From d5aa7c46692474376a3c31704cfc4783c86338f2 Mon Sep 17 00:00:00 2001
From: Jacques Nadeau <jacques@apache.org>
Date: Fri, 5 Feb 2016 12:08:35 -0800
Subject: [PATCH 0001/1644] Initial Commit

---
 README.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000..e2dc7471c204d
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+arrow

From cbc56bf8ac423c585c782d5eda5c517ea8df8e3c Mon Sep 17 00:00:00 2001
From: Jacques Nadeau <jacques@apache.org>
Date: Tue, 16 Feb 2016 21:35:38 -0800
Subject: [PATCH 0002/1644] Update readme and add license in root.

---
 LICENSE.txt | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 README.md   |  14 +++-
 2 files changed, 215 insertions(+), 1 deletion(-)
 create mode 100644 LICENSE.txt

diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/README.md b/README.md
index e2dc7471c204d..4423a91351381 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,13 @@
-arrow
+## Apache Arrow
+
+#### Powering Columnar In-Memory Analytics
+
+Arrow is a set of technologies that enable big-data systems to process and move data fast.
+
+Initial implementations include:
+
+ - [The Arrow Format](https://github.com/apache/arrow/tree/master/format)
+ - [Arrow Structures and APIs in C++](https://github.com/apache/arrow/tree/master/cpp)
+ - [Arrow Structures and APIs in Java](https://github.com/apache/arrow/tree/master/java)
+
+Arrow is an [Apache Software Foundation](www.apache.org) project. More info can be found at [arrow.apache.org](http://arrow.apache.org).

From fa5f0299f046c46e1b2f671e5e3b4f1956522711 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Wed, 17 Feb 2016 04:37:53 -0800
Subject: [PATCH 0003/1644] ARROW-1: Initial Arrow Code Commit

---
 java/.gitignore                               |  22 +
 java/memory/pom.xml                           |  50 +
 .../main/java/io/netty/buffer/ArrowBuf.java   | 863 ++++++++++++++++++
 .../io/netty/buffer/ExpandableByteBuf.java    |  55 ++
 .../java/io/netty/buffer/LargeBuffer.java     |  59 ++
 .../netty/buffer/MutableWrappedByteBuf.java   | 336 +++++++
 .../netty/buffer/PooledByteBufAllocatorL.java | 272 ++++++
 .../buffer/UnsafeDirectLittleEndian.java      | 270 ++++++
 .../org/apache/arrow/memory/Accountant.java   | 272 ++++++
 .../arrow/memory/AllocationManager.java       | 433 +++++++++
 .../arrow/memory/AllocationReservation.java   |  86 ++
 .../memory/AllocatorClosedException.java      |  31 +
 .../apache/arrow/memory/BaseAllocator.java    | 781 ++++++++++++++++
 .../apache/arrow/memory/BoundsChecking.java   |  35 +
 .../apache/arrow/memory/BufferAllocator.java  | 151 +++
 .../apache/arrow/memory/BufferManager.java    |  66 ++
 .../apache/arrow/memory/ChildAllocator.java   |  53 ++
 .../arrow/memory/DrillByteBufAllocator.java   | 141 +++
 .../arrow/memory/OutOfMemoryException.java    |  50 +
 .../java/org/apache/arrow/memory/README.md    | 121 +++
 .../apache/arrow/memory/RootAllocator.java    |  39 +
 .../org/apache/arrow/memory/package-info.java |  24 +
 .../arrow/memory/util/AssertionUtil.java      |  37 +
 .../arrow/memory/util/AutoCloseableLock.java  |  43 +
 .../arrow/memory/util/HistoricalLog.java      | 185 ++++
 .../org/apache/arrow/memory/util/Metrics.java |  40 +
 .../org/apache/arrow/memory/util/Pointer.java |  28 +
 .../apache/arrow/memory/util/StackTrace.java  |  70 ++
 .../src/main/resources/drill-module.conf      |  25 +
 .../apache/arrow/memory/TestAccountant.java   | 164 ++++
 .../arrow/memory/TestBaseAllocator.java       | 648 +++++++++++++
 .../apache/arrow/memory/TestEndianess.java    |  43 +
 java/pom.xml                                  | 470 ++++++++++
 java/vector/pom.xml                           | 165 ++++
 java/vector/src/main/codegen/config.fmpp      |  24 +
 .../main/codegen/data/ValueVectorTypes.tdd    | 168 ++++
 .../src/main/codegen/includes/license.ftl     |  18 +
 .../src/main/codegen/includes/vv_imports.ftl  |  62 ++
 .../templates/AbstractFieldReader.java        | 124 +++
 .../templates/AbstractFieldWriter.java        | 147 +++
 .../AbstractPromotableFieldWriter.java        | 142 +++
 .../main/codegen/templates/BaseReader.java    |  73 ++
 .../main/codegen/templates/BaseWriter.java    | 117 +++
 .../codegen/templates/BasicTypeHelper.java    | 538 +++++++++++
 .../main/codegen/templates/ComplexCopier.java | 133 +++
 .../codegen/templates/ComplexReaders.java     | 183 ++++
 .../codegen/templates/ComplexWriters.java     | 151 +++
 .../codegen/templates/FixedValueVectors.java  | 813 +++++++++++++++++
 .../codegen/templates/HolderReaderImpl.java   | 290 ++++++
 .../main/codegen/templates/ListWriters.java   | 234 +++++
 .../main/codegen/templates/MapWriters.java    | 240 +++++
 .../main/codegen/templates/NullReader.java    | 138 +++
 .../templates/NullableValueVectors.java       | 630 +++++++++++++
 .../templates/RepeatedValueVectors.java       | 421 +++++++++
 .../codegen/templates/UnionListWriter.java    | 185 ++++
 .../main/codegen/templates/UnionReader.java   | 194 ++++
 .../main/codegen/templates/UnionVector.java   | 467 ++++++++++
 .../main/codegen/templates/UnionWriter.java   | 228 +++++
 .../main/codegen/templates/ValueHolders.java  | 116 +++
 .../templates/VariableLengthVectors.java      | 644 +++++++++++++
 .../apache/arrow/vector/AddOrGetResult.java   |  38 +
 .../apache/arrow/vector/AllocationHelper.java |  61 ++
 .../arrow/vector/BaseDataValueVector.java     |  91 ++
 .../apache/arrow/vector/BaseValueVector.java  | 125 +++
 .../org/apache/arrow/vector/BitVector.java    | 450 +++++++++
 .../apache/arrow/vector/FixedWidthVector.java |  35 +
 .../apache/arrow/vector/NullableVector.java   |  23 +
 .../NullableVectorDefinitionSetter.java       |  23 +
 .../org/apache/arrow/vector/ObjectVector.java | 220 +++++
 .../arrow/vector/SchemaChangeCallBack.java    |  52 ++
 .../arrow/vector/ValueHolderHelper.java       | 203 ++++
 .../org/apache/arrow/vector/ValueVector.java  | 222 +++++
 .../arrow/vector/VariableWidthVector.java     |  51 ++
 .../apache/arrow/vector/VectorDescriptor.java |  83 ++
 .../apache/arrow/vector/VectorTrimmer.java    |  33 +
 .../org/apache/arrow/vector/ZeroVector.java   | 181 ++++
 .../complex/AbstractContainerVector.java      | 143 +++
 .../vector/complex/AbstractMapVector.java     | 278 ++++++
 .../complex/BaseRepeatedValueVector.java      | 260 ++++++
 .../vector/complex/ContainerVectorLike.java   |  43 +
 .../vector/complex/EmptyValuePopulator.java   |  54 ++
 .../arrow/vector/complex/ListVector.java      | 321 +++++++
 .../arrow/vector/complex/MapVector.java       | 374 ++++++++
 .../arrow/vector/complex/Positionable.java    |  22 +
 .../complex/RepeatedFixedWidthVectorLike.java |  40 +
 .../vector/complex/RepeatedListVector.java    | 428 +++++++++
 .../vector/complex/RepeatedMapVector.java     | 584 ++++++++++++
 .../vector/complex/RepeatedValueVector.java   |  85 ++
 .../RepeatedVariableWidthVectorLike.java      |  35 +
 .../arrow/vector/complex/StateTool.java       |  34 +
 .../vector/complex/VectorWithOrdinal.java     |  30 +
 .../complex/impl/AbstractBaseReader.java      | 100 ++
 .../complex/impl/AbstractBaseWriter.java      |  59 ++
 .../complex/impl/ComplexWriterImpl.java       | 193 ++++
 .../complex/impl/MapOrListWriterImpl.java     | 112 +++
 .../vector/complex/impl/PromotableWriter.java | 196 ++++
 .../complex/impl/RepeatedListReaderImpl.java  | 145 +++
 .../complex/impl/RepeatedMapReaderImpl.java   | 192 ++++
 .../impl/SingleLikeRepeatedMapReaderImpl.java |  89 ++
 .../complex/impl/SingleListReaderImpl.java    |  88 ++
 .../complex/impl/SingleMapReaderImpl.java     | 108 +++
 .../vector/complex/impl/UnionListReader.java  |  98 ++
 .../vector/complex/reader/FieldReader.java    |  29 +
 .../vector/complex/writer/FieldWriter.java    |  27 +
 .../arrow/vector/holders/ComplexHolder.java   |  25 +
 .../arrow/vector/holders/ObjectHolder.java    |  38 +
 .../vector/holders/RepeatedListHolder.java    |  23 +
 .../vector/holders/RepeatedMapHolder.java     |  23 +
 .../arrow/vector/holders/UnionHolder.java     |  37 +
 .../arrow/vector/holders/ValueHolder.java     |  31 +
 .../arrow/vector/types/MaterializedField.java | 217 +++++
 .../org/apache/arrow/vector/types/Types.java  | 132 +++
 .../vector/util/ByteFunctionHelpers.java      | 233 +++++
 .../apache/arrow/vector/util/CallBack.java    |  23 +
 .../arrow/vector/util/CoreDecimalUtility.java |  91 ++
 .../apache/arrow/vector/util/DateUtility.java | 682 ++++++++++++++
 .../arrow/vector/util/DecimalUtility.java     | 737 +++++++++++++++
 .../vector/util/JsonStringArrayList.java      |  57 ++
 .../arrow/vector/util/JsonStringHashMap.java  |  76 ++
 .../arrow/vector/util/MapWithOrdinal.java     | 248 +++++
 .../util/OversizedAllocationException.java    |  49 +
 .../util/SchemaChangeRuntimeException.java    |  41 +
 .../org/apache/arrow/vector/util/Text.java    | 621 +++++++++++++
 .../arrow/vector/util/TransferPair.java       |  27 +
 124 files changed, 22077 insertions(+)
 create mode 100644 java/.gitignore
 create mode 100644 java/memory/pom.xml
 create mode 100644 java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
 create mode 100644 java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
 create mode 100644 java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
 create mode 100644 java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
 create mode 100644 java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
 create mode 100644 java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/README.md
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/package-info.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/util/Metrics.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/util/Pointer.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java
 create mode 100644 java/memory/src/main/resources/drill-module.conf
 create mode 100644 java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
 create mode 100644 java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
 create mode 100644 java/memory/src/test/java/org/apache/arrow/memory/TestEndianess.java
 create mode 100644 java/pom.xml
 create mode 100644 java/vector/pom.xml
 create mode 100644 java/vector/src/main/codegen/config.fmpp
 create mode 100644 java/vector/src/main/codegen/data/ValueVectorTypes.tdd
 create mode 100644 java/vector/src/main/codegen/includes/license.ftl
 create mode 100644 java/vector/src/main/codegen/includes/vv_imports.ftl
 create mode 100644 java/vector/src/main/codegen/templates/AbstractFieldReader.java
 create mode 100644 java/vector/src/main/codegen/templates/AbstractFieldWriter.java
 create mode 100644 java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
 create mode 100644 java/vector/src/main/codegen/templates/BaseReader.java
 create mode 100644 java/vector/src/main/codegen/templates/BaseWriter.java
 create mode 100644 java/vector/src/main/codegen/templates/BasicTypeHelper.java
 create mode 100644 java/vector/src/main/codegen/templates/ComplexCopier.java
 create mode 100644 java/vector/src/main/codegen/templates/ComplexReaders.java
 create mode 100644 java/vector/src/main/codegen/templates/ComplexWriters.java
 create mode 100644 java/vector/src/main/codegen/templates/FixedValueVectors.java
 create mode 100644 java/vector/src/main/codegen/templates/HolderReaderImpl.java
 create mode 100644 java/vector/src/main/codegen/templates/ListWriters.java
 create mode 100644 java/vector/src/main/codegen/templates/MapWriters.java
 create mode 100644 java/vector/src/main/codegen/templates/NullReader.java
 create mode 100644 java/vector/src/main/codegen/templates/NullableValueVectors.java
 create mode 100644 java/vector/src/main/codegen/templates/RepeatedValueVectors.java
 create mode 100644 java/vector/src/main/codegen/templates/UnionListWriter.java
 create mode 100644 java/vector/src/main/codegen/templates/UnionReader.java
 create mode 100644 java/vector/src/main/codegen/templates/UnionVector.java
 create mode 100644 java/vector/src/main/codegen/templates/UnionWriter.java
 create mode 100644 java/vector/src/main/codegen/templates/ValueHolders.java
 create mode 100644 java/vector/src/main/codegen/templates/VariableLengthVectors.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/VectorWithOrdinal.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/holders/ComplexHolder.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedListHolder.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/CallBack.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/SchemaChangeRuntimeException.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/TransferPair.java

diff --git a/java/.gitignore b/java/.gitignore
new file mode 100644
index 0000000000000..73c1be4912297
--- /dev/null
+++ b/java/.gitignore
@@ -0,0 +1,22 @@
+.project
+.buildpath
+.classpath
+.checkstyle
+.settings/
+.idea/
+TAGS
+*.log
+*.lck
+*.iml
+target/
+*.DS_Store
+*.patch
+*~
+git.properties
+contrib/native/client/build/
+contrib/native/client/build/*
+CMakeCache.txt
+CMakeFiles
+Makefile
+cmake_install.cmake
+install_manifest.txt
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
new file mode 100644
index 0000000000000..44332f5ed14a8
--- /dev/null
+++ b/java/memory/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+  license agreements. See the NOTICE file distributed with this work for additional 
+  information regarding copyright ownership. The ASF licenses this file to 
+  You under the Apache License, Version 2.0 (the "License"); you may not use 
+  this file except in compliance with the License. You may obtain a copy of 
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+  by applicable law or agreed to in writing, software distributed under the 
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+  OF ANY KIND, either express or implied. See the License for the specific 
+  language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.arrow</groupId>
+    <artifactId>arrow-java-root</artifactId>
+    <version>0.1-SNAPSHOT</version>
+  </parent>
+  <artifactId>arrow-memory</artifactId>
+  <name>arrow-memory</name>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>com.codahale.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>3.0.1</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+      <version>3.0.1</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.carrotsearch</groupId>
+      <artifactId>hppc</artifactId>
+      <version>0.7.1</version>
+    </dependency>
+  </dependencies>
+
+
+  <build>
+  </build>
+
+
+
+</project>
diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
new file mode 100644
index 0000000000000..f033ba6538e83
--- /dev/null
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -0,0 +1,863 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.nio.charset.Charset;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.memory.BoundsChecking;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.BufferManager;
+import org.apache.arrow.memory.AllocationManager.BufferLedger;
+import org.apache.arrow.memory.BaseAllocator.Verbosity;
+import org.apache.arrow.memory.util.HistoricalLog;
+
+import com.google.common.base.Preconditions;
+
+public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ArrowBuf.class);
+
+  private static final AtomicLong idGenerator = new AtomicLong(0);
+
+  private final long id = idGenerator.incrementAndGet();
+  private final AtomicInteger refCnt;
+  private final UnsafeDirectLittleEndian udle;
+  private final long addr;
+  private final int offset;
+  private final BufferLedger ledger;
+  private final BufferManager bufManager;
+  private final ByteBufAllocator alloc;
+  private final boolean isEmpty;
+  private volatile int length;
+  private final HistoricalLog historicalLog = BaseAllocator.DEBUG ?
+      new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "DrillBuf[%d]", id) : null;
+
+  public ArrowBuf(
+      final AtomicInteger refCnt,
+      final BufferLedger ledger,
+      final UnsafeDirectLittleEndian byteBuf,
+      final BufferManager manager,
+      final ByteBufAllocator alloc,
+      final int offset,
+      final int length,
+      boolean isEmpty) {
+    super(byteBuf.maxCapacity());
+    this.refCnt = refCnt;
+    this.udle = byteBuf;
+    this.isEmpty = isEmpty;
+    this.bufManager = manager;
+    this.alloc = alloc;
+    this.addr = byteBuf.memoryAddress() + offset;
+    this.ledger = ledger;
+    this.length = length;
+    this.offset = offset;
+
+    if (BaseAllocator.DEBUG) {
+      historicalLog.recordEvent("create()");
+    }
+
+  }
+
+  public ArrowBuf reallocIfNeeded(final int size) {
+    Preconditions.checkArgument(size >= 0, "reallocation size must be non-negative");
+
+    if (this.capacity() >= size) {
+      return this;
+    }
+
+    if (bufManager != null) {
+      return bufManager.replace(this, size);
+    } else {
+      throw new UnsupportedOperationException("Realloc is only available in the context of an operator's UDFs");
+    }
+  }
+
+  @Override
+  public int refCnt() {
+    if (isEmpty) {
+      return 1;
+    } else {
+      return refCnt.get();
+    }
+  }
+
+  private long addr(int index) {
+    return addr + index;
+  }
+
+  private final void checkIndexD(int index, int fieldLength) {
+    ensureAccessible();
+    if (fieldLength < 0) {
+      throw new IllegalArgumentException("length: " + fieldLength + " (expected: >= 0)");
+    }
+    if (index < 0 || index > capacity() - fieldLength) {
+      if (BaseAllocator.DEBUG) {
+        historicalLog.logHistory(logger);
+      }
+      throw new IndexOutOfBoundsException(String.format(
+          "index: %d, length: %d (expected: range(0, %d))", index, fieldLength, capacity()));
+    }
+  }
+
+  /**
+   * Allows a function to determine whether not reading a particular string of bytes is valid.
+   *
+   * Will throw an exception if the memory is not readable for some reason. Only doesn't something in the case that
+   * AssertionUtil.BOUNDS_CHECKING_ENABLED is true.
+   *
+   * @param start
+   *          The starting position of the bytes to be read.
+   * @param end
+   *          The exclusive endpoint of the bytes to be read.
+   */
+  public void checkBytes(int start, int end) {
+    if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
+      checkIndexD(start, end - start);
+    }
+  }
+
+  private void chk(int index, int width) {
+    if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
+      checkIndexD(index, width);
+    }
+  }
+
+  private void ensure(int width) {
+    if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
+      ensureWritable(width);
+    }
+  }
+
+  /**
+   * Create a new DrillBuf that is associated with an alternative allocator for the purposes of memory ownership and
+   * accounting. This has no impact on the reference counting for the current DrillBuf except in the situation where the
+   * passed in Allocator is the same as the current buffer.
+   *
+   * This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
+   * reference count of 1 (in the case that this is the first time this memory is being associated with the new
+   * allocator) or the current value of the reference count + 1 for the other AllocationManager/BufferLedger combination
+   * in the case that the provided allocator already had an association to this underlying memory.
+   *
+   * @param target
+   *          The target allocator to create an association with.
+   * @return A new DrillBuf which shares the same underlying memory as this DrillBuf.
+   */
+  public ArrowBuf retain(BufferAllocator target) {
+
+    if (isEmpty) {
+      return this;
+    }
+
+    if (BaseAllocator.DEBUG) {
+      historicalLog.recordEvent("retain(%s)", target.getName());
+    }
+    final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
+    return otherLedger.newDrillBuf(offset, length, null);
+  }
+
+  /**
+   * Transfer the memory accounting ownership of this DrillBuf to another allocator. This will generate a new DrillBuf
+   * that carries an association with the underlying memory of this DrillBuf. If this DrillBuf is connected to the
+   * owning BufferLedger of this memory, that memory ownership/accounting will be transferred to the taret allocator. If
+   * this DrillBuf does not currently own the memory underlying it (and is only associated with it), this does not
+   * transfer any ownership to the newly created DrillBuf.
+   *
+   * This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
+   * reference count of 1 (in the case that this is the first time this memory is being associated with the new
+   * allocator) or the current value of the reference count for the other AllocationManager/BufferLedger combination in
+   * the case that the provided allocator already had an association to this underlying memory.
+   *
+   * Transfers will always succeed, even if that puts the other allocator into an overlimit situation. This is possible
+   * due to the fact that the original owning allocator may have allocated this memory out of a local reservation
+   * whereas the target allocator may need to allocate new memory from a parent or RootAllocator. This operation is done
+   * in a mostly-lockless but consistent manner. As such, the overlimit==true situation could occur slightly prematurely
+   * to an actual overlimit==true condition. This is simply conservative behavior which means we may return overlimit
+   * slightly sooner than is necessary.
+   *
+   * @param target
+   *          The allocator to transfer ownership to.
+   * @return A new transfer result with the impact of the transfer (whether it was overlimit) as well as the newly
+   *         created DrillBuf.
+   */
+  public TransferResult transferOwnership(BufferAllocator target) {
+
+    if (isEmpty) {
+      return new TransferResult(true, this);
+    }
+
+    final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
+    final ArrowBuf newBuf = otherLedger.newDrillBuf(offset, length, null);
+    final boolean allocationFit = this.ledger.transferBalance(otherLedger);
+    return new TransferResult(allocationFit, newBuf);
+  }
+
+  /**
+   * The outcome of a Transfer.
+   */
+  public class TransferResult {
+
+    /**
+     * Whether this transfer fit within the target allocator's capacity.
+     */
+    public final boolean allocationFit;
+
+    /**
+     * The newly created buffer associated with the target allocator.
+     */
+    public final ArrowBuf buffer;
+
+    private TransferResult(boolean allocationFit, ArrowBuf buffer) {
+      this.allocationFit = allocationFit;
+      this.buffer = buffer;
+    }
+
+  }
+
+  @Override
+  public boolean release() {
+    return release(1);
+  }
+
+  /**
+   * Release the provided number of reference counts.
+   */
+  @Override
+  public boolean release(int decrement) {
+
+    if (isEmpty) {
+      return false;
+    }
+
+    if (decrement < 1) {
+      throw new IllegalStateException(String.format("release(%d) argument is not positive. Buffer Info: %s",
+          decrement, toVerboseString()));
+    }
+
+    final int refCnt = ledger.decrement(decrement);
+
+    if (BaseAllocator.DEBUG) {
+      historicalLog.recordEvent("release(%d). original value: %d", decrement, refCnt + decrement);
+    }
+
+    if (refCnt < 0) {
+      throw new IllegalStateException(
+          String.format("DrillBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString()));
+    }
+
+    return refCnt == 0;
+
+  }
+
+  @Override
+  public int capacity() {
+    return length;
+  }
+
+  @Override
+  public synchronized ArrowBuf capacity(int newCapacity) {
+
+    if (newCapacity == length) {
+      return this;
+    }
+
+    Preconditions.checkArgument(newCapacity >= 0);
+
+    if (newCapacity < length) {
+      length = newCapacity;
+      return this;
+    }
+
+    throw new UnsupportedOperationException("Buffers don't support resizing that increases the size.");
+  }
+
+  @Override
+  public ByteBufAllocator alloc() {
+    return udle.alloc();
+  }
+
+  @Override
+  public ByteOrder order() {
+    return ByteOrder.LITTLE_ENDIAN;
+  }
+
+  @Override
+  public ByteBuf order(ByteOrder endianness) {
+    return this;
+  }
+
+  @Override
+  public ByteBuf unwrap() {
+    return udle;
+  }
+
+  @Override
+  public boolean isDirect() {
+    return true;
+  }
+
+  @Override
+  public ByteBuf readBytes(int length) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ByteBuf readSlice(int length) {
+    final ByteBuf slice = slice(readerIndex(), length);
+    readerIndex(readerIndex() + length);
+    return slice;
+  }
+
+  @Override
+  public ByteBuf copy() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ByteBuf copy(int index, int length) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ByteBuf slice() {
+    return slice(readerIndex(), readableBytes());
+  }
+
+  public static String bufferState(final ByteBuf buf) {
+    final int cap = buf.capacity();
+    final int mcap = buf.maxCapacity();
+    final int ri = buf.readerIndex();
+    final int rb = buf.readableBytes();
+    final int wi = buf.writerIndex();
+    final int wb = buf.writableBytes();
+    return String.format("cap/max: %d/%d, ri: %d, rb: %d, wi: %d, wb: %d",
+        cap, mcap, ri, rb, wi, wb);
+  }
+
+  @Override
+  public ArrowBuf slice(int index, int length) {
+
+    if (isEmpty) {
+      return this;
+    }
+
+    /*
+     * Re the behavior of reference counting, see http://netty.io/wiki/reference-counted-objects.html#wiki-h3-5, which
+     * explains that derived buffers share their reference count with their parent
+     */
+    final ArrowBuf newBuf = ledger.newDrillBuf(offset + index, length);
+    newBuf.writerIndex(length);
+    return newBuf;
+  }
+
+  @Override
+  public ArrowBuf duplicate() {
+    return slice(0, length);
+  }
+
+  @Override
+  public int nioBufferCount() {
+    return 1;
+  }
+
+  @Override
+  public ByteBuffer nioBuffer() {
+    return nioBuffer(readerIndex(), readableBytes());
+  }
+
+  @Override
+  public ByteBuffer nioBuffer(int index, int length) {
+    return udle.nioBuffer(offset + index, length);
+  }
+
+  @Override
+  public ByteBuffer internalNioBuffer(int index, int length) {
+    return udle.internalNioBuffer(offset + index, length);
+  }
+
+  @Override
+  public ByteBuffer[] nioBuffers() {
+    return new ByteBuffer[] { nioBuffer() };
+  }
+
+  @Override
+  public ByteBuffer[] nioBuffers(int index, int length) {
+    return new ByteBuffer[] { nioBuffer(index, length) };
+  }
+
+  @Override
+  public boolean hasArray() {
+    return udle.hasArray();
+  }
+
+  @Override
+  public byte[] array() {
+    return udle.array();
+  }
+
+  @Override
+  public int arrayOffset() {
+    return udle.arrayOffset();
+  }
+
+  @Override
+  public boolean hasMemoryAddress() {
+    return true;
+  }
+
+  @Override
+  public long memoryAddress() {
+    return this.addr;
+  }
+
+  @Override
+  public String toString() {
+    return String.format("DrillBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity());
+  }
+
+  @Override
+  public String toString(Charset charset) {
+    return toString(readerIndex, readableBytes(), charset);
+  }
+
+  @Override
+  public String toString(int index, int length, Charset charset) {
+
+    if (length == 0) {
+      return "";
+    }
+
+    return ByteBufUtil.decodeString(nioBuffer(index, length), charset);
+  }
+
+  @Override
+  public int hashCode() {
+    return System.identityHashCode(this);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    // identity equals only.
+    return this == obj;
+  }
+
+  @Override
+  public ByteBuf retain(int increment) {
+    Preconditions.checkArgument(increment > 0, "retain(%d) argument is not positive", increment);
+
+    if (isEmpty) {
+      return this;
+    }
+
+    if (BaseAllocator.DEBUG) {
+      historicalLog.recordEvent("retain(%d)", increment);
+    }
+
+    final int originalReferenceCount = refCnt.getAndAdd(increment);
+    Preconditions.checkArgument(originalReferenceCount > 0);
+    return this;
+  }
+
+  @Override
+  public ByteBuf retain() {
+    return retain(1);
+  }
+
+  @Override
+  public long getLong(int index) {
+    chk(index, 8);
+    final long v = PlatformDependent.getLong(addr(index));
+    return v;
+  }
+
+  @Override
+  public float getFloat(int index) {
+    return Float.intBitsToFloat(getInt(index));
+  }
+
+  @Override
+  public double getDouble(int index) {
+    return Double.longBitsToDouble(getLong(index));
+  }
+
+  @Override
+  public char getChar(int index) {
+    return (char) getShort(index);
+  }
+
+  @Override
+  public long getUnsignedInt(int index) {
+    return getInt(index) & 0xFFFFFFFFL;
+  }
+
+  @Override
+  public int getInt(int index) {
+    chk(index, 4);
+    final int v = PlatformDependent.getInt(addr(index));
+    return v;
+  }
+
+  @Override
+  public int getUnsignedShort(int index) {
+    return getShort(index) & 0xFFFF;
+  }
+
+  @Override
+  public short getShort(int index) {
+    chk(index, 2);
+    short v = PlatformDependent.getShort(addr(index));
+    return v;
+  }
+
+  @Override
+  public ByteBuf setShort(int index, int value) {
+    chk(index, 2);
+    PlatformDependent.putShort(addr(index), (short) value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setInt(int index, int value) {
+    chk(index, 4);
+    PlatformDependent.putInt(addr(index), value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setLong(int index, long value) {
+    chk(index, 8);
+    PlatformDependent.putLong(addr(index), value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setChar(int index, int value) {
+    chk(index, 2);
+    PlatformDependent.putShort(addr(index), (short) value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setFloat(int index, float value) {
+    chk(index, 4);
+    PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value));
+    return this;
+  }
+
+  @Override
+  public ByteBuf setDouble(int index, double value) {
+    chk(index, 8);
+    PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value));
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeShort(int value) {
+    ensure(2);
+    PlatformDependent.putShort(addr(writerIndex), (short) value);
+    writerIndex += 2;
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeInt(int value) {
+    ensure(4);
+    PlatformDependent.putInt(addr(writerIndex), value);
+    writerIndex += 4;
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeLong(long value) {
+    ensure(8);
+    PlatformDependent.putLong(addr(writerIndex), value);
+    writerIndex += 8;
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeChar(int value) {
+    ensure(2);
+    PlatformDependent.putShort(addr(writerIndex), (short) value);
+    writerIndex += 2;
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeFloat(float value) {
+    ensure(4);
+    PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value));
+    writerIndex += 4;
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeDouble(double value) {
+    ensure(8);
+    PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value));
+    writerIndex += 8;
+    return this;
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+    udle.getBytes(index + offset, dst, dstIndex, length);
+    return this;
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, ByteBuffer dst) {
+    udle.getBytes(index + offset, dst);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setByte(int index, int value) {
+    chk(index, 1);
+    PlatformDependent.putByte(addr(index), (byte) value);
+    return this;
+  }
+
+  public void setByte(int index, byte b) {
+    chk(index, 1);
+    PlatformDependent.putByte(addr(index), b);
+  }
+
+  public void writeByteUnsafe(byte b) {
+    PlatformDependent.putByte(addr(readerIndex), b);
+    readerIndex++;
+  }
+
+  @Override
+  protected byte _getByte(int index) {
+    return getByte(index);
+  }
+
+  @Override
+  protected short _getShort(int index) {
+    return getShort(index);
+  }
+
+  @Override
+  protected int _getInt(int index) {
+    return getInt(index);
+  }
+
+  @Override
+  protected long _getLong(int index) {
+    return getLong(index);
+  }
+
+  @Override
+  protected void _setByte(int index, int value) {
+    setByte(index, value);
+  }
+
+  @Override
+  protected void _setShort(int index, int value) {
+    setShort(index, value);
+  }
+
+  @Override
+  protected void _setMedium(int index, int value) {
+    setMedium(index, value);
+  }
+
+  @Override
+  protected void _setInt(int index, int value) {
+    setInt(index, value);
+  }
+
+  @Override
+  protected void _setLong(int index, long value) {
+    setLong(index, value);
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+    udle.getBytes(index + offset, dst, dstIndex, length);
+    return this;
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
+    udle.getBytes(index + offset, out, length);
+    return this;
+  }
+
+  @Override
+  protected int _getUnsignedMedium(int index) {
+    final long addr = addr(index);
+    return (PlatformDependent.getByte(addr) & 0xff) << 16 |
+        (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
+        PlatformDependent.getByte(addr + 2) & 0xff;
+  }
+
+  @Override
+  public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
+    return udle.getBytes(index + offset, out, length);
+  }
+
+  @Override
+  public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+    udle.setBytes(index + offset, src, srcIndex, length);
+    return this;
+  }
+
+  public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
+    if (src.isDirect()) {
+      checkIndex(index, length);
+      PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this.memoryAddress() + index,
+          length);
+    } else {
+      if (srcIndex == 0 && src.capacity() == length) {
+        udle.setBytes(index + offset, src);
+      } else {
+        ByteBuffer newBuf = src.duplicate();
+        newBuf.position(srcIndex);
+        newBuf.limit(srcIndex + length);
+        udle.setBytes(index + offset, src);
+      }
+    }
+
+    return this;
+  }
+
+  @Override
+  public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+    udle.setBytes(index + offset, src, srcIndex, length);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setBytes(int index, ByteBuffer src) {
+    udle.setBytes(index + offset, src);
+    return this;
+  }
+
+  @Override
+  public int setBytes(int index, InputStream in, int length) throws IOException {
+    return udle.setBytes(index + offset, in, length);
+  }
+
+  @Override
+  public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
+    return udle.setBytes(index + offset, in, length);
+  }
+
+  @Override
+  public byte getByte(int index) {
+    chk(index, 1);
+    return PlatformDependent.getByte(addr(index));
+  }
+
+  @Override
+  public void close() {
+    release();
+  }
+
+  /**
+   * Returns the possible memory consumed by this DrillBuf in the worse case scenario. (not shared, connected to larger
+   * underlying buffer of allocated memory)
+   *
+   * @return Size in bytes.
+   */
+  public int getPossibleMemoryConsumed() {
+    return ledger.getSize();
+  }
+
+  /**
+   * Return that is Accounted for by this buffer (and its potentially shared siblings within the context of the
+   * associated allocator).
+   *
+   * @return Size in bytes.
+   */
+  public int getActualMemoryConsumed() {
+    return ledger.getAccountedSize();
+  }
+
+  private final static int LOG_BYTES_PER_ROW = 10;
+
+  /**
+   * Return the buffer's byte contents in the form of a hex dump.
+   *
+   * @param start
+   *          the starting byte index
+   * @param length
+   *          how many bytes to log
+   * @return A hex dump in a String.
+   */
+  public String toHexString(final int start, final int length) {
+    final int roundedStart = (start / LOG_BYTES_PER_ROW) * LOG_BYTES_PER_ROW;
+
+    final StringBuilder sb = new StringBuilder("buffer byte dump\n");
+    int index = roundedStart;
+    for (int nLogged = 0; nLogged < length; nLogged += LOG_BYTES_PER_ROW) {
+      sb.append(String.format(" [%05d-%05d]", index, index + LOG_BYTES_PER_ROW - 1));
+      for (int i = 0; i < LOG_BYTES_PER_ROW; ++i) {
+        try {
+          final byte b = getByte(index++);
+          sb.append(String.format(" 0x%02x", b));
+        } catch (IndexOutOfBoundsException ioob) {
+          sb.append(" <ioob>");
+        }
+      }
+      sb.append('\n');
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Get the integer id assigned to this DrillBuf for debugging purposes.
+   *
+   * @return integer id
+   */
+  public long getId() {
+    return id;
+  }
+
+  public String toVerboseString() {
+    if (isEmpty) {
+      return toString();
+    }
+
+    StringBuilder sb = new StringBuilder();
+    ledger.print(sb, 0, Verbosity.LOG_WITH_STACKTRACE);
+    return sb.toString();
+  }
+
+  public void print(StringBuilder sb, int indent, Verbosity verbosity) {
+    BaseAllocator.indent(sb, indent).append(toString());
+
+    if (BaseAllocator.DEBUG && !isEmpty && verbosity.includeHistoricalLog) {
+      sb.append("\n");
+      historicalLog.buildHistory(sb, indent + 1, verbosity.includeStackTraces);
+    }
+  }
+
+}
diff --git a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java b/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
new file mode 100644
index 0000000000000..59886474923f3
--- /dev/null
+++ b/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import org.apache.arrow.memory.BufferAllocator;
+
+/**
+ * Allows us to decorate DrillBuf to make it expandable so that we can use them in the context of the Netty framework
+ * (thus supporting RPC level memory accounting).
+ */
+public class ExpandableByteBuf extends MutableWrappedByteBuf {
+
+  private final BufferAllocator allocator;
+
+  public ExpandableByteBuf(ByteBuf buffer, BufferAllocator allocator) {
+    super(buffer);
+    this.allocator = allocator;
+  }
+
+  @Override
+  public ByteBuf copy(int index, int length) {
+    return new ExpandableByteBuf(buffer.copy(index, length), allocator);
+  }
+
+  @Override
+  public ByteBuf capacity(int newCapacity) {
+    if (newCapacity > capacity()) {
+      ByteBuf newBuf = allocator.buffer(newCapacity);
+      newBuf.writeBytes(buffer, 0, buffer.capacity());
+      newBuf.readerIndex(buffer.readerIndex());
+      newBuf.writerIndex(buffer.writerIndex());
+      buffer.release();
+      buffer = newBuf;
+      return newBuf;
+    } else {
+      return super.capacity(newCapacity);
+    }
+  }
+
+}
diff --git a/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java b/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
new file mode 100644
index 0000000000000..5f5e904fb0429
--- /dev/null
+++ b/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A MutableWrappedByteBuf that also maintains a metric of the number of huge buffer bytes and counts.
+ */
+public class LargeBuffer extends MutableWrappedByteBuf {
+
+  private final AtomicLong hugeBufferSize;
+  private final AtomicLong hugeBufferCount;
+
+  private final int initCap;
+
+  public LargeBuffer(ByteBuf buffer, AtomicLong hugeBufferSize, AtomicLong hugeBufferCount) {
+    super(buffer);
+    initCap = buffer.capacity();
+    this.hugeBufferCount = hugeBufferCount;
+    this.hugeBufferSize = hugeBufferSize;
+  }
+
+  @Override
+  public ByteBuf copy(int index, int length) {
+    return new LargeBuffer(buffer.copy(index, length), hugeBufferSize, hugeBufferCount);
+  }
+
+  @Override
+  public boolean release() {
+    return release(1);
+  }
+
+  @Override
+  public boolean release(int decrement) {
+    boolean released = unwrap().release(decrement);
+    if (released) {
+      hugeBufferSize.addAndGet(-initCap);
+      hugeBufferCount.decrementAndGet();
+    }
+    return released;
+  }
+
+}
diff --git a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
new file mode 100644
index 0000000000000..5709473135e4b
--- /dev/null
+++ b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+
+/**
+ * This is basically a complete copy of DuplicatedByteBuf. We copy because we want to override some behaviors and make
+ * buffer mutable.
+ */
+abstract class MutableWrappedByteBuf extends AbstractByteBuf {
+
+  @Override
+  public ByteBuffer nioBuffer(int index, int length) {
+    return unwrap().nioBuffer(index, length);
+  }
+
+  ByteBuf buffer;
+
+  public MutableWrappedByteBuf(ByteBuf buffer) {
+    super(buffer.maxCapacity());
+
+    if (buffer instanceof MutableWrappedByteBuf) {
+      this.buffer = ((MutableWrappedByteBuf) buffer).buffer;
+    } else {
+      this.buffer = buffer;
+    }
+
+    setIndex(buffer.readerIndex(), buffer.writerIndex());
+  }
+
+  @Override
+  public ByteBuf unwrap() {
+    return buffer;
+  }
+
+  @Override
+  public ByteBufAllocator alloc() {
+    return buffer.alloc();
+  }
+
+  @Override
+  public ByteOrder order() {
+    return buffer.order();
+  }
+
+  @Override
+  public boolean isDirect() {
+    return buffer.isDirect();
+  }
+
+  @Override
+  public int capacity() {
+    return buffer.capacity();
+  }
+
+  @Override
+  public ByteBuf capacity(int newCapacity) {
+    buffer.capacity(newCapacity);
+    return this;
+  }
+
+  @Override
+  public boolean hasArray() {
+    return buffer.hasArray();
+  }
+
+  @Override
+  public byte[] array() {
+    return buffer.array();
+  }
+
+  @Override
+  public int arrayOffset() {
+    return buffer.arrayOffset();
+  }
+
+  @Override
+  public boolean hasMemoryAddress() {
+    return buffer.hasMemoryAddress();
+  }
+
+  @Override
+  public long memoryAddress() {
+    return buffer.memoryAddress();
+  }
+
+  @Override
+  public byte getByte(int index) {
+    return _getByte(index);
+  }
+
+  @Override
+  protected byte _getByte(int index) {
+    return buffer.getByte(index);
+  }
+
+  @Override
+  public short getShort(int index) {
+    return _getShort(index);
+  }
+
+  @Override
+  protected short _getShort(int index) {
+    return buffer.getShort(index);
+  }
+
+  @Override
+  public int getUnsignedMedium(int index) {
+    return _getUnsignedMedium(index);
+  }
+
+  @Override
+  protected int _getUnsignedMedium(int index) {
+    return buffer.getUnsignedMedium(index);
+  }
+
+  @Override
+  public int getInt(int index) {
+    return _getInt(index);
+  }
+
+  @Override
+  protected int _getInt(int index) {
+    return buffer.getInt(index);
+  }
+
+  @Override
+  public long getLong(int index) {
+    return _getLong(index);
+  }
+
+  @Override
+  protected long _getLong(int index) {
+    return buffer.getLong(index);
+  }
+
+  @Override
+  public abstract ByteBuf copy(int index, int length);
+
+  @Override
+  public ByteBuf slice(int index, int length) {
+    return new SlicedByteBuf(this, index, length);
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+    buffer.getBytes(index, dst, dstIndex, length);
+    return this;
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+    buffer.getBytes(index, dst, dstIndex, length);
+    return this;
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, ByteBuffer dst) {
+    buffer.getBytes(index, dst);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setByte(int index, int value) {
+    _setByte(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setByte(int index, int value) {
+    buffer.setByte(index, value);
+  }
+
+  @Override
+  public ByteBuf setShort(int index, int value) {
+    _setShort(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setShort(int index, int value) {
+    buffer.setShort(index, value);
+  }
+
+  @Override
+  public ByteBuf setMedium(int index, int value) {
+    _setMedium(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setMedium(int index, int value) {
+    buffer.setMedium(index, value);
+  }
+
+  @Override
+  public ByteBuf setInt(int index, int value) {
+    _setInt(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setInt(int index, int value) {
+    buffer.setInt(index, value);
+  }
+
+  @Override
+  public ByteBuf setLong(int index, long value) {
+    _setLong(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setLong(int index, long value) {
+    buffer.setLong(index, value);
+  }
+
+  @Override
+  public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+    buffer.setBytes(index, src, srcIndex, length);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+    buffer.setBytes(index, src, srcIndex, length);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setBytes(int index, ByteBuffer src) {
+    buffer.setBytes(index, src);
+    return this;
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, OutputStream out, int length)
+      throws IOException {
+    buffer.getBytes(index, out, length);
+    return this;
+  }
+
+  @Override
+  public int getBytes(int index, GatheringByteChannel out, int length)
+      throws IOException {
+    return buffer.getBytes(index, out, length);
+  }
+
+  @Override
+  public int setBytes(int index, InputStream in, int length)
+      throws IOException {
+    return buffer.setBytes(index, in, length);
+  }
+
+  @Override
+  public int setBytes(int index, ScatteringByteChannel in, int length)
+      throws IOException {
+    return buffer.setBytes(index, in, length);
+  }
+
+  @Override
+  public int nioBufferCount() {
+    return buffer.nioBufferCount();
+  }
+
+  @Override
+  public ByteBuffer[] nioBuffers(int index, int length) {
+    return buffer.nioBuffers(index, length);
+  }
+
+  @Override
+  public ByteBuffer internalNioBuffer(int index, int length) {
+    return nioBuffer(index, length);
+  }
+
+  @Override
+  public int forEachByte(int index, int length, ByteBufProcessor processor) {
+    return buffer.forEachByte(index, length, processor);
+  }
+
+  @Override
+  public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
+    return buffer.forEachByteDesc(index, length, processor);
+  }
+
+  @Override
+  public final int refCnt() {
+    return unwrap().refCnt();
+  }
+
+  @Override
+  public final ByteBuf retain() {
+    unwrap().retain();
+    return this;
+  }
+
+  @Override
+  public final ByteBuf retain(int increment) {
+    unwrap().retain(increment);
+    return this;
+  }
+
+  @Override
+  public boolean release() {
+    return release(1);
+  }
+
+  @Override
+  public boolean release(int decrement) {
+    boolean released = unwrap().release(decrement);
+    return released;
+  }
+
+}
diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
new file mode 100644
index 0000000000000..1610028df9de3
--- /dev/null
+++ b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import io.netty.util.internal.StringUtil;
+
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.arrow.memory.OutOfMemoryException;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+
+/**
+ * The base allocator that we use for all of Drill's memory management. Returns UnsafeDirectLittleEndian buffers.
+ */
+public class PooledByteBufAllocatorL {
+  private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
+
+  private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
+
+
+  public static final String METRIC_PREFIX = "drill.allocator.";
+
+  private final MetricRegistry registry;
+  private final AtomicLong hugeBufferSize = new AtomicLong(0);
+  private final AtomicLong hugeBufferCount = new AtomicLong(0);
+  private final AtomicLong normalBufferSize = new AtomicLong(0);
+  private final AtomicLong normalBufferCount = new AtomicLong(0);
+
+  private final InnerAllocator allocator;
+  public final UnsafeDirectLittleEndian empty;
+
+  public PooledByteBufAllocatorL(MetricRegistry registry) {
+    this.registry = registry;
+    allocator = new InnerAllocator();
+    empty = new UnsafeDirectLittleEndian(new DuplicatedByteBuf(Unpooled.EMPTY_BUFFER));
+  }
+
+  public UnsafeDirectLittleEndian allocate(int size) {
+    try {
+      return allocator.directBuffer(size, Integer.MAX_VALUE);
+    } catch (OutOfMemoryError e) {
+      throw new OutOfMemoryException("Failure allocating buffer.", e);
+    }
+
+  }
+
+  public int getChunkSize() {
+    return allocator.chunkSize;
+  }
+
+  private class InnerAllocator extends PooledByteBufAllocator {
+
+
+    private final PoolArena<ByteBuffer>[] directArenas;
+    private final MemoryStatusThread statusThread;
+    private final Histogram largeBuffersHist;
+    private final Histogram normalBuffersHist;
+    private final int chunkSize;
+
+    public InnerAllocator() {
+      super(true);
+
+      try {
+        Field f = PooledByteBufAllocator.class.getDeclaredField("directArenas");
+        f.setAccessible(true);
+        this.directArenas = (PoolArena<ByteBuffer>[]) f.get(this);
+      } catch (Exception e) {
+        throw new RuntimeException("Failure while initializing allocator.  Unable to retrieve direct arenas field.", e);
+      }
+
+      this.chunkSize = directArenas[0].chunkSize;
+
+      if (memoryLogger.isTraceEnabled()) {
+        statusThread = new MemoryStatusThread();
+        statusThread.start();
+      } else {
+        statusThread = null;
+      }
+      removeOldMetrics();
+
+      registry.register(METRIC_PREFIX + "normal.size", new Gauge<Long>() {
+        @Override
+        public Long getValue() {
+          return normalBufferSize.get();
+        }
+      });
+
+      registry.register(METRIC_PREFIX + "normal.count", new Gauge<Long>() {
+        @Override
+        public Long getValue() {
+          return normalBufferCount.get();
+        }
+      });
+
+      registry.register(METRIC_PREFIX + "huge.size", new Gauge<Long>() {
+        @Override
+        public Long getValue() {
+          return hugeBufferSize.get();
+        }
+      });
+
+      registry.register(METRIC_PREFIX + "huge.count", new Gauge<Long>() {
+        @Override
+        public Long getValue() {
+          return hugeBufferCount.get();
+        }
+      });
+
+      largeBuffersHist = registry.histogram(METRIC_PREFIX + "huge.hist");
+      normalBuffersHist = registry.histogram(METRIC_PREFIX + "normal.hist");
+
+    }
+
+
+    private synchronized void removeOldMetrics() {
+      registry.removeMatching(new MetricFilter() {
+        @Override
+        public boolean matches(String name, Metric metric) {
+          return name.startsWith("drill.allocator.");
+        }
+
+      });
+    }
+
+    private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCapacity) {
+      PoolThreadCache cache = threadCache.get();
+      PoolArena<ByteBuffer> directArena = cache.directArena;
+
+      if (directArena != null) {
+
+        if (initialCapacity > directArena.chunkSize) {
+          // This is beyond chunk size so we'll allocate separately.
+          ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity);
+
+          hugeBufferCount.incrementAndGet();
+          hugeBufferSize.addAndGet(buf.capacity());
+          largeBuffersHist.update(buf.capacity());
+          // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
+          return new UnsafeDirectLittleEndian(new LargeBuffer(buf, hugeBufferSize, hugeBufferCount));
+
+        } else {
+          // within chunk, use arena.
+          ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity);
+          if (!(buf instanceof PooledUnsafeDirectByteBuf)) {
+            fail();
+          }
+
+          normalBuffersHist.update(buf.capacity());
+          if (ASSERT_ENABLED) {
+            normalBufferSize.addAndGet(buf.capacity());
+            normalBufferCount.incrementAndGet();
+          }
+
+          return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount,
+              normalBufferSize);
+        }
+
+      } else {
+        throw fail();
+      }
+    }
+
+    private UnsupportedOperationException fail() {
+      return new UnsupportedOperationException(
+          "Drill requries that the JVM used supports access sun.misc.Unsafe.  This platform didn't provide that functionality.");
+    }
+
+    public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
+      if (initialCapacity == 0 && maxCapacity == 0) {
+        newDirectBuffer(initialCapacity, maxCapacity);
+      }
+      validate(initialCapacity, maxCapacity);
+      return newDirectBufferL(initialCapacity, maxCapacity);
+    }
+
+    @Override
+    public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
+      throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+    }
+
+
+    private void validate(int initialCapacity, int maxCapacity) {
+      if (initialCapacity < 0) {
+        throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)");
+      }
+      if (initialCapacity > maxCapacity) {
+        throw new IllegalArgumentException(String.format(
+            "initialCapacity: %d (expected: not greater than maxCapacity(%d)",
+            initialCapacity, maxCapacity));
+      }
+    }
+
+    private class MemoryStatusThread extends Thread {
+
+      public MemoryStatusThread() {
+        super("memory-status-logger");
+        this.setDaemon(true);
+        this.setName("allocation.logger");
+      }
+
+      @Override
+      public void run() {
+        while (true) {
+          memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString());
+          try {
+            Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000);
+          } catch (InterruptedException e) {
+            return;
+          }
+
+        }
+      }
+
+    }
+
+    public String toString() {
+      StringBuilder buf = new StringBuilder();
+      buf.append(directArenas.length);
+      buf.append(" direct arena(s):");
+      buf.append(StringUtil.NEWLINE);
+      for (PoolArena<ByteBuffer> a : directArenas) {
+        buf.append(a);
+      }
+
+      buf.append("Large buffers outstanding: ");
+      buf.append(hugeBufferCount.get());
+      buf.append(" totaling ");
+      buf.append(hugeBufferSize.get());
+      buf.append(" bytes.");
+      buf.append('\n');
+      buf.append("Normal buffers outstanding: ");
+      buf.append(normalBufferCount.get());
+      buf.append(" totaling ");
+      buf.append(normalBufferSize.get());
+      buf.append(" bytes.");
+      return buf.toString();
+    }
+
+
+  }
+
+  public static final boolean ASSERT_ENABLED;
+
+  static {
+    boolean isAssertEnabled = false;
+    assert isAssertEnabled = true;
+    ASSERT_ENABLED = isAssertEnabled;
+  }
+
+}
diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
new file mode 100644
index 0000000000000..6495d5d371e76
--- /dev/null
+++ b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.netty.buffer;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.nio.ByteOrder;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * The underlying class we use for little-endian access to memory. Is used underneath DrillBufs to abstract away the
+ * Netty classes and underlying Netty memory management.
+ */
+public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
+  private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
+  private static final AtomicLong ID_GENERATOR = new AtomicLong(0);
+
+  public final long id = ID_GENERATOR.incrementAndGet();
+  private final AbstractByteBuf wrapped;
+  private final long memoryAddress;
+
+  private final AtomicLong bufferCount;
+  private final AtomicLong bufferSize;
+  private final long initCap;
+
+  UnsafeDirectLittleEndian(DuplicatedByteBuf buf) {
+    this(buf, true, null, null);
+  }
+
+  UnsafeDirectLittleEndian(LargeBuffer buf) {
+    this(buf, true, null, null);
+  }
+
+  UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong bufferCount, AtomicLong bufferSize) {
+    this(buf, true, bufferCount, bufferSize);
+
+  }
+
+  private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake, AtomicLong bufferCount, AtomicLong bufferSize) {
+    super(buf);
+    if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
+      throw new IllegalStateException("Drill only runs on LittleEndian systems.");
+    }
+
+    this.bufferCount = bufferCount;
+    this.bufferSize = bufferSize;
+
+    // initCap is used if we're tracking memory release. If we're in non-debug mode, we'll skip this.
+    this.initCap = ASSERT_ENABLED ? buf.capacity() : -1;
+
+    this.wrapped = buf;
+    this.memoryAddress = buf.memoryAddress();
+  }
+    private long addr(int index) {
+        return memoryAddress + index;
+    }
+
+    @Override
+    public long getLong(int index) {
+//        wrapped.checkIndex(index, 8);
+        long v = PlatformDependent.getLong(addr(index));
+        return v;
+    }
+
+    @Override
+    public float getFloat(int index) {
+        return Float.intBitsToFloat(getInt(index));
+    }
+
+  @Override
+  public ByteBuf slice() {
+    return slice(this.readerIndex(), readableBytes());
+  }
+
+  @Override
+  public ByteBuf slice(int index, int length) {
+    return new SlicedByteBuf(this, index, length);
+  }
+
+  @Override
+  public ByteOrder order() {
+    return ByteOrder.LITTLE_ENDIAN;
+  }
+
+  @Override
+  public ByteBuf order(ByteOrder endianness) {
+    return this;
+  }
+
+  @Override
+  public double getDouble(int index) {
+    return Double.longBitsToDouble(getLong(index));
+  }
+
+  @Override
+  public char getChar(int index) {
+    return (char) getShort(index);
+  }
+
+  @Override
+  public long getUnsignedInt(int index) {
+    return getInt(index) & 0xFFFFFFFFL;
+  }
+
+  @Override
+  public int getInt(int index) {
+    int v = PlatformDependent.getInt(addr(index));
+    return v;
+  }
+
+  @Override
+  public int getUnsignedShort(int index) {
+    return getShort(index) & 0xFFFF;
+  }
+
+  @Override
+  public short getShort(int index) {
+    short v = PlatformDependent.getShort(addr(index));
+    return v;
+  }
+
+  @Override
+  public ByteBuf setShort(int index, int value) {
+    wrapped.checkIndex(index, 2);
+    _setShort(index, value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setInt(int index, int value) {
+    wrapped.checkIndex(index, 4);
+    _setInt(index, value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setLong(int index, long value) {
+    wrapped.checkIndex(index, 8);
+    _setLong(index, value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setChar(int index, int value) {
+    setShort(index, value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf setFloat(int index, float value) {
+    setInt(index, Float.floatToRawIntBits(value));
+    return this;
+  }
+
+  @Override
+  public ByteBuf setDouble(int index, double value) {
+    setLong(index, Double.doubleToRawLongBits(value));
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeShort(int value) {
+    wrapped.ensureWritable(2);
+    _setShort(wrapped.writerIndex, value);
+    wrapped.writerIndex += 2;
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeInt(int value) {
+    wrapped.ensureWritable(4);
+    _setInt(wrapped.writerIndex, value);
+    wrapped.writerIndex += 4;
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeLong(long value) {
+    wrapped.ensureWritable(8);
+    _setLong(wrapped.writerIndex, value);
+    wrapped.writerIndex += 8;
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeChar(int value) {
+    writeShort(value);
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeFloat(float value) {
+    writeInt(Float.floatToRawIntBits(value));
+    return this;
+  }
+
+  @Override
+  public ByteBuf writeDouble(double value) {
+    writeLong(Double.doubleToRawLongBits(value));
+    return this;
+  }
+
+  private void _setShort(int index, int value) {
+    PlatformDependent.putShort(addr(index), (short) value);
+  }
+
+  private void _setInt(int index, int value) {
+    PlatformDependent.putInt(addr(index), value);
+  }
+
+  private void _setLong(int index, long value) {
+    PlatformDependent.putLong(addr(index), value);
+  }
+
+  @Override
+  public byte getByte(int index) {
+    return PlatformDependent.getByte(addr(index));
+  }
+
+  @Override
+  public ByteBuf setByte(int index, int value) {
+    PlatformDependent.putByte(addr(index), (byte) value);
+    return this;
+  }
+
+  @Override
+  public boolean release() {
+    return release(1);
+  }
+
+  @Override
+  public boolean release(int decrement) {
+    final boolean released = super.release(decrement);
+    if (ASSERT_ENABLED && released && bufferCount != null && bufferSize != null) {
+      bufferCount.decrementAndGet();
+      bufferSize.addAndGet(-initCap);
+    }
+    return released;
+  }
+
+  @Override
+  public int hashCode() {
+    return System.identityHashCode(this);
+  }
+
+  public static final boolean ASSERT_ENABLED;
+
+  static {
+    boolean isAssertEnabled = false;
+    assert isAssertEnabled = true;
+    ASSERT_ENABLED = isAssertEnabled;
+  }
+
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
new file mode 100644
index 0000000000000..dc75e5d7231a8
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Provides a concurrent way to manage account for memory usage without locking. Used as basis for Allocators. All
+ * operations are threadsafe (except for close).
+ */
+@ThreadSafe
+class Accountant implements AutoCloseable {
+  // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Accountant.class);
+
+  /**
+   * The parent allocator
+   */
+  protected final Accountant parent;
+
+  /**
+   * The amount of memory reserved for this allocator. Releases below this amount of memory will not be returned to the
+   * parent Accountant until this Accountant is closed.
+   */
+  protected final long reservation;
+
+  private final AtomicLong peakAllocation = new AtomicLong();
+
+  /**
+   * Maximum local memory that can be held. This can be externally updated. Changing it won't cause past memory to
+   * change but will change responses to future allocation efforts
+   */
+  private final AtomicLong allocationLimit = new AtomicLong();
+
+  /**
+   * Currently allocated amount of memory;
+   */
+  private final AtomicLong locallyHeldMemory = new AtomicLong();
+
+  public Accountant(Accountant parent, long reservation, long maxAllocation) {
+    Preconditions.checkArgument(reservation >= 0, "The initial reservation size must be non-negative.");
+    Preconditions.checkArgument(maxAllocation >= 0, "The maximum allocation limit must be non-negative.");
+    Preconditions.checkArgument(reservation <= maxAllocation,
+        "The initial reservation size must be <= the maximum allocation.");
+    Preconditions.checkArgument(reservation == 0 || parent != null, "The root accountant can't reserve memory.");
+
+    this.parent = parent;
+    this.reservation = reservation;
+    this.allocationLimit.set(maxAllocation);
+
+    if (reservation != 0) {
+      // we will allocate a reservation from our parent.
+      final AllocationOutcome outcome = parent.allocateBytes(reservation);
+      if (!outcome.isOk()) {
+        throw new OutOfMemoryException(String.format(
+            "Failure trying to allocate initial reservation for Allocator. "
+                + "Attempted to allocate %d bytes and received an outcome of %s.", reservation, outcome.name()));
+      }
+    }
+  }
+
+  /**
+   * Attempt to allocate the requested amount of memory. Either completely succeeds or completely fails. Constructs a a
+   * log of delta
+   *
+   * If it fails, no changes are made to accounting.
+   *
+   * @param size
+   *          The amount of memory to reserve in bytes.
+   * @return True if the allocation was successful, false if the allocation failed.
+   */
+  AllocationOutcome allocateBytes(long size) {
+    final AllocationOutcome outcome = allocate(size, true, false);
+    if (!outcome.isOk()) {
+      releaseBytes(size);
+    }
+    return outcome;
+  }
+
+  private void updatePeak() {
+    final long currentMemory = locallyHeldMemory.get();
+    while (true) {
+
+      final long previousPeak = peakAllocation.get();
+      if (currentMemory > previousPeak) {
+        if (!peakAllocation.compareAndSet(previousPeak, currentMemory)) {
+          // peak allocation changed underneath us. try again.
+          continue;
+        }
+      }
+
+      // we either succeeded to set peak allocation or we weren't above the previous peak, exit.
+      return;
+    }
+  }
+
+
+  /**
+   * Increase the accounting. Returns whether the allocation fit within limits.
+   *
+   * @param size
+   *          to increase
+   * @return Whether the allocation fit within limits.
+   */
+  boolean forceAllocate(long size) {
+    final AllocationOutcome outcome = allocate(size, true, true);
+    return outcome.isOk();
+  }
+
+  /**
+   * Internal method for allocation. This takes a forced approach to allocation to ensure that we manage reservation
+   * boundary issues consistently. Allocation is always done through the entire tree. The two options that we influence
+   * are whether the allocation should be forced and whether or not the peak memory allocation should be updated. If at
+   * some point during allocation escalation we determine that the allocation is no longer possible, we will continue to
+   * do a complete and consistent allocation but we will stop updating the peak allocation. We do this because we know
+   * that we will be directly unwinding this allocation (and thus never actually making the allocation). If force
+   * allocation is passed, then we continue to update the peak limits since we now know that this allocation will occur
+   * despite our moving past one or more limits.
+   *
+   * @param size
+   *          The size of the allocation.
+   * @param incomingUpdatePeak
+   *          Whether we should update the local peak for this allocation.
+   * @param forceAllocation
+   *          Whether we should force the allocation.
+   * @return The outcome of the allocation.
+   */
+  private AllocationOutcome allocate(final long size, final boolean incomingUpdatePeak, final boolean forceAllocation) {
+    final long newLocal = locallyHeldMemory.addAndGet(size);
+    final long beyondReservation = newLocal - reservation;
+    final boolean beyondLimit = newLocal > allocationLimit.get();
+    final boolean updatePeak = forceAllocation || (incomingUpdatePeak && !beyondLimit);
+
+    AllocationOutcome parentOutcome = AllocationOutcome.SUCCESS;
+    if (beyondReservation > 0 && parent != null) {
+      // we need to get memory from our parent.
+      final long parentRequest = Math.min(beyondReservation, size);
+      parentOutcome = parent.allocate(parentRequest, updatePeak, forceAllocation);
+    }
+
+    final AllocationOutcome finalOutcome = beyondLimit ? AllocationOutcome.FAILED_LOCAL :
+        parentOutcome.ok ? AllocationOutcome.SUCCESS : AllocationOutcome.FAILED_PARENT;
+
+    if (updatePeak) {
+      updatePeak();
+    }
+
+    return finalOutcome;
+  }
+
+  public void releaseBytes(long size) {
+    // reduce local memory. all memory released above reservation should be released up the tree.
+    final long newSize = locallyHeldMemory.addAndGet(-size);
+
+    Preconditions.checkArgument(newSize >= 0, "Accounted size went negative.");
+
+    final long originalSize = newSize + size;
+    if(originalSize > reservation && parent != null){
+      // we deallocated memory that we should release to our parent.
+      final long possibleAmountToReleaseToParent = originalSize - reservation;
+      final long actualToReleaseToParent = Math.min(size, possibleAmountToReleaseToParent);
+      parent.releaseBytes(actualToReleaseToParent);
+    }
+
+  }
+
+  /**
+   * Set the maximum amount of memory that can be allocated in the this Accountant before failing an allocation.
+   *
+   * @param newLimit
+   *          The limit in bytes.
+   */
+  public void setLimit(long newLimit) {
+    allocationLimit.set(newLimit);
+  }
+
+  public boolean isOverLimit() {
+    return getAllocatedMemory() > getLimit() || (parent != null && parent.isOverLimit());
+  }
+
+  /**
+   * Close this Accountant. This will release any reservation bytes back to a parent Accountant.
+   */
+  public void close() {
+    // return memory reservation to parent allocator.
+    if (parent != null) {
+      parent.releaseBytes(reservation);
+    }
+  }
+
+  /**
+   * Return the current limit of this Accountant.
+   *
+   * @return Limit in bytes.
+   */
+  public long getLimit() {
+    return allocationLimit.get();
+  }
+
+  /**
+   * Return the current amount of allocated memory that this Accountant is managing accounting for. Note this does not
+   * include reservation memory that hasn't been allocated.
+   *
+   * @return Currently allocate memory in bytes.
+   */
+  public long getAllocatedMemory() {
+    return locallyHeldMemory.get();
+  }
+
+  /**
+   * The peak memory allocated by this Accountant.
+   *
+   * @return The peak allocated memory in bytes.
+   */
+  public long getPeakMemoryAllocation() {
+    return peakAllocation.get();
+  }
+
+  /**
+   * Describes the type of outcome that occurred when trying to account for allocation of memory.
+   */
+  public static enum AllocationOutcome {
+
+    /**
+     * Allocation succeeded.
+     */
+    SUCCESS(true),
+
+    /**
+     * Allocation succeeded but only because the allocator was forced to move beyond a limit.
+     */
+    FORCED_SUCESS(true),
+
+    /**
+     * Allocation failed because the local allocator's limits were exceeded.
+     */
+    FAILED_LOCAL(false),
+
+    /**
+     * Allocation failed because a parent allocator's limits were exceeded.
+     */
+    FAILED_PARENT(false);
+
+    private final boolean ok;
+
+    AllocationOutcome(boolean ok) {
+      this.ok = ok;
+    }
+
+    public boolean isOk() {
+      return ok;
+    }
+  }
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
new file mode 100644
index 0000000000000..0db61443266c6
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -0,0 +1,433 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import static org.apache.arrow.memory.BaseAllocator.indent;
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.PooledByteBufAllocatorL;
+import io.netty.buffer.UnsafeDirectLittleEndian;
+
+import java.util.IdentityHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.arrow.memory.BaseAllocator.Verbosity;
+import org.apache.arrow.memory.util.AutoCloseableLock;
+import org.apache.arrow.memory.util.HistoricalLog;
+import org.apache.arrow.memory.util.Metrics;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Manages the relationship between one or more allocators and a particular UDLE. Ensures that one allocator owns the
+ * memory that multiple allocators may be referencing. Manages a BufferLedger between each of its associated allocators.
+ * This class is also responsible for managing when memory is allocated and returned to the Netty-based
+ * PooledByteBufAllocatorL.
+ *
+ * The only reason that this isn't package private is we're forced to put DrillBuf in Netty's package which need access
+ * to these objects or methods.
+ *
+ * Threading: AllocationManager manages thread-safety internally. Operations within the context of a single BufferLedger
+ * are lockless in nature and can be leveraged by multiple threads. Operations that cross the context of two ledgers
+ * will acquire a lock on the AllocationManager instance. Important note, there is one AllocationManager per
+ * UnsafeDirectLittleEndian buffer allocation. As such, there will be thousands of these in a typical query. The
+ * contention of acquiring a lock on AllocationManager should be very low.
+ *
+ */
+public class AllocationManager {
+  // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationManager.class);
+
+  private static final AtomicLong MANAGER_ID_GENERATOR = new AtomicLong(0);
+  private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0);
+  static final PooledByteBufAllocatorL INNER_ALLOCATOR = new PooledByteBufAllocatorL(Metrics.getInstance());
+
+  private final RootAllocator root;
+  private final long allocatorManagerId = MANAGER_ID_GENERATOR.incrementAndGet();
+  private final int size;
+  private final UnsafeDirectLittleEndian underlying;
+  private final IdentityHashMap<BufferAllocator, BufferLedger> map = new IdentityHashMap<>();
+  private final ReadWriteLock lock = new ReentrantReadWriteLock();
+  private final AutoCloseableLock readLock = new AutoCloseableLock(lock.readLock());
+  private final AutoCloseableLock writeLock = new AutoCloseableLock(lock.writeLock());
+  private final long amCreationTime = System.nanoTime();
+
+  private volatile BufferLedger owningLedger;
+  private volatile long amDestructionTime = 0;
+
+  AllocationManager(BaseAllocator accountingAllocator, int size) {
+    Preconditions.checkNotNull(accountingAllocator);
+    accountingAllocator.assertOpen();
+
+    this.root = accountingAllocator.root;
+    this.underlying = INNER_ALLOCATOR.allocate(size);
+
+    // we do a no retain association since our creator will want to retrieve the newly created ledger and will create a
+    // reference count at that point
+    this.owningLedger = associate(accountingAllocator, false);
+    this.size = underlying.capacity();
+  }
+
+  /**
+   * Associate the existing underlying buffer with a new allocator. This will increase the reference count to the
+   * provided ledger by 1.
+   * @param allocator
+   *          The target allocator to associate this buffer with.
+   * @return The Ledger (new or existing) that associates the underlying buffer to this new ledger.
+   */
+  BufferLedger associate(final BaseAllocator allocator) {
+    return associate(allocator, true);
+  }
+
+  private BufferLedger associate(final BaseAllocator allocator, final boolean retain) {
+    allocator.assertOpen();
+
+    if (root != allocator.root) {
+      throw new IllegalStateException(
+          "A buffer can only be associated between two allocators that share the same root.");
+    }
+
+    try (AutoCloseableLock read = readLock.open()) {
+
+      final BufferLedger ledger = map.get(allocator);
+      if (ledger != null) {
+        if (retain) {
+          ledger.inc();
+        }
+        return ledger;
+      }
+
+    }
+    try (AutoCloseableLock write = writeLock.open()) {
+      // we have to recheck existing ledger since a second reader => writer could be competing with us.
+
+      final BufferLedger existingLedger = map.get(allocator);
+      if (existingLedger != null) {
+        if (retain) {
+          existingLedger.inc();
+        }
+        return existingLedger;
+      }
+
+      final BufferLedger ledger = new BufferLedger(allocator, new ReleaseListener(allocator));
+      if (retain) {
+        ledger.inc();
+      }
+      BufferLedger oldLedger = map.put(allocator, ledger);
+      Preconditions.checkArgument(oldLedger == null);
+      allocator.associateLedger(ledger);
+      return ledger;
+    }
+  }
+
+
+  /**
+   * The way that a particular BufferLedger communicates back to the AllocationManager that it now longer needs to hold
+   * a reference to particular piece of memory.
+   */
+  private class ReleaseListener {
+
+    private final BufferAllocator allocator;
+
+    public ReleaseListener(BufferAllocator allocator) {
+      this.allocator = allocator;
+    }
+
+    /**
+     * Can only be called when you already hold the writeLock.
+     */
+    public void release() {
+      allocator.assertOpen();
+
+      final BufferLedger oldLedger = map.remove(allocator);
+      oldLedger.allocator.dissociateLedger(oldLedger);
+
+      if (oldLedger == owningLedger) {
+        if (map.isEmpty()) {
+          // no one else owns, lets release.
+          oldLedger.allocator.releaseBytes(size);
+          underlying.release();
+          amDestructionTime = System.nanoTime();
+          owningLedger = null;
+        } else {
+          // we need to change the owning allocator. we've been removed so we'll get whatever is top of list
+          BufferLedger newLedger = map.values().iterator().next();
+
+          // we'll forcefully transfer the ownership and not worry about whether we exceeded the limit
+          // since this consumer can't do anything with this.
+          oldLedger.transferBalance(newLedger);
+        }
+      } else {
+        if (map.isEmpty()) {
+          throw new IllegalStateException("The final removal of a ledger should be connected to the owning ledger.");
+        }
+      }
+
+
+    }
+  }
+
+  /**
+   * The reference manager that binds an allocator manager to a particular BaseAllocator. Also responsible for creating
+   * a set of DrillBufs that share a common fate and set of reference counts.
+   * As with AllocationManager, the only reason this is public is due to DrillBuf being in io.netty.buffer package.
+   */
+  public class BufferLedger {
+
+    private final IdentityHashMap<ArrowBuf, Object> buffers =
+        BaseAllocator.DEBUG ? new IdentityHashMap<ArrowBuf, Object>() : null;
+
+    private final long ledgerId = LEDGER_ID_GENERATOR.incrementAndGet(); // unique ID assigned to each ledger
+    private final AtomicInteger bufRefCnt = new AtomicInteger(0); // start at zero so we can manage request for retain
+                                                                  // correctly
+    private final long lCreationTime = System.nanoTime();
+    private volatile long lDestructionTime = 0;
+    private final BaseAllocator allocator;
+    private final ReleaseListener listener;
+    private final HistoricalLog historicalLog = BaseAllocator.DEBUG ? new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH,
+        "BufferLedger[%d]", 1)
+        : null;
+
+    private BufferLedger(BaseAllocator allocator, ReleaseListener listener) {
+      this.allocator = allocator;
+      this.listener = listener;
+    }
+
+    /**
+     * Transfer any balance the current ledger has to the target ledger. In the case that the current ledger holds no
+     * memory, no transfer is made to the new ledger.
+     * @param target
+     *          The ledger to transfer ownership account to.
+     * @return Whether transfer fit within target ledgers limits.
+     */
+    public boolean transferBalance(final BufferLedger target) {
+      Preconditions.checkNotNull(target);
+      Preconditions.checkArgument(allocator.root == target.allocator.root,
+          "You can only transfer between two allocators that share the same root.");
+      allocator.assertOpen();
+
+      target.allocator.assertOpen();
+      // if we're transferring to ourself, just return.
+      if (target == this) {
+        return true;
+      }
+
+      // since two balance transfers out from the allocator manager could cause incorrect accounting, we need to ensure
+      // that this won't happen by synchronizing on the allocator manager instance.
+      try (AutoCloseableLock write = writeLock.open()) {
+        if (owningLedger != this) {
+          return true;
+        }
+
+        if (BaseAllocator.DEBUG) {
+          this.historicalLog.recordEvent("transferBalance(%s)", target.allocator.name);
+          target.historicalLog.recordEvent("incoming(from %s)", owningLedger.allocator.name);
+        }
+
+        boolean overlimit = target.allocator.forceAllocate(size);
+        allocator.releaseBytes(size);
+        owningLedger = target;
+        return overlimit;
+      }
+
+    }
+
+    /**
+     * Print the current ledger state to a the provided StringBuilder.
+     * @param sb
+     *          The StringBuilder to populate.
+     * @param indent
+     *          The level of indentation to position the data.
+     * @param verbosity
+     *          The level of verbosity to print.
+     */
+    public void print(StringBuilder sb, int indent, Verbosity verbosity) {
+      indent(sb, indent)
+          .append("ledger[")
+          .append(ledgerId)
+          .append("] allocator: ")
+          .append(allocator.name)
+          .append("), isOwning: ")
+          .append(owningLedger == this)
+          .append(", size: ")
+          .append(size)
+          .append(", references: ")
+          .append(bufRefCnt.get())
+          .append(", life: ")
+          .append(lCreationTime)
+          .append("..")
+          .append(lDestructionTime)
+          .append(", allocatorManager: [")
+          .append(AllocationManager.this.allocatorManagerId)
+          .append(", life: ")
+          .append(amCreationTime)
+          .append("..")
+          .append(amDestructionTime);
+
+      if (!BaseAllocator.DEBUG) {
+        sb.append("]\n");
+      } else {
+        synchronized (buffers) {
+          sb.append("] holds ")
+              .append(buffers.size())
+              .append(" buffers. \n");
+          for (ArrowBuf buf : buffers.keySet()) {
+            buf.print(sb, indent + 2, verbosity);
+            sb.append('\n');
+          }
+        }
+      }
+
+    }
+
+    private void inc() {
+      bufRefCnt.incrementAndGet();
+    }
+
+    /**
+     * Decrement the ledger's reference count. If the ledger is decremented to zero, this ledger should release its
+     * ownership back to the AllocationManager
+     */
+    public int decrement(int decrement) {
+      allocator.assertOpen();
+
+      final int outcome;
+      try (AutoCloseableLock write = writeLock.open()) {
+        outcome = bufRefCnt.addAndGet(-decrement);
+        if (outcome == 0) {
+          lDestructionTime = System.nanoTime();
+          listener.release();
+        }
+      }
+
+      return outcome;
+    }
+
+    /**
+     * Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator doesn't currently have a
+     * ledger associated with this AllocationManager, a new one is created. This is placed on BufferLedger rather than
+     * AllocationManager directly because DrillBufs don't have access to AllocationManager and they are the ones
+     * responsible for exposing the ability to associate multiple allocators with a particular piece of underlying
+     * memory. Note that this will increment the reference count of this ledger by one to ensure the ledger isn't
+     * destroyed before use.
+     *
+     * @param allocator
+     * @return
+     */
+    public BufferLedger getLedgerForAllocator(BufferAllocator allocator) {
+      return associate((BaseAllocator) allocator);
+    }
+
+    /**
+     * Create a new DrillBuf associated with this AllocationManager and memory. Does not impact reference count.
+     * Typically used for slicing.
+     * @param offset
+     *          The offset in bytes to start this new DrillBuf.
+     * @param length
+     *          The length in bytes that this DrillBuf will provide access to.
+     * @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
+     */
+    public ArrowBuf newDrillBuf(int offset, int length) {
+      allocator.assertOpen();
+      return newDrillBuf(offset, length, null);
+    }
+
+    /**
+     * Create a new DrillBuf associated with this AllocationManager and memory.
+     * @param offset
+     *          The offset in bytes to start this new DrillBuf.
+     * @param length
+     *          The length in bytes that this DrillBuf will provide access to.
+     * @param manager
+     *          An optional BufferManager argument that can be used to manage expansion of this DrillBuf
+     * @param retain
+     *          Whether or not the newly created buffer should get an additional reference count added to it.
+     * @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
+     */
+    public ArrowBuf newDrillBuf(int offset, int length, BufferManager manager) {
+      allocator.assertOpen();
+
+      final ArrowBuf buf = new ArrowBuf(
+          bufRefCnt,
+          this,
+          underlying,
+          manager,
+          allocator.getAsByteBufAllocator(),
+          offset,
+          length,
+          false);
+
+      if (BaseAllocator.DEBUG) {
+        historicalLog.recordEvent(
+            "DrillBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == "
+                + "%d](%s)) => ledger hc == %d",
+            allocator.name, System.identityHashCode(buf), buf.toString(),
+            System.identityHashCode(this));
+
+        synchronized (buffers) {
+          buffers.put(buf, null);
+        }
+      }
+
+      return buf;
+
+    }
+
+    /**
+     * What is the total size (in bytes) of memory underlying this ledger.
+     *
+     * @return Size in bytes
+     */
+    public int getSize() {
+      return size;
+    }
+
+    /**
+     * How much memory is accounted for by this ledger. This is either getSize() if this is the owning ledger for the
+     * memory or zero in the case that this is not the owning ledger associated with this memory.
+     *
+     * @return Amount of accounted(owned) memory associated with this ledger.
+     */
+    public int getAccountedSize() {
+      try (AutoCloseableLock read = readLock.open()) {
+        if (owningLedger == this) {
+          return size;
+        } else {
+          return 0;
+        }
+      }
+    }
+
+    /**
+     * Package visible for debugging/verification only.
+     */
+    UnsafeDirectLittleEndian getUnderlying() {
+      return underlying;
+    }
+
+    /**
+     * Package visible for debugging/verification only.
+     */
+    boolean isOwningLedger() {
+      return this == owningLedger;
+    }
+
+  }
+
+}
\ No newline at end of file
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
new file mode 100644
index 0000000000000..68d1244d1e328
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Supports cumulative allocation reservation. Clients may increase the size of the reservation repeatedly until they
+ * call for an allocation of the current total size. The reservation can only be used once, and will throw an exception
+ * if it is used more than once.
+ * <p>
+ * For the purposes of airtight memory accounting, the reservation must be close()d whether it is used or not.
+ * This is not threadsafe.
+ */
+public interface AllocationReservation extends AutoCloseable {
+
+  /**
+   * Add to the current reservation.
+   *
+   * <p>Adding may fail if the allocator is not allowed to consume any more space.
+   *
+   * @param nBytes the number of bytes to add
+   * @return true if the addition is possible, false otherwise
+   * @throws IllegalStateException if called after buffer() is used to allocate the reservation
+   */
+  boolean add(final int nBytes);
+
+  /**
+   * Requests a reservation of additional space.
+   *
+   * <p>The implementation of the allocator's inner class provides this.
+   *
+   * @param nBytes the amount to reserve
+   * @return true if the reservation can be satisfied, false otherwise
+   */
+  boolean reserve(int nBytes);
+
+  /**
+   * Allocate a buffer whose size is the total of all the add()s made.
+   *
+   * <p>The allocation request can still fail, even if the amount of space
+   * requested is available, if the allocation cannot be made contiguously.
+   *
+   * @return the buffer, or null, if the request cannot be satisfied
+   * @throws IllegalStateException if called called more than once
+   */
+  ArrowBuf allocateBuffer();
+
+  /**
+   * Get the current size of the reservation (the sum of all the add()s).
+   *
+   * @return size of the current reservation
+   */
+  int getSize();
+
+  /**
+   * Return whether or not the reservation has been used.
+   *
+   * @return whether or not the reservation has been used
+   */
+  public boolean isUsed();
+
+  /**
+   * Return whether or not the reservation has been closed.
+   *
+   * @return whether or not the reservation has been closed
+   */
+  public boolean isClosed();
+
+  public void close();
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
new file mode 100644
index 0000000000000..566457981c7ed
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+/**
+ * Exception thrown when a closed BufferAllocator is used. Note
+ * this is an unchecked exception.
+ *
+ * @param message string associated with the cause
+ */
+@SuppressWarnings("serial")
+public class AllocatorClosedException extends RuntimeException {
+  public AllocatorClosedException(String message) {
+    super(message);
+  }
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
new file mode 100644
index 0000000000000..72f77ab0c7bc2
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -0,0 +1,781 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.UnsafeDirectLittleEndian;
+
+import java.util.Arrays;
+import java.util.IdentityHashMap;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.arrow.memory.AllocationManager.BufferLedger;
+import org.apache.arrow.memory.util.AssertionUtil;
+import org.apache.arrow.memory.util.HistoricalLog;
+
+import com.google.common.base.Preconditions;
+
+public abstract class BaseAllocator extends Accountant implements BufferAllocator {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseAllocator.class);
+
+  public static final String DEBUG_ALLOCATOR = "arrow.memory.debug.allocator";
+
+  private static final AtomicLong ID_GENERATOR = new AtomicLong(0);
+  private static final int CHUNK_SIZE = AllocationManager.INNER_ALLOCATOR.getChunkSize();
+
+  public static final int DEBUG_LOG_LENGTH = 6;
+  public static final boolean DEBUG = AssertionUtil.isAssertionsEnabled()
+      || Boolean.parseBoolean(System.getProperty(DEBUG_ALLOCATOR, "false"));
+  private final Object DEBUG_LOCK = DEBUG ? new Object() : null;
+
+  private final BaseAllocator parentAllocator;
+  private final ByteBufAllocator thisAsByteBufAllocator;
+  private final IdentityHashMap<BaseAllocator, Object> childAllocators;
+  private final ArrowBuf empty;
+
+  private volatile boolean isClosed = false; // the allocator has been closed
+
+  // Package exposed for sharing between AllocatorManger and BaseAllocator objects
+  final String name;
+  final RootAllocator root;
+
+  // members used purely for debugging
+  private final IdentityHashMap<BufferLedger, Object> childLedgers;
+  private final IdentityHashMap<Reservation, Object> reservations;
+  private final HistoricalLog historicalLog;
+
+  protected BaseAllocator(
+      final BaseAllocator parentAllocator,
+      final String name,
+      final long initReservation,
+      final long maxAllocation) throws OutOfMemoryException {
+    super(parentAllocator, initReservation, maxAllocation);
+
+    if (parentAllocator != null) {
+      this.root = parentAllocator.root;
+      empty = parentAllocator.empty;
+    } else if (this instanceof RootAllocator) {
+      this.root = (RootAllocator) this;
+      empty = createEmpty();
+    } else {
+      throw new IllegalStateException("An parent allocator must either carry a root or be the root.");
+    }
+
+    this.parentAllocator = parentAllocator;
+    this.name = name;
+
+    this.thisAsByteBufAllocator = new DrillByteBufAllocator(this);
+
+    if (DEBUG) {
+      childAllocators = new IdentityHashMap<>();
+      reservations = new IdentityHashMap<>();
+      childLedgers = new IdentityHashMap<>();
+      historicalLog = new HistoricalLog(DEBUG_LOG_LENGTH, "allocator[%s]", name);
+      hist("created by \"%s\", owned = %d", name, this.getAllocatedMemory());
+    } else {
+      childAllocators = null;
+      reservations = null;
+      historicalLog = null;
+      childLedgers = null;
+    }
+
+  }
+
+  public void assertOpen() {
+    if (AssertionUtil.ASSERT_ENABLED) {
+      if (isClosed) {
+        throw new IllegalStateException("Attempting operation on allocator when allocator is closed.\n"
+            + toVerboseString());
+      }
+    }
+  }
+
+  @Override
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public ArrowBuf getEmpty() {
+    assertOpen();
+    return empty;
+  }
+
+  /**
+   * For debug/verification purposes only. Allows an AllocationManager to tell the allocator that we have a new ledger
+   * associated with this allocator.
+   */
+  void associateLedger(BufferLedger ledger) {
+    assertOpen();
+    if (DEBUG) {
+      synchronized (DEBUG_LOCK) {
+        childLedgers.put(ledger, null);
+      }
+    }
+  }
+
+  /**
+   * For debug/verification purposes only. Allows an AllocationManager to tell the allocator that we are removing a
+   * ledger associated with this allocator
+   */
+  void dissociateLedger(BufferLedger ledger) {
+    assertOpen();
+    if (DEBUG) {
+      synchronized (DEBUG_LOCK) {
+        if (!childLedgers.containsKey(ledger)) {
+          throw new IllegalStateException("Trying to remove a child ledger that doesn't exist.");
+        }
+        childLedgers.remove(ledger);
+      }
+    }
+  }
+
+  /**
+   * Track when a ChildAllocator of this BaseAllocator is closed. Used for debugging purposes.
+   *
+   * @param childAllocator
+   *          The child allocator that has been closed.
+   */
+  private void childClosed(final BaseAllocator childAllocator) {
+    assertOpen();
+
+    if (DEBUG) {
+      Preconditions.checkArgument(childAllocator != null, "child allocator can't be null");
+
+      synchronized (DEBUG_LOCK) {
+        final Object object = childAllocators.remove(childAllocator);
+        if (object == null) {
+          childAllocator.historicalLog.logHistory(logger);
+          throw new IllegalStateException("Child allocator[" + childAllocator.name
+              + "] not found in parent allocator[" + name + "]'s childAllocators");
+        }
+      }
+    }
+  }
+
+  private static String createErrorMsg(final BufferAllocator allocator, final int rounded, final int requested) {
+    if (rounded != requested) {
+      return String.format(
+          "Unable to allocate buffer of size %d (rounded from %d) due to memory limit. Current allocation: %d",
+          rounded, requested, allocator.getAllocatedMemory());
+    } else {
+      return String.format("Unable to allocate buffer of size %d due to memory limit. Current allocation: %d",
+          rounded, allocator.getAllocatedMemory());
+    }
+  }
+
+  @Override
+  public ArrowBuf buffer(final int initialRequestSize) {
+    assertOpen();
+
+    return buffer(initialRequestSize, null);
+  }
+
+  private ArrowBuf createEmpty(){
+    assertOpen();
+
+    return new ArrowBuf(new AtomicInteger(), null, AllocationManager.INNER_ALLOCATOR.empty, null, null, 0, 0, true);
+  }
+
+  @Override
+  public ArrowBuf buffer(final int initialRequestSize, BufferManager manager) {
+    assertOpen();
+
+    Preconditions.checkArgument(initialRequestSize >= 0, "the requested size must be non-negative");
+
+    if (initialRequestSize == 0) {
+      return empty;
+    }
+
+    // round to next largest power of two if we're within a chunk since that is how our allocator operates
+    final int actualRequestSize = initialRequestSize < CHUNK_SIZE ?
+        nextPowerOfTwo(initialRequestSize)
+        : initialRequestSize;
+    AllocationOutcome outcome = this.allocateBytes(actualRequestSize);
+    if (!outcome.isOk()) {
+      throw new OutOfMemoryException(createErrorMsg(this, actualRequestSize, initialRequestSize));
+    }
+
+    boolean success = false;
+    try {
+      ArrowBuf buffer = bufferWithoutReservation(actualRequestSize, manager);
+      success = true;
+      return buffer;
+    } finally {
+      if (!success) {
+        releaseBytes(actualRequestSize);
+      }
+    }
+
+  }
+
+  /**
+   * Used by usual allocation as well as for allocating a pre-reserved buffer. Skips the typical accounting associated
+   * with creating a new buffer.
+   */
+  private ArrowBuf bufferWithoutReservation(final int size, BufferManager bufferManager) throws OutOfMemoryException {
+    assertOpen();
+
+    final AllocationManager manager = new AllocationManager(this, size);
+    final BufferLedger ledger = manager.associate(this); // +1 ref cnt (required)
+    final ArrowBuf buffer = ledger.newDrillBuf(0, size, bufferManager);
+
+    // make sure that our allocation is equal to what we expected.
+    Preconditions.checkArgument(buffer.capacity() == size,
+        "Allocated capacity %d was not equal to requested capacity %d.", buffer.capacity(), size);
+
+    return buffer;
+  }
+
+  @Override
+  public ByteBufAllocator getAsByteBufAllocator() {
+    return thisAsByteBufAllocator;
+  }
+
+  @Override
+  public BufferAllocator newChildAllocator(
+      final String name,
+      final long initReservation,
+      final long maxAllocation) {
+    assertOpen();
+
+    final ChildAllocator childAllocator = new ChildAllocator(this, name, initReservation, maxAllocation);
+
+    if (DEBUG) {
+      synchronized (DEBUG_LOCK) {
+        childAllocators.put(childAllocator, childAllocator);
+        historicalLog.recordEvent("allocator[%s] created new child allocator[%s]", name, childAllocator.name);
+      }
+    }
+
+    return childAllocator;
+  }
+
+  public class Reservation implements AllocationReservation {
+    private int nBytes = 0;
+    private boolean used = false;
+    private boolean closed = false;
+    private final HistoricalLog historicalLog;
+
+    public Reservation() {
+      if (DEBUG) {
+        historicalLog = new HistoricalLog("Reservation[allocator[%s], %d]", name, System.identityHashCode(this));
+        historicalLog.recordEvent("created");
+        synchronized (DEBUG_LOCK) {
+          reservations.put(this, this);
+        }
+      } else {
+        historicalLog = null;
+      }
+    }
+
+    public boolean add(final int nBytes) {
+      assertOpen();
+
+      Preconditions.checkArgument(nBytes >= 0, "nBytes(%d) < 0", nBytes);
+      Preconditions.checkState(!closed, "Attempt to increase reservation after reservation has been closed");
+      Preconditions.checkState(!used, "Attempt to increase reservation after reservation has been used");
+
+      // we round up to next power of two since all reservations are done in powers of two. This may overestimate the
+      // preallocation since someone may perceive additions to be power of two. If this becomes a problem, we can look
+      // at
+      // modifying this behavior so that we maintain what we reserve and what the user asked for and make sure to only
+      // round to power of two as necessary.
+      final int nBytesTwo = BaseAllocator.nextPowerOfTwo(nBytes);
+      if (!reserve(nBytesTwo)) {
+        return false;
+      }
+
+      this.nBytes += nBytesTwo;
+      return true;
+    }
+
+    public ArrowBuf allocateBuffer() {
+      assertOpen();
+
+      Preconditions.checkState(!closed, "Attempt to allocate after closed");
+      Preconditions.checkState(!used, "Attempt to allocate more than once");
+
+      final ArrowBuf drillBuf = allocate(nBytes);
+      used = true;
+      return drillBuf;
+    }
+
+    public int getSize() {
+      return nBytes;
+    }
+
+    public boolean isUsed() {
+      return used;
+    }
+
+    public boolean isClosed() {
+      return closed;
+    }
+
+    @Override
+    public void close() {
+      assertOpen();
+
+      if (closed) {
+        return;
+      }
+
+      if (DEBUG) {
+        if (!isClosed()) {
+          final Object object;
+          synchronized (DEBUG_LOCK) {
+            object = reservations.remove(this);
+          }
+          if (object == null) {
+            final StringBuilder sb = new StringBuilder();
+            print(sb, 0, Verbosity.LOG_WITH_STACKTRACE);
+            logger.debug(sb.toString());
+            throw new IllegalStateException(
+                String.format("Didn't find closing reservation[%d]", System.identityHashCode(this)));
+          }
+
+          historicalLog.recordEvent("closed");
+        }
+      }
+
+      if (!used) {
+        releaseReservation(nBytes);
+      }
+
+      closed = true;
+    }
+
+    public boolean reserve(int nBytes) {
+      assertOpen();
+
+      final AllocationOutcome outcome = BaseAllocator.this.allocateBytes(nBytes);
+
+      if (DEBUG) {
+        historicalLog.recordEvent("reserve(%d) => %s", nBytes, Boolean.toString(outcome.isOk()));
+      }
+
+      return outcome.isOk();
+    }
+
+    /**
+     * Allocate the a buffer of the requested size.
+     *
+     * <p>
+     * The implementation of the allocator's inner class provides this.
+     *
+     * @param nBytes
+     *          the size of the buffer requested
+     * @return the buffer, or null, if the request cannot be satisfied
+     */
+    private ArrowBuf allocate(int nBytes) {
+      assertOpen();
+
+      boolean success = false;
+
+      /*
+       * The reservation already added the requested bytes to the allocators owned and allocated bytes via reserve().
+       * This ensures that they can't go away. But when we ask for the buffer here, that will add to the allocated bytes
+       * as well, so we need to return the same number back to avoid double-counting them.
+       */
+      try {
+        final ArrowBuf drillBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);
+
+        if (DEBUG) {
+          historicalLog.recordEvent("allocate() => %s", String.format("DrillBuf[%d]", drillBuf.getId()));
+        }
+        success = true;
+        return drillBuf;
+      } finally {
+        if (!success) {
+          releaseBytes(nBytes);
+        }
+      }
+    }
+
+    /**
+     * Return the reservation back to the allocator without having used it.
+     *
+     * @param nBytes
+     *          the size of the reservation
+     */
+    private void releaseReservation(int nBytes) {
+      assertOpen();
+
+      releaseBytes(nBytes);
+
+      if (DEBUG) {
+        historicalLog.recordEvent("releaseReservation(%d)", nBytes);
+      }
+    }
+
+  }
+
+  @Override
+  public AllocationReservation newReservation() {
+    assertOpen();
+
+    return new Reservation();
+  }
+
+
+  @Override
+  public synchronized void close() {
+    /*
+     * Some owners may close more than once because of complex cleanup and shutdown
+     * procedures.
+     */
+    if (isClosed) {
+      return;
+    }
+
+    isClosed = true;
+
+    if (DEBUG) {
+      synchronized(DEBUG_LOCK) {
+        verifyAllocator();
+
+        // are there outstanding child allocators?
+        if (!childAllocators.isEmpty()) {
+          for (final BaseAllocator childAllocator : childAllocators.keySet()) {
+            if (childAllocator.isClosed) {
+              logger.warn(String.format(
+                  "Closed child allocator[%s] on parent allocator[%s]'s child list.\n%s",
+                  childAllocator.name, name, toString()));
+            }
+          }
+
+          throw new IllegalStateException(
+              String.format("Allocator[%s] closed with outstanding child allocators.\n%s", name, toString()));
+        }
+
+        // are there outstanding buffers?
+        final int allocatedCount = childLedgers.size();
+        if (allocatedCount > 0) {
+          throw new IllegalStateException(
+              String.format("Allocator[%s] closed with outstanding buffers allocated (%d).\n%s",
+                  name, allocatedCount, toString()));
+        }
+
+        if (reservations.size() != 0) {
+          throw new IllegalStateException(
+              String.format("Allocator[%s] closed with outstanding reservations (%d).\n%s", name, reservations.size(),
+                  toString()));
+        }
+
+      }
+    }
+
+    // Is there unaccounted-for outstanding allocation?
+    final long allocated = getAllocatedMemory();
+    if (allocated > 0) {
+      throw new IllegalStateException(
+          String.format("Memory was leaked by query. Memory leaked: (%d)\n%s", allocated, toString()));
+    }
+
+    // we need to release our memory to our parent before we tell it we've closed.
+    super.close();
+
+    // Inform our parent allocator that we've closed
+    if (parentAllocator != null) {
+      parentAllocator.childClosed(this);
+    }
+
+    if (DEBUG) {
+      historicalLog.recordEvent("closed");
+      logger.debug(String.format(
+          "closed allocator[%s].",
+          name));
+    }
+
+
+  }
+
+  public String toString() {
+    final Verbosity verbosity = logger.isTraceEnabled() ? Verbosity.LOG_WITH_STACKTRACE
+        : Verbosity.BASIC;
+    final StringBuilder sb = new StringBuilder();
+    print(sb, 0, verbosity);
+    return sb.toString();
+  }
+
+  /**
+   * Provide a verbose string of the current allocator state. Includes the state of all child allocators, along with
+   * historical logs of each object and including stacktraces.
+   *
+   * @return A Verbose string of current allocator state.
+   */
+  public String toVerboseString() {
+    final StringBuilder sb = new StringBuilder();
+    print(sb, 0, Verbosity.LOG_WITH_STACKTRACE);
+    return sb.toString();
+  }
+
+  private void hist(String noteFormat, Object... args) {
+    historicalLog.recordEvent(noteFormat, args);
+  }
+
+  /**
+   * Rounds up the provided value to the nearest power of two.
+   *
+   * @param val
+   *          An integer value.
+   * @return The closest power of two of that value.
+   */
+  static int nextPowerOfTwo(int val) {
+    int highestBit = Integer.highestOneBit(val);
+    if (highestBit == val) {
+      return val;
+    } else {
+      return highestBit << 1;
+    }
+  }
+
+
+  /**
+   * Verifies the accounting state of the allocator. Only works for DEBUG.
+   *
+   * @throws IllegalStateException
+   *           when any problems are found
+   */
+  void verifyAllocator() {
+    final IdentityHashMap<UnsafeDirectLittleEndian, BaseAllocator> buffersSeen = new IdentityHashMap<>();
+    verifyAllocator(buffersSeen);
+  }
+
+  /**
+   * Verifies the accounting state of the allocator. Only works for DEBUG.
+   *
+   * <p>
+   * This overload is used for recursive calls, allowing for checking that DrillBufs are unique across all allocators
+   * that are checked.
+   * </p>
+   *
+   * @param buffersSeen
+   *          a map of buffers that have already been seen when walking a tree of allocators
+   * @throws IllegalStateException
+   *           when any problems are found
+   */
+  private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, BaseAllocator> buffersSeen) {
+    synchronized (DEBUG_LOCK) {
+
+      // The remaining tests can only be performed if we're in debug mode.
+      if (!DEBUG) {
+        return;
+      }
+
+      final long allocated = getAllocatedMemory();
+
+      // verify my direct descendants
+      final Set<BaseAllocator> childSet = childAllocators.keySet();
+      for (final BaseAllocator childAllocator : childSet) {
+        childAllocator.verifyAllocator(buffersSeen);
+      }
+
+      /*
+       * Verify my relationships with my descendants.
+       *
+       * The sum of direct child allocators' owned memory must be <= my allocated memory; my allocated memory also
+       * includes DrillBuf's directly allocated by me.
+       */
+      long childTotal = 0;
+      for (final BaseAllocator childAllocator : childSet) {
+        childTotal += Math.max(childAllocator.getAllocatedMemory(), childAllocator.reservation);
+      }
+      if (childTotal > getAllocatedMemory()) {
+        historicalLog.logHistory(logger);
+        logger.debug("allocator[" + name + "] child event logs BEGIN");
+        for (final BaseAllocator childAllocator : childSet) {
+          childAllocator.historicalLog.logHistory(logger);
+        }
+        logger.debug("allocator[" + name + "] child event logs END");
+        throw new IllegalStateException(
+            "Child allocators own more memory (" + childTotal + ") than their parent (name = "
+                + name + " ) has allocated (" + getAllocatedMemory() + ')');
+      }
+
+      // Furthermore, the amount I've allocated should be that plus buffers I've allocated.
+      long bufferTotal = 0;
+
+      final Set<BufferLedger> ledgerSet = childLedgers.keySet();
+      for (final BufferLedger ledger : ledgerSet) {
+        if (!ledger.isOwningLedger()) {
+          continue;
+        }
+
+        final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
+        /*
+         * Even when shared, DrillBufs are rewrapped, so we should never see the same instance twice.
+         */
+        final BaseAllocator otherOwner = buffersSeen.get(udle);
+        if (otherOwner != null) {
+          throw new IllegalStateException("This allocator's drillBuf already owned by another allocator");
+        }
+        buffersSeen.put(udle, this);
+
+        bufferTotal += udle.capacity();
+      }
+
+      // Preallocated space has to be accounted for
+      final Set<Reservation> reservationSet = reservations.keySet();
+      long reservedTotal = 0;
+      for (final Reservation reservation : reservationSet) {
+        if (!reservation.isUsed()) {
+          reservedTotal += reservation.getSize();
+        }
+      }
+
+      if (bufferTotal + reservedTotal + childTotal != getAllocatedMemory()) {
+        final StringBuilder sb = new StringBuilder();
+        sb.append("allocator[");
+        sb.append(name);
+        sb.append("]\nallocated: ");
+        sb.append(Long.toString(allocated));
+        sb.append(" allocated - (bufferTotal + reservedTotal + childTotal): ");
+        sb.append(Long.toString(allocated - (bufferTotal + reservedTotal + childTotal)));
+        sb.append('\n');
+
+        if (bufferTotal != 0) {
+          sb.append("buffer total: ");
+          sb.append(Long.toString(bufferTotal));
+          sb.append('\n');
+          dumpBuffers(sb, ledgerSet);
+        }
+
+        if (childTotal != 0) {
+          sb.append("child total: ");
+          sb.append(Long.toString(childTotal));
+          sb.append('\n');
+
+          for (final BaseAllocator childAllocator : childSet) {
+            sb.append("child allocator[");
+            sb.append(childAllocator.name);
+            sb.append("] owned ");
+            sb.append(Long.toString(childAllocator.getAllocatedMemory()));
+            sb.append('\n');
+          }
+        }
+
+        if (reservedTotal != 0) {
+          sb.append(String.format("reserved total : %d bytes.", reservedTotal));
+          for (final Reservation reservation : reservationSet) {
+            reservation.historicalLog.buildHistory(sb, 0, true);
+            sb.append('\n');
+          }
+        }
+
+        logger.debug(sb.toString());
+
+        final long allocated2 = getAllocatedMemory();
+
+        if (allocated2 != allocated) {
+          throw new IllegalStateException(String.format(
+              "allocator[%s]: allocated t1 (%d) + allocated t2 (%d). Someone released memory while in verification.",
+              name, allocated, allocated2));
+
+        }
+        throw new IllegalStateException(String.format(
+            "allocator[%s]: buffer space (%d) + prealloc space (%d) + child space (%d) != allocated (%d)",
+            name, bufferTotal, reservedTotal, childTotal, allocated));
+      }
+    }
+  }
+
+  void print(StringBuilder sb, int level, Verbosity verbosity) {
+
+    indent(sb, level)
+        .append("Allocator(")
+        .append(name)
+        .append(") ")
+        .append(reservation)
+        .append('/')
+        .append(getAllocatedMemory())
+        .append('/')
+        .append(getPeakMemoryAllocation())
+        .append('/')
+        .append(getLimit())
+        .append(" (res/actual/peak/limit)")
+        .append('\n');
+
+    if (DEBUG) {
+      indent(sb, level + 1).append(String.format("child allocators: %d\n", childAllocators.size()));
+      for (BaseAllocator child : childAllocators.keySet()) {
+        child.print(sb, level + 2, verbosity);
+      }
+
+      indent(sb, level + 1).append(String.format("ledgers: %d\n", childLedgers.size()));
+      for (BufferLedger ledger : childLedgers.keySet()) {
+        ledger.print(sb, level + 2, verbosity);
+      }
+
+      final Set<Reservation> reservations = this.reservations.keySet();
+      indent(sb, level + 1).append(String.format("reservations: %d\n", reservations.size()));
+      for (final Reservation reservation : reservations) {
+        if (verbosity.includeHistoricalLog) {
+          reservation.historicalLog.buildHistory(sb, level + 3, true);
+        }
+      }
+
+    }
+
+  }
+
+  private void dumpBuffers(final StringBuilder sb, final Set<BufferLedger> ledgerSet) {
+    for (final BufferLedger ledger : ledgerSet) {
+      if (!ledger.isOwningLedger()) {
+        continue;
+      }
+      final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
+      sb.append("UnsafeDirectLittleEndian[dentityHashCode == ");
+      sb.append(Integer.toString(System.identityHashCode(udle)));
+      sb.append("] size ");
+      sb.append(Integer.toString(udle.capacity()));
+      sb.append('\n');
+    }
+  }
+
+
+  public static StringBuilder indent(StringBuilder sb, int indent) {
+    final char[] indentation = new char[indent * 2];
+    Arrays.fill(indentation, ' ');
+    sb.append(indentation);
+    return sb;
+  }
+
+  public static enum Verbosity {
+    BASIC(false, false), // only include basic information
+    LOG(true, false), // include basic
+    LOG_WITH_STACKTRACE(true, true) //
+    ;
+
+    public final boolean includeHistoricalLog;
+    public final boolean includeStackTraces;
+
+    Verbosity(boolean includeHistoricalLog, boolean includeStackTraces) {
+      this.includeHistoricalLog = includeHistoricalLog;
+      this.includeStackTraces = includeStackTraces;
+    }
+  }
+
+  public static boolean isDebug() {
+    return DEBUG;
+  }
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java b/java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java
new file mode 100644
index 0000000000000..4e88c734ab4be
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+public class BoundsChecking {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BoundsChecking.class);
+
+  public static final boolean BOUNDS_CHECKING_ENABLED;
+
+  static {
+    boolean isAssertEnabled = false;
+    assert isAssertEnabled = true;
+    BOUNDS_CHECKING_ENABLED = isAssertEnabled
+        || !"true".equals(System.getProperty("drill.enable_unsafe_memory_access"));
+  }
+
+  private BoundsChecking() {
+  }
+
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
new file mode 100644
index 0000000000000..16a68128b704f
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Wrapper class to deal with byte buffer allocation. Ensures users only use designated methods.
+ */
+public interface BufferAllocator extends AutoCloseable {
+  /**
+   * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
+   * requested size for rounding purposes. However, the buffer's capacity will be set to the configured size.
+   *
+   * @param size
+   *          The size in bytes.
+   * @return a new DrillBuf, or null if the request can't be satisfied
+   * @throws OutOfMemoryException
+   *           if buffer cannot be allocated
+   */
+  public ArrowBuf buffer(int size);
+
+  /**
+   * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
+   * requested size for rounding purposes. However, the buffer's capacity will be set to the configured size.
+   *
+   * @param size
+   *          The size in bytes.
+   * @param manager
+   *          A buffer manager to manage reallocation.
+   * @return a new DrillBuf, or null if the request can't be satisfied
+   * @throws OutOfMemoryException
+   *           if buffer cannot be allocated
+   */
+  public ArrowBuf buffer(int size, BufferManager manager);
+
+  /**
+   * Returns the allocator this allocator falls back to when it needs more memory.
+   *
+   * @return the underlying allocator used by this allocator
+   */
+  public ByteBufAllocator getAsByteBufAllocator();
+
+  /**
+   * Create a new child allocator.
+   *
+   * @param name
+   *          the name of the allocator.
+   * @param initReservation
+   *          the initial space reservation (obtained from this allocator)
+   * @param maxAllocation
+   *          maximum amount of space the new allocator can allocate
+   * @return the new allocator, or null if it can't be created
+   */
+  public BufferAllocator newChildAllocator(String name, long initReservation, long maxAllocation);
+
+  /**
+   * Close and release all buffers generated from this buffer pool.
+   *
+   * <p>When assertions are on, complains if there are any outstanding buffers; to avoid
+   * that, release all buffers before the allocator is closed.
+   */
+  @Override
+  public void close();
+
+  /**
+   * Returns the amount of memory currently allocated from this allocator.
+   *
+   * @return the amount of memory currently allocated
+   */
+  public long getAllocatedMemory();
+
+  /**
+   * Set the maximum amount of memory this allocator is allowed to allocate.
+   *
+   * @param newLimit
+   *          The new Limit to apply to allocations
+   */
+  public void setLimit(long newLimit);
+
+  /**
+   * Return the current maximum limit this allocator imposes.
+   *
+   * @return Limit in number of bytes.
+   */
+  public long getLimit();
+
+  /**
+   * Returns the peak amount of memory allocated from this allocator.
+   *
+   * @return the peak amount of memory allocated
+   */
+  public long getPeakMemoryAllocation();
+
+  /**
+   * Create an allocation reservation. A reservation is a way of building up
+   * a request for a buffer whose size is not known in advance. See
+   * {@see AllocationReservation}.
+   *
+   * @return the newly created reservation
+   */
+  public AllocationReservation newReservation();
+
+  /**
+   * Get a reference to the empty buffer associated with this allocator. Empty buffers are special because we don't
+   * worry about them leaking or managing reference counts on them since they don't actually point to any memory.
+   */
+  public ArrowBuf getEmpty();
+
+  /**
+   * Return the name of this allocator. This is a human readable name that can help debugging. Typically provides
+   * coordinates about where this allocator was created
+   */
+  public String getName();
+
+  /**
+   * Return whether or not this allocator (or one if its parents) is over its limits. In the case that an allocator is
+   * over its limit, all consumers of that allocator should aggressively try to addrss the overlimit situation.
+   */
+  public boolean isOverLimit();
+
+  /**
+   * Return a verbose string describing this allocator. If in DEBUG mode, this will also include relevant stacktraces
+   * and historical logs for underlying objects
+   *
+   * @return A very verbose description of the allocator hierarchy.
+   */
+  public String toVerboseString();
+
+  /**
+   * Asserts (using java assertions) that the provided allocator is currently open. If assertions are disabled, this is
+   * a no-op.
+   */
+  public void assertOpen();
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
new file mode 100644
index 0000000000000..0610ff09276bf
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
@@ -0,0 +1,66 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.memory;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Manages a list of {@link ArrowBuf}s that can be reallocated as needed. Upon
+ * re-allocation the old buffer will be freed. Managing a list of these buffers
+ * prevents some parts of the system from needing to define a correct location
+ * to place the final call to free them.
+ *
+ * The current uses of these types of buffers are within the pluggable components of Drill.
+ * In UDFs, memory management should not be a concern. We provide access to re-allocatable
+ * DrillBufs to give UDF writers general purpose buffers we can account for. To prevent the need
+ * for UDFs to contain boilerplate to close all of the buffers they request, this list
+ * is tracked at a higher level and all of the buffers are freed once we are sure that
+ * the code depending on them is done executing (currently {@link FragmentContext}
+ * and {@link QueryContext}.
+ */
+public interface BufferManager extends AutoCloseable {
+
+  /**
+   * Replace an old buffer with a new version at least of the provided size. Does not copy data.
+   *
+   * @param old
+   *          Old Buffer that the user is no longer going to use.
+   * @param newSize
+   *          Size of new replacement buffer.
+   * @return
+   */
+  public ArrowBuf replace(ArrowBuf old, int newSize);
+
+  /**
+   * Get a managed buffer of indeterminate size.
+   *
+   * @return A buffer.
+   */
+  public ArrowBuf getManagedBuffer();
+
+  /**
+   * Get a managed buffer of at least a certain size.
+   *
+   * @param size
+   *          The desired size
+   * @return A buffer
+   */
+  public ArrowBuf getManagedBuffer(int size);
+
+  public void close();
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
new file mode 100644
index 0000000000000..6f120e5328bd4
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+
+/**
+ * Child allocator class. Only slightly different from the {@see RootAllocator},
+ * in that these can't be created directly, but must be obtained from
+ * {@see BufferAllocator#newChildAllocator(AllocatorOwner, long, long, int)}.
+
+ * <p>Child allocators can only be created by the root, or other children, so
+ * this class is package private.</p>
+ */
+class ChildAllocator extends BaseAllocator {
+  /**
+   * Constructor.
+   *
+   * @param parentAllocator parent allocator -- the one creating this child
+   * @param allocatorOwner a handle to the object making the request
+   * @param allocationPolicy the allocation policy to use; the policy for all
+   *   allocators must match for each invocation of a drillbit
+   * @param initReservation initial amount of space to reserve (obtained from the parent)
+   * @param maxAllocation maximum amount of space that can be obtained from this allocator;
+   *   note this includes direct allocations (via {@see BufferAllocator#buffer(int, int)}
+   *   et al) and requests from descendant allocators. Depending on the allocation policy in
+   *   force, even less memory may be available
+   * @param flags one or more of BaseAllocator.F_* flags
+   */
+  ChildAllocator(
+      BaseAllocator parentAllocator,
+      String name,
+      long initReservation,
+      long maxAllocation) {
+    super(parentAllocator, name, initReservation, maxAllocation);
+  }
+
+
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java
new file mode 100644
index 0000000000000..23d644841e13f
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.CompositeByteBuf;
+import io.netty.buffer.ExpandableByteBuf;
+
+/**
+ * An implementation of ByteBufAllocator that wraps a Drill BufferAllocator. This allows the RPC layer to be accounted
+ * and managed using Drill's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is
+ * the signature and the fact that this Allocator returns ExpandableByteBufs which enable otherwise non-expandable
+ * DrillBufs to be expandable.
+ */
+public class DrillByteBufAllocator implements ByteBufAllocator {
+
+  private static final int DEFAULT_BUFFER_SIZE = 4096;
+  private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16;
+
+  private final BufferAllocator allocator;
+
+  public DrillByteBufAllocator(BufferAllocator allocator) {
+    this.allocator = allocator;
+  }
+
+  @Override
+  public ByteBuf buffer() {
+    return buffer(DEFAULT_BUFFER_SIZE);
+  }
+
+  @Override
+  public ByteBuf buffer(int initialCapacity) {
+    return new ExpandableByteBuf(allocator.buffer(initialCapacity), allocator);
+  }
+
+  @Override
+  public ByteBuf buffer(int initialCapacity, int maxCapacity) {
+    return buffer(initialCapacity);
+  }
+
+  @Override
+  public ByteBuf ioBuffer() {
+    return buffer();
+  }
+
+  @Override
+  public ByteBuf ioBuffer(int initialCapacity) {
+    return buffer(initialCapacity);
+  }
+
+  @Override
+  public ByteBuf ioBuffer(int initialCapacity, int maxCapacity) {
+    return buffer(initialCapacity);
+  }
+
+  @Override
+  public ByteBuf directBuffer() {
+    return buffer();
+  }
+
+  @Override
+  public ByteBuf directBuffer(int initialCapacity) {
+    return allocator.buffer(initialCapacity);
+  }
+
+  @Override
+  public ByteBuf directBuffer(int initialCapacity, int maxCapacity) {
+    return buffer(initialCapacity, maxCapacity);
+  }
+
+  @Override
+  public CompositeByteBuf compositeBuffer() {
+    return compositeBuffer(DEFAULT_MAX_COMPOSITE_COMPONENTS);
+  }
+
+  @Override
+  public CompositeByteBuf compositeBuffer(int maxNumComponents) {
+    return new CompositeByteBuf(this, true, maxNumComponents);
+  }
+
+  @Override
+  public CompositeByteBuf compositeDirectBuffer() {
+    return compositeBuffer();
+  }
+
+  @Override
+  public CompositeByteBuf compositeDirectBuffer(int maxNumComponents) {
+    return compositeBuffer(maxNumComponents);
+  }
+
+  @Override
+  public boolean isDirectBufferPooled() {
+    return false;
+  }
+
+  @Override
+  public ByteBuf heapBuffer() {
+    throw fail();
+  }
+
+  @Override
+  public ByteBuf heapBuffer(int initialCapacity) {
+    throw fail();
+  }
+
+  @Override
+  public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
+    throw fail();
+  }
+
+  @Override
+  public CompositeByteBuf compositeHeapBuffer() {
+    throw fail();
+  }
+
+  @Override
+  public CompositeByteBuf compositeHeapBuffer(int maxNumComponents) {
+    throw fail();
+  }
+
+  private RuntimeException fail() {
+    throw new UnsupportedOperationException("Allocator doesn't support heap-based memory.");
+  }
+
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java b/java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java
new file mode 100644
index 0000000000000..6ba0284d8d449
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+
+public class OutOfMemoryException extends RuntimeException {
+  private static final long serialVersionUID = -6858052345185793382L;
+
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutOfMemoryException.class);
+
+  public OutOfMemoryException() {
+    super();
+  }
+
+  public OutOfMemoryException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+
+  public OutOfMemoryException(String message, Throwable cause) {
+    super(message, cause);
+
+  }
+
+  public OutOfMemoryException(String message) {
+    super(message);
+
+  }
+
+  public OutOfMemoryException(Throwable cause) {
+    super(cause);
+
+  }
+
+
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/README.md b/java/memory/src/main/java/org/apache/arrow/memory/README.md
new file mode 100644
index 0000000000000..09e4257ed0f72
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/README.md
@@ -0,0 +1,121 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+ http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+# Memory: Allocation, Accounting and Management
+ 
+The memory management package contains all the memory allocation related items that Arrow uses to manage memory.
+
+
+## Key Components
+Memory management can be broken into the following main components:
+
+- Memory chunk allocation and fragmentation management
+  - `PooledByteBufAllocatorL` - A LittleEndian clone of Netty's jemalloc implementation
+  - `UnsafeDirectLittleEndian` - A base level memory access interface
+  - `LargeBuffer` - A buffer backing implementation used when working with data larger than one Netty chunk (default to 16mb)
+- Memory limits & Accounting
+  - `Accountant` - A nestable class of lockfree memory accountors.
+- Application-level memory allocation
+  - `BufferAllocator` - The public interface application users should be leveraging
+  - `BaseAllocator` - The base implementation of memory allocation, contains the meat of our the Arrow allocator implementation
+  - `RootAllocator` - The root allocator. Typically only one created for a JVM
+  - `ChildAllocator` - A child allocator that derives from the root allocator
+- Buffer ownership and transfer capabilities
+  - `AllocationManager` - Responsible for managing the relationship between multiple allocators and a single chunk of memory
+  - `BufferLedger` - Responsible for allowing maintaining the relationship between an `AllocationManager`, a `BufferAllocator` and one or more individual `ArrowBuf`s 
+- Memory access
+  - `ArrowBuf` - The facade for interacting directly with a chunk of memory.
+ 
+
+## Memory Management Overview
+Arrow's memory model is based on the following basic concepts:
+
+ - Memory can be allocated up to some limit. That limit could be a real limit (OS/JVM) or a locally imposed limit.
+ - Allocation operates in two phases: accounting then actual allocation. Allocation could fail at either point.
+ - Allocation failure should be recoverable. In all cases, the Allocator infrastructure should expose memory allocation failures (OS or internal limit-based) as `OutOfMemoryException`s.
+ - Any allocator can reserve memory when created. This memory shall be held such that this allocator will always be able to allocate that amount of memory.
+ - A particular application component should work to use a local allocator to understand local memory usage and better debug memory leaks.
+ - The same physical memory can be shared by multiple allocators and the allocator must provide an accounting paradigm for this purpose.
+
+## Allocator Trees
+
+Arrow provides a tree-based model for memory allocation. The RootAllocator is created first, then all allocators are created as children of that allocator. The RootAllocator is responsible for being the master bookeeper for memory allocations. All other allocators are created as children of this tree. Each allocator can first determine whether it has enough local memory to satisfy a particular request. If not, the allocator can ask its parent for an additional memory allocation.
+
+## Reserving Memory
+
+Arrow provides two different ways to reserve memory:
+
+  - BufferAllocator accounting reservations: 
+      When a new allocator (other than the `RootAllocator`) is initialized, it can set aside memory that it will keep locally for its lifetime. This is memory that will never be released back to its parent allocator until the allocator is closed.
+  - `AllocationReservation` via BufferAllocator.newReservation(): Allows a short-term preallocation strategy so that a particular subsystem can ensure future memory is available to support a particular request.
+  
+## Memory Ownership, Reference Counts and Sharing
+Many BufferAllocators can reference the same piece of memory at the same time. The most common situation for this is in the case of a Broadcast Join: in this situation many downstream operators in the same Arrowbit will receive the same physical memory. Each of these operators will be operating within its own Allocator context. We therefore have multiple allocators all pointing at the same physical memory. It is the AllocationManager's responsibility to ensure that in this situation, that all memory is accurately accounted for from the Root's perspective and also to ensure that the memory is correctly released once all BufferAllocators have stopped using that memory.
+
+For simplicity of accounting, we treat that memory as being used by one of the BufferAllocators associated with the memory. When that allocator releases its claim on that memory, the memory ownership is then moved to another BufferLedger belonging to the same AllocationManager. Note that because a ArrowBuf.release() is what actually causes memory ownership transfer to occur, we always precede with ownership transfer (even if that violates an allocator limit). It is the responsibility of the application owning a particular allocator to frequently confirm whether the allocator is over its memory limit (BufferAllocator.isOverLimit()) and if so, attempt to aggresively release memory to ameliorate the situation.
+
+All ArrowBufs (direct or sliced) related to a single BufferLedger/BufferAllocator combination share the same reference count and either all will be valid or all will be invalid.
+
+## Object Hierarchy
+
+There are two main ways that someone can look at the object hierarchy for Arrow's memory management scheme. The first is a memory based perspective as below:
+
+### Memory Perspective
+<pre>
++ AllocationManager
+|
+|-- UnsignedDirectLittleEndian (One per AllocationManager)
+|
+|-+ BufferLedger 1 ==> Allocator A (owning)
+| ` - ArrowBuf 1
+|-+ BufferLedger 2 ==> Allocator B (non-owning)
+| ` - ArrowBuf 2
+|-+ BufferLedger 3 ==> Allocator C (non-owning)
+  | - ArrowBuf 3
+  | - ArrowBuf 4
+  ` - ArrowBuf 5
+</pre>
+
+In this picture, a piece of memory is owned by an allocator manager. An allocator manager is responsible for that piece of memory no matter which allocator(s) it is working with. An allocator manager will have relationships with a piece of raw memory (via its reference to UnsignedDirectLittleEndian) as well as references to each BufferAllocator it has a relationship to. 
+
+### Allocator Perspective
+<pre>
++ RootAllocator
+|-+ ChildAllocator 1
+| | - ChildAllocator 1.1
+| ` ...
+|
+|-+ ChildAllocator 2
+|-+ ChildAllocator 3
+| |
+| |-+ BufferLedger 1 ==> AllocationManager 1 (owning) ==> UDLE
+| | `- ArrowBuf 1
+| `-+ BufferLedger 2 ==> AllocationManager 2 (non-owning)==> UDLE
+| 	`- ArrowBuf 2
+|
+|-+ BufferLedger 3 ==> AllocationManager 1 (non-owning)==> UDLE
+| ` - ArrowBuf 3
+|-+ BufferLedger 4 ==> AllocationManager 2 (owning) ==> UDLE
+  | - ArrowBuf 4
+  | - ArrowBuf 5
+  ` - ArrowBuf 6
+</pre>
+
+In this picture, a RootAllocator owns three ChildAllocators. The first ChildAllocator (ChildAllocator 1) owns a subsequent ChildAllocator. ChildAllocator has two BufferLedgers/AllocationManager references. Coincidentally, each of these AllocationManager's is also associated with the RootAllocator. In this case, one of the these AllocationManagers is owned by ChildAllocator 3 (AllocationManager 1) while the other AllocationManager (AllocationManager 2) is owned/accounted for by the RootAllocator. Note that in this scenario, ArrowBuf 1 is sharing the underlying memory as ArrowBuf 3. However the subset of that memory (e.g. through slicing) might be different. Also note that ArrowBuf 2 and ArrowBuf 4, 5 and 6 are also sharing the same underlying memory. Also note that ArrowBuf 4, 5 and 6 all share the same reference count and fate.
+
+## Debugging Issues
+The Allocator object provides a useful set of tools to better understand the status of the allocator. If in `DEBUG` mode, the allocator and supporting classes will record additional debug tracking information to better track down memory leaks and issues. To enable DEBUG mode, either enable Java assertions with `-ea` or pass the following system property to the VM when starting `-Darrow.memory.debug.allocator=true`. The BufferAllocator also provides a `BufferAllocator.toVerboseString()` which can be used in DEBUG mode to get extensive stacktrace information and events associated with various Allocator behaviors.
\ No newline at end of file
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
new file mode 100644
index 0000000000000..571fc37577209
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * The root allocator for using direct memory inside a Drillbit. Supports creating a
+ * tree of descendant child allocators.
+ */
+public class RootAllocator extends BaseAllocator {
+
+  public RootAllocator(final long limit) {
+    super(null, "ROOT", 0, limit);
+  }
+
+  /**
+   * Verify the accounting state of the allocation system.
+   */
+  @VisibleForTesting
+  public void verify() {
+    verifyAllocator();
+  }
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/package-info.java b/java/memory/src/main/java/org/apache/arrow/memory/package-info.java
new file mode 100644
index 0000000000000..712af3026e29c
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/package-info.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *  Memory Allocation, Account and Management
+ *
+ *  See the README.md file in this directory for detailed information about Drill's memory allocation subsystem.
+ *
+ */
+package org.apache.arrow.memory;
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java b/java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java
new file mode 100644
index 0000000000000..28d078528974e
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory.util;
+
+public class AssertionUtil {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
+
+  public static final boolean ASSERT_ENABLED;
+
+  static{
+    boolean isAssertEnabled = false;
+    assert isAssertEnabled = true;
+    ASSERT_ENABLED = isAssertEnabled;
+  }
+
+  public static boolean isAssertionsEnabled(){
+    return ASSERT_ENABLED;
+  }
+
+  private AssertionUtil() {
+  }
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java b/java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java
new file mode 100644
index 0000000000000..94e5cc5fded4f
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory.util;
+
+import java.util.concurrent.locks.Lock;
+
+/**
+ * Simple wrapper class that allows Locks to be released via an try-with-resources block.
+ */
+public class AutoCloseableLock implements AutoCloseable {
+
+  private final Lock lock;
+
+  public AutoCloseableLock(Lock lock) {
+    this.lock = lock;
+  }
+
+  public AutoCloseableLock open() {
+    lock.lock();
+    return this;
+  }
+
+  @Override
+  public void close() {
+    lock.unlock();
+  }
+
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
new file mode 100644
index 0000000000000..38cb779343ab6
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory.util;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+
+import org.slf4j.Logger;
+
+/**
+ * Utility class that can be used to log activity within a class
+ * for later logging and debugging. Supports recording events and
+ * recording the stack at the time they occur.
+ */
+public class HistoricalLog {
+  private static class Event {
+    private final String note; // the event text
+    private final StackTrace stackTrace; // where the event occurred
+    private final long time;
+
+    public Event(final String note) {
+      this.note = note;
+      this.time = System.nanoTime();
+      stackTrace = new StackTrace();
+    }
+  }
+
+  private final LinkedList<Event> history = new LinkedList<>();
+  private final String idString; // the formatted id string
+  private Event firstEvent; // the first stack trace recorded
+  private final int limit; // the limit on the number of events kept
+
+  /**
+   * Constructor. The format string will be formatted and have its arguments
+   * substituted at the time this is called.
+   *
+   * @param idStringFormat {@link String#format} format string that can be used
+   *     to identify this object in a log. Including some kind of unique identifier
+   *     that can be associated with the object instance is best.
+   * @param args for the format string, or nothing if none are required
+   */
+  public HistoricalLog(final String idStringFormat, Object... args) {
+    this(Integer.MAX_VALUE, idStringFormat, args);
+  }
+
+  /**
+   * Constructor. The format string will be formatted and have its arguments
+   * substituted at the time this is called.
+   *
+   * <p>This form supports the specification of a limit that will limit the
+   * number of historical entries kept (which keeps down the amount of memory
+   * used). With the limit, the first entry made is always kept (under the
+   * assumption that this is the creation site of the object, which is usually
+   * interesting), and then up to the limit number of entries are kept after that.
+   * Each time a new entry is made, the oldest that is not the first is dropped.
+   * </p>
+   *
+   * @param limit the maximum number of historical entries that will be kept,
+   *   not including the first entry made
+   * @param idStringFormat {@link String#format} format string that can be used
+   *     to identify this object in a log. Including some kind of unique identifier
+   *     that can be associated with the object instance is best.
+   * @param args for the format string, or nothing if none are required
+   */
+  public HistoricalLog(final int limit, final String idStringFormat, Object... args) {
+    this.limit = limit;
+    this.idString = String.format(idStringFormat, args);
+  }
+
+  /**
+   * Record an event. Automatically captures the stack trace at the time this is
+   * called. The format string will be formatted and have its arguments substituted
+   * at the time this is called.
+   *
+   * @param noteFormat {@link String#format} format string that describes the event
+   * @param args for the format string, or nothing if none are required
+   */
+  public synchronized void recordEvent(final String noteFormat, Object... args) {
+    final String note = String.format(noteFormat, args);
+    final Event event = new Event(note);
+    if (firstEvent == null) {
+      firstEvent = event;
+    }
+    if (history.size() == limit) {
+      history.removeFirst();
+    }
+    history.add(event);
+  }
+
+  /**
+   * Write the history of this object to the given {@link StringBuilder}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param sb {@link StringBuilder} to write to
+   */
+  public void buildHistory(final StringBuilder sb, boolean includeStackTrace) {
+    buildHistory(sb, 0, includeStackTrace);
+  }
+
+  /**
+   * Write the history of this object to the given {@link StringBuilder}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param sb {@link StringBuilder} to write to
+   * @param additional an extra string that will be written between the identifying
+   *     information and the history; often used for a current piece of state
+   */
+
+  /**
+   *
+   * @param sb
+   * @param indexLevel
+   * @param includeStackTrace
+   */
+  public synchronized void buildHistory(final StringBuilder sb, int indent, boolean includeStackTrace) {
+    final char[] indentation = new char[indent];
+    final char[] innerIndentation = new char[indent + 2];
+    Arrays.fill(indentation, ' ');
+    Arrays.fill(innerIndentation, ' ');
+
+    sb.append(indentation)
+        .append("event log for: ")
+        .append(idString)
+        .append('\n');
+
+
+    if (firstEvent != null) {
+      sb.append(innerIndentation)
+          .append(firstEvent.time)
+          .append(' ')
+          .append(firstEvent.note)
+          .append('\n');
+      if (includeStackTrace) {
+        firstEvent.stackTrace.writeToBuilder(sb, indent + 2);
+      }
+
+      for(final Event event : history) {
+        if (event == firstEvent) {
+          continue;
+        }
+        sb.append(innerIndentation)
+            .append("  ")
+            .append(event.time)
+            .append(' ')
+            .append(event.note)
+            .append('\n');
+
+        if (includeStackTrace) {
+          event.stackTrace.writeToBuilder(sb, indent + 2);
+          sb.append('\n');
+        }
+      }
+    }
+  }
+
+  /**
+   * Write the history of this object to the given {@link Logger}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param logger {@link Logger} to write to
+   */
+  public void logHistory(final Logger logger) {
+    final StringBuilder sb = new StringBuilder();
+    buildHistory(sb, 0, true);
+    logger.debug(sb.toString());
+  }
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/Metrics.java b/java/memory/src/main/java/org/apache/arrow/memory/util/Metrics.java
new file mode 100644
index 0000000000000..5177a2478b53a
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/Metrics.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory.util;
+
+import com.codahale.metrics.MetricRegistry;
+
+public class Metrics {
+
+  private Metrics() {
+
+  }
+
+  private static class RegistryHolder {
+    public static final MetricRegistry REGISTRY;
+
+    static {
+      REGISTRY = new MetricRegistry();
+    }
+
+  }
+
+  public static MetricRegistry getInstance() {
+    return RegistryHolder.REGISTRY;
+  }
+}
\ No newline at end of file
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/Pointer.java b/java/memory/src/main/java/org/apache/arrow/memory/util/Pointer.java
new file mode 100644
index 0000000000000..58ab13b0a16ab
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/Pointer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory.util;
+
+public class Pointer<T> {
+  public T value;
+
+  public Pointer(){}
+
+  public Pointer(T value){
+    this.value = value;
+  }
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java b/java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java
new file mode 100644
index 0000000000000..638c2fb9a959e
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory.util;
+
+import java.util.Arrays;
+
+/**
+ * Convenient way of obtaining and manipulating stack traces for debugging.
+ */
+public class StackTrace {
+  private final StackTraceElement[] stackTraceElements;
+
+  /**
+   * Constructor. Captures the current stack trace.
+   */
+  public StackTrace() {
+    // skip over the first element so that we don't include this constructor call
+    final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
+    stackTraceElements = Arrays.copyOfRange(stack, 1, stack.length - 1);
+  }
+
+  /**
+   * Write the stack trace to a StringBuilder.
+   * @param sb
+   *          where to write it
+   * @param indent
+   *          how many double spaces to indent each line
+   */
+  public void writeToBuilder(final StringBuilder sb, final int indent) {
+    // create the indentation string
+    final char[] indentation = new char[indent * 2];
+    Arrays.fill(indentation, ' ');
+
+    // write the stack trace in standard Java format
+    for(StackTraceElement ste : stackTraceElements) {
+      sb.append(indentation)
+          .append("at ")
+          .append(ste.getClassName())
+          .append('.')
+          .append(ste.getMethodName())
+          .append('(')
+          .append(ste.getFileName())
+          .append(':')
+          .append(Integer.toString(ste.getLineNumber()))
+          .append(")\n");
+    }
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder();
+    writeToBuilder(sb, 0);
+    return sb.toString();
+  }
+}
diff --git a/java/memory/src/main/resources/drill-module.conf b/java/memory/src/main/resources/drill-module.conf
new file mode 100644
index 0000000000000..593ef8e41e76b
--- /dev/null
+++ b/java/memory/src/main/resources/drill-module.conf
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//  This file tells Drill to consider this module when class path scanning.
+//  This file can also include any supplementary configuration information.
+//  This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+drill: {
+  memory: {
+    debug.error_on_leak: true,
+    top.max: 1000000000000
+  }
+
+}
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java b/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
new file mode 100644
index 0000000000000..86bccf5064a60
--- /dev/null
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.arrow.memory.Accountant;
+import org.apache.arrow.memory.Accountant.AllocationOutcome;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAccountant {
+
+  @Test
+  public void basic() {
+    ensureAccurateReservations(null);
+  }
+
+  @Test
+  public void nested() {
+    final Accountant parent = new Accountant(null, 0, Long.MAX_VALUE);
+    ensureAccurateReservations(parent);
+    assertEquals(0, parent.getAllocatedMemory());
+  }
+
+  @Test
+  public void multiThread() throws InterruptedException {
+    final Accountant parent = new Accountant(null, 0, Long.MAX_VALUE);
+
+    final int numberOfThreads = 32;
+    final int loops = 100;
+    Thread[] threads = new Thread[numberOfThreads];
+
+    for (int i = 0; i < numberOfThreads; i++) {
+      Thread t = new Thread() {
+
+        @Override
+        public void run() {
+          try {
+            for (int i = 0; i < loops; i++) {
+              ensureAccurateReservations(parent);
+            }
+          } catch (Exception ex) {
+            ex.printStackTrace();
+            Assert.fail(ex.getMessage());
+          }
+        }
+
+      };
+      threads[i] = t;
+      t.start();
+    }
+
+    for (Thread thread : threads) {
+      thread.join();
+    }
+
+    assertEquals(0, parent.getAllocatedMemory());
+  }
+
+  private void ensureAccurateReservations(Accountant outsideParent) {
+    final Accountant parent = new Accountant(outsideParent, 0, 10);
+    assertEquals(0, parent.getAllocatedMemory());
+
+    final Accountant child = new Accountant(parent, 2, Long.MAX_VALUE);
+    assertEquals(2, parent.getAllocatedMemory());
+
+    {
+      AllocationOutcome first = child.allocateBytes(1);
+      assertEquals(AllocationOutcome.SUCCESS, first);
+    }
+
+    // child will have new allocation
+    assertEquals(1, child.getAllocatedMemory());
+
+    // root has no change since within reservation
+    assertEquals(2, parent.getAllocatedMemory());
+
+    {
+      AllocationOutcome first = child.allocateBytes(1);
+      assertEquals(AllocationOutcome.SUCCESS, first);
+    }
+
+    // child will have new allocation
+    assertEquals(2, child.getAllocatedMemory());
+
+    // root has no change since within reservation
+    assertEquals(2, parent.getAllocatedMemory());
+
+    child.releaseBytes(1);
+
+    // child will have new allocation
+    assertEquals(1, child.getAllocatedMemory());
+
+    // root has no change since within reservation
+    assertEquals(2, parent.getAllocatedMemory());
+
+    {
+      AllocationOutcome first = child.allocateBytes(2);
+      assertEquals(AllocationOutcome.SUCCESS, first);
+    }
+
+    // child will have new allocation
+    assertEquals(3, child.getAllocatedMemory());
+
+    // went beyond reservation, now in parent accountant
+    assertEquals(3, parent.getAllocatedMemory());
+
+    {
+      AllocationOutcome first = child.allocateBytes(7);
+      assertEquals(AllocationOutcome.SUCCESS, first);
+    }
+
+    // child will have new allocation
+    assertEquals(10, child.getAllocatedMemory());
+
+    // went beyond reservation, now in parent accountant
+    assertEquals(10, parent.getAllocatedMemory());
+
+    child.releaseBytes(9);
+
+    assertEquals(1, child.getAllocatedMemory());
+
+    // back to reservation size
+    assertEquals(2, parent.getAllocatedMemory());
+
+    AllocationOutcome first = child.allocateBytes(10);
+    assertEquals(AllocationOutcome.FAILED_PARENT, first);
+
+    // unchanged
+    assertEquals(1, child.getAllocatedMemory());
+    assertEquals(2, parent.getAllocatedMemory());
+
+    boolean withinLimit = child.forceAllocate(10);
+    assertEquals(false, withinLimit);
+
+    // at new limit
+    assertEquals(child.getAllocatedMemory(), 11);
+    assertEquals(parent.getAllocatedMemory(), 11);
+
+
+    child.releaseBytes(11);
+    assertEquals(child.getAllocatedMemory(), 0);
+    assertEquals(parent.getAllocatedMemory(), 2);
+
+    child.close();
+    parent.close();
+  }
+}
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
new file mode 100644
index 0000000000000..e13dabb9533da
--- /dev/null
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -0,0 +1,648 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.ArrowBuf.TransferResult;
+
+import org.apache.arrow.memory.AllocationReservation;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.memory.RootAllocator;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestBaseAllocator {
+  // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBaseAllocator.class);
+
+  private final static int MAX_ALLOCATION = 8 * 1024;
+
+/*
+  // ---------------------------------------- DEBUG -----------------------------------
+
+  @After
+  public void checkBuffers() {
+    final int bufferCount = UnsafeDirectLittleEndian.getBufferCount();
+    if (bufferCount != 0) {
+      UnsafeDirectLittleEndian.logBuffers(logger);
+      UnsafeDirectLittleEndian.releaseBuffers();
+    }
+
+    assertEquals(0, bufferCount);
+  }
+
+//  @AfterClass
+//  public static void dumpBuffers() {
+//    UnsafeDirectLittleEndian.logBuffers(logger);
+//  }
+
+  // ---------------------------------------- DEBUG ------------------------------------
+*/
+
+
+  @Test
+  public void test_privateMax() throws Exception {
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      final ArrowBuf drillBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2);
+      assertNotNull("allocation failed", drillBuf1);
+
+      try(final BufferAllocator childAllocator =
+          rootAllocator.newChildAllocator("noLimits", 0, MAX_ALLOCATION)) {
+        final ArrowBuf drillBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2);
+        assertNotNull("allocation failed", drillBuf2);
+        drillBuf2.release();
+      }
+
+      drillBuf1.release();
+    }
+  }
+
+  @Test(expected=IllegalStateException.class)
+  public void testRootAllocator_closeWithOutstanding() throws Exception {
+    try {
+      try(final RootAllocator rootAllocator =
+          new RootAllocator(MAX_ALLOCATION)) {
+        final ArrowBuf drillBuf = rootAllocator.buffer(512);
+        assertNotNull("allocation failed", drillBuf);
+      }
+    } finally {
+      /*
+       * We expect there to be one unreleased underlying buffer because we're closing
+       * without releasing it.
+       */
+/*
+      // ------------------------------- DEBUG ---------------------------------
+      final int bufferCount = UnsafeDirectLittleEndian.getBufferCount();
+      UnsafeDirectLittleEndian.releaseBuffers();
+      assertEquals(1, bufferCount);
+      // ------------------------------- DEBUG ---------------------------------
+*/
+    }
+  }
+
+  @Test
+  public void testRootAllocator_getEmpty() throws Exception {
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      final ArrowBuf drillBuf = rootAllocator.buffer(0);
+      assertNotNull("allocation failed", drillBuf);
+      assertEquals("capacity was non-zero", 0, drillBuf.capacity());
+      drillBuf.release();
+    }
+  }
+
+  @Ignore // TODO(DRILL-2740)
+  @Test(expected = IllegalStateException.class)
+  public void testAllocator_unreleasedEmpty() throws Exception {
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      @SuppressWarnings("unused")
+      final ArrowBuf drillBuf = rootAllocator.buffer(0);
+    }
+  }
+
+  @Test
+  public void testAllocator_transferOwnership() throws Exception {
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      final BufferAllocator childAllocator1 =
+          rootAllocator.newChildAllocator("changeOwnership1", 0, MAX_ALLOCATION);
+      final BufferAllocator childAllocator2 =
+          rootAllocator.newChildAllocator("changeOwnership2", 0, MAX_ALLOCATION);
+
+      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4);
+      rootAllocator.verify();
+      TransferResult transferOwnership = drillBuf1.transferOwnership(childAllocator2);
+      final boolean allocationFit = transferOwnership.allocationFit;
+      rootAllocator.verify();
+      assertTrue(allocationFit);
+
+      drillBuf1.release();
+      childAllocator1.close();
+      rootAllocator.verify();
+
+      transferOwnership.buffer.release();
+      childAllocator2.close();
+    }
+  }
+
+  @Test
+  public void testAllocator_shareOwnership() throws Exception {
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+      final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("shareOwnership1", 0, MAX_ALLOCATION);
+      final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("shareOwnership2", 0, MAX_ALLOCATION);
+      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4);
+      rootAllocator.verify();
+
+      // share ownership of buffer.
+      final ArrowBuf drillBuf2 = drillBuf1.retain(childAllocator2);
+      rootAllocator.verify();
+      assertNotNull(drillBuf2);
+      assertNotEquals(drillBuf2, drillBuf1);
+
+      // release original buffer (thus transferring ownership to allocator 2. (should leave allocator 1 in empty state)
+      drillBuf1.release();
+      rootAllocator.verify();
+      childAllocator1.close();
+      rootAllocator.verify();
+
+      final BufferAllocator childAllocator3 = rootAllocator.newChildAllocator("shareOwnership3", 0, MAX_ALLOCATION);
+      final ArrowBuf drillBuf3 = drillBuf1.retain(childAllocator3);
+      assertNotNull(drillBuf3);
+      assertNotEquals(drillBuf3, drillBuf1);
+      assertNotEquals(drillBuf3, drillBuf2);
+      rootAllocator.verify();
+
+      drillBuf2.release();
+      rootAllocator.verify();
+      childAllocator2.close();
+      rootAllocator.verify();
+
+      drillBuf3.release();
+      rootAllocator.verify();
+      childAllocator3.close();
+    }
+  }
+
+  @Test
+  public void testRootAllocator_createChildAndUse() throws Exception {
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+      try (final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createChildAndUse", 0,
+          MAX_ALLOCATION)) {
+        final ArrowBuf drillBuf = childAllocator.buffer(512);
+        assertNotNull("allocation failed", drillBuf);
+        drillBuf.release();
+      }
+    }
+  }
+
+  @Test(expected=IllegalStateException.class)
+  public void testRootAllocator_createChildDontClose() throws Exception {
+    try {
+      try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+        final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createChildDontClose", 0,
+            MAX_ALLOCATION);
+        final ArrowBuf drillBuf = childAllocator.buffer(512);
+        assertNotNull("allocation failed", drillBuf);
+      }
+    } finally {
+      /*
+       * We expect one underlying buffer because we closed a child allocator without
+       * releasing the buffer allocated from it.
+       */
+/*
+      // ------------------------------- DEBUG ---------------------------------
+      final int bufferCount = UnsafeDirectLittleEndian.getBufferCount();
+      UnsafeDirectLittleEndian.releaseBuffers();
+      assertEquals(1, bufferCount);
+      // ------------------------------- DEBUG ---------------------------------
+*/
+    }
+  }
+
+  private static void allocateAndFree(final BufferAllocator allocator) {
+    final ArrowBuf drillBuf = allocator.buffer(512);
+    assertNotNull("allocation failed", drillBuf);
+    drillBuf.release();
+
+    final ArrowBuf drillBuf2 = allocator.buffer(MAX_ALLOCATION);
+    assertNotNull("allocation failed", drillBuf2);
+    drillBuf2.release();
+
+    final int nBufs = 8;
+    final ArrowBuf[] drillBufs = new ArrowBuf[nBufs];
+    for(int i = 0; i < drillBufs.length; ++i) {
+      ArrowBuf drillBufi = allocator.buffer(MAX_ALLOCATION / nBufs);
+      assertNotNull("allocation failed", drillBufi);
+      drillBufs[i] = drillBufi;
+    }
+    for(ArrowBuf drillBufi : drillBufs) {
+      drillBufi.release();
+    }
+  }
+
+  @Test
+  public void testAllocator_manyAllocations() throws Exception {
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      try(final BufferAllocator childAllocator =
+          rootAllocator.newChildAllocator("manyAllocations", 0, MAX_ALLOCATION)) {
+        allocateAndFree(childAllocator);
+      }
+    }
+  }
+
+  @Test
+  public void testAllocator_overAllocate() throws Exception {
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      try(final BufferAllocator childAllocator =
+          rootAllocator.newChildAllocator("overAllocate", 0, MAX_ALLOCATION)) {
+        allocateAndFree(childAllocator);
+
+        try {
+          childAllocator.buffer(MAX_ALLOCATION + 1);
+          fail("allocated memory beyond max allowed");
+        } catch (OutOfMemoryException e) {
+          // expected
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testAllocator_overAllocateParent() throws Exception {
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      try(final BufferAllocator childAllocator =
+          rootAllocator.newChildAllocator("overAllocateParent", 0, MAX_ALLOCATION)) {
+        final ArrowBuf drillBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2);
+        assertNotNull("allocation failed", drillBuf1);
+        final ArrowBuf drillBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2);
+        assertNotNull("allocation failed", drillBuf2);
+
+        try {
+          childAllocator.buffer(MAX_ALLOCATION / 4);
+          fail("allocated memory beyond max allowed");
+        } catch (OutOfMemoryException e) {
+          // expected
+        }
+
+        drillBuf1.release();
+        drillBuf2.release();
+      }
+    }
+  }
+
+  private static void testAllocator_sliceUpBufferAndRelease(
+      final RootAllocator rootAllocator, final BufferAllocator bufferAllocator) {
+    final ArrowBuf drillBuf1 = bufferAllocator.buffer(MAX_ALLOCATION / 2);
+    rootAllocator.verify();
+
+    final ArrowBuf drillBuf2 = drillBuf1.slice(16, drillBuf1.capacity() - 32);
+    rootAllocator.verify();
+    final ArrowBuf drillBuf3 = drillBuf2.slice(16, drillBuf2.capacity() - 32);
+    rootAllocator.verify();
+    @SuppressWarnings("unused")
+    final ArrowBuf drillBuf4 = drillBuf3.slice(16, drillBuf3.capacity() - 32);
+    rootAllocator.verify();
+
+    drillBuf3.release(); // since they share refcounts, one is enough to release them all
+    rootAllocator.verify();
+  }
+
+  @Test
+  public void testAllocator_createSlices() throws Exception {
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+      testAllocator_sliceUpBufferAndRelease(rootAllocator, rootAllocator);
+
+      try (final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) {
+        testAllocator_sliceUpBufferAndRelease(rootAllocator, childAllocator);
+      }
+      rootAllocator.verify();
+
+      testAllocator_sliceUpBufferAndRelease(rootAllocator, rootAllocator);
+
+      try (final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) {
+        try (final BufferAllocator childAllocator2 =
+            childAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) {
+          final ArrowBuf drillBuf1 = childAllocator2.buffer(MAX_ALLOCATION / 8);
+          @SuppressWarnings("unused")
+          final ArrowBuf drillBuf2 = drillBuf1.slice(MAX_ALLOCATION / 16, MAX_ALLOCATION / 16);
+          testAllocator_sliceUpBufferAndRelease(rootAllocator, childAllocator);
+          drillBuf1.release();
+          rootAllocator.verify();
+        }
+        rootAllocator.verify();
+
+        testAllocator_sliceUpBufferAndRelease(rootAllocator, childAllocator);
+      }
+      rootAllocator.verify();
+    }
+  }
+
+  @Test
+  public void testAllocator_sliceRanges() throws Exception {
+//    final AllocatorOwner allocatorOwner = new NamedOwner("sliceRanges");
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      // Populate a buffer with byte values corresponding to their indices.
+      final ArrowBuf drillBuf = rootAllocator.buffer(256);
+      assertEquals(256, drillBuf.capacity());
+      assertEquals(0, drillBuf.readerIndex());
+      assertEquals(0, drillBuf.readableBytes());
+      assertEquals(0, drillBuf.writerIndex());
+      assertEquals(256, drillBuf.writableBytes());
+
+      final ArrowBuf slice3 = (ArrowBuf) drillBuf.slice();
+      assertEquals(0, slice3.readerIndex());
+      assertEquals(0, slice3.readableBytes());
+      assertEquals(0, slice3.writerIndex());
+//      assertEquals(256, slice3.capacity());
+//      assertEquals(256, slice3.writableBytes());
+
+      for(int i = 0; i < 256; ++i) {
+        drillBuf.writeByte(i);
+      }
+      assertEquals(0, drillBuf.readerIndex());
+      assertEquals(256, drillBuf.readableBytes());
+      assertEquals(256, drillBuf.writerIndex());
+      assertEquals(0, drillBuf.writableBytes());
+
+      final ArrowBuf slice1 = (ArrowBuf) drillBuf.slice();
+      assertEquals(0, slice1.readerIndex());
+      assertEquals(256, slice1.readableBytes());
+      for(int i = 0; i < 10; ++i) {
+        assertEquals(i, slice1.readByte());
+      }
+      assertEquals(256 - 10, slice1.readableBytes());
+      for(int i = 0; i < 256; ++i) {
+        assertEquals((byte) i, slice1.getByte(i));
+      }
+
+      final ArrowBuf slice2 = (ArrowBuf) drillBuf.slice(25, 25);
+      assertEquals(0, slice2.readerIndex());
+      assertEquals(25, slice2.readableBytes());
+      for(int i = 25; i < 50; ++i) {
+        assertEquals(i, slice2.readByte());
+      }
+
+/*
+      for(int i = 256; i > 0; --i) {
+        slice3.writeByte(i - 1);
+      }
+      for(int i = 0; i < 256; ++i) {
+        assertEquals(255 - i, slice1.getByte(i));
+      }
+*/
+
+      drillBuf.release(); // all the derived buffers share this fate
+    }
+  }
+
+  @Test
+  public void testAllocator_slicesOfSlices() throws Exception {
+//    final AllocatorOwner allocatorOwner = new NamedOwner("slicesOfSlices");
+    try(final RootAllocator rootAllocator =
+        new RootAllocator(MAX_ALLOCATION)) {
+      // Populate a buffer with byte values corresponding to their indices.
+      final ArrowBuf drillBuf = rootAllocator.buffer(256);
+      for(int i = 0; i < 256; ++i) {
+        drillBuf.writeByte(i);
+      }
+
+      // Slice it up.
+      final ArrowBuf slice0 = drillBuf.slice(0, drillBuf.capacity());
+      for(int i = 0; i < 256; ++i) {
+        assertEquals((byte) i, drillBuf.getByte(i));
+      }
+
+      final ArrowBuf slice10 = slice0.slice(10, drillBuf.capacity() - 10);
+      for(int i = 10; i < 256; ++i) {
+        assertEquals((byte) i, slice10.getByte(i - 10));
+      }
+
+      final ArrowBuf slice20 = slice10.slice(10, drillBuf.capacity() - 20);
+      for(int i = 20; i < 256; ++i) {
+        assertEquals((byte) i, slice20.getByte(i - 20));
+      }
+
+      final ArrowBuf slice30 = slice20.slice(10,  drillBuf.capacity() - 30);
+      for(int i = 30; i < 256; ++i) {
+        assertEquals((byte) i, slice30.getByte(i - 30));
+      }
+
+      drillBuf.release();
+    }
+  }
+
+  @Test
+  public void testAllocator_transferSliced() throws Exception {
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+      final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("transferSliced1", 0, MAX_ALLOCATION);
+      final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferSliced2", 0, MAX_ALLOCATION);
+
+      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
+      final ArrowBuf drillBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8);
+
+      final ArrowBuf drillBuf1s = drillBuf1.slice(0, drillBuf1.capacity() / 2);
+      final ArrowBuf drillBuf2s = drillBuf2.slice(0, drillBuf2.capacity() / 2);
+
+      rootAllocator.verify();
+
+      TransferResult result1 = drillBuf2s.transferOwnership(childAllocator1);
+      rootAllocator.verify();
+      TransferResult result2 = drillBuf1s.transferOwnership(childAllocator2);
+      rootAllocator.verify();
+
+      result1.buffer.release();
+      result2.buffer.release();
+
+      drillBuf1s.release(); // releases drillBuf1
+      drillBuf2s.release(); // releases drillBuf2
+
+      childAllocator1.close();
+      childAllocator2.close();
+    }
+  }
+
+  @Test
+  public void testAllocator_shareSliced() throws Exception {
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+      final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("transferSliced", 0, MAX_ALLOCATION);
+      final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferSliced", 0, MAX_ALLOCATION);
+
+      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
+      final ArrowBuf drillBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8);
+
+      final ArrowBuf drillBuf1s = drillBuf1.slice(0, drillBuf1.capacity() / 2);
+      final ArrowBuf drillBuf2s = drillBuf2.slice(0, drillBuf2.capacity() / 2);
+
+      rootAllocator.verify();
+
+      final ArrowBuf drillBuf2s1 = drillBuf2s.retain(childAllocator1);
+      final ArrowBuf drillBuf1s2 = drillBuf1s.retain(childAllocator2);
+      rootAllocator.verify();
+
+      drillBuf1s.release(); // releases drillBuf1
+      drillBuf2s.release(); // releases drillBuf2
+      rootAllocator.verify();
+
+      drillBuf2s1.release(); // releases the shared drillBuf2 slice
+      drillBuf1s2.release(); // releases the shared drillBuf1 slice
+
+      childAllocator1.close();
+      childAllocator2.close();
+    }
+  }
+
+  @Test
+  public void testAllocator_transferShared() throws Exception {
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+      final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("transferShared1", 0, MAX_ALLOCATION);
+      final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferShared2", 0, MAX_ALLOCATION);
+      final BufferAllocator childAllocator3 = rootAllocator.newChildAllocator("transferShared3", 0, MAX_ALLOCATION);
+
+      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
+
+      boolean allocationFit;
+
+      ArrowBuf drillBuf2 = drillBuf1.retain(childAllocator2);
+      rootAllocator.verify();
+      assertNotNull(drillBuf2);
+      assertNotEquals(drillBuf2, drillBuf1);
+
+      TransferResult result = drillBuf1.transferOwnership(childAllocator3);
+      allocationFit = result.allocationFit;
+      final ArrowBuf drillBuf3 = result.buffer;
+      assertTrue(allocationFit);
+      rootAllocator.verify();
+
+      // Since childAllocator3 now has childAllocator1's buffer, 1, can close
+      drillBuf1.release();
+      childAllocator1.close();
+      rootAllocator.verify();
+
+      drillBuf2.release();
+      childAllocator2.close();
+      rootAllocator.verify();
+
+      final BufferAllocator childAllocator4 = rootAllocator.newChildAllocator("transferShared4", 0, MAX_ALLOCATION);
+      TransferResult result2 = drillBuf3.transferOwnership(childAllocator4);
+      allocationFit = result.allocationFit;
+      final ArrowBuf drillBuf4 = result2.buffer;
+      assertTrue(allocationFit);
+      rootAllocator.verify();
+
+      drillBuf3.release();
+      childAllocator3.close();
+      rootAllocator.verify();
+
+      drillBuf4.release();
+      childAllocator4.close();
+      rootAllocator.verify();
+    }
+  }
+
+  @Test
+  public void testAllocator_unclaimedReservation() throws Exception {
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+      try (final BufferAllocator childAllocator1 =
+          rootAllocator.newChildAllocator("unclaimedReservation", 0, MAX_ALLOCATION)) {
+        try(final AllocationReservation reservation = childAllocator1.newReservation()) {
+          assertTrue(reservation.add(64));
+        }
+        rootAllocator.verify();
+      }
+    }
+  }
+
+  @Test
+  public void testAllocator_claimedReservation() throws Exception {
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+
+      try (final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("claimedReservation", 0,
+          MAX_ALLOCATION)) {
+
+        try (final AllocationReservation reservation = childAllocator1.newReservation()) {
+          assertTrue(reservation.add(32));
+          assertTrue(reservation.add(32));
+
+          final ArrowBuf drillBuf = reservation.allocateBuffer();
+          assertEquals(64, drillBuf.capacity());
+          rootAllocator.verify();
+
+          drillBuf.release();
+          rootAllocator.verify();
+        }
+        rootAllocator.verify();
+      }
+    }
+  }
+
+  @Test
+  public void multiple() throws Exception {
+    final String owner = "test";
+    try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
+
+      final int op = 100000;
+
+      BufferAllocator frag1 = allocator.newChildAllocator(owner, 1500000, Long.MAX_VALUE);
+      BufferAllocator frag2 = allocator.newChildAllocator(owner, 500000, Long.MAX_VALUE);
+
+      allocator.verify();
+
+      BufferAllocator allocator11 = frag1.newChildAllocator(owner, op, Long.MAX_VALUE);
+      ArrowBuf b11 = allocator11.buffer(1000000);
+
+      allocator.verify();
+
+      BufferAllocator allocator12 = frag1.newChildAllocator(owner, op, Long.MAX_VALUE);
+      ArrowBuf b12 = allocator12.buffer(500000);
+
+      allocator.verify();
+
+      BufferAllocator allocator21 = frag1.newChildAllocator(owner, op, Long.MAX_VALUE);
+
+      allocator.verify();
+
+      BufferAllocator allocator22 = frag2.newChildAllocator(owner, op, Long.MAX_VALUE);
+      ArrowBuf b22 = allocator22.buffer(2000000);
+
+      allocator.verify();
+
+      BufferAllocator frag3 = allocator.newChildAllocator(owner, 1000000, Long.MAX_VALUE);
+
+      allocator.verify();
+
+      BufferAllocator allocator31 = frag3.newChildAllocator(owner, op, Long.MAX_VALUE);
+      ArrowBuf b31a = allocator31.buffer(200000);
+
+      allocator.verify();
+
+      // Previously running operator completes
+      b22.release();
+
+      allocator.verify();
+
+      allocator22.close();
+
+      b31a.release();
+      allocator31.close();
+
+      b12.release();
+      allocator12.close();
+
+      allocator21.close();
+
+      b11.release();
+      allocator11.close();
+
+      frag1.close();
+      frag2.close();
+      frag3.close();
+
+    }
+  }
+}
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestEndianess.java b/java/memory/src/test/java/org/apache/arrow/memory/TestEndianess.java
new file mode 100644
index 0000000000000..25357dc7b07ef
--- /dev/null
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestEndianess.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import static org.junit.Assert.assertEquals;
+import io.netty.buffer.ByteBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.junit.Test;
+
+
+public class TestEndianess {
+
+  @Test
+  public void testLittleEndian() {
+    final BufferAllocator a = new RootAllocator(10000);
+    final ByteBuf b = a.buffer(4);
+    b.setInt(0, 35);
+    assertEquals(b.getByte(0), 35);
+    assertEquals(b.getByte(1), 0);
+    assertEquals(b.getByte(2), 0);
+    assertEquals(b.getByte(3), 0);
+    b.release();
+    a.close();
+  }
+
+}
diff --git a/java/pom.xml b/java/pom.xml
new file mode 100644
index 0000000000000..8a3b192e13e40
--- /dev/null
+++ b/java/pom.xml
@@ -0,0 +1,470 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache</groupId>
+    <artifactId>apache</artifactId>
+    <version>14</version>
+  </parent>
+
+  <groupId>org.apache.arrow</groupId>
+  <artifactId>arrow-java-root</artifactId>
+  <version>0.1-SNAPSHOT</version>
+  <packaging>pom</packaging>
+
+  <name>Apache Arrow Java Root POM</name>
+  <description>Apache arrow is an open source, low latency SQL query engine for Hadoop and NoSQL.</description>
+  <url>http://arrow.apache.org/</url>
+
+  <properties>
+    <target.gen.source.path>${project.basedir}/target/generated-sources</target.gen.source.path>
+    <dep.junit.version>4.11</dep.junit.version>
+    <dep.slf4j.version>1.7.6</dep.slf4j.version>
+    <dep.guava.version>18.0</dep.guava.version>
+    <forkCount>2</forkCount>
+    <jackson.version>2.7.1</jackson.version>
+    <hadoop.version>2.7.1</hadoop.version>
+    <fmpp.version>0.9.15</fmpp.version>
+    <freemarker.version>2.3.21</freemarker.version>
+  </properties>
+
+  <scm>
+    <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
+    <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
+    <url>https://github.com/apache/arrow</url>
+    <tag>HEAD</tag>
+  </scm>
+
+  <mailingLists>
+    <mailingList>
+      <name>Developer List</name>
+      <subscribe>dev-subscribe@arrow.apache.org</subscribe>
+      <unsubscribe>dev-unsubscribe@arrow.apache.org</unsubscribe>
+      <post>dev@arrow.apache.org</post>
+      <archive>http://mail-archives.apache.org/mod_mbox/arrow-dev/</archive>
+    </mailingList>
+    <mailingList>
+      <name>Commits List</name>
+      <subscribe>commits-subscribe@arrow.apache.org</subscribe>
+      <unsubscribe>commits-unsubscribe@arrow.apache.org</unsubscribe>
+      <post>commits@arrow.apache.org</post>
+      <archive>http://mail-archives.apache.org/mod_mbox/arrow-commits/</archive>
+    </mailingList>
+    <mailingList>
+      <name>Issues List</name>
+      <subscribe>issues-subscribe@arrow.apache.org</subscribe>
+      <unsubscribe>issues-unsubscribe@arrow.apache.org</unsubscribe>
+      <archive>http://mail-archives.apache.org/mod_mbox/arrow-issues/</archive>
+    </mailingList>
+  </mailingLists>
+
+  <repositories>
+
+  </repositories>
+
+  <issueManagement>
+    <system>Jira</system>
+    <url>https://issues.apache.org/jira/browse/arrow</url>
+  </issueManagement>
+
+  <build>
+
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>rat-checks</id>
+            <phase>validate</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <excludeSubprojects>false</excludeSubprojects>
+          <excludes>
+            <exclude>**/*.log</exclude>
+            <exclude>**/*.css</exclude>
+            <exclude>**/*.js</exclude>
+            <exclude>**/*.md</exclude>
+            <exclude>**/*.eps</exclude>
+            <exclude>**/*.json</exclude>
+            <exclude>**/*.seq</exclude>
+            <exclude>**/*.parquet</exclude>
+            <exclude>**/*.sql</exclude>
+            <exclude>**/git.properties</exclude>
+            <exclude>**/*.csv</exclude>
+            <exclude>**/*.csvh</exclude>
+            <exclude>**/*.csvh-test</exclude>
+            <exclude>**/*.tsv</exclude>
+            <exclude>**/*.txt</exclude>
+            <exclude>**/*.ssv</exclude>
+            <exclude>**/arrow-*.conf</exclude>
+            <exclude>**/.buildpath</exclude>
+            <exclude>**/*.proto</exclude>
+            <exclude>**/*.fmpp</exclude>
+            <exclude>**/target/**</exclude>
+            <exclude>**/*.iml</exclude>
+            <exclude>**/*.tdd</exclude>
+            <exclude>**/*.project</exclude>
+            <exclude>**/TAGS</exclude>
+            <exclude>**/*.checkstyle</exclude>
+            <exclude>**/.classpath</exclude>
+            <exclude>**/.settings/**</exclude>
+            <exclude>.*/**</exclude>
+            <exclude>**/*.patch</exclude>
+            <exclude>**/*.pb.cc</exclude>
+            <exclude>**/*.pb.h</exclude>
+            <exclude>**/*.linux</exclude>
+            <exclude>**/client/build/**</exclude>
+            <exclude>**/*.tbl</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>**/logging.properties</exclude>
+            <exclude>**/logback-test.xml</exclude>
+            <exclude>**/logback.out.xml</exclude>
+            <exclude>**/logback.xml</exclude>
+          </excludes>
+          <archive>
+            <index>true</index>
+            <manifest>
+              <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+              <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+            </manifest>
+            <manifestEntries>
+              <Extension-Name>org.apache.arrow</Extension-Name>
+              <Built-By>${username}</Built-By>
+              <url>http://arrow.apache.org/</url>
+            </manifestEntries>
+          </archive>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+            <configuration>
+              <skipIfEmpty>true</skipIfEmpty>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-resources-plugin</artifactId>
+        <configuration>
+          <encoding>UTF-8</encoding>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.7</source>
+          <target>1.7</target>
+          <maxmem>2048m</maxmem>
+          <useIncrementalCompilation>false</useIncrementalCompilation>
+          <fork>true</fork>
+        </configuration>
+      </plugin>
+      <plugin>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>validate_java_and_maven_version</id>
+            <phase>verify</phase>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <inherited>false</inherited>
+            <configuration>
+              <rules>
+                <requireMavenVersion>
+                  <version>[3.0.4,4)</version>
+                </requireMavenVersion>
+              </rules>
+            </configuration>
+          </execution>
+          <execution>
+            <id>avoid_bad_dependencies</id>
+            <phase>verify</phase>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <bannedDependencies>
+                  <excludes>
+                    <exclude>commons-logging</exclude>
+                    <exclude>javax.servlet:servlet-api</exclude>
+                    <exclude>org.mortbay.jetty:servlet-api</exclude>
+                    <exclude>org.mortbay.jetty:servlet-api-2.5</exclude>
+                    <exclude>log4j:log4j</exclude>
+                  </excludes>
+                </bannedDependencies>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>pl.project13.maven</groupId>
+        <artifactId>git-commit-id-plugin</artifactId>
+        <version>2.1.9</version>
+        <executions>
+          <execution>
+            <id>for-jars</id>
+            <inherited>true</inherited>
+            <goals>
+              <goal>revision</goal>
+            </goals>
+            <configuration>
+              <generateGitPropertiesFilename>target/classes/git.properties</generateGitPropertiesFilename>
+            </configuration>
+          </execution>
+          <execution>
+            <id>for-source-tarball</id>
+            <goals>
+              <goal>revision</goal>
+            </goals>
+            <inherited>false</inherited>
+            <configuration>
+              <generateGitPropertiesFilename>./git.properties</generateGitPropertiesFilename>
+            </configuration>
+          </execution>
+        </executions>
+
+        <configuration>
+          <dateFormat>dd.MM.yyyy '@' HH:mm:ss z</dateFormat>
+          <verbose>true</verbose>
+          <skipPoms>false</skipPoms>
+          <generateGitPropertiesFile>true</generateGitPropertiesFile>
+          <failOnNoGitDirectory>false</failOnNoGitDirectory>
+          <gitDescribe>
+            <skip>false</skip>
+            <always>false</always>
+            <abbrev>7</abbrev>
+            <dirty>-dirty</dirty>
+            <forceLongFormat>true</forceLongFormat>
+          </gitDescribe>
+        </configuration>
+      </plugin>
+    </plugins>
+    <pluginManagement>
+
+      <plugins>
+        <plugin>
+          <groupId>org.apache.rat</groupId>
+          <artifactId>apache-rat-plugin</artifactId>
+          <version>0.11</version>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-resources-plugin</artifactId>
+          <version>2.6</version>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-enforcer-plugin</artifactId>
+          <version>1.3.1</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>2.17</version>
+          <configuration>
+            <argLine>-ea</argLine>
+            <forkCount>${forkCount}</forkCount>
+            <reuseForks>true</reuseForks>
+            <systemPropertyVariables>
+              <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
+            </systemPropertyVariables>
+          </configuration>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-release-plugin</artifactId>
+          <version>2.5.2</version>
+          <configuration>
+            <useReleaseProfile>false</useReleaseProfile>
+            <pushChanges>false</pushChanges>
+            <goals>deploy</goals>
+            <arguments>-Papache-release ${arguments}</arguments>
+          </configuration>
+        </plugin>
+
+        <!--This plugin's configuration is used to store Eclipse m2e settings
+          only. It has no influence on the Maven build itself. -->
+        <plugin>
+          <groupId>org.eclipse.m2e</groupId>
+          <artifactId>lifecycle-mapping</artifactId>
+          <version>1.0.0</version>
+          <configuration>
+            <lifecycleMappingMetadata>
+              <pluginExecutions>
+                <pluginExecution>
+                  <pluginExecutionFilter>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-antrun-plugin</artifactId>
+                    <versionRange>[1.6,)</versionRange>
+                    <goals>
+                      <goal>run</goal>
+                    </goals>
+                  </pluginExecutionFilter>
+                  <action>
+                    <ignore />
+                  </action>
+                </pluginExecution>
+                <pluginExecution>
+                  <pluginExecutionFilter>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-enforcer-plugin</artifactId>
+                    <versionRange>[1.2,)</versionRange>
+                    <goals>
+                      <goal>enforce</goal>
+                    </goals>
+                  </pluginExecutionFilter>
+                  <action>
+                    <ignore />
+                  </action>
+                </pluginExecution>
+                <pluginExecution>
+                  <pluginExecutionFilter>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>
+                      maven-remote-resources-plugin
+                    </artifactId>
+                    <versionRange>[1.1,)</versionRange>
+                    <goals>
+                      <goal>process</goal>
+                    </goals>
+                  </pluginExecutionFilter>
+                  <action>
+                    <ignore />
+                  </action>
+                </pluginExecution>
+                <pluginExecution>
+                  <pluginExecutionFilter>
+                    <groupId>org.apache.rat</groupId>
+                    <artifactId>apache-rat-plugin</artifactId>
+                    <versionRange>[0.10,)</versionRange>
+                    <goals>
+                      <goal>check</goal>
+                    </goals>
+                  </pluginExecutionFilter>
+                  <action>
+                    <ignore />
+                  </action>
+                </pluginExecution>
+              </pluginExecutions>
+            </lifecycleMappingMetadata>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+  <dependencies>
+
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-handler</artifactId>
+      <version>4.0.27.Final</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${dep.guava.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${dep.slf4j.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jul-to-slf4j</artifactId>
+      <version>${dep.slf4j.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jcl-over-slf4j</artifactId>
+      <version>${dep.slf4j.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>log4j-over-slf4j</artifactId>
+      <version>${dep.slf4j.version}</version>
+    </dependency>
+
+    <!-- Test Dependencies -->
+    <dependency>
+      <!-- JMockit needs to be on class path before JUnit. -->
+      <groupId>com.googlecode.jmockit</groupId>
+      <artifactId>jmockit</artifactId>
+      <version>1.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>${dep.junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <!-- Mockito needs to be on the class path after JUnit (or Hamcrest) as 
+           long as Mockito _contains_ older Hamcrest classes.  See arrow-2130. --> 
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <version>1.9.5</version>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.0.13</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>de.huxhorn.lilith</groupId>
+      <artifactId>de.huxhorn.lilith.logback.appender.multiplex-classic</artifactId>
+      <version>0.9.44</version>
+      <scope>test</scope>
+    </dependency>
+
+  </dependencies>
+
+  <modules>
+    <module>memory</module>
+    <module>vector</module>
+  </modules>
+</project>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
new file mode 100644
index 0000000000000..e693344221b9a
--- /dev/null
+++ b/java/vector/pom.xml
@@ -0,0 +1,165 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+  license agreements. See the NOTICE file distributed with this work for additional 
+  information regarding copyright ownership. The ASF licenses this file to 
+  You under the Apache License, Version 2.0 (the "License"); you may not use 
+  this file except in compliance with the License. You may obtain a copy of 
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+  by applicable law or agreed to in writing, software distributed under the 
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+  OF ANY KIND, either express or implied. See the License for the specific 
+  language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.arrow</groupId>
+    <artifactId>arrow-java-root</artifactId>
+    <version>0.1-SNAPSHOT</version>
+  </parent>
+  <artifactId>vector</artifactId>
+  <name>vectors</name>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>joda-time</groupId>
+      <artifactId>joda-time</artifactId>
+      <version>2.9</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+      <version>2.7.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.7.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.carrotsearch</groupId>
+      <artifactId>hppc</artifactId>
+      <version>0.7.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.4</version>
+    </dependency>
+
+
+  </dependencies>
+
+    <pluginRepositories>
+        <pluginRepository>
+            <id>apache</id>
+            <name>apache</name>
+            <url>https://repo.maven.apache.org/</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </pluginRepository>
+    </pluginRepositories>  
+    
+  <build>
+
+    <resources>
+      <resource>
+        <!-- Copy freemarker template and fmpp configuration files of Vector's 
+          to allow clients to leverage definitions. -->
+        <directory>${basedir}/src/main/codegen</directory>
+        <targetPath>codegen</targetPath>
+      </resource>
+    </resources>
+
+    <plugins>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
+        <executions>
+          <execution> <!-- copy all templates in the same location to compile them at once -->
+            <id>copy-fmpp-resources</id>
+            <phase>initialize</phase>
+            <goals>
+              <goal>copy-resources</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/codegen</outputDirectory>
+              <resources>
+                <resource>
+                  <directory>src/main/codegen</directory>
+                  <filtering>false</filtering>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin> <!-- generate sources from fmpp -->
+        <groupId>org.apache.drill.tools</groupId>
+        <artifactId>drill-fmpp-maven-plugin</artifactId>
+        <version>1.4.0</version>
+        <executions>
+          <execution>
+            <id>generate-fmpp</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>generate</goal>
+            </goals>
+            <configuration>
+              <config>src/main/codegen/config.fmpp</config>
+              <output>${project.build.directory}/generated-sources</output>
+              <templates>${project.build.directory}/codegen/templates</templates>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+    <pluginManagement>
+      <plugins>
+        <!--This plugin's configuration is used to store Eclipse m2e settings 
+          only. It has no influence on the Maven build itself. -->
+        <plugin>
+          <groupId>org.eclipse.m2e</groupId>
+          <artifactId>lifecycle-mapping</artifactId>
+          <version>1.0.0</version>
+          <configuration>
+            <lifecycleMappingMetadata>
+              <pluginExecutions>
+                <pluginExecution>
+                  <pluginExecutionFilter>
+                    <groupId>org.apache.drill.tools</groupId>
+                    <artifactId>drill-fmpp-maven-plugin</artifactId>
+                    <versionRange>[1.0,)</versionRange>
+                    <goals>
+                      <goal>generate</goal>
+                    </goals>
+                  </pluginExecutionFilter>
+                  <action>
+                    <execute>
+                      <runOnIncremental>false</runOnIncremental>
+                      <runOnConfiguration>true</runOnConfiguration>
+                    </execute>
+                  </action>
+                </pluginExecution>
+              </pluginExecutions>
+            </lifecycleMappingMetadata>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+    
+  
+  </build>
+
+
+
+</project>
diff --git a/java/vector/src/main/codegen/config.fmpp b/java/vector/src/main/codegen/config.fmpp
new file mode 100644
index 0000000000000..663677cbb5a76
--- /dev/null
+++ b/java/vector/src/main/codegen/config.fmpp
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data: {
+    # TODO:  Rename to ~valueVectorModesAndTypes for clarity.
+    vv:                       tdd(../data/ValueVectorTypes.tdd),
+
+}
+freemarkerLinks: {
+    includes: includes/
+}
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
new file mode 100644
index 0000000000000..e747c30c5d1cb
--- /dev/null
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -0,0 +1,168 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  modes: [
+    {name: "Optional", prefix: "Nullable"},
+    {name: "Required", prefix: ""},
+    {name: "Repeated", prefix: "Repeated"}
+    ],
+  types: [
+    {
+      major: "Fixed",
+      width: 1,
+      javaType: "byte",
+      boxedType: "Byte",
+      fields: [{name: "value", type: "byte"}],
+      minor: [
+        { class: "TinyInt", valueHolder: "IntHolder" },
+        { class: "UInt1", valueHolder: "UInt1Holder" }
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 2,
+      javaType: "char",
+      boxedType: "Character",
+      fields: [{name: "value", type: "char"}],
+      minor: [
+        { class: "UInt2", valueHolder: "UInt2Holder"}
+      ]
+    },    {
+      major: "Fixed",
+      width: 2,
+      javaType: "short",
+      boxedType: "Short",
+      fields: [{name: "value", type: "short"}],
+      minor: [
+        { class: "SmallInt", valueHolder: "Int2Holder"},
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 4,
+      javaType: "int",
+      boxedType: "Integer",
+      fields: [{name: "value", type: "int"}],
+      minor: [
+        { class: "Int", valueHolder: "IntHolder"},
+        { class: "UInt4", valueHolder: "UInt4Holder" },
+        { class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]},
+        { class: "Time", javaType: "int", friendlyType: "DateTime" },
+        { class: "IntervalYear", javaType: "int", friendlyType: "Period" }
+        { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal", fields: [{name:"value", type:"int"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 8,
+      javaType: "long",
+      boxedType: "Long",
+      fields: [{name: "value", type: "long"}],
+      minor: [
+        { class: "BigInt"},
+        { class: "UInt8" },
+        { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
+        { class: "Date", javaType: "long", friendlyType: "DateTime" },
+        { class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
+        { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal", fields: [{name:"value", type:"long"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
+        <#--
+        { class: "Money", maxPrecisionDigits: 2, scale: 1, },
+        -->
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 12,
+      javaType: "ArrowBuf",
+      boxedType: "ArrowBuf",
+      minor: [
+        { class: "IntervalDay", millisecondsOffset: 4, friendlyType: "Period", fields: [ {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 16,
+      javaType: "ArrowBuf"
+      boxedType: "ArrowBuf",      
+      minor: [
+        { class: "Interval", daysOffset: 4, millisecondsOffset: 8, friendlyType: "Period", fields: [ {name: "months", type: "int"}, {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 12,
+      javaType: "ArrowBuf",
+      boxedType: "ArrowBuf",
+      minor: [
+        <#--
+        { class: "TimeTZ" },
+        { class: "Interval" }
+        -->
+        { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 16,
+      javaType: "ArrowBuf",
+      boxedType: "ArrowBuf",
+      
+      minor: [
+        { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 24,
+      javaType: "ArrowBuf",
+      boxedType: "ArrowBuf",
+      minor: [
+        { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 20,
+      javaType: "ArrowBuf",
+      boxedType: "ArrowBuf",
+      minor: [
+        { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+      ]
+    },
+    {
+      major: "VarLen",
+      width: 4,
+      javaType: "int",
+      boxedType: "ArrowBuf",
+      fields: [{name: "start", type: "int"}, {name: "end", type: "int"}, {name: "buffer", type: "ArrowBuf"}],
+      minor: [
+        { class: "VarBinary" , friendlyType: "byte[]" },
+        { class: "VarChar" , friendlyType: "Text" },
+        { class: "Var16Char" , friendlyType: "String" }
+      ]
+    },
+    {
+      major: "Bit",
+      width: 1,
+      javaType: "int",
+      boxedType: "Integer",
+      minor: [
+        { class: "Bit" , friendlyType: "Boolean", fields: [{name: "value", type: "int"}] }
+      ]
+    }
+  ]
+}
diff --git a/java/vector/src/main/codegen/includes/license.ftl b/java/vector/src/main/codegen/includes/license.ftl
new file mode 100644
index 0000000000000..0455fd87ddcb5
--- /dev/null
+++ b/java/vector/src/main/codegen/includes/license.ftl
@@ -0,0 +1,18 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
\ No newline at end of file
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
new file mode 100644
index 0000000000000..2d808b1b3cb3f
--- /dev/null
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -0,0 +1,62 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+  license agreements. See the NOTICE file distributed with this work for additional 
+  information regarding copyright ownership. The ASF licenses this file to 
+  You under the Apache License, Version 2.0 (the "License"); you may not use 
+  this file except in compliance with the License. You may obtain a copy of 
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+  by applicable law or agreed to in writing, software distributed under the 
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+  OF ANY KIND, either express or implied. See the License for the specific 
+  language governing permissions and limitations under the License. -->
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.ObjectArrays;
+import com.google.common.base.Charsets;
+import com.google.common.collect.ObjectArrays;
+
+import com.google.common.base.Preconditions;
+import io.netty.buffer.*;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+import org.apache.arrow.memory.*;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.*;
+import org.apache.arrow.vector.types.*;
+import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.holders.*;
+import org.apache.arrow.vector.util.*;
+import org.apache.arrow.vector.complex.*;
+import org.apache.arrow.vector.complex.reader.*;
+import org.apache.arrow.vector.complex.impl.*;
+import org.apache.arrow.vector.complex.writer.*;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.List;
+
+import java.io.Closeable;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.joda.time.DateTime;
+import org.joda.time.Period;
+
+
+
+
+
+
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
new file mode 100644
index 0000000000000..b83dba2879190
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/AbstractFieldReader.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+@SuppressWarnings("unused")
+abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
+  
+  AbstractFieldReader(){
+    super();
+  }
+
+  /**
+   * Returns true if the current value of the reader is not null
+   * @return
+   */
+  public boolean isSet() {
+    return true;
+  }
+
+  <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
+          "Character", "DateTime", "Period", "Double", "Float",
+          "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
+  <#assign safeType=friendlyType />
+  <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+  
+  public ${friendlyType} read${safeType}(int arrayIndex){
+    fail("read${safeType}(int arrayIndex)");
+    return null;
+  }
+  
+  public ${friendlyType} read${safeType}(){
+    fail("read${safeType}()");
+    return null;
+  }
+  
+  </#list>
+  
+  public void copyAsValue(MapWriter writer){
+    fail("CopyAsValue MapWriter");
+  }
+  public void copyAsField(String name, MapWriter writer){
+    fail("CopyAsField MapWriter");
+  }
+
+  public void copyAsField(String name, ListWriter writer){
+    fail("CopyAsFieldList");
+  }
+  
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign boxedType = (minor.boxedType!type.boxedType) />
+
+  public void read(${name}Holder holder){
+    fail("${name}");
+  }
+
+  public void read(Nullable${name}Holder holder){
+    fail("${name}");
+  }
+  
+  public void read(int arrayIndex, ${name}Holder holder){
+    fail("Repeated${name}");
+  }
+  
+  public void read(int arrayIndex, Nullable${name}Holder holder){
+    fail("Repeated${name}");
+  }
+  
+  public void copyAsValue(${name}Writer writer){
+    fail("CopyAsValue${name}");
+  }
+  public void copyAsField(String name, ${name}Writer writer){
+    fail("CopyAsField${name}");
+  }
+  </#list></#list>
+  
+  public FieldReader reader(String name){
+    fail("reader(String name)");
+    return null;
+  }
+
+  public FieldReader reader(){
+    fail("reader()");
+    return null;
+    
+  }
+  
+  public int size(){
+    fail("size()");
+    return -1;
+  }
+  
+  private void fail(String name){
+    throw new IllegalArgumentException(String.format("You tried to read a [%s] type when you are using a field reader of type [%s].", name, this.getClass().getSimpleName()));
+  }
+  
+  
+}
+
+
+
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
new file mode 100644
index 0000000000000..6ee9dad44e929
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/AbstractFieldWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter {
+  AbstractFieldWriter(FieldWriter parent) {
+    super(parent);
+  }
+
+  @Override
+  public void start() {
+    throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
+  }
+
+  @Override
+  public void end() {
+    throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
+  }
+
+  @Override
+  public void startList() {
+    throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
+  }
+
+  @Override
+  public void endList() {
+    throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  @Override
+  public void write(${name}Holder holder) {
+    fail("${name}");
+  }
+
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    fail("${name}");
+  }
+
+  </#list></#list>
+
+  public void writeNull() {
+    fail("${name}");
+  }
+
+  /**
+   * This implementation returns {@code false}.
+   * <p>  
+   *   Must be overridden by map writers.
+   * </p>  
+   */
+  @Override
+  public boolean isEmptyMap() {
+    return false;
+  }
+
+  @Override
+  public MapWriter map() {
+    fail("Map");
+    return null;
+  }
+
+  @Override
+  public ListWriter list() {
+    fail("List");
+    return null;
+  }
+
+  @Override
+  public MapWriter map(String name) {
+    fail("Map");
+    return null;
+  }
+
+  @Override
+  public ListWriter list(String name) {
+    fail("List");
+    return null;
+  }
+
+  <#list vv.types as type><#list type.minor as minor>
+  <#assign lowerName = minor.class?uncap_first />
+  <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+  <#assign upperName = minor.class?upper_case />
+  <#assign capName = minor.class?cap_first />
+  <#if minor.class?starts_with("Decimal") >
+  public ${capName}Writer ${lowerName}(String name, int scale, int precision) {
+    fail("${capName}");
+    return null;
+  }
+  </#if>
+
+  @Override
+  public ${capName}Writer ${lowerName}(String name) {
+    fail("${capName}");
+    return null;
+  }
+
+  @Override
+  public ${capName}Writer ${lowerName}() {
+    fail("${capName}");
+    return null;
+  }
+
+  </#list></#list>
+
+  public void copyReader(FieldReader reader) {
+    fail("Copy FieldReader");
+  }
+
+  public void copyReaderToField(String name, FieldReader reader) {
+    fail("Copy FieldReader to STring");
+  }
+
+  private void fail(String name) {
+    throw new IllegalArgumentException(String.format("You tried to write a %s type when you are using a ValueWriter of type %s.", name, this.getClass().getSimpleName()));
+  }
+}
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
new file mode 100644
index 0000000000000..549dbf107ea67
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/AbstractPromotableFieldWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * A FieldWriter which delegates calls to another FieldWriter. The delegate FieldWriter can be promoted to a new type
+ * when necessary. Classes that extend this class are responsible for handling promotion.
+ *
+ * This class is generated using freemarker and the ${.template_name} template.
+ *
+ */
+@SuppressWarnings("unused")
+abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
+  AbstractPromotableFieldWriter(FieldWriter parent) {
+    super(parent);
+  }
+
+  /**
+   * Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type
+   * @param type
+   * @return
+   */
+  abstract protected FieldWriter getWriter(MinorType type);
+
+  /**
+   * Return the current FieldWriter
+   * @return
+   */
+  abstract protected FieldWriter getWriter();
+
+  @Override
+  public void start() {
+    getWriter(MinorType.MAP).start();
+  }
+
+  @Override
+  public void end() {
+    getWriter(MinorType.MAP).end();
+  }
+
+  @Override
+  public void startList() {
+    getWriter(MinorType.LIST).startList();
+  }
+
+  @Override
+  public void endList() {
+    getWriter(MinorType.LIST).endList();
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#if !minor.class?starts_with("Decimal") >
+  @Override
+  public void write(${name}Holder holder) {
+    getWriter(MinorType.${name?upper_case}).write(holder);
+  }
+
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+  }
+
+  </#if>
+  </#list></#list>
+
+  public void writeNull() {
+  }
+
+  @Override
+  public MapWriter map() {
+    return getWriter(MinorType.LIST).map();
+  }
+
+  @Override
+  public ListWriter list() {
+    return getWriter(MinorType.LIST).list();
+  }
+
+  @Override
+  public MapWriter map(String name) {
+    return getWriter(MinorType.MAP).map(name);
+  }
+
+  @Override
+  public ListWriter list(String name) {
+    return getWriter(MinorType.MAP).list(name);
+  }
+
+  <#list vv.types as type><#list type.minor as minor>
+  <#assign lowerName = minor.class?uncap_first />
+  <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+  <#assign upperName = minor.class?upper_case />
+  <#assign capName = minor.class?cap_first />
+  <#if !minor.class?starts_with("Decimal") >
+
+  @Override
+  public ${capName}Writer ${lowerName}(String name) {
+    return getWriter(MinorType.MAP).${lowerName}(name);
+  }
+
+  @Override
+  public ${capName}Writer ${lowerName}() {
+    return getWriter(MinorType.LIST).${lowerName}();
+  }
+
+  </#if>
+  </#list></#list>
+
+  public void copyReader(FieldReader reader) {
+    getWriter().copyReader(reader);
+  }
+
+  public void copyReaderToField(String name, FieldReader reader) {
+    getWriter().copyReaderToField(name, reader);
+  }
+}
diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java
new file mode 100644
index 0000000000000..8f12b1da80424
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/BaseReader.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/reader/BaseReader.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.reader;
+
+<#include "/@includes/vv_imports.ftl" />
+
+
+
+@SuppressWarnings("unused")
+public interface BaseReader extends Positionable{
+  MajorType getType();
+  MaterializedField getField();
+  void reset();
+  void read(UnionHolder holder);
+  void read(int index, UnionHolder holder);
+  void copyAsValue(UnionWriter writer);
+  boolean isSet();
+
+  public interface MapReader extends BaseReader, Iterable<String>{
+    FieldReader reader(String name);
+  }
+  
+  public interface RepeatedMapReader extends MapReader{
+    boolean next();
+    int size();
+    void copyAsValue(MapWriter writer);
+  }
+  
+  public interface ListReader extends BaseReader{
+    FieldReader reader(); 
+  }
+  
+  public interface RepeatedListReader extends ListReader{
+    boolean next();
+    int size();
+    void copyAsValue(ListWriter writer);
+  }
+  
+  public interface ScalarReader extends  
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, </#list></#list> 
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> Repeated${name}Reader, </#list></#list>
+  BaseReader {}
+  
+  interface ComplexReader{
+    MapReader rootAsMap();
+    ListReader rootAsList();
+    boolean rootIsMap();
+    boolean ok();
+  }
+}
+
diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java
new file mode 100644
index 0000000000000..299b2389bb35c
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/BaseWriter.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/writer/BaseWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.writer;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * File generated from ${.template_name} using FreeMarker.
+ */
+@SuppressWarnings("unused")
+  public interface BaseWriter extends AutoCloseable, Positionable {
+  FieldWriter getParent();
+  int getValueCapacity();
+
+  public interface MapWriter extends BaseWriter {
+
+    MaterializedField getField();
+
+    /**
+     * Whether this writer is a map writer and is empty (has no children).
+     * 
+     * <p>
+     *   Intended only for use in determining whether to add dummy vector to
+     *   avoid empty (zero-column) schema, as in JsonReader.
+     * </p>
+     * 
+     */
+    boolean isEmptyMap();
+
+    <#list vv.types as type><#list type.minor as minor>
+    <#assign lowerName = minor.class?uncap_first />
+    <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+    <#assign upperName = minor.class?upper_case />
+    <#assign capName = minor.class?cap_first />
+    <#if minor.class?starts_with("Decimal") >
+    ${capName}Writer ${lowerName}(String name, int scale, int precision);
+    </#if>
+    ${capName}Writer ${lowerName}(String name);
+    </#list></#list>
+
+    void copyReaderToField(String name, FieldReader reader);
+    MapWriter map(String name);
+    ListWriter list(String name);
+    void start();
+    void end();
+  }
+
+  public interface ListWriter extends BaseWriter {
+    void startList();
+    void endList();
+    MapWriter map();
+    ListWriter list();
+    void copyReader(FieldReader reader);
+
+    <#list vv.types as type><#list type.minor as minor>
+    <#assign lowerName = minor.class?uncap_first />
+    <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+    <#assign upperName = minor.class?upper_case />
+    <#assign capName = minor.class?cap_first />
+    ${capName}Writer ${lowerName}();
+    </#list></#list>
+  }
+
+  public interface ScalarWriter extends
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Writer, </#list></#list> BaseWriter {}
+
+  public interface ComplexWriter {
+    void allocate();
+    void clear();
+    void copyReader(FieldReader reader);
+    MapWriter rootAsMap();
+    ListWriter rootAsList();
+
+    void setPosition(int index);
+    void setValueCount(int count);
+    void reset();
+  }
+
+  public interface MapOrListWriter {
+    void start();
+    void end();
+    MapOrListWriter map(String name);
+    MapOrListWriter listoftmap(String name);
+    MapOrListWriter list(String name);
+    boolean isMapWriter();
+    boolean isListWriter();
+    VarCharWriter varChar(String name);
+    IntWriter integer(String name);
+    BigIntWriter bigInt(String name);
+    Float4Writer float4(String name);
+    Float8Writer float8(String name);
+    BitWriter bit(String name);
+    VarBinaryWriter binary(String name);
+  }
+}
diff --git a/java/vector/src/main/codegen/templates/BasicTypeHelper.java b/java/vector/src/main/codegen/templates/BasicTypeHelper.java
new file mode 100644
index 0000000000000..bb6446e8d6b19
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/BasicTypeHelper.java
@@ -0,0 +1,538 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/util/BasicTypeHelper.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.util;
+
+<#include "/@includes/vv_imports.ftl" />
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.complex.RepeatedMapVector;
+import org.apache.arrow.vector.util.CallBack;
+
+public class BasicTypeHelper {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicTypeHelper.class);
+
+  private static final int WIDTH_ESTIMATE = 50;
+
+  // Default length when casting to varchar : 65536 = 2^16
+  // This only defines an absolute maximum for values, setting
+  // a high value like this will not inflate the size for small values
+  public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
+
+  protected static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
+    return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
+  }
+
+  protected static String buildErrorMessage(final String operation, final MajorType type) {
+    return buildErrorMessage(operation, type.getMinorType(), type.getMode());
+  }
+
+  public static int getSize(MajorType major) {
+    switch (major.getMinorType()) {
+<#list vv.types as type>
+  <#list type.minor as minor>
+    case ${minor.class?upper_case}:
+      return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
+                               minor.class?substring(0, 3) == "PRO" ||
+                               minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
+  </#list>
+</#list>
+//      case FIXEDCHAR: return major.getWidth();
+//      case FIXED16CHAR: return major.getWidth();
+//      case FIXEDBINARY: return major.getWidth();
+    }
+    throw new UnsupportedOperationException(buildErrorMessage("get size", major));
+  }
+
+  public static ValueVector getNewVector(String name, BufferAllocator allocator, MajorType type, CallBack callback){
+    MaterializedField field = MaterializedField.create(name, type);
+    return getNewVector(field, allocator, callback);
+  }
+  
+  
+  public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
+    switch (type) {
+    case UNION:
+      return UnionVector.class;
+    case MAP:
+      switch (mode) {
+      case OPTIONAL:
+      case REQUIRED:
+        return MapVector.class;
+      case REPEATED:
+        return RepeatedMapVector.class;
+      }
+      
+    case LIST:
+      switch (mode) {
+      case REPEATED:
+        return RepeatedListVector.class;
+      case REQUIRED:
+      case OPTIONAL:
+        return ListVector.class;
+      }
+    
+<#list vv.types as type>
+  <#list type.minor as minor>
+      case ${minor.class?upper_case}:
+        switch (mode) {
+          case REQUIRED:
+            return ${minor.class}Vector.class;
+          case OPTIONAL:
+            return Nullable${minor.class}Vector.class;
+          case REPEATED:
+            return Repeated${minor.class}Vector.class;
+        }
+  </#list>
+</#list>
+    case GENERIC_OBJECT      :
+      return ObjectVector.class  ;
+    default:
+      break;
+    }
+    throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
+  }
+  public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
+    switch (type) {
+    case MAP:
+      switch (mode) {
+      case REQUIRED:
+        if (!isSingularRepeated)
+          return SingleMapReaderImpl.class;
+        else
+          return SingleLikeRepeatedMapReaderImpl.class;
+      case REPEATED: 
+          return RepeatedMapReaderImpl.class;
+      }
+    case LIST:
+      switch (mode) {
+      case REQUIRED:
+        return SingleListReaderImpl.class;
+      case REPEATED:
+        return RepeatedListReaderImpl.class;
+      }
+      
+<#list vv.types as type>
+  <#list type.minor as minor>
+      case ${minor.class?upper_case}:
+        switch (mode) {
+          case REQUIRED:
+            return ${minor.class}ReaderImpl.class;
+          case OPTIONAL:
+            return Nullable${minor.class}ReaderImpl.class;
+          case REPEATED:
+            return Repeated${minor.class}ReaderImpl.class;
+        }
+  </#list>
+</#list>
+      default:
+        break;
+      }
+      throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
+  }
+  
+  public static Class<?> getWriterInterface( MinorType type, DataMode mode){
+    switch (type) {
+    case UNION: return UnionWriter.class;
+    case MAP: return MapWriter.class;
+    case LIST: return ListWriter.class;
+<#list vv.types as type>
+  <#list type.minor as minor>
+      case ${minor.class?upper_case}: return ${minor.class}Writer.class;
+  </#list>
+</#list>
+      default:
+        break;
+      }
+      throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
+  }
+  
+  public static Class<?> getWriterImpl( MinorType type, DataMode mode){
+    switch (type) {
+    case UNION:
+      return UnionWriter.class;
+    case MAP:
+      switch (mode) {
+      case REQUIRED:
+      case OPTIONAL:
+        return SingleMapWriter.class;
+      case REPEATED:
+        return RepeatedMapWriter.class;
+      }
+    case LIST:
+      switch (mode) {
+      case REQUIRED:
+      case OPTIONAL:
+        return UnionListWriter.class;
+      case REPEATED:
+        return RepeatedListWriter.class;
+      }
+      
+<#list vv.types as type>
+  <#list type.minor as minor>
+      case ${minor.class?upper_case}:
+        switch (mode) {
+          case REQUIRED:
+            return ${minor.class}WriterImpl.class;
+          case OPTIONAL:
+            return Nullable${minor.class}WriterImpl.class;
+          case REPEATED:
+            return Repeated${minor.class}WriterImpl.class;
+        }
+  </#list>
+</#list>
+      default:
+        break;
+      }
+      throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
+  }
+
+  public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
+    switch (type) {      
+<#list vv.types as type>
+  <#list type.minor as minor>
+      case ${minor.class?upper_case}:
+        switch (mode) {
+          case REQUIRED:
+            return ${minor.class}HolderReaderImpl.class;
+          case OPTIONAL:
+            return Nullable${minor.class}HolderReaderImpl.class;
+          case REPEATED:
+            return Repeated${minor.class}HolderReaderImpl.class;
+        }
+  </#list>
+</#list>
+      default:
+        break;
+      }
+      throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
+  }
+  
+  public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
+    return getNewVector(field, allocator, null);
+  }
+  public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+    MajorType type = field.getType();
+
+    switch (type.getMinorType()) {
+    
+    case UNION:
+      return new UnionVector(field, allocator, callBack);
+
+    case MAP:
+      switch (type.getMode()) {
+      case REQUIRED:
+      case OPTIONAL:
+        return new MapVector(field, allocator, callBack);
+      case REPEATED:
+        return new RepeatedMapVector(field, allocator, callBack);
+      }
+    case LIST:
+      switch (type.getMode()) {
+      case REPEATED:
+        return new RepeatedListVector(field, allocator, callBack);
+      case OPTIONAL:
+      case REQUIRED:
+        return new ListVector(field, allocator, callBack);
+      }
+<#list vv.  types as type>
+  <#list type.minor as minor>
+    case ${minor.class?upper_case}:
+      switch (type.getMode()) {
+        case REQUIRED:
+          return new ${minor.class}Vector(field, allocator);
+        case OPTIONAL:
+          return new Nullable${minor.class}Vector(field, allocator);
+        case REPEATED:
+          return new Repeated${minor.class}Vector(field, allocator);
+      }
+  </#list>
+</#list>
+    case GENERIC_OBJECT:
+      return new ObjectVector(field, allocator)        ;
+    default:
+      break;
+    }
+    // All ValueVector types have been handled.
+    throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
+  }
+
+  public static ValueHolder getValue(ValueVector vector, int index) {
+    MajorType type = vector.getField().getType();
+    ValueHolder holder;
+    switch(type.getMinorType()) {
+<#list vv.types as type>
+  <#list type.minor as minor>
+    case ${minor.class?upper_case} :
+      <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
+        minor.class?starts_with("Decimal28") ||  minor.class?starts_with("Decimal38")>
+        switch (type.getMode()) {
+          case REQUIRED:
+            holder = new ${minor.class}Holder();
+            ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
+            return holder;
+          case OPTIONAL:
+            holder = new Nullable${minor.class}Holder();
+            ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
+            if (((Nullable${minor.class}Holder)holder).isSet == 1) {
+              ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
+            }
+            return holder;
+        }
+      <#else>
+      switch (type.getMode()) {
+        case REQUIRED:
+          holder = new ${minor.class}Holder();
+          ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
+          return holder;
+        case OPTIONAL:
+          holder = new Nullable${minor.class}Holder();
+          ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
+          if (((Nullable${minor.class}Holder)holder).isSet == 1) {
+            ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
+          }
+          return holder;
+      }
+      </#if>
+  </#list>
+</#list>
+    case GENERIC_OBJECT:
+      holder = new ObjectHolder();
+      ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index)         ;
+      break;
+    }
+
+    throw new UnsupportedOperationException(buildErrorMessage("get value", type));
+  }
+
+  public static void setValue(ValueVector vector, int index, ValueHolder holder) {
+    MajorType type = vector.getField().getType();
+
+    switch(type.getMinorType()) {
+<#list vv.types as type>
+  <#list type.minor as minor>
+    case ${minor.class?upper_case} :
+      switch (type.getMode()) {
+        case REQUIRED:
+          ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
+          return;
+        case OPTIONAL:
+          if (((Nullable${minor.class}Holder) holder).isSet == 1) {
+            ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
+          }
+          return;
+      }
+  </#list>
+</#list>
+    case GENERIC_OBJECT:
+      ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
+      return;
+    default:
+      throw new UnsupportedOperationException(buildErrorMessage("set value", type));
+    }
+  }
+
+  public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
+    MajorType type = vector.getField().getType();
+
+    switch(type.getMinorType()) {
+      <#list vv.types as type>
+      <#list type.minor as minor>
+      case ${minor.class?upper_case} :
+      switch (type.getMode()) {
+        case REQUIRED:
+          ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
+          return;
+        case OPTIONAL:
+          if (((Nullable${minor.class}Holder) holder).isSet == 1) {
+            ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
+          } else {
+            ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
+          }
+          return;
+      }
+      </#list>
+      </#list>
+      case GENERIC_OBJECT:
+        ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
+      default:
+        throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
+    }
+  }
+
+  public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
+    MajorType type1 = v1.getField().getType();
+    MajorType type2 = v2.getField().getType();
+
+    if (type1.getMinorType() != type2.getMinorType()) {
+      return false;
+    }
+
+    switch(type1.getMinorType()) {
+<#list vv.types as type>
+  <#list type.minor as minor>
+    case ${minor.class?upper_case} :
+      if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) == 
+           ((${minor.class}Vector) v2).getAccessor().get(v2index) ) 
+        return true;
+      break;
+  </#list>
+</#list>
+    default:
+      break;
+    }
+    return false;
+  }
+
+  /**
+   *  Create a ValueHolder of MajorType.
+   * @param type
+   * @return
+   */
+  public static ValueHolder createValueHolder(MajorType type) {
+    switch(type.getMinorType()) {
+      <#list vv.types as type>
+      <#list type.minor as minor>
+      case ${minor.class?upper_case} :
+
+        switch (type.getMode()) {
+          case REQUIRED:
+            return new ${minor.class}Holder();
+          case OPTIONAL:
+            return new Nullable${minor.class}Holder();
+          case REPEATED:
+            return new Repeated${minor.class}Holder();
+        }
+      </#list>
+      </#list>
+      case GENERIC_OBJECT:
+        return new ObjectHolder();
+      default:
+        throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
+    }
+  }
+
+  public static boolean isNull(ValueHolder holder) {
+    MajorType type = getValueHolderType(holder);
+
+    switch(type.getMinorType()) {
+      <#list vv.types as type>
+      <#list type.minor as minor>
+      case ${minor.class?upper_case} :
+
+      switch (type.getMode()) {
+        case REQUIRED:
+          return true;
+        case OPTIONAL:
+          return ((Nullable${minor.class}Holder) holder).isSet == 0;
+        case REPEATED:
+          return true;
+      }
+      </#list>
+      </#list>
+      default:
+        throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
+    }
+  }
+
+  public static ValueHolder deNullify(ValueHolder holder) {
+    MajorType type = getValueHolderType(holder);
+
+    switch(type.getMinorType()) {
+      <#list vv.types as type>
+      <#list type.minor as minor>
+      case ${minor.class?upper_case} :
+
+        switch (type.getMode()) {
+          case REQUIRED:
+            return holder;
+          case OPTIONAL:
+            if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
+              ${minor.class}Holder newHolder = new ${minor.class}Holder();
+
+              <#assign fields = minor.fields!type.fields />
+              <#list fields as field>
+              newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
+              </#list>
+
+              return newHolder;
+            } else {
+              throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
+            }
+          case REPEATED:
+            return holder;
+        }
+      </#list>
+      </#list>
+      default:
+        throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
+    }
+  }
+
+  public static ValueHolder nullify(ValueHolder holder) {
+    MajorType type = getValueHolderType(holder);
+
+    switch(type.getMinorType()) {
+      <#list vv.types as type>
+      <#list type.minor as minor>
+      case ${minor.class?upper_case} :
+        switch (type.getMode()) {
+          case REQUIRED:
+            Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
+            newHolder.isSet = 1;
+            <#assign fields = minor.fields!type.fields />
+            <#list fields as field>
+            newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
+            </#list>
+            return newHolder;
+          case OPTIONAL:
+            return holder;
+          case REPEATED:
+            throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
+        }
+      </#list>
+      </#list>
+      default:
+        throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
+    }
+  }
+
+  public static MajorType getValueHolderType(ValueHolder holder) {
+
+    if (0 == 1) {
+      return null;
+    }
+    <#list vv.types as type>
+    <#list type.minor as minor>
+      else if (holder instanceof ${minor.class}Holder) {
+        return ((${minor.class}Holder) holder).TYPE;
+      } else if (holder instanceof Nullable${minor.class}Holder) {
+      return ((Nullable${minor.class}Holder) holder).TYPE;
+      }
+    </#list>
+    </#list>
+
+    throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
+
+  }
+
+}
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
new file mode 100644
index 0000000000000..3614231c8342e
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/ComplexCopier.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ComplexCopier {
+
+  /**
+   * Do a deep copy of the value in input into output
+   * @param in
+   * @param out
+   */
+  public static void copy(FieldReader input, FieldWriter output) {
+    writeValue(input, output);
+  }
+
+  private static void writeValue(FieldReader reader, FieldWriter writer) {
+    final DataMode m = reader.getType().getMode();
+    final MinorType mt = reader.getType().getMinorType();
+
+    switch(m){
+    case OPTIONAL:
+    case REQUIRED:
+
+
+      switch (mt) {
+
+      case LIST:
+        writer.startList();
+        while (reader.next()) {
+          writeValue(reader.reader(), getListWriterForReader(reader.reader(), writer));
+        }
+        writer.endList();
+        break;
+      case MAP:
+        writer.start();
+        if (reader.isSet()) {
+          for(String name : reader){
+            FieldReader childReader = reader.reader(name);
+            if(childReader.isSet()){
+              writeValue(childReader, getMapWriterForReader(childReader, writer, name));
+            }
+          }
+        }
+        writer.end();
+        break;
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#assign uncappedName = name?uncap_first/>
+  <#if !minor.class?starts_with("Decimal")>
+
+      case ${name?upper_case}:
+        if (reader.isSet()) {
+          Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
+          reader.read(${uncappedName}Holder);
+          if (${uncappedName}Holder.isSet == 1) {
+            writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list>);
+          }
+        }
+        break;
+
+  </#if>
+  </#list></#list>
+      }
+              break;
+    }
+ }
+
+  private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
+    switch (reader.getType().getMinorType()) {
+    <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+    <#assign fields = minor.fields!type.fields />
+    <#assign uncappedName = name?uncap_first/>
+    <#if !minor.class?starts_with("Decimal")>
+    case ${name?upper_case}:
+      return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name);
+    </#if>
+    </#list></#list>
+    case MAP:
+      return (FieldWriter) writer.map(name);
+    case LIST:
+      return (FieldWriter) writer.list(name);
+    default:
+      throw new UnsupportedOperationException(reader.getType().toString());
+    }
+  }
+
+  private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) {
+    switch (reader.getType().getMinorType()) {
+    <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+    <#assign fields = minor.fields!type.fields />
+    <#assign uncappedName = name?uncap_first/>
+    <#if !minor.class?starts_with("Decimal")>
+    case ${name?upper_case}:
+    return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>();
+    </#if>
+    </#list></#list>
+    case MAP:
+      return (FieldWriter) writer.map();
+    case LIST:
+      return (FieldWriter) writer.list();
+    default:
+      throw new UnsupportedOperationException(reader.getType().toString());
+    }
+  }
+}
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
new file mode 100644
index 0000000000000..34c657126015e
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+import java.util.List;
+
+import org.apache.arrow.record.TransferPair;
+import org.apache.arrow.vector.complex.IndexHolder;
+import org.apache.arrow.vector.complex.writer.IntervalWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#list ["", "Repeated"] as mode>
+<#assign lowerName = minor.class?uncap_first />
+<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+<#assign name = mode + minor.class?cap_first />
+<#assign javaType = (minor.javaType!type.javaType) />
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+<#assign safeType=friendlyType />
+<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+
+<#assign hasFriendly = minor.friendlyType!"no" == "no" />
+
+<#list ["", "Nullable"] as nullMode>
+<#if (mode == "Repeated" && nullMode  == "") || mode == "" >
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${nullMode}${name}ReaderImpl.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+@SuppressWarnings("unused")
+public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
+  
+  private final ${nullMode}${name}Vector vector;
+  
+  public ${nullMode}${name}ReaderImpl(${nullMode}${name}Vector vector){
+    super();
+    this.vector = vector;
+  }
+
+  public MajorType getType(){
+    return vector.getField().getType();
+  }
+
+  public MaterializedField getField(){
+    return vector.getField();
+  }
+  
+  public boolean isSet(){
+    <#if nullMode == "Nullable">
+    return !vector.getAccessor().isNull(idx());
+    <#else>
+    return true;
+    </#if>
+  }
+
+
+  
+  
+  <#if mode == "Repeated">
+
+  public void copyAsValue(${minor.class?cap_first}Writer writer){
+    Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer;
+    impl.vector.copyFromSafe(idx(), impl.idx(), vector);
+  }
+  
+  public void copyAsField(String name, MapWriter writer){
+    Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl)  writer.list(name).${lowerName}();
+    impl.vector.copyFromSafe(idx(), impl.idx(), vector);
+  }
+  
+  public int size(){
+    return vector.getAccessor().getInnerValueCountAt(idx());
+  }
+  
+  public void read(int arrayIndex, ${minor.class?cap_first}Holder h){
+    vector.getAccessor().get(idx(), arrayIndex, h);
+  }
+  public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h){
+    vector.getAccessor().get(idx(), arrayIndex, h);
+  }
+  
+  public ${friendlyType} read${safeType}(int arrayIndex){
+    return vector.getAccessor().getSingleObject(idx(), arrayIndex);
+  }
+
+  
+  public List<Object> readObject(){
+    return (List<Object>) (Object) vector.getAccessor().getObject(idx());
+  }
+  
+  <#else>
+  
+  public void copyAsValue(${minor.class?cap_first}Writer writer){
+    ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer;
+    impl.vector.copyFromSafe(idx(), impl.idx(), vector);
+  }
+  
+  public void copyAsField(String name, MapWriter writer){
+    ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
+    impl.vector.copyFromSafe(idx(), impl.idx(), vector);
+  }
+
+  <#if nullMode != "Nullable">
+  public void read(${minor.class?cap_first}Holder h){
+    vector.getAccessor().get(idx(), h);
+  }
+  </#if>
+
+  public void read(Nullable${minor.class?cap_first}Holder h){
+    vector.getAccessor().get(idx(), h);
+  }
+  
+  public ${friendlyType} read${safeType}(){
+    return vector.getAccessor().getObject(idx());
+  }
+  
+  public void copyValue(FieldWriter w){
+    
+  }
+  
+  public Object readObject(){
+    return vector.getAccessor().getObject(idx());
+  }
+
+  
+  </#if>
+}
+</#if>
+</#list>
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/reader/${name}Reader.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.reader;
+
+<#include "/@includes/vv_imports.ftl" />
+@SuppressWarnings("unused")
+public interface ${name}Reader extends BaseReader{
+  
+  <#if mode == "Repeated">
+  public int size();
+  public void read(int arrayIndex, ${minor.class?cap_first}Holder h);
+  public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h);
+  public Object readObject(int arrayIndex);
+  public ${friendlyType} read${safeType}(int arrayIndex);
+  <#else>
+  public void read(${minor.class?cap_first}Holder h);
+  public void read(Nullable${minor.class?cap_first}Holder h);
+  public Object readObject();
+  public ${friendlyType} read${safeType}();
+  </#if>  
+  public boolean isSet();
+  public void copyAsValue(${minor.class}Writer writer);
+  public void copyAsField(String name, ${minor.class}Writer writer);
+  
+}
+
+
+
+</#list>
+</#list>
+</#list>
+
+
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
new file mode 100644
index 0000000000000..8f9a6e7b97117
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#list ["", "Nullable", "Repeated"] as mode>
+<#assign name = mode + minor.class?cap_first />
+<#assign eName = name />
+<#assign javaType = (minor.javaType!type.javaType) />
+<#assign fields = minor.fields!type.fields />
+
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${eName}WriterImpl.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using FreeMarker on the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ${eName}WriterImpl extends AbstractFieldWriter {
+
+  private final ${name}Vector.Mutator mutator;
+  final ${name}Vector vector;
+
+  public ${eName}WriterImpl(${name}Vector vector, AbstractFieldWriter parent) {
+    super(parent);
+    this.mutator = vector.getMutator();
+    this.vector = vector;
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return vector.getField();
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return vector.getValueCapacity();
+  }
+
+  @Override
+  public void allocate() {
+    vector.allocateNew();
+  }
+
+  @Override
+  public void close() {
+    vector.close();
+  }
+
+  @Override
+  public void clear() {
+    vector.clear();
+  }
+
+  @Override
+  protected int idx() {
+    return super.idx();
+  }
+
+  <#if mode == "Repeated">
+
+  public void write(${minor.class?cap_first}Holder h) {
+    mutator.addSafe(idx(), h);
+    vector.getMutator().setValueCount(idx()+1);
+  }
+
+  public void write(Nullable${minor.class?cap_first}Holder h) {
+    mutator.addSafe(idx(), h);
+    vector.getMutator().setValueCount(idx()+1);
+  }
+
+  <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+    vector.getMutator().setValueCount(idx()+1);
+  }
+  </#if>
+
+  public void setPosition(int idx) {
+    super.setPosition(idx);
+    mutator.startNewValue(idx);
+  }
+
+
+  <#else>
+
+  public void write(${minor.class}Holder h) {
+    mutator.setSafe(idx(), h);
+    vector.getMutator().setValueCount(idx()+1);
+  }
+
+  public void write(Nullable${minor.class}Holder h) {
+    mutator.setSafe(idx(), h);
+    vector.getMutator().setValueCount(idx()+1);
+  }
+
+  <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    mutator.setSafe(idx(), <#if mode == "Nullable">1, </#if><#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+    vector.getMutator().setValueCount(idx()+1);
+  }
+
+  <#if mode == "Nullable">
+
+  public void writeNull() {
+    mutator.setNull(idx());
+    vector.getMutator().setValueCount(idx()+1);
+  }
+  </#if>
+  </#if>
+  </#if>
+}
+
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/writer/${eName}Writer.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.writer;
+
+<#include "/@includes/vv_imports.ftl" />
+@SuppressWarnings("unused")
+public interface ${eName}Writer extends BaseWriter {
+  public void write(${minor.class}Holder h);
+
+  <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
+  </#if>
+}
+
+</#list>
+</#list>
+</#list>
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
new file mode 100644
index 0000000000000..18fcac93bb6f0
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -0,0 +1,813 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+
+<#if type.major == "Fixed">
+<@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * ${minor.class} implements a vector of fixed width values.  Elements in the vector are accessed
+ * by position, starting from the logical start of the vector.  Values should be pushed onto the
+ * vector sequentially, but may be randomly accessed.
+ *   The width of each element is ${type.width} byte(s)
+ *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+  private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+
+  private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
+  private int allocationMonitor = 0;
+
+  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  @Override
+  public FieldReader getReader(){
+    return reader;
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+    return valueCount * ${type.width};
+  }
+
+  @Override
+  public int getValueCapacity(){
+    return (int) (data.capacity() *1.0 / ${type.width});
+  }
+
+  @Override
+  public Accessor getAccessor(){
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator(){
+    return mutator;
+  }
+
+  @Override
+  public void setInitialCapacity(final int valueCount) {
+    final long size = 1L * valueCount * ${type.width};
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+    allocationSizeInBytes = (int)size;
+  }
+
+  @Override
+  public void allocateNew() {
+    if(!allocateNewSafe()){
+      throw new OutOfMemoryException("Failure while allocating buffer.");
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    long curAllocationSize = allocationSizeInBytes;
+    if (allocationMonitor > 10) {
+      curAllocationSize = Math.max(8, curAllocationSize / 2);
+      allocationMonitor = 0;
+    } else if (allocationMonitor < -2) {
+      curAllocationSize = allocationSizeInBytes * 2L;
+      allocationMonitor = 0;
+    }
+
+    try{
+      allocateBytes(curAllocationSize);
+    } catch (RuntimeException ex) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Allocate a new buffer that supports setting at least the provided number of values. May actually be sized bigger
+   * depending on underlying buffer rounding size. Must be called prior to using the ValueVector.
+   *
+   * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
+   *
+   * @param valueCount
+   * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
+   */
+  @Override
+  public void allocateNew(final int valueCount) {
+    allocateBytes(valueCount * ${type.width});
+  }
+
+  @Override
+  public void reset() {
+    allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+    allocationMonitor = 0;
+    zeroVector();
+    super.reset();
+    }
+
+  private void allocateBytes(final long size) {
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+
+    final int curSize = (int)size;
+    clear();
+    data = allocator.buffer(curSize);
+    data.readerIndex(0);
+    allocationSizeInBytes = curSize;
+  }
+
+/**
+ * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+ *
+ * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
+ */
+  public void reAlloc() {
+    final long newAllocationSize = allocationSizeInBytes * 2L;
+    if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
+      throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
+    }
+
+    logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize);
+    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    newBuf.setBytes(0, data, 0, data.capacity());
+    final int halfNewCapacity = newBuf.capacity() / 2;
+    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    newBuf.writerIndex(data.writerIndex());
+    data.release(1);
+    data = newBuf;
+    allocationSizeInBytes = (int)newAllocationSize;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void zeroVector() {
+    data.setZero(0, data.capacity());
+  }
+
+//  @Override
+//  public void load(SerializedField metadata, ArrowBuf buffer) {
+//    Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
+//    final int actualLength = metadata.getBufferLength();
+//    final int valueCount = metadata.getValueCount();
+//    final int expectedLength = valueCount * ${type.width};
+//    assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength);
+//
+//    clear();
+//    if (data != null) {
+//      data.release(1);
+//    }
+//    data = buffer.slice(0, actualLength);
+//    data.retain(1);
+//    data.writerIndex(actualLength);
+//    }
+
+  public TransferPair getTransferPair(BufferAllocator allocator){
+    return new TransferImpl(getField(), allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+    return new TransferImpl(getField().withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((${minor.class}Vector) to);
+  }
+
+  public void transferTo(${minor.class}Vector target){
+    target.clear();
+    target.data = data.transferOwnership(target.allocator).buffer;
+    target.data.writerIndex(data.writerIndex());
+    clear();
+  }
+
+  public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
+    final int startPoint = startIndex * ${type.width};
+    final int sliceLength = length * ${type.width};
+    target.clear();
+    target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
+    target.data.writerIndex(sliceLength);
+  }
+
+  private class TransferImpl implements TransferPair{
+    private ${minor.class}Vector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator){
+      to = new ${minor.class}Vector(field, allocator);
+    }
+
+    public TransferImpl(${minor.class}Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public ${minor.class}Vector getTo(){
+      return to;
+    }
+
+    @Override
+    public void transfer(){
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
+    }
+  }
+
+  public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    <#if (type.width > 8)>
+    from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
+    <#else> <#-- type.width <= 8 -->
+    data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width},
+        from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width})
+    );
+    </#if> <#-- type.width -->
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    while(thisIndex >= getValueCapacity()) {
+        reAlloc();
+    }
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+  public void decrementAllocationMonitor() {
+    if (allocationMonitor > 0) {
+      allocationMonitor = 0;
+    }
+    --allocationMonitor;
+  }
+
+  private void incrementAllocationMonitor() {
+    ++allocationMonitor;
+  }
+
+  public final class Accessor extends BaseDataValueVector.BaseAccessor {
+    @Override
+    public int getValueCount() {
+      return data.writerIndex() / ${type.width};
+    }
+
+    @Override
+    public boolean isNull(int index){
+      return false;
+    }
+
+    <#if (type.width > 8)>
+
+    public ${minor.javaType!type.javaType} get(int index) {
+      return data.slice(index * ${type.width}, ${type.width});
+    }
+
+    <#if (minor.class == "Interval")>
+    public void get(int index, ${minor.class}Holder holder){
+
+      final int offsetIndex = index * ${type.width};
+      holder.months = data.getInt(offsetIndex);
+      holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
+      holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+    }
+
+    public void get(int index, Nullable${minor.class}Holder holder){
+      final int offsetIndex = index * ${type.width};
+      holder.isSet = 1;
+      holder.months = data.getInt(offsetIndex);
+      holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
+      holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+    }
+
+    @Override
+    public ${friendlyType} getObject(int index) {
+      final int offsetIndex = index * ${type.width};
+      final int months  = data.getInt(offsetIndex);
+      final int days    = data.getInt(offsetIndex + ${minor.daysOffset});
+      final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+      final Period p = new Period();
+      return p.plusMonths(months).plusDays(days).plusMillis(millis);
+    }
+
+    public StringBuilder getAsStringBuilder(int index) {
+
+      final int offsetIndex = index * ${type.width};
+
+      int months  = data.getInt(offsetIndex);
+      final int days    = data.getInt(offsetIndex + ${minor.daysOffset});
+      int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+
+      final int years  = (months / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+      months = (months % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+
+      final int hours  = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
+      millis     = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
+
+      final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
+      millis      = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
+
+      final long seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
+      millis      = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
+
+      final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
+      final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
+      final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
+
+
+      return(new StringBuilder().
+             append(years).append(yearString).
+             append(months).append(monthString).
+             append(days).append(dayString).
+             append(hours).append(":").
+             append(minutes).append(":").
+             append(seconds).append(".").
+             append(millis));
+    }
+
+    <#elseif (minor.class == "IntervalDay")>
+    public void get(int index, ${minor.class}Holder holder){
+
+      final int offsetIndex = index * ${type.width};
+      holder.days = data.getInt(offsetIndex);
+      holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+    }
+
+    public void get(int index, Nullable${minor.class}Holder holder){
+      final int offsetIndex = index * ${type.width};
+      holder.isSet = 1;
+      holder.days = data.getInt(offsetIndex);
+      holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+    }
+
+    @Override
+    public ${friendlyType} getObject(int index) {
+      final int offsetIndex = index * ${type.width};
+      final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+      final int  days   = data.getInt(offsetIndex);
+      final Period p = new Period();
+      return p.plusDays(days).plusMillis(millis);
+    }
+
+
+    public StringBuilder getAsStringBuilder(int index) {
+      final int offsetIndex = index * ${type.width};
+
+      int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+      final int  days   = data.getInt(offsetIndex);
+
+      final int hours  = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
+      millis     = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
+
+      final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
+      millis      = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
+
+      final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
+      millis      = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
+
+      final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
+
+      return(new StringBuilder().
+              append(days).append(dayString).
+              append(hours).append(":").
+              append(minutes).append(":").
+              append(seconds).append(".").
+              append(millis));
+    }
+
+    <#elseif (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+
+    public void get(int index, ${minor.class}Holder holder) {
+        holder.start = index * ${type.width};
+        holder.buffer = data;
+        holder.scale = getField().getScale();
+        holder.precision = getField().getPrecision();
+    }
+
+    public void get(int index, Nullable${minor.class}Holder holder) {
+        holder.isSet = 1;
+        holder.start = index * ${type.width};
+        holder.buffer = data;
+        holder.scale = getField().getScale();
+        holder.precision = getField().getPrecision();
+    }
+
+      @Override
+      public ${friendlyType} getObject(int index) {
+      <#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")>
+      // Get the BigDecimal object
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
+      <#else>
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
+      </#if>
+    }
+
+    <#else>
+    public void get(int index, ${minor.class}Holder holder){
+      holder.buffer = data;
+      holder.start = index * ${type.width};
+    }
+
+    public void get(int index, Nullable${minor.class}Holder holder){
+      holder.isSet = 1;
+      holder.buffer = data;
+      holder.start = index * ${type.width};
+    }
+
+    @Override
+    public ${friendlyType} getObject(int index) {
+      return data.slice(index * ${type.width}, ${type.width})
+    }
+
+    </#if>
+    <#else> <#-- type.width <= 8 -->
+
+    public ${minor.javaType!type.javaType} get(int index) {
+      return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+    }
+
+    <#if type.width == 4>
+    public long getTwoAsLong(int index) {
+      return data.getLong(index * ${type.width});
+    }
+
+    </#if>
+
+    <#if minor.class == "Date">
+    @Override
+    public ${friendlyType} getObject(int index) {
+        org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+        date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+        return date;
+    }
+
+    <#elseif minor.class == "TimeStamp">
+    @Override
+    public ${friendlyType} getObject(int index) {
+        org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+        date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+        return date;
+    }
+
+    <#elseif minor.class == "IntervalYear">
+    @Override
+    public ${friendlyType} getObject(int index) {
+
+      final int value = get(index);
+
+      final int years  = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+      final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+      final Period p = new Period();
+      return p.plusYears(years).plusMonths(months);
+    }
+
+    public StringBuilder getAsStringBuilder(int index) {
+
+      int months  = data.getInt(index);
+
+      final int years  = (months / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+      months = (months % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+
+      final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
+      final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
+
+      return(new StringBuilder().
+             append(years).append(yearString).
+             append(months).append(monthString));
+    }
+
+    <#elseif minor.class == "Time">
+    @Override
+    public DateTime getObject(int index) {
+
+        org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+        time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+        return time;
+    }
+
+    <#elseif minor.class == "Decimal9" || minor.class == "Decimal18">
+    @Override
+    public ${friendlyType} getObject(int index) {
+
+        final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value());
+        return new BigDecimal(value, getField().getScale());
+    }
+
+    <#else>
+    @Override
+    public ${friendlyType} getObject(int index) {
+      return get(index);
+    }
+    public ${minor.javaType!type.javaType} getPrimitiveObject(int index) {
+      return get(index);
+    }
+    </#if>
+
+    public void get(int index, ${minor.class}Holder holder){
+      <#if minor.class.startsWith("Decimal")>
+      holder.scale = getField().getScale();
+      holder.precision = getField().getPrecision();
+      </#if>
+
+      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+    }
+
+    public void get(int index, Nullable${minor.class}Holder holder){
+      holder.isSet = 1;
+      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+    }
+
+
+   </#if> <#-- type.width -->
+ }
+
+ /**
+  * ${minor.class}.Mutator implements a mutable vector of fixed width values.  Elements in the
+  * vector are accessed by position from the logical start of the vector.  Values should be pushed
+  * onto the vector sequentially, but may be randomly accessed.
+  *   The width of each element is ${type.width} byte(s)
+  *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+  *
+  * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+  */
+  public final class Mutator extends BaseDataValueVector.BaseMutator {
+
+    private Mutator(){};
+   /**
+    * Set the element at the given index to the given value.  Note that widths smaller than
+    * 32 bits are handled by the ArrowBuf interface.
+    *
+    * @param index   position of the bit to set
+    * @param value   value to set
+    */
+  <#if (type.width > 8)>
+   public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+     data.setBytes(index * ${type.width}, value, 0, ${type.width});
+   }
+
+   public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     data.setBytes(index * ${type.width}, value, 0, ${type.width});
+   }
+
+  <#if (minor.class == "Interval")>
+   public void set(int index, int months, int days, int milliseconds){
+     final int offsetIndex = index * ${type.width};
+     data.setInt(offsetIndex, months);
+     data.setInt((offsetIndex + ${minor.daysOffset}), days);
+     data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
+   }
+
+   protected void set(int index, ${minor.class}Holder holder){
+     set(index, holder.months, holder.days, holder.milliseconds);
+   }
+
+   protected void set(int index, Nullable${minor.class}Holder holder){
+     set(index, holder.months, holder.days, holder.milliseconds);
+   }
+
+   public void setSafe(int index, int months, int days, int milliseconds){
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     set(index, months, days, milliseconds);
+   }
+
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
+     setSafe(index, holder.months, holder.days, holder.milliseconds);
+   }
+
+   public void setSafe(int index, ${minor.class}Holder holder){
+     setSafe(index, holder.months, holder.days, holder.milliseconds);
+   }
+
+   <#elseif (minor.class == "IntervalDay")>
+   public void set(int index, int days, int milliseconds){
+     final int offsetIndex = index * ${type.width};
+     data.setInt(offsetIndex, days);
+     data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
+   }
+
+   protected void set(int index, ${minor.class}Holder holder){
+     set(index, holder.days, holder.milliseconds);
+   }
+   protected void set(int index, Nullable${minor.class}Holder holder){
+     set(index, holder.days, holder.milliseconds);
+   }
+
+   public void setSafe(int index, int days, int milliseconds){
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     set(index, days, milliseconds);
+   }
+
+   public void setSafe(int index, ${minor.class}Holder holder){
+     setSafe(index, holder.days, holder.milliseconds);
+   }
+
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
+     setSafe(index, holder.days, holder.milliseconds);
+   }
+
+   <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+
+   public void set(int index, ${minor.class}Holder holder){
+     set(index, holder.start, holder.buffer);
+   }
+
+   void set(int index, Nullable${minor.class}Holder holder){
+     set(index, holder.start, holder.buffer);
+   }
+
+   public void setSafe(int index,  Nullable${minor.class}Holder holder){
+     setSafe(index, holder.start, holder.buffer);
+   }
+   public void setSafe(int index,  ${minor.class}Holder holder){
+     setSafe(index, holder.start, holder.buffer);
+   }
+
+   public void setSafe(int index, int start, ArrowBuf buffer){
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     set(index, start, buffer);
+   }
+
+   public void set(int index, int start, ArrowBuf buffer){
+     data.setBytes(index * ${type.width}, buffer, start, ${type.width});
+   }
+
+   <#else>
+
+   protected void set(int index, ${minor.class}Holder holder){
+     set(index, holder.start, holder.buffer);
+   }
+
+   public void set(int index, Nullable${minor.class}Holder holder){
+     set(index, holder.start, holder.buffer);
+   }
+
+   public void set(int index, int start, ArrowBuf buffer){
+     data.setBytes(index * ${type.width}, buffer, start, ${type.width});
+   }
+
+   public void setSafe(int index, ${minor.class}Holder holder){
+     setSafe(index, holder.start, holder.buffer);
+   }
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
+     setSafe(index, holder.start, holder.buffer);
+   }
+
+   public void setSafe(int index, int start, ArrowBuf buffer){
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     set(index, holder);
+   }
+
+   public void set(int index, Nullable${minor.class}Holder holder){
+     data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
+   }
+   </#if>
+
+   @Override
+   public void generateTestData(int count) {
+     setValueCount(count);
+     boolean even = true;
+     final int valueCount = getAccessor().getValueCount();
+     for(int i = 0; i < valueCount; i++, even = !even) {
+       final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE;
+       for(int w = 0; w < ${type.width}; w++){
+         data.setByte(i + w, b);
+       }
+     }
+   }
+
+   <#else> <#-- type.width <= 8 -->
+   public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
+   }
+
+   public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     set(index, value);
+   }
+
+   protected void set(int index, ${minor.class}Holder holder){
+     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
+   }
+
+   public void setSafe(int index, ${minor.class}Holder holder){
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     set(index, holder);
+   }
+
+   protected void set(int index, Nullable${minor.class}Holder holder){
+     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
+   }
+
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     set(index, holder);
+   }
+
+   @Override
+   public void generateTestData(int size) {
+     setValueCount(size);
+     boolean even = true;
+     final int valueCount = getAccessor().getValueCount();
+     for(int i = 0; i < valueCount; i++, even = !even) {
+       if(even){
+         set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE);
+       }else{
+         set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE);
+       }
+     }
+   }
+
+   public void generateTestDataAlt(int size) {
+     setValueCount(size);
+     boolean even = true;
+     final int valueCount = getAccessor().getValueCount();
+     for(int i = 0; i < valueCount; i++, even = !even) {
+       if(even){
+         set(i, (${(minor.javaType!type.javaType)}) 1);
+       }else{
+         set(i, (${(minor.javaType!type.javaType)}) 0);
+       }
+     }
+   }
+
+  </#if> <#-- type.width -->
+
+   @Override
+   public void setValueCount(int valueCount) {
+     final int currentValueCapacity = getValueCapacity();
+     final int idx = (${type.width} * valueCount);
+     while(valueCount > getValueCapacity()) {
+       reAlloc();
+     }
+     if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
+       incrementAllocationMonitor();
+     } else if (allocationMonitor > 0) {
+       allocationMonitor = 0;
+     }
+     VectorTrimmer.trim(data, idx);
+     data.writerIndex(valueCount * ${type.width});
+   }
+ }
+}
+
+</#if> <#-- type.major -->
+</#list>
+</#list>
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
new file mode 100644
index 0000000000000..3005fca0385aa
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#list ["", "Nullable", "Repeated"] as holderMode>
+<#assign nullMode = holderMode />
+<#if holderMode == "Repeated"><#assign nullMode = "Nullable" /></#if>
+
+<#assign lowerName = minor.class?uncap_first />
+<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+<#assign name = minor.class?cap_first />
+<#assign javaType = (minor.javaType!type.javaType) />
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+<#assign safeType=friendlyType />
+<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+<#assign fields = minor.fields!type.fields />
+
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${holderMode}${name}HolderReaderImpl.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.joda.time.Period;
+
+// Source code generated using FreeMarker template ${.template_name}
+
+@SuppressWarnings("unused")
+public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
+
+  private ${nullMode}${name}Holder holder;
+<#if holderMode == "Repeated" >
+  private int index = -1;
+  private ${holderMode}${name}Holder repeatedHolder;
+</#if>
+
+  public ${holderMode}${name}HolderReaderImpl(${holderMode}${name}Holder holder) {
+<#if holderMode == "Repeated" >
+    this.holder = new ${nullMode}${name}Holder();
+    this.repeatedHolder = holder;
+<#else>
+    this.holder = holder;
+</#if>
+  }
+
+  @Override
+  public int size() {
+<#if holderMode == "Repeated">
+    return repeatedHolder.end - repeatedHolder.start;
+<#else>
+    throw new UnsupportedOperationException("You can't call size on a Holder value reader.");
+</#if>
+  }
+
+  @Override
+  public boolean next() {
+<#if holderMode == "Repeated">
+    if(index + 1 < repeatedHolder.end) {
+      index++;
+      repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
+      return true;
+    } else {
+      return false;
+    }
+<#else>
+    throw new UnsupportedOperationException("You can't call next on a single value reader.");
+</#if>
+
+  }
+
+  @Override
+  public void setPosition(int index) {
+    throw new UnsupportedOperationException("You can't call next on a single value reader.");
+  }
+
+  @Override
+  public MajorType getType() {
+<#if holderMode == "Repeated">
+    return this.repeatedHolder.TYPE;
+<#else>
+    return this.holder.TYPE;
+</#if>
+  }
+
+  @Override
+  public boolean isSet() {
+    <#if holderMode == "Repeated">
+    return this.repeatedHolder.end!=this.repeatedHolder.start;
+    <#elseif nullMode == "Nullable">
+    return this.holder.isSet == 1;
+    <#else>
+    return true;
+    </#if>
+    
+  }
+
+<#if holderMode != "Repeated">
+@Override
+  public void read(${name}Holder h) {
+  <#list fields as field>
+    h.${field.name} = holder.${field.name};
+  </#list>
+  }
+
+  @Override
+  public void read(Nullable${name}Holder h) {
+  <#list fields as field>
+    h.${field.name} = holder.${field.name};
+  </#list>
+    h.isSet = isSet() ? 1 : 0;
+  }
+</#if>
+
+<#if holderMode == "Repeated">
+  @Override
+  public ${friendlyType} read${safeType}(int index){
+    repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
+    ${friendlyType} value = read${safeType}();
+    if (this.index > -1) {
+      repeatedHolder.vector.getAccessor().get(repeatedHolder.start + this.index, holder);
+    }
+    return value;
+  }
+</#if>
+
+  @Override
+  public ${friendlyType} read${safeType}(){
+<#if nullMode == "Nullable">
+    if (!isSet()) {
+      return null;
+    }
+</#if>
+
+<#if type.major == "VarLen">
+
+      int length = holder.end - holder.start;
+      byte[] value = new byte [length];
+      holder.buffer.getBytes(holder.start, value, 0, length);
+
+<#if minor.class == "VarBinary">
+      return value;
+<#elseif minor.class == "Var16Char">
+      return new String(value);
+<#elseif minor.class == "VarChar">
+      Text text = new Text();
+      text.set(value);
+      return text;
+</#if>
+
+<#elseif minor.class == "Interval">
+      Period p = new Period();
+      return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
+
+<#elseif minor.class == "IntervalDay">
+      Period p = new Period();
+      return p.plusDays(holder.days).plusMillis(holder.milliseconds);
+
+<#elseif minor.class == "Decimal9" ||
+         minor.class == "Decimal18" >
+      BigInteger value = BigInteger.valueOf(holder.value);
+      return new BigDecimal(value, holder.scale);
+
+<#elseif minor.class == "Decimal28Dense" ||
+         minor.class == "Decimal38Dense">
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
+                                                                                holder.start,
+                                                                                holder.nDecimalDigits,
+                                                                                holder.scale,
+                                                                                holder.maxPrecision,
+                                                                                holder.WIDTH);
+
+<#elseif minor.class == "Decimal28Sparse" ||
+         minor.class == "Decimal38Sparse">
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
+                                                                                 holder.start,
+                                                                                 holder.nDecimalDigits,
+                                                                                 holder.scale);
+
+<#elseif minor.class == "Bit" >
+      return new Boolean(holder.value != 0);
+<#else>
+      ${friendlyType} value = new ${friendlyType}(this.holder.value);
+      return value;
+</#if>
+
+  }
+
+  @Override
+  public Object readObject() {
+<#if holderMode == "Repeated" >
+    List<Object> valList = Lists.newArrayList();
+    for (int i = repeatedHolder.start; i < repeatedHolder.end; i++) {
+      valList.add(repeatedHolder.vector.getAccessor().getObject(i));
+    }
+    return valList;
+<#else>
+    return readSingleObject();
+</#if>
+  }
+
+  private Object readSingleObject() {
+<#if nullMode == "Nullable">
+    if (!isSet()) {
+      return null;
+    }
+</#if>
+
+<#if type.major == "VarLen">
+      int length = holder.end - holder.start;
+      byte[] value = new byte [length];
+      holder.buffer.getBytes(holder.start, value, 0, length);
+
+<#if minor.class == "VarBinary">
+      return value;
+<#elseif minor.class == "Var16Char">
+      return new String(value);
+<#elseif minor.class == "VarChar">
+      Text text = new Text();
+      text.set(value);
+      return text;
+</#if>
+
+<#elseif minor.class == "Interval">
+      Period p = new Period();
+      return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
+
+<#elseif minor.class == "IntervalDay">
+      Period p = new Period();
+      return p.plusDays(holder.days).plusMillis(holder.milliseconds);
+
+<#elseif minor.class == "Decimal9" ||
+         minor.class == "Decimal18" >
+      BigInteger value = BigInteger.valueOf(holder.value);
+      return new BigDecimal(value, holder.scale);
+
+<#elseif minor.class == "Decimal28Dense" ||
+         minor.class == "Decimal38Dense">
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
+                                                                                holder.start,
+                                                                                holder.nDecimalDigits,
+                                                                                holder.scale,
+                                                                                holder.maxPrecision,
+                                                                                holder.WIDTH);
+
+<#elseif minor.class == "Decimal28Sparse" ||
+         minor.class == "Decimal38Sparse">
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
+                                                                                 holder.start,
+                                                                                 holder.nDecimalDigits,
+                                                                                 holder.scale);
+
+<#elseif minor.class == "Bit" >
+      return new Boolean(holder.value != 0);
+<#else>
+      ${friendlyType} value = new ${friendlyType}(this.holder.value);
+      return value;
+</#if>
+  }
+
+<#if holderMode != "Repeated" && nullMode != "Nullable">
+  public void copyAsValue(${minor.class?cap_first}Writer writer){
+    writer.write(holder);
+  }
+</#if>
+}
+
+</#list>
+</#list>
+</#list>
diff --git a/java/vector/src/main/codegen/templates/ListWriters.java b/java/vector/src/main/codegen/templates/ListWriters.java
new file mode 100644
index 0000000000000..cf9fa30fa4784
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ListWriters.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+
+<#list ["Single", "Repeated"] as mode>
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}ListWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+<#if mode == "Single">
+  <#assign containerClass = "AbstractContainerVector" />
+  <#assign index = "idx()">
+<#else>
+  <#assign containerClass = "RepeatedListVector" />
+  <#assign index = "currentChildIndex">
+</#if>
+
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using FreeMarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ${mode}ListWriter extends AbstractFieldWriter {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${mode}ListWriter.class);
+
+  static enum Mode { INIT, IN_MAP, IN_LIST <#list vv.types as type><#list type.minor as minor>, IN_${minor.class?upper_case}</#list></#list> }
+
+  private final String name;
+  protected final ${containerClass} container;
+  private Mode mode = Mode.INIT;
+  private FieldWriter writer;
+  protected RepeatedValueVector innerVector;
+
+  <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
+  public ${mode}ListWriter(String name, ${containerClass} container, FieldWriter parent){
+    super(parent);
+    this.name = name;
+    this.container = container;
+  }
+
+  public ${mode}ListWriter(${containerClass} container, FieldWriter parent){
+    super(parent);
+    this.name = null;
+    this.container = container;
+  }
+
+  @Override
+  public void allocate() {
+    if(writer != null) {
+      writer.allocate();
+    }
+
+    <#if mode == "Repeated">
+    container.allocateNew();
+    </#if>
+  }
+
+  @Override
+  public void clear() {
+    if (writer != null) {
+      writer.clear();
+    }
+  }
+
+  @Override
+  public void close() {
+    clear();
+    container.close();
+    if (innerVector != null) {
+      innerVector.close();
+    }
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return innerVector == null ? 0 : innerVector.getValueCapacity();
+  }
+
+  public void setValueCount(int count){
+    if(innerVector != null) innerVector.getMutator().setValueCount(count);
+  }
+
+  @Override
+  public MapWriter map() {
+    switch(mode) {
+    case INIT:
+      int vectorCount = container.size();
+      final RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class);
+      innerVector = vector;
+      writer = new RepeatedMapWriter(vector, this);
+      if(vectorCount != container.size()) {
+        writer.allocate();
+      }
+      writer.setPosition(${index});
+      mode = Mode.IN_MAP;
+      return writer;
+    case IN_MAP:
+      return writer;
+    }
+
+    throw new RuntimeException(getUnsupportedErrorMsg("MAP", mode.name()));
+
+  }
+
+  @Override
+  public ListWriter list() {
+    switch(mode) {
+    case INIT:
+      final int vectorCount = container.size();
+      final RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class);
+      innerVector = vector;
+      writer = new RepeatedListWriter(null, vector, this);
+      if(vectorCount != container.size()) {
+        writer.allocate();
+      }
+      writer.setPosition(${index});
+      mode = Mode.IN_LIST;
+      return writer;
+    case IN_LIST:
+      return writer;
+    }
+
+    throw new RuntimeException(getUnsupportedErrorMsg("LIST", mode.name()));
+
+  }
+
+  <#list vv.types as type><#list type.minor as minor>
+  <#assign lowerName = minor.class?uncap_first />
+  <#assign upperName = minor.class?upper_case />
+  <#assign capName = minor.class?cap_first />
+  <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+
+  private static final MajorType ${upperName}_TYPE = Types.repeated(MinorType.${upperName});
+
+  @Override
+  public ${capName}Writer ${lowerName}() {
+    switch(mode) {
+    case INIT:
+      final int vectorCount = container.size();
+      final Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class);
+      innerVector = vector;
+      writer = new Repeated${capName}WriterImpl(vector, this);
+      if(vectorCount != container.size()) {
+        writer.allocate();
+      }
+      writer.setPosition(${index});
+      mode = Mode.IN_${upperName};
+      return writer;
+    case IN_${upperName}:
+      return writer;
+    }
+
+    throw new RuntimeException(getUnsupportedErrorMsg("${upperName}", mode.name()));
+
+  }
+  </#list></#list>
+
+  public MaterializedField getField() {
+    return container.getField();
+  }
+
+  <#if mode == "Repeated">
+
+  public void startList() {
+    final RepeatedListVector list = (RepeatedListVector) container;
+    final RepeatedListVector.RepeatedMutator mutator = list.getMutator();
+
+    // make sure that the current vector can support the end position of this list.
+    if(container.getValueCapacity() <= idx()) {
+      mutator.setValueCount(idx()+1);
+    }
+
+    // update the repeated vector to state that there is current+1 objects.
+    final RepeatedListHolder h = new RepeatedListHolder();
+    list.getAccessor().get(idx(), h);
+    if (h.start >= h.end) {
+      mutator.startNewValue(idx());
+    }
+    currentChildIndex = container.getMutator().add(idx());
+    if(writer != null) {
+      writer.setPosition(currentChildIndex);
+    }
+  }
+
+  public void endList() {
+    // noop, we initialize state at start rather than end.
+  }
+  <#else>
+
+  public void setPosition(int index) {
+    super.setPosition(index);
+    if(writer != null) {
+      writer.setPosition(index);
+    }
+  }
+
+  public void startList() {
+    // noop
+  }
+
+  public void endList() {
+    // noop
+  }
+  </#if>
+
+  private String getUnsupportedErrorMsg(String expected, String found) {
+    final String f = found.substring(3);
+    return String.format("In a list of type %s, encountered a value of type %s. "+
+      "Drill does not support lists of different types.",
+       f, expected
+    );
+  }
+}
+</#list>
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
new file mode 100644
index 0000000000000..7001367bb3774
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<#list ["Single", "Repeated"] as mode>
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" />
+<#if mode == "Single">
+<#assign containerClass = "MapVector" />
+<#assign index = "idx()">
+<#else>
+<#assign containerClass = "RepeatedMapVector" />
+<#assign index = "currentChildIndex">
+</#if>
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+import java.util.Map;
+
+import org.apache.arrow.vector.holders.RepeatedMapHolder;
+import org.apache.arrow.vector.AllocationHelper;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+
+import com.google.common.collect.Maps;
+
+/*
+ * This class is generated using FreeMarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ${mode}MapWriter extends AbstractFieldWriter {
+
+  protected final ${containerClass} container;
+  private final Map<String, FieldWriter> fields = Maps.newHashMap();
+  <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
+
+  private final boolean unionEnabled;
+
+  public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled) {
+    super(parent);
+    this.container = container;
+    this.unionEnabled = unionEnabled;
+  }
+
+  public ${mode}MapWriter(${containerClass} container, FieldWriter parent) {
+    this(container, parent, false);
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return container.getValueCapacity();
+  }
+
+  @Override
+  public boolean isEmptyMap() {
+    return 0 == container.size();
+  }
+
+  @Override
+  public MaterializedField getField() {
+      return container.getField();
+  }
+
+  @Override
+  public MapWriter map(String name) {
+      FieldWriter writer = fields.get(name.toLowerCase());
+    if(writer == null){
+      int vectorCount=container.size();
+        MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class);
+      if(!unionEnabled){
+        writer = new SingleMapWriter(vector, this);
+      } else {
+        writer = new PromotableWriter(vector, container);
+      }
+      if(vectorCount != container.size()) {
+        writer.allocate();
+      }
+      writer.setPosition(${index});
+      fields.put(name.toLowerCase(), writer);
+    }
+    return writer;
+  }
+
+  @Override
+  public void close() throws Exception {
+    clear();
+    container.close();
+  }
+
+  @Override
+  public void allocate() {
+    container.allocateNew();
+    for(final FieldWriter w : fields.values()) {
+      w.allocate();
+    }
+  }
+
+  @Override
+  public void clear() {
+    container.clear();
+    for(final FieldWriter w : fields.values()) {
+      w.clear();
+    }
+  }
+
+  @Override
+  public ListWriter list(String name) {
+    FieldWriter writer = fields.get(name.toLowerCase());
+    int vectorCount = container.size();
+    if(writer == null) {
+      if (!unionEnabled){
+        writer = new SingleListWriter(name,container,this);
+      } else{
+        writer = new PromotableWriter(container.addOrGet(name, Types.optional(MinorType.LIST), ListVector.class), container);
+      }
+      if (container.size() > vectorCount) {
+        writer.allocate();
+      }
+      writer.setPosition(${index});
+      fields.put(name.toLowerCase(), writer);
+    }
+    return writer;
+  }
+
+  <#if mode == "Repeated">
+  public void start() {
+      // update the repeated vector to state that there is current+1 objects.
+    final RepeatedMapHolder h = new RepeatedMapHolder();
+    final RepeatedMapVector map = (RepeatedMapVector) container;
+    final RepeatedMapVector.Mutator mutator = map.getMutator();
+
+    // Make sure that the current vector can support the end position of this list.
+    if(container.getValueCapacity() <= idx()) {
+      mutator.setValueCount(idx()+1);
+    }
+
+    map.getAccessor().get(idx(), h);
+    if (h.start >= h.end) {
+      container.getMutator().startNewValue(idx());
+    }
+    currentChildIndex = container.getMutator().add(idx());
+    for(final FieldWriter w : fields.values()) {
+      w.setPosition(currentChildIndex);
+    }
+  }
+
+
+  public void end() {
+    // noop
+  }
+  <#else>
+
+  public void setValueCount(int count) {
+    container.getMutator().setValueCount(count);
+  }
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    for(final FieldWriter w: fields.values()) {
+      w.setPosition(index);
+    }
+  }
+
+  @Override
+  public void start() {
+  }
+
+  @Override
+  public void end() {
+  }
+
+  </#if>
+
+  <#list vv.types as type><#list type.minor as minor>
+  <#assign lowerName = minor.class?uncap_first />
+  <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+  <#assign upperName = minor.class?upper_case />
+  <#assign capName = minor.class?cap_first />
+  <#assign vectName = capName />
+  <#assign vectName = "Nullable${capName}" />
+
+  <#if minor.class?starts_with("Decimal") >
+  public ${minor.class}Writer ${lowerName}(String name) {
+    // returns existing writer
+    final FieldWriter writer = fields.get(name.toLowerCase());
+    assert writer != null;
+    return writer;
+  }
+
+  public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) {
+    final MajorType ${upperName}_TYPE = new MajorType(MinorType.${upperName}, DataMode.OPTIONAL, scale, precision, null, null);
+  <#else>
+  private static final MajorType ${upperName}_TYPE = Types.optional(MinorType.${upperName});
+  @Override
+  public ${minor.class}Writer ${lowerName}(String name) {
+  </#if>
+    FieldWriter writer = fields.get(name.toLowerCase());
+    if(writer == null) {
+      ValueVector vector;
+      ValueVector currentVector = container.getChild(name);
+      if (unionEnabled){
+        ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
+        writer = new PromotableWriter(v, container);
+        vector = v;
+      } else {
+        ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
+        writer = new ${vectName}WriterImpl(v, this);
+        vector = v;
+      }
+      if (currentVector == null || currentVector != vector) {
+        vector.allocateNewSafe();
+      } 
+      writer.setPosition(${index});
+      fields.put(name.toLowerCase(), writer);
+    }
+    return writer;
+  }
+
+  </#list></#list>
+
+}
+</#list>
diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java
new file mode 100644
index 0000000000000..3ef6c7dcc49a6
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/NullReader.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/NullReader.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+
+@SuppressWarnings("unused")
+public class NullReader extends AbstractBaseReader implements FieldReader{
+  
+  public static final NullReader INSTANCE = new NullReader();
+  public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(Types.repeated(MinorType.NULL));
+  public static final NullReader EMPTY_MAP_INSTANCE = new NullReader(Types.required(MinorType.MAP));
+  private MajorType type;
+  
+  private NullReader(){
+    super();
+    type = Types.required(MinorType.NULL);
+  }
+
+  private NullReader(MajorType type){
+    super();
+    this.type = type;
+  }
+
+  @Override
+  public MajorType getType() {
+    return type;
+  }
+  
+  public void copyAsValue(MapWriter writer) {}
+
+  public void copyAsValue(ListWriter writer) {}
+
+  public void copyAsValue(UnionWriter writer) {}
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  public void read(${name}Holder holder){
+    throw new UnsupportedOperationException("NullReader cannot write into non-nullable holder");
+  }
+
+  public void read(Nullable${name}Holder holder){
+    holder.isSet = 0;
+  }
+
+  public void read(int arrayIndex, ${name}Holder holder){
+    throw new ArrayIndexOutOfBoundsException();
+  }
+  
+  public void copyAsValue(${minor.class}Writer writer){}
+  public void copyAsField(String name, ${minor.class}Writer writer){}
+
+  public void read(int arrayIndex, Nullable${name}Holder holder){
+    throw new ArrayIndexOutOfBoundsException();
+  }
+  </#list></#list>
+  
+  public int size(){
+    return 0;
+  }
+  
+  public boolean isSet(){
+    return false;
+  }
+  
+  public boolean next(){
+    return false;
+  }
+  
+  public RepeatedMapReader map(){
+    return this;
+  }
+  
+  public RepeatedListReader list(){
+    return this;
+  }
+  
+  public MapReader map(String name){
+    return this;
+  }
+  
+  public ListReader list(String name){
+    return this;
+  }
+  
+  public FieldReader reader(String name){
+    return this;
+  }
+  
+  public FieldReader reader(){
+    return this;
+  }
+  
+  private void fail(String name){
+    throw new IllegalArgumentException(String.format("You tried to read a %s type when you are using a ValueReader of type %s.", name, this.getClass().getSimpleName()));
+  }
+  
+  <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
+          "Character", "DateTime", "Period", "Double", "Float",
+          "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
+  <#assign safeType=friendlyType />
+  <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+  
+  public ${friendlyType} read${safeType}(int arrayIndex){
+    return null;
+  }
+  
+  public ${friendlyType} read${safeType}(){
+    return null;
+  }
+  </#list>
+  
+}
+
+
+
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
new file mode 100644
index 0000000000000..6893a25efbe18
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -0,0 +1,630 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign className = "Nullable${minor.class}Vector" />
+<#assign valuesName = "${minor.class}Vector" />
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+
+<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * Nullable${minor.class} implements a vector of values which could be null.  Elements in the vector
+ * are first checked against a fixed length vector of boolean values.  Then the element is retrieved
+ * from the base class (if not null).
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+@SuppressWarnings("unused")
+public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector{
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
+
+  private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
+
+  private final MaterializedField bitsField = MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED));
+  private final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
+  private final ${valuesName} values = new ${minor.class}Vector(field, allocator);
+
+  private final Mutator mutator = new Mutator();
+  private final Accessor accessor = new Accessor();
+
+  public ${className}(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  @Override
+  public FieldReader getReader(){
+    return reader;
+  }
+
+  @Override
+  public int getValueCapacity(){
+    return Math.min(bits.getValueCapacity(), values.getValueCapacity());
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), ArrowBuf.class);
+    if (clear) {
+      for (final ArrowBuf buffer:buffers) {
+        buffer.retain(1);
+      }
+      clear();
+    }
+    return buffers;
+  }
+
+  @Override
+  public void close() {
+    bits.close();
+    values.close();
+    super.close();
+  }
+
+  @Override
+  public void clear() {
+    bits.clear();
+    values.clear();
+    super.clear();
+  }
+
+  @Override
+  public int getBufferSize(){
+    return values.getBufferSize() + bits.getBufferSize();
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    return values.getBufferSizeFor(valueCount)
+        + bits.getBufferSizeFor(valueCount);
+  }
+
+  @Override
+  public ArrowBuf getBuffer() {
+    return values.getBuffer();
+  }
+
+  @Override
+  public ${valuesName} getValuesVector() {
+    return values;
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    bits.setInitialCapacity(numRecords);
+    values.setInitialCapacity(numRecords);
+  }
+
+//  @Override
+//  public SerializedField.Builder getMetadataBuilder() {
+//    return super.getMetadataBuilder()
+//      .addChild(bits.getMetadata())
+//      .addChild(values.getMetadata());
+//  }
+
+  @Override
+  public void allocateNew() {
+    if(!allocateNewSafe()){
+      throw new OutOfMemoryException("Failure while allocating buffer.");
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* Boolean to keep track if all the memory allocations were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      success = values.allocateNewSafe() && bits.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    bits.zeroVector();
+    mutator.reset();
+    accessor.reset();
+    return success;
+  }
+
+  <#if type.major == "VarLen">
+  @Override
+  public void allocateNew(int totalBytes, int valueCount) {
+    try {
+      values.allocateNew(totalBytes, valueCount);
+      bits.allocateNew(valueCount);
+    } catch(RuntimeException e) {
+      clear();
+      throw e;
+    }
+    bits.zeroVector();
+    mutator.reset();
+    accessor.reset();
+  }
+
+  public void reset() {
+    bits.zeroVector();
+    mutator.reset();
+    accessor.reset();
+    super.reset();
+  }
+
+  @Override
+  public int getByteCapacity(){
+    return values.getByteCapacity();
+  }
+
+  @Override
+  public int getCurrentSizeInBytes(){
+    return values.getCurrentSizeInBytes();
+  }
+
+  <#else>
+  @Override
+  public void allocateNew(int valueCount) {
+    try {
+      values.allocateNew(valueCount);
+      bits.allocateNew(valueCount+1);
+    } catch(OutOfMemoryException e) {
+      clear();
+      throw e;
+    }
+    bits.zeroVector();
+    mutator.reset();
+    accessor.reset();
+  }
+
+  @Override
+  public void reset() {
+    bits.zeroVector();
+    mutator.reset();
+    accessor.reset();
+    super.reset();
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void zeroVector() {
+    bits.zeroVector();
+    values.zeroVector();
+  }
+  </#if>
+
+
+//  @Override
+//  public void load(SerializedField metadata, ArrowBuf buffer) {
+//    clear();
+    // the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
+//    final SerializedField bitsField = metadata.getChild(0);
+//    bits.load(bitsField, buffer);
+//
+//    final int capacity = buffer.capacity();
+//    final int bitsLength = bitsField.getBufferLength();
+//    final SerializedField valuesField = metadata.getChild(1);
+//    values.load(valuesField, buffer.slice(bitsLength, capacity - bitsLength));
+//  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator){
+    return new TransferImpl(getField(), allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+    return new TransferImpl(getField().withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((Nullable${minor.class}Vector) to);
+  }
+
+  public void transferTo(Nullable${minor.class}Vector target){
+    bits.transferTo(target.bits);
+    values.transferTo(target.values);
+    <#if type.major == "VarLen">
+    target.mutator.lastSet = mutator.lastSet;
+    </#if>
+    clear();
+  }
+
+  public void splitAndTransferTo(int startIndex, int length, Nullable${minor.class}Vector target) {
+    bits.splitAndTransferTo(startIndex, length, target.bits);
+    values.splitAndTransferTo(startIndex, length, target.values);
+    <#if type.major == "VarLen">
+    target.mutator.lastSet = length - 1;
+    </#if>
+  }
+
+  private class TransferImpl implements TransferPair {
+    Nullable${minor.class}Vector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator){
+      to = new Nullable${minor.class}Vector(field, allocator);
+    }
+
+    public TransferImpl(Nullable${minor.class}Vector to){
+      this.to = to;
+    }
+
+    @Override
+    public Nullable${minor.class}Vector getTo(){
+      return to;
+    }
+
+    @Override
+    public void transfer(){
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, Nullable${minor.class}Vector.this);
+    }
+  }
+
+  @Override
+  public Accessor getAccessor(){
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator(){
+    return mutator;
+  }
+
+  public ${minor.class}Vector convertToRequiredVector(){
+    ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
+    if (v.data != null) {
+      v.data.release(1);
+    }
+    v.data = values.data;
+    v.data.retain(1);
+    clear();
+    return v;
+  }
+
+  public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
+    final Accessor fromAccessor = from.getAccessor();
+    if (!fromAccessor.isNull(fromIndex)) {
+      mutator.set(thisIndex, fromAccessor.get(fromIndex));
+    }
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    <#if type.major == "VarLen">
+    mutator.fillEmpties(thisIndex);
+    </#if>
+    values.copyFromSafe(fromIndex, thisIndex, from);
+    bits.getMutator().setSafe(thisIndex, 1);
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
+    <#if type.major == "VarLen">
+    mutator.fillEmpties(thisIndex);
+    </#if>
+    bits.copyFromSafe(fromIndex, thisIndex, from.bits);
+    values.copyFromSafe(fromIndex, thisIndex, from.values);
+  }
+
+  public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
+    final UInt1Vector.Accessor bAccessor = bits.getAccessor();
+    final ${valuesName}.Accessor vAccessor = values.getAccessor();
+
+    /**
+     * Get the element at the specified position.
+     *
+     * @param   index   position of the value
+     * @return  value of the element, if not null
+     * @throws  NullValueException if the value is null
+     */
+    public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+      if (isNull(index)) {
+          throw new IllegalStateException("Can't get a null value");
+      }
+      return vAccessor.get(index);
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return isSet(index) == 0;
+    }
+
+    public int isSet(int index){
+      return bAccessor.get(index);
+    }
+
+    <#if type.major == "VarLen">
+    public long getStartEnd(int index){
+      return vAccessor.getStartEnd(index);
+    }
+
+    @Override
+    public int getValueLength(int index) {
+      return values.getAccessor().getValueLength(index);
+    }
+    </#if>
+
+    public void get(int index, Nullable${minor.class}Holder holder){
+      vAccessor.get(index, holder);
+      holder.isSet = bAccessor.get(index);
+
+      <#if minor.class.startsWith("Decimal")>
+      holder.scale = getField().getScale();
+      holder.precision = getField().getPrecision();
+      </#if>
+    }
+
+    @Override
+    public ${friendlyType} getObject(int index) {
+      if (isNull(index)) {
+          return null;
+      }else{
+        return vAccessor.getObject(index);
+      }
+    }
+
+    <#if minor.class == "Interval" || minor.class == "IntervalDay" || minor.class == "IntervalYear">
+    public StringBuilder getAsStringBuilder(int index) {
+      if (isNull(index)) {
+          return null;
+      }else{
+        return vAccessor.getAsStringBuilder(index);
+      }
+    }
+    </#if>
+
+    @Override
+    public int getValueCount(){
+      return bits.getAccessor().getValueCount();
+    }
+
+    public void reset(){}
+  }
+
+  public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> {
+    private int setCount;
+    <#if type.major = "VarLen"> private int lastSet = -1;</#if>
+
+    private Mutator(){
+    }
+
+    public ${valuesName} getVectorWithValues(){
+      return values;
+    }
+
+    @Override
+    public void setIndexDefined(int index){
+      bits.getMutator().set(index, 1);
+    }
+
+    /**
+     * Set the variable length element at the specified index to the supplied byte array.
+     *
+     * @param index   position of the bit to set
+     * @param bytes   array of bytes to write
+     */
+    public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+      setCount++;
+      final ${valuesName}.Mutator valuesMutator = values.getMutator();
+      final UInt1Vector.Mutator bitsMutator = bits.getMutator();
+      <#if type.major == "VarLen">
+      for (int i = lastSet + 1; i < index; i++) {
+        valuesMutator.set(i, emptyByteArray);
+      }
+      </#if>
+      bitsMutator.set(index, 1);
+      valuesMutator.set(index, value);
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
+    <#if type.major == "VarLen">
+
+    private void fillEmpties(int index){
+      final ${valuesName}.Mutator valuesMutator = values.getMutator();
+      for (int i = lastSet; i < index; i++) {
+        valuesMutator.setSafe(i + 1, emptyByteArray);
+      }
+      while(index > bits.getValueCapacity()) {
+        bits.reAlloc();
+      }
+      lastSet = index;
+    }
+
+    @Override
+    public void setValueLengthSafe(int index, int length) {
+      values.getMutator().setValueLengthSafe(index, length);
+      lastSet = index;
+    }
+    </#if>
+
+    public void setSafe(int index, byte[] value, int start, int length) {
+      <#if type.major != "VarLen">
+      throw new UnsupportedOperationException();
+      <#else>
+      fillEmpties(index);
+
+      bits.getMutator().setSafe(index, 1);
+      values.getMutator().setSafe(index, value, start, length);
+      setCount++;
+      <#if type.major == "VarLen">lastSet = index;</#if>
+      </#if>
+    }
+
+    public void setSafe(int index, ByteBuffer value, int start, int length) {
+      <#if type.major != "VarLen">
+      throw new UnsupportedOperationException();
+      <#else>
+      fillEmpties(index);
+
+      bits.getMutator().setSafe(index, 1);
+      values.getMutator().setSafe(index, value, start, length);
+      setCount++;
+      <#if type.major == "VarLen">lastSet = index;</#if>
+      </#if>
+    }
+
+    public void setNull(int index){
+      bits.getMutator().setSafe(index, 0);
+    }
+
+    public void setSkipNull(int index, ${minor.class}Holder holder){
+      values.getMutator().set(index, holder);
+    }
+
+    public void setSkipNull(int index, Nullable${minor.class}Holder holder){
+      values.getMutator().set(index, holder);
+    }
+
+
+    public void set(int index, Nullable${minor.class}Holder holder){
+      final ${valuesName}.Mutator valuesMutator = values.getMutator();
+      <#if type.major == "VarLen">
+      for (int i = lastSet + 1; i < index; i++) {
+        valuesMutator.set(i, emptyByteArray);
+      }
+      </#if>
+      bits.getMutator().set(index, holder.isSet);
+      valuesMutator.set(index, holder);
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
+    public void set(int index, ${minor.class}Holder holder){
+      final ${valuesName}.Mutator valuesMutator = values.getMutator();
+      <#if type.major == "VarLen">
+      for (int i = lastSet + 1; i < index; i++) {
+        valuesMutator.set(i, emptyByteArray);
+      }
+      </#if>
+      bits.getMutator().set(index, 1);
+      valuesMutator.set(index, holder);
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
+    public boolean isSafe(int outIndex) {
+      return outIndex < Nullable${minor.class}Vector.this.getValueCapacity();
+    }
+
+    <#assign fields = minor.fields!type.fields />
+    public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){
+      final ${valuesName}.Mutator valuesMutator = values.getMutator();
+      <#if type.major == "VarLen">
+      for (int i = lastSet + 1; i < index; i++) {
+        valuesMutator.set(i, emptyByteArray);
+      }
+      </#if>
+      bits.getMutator().set(index, isSet);
+      valuesMutator.set(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
+    public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
+      <#if type.major == "VarLen">
+      fillEmpties(index);
+      </#if>
+
+      bits.getMutator().setSafe(index, isSet);
+      values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
+      setCount++;
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
+
+    public void setSafe(int index, Nullable${minor.class}Holder value) {
+
+      <#if type.major == "VarLen">
+      fillEmpties(index);
+      </#if>
+      bits.getMutator().setSafe(index, value.isSet);
+      values.getMutator().setSafe(index, value);
+      setCount++;
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
+    public void setSafe(int index, ${minor.class}Holder value) {
+
+      <#if type.major == "VarLen">
+      fillEmpties(index);
+      </#if>
+      bits.getMutator().setSafe(index, 1);
+      values.getMutator().setSafe(index, value);
+      setCount++;
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
+    <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")>
+      public void setSafe(int index, ${minor.javaType!type.javaType} value) {
+        <#if type.major == "VarLen">
+        fillEmpties(index);
+        </#if>
+        bits.getMutator().setSafe(index, 1);
+        values.getMutator().setSafe(index, value);
+        setCount++;
+      }
+
+    </#if>
+
+    @Override
+    public void setValueCount(int valueCount) {
+      assert valueCount >= 0;
+      <#if type.major == "VarLen">
+      fillEmpties(valueCount);
+      </#if>
+      values.getMutator().setValueCount(valueCount);
+      bits.getMutator().setValueCount(valueCount);
+    }
+
+    @Override
+    public void generateTestData(int valueCount){
+      bits.getMutator().generateTestDataAlt(valueCount);
+      values.getMutator().generateTestData(valueCount);
+      <#if type.major = "VarLen">lastSet = valueCount;</#if>
+      setValueCount(valueCount);
+    }
+
+    @Override
+    public void reset(){
+      setCount = 0;
+      <#if type.major = "VarLen">lastSet = -1;</#if>
+    }
+  }
+}
+</#list>
+</#list>
diff --git a/java/vector/src/main/codegen/templates/RepeatedValueVectors.java b/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
new file mode 100644
index 0000000000000..5ac80f57737ff
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
@@ -0,0 +1,421 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+<#assign fields = minor.fields!type.fields />
+
+<@pp.changeOutputFile name="/org/apache/arrow/vector/Repeated${minor.class}Vector.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
+ * repeated protobuf field).  The implementation uses two additional value vectors; one to convert
+ * the index offset to the underlying element offset, and another to store the number of values
+ * in the vector.
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+
+public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>VectorLike {
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Repeated${minor.class}Vector.class);
+
+  // we maintain local reference to concrete vector type for performance reasons.
+  private ${minor.class}Vector values;
+  private final FieldReader reader = new Repeated${minor.class}ReaderImpl(Repeated${minor.class}Vector.this);
+  private final Mutator mutator = new Mutator();
+  private final Accessor accessor = new Accessor();
+
+  public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    addOrGetVector(VectorDescriptor.create(new MajorType(field.getType().getMinorType(), DataMode.REQUIRED)));
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public ${minor.class}Vector getDataVector() {
+    return values;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new TransferImpl(getField(), allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+    return new TransferImpl(getField().withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((Repeated${minor.class}Vector) to);
+  }
+
+  @Override
+  public AddOrGetResult<${minor.class}Vector> addOrGetVector(VectorDescriptor descriptor) {
+    final AddOrGetResult<${minor.class}Vector> result = super.addOrGetVector(descriptor);
+    if (result.isCreated()) {
+      values = result.getVector();
+    }
+    return result;
+  }
+
+  public void transferTo(Repeated${minor.class}Vector target) {
+    target.clear();
+    offsets.transferTo(target.offsets);
+    values.transferTo(target.values);
+    clear();
+  }
+
+  public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) {
+    final UInt4Vector.Accessor a = offsets.getAccessor();
+    final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+    final int startPos = a.get(startIndex);
+    final int endPos = a.get(startIndex + groups);
+    final int valuesToCopy = endPos - startPos;
+
+    values.splitAndTransferTo(startPos, valuesToCopy, to.values);
+    to.offsets.clear();
+    to.offsets.allocateNew(groups + 1);
+    int normalizedPos = 0;
+    for (int i=0; i < groups + 1;i++ ) {
+      normalizedPos = a.get(startIndex+i) - startPos;
+      m.set(i, normalizedPos);
+    }
+    m.setValueCount(groups == 0 ? 0 : groups + 1);
+  }
+
+  private class TransferImpl implements TransferPair {
+    final Repeated${minor.class}Vector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
+      this.to = new Repeated${minor.class}Vector(field, allocator);
+    }
+
+    public TransferImpl(Repeated${minor.class}Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public Repeated${minor.class}Vector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, Repeated${minor.class}Vector.this);
+    }
+  }
+
+    public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
+      final Accessor vAccessor = v.getAccessor();
+      final int count = vAccessor.getInnerValueCountAt(inIndex);
+      mutator.startNewValue(outIndex);
+      for (int i = 0; i < count; i++) {
+        mutator.add(outIndex, vAccessor.get(inIndex, i));
+      }
+    }
+
+    public void copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
+      final Accessor vAccessor = v.getAccessor();
+      final int count = vAccessor.getInnerValueCountAt(inIndex);
+      mutator.startNewValue(outIndex);
+      for (int i = 0; i < count; i++) {
+        mutator.addSafe(outIndex, vAccessor.get(inIndex, i));
+      }
+    }
+
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      if(!offsets.allocateNewSafe()) return false;
+      if(!values.allocateNewSafe()) return false;
+      success = true;
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    offsets.zeroVector();
+    mutator.reset();
+    return true;
+  }
+
+  @Override
+  public void allocateNew() {
+    try {
+      offsets.allocateNew();
+      values.allocateNew();
+    } catch (OutOfMemoryException e) {
+      clear();
+      throw e;
+    }
+    offsets.zeroVector();
+    mutator.reset();
+  }
+
+  <#if type.major == "VarLen">
+//  @Override
+//  protected SerializedField.Builder getMetadataBuilder() {
+//    return super.getMetadataBuilder()
+//            .setVarByteLength(values.getVarByteLength());
+//  }
+
+  public void allocateNew(int totalBytes, int valueCount, int innerValueCount) {
+    try {
+      offsets.allocateNew(valueCount + 1);
+      values.allocateNew(totalBytes, innerValueCount);
+    } catch (OutOfMemoryException e) {
+      clear();
+      throw e;
+    }
+    offsets.zeroVector();
+    mutator.reset();
+  }
+
+  public int getByteCapacity(){
+    return values.getByteCapacity();
+  }
+
+  <#else>
+
+  @Override
+  public void allocateNew(int valueCount, int innerValueCount) {
+    clear();
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to//
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      offsets.allocateNew(valueCount + 1);
+      values.allocateNew(innerValueCount);
+    } catch(OutOfMemoryException e){
+      clear();
+      throw e;
+    }
+    offsets.zeroVector();
+    mutator.reset();
+  }
+
+  </#if>
+
+  // This is declared a subclass of the accessor declared inside of FixedWidthVector, this is also used for
+  // variable length vectors, as they should ahve consistent interface as much as possible, if they need to diverge
+  // in the future, the interface shold be declared in the respective value vector superclasses for fixed and variable
+  // and we should refer to each in the generation template
+  public final class Accessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
+    @Override
+    public List<${friendlyType}> getObject(int index) {
+      final List<${friendlyType}> vals = new JsonStringArrayList<>();
+      final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
+      final int start = offsetsAccessor.get(index);
+      final int end = offsetsAccessor.get(index + 1);
+      final ${minor.class}Vector.Accessor valuesAccessor = values.getAccessor();
+      for(int i = start; i < end; i++) {
+        vals.add(valuesAccessor.getObject(i));
+      }
+      return vals;
+    }
+
+    public ${friendlyType} getSingleObject(int index, int arrayIndex) {
+      final int start = offsets.getAccessor().get(index);
+      return values.getAccessor().getObject(start + arrayIndex);
+    }
+
+    /**
+     * Get a value for the given record.  Each element in the repeated field is accessed by
+     * the positionIndex param.
+     *
+     * @param  index           record containing the repeated field
+     * @param  positionIndex   position within the repeated field
+     * @return element at the given position in the given record
+     */
+    public <#if type.major == "VarLen">byte[]
+           <#else>${minor.javaType!type.javaType}
+           </#if> get(int index, int positionIndex) {
+      return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
+    }
+
+    public void get(int index, Repeated${minor.class}Holder holder) {
+      holder.start = offsets.getAccessor().get(index);
+      holder.end =  offsets.getAccessor().get(index+1);
+      holder.vector = values;
+    }
+
+    public void get(int index, int positionIndex, ${minor.class}Holder holder) {
+      final int offset = offsets.getAccessor().get(index);
+      assert offset >= 0;
+      assert positionIndex < getInnerValueCountAt(index);
+      values.getAccessor().get(offset + positionIndex, holder);
+    }
+
+    public void get(int index, int positionIndex, Nullable${minor.class}Holder holder) {
+      final int offset = offsets.getAccessor().get(index);
+      assert offset >= 0;
+      if (positionIndex >= getInnerValueCountAt(index)) {
+        holder.isSet = 0;
+        return;
+      }
+      values.getAccessor().get(offset + positionIndex, holder);
+    }
+  }
+
+  public final class Mutator extends BaseRepeatedValueVector.BaseRepeatedMutator implements RepeatedMutator {
+    private Mutator() {}
+
+    /**
+     * Add an element to the given record index.  This is similar to the set() method in other
+     * value vectors, except that it permits setting multiple values for a single record.
+     *
+     * @param index   record of the element to add
+     * @param value   value to add to the given row
+     */
+    public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+      int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().set(nextOffset, value);
+      offsets.getMutator().set(index+1, nextOffset+1);
+    }
+
+    <#if type.major == "VarLen">
+    public void addSafe(int index, byte[] bytes) {
+      addSafe(index, bytes, 0, bytes.length);
+    }
+
+    public void addSafe(int index, byte[] bytes, int start, int length) {
+      final int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setSafe(nextOffset, bytes, start, length);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
+    <#else>
+
+    public void addSafe(int index, ${minor.javaType!type.javaType} srcValue) {
+      final int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setSafe(nextOffset, srcValue);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
+    </#if>
+
+    public void setSafe(int index, Repeated${minor.class}Holder h) {
+      final ${minor.class}Holder ih = new ${minor.class}Holder();
+      final ${minor.class}Vector.Accessor hVectorAccessor = h.vector.getAccessor();
+      mutator.startNewValue(index);
+      for(int i = h.start; i < h.end; i++){
+        hVectorAccessor.get(i, ih);
+        mutator.addSafe(index, ih);
+      }
+    }
+
+    public void addSafe(int index, ${minor.class}Holder holder) {
+      int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setSafe(nextOffset, holder);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
+    public void addSafe(int index, Nullable${minor.class}Holder holder) {
+      final int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setSafe(nextOffset, holder);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
+    <#if (fields?size > 1) && !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+    public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+      int nextOffset = offsets.getAccessor().get(arrayIndex+1);
+      values.getMutator().setSafe(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+      offsets.getMutator().setSafe(arrayIndex+1, nextOffset+1);
+    }
+    </#if>
+
+    protected void add(int index, ${minor.class}Holder holder) {
+      int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().set(nextOffset, holder);
+      offsets.getMutator().set(index+1, nextOffset+1);
+    }
+
+    public void add(int index, Repeated${minor.class}Holder holder) {
+
+      ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
+      ${minor.class}Holder innerHolder = new ${minor.class}Holder();
+
+      for(int i = holder.start; i < holder.end; i++) {
+        accessor.get(i, innerHolder);
+        add(index, innerHolder);
+      }
+    }
+
+    @Override
+    public void generateTestData(final int valCount) {
+      final int[] sizes = {1, 2, 0, 6};
+      int size = 0;
+      int runningOffset = 0;
+      final UInt4Vector.Mutator offsetsMutator = offsets.getMutator();
+      for(int i = 1; i < valCount + 1; i++, size++) {
+        runningOffset += sizes[size % sizes.length];
+        offsetsMutator.set(i, runningOffset);
+      }
+      values.getMutator().generateTestData(valCount * 9);
+      setValueCount(size);
+    }
+
+    @Override
+    public void reset() {
+    }
+  }
+}
+</#list>
+</#list>
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
new file mode 100644
index 0000000000000..9a6b08fc561f9
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.UnsupportedOperationException;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionListWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+
+@SuppressWarnings("unused")
+public class UnionListWriter extends AbstractFieldWriter {
+
+  private ListVector vector;
+  private UInt4Vector offsets;
+  private PromotableWriter writer;
+  private boolean inMap = false;
+  private String mapName;
+  private int lastIndex = 0;
+
+  public UnionListWriter(ListVector vector) {
+    super(null);
+    this.vector = vector;
+    this.writer = new PromotableWriter(vector.getDataVector(), vector);
+    this.offsets = vector.getOffsetVector();
+  }
+
+  public UnionListWriter(ListVector vector, AbstractFieldWriter parent) {
+    this(vector);
+  }
+
+  @Override
+  public void allocate() {
+    vector.allocateNew();
+  }
+
+  @Override
+  public void clear() {
+    vector.clear();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return null;
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return vector.getValueCapacity();
+  }
+
+  @Override
+  public void close() throws Exception {
+
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#assign uncappedName = name?uncap_first/>
+
+  <#if !minor.class?starts_with("Decimal")>
+
+  @Override
+  public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>() {
+    return this;
+  }
+
+  @Override
+  public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
+    assert inMap;
+    mapName = name;
+    final int nextOffset = offsets.getAccessor().get(idx() + 1);
+    vector.getMutator().setNotNull(idx());
+    writer.setPosition(nextOffset);
+    ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name);
+    return ${uncappedName}Writer;
+  }
+
+  </#if>
+
+  </#list></#list>
+
+  @Override
+  public MapWriter map() {
+    inMap = true;
+    return this;
+  }
+
+  @Override
+  public ListWriter list() {
+    final int nextOffset = offsets.getAccessor().get(idx() + 1);
+    vector.getMutator().setNotNull(idx());
+    offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
+    writer.setPosition(nextOffset);
+    return writer;
+  }
+
+  @Override
+  public ListWriter list(String name) {
+    final int nextOffset = offsets.getAccessor().get(idx() + 1);
+    vector.getMutator().setNotNull(idx());
+    writer.setPosition(nextOffset);
+    ListWriter listWriter = writer.list(name);
+    return listWriter;
+  }
+
+  @Override
+  public MapWriter map(String name) {
+    MapWriter mapWriter = writer.map(name);
+    return mapWriter;
+  }
+
+  @Override
+  public void startList() {
+    vector.getMutator().startNewValue(idx());
+  }
+
+  @Override
+  public void endList() {
+
+  }
+
+  @Override
+  public void start() {
+    assert inMap;
+    final int nextOffset = offsets.getAccessor().get(idx() + 1);
+    vector.getMutator().setNotNull(idx());
+    offsets.getMutator().setSafe(idx() + 1, nextOffset);
+    writer.setPosition(nextOffset);
+  }
+
+  @Override
+  public void end() {
+    if (inMap) {
+      inMap = false;
+      final int nextOffset = offsets.getAccessor().get(idx() + 1);
+      offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
+    }
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#assign uncappedName = name?uncap_first/>
+
+  <#if !minor.class?starts_with("Decimal")>
+
+  @Override
+  public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    assert !inMap;
+    final int nextOffset = offsets.getAccessor().get(idx() + 1);
+    vector.getMutator().setNotNull(idx());
+    writer.setPosition(nextOffset);
+    writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+    offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
+  }
+
+  </#if>
+
+  </#list></#list>
+
+}
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
new file mode 100644
index 0000000000000..44c3e55dcc6f1
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionReader.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+@SuppressWarnings("unused")
+public class UnionReader extends AbstractFieldReader {
+
+  private BaseReader[] readers = new BaseReader[43];
+  public UnionVector data;
+  
+  public UnionReader(UnionVector data) {
+    this.data = data;
+  }
+
+  private static MajorType[] TYPES = new MajorType[43];
+
+  static {
+    for (MinorType minorType : MinorType.values()) {
+      TYPES[minorType.ordinal()] = new MajorType(minorType, DataMode.OPTIONAL);
+    }
+  }
+
+  public MajorType getType() {
+    return TYPES[data.getTypeValue(idx())];
+  }
+
+  public boolean isSet(){
+    return !data.getAccessor().isNull(idx());
+  }
+
+  public void read(UnionHolder holder) {
+    holder.reader = this;
+    holder.isSet = this.isSet() ? 1 : 0;
+  }
+
+  public void read(int index, UnionHolder holder) {
+    getList().read(index, holder);
+  }
+
+  private FieldReader getReaderForIndex(int index) {
+    int typeValue = data.getTypeValue(index);
+    FieldReader reader = (FieldReader) readers[typeValue];
+    if (reader != null) {
+      return reader;
+    }
+    switch (MinorType.values()[typeValue]) {
+    case LATE:
+      return NullReader.INSTANCE;
+    case MAP:
+      return (FieldReader) getMap();
+    case LIST:
+      return (FieldReader) getList();
+    <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+    <#assign uncappedName = name?uncap_first/>
+    <#if !minor.class?starts_with("Decimal")>
+    case ${name?upper_case}:
+      return (FieldReader) get${name}();
+    </#if>
+    </#list></#list>
+    default:
+      throw new UnsupportedOperationException("Unsupported type: " + MinorType.values()[typeValue]);
+    }
+  }
+
+  private SingleMapReaderImpl mapReader;
+
+  private MapReader getMap() {
+    if (mapReader == null) {
+      mapReader = (SingleMapReaderImpl) data.getMap().getReader();
+      mapReader.setPosition(idx());
+      readers[MinorType.MAP.ordinal()] = mapReader;
+    }
+    return mapReader;
+  }
+
+  private UnionListReader listReader;
+
+  private FieldReader getList() {
+    if (listReader == null) {
+      listReader = new UnionListReader(data.getList());
+      listReader.setPosition(idx());
+      readers[MinorType.LIST.ordinal()] = listReader;
+    }
+    return listReader;
+  }
+
+  @Override
+  public java.util.Iterator<String> iterator() {
+    return getMap().iterator();
+  }
+
+  @Override
+  public void copyAsValue(UnionWriter writer) {
+    writer.data.copyFrom(idx(), writer.idx(), data);
+  }
+
+  <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
+          "Character", "DateTime", "Period", "Double", "Float",
+          "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
+  <#assign safeType=friendlyType />
+  <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+
+  @Override
+  public ${friendlyType} read${safeType}() {
+    return getReaderForIndex(idx()).read${safeType}();
+  }
+
+  </#list>
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+          <#assign uncappedName = name?uncap_first/>
+  <#assign boxedType = (minor.boxedType!type.boxedType) />
+  <#assign javaType = (minor.javaType!type.javaType) />
+  <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+  <#assign safeType=friendlyType />
+  <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+  <#if !minor.class?starts_with("Decimal")>
+
+  private Nullable${name}ReaderImpl ${uncappedName}Reader;
+
+  private Nullable${name}ReaderImpl get${name}() {
+    if (${uncappedName}Reader == null) {
+      ${uncappedName}Reader = new Nullable${name}ReaderImpl(data.get${name}Vector());
+      ${uncappedName}Reader.setPosition(idx());
+      readers[MinorType.${name?upper_case}.ordinal()] = ${uncappedName}Reader;
+    }
+    return ${uncappedName}Reader;
+  }
+
+  public void read(Nullable${name}Holder holder){
+    getReaderForIndex(idx()).read(holder);
+  }
+
+  public void copyAsValue(${name}Writer writer){
+    getReaderForIndex(idx()).copyAsValue(writer);
+  }
+  </#if>
+  </#list></#list>
+
+  @Override
+  public void copyAsValue(ListWriter writer) {
+    ComplexCopier.copy(this, (FieldWriter) writer);
+  }
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    for (BaseReader reader : readers) {
+      if (reader != null) {
+        reader.setPosition(index);
+      }
+    }
+  }
+  
+  public FieldReader reader(String name){
+    return getMap().reader(name);
+  }
+
+  public FieldReader reader() {
+    return getList().reader();
+  }
+
+  public boolean next() {
+    return getReaderForIndex(idx()).next();
+  }
+}
+
+
+
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
new file mode 100644
index 0000000000000..ba94ac22a05f6
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -0,0 +1,467 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/UnionVector.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex;
+
+<#include "/@includes/vv_imports.ftl" />
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.apache.arrow.vector.complex.impl.ComplexCopier;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+
+
+/**
+ * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each
+ * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods,
+ * as well as the addOrGet method.
+ *
+ * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
+ * each time the vector is accessed.
+ */
+public class UnionVector implements ValueVector {
+
+  private MaterializedField field;
+  private BufferAllocator allocator;
+  private Accessor accessor = new Accessor();
+  private Mutator mutator = new Mutator();
+  private int valueCount;
+
+  private MapVector internalMap;
+  private UInt1Vector typeVector;
+
+  private MapVector mapVector;
+  private ListVector listVector;
+
+  private FieldReader reader;
+  private NullableBitVector bit;
+
+  private int singleType = 0;
+  private ValueVector singleVector;
+  private MajorType majorType;
+
+  private final CallBack callBack;
+
+  public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    this.field = field.clone();
+    this.allocator = allocator;
+    this.internalMap = new MapVector("internal", allocator, callBack);
+    this.typeVector = internalMap.addOrGet("types", new MajorType(MinorType.UINT1, DataMode.REQUIRED), UInt1Vector.class);
+    this.field.addChild(internalMap.getField().clone());
+    this.majorType = field.getType();
+    this.callBack = callBack;
+  }
+
+  public BufferAllocator getAllocator() {
+    return allocator;
+  }
+
+  public List<MinorType> getSubTypes() {
+    return majorType.getSubTypes();
+  }
+
+  public void addSubType(MinorType type) {
+    if (majorType.getSubTypes().contains(type)) {
+      return;
+    }
+    List<MinorType> subTypes = this.majorType.getSubTypes();
+    List<MinorType> newSubTypes = new ArrayList<>(subTypes);
+    newSubTypes.add(type);
+    majorType =  new MajorType(this.majorType.getMinorType(), this.majorType.getMode(), this.majorType.getPrecision(),
+            this.majorType.getScale(), this.majorType.getTimezone(), newSubTypes);
+    field = MaterializedField.create(field.getName(), majorType);
+    if (callBack != null) {
+      callBack.doWork();
+    }
+  }
+
+  private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
+
+  public MapVector getMap() {
+    if (mapVector == null) {
+      int vectorCount = internalMap.size();
+      mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class);
+      addSubType(MinorType.MAP);
+      if (internalMap.size() > vectorCount) {
+        mapVector.allocateNew();
+      }
+    }
+    return mapVector;
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#assign uncappedName = name?uncap_first/>
+  <#if !minor.class?starts_with("Decimal")>
+
+  private Nullable${name}Vector ${uncappedName}Vector;
+  private static final MajorType ${name?upper_case}_TYPE = new MajorType(MinorType.${name?upper_case}, DataMode.OPTIONAL);
+
+  public Nullable${name}Vector get${name}Vector() {
+    if (${uncappedName}Vector == null) {
+      int vectorCount = internalMap.size();
+      ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class);
+      addSubType(MinorType.${name?upper_case});
+      if (internalMap.size() > vectorCount) {
+        ${uncappedName}Vector.allocateNew();
+      }
+    }
+    return ${uncappedName}Vector;
+  }
+
+  </#if>
+
+  </#list></#list>
+
+  private static final MajorType LIST_TYPE = new MajorType(MinorType.LIST, DataMode.OPTIONAL);
+
+  public ListVector getList() {
+    if (listVector == null) {
+      int vectorCount = internalMap.size();
+      listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class);
+      addSubType(MinorType.LIST);
+      if (internalMap.size() > vectorCount) {
+        listVector.allocateNew();
+      }
+    }
+    return listVector;
+  }
+
+  public int getTypeValue(int index) {
+    return typeVector.getAccessor().get(index);
+  }
+
+  public UInt1Vector getTypeVector() {
+    return typeVector;
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    internalMap.allocateNew();
+    if (typeVector != null) {
+      typeVector.zeroVector();
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    boolean safe = internalMap.allocateNewSafe();
+    if (safe) {
+      if (typeVector != null) {
+        typeVector.zeroVector();
+      }
+    }
+    return safe;
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity());
+  }
+
+  @Override
+  public void close() {
+  }
+
+  @Override
+  public void clear() {
+    internalMap.clear();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new TransferImpl(field, allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(field.withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector target) {
+    return new TransferImpl((UnionVector) target);
+  }
+
+  public void transferTo(UnionVector target) {
+    internalMap.makeTransferPair(target.internalMap).transfer();
+    target.valueCount = valueCount;
+    target.majorType = majorType;
+  }
+
+  public void copyFrom(int inIndex, int outIndex, UnionVector from) {
+    from.getReader().setPosition(inIndex);
+    getWriter().setPosition(outIndex);
+    ComplexCopier.copy(from.reader, mutator.writer);
+  }
+
+  public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
+    copyFrom(inIndex, outIndex, from);
+  }
+
+  public ValueVector addVector(ValueVector v) {
+    String name = v.getField().getType().getMinorType().name().toLowerCase();
+    MajorType type = v.getField().getType();
+    Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
+    final ValueVector newVector = internalMap.addOrGet(name, type, (Class<ValueVector>) BasicTypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()));
+    v.makeTransferPair(newVector).transfer();
+    internalMap.putChild(name, newVector);
+    addSubType(v.getField().getType().getMinorType());
+    return newVector;
+  }
+
+  private class TransferImpl implements TransferPair {
+
+    UnionVector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
+      to = new UnionVector(field, allocator, null);
+    }
+
+    public TransferImpl(UnionVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      this.to.copyFrom(from, to, UnionVector.this);
+    }
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    if (reader == null) {
+      reader = new UnionReader(this);
+    }
+    return reader;
+  }
+
+  public FieldWriter getWriter() {
+    if (mutator.writer == null) {
+      mutator.writer = new UnionWriter(this);
+    }
+    return mutator.writer;
+  }
+
+//  @Override
+//  public UserBitShared.SerializedField getMetadata() {
+//    SerializedField.Builder b = getField() //
+//            .getAsBuilder() //
+//            .setBufferLength(getBufferSize()) //
+//            .setValueCount(valueCount);
+//
+//    b.addChild(internalMap.getMetadata());
+//    return b.build();
+//  }
+
+  @Override
+  public int getBufferSize() {
+    return internalMap.getBufferSize();
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    long bufferSize = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      bufferSize += v.getBufferSizeFor(valueCount);
+    }
+
+    return (int) bufferSize;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    return internalMap.getBuffers(clear);
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    List<ValueVector> vectors = Lists.newArrayList(internalMap.iterator());
+    vectors.add(typeVector);
+    return vectors.iterator();
+  }
+
+  public class Accessor extends BaseValueVector.BaseAccessor {
+
+
+    @Override
+    public Object getObject(int index) {
+      int type = typeVector.getAccessor().get(index);
+      switch (MinorType.values()[type]) {
+      case LATE:
+        return null;
+      <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#if !minor.class?starts_with("Decimal")>
+      case ${name?upper_case}:
+        return get${name}Vector().getAccessor().getObject(index);
+      </#if>
+
+      </#list></#list>
+      case MAP:
+        return getMap().getAccessor().getObject(index);
+      case LIST:
+        return getList().getAccessor().getObject(index);
+      default:
+        throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[type]);
+      }
+    }
+
+    public byte[] get(int index) {
+      return null;
+    }
+
+    public void get(int index, ComplexHolder holder) {
+    }
+
+    public void get(int index, UnionHolder holder) {
+      FieldReader reader = new UnionReader(UnionVector.this);
+      reader.setPosition(index);
+      holder.reader = reader;
+    }
+
+    @Override
+    public int getValueCount() {
+      return valueCount;
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return typeVector.getAccessor().get(index) == 0;
+    }
+
+    public int isSet(int index) {
+      return isNull(index) ? 0 : 1;
+    }
+  }
+
+  public class Mutator extends BaseValueVector.BaseMutator {
+
+    UnionWriter writer;
+
+    @Override
+    public void setValueCount(int valueCount) {
+      UnionVector.this.valueCount = valueCount;
+      internalMap.getMutator().setValueCount(valueCount);
+    }
+
+    public void setSafe(int index, UnionHolder holder) {
+      FieldReader reader = holder.reader;
+      if (writer == null) {
+        writer = new UnionWriter(UnionVector.this);
+      }
+      writer.setPosition(index);
+      MinorType type = reader.getType().getMinorType();
+      switch (type) {
+      <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#if !minor.class?starts_with("Decimal")>
+      case ${name?upper_case}:
+        Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
+        reader.read(${uncappedName}Holder);
+        setSafe(index, ${uncappedName}Holder);
+        break;
+      </#if>
+      </#list></#list>
+      case MAP: {
+        ComplexCopier.copy(reader, writer);
+        break;
+      }
+      case LIST: {
+        ComplexCopier.copy(reader, writer);
+        break;
+      }
+      default:
+        throw new UnsupportedOperationException();
+      }
+    }
+
+    <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+    <#assign fields = minor.fields!type.fields />
+    <#assign uncappedName = name?uncap_first/>
+    <#if !minor.class?starts_with("Decimal")>
+    public void setSafe(int index, Nullable${name}Holder holder) {
+      setType(index, MinorType.${name?upper_case});
+      get${name}Vector().getMutator().setSafe(index, holder);
+    }
+
+    </#if>
+    </#list></#list>
+
+    public void setType(int index, MinorType type) {
+      typeVector.getMutator().setSafe(index, type.ordinal());
+    }
+
+    @Override
+    public void reset() { }
+
+    @Override
+    public void generateTestData(int values) { }
+  }
+}
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
new file mode 100644
index 0000000000000..c9c29e0dd5f92
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -0,0 +1,228 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
+
+  UnionVector data;
+  private MapWriter mapWriter;
+  private UnionListWriter listWriter;
+  private List<BaseWriter> writers = Lists.newArrayList();
+
+  public UnionWriter(BufferAllocator allocator) {
+    super(null);
+  }
+
+  public UnionWriter(UnionVector vector) {
+    super(null);
+    data = vector;
+  }
+
+  public UnionWriter(UnionVector vector, FieldWriter parent) {
+    super(null);
+    data = vector;
+  }
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    for (BaseWriter writer : writers) {
+      writer.setPosition(index);
+    }
+  }
+
+
+  @Override
+  public void start() {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    getMapWriter().start();
+  }
+
+  @Override
+  public void end() {
+    getMapWriter().end();
+  }
+
+  @Override
+  public void startList() {
+    getListWriter().startList();
+    data.getMutator().setType(idx(), MinorType.LIST);
+  }
+
+  @Override
+  public void endList() {
+    getListWriter().endList();
+  }
+
+  private MapWriter getMapWriter() {
+    if (mapWriter == null) {
+      mapWriter = new SingleMapWriter(data.getMap(), null, true);
+      mapWriter.setPosition(idx());
+      writers.add(mapWriter);
+    }
+    return mapWriter;
+  }
+
+  public MapWriter asMap() {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    return getMapWriter();
+  }
+
+  private ListWriter getListWriter() {
+    if (listWriter == null) {
+      listWriter = new UnionListWriter(data.getList());
+      listWriter.setPosition(idx());
+      writers.add(listWriter);
+    }
+    return listWriter;
+  }
+
+  public ListWriter asList() {
+    data.getMutator().setType(idx(), MinorType.LIST);
+    return getListWriter();
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#assign uncappedName = name?uncap_first/>
+
+          <#if !minor.class?starts_with("Decimal")>
+
+  private ${name}Writer ${name?uncap_first}Writer;
+
+  private ${name}Writer get${name}Writer() {
+    if (${uncappedName}Writer == null) {
+      ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null);
+      ${uncappedName}Writer.setPosition(idx());
+      writers.add(${uncappedName}Writer);
+    }
+    return ${uncappedName}Writer;
+  }
+
+  public ${name}Writer as${name}() {
+    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    return get${name}Writer();
+  }
+
+  @Override
+  public void write(${name}Holder holder) {
+    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    get${name}Writer().setPosition(idx());
+    get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
+  }
+
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    get${name}Writer().setPosition(idx());
+    get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+  }
+  </#if>
+
+  </#list></#list>
+
+  public void writeNull() {
+  }
+
+  @Override
+  public MapWriter map() {
+    data.getMutator().setType(idx(), MinorType.LIST);
+    getListWriter().setPosition(idx());
+    return getListWriter().map();
+  }
+
+  @Override
+  public ListWriter list() {
+    data.getMutator().setType(idx(), MinorType.LIST);
+    getListWriter().setPosition(idx());
+    return getListWriter().list();
+  }
+
+  @Override
+  public ListWriter list(String name) {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().list(name);
+  }
+
+  @Override
+  public MapWriter map(String name) {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().map(name);
+  }
+
+  <#list vv.types as type><#list type.minor as minor>
+  <#assign lowerName = minor.class?uncap_first />
+  <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+  <#assign upperName = minor.class?upper_case />
+  <#assign capName = minor.class?cap_first />
+  <#if !minor.class?starts_with("Decimal")>
+  @Override
+  public ${capName}Writer ${lowerName}(String name) {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().${lowerName}(name);
+  }
+
+  @Override
+  public ${capName}Writer ${lowerName}() {
+    data.getMutator().setType(idx(), MinorType.LIST);
+    getListWriter().setPosition(idx());
+    return getListWriter().${lowerName}();
+  }
+  </#if>
+  </#list></#list>
+
+  @Override
+  public void allocate() {
+    data.allocateNew();
+  }
+
+  @Override
+  public void clear() {
+    data.clear();
+  }
+
+  @Override
+  public void close() throws Exception {
+    data.close();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return data.getField();
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return data.getValueCapacity();
+  }
+}
diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java
new file mode 100644
index 0000000000000..2b14194574a58
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ValueHolders.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+<#list vv.modes as mode>
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign className="${mode.prefix}${minor.class}Holder" />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/holders/${className}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.holders;
+
+<#include "/@includes/vv_imports.ftl" />
+
+public final class ${className} implements ValueHolder{
+  
+  public static final MajorType TYPE = new MajorType(MinorType.${minor.class?upper_case}, DataMode.${mode.name?upper_case});
+
+  public MajorType getType() {return TYPE;}
+  
+    <#if mode.name == "Repeated">
+    
+    /** The first index (inclusive) into the Vector. **/
+    public int start;
+    
+    /** The last index (exclusive) into the Vector. **/
+    public int end;
+    
+    /** The Vector holding the actual values. **/
+    public ${minor.class}Vector vector;
+    
+    <#else>
+    public static final int WIDTH = ${type.width};
+    
+    <#if mode.name == "Optional">public int isSet;</#if>
+    <#assign fields = minor.fields!type.fields />
+    <#list fields as field>
+    public ${field.type} ${field.name};
+    </#list>
+    
+    <#if minor.class.startsWith("Decimal")>
+    public static final int maxPrecision = ${minor.maxPrecisionDigits};
+    <#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")>
+    public static final int nDecimalDigits = ${minor.nDecimalDigits};
+    
+    public static int getInteger(int index, int start, ArrowBuf buffer) {
+      int value = buffer.getInt(start + (index * 4));
+
+      if (index == 0) {
+          /* the first byte contains sign bit, return value without it */
+          <#if minor.class.endsWith("Sparse")>
+          value = (value & 0x7FFFFFFF);
+          <#elseif minor.class.endsWith("Dense")>
+          value = (value & 0x0000007F);
+          </#if>
+      }
+      return value;
+    }
+
+    public static void setInteger(int index, int value, int start, ArrowBuf buffer) {
+        buffer.setInt(start + (index * 4), value);
+    }
+  
+    public static void setSign(boolean sign, int start, ArrowBuf buffer) {
+      // Set MSB to 1 if sign is negative
+      if (sign == true) {
+        int value = getInteger(0, start, buffer);
+        setInteger(0, (value | 0x80000000), start, buffer);
+      }
+    }
+  
+    public static boolean getSign(int start, ArrowBuf buffer) {
+      return ((buffer.getInt(start) & 0x80000000) != 0);
+    }
+    </#if></#if>
+    
+    @Deprecated
+    public int hashCode(){
+      throw new UnsupportedOperationException();
+    }
+
+    /*
+     * Reason for deprecation is that ValueHolders are potential scalar replacements
+     * and hence we don't want any methods to be invoked on them.
+     */
+    @Deprecated
+    public String toString(){
+      throw new UnsupportedOperationException();
+    }
+    </#if>
+    
+    
+    
+    
+}
+
+</#list>
+</#list>
+</#list>
\ No newline at end of file
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
new file mode 100644
index 0000000000000..13d53b8e846ab
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -0,0 +1,644 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+
+<#if type.major == "VarLen">
+<@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * ${minor.class}Vector implements a vector of variable width values.  Elements in the vector
+ * are accessed by position from the logical start of the vector.  A fixed width offsetVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * ArrowBuf.  Size is inferred by adjacent elements.
+ *   The width of each element is ${type.width} byte(s)
+ *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+  private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
+  private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
+  private static final int MIN_BYTE_COUNT = 4096;
+
+  public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+  private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
+  private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
+  private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
+
+  private final Accessor accessor;
+  private final Mutator mutator;
+
+  private final UInt${type.width}Vector.Accessor oAccessor;
+
+  private int allocationSizeInBytes = INITIAL_BYTE_COUNT;
+  private int allocationMonitor = 0;
+
+  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    this.oAccessor = offsetVector.getAccessor();
+    this.accessor = new Accessor();
+    this.mutator = new Mutator();
+  }
+
+  @Override
+  public FieldReader getReader(){
+    return reader;
+  }
+
+  @Override
+  public int getBufferSize(){
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return offsetVector.getBufferSize() + data.writerIndex();
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    final int idx = offsetVector.getAccessor().get(valueCount);
+    return offsetVector.getBufferSizeFor(valueCount + 1) + idx;
+  }
+
+  @Override
+  public int getValueCapacity(){
+    return Math.max(offsetVector.getValueCapacity() - 1, 0);
+  }
+
+  @Override
+  public int getByteCapacity(){
+    return data.capacity();
+  }
+
+  @Override
+  public int getCurrentSizeInBytes() {
+    return offsetVector.getAccessor().get(getAccessor().getValueCount());
+  }
+
+  /**
+   * Return the number of bytes contained in the current var len byte vector.
+   * @return
+   */
+  public int getVarByteLength(){
+    final int valueCount = getAccessor().getValueCount();
+    if(valueCount == 0) {
+      return 0;
+    }
+    return offsetVector.getAccessor().get(valueCount);
+  }
+
+//  @Override
+//  public SerializedField getMetadata() {
+//    return getMetadataBuilder() //
+//             .addChild(offsetVector.getMetadata())
+//             .setValueCount(getAccessor().getValueCount()) //
+//             .setBufferLength(getBufferSize()) //
+//             .build();
+//  }
+//
+//  @Override
+//  public void load(SerializedField metadata, ArrowBuf buffer) {
+//     the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
+//    final SerializedField offsetField = metadata.getChild(0);
+//    offsetVector.load(offsetField, buffer);
+//
+//    final int capacity = buffer.capacity();
+//    final int offsetsLength = offsetField.getBufferLength();
+//    data = buffer.slice(offsetsLength, capacity - offsetsLength);
+//    data.retain();
+//  }
+
+  @Override
+  public void clear() {
+    super.clear();
+    offsetVector.clear();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(false), super.getBuffers(false), ArrowBuf.class);
+    if (clear) {
+      // does not make much sense but we have to retain buffers even when clear is set. refactor this interface.
+      for (final ArrowBuf buffer:buffers) {
+        buffer.retain(1);
+      }
+      clear();
+    }
+    return buffers;
+  }
+
+  public long getOffsetAddr(){
+    return offsetVector.getBuffer().memoryAddress();
+  }
+
+  public UInt${type.width}Vector getOffsetVector(){
+    return offsetVector;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator){
+    return new TransferImpl(getField(), allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+    return new TransferImpl(getField().withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((${minor.class}Vector) to);
+  }
+
+  public void transferTo(${minor.class}Vector target){
+    target.clear();
+    this.offsetVector.transferTo(target.offsetVector);
+    target.data = data.transferOwnership(target.allocator).buffer;
+    target.data.writerIndex(data.writerIndex());
+    clear();
+  }
+
+  public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
+    UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor();
+    final int startPoint = offsetVectorAccessor.get(startIndex);
+    final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint;
+    target.clear();
+    target.offsetVector.allocateNew(length + 1);
+    offsetVectorAccessor = this.offsetVector.getAccessor();
+    final UInt4Vector.Mutator targetOffsetVectorMutator = target.offsetVector.getMutator();
+    for (int i = 0; i < length + 1; i++) {
+      targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
+    }
+    target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
+    target.getMutator().setValueCount(length);
+}
+
+  protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
+    final int start = fromOffsetVectorAccessor.get(fromIndex);
+    final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
+    final int len = end - start;
+
+    final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
+    from.data.getBytes(start, data, outputStart, len);
+    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
+  }
+
+  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
+    final int start = fromOffsetVectorAccessor.get(fromIndex);
+    final int end =   fromOffsetVectorAccessor.get(fromIndex + 1);
+    final int len = end - start;
+    final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
+
+    while(data.capacity() < outputStart + len) {
+        reAlloc();
+    }
+
+    offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len);
+    from.data.getBytes(start, data, outputStart, len);
+    return true;
+  }
+
+  private class TransferImpl implements TransferPair{
+    ${minor.class}Vector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator){
+      to = new ${minor.class}Vector(field, allocator);
+    }
+
+    public TransferImpl(${minor.class}Vector to){
+      this.to = to;
+    }
+
+    @Override
+    public ${minor.class}Vector getTo(){
+      return to;
+    }
+
+    @Override
+    public void transfer(){
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
+    }
+  }
+
+  @Override
+  public void setInitialCapacity(final int valueCount) {
+    final long size = 1L * valueCount * ${type.width};
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+    allocationSizeInBytes = (int)size;
+    offsetVector.setInitialCapacity(valueCount + 1);
+  }
+
+  @Override
+  public void allocateNew() {
+    if(!allocateNewSafe()){
+      throw new OutOfMemoryException("Failure while allocating buffer.");
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    long curAllocationSize = allocationSizeInBytes;
+    if (allocationMonitor > 10) {
+      curAllocationSize = Math.max(MIN_BYTE_COUNT, curAllocationSize / 2);
+      allocationMonitor = 0;
+    } else if (allocationMonitor < -2) {
+      curAllocationSize = curAllocationSize * 2L;
+      allocationMonitor = 0;
+    }
+
+    if (curAllocationSize > MAX_ALLOCATION_SIZE) {
+      return false;
+    }
+
+    clear();
+    /* Boolean to keep track if all the memory allocations were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    try {
+      final int requestedSize = (int)curAllocationSize;
+      data = allocator.buffer(requestedSize);
+      allocationSizeInBytes = requestedSize;
+      offsetVector.allocateNew();
+    } catch (OutOfMemoryException e) {
+      clear();
+      return false;
+    }
+    data.readerIndex(0);
+    offsetVector.zeroVector();
+    return true;
+  }
+
+  @Override
+  public void allocateNew(int totalBytes, int valueCount) {
+    clear();
+    assert totalBytes >= 0;
+    try {
+      data = allocator.buffer(totalBytes);
+      offsetVector.allocateNew(valueCount + 1);
+    } catch (RuntimeException e) {
+      clear();
+      throw e;
+    }
+    data.readerIndex(0);
+    allocationSizeInBytes = totalBytes;
+    offsetVector.zeroVector();
+  }
+
+  @Override
+  public void reset() {
+    allocationSizeInBytes = INITIAL_BYTE_COUNT;
+    allocationMonitor = 0;
+    data.readerIndex(0);
+    offsetVector.zeroVector();
+    super.reset();
+  }
+
+  public void reAlloc() {
+    final long newAllocationSize = allocationSizeInBytes*2L;
+    if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
+      throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    newBuf.setBytes(0, data, 0, data.capacity());
+    data.release();
+    data = newBuf;
+    allocationSizeInBytes = (int)newAllocationSize;
+  }
+
+  public void decrementAllocationMonitor() {
+    if (allocationMonitor > 0) {
+      allocationMonitor = 0;
+    }
+    --allocationMonitor;
+  }
+
+  private void incrementAllocationMonitor() {
+    ++allocationMonitor;
+  }
+
+  @Override
+  public Accessor getAccessor(){
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor {
+    final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
+    public long getStartEnd(int index){
+      return oAccessor.getTwoAsLong(index);
+    }
+
+    public byte[] get(int index) {
+      assert index >= 0;
+      final int startIdx = oAccessor.get(index);
+      final int length = oAccessor.get(index + 1) - startIdx;
+      assert length >= 0;
+      final byte[] dst = new byte[length];
+      data.getBytes(startIdx, dst, 0, length);
+      return dst;
+    }
+
+    @Override
+    public int getValueLength(int index) {
+      final UInt${type.width}Vector.Accessor offsetVectorAccessor = offsetVector.getAccessor();
+      return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index);
+    }
+
+    public void get(int index, ${minor.class}Holder holder){
+      holder.start = oAccessor.get(index);
+      holder.end = oAccessor.get(index + 1);
+      holder.buffer = data;
+    }
+
+    public void get(int index, Nullable${minor.class}Holder holder){
+      holder.isSet = 1;
+      holder.start = oAccessor.get(index);
+      holder.end = oAccessor.get(index + 1);
+      holder.buffer = data;
+    }
+
+
+    <#switch minor.class>
+    <#case "VarChar">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      Text text = new Text();
+      text.set(get(index));
+      return text;
+    }
+    <#break>
+    <#case "Var16Char">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      return new String(get(index), Charsets.UTF_16);
+    }
+    <#break>
+    <#default>
+    @Override
+    public ${friendlyType} getObject(int index) {
+      return get(index);
+    }
+    </#switch>
+
+    @Override
+    public int getValueCount() {
+      return Math.max(offsetVector.getAccessor().getValueCount()-1, 0);
+    }
+
+    @Override
+    public boolean isNull(int index){
+      return false;
+    }
+
+    public UInt${type.width}Vector getOffsetVector(){
+      return offsetVector;
+    }
+  }
+
+  /**
+   * Mutable${minor.class} implements a vector of variable width values.  Elements in the vector
+   * are accessed by position from the logical start of the vector.  A fixed width offsetVector
+   * is used to convert an element's position to it's offset from the start of the (0-based)
+   * ArrowBuf.  Size is inferred by adjacent elements.
+   *   The width of each element is ${type.width} byte(s)
+   *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+   *
+   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+   */
+  public final class Mutator extends BaseValueVector.BaseMutator implements VariableWidthVector.VariableWidthMutator {
+
+    /**
+     * Set the variable length element at the specified index to the supplied byte array.
+     *
+     * @param index   position of the bit to set
+     * @param bytes   array of bytes to write
+     */
+    protected void set(int index, byte[] bytes) {
+      assert index >= 0;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
+      data.setBytes(currentOffset, bytes, 0, bytes.length);
+    }
+
+    public void setSafe(int index, byte[] bytes) {
+      assert index >= 0;
+
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      while (data.capacity() < currentOffset + bytes.length) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length);
+      data.setBytes(currentOffset, bytes, 0, bytes.length);
+    }
+
+    /**
+     * Set the variable length element at the specified index to the supplied byte array.
+     *
+     * @param index   position of the bit to set
+     * @param bytes   array of bytes to write
+     * @param start   start index of bytes to write
+     * @param length  length of bytes to write
+     */
+    protected void set(int index, byte[] bytes, int start, int length) {
+      assert index >= 0;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, bytes, start, length);
+    }
+
+    public void setSafe(int index, ByteBuffer bytes, int start, int length) {
+      assert index >= 0;
+
+      int currentOffset = offsetVector.getAccessor().get(index);
+
+      while (data.capacity() < currentOffset + length) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, bytes, start, length);
+    }
+
+    public void setSafe(int index, byte[] bytes, int start, int length) {
+      assert index >= 0;
+
+      final int currentOffset = offsetVector.getAccessor().get(index);
+
+      while (data.capacity() < currentOffset + length) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, bytes, start, length);
+    }
+
+    @Override
+    public void setValueLengthSafe(int index, int length) {
+      final int offset = offsetVector.getAccessor().get(index);
+      while(data.capacity() < offset + length ) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length);
+    }
+
+
+    public void setSafe(int index, int start, int end, ArrowBuf buffer){
+      final int len = end - start;
+      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+      while(data.capacity() < outputStart + len) {
+        reAlloc();
+      }
+
+      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+      buffer.getBytes(start, data, outputStart, len);
+    }
+
+    public void setSafe(int index, Nullable${minor.class}Holder holder){
+      assert holder.isSet == 1;
+
+      final int start = holder.start;
+      final int end =   holder.end;
+      final int len = end - start;
+
+      int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+      while(data.capacity() < outputStart + len) {
+        reAlloc();
+      }
+
+      holder.buffer.getBytes(start, data, outputStart, len);
+      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+    }
+
+    public void setSafe(int index, ${minor.class}Holder holder){
+      final int start = holder.start;
+      final int end =   holder.end;
+      final int len = end - start;
+      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+      while(data.capacity() < outputStart + len) {
+        reAlloc();
+      }
+
+      holder.buffer.getBytes(start, data, outputStart, len);
+      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+    }
+
+    protected void set(int index, int start, int length, ArrowBuf buffer){
+      assert index >= 0;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      final ArrowBuf bb = buffer.slice(start, length);
+      data.setBytes(currentOffset, bb);
+    }
+
+    protected void set(int index, Nullable${minor.class}Holder holder){
+      final int length = holder.end - holder.start;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, holder.buffer, holder.start, length);
+    }
+
+    protected void set(int index, ${minor.class}Holder holder){
+      final int length = holder.end - holder.start;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, holder.buffer, holder.start, length);
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      final int currentByteCapacity = getByteCapacity();
+      final int idx = offsetVector.getAccessor().get(valueCount);
+      data.writerIndex(idx);
+      if (valueCount > 0 && currentByteCapacity > idx * 2) {
+        incrementAllocationMonitor();
+      } else if (allocationMonitor > 0) {
+        allocationMonitor = 0;
+      }
+      VectorTrimmer.trim(data, idx);
+      offsetVector.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+    }
+
+    @Override
+    public void generateTestData(int size){
+      boolean even = true;
+      <#switch minor.class>
+      <#case "Var16Char">
+      final java.nio.charset.Charset charset = Charsets.UTF_16;
+      <#break>
+      <#case "VarChar">
+      <#default>
+      final java.nio.charset.Charset charset = Charsets.UTF_8;
+      </#switch>
+      final byte[] evenValue = new String("aaaaa").getBytes(charset);
+      final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset);
+      for(int i =0; i < size; i++, even = !even){
+        set(i, even ? evenValue : oddValue);
+        }
+      setValueCount(size);
+    }
+  }
+}
+
+</#if> <#-- type.major -->
+</#list>
+</#list>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
new file mode 100644
index 0000000000000..388eb9c447977
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import com.google.common.base.Preconditions;
+
+public class AddOrGetResult<V extends ValueVector> {
+  private final V vector;
+  private final boolean created;
+
+  public AddOrGetResult(V vector, boolean created) {
+    this.vector = Preconditions.checkNotNull(vector);
+    this.created = created;
+  }
+
+  public V getVector() {
+    return vector;
+  }
+
+  public boolean isCreated() {
+    return created;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
new file mode 100644
index 0000000000000..54c3cd7331e0f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.vector.complex.RepeatedFixedWidthVectorLike;
+import org.apache.arrow.vector.complex.RepeatedVariableWidthVectorLike;
+
+public class AllocationHelper {
+//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class);
+
+  public static void allocate(ValueVector v, int valueCount, int bytesPerValue) {
+    allocate(v, valueCount, bytesPerValue, 5);
+  }
+
+  public static void allocatePrecomputedChildCount(ValueVector v, int valueCount, int bytesPerValue, int childValCount){
+    if(v instanceof FixedWidthVector) {
+      ((FixedWidthVector) v).allocateNew(valueCount);
+    } else if (v instanceof VariableWidthVector) {
+      ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount);
+    } else if(v instanceof RepeatedFixedWidthVectorLike) {
+      ((RepeatedFixedWidthVectorLike) v).allocateNew(valueCount, childValCount);
+    } else if(v instanceof RepeatedVariableWidthVectorLike) {
+      ((RepeatedVariableWidthVectorLike) v).allocateNew(childValCount * bytesPerValue, valueCount, childValCount);
+    } else {
+      v.allocateNew();
+    }
+  }
+
+  public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){
+    allocatePrecomputedChildCount(v, valueCount, bytesPerValue, repeatedPerTop * valueCount);
+  }
+
+  /**
+   * Allocates the exact amount if v is fixed width, otherwise falls back to dynamic allocation
+   * @param v value vector we are trying to allocate
+   * @param valueCount  size we are trying to allocate
+   * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the memory
+   */
+  public static void allocateNew(ValueVector v, int valueCount) {
+    if (v instanceof  FixedWidthVector) {
+      ((FixedWidthVector) v).allocateNew(valueCount);
+    } else {
+      v.allocateNew();
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
new file mode 100644
index 0000000000000..b129ea9bcb95f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.MaterializedField;
+
+
+public abstract class BaseDataValueVector extends BaseValueVector {
+
+  protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
+
+  protected ArrowBuf data;
+
+  public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    data = allocator.getEmpty();
+  }
+
+  @Override
+  public void clear() {
+    if (data != null) {
+      data.release();
+    }
+    data = allocator.getEmpty();
+    super.clear();
+  }
+
+  @Override
+  public void close() {
+    clear();
+    if (data != null) {
+      data.release();
+      data = null;
+    }
+    super.close();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    ArrowBuf[] out;
+    if (getBufferSize() == 0) {
+      out = new ArrowBuf[0];
+    } else {
+      out = new ArrowBuf[]{data};
+      data.readerIndex(0);
+      if (clear) {
+        data.retain(1);
+      }
+    }
+    if (clear) {
+      clear();
+    }
+    return out;
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return data.writerIndex();
+  }
+
+  public ArrowBuf getBuffer() {
+    return data;
+  }
+
+  /**
+   * This method has a similar effect of allocateNew() without actually clearing and reallocating
+   * the value vector. The purpose is to move the value vector to a "mutate" state
+   */
+  public void reset() {}
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
new file mode 100644
index 0000000000000..8bca3c005370e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Iterator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.TransferPair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class BaseValueVector implements ValueVector {
+  private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
+
+  public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
+  public static final int INITIAL_VALUE_ALLOCATION = 4096;
+
+  protected final BufferAllocator allocator;
+  protected final MaterializedField field;
+
+  protected BaseValueVector(MaterializedField field, BufferAllocator allocator) {
+    this.field = Preconditions.checkNotNull(field, "field cannot be null");
+    this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
+  }
+
+  @Override
+  public String toString() {
+    return super.toString() + "[field = " + field + ", ...]";
+  }
+
+  @Override
+  public void clear() {
+    getMutator().reset();
+  }
+
+  @Override
+  public void close() {
+    clear();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  public MaterializedField getField(String ref){
+    return getField().withPath(ref);
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return getTransferPair(getField().getPath(), allocator);
+  }
+
+//  public static SerializedField getMetadata(BaseValueVector vector) {
+//    return getMetadataBuilder(vector).build();
+//  }
+//
+//  protected static SerializedField.Builder getMetadataBuilder(BaseValueVector vector) {
+//    return SerializedFieldHelper.getAsBuilder(vector.getField())
+//        .setValueCount(vector.getAccessor().getValueCount())
+//        .setBufferLength(vector.getBufferSize());
+//  }
+
+  public abstract static class BaseAccessor implements ValueVector.Accessor {
+    protected BaseAccessor() { }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+  }
+
+  public abstract static class BaseMutator implements ValueVector.Mutator {
+    protected BaseMutator() { }
+
+    @Override
+    public void generateTestData(int values) {}
+
+    //TODO: consider making mutator stateless(if possible) on another issue.
+    public void reset() {}
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return Iterators.emptyIterator();
+  }
+
+  public static boolean checkBufRefs(final ValueVector vv) {
+    for(final ArrowBuf buffer : vv.getBuffers(false)) {
+      if (buffer.refCnt() <= 0) {
+        throw new IllegalStateException("zero refcount");
+      }
+    }
+
+    return true;
+  }
+
+  @Override
+  public BufferAllocator getAllocator() {
+    return allocator;
+  }
+}
+
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
new file mode 100644
index 0000000000000..952e9028e0668
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -0,0 +1,450 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.impl.BitReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.BitHolder;
+import org.apache.arrow.vector.holders.NullableBitHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start
+ * of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0'
+ * or '1'.
+ */
+public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
+
+  private final FieldReader reader = new BitReaderImpl(BitVector.this);
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+
+  private int valueCount;
+  private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+  private int allocationMonitor = 0;
+
+  public BitVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public int getBufferSize() {
+    return getSizeFromCount(valueCount);
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    return getSizeFromCount(valueCount);
+  }
+
+  private int getSizeFromCount(int valueCount) {
+    return (int) Math.ceil(valueCount / 8.0);
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return (int)Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L);
+  }
+
+  private int getByteIndex(int index) {
+    return (int) Math.floor(index / 8.0);
+  }
+
+  @Override
+  public void setInitialCapacity(final int valueCount) {
+    allocationSizeInBytes = getSizeFromCount(valueCount);
+  }
+
+  @Override
+  public void allocateNew() {
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryException();
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    long curAllocationSize = allocationSizeInBytes;
+    if (allocationMonitor > 10) {
+      curAllocationSize = Math.max(8, allocationSizeInBytes / 2);
+      allocationMonitor = 0;
+    } else if (allocationMonitor < -2) {
+      curAllocationSize = allocationSizeInBytes * 2L;
+      allocationMonitor = 0;
+    }
+
+    try {
+      allocateBytes(curAllocationSize);
+    } catch (OutOfMemoryException ex) {
+      return false;
+    }
+    return true;
+  }
+
+  @Override
+  public void reset() {
+    valueCount = 0;
+    allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+    allocationMonitor = 0;
+    zeroVector();
+    super.reset();
+  }
+
+  /**
+   * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+   *
+   * @param valueCount
+   *          The number of values which can be contained within this vector.
+   */
+  @Override
+  public void allocateNew(int valueCount) {
+    final int size = getSizeFromCount(valueCount);
+    allocateBytes(size);
+  }
+
+  private void allocateBytes(final long size) {
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+
+    final int curSize = (int) size;
+    clear();
+    data = allocator.buffer(curSize);
+    zeroVector();
+    allocationSizeInBytes = curSize;
+  }
+
+  /**
+   * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+   */
+  public void reAlloc() {
+    final long newAllocationSize = allocationSizeInBytes * 2L;
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+
+    final int curSize = (int)newAllocationSize;
+    final ArrowBuf newBuf = allocator.buffer(curSize);
+    newBuf.setZero(0, newBuf.capacity());
+    newBuf.setBytes(0, data, 0, data.capacity());
+    data.release();
+    data = newBuf;
+    allocationSizeInBytes = curSize;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void zeroVector() {
+    data.setZero(0, data.capacity());
+  }
+
+  public void copyFrom(int inIndex, int outIndex, BitVector from) {
+    this.mutator.set(outIndex, from.accessor.get(inIndex));
+  }
+
+  public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) {
+    if (outIndex >= this.getValueCapacity()) {
+      decrementAllocationMonitor();
+      return false;
+    }
+    copyFrom(inIndex, outIndex, from);
+    return true;
+  }
+
+//  @Override
+//  public void load(SerializedField metadata, DrillBuf buffer) {
+//    Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
+//    final int valueCount = metadata.getValueCount();
+//    final int expectedLength = getSizeFromCount(valueCount);
+//    final int actualLength = metadata.getBufferLength();
+//    assert expectedLength == actualLength: "expected and actual buffer sizes do not match";
+//
+//    clear();
+//    data = buffer.slice(0, actualLength);
+//    data.retain();
+//    this.valueCount = valueCount;
+//  }
+
+  @Override
+  public Mutator getMutator() {
+    return new Mutator();
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return new Accessor();
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new TransferImpl(getField(), allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(getField().withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((BitVector) to);
+  }
+
+
+  public void transferTo(BitVector target) {
+    target.clear();
+    if (target.data != null) {
+      target.data.release();
+    }
+    target.data = data;
+    target.data.retain(1);
+    target.valueCount = valueCount;
+    clear();
+  }
+
+  public void splitAndTransferTo(int startIndex, int length, BitVector target) {
+    assert startIndex + length <= valueCount;
+    int firstByte = getByteIndex(startIndex);
+    int byteSize = getSizeFromCount(length);
+    int offset = startIndex % 8;
+    if (offset == 0) {
+      target.clear();
+      // slice
+      if (target.data != null) {
+        target.data.release();
+      }
+      target.data = (ArrowBuf) data.slice(firstByte, byteSize);
+      target.data.retain(1);
+    } else {
+      // Copy data
+      // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
+      // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
+      // The last byte copied to target is a bit tricky :
+      //   1) if length requires partly byte (length % 8 !=0), copy the remaining bits only.
+      //   2) otherwise, copy the last byte in the same way as to the prior bytes.
+      target.clear();
+      target.allocateNew(length);
+      // TODO maybe do this one word at a time, rather than byte?
+      for(int i = 0; i < byteSize - 1; i++) {
+        target.data.setByte(i, (((this.data.getByte(firstByte + i) & 0xFF) >>> offset) + (this.data.getByte(firstByte + i + 1) <<  (8 - offset))));
+      }
+      if (length % 8 != 0) {
+        target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
+      } else {
+        target.data.setByte(byteSize - 1,
+            (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) <<  (8 - offset))));
+      }
+    }
+    target.getMutator().setValueCount(length);
+  }
+
+  private class TransferImpl implements TransferPair {
+    BitVector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
+      this.to = new BitVector(field, allocator);
+    }
+
+    public TransferImpl(BitVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public BitVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, BitVector.this);
+    }
+  }
+
+  private void decrementAllocationMonitor() {
+    if (allocationMonitor > 0) {
+      allocationMonitor = 0;
+    }
+    --allocationMonitor;
+  }
+
+  private void incrementAllocationMonitor() {
+    ++allocationMonitor;
+  }
+
+  public class Accessor extends BaseAccessor {
+
+    /**
+     * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the bit was not set.
+     *
+     * @param index
+     *          position of the bit in the vector
+     * @return 1 if set, otherwise 0
+     */
+    public final int get(int index) {
+      int byteIndex = index >> 3;
+      byte b = data.getByte(byteIndex);
+      int bitIndex = index & 7;
+      return Long.bitCount(b &  (1L << bitIndex));
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+
+    @Override
+    public final Boolean getObject(int index) {
+      return new Boolean(get(index) != 0);
+    }
+
+    @Override
+    public final int getValueCount() {
+      return valueCount;
+    }
+
+    public final void get(int index, BitHolder holder) {
+      holder.value = get(index);
+    }
+
+    public final void get(int index, NullableBitHolder holder) {
+      holder.isSet = 1;
+      holder.value = get(index);
+    }
+  }
+
+  /**
+   * MutableBit implements a vector of bit-width values. Elements in the vector are accessed by position from the
+   * logical start of the vector. Values should be pushed onto the vector sequentially, but may be randomly accessed.
+   *
+   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+   */
+  public class Mutator extends BaseMutator {
+
+    private Mutator() {
+    }
+
+    /**
+     * Set the bit at the given index to the specified value.
+     *
+     * @param index
+     *          position of the bit to set
+     * @param value
+     *          value to set (either 1 or 0)
+     */
+    public final void set(int index, int value) {
+      int byteIndex = index >> 3;
+      int bitIndex = index & 7;
+      byte currentByte = data.getByte(byteIndex);
+      byte bitMask = (byte) (1L << bitIndex);
+      if (value != 0) {
+        currentByte |= bitMask;
+      } else {
+        currentByte -= (bitMask & currentByte);
+      }
+
+      data.setByte(byteIndex, currentByte);
+    }
+
+    public final void set(int index, BitHolder holder) {
+      set(index, holder.value);
+    }
+
+    final void set(int index, NullableBitHolder holder) {
+      set(index, holder.value);
+    }
+
+    public void setSafe(int index, int value) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, value);
+    }
+
+    public void setSafe(int index, BitHolder holder) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, holder.value);
+    }
+
+    public void setSafe(int index, NullableBitHolder holder) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, holder.value);
+    }
+
+    @Override
+    public final void setValueCount(int valueCount) {
+      int currentValueCapacity = getValueCapacity();
+      BitVector.this.valueCount = valueCount;
+      int idx = getSizeFromCount(valueCount);
+      while(valueCount > getValueCapacity()) {
+        reAlloc();
+      }
+      if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
+        incrementAllocationMonitor();
+      } else if (allocationMonitor > 0) {
+        allocationMonitor = 0;
+      }
+      VectorTrimmer.trim(data, idx);
+    }
+
+    @Override
+    public final void generateTestData(int values) {
+      boolean even = true;
+      for(int i = 0; i < values; i++, even = !even) {
+        if (even) {
+          set(i, 1);
+        }
+      }
+      setValueCount(values);
+    }
+
+  }
+
+  @Override
+  public void clear() {
+    this.valueCount = 0;
+    super.clear();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
new file mode 100644
index 0000000000000..59057000bbca9
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+
+public interface FixedWidthVector extends ValueVector{
+
+  /**
+   * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
+   *
+   * @param valueCount   Number of values in the vector.
+   */
+  void allocateNew(int valueCount);
+
+/**
+ * Zero out the underlying buffer backing this vector.
+ */
+  void zeroVector();
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
new file mode 100644
index 0000000000000..00c33fc2d6e6c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+public interface NullableVector extends ValueVector{
+
+  ValueVector getValuesVector();
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
new file mode 100644
index 0000000000000..b819c5d39e99c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+public interface NullableVectorDefinitionSetter {
+
+  public void setIndexDefined(int index);
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
new file mode 100644
index 0000000000000..b806b180e7014
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ObjectHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class ObjectVector extends BaseValueVector {
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+  private int maxCount = 0;
+  private int count = 0;
+  private int allocationSize = 4096;
+
+  private List<Object[]> objectArrayList = new ArrayList<>();
+
+  public ObjectVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  public void addNewArray() {
+    objectArrayList.add(new Object[allocationSize]);
+    maxCount += allocationSize;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  public final class Mutator implements ValueVector.Mutator {
+
+    public void set(int index, Object obj) {
+      int listOffset = index / allocationSize;
+      if (listOffset >= objectArrayList.size()) {
+        addNewArray();
+      }
+      objectArrayList.get(listOffset)[index % allocationSize] = obj;
+    }
+
+    public boolean setSafe(int index, long value) {
+      set(index, value);
+      return true;
+    }
+
+    protected void set(int index, ObjectHolder holder) {
+      set(index, holder.obj);
+    }
+
+    public boolean setSafe(int index, ObjectHolder holder){
+      set(index, holder);
+      return true;
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      count = valueCount;
+    }
+
+    @Override
+    public void reset() {
+      count = 0;
+      maxCount = 0;
+      objectArrayList = new ArrayList<>();
+      addNewArray();
+    }
+
+    @Override
+    public void generateTestData(int values) {
+    }
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    // NoOp
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    addNewArray();
+  }
+
+  public void allocateNew(int valueCount) throws OutOfMemoryException {
+    while (maxCount < valueCount) {
+      addNewArray();
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    allocateNew();
+    return true;
+  }
+
+  @Override
+  public int getBufferSize() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public void close() {
+    clear();
+  }
+
+  @Override
+  public void clear() {
+    objectArrayList.clear();
+    maxCount = 0;
+    count = 0;
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return maxCount;
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+//  @Override
+//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+//    throw new UnsupportedOperationException("ObjectVector does not support this");
+//  }
+//
+//  @Override
+//  public UserBitShared.SerializedField getMetadata() {
+//    throw new UnsupportedOperationException("ObjectVector does not support this");
+//  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  public final class Accessor extends BaseAccessor {
+    @Override
+    public Object getObject(int index) {
+      int listOffset = index / allocationSize;
+      if (listOffset >= objectArrayList.size()) {
+        addNewArray();
+      }
+      return objectArrayList.get(listOffset)[index % allocationSize];
+    }
+
+    @Override
+    public int getValueCount() {
+      return count;
+    }
+
+    public Object get(int index) {
+      return getObject(index);
+    }
+
+    public void get(int index, ObjectHolder holder){
+      holder.obj = getObject(index);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java b/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
new file mode 100644
index 0000000000000..fc0a066749a91
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.vector.util.CallBack;
+
+
+public class SchemaChangeCallBack implements CallBack {
+  private boolean schemaChanged = false;
+
+  /**
+   * Constructs a schema-change callback with the schema-changed state set to
+   * {@code false}.
+   */
+  public SchemaChangeCallBack() {
+  }
+
+  /**
+   * Sets the schema-changed state to {@code true}.
+   */
+  @Override
+  public void doWork() {
+    schemaChanged = true;
+  }
+
+  /**
+   * Returns the value of schema-changed state, <strong>resetting</strong> the
+   * schema-changed state to {@code false}.
+   */
+  public boolean getSchemaChangedAndReset() {
+    final boolean current = schemaChanged;
+    schemaChanged = false;
+    return current;
+  }
+}
+
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
new file mode 100644
index 0000000000000..61ce285d61b0c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.math.BigDecimal;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.holders.BigIntHolder;
+import org.apache.arrow.vector.holders.BitHolder;
+import org.apache.arrow.vector.holders.DateHolder;
+import org.apache.arrow.vector.holders.Decimal18Holder;
+import org.apache.arrow.vector.holders.Decimal28SparseHolder;
+import org.apache.arrow.vector.holders.Decimal38SparseHolder;
+import org.apache.arrow.vector.holders.Decimal9Holder;
+import org.apache.arrow.vector.holders.Float4Holder;
+import org.apache.arrow.vector.holders.Float8Holder;
+import org.apache.arrow.vector.holders.IntHolder;
+import org.apache.arrow.vector.holders.IntervalDayHolder;
+import org.apache.arrow.vector.holders.IntervalYearHolder;
+import org.apache.arrow.vector.holders.NullableBitHolder;
+import org.apache.arrow.vector.holders.TimeHolder;
+import org.apache.arrow.vector.holders.TimeStampHolder;
+import org.apache.arrow.vector.holders.VarCharHolder;
+import org.apache.arrow.vector.util.DecimalUtility;
+
+import com.google.common.base.Charsets;
+
+
+public class ValueHolderHelper {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
+
+  public static IntHolder getIntHolder(int value) {
+    IntHolder holder = new IntHolder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static BigIntHolder getBigIntHolder(long value) {
+    BigIntHolder holder = new BigIntHolder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static Float4Holder getFloat4Holder(float value) {
+    Float4Holder holder = new Float4Holder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static Float8Holder getFloat8Holder(double value) {
+    Float8Holder holder = new Float8Holder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static DateHolder getDateHolder(long value) {
+    DateHolder holder = new DateHolder();
+    holder.value = value;
+    return holder;
+  }
+
+  public static TimeHolder getTimeHolder(int value) {
+    TimeHolder holder = new TimeHolder();
+    holder.value = value;
+    return holder;
+  }
+
+  public static TimeStampHolder getTimeStampHolder(long value) {
+    TimeStampHolder holder = new TimeStampHolder();
+    holder.value = value;
+    return holder;
+  }
+
+  public static BitHolder getBitHolder(int value) {
+    BitHolder holder = new BitHolder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static NullableBitHolder getNullableBitHolder(boolean isNull, int value) {
+    NullableBitHolder holder = new NullableBitHolder();
+    holder.isSet = isNull? 0 : 1;
+    if (! isNull) {
+      holder.value = value;
+    }
+
+    return holder;
+  }
+
+  public static VarCharHolder getVarCharHolder(ArrowBuf buf, String s){
+    VarCharHolder vch = new VarCharHolder();
+
+    byte[] b = s.getBytes(Charsets.UTF_8);
+    vch.start = 0;
+    vch.end = b.length;
+    vch.buffer = buf.reallocIfNeeded(b.length);
+    vch.buffer.setBytes(0, b);
+    return vch;
+  }
+
+  public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
+    VarCharHolder vch = new VarCharHolder();
+
+    byte[] b = s.getBytes(Charsets.UTF_8);
+    vch.start = 0;
+    vch.end = b.length;
+    vch.buffer = a.buffer(b.length); //
+    vch.buffer.setBytes(0, b);
+    return vch;
+  }
+
+
+  public static IntervalYearHolder getIntervalYearHolder(int intervalYear) {
+    IntervalYearHolder holder = new IntervalYearHolder();
+
+    holder.value = intervalYear;
+    return holder;
+  }
+
+  public static IntervalDayHolder getIntervalDayHolder(int days, int millis) {
+      IntervalDayHolder dch = new IntervalDayHolder();
+
+      dch.days = days;
+      dch.milliseconds = millis;
+      return dch;
+  }
+
+  public static Decimal9Holder getDecimal9Holder(int decimal, int scale, int precision) {
+    Decimal9Holder dch = new Decimal9Holder();
+
+    dch.scale = scale;
+    dch.precision = precision;
+    dch.value = decimal;
+
+    return dch;
+  }
+
+  public static Decimal18Holder getDecimal18Holder(long decimal, int scale, int precision) {
+    Decimal18Holder dch = new Decimal18Holder();
+
+    dch.scale = scale;
+    dch.precision = precision;
+    dch.value = decimal;
+
+    return dch;
+  }
+
+  public static Decimal28SparseHolder getDecimal28Holder(ArrowBuf buf, String decimal) {
+
+    Decimal28SparseHolder dch = new Decimal28SparseHolder();
+
+    BigDecimal bigDecimal = new BigDecimal(decimal);
+
+    dch.scale = bigDecimal.scale();
+    dch.precision = bigDecimal.precision();
+    Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
+    dch.start = 0;
+    dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE);
+    DecimalUtility
+        .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+
+    return dch;
+  }
+
+  public static Decimal38SparseHolder getDecimal38Holder(ArrowBuf buf, String decimal) {
+
+      Decimal38SparseHolder dch = new Decimal38SparseHolder();
+
+      BigDecimal bigDecimal = new BigDecimal(decimal);
+
+      dch.scale = bigDecimal.scale();
+      dch.precision = bigDecimal.precision();
+      Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
+      dch.start = 0;
+    dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE);
+    DecimalUtility
+        .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+
+      return dch;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
new file mode 100644
index 0000000000000..c05f0e7c50fd2
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.io.Closeable;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * An abstraction that is used to store a sequence of values in an individual column.
+ *
+ * A {@link ValueVector value vector} stores underlying data in-memory in a columnar fashion that is compact and
+ * efficient. The column whose data is stored, is referred by {@link #getField()}.
+ *
+ * A vector when instantiated, relies on a {@link org.apache.drill.exec.record.DeadBuf dead buffer}. It is important
+ * that vector is allocated before attempting to read or write.
+ *
+ * There are a few "rules" around vectors:
+ *
+ * <ul>
+ *   <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
+ *   <li>null vectors start with all values as null before writing anything</li>
+ *   <li>for variable width types, the offset vector should be all zeros before writing</li>
+ *   <li>you must call setValueCount before a vector can be read</li>
+ *   <li>you should never write to a vector once it has been read.</li>
+ * </ul>
+ *
+ * Please note that the current implementation doesn't enfore those rules, hence we may find few places that
+ * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)
+ *
+ * This interface "should" strive to guarantee this order of operation:
+ * <blockquote>
+ * allocate > mutate > setvaluecount > access > clear (or allocate to start the process over).
+ * </blockquote>
+ */
+public interface ValueVector extends Closeable, Iterable<ValueVector> {
+  /**
+   * Allocate new buffers. ValueVector implements logic to determine how much to allocate.
+   * @throws OutOfMemoryException Thrown if no memory can be allocated.
+   */
+  void allocateNew() throws OutOfMemoryException;
+
+  /**
+   * Allocates new buffers. ValueVector implements logic to determine how much to allocate.
+   * @return Returns true if allocation was succesful.
+   */
+  boolean allocateNewSafe();
+
+  BufferAllocator getAllocator();
+
+  /**
+   * Set the initial record capacity
+   * @param numRecords
+   */
+  void setInitialCapacity(int numRecords);
+
+  /**
+   * Returns the maximum number of values that can be stored in this vector instance.
+   */
+  int getValueCapacity();
+
+  /**
+   * Alternative to clear(). Allows use as an AutoCloseable in try-with-resources.
+   */
+  @Override
+  void close();
+
+  /**
+   * Release the underlying DrillBuf and reset the ValueVector to empty.
+   */
+  void clear();
+
+  /**
+   * Get information about how this field is materialized.
+   */
+  MaterializedField getField();
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
+   * the same type.
+   */
+  TransferPair getTransferPair(BufferAllocator allocator);
+
+  TransferPair getTransferPair(String ref, BufferAllocator allocator);
+
+  /**
+   * Returns a new {@link org.apache.arrow.vector.util.TransferPair transfer pair} that is used to transfer underlying
+   * buffers into the target vector.
+   */
+  TransferPair makeTransferPair(ValueVector target);
+
+  /**
+   * Returns an {@link org.apache.arrow.vector.ValueVector.Accessor accessor} that is used to read from this vector
+   * instance.
+   */
+  Accessor getAccessor();
+
+  /**
+   * Returns an {@link org.apache.arrow.vector.ValueVector.Mutator mutator} that is used to write to this vector
+   * instance.
+   */
+  Mutator getMutator();
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.complex.reader.FieldReader field reader} that supports reading values
+   * from this vector.
+   */
+  FieldReader getReader();
+
+  /**
+   * Get the metadata for this field. Used in serialization
+   *
+   * @return FieldMetadata for this field.
+   */
+//  SerializedField getMetadata();
+
+  /**
+   * Returns the number of bytes that is used by this vector instance.
+   */
+  int getBufferSize();
+
+  /**
+   * Returns the number of bytes that is used by this vector if it holds the given number
+   * of values. The result will be the same as if Mutator.setValueCount() were called, followed
+   * by calling getBufferSize(), but without any of the closing side-effects that setValueCount()
+   * implies wrt finishing off the population of a vector. Some operations might wish to use
+   * this to determine how much memory has been used by a vector so far, even though it is
+   * not finished being populated.
+   *
+   * @param valueCount the number of values to assume this vector contains
+   * @return the buffer size if this vector is holding valueCount values
+   */
+  int getBufferSizeFor(int valueCount);
+
+  /**
+   * Return the underlying buffers associated with this vector. Note that this doesn't impact the reference counts for
+   * this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus
+   * external classes shouldn't hold a reference to it (unless they change it).
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted;
+   *   but the returned array will be the only reference to them
+   *
+   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this vector instance.
+   */
+  ArrowBuf[] getBuffers(boolean clear);
+
+  /**
+   * Load the data provided in the buffer. Typically used when deserializing from the wire.
+   *
+   * @param metadata
+   *          Metadata used to decode the incoming buffer.
+   * @param buffer
+   *          The buffer that contains the ValueVector.
+   */
+//  void load(SerializedField metadata, DrillBuf buffer);
+
+  /**
+   * An abstraction that is used to read from this vector instance.
+   */
+  interface Accessor {
+    /**
+     * Get the Java Object representation of the element at the specified position. Useful for testing.
+     *
+     * @param index
+     *          Index of the value to get
+     */
+    Object getObject(int index);
+
+    /**
+     * Returns the number of values that is stored in this vector.
+     */
+    int getValueCount();
+
+    /**
+     * Returns true if the value at the given index is null, false otherwise.
+     */
+    boolean isNull(int index);
+  }
+
+  /**
+   * An abstractiong that is used to write into this vector instance.
+   */
+  interface Mutator {
+    /**
+     * Sets the number of values that is stored in this vector to the given value count.
+     *
+     * @param valueCount  value count to set.
+     */
+    void setValueCount(int valueCount);
+
+    /**
+     * Resets the mutator to pristine state.
+     */
+    void reset();
+
+    /**
+     * @deprecated  this has nothing to do with value vector abstraction and should be removed.
+     */
+    @Deprecated
+    void generateTestData(int values);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
new file mode 100644
index 0000000000000..e227bb4c4176c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+public interface VariableWidthVector extends ValueVector{
+
+  /**
+   * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
+   *
+   * @param totalBytes   Desired size of the underlying data buffer.
+   * @param valueCount   Number of values in the vector.
+   */
+  void allocateNew(int totalBytes, int valueCount);
+
+  /**
+   * Provide the maximum amount of variable width bytes that can be stored int his vector.
+   * @return
+   */
+  int getByteCapacity();
+
+  VariableWidthMutator getMutator();
+
+  VariableWidthAccessor getAccessor();
+
+  interface VariableWidthAccessor extends Accessor {
+    int getValueLength(int index);
+  }
+
+  int getCurrentSizeInBytes();
+
+  interface VariableWidthMutator extends Mutator {
+    void setValueLengthSafe(int index, int length);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
new file mode 100644
index 0000000000000..fdad99a333258
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.util.Collection;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MajorType;
+
+public class VectorDescriptor {
+  private static final String DEFAULT_NAME = "NONE";
+
+  private final MaterializedField field;
+
+  public VectorDescriptor(final MajorType type) {
+    this(DEFAULT_NAME, type);
+  }
+
+  public VectorDescriptor(final String name, final MajorType type) {
+    this(MaterializedField.create(name, type));
+  }
+
+  public VectorDescriptor(final MaterializedField field) {
+    this.field = Preconditions.checkNotNull(field, "field cannot be null");
+  }
+
+  public MaterializedField getField() {
+    return field;
+  }
+
+  public MajorType getType() {
+    return field.getType();
+  }
+
+  public String getName() {
+    return field.getLastName();
+  }
+
+  public Collection<MaterializedField> getChildren() {
+    return field.getChildren();
+  }
+
+  public boolean hasName() {
+    return getName() != DEFAULT_NAME;
+  }
+
+  public VectorDescriptor withName(final String name) {
+    return new VectorDescriptor(field.withPath(name));
+  }
+
+  public VectorDescriptor withType(final MajorType type) {
+    return new VectorDescriptor(field.withType(type));
+  }
+
+  public static VectorDescriptor create(final String name, final MajorType type) {
+    return new VectorDescriptor(name, type);
+  }
+
+  public static VectorDescriptor create(final MajorType type) {
+    return new VectorDescriptor(type);
+  }
+
+  public static VectorDescriptor create(final MaterializedField field) {
+    return new VectorDescriptor(field);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
new file mode 100644
index 0000000000000..055857e956084
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ArrowBuf;
+
+public class VectorTrimmer {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorTrimmer.class);
+
+  public static void trim(ByteBuf data, int idx) {
+    data.writerIndex(idx);
+    if (data instanceof ArrowBuf) {
+      // data.capacity(idx);
+      data.writerIndex(idx);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
new file mode 100644
index 0000000000000..78de8706fb7d4
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Iterator;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.impl.NullReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.collect.Iterators;
+
+public class ZeroVector implements ValueVector {
+  public final static ZeroVector INSTANCE = new ZeroVector();
+
+  private final MaterializedField field = MaterializedField.create("[DEFAULT]", Types.required(MinorType.LATE));
+
+  private final TransferPair defaultPair = new TransferPair() {
+    @Override
+    public void transfer() { }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) { }
+
+    @Override
+    public ValueVector getTo() {
+      return ZeroVector.this;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) { }
+  };
+
+  private final Accessor defaultAccessor = new Accessor() {
+    @Override
+    public Object getObject(int index) {
+      return null;
+    }
+
+    @Override
+    public int getValueCount() {
+      return 0;
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return true;
+    }
+  };
+
+  private final Mutator defaultMutator = new Mutator() {
+    @Override
+    public void setValueCount(int valueCount) { }
+
+    @Override
+    public void reset() { }
+
+    @Override
+    public void generateTestData(int values) { }
+  };
+
+  public ZeroVector() { }
+
+  @Override
+  public void close() { }
+
+  @Override
+  public void clear() { }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return defaultPair;
+  }
+
+//  @Override
+//  public UserBitShared.SerializedField getMetadata() {
+//    return getField()
+//        .getAsBuilder()
+//        .setBufferLength(getBufferSize())
+//        .setValueCount(getAccessor().getValueCount())
+//        .build();
+//  }
+
+  @Override
+  public Iterator iterator() {
+    return Iterators.emptyIterator();
+  }
+
+  @Override
+  public int getBufferSize() {
+    return 0;
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    return 0;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    return new ArrowBuf[0];
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    allocateNewSafe();
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    return true;
+  }
+
+  @Override
+  public BufferAllocator getAllocator() {
+    throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) { }
+
+  @Override
+  public int getValueCapacity() {
+    return 0;
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return defaultPair;
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector target) {
+    return defaultPair;
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return defaultAccessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return defaultMutator;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return NullReader.INSTANCE;
+  }
+
+//  @Override
+//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
new file mode 100644
index 0000000000000..c671c9e0b3c55
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import java.util.Collection;
+
+import javax.annotation.Nullable;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+/**
+ * Base class for composite vectors.
+ *
+ * This class implements common functionality of composite vectors.
+ */
+public abstract class AbstractContainerVector implements ValueVector {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
+
+  protected MaterializedField field;
+  protected final BufferAllocator allocator;
+  protected final CallBack callBack;
+
+  protected AbstractContainerVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    this.field = Preconditions.checkNotNull(field);
+    this.allocator = allocator;
+    this.callBack = callBack;
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryException();
+    }
+  }
+
+  public BufferAllocator getAllocator() {
+    return allocator;
+  }
+
+  /**
+   * Returns the field definition of this instance.
+   */
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
+   */
+  public ValueVector getChild(String name) {
+    return getChild(name, ValueVector.class);
+  }
+
+  /**
+   * Returns a sequence of field names in the order that they show up in the schema.
+   */
+  protected Collection<String> getChildFieldNames() {
+    return Sets.newLinkedHashSet(Iterables.transform(field.getChildren(), new Function<MaterializedField, String>() {
+      @Nullable
+      @Override
+      public String apply(MaterializedField field) {
+        return Preconditions.checkNotNull(field).getLastName();
+      }
+    }));
+  }
+
+  /**
+   * Clears out all underlying child vectors.
+   */
+ @Override
+  public void close() {
+    for (ValueVector vector:(Iterable<ValueVector>)this) {
+      vector.close();
+    }
+  }
+
+  protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
+    if (clazz.isAssignableFrom(v.getClass())) {
+      return (T) v;
+    }
+    throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s].  Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
+  }
+
+  MajorType getLastPathType() {
+    if((this.getField().getType().getMinorType() == MinorType.LIST  &&
+        this.getField().getType().getMode() == DataMode.REPEATED)) {  // Use Repeated scalar type instead of Required List.
+      VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
+      ValueVector v = vord.vector;
+      if (! (v instanceof  AbstractContainerVector)) {
+        return v.getField().getType();
+      }
+    } else if (this.getField().getType().getMinorType() == MinorType.MAP  &&
+        this.getField().getType().getMode() == DataMode.REPEATED) {  // Use Required Map
+      return new MajorType(MinorType.MAP, DataMode.REQUIRED);
+    }
+
+    return this.getField().getType();
+  }
+
+  protected boolean supportsDirectRead() {
+    return false;
+  }
+
+  // return the number of child vectors
+  public abstract int size();
+
+  // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
+  public abstract <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz);
+
+  // return the child vector with the input name
+  public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
+
+  // return the child vector's ordinal in the composite container
+  public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
new file mode 100644
index 0000000000000..d4189b2314a6a
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -0,0 +1,278 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.MapWithOrdinal;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/*
+ * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
+ */
+public abstract class AbstractMapVector extends AbstractContainerVector {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
+
+  // Maintains a map with key as field name and value is the vector itself
+  private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
+
+  protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    super(field.clone(), allocator, callBack);
+    MaterializedField clonedField = field.clone();
+    // create the hierarchy of the child vectors based on the materialized field
+    for (MaterializedField child : clonedField.getChildren()) {
+      if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
+        final String fieldName = child.getLastName();
+        final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
+        putVector(fieldName, v);
+      }
+    }
+  }
+
+  @Override
+  public void close() {
+    for(final ValueVector valueVector : vectors.values()) {
+      valueVector.close();
+    }
+    vectors.clear();
+
+    super.close();
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      for (final ValueVector v : vectors.values()) {
+        if (!v.allocateNewSafe()) {
+          return false;
+        }
+      }
+      success = true;
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
+   * {@link org.apache.arrow.vector.ValueVector}.
+   *
+   * Execution takes place in the following order:
+   * <ul>
+   *   <li>
+   *     if field is new, create and insert a new vector of desired type.
+   *   </li>
+   *   <li>
+   *     if field exists and existing vector is of desired vector type, return the vector.
+   *   </li>
+   *   <li>
+   *     if field exists and null filled, clear the existing vector; create and insert a new vector of desired type.
+   *   </li>
+   *   <li>
+   *     otherwise, throw an {@link java.lang.IllegalStateException}
+   *   </li>
+   * </ul>
+   *
+   * @param name name of the field
+   * @param type type of the field
+   * @param clazz class of expected vector type
+   * @param <T> class type of expected vector type
+   * @throws java.lang.IllegalStateException raised if there is a hard schema change
+   *
+   * @return resultant {@link org.apache.arrow.vector.ValueVector}
+   */
+  @Override
+  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+    final ValueVector existing = getChild(name);
+    boolean create = false;
+    if (existing == null) {
+      create = true;
+    } else if (clazz.isAssignableFrom(existing.getClass())) {
+      return (T) existing;
+    } else if (nullFilled(existing)) {
+      existing.clear();
+      create = true;
+    }
+    if (create) {
+      final T vector = (T) BasicTypeHelper.getNewVector(name, allocator, type, callBack);
+      putChild(name, vector);
+      if (callBack!=null) {
+        callBack.doWork();
+      }
+      return vector;
+    }
+    final String message = "Drill does not support schema change yet. Existing[%s] and desired[%s] vector types mismatch";
+    throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(), clazz.getSimpleName()));
+  }
+
+  private boolean nullFilled(ValueVector vector) {
+    for (int r = 0; r < vector.getAccessor().getValueCount(); r++) {
+      if (!vector.getAccessor().isNull(r)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given ordinal identifier.
+   */
+  public ValueVector getChildByOrdinal(int id) {
+    return vectors.getByOrdinal(id);
+  }
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.ValueVector} instance of subtype of <T> corresponding to the given
+   * field name if exists or null.
+   */
+  @Override
+  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+    final ValueVector v = vectors.get(name.toLowerCase());
+    if (v == null) {
+      return null;
+    }
+    return typeify(v, clazz);
+  }
+
+  /**
+   * Inserts the vector with the given name if it does not exist else replaces it with the new value.
+   *
+   * Note that this method does not enforce any vector type check nor throws a schema change exception.
+   */
+  protected void putChild(String name, ValueVector vector) {
+    putVector(name, vector);
+    field.addChild(vector.getField());
+  }
+
+  /**
+   * Inserts the input vector into the map if it does not exist, replaces if it exists already
+   * @param name  field name
+   * @param vector  vector to be inserted
+   */
+  protected void putVector(String name, ValueVector vector) {
+    final ValueVector old = vectors.put(
+        Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
+        Preconditions.checkNotNull(vector, "vector cannot be null")
+    );
+    if (old != null && old != vector) {
+      logger.debug("Field [{}] mutated from [{}] to [{}]", name, old.getClass().getSimpleName(),
+                   vector.getClass().getSimpleName());
+    }
+  }
+
+  /**
+   * Returns a sequence of underlying child vectors.
+   */
+  protected Collection<ValueVector> getChildren() {
+    return vectors.values();
+  }
+
+  /**
+   * Returns the number of underlying child vectors.
+   */
+  @Override
+  public int size() {
+    return vectors.size();
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return vectors.values().iterator();
+  }
+
+  /**
+   * Returns a list of scalar child vectors recursing the entire vector hierarchy.
+   */
+  public List<ValueVector> getPrimitiveVectors() {
+    final List<ValueVector> primitiveVectors = Lists.newArrayList();
+    for (final ValueVector v : vectors.values()) {
+      if (v instanceof AbstractMapVector) {
+        AbstractMapVector mapVector = (AbstractMapVector) v;
+        primitiveVectors.addAll(mapVector.getPrimitiveVectors());
+      } else {
+        primitiveVectors.add(v);
+      }
+    }
+    return primitiveVectors;
+  }
+
+  /**
+   * Returns a vector with its corresponding ordinal mapping if field exists or null.
+   */
+  @Override
+  public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
+    final int ordinal = vectors.getOrdinal(name.toLowerCase());
+    if (ordinal < 0) {
+      return null;
+    }
+    final ValueVector vector = vectors.getByOrdinal(ordinal);
+    return new VectorWithOrdinal(vector, ordinal);
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final List<ArrowBuf> buffers = Lists.newArrayList();
+
+    for (final ValueVector vector : vectors.values()) {
+      for (final ArrowBuf buf : vector.getBuffers(false)) {
+        buffers.add(buf);
+        if (clear) {
+          buf.retain(1);
+        }
+      }
+      if (clear) {
+        vector.clear();
+      }
+    }
+
+    return buffers.toArray(new ArrowBuf[buffers.size()]);
+  }
+
+  @Override
+  public int getBufferSize() {
+    int actualBufSize = 0 ;
+
+    for (final ValueVector v : vectors.values()) {
+      for (final ArrowBuf buf : v.getBuffers(false)) {
+        actualBufSize += buf.writerIndex();
+      }
+    }
+    return actualBufSize;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
new file mode 100644
index 0000000000000..6518897fb780d
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
+
+  public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
+  public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+  public final static String DATA_VECTOR_NAME = "$data$";
+
+  public final static MaterializedField OFFSETS_FIELD =
+    MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
+
+  protected final UInt4Vector offsets;
+  protected ValueVector vector;
+
+  protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
+    this(field, allocator, DEFAULT_DATA_VECTOR);
+  }
+
+  protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
+    super(field, allocator);
+    this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
+    this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      if (!offsets.allocateNewSafe()) {
+        return false;
+      }
+      success = vector.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    offsets.zeroVector();
+    return success;
+  }
+
+
+  @Override
+  public UInt4Vector getOffsetVector() {
+    return offsets;
+  }
+
+  @Override
+  public ValueVector getDataVector() {
+    return vector;
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    offsets.setInitialCapacity(numRecords + 1);
+    vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+  }
+
+  @Override
+  public int getValueCapacity() {
+    final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
+    if (vector == DEFAULT_DATA_VECTOR) {
+      return offsetValueCapacity;
+    }
+    return Math.min(vector.getValueCapacity(), offsetValueCapacity);
+  }
+
+//  @Override
+//  protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+//    return super.getMetadataBuilder()
+//        .addChild(offsets.getMetadata())
+//        .addChild(vector.getMetadata());
+//  }
+
+  @Override
+  public int getBufferSize() {
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return offsets.getBufferSize() + vector.getBufferSize();
+  }
+
+  @Override
+  public int getBufferSizeFor(int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount);
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return Collections.singleton(getDataVector()).iterator();
+  }
+
+  @Override
+  public void clear() {
+    offsets.clear();
+    vector.clear();
+    super.clear();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
+    if (clear) {
+      for (ArrowBuf buffer:buffers) {
+        buffer.retain();
+      }
+      clear();
+    }
+    return buffers;
+  }
+
+//  @Override
+//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+//    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+//    offsets.load(offsetMetadata, buffer);
+//
+//    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
+//    if (getDataVector() == DEFAULT_DATA_VECTOR) {
+//      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+//    }
+//
+//    final int offsetLength = offsetMetadata.getBufferLength();
+//    final int vectorLength = vectorMetadata.getBufferLength();
+//    vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
+//  }
+
+  /**
+   * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
+   *
+   * @see {@link ContainerVectorLike#size}
+   */
+  @Override
+  public int size() {
+    return vector == DEFAULT_DATA_VECTOR ? 0:1;
+  }
+
+  @Override
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+    boolean created = false;
+    if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != MinorType.LATE) {
+      final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
+      vector = BasicTypeHelper.getNewVector(field, allocator);
+      // returned vector must have the same field
+      assert field.equals(vector.getField());
+      getField().addChild(field);
+      created = true;
+    }
+
+    final MajorType actual = vector.getField().getType();
+    if (!actual.equals(descriptor.getType())) {
+      final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
+          descriptor.getType(), actual);
+      throw new SchemaChangeRuntimeException(msg);
+    }
+
+    return new AddOrGetResult<>((T)vector, created);
+  }
+
+  protected void replaceDataVector(ValueVector v) {
+    vector.clear();
+    vector = v;
+  }
+
+  public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
+
+    @Override
+    public int getValueCount() {
+      return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
+    }
+
+    @Override
+    public int getInnerValueCount() {
+      return vector.getAccessor().getValueCount();
+    }
+
+    @Override
+    public int getInnerValueCountAt(int index) {
+      return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+
+    @Override
+    public boolean isEmpty(int index) {
+      return false;
+    }
+  }
+
+  public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
+
+    @Override
+    public void startNewValue(int index) {
+      while (offsets.getValueCapacity() <= index) {
+        offsets.reAlloc();
+      }
+      offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
+      setValueCount(index+1);
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      // TODO: populate offset end points
+      offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+      final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+      vector.getMutator().setValueCount(childValueCount);
+    }
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
new file mode 100644
index 0000000000000..e50b0d0d0a5ea
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+
+/**
+ * A mix-in used for introducing container vector-like behaviour.
+ */
+public interface ContainerVectorLike {
+
+  /**
+   * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
+   *
+   * @param  descriptor vector descriptor
+   * @return  result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
+   * @throws org.apache.drill.common.exceptions.DrillRuntimeException
+   *    if schema change is not permissible between the given and existing data vector types.
+   */
+  <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
+
+  /**
+   * Returns the number of child vectors in this container vector-like instance.
+   */
+  int size();
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
new file mode 100644
index 0000000000000..df699755770a5
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.UInt4Vector;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A helper class that is used to track and populate empty values in repeated value vectors.
+ */
+public class EmptyValuePopulator {
+  private final UInt4Vector offsets;
+
+  public EmptyValuePopulator(UInt4Vector offsets) {
+    this.offsets = Preconditions.checkNotNull(offsets, "offsets cannot be null");
+  }
+
+  /**
+   * Marks all values since the last set as empty. The last set value is obtained from underlying offsets vector.
+   *
+   * @param lastIndex  the last index (inclusive) in the offsets vector until which empty population takes place
+   * @throws java.lang.IndexOutOfBoundsException  if lastIndex is negative or greater than offsets capacity.
+   */
+  public void populate(int lastIndex) {
+    if (lastIndex < 0) {
+      throw new IndexOutOfBoundsException("index cannot be negative");
+    }
+    final UInt4Vector.Accessor accessor = offsets.getAccessor();
+    final UInt4Vector.Mutator mutator = offsets.getMutator();
+    final int lastSet = Math.max(accessor.getValueCount() - 1, 0);
+    final int previousEnd = accessor.get(lastSet);//0 ? 0 : accessor.get(lastSet);
+    for (int i = lastSet; i < lastIndex; i++) {
+      mutator.setSafe(i + 1, previousEnd);
+    }
+    mutator.setValueCount(lastIndex+1);
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
new file mode 100644
index 0000000000000..8387c9e5ba667
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -0,0 +1,321 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.complex.impl.ComplexCopier;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.collect.ObjectArrays;
+
+public class ListVector extends BaseRepeatedValueVector {
+
+  private UInt4Vector offsets;
+  private final UInt1Vector bits;
+  private Mutator mutator = new Mutator();
+  private Accessor accessor = new Accessor();
+  private UnionListWriter writer;
+  private UnionListReader reader;
+  private CallBack callBack;
+
+  public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    super(field, allocator);
+    this.bits = new UInt1Vector(MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED)), allocator);
+    offsets = getOffsetVector();
+    this.field.addChild(getDataVector().getField());
+    this.writer = new UnionListWriter(this);
+    this.reader = new UnionListReader(this);
+    this.callBack = callBack;
+  }
+
+  public UnionListWriter getWriter() {
+    return writer;
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    super.allocateNewSafe();
+  }
+
+  public void transferTo(ListVector target) {
+    offsets.makeTransferPair(target.offsets).transfer();
+    bits.makeTransferPair(target.bits).transfer();
+    if (target.getDataVector() instanceof ZeroVector) {
+      target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+    }
+    getDataVector().makeTransferPair(target.getDataVector()).transfer();
+  }
+
+  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
+    copyFrom(inIndex, outIndex, from);
+  }
+
+  public void copyFrom(int inIndex, int outIndex, ListVector from) {
+    FieldReader in = from.getReader();
+    in.setPosition(inIndex);
+    FieldWriter out = getWriter();
+    out.setPosition(outIndex);
+    ComplexCopier.copy(in, out);
+  }
+
+  @Override
+  public ValueVector getDataVector() {
+    return vector;
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(field.withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector target) {
+    return new TransferImpl((ListVector) target);
+  }
+
+  private class TransferImpl implements TransferPair {
+
+    ListVector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
+      to = new ListVector(field, allocator, null);
+      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+    }
+
+    public TransferImpl(ListVector to) {
+      this.to = to;
+      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      to.allocateNew();
+      for (int i = 0; i < length; i++) {
+        copyValueSafe(startIndex + i, i);
+      }
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      this.to.copyFrom(from, to, ListVector.this);
+    }
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      if (!offsets.allocateNewSafe()) {
+        return false;
+      }
+      success = vector.allocateNewSafe();
+      success = success && bits.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    if (success) {
+      offsets.zeroVector();
+      bits.zeroVector();
+    }
+    return success;
+  }
+
+//  @Override
+//  protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+//    return getField().getAsBuilder()
+//            .setValueCount(getAccessor().getValueCount())
+//            .setBufferLength(getBufferSize())
+//            .addChild(offsets.getMetadata())
+//            .addChild(bits.getMetadata())
+//            .addChild(vector.getMetadata());
+//  }
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+    AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+    reader = new UnionListReader(this);
+    return result;
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize();
+  }
+
+  @Override
+  public void clear() {
+    offsets.clear();
+    vector.clear();
+    bits.clear();
+    lastSet = 0;
+    super.clear();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false),
+            vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
+    if (clear) {
+      for (ArrowBuf buffer:buffers) {
+        buffer.retain();
+      }
+      clear();
+    }
+    return buffers;
+  }
+
+//  @Override
+//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+//    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+//    offsets.load(offsetMetadata, buffer);
+//
+//    final int offsetLength = offsetMetadata.getBufferLength();
+//    final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
+//    final int bitLength = bitMetadata.getBufferLength();
+//    bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
+//
+//    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
+//    if (getDataVector() == DEFAULT_DATA_VECTOR) {
+//      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+//    }
+//
+//    final int vectorLength = vectorMetadata.getBufferLength();
+//    vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
+//  }
+
+  public UnionVector promoteToUnion() {
+    MaterializedField newField = MaterializedField.create(getField().getPath(), new MajorType(MinorType.UNION, DataMode.OPTIONAL));
+    UnionVector vector = new UnionVector(newField, allocator, null);
+    replaceDataVector(vector);
+    reader = new UnionListReader(this);
+    return vector;
+  }
+
+  private int lastSet;
+
+  public class Accessor extends BaseRepeatedAccessor {
+
+    @Override
+    public Object getObject(int index) {
+      if (isNull(index)) {
+        return null;
+      }
+      final List<Object> vals = new JsonStringArrayList<>();
+      final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
+      final int start = offsetsAccessor.get(index);
+      final int end = offsetsAccessor.get(index + 1);
+      final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor();
+      for(int i = start; i < end; i++) {
+        vals.add(valuesAccessor.getObject(i));
+      }
+      return vals;
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return bits.getAccessor().get(index) == 0;
+    }
+  }
+
+  public class Mutator extends BaseRepeatedMutator {
+    public void setNotNull(int index) {
+      bits.getMutator().setSafe(index, 1);
+      lastSet = index + 1;
+    }
+
+    @Override
+    public void startNewValue(int index) {
+      for (int i = lastSet; i <= index; i++) {
+        offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
+      }
+      setNotNull(index);
+      lastSet = index + 1;
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      // TODO: populate offset end points
+      if (valueCount == 0) {
+        offsets.getMutator().setValueCount(0);
+      } else {
+        for (int i = lastSet; i < valueCount; i++) {
+          offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
+        }
+        offsets.getMutator().setValueCount(valueCount + 1);
+      }
+      final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+      vector.getMutator().setValueCount(childValueCount);
+      bits.getMutator().setValueCount(valueCount);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
new file mode 100644
index 0000000000000..1bbce73d6ff82
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -0,0 +1,374 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.RepeatedMapVector.MapSingleCopier;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringHashMap;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
+
+public class MapVector extends AbstractMapVector {
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
+
+  public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
+
+  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+  private int valueCount;
+
+  public MapVector(String path, BufferAllocator allocator, CallBack callBack){
+    this(MaterializedField.create(path, TYPE), allocator, callBack);
+  }
+
+  public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+    super(field, allocator, callBack);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    //return new SingleMapReaderImpl(MapVector.this);
+    return reader;
+  }
+
+  transient private MapTransferPair ephPair;
+  transient private MapSingleCopier ephPair2;
+
+  public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
+    if(ephPair == null || ephPair.from != from) {
+      ephPair = (MapTransferPair) from.makeTransferPair(this);
+    }
+    ephPair.copyValueSafe(fromIndex, thisIndex);
+  }
+
+  public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
+    if(ephPair2 == null || ephPair2.from != from) {
+      ephPair2 = from.makeSingularCopier(this);
+    }
+    ephPair2.copySafe(fromSubIndex, thisIndex);
+  }
+
+  @Override
+  protected boolean supportsDirectRead() {
+    return true;
+  }
+
+  public Iterator<String> fieldNameIterator() {
+    return getChildFieldNames().iterator();
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      v.setInitialCapacity(numRecords);
+    }
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (valueCount == 0 || size() == 0) {
+      return 0;
+    }
+    long buffer = 0;
+    for (final ValueVector v : (Iterable<ValueVector>)this) {
+      buffer += v.getBufferSize();
+    }
+
+    return (int) buffer;
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    long bufferSize = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      bufferSize += v.getBufferSizeFor(valueCount);
+    }
+
+    return (int) bufferSize;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    int expectedSize = getBufferSize();
+    int actualSize   = super.getBufferSize();
+
+    Preconditions.checkArgument(expectedSize == actualSize);
+    return super.getBuffers(clear);
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new MapTransferPair(this, getField().getPath(), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new MapTransferPair(this, (MapVector) to);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new MapTransferPair(this, ref, allocator);
+  }
+
+  protected static class MapTransferPair implements TransferPair{
+    private final TransferPair[] pairs;
+    private final MapVector from;
+    private final MapVector to;
+
+    public MapTransferPair(MapVector from, String path, BufferAllocator allocator) {
+      this(from, new MapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
+    }
+
+    public MapTransferPair(MapVector from, MapVector to) {
+      this(from, to, true);
+    }
+
+    protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
+      this.from = from;
+      this.to = to;
+      this.pairs = new TransferPair[from.size()];
+      this.to.ephPair = null;
+      this.to.ephPair2 = null;
+
+      int i = 0;
+      ValueVector vector;
+      for (String child:from.getChildFieldNames()) {
+        int preSize = to.size();
+        vector = from.getChild(child);
+        if (vector == null) {
+          continue;
+        }
+        //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
+        // the child fields may be nested fields of the top level child. For example if the structure
+        // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
+        // But the children member of a Materialized field is a HashSet. If the fields are added in the
+        // children HashSet, and the hashCode of the Materialized field includes the hash code of the
+        // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
+        // (This is similar to what happens in ScanBatch where the children cannot be added till they are
+        // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
+        // include the hashCode of the children but is based only on MaterializedField$key.
+        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+        if (allocate && to.size() != preSize) {
+          newVector.allocateNew();
+        }
+        pairs[i++] = vector.makeTransferPair(newVector);
+      }
+    }
+
+    @Override
+    public void transfer() {
+      for (final TransferPair p : pairs) {
+        p.transfer();
+      }
+      to.valueCount = from.valueCount;
+      from.clear();
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      for (TransferPair p : pairs) {
+        p.copyValueSafe(from, to);
+      }
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      for (TransferPair p : pairs) {
+        p.splitAndTransfer(startIndex, length);
+      }
+      to.getMutator().setValueCount(length);
+    }
+  }
+
+  @Override
+  public int getValueCapacity() {
+    if (size() == 0) {
+      return 0;
+    }
+
+    final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
+      @Override
+      public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
+        return Ints.compare(
+            Preconditions.checkNotNull(left).getValueCapacity(),
+            Preconditions.checkNotNull(right).getValueCapacity()
+        );
+      }
+    };
+
+    return natural.min(getChildren()).getValueCapacity();
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+//  @Override
+//  public void load(SerializedField metadata, DrillBuf buf) {
+//    final List<SerializedField> fields = metadata.getChildList();
+//    valueCount = metadata.getValueCount();
+//
+//    int bufOffset = 0;
+//    for (final SerializedField child : fields) {
+//      final MaterializedField fieldDef = SerializedFieldHelper.create(child);
+//
+//      ValueVector vector = getChild(fieldDef.getLastName());
+//      if (vector == null) {
+//         if we arrive here, we didn't have a matching vector.
+//        vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
+//        putChild(fieldDef.getLastName(), vector);
+//      }
+//      if (child.getValueCount() == 0) {
+//        vector.clear();
+//      } else {
+//        vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
+//      }
+//      bufOffset += child.getBufferLength();
+//    }
+//
+//    assert bufOffset == buf.capacity();
+//  }
+//
+//  @Override
+//  public SerializedField getMetadata() {
+//    SerializedField.Builder b = getField() //
+//        .getAsBuilder() //
+//        .setBufferLength(getBufferSize()) //
+//        .setValueCount(valueCount);
+//
+//
+//    for(ValueVector v : getChildren()) {
+//      b.addChild(v.getMetadata());
+//    }
+//    return b.build();
+//  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  public class Accessor extends BaseValueVector.BaseAccessor {
+
+    @Override
+    public Object getObject(int index) {
+      Map<String, Object> vv = new JsonStringHashMap<>();
+      for (String child:getChildFieldNames()) {
+        ValueVector v = getChild(child);
+        // TODO(DRILL-4001):  Resolve this hack:
+        // The index/value count check in the following if statement is a hack
+        // to work around the current fact that RecordBatchLoader.load and
+        // MapVector.load leave child vectors with a length of zero (as opposed
+        // to matching the lengths of siblings and the parent map vector)
+        // because they don't remove (or set the lengths of) vectors from
+        // previous batches that aren't in the current batch.
+        if (v != null && index < v.getAccessor().getValueCount()) {
+          Object value = v.getAccessor().getObject(index);
+          if (value != null) {
+            vv.put(child, value);
+          }
+        }
+      }
+      return vv;
+    }
+
+    public void get(int index, ComplexHolder holder) {
+      reader.setPosition(index);
+      holder.reader = reader;
+    }
+
+    @Override
+    public int getValueCount() {
+      return valueCount;
+    }
+  }
+
+  public ValueVector getVectorById(int id) {
+    return getChildByOrdinal(id);
+  }
+
+  public class Mutator extends BaseValueVector.BaseMutator {
+
+    @Override
+    public void setValueCount(int valueCount) {
+      for (final ValueVector v : getChildren()) {
+        v.getMutator().setValueCount(valueCount);
+      }
+      MapVector.this.valueCount = valueCount;
+    }
+
+    @Override
+    public void reset() { }
+
+    @Override
+    public void generateTestData(int values) { }
+  }
+
+  @Override
+  public void clear() {
+    for (final ValueVector v : getChildren()) {
+      v.clear();
+    }
+    valueCount = 0;
+  }
+
+  @Override
+  public void close() {
+    final Collection<ValueVector> vectors = getChildren();
+    for (final ValueVector v : vectors) {
+      v.close();
+    }
+    vectors.clear();
+    valueCount = 0;
+
+    super.close();
+ }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
new file mode 100644
index 0000000000000..93451181ca949
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+public interface Positionable {
+  public void setPosition(int index);
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
new file mode 100644
index 0000000000000..23850bc9034df
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+/**
+ * A {@link org.apache.arrow.vector.ValueVector} mix-in that can be used in conjunction with
+ * {@link RepeatedValueVector} subtypes.
+ */
+public interface RepeatedFixedWidthVectorLike {
+  /**
+   * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
+   *
+   * @param valueCount   Number of separate repeating groupings.
+   * @param innerValueCount   Number of supported values in the vector.
+   */
+  void allocateNew(int valueCount, int innerValueCount);
+
+  /**
+   * Load the records in the provided buffer based on the given number of values.
+   * @param valueCount   Number of separate repeating groupings.
+   * @param innerValueCount Number atomic values the buffer contains.
+   * @param buf Incoming buffer.
+   * @return The number of bytes of the buffer that were consumed.
+   */
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
new file mode 100644
index 0000000000000..778fe81b5da6a
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
@@ -0,0 +1,428 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.complex.impl.NullReader;
+import org.apache.arrow.vector.complex.impl.RepeatedListReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.holders.RepeatedListHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class RepeatedListVector extends AbstractContainerVector
+    implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
+
+  public final static MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
+  private final RepeatedListReaderImpl reader = new RepeatedListReaderImpl(null, this);
+  private final DelegateRepeatedVector delegate;
+
+  protected static class DelegateRepeatedVector extends BaseRepeatedValueVector {
+
+    private final RepeatedListAccessor accessor = new RepeatedListAccessor();
+    private final RepeatedListMutator mutator = new RepeatedListMutator();
+    private final EmptyValuePopulator emptyPopulator;
+    private transient DelegateTransferPair ephPair;
+
+    public class RepeatedListAccessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
+
+      @Override
+      public Object getObject(int index) {
+        final List<Object> list = new JsonStringArrayList<>();
+        final int start = offsets.getAccessor().get(index);
+        final int until = offsets.getAccessor().get(index+1);
+        for (int i = start; i < until; i++) {
+          list.add(vector.getAccessor().getObject(i));
+        }
+        return list;
+      }
+
+      public void get(int index, RepeatedListHolder holder) {
+        assert index <= getValueCapacity();
+        holder.start = getOffsetVector().getAccessor().get(index);
+        holder.end = getOffsetVector().getAccessor().get(index+1);
+      }
+
+      public void get(int index, ComplexHolder holder) {
+        final FieldReader reader = getReader();
+        reader.setPosition(index);
+        holder.reader = reader;
+      }
+
+      public void get(int index, int arrayIndex, ComplexHolder holder) {
+        final RepeatedListHolder listHolder = new RepeatedListHolder();
+        get(index, listHolder);
+        int offset = listHolder.start + arrayIndex;
+        if (offset >= listHolder.end) {
+          holder.reader = NullReader.INSTANCE;
+        } else {
+          FieldReader r = getDataVector().getReader();
+          r.setPosition(offset);
+          holder.reader = r;
+        }
+      }
+    }
+
+    public class RepeatedListMutator extends BaseRepeatedValueVector.BaseRepeatedMutator {
+
+      public int add(int index) {
+        final int curEnd = getOffsetVector().getAccessor().get(index+1);
+        getOffsetVector().getMutator().setSafe(index + 1, curEnd + 1);
+        return curEnd;
+      }
+
+      @Override
+      public void startNewValue(int index) {
+        emptyPopulator.populate(index+1);
+        super.startNewValue(index);
+      }
+
+      @Override
+      public void setValueCount(int valueCount) {
+        emptyPopulator.populate(valueCount);
+        super.setValueCount(valueCount);
+      }
+    }
+
+
+    public class DelegateTransferPair implements TransferPair {
+      private final DelegateRepeatedVector target;
+      private final TransferPair[] children;
+
+      public DelegateTransferPair(DelegateRepeatedVector target) {
+        this.target = Preconditions.checkNotNull(target);
+        if (target.getDataVector() == DEFAULT_DATA_VECTOR) {
+          target.addOrGetVector(VectorDescriptor.create(getDataVector().getField()));
+          target.getDataVector().allocateNew();
+        }
+        this.children = new TransferPair[] {
+            getOffsetVector().makeTransferPair(target.getOffsetVector()),
+            getDataVector().makeTransferPair(target.getDataVector())
+        };
+      }
+
+      @Override
+      public void transfer() {
+        for (TransferPair child:children) {
+          child.transfer();
+        }
+      }
+
+      @Override
+      public ValueVector getTo() {
+        return target;
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+        target.allocateNew();
+        for (int i = 0; i < length; i++) {
+          copyValueSafe(startIndex + i, i);
+        }
+      }
+
+      @Override
+      public void copyValueSafe(int srcIndex, int destIndex) {
+        final RepeatedListHolder holder = new RepeatedListHolder();
+        getAccessor().get(srcIndex, holder);
+        target.emptyPopulator.populate(destIndex+1);
+        final TransferPair vectorTransfer = children[1];
+        int newIndex = target.getOffsetVector().getAccessor().get(destIndex);
+        //todo: make this a bulk copy.
+        for (int i = holder.start; i < holder.end; i++, newIndex++) {
+          vectorTransfer.copyValueSafe(i, newIndex);
+        }
+        target.getOffsetVector().getMutator().setSafe(destIndex + 1, newIndex);
+      }
+    }
+
+    public DelegateRepeatedVector(String path, BufferAllocator allocator) {
+      this(MaterializedField.create(path, TYPE), allocator);
+    }
+
+    public DelegateRepeatedVector(MaterializedField field, BufferAllocator allocator) {
+      super(field, allocator);
+      emptyPopulator = new EmptyValuePopulator(getOffsetVector());
+    }
+
+    @Override
+    public void allocateNew() throws OutOfMemoryException {
+      if (!allocateNewSafe()) {
+        throw new OutOfMemoryException();
+      }
+    }
+
+    @Override
+    public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      return makeTransferPair(new DelegateRepeatedVector(ref, allocator));
+    }
+
+    @Override
+    public TransferPair makeTransferPair(ValueVector target) {
+      return new DelegateTransferPair(DelegateRepeatedVector.class.cast(target));
+    }
+
+    @Override
+    public RepeatedListAccessor getAccessor() {
+      return accessor;
+    }
+
+    @Override
+    public RepeatedListMutator getMutator() {
+      return mutator;
+    }
+
+    @Override
+    public FieldReader getReader() {
+      throw new UnsupportedOperationException();
+    }
+
+    public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) {
+      if(ephPair == null || ephPair.target != from) {
+        ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this));
+      }
+      ephPair.copyValueSafe(fromIndex, thisIndex);
+    }
+
+  }
+
+  protected class RepeatedListTransferPair implements TransferPair {
+    private final TransferPair delegate;
+
+    public RepeatedListTransferPair(TransferPair delegate) {
+      this.delegate = delegate;
+    }
+
+    public void transfer() {
+      delegate.transfer();
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      delegate.splitAndTransfer(startIndex, length);
+    }
+
+    @Override
+    public ValueVector getTo() {
+      final DelegateRepeatedVector delegateVector = DelegateRepeatedVector.class.cast(delegate.getTo());
+      return new RepeatedListVector(getField(), allocator, callBack, delegateVector);
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      delegate.copyValueSafe(from, to);
+    }
+  }
+
+  public RepeatedListVector(String path, BufferAllocator allocator, CallBack callBack) {
+    this(MaterializedField.create(path, TYPE), allocator, callBack);
+  }
+
+  public RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    this(field, allocator, callBack, new DelegateRepeatedVector(field, allocator));
+  }
+
+  protected RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack, DelegateRepeatedVector delegate) {
+    super(field, allocator, callBack);
+    this.delegate = Preconditions.checkNotNull(delegate);
+
+    final List<MaterializedField> children = Lists.newArrayList(field.getChildren());
+    final int childSize = children.size();
+    assert childSize < 3;
+    final boolean hasChild = childSize > 0;
+    if (hasChild) {
+      // the last field is data field
+      final MaterializedField child = children.get(childSize-1);
+      addOrGetVector(VectorDescriptor.create(child));
+    }
+  }
+
+
+    @Override
+  public RepeatedListReaderImpl getReader() {
+    return reader;
+  }
+
+  @Override
+  public DelegateRepeatedVector.RepeatedListAccessor getAccessor() {
+    return delegate.getAccessor();
+  }
+
+  @Override
+  public DelegateRepeatedVector.RepeatedListMutator getMutator() {
+    return delegate.getMutator();
+  }
+
+  @Override
+  public UInt4Vector getOffsetVector() {
+    return delegate.getOffsetVector();
+  }
+
+  @Override
+  public ValueVector getDataVector() {
+    return delegate.getDataVector();
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    delegate.allocateNew();
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    return delegate.allocateNewSafe();
+  }
+
+  @Override
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+    final AddOrGetResult<T> result = delegate.addOrGetVector(descriptor);
+    if (result.isCreated() && callBack != null) {
+      callBack.doWork();
+    }
+    this.field = delegate.getField();
+    return result;
+  }
+
+  @Override
+  public int size() {
+    return delegate.size();
+  }
+
+  @Override
+  public int getBufferSize() {
+    return delegate.getBufferSize();
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    return delegate.getBufferSizeFor(valueCount);
+  }
+
+  @Override
+  public void close() {
+    delegate.close();
+  }
+
+  @Override
+  public void clear() {
+    delegate.clear();
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new RepeatedListTransferPair(delegate.getTransferPair(allocator));
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new RepeatedListTransferPair(delegate.getTransferPair(ref, allocator));
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    final RepeatedListVector target = RepeatedListVector.class.cast(to);
+    return new RepeatedListTransferPair(delegate.makeTransferPair(target.delegate));
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return delegate.getValueCapacity();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    return delegate.getBuffers(clear);
+  }
+
+
+//  @Override
+//  public void load(SerializedField metadata, DrillBuf buf) {
+//    delegate.load(metadata, buf);
+//  }
+
+//  @Override
+//  public SerializedField getMetadata() {
+//    return delegate.getMetadata();
+//  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return delegate.iterator();
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    delegate.setInitialCapacity(numRecords);
+  }
+
+  /**
+   * @deprecated
+   *   prefer using {@link #addOrGetVector(org.apache.arrow.vector.VectorDescriptor)} instead.
+   */
+  @Override
+  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+    final AddOrGetResult<T> result = addOrGetVector(VectorDescriptor.create(type));
+    return result.getVector();
+  }
+
+  @Override
+  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+    if (name != null) {
+      return null;
+    }
+    return typeify(delegate.getDataVector(), clazz);
+  }
+
+  @Override
+  public void allocateNew(int valueCount, int innerValueCount) {
+    clear();
+    getOffsetVector().allocateNew(valueCount + 1);
+    getMutator().reset();
+  }
+
+  @Override
+  public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
+    if (name != null) {
+      return null;
+    }
+    return new VectorWithOrdinal(delegate.getDataVector(), 0);
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) {
+    delegate.copyFromSafe(fromIndex, thisIndex, from.delegate);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
new file mode 100644
index 0000000000000..e7eacd3c67c40
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
@@ -0,0 +1,584 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.AllocationHelper;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.complex.impl.NullReader;
+import org.apache.arrow.vector.complex.impl.RepeatedMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.holders.RepeatedMapHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.TransferPair;
+import org.apache.commons.lang3.ArrayUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+public class RepeatedMapVector extends AbstractMapVector
+    implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
+
+  public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.REPEATED);
+
+  private final UInt4Vector offsets;   // offsets to start of each record (considering record indices are 0-indexed)
+  private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
+  private final RepeatedMapAccessor accessor = new RepeatedMapAccessor();
+  private final Mutator mutator = new Mutator();
+  private final EmptyValuePopulator emptyPopulator;
+
+  public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+    super(field, allocator, callBack);
+    this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator);
+    this.emptyPopulator = new EmptyValuePopulator(offsets);
+  }
+
+  @Override
+  public UInt4Vector getOffsetVector() {
+    return offsets;
+  }
+
+  @Override
+  public ValueVector getDataVector() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    offsets.setInitialCapacity(numRecords + 1);
+    for(final ValueVector v : (Iterable<ValueVector>) this) {
+      v.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+    }
+  }
+
+  @Override
+  public RepeatedMapReaderImpl getReader() {
+    return reader;
+  }
+
+  @Override
+  public void allocateNew(int groupCount, int innerValueCount) {
+    clear();
+    try {
+      offsets.allocateNew(groupCount + 1);
+      for (ValueVector v : getChildren()) {
+        AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
+      }
+    } catch (OutOfMemoryException e){
+      clear();
+      throw e;
+    }
+    offsets.zeroVector();
+    mutator.reset();
+  }
+
+  public Iterator<String> fieldNameIterator() {
+    return getChildFieldNames().iterator();
+  }
+
+  @Override
+  public List<ValueVector> getPrimitiveVectors() {
+    final List<ValueVector> primitiveVectors = super.getPrimitiveVectors();
+    primitiveVectors.add(offsets);
+    return primitiveVectors;
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    long bufferSize = offsets.getBufferSize();
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      bufferSize += v.getBufferSize();
+    }
+    return (int) bufferSize;
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    long bufferSize = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      bufferSize += v.getBufferSizeFor(valueCount);
+    }
+
+    return (int) bufferSize;
+  }
+
+  @Override
+  public void close() {
+    offsets.close();
+    super.close();
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new RepeatedMapTransferPair(this, getField().getPath(), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new RepeatedMapTransferPair(this, (RepeatedMapVector)to);
+  }
+
+  MapSingleCopier makeSingularCopier(MapVector to) {
+    return new MapSingleCopier(this, to);
+  }
+
+  protected static class MapSingleCopier {
+    private final TransferPair[] pairs;
+    public final RepeatedMapVector from;
+
+    public MapSingleCopier(RepeatedMapVector from, MapVector to) {
+      this.from = from;
+      this.pairs = new TransferPair[from.size()];
+
+      int i = 0;
+      ValueVector vector;
+      for (final String child:from.getChildFieldNames()) {
+        int preSize = to.size();
+        vector = from.getChild(child);
+        if (vector == null) {
+          continue;
+        }
+        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+        if (to.size() != preSize) {
+          newVector.allocateNew();
+        }
+        pairs[i++] = vector.makeTransferPair(newVector);
+      }
+    }
+
+    public void copySafe(int fromSubIndex, int toIndex) {
+      for (TransferPair p : pairs) {
+        p.copyValueSafe(fromSubIndex, toIndex);
+      }
+    }
+  }
+
+  public TransferPair getTransferPairToSingleMap(String reference, BufferAllocator allocator) {
+    return new SingleMapTransferPair(this, reference, allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new RepeatedMapTransferPair(this, ref, allocator);
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      if (!offsets.allocateNewSafe()) {
+        return false;
+      }
+      success =  super.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    offsets.zeroVector();
+    return success;
+  }
+
+  protected static class SingleMapTransferPair implements TransferPair {
+    private final TransferPair[] pairs;
+    private final RepeatedMapVector from;
+    private final MapVector to;
+    private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.REQUIRED);
+
+    public SingleMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
+      this(from, new MapVector(MaterializedField.create(path, MAP_TYPE), allocator, from.callBack), false);
+    }
+
+    public SingleMapTransferPair(RepeatedMapVector from, MapVector to) {
+      this(from, to, true);
+    }
+
+    public SingleMapTransferPair(RepeatedMapVector from, MapVector to, boolean allocate) {
+      this.from = from;
+      this.to = to;
+      this.pairs = new TransferPair[from.size()];
+      int i = 0;
+      ValueVector vector;
+      for (final String child : from.getChildFieldNames()) {
+        int preSize = to.size();
+        vector = from.getChild(child);
+        if (vector == null) {
+          continue;
+        }
+        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+        if (allocate && to.size() != preSize) {
+          newVector.allocateNew();
+        }
+        pairs[i++] = vector.makeTransferPair(newVector);
+      }
+    }
+
+
+    @Override
+    public void transfer() {
+      for (TransferPair p : pairs) {
+        p.transfer();
+      }
+      to.getMutator().setValueCount(from.getAccessor().getValueCount());
+      from.clear();
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      for (TransferPair p : pairs) {
+        p.copyValueSafe(from, to);
+      }
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      for (TransferPair p : pairs) {
+        p.splitAndTransfer(startIndex, length);
+      }
+      to.getMutator().setValueCount(length);
+    }
+  }
+
+  private static class RepeatedMapTransferPair implements TransferPair{
+
+    private final TransferPair[] pairs;
+    private final RepeatedMapVector to;
+    private final RepeatedMapVector from;
+
+    public RepeatedMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
+      this(from, new RepeatedMapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
+    }
+
+    public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to) {
+      this(from, to, true);
+    }
+
+    public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to, boolean allocate) {
+      this.from = from;
+      this.to = to;
+      this.pairs = new TransferPair[from.size()];
+      this.to.ephPair = null;
+
+      int i = 0;
+      ValueVector vector;
+      for (final String child : from.getChildFieldNames()) {
+        final int preSize = to.size();
+        vector = from.getChild(child);
+        if (vector == null) {
+          continue;
+        }
+
+        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+        if (to.size() != preSize) {
+          newVector.allocateNew();
+        }
+
+        pairs[i++] = vector.makeTransferPair(newVector);
+      }
+    }
+
+    @Override
+    public void transfer() {
+      from.offsets.transferTo(to.offsets);
+      for (TransferPair p : pairs) {
+        p.transfer();
+      }
+      from.clear();
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int srcIndex, int destIndex) {
+      RepeatedMapHolder holder = new RepeatedMapHolder();
+      from.getAccessor().get(srcIndex, holder);
+      to.emptyPopulator.populate(destIndex + 1);
+      int newIndex = to.offsets.getAccessor().get(destIndex);
+      //todo: make these bulk copies
+      for (int i = holder.start; i < holder.end; i++, newIndex++) {
+        for (TransferPair p : pairs) {
+          p.copyValueSafe(i, newIndex);
+        }
+      }
+      to.offsets.getMutator().setSafe(destIndex + 1, newIndex);
+    }
+
+    @Override
+    public void splitAndTransfer(final int groupStart, final int groups) {
+      final UInt4Vector.Accessor a = from.offsets.getAccessor();
+      final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+      final int startPos = a.get(groupStart);
+      final int endPos = a.get(groupStart + groups);
+      final int valuesToCopy = endPos - startPos;
+
+      to.offsets.clear();
+      to.offsets.allocateNew(groups + 1);
+
+      int normalizedPos;
+      for (int i = 0; i < groups + 1; i++) {
+        normalizedPos = a.get(groupStart + i) - startPos;
+        m.set(i, normalizedPos);
+      }
+
+      m.setValueCount(groups + 1);
+      to.emptyPopulator.populate(groups);
+
+      for (final TransferPair p : pairs) {
+        p.splitAndTransfer(startPos, valuesToCopy);
+      }
+    }
+  }
+
+
+  transient private RepeatedMapTransferPair ephPair;
+
+  public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) {
+    if (ephPair == null || ephPair.from != from) {
+      ephPair = (RepeatedMapTransferPair) from.makeTransferPair(this);
+    }
+    ephPair.copyValueSafe(fromIndex, thisIndex);
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return Math.max(offsets.getValueCapacity() - 1, 0);
+  }
+
+  @Override
+  public RepeatedMapAccessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final int expectedBufferSize = getBufferSize();
+    final int actualBufferSize = super.getBufferSize();
+
+    Preconditions.checkArgument(expectedBufferSize == actualBufferSize + offsets.getBufferSize());
+    return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
+  }
+
+
+//  @Override
+//  public void load(SerializedField metadata, DrillBuf buffer) {
+//    final List<SerializedField> children = metadata.getChildList();
+//
+//    final SerializedField offsetField = children.get(0);
+//    offsets.load(offsetField, buffer);
+//    int bufOffset = offsetField.getBufferLength();
+//
+//    for (int i = 1; i < children.size(); i++) {
+//      final SerializedField child = children.get(i);
+//      final MaterializedField fieldDef = SerializedFieldHelper.create(child);
+//      ValueVector vector = getChild(fieldDef.getLastName());
+//      if (vector == null) {
+        // if we arrive here, we didn't have a matching vector.
+//        vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
+//        putChild(fieldDef.getLastName(), vector);
+//      }
+//      final int vectorLength = child.getBufferLength();
+//      vector.load(child, buffer.slice(bufOffset, vectorLength));
+//      bufOffset += vectorLength;
+//    }
+//
+//    assert bufOffset == buffer.capacity();
+//  }
+//
+//
+//  @Override
+//  public SerializedField getMetadata() {
+//    SerializedField.Builder builder = getField() //
+//        .getAsBuilder() //
+//        .setBufferLength(getBufferSize()) //
+        // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
+//        .setValueCount(accessor.getValueCount());
+//    builder.addChild(offsets.getMetadata());
+//    for (final ValueVector child : getChildren()) {
+//      builder.addChild(child.getMetadata());
+//    }
+//    return builder.build();
+//  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  public class RepeatedMapAccessor implements RepeatedAccessor {
+    @Override
+    public Object getObject(int index) {
+      final List<Object> list = new JsonStringArrayList<>();
+      final int end = offsets.getAccessor().get(index+1);
+      String fieldName;
+      for (int i =  offsets.getAccessor().get(index); i < end; i++) {
+        final Map<String, Object> vv = Maps.newLinkedHashMap();
+        for (final MaterializedField field : getField().getChildren()) {
+          if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
+            fieldName = field.getLastName();
+            final Object value = getChild(fieldName).getAccessor().getObject(i);
+            if (value != null) {
+              vv.put(fieldName, value);
+            }
+          }
+        }
+        list.add(vv);
+      }
+      return list;
+    }
+
+    @Override
+    public int getValueCount() {
+      return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
+    }
+
+    @Override
+    public int getInnerValueCount() {
+      final int valueCount = getValueCount();
+      if (valueCount == 0) {
+        return 0;
+      }
+      return offsets.getAccessor().get(valueCount);
+    }
+
+    @Override
+    public int getInnerValueCountAt(int index) {
+      return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+    }
+
+    @Override
+    public boolean isEmpty(int index) {
+      return false;
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+
+    public void get(int index, RepeatedMapHolder holder) {
+      assert index < getValueCapacity() :
+        String.format("Attempted to access index %d when value capacity is %d",
+            index, getValueCapacity());
+      final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
+      holder.start = offsetsAccessor.get(index);
+      holder.end = offsetsAccessor.get(index + 1);
+    }
+
+    public void get(int index, ComplexHolder holder) {
+      final FieldReader reader = getReader();
+      reader.setPosition(index);
+      holder.reader = reader;
+    }
+
+    public void get(int index, int arrayIndex, ComplexHolder holder) {
+      final RepeatedMapHolder h = new RepeatedMapHolder();
+      get(index, h);
+      final int offset = h.start + arrayIndex;
+
+      if (offset >= h.end) {
+        holder.reader = NullReader.INSTANCE;
+      } else {
+        reader.setSinglePosition(index, arrayIndex);
+        holder.reader = reader;
+      }
+    }
+  }
+
+  public class Mutator implements RepeatedMutator {
+    @Override
+    public void startNewValue(int index) {
+      emptyPopulator.populate(index + 1);
+      offsets.getMutator().setSafe(index + 1, offsets.getAccessor().get(index));
+    }
+
+    @Override
+    public void setValueCount(int topLevelValueCount) {
+      emptyPopulator.populate(topLevelValueCount);
+      offsets.getMutator().setValueCount(topLevelValueCount == 0 ? 0 : topLevelValueCount + 1);
+      int childValueCount = offsets.getAccessor().get(topLevelValueCount);
+      for (final ValueVector v : getChildren()) {
+        v.getMutator().setValueCount(childValueCount);
+      }
+    }
+
+    @Override
+    public void reset() {}
+
+    @Override
+    public void generateTestData(int values) {}
+
+    public int add(int index) {
+      final int prevEnd = offsets.getAccessor().get(index + 1);
+      offsets.getMutator().setSafe(index + 1, prevEnd + 1);
+      return prevEnd;
+    }
+  }
+
+  @Override
+  public void clear() {
+    getMutator().reset();
+
+    offsets.clear();
+    for(final ValueVector vector : getChildren()) {
+      vector.clear();
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
new file mode 100644
index 0000000000000..99c0a0aeb1e2c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+
+/**
+ * An abstraction representing repeated value vectors.
+ *
+ * A repeated vector contains values that may either be flat or nested. A value consists of zero or more cells(inner values).
+ * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector
+ * uses the offset vector to determine the sequence of cells pertaining to an individual value.
+ *
+ */
+public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
+
+  final static int DEFAULT_REPEAT_PER_RECORD = 5;
+
+  /**
+   * Returns the underlying offset vector or null if none exists.
+   *
+   * TODO(DRILL-2995): eliminate exposing low-level interfaces.
+   */
+  UInt4Vector getOffsetVector();
+
+  /**
+   * Returns the underlying data vector or null if none exists.
+   */
+  ValueVector getDataVector();
+
+  @Override
+  RepeatedAccessor getAccessor();
+
+  @Override
+  RepeatedMutator getMutator();
+
+  interface RepeatedAccessor extends ValueVector.Accessor {
+    /**
+     * Returns total number of cells that vector contains.
+     *
+     * The result includes empty, null valued cells.
+     */
+    int getInnerValueCount();
+
+
+    /**
+     * Returns number of cells that the value at the given index contains.
+     */
+    int getInnerValueCountAt(int index);
+
+    /**
+     * Returns true if the value at the given index is empty, false otherwise.
+     *
+     * @param index  value index
+     */
+    boolean isEmpty(int index);
+  }
+
+  interface RepeatedMutator extends ValueVector.Mutator {
+    /**
+     * Starts a new value that is a container of cells.
+     *
+     * @param index  index of new value to start
+     */
+    void startNewValue(int index);
+
+
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
new file mode 100644
index 0000000000000..93b744e108719
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+public interface RepeatedVariableWidthVectorLike {
+  /**
+   * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
+   *
+   * @param totalBytes   Desired size of the underlying data buffer.
+   * @param parentValueCount   Number of separate repeating groupings.
+   * @param childValueCount   Number of supported values in the vector.
+   */
+  void allocateNew(int totalBytes, int parentValueCount, int childValueCount);
+
+  /**
+   * Provide the maximum amount of variable width bytes that can be stored int his vector.
+   * @return
+   */
+  int getByteCapacity();
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
new file mode 100644
index 0000000000000..852c72c549729
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import java.util.Arrays;
+
+public class StateTool {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StateTool.class);
+
+  public static <T extends Enum<?>> void check(T currentState, T... expectedStates) {
+    for (T s : expectedStates) {
+      if (s == currentState) {
+        return;
+      }
+    }
+    throw new IllegalArgumentException(String.format("Expected to be in one of these states %s but was actuall in state %s", Arrays.toString(expectedStates), currentState));
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/VectorWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/VectorWithOrdinal.java
new file mode 100644
index 0000000000000..d04fc1c022c05
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/VectorWithOrdinal.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.ValueVector;
+
+public class VectorWithOrdinal {
+  public final ValueVector vector;
+  public final int ordinal;
+
+  public VectorWithOrdinal(ValueVector v, int ordinal) {
+    this.vector = v;
+    this.ordinal = ordinal;
+  }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
new file mode 100644
index 0000000000000..264e241e73935
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.impl;
+
+import java.util.Iterator;
+
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.holders.UnionHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+
+
+abstract class AbstractBaseReader implements FieldReader{
+
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseReader.class);
+  private static final MajorType LATE_BIND_TYPE = new MajorType(MinorType.LATE, DataMode.OPTIONAL);
+
+  private int index;
+
+  public AbstractBaseReader() {
+    super();
+  }
+
+  public void setPosition(int index){
+    this.index = index;
+  }
+
+  int idx(){
+    return index;
+  }
+
+  @Override
+  public void reset() {
+    index = 0;
+  }
+
+  @Override
+  public Iterator<String> iterator() {
+    throw new IllegalStateException("The current reader doesn't support reading as a map.");
+  }
+
+  public MajorType getType(){
+    throw new IllegalStateException("The current reader doesn't support getting type information.");
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return MaterializedField.create("unknown", LATE_BIND_TYPE);
+  }
+
+  @Override
+  public boolean next() {
+    throw new IllegalStateException("The current reader doesn't support getting next information.");
+  }
+
+  @Override
+  public int size() {
+    throw new IllegalStateException("The current reader doesn't support getting size information.");
+  }
+
+  @Override
+  public void read(UnionHolder holder) {
+    holder.reader = this;
+    holder.isSet = this.isSet() ? 1 : 0;
+  }
+
+  @Override
+  public void read(int index, UnionHolder holder) {
+    throw new IllegalStateException("The current reader doesn't support reading union type");
+  }
+
+  @Override
+  public void copyAsValue(UnionWriter writer) {
+    throw new IllegalStateException("The current reader doesn't support reading union type");
+  }
+
+  @Override
+  public void copyAsValue(ListWriter writer) {
+    ComplexCopier.copy(this, (FieldWriter)writer);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
new file mode 100644
index 0000000000000..4e1e103a12e7c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.impl;
+
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+
+
+abstract class AbstractBaseWriter implements FieldWriter {
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseWriter.class);
+
+  final FieldWriter parent;
+  private int index;
+
+  public AbstractBaseWriter(FieldWriter parent) {
+    this.parent = parent;
+  }
+
+  @Override
+  public String toString() {
+    return super.toString() + "[index = " + index + ", parent = " + parent + "]";
+  }
+
+  @Override
+  public FieldWriter getParent() {
+    return parent;
+  }
+
+  public boolean isRoot() {
+    return parent == null;
+  }
+
+  int idx() {
+    return index;
+  }
+
+  @Override
+  public void setPosition(int index) {
+    this.index = index;
+  }
+
+  @Override
+  public void end() {
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
new file mode 100644
index 0000000000000..4e2051fd4efee
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.impl;
+
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.StateTool;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+
+import com.google.common.base.Preconditions;
+
+public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
+//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
+
+  private SingleMapWriter mapRoot;
+  private SingleListWriter listRoot;
+  private final MapVector container;
+
+  Mode mode = Mode.INIT;
+  private final String name;
+  private final boolean unionEnabled;
+
+  private enum Mode { INIT, MAP, LIST };
+
+  public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){
+    super(null);
+    this.name = name;
+    this.container = container;
+    this.unionEnabled = unionEnabled;
+  }
+
+  public ComplexWriterImpl(String name, MapVector container){
+    this(name, container, false);
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return container.getField();
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return container.getValueCapacity();
+  }
+
+  private void check(Mode... modes){
+    StateTool.check(mode, modes);
+  }
+
+  @Override
+  public void reset(){
+    setPosition(0);
+  }
+
+  @Override
+  public void close() throws Exception {
+    clear();
+    mapRoot.close();
+    if (listRoot != null) {
+      listRoot.close();
+    }
+  }
+
+  @Override
+  public void clear(){
+    switch(mode){
+    case MAP:
+      mapRoot.clear();
+      break;
+    case LIST:
+      listRoot.clear();
+      break;
+    }
+  }
+
+  @Override
+  public void setValueCount(int count){
+    switch(mode){
+    case MAP:
+      mapRoot.setValueCount(count);
+      break;
+    case LIST:
+      listRoot.setValueCount(count);
+      break;
+    }
+  }
+
+  @Override
+  public void setPosition(int index){
+    super.setPosition(index);
+    switch(mode){
+    case MAP:
+      mapRoot.setPosition(index);
+      break;
+    case LIST:
+      listRoot.setPosition(index);
+      break;
+    }
+  }
+
+
+  public MapWriter directMap(){
+    Preconditions.checkArgument(name == null);
+
+    switch(mode){
+
+    case INIT:
+      MapVector map = (MapVector) container;
+      mapRoot = new SingleMapWriter(map, this, unionEnabled);
+      mapRoot.setPosition(idx());
+      mode = Mode.MAP;
+      break;
+
+    case MAP:
+      break;
+
+    default:
+        check(Mode.INIT, Mode.MAP);
+    }
+
+    return mapRoot;
+  }
+
+  @Override
+  public MapWriter rootAsMap() {
+    switch(mode){
+
+    case INIT:
+      MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
+      mapRoot = new SingleMapWriter(map, this, unionEnabled);
+      mapRoot.setPosition(idx());
+      mode = Mode.MAP;
+      break;
+
+    case MAP:
+      break;
+
+    default:
+        check(Mode.INIT, Mode.MAP);
+    }
+
+    return mapRoot;
+  }
+
+
+  @Override
+  public void allocate() {
+    if(mapRoot != null) {
+      mapRoot.allocate();
+    } else if(listRoot != null) {
+      listRoot.allocate();
+    }
+  }
+
+  @Override
+  public ListWriter rootAsList() {
+    switch(mode){
+
+    case INIT:
+      listRoot = new SingleListWriter(name, container, this);
+      listRoot.setPosition(idx());
+      mode = Mode.LIST;
+      break;
+
+    case LIST:
+      break;
+
+    default:
+        check(Mode.INIT, Mode.MAP);
+    }
+
+    return listRoot;
+  }
+
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java
new file mode 100644
index 0000000000000..f8a9d4232aadc
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.impl;
+
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapOrListWriter;
+import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.BitWriter;
+import org.apache.arrow.vector.complex.writer.Float4Writer;
+import org.apache.arrow.vector.complex.writer.Float8Writer;
+import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.complex.writer.VarBinaryWriter;
+import org.apache.arrow.vector.complex.writer.VarCharWriter;
+
+public class MapOrListWriterImpl implements MapOrListWriter {
+
+  public final BaseWriter.MapWriter map;
+  public final BaseWriter.ListWriter list;
+
+  public MapOrListWriterImpl(final BaseWriter.MapWriter writer) {
+    this.map = writer;
+    this.list = null;
+  }
+
+  public MapOrListWriterImpl(final BaseWriter.ListWriter writer) {
+    this.map = null;
+    this.list = writer;
+  }
+
+  public void start() {
+    if (map != null) {
+      map.start();
+    } else {
+      list.startList();
+    }
+  }
+
+  public void end() {
+    if (map != null) {
+      map.end();
+    } else {
+      list.endList();
+    }
+  }
+
+  public MapOrListWriter map(final String name) {
+    assert map != null;
+    return new MapOrListWriterImpl(map.map(name));
+  }
+
+  public MapOrListWriter listoftmap(final String name) {
+    assert list != null;
+    return new MapOrListWriterImpl(list.map());
+  }
+
+  public MapOrListWriter list(final String name) {
+    assert map != null;
+    return new MapOrListWriterImpl(map.list(name));
+  }
+
+  public boolean isMapWriter() {
+    return map != null;
+  }
+
+  public boolean isListWriter() {
+    return list != null;
+  }
+
+  public VarCharWriter varChar(final String name) {
+    return (map != null) ? map.varChar(name) : list.varChar();
+  }
+
+  public IntWriter integer(final String name) {
+    return (map != null) ? map.integer(name) : list.integer();
+  }
+
+  public BigIntWriter bigInt(final String name) {
+    return (map != null) ? map.bigInt(name) : list.bigInt();
+  }
+
+  public Float4Writer float4(final String name) {
+    return (map != null) ? map.float4(name) : list.float4();
+  }
+
+  public Float8Writer float8(final String name) {
+    return (map != null) ? map.float8(name) : list.float8();
+  }
+
+  public BitWriter bit(final String name) {
+    return (map != null) ? map.bit(name) : list.bit();
+  }
+
+  public VarBinaryWriter binary(final String name) {
+    return (map != null) ? map.varBinary(name) : list.varBinary();
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
new file mode 100644
index 0000000000000..ea62e02360802
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.impl;
+
+import java.lang.reflect.Constructor;
+
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.complex.AbstractMapVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * This FieldWriter implementation delegates all FieldWriter API calls to an inner FieldWriter. This inner field writer
+ * can start as a specific type, and this class will promote the writer to a UnionWriter if a call is made that the specifically
+ * typed writer cannot handle. A new UnionVector is created, wrapping the original vector, and replaces the original vector
+ * in the parent vector, which can be either an AbstractMapVector or a ListVector.
+ */
+public class PromotableWriter extends AbstractPromotableFieldWriter {
+
+  private final AbstractMapVector parentContainer;
+  private final ListVector listVector;
+  private int position;
+
+  private enum State {
+    UNTYPED, SINGLE, UNION
+  }
+
+  private MinorType type;
+  private ValueVector vector;
+  private UnionVector unionVector;
+  private State state;
+  private FieldWriter writer;
+
+  public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
+    super(null);
+    this.parentContainer = parentContainer;
+    this.listVector = null;
+    init(v);
+  }
+
+  public PromotableWriter(ValueVector v, ListVector listVector) {
+    super(null);
+    this.listVector = listVector;
+    this.parentContainer = null;
+    init(v);
+  }
+
+  private void init(ValueVector v) {
+    if (v instanceof UnionVector) {
+      state = State.UNION;
+      unionVector = (UnionVector) v;
+      writer = new UnionWriter(unionVector);
+    } else if (v instanceof ZeroVector) {
+      state = State.UNTYPED;
+    } else {
+      setWriter(v);
+    }
+  }
+
+  private void setWriter(ValueVector v) {
+    state = State.SINGLE;
+    vector = v;
+    type = v.getField().getType().getMinorType();
+    Class writerClass = BasicTypeHelper
+        .getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
+    if (writerClass.equals(SingleListWriter.class)) {
+      writerClass = UnionListWriter.class;
+    }
+    Class vectorClass = BasicTypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField()
+        .getDataMode());
+    try {
+      Constructor constructor = null;
+      for (Constructor c : writerClass.getConstructors()) {
+        if (c.getParameterTypes().length == 3) {
+          constructor = c;
+        }
+      }
+      if (constructor == null) {
+        constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class);
+        writer = (FieldWriter) constructor.newInstance(vector, null);
+      } else {
+        writer = (FieldWriter) constructor.newInstance(vector, null, true);
+      }
+    } catch (ReflectiveOperationException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    FieldWriter w = getWriter();
+    if (w == null) {
+      position = index;
+    } else {
+      w.setPosition(index);
+    }
+  }
+
+  protected FieldWriter getWriter(MinorType type) {
+    if (state == State.UNION) {
+      return writer;
+    }
+    if (state == State.UNTYPED) {
+      if (type == null) {
+        return null;
+      }
+      ValueVector v = listVector.addOrGetVector(new VectorDescriptor(new MajorType(type, DataMode.OPTIONAL))).getVector();
+      v.allocateNew();
+      setWriter(v);
+      writer.setPosition(position);
+    }
+    if (type != this.type) {
+      return promoteToUnion();
+    }
+    return writer;
+  }
+
+  @Override
+  public boolean isEmptyMap() {
+    return writer.isEmptyMap();
+  }
+
+  protected FieldWriter getWriter() {
+    return getWriter(type);
+  }
+
+  private FieldWriter promoteToUnion() {
+    String name = vector.getField().getLastName();
+    TransferPair tp = vector.getTransferPair(vector.getField().getType().getMinorType().name().toLowerCase(), vector.getAllocator());
+    tp.transfer();
+    if (parentContainer != null) {
+      unionVector = parentContainer.addOrGet(name, new MajorType(MinorType.UNION, DataMode.OPTIONAL), UnionVector.class);
+    } else if (listVector != null) {
+      unionVector = listVector.promoteToUnion();
+    }
+    unionVector.addVector(tp.getTo());
+    writer = new UnionWriter(unionVector);
+    writer.setPosition(idx());
+    for (int i = 0; i < idx(); i++) {
+      unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
+    }
+    vector = null;
+    state = State.UNION;
+    return writer;
+  }
+
+  @Override
+  public void allocate() {
+    getWriter().allocate();
+  }
+
+  @Override
+  public void clear() {
+    getWriter().clear();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return getWriter().getField();
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return getWriter().getValueCapacity();
+  }
+
+  @Override
+  public void close() throws Exception {
+    getWriter().close();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
new file mode 100644
index 0000000000000..dd1a152e2f603
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
@@ -0,0 +1,145 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex.impl;
+
+
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.RepeatedListVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.holders.RepeatedListHolder;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+
+public class RepeatedListReaderImpl extends AbstractFieldReader{
+  private static final int NO_VALUES = Integer.MAX_VALUE - 1;
+  private static final MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
+  private final String name;
+  private final RepeatedListVector container;
+  private FieldReader reader;
+
+  public RepeatedListReaderImpl(String name, RepeatedListVector container) {
+    super();
+    this.name = name;
+    this.container = container;
+  }
+
+  @Override
+  public MajorType getType() {
+    return TYPE;
+  }
+
+  @Override
+  public void copyAsValue(ListWriter writer) {
+    if (currentOffset == NO_VALUES) {
+      return;
+    }
+    RepeatedListWriter impl = (RepeatedListWriter) writer;
+    impl.container.copyFromSafe(idx(), impl.idx(), container);
+  }
+
+  @Override
+  public void copyAsField(String name, MapWriter writer) {
+    if (currentOffset == NO_VALUES) {
+      return;
+    }
+    RepeatedListWriter impl = (RepeatedListWriter) writer.list(name);
+    impl.container.copyFromSafe(idx(), impl.idx(), container);
+  }
+
+  private int currentOffset;
+  private int maxOffset;
+
+  @Override
+  public void reset() {
+    super.reset();
+    currentOffset = 0;
+    maxOffset = 0;
+    if (reader != null) {
+      reader.reset();
+    }
+    reader = null;
+  }
+
+  @Override
+  public int size() {
+    return maxOffset - currentOffset;
+  }
+
+  @Override
+  public void setPosition(int index) {
+    if (index < 0 || index == NO_VALUES) {
+      currentOffset = NO_VALUES;
+      return;
+    }
+
+    super.setPosition(index);
+    RepeatedListHolder h = new RepeatedListHolder();
+    container.getAccessor().get(index, h);
+    if (h.start == h.end) {
+      currentOffset = NO_VALUES;
+    } else {
+      currentOffset = h.start-1;
+      maxOffset = h.end;
+      if(reader != null) {
+        reader.setPosition(currentOffset);
+      }
+    }
+  }
+
+  @Override
+  public boolean next() {
+    if (currentOffset +1 < maxOffset) {
+      currentOffset++;
+      if (reader != null) {
+        reader.setPosition(currentOffset);
+      }
+      return true;
+    } else {
+      currentOffset = NO_VALUES;
+      return false;
+    }
+  }
+
+  @Override
+  public Object readObject() {
+    return container.getAccessor().getObject(idx());
+  }
+
+  @Override
+  public FieldReader reader() {
+    if (reader == null) {
+      ValueVector child = container.getChild(name);
+      if (child == null) {
+        reader = NullReader.INSTANCE;
+      } else {
+        reader = child.getReader();
+      }
+      reader.setPosition(currentOffset);
+    }
+    return reader;
+  }
+
+  public boolean isSet() {
+    return true;
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
new file mode 100644
index 0000000000000..09a831d8329fc
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
@@ -0,0 +1,192 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex.impl;
+
+import java.util.Map;
+
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.RepeatedMapVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.holders.RepeatedMapHolder;
+import org.apache.arrow.vector.types.Types.MajorType;
+
+import com.google.common.collect.Maps;
+
+@SuppressWarnings("unused")
+public class RepeatedMapReaderImpl extends AbstractFieldReader{
+  private static final int NO_VALUES = Integer.MAX_VALUE - 1;
+
+  private final RepeatedMapVector vector;
+  private final Map<String, FieldReader> fields = Maps.newHashMap();
+
+  public RepeatedMapReaderImpl(RepeatedMapVector vector) {
+    this.vector = vector;
+  }
+
+  private void setChildrenPosition(int index) {
+    for (FieldReader r : fields.values()) {
+      r.setPosition(index);
+    }
+  }
+
+  @Override
+  public FieldReader reader(String name) {
+    FieldReader reader = fields.get(name);
+    if (reader == null) {
+      ValueVector child = vector.getChild(name);
+      if (child == null) {
+        reader = NullReader.INSTANCE;
+      } else {
+        reader = child.getReader();
+      }
+      fields.put(name, reader);
+      reader.setPosition(currentOffset);
+    }
+    return reader;
+  }
+
+  @Override
+  public FieldReader reader() {
+    if (currentOffset == NO_VALUES) {
+      return NullReader.INSTANCE;
+    }
+
+    setChildrenPosition(currentOffset);
+    return new SingleLikeRepeatedMapReaderImpl(vector, this);
+  }
+
+  private int currentOffset;
+  private int maxOffset;
+
+  @Override
+  public void reset() {
+    super.reset();
+    currentOffset = 0;
+    maxOffset = 0;
+    for (FieldReader reader:fields.values()) {
+      reader.reset();
+    }
+    fields.clear();
+  }
+
+  @Override
+  public int size() {
+    if (isNull()) {
+      return 0;
+    }
+    return maxOffset - (currentOffset < 0 ? 0 : currentOffset);
+  }
+
+  @Override
+  public void setPosition(int index) {
+    if (index < 0 || index == NO_VALUES) {
+      currentOffset = NO_VALUES;
+      return;
+    }
+
+    super.setPosition(index);
+    RepeatedMapHolder h = new RepeatedMapHolder();
+    vector.getAccessor().get(index, h);
+    if (h.start == h.end) {
+      currentOffset = NO_VALUES;
+    } else {
+      currentOffset = h.start-1;
+      maxOffset = h.end;
+      setChildrenPosition(currentOffset);
+    }
+  }
+
+  public void setSinglePosition(int index, int childIndex) {
+    super.setPosition(index);
+    RepeatedMapHolder h = new RepeatedMapHolder();
+    vector.getAccessor().get(index, h);
+    if (h.start == h.end) {
+      currentOffset = NO_VALUES;
+    } else {
+      int singleOffset = h.start + childIndex;
+      assert singleOffset < h.end;
+      currentOffset = singleOffset;
+      maxOffset = singleOffset + 1;
+      setChildrenPosition(singleOffset);
+    }
+  }
+
+  @Override
+  public boolean next() {
+    if (currentOffset +1 < maxOffset) {
+      setChildrenPosition(++currentOffset);
+      return true;
+    } else {
+      currentOffset = NO_VALUES;
+      return false;
+    }
+  }
+
+  public boolean isNull() {
+    return currentOffset == NO_VALUES;
+  }
+
+  @Override
+  public Object readObject() {
+    return vector.getAccessor().getObject(idx());
+  }
+
+  @Override
+  public MajorType getType() {
+    return vector.getField().getType();
+  }
+
+  @Override
+  public java.util.Iterator<String> iterator() {
+    return vector.fieldNameIterator();
+  }
+
+  @Override
+  public boolean isSet() {
+    return true;
+  }
+
+  @Override
+  public void copyAsValue(MapWriter writer) {
+    if (currentOffset == NO_VALUES) {
+      return;
+    }
+    RepeatedMapWriter impl = (RepeatedMapWriter) writer;
+    impl.container.copyFromSafe(idx(), impl.idx(), vector);
+  }
+
+  public void copyAsValueSingle(MapWriter writer) {
+    if (currentOffset == NO_VALUES) {
+      return;
+    }
+    SingleMapWriter impl = (SingleMapWriter) writer;
+    impl.container.copyFromSafe(currentOffset, impl.idx(), vector);
+  }
+
+  @Override
+  public void copyAsField(String name, MapWriter writer) {
+    if (currentOffset == NO_VALUES) {
+      return;
+    }
+    RepeatedMapWriter impl = (RepeatedMapWriter) writer.map(name);
+    impl.container.copyFromSafe(idx(), impl.idx(), vector);
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
new file mode 100644
index 0000000000000..086d26e119440
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.complex.impl;
+
+import java.util.Iterator;
+
+import org.apache.arrow.vector.complex.RepeatedMapVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+
+public class SingleLikeRepeatedMapReaderImpl extends AbstractFieldReader{
+
+  private RepeatedMapReaderImpl delegate;
+
+  public SingleLikeRepeatedMapReaderImpl(RepeatedMapVector vector, FieldReader delegate) {
+    this.delegate = (RepeatedMapReaderImpl) delegate;
+  }
+
+  @Override
+  public int size() {
+    throw new UnsupportedOperationException("You can't call size on a single map reader.");
+  }
+
+  @Override
+  public boolean next() {
+    throw new UnsupportedOperationException("You can't call next on a single map reader.");
+  }
+
+  @Override
+  public MajorType getType() {
+    return Types.required(MinorType.MAP);
+  }
+
+
+  @Override
+  public void copyAsValue(MapWriter writer) {
+    delegate.copyAsValueSingle(writer);
+  }
+
+  public void copyAsValueSingle(MapWriter writer){
+    delegate.copyAsValueSingle(writer);
+  }
+
+  @Override
+  public FieldReader reader(String name) {
+    return delegate.reader(name);
+  }
+
+  @Override
+  public void setPosition(int index) {
+    delegate.setPosition(index);
+  }
+
+  @Override
+  public Object readObject() {
+    return delegate.readObject();
+  }
+
+  @Override
+  public Iterator<String> iterator() {
+    return delegate.iterator();
+  }
+
+  @Override
+  public boolean isSet() {
+    return ! delegate.isNull();
+  }
+
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
new file mode 100644
index 0000000000000..f16f628603d69
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
@@ -0,0 +1,88 @@
+
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex.impl;
+
+
+import org.apache.arrow.vector.complex.AbstractContainerVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+
+@SuppressWarnings("unused")
+public class SingleListReaderImpl extends AbstractFieldReader{
+
+  private static final MajorType TYPE = Types.optional(MinorType.LIST);
+  private final String name;
+  private final AbstractContainerVector container;
+  private FieldReader reader;
+
+  public SingleListReaderImpl(String name, AbstractContainerVector container) {
+    super();
+    this.name = name;
+    this.container = container;
+  }
+
+  @Override
+  public MajorType getType() {
+    return TYPE;
+  }
+
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    if (reader != null) {
+      reader.setPosition(index);
+    }
+  }
+
+  @Override
+  public Object readObject() {
+    return reader.readObject();
+  }
+
+  @Override
+  public FieldReader reader() {
+    if (reader == null) {
+      reader = container.getChild(name).getReader();
+      setPosition(idx());
+    }
+    return reader;
+  }
+
+  @Override
+  public boolean isSet() {
+    return false;
+  }
+
+  @Override
+  public void copyAsValue(ListWriter writer) {
+    throw new UnsupportedOperationException("Generic list copying not yet supported.  Please resolve to typed list.");
+  }
+
+  @Override
+  public void copyAsField(String name, MapWriter writer) {
+    throw new UnsupportedOperationException("Generic list copying not yet supported.  Please resolve to typed list.");
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
new file mode 100644
index 0000000000000..84b99801419c4
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -0,0 +1,108 @@
+
+
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex.impl;
+
+
+import java.util.Map;
+
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.Types.MajorType;
+
+import com.google.common.collect.Maps;
+
+@SuppressWarnings("unused")
+public class SingleMapReaderImpl extends AbstractFieldReader{
+
+  private final MapVector vector;
+  private final Map<String, FieldReader> fields = Maps.newHashMap();
+
+  public SingleMapReaderImpl(MapVector vector) {
+    this.vector = vector;
+  }
+
+  private void setChildrenPosition(int index){
+    for(FieldReader r : fields.values()){
+      r.setPosition(index);
+    }
+  }
+
+  @Override
+  public FieldReader reader(String name){
+    FieldReader reader = fields.get(name);
+    if(reader == null){
+      ValueVector child = vector.getChild(name);
+      if(child == null){
+        reader = NullReader.INSTANCE;
+      }else{
+        reader = child.getReader();
+      }
+      fields.put(name, reader);
+      reader.setPosition(idx());
+    }
+    return reader;
+  }
+
+  @Override
+  public void setPosition(int index){
+    super.setPosition(index);
+    for(FieldReader r : fields.values()){
+      r.setPosition(index);
+    }
+  }
+
+  @Override
+  public Object readObject() {
+    return vector.getAccessor().getObject(idx());
+  }
+
+  @Override
+  public boolean isSet() {
+    return true;
+  }
+
+  @Override
+  public MajorType getType(){
+    return vector.getField().getType();
+  }
+
+  @Override
+  public java.util.Iterator<String> iterator(){
+    return vector.fieldNameIterator();
+  }
+
+  @Override
+  public void copyAsValue(MapWriter writer){
+    SingleMapWriter impl = (SingleMapWriter) writer;
+    impl.container.copyFromSafe(idx(), impl.idx(), vector);
+  }
+
+  @Override
+  public void copyAsField(String name, MapWriter writer){
+    SingleMapWriter impl = (SingleMapWriter) writer.map(name);
+    impl.container.copyFromSafe(idx(), impl.idx(), vector);
+  }
+
+
+}
+
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
new file mode 100644
index 0000000000000..9b54d02e571de
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
@@ -0,0 +1,98 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex.impl;
+
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.holders.UnionHolder;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+
+public class UnionListReader extends AbstractFieldReader {
+
+  private ListVector vector;
+  private ValueVector data;
+  private UInt4Vector offsets;
+
+  public UnionListReader(ListVector vector) {
+    this.vector = vector;
+    this.data = vector.getDataVector();
+    this.offsets = vector.getOffsetVector();
+  }
+
+  @Override
+  public boolean isSet() {
+    return true;
+  }
+
+  MajorType type = new MajorType(MinorType.LIST, DataMode.OPTIONAL);
+
+  public MajorType getType() {
+    return type;
+  }
+
+  private int currentOffset;
+  private int maxOffset;
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    currentOffset = offsets.getAccessor().get(index) - 1;
+    maxOffset = offsets.getAccessor().get(index + 1);
+  }
+
+  @Override
+  public FieldReader reader() {
+    return data.getReader();
+  }
+
+  @Override
+  public Object readObject() {
+    return vector.getAccessor().getObject(idx());
+  }
+
+  @Override
+  public void read(int index, UnionHolder holder) {
+    setPosition(idx());
+    for (int i = -1; i < index; i++) {
+      next();
+    }
+    holder.reader = data.getReader();
+    holder.isSet = data.getReader().isSet() ? 1 : 0;
+  }
+
+  @Override
+  public boolean next() {
+    if (currentOffset + 1 < maxOffset) {
+      data.getReader().setPosition(++currentOffset);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  public void copyAsValue(ListWriter writer) {
+    ComplexCopier.copy(this, (FieldWriter) writer);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
new file mode 100644
index 0000000000000..c4eb3dc739a49
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.reader;
+
+import org.apache.arrow.vector.complex.reader.BaseReader.ListReader;
+import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.reader.BaseReader.RepeatedListReader;
+import org.apache.arrow.vector.complex.reader.BaseReader.RepeatedMapReader;
+import org.apache.arrow.vector.complex.reader.BaseReader.ScalarReader;
+
+
+
+public interface FieldReader extends MapReader, ListReader, ScalarReader, RepeatedMapReader, RepeatedListReader {
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
new file mode 100644
index 0000000000000..ecffe0bec0e84
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.writer;
+
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ScalarWriter;
+
+public interface FieldWriter extends MapWriter, ListWriter, ScalarWriter {
+  void allocate();
+  void clear();
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ComplexHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ComplexHolder.java
new file mode 100644
index 0000000000000..0f9310da55b79
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/ComplexHolder.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.holders;
+
+import org.apache.arrow.vector.complex.reader.FieldReader;
+
+public class ComplexHolder implements ValueHolder {
+  public FieldReader reader;
+  public int isSet;
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
new file mode 100644
index 0000000000000..5a5fe0305d83a
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.holders;
+
+import org.apache.arrow.vector.types.Types;
+
+/*
+ * Holder class for the vector ObjectVector. This holder internally stores a
+ * reference to an object. The ObjectVector maintains an array of these objects.
+ * This holder can be used only as workspace variables in aggregate functions.
+ * Using this holder should be avoided and we should stick to native holder types.
+ */
+@Deprecated
+public class ObjectHolder implements ValueHolder {
+  public static final Types.MajorType TYPE = Types.required(Types.MinorType.GENERIC_OBJECT);
+
+  public Types.MajorType getType() {
+    return TYPE;
+  }
+
+  public Object obj;
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedListHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedListHolder.java
new file mode 100644
index 0000000000000..83506cdc17549
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedListHolder.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.holders;
+
+public final class RepeatedListHolder implements ValueHolder{
+  public int start;
+  public int end;
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java
new file mode 100644
index 0000000000000..85d782b381835
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.holders;
+
+public final class RepeatedMapHolder implements ValueHolder{
+  public int start;
+  public int end;
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
new file mode 100644
index 0000000000000..b868a620f985b
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.holders;
+
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+
+public class UnionHolder implements ValueHolder {
+  public static final MajorType TYPE = new MajorType(MinorType.UNION, DataMode.OPTIONAL);
+  public FieldReader reader;
+  public int isSet;
+
+  public MajorType getType() {
+    return reader.getType();
+  }
+
+  public boolean isSet() {
+    return isSet == 1;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
new file mode 100644
index 0000000000000..88cbcd4a8c308
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.holders;
+
+/**
+ * Wrapper object for an individual value in Drill.
+ *
+ * ValueHolders are designed to be mutable wrapper objects for defining clean
+ * APIs that access data in Drill. For performance, object creation is avoided
+ * at all costs throughout execution. For this reason, ValueHolders are
+ * disallowed from implementing any methods, this allows for them to be
+ * replaced by their java primitive inner members during optimization of
+ * run-time generated code.
+ */
+public interface ValueHolder {
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java b/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
new file mode 100644
index 0000000000000..c73098b2a85d7
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.types;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Objects;
+
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+
+
+public class MaterializedField {
+  private final String name;
+  private final MajorType type;
+  // use an ordered set as existing code relies on order (e,g. parquet writer)
+  private final LinkedHashSet<MaterializedField> children;
+
+  MaterializedField(String name, MajorType type, LinkedHashSet<MaterializedField> children) {
+    this.name = name;
+    this.type = type;
+    this.children = children;
+  }
+
+  public Collection<MaterializedField> getChildren() {
+    return new ArrayList<>(children);
+  }
+
+  public MaterializedField newWithChild(MaterializedField child) {
+    MaterializedField newField = clone();
+    newField.addChild(child);
+    return newField;
+  }
+
+  public void addChild(MaterializedField field){
+    children.add(field);
+  }
+
+  public MaterializedField clone() {
+    return withPathAndType(name, getType());
+  }
+
+  public MaterializedField withType(MajorType type) {
+    return withPathAndType(name, type);
+  }
+
+  public MaterializedField withPath(String name) {
+    return withPathAndType(name, getType());
+  }
+
+  public MaterializedField withPathAndType(String name, final MajorType type) {
+    final LinkedHashSet<MaterializedField> newChildren = new LinkedHashSet<>(children.size());
+    for (final MaterializedField child:children) {
+      newChildren.add(child.clone());
+    }
+    return new MaterializedField(name, type, newChildren);
+  }
+
+//  public String getLastName(){
+//    PathSegment seg = key.path.getRootSegment();
+//    while (seg.getChild() != null) {
+//      seg = seg.getChild();
+//    }
+//    return seg.getNameSegment().getPath();
+//  }
+
+
+  // TODO: rewrite without as direct match rather than conversion then match.
+//  public boolean matches(SerializedField booleanfield){
+//    MaterializedField f = create(field);
+//    return f.equals(this);
+//  }
+
+  public static MaterializedField create(String name, MajorType type){
+    return new MaterializedField(name, type, new LinkedHashSet<MaterializedField>());
+  }
+
+//  public String getName(){
+//    StringBuilder sb = new StringBuilder();
+//    boolean first = true;
+//    for(NamePart np : def.getNameList()){
+//      if(np.getType() == Type.ARRAY){
+//        sb.append("[]");
+//      }else{
+//        if(first){
+//          first = false;
+//        }else{
+//          sb.append(".");
+//        }
+//        sb.append('`');
+//        sb.append(np.getName());
+//        sb.append('`');
+//
+//      }
+//    }
+//    return sb.toString();
+//  }
+
+  public String getPath() {
+    return getName();
+  }
+
+  public String getLastName() {
+    return getName();
+  }
+
+  public String getName() {
+    return name;
+  }
+
+//  public int getWidth() {
+//    return type.getWidth();
+//  }
+
+  public MajorType getType() {
+    return type;
+  }
+
+  public int getScale() {
+      return type.getScale();
+  }
+  public int getPrecision() {
+      return type.getPrecision();
+  }
+  public boolean isNullable() {
+    return type.getMode() == DataMode.OPTIONAL;
+  }
+
+  public DataMode getDataMode() {
+    return type.getMode();
+  }
+
+  public MaterializedField getOtherNullableVersion(){
+    MajorType mt = type;
+    DataMode newDataMode = null;
+    switch (mt.getMode()){
+    case OPTIONAL:
+      newDataMode = DataMode.REQUIRED;
+      break;
+    case REQUIRED:
+      newDataMode = DataMode.OPTIONAL;
+      break;
+    default:
+      throw new UnsupportedOperationException();
+    }
+    return new MaterializedField(name, new MajorType(mt.getMinorType(), newDataMode, mt.getPrecision(), mt.getScale(), mt.getTimezone(), mt.getSubTypes()), children);
+  }
+
+  public Class<?> getValueClass() {
+    return BasicTypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(this.name, this.type, this.children);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+    MaterializedField other = (MaterializedField) obj;
+    // DRILL-1872: Compute equals only on key. See also the comment
+    // in MapVector$MapTransferPair
+
+    return this.name.equalsIgnoreCase(other.name) &&
+            Objects.equals(this.type, other.type);
+  }
+
+
+  @Override
+  public String toString() {
+    final int maxLen = 10;
+    String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : "";
+    return name + "(" + type.getMinorType().name() + ":" + type.getMode().name() + ")" + childStr;
+  }
+
+
+  private String toString(Collection<?> collection, int maxLen) {
+    StringBuilder builder = new StringBuilder();
+    builder.append("[");
+    int i = 0;
+    for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
+      if (i > 0){
+        builder.append(", ");
+      }
+      builder.append(iterator.next());
+    }
+    builder.append("]");
+    return builder.toString();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
new file mode 100644
index 0000000000000..cef892ce88030
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.types;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class Types {
+  public enum MinorType {
+    LATE,   //  late binding type
+    MAP,   //  an empty map column.  Useful for conceptual setup.  Children listed within here
+
+    TINYINT,   //  single byte signed integer
+    SMALLINT,   //  two byte signed integer
+    INT,   //  four byte signed integer
+    BIGINT,   //  eight byte signed integer
+    DECIMAL9,   //  a decimal supporting precision between 1 and 9
+    DECIMAL18,   //  a decimal supporting precision between 10 and 18
+    DECIMAL28SPARSE,   //  a decimal supporting precision between 19 and 28
+    DECIMAL38SPARSE,   //  a decimal supporting precision between 29 and 38
+    MONEY,   //  signed decimal with two digit precision
+    DATE,   //  days since 4713bc
+    TIME,   //  time in micros before or after 2000/1/1
+    TIMETZ,  //  time in micros before or after 2000/1/1 with timezone
+    TIMESTAMPTZ,   //  unix epoch time in millis
+    TIMESTAMP,   //  TBD
+    INTERVAL,   //  TBD
+    FLOAT4,   //  4 byte ieee 754
+    FLOAT8,   //  8 byte ieee 754
+    BIT,  //  single bit value (boolean)
+    FIXEDCHAR,  //  utf8 fixed length string, padded with spaces
+    FIXED16CHAR,
+    FIXEDBINARY,   //  fixed length binary, padded with 0 bytes
+    VARCHAR,   //  utf8 variable length string
+    VAR16CHAR, // utf16 variable length string
+    VARBINARY,   //  variable length binary
+    UINT1,  //  unsigned 1 byte integer
+    UINT2,  //  unsigned 2 byte integer
+    UINT4,   //  unsigned 4 byte integer
+    UINT8,   //  unsigned 8 byte integer
+    DECIMAL28DENSE, // dense decimal representation, supporting precision between 19 and 28
+    DECIMAL38DENSE, // dense decimal representation, supporting precision between 28 and 38
+    NULL, // a value of unknown type (e.g. a missing reference).
+    INTERVALYEAR, // Interval type specifying YEAR to MONTH
+    INTERVALDAY, // Interval type specifying DAY to SECONDS
+    LIST,
+    GENERIC_OBJECT,
+    UNION
+  }
+
+  public enum DataMode {
+    REQUIRED,
+    OPTIONAL,
+    REPEATED
+  }
+
+  public static class MajorType {
+    private MinorType minorType;
+    private DataMode mode;
+    private Integer precision;
+    private Integer scale;
+    private Integer timezone;
+    private List<MinorType> subTypes;
+
+    public MajorType(MinorType minorType, DataMode mode) {
+      this(minorType, mode, null, null, null, null);
+    }
+
+    public MajorType(MinorType minorType, DataMode mode, Integer precision, Integer scale) {
+      this(minorType, mode, precision, scale, null, null);
+    }
+
+    public MajorType(MinorType minorType, DataMode mode, Integer precision, Integer scale, Integer timezone, List<MinorType> subTypes) {
+      this.minorType = minorType;
+      this.mode = mode;
+      this.precision = precision;
+      this.scale = scale;
+      this.timezone = timezone;
+      this.subTypes = subTypes;
+    }
+
+    public MinorType getMinorType() {
+      return minorType;
+    }
+
+    public DataMode getMode() {
+      return mode;
+    }
+
+    public Integer getPrecision() {
+      return precision;
+    }
+
+    public Integer getScale() {
+      return scale;
+    }
+
+    public Integer getTimezone() {
+      return timezone;
+    }
+
+    public List<MinorType> getSubTypes() {
+      return subTypes;
+    }
+  }
+
+  public static MajorType required(MinorType minorType) {
+    return new MajorType(minorType, DataMode.REQUIRED);
+  }
+  public static MajorType optional(MinorType minorType) {
+    return new MajorType(minorType, DataMode.OPTIONAL);
+  }
+  public static MajorType repeated(MinorType minorType) {
+    return new MajorType(minorType, DataMode.REPEATED);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
new file mode 100644
index 0000000000000..2bdfd70b22956
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
@@ -0,0 +1,233 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.util;
+
+import io.netty.buffer.ArrowBuf;
+import io.netty.util.internal.PlatformDependent;
+
+import org.apache.arrow.memory.BoundsChecking;
+
+import com.google.common.primitives.UnsignedLongs;
+
+public class ByteFunctionHelpers {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
+
+  /**
+   * Helper function to check for equality of bytes in two DrillBuffers
+   *
+   * @param left Left DrillBuf for comparison
+   * @param lStart start offset in the buffer
+   * @param lEnd end offset in the buffer
+   * @param right Right DrillBuf for comparison
+   * @param rStart start offset in the buffer
+   * @param rEnd end offset in the buffer
+   * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+   */
+  public static final int equal(final ArrowBuf left, int lStart, int lEnd, final ArrowBuf right, int rStart, int rEnd){
+    if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
+      left.checkBytes(lStart, lEnd);
+      right.checkBytes(rStart, rEnd);
+    }
+    return memEqual(left.memoryAddress(), lStart, lEnd, right.memoryAddress(), rStart, rEnd);
+  }
+
+  private static final int memEqual(final long laddr, int lStart, int lEnd, final long raddr, int rStart,
+      final int rEnd) {
+
+    int n = lEnd - lStart;
+    if (n == rEnd - rStart) {
+      long lPos = laddr + lStart;
+      long rPos = raddr + rStart;
+
+      while (n > 7) {
+        long leftLong = PlatformDependent.getLong(lPos);
+        long rightLong = PlatformDependent.getLong(rPos);
+        if (leftLong != rightLong) {
+          return 0;
+        }
+        lPos += 8;
+        rPos += 8;
+        n -= 8;
+      }
+      while (n-- != 0) {
+        byte leftByte = PlatformDependent.getByte(lPos);
+        byte rightByte = PlatformDependent.getByte(rPos);
+        if (leftByte != rightByte) {
+          return 0;
+        }
+        lPos++;
+        rPos++;
+      }
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  /**
+   * Helper function to compare a set of bytes in two DrillBuffers.
+   *
+   * Function will check data before completing in the case that
+   *
+   * @param left Left DrillBuf to compare
+   * @param lStart start offset in the buffer
+   * @param lEnd end offset in the buffer
+   * @param right Right DrillBuf to compare
+   * @param rStart start offset in the buffer
+   * @param rEnd end offset in the buffer
+   * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+   */
+  public static final int compare(final ArrowBuf left, int lStart, int lEnd, final ArrowBuf right, int rStart, int rEnd){
+    if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
+      left.checkBytes(lStart, lEnd);
+      right.checkBytes(rStart, rEnd);
+    }
+    return memcmp(left.memoryAddress(), lStart, lEnd, right.memoryAddress(), rStart, rEnd);
+  }
+
+  private static final int memcmp(final long laddr, int lStart, int lEnd, final long raddr, int rStart, final int rEnd) {
+    int lLen = lEnd - lStart;
+    int rLen = rEnd - rStart;
+    int n = Math.min(rLen, lLen);
+    long lPos = laddr + lStart;
+    long rPos = raddr + rStart;
+
+    while (n > 7) {
+      long leftLong = PlatformDependent.getLong(lPos);
+      long rightLong = PlatformDependent.getLong(rPos);
+      if (leftLong != rightLong) {
+        return UnsignedLongs.compare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+      }
+      lPos += 8;
+      rPos += 8;
+      n -= 8;
+    }
+
+    while (n-- != 0) {
+      byte leftByte = PlatformDependent.getByte(lPos);
+      byte rightByte = PlatformDependent.getByte(rPos);
+      if (leftByte != rightByte) {
+        return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+      }
+      lPos++;
+      rPos++;
+    }
+
+    if (lLen == rLen) {
+      return 0;
+    }
+
+    return lLen > rLen ? 1 : -1;
+
+  }
+
+  /**
+   * Helper function to compare a set of bytes in DrillBuf to a ByteArray.
+   *
+   * @param left Left DrillBuf for comparison purposes
+   * @param lStart start offset in the buffer
+   * @param lEnd end offset in the buffer
+   * @param right second input to be compared
+   * @param rStart start offset in the byte array
+   * @param rEnd end offset in the byte array
+   * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+   */
+  public static final int compare(final ArrowBuf left, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
+    if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
+      left.checkBytes(lStart, lEnd);
+    }
+    return memcmp(left.memoryAddress(), lStart, lEnd, right, rStart, rEnd);
+  }
+
+
+  private static final int memcmp(final long laddr, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
+    int lLen = lEnd - lStart;
+    int rLen = rEnd - rStart;
+    int n = Math.min(rLen, lLen);
+    long lPos = laddr + lStart;
+    int rPos = rStart;
+
+    while (n-- != 0) {
+      byte leftByte = PlatformDependent.getByte(lPos);
+      byte rightByte = right[rPos];
+      if (leftByte != rightByte) {
+        return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+      }
+      lPos++;
+      rPos++;
+    }
+
+    if (lLen == rLen) {
+      return 0;
+    }
+
+    return lLen > rLen ? 1 : -1;
+  }
+
+  /*
+   * Following are helper functions to interact with sparse decimal represented in a byte array.
+   */
+
+  // Get the integer ignore the sign
+  public static int getInteger(byte[] b, int index) {
+    return getInteger(b, index, true);
+  }
+  // Get the integer, ignore the sign
+  public static int getInteger(byte[] b, int index, boolean ignoreSign) {
+    int startIndex = index * DecimalUtility.INTEGER_SIZE;
+
+    if (index == 0 && ignoreSign == true) {
+      return (b[startIndex + 3] & 0xFF) |
+             (b[startIndex + 2] & 0xFF) << 8 |
+             (b[startIndex + 1] & 0xFF) << 16 |
+             (b[startIndex] & 0x7F) << 24;
+    }
+
+    return ((b[startIndex + 3] & 0xFF) |
+        (b[startIndex + 2] & 0xFF) << 8 |
+        (b[startIndex + 1] & 0xFF) << 16 |
+        (b[startIndex] & 0xFF) << 24);
+
+  }
+
+  // Set integer in the byte array
+  public static void setInteger(byte[] b, int index, int value) {
+    int startIndex = index * DecimalUtility.INTEGER_SIZE;
+    b[startIndex] = (byte) ((value >> 24) & 0xFF);
+    b[startIndex + 1] = (byte) ((value >> 16) & 0xFF);
+    b[startIndex + 2] = (byte) ((value >> 8) & 0xFF);
+    b[startIndex + 3] = (byte) ((value) & 0xFF);
+  }
+
+  // Set the sign in a sparse decimal representation
+  public static void setSign(byte[] b, boolean sign) {
+    int value = getInteger(b, 0);
+    if (sign == true) {
+      setInteger(b, 0, value | 0x80000000);
+    } else {
+      setInteger(b, 0, value & 0x7FFFFFFF);
+    }
+  }
+
+  // Get the sign
+  public static boolean getSign(byte[] b) {
+    return ((getInteger(b, 0, false) & 0x80000000) != 0);
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/CallBack.java b/java/vector/src/main/java/org/apache/arrow/vector/util/CallBack.java
new file mode 100644
index 0000000000000..249834270b3fe
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/CallBack.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+
+public interface CallBack {
+  public void doWork();
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
new file mode 100644
index 0000000000000..1eb2c13cd4c59
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import java.math.BigDecimal;
+
+import org.apache.arrow.vector.types.Types;
+
+public class CoreDecimalUtility {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoreDecimalUtility.class);
+
+  public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+    // Truncate or pad to set the input to the correct scale
+    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+    return (input.unscaledValue().longValue());
+  }
+
+  public static int getMaxPrecision(Types.MinorType decimalType) {
+    if (decimalType == Types.MinorType.DECIMAL9) {
+      return 9;
+    } else if (decimalType == Types.MinorType.DECIMAL18) {
+      return 18;
+    } else if (decimalType == Types.MinorType.DECIMAL28SPARSE) {
+      return 28;
+    } else if (decimalType == Types.MinorType.DECIMAL38SPARSE) {
+      return 38;
+    }
+    return 0;
+  }
+
+  /*
+   * Function returns the Minor decimal type given the precision
+   */
+  public static Types.MinorType getDecimalDataType(int precision) {
+    if (precision <= 9) {
+      return Types.MinorType.DECIMAL9;
+    } else if (precision <= 18) {
+      return Types.MinorType.DECIMAL18;
+    } else if (precision <= 28) {
+      return Types.MinorType.DECIMAL28SPARSE;
+    } else {
+      return Types.MinorType.DECIMAL38SPARSE;
+    }
+  }
+
+  /*
+   * Given a precision it provides the max precision of that decimal data type;
+   * For eg: given the precision 12, we would use DECIMAL18 to store the data
+   * which has a max precision range of 18 digits
+   */
+  public static int getPrecisionRange(int precision) {
+    return getMaxPrecision(getDecimalDataType(precision));
+  }
+  public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) {
+    // Truncate/ or pad to set the input to the correct scale
+    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+    return (input.unscaledValue().intValue());
+  }
+
+  /*
+   * Helper function to detect if the given data type is Decimal
+   */
+  public static boolean isDecimalType(Types.MajorType type) {
+    return isDecimalType(type.getMinorType());
+  }
+
+  public static boolean isDecimalType(Types.MinorType minorType) {
+    if (minorType == Types.MinorType.DECIMAL9 || minorType == Types.MinorType.DECIMAL18 ||
+        minorType == Types.MinorType.DECIMAL28SPARSE || minorType == Types.MinorType.DECIMAL38SPARSE) {
+      return true;
+    }
+    return false;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
new file mode 100644
index 0000000000000..f4fc1736032c0
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
@@ -0,0 +1,682 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.util;
+
+import org.joda.time.Period;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.DateTimeFormatterBuilder;
+import org.joda.time.format.DateTimeParser;
+
+import com.carrotsearch.hppc.ObjectIntHashMap;
+
+// Utility class for Date, DateTime, TimeStamp, Interval data types
+public class DateUtility {
+
+
+    /* We have a hashmap that stores the timezone as the key and an index as the value
+     * While storing the timezone in value vectors, holders we only use this index. As we
+     * reconstruct the timestamp, we use this index to index through the array timezoneList
+     * and get the corresponding timezone and pass it to joda-time
+     */
+  public static ObjectIntHashMap<String> timezoneMap = new ObjectIntHashMap<String>();
+
+    public static String[] timezoneList =  {"Africa/Abidjan",
+                                            "Africa/Accra",
+                                            "Africa/Addis_Ababa",
+                                            "Africa/Algiers",
+                                            "Africa/Asmara",
+                                            "Africa/Asmera",
+                                            "Africa/Bamako",
+                                            "Africa/Bangui",
+                                            "Africa/Banjul",
+                                            "Africa/Bissau",
+                                            "Africa/Blantyre",
+                                            "Africa/Brazzaville",
+                                            "Africa/Bujumbura",
+                                            "Africa/Cairo",
+                                            "Africa/Casablanca",
+                                            "Africa/Ceuta",
+                                            "Africa/Conakry",
+                                            "Africa/Dakar",
+                                            "Africa/Dar_es_Salaam",
+                                            "Africa/Djibouti",
+                                            "Africa/Douala",
+                                            "Africa/El_Aaiun",
+                                            "Africa/Freetown",
+                                            "Africa/Gaborone",
+                                            "Africa/Harare",
+                                            "Africa/Johannesburg",
+                                            "Africa/Juba",
+                                            "Africa/Kampala",
+                                            "Africa/Khartoum",
+                                            "Africa/Kigali",
+                                            "Africa/Kinshasa",
+                                            "Africa/Lagos",
+                                            "Africa/Libreville",
+                                            "Africa/Lome",
+                                            "Africa/Luanda",
+                                            "Africa/Lubumbashi",
+                                            "Africa/Lusaka",
+                                            "Africa/Malabo",
+                                            "Africa/Maputo",
+                                            "Africa/Maseru",
+                                            "Africa/Mbabane",
+                                            "Africa/Mogadishu",
+                                            "Africa/Monrovia",
+                                            "Africa/Nairobi",
+                                            "Africa/Ndjamena",
+                                            "Africa/Niamey",
+                                            "Africa/Nouakchott",
+                                            "Africa/Ouagadougou",
+                                            "Africa/Porto-Novo",
+                                            "Africa/Sao_Tome",
+                                            "Africa/Timbuktu",
+                                            "Africa/Tripoli",
+                                            "Africa/Tunis",
+                                            "Africa/Windhoek",
+                                            "America/Adak",
+                                            "America/Anchorage",
+                                            "America/Anguilla",
+                                            "America/Antigua",
+                                            "America/Araguaina",
+                                            "America/Argentina/Buenos_Aires",
+                                            "America/Argentina/Catamarca",
+                                            "America/Argentina/ComodRivadavia",
+                                            "America/Argentina/Cordoba",
+                                            "America/Argentina/Jujuy",
+                                            "America/Argentina/La_Rioja",
+                                            "America/Argentina/Mendoza",
+                                            "America/Argentina/Rio_Gallegos",
+                                            "America/Argentina/Salta",
+                                            "America/Argentina/San_Juan",
+                                            "America/Argentina/San_Luis",
+                                            "America/Argentina/Tucuman",
+                                            "America/Argentina/Ushuaia",
+                                            "America/Aruba",
+                                            "America/Asuncion",
+                                            "America/Atikokan",
+                                            "America/Atka",
+                                            "America/Bahia",
+                                            "America/Bahia_Banderas",
+                                            "America/Barbados",
+                                            "America/Belem",
+                                            "America/Belize",
+                                            "America/Blanc-Sablon",
+                                            "America/Boa_Vista",
+                                            "America/Bogota",
+                                            "America/Boise",
+                                            "America/Buenos_Aires",
+                                            "America/Cambridge_Bay",
+                                            "America/Campo_Grande",
+                                            "America/Cancun",
+                                            "America/Caracas",
+                                            "America/Catamarca",
+                                            "America/Cayenne",
+                                            "America/Cayman",
+                                            "America/Chicago",
+                                            "America/Chihuahua",
+                                            "America/Coral_Harbour",
+                                            "America/Cordoba",
+                                            "America/Costa_Rica",
+                                            "America/Cuiaba",
+                                            "America/Curacao",
+                                            "America/Danmarkshavn",
+                                            "America/Dawson",
+                                            "America/Dawson_Creek",
+                                            "America/Denver",
+                                            "America/Detroit",
+                                            "America/Dominica",
+                                            "America/Edmonton",
+                                            "America/Eirunepe",
+                                            "America/El_Salvador",
+                                            "America/Ensenada",
+                                            "America/Fort_Wayne",
+                                            "America/Fortaleza",
+                                            "America/Glace_Bay",
+                                            "America/Godthab",
+                                            "America/Goose_Bay",
+                                            "America/Grand_Turk",
+                                            "America/Grenada",
+                                            "America/Guadeloupe",
+                                            "America/Guatemala",
+                                            "America/Guayaquil",
+                                            "America/Guyana",
+                                            "America/Halifax",
+                                            "America/Havana",
+                                            "America/Hermosillo",
+                                            "America/Indiana/Indianapolis",
+                                            "America/Indiana/Knox",
+                                            "America/Indiana/Marengo",
+                                            "America/Indiana/Petersburg",
+                                            "America/Indiana/Tell_City",
+                                            "America/Indiana/Vevay",
+                                            "America/Indiana/Vincennes",
+                                            "America/Indiana/Winamac",
+                                            "America/Indianapolis",
+                                            "America/Inuvik",
+                                            "America/Iqaluit",
+                                            "America/Jamaica",
+                                            "America/Jujuy",
+                                            "America/Juneau",
+                                            "America/Kentucky/Louisville",
+                                            "America/Kentucky/Monticello",
+                                            "America/Knox_IN",
+                                            "America/Kralendijk",
+                                            "America/La_Paz",
+                                            "America/Lima",
+                                            "America/Los_Angeles",
+                                            "America/Louisville",
+                                            "America/Lower_Princes",
+                                            "America/Maceio",
+                                            "America/Managua",
+                                            "America/Manaus",
+                                            "America/Marigot",
+                                            "America/Martinique",
+                                            "America/Matamoros",
+                                            "America/Mazatlan",
+                                            "America/Mendoza",
+                                            "America/Menominee",
+                                            "America/Merida",
+                                            "America/Metlakatla",
+                                            "America/Mexico_City",
+                                            "America/Miquelon",
+                                            "America/Moncton",
+                                            "America/Monterrey",
+                                            "America/Montevideo",
+                                            "America/Montreal",
+                                            "America/Montserrat",
+                                            "America/Nassau",
+                                            "America/New_York",
+                                            "America/Nipigon",
+                                            "America/Nome",
+                                            "America/Noronha",
+                                            "America/North_Dakota/Beulah",
+                                            "America/North_Dakota/Center",
+                                            "America/North_Dakota/New_Salem",
+                                            "America/Ojinaga",
+                                            "America/Panama",
+                                            "America/Pangnirtung",
+                                            "America/Paramaribo",
+                                            "America/Phoenix",
+                                            "America/Port-au-Prince",
+                                            "America/Port_of_Spain",
+                                            "America/Porto_Acre",
+                                            "America/Porto_Velho",
+                                            "America/Puerto_Rico",
+                                            "America/Rainy_River",
+                                            "America/Rankin_Inlet",
+                                            "America/Recife",
+                                            "America/Regina",
+                                            "America/Resolute",
+                                            "America/Rio_Branco",
+                                            "America/Rosario",
+                                            "America/Santa_Isabel",
+                                            "America/Santarem",
+                                            "America/Santiago",
+                                            "America/Santo_Domingo",
+                                            "America/Sao_Paulo",
+                                            "America/Scoresbysund",
+                                            "America/Shiprock",
+                                            "America/Sitka",
+                                            "America/St_Barthelemy",
+                                            "America/St_Johns",
+                                            "America/St_Kitts",
+                                            "America/St_Lucia",
+                                            "America/St_Thomas",
+                                            "America/St_Vincent",
+                                            "America/Swift_Current",
+                                            "America/Tegucigalpa",
+                                            "America/Thule",
+                                            "America/Thunder_Bay",
+                                            "America/Tijuana",
+                                            "America/Toronto",
+                                            "America/Tortola",
+                                            "America/Vancouver",
+                                            "America/Virgin",
+                                            "America/Whitehorse",
+                                            "America/Winnipeg",
+                                            "America/Yakutat",
+                                            "America/Yellowknife",
+                                            "Antarctica/Casey",
+                                            "Antarctica/Davis",
+                                            "Antarctica/DumontDUrville",
+                                            "Antarctica/Macquarie",
+                                            "Antarctica/Mawson",
+                                            "Antarctica/McMurdo",
+                                            "Antarctica/Palmer",
+                                            "Antarctica/Rothera",
+                                            "Antarctica/South_Pole",
+                                            "Antarctica/Syowa",
+                                            "Antarctica/Vostok",
+                                            "Arctic/Longyearbyen",
+                                            "Asia/Aden",
+                                            "Asia/Almaty",
+                                            "Asia/Amman",
+                                            "Asia/Anadyr",
+                                            "Asia/Aqtau",
+                                            "Asia/Aqtobe",
+                                            "Asia/Ashgabat",
+                                            "Asia/Ashkhabad",
+                                            "Asia/Baghdad",
+                                            "Asia/Bahrain",
+                                            "Asia/Baku",
+                                            "Asia/Bangkok",
+                                            "Asia/Beirut",
+                                            "Asia/Bishkek",
+                                            "Asia/Brunei",
+                                            "Asia/Calcutta",
+                                            "Asia/Choibalsan",
+                                            "Asia/Chongqing",
+                                            "Asia/Chungking",
+                                            "Asia/Colombo",
+                                            "Asia/Dacca",
+                                            "Asia/Damascus",
+                                            "Asia/Dhaka",
+                                            "Asia/Dili",
+                                            "Asia/Dubai",
+                                            "Asia/Dushanbe",
+                                            "Asia/Gaza",
+                                            "Asia/Harbin",
+                                            "Asia/Hebron",
+                                            "Asia/Ho_Chi_Minh",
+                                            "Asia/Hong_Kong",
+                                            "Asia/Hovd",
+                                            "Asia/Irkutsk",
+                                            "Asia/Istanbul",
+                                            "Asia/Jakarta",
+                                            "Asia/Jayapura",
+                                            "Asia/Jerusalem",
+                                            "Asia/Kabul",
+                                            "Asia/Kamchatka",
+                                            "Asia/Karachi",
+                                            "Asia/Kashgar",
+                                            "Asia/Kathmandu",
+                                            "Asia/Katmandu",
+                                            "Asia/Kolkata",
+                                            "Asia/Krasnoyarsk",
+                                            "Asia/Kuala_Lumpur",
+                                            "Asia/Kuching",
+                                            "Asia/Kuwait",
+                                            "Asia/Macao",
+                                            "Asia/Macau",
+                                            "Asia/Magadan",
+                                            "Asia/Makassar",
+                                            "Asia/Manila",
+                                            "Asia/Muscat",
+                                            "Asia/Nicosia",
+                                            "Asia/Novokuznetsk",
+                                            "Asia/Novosibirsk",
+                                            "Asia/Omsk",
+                                            "Asia/Oral",
+                                            "Asia/Phnom_Penh",
+                                            "Asia/Pontianak",
+                                            "Asia/Pyongyang",
+                                            "Asia/Qatar",
+                                            "Asia/Qyzylorda",
+                                            "Asia/Rangoon",
+                                            "Asia/Riyadh",
+                                            "Asia/Saigon",
+                                            "Asia/Sakhalin",
+                                            "Asia/Samarkand",
+                                            "Asia/Seoul",
+                                            "Asia/Shanghai",
+                                            "Asia/Singapore",
+                                            "Asia/Taipei",
+                                            "Asia/Tashkent",
+                                            "Asia/Tbilisi",
+                                            "Asia/Tehran",
+                                            "Asia/Tel_Aviv",
+                                            "Asia/Thimbu",
+                                            "Asia/Thimphu",
+                                            "Asia/Tokyo",
+                                            "Asia/Ujung_Pandang",
+                                            "Asia/Ulaanbaatar",
+                                            "Asia/Ulan_Bator",
+                                            "Asia/Urumqi",
+                                            "Asia/Vientiane",
+                                            "Asia/Vladivostok",
+                                            "Asia/Yakutsk",
+                                            "Asia/Yekaterinburg",
+                                            "Asia/Yerevan",
+                                            "Atlantic/Azores",
+                                            "Atlantic/Bermuda",
+                                            "Atlantic/Canary",
+                                            "Atlantic/Cape_Verde",
+                                            "Atlantic/Faeroe",
+                                            "Atlantic/Faroe",
+                                            "Atlantic/Jan_Mayen",
+                                            "Atlantic/Madeira",
+                                            "Atlantic/Reykjavik",
+                                            "Atlantic/South_Georgia",
+                                            "Atlantic/St_Helena",
+                                            "Atlantic/Stanley",
+                                            "Australia/ACT",
+                                            "Australia/Adelaide",
+                                            "Australia/Brisbane",
+                                            "Australia/Broken_Hill",
+                                            "Australia/Canberra",
+                                            "Australia/Currie",
+                                            "Australia/Darwin",
+                                            "Australia/Eucla",
+                                            "Australia/Hobart",
+                                            "Australia/LHI",
+                                            "Australia/Lindeman",
+                                            "Australia/Lord_Howe",
+                                            "Australia/Melbourne",
+                                            "Australia/NSW",
+                                            "Australia/North",
+                                            "Australia/Perth",
+                                            "Australia/Queensland",
+                                            "Australia/South",
+                                            "Australia/Sydney",
+                                            "Australia/Tasmania",
+                                            "Australia/Victoria",
+                                            "Australia/West",
+                                            "Australia/Yancowinna",
+                                            "Brazil/Acre",
+                                            "Brazil/DeNoronha",
+                                            "Brazil/East",
+                                            "Brazil/West",
+                                            "CET",
+                                            "CST6CDT",
+                                            "Canada/Atlantic",
+                                            "Canada/Central",
+                                            "Canada/East-Saskatchewan",
+                                            "Canada/Eastern",
+                                            "Canada/Mountain",
+                                            "Canada/Newfoundland",
+                                            "Canada/Pacific",
+                                            "Canada/Saskatchewan",
+                                            "Canada/Yukon",
+                                            "Chile/Continental",
+                                            "Chile/EasterIsland",
+                                            "Cuba",
+                                            "EET",
+                                            "EST",
+                                            "EST5EDT",
+                                            "Egypt",
+                                            "Eire",
+                                            "Etc/GMT",
+                                            "Etc/GMT+0",
+                                            "Etc/GMT+1",
+                                            "Etc/GMT+10",
+                                            "Etc/GMT+11",
+                                            "Etc/GMT+12",
+                                            "Etc/GMT+2",
+                                            "Etc/GMT+3",
+                                            "Etc/GMT+4",
+                                            "Etc/GMT+5",
+                                            "Etc/GMT+6",
+                                            "Etc/GMT+7",
+                                            "Etc/GMT+8",
+                                            "Etc/GMT+9",
+                                            "Etc/GMT-0",
+                                            "Etc/GMT-1",
+                                            "Etc/GMT-10",
+                                            "Etc/GMT-11",
+                                            "Etc/GMT-12",
+                                            "Etc/GMT-13",
+                                            "Etc/GMT-14",
+                                            "Etc/GMT-2",
+                                            "Etc/GMT-3",
+                                            "Etc/GMT-4",
+                                            "Etc/GMT-5",
+                                            "Etc/GMT-6",
+                                            "Etc/GMT-7",
+                                            "Etc/GMT-8",
+                                            "Etc/GMT-9",
+                                            "Etc/GMT0",
+                                            "Etc/Greenwich",
+                                            "Etc/UCT",
+                                            "Etc/UTC",
+                                            "Etc/Universal",
+                                            "Etc/Zulu",
+                                            "Europe/Amsterdam",
+                                            "Europe/Andorra",
+                                            "Europe/Athens",
+                                            "Europe/Belfast",
+                                            "Europe/Belgrade",
+                                            "Europe/Berlin",
+                                            "Europe/Bratislava",
+                                            "Europe/Brussels",
+                                            "Europe/Bucharest",
+                                            "Europe/Budapest",
+                                            "Europe/Chisinau",
+                                            "Europe/Copenhagen",
+                                            "Europe/Dublin",
+                                            "Europe/Gibraltar",
+                                            "Europe/Guernsey",
+                                            "Europe/Helsinki",
+                                            "Europe/Isle_of_Man",
+                                            "Europe/Istanbul",
+                                            "Europe/Jersey",
+                                            "Europe/Kaliningrad",
+                                            "Europe/Kiev",
+                                            "Europe/Lisbon",
+                                            "Europe/Ljubljana",
+                                            "Europe/London",
+                                            "Europe/Luxembourg",
+                                            "Europe/Madrid",
+                                            "Europe/Malta",
+                                            "Europe/Mariehamn",
+                                            "Europe/Minsk",
+                                            "Europe/Monaco",
+                                            "Europe/Moscow",
+                                            "Europe/Nicosia",
+                                            "Europe/Oslo",
+                                            "Europe/Paris",
+                                            "Europe/Podgorica",
+                                            "Europe/Prague",
+                                            "Europe/Riga",
+                                            "Europe/Rome",
+                                            "Europe/Samara",
+                                            "Europe/San_Marino",
+                                            "Europe/Sarajevo",
+                                            "Europe/Simferopol",
+                                            "Europe/Skopje",
+                                            "Europe/Sofia",
+                                            "Europe/Stockholm",
+                                            "Europe/Tallinn",
+                                            "Europe/Tirane",
+                                            "Europe/Tiraspol",
+                                            "Europe/Uzhgorod",
+                                            "Europe/Vaduz",
+                                            "Europe/Vatican",
+                                            "Europe/Vienna",
+                                            "Europe/Vilnius",
+                                            "Europe/Volgograd",
+                                            "Europe/Warsaw",
+                                            "Europe/Zagreb",
+                                            "Europe/Zaporozhye",
+                                            "Europe/Zurich",
+                                            "GB",
+                                            "GB-Eire",
+                                            "GMT",
+                                            "GMT+0",
+                                            "GMT-0",
+                                            "GMT0",
+                                            "Greenwich",
+                                            "HST",
+                                            "Hongkong",
+                                            "Iceland",
+                                            "Indian/Antananarivo",
+                                            "Indian/Chagos",
+                                            "Indian/Christmas",
+                                            "Indian/Cocos",
+                                            "Indian/Comoro",
+                                            "Indian/Kerguelen",
+                                            "Indian/Mahe",
+                                            "Indian/Maldives",
+                                            "Indian/Mauritius",
+                                            "Indian/Mayotte",
+                                            "Indian/Reunion",
+                                            "Iran",
+                                            "Israel",
+                                            "Jamaica",
+                                            "Japan",
+                                            "Kwajalein",
+                                            "Libya",
+                                            "MET",
+                                            "MST",
+                                            "MST7MDT",
+                                            "Mexico/BajaNorte",
+                                            "Mexico/BajaSur",
+                                            "Mexico/General",
+                                            "NZ",
+                                            "NZ-CHAT",
+                                            "Navajo",
+                                            "PRC",
+                                            "PST8PDT",
+                                            "Pacific/Apia",
+                                            "Pacific/Auckland",
+                                            "Pacific/Chatham",
+                                            "Pacific/Chuuk",
+                                            "Pacific/Easter",
+                                            "Pacific/Efate",
+                                            "Pacific/Enderbury",
+                                            "Pacific/Fakaofo",
+                                            "Pacific/Fiji",
+                                            "Pacific/Funafuti",
+                                            "Pacific/Galapagos",
+                                            "Pacific/Gambier",
+                                            "Pacific/Guadalcanal",
+                                            "Pacific/Guam",
+                                            "Pacific/Honolulu",
+                                            "Pacific/Johnston",
+                                            "Pacific/Kiritimati",
+                                            "Pacific/Kosrae",
+                                            "Pacific/Kwajalein",
+                                            "Pacific/Majuro",
+                                            "Pacific/Marquesas",
+                                            "Pacific/Midway",
+                                            "Pacific/Nauru",
+                                            "Pacific/Niue",
+                                            "Pacific/Norfolk",
+                                            "Pacific/Noumea",
+                                            "Pacific/Pago_Pago",
+                                            "Pacific/Palau",
+                                            "Pacific/Pitcairn",
+                                            "Pacific/Pohnpei",
+                                            "Pacific/Ponape",
+                                            "Pacific/Port_Moresby",
+                                            "Pacific/Rarotonga",
+                                            "Pacific/Saipan",
+                                            "Pacific/Samoa",
+                                            "Pacific/Tahiti",
+                                            "Pacific/Tarawa",
+                                            "Pacific/Tongatapu",
+                                            "Pacific/Truk",
+                                            "Pacific/Wake",
+                                            "Pacific/Wallis",
+                                            "Pacific/Yap",
+                                            "Poland",
+                                            "Portugal",
+                                            "ROC",
+                                            "ROK",
+                                            "Singapore",
+                                            "Turkey",
+                                            "UCT",
+                                            "US/Alaska",
+                                            "US/Aleutian",
+                                            "US/Arizona",
+                                            "US/Central",
+                                            "US/East-Indiana",
+                                            "US/Eastern",
+                                            "US/Hawaii",
+                                            "US/Indiana-Starke",
+                                            "US/Michigan",
+                                            "US/Mountain",
+                                            "US/Pacific",
+                                            "US/Pacific-New",
+                                            "US/Samoa",
+                                            "UTC",
+                                            "Universal",
+                                            "W-SU",
+                                            "WET",
+                                            "Zulu"};
+
+    static {
+      for (int i = 0; i < timezoneList.length; i++) {
+        timezoneMap.put(timezoneList[i], i);
+      }
+    }
+
+    public static final DateTimeFormatter formatDate        = DateTimeFormat.forPattern("yyyy-MM-dd");
+    public static final DateTimeFormatter formatTimeStamp    = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+    public static final DateTimeFormatter formatTimeStampTZ = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ");
+    public static final DateTimeFormatter formatTime        = DateTimeFormat.forPattern("HH:mm:ss.SSS");
+
+    public static DateTimeFormatter dateTimeTZFormat = null;
+    public static DateTimeFormatter timeFormat = null;
+
+    public static final int yearsToMonths = 12;
+    public static final int hoursToMillis = 60 * 60 * 1000;
+    public static final int minutesToMillis = 60 * 1000;
+    public static final int secondsToMillis = 1000;
+    public static final int monthToStandardDays = 30;
+    public static final long monthsToMillis = 2592000000L; // 30 * 24 * 60 * 60 * 1000
+    public static final int daysToStandardMillis = 24 * 60 * 60 * 1000;
+
+
+  public static int getIndex(String timezone) {
+        return timezoneMap.get(timezone);
+    }
+
+    public static String getTimeZone(int index) {
+        return timezoneList[index];
+    }
+
+    // Function returns the date time formatter used to parse date strings
+    public static DateTimeFormatter getDateTimeFormatter() {
+
+        if (dateTimeTZFormat == null) {
+            DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");
+            DateTimeParser optionalTime = DateTimeFormat.forPattern(" HH:mm:ss").getParser();
+            DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
+            DateTimeParser optionalZone = DateTimeFormat.forPattern(" ZZZ").getParser();
+
+            dateTimeTZFormat = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).appendOptional(optionalZone).toFormatter();
+        }
+
+        return dateTimeTZFormat;
+    }
+
+    // Function returns time formatter used to parse time strings
+    public static DateTimeFormatter getTimeFormatter() {
+        if (timeFormat == null) {
+            DateTimeFormatter timeFormatter = DateTimeFormat.forPattern("HH:mm:ss");
+            DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
+            timeFormat = new DateTimeFormatterBuilder().append(timeFormatter).appendOptional(optionalSec).toFormatter();
+        }
+        return timeFormat;
+    }
+
+    public static int monthsFromPeriod(Period period){
+      return (period.getYears() * yearsToMonths) + period.getMonths();
+    }
+
+    public static int millisFromPeriod(final Period period){
+      return (period.getHours() * hoursToMillis) +
+      (period.getMinutes() * minutesToMillis) +
+      (period.getSeconds() * secondsToMillis) +
+      (period.getMillis());
+    }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
new file mode 100644
index 0000000000000..576a5b6351ad1
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -0,0 +1,737 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.UnpooledByteBufAllocator;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.arrow.vector.holders.Decimal38SparseHolder;
+
+public class DecimalUtility extends CoreDecimalUtility{
+
+    public final static int MAX_DIGITS = 9;
+    public final static int DIGITS_BASE = 1000000000;
+    public final static int DIGITS_MAX = 999999999;
+    public final static int INTEGER_SIZE = (Integer.SIZE/8);
+
+    public final static String[] decimalToString = {"",
+            "0",
+            "00",
+            "000",
+            "0000",
+            "00000",
+            "000000",
+            "0000000",
+            "00000000",
+            "000000000"};
+
+    public final static long[] scale_long_constants = {
+        1,
+        10,
+        100,
+        1000,
+        10000,
+        100000,
+        1000000,
+        10000000,
+        100000000,
+        1000000000,
+        10000000000l,
+        100000000000l,
+        1000000000000l,
+        10000000000000l,
+        100000000000000l,
+        1000000000000000l,
+        10000000000000000l,
+        100000000000000000l,
+        1000000000000000000l};
+
+    /*
+     * Simple function that returns the static precomputed
+     * power of ten, instead of using Math.pow
+     */
+    public static long getPowerOfTen(int power) {
+      assert power >= 0 && power < scale_long_constants.length;
+      return scale_long_constants[(power)];
+    }
+
+    /*
+     * Math.pow returns a double and while multiplying with large digits
+     * in the decimal data type we encounter noise. So instead of multiplying
+     * with Math.pow we use the static constants to perform the multiplication
+     */
+    public static long adjustScaleMultiply(long input, int factor) {
+      int index = Math.abs(factor);
+      assert index >= 0 && index < scale_long_constants.length;
+      if (factor >= 0) {
+        return input * scale_long_constants[index];
+      } else {
+        return input / scale_long_constants[index];
+      }
+    }
+
+    public static long adjustScaleDivide(long input, int factor) {
+      int index = Math.abs(factor);
+      assert index >= 0 && index < scale_long_constants.length;
+      if (factor >= 0) {
+        return input / scale_long_constants[index];
+      } else {
+        return input * scale_long_constants[index];
+      }
+    }
+
+    /* Given the number of actual digits this function returns the
+     * number of indexes it will occupy in the array of integers
+     * which are stored in base 1 billion
+     */
+    public static int roundUp(int ndigits) {
+        return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+    }
+
+    /* Returns a string representation of the given integer
+     * If the length of the given integer is less than the
+     * passed length, this function will prepend zeroes to the string
+     */
+    public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
+        String value = ((Integer) number).toString();
+        int length = value.length();
+
+        StringBuilder str = new StringBuilder();
+        str.append(decimalToString[desiredLength - length]);
+        str.append(value);
+
+        return str;
+    }
+
+    public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
+        String value = ((Long) number).toString();
+        int length = value.length();
+
+        StringBuilder str = new StringBuilder();
+
+        // Desired length can be > MAX_DIGITS
+        int zeroesLength = desiredLength - length;
+        while (zeroesLength > MAX_DIGITS) {
+            str.append(decimalToString[MAX_DIGITS]);
+            zeroesLength -= MAX_DIGITS;
+        }
+        str.append(decimalToString[zeroesLength]);
+        str.append(value);
+
+        return str;
+    }
+
+  public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
+
+        // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
+        return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false);
+    }
+
+    public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
+
+        // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
+        return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true);
+    }
+
+    public static BigDecimal getBigDecimalFromDrillBuf(ArrowBuf bytebuf, int start, int length, int scale) {
+      byte[] value = new byte[length];
+      bytebuf.getBytes(start, value, 0, length);
+      BigInteger unscaledValue = new BigInteger(value);
+      return new BigDecimal(unscaledValue, scale);
+    }
+
+  public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
+    byte[] value = new byte[length];
+    bytebuf.get(value);
+    BigInteger unscaledValue = new BigInteger(value);
+    return new BigDecimal(unscaledValue, scale);
+  }
+
+    /* Create a BigDecimal object using the data in the DrillBuf.
+     * This function assumes that data is provided in a non-dense format
+     * It works on both sparse and intermediate representations.
+     */
+  public static BigDecimal getBigDecimalFromDrillBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale,
+      boolean truncateScale) {
+
+        // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
+        int actualDigits;
+
+        // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out
+        BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
+
+        BigInteger base = BigInteger.valueOf(DIGITS_BASE);
+
+        for (int i = 1; i < nDecimalDigits; i++) {
+
+            BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
+            decimalDigits = decimalDigits.multiply(base);
+            decimalDigits = decimalDigits.add(temp);
+        }
+
+        // Truncate any additional padding we might have added
+        if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
+            BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
+            decimalDigits = decimalDigits.divide(truncate);
+        }
+
+        // set the sign
+        if ((data.getInt(startIndex) & 0x80000000) != 0) {
+            decimalDigits = decimalDigits.negate();
+        }
+
+        BigDecimal decimal = new BigDecimal(decimalDigits, scale);
+
+        return decimal;
+    }
+
+    /* This function returns a BigDecimal object from the dense decimal representation.
+     * First step is to convert the dense representation into an intermediate representation
+     * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object
+     */
+    public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
+
+        /* This method converts the dense representation to
+         * an intermediate representation. The intermediate
+         * representation has one more integer than the dense
+         * representation.
+         */
+        byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
+
+        // Start storing from the least significant byte of the first integer
+        int intermediateIndex = 3;
+
+        int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
+        int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
+
+        int maskIndex;
+        int shiftOrder;
+        byte shiftBits;
+
+        // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
+        if (maxPrecision == 38) {
+            maskIndex = 0;
+            shiftOrder = 6;
+            shiftBits = 0x00;
+            intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
+        } else if (maxPrecision == 28) {
+            maskIndex = 1;
+            shiftOrder = 4;
+            shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
+            intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
+        } else {
+            throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
+        }
+
+        int inputIndex = 1;
+        boolean sign = false;
+
+        if ((data.getByte(startIndex) & 0x80) != 0) {
+            sign = true;
+        }
+
+        while (inputIndex < width) {
+
+            intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
+
+            shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
+
+            inputIndex++;
+            intermediateIndex++;
+
+            if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
+                shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
+                maskIndex++;
+                shiftOrder -= 2;
+            }
+
+        }
+        /* copy the last byte */
+        intermediateBytes[intermediateIndex] = shiftBits;
+
+        if (sign == true) {
+            intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
+        }
+
+    final ByteBuf intermediate = UnpooledByteBufAllocator.DEFAULT.buffer(intermediateBytes.length);
+    try {
+        intermediate.setBytes(0, intermediateBytes);
+
+      BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
+      return ret;
+    } finally {
+      intermediate.release();
+    }
+
+    }
+
+    /*
+     * Function converts the BigDecimal and stores it in out internal sparse representation
+     */
+  public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
+      int nDecimalDigits) {
+
+        // Initialize the buffer
+        for (int i = 0; i < nDecimalDigits; i++) {
+          data.setInt(startIndex + (i * INTEGER_SIZE), 0);
+        }
+
+        boolean sign = false;
+
+        if (input.signum() == -1) {
+            // negative input
+            sign = true;
+            input = input.abs();
+        }
+
+        // Truncate the input as per the scale provided
+        input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+        // Separate out the integer part
+        BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
+
+        int destIndex = nDecimalDigits - roundUp(scale) - 1;
+
+        // we use base 1 billion integer digits for out integernal representation
+        BigDecimal base = new BigDecimal(DIGITS_BASE);
+
+        while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
+            // store the modulo as the integer value
+            data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
+            destIndex--;
+            // Divide by base 1 billion
+            integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
+        }
+
+        /* Sparse representation contains padding of additional zeroes
+         * so each digit contains MAX_DIGITS for ease of arithmetic
+         */
+        int actualDigits;
+        if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
+            // Pad additional zeroes
+            scale = scale + (MAX_DIGITS - actualDigits);
+            input = input.setScale(scale, BigDecimal.ROUND_DOWN);
+        }
+
+        //separate out the fractional part
+        BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
+
+        destIndex = nDecimalDigits - 1;
+
+        while (scale > 0) {
+            // Get next set of MAX_DIGITS (9) store it in the DrillBuf
+            fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
+            BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
+
+            data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
+            destIndex--;
+
+            fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
+            scale -= MAX_DIGITS;
+        }
+
+        // Set the negative sign
+        if (sign == true) {
+            data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
+        }
+
+    }
+
+
+    public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+        // Truncate or pad to set the input to the correct scale
+        input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+        return (input.unscaledValue().longValue());
+    }
+
+    public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
+      return BigDecimal.valueOf(input, scale);
+    }
+
+    public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
+      return BigDecimal.valueOf(input, scale);
+    }
+
+
+    public static int compareDenseBytes(ArrowBuf left, int leftStart, boolean leftSign, ArrowBuf right, int rightStart, boolean rightSign, int width) {
+
+      int invert = 1;
+
+      /* If signs are different then simply look at the
+       * sign of the two inputs and determine which is greater
+       */
+      if (leftSign != rightSign) {
+
+        return((leftSign == true) ? -1 : 1);
+      } else if(leftSign == true) {
+        /* Both inputs are negative, at the end we will
+         * have to invert the comparison
+         */
+        invert = -1;
+      }
+
+      int cmp = 0;
+
+      for (int i = 0; i < width; i++) {
+        byte leftByte  = left.getByte(leftStart + i);
+        byte rightByte = right.getByte(rightStart + i);
+        // Unsigned byte comparison
+        if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
+          cmp = 1;
+          break;
+        } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
+          cmp = -1;
+          break;
+        }
+      }
+      cmp *= invert; // invert the comparison if both were negative values
+
+      return cmp;
+    }
+
+    public static int getIntegerFromSparseBuffer(ArrowBuf buffer, int start, int index) {
+      int value = buffer.getInt(start + (index * 4));
+
+      if (index == 0) {
+        /* the first byte contains sign bit, return value without it */
+        value = (value & 0x7FFFFFFF);
+      }
+      return value;
+    }
+
+    public static void setInteger(ArrowBuf buffer, int start, int index, int value) {
+      buffer.setInt(start + (index * 4), value);
+    }
+
+    public static int compareSparseBytes(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
+
+      int invert = 1;
+
+      if (absCompare == false) {
+        if (leftSign != rightSign) {
+          return (leftSign == true) ? -1 : 1;
+        }
+
+        // Both values are negative invert the outcome of the comparison
+        if (leftSign == true) {
+          invert = -1;
+        }
+      }
+
+      int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
+      return cmp * invert;
+    }
+    public static int compareSparseBytesInner(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
+      /* compute the number of integer digits in each decimal */
+      int leftInt  = leftPrecision - leftScale;
+      int rightInt = rightPrecision - rightScale;
+
+      /* compute the number of indexes required for storing integer digits */
+      int leftIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftInt);
+      int rightIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightInt);
+
+      /* compute number of indexes required for storing scale */
+      int leftScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftScale);
+      int rightScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightScale);
+
+      /* compute index of the most significant integer digits */
+      int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+      int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+
+      int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
+      int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
+
+      /* Discard the zeroes in the integer part */
+      while (leftIndex1 < leftStopIndex) {
+        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+          break;
+        }
+
+        /* Digit in this location is zero, decrement the actual number
+         * of integer digits
+         */
+        leftIntRoundedUp--;
+        leftIndex1++;
+      }
+
+      /* If we reached the stop index then the number of integers is zero */
+      if (leftIndex1 == leftStopIndex) {
+        leftIntRoundedUp = 0;
+      }
+
+      while (rightIndex1 < rightStopIndex) {
+        if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+          break;
+        }
+
+        /* Digit in this location is zero, decrement the actual number
+         * of integer digits
+         */
+        rightIntRoundedUp--;
+        rightIndex1++;
+      }
+
+      if (rightIndex1 == rightStopIndex) {
+        rightIntRoundedUp = 0;
+      }
+
+      /* We have the accurate number of non-zero integer digits,
+       * if the number of integer digits are different then we can determine
+       * which decimal is larger and needn't go down to comparing individual values
+       */
+      if (leftIntRoundedUp > rightIntRoundedUp) {
+        return 1;
+      }
+      else if (rightIntRoundedUp > leftIntRoundedUp) {
+        return -1;
+      }
+
+      /* The number of integer digits are the same, set the each index
+       * to the first non-zero integer and compare each digit
+       */
+      leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+      rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+
+      while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+          return 1;
+        }
+        else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+          return -1;
+        }
+
+        leftIndex1++;
+        rightIndex1++;
+      }
+
+      /* The integer part of both the decimal's are equal, now compare
+       * each individual fractional part. Set the index to be at the
+       * beginning of the fractional part
+       */
+      leftIndex1 = leftStopIndex;
+      rightIndex1 = rightStopIndex;
+
+      /* Stop indexes will be the end of the array */
+      leftStopIndex = nDecimalDigits;
+      rightStopIndex = nDecimalDigits;
+
+      /* compare the two fractional parts of the decimal */
+      while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+          return 1;
+        }
+        else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+          return -1;
+        }
+
+        leftIndex1++;
+        rightIndex1++;
+      }
+
+      /* Till now the fractional part of the decimals are equal, check
+       * if one of the decimal has fractional part that is remaining
+       * and is non-zero
+       */
+      while (leftIndex1 < leftStopIndex) {
+        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+          return 1;
+        }
+        leftIndex1++;
+      }
+
+      while(rightIndex1 < rightStopIndex) {
+        if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+          return -1;
+        }
+        rightIndex1++;
+      }
+
+      /* Both decimal values are equal */
+      return 0;
+    }
+
+    public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
+      byte[] value = Arrays.copyOfRange(bytes, start, start + length);
+      BigInteger unscaledValue = new BigInteger(value);
+      return new BigDecimal(unscaledValue, scale);
+    }
+
+  public static void roundDecimal(ArrowBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
+    int newScaleRoundedUp  = org.apache.arrow.vector.util.DecimalUtility.roundUp(desiredScale);
+    int origScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(currentScale);
+
+    if (desiredScale < currentScale) {
+
+      boolean roundUp = false;
+
+      //Extract the first digit to be truncated to check if we need to round up
+      int truncatedScaleIndex = desiredScale + 1;
+      if (truncatedScaleIndex <= currentScale) {
+        int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
+        extractDigitIndex += org.apache.arrow.vector.util.DecimalUtility.roundUp(truncatedScaleIndex);
+        int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
+        int temp = org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS);
+        if (temp != 0) {
+          extractDigit = extractDigit / (int) (Math.pow(10, temp));
+        }
+        if ((extractDigit % 10)  > 4) {
+          roundUp = true;
+        }
+      }
+
+      // Get the source index beyond which we will truncate
+      int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
+      int srcIndex = srcIntIndex + newScaleRoundedUp;
+
+      // Truncate the remaining fractional part, move the integer part
+      int destIndex = nDecimalDigits - 1;
+      if (srcIndex != destIndex) {
+        while (srcIndex >= 0) {
+          setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
+        }
+
+        // Set the remaining portion of the decimal to be zeroes
+        while (destIndex >= 0) {
+          setInteger(result, start, destIndex--, 0);
+        }
+        srcIndex = nDecimalDigits - 1;
+      }
+
+      // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
+      int truncateFactor = org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS);
+      if (truncateFactor != org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS) {
+        truncateFactor = (int) Math.pow(10, truncateFactor);
+        int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
+        fractionalDigits /= truncateFactor;
+        setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
+      }
+
+      // Finally round up the digit if needed
+      if (roundUp == true) {
+        srcIndex = nDecimalDigits - 1;
+        int carry;
+        if (truncateFactor != org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS) {
+          carry = truncateFactor;
+        } else {
+          carry = 1;
+        }
+
+        while (srcIndex >= 0) {
+          int value = getIntegerFromSparseBuffer(result, start, srcIndex);
+          value += carry;
+
+          if (value >= org.apache.arrow.vector.util.DecimalUtility.DIGITS_BASE) {
+            setInteger(result, start, srcIndex--, value % org.apache.arrow.vector.util.DecimalUtility.DIGITS_BASE);
+            carry = value / org.apache.arrow.vector.util.DecimalUtility.DIGITS_BASE;
+          } else {
+            setInteger(result, start, srcIndex--, value);
+            carry = 0;
+            break;
+          }
+        }
+      }
+    } else if (desiredScale > currentScale) {
+      // Add fractional digits to the decimal
+
+      // Check if we need to shift the decimal digits to the left
+      if (newScaleRoundedUp > origScaleRoundedUp) {
+        int srcIndex  = 0;
+        int destIndex = newScaleRoundedUp - origScaleRoundedUp;
+
+        // Check while extending scale, we are not overwriting integer part
+        while (srcIndex < destIndex) {
+          if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
+            throw new RuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
+          }
+        }
+
+        srcIndex = 0;
+        while (destIndex < nDecimalDigits) {
+          setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
+        }
+
+        // Clear the remaining part
+        while (srcIndex < nDecimalDigits) {
+          setInteger(result, start, srcIndex++, 0);
+        }
+      }
+    }
+  }
+
+  public static int getFirstFractionalDigit(int decimal, int scale) {
+    if (scale == 0) {
+      return 0;
+    }
+    int temp = (int) adjustScaleDivide(decimal, scale - 1);
+    return Math.abs(temp % 10);
+  }
+
+  public static int getFirstFractionalDigit(long decimal, int scale) {
+    if (scale == 0) {
+      return 0;
+    }
+    long temp = adjustScaleDivide(decimal, scale - 1);
+    return (int) (Math.abs(temp % 10));
+  }
+
+  public static int getFirstFractionalDigit(ArrowBuf data, int scale, int start, int nDecimalDigits) {
+    if (scale == 0) {
+      return 0;
+    }
+
+    int index = nDecimalDigits - roundUp(scale);
+    return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1));
+  }
+
+  public static int compareSparseSamePrecScale(ArrowBuf left, int lStart, byte[] right, int length) {
+    // check the sign first
+    boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
+    boolean rSign = ByteFunctionHelpers.getSign(right);
+    int cmp = 0;
+
+    if (lSign != rSign) {
+      return (lSign == false) ? 1 : -1;
+    }
+
+    // invert the comparison if we are comparing negative numbers
+    int invert = (lSign == true) ? -1 : 1;
+
+    // compare byte by byte
+    int n = 0;
+    int lPos = lStart;
+    int rPos = 0;
+    while (n < length/4) {
+      int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
+      int rightInt = ByteFunctionHelpers.getInteger(right, n);
+      if (leftInt != rightInt) {
+        cmp =  (leftInt - rightInt ) > 0 ? 1 : -1;
+        break;
+      }
+      n++;
+    }
+    return cmp * invert;
+  }
+}
+
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
new file mode 100644
index 0000000000000..7aeaa12ef9fcf
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class JsonStringArrayList<E> extends ArrayList<E> {
+
+  private static ObjectMapper mapper;
+
+  static {
+    mapper = new ObjectMapper();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof List)) {
+      return false;
+    }
+    List other = (List) obj;
+    return this.size() == other.size() && this.containsAll(other);
+  }
+
+  @Override
+  public final String toString() {
+    try {
+      return mapper.writeValueAsString(this);
+    } catch(JsonProcessingException e) {
+      throw new IllegalStateException("Cannot serialize array list to JSON string", e);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
new file mode 100644
index 0000000000000..750dd592aa49c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/*
+ * Simple class that extends the regular java.util.HashMap but overrides the
+ * toString() method of the HashMap class to produce a JSON string instead
+ */
+public class JsonStringHashMap<K, V> extends LinkedHashMap<K, V> {
+
+  private static ObjectMapper mapper;
+
+  static {
+    mapper = new ObjectMapper();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof Map)) {
+      return false;
+    }
+    Map other = (Map) obj;
+    if (this.size() != other.size()) {
+      return false;
+    }
+    for (K key : this.keySet()) {
+      if (this.get(key) == null ) {
+        if (other.get(key) == null) {
+          continue;
+        } else {
+          return false;
+        }
+      }
+      if ( ! this.get(key).equals(other.get(key))) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public final String toString() {
+    try {
+      return mapper.writeValueAsString(this);
+    } catch(JsonProcessingException e) {
+      throw new IllegalStateException("Cannot serialize hash map to JSON string", e);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
new file mode 100644
index 0000000000000..dea433e99e80f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import java.util.AbstractMap;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import io.netty.util.collection.IntObjectHashMap;
+import io.netty.util.collection.IntObjectMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation of map that supports constant time look-up by a generic key or an ordinal.
+ *
+ * This class extends the functionality a regular {@link Map} with ordinal lookup support.
+ * Upon insertion an unused ordinal is assigned to the inserted (key, value) tuple.
+ * Upon update the same ordinal id is re-used while value is replaced.
+ * Upon deletion of an existing item, its corresponding ordinal is recycled and could be used by another item.
+ *
+ * For any instance with N items, this implementation guarantees that ordinals are in the range of [0, N). However,
+ * the ordinal assignment is dynamic and may change after an insertion or deletion. Consumers of this class are
+ * responsible for explicitly checking the ordinal corresponding to a key via
+ * {@link org.apache.arrow.vector.util.MapWithOrdinal#getOrdinal(Object)} before attempting to execute a lookup
+ * with an ordinal.
+ *
+ * @param <K> key type
+ * @param <V> value type
+ */
+
+public class MapWithOrdinal<K, V> implements Map<K, V> {
+  private final static Logger logger = LoggerFactory.getLogger(MapWithOrdinal.class);
+
+  private final Map<K, Entry<Integer, V>> primary = Maps.newLinkedHashMap();
+  private final IntObjectHashMap<V> secondary = new IntObjectHashMap<>();
+
+  private final Map<K, V> delegate = new Map<K, V>() {
+    @Override
+    public boolean isEmpty() {
+      return size() == 0;
+    }
+
+    @Override
+    public int size() {
+      return primary.size();
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+      return primary.containsKey(key);
+    }
+
+    @Override
+    public boolean containsValue(Object value) {
+      return primary.containsValue(value);
+    }
+
+    @Override
+    public V get(Object key) {
+      Entry<Integer, V> pair = primary.get(key);
+      if (pair != null) {
+        return pair.getValue();
+      }
+      return null;
+    }
+
+    @Override
+    public V put(K key, V value) {
+      final Entry<Integer, V> oldPair = primary.get(key);
+      // if key exists try replacing otherwise, assign a new ordinal identifier
+      final int ordinal = oldPair == null ? primary.size():oldPair.getKey();
+      primary.put(key, new AbstractMap.SimpleImmutableEntry<>(ordinal, value));
+      secondary.put(ordinal, value);
+      return oldPair==null ? null:oldPair.getValue();
+    }
+
+    @Override
+    public V remove(Object key) {
+      final Entry<Integer, V> oldPair = primary.remove(key);
+      if (oldPair!=null) {
+        final int lastOrdinal = secondary.size();
+        final V last = secondary.get(lastOrdinal);
+        // normalize mappings so that all numbers until primary.size() is assigned
+        // swap the last element with the deleted one
+        secondary.put(oldPair.getKey(), last);
+        primary.put((K) key, new AbstractMap.SimpleImmutableEntry<>(oldPair.getKey(), last));
+      }
+      return oldPair==null ? null:oldPair.getValue();
+    }
+
+    @Override
+    public void putAll(Map<? extends K, ? extends V> m) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void clear() {
+      primary.clear();
+      secondary.clear();
+    }
+
+    @Override
+    public Set<K> keySet() {
+      return primary.keySet();
+    }
+
+    @Override
+    public Collection<V> values() {
+      return Lists.newArrayList(Iterables.transform(secondary.entries(), new Function<IntObjectMap.Entry<V>, V>() {
+        @Override
+        public V apply(IntObjectMap.Entry<V> entry) {
+          return Preconditions.checkNotNull(entry).value();
+        }
+      }));
+    }
+
+    @Override
+    public Set<Entry<K, V>> entrySet() {
+      return Sets.newHashSet(Iterables.transform(primary.entrySet(), new Function<Entry<K, Entry<Integer, V>>, Entry<K, V>>() {
+        @Override
+        public Entry<K, V> apply(Entry<K, Entry<Integer, V>> entry) {
+          return new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), entry.getValue().getValue());
+        }
+      }));
+    }
+  };
+
+  /**
+   * Returns the value corresponding to the given ordinal
+   *
+   * @param id ordinal value for lookup
+   * @return an instance of V
+   */
+  public V getByOrdinal(int id) {
+    return secondary.get(id);
+  }
+
+  /**
+   * Returns the ordinal corresponding to the given key.
+   *
+   * @param key key for ordinal lookup
+   * @return ordinal value corresponding to key if it exists or -1
+   */
+  public int getOrdinal(K key) {
+    Entry<Integer, V> pair = primary.get(key);
+    if (pair != null) {
+      return pair.getKey();
+    }
+    return -1;
+  }
+
+  @Override
+  public int size() {
+    return delegate.size();
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return delegate.isEmpty();
+  }
+
+  @Override
+  public V get(Object key) {
+    return delegate.get(key);
+  }
+
+  /**
+   * Inserts the tuple (key, value) into the map extending the semantics of {@link Map#put} with automatic ordinal
+   * assignment. A new ordinal is assigned if key does not exists. Otherwise the same ordinal is re-used but the value
+   * is replaced.
+   *
+   * {@see java.util.Map#put}
+   */
+  @Override
+  public V put(K key, V value) {
+    return delegate.put(key, value);
+  }
+
+  @Override
+  public Collection<V> values() {
+    return delegate.values();
+  }
+
+  @Override
+  public boolean containsKey(Object key) {
+    return delegate.containsKey(key);
+  }
+
+  @Override
+  public boolean containsValue(Object value) {
+    return delegate.containsValue(value);
+  }
+
+  /**
+   * Removes the element corresponding to the key if exists extending the semantics of {@link Map#remove} with ordinal
+   * re-cycling. The ordinal corresponding to the given key may be re-assigned to another tuple. It is important that
+   * consumer checks the ordinal value via {@link #getOrdinal(Object)} before attempting to look-up by ordinal.
+   *
+   * {@see java.util.Map#remove}
+   */
+  @Override
+  public V remove(Object key) {
+    return delegate.remove(key);
+  }
+
+  @Override
+  public void putAll(Map<? extends K, ? extends V> m) {
+    delegate.putAll(m);
+  }
+
+  @Override
+  public void clear() {
+    delegate.clear();
+  }
+
+  @Override
+  public Set<K> keySet() {
+    return delegate.keySet();
+  }
+
+  @Override
+  public Set<Entry<K, V>> entrySet() {
+    return delegate.entrySet();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java b/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
new file mode 100644
index 0000000000000..ec628b22c2d90
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+
+/**
+ * An exception that is used to signal that allocation request in bytes is greater than the maximum allowed by
+ * {@link org.apache.arrow.memory.BufferAllocator#buffer(int) allocator}.
+ *
+ * <p>Operators should handle this exception to split the batch and later resume the execution on the next
+ * {@link RecordBatch#next() iteration}.</p>
+ *
+ */
+public class OversizedAllocationException extends RuntimeException {
+  public OversizedAllocationException() {
+    super();
+  }
+
+  public OversizedAllocationException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+
+  public OversizedAllocationException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public OversizedAllocationException(String message) {
+    super(message);
+  }
+
+  public OversizedAllocationException(Throwable cause) {
+    super(cause);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/SchemaChangeRuntimeException.java b/java/vector/src/main/java/org/apache/arrow/vector/util/SchemaChangeRuntimeException.java
new file mode 100644
index 0000000000000..c281561430707
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/SchemaChangeRuntimeException.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+
+public class SchemaChangeRuntimeException extends RuntimeException {
+  public SchemaChangeRuntimeException() {
+    super();
+  }
+
+  public SchemaChangeRuntimeException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+
+  public SchemaChangeRuntimeException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public SchemaChangeRuntimeException(String message) {
+    super(message);
+  }
+
+  public SchemaChangeRuntimeException(Throwable cause) {
+    super(cause);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
new file mode 100644
index 0000000000000..3919f0606cb20
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
@@ -0,0 +1,621 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.MalformedInputException;
+import java.text.CharacterIterator;
+import java.text.StringCharacterIterator;
+import java.util.Arrays;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+
+/**
+ * A simplified byte wrapper similar to Hadoop's Text class without all the dependencies. Lifted from Hadoop 2.7.1
+ */
+@JsonSerialize(using = Text.TextSerializer.class)
+public class Text {
+
+  private static ThreadLocal<CharsetEncoder> ENCODER_FACTORY =
+      new ThreadLocal<CharsetEncoder>() {
+        @Override
+        protected CharsetEncoder initialValue() {
+          return Charset.forName("UTF-8").newEncoder().
+              onMalformedInput(CodingErrorAction.REPORT).
+              onUnmappableCharacter(CodingErrorAction.REPORT);
+        }
+      };
+
+  private static ThreadLocal<CharsetDecoder> DECODER_FACTORY =
+      new ThreadLocal<CharsetDecoder>() {
+        @Override
+        protected CharsetDecoder initialValue() {
+          return Charset.forName("UTF-8").newDecoder().
+              onMalformedInput(CodingErrorAction.REPORT).
+              onUnmappableCharacter(CodingErrorAction.REPORT);
+        }
+      };
+
+  private static final byte[] EMPTY_BYTES = new byte[0];
+
+  private byte[] bytes;
+  private int length;
+
+  public Text() {
+    bytes = EMPTY_BYTES;
+  }
+
+  /**
+   * Construct from a string.
+   */
+  public Text(String string) {
+    set(string);
+  }
+
+  /** Construct from another text. */
+  public Text(Text utf8) {
+    set(utf8);
+  }
+
+  /**
+   * Construct from a byte array.
+   */
+  public Text(byte[] utf8) {
+    set(utf8);
+  }
+
+  /**
+   * Get a copy of the bytes that is exactly the length of the data. See {@link #getBytes()} for faster access to the
+   * underlying array.
+   */
+  public byte[] copyBytes() {
+    byte[] result = new byte[length];
+    System.arraycopy(bytes, 0, result, 0, length);
+    return result;
+  }
+
+  /**
+   * Returns the raw bytes; however, only data up to {@link #getLength()} is valid. Please use {@link #copyBytes()} if
+   * you need the returned array to be precisely the length of the data.
+   */
+  public byte[] getBytes() {
+    return bytes;
+  }
+
+  /** Returns the number of bytes in the byte array */
+  public int getLength() {
+    return length;
+  }
+
+  /**
+   * Returns the Unicode Scalar Value (32-bit integer value) for the character at <code>position</code>. Note that this
+   * method avoids using the converter or doing String instantiation
+   *
+   * @return the Unicode scalar value at position or -1 if the position is invalid or points to a trailing byte
+   */
+  public int charAt(int position) {
+    if (position > this.length)
+    {
+      return -1; // too long
+    }
+    if (position < 0)
+    {
+      return -1; // duh.
+    }
+
+    ByteBuffer bb = (ByteBuffer) ByteBuffer.wrap(bytes).position(position);
+    return bytesToCodePoint(bb.slice());
+  }
+
+  public int find(String what) {
+    return find(what, 0);
+  }
+
+  /**
+   * Finds any occurence of <code>what</code> in the backing buffer, starting as position <code>start</code>. The
+   * starting position is measured in bytes and the return value is in terms of byte position in the buffer. The backing
+   * buffer is not converted to a string for this operation.
+   *
+   * @return byte position of the first occurence of the search string in the UTF-8 buffer or -1 if not found
+   */
+  public int find(String what, int start) {
+    try {
+      ByteBuffer src = ByteBuffer.wrap(this.bytes, 0, this.length);
+      ByteBuffer tgt = encode(what);
+      byte b = tgt.get();
+      src.position(start);
+
+      while (src.hasRemaining()) {
+        if (b == src.get()) { // matching first byte
+          src.mark(); // save position in loop
+          tgt.mark(); // save position in target
+          boolean found = true;
+          int pos = src.position() - 1;
+          while (tgt.hasRemaining()) {
+            if (!src.hasRemaining()) { // src expired first
+              tgt.reset();
+              src.reset();
+              found = false;
+              break;
+            }
+            if (!(tgt.get() == src.get())) {
+              tgt.reset();
+              src.reset();
+              found = false;
+              break; // no match
+            }
+          }
+          if (found) {
+            return pos;
+          }
+        }
+      }
+      return -1; // not found
+    } catch (CharacterCodingException e) {
+      // can't get here
+      e.printStackTrace();
+      return -1;
+    }
+  }
+
+  /**
+   * Set to contain the contents of a string.
+   */
+  public void set(String string) {
+    try {
+      ByteBuffer bb = encode(string, true);
+      bytes = bb.array();
+      length = bb.limit();
+    } catch (CharacterCodingException e) {
+      throw new RuntimeException("Should not have happened ", e);
+    }
+  }
+
+  /**
+   * Set to a utf8 byte array
+   */
+  public void set(byte[] utf8) {
+    set(utf8, 0, utf8.length);
+  }
+
+  /** copy a text. */
+  public void set(Text other) {
+    set(other.getBytes(), 0, other.getLength());
+  }
+
+  /**
+   * Set the Text to range of bytes
+   *
+   * @param utf8
+   *          the data to copy from
+   * @param start
+   *          the first position of the new string
+   * @param len
+   *          the number of bytes of the new string
+   */
+  public void set(byte[] utf8, int start, int len) {
+    setCapacity(len, false);
+    System.arraycopy(utf8, start, bytes, 0, len);
+    this.length = len;
+  }
+
+  /**
+   * Append a range of bytes to the end of the given text
+   *
+   * @param utf8
+   *          the data to copy from
+   * @param start
+   *          the first position to append from utf8
+   * @param len
+   *          the number of bytes to append
+   */
+  public void append(byte[] utf8, int start, int len) {
+    setCapacity(length + len, true);
+    System.arraycopy(utf8, start, bytes, length, len);
+    length += len;
+  }
+
+  /**
+   * Clear the string to empty.
+   *
+   * <em>Note</em>: For performance reasons, this call does not clear the underlying byte array that is retrievable via
+   * {@link #getBytes()}. In order to free the byte-array memory, call {@link #set(byte[])} with an empty byte array
+   * (For example, <code>new byte[0]</code>).
+   */
+  public void clear() {
+    length = 0;
+  }
+
+  /*
+   * Sets the capacity of this Text object to <em>at least</em> <code>len</code> bytes. If the current buffer is longer,
+   * then the capacity and existing content of the buffer are unchanged. If <code>len</code> is larger than the current
+   * capacity, the Text object's capacity is increased to match.
+   *
+   * @param len the number of bytes we need
+   *
+   * @param keepData should the old data be kept
+   */
+  private void setCapacity(int len, boolean keepData) {
+    if (bytes == null || bytes.length < len) {
+      if (bytes != null && keepData) {
+        bytes = Arrays.copyOf(bytes, Math.max(len, length << 1));
+      } else {
+        bytes = new byte[len];
+      }
+    }
+  }
+
+  /**
+   * Convert text back to string
+   *
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    try {
+      return decode(bytes, 0, length);
+    } catch (CharacterCodingException e) {
+      throw new RuntimeException("Should not have happened ", e);
+    }
+  }
+
+  /**
+   * Read a Text object whose length is already known. This allows creating Text from a stream which uses a different
+   * serialization format.
+   */
+  public void readWithKnownLength(DataInput in, int len) throws IOException {
+    setCapacity(len, false);
+    in.readFully(bytes, 0, len);
+    length = len;
+  }
+
+  /** Returns true iff <code>o</code> is a Text with the same contents. */
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof Text)) {
+      return false;
+    }
+
+    final Text that = (Text) o;
+    if (this.getLength() != that.getLength()) {
+      return false;
+    }
+
+    byte[] thisBytes = Arrays.copyOf(this.getBytes(), getLength());
+    byte[] thatBytes = Arrays.copyOf(that.getBytes(), getLength());
+    return Arrays.equals(thisBytes, thatBytes);
+
+  }
+
+  @Override
+  public int hashCode() {
+    return super.hashCode();
+  }
+
+  // / STATIC UTILITIES FROM HERE DOWN
+  /**
+   * Converts the provided byte array to a String using the UTF-8 encoding. If the input is malformed, replace by a
+   * default value.
+   */
+  public static String decode(byte[] utf8) throws CharacterCodingException {
+    return decode(ByteBuffer.wrap(utf8), true);
+  }
+
+  public static String decode(byte[] utf8, int start, int length)
+      throws CharacterCodingException {
+    return decode(ByteBuffer.wrap(utf8, start, length), true);
+  }
+
+  /**
+   * Converts the provided byte array to a String using the UTF-8 encoding. If <code>replace</code> is true, then
+   * malformed input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a
+   * MalformedInputException.
+   */
+  public static String decode(byte[] utf8, int start, int length, boolean replace)
+      throws CharacterCodingException {
+    return decode(ByteBuffer.wrap(utf8, start, length), replace);
+  }
+
+  private static String decode(ByteBuffer utf8, boolean replace)
+      throws CharacterCodingException {
+    CharsetDecoder decoder = DECODER_FACTORY.get();
+    if (replace) {
+      decoder.onMalformedInput(
+          java.nio.charset.CodingErrorAction.REPLACE);
+      decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
+    }
+    String str = decoder.decode(utf8).toString();
+    // set decoder back to its default value: REPORT
+    if (replace) {
+      decoder.onMalformedInput(CodingErrorAction.REPORT);
+      decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
+    }
+    return str;
+  }
+
+  /**
+   * Converts the provided String to bytes using the UTF-8 encoding. If the input is malformed, invalid chars are
+   * replaced by a default value.
+   *
+   * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit()
+   */
+
+  public static ByteBuffer encode(String string)
+      throws CharacterCodingException {
+    return encode(string, true);
+  }
+
+  /**
+   * Converts the provided String to bytes using the UTF-8 encoding. If <code>replace</code> is true, then malformed
+   * input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a
+   * MalformedInputException.
+   *
+   * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit()
+   */
+  public static ByteBuffer encode(String string, boolean replace)
+      throws CharacterCodingException {
+    CharsetEncoder encoder = ENCODER_FACTORY.get();
+    if (replace) {
+      encoder.onMalformedInput(CodingErrorAction.REPLACE);
+      encoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
+    }
+    ByteBuffer bytes =
+        encoder.encode(CharBuffer.wrap(string.toCharArray()));
+    if (replace) {
+      encoder.onMalformedInput(CodingErrorAction.REPORT);
+      encoder.onUnmappableCharacter(CodingErrorAction.REPORT);
+    }
+    return bytes;
+  }
+
+  static final public int DEFAULT_MAX_LEN = 1024 * 1024;
+
+  // //// states for validateUTF8
+
+  private static final int LEAD_BYTE = 0;
+
+  private static final int TRAIL_BYTE_1 = 1;
+
+  private static final int TRAIL_BYTE = 2;
+
+  /**
+   * Check if a byte array contains valid utf-8
+   *
+   * @param utf8
+   *          byte array
+   * @throws MalformedInputException
+   *           if the byte array contains invalid utf-8
+   */
+  public static void validateUTF8(byte[] utf8) throws MalformedInputException {
+    validateUTF8(utf8, 0, utf8.length);
+  }
+
+  /**
+   * Check to see if a byte array is valid utf-8
+   *
+   * @param utf8
+   *          the array of bytes
+   * @param start
+   *          the offset of the first byte in the array
+   * @param len
+   *          the length of the byte sequence
+   * @throws MalformedInputException
+   *           if the byte array contains invalid bytes
+   */
+  public static void validateUTF8(byte[] utf8, int start, int len)
+      throws MalformedInputException {
+    int count = start;
+    int leadByte = 0;
+    int length = 0;
+    int state = LEAD_BYTE;
+    while (count < start + len) {
+      int aByte = utf8[count] & 0xFF;
+
+      switch (state) {
+      case LEAD_BYTE:
+        leadByte = aByte;
+        length = bytesFromUTF8[aByte];
+
+        switch (length) {
+        case 0: // check for ASCII
+          if (leadByte > 0x7F) {
+            throw new MalformedInputException(count);
+          }
+          break;
+        case 1:
+          if (leadByte < 0xC2 || leadByte > 0xDF) {
+            throw new MalformedInputException(count);
+          }
+          state = TRAIL_BYTE_1;
+          break;
+        case 2:
+          if (leadByte < 0xE0 || leadByte > 0xEF) {
+            throw new MalformedInputException(count);
+          }
+          state = TRAIL_BYTE_1;
+          break;
+        case 3:
+          if (leadByte < 0xF0 || leadByte > 0xF4) {
+            throw new MalformedInputException(count);
+          }
+          state = TRAIL_BYTE_1;
+          break;
+        default:
+          // too long! Longest valid UTF-8 is 4 bytes (lead + three)
+          // or if < 0 we got a trail byte in the lead byte position
+          throw new MalformedInputException(count);
+        } // switch (length)
+        break;
+
+      case TRAIL_BYTE_1:
+        if (leadByte == 0xF0 && aByte < 0x90) {
+          throw new MalformedInputException(count);
+        }
+        if (leadByte == 0xF4 && aByte > 0x8F) {
+          throw new MalformedInputException(count);
+        }
+        if (leadByte == 0xE0 && aByte < 0xA0) {
+          throw new MalformedInputException(count);
+        }
+        if (leadByte == 0xED && aByte > 0x9F) {
+          throw new MalformedInputException(count);
+        }
+        // falls through to regular trail-byte test!!
+      case TRAIL_BYTE:
+        if (aByte < 0x80 || aByte > 0xBF) {
+          throw new MalformedInputException(count);
+        }
+        if (--length == 0) {
+          state = LEAD_BYTE;
+        } else {
+          state = TRAIL_BYTE;
+        }
+        break;
+      default:
+        break;
+      } // switch (state)
+      count++;
+    }
+  }
+
+  /**
+   * Magic numbers for UTF-8. These are the number of bytes that <em>follow</em> a given lead byte. Trailing bytes have
+   * the value -1. The values 4 and 5 are presented in this table, even though valid UTF-8 cannot include the five and
+   * six byte sequences.
+   */
+  static final int[] bytesFromUTF8 =
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0,
+      // trail bytes
+      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
+      3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5 };
+
+  /**
+   * Returns the next code point at the current position in the buffer. The buffer's position will be incremented. Any
+   * mark set on this buffer will be changed by this method!
+   */
+  public static int bytesToCodePoint(ByteBuffer bytes) {
+    bytes.mark();
+    byte b = bytes.get();
+    bytes.reset();
+    int extraBytesToRead = bytesFromUTF8[(b & 0xFF)];
+    if (extraBytesToRead < 0)
+    {
+      return -1; // trailing byte!
+    }
+    int ch = 0;
+
+    switch (extraBytesToRead) {
+    case 5:
+      ch += (bytes.get() & 0xFF);
+      ch <<= 6; /* remember, illegal UTF-8 */
+    case 4:
+      ch += (bytes.get() & 0xFF);
+      ch <<= 6; /* remember, illegal UTF-8 */
+    case 3:
+      ch += (bytes.get() & 0xFF);
+      ch <<= 6;
+    case 2:
+      ch += (bytes.get() & 0xFF);
+      ch <<= 6;
+    case 1:
+      ch += (bytes.get() & 0xFF);
+      ch <<= 6;
+    case 0:
+      ch += (bytes.get() & 0xFF);
+    }
+    ch -= offsetsFromUTF8[extraBytesToRead];
+
+    return ch;
+  }
+
+  static final int offsetsFromUTF8[] =
+  { 0x00000000, 0x00003080,
+      0x000E2080, 0x03C82080, 0xFA082080, 0x82082080 };
+
+  /**
+   * For the given string, returns the number of UTF-8 bytes required to encode the string.
+   *
+   * @param string
+   *          text to encode
+   * @return number of UTF-8 bytes required to encode
+   */
+  public static int utf8Length(String string) {
+    CharacterIterator iter = new StringCharacterIterator(string);
+    char ch = iter.first();
+    int size = 0;
+    while (ch != CharacterIterator.DONE) {
+      if ((ch >= 0xD800) && (ch < 0xDC00)) {
+        // surrogate pair?
+        char trail = iter.next();
+        if ((trail > 0xDBFF) && (trail < 0xE000)) {
+          // valid pair
+          size += 4;
+        } else {
+          // invalid pair
+          size += 3;
+          iter.previous(); // rewind one
+        }
+      } else if (ch < 0x80) {
+        size++;
+      } else if (ch < 0x800) {
+        size += 2;
+      } else {
+        // ch < 0x10000, that is, the largest char value
+        size += 3;
+      }
+      ch = iter.next();
+    }
+    return size;
+  }
+
+  public static class TextSerializer extends StdSerializer<Text> {
+
+    public TextSerializer() {
+      super(Text.class);
+    }
+
+    @Override
+    public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
+        throws IOException, JsonGenerationException {
+      jsonGenerator.writeString(text.toString());
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/TransferPair.java b/java/vector/src/main/java/org/apache/arrow/vector/util/TransferPair.java
new file mode 100644
index 0000000000000..6e68d55226266
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/TransferPair.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import org.apache.arrow.vector.ValueVector;
+
+public interface TransferPair {
+  public void transfer();
+  public void splitAndTransfer(int startIndex, int length);
+  public ValueVector getTo();
+  public void copyValueSafe(int from, int to);
+}

From 16e44e3d456219c48595142d0a6814c9c950d30c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes@cloudera.com>
Date: Tue, 16 Feb 2016 16:02:46 -0800
Subject: [PATCH 0004/1644] ARROW-3: This patch includes a WIP draft
 specification document for the physical Arrow memory layout produced over a
 series of discussions amongst the to-be Arrow committers during late 2015.
 There are also a few small PNG diagrams that illustrate some of the Arrow
 layout concepts.

---
 format/Layout.md                           | 253 +++++++++++++++++++++
 format/README.md                           |   5 +
 format/diagrams/layout-dense-union.png     | Bin 0 -> 47999 bytes
 format/diagrams/layout-list-of-list.png    | Bin 0 -> 40105 bytes
 format/diagrams/layout-list-of-struct.png  | Bin 0 -> 60600 bytes
 format/diagrams/layout-list.png            | Bin 0 -> 15906 bytes
 format/diagrams/layout-primitive-array.png | Bin 0 -> 10907 bytes
 format/diagrams/layout-sparse-union.png    | Bin 0 -> 43020 bytes
 8 files changed, 258 insertions(+)
 create mode 100644 format/Layout.md
 create mode 100644 format/README.md
 create mode 100644 format/diagrams/layout-dense-union.png
 create mode 100644 format/diagrams/layout-list-of-list.png
 create mode 100644 format/diagrams/layout-list-of-struct.png
 create mode 100644 format/diagrams/layout-list.png
 create mode 100644 format/diagrams/layout-primitive-array.png
 create mode 100644 format/diagrams/layout-sparse-union.png

diff --git a/format/Layout.md b/format/Layout.md
new file mode 100644
index 0000000000000..c393163bf894b
--- /dev/null
+++ b/format/Layout.md
@@ -0,0 +1,253 @@
+# Arrow: Physical memory layout
+
+## Definitions / Terminology
+
+Since different projects have used differents words to describe various
+concepts, here is a small glossary to help disambiguate.
+
+* Array: a sequence of values with known length all having the same type.
+* Slot or array slot: a single logical value in an array of some particular data type
+* Contiguous memory region: a sequential virtual address space with a given
+  length. Any byte can be reached via a single pointer offset less than the
+  region’s length.
+* Primitive type: a data type that occupies a fixed-size memory slot specified
+  in bit width or byte width
+* Nested or parametric type: a data type whose full structure depends on one or
+  more other child relative types. Two fully-specified nested types are equal
+  if and only if their child types are equal. For example, `List<U>` is distinct
+  from `List<V>` iff U and V are different relative types.
+* Relative type or simply type (unqualified): either a specific primitive type
+  or a fully-specified nested type. When we say slot we mean a relative type
+  value, not necessarily any physical storage region.
+* Logical type: A data type that is implemented using some relative (physical)
+  type. For example, a Decimal value stored in 16 bytes could be stored in a
+  primitive array with slot size 16 bytes. Similarly, strings can be stored as
+  `List<1-byte>`.
+* Parent and child arrays: names to express relationships between physical
+  value arrays in a nested type structure. For example, a `List<T>`-type parent
+  array has a T-type array as its child (see more on lists below).
+* Leaf node or leaf: A primitive value array that may or may not be a child
+  array of some array with a nested type.
+
+## Requirements, goals, and non-goals
+
+Base requirements
+
+* A physical memory layout enabling zero-deserialization data interchange
+  amongst a variety of systems handling flat and nested columnar data, including
+  such systems as Spark, Drill, Impala, Kudu, Ibis, Spark, ODBC protocols, and
+  proprietary systems that utilize the open source components.
+* All array slots are accessible in constant time, with complexity growing
+  linearly in the nesting level
+* Capable of representing fully-materialized and decoded / decompressed Parquet
+  data
+* All leaf nodes (primitive value arrays) use contiguous memory regions
+* Each relative type can be nullable or non-nullable
+* Arrays are immutable once created. Implementations can provide APIs to mutate
+  an array, but applying mutations will require a new array data structure to
+  be built.
+* Arrays are relocatable (e.g. for RPC/transient storage) without pointer
+  swizzling. Another way of putting this is that contiguous memory regions can
+  be migrated to a different address space (e.g. via a memcpy-type of
+  operation) without altering their contents.
+
+## Goals (for this document)
+
+* To describe relative types (physical value types and a preliminary set of
+  nested types) sufficient for an unambiguous implementation
+* Memory layout and random access patterns for each relative type
+* Null representation for nullable types
+
+## Non-goals (for this document
+
+* To enumerate or specify logical types that can be implemented as primitive
+  (fixed-width) value types. For example: signed and unsigned integers,
+  floating point numbers, boolean, exact decimals, date and time types,
+  CHAR(K), VARCHAR(K), etc.
+* To specify standardized metadata or a data layout for RPC or transient file
+  storage.
+* To define a selection or masking vector construct
+* Implementation-specific details
+* Details of a user or developer C/C++/Java API.
+* Any “table” structure composed of named arrays each having their own type or
+  any other structure that composes arrays.
+* Any memory management or reference counting subsystem
+* To enumerate or specify types of encodings or compression support
+
+## Array lengths
+
+Any array has a known and fixed length, stored as a 32-bit signed integer, so a
+maximum of 2^31 - 1 elements. We choose a signed int32 for a couple reasons:
+
+* Enhance compatibility with Java and client languages which may have varying quality of support for unsigned integers.
+* To encourage developers to compose smaller arrays (each of which contains
+  contiguous memory in its leaf nodes) to create larger array structures
+  possibly exceeding 2^31 - 1 elements, as opposed to allocating very large
+  contiguous memory blocks.
+
+## Nullable and non-nullable arrays
+
+Any relative type can be nullable or non-nullable.
+
+Nullable arrays have a contiguous memory buffer, known as the null bitmask,
+whose length is large enough to have 1 bit for each array slot. Whether any
+array slot is null is encoded in the respective bits of this bitmask, i.e.:
+
+```
+is_null[j] -> bitmask[j / 8] & (1 << (j % 8))
+```
+
+Physically, non-nullable (NN) arrays do not have a null bitmask.
+
+For nested types, if the top-level nested type is nullable, it has its own
+bitmask regardless of whether the child types are nullable.
+
+## Primitive value arrays
+
+A primitive value array represents a fixed-length array of values each having
+the same physical slot width typically measured in bytes, though the spec also
+provides for bit-packed types (e.g. boolean values encoded in bits).
+
+Internally, the array contains a contiguous memory buffer whose total size is
+equal to the slot width multiplied by the array length. For bit-packed types,
+the size is rounded up to the nearest byte.
+
+The associated null bitmask (for nullable types) is contiguously allocated (as
+described above) but does not need to be adjacent in memory to the values
+buffer.
+
+(diagram not to scale)
+
+<img src="diagrams/layout-primitive-array.png" width="400"/>
+
+## List type
+
+List is a nested type in which each array slot contains a variable-size
+sequence of values all having the same relative type (heterogeneity can be
+achieved through unions, described later).
+
+A list type is specified like `List<T>`, where `T` is any relative type
+(primitive or nested).
+
+A list-array is represented by the combination of the following:
+
+* A values array, a child array of type T. T may also be a nested type.
+* An offsets array containing 32-bit signed integers with length equal to the
+  length of the top-level array plus one. Note that this limits the size of the
+  values array to 2^31 -1.
+
+The offsets array encodes a start position in the values array, and the length
+of the value in each slot is computed using the first difference with the next
+element in the offsets array. For example. the position and length of slot j is
+computed as:
+
+```
+slot_position = offsets[j]
+slot_length = offsets[j + 1] - offsets[j]  // (for 0 <= j < length)
+```
+
+The first value in the offsets array is 0, and the last element is the length
+of the values array.
+
+Let’s consider an example, the type `List<Char>`, where Char is a 1-byte
+logical type.
+
+For an array of length 3 with respective values:
+
+[[‘j’, ‘o’, ‘e’], null, [‘m’, ‘a’, ‘r’, ‘k’]]
+
+We have the following offsets and values arrays
+
+<img src="diagrams/layout-list.png" width="400"/>
+
+Let’s consider an array of a nested type, `List<List<byte>>`
+
+<img src="diagrams/layout-list-of-list.png" width="400"/>
+
+## Struct type
+
+A struct is a nested type parameterized by an ordered sequence of relative
+types (which can all be distinct), called its fields.
+
+Typically the fields have names, but the names and their types are part of the
+type metadata, not the physical memory layout.
+
+A struct does not have any additional allocated physical storage.
+
+Physically, a struct type has one child array for each field.
+
+For example, the struct (field names shown here as strings for illustration
+purposes)
+
+```
+Struct [nullable] <
+  name: String (= List<char>) [nullable],
+  age: Int32 [not-nullable]
+>
+```
+
+has two child arrays, one List<char> array (layout as above) and one
+non-nullable 4-byte physical value array having Int32 (not-null) logical
+type. Here is a diagram showing the full physical layout of this struct:
+
+<img src="diagrams/layout-list-of-struct.png" width="400"/>
+
+While a struct does not have physical storage for each of its semantic slots
+(i.e. each scalar C-like struct), an entire struct slot can be set to null via
+the bitmask. Whether each of the child field arrays can have null values
+depends on whether or not the respective relative type is nullable.
+
+## Dense union type
+
+A dense union is semantically similar to a struct, and contains an ordered
+sequence of relative types. While a struct contains multiple arrays, a union is
+semantically a single array in which each slot can have a different type.
+
+The union types may be named, but like structs this will be a matter of the
+metadata and will not affect the physical memory layout.
+
+We define two distinct union types that are optimized for different use
+cases. This first, the dense union, represents a mixed-type array with 6 bytes
+of overhead for each value. Its physical layout is as follows:
+
+* One child array for each relative type
+* Types array: An array of unsigned integers, enumerated from 0 corresponding
+  to each type, with the smallest byte width capable of representing the number
+  of types in the union.
+* Offsets array: An array of signed int32 values indicating the relative offset
+  into the respective child array for the type in a given slot. The respective
+  offsets for each child value array must be in order / increasing.
+
+Alternate proposal (TBD): the types and offset values may be packed into an
+int48 with 2 bytes for the type and 4 bytes for the offset.
+
+Critically, the dense union allows for minimal overhead in the ubiquitous
+union-of-structs with non-overlapping-fields use case (Union<s1: Struct1, s2:
+Struct2, s3: Struct3, …>)
+
+Here is a diagram of an example dense union:
+
+<img src="diagrams/layout-dense-union.png" width="400"/>
+
+## Sparse union type
+
+A sparse union has the same structure as a dense union, with the omission of
+the offsets array. In this case, the child arrays are each equal in length to
+the length of the union. This is analogous to a large struct in which all
+fields are nullable.
+
+While a sparse union may use significantly more space compared with a dense
+union, it has some advantages that may be desirable in certain use cases:
+
+<img src="diagrams/layout-sparse-union.png" width="400"/>
+
+More amenable to vectorized expression evaluation in some use cases.
+Equal-length arrays can be interpreted as a union by only defining the types array
+
+Note that nested types in a sparse union must be internally consistent
+(e.g. see the List in the diagram), i.e. random access at any index j yields
+the correct value.
+
+## References
+
+Drill docs https://drill.apache.org/docs/value-vectors/
diff --git a/format/README.md b/format/README.md
new file mode 100644
index 0000000000000..1120e6282a50a
--- /dev/null
+++ b/format/README.md
@@ -0,0 +1,5 @@
+## Arrow specification documents
+
+> **Work-in-progress specification documents**. These are discussion documents
+> created by the Arrow developers during late 2015 and in no way represents a
+> finalized specification.
diff --git a/format/diagrams/layout-dense-union.png b/format/diagrams/layout-dense-union.png
new file mode 100644
index 0000000000000000000000000000000000000000..5f1f3811bf0056defe19abf494afcaba1bedbb77
GIT binary patch
literal 47999
zcmeFZbx>W~*DVMH2n2TxF2OB8a0%|gHMj+L2n2V6yF-8wT!RG-8k``(-Q6Wvuan&S
z``&x?y1J^XySo4QZk^nO&E9M8z1CcFjycAhCrn8}66G1;Gbku16lp0jWhf|^Z78TG
zIS4S|$g&u=Blrv2Nm)`9s(hGu2mF9&FQw%K1%*ou`2#JjOmz%Sn6y;Ybk>xY<2AOk
zVKy|eGcsj%x3LFjLqYMo^MZfcm^vGhx!YLVI`O&-Q2aTA7yKJ?n1zDu&neDU0u-9^
zN@TC?98JkMnO`!qQV2dHBO~K?G%@2<7L)kra`2M?g@v=TJueH3o0}W68wazUqd5y3
z4-XFuD?1B2I}<pA$;rdk+0dQI)`{}(ApeddX6j_@Xld_kX=h6Y8Q0Lr&c#`Pf&y}*
z|NQs&Ih`%d{_9S*PXD|XctIA(Z&=uvSy}!wHn@}@a+LSAosGStsgn~JUyzgk&zb+z
zvH!Zy-}99$?40euEI3*kOWQh|I)aOx4Ixh_$o|jO|L14?uVX1XTAG5l{&O|kKUe>c
z$No8=p9S*b|HDN5o#j7A!88j#<7fHLk_kSmTw`B=f)a+3786l*hu+WhSjHKDd@Nx6
zfr%L=6;bv{oa$DcJ}z4Eh1HK%FO#L3{JvIdMCmEUTsRDI@$|lOWvV#zT&cF+Uwc<P
z*=!tFuE~v80!~L~_mY`Bw;mad92Fj!zdSN0)yK0DLVf=q`Urbqe5rNZ=szW_l3;+L
zAp7%)(S#|Xt=rmd`1dBlGZ-<;%C0Kq&=G%cEbNFVWiNd=U$phlB}%TaR1FCW30Cd@
zj-d;KsT;@Vw%hp6APLW*5t&W7HwY{LIY5={3!A(?ocW&lpZl6h3j52>$Tz|>{&V05
zHCYFBqf=qlzjyub$N0ZF8#toNooGV*XM-F^O?%}+57&93yg3r$!mwgFFA-GfA`LCD
zDPn@#^?mL+ZRcMNRL<FcD{Hx&%}o`$UJf{Fy^YHMET3*$*#V#H;TXk#+DB<VoFRSU
z@=Nb35KE!2kpu@D8Ux*rot7+8HgyTgl+7!F`1axIXwI%po^$hU%X=;6cvfZ80rH2!
zZ13N9XCr0H6GbXcUqr;kz#|C5e)NUKTR@~xvWMBW8y0%J>rZ5`YrWYt&;H=rF{knb
zVp2*?5Cc1$e`n%AVSn)Tm-pEa_u1x<)o6}L#_xhw$c7O9j?mXbZHV>xkS19;l8`fQ
z&f`KCGQ5$fFky0vTwoLVzZPEk4p^u9kQC3~`~N*hR-7#r!G-cO?As3)lRM*u<BhM;
z1ZoXCgJIQ6HBYYBqS2o7p65jiYC8>a>eO2JtuXO@`S1i^$omew=Rw4JwbifkukR+G
z^@Q{crg2(zg<_0<md`_Ld-%;zc3l7Xa9eLZ!=Cb3zvTVx_V8PnxqSaYLl%emFjM_J
zyXDXPo3kyuuaoMM&s`>d&}Qp`kqjKV|GXDFLg_>{LISVflF#G9Jz?ARYJ_U1z-lNZ
z<6OF3>(1#i=P*SG3XufW_x{T{yQ6nC?b=ploQB_FHM(H%^u4NB()B1_Upszo^15CL
zDN!rIEj78A)M(qU9(cGrQ`GMekHFR1l!zpZ4ZKQYsA#U1!FlYJ?u~jbeVRCQce>!V
zbTCnbWn=nQj0a*dWTG(I^v_!Huqk0yV9_m(^+WbW*}d15yvbgpzpb+-I67r%yZfnw
z90slEb~xhro+BUe4C!dG<<_D1dA;+Fa`i#OTGT?53*TO8jZ&==laVC&<rSDhQlC3E
zHR|<_jRnqMx&dCVN$S{nI7g0GI>C6Ss&M07BPnca$DL^A6C;ARi~@4;D$;)}Ll_3*
znMkn=?5Z>wX5_#@{nFbSGkNcDBAePVuz)7dqAOZ|7Xu@_yxpIvF;80|b)7a$V#10<
zYQi1($Q)8xKO2TctIgf#D~oRzr?IvwhNa#h_d_h?_F~U`QoUFeG24BydZ+WVX>J0J
zzFo^jOzKa(H^mSyh8^1e4z5^ZGF;dTL5!jy$7tniOE<2L0psfPSy`4AlM1Ki!}n_+
zRtDklY8k&;{i@>y=5MJlpzD1(V{TPi1y3;An)Bd5kg;4n%)6Ho<JZ$&plxuq;8vfu
z<i3WYue@!lWMY`OdwbM|HdHEwfewxF0tT81XU16fF}$4~;qG*RWm_oAW0RnH>3vO|
z?ZPax1M^no2#?pCO+7i$8?UWQH#18bO~w1)7h2*JB-H%{$!`e<eF<1bQ2TjXA_mrA
zlhrP#>RD?ae($Le^{4NZe?!fTFHr)!8Qh8;cx-X_@=Z*+<t)#`8PjAvuP5!^Q|F|i
zNUizv%vJeS{=>|#G|?`Zad+5nPxmW3Q3IZ$xkm6YyL6&(*V?Bo{uaX>W@uOqXF%7o
ze_)91j#Y)YoKFbXZp5mpLKEc(T^Uf{LaSm>4ybS&GS*Tguo0LN=}*_9h3fJT>gJf$
zvTW)XaC2e5KI_$DdD}cqt5{habYEI#bMr2z1Q%;oM=YrXtZw60R-RckdG+1aQQPih
z$?fh~0c3epv7s$>>BF06BcDUNuIBHIi#&avK%Y2{C2eH)rWiT3*Eov9?NE21?P5x&
zkUsRTYUgvh_AYI<*Xn?Th!f9R#MaLynf6~g*-3EHK{eH5)yEb2Nn{d`O{q!-85+Gc
zjTsaAs_wJEICUV`egb+2PSWUIP^d{wbbK}|gY-CTQO2wk{7YubYSXxv_RH-m3OwDh
zGiK>FJyFkf3KcTt`st(p7*eY5%Rf_1?+f+KZrZ{ag>Q#y8ZCKU#{+SbQ5X3=HW!i3
zy<jhEtHey04cqkY@3`ug{jt2K>9uHr*9Hu)JTI;H{n@_jRhvlHsnNCl88Z#US5YkW
z`zo^KDL(Iqn=RGnj?%+#WQsH3B742NcipTcIyPBt*w~Q18Ei?>q%fV~>&`_I^TcsH
zlk>V+2_a?eE7PfG&N?Jc8njYT_+t{pz$Ai<OVW(Rp!ihwBAF^+J$n$v%oxA(2-@z+
zv+7_NWyl|<uo)|JZpFoDkJ1+OC)qSBdKqGbGmUBvp6`r@EG*AVeZKp%aS?zMekP*k
z(h~*Md!?EEO7%sp6S0W}??*11S=F3Kqz}d`J?~!`v_t8(z<EZnJ&OzMuK7imLOzm`
zbM0S`-<90@;a0}$M|@d000`@f06jb)tG$LnsI{4!dOy?V<70xYAFYML^KiY!RPD4i
z+&V#_fB5y+Ti1C?X6IPhW?P>!+I>y$$NMY98kZN570LYH4zTpmbD100F>#}98&=@?
zMCe9-tNjTgj+6k5aNE)+vceG0)SUDXja!;v2jaU9EqiqL_%0<d>orY}g?&40vdV=^
zF3ftr%RHj5#GI?az35KKpbl$4@eO#bk0b(VYDCzgYrWAzzqj)+bW69mm%M6jT5ASG
z@Ak0tU#a=3RAA_vTq`5Y7OMbXiv<%~7G?f3<up`S9OKKIJoqdVKG*$FCZwb?;<7=(
zJX|Ji`@<1`IHXdI{cQqvlTf!KXY%(EwJT+~B3(Q|PA5|EjF#l0V*=)^vc22Q8y!>E
z9ZyO3M{@%hlt+ETSdN;{P;K>q?}rnfZ*N#ogtO63{DZclfk6tAH8E<z9L&{A8n5uz
z<hJ8;+h)XTzjCxVUH^eX_%igz^-K)E%kG46w7`@q%Xxo3uAIYIhg`PcLguVyEvn~N
z791;h)mrirasUqSF|UOxR@nBFny6slDB4k?*#pYj+@(8gjpOCHI>re_g=@@)o=q5K
zO;S`;ehkX@&!4Qd9Pf!F;(J3glJ<8E;-tVDW17o<!2RQ(0OFv|Dfq;m>5ifUiSyv|
zzd3`fqd<=KZRfw2P+`U_$8~9`WJ6*Z#2d*0C`&gU_N(%r17u+UX;3*#O_)P09^wa8
zfSvX`uX0-db3lp$>;mZw&I_%7?%JOKw%Y#(kKsozy<hvA5yn*1V6oW)jsLhEm`E##
z3L68-c8wcJp8Hk3(zKbi;|hG;f$g8fBg}>~Ql-u79&TCUGlB-re(C$P++WU7N_<>i
z0&a8{pot<7C^LCopyt5jgUf6(U8ba$ekahWK%z{r+zCJ_6#yAVK0a817#@Q9{AaQn
zh=sy0VbymPfx#xh!lMwX0`RhY)T~?}gQw>_2I4wK$XlVSd8ZDN;GXl{NrqEY%E9}~
ztEF%p2KJ(6o4LB&4>f&5U?&_ecjR6$zHM$WAIZ|O`Kp$-`t7Kd&q&w8K|I_SK}-tZ
zxR+=Y)M3Iw7!6m8o^PNu&n)1sA;B_I-}B(>)sj#4(jyA@e0#}x>+NAl<wJMgitR#^
z06?tgrh_SLAdo)N^Id?y;?0LRf6arC2I58i5Z4?elmY`J<Z)cDHXUdM@el-|R<ofr
zksJ8QUjU*RUF2n1P8Q=lva<kJ@&UM0JZ4>~pMi*2yl#hgyCoIn)kOUOsK%fH<G2}B
z(ZYCv;0K=ZXuoEpu6TrJE5rN#cUe^rq3aLNv}6WZk|$45=K;cP0+1+%#6))oJTQQT
zNAGL6?G}mepR#oZqm<~kwaU&~*9_@4e1a+6b>5vQ0vJ=uZ&7eCm1C=qGCG6bgFUvi
z3-UCj&>kG<ize~%@6WfP?}AZ?-p3y{tf6j(?vP>rh!YnuA+&=J0wzvDgPwaSp%w<F
zQ`Fb3@P4?;0+AU}&%OgLX|`0WMk+0$E{<O9;c*K+m~3o{xbz^!IEoG1Lf5}Yt9vj@
zfJpx1B$vagnEnh%780rH^z*-oDOvr@4@(sHcQ|<l0Y(14t~fK}5|81+x(JL~)$3Kg
zBzfGBFb%Nt?mE;jA!Is6ew*3aB9+1>ceqn>rFvUETCZzoDb<LoH%d>}(u_Z&vmH40
zd?a9g3pSviC^|?!5sHC7bfopItOoPdpH3cBL4iw2;&m*7N5_E|Lh&fP(Vtir_inP-
zRA&y%s+ssSU!slH1|Bp<U%0QeZuR$qL5dhUzg|+GjOR7$W(!Xlh7H(WGSIkIrgmEu
zpN_sp)7p%B&i7lUO55S|2QAhQhN*q27{|lrGllw|l>s4;7Rkx90T9nI!jDCSs{gM0
ztx*^6hu_Y>?YDD7(k2loVP;2jUjwgBDKP<d+S!OuwqAet4(<2~HdD%u$>a;u-e^)q
z3@)#)c1waK)%_sJjp6hYEfp$)MAmv&-vaX&ku0s(-@VJf9n__Bcfu1{8=R1OU*~dX
z9S7mPwUCDFR=8`tHDrCRZzgRYOTw-3?}E)D_<xq!6x2J2Bvq3ebjp&VtJf`gdvd=(
zD*Za8Z9OSH-ictN8zpe5X?5=V1XXI>$Re&r+UQKf0{)*ZB`e-4S->owLxET+bv~{r
zq=i=5bnFWE!S|p~;JE$C$=Y|>1?8Epy`jzT;PF0{giASdFw@!kf$fU5gEA8oRR)5d
zg;pgY`5>+-7j%_k@0?*R;o-=oXnc6>3PlU_yW=Dj6qUy@Nm2omK6GUVi7mY_q@kmT
zuGh==S4)cP5=1r=AXw`cGegXi7I^f&*kVzPzD(bEQFkBDG7&9_m70eUp~o(ngnG&&
z7i&7fn{_O0M{%z*NHl*T^up*QwSnhkQYl0BZY0b5_fDT?^L6Z?(x*yviFL38BSRAH
z+U_swKvsLRnNGQn9N}OQAl$D_@|*sp&gRlaP5>$)t3f-ij&-buj&X04z{P~hkCBgL
zCnjoKXX*(c4!6q+`$r@Pf2UCDfX%@6Cwgh8;qEa9|B0cz0cOOKYK5PvWm&mIy-dfh
z1uSoCDZ$DYraJ!t;9exqJT}&XNu5ZETn=Wx9s#5Zz>U4-XV$15J-3C8@2{FTw-~^7
zs9$hpUWA)(N{k?9^}0UJ>C|86cc!h~8%h=Yhzdd0#%;Vahpo4^A6<TKX+S(K6$n#e
z{%;Z?zBgl8IZpybgYs~!y!ljy`6N5IrMSE1iy^c|*M=MP+pX;<#}RG)l<vB*t50LJ
zH7{48$Y{T6lD@j{g>f&txZM57F>Ie4Pb1IX0S8Ut^aD8;ed~VP-TUrDQUfP=kgWUX
z#^-d~)9-EsUeI#Mr~q*eI)?SeR_*9{c{tO7<QxH}+JO#MZITX#2NPj|v`ZG_sRg4%
zCDM<^QR<#q1g6zA$qjV#qVC^ej&P)R<s-NTiq$AnkrT<}Te4FDl^-7f+WbOr^Te18
zH|z_FPuCZF%}><_+m^e9uN*PWU&e)Sq-2aar%}jYLSux6gJAnbs+a^L)Yo^MH)Rrz
z)9B@I;J&ehy*m6Szpl)WHc~LtX+d+j*Tmt>n%A6$>DoKk7_xRX5fP>6zu%$8Ew;wO
zrD55bq}A-%2kH3dr%ji$RuxUV^cuMkhCrMQU<QPk$v^}Xnf4h?FOZNWtSmW=2qZ)O
zl+G6X(EL>sUiP$ymf3nXYScDevO&%jJ_vV&9t4ggG@(0IzPK0NX4G5b-5Q7`vZd%%
z?vnf*jyD_0x9r=>0C0?gg~Lv(+{N=_M6k!qZQ;`jZohs~1b5|-&^s{yd#~cCEc64n
z>yF8DyW;OW=<Q9eg_*9^{wkm45T|?*g4j^_;-s23)2x_VuU8F7AlV4^zfj*)rky{Q
z)qS2h2p}w5R+0dm8sq}(x108qvLtW6gphhSaC4w;;xTF(uu1FFpGK8@?`+unDi}xt
z<I+!8*bGpuN&6Af+&B8>`dwdy1X>WvHh7K*T{;jAgbZyei)g3u6~r?3G_y_C232ir
z5JH$2QQFmULdbQXT##ha4DL-a*A(y^58s^<zp5<@UUm>??R8F84NMmafZv#j`GeEt
z0EQdhR#k%iiDB>Qk&#-)&0TKCQ`$?k_FdrpxxFFGFR(F^bZmDb$?y6ZDm3M;4i_@t
z^(Ge*Gjq?|zb5+ZX6z*R#O*Va!iCXS@()cm(;&)L#LJvn%9;WP@-I%OhAAOimQ(h!
zm3nbGZ5&<T05{;lZF}+bU_i?}bHfU;mnlyok^(apuA@77_ge;^Te&B}ZJf;VPafSH
z;0Dh~QIYKTnX2D=`45C*P?jR9>JFuG4iqwF92gNBbEGH=&@;f80=FPLqtF8XRuhKr
zupy7rS5fWAXf$3?(9-<WVO_}r?YMch?y2!pW9H#z9*=`m+(H0$hEX!P1Ca8XJ35#p
zANPSr$YoGxHWz&Y+-S)~hB$G(&^#Zm9o6PwZW2QhX{U|?5o>4Oa3<fc3jHOW5rE6~
zF{^%-7Pb_nhG4v&k%{bVay^(`fucF{?+1Hv8KKXq{B<q}@6x_kScDi}X@*38Ok0Sw
z8SN6ev{~*(=i6g1$D2LQzyo&<zea6E&`QLAWlz?7Ug@_H^BT27>=!I{U<#lwCZe8n
z@#wt2Rl19vUh#R^OBYQ1eZD|8`Gw@Qo1QhY1kp-a68Ri;FR$aiudGoUo6jM8TsuH9
z1H6QVRF1_MQERqs_pZ*Wj6WS<#;#A+X2RhFre;f-ntL2fxQi12x<v6bFDxTZH04W~
z!*~|T*nZ_QV(Dx@;n{^Dd9C~*Ip?x_ntFUoprgv|Xfaeq5RtAnhXO3|gNU!4$UQ(X
z*#1z%YmX<!w?}Xlih-DmdANQ$jRt$}Iaegr(_h|D>MnoVpnc{zVN{vG9gOAiFt_}8
z?*phWPnEo9PtopAxOP$U434>Wt>Zn0Jl9%Q02WkvLfjnN%E}&owGfTHEsz}=5Tomz
z&u!;}uA367UZ$P8#3TEibm1cbhb{)5W<lj@*-YXo-1JiG;`==hr*cQk51f!q4h1-}
zQ$Y!AT)!8H8E%UnTNzGKb2p-s&6EyWJy$J9$lV4djEDB<(M2Xw;uoh=I(9SFV>Id3
zr+^$SygHP@z@?lw>~0>=utdSye@$VuJ5`26KzuxR1I0gK4VL|Ix{A_PWB$doclXb@
z8dEbI#5ZQEU17x9VkBbE)a{j2I4ys6;e>IuD^O=|a4NXx*@%7(6Xrqx(sk5K+*8CB
z4$?^jwjk4p?g+d*{AC2AiYb#sz16L-gaC5d67meuk7WLFzR7HbnB~=s1fzSEyM9P~
zxRQii)+y4l>zf@?a2zJYy}8cN*m2o&PV3Jnz6T*`%^7uv5Zv5sW&dbASn}z6x!6r<
z7i-bE{Y8^i^gLf%{a%wPyv}=)28J4vtN&1}f`0)+fzAR$+&^8RFZ4T5=^G$odP|&k
zBN4i?e3QbBgv57qK%}$8sFeS;Uz#+btc_JNh8+g{KZTz<2ZTd_g1+oD_K^3LSA?Me
zfa(cwSUkci7?4D*FAa)D5})WfNP-f&TpR)0Qe7?z2U0$D9eFP1CZs>fA3p4HTnHbj
z_V<qE4_6CY3%2SXHz$J;5}rZx<S-<rPP_tVt;$2Wc?o!4c2Oe_8?P>?TqwXA$%9Np
zQ!$#C_FqPMnVO6Uj|BySupt8j(pet7V`-seCf46v6&m9hWS*-9x&P`Pp~9sg^9;K<
z|KbFBlRuxgAk%DqiTzKm3Su26c%iB+5*LMkM*S}&^6w>8G=P_w8qXUB`vHUk@Nq{3
z9smMX$Y7{t4KDX*knot1QSBny9{o?kF~Fd-&>q8)Tg_iID2~x!07tD;ZG!klKJDd<
zMPB4Xta>h91TIq=kJIP9?;x#tTc-UL5dFzM+;i47^lC=umJ@lB(IA6ba+!o<2F$Eg
zMaz^PJ$M{+Cm6Q(rCk{bO|&pLq6Rmo8^rcbAa?5P0Z;pGuFfXtoT_%L9&8i<XOQGY
zT`gy-H{kGe?T<U2hL|bkL!NzISeWbY3ckSf<8j&q2`q`iJ38eO49*(oou5_4y(I6q
zb93KEg%bYwlqSpEXcD&-J;3;kp#<28HG<R{W_pp5q*(iFD65ZxAAX-7E&0q;n~G_a
z>w0vD<7Ayr!$I!9mE{>_dk-S5Gf6<)QxbPywS1sD#B-*}b1$3&A6PW4V0U(vy%*pX
zG$8)x)qKFhtO`l)8qQ$2pN~l@@@ypNd*1}bR)B==m#)(g2(7OGxMgNgE5=qN22*mW
zz<02IFbBN*0R2zEVTTd(IGO|@F)y~#L6E;L@DDN&{S2%SU{h$mzO-As1JpGL=@Daq
zeW=yYbqI2~z4)cCX!JM<m<Yh1#4KZWO20|&&xG=Hnys}AC^za)pp#GMl4p+keX*yr
z1=eC<JxWvn<UFA`fF<sJ$2*bwll&Mc7Z9iSaVtMw0m6gZ8(7=50#PS)$?I(BoAb`<
zkgv-?npOEu<PRo(+xZ3^+r|yBzCu7PiUwNGJpGTtiTOsSvTIbn{g~n`pl2Yh;U~zA
z7b=>ITbGp8iK<&omCj&D5_i<?sunPMPnBu`M!}%sm1$L3E}0KxsBeHWSA~Gq|LBHQ
zH0|m+Accr$cHNzzQ?-ZijnnjU!3(Z)L$nQI6kjWgLY@l<-JfI0k79|103HaBV5Tb!
ztE?MPUV&VZ1V?PW#Ag5>G^jdr^ppY6ttfay<!z-K$KpFSVp*8k!>V_;mF@E|p`w^u
zVbnW1siB`adcny}3~>RXh!#aJup=nM$xdM3dYy>JYg<#{-{E>lS-dX?FpyBsJky1B
z9I-Cy2KBO;=BqVOzMejJMvt2ZPx-b)or&EC<cJ_%?}3yr01jB!OpE8$S`^=Ke$sQd
z`6``yTQC4_IIiwu?)ROAZ&%*82UmbkfFGcj0368<Y`%l^Z{*@f3UmbnU@EIj!?q@}
zX3*69`i4erfCK=+-skq9UUj7W3$M%W_3RC}HVUYAKokqU7Py)}OEu3bdXpEtL5`@_
zpCb|gp)djH@W}pfS)})^uKt=bn*aM-E2Al(1Nl99N&;LUGma$Fb3RwG*iLo=qj6wq
zhjG{tZVLtGXG1~{|2u4oWm@0GRS<|a5&<=r`6CEdmvx5Va!H6P7?26o@d%VSsx&}P
z$ZB&cDRUhWxXMu>&&pIW?)m7TjKsDN)oZ>GMw=ytKY){67mUBfhsqhJ@s3<*&-T?>
z0sQxuCnvq6SffC>7N&z}$uc!XRiW)d2nO-D-9Vzczxb81jXMjCTy}Tc=JQ3<Iu*Vw
zY{aYgcP}Oy*9+XuFF8u-SwNcl8O1Alp2}t%Of(HLCN7@`FJ=&2&8HG|n~^FL!_=sl
z$GG&~?-cwc4t_##{L9TZdFX!jFwM7YY>0{sUi)u=!UneRw-)j84nUqm&6zN3i5i*q
zc6&Z)SUniceZr<$n<M5RZPqkEZIl{u4p-?`g`Yf=ERqZy>VHYn!XFWGwlpgZpl{!Q
zR><tX+<d&-5+br~AZLF%_Z=9(5F*rKYZDH!dcw~y{vh?_W7s%#+WF2rsZ$dKl0BUg
zT$OYXe4lc>fS5!hgmZK|oN&Sj#ei1jOSh@hYI%QQ-{Fr5cSfjN^YYz9D06jnXB4@d
zt)AOZQ;_TgJu^GLeG#+JW|M-?nsNXfkB{$oEaPG#ikZqsFw2-Awn(p*7L8EEK*1oe
z?VrpTMKEnMWUT^YVEbiR2&d(9f1zi>x~#0{4HBif8ONpx0}&(LVg{l#`B?C#VGzFC
zLFF?lZ2*+*XlWi}a^U4PNS2TySql09&yt$?EZ{wmu@*+z`Jc5v$6h|olhgw0SSd;J
zZt(^DAkY=cOS#nyf$Lg_P_FZhDkH}7IpE5vHb754<!dtPerB$dCR2@=Bx5uP(sY}J
z2f&r-K4yA^>s&cVFZ#;5+(>tOcb_(>+F>9W&wekr{JtL{ae_6Nga1E|+J3On!ALnh
z7Qx)K4fZ)oPx9MAfN-sq8sZ)Doc#Mdk6J=%Uf~^~R8pOTL{IDEz(@ho+I@HW5|OL4
z6jNQ{Y<e*W%fg=^lXiYybaq0$Y|S{IK6im*6%9zy89m&WzS^rm31zatvZU9qvz`HR
zx-~&_FAllla$L51KJTFWYpeF;avQQ$beZ}^v0)N{_nHsTPvGr~g)12e85N3Z-HDIj
zIm9qIT=zBk*MK}tA2+KS25!0umfmDohDV$X35GbwxRJ+Z>>GYphhGMs{7=89Bq`!9
zJf<7Z+W<#tZ@?~i@mizQv6ypQR0`+SMXxCmT;(+npetIaV>Jemx68jBI;P<yLmR(M
zP<HhIyULhhE$&snZ$MsSR|HUhlyZZtzCl=0<o&O%dk*F9RM2-_WbO<0ohX2v*qbcT
zpmHh31e1%5sCp`>%)yvv4NReg%9jso_^yVDRg%PG9g;~j@+<P6bZw92(M4o?qR!6%
z|Ax3$u$cMDdUBO(sE~%7LlQIY{qLPZ9lC)ZZ4EtFt-ti{K#bfDq@_3!BHHbm+8AMi
z@Eo0}d>^Y*hE|{aqF)K6Ag<4eWMy$;oI)nye>Khf6mtQr9CimLoaK|R&m=YTa^9~+
za)SZ^ukp`7yk<t8-`X=B&fq~FDoy7J#TbL0u=We)v3)opHePUF3BqeGCX8@?T%G(y
z)dxry_nI-C?2jAan!cSVN-&KoML!^v)DPHcN$F$?C1Dr@=HCFSy@2&gw{4<3FZ|X^
z;(7e8m!PeHqTyP**Hg!$V^pec<T`wgnj(l>C{Dzpj-T>`bIXPrzb8{DLNK>%c?Rxl
zxW<c<nLI$$B9774?gPDx=%vN7O*yErIjnwvv1@JgjA@&K2ldCs6+F7HZq{f!(6(1y
zxwJYS?r%djRRIPPCP%O+>bvbz)`a~;)xP2n$sDeN!u{yqAr3l%KucgMYv@uHZYZcK
zB}TTr*y5>KzP*Mn7DEJ5gt=n15<p2oa6unw?$Brn5XhqN+59gU2bPDcHwVRssYF+E
zi$|-8RRg_)uy7X+nYiA?#`E&X6IS({3+I}ieH3fk*tagFufO2^E4ly$N}IkvFb)V%
zhp?KzK(zzE?Wm@FyI?25!RA}z9_q0@Wa3vboOrmVtk-=Fx^O1>@lr&Lb~d1t;d*hw
zae#?qANKSa180KiBgYhfBoVjkC%ItG^&}Lv@c`g{LB6Bn)fYqFOY9_Pd+{X!Oh6rE
zL&(bAK%n&UBZ4kiN}yj;`dX)L-a&Uj7@#DO9DLd+ngx-(Kgc&#_E)){j<Rf{Tw@Ob
z8U(1Zq3s4iwlhLy*@yWC2j`n%z9Tu+-bpPCDSiN`!?HzDAj(e)Y$fq`GlXvjgr*v*
z^y6zruF^JP!de(Hk+NFy_3)q`DuxBx>*sP}h<209p6k+wD-XBG4V|z3Kp`^`&XIxl
zND%*n%a<RqzNnR`k7Nm)EDm_vi2#uOLJ$bV;Ul_CK)3h{%h#nK(AHmt?H?5h-ES2O
zL_)94cW?d$_hgdb=^bGUjlib+`{||yA?$i^^3y+qgbGK2*;z+W=>D5!fd7X>lBtu_
z67m0xOAOrD4~&fP|6~N%*&t<F<<N-u&$v98F<bM*9YrZ%ngU@3?~Xe?4;#$^@yxf8
zpfaUt9Eo71V4<e(b<zzm+UqSMF6)Ld2_kA|?A<yD!4HIGCACZnbZ9>y?wo*(kCej<
z;&C@evfC`iJ^?t&4Juk3)_NWvuG)ZP@mz%Hjq1sadG?sVyHBz}JM1g#2N=b1b1*dl
zustpZR5|mzVLnBc<DXH8U!jW}0Qy|N$+<|x3Zy5umj`<jMQgu(K*`S%NJVr}iMaOl
zAAcuDXn<8T_?igWx1j{k>xp$XbD!VvtCpxAJlq~dOIM8WpUHW~VfK~DhFSnj?wXZo
zGI@TwG2nB1YX{N+Eu2BxGXPJ0+M%E3d+fP0xT#|3HlI8L%PX}Ox5@U23gZht3`AA~
zp3c$}2u9W*V*x0*641zxAbkLooCl~y3f+>uUyoNT*Qi(&yxmt})B9LEiNZC#3~c*o
zyl@0ir;tD?sqN-&O1ysuWSf{^Q7Zu1&_=K=_ZuA6c!8oDDp3X`65dE4I1H;5tBUAz
z8uuVSMAmYu(ZCcyC^Nc|@}4AJ7?7<j7Xf)0<V+f6+LL_YAS9=A1_52`p`z`9kljSe
z5`h`{7(_V%;5oSF?K?p+m1nOG;R=usp5x2{juKx16iyuhVGXET9wA(=`_rKKW`QUV
z5&`Lf0V>D7X+}`R_XC_=L3QsunDL{QtGCv=+w*`FQZ>|L6z=4FH}OKa3`d`OK>16r
z+&jyOFBrU8kjjguuOQ=UmC^%NABZ{Xx~;?*j_nst6(jSmO7NiNY7w`c3U^Xg+*Q@a
zTzDBwOm!mJzB@qR-8mC+BG3Cl4u32J9^s)^lpI3}7TO%G3Z!c`fS3&1cA9ZLjTU-n
z03@Z&_5S(dyOOeg+#q;nm^T=s%5PMu)S?mY0@?vmegF^v!{w(mZIHJKi&izP&Qo>+
z_c=gZ`Cuz}vKe(D$o5WC+w@GbBD8EI>5TSQiU-~I>~P@mxZGWzOz*)Gflz1#vR?m_
zT(&;@``fE6eI*vnN~lEmIAWrYI$z%b(ZTbp?E-JDDjQ{$?afYMwgpDOJOCdrvOmv2
zb<1-)Z;wU`{<er<s%y8{1j+6kD193LB~3TMZ*7qVm;^u)R|1967<#Cf_&%46v$A(l
z=Lv%r$jQpN!w9oJTx!jlK3=YNN3bC-xz5={^K9Z5-kjlTXu^hz2osfnP5(^9AX?zE
zl9V<2cJ%iNs~G_}t$q|aHx?8a?eOlEzV47oq?ga)=Xn_|{s6e~{i!ON>npGTvhh6G
zRP_$rB?qIB(y+aBZdR~U<#+>V)<B+EUtdw*fK&mLBg@^D7AqbP#ljy+uQBcJ6*75O
zoO}DT9C`?*(0in{CO5y#x>^J#f<$65au^2To#w^oB3B^a_7SuyFUnG3>O07NXX*r;
zx#Z8Mq+Y&H`2mE}cbQZxC&Sf9(P@;0gYe+Bs@QkZASE)AfubG)snu|+#qZvS6_DZO
z@4Q%=xh%8uAo*}U_A)ghyGy}f@!KIwBm(9vcP(CX;*7R9{cX4+-$5-1c>c)(t+Uet
zB-*fVM($_cT2!xSc8kA%VkS$ZtynDO%E`e8(qvA{_Uf5Sm+uWJvTF8;Cm@%i`N>p`
z#N#k3i50vDK#|b-=<5?xJnc^EWL<x2ZmTIqS@B<@)Iig5+57^j#|kH7P<bNC`kpcT
z>0H_?2a3GcZ5JQ|g;l|!gc-wnQEQU_TRev!QLNlSq_=(7R{+L6vr>avU>dS9SK0;G
z-HY-^vYm>@QA`RPPB?qhf?PO3720p)fsfIxU`zLF;3NZ5f!ZjNWihvp%2#fgA#3wz
zEn}a6I!B%4wN=%f!j9>)k1rFi<A~Mjq<@ua#iy~&cmw$nQpV?4{c=;44j9*Y;>Ic9
zZCpFWGDI6h`f9BEOhCqyQ1E%IVY%K76kCwb0d^h2aqBqb8!}9nXt4bH;%0RPv}cH=
ze4_$OgAV&}rR(I@2#G>$RK8^&JqkKY{jMYl+nT047{zT{E~Wsqs5T31`?}ho#em8^
z)mKVB$Qbo%HPj$4#h2J^-l5g~EXOW3lv5q?6#6OEjicjE;*Eb1GdvR#g+C&-7)AB6
zFq-uDb)0<C!gKbOPMkr0dr)D<7Vr@vo@_9;l+2Z%_`pOvLqq?=qy|w4tuBZXxHEtz
zOmJF6cIOncF=kVkJ~gXkweqMPz`!b%C|KGIF1=X}K)XC#NUx!zkxQ{36?sbDTA3t9
zrdehrMh2m9PR+j%isdZBGB+&4EPj1i13+~S-w%5Z3_DXf+|i*FEx8U)tx|Y&gi<<W
z1+7;D_Rhoefz{1c^7bO*@##`qSX!E)A|^bbl%eJ5ItU4mm+EtieRAs47?R3g9-FD=
z852gPnLAaP-P>j88s6Jmp*!=9HHz;}%w$PF>m~5cTQ6b+;fe(Jyg{LrBX89ODHu&F
znX0rGVs}B`9U`b+^$9K(fqf?H8?E8xHwupnKP_j{x)V%cfy(Yxf#-@GI~8@NH6BXV
z0Fezu5?|++-7uWZROje+G{dV{Wjy{~Jb7D|B8~ab#Qsg95kTZ`*Qbte6DLPfWf!TE
zevHPVzos_|<fbcO{mk0eFDQ-?`6*|WWt$0I9QC%vCL6Crwl?Up+ge|fzrWbh6xZ(@
zfQ|gEJXMS@1~3-O6^)yOyl;S7|B-qF0T#O%kEQuoG#wvRr>7k^x)vn0=^CAHLM@TP
z8T%4K8RgJJaA0-T%Z?Zgf%rOS5G-e;WLEhTFAZzlY_;nB$!DPA#%toBJN1!=yyC;R
z3r6mI5tE_Zi?b4nXdQTSR$QS=58Z{A;I*IsdfLb%0zZu9t>80BQ4kBM@YTW)umhna
zg~5K=P>rlX$xi}A33h%eta><q2p;hkS%}^@#&Lt24w2vng)lE+!Pp?zDg|+q75!f!
z26-9Z)^RWVk$5}qa@i~y#rJh-=d?HatDl(}HkUQ@5yRXN2LxSz7i8&522oOHRTlMF
zjY<Fj$DNxOYOwSlp`tbii{PKZ=i{ohX=vK5{<4oI5nwHB0I1+OfioIVs`+~#Dl7e8
zV5ucPMJ3)|SpS%c;fUOn^pYaNYW9HCd#|is*F@yWtJRv{AF?){&01J&q-<sY&z<dc
zipXI^Ab0CHrDYzpsgaLh0x5Y4&PkAmt)^U_LXfi~W!&eGKQ>)%i7=&BkgZZ85cFyo
zVI5Cyv}T#<?!?Ukq^U20(D$etP_v_5XT9U~W&OufL_|(_S{EGw{HbN0U>m-MNiEZ%
z@(EBZq@KzXhin~C|9(`cpcuV#PE{JC$LY$^mxn!jW?qSds4%sN@2HvMySl0x0m$N&
zgX7D|@dAV=i`B{qxGrGl)>03NU0|t+j7Gm!>IW|rHykdHk(P?EML{vpTpJokyrO_h
zA~3sw7T}60WDol)j{EMIAhdz^19VqTETv>M`3x+E*OqZi#3TlTI-p={SZc0*-V}CV
z%!AaaMH=qex3XaXyY(!vq+DLLRpl?Y;AvCMNqsSzjzD2a|2gE78OEqms9^fJx=j8?
zyd5+UM4*K-i<1$;*s?^=ecLO;>#eZ7{C+&zt3`h^MzO58prf?%61GyeM~`TGPyjIv
ze~M1@j88|KzdOhbA@V$og?K&y4l_aoP{+~*ox<BDmOtKglF`)8T2NvXmWob|`E%xN
z@tt&G#ij>H$99obSJDy21}%(DHz*Z~B}ugxW3m7{c}w^#Lf8b>e&5gqZiy5GYWySu
zOYQFt83mh<9-v%Emyd%*-ibP?%gnMc&r#6zgR%2B4|&pMS$Q7<Y<=dIbLB3)vkN>%
z*=BJgpEX>0|CvwW4*s&oF;BfV9iuQ0cXV}jelh6P$-#s&2Ee0yr$`Y{avk;+OOk7#
zQ$&R%2vG9kVg{42^HnX@rN{K^GiCjl@EZ^$JE}c8I=#d31%=mn8$FG-Q|cqf_52o7
zR89OSP6_f}m<|{An<wzdl^p5*A6{b^u_!<peZ(9TMwTSe=I<|4TA_LQQzuc;YX!k}
z7+%z(VI`O-iCJ%VC~SCeh*OfTfLHs)WN}h`vE29JY!Wl&eHkZlkQ0m?bXkT@WApjF
z!s2+Y@wymFppbmnXuZemYm_ynmU5M{7ZvO}4>#t=Vlmh|SWv?*WGy~Z+<((4N&4Ys
zniE%-AeiyZI?}Rsz^)o@-On}!GmOB55WKCVGfuo^VO`!BNxXn}7iqe}Eukf52O=e&
zU2tFLbapWBa(%FrqvU$NCE*#;)>yW6Gt!V*w@zTLp@V<UXp5(t347fLS$1rs3u&Ex
z)lx6<!P*k_WDn_OV<jG<zD?+>45D@zETUM=g|!ww?>qNWNeBPc=1tG^%OH6#4?u@%
z!W&(O^11g*W=ed;NBuS!sx^;ho!#>)7<H`}Ro`ZK7#@$s`rvS>t*xfS1<na!)M@xf
zb%BIe`|=h%T3yPo98_o|ZX$J+?Np^lB@cB5(<0u^A6m6WMWKwxZvJnIDZ=smnh}nf
zz6b$UMiiaZt-fQ4_6TQDt+|IWY^$~2rGiFCa|*BX_NUGik=bo9H;3e|y9y;TAA_--
zQwQrNnI)l%L}{ykC`^a4ZFJSZ;4MDSh#sbaB1%kvsU9+WxLYSHr~3%)Y9LuMN2f2x
zLV*ji@rQ!xf_=3|vMKv%cF{oq(|4N=m{z|~IJ{qLF~k~(nQo2^jJ<0%iZ3B;s}CJE
zvF~Zr`{sI)`1ScU_K!PNsj~_HK7ZEr5_v$ZAOQZrQRp3<7?}|am@ksTo(43lcfA7q
zM|OjbAF?JWZLg1Z0pK>b!!CEH&dYcq#(<+td!)Kv31HwWQyer&<X5E})+H+zv|N?>
zA%oSZt!kQ@Y`r%V2I0(_E8WbAWQk_YKx+KOmh1xp`s?I?XuUlg&sZ9T45^Q2U-qos
z^<p!g#E7_dq*s>sJ>-Om)Uw~}uJuGR{qQJp;&DPq>4ULP>&tBjhfojnCr(3kA5S3}
zzJhN7>7pY3+v5*y0PyqVEeu-q@zAn}XA~NG`w^+gp*xn-*1?^ejC?|#B4VsQwv|v?
zL%SU<C8InsXd9$4EP-YUopVwR^)v1aWdAtH%(Y;E5@7ZPq*WO^#(TjSb$wIh#2e(L
zIUhtyw$xa}r^Aae|F@g~AYlG4v-#yX%2zo5ks1N}fXLi!UXJ@8Int2|(7BTK^fdnv
z3lvIT01y4I1ckqB)UvAFzjQ>-HE<=^*^Sze?w&uNzF3g@|D_H7(i*}ifNP)=bovUE
z+`pVf3?;br|Bq;n|5qo5k;@NgH4x#g+g9!1FW-Xg3+4z$SS^u_lGS0g3owthwhO-#
zHQt74{hE0GXCnRf5dcFImF!3xD-5bib}m5`DJt(aSs06ShRENlVo0GqY7j)g{h2Qf
z9P$^HLR3oU)d<{Z8P`@$R^SZcXp}5L(Id5hFp2`~gODE})gj$lAkDuAObp{YpuD#2
z7MCb>hQ|IOkA#)MO{M$gin7Phg}pqM{hp%hgIZzPk~gceFHO`UASYo5foK0@piap~
z>3hv?F-q2z4C*%fn`lm`%~bLaAw-t&EAV71y4X6Xut`ih!Ph?v6-mQbHdq%3^T01H
zA+?%>XJ6i|D=dR20dh*`bd|9ljyWhF&U9PcVaEAuNTF!Z2lT-DzR-5lR2p@MlfMpr
zX)`+^%><@{2q2pLB%_hQS%^LReMD6K@v4wLTzCil8qh+Z<oNx9%P&aDD;K>Ot}Fg?
zAP5(5NJWZ4%vJ_spvoC)k#wBUVz{-8K$gqmv`X5D>x)ZEQU)i8kwH2YyKH49a$zup
zO$hkiS^2`uMzU<=rA+A1->Lm+f(!!MM}fVynv*WV9nh_4szJGJ?{BJDYG$WA??01}
z1Y2A3Le`)^OBiM-ojaS$Ce1~cLKcQrNe`mk`#oaB^lbPrV$-X^0P)?#B9sxRIlSvJ
zuVq|P2$LZ*od79S*kB#l5R?#00^FM<g>2)}Q$o<WAuNp!_zY>eJotax)iy+gQMdbz
zvp8k`*@iC=!QPcIBIHE)_W*?U{O>dVdp`c|E?^&OGDg@QD64>Seg4v7l)eN~E?+-w
z6rpCO63k9}40s#BqFRGCt^J<`vUM*&dnebsi7#NYzgzhp1n9l#yQ8*EVA_BMnJa#l
z-~*~T6oDEK7{FREQnveQ7~DJR(|1U)xP%kLvvWYUXL|WIR+@^{Y!Kf&1(Za{gB}3?
z`$w>SfN*WR3Y3dRkZr?qbn$Q2j0mnBNYFyY1DZ(CC6}$8G$M1!P?mrvXqwEe*@)By
z+^-(k;n{$t`OHnH<_^jXih-zFtlFJS;s*$&<9Vl%dqBJmgSz@s*kr95v*&<_2J)kN
z4B`ozz_}7@tEiCz$S^wso{qnKR7hpt0Q^RIjIGI{g-C}t@Qr}rdv<`V7zXIk5E?p6
z7zXvyWx6IH)ByJI0}$oInXm-OL1kU|vM5>wXki=mKCOYM!=RtS`DeS|lh<cmTydm*
zfOoqBj&AIA76%rM+%)(xpk?TiiJ+8x79WxMacw3?<cV11quwJ&Eokx6gvA_$)cKEe
z9hVFM=?BoXD2fd48An1v>F5}vVmRo0tl2(D_6`6wHk&`XDSeY?fL@(=;|sE&g(s-5
zmH}N|PvJEQ>7j(=zM%7my&@lh2ao0L9pLZ21^>P}^4Tpa<g%Tg?(SlW!LD>zQxANY
zcOI7qbRQc^8KCw6RqTnOJ(!D_KjQOvy9AIBptvrEU#)!vR4;dkjL}<@c3)_kucM0I
z%gr^Qlc~rc8xZiF9Q62hY#`gZ2TYxE0WQT`|AEIZS&sd50M;!&mdi2(_UX2Hf9R<N
z%GpO!q1IY>jg!1o4s%p}GkzGQdb5HMgT@s|H$hRsn>>lv!N?-mmC-J(m+lu77V3A$
z4~Nj0I6gEFn0_zrdWh`0)a~hDrscV2Oh5o!oZ$h^Q>oM47#kJS9njIB%(>sdYu*S#
z<mBV7<9G*4>ptlZq#e*%v<4O27jK(Do0iYvPrPN>2T(+FC60Tp>@R!h&OK|92O2^O
z=$3$@^Rh^Wo*D)ei3lz+%wvK^Eo1ey?<VU&mdfb^EdMcJ(cWiM2BPt9=ajx<urL#h
z(n2_+A8Z48zJFbc;)Xa>#HJ&tR=se=6kC)H3V=m_5zTk_>H&pj@sXwE-|qtGrr!i*
z?ror*ar-A|ztk>PM#?j${tY-dhkiX0BVe|@gjS^(C5;HK68^F=(x2IyHGqJUY-zi}
zU<u=u%GR3cI*XVorvAj#@5ia}@2y1b>f(Hh)G?|&wpP%v8I1Z|CJ$6^(P@v*a3wHS
z&O|Xc-axu}2sq5T+8!?>K4%x)--k`v1F^-6-Owr;2lkTrBw0Hpp}`_|&4;ukgd!%q
z9W-V)S&UIA-)MS(O67$#hvK`%>mTv7a~nY0-HMF1Nx5fNXb(T*5O6Rs=3B#X{h0U_
zR6F=R1D()sz*a}FifX1er;ASEbUU2y+ly8V+71IfM8@9~SMMS&G2{PMzO9S;a(XiK
zNFb6xXY#oXH{)`=TVn`$K9QPSJe^@Jkn?1F1gJ%NRRYrE?=d?*6DS~3LCUlkF{`pe
z;W%9jQpq(6WD-|-US8z^rKLuz=L0B{S(_P=dcToh`|9ZUG)h%g9Aw{*0Z?dZAgYGE
zXsz^E)tm*kFR~7*hY$^{qu<3IIDz!ufqKT>pB9sO#z>{*y$jwEjBFBPBVgbgy5yzP
z1UlXE45VRdUJZutKpo0m&O0gYYZ_nYgA%x*`E3LPJ&~=M{LM%?oVqKk7hhDxL(Q=$
zJ+B=o*g-?c=c1h@NYRteKoZk<LVsaSNfJsnf0xHzm)DlLtpNdz-4u8n46LWN{Pd6_
z4DJPImHFM2#B;-t=VlUzs8U(bb>)?Z!B!cd*Xpcur8AjjIg)!3U)IL<`Ux68x~X>>
z4dyvV<MyYBZ6;&G3hGb7X(HX^VWeLPvd0L#jX*(6;Yb$U2U;P}B_U_xt^nBK5$tsl
zbDhuQ$=`c?*85`T-M8HvmKyf9gKaD>1Ov<B=CejRQ0)g+TK(Kf$Sg_vI{a}6I;VWu
zAB~N(<e~iJl$LG#PS1viseI&4y?|eyvT?;Rou)Q_-*`66rxN(|H|TW0-Oc>2Lw=$2
zx^_F#&5(*@p#4)o#5w#l$q#?4ZQHcVgSi^$-gF*RLov3uTwE`1lUY*acza|APV<`j
zsM<C>KyUhMCQGs&*f-|I#g5DtIH#(h?>rtc>cySPSIm&>`SzbqFQJRP>}TXw!Wf32
zGNb!n)+{qU03TZ5&><Y*h3S6g+4<vlDsT-9u3MG(9r1yF(}$}-t(oKdavfOl?T|O!
z;T8{(U2O^7i3u(H-FnC2b!pA;GjIL-EulwpIyLGBp;w?-yhK&HNT%I;Sc9v6tEK`o
zgYXzy`f=uXF!+`JyWnJCchMN_82Z8Dr-PS1o`eH;Ean}c9SND9)O{J6G5tDjnFo_Q
z4gkLDTe^53hmz&JwyThifD}9Vk`O#9QKD=NTeP5hgDiif#kuRz+F(s#%egwHbj%x6
z`=v|^_UYI?8#YwKdGZEHwPTF6y<E?hf^58aycW}{tP@+4&IP1oe(E>14U#1NZ_i;D
zl>?oax669(mf7f%bsfXF1qD|!I%d%aYeLhpotgSTL08LHjo>IHdl7W!Q|7oAbMlRo
zJ5PuNDt#p~r4^MCnZb-5SJeY0jGZ{)DL;LrE9h2#N&QyC+H!M&p#jntcX-n6J~r7n
z7uF73Dd}%x5rKgDWye#h73hdI>qB8Y0<YrrORq~8-??-er6a@sHQD|<;B7ZcDF#w&
zmRD`*knHPElDcsPs<Srq-URw1dBPCT7_S$FYk2Mv;<|FV+9{r4>&{0IK*P03l$&`M
z0M_*zXGk749%(j|LdOoIFml+K_d|uFciSg+Hki<lfh}b@39ei<rTw1q2wS?NI%sco
z+@nupE+l&BGdMIQI&ATb2iicv$u;;BT5F#7TFI3JZoIj@Z=W|NtOqV^T)PWJ;y7z%
zk9Y~cv_A+_VQQp{37FGrFqTtCkoPIH{iK@43R4<s@}4W}GHmkU!nZkPq!H<6{|}(5
z#>7nR*A1^IdIwK@1w2WiLaX7AYfwvav4}i8LDRAvcXKxLi}w~5u({H%IM=H+O$p0!
zIOP`XlGF_ig(R5VJ!fOi%x&7BEd2SP*nYHr&INjU|6|YnnhzuMhcce9tPv6K@H@lw
z)gi3LW&h22-Y}R{I5Rt)KrPSeviHp-MqidrFSv%NX`kS~&nG5gK`Zr<>si9nMDk!T
zq?;w^G?G}8uV!7U?g8BnfI?emU)LR-j7bm%-sd?kQ*hKyd#{Lnb1BTU94IDq>QKdt
zPh4KLrWw3xDKSa(Gr5<8G<&}8ZRMdwV(+;|VlK%*7+8kI8-M0R$!21wMTK9OVFuTq
zE_S`07tJNu$1TOvs#=3b^;+pMZSHJF!-pyvQ<tJ6MUQ&s8MBx7g3sf<CXF)sBr_?N
zHPuO~`6&tg>^uG7yrJHnZf}i`zkaE&`y<g{PvMfHx7O`r65$QBpXyo!n~3wOAis4l
z+4lfn5Mi_5A(_H3z@5F}mme|Mm{UUT;8)tt&RdU{`P?u!fGKeKJ9n)-uH<|WR~kLL
zh!EBU;w7*72GSbWQK*cfy&;#Dnwa0p^UpyGVfpyg?kbt}_|GT=JmVQN!j_HP-UN?q
z8DQP?V(Z(Fxmu*3Gd^_d(NsJoG08ylO;;_wv3KFLTjXzzUZ8Pz{t!a){>7u&8@uKj
ziVV1s*X`voUo{^ncK8}9ztk7cYlK#8f`UnHIZIFV<H%X&8|M0UP_9Xk<|kGEdzl?;
z=lcw<+@wrcw=te&*6@vb&$C>MJSSQy4l9d&8=?t%9mT1UKpZT-C(u`=J3}!VJjY@t
z2A&FEth@^KjC*6;$eIQBPr<jYw2yYBC#W?bjBJ(uR@ok;vq?c(un(IYp*-qqdt_xX
zc%D4Al7Nhb<wTfy-f{XhFpln3IQT9TI7+9ej!)gg9fqdilOG=Z)pBLNN>OziytC}h
z>8yeN9E;6HYa$N5rDze?6&70Dky&$=e_?wcD^*s3P^PV*4=0SO9hVh1YgF36&{utb
zGu>`8GvuTnl<y5ueSSv)CVsIWImWNn>ep~lsd8iq>?+%U)2#fG{XF5PtB&qwV|G)X
z8K-rIxEoW4tdZ2KLIfV_1~Fs;X}CS{`9kH2vt29)BJql-!p7SboN@<2uQEq#_KZz3
zHK)ZTpJM(3>}dID74{hk-_bEcqQKm2CT}NFM+KK*xYv6uKcw80KDpsZV@UK%5X~P~
zPVs|e1ay<WGKv@CO0nPPYI;_`)i;-#e<%0qp{F^(z#%AnR>alVQ01jz>Dy*$3sX>?
zxN%Cw;M>(G<JMj$Y__(#mQ;cCy|0L%>BBQP<%}&Si*Xzrx(;fiQ%1GZArpTjqCq9{
zS2d-fI`oaZ(<_$7-w!Z}K8#0?c%kKwsK^wxuVC0wp4!GCN5!gQ>F%edICU5d6%ei|
zq(n^B=+ZQ}ORp^lmc3@=&C95ylX%hFgT$b~Usn~lzVJ~#Q~D<LSiP>?$nLzt=bixv
zg-enA?Kf3qPJh{-%$$wT=0@8e>>G)qM??jA;`o|YuXO!o&j&LLkkW*?+zez0J`k~$
zgT{ZYN`!+=(Eo@_cvLrNutVIWfUR9DM=_R{gWgjQyS9<e7rFMve8{|FB?Bo5Q8W0*
zXEdVHZ%+b@dxa!f*^BX!$);(7o7uOIgmkx+IlSM@Ack_m;Qd%kr2Hj)FZXFxKk$9V
z@~T?0`gNp6pda2BL`3GXfQxdY<$UJf@Q%e}9?dR9%#OuolR+8P33f#Ww&$J0ViUA8
z4cRY=a;t<*%+)L1d3q~j*i8Ge5h~3RWb8X?gYZZ^SeqO-33>1sv1S!K?acB!-UQr(
z!k23IC3&L33@2m;vFo@-5b0;+A|hEs^ptRh{p=lfxFRs(6S5H>#+$8#MHt;HR)~UI
zP;JdoJes|y{lzf}I|VoZi@(R8&Ncr+-D52lt1NwkWj=E2cYF@$S{4`5BIYwntd?3A
zjQ(UtGeEdN;2pHMXF_W=$~Uzi@L}DpPhr!5pN)TKFfKUz1xdE(S%Oo3%@5A_P$g_;
z#G>w-pF@oX7FY#%T0kWwO1O&eM2Syk@YEV2>lDXjZhs0-(-bOWJHk;NQQruxPzn8$
zw9J`#w(<*$7LPcq<<(3|{cF(i&}r=}CD>!HysNwr84|^G4)3`hD;bPNdU(pP3`jhO
z7oQNxa6thkmO369S-mZ$_ATf*K6dRUJ;Z*^A%@@B0Z(*{qY%WO`C2^u%d8_Ve#b8;
zA%i-F{C5cd2YYWFRn;2Bivohu(uhH~bV*2ugrIavmw-r@AR!@*G)Si)-5^rZ9nv6O
zBB-Q*z?=K%Iq$vu?;GQed*8TYoH5Q2*lVw~_S)Y!=WqT(BPG+a;Ks(rSdX!vB5z9f
z%8P?dBWH(3q-%U=+Yc5U^Pep5j+u2M7T9Pc-H4wS-(TQ7P;pF)t2@4?q~N7)lpTn~
z>A3!oG~Ft5%<MI7i^9D7#iIGqgm#T+%+wdrFVh%-+Fy8UYiNy%!vcHO?C+5e3D$n>
z&NXJ-nh#M=xiwTxevEYe1J~uJ+x_%Rn=K5n*tU1=uaPJF|6XFz(TY#I@9tp97urtJ
zN@iN#jCAgu-2aAFVC9x4GwR*dlg&eGxvLvVoZ(%ShQf(tHy;h^{?vGV-=g<M#NA44
z&!;CLSB6vN{AZn;Bs|ILOo#BVy~_RBsP9?Us$_m+C4T2IubIl~YnliSr<Nu0^}EX{
zG%Guut|_{MXkLn|o?8>wwnWeS>(j8Y%+Zbjez;foJJdvO67?RzF~=KEMj;$;4-0$S
z{X2K;4*jp<E!)qxmShd9QFHcH>!(7y(R4reCwEUT@{>2;DJXs%Bn&#TIRvsZh^(b-
zGZCeuGd5vw4%x)pFZ=O+R9MsF&~<dpr!yQ~Rbh=nY@2el2_3Ft`I01>=6CB-yf04X
ztWvr^ZO6nZkW)7r*q8Y5Ebc0E**U#9eF2U-C1tu37i-uV2PgZF))eNeyZf$uGV&N~
z5V7v}%-9$DnX2_>_WBjp+A;2>nLu`%+vP%NPl5OKbg}{8Ly$)!q$CERXb8eytO%Y&
zns^4Id=w$v{-ZT07!}cSei^DBn#TRq{%BBuLp11*UVgpioqp76qPS}!?g7%8wcv(d
zlZV3xQ`g{Qe(Q;{Ik^}0>UF&;(|*p^Y<_;UpLv>J&QK2~4vb8`a@c2HDg0k*pD*wf
z3$+G(y8UBg^`bG~R@+{vJo;3Ch>$_<zTA|F1OWlVnf+tM>B!gKNppswbU(Oz^z4`}
z&IwV|=sPjkz6D6F1q%?dQMlU~C9O^jOfv<KsFV@Z*7T+NEZw}<50k;ARiu|Io4>!+
ze$p%{b+a9^Fra&**MLhpV9*Ft3pCPQ?qdrDj@@X6JC8o9f5A&(*$KfjJnmu+T}^MD
zd{?^h2A$1mw8!rYGQs+x&j;4~-nyRe7A5S{w)6v-lTzygiB}hRD4Lo$Nv_%0s>bY`
z#cBMKIj;A#GoLgRFI314?X5SomE4avR=_2>Zo7+4dGLZ;ihPiSrB~VDdU0(d_b}0@
zutmx&7(wxTboL8%E%Sf196_kpC{P}<vb7YBLH=w-=h5j9ts_uAk0W2D$hBaweF@$V
zEayK;yR-g&?5t;rwZr08I-RLY6P4$36Dlc1&t8Mm^J(makY6_)&v+Y;lL7?1X+N1b
zO@02_FJ4si=%Yc#(?!)5*9wB-m17Li<}Yz~bcQy9TCGZs1A4pe7Cvtxf9DeChvhyM
z=@Z>~jF+E&a66OZV76nakg=@fO479ouV$SAXpNWJ3G+S5-(H)J_>Q0$<UAGL@O%2T
zpCeei^5sl3fA~O|q3yH-QeRir<c8oed4yBBZ8<Lg_fNl%#+;ZOThYd@;bJUt7+LUs
zUz1EM-N{(*pLELdrcc6R3v1{;j*N6X?Njr^s>Nf&;AiKpE8H%Q%4b_HIDQ%&z{O5e
z`*@f3mcH;8zmxmv9}ib4*qS=mE*oE+z2sPP39f%vd|_8=x_gG+ryAB_rxYU2@4(5f
zR>QQH<9IFK%n$zq$;6~GR%JlkwXE6gbh+vK`(-bOrbik#%9xhaREo6xR3y?A9}8gG
zk=ULbW?es)dzz#c@yfgCXF*T`PoT~3Oq(AUZK~=jzJ{vKOnYu=)}Ec250=)S&QvkQ
zY!aHORKupOMr?j#$PX(t(EM>5%@mojg;pc_H07uBKq3Ar_IFo8S0ZuA`p9XHXqS#(
zAAGvk@TfH2{Fm7tckt!;D|EkHN_n#;yr>86`ewgkOdYf<(m9M!uCqNf`qA>1r8f7Q
z&AlPr?qg{O(bVRuxPzM&<8@L(`XY@FjT!yI6{>_I-JhY{`h>B}<NN|Se-gW+Y$JCj
z7pi!t%l3T{mZnbW*~~G|9wM3^7xIy4?&~Zv@6Cn-TRQEcmCc9mrrZ)>x8nR}s_o3A
zElM~qYUb>iM^<!y<KjAQP<sOo`(^0)^VeqG8Be>t2_1h^m}ngds?-5msl~dg6&iwu
zj&6e_*YV)Tnei*^bHi<^j$;{Nt!LH$yt~?>aHuNTul%n_)~N;2n8Xy1##K3Ati^|n
z(nNS>PnH>d>bDnqKcm%8<k+1%AK%jX>}5IiYMsd}Q{eV$opb5^vzUzZ;N5i0X6d_M
zS(B_5N9&3L=N20~bKg9nu8jxU!#VGb<)skJYfev-HB>ZEQTgubNq67(IIyfK9mBhp
zzd~4#&RQ6(P9QrxBy~K*Y*9H(#*8?U-i&%|$ivo^A^sI?9=55~nVJ^AT1`G2bep-n
zo5I}J+?GDp0-@V}TX*`10cu*fEiV%CZq}hDsF@P6|2yi$6GE9$C;Gso9E2Dw{qvs+
zh?}BA!j}J=yBZ@*Ex+YS-2vYEA8=k)8h*me6#_E7zq2reGiLc)0%zI32a%#AhMyF4
zGwT2MgMWusRDWNC=c@)|tQ_iWxBq@H1#eC8e@^EA+?l@*<9~zB|DQiI8S;gw9id`_
zs57<CMIBbVQ$pWoz!cRJ9ePk(m?!<*iu9mHiPI?2W5T4yWAz1Mw6t!f70j03iJ^EX
zs!@9K-k}BK(x_nLEGoQ>pwwWEuSO!=tbr%Pl@)SEiI=kKuMpN#d~En_YU=+fC<4Ho
zHn+@E{lkCC41f4WN};PFtd$TmVwk4K!!WvSPkraV7vUpByi^a>>Hb|p9&5qi7PC*Y
z8VDzUUIg#tr{K?_9{1mIjSv(I7K8DBeR8Sss9<^cpL8-A^8U^R%=oY45lW>!X!l?K
zI9>qAbsorrP;%LW5LD0WkRNjn)CPl~5{4Bu3{Ywg9o&=8M<9}W2xelZ2U`ez0{Hw?
z!JM!fL=-TuJNc|#SMk&I#^o7@cBNk3*#q7LEMJgv_`buKiewnlc@Rnd1T_1L-|b++
zA|2gRxhLzfxr1PYnL6X%<{!S5?#Lfajtic50LWeE`ZiU*f-rY-TqrW66+SqkO2dZP
z2EBFsNA8j`^i>(d>ba0L4J#O9HlLr@0^?xL>T!V-UiW333yhN*CZ7S7;w^AB%8VpQ
zE5TevDTSYtg6QOUc=i-T#O~04>7;Vv>j7)O6k&IVuM2!+7)SyUT354Bt%o|eN7aSN
zJ@BD@|MySr0~N=bAg5S;nDNf94|K{V*FSKYbgs^s5QmBS-Zyy4QujlrTHfR6+X)aF
zW}^@TC6KP@{6<Ji;O?X-uTD!o|6l+%v{XU2jfwZ3KSWJ{i<@hx0yTv2<?nqM8U-CD
zp=ApnO?`^KdC&Iuc4@OmSG4*Rb5EgcVgJ9kHO1<Pm>p#rA-`nhTr=QByf?VERo)yF
zIkPb^Y4j<#FIQ^{^eRX)SfBochD8EOc3!9Jyn$mzTB-JUdU#tk)8@BMUvUm29Bw}$
zthZR66xBAKc)}~<`6T(<e~Ub9XR&xG#FP(wDR^+#i(<J|QBE31wPfbNi|ie;wVoPF
zE4_jH8u5FI9R&{FPl=u$vV>qYRB_YqvQym7eJeMi$YiR*AH^L+xvZbS4*)NjDn|A_
zRZL2zyYV2$#jz2!xq#DGVkQ)Kg5i>^#rF?5iRBQYc^HY_5uS2g@PY}-n4u;NM}2Pc
zTCwO<S;E7h^-#T7hc_{Qm{Y59RQUPb19upIeM!W`_J;A+ga-_nBtMh`KU(WI7^+Bc
z6H#%k|MS2Y$A{X{6a;etT`CuwL3l#aMOR^}`fXu0EjVKmsGop^>@;O6v+mL|2yf1S
z7dD`K38q<IU|Hg3!psrY^SZieTthd&<w_-0Xek(mSO!u{b*32o{{Bg--vQs2mRC6t
z)tI?%k-zoZ2`42#Xg9}1^?~}jk1!?Z0VlN{a4D5qeDi?ZD9#tjY@j~|M^Q2e*9)8w
zrsP`3ZVO?=uEOBW=#=Ra)UqH!FM&IdKxzuanYZ_zD~Uf0^D@V2l4l$TORbBw9zRZ0
z1@c?Z8|^{V@>jZIM7-2js=$uSLcLP~WVA1C&+EXN&P8q=6Vy+s!A3VsmOxw`A7RvX
zdj=FEFNT)Ohj0cmx_>PMC7aFXnki7``C*M-)url;q<Bu~2q<=LQYOd&aKRE1o2c9d
zabtjdj7|a44Yh!Y<AuwM+7|?w#vR3E68)-=gvSA_7*0M(7Tu{DyLz3xKTxvIhsaO=
z4pt!8(=5YqxidktjB?iak>D2X9CMvYd{7?KSsNBR!q5ycyDrNxzKtv;j8#d#O6`Tr
z?MF4*t_tR1KiLyHJ8oYctetZRRi1%ydBELRvESb%2OKurnJR9h6Y1Gus4E%f;bA5u
z2$s4iG=PNE=KD5SJ~)9;HJ{7u7jPBu#Tos`bG^=vTCLtF*;)EhN)RO9I?_%|`yI;O
zWC=pHndFS15;}!=kodH==kbD64uo7ApPVj!pCkqt_i`AwzlJpq=hfB5;9tofU`7|^
z%;Eq8a+!{{S<?LY`=}%Ly<yVT^t%C@pe%`6Bs_V5P7V_rMbE4shQn4jN&B6CVH9zB
zyPM0NdcD(T8#=3bC<E?%bQTdnBon{mx9nePb#U887^fI>?OGF~tg$-dCdNxJ%Yy~t
z{t#t*G7L}N3UZp8A6_HIMb5#i;stub$I^9>;!Ng{XxyD*|0<%7Amqd1?Ahjz!EyzG
z&%ajv;s@^1AHBM%x67<)EPwh8c3Rw6Vl64dH&h-%X`Oa@try+*Et*JH5=jqinid%(
z(?H_pV<45>Z<e-d%j#N*=37UbAdr~<i!*r*p*57Ie+;IAOszN=&rsj<>A^@LX)wlL
zf$ni`ujuKk(Oh5>v?Gig^HUr`R6E^56Pd8(=0CU|ck2EEc-wHL2PV9qxWf0J2I_Hv
z-8j(DWA1~-U>5ip;qR7uD>T<Y9lb~}OZ3p9Q3J)sVJS0GnX2~)Tt9}hY8Ci+Fn8hM
z6)y<*#ac7940H)c940*ZmV!+!SF8cC90iBx5o!X&-_Z>V155~TS*Zi7`S`{h7(S?>
zd7=_{P*}x#?Z4}bru}hUJQI&*wO>G!HSN~7otINaLFE_A%c^KUBz(8%u8oyFe8u5A
z63o5<E3j?nB}}zlT&{n)zp17*@l&2PK15W5mN}N~HE&{LVlA5jmv114d!tHd?!<e|
z>t_ZUp^wuWVF!UBc6!Htscub6G%4LM$hTIaiMmsKV$f1cM6@R8&@{LGl8CRAAZ$ef
zZ`pGAMjD__Y|r&j#{`Nk@4^F~Jgue@jISj;xU!TT6sNOIWi7ZV;Q7&PKceH8#xp&>
zZ<=W^ipsaP|9N|6+0&oWqRx%Qe9FQ);l@#e+w+z?$ty)(tZ~8*@;I|3_$WS}etrK%
z1=B*d1;evDwjc!Yv8pR|>=(48PJeKHo<kD*8nx_U!sm+KTc752cFnO!?B&=TIUd;?
z^YnEr6=@htWGvoHxZl+??X3C4nV7x`U?-E@U}2#W2^#~1xQ(q`e2Hch{1!l4y8n;U
zm%R~*t<57$-Vfbi3OO@*e?9%3<1a@!HV&X4VZFd2kd!3Vcgxi7FAGAyV)|LysZJq*
z%_i5SgIM;Fk9#?Ggd<0@f;?}1EA<-LtJA0e<&+jKPnV&J*%IJ-tOZlgDl=?~d}gr_
zs_#vALa`^+<0nfj7Stj_xmihIqmgn4TQ@`T+RBsgy@4)-J8me3zQ-ZwAQ|08eTwsh
zc5DBa8|jM@1nHHV4tNH`b9Oz{q@_5wJIh>OA?5LXS;h<^K+5RIx0l+ERuRy`Dz2ZS
zev?^FVDcH{^j)~5$PIJotK465=7N&fJk7Pu2<0hv{jjt+E!V_^q%h+R?WnuhsHjg~
zHv;(peNl>SJXz_KNj%hqjBmlyF<~Ew85JC}iJDu}{kp^BK4H$El^>MjU{)FszIr^e
zVi+jOCnpTTwf996>gs986TCKy1&t#nq6gq1A9bzvCCEAFY2EgC3j)-sjw;3rR2kr-
z3?h(ihs6aqRY*Es#P|M?U~eJ6hy~6j7&liiyLH`h)e`@v5Oswy^tK^q;_%h?4z_C?
znK*f1KXcF4N;#$OKEGG_FE9<R@18?BaQyXSsV@HKuBQ>}?b(Dm=7*1&OcEV~=50xl
zuHLPO`El_}+8INuKu-4~j;UT~`iqoICk(A{IZqDoX0cRnl*I-fq)2Wj5%uH;Wgd2g
zPzi^fcqI<Q*?CBZh-sjPSf%teL<)a%E6}C<<Xuk0$^8bHV?<rmv0RmUMXs-O!?_9^
zLFFA^4C-NhA&h(q^SG|~7hDCbJl9_#ZVjlIrHQ2ZGh5lBZ7i{FC=B=VR|QstJ2CT2
zOwQ@EX8SrTG`~D&`Bo9f`cCc2#-y*ev<f&~8e6=7!(6$nif?yM>po%|DGXNi>>*_Q
zs`5tY7Nc$wQf<AZh8rd~1=Oc^EWU+<wKE6q+FUtr(j(%&hk4TZ=zW4;{H9L!GVTcv
z0QZEO^UY6^>o}DuzRI*;kt0^6-?ax|-@Rk$B*A@`07VO}oF1LdR3v4&ETkE%(WqL>
zE^nVvRi<^SMhIw-lv5(}Bui0GeCpLud*#j&7@%+`Ee=zu|KIc4BJ7qIjj=E{#k~?T
zAYY@t6u8hM+}K$}e!I$lBIacQI%ivm39WB7sR0f>`5WJBg1uS>Rw4<5SJ*{%_X-#;
z%y6EIoFnYwVFG4$h<dqpwI$SE<Zxp_(qOw>XC3|UDN(yD_DfXHD?Ps}s>p5z@h=2n
zq^|Wp=U5{GC?cdmtqQ0}GoK>D2lVd}9?Rl{SAB+sZOHw+>y79emTl7)qnH*H##BGk
zFMoG^ZI|@^K`W{zbR@NS)&zu~rWJx>%DW<N->PW}SOdK|Eyy?wk>uDqZ)r8Ul|Nfv
z6G}7*Ym~#hhLMrFY{}qB_`xKY<+=q3vv5j@XN<`G%{f^Js3P*^)jc&(jbFgpU77sA
zjExtifHqm8-DGANTi#B-FDami%)X)%w#hI#6DUBnTII&rf{@*7*;b<2)XnqlmtGMV
zp}f%YoL{~k*cJ6wwB|C0Iik3!FECv%cBe>t*qx6$b7d)o<|rY02~hIG3Bx1KcL4}H
ziX!>)XwCqfAQtLne%2A#mNr?{qm8mjdxr6DyUVN)<M9%|bfqqck*EG<r1oTQJe&N~
zXVs33N^(9(;D;tYJ|mv*IHub3LaR4OjT%hvC{hea(zv$JNWYqnPm{|`HC)wn*UMJ2
zEAMahF6c7D#VLC}y`l7F_R)L1+|b%0?r+WmoPrztbJTDvQ#Wn~)pkq;J}WEVA-|(~
zeLwvUC()<h81Js)cl(MdCxp0-F4e`*-N0q8aKzav-{Fv8Su~{{<fzCuD)hF0U@iBp
z2)AS}hk0$8bFYJ#eIv@=DmxS#XOT59^+BRHwWrqAT0$$3S(pd}%kv+P=#o@pZtzlb
z*HecLvdHxWmY_ey7;4m6`5~dI^%uOVpne>jV6B0c)!Tn7)%6l8L$`s-H$k7gQg)st
z(}N7DCF{Dh)@yrx62B8|f-Ljp=_Ef{|1IofeV1cONB#n8EoPpppNJk>O=0>Ezxv!^
z0E(x%rrlv`U{>c4U6XtX{U4v;)jA;9IoPRKUW|tFbQ3#Uo3*J5_xr1y-ZxH6dqiJg
zoQOfUhL23)i%F>GdEUJm{b?!cdG`qnR{NeUqqA<JlJfCY3KAwicoRUA(Pe*O)A1O4
zpCQT{=V$ZveTr-=+1tLx?8Ix>=(qG|ar|pu(fB%)JV5&o=8IOKB=f^kQK+?@=fFee
zIih3vK$}qaU$hN%6t{1d$KXI?LJ_9D|1d(VZP@B37eg-$Y@!?0k8fXR<S*@OBwzba
zX~9T`!bz<6qj41ExfGTxns(X7K^MQG5pJu0bQX_s5p9Iqh{+v|Qhi~)#xm(?mX5}P
zOd}-}uG+gI1}vFL&@%li${<4FoLBcB(&j^5Vv1vJw-MC`gQ{p0wR3;5->%f}f=(9R
z88Zf+5I)8M6FuKVOg4cqDd}ZNvC^vF8xq(Z9Ns7=lX%doUvd@(w&TlomSnXXsR#Xc
z`IzF+`%%3s{3j#;fol5tAxP}vW%2I0dHQUQ=r^0tW3hetDHfEuqIjeFF~DCz;O<&~
ztk?&a9j<{wPdLw?^uy3ZxVb+5cvlEvO7;H^6B_+%kMx(|EQH{@+xGGr{tK$I0VL9s
zD)cAt1rV^_2*6${-a6p?`xr(eygD5=lo0l4|D0h3g5>`#Pw&dV&hY=)6#c!d41`L9
z=>yZG&)*;XKU|vr=bnZd(+7O(mH-CC4FcIDNz<KbcnBaa2b#KDoW@eEZ<Vi5M{)Yf
z-_<PX=dT2<gbXUPdSR@xh#A1nvM9(O>7zLm%sxMOs@H2-5qkeKzv#DzRVvr_>*?A#
zC_FUyn6URL3as#a(GePCi9`rf`3oXulmVzk6klrff1}k=aAVPp3Yq@{p4$+@P_3rI
zC;z?(5@LY=<6ltt_aI<bfVlZaCxtHmy$BsZtzf*o!vA1#Mi%%41J>Tp{P$2%BP2`z
zlTJo0xU6JZjTc%hpe6(Eq6v#jk#ctDy7wh8e;NU=+7c=Q142Zw1K5~neFNHfG36zA
zXOzQ^xz)#9E%8qp3nI4Yn)f9lJP9nGX2ACW2v$SmnvtO`#pE(COaLSk0Q-31l`w8x
zP-9wJM-=iMaxh01J^*v`mb>?vyb);A_C&E{kOvTh^C_ekn!N5!8NM2>a9r<~GtelQ
zCg2#Hj^A6O5cY5aY|T%#1~y*mo?l!RFAB&z{WD#cf4wVnC*!xvxF@n<M_c*r*%RS|
z(Z^b-%;3k~r1$DOgF!#Eh<&#}g#4xA7o0r;kOQEi$tE)a!EFaWw%-UFa4TRix<GRe
zlyq($M``H`Xpftr*@bpeU5#RSY}W_*%1q5OAs|LIsMR~HAtVGgQ^O6%px<;908@#O
zonnt0te3=hH4vWvGlJR0io#*>`BsF!a~Nu|?Xl_~fFWxV-g33aqZQ=70D(MuzXU*&
z7eGS#?rU*s#JG0^KaXrpmM#Edz_J0xEf(P6@AhNypQNB^<4y|4e$I&@fYyjA=OwDl
zuJqKKlVD9eaeeYW2WHO*J+b;Bc*al;Xd$4rmM3o<3BOXIGRJ4S?rMD$21dD!%CCEY
zH%SkNg$zKk+d%+E0dI%O4gqBROm21G>Wvx~i`n#C!YO#v#HqJZTf{Fe`?zjT4CKMG
zJ>>F+M2tNkJTw=`Pam5;_p=uV_V1rL6b2#n#4G_ZzXas+d5MP$061$uI|2-k^pN9H
z`{>*b7V$TDvVn^ZbM{@+gwLkRjI?oHfKR6ep0J_@T4=hqUnu7Ts^%X0w;a}5D_O=E
zN~kluZ&j<<TR&r)6a`{)pH&nDU8SW6{2XOo&Q9+a>e0=!CeYWk5CPI>0+gO9bsqOU
z)3bAgw>3%kkA)O0bFJnIX(b;fM8Lp1gpm;-W{R&4DePL1ztNlYJcq{^AOYQ<uKoo<
zt#Un7=CG$)gnUFxItKo}hY2%Z5xEA7cQqSI0XnVZKVFSDT7a}(4)Wi;n{t+o#I#HY
z#5Z)L-ViIaJezCzC8+*L7NQhr`PeK!r~Fhc?0?hsKIjeQJaEY&r-ApmIzsyaj*&~@
z1yf}nTIIjT8_;v8AKz~^`(qy2f`ye=jR?-VCHZR>;EA!ubiI7Vi+B1T8$x>!e9WR=
z<C9Q1cE0BQjiD~a>(@9CX-}XU==UJJO5&DSRvo71^$R|uvV5akM5W^p$F*z{s8WP_
zLb9hyJj5q5^u>HuFLU1{$yyQQF_h}Mhuf32v|OYHcFNaP2SWdu-({eqhS1hwlX0y=
zfQBTBon#vm*6!jDT9+O$2!S=i#@$)235g!fQ<pI5(^d0H<8?D_AAVrv%=h&OzLMrZ
zPRTUFn*n(opXQn1(}aT<qYBlYXHR8z#WhR@F2F3&4S*ng9|^4W+ewd}JK9!dpq0O_
zI@P`|KORMRo6}D&F5m9wSEHq=LUY8jsYU1OhsCC~yeNYJKFUgdc!dd(G?p!fzq%(l
z;|E4ykrbjYp%)X$?S`JZM|VL8>3~8!7mZx|@se95BCmzyBd7n1-{%{lp1buMc5k=C
ze=mo!Kh}U?r9@f5>5?ZOU`4o@8RAM@JZ4d|?g4CS82oR6=Ea06TAejk4$Qmqh&ppv
zzGOd}0fbPi?KtFd>(XPATe#1w*}`HH0H5o7LeQNL7OTd@d{l94t)l8(dIS$u)|m7f
zJ+%013jYaA(U8kn^y>rrCu0>X_9sNwK`v6XO(=8LjpMdUanE*69Z&;6L_Xw%V9tSm
z7=5)NAL64tP1&g+bp^z#phz4sb%@c?T5*FlavmSm(i&Ivw;g7MzZ0xa2nQVHaJKX>
zGUGwr+pWzJ<4K*R%?b`lG~Z>CF`s0Y+-mJ->oLP!XVMF^{_IRM6%W6A<qmOXP=s*O
zKGteQ(<RaHvzhv3ymY=J?Q<37QC*$6IAt9Gv|6UqwjXx%0s)DcAo0**UiQRWR_;M~
zld!!)^TWafQBhlayNZF^652y`bl0s>Gf)~G$326540eS6vHXV4u^elA_Je6yGQ6a}
zMbq7i8JcPjr4ACWx0yn#l=aV~_+*+?@Zxw^$$t4%&YUFC=K5!#k8Z6kel1Yx5Y-kq
zo){<l<TECb8LlucoUr<(slWlGIKnLdxbMRikP-Z7#^Pp)H$uc6gjW-ARc5Tj=oDQo
zV);4Jwv1bv1rn|UIHEmox?LLvi3_mM)dai-%w&nhEbSbw@Zr(dl||kRrc%cz{T@%Q
zH74iG2EZTA9Eja7OFzE2-e@&=EGn{a8XwY|gT)lW7Av1?OvHQRgk+hBxoa(hi=Jlu
zc0v@*gRPuw)iBOat9;oU4X<efUfVLxD`+w`$iL}5J{A<&s5PIh{qDL`eWa6KeY}5m
z?S=P*g_p?pA(Ne^g=78gN|C<CarsiE?@DOGri=HQPr^$`ww4ndcsKHPr<OK<$H?h{
zfG4l-;b7BKebd{|lKCVTR!yA;iIdu+QWVQ4=Lz1M75Jl9qAFCPQ~HQKMXfk1q8>2k
z72jl$D6|BT)N?+8IMs3N3_S+lhwLF%;6OL1D7W=tLh_1}Ye~UarNvNATq>I9O5Qnr
zv0iUme-^9Rijn$zwSwp^9qX!`kk;u{H6?ZKTWd=BN;iT(^1n^#4R)^3Tz2~E|7I$o
zvZo+LD~JFUPpn14_gk91cDZo?nd7K6g}Exot(<qCIJ)t^Qr}nbZ2sO!d<wpj#aP10
zaanzW$-MVLh6m=6mR_z;s2tevX!+{wmLwc;#bRVEE)Qa~9=N`GE3v=W(vC?MYfAF$
z>K(Lt1t<d^x<+Uje;=eDSX7!z-XWmwui@CC@j*g?3s0=uln8Wm5kFqf!IX%R;K6F2
zn*^Pk{J>*5b#5PrbE?n6@(muj40j5JaK!I9Y;OSix{2_bZ}N&+SBIGxSNT(w2uBnq
z^(qm1wj}Ad%p3~6sgd%@(rT%S2t)ZFMubK`)(>8T&nQ2NcB5LA#SpoL03mDIj#JXF
zwrQ)~O}QdG(;ldW@^2;bu1Li=q|~SP^6{WO2=*h#+Q9A&mZ#w@<yF<%Fdx1AT=Hag
zdgLq=JL~YvQw`l=53fA0dDiqQnt@*$e(x1^2smQf1JGm0Nip%okbDhME8IGzH<Xz*
zOKeC)Mg)S}ncfegrylC05G$uD8hso#=GO7iI|-r3-R^qGIg$QwzNR=uX3WN<^fp2-
zNIwX(JPpkdzsPCzs@9;5!UtYgovy{lk57(%ieY%vv>L6j>@u0Ey^i|@;=OXTYdLJ$
zDHaGaBNKT7;A-oLyO+sG1F}x;(o=3ED;s+z&t&<Ftu;(*xh~LZpqEQ&zOE`P^#d}G
zpKTb|n?@KO4tjXxwb#2qDXF6Pd4R=m<9V55Y3ZzsJX!Iq`K!|J5A7yK`3F-~>9GfF
z$pjp7+;`My+N6-Lb?JO+Ph=8DLPnLRiJEWkNQ$w}3!}y9p3W(vvoDCw`GBhTE>=R-
z38EbkHcCk)aUa?{Gpg77{S_5C(}Y}YzvYcKyQEPh*?%=TW#*h24I<JQ0Bz*kSk3D%
zMxAS6oBygkEXbPQMZc-!z<pf@>o(}PlF(H#rk*D<JM<!M_5sSYqF&Wt(-vay0#)||
zW5!d3Qtr#V#z*NQ!NlK~lqE7OZX~%KxbpCEC<csh6DbDoN{0x^;puV@SX<D21V&A6
zfkv<5Q}{ArQE7~MK2v->ad4+2L;C0d8&x<(mM{$AzRX=+4~dlUJ)vpiK{>jWhmod;
z_zvIn{YKA*aKm(0`e<9LtRMZ0^a-hPJd1YLud05z<ZAs<?3OM?oqU6W;5>~St1<0@
z3GKm!f&%A_{+P?ECOr@l5HkEI%qw}lMHb#LKQn#zC}6!Hu~tXo9vdguntgzNR`|~3
z;3V__+BG&w6WC7V3Z~1SpFGj|tabNrt9*7zM#+EabW8{ZeYMnQ(B1U#fyG&B(Annm
zO16s4&!KxagwtgWGVZt*52Q0=TMP!Vb>$Ss6zYX`C$SwwWvDr)>@{B;mgZILv5rr^
z3OcZnem1W5I2CP#Fmpgr=Cst$^~gcrlB<;ac5%#EzUaA}iRKRv+!~pCc8g;BL_H(f
z4}?CnH$7HNRpVf_q^{hx8ReyMF0Qsa2xAaV6?<RPK5=~GnT={wGhg<PT<u&RkNBL4
z<1tBMr=dpnXNCCG2W(r}na76FHDdm9+Iqw@&o{03)dq2V4Ny6-i^H-uaFe}})|(rm
zmKw*?9fb!u6mPULuqoMiWFF3&Jf~C%PD_Yo)}bS`4;^M0oEeczQ2TIqB=}>g{CVeY
z8nb;5|BRZE8~Z&f_vZ{RSn)Dlb+I`Q*u=2VO(~=16{OcS-|lve%v5-=RXKi@j-(i+
zKUaqggY40p!l_-Kv5%WC_t-D9w%I*yXLCQ&_-V=fPD{~H&?Lu^L7q~j17WiG)ijJg
zT(CYO#(7KW=@X?<XU1aV1gm3EN^Z-?kgMg~DpGrJo;;71+hy6s<Q~bVej^sK0HRYG
z7PnlMR8wf#^=ej$1aT1$99e=TJY;>PsL_|*kRN+9ipFZ6SFL-HjisC1nDt(*VVZxp
zQ8_u!X>u2#iUgxSoX1<QUsgT;=AOrMW))LGm&@M}DTfi&unYFzHf+STrN@>lHxDi*
zR-cK#teWe5=x^IWux*lPxm<%PS!?|Hff$M}J=^PzV5H`Vj}(@O8?*Qk?MyW2OP4Ix
z8$$GD73I~(=r@pIyMQnFHzG>=^8GqIVmNoi1<>MC+ROP#;Q#&S6qDGp@rFGW^Pj)Q
zlcg40jz$fj{`=1|RiE~b!4(O$KQE??lJaS%!@!jO`%lAsq~Orhu7G#{ycM1pWiV2(
z0s&FT-+#6!A!`g)?*qM%xpKz5s!Qwuo$|*en$1hkuFJ?bmm6=8EcEs^4X^Qi&dSPa
zInAO$#p^B<OJI9aF(VRAv#8<2&1T#2E)sFu;nXuR>^kXqJOpLnst|8bi$eSV{&iHH
zzs#eqzo!PTZtp0*aR^Hd;?F60kO>loH@E)1EpwPrb#=tqB7bj#KfTy8!A;>ie?DU$
zA6;YuQpBzP^H#pYsIqzz+4&}a?~2`BvE^lru+zV9g_jIhZF%4n_V+|>(PZ`JLX;`~
zegYDNl=c|mX%T<Vq@@_{nJ(wozi%aG373&MGRFD$OzMa!?d|W%c>R4VbVaz~|L3Cr
z&nKOU@xROCEtWZZ=N$zG!pqkltfRp-w;xO)0Ls>Zeg@(k^$y1#f$W-^6V<Q3rUeXd
zz-^#Nzfi4U6#S-bTpygF=a7OEek&OC?}6dOvoDP&z?OLTI8&lB{UuP1zCml5w~$cm
zPWSk+o?eRPmBFA^@GeLIfC=1CU6u{OiGts#%^y%FVS7S4ce)=y0@1hcAco>|^d0c!
z^K0FCJ+E3vw!ncvTXzJS)aO%WRO)KYr{ksckJ3E*Ibp9AbJvGkzB;q?rPAUN#EpPI
zuG!TIXcMs5>JVupu;iZtflW;dz$&mRP6g+JdOn9vRY7DgP0-5sFSfqAPJZ9=h|-=Z
z^>%dy^$+lu5H!dIU+ov*#<$fBs8H434Z6F)%Zl4NoP^CBY+))1ki%sZeH5gXg^cMs
zW^11t#8hwY4W|e=DhVH|RYHn(wT^T)5|Y4OIVo{9Aw*_Zz2gSRNsr+o3?Qx-_96wV
zu$_8(4oFf38=TEY0ax>`w=8`Ma(w0=#^8qnW*l`8Ok4txB8*%+0!y*aV7okryoZ7|
z0C;p6(BsD1H5Q#;VsZ%F%a1sNs0;?Bly_9bL`3%id{mmqB1knuU}^`#GNsNa<`5ks
z!0T{THM#{nPfPcA<n<LNvQ+KCgJ0>iDz@t0EPi*qPc-rQGQwkcrGn&8v)s5t^x|hE
zxPjdBf>KXdSA5tu2Nqar=Q?@Jqap8$%DlD035eaa8<8gMP`?BK5K=6JRDVGg!(@B+
zxufnP7-k4A<Z*tDYz4(>_GG#HGuz12k!hBU>pAO7c_C#Flei#f_MWcdU}_=#+6hfd
zZq}T@gy7C)Li6vAunllo4ZD?S&Id5z1MiOyPk?MuQg8-0Eckur>}?W$NV){;`v!!M
z1!R2<EE#cC-6quf=uu6k3=f%ujU_e%H*6>Ua}{+7gUMezDV&FM<YHhC%Mtzve!>k;
zAHQvI{l5F#Bn`ksur*TC0!ss6%-pkmkQ#=tc~i4A=VR=12ODQd%pI(=YOL0#Tp0?y
zn)@QH`h^kI<?1t$Hs$bvJ_hddL}!Io=<f3c8#1QBbod}{I+-2ilzK2y9R*G={^Z9K
zz>W$~#*WQZACQHqT0kCG-s9SO!GQGuf}7kQq&2U7E-G*Vv*~vAmEtuNGH<@<2x~Av
zUyrc^hf(@sF)EZ?q94aGWV2gf3t%adtJ6zYlQMb2(sG-0Ka!gC!dj`;k{0l+$Xkyt
zFU}!4tGJW}?E3w=D_aXFG?Qv!6ZFABc_e%ot3-U4V4mg#j6-Je2so!GTii`<Mfizw
zS{BQVMeVYNJ^Kk{oAG5O<`LHfz+(BwW%Z2X+`^GiG%zf;3V<}mL_09#1?;J$D2wqC
zjC(f*>-@LBTYOidR)Gli{3lPSF6O_#%&&!;=>b^R;W8x9aOtL=Dw*jc>H|J>Pl!<g
zLL#>{ZN#m6yU_}4<y6eZWl*woL|<8wyaC4nZ-)GOL{wRdF@hjCz>`3z#NTQ&-a)gm
zYnd;?4t0gE9U7p9+2gnS<rg4$hGXMN8&2?-Vge|mzS_BA=6Rm%*`EP%>`bRgF5r}P
zagOq%_vK4fMOFQ}E?T89xF2{WKI>uWV4UE=Q`B|?Mv@d4KZ5L5GF`EtFkrdQNEWE}
z!R1+H{)Fs!C}_&X-&<O**OOm6hi6S7+;*X9zHve=(P6DuV672RytEMyFROUazp*XR
zxPkCo1h|#d_WprwK2nV1&V9BdDQAGK-p)5&oLYYzCYaHgF<{?qzI<s>O!|>m_pnXQ
z=^&W`Q32^8KWUt(Uvl1_U<w1#v1~&9r|_UJ#l(wYq18d1&W1|VS$Eq3Lq`~9B|}lW
z28g`ztivv|13~IkaU-Ja@*^G2`jQ-OJu&_}^77a9iulFr;{G11VWG~~6pk;F5}o20
z)a~RL#s!1>z%-2KgLFbjm263x9MLh6W#!8f9ha~9UjrOwQMoQ`ruguddpME5r#}>)
z`10H8>vUy)T5g<)2zHkN)L!1gUY3fHx4DZM_ho11U|~E=v8ZI?s|QX-IFpkK;XJxm
zCz(Y%U)Jf&d~Ex(km62jAvDBM1#V~ck|9T?KV)r07%V>BOo-lZsZMD%qHMylE9iDr
zDaEm3D}LJ6Umk9)q`K?u9hJsIq*+ocuLnfWrP#@0T~`3m2X*(2d!p%J)gRl=ju4bz
zuvxCzn{e<A6TKB58TuoOZE-E8;JGD&8;C#rBuWXF$eD0?C=JTMBO7iT?dzYoUnCbC
zmKSegj)0X#4OqxbwM|WWTvYCByys&Qvj37yI#`?ZzNDcLK4yDeck6negrUGSNt#Ak
zEVmQWCdiM3_|MTFyxS$v<-T#D{ACCWq@TUZNFOxImIM(H_3}@$Pnw`=mW8;rxZJaG
z)t@PQQ=}d~S^MK%EdF>O)VW=F*E(X`^Z3i1-JVq<-e6#by5=3?4cK(5y0Hx+l|MtT
zkWYDmgqpZQfK>5;?ckB-HanQV$n7hGl@c$^M(7ojEh~0_$pfK)xitri<?so3!I#hD
z&hdA>g%_}$=Lnq4IDwZwg?&4I1Gy=QwKDE!;B`5k$Td@@U~umGWxKpX(HFTTNcBCZ
z*c?yDb(e=db_K$5`CLq4#sj7tsx}y<__8!`PoFB)zC&a}Z-3BE*01VG`jXIeQ`b>X
z%=^wsxuTdtF|EjwHicAi`1>O15q*G3*Bws~-acrhTkO3GDUXUvu00jcM}&OSA!vXE
z8Fe4sZV@9sH67YdC)lySB!*VPFwyOGsH1)|RJNPa9j#dP7-G=qA+r&8Tx6H6Rg?4O
zaI=-a?_}Z<p3=5ZEP<iHf%4ZTDPi1G2z}t`JNXV<E%-<1*E{4+Z-Z=dxZvjgaMr<2
z*|-uidpeHL11Yt7sul+Bdl4qIGRhf^Oa#8`Mo~c!Z2iQqSD;X%xX^jk`1=j!@WA!<
z#vg51z&sld@PV-zujQ~9p~Ed+E45^=0+V{5E_fcYX&N^%fAoaymWMuGrDekx)bbDc
zr^gUOJj{#>qR3%?nhcn<M%77e8sanm=^{?ppo{ySj@0?5cY;g?*j}B=(24(PNqjv?
zDeZfXA0e&$>2F}b`ac{T>Yg(OpPzqP6wn6{B%t46k&yn=NiD!Cws%h0^6O7S^EwJ1
zdw*P#OzMC7<EZa&u#*c6!{2|pBPm&EU?i@EKK}F9-L7yj&r4K}mwy@@YFcQ#GG9r*
z`}5bUAK+k3-Y7Rj|6B+@EW5$Ot<-9N{(A2T9PEF%Y=8GMMCi5Qd88lq7^A`2dgYtz
z57M%4PbkZXu(%tXvNE!<9pzHp>=6@_Q@Hv9tr?3VdU4>y2K70)XXNd*#R0|0?jXAF
z3e5pcRT4D%s5VMiS$jjXq0Iwld(&kno4<%}J(y;ftKFgeuF)TTs>fWhDu#FUZ?_i9
zBDSnjfzSJI!%F~bmjW)T(Lb#=piDh*Gw5z9U-{b#vOx1){=ewU8N*u;b>z;hW^(f9
z9<2<H72)ZTtG<X4G>jVLDCCrtm7%&En>4Z(LH`Xw91B3cgCui@Ry5MpPj34bVL;U0
zfl4+0;y^W1A^=!t2nyXDK_^Dlz|uTqFU)v$gO|W=18TW7WSe{iam^bb-(bG`{(C#H
zoo=Sf6NZ)v`W{bDIT_DO)e2Nj1#Jf0ePQ4W@t887zJaUj;B~&ztNdWM1hib6xjIhU
z&p=st0S0BsJWmNU2Xz<F_#pW1y}`NshE7U6B)x7ot_L6<$flU9GmDUbaAhZQFpJ3!
z=31GOK{DJfSxO8)+6f;WY)=k-9u6J{i@NR5UT8zq_P~GzCfO<$rilK<NwI98vIQv4
zHWehPYp_>Y9DW@Kdelz6`b}9tP@<#h32q^<L;knC)<@74<YCr<D5QW-Iqf<MBK8Ev
z#`n#7V?A%!=!Yv?Xs2xBBWUel<mC<gOF2W>XyC@o1F`-mya}Xi;2@eOXfpQY=@(m*
z*`v_V(7eyL4S>U*<MU$Z)2^&ume=rrXd(>S5s8!|<QOAo%w@2(Go*VuK*9;|4Ss`p
zeMo4i*R!<FBD}|WS<|eT2sQQ{UaMF_Rp`WG^Dj;(n_q%l1uE1d@b8C@esa1OYXJ>W
zNmLh9vY4_q&C)-MD$teF&Qkt`7%S+1)G0?KVvoE|WPC+1Ogjgasb-AM7w=22iujDr
zS9Ugl565FM!gEV=Ok(AfjfLg&wH6{U4|@-r+*=&vRQn#8%Bm#B7P17XnC{rk53%cP
zjC0`3D@AXKf+Ljyvu&6@gzU&0V2Z*_e2PH-fsP??;nzc(aI2KcX(QIad}`7ZE7f_d
znC&$B6WxjwbtD}jG${9;r`;T>7C6^DCd47XKUL)pQ|Y8W7xE8=$2su%Bd%bxzke-E
zfd^O1M;YNl2ObtKHbEdma)9T=M8WOOQ{;JX>*eS0SrqzOj)9H?bU{BrwZDCJOKqHS
z=%HBUYo~?y%F<oh)6&aR_I(X9?BGzEl`|{?(gnF!*2A=H3@LVgP5O}#j0XhzQ~6>c
ztb<$fW6s`ZvkP25>FQpIKSD>noAKT^?-R70d0*>6E@2+S98;I0(p#s;&=6K;E@w?%
z#Wn;?-=V6`QEjsK@#YT*5=I%db-HNTAT2A$)>W^}(3dcmzG<_tX8j&Eg@_*3{WA18
znon4$4l;{i0?1=Ba+sm^su1%0rgM5`3cfy0D{t|>ILMxlBl_C*v57HRE{N=2cSJ7W
zf=qO=;p6=uhu6b?!Z>9T;xM5N)V_S_L>m0^(U#wPTfgzID4@y;3LhaPL_v4M8%Cq7
ztZiz@vFe;YfQL7~P*MQp4ZvH$b5v)|K5@y31@0hxf(Nq8w(+No1#N?*N>49D(x_$K
zqDKh6pyL$Kb@xR=5unVlCNw8{s#G8dqb>msyFedU_|-(Oz)lINGq2OsonY>;rKj+X
zgNNy%s(x$}SODrTRK>O<?6}H_5nY2ashw@pc&VS7i9MeH*~7x*Sen0ERZ)6?4v4)F
z;Wif1qTc(Ek@GN-L%dcT#1}7{-nc@E{>+yqg1-&BrQST2pWK4*?gstxV}d8hBpIQ*
z<#(y}5G)+3^1>YA1(Wm(^)_1@86;mubpA(5;|VcpKN06#V5JiA@4l)RG*f49QW<If
zt>H`39hlM?Bd0p@jub!nz^9%#n1>iG{<%|&0L|h~U!oQD)HnDP$6Xm^nOL6V(1wQQ
zT|`VBz8GbaPo=4BGTNIzqp%7-MoG$Olob2mRU_45Xgx=3-vL4`Wbumdfw5B6=3b5C
zSGsasI=CKb--%+~w`HWPI<Y(u{&ifcSFiFm0+L<`5^|)7WYl2a%(}Kqs^g@Txt*5@
zLL=6#8&KPs4%Th@A_;QFRN)4|QIwI3rwgwMG5&(|OH<H91!ucK9!UR_;U^7FMpTnh
z6!KXBeG-hD<XcR<9j>lwuKZQYoye+N1IefhjpN_h^HWICDWkYiAUK4_1ZcNna|Djb
z^IQ(AvX$cn@!VG|DBYnnAsD~zLw;^Wk~+ra<iYi(f-seH!m3_J=@=GW7;g6f_lKtj
zC<sDKN@_v!b8y6mVvqK?HGL1UJD`nr1?`mw$pI|Z2rl;d`+LU{+davkl~Ag=XNE4N
z?sFGiS=b@u%===)8(KN;7e_Y2aR(*oFPE>5LFC8A(_H(vZlQ<=2tcJaun&C!aK_iH
zvUuy!grqwBYRi$fqKrGRouE>)!b*isA#y$gd)5T7t`SMCQ%05pR|kBOU=M<LpxygL
zv^6ND@Y&r%1uP_6;1?oRe6HDvbtMOld1ENu0Dz*o8%)_^E%=O3inx6FL2Za!X<5r2
zB&N_`tW_?N5FXpUsT}&U_{WZkKk}^1N1?N|oyg&fA)m;=Q{D>u6&X5C*aSc4%<EKJ
zop5<l&>YZ_H{UGx);oHzIa%JFafcuQd)RZf<03Ne9RJjXgT=c0rTZC8)wd{mjINiG
zXEYDnbV8`ly3D-&a}5VIh}nqE6yja}g`=S$gZ1kv-ro|Xf)2JbR_&QT#U2teqT2Ih
z$QA#KM|a1=_q4|JDe2!oN0gF3Q)93GJsPDhEVk3pxljK7`41@AI$Tw5O8o`WZJ<sy
z{r~XhVn#JL^X65s5`&(`H3$;#%lZt-h}^b`aL{@{c2c$~eAGuq8tL;b;x7{4iWnL2
zZ>tFtlal7o5P1kJ$hHZidh)d$Ue9xlaNrmW{z#c_Nf1mhSSPj2flvFls{37v6pWRA
z<?l7T4ueZJ#U=LrQ{civg_ZvI>hb>9`kk1ODK*`&3f4CuLx9cxOtbeT6ds<CT^R2n
z%DkS@0!4=qh@Wyr&(+>NUIHGHH9WGg<o3ocDJbmT-UUj|OMpc}0g(lS0%|K*2Uej~
zL+qFJ+!8W=Vu*WzEmP8Xo5QeW;o|6CcpmT%>$&JV-vqY+oOX7wmECM*FH~zixeNA6
zR$#WOC-9tx0a0<`)5u`o5j+4~w7m5$=gvrW7vc+=<}@l-zSby1c@6c~MBJ5YKP@B^
z@4N58Ms!k1{be<`h49h+%>HpxRxpWp2K6U|KZp4@rEc$l6<=ZkM?LrE^71l3Z$Ltv
zHD!fw>3taL#ug9}5g{CGL3ptY8qhLez|4_x(1<TGK>Lw$N@+7&qevR;U?q!9PlLR6
z7s^ns4_}~sJ>7<He^%UGO;Gxbrv#k=a%@|-boKmR_WE=WrY|A^81ASVBj91_P;y+H
zG=pPhgHDy*QXBUmzvLSHUZ5GTh+hqL={<~jN>hyPxj@L|OQ3-ETVtH@`!4{cXrXkP
zHDu*aEe7S^9W!I7I46L~&eJ9RJmF6<7DRCu-;_3^k_EB-x(M%e(}14t92CX;1Dj6C
zgN@|CRb(Dniue$+s<wiLf)7)p;7wlzf*Lj)&`7H%TmgTlG*fPAcbbeo)KkaYB`dyN
zs4$7|?Y6bGLDe?-Z&e2UF<8BO|Lh<Qse#yIQ0wUFO)-!B)*~k8w-Y0?n$!3~jrSCp
zenaN;$2s>efFmGM@3ATws5K1=RC2v-7u7E?`qM=(0Ysgs6LS8VqskJD1UKp>l*r)H
zBLY^<h$J#7hl8CN%k>NaSD}}1ykOzfVR$=&mJP)ptwu(Bw$J(vKm!yn=bY|n1W6t{
z!k#Ff{{bA63r}RoZ&@7kz{2v>j=B;EWRjOqfE9kofGp#F?jbb-!lYpg9xRkoHd~!%
zi0W;<TCUmg&{@xW-0785p;fHz-3+IFaNpH%D~KjI=jCm6G*RZm%`zu?@?xjD!G7qe
zC*YxA0`2>S*%}eMRA>6V``3jZEL@{_H$B1Go`<&-D7(qicoN4>Pm@}pczY;?bBpgG
zu?{>q{q*Ye_HwzVjwfrP=Q$ydNs%`OQ{NXi915$O$5<KkDiFai!&m7&qpR`&(0n0O
zc^c-xr?AJFmOMvEwsZrOd#m;*0NJX>s%qV0uv1^*`QS35SPXfVhA>=+T}fa!_)@a?
znjYI1il|Yi_uJwHr=|ZCcxPcM<C*szP?EpoJUjsCd>^6(ov9=h4iQ-Wn_B^E$XFWt
zArB`UfTGs~a6^Nvsd0Ss%UgHNMotBF^frL8ry*f-VwZJ;JXnD!az@l-B9*2=HpiP5
zm}V505*J(13#CrT-AJ1+W7G%+^FVknqJF2BhBiYth$mfc+f3*|Gw9ZbM9zvFCAIxK
zjosg=@Fpzrq?*AVovgiyTI77`6fa$gU46qh`o{S&Py!ul_S)EycbWGTxKdoIWOrd_
zbz)eB7L!Z~^}s6Kr&3TM+4($GQOWGAQ(TG-)#;`o$g5%QAUsg7MUvs@tJP3A`0OoY
ziObqdEPtB^VY0BZ`181l&8%qEL9AwgI(|oEXQqN2Vk@Q0)_puxs-cV5AANIF9+@@8
zYI(G1M-3S&IBa@##a<e!j0L32UtOYA<f#+uBg?|4qotufr-Uznsk}2+&rKltS}IEZ
zU|{~b&cV=}UD}Nrw&}O-V>7FBzvHyRqr+fx<jTWxfAGYJ*V_1NH1i`>tP?}6iE|*M
z8c~OwU)&+ht_*fpw&%k4dXK6$8CcbM39i`3V02jmy;S8PFUgb7cJVvGo5Mwzx>$tU
zqm2?)AU;cONhzLNNs?h&QKV%f8w+U6p0J+Oxuesz1T;J>0iuT5hb`LO4}dv_Iq(m2
zEM|`p)3RX`f*&4g6kfARHjB;X$D1{t$aN=-uD=n>e1D@vIR7P-$QG0>hXL5bZ40oR
zvVs6Kd3pFd&3qgwX%vYQCmjHl<Gl+W#fMiBj@U4l7*O|xB&@{Dty!Z&PQkY@yn0)R
zyT*rQxtrfMW+uMTYp7R+fZFK4S!bfs6$m0{C*{}^yS?=lgYR#@ezJ0`x$DN>9J}{P
zAJ6zyY!d7q=C(E84uSlgk72g;NVE_kLO}e=1O?U$Zkkh|W~$%}Ogk+HGeFT?UyYI+
zqMIvE>?J*dNsZvF6K_BodtSS=?tpx6L7}p~2l58~_EssffNrX_OzwBTWICAa$I6<T
zpec5LJQ!6_mChUENFFFD)^Y3ojpKMxo;wq;r$=Bp-%w2!QR!kex9K!iltxr6AsiIh
zPKV1t@Z33G+8f{5b8P6`sYJD`Gs7TA9gA{{HU#D%USFZGRRbr0r+p2%eX%ihw1+=+
z-u0)Zm!bBz?tgUMzb6+C{`4#4w}w>w$+LTd=7oEO_N$$@wW%kgs3aerA)M@&Csg9y
z!pdSluDbBD2STI---l*6QMouL@#%+#qn-&l#PX_}bZw(m6dHHfS4QJpD?h|PmAiQf
zy}hyR1UHk?s30AQM{rqtXSTj?PyPBzt2cf(lP~z)C;zm}O|PR{=lNg;;EMTsRo48g
z=@hT(A-JPo>~KR>Kz=TRw=@^bFa)df_F&hnD{efb=CGOKh{2D5q2<@l1@?+tTAPCy
z+*jm|9q)kT9zM<OS7M*JOJ%#BN2Gt3glWO|oYs4rKPxkvbq58kgdZvQxmv})O3U3t
z%n^a@#}N)gqWQkM`!buk+bPqN*+!-MA0J{RbDqM0<7wwxHvP@zH%irlgjKxK@RuLS
zLfykSR$f+q%<i&(n(Hv;cQFf9`uW*t<%DyV^syxW(DX2x2kU9{PqF(liT2HJrpm>X
z1+V##B4Z`aB72IWCX<?9SZ*CBTqwJfsl`(t5U8Ym+kWyAaD!y@wfnV<gNpm<Ra!~p
zCbu;8Piw!>uy`5N>x^YDL1A-{(}8e&h>IkV8`}vQbpE6^J{tQXj6<<+$###Q?R<2i
zq8@N&&BB0u0IrLi#ZQjoa<lt$vshY3Sx~63HpTG}zTDc@8|6~;lwLxUEA4R;ZYRYz
z_og;47?1~wU&6F64b1o2d8M5a&Y%@`+8bAxKyhS7UK74@4fk_p2IRZ_o(dJ$Y1Z1l
z*uqGFgxZw_O=jWmYbo|WZ4=$G$SW`Uh1d?$jvi=z&6Fz2irhPOf#dvb3eq`+LxxDP
z5IsQ}Bj0j%<}Tgm0*yPCiX9*X$Ta8+`gJ2y{aW{eeX&8a{x+7ZzUy;=ZH*VVOPldM
zusdt!<@DO_Qy+Qr>_l?(uOtjK?IhHSw%L$Vc+b=<3}2vg3lSVe_QC&Sj?LKFh)>#N
zY5oAUlz9H|UB>G@fM))mpERLI)&2Tk_18qFqnP1t_#D^A_C`_8)(er+!}WwK%2$>d
zyR1-8Zk|w9dwmP#W)m@Puq}M|uKQ5L_xmsmTU*X@J{kO&UX=WYI7N(T-;^4BB@lgq
zZxLqV^WS`z1T~^<{kT!*D;4nnRj+9{N)yK@`rk$Zl?s|&e=e`TaIg<M6fyt1{`wzy
z{GX4S3eyIrtU#TIYWD=3W~?Sl5dJ-oSi1>de{@dT1{q2PxKeGuTy3~ll<2-tqWfOC
z2H&?MMD>5J{_e5SaE3UPD31W+H0g5>7yFEcd5{@-dgC=>pz%6OOkCWz?dX?+8$?H3
zo-bS$yY$Ah&H~^JEpRau96;y1x(dVq3YeF4@;6B(+JB#(9l5ZdrZ75+Pzq+{?#h*-
zW=7ohcuZ(IKAW-vuBZdW6hiegl=cy<i<wmO0ArRGhp<HiWxATjqfLU62HJzLsVNLN
z33OdVsR{^fbJjb9<_ln>IKX&W2;X^U=FqkIEcMG*r$ImjQXTXuqprZ!3fSS1p#YgP
zy9JBAF~cZ<Np&5BM}nuD*(>P4@~;7GvmlC_&7P|<yaCBkFY6_0FkEXE^P(B~E2Z2N
zXoI#VOGUSfDn%f6DF+p}AD7U5eC`8eE}=>yI6$A8rTiz}-Cta6cl-cXA4nJV9?6mx
zJj;Ds4>x5Tcps^|h1!*QT_Yz1nO!%E=2Jg}6bbX72ErFKhbU3VwWNaN85=XRym|r5
z_5hmRRTisF1^rC*SpFl|nUN(9vYd)|M6wc?qd4*XZ2L2ZeTsv)7ZPFy2a5Md_3V7z
zGeuJnRxAMY7L{I{e({ERy=IA?!0X3*rIFzIvso45K#KFE+aleort03anH(Dxg)L9$
ze{`fCj72)Jdt6MTavBpPP{wW{rXoKyVYsLGO2^C=AXD&8iLs>**;xj_?lw?pn$KsO
zJ4p6sKKPX29RfOS$_k>167!qTDGk6b8wmw+hD!XWpzGU#r9ctV!U#Vme@dm~(^GFx
z*@yrzs3{FE?5ybho7T1q{d;tHq=ALhKz|B`Ec1Y|Lo$mq>Q%x5JgCXF6|I!J1&evX
zi<ZG!IhbC4o2BnHKFxuSTo?&b*ynC)LYDs<1c>kvSjm#!fa#z|>J@~iy(a8%|5tn0
z{torF#$%6AXlBSglm^*um0Y_>N|GpLw<Ri)TWJ)jkV{0i$gVwZg%ZhT=d#P4N}|x@
zI*O7@SG%H;qS@#3{m#rEaGw3kIp;Z^C(kp@n6<uZt@mB;`}us{wS0T@bXVkMyg)S#
zTVq=ot~L0qHDjN?Yp|R=2z759`J}SAiMf9QNLgQ6QQ11V^8}|1zAOI78fSv%kCb&d
zb4hOwQe>o<(-SxALLq8WGW6ON$sT4y%0q<oFaCl75#ocap$Ct+hgsdo91TqhFLVIA
z&AHsUR5n9;XXO(u?|HLo;Ez$x>qZ!$vQVi((YGR}!n7CL7S3eiEP5Oox@p8rv=n)3
z!>n^2YS)AQa&C2dbOr#xrqZg?E)P?0pOQsCz#1gj91&s#{5(M1wL;5)tRb%8VX7UW
zv(U$7jfw%~746><#J4!?=~=W%Ipfg8gZ#|xr}(k4yEV}PLm^Ejv$jt&W^kIhvv|+i
zL8lE%sO+!0rL2mrtZUJL?$Ye)rbqN0k<7RgCit2|BO0}<Z~h4=e3G`9U6VmN*wZvb
zpG6d$%jEGUuWshBU!c^l1+G^!T#+=@%-KPJP)Z_t6~pvW?P*+1zUA&qKOfNf*UZjU
z(cz(DOye$z!c&xLcGW_7k=<)Gn0&g))-*0&Zp8$#UT)La<MWBC&1cZo5Lz#K6#Ige
zbFk}(vr9uuyWQ_=Y-S2?Q23MeTeE_Z-}d%?)SY!8>UsY-M<x%e%I2-ZtF=J9;#H;y
zuqa#H1%-!icCbzZwau%$T+H;^%fiVshj4_6_Go|A&85AJ8&ZEPm3FAzqf%8E9S6`_
zv8M@qOxe3kr%cDMIC3W4=jq`-*Xlq<eMk0A=4J)LX6)*Rq{Bb=6eEs<M((t5J45_?
zYNx4uzeZtt=Dh@^JW;uVYRl+Z#JX;bi@zrp`s;Yr{KTR50IqY-Gs>0Ui-CtgC*nN_
zx~|M-&PcaLYs}dYxW^^&{q84I0~OELfME*AiIo<j1lZM2scP<=(|+u4<naf#q``Z5
z@2&*%#OQZUvwTncEtfU$HdvXp$;-R953;ETsa$2_lhE~{g3R?tp~mwpZs9#K+h?0)
zd;{{ePFp9WV4foWp34?Lj~ii#8_A2mNWS*J*iCyhG&k_gU4%zWZ6WbFfEn!L%uvVa
zu0M0GtgMl5w;#k!X8j`!Uz;xdk#$f9htNsE4kcvTi4frw2Hx=Q(6Fm>D50m{3sy)^
zq#YB#^Ck5AydI2y;I{Q0S0DjAhO@OM>4kN)Z}j992}$LgLpre(A<DRIJ$Kcn3G0+z
z%|>z`@f@!my?bSWo4hyPCHdV*Dule2sVfC!uN;um|J1i5EZv;5X+*yxS|}|;pO!pf
z)l`eAy<e-}5NbL^U$Angdbow_Y?--pdslH$jlxk~NPBVKVl|V=bd^lGMM9m!72&2m
zdp>5ano^e-Q*TgUQO8~@^2eeNKyLZROGf-a=cE<_9r8=ba4_5)LvzHnlP}edpdwGU
zY{>KFjU)A-CiK&7@s9<e<RyfXv6F|v>5%{P4k>)!(alQyrYyl!iPJKR1c3DgxJ+|>
zAAx>47JKjf!H#A8o|J=*V!m9vpWl?&X1E5ckV=7Q{0Kh%uKKE=srg72PsZB!knPf>
zQ<m)AFq9bvVnn8xeTdO_+eJZah%X?2D~#iXTH$Xy6I?xNmYS{C05{;FeQrJ}dk7Cd
zk8gHTk&U$EL){*D-Mm$64ezDUj~IgAm+Mf7N*^LFX{MxPq;HH~!t3XT`35Jr)2M|G
z#hwoMwFt*qVXdMdnmFGX#f>F0#~m5caE%)~IilpiM8rvJ;&SpSYzQ%X;Eh4#UUW4E
z{SA>Fg8h|Ny;!czG1~iib6HNnwKs_)l0H1YgYP6RJUXAN<5GU`Y4uUKgRWi{5KMQF
zKH3kN(FQTrgY+1)Q1WlBTr7Tyoi%HgO4#Ic6#1C10sn1}u*tI}r5NZhco3bROL8_U
zyht`%&aIx&scP*^lonB&j3a>gHhAhPhIL)rmS>mVjVhlSS5(8a-5*Z3nK8s*iqA;Z
zBh4wmE(?AI#Ff4Bw%xm*L*usZzV#X9)a?=UL=2>gQ13kAsnt|P40Uh9wb@t~E!6`C
z*Qs&gu<7Af4((<#p3q_>FP>-4d)`07FP5cve()?fU_zY2nC9bmp9if4@%nZ|sPJ)e
zOcvq+&$mz(bZm8ZoQZ}MvWjOD69k3L=+hc=$y?9}w{!!y%gb-fmB7MWV&N`F%o#*9
z>rY5}!IRG<?qL0@H}YlD#${Ue$_nz`<y7__-TZa=9lEn+b=ciFJ^}Y*t8ZI&e<{{y
zb4x-E`PAdi0|-64qE08%!EbVL%~MY<vhtj{TAxsdF}OA{lw9=VKMkb$7CrhHH`s`+
z|2;y4t<k`5mgx^_f9SqE!<KuT;&Fu6DH-+xJ={Hqv*r2;)nO@aiR*N<?@~YgDQT^4
zUKQ}zI2VL`ul-Zote357+&DORYQcUoJsD1|$5my3!njNzG0Xz4Qp}D66?@@wkFKM$
z4_0iIp&It{KA9Zu*^jkw3YEFOqmmDSAN%)LekKI2IKcplwvqPWzZ7iYONv)KqKfm@
zd#^0p@j_R$+X-&j@^@I{H~e}LLZun|7xmpeKFwYCYK2OHGpVtpN{|!KEi!69do<DD
zt>~gaQ5WqE`|rpSkBC)peIvHmk{>KhNJ9cU=nNbGQ*sCIU5ol9a6S*D#GaF##Z>yh
zY&wnn$a%*XJisjKn5&(P;#*=qEOX0~>qL}8e7z1PsCYD-D#h!!SdJ_TX8u9r5w|<j
zIgw;r7HzXqL=|3)eh4_SD}&kG@BW~P*XJfvp|<kjzWIAC$Va+DMW(5@w(#J!gGO2j
z=KEaQ2d)z-3DLb#LWue{m?|mg9#PK|bw@)mHntd>qM_A>g0WeLv5}NY7mSS_#-{3w
z=p(_{C}M03P5k&yOQ;HCGis({B^VoTy#B9kJj<Zf$cXQrEw)5NiO$)SV(?rQa4*1j
z5JT~xLWlz^@DC2K?rW<*c(7`VEk>%dW)sR8uNV#9u*Jp_`RIbr<1gW4pH|_zgc}F?
z(+)QPyHi;e76Ml4*8{qC*<ZwLJ8Z4qLW%dOo<{DB7BRvq9g#p1K&^y9X7Z*M)&W}0
z_sNaFAV6CA?Ug#i5P%_&LP@0{K<@kGpXrA^Uwr$Qsf30z3-!*LY^1~yOJ^WG{9aiW
z5?`D&wZ$+Ie+cCj{kZ4e!G<_9>C1MIoMbHXb!wymyBZ2pFIJDUT_`2JnskMe8UqYn
zwn*-qu7dY{6(4a$**8do=<WcT$FQBjHI;$*X(+s+Ge+ONs4DirGLSuAI5bnz4B5iw
zL8C;D)8&ikG<hNvj8%RJl;D+~4Kn>XRw}bihX!d${iLH>ppkUtqk^cEGEqq{Dj~ni
zL`rpK26Hg^ikNy(#+H*06j(9_GQ6ET$!Ql<VAwx)RtJxrp8Mm<sn;(-;@MV-DurU_
z8WTFp8~t%>iw>9EO&7j>xHHH1Onk+uCWWeZ29E(7V;nEgT_+(Z7&Kob$#eT9^;xT<
zRE`eQ5**|XJKsD|s1%EAp%cgf<{V;x(yLcv8yE$rGV!b0kmQMz9C4>Oa!YV}J*^Kn
zxCHnFZ<d$vrCUGOqNx*O5I#>B$*4iZFY=2Qcz2f|(KBDr+N-tRC18TR#DFxsXCmOS
zJRWxrR>URv_m<qvPbxo8cZ2bS?9SmuVb#>D+X{YDAOup%#sgS66K}fSg2OZzsWdk$
z67~;;9gjtyT}Q6Camy@_VV)TX2)^S?-@5%zj8>Gbw_l`a9`t*6ODHhK`z(EbZLj7N
zg-$Fvty@ybBCjGp=Nc1ziMTFOt@9ogh;7S1Zq)5dHW6Dk_5N?}d(>O>9F;5m6N=MH
zrdhFP9D_s@uKqOei*0A38c*y=(R6@0;O@#H@MU^B(=Yc|K+xm+=u-53Ra}g0i`)u%
zx3SROl_wULQxDZ>!fkz{d4`bQW|iEBk&u{+rRCVkK~$Bv)*OSDm{~;4|FLj`F?eTe
zqr?rcMs;70%DjI1mA<=D>QZ$=1LMARB8OZ*eX}{4vK-ay<AKj62DY5_KWBK7@1%R-
zg`4?R=){!xW3R5=+)m|tL6049b5CxU32Fsv5uQ%2vsokXg8Sj-{APCu%Kau$?l-8!
zkQ6wF8`uI$bT>}%KR%F-5c8ijKK(JjuXdTB`R)3<R6M*wQz-KK2pNNEPx+y1_@9vW
z6i=|J@}#B7ioofVaw>1;nkt=ZOk4`OlOuTiAUqEzaCz)Z{w~mt>^|q#Ziuo-N6ulO
zinC`-O_9F~tXD@UH1iZu=uC{?E*73==e;N|{w|=FfPh<3S>PMPcVaMu=FPdp-ytJ-
zT<iZp<G{T@Ukzl2VPv;?gGN-hKI8YS_B&3qtoCe8J6BbFE8QadW`8d)*N9j?Hw6R_
zkOIR=R7~mPLUUkYeI<M2i%k$?UcBUSf&+Prajqz7Xc@ATuo5&-7CIHQ@7&(|2Up*@
z_2H{X$;zm9WpYYu#U<noEPl2{K!vp27&Y)XlGfO#Q|lR3SV@1Mi}2-K)V#uG`(%Q_
zz&|v7sr>y5QF7Eudu*uPR?azW7)!<_YLUVS_`W}I<63spTe8)e;*F`U<ZK2w4M2wO
z1vs@$>S0O5+4Ai{7Gh+EpyneC@olVg6YUJEfpJ?jkxkl|ANA^vn?;hlhixPl>KzTp
zQqUl~dV>NO_d;Q=xAAJ2Qp8DN-csrlI|uUFF7VAU#2sS|_uBYNikgEZl=S$MAf34g
zU$=a$cqxBDQXq<@=NN?wk|W42<m^L61r@TZk#<q??c4<RW|NgwUe0m+f&T%E917A6
znLk=ja1j7D{LuD4l6pqJyXMGWX&F=l`(15u*~2|*CPT??_tGDc`xlXqCxUvAn~IWP
z+>Qip?qHVd$hT?>ln(Fve`PV^^M9wDdh)~!ZevZ!=c4Kv__0~L$*Rb5SLD9{2x%!u

literal 0
HcmV?d00001

diff --git a/format/diagrams/layout-list-of-list.png b/format/diagrams/layout-list-of-list.png
new file mode 100644
index 0000000000000000000000000000000000000000..5bc00784641ab3cb3c3275ce1e52e7ccd3b2ec37
GIT binary patch
literal 40105
zcmd?RWmH_-7Bz?l4Z+=m26rd8LvVL@ch}(V?hqij1`F;Iyl{7yAOVs-<lgsQ^7ZK7
zJw`X<P!y;3*|N5+HRs$FE-xqk8XgxO3=HhGq=bkf7#L&^7#PG8ECg_;C?+!z_yg{w
zC@ut6HGzK!`~zn%q2UAuhDrYN3ofZhb^#=qvrtxdR+p9GHnOv!H!!v{G@*C5u?JFv
zf$_R?18;3ioDGQGZLDpbxZU|kf2ZIE-oM;tASM2t#Mz3ER9#k{SlG_dgqV$<m7bB5
zAD)<)nAg$Rlv`0m?BB<M|M*DFot^Eu85rE$-00m{=<OWM7?`-YxEL6j8JL;rfE092
z9=6T~?sT?JZ~w{UKlzB5I2k!w*gIR;*%H6xYhY;S;><@%`clw;fBvbbvxVvZO0sqO
zw^={~8D2hNV4`Pa_;238qr5M7xrOa)>>W*<oPhlK*?51a{OjKT%K0a~yt$pT9bg4V
z3nNKeXA?)@VP}Jv+VM00`}F^FivKgO4~`ZlK&yX0&Ghfnf8P5yJukyc!~ZbEKUV&I
z7cetFJTJq4yNn;cW}A5x3`_t_QbbVM9sDF4`U$=Naqt;lDm2iaI6p8T8roGK%+SrP
z8MGRnotB18RS>os5MgMjPaGj3S=#q&{Cw(x!RvChW3_F$?Qtb?A$4&v+-!}->u!u?
za!i#@yTu;PM??VX&#MPnoms}P7y}#y`HxFoM8L9PAOJSZ|9@971u+{pE3?U;>5#$8
zrf?xii6Q=XA!qo{9()^068x)}A~7q(KQjvxFo#eq6n?Ap>aTL+a4VVq%#0W!A5J0~
zDOv{S&pQ9E+#fXqGXDQW<#M`u%yZ9H+d`|Axc+KyUK*6VVWCAgLd6hKF|>+`ib5St
z_5XSXXg_+0fCyWQDXow=Iw>ZOa@b!VLXjXxlBjo13Chz%|MdZ_n1JPtA*VCRAN@xR
zAqWeYTu9R5uPW#>0sV#5n~(5Uf58d~SUMw1Nxl6mgGiJ(+zxCcoxeLA8OR_=UJ>`N
z4Ai55(b8{!AoSO02}F*=B?wNK{wsqnL#QG#hL6Hn{~K$-P(cm`G9XS$68kFy%(ujB
zo=o_ye~r}uVju&U{36!BGN{1@#&G23w~)Uq14RN1c3ARals_f~azH;|8C}Fs`oAng
z3=3o+NJahsbJJSBk}P7~mD0bz*nI!<e!KeXPt9uGjubkrCaVS8akd58g#pZF9ixFL
zkG*uWc}=Uz%BH3AmV}+gTfv}L&$r8VkLO?gwg;lGH5G1p4&zCLKgv(Y@tiiyDoTqo
z$dmsw6qIoT%0m#a#4MatR4^9bVxo$I?T$xx-L3oijZl}i|2V2xJE5s+*<$$mv=u!}
z$Ln^sQk<&G-P~@Hew^#~Q^O=#MJxti@>tjVvY(L8vu}AU80_oanpZ6U5A%$s&5wA#
zSHqN-P4UxPHhb5T@4p^bwJDwyNhV&E-%Mt5*bV>kF@!0A5dFP`I#M7<zS8oPBk=q6
z?Dlwjs`g`jzr3Pq1V@kDEQ{SnbD>ISD&K1Ub2x>=yoMQp%dEn(P4nsqO*K9W0$<$#
zu2q)rk3*DEN$cAMU9a(hwaNFt@U@5D|9YBnHm~!(nG)}2QO#uj(B)*c7fR%74)KP7
z8$G#Mb2Er?NQor~Ax=w*q9Uy|RmXKP9F1~MHPA8q_mW7KE^x@@f{4x7Wx@D{@8+Aw
zes-#H=Wx>LOUZnmtlT$V=N$`Qoc6IpywASu+oLaf9#<OCvM7j-r!heiVT<&1;cM5h
z+bmhPoi^FEUy232dVRa%G`WFOA)8LQ@rCW$9}>y^>2gpZ53a8bKfcsdHIl;+qO<J6
zF(0|!Lxfg#@gMsnh~?eP9+c+#;)dBQH}yk1^@XA6`Fx*SvZ~Hz{S247|9rPD-qUfr
zM59Z<y;(_L;BKx^-F`K)pX(R1EHhIg8%ZT;Id{E3S*=hgX6cQ@MC-sfcz;~|bzaM^
z1HW#cO*(3PtRkx$VNfkN-M;H49~FjHY51S1DK%CG$si`<>8y3W)^R*rChTvcb<|%2
z1yYs3Jf%P!rIYehR^k1KRxCW3!<X6NuO6eoRLDA>nHpW5&FvzUbje0Ee#8ES${=c`
z0vM+?2EBo^NC5<HE0?oMAEIjugJvs!uOjldP=UBCs1OjbNSHHt?AD-qvq=J1P=NI8
z!6<#(7$qUak#2`)t^e9rKR?^}s8f-H_3y_EgFih_8yD!CR~*-9&2l{UQuTa20E2E*
zMKwcql7f)@?l(hY2za(wqsljBW$wQ(=uvh9cKk8(;k|4+vq`#1N;h|mM3MjoJyYYl
z(<GnvntktU3)ww!e22dCufHbfP+PPOAYr`+3EVD%e)MDM$@7d{Vrp4CY=u*ra(Q?0
zvTdR@RW%$|x1DkOgXomL!64zAgQxmtyUc(2`Q!M@PN?^J$K3$ea(@deru*Y*^Z0J|
zO%cge%XW;B!&46e!;1Iyc=ykf`pGR_UXQCCS&r44U^F?9b^W+EIcOJ#$opa^@q;5e
z<n5G-noe@T1t{k1DZ8mkCLZz4$Kdlbwgv5=g2PDXRK(Iu+t%=BkIxPENl=uZSnbxb
zPQ#s<z@;(aTWA)sx=h;vE6St%c#an*3@2GkCBEMUT65xbKkz<B)tH(@VSGo@z}I&{
z$BoI9|M7(76&xWambQJswKUv!;u)(@e8;dxpPpbiO#R)#JKmCKIap3}Cc0dw2`*lA
zUX)ngt0CLA(-dg7X8Z8X<=8jt7SBJvTg`q_xV>qm`MskdCxL}pi#({aDM-O4H)}Q9
z&bFL)J|<Myt#_(EHn000)9h3>uWezzgEdW4R-09=)Ql$Jna?yCrE4bR^TcaA1HCcs
zZ~dyK?GVCo`B|fWg-$Qe<#>KA!GXshh~MJdZ9BPo+NpTTGccxQJMb7k65T5YP^YN4
zcqg%77%GaEEofZ@zz`m6$9WD@+tDLP;lZiLif!7*)7Px~K006}$3G;9R<)hA_N}<B
z`;=z8W>oJiHCnL<I(q1U5%hDt<1`7o6d<9OwFPHIP26ld@2qHoa%Rr$(ysGjP!h|s
z@kZlK&7RtF41qplZUs9dr}<n15#l5S&4Pw?4w0WR3^hG$0dC4!^=Vc>C}a2p*McdF
z<#-_M%nIT<;&7%k^CF+ncu%ggU|N@1=O!MG%{(2>u6~@I;=LAuw`j30|79N<FKyAw
zw2x<G3ov|{oq5x+pj4+mRR-<~`VmS@5(xM%90Jp+?hy%0Ux_g`WUGd1D5b(LR_`xp
zOWqLjVpr?{%ciLYDD7a}(315y=V^MjT;;eRl3vJavjN<RV4zQWj-JnwwaH8_a(kx^
z*myjj?l0H9ucMa_TOQBa+LEAJ-OiCn$L6SR*4%s2)V_oygbt6m(QFjggWJ^&Q>gEX
zN8^SB;L~JS#HmBoYHBEiMQCpD&s%}ab)ulD$n@vdj0FVP8s4}n!LI|$>95H9J3swa
z+A}KF6lm=t=o-a^O087X3C?Xqx6xViBJvz+<Z{N2UTk&Whuv@kOu^WeJ}6sYCEYqa
zsHpDrTxoImv<*6x-S<ZvMOy1K=*_>8&bU7FBf7yZ+JcO4NS?^zO0>X+V_!0V>zU#p
z+QzvBiLiUxygpGpih<cOY7SnaJ4W>hwi*ocwYS?TvoZ_LuV0N<t3ZmJg!)fY!xklF
zQ&DJ{3N;6Pl5cqh*~bep)-G|L1WN*JFZ6&qCH0095YSIUtW~3@?K7I=fM{pcAKdqf
ze0%RbQ}!oJ5o9x1B~`Y3!d<9eW`W6vuFm&w+3S;L4}hs1$6mN6m(4wwF}_5@#3_1D
zm4<8Mf4gXw%jLA!0`<8HHCArV5H<~0mUlG!y!|?+EyPzpBoFe-kPJ9a4n$}oGy-M|
zD@ckZSs&@u&^Br)d>-jF{TujF-CV*4U~$zbc3bl(P4>83b<>i${rZ#=nn1#K`3lck
z0K=m4e66nu`Ze1Go+PkB60yT|&>)tGp~V_~T<|}QQS;gefYIjS^h)!{X@tg-1k>I_
znE2wmc3-n8|D(9s-mP7Rf-NJ!%w~}DXgrM%AGw7FYP6V(zP<*M2J0MQXJsr1i91Zy
zSfMa+Mn<VH9+UWf%pGonQH=Ld@KUhueudDuq<X$|O(U+E6hR!PLBh5KeD8_#s0i~c
zx*L4#72+){(!=eVqWX&``Yzhv=e@Y~chb}##Rm`$0Z2AmshSJAz;#0+qX`2vK<IO$
z(ERyBMl{D-+SeA{qLTemXkmhw{xi&1I@d08nQP?rT|g7%z%X*x-?>F%+2{-GPVa%y
z6Xxp7R3P!3Fk<1nyO@?{Fi0jth=`|KXg&F~r_TDN&u*Huc7&zTo=B-kj{A@d%LqKv
zsKxOLt0vW1U-|Z7mj4!Hq#SYKUi@e|s3@JN)nLB4)#iK{NIJ1Q%JB7s^zr5}DEp1q
z@-SuY2`QX@s3~{=7@cVG?V6MXsA-H8vFX}7tj`$LE)MXHa;p)ep3U7wFXLtPSu`fD
zzSJx$4gbKH=$prS@QR~;KawZvIPm^h$GiAqqXv{<a-kn>x(YT5kz(!YQ={e2<(G2P
zO8$G9jms7o{xpSc&A2^<eeR3v3Y8(2cG!+IR$*H1fNnqCp9w*LVA3P1=}(|$x^_FT
zy&?Fc0S}4}r*x<8eR!;r)HgM4ICHxa-FlhWnUF{hAC%Y2nB6+w7ygIeK8qL1mh3T8
zI_{SYPg1^iBMO?&7lr}sqO~eVM|X>j`E9%!Ot_9F>r2F!GHfO+JMeZBMFMq)h0QY}
zzo$*qOr#OxVIH@-Hq)Qtp<;ZFhlyxi^D*_vSKqu#a4eq7ol-EHoS6z^`te=AZGE)*
zdC7&9<}?7sp7`9-xd&<?oqk>U#5XjH>hSHV^84i=QGL$<2jVf<g#DRSk`8mUK`IFg
zOeYlR)d*B(y5CY#1p4oQ<5S9uL7v$41vi_CzG^MJo0Wg;9xW1kbT((M`ug+7Q?Rz@
zVR4C6YXNK#1x11|uf(}mF6>!sOGQCNEgF(Eqb`(pd!R?;E+<moz|r=tNWt*tl5|GU
z8~(IhGrO$~gZjkut>lafoi<mIx}>pq*fO_B!k}Y>M-dcAr1rz!9g;Cz?)TTOG+}YB
zzW7Le^tA??@({=9=Xze(EYyokd()JZ{%f(B;9_ncVGzz}I}7hvB1aphQMDaEZQaE-
zCeQ-TO(c&vsXH7~32nG(s}o#@FwKWI%+`f(G)wxdS46CHC@8Wf)~S^S4RH0%9a1p}
zqh`_*sC9pbal||U2?~Ydg^<NK9f>MlcinTzn2uKh-7-<wxAt6JZ+nmsj7TsREcw$G
zr|ZOK*wmyy`<p@xevS#0-pb<R9*XqH`HGD-qk5Dwp%i^)F@wcF{lpCl-Y*d_%@>EW
zcYU@jHZ&E*O>#_~ma#%a9lGbY*6pJVjy*MW)<{6>Ldb#?0O43ny!-leh2S*{yB*W>
zsxE6c_*HD|=~Q2K5E`Tt{kxQ?VY?D7dB>kdth_+O$i;~oyom~4*K6QRvdWzqO*-7>
z7}moKQY5_6@}FF%^%7hC2V=QFrFzYVn^^AQT;m$PTyYm<R?`U6Y--ngg*A{C%7r!;
zJ#)#ZYOrIp%#)jG*U6<+Jc#dv*!(R*LaggX6p44>V-mG=<0VaH1F8ixjw)A7MjQ4h
z*VN~wf`|nF!_ZR#)~`s`Rjz^JpUOfWhV4J5T0t!Lz_QpGuo#wnXq5U$Q+WaIt;30m
zZIThx#q~4f`doc*!@LJ@BRL`%PnVm`@|gGgG>1{*&dS0SO_SQ;>?mT(8z=S%tBN1o
z<EVMJ@^;Oob4obYy%QU;Vi+3W$`co{^mU{Ccy(1yr1$J4JLA*Te@q`J{?n7h0n_q@
zmj_+YTwaM;3M^iAVQx%HmrkN2>fgtZ35x`7Dbw{Bd<hPiTtx(^e3%+BmAa?ij@O(@
z>;Gu|0C!P5JXq{O=Eb35TGsD-zIfdy6(~!0AzGVwShwTY%qUt9-)&Nkdm5we5H3yM
zg%4?h<ukFyu+1Cj{TdD_3l3wDe+wpxw7)EjN>PD!g`df>Q^u5KTmy%Of98$dOb`Fi
zJWo>Gb2uX#ceCK=G2$XYx37sRT@Ca7%4B;F+8&wpZOVn`TkR$FgO`xUr!CRv@SjbQ
z6j-w?K27aSb+@hGvq?SYO-)N!r<(?juWf|6m965#dAD7vr|dd^kqOy$v02_rdy~e&
z2;+aj{8i>KsIT5AD%T^pwi(j1zUC1*Rosx&@@S12@}mtN(UB%{t~F8{+b{SM0sK!w
z80tTieCAnHq7xB$y43q`I}HY(%xY{3NsUL;g)XBKvt5_q0xo?ey}Bf*HW0C6<beJo
z)|ekegA4LOH76@iAEQ6v%fMd?)r(900D@Wc3`GUa?b8inO?Z6UO6yKR?DPhejpcLj
z7q6=!B&wj>b_Tj1aH1uaB{2lh{}adwp>?rh>>7O+rOsb!=LFAG2wiV#2p=a4C$r{l
zhQhZr4fI%lPGPSWE%*?(rW=z+j?*l!b()HSNAgt{Gic&y^(gh0+~Dcs;v#I5*Wowe
zH{-erac!pK#p;OC>c_)9rp!yWc0ES(n{plBlT8!)zGDj8ZJy|}T*=@aV6}x<v#w{3
zh1j?n$jIl-P_V4OnNV;zFB8n)p*uE}=w0qkHU;;xK+>&*754Ku-&%<Icp43d=L++#
zJ~Gk;Y<*BNP1Cw@UyS8VALFE92nPcs=&4A-zx|RPAG5_|M+7}aqOdgWwIf*=%{a89
zttpU{vNZi{#oWAw5}JWJW;Pg6SIZ*S0O>n1i@c5Xbvf~#i+3`Pfhoi7>85`uLBqk1
zGn16hjPmg`?`-^csS|^37A?GZZRkZKfN)#@2aVB%+z?WF<2%h+GX9C#<g6x7T2IFf
ziJv7aM3&F(;>R2nM{QIaG*tGfNQO?()=c@7@lFGjbeCS!oSCKcg?o~aS1cSf^UQ_~
zt*sNGFVY%E+Wp8_E^hQp<lT^&Z+*HxF<z5XE~2Z&pg1<>Y-)prOf<#o0p&3-?bg>x
zpnNX2{?tee4=r*^gmU&!KNeYtq%xEa&+k{oZ(I&<q!SnKQjgk;^AGgDHoWQ4GKBcm
zKPS@Ycz9ZwveNS_6cQ6hvkK!=yRF>Tv=%X~eQI{;ur61)&Gh34i8OM1dOOS%Tc|%B
z#l?as^K1KLR0%njY+Tu|7ql!wV76D=^kW8Ny0smf@^9*R+Ojp>kuU1&9Kv*u=~!m-
zOdQPzwILQe42QKc7-a0OphLi{r%ZW5&#QiqyF6w%P^@ELQ>s6%4a3bIhO<V-h!<2>
z_lLY^i5-<_F$X~-*HS+HtO4fe0GOZ@7fA}XrR+GM2c)%xR(rmUcB|;VL7P=Ya+Oe_
z@95wA5M6?q<fRK|Pz@l=%BlU+=My^<MJ;K#<0Y(Unat^Qpof+<F^}FB0sADWOm-U?
z8b`yt=|vCfwZ8nG?(fnO{l7rv6E|3Dm})$sD~reD9pSa%hBY6oirjE|A>44d)&j~9
zN|8!qQZ~j#qHp1|a0h0(<|K%@#*GibwD`}EU4yw2Sb7bDYlad{R01f1ed4ATaCU5O
z6j314nq$0bk&)QU7t==NtWFL(g{F0<r({q$N8UCT&XlMXy+D)75a5s95uoXJSEWC%
zpktZX$PG!lE2QP5CO%+uIg)+##?~93R;siHu&`f1xGc%+jPCZ)q45g@g?<NiGkDB-
zP5%%7FIO3ep)jnx&gj-&6TT-E@fKhoo!F-xwDV4nugEu0U;Ar~z2=0Afyj@J=V8-W
z!uBTnc6~sL#AH7IIbA7jnnF&940p1{VH*Vh3TG_kAW!!@O<S;_|D#kF4%m!o$=W4(
z%q6qM_>SO5!ecPOrj^}W;+cH+D8|KstL$g{DE}bL^p5>YlXC(w^q_T~im@DBx0gjG
z?*y)|pj#@M{u^gE2C5H=5trX$0+BYyyEt#r@)Ec6HFphpE}k{!4wwG2DUBF(+JyV^
zrY+b5!KxN|wICpekt6|HOJ|F$S%o9MFb|7ZzFUFN0Z1r4$-{5+`B`)QIK#mKiISo8
z->Xuec0~l<IC6b2St5OLKBwSjqjZC%hD7|8ia`mJI36ct+YG}m;o6#54z&+?A0$+$
zemgBl+uW=DLaT<OMp@{3Za|SN59SNHmn?JS%Pa>6aN&u!#=6wZ_8S5f{#fviUD|#1
z1=mzo`4M}_we=I+qjLjIluaHU>wBlR)Wi^QBzZ7gBm^W(F99*J1%ONM*NU^RdppIn
zY(=q8Xm69_4lh2X4zjDMv1&bY79(}wP-Lm2?sTjGAv_O^%OafK97gpcDe`-8zM)GN
zmGBoxTOJdtp}&T<RU!IM*c(<xKnoGFP*LbF+&wJWU+yRjqZA(P4}kj<A^_3T6z0xJ
z{gF069bjN${{N7YjixZ&(u53x&-2FpusD?zBY*6_Ou-?n^ZA<3w>-haGb;K+sB%nt
zZEVqj^740e;<RKDU?W;!5c8wZpCnZNr7Q%XQR4_z?&V@dgOb@=F9_7i*^WpU=5i8O
z{>f63oLE>z(TgHZ`zjhKSTg({*z(`kE@^;$Oa0jM-_l>+Acp}&hd^4A#9u^5J1}&-
zEU0LIhfzSP-&ep3f%E@rrn;{PVG~iq1<KIUm`z3ieDmQ2yUgMBc-T#~n=608fLDQ4
zMA1$SVwZ@;6rd-0>j6%{rg2Uc4t<gawEzi+IXDgQZ7j-)Q?CAvsYUWMf$>=(S_DNZ
zzXY{6nd=8YlX^fLlt>|a)l>vvasWJ0!$A>xvGT<!RoC+npE<=U#qZ~XS)wdE4zNbP
z22g)2ubTrDJLzrcpD!V9=i`~g5NPNw>z(fKc5(Ui@9D?^;dnF@o#}`X*afVmtDQcN
z$v^MD9F<qVE0}Q2I~-Rw=Xf5K9Y|Jut{Y(jobv%3vy;XJT|C<s0{}lPWKAX1Fp6fA
zd;fex6+*^!ZdKi3YIF{u+DP=k%TEh#Rt^3i2Zb+qx<+f4wAz&=k0=~a@VhMk&r=xu
zV5qB4;E=C4U+?9I5^@6w3M!o~X-FocL60bwPOv#PL9Y8&I5yqjB;Vc2#b$5wn&+`d
z6VI!B)3LNm0326DezmvJ-)J#o3a|}6+tD1jtmZZQIbN?O)on(mp><mm)Rw8GfMFuP
z31u8zbpalCum{w1W@w?wM(bw`Hq#aWQAstlgrlDU{zw|X4|jnMfZj13eG9j?A`LDt
zFZbL{R<lpJ-%Zt<*Ku1F8M?_uZn=BHqD}m@Kc1;ZL8rh1G)5{B7_HMM&G2!!lu$CB
ztPQ{#?s7Cs($!Q6Z|*iG3pmq6)U^O!+A`Ru-4HP<kS7+6dq2i%v&7o_OezAP@?O85
z@7{0tLykx$60(|qv#h8D;Po0(FqPz&L4NTb=%jA~5>(*4-0CuRpc(_<5jT(l481}j
zL)%H6vcn|rwai2wCqPBHt-35s6^d{F4B)x|n3Mxh?Y4qALCEdKc%WpPTD2~1%iZ~U
zu6u;f&xie~1qbxAl`q#of3~eSj(W@h!S->|?O|!I;{bH14sHDyv-%)P_)&Xjf08TE
z1#y=j@oYDs5bfYF`M1K}`T|stZ9MUJ+lw^c=N}|*%6kD2p%xfAj`)2Jg9ObJ+{ec6
z47U2hSIX6@*Ak=$7|lA*T93gDUD5;jfxQEP6C8Q($8q&OYP1ai!RN5sWJEex&|U!a
zSmr3tVI>@q@BJous`hl1P8+|^560Nm>j|Fl2LOpz)%7spC8uxM2yA%uU8CM~6)3K#
z@0E75txg)F{(&cAZzt}x+wg_H#Qn(Ey$maEm!qA?8{N)_z05bf7hp7WoD2r)HnX2l
z)<BOycC~|qR>MNzcPS^Ye*ydhU{M77Z|5(FIZ(3axn`OXH`ojA?sL%%V-XN~I-W%?
z?oXNPop3FD@yQ^Vh4JF68Q8t?s|Bw8EXd|@j`b(NCs_o5TyHq%-=Ja$aSh|rph%SJ
zM@PzR0R_)5FW@?`xjhPnvXIMswEOzZjxnF2Wjm~+T&t$%<FaUy+L&~9j-jT#IerD8
z<qimqsE^v^$L|KaULf~k@%K)<_Yjp@&9@~_01ztkfp0SyP1_ag{^#R49Z%b{wlN3{
zzLubTBM}Pv;A)g%QaMAGw=02K@Y6ly8{ThfUPt7m?O&J`mL%lsO=Xc(o&JT`n?0`g
z#!_h$G+r~+^dDz*USP!h3=N7HwFR#}4^|r<u;8&N?mW?;zdT?A2Ue3d+Tjz-^3o5Q
z^ovDl)Z2m32rmfx>Oi{*A_8w?E5Px2hhV7Y*&}V~L+nZGcct4gc33ag6(@E>@NeXM
zvTzKi8y6gFomO#pXr8y7^XKkpUx@&`3OkwDPnzm><`)b-j|>*%&j6<|Dl?=$#DIb#
zMJptdB)E7jn|IMMhC|>QLFY=XIe1Q9j%}ESUcx><djJquwXUa2?hG}BNR2KB-}2Q{
zhXcpcrnTvn3aMX5GP`Mb6Y;t;x>hgSwvCtTzb%xG!i_$CNK%s9{67HoFvVj<)PuDS
z9%#cVF~Pe5w9bnjgfAhQT_4}CS8k-LuTu3TnbJm5oq?um+fj!@PlKwcth=ZLZkbpJ
zWgi)~)_*a5Xuau%{EesY`^ukB@SH#C^?m&%yoM2$^()a9ZtMv_rq#=8@tsl=g;(fi
zjt<1xj~oz0(ePNL!ArVOwrOB0GgxZE2N52&W8aGrl*XQQKHAcxLrQ87b3qA}O|kB;
z{^GxZCcO-hDDmV9jDz|Vb>OF7w>S~Tltzx8!uDxl7an27qyelrKS&W};tehC{h6ah
z7Y<>Gr5OxiIfql+(*T~Gh;ndutRDc?-%#W{mry}QV3aC9NcLJHe!1VMOZah6Y^o3i
zOBU__dZS2E1@5X(NY~Pt95MNmO_a%}5Y(-17@{IY=dCiOQcbb>oA7RG(nfrM)ELdU
z=Q(fRM$Nm**wMw9nKh3z`|b_E{q%gASeKvQwx#_8lA~Ei(7%?lv2<}=a9*U+TvhtO
zXA)9)(}oK-mc`}t{C!~`^N3&S>#F5cK?wQ871n0Q?hf*re$Z>G_m$sQwUz)zSDbm4
z(~b;)kbOaY*Hw&Zs^Yx2qED>t8W%IfzWo?#eQjO+nU&eMtqz9GX+9d8?yxO=29Wu)
z=60KB?&-9a4X;VoWyZ85EIEnoQ&FH-S2Wc@o7p_{Yk05-yvla15rGmh!A(hhbqS*q
zVEfSkrI)z20<dZAmurl|{d-+wS(o~kFB^9Pz#e=~U@&`-#720)ULlu7dd3LAh66Bh
zY&i$RCd2W&_+m4B$r;t)bj;;tfTi3#b1bj@!m#bLNvfj{?C^d76~5ZTbctuz&g3^k
z;rfNv9zg%m1YCMCmRGk#@%BRXoxreSQ%et3LJ3^cA`$VrMpI~qqNY|eQ^!-?osoka
zaE(5#213?DKdKk&KfMgX#hhI_O25s*SpJM24WbKzf+vtsEXvm@F}%&598I1dwUcuo
z+@4l&3N7i_K$Dm~&cU`9JJqInV2H9eUXb;Hsx+1o`W>juxYF^rlpKJa<GMbyHn$Po
zG;L8D0U`_4@p$}1Wm%_vQ-jTNx@h>t*^rGT$k-(|#BRB1xl2J{QP%Z+Fu@FTpT|}J
zRj$%O|MoqqUqS(lYWT?zFjwjs-}dE?Ab#_bMwxw=jbeRFE4yS)_YL`Y*GVFpA=7Th
zLx&@BrmtI6MD){m8*J8}+pis@;C@6%cb#{XEm-&9u~{blM#J&QL82ew^{)ri(rDeS
zicI^Y#OM09@$}t1A_;A3-|T6TrW-wBU7r8Sv}w|?RCoyM{?PRV50c&=F{?7v!iY|b
zbPGWHz&jN}=l>WD4brjzSGF`|4jy0aCp~2l4(ad=_E{$Cu!z7jZ6O6qaX)DrYXf!Q
zrt;Pl$Ktrq?$xZOq>II^77O~nHL&f#DYCe?i;F7z>}G0(W5WbPkU==9j6TV)%STIo
zd0^9eSYm5IrR3TFJF=n$>@gUq5Yi{<pDUnE_;5OSKA#+kYZ;*og!`pvT4E<j00!Ji
zO2)>whC;SiyjW7L1?MiiF)&ii!?Ss@XtJ6ftHwx40v@w1VBMhGOqsCm>%_RxxLk7L
zIRk@eWHE4<*ynCrWSUQ6>M?Fy`e9sg2)kVz#3CMT-vtROr--t9g{jeoAwuZpi(d`b
za5F(W{?zxQ|8XH|Jr6<AeRD>Co^3o?l`$2Fka5sNQ&7^%MNwXre^%2^Dvr~#{jwV;
zj5PeEL3slmL(Q&4%A{jBooh|5fzg=O=XUMnTYfJfdx-@hvb39Alz*rcW%}*vGXRRB
z%PVCICS=A~q$JZDxp|LOn0lIGrz-)>LS@7`uZXUi(9v#5%53A%kXqd+B6cF4?D52l
zbz;k^FBgW;tcmQVF6<o+JQ3nBdma^FkA{}TeQBWr6mLD3XhWh{KY+1sexZ)jIGA$D
z()p_Ck*1-6%gTk2MxDWUD-EFGEifA_(Vm(%!~IfSRjS&}4&Tf?Ki-a=#1|S}K#F0F
zX^;`(5E4bzCHeCV)=%<hIw{x|_wO^JN&jwP_*19vPQvR#XL6!tu8fQY!}|g-iZ*u*
zugx9)d1K~c<@OgmqfZo&j%kyr$nip{ud>G?rI=3zw-YtVXb@nZZk2s*Rb<>V028*`
zrs9jr!+zjAN}B|{h%D-3z~k<vlQVdW13rUwnb)P9@`8r%<qFiv%yY^W?a?pam(!I~
z!KL|v9-qdlAuGOQKOkFLZax{S0$zkw$U_CPX_tMI&e8{TzB;ha^4~dOvy`Z@DcQjx
zk>qZxYPp8Gyj-zAjd3xJ6r{G5QdlcQ8H^fAeHkT`wric%?(#%~wRoH&tv4TV=Oe-1
zw>RqA(|$kmU1t;iCGr!<c|iiu)G;#Of?GSzRax;An#Q<5jecZ{(KDGgoMRDokMlkd
zaQ#XD8}ddK`VD!@G~Fs<Pb5fpNEdNQ-_K^xCS4Asy{T|ZG)cLVtu#O*qk&^@+#sNF
zP0>ivUel7PXzBZQu-3OZYwkt{%x}05rn@btF$yjoJgBzMJOaJT5gT1xq#0;B_P7QC
zfo}8iEnLtZ(iQC16C-J%O6z_&O&mcIwjJqOYIS-|(y8>;J0=vvoRY_VDmGrp|7e?t
z0r?w@`AC8|#BYTA+oa-q^M=`Ix|x$|k0hgT@=pN5J$4P63rXo+{=QPo;9pv2lWD}S
zzt)w7nc!-b$6$0Tz05Z(O;w})L(wsbnMu8EQDe6)c|lsqOkp9=I(Az|Pb=OGz7L!l
z_G9t-aZrwqGIvkLFbcl;bV{0v?{(FtO4yuSM_p=eT-yl`tCY2oECw~Y{j6S=81^Sb
z2_)}60O7W^OiwqIZ##ufgduQ7w#Dc)Ht=Ez+nK=rb-x|^SU&1Ie2ltS(WMj~$%}MZ
zCgpc~@{g_~9#+39?5ei4Fe$Hnm{Pt*cK{BAsP+_G+8=1t0OvF#%kc&36Re2m`tzP`
z@heXDQK=2srrEYNY-J85V*B9@@eHlL_M2(xH6LI6n9XrR1H>v<_CT9y?wfDIN;ARH
z1|h4#-ENqX*(QTHvmL2Wk0wjZi>#7f8i#UT98zRFpeVALS3mkAgtChtEFv2hLzn~u
z<TA%J3>Gc-CUyw99E-C11LEi0-Z4tG4|gqwb-KMKg|ovnjx^IW2oz~`DqQ^hF{tf4
zaaWdmE(cF`MjrD}sT$k7%Jnk|os`ciF?BCQa}O6vNa6-n;v%s?I~%HNm*gJO&VU<i
zj&XPPFwBa{o9Rq?2Z7;JA4$-j<YfI`9h!bB#2Nh>b`P@Cxm4c`VR5w&`LQAtyku8r
zAG}845i>+>+`_RMYgRcW@~C`8C+;K-7pN(hnr{MZC}?+%d?*oD_Q5RIr>T(u8fj8`
z*(!wXW-F4R>At~|6{n+=r`~8R9GAR{ha!sv^DD}H^>{|)$V87%Ny2rh*!Im9-&J~{
za+*FP!d63LLl5W@W#Ba(oegg3Ts-1jZtsRWQ;SJ=Wk}=WtTz6;lg~?_Qit5#AjxU<
z;!a-P#$XvHxoCE9Z@)#XgYhu_(A5Qg^*}nNfL~jd1srWRRG(A8+z3QW%rDuc@O1i&
zO5LbC0&2Q+baN=YcF9U?&mY4jHlQbnVK^r+2GG*((_4hR1Bg2g>!6bGIZ*dMLF!KU
z>`#f2WqN);nj6KC0OP|GN3s{hE^NC?=^>TZggRNyYcQsNV_`(Xz}Cw9O4wHmWmF!4
zw4fsoFUTsT572ILUtLH~=De3qdC!)S>&I&9#t(L@Pdk%#lZBfk8STom%vfV$Aq~>%
z{rI|<-fG_w47^(&$0s7bdhDreN;x<>jzbT`V{09kH)E!OX%FLWHa(f8y6-FnFEYgo
zQ2HJu?bQ}%s93IYWDPTAvWPzm^0&OAyQj$!@L-F@w*5}Z<?y<a`{BCoy9xUDX}jZ0
z-%^-Kv>BJZm;sJfo6)74EesN0hYZ~MD9zU))c!{PIX!>#IjeugmRxx`b1ecwYQljY
zKdQH4LpGTsAqrfiZpN<Nq|PXrZU{8h5f2|JFpN^;pzCOu8qif6R}CSu#PIANe`|tO
z8|Uvrtx*fB?9NaR6{`pS{sLPsn~&2=Zw1gz?%4*s+mq`U&mq6p*%9;+$LQ@lK+Nr~
z-+xPv^yWbFlVKWTcFK7LECT}4Vpk&g>*=QI2UPoT;V+v)r*HYjm>kXCeM`-@BjMdk
z*WQfqC(m^1>?luTa6qMZ5ZX{-zDF=%9=so6Mlggyr-ilGQ&0|t=0K1!SE5BAfZZO=
z^u#IRP;x>+lXr9-zdFMI$SuR!_|4*>lR18Iyv4g2h*XP(@_PIXB0;jXajCdEeT1~>
z<wiJ`<TFb{fd$z43<Z;ySno}(E#f0-Lm@ox&c<7L=?n(7N}8)U*)sTC@3E7F>qi@2
z3wqYeWLBAm3yeYCAmFil>+0_p(tUps;6alQ3q|aYVe=cWHXZ&61j+G8He>cy{QFr>
zu#l9ANCf^>31or(o-8P<#3X==h$@h03Sk@vODcZ_1;9|$Na)*9ysinCskbZc+tKYh
z?K^mw`QTvPBxx=lO@-+*EV>i7uTo%(-&r}pe&UGi)oBEHDY8reoiV<PEuukzXje4e
z=n9~q)%^lMNme)Y1=2-_IQ=`H8(vWB)jiSi6H}+^E7wc-Bq;FjrjpKy!GJ`8iV^To
zpkGxTjU<A>nA^|eRi|$xRRMV2u-p;dWXUUAjLcql95zoD8P3{ZVbSvOk$4Yr0vGh*
zllu$H85+TE*qv{g(224bFPc?>T3GW)gW91~#g5vR%~Zu8@```8*yzFjz6J<^T(acA
z+Jo84YK-xZUgS5*33y7205m-@A^w{}_`Pra_Mu+`b+gE$lKKlMtwW2$&CZTJdtv4N
zdkNqIfZ~jRLed}i{9%nLfz6MH=wH76Uo1U<74Sq$0}81AfX2a5hU5Sz#+CnHwcvlM
zWdS$=a*(15|Ddt|zQ$z(Hb3!pR{N`3(}2=F`&C52OU3_t5%T~oSbR9B`m0(+FOu}v
z=%W9~eE)+py8<n!zn;?ht6CDmd8Qehzf%f)K|Yu&9F%70P^Pa`;1S6yfcqE8#)uY(
z{($|Do*JdBygcFr1tFZ&{|eLJ-}4JtoQh};Ea+fdY4e>bz|V9+jeJ=D#$X)~8dpJ&
zc~vLIlk?*6B+&(2Js#36S1I<25JQ5?hO_u9I5WWwakQr0s5JjkeSvV)I9#0q0?OKd
zIOTsD8zBjpkLj?&_K(^Jppp}_wTke_8NLJn|GCg#@I=FDwbeguN-hOF9`2gehxrGI
zK}7)=ufM3e_0NK&l7PoiPdUTs{s7A5Wgtv`{z%^J4SjJJ0HDtbphY#CtQP@@3y5b~
z(9Y$<SuhIR|E!!8v3!%|&7XR6U2TVc48Mn6EW4->VRE@~LZh*EkL#4Db;<!|l|KiT
zzzNidQZxYRy(q)QV+nQYO~+pl?qn6!bXJS$q4>8moyeTZTOgh;jIej>6>8Ohr1K7d
zjCFx1Lc1Mem<-wSlvH`kxJgYhTp$XnL?-<T0AYXL%zQ|s`~U=#yIPHwjkily9JVW*
z1Mf!4J~0^$AObi#=A}~;prTJ!nwiB!p>l!PEfdr8*i9}Geo?+>^SDi{N#>)PF}xem
zY1jv}_zFV60h1)hB#w0`p-P=#sVueO#&7Xz?eBPr10JsT0o~~-0IpM)ytYdLpm9R)
zi|(~F{GMIzq4y6viK}kw1jGWPcg98liV6%mXQYc{QK)x`0v_d4fHawZ^$Q?&SE&$A
zq*dM;!6&zDeEYjmhH9cG;B?M?fj2;+HS@w~Oj>mT3voh^9YAl1C0(1WIt1u!J%1dQ
z&RAl?y<pxyz8}ZpvPBdNK`6+Z$BE~gc~rL^l@AhnMadcDpmne&6GLDK<?+s$H(^Zs
zX}F?!dT}Tl+KGkG#0h+AjFb<*hEYNwsI4Jj)CVK0{1i&)DPif1q)aK3W<Mp06_2uf
z1jM^2g>y<lyrE|oCLS+J`S-%nxBY-iBtnc3fGS_;FbU|Pc|N|zy9G4K`n3XFYle|@
za=e#)wDg8(lyXJmuW?z2rXAgA+zp=|Y+Da0<`=}GumkFGOZ@<$Gkzxg7K%M|YuCSO
z>8+Uo1(J%Y)+6rY&p|k*0LJbLN3{b4kG0ai`n3Kis|yVP!2muem>MadF4^vZ5hFxA
zyCnLlPoYS04FcI6Filx8ptJw_e2cdZ1_*AG&&z225sP)9*EeTMKa^s;fT&U2&SggN
zPrw{1mL@*5ya8f2uUQ50+N>kwdK&2ZZ*OCHd#+}VDyq-6t_cJv*8seBAeQL-T?R0c
z0J^3Vm?BXzWtPAK(lN>ZXpIjpIr-+*;={%-H$d<Vh|f_}hsM%b^t?{l)bgO$d>?J}
zq9()R9EBveXk=3rV1qT=V3Ro`lt(h{Ab5j-vjwjjFSLoAzX<{}!4o(YQY04IZ~H-{
zQnPWX*^Y?4dipm600Bp)8NKIb1?-XjOB9*bfdp8CeYHk|{J);A)tMRPj2ca<-J)^X
zB)az60m0PbCEG92s6I#-B<Faj7`|I|c*JB--F3wmVBqEpzlB6H1|`=<UXMnk<49((
zT5MH@Fp62ltZrl>i-=n2nx>m{KHq9(T<m4q5ju@A0XENn26IYkR4QcB)N|hq+37#n
zT^8ATZbYI9|G=o@31^-!j6kq<8Am=g`}{)h{NJdE*%ysJ#k%w$Km`C&?Gdov=j~)Q
z-3I`h<90m>6V(TZjyu!tqH!$fW7FryJIWlx2j6Y4*lMg>7j3W8#!g``fP-M5MabH}
z214MPlz!R{*-Z8%?<I>;oA%3rD&s9+9@}YOKa{ir33b{L7}fT@qNL9f2Kqx^-yAQ*
z&R-JT1Etq&GyxnNfCW2fz&8}XpTFdsW3kpLI~1<m+wI!vee=G3CqdeMSs<6+=f3Id
ze~1Njg%#Ij8;0M+f=g$?!yyUzaSEVe4}pXHSQTq>&WVsL;RrAR1~QT~k|0xEfNcRJ
z*sZmVl1T3}Fzvf$VhY;XXH+VI%>oR3!awq=7E_FxL`nBn`x;v0p~dpD&ohzl`>c{G
z!@3owA0YdMF?gY^68M!3X?|t9t<_#VEM7|`u#ch00fzx-4<Ev>DDp)2sybF(mp!)W
zo`4ub6NBow%uPpzN>aNj3>=NtkVJAcX3KEZxYEgQLCcO0ABsiN>isY4#F3;fJR(M>
zP9lDrVYuD_YP$eTvAf2_x5F!MxXOop?pF6v>&>^}hfxKx9W`Ksllw9`Y7MM>MJ`)i
zj>WhDbf00jro`BUnM01w6&Ue5^ppId^QA~Nz>%Y#%w1rGB4zU7@VY$#sJWR47eB67
zNWB3Zd<fcSRSf=`C$hs75~z%c;tMweTG)kPZW_WywsD>u5{U4%*#w(tkbLLAd5OUP
z@DepaytHVhYz<X_wY^EcVRfcBmb^oH#~<Lq(A|1mWktPOKTQ7B(1NEYm2ak4643Cz
zK^{mwNeH&wdpR>eL-*YgIJ|NNmr`o_!&0(?by0C!ny*BAIr+&qJmd)#Wf><D*}}yB
zqjAaiGPoLeC6`osofAw|BHy@`!M$h7Z0Cf?+Ad%{mir+eUg>=TKR*qi$($<KdWhFv
zs73`jEM_t1^Bs3<qjbK@ihdms<x}}&mvcGS#f^OXCvam2h>Wz(`l>sh{Xe{aM!!xd
zE4BJu`^t$BEz@`NZrTSC`XdJ*!-XgD`P8S%AH(+~r$j4^Tb3rmWbp)ui)-0y_4r-n
z`jx+ZL`EREM`wz=;8-=yv<VY&Kgp|7$K*Z<(lj!x|8}Xj^7Yp?y~*JNo+@f8(<Zo;
z7dlD4@iGSJNPx>cj<*LrQZY^QbJ-NnS&Px>QsbqrzBqwXe!g(%Qkh^{ddJv<ZUMej
z?~ts&OL=z?O#Ofhz?*7dSR3D{b_ZgV$VSD>xZ6(+yc%1|kexX6`P@Ta`&iIbbV^F+
zpNvaGJ(uk;fYYhf%>d{joun9x(_4{W@wDRbhnYR*(ZIL0c&S^P!OA)@VS-Fjb+mtM
zfICGX@DKGeGwmk(_U(KqRF~Ws3}b+wQ3}7T6|LAqwBDL30*=oa<;dpfeP$QS+@0Bw
zy)I^z+-T~7r%|Sn{A9Y7bkK<g+85pgh=lscTCl<enh!rS4lBuRurKe6I#j0PcbBSm
z+@xrC3S)-UgC$0BLyh3~wcehtzVNwipo{6X*XGa-?SiO<$n}k3xDTX89us$sTZ!HC
zHf?U_XOa7;rATiF%G1wn1=zTg$U5IScCw~BP2@C0X&y){zZKOukf4!lkDn4h2js*O
zBnN#;rE+YG>NAwD>?)KMWTW?N3&o-Zj!-u~8YH-@07`l~-&~*DOS$%`)-bc4J*gpd
zdjdc~$npC4T~f`GWrfFEI3t7_+!@UopYR#o*6#vIzIG5dO91Pb+w;)mEFDIt*-&#-
z_tdX7pe2X2>A#+vpO$tN0Rov^%v?zI8-H}yjf_MCWZzmmr!L!lwoH$8O)MO);lj{8
zQmEzWatW{5L10`TI5(lN64~RvnwC^f;8|dIFzuqyFSwF_bu%Q%)n;3Y*;;p=k)62>
zey#}sdlNAWCCF-&N~3Q`%V)~QdiJ8WP03fbNi#eR4JOjjF7P4prK9Dk_onoRhWLpX
z$Jz4RP5}3_tL=4y1NcjrWh{o!8jg}$Ld<UqV|7dV(5DN<-z*5Y)z}_!_?JzkS!ih4
zAJX|#XE=iTH@C8JJAgBgkE);SS4o(b$+Wj<!Ix_X*%9q<qvq#b*8_9P{6G7;qlLn|
z9nFR(#B;GN0M3<h@)nDY+Z2ICiO<WCfU_-xpD$F}b<XG60-=7Wr=N)Zebm)uw(mGY
zv{<qsu~*IX&O3(@EA^{mG~zECkmjgTIg`kC;OAu2q><UUtH+%ajN!4bW!$7WI$xFv
z!tfAp$2s$q6myb_ISa2IpUB#EQZ_+>aS-#X4`YUyG>7VLh<tv-UcjMAp0-E~5TLBL
zt&I!SAf2I{qeZze>new(l6;rGV)(h^%dV_GJ)LvWX;J~Lb5Szc+E7?wsvY+A!9{D%
zwyZ&p5r3>P83!A#`2MWw###SLCONem0^M|9uPQLrx)QL$FX05?u2&sX@x8+%VTrsG
zsbJo#xkZAxUA9sXQbJ)(Qixt__bhh3@Em^qlL3Yu0@`q&G*mk$y{+K3zDK(zqr%=l
z<3xlj=_>h|tiWx#cV%MRY7x#TRO@5z#i@qX%BOqMNC0~X6{9jsHkjb-0kCaJCQyS3
z^QWJs*m{3yqIuv$1t_d0=}i3EU;Qi6=`Ab8bT|xVTZ^RIJhP-WTP>syvfM!64)DM{
zC*Ak7TZ%ml_-*48e0SP*7@uz(!iPio$kX|6>-Wbo-r>t9$eXCj8OF%w)$SE1E?Khg
zp@l2GeJ@((W)y2c#QA*G@SpDjK>Zp|>%3tZc+C6N_&&CP*MZr=Qip<zlB-5?^lnod
zRyy}EUXpGc3rz}DZ;iRLZx?GW4z6DIq_WtWvo|)Kk^j<S**0gqQwx79fH3{`J7(FT
z#w;ll!#KOoDn@_mn<Hi?7mB0%Xb@x9+6ih4*nq;Vh-0`Cc&U>nuffd*AQb@7rE<Fa
zgAaU^m@f4R%=%@!1diXH!dn~_DG40Zloe`XQt!%<?ZH0unWRW1^0`52cJyVhjA-yv
zA}7(%9+#=CRUem07GWZdLxvbuDI6~uU1LDcP*!(sVU7tZYlcVeMM9;`I8j{#sP0b_
zrz_3nveB+pap1cS%dLyxIE)Cxb<(KqSrJm<Z;?;?k11>_rWk2+(w&!}YQ?9+$HrHL
zV74PfjWPM;xOUNJ)U>1TwNVF_LN7XFwVrF9W$2|p3D`>tVQ5W<pH|XPjq7vFR6p;w
z6D)my*w3whTciKl_yZmi27Y=hSJnd|yIj<|_qWoS5qp4nM4QYJDm_a~#YS2jN+P%6
zT9fed4l{uA+|~i*<OTBaSFQO6sHZX3*By<;2i5RpLfuo`@nve)ZD%D9n~rBdqcgiR
zW5(0gp1T;;va+Z|8M3jcp!SCeD8h;~v}&=CVkWTjEbCc2Afv@FfQSA}3wia%Vx6uH
z@vKx3*miy9bezW7kh{Qoo6Fn!yQ@Bi=+R<n*mPB*8eUYfJ6HxbBotAM(oy(zWM%TA
zIAsI6qz2eRyCuhyEFDa4V5Mi%6e4drS`i9m5?8Lq#%>MU-{n~wvg`$4X!F@MPVGzO
z98OgbV|Z7Xxvt4o=I67oSumS05HnDL8R>K+;Z>K<z>}kY+u_y)4r8wwc2}ix<aAU6
zB@-P76Ot+2OqzStd`d^>1uC0rhorHQx+d%9<M?hnmCY?v<S$d#qR8k|7vBX18gWK|
zIBUq7-P^}c6h+nL)K4h-0S#?klK3u6NnAJckZc&eCdIT>tM&3_@4Azq-AS~H-phtK
zNz0Ev0kv#J&{*HbqCvZwHx1A5she2i6$A_&>mhL&q9Hzh>u9WL?{C8h>sH*=NfaHe
z?X7fQMxZC6o%eEbkkDA#meU|<+~u-fKG}66guZ7slyM{vPUzcJ1XOki1iHBCJx{O9
zG#rPMD#o@h57KztY;VQ5gzavPr_Ps3_^dmCSpU5*1ULd~1(}_7x20`~`54Gm$gm65
zdzjH^nQ7nM?$OpkvWP1V-M#GFxNG3&JyEkB4_8*qQV*V<*cv(^RM<8o$rLK0i8eAG
ziqcsu;g2>YOk+qj>f=#?5+0(&)ldl)Vl&n$O<XN7bLoJ}$#LJE+L!l&wr@OW!QTjh
z$x2TbTw{}={Q_bS-}n7R_)fB1J0s$PP0DI5i327~riC|ijZbJb-mKpIu7=gNzM5Q(
z`1$@m49h;?hi*$xlCKA~KgNNb&7O!DJj7l}P6`%HTy?M3BowI@VgG<wk{KJjIDzh=
zE_a>vC!pTwP#Iz@d^t;6AoelN4cF9+cEjZd0Fc0>4n9w}MStnOHccRNha=CP=H+&G
zoT5{FpVcOF3{o`JV!K*Zye2fLJH0N^@2e9?EpwGw*xL_k{dG_nn{WcevX>89W+R!@
zYZH$~ktAZ^dblfcVFo3DlDt#PqXUs`=q>YRMQ_G%esI%sQpsk$LA9Onf<>cJb&LGQ
zjfq|14)pg<1wjmhObbM=zHc}Vio3K@EK2?IfDj{0K%}Crk<5@pqwd<L&y|LUZs@4T
zGSW7}fK$3kKJriywf=5&V(#u1U)SNqi~VY2K`scs^~{j9eE!^DK%Zn}4YH_59^N9T
zhd#dnTO~kfkj@Pvqqmy~@V;}L2;9w@I$^apEg!%x#J|<4?XfjwpQSi=s9DntjKWOi
zmoQ9Xqz%Eo(58ownZ%~lcVobIbhF71`m#vZyt3s&P6}?<F5K%%tpg^yg2|2Dd$3MB
zvP?%|y#{I9E44N9!JT-2>X?^XF?PF#qV0hWO&^BDPmUl;#R|dJNR=Riuk=Zi%x{L<
zB`%?w`fLe%<CtnjEyYTx5q#xN)}mKn!yC>X$L*Ya(~H)I<iY6(mUAduSul-L4W9jb
z>$M{Al?N8pIR<V#(jip>5L9{dSnWy!TwdxBoyP8N96|sQ{)R{LV!%O9)>^%Y)N&_v
zh%%gpOM*qmuv1aS*O94-j&RrqlEUw|e~{+>*LL{HY_N}11bH5xtIv^SeO3^4pZfLB
zFmJAc*{z~5@z6)^QC))2nakwhi>F)Ui5$>bofQ;=*pbkbWF|j{<TNG*S)G0dzDSz-
zbe_W!9=tiObb7nb_-Tt_{@YP2>Ok`WxgO$h4GY0nS%L0@3@lV~*Dj+aw9YL=ENT>1
zE%e~~F_%4}H|$G>Qi{OWO=wHm`4{6Ii&!xroTaQiGGEjrxQ~8t1mWfc1)s5z^PZ&F
zEq|e3+dc(Ir)!%HB#NH>=m1Muqj|IQ-H9SGo;!`{LU$8O=}+z{8(AbN`J<mFPoSeb
z)?Q_leFBJ>S#0Atp6)<KhY^|;b=X~Mr)nhq&hf{$Iu`W);e(5^j4O_$XhbV?Q7T>M
zerg#A?&JoD#Bh2Lswj=^PgkS##SF+r>GTL+d7^0Wl%GrvF$yvCV#^pHCzV*T$<iSx
zv`X5F@${-rrF>)iggT994|eXvaip<F<)v|n=$y$6@(aNVN%yC$7!C$DGDL>}O>Op-
zf;s2?Si&xFzQNmcTc+W&=#%Ia7XXdm46{Rz6p1Cq<7DiP6gZk#m@a8q3-!|<%Qbbo
zKqXH-ZVk+8d#DtaA-S+TTIQ$~S*COD&JhH5lYwWa$ue}(<*XTUVc}Q5Up71Rj4*rc
z=j_ecP8}2qKxF=9X#@jR>MPS}tu^HXl_$H(%QAMZCkOz>(B45g6kIxNaW}eGn64{1
z{ZY24p;_wxYVSRuqFTN^VQEppL>2`J5=8_=lH^til9PaB5tSr4v}9UDP?Cs<fCNDV
zM3Ur;WF$5@HBF|;Ifwr0fO7rMn)S`hn^`mO&Ahu@cj@Umb?WTe;kQHWBlH7;lZpUN
zQp{z)u-!2y8}aoX)5x)<@P|8X+%(^~9P%PdxC`^*%9?uB%`=u?FQzEJ&LCS+v-t4L
z;3kGSjsClK#(;nFS!3r7vKhIFk}s}ZFSPFc{OV29;L=c0AoEUJ(e}|VtJRUhU%rw)
zPva$to9%_Znrl>A59ZLA-nKm<U%6=1!pmp|f~~jn&U|D~um46f)<OG{kXnB3mLu)y
zw$a2>b^;c4gZS=bOCODGy=zYdv5Gz!z2TBZk09|v-~N8AN6z|(F=1O{HnZ}HYkUT5
zu5yn}ZTBO>HNGvod`JxkmMc5zV-hII>z2PC!sJgt9)0X|LP=T4qIwb4*5~n2&f_OH
zgYDAT@Uk)_%uj_rmW_#I3YS%L6(e}qR~5|oygm=@NG-%WB`DAHJk66WeBm?vl)Uc8
z$DQYGs|c!JVzgHuiv1jCJwt7@iQhtcgZZnCF-<zZT+&l#xk_{IAORMOTOLjFI#CRg
z^hkVCH&tnvlME9SyS{n_mcJOWUJQbxW4jX-q-IvcD(o+1ym@)<1%ZY@V?m(qc*`i|
z9TYH+CBX0Tuj_Tp*U6}CyzFX2<_r#1zoD0X^zAz_3q@OF)me5z2AJHvfG&C{F@pTT
zP`+er9OdxAybUy9s65HH(Pu_!pn^PYsCuR8d(!8W8kSEW^!^g@whB6FfV9T9wuV|(
zpdtl{g1u@4;f%Wbw(?QO$pT%3r<3A6ky#GO^!1;^6WVy|*+9rUHbm%MFIWA$Yah_x
z>&@PMNxlf9DxqlaG&2b&yV`AB0P)rE#WPM8-16?ya(|z3Kvsb+L{a&i!SKIR2K7*G
zXUmO=@bByZsWiwVj1)@1{!Xnh#ej8>Z3HtNp2$5+1Kqm}k_)eANC}z#$|JmW1v!`y
zjPyI@zjHV?AYa6qCdc&lpCpDad>JIH6y51${*NtmJY8x7<b$p)(Y=)4e-Qjq3eG6Y
zcpLu(acc<aVZ){OyuVxQ96NpOpu>)**-FIAP8N3O7h^o~KVEvziUlOyX$#cXTrP``
z%i|7b#>&V%EqkAKe|-CTn0e7V@wP`-40yKY{3R={kiDu^RTMitHIQv2_C`J2iTTFk
zs;c4-bP0>iG!YW@X~hi0jL6JNpPC2S#gcpr`L~NteB%XdBFm_}uSgTx6>s);o2!F!
zHk`*EtN-1vO4$++I!q_%o9N&Bd;%1(6oMqy-(8*H1tdxDIbLP1_RofAy&~Fqx;bgU
zXWC`Iw$^2q_e(aDJmt={59PC3JU$+L&o(t<1hmq6r_EElEyvZbUGS#j{JrJO5Vy2)
zx@;r}aEcUmf9kc<eHDFb)@J=2Wv8wg8)dvF0Y2;E{0x`g^Jx{kiAF~)=Xr;r-mt+~
zl5x+g^f0N1jXgGb=<bb>ma<P`b~T0O_9A=vjG9_bl0Tt#Q^+`ws-#Ieqwh3PEf*wi
zTu;S!UT_WTQ1o5=Dd@o9g4wGu+Dnv=a79<<ow#=AwM6;dp-^OtuJeSApHxR?Vrr3L
z7`KPN^CCD1E3$`uFXLf==w^(5dHM}JfoyZCJJ^NJX=IE|Ee~V1BIIC3ba8jue4!ZE
zqbGnEd{`nY9ls93?&Oj1gkpE*Uj?X@iKc!FblND6%BQASa>F9EK`Ll>FE-UNM$oN>
z(Mr7Eu&Fa{H9m?uYAh_W%2vXOT;laFbKba5)Fmbo7CG~qnMSQ2^RRt;7!p3SvD-z@
z`0Tc`rB9Z&8?N!i^Y%6eC@%ZHX)DZd;&ZMf^<ysy4sy2yO^R4!JeFqfA`sdxD7z)J
zZ%pBw9d<e;n{Hs?vu|8q0m@mG;nDR6>$Ywm8glebP+t_(B!{%X_h?){SLv(sCF@c%
zf;15#seL(v<FTTIA~u?sX+P!DB#Xq)cRDFipS$O?gWEqB^nOz)(pwjLTTVtaI4+YC
zOK_hTPad`4dh2l>t?<Y5=lnW_vllYaT`?YO=Wy4k0;srP3~!GCc~4n*iO_fUlIUN^
z3=x=0TyR~L#oexQ-J7BueC{m?PO2;v=bDT?!`PvC8{NeA7Na!vu`T1>F&-lf6vW8W
ztcyQcV{XSvcc%taf+BIt;+az8QzaGS-IzV)Ve1apLpHFoU(>r`xoW^<);)}&;Q+mm
z$~GeUcLoP)KOC<KxTNDoB6|f!3mdKRpZJFUMRVN}<y$x&QHsE<F8ERj*Mw-?CP^}Z
zdF<z4&*Y9Tb8HG*?!FvtDP1hN5@Xr>WE3H`{n5>bB0wsD5|8BGF+45kt!z9vRpTRh
zt$rC))GoNz-4#Grj?cA-KFD(~5zKm;h054geSZgkpYg_gx;P=fX`g>tQb#Ra>&oXM
z%V$4ySNiE_mGj)a1hb44!bPUavbPQJuSJyS6Ox&Fv_GLL+xz)8713{nr8L0Uj#uXy
z*s%qV)^HUj1VLJoJQ%3l>0zZPkx+?qRD8}0<9X)?Z1cFp_lsBfBSn|j?V|Y6)>GPY
zcIMmfO*_Nl$Qc&tkjZKnnu@4+&2|T=x#;p&J{Y*K+e|j)85$cV1_<amPvHWRlGXO*
zQgD(W_swh8<GF6&(vRM`?C_}lNtoN(l=M0}b@IJw`|D>NCMb@B{)n-@_+uR^qPt%?
zK16a`Y;#cR?et}Fbtb8dZk|yR$Br>R@G$0Z|Ap~ey)sy%>c$3k0Y_re_r2PGs@<|+
zyP{j))e+@8?SQ&19kaXO8?h>o{G7VM!*s0-@E7t!Q)K(Tbj&5|a@rz&C--!r9CJHC
z$6fsWzSK8g<7*D24<dw1Z0|CMJM|K~ko}5AgMs_Pxy!wVPAp2Bks<G>_$}`jSw>CE
zshlR+41#A};PBXaf)56c>*%?!xPgHP7M!Rd%GazT-!ZBXm(Yo0P)6ZvnDs}jFXZhs
z3+THekN@nYdD*S)rVgnri2UBrQ!@pua!e2nR8d`}OZ7$69PK?%Hl49yFEt-z2KcGK
zfvJGIu=G;ReD@f2Q{p!(_HS5T%TOXrZ*^pb=DB|hnaV#mV77JPmXuNye~F>jcI!p)
zr3LqvlDbTU^Pa4C&^bN>{C9A8DXMd$;)7`6JCo`6Y+$=rztVPC9SajB>OoT{XAs<%
zd)P4N{PrZ#)X8(yPGm&RxpZNo8zSWgzuiPe@R>*33<O{PmD%u9T2yeT+9&$?)4aQZ
z(IQWNc^iMaU;m)P5eU&sP+y_vCjBh8Qp2JzZ)a~3GhZ;R5#|iXBo0#VKHYnDu5hj-
z%kMt)UOmquY5KnR>~92%W9-`1Irlu>-7qdj)_W(<70ngqL4zcQJVg;DGu!OXFKyOO
z>e<eRc4wa3crBJ^QB?hv;h+nmBx`IK^3?Q;Ak6mA>|MoZQM<0`nbd+lA#?xDi5;s~
z?yu!m`H0XJ$u!6}b)W75@ZZSt8Zm1>d6nyUqB>pK#zZgc<(n>Djh)@vRylm#mzK~V
zx4=yvhBcqZHYC%KM5cxzUGTI$ro3D9S-c6Ux3FBt5pA}cdT61XneLRyM+_%dZ!Y>C
zJM*bTH)h<{LKEpDeZCVE9GTfGFEf_z4M((d?(e4zuS_<js3)$2jTTxO(WenkrQr3L
zuvd-e$&wkNaam;ehT?rzwSm#M`)P{t*cp!_Q7>wyGDiq4&B_U-q1l`x6Qo=&)htU+
zaVQUa;`_V$o7tB;d$ER4(R@-TRsArNr3)OHijVc2cdw7-RNoLD!<F?L2B<H1<X{E_
z?P~q0&}P+L&u8~=qTUkpBFOV8sWv#C^yVlY<3_AXYTx%H5(?VSw!bSt;M|1wD9=jA
zY@}xIEu;<&+g{5d_`;ZBpQ>%4xxJA(G&Rqz_y>K_e+K7ZZ+xa<%>1XM29Ezu9+>TS
zF48~#1>`|>Auk>Mt4RN<Ah~xFfIOLbntGPML7p;D`ec3Fq;h{A1Gs^{9s+p<2&Txt
zLEaq*<eAjoyZ(3K2oqF)G&UIg>~EAp2ufXq{%00?2-Skq|5We5bDR_^%t9zcZ!ceB
z(W<!P7$bH6im7xu`xZt8)&H@PboC2UmVvD1w?n&GPiG41+5jM?;rK5AaqQC(re8tu
zyDVp0lzwu*?<+8^y(p_f;(Y+%Dl{&}<pKcr@?Po}hPO&EBOjtB<E7wVmV-84_qIZ{
zfEzJFSVyI|bQgZ??=7f+fK7DxpFKTLI`>5Yu_?sy@sFyXAi6>qhr`)?{w8ZR*#Ua9
zUyX^EBEs#aU_7=`)%&|<EPve28jq}1{eC+60kC!4W&pb3#Fi&DQu`q&^prze<N9en
z-QU|L8whqBuE#&$QpPE=-N@zRI-%PAO>HNCMG3vp<E7^hPTGDd&*zSZuoIWhc^fUW
z&NRD$smTSG9ve<6h)tx2Hc&>E`B^Pt7*|A>Os(TWVW^+x!TR6a15CEIXx#MCsmIYp
z>9X)=V7MFx9kZT9@;l)=^9<(1u@h&~O#2P1Ir}a3f7|b@r_HyX1<kPp()By7SO{5V
z&izpM;DSrl)OY)gYtqzfbsSeOe3qHYn)-r4c?!QqX?JJBg%-Or#)I4CA-WKfD4%T4
zU{(ey=^lZ~v$Rt7>5dea5lQ35Nb$1C{UV*|xeYAnp&0N}>746q;Hd;DO(kWv3#`hC
zmL+5@h0h0zoq%E;!jN>@fsz`t^R$a&MjL2OK4%;4!fz%po~O;zZ5A7*?THJ6Mzx@w
z<~NKT*L~jee4tkky&({QQ`2uPcV<WAx*hT?lThMBI22{N90|dl*#5Mg&36uSnbc;h
zE#gcYqPNA6pn1v(dgXM>D78Q_`xFJC^iG?cnFXwxC)FWT$A|p0TPfA5U1@qQndW-F
za9$Xd3Tl%L(Mx)3s_t$>u7%l0@S?BDT9F7r$Du_-gAlv%z?89%2E7swZD5sUR8m#H
zQl^_0{p?VozuAW=6zsXK^V7cf$x7&*i-H?Mk8rxn>_G(A6(9C>uqlb!a@n}h*pH1M
z?2qaT-$L?!7tsj78K>}4@sd%EW#Qfj9sBOjr$6Ie^9m}DBXQ^=9K~(hwWsD`;>-~k
zJ-wsaLMEtM&3f;L^>9UsVhOh7r&X?g!5lL}PY1Lu<t$V6D<8qX5RkN@<hBs-6R@dk
z{W*lW1V^B>=5OurRu??Ia;@pyr0&wlXiM(1q6(T8(Vke9O6%}YZG2UN0qdz=F0qvd
z4P6!s&o>KziA)&KeECOQ;CBeIUjk!kygC<OWpORw;AP|e*@Z^sUQOD$1QBXe67)1k
zNyh26tZni2@@4kyg3;i~NDj<3y=&D++K~xTjc^*T8as42FubiHshVc&G(yBTnSaSX
z1UNIl+fGTOqi{|ZA#xKgWUrxJsD;9xcO44G>|Z?hf~!=^{@XSW*V+B${sn@|2rP20
z@4GU}qytpbAs+!fpyx&uyQJGc(}w33Gt!Ach-@ggUuaN|k!F*i=YbX%$8&OZrr$B)
zJhoLK()0-<&vkV2a~N0Yc8#%t+vp9~#d_s^n|haX2~xt>p;E-(=!M7)K>&g%HU3m3
zh)K)Rvva~}eUk~2KA0&+7#`u-J*=L+32D%lQO97v)oeqjduO0q^yOK<js$}H*O6h{
zqW3c`G%G=HfE23iT%)OSdi=p;{!_v3%9D?Lb|1K;!a(Dn@)uhx96#fhcoBhIpsOQ~
z8+YMRkl-FYP+gsfUo|>bLXRsLVno~|=r6#itxB8Tl8_m($$I}bXkfv9p683%FF17`
zwwgRF9#yV@yjkl>8T-GD$0HER(7V3;?eE&r*q6Y}a1J+zVE?u#)eOMzeK4l}FK#m-
zMDL0yR@C1?k2lcyo6Nct{l6bX3VM7Oz7$r9gDS87`~eFEwU@z~C4WDN88WAYh-eAm
zI{Yq?1(Bu;;N`0s(w+Z)knBfrJL$a0>pu=OG>!6gnEDw?PJdOzpGLm_mM{aCx&5-R
zHtCVvfh98q!kemO-qRsp$zP0)d0})l$F4n)`+Xe4X}O1WzOTJgZ%9}~3G)&LYs*N|
z$KcnTr;BJAVGKnVkDd%(X3qkurJC$5U*U+4=dmLKfWN$i^pXhA_|dC^za%{eLdxz=
zK=6l9&<$|tVkM8-`G2TFjC{kY%%*de(Qy?~S$b(_(jeLUI;L>^o4L8y!)q6-@82CN
zO7{t%;Su6PEd$_FSksrh^yaAZ^gSv-RHXA4j3`mra-(0e3_{SP#m=hmI-fqsr$)ic
zeCFMs4DlqJI`;0w+NBo@Lf4h+@uTT>`(nE>yCabcX5HF_4JV_Tg{FE~8hFzl)0-86
zmEM8cM4(6j;Er$lWl?geFFo_yhN>Y6fn@BY{;6EkydMH`JDpvN`qw~v3G3+-VWLa3
z<?HOZGsXLbWxMM;clV;5b}Fw}iFN~h-uL1F{Z>PW1|J9N-13*YB45?T0)P+-@3t&@
zR=mZp6@vXra9;@7Jd>X1Slf9X<6(8&rYk{n!|jJh+2qN>kFuw}90VVN0LP4EYBn7*
zTqj8nA}@N72^3bnqS~9=5O-#zfGU2Gam1}MG?G8lWDnWR_R0;9eb|rA6di%E#$N}^
z$s~OXmeU+lMy-;lkFIIpZ2#ia?tE_Ar!E;REU>(Vu{2#R-F@`Q2aC3Bt&)@H+L={8
z5%_{u<2~keYZvtRwXbU&^cz=(TBF=mE?bM6(VccEdX~{0ykk|=5C|ax0=c2C6fS$M
zE_6nN{wbOqi#$DD03+yOVlX<7_3FJK(VV628Ar4&4~FBwM!<)G=$@}~ZJ6)RH7kP2
zOg8TskMt~6ie28WmiF5ZpyRQ`7!jSn|C02zr%(9@y`nrbbf7P2*(`rq+_Hz2&UI-@
zdY9T`kYb0{@e}Lk7vNDi(oM@MrBgA_w`~<ZfZo04_>vIdc?XX*8?NwH8B6;1jlrer
zP8@-<g?@;2-OJ(5uV<ql{(NzQXu!JLts8xNI32faBXKIIS%I>ooO$bP$aTZTQorbB
z5Vd*qXf;lk5X6?|x-p1;2k;l;G70-d?7(M2=9IFt>1$yo`9C!>JcH?kT}`g@?QgE6
z7tuMNl@#sHQ}PMzt5$uw5=gBaSR-})#u?>%1!4^#R1Q1)+E*PMn$|gMH{}f)8Z6gk
zDVk(IGb;5c+kX6jQ}SvM!E=J^R4S}7Xp`_KlmSVGG0yW>ufiBIs))q*e%$j(`!<am
zLfcHWR$PNfjRF*HjvBtHK-s(dwCQ8G3&}4w5E*IU)e20};`!<_LQam9WnJ9!jR8Hm
zJCc@mE8R3K%2X3Zm3IL1ypsUlsNj0d%d(oF?J3Qh=~*_3m;IFaLA2;2h;yiv`O`m&
zIw#~3M?sE!!*auX{h2-XbqI^qkKygSRj1a5kyf{j+3fX*!HeDyE5?=sB|Zt;O$ULC
zp*o&LW3=VQyg~%m5Bh0Fm<qw}+9Vz2C)=txlAQK=g%9Sm30KD{khTmNCYl>x>a($x
zO5!??Y!HbS{QVMU_JV6Upixl*1T16qg)L%?sCX?Olo@($txN>$&k-q3(`P77u5snB
z{eo|W-owAZYTZZTB4E;y@9y;qY}G>9xt-NYNk%iTW=dVV{t-e4W*CJ601_u{EL6Xn
z1e$@vm~eMU6I8A3aFs>z4qM4-+a)Jx=$BL?S!puSmgr}Z#<2{b?~Dk3DQ==c#zQH?
z7t9yhNhC5oHs?J1m1G?GCH4>0g1-I$N4I-m*~w}ZK5|<~?Y&wHvuc0S#EJaD7EJJ%
zz7^Nh!lHOlz-T*rdySg2><)E6(%Dbse+u*KAqAq0=}0vQ@<6`377WhEDCg<B?Vq@p
z#B`_*!2b!EV~;P8{(E!AfSUea?=>cA=UW?+Z-Krmb+nmpufpy}b~AnF*Fupo#TdS0
zzW#W_RD$B2<7(kM91TTDK>hXGO0qY166_PM{Z5VqUpTY1oF@g^uZV8B^~OsFjh$ip
z7hmr(68v2B_1aq=<b$Wr1iL|<pOg`>lrm(E_3Z_B$ttu(Z_rbfSUx1b^>D2AJe~HC
zvzKN!FqceSA*7!k*r*+P`q$=72vuFKk2PSZXz&yM^~(+LtB~>>R(<rBO~fPl@BYRJ
zGUCo5Vkp~RtqiIxqutx7+dfB?YlgOrSIE;gn@aFw_Mblbf(ZcxY_96cUWY9af8=-g
z<v<y=)E7HhT+PuPH@>BbZUa76F?3+fo9y{s)W(8aHpbULsyUn|@rmC=q-F8XB_EoO
z?^AK3CaVYG_z~#jithfmk~@UDHq9~%#r!6nf#wVO4f-kRXw$J4@zRxPKf02g8Yfn|
zL90cdChhF+%A97o1Yd|Zt+A89ed=>=Z$5K@b9b@5W!`xarHOG~=uD(bGf3B$)s{Q}
z4Yvug1K1ZMoTy)q1a&ec`lGMqqI_*FS3&UH*)o@9Tx7rP6nm~g=eLid<`bMcX*^Hb
zFku7M0)-A{B|nYqEwjTyrP9Nsu2f2it=q+K2u@e!iC=6>+~_Q_@8^7EXK!gQ;d!E2
zKdr^KAUCDFlF?`S(Uo+LSAuUfFOc|3JZ!5X9JBhNYnOP}POC@}9V+}?@u6vfv~*lb
z;}gY)dGo@p6;nJ<-#ZC+iU6~#3guSFon>x*pafBy^w-p@AZUD!cJYh(rT%l`xG^v7
z95Y1kkCcOeS-JF(pVcd}c87*0*QEoPj09l-EBoQ_lp0yq@;hzOr!L!QE;Mk7u54PB
z?e1ofHE%AaYFd>}!}2e?KS~#J7+=zwVw9NMAh{@DII>75B_Aoa7eyrQKH>dSK+oN9
zWjx|fx6#u|VCTWaQX~&X-Wn;5KPz&vESwIPUT|4oIt5OYl!qfge7$F)fBvcv&>~{6
zyRXg*Pp{ftx9wbK=X<7%34@10I!2isG#9#h+EM))Ozv6F6ThxG4H&)<uLmw%7y%+;
zjWOZ|CcwD5;#&`I&aP2nT!yE7EQ<_#YXX99t<CCei(uvA<se?aDuNf)q4InpLDT4Z
zk(OL~?<FRFDiy`)*dYM}+ULWj(oHb}RxQVa1{^SVquujc$`k4I@~wv_`W=hbn|;D`
zZ?^4fusS?1QjG7Gi+^j#NVg+H16=-exjZzBV)Sl%tyOoD3Ztx_y{a_1vv-Tm;r=c%
zGCd2(d>gcCPNVTC0pN4b@=t<UT;e+t_E<R=O#94;d4GD}(g3?v>2P$ddB^vyL255l
zqNEJ)=gE)>AHMbGI;4PZ-upaO`d@k9KmZ?E0mV@-Hh+`E-;XJ;T`1!-pYV4OnEg@N
z>{8gHkT>WaRzftgyeDAb7z{aMn*{!s*J>vXJc<S*Xw196jTa>x666M+1^4wE*6OS<
zIafh@0`d<QeT}tok_}x|jZr+cP~0HtF^ouT$=(Yb@aanCap=pYbKf6-y?^{OYUhU}
zYd`h~W)|k}s=F|gK1i-K<J}qE7~FNJ+F;-3Y8+oRRWrD>$&1Vpwz&UVx#KBG9en6(
z(E^>YJBr$G^Ll%dN?w{yl;bRkg?3K%#XU%IAQ?zoFH|yAaDn5UHkDlAT^g)ODBmB|
z#2Bp6`rHzDuMS<aWKJD>i-J3m^&u?otIacNiSji{+tI8iJIz}&gxG$dyD!}3q6yIF
z3b^wsz;%`56_9)?aT^BVGdjHHZ?uGJSimVqe*l5*|0@JCv#-)m@}12*oAcw!6W+Ka
zRP4u_Qwu&j12<z0FqhgBFNMDfpt5^hyYCaz`n;EgL`x=M9oB(eYrFcmr%qZkeuXuw
z1G}5qBwc@)fL-Q3Il{qnVOcePbngA~b&e08HTX^Zr6l<JP#<}1Dyj%aBX})x<wJS~
ze}Ntv!_2X$nFN)(j9_rS_1?s|p621|#)4`d)HMhkga#YJkM-I4pSBkaQSAqO5B&jw
zANwJ_3L%LRNPqqL^Bx9lU_z_hJC(nKq)?2A_4{3>z<&n%!r35(I$gqa>2Ft&6nu}!
z{@>rEhc;L+<&O=%YK2FT6jt&!;FSPNxS?oh7Z+7YHC2qW>Kn;Y-5HMGme&T9C1`=V
z0Mvh37huMs$N&lm65x<S`kErk&WGi7sKN_%Dku}ql3U*-Z29&<FB25zk|74ZC{v-#
zoqut0A%Fbwn$MRb7ZFl5guuqi#YHk5Gz<N`Bk&*4NmLDwdw+j)3Hgpbmg5yON`dr;
z2%$gK*2;!~2lw_yFnhM1X+Vx}&NFnbNPn`%??4b-5Hf#(CExg&qs#qG)vSuvNAQ%y
zW|>k|;%iWe(TG+QD_6mnF5k-(CnSb&gg`msdU#rJ<^wN*t5`C%d<KVmb>~2<GJzHP
z;p=y5@FQ0g#db$GFxo~@Yea(c*&s%{zd5IwXW%iVSHhKE(4h*9!ks5sDl?MwqDYr*
zF|oz*?g9bxuGv*K@`G@RH#2>arlC^`KgokfuQd&@LFdth=wArY^@<Dqjye_r4vvF6
zvHd$@ry`}d2X2~AoV^Vq{EH+P1*e0|t%_W_E&BzYDdRl2TGr6r2@qeBBLPtv;F_Tn
zipOZEV8U~mxV9WKLGl!PUGu0lf#}_hDue;uEJ)tsk=%e8>$*y!g&fjEaPoKuAYlwT
z$NQ=DEmP>7hX4|^bzI=kT;#h*NA#%22I~hN0rFzCJL0(g+Pmo@Wtt%Km~2gMRtmN_
z4%(voh<hL3&}ZnbjGIn=M8&(N>V2)Z#rfgTzcu1$jZ*$*VPq~y*`r-%AJktyNN(YS
zt%)jE488}9bV&gJXYV~amxmSkWx8s{@-gcTz6%APcx{Rg^H!hO({%E!U(axgX<j`L
zU{icxHQJmdmfwpbMc2i5w^-?<Ob65ZS*WdY9YV$<aNN_~Ew75jUAp9{T7J@u2f;bW
zkq_}tmh5-Mb`Q?F^`2-Du)1*FM-^fe_%L`2&dcVVOfV@=wYhn>>G2|GgQaG3FMSWw
zW5Esl7F{{DmVD+NLrkXn)eOpjABaElAm#<b^!#8Y@}pAN_L{L6j|qivp@0VFly8L=
zMWOJ6iAf0Wb3D$uUXeKM!Ac}~o$1crFKq(Y^b@Wp@!XHAB&;4c4^Op!jZ+6_YJEsK
zryOP*TEyMZ=pftE`pqZaWdhmS?Nl>8%jX>LpK#SFV7F)Zt{vy&kuUW$0&@{;E=>`9
z(jI(F?P&TXAa?MZqHMPxFkewP)PYR?;j=e^liG!yee0l!zbqFEd@7Bfv%Og$p^t`8
zj;?vhu10d3WUK@MuVv3rD&C%J#e6nym_vtm#%`2PNVHEHL=5R+-XMHbxY(#3Pw?f6
z^<K5H;$`u5tXhjj-*?G~{x_t^UyRBiJ|CEpy^&KS6~H{xkrqX@AoumqW*!4d^@Qj5
zYZCN~NGD+aXWMb+3yS6AR~uZB?e&C&k6Dp$&kVvlob+<7o>K1>L{IfH(i<K_B7fe3
zG58BT?`*hANy2sMXTfab@vDBsS|@JjX1vt%A|`$;&UEDpq09{?w=+9p!%R%9dgxAd
z1NFU$;qXL7gd1kVy)3V9B@x@cS1vjEX=}5R=)1t9cpGUqBkx4#7sv2m_yoiRGI0O%
z$<$_S0p&8at`6HeQK-2|EYE_ORg-^d$D)BENJ?%{eK=VuMOa1rak~}G-RyV`+^Nk*
zFO{d-J%F!4zc~9oF}}rzwsi2Mu)AZogY>i2>XO>7(hBPni7p?1KSmKeCXM;@HLvn2
zml)1U{<+|TAIKk+=Uu#zUyv*1c{q^1^%@>`q56*rX87+bS2H{>>J3fwQ5~$89_|TG
z!LPz<)JKH<z5^ax4tr#ej5Id~?*o1Zms3fJ<8_$L^`SR}t{`D&d{HMX(^5$ef5#`t
zmWBIUhJJS<{Cx-h>Fe}Tl>hS!2{p^<Yh1q;D$843FB8F5-b0RWU%+mw5I@6cNb92B
zwLLQ3G`+nlTQ6IhG;?7sN4LYx+k=LMVZqXVGeG0Y(K}khEPkqgp@%sUKewLtf7o!j
zWa{lhbF~M(_(F;D=N%g=r2<dq@_UoezdeJ8h_^$rNRs|}wU^BFQa+tToqyhpVC)Xu
zzgaSpT;>mri9dK=q|p$+fA`M?Fp0;bkAD1YKj>jxC5r=UL1Mg*n+#+2cWKejruMwg
zEEJ4R6xcmoSS>*Nq~$mY7(2f?q<$TsW3%jF;W+8@l{HPSkBenirTgEi{U|*i?I@*Q
zm6C256kF<BVg0n<*T5wS$2m<oJhCs~Brn?YEoau$x0_SQ<6K#+9}T+1<LvW<c9juO
zK=lG~yxPe;-6{ORXzsapb}fbTzx?7|8qrq!gThvO=~;>{!v)5L;!RojvdoLiGwI^H
zJ!COk9Z9*lL>`7YYqMI{@FSVw^zuj|oF-f2_I*kE1y~}j$7(#$V1GK!(_?!AG0_<9
zu;oOiu~m!k;LJ*Ar43)5Ga`CTdpT#+xHe|lt8$~4S*W>Tqnsm)Qk?Xj?oeEohmbWK
zSCz;IZBhgYaeO3-=R^6g6YJt*=8+be<HNOX=0mYRQ9D8!yhCL1QVYuoB^y-2ma89$
zVA(jvLw1Hy62~96db@1PmC-=C@n&CZia(CJMLgOXC97lyJKTNHDflAIJ;wL9q1|C8
zzcuk?VSS++(RAgm<K>(=O0lw)Zz=2A!^zQL!{OS1^|jl&u0Tw_31jS{*-xI!3m5R%
ze{Y=Vs0fZ~rl$=)_aZ4PnJqRnKOZ0-P69;Yo+{j5QMFwcl8(3E%ZE?V5bvo)uH*;|
z7LR>1XZhq&k5CjIjmv7e=XfOxhzxz2rxmw9`-CebPjylc(GFevgM_0#XK8t-f@^;y
zmwyA-u?$0&_LOIRph<~~;X$AUti#sXsi7HldOVmn1#osVQggV$4CG1=M=TJ)4IEvG
z8O(gHlf?+bw`B!4&uf2u2!4NmZ>U>fH|>XP0%!@$W84(MyR|f9S?G|Zz`y;ozBbIa
zT!WC9KpiNQ9G+<g=*87F3PvI@Vyjp(zSr1xLB(6^bw)%xW;pk~9v}qtc1YPAZHuqI
zePI*=f-*cOE=Ioa<U~lOnTp(Ukm^2FBv?A!9Tv0U;jSudxL=HzED_F2)l5fvbtQ=|
z`^lyEo3uBX*Pe1WTpbY1GOy+N>L7LM_{nqgo#vg`hqY0NosAgzB+;BUoVauQ6_lWU
zDBo4h1zE0-7owCy7Czl9sNbS2-WbfDvT8BNco=19&Jr2z6L6WLb0W0uv6m{fen+>h
z%0_6PH)`?D>$)OpbVeBxf11=c&W?kOOK=AWUzo`9ew?HnZ4j+4X}md6b2ML;<BiEE
zEjdLyc<g<coAZXg2iie4=Ly90T(Ehm$pkg6uJF3D%qIm%iS9Jh4i5n#6zG21h{mSL
zrwyi2nbLNTMV3*c3nS10mQfb$PdTT}3zBxss`=i|etKQE^Cj?c<Rktg?Ws!Oi%Kxb
zQaDmEOziYhtTU$nAx0=CYos(aK1jYFH>{x8CXZM1l|HsXFW8()kB)GqC*?;MrKXie
zccP}O#<HYdUQ+sa+&ug4%ncGJM0}-H_UTGe6DupCl-P0gb`<D|>LdEwQZcarj(zn0
zMD*2Q4<q)Bpj7YcL(@&)g!#6b*+R0eJxrK6Pc3*&sMaG-i|sx=EBS${KVjQzuO`K?
zJTHY{oMrn%Up!a?PzzxTBP#z~K?9Pbho1k|vOv<ymrqn-$BUkWQ~19}!3BWq3*3r^
zMBwmRhAT36TzvL$8e)I=suLArzz$auLhC-hCh)ZJlT)jh!T$8V%PF;}I|Rg6lYwPz
ztLf(lcP}$6wy~$vIu8)Glq^2NbZTO(tDNt4KZw_iG|+dNiWVXy*DA-6QWO-e0|Fq6
z&oq8oYt|nP)YeUncBwJlad7>-u9EJAdB06aeDxKW*I%sf1eo^?+l}EDa$lRC@4RYg
z-WhZ3C%5x)l=OsKp4feDXsMHE4PNFwe5mhrp8ue`&ZR>9)Lco}7MZnxkcX%mH<&<{
zVKGnBVCECQTNt<S-fkJE>msTaGy5sGxzBuVIXo&*WQ<+ZW&Jwg3ky<u;_seCsGqq~
zhCm5jErL(#v@^V5PcK?}W1eP!n9Hf1&A+>PGDuAc*Be|Lqv}Yt!wRQA2JGi?lNY2A
zQh4YDTbbbUrKLrbi;3_3M7XSO4wbfyRQrKXF;;LO2KN??l%6NnE;ouudBUB6q$L_#
zFhkqn%Y4H=;3sNpZ+kuU2^M=xsK=v)OY)SG!j|0k<WKT|VpU-Ks$>-IL3>qCaMr;1
z0k6txD7#)_%6+lTWDwugqqB#`t%7l@O{oXt90<R2o3wT@3Y4ux@5Czw>E9g66(1m@
zS;%PWlB2bORG&P+Lh?Nvve2{+grBTuRnCBB5@nS^1^W~I`_>D`S)-l%tgH(5e7$Nd
zOQ%>I0uZEts@g0EVj-`;b0?XF)%5EzBQ@gIaq_nuC<Z)DE5C=DLKX_8cE83#qYK)|
zSOy(ylM`dsq$YJx;i#g7S~`awaXM$~6}smF&&HBW?v#~jOw-j7O1g1SIv5~z`O)u#
zeTy%m12Pi#%is)QR0^<gf_!Hl!=MYP^{haJ>3rn2>xz@~>!K23N~U>@l9Bn!qoiWc
zAc@u#(v;*$VPaS~z7S!G%!F*o+jr$}e|o8E*WU#HN>VC!iJ!5vy#U?;+Au%?P9O!p
z&Ca19Op_7+M;`*A$nn}9pHNw5s`fFaOiey|Jv=?d?tU|w)A`OJ9sWF;3S!N^&zjeE
z*r_)*Qx1V-^@4RDBWW-~Kxi&kE9j4&h&=|gIML(90Z|M75f-J!;-mH8erHB-{EJit
zASh#Z%F$TeY?<KS1jU0H68fSH*Y5HiT__+X{x?8OrD=i`x^D-_gc*hqUmR*_E_qlV
zq(|g#|BJBSa!tHYYy@%-*Z?+X^U?`~7CcSy++%phZn9Z#Vm9w<%g6oo#$LM*Zm!|&
zyFen48enMZ1%7d8_j#{o^f$-6-q(7%IDg9Dk9KR!G3LR9jW{q;^iS{>0u(O|*g|j6
znqNML!0dFqFS~5lb0uYMs(tgi6f}qsk6VaP((ZsR*I>rcc7vY#dx5t$J@0E|5r8KY
z0zN8gU%CuE>?NahE!~34a!bo)r#j@`&f1y8YoWyS%G$tL$xh9)0Yk>*hZ^l#4BT?W
zq=9TSPit`E!A?*R$DiqY&<|8JJQIJucyhlsW`E_g(FXO+=gOImPX)kj0LV0(Pj)DP
z*Ha~w$cxjQ>zv6?r_uhV*MA)kb_Klg6@;_}^d^hQa=i@&=h<X&n*5e%^Yk1R67XDn
z270MeZ3IuC_b5MoB8~_PH4NK~Kd&=<=>Q^50|A*HyTAkq%PklJgBu!SjTy{y3p&?r
zz}0+f63`+x2aGXyYf^;Nh{EaJ)lVyNm-g4Nj1iw|Aq~yMKGaYivc^jd(Sqr^b-Rbb
z`$hk+0Mx=sffi)_Y;qp_a43)O0QWGP=g<8E7OnuW@PEySLpY)KuW*8mUMEfxmNt0d
zXPSLq#ICGT!!CYV*-fk4Q73@mUwNh(ee4*Ox{So_`@rAp?Cm<MBquS5#-0^6F+HMN
z@#*aG9BT(MNK6+YF+HU-`oC33pWx5Cm@MN;RyAAp)~d^JZo|H7uRNZl2YRjV*@DkA
zgb6kUfla=r1@y)HpB&N<uW;k170=h~1>IWjrPRN?l1$@BlhqEYr5r{!l!WId%Y3=L
z)qsoLN`;4c)I-(%s41dOV2|9d%F>n*LQCp=TNB~}&M7VKHv2D51fjQz@b-2jUBX3;
z=W~nfB|?<kuxWjy=d3c;3s9xi8HVNWPaUed1Xm~C3|$CF3D8?X!U_U2i{MiMEpCW;
zkOWk1Xhpq4{oJnX1|PZ2+l-Vt{{6NmJP$B=o=Eh8Ak5<T_4PUK%^K4>?bgesm*#3z
z=adz#g(tV<Ev#?Ritg_JO5@gGfe+Ea_N!XG4=l^@>D|S#Q3IFJrVx+K5X_+K&Rf2d
zO2u=5SMSC+k8E1AYh1#EC6Fu*V%xF{Uk9)(?|)|L5_k_E1)Nbc7eTERVzpm_QlYSF
ziQs$MlG%=lzC=Srb!%vP#Ur!n17l>$yccSa8eMD{!E*=~6Pzq~b2S*2_v0TJZAwA0
z3()aUE!w4YbSNEkRoPrsQ`*`N#_@XWzgYJQ%AfzL)Bd2J3hWV91MHEdlgJhkQer<K
zfEmC9-IDMOX}>NA_Lr#b?1SV+@!WI+|K&`5-Z?Ulh#w2XwGyX7Jj8A4hHf{_Hlwjm
zSKH`x4t<k-3Q|P1#27jtuwnkd2fQaU#H9w>X(g1=ERL^hzY5d{7;GkS+xrDt=f&){
zzkmNG<1E+|4SIU1^kX%ZkZO3FQ!bn}DZyO4|J1nq4#1)md$*pY@)e0;AsaAtX7fwn
zKd|RnR%e}pz~JJ9@cj06nVUyYRILP#WWCvCAxP}I(Q1-KUXOi9;aBTJC^J-o;)v&H
zt$JPTkp!pBea8wQj0XB!y#Q_ytEulY>Jh~^{1t<taHEjR4(rr3m`mB2b6egtI(Z(8
z8S5$*D3mFG??ZV2(Esq&Bm3ja3DocE5;TPPsL$Um?~GKZGh1$|#cJ>QUEpxuYszz8
zGo#L(XzgokT9EU%9K6wV>CLt_=X7aXF+Y_KJBgFkw8>e_RY<z;Da9JMM9-ubDLT(+
zy#_DqK(I+xDfIT!9Y}&8+Q1*0;&D5h5{Vy$r-+m8!u`{Z*Rvh`d@$|<h>C=*FOvPf
z1t@bNfc^emPXqbgdfUOz#nrEcn!rOss&v|Vlh0snfi;qs2Q<C-r17=cGWV$`)`b{@
zOR)mkj}JlMT?Iu4Wk|Ra`*{40XhMEBHuYkE#u<oY1j?}hogPd17Rn0{C<u&Q#Df(O
z1FwP7LY3(ch2Rf>z(3eiK6iA16Hx4b80rj#2vtn`m>ulXC8RX1$MS$5CvGK64Sa03
z1l^_MzOk8}&SzhDb8-_3YEGTWr;aKupnxKavHD1$Th<#!1D%(~H8<uqE3k8UCBq;3
z5xpBj-I&?$h~gD6Z|hA+J>KktDSU#|V_l`jG83d8b@;Ye#09oNTxKBI4aXsXnSIsJ
zVr9NFBpxG$__`r7R#$m`0Oya}iekyr*cmVZlQ0>XRr(z;qH56%D*daF8%w<|@D}1U
zZFXy1We60E_-xRd{^4_cwPVjZS-(}uxO%q|xrdINl_Nl40PD@bZBIiqdwgY4YBJlT
z7kPXeq37z?B(^??!3_#pICr0-0d|Vs4In5tJ9fy@!}VZv{U%=6VwoV3uC>|Q$4^O#
zmo9HpA9*#ds?ysN8V9<!5Z@8w)W||@X}wRO$&tnxxL>-6c3N2|#r>qVDq^H6)&?IN
zQW$0eE6A2FkGYuZp9?2rUFP3~IrYd!15Kvokivo)i@7LOElu}psRXw<g$|*O5U|<@
ztq(7FPl4#8wBfe5^nOpe73Qhqq|(_)l$^)2L*Clfyf;8W27zSGq7`V5K!$4AoTzA1
zyv2V%*1CBBacXselEZZfHR_YbonL=8YVhQD2~W7q37{zQOLBO?s>mhj3$5-qv~ra!
zPu~p^LuJ;cN%n@j0VQVyC{o*`CIqMlLX(XhlR=v`G5~X~psi>%zXd>u&hYeeHR+FT
zDijcfUFjDisjb|n#x>)VN_S>3`q8{wnItpk#TUNEoTa+uc^(Y934#MWR!G<Z1A4<W
z^*lxygT(nBl?+p-l?^7dx{oH^C7K@P5z<H+Cj`+`ibWZkWZOymOo{ynzC;&I$H5a2
z$ud|lyQ~h`Y3?x{D4~9hM%@G5-ucFyzE_Yecw-Rze^1Egza5<VuLgzw|54n(W8?-H
z-T={VLAgIYz3;ZOzg=M`c6vFI+HxDcfLmPUX$gFQ*rbi2Ug47VZ)m8!?_tkHdj(Ek
z2DGEQOQJ2r(8@zy;=4<$Ssq(}XX8uG8MyRFBR1n>6mCscG~^D9j2a9)nS^{7q85>z
zVAkkD>?{w{RFJmQ%VWq75JSs3DI7Qt-1yDg<0pIqa9zT76Y`r&L?48&%QmXK8r+d9
z!tD!$;Wk|`rCY!JYUev`R%k%8Nz9k==!M&msDl)dWmIbTflNljYv+qxaXL7msSh*f
z2aH?B1B(~mq{P>j;Zm*Eu}eP}(y`}PWol_?Xg+yVt2c;Zynt0OiSiV1$Zqaxgsdj9
z($FYf|E=xBNbU_$5vNEF=`;b?xjsQP+;$s^byvs!a*0x5qzV*1gR|og1Xy~16Fhjf
z6Vm$(CWhaw=AD4Nfx}*vXiZ=wzfnzHlh_4OTG7becTa&9D}MvxG!*^MUqOoX4yiAx
z-TmfPzLRA;^Tp{gw$lP-8<lgpgUe|%>6dewx=^F-7<b_>k2>#o!k11IC&zD!=H7+q
ze)r*!Cmu>F+GEM8h~~z8f~Nl7`lgYh$a)=Cp$!$oBo)*h%~7K`_u?2*7eoXLOhOo-
zH9ib2O-bjE^(Z?<>tx>(aW}nWGfa3c!|hyxUEeCu*6F3%bW(r__teTGh1xoD55gt9
z%o$538RM1&_BN{JZm!M>+QmF~{-vSgTAb{3js;M`6)3OLg9HKn+u-v2%pECqrQ*G2
z<{1O)YIP7+S^zO5PNyn7@WyQ<i(donKcyZ(7<S6%^WK#43&#v>2fz1bDQ%&qP(h#)
zZ&+pY{G0PN^PTR^l3{zbEgnl<t5f|JIZr?g&M$qpDN$T-TCT|5vi|BRXsyz*s;L^0
zTFL9<DgOj&|5YYkC*Rkzi)PIbqQXdMS6eXH?R#_28?iKL1=z5h(eSLn*JAOELMsbr
zDATx2ly@v+c9zOw23$JT#f84@NWTJ6_o$%_%?;UMkwaul(Hxo;CrcVk8LwG2vqW@)
zw4g)mQ!@#Y9B53oM_f7g6~rw4_08T`diaVl3vD-4Fk>p)<_a1-RQ>f!<&uJaC-9tU
z+#YB$EDuueT}cAT43H!7gQx3TG(E-^KU0eW(F2fM^ckSZIf63y*MQI7ck#*5h5yha
z4wUJ8jsFps_`jx1$u(k2N<?={9PI618hkkoay*b1^H7IO+3@u)wGMnoMK&B5|0mPA
zB^R_}e+TN#x&H*}A-OmJ$}d3N#>@Qm3&>s`T&MsBv~nRB%=n)pAa(WsJUvX#q%oWT
zvPW-<ludZI`hEh+&LV#R;I6mMUcOIAt^+u+q=!0N7&cCV#bU3vJe}jEgh5eR`Oi5n
zBLAYE4hkt`uMP=3(@;=?qB`IEjP)QT05Kc#GKk*^T1K}W@!k(mnvnG_4+IAfhrk1Y
zA-OD|H1+!qJeU>;plQ|X9F;fF*8)Qw7@25)H0gKX#(W=3mHXpA{tY`1o(tCiw`eSP
z9Tid3WqMwW8l7N^g7zO8LeCDAJ-gG)QIu}%0f<A|th_(4;?R({q2QL#Vuz!Z+X5{%
zneQaadMH&u%l*#%AFW&vxTtmmA_-6o>)=6;K`DYUv(<l+<3GuP2m4>c9euXCMQ!aA
zN%yZ!ahmY`5u$t$h4$%>w7zBzlC_B!L1-QlJ_bU2dwX(I;e<Z91npG*tl>VIUx(SQ
zBc>*(BE84-vyhsmVW9qqkMYHT6RZb;lHZK{2uN{e{n<ZkO$_2%MJJDbC8z>?ZxnLr
zk6h5h;r@>2{@e?agB<iywJbb;q!sSzd2x>FpSg&2;QoOMPya|J7{=Yl_s?APxu6^*
zvhmm->BoaT$~OOJF8DJb$o%1W|17r3r2C(_kUc;XJiB@PpT+v3(*EckFf0>jpPt7j
z|5+?}@#Fu@^`9^PhtmJg?s)45g4+qIGLLCP{$Y|jZ+I3WzQ4cUE>r2B!ZYFr46UF>
zVk|$?3o5vI_C@6%Mv`T3oke5YDtd4IVLo^`V7;3N%;1qMQjrtKQ@`Z#|HHkpjevK<
zi>wxora5=o7j>@U(owOX@>7t1YeeTE{s;&1B$)K%@Y&zxR-g(X3)CV^%=Zf)J=P0M
z>Sg`v2ztNQg8OsYZOk2+#@O3n(%ZKxk3{AdFnnPJJL^ZLu{GB7BAWgaRAq1|#!r9(
zXc%yFIWmnV%D|Z=ZoYA}uA4xEJee;oIWmpBSLvk$2#i78@^F@a8##8xL(>Y>xfkcp
R9s~c$NGeLC-Ff)p{{zlJ;^+VX

literal 0
HcmV?d00001

diff --git a/format/diagrams/layout-list-of-struct.png b/format/diagrams/layout-list-of-struct.png
new file mode 100644
index 0000000000000000000000000000000000000000..00d6c6fa441769a3c86044a52186d71c0bc23d54
GIT binary patch
literal 60600
zcmeFZWn9%;)HR9-N+{ALozfsF-60*)-Q5k+-QArcAtfP5cZUL-kVa7&M4G$yIeH%T
z`Q7{Je!BPd3){W_v0|<{#~gF46|NvB@em0g2?hq{p_HVk5)2I71Pshw5(HTAjg1Kc
zCHQm4Nl8K&rfQ5}8~g{+UQ)ve1_m4afK7QvN{QkCT=3CCMcr9lR)*Wi&W6#z*v`;|
z(cQ)#Tnz)m>&^{6+L$;Skht4e+d6T(^O5~?1vmH%{hEo4<ey8Nt@y~)Wfe$7>>N!<
z*csUvnaTK(NJvO{9gR)7l|;pV-VXl9M`rHqY|qWa<mTqa=*G%u=V->n^6c3&Cg!J1
zPoFY?D;S(SY@H3<8El=%|9!}>=ZKm(897?mJ6qV<l0ctpU})#!%tuBBebIma{%xnT
zh3Wsi$=2y-w?GG(puaG&Ffud!_iS)0FZ5e(5jz`uM-wL}@O*xD-hZzA<GcTP&%f6z
znA<tqfl+X@Fp{!$HgN<uI~zb-$N%)_-T(Jj{QWF>M++0s>woTM`FZ#6-~GIvmkHYO
z-v;8}QU2#!FwFc&yiEU189!3>`qL#C7(o~*Q6UxgJ3HAfc?2IJx66Wg>JA8UB!V?7
zdvK}PGzyCr$e48K^>CVb6EI?$5!iMPj}#V{bOP_W{a7yLZU51ln7;JhYj*EbOYZgh
z)<}9`Zu-)QTh3?R(&a}2R|f(<6eM>LB?V#8<NV-$evGDK7B75U&-d5Y=y4ssh)=lQ
zKl=NQJQ_I6EZI)W!rz|p&&Rx#6Qcil?7zSIm!K-xbHJBJKmX&M;KzVzufM;7%Mcdq
zgkV;1(Ei;<B)R>+zd|=hx&v>fTs{BlS4V!e2Q>Qr{|_6zg?g6XHvYBz&8I8}$m_oc
z<%HtShVDzx!%&|OZ_MspUwkhwedj{?moenoAbz01z2r1DbU`T~HL6DY+a}C0{X=kZ
zbh;t0h7EXrjrT8eV#dD%|Jj_2_kY=dI2y2lh@#zJ7yf!{hdHpM!qtQSWe50Tzz&`y
zG?4#2>4o4)F(KV||9889Efh-5$Nb&xC~?8gXZR@p+lUpxtE0&*{!b_VnUw#p&qSVY
zA%1Q4De1v#C~*w?jQV=4fY0TFvQqJ*?neSq`0SP&1MzQKkA^4`Wu8vUu#R%AN3F;3
z*vd!u`~1E}CKTa3{ezD)?b<ZlSA%^|*Ui3FHtxL|o~|6^|NcSwTSeVi4~BeHZH9gT
zV#p)nPnBO^Rvg?X_O}0$A1cT9{ll~@=hBok)B8cH%!;;)uTCQ@_qnfd+Ru8ge=INe
zZ!N@a6Syrp9EyFL8Lhjy_@-<%DTZS+fa}>BKP|_@K(grZIh&6^O%-J&L7Fi<eE;3I
zDu!>Vxj(e*-hV$^k6pHH*=>NBK5nbS(Q%%f)$`Tyfbnj5(|&W&eznO28vU&Bx?kYt
za3%1OYSo&?uQlWuA;?s1uj|l@B}LSF(1q-KwN6}`?K+>c_Oao*?R4Hsn!X0EVU(>)
zp8uSpc$kvNeZ_y6sXeRiv27EnVTE>$ZmREQvYKuEq<F5+g@wb$9B*^snf1Hv60;l+
z({`V0N0S)dqk&j~8}|;sJ3KqpB6})Ygd9d$`)!wdb>sZQY|EfuU#TyThMAPDrlhDV
z+nx_5jHW3L#?YFPnZkB`&h^!8HYVn|oRlE$?fIPT_9;ZBav1qyBT@c{-5U(8<C3hv
z^~vRa`|TUw>u=gV1UWBHAv}9c<a$$!khe3M-wwJTNw9uC;!FA&^zzIQPhdMehG<GV
z%%fQP&ThcHmwPSK@&Yg0uTQ^eSlY~8qDc^-Sz=fbX<8i(Qs<gxSiB>kHuVpD#Ghu!
zYj}#+=`pM0cKubY9qBFeviGq~O)s|7*fVOuPu|C)pusr{+co_}Mnk*}W4y;B_X(X9
zVx|-&2!nC<RyqS0wCzt*9JUTrxpur*Uo%7GxL%wGe6sI;XjRp`m8$217;$rbc^tjI
z&pOT<nBe{Wqnht=<}&mz&u+akHl|(V8F#MBj7EGo@qGbpFu~k!c)464CaIHDcGMrU
zG|WqW8ZDn(-ja@`$59iQ#m@TOh6OqHk17=Q?(2|({?4}xFhnm;6hngEqX@NKpDwI#
zHf<zGH!P57F%x}yUh)2PKA$5Lg)rg}F196EP1nO89=oI16%4=6M!dw$X=8h@WOd52
z^Rz4j?u+HUfXm%D5tKuAh2^%P=xoBA`u5AEePNqDFbZKZx;{%@`|p93VyiyaWtv*r
zYuUd6b7FSMcYAd#Vsmp+D$vV*SL<ZAad`~M#^*Xk(~{=-+J_GLWgI>4_(~JapLoST
zhjL;s0OLF;$Zs-5%Qk$0IviYZ*!S3O2Rdh7XM%<wdn{)|x~(8_?kC=tyN%l@!RMa^
zZky+rG6<$O8C!O`Xg+CI+g7UgW!a+5S(aCxF1al)-`@OKdTp{QM`aJ@8ko{F>YPpE
zl6pk?ntL?s7?;=XTbwr$T)|HshTg*TcFEbYP7Nqx+uvM$7U(C#FWt)Vvi&4I#(%j(
zu+|<|i3NW}_cLw@K0-IQ>I^69$U)53Fikf(?8Twpla528ULY!(Y0$w@qiEPw$Yf)t
zCz?EZI!v#eAGyI8d217Fn3LcKmmvhxD_J)%3~ae{?gsRcARV%q&AIH&YWH|h!jiFe
zfUH?_J(}|q@y$W#fU$+=UPb7ykbFYo-=Nz)7?72!>xsw>m$I@ijLgupMu2^;pX_;i
z{UcK7xjQ_O&r7<hrZw8~3Sq1@;vD-f#GsOn_LF_zTd!33SLnH)bG%{(Leqv3`nu;O
zh1(5zwT+_LIp*~57UYQI_1VuEgvL`HY*!m}lhvK2x}Bp%8129Rkz-vu7y%#g?8z<~
zx%i^n^2?_5!>*jFn%Pl-+bepH496jwx=p6071*nZfljRG_Mq!Ir*Xc*4*&Z^1Nf!Z
zt4V!|=W|A}o6H0bQdtgtc%<(U5FWh8-|8(~ZT_4itz1E3Qvcuvpd8aH=Q}JJn|Seu
zYM&~C)%k(rykA~j?Zd2`l8m0}9*m_bNxuKI^ORb##4&FeVw(9-EP^4)aM|aQp(H_B
zo{wFI5iM#144))Lb6L}!*UcVb&Rtz~{&(-Ae<<2e@MtaJ(aozCWl-~rP&ai~HN9Be
z;EJ0bdrnu`Q}8zTYJ<AWK6PD&vRsN8w-x)`^o4ShHr^iP-jkd&CuBnn2iX&<J*1Se
zE{qVFhy|D*?g&LQnyiJihb4-|aLbWruBeLPbX}hM0n_%>rLfJS>;i;}@LY|Hj>s0+
z`p2&k?knmR*Px=Q&V94zEN*esTh%U=<=aq8VxrQq_mXFu6b0_|%muih1)IjhT`C_f
zG)Z<Gwi(4^FaBOkWz75X_S*vwg7GZ&Y9xy|W1Azc21HTrN|)l=Wa`U^JSF|tKEH6m
znrC|Qcb4a1(|({lk**)VYlRt!z-fH*{f#e!s*Hn%2Xhx1=OR@hd`5wHD*Tgf@-x#+
z8`UwD0TI?md1Y4G%>^$-g@;C{fbG}CshgKt`vkMq*a$O73`5$@tARVC(rWDDXOWz+
z9<;v5@!A{Y$dp|tk{ZO{5_^zi9y;-c{;eVTgmqnf4u}=*0V6RK#SVyl_KFq4D3_G>
z?8<{fh^H!H;uEKm{X@ZZW@=hCZxh}p;*m;$=@KS7a3x=G<lsA<Gn!rj!6SE+VwAfc
z9!I-+1IB&H4^?)vg}{tf<5{P34F83>E2Y?UNy4Cbke1QV&Qr8U7p*WuDWXI{kwy+m
z-pvQmGQpYLg<pz2Ta)Cqy1>f5HJG#;%!OrRWg?lvB%7?vMR{E<&a$Gw@`;X8)!L!w
zlE%m8Oj;pJTSAr_dO6-*eLUlaNVuRzJFw_19F1iQDdXX%HsLpA;TF;l@Z(eJ2#eV^
z3LR#QN=HtVVsqalrOQ%e54K6rw13A{o~8Pkvb^*Yyo%NSEIL2ib^gg~|Iogh0<vo~
zktf8W^~t86m8Hb5A}6X2($1Sr0Pq71b%8>W2>K=x>K#fkp$&N6_v6gn4|pvMVg$Pk
zb}3*VhO0Y^uJ{m1S?p#xjnUBHz6#zdf)Qq1@M>o3!djO~qeG1HcP1U99p8JUU4)PQ
zjA*NxoY21G&LnJ$TTl(cFp+!*FY*z$fL!#mt+Y+g&O`=F(m}&s@{&?iX*S79&n;jo
z*O8yaa%dEvNBNedn|4`-;$a5|zQr)YKMKFvnPX-!?4#hIPk)8ESQ<Zr*;fzCC)0x<
zca!3MGTtDPF-MBIuGM)RXideB>bh-7`*dDEfXm?EiAa?fo33{kqucSuJ>^Z(Y2&*u
z9&9D$0w+~-M>t$M@DXXBN436M|NMBRq`z%+-3E*~YUV~bbq=mUw9{;Xaq^z&Q+HAJ
z{qR>|C7bX_-BcLIF0bGlv?!=|I!1F88xKs)JOS3w0XWV(7F)}j#qbr`?U`o?)01|A
z9iE7xp_Kg@d5H$HWg&Y5A=&gRU4x5t*cy~{^*2pv1#;6?c`6mXh>MJ2C*7%$UL~+`
zj7(4Jy3Mt159SJ{BV%{Qp0viT-A~t?Q<0vLy}^%Sz%mz!vatR&$=Skra-H~TU#*Lb
zU98h4x%g`!u&?3=pdh5~yeRQuME{TK>ciJWX3Rw1@u(j$C8a~$O9t_<-L<!xS>K5x
ziq5)s1|Rj@f50kc%S%wzviD9!s7Ak$K4)~3q}*OSnlVMk6|w_^D=@P(5yxA_GN!%c
zj?nGFo~m~qC-L<QO1FN4@x$Zi*mC%yE31@gHgOVW^cIsiBGJR~Nc6x9WeKkvI1S^o
zPR^luC)D)_$!^k7HyIQZo<#;xtzj~uC1kfcx)7z74X|2Osi(T(Y+6;@J1$U2lXNAN
z?tWlOs%Gn>KhPOT$d_q$DEZn8C%?h<g=+mcj#MNL6=Jw8)Gmi?R?B)UJ2Dp5#@KeY
z#(Ye+U#F93+ZScok@>(gPJ)3vp>BQSqmnjSfs@H`%jLAtlJV*TH5Avq)oX+|DQC5!
z!>whVLVWM{1sm6gGj{81wiy|+u8v@WOr{_{!*}v!+`vrMoUQg>$Xr}Z=iVgkzH-c%
zwc%3eDQ^Dyy1X`6D~e^|t;N3}SkR6d^|o>5S>)}#uN9xC<Ajn31Y5^GLokQs8#)gA
ziHWlfbqBT*<r4g0d5aIx-?zXg44FK9QN<WqBOc#}xR^V(_85D;vm*5=d1u6P>+md-
zNwqzd!S0ip+|3NiO)F2#KB6?M{F;Eoz80Q06)PiUqqwvz)FSGzL0uMC0DZLICCN-G
z8cFgaZRt>kVbDpCqD_}2DT<W#wt(xVkw38SD@kI=l}+_}zIU`&5RVUM_U?;$T^35I
z(5YMaHtSgywL!Ck>F0#0#Vqx$Kvunn^rmOTC3Q5DPdd{Z<c*tMEO{x(7TE$jopYZ`
zu^EZ8ugESxWqX9IT}K|}?CH|wmG0aSk{d(8CDa*Vm||NpQ;W#^4sjE|5u4J%V`W!r
zGM`hcxWSn{pw5#dD*pm`Xo2m)fP?Q54v()Ko9i?3Q`PXVV!q#BEj*3VLHYO#t?7!A
z+;~+wb=c5yO_Pb_bUCpa*XXq#eMWrnFpe)X8SYE*PMM^ehMzso7j^#TltRylhFA$J
zOWt6yUe_crl$L!>l5ntt*|Z)5+~ZHix8f*~_358~>J{A_OtENM+1YDNSm{yd=68qG
z6g$tKB<#*BQ+>_QAcr636Dj?yAN8?JH5Jj@!GqIZ@m-M>3ll7=nEP{_1)3j8i<_e&
z%IpYZP4=iA-<r5^9o11I?~@bPD>n#&aUAwb%0!)a&|58vb`Ga3gBa2*LB~;22(tB{
z?j-n<ljrf_;CpGyf2<YGjQ&c_UMaxDzcVp7m8R~WVQdydWKYVVO&45T(|sKFCHn`{
z4prB;w1i-*VHv%8M=P!t_#hS)Cn2L(&FXKPB=f#Hefu~p8==>G;u@7E?w+4YC*_08
ztomh1`CmlmuPZ^4<MEZiH4{m0t*7V2zJqQo?uCa#O0o*1PD84rw&S0X%u}b_T!~35
zATfofhF{B1DfQS<EaMITsHT_gP|J(HuFKJGoY7wrCr<B4uR-!OobR!dS$CRsO$f3a
z{{9RYO=BAySN-M_kjD<49e#a4h08z=XLUF#{5ni9f(B!R#zZx@LkO)iHueihW3wAB
zBkxUyilwXV(KKgdmbL`2>YOxhWt3mwi48igN7aqs^RUJ=u_z-MrO3BsY+hI<t%%Z!
zwqJ?S)}f=@4LW9*T!oCk;%Jq!Ut7uUOOdUHVK<eCf>`@BJ2yj(vF2TBdFHU0xV3B+
zwv!=Z5gMF#mAXx+4${~jY?-%EU7+Z@b(!HGCj;cM#tzmiy<J(C!Q-moY&j!`eG#<u
zv-B2?Of#0WeTHsZKdC2ACjT?}=CQ*@qX`mg<yF%`FIx^wsWwX%gPRf^GIs(16K#}y
zp6fG49-z}ur+%r0QTI@e!6((GASS|__Icb~!bMTr3;b;aNhcisaLquEXLkS{dh1L1
z^6Cf|<uPQYRzC^K&}506u|8G0GEY|Yx1)5?)f;hKku3M$hDY^i4p(@b6;kw#PZUsE
ziBUyr(q%EYAr;<Nq&ruDnWh_?+MyINCTBb>&Gi`?j^x>IW$l7bwjxbk-W*M&(_4+x
z@m(u$d-ek<dPB~jK@1nwhz9d%q~aunI{x(3sP?%jLG658>Qe+((|&wO{*xG{Cl0Z5
z&iBW^Dr~)Q)j5v(+OtsELN;2g1>k0+8fls5CY<zxgQAtdA_-iK=Sacg&TKxebG_8*
zwRD_H$B=nHm1#nB?lSX{^$HV8ZnOr6(_u;(O@vu(o-5mxpm8fqT}=lhy?dWLDDTH<
zuPz#l8;(5e6Zz>>bEzj{0x*cX+*P+G89d%3x1#qv^jr(4u147$D>1QazH(O2hnalo
zt}8Jl{$WA|-U#+vMHqLcqY|vtAQyx)mO2NRz$jVy6J-<cd4z4L^PzE>?R2JO`EGOf
zf(DP(9W?`ra9*Xv^x9Qp%)Ya}QeiPUF~OxF6WZL~u0e5RZv^t7_etg*?&ftubpC>L
zY9l$a$^)h|Dffg`WJ(6+=Lcuj4ERb!7HN2~hHYvRE2pB^%8yiaTt4QDgrdXWhYY&i
zp|<CLMP3EV_#o=xx-YxVEN>zAVIS;{r8usM<=J*v1pA_5^=9D9NBx?ga!{K??Ih-9
zi!NB)%8lLpoy!lc4Z3jZUX!J~t|fMye@K_SD!%5HOzi83i)F{zH2s`=yp)y_gD;?F
zLlU_Df2hk71K8asTxeb&F-n;)nAO@g9v@n#JHjJG4rMxuA_RwX2eRMuWnd$65H-uR
z;ea(w!ps7c)u89M%Fy{9&pum%9T=(a%u|L_QY?~ncb}9f$30fX!x}Hiq6pk(GvT07
zTxg~J*2l)f8;8d+(}l~MIfS}fcqCf=LY1*&(};%9#GGO6ZYte45(|-1O@RW8m`auU
z;FTk%{9sqJ%%^-~r-Y9zpJa{YKW53rF3oiuGWQ6vJdP>eGPQa&FzO3UMxr|iB=pU8
zvR1#uWT(H>K!_XWUY!dmbzEm3e}(5h{P1l<Zj5<l15X6}yDKujqXF_s$pRJIpxwmr
zMv^uJuF;goShjsNd@nNeX+0Jl`fD(sY2Tfr3s%FW7m+k;$^7yjw}nS}So&Yi7;sk$
zh;mnI4i0elU8RYnh8;TllO-OG9VrOYM{gKE-ZWYcc61qJQu9j=%I9$SRscEa(9AF6
z3jRS8fC2IINZCDO&OQtpEvsz&^ltc_B8<@E@S(QaZEKV9kF-3ywdw#QNi#}w#CRER
zgqCsiN*P|vSw2@S<HyjFsCZ*UJUkST$k9AQ%B5)HzWHMPlz^PZ?b!L59Eauz;;{21
zo#6T;Z-b*r(8shi_03@(6kY~Ja)9xe1nqhHH+gI(<~O+>Yx(_Pvd$e2H9eH3PQYLZ
z3U>_*&lxBnrKSC$ejJtI-?*q?b+et<x%Wx2^=;=~=pI=$!$+35T6^wFFDA~e)4_f`
zuG33{$vaHnpQF%w%=JBTr0u?L1&OZ1fEA|9@>xWw@{%kFEKAvMHh&RAVtG5rTRHd&
z>aV_JyF6!2x~%|?d(|`IdILCin*sAfDUtpu%uYvlU8dKg#@TeP{j?PmaszT>=#;@Y
z2lOh^L=C;P%>-n+L6hi%qwDT_*4#ajTvL^3S?nWps#PU-%rIbDZ*MLKI|b6u#mPo^
z=Gtm0>nuKyOlq7_+%D`utTRfwII$|lU`lt&3^SNH;=|+CmFX2k<er39zsD>uO@-ga
z-0ROLh0rq3)RrjI_{XDKG3Od|hZK(>A?_^zA{En28vCr~=3z|)B7(=bDaDCWzTsW@
znlrpq$-NEjOz(U|lb&K@4~|mF4WuOA3J>wo4lT!^?^n)A*45fn<w_oIFygwDc^r_2
z)b|<8Rkbpy`zp<vyD7O@{9#xsw$>Jdg08)Amn|Xl8~wz?W}}vMcKY<8W8Oc#TML>w
zQzu>vR|momW}*zb?&t9DHUqTsW$ASsT+516Xf-N|1Z-%-S>o8XAo1i9Ap#67$j)?J
z&*2(`NUPU9EIBe)@aQzfGg2NKH+YLoEoILF8yrF^Ptd8WxlgC_4C~OVP^O1KoJ7a>
z`a7jn4LPxa=^4rU=hy?L;uHpKmh-7hUL2WnUiB1O&rhvww&PlO$5JYMfPdXOa-h-n
zKI|RrIU9W1x(MN-a5ke*vfoPRjYt;>dC)1b0w8VwNK66$e(N#w)mkF~4*~WtCv)g`
zm!=X{zXTif1u9+(BemEBljo+cZsM}NnS=(}0)&u~p<)l^Zo0d=1P2#>xd^uRZ1q2p
zta#49nx)))XY1~BWVLeNe?X9OLEcM&0nk&5ez`GcN=B2tzHlFPY7I9>+Jb71WcYrv
zW9H<r0#zkt*YYDprx%5@J<PZd4b$<iWx~k=vc(N5C|lP{MdzE=QroXg!jW{aB;})d
z*DyIB^~7YT60AN_wI<fGazDM<;%rB}ivnAwULibI{-%xlllwXg%>$EOKI-tvp{brU
zM+e`4`35nA$Y68k#(pE$xEEt|GA)z%*ilVn5LfkToJeMm%($icd(;bal@zdZrsZ+s
ziK08QbA}|N*$hNvZiyE7i6n2BBTNPi_zX1BG8XiywB-R?v8CP1xX(Cb<o*b@*XEmt
z7MdG8e681XDOQo_#A>i`m2ysOw`GK|h?v=U%44$zndcberhs{i{n-1!uLWKWHfKfI
zUy{vI3$t}7@GenZGb0zpjZ@a<dy!LXR#TO`pXwQq8ZLGYl(-ig(qXaZ2P$?;$!#JN
zmd07Xqa*r+H95I2p!ZaVu5(9jbE#Fh4@AJsz#ow6IviRLW-&I3l@UqS>+J=s!NwVH
zkcJW{5j+rRI5b(+iY;SBDjLvrpTaIz87spx+BE$&A775)))0%vEE;C{vw9vxX&CgE
zn;IGr79c&EPTE&OS7G$ylCs)RcMQ>@+Vofyx(qh6*0c;RhWF(#%-y($hy2kHse*Vo
zyK;-!rcR2y{d(7q<-*|7kfOpn)B9*pd033_MdnJ+x2N->J2gQG!3Jq45$hGEx>EM{
zM0iatP~*dOdoe|_27zq~Ty(9Zd?T~Dj9Gv3Xo$C;)k<7=XSrTa(sqAZ8yV7FL5gOC
z&@FgQZq=L-#ISjoeO7NFB6q4YFhiMr`bv=f&|KO|g`)jv6eGHblPWE*Y#>1_i^0DY
z6j2UgMzJ2;u2!rRk)<4v1l}{PcbCO*M?4JA=gCvV(K2z4zX7vFt01F~XCz+ZnL-rb
z#Jh(PLYpiTfT8uIDl=Ii&A8A|JvrPnd0yx=6cU|^JVN-f|2Z#5zQ4c)HGC;0zx`|B
z0zB3N$lh9|kp0Z*w4#Rn5jV_owDE(Dt<|s9T)fg1!raY^pH*H4(>}z>!eV?9^5ev5
z=tynt+mUcI@ryS4IqT_iJcOb<p;~aI3OU}*wBBbBskdKy(&kxyF?TYshf$iJ7Z)V$
z(K#t)!gyVA-f9ne9ChGh+MD#;OczrC==z9dQvZ+%7QJzN!o>l*e{%6vt<@B&Tyxz=
zf+MP&&r2syR0-~7v~Gx)*#uzJ#8>D};*7=hI;gU~q2SWE$xtzLxUzOvP#M<dAu36{
z)2xo%t5yFDr$61X+NQ+qe(#JDN<!P*j?jI=BqY%Qp2U=vR*xqlR!s?8ulq@7F(+|n
z=Cf9MA~KUtn!oZI9OIM8GEfdhvtm6-?P%-C;ge=;Hg@i5qbV~+&uV5#;x(SldZmt;
zwNctTNBJ!wJVW+c96F4f{|w`l^g{u{WO(Xww!4Jwywq9tuvQ69O|--ffh!f>LWlIP
zs@MGV?h=^SIYqr#-n0cg<SBzvHg4TrW`>_sry%U1fOR&8T_Jl`liN}2SOqh+Y9V%k
zu{rCfn7Z%6r|E?aY~Qdv6y&J#%!BS{1NuK@Q)x##)`fAqAW8CNjiMRvXM?#_2i{G!
zTtS@!+z7F-m1xKkeKpQQB3@j1O;PFj=S3H$!^u6Up{Ihk%>8T5N|m;wktNlWyRFA#
zX>`iS+EiEmXvJxkiNjXh$c#C%cCClv98;1Z6I{cv(jy`6rc{LzuV0Y*3wM{|_;zVq
zDZ&fzVZ+qTr+hv$th4?SI?VQ%A;`-qST~~7{9!l!wdrR`wp37=!K0&#S?i_UP=I-g
z`yk!gmXAo#Pvfb!!sz;ju#s{XQjblaE58^l<$cJKOdE~%1gMD7x#>+3{&;-MuxI+A
z2=8M^W6N>nacSWkp+JZ_f&*jkZ5Ai_AeA1y#xv^vcE{DeoJwyYk47<EX4u<}l0G#*
z)afQXqh0xR1082ZZ5!?v&nBNlC3_uK{3^d=A)JU*t8y)Embq(I*2Ob=SvcIuQB)_v
zcdhi;N;msjRyu|P)6NtCsTD9&JgeD;&!<;s>eu8aaHxB0DSez4ypQLT7GxO(U-0&=
zvRS3&7U(py9PqkS{N*Q-3^NVU`0PmSs?4g)3#Os>y%*Cw2Kqwd)RR8$Os7@mNUtk|
z9&LzFP+N;?6W+T!f`E`kx7Z<K-Ov>4h%-t4wDuy@XQZ9pVdGOKs4%c#recmfB%qc>
zt86U0Bj@nQl)FNHOAXdcbL+*`TD`USXrI=xyhOa<^(_2~_jbdgqx520<Zh!p%KFCX
z3f&{VJ&(P(MXOZL5KVzTPtV4xe+MD25yL!eM%kX@h=asZfQnqR$2ttHEME49xz@*#
z@+Vwg&I7KD{&LiQHH1b_7hL*$>at_H&wt`tiiwx757Dx0Hk1}#!rG_>hS5ibFWoBD
zPc=L?l)d}4{P;CuNHWC)GD7t>-D+)(PH2k|6K;*jfA!5+bOrkvFN1J9Q4FrY8Y+ec
z*F|8_wbEx_ck@EO{`lze9aXL+Sx_3W+JZVh9n!;Yb2~k?w=46XF6fHM|K>LHU>P3~
zl|IP+t@V264KbTg2dSf_b$Ryd?;kw?zuQmdZ;D2uMCXHTaX-V<#i-?1^+__14i;&w
z3c<xCCk&@YGfGLcsd*&;nYQO*|8)%q_p;jOkK$p35pWseejW3&Rtq8j)z}3&5uR|h
z+$lF!3=phMru2GTONumHT<evdM8B-{6mVyoDO!7g>Nj)k+23Vwa?bkimC$@ZP#F=o
zxS`D5M*F{h{FC<!zM;@hn%DJ!5Y+!N(4Vvl3Mu#jrFlY5`L8?5f%HjBcJbyf9g#Bv
z(3ym4o0ro3sZ7cH2;?M{RJOi<w8syQJIF7fPK2?3`q#|-l)6k<f&o&+XcYKWg#Y;-
zLZ|?W6ldPzFS(RCJ9uw-_(JagYJc9!P8{+s+kYeYZyH&j0p9zuXIK8OZiNChmg_*U
zYsO#g)RYD9wOl);{;OLeU^;j<gn4BDYG)-?-ikH5!)4-sDzbkbI6(@AfzUBOyX3ES
zs$wYEH)J$jOZ_SJ`GN|2&10Oykhyj7?al3%!dN-p!=4|DPV&RtHkJ}l5dzR_$l=_A
z`bw(0VR*@@*gCQA)geRUqEd4zV6|S3u=H1G_tpK;8^KucGLAF!)~lo8aZs2lN|c*l
zam`#@jF%t^pL)!!-+{uj6Zh>aDCO&m8~faxe{NPD<!U|O%AD1-svKhQJ%yNova0Up
z)12f#y62e%2DP6_NbAAMtHKx{8u<Wf`H=y2z3bili)%!?a(@iAIXalK#EyCEXgDIT
z9TrQnm$%2cw@FrpAgN_=8~T|>{Cq*)OEAz2pEbD>ZqX5tKf@x^Gk;W3mJ<EriC;d#
zMJX+mGwJC)zJiv$7oEmg%0aW=ycr!m|6N;^cZ^`ZYQ4?w3lsxP%^L}qzE@E30FXCa
zzb>zOH_EXf!_te{?6A{*dxNR&JSjG-Wye?WUHj$tkBv*N3!n-wXP*1)bCwf8xe(wt
zsM3uSv@4z6LAf#8=VJR2|Je#uO#*~64DU~-=zwshc9@aJabuwU_Ie+4!7!2ml*IQy
zjrIF>Y3^2>>G>tJLJ!KflLJ1NyAl;kl4iG4M|fsgoy(v=?6%V3{~!cc08|T28y;*+
z5P59u)Q!iC0G&sA`W8st0D7bbl@T7fQNV{$BB%Yi_D-l^1wQ-;FPoVa&wc$K=vTUf
zp1keV0C#&}$g(LOZl!^WG?C|*eC=~Ysx-fZ1=|(|G!3+fUQMW`2Axff4&@Qoil3o}
zBUIC{1eg?uuWOM^B?mwppju0j9<&fW1GGFuCh98T&O5Vj^lFgSuYkGM1)k5g-y}ys
zl?@POtSReG^tYdUZ?BPhc`5%`MQoXZyy>%$NbE6+$c)Uh>wRC}lSzr``<EW{bt6yJ
z4?cN(Cfo@EgY^MCsj&-*P8%eA$KwLNwy1hO=ai=+r^TYWhPzon2LN*VU1-T3WQv~G
z-)A`&tXT;ZE`Qv%N1lyxF6UhXt%=M$o>fJSxjoRMbPGmUU`lVr03{3%92vKy0TBxj
zAz(XJdmCyJc<t1fr5S`giL>Ewl;hqm0@P*PLuQM_JpMU>nh#H{$8tca{y}V2q4B7%
z(H&}|G*Ehj7M;HV_7h<A(tbb~(2;$4S6ye?p!&Oe6MTgFO1Zi(?3;BF(+ehx)z<Bj
zOjyj<JLjKs<&Up{)F%GLdK4>)S&h@$Y=&HLuu~MvP0PTkTl*zp<l|`?jy`SUc(DSZ
zdXph)hn<>!aemnEzzdA!a07W&E&fqKB;!zyn1>RYV(Ms=bFICSejlD$D^P3Inv4OP
z{o(-d#n{%Y=_<ifV}#04mVN?chI$3I%<IUPXF(Wpg1BZpPk(~KpF%~pKtbM!S;&#^
z_o&<w^t00iJGXfY>UU|N|8L%XeJvDtuXxX1zW1v;kQyZfX0y^w6+DP=UUHe`+iO}E
zP7<do>19rFJnOjoNR-M~(J($XAKh+Y@#z5IDA)!kRf@F`KkS>N>NNDAOB-9+v>x`+
zRo0_FV*+KbP-34nIh&@{kSJ<?(yr+;r0_Wm_;RL(>9W!s&%$U9Lk2pyuIh83d$Iw#
zBnbis(O~SqtgHSJa5K|ydc)Aj50Hy9U;bFaVKuF32jxHbW0Lc;3hZ$r#{tq_8P|v5
zZ<F4<Kh@;++AdCA!y6{$jusT65uj}N<oV6exxl$$xCdBuE=G-_Nhdr`l&CB728qJ@
zw8^ld-<fz5USc2K84z3%0j~!Z@+Hubp&8NVbU!(M;yNL8pAXdkUo=>-{1GPd?t>tj
zi6SsVIW-Dmwq7l(3dC`kp`T_Xc@DA8#GcG#$&`b73YS;WqzJJpacpe|-vZ7Is0n<I
z5_6&YBS@BQvBo=%THbFA<N`ct36ykZhDd!mQrct%7@H}9Yx+_9Eb%TD{w|eZ<+yE@
z@cz=>{0xG3{cj^Vr%?*G6OHuCW7BY|@+n1H9aBuw6+;vR4R)it!@gi_Gv2nHOi3%2
z6nWGQE9*2YS9Ur>G;2D5t|^qSE-lwFhWqO)uo`nWD}vt3rvED7{u^=p8z?~>yeNiZ
z+JaadUHAOqF$CdEhD4or>7z{PrsRnO#LW7FgRsUaC)*5aT&dvc8(%p#;zmv9cus))
zO(dmGw|;{o|Kaw4_#aTiKLo0qhBGP_55vMNMI;PTwkj^TJr;mU)HY$U#;9gh*B&J=
z9wfelW)Z>I^S2+yeNI{7sMmXJvyk;Z+yuG>{T>y+umQkPqUXSI-n5lr^(ACU7F0SK
zw!U+MMFo|t)yp5`ROs0@^e^I1?vxS9W3YvpWbG^|8PttZqz%)2CCUxH`=_oK{Z8OL
zO3cycBwU5S%lo|3Cbk~)7zVP2wu{%=>5Ru{S&Ca=)|oWYKE|qPJBSS`@5T+H(BZmk
z_ziIlM^j_>)b1mfj5(T&r2Y|Q=b?~k8z;bq7ERJw+2F2}D*>s(E@&+_rPbqU5TV((
z(u|aTfDl2IWK|PiH^xmTMopWudzcj#FgyUMtl!X|gJ+}9Iv(XD4w*<W`NgdN43P7o
z(8tdjf}jJXD7Hgvq~}zQ!9pBx@c@p5_v)bAxHem=nYp2vmh#cz>=LlpXWwVE>}sy1
z#?(DQ7_D118;mdxnYac`7{dBu%g;K0?JEGfZOQ&I^Z_1vuPabSVZzY7FxHa8?(ADN
z2qjE8lKnkC3&Pq+3$iNGF+N<7&H(`xU>cTi5SMI#nl#7v<{a^2C?t`7zZKXeUe5<*
z`E0lsqek2Z9e07xFdRvn)%Ds1UhpDP@5*qHGS#>x2`T{XLS}-7C6-2Aw<VX}(?gSD
zTsmGfFVTQrpRho>7Cmff)%3ty=9V<k)rXvm)fxJdAd-+R*g@Sq-~q<~miYlO%YC7$
zVFJV^P%$jfMh$%6zxYCqzi6B&o4{7-$c+6b;PZse71ms7njz9iX8Xm*H|halm>Y@T
zuq;A}X&!_5Z}AGNY5_Db<ls2|_S#rW-D5KuG;bC)Yt=h*jO<jQivX%Si-(rvXUMsf
zhnJp16|nSXlqZwax!$$=vANnV*1g@KkH$?rpZpP4>#6VPSZX=zFh3@P2eIBPV9mJ2
zxG;&(Z4u3jgk#*=XVhYC`V){yb%#9EG6=>ZOQKs)GyEzlwe}+O^n4_I0Tg-m-p=Yq
zK|OML^S28+!|(4wsA6nd2>|23w}4-07)&jktq}!+9z^+<r1HOoKk=ris9Fms;zC_G
zrYeap1r4-CSOg7wy7e%lXE(R9*yYo|2`3R|xlB(@2wZQ=-zTt#d04APEnL5qqNQ*M
zbjPBh>$;I=a364u&K!&2KkN$sj`nVI1x4y}0_ye`<$K&-?lpf-dqmW_*-)DWl3*yF
zq|Y5_Zew*aBl}}e*fIn~;7U~s#r0BhtR8t$`IJgN=g^>e76HU~c^uE)&Rdoz)9e3@
z&hLS2kptEA5hptcXy`#HLW=-r1a$*Vs9oJ%GVHy3kF5n!c=D=wq)(H6DklD0vZEw<
z(l)HGPSrEzc@N?8%^|f3Xfpig8=@JMw!7cnw9B;?fLKPGGJmuB$d~I&UZAISm@EAs
zjXt>t`_Q56h5c!W^inoU48b!;;F=7~;teTEp=!KMEUfwLQ~E!ByE7RGd(^t$X+K{9
zu%_z@F=1-IIs&LvDrsKB_lmxz2Yru5iBd4~_lzU28w4btt9$krzq>}@)-uHMNCT$x
zght=-HvIV>$^VCYK*~g4`~S-IUf&+t4>L4g?evcW)63pcc*>g1Km~wagn1C3N`x8R
zhI3;flI+{_vhTBc#G;Xm@60n}r&s^z+e5??yKa_eMV1KOAb|mi4?U&8*-D^F8`y-T
zCF6Z{*guI7z6L>hw#n_e;(wp9{mPNn7C;XmB9jQl_8g`;eF{Gb&6Tw7?Qs2`PEPQH
zq;A)V;i5~D$orrZs(?-(0Z9C%i2Z4B`5M9Race>ZT+0X7mg*Lnx(#UOE(&arsOa)z
zg);py<!q6_acCG{y0!$~dpHl0fXh<?t!FJjnQzn+hE};V+4hy9`r>Q3p3BD<E*yp~
z|KW;>&xp8eug3KZsppUpUU7JJis8Qu)2*HKaHsh11<x@ix->GRN1LIFaF8U5DNF+g
zz7K+UJ9wlW5XmptwP(NL1$ynzK%89%Jg@B2K?=wz@S>^(y$Q$P^lm+8G|WbGB2J+C
zP@bYj0KC)9a+=li6rcTAaw8_JK{*DXEZg?oHny(&su;G0@hJN&0Z2`qba;V$7;5q9
zRsd}Qo&Gt{s)7YuL!iX+JO|KC!2RJ^;6wBSA4(Qk1eh0uQ+b}PhJtJ70Zv~B$T}A!
zLgvQ6NoLzLKnl3efsmiC#y;2g#zP$ToesDiID?qF=60Yu9|f4&7lZ)jC~(nx-`ks>
zGYx%z_$D}LPKIl=vHaQ_G#6;MH=)|ubpRFm!2SeYkXo`#gS^f)MZ<KEU3c{Wgg^Zm
z&;)!FAmGCnpL?U%j<ngrD;B7eGKd|vSAiv?DNpXjAoGiz0vmXes5+)C`Et$cbFeSe
zmq2C(LnVo`)&vyRwPIiX&8&F7t#*e>)PRJ6F*A26l*rQ@Q_!z?WMeFJt1K&h3>(8j
zIvFV3F-AGSv}4_mlJo)kXeSq^_sc*L+#3;J_fb_#Y>7=46aShF+3AH~Xnl!&1<Gi)
zm>hx22gPOs`>+@>#x11MZdne<2AI;(8Q~<T3cE~@LkQ;#(F{3JoLTuEeCDvkTU*ds
zMZ$DV(Ih&(bLROn?GpDHO%1ADd2$}j>HnNT-ZkQfQ}5(VwVQka{O@+zaBYA>0BkAp
z%dpm-`S30eASh=sY6HfUw!*PsT?YixRj-=9?$qaey}ba*Ajt1Y3aURCSZ7(3q|dEv
z0Qu%Mz|72=jP(=3)~>=n&5QtIWK};j955-t6oCslB?9QMY6eWYFR+2z4QhdWp%oB|
zdD31$XBXt_&Y^^^evi%iL0AKk8QzxI=WG?EdQn2jkKLeX9-7Z-+jo9g0n>U2071ah
zjVqw0FJ^y`{6q$h`89NBg8<0n5mH)!4U1ugG=eS}&289&h7=)hNmJc>@p=M&J|_#j
zfW<}e>77@ATzR^20p9KgOd7LPqqZRZzX1a5%@=yuXLsh8y8S*bX})aoepO+=ezmmk
z^P?9>uV}E=fXeYnn1Q0q`QXWHAhw)%@XP%$@)*fNGXFR&i{fO%4z9NawNsF+VAjoC
z=P4=08-Qb;r^ic97r|83S=*)~t;|OBL`PipzDi@qT`P!s9aRUyxAa@dJUQ~)G^3b7
zji)SgePw_JWPD+u65l+={cXY;<cYv=Vu21gOYE81aU`}J?afhYdz34)AvDv59K9_r
ze7iUERy9Taa(W<U=6^^)b6?oEe0rN($$Fo#qc0leVCQDk<J|#4V{joYTcdga7^xVM
zolu%~S@bz9z)Owq=&M^u8DqjdPKR!WQJvU<WK35eKw(2|`3sg%szS1cQQzCkWsU1_
zl&fyy8`Py@_5uP&I-@>{&KY{(G><_i(g~*kP$Q-ju9+_Jjw+6a;%^b%5@ksRM!e8D
z0V54l?qpg0mJ(wGwV5}D3t0dl)xrP1_5hX)_X}XnH@7d?5Ednah?*xNC6siozp82m
zTUERPZ$_KayZ-|uq)}JUtv|i6>>F3m4I4J>Y7Pf0o$-OgRxHfuiot6;z&vtk+kSlV
zk`zrDHk6V}hlr|YA}$5p#9q})eh)xq6ntZfoY>bfOs2%&OCqJ1;L>byn*jKS3xGFA
z+7t%fZsq^M^{wEKl`P(!yC*GPB9xow(FV3LRoGh&A-sb;>2w*gpr+4KC)aR}h{tIB
zc#)cuqKzDJ&KWY2FuXR0_xPSvlZiZO0vOs2Yq{s^-#xn!sZnh?H$?;T-ii{D$%f5?
z%`xd=og39>tYdPw>(pwLglpWxGm!xQQ9p}hXi$gSmNokeYKwuyhb-~lhO?Dh^t}@`
zHw=MG$T)3%rv@B#ODxIMy@&WJ`Nr&}2_1AA?$F#koj7GRp>#m>VJ$K;N@Ea?VJ;Sb
zMGgZt)$@rv_Xz}zNoESspM*WS$bBUMHb(VC&{c)iqZFFgkRiE3_Z-Abh~jpkavKlC
z8A>_*9vVsfMFmsmg9_rq9cQ}4PbViD=DyVmd#T^7?EvaUAz%HQ{3^Ee&T?_8ZibDt
zDp0xufC_af#+-^DsL-sWE!Ux;H(Ld4jww?=4v#3|WeN2G7sx-`OafjP`Oq;Eqca`*
zZMX~PDQ5992p%ET?|Kt{N#B09<zK^D7Ik=Q(A|&E6PaX{a`w*Qb*)3K_V=yLeXv~z
zuKGPdenLI2G%a9<7En}&?Z)q#?SX6+0bgZ@(2Hs^<VklM75$ywaL$p^0Tm8*u{Ouo
z>XGK+sXDHBccPzkK!5<A2FIvOy_ev>zzsr%GZ#SYUY!q_EkC?6jYZdkAn6pLO4w}D
zdV>(w6Cfizj}G^wMZhe$xkx!tmIIGY?L6*yi0(}_&l9hAD;d&Bn$9~KX4%Ql6FQDl
zUOumhOfdbv1-xY>x7cD8V8oi{O{FtZ{zIlvuZ4bxAq!C~mU8NbfS#WISr`D%CB{B?
zSs}3X+E!_t03Y~_)!z1FD$6KVsee13PJ>Jo7tPCrK-f4MBn{3t0f(9FNXF9rfEBgi
z6zn)cRHhEsz9Z9;&oM|HgznQ!(N%MrD9=BLb*k_?ct=M7Mwio$Q34ojTsYl~-br3S
zkXZZ}Sc8afR#(IqM^S3=01|A@J3+a}oIxY6^r<rHJOz9mc_)1qQqfgq!+Zox-DbeG
zej@?1K1^_pld7u`RqHNUB)zZ{eRrCO^5tJd1U-PmDdOWrW?;|Emoz_4i~fNDQu<NA
zba$9iFyKPAGVS;QL8~p%<3Pw^jhh}c7cN>QxW}wvrG=_M-9vg8N^UuJcYJLEyQxZT
zRKi#u!vhVu26_;@;dj$2q)>H!azj+421f;L^DNhS3l<Fmhs4Uab0yFI2oA>o^!oyt
zzPbY-t8T7?Qwlhg6mfcMXmU^xM#uw2oLmbZ{LxiYABu(TaC@KGi&8BF8LD2d6W<Q-
zlRXHjYMe8S9MUT7IZ)!&Y9Pv|QGJHJr2LpYJ{U)rE2GQS1E4n4?a&DcW@C0Lk>ced
zA3Lzs5O}cIuE#d@iaX@*pfw_37I4^}0Q`gtUV;Pxn&b{10mjnG_1OUU6G^CAfEyp`
zT(O&w2LIbC<&N_c3iP}wFp}Zfh!dI7^|G0<*Zsau%|pcz+B1OH;$VFIIMWp7_N-rE
z&brQc<0A;0hagH+mGlEBmaEVEif1NUT=`XJHJz&Rdo2?%Qq;~=+~<H&{{$GNIv1gL
zprzy=YMc_<`(kqeOiO^NGuL-zyYt;az^C>t*?WMNY>{)^{eGPV0C>OND?^l)YHh&N
zgDw2bETIT-k!5x2eSfa4r@+!}9KN%Z)im}4Vj@R8(Acmg2GD^&=B?gPidqiYDFv7d
zrd={$m~irU3QIB*J*b>}P)^Vdw!Aum@Ci+y*tq&|N8h(lWtbNL40^8&Pkfl9n@b}?
z>IQHaA<bNsvC%r5G<5*~)Y%OICMNkYQ3oDJVx^MiH0H}y^~djX0aYJVQ+fQAImGs5
zL5uB+{OY&0Jzw(`KW{d?`!7s<frGuKy)UtiF2vhD1CdO9?<{x6IxubLV4Q+#6q<ot
zg!=;are~gX4P~rG9U)Q8@|%Rx%UCWLqs*6}GII}wfD1qx)BceR^(0<ePY}^Gbe^xD
zEu$ZO21TLGy==?ZW$IEqn@P&5%N9n`h5>{uhN4Cb^>0hiR}u_Z?JRLP82`%_JX8Kn
zw9S|b-v5Sgp?r<G1hfb{>f&iFVI3p&#{vhwMH5=9LmzSdGXs|d#TSrd?q~2A{qFm~
z;}}Q*SBPlJ6aR;YqL}aiDO0xFXm-ti1N0Bk=AQx>zy`Rw>_3?!^e?<%jg7Tzjj8|H
zAVF#HUabI#+;8R#{pd^tT*GG_C$m43Zpl1K@Lq;aYwF(@FnBX}EJ)3ctD0H=^rRFM
z=-|C6`VAO=bt_u}41>q)BHrJFhX~#)&Abr)SGUH<LCLb!YA@>d922y&9?B(ArtSV&
z^NJJv47U8ceqwj)`>UP*XZZgy;r}!I|Ev#S{U>%(Owh&nL0CVUeFhgSc?R~%|2aV-
zLq;Evx9xyv%vcN`{ox6F4N|F4PpZi`dIGO!A(DJ|tE=EHnElUuzON#3v;VSkPxd=6
z1mCoI>!7reYRV&<rDidZ_!CjB$a_4}gK7CiUFQjQ%&x&xm?lvH{5z#R@@L_gg2bO2
zFi+p$%>ReT02U8z_!Cfg*T{%${>!*26sW=K$%d)?A3ybTSr>x`Njq7n?17y943I_a
zuh|#=_9&6S6J$Ep<amB91G~h{fFv}>I6;akO+Nr6<kO>o6a7&~EC8qjAHMgW0KYvs
z1|GCj>B<PU|G#nvX<t$WwAKTvzv&{OC`-^%$iv63peob~_K{OcXZ%@!mCTcdwT+yj
zEU?89EMfq}+Z>1TD8TSRY`(cKPc_gX%4IU&5oJmy$G}Ml3NIO+`+%fOk)<!O@)9Jz
zP{avx7DtZ66L9E4;L*khXokeI`Da!^;y(nuRr2`<$5W6`B1km@YTOPSH~_HMIoq_1
zvb0F$>%agV`%ZYklbnCCkDw_@16?P6c`^x-=o4^U!6GOTL22$`#+Hf~-8IC&=J%gr
z&5MOrTW*BOKAI<u+C%REY|k?n=2>kgLQp*;fDQ~z9U(dZaq_Q5LAisdaRHD%L5d*0
zx<O0v02-bH008Q>n@#&`W(z2VE)V{gQb9YY_rFT^1pDXBG0RO=Z4$aK2Epx(hu0rx
z*FXcW8r!d<Ir=~?0PsL}NAY%7MOr}eB+s=P1ZwuQ&%h@1S*lFyfQYI$oK}DOT0+qd
zD0eycUMy1w9OJO;d&`IEbNx+^`fYk%MePtRklIm%Ldl=^KR#vtU5Y^elHi9u#)$Sp
z%P*~pbP!Yn8NQ<Cn~n4Rh#I<2<W5p!36)%cUGRZf&@u=(k%FNi^0d3p9f(<>TjiDI
zo^67is5RinH`QH8y$M*3>io}npfp(^sW)1Ps^*;Qg3W!9X4THTP}9LW2ggPj48qcj
zn5bkG7!MkxR+#+7M+3|6-n(;(|7^E@TFFT?AbLq*K*)oPE7xnUd4QBcUat8)(9oMp
z2N}Kw1N27VWI_ZmF?--GZot%4Etp<{0}5sUQwC;+WdUXF7^pQ2&!2eJ7hNUk-^ZCU
z0#pG=xl^Ckg{x0V$t8{C{8bQ^%!37OfahA`Vb|~Q0~_~)xuNwCz_Bg0``!XhS{x(!
z2WDqZIGW>!SBjX>(<Z>H97ZxliQa)>2qzv^rEF?FYLp(8BGIUb2lfHKo5ujjFab!#
zy3zv-4M24ZVY&rdeeP3wWOwtepJf?@;7w%F2xNvy#dlkdTetnqvnPXfPj|fC2NChT
zQt&sgA80Kr$v{dpW&aMyNv{u7p8McHg=J7?PN)V1`Ub%JfWvRh1ume6_%(rj*tImY
zETxdCWnBaRB4Ownl+*#e-`))}goXv%;posaaX=6;r0*;x@Fb&9=eQg8u=YZki6o0Z
z>JmU@l?wL&{{v;g7B&}HP_}L7rWZgzan}_vmPH(c>45rD=fLy#mg8E5tBA)6Ir`Cl
z%V%8_lyVI=NvD6HdSQ;iz-v8MZTKCO%xIq?Dr)gQex^bU%u}+^VmO&a?=M&r4`9JS
zQSmc%_U%W+^B^b2<}p=EfTqIUUAIulN-r$UXW%o!PXOJP{i3q01A2ggb@Dbddee=#
zVH6AP5~2eK))@O*K9FO<pfw*j?#B&s|Bs?+0u?sdSX_j=Ivxp*S#!->`c4B8R73GR
z1t<t4H=quM5;^W1tomB}q#i&5z3+fUPNF{rn@a0(Lph|T?JF2G;zzi&G?G%vt$Y^H
z&%odVTG*f=bW>#HbpxSU?4u}q4Nz!B*|$>1mpyrd`Jkfz{><8)p31J}4ZO)s7~AH}
zlvgf*ws+HLQo~%ipMnQ1O1(F%M^XBBG$v5sw7VgRYO5d#3X^Eew0##o@isXWee&h=
zl>F?0H;V(-b)zZeH;K7d6ppk4LVXZN#b8Gp&p9mwRIGG9cf<Nc;a}&ve0qA85{>a0
zcFoRUL52_AkNfvCU0OE{d<z|WJr((-JAwCx;M(sjYK^Y2AV^dK*-u`^(dJP7BYD-W
z;{ZG0W1n!U9RGrG0|2LO`PH8s0M{9Q<DNCjKbb-R7jG)Pm6*QPuZsPkhJbBBl75VW
zy5OOSd@-A;x{LuqN!NB#xUTF<lCqq^6>in!BkfI4;m%%BdhaPdihBh;gJoGZfENQk
z<3MxAX2C=PG(w-Nd;H(>gX^=+66`oL<-_@=e?ewm8Hf_Hy$fRN^h;6b$e@%5)g3gS
zh+@d9R0&3~Tmm1RN+)sy_|_~Q9*A*bH1M^rMt5eio1YGPYknwiP9QoZg9V296;uX?
z{QzsMW$h3|Vv3_dMoP`}O_wdT1SR=lggG%if@6Rf=i4`Jvs|A_cVI?Z=XT5XN5}Af
zm1Gni0uKVVfWDwg=eKvg9@e{2a^f6J<OuIhknQCjUn~PWaD23&0I05FN=iwcgX4Q7
zu%2&y^_=rUB+2b!0a3ZJ$MqVjFiJ(?{Nzrq)phzwirr|#nozV(o&7~7`wAhOxx(gk
zmZkYL?XOUC!U$`0Qnh+oGl?F=v3MNky;&mOK?G}eAa&E&n?U~r>5`f*o3?2yM2t?2
zaNTlar9Ooxd03uO8k@Ba2Th-k=v3w&?=XFxzUMrs_!=Ip8ZSgrP1WXGp4s48@_;i*
zenv~Qi7rStjx&H+T<05!#6VmHDt$h^SZee!A(dR9K5cVE6XL5`Be-z|h*(i!)X_CN
z-%G8*WOpcu3Qk<nski}T^%^*mfHXIaMd0>)+$rCD`Q;BC*Rv?iBzjfOQciTh6g<}T
z+@=(71N7>VTYfP1+RR#XA7!fc5n#3Dfn=#yN(a=FTfdX!vPZvSGVI8cV**4N7_C4Y
zJk}9rkwurq?(&LSq+VN4*x`t!L|{crn`RK@XHmO3jP+e9ea=jz8-se0?8fMg49+zP
zgjRpQUjp}$*+@;`cswmUG)|@d4q7@(>@JTpgxdnASY&AKF!ylAaMF($ww-U0?>%|V
z^Lg3_U?V8NQ1J#8{If{p4iJv;b_u{iNj<Me8`7tm5TSA_f)H}Xa1cn>fr5%mNKk);
z0J!8L$Z5kyTT|9rHzyzLZgnr3-2Q{_V2~(tv5Fpi2c(<P)q@;H67jl7`r3gEt11lx
zI-RfRBwvB5i$olAzu{n`zcHYxhoRbZdI_k8hF~IQ(Nw9kou_~RZm@EI_7EB-9!2Vb
zql%Jb4hLzEH*)efZC{GV5QOvjcKF?C)`U_s?H!Ab-eK{5L<8VJrb{qa3s7~J+EcHA
zeTGb;_4pVosoR@tkaUkhr7mDukY_kFs6S_w4TwHwFw%wdFGN~qui6IUxyGX*Ix}A{
zNZ{)(oUcvmG3)nMrb<(OakBFnFby%D{49;0nqRzJJa%A^#{-(nbiigM|8Jl&B~$9Y
zV~Rz2&U<)FpL0asX~ZZb!6nV5K@>xdfr!6*V4gX2Jb^85!HX1&%R;s(5zt>@9i~a-
zsdVR^<fYTmRVug#w*3qA*sRg)qiT3KSf6g5wHkg+sLHeOFw9<rs_iXh%L2e%g-BFC
z`iZ>gA9IE3h$yYhfD^cg<(GhLiW2EKwXHyi))pM4wJ8(47=%#w;X81<xC|u4N~Vks
zK<y-k_rYRMrMVkuPa?1Vr{-!A>pANrvFLU%0^wwP9T9cE883{KG|pSnA=No38W&tU
zx~#`lSCJA<2E`|GU_Lt@VLjhUSrkCb4wP^nyyt+pC8p#`LGvDHFCA*0&(NBN+;#n6
zQt4hJIGB|npqeD={(rIemSI_DZM?UX($Xc;H{IRc-Q5iW(j_h3b<-u?NQjgI(%ncZ
zpp+sif(QccbvZN7%shMV5BtM=yvKXIdp>Z?Tyt}C)mrOZ=lT2p&lF(3;Y7EJ+`=<e
zp;81BVarw00Ygr_kfz5A1-P3Wr0aGX)THOT^gP&tp{73sbjxa*Lf$A;<TGi+zz#rp
zT+N=9Lf(V<Bcj|G-m3|bh0j3wkF?G0%z%^64O-(R%TNb749kpL3%}FNrq*3a_)IrY
zKd$z|{G|WFBqF2#Gk7<`(-gL$;kuIclm&_oRs`mECXuiOsnl9Gc*q|vm))nReL8IE
zw_}tsQcs&A>IqQ)J>8aYTNH+=lXP+BWbpp42+iHC);wYc6*RR`q&I_JIkzoK?T~l6
zc-uve8&|MnHpR6koX$An8SU)xmD~Yo%ZySJzimY}B~JMZPz148KN8H?*QDN*2bnif
z^>Gq<rog=L&qD({Mqc4xFe13UJ5tmZ&XTmD-MJ94(^s_RqswB!K3}o9i<!Uios09O
zTTfmb(GUHdhm#C%-hCnb@%5lL^p;^`Hb775;w$_u9TP{lKdLr74V}OLb~!z%0F)QF
zNLa_TsL-ZCNs>*L%$bt94a{;@rK*>t*iih{sP=N=aOoDkNY35+!kNDF8VFZ)x~y0P
z1^!lp<eeQW1Jnaz0~4_X#8}o}LmE_yIxmRslJfid(_kv3Y+M*d>u^QzFz_(-`fOye
zI@!-L(T~{L4ecm5>%x#3J|mFQxzca|_|~lUiyQmq7^+rcZVft`{;R6<KvQm)P2FvD
zsO&oS^rA}Qnn|zbDr#pv{7}SxFegAuaqGG6>qszfhiBo&QkG`<I4FoomrxrR^{@VT
z@Y<|@EWx$t`MiyS>FXm4^AHD0^a(WeqWGsA@K&m?me~}(UJ>}nW9hk{1m;BvcKVn6
zRoe|w!aI=l2#(}&fpG~dd()~jQ!-=Y>(whohJMmzEnbEegnT-;W&*{T5fJjIlO$-$
zc6eAzCB^ud8_HG%iMZ2GwrhNOKSX50I(<?EE0m6rrN3V>nW-pV2f3S#*Qa)X&0QHE
zIr<l)sjR`zejsS}Ao`jrCfBv`TF8LaFZOoqZH}&k(jz8sU4Py$*o+V6l<i1v5zajy
z#7dr|6CB%LFN}07i24Prnh1iq7VCLS`+j4GZTRv%M-Nfy2P?9SnVoUv_|MF#=Op_{
z+i+)$^7+lY7%)k>>Ews%TOTux@R-nN%Rk=kDHIjabi5MXt6yl&3!|9pT78sKDVfQ?
zKG_vw`=TcFy!)*koPwD6e2+w8@8Ad&l^-;WN&rC3^L3%cJHeqLWxcK|X)CmS2O*S}
z9h5t_e|a|9Yq$PR-t%TjUIp>BtC>6yK*Z`CN1q<HSoYMYhM76*oQob?dMIbbbadt9
ze_@*xXVQLIu$5S8NEO*_Ai5||3;caXEUmQB(R2t3=2-bt>&<s-vMV{c%cb%hxx%A^
z#=LB>o781dC-_>cti|ZWPQFv;Qy$U@GKFzastB~&&41%^=yWyg$PrT!t}(^@+^Iu8
zOeXh%0%SxES*`hwhwEYdg=)<jb<HnqaFt&@ZnAQwgOP;#BXHg}rPL9R^R(D^$+hl^
zkX!>(z+u!(C@4dj%86pU+K3lds%nYgsp2og9B#wOYRg@6@+bSXG83BZP_?Wb@sfs_
zZ~B)ITIRSdeiB|mY-vY530mtX1>q=-cML3#+7V}5niUsNRBSU$W#JwV%Vs@c+eGYi
zJx4Z&_Q$L-hxfwO1Pj$7T7S4jaR2yt+pA|>hj4bN%kj-&`J@Ytn>tZ;<s^HrrdB$}
zh_%hKh-}r>(L*SMUO`%8gbzhYWGYl4L1gQd4bSd{N1NKAI6N}b$0QJB+(8<CtitWF
z$SGS?;g7>Gi(&G_yH41OGgdi5DAp&?anWt*nfm}5?|YS^`+CL%F-+X~8hIn$Z9u`U
zIQ7l#*afeEq%vNkZFS!U8S;gZO_mxtVQQZ=$|@aFZEbw_zKYY>Nz1}n$VCJ~yT@x%
z6+>$MIkQ-Znq5h5<0iJ)k+BPU1SO5R0dRXYS<!|lLFWpwTt~^>T5o!7K^p;2VAESi
zcd}M+D>d#D#*0F>g88v*3KQ>f)@(;$t8p-_nHI2;m9W`d8TKi_6aA@jztU3}&2{-}
zS4OT=!|4zCfvS~5AUiGw=G3eBmMeI+TL(LXS`jkC@%ZYiv^~VA0{VCp;Ryj2w-C%)
zCZ2pMH%(3@nNKGe@CNm~=XA(E35^I7&$~Nx`3#1BJ3O4>$G&a6TyKu@r3b4;{g%tV
z%F&8&p`^^q_eQZ!)OJcTl*TN9gCs=_a>8iSZS*`QR6m0f)0iLCPVi1QGrh4_cc)iM
zZJm?5GK@B^Xcd?m?uoB_%`2*Fbl^&JT&ao&#E~^qM|n#JxVxYV@jY%)$wo1P=K-&Z
zgAPri&kr62Gi2@hMYfklih8Ik_Y4gsa=9m9WH3P|i2zNTg@Gx)?&=oP&t~ZAwxS7a
zHdcsT7-xFio*M1f-Ud|4trzeGYC(^FHUn0o&=&p<({LB@iIlIt$#s2i(<53hIS63D
zOVXl>Z;V9u>w&tf{ZBrK8ZV(n-r29CR$EnE$8#LjyJ?adNI|BXMMM6=j47Z}G}?>;
z6L`&gp2|p0Bly)x^<q^vzBheJ)#=DdmP;vS2)Vav<(Xz8N$d|Qe$H3K33AwY5x2p<
z+f`MDF+-P?H7k+-6IGr%T(w-xg(VTICVGj0)x{{hklAQVncvWXzSa^f19BBMYihWU
zzq-FX?)vCRitnNL)|7+rwuaKQb%4Ap4K2vE$PzG$5{!=g$hA({B~y*5uT?uFqFd>U
zs}|k3Js3-)jZMkRpS;lF+)BYm5OOvkse?r+F8V!4`hnDEuw4(f(+8GR>uCq+8Xlau
zT-7;m?XYi~>o<C?(_+Wb4aVd*NH>5{w%RqBhjY+V*iV}CBK81PhD88L5;YAT>DV6s
zXZAu81FA{fT4kr3B|_#TiDtrE%9cLKBqvSze57jViT}#*hd~QbF5&bJO)}RVri7Mp
zeaPWUi>xEmfrmB;mCl(FgE&BVWF-BXf6U<vwP5omeEkdOYu$7r(V%^GySqXOTjjx`
z{wmXACLKk*rKmTC&!3vp_s30<b}F_Grn9a3`xehzhs#pbjLD`C)7Rn@TVJ?fr;9&Y
zIpW=p>dYHt9Nr>dFYtWG87%Id#K0-F=>J-CiW6ta(LmXg*b0F@{w{h$UVNrfT5cS#
zi5*<GzI!JrKVGCjBsW;0wcVO*@N;&@h!bZ`f@1kZ-b9l-DuS=j&g}uaJPM}(0PWcJ
z8qozu8AEoC(AJ3;s;vaLxEdjTlB1GKs<k&Hm=^w7!R(Zul$oKHDUuL5kd{8=LO>se
zp1?Clm=hPPBR7}{GQAz;TjiYMeW4c<7XIcVqoffTLrzEL6E#Qt469rO3%D&&K|8T~
z-@V6(_FTyIYiwG5y5W_<ws_7NILg43(cHxYci2G}K~=e3-$<ace?*>1$qxbRd^#Pg
za<Yw5Y23*dv0pxn(Kp;$LYr#qW?Tfjba9>|VI}|`Hpd;{VL*b}9=PF{=H8?9gocBD
zevoA`UI$k_*Zbg#ed4uF7?wYv<Qa`E`%G~oL_OARjCNn#MjD296>PDdM}6?*B<VuH
z3oLro{6szb8S9^!Z^zqUhs;yGR*AfpeCwvp(13(!rh`^Kkx@Xr;2Jga)hdXtF~m{}
zTy!qlB&^2A=*J7?q$Ae`yHo5JT0`31#&}B!UdR%Vo|24b?C;^&gTRvZiK((all_T1
z5YW8NNI__g_(kWy^w_fQf~j}Kr<1(L8Mi?3{^YbQ5pd3_%6mnq>uv05)G~#$U4Z^@
zW~Dq+W)J&$x?Z3t{@KvUreXhtwv06F$vDj{iwGKi?Q@kGy<-8F&K^=BkoF94=#N>}
z^y^6XWoo9v4@Tf0Vw~pck0A5R+^uYu<TWVClc9-;S%XM0Lk_afYPB_rxt<o)=u2^3
zk){keLQPI2JdCNwOzEys@&9=};`&I`oL}3+fO{uRYUxuva(IW}t~ex&`cY^dpz_FI
zSqN8@f3BJL)kzNsu3M`4Gkh9t6&!g^R%20C<5;cAsIWbrHJINjCk(E4W?4naWevl$
zu<%yw<i?tGU~+MM6^k}lo<oM<lpf>S++?_IG|6x$EC*d_%ZO|}JQZ~{ajwahkQ<FP
zQgA!t>fr?3#qy?Fs|*!B$#Bmx{`*{v3$OH1@k$&)v+VmN=S#WF79AFraHDQP%f4F?
zE(58r(UCK)=RBi`=Idfo3t=iiy~#oKG}XYZ)^;;9mz`XGGu^rM<hppRreN!4dl%(i
zxV5;Pe_5&r=U(I9va-pMzFYsFV!m*CkWb=AtDjOXNkYYm;A`L5Y7PPZ$qk|{x+Ny;
z9fWIuMZNRxNLz(XuZ3i01on*<|D$6pV~;Zzk~QWacl1`hCd?0g6^Hp8BnPcJEstHu
zE9DxH1p#fbol9NLC6Yoe#68&#B`(%{fbtm;lTUAwA!aDG9u<b`;Rbnc3^#GtcMrvr
z^y4rDeQ{(fa@EHEfW7Z5ZaS52Ru2-wX>Zv`qfNN2DxT_jE}WPnm4mbt4+hma$g6r9
z6kMy9(n=qdEXP0IE>v;UdV?KV=Gm;uCj|c-O5o3|srF!qJ9Zo7qKUSX=J&9v5f=b2
z8-b|cIdUvD#QYbvW2^+KtIBKNVc|v8eewz0bK?s^LZAD2Tj&HIc5L6K*w6`?LOW8|
z%6^t-*y#CbOHoO7YBaI0wiD5xFuI4X8WTZXNCd!0AZfi9uO~Q|HS|4^z%t(`NPj^;
zwFbRejMf@UUy7U?d`;YYG$pWFP<w0ov)cSXx6<h^<|lii{54+FZ{?{K!IrBuT`8wl
z0L?OcRwDsFlls?elner@eiE8vb>XE&WQ5YvU|-=lx%|%Kr#1@e%wtx3qD;f4v5l$+
z25r0oXz6FOGIw|Nt?sFsj(gpoy8%i><vrkqEjznq%1U_xIUIkfj7Dw}_4MiX$vF-k
z3~i*be*^ei!N6_Zv7ZtKut%|3@>*qeXgPzdkmc@6s_J8~kr!w%na&dUOXg3CDEe#Z
zTb$JEg$UqoAMS}vc6J?q=YjD+x_)DmOUOz8fHT1bQ^W+g?^$f5hwmwJH$RILz6%hR
zJF}(5w`JH(lN{apLZ#-3m=^Yt^`2aLSK8=BkTv-P;lW9&WXU0JL4`N+7uj%2R=Hf;
zU44(T>9z0MvUMZR2J*9EnGhH9bc*Os?&dOz!5eD?&sY0d%MsMW=469s1V8VY6yc<J
zgKSdD%h|e4B9QJa7K{o{u4wh2c@>^!Ijx3W(fLBdi>-X1o<gR-be1`kgu<u)9#S82
zcAhf-WdGwkw5h0!QAzB#v%<a@Bd5I8txzti38H&SK5u!%^;nNQgJZD8?#roq@~U9Q
zyYz8vuA>AmQ1fK?$$*ySQE6-yLC>@M9?yfv;e31WwEz_kWo%uXvMd@t9U6vh&$ZII
zM8}roV~`vmpo-<q7GZ0qM<=$KP8$3j?M>0+NVrWM4}E_Gc!|c7&~uV%$S84<=hjE%
zlDqhhqvse23Zi=WiEY)p%JVAp3ciW4`!|8e3}%+niJb($Xxg4$+4@<4M3rDIfwK=N
z)HHe$b;_r9;1*X3-u6tjjSV*?#D0KZz<MWXrL~5GUxbN28BrAlCvph!Jzf9U$?I}F
zorX-yj~J1`jVO)xO%mzU;OJGzftaw|I=vH47D1gebTnJKPrwl?a$U|@zdAvOvD@1|
zMhiCM0?(9}w0@?Z=5I8TubyTZIfiINj;B)iRc$J(Ny?r3t!G+HT7T>>fE^3?Br#WZ
zk4NCkFcQ{8ltFg4M9E02Qop&fSu(`0>q~hrk1DEAdEYaC)Xu*t6QB-%sT6jr=m`-g
z1HFpPZjCl~Jv&JdF!IhDkE#=z2$U-&+KouIt{-@v#zwkIhvTI-FUOSTntQFagF`&X
z?2rb+YfDA{V(Zr3Ly2{i0n1JLQi0spo`eqsZQtPsdE<`p7JXuQO<-P#JB#|zaKp~a
z!7GM0$|$iSZuKnElzwNHO4%sZFmYJ671)g9sMj8~r(-^uV?EA}%%CRnP|P;9P!p5=
z`J~Omm+hD}9?Ko1gk<n58N0dCnFF%O`CnM#%lnqD;HC=gPSzS66+o`;{;f;ln8MQR
zOHc9b+wad$W^>>ceXx3v)M{D}tlG1$K-DO5o#PYAIBY_XgW_aTDj(E|hkEm#t+wf~
z+Wgss0ce^}-}N>I%H!W<@!{8=xmC{+F0NXV65Q**%-}*#WDvpya*0J7&e{b|0jCb3
z)9EHmG}RC>)yTJPs1kj7wCwr?B<vT6i;_g^pIhQ*dfE&a;G;#Rz-|fzNwQg^f4SG!
z${AZumtsD1CxWeD8lxtTQMcNjWugVS^e{5|(#SnB;QUy@Jf)6Pd*>(hy)K*)k{Acf
z{2cez+WKvzzV`|Db5I*$CdneH2$ntymHejhHdVVfRm53vYoQr)!A5Mi`D#;_UNgL{
zkB{*^qkd?^92}rxWq36HfsF<e*8YM3nail2zWev2{C=ZTHAJk7^m8m;(O7)U+rhC&
zR&R4~TRRR)6Rc5~_GUfPoZ=%|XH?(D{Xr}=(QoM}1F}F5kOc-t*=x*BCcuB;fD7>m
zHDay&Co2EC%{xc$5f>_@43dSX?)+V@>wl}ST&2LeK$P7t1-?rN(O=?(8b3wndlyxB
zo$K@Vx08G;^f9uc)^r`3z~X%lSq(N-JG65$!B5&S@Eb1r4IlX69auO7*Fm*p5R82%
zq^oAdU^ViM6)nSPzg5F#s>u^=;FUk9L4(QBZSm9tH3>&Ct{~5(we*iN0Q75KfC=L7
zmBiLSEK~`*ru#8?rDfN9G4=5B76#4Ba#y*y4KI)yC%LY;YpP%MXW*AADTH_%i&<ef
z2<=ofJeqa!_`+$CHv+1<11n*cZljX%XiU<U4bO?CMb&k^E>Pn|Q&b7l=a;4az`7o<
zPeCKHfRmBMJ@}dJm{z;9(c#Yu)D-RPm<H2S*_s=hz<XvE!Szoq!W3v%lu$n8SCf#=
zUquqd5-H>$Whb1`cDG7ugVho#*Wjf41^>CGKj5o3D&YP;YAU4ti|_zf3;mzjWl&*q
z4Gtjx|HLnpgnqu?_4^WL6Hx1$hsi*EeISSATOADV1G1EmB|U?)rBxx0-!O!uz@JFW
z@Z!Ib7=&Jn;KQBar6Uy{PxcqtVZR=nzj?4f3IoC*a36~w%=M&!+3>-(%NvvyJLszd
z5E}w!AC89Jy?Bx>)ZWBx*_E~#7`;hSAou5^I>U9xatOYv|F;es{H7h)FHVGUnf!6H
z|9WA<kUs!utUjgq8En=HSsX|-XC!|0yf0@dU^aTA`ZviDF0KKd+CE1!ex@W6s_5Sx
zIYh?*<;dBO40MDVNq{cT0NNa1VIU$=sL=T*NGn_b@K5320<nX<KJcOTA3F(<hCnOx
zsQ`c*mC7wZ!5NliSo*%E5jbfMBHa~rzu3)Qn2f&XCOAUwwYXV?^?P*xx!=nH`@Mci
zS9QeTls>4KneGY1*KZullIX#=fvo|3CDffjNSzHh0aU*PDt)MJ#skuDK+1dpZ8|Eu
z!E|fTzsh~&`?tKQmI2(0A~*~-`ve(k;9x&JjRgd_my<j*1r=X_cm7Zf*bCV}C4hgc
z7PrWLcTp<AKm=SjJM5|R-XeXj2YOwlb1Oz{(cie=eJJKk%idH{Dfx+P{qWLMY}qSa
z=R~)${6GB#z%|}i1UNf(&w+Yp85$L?LL!L=sNXf;!izCH%7Xact8X*CfVTy7NTv}l
zG0><0_(WwIDD=iGmYsowsc03k9Ljv$v`{Mx^8p|Lz(5l5xY`GR0PO-27>mHeiiZF$
zs09TqppgwdtFype@&$$f(|mC3CLoQ2<Wisz7<zeFMT(ce0|4-kg!+cmge9v`a5wPf
z3>Gf)8Bic{)}U(s2I#G{$DocQ04Ns)RY<6;4^;{^yArL^<cL-QJ>rmSm!&%iCCmR{
zvr;89P){+C<fa;8?8>I)osT!;@ebCJ$!@NpI#)Zul%Y3d6wApEtYeLUvpHM@svxg5
zr>cqpW_-X#U~5G&s){~=N=st~^|41ifJ_D+43!&9<lb{xsMDt6F~Bi%6EZEAOznqM
zea)Cc0Q}b@<LUP?Fm{6JK4_-<1ZZCI7;;a=ue#qn$wK&8Fqd93_tl(aV%8zs0l|HV
z4O-iYUyBX(`H#VSp73Yx-+%ac-@<dmMR1AE;hHNrJqMtIHz4c_z}<XGWPJ}5+lZU>
z8r#>LLA(HI8gP)vfD9=I!2%RhK{Jsqpd%ZK$^%3acjd%U4KPs1_yB2Iw7ltt$sW{G
z%S2mB4V44NMA4V4Ut75fK+N3w`l`uu#GzW?FDfLF$pwPg)`6CYmy>@Ik|u)-5CMY@
zxQ#rzIT;WD70|&x@tYs-IGe4X+xlrTbMHB(>Hq{vG+GTC0POOSyzQpk&4q73Jk~=i
z0`Q)(RgaPTWj&#Pu?Ao!&XW$i(*!Dsm4U{9+oRJ=mPG2csOB?uDC-9h%y*!Z)F`x$
zwDn5EiMIhQUS#fyWva_rgRrcWCN13ag1i({h!0@>@6~~QS5DBH?3BF*^~G#4S0Z$>
zkDLLJK}%^C2!=k-nm0OE&!V=99HC1(Q!(;QS6V2jFWP-m%D}GzbX_5Yu{`<?Aam>(
zbKM90y{m^L8m*7wzJIC<ndYu9e!>O@OK`p<ODMxn{xRPoGnvK~smF+Ap|IHzv;qLB
zv98&sliY)LbBT|fUd5``=YfQepb+B=AZVg3tYr&#I0Eg8)HlJ9c?6Qf6Z{c&gzp^Q
zZvZUsa1qQ8et?zs8dZe&n=fx;Tm9dd1JR`awx>T(&t#s?C8e+S&XEa$ffwNRf)Nk$
zt=J{_uCd|P`s|;YBHk=8RW*ycsv){6hY`ZLTFR1znG<t6!;jk*)PYEo245)to9nU5
zb8KAyp1Lx0sW=EPG~Fz|K5}5)0<_9KKfo7xGrwmSZv(-gS}qgUb|k*_<6WF!f3OFs
zx<W@Y4S2`;B*mI@^%ZckKt|fi*tusn^H0HsBLPSW{qEwmmVJwb6QT!)W1ew}8YI}r
z!GPAXGA%`um8e?$PB2Nd+EE4Y^}vyzFntVw1xyUqj06t(2|yC2e>c+4i?nX+ejob?
zRDJi=;i9%lj_>hHeZkZ_JOIfxlwi%-+_%Jmajw!h@1Kr(zZZ9R5STa%YTwJ0T*n2C
z0f=AW6MnRg>aS>hqRx_ebVBgN8gk09eGkydn<`3)!80T4I9O_)Y;qyclD4vOie?$5
zKhF3B?6dR4^i*3f+j5L1sg-f0vqvRW3nZn2Qd^LftT7?oeXU3YCLv4S-*_tBtR$kb
z;62oZO1jxfWEw-3#~4+so@J^gI+Z88%D%ZXd%1T-zw`m=Km65jOD@A)pkc8RDve7%
z)NI1}Jp|qG{tTG+>D0)Kd)ijx7KLoyA=?4l((J&&SIh3`0E^QksMB}_`T2oHK`Rz*
z(infGouCD)Is-s}!U)D@h%dtWzzj@Qb_LcSX3TRid8M>sdGOkO!_&8K@y#TBa&#2{
z&01jVEn8E8fw+y=>G0#rHqZ)*#EkYH_<?wDaEfn!_eLFbDcMyg{MPI+{~7Y<09+}K
zJgNvoe$&(PZwVYP1wr4AUzCr|?=uSNNw{p()hQ~(>jJ#>ez_ukdAEvAU^We6f+DWF
zR99HM?jco4SQvp*cWJ~tE)JoUl+@vTIYu?XdGN;PpCWtqm#TZ}kB%w*c*ej#47uGG
zdu?d=^c!GOXQE@@+~h-NED1yW{eumi#JwH$;28ZhtPjZBNGj^z=<~a(xjrzF5E#xk
z^F6t1G}m<o^_N+sL6i_b2Gw0?N3>L{8#5fvxDRO6^-C!)p-DiUnrPfuiFw6C+LFnt
zSKcMM72I?j@6Pw;e`+D15NJw5KS~%8@)wC<PU7g^O?|-c2M?Z=NUBnzS5vYmIb2DE
zGz}C<dyQifs~&-)xs?r*3epwUlgiR+v})!?GgLBMY8Xo?@bT}UVG?&cn?%jEYei=B
z4fwB0)C!yT+;)sIi|HDZJ+GwCf;MXZyh3!w+4wL72bRF&I>{MSm#6)=GtYoz)E2q8
zCF&)#@^`++<HyaL|J>}7Cv2b{KF0X$3KEyt`6<baR?@$;8BU~zskwp)DY~VblaPV-
z`il(yY73et3mD;d&{K!or(DgzdtVA;+x9ywXX+db(fxg`&<{uBzOYs&VL#oHY~)y;
zsyt_AW!1MskObJk`vV>1t7Fe)bqz9I+|cBU$9tOg#F18v&7f?--p!A`FOiRF_el(e
zFpzGvcmhnbq~Za)SZ-<<Tt}4)<?-9f2Gw1>3CQupm=pt|Vlgl=yumo#<NunLH$%b|
zJ1bl(tg_G{|M&g(K|q>b#+s4}r@+YpiPQrgC8^wu3@c-(an8MmJ!x~^)hke>So76J
z)2Mwu!SiUPnXTw9JVq-<DG7$V2_{%T4y15NpvRP<v9tyco92NP;!$2aq#n1uWs<Ct
zV5HBzhST6md)?vjE@v{svjuMTu(h9a!AV{#_a>4$7U}@DDC-M|E^@vPy}^sUrxhHZ
zz--<|1PPvNw!jWeVFp+i>3-`3F^nAcjsD-aGp0{7VQQ?@vt&qh&BJ=HMv@Yh;OF(u
zqwz2?2+rklFe-p>?ih(HD>5Gb3M23w8Pp3DM=ybGNCmvP&tJm3r`+Z}PzR@N%IYiu
z4e>6BnPwU%*?HI)&I+75sQ9_caqx3B@$V5ftPVC$c1U1;$!JI|mt+6ti<ezF*j8?b
zKKo-7ajD%&=Ma(l7>%Jov+JeWgZLMDOBjdEwLceI^#%enRm;K<AG30%hcet*?QGpQ
zAJuD*Zfzx=D`VmGM7$caBufXmQ7<qBOfKVV<9C4A2{({SP0^`s)$}5#Cvse#nF3A6
zOUOMILX@p-%hcvM%V#0;H-d3fdZ5Dd@#Ua-1BD30uIXXNm6M;wRGAawPjAlmf8~7w
zn9=)T$&OAn!KWB%t_s|1lSRoBe+~J*EmXq+A;H&cJfFnErQa~?34a3k-xcULW$C{H
zLEJCX;OWmgiUL*aX#zkjY2m+(Qi)VyogqBtcw`K7U=}(YQ`$TP(X}XOR`y&_Zx7fE
zE2FS3wJv_u4g-&|ySk%moGH`I`Qy^${b`$B$-E!cR``y`rD2Fju-|tCy}eGoJ+;oI
zZ&mO0;wwf@)sj?>i>Kt*vqIr%J$V#$XCLdWK1T0>De9UH`h}k+E%`HzsdWVpBbTQb
zTvq>_Oa6K*0u{j+i($`_)BEel?+*kjfy3F2S$6+*`2X<Jmb!Q0OQp=<WL~i2l)Xrr
zH1e-sF8sOl)Q8mT&pAyU;ep`CNbX!Pqt2%H9n6)Pti3HHWK^`A%;7@WwdMcg)-Mvd
zQ%tYPS8A5%AVy6FFV?(zWcuf{cwmE2tbO@or&5RIJ+sL`uBh1`-~6BJVbliKqNw&w
z`#(p1e?SrOc0t@Ll)om|KmKPp4GN}U!Xhune;uZ??8{{|{r}q+%M<421~7rAhs4Th
zKbZqoK}^oDszzvSm}L*2D^p0Jlz3^@ncsx?iGU=J*GZe@;w~0e^P_~^Htk#IdL=0+
zS0QK1LrEW(Ozhg*em}_<G_$TV>*5us<An>#LKvOn%kJ%-Xd4D#k{!kHUPqF885Ht+
z$~xdfuyFjoM%dajfP6F{lsItruR-|d(j<pQ0ups#G5pVR{QKVqkTC^gJbp<1XV(7x
zV8POfG+jzJrSR{=mN{Vv`<&Pk|NaVe0+?l=QD*tYe;vle=iDM6-oEo+??x&3iwWtx
zl>c?u3cOzff>+?~HDGK2v;{-$rV}^{n#@d*|2+l+4J>s(6K$rmlt}uaL$;BSD%0)}
zXo1<)8kwd1pR={JMXui_Cv`{x6Tv?&7_$JJf1jDBhT(rc30*D;DY)7+kME5C_$XsZ
zUmv(**c-6(NWy=<$l+<Y?z1JoKkaS41;i(vK$ycc0)Qu=iMa=7n-x3X_y$<;NHtL2
zx^V!C_sl)OgIZl{gkD1CBp)Ec6BPFBfm#^o%>#kj+6d@N-u}8-YigXIgCax}B15X(
z0-_MG?FJFP#JhK(xhOv{rKmV-?(-13e8}+Y>L>X8%7}P7pb(%<0!U$~u-hqzf<4hD
z*scMuK*6YDph91<0~?h6L5b5Rcj22u@V~Eb2t(ER69f99F~xC17~(EU{B?+0fgl(6
z)nb*n-i?0-y=&_ff7%|HsTG}K3DH)pgAYGL&I5oSR*N$tkq8N*gSM3mm!E?_KWYY`
z@s*u%u9@KFK^ZMVYpw0;)*uk^tAn&+MNARMwS<5@>^|PNub>=w(RW8&Gp~p*vM>d7
zhC5`Bs*j^3N{8qE@$#f1cYXVneaxyS?ezpFT)MG^ub}_nA<CtNC4}DttD%|!sO#u4
z*v~=~o8L4*jC^&&wS78AX%YtYUoB$suWUg=)OX;nbOXN)4$x6cHV5*(f=<5}*qH*`
z3z`Tq;D+ScNqFV~HabFJvoz`oavi1fK5?+DAK>1GfUTV;5eIPC1iNmJ0W1O5XN%g0
z?vN`g2sD2D=)W7{4t7=!_<DJe_XXc;({DiPtEHn1XGMkR=e41~C(3Ys@E*0JExQmN
z$cF<la*<&|`kZ(>?-rdWEuGCrpa98N)}SMu+f!mF<n=vTahckN91B4^oV4`&>jdtQ
zcp29f?*MOgTH@E07N_|dt|6o_3I)k8;F`*|qZF~d;tf1So=OVU<XlfWh~u+dT&ILy
zJ^tRty8+?v()U>jTb(j6{(k(fc-5QVYSr%Wn}D}JAe^mzg(i>_k0p!Z8r(4l*fkFT
zXzNy#O%?d)Cs^vAz5+~jZW6yy=bs*3tQ%kllJ;`t?P3}9lEP+ELQcyfW0}@$r)Fny
z<{q}aswnyfcIC-3y&#Fld)qz$qJrh~>#1pA@=3P6pvR$2|J#3@4k7233rkZj%d8oq
zcc3XqTQDv_j|ytnVIo-C4Oaa;zBWNvUxM18Ct%l!0TsFrZPD`u*)({jcv*Xa^rx*s
zjU~Z2hto=0HU)~$U>sB5A6Wb}#^f0MA=wOf&!ft<Tf<z~<9|K@8gk0s(P9<?=11on
z^b*j5`^U+&$$+)*el3Xg&e3OU`O%5X<|-rbqUS{6DIjJwGluZtVzCQ`rGQE`Ou|Q?
zl6G_K*Q@dX-Qow((31p&9D5bB<O`L(gHk*$klvgWJvIuuYs~y;Vc9^=1c_%xdAd1X
zNRB0`*RI0ZS^oaxN`2x5t`U4DC|$Z+z<fefCyw(N8(|;xu7xWH0K)&NYF|45^DE)2
zC|!Qsx_70vif>LCm??n#4aj@cG=H3cZzl=Ro>0EG)^gDcXz=EuG2_zqZ-8mHYa#kx
z@#f4VBb3D24}2HhV)=6hS_b&k$UkO<PdMN?#MA~b&%QuR+*I~~(ca?n#rBFH{Lu(V
z7K&C=ZO1OVkRJn-(^*ACwE%|#YO0@Rdu@}?^6jW7FIA!5l~a`*?zz6bX(nPZ&^Dfi
zOJ+}dI{=JX&?!|S2xJ9}F2L!S30=h4O`n6dwKcq0PVcuqOA%Sdvc9B+jBBZA{g3~6
z90ZJDVWFxtVw$~&*j5YMDzDe|l<yl0Yg`Oyp<gfJSC3A!1aqB#R*enhyrYJ%e;eHx
zm@wrCL^T$H<+hQ5EU~`!SMtm3Ro)sFCYd{Gz<YO9tzts;X|6sVPw2kL+Y8{$a=rfc
z<~x)Nen)W)dRhbkf<;L(@j8m3c%A+)F9439a3v|A9($YlGKCnjjdk{Ww*ln|YeRqj
zur(3l3%r`w%b6>;F4==ppd!6+%saLg$BZu@6dZwl%t#?Wm4n$K@1foeVC^}JNRybD
zD{9(83VoIXKvf!ahECzp@u=W!Gyn@ype}d|6*vlp^h*K|B2@QepLo$C?1N;Wj69e4
zC&)HyoVm{STg4hr=@j_x>~L2!!B&$5vk;IA^(56cfG(glojSo<aKkj>17%!lu3si)
zx>nRgs$#(e)aQ2uNZ?Yk28w?SGSu$R-Z9BFgjb8GM;ahO2EagQuzF`-?}!O<(?V9^
zy4H9<Nhi#A*8_O-;Ev4zE7-@Pw+UU>fCkp8-m%|Yr@j||_VpGJt?rTAShmY+WSJkd
z30r+>m-lm_dGXxsk8r#JRnSrM_rACZeQ;JvbZ7Q5GrDE2$QSs{jrqV;Kd;pcGTw8o
zWY6TIwiYHz$NdRcD<GS#Y(Oq(p3E>c&NbP>O$v9ou)?9`$9?vO?}CAtlk*A2eJS86
z#vhWAxTeeO<7ITVI|BBGG2g7cVIj02%e;TS>(uVA-A0n)m_f`wSvZhDGVI)#%kY^a
zS#T#d=lq$yHU8=ITNaI=s#<R_J!kUVpfPh-x8xy+QGV`e2hyPPQauH8%Sd^isL6uR
zH-HV;?Sljif3_T1q%R0Nrd@Yv=ggEfyZD~d>($S>us)AV(M^Z)gPmu*DhZr$f2!w9
z!Q~4~4M9u+<>iPOTFw`ppem`IcO;EY_u($QwflVe1ibZf3D)D2gZzmFgNRsED*5c^
z%SzA6+PEmVADW<!f(^5P8*?7z^LLVatJ#Q&hVnAk0N-fin3>Li!gOO|Fm5+z$q5d`
z5MJHw!qjpj8<&4%EPspHwO!e(Ur?Y+d`~@{S@t$%AP^?n1nY9{&D%;$gGl9T+Dr4!
z{ut1?1&a)p)eb9*`wSE^<K~fRnAIj^OkAw9pu_PUJQt9Y)e@M{Z1W(n-%CtR_{w!{
z|2^AKooS)+#)K*e{R2nvp$a5#WKb;h#XIQz=6ab}iu4WSy>OB43lye^U?<;_Nt=@_
zRL}28EJG4<eSK$tLre#RfF)WW_L3h`ZIUcTm91L?vXI1;L>=aaH@0@(VYa~g!nb5P
zRm#FCKrv$p$X)G#6seT@fivTTBMVrbwF${8UqbzMl=_bEfpiPFqJaZed<JzO8pg;r
z^&A6f2Cnu!&mUk|kEycN+lHf=Ri{82)#{F)U=f$(BOvs1$pd0LQawgrI_YF#RT^6g
zWHdJ+slaS4er1(>E6-T7%MfPuoo^&Gv5mE0F+}uPlOjlRu~~g<6mI(H<cL0-a1@PG
z5a*rQf>Xw8tBTLBJmapb!HR;}5eAaCKd`nA?>62l+nWT<VUbf+tHwlew2-piQKiyO
zmt(*(l5raBo}YN%Bs~mz%$OFazMPVKHGs^#%DZYeucI;+MCa{GCUL>g4YzOB^@M~m
zt8i8xJH{C=kX07?<~%RVOBQPn?7*jZZTW#_x&j`Vc{*sLNEhM@#%r56n`XC4&^@;J
zh4CAcE%LH6Pty*&ETd>YWbe$1-eYK254>>p)N{u_JjND|g+sjCV#vbO+3A9YLeck%
zQ<wT&D!Wk?gh2jzb$GvjM}NmuwdniX+n1Fra-`OM%=A<>^A}K^B$>;pV->2fc*inh
zh+b>taE1$zwVm%#Y|ZwVT9&Xzdn&!77&+Gc4A$ID){#@89t11inw7ye;SFQElvfkU
z2fE~X2fPXzj}R2$p7{GiKhr1y(?g!z6{xJT9s0esSSth*_uY#bpeIncsMA=MhTXFu
z8jdaXFJLk&>HPKD&?c$HK`R)BMFc7hWWxR#_s%gbNb~N^4i-MwT^dQSuQgx0|0N$S
zV<pPz;f08f9Zq4O8=R3j)WqLmkQ?jh!bUp7-@K3$kL2dch~xsH&mZ@D=S(pBgrb;G
zE55^P-WQW^ZEwEp%!u^~MBRMbxMeyHs*yDL`#=>ZO}6~S@0m8OJEF~-yXn!YkUWR^
zIs4lMm%X0X?;fHE+>+S1ySM{}Ru>Rj;D=_kj+sK+whwq)zxMP)c1RlDZ@olk)Kn_j
zeB0C%w9DdwN`cM-4=;&|wOg$I^`d}Ac(*eqM%4ESpUZ~=r5AR#(#RyMCzlf0Y#OCd
zLu?zC-T?Ra%{hmR{v~nY35H`fc5WQCjF6wd{J5G(OW$LTg}XTO@A1B+DYTWPhzk$#
zJ8=}f0s+dy527l5Otxy|)v5KyGm(s9;{zSaY4>P12J=AXs+so}FG!q@$*Q#xs=C?A
zUFM-F%5*!{j(65Oy*)zIXxq4TI15$JTz2F58z$Mm1d&q{2`Kpn!|lj~<=YCx+*rNN
zNKXa3AFiI`rsd%9=5u$esy%YrYmSb2q|NVPzW7Z6TjFvq#oy+YLH#LC&&Cru?{+$;
zTuleCh;eL2@mT;rxz0Q$J28wPsqthSc{kb5$fc{QIaZ9xKOxgS4yRQW4He`!`nKoR
zoli`nPMs)deG|`74})^&>9YeA#O2>2D-W5Z)1%{X@g#Kxq-nMHOUQ(B`J3?Xd&^2+
z9$g-tfMgeirfe18!_79@Yb~gtOIJYTe>gCamN^-fMi5)anXXX3Z@6hx8>sA-cZq`D
zIsWT@Lo>Ws8huU~VZ$I?oamtFGqIX>?s$4mY86(*K)n!OR%C2XoWuCY(IxZ5VhE=t
ztB;_q3C!q_zIH;8$mhPO^$qa4WL8Tx8+gog_ds{ytD*deob!hd99I0rm)YI>fz;%D
zT9e<?#*6J$VKmAt3)1{W<qehvDp!?jNw$3WsnK8PKPg5rvo+T+m+RB{+%5wAPCziu
zog^VT#Tk<@-NYklvZyHKhNDa630+M4yRuQ3XQgd0buk{anAom`<9EYJ@0f~<9Z9E6
z;A6!%KQa0;S)cbE%C=O~IlOHnems)e5%*0Qfq`1eR>e`F>?<fo6bG`+O2Y%4UJ7=e
zd@-Z)pxbiA-h<qAFUmh?qML?Jj-9hCh3Ld7_y|O|iI+5TWB1cNpNr*<PeYsrPLYww
z1Vy#{UFNOCKZn_`*pvFny7UC~(FIBCTtMieF4QV{-(|X-mQv@3L14>SW=wai*Y@q?
zWXv9K%P%}H$-Mja>N2OZIc&P}BENbJWaMOH+oj;y1beAnJU05aoJ20@NbUyqaXTu5
zp9br4M`O-xhb(@!jCk6apzM^ko#Ck2zA3D5-jUs?h?fNTNmy(A(|m%1r;^kbC)mXU
zRQF##Y9#i_(IQzJv~RjqC}bnnbhqWBk@{D<cdqhS_Ym$j@YYc67D|hu`IF-2;upvs
zz_Fz&ury|x;oGq`VhuD^1)j?K^BCFBk(fM;V})E|k!qf+vaqmyAt<`sZ3KSytB9H*
zJCmsItLd2=O5+^knZGLMW}8u3QRuAgwS&>Xv?cqvT9DZCTT#?r{TM&Hf-vLM+^X!x
zBT!E)*NklFaaDS(GLmEBl&y?m8nt*U{QJzRfeLx%fxx(AS+sWvfqP3_cz;VXqk__7
zIs9)9ch?RYUcF@6O00ckJq;PEp?kTv2>H2priSrE#vK;{&#0r$)PNu(MM*QeHCao_
z?vgb9$|#bMTx9QLMHZuK?~g5UZ+@gv4~$j}<JRxTFYso+#&W4B%6RC_FO#hxN!2)k
zN?!d$#@nHWOc(|KH38#$=6mkT-u&T+_wgCutGsERzxyT}yv%ZS>Jd1`#12g|r+cLZ
zZJv{jK$cbD0vj%4FfT*X&b&9(;MJ`_DaSp-kFf#r9f(P{b30E@s}rl%nLoW~?bh5H
zjX%z_8+c7Xrsa^cnYL?<q|WVf+S7vIz`mAPsx3qr4%=7U(G9Uq5qu!8s>ZT_814On
z7NObOC?k`bD4m+)zNZx%T6JFX#%Su3UhC{~WxAQvxlYWB9BI3qFKy4_OkgMl<b_04
z-kYkOdKM+Z$(Hpu#Vvjh&twLdOA*ymC-|=wTWP*&Q<<q*zj7QdaGgk1R(TlJA?C?p
z#Eep~VgI%2@%Mc@mW$jz9&Qk%^DJnYl8}a{Qs&w$g=T`3#`Ug%ofu-2sU6aEV6|GA
zqysYgMY><*0aCysd#5=vk`~^p=nbEeO*8KiJ6Up?$)jbLKBb<_O@awNe9k;<4_1lJ
z0vo@}8@U>6&;=w@2us!mRyJ1-Z<c&GyfWCw*{NnHQAr=FJC~^@8N3M?5Xyl0AN={1
zU&31Fpm9v-he?vbW@4<ar*T@M&Jx!sMEmOTyn1?Cba>vo%$j=jM|#GncjkGrv{!jg
zIVy%&B@TmS(FM!O9v+(4peHA1J3YXEu8b~NQ)6HKBvWq+#oRqovaq*ir77286<H3Y
z00eI?t5-%!OTh&wFYamQEqZshB|PM%-=|lKvw5Q3H|EE7!m&P;9~^*%8!$g1kX=Q&
z&W4j+xvAj9`wQHlG6$>658wvvYNaLg{!Y4G%225VX$vo5eZ{o-z*&o5CN?LNE^cP0
zA*7d>xj8=Gf_88)q_Jv-%3mG9vMGBYohoy_206Pe_twl&F_j%Bf7Mb*p)l*Ral_iy
z$F!dq)vlrE%#OY^kM51<(q+6m@O=-D05rk+RyyLe_%Vwp`Wv)=jD5CM8j(PJNA$W?
zRT<mBw}6_z0r(Q65jI=H=d6_;Qx{Sj8G=q3^FfNu4=>d?hczy9%oNosTUm^fostYo
zTEtwPcl4Eh&MymeiB$3}5m#^8t$g_4)MB!bpKvsuX4-h#P>u;1M;5vml{B!@F38sy
z27olvQ&l8;`Kx&N)m{TSH6sT6C7j95SW(U03+YQG@}i9wWjb5gyAzSKWOcOP+4XTy
zQO&aRsDLue8HkBTWKYPjKE?KHuMc^1I~il_%(tYp>t@^CVNw#QjLJjvH&CXMWg~sJ
z`ei_e_5S?)aVH8zHZZB<V3w;vRHOZfS0IZd{lU0Qng6-LG3`5Q+~DSxz_EqpSaq*l
zf}hQu-;`mjCM4~6?(@fK+wL_DyY)5gFc`deuoM(9Sq>NTvL@I$DSSwlCu@(6u~5R9
zVxryPD|Jr}(#&$6GgZ1N-}$tM^3s&@VpM?{&bdjkahnM%jo}fOmmd38BG>aDFRO(7
z7uexMJJ%>ChvN6yd{lJid4=^o6-eE)E}_cezpq!<R&L#ZVpr$EaErsM4k-}!mkYBz
z=Q~@NXwaanqqFp;jBcdPoYHq_rd={U?Pd{CW1)ua+M=rE!7IZu&^2B$``S(}d2cXd
zoZ2Qd)Xcf84l*$+A5PG-*xGsR>j`ITv~wyIX=d;wp)28Wx?@YMW>!of(^asvKA5K(
zb}B79TF#q*GuyL#S{ZD>GlE4b3!R@{UOo)-&eL_{NoO+)&9%vhAIX`IZVE*%Xsc_&
z>#tO`rry*#v>%0yhSUcr*x`5AgK`18WuMTNKv-2%7-BFib!}nUOP`Kje6k|(#;g;*
zngYk09ZdFxd*j2nyclMmB~IH6`X_p{fVN)TrK+o88C6#Hi&Sg2Pl<+=t!}CMrYIW^
zODR3Bw8zlwDzHCMT;iy3GR%0jbxGMjv8OXLzBgSLxREo>POtpLoJCnWQL6^amf3mL
ziEu%mNV{wU^{h1?-NNlyKh9dW>l&TIR)O2);I<;nHaYj;LJ8)8T!2YDNj^kZ-7Cfx
zw{U-`y1j-;R#@5XNiXfMjj>Bme>Wxo+fm%vO)!MR-Ed2;KLDpjDW~e^1t2$9VLdnO
z14288k0`{ZUp_qy6*;$6vKUbo8NO6Tt5%6~^a2@K>UDxX-I=6@HgVV%-Sm?{U%*X6
zKuNA})Y+K?3;yh|R_$}Rnu*TB;7XlZZ@wVM+PFTGY9tO_Z0*|Odg=nho9Y%DvHWx^
zIJf3Q`FvN;vO=#iy}VU!-r%}arHXZq{Ls(r5}@>~Dk*I+)>4()ji60-eCLfI=Q`&@
z=MH=l119rTPAx1O?kZb8;*PS$-kZSmPXItT45{wSMgs+j@I1ubU)4(+Q^>zlRh#sE
zTg6&*w}Y?!cPVeD#8p-;GD0j+Q~}DTkJf{@w-JN^GN)WNr7L0(C>XdacSP{`8TT4P
zAsVTIz_X`po3Q8hz!D%%Sw-4y4{^*@jnYYue1aC=9RvkV9};x;+HP$@HXh>h;<?|F
zqN<O}*zA<EOf3DWqjR?zo@GtK6e*o1NxA5zxXT_mu4NKIwj0a_7eYr*2j<7+iH$q;
z$i#>aYX(xdUti>ZR>xOhawazKc5i0h&2mdw6cnv^|LOL`B$3R@*lUL9$%-1+iWR<3
zU{Y)1tliM!RVWUU*)%<*AG1Gb0doxwsMWkd0~anSfyz1Vs0x$5zCF;aFHw<M@`S!a
z@5Hu!V5JMBAG6i;+r?a!NmC|tXT$+=q9ruoSUp(4BsCQU^7LW6i@m`gP2Mu+{{U=Y
zLIIi@3t`Jqazi~>)`2Sz(QRV3N3!-bPpVk*TL!)-lk!CKO}3D{%gj0+jUB{f8Nb_X
zeYku?Mw0>1!*D*;KE;X2qsq2|%Bk3Qm1kINZT@w<-mG}|uej*Ft|e4e77nQ6)r#%&
z#_T4Dub_-5%)l9x1*Tjp<^|1V+&?)|N%A1bUsG>Q!s>3)lyu4vb19y{wNHlL`&uI(
z-84EkdS8O{T=*Wy4dr`zK_$<UZsZ(CF5|XF;2;l?31_b=!oGoiTXe&9MIA*AKt=a}
zuB?e3i^HoMry8<uZEkTHLX|m^oiMjoEB{H#^zos_y;VF1b-pECR>;7!qW=@9bq}QW
z@aog`|4C{Lnm1e}-#K=)HfAcOpH47SZbGHksu^96bosOccnvbIvDw=#-gM=_BC8NN
z6U#(+T%Xz27$Rp!PM`*%_M?g>H5Z})>-aJHtn!ho@!JFGgy+g-G4J&E<>ap=;O2(B
zfg6<z3nb!Zv6*P#R9*}d2LWP2RWFSR!Otc#8?%GZgNbJcD#J5L`97I%P`L!DUwG!m
zLxuZ3x`)KM&nbfXl_W`-v260iQlaX9-!I~=yUHvQX(}v29ZiVnx_DoH>M2!<y!QJZ
zSfPCP!`BaA<36yCPi_R4r!>1>K*qP(n=b+pYb~^^#4D|~YBEnFB_sRixcQf2zL2X9
zNPI~1R2Wl#Bl)X&V>wV<-ks1OeBYk=y1KrME>cNVlI-JHZt0$987ZfUlTMzA#kEad
z@#ByxsN4qVtTCqSW9^v$ITPz~oM>IWIsDI{=u$Stqf<ogF^CHh4w=P*qJFU+?}mry
z^AhKMvi6b-_<T$guk+~zg+L<*uSa^9IhCunsnp|=w6@XBPdsxtS;r-Fq+~OFHI_Wu
zzN@mdVlerZS~kDzG_eB-j-&-!>?N0*%~tXTL#FqT!=G$TDV)_J1NFVXR3C9}r!xs>
zRu)DL3q4v6euc|Tr~ZPjPnO#7I7k{`6)h~*Os-C7K0ghzH>S;#)-2B{i#<0w<!i|7
z{J2hsa~_({{ZM!v1WXH6?~?7H-RNhQe4lF;3?|v@7@i7%K5U)WJ!kj3(?pE)Le_z8
zCvx7fk=3Tw)0PbLgGTL0#(Pa31Y85u%p3a2vQ>vcpJsS;S*NyLcO$OgZu|1LG<4OS
zVuIByqQ^M}mC+SOe8ldMJwd08JE1a9(|(V3pVIkj0@xn+{=1cC8<jG<#``|)dR-rI
zztZR!KW%w%fRLY*>SaeoL{mF+ydv8-cC5C+jD+XT-D-E(H+RU)$;sMutL3J1m)DTX
z=ZD@6wr;Nv+F`53lJ}`$$rF*JQm>J-cc`e5zy7DK#d<kE$u%fgqMttP1M(EgPjWTt
zs(eXHGclRC`CQ?TFx8C2f~Oiz4Qv-nH{9uUm!bZzZNnfD2&*Yn@z63>T;<ksNZggF
z(*iNT*%=28Hg47i$e~Y;rn~a0Jevgl^M<`d^}L*RVlG=7GEc&guvb81IZzIK<2Y`@
zWF)hXLX0~E%3(@AcH^lNgQmpD6}n4V$-~IRUy)>nW+(adB8Ojsgb3r3TcR8}+@5KG
z8Dr7SgNk`%MWD7-+xn!+W%c$8C!I13wl~a46phd4KLfj~jcSn2^=@C$Wp;bEq%Ru?
z@^V4G!EU)1TGm)`byxSRLq2*Qk`~41r#=$Nm%wTb*zNb1R`ts?`wUITnmU%T9qsm|
z%xF8k%|tvk>Am>`B(^|0@WSWpg}Eumj(OgdoY`B*i}|CPvW>Cuos&LevL9R>7)T$*
zL5c{oR7GZEnL}&}m(qU=Vc5eiJ!fN)qrt&E6#J}z7{UCsw&E+@Ky#|lBV4oDvrJ_+
zk~-aWdlf|SsT?#kN9jq)!J6l`{A$+|i<Mww^4L__{&mS4KH{n6VvUMSwYSW0?xWT-
z)d+5n9a-n67cUouelj-E<w)b-p&8CeZI#1!?s3vo`*_=pVP^(AB*j=XO?>5r$gj}D
zY3;NA-Q!xF-+M~mCBA2z=$O!^Qk+#mqD`mo<5K)Fp@Atb(mLk{G#R({bUlMlCO|fQ
zyTHr`ov6|Pxq56LHi4(YiYxQ;RJobWK@4&8;+KkFRxO3b!Xk$z;G~pXueU0H6LgN*
zx85D6^3k4a4y-oXc+<QM5o=O644iaEoUD;)Q;H{XYB$z?4#6F?EFfBIc|)NFF$7z`
zkgca>^7QBX)3*(c06_I9+gInR_r2J+hrLAxGp1qTOpI7=7u()W3DbiTth0tM<_-n!
z$WP$wNroq*C4N{*6MG8P9{VgYIQ=mPmbjDdd<&)O50j}sL`nyzAl|_n?2GQL3zfbP
zx0OcB%s1S9RZDg3I!eSvf;9l25hw?WMZc>SAfI@t1^|co_u+}|UVtOuI+Ug?um$Hq
zX_|K<f<mvn%P^fFaNpu$+~D#vxfm@9!2yKmnfFFyS%8Z<&_h_enF4SnRO`bp<rIG4
z%bsB_Rx!5gcG?4(;)CRHv{|8hX;stDQQ{DgiWWH;r3E&JOmAM-gYD6VBCW;k6e#n(
zNZ_8imM2TESz?12m590gI|=Z@{ITDZm<_mTmUMqs`Ii(;Q>29DK`I6``a8_}YWi=|
z0vE)Q<WeqoEB-3a{$>O))4?@TnYqsB{Tt$t0+)h2_+9gNJoeX*J}L;}m|yz-f4};V
zI`jW4H-v;QV&vqH2lImYPc-V(`H;<z5-pKJEb!{b*I;kyqIhT*yN@W7L9gb#OAr$s
zz4Bs6)tD*lC-dwGVn}b`$DQxzXYs8c8(6HJe(LpAC5OCdDKdbCA7A~d5K++^n?0LC
zBk+VbY`b8;eF25&QUs2tQoWtOsyA33J*?5HO*4B3O_wE-*%&J^trHxnLpos0{DUsi
zgN9O5!S3G>C`A5rnS#Ur&wbiStLqBUNc7M1a5UTk7M3r8ddK=X5Kedl@_xe3V_u0t
z4B$5Pkd9<H`F{TT<-7{VeE_dEdqVAUf$^*t+7&eN7&WT^e_W_Z|L2v37vXk*NeOBm
z$_TZQFd=g{2b?OH^zWj9ZgcSYuy0`bQRxPvuB+PxVqV*9)rjcc#bIjB-JkOeY5s9@
zK(xyNRYiBW1M&VEUBAbr<T)CqS{`o=TpR0u0keJj{BdbxwbS4`g?^b1+3LHldPxp+
z-t*UGn+|Ir7kh4W*Z2mc#2yIV`PBY3=)cB49vJ`Bj&}jy`j5YenFq#yiJ8X?1i}4H
zW|2@s<3Hj%=RZCQ=G783!Mp$Yw4*uI?YoZNq37fAHFI6^{CBc{|3UD8$)?$RC5bek
zSr8o}_59?7SO@zs5=nXNx3G|*zXA|XC#}FkXQO2)d`4Tw4^lE8w=M!-2*-YY0_}4_
z1z`G@MOUkCaL_&P_lqDVTXy$-zZRhR&vVo)`2~2%pTEc+MgQw8B^AJ(6?t_m_3!Jy
z0cG8zUk{E*{{~c`8Sp+XIE>nT=ijX9r3H9Pe>F8b{C*Ao1C@Bog2S{&6uSSqSqGuu
zX5I7-7d`pson@TG1&5QLKKj2W{l{(j-`#0T-CxC}zi)J6N#E@3EGos63=|steyC<_
zmRJr378Dp&VTfXMoPnnY6wD2O@WKbw;aoPur<zV|wTMRHPn0s~?IZ-c$UmQ$OqS0*
z&NB~3!2R-*kBWG0;SZ|6WFM$J4?9(sWy61?pZwIH+5h$8gdsiyFVus3xzhi7yqYCw
z;6c*QZpi)D;dU@NxYFwU7pTcgZP~YqoY(zdhZl)Bx9VMO(f{=ZONP@!`(H^f;{Q6l
z266(=l2o7kYpgQPMuNZi2LbZW6W25WZ)JNL(?7;9G{*8E{s?i(?7t2>W01Hn^Z$?b
zzA`MUuIrXmQjii*q`Ol<N<m7xL%I<VkXA|%K|#6^Q94ybl#uR{R*(>oRuDuffipMi
zqdb4lb*}Hc=e+*#(gl0p_g;IhwdNXYj4}CcABoirDPs6U<&27Z|7v!GM@Oj7N2YcX
z+l%mt*aDvFN1O9FfnoH%jpXky2QN|?i`-c#aPGgI{kvcOV>-L{I2u?V_5j1L3=9DU
z-MG$8-sHcrb{(i_LS+_j7|*8hf3CP}5%_DS&B(=+{pPmFSq(`mD^hc8f0o-HaS2~m
zqHae{@j(xmGz(Sy7+=l+v33(Y&k+U`D6VceX>$7&x#qm~5>3$mJ<Jw}Ao#c{2+5=a
z&k_rR%<>_4wgOSLND{5*?7+)c|8!?01laes+oef3ZWjYu$^b_pku!!$siH17BlG_m
zLcQ-H#9#aFalA%zBuxT76Cy(^KikmtXPwy5tGf^cbbcTE-}gSeT8bI7XzEi|O{-1@
zJsL16B@x#~P=aj&VQ(CEmp>L&{e6JcgA`Fs&@&Ia;=K<9y1fs_B~52#p_Wg-BKxyC
zIi1qF&<j8OKv4?5I+A2tBEXupPBJ@2Rs%i*`@oYm2Gjj>;N^@6fRXa{Pzu}$JpYgn
zv|Jx~<L3b^;u{CM2fO9Vz;&(!rZffZcK}~@;e&)B3AqLeDMU*QMWEIu&)hkH-THaS
z?RHhaoq>lE0M7zddQvumQV=*QTtLP>V+BV1_wnwmu1RR5gM9PM9tf9hHi0Vp*@Ala
z9VAXYi#HdfsXG`;*4OHfQ;UHRn64f^uL@|Wx(c6|{+Y7^53mseG@m^u!KRK>;5r(n
zU#-Mim}=B}c;|8%@@<>7lBfqSW$L~^SiQ-tdw*@FwH1*siU|24b-`-I*AFDtnuwzj
z!i(Aap7X0+@5-&t`y-{WFdk+_YRvLIrdkD|T@rgc6!p9KD(P~v1VFOGcy+1oXMeGv
zqxvmRVV$QaKbKmwp){b=untg_h==5KD1HBqA}-Jud(L*wuAKT1qg|PD$O|ONV<7aO
zV0U!)^~X9)bU*Fe&|c1yGDB0^zo0w^<R@u&I?_~`pr&AiAPWpy8@f6$i2Xp|zl9$E
zP+D3#{)7mK5oO}GKvsopRtJgwe#}ruHff1iZ~A~RT*t_L2;m~a>E#=?!XiE>x}cfs
z11V<AT;*~JvfTr*)ERpkiuv#S54XEORJRV!LMOi=Rm%{|K5U)*cBKknl7l1+s(;1{
zBGo9}bG6!u%#d=x&^zpmxm^!;26OYwm8%d$6d@td+f+hvP;Q`Dlf?6xfKmx5huNB~
zL2@TAM)^iO*AHNC(YDO>VthmZJs{B-q%gg-ZV1M%5cK=xips5@XhL%jR$JIUyhr0a
zHtDj?X*~@TQZ=27x>enBuIP84pf+FE*)<?HKLo|#j5AQf6h^hOCnDQoH-hgWhxL>y
z<wUQ}f+@6}IIh;?7IpJw*4#9BZ$yORvJU`@jkYnU1@3K&G59-clrY80mOrxtsOIvb
zeuilg^SOq_Pd;p3VUiY!P*9G?k?lqf;u)ESJ3A%C>hkS&TeVM0fALDZ;G$oIgPjqw
zzc{9uQBgY-WH(f0W7MYY(VJnq@e(bFPU@<LsM7lR8x06^P22xj2w`+V=cnPE#M<vK
zo=}0?_sC&YsbuRE*5Oa3om>{J_q&I4O284T&oa@L-{)>jOl=>=V-7XyD$p#?8MGtX
zU2tQkF+>FTZ3&ZQZN5-~R!<Lh0r?Wj%la9&;yT{wA$3<pt5{$DfZ$GRKqSRYX~CIi
zSQ6_Z|I90bNUlMYEEQNoL}P7H^q^nU0A7|IYNrpK__1|X<=zF#(Nd&NZN=+6w(zZ}
zZ-}X^6FW8bNjRvcXhuWF^@Tc_S<#=NCaXF~0MxPD7i6>&WYO+lOzKz%dWq`g@iAj1
zt`43{NI`24gq@wU5gmBnwyASLabto3{3ZWpv46wGTfVM+-~<Rx;8{|Z({s%gYP0eT
zEqqkm6s<3siL<7hM%skjQyDHS(&m6~|Cr*%8bo6Ll23w<Yz2x6^khg>6kA?wW*X<l
zzJRc^fe3iV#U(S^F^l88gGR?kfXpU<j9^>UnF4)i3Ydy=>R+-ZTSVVQ%SFr%aYwzk
zASbyV9jPUHykOsa04(A(Se5sd?WzsL*R(X1wNq%`H<Y`v&+PRw$9g5%>rr;EOEX&9
z@eQXzX$_^s1~;a`n|@c<34+Fu+QY#qSBqZ^L!HMf9^5+py0Laz8dj_iRNjK-eJQ&j
zE`IsX3gDozES^z^g53ow1nnLn_fM8-@7UU9{fIwX*v$fptd-pVB&N?TnKYKyI!uV-
zS%RIG;ysyftU=R=iUK*KaIhL2Lk`VVvcTvZ>$zvYZMhrwaBqFQZESB`z590Cs|$E4
zhC<owe?HV64xWb@_yW=P=n47nX^B$j$cHHlvNN;K_U3#ufr_k)>Iy9KAKM+cfWyBE
z6Pv=@4p45VdpytAv|U*MQt6%ihe)wx3e`}3y`<xeFW&C$f&2Sy?weB}DeU}AM2*nP
z5%O0))#|yLHGHPe9I5?(an*+~9W)HbTZ?fv;!(b;YWTLs#aO*rf|q*#+lYu(N6wk{
zc%?RkTYbF=(UHPXq5t5puuv9~O>Di<Xt^<thziDFwesK_0TD@(GoxpzOt}(KW2hV+
zNt65~C9;l>*|0XzOOK8|EvNM+{)ecK73aSa2Zzx+8Xo`_4l@hvNT(X|c_wixC|+62
zoLw_NuY?1b%y&n$vVJLyfyXG>uSk}paU1noKI`l`>U{3ku=bW>>oN5(jE2Wg?oMpZ
zrCnD^xUjpwS^MzvqTHW_Dg{yi!7!x^DHCi$nRs#PoEpmvDk1d)h1C91JJC`;{EM9&
z53(_7%9XJDb=Y_)vJ)O6@H24ndaH#b5{?y)SVCEcJ*atb@@h-O>9>wXhR-j#|H)WX
zAY)mn=1Ncu^uSh1;+cXRsAsl^ga(ZC_CRFhIeZP$1S@8a_RkGh5{n*H*g5%5Q4i#S
zwzbPj1#=E?!Ap5(mXjG7$@+_!ZGg`XvZwU-_aZkAy0STxiLXN4n%5w`NwJcx8V5SB
zxz8KLFe5nV{xz*cfk><?a7#VVkBmk!i+&nTtja(y0mTQh5p(>;DV`w)?G%`=v29dt
zAuR2)x5)bwfQRq932RG4?+Uoo<=M2#4#;d?{!TZM*%A|Yi+m^a*mK}w;e377`%dtp
z2~ltNI;0=wf#iZqJYnkjr>x}PBShoS=;EwNa2ZS54Yow)g~p*$wY_;z;G2t5^pAnj
z2vQvBTa(kDjdmLju-J7PtuS}41lXz9z$CQ-%V_w$M%5LEFj|WE+@2F@8}1A!^mm}o
z#n)-=A3%cO>RrAcjYe$4w>AfhgH%ycchrYuTFX;Vwq{90U($O04J?hVe6)-Md1I2>
zcWC`fZ#44_xB@3TVq=M8JkF3G=)JuBIJBb6kpD@Y&|p^QqW&XVL1!_=pGPrvQ}xck
zObU(Bc~OxfzO)lMvVhlKmf*g`L&5a?3ZLsIWDc6=>SoNOpG6Eh@<i+h4~(ODOWpuL
zUcX~;UnIoE%EFd<T@{t#hI+EX%v68ATz(0|#~Mh<mhD8&SWO^hDbgO<>UGN3c0agx
z<DU(05EIWJ@TroRK`4C;fBdU<hDu<$Uu_bG)pQe7hPh&&pAD|u_eJjw)>|T_4q<9+
zs7^sr44B{24Ei+~=%iZGt#*15Mc3)X(y)9rAf_p(B;fQTl{)TrBYL>RoOiMID+K@~
zvANaCmX{ssW`bCI$0nER%a+EUb0NI5?<@GG?Ykx*@%297cORh;@*D!_kD|(i-p~Mn
zdIIZ7Y*(&*Gd1|OqJPRY{@|@lu_rA@1(G;8qOBdT?9-^w)p1}DEN%T~fZ&ikC9nxh
zvPLlI>T4SdeQX;EDCify=>&2X|BPXA2GiKlvz2;e?jJC4>{@u7?b8n-eYl{^q%Jq^
z`)SX8tV4+Fygw~x;P1dcgbR8TVRZX?>x;AgGvJY}M8;QWeo%WX;pk>{;WCWXn(CtT
zxT81mhUOj<^vk~yh@V4iBOA8fCdN9de`Jx!?G~z`dHzWt<IxTB4x;+y<}PGD3bwl$
zL-W_Sbwo$80bf=?3=$KE4%G*LcI%?t(7b)s6d_3b-V#Bt*8q<Hsk3uQ;h&H>{Duz@
zc<YDY)1M5A-)V-F2`nw&XRW?v9lc2pfB+u*lzM1Kdx{$Y)YeOVG=5)yII~F2L-Tom
ze2)LP?l#q$zlZc+MgagzYD1r${`fnnWn4x!b^W*&&q$<9ArwXbc`4(p2&%rqk(9l}
zc8Dvz(8$QUVaFS*gM;wOj|9eTToR&X{YL#QcV6iueJK@q0A24xee~CH%8&yl&LZnS
z9v6qVGp$<`KXP9fL}U<%<xf-ckA!DN0BY{$KlA<1OQZ||M0(#~pYyNp2DB3Ne>FJ*
z+UbB>n0yoiMZPG(|KJzRWPrFXAtmRx?h3{xt}OQ=Q#uUzKqo?Sai5K?wVA3%3<;pB
zR>TZpQclsTEZ-M|{TG81Z|n|%jUwhTMVcBDboY+*qMv=GjJ3rvia>B`ni)}gZzy|G
zdf@b|5pAD&bPD&w)M4UPsaKcy+~}qYM+({j=~IH-&7;Ac@gLu*|98Co-s(E=%RuC-
z`=AN5lfd2lX9(_q{QMw2#U>{Jts4t;av})h^^-SJ!WYnq?vMSeyVa2IB88wq%)9Yx
zlMQ*wXGn@BR+)|Fejvz(-^}Sr(?+=YHlnalx+_#f*H#P5ERrTpKxPxX19b`x5C{wW
z7ySXxC~z$U2x=CJtRYD<@DVoOK*2^t_mK$tbFSLgAP(>528VJ`&JM$VrMyRUSxcl!
zG4<Ut03^t5_dy0+szLeoXA;}1gACO5lnS)h%+BlW%#vX1r@jIy&@RY+R$pJY^$!ax
zg(^p{+|rQBWpE%zNlB@AwS(&^uFE2zYnAYjiORj#*-b#X{>l&mkS^kw#4Z5jG6wln
z=;`Zsc0rx~z5ANqbtp{UUK|;D%GhfPA&Z#cIDom$<aUo|sBY)V&{{lzQs=mRSy}*J
zcz?wVvCORfG0R;|ZRfNZ^1Z<OGi672)8UlhOV86Jt-;PrOrXbU-TzB=LB75E7Lvi&
zp!AmBL2~EHOvoNrfN_HqBHkgd2-DmJoJs=f-An~_!M^iZ7hh*J6gdGU&eiTBNV*Xh
zAZzgU%Je9)o=%r{2AbjO)5a4d?qGPm2S`-EPjVsgIOJmBJ2wH!W$qIK_Jb@8+ek@0
zkxMbvT-!^!wJDs(4#XV+`2w@O3q@&=^o2$ofU;B^FrqcdQ&;@}$MP`i@<x1^2;`o5
zuWG)*PY_?zGX}CTmhe0r_#v9;Y|wCO?v0Y(;JB_y!_KpQplcnuQVPZ-YHgdH;LPqD
z>*awX!1N`E`bPkj#Fs#V@YCn+W2Vm_mA6_-$pZ#~0UNH8n(9!B{mr7upc@k%T}t~S
zYky3_7m3H+VDPy-mIm<Os@J7;V258P^}5th{~dZy^2ZdxNG8!SxiW?4HW!5{hbO?i
zNg$YM4soSZ*akwIGbYp>^0(ex+TvrrWHEaGR3qBY$+x!>vI)dL1>}3ZW*nod#o%-a
zL2=4mE{)tsbob-N-Sc4UMgJV!Z?a!q>CO*9!LL35s4+S;B*%>(@o%!Y1MpPiF4Thu
zKfF4*TN;l$;`(6sfxr~&(asxS@0jt>W^AYkG1x(>oeadrTKTwJZz@)KI-3DA#}BaG
zl;xKr_2xHhe=cZ>u=-zCzI){>cuB1jnpobS%a(eNk%#-3n{EX3qUAtjyDs)ggNj6T
z+wr^ufz6rEN<Lc{J_QRh;c<yy_zEVkTTR**7Mm-V-<VfEH!j00J4|fPC!^+c%0Px1
zijqEySqpYGJioODIczmpFpz=8B4X=6#SwDn=h<-vA?k5_wqV|e?Qn?OboFpc9G+J%
zWZW`<5`F>Aj-aI#(qLDBM>|(FH<17kIo2W!<QWRF%$2<fPtf*LyY*hkva#)6l2+qE
zV^AJMphjC${R3L1kpVQ93Iqy+%`$EpmgieoZuCukDPIcRxTQtiO!Wdzk7eC_fWp^1
z5^iRsV_=t!?3M0VJL!6CBYJfQ*{iZkm%%*Ge&@DE#YULwO*+?1ox6I8YbpqxL-eis
z@n)LGc+`qPLvT8LK1DHReKyfV0<w6$j9`#t>U}cYN%eV3-V);C`|8eR;<%DW+>@{l
z0)lup#a3fP5I{!f4Jh?<cjXp>i)i{xP*Z{B=gq7E)1s``E;2LV!)Bh5-86!ULa*;8
z_;CaAZI?&+06P=vz8(OegAF}e%arakg5>jdB&ghTp)x%#*;WW26)tm|UH_R4#KnQQ
zBa+Att0XU{L?Vh{G-<#$`XOSLh*#P8@HRL#*a4wdq1DkR@ca~*Ggx^ZTTccvON8RI
zj{<mgaa>foAA;7sCZ1B>)@4qCGNAJ9*;D5paCRm*12hpiM%NH-)x%3x4!^C}wuf+V
zEbuWq9hU~>R*Wf+4HWGm-cQfQbC;`C!PzS&#7W+`K&RsM%qU}*?5b%0Lx89~$6StE
zd|tnEt7c#<4|&Zd2}?mlx%8EXiBPXMYI>Q$b{MfO<DLEYi3c>-51QAB9t>1Z-(|mb
z@8#R#H1{SH%jPfS__ML2vg^(!zO7!5XDgoi@VzWIG-aqYkqL7VE;u(+?RDQ$j8uon
z?WAEsx`Wo8n1GC15)mf5qi^Ys`+b#>_eoE+zz?#NrAZ|EqQS(+z&Az9SNw*PkooB~
zQGsq6dQ@Hwu<yv|3eL>h1_zLMBNMA_uvaP^bdDxwrc~NRd|NA~vm?6<qGRH@c&y|V
zlMQKNJ}RpfF`%%IpBp{{^uXv4Q<`C!S3RaY7|#%A!s{dJ(NYgG$}~d2<%HG6bkSJm
z=^&xbXrbF405YeNd-3_IzTKGIH7-JIM4x~<YA{OhhKxZ(&TfI|NfG|dE|`T{FB7`<
zoMquWm_Ah-+J$|ozhlE3kx?olgv*{Wa273HKL0{ViB#}ewOj_?;?S{GmlSpl;!TCj
zbC%s#6BQ>Gj9PJ-&{jW*$0lH(_kJGb>S0!gIo(3;jG<}*)*K}3d!-@#dk5O@(Bee~
zgQKYSs>;V)FQ+hRm}%&(e0~$^`F`+5*@5uz`xI3a6gE8tnG0HHA5~#I_KD>?=`7ox
zXS5ApZ7Y*{FSN#T!}&cd6oqj~%s1?NG;e8fQe17Dy8dwYjG$`;MtDtQkl#%c@`2l^
zuHK_e4Gdx#Lg6$Cch6sJ&6Be-S^AEMHOcf;#f>{SIn{D0Bi>mo43^dOqXn{?okcH;
z3P4k<fAulu@_J9GqTg8?Rw04`j2mT4G^pyDVm16-cxG%@L**&j&XH{yE#9d`d1j$B
zs?d%1l}^@tT_zw*g9ZIX4dL3UOaba?B^`~m%L4J^9qk&i3ge%SOKx3|($O=+EQn?m
z3&inpyK0F+RphGdzX{7Iv5Iiu%~wqrg<8S=?D4O?aO<3_Eahjk*}R?~xA#ddurToW
zQcF{g$JW^?+?DCV@ucq6bnpX9;|HM_W9oxj+HA)J%ko_uJgi9!`dhv{AIwh}KZk!x
zX{3XRd@8=o-;A6AC)SvkL?a+%C06#D{8CBb*hK!1hzWOr=+N?kvG;(|iJoIHIa71t
z2j}BOLKBS?N$2@_<JFwTik~Cs5*nv$qYD$H1U6Mj5Iy}8+{&{tqM6xu^7Go()B_(6
z-^|X>g;O6dJUrK{p5M%>*Eq6}4dw=+zO{L_bRN^rVw;-raVo)tEFDr7on?HSS(GXU
zvFmE`iUk$6^6dh6(Fz4qSq-tp!5T2<vIXLww6=_~_NCG59=0rv_t_HOfTbegzPtb_
z@IU?d5+(WxX`i1`KZ9yETG)?9OQCEt!*bKnt~*sjW1s%MFrB?Fmb+`bU%o@Il3k(?
z+4MN^lnfW|kiI(JP1n?|q+4`MKjaN(PD*~KW6J$5=x83OSRjz2Qa}=cA@D8oHP-i6
z8g|?Qwl6LdD3oiwQE0Xs)v|hIjnC|ZIOvN9K2Ih@m!rsg)LmjzL8Q*5zauY#Xs0g~
z)3ojkzpRND8oKhK9I7HWXEOXSDo(B$$KJAc?p)JW+%!}dOmn&aZIW4eEb4<+g@^K(
z`x*`kp<e0w6_@EZ!t^=1-A=CFrkK-y<nIPHn@DX7rD!LgY`)NI0u`(|MWy4WSNuec
zZSj|YSU+1a<0fHVmkTqVyDE>~bLqcLuxY9>$v;W;AmTYChI`*cyxexuOP(r0%(S>z
z@<^P#PdW*jbqZ+q{R>q|%2L77@l>gKiJc{?op>n?I`&ijlw`KL?TUW37n6<N@O?Xb
zlKW)qoKXB;l6_a680+bp9FIp@J66|O#6B>5xu+<<bIY25q?5OYEjMkNa5y(0w*0o%
z<%E?>uLmr1H+c!Kik>%ckbQ1B!L62RQ27>8Wa=A3BkQ&40kH<d{pbL^8@SLI(^<aM
z!8Ryu*PDxwD)+coFee)%yYig{eQ33!i{1C5J1sZn(!o@`a(0pH^0rX3;B0k&4jOt7
z#4(eTe8-i=Q9+lt>a%Aju?i+*Weq~PGOwJkJd+#Gg;y$dGy1rG-F4jzU$omdYrzex
zR_dg455V`$HsT_!4d^|J=vWla<%>AO4;?|8>T@E|#|%L{D>k|!iJCL|9Go<TJ}{i<
za-6jQHC~Pv*crHDd*x(_FiH-3&I&cRYV9QzyJeSfbUcT1x%uypzg=@Mzw&r2r<6MX
zOW1~1EeHgqyMaF~+28BM4P)U0gUvDHyXf)Xl`<)#lx3RpQ8=;!V%O42m!Eg`jo$LL
zW1wD+4$DmQnhqO@v!hrNK7;Qz(7}c)h~Bf%ZI~imAf0*rn#a}b48~NZgzFYVL=Op@
ztarx}R$_*otp-wYcS`#MMH~xiq+Y&5Df;SJa`kF_*EU@ga}fE-^lVGaV`=!~%p@6o
zr(=xRjH;McD#@=A5E{G3zZMOb{SF>jcTdo?cig@}7P>k^gA3j;WvUrbt8eq#nkS2K
z)l}stPp^mVoBr@mKK57)42$h@JA@<DGaKHe3Rts?(>b0*ETTk(*LD(ij;DvfdMsC<
zqo}E)FKuyYeQGMHf#=F`Rg{l)nMyuT;4x9RP0Y{g?elAH!JQ>;6(*Pq_@v4<cE(8t
z<T<fc@6O&LmLM|djgOwMEn}^UvY#qtAK+``%SB-=&brM>-;onJg6SkLtDt~hX?d=X
zG|E#{EC@Dk@mUJ^atg#$RQWMf;y!L3iKU$ax#9H=kh~fjd`?i7e#k4EA2#9EbP4=)
zl}0RY7iG<xZ8$LeuqMojW1gHP_lU1OW@qIQsF?n#^%I|W3roE~M?}xVe0MH)A9y3l
z%lD4SeN<1MJb5#xSefJI9REYsD%}lVE8@nb1ZB1Bj9)X<9DCfeMDOdOZ0u%hY(h$J
zsW!mIWO+G)JxS}-^R9rGV?P#$q`QOGnVvmr*In*PjtZX}8GQ5AA0q<Oo-7FC$GHo(
z211PD!I$NN3(Pfq9!5D1oU^oxPI<NbO8nJzKPVy7d*aqA)t=<`>6p(ph8`J3{nAy3
zW}q3TF_v~Bi%;W>?pb~4-hsR#wL@W6)jm@0qe7hmzG{&^?1EfTnH}GIq1@HSpgUd}
zyC${)eo9-gY2|Z#|JX&nqq-k$NOQyNUh6;{0YQ;KZ072MoWy!`>G=eeZ!HJQ!L6XA
zD?PvnW$-STsSZV<FYkIlA7L=*v&@+?xGrVlA*DMJ+0@HXK{e2t@@Anr<54adL5`U0
z_0_dTk`s2DV`z~LgH`LK7F$sZRElWR3AQqd-P+=jpWL412t`sACW<*mEC}evph*%D
zHnj0vstAa68aiWzl;J-A?0A*dj<JnugrtP}pa2=KKMUKQ_M?ca40BJ^mQigM?}fBg
zf151Bzdu)Rc}eObzpgCvw8{jSVB|lG94#MH_lmC16C2d%Z%I-P7H~r=TIb_>d7e7V
zb(_^_vV(F$ecdzt%oe+mO@N~L4sXBg>rXD*kwI%qrKyy!Ro}g=IL&vXw`^?8sP!PX
zI?c*tf2&t8k$Qr$Z!S$Ow&0Uurc`#&uUu0cK;_#;xGl-ya_q9JAF|I!Rs#sH`thtZ
z99j~ozSFuW0sm*ncuBf9;wAi{J#~>T&pHbyRF1_@*GE)oWc_~Hop3Lud5XA$q_ET5
zlPQT!V$DPw!#5XH^(%OU(Z-DOwPByKmG8Qu)ewg-ym(f}id#rqQ3{u^CEm)^W^UwO
z#<|(X%M`{<gk(8_c7ahb$CBZwur`zo6eYK9w6+FCBsho6qs_i~(Klaa^<|TNhKon~
zHJjTP8)u(Jt%}^@%QP`6b0c3oK1aTDp+<O9I=LoscUexy%8&89(ic6-eTC&cO~nkS
z&wFshr@>wv)!z@05<H)abFMQ(r#FwG(?3twP&azRQZsZQJnw2dAh&eZ&bViCe(>h&
zFX)0k57h$ZoZ3nmudS5Of1kT2wM%2~n>kKMAGjtse<!oIA<ZT1dXP|}c;(NdP*E;m
zkG*3Q_E{$w;Vf;V3-+$Qr~CqED6$^UPbO7hV&?UEeu|vBn6*jor#Y>ZU$xcVsJ(8d
zBbcmy3x-5Bj-O)AW_ej#>0mPnSfqz7F}f_H;PhfdQw%MOx!99<^W~75bNP;pW;vuN
zVw1mMl+B+iXKs4T%Z!fMhY`Si@`;fxg`p4^-NxgkAlulMsPM$bsDd->qXX;dw_gg4
zNb4HhU%UG*g>9oFM+sk|v*m`Br=1Vu(3QTv`-(kS!gdy_jb403gXu!nartgdZuVVU
zM0RQsn8&KKhs*l!Ka*%K9Ey$T8eVwN@Qh7x$(s1u7?v=|mE}sb;9zhQ=p(Cn1J)%Z
zvX^pL-MN%uJhn<!&ho!K!N2jbBdSt}CxB;CqI`W`+_>~h^P?s1L<xyGgWjj5eY*_n
zcGDpohUq!0q7%4ZCg~iV7wj4m9~?Az`FF6qICGm!s`~b$KD8wl<L|vChv?hFF+i+N
zm|zxg1D}Bjr+)eS*s^2#9WTO1-Q-4}77BItti&`EE>W>sXsMaSKh|-zPS#7h(<yM=
z=gymvM$cH3_yqMc_I=53OEmiOC?BuKR?;7vWA$yAaaYKuvhW?j2)88eV|poSltJHA
zkghtaSZIKzQXsHc-O7HNO=|jv7AwzXkWiDOB}NXt@5m|6cXaG{^aayibI~A-zn(vu
zc{QBwuC+n5qHo3aj8wOCHaW*F?T+LnyL`dNYm!%OeN!-SB%?hn<!zrG?C-|%<L3@|
z300tHla*JKwL47{m@3V7<dv6Ib5<)meAiRvmV4N*Sf$B(+Wk2k%)4fdfG&xRk;`au
z!Iwxxz(qRYY>FV~g!_+EEqaPSqDY_M*>*Ng;!iYxiB33=r|`gOwN%a%5rt<+wPH?8
zt`XsRp_+F|8>0}bCTQ?DUlhQbsgJIhQzXt3tbQ74l|pHq*&P=kp)$gX(eHEWn0~-g
zJ&?I@PzzR*PThGX%eL1$%Ab)Z#;wRvc_rrb<gL;^>w*keTR#(IzC3NlY?wTJn_1mQ
zF_)}7yXxbxn(FEJ1Oo95HA*7>#E{6ryrzzmSi&NNfdaLQsdDeyDVp5`Kj`AHVpjm1
zz+$=LNu;9sF;JF|a%GMvH&Wm%ld7MqMOZclmA*Xb+wcjke3bkVe!-8d-H6zykI6AN
zf7^j~R191j^SDY5?%=d*#gyV|O`qh`-cjJDBp>DM7uZb6fG8t<(RZ2;M69;cB{O@h
z_*vWSCvGdJLul>t;`^Jo7%jbCyb&Xw@YAl?2)yP-+V3@>TZQM;)up;6u^Pq+G`gpR
zJrbaBe6}Fm;bZ(IH<`h;-Rq+LLWGx-r{wo+DL1c~!B^)B&OQCO<^<KYnHZ<^2RX}5
z#2kC194|B?<~^;1UAjErDuy$#9osUmPAvaTg4ae-?YdbIz7P$5F&RZ<mV^0pY+G_~
znFB4rp&g0_*s}Mm42CppE_ZXTMrxFq2+a!!>pQUz_3?&N)Z3@*g!bt?E>G@xT(Pl8
zA717-hbc4Svl!<Pd`hGDVr0z~&q=KqTepr@pcioI^uMKHdY|RZlU}W=5SD64<2tkI
zxF@OIQgaurw=6YLBHquolBgZ2ULY@Ru|B6b(d!#?qOS=*`MWYMr2jvn&E%Q=w&=2M
zmg44%$i~0b7}@ygSLmh&XF@?+tC{`=gS0&OG<(Ax`&sA4!pZgXi}D|y7SU2{+?g|u
z&~5Kke$5qCz+QMqAdRVyDKg!qCZE*lUhW%V!)R$ci;D$<d0}*xcQLFAM;E5fT;L<k
zI*I;vGF_;YvBwg{I6+ln+p>l0Zv5i8liZmVG7gPj7;Xy?h7xkQcU^8(#AOJ*THnTU
zl54$#6<;zT#_AN4Qj%0`-zUlhirev(FKqP{-?WevN0UCn#U`{+(qapBJx|obdlid)
zq|m`I?8;7qqz>I<>Knn))tD?)da35U{d|-@&bnEfZf>%nTpa>?c@oYl511PSM5<@<
z4s!de!bJ&mO-muecN+_FYEyh$5?004?bE<;uFsC2l$EUii#4kiS0MGz(;7K&b9_)u
zX7@=#)-6vW4?Vkx&z#|_c9UHf_>F7#<Q9gY<ZGy$iwH;Q9!l~Jw&>I29m3KU5zsd7
z^Na8BNdGh-c30aYza~sYU!C#NadlCvKHf;RKyJ*;Fro{6MRD_QLq)vl8--bTh1EJ@
z3ux_c<e`{N(CDm7WcMk`(-H0j@OCrFnbA<t@$w-8NV|HKRPh}#ekG@mU08XGuZlUo
zP%-SAP1A*JJr+l^&?nZNx3@|bOgxbmJf`GS|Exf+k5GMHHdg(X2aak4;b$f+rm)i#
zC+?0bL@if$Gdu)3j|9OQ2M-SJHvb1^{PX_6?dp9UYcKHqr=MD0JinvQq@v8=08!w;
zt)P3&+kAKNd_C<%JAFpOz7PdhE`&Z@%ubd)dp}<}EX~{I!h%LNI67R8Wy3ZnZZy%R
zL@k`Fp;GE?@{Lh6Jiymq9F5k_^rX6$nb-I+{bS4AjqU)|!OG_`lotE;eN$p7Kb$jl
z><42zR&V{9_}~QC8@vCVubXM10KpA5lSP>cJK_Q67T4?xxE5DOHk(+9PaRXH(bgLN
zWKG7WRsPjW?o6Ab2b|{k&$64?sfR~}I7jycS1Uc5W=TDL5=~nJ2%TOVuTJxC=avxx
z(IUUu=vy5}s<%h7!YNzF9>U&W-mdI=RS)(~0a-?`!og|r#8-22x0c8`Qc_;ZEA2zp
z*p^&-tmz#?hJoNt0jv*66tZVq7e7-_Njzu8Mj?-2b^j2qEHJ`Im;IdGg_7QXJ$C%I
zRY#+<)$PIKbFL-x7Nm6M7rA;7{pmd&*|cfqMA~}h2SfRZD$Cy9^?ZRW%L5&yxYcNR
z%pW+au5CVXn3hmr)`rfTMRubnHTtzvxvGA|?RoH%h+Wg}_xhAy$8`wgyFEZTAbz<{
zJU^g#V(;ZOCHZduFBO{iMj8n-FER;bKT@=PopAZetVQ@ERf#h6aLfGl>rlX|xRn0Q
zu*>VM$ZADCp=Q&_!9*R^xnU37ven&7n3REJ+h4;~ZW}*fj%mp?vQD;o#~3mH43T<w
z(`L^{KXX7U*u@geU%1CVd+rg{OeP^_bowhg?a_hBQil_~2?PWW$^tl3Gt%RgHd~*&
zmiKX7GN&|@4Sl5GXm>)r>Y^R?=;fENjZ3^3Vqm2O^4!ZO1XNj+sohf@7;)IfDX-Fn
zXFX!2w&P4LoSzy(c_ylC&;Gf`5S{LvVuN}=fne9*?6ALNly^!~eEP!?i|n&Ocvs?#
z9wun=o60gPg7=Yv%siTkV+U)lQnPj7@+He2;gXS}g|Sp+P2UO}ccC{Yja7<Gy`=lb
z9vN1vPG@`@i^?j3^Qza&NpYdOSsUZWSM9yq8@@UAG`-$YrKuU;^E6J6MjNMr)10fJ
z{Q3Kn&fkjr+H;$g3aB5?m#<T%7ohJ)S~Bc7OnFmN$CF~D@lIeHdZ*7_NM>@3Z;^Y%
zt+Syz<LD%2_|;j#mtdF3Imh~%7&~!|hG6}L=Wn`4(7>n{hxa2>WTzOF^|{#u@4=iA
zBCDA`{+I2M^sGEIliX+7lgkn+HFtd0YC`7xv6H<bhl`t9#$rca2@v(B#F?2Uok+*V
z1NQA`neG5CM%~zUS@K82GgLCEJq7Vp$!@$gqXK6*&--yF516H1(8m_mJmpkVEOX^Y
z^M_T1y>5x^stZr#C#ZVy;<ug$*^Eb#A1r=(d4I`eg^(plBxmEhElXp)!dR29#vA+*
zLQ`o^s^TUJ(n#~K_VFPbKXltjH(k$ROl*c7|9H?*Q+q$%)Ds8b*&UPSi=OG0lZkhG
zqgPnmG#@=V>CTZof7y2q&Iy`)VziT!0@;{Eh1Kb04ziQbX%b1PzV~AgR4DstWjsd{
zlSxp~roQwTt-K?H#toHml)-9*%u_jy<RrJG@+ZAdTz%~EDHDo))+3E;G4~#prWO=s
z75kA+v7PtuAU-%~CXsl+Da(EKb=YoJ#pzNU75LJhPkE6xsK(=sXCbVH-fpz}8t<nH
zrTO35o^BEQWJgq?#pGiodtR@Asvt7t4NHnT7!XHf^LuDqYm7=~H9SSKK!9re#F<V^
zO0#i<+NMdL=ZPua97$vQ_ik@*I&roS_#>I8dd!id+HrbFN~W$}cs>O0<le<z-8;s5
zN?+;@R^YsuXfFl$d~&KH2361E{Y>I1J6f=NA9Eb*&x(m%6H<7y<rn@8Q8MuF&);P)
z+4|B*&?<J9O2UlbiPTjz-e<J<tc#`;^vk`eg?Sfz4cHC7-WWKMK>a<pL8ztrQKCV2
zg?)--`VBTU?Yh&YY>}rA7d1-arwWc1#-HU83(fUcU2cwUndX)~{n8F_@Z2?~Rw?l^
zL|qh%HQ6xoEVoi&ovKhhs3oMo8iioDm$>N%rW0oz@JmOo^WEO$BT45~*N9w5KcA>&
zI!eQ%fJeohsY$mSDLjpp!=?cKMy3py)wFUN?=2A8kS{l=s^+<QskB0wlUq`J+O9A%
zWH$2+LCKpdgj61diSZLoO(o&CJ3s7?(4I+t9VJTaQ<WXaTR1<XgITE`&}VLYg0PS2
z>pJgeIVXc?C$5|ErM$%4K@sZLFH>Gz7`Azk@5*tGRZSC~mMQutfoTbmF^fKuWub^n
z-T0W55R@-SArLdaICnxRh@~=+*B^zSg71{c_2gH&t(Z6z6h$*O>JF&n#`jR|@>^{1
zioqzh+(+}PIl*|%m@T$HxJC1QVl&wWd-4Y&D_76n)71hwC)2g)Bx=NmmZdjJiujzR
z^meLVpgHD`+EP7L<QNz^`wif~yi|%d&ZyRwSY4&k-v9ILRV^>zK$on;&iYUlziSmc
zYL?^oH}MJ>DJ+VRsIe$~{g>>CoC!4S|H(B1{w4{j#lac0dd%EWnJAR{s0+AGeuw%9
zjSLm@E{IAw6#H-ft2qSGiO+lq`bo$6eS0Z&z)UVFq(mNVuH%8-sM(qScJ@ECuK%w)
zo2MWzybG)tcJz=cEW}Jo3p2zUg9aymF~JPm>wOPrPhj<#9MZuCJ1YLAm9CuMC1qT+
zuvaa<gnDqw_Ru^l!!B@TyS8*u(^(F)QU@UKM8Pzp8@`-LXhzIDd>1a!`j0M^ebsZn
zsI5R2qD}Nl;TPrdm_F=0f^L2j_dG)H%ZPwjc5D*7dZg7f3b+!-x0i1GvRwZ;wPZ-0
zhUWL7SBHO(gu^#oM*}3x6YE2pW51r~C(AQX2<Uj_C*KwwZM^{c(K8<lS1bNu&L7^)
z5pf5jFVR2R8pHuJQk(ANb;JPu&)6JAcIC2Hp8sT3{QeO<VZimMCpGhjwN(z!0g=DT
z)M2uVcx8<k#Qb^=_!nL&;um4!`^M<+8<W^(!mM%HrQ6_0E7+!>_gtviNw50*7A(E8
z&@42a^5RG<PaE{;e>?hjZ~sqpbmnN%ONb;Xe{4K_{D^nh-bKUta{W)g^{4!v$$<tf
z_={qVLk1(#tzIO>2z)0f_;VtAS~Pzxj8Mo3;b{&k`mG&1;yUq1T6r65%ioGN-1@PU
ze&EF#lVRtC7w(4`+wbQGw5<>Vq8O*L_DGmyNa;iLlydw;GXE^lwus$?_Jz%(5l_Mb
z{xbRre-~>x4CZ2FjIk~3{}tItj9bAt?JRwzaU_b>GPL0nbuh;fTbDl{sFqO#F|SXi
z`!6nS85<0;f{fjxBN8VLs&a62PAeX{g;bype4-(#oBzL>OX1P2^soF8uO1G|-U9f<
z!dokU1RsaZYC7=f9QXbb6dis|1Eda!cJJPidE|)e<Kfjm|N4KYv%egSKDQtdxkZC=
z3vX=MWB(wX^rEwlOUqE!qt9Pt?s4T?bSNaCprU04d^n?jhKNQg0RDr4MF{_>(~)v^
z!W?J*s|h7gUCQ}B#|doY=f8i^l$}QS<*gOsznTK-SZIXh$nN~S(XU^8N=le!cwzJC
zwIqaT^!KEHed5nwe20V8Qw$}F55L5(Yk8n8x|#O(S37KVz{Hr>>GeHwEjhHcO-KE@
z>gRVXN4s^MeL>>Lwc*fa=_3EnGa&ykM6ItNb$fW^T3OV3ky2rXKMxy#5?CDYVGO5I
z^~kmVk=<8Qjy}dLMML(ggnj>1_@DP2g|&5?-%K#BjvnpF+vRAHki)kfNI>>#bvMXl
z-VS5mGjMs&@7lzd)t8C0QQiJ^Mn_(P*Od>^!(VadXC|Aaj{IN?BIPsckr%;$b_@Ba
zBb|<sOZ9gwp#;iEIWrZ790|^V4`t{_A|wh6t0#C!?a1d5rg<EBOBh6`^~p~u6^}ID
z8aVNHGzUo0(FhakNFNDPDd*86-<nV!1H-1lp5;g*mdg3R;{+8gEZ~EZ9zNrd#(KNs
zf5){H%}K&E?7~M!9tAq-NPl4cx0C+cN&n{}spc&0tCD|)`R|afYsGT#pSt<lnA&2B
z_DFbs;dGwd+<fT2c?TwegOwb^BR`e;a6Irccba7QcikiH&ZPk##I~1jod5j+NQhrV
zt*4l#RQ_u`2FAh+-flAa{_pWe2(v%oTid_Kb{Yqbu<HKKgv#I7s!2JAypsKUA^Kp6
zTJOF7cGKhUYlC6VX%bBNdyFyTV)ew<*1Y`tS{hiJXiNJ3ehOiIDd+Sb-(ElX``SkV
zAMjivnEo2(b?s=kzJ7eN`uXo`u`n@gLaz$^{rD<mglWwFzSsUty;1>EuyRUY$^85A
e|M$A?2dV*+bTK1SWh4sxQMjZcQ*zPt*8c)ByOAgW

literal 0
HcmV?d00001

diff --git a/format/diagrams/layout-list.png b/format/diagrams/layout-list.png
new file mode 100644
index 0000000000000000000000000000000000000000..167b10b11e37e761de81de8fa9fc8c5c9a30e4f8
GIT binary patch
literal 15906
zcmd_R1yEhVwk?VU34~z5-7UC#aCZp7U4mO+<H6l6c!GrxJh;0<2niAh8r<FC_vV~?
z-o3x-)<6H(t6TNzr$Vy#THU?6d-a@ij4>w>s>(9Yo)SKVfq{7@Co8EA0|UDceqxc~
zz?Fsf9(v$ESXXr!ahUQk;w|t6#Yy&!D+~-S4fF?APM!J?H27tsspF=jq$ptKXwPC|
z?r3Vk;%V;$TEoByc?y7!_7-j?<ev6+4z2>8!jz9K1i)wLWmZb^$0lyJ!jw8ns^k)m
zE*9k6EL<#Xlp;^b$;pLW%q<1fC8ht_9efj}w03iI5@2QZ@bF;q;AC-hv0`QC=jUf-
z<6!0BcnMm(bcHy$nRvc*aHaakApbFrq=l=Qi;a_;jiUoObX*ftM|U@2N=oR1{`Kb{
z^K`SZ{9jLUaQ)}9zyevJcUak3*jWEHHs~q@y(%E#Xz%1=;pz&;7vUCqZ24cV{jcZz
zqrIxNqnjhx1s5ALIR`fj7tq<w1Ufqrj(_(4?_2!e$5L^zu>h-n?9Kkq-v9a9Kidnj
zLKps@HsT+<{CE{?v&d5+)_=V+k*Afb9DiV71lQ#xUuk;6?qwtS;7C36Q3R_deiuK`
zDnOP!$ZOO2{G*xt$15pJ*eAX()EFq$@2JShpVDhJxiPTPB!8z!R>c;R6-PmUyCoww
z=x-jj;BEZV{Kt2{n{H%G`Nr4Zb$o1WWWUhw_R^WrPc{Gn{2fv37$1qTnv;9{mV=R(
zTtgNQgAyOGg`zs7bRbSB21hLhm+W#5i$xBL#U>0Z-?(KU3;)<zeN5!>ZV{ar18*y;
z4d?~~dd(t1?{=Ib(61GFP&@<O$YJIAZyxWa=fD)spRyFdf<dXp7_7)1?@nUpC1);#
za6Qg}er*>Xde_(sJa)1<9uYbnn8GaI$GeIu0qPE3ct2pE(*cux{(sP9^(vpAZcT3V
z;u+01xii`6HNUg3Fl-%4VR~I@+<Cq?=QvmQl0B&2aq)gi&rz#Pms=*5xYl-hf4X9E
zYqB^Sn@q?PBc$GLRy|9%$!%+=qCK`tA&uovnO?o^bUDA(tC7AKB0=|U%`)BE-Psze
z!Nfw<kK*IP7xVRtemfjTw^t`>mLX4H#5ZV{>9%@bprfL?Dl?2o);PaCTIoI8nSPFf
z!l2*aG+CtdoS1JZ%TObFs%yH!NYwf}`8Nvx``gaWP8juK<?M-knZ-6=y>bumZyr+j
z>gsB<$3YI$>+*7g<{V*fTpPdZtzs-1`GAqjqt!KLN}r4UKUHS^b=DJbYocdAHI_u}
ze}C(bBNHq(k5)p-<2^KEhr@Vvx(*iEeljMA{zt&~`gAz;rE47?M*^)9VFDjHxflW*
zJiJ=C#EMcT&oZ;}pJtB=tx|3DwT<Dl1F(ZGsWNe-?wj8`DL(MoO_yh9VJ+`WS4fm)
zVHYc9zUv9cM2`sh;JQB2Svy35s@Rb-LeW|zp%fxvMM9WIj7zU7hFI<JM<8o^wuY`e
zSv~`7m^YY!pWj1t>G|#~RsH$M2hLBz1D`yRO!U#IsHoPG(dJUc80T~yOuBog+sp?O
zFY6@qDEh&wj4F*^<p@DAl3wcZ$i~^-H#o0i92^`xfY+|y>UGu`goOV~<l*)tJh;tF
zl9>)yJ+?<FgX>iwqt=(NhOLCFLlM|?lvwO!K8NqK{C*V3eacFplF1Qp-yX{roS2xP
zQ_j-)W<8NeqwqHw*^D}>N4&w~V8M5Po?qWH2eM8exx(qVAjIFwhTW}`C7uB`iA3<7
z-9S7AJ`NQkFRqJenTb+J5M2hWlwp-=??;I+CtH`xLlY;@ADfybPR{CY@zl|+VQ$va
zj40W$l3(i8xvUK}kOi`!!gYr})2KGqXC)4!-TPgOy71}>f7)s!9TTEg?@(4?-Z2IY
z9{02@p2BB*IE|Gx?)SUhnfJ0jmSb5Z=_J_L@Xj)S)7j0EWf8Eju*liN9lz(jCuS|U
zm18JTueG1U#eOpMX5cBMGm4riM#lhVLUq=l)!N~>=)-NncQ7;i_|?^w6|hOco&$wG
znCc^6_3HOWGkH^(<7KIpvju9ibofg-mJ+%`_C9b}bd^PXel5$N5f(v(8L>r6EIu1Y
z%pb^g(00Ru&aPGX?PO9#R7!DwzR`ln8F$*UXElg{mWCa^j^=P&IC~TK)|LRSp|)S_
z1y-JpX1;VxajqX>nC;Y8HA?n1uhqV1BOGR*^dlr;Rsv0E;l4WzH2KH}vu_A`?CW3o
zk!dXRg<K)<JBOkXIE9<=1#886prfKlSFyI=R{P-67{7bMj$iO_e{*om5bV6x;nRvA
z5v8Fjhj5r8?G@Ie`&>hZ4~KwTJdo9(>8~m`b5}27LZHgVSa#J+K)2i0kL3!EZuH~y
z8*wz+6$>KFAXK{bQA|~q=g22!dcb22H<7|Gw?Iw=(!=z;d-Pj8)gthif>ll4B=^pm
z_C`{`aAWSc$m3#me2Y$3Q(LLp<n_69(#Imtmp8Am9L4C1Kbf`4y#uo|Gc!xkaDkEd
znJ*JZSp>|^r3HC)3?>aXm|bes3AZVi+g6(&D~-lE`~xw+2*L8-7k&@-zM3&A>mwhe
z3L88UZ2Rrw@hhp+kp=c0CE=VuQzJt3QJm;$4U;_);;XGEJ|SlbLDXZdW^xd66djJu
z$lzwM;mlZ#+P<boA8~XM@z`NZGRq6WJtq^v3BzA7bq(=(YJx-}gI4saM=?ALOTi}{
zvoD&Tv5Wr2oSGqI?FE`R`!5zFA=M{SY6)s1zQZ)+(M@b{QW=MbPGK5jVd!r+v&d=K
zBVNS5Ay?G9JM4ariHVs$oXToB_A6pIk(jjtGIZ1>7~VM=>H5p`Fy}b)d82fwk~|1F
z>isk|vpQT7$|DPmSG{@9S129!Qamv+Y;9L6x*5*GbTE5G22ikRjh|tA#k5~8{f61=
zUMZkQ(m1DFswUv0p{1NxD7dDD@X)L$JLyqTzfefl>F80W(1gv`cgWW?YCc5&v0A;s
zd=aQMmc{=IZy;HlMvS5Ui<QBg`~58>i(czXvP+X}0ww--+vyL5nT#??FCo>F@p0h$
zQ(28-_kw2jV&#dr-v+}9)*0suu)hv~L70GWz#ieT=yw>gPsatQ_=1=bXt@g4=(H?8
z@`S;Dt_4!Vk?=|K?b8lHN@3cbBd36(7DYQIxGbjngl2Z?ekw+`nO8%YL8GpS0Ucc=
z=jmn>FXk}RNJ6u8s&LLsbqV5VDToOPx971WP4TE!deId&5OmF)P-Yul>1a1=E!~vk
zAWlLsA(!vrk%k2Jvp5LYBa?|%^US}+zwrqMI6#KjM3U2TG$Z7R%I@z}vb|1EVt2FX
z=;CKF);+oI#xFG$e;eDYmYjy*8H(XIFICkWMBuzv%a^7I;3udO>|OpkC1H2jjf{5#
zA>^@3cj4~e$6>dpIZvXG=?r;0l*}+>att>U5KK}yaC5bhYsSqD31LekK~^1lxnuFQ
zeR)~VgcO&pYD?9Q*XN-10(8hv{{t*%`&a2I25MsZ*t;1IWG(jvu1|lJI08!Sg{D_4
zVG+A^geGF$#m&QV7(>L$*sh``c)3Uf4JC!XwSf?n-q1nuu5Fm9P+nRI_N$!V-PtrH
z>N<oC=@Ub<tEnPJ1i$U>cqbYbRscm^-|E|=Vb*rO;%P#vb89%{_58!nX;}vImJFqn
z#|)wNGB8Fwe$3Khkd`M=A7)g;C8{jL$6}87xPEpT?#gI7{H-!X`dIP0g7h3ca*rk6
zkH7Ue9K&2H)`LJ>6Y+$uHwv%ci=1>=^IXO(a+YHH?_iSiBzar#j9kPA=PG-6Cck6&
zc*-<3?94@yr=6U+pMoFm4*b!Josb#Q>I21wvM0%<HI{>rp5t<_K*(n3(Sj++{BNtm
z``1)|Ql1BxG0K%_+Tr&~N%e$VNmfy5$OXXIY5tCOZqo6DQR#eoqw}{1Z7vBW_xxMl
zVhGwYH^Wd45<No^ZKWeMyDfW`&&en+dxs`9JYn!IhKAzsK;t)g{&gf`p2EDR-!RN{
z8g*z|{2%<7y&0!TN4mut4<oT?-eY|)sFP<wsJt?uXEv0lA#F?^kE|@}a?XYQrG0+Y
zVYoXo3xI(Ojl##|e7W@4ep-S{&2Rd*A$muCPwbw$IEb3<N|SS=r3xZ-<B|u!awGo6
zA6m|4oS_S=B3kJA^n+!@aVY^CnYiwQSz2b$yU)x$6a=)yY%+0;X3=Eq+x^3+*%y_9
z=C9z%{o}=3o_vSk)k&<F6PtFU>?{9WTQtX6iGO0>SAfeB>FJ=+Snnj8)Ft7gSCxbm
z_r$m!4S%IK^0<&UB!siM1jUK-Q^}t+$yN5c5C__X&Bfx6e;TlE1z8GZJUe7;Ih`b6
zFjZ`NBQ`v)$kkT{wX$Djro2q~!eOm~u-uo8E%LjDBdiz2X?Ai2y$Ax}@;h?iFM`r+
zr~q{D1t9$8zHx>O0MSTLbIwaUQkHCiRa>aGe??`31)4|(2<lMdqa^}p9l(#`6xf^P
zXI2X30AiG43~9z}vQX5&4iGwT@#4%|DAtY#c0=>89;!p5TfB7{PzFH_f({B|&?975
z2A^H8W6Efum_GuhP(QGf97+n(lYqJZ|K<^w3MRRzeSLkQAtBiU?u@jwv?@75SI27r
zA?v3yztQK>)G$q>VM|2t`2f@QB$LP9#o7cQD%ZaQ%NT<af>@rrNx8wn!AGn8R-+le
zI%#UG#@{(q6!nbF3KL<IksZv!y&*(FhV#DIcfS-^a{ANc{#OEyd?`Qw7=)mfoBig^
z5*6XIpRVJ=csMw2_qSK!;m8PC-ZhUSyf^{A@ZoYb&U&vvE=fp>c9;S-M6AwkR={JQ
z8Qt+uQ?1*U`Wg;Ab^H(#tMzob0e0e_Pf}6TNR1g-J8$jQhEgVSug`XhZC^5HA&4yj
zLL=V7$7tn`f#Bmjd(!b99`F}wmAjQFRya7gc_T+|ns-Jx<BtbT4(0Mwa;^{ri)*q_
z;dP7W5p~sD`7CoA#YYxtNo^dQJf181a5imJ0f-C0-L}8J{A%`a+*NRf-vQ6$Bf`hu
z+(>gf;i6e+a5e)><cn5mXsR)t;z#iiw%WVB`o$)@5kL+4V@aaqasb~k8)~x{P6dRf
zyEQt?^K^59_5KbMFU83J?(87E{ho}=IzCIv*7NTAjQ*qF-L<u}lao_t;FF!7B|%=j
zF;NmWvJtwy%-e+uY0|O8Z|WWX=r_7_j^z4X4^kWfAfJH$F2eI~V5X|y){nww_npuW
zOkt;LYHAuqiZ)>9W~(hE#KmDSh$5XDY^PcH*T6mouhARDzV32tIYPv0xmfU)kLHF$
z#bdz<B@=KPD3r?Ewrik$XkwnS_!~<iC~1~;gM#bsv>o(Z#A)nv(fDv9m+!~F(fi#Q
z>6POTPc)`K&~7ppe4n+qnJgk<H<dKi@(0vwAW$l&2W#Mn5nI&j^s_`5`q;}?Xg#tl
zs=?z~{6V&HvaBk{V%UJiq?|UeH85+$20vxmszp5fl^B)S{p2Z5gUec^#QU@DUlJMY
zW_`EkbB-|d#WI&Q<046uQQ7tl<_rI1UlCPjlOJ!vMhF(Dp?^e|9|R~Bd)hc{7!mLw
zyv9nMGV?0wCBQ{XG>Svr9oz5fYNW!Rli`K}dw9cN`m0^Ok9uP$MQUPm)0!p?`5RyX
zFCklnX~Zj9=a2-29lW8BEWBa5A1cRD)2~^)O>e`Mse-|=|8tqhs#mxB!72yS*K45#
z5-BtH&E_PxU9>myg#?_x|3f>~{dN1B8cWL7q8`eq8LdRXBGELf)b0R>+FSt?ssqDi
zFp)-{QxXQAz|KsDC~qC#Ma1(EMnV&9fzxs%d|;|fk7<kjzN;ze()e#|4S>IQf-4md
z5)u+Ar@HW)Qw{C`Twisn;M_kFs3KShg$x%+y#J)vetSF&SZqJqFBvjlge^(<FRv?N
z>rB|M05g}zrC3{<FOb8|DpM~U0wCj~3^tvzB(<q(LL&bA<?gW68&UV|pZ0Tgohf*%
zhTx>pNN>ubRNh=1yvE|So54t!QHYER#$te@M<{l3zg>+JC02g3gpqHc6Be^@aj@vY
zwsCi{*k6K1z+vp&;38k@pAt=)=5pq1`0dpsp1e(6Mu0Z$?n&U6SoQmpanbyP75Mk@
z2Kab*uS?WvdL#;#c$^tM_UHYJ+%M?B2_BSXt~Tb1`IKF&;*gGoOXbV8wLY3DPYs8P
z%Z#x%*XG+y%-SWrZ;ZJPgN!KSq99o>O~q+7)?zV?yOzvBoy?khd_s#~GME|S)F@6t
zqmZg8FUuUDi;z^&b}cU#gjyG#T#9Yg>#o~I%-G0`p?WQZJg_e0bIIno&|-Z2+qPox
zC?W*=q#|))RlU4+-1b_WVN_X;oE^G^w?C35Rs=spw;TCg(YTmo1-|-}MFNjKQcjLY
z)x7BcV35_di9bXsl3pc6y&m!qd%5Eu4V#ppWB#fT*%&`IDlA-Xmz9*S3x|Kq6dm0@
zKIv5en@J3Qbp4v=YInLjpf5j+#pfH}%_F|2vm8v(uoz0lyb)x#`XtE+dua`5<FOD{
zDz?*<g(*0;dAL;qzNedNYfz@hCK}Xz(EeZ;A!!VZK|ClpYE5Gbd9@F7n61(RVEbyR
zO|~>vBWcArQsKXsJze{EnL6d>{dD=`r4QmX&06RLe>hEYr*|+ZqH%jYh>|GKSvpOE
z5T7|I#D09bF3=-vGn&DjVh$*`6n<MLa}5>)5*H0F!4ZjG?b}~ap}_b2&B&FhXRLFn
zei2q58ay>tnCUy&2pJg8k5W;%y*+_C%gHJP*nRebMWL!`KRnS2vr9v(SX&MQC4^}8
z4)sxR8A(>0`5fl?*z-SS!9hlXI}G&?dv7aT<WT~!gu=VC_+7i=Tojh{eU6+u`>tQ)
z>~*Ps83>X2j1e$|TFfN%hBjcBqVLaAqtBJghy}y&3-MXtFl~ga4*GOb?%g%SkCkcu
zvDykbd6pE#nT@jy<4&wdC&WG~mV+;B&4CE;h-gkCr7koY=9;+;{tx%^4;`=3+xk=C
zA=<BS!dXAchLRAb@3~(t!3EXKgssloWz*{9{Yh^r*R5s5nS7)5@uQ1E&uGzbh(7-Z
z<W)-<oyaS<jT#m+ps`I?naQAa_vR@sN5E(YtTB6Xy=M07dPPa9fWBf+0eoeOhA#8h
z`E2^(Oi#i1j*w<yD>Lq^*Ms}BD5-zXASp)lg`?Z>Q1^>k=@m0nRY%pfq2rU(ekiZw
z=D;N$F(C-^%5Jeum`B-|6nohDGD;-^iv~y_3~?zv@x(Fj=LHW!an{AO&K%0<$ux-5
zcRCbt(5tXeU-f&@c6IFdNfeWg0v`7>o<c&E<F{&oEaql)!?z`dm!DH4Vj48fVsM73
z0Z#{n2VZ^V6AKTYuKPPF48RX@%1?EaI&T86&-bEaqw(3w=n!|&lYM&H2X{^w+f~tN
z=^&d_ijsn(KR13EF>)7b7TkuB`8j>S^j?BNoukMa|3YR>1wh6p7B~I*EeY%3`btma
zGR=(l&l)L7bxS0_plq@;;>Ad!RyC?R1aai?*lD8MV|B%-@t>y;#b9<*hN<ym(cdgr
z-r$D~AXhfL9fG_h(AGu<VGI$z_tTR)Zvi5x$4AK_Nug^Vk2-I75NEL0Cip%i*{;^u
zc-@j4|1$2O*Sx$Tvrn0CqYIN^-KR>X>+0(QyUs`t=?S;(lra(Iwz!O@BvKf@V<aMd
zvxsILQa4M1TO)c&HA0wq`t5sUN|&XM_c{hne71+nziVxew0c^9MiX!*GZ<gkcdQi8
z)V)7qn14yG=Pk<;VC;+h+9<dxJ-QIC>U_0jsRTGof=UpK2OPS%UVC~r5C=I`-?Y&8
z-$CQhMOYJuvr+BI8`m>PYXPx-&(IveK0v?S4nV1Y8Q%F9(LsttnE(>L<x-_Z>^+j=
zRfSaLb9KzuW5%lEqPCKR7H<D8UEL|iUY4U`7@>i2DJo3EbS^p(ahg=fy@Z$$yJHBS
zvFjWyjDVAwmhN1WE>*p^9A44fgpBSdakI5POAwJ+ph<0>h6&^IUw6kFNNF8iA*id%
z$YF?^+eQ{8ZyMiA>{-i!8C5PVe@`^OfP}f|`+X$-(IEOWCF15w_IO6=_zwMAytvQa
z(UZ_3vz>7AClft~2sL4RJrWV7)H`SsHSC+6GfT|G`N`2cqNM))v>rV_KdVVZ$cWkH
zOJ=f{Yvw59pyZWZ(uQXfx!?x(BA`b0?m&)Oq7z9`1JD_?NLS|Q-LQFgXM6(nXShNY
zp1k`gMyZLrQewUWOOJNcQ-&7VyDz}(1DoJO;r1DQeABEIw#I%%PEC;`TsQ%J@T<p6
ztG5S!NVYP7XxwlO1^gAwU~&3FJv?-7Kw3V}dzOyd{RA<LrQI##x@C$r+3R6@NYa;6
zQa<Sn;wjbs9QUX<^?6=rD-VxZ#Fe(2EprHsc16*aBX`7$HK~oheN<-Bl?V-sanCds
zgDky3BxWg%376<Mm3NqAe_yOGP4I0nV>q|8N^Y^txP%po`q--<FSQ+i6{?%eoBcFu
z-Q7>-&&<eSS592PBt|4z$c7!|O{rgHDz&>SIRh!zCr`{}>@fQ@Hvg3{zvU8EkJA~&
z&8CR<A&||%tp~~C={@o-$wu7t?m(^2@gN=LvwOzOST<)?(uUsTZ&~;zXTBl?M)YA#
z^TcmW16@utnRs94PTO*RT49V<sd}aNp|QJ_j!VrCLrty(;pb-n=r<whoX+V40qU#6
zp|2v<lO<qIe$EI}rdp*`Ag~|;@wDaSWRsE=cd^Rm6{+XSr$QIoJc+#(j5MUrb?HyU
ziHzY=pB>q940#N)G01pVcD)~t2!xyd@?7*5!&OzB5G&~z$Ouu)o{#_qqB6pgTR0R4
zeXJU_B!6u0i1-yu#nIsx%jj6}v=(Tw4iRZcc{_f9)(@;OuMLz|kJBBX($tSC>0o0g
z=jSHpJ7J$)(TxYt8@y_V$-xiY%tKnrks?ste4Ra<$S(N4*qd!4nw`6Dav5E7^qAz*
zST<x3f!Ulwc!A?{^pwFAC1uqPdUfnDjP&V{#fXyEMX}iP8m@G+PA8qlVdq?__8J|O
zj&qbvKGn}9PlgLj>tH!K0o3@C!Te4kztq21fbJAi4=tJfy>}Pmjn8qL>VB#A7&ts7
zQWhOJz6<0GuOp%dnWbUyloflI#KYTXF#ie}*|B@uprd#K!)e%+EB>7Yqup}E;kL=Q
z9&+dj>ps7J*NMLFT{XUQ_N|y?UYf`Nt78tf$}rf#LMNo#sGw6&uBVyJpva0EB|STU
z^%SC9ncj+brGHb^h%Z`5oe^^7*22(<?mg%`_RKE+J%Z22aaT=;yzV`<7d*@+(+H&L
zU77AgybjS93A5b}x;92|ZQnQe8S5P%^49?0PNnFuP#JDW3<V5(CJiv`wzlGbTnAb}
zG?NW7AospN3)tK7)>Q^t00C4U{-1Adti;gon6=uU?0Xv|ty@Gifsus8cA42F0#Zv=
z>o-FbE7ue!#)FqOuFq5ZKFNwP!LGJ5?4IcQ7I3tt_)A$^UAm{>M5}1GN`@b$R7+zy
z$t?{{sHPp|#EfV)Ra>E&+i<|zDpwBRawtMoi&T(}tZ#|30m4JT8CZSUftfClGmIh7
z)i<XjCI)Pv>@IlZjpvz&vg$2geR^YBjjRKi<$NPX%=xF8$m4YBJ|F2K<nO?n?GIL{
zc-}+oPvYb{{1Cia1i7}iYkQfde}2|45o;4VhhK8V)foEI`1j(hH1zo<CYisA+|L5{
z;?CUYGnGv32$KK=J6UptBE|S)H&|Fe3~D}JiCA+K3aJ%;y>nlt3%b56l$$KP5Pb3F
zXLyU}{SAZB%LfYwyVIBn4S!iKB9-`nXUMmgd%VNi58C<iy~Gr=cE;r>ir<c@#2IE^
z=fuZ<Z}AJUeNFlCxBJ7Ss^{^KtMIYW?fyS)XWbQg^2v;uSQnALYd-pvG-w}q*ARNB
z#HII{ta4Qvr}a}i)EX6-OL!jY-~<fIO|_)@6IhHoyGC_AGA`~kZ*Qzm&8PVZ6yUU}
z54UZFu}MYTD!+;?2x;{DtOJpv0wju?``Rr2$Y*(0$uV7>=8))3u^0uXOWeF0K0-?h
zSVPGYAUZl+rs^F*LP4o_Sd5L0&@#n8WC00e+xyp4tjm(irZhcUfDZayjHcDJ-|5cB
z8@6I?2@sjdb$miVcf3=ovvZtT4&7@WVn_2%gSkj)UiatT=1>kcACG8{KWih&9LW%1
zp_VC^b4|5Gdm7vB!OKJY+g-WuQEXTS*{Je_#psyMh^3J?bq+d{B{r><t<7JiA*7|*
zFG_zsMMmx@nd61nZV{500qHB6t^);kdZcD!#CPMo%&`r>&7pUN%jU<a??L{U@M72Q
zQ%l`S0!+wZPy9|+G!0bY>PB(W+$V&L%|o2w1ag>E+Db&rJj<r%OjCu{`^aXLQW#uQ
zPYF1h#s+nu5*k|;j0~8QM5ps7#Vh&MBG!Q{n$pD0@AtyU8?2g|(BlI1v=-%ne3xmK
z&(lkT6}2XPLVOyQ{JlfHC(bhkS~C2JCeV@oxsA2C{w2l<D|nR34~x^eObn@GxvlQq
z+j&ch6)nG|Bh3F}JZ$E2gMbf5@4MUa=POXp6Au>JuC}Vzy5mShe8_P${^t5}24Sz2
zm$0XTyuAVfAEOG6Pp!bX|HTQRVp^`yJL|!gH_QETnqPHohvqG2QqrwSbI4)6F+8&(
z+u8Q@a@If3XN$@U8CdppE$;XC%NPpeRv2UY`>(i!MQ!X#EPbC?W#cj0*FR^U?SEOw
zL$j9`;I{Sj#oZ3deSrq&6O@^Nj*J*RV~Yq=DLdm*e#L_1%x)Ds8Whijj<N4|etsoP
zujw~7F3jcM{YkZ-^9Wm+XheUR38~Whs!I#xN*gv{_)CFeo9~0D`jke}DSX?68XYD3
z_2XhY+X74uPFXb)JxW=eAT*Kc`!9CCCL@eUE;R~pk7qSwygQvCn|2f%%`G*^m)SUf
zF_g-a^i124$zN(fkaWEQ84@OFd&diVB=Gb;@_Tzp`f+$BrqUJ<&!hBd4qVCy#O_Dw
zQyzTQXo`t>ls-QI>9eSl?NRzn1dch`!7Tq#UH#W|#o&sd>T2QKj5YLy#-r#!j2i8V
z9Yc@iT-s*Y>V{|$dfHs5Vk~l}BY0%i1m0UzJwApQmPxdso%6QInVfryI_Landc>fd
z7#^Io$X-blEHjrDW9-l@>c3flVZ3HBj^l+Vx&^ru+uTZ$$**|8OlEM+@p-~H{h26>
z!S!jB#k<Gz#s@18`6Mgq+kSAZrLZ<h5ESU$CR|`Jz2HLqxcnt|xa4C=1Jz2|BlJJl
zB3e!QQ+5=^%A|8Oz9(a4ME!BMm14`LwcEUpUJa@`FC%=t{Z`z6k>X*ed2>2w_AIj=
z_?*8uUXksT$>ISc`<QUIFDT-SPea5Tc1t=@F`Nt9UJ&I|JWomt&_bSXbKN*OO44)N
zGx)OZ|Camhue)rDhq;<@#=Z{==*pGWPRq5WYO)4NHB5x*>?Kmj!~$NoF|a}71}(`v
zW&RPLq*$b+vvrB8R_dP}ocpcb@iuz>^<{O{ET2M}<E_xUT^$j+LY?xiO870LC<B*D
z9uIu&#+}LcBNltToZqb$lZg;I4+p}V!La*#o3;!AU&k2}1sxj)Dn3S!jrJ4F;(i~Y
zO(5zd;EhWeHmTNX-;K9@NMLZq^?-QBA~4Gs?{^h0TTwEmFbQ6-evEX^*5ef=_4B%(
zzcMRU%)Ic&PWD7>5eJ^YW9DF?(e>|A8;D7-`V$-$_xJKe2|ZV@x&LFNn|;-ppC#_D
zNW%i(>_XGK`;`THPQH7>5vMep8CEMc#@tZFuQ<$;2(fVau2}DKiF$AMjP&jR4fcSA
zZag6%1UXB<8+{6gNq?ZTsO##4Vtt#;e?Fp_D4jj1`wc0f9GHdHH%@c3uj(}m!Ed5m
zHb&#_!smF(wNhdtWIM1B9R4&t6Rcd?GFB231Ifl)%oj*pp=9bq+~obTEpAux0~mlJ
zgxV>}!_m^IHW>TqI4!Fe1d*hD%Guo-N@B7yHkJ<45aLP)Mg#A3oBC_^aCJcSljfUe
zv+2nQipqJO30;-OdqpqN-!bd8qO-5bi|%2JORR#}{);0nuALV$LYGe84}Z7Dq@|G=
zvnAVci_qk5okXjY0b+Ie>}6cWfuN`7@d&@Cp_^)Z&N}bJDK4L~--iYA=LAkWmb*8;
z0&7D_qHOe=CgNScm=a{V&ikYn2eP>5J~O!fuv*$H2otNp=fv>*qLa{8;gQ4VtDSZa
zzcck&x7t&UIe34jmG}Ps&coZoKQ!Nk%!zWbbLBUR>R#HHRMoMotE;uK*LiZKvG#^L
z1o${#d*}o^JSwS)NI1RX1ByH<Z<{Rd29A;_$BZQ5*<@MYd>ueKICWI5s2e)}%V<Er
zz^+ijA)I6b(lRPIc!=xM6$TOQ9<20y8s#llcXz@NBnbwUWI+{tS+S(#sCR!Fu|3#1
zzNo(U*_tez{HZSANUN`l?X8~kZEA0I>xc>a_9F_}1!ks|5UibcCqmTGS+(>9Lc{J>
z&hvk0^e7<BqqIcLlK>4x^Pn!Pd851L(euCo2ZzlUp+pUhd>0_P?q3};KYF+T;NcLI
zO|hXKt^-;j<}D(m0EJ?-ASuQBK50f0>O-7B<<|bc^B8Tv?9_wR4ZRhgMVTpwkDl47
z2|U%~JCd3CF4-N`|A?fq0V#{o4gcW=4P8pvv6GSlSP;sIpZzMF_s3&2bh$a;BZWkR
zh)XIl82IQA^}wVTD961X=V=DM{C9W0w(Ncom%UhBeNKif?0al|dL|mkVKJ6eA+|>7
zeueGKP%yL$mgfaP5S{$l<i$<7QLUy(HamU8#+<}&_q~@g|A3-~0E(*oeDWj;=iqzU
zfkxr5#VD@J_TMVuIo^Z$Cg<CM=cqU=+NFL$k~K#N(5=K*?if<>_VO`;^h?*O<O)qy
zLYm{GRg^y#gJ2+Xl`0-Rk|PecVxvNJky0~W{M{nLVx!v1)Lg$uq6gEPiKnlZUpgV+
z7mASAsrBzH$ewqrR?)kl-!Xb;#C)eq6RX<1zq!uUhPWHHfB&K{DBmKg0S00~B;o@V
z_la&dkYI@^)q{kArG7z~Z*eE06uBL)KqnDYYqPU?;JP2yuJA52m?rak^Qj4q_GbbN
zozBm$KW&gd-~9~4UDh+I4(shG_<vd|P#Zj@)4^pmoClecxq3%I95NgnF7NYQ+?Tq1
zJty%9`dF%>ogC&ppAR<fezYtrtjhQ$ix>zb*LiiX^CF@t<Hf)0|I!O_S#x{SM$AAu
zzkv`n(i3D7JM^bhh|KwVl;!31ySGyS`@R?ct@T;m?EHrk!?d%9))zh`635x<!kSnr
zCQ9e0D4ICk)YN&Y9ajoz?APdn85X;AWXL2kW3LTAFP{JEbM`p6JLJ5{)J?oD%ujAM
z^S&6%rb4<P5pG~T{_&(>uHO4*GY)Z~VSiK(P4T0#(0=(ui8=+WmF_&%${Qtwx`U@m
zva+8wTHmS~=VevNhq&+wL1@Ke&1aYp8VDvx&J!($?B~he<9Mv@FFAkQ!j>^BzM9pb
zvt%*MWijyb;CC0~HR(b~TXk*}rxvTmq>>&;KuZ#<2GH3X9a4WVJC~<L*BC#er&4su
zqTj?U!?F7Uc=gh*911hzl#Dal5#3}#N55T}`8wyaFLnadx@|s<8S1aU>U^Xnn2Zk~
zg!5QgiD!<<H4^*%?YNzIwpyFzL2)aQ<|j@UR}<FKXMK3X=pKc%3LN(<)4#nZgw+tP
zEx=NKC3`G&z^?j}ODlEzP$+@AFOaz0LVd}uK=gEdJgjJ{LhCLwfh@Cd^qno8%o?Fk
zZ0|+QvI@_SRC+CK2_D)rcDPdQvg0b;D=*i`7cAkl`@cPwI)f0597_%6WEfuWTwPvz
zC-wUTMz{!lH)>PouALRiF4w6teH-~z^&=dM&@CUfszjU(F>;G)v1L_P$XHJE;UfF$
z!J_|?Q)*b7SO5UDt^bDS9@rGfysrY)bttH)3e`Cv2}AcG6h&5y1{UOq#zWC}uh+dN
zH^z+WA+|A$FHY8n+z<S2*9MqmR6<pVE~HAZdbMzS%>^)*sKm=V#cPeeT#uW^=H?ph
z`#j0`wmq|6oAdrvXZeYBgwi>c_>2Xp%Tv3^)H%%i{(;lV<*)3PtD-I%hmvkpaKJ>9
zjqX<~iC?0W!P-z_)M;gQ9!wG))yzZ~av(KmDz%|V@S4qNfjV<gCOA<tIS~QCsVx&@
zzq{Wlm);kFft+hYnPl)ul1Xnx6YLmdv{o=Fc->iazlA^dq3sJ(XGxl%UX$+|R(t))
z%+V4ZeT+0w3t}qEpQ#g|o`X*i*P}5c9H{0>!I0ph*@tw&jr|}9UwG??rE;&5lRQlG
z`MJi>uNrx2c@}S4_>7Ah<L@yH$tK;QCJ^jNhD-7Q2}u{Becy}9C)CD#u(3nWWbUm(
z7>HB6iziY)r{ufL#t^3rb19Ml9`JMDY!&KXmr$TJzp#(-;eTt*Z{_i1PTPS&Tkg0g
z*^!OPhsHdL{!J1~0F&AN3rtpNYxvcmU0P}b<O6q<)_D1rpkCA$HS$<alum2bpEa^J
zsCqIng^gGgZj(Te*hPK}a{tdY$Cby`R`M<0x3W4v3sECd;<>41uH=7V2@ZQggB%+G
zG4tGqxmGHAwdQR*t|jVPWm^7SU!@|lErcFDydsE#gq|rG?f_7^iFmBY&Md=!-1=eV
zB+`rj2(@vc4EY!kINXP<ad!vZ=Lr>cJ1ahs?9d4=y@b<0|0#A41c$CT5*D>Dc4gPk
zxXjJeu~cP@=j!Z18t`@6)OSt$ri+K`IRZRZ8Sbs0t=mzfmaLu}%?-v|k;i97C9{=l
z<*i6)_RpAHrNDF|<td#xX=6J-l>06y{T0fWBbP!zM8l7#Q^~o%y9o;=Ji08ruG@bm
z==N9A^WYVkVQge%tNTv7S>FM_lT$~fN==x#Ac?cXce>nC)<_0-fK^_jzS{}qri&$I
z)+xAjW#v}O4BL_wqk0d-EX5q5M)?#*ke7!RrfO;?_raaYw^{U$g`?t4v)(!z=aM=*
zdw92ByxQP-$-{hIr5OP-w%Luo8?lBjYV_Sf+}`FZl;<{Ev8MI~WZTa#xCobUk|whQ
zq`wfK5hjBQLSX5v`VHTAuvvK>|B&8V9(;YnVfB3QX};6ta)2Llu=#Lc`*glpqCEqr
zS1^p0+>5@wOuugBlX=KV`~0`sFKCLAa5?0Mn-lq<IJ*v127|olLbm51pq9<AbLZoV
zjue90%QAU}vKa7T*{dP9n^klp76gMJ-oNg3xXzV0Ru*131rm@JA}u9s99DR2?D>9=
zf|wX^#1D3!s82^W)oyfj<N<_DqHXX!X)+5tw?ZA90NT+bA>Nehn5H#6{gz^3OMW<R
zp)En1N%NxS?_TqnQW{I^sAB!s#=ZxySSz*22)vP2^U;i2khwBwepja7SZ!5mY_m?;
zp$5Uazg|2biY1-L157Roy@D<wxp-78516&3HSL@%+;3zj%^U<)c7=s&s3TLsg<P@=
z!bY3>P9a2p1xfbOB8ZNi&sxfBHhjn5-xw^bewmvP+Cm}W$pPSWam2{I!#cm)rX~#p
za;RYQk&_uT5T7oTXwdE2uzyjWGJ^s}59JII6C7n;$7veZDZBk2N5m+biUF=G0`NS(
zM<ryUbxWbDRiD>xL?`X!E{h(NJ;z}-P;gu6)O&b)2XbwIS(o@bYnEf{;3~i|P<BKl
z<1E)0S;{xw`6x&ErbEkCe%CvWkA{fsQj*ydf|5d#Zp+#D4p4UvBG3>C1CN{UkWx`{
zaTbLFIqW#bXM^TCtEZ+6=Tsds!Mhu-x<EKE<_Gf}ZcIi@04i!6Fke5B;$C2DsZ>Bm
z%u=wy0A%$JpyIv%H_dgfKwkSmsZDp_!q<Ecm+-+>e)mM+jNUeONa;v{J%*6`$6yIF
zRBmxV_zD!=OgtM_6zVt_BbQ7e2a)vQ3_v6hMrpy6;sgA^iNCJ07<_b47{EKc%6sz+
z>Pt3d0@TaN^im&XPejbTWFmx&U*P(`-(|_sn-JP3paexStDVnru%h__)GT@LPwmZ7
z%vR&DsyE0Kzk4X@U!*!i>37lG$96ELlI`e6`KsG)bNetl-zv&%AolxH2&!RhQxoK7
zEY=rMF_Uw=U(X?F{ixG7lP|+EF#r{An{0lu%F1zfRPW4IYdl^=_-PX+Hs^Bl8;$l{
zqIjOYE#Y<~gaUW9>-Kz?l=5igHszf~O`&&^3>WzFxRP*1RC$I&*-G793XNW@{eN2P
zO(x<j6jI!jmoOw4JYZI4C6p^YQ|T0_V4E?J(Vu#K(NWX!rlOr<;kl{I);~Wr*{HOk
zUQ$5liY31BTVMseo3cW`)#HFP1K0Hg_nz?KhpW-26(;+S$v2U7Gw4`{7H~;cBUh)T
zDApz+)rCx{Oqo*;5pH(B&)YX=bd>#X0$1tR+LoTw0m(k2W7ZW<Iy!Un;Glh}S+2HJ
z_)RNLZra?t2oaoNwW~#;mMhVXp;ozE_oM&@<ls<}uJ6r0Tg4V6{-9)F{!ep}bvp^6
z+&^pg3c_LD5mWmAtQ92{lErvp(10`BVq1Y-Jhl{%&CC$E^Km50_y|K5R|0N|MzvIT
zZRA5BtCTdi-D0hB%on<@al1v5$kxAw@;IWnenU;~G6y+gherRttV=;?5hwSq`|3&?
z)17pU6i3>By-{c2y{%bM4cWTgPRTu!_tq)lgcYIuI}*%JZHxz}ONM#9D(&W`XqOOp
zRfX^+1(OW(hrFVO*XaFsO#$pA7!;idn9z+&E1D=~GgLYq=2dw@abQeFz?g2Xb5fxV
zLV+=HlowEdgL@7wXRbJ~;RlD)0$PFlzqxtGw9V}F#kVYk=m%Kxwt4+x!+y0!W#c!G
z1QlN<z)_kjmfMyRc99wy{9&JAvo><)QF?24sx=DVDMKqCnL#9UVDR_>4Z_~wH;d*b
z5557BuR|SJU#bgT3<WG!B^@xn|1ZtOh~8IewAs!d#h5J@lL~ur`R?Cp|G7^7a+NDh
z(o{BAt)%x#vFte*T}1y4BZ^K`$A@rN@AEJzX@k9^{qN63$%y=DSC;d`J|dyN+vYSr
zInSP|{*^ohsQRN6*&@DF_FVK$J!B_%{K4CJ@{2E#_BY!Fgwb>^v%1%W-SQUTt^7l%
z+((w|?-|P$Xxl4|Ig;<q>*%QWa+BINqf==d?sg5uSlao9);S^%m30tLeOeu2zC<!K
zx~_jUDs9RolL>x`Rq=#=%}`=IQm(@dp*qYwX^uFZy*u!Lb`Xg4Q828<{}#di@2U5H
zxuY)XcRQo!USkN}$pKbJl8ALcg6O1T=fvPDamo%tYVV`c?+o5hUzMd2)Ia87gH8NT
z?_J%2*mg=1sznF@|M)-A1#F-j+_0%!2LeLR&bMegdZXe(Z2Q<Dcj!f^K*9%i->21^
z0Tc<J#DE0W#xpq>ZEGBvHSI%6dw?J-`N0ZC1z|?%z-Kt{`9+H?5om0u9N@afa{$AY
zi=c^#&+G2xrnsB=r}^(lf&l;-9H{`q_8nW=QyVM8ouo16o_Dtpu~)6zwVmlHG1Rxk
z*-!w>TLXCG<$)<7G#%y(j>OJQIxg5kogA1n;eT9j!f3$uxX5141^!&O{B$z+8Oi2G
zzw<|yEG=C5YsOdX1Ss(xq6jPd3p+opI;wx|>@T)I_&pf@Jo5kU$pHK{mZ!D?ToH{E
ztH^z=-@)!oO}%Mjb^AzqB8@BUWQJ*WsB2)aKwAL~wK2^5%bSblQ?ds4jhvyi+z$_b
zdY1CY;TbJn$N_39Bqs?i3q8bpfAgHQ<$fw}bk0M&Ovj}2RPLwq;uZmM%1tcm3>Ma*
zJDh|}^wVd8nxDn)uO?{`Ek~AiMr$>0_!#Sx>TQ@%3Zlj8W9{KNErmRmS6JK272n8>
z?3sYYKZmz)rZzT#vye|szZ%^HnI1F(_U&ozotJhcTZ;L~nPV1qY$%>dVt2Z~fS8NZ
zzBA=<Sou=$9i)cgtKPrLr1JvB`n4<2H-GJQ1_hTpw$?v<$9%z`F6jXt@dvX84y%&%
zJ%!_$&BW$fLqju(S1XAhZI>5f#uz2|O-lVf#H4@=lmg}KYe1-{KkFY!D=KF0-vMzN
z(ynyw@apXdWHu9~j1Kk&D}b`wlghv7Ltws!<LYO9x=JW~0BI`#Q+OYJ8!(b=;9CFF
zK}&X7)v2L{vWoA}V(kBkF3{5s1#7hA<c7r-<`P!|FSD(Itp3l7-z!ZdsPtw(t5H#v
z)@T_Jr_2rg;ex`T43KrIFEX-u0gh=IScUzX10m25jg`PQ+Fqw<L%~@jh$q_nMbJdZ
h6iA-@pWFQK0gXQMMp4Xh2l|WKa#G5Y<>JPH{}-IO`fUIJ

literal 0
HcmV?d00001

diff --git a/format/diagrams/layout-primitive-array.png b/format/diagrams/layout-primitive-array.png
new file mode 100644
index 0000000000000000000000000000000000000000..bd212f081151234c01f5814be4a4e4e1e4841835
GIT binary patch
literal 10907
zcmeHtWmJ^yzb*_zBOpk3Bi#%=gp{;|bk|EaNDU<*l1d6PD2f4!fRfVE2+|1BEdqjc
zog0|<egAu}b3X31&WG(<>fF!V&z--xuIo4PI$A2%@Tl?7(9o`_sVeEAp+PFZ=MEeQ
z_&0-Pkskbq?xUxofcEYy?J8*Cda4@xprMhoUHnB?(_`HN9fqCt@A}@=)R46GaO1PK
z^RTh!3v}}Yz0uI510}&nH+x@e#y~e$cOS_>8Rp9#lHl{jV}54F%PzhyGR${1br=;r
zyzLo9`9$~xm}T)885yO$?HnZal$24ggO&`lldrF*BtL&ZKmcEWFrSCFBfp@8gap5U
z5WkQRFX+MR6Xfn|9mwnM!*Vsq-{UCR``CIrd-^(ixHDdiYi;A<=PSd^d@<2Kf3DW)
z>+JCFOzu9YZGjE)Uwp$a$S1)6&)DEq>5He5iXLvB-u6B|V0>9o>C2w~<=MaUT=my+
z^6>Qlr{L{utLE-&?+sq|wZ2%LtPtw$|96l7K9;t(vpv}L<=cX&xBv4Qs=qY<#m4{R
zAg)gN@+mlGSv+a}f38dxuXJAM2O1iurJ9nwejxg<Jo_rzTO$ZusKVBa7S6^-h82@M
zgvTAdd84e!m^RXtJkTnY&3d+pB9f=R%2d^}67CsUtJuG3zE)6Q^TTJkt7mdmDrmX=
z#nI!v3C*=XukRGqoE-gpPkQ(W0cB-G$5n+x8c4Ga_M^|!|9NoEg7lsVzWS0R1x^a#
z!q)T*WV-B3l7i@p*!-ufCeABbk*3+0zsKUjuEzjfE}J4uNbi1VEXCjLSF{I&V7kj@
ze>9>?o|M+&?>3lF>VGG^JmLSLBQ<|wX0U71&086=l<oNPiF-{fLs-?DYojF}cKYPb
zqe-=p$M}3zR>KBHCA+ZtlkM)~joSAi{HMqJ`F3e)U73PLsGDPli0Ttgh3xcxux;0p
zJN@HEt@8svKV-YJr>ne2sC%X<N;v9py}YlAC3-sS*p2-ydFK99@FoHsqpR|ySA1GS
z@XL+G?kw&rb8;e$pytAMqgL_4%lGcOxmka_L2{cX)re3sFlEeKvPz{EWvFoYBC_gz
zQ+c0k_l(XR_lDEGCFSwY!TU>Pk8fWY)pr%pC3I5qnPDj{PwjeH`BzC7l|POQSN0b%
zjZ<Y>faGrum=4|F?&cS+dXZ^SgH-grdu5PF=$eBBrMUa#$5D&pF^`7d^X+}2qh>U&
zrA<irD2E*9mjr7pNw7!=cS*$nxv6Ue#|&@fr1$yheloQvUS-oQ3Ul91w#-S7hSPh0
zR#IP+@iT;MHHDuaPiss6Zav;?klU==dH11>`se$GGZUARUEqcJUTsu2XkeZ*a|!%m
zQ$fq059Sig26)1!gEv285J?M&cpuEhmM=3G4mMPT%*Ie`@6RX8ZH$+5L^XwbK2H==
z?@kFjS}o$QKbT{j1n*Q2YDA=E^4D!OgDce-(YwuSP=-*r@NwYAB{(N0<<ofF#f`}E
zl9bI^a0dDQ-<@|VMOp}ir4JV#WpCfn6eVK|m~a!9-pL-?`M8v2OE?sxQQ|3K*O3wu
z#Cm=-+Mt|7OI7x+dWjlu|L5p?JGE%nTAY$R-vPA(0xdL+9rGIBP9y7FxsxqL$*gX^
zS|0M?g;bS+Vj<^Gn)gle!;YM@Z5-6D?(2;fk}xCcbbtDM=#IAik}c6xj?25w%-<>u
zp{@J-S%S8*U{QV}#>-!&Hq9Ke?dPJYNv6ZkWu_O`k5$MTACzu2<A;L-5})*J!3Kv7
zx~*5uCsEGU`k$Up9R6-+$S&fqSz&&oExB4KQnlU1Z5!CQfQYe5mf0H%JN`Au=a3(|
zcetF-fJ=@kJI?XMwXAyisniYCC)d(|nH|4*`;NI0&23H~&Kw;dr-glg(x=Zp(3ejA
z)3Jvq%!*)8L%4UfC_k*7E<hT5_or&iCLt#8h9q;vlM*tvi)e<#gcul!+eESPVuKFa
zl*Ga--0$m1@Xx{7YeYQKmX)X;BP#vXUTh_i6*zZZt+Ealz|+>w10gD|ntM<n{|CXz
zA?P|*%BYeZxc0hep!hnAa^cSDu#sin(la)lR9qO@ecuXN9jm87OH?xVd`*_pZu8;F
z?5z!Rr&soQm+pOc8T`(iz*7eNIo4sYVa)ui^p3{ktO}MvlvlhV&U(dF^4OL2J?)lg
z-<s^B$sXap9?8&pazU)-8-XLnj$bB`w2afh-=7PxXB8->hwhDY;DsNpa!lf<*rEmg
z`Z)Sbnmw{o{?!=aBAO=KbI<6EQB#l4{p=>U*At-Sfd?;|w^xVuZ9a`VO!!Z@{r-%A
z|0MuBJDBelw|RG!H;{DDvwevv>xL&D^;8qEA9Vg>)>+uwr$S9T<B7Djp|bm9w#mKt
z<Es=SXCS8h$~vAtKg%3O=gYnml|fvtU^8kSpy@cR;hvDGbf0gPqiWK-JEmsUDg~(#
zZJEFu3pvm#8&b%@O!fIH5sTEww79q0U!eHS_O6VzChH9;VCM?0ZH8hlf+$yU!SQ|N
zO{{pfyE<63Fq0)8yuYD20$lGaMYsu;qGTA>Yt};Jcq=Tn#CeOJ#3ktCC{7L1A2ogx
zzJr=~)Se7fQ9tHM4u_`{Mcp(mt0V5l_J@Sf{|@P?;<r+<K8>LxuRt)gF~!pA)!+2F
zG!!_*jcx0kKE$nLJZamt5C5Y;a;EPt2Sro}XVeqxQs(IH8D7c(-m}p8L~(`eG>0hi
zaue4E%_!#&A44J}mUBs^<SppgiW|(lzK~T}mYy=<X`(Xg8wI3ddKB34O@bp}Hf~;j
zT$NS3p_X6t_1JrtT2i_X)q2UetKWSyn|)ZP8Pz#d(#1xj^m{7s&o>p_0@?GEoiuC8
zHv_4Om*X`;Rl#r=C&U<YpZC7ww}b^vmJpEZm!&#QIeOT2BJhTodvrT%@@`f2E8rK<
z3Ye@5(s_pYIyTM0t{lP}LidKYRB&^|)Wg1iUDeF936q;4)Q6X+v=NdUpuv|EvHD%f
zM0S2K#bH<`QIXjld%aR)pWRNGEd9w+MZQK5KkB?nSo;eI<2kXd?=tClbuy8hAXHUO
zaI($H;4aae*ac&|qn4Is(r<U6I$YTX&{<%3v9r)v+_v$_z(@@2U{OIP5hLa5f6Px(
zUxJYU?Ge>;5UWX_&~Vp6$4lX@Ejne@Hj9g`daPND_OKze%(D>dImTOk?IG?z*sR5&
zEDftKpQsZq)%<*ipMjBNuZ|&^&Qq-OPoe~S^_2X!QM>u|VC*l|en_*BU0augGL%B0
zgS>U=ld#>J$8~nSHp_Vd><KenuEsr-Ot6}G1RfhVGl4qpGEBq>S>nC1^1k65=Ke->
zSTRsZoseD{CTZ06AX=u;nZ=9*`)1dF-gGD-rQpQA1WW2l2P=#>-byL=9%4t1=EzHh
zG4Q%Q4h|(pga7n`t_t0kc=ar7_u5}#3Zfomo$PN*K2e*}N_fQEUWd`(3S(iDFHE#Q
z)fSW{$;ukP-WVXEd`9^~(6as(t_!E&_Pp4H>mapCx8JCF<`x$1Ae@sGBEfKPr6BZp
zFVCa?aO&RAlG@GsC#J_BudQUxGd-jJHYdP?!go$=5lM!Gakt~ft$vnNI&|hbKe2Q`
zYP5_EjIZRUR_>-rWVNK=ylRZ^oXz~y__`P15Dy)~W{6Ucy^XTVY+4I`WJ-A<w=SVg
zx0^ZvgDmYn{v=n!(Hb{IeY;z~9qZB4-IUH<J=7(*L+EOXiIqE-VS{3`=R(&Xj7_Rq
zS=E>Cs0e-9+#4%uv`z;3MubBKD>JMa!%D$w#L=FqO}B%S`68vrW7D;c!9>bJPYqv?
zkK7=9T?4X+uon{*89gqi!Q_|6ceISt>s^(uCQBh%NwYlAE_camlm6-`!_Yp~lbviz
z5Z1r(+w)1_)(5NW=2T~A`wc`?LeW_9F^GZwjSMLyvvpE`_FWseF4_(Gmr}?(wU_WY
z0yhJmmmWp!)!vtn@xUq9;Ng-2CQtQ38c0k$<d`p~sIRp+pnM@O6>b|AWy8_9_w`R-
zTSuC9%ns8xf{wM(_m-UyJX9xAo`^Pm!kU21uMSZ9wO6{&$H?U?n*N}dbJ&2}S7?6s
zC9NG^6r+g(Q|uy;#OmV4J0WL>%j0QqRkQ)&uP_!V!m)-%!CMCkNmRn@fhM#;3#sk2
zK0VQU9iJ^xu`>vA?Jce61OPcl05^?Y%L=Grr6rv@b0Sz={TuXUq2Y>@c@xuf%s#{s
z)5w+A61J&?wDWDvQ!4JGYa!m(R`Oh;!xE<cgD!a&&b!K22h13uiWH<Hng5lIq8O^k
z5~x1mclDlaR64`r0GwlnXlw$78Y|KP&oGTQwm_M~4CqdMf*u@<>i>zv3A#7~K&R<S
z<@6T{OWMtbLV=|vQh+HGouWgN4=07vKKbjvd>!d{J%L&YU_ZU4_SLOR=7dy$m~$6f
zMz38Y$FUE#RGp6bq2%p%^eBCH;`tj+ghUjW^G%fp%v6%)&JI98(*EKGl3?mB@|%MS
zj-AYg&^WxXNPEYK)yv>`BOA`biT!2axRq4<<d;Xdt?O;xkNUP3kJevZO}GYD&vom;
z3z^)hiY~6V`rk8lQMZf9i1gN3Rm=k3M<}&ks&4w>Y!fO5HbFRacG`9B4}kz6e-W)D
zo8WZMpsf9WK@w+D#@3#EwBleyI`rex>kTp;b>wVssY4Rs$om;l0PEVE9Q=k=dVKgo
zevx$l1MMa^{5Q0#M=RmYT8ou(vZ@_E7g#95T?l}@`*)_(%!ZQ}M(-^SA;W3XM=Q^L
zqo*5%`=o`JbA8^GHsf)>tC~}$aeFNHZWWxpGyupe0fU<2=58N{+Y)KSUB?9YUQ=TE
z0OTusFcVeiYzHir`ML+G(x8^quRGc@eiLq$iS#l@05zGqe)!^Kx6><b2~vup-$KAt
z5JxbxMbILdGNdMS|3|>@FBC-8*$v<(=E;oAyZ~$wBKTUg@%<zBS=0qkO-5i8b3BE;
z#XmdQ=@Xv{oCowkeSY)v{uD^+;y;Vay^CspeVPDY<~!H*uq|77j{>MKW)Co=zw+3N
z+nGX(rafH@g~s*jD_>0_^x6kG&wYCUsg`6wKa(l>=JDqBO9=KL!P$vR$wx!_Uf?zn
zH7R>x18N*GHUX1fDqP8{^GQ;CH>}DYHaiX2g>msetUgql#0vU$i$btL3t-qo^ymGa
zZMHAnU+x3)2D21Y^&hv-)DfRzFTZ&z6yx{?K`0v6-1LO$QMcS_QLeYT1EtwBAAygK
zc<tT4DyPo?R#kT?fb#%;@ByBK#2--~XfD)s=ZyN4n)=-O<acs7!LaVl?Kq*uZp-`r
zU74NBCfxRVA=Il*7f}I|H6<^jS&rM_<QQOfB1!>7HLU>}H>CU<SWH~dOcasq%%DF2
z!y$)@nLFjSWZ3;9FZlouoA7AJ!%gP5lAQyI&O(e$cDsYcGIW=$4Y8PM`SjX@jwdeu
z+npRl)032{Rd*f;T4T#NXBazl4oRN=8PtZ&M!Wy0`Nrred$hvRLq^Mc3OtF3UYgQ<
z!GCk=v&VG3=P|exa8VWnWOd5ebiVz7MpOd?_ysFJ{OnLuY&=>>_2tuhf0?wPA;b`6
zkyv}({0(N_oouc-u#>G8MY8*&McNDmSY}^=Y5F#uGPml;lhwe96wBJ_gIoF3+0j1k
ze3il;{%iF`X8rDn=}x6_*|RHF#mN`({U(pKGQrDrnf=Kfkn|ElucshhbHitE%XH0i
z_|frVKYXycM5-+<nDhYpXai(M3ap3L`BZ{S4m{YpVSD3_vD-$Lp?){+E7fV@m4?8#
z5O>igWY<(*by&)d3=(-|IQOF|Y<yE;pmRxuKf>7trq653)JRNHc5i=!?ybNZ#!1)m
zUMI0GH8Y=H(MOtMRMk!rucbXCkaG-YTe1CwZ3TcIGV@N?`#P1B_?4uSaLXq8R=i|m
zjDV1I@F~QG8@@Tn^e{B+k0oOG%Hijl@$tg7M9sF%=@&88wmyf=L}0&+Y7$0!&&_TW
z4ag;ABu1q2hqE;02JpL~T8OaX$=!B`t^D&7j$yK_@?`1lj(+n$&NFEg_&aF0pYCh`
zA9|!Aq$bJAOoK&)wa%cMbe*{{;Jb4X60enK2$P;FbBrvKM!rX#bS#>a>M_9yZz&bv
zRwB;mS1hGp4R^>)0&%-D#;0x}aza0QwTMr{G1sdNU{LO9Q5OakIzRoC^Ld8z4hpAO
zJK*f?zWJ}KT1jq0aUIU<2HEHnnDz&?<r2x{4i`vY`b<id_A6QE;26?QDtsG`l$JCw
z3tCKHA$PVOpw#lbH<KCANWk`+B|exi;D_Jn+jliw24_MgK{qdRQeFbQH<a*TiEnkc
z&8+$4PYaP;;Ulx_W3yGGlhUz^i1#76tOE2_HSZ@4o)vCF9MT&L?96VpwZ+O#>KSs1
zfXJuC74OV*?)Wr_eS+^9E9+Pt!idHB6BjW>b{>nWry)+Y&*@&betaemL+QuU2yA+I
z82}xzz0`uE(U*AD*B>HkMEJde<sFVleBfON%7tgNrM4-!>02uH>Yv!Twc6P5wSmaO
zw%9bMCxp)=EH1@i?Az#<`Z@JvYp;W%4a2J=g(;A3PA-C!S<Y8f8qdhwZy10%3Q<l<
znr*UgVu1DKRZ&LoGh^KcYKWAjUi{U>dJs`dL#GYyy3)EhHXR4o5Z%~GV7wnSJ|y+o
zI3r{jU5Y<onO-~I{TZ;;gnbc-m0aM$R?7-%GWVY|jGa~;t-Zx&z!ahHn(wRn-A2<V
zz0-5NUQTq(zAfGy3CTe>d_-Ix5r%z@x^?eCh#K3KGIM$RulO=g)&l(|iawUolvX>;
zHzhC19X)w#A$5q3P1~r9Wup-h$D0&>`qQ#c@+VLI>bTHRzar6>6!mO$D~`rTsY2_w
zTJ0lZ9$IC<UobjB-(VSMbD7QE>}(o=-PS{NgMiO6-;n+O*%g$NydIIm&#j;N1$eaS
z53HS%4;CpD%4)v(JVV~NOxL3@&`oJP1vKUvT2nLCdIVYs0!_A__W;&=>ksChb(wUN
zJEm{a1y}bo4rL^S?qNj-(s(Yh=0G3d$Wt1-tKQDZxX?%=wA7JzGoq@Bl47>f;FkAR
zU!-kw;o*MhH*?9lMjyN`=sjL84rB&VqvH8RuO8QCp4U`LQ7W`F6VzT5l)?@gkE2_G
zDA8+(8EJ?~3W?W1*I<_yh9?C(z%nFx_=q{U?u^~O#APcFp$}fr65kSpx9a5;sg)wD
zq~!8Vd2ze9cs^&kCf*?#o75(+KVrilx=kt0(&{Wlhr5(<m$55;u}3*-i5B+*b~HSe
z>78zIx^`(VQ)_wiTx2$rB^+fJl8`TYgmLpUICB~l0Snk_ahyF^&4h3bGvugcxXj@7
zvlA$rQrxkHsvVvk-*d7;jNDzD);W&+5FufL`lZSV_u8u6)X!K2aM&+MW()Bl><)f{
z!j`hq1ZRT;gnBtYJgmx?oaNaymO*6Zl}*Is+_){lC`1xV6G4<9zTp?rw6oat8ePL`
zy{z`<yIKWz&BL<Kk(!9W1JVV`7=!>p7WaKZx~?fXEX7d_Y(k!fAA$t@8!lG0jG7;~
z^2NUy{Zv-4vNdU;Z=}Ktygp}$t5}Ck$#tr%8H0GJr15Ow(}X+CoQS6oM4rsc<m6yJ
zc|0CovE<VyBW`j77FYH~o+LBgJhgz*B9V`;Lbc}-Z9dRK49`)YHY*3sorsKvWqpXl
z$Rf9rvtU2+VC`5s4%9#+wJ0{&_KiwwzRQ*$$IuTB#GASgwntCWDz?NeT{2~+;zR~J
zdNI6HB$Oeo5UD0JWYeZ8^}w0%uV-i?&!7fKyTY3fe*XyXV2UMP<xGn1xL;tC(3AiR
zlJJ*j4*#Uk6NCHRbS09M`gt8)e7~wGw+T0!;lKG@ho>9;{MGN*U14>;Z*1?`CeU%<
zOarD|)fF3l@Yg<twxaCw!z?Xv?q}^YE3uHK%0TY+{^u%0S^^5Y1x=nu`oG#VDAkk1
z#;jxM29SvlNk`Ea+8M&79kN#XuKV#f)0dB?l0Fe?!GN_aIm{nY@FwNf-!;SeO!_wU
z7Xh!~O-qDe#VUn~%N;U2Q?M}H#BJ?CGoYFcUSEl9*6WeCJ#DK?PNWVOhaCH!Fyi7x
zGcZhr@DP{gUgzDRlS>=fkE@PEOQQWXR_>^+m8{YJkO<p^f||%tyigLF2(-RD&&evU
zI)dDj=w)pA6q9LZzuRprG;bz~ONNuslGod4Jc`YaKLM)YD;+V%g-zGjOpk)oBnbFy
z23ZK#6-z+J9m)hWBJ^;g79|*X0w9jJuS%POYS)AITKDu6>h%~<K+&mEnM0r|EW2!=
zZF#oGcbP#*IBC(-qnc$%uvxF#ulPaxXuC21MWJRvQRT))DvS};4#j;%JCxb*8AZ+T
zE_80s<W_W)j=FXmwChiPnO|`n9T=$B#00Yvs-oJKpuM~3v5BHQ&DV7#I44=W;HdTy
ziH^i8{_x<;tE$b7B(T+4`gn}AtM>nH{r}fnN4kX80-LeJajlxYFFqaOqrCFs(jF_G
z!q3l6J!)28j%4;_u5W{i!_o_NRvmr_kmp|;jqY94UXa!}`mkyj38yUaf&%x#=FuAK
zyHEo)>e&c&me3C`m=YhubKZ4vWXNLQ)I*8I=(rj~)#*_eM6e)++{F1sdPv&e9-^NB
zU$!s|1O%)-iB1XtcO(4RUw>Ek`m1@dW#z7{83BEx1?+WA)$MPctQwfN@5b`-3d(y7
z0EgG{D-`3r@{W~B=wEXwFz<y98uu!60YCx2gYuP0TL1H8*?V8ej8Z-VB=Np_Dd$5E
z;Ms8}-(B9*fijNNX5HXN7o(GFndbiEyr0Hw0B^_-oXhx_jbgSk!F!Xo$uc(O&T{}M
zoB}0(3GfBE^TWQ^(uJ#;Qw+zyRHOi;9>t<@6>#cDW;^8Ev^)UGs2<G6|M|~=5nz`9
z+JaKrszaXN$OWsc{NexnG$iN>xPbHyM;ve&oXY?eb@nTTv0dsC^)!f;3fap~gZVP#
z-&%RQC4JwU@aQX~$A5I!I);q%;a&<}QO;jn(JO4P1vItA#_evb3>Il~5v&5v5&|G|
zFOk)KDu+P8mxLE}S;#E{XG5%6TamE!(iWk`&#u=Z8p}o5IHH9HRP)P33R6i`2kU7J
zigZc4o1P``byXy~S@mQI0u5;$RN1aSH9I*5_!T+`VUPGiZpzc4dlSs~2o#TUyI&VQ
z%~`<=iF|%d8jz%YP<R271|UtDNH^NBgV_&xYwUaAA)pHBVntatcn_(7TrmW+v_F6e
zdQ)5!1SNkK`vA(v2g-s4gWzqFT}Tvq=1z~W_H=zH&`SKHpQ|K0G)fl%-b<+FGm@~D
zG&zC_nZv9pZ)glOK*ea)`Z49<PH!6qQ`SS<4Nyi@NWbDhIQYa1dmBHbGB|W^bvFV2
zMS4%=rgc~HXXIl~epGQK>r8P3$li2a9zc01mTlc;1+}Xjt6VF4rt~N7*COB_|1YQY
zZJQ(vF*VXK;aaYl#rVE%yDQ`eU;%hcXHXzGn0G9;2@s`N*ryxpfx4y|dp2vyE!I{{
z3NzwPmSpL)uD9vR6yq-MKRdaBN~^M0Hqc~jx_OAVc9W^laQ8072D=}iTF3MO>?20%
zTIR_u^D!VHsUhv^PWL8s-4#$`)lHlsb%`04U{L9UG~EU!2k3ASPgeZ<+6_ro7^M(Z
zEYFfTyZ*_+9sq$Ao*LNNkwB8j&_3iWHjEUG^a@3a(qd8u)sKUcO%YIGIxg-#MQXM1
z(S7K5q8!baUGW@G1NS`|ZWiSVJzBg99;(P0=wB~@f(Kv{fj9#NW;5HPr3>XC$76$W
zcJra|*TAnH4NkY?j46KsUkR55g1N5VZY$u<q~BW;k#_UzX;!zQiUANf0px)t!hux1
z%)mNPbi^us_U-$6Jd(4r_K24UyDQ&V{;5((Vn*b^Sno@1j=pD@INw@Tlz{Dx*|H>+
zn#lEv3@UL%Ws#9shWy5KM`(;)uvvybCqT2S)yQ76s0UTR0{LQ!UONB7amV}^A=lE@
z2MFxB6eh*H?pER~T5%_(Y&^hS!40kfp+15Ax(3}aT~P@tdrC&a*S_fNii{k15{2x?
zwi(~SLC1)rO<7IajRb)+Y<JkG4_mlVP2dP!fU5n$u(zt+pwds{d4cQL2(3nhFg=OL
z@aNzFX^yxM;@deu+}>g|F|L|TP-6*g9Y|4DXVg4)*UwYF#)~b@Ye+tJ4-T1^AA_p0
zVz}a}4v5BVnGwNe5O=gBVnn7lKy{v$VH(E72dW|KeyySf;pb=X>yQ58G@i9&bTEl~
zM^we;|8s=*(UWSsPDb@o(ixTbme!KWsRs$nHINa)@9jgNS}0&frJP7D+Ranmy3LMq
zr#uAwjr$NYT_mklEK%Zxe8L&c->Q}3-@ey?<^oF3Y%uE_vH>2wXb71JVdP&5cVN&~
z3lN$b`e7T7bjRwUMXC&eGZBJmBp;rBAN>k7<(7`a-P;ou=S#d1_aA{qF}8K9BGQyu
zE2L_>fS5R&QsRz-gY$Wk)CSHX&~l(XSWJ0miMP*Qm)c3X@+ie}W=r|6kJaUlsmpf+
zgv*Mnj%5Z;d&tqUrk1`=vr=hQ>3W<Wyiq;=2HxZu#VUkZ_ZS6Ja^bz{7&i2=Y(a+P
z^no+z8M91fAdWaxwf5rh%sQ4;g#aDEHX_7bSqMnfj6tAsCbaDt1gP!eBB(o9=-kX2
z-KmyWD8@9DqzvM$1{z>7@O0p}{+-)OMg(thVMfpT(|Mkw(v*(I1?z@!qqm2-KMh1<
z+Pgj#!}Ls7ot3Z0Ct9A1qnBKMPn^U{qN61?_@%}@H33^b5FSG_PVPuJLueBz1({hE
z((IFd1We~awJ=<ZJG7T6K=cOj^Hzjg<qxqe=!l66FW>4SsF|>*9$M=%;xpA~wJE3F
zT>aZIJ)r|J52#;A^m0y$$)2#rDd%oFt@IZZ5Y$I#RQ41Xvq=BxS8RzT$_FxY#{*rQ
z?3oL-vtOa}2cA4;0iGaMDQO+huNXl1HV^m_Q_i#_f=&k8MobPZBStt|Ho63`DyZ6e
zAW*+PnSC?DXx1zi<A>2L849C;hFDDg*G(KY*`^3A!%sym9<{o}wX5WT-!B|KG0?jY
z4}{v|Y3LXas4`UkYYc+A-)#~HE=u-}RI#xt0tnd0q90yUdfB9eL=p?cHrHav4{B%c
z)7-CNOUIC=5jbdXc7|NT@PUYhM^h~#uq0R>usk2wcFw87^ip(KIj!6IdK-UX@S-4e
zFZrdZBBPb7W+!gJLy>_Zo@>s9(1D6g<zeTc5aS5Pwxvek7LwA^{;U}fwI2q{)uMap
z6Xeeq2;@LXCAL8=_bCQVlrXySVHKnx<RBpkVID*KL3Nk2oeQp^_=&E~oJf$Ho!m0S
zOH7?OA3;Z|s~Y*+rw&s8;J0d9$_2$F`IV}pmEA}{fYys~tOA=CbnB`>v4I0<4buI_
zUyqdx!+;d}+q9Jkv3XFH)5s`@ODu*M%pIrCUL)>J%tzd&JakT^gKC^)->j=Pkxyb_
zQc;-MfN!z+#%wnc2?Uo$2^Pajk%&P>f?S<?3ZETlWP<WHi|!MclrnST<MMxxS{Js+
z_*rp~lo9=$Lz|)I?=J>aaSK=9)X~s=C{6*2(bf8ZlzV}&wy+nj>KJecohCqJ&*LZZ
zQ5(3ZlHi>oXrH3m&o2;~h}SWysB$A0P=qFilonLmR|N2vBzE_36r+eF75ECi?}v)l
zl_~~A*1CDz1sM%)fU8fbuGKAXkbghXKm&m`K~`yf`C~D5CrB<_fA-Wx;uSG+7=;28
zmEqm;&t-HsRZujB2~2{AZh~St{Z3$w)W72I$)P6218daB>0d)Nl>r6ochVj@PQ8j3
z`l?`y3J60K%MqalqbUbJn@UD)Q2~%7GDt-Ys`>5WcWvKEcf(Nelk9>d{g=fpBELO-
zwa<cLaNt*Q@-b*7D1zrC0!B;RQ~Jz+VsJ5Fv0P9mKUDKMpkPiF9BZd2fXu}T?t=;b
m7De!=6TxWzgIkX{$KX1Ei4}Yz&%5}=sM;+prFRNek^cvN5e8%c

literal 0
HcmV?d00001

diff --git a/format/diagrams/layout-sparse-union.png b/format/diagrams/layout-sparse-union.png
new file mode 100644
index 0000000000000000000000000000000000000000..450ea29fd84f83578567160de5878b8101ba9785
GIT binary patch
literal 43020
zcmeFZbySt#+BQme=c2oF(cLK>(n@!CNJ!VBkq{&mP(n&V5F{4e-JKFr((yg~_I};_
zoH5RS-+$+I42LkC&+NG8bzS!ztEsMtiAIVB0|SGptR$xm0|Rdc0|R%A0tcihif{Y|
z{(<$>R+NFM8X?~YenE9pGW3LjA!2y`3#+V6e*$Efw%0ZAGEh?$v2=CjG`DiKu;%o0
zb^~(5z=-*Y06#igdzn-FIXk&{iuj4s{*yxl`1v`Ri<bJIEMAV{v<7OL)UvJ~*3|r*
ze4O005@^)a)M6f1HX_<`3jh8b_)VPF&dbY9gp149*O$|mm($h5mJ2K_EX>8t!^OkH
z0p#HD^mp+x_v3K!r2AJP|5J{fwWp<ry_=W4s|)pWx#kwG-d^Igw9gg&{qwJOdfD6j
zrzRKAe|HP$AlLK%aDh3wx&D?7d@A;wDkAIZ?B-$Z=?RoC!7uht&i_Z+f9m;HeoZ@9
zFIQj`JnSu%UA(M4fRDY*pIaxv^Y5?!zjOS*%hK?$w+4Fs&)498fBoNS|IRPQ_1y9Q
z8i;?5@;|A-FiW6`as8b#3AE}>o@E#qDHvrrX<a|q!yn#j9P&5UPe0uDkB^Uc;KJ&=
z84z$;5Mh;JSr9+Uk|HXn=%wS>?`IJ0kEr<iF0G>P>;_&Q=XoCc=jV#=dO!NL+cM?*
z4<8=a@s18*kV*tzKb<qdqAE$j;U&LCpr;Of!Ub)IU_Gh)PXh3NYsPS_F_-ZFl?fN6
zj5_gc7y5saU<u{!p1=QhQ9n@-ba51au1EeS=`C7O(0>(bh5!^ww{{})?~0#4L>!m+
zp8|lgxZz+4yK&w8X#Uf@f-pO(f13HPBv9yEv<(!Y*8fV<l&;7A+Zk9?h?EqfI|BLZ
z|4J&OUO@ia&bJ5()CFNea9CXbm86Sv1plwm_@^2=98GC**l6PaN&<%CKaHXe{r~%L
zguelG6emdpY{W8+2=C?^yZ-oNSDY}@#sAF#FIkET&+l^HcGYh)!2(+PaIAkZTPlap
zcC(Ws`LM5nEpbi#`zs=mL4jOqqw>_(w%cDTZ+<hJ*Yr}pdA!+m)nT#8Vc@pAK3ShX
z&}-gbYFzSA>2r={$N+f`f!TKM`{<HdfuayN>c?d8Rk@|1LYQwx1XlyGCGWZWM};;a
z!vc#H>wy7WG?kw7)|nrIF6U1|j=s$@=J{<H+J5j3{~f&aCU)wovT+4%SWw&F=Kff4
z!p=DXW}@Zz`#~IA)!J_rt{(`~s7$$TxBI2}F5ma}78hMwoSmzG|F~NDl=(Vf*?Sg~
zOyF=k)3oHc9tB(a_9Q9fe$&1ryQH$ym*M>GxaD^^wnTy57iH|jI1EDcVeeKg1rqCA
z_Z|}8CHEWA?Q|V1%G@!<KU~45@sdwBqfnzmplD<YFU|4xLka6<wY-2|EXNjhp1UIg
zi%tpV?I)e+%~#5z=R>@bH(P1Ni6Rb`Hy`uk6gxhb<_)|ZnbeScIPIlEwkxlw9T+Q=
z^^Z`vSq;8BQfB&)KIp&u;i@p2GVW1iK?w%2hN68`;*K5d;r@J7pUvXOpBeOVmzJXj
z2Y1r*T`jzC6Q+-6nF-btX`f&FZ@=5lFy?GfpXsPOY=3%irTx}+y%AqmJMH$n{pqpt
z9f|Yr$=|{GA&>qaguj0O^@X24H{jRjQIUg+IKx8X)P^1u!2r}3QqX#aKM}t!7hS2`
z9}Y~PUVNeD7ol{nxj!2;1*(WeX$5LgfEXg9S<vD&ynuKtGq$?Rc=Db9nzu|6mthP{
zJL#BbO`Sfw*@)-rl>L41wY`d{bo+77pRKZ9yWwDGX&&#!1HF(r#xP|0h%R|-)~(8x
z&%fVq3+R{6USYI47)l^|++>;7{73cdYBhxFDEareX0bexq1E;~{ouozUYFKWg(%PK
zlZVHv_K>S(UtX?N-j^-~v_nhaw~C#>l;6%6yH+wqpUCsw?iF)2%vsb8@t{*ZPz7C#
z^Nxyz^Mp!7Xt(W`<fu#BY{`qCT}*1U|N2tS99AFC)pot)Ihr7F)6bO8o$b+uDY!Pu
z^heLwSw3%i>NcQTg=xfdRQxI}|9&g)&9SoL$6JO%jwYQwJf;s`+WgxUk4x{9ns@Ww
z*g%`MQdRsm`175^gfAAIW$>LEmwiHj0VRoDGLNS4i~rW5C9oQJ-hRK$mEI}V1FYn^
z<Bh(2;ht}^uX75?mmrfmdtvCDF#<!5d!fjbYp%#v)iDbQ0eaFO7gTEx>nGLm)%diq
zu}5ZImMxff*ri3Sw3D<Jn|I!`b(>!V5A)BqZYGLK+@ExL&%VmUZmE*BEZcd!`#q`$
z>z(n{I4~;^70m?2dV`CslFel&Y!=vT34$wrsmhG~Xvx#jewn<{c*qTwd|nsM4f8hN
zruE1iD{6|Eux!e+tqScy<6^tHo&#0X797#HVGfZ!sz-Zfh5U6d)dv!`Wk70jriy+N
zWSTnMZ+;TRHNRS!U*cIN%=(QJtQ94q#$$2sA$+?f5StL@jh5!R^q8h5V0dvz(yJ!8
z+&$~zM{r`+yWp`NNjx%>$W;TZDuILI7IcIKKB4(PEIR_z<x}%*z!nQE8}CQLB-PDj
z+|(hgYaT<)9#gI^8NZJwHaCXKCZLmpq@CKR3D6agWuwa0nlb!gw*qT3iA~(3m1170
z&$N5F^vG~;qI;W5P^g`jN<P>(S;6ps{?*N5(=#3)2HwH#Rw(6)@|k5?pu1cl@N9NT
z%TNXTHgP%E_A?e*@`lYF7mcGA+cTa+f4AMah%<;lO|U=B^xWvZI+<lc@C)H4lDBT0
zvq0}D_xNV>f`BZ2t^dk|b80XOb%<|L)qVJti)y-<<JVCGpLuK9=Ca?dbYvNY2dLRP
zCE+wk#fT7yh3WJknqNIxhjpV8G&Hi*3EO60kWeoAohM+jc^HL%l@)hl@JzB~ng;um
zXE9s83NW@I>|SwMeJZDz^e_G3W7O1xa`*(-=6O0FT4)S9=?(XWXNL|$PixKLv`8%8
zb-m?@jS|{S5Pa=&79e%wCC(!|%dXO`gtho0!`t^&UKC<X117K|EV{C@s=pxw><HL%
zWbk?3oegp&32nqslQC-W4spxA_h2PN(5{zmeY`y|REc+Uto$|uG&vD|d*^q^lW3hT
zd#PE2%eXwTHSCL1{_s<waD1o_l*j4I=MvUi{R?w**qnFOCzK@B;zb0(bECCy3qbia
zjri(9upt?%y&t@|MMZbAERS2y)fIyF9u6tPkKgffvDv(A$7T+@j8Zd(x^iXYM9@fO
zh&t7eE70sZe*Ka039rep4Zru3MIi>~O5pjh3DO|<Tg!_m)IdnGr<tiL1<9lZ&!;k%
z9a`V(+V#gtMq&FzA-BHk`L{0$*`lsNbXd&hqO^iDr(yI%EWQ4hHdY4#y<O^beWM;l
z4us~pdHxPaYW`8x{>a#{c{EuDHTJT(wqG-Rq(<`{vOn&xsy=;wRebB3R&|w6O1S>{
z?r!y|4HU`}L-nwyDE|~Eu{V%)bC&7bUH-Lg0EdDq160P|yp_r>b-!}erVoC}Bu)}w
zTS?DwHh!J6dSB9Cw7HmpYFANzgZ8kF9b!;_K9T`v8^uvm*Lxj3e+1kmRBtR+);b>J
zwod7|KU$24i~Y335xzgqf0Co=0PTP9T`@;oC-)rcAP+i)zUOF6Kb{#e=v{hWBMm_!
zuaKvfP48^Vprxg>gn(VzA1)0{+`5BcE$Y{#^ar68KK(Oq348taMA{#YT*2z23`2(U
zCl+(Qt3kz|O$gvy7{Wa4586Na*209<%HUya%8Xo$#ATZ;kkeMfTSIWD#DOay_yPlq
zptr!TQ`o5J{?Jx(P(D6$h|or`(u!>sG^MAAV7!P_v;K%5=~{_t!Pj6Z8*Lybj+KEQ
zxr|>!BEMM1q-?y&H*@+WmO!%XSEKZyYHzu7xZhbnQ*=)J4-PiHqd&8zw!-o;X?KIh
z5!=~>I*6m85+NRj3@+hw3y=5*ub-SJvLD0EPH`jX286X0FF$1;3A68j?2Tx-d3yCw
zr3d>NlU!cE=Rb?pc+6O&-dQnqk1wZ@jhad>1?lcC`Xunrz{rFBPt04u7tMVi_oU7T
zKS39S<Tm@IaB`5N-OEiD9xfNT7_Hz3fjJd<J@ea6$oxSbgW2Yn&QL|Y{GhbrNAUIa
zj;z=cjA~R7@3xJ<CiS&fOgm}giz@r(2+|CHJNZMaj%~5?p|0AHOJcLNWGAiPSi)P$
zgn9-v75!X{z0pFDPZWM@p?fJN77=989NC6lPOTPS7V=uv4Rlm1WxrNZH2c_rB<QhL
zi<`4_(*g=3b3F!Fw1|^VJyV*u&e;Otq6&cPs}aP6n}2&Z3X4n*6M7PFnoR$czl!qY
z2&uGTQ-4}=>%t_7ho(`O&R2_*87&Ss(W#As1ja#Jwo|P>_rWMf0lk)exw9~3T1#xf
zi9F#Qg~X5<{OF`Is{1wMmfvlnM<7Z>4Y5-ME_o!%xP<BZ^U^F^$!m-~f&G9d68re>
zOyb1Q#3o+S=*uq080|T(C~VN7^|&TUP=&z4hoG`ZaTldQU#A(#@5QhaN{+a6x`82)
z%PlcocBT+vO~r&pj>@4SZzzVpOR+4k5|s>zs9cL#UD)kQ#GQF1JRi7qBwLe=EoiL)
zMCl0h6*DQ!BEmrelc^%ZyzUWRo9Q4a_W?w@r!aqd9{=*#i9!*xMpFVX)w*CP>2^=i
z%`#v4e5S#J?+jHJr;7<}iS7%uSs$@c$w36i{LF}(FE|ub(}m3@!G5W*MwcYUV>Rd;
zrTtFLc#)#*yAollWmpszaVp;>4dw28FwO{3!Y`6Ivt6wFr_09nNR>R6c{@YF@0R34
zbUas50`)=126i9lV&H#C9yU<0z8*DA509IcLtFx`?2td@rr_+pbXtv@WU@Fo9U8(k
z-HR=Bq2sK+^B8d_Hid&~TT6<s?~^eh)IKGfFUh2>N^_bfHIEt5ygz1x=rB0ihIj_u
z6m%;@r2|8vsMA#_Ex^n|q;DR#c=j7{yF*tPKkQ5Ly$FoNv_k;TzBe~Ylh7GOu?WPk
zk%uv++N#}1O`6_#{ZJ@7bCc;C0A6&E;0HFvB@0I1%fZ4us)N4H-33?U6@sDr9S<`U
z>X~#`0Vl~Z9$9iPQ}zxSy9C_s%_K<)C*f`^xu@%1Ng0oe@nr5duVsA(^FC!5{d(_M
z)sL!kRJ$HA&Equjw3t%ob8l2cjr4{hkr0~2#YP|ASg1V5%5{+MIs275YBNYCk1vpu
ziW9t#V*6_B95e#KwaDBqq)Hc7z_GM*8<Uh72@Ko{V|`8J%3;%3*CNJunV0yxq2e}v
zB+egg!#o1_k~BskS5-uxfXnakG%19hrtx+}^f$R{9qFkt9-k(nyVOJUQAhJr7Z(IC
zG*WHnaaXYOsmNcp%<Ryk5ZYy`8WM*l?MxlG6IBnfn(TE@1Sao!2K+FYW2~k_6Nw4s
zRQ>#Cqo_*gIxxc3p*UvH7thBvN@dh1man-Mx%p4`yEJf<{_|r|`I7UhZdW;Bb3dIY
zLLX&iy<SUO8Jh1I!Pi>$OH9qJOK!bAF?`zNG&0!!ZrW7w@EAvCt*Q5|)DG%&N<OwI
zP`YdG4$>&Mm?|-lTXJhFYv^IwIgv3}i90-KDhlgErkQk~_k`1e|CR0KB4l2pz;UW<
zB>fQ^oY$Kpsy|(Aid2ZfuoG6BdFm`afh$*%)b*pIFHj)5LKF*z`y0iBWhQ3-4JV6(
z$`E#xW(lcfXI$b^B@{+zRe8P*q|FbEtPTD=1x|JpO0BshB~Nda@byXN&2XFtl7_^p
zf6a8v*jYg+Ym9DT&EsS>CT)3&K?)9EYz!{44ybjgR`vz|S~pgBz(AH_LKX#<L&fGx
z{8}(#%F@E}X;D~~bH|><@78nHnV4w2H)qjy&6zroQD%?5u0U0fI>jeo?7-1$du)g`
z>x&<kAEF!fTEjq0VSaXemj*5?NHuv^`X-yu51rL2Rq(TIz+h#(6PaJ)#zw!g8Ap(m
zD_4}JN7luHK<b3`b#Y`7+VH@JE>dbvLgsqa8BIEk(qgCG_?ww(c4m*YiS&V(2<Vl2
ztJUy2b;$xNbI=OzDfB$B2Pc+iEo8k{c^yP*O2+?jc;11VC;A5Ee2N@N=M8+d!GIEl
ze0)a8)7@&vMxHqC=8G+6dcs&VL+B7HrV2A_6ywHx_9H&qpgvt?j{!SqoX6jXyIGYv
zAM*lprs?!BF@;S8)M|B`O08<lJriqFK}dI1YU?XXsiQUb&SurKkdXUT>9YH|L`;tT
zE@_MaG>lRsZ+R&Q%qOA!-v6<C>9%8h$a!M<ak6B_d`v{F$}l_LA6`_r{&PXx{LIs1
zT$kTgc*$#UV}O|AZNo6uOP(Ji&Foex`|Wqjzkl_&TadS%`$H7p)ZrjIM^OB^slgJ-
zjPvE3Nj85aTsINHV@K+QpY1gwKk`~2mpeyUWl5PW2GU!t{n0`^+*^OjhPCJO>N#MQ
zV5ivcIDD?=L>r*ml$sOVDgs{*37UYyMiAv?>*FSXothWKOrrWyahNC5+%j`0Gcv;b
zGQaT9%qu1}nuzcmVrmaEwsPG870M+h{>i7s_u=0#PWxe>rDrxHk+p302hWx*PWi-~
zKGvtVLxvUu4+N<Isf~`{m{sDz|6;xc-s&MBz3%#SW4z_Wb)TV8e|%eL%qULOuXxi~
zuuWiR_cemNFGj*?aq;CjlfnUVbfa?3yK%{2;GKpXa%pE-L?SEhM9H1Yq2N+DDz$2A
zr|&M0BI}-@tUbCYz)pyH1H7hhmKvIb$#^-<><aAks0pr6JAKg0d4t`(eK>9F#VPN^
z6#PalxD@Rk94stRJ>mEOm;*5-<sIon^mt|*Zs$lBq+1VzBAE$Iha7tizum~_qSenr
zq#W0CTnV)6<|*x-wv}7oY#i^eTw$Vxd1IVgJhVOk)4H0Uo(7FG#J_K%)ZOfRPNg>r
zRFXBlN#*uH^@)UqoT$_CiX<H=1C0k)2aI|i-f>3y1L25U6s!eq%R6HHnT*jJc&^al
zkt;1N5NA<Gv(l0t|B~+qn<xTaKI(kSXJ&I?Sd7o^S%{{}B~}=xjWQHP7FT<Gk!2S^
zG3Rro2MwE3`8_M9t(copPhWpr8h!gdb@_GIIp-xuH5i?whu`Yk`Bx&+qw8|{vh}3*
zG19~iv}8=!x@u1B=9uQcupKhPK;LNPPzT<~XKxf3CAc2Pp&IY?%OuQYbKjMxcw%rQ
zQqL&81n*7hXbii&4qr#PBD0J>y{yXmLNZ!*GOkE6AaYpU)rU#q7h_eoJxt75ER&5n
zy$pgSYN)$YRvOfzg{sc&3u18c;Jx($?a;-IU+i<jrpvz;Sic#uA#v}Q)P~y11isKE
z;E(FLe&zqDo6ZrW;hMJK-2HK^+Po`K;&$&E#+>?SLWRqkpDDwr4Du~)mTF<>jxCtZ
zx|01D*Z0t@@oI@<(-|p74&^Yld!I$;-d-9GJRXeV*zvcOMlAo>93LK!v~lWuD7Dbp
z2MEHAtnKyLEj~&>&}(B=V=@*Ez?m7}$~sDKf_pv;<uFpl40#bASYk<Bn>G9yAT(-)
z$X-nI{jRu?SEVGBBYMyj(HUYs4Rp78=}%kyA%Y$pJS45ii?E>(HKf0Dw%(Dj&)0I!
zyF8ka-Jj}OvANA^e5s<E7)C)u7|RCl)@v(sWUB<sYs=aF{tGs~`H%sR9Ir@c_H<v8
zmi8L*I3`p^&J^L*Ppa14;wYB)J><i!?Hk5iRjn?nQMWoZQ(SUY;9t1<Z8Q1K@L>PG
zI=y%9*=G34SVQV#+U!+dKNxqRI@*d(jQOy{5RfBxJ%bMp9^D1G>$cynozIf!M%}BL
z4RAcaN`HXzd?gW$%HnNgLyrS%q6=aW9kZ);Ttl+MVi*)gL^eJLG5Z@nt(7pz>s(bJ
zk<8u}z@1IYSQW`vGJIqRu6G$hBt-vZo?pgii)r}2!nIywm|_~<vj=P62&1TOo@q3Z
z5#7P`tViVa-PXR|$f*>^&}cEncwlu8Z$uxMucx<;Z8V?Rzn6UYA|G#=kz&)Zm^#mF
z$f#N=u@ISkx2Rvj(YwwEcb**ScFghmfzX%O977`aa}HrK#qsrF0+MdC_E_HhFsq-9
z>HB88VUsgf^H6kO@o4vmpoVaVK0jEFmvp;P_%nTVj|Bp(G47H>4kbusJRLaE$cm1{
zR#T==IJ^=S!q#O*3!n%e&`kWxcwy?NCFh=Q{3tt&PEQ}_Z?5l~e1ux&t=Q8aE{ao$
z#oY_2dkob%v%}4P@{qv_=Ex&<nIAJ}IPA#W?ebjqkRtD5q$}`=581DX@psbdNn8or
z+850G@B{-UN*bO|3A@dyjPxmPuXSLQ+nLq3cd%Kt-88u%$s7R*t2n|~>B^u-0(6Ib
z4c>=RxklwX7=dJ7^kpHgw@j+s8x~ez()Jw>$HT(epLg-DW~yIg_CZU#-FCg!Vq)wo
z&GXt>ZkFNN^(hs$=*xC~<OM$;MtWY)`KEY2d@;8~HD(SZBkVRCPN*<%{ld%_U)h;O
zj^5s47an7Vk%fFZR}v9~9A9+yGPH;2-jK`~GWJp9RA3Q!#%to&5U_uV9fy3LTL#B<
z!hCwqZP*Wfm$3R0{AgeLfoSP)8@F`X(r%=~KuI^iL)yB13reNuCfig*Br%4@GezJ_
zJ<1Yu3OQU@P5G1FSC6RJTJO41vYx1)bn>Tn-=){`v!U1@`o;7;XY;T8BC1Z4CZ9OJ
z?-f;>k+6jcX{hB^Nrr^wvRviI)ZZ+AY<XasWf1Y~XB2XGpgq<B?&#wQyq-pn!f#y2
z7mu{%X0S={cYQ+hMlCTmGR0vAVuTrs*$D~4@7Z=t$EDb!u=+wB`q<vxzzSHR?%qv*
zB^AhUliHXjB$+N%YcF#yaNtgS&xUB(;&;e3oPn9Z`;qvypZSdB0oM5}B-d8l^H?Zd
zd?KM%xo_SslsM<rq`Ze=O(1<lt!{SgGKkbwpniQX+q3(eu!+q>nviSrqbGC9oj_SE
z_zYK(95-xk@GUrYZ<MHQoh%5zQL%76N((Kqs3bRPBp|5a@c8*nbQ<oSXC_u9tYO~h
zIZU2O+cJw@B*-oYVh&#cW2KQ-2tZ-trVE$rw!rGvsYM`;>jLlz3?@hFV#&KaB33A}
zcG??@-j=Aay{hR5`<`}Sm}|y*1O|?!dgIBg&&TuAa($9Sqb8mV58oyH6sdF5IoXFY
z2MeGGh9kSLAsC05*jbL|n=lSLrGM~vvS0tlGA@?O67q1?eX_p4ui7p9F|BX<LCv|`
zIc2y^5p=X8jITb#8i%WaxuX;PdCnptE)Qx)7Zegnf64&uWB8h9A#rVG`prb(zCj)n
zGs*O^wv{mh>Y3xqoL7c_+64~*%eX~shs&{1)~3djaHZsi6<NGkC?M@YnI`||@J@RS
z$Nixu8B<sAp(<9M;uZNHL~B}%p%ao}l1uX(!gSEed*@Udt9XCdJDqM?KLt58=76&^
z%UE&^+O{*p;gK;xF7Kkkh1wuo(fr6P$mq8>zCTa+qvE14AV}`|-0X7AfngC^;z+E1
z*o6h$2i<bqTt<wi9dD*z>eJ9u#rIP9nrOCpZq`Cz-0!94?33P|vqNb8%SJdFmugRY
z$fC!2dIX9(EVk0TPhLa5(lT<5b!|wG&fBSd&&Dj2ug%uncIzjW_d&LgIUVeAT1oh%
zWMCBy6^!GT{SruK1|wB!T~3OTaMKx!#+HLW-bkVm#=0yK98ZT72X|)~X-_f3a5B|5
zpg?gm&>h_X+LsY<NptW;>*OZE_3VMX^TgfYiP>BISiGL7*nE+8-HC1yu}CGD^t%E~
zB{6ip#m+CAI^xx;NfhG$&SS)&srv%Xlk>Q@noK}#^rP*4H>SH#c)WDLYi`KxdAB7z
ziwlF#g*7>8m#NDSZqbfm`QLG$5ELErCB=@5wziV$PZ=`V1Wg!MLX5m~yqcbVW?<pJ
ze4_Oa+ra$^9oy?lwr$|wx5-^0kqr<3Vm{smpzt0W#AMc<zt8zQ@6hQ;ZYfP6V%;&Z
z5(KkITOMYwID8b@i#FuL-*Mnidg&q&>|Hjl1hADe&&gHWy-OdAP@#k>X>UFdrg9th
zo<}e-2@=40(j&Zip(v{4A{fOjgvn6F#X_Xpj-7fQh?OC>RH*vO<Ah=-31S`^*+@_2
z1j#`XuVt{2S7EO~_=JF}g}F{eIJRIRm|N}BkPt2K*=$BVF~!NrL+)eacwOCBAH9=t
z;1RH~tIta(wMC+c<LRZ@?7<MF5-{XcH=L1>OPF`}#V<^+Yn6`>kvYJoy{DGSZM{07
zqO4=s`DZ2c6*1?&bF)}Yecy<4p}R+>0-3)y*g^XUpVtQZr93(0dN7_F>Dw4CvPZ4;
zORr*O+d=1-PpI#`*uhukZ!3M-`NRS&R0-rrE&3TUhZf3CUyY#XsN#{&e@@71eNU%9
z?4)N$FP!#vNm*yUZ@7u<*zb_%Rpo2>CkI7+ris(J3CL()9B7{q1Zyifw4pCo#&~Wx
zef$+EPg0m<!`2v!B~FIZ=9uU7aq`RJN8ShIhEv>h$CxFCot@^WzSlbvWflu@#i{PU
zvp@;zeYUo(-5sQ_SFq-3JzTsx4)qy?t={~04Y^x1xlehm_a-DV;!vV%g+fav&rFP0
zi)b^r3&A?xZpV3$N#_`qWMEsCk35fr_%pX=DhGk@ebbi^1n^-*q(jU^_F(0u^|P6>
z=(dM~Ip)U6k7x0=Z%pM7!4SF5FVL8HvZtlI3yn7QgyCT_|B<h%0e!+Fjk}~z-$`rA
z=DQQP(LrKeU~vDN=uy3nUWh#Lx;wtj8%O3Cf{tWDs`vAvk^*S(x`C1}@#${Qi7+0r
zBDQpnyeT&YclN%A#moglYhQY7;gin=6_;C=W@U?8QH!Z^?a+Gmhwq<sVSg3EUHu|R
zRm3xq&G9|7)c?cqltHg!Y}j07VsB3BZ&^l&UuexV%%G-2odp8J2xG6pdL67|+5#%_
z?S23DOm&!&1aMpFEk8p$KOtYM=WYopICOEw8E-Ug@#yZ{@z9|=p)8_|?>#6={9=)s
zH{#fKB%%OIZc#u)4;(BmLAYv4LsJ(Y7_Gn96_LZ76O!g?yu-2i9i<N@E=B&k9y4v8
zO?PHsgXR(ddO(P=bontNfj1zNXE`Twiq@fmvxg>v11+{%)YojQbqqoESTjHDhwo|(
z0@)m0PD3H5(xarLPH%Q|T-$O7=ByqoUnGtUtVy2d1zv*r$Yyb-(j>9r7#+ZmYo9w9
z)3<PJv<{S2$WwWZUpVr7HadbY7^#>@ib5KKpw{ny_d;q5V3UY6Y#GQEO42flK&BGm
zldZcSqo_+8IV$I3<;Nd3FN#ohv&ml6)#Ii3s+{s->a<DJFN*uBml$#Mdz0^}5~|n1
zjU@m|PJW~cEyeNd&h`Dd#h%2HC$lciSI`J=T4$F#_cEFsG#&4lyC@Aa$%y3JtaCll
z14&I$FF7;#{+@T;uFdY4y~JJ9;Kt52fJ4+|A#g`LTb7BSI}gG36MZr@?1-iKSsxuP
z21S+7L3+W@xO4*8I}PpOLT<c0&QJjt&&lOD!|w>U$uLyCQ|~q0zI91BJ?xY_J=y0K
z3M&=Z0#c4kRUM<0U?w+!aIYh!K%*0H`n;-a`1AtdQ1V^b9#iNeLqXVFyR+-?n<$tJ
ze^7XBTO4UkT*jwbaUFiP30Szxd@QL36x8&@w*oPr9C;8kr1D_ioJ9te&e!I@o1Nc2
zx+#0DQXuv7QAesM8iD@gX`xB0!Ha=96nIIyYBjK!ufDo8-&c*zm{0+#{4w=@Mjoz`
zb`8a%T<Lw%Uj@k?F5Ck3rIM@>=3m*#7%Lzmsid`+_^Zt*P<-ys|7QB%A^5+x3NVgS
z0{C<jLLSfa0n&3pDJ?JXtbbBn^hl|gq;M4VnU(}B>2*1LHGm!@!z6$PT`gaI1-STU
zaR|VEGJf!y_}qTSQQJq0&eeJ%Rp+2{53rsaQ~+!Z*qc8NYX=|icO?O-3&&`KnpvUr
z2;rw!7O@Ph0NtqoH~Y$$LU0FQ=!HgvHVL*v0M!X)$i4eLQ-0uCwryTBh{ip?G%p~o
zkxa$~kN^Nm1QmJvAQipt38uWDN@|I_!#aHvcc_x_EdZBexdZx|@@M(V$J1EuE)3ph
zrua`cpSmj!AeG_uS@GWm6cHgnH%XJju7`L>oxbmvJOMI@c=8iTfWwG?e04hSP+mDA
za=;$9^DKaQ(}~LDB@ekpcV?RdWH+9`<zB&zp%JU4Nb$`gIS=Rh_Ta0392S*CV7ivV
z4Ny`(JHEuUz>1$GfgJskZGI5zK)-WK52+(gJ)rvlgf5>6H|Y=hF?j&`KlprDP^r&z
z2hfV-TP6y}O<-Mc7!FD2pW*E}B7y-aOx<wQ3t}$ML_jPgdR@T6*?xc4JQkPv256Wx
zGKDU<>^x5lP}JOe4s6GY0a6X$*KhN<PQR2_ZD4j36MZrd#}p`QZv%S#^n$JO0;ndM
z9-<BOTI_jx0GiDxgpnfV1_1HzAAW}<Ap-NL{9Gx(_Ywzbd!{2gIV_1#k3M%5P%CMU
zfh-}X^S1fk1>Hz-QiD$b2M{>O(Fg%l4~wqt4^5kii&BiSOu~TPC(I`%cWJG)j<abk
ztY`?(>xhq*6@{DtaN^@RAQ#d@pKd<{1a$4r9e^$=09XPXZ;bWQ@-=<}DvEbw|D!-5
zQhVw_5xZLQgz%k|S--cD+H_FWaRJl=(xSP(EB>RxyX3l6+}GWEh^$y>L6)@N#0cNB
zwVz~5?*w`*0iEb)jiG=MowrI`21G%IBaT(gj9qbtRLk;kfGTUP?e5t4LSxt#5Wpyk
z%5iN8ux~E8i;HCwjViPNZ-ctb@r0rwuGc%{kgO%INi|^&t~XI{(JbiHgaLrt70`F_
zYX|>+vs#Yppvi?h=iIn_{aB(x?%s_H|Am+_C`s%LMR^C%xB%p>#9eF8LpasFnN;&P
zex7h~)dE|lh1YM6zBOdbsqRp~3@Y6x)5zA!gfYMQ+@b^neXy0hBMFI@R=yFt7?TaT
z?xMo62Oz><FOQNJ%sV`Wz`fo5Xp|PF!8hB03@t;~m|4{qwTxPzGym!;MUiq|>BZy)
zIjjO{*(4k~`%lo~Th`ugub3p%Ks`}%O-73NAq)auwO*=_d()>|eY6h%1S_g{A%)IL
zJQq4s)q098wDuO6(vLdmri&sW_s7GBQReF|Y{{Wg7RQn5yiQ83BkqThd2>{q?5<yK
z-#FCaIA~6`(lpxs%otlid$0IsP4Jxqe)Zo2oktzlGcb<~*zgs{I`Ax}%{gnS^D=8$
zsinYqOL7C0nHxs*%4@u)hqN4?LBK@L1RN!D*NM)>m#cA$ntf$p{qAKa>rnddy$j{b
zFA=OUKNf0d%JC7ZKchjbxziMnfW2oL^0}ia=c|uT&u7V(zLvF9xUgBiIX3U@zq2VP
zObntlD0l}@@1<!-n0sXZ8C31ALz4h4V3G1M4}XE?1ON}j?)%NEmg5&XBN<AB-`3Oh
z(;GjQ*i68{mS&N?ow-x?E6DfTh@DEqrOP`k>>QPR63YWbZOB+eS73&Y29#*Z0K4uE
z8GWC#%~XaTfab#dA}R5Cb!mj$;%7RY>sxYxe0D8Z@z32dW~pHoVL5>VtYTN`B!LIm
zCOr%5KW{PD4GHr9RP0~{sLE?jr>>3e5F8G7GX_2Zd(yrnvhA+2X(J90R{1694%0*C
zYg3T9R922-fA5Wh48Fu7dERViT81g~(xm}}K}Z}+iJ9?0wb5R*<h{AUAz>)rzS(E?
z^FK};H-PHIbCWx{D`tc~O|>(QLF9gb?duS|Spw|~ublfG!FFk*U!5yhWhVTr?__WU
zy}7XzKrS_s0M#e_W^zGB0$BA~+-JD}CW)=w7tZMvsUu40xLnnG_CkcxGUGxpfWjgI
zldl$gJz3dubhik`OSoaXGap$g0oemwDDfck7BM#H&ax|VY)I%==+A~S8j0^(;-~nr
z=YV_+xyBG`QmW*R+6hRul<;2XK1(Ioj*l~jlka7v)3g30x$q6*g!T{YWqVg2AtHeZ
z^?zegh@>O7&-bx3Y5Dtqj(!o@9+)yq2O5x$8PzlqKb6e|R6i|6pXj>nsLv`gUUs3%
zMz#K!0S=gg0$weH%5;KzGWhr{h||kOHlca~QKZ>2vn7>w1vaa+Re-_gZ<r~PllW14
zT6YX6^k!ebao!VFwK$_Mq&1(-0{4>h8h`d$LZ%9IvpYQ@R{QbZDNsE^C=d@n`j4RK
zVYTe3=%s%9Eur)@kGXMq{^LbyII=!PMMs-(W<$-cCnvr@`GzdT&3s=J;&t<w2v?@D
zOS9Bl%&r4fi+ViVYoFe@wg>(S>a%yFongS~Nr7(K2JwGg+T-^0aD{#4EFLnS!Oj$N
z3EPdHRFjsB9fvXi&WE}W+X$Gaq0zRe^19)nS!{R%D2R0(YMbMx5E`B5VSOqG7*TLB
z7OF3Es+E_<{RQ{iBE#PHe7%974e?0bE85J~1dtHqD-mNX=xU&(bJ?xaNy5C{h141l
z><!(PZn}V+%4FuUxdhNGqXWB!fD3ir^&Jsw4D-@y3tU3Jv)taVzMq}idT?N|B>HKB
zZeZL3bu|Q+y>z2HxGCR-`fQ-l(uK@(6g$?x(UZR4D^A+jjKAf%ByYh_Tzxq9C3Gc<
zA{!sIP|c$kGZU7Kr38t`YrN~p8x@A#TOsIh<1fi*F1Y1-iF084vL}l*W_2YBo<jOw
zKy9b@Rf0g*LZerJ6gr{YWk5mIR7xXpX@8`Mg*_#VRr%7#Fc2X%f%K(mg77(?iNohl
zGVV@%pod9!HxO|r)tkebRxC%A>1xU@95d>-4vC`1XlaJ-M<{wa^VCb2OhY?eRndEF
z0P&iq+)Rd~!eQ`=Z0^h#ki`OeNX79lvPaj{v7-62nglWAh-J12xWf|ZeF7d(i7q?*
zH-q3D(-b#A`({_rDd$|F4vc}vS>ZQ8uIe6uBdAZ%vKNE4L1gTNTVm(0UR!?5_O(^U
zszQ3>bs=-0lgs<T&|do~IxxR5a0}Lbpm{Gdqla@4BC-3z)->#VR5E|sTlfj~)fZ|k
z6#aeLa_Zo_0vBy*jM0F{GsJ$29|R&D%f71#B&OG=AJOq@8`eUC>F#Z}rPX(?BL_~9
zlEi}NuxvK#O{A@QI8ge#@sXtFYu$KlDH_by_;c6%oYfadGjsch$BNaWZqF=0h2XZ9
zpYM5dP)lUowNPvl=L^<soET5c`Az`?B_5Yl9puRzUkt9E!iw*Do48~4`-??Twm1fd
z_bHT^ShjfdEkU>4BoWq!_-x3OR@LHGQ9LKwo<S*<x67}rwlJZIC=}eHFp!P#o|3L`
z4v`vIy@A{F17_!s^(ZnViM<rNwB|9hQMHJTKpEhGF?r-@)|GZb?Xlw3lD@yhs??FT
z5?Tig3|^w!Zrc|zNtQ%aI~gn|Z5qR*#Uy0iC-<yp_RV22vBu2t*c)-a=mIf-mf)6<
zE`5;q{Xn4sME6V<OPGG@$Mu&Ody*~`cWM%GvgaOsu#__U{A$TEhH#INZ4nDG1zb30
z09dCUP?>W7j6DH!KlU=Up)AY5+)AzAEHU+rGbk8+*5xEipqJ4NJ!9Ntq`t=VG^Q-T
zXjLgnV+haQ1acCMw<e)koMY&WbU3?oa_G&PCTgtl521&9-Utc6m-{qqQ3yDJHpSiX
zS$0rnj2t;0XWm(o&=w*PkJ5CvQ5us3b9{zvGUy<#jWeBY&IQcO%eBcx(X?#`3QiJR
zw8UpsGa=m?i7khvm00nc0@U(%6PAw^*eFa8@MZWrgI<T|P)p``J<Ad<_7+O}{q65+
zS@xK=*d8W|N83Vlm3}O=)#ObZDPHdNS%d@gaVNJZnK{l4BvEhY3t*c?PXZ~E#T1Si
z;9lyh|IubxG$MY@A$fBgeM9UckPhAH&w`2SdcR=gh0L>ikS{%HA^nWRkJp(~v14@!
zw-4D_20_1O5$bSv;3u?#78WHS%F$>uV}0QAUc9$SS}OzVdy=Vqxz&ySJqwXERGh-f
z&6jvPSsxj&GIDv%he2dSW{@*VVDs$}frvR<4r}==zCB-J1rM6ithK-cpqFOO;m<GF
zkfD-IyPztO@k{EvwW4LPJto+2Ggn3X$=Wt=^v|H>5G5V(?yp6Xr|XW(-{k89A4@FM
zF14iNx(;8zb<9V>*oqjc$&RNppN<=puq}t~GlV#CZWh=Z_!-pr%(+;cP(sr7qE$GX
zt-4Uzc2?8!3^1Ytto18>3QE`yt;tlJbOFmkB(C_=$jxzY*bJZpj;d*F>JesBHFUvG
zyuxfbFvlHCSBW4$=(G`Rs(CSl1RfBY*42RuZF>~VCtdhuao14(f>Ohox;NcWmU%8t
z#Yu^r2UZ!>=RE*^A2rsM5EfKR7-?y%O_-?eB7cv;fxU6aTr=Gi<j8#f=O-VIu9a8c
zHZQc9-TdA~ky;D3GtZ=q6;Ma{YILE-3txS~{-RoD<mmge3YmBShYX=sm&B3~X;B9M
z{4^Ysvz%nk@crlALq^dp!;D3QBc_-*@x7$ri(&^pgNk8WRSalymE8cI?BGER<Jvs3
z{-IxB@VUMI$^whOE54?*ll7f~ikuCOf$BhNnBGoa;Oo<%9ywcibIFH`$+$=NSWdQJ
z))6j!)oG?Y2s%$n_lnb0Cio*MvX)%=Wx+8p{Te^D)rhjA26`wBDn#!V3B`@K!$V02
zC&tk=?v8s?Sht3RNe;z+?vX0)SlQ+lC$BJ8Zmfm|0ty(4$KL_H2U1XuB4hw9X{g1m
ztIKyPSstj!>0`@#i%0AykYZT*(LFV?Ce%V~=iP>OY%UTB)dwfHa0Ussj`;q&`DoNJ
z;FU!TH^i^6g98?qtD(kHs2R_#T8{{l5goaW+$Mk;g+Bns3d&=%x-n#_8k=|U5ZVL?
znQN8N9Ib_{QY+?6(kibvH@pG?lDK-*)91J;CUTkTM$c;4)TCb}_b_-IO3(18aC9l)
zpi<h8h^xm4Qoo6trnQCB9z?-4P%H~ah}sm_Oq+ouF09A@(aJ+O_*gJY{~%S3Z(NS1
z&gW|-7E`n~k-$s&Q~#A3S+#L0sukvri4;W|(MdhX<Y=C|^9ydn@)%mALW-Wcm!#?j
zH8Csaxq#n>%%|FXpPKi~;6p7ujGZFeJMzH38ePP9e8<+|iCG*~?2V}KD-g)cb)<$K
zBYCa+faBXN=i=Nu06!+%QSdt(A2de;^ZAJNK-he$p9T1tCLYR=ud;BjXn_61b694&
z`@#2hUm|i)u4|>iPrzLUTGe_#_qyF8f+#?py8{8jnnL5+5%<Nod?1d<13Jrr{UXCb
zj}1jH$Ej8s#TsnJE8~Ob^zB#u(i3s}wR~Ti<%3Z2_bkacex?ryA&DU||J*fpvEh|g
zLW-QF$My8PjJ&bG?Xhb5tRpeOaSOoHg8ZSJM5!Bv!@WQZ`r&+UXkP5@Qb4)k$;!BN
zuZNwU+*1ap+y?|saJT5)G$hXC?bXDXuTyCkojkSJ7ok51LGb%_afDsB2UFBEZYR%3
zZut{zLKr@FUIeAFkp*sJ4X9lQzHyCR)=+@L4yL1=i?SwPqLoIiMO15a?JhYE&GD3}
zrR|P|MAjm5UM^G4FwPHQ3{H>3pqQV1(wxFDo7scpzCHThoN$>H?mKU9NSX4yDV?<X
zM1nTFBxOtg&>I3?MsOl)C!Ku^vC-Sr#`NcRfbb39hS$D_;s`C7N;s$gOI|%Z+?)MR
znMhymt>N#pBq?GgAWZO*?l#qzB{S|dl5pLDtRmM276+nwV0{c|+BS>y7eLG`Ok67H
zk_PLXzP$dJ2LJkwC+}mJL}q;d%B=f=9@3c>f+sEr8>@1NF{XkwzA*O<rIYstOJr7V
z&)AyX;Z5$~M}|z7^kqCqzs-g+>*~r!Z`K&Vr-m#=*d4@=IEY6%vj=ukkD4LON0bxV
zD+qUJyaHgFbLY=bckVszN7uvAJZBf}6Kz4tb#IaSYg#&yVqlr_6r1sd$=<+#MTvUV
z=1Eg=>9C~Y@!8_pYf-k5z;nz^1}qhSijyU;JFtb8k}9qJmo2t~0FS4dPVgM#BLKW`
zm0(JIKsHuOO}WKZN}>_el*8AUa-8EykLgiG@97FFL@qNK2dug6$vK8{!}&!X0XEOo
zT(mg@J?|1y<~hH&LaW{Ya>UUx3;<S~7~kmtg#lM$zIZf+j{znLAPS_3JB%g`nb(2J
zdhor4*#ac#5;>4FdUG{4X}imcC+jl)CAs@g?(qzlKb^JP!{%-tbr4>6F|o!R{`iR_
zvGc`AZ8RKxaNN$dYXk%Ij)*r_fW`|6BYvOznWmg-Z0aTH!n(pLMs40o5YP9_uHWK8
z%U!rJ=lJ{~y9}`GDkzh~K$OSpu36s`BwDrnS|IiM&uV5cDwbYOBA@a(u1Dk!y=!b(
zOtQPCfU)#aYt%eZ7Jg$`h&*D-R;y?oK3Fbs$BTw0#1Ug_>(?Afbprpe)F03b^C6Z-
zYa^3j1fu#~2g`CyCZ`IAd0lqc9$JnD3M`4LT3n-JN@fG3J5M|$dw)URvjji)lz|>H
zR}<NVGHM&P_?9v~1w=3zez|oXrD6WY_>Y>4+EvCsY%w2xOX7NnCjyzsI1g;L)h?Yb
zO!DS6TAWxVl5XzlJeJPct_(kgWzp?h=Ku$F@EgmS&(u8Mab}}!e($dB1Wt=2q1x1?
z+Eo!x?35rbPR#`_?5HP}cwLrmFJ&G=xIYP=!*nh%mN#*Oy3$EZyR1aD5mSUJLafHX
zSG98L^oloC*Bq4Y-snyQ1;M>if3lL#fhXI~*eZ8suotD5UE5~<o8xBEPXMK;kmrpw
z?jSvKk|`_qOLfkm9_c5~b&B`_q^-Au)qn~KcEg@;>5gCSi>Y)sUqY9#;DTLoQvmxQ
zvS6_yG-b_X{8O9SFMXWP-qCj&iJnJ;4rsKmVk%1#B_FSg_>}}?^S%cnS><O`Os!iJ
zU^*2c^_Mf{RA=vc3gquBCsGTHa&MEjiRZ9g`TfdQ$xw?hv^?&=y|S}LL{gX}*Xw}e
zYdhC8nF|gRL_@Q9*sB)pevI9xZ(v-9;$IDXk&KSAv@rDsfUIH-vvUBYJU6#CnCc-N
z5u>N~WE|nHSZC=MGg2q3V)DeAvHU}IQ-EPUHk<^wxuw+`F}^}DR{NN7cm8N*5auo|
zGAe`4AxF2P+vC6>kXvmO<$cSm6<;IGeZBdTn3#S#yA^(%S~kRT(Ln*9WaW6MZGNwI
z9gLSpzm#N`4H4e%iFMR%*KkR@xWoQJnX#k_@PJV(qvu~nj8l7{Q@a2Lx!V!>nU<+G
z{lE{=haF9H_KaeBeKW>ki5Te{L~3BcvBf(6ywE$5x<QW;2`|iM29~UQ=0%>LOPE@9
zsLyK9)ydEo@rmq$tskt~%`W(N8t)&*aJ%^yKfTm$9#$LFJqooU&=W~xeczAGhrFpz
zPdyiB@cp2FjU298*ly{lx#i3t;Q6~GaXq*XOl!dT#RDLy=bLbg7G!3O+I-7w3S`OH
z@iKE=Ve|m%BBW;x|H5&L6am2)UcE7_sofx1WuJZeFaskahe5MyvHLO2xdp3bJ{|XQ
z8V#yYWR(2)L#p7VY>M>|lR9|Tr7weP?b)BPE7abA!hGC7!OMPteQs(2(eW0r2N^3=
zcbPv-bsuQ>1NQ4&$oCcbmOmMX0Z>91xFGScm|!B`#;QWPh9!rrci@U$DD?*p&w1t+
zg>fXzn73xg5#{hHfF~kr14B{4u;6MguZM_ckvt?_Di-MAsFHErGVb?38BV@GhT19N
zf_}s`Q%W7h)B%1LykUI{>D9}!qH%kpVu(3LExeLY*zMe%TEOY9gwc3)K&J@k*<|C7
zskl<W(ga89X(Ih4rIK-A^}$S2pg7vUKEm37lW~>|8*}(yGD|52z)EEj3Yq^hSJa3A
zm&H;a#`D)#IWz*;^o;*o(EpCnzt+|@S&qT~!~%GW@V{&7|Jrs5*TIn?35fa$_)33m
zJsZgToJg1=C5dkk^k@1j04@hJnk_SojUpgnQc1MtAGj_XtC)=yNLLPzEdc=&IPp#p
z$K|hyI6{x{+z@J7?%&j1q?@uzN51T|3CP39DI}iAh%`Uv=SN0WD)@*{FU@Y>#nSNi
zAh06_q8#|7@SFZUKQy(0gMh(V2%7OXpu!O?@NnR)-C_HCqP%zkJa9i^md^b>KMGy}
zjM1X}Od!Mm#88k@23U`b`gH>1zcCi-<AGzRrSJXMzx6Y-0QIK|<XZfV_z>!Y3p`JW
z4bR{Gt)Br902oX}={Wsu<QON=$Rk?Ew*S=s5vX5f!7YyCZzC7vfkwWv8rJ_?e|q}Q
zwbX&ixgTn4K)`_SM-7XVH^+j2@DCu_S^<J_lxL8gcYv8Cc;eXu8DL90fg7>?Y=EtF
zk`s759zT<?&wM1d_{CT^zQ(hYFekv(0lcc42%Zgta+>+=L$yg|uCx0x5P9GiLeyQF
z)B#Gw@rqTj{wwck07^G)^^C$G3G11&+TlpyX6pjne~He-{!HoAJz};P)2;*Y3wR%|
zH;GBAeq1gjNj~~@)ka1<=ahYqG5I@s+hN-&{s5Sn38t~a(=0B4a;*Cu444#Zr=LD0
zA<%~Es0&*I!16ObB>{Nm{{qYfK7fOb9s}YD#18_!<W)HSz6r2iD<Mx0Eu0E^Fq9&a
zck`pcx0Ct7CmqPwNq}?u>fOqf4G<teIvmN}AD~p(0nsSn@g^sr?7NK0gP^qWQ0-N@
zv@Bt>5A>1_L8Q3rAKI_!psJ;cm%-?TK1zh1OOqX=gO;5PBLDymMpFfYY6F0{2V<n+
zPk;yd!_98Kpov@>p3Pto@IIO`@f=R33Bf0kz6Z>9N;y*3)>F<O7us#ZVMD5XTt6-*
zXzTz+?+&1`YQ0I4MXh)W=%;B*`?(*24XX62W^QkR)x&~+9r6KJ9kGTtx8?PVv@cyd
zIj)o*bMCzqp5D^q9Z2AaXX3nh?3ODh{RAMs#ZP|1<0hFmb2Ae2Y`LPL(SCVe01$4F
zBhT9Vt}}qHv5W;FD0KBj&V-P7lt~72+H7E?2%H8R#OaOjf6oNEzK`N%7bo1Rb<Y+`
zioh+|1oSlK=a+O~=GcfmPu4ns*bA;Ftw59l&q9h8D<@QNA~3kmZZk_q6l{$nVL&v8
z5&#^`=)-WXrDw==0G$65F$~*N(U0b=H=d{alihZrHxqj5TZ9F<pD%};;T#*_8QlQD
z|2gOcy)nQrmS$PRS=_$UGqRshW(qo}Yy_eQ^Z)@EyeevXSbOXnvDqYXxQx+wBmrW`
zFrV|gY2YbT^Y;05pap>v8M9xn0*<bPbY4Gs-J|xey0q&(kVR7E0UN;iS#+b<Y9j#y
z=g#8jzQ?vBGxkpZbw2^wrj=75{sZ8&3^2AXP3uuk?#~WzKum%k6&l3LSr?HLwpr#;
zaTaDXB=@}}wz+Y*B5T<~T^D!eEJn@n6#cZosnvj|qz>R>VD_OhUZmFmtF3l4JNHbb
z*pTf4PAAW<mLG2iO&>eMkjOt`4pIP{EWZ8m&sRVqw>q~R-pJ_V1h`_}0ip+8vI89r
zaqs)nUVuk`V5SAQ#2eapb{QFV7Ec@7W>qC`U|>!&nS`SBRNtxd^wUqO@(!n}aNYp?
z9aZ@^lR&OIge-rI%(`I#V?d3Z=RITC#gNicL~RqPJT^RZ2DlZ6nz3(xW?liJXn^hw
zOSQpadHYwg#`CXPVNt!_N_n8s8d0cl0dT;K=FKbL<!?xlU09;e=La2}vP?a&!7qW3
zBG6};Je9Cc{+Y4c#Lxds(qO7!*t!;mK6hAbF#=I{2A=3ZKoGs96lCY58=(y9XAT`$
z90G5!i1CB}OzQ(26vJXFRF?)zK8<MdJGciF9uyph`X?4a^K*EP{}!N*Ru!1H>R<{)
z+LHNUMRiDOY+g6En`N1Znx0|gz~cUrJ~>#j{vDe@ZXU399Z~DGq$`BxqDTy(Xa!=0
zGLZ$VkC%<+8vg|J02)(n<M7<@!e1#7?j41)(L{Cf=vIP41QKDOSgTjIS*FhcQuIxO
zycMY8MN#X#mq;BeaHl2#4T>|AP~WOx8;D5KcDdjP=5GOXco_y3s2C#(K#U3#_x77%
z*D;pxT>zi!OQtcxiY7m)HShT~u1GZuy{c8Bt|xfpk4msiUhu(PQwlP;#aF-q7jF@P
z?C&2TnwP<LY*)7J-)8XaghK)61I)qi1FYrTt-Vsz#d46vnomm53eb=xmta_y8fV_m
z%b)Pr&ngnJ11%~Ip>>2at6`S99t`CTsYT0+Q{VQxSDXMEwjyj7J>*e~twav-fA}*C
zC?1x9bN|Es=5suhvdZ}gKtSOG%91X9mOI1~*+ZmvFlKA}y^-NS@Cb@E{B%Vi0=iym
z3VxKg2LL37*ofiIm}Lfc97CFNC&uhkIX53CWuXZ%flT^S88@E~acIE-GJ^y(+64oI
z-aXQ~A0_}J&<+u;fohUFS)~53!g2rLJ=3upLzg%Q;E$VY76R(rFX`>#IU4$Ik*{oW
zoO@(u0l`Y*{&Le5&+Y1S(2ico#}uJwH3F8<8jSAutc-6prTM`knR8ec3t8VbEVMYd
z^dSe?A4LJt0t=W;yNnbS@CBYP@`-r@*6QFoQ4KW(A!;{`?al$4!cSf#sG!%ktXoC6
ztclths%W0gF7gN9)Kt?*EEMrTc(H)(ylo;H@PFES>!>WZXm3<rUyzdSknZl31_?o0
zT0mMrN)YLg?rsSYq)S3jP!VZC0Rd4^DG`tmR7AP+ac}XQbH8!NH^v=zjC;Ou_8;uc
zyLs2Mo)vTb=5PMyTwEA9q8irddo=@HUIT3qHd}qN?W7og0C}PPU2b3Mmlt|XvKT$K
z_+NH%x=~>++cuG!n7FosX6amKiz{6pb}qrFOEgo)uGE+XwTenmFRRcxH(hx6`<Q%5
z(+$DMQ8<umB5bm9-zn_(nA_YW50?-5;KRXVMm1sA)<Jc<EEuJ?lH?uuv_+^=a_CY*
z4PiTh^lC8>jIx|lbHp|8OF6XeV1>QdQ#S8VyF$YX-967Zp4jv~CfS@#gmSpuvM!33
zf<Ufg<}KCfACDnsC!a*G7D7;-7_eUQTs4H3W=5XPqN#`Nz>$7iccFcaEwZUkDj_tD
zDZa42``3Xh_nT`ZF>zw|1Maa3ajMl(7=*|Ttma13d122oMZA|ZDJl1re33-lX}s?G
z6!{DOzK;pXwR6;?S<GQ_tYdw|z3k4v7CGc+V1=1z3cF4VI#imn(24YlZ$myUNA@(f
zFek~+<|L%P5iH}O0~HC<hc>5qrOsU-$igl&;Ek}po^}a8F3GXTpS`^qXS<{rWFxO0
zm8Oaf@%Gw$BMk#8gr&7VcHr6syFzCqcB;2fFn-lk`%~7RAD?(gVv%18`6)K(c#+Kb
z>vK?J{fxCOzQ?ZzcH-1*dtY`adW|EcE?g3daWsii4c*|5m|+mVIj)mh>-IRu)|W!&
z;>x74Ip2yw*Km?R-xD0kdS4*qIMqfE7Rgk?sfuh|NyJi9)%kPjR`!OY;%msChH=@6
zcWlO?Pf#6Rp*v3!QRRlV6j}-1T{Oq!v2r>4E|RqA%uT|~V3x7-K|?QKl{uKrw2-9q
zsN>wOW@MJelOd4ZQZSUb<36*VXEbnKm~#heo8X+vx4f8-S7+&_#-<zzWr(EvK%nt8
zB7ehK?fyw`ahqWbOJ9&%S<i?KT?)THFmZ-fbmy|(J}B6vUbtR1{`t{bPSmh#mdj(>
z4^dL|+Whb&?{z7ROt%d8?ROQvvI1M_z(r`UUcH#?=a_NbwC@c~1yn6-k5?kWp=1Or
zb`VmAsS-q1GW@K)zX*fkTUN~<MVGTat$DxyikFQm>irXqhr|pg!-NR(*|Cm5@g#9W
zd|6=>!c&p8)Xh9@jYf~f8yoSpGPng@O%~ekQ-nS}kJ#T7`AY;($E1VuG9d5vCZ!eC
z&f@E0s@c#BA|$r7lvq7E;VV4lVJ|*uyhw;7-@OpWM8f`-P-}LOJm2a9jyj_$ir(z@
z<3P{rcZ+2)&lQF=F5)=^!V$@ODmR6j_ks<%mZ!7#S^u|QLwroL02VXuU5?+?943f%
zet*NOnkss*?%tFclwgrMdCT>E|G|k;p#xRRxN0}<<*~LRPxT!C@9pBUV?763Pzeiv
zldATrqEWS^->Dxxr1wFTrec2IC3UY}jnTJ3^hZgpMeVyUu%yh?i<J$Ax7R;}ty$5P
zJyvJBDzg>{*b|Yx0Zb-$7k`-aY-cJt;^RlbA8I6bBARhbKR?D3OOSaBXl^e@&>{Vz
zkI3o4_a~w{AvP~0M?4*#O0BQP$)h$!iZXGt)jGlEb45I}chDsalS0A3?h@0w%oA>A
z-#q+N8|pRcp~N?2zXygJHS}`!;mA?RUYEQsCK(GydWYGQj{+DtEolmled6OA;bYTe
z1T88_-SG!(YfAmmrwMqL4Fck*uko=yi%Co*@s>i1Wngzn=JpQU6!)gSU}qz+>9C_A
z@7D7=p#WgjS3?7B@*S>U<tII=v_-y42-wtpxqasx&nAvpZw%oN<t1*2bzT+0r_<kz
zW^-FQkjW7OQ8F0<D*fSLd{_QBO3aj}=i6G<Dz(xx+N)jHXV08keK+8<rEnqbC&1eV
zYsj+-Cf`}Klcgx@eGw5+GPt#9-{pd+J~be6F-+zij<-@{7a=Gi3wkE(!5eB{!37`{
zceJsHzMNOqofL<ra$8z;8Lf(a#vknj9*t0dT=S=VnNvoKQ5@+*f%}GdlOYP|w2kQ2
zozU6wegF<n&{!qq;sx!$B8NAcPMe#&9%5okSlm9ee}g3wTM#ttZsj^jagI8Eru4Fq
z<qtU2Z!)%yI-Dx76%FjqQZDcmRCw$}`f}{Nt}s;(`}8|b%5W2R3%C6ZkQ*F~CkK;x
zEzglTh|El+Tq~B%kZQj<LOf#=ZMriLewW@WR$(I+)RTfrG1m<361uVQk4!?K^>gs`
z#=o%vdZ|FD3l%vWuqU7?XmmxRf|=BqxP;f%p{F#g{ege_su`zEdYYkDUF2l{#OHaX
zsQo7gjkE?Rl2tC0>@VqEWy)c*(>v(oA2cYkmp!w#*5qXEW!K3aqc6`&KDDS#?dMRd
zTeX(Yn-g+3l~QiSXLfkmhsocl>q1Gmj#~t{QajzdY=$KLTJ#`0&zW#8o84L#Woq#Y
zJ#>J0dvRb|b0u(}xSoIGMYIdbz&<vM$8io50y<X93CmkkclWbXyNG&|q@K>abGch8
zU#3ep^4Nd>9G$1imzq`UHdSNkW_)IRB_fWxt|F*=k{iXI4)RRt{4DB~lLV3{-iWf7
z_}Zvqj!!4i)@G!e70`|-z5L@0D>R{{MV}Vb%IA?vQ;<HYG~lIk&=Idw98eqTcx7@q
zk*BUboYhsC)Q@ZE={sQ^p=J__Nz#{U=W%k2)Y{@_&4r9q8Sqc#(l~q+8f2U0>z;AQ
z=i7W`AT$80Qn3o`!*}ovaPF0bLs`62r<5-y%m)-<2p@Ph85ulY&R2^Pb}b5(0<R*L
zbTvGD`a~*3a!#L3i=@KtQ!OLkjm2fAof51z7fLGHU@2Z3lQpLvZ16{otN>tjeVDWi
zjKT!tSO?cKjnV|c0aAD^j!5mZ<W9S<o$@i=5Q@%YO@Sm7-*)hgF87=}+jw;Oe(L~k
zzQwWouU;%3Mo@TL-#o^Zvw6VP9Z~rriey0AD}eMaBoM^*5QMun6(-*O%UP%w)1%W2
zh>+9%=hu1pi$=V7QqpX6E(Lfy`v0Njd&nO(Pd?GSk!9*&6R*-*ghTOiW-QKKN1EU@
z#qz#z4V5Vv<Ub`S!<fSsjfK3ih4E4@1#F15M%kEC>zH%l)?R2rzqs_hGIGT4%<n-!
zUY=ElQ3_MY(<<&_erT1lo7`UHV8a>Bdms86#BbeDdg*9#fiHpOH?-L^8a{n-IP#bk
zQ9vakY)V4@-Qxn23eQPA;yN7jTY1RwUg}$N%_@P>$s$$%K0TauaS9)P;l(&d)GX>F
zp=Awj$5nB>m*8DH6H!U~IMIZ2LP^arOjHu1FUEaUI0FB9QdkC#WO$C(C(it1VrLa8
zpUJP#rhfd+DZ=nchWWXfzg?jOhWWoXRLOW}PI{32n8mq|Xesc<`2G&|?eoBSFHPx*
zA2$s>;*PNlbx%EhaYlQD&TdO;(m3uTKNQkjNBP8O9P>(CAcQ#IaV<mpcpiPAq^B)q
zvH9)e_fXG;PX=qtT#uVB#sj9Q^TZqG<E5uC2%i|Z)7z&XH*E=&`!wb%1pwrK=Uks1
zNcjI5qW>A9f0ow&tfK#&jfUk{az5UbBMcGrL)c#is7wa`#Z*M_3)Nl`vPR<Dro{gA
z<+f(KUH;Qyh$Vk?ia})#QCmZaMgGJ&+}SepCZvkQL+blGAR>Wpv;yJ`M8e(=fjR}M
za|#Z?Gqo*~X~5VbB)jZ2>dbm~6jBTp$P7p>kRUf0hI|i%nF7wj1ph@vJ@y<xc%V^#
zK@d{CHX5J$vHc~Z`N2EN3n1w{`)wft$-e;p7;o~n87NwT&3vK3o_pI2{SyqVIAW7Y
zr}^!EK*dS~^#UNb;|xfJBA`%%6j8jwvAPX>Q;#>Wd7VHb!a+64kL{R>L$q*Z3rXdG
zBp={@hZvJ^ZBZjHx|wBx&D>7E1}>tx*8tmh8StY@1xm>tLaGGD2rf^J2hbW{1=z3f
zhw&4G`PW)_t-B7C9es=Poy@Di_8^5d$##J4u802u3H>t|sdgwqD+g==QsI?$P-5sd
zjs*-H2(KjS?QY*b83;{k#8-R#RJ4Yqj~`D3f*Yae=qPAOW@vWVnjuB05b|lRzXiEA
z{`E2$2bNXft!K4+RJk|U=SmKvPt*E7!lcxlE>?^{*bkKrA_fOX{;n;hheMts#n&N=
z-&dVIUL&FhSg<}>dIlV*0qR6;FBGG8RYO^=eDxfP^QLS&QbMl47dsxZ6g~kzTm<P7
zC$k}W<~{?4sUwb0CRp!_+ii7Q2$REmfvKAh>ZL-t&msFmfTlrEQ}@v>Bm)>g;mWt{
zTR=;T1fQq(|F{Y2^!Da?!UvXqSF(jMq**`dRcS~La+dKTWVC4DYs2`v-Ea~Tpmc=V
zB3vqV?e1>@i-K2<`0@kf$xAa(RP|;S(cuwrmW_rUv(6PGs=H0mIOoMC4MD0o15B_a
zK=OSLv0Afa;_GX7E0huu%Kc1r-eju60h}kKE-Vh_CQb<o2)VOXU2mn=R9$yJZiErr
z47kSk%Db~y>XA|hTLsqPI9b5tSSVppB4AiHVu^Zo5dsO#kR!fa>Ypnd7WD%w(^h~s
zjquF+NRby_wo$9Y4TlZs=9W!Z18f9QM9R?wwe&N(5F%Pa2RgRUIe(zu=@`Z}cySe`
z7R2Ja5Fi0|oR#2pq$=h%Qv0a9u4$0FEK%IdBf8r)a2HDH(nvJNQQfjBijnDi-ZxSB
z&r{$3tn$tA`jBPz`-yR%?|-Vxc}{X_W9$K?OS7;cYJ;)(w5?2~uk>@lK=r#KBccX%
z>-4_U?_k3!t0E&*x@4MgD}KEktR}cd1;Ga&&j&H$1PB*EHWXLPZG4hd2|y$JO7Y(N
z3YAy^YIdKzLC@Tb!zTxX=@QUKq-A#*9k@Qi+Dl{ZnW_b@`80E*&QgNN=$N(i-6_4C
zUrdndDbRAw@EbcMwV5#RUJw30bjeHVDVYJ1FC^(u<iAWr^Wj;)Gy+_1D=fgKJpzJ@
zYTrv9wi)PuYsqfwW8r*Z0(@iT^rdo3Tl(YeMmgBV=ct5b!8ko(j{$Bw2;2UJ)aDJ+
z51){N;bWF4py@?BeQu`TA^>(?FlpK{mDds#qy=z?GA|gk>(7lofY&4OdiedoOL6zW
zT`1Zc=~QRd{|*73tI}a$kAi_}4TQK#iXD=AQ2psXe<n)3_$mb6_LW{<IJ-AdLSr*N
zPkAZu(MKcK5J~6@k}uO31uMwd<8oDw`8wT))!*ImN4bs;<_@{$DUU<x=P{_){K}u2
zR}V$fPvViPnWbj>261qReu9vp_tCf!-QDO9p(hDq%-?xXD|xEiBx#dJfX*AX&~ZF(
zdRVb^wsbdEOLCt1Ts)%%K0hNe#y%01TQ-k&A+_L)@3hT?OUX@&Sumr`lkwF#zv~+z
z?n?14T5>Vgh|A{a=p>=6?nc9|wskB?rFsu>FGxi`3=|QRSUW^lYv_7<Y8VnBwP|Yz
zb8VN$$VM%euZrLDD8iK!3Klo?8${D=(sEtYoVgLIuzS`ULL&spcTP*pr8GYd2#qyL
zQMd|t@O&k`)O|Vih^UKtHHbBb>?V9gQ#P;0rxbWadYE<*`KpwbqWL)68fOB%6la|7
zzhoWrm{)I>)|?bU=dI@=OflrFV4gmsxvlWMDD1K-tF8;!1dfTo8ZMj{k@wi8229>R
zyJ4rJKl<G}HoUzDg{N`^W^!{Of-ouj(e{`!;R9ep+Yyo3!wK&aD|VgYcMnsqKOn1L
zN9TRa4a5Ora6jxsC%=ki^ZNOWOQ;8mfPPDS=|T>-BSdYuE*T3uWFh5^fqj+vlzZ(e
z7`N@TH}>PG5B46t^}DCVr>)@-8M=8RS2E1uK<&MeL6<(M(;Y`E&!^-F5j6RX`v!7S
zik-Tdt?(I6rt_8BNC2mJ8ac+0g6A0tsoju0mV6Qm8%Yc;4Vh{jV9)Qp@Vz0`L_8yt
zCx;d5Rt<M<VwsFRYUE2|#;0;udycX5&%0o9l=i^AJT30XCxg3`t&mJSh9y*jwM0B1
z*9fXqoc(ZGx0Z28Ig~>ZC{`EYWD5YD6miDMbtDpwa20-={)2vpP&~VjMj^cQjqhtD
z@ZrDMFvE0z8Nyg}cWV8K&P6<pbgAQH0nTvVnpq(DOr#g!fWn0Z^Y`CABtNH+doH`n
z3&$lS7m`#e7UvdGM-l!`fO|*VuEvE{-TgE>jt}8xGqn?zy_clX(^VdKaviu^`5nuP
z>6ur6Zl_QJAx~AobPh-9(bJY;mgens2n-PiEf;v}kd0Vy3k)g2v^`??ZX`^YXar_H
z3p2)x9X>0P(k~Z$ZAp5!##X&J1Y7cHK@j@d{D{q90;7Xpo7x@wEYm2x9-w+x?k!#J
zjs-Mv9oamx-(H8kgCg=<QMB%`VL6nlpaSfzl4l;A-W!4V+UW`+K`bLQTt-)k8XWi4
zWX$kz7P0Y_C`bMMXRe_x1?`b{$``-Z1}*QpU!i4*MY8KOFXa1mAZd@JF+lTd*EEj~
zKF)x2(Fa8>eHsM{=|L>pvs_mGIMX6SuYEQ7!{Xow;;|hEcI4md4Za9K(%p}?joOZ5
z->4GqOlV}LIQ!QTIwAodYreH@`Oml1i<!YH#+iKi>vjD^EL_Ozw|Rd<{{O3ok@x;`
z3%J66>_2fs(={=we|_k&Zs_8Fiw<SA20%KTkM53qq^k@z9w$-5RkD;X#nH8MwQ8Ax
zZT;r%AF+|kFs^@$+{6geBhb>r>pZuTKk#tef4NX#+;~%VSGRPrQ}wNv);VjBkF1+)
zVFt$CgF93k<-byD<AYeV$}YZX^=dOS-g(1(wS$d$fy_IWMoaJSyl^0nc6gKhU;cpv
zb~>-#Wv;5@*<3}#OzG&h;{WkjS#_BaDToWN>R%~FA>!BK3}*ek6+=@6rc))cyz=jq
zlfuNvD9#A~b@XH5`B{~XWJbr!$tN6;83(6%)baB<iQ)MqVp*ksX$2j`Wk=f&{ksIA
z_iP|VF)8iSH$3j-7!Gu@u-KLQ_w&bIqk%Mk0LqE>wogtQ2XQ?M53Ae+jP?OdBgD{^
zS581`FkG9^yYfuP5Jkc%5S9$3@LFHH2P}5ZL$Fi|gdn4kgFVFCJj_}4ZF|1NI0)kM
z5Xh`YKsiuA6+m^%dO@=LC*2temM~vfm#!Rde3yg3QK*I3|2?e*Z(sOI9sM%|P~Aq?
zfg@4dI_omsU?cS%Tm>&cov_L0q(TJDbr!%KK|BO1nFOhjZ||bVYn4&uKoyuwEe8N6
zKE!?B3-*aNDT0NdQhk2PX;MriOvMn17;iZw&kbn%Ui>E{TD;>vQVzQDJxD>U7G*Uz
z38-}pmoE4KxU9?U5yZ)eh9UQ#by3k^xUtG43k32*2;n>QrW|!b?6&dOXQgbwB|K;<
zv4q_mM79AHlm$22Kj?S7x)c%w0rF0`MgWP#1BQMNS1^2rgZwQ|rcUFTcgjJaNP-a5
zGJwQ_(wITkFf}CQS+nl-$Hx)QaJ(RlWhY<!61)pNTc}#h9+N7xoCFZuhc^JyViD~G
zl8lU8-T}b;ER+LieS6~*RaMy%8Y-M?@7PTM6j*W{Y50T|wF49-%G`s*WT1Xxsu#Es
zuB;E70U^F1`3A2wh9bUY#KoK3KY95-t1zruBTlfyZ1hjbvH|`A5-U3-qFh9L3yf>}
zb%+%>3F4&#VYI}GXe=Vbsv(43#Pr0L4dfMU3Gbjkd|6XFLU-?K5bLGOL8*Nte@~lz
z_~UN&-Vo>!0{Xq-u*Q4&Q=7uZ3&*DrhWk~1F)_w_T=H{7w2vHVa58dF{HQ{210dUW
z*<)M<%K8wlzJUBNQ}Ks)e<##iFAYd<=VWCLouj{I*sfw=sF+~qlaU8aapeneA~^CV
zoM~|QKLg(I9&*nJ54AsOgR4C_aW4fMGd8NmNGc<GK$1jc3zv_$PL?89U@*n>bH_|_
zimgBIabp~IVVXKAR4Wf-95pw0qt&L0{d^)kz=cmgTUJq#&auFAy6-(~qeN%H^Sx`U
zNK^}oQBE_CjCiq7)03=HBfhaqCEjlTB)1qO@ZT&3hrHH|?k6q#R_Etn5O()1L=)~<
zlfCtbV$V<rN4y!=jMR9t{dOD?Z3AtwNeDMdzz*9gxDRqye+Fx!X3%0a$-Fufur9RI
zl>pWyBklz0Ges*o;%b~T5l9&c7f&R1#Ht$sJv}WMi&cWcBHaVE!3GGi=AgZ@;`P8V
z{p(N=V&RDz6-bdbp-TT5<!fLx&MJgTjp)YH7@#2nyBQW|jtToR9WhcGonrPNv%82m
zwT?pd&7;x@^bR6VC27)CCVJtw_|%gbPk}k$q*e60MRXlT9r3#3GN1Zf3&&xvq`K|M
zh25)dbe+tQR1!;7yV2}C`1cq%hODrGiM3u7A$2;CS`f;;U|E)7svQI38;F$?bFlWf
zF<Lc~vRpc(k%PGaCu{dhlM1>Iia0R36<p!RWr&c~D91j>06jf}a?h|NEKhry3s3;b
z-H8M@JiED~Kf8t|4m5d&%z^;yPhFKm@!SmgL>Xbb2$9O)W0e}p5<}jY@zxt2ZY^P1
z^=L2?ic?p8*=t0dWbNVHfjmilp>e2gfE4?nKn*jY^=|nu=eh_#D+Y{O2Q!Mi%>>wd
zlao57ioE)ddSVLOTwHyrDk$%XYqgNS=AFW#bsbwK8Gd*L{XQ#TJ=jb;hDsDldl<%-
zfrHEl!l(k<It-5S;zaS;7`?Duhdi!PW^zpX1)>YX$yA!WQER|lXJq$)W``uQ8xSM+
zRQRJ097Wf|i-9mv0ery>_#!M7tSlW@=^}H3l(=xC`ggccq>^)vE!%=bjYujcIjk?v
z6M-0-nxP`H%XwG$BfF1&{XmpswtNI2)@$e-0d?is>9YQRN0d8ePgj$N0M9T&f$lCP
z;hQ+9XgA1%<c7B|o+bYId(|j2)+x^ej`TLSxg2zTG4)(jUZ2xh15X}MR{JEtGJK}n
zR#ObxD=ych5#O1Dt^|ty^Q09O9$kY-Nf3av-C!)qT7iQ(XVheGpP!0mO*s*U5JV=)
zvaqCwAUhIF&k>@X`tm{xruK|q(DVlfkP?|gdN?VOF)@swJ_(g*;I}P(Tr01VC&q23
zpEwR%1IX`$8LG0G25<^8zsl=##`m1W4)Ze_)JCx;*;C}ce_^Z9mb^K(Ox>jY2o5j$
z?6M{9d*FV6)=msWQvRdezlN6CYPxH5`dr-}JT@J!;UT=W#CKQy3Y%-Jy2FcGXgWn+
z`_5#k<s2v>@&S|5LJ(Le2cDiHM-xsf*fhwUFe0-y;ujFDV>lX##7>pEf#=jAQ|P8U
zqt39`NWCHW=zu#7P8A!(+ERTxx@3XZO+15}?>(90u4Y;Q@Yn2GlQACKI>GxFS$RmW
z&v$Q|_ou|h*gT0U;Y|{oGU2t#wT<I4)Bj1KIZfhkmZ@>Dr+MkI@x7bhS#<z<$ehmj
z#ljeb)A`bCLqIHO=cCs2FJqV1$S>4)1Z*z5r362Ck=-D)bBCKA@T=A<J6zslNI8oj
z(FcJemdB_BYDnSQe#|=`_7^$@z(LlUUElE+t-e5@lVv(r!!dMm5&-D3za!>%j7oH+
z0hRnNMs*w#*gzC@Ru7l|LR!sp@O&TbJ3W7)(*v+duZLdE{eAvN1Ul6XkeM8JvLzcj
z8Sp)?^Z5Ck^kI+6)7!Jr1kp`j^%HBfs<Edf%v6P!0$rEf?<M^M9aX@C{;%)~6vLz$
z)tN%DzFZV7zkbE*?9{fNU*$g<&2Z=Oi}G$H8=%ZxWeKB$kdU`R*M9)<$k5LRg#NZO
zU?afFO2HdTF<SjaZ^0b!!k^FfAnMG2-VoX$@1-LW`1^L<5Q16%ddB*<q4|wY6_6<)
zhyi`&zav*XkI)$bgblxrKVpP|6FSwabv)c-pJCj7JUC?%a{N&WWDn<@e2)-E|4jI*
zA9V2N+mkhjUHa#vewfK&jP1Xnr7tqj!QjsrQ<siM8yu@q%K(Vue8KvEhKi=sw0wtd
ze6CAxowcqohxNj|u}?9^v4j~DwL_4xZPuk4*7TAvt*~7`T;+whyeo$fcv&#aI}&Q|
z43lu;g(*IXCKQOTdBC~?q-dqi;MU9RhU^37R{oK6`9Jt?r4sm9NQprp0Kyw@QA4UV
z!jH3MnJ}BLhR%&>Nfy7qA?&q_@3zz@t+u;J%hUGD@S~PVh-HE)kQ3sW_rH@_q5M$^
zG@F2XANhd{%pYnNfgAo1?CyKGp$V}*+eZ*+1+o<H*RsJ4%QL7}FaKFh4K`h47)5(G
zsiiChRn#duj}yMm=#77n+!y#4ZK*a~a0M+=i^o_1ur32Iq4sp~Mpk%etwo%JzB&$I
z8H3?B&JVwE5W~il;d}K<KcDFcXr=Q|nh4&5y;>hgT>p&61ziZc0uU*IqbwFQP2FB4
z*Q~@@YzkGs1hzO(%?^(EBs=s=7`Xp*hGF`3pQ?ER7D4CBz>o5%zb%JABbXl^gGvbz
z2-I+{+ofyogTYM%9sy39P2m2Ggmm1VJFwOidLKoKKN&N2zjI4eSuQL7yX4nb0MjFW
z_9+vd6$OnpA_&KWtKI-D3&G82&zusa%$RL1tLAH>Oi^{Vqc6tz!n{bY1NK+Kq(C5^
z7*mYjYb0y{P^4*F`<~MQhgOgAru$2zYFEDdeJ-v3GOC0r<aRWRQu-pe9S-hH)5L$l
z6t+V$hSk{RR<TIZPkJ5jbNDNyN+#it`91L5X$pjPj+72B5uh61xh{=LOaoim38o#r
z;Q>_76GKFEIt>@z0U&pX^anKvRhK*@dP)oAuxSFpOQ+~6r7x19@Cv&N)@8{$_y{d|
zMCXN~8K=Z9#+QW*q7kb>ATk8Uk{C-2b^~B@1f-njKV>6`e|9lN*+ljWoLk7fYv4dE
zLri3vPm1p%R2l-7PfIW_7M;e})9aaX83^sW?7H{|@cLCx$K~x|5w)2iv5ZHnDKCU1
zkWnb}jUhZmwgG%QL>wgDGoH3j8n&t2v%nF(02anSYMC+25WDxy=_>^l2(mG}eKF|(
zyEiIc+w?s0yDeQ1mgYjk69PO9z&KQ`#RBZ}Z~>V^u06$}9z$*dzuMS2a2kI*#5@9P
zP=xFkfcoX2?#?2hk~wm$!{Zch><fl9{7S_|RJM~<)DfiwdVF_&Ig>PGcD~@RoP8%*
z3aG`oV=1ee^W-x!<bJI4qNIas%Rmjb*ovNI)5S$>z0;jRrMia4xZ^ZjlqWyr0MjVA
z^3nb|sr7E2?lVa|(;6@Q_x8s4lN>=X3KwTrL3{S-VM-|?(4|53j$LoJPwdCifVj1e
zPz4eI4+|rv0t#6<91wqt*&ql^FUH>u$wS*dd3uL;kow<uS6Kj4La_aW)$3l1XLcx6
zH-bP@B*9#hUM)6zZ*vM^jSUlo?D|vUe%;s?4}-$e;)W^XNiJfEir}zX-jh72I1g)5
z^V4~_8luF)H>aMd2O-{+IP#oIX~yx>en%r-UM;I4ac_8pW%LApRSP9DDtWo`E66M@
zPAlN&*+fzQncj<Lj&R!|;nGx*Oi~HdL+5#6R*BBzfbrkOA5lczB{H!~RSoB~5lJdF
z!IzQ;t=+c^Aj;;BQI#v-YJ>ql@5)Gng()?#44!s9`xDUSQimH7RWrTl=7rro^g@4-
zD=d|?=a(}qJm$CXM<Ec<qaa1dW}EqR&?y_wRg~qZ+?v3i9(?1H3hK4^QpocBfcXEV
z_=FHA15Vv9T$c^1<9$Rxqk<`Nz?PS17@)2Z$5g*ULWpz5(<TgOAkfH^O;uZWJ~4wT
zY&x*ji^Cl2fgtaJ6nL&ku{Voxgro%A^N4@u+h?$zNP#K-@N5!JB&o*hUiJ%3YgROp
z6e-xY2V204hKhC{sl$cfQoxnZ9;(|U=OfIjh~2rg)JHg^$cmHXB@CBU6~TXQa|6kD
z{`v}($Rj=P;0DMZ>JjA12kfV2o<BC)KF38;aEDQlQ1!k`;~O5WSz{$?y{}Cjd_Xc4
zd*5d~k0S(p*c#v?CinXn#e6{-bsp{mJe}@r^*AbiGdFTXq$QK~6TWuZO3BDRG5C@f
z1t&QNdnynvdFiJSYuSa5*plHU_t_1saS|eXY~%U%!QU5tO`dE?fHPh3p*Vs-snKb9
zLoBW7^d=b`Vg|R*ai03=1@`s<IE>&jm#?$#EIf)h>fYa!m5yHvT#yV4#fquHN(_%`
zuw&vysU_==8{lE#nmh{EcP2D4tg59+ZfT7x$5szBARs%-eU`I$ftXXbvba`nUmL&w
zVWg;QQyf|)>r1NhGr`Y~JBPcxKM%i>5AOEcoa^}7aZ6@NHg`$K%Ia;q=LZU2lb)l8
z{7jdX%`xg2?~7dX{6(H0T0EI7h=E<_V^h{;vBAm5muV-&#-?gz<vdZD%E!_qq-6en
zypfppbna56&I#J3^7*th5>pB8vuDprNW40HtZlfBHg1)8+_TT6$f$%t(u-Z<T6gD{
zJjU8Nr`-(BmjwqqYv(G5?V|8if|U1I@MCFxqYpml%Rg@DS^tuBqAllvNOl4m15KoG
z0)8~8hN8zQ)MyluUyx6_SQNrn3p?ysu^B^8px{R+ABr`mRqzEi@(c3G0zC$a%I9G=
zmgDcZ5;H2c+)P#br%7n#Gj<B&4M%=zoIl^Ojz57@%9kJ-{<}%|as3=TSH^ol{rEfP
zXmuhto7=?yc1)p12A=yiS4-{qJJZYx;ZArgiT`%&1nMoZ`u?b{C|&Hfp6}_C9Cptz
zB_-%MjEhe?%+zWqyyf#hm{bqtiJ{S^C}IBRWlo@Ci5B2i%?#x|Jv}{Qm+VhVhc`Di
z$voWKKi$fCo4eF2d1^mAuo#C8+XUSn?u21O`e&rJI^h-OpP%SU{__qd&|^ME*W;#x
zDH#>N{GjgnM*Mqd;Ro{8n33AYuk;ySDOmm_mHhFzAR`yr5c{XeZ<a7DpB+zpIQM5h
zc;PD)MB>LycH^Zm9{j|&K0KZuWCl@>+5a@T;0M#weG4Oj;ZM&SU^dJYrd9qldB_h_
z{8c1$o8`}-P=whHiRb@r@jiBetiRVl8rvm{!_NR#TqmoH+}+(PbPB7Cmy$y)ms2E(
zbqzz&dgHN7yzpX-T|JO5k0@_5vcRifT3NYJBrC<i@l1ad=@L<gg1)M%>f*h-&e;n3
z`e~x&!VeFKXT>P1?v*|4Iv}nRuL#@jz4{HD!?OU?)P~@3SKi#e=I{T#)$Mgc8jY9>
z89DhIpjN^x&zX>fL=yXEEV!&CYyHF~m9_U-$vV--G?!L!3F(O%Mu&@uiHQ^4A>_C*
z$Oi{PJLYZfXFV1aUe#HaAA${|m-Hw;oTrjm5JVGgJ%HKyd;#MGk%EFm7+7MBBDsu8
zh`tU8UGAHxvwTgZSf-Y^Jd)4xoN;jZj&?p|rgVQ47k;mL0>cR1{&aoxd}Ro!;!o^1
zxl$_uMwhI9Vi(>jza9|tZ0qq&yjS4j!a=ZDn*rNb1D9_+5O`J4<F=6vPWIl!qcq!B
zYp1>duk%h*Z0}ilaAe(ix<YoVP_)qY^88-q@BO?wIxIC`jqK;uL*A#am1U!NF4l>j
z;qQW&I@lH>(G1WS9BjRNf>bwcy!=dGUq5^b=MvS0$am(xe)W7G(w;q24c=Ru^l}V&
z5_S6H$B$5H&7jkF<G$bLqO2^6l@o5Uf}>QDbQr`7d=5G%VGtEK!bb8XXx||hI+nc8
zvP^qaAm!}<!`HN>UFnkX4ALkT9$}os>gQQpZuFw}n`yFd;+lB}AZ@Z-i_)7?o@nf1
zs3Xx{p$s!DVR_v3ndWCyzPrmK%8A{k%?^G1&0dvx8Rys<97giQ-Das~Y+PKb=6V^x
z9f8Afy2aVRd1XkTJ7efRbU40Bo13Nh&WVdelLeHeO0a&z@z&?N=cmWv@|Xp&D#KLs
z;XZ!<dMQKt!7}m`nV|@*(`h_0UJLYob3#D}J{=vdqRfO*P<hqD>Wp;AuS1|;;96pa
zHn-Q}v-@|GL-kP>iiR%I*jTVt+7m+OB*k4Os*HFHjl5q$i3HLrf~9fXE+7YY#!Bee
zl4+56xPvbz+s?;4`NMaFusYJ_(|GgZiBw->?h)&PNz*IrU?hQIT-ZS^uSMBBtZ>{E
z=DGCH(#1l@da*xMEfZ#G@cm-gBoYB19zM^N{YB4R?UZpb8Y2;wRL!`Zsq&j9`=Oa;
z$J%nrwtP`%eb{-$ot$4HPOL;lJI-7mGQzqJCynkCQNP?=TVyOb8Sl1km*gcRB#aiy
zoI0y1XZ^A}B&1`UH(+UF8A3kjS~Vt=8Q1_9A?-&IT|i0>SETZaExyYM{&`@EJM$$O
zX?n$sqFzYN@$D4?x|$9x{j8n`8NG}R-YYu(4|NG#VVZNEj67I&Bz$7%K#AhU^N2TH
znqd-$SMA}}ZFbNuTmAC!<y6qoytrR(=23U_5kpx|61_|UOJaV1iJ8zmgT$xkDh(-A
z9G(ZJ-Zl;{Ws&SD@3o1$#bP_VyAfFitE3pCrwK10i2iDVPMz~jnmZlUo*(*3)^yKg
zD*QayOFU@Wzn*h4`OI0D_1cT|Ryi)~N(w<>>h@MR?IMGI?4RyPrvF8;Ft@dJh}^T?
zw<_*FN9B^UTu1BP&V(!w5~uOfn|ktx4=#Jx!<wnGs!Rx#M$Oq8JMzaM=V5!7!jJup
zM9sc5p0HE>i8pJ_+cf=QMX3{=46@}(EoV!N58q+ShaJal$iib>Bj}sT7{y#_K?$9(
zbcmA#6CL&$y~x$JzV7<H5yrN|dj+dBflBDH$(uW1hZHYf#*vLoQ%B$UvdT0$I5c$h
zr9vLF_6YPo-Jq^&($XIQ3w73s9+!9wY`R`{F0&4@ltNI;G?*Zx$gt;f^*`|VvO2DI
zwwcDG6<giPsq2eK@Zq=byDR3ib8T1Wd%{lIXS#ABqO-NB+ADW^*ikodQW)37E~&)B
z5o72NusBgB2P{C^)rSL3!_kBc{5(AQLYJdzJnU*bUv~Ok{jM))B97btNs(FL^dP3$
z_nf!j$g%I7OQK7~izemJ$+#+7Z`o<QKlm~(;uK+P<E1>M9`Z2O@9npF=znw>T)u0b
zxjmUYR$Y<u8;AVrAnty*Hd!68zfSGKW~gCI!Dj{s1%-&}A#vZ_y|Pn%QvBb~nP*{t
zd-FNiajusT-zaT|%is<p1E+GVFKo{V_d#qbn4QWOuywTa#Rtx6?=}x%#nto)%V3&W
zpE$wRuA!o62suj{3?BycwobD#eqVynS##j`9fn_9(V177IXF00mX;D}CEOo{WA;bC
zD6<gw4aYA1CpcDu97p$sJ@u}+l$BV#<o-{<OwTYivdZ;xWXXMx^0~L3nHd@VaY|rM
zJ32a+Dc#@GWj`S9qsEW^VG3t);mpOOTPOBdd>oexJ{tx3!+!;5z28to&{%>S!30D0
z0dXJubG=y{*hHONEY>i|zmMh~Q&>rkLC*Al4(>4>SO))1{({l8Gv)zV10XSh#F)lu
zww?}~?AwDIf<x{j1Uv2nFYDEi$#iPMCjawx>XBgBNlWOelH=mybUQ5sRs>xADnh-V
zSRVC$8Rg~ZlWL$`JgLp8*u&gIu7ZdC4?aN<={%-7(a&Hk!QkVOLy+PBAy|#oymbbk
z+Gm_YV4L1HJ3a^R1rZp1P8@yC0;|%p!^_^&a~o{^%a<=jL_}o#HVX<1HylY<`sA*@
z?RxxJ$@vKKAU@7CS9sHp3!}Kf@>dDs^@fH<pv+rcspW&ae8wIG_?_x3`PhqFARiJu
zQK<*gyR!lNu1iJL#<jB*7j8i^MThgtJDEe@>AQPkNjU(Ev2XMze={$`M3Vu`H4F^L
zCJbvwvnHm{G1sw6nw$-TNemAUSB<BjE-5cBx3{;?%*@<)```y)ArXgRlLt5UX7643
zG*w$|wXPC*0(Ap>3%~Ttu1W?r95a5Ok$CyV7dNSrmWVPj0c$zfMN?B#LvN(7Z&~Z<
zrOw%B-EM$hl6<eT>^u|v%_cKi!4;daxE<CI`7OrB%8b%sCr=R|WDXd$EDIYrvAphb
z-T!d6wkveb;Eb?cOr$9^SFV+fRMmFq-<Z9ZFXQJ9j#?_qxy3N4yQU9rT=N!?hrJlq
z59^Dit&|-_gWW&G34~iXRv3wG4Ju#aYSxt}USEJ0<#vo#3(#l&e76fiJdUFUGk{$3
z#ocT-qCdcq_zFP#!=hlU&u7CxF1ZG}#TwZ<=)K!qd!rXl_dN=T!bcE?z|kH~&^t`C
zq$Yx6tIKSTyQKIWpkbYPo6cKiNcH$6<GX$x3cwFlTYK5R%G!5EWB5VW-t1NQp<O9b
z*mRI&bR%~|e)hf1H=Vy5R@cz@IN#lk&t$m6!4)s0U}fj)UOEqh3m|<g@TP?o;Q@f-
z^3JJR^XIuD=Za2WtGi;R#wY|Gj`h4liJa$W!fsg{3K*KF74lwsXtBS%vJySWs7$Zr
zABh54LQ)qis}{O{HXXDF`L`o>`s=&+kW+hTQpNv*dJ#Hv2iv@i2IYbat4>Nl<t7|E
zDol*kulqk#fO4S^yyC@?+XV#$cMU4kJY|i_hyk`~XlOK9>;wCD<;+(|?;XI+h^G{g
zgJTjj4$K!&{dpP7R>6$Lg(oWNeGSxlzy$PsMR{d6!x<}fWh#Squt{$J<I9SGy*0L5
z7oDeSX1->Avu_Kn(YKP4$yok0akts}5z1d#J$UGoqF3hlrkpcjSR{0iJtKb<vAqG9
zi#OiE@v>NG-#kOkV~o!rcld4lg9DEII=Rzy!#mI_5l=gKdF_BQ$|(aNh;##r4cm#e
zhtDCAeTb$iP-!GkWwxnaBb|L`rKUUu9KNlhP*yhByjc>C_8a-n<$j#s6dp3mVL6wS
z5xuzRTVY}G<!y>3Ln>rZfqWM0v~nGz_|><806@Hmx92*%g=G}0^<qK~Pz9Z9&-MDT
z5zXq)A74+sIU98F1>PI;jVcp((HaV8e9{93`(lEjD2P0e`Ieo&nWq*OEgejWy&p&7
zSkcrlUv@^&o4Q>+Ys}88PN;vMn@fF%y^~1SG}L^U`#C`)6S#rnIQ;?lL>xzJ^va^>
z#Qk>`(>X4v`N`g6AxbeKd@|tz3%Bv2Trkxg8u+3GIMs0%blbQ3lIR0|gE+gJ@x@DI
z%%P#1sGY}6_=;~!GmpZP*m!uV-Cnmgt;KaUFlRR{{s@4VqH0$7wj#5Xs#~@+F)@+T
zxuAfy0E^-iglkJ!e0Ba|Bg1eiZ0wTIt~6$bAua)dZ2SEzUQ-embgkLFG_>CSpr4P@
zWQnW;Mu#8W@^n*^@MQskv}irs>;nYMg(Srg!3p#2R;Y!T=$TLv`F6?-hi8ya?GIk;
z7%o{K3**(JNNEdxhC0(`wSCPqAwalBSE)&%*1)r_l0T;s14}kgfbo(+Mk3&y{&QsA
z0<~YX-dc)yk@=t5hIy!NiN3Xu4SpiW4%2e6n|7}75~x^GEio}M6?t@HLiu4c-5)0N
zM2LZToSf%bWE2#euiKu7Ge4&v{3@GJkv^cuD=F@d$l7;)+G|B<W%GkFtCGyP+IZxf
z4T9Qe0JXV@nnuPUsO=+IQBa{W60xzd`TP5)vum4m2S2dTorBE;4&ZF5DxQXcU42&-
z^Q|{ehHx-U(Rx`WnSL0c=aY|b4K)7@g!373SCedh%f*O*7au|~PCgd6`9a07B{uW{
zs_V9GJx49F29od+804+qMyAjfrsc^3T^aV%-p+0~PpsN^V>YalGxm~#oFPj`dl!@Y
z_t~OxgjUlS`Ec$I<}1wh@YMq{BIqn5QK150W;hJMh*Pu^H1j@|@O<A3Seho8TkRG9
zJr>fGwhM~^QApHi>87jQ#=!o@>kEa_Mid<NRu692@#*1VT*FMW8%JL&+jY0=LpgAN
zsAed+_?%#GX);;&r}q;zWmDXM*R5`F%Is*y1uoXY*IM!C`>@?&pVg_Y(`N_rt3EVg
zxe%^t_4fX?E0=?IB$e+;JeG`j=OGVMd|%|p+7W4SaOCE2!R~`9f%@Rc(dYVIXDhE(
z21hoIg#G5qrWd6&WiZ@@u1MIU6Oh+Jm=Z}I;7w`-YL97?S!9;}oBRdq70SkJl(^jB
zW;wACMJHNPVm-~UgC3Z&8O8r$%KinC|GP|?8D2(qHd4(7PVgHLX}j<9v2@P_`wRw}
zpGVis?7r91;QROQ3uXN7R8%-RIQVYPNy*6kXgZ~j0j3FCKp~v!?YZU$h%_F894v}{
z8?#2uV>px;=u%=ypeZWB2ouoX*k7L+d}ahz@4_uPD6-`B@{Tr7_AV_n5u*t_5jTH1
z&+|f=uW<NYlvW8>K7k>F=H}+c#l;05%T6>4{>jWVGB$1v__~Ql$v>PUbQ!3iE3fKc
zGuF{R7{@|kf71i5=B)6iQ-o(S@`uh)<39oP(}02pi8MU{Lcyroc;M(TJ}z#NznS$W
z*r*g`=<8E?Y#W>z;4HI&z3jqI;qMHATm?i9B<oi>6iCd062j|0yTqg%)&K6E%cR+T
zA7?PYBX<D*arlwYh&nN{0U^{p7bIT2eG=ZLz2*eYd;3_?$}M04M{db?1Bir!a)Qx9
zN##VCBpMFJF2B!K&Pc&AMB`P?#+CZj@b=blt_VP`rgeLSqm9vpS7C|NZ<__I3L3y%
zm&q!y%;mr{tzl`}VKZ7s_e7nhO>mIDwno-+R^&<s?TFiK3yxXz<Ll_f7wgcN<MX1Q
zAc8)Zwzs&rc(C_r9<r_;1bqDk3E`}tah|@Ee8yZH1Dqi<De+~hmC-_^Vih|e8@>w2
z@=#5CN=5q~XvHxVMmDzckfUEDu9b^dfr9MQy`DM~%LOH^FCf86K&giAfzON1g5Yij
z&W-ON!XKPp+o23tdT^+wa1MMxddH0)Igpyfo?HozE27Go&)&fUZk}f=J^tt>%{*{&
zSpt{%19o?JmF-hXlFAR8P!Q@>1!$7QAkQymU|@g=I@7CJ@)P!Ani_OV<X-+E>Qg0w
zP%TH`H5oZ_grk4cFmL0?d7=mzz=ZC>Ej!g}i8Sx3ng}t_vV4qpV=a1-yPNtFX*uE-
zU`WmvN>9FI@pbe_`%XRc4yhmi2(GMnsQ(SX`#`2D&L;@Pu`2pF-dLS9hPN=6BAuQd
zNat8uZsN`QfyD8Fb_+ZEk{pH2$;$ZriI~fT`omkSSIenvU|oLNXqyLU->Rf{gPClg
zYe~2Se+eSwL2%0QT~6B@SkhMXA3e9ZX5!lz8=3?stn}!tpU}`b1K;&&CLW7J;ASVU
z;rAv&r6@f0DX;bE#=#M)uM`k6mZBJ}8zwpW3`?qiBglCL&^Z2VUTTI`6F;SozY!ZJ
z@y&ma+xh5&uiZzMn{1Yl0k<RC(xveACqXciho1FN7GKF!&y^RqH43G#dF&*F9^x_9
zzZ}eq*^#d8;$S)w?*BpB^ggrZ_Jix&w*H+{A}rT}7*ErR<L}+$?g*4|e!;rC$M{N_
z?F9NgrFvSo+=r_-DN4!JH8s<eZJ%V<v-g{GMHA4|fTIN3O3RKb9i8J%_F7!2Ayr5X
z`@zoApY#_Z$M;o#4x(E7D=I2J1IG7iUj7WI_v@!u&*u`}Me`B&jn!upc#e9^KMdO8
z^1aMO=dbP#7_hNJiqgh)d-__2<p&70*Du-^zkhH0{j%$ar$;)s<lL~h2#<OLF~{q$
zN36evDY&960^>$TM#``V=s0Iec*FMd=MVcvBv--kOpvB@pSFvY^AGtBR8^s@f0Kpg
zqKyizqsFhQ?0OOiMDoB$rT^&9G|!UABV>@{Wst40=)a2oLcV}oD&tMLJ0IiazW^92
zy?80#3^?*cwi~8I+aLu?eLOd3(UHz6kA+XQ1-C*@=<}tG<gg^%_FRi3ZczONg0;+3
zCE@$bw`O9z{bkLW@oDlya6br%?vOY?a$@DoW98|qCh>Y{@T(Tvj3cmQ{JN{3)Z@1G
z&9{=z*e<bA$@tdbE6%27HZFx|@7QXB0b05>*V57gJ}bCfgH{lqlNtKr9uNMcn~hHi
zvHvvZ`T8c4>_ZN6chJ)cPEIcwg*^=VapRhk97DF9E7S<dJmk9m+O3Vxw23uC4SdT4
zwBDw8WR<di3u>pPB9-33&aS^yeB*Unax3N<OK~cg4z=_>%Sw1WY=31;zAN*Az$iCe
zbQgRBjOC8<uluKsa=pNW=qT`840!i>S&khpSdB6DlbllPu$aL$3wun3UMy$$00)E`
z<_G~_efqO8%xK2qBxqXNn^=!^N2fH9i^<SvmyO(CnQb)NQ0bbY6bieYhmm0|1-;!H
zONy8v$lIi(O6a=P;sEIz*(=0}9QvqFx+D>y-_yLtWJGojlTDeT`lQepljv`E+FeAH
zWB~vvNMh`M4+DBZLzCBN$<oZUappS8=uB+$y1mF54AYc3N!cx&TS$chxTr0KG9m$Y
zxpZmU=UF2>9dJ{f`~w6xjpO)yerxGPEOqDhZ589LG_l&L`bz?oWcIb0P{zxsh}-nO
z&)xNr!NIiWo2suw?>)iokH$&(+5PRQz+(PT)BBu2-wO{P^IpLj2s$b)W!_D%KuRIn
z8*MZsHj-jYVqTA*DWs0u)xNK*nBSITdP;~a^<ashkXD?);wQEnU5&E#Q)j2Z7f(!@
z7@Z<_D4s$kG{O#QdY`MTH$By;G$`vPVjV4-_TRznzsz_HSZe$ugB4fI+T7h1H}y+^
z#BE~Ku^t)9>Lq<=Pg{XPr2?HT<i+EHKz&VYQKvD5x{Cv<HNfm^DFESEB<cAcJnaty
z1bl8y&e;7NF~1;oz7M;1=WBJ<#d>}LOwI2x#4O<I+Q@w|tu+&yBN{UU?`re>t%DGj
zDuxmnk(I#uwIM*jyUk=*3B_p!XED7#6}Pk)T4|(Weg|fi@~Q`AavKsgZmcFY@4E*4
zFf6&DA%!Lr6&;-<e@D~s3?pv)?<Erxh8;gHNhK%92AeY{bji3d6Cp4nBcXZeA3!`h
zni5c3jR4}!5v$v0s}Vq)YIe!&!cWh;4DZTIUHy&0V=O8(!;1O?iF?@r%jDCZL)?eo
zAi0PfVxa32!f+7&LuxXLLx_5MP7)W+=-;1g!D(OWSdkTW{GCRC(S6H(djFto;P)JX
z<TxvH4<G-}cVe`mqv8@tNC@b6E3(Lvnz?Il{pUMiwpe2tJ3Pt$#p;X;2$}j^hwmid
zqJO?)OUkI2Fp^mIZ^uqxJjZ1X@U@f1AO6PKNsvl%xfH_3Bz&7{A;V3yYx8@tUFJJq
zs#C!rh~@DrLD27?W=NQN$a_^dqu0t{{Y~({8v2g+`VH5@oX0D9p$8&Ev7Q9$nBNsh
zHHjLqJuv@Y2WH_V{(CT=16gkW1gU+9LDTTRCMj#R4toJM4g&SUEhC0TR#7C`e*wIi
zRsWTNgd-hWg)PMp;`NglhOMfMTwEHndeoO-Kj&g&(&{0BSMma5Ep<<tL3w&0KrCCF
zQpa7*%*>39jZIBwfEr5-F=Cam==601IL7x=?IE>eWU61rLf(IjBR^-n&vv03F?0$4
z2>{c}UT>(ZRNoZV(w{C-KIQBK!BEI}@JOijrh>($TaJ5?=#3QK#1XCD{~{EuyUC?x
z)eiio**zDx`G-FN;nvcP{QCK0cV{PwPRa{OorM(k$?A*sT@pL<+bUH40<raXf3;5}
z3vM^7CD$Q~h}>5iS$?);(_Q|}cF|s2KtS@&l#pr0c2zLTj}}NZJHKTA{wM_LJU;9L
zp<+)&t+9~z8}UAr0mLhc{TuNDKS9;qbNGGt)`M@iw#Qb0^ioSc69h`0q4-5$8)LeI
z%oKXAVZ62V4Z_7nmVAuTzi`$|0l;?nnO65V(q#pdxe*7``rjZ2JDFNUEssXZ>$b(q
zjThxZa6%?zfmh<FMJXB{+6kT6Ja{PsogpghxxoL6axF*7d-=cciufqRc>s}>|F<oh
zT@zF1{=*SM1Tx4@`EAC~K_|Z5>VZSrE-@dGF(SuHevy=vltG1#DhCD%360qSS{5&m
z^enx-{4AN_>UgOt>YyACT|fb_<K^QPM6w~t$tWNu;WDv0ao1qSIBr}U=!UnTc?7RS
zW|~L)=#q6`0yT1V#Nv?^sMC)GJ{++uU_zkih7ASvN${1773wGlX-OaeOaRe8D!IUR
zi@}{*4dzo;;G#jeIDK6(b@@vs6?}!gZzk2M)pf@9`(W0LHV`&Il3jM+d2$t1K>I6q
z8?ZD4c!%<0n)XMKdzp|FQb`Ymw*@JgEtCRQ7Tv)?N4o98FMfRPMuO&bKH3%7lH5H?
zGf6v9u5(%Cy1@L~>&+Dhoxdgb;8J(6qMxdXlU5}0zUn;$A9}kRvKwu+@io)=SnTlE
z2Ht#bg6jxjds6oE*k&6|H%C>k&(zV^TEA+?#luT%%RywVa*(_s-sgHTQann<;1}5f
zMYH@$p&m<Us08NX)d=v7rXWZVCGih2c*DD3Frf;fvxB`#jM&p$z=O(&k<Lv`H(p*E
z_)=Urh)bzB#PuZ>i7RuMS(#*w4_ssapl`6C-rUp#{u$4_lU&e2OJF=i5`Dw2aW-lx
z2zzzrmC~a#NFcW0B3+7i2bjQddIh(6Y!CPA*Q%B>Gk{31>gqZKv9NiQokpcitIMP&
ziONjC>R5^HHO@BGo_3QfS_ZJ_^bs2H-=ID=2jj)l5cS{^8z$VutaS_C;FFM$aKgkz
zg0)+)I^P9NBazWcwv@s`K8!**uR+a6NR!*u*$UujhRsjD9*PzhU*km7Gmn8jiu!6J
z0L!^JEA;zeXqWnx;G<XgdPOqpFDsI_eY_ME!%RmB;h<pThGVa?y%Hgh+If+9%lD!p
zv>qs6va=<eU)<jN^lAw6DL@3I^wUmZ8^k3Bt^AGbfz#uEruh{fP<uk)u9M6)bkYe_
zFgX>dS-_MbSBk;;4rN7TQ<zv-E>G2%2D%C8MuFr2{9m3JVkN!Puw8Uj!9ZYv>;@e9
zq2b|{Ue=)<ZP*<Ued#NU%X;uzvP6u##g>U^#~gUA*?Eko=jwt+?lP1FT{fy_`%Sgg
zQlF+sOpIzG+Xu%@Q%eix0K_n_sxJzdMC)bGP}xib$@qsNbeR&zN^~nZOr4LT@Wc^^
z!vD<<>dx!_#J-$=^0&~Xalkvn7v^Z{-^lK7RN!F@kyF!4X2-gx7C5c{#D4$({uk;~
z{I&v^u7MeEv{-C;eQ`Y8r`hg$$qYA{H%A<Uh&Q=4;o&Gh>7+Za+~3?+H7LBl4{Lk+
zgE|j3nsg!L(j!AoWyYA_YN-_4xZfa!j#RMD`tITL`U7HVaVb$)OY=A$n5nyepa_Tq
zNMM<1NlyQRwUEyl6&LCL{?@J%iryw65I@E|AX$?-_4hY^gu#i7IQ<W_K|Uu@8HHB<
zZ6Qnx{$AjnHNw*T^HC@|)%*X$$A{Tr|9+4t$#k>h1y4`UNQ{-0m5TCmRg+v7INn0J
zAkeG7h4g1k0f-YhDp4oh$S{7Bh6V8SV;h7gk*^y7Ol8MJLV5r8`yr`%Rb{0D3i*{W
z6$SC87nZI^hd|0-g*{<Y`oQDe<NDyeYCiMU7N>Dr(4bvgo4_%Uny9<Zj6cb~1lutc
zl#_pOVAbtQhXVM$5Dnr)^h;|%$Jh=LJP{q`hrmxP0LCEUaeN@9$Vfak59UAu1-e{0
z>a`lv<{Nu!<hwbA@q!?ugD?dNSC2?c?8F@b=vR+o7?=YjCg8P#&=5hgGbIsBFGNw;
zPDuP)n>|jDSfNK5rDJjH2T(gZJ3GL9j04Ml?bDP|EzPU>BOR615*#~D?CX~p>p~Q}
zo<G4xRD;@RIM-NaO3*U4y)lx_%Gzq@a{wC@LpSMr%bk%CsSMqCwX!(r;P2Q078a_)
ze1q)H2wxhbk_}?DNtsM|?lnX)A%}@ZaDFe3pj|)F06}>sgiiZjQ#?Z)yPAl8Ddc~$
zTJ^mB<PAEtF3=P&y~#l0x>i=L_LBQ*PFm`S05p~iQDENt2!s|<!=(jCBQ0xO_O-Do
z`C0pt8O~XKAo7BwHt=aM&=%wk+Xr(H*sTM%a(Cr9a_efR-)C{Nq!15rHL4K77I<VV
zCA9Z`<RdZCp0@SDmyc=mn%<xYy2m5a!Y4@sPJ<gb6!!|HQHXV^GRzgWk9#y$e@R{z
zSs|UU#5dAz(<1{hjskq)@9Ek2B@d2+PnS9F2ONO^@EHK>4L^5kYKXX5E<m1xNYpe)
zW$KeNx0yl}l%R3aL2XB5c_yN~Fvp#0A?PJ8u4Mq9oWNmTfl*;BuI_<+TbszD^z^b%
zR58G%WRO|s(y~?DEQYCniE8C#X1S%Kp{<=sPIP6WlBqjAJ>5;TnN5lILkSm}NWB53
zixgkS4|&L0BoHk=_00WkSI2{ylC#_6xG3yZUWM?=-F0HXTHj1Ue+<bL1x6)>egHe`
zTaZJ5yl8yrfrJN0bCd=_m^sY@YbxNZGjd$~-X>!pJ=FWbWp@JUG;}x=LPM-E`!`<D
zAiw;61w=2oZ_p2V$$$4%0KECe-fgjen@~^)0Q0d?k%IAW6DLsnFmkh>G_D*sx(|)c
zUSK-;rwQ~rIE*c?(x;9a-9YB*mJ-%Ky?_T#gRkfGZ;0x5JF{>MyU5?f|FgS^8ZN@&
zVRgam*wt&HRG$h)|Net91H@l=-V+PUsY?J}`tx%k7C6i&(Lj_N;&BNUfi=dY(1(R7
zM~qmBtpQ!?V=-;00xQT?5>#TPkKZ5})=QA+4F0i>B?fG)5DDAb@k`6WN%&>lPQ>uf
zxG*ACT0NsO;`hM9k8kSG;dALDx_@N7Vd~H-i(V<gpTS;WfuUs37FRy@JDI^${~y$m
bup<;phmJF=)LWku@Q;S7j!K=9b@=}Ulqxs$

literal 0
HcmV?d00001


From 23c4b08d154f8079806a1f0258d7e4af17bdf5fd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes@cloudera.com>
Date: Tue, 16 Feb 2016 17:56:05 -0800
Subject: [PATCH 0005/1644] ARROW-4: This provides an partial C++11
 implementation of the Apache Arrow data structures along with a cmake-based
 build system. The codebase generally follows Google C++ style guide, but more
 cleaning to be more conforming is needed. It uses googletest for unit
 testing.

Feature-wise, this patch includes:

* A small logical data type object model
* Immutable array accessor containers for fixed-width primitive and list types
* A String array container implemented as a List<byte>
* Builder classes for the primitive arrays and list types
* A simple memory management model using immutable and immutable buffers and
  C++ RAII idioms
* Modest unit test coverage for the above features.
---
 cpp/.gitignore                            |   21 +
 cpp/CMakeLists.txt                        |  483 ++
 cpp/LICENSE.txt                           |  202 +
 cpp/README.md                             |   48 +
 cpp/build-support/asan_symbolize.py       |  360 ++
 cpp/build-support/bootstrap_toolchain.py  |  114 +
 cpp/build-support/cpplint.py              | 6323 +++++++++++++++++++++
 cpp/build-support/run-test.sh             |  195 +
 cpp/build-support/stacktrace_addr2line.pl |   92 +
 cpp/cmake_modules/CompilerInfo.cmake      |   46 +
 cpp/cmake_modules/FindGPerf.cmake         |   69 +
 cpp/cmake_modules/FindGTest.cmake         |   91 +
 cpp/cmake_modules/FindParquet.cmake       |   80 +
 cpp/cmake_modules/san-config.cmake        |   92 +
 cpp/setup_build_env.sh                    |   12 +
 cpp/src/arrow/CMakeLists.txt              |   33 +
 cpp/src/arrow/api.h                       |   21 +
 cpp/src/arrow/array-test.cc               |   92 +
 cpp/src/arrow/array.cc                    |   44 +
 cpp/src/arrow/array.h                     |   79 +
 cpp/src/arrow/builder.cc                  |   63 +
 cpp/src/arrow/builder.h                   |  101 +
 cpp/src/arrow/field-test.cc               |   38 +
 cpp/src/arrow/field.h                     |   48 +
 cpp/src/arrow/parquet/CMakeLists.txt      |   35 +
 cpp/src/arrow/test-util.h                 |   97 +
 cpp/src/arrow/type.cc                     |   22 +
 cpp/src/arrow/type.h                      |  180 +
 cpp/src/arrow/types/CMakeLists.txt        |   63 +
 cpp/src/arrow/types/binary.h              |   33 +
 cpp/src/arrow/types/boolean.h             |   35 +
 cpp/src/arrow/types/collection.h          |   45 +
 cpp/src/arrow/types/construct.cc          |   88 +
 cpp/src/arrow/types/construct.h           |   32 +
 cpp/src/arrow/types/datetime.h            |   79 +
 cpp/src/arrow/types/decimal.h             |   32 +
 cpp/src/arrow/types/floating.cc           |   22 +
 cpp/src/arrow/types/floating.h            |   43 +
 cpp/src/arrow/types/integer.cc            |   22 +
 cpp/src/arrow/types/integer.h             |   88 +
 cpp/src/arrow/types/json.cc               |   42 +
 cpp/src/arrow/types/json.h                |   38 +
 cpp/src/arrow/types/list-test.cc          |  166 +
 cpp/src/arrow/types/list.cc               |   31 +
 cpp/src/arrow/types/list.h                |  206 +
 cpp/src/arrow/types/null.h                |   34 +
 cpp/src/arrow/types/primitive-test.cc     |  345 ++
 cpp/src/arrow/types/primitive.cc          |   50 +
 cpp/src/arrow/types/primitive.h           |  240 +
 cpp/src/arrow/types/string-test.cc        |  242 +
 cpp/src/arrow/types/string.cc             |   40 +
 cpp/src/arrow/types/string.h              |  181 +
 cpp/src/arrow/types/struct-test.cc        |   61 +
 cpp/src/arrow/types/struct.cc             |   38 +
 cpp/src/arrow/types/struct.h              |   51 +
 cpp/src/arrow/types/test-common.h         |   50 +
 cpp/src/arrow/types/union.cc              |   49 +
 cpp/src/arrow/types/union.h               |   86 +
 cpp/src/arrow/util/CMakeLists.txt         |   81 +
 cpp/src/arrow/util/bit-util-test.cc       |   44 +
 cpp/src/arrow/util/bit-util.cc            |   46 +
 cpp/src/arrow/util/bit-util.h             |   68 +
 cpp/src/arrow/util/buffer-test.cc         |   58 +
 cpp/src/arrow/util/buffer.cc              |   53 +
 cpp/src/arrow/util/buffer.h               |  133 +
 cpp/src/arrow/util/macros.h               |   26 +
 cpp/src/arrow/util/random.h               |  128 +
 cpp/src/arrow/util/status.cc              |   38 +
 cpp/src/arrow/util/status.h               |  152 +
 cpp/src/arrow/util/test_main.cc           |   26 +
 cpp/thirdparty/build_thirdparty.sh        |   62 +
 cpp/thirdparty/download_thirdparty.sh     |   20 +
 cpp/thirdparty/versions.sh                |    3 +
 73 files changed, 12551 insertions(+)
 create mode 100644 cpp/.gitignore
 create mode 100644 cpp/CMakeLists.txt
 create mode 100644 cpp/LICENSE.txt
 create mode 100644 cpp/README.md
 create mode 100755 cpp/build-support/asan_symbolize.py
 create mode 100755 cpp/build-support/bootstrap_toolchain.py
 create mode 100755 cpp/build-support/cpplint.py
 create mode 100755 cpp/build-support/run-test.sh
 create mode 100755 cpp/build-support/stacktrace_addr2line.pl
 create mode 100644 cpp/cmake_modules/CompilerInfo.cmake
 create mode 100644 cpp/cmake_modules/FindGPerf.cmake
 create mode 100644 cpp/cmake_modules/FindGTest.cmake
 create mode 100644 cpp/cmake_modules/FindParquet.cmake
 create mode 100644 cpp/cmake_modules/san-config.cmake
 create mode 100755 cpp/setup_build_env.sh
 create mode 100644 cpp/src/arrow/CMakeLists.txt
 create mode 100644 cpp/src/arrow/api.h
 create mode 100644 cpp/src/arrow/array-test.cc
 create mode 100644 cpp/src/arrow/array.cc
 create mode 100644 cpp/src/arrow/array.h
 create mode 100644 cpp/src/arrow/builder.cc
 create mode 100644 cpp/src/arrow/builder.h
 create mode 100644 cpp/src/arrow/field-test.cc
 create mode 100644 cpp/src/arrow/field.h
 create mode 100644 cpp/src/arrow/parquet/CMakeLists.txt
 create mode 100644 cpp/src/arrow/test-util.h
 create mode 100644 cpp/src/arrow/type.cc
 create mode 100644 cpp/src/arrow/type.h
 create mode 100644 cpp/src/arrow/types/CMakeLists.txt
 create mode 100644 cpp/src/arrow/types/binary.h
 create mode 100644 cpp/src/arrow/types/boolean.h
 create mode 100644 cpp/src/arrow/types/collection.h
 create mode 100644 cpp/src/arrow/types/construct.cc
 create mode 100644 cpp/src/arrow/types/construct.h
 create mode 100644 cpp/src/arrow/types/datetime.h
 create mode 100644 cpp/src/arrow/types/decimal.h
 create mode 100644 cpp/src/arrow/types/floating.cc
 create mode 100644 cpp/src/arrow/types/floating.h
 create mode 100644 cpp/src/arrow/types/integer.cc
 create mode 100644 cpp/src/arrow/types/integer.h
 create mode 100644 cpp/src/arrow/types/json.cc
 create mode 100644 cpp/src/arrow/types/json.h
 create mode 100644 cpp/src/arrow/types/list-test.cc
 create mode 100644 cpp/src/arrow/types/list.cc
 create mode 100644 cpp/src/arrow/types/list.h
 create mode 100644 cpp/src/arrow/types/null.h
 create mode 100644 cpp/src/arrow/types/primitive-test.cc
 create mode 100644 cpp/src/arrow/types/primitive.cc
 create mode 100644 cpp/src/arrow/types/primitive.h
 create mode 100644 cpp/src/arrow/types/string-test.cc
 create mode 100644 cpp/src/arrow/types/string.cc
 create mode 100644 cpp/src/arrow/types/string.h
 create mode 100644 cpp/src/arrow/types/struct-test.cc
 create mode 100644 cpp/src/arrow/types/struct.cc
 create mode 100644 cpp/src/arrow/types/struct.h
 create mode 100644 cpp/src/arrow/types/test-common.h
 create mode 100644 cpp/src/arrow/types/union.cc
 create mode 100644 cpp/src/arrow/types/union.h
 create mode 100644 cpp/src/arrow/util/CMakeLists.txt
 create mode 100644 cpp/src/arrow/util/bit-util-test.cc
 create mode 100644 cpp/src/arrow/util/bit-util.cc
 create mode 100644 cpp/src/arrow/util/bit-util.h
 create mode 100644 cpp/src/arrow/util/buffer-test.cc
 create mode 100644 cpp/src/arrow/util/buffer.cc
 create mode 100644 cpp/src/arrow/util/buffer.h
 create mode 100644 cpp/src/arrow/util/macros.h
 create mode 100644 cpp/src/arrow/util/random.h
 create mode 100644 cpp/src/arrow/util/status.cc
 create mode 100644 cpp/src/arrow/util/status.h
 create mode 100644 cpp/src/arrow/util/test_main.cc
 create mode 100755 cpp/thirdparty/build_thirdparty.sh
 create mode 100755 cpp/thirdparty/download_thirdparty.sh
 create mode 100755 cpp/thirdparty/versions.sh

diff --git a/cpp/.gitignore b/cpp/.gitignore
new file mode 100644
index 0000000000000..ab30247d49378
--- /dev/null
+++ b/cpp/.gitignore
@@ -0,0 +1,21 @@
+thirdparty/
+CMakeFiles/
+CMakeCache.txt
+CTestTestfile.cmake
+Makefile
+cmake_install.cmake
+build/
+Testing/
+
+#########################################
+# Editor temporary/working/backup files #
+.#*
+*\#*\#
+[#]*#
+*~
+*$
+*.bak
+*flymake*
+*.kdev4
+*.log
+*.swp
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
new file mode 100644
index 0000000000000..90e55dfddbf30
--- /dev/null
+++ b/cpp/CMakeLists.txt
@@ -0,0 +1,483 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 2.7)
+project(arrow)
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
+
+include(CMakeParseArguments)
+
+set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
+set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
+
+# Allow "make install" to not depend on all targets.
+#
+# Must be declared in the top-level CMakeLists.txt.
+set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
+
+# Generate a Clang compile_commands.json "compilation database" file for use
+# with various development tools, such as Vim's YouCompleteMe plugin.
+# See http://clang.llvm.org/docs/JSONCompilationDatabase.html
+if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
+  set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
+endif()
+
+# Enable using a custom GCC toolchain to build Arrow
+if (NOT "$ENV{ARROW_GCC_ROOT}" STREQUAL "")
+  set(GCC_ROOT $ENV{ARROW_GCC_ROOT})
+  set(CMAKE_C_COMPILER ${GCC_ROOT}/bin/gcc)
+  set(CMAKE_CXX_COMPILER ${GCC_ROOT}/bin/g++)
+endif()
+
+# ----------------------------------------------------------------------
+# cmake options
+
+# Top level cmake dir
+if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
+  option(ARROW_WITH_PARQUET
+    "Build the Parquet adapter and link to libparquet"
+    OFF)
+
+  option(ARROW_BUILD_TESTS
+    "Build the Arrow googletest unit tests"
+    ON)
+endif()
+
+if(NOT ARROW_BUILD_TESTS)
+  set(NO_TESTS 1)
+endif()
+
+
+############################################################
+# Compiler flags
+############################################################
+
+# compiler flags that are common across debug/release builds
+#  - msse4.2: Enable sse4.2 compiler intrinsics.
+#  - Wall: Enable all warnings.
+#  - Wno-sign-compare: suppress warnings for comparison between signed and unsigned
+#    integers
+#  -Wno-deprecated: some of the gutil code includes old things like ext/hash_set, ignore that
+#  - pthread: enable multithreaded malloc
+#  - -D__STDC_FORMAT_MACROS: for PRI* print format macros
+#  -fno-strict-aliasing
+#     Assume programs do not follow strict aliasing rules.
+#     GCC cannot always verify whether strict aliasing rules are indeed followed due to
+#     fundamental limitations in escape analysis, which can result in subtle bad code generation.
+#     This has a small perf hit but worth it to avoid hard to debug crashes.
+set(CXX_COMMON_FLAGS "-std=c++11 -fno-strict-aliasing -msse3 -Wall -Wno-deprecated -pthread -D__STDC_FORMAT_MACROS")
+
+# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
+# For all builds:
+# For CMAKE_BUILD_TYPE=Debug
+#   -ggdb: Enable gdb debugging
+# For CMAKE_BUILD_TYPE=FastDebug
+#   Same as DEBUG, except with some optimizations on.
+# For CMAKE_BUILD_TYPE=Release
+#   -O3: Enable all compiler optimizations
+#   -g: Enable symbols for profiler tools (TODO: remove for shipping)
+set(CXX_FLAGS_DEBUG "-ggdb")
+set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
+set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
+
+set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate")
+set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use")
+
+# if no build build type is specified, default to debug builds
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Debug)
+endif(NOT CMAKE_BUILD_TYPE)
+
+string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
+
+
+# Set compile flags based on the build type.
+message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_DEBUG})
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
+  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_FASTDEBUG})
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
+  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE})
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN")
+  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_PROFILE_GEN})
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD")
+  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_PROFILE_BUILD})
+else()
+  message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
+endif ()
+
+# Add common flags
+set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
+
+# Required to avoid static linking errors with dependencies
+add_definitions(-fPIC)
+
+# Determine compiler version
+include(CompilerInfo)
+
+if ("${COMPILER_FAMILY}" STREQUAL "clang")
+  # Clang helpfully provides a few extensions from C++11 such as the 'override'
+  # keyword on methods. This doesn't change behavior, and we selectively enable
+  # it in src/gutil/port.h only on clang. So, we can safely use it, and don't want
+  # to trigger warnings when we do so.
+  # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++11-extensions")
+
+  # Using Clang with ccache causes a bunch of spurious warnings that are
+  # purportedly fixed in the next version of ccache. See the following for details:
+  #
+  #   http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html
+  #   http://petereisentraut.blogspot.com/2011/09/ccache-and-clang-part-2.html
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
+
+  # Only hardcode -fcolor-diagnostics if stderr is opened on a terminal. Otherwise
+  # the color codes show up as noisy artifacts.
+  #
+  # This test is imperfect because 'cmake' and 'make' can be run independently
+  # (with different terminal options), and we're testing during the former.
+  execute_process(COMMAND test -t 2 RESULT_VARIABLE ARROW_IS_TTY)
+  if ((${ARROW_IS_TTY} EQUAL 0) AND (NOT ("$ENV{TERM}" STREQUAL "dumb")))
+    message("Running in a controlling terminal")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcolor-diagnostics")
+  else()
+    message("Running without a controlling terminal or in a dumb terminal")
+  endif()
+
+  # Use libstdc++ and not libc++. The latter lacks support for tr1 in OSX
+  # and since 10.9 is now the default.
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++")
+endif()
+
+# Sanity check linking option.
+if (NOT ARROW_LINK)
+  set(ARROW_LINK "d")
+elseif(NOT ("auto" MATCHES "^${ARROW_LINK}" OR
+            "dynamic" MATCHES "^${ARROW_LINK}" OR
+            "static" MATCHES "^${ARROW_LINK}"))
+  message(FATAL_ERROR "Unknown value for ARROW_LINK, must be auto|dynamic|static")
+else()
+  # Remove all but the first letter.
+  string(SUBSTRING "${ARROW_LINK}" 0 1 ARROW_LINK)
+endif()
+
+# ASAN / TSAN / UBSAN
+include(san-config)
+
+# For any C code, use the same flags.
+set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
+
+# Code coverage
+if ("${ARROW_GENERATE_COVERAGE}")
+  if("${CMAKE_CXX_COMPILER}" MATCHES ".*clang.*")
+    # There appears to be some bugs in clang 3.3 which cause code coverage
+    # to have link errors, not locating the llvm_gcda_* symbols.
+    # This should be fixed in llvm 3.4 with http://llvm.org/viewvc/llvm-project?view=revision&revision=184666
+    message(SEND_ERROR "Cannot currently generate coverage with clang")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -DCOVERAGE_BUILD")
+
+  # For coverage to work properly, we need to use static linkage. Otherwise,
+  # __gcov_flush() doesn't properly flush coverage from every module.
+  # See http://stackoverflow.com/questions/28164543/using-gcov-flush-within-a-library-doesnt-force-the-other-modules-to-yield-gc
+  if("${ARROW_LINK}" STREQUAL "a")
+    message("Using static linking for coverage build")
+    set(ARROW_LINK "s")
+  elseif("${ARROW_LINK}" STREQUAL "d")
+    message(SEND_ERROR "Cannot use coverage with dynamic linking")
+  endif()
+endif()
+
+# If we still don't know what kind of linking to perform, choose based on
+# build type (developers like fast builds).
+if ("${ARROW_LINK}" STREQUAL "a")
+  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
+      "${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
+    message("Using dynamic linking for ${CMAKE_BUILD_TYPE} builds")
+    set(ARROW_LINK "d")
+  else()
+    message("Using static linking for ${CMAKE_BUILD_TYPE} builds")
+    set(ARROW_LINK "s")
+  endif()
+endif()
+
+# Are we using the gold linker? It doesn't work with dynamic linking as
+# weak symbols aren't properly overridden, causing tcmalloc to be omitted.
+# Let's flag this as an error in RELEASE builds (we shouldn't release a
+# product like this).
+#
+# See https://sourceware.org/bugzilla/show_bug.cgi?id=16979 for details.
+#
+# The gold linker is only for ELF binaries, which OSX doesn't use. We can
+# just skip.
+if (NOT APPLE)
+  execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Wl,--version OUTPUT_VARIABLE LINKER_OUTPUT)
+endif ()
+if (LINKER_OUTPUT MATCHES "gold")
+  if ("${ARROW_LINK}" STREQUAL "d" AND
+      "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
+    message(SEND_ERROR "Cannot use gold with dynamic linking in a RELEASE build "
+      "as it would cause tcmalloc symbols to get dropped")
+  else()
+    message("Using gold linker")
+  endif()
+  set(ARROW_USING_GOLD 1)
+else()
+  message("Using ld linker")
+endif()
+
+# Having set ARROW_LINK due to build type and/or sanitizer, it's now safe to
+# act on its value.
+if ("${ARROW_LINK}" STREQUAL "d")
+  set(BUILD_SHARED_LIBS ON)
+
+  # Position independent code is only necessary when producing shared objects.
+  add_definitions(-fPIC)
+endif()
+
+# set compile output directory
+string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
+
+# If build in-source, create the latest symlink. If build out-of-source, which is
+# preferred, simply output the binaries in the build folder
+if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
+  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/${BUILD_SUBDIR_NAME}/")
+  # Link build/latest to the current build directory, to avoid developers
+  # accidentally running the latest debug build when in fact they're building
+  # release builds.
+  FILE(MAKE_DIRECTORY ${BUILD_OUTPUT_ROOT_DIRECTORY})
+  if (NOT APPLE)
+    set(MORE_ARGS "-T")
+  endif()
+EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
+  ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
+else()
+  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
+endif()
+
+# where to put generated archives (.a files)
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
+# where to put generated libraries (.so files)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
+# where to put generated binaries
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+include_directories(src)
+
+############################################################
+# Visibility
+############################################################
+# For generate_export_header() and add_compiler_export_flags().
+include(GenerateExportHeader)
+
+############################################################
+# Testing
+############################################################
+
+# Add a new test case, with or without an executable that should be built.
+#
+# REL_TEST_NAME is the name of the test. It may be a single component
+# (e.g. monotime-test) or contain additional components (e.g.
+# net/net_util-test). Either way, the last component must be a globally
+# unique name.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+function(ADD_ARROW_TEST REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
+    # This test has a corresponding .cc file, set it up as an executable.
+    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+    add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
+    target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+  else()
+    # No executable, just invoke the test (probably a script) directly.
+    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
+  endif()
+
+  add_test(${TEST_NAME}
+    ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
+  if(ARGN)
+    set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
+  endif()
+endfunction()
+
+# A wrapper for add_dependencies() that is compatible with NO_TESTS.
+function(ADD_ARROW_TEST_DEPENDENCIES REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  add_dependencies(${TEST_NAME} ${ARGN})
+endfunction()
+
+enable_testing()
+
+############################################################
+# Dependencies
+############################################################
+function(ADD_THIRDPARTY_LIB LIB_NAME)
+  set(options)
+  set(one_value_args SHARED_LIB STATIC_LIB)
+  set(multi_value_args DEPS)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(("${ARROW_LINK}" STREQUAL "s" AND ARG_STATIC_LIB) OR (NOT ARG_SHARED_LIB))
+    if(NOT ARG_STATIC_LIB)
+      message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
+    endif()
+    add_library(${LIB_NAME} STATIC IMPORTED)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
+  else()
+    add_library(${LIB_NAME} SHARED IMPORTED)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+  endif()
+
+  if(ARG_DEPS)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+  endif()
+endfunction()
+
+## GTest
+if ("$ENV{GTEST_HOME}" STREQUAL "")
+  set(GTest_HOME ${THIRDPARTY_DIR}/googletest-release-1.7.0)
+endif()
+find_package(GTest REQUIRED)
+include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(gtest
+  STATIC_LIB ${GTEST_STATIC_LIB})
+
+## Google PerfTools
+##
+## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment
+## near definition of ARROW_USING_GOLD).
+# find_package(GPerf REQUIRED)
+# if (NOT "${ARROW_USE_ASAN}" AND
+#     NOT "${ARROW_USE_TSAN}" AND
+#     NOT ("${ARROW_USING_GOLD}" AND "${ARROW_LINK}" STREQUAL "d"))
+#   ADD_THIRDPARTY_LIB(tcmalloc
+#     STATIC_LIB "${TCMALLOC_STATIC_LIB}"
+#     SHARED_LIB "${TCMALLOC_SHARED_LIB}")
+#   ADD_THIRDPARTY_LIB(profiler
+#     STATIC_LIB "${PROFILER_STATIC_LIB}"
+#     SHARED_LIB "${PROFILER_SHARED_LIB}")
+#   list(APPEND ARROW_BASE_LIBS tcmalloc profiler)
+#   add_definitions("-DTCMALLOC_ENABLED")
+#   set(ARROW_TCMALLOC_AVAILABLE 1)
+# endif()
+
+############################################################
+# Linker setup
+############################################################
+set(ARROW_MIN_TEST_LIBS arrow arrow_test_main arrow_test_util ${ARROW_BASE_LIBS})
+set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
+
+############################################################
+# "make ctags" target
+############################################################
+if (UNIX)
+  add_custom_target(ctags ctags -R --languages=c++,c)
+endif (UNIX)
+
+############################################################
+# "make etags" target
+############################################################
+if (UNIX)
+  add_custom_target(tags etags --members --declarations
+  `find ${CMAKE_CURRENT_SOURCE_DIR}/src
+   -name \\*.cc -or -name \\*.hh -or -name \\*.cpp -or -name \\*.h -or -name \\*.c -or
+   -name \\*.f`)
+  add_custom_target(etags DEPENDS tags)
+endif (UNIX)
+
+############################################################
+# "make cscope" target
+############################################################
+if (UNIX)
+  add_custom_target(cscope find ${CMAKE_CURRENT_SOURCE_DIR}
+  ( -name \\*.cc -or -name \\*.hh -or -name \\*.cpp -or
+    -name \\*.h -or -name \\*.c -or -name \\*.f )
+  -exec echo \"{}\" \; > cscope.files && cscope -q -b VERBATIM)
+endif (UNIX)
+
+############################################################
+# "make lint" target
+############################################################
+if (UNIX)
+  # Full lint
+  add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
+  --verbose=2
+  --linelength=90
+  --filter=-whitespace/comments,-readability/todo,-build/header_guard
+    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h`)
+endif (UNIX)
+
+#----------------------------------------------------------------------
+# Parquet adapter
+
+if(ARROW_WITH_PARQUET)
+  find_package(Parquet REQUIRED)
+  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(parquet
+    STATIC_LIB ${PARQUET_STATIC_LIB}
+    SHARED_LIB ${PARQUET_SHARED_LIB})
+
+  add_subdirectory(src/arrow/parquet)
+  list(APPEND LINK_LIBS arrow_parquet parquet)
+endif()
+
+############################################################
+# Subdirectories
+############################################################
+
+add_subdirectory(src/arrow)
+add_subdirectory(src/arrow/util)
+add_subdirectory(src/arrow/types)
+
+set(LINK_LIBS
+  arrow_util
+  arrow_types)
+
+set(ARROW_SRCS
+  src/arrow/array.cc
+  src/arrow/builder.cc
+  src/arrow/type.cc
+)
+
+add_library(arrow SHARED
+  ${ARROW_SRCS}
+)
+target_link_libraries(arrow ${LINK_LIBS})
+set_target_properties(arrow PROPERTIES LINKER_LANGUAGE CXX)
+
+install(TARGETS arrow
+  LIBRARY DESTINATION lib)
diff --git a/cpp/LICENSE.txt b/cpp/LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/cpp/LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/cpp/README.md b/cpp/README.md
new file mode 100644
index 0000000000000..378dc4e28de76
--- /dev/null
+++ b/cpp/README.md
@@ -0,0 +1,48 @@
+# Arrow C++
+
+## Setup Build Environment
+
+Arrow uses CMake as a build configuration system. Currently, it supports in-source and
+out-of-source builds with the latter one being preferred.
+
+Arrow requires a C++11-enabled compiler. On Linux, gcc 4.8 and higher should be
+sufficient.
+
+To build the thirdparty build dependencies, run:
+
+```
+./thirdparty/download_thirdparty.sh
+./thirdparty/build_thirdparty.sh
+```
+
+You can also run from the root of the C++ tree
+
+```
+source setup_build_env.sh
+```
+
+Arrow is configured to use the `thirdparty` directory by default for its build
+dependencies. To set up a custom toolchain see below.
+
+Simple debug build:
+
+    mkdir debug
+    cd debug
+    cmake ..
+    make
+    ctest
+
+Simple release build:
+
+    mkdir release
+    cd release
+    cmake .. -DCMAKE_BUILD_TYPE=Release
+    make
+    ctest
+
+### Third-party environment variables
+
+To set up your own specific build toolchain, here are the relevant environment
+variables
+
+* Googletest: `GTEST_HOME` (only required to build the unit tests)
diff --git a/cpp/build-support/asan_symbolize.py b/cpp/build-support/asan_symbolize.py
new file mode 100755
index 0000000000000..839a1984bd349
--- /dev/null
+++ b/cpp/build-support/asan_symbolize.py
@@ -0,0 +1,360 @@
+#!/usr/bin/env python
+#===- lib/asan/scripts/asan_symbolize.py -----------------------------------===#
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===------------------------------------------------------------------------===#
+import bisect
+import os
+import re
+import subprocess
+import sys
+
+llvm_symbolizer = None
+symbolizers = {}
+filetypes = {}
+vmaddrs = {}
+DEBUG = False
+
+
+# FIXME: merge the code that calls fix_filename().
+def fix_filename(file_name):
+  for path_to_cut in sys.argv[1:]:
+    file_name = re.sub('.*' + path_to_cut, '', file_name)
+  file_name = re.sub('.*asan_[a-z_]*.cc:[0-9]*', '_asan_rtl_', file_name)
+  file_name = re.sub('.*crtstuff.c:0', '???:0', file_name)
+  return file_name
+
+
+class Symbolizer(object):
+  def __init__(self):
+    pass
+
+  def symbolize(self, addr, binary, offset):
+    """Symbolize the given address (pair of binary and offset).
+
+    Overriden in subclasses.
+    Args:
+        addr: virtual address of an instruction.
+        binary: path to executable/shared object containing this instruction.
+        offset: instruction offset in the @binary.
+    Returns:
+        list of strings (one string for each inlined frame) describing
+        the code locations for this instruction (that is, function name, file
+        name, line and column numbers).
+    """
+    return None
+
+
+class LLVMSymbolizer(Symbolizer):
+  def __init__(self, symbolizer_path):
+    super(LLVMSymbolizer, self).__init__()
+    self.symbolizer_path = symbolizer_path
+    self.pipe = self.open_llvm_symbolizer()
+
+  def open_llvm_symbolizer(self):
+    if not os.path.exists(self.symbolizer_path):
+      return None
+    cmd = [self.symbolizer_path,
+           '--use-symbol-table=true',
+           '--demangle=false',
+           '--functions=true',
+           '--inlining=true']
+    if DEBUG:
+      print ' '.join(cmd)
+    return subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE)
+
+  def symbolize(self, addr, binary, offset):
+    """Overrides Symbolizer.symbolize."""
+    if not self.pipe:
+      return None
+    result = []
+    try:
+      symbolizer_input = '%s %s' % (binary, offset)
+      if DEBUG:
+        print symbolizer_input
+      print >> self.pipe.stdin, symbolizer_input
+      while True:
+        function_name = self.pipe.stdout.readline().rstrip()
+        if not function_name:
+          break
+        file_name = self.pipe.stdout.readline().rstrip()
+        file_name = fix_filename(file_name)
+        if (not function_name.startswith('??') and
+            not file_name.startswith('??')):
+          # Append only valid frames.
+          result.append('%s in %s %s' % (addr, function_name,
+                                         file_name))
+    except Exception:
+      result = []
+    if not result:
+      result = None
+    return result
+
+
+def LLVMSymbolizerFactory(system):
+  symbolizer_path = os.getenv('LLVM_SYMBOLIZER_PATH')
+  if not symbolizer_path:
+    # Assume llvm-symbolizer is in PATH.
+    symbolizer_path = 'llvm-symbolizer'
+  return LLVMSymbolizer(symbolizer_path)
+
+
+class Addr2LineSymbolizer(Symbolizer):
+  def __init__(self, binary):
+    super(Addr2LineSymbolizer, self).__init__()
+    self.binary = binary
+    self.pipe = self.open_addr2line()
+
+  def open_addr2line(self):
+    cmd = ['addr2line', '-f', '-e', self.binary]
+    if DEBUG:
+      print ' '.join(cmd)
+    return subprocess.Popen(cmd,
+                            stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+  def symbolize(self, addr, binary, offset):
+    """Overrides Symbolizer.symbolize."""
+    if self.binary != binary:
+      return None
+    try:
+      print >> self.pipe.stdin, offset
+      function_name = self.pipe.stdout.readline().rstrip()
+      file_name = self.pipe.stdout.readline().rstrip()
+    except Exception:
+      function_name = ''
+      file_name = ''
+    file_name = fix_filename(file_name)
+    return ['%s in %s %s' % (addr, function_name, file_name)]
+
+
+class DarwinSymbolizer(Symbolizer):
+  def __init__(self, addr, binary):
+    super(DarwinSymbolizer, self).__init__()
+    self.binary = binary
+    # Guess which arch we're running. 10 = len('0x') + 8 hex digits.
+    if len(addr) > 10:
+      self.arch = 'x86_64'
+    else:
+      self.arch = 'i386'
+    self.vmaddr = None
+    self.pipe = None
+
+  def write_addr_to_pipe(self, offset):
+    print >> self.pipe.stdin, '0x%x' % int(offset, 16)
+
+  def open_atos(self):
+    if DEBUG:
+      print 'atos -o %s -arch %s' % (self.binary, self.arch)
+    cmdline = ['atos', '-o', self.binary, '-arch', self.arch]
+    self.pipe = subprocess.Popen(cmdline,
+                                 stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
+
+  def symbolize(self, addr, binary, offset):
+    """Overrides Symbolizer.symbolize."""
+    if self.binary != binary:
+      return None
+    self.open_atos()
+    self.write_addr_to_pipe(offset)
+    self.pipe.stdin.close()
+    atos_line = self.pipe.stdout.readline().rstrip()
+    # A well-formed atos response looks like this:
+    #   foo(type1, type2) (in object.name) (filename.cc:80)
+    match = re.match('^(.*) \(in (.*)\) \((.*:\d*)\)$', atos_line)
+    if DEBUG:
+      print 'atos_line: ', atos_line
+    if match:
+      function_name = match.group(1)
+      function_name = re.sub('\(.*?\)', '', function_name)
+      file_name = fix_filename(match.group(3))
+      return ['%s in %s %s' % (addr, function_name, file_name)]
+    else:
+      return ['%s in %s' % (addr, atos_line)]
+
+
+# Chain several symbolizers so that if one symbolizer fails, we fall back
+# to the next symbolizer in chain.
+class ChainSymbolizer(Symbolizer):
+  def __init__(self, symbolizer_list):
+    super(ChainSymbolizer, self).__init__()
+    self.symbolizer_list = symbolizer_list
+
+  def symbolize(self, addr, binary, offset):
+    """Overrides Symbolizer.symbolize."""
+    for symbolizer in self.symbolizer_list:
+      if symbolizer:
+        result = symbolizer.symbolize(addr, binary, offset)
+        if result:
+          return result
+    return None
+
+  def append_symbolizer(self, symbolizer):
+    self.symbolizer_list.append(symbolizer)
+
+
+def BreakpadSymbolizerFactory(binary):
+  suffix = os.getenv('BREAKPAD_SUFFIX')
+  if suffix:
+    filename = binary + suffix
+    if os.access(filename, os.F_OK):
+      return BreakpadSymbolizer(filename)
+  return None
+
+
+def SystemSymbolizerFactory(system, addr, binary):
+  if system == 'Darwin':
+    return DarwinSymbolizer(addr, binary)
+  elif system == 'Linux':
+    return Addr2LineSymbolizer(binary)
+
+
+class BreakpadSymbolizer(Symbolizer):
+  def __init__(self, filename):
+    super(BreakpadSymbolizer, self).__init__()
+    self.filename = filename
+    lines = file(filename).readlines()
+    self.files = []
+    self.symbols = {}
+    self.address_list = []
+    self.addresses = {}
+    # MODULE mac x86_64 A7001116478B33F18FF9BEDE9F615F190 t
+    fragments = lines[0].rstrip().split()
+    self.arch = fragments[2]
+    self.debug_id = fragments[3]
+    self.binary = ' '.join(fragments[4:])
+    self.parse_lines(lines[1:])
+
+  def parse_lines(self, lines):
+    cur_function_addr = ''
+    for line in lines:
+      fragments = line.split()
+      if fragments[0] == 'FILE':
+        assert int(fragments[1]) == len(self.files)
+        self.files.append(' '.join(fragments[2:]))
+      elif fragments[0] == 'PUBLIC':
+        self.symbols[int(fragments[1], 16)] = ' '.join(fragments[3:])
+      elif fragments[0] in ['CFI', 'STACK']:
+        pass
+      elif fragments[0] == 'FUNC':
+        cur_function_addr = int(fragments[1], 16)
+        if not cur_function_addr in self.symbols.keys():
+          self.symbols[cur_function_addr] = ' '.join(fragments[4:])
+      else:
+        # Line starting with an address.
+        addr = int(fragments[0], 16)
+        self.address_list.append(addr)
+        # Tuple of symbol address, size, line, file number.
+        self.addresses[addr] = (cur_function_addr,
+                                int(fragments[1], 16),
+                                int(fragments[2]),
+                                int(fragments[3]))
+    self.address_list.sort()
+
+  def get_sym_file_line(self, addr):
+    key = None
+    if addr in self.addresses.keys():
+      key = addr
+    else:
+      index = bisect.bisect_left(self.address_list, addr)
+      if index == 0:
+        return None
+      else:
+        key = self.address_list[index - 1]
+    sym_id, size, line_no, file_no = self.addresses[key]
+    symbol = self.symbols[sym_id]
+    filename = self.files[file_no]
+    if addr < key + size:
+      return symbol, filename, line_no
+    else:
+      return None
+
+  def symbolize(self, addr, binary, offset):
+    if self.binary != binary:
+      return None
+    res = self.get_sym_file_line(int(offset, 16))
+    if res:
+      function_name, file_name, line_no = res
+      result = ['%s in %s %s:%d' % (
+          addr, function_name, file_name, line_no)]
+      print result
+      return result
+    else:
+      return None
+
+
+class SymbolizationLoop(object):
+  def __init__(self, binary_name_filter=None):
+    # Used by clients who may want to supply a different binary name.
+    # E.g. in Chrome several binaries may share a single .dSYM.
+    self.binary_name_filter = binary_name_filter
+    self.system = os.uname()[0]
+    if self.system in ['Linux', 'Darwin']:
+      self.llvm_symbolizer = LLVMSymbolizerFactory(self.system)
+    else:
+      raise Exception('Unknown system')
+
+  def symbolize_address(self, addr, binary, offset):
+    # Use the chain of symbolizers:
+    # Breakpad symbolizer -> LLVM symbolizer -> addr2line/atos
+    # (fall back to next symbolizer if the previous one fails).
+    if not binary in symbolizers:
+      symbolizers[binary] = ChainSymbolizer(
+          [BreakpadSymbolizerFactory(binary), self.llvm_symbolizer])
+    result = symbolizers[binary].symbolize(addr, binary, offset)
+    if result is None:
+      # Initialize system symbolizer only if other symbolizers failed.
+      symbolizers[binary].append_symbolizer(
+          SystemSymbolizerFactory(self.system, addr, binary))
+      result = symbolizers[binary].symbolize(addr, binary, offset)
+    # The system symbolizer must produce some result.
+    assert result
+    return result
+
+  def print_symbolized_lines(self, symbolized_lines):
+    if not symbolized_lines:
+      print self.current_line
+    else:
+      for symbolized_frame in symbolized_lines:
+        print '    #' + str(self.frame_no) + ' ' + symbolized_frame.rstrip()
+        self.frame_no += 1
+
+  def process_stdin(self):
+    self.frame_no = 0
+    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+
+    while True:
+      line = sys.stdin.readline()
+      if not line: break
+      self.current_line = line.rstrip()
+      #0 0x7f6e35cf2e45  (/blah/foo.so+0x11fe45)
+      stack_trace_line_format = (
+          '^( *#([0-9]+) *)(0x[0-9a-f]+) *\((.*)\+(0x[0-9a-f]+)\)')
+      match = re.match(stack_trace_line_format, line)
+      if not match:
+        print self.current_line
+        continue
+      if DEBUG:
+        print line
+      _, frameno_str, addr, binary, offset = match.groups()
+      if frameno_str == '0':
+        # Assume that frame #0 is the first frame of new stack trace.
+        self.frame_no = 0
+      original_binary = binary
+      if self.binary_name_filter:
+        binary = self.binary_name_filter(binary)
+      symbolized_line = self.symbolize_address(addr, binary, offset)
+      if not symbolized_line:
+        if original_binary != binary:
+          symbolized_line = self.symbolize_address(addr, binary, offset)
+      self.print_symbolized_lines(symbolized_line)
+
+
+if __name__ == '__main__':
+  loop = SymbolizationLoop()
+  loop.process_stdin()
diff --git a/cpp/build-support/bootstrap_toolchain.py b/cpp/build-support/bootstrap_toolchain.py
new file mode 100755
index 0000000000000..128be78bbacc9
--- /dev/null
+++ b/cpp/build-support/bootstrap_toolchain.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# Copyright (c) 2015, Cloudera, inc.
+# Confidential Cloudera Information: Covered by NDA.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Bootstrapping the native toolchain with prebuilt binaries
+#
+# The purpose of this script is to download prebuilt artifacts of the native toolchain to
+# satisfy the third-party dependencies. The script checks for the presence of
+# NATIVE_TOOLCHAIN. NATIVE_TOOLCHAIN indicates the location where the prebuilt artifacts
+# should be extracted to.
+#
+# The script is called as follows without any additional parameters:
+#
+#     python bootstrap_toolchain.py
+import sh
+import os
+import sys
+import re
+
+HOST = "https://native-toolchain.s3.amazonaws.com/build"
+
+OS_MAPPING = {
+  "centos6" : "ec2-package-centos-6",
+  "centos5" : "ec2-package-centos-5",
+  "centos7" : "ec2-package-centos-7",
+  "debian6" : "ec2-package-debian-6",
+  "debian7" : "ec2-package-debian-7",
+  "suselinux11": "ec2-package-sles-11",
+  "ubuntu12.04" : "ec2-package-ubuntu-12-04",
+  "ubuntu14.04" : "ec2-package-ubuntu-14-04"
+}
+
+def get_release_label():
+  """Gets the right package label from the OS version"""
+  release = "".join(map(lambda x: x.lower(), sh.lsb_release("-irs").split()))
+  for k, v in OS_MAPPING.iteritems():
+    if re.search(k, release):
+      return v
+
+  print("Pre-built toolchain archives not available for your platform.")
+  print("Clone and build native toolchain from source using this repository:")
+  print("    https://github.com/cloudera/native-toolchain")
+  raise Exception("Could not find package label for OS version: {0}.".format(release))
+
+def download_package(destination, product, version, compiler):
+  label = get_release_label()
+  file_name = "{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
+  url_path="/{0}/{1}-{2}/{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
+  download_path = HOST + url_path
+
+  print "URL {0}".format(download_path)
+  print "Downloading {0} to {1}".format(file_name, destination)
+  # --no-clobber avoids downloading the file if a file with the name already exists
+  sh.wget(download_path, directory_prefix=destination, no_clobber=True)
+  print "Extracting {0}".format(file_name)
+  sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
+  sh.rm(os.path.join(destination, file_name))
+
+
+def bootstrap(packages):
+  """Validates the presence of $NATIVE_TOOLCHAIN in the environment. By checking
+  $NATIVE_TOOLCHAIN is present, we assume that {LIB}_VERSION will be present as well. Will
+  create the directory specified by $NATIVE_TOOLCHAIN if it does not yet exist. Each of
+  the packages specified in `packages` is downloaded and extracted into $NATIVE_TOOLCHAIN.
+  """
+  # Create the destination directory if necessary
+  destination = os.getenv("NATIVE_TOOLCHAIN")
+  if not destination:
+    print("Build environment not set up correctly, make sure "
+          "$NATIVE_TOOLCHAIN is present.")
+    sys.exit(1)
+
+  if not os.path.exists(destination):
+    os.makedirs(destination)
+
+  # Detect the compiler
+  if "SYSTEM_GCC" in os.environ:
+    compiler = "gcc-system"
+  else:
+    compiler = "gcc-{0}".format(os.environ["GCC_VERSION"])
+
+  for p in packages:
+    pkg_name, pkg_version = unpack_name_and_version(p)
+    download_package(destination, pkg_name, pkg_version, compiler)
+
+def unpack_name_and_version(package):
+  """A package definition is either a string where the version is fetched from the
+  environment or a tuple where the package name and the package version are fully
+  specified.
+  """
+  if isinstance(package, basestring):
+    env_var = "{0}_VERSION".format(package).replace("-", "_").upper()
+    try:
+      return package, os.environ[env_var]
+    except KeyError:
+      raise Exception("Could not find version for {0} in environment var {1}".format(
+        package, env_var))
+  return package[0], package[1]
+
+if __name__ == "__main__":
+  packages = [("gcc","4.9.2"), ("gflags", "2.0"), ("glog", "0.3.3-p1"),
+              ("gperftools", "2.3"), ("libunwind", "1.1"), ("googletest", "20151222")]
+  bootstrap(packages)
diff --git a/cpp/build-support/cpplint.py b/cpp/build-support/cpplint.py
new file mode 100755
index 0000000000000..ccc25d4c56b1a
--- /dev/null
+++ b/cpp/build-support/cpplint.py
@@ -0,0 +1,6323 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2009 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Does google-lint on c++ files.
+
+The goal of this script is to identify places in the code that *may*
+be in non-compliance with google style.  It does not attempt to fix
+up these problems -- the point is to educate.  It does also not
+attempt to find all problems, or to ensure that everything it does
+find is legitimately a problem.
+
+In particular, we can get very confused by /* and // inside strings!
+We do a small hack, which is to ignore //'s with "'s after them on the
+same line, but it is far from perfect (in either direction).
+"""
+
+import codecs
+import copy
+import getopt
+import math  # for log
+import os
+import re
+import sre_compile
+import string
+import sys
+import unicodedata
+
+
+_USAGE = """
+Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...]
+                   [--counting=total|toplevel|detailed] [--root=subdir]
+                   [--linelength=digits]
+        <file> [file] ...
+
+  The style guidelines this tries to follow are those in
+    http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
+
+  Every problem is given a confidence score from 1-5, with 5 meaning we are
+  certain of the problem, and 1 meaning it could be a legitimate construct.
+  This will miss some errors, and is not a substitute for a code review.
+
+  To suppress false-positive errors of a certain category, add a
+  'NOLINT(category)' comment to the line.  NOLINT or NOLINT(*)
+  suppresses errors of all categories on that line.
+
+  The files passed in will be linted; at least one file must be provided.
+  Default linted extensions are .cc, .cpp, .cu, .cuh and .h.  Change the
+  extensions with the --extensions flag.
+
+  Flags:
+
+    output=vs7
+      By default, the output is formatted to ease emacs parsing.  Visual Studio
+      compatible output (vs7) may also be used.  Other formats are unsupported.
+
+    verbose=#
+      Specify a number 0-5 to restrict errors to certain verbosity levels.
+
+    filter=-x,+y,...
+      Specify a comma-separated list of category-filters to apply: only
+      error messages whose category names pass the filters will be printed.
+      (Category names are printed with the message and look like
+      "[whitespace/indent]".)  Filters are evaluated left to right.
+      "-FOO" and "FOO" means "do not print categories that start with FOO".
+      "+FOO" means "do print categories that start with FOO".
+
+      Examples: --filter=-whitespace,+whitespace/braces
+                --filter=whitespace,runtime/printf,+runtime/printf_format
+                --filter=-,+build/include_what_you_use
+
+      To see a list of all the categories used in cpplint, pass no arg:
+         --filter=
+
+    counting=total|toplevel|detailed
+      The total number of errors found is always printed. If
+      'toplevel' is provided, then the count of errors in each of
+      the top-level categories like 'build' and 'whitespace' will
+      also be printed. If 'detailed' is provided, then a count
+      is provided for each category like 'build/class'.
+
+    root=subdir
+      The root directory used for deriving header guard CPP variable.
+      By default, the header guard CPP variable is calculated as the relative
+      path to the directory that contains .git, .hg, or .svn.  When this flag
+      is specified, the relative path is calculated from the specified
+      directory. If the specified directory does not exist, this flag is
+      ignored.
+
+      Examples:
+        Assuming that src/.git exists, the header guard CPP variables for
+        src/chrome/browser/ui/browser.h are:
+
+        No flag => CHROME_BROWSER_UI_BROWSER_H_
+        --root=chrome => BROWSER_UI_BROWSER_H_
+        --root=chrome/browser => UI_BROWSER_H_
+
+    linelength=digits
+      This is the allowed line length for the project. The default value is
+      80 characters.
+
+      Examples:
+        --linelength=120
+
+    extensions=extension,extension,...
+      The allowed file extensions that cpplint will check
+
+      Examples:
+        --extensions=hpp,cpp
+
+    cpplint.py supports per-directory configurations specified in CPPLINT.cfg
+    files. CPPLINT.cfg file can contain a number of key=value pairs.
+    Currently the following options are supported:
+
+      set noparent
+      filter=+filter1,-filter2,...
+      exclude_files=regex
+      linelength=80
+
+    "set noparent" option prevents cpplint from traversing directory tree
+    upwards looking for more .cfg files in parent directories. This option
+    is usually placed in the top-level project directory.
+
+    The "filter" option is similar in function to --filter flag. It specifies
+    message filters in addition to the |_DEFAULT_FILTERS| and those specified
+    through --filter command-line flag.
+
+    "exclude_files" allows to specify a regular expression to be matched against
+    a file name. If the expression matches, the file is skipped and not run
+    through liner.
+
+    "linelength" allows to specify the allowed line length for the project.
+
+    CPPLINT.cfg has an effect on files in the same directory and all
+    sub-directories, unless overridden by a nested configuration file.
+
+      Example file:
+        filter=-build/include_order,+build/include_alpha
+        exclude_files=.*\.cc
+
+    The above example disables build/include_order warning and enables
+    build/include_alpha as well as excludes all .cc from being
+    processed by linter, in the current directory (where the .cfg
+    file is located) and all sub-directories.
+"""
+
+# We categorize each error message we print.  Here are the categories.
+# We want an explicit list so we can list them all in cpplint --filter=.
+# If you add a new error message with a new category, add it to the list
+# here!  cpplint_unittest.py should tell you if you forget to do this.
+_ERROR_CATEGORIES = [
+    'build/class',
+    'build/c++11',
+    'build/deprecated',
+    'build/endif_comment',
+    'build/explicit_make_pair',
+    'build/forward_decl',
+    'build/header_guard',
+    'build/include',
+    'build/include_alpha',
+    'build/include_order',
+    'build/include_what_you_use',
+    'build/namespaces',
+    'build/printf_format',
+    'build/storage_class',
+    'legal/copyright',
+    'readability/alt_tokens',
+    'readability/braces',
+    'readability/casting',
+    'readability/check',
+    'readability/constructors',
+    'readability/fn_size',
+    'readability/function',
+    'readability/inheritance',
+    'readability/multiline_comment',
+    'readability/multiline_string',
+    'readability/namespace',
+    'readability/nolint',
+    'readability/nul',
+    'readability/strings',
+    'readability/todo',
+    'readability/utf8',
+    'runtime/arrays',
+    'runtime/casting',
+    'runtime/explicit',
+    'runtime/int',
+    'runtime/init',
+    'runtime/invalid_increment',
+    'runtime/member_string_references',
+    'runtime/memset',
+    'runtime/indentation_namespace',
+    'runtime/operator',
+    'runtime/printf',
+    'runtime/printf_format',
+    'runtime/references',
+    'runtime/string',
+    'runtime/threadsafe_fn',
+    'runtime/vlog',
+    'whitespace/blank_line',
+    'whitespace/braces',
+    'whitespace/comma',
+    'whitespace/comments',
+    'whitespace/empty_conditional_body',
+    'whitespace/empty_loop_body',
+    'whitespace/end_of_line',
+    'whitespace/ending_newline',
+    'whitespace/forcolon',
+    'whitespace/indent',
+    'whitespace/line_length',
+    'whitespace/newline',
+    'whitespace/operators',
+    'whitespace/parens',
+    'whitespace/semicolon',
+    'whitespace/tab',
+    'whitespace/todo',
+    ]
+
+# These error categories are no longer enforced by cpplint, but for backwards-
+# compatibility they may still appear in NOLINT comments.
+_LEGACY_ERROR_CATEGORIES = [
+    'readability/streams',
+    ]
+
+# The default state of the category filter. This is overridden by the --filter=
+# flag. By default all errors are on, so only add here categories that should be
+# off by default (i.e., categories that must be enabled by the --filter= flags).
+# All entries here should start with a '-' or '+', as in the --filter= flag.
+_DEFAULT_FILTERS = ['-build/include_alpha']
+
+# We used to check for high-bit characters, but after much discussion we
+# decided those were OK, as long as they were in UTF-8 and didn't represent
+# hard-coded international strings, which belong in a separate i18n file.
+
+# C++ headers
+_CPP_HEADERS = frozenset([
+    # Legacy
+    'algobase.h',
+    'algo.h',
+    'alloc.h',
+    'builtinbuf.h',
+    'bvector.h',
+    'complex.h',
+    'defalloc.h',
+    'deque.h',
+    'editbuf.h',
+    'fstream.h',
+    'function.h',
+    'hash_map',
+    'hash_map.h',
+    'hash_set',
+    'hash_set.h',
+    'hashtable.h',
+    'heap.h',
+    'indstream.h',
+    'iomanip.h',
+    'iostream.h',
+    'istream.h',
+    'iterator.h',
+    'list.h',
+    'map.h',
+    'multimap.h',
+    'multiset.h',
+    'ostream.h',
+    'pair.h',
+    'parsestream.h',
+    'pfstream.h',
+    'procbuf.h',
+    'pthread_alloc',
+    'pthread_alloc.h',
+    'rope',
+    'rope.h',
+    'ropeimpl.h',
+    'set.h',
+    'slist',
+    'slist.h',
+    'stack.h',
+    'stdiostream.h',
+    'stl_alloc.h',
+    'stl_relops.h',
+    'streambuf.h',
+    'stream.h',
+    'strfile.h',
+    'strstream.h',
+    'tempbuf.h',
+    'tree.h',
+    'type_traits.h',
+    'vector.h',
+    # 17.6.1.2 C++ library headers
+    'algorithm',
+    'array',
+    'atomic',
+    'bitset',
+    'chrono',
+    'codecvt',
+    'complex',
+    'condition_variable',
+    'deque',
+    'exception',
+    'forward_list',
+    'fstream',
+    'functional',
+    'future',
+    'initializer_list',
+    'iomanip',
+    'ios',
+    'iosfwd',
+    'iostream',
+    'istream',
+    'iterator',
+    'limits',
+    'list',
+    'locale',
+    'map',
+    'memory',
+    'mutex',
+    'new',
+    'numeric',
+    'ostream',
+    'queue',
+    'random',
+    'ratio',
+    'regex',
+    'set',
+    'sstream',
+    'stack',
+    'stdexcept',
+    'streambuf',
+    'string',
+    'strstream',
+    'system_error',
+    'thread',
+    'tuple',
+    'typeindex',
+    'typeinfo',
+    'type_traits',
+    'unordered_map',
+    'unordered_set',
+    'utility',
+    'valarray',
+    'vector',
+    # 17.6.1.2 C++ headers for C library facilities
+    'cassert',
+    'ccomplex',
+    'cctype',
+    'cerrno',
+    'cfenv',
+    'cfloat',
+    'cinttypes',
+    'ciso646',
+    'climits',
+    'clocale',
+    'cmath',
+    'csetjmp',
+    'csignal',
+    'cstdalign',
+    'cstdarg',
+    'cstdbool',
+    'cstddef',
+    'cstdint',
+    'cstdio',
+    'cstdlib',
+    'cstring',
+    'ctgmath',
+    'ctime',
+    'cuchar',
+    'cwchar',
+    'cwctype',
+    ])
+
+
+# These headers are excluded from [build/include] and [build/include_order]
+# checks:
+# - Anything not following google file name conventions (containing an
+#   uppercase character, such as Python.h or nsStringAPI.h, for example).
+# - Lua headers.
+_THIRD_PARTY_HEADERS_PATTERN = re.compile(
+    r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$')
+
+
+# Assertion macros.  These are defined in base/logging.h and
+# testing/base/gunit.h.  Note that the _M versions need to come first
+# for substring matching to work.
+_CHECK_MACROS = [
+    'DCHECK', 'CHECK',
+    'EXPECT_TRUE_M', 'EXPECT_TRUE',
+    'ASSERT_TRUE_M', 'ASSERT_TRUE',
+    'EXPECT_FALSE_M', 'EXPECT_FALSE',
+    'ASSERT_FALSE_M', 'ASSERT_FALSE',
+    ]
+
+# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE
+_CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS])
+
+for op, replacement in [('==', 'EQ'), ('!=', 'NE'),
+                        ('>=', 'GE'), ('>', 'GT'),
+                        ('<=', 'LE'), ('<', 'LT')]:
+  _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement
+  _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement
+  _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement
+  _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement
+  _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement
+  _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement
+
+for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'),
+                            ('>=', 'LT'), ('>', 'LE'),
+                            ('<=', 'GT'), ('<', 'GE')]:
+  _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement
+  _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement
+  _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement
+  _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement
+
+# Alternative tokens and their replacements.  For full list, see section 2.5
+# Alternative tokens [lex.digraph] in the C++ standard.
+#
+# Digraphs (such as '%:') are not included here since it's a mess to
+# match those on a word boundary.
+_ALT_TOKEN_REPLACEMENT = {
+    'and': '&&',
+    'bitor': '|',
+    'or': '||',
+    'xor': '^',
+    'compl': '~',
+    'bitand': '&',
+    'and_eq': '&=',
+    'or_eq': '|=',
+    'xor_eq': '^=',
+    'not': '!',
+    'not_eq': '!='
+    }
+
+# Compile regular expression that matches all the above keywords.  The "[ =()]"
+# bit is meant to avoid matching these keywords outside of boolean expressions.
+#
+# False positives include C-style multi-line comments and multi-line strings
+# but those have always been troublesome for cpplint.
+_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile(
+    r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)')
+
+
+# These constants define types of headers for use with
+# _IncludeState.CheckNextIncludeOrder().
+_C_SYS_HEADER = 1
+_CPP_SYS_HEADER = 2
+_LIKELY_MY_HEADER = 3
+_POSSIBLE_MY_HEADER = 4
+_OTHER_HEADER = 5
+
+# These constants define the current inline assembly state
+_NO_ASM = 0       # Outside of inline assembly block
+_INSIDE_ASM = 1   # Inside inline assembly block
+_END_ASM = 2      # Last line of inline assembly block
+_BLOCK_ASM = 3    # The whole block is an inline assembly block
+
+# Match start of assembly blocks
+_MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)'
+                        r'(?:\s+(volatile|__volatile__))?'
+                        r'\s*[{(]')
+
+
+_regexp_compile_cache = {}
+
+# {str, set(int)}: a map from error categories to sets of linenumbers
+# on which those errors are expected and should be suppressed.
+_error_suppressions = {}
+
+# The root directory used for deriving header guard CPP variable.
+# This is set by --root flag.
+_root = None
+
+# The allowed line length of files.
+# This is set by --linelength flag.
+_line_length = 80
+
+# The allowed extensions for file names
+# This is set by --extensions flag.
+_valid_extensions = set(['cc', 'h', 'cpp', 'cu', 'cuh'])
+
+def ParseNolintSuppressions(filename, raw_line, linenum, error):
+  """Updates the global list of error-suppressions.
+
+  Parses any NOLINT comments on the current line, updating the global
+  error_suppressions store.  Reports an error if the NOLINT comment
+  was malformed.
+
+  Args:
+    filename: str, the name of the input file.
+    raw_line: str, the line of input text, with comments.
+    linenum: int, the number of the current line.
+    error: function, an error handler.
+  """
+  matched = Search(r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?', raw_line)
+  if matched:
+    if matched.group(1):
+      suppressed_line = linenum + 1
+    else:
+      suppressed_line = linenum
+    category = matched.group(2)
+    if category in (None, '(*)'):  # => "suppress all"
+      _error_suppressions.setdefault(None, set()).add(suppressed_line)
+    else:
+      if category.startswith('(') and category.endswith(')'):
+        category = category[1:-1]
+        if category in _ERROR_CATEGORIES:
+          _error_suppressions.setdefault(category, set()).add(suppressed_line)
+        elif category not in _LEGACY_ERROR_CATEGORIES:
+          error(filename, linenum, 'readability/nolint', 5,
+                'Unknown NOLINT error category: %s' % category)
+
+
+def ResetNolintSuppressions():
+  """Resets the set of NOLINT suppressions to empty."""
+  _error_suppressions.clear()
+
+
+def IsErrorSuppressedByNolint(category, linenum):
+  """Returns true if the specified error category is suppressed on this line.
+
+  Consults the global error_suppressions map populated by
+  ParseNolintSuppressions/ResetNolintSuppressions.
+
+  Args:
+    category: str, the category of the error.
+    linenum: int, the current line number.
+  Returns:
+    bool, True iff the error should be suppressed due to a NOLINT comment.
+  """
+  return (linenum in _error_suppressions.get(category, set()) or
+          linenum in _error_suppressions.get(None, set()))
+
+
+def Match(pattern, s):
+  """Matches the string with the pattern, caching the compiled regexp."""
+  # The regexp compilation caching is inlined in both Match and Search for
+  # performance reasons; factoring it out into a separate function turns out
+  # to be noticeably expensive.
+  if pattern not in _regexp_compile_cache:
+    _regexp_compile_cache[pattern] = sre_compile.compile(pattern)
+  return _regexp_compile_cache[pattern].match(s)
+
+
+def ReplaceAll(pattern, rep, s):
+  """Replaces instances of pattern in a string with a replacement.
+
+  The compiled regex is kept in a cache shared by Match and Search.
+
+  Args:
+    pattern: regex pattern
+    rep: replacement text
+    s: search string
+
+  Returns:
+    string with replacements made (or original string if no replacements)
+  """
+  if pattern not in _regexp_compile_cache:
+    _regexp_compile_cache[pattern] = sre_compile.compile(pattern)
+  return _regexp_compile_cache[pattern].sub(rep, s)
+
+
+def Search(pattern, s):
+  """Searches the string for the pattern, caching the compiled regexp."""
+  if pattern not in _regexp_compile_cache:
+    _regexp_compile_cache[pattern] = sre_compile.compile(pattern)
+  return _regexp_compile_cache[pattern].search(s)
+
+
+class _IncludeState(object):
+  """Tracks line numbers for includes, and the order in which includes appear.
+
+  include_list contains list of lists of (header, line number) pairs.
+  It's a lists of lists rather than just one flat list to make it
+  easier to update across preprocessor boundaries.
+
+  Call CheckNextIncludeOrder() once for each header in the file, passing
+  in the type constants defined above. Calls in an illegal order will
+  raise an _IncludeError with an appropriate error message.
+
+  """
+  # self._section will move monotonically through this set. If it ever
+  # needs to move backwards, CheckNextIncludeOrder will raise an error.
+  _INITIAL_SECTION = 0
+  _MY_H_SECTION = 1
+  _C_SECTION = 2
+  _CPP_SECTION = 3
+  _OTHER_H_SECTION = 4
+
+  _TYPE_NAMES = {
+      _C_SYS_HEADER: 'C system header',
+      _CPP_SYS_HEADER: 'C++ system header',
+      _LIKELY_MY_HEADER: 'header this file implements',
+      _POSSIBLE_MY_HEADER: 'header this file may implement',
+      _OTHER_HEADER: 'other header',
+      }
+  _SECTION_NAMES = {
+      _INITIAL_SECTION: "... nothing. (This can't be an error.)",
+      _MY_H_SECTION: 'a header this file implements',
+      _C_SECTION: 'C system header',
+      _CPP_SECTION: 'C++ system header',
+      _OTHER_H_SECTION: 'other header',
+      }
+
+  def __init__(self):
+    self.include_list = [[]]
+    self.ResetSection('')
+
+  def FindHeader(self, header):
+    """Check if a header has already been included.
+
+    Args:
+      header: header to check.
+    Returns:
+      Line number of previous occurrence, or -1 if the header has not
+      been seen before.
+    """
+    for section_list in self.include_list:
+      for f in section_list:
+        if f[0] == header:
+          return f[1]
+    return -1
+
+  def ResetSection(self, directive):
+    """Reset section checking for preprocessor directive.
+
+    Args:
+      directive: preprocessor directive (e.g. "if", "else").
+    """
+    # The name of the current section.
+    self._section = self._INITIAL_SECTION
+    # The path of last found header.
+    self._last_header = ''
+
+    # Update list of includes.  Note that we never pop from the
+    # include list.
+    if directive in ('if', 'ifdef', 'ifndef'):
+      self.include_list.append([])
+    elif directive in ('else', 'elif'):
+      self.include_list[-1] = []
+
+  def SetLastHeader(self, header_path):
+    self._last_header = header_path
+
+  def CanonicalizeAlphabeticalOrder(self, header_path):
+    """Returns a path canonicalized for alphabetical comparison.
+
+    - replaces "-" with "_" so they both cmp the same.
+    - removes '-inl' since we don't require them to be after the main header.
+    - lowercase everything, just in case.
+
+    Args:
+      header_path: Path to be canonicalized.
+
+    Returns:
+      Canonicalized path.
+    """
+    return header_path.replace('-inl.h', '.h').replace('-', '_').lower()
+
+  def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path):
+    """Check if a header is in alphabetical order with the previous header.
+
+    Args:
+      clean_lines: A CleansedLines instance containing the file.
+      linenum: The number of the line to check.
+      header_path: Canonicalized header to be checked.
+
+    Returns:
+      Returns true if the header is in alphabetical order.
+    """
+    # If previous section is different from current section, _last_header will
+    # be reset to empty string, so it's always less than current header.
+    #
+    # If previous line was a blank line, assume that the headers are
+    # intentionally sorted the way they are.
+    if (self._last_header > header_path and
+        Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])):
+      return False
+    return True
+
+  def CheckNextIncludeOrder(self, header_type):
+    """Returns a non-empty error message if the next header is out of order.
+
+    This function also updates the internal state to be ready to check
+    the next include.
+
+    Args:
+      header_type: One of the _XXX_HEADER constants defined above.
+
+    Returns:
+      The empty string if the header is in the right order, or an
+      error message describing what's wrong.
+
+    """
+    error_message = ('Found %s after %s' %
+                     (self._TYPE_NAMES[header_type],
+                      self._SECTION_NAMES[self._section]))
+
+    last_section = self._section
+
+    if header_type == _C_SYS_HEADER:
+      if self._section <= self._C_SECTION:
+        self._section = self._C_SECTION
+      else:
+        self._last_header = ''
+        return error_message
+    elif header_type == _CPP_SYS_HEADER:
+      if self._section <= self._CPP_SECTION:
+        self._section = self._CPP_SECTION
+      else:
+        self._last_header = ''
+        return error_message
+    elif header_type == _LIKELY_MY_HEADER:
+      if self._section <= self._MY_H_SECTION:
+        self._section = self._MY_H_SECTION
+      else:
+        self._section = self._OTHER_H_SECTION
+    elif header_type == _POSSIBLE_MY_HEADER:
+      if self._section <= self._MY_H_SECTION:
+        self._section = self._MY_H_SECTION
+      else:
+        # This will always be the fallback because we're not sure
+        # enough that the header is associated with this file.
+        self._section = self._OTHER_H_SECTION
+    else:
+      assert header_type == _OTHER_HEADER
+      self._section = self._OTHER_H_SECTION
+
+    if last_section != self._section:
+      self._last_header = ''
+
+    return ''
+
+
+class _CppLintState(object):
+  """Maintains module-wide state.."""
+
+  def __init__(self):
+    self.verbose_level = 1  # global setting.
+    self.error_count = 0    # global count of reported errors
+    # filters to apply when emitting error messages
+    self.filters = _DEFAULT_FILTERS[:]
+    # backup of filter list. Used to restore the state after each file.
+    self._filters_backup = self.filters[:]
+    self.counting = 'total'  # In what way are we counting errors?
+    self.errors_by_category = {}  # string to int dict storing error counts
+
+    # output format:
+    # "emacs" - format that emacs can parse (default)
+    # "vs7" - format that Microsoft Visual Studio 7 can parse
+    self.output_format = 'emacs'
+
+  def SetOutputFormat(self, output_format):
+    """Sets the output format for errors."""
+    self.output_format = output_format
+
+  def SetVerboseLevel(self, level):
+    """Sets the module's verbosity, and returns the previous setting."""
+    last_verbose_level = self.verbose_level
+    self.verbose_level = level
+    return last_verbose_level
+
+  def SetCountingStyle(self, counting_style):
+    """Sets the module's counting options."""
+    self.counting = counting_style
+
+  def SetFilters(self, filters):
+    """Sets the error-message filters.
+
+    These filters are applied when deciding whether to emit a given
+    error message.
+
+    Args:
+      filters: A string of comma-separated filters (eg "+whitespace/indent").
+               Each filter should start with + or -; else we die.
+
+    Raises:
+      ValueError: The comma-separated filters did not all start with '+' or '-'.
+                  E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter"
+    """
+    # Default filters always have less priority than the flag ones.
+    self.filters = _DEFAULT_FILTERS[:]
+    self.AddFilters(filters)
+
+  def AddFilters(self, filters):
+    """ Adds more filters to the existing list of error-message filters. """
+    for filt in filters.split(','):
+      clean_filt = filt.strip()
+      if clean_filt:
+        self.filters.append(clean_filt)
+    for filt in self.filters:
+      if not (filt.startswith('+') or filt.startswith('-')):
+        raise ValueError('Every filter in --filters must start with + or -'
+                         ' (%s does not)' % filt)
+
+  def BackupFilters(self):
+    """ Saves the current filter list to backup storage."""
+    self._filters_backup = self.filters[:]
+
+  def RestoreFilters(self):
+    """ Restores filters previously backed up."""
+    self.filters = self._filters_backup[:]
+
+  def ResetErrorCounts(self):
+    """Sets the module's error statistic back to zero."""
+    self.error_count = 0
+    self.errors_by_category = {}
+
+  def IncrementErrorCount(self, category):
+    """Bumps the module's error statistic."""
+    self.error_count += 1
+    if self.counting in ('toplevel', 'detailed'):
+      if self.counting != 'detailed':
+        category = category.split('/')[0]
+      if category not in self.errors_by_category:
+        self.errors_by_category[category] = 0
+      self.errors_by_category[category] += 1
+
+  def PrintErrorCounts(self):
+    """Print a summary of errors by category, and the total."""
+    for category, count in self.errors_by_category.iteritems():
+      sys.stderr.write('Category \'%s\' errors found: %d\n' %
+                       (category, count))
+    sys.stderr.write('Total errors found: %d\n' % self.error_count)
+
+_cpplint_state = _CppLintState()
+
+
+def _OutputFormat():
+  """Gets the module's output format."""
+  return _cpplint_state.output_format
+
+
+def _SetOutputFormat(output_format):
+  """Sets the module's output format."""
+  _cpplint_state.SetOutputFormat(output_format)
+
+
+def _VerboseLevel():
+  """Returns the module's verbosity setting."""
+  return _cpplint_state.verbose_level
+
+
+def _SetVerboseLevel(level):
+  """Sets the module's verbosity, and returns the previous setting."""
+  return _cpplint_state.SetVerboseLevel(level)
+
+
+def _SetCountingStyle(level):
+  """Sets the module's counting options."""
+  _cpplint_state.SetCountingStyle(level)
+
+
+def _Filters():
+  """Returns the module's list of output filters, as a list."""
+  return _cpplint_state.filters
+
+
+def _SetFilters(filters):
+  """Sets the module's error-message filters.
+
+  These filters are applied when deciding whether to emit a given
+  error message.
+
+  Args:
+    filters: A string of comma-separated filters (eg "whitespace/indent").
+             Each filter should start with + or -; else we die.
+  """
+  _cpplint_state.SetFilters(filters)
+
+def _AddFilters(filters):
+  """Adds more filter overrides.
+
+  Unlike _SetFilters, this function does not reset the current list of filters
+  available.
+
+  Args:
+    filters: A string of comma-separated filters (eg "whitespace/indent").
+             Each filter should start with + or -; else we die.
+  """
+  _cpplint_state.AddFilters(filters)
+
+def _BackupFilters():
+  """ Saves the current filter list to backup storage."""
+  _cpplint_state.BackupFilters()
+
+def _RestoreFilters():
+  """ Restores filters previously backed up."""
+  _cpplint_state.RestoreFilters()
+
+class _FunctionState(object):
+  """Tracks current function name and the number of lines in its body."""
+
+  _NORMAL_TRIGGER = 250  # for --v=0, 500 for --v=1, etc.
+  _TEST_TRIGGER = 400    # about 50% more than _NORMAL_TRIGGER.
+
+  def __init__(self):
+    self.in_a_function = False
+    self.lines_in_function = 0
+    self.current_function = ''
+
+  def Begin(self, function_name):
+    """Start analyzing function body.
+
+    Args:
+      function_name: The name of the function being tracked.
+    """
+    self.in_a_function = True
+    self.lines_in_function = 0
+    self.current_function = function_name
+
+  def Count(self):
+    """Count line in current function body."""
+    if self.in_a_function:
+      self.lines_in_function += 1
+
+  def Check(self, error, filename, linenum):
+    """Report if too many lines in function body.
+
+    Args:
+      error: The function to call with any errors found.
+      filename: The name of the current file.
+      linenum: The number of the line to check.
+    """
+    if Match(r'T(EST|est)', self.current_function):
+      base_trigger = self._TEST_TRIGGER
+    else:
+      base_trigger = self._NORMAL_TRIGGER
+    trigger = base_trigger * 2**_VerboseLevel()
+
+    if self.lines_in_function > trigger:
+      error_level = int(math.log(self.lines_in_function / base_trigger, 2))
+      # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ...
+      if error_level > 5:
+        error_level = 5
+      error(filename, linenum, 'readability/fn_size', error_level,
+            'Small and focused functions are preferred:'
+            ' %s has %d non-comment lines'
+            ' (error triggered by exceeding %d lines).'  % (
+                self.current_function, self.lines_in_function, trigger))
+
+  def End(self):
+    """Stop analyzing function body."""
+    self.in_a_function = False
+
+
+class _IncludeError(Exception):
+  """Indicates a problem with the include order in a file."""
+  pass
+
+
+class FileInfo(object):
+  """Provides utility functions for filenames.
+
+  FileInfo provides easy access to the components of a file's path
+  relative to the project root.
+  """
+
+  def __init__(self, filename):
+    self._filename = filename
+
+  def FullName(self):
+    """Make Windows paths like Unix."""
+    return os.path.abspath(self._filename).replace('\\', '/')
+
+  def RepositoryName(self):
+    """FullName after removing the local path to the repository.
+
+    If we have a real absolute path name here we can try to do something smart:
+    detecting the root of the checkout and truncating /path/to/checkout from
+    the name so that we get header guards that don't include things like
+    "C:\Documents and Settings\..." or "/home/username/..." in them and thus
+    people on different computers who have checked the source out to different
+    locations won't see bogus errors.
+    """
+    fullname = self.FullName()
+
+    if os.path.exists(fullname):
+      project_dir = os.path.dirname(fullname)
+
+      if os.path.exists(os.path.join(project_dir, ".svn")):
+        # If there's a .svn file in the current directory, we recursively look
+        # up the directory tree for the top of the SVN checkout
+        root_dir = project_dir
+        one_up_dir = os.path.dirname(root_dir)
+        while os.path.exists(os.path.join(one_up_dir, ".svn")):
+          root_dir = os.path.dirname(root_dir)
+          one_up_dir = os.path.dirname(one_up_dir)
+
+        prefix = os.path.commonprefix([root_dir, project_dir])
+        return fullname[len(prefix) + 1:]
+
+      # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by
+      # searching up from the current path.
+      root_dir = os.path.dirname(fullname)
+      while (root_dir != os.path.dirname(root_dir) and
+             not os.path.exists(os.path.join(root_dir, ".git")) and
+             not os.path.exists(os.path.join(root_dir, ".hg")) and
+             not os.path.exists(os.path.join(root_dir, ".svn"))):
+        root_dir = os.path.dirname(root_dir)
+
+      if (os.path.exists(os.path.join(root_dir, ".git")) or
+          os.path.exists(os.path.join(root_dir, ".hg")) or
+          os.path.exists(os.path.join(root_dir, ".svn"))):
+        prefix = os.path.commonprefix([root_dir, project_dir])
+        return fullname[len(prefix) + 1:]
+
+    # Don't know what to do; header guard warnings may be wrong...
+    return fullname
+
+  def Split(self):
+    """Splits the file into the directory, basename, and extension.
+
+    For 'chrome/browser/browser.cc', Split() would
+    return ('chrome/browser', 'browser', '.cc')
+
+    Returns:
+      A tuple of (directory, basename, extension).
+    """
+
+    googlename = self.RepositoryName()
+    project, rest = os.path.split(googlename)
+    return (project,) + os.path.splitext(rest)
+
+  def BaseName(self):
+    """File base name - text after the final slash, before the final period."""
+    return self.Split()[1]
+
+  def Extension(self):
+    """File extension - text following the final period."""
+    return self.Split()[2]
+
+  def NoExtension(self):
+    """File has no source file extension."""
+    return '/'.join(self.Split()[0:2])
+
+  def IsSource(self):
+    """File has a source file extension."""
+    return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx')
+
+
+def _ShouldPrintError(category, confidence, linenum):
+  """If confidence >= verbose, category passes filter and is not suppressed."""
+
+  # There are three ways we might decide not to print an error message:
+  # a "NOLINT(category)" comment appears in the source,
+  # the verbosity level isn't high enough, or the filters filter it out.
+  if IsErrorSuppressedByNolint(category, linenum):
+    return False
+
+  if confidence < _cpplint_state.verbose_level:
+    return False
+
+  is_filtered = False
+  for one_filter in _Filters():
+    if one_filter.startswith('-'):
+      if category.startswith(one_filter[1:]):
+        is_filtered = True
+    elif one_filter.startswith('+'):
+      if category.startswith(one_filter[1:]):
+        is_filtered = False
+    else:
+      assert False  # should have been checked for in SetFilter.
+  if is_filtered:
+    return False
+
+  return True
+
+
+def Error(filename, linenum, category, confidence, message):
+  """Logs the fact we've found a lint error.
+
+  We log where the error was found, and also our confidence in the error,
+  that is, how certain we are this is a legitimate style regression, and
+  not a misidentification or a use that's sometimes justified.
+
+  False positives can be suppressed by the use of
+  "cpplint(category)"  comments on the offending line.  These are
+  parsed into _error_suppressions.
+
+  Args:
+    filename: The name of the file containing the error.
+    linenum: The number of the line containing the error.
+    category: A string used to describe the "category" this bug
+      falls under: "whitespace", say, or "runtime".  Categories
+      may have a hierarchy separated by slashes: "whitespace/indent".
+    confidence: A number from 1-5 representing a confidence score for
+      the error, with 5 meaning that we are certain of the problem,
+      and 1 meaning that it could be a legitimate construct.
+    message: The error message.
+  """
+  if _ShouldPrintError(category, confidence, linenum):
+    _cpplint_state.IncrementErrorCount(category)
+    if _cpplint_state.output_format == 'vs7':
+      sys.stderr.write('%s(%s):  %s  [%s] [%d]\n' % (
+          filename, linenum, message, category, confidence))
+    elif _cpplint_state.output_format == 'eclipse':
+      sys.stderr.write('%s:%s: warning: %s  [%s] [%d]\n' % (
+          filename, linenum, message, category, confidence))
+    else:
+      sys.stderr.write('%s:%s:  %s  [%s] [%d]\n' % (
+          filename, linenum, message, category, confidence))
+
+
+# Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard.
+_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile(
+    r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)')
+# Match a single C style comment on the same line.
+_RE_PATTERN_C_COMMENTS = r'/\*(?:[^*]|\*(?!/))*\*/'
+# Matches multi-line C style comments.
+# This RE is a little bit more complicated than one might expect, because we
+# have to take care of space removals tools so we can handle comments inside
+# statements better.
+# The current rule is: We only clear spaces from both sides when we're at the
+# end of the line. Otherwise, we try to remove spaces from the right side,
+# if this doesn't work we try on left side but only if there's a non-character
+# on the right.
+_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile(
+    r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' +
+    _RE_PATTERN_C_COMMENTS + r'\s+|' +
+    r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' +
+    _RE_PATTERN_C_COMMENTS + r')')
+
+
+def IsCppString(line):
+  """Does line terminate so, that the next symbol is in string constant.
+
+  This function does not consider single-line nor multi-line comments.
+
+  Args:
+    line: is a partial line of code starting from the 0..n.
+
+  Returns:
+    True, if next character appended to 'line' is inside a
+    string constant.
+  """
+
+  line = line.replace(r'\\', 'XX')  # after this, \\" does not match to \"
+  return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1
+
+
+def CleanseRawStrings(raw_lines):
+  """Removes C++11 raw strings from lines.
+
+    Before:
+      static const char kData[] = R"(
+          multi-line string
+          )";
+
+    After:
+      static const char kData[] = ""
+          (replaced by blank line)
+          "";
+
+  Args:
+    raw_lines: list of raw lines.
+
+  Returns:
+    list of lines with C++11 raw strings replaced by empty strings.
+  """
+
+  delimiter = None
+  lines_without_raw_strings = []
+  for line in raw_lines:
+    if delimiter:
+      # Inside a raw string, look for the end
+      end = line.find(delimiter)
+      if end >= 0:
+        # Found the end of the string, match leading space for this
+        # line and resume copying the original lines, and also insert
+        # a "" on the last line.
+        leading_space = Match(r'^(\s*)\S', line)
+        line = leading_space.group(1) + '""' + line[end + len(delimiter):]
+        delimiter = None
+      else:
+        # Haven't found the end yet, append a blank line.
+        line = '""'
+
+    # Look for beginning of a raw string, and replace them with
+    # empty strings.  This is done in a loop to handle multiple raw
+    # strings on the same line.
+    while delimiter is None:
+      # Look for beginning of a raw string.
+      # See 2.14.15 [lex.string] for syntax.
+      matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line)
+      if matched:
+        delimiter = ')' + matched.group(2) + '"'
+
+        end = matched.group(3).find(delimiter)
+        if end >= 0:
+          # Raw string ended on same line
+          line = (matched.group(1) + '""' +
+                  matched.group(3)[end + len(delimiter):])
+          delimiter = None
+        else:
+          # Start of a multi-line raw string
+          line = matched.group(1) + '""'
+      else:
+        break
+
+    lines_without_raw_strings.append(line)
+
+  # TODO(unknown): if delimiter is not None here, we might want to
+  # emit a warning for unterminated string.
+  return lines_without_raw_strings
+
+
+def FindNextMultiLineCommentStart(lines, lineix):
+  """Find the beginning marker for a multiline comment."""
+  while lineix < len(lines):
+    if lines[lineix].strip().startswith('/*'):
+      # Only return this marker if the comment goes beyond this line
+      if lines[lineix].strip().find('*/', 2) < 0:
+        return lineix
+    lineix += 1
+  return len(lines)
+
+
+def FindNextMultiLineCommentEnd(lines, lineix):
+  """We are inside a comment, find the end marker."""
+  while lineix < len(lines):
+    if lines[lineix].strip().endswith('*/'):
+      return lineix
+    lineix += 1
+  return len(lines)
+
+
+def RemoveMultiLineCommentsFromRange(lines, begin, end):
+  """Clears a range of lines for multi-line comments."""
+  # Having // dummy comments makes the lines non-empty, so we will not get
+  # unnecessary blank line warnings later in the code.
+  for i in range(begin, end):
+    lines[i] = '/**/'
+
+
+def RemoveMultiLineComments(filename, lines, error):
+  """Removes multiline (c-style) comments from lines."""
+  lineix = 0
+  while lineix < len(lines):
+    lineix_begin = FindNextMultiLineCommentStart(lines, lineix)
+    if lineix_begin >= len(lines):
+      return
+    lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin)
+    if lineix_end >= len(lines):
+      error(filename, lineix_begin + 1, 'readability/multiline_comment', 5,
+            'Could not find end of multi-line comment')
+      return
+    RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1)
+    lineix = lineix_end + 1
+
+
+def CleanseComments(line):
+  """Removes //-comments and single-line C-style /* */ comments.
+
+  Args:
+    line: A line of C++ source.
+
+  Returns:
+    The line with single-line comments removed.
+  """
+  commentpos = line.find('//')
+  if commentpos != -1 and not IsCppString(line[:commentpos]):
+    line = line[:commentpos].rstrip()
+  # get rid of /* ... */
+  return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line)
+
+
+class CleansedLines(object):
+  """Holds 4 copies of all lines with different preprocessing applied to them.
+
+  1) elided member contains lines without strings and comments.
+  2) lines member contains lines without comments.
+  3) raw_lines member contains all the lines without processing.
+  4) lines_without_raw_strings member is same as raw_lines, but with C++11 raw
+     strings removed.
+  All these members are of <type 'list'>, and of the same length.
+  """
+
+  def __init__(self, lines):
+    self.elided = []
+    self.lines = []
+    self.raw_lines = lines
+    self.num_lines = len(lines)
+    self.lines_without_raw_strings = CleanseRawStrings(lines)
+    for linenum in range(len(self.lines_without_raw_strings)):
+      self.lines.append(CleanseComments(
+          self.lines_without_raw_strings[linenum]))
+      elided = self._CollapseStrings(self.lines_without_raw_strings[linenum])
+      self.elided.append(CleanseComments(elided))
+
+  def NumLines(self):
+    """Returns the number of lines represented."""
+    return self.num_lines
+
+  @staticmethod
+  def _CollapseStrings(elided):
+    """Collapses strings and chars on a line to simple "" or '' blocks.
+
+    We nix strings first so we're not fooled by text like '"http://"'
+
+    Args:
+      elided: The line being processed.
+
+    Returns:
+      The line with collapsed strings.
+    """
+    if _RE_PATTERN_INCLUDE.match(elided):
+      return elided
+
+    # Remove escaped characters first to make quote/single quote collapsing
+    # basic.  Things that look like escaped characters shouldn't occur
+    # outside of strings and chars.
+    elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided)
+
+    # Replace quoted strings and digit separators.  Both single quotes
+    # and double quotes are processed in the same loop, otherwise
+    # nested quotes wouldn't work.
+    collapsed = ''
+    while True:
+      # Find the first quote character
+      match = Match(r'^([^\'"]*)([\'"])(.*)$', elided)
+      if not match:
+        collapsed += elided
+        break
+      head, quote, tail = match.groups()
+
+      if quote == '"':
+        # Collapse double quoted strings
+        second_quote = tail.find('"')
+        if second_quote >= 0:
+          collapsed += head + '""'
+          elided = tail[second_quote + 1:]
+        else:
+          # Unmatched double quote, don't bother processing the rest
+          # of the line since this is probably a multiline string.
+          collapsed += elided
+          break
+      else:
+        # Found single quote, check nearby text to eliminate digit separators.
+        #
+        # There is no special handling for floating point here, because
+        # the integer/fractional/exponent parts would all be parsed
+        # correctly as long as there are digits on both sides of the
+        # separator.  So we are fine as long as we don't see something
+        # like "0.'3" (gcc 4.9.0 will not allow this literal).
+        if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head):
+          match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail)
+          collapsed += head + match_literal.group(1).replace("'", '')
+          elided = match_literal.group(2)
+        else:
+          second_quote = tail.find('\'')
+          if second_quote >= 0:
+            collapsed += head + "''"
+            elided = tail[second_quote + 1:]
+          else:
+            # Unmatched single quote
+            collapsed += elided
+            break
+
+    return collapsed
+
+
+def FindEndOfExpressionInLine(line, startpos, stack):
+  """Find the position just after the end of current parenthesized expression.
+
+  Args:
+    line: a CleansedLines line.
+    startpos: start searching at this position.
+    stack: nesting stack at startpos.
+
+  Returns:
+    On finding matching end: (index just after matching end, None)
+    On finding an unclosed expression: (-1, None)
+    Otherwise: (-1, new stack at end of this line)
+  """
+  for i in xrange(startpos, len(line)):
+    char = line[i]
+    if char in '([{':
+      # Found start of parenthesized expression, push to expression stack
+      stack.append(char)
+    elif char == '<':
+      # Found potential start of template argument list
+      if i > 0 and line[i - 1] == '<':
+        # Left shift operator
+        if stack and stack[-1] == '<':
+          stack.pop()
+          if not stack:
+            return (-1, None)
+      elif i > 0 and Search(r'\boperator\s*$', line[0:i]):
+        # operator<, don't add to stack
+        continue
+      else:
+        # Tentative start of template argument list
+        stack.append('<')
+    elif char in ')]}':
+      # Found end of parenthesized expression.
+      #
+      # If we are currently expecting a matching '>', the pending '<'
+      # must have been an operator.  Remove them from expression stack.
+      while stack and stack[-1] == '<':
+        stack.pop()
+      if not stack:
+        return (-1, None)
+      if ((stack[-1] == '(' and char == ')') or
+          (stack[-1] == '[' and char == ']') or
+          (stack[-1] == '{' and char == '}')):
+        stack.pop()
+        if not stack:
+          return (i + 1, None)
+      else:
+        # Mismatched parentheses
+        return (-1, None)
+    elif char == '>':
+      # Found potential end of template argument list.
+
+      # Ignore "->" and operator functions
+      if (i > 0 and
+          (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))):
+        continue
+
+      # Pop the stack if there is a matching '<'.  Otherwise, ignore
+      # this '>' since it must be an operator.
+      if stack:
+        if stack[-1] == '<':
+          stack.pop()
+          if not stack:
+            return (i + 1, None)
+    elif char == ';':
+      # Found something that look like end of statements.  If we are currently
+      # expecting a '>', the matching '<' must have been an operator, since
+      # template argument list should not contain statements.
+      while stack and stack[-1] == '<':
+        stack.pop()
+      if not stack:
+        return (-1, None)
+
+  # Did not find end of expression or unbalanced parentheses on this line
+  return (-1, stack)
+
+
+def CloseExpression(clean_lines, linenum, pos):
+  """If input points to ( or { or [ or <, finds the position that closes it.
+
+  If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the
+  linenum/pos that correspond to the closing of the expression.
+
+  TODO(unknown): cpplint spends a fair bit of time matching parentheses.
+  Ideally we would want to index all opening and closing parentheses once
+  and have CloseExpression be just a simple lookup, but due to preprocessor
+  tricks, this is not so easy.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    pos: A position on the line.
+
+  Returns:
+    A tuple (line, linenum, pos) pointer *past* the closing brace, or
+    (line, len(lines), -1) if we never find a close.  Note we ignore
+    strings and comments when matching; and the line we return is the
+    'cleansed' line at linenum.
+  """
+
+  line = clean_lines.elided[linenum]
+  if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]):
+    return (line, clean_lines.NumLines(), -1)
+
+  # Check first line
+  (end_pos, stack) = FindEndOfExpressionInLine(line, pos, [])
+  if end_pos > -1:
+    return (line, linenum, end_pos)
+
+  # Continue scanning forward
+  while stack and linenum < clean_lines.NumLines() - 1:
+    linenum += 1
+    line = clean_lines.elided[linenum]
+    (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack)
+    if end_pos > -1:
+      return (line, linenum, end_pos)
+
+  # Did not find end of expression before end of file, give up
+  return (line, clean_lines.NumLines(), -1)
+
+
+def FindStartOfExpressionInLine(line, endpos, stack):
+  """Find position at the matching start of current expression.
+
+  This is almost the reverse of FindEndOfExpressionInLine, but note
+  that the input position and returned position differs by 1.
+
+  Args:
+    line: a CleansedLines line.
+    endpos: start searching at this position.
+    stack: nesting stack at endpos.
+
+  Returns:
+    On finding matching start: (index at matching start, None)
+    On finding an unclosed expression: (-1, None)
+    Otherwise: (-1, new stack at beginning of this line)
+  """
+  i = endpos
+  while i >= 0:
+    char = line[i]
+    if char in ')]}':
+      # Found end of expression, push to expression stack
+      stack.append(char)
+    elif char == '>':
+      # Found potential end of template argument list.
+      #
+      # Ignore it if it's a "->" or ">=" or "operator>"
+      if (i > 0 and
+          (line[i - 1] == '-' or
+           Match(r'\s>=\s', line[i - 1:]) or
+           Search(r'\boperator\s*$', line[0:i]))):
+        i -= 1
+      else:
+        stack.append('>')
+    elif char == '<':
+      # Found potential start of template argument list
+      if i > 0 and line[i - 1] == '<':
+        # Left shift operator
+        i -= 1
+      else:
+        # If there is a matching '>', we can pop the expression stack.
+        # Otherwise, ignore this '<' since it must be an operator.
+        if stack and stack[-1] == '>':
+          stack.pop()
+          if not stack:
+            return (i, None)
+    elif char in '([{':
+      # Found start of expression.
+      #
+      # If there are any unmatched '>' on the stack, they must be
+      # operators.  Remove those.
+      while stack and stack[-1] == '>':
+        stack.pop()
+      if not stack:
+        return (-1, None)
+      if ((char == '(' and stack[-1] == ')') or
+          (char == '[' and stack[-1] == ']') or
+          (char == '{' and stack[-1] == '}')):
+        stack.pop()
+        if not stack:
+          return (i, None)
+      else:
+        # Mismatched parentheses
+        return (-1, None)
+    elif char == ';':
+      # Found something that look like end of statements.  If we are currently
+      # expecting a '<', the matching '>' must have been an operator, since
+      # template argument list should not contain statements.
+      while stack and stack[-1] == '>':
+        stack.pop()
+      if not stack:
+        return (-1, None)
+
+    i -= 1
+
+  return (-1, stack)
+
+
+def ReverseCloseExpression(clean_lines, linenum, pos):
+  """If input points to ) or } or ] or >, finds the position that opens it.
+
+  If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the
+  linenum/pos that correspond to the opening of the expression.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    pos: A position on the line.
+
+  Returns:
+    A tuple (line, linenum, pos) pointer *at* the opening brace, or
+    (line, 0, -1) if we never find the matching opening brace.  Note
+    we ignore strings and comments when matching; and the line we
+    return is the 'cleansed' line at linenum.
+  """
+  line = clean_lines.elided[linenum]
+  if line[pos] not in ')}]>':
+    return (line, 0, -1)
+
+  # Check last line
+  (start_pos, stack) = FindStartOfExpressionInLine(line, pos, [])
+  if start_pos > -1:
+    return (line, linenum, start_pos)
+
+  # Continue scanning backward
+  while stack and linenum > 0:
+    linenum -= 1
+    line = clean_lines.elided[linenum]
+    (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack)
+    if start_pos > -1:
+      return (line, linenum, start_pos)
+
+  # Did not find start of expression before beginning of file, give up
+  return (line, 0, -1)
+
+
+def CheckForCopyright(filename, lines, error):
+  """Logs an error if no Copyright message appears at the top of the file."""
+
+  # We'll say it should occur by line 10. Don't forget there's a
+  # dummy line at the front.
+  for line in xrange(1, min(len(lines), 11)):
+    if re.search(r'Copyright', lines[line], re.I): break
+  else:                       # means no copyright line was found
+    error(filename, 0, 'legal/copyright', 5,
+          'No copyright message found.  '
+          'You should have a line: "Copyright [year] <Copyright Owner>"')
+
+
+def GetIndentLevel(line):
+  """Return the number of leading spaces in line.
+
+  Args:
+    line: A string to check.
+
+  Returns:
+    An integer count of leading spaces, possibly zero.
+  """
+  indent = Match(r'^( *)\S', line)
+  if indent:
+    return len(indent.group(1))
+  else:
+    return 0
+
+
+def GetHeaderGuardCPPVariable(filename):
+  """Returns the CPP variable that should be used as a header guard.
+
+  Args:
+    filename: The name of a C++ header file.
+
+  Returns:
+    The CPP variable that should be used as a header guard in the
+    named file.
+
+  """
+
+  # Restores original filename in case that cpplint is invoked from Emacs's
+  # flymake.
+  filename = re.sub(r'_flymake\.h$', '.h', filename)
+  filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename)
+  # Replace 'c++' with 'cpp'.
+  filename = filename.replace('C++', 'cpp').replace('c++', 'cpp')
+  
+  fileinfo = FileInfo(filename)
+  file_path_from_root = fileinfo.RepositoryName()
+  if _root:
+    file_path_from_root = re.sub('^' + _root + os.sep, '', file_path_from_root)
+  return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_'
+
+
+def CheckForHeaderGuard(filename, clean_lines, error):
+  """Checks that the file contains a header guard.
+
+  Logs an error if no #ifndef header guard is present.  For other
+  headers, checks that the full pathname is used.
+
+  Args:
+    filename: The name of the C++ header file.
+    clean_lines: A CleansedLines instance containing the file.
+    error: The function to call with any errors found.
+  """
+
+  # Don't check for header guards if there are error suppression
+  # comments somewhere in this file.
+  #
+  # Because this is silencing a warning for a nonexistent line, we
+  # only support the very specific NOLINT(build/header_guard) syntax,
+  # and not the general NOLINT or NOLINT(*) syntax.
+  raw_lines = clean_lines.lines_without_raw_strings
+  for i in raw_lines:
+    if Search(r'//\s*NOLINT\(build/header_guard\)', i):
+      return
+
+  cppvar = GetHeaderGuardCPPVariable(filename)
+
+  ifndef = ''
+  ifndef_linenum = 0
+  define = ''
+  endif = ''
+  endif_linenum = 0
+  for linenum, line in enumerate(raw_lines):
+    linesplit = line.split()
+    if len(linesplit) >= 2:
+      # find the first occurrence of #ifndef and #define, save arg
+      if not ifndef and linesplit[0] == '#ifndef':
+        # set ifndef to the header guard presented on the #ifndef line.
+        ifndef = linesplit[1]
+        ifndef_linenum = linenum
+      if not define and linesplit[0] == '#define':
+        define = linesplit[1]
+    # find the last occurrence of #endif, save entire line
+    if line.startswith('#endif'):
+      endif = line
+      endif_linenum = linenum
+
+  if not ifndef or not define or ifndef != define:
+    error(filename, 0, 'build/header_guard', 5,
+          'No #ifndef header guard found, suggested CPP variable is: %s' %
+          cppvar)
+    return
+
+  # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__
+  # for backward compatibility.
+  if ifndef != cppvar:
+    error_level = 0
+    if ifndef != cppvar + '_':
+      error_level = 5
+
+    ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum,
+                            error)
+    error(filename, ifndef_linenum, 'build/header_guard', error_level,
+          '#ifndef header guard has wrong style, please use: %s' % cppvar)
+
+  # Check for "//" comments on endif line.
+  ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum,
+                          error)
+  match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif)
+  if match:
+    if match.group(1) == '_':
+      # Issue low severity warning for deprecated double trailing underscore
+      error(filename, endif_linenum, 'build/header_guard', 0,
+            '#endif line should be "#endif  // %s"' % cppvar)
+    return
+
+  # Didn't find the corresponding "//" comment.  If this file does not
+  # contain any "//" comments at all, it could be that the compiler
+  # only wants "/**/" comments, look for those instead.
+  no_single_line_comments = True
+  for i in xrange(1, len(raw_lines) - 1):
+    line = raw_lines[i]
+    if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line):
+      no_single_line_comments = False
+      break
+
+  if no_single_line_comments:
+    match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif)
+    if match:
+      if match.group(1) == '_':
+        # Low severity warning for double trailing underscore
+        error(filename, endif_linenum, 'build/header_guard', 0,
+              '#endif line should be "#endif  /* %s */"' % cppvar)
+      return
+
+  # Didn't find anything
+  error(filename, endif_linenum, 'build/header_guard', 5,
+        '#endif line should be "#endif  // %s"' % cppvar)
+
+
+def CheckHeaderFileIncluded(filename, include_state, error):
+  """Logs an error if a .cc file does not include its header."""
+
+  # Do not check test files
+  if filename.endswith('_test.cc') or filename.endswith('_unittest.cc'):
+    return
+
+  fileinfo = FileInfo(filename)
+  headerfile = filename[0:len(filename) - 2] + 'h'
+  if not os.path.exists(headerfile):
+    return
+  headername = FileInfo(headerfile).RepositoryName()
+  first_include = 0
+  for section_list in include_state.include_list:
+    for f in section_list:
+      if headername in f[0] or f[0] in headername:
+        return
+      if not first_include:
+        first_include = f[1]
+
+  error(filename, first_include, 'build/include', 5,
+        '%s should include its header file %s' % (fileinfo.RepositoryName(),
+                                                  headername))
+
+
+def CheckForBadCharacters(filename, lines, error):
+  """Logs an error for each line containing bad characters.
+
+  Two kinds of bad characters:
+
+  1. Unicode replacement characters: These indicate that either the file
+  contained invalid UTF-8 (likely) or Unicode replacement characters (which
+  it shouldn't).  Note that it's possible for this to throw off line
+  numbering if the invalid UTF-8 occurred adjacent to a newline.
+
+  2. NUL bytes.  These are problematic for some tools.
+
+  Args:
+    filename: The name of the current file.
+    lines: An array of strings, each representing a line of the file.
+    error: The function to call with any errors found.
+  """
+  for linenum, line in enumerate(lines):
+    if u'\ufffd' in line:
+      error(filename, linenum, 'readability/utf8', 5,
+            'Line contains invalid UTF-8 (or Unicode replacement character).')
+    if '\0' in line:
+      error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.')
+
+
+def CheckForNewlineAtEOF(filename, lines, error):
+  """Logs an error if there is no newline char at the end of the file.
+
+  Args:
+    filename: The name of the current file.
+    lines: An array of strings, each representing a line of the file.
+    error: The function to call with any errors found.
+  """
+
+  # The array lines() was created by adding two newlines to the
+  # original file (go figure), then splitting on \n.
+  # To verify that the file ends in \n, we just have to make sure the
+  # last-but-two element of lines() exists and is empty.
+  if len(lines) < 3 or lines[-2]:
+    error(filename, len(lines) - 2, 'whitespace/ending_newline', 5,
+          'Could not find a newline character at the end of the file.')
+
+
+def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error):
+  """Logs an error if we see /* ... */ or "..." that extend past one line.
+
+  /* ... */ comments are legit inside macros, for one line.
+  Otherwise, we prefer // comments, so it's ok to warn about the
+  other.  Likewise, it's ok for strings to extend across multiple
+  lines, as long as a line continuation character (backslash)
+  terminates each line. Although not currently prohibited by the C++
+  style guide, it's ugly and unnecessary. We don't do well with either
+  in this lint program, so we warn about both.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # Remove all \\ (escaped backslashes) from the line. They are OK, and the
+  # second (escaped) slash may trigger later \" detection erroneously.
+  line = line.replace('\\\\', '')
+
+  if line.count('/*') > line.count('*/'):
+    error(filename, linenum, 'readability/multiline_comment', 5,
+          'Complex multi-line /*...*/-style comment found. '
+          'Lint may give bogus warnings.  '
+          'Consider replacing these with //-style comments, '
+          'with #if 0...#endif, '
+          'or with more clearly structured multi-line comments.')
+
+  if (line.count('"') - line.count('\\"')) % 2:
+    error(filename, linenum, 'readability/multiline_string', 5,
+          'Multi-line string ("...") found.  This lint script doesn\'t '
+          'do well with such strings, and may give bogus warnings.  '
+          'Use C++11 raw strings or concatenation instead.')
+
+
+# (non-threadsafe name, thread-safe alternative, validation pattern)
+#
+# The validation pattern is used to eliminate false positives such as:
+#  _rand();               // false positive due to substring match.
+#  ->rand();              // some member function rand().
+#  ACMRandom rand(seed);  // some variable named rand.
+#  ISAACRandom rand();    // another variable named rand.
+#
+# Basically we require the return value of these functions to be used
+# in some expression context on the same line by matching on some
+# operator before the function name.  This eliminates constructors and
+# member function calls.
+_UNSAFE_FUNC_PREFIX = r'(?:[-+*/=%^&|(<]\s*|>\s+)'
+_THREADING_LIST = (
+    ('asctime(', 'asctime_r(', _UNSAFE_FUNC_PREFIX + r'asctime\([^)]+\)'),
+    ('ctime(', 'ctime_r(', _UNSAFE_FUNC_PREFIX + r'ctime\([^)]+\)'),
+    ('getgrgid(', 'getgrgid_r(', _UNSAFE_FUNC_PREFIX + r'getgrgid\([^)]+\)'),
+    ('getgrnam(', 'getgrnam_r(', _UNSAFE_FUNC_PREFIX + r'getgrnam\([^)]+\)'),
+    ('getlogin(', 'getlogin_r(', _UNSAFE_FUNC_PREFIX + r'getlogin\(\)'),
+    ('getpwnam(', 'getpwnam_r(', _UNSAFE_FUNC_PREFIX + r'getpwnam\([^)]+\)'),
+    ('getpwuid(', 'getpwuid_r(', _UNSAFE_FUNC_PREFIX + r'getpwuid\([^)]+\)'),
+    ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'),
+    ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'),
+    ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'),
+    ('strtok(', 'strtok_r(',
+     _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'),
+    ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'),
+    )
+
+
+def CheckPosixThreading(filename, clean_lines, linenum, error):
+  """Checks for calls to thread-unsafe functions.
+
+  Much code has been originally written without consideration of
+  multi-threading. Also, engineers are relying on their old experience;
+  they have learned posix before threading extensions were added. These
+  tests guide the engineers to use thread-safe functions (when using
+  posix directly).
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+  for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST:
+    # Additional pattern matching check to confirm that this is the
+    # function we are looking for
+    if Search(pattern, line):
+      error(filename, linenum, 'runtime/threadsafe_fn', 2,
+            'Consider using ' + multithread_safe_func +
+            '...) instead of ' + single_thread_func +
+            '...) for improved thread safety.')
+
+
+def CheckVlogArguments(filename, clean_lines, linenum, error):
+  """Checks that VLOG() is only used for defining a logging level.
+
+  For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and
+  VLOG(FATAL) are not.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+  if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line):
+    error(filename, linenum, 'runtime/vlog', 5,
+          'VLOG() should be used with numeric verbosity level.  '
+          'Use LOG() if you want symbolic severity levels.')
+
+# Matches invalid increment: *count++, which moves pointer instead of
+# incrementing a value.
+_RE_PATTERN_INVALID_INCREMENT = re.compile(
+    r'^\s*\*\w+(\+\+|--);')
+
+
+def CheckInvalidIncrement(filename, clean_lines, linenum, error):
+  """Checks for invalid increment *count++.
+
+  For example following function:
+  void increment_counter(int* count) {
+    *count++;
+  }
+  is invalid, because it effectively does count++, moving pointer, and should
+  be replaced with ++*count, (*count)++ or *count += 1.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+  if _RE_PATTERN_INVALID_INCREMENT.match(line):
+    error(filename, linenum, 'runtime/invalid_increment', 5,
+          'Changing pointer instead of value (or unused value of operator*).')
+
+
+def IsMacroDefinition(clean_lines, linenum):
+  if Search(r'^#define', clean_lines[linenum]):
+    return True
+
+  if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]):
+    return True
+
+  return False
+
+
+def IsForwardClassDeclaration(clean_lines, linenum):
+  return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum])
+
+
+class _BlockInfo(object):
+  """Stores information about a generic block of code."""
+
+  def __init__(self, seen_open_brace):
+    self.seen_open_brace = seen_open_brace
+    self.open_parentheses = 0
+    self.inline_asm = _NO_ASM
+    self.check_namespace_indentation = False
+
+  def CheckBegin(self, filename, clean_lines, linenum, error):
+    """Run checks that applies to text up to the opening brace.
+
+    This is mostly for checking the text after the class identifier
+    and the "{", usually where the base class is specified.  For other
+    blocks, there isn't much to check, so we always pass.
+
+    Args:
+      filename: The name of the current file.
+      clean_lines: A CleansedLines instance containing the file.
+      linenum: The number of the line to check.
+      error: The function to call with any errors found.
+    """
+    pass
+
+  def CheckEnd(self, filename, clean_lines, linenum, error):
+    """Run checks that applies to text after the closing brace.
+
+    This is mostly used for checking end of namespace comments.
+
+    Args:
+      filename: The name of the current file.
+      clean_lines: A CleansedLines instance containing the file.
+      linenum: The number of the line to check.
+      error: The function to call with any errors found.
+    """
+    pass
+
+  def IsBlockInfo(self):
+    """Returns true if this block is a _BlockInfo.
+
+    This is convenient for verifying that an object is an instance of
+    a _BlockInfo, but not an instance of any of the derived classes.
+
+    Returns:
+      True for this class, False for derived classes.
+    """
+    return self.__class__ == _BlockInfo
+
+
+class _ExternCInfo(_BlockInfo):
+  """Stores information about an 'extern "C"' block."""
+
+  def __init__(self):
+    _BlockInfo.__init__(self, True)
+
+
+class _ClassInfo(_BlockInfo):
+  """Stores information about a class."""
+
+  def __init__(self, name, class_or_struct, clean_lines, linenum):
+    _BlockInfo.__init__(self, False)
+    self.name = name
+    self.starting_linenum = linenum
+    self.is_derived = False
+    self.check_namespace_indentation = True
+    if class_or_struct == 'struct':
+      self.access = 'public'
+      self.is_struct = True
+    else:
+      self.access = 'private'
+      self.is_struct = False
+
+    # Remember initial indentation level for this class.  Using raw_lines here
+    # instead of elided to account for leading comments.
+    self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum])
+
+    # Try to find the end of the class.  This will be confused by things like:
+    #   class A {
+    #   } *x = { ...
+    #
+    # But it's still good enough for CheckSectionSpacing.
+    self.last_line = 0
+    depth = 0
+    for i in range(linenum, clean_lines.NumLines()):
+      line = clean_lines.elided[i]
+      depth += line.count('{') - line.count('}')
+      if not depth:
+        self.last_line = i
+        break
+
+  def CheckBegin(self, filename, clean_lines, linenum, error):
+    # Look for a bare ':'
+    if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]):
+      self.is_derived = True
+
+  def CheckEnd(self, filename, clean_lines, linenum, error):
+    # If there is a DISALLOW macro, it should appear near the end of
+    # the class.
+    seen_last_thing_in_class = False
+    for i in xrange(linenum - 1, self.starting_linenum, -1):
+      match = Search(
+          r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' +
+          self.name + r'\)',
+          clean_lines.elided[i])
+      if match:
+        if seen_last_thing_in_class:
+          error(filename, i, 'readability/constructors', 3,
+                match.group(1) + ' should be the last thing in the class')
+        break
+
+      if not Match(r'^\s*$', clean_lines.elided[i]):
+        seen_last_thing_in_class = True
+
+    # Check that closing brace is aligned with beginning of the class.
+    # Only do this if the closing brace is indented by only whitespaces.
+    # This means we will not check single-line class definitions.
+    indent = Match(r'^( *)\}', clean_lines.elided[linenum])
+    if indent and len(indent.group(1)) != self.class_indent:
+      if self.is_struct:
+        parent = 'struct ' + self.name
+      else:
+        parent = 'class ' + self.name
+      error(filename, linenum, 'whitespace/indent', 3,
+            'Closing brace should be aligned with beginning of %s' % parent)
+
+
+class _NamespaceInfo(_BlockInfo):
+  """Stores information about a namespace."""
+
+  def __init__(self, name, linenum):
+    _BlockInfo.__init__(self, False)
+    self.name = name or ''
+    self.starting_linenum = linenum
+    self.check_namespace_indentation = True
+
+  def CheckEnd(self, filename, clean_lines, linenum, error):
+    """Check end of namespace comments."""
+    line = clean_lines.raw_lines[linenum]
+
+    # Check how many lines is enclosed in this namespace.  Don't issue
+    # warning for missing namespace comments if there aren't enough
+    # lines.  However, do apply checks if there is already an end of
+    # namespace comment and it's incorrect.
+    #
+    # TODO(unknown): We always want to check end of namespace comments
+    # if a namespace is large, but sometimes we also want to apply the
+    # check if a short namespace contained nontrivial things (something
+    # other than forward declarations).  There is currently no logic on
+    # deciding what these nontrivial things are, so this check is
+    # triggered by namespace size only, which works most of the time.
+    if (linenum - self.starting_linenum < 10
+        and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)):
+      return
+
+    # Look for matching comment at end of namespace.
+    #
+    # Note that we accept C style "/* */" comments for terminating
+    # namespaces, so that code that terminate namespaces inside
+    # preprocessor macros can be cpplint clean.
+    #
+    # We also accept stuff like "// end of namespace <name>." with the
+    # period at the end.
+    #
+    # Besides these, we don't accept anything else, otherwise we might
+    # get false negatives when existing comment is a substring of the
+    # expected namespace.
+    if self.name:
+      # Named namespace
+      if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) +
+                    r'[\*/\.\\\s]*$'),
+                   line):
+        error(filename, linenum, 'readability/namespace', 5,
+              'Namespace should be terminated with "// namespace %s"' %
+              self.name)
+    else:
+      # Anonymous namespace
+      if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line):
+        # If "// namespace anonymous" or "// anonymous namespace (more text)",
+        # mention "// anonymous namespace" as an acceptable form
+        if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', line):
+          error(filename, linenum, 'readability/namespace', 5,
+                'Anonymous namespace should be terminated with "// namespace"'
+                ' or "// anonymous namespace"')
+        else:
+          error(filename, linenum, 'readability/namespace', 5,
+                'Anonymous namespace should be terminated with "// namespace"')
+
+
+class _PreprocessorInfo(object):
+  """Stores checkpoints of nesting stacks when #if/#else is seen."""
+
+  def __init__(self, stack_before_if):
+    # The entire nesting stack before #if
+    self.stack_before_if = stack_before_if
+
+    # The entire nesting stack up to #else
+    self.stack_before_else = []
+
+    # Whether we have already seen #else or #elif
+    self.seen_else = False
+
+
+class NestingState(object):
+  """Holds states related to parsing braces."""
+
+  def __init__(self):
+    # Stack for tracking all braces.  An object is pushed whenever we
+    # see a "{", and popped when we see a "}".  Only 3 types of
+    # objects are possible:
+    # - _ClassInfo: a class or struct.
+    # - _NamespaceInfo: a namespace.
+    # - _BlockInfo: some other type of block.
+    self.stack = []
+
+    # Top of the previous stack before each Update().
+    #
+    # Because the nesting_stack is updated at the end of each line, we
+    # had to do some convoluted checks to find out what is the current
+    # scope at the beginning of the line.  This check is simplified by
+    # saving the previous top of nesting stack.
+    #
+    # We could save the full stack, but we only need the top.  Copying
+    # the full nesting stack would slow down cpplint by ~10%.
+    self.previous_stack_top = []
+
+    # Stack of _PreprocessorInfo objects.
+    self.pp_stack = []
+
+  def SeenOpenBrace(self):
+    """Check if we have seen the opening brace for the innermost block.
+
+    Returns:
+      True if we have seen the opening brace, False if the innermost
+      block is still expecting an opening brace.
+    """
+    return (not self.stack) or self.stack[-1].seen_open_brace
+
+  def InNamespaceBody(self):
+    """Check if we are currently one level inside a namespace body.
+
+    Returns:
+      True if top of the stack is a namespace block, False otherwise.
+    """
+    return self.stack and isinstance(self.stack[-1], _NamespaceInfo)
+
+  def InExternC(self):
+    """Check if we are currently one level inside an 'extern "C"' block.
+
+    Returns:
+      True if top of the stack is an extern block, False otherwise.
+    """
+    return self.stack and isinstance(self.stack[-1], _ExternCInfo)
+
+  def InClassDeclaration(self):
+    """Check if we are currently one level inside a class or struct declaration.
+
+    Returns:
+      True if top of the stack is a class/struct, False otherwise.
+    """
+    return self.stack and isinstance(self.stack[-1], _ClassInfo)
+
+  def InAsmBlock(self):
+    """Check if we are currently one level inside an inline ASM block.
+
+    Returns:
+      True if the top of the stack is a block containing inline ASM.
+    """
+    return self.stack and self.stack[-1].inline_asm != _NO_ASM
+
+  def InTemplateArgumentList(self, clean_lines, linenum, pos):
+    """Check if current position is inside template argument list.
+
+    Args:
+      clean_lines: A CleansedLines instance containing the file.
+      linenum: The number of the line to check.
+      pos: position just after the suspected template argument.
+    Returns:
+      True if (linenum, pos) is inside template arguments.
+    """
+    while linenum < clean_lines.NumLines():
+      # Find the earliest character that might indicate a template argument
+      line = clean_lines.elided[linenum]
+      match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:])
+      if not match:
+        linenum += 1
+        pos = 0
+        continue
+      token = match.group(1)
+      pos += len(match.group(0))
+
+      # These things do not look like template argument list:
+      #   class Suspect {
+      #   class Suspect x; }
+      if token in ('{', '}', ';'): return False
+
+      # These things look like template argument list:
+      #   template <class Suspect>
+      #   template <class Suspect = default_value>
+      #   template <class Suspect[]>
+      #   template <class Suspect...>
+      if token in ('>', '=', '[', ']', '.'): return True
+
+      # Check if token is an unmatched '<'.
+      # If not, move on to the next character.
+      if token != '<':
+        pos += 1
+        if pos >= len(line):
+          linenum += 1
+          pos = 0
+        continue
+
+      # We can't be sure if we just find a single '<', and need to
+      # find the matching '>'.
+      (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1)
+      if end_pos < 0:
+        # Not sure if template argument list or syntax error in file
+        return False
+      linenum = end_line
+      pos = end_pos
+    return False
+
+  def UpdatePreprocessor(self, line):
+    """Update preprocessor stack.
+
+    We need to handle preprocessors due to classes like this:
+      #ifdef SWIG
+      struct ResultDetailsPageElementExtensionPoint {
+      #else
+      struct ResultDetailsPageElementExtensionPoint : public Extension {
+      #endif
+
+    We make the following assumptions (good enough for most files):
+    - Preprocessor condition evaluates to true from #if up to first
+      #else/#elif/#endif.
+
+    - Preprocessor condition evaluates to false from #else/#elif up
+      to #endif.  We still perform lint checks on these lines, but
+      these do not affect nesting stack.
+
+    Args:
+      line: current line to check.
+    """
+    if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line):
+      # Beginning of #if block, save the nesting stack here.  The saved
+      # stack will allow us to restore the parsing state in the #else case.
+      self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack)))
+    elif Match(r'^\s*#\s*(else|elif)\b', line):
+      # Beginning of #else block
+      if self.pp_stack:
+        if not self.pp_stack[-1].seen_else:
+          # This is the first #else or #elif block.  Remember the
+          # whole nesting stack up to this point.  This is what we
+          # keep after the #endif.
+          self.pp_stack[-1].seen_else = True
+          self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack)
+
+        # Restore the stack to how it was before the #if
+        self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if)
+      else:
+        # TODO(unknown): unexpected #else, issue warning?
+        pass
+    elif Match(r'^\s*#\s*endif\b', line):
+      # End of #if or #else blocks.
+      if self.pp_stack:
+        # If we saw an #else, we will need to restore the nesting
+        # stack to its former state before the #else, otherwise we
+        # will just continue from where we left off.
+        if self.pp_stack[-1].seen_else:
+          # Here we can just use a shallow copy since we are the last
+          # reference to it.
+          self.stack = self.pp_stack[-1].stack_before_else
+        # Drop the corresponding #if
+        self.pp_stack.pop()
+      else:
+        # TODO(unknown): unexpected #endif, issue warning?
+        pass
+
+  # TODO(unknown): Update() is too long, but we will refactor later.
+  def Update(self, filename, clean_lines, linenum, error):
+    """Update nesting state with current line.
+
+    Args:
+      filename: The name of the current file.
+      clean_lines: A CleansedLines instance containing the file.
+      linenum: The number of the line to check.
+      error: The function to call with any errors found.
+    """
+    line = clean_lines.elided[linenum]
+
+    # Remember top of the previous nesting stack.
+    #
+    # The stack is always pushed/popped and not modified in place, so
+    # we can just do a shallow copy instead of copy.deepcopy.  Using
+    # deepcopy would slow down cpplint by ~28%.
+    if self.stack:
+      self.previous_stack_top = self.stack[-1]
+    else:
+      self.previous_stack_top = None
+
+    # Update pp_stack
+    self.UpdatePreprocessor(line)
+
+    # Count parentheses.  This is to avoid adding struct arguments to
+    # the nesting stack.
+    if self.stack:
+      inner_block = self.stack[-1]
+      depth_change = line.count('(') - line.count(')')
+      inner_block.open_parentheses += depth_change
+
+      # Also check if we are starting or ending an inline assembly block.
+      if inner_block.inline_asm in (_NO_ASM, _END_ASM):
+        if (depth_change != 0 and
+            inner_block.open_parentheses == 1 and
+            _MATCH_ASM.match(line)):
+          # Enter assembly block
+          inner_block.inline_asm = _INSIDE_ASM
+        else:
+          # Not entering assembly block.  If previous line was _END_ASM,
+          # we will now shift to _NO_ASM state.
+          inner_block.inline_asm = _NO_ASM
+      elif (inner_block.inline_asm == _INSIDE_ASM and
+            inner_block.open_parentheses == 0):
+        # Exit assembly block
+        inner_block.inline_asm = _END_ASM
+
+    # Consume namespace declaration at the beginning of the line.  Do
+    # this in a loop so that we catch same line declarations like this:
+    #   namespace proto2 { namespace bridge { class MessageSet; } }
+    while True:
+      # Match start of namespace.  The "\b\s*" below catches namespace
+      # declarations even if it weren't followed by a whitespace, this
+      # is so that we don't confuse our namespace checker.  The
+      # missing spaces will be flagged by CheckSpacing.
+      namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line)
+      if not namespace_decl_match:
+        break
+
+      new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum)
+      self.stack.append(new_namespace)
+
+      line = namespace_decl_match.group(2)
+      if line.find('{') != -1:
+        new_namespace.seen_open_brace = True
+        line = line[line.find('{') + 1:]
+
+    # Look for a class declaration in whatever is left of the line
+    # after parsing namespaces.  The regexp accounts for decorated classes
+    # such as in:
+    #   class LOCKABLE API Object {
+    #   };
+    class_decl_match = Match(
+        r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?'
+        r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))'
+        r'(.*)$', line)
+    if (class_decl_match and
+        (not self.stack or self.stack[-1].open_parentheses == 0)):
+      # We do not want to accept classes that are actually template arguments:
+      #   template <class Ignore1,
+      #             class Ignore2 = Default<Args>,
+      #             template <Args> class Ignore3>
+      #   void Function() {};
+      #
+      # To avoid template argument cases, we scan forward and look for
+      # an unmatched '>'.  If we see one, assume we are inside a
+      # template argument list.
+      end_declaration = len(class_decl_match.group(1))
+      if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration):
+        self.stack.append(_ClassInfo(
+            class_decl_match.group(3), class_decl_match.group(2),
+            clean_lines, linenum))
+        line = class_decl_match.group(4)
+
+    # If we have not yet seen the opening brace for the innermost block,
+    # run checks here.
+    if not self.SeenOpenBrace():
+      self.stack[-1].CheckBegin(filename, clean_lines, linenum, error)
+
+    # Update access control if we are inside a class/struct
+    if self.stack and isinstance(self.stack[-1], _ClassInfo):
+      classinfo = self.stack[-1]
+      access_match = Match(
+          r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?'
+          r':(?:[^:]|$)',
+          line)
+      if access_match:
+        classinfo.access = access_match.group(2)
+
+        # Check that access keywords are indented +1 space.  Skip this
+        # check if the keywords are not preceded by whitespaces.
+        indent = access_match.group(1)
+        if (len(indent) != classinfo.class_indent + 1 and
+            Match(r'^\s*$', indent)):
+          if classinfo.is_struct:
+            parent = 'struct ' + classinfo.name
+          else:
+            parent = 'class ' + classinfo.name
+          slots = ''
+          if access_match.group(3):
+            slots = access_match.group(3)
+          error(filename, linenum, 'whitespace/indent', 3,
+                '%s%s: should be indented +1 space inside %s' % (
+                    access_match.group(2), slots, parent))
+
+    # Consume braces or semicolons from what's left of the line
+    while True:
+      # Match first brace, semicolon, or closed parenthesis.
+      matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line)
+      if not matched:
+        break
+
+      token = matched.group(1)
+      if token == '{':
+        # If namespace or class hasn't seen a opening brace yet, mark
+        # namespace/class head as complete.  Push a new block onto the
+        # stack otherwise.
+        if not self.SeenOpenBrace():
+          self.stack[-1].seen_open_brace = True
+        elif Match(r'^extern\s*"[^"]*"\s*\{', line):
+          self.stack.append(_ExternCInfo())
+        else:
+          self.stack.append(_BlockInfo(True))
+          if _MATCH_ASM.match(line):
+            self.stack[-1].inline_asm = _BLOCK_ASM
+
+      elif token == ';' or token == ')':
+        # If we haven't seen an opening brace yet, but we already saw
+        # a semicolon, this is probably a forward declaration.  Pop
+        # the stack for these.
+        #
+        # Similarly, if we haven't seen an opening brace yet, but we
+        # already saw a closing parenthesis, then these are probably
+        # function arguments with extra "class" or "struct" keywords.
+        # Also pop these stack for these.
+        if not self.SeenOpenBrace():
+          self.stack.pop()
+      else:  # token == '}'
+        # Perform end of block checks and pop the stack.
+        if self.stack:
+          self.stack[-1].CheckEnd(filename, clean_lines, linenum, error)
+          self.stack.pop()
+      line = matched.group(2)
+
+  def InnermostClass(self):
+    """Get class info on the top of the stack.
+
+    Returns:
+      A _ClassInfo object if we are inside a class, or None otherwise.
+    """
+    for i in range(len(self.stack), 0, -1):
+      classinfo = self.stack[i - 1]
+      if isinstance(classinfo, _ClassInfo):
+        return classinfo
+    return None
+
+  def CheckCompletedBlocks(self, filename, error):
+    """Checks that all classes and namespaces have been completely parsed.
+
+    Call this when all lines in a file have been processed.
+    Args:
+      filename: The name of the current file.
+      error: The function to call with any errors found.
+    """
+    # Note: This test can result in false positives if #ifdef constructs
+    # get in the way of brace matching. See the testBuildClass test in
+    # cpplint_unittest.py for an example of this.
+    for obj in self.stack:
+      if isinstance(obj, _ClassInfo):
+        error(filename, obj.starting_linenum, 'build/class', 5,
+              'Failed to find complete declaration of class %s' %
+              obj.name)
+      elif isinstance(obj, _NamespaceInfo):
+        error(filename, obj.starting_linenum, 'build/namespaces', 5,
+              'Failed to find complete declaration of namespace %s' %
+              obj.name)
+
+
+def CheckForNonStandardConstructs(filename, clean_lines, linenum,
+                                  nesting_state, error):
+  r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2.
+
+  Complain about several constructs which gcc-2 accepts, but which are
+  not standard C++.  Warning about these in lint is one way to ease the
+  transition to new compilers.
+  - put storage class first (e.g. "static const" instead of "const static").
+  - "%lld" instead of %qd" in printf-type functions.
+  - "%1$d" is non-standard in printf-type functions.
+  - "\%" is an undefined character escape sequence.
+  - text after #endif is not allowed.
+  - invalid inner-style forward declaration.
+  - >? and <? operators, and their >?= and <?= cousins.
+
+  Additionally, check for constructor/destructor style violations and reference
+  members, as it is very convenient to do so while checking for
+  gcc-2 compliance.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    error: A callable to which errors are reported, which takes 4 arguments:
+           filename, line number, error level, and message
+  """
+
+  # Remove comments from the line, but leave in strings for now.
+  line = clean_lines.lines[linenum]
+
+  if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line):
+    error(filename, linenum, 'runtime/printf_format', 3,
+          '%q in format strings is deprecated.  Use %ll instead.')
+
+  if Search(r'printf\s*\(.*".*%\d+\$', line):
+    error(filename, linenum, 'runtime/printf_format', 2,
+          '%N$ formats are unconventional.  Try rewriting to avoid them.')
+
+  # Remove escaped backslashes before looking for undefined escapes.
+  line = line.replace('\\\\', '')
+
+  if Search(r'("|\').*\\(%|\[|\(|{)', line):
+    error(filename, linenum, 'build/printf_format', 3,
+          '%, [, (, and { are undefined character escapes.  Unescape them.')
+
+  # For the rest, work with both comments and strings removed.
+  line = clean_lines.elided[linenum]
+
+  if Search(r'\b(const|volatile|void|char|short|int|long'
+            r'|float|double|signed|unsigned'
+            r'|schar|u?int8|u?int16|u?int32|u?int64)'
+            r'\s+(register|static|extern|typedef)\b',
+            line):
+    error(filename, linenum, 'build/storage_class', 5,
+          'Storage class (static, extern, typedef, etc) should be first.')
+
+  if Match(r'\s*#\s*endif\s*[^/\s]+', line):
+    error(filename, linenum, 'build/endif_comment', 5,
+          'Uncommented text after #endif is non-standard.  Use a comment.')
+
+  if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line):
+    error(filename, linenum, 'build/forward_decl', 5,
+          'Inner-style forward declarations are invalid.  Remove this line.')
+
+  if Search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?',
+            line):
+    error(filename, linenum, 'build/deprecated', 3,
+          '>? and <? (max and min) operators are non-standard and deprecated.')
+
+  if Search(r'^\s*const\s*string\s*&\s*\w+\s*;', line):
+    # TODO(unknown): Could it be expanded safely to arbitrary references,
+    # without triggering too many false positives? The first
+    # attempt triggered 5 warnings for mostly benign code in the regtest, hence
+    # the restriction.
+    # Here's the original regexp, for the reference:
+    # type_name = r'\w+((\s*::\s*\w+)|(\s*<\s*\w+?\s*>))?'
+    # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;'
+    error(filename, linenum, 'runtime/member_string_references', 2,
+          'const string& members are dangerous. It is much better to use '
+          'alternatives, such as pointers or simple constants.')
+
+  # Everything else in this function operates on class declarations.
+  # Return early if the top of the nesting stack is not a class, or if
+  # the class head is not completed yet.
+  classinfo = nesting_state.InnermostClass()
+  if not classinfo or not classinfo.seen_open_brace:
+    return
+
+  # The class may have been declared with namespace or classname qualifiers.
+  # The constructor and destructor will not have those qualifiers.
+  base_classname = classinfo.name.split('::')[-1]
+
+  # Look for single-argument constructors that aren't marked explicit.
+  # Technically a valid construct, but against style. Also look for
+  # non-single-argument constructors which are also technically valid, but
+  # strongly suggest something is wrong.
+  explicit_constructor_match = Match(
+      r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*'
+      r'\(((?:[^()]|\([^()]*\))*)\)'
+      % re.escape(base_classname),
+      line)
+
+  if explicit_constructor_match:
+    is_marked_explicit = explicit_constructor_match.group(1)
+
+    if not explicit_constructor_match.group(2):
+      constructor_args = []
+    else:
+      constructor_args = explicit_constructor_match.group(2).split(',')
+
+    # collapse arguments so that commas in template parameter lists and function
+    # argument parameter lists don't split arguments in two
+    i = 0
+    while i < len(constructor_args):
+      constructor_arg = constructor_args[i]
+      while (constructor_arg.count('<') > constructor_arg.count('>') or
+             constructor_arg.count('(') > constructor_arg.count(')')):
+        constructor_arg += ',' + constructor_args[i + 1]
+        del constructor_args[i + 1]
+      constructor_args[i] = constructor_arg
+      i += 1
+
+    defaulted_args = [arg for arg in constructor_args if '=' in arg]
+    noarg_constructor = (not constructor_args or  # empty arg list
+                         # 'void' arg specifier
+                         (len(constructor_args) == 1 and
+                          constructor_args[0].strip() == 'void'))
+    onearg_constructor = ((len(constructor_args) == 1 and  # exactly one arg
+                           not noarg_constructor) or
+                          # all but at most one arg defaulted
+                          (len(constructor_args) >= 1 and
+                           not noarg_constructor and
+                           len(defaulted_args) >= len(constructor_args) - 1))
+    initializer_list_constructor = bool(
+        onearg_constructor and
+        Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0]))
+    copy_constructor = bool(
+        onearg_constructor and
+        Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&'
+              % re.escape(base_classname), constructor_args[0].strip()))
+
+    if (not is_marked_explicit and
+        onearg_constructor and
+        not initializer_list_constructor and
+        not copy_constructor):
+      if defaulted_args:
+        error(filename, linenum, 'runtime/explicit', 5,
+              'Constructors callable with one argument '
+              'should be marked explicit.')
+      else:
+        error(filename, linenum, 'runtime/explicit', 5,
+              'Single-parameter constructors should be marked explicit.')
+    elif is_marked_explicit and not onearg_constructor:
+      if noarg_constructor:
+        error(filename, linenum, 'runtime/explicit', 5,
+              'Zero-parameter constructors should not be marked explicit.')
+      else:
+        error(filename, linenum, 'runtime/explicit', 0,
+              'Constructors that require multiple arguments '
+              'should not be marked explicit.')
+
+
+def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error):
+  """Checks for the correctness of various spacing around function calls.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # Since function calls often occur inside if/for/while/switch
+  # expressions - which have their own, more liberal conventions - we
+  # first see if we should be looking inside such an expression for a
+  # function call, to which we can apply more strict standards.
+  fncall = line    # if there's no control flow construct, look at whole line
+  for pattern in (r'\bif\s*\((.*)\)\s*{',
+                  r'\bfor\s*\((.*)\)\s*{',
+                  r'\bwhile\s*\((.*)\)\s*[{;]',
+                  r'\bswitch\s*\((.*)\)\s*{'):
+    match = Search(pattern, line)
+    if match:
+      fncall = match.group(1)    # look inside the parens for function calls
+      break
+
+  # Except in if/for/while/switch, there should never be space
+  # immediately inside parens (eg "f( 3, 4 )").  We make an exception
+  # for nested parens ( (a+b) + c ).  Likewise, there should never be
+  # a space before a ( when it's a function argument.  I assume it's a
+  # function argument when the char before the whitespace is legal in
+  # a function name (alnum + _) and we're not starting a macro. Also ignore
+  # pointers and references to arrays and functions coz they're too tricky:
+  # we use a very simple way to recognize these:
+  # " (something)(maybe-something)" or
+  # " (something)(maybe-something," or
+  # " (something)[something]"
+  # Note that we assume the contents of [] to be short enough that
+  # they'll never need to wrap.
+  if (  # Ignore control structures.
+      not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b',
+                 fncall) and
+      # Ignore pointers/references to functions.
+      not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and
+      # Ignore pointers/references to arrays.
+      not Search(r' \([^)]+\)\[[^\]]+\]', fncall)):
+    if Search(r'\w\s*\(\s(?!\s*\\$)', fncall):      # a ( used for a fn call
+      error(filename, linenum, 'whitespace/parens', 4,
+            'Extra space after ( in function call')
+    elif Search(r'\(\s+(?!(\s*\\)|\()', fncall):
+      error(filename, linenum, 'whitespace/parens', 2,
+            'Extra space after (')
+    if (Search(r'\w\s+\(', fncall) and
+        not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and
+        not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and
+        not Search(r'\bcase\s+\(', fncall)):
+      # TODO(unknown): Space after an operator function seem to be a common
+      # error, silence those for now by restricting them to highest verbosity.
+      if Search(r'\boperator_*\b', line):
+        error(filename, linenum, 'whitespace/parens', 0,
+              'Extra space before ( in function call')
+      else:
+        error(filename, linenum, 'whitespace/parens', 4,
+              'Extra space before ( in function call')
+    # If the ) is followed only by a newline or a { + newline, assume it's
+    # part of a control statement (if/while/etc), and don't complain
+    if Search(r'[^)]\s+\)\s*[^{\s]', fncall):
+      # If the closing parenthesis is preceded by only whitespaces,
+      # try to give a more descriptive error message.
+      if Search(r'^\s+\)', fncall):
+        error(filename, linenum, 'whitespace/parens', 2,
+              'Closing ) should be moved to the previous line')
+      else:
+        error(filename, linenum, 'whitespace/parens', 2,
+              'Extra space before )')
+
+
+def IsBlankLine(line):
+  """Returns true if the given line is blank.
+
+  We consider a line to be blank if the line is empty or consists of
+  only white spaces.
+
+  Args:
+    line: A line of a string.
+
+  Returns:
+    True, if the given line is blank.
+  """
+  return not line or line.isspace()
+
+
+def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line,
+                                 error):
+  is_namespace_indent_item = (
+      len(nesting_state.stack) > 1 and
+      nesting_state.stack[-1].check_namespace_indentation and
+      isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and
+      nesting_state.previous_stack_top == nesting_state.stack[-2])
+
+  if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item,
+                                     clean_lines.elided, line):
+    CheckItemIndentationInNamespace(filename, clean_lines.elided,
+                                    line, error)
+
+
+def CheckForFunctionLengths(filename, clean_lines, linenum,
+                            function_state, error):
+  """Reports for long function bodies.
+
+  For an overview why this is done, see:
+  http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions
+
+  Uses a simplistic algorithm assuming other style guidelines
+  (especially spacing) are followed.
+  Only checks unindented functions, so class members are unchecked.
+  Trivial bodies are unchecked, so constructors with huge initializer lists
+  may be missed.
+  Blank/comment lines are not counted so as to avoid encouraging the removal
+  of vertical space and comments just to get through a lint check.
+  NOLINT *on the last line of a function* disables this check.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    function_state: Current function name and lines in body so far.
+    error: The function to call with any errors found.
+  """
+  lines = clean_lines.lines
+  line = lines[linenum]
+  joined_line = ''
+
+  starting_func = False
+  regexp = r'(\w(\w|::|\*|\&|\s)*)\('  # decls * & space::name( ...
+  match_result = Match(regexp, line)
+  if match_result:
+    # If the name is all caps and underscores, figure it's a macro and
+    # ignore it, unless it's TEST or TEST_F.
+    function_name = match_result.group(1).split()[-1]
+    if function_name == 'TEST' or function_name == 'TEST_F' or (
+        not Match(r'[A-Z_]+$', function_name)):
+      starting_func = True
+
+  if starting_func:
+    body_found = False
+    for start_linenum in xrange(linenum, clean_lines.NumLines()):
+      start_line = lines[start_linenum]
+      joined_line += ' ' + start_line.lstrip()
+      if Search(r'(;|})', start_line):  # Declarations and trivial functions
+        body_found = True
+        break                              # ... ignore
+      elif Search(r'{', start_line):
+        body_found = True
+        function = Search(r'((\w|:)*)\(', line).group(1)
+        if Match(r'TEST', function):    # Handle TEST... macros
+          parameter_regexp = Search(r'(\(.*\))', joined_line)
+          if parameter_regexp:             # Ignore bad syntax
+            function += parameter_regexp.group(1)
+        else:
+          function += '()'
+        function_state.Begin(function)
+        break
+    if not body_found:
+      # No body for the function (or evidence of a non-function) was found.
+      error(filename, linenum, 'readability/fn_size', 5,
+            'Lint failed to find start of function body.')
+  elif Match(r'^\}\s*$', line):  # function end
+    function_state.Check(error, filename, linenum)
+    function_state.End()
+  elif not Match(r'^\s*$', line):
+    function_state.Count()  # Count non-blank/non-comment lines.
+
+
+_RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?')
+
+
+def CheckComment(line, filename, linenum, next_line_start, error):
+  """Checks for common mistakes in comments.
+
+  Args:
+    line: The line in question.
+    filename: The name of the current file.
+    linenum: The number of the line to check.
+    next_line_start: The first non-whitespace column of the next line.
+    error: The function to call with any errors found.
+  """
+  commentpos = line.find('//')
+  if commentpos != -1:
+    # Check if the // may be in quotes.  If so, ignore it
+    # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison
+    if (line.count('"', 0, commentpos) -
+        line.count('\\"', 0, commentpos)) % 2 == 0:   # not in quotes
+      # Allow one space for new scopes, two spaces otherwise:
+      if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and
+          ((commentpos >= 1 and
+            line[commentpos-1] not in string.whitespace) or
+           (commentpos >= 2 and
+            line[commentpos-2] not in string.whitespace))):
+        error(filename, linenum, 'whitespace/comments', 2,
+              'At least two spaces is best between code and comments')
+
+      # Checks for common mistakes in TODO comments.
+      comment = line[commentpos:]
+      match = _RE_PATTERN_TODO.match(comment)
+      if match:
+        # One whitespace is correct; zero whitespace is handled elsewhere.
+        leading_whitespace = match.group(1)
+        if len(leading_whitespace) > 1:
+          error(filename, linenum, 'whitespace/todo', 2,
+                'Too many spaces before TODO')
+
+        username = match.group(2)
+        if not username:
+          error(filename, linenum, 'readability/todo', 2,
+                'Missing username in TODO; it should look like '
+                '"// TODO(my_username): Stuff."')
+
+        middle_whitespace = match.group(3)
+        # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison
+        if middle_whitespace != ' ' and middle_whitespace != '':
+          error(filename, linenum, 'whitespace/todo', 2,
+                'TODO(my_username) should be followed by a space')
+
+      # If the comment contains an alphanumeric character, there
+      # should be a space somewhere between it and the // unless
+      # it's a /// or //! Doxygen comment.
+      if (Match(r'//[^ ]*\w', comment) and
+          not Match(r'(///|//\!)(\s+|$)', comment)):
+        error(filename, linenum, 'whitespace/comments', 4,
+              'Should have a space between // and comment')
+
+
+def CheckAccess(filename, clean_lines, linenum, nesting_state, error):
+  """Checks for improper use of DISALLOW* macros.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]  # get rid of comments and strings
+
+  matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|'
+                   r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line)
+  if not matched:
+    return
+  if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo):
+    if nesting_state.stack[-1].access != 'private':
+      error(filename, linenum, 'readability/constructors', 3,
+            '%s must be in the private: section' % matched.group(1))
+
+  else:
+    # Found DISALLOW* macro outside a class declaration, or perhaps it
+    # was used inside a function when it should have been part of the
+    # class declaration.  We could issue a warning here, but it
+    # probably resulted in a compiler error already.
+    pass
+
+
+def CheckSpacing(filename, clean_lines, linenum, nesting_state, error):
+  """Checks for the correctness of various spacing issues in the code.
+
+  Things we check for: spaces around operators, spaces after
+  if/for/while/switch, no spaces around parens in function calls, two
+  spaces between code and comment, don't start a block with a blank
+  line, don't end a function with a blank line, don't add a blank line
+  after public/protected/private, don't have too many blank lines in a row.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    error: The function to call with any errors found.
+  """
+
+  # Don't use "elided" lines here, otherwise we can't check commented lines.
+  # Don't want to use "raw" either, because we don't want to check inside C++11
+  # raw strings,
+  raw = clean_lines.lines_without_raw_strings
+  line = raw[linenum]
+
+  # Before nixing comments, check if the line is blank for no good
+  # reason.  This includes the first line after a block is opened, and
+  # blank lines at the end of a function (ie, right before a line like '}'
+  #
+  # Skip all the blank line checks if we are immediately inside a
+  # namespace body.  In other words, don't issue blank line warnings
+  # for this block:
+  #   namespace {
+  #
+  #   }
+  #
+  # A warning about missing end of namespace comments will be issued instead.
+  #
+  # Also skip blank line checks for 'extern "C"' blocks, which are formatted
+  # like namespaces.
+  if (IsBlankLine(line) and
+      not nesting_state.InNamespaceBody() and
+      not nesting_state.InExternC()):
+    elided = clean_lines.elided
+    prev_line = elided[linenum - 1]
+    prevbrace = prev_line.rfind('{')
+    # TODO(unknown): Don't complain if line before blank line, and line after,
+    #                both start with alnums and are indented the same amount.
+    #                This ignores whitespace at the start of a namespace block
+    #                because those are not usually indented.
+    if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1:
+      # OK, we have a blank line at the start of a code block.  Before we
+      # complain, we check if it is an exception to the rule: The previous
+      # non-empty line has the parameters of a function header that are indented
+      # 4 spaces (because they did not fit in a 80 column line when placed on
+      # the same line as the function name).  We also check for the case where
+      # the previous line is indented 6 spaces, which may happen when the
+      # initializers of a constructor do not fit into a 80 column line.
+      exception = False
+      if Match(r' {6}\w', prev_line):  # Initializer list?
+        # We are looking for the opening column of initializer list, which
+        # should be indented 4 spaces to cause 6 space indentation afterwards.
+        search_position = linenum-2
+        while (search_position >= 0
+               and Match(r' {6}\w', elided[search_position])):
+          search_position -= 1
+        exception = (search_position >= 0
+                     and elided[search_position][:5] == '    :')
+      else:
+        # Search for the function arguments or an initializer list.  We use a
+        # simple heuristic here: If the line is indented 4 spaces; and we have a
+        # closing paren, without the opening paren, followed by an opening brace
+        # or colon (for initializer lists) we assume that it is the last line of
+        # a function header.  If we have a colon indented 4 spaces, it is an
+        # initializer list.
+        exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)',
+                           prev_line)
+                     or Match(r' {4}:', prev_line))
+
+      if not exception:
+        error(filename, linenum, 'whitespace/blank_line', 2,
+              'Redundant blank line at the start of a code block '
+              'should be deleted.')
+    # Ignore blank lines at the end of a block in a long if-else
+    # chain, like this:
+    #   if (condition1) {
+    #     // Something followed by a blank line
+    #
+    #   } else if (condition2) {
+    #     // Something else
+    #   }
+    if linenum + 1 < clean_lines.NumLines():
+      next_line = raw[linenum + 1]
+      if (next_line
+          and Match(r'\s*}', next_line)
+          and next_line.find('} else ') == -1):
+        error(filename, linenum, 'whitespace/blank_line', 3,
+              'Redundant blank line at the end of a code block '
+              'should be deleted.')
+
+    matched = Match(r'\s*(public|protected|private):', prev_line)
+    if matched:
+      error(filename, linenum, 'whitespace/blank_line', 3,
+            'Do not leave a blank line after "%s:"' % matched.group(1))
+
+  # Next, check comments
+  next_line_start = 0
+  if linenum + 1 < clean_lines.NumLines():
+    next_line = raw[linenum + 1]
+    next_line_start = len(next_line) - len(next_line.lstrip())
+  CheckComment(line, filename, linenum, next_line_start, error)
+
+  # get rid of comments and strings
+  line = clean_lines.elided[linenum]
+
+  # You shouldn't have spaces before your brackets, except maybe after
+  # 'delete []' or 'return []() {};'
+  if Search(r'\w\s+\[', line) and not Search(r'(?:delete|return)\s+\[', line):
+    error(filename, linenum, 'whitespace/braces', 5,
+          'Extra space before [')
+
+  # In range-based for, we wanted spaces before and after the colon, but
+  # not around "::" tokens that might appear.
+  if (Search(r'for *\(.*[^:]:[^: ]', line) or
+      Search(r'for *\(.*[^: ]:[^:]', line)):
+    error(filename, linenum, 'whitespace/forcolon', 2,
+          'Missing space around colon in range-based for loop')
+
+
+def CheckOperatorSpacing(filename, clean_lines, linenum, error):
+  """Checks for horizontal spacing around operators.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # Don't try to do spacing checks for operator methods.  Do this by
+  # replacing the troublesome characters with something else,
+  # preserving column position for all other characters.
+  #
+  # The replacement is done repeatedly to avoid false positives from
+  # operators that call operators.
+  while True:
+    match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line)
+    if match:
+      line = match.group(1) + ('_' * len(match.group(2))) + match.group(3)
+    else:
+      break
+
+  # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )".
+  # Otherwise not.  Note we only check for non-spaces on *both* sides;
+  # sometimes people put non-spaces on one side when aligning ='s among
+  # many lines (not that this is behavior that I approve of...)
+  if ((Search(r'[\w.]=', line) or
+       Search(r'=[\w.]', line))
+      and not Search(r'\b(if|while|for) ', line)
+      # Operators taken from [lex.operators] in C++11 standard.
+      and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line)
+      and not Search(r'operator=', line)):
+    error(filename, linenum, 'whitespace/operators', 4,
+          'Missing spaces around =')
+
+  # It's ok not to have spaces around binary operators like + - * /, but if
+  # there's too little whitespace, we get concerned.  It's hard to tell,
+  # though, so we punt on this one for now.  TODO.
+
+  # You should always have whitespace around binary operators.
+  #
+  # Check <= and >= first to avoid false positives with < and >, then
+  # check non-include lines for spacing around < and >.
+  #
+  # If the operator is followed by a comma, assume it's be used in a
+  # macro context and don't do any checks.  This avoids false
+  # positives.
+  #
+  # Note that && is not included here.  Those are checked separately
+  # in CheckRValueReference
+  match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line)
+  if match:
+    error(filename, linenum, 'whitespace/operators', 3,
+          'Missing spaces around %s' % match.group(1))
+  elif not Match(r'#.*include', line):
+    # Look for < that is not surrounded by spaces.  This is only
+    # triggered if both sides are missing spaces, even though
+    # technically should should flag if at least one side is missing a
+    # space.  This is done to avoid some false positives with shifts.
+    match = Match(r'^(.*[^\s<])<[^\s=<,]', line)
+    if match:
+      (_, _, end_pos) = CloseExpression(
+          clean_lines, linenum, len(match.group(1)))
+      if end_pos <= -1:
+        error(filename, linenum, 'whitespace/operators', 3,
+              'Missing spaces around <')
+
+    # Look for > that is not surrounded by spaces.  Similar to the
+    # above, we only trigger if both sides are missing spaces to avoid
+    # false positives with shifts.
+    match = Match(r'^(.*[^-\s>])>[^\s=>,]', line)
+    if match:
+      (_, _, start_pos) = ReverseCloseExpression(
+          clean_lines, linenum, len(match.group(1)))
+      if start_pos <= -1:
+        error(filename, linenum, 'whitespace/operators', 3,
+              'Missing spaces around >')
+
+  # We allow no-spaces around << when used like this: 10<<20, but
+  # not otherwise (particularly, not when used as streams)
+  #
+  # We also allow operators following an opening parenthesis, since
+  # those tend to be macros that deal with operators.
+  match = Search(r'(operator|[^\s(<])(?:L|UL|ULL|l|ul|ull)?<<([^\s,=<])', line)
+  if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and
+      not (match.group(1) == 'operator' and match.group(2) == ';')):
+    error(filename, linenum, 'whitespace/operators', 3,
+          'Missing spaces around <<')
+
+  # We allow no-spaces around >> for almost anything.  This is because
+  # C++11 allows ">>" to close nested templates, which accounts for
+  # most cases when ">>" is not followed by a space.
+  #
+  # We still warn on ">>" followed by alpha character, because that is
+  # likely due to ">>" being used for right shifts, e.g.:
+  #   value >> alpha
+  #
+  # When ">>" is used to close templates, the alphanumeric letter that
+  # follows would be part of an identifier, and there should still be
+  # a space separating the template type and the identifier.
+  #   type<type<type>> alpha
+  match = Search(r'>>[a-zA-Z_]', line)
+  if match:
+    error(filename, linenum, 'whitespace/operators', 3,
+          'Missing spaces around >>')
+
+  # There shouldn't be space around unary operators
+  match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line)
+  if match:
+    error(filename, linenum, 'whitespace/operators', 4,
+          'Extra space for operator %s' % match.group(1))
+
+
+def CheckParenthesisSpacing(filename, clean_lines, linenum, error):
+  """Checks for horizontal spacing around parentheses.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # No spaces after an if, while, switch, or for
+  match = Search(r' (if\(|for\(|while\(|switch\()', line)
+  if match:
+    error(filename, linenum, 'whitespace/parens', 5,
+          'Missing space before ( in %s' % match.group(1))
+
+  # For if/for/while/switch, the left and right parens should be
+  # consistent about how many spaces are inside the parens, and
+  # there should either be zero or one spaces inside the parens.
+  # We don't want: "if ( foo)" or "if ( foo   )".
+  # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed.
+  match = Search(r'\b(if|for|while|switch)\s*'
+                 r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$',
+                 line)
+  if match:
+    if len(match.group(2)) != len(match.group(4)):
+      if not (match.group(3) == ';' and
+              len(match.group(2)) == 1 + len(match.group(4)) or
+              not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)):
+        error(filename, linenum, 'whitespace/parens', 5,
+              'Mismatching spaces inside () in %s' % match.group(1))
+    if len(match.group(2)) not in [0, 1]:
+      error(filename, linenum, 'whitespace/parens', 5,
+            'Should have zero or one spaces inside ( and ) in %s' %
+            match.group(1))
+
+
+def CheckCommaSpacing(filename, clean_lines, linenum, error):
+  """Checks for horizontal spacing near commas and semicolons.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  raw = clean_lines.lines_without_raw_strings
+  line = clean_lines.elided[linenum]
+
+  # You should always have a space after a comma (either as fn arg or operator)
+  #
+  # This does not apply when the non-space character following the
+  # comma is another comma, since the only time when that happens is
+  # for empty macro arguments.
+  #
+  # We run this check in two passes: first pass on elided lines to
+  # verify that lines contain missing whitespaces, second pass on raw
+  # lines to confirm that those missing whitespaces are not due to
+  # elided comments.
+  if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and
+      Search(r',[^,\s]', raw[linenum])):
+    error(filename, linenum, 'whitespace/comma', 3,
+          'Missing space after ,')
+
+  # You should always have a space after a semicolon
+  # except for few corner cases
+  # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more
+  # space after ;
+  if Search(r';[^\s};\\)/]', line):
+    error(filename, linenum, 'whitespace/semicolon', 3,
+          'Missing space after ;')
+
+
+def CheckBracesSpacing(filename, clean_lines, linenum, error):
+  """Checks for horizontal spacing near commas.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # Except after an opening paren, or after another opening brace (in case of
+  # an initializer list, for instance), you should have spaces before your
+  # braces. And since you should never have braces at the beginning of a line,
+  # this is an easy test.
+  match = Match(r'^(.*[^ ({>]){', line)
+  if match:
+    # Try a bit harder to check for brace initialization.  This
+    # happens in one of the following forms:
+    #   Constructor() : initializer_list_{} { ... }
+    #   Constructor{}.MemberFunction()
+    #   Type variable{};
+    #   FunctionCall(type{}, ...);
+    #   LastArgument(..., type{});
+    #   LOG(INFO) << type{} << " ...";
+    #   map_of_type[{...}] = ...;
+    #   ternary = expr ? new type{} : nullptr;
+    #   OuterTemplate<InnerTemplateConstructor<Type>{}>
+    #
+    # We check for the character following the closing brace, and
+    # silence the warning if it's one of those listed above, i.e.
+    # "{.;,)<>]:".
+    #
+    # To account for nested initializer list, we allow any number of
+    # closing braces up to "{;,)<".  We can't simply silence the
+    # warning on first sight of closing brace, because that would
+    # cause false negatives for things that are not initializer lists.
+    #   Silence this:         But not this:
+    #     Outer{                if (...) {
+    #       Inner{...}            if (...){  // Missing space before {
+    #     };                    }
+    #
+    # There is a false negative with this approach if people inserted
+    # spurious semicolons, e.g. "if (cond){};", but we will catch the
+    # spurious semicolon with a separate check.
+    (endline, endlinenum, endpos) = CloseExpression(
+        clean_lines, linenum, len(match.group(1)))
+    trailing_text = ''
+    if endpos > -1:
+      trailing_text = endline[endpos:]
+    for offset in xrange(endlinenum + 1,
+                         min(endlinenum + 3, clean_lines.NumLines() - 1)):
+      trailing_text += clean_lines.elided[offset]
+    if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text):
+      error(filename, linenum, 'whitespace/braces', 5,
+            'Missing space before {')
+
+  # Make sure '} else {' has spaces.
+  if Search(r'}else', line):
+    error(filename, linenum, 'whitespace/braces', 5,
+          'Missing space before else')
+
+  # You shouldn't have a space before a semicolon at the end of the line.
+  # There's a special case for "for" since the style guide allows space before
+  # the semicolon there.
+  if Search(r':\s*;\s*$', line):
+    error(filename, linenum, 'whitespace/semicolon', 5,
+          'Semicolon defining empty statement. Use {} instead.')
+  elif Search(r'^\s*;\s*$', line):
+    error(filename, linenum, 'whitespace/semicolon', 5,
+          'Line contains only semicolon. If this should be an empty statement, '
+          'use {} instead.')
+  elif (Search(r'\s+;\s*$', line) and
+        not Search(r'\bfor\b', line)):
+    error(filename, linenum, 'whitespace/semicolon', 5,
+          'Extra space before last semicolon. If this should be an empty '
+          'statement, use {} instead.')
+
+
+def IsDecltype(clean_lines, linenum, column):
+  """Check if the token ending on (linenum, column) is decltype().
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: the number of the line to check.
+    column: end column of the token to check.
+  Returns:
+    True if this token is decltype() expression, False otherwise.
+  """
+  (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column)
+  if start_col < 0:
+    return False
+  if Search(r'\bdecltype\s*$', text[0:start_col]):
+    return True
+  return False
+
+
+def IsTemplateParameterList(clean_lines, linenum, column):
+  """Check if the token ending on (linenum, column) is the end of template<>.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: the number of the line to check.
+    column: end column of the token to check.
+  Returns:
+    True if this token is end of a template parameter list, False otherwise.
+  """
+  (_, startline, startpos) = ReverseCloseExpression(
+      clean_lines, linenum, column)
+  if (startpos > -1 and
+      Search(r'\btemplate\s*$', clean_lines.elided[startline][0:startpos])):
+    return True
+  return False
+
+
+def IsRValueType(typenames, clean_lines, nesting_state, linenum, column):
+  """Check if the token ending on (linenum, column) is a type.
+
+  Assumes that text to the right of the column is "&&" or a function
+  name.
+
+  Args:
+    typenames: set of type names from template-argument-list.
+    clean_lines: A CleansedLines instance containing the file.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    linenum: the number of the line to check.
+    column: end column of the token to check.
+  Returns:
+    True if this token is a type, False if we are not sure.
+  """
+  prefix = clean_lines.elided[linenum][0:column]
+
+  # Get one word to the left.  If we failed to do so, this is most
+  # likely not a type, since it's unlikely that the type name and "&&"
+  # would be split across multiple lines.
+  match = Match(r'^(.*)(\b\w+|[>*)&])\s*$', prefix)
+  if not match:
+    return False
+
+  # Check text following the token.  If it's "&&>" or "&&," or "&&...", it's
+  # most likely a rvalue reference used inside a template.
+  suffix = clean_lines.elided[linenum][column:]
+  if Match(r'&&\s*(?:[>,]|\.\.\.)', suffix):
+    return True
+
+  # Check for known types and end of templates:
+  #   int&& variable
+  #   vector<int>&& variable
+  #
+  # Because this function is called recursively, we also need to
+  # recognize pointer and reference types:
+  #   int* Function()
+  #   int& Function()
+  if (match.group(2) in typenames or
+      match.group(2) in ['char', 'char16_t', 'char32_t', 'wchar_t', 'bool',
+                         'short', 'int', 'long', 'signed', 'unsigned',
+                         'float', 'double', 'void', 'auto', '>', '*', '&']):
+    return True
+
+  # If we see a close parenthesis, look for decltype on the other side.
+  # decltype would unambiguously identify a type, anything else is
+  # probably a parenthesized expression and not a type.
+  if match.group(2) == ')':
+    return IsDecltype(
+        clean_lines, linenum, len(match.group(1)) + len(match.group(2)) - 1)
+
+  # Check for casts and cv-qualifiers.
+  #   match.group(1)  remainder
+  #   --------------  ---------
+  #   const_cast<     type&&
+  #   const           type&&
+  #   type            const&&
+  if Search(r'\b(?:const_cast\s*<|static_cast\s*<|dynamic_cast\s*<|'
+            r'reinterpret_cast\s*<|\w+\s)\s*$',
+            match.group(1)):
+    return True
+
+  # Look for a preceding symbol that might help differentiate the context.
+  # These are the cases that would be ambiguous:
+  #   match.group(1)  remainder
+  #   --------------  ---------
+  #   Call         (   expression &&
+  #   Declaration  (   type&&
+  #   sizeof       (   type&&
+  #   if           (   expression &&
+  #   while        (   expression &&
+  #   for          (   type&&
+  #   for(         ;   expression &&
+  #   statement    ;   type&&
+  #   block        {   type&&
+  #   constructor  {   expression &&
+  start = linenum
+  line = match.group(1)
+  match_symbol = None
+  while start >= 0:
+    # We want to skip over identifiers and commas to get to a symbol.
+    # Commas are skipped so that we can find the opening parenthesis
+    # for function parameter lists.
+    match_symbol = Match(r'^(.*)([^\w\s,])[\w\s,]*$', line)
+    if match_symbol:
+      break
+    start -= 1
+    line = clean_lines.elided[start]
+
+  if not match_symbol:
+    # Probably the first statement in the file is an rvalue reference
+    return True
+
+  if match_symbol.group(2) == '}':
+    # Found closing brace, probably an indicate of this:
+    #   block{} type&&
+    return True
+
+  if match_symbol.group(2) == ';':
+    # Found semicolon, probably one of these:
+    #   for(; expression &&
+    #   statement; type&&
+
+    # Look for the previous 'for(' in the previous lines.
+    before_text = match_symbol.group(1)
+    for i in xrange(start - 1, max(start - 6, 0), -1):
+      before_text = clean_lines.elided[i] + before_text
+    if Search(r'for\s*\([^{};]*$', before_text):
+      # This is the condition inside a for-loop
+      return False
+
+    # Did not find a for-init-statement before this semicolon, so this
+    # is probably a new statement and not a condition.
+    return True
+
+  if match_symbol.group(2) == '{':
+    # Found opening brace, probably one of these:
+    #   block{ type&& = ... ; }
+    #   constructor{ expression && expression }
+
+    # Look for a closing brace or a semicolon.  If we see a semicolon
+    # first, this is probably a rvalue reference.
+    line = clean_lines.elided[start][0:len(match_symbol.group(1)) + 1]
+    end = start
+    depth = 1
+    while True:
+      for ch in line:
+        if ch == ';':
+          return True
+        elif ch == '{':
+          depth += 1
+        elif ch == '}':
+          depth -= 1
+          if depth == 0:
+            return False
+      end += 1
+      if end >= clean_lines.NumLines():
+        break
+      line = clean_lines.elided[end]
+    # Incomplete program?
+    return False
+
+  if match_symbol.group(2) == '(':
+    # Opening parenthesis.  Need to check what's to the left of the
+    # parenthesis.  Look back one extra line for additional context.
+    before_text = match_symbol.group(1)
+    if linenum > 1:
+      before_text = clean_lines.elided[linenum - 1] + before_text
+    before_text = match_symbol.group(1)
+
+    # Patterns that are likely to be types:
+    #   [](type&&
+    #   for (type&&
+    #   sizeof(type&&
+    #   operator=(type&&
+    #
+    if Search(r'(?:\]|\bfor|\bsizeof|\boperator\s*\S+\s*)\s*$', before_text):
+      return True
+
+    # Patterns that are likely to be expressions:
+    #   if (expression &&
+    #   while (expression &&
+    #   : initializer(expression &&
+    #   , initializer(expression &&
+    #   ( FunctionCall(expression &&
+    #   + FunctionCall(expression &&
+    #   + (expression &&
+    #
+    # The last '+' represents operators such as '+' and '-'.
+    if Search(r'(?:\bif|\bwhile|[-+=%^(<!?:,&*]\s*)$', before_text):
+      return False
+
+    # Something else.  Check that tokens to the left look like
+    #   return_type function_name
+    match_func = Match(r'^(.*\S.*)\s+\w(?:\w|::)*(?:<[^<>]*>)?\s*$',
+                       match_symbol.group(1))
+    if match_func:
+      # Check for constructors, which don't have return types.
+      if Search(r'\b(?:explicit|inline)$', match_func.group(1)):
+        return True
+      implicit_constructor = Match(r'\s*(\w+)\((?:const\s+)?(\w+)', prefix)
+      if (implicit_constructor and
+          implicit_constructor.group(1) == implicit_constructor.group(2)):
+        return True
+      return IsRValueType(typenames, clean_lines, nesting_state, linenum,
+                          len(match_func.group(1)))
+
+    # Nothing before the function name.  If this is inside a block scope,
+    # this is probably a function call.
+    return not (nesting_state.previous_stack_top and
+                nesting_state.previous_stack_top.IsBlockInfo())
+
+  if match_symbol.group(2) == '>':
+    # Possibly a closing bracket, check that what's on the other side
+    # looks like the start of a template.
+    return IsTemplateParameterList(
+        clean_lines, start, len(match_symbol.group(1)))
+
+  # Some other symbol, usually something like "a=b&&c".  This is most
+  # likely not a type.
+  return False
+
+
+def IsDeletedOrDefault(clean_lines, linenum):
+  """Check if current constructor or operator is deleted or default.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+  Returns:
+    True if this is a deleted or default constructor.
+  """
+  open_paren = clean_lines.elided[linenum].find('(')
+  if open_paren < 0:
+    return False
+  (close_line, _, close_paren) = CloseExpression(
+      clean_lines, linenum, open_paren)
+  if close_paren < 0:
+    return False
+  return Match(r'\s*=\s*(?:delete|default)\b', close_line[close_paren:])
+
+
+def IsRValueAllowed(clean_lines, linenum, typenames):
+  """Check if RValue reference is allowed on a particular line.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    typenames: set of type names from template-argument-list.
+  Returns:
+    True if line is within the region where RValue references are allowed.
+  """
+  # Allow region marked by PUSH/POP macros
+  for i in xrange(linenum, 0, -1):
+    line = clean_lines.elided[i]
+    if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line):
+      if not line.endswith('PUSH'):
+        return False
+      for j in xrange(linenum, clean_lines.NumLines(), 1):
+        line = clean_lines.elided[j]
+        if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line):
+          return line.endswith('POP')
+
+  # Allow operator=
+  line = clean_lines.elided[linenum]
+  if Search(r'\boperator\s*=\s*\(', line):
+    return IsDeletedOrDefault(clean_lines, linenum)
+
+  # Allow constructors
+  match = Match(r'\s*(?:[\w<>]+::)*([\w<>]+)\s*::\s*([\w<>]+)\s*\(', line)
+  if match and match.group(1) == match.group(2):
+    return IsDeletedOrDefault(clean_lines, linenum)
+  if Search(r'\b(?:explicit|inline)\s+[\w<>]+\s*\(', line):
+    return IsDeletedOrDefault(clean_lines, linenum)
+
+  if Match(r'\s*[\w<>]+\s*\(', line):
+    previous_line = 'ReturnType'
+    if linenum > 0:
+      previous_line = clean_lines.elided[linenum - 1]
+    if Match(r'^\s*$', previous_line) or Search(r'[{}:;]\s*$', previous_line):
+      return IsDeletedOrDefault(clean_lines, linenum)
+
+  # Reject types not mentioned in template-argument-list
+  while line:
+    match = Match(r'^.*?(\w+)\s*&&(.*)$', line)
+    if not match:
+      break
+    if match.group(1) not in typenames:
+      return False
+    line = match.group(2)
+
+  # All RValue types that were in template-argument-list should have
+  # been removed by now.  Those were allowed, assuming that they will
+  # be forwarded.
+  #
+  # If there are no remaining RValue types left (i.e. types that were
+  # not found in template-argument-list), flag those as not allowed.
+  return line.find('&&') < 0
+
+
+def GetTemplateArgs(clean_lines, linenum):
+  """Find list of template arguments associated with this function declaration.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: Line number containing the start of the function declaration,
+             usually one line after the end of the template-argument-list.
+  Returns:
+    Set of type names, or empty set if this does not appear to have
+    any template parameters.
+  """
+  # Find start of function
+  func_line = linenum
+  while func_line > 0:
+    line = clean_lines.elided[func_line]
+    if Match(r'^\s*$', line):
+      return set()
+    if line.find('(') >= 0:
+      break
+    func_line -= 1
+  if func_line == 0:
+    return set()
+
+  # Collapse template-argument-list into a single string
+  argument_list = ''
+  match = Match(r'^(\s*template\s*)<', clean_lines.elided[func_line])
+  if match:
+    # template-argument-list on the same line as function name
+    start_col = len(match.group(1))
+    _, end_line, end_col = CloseExpression(clean_lines, func_line, start_col)
+    if end_col > -1 and end_line == func_line:
+      start_col += 1  # Skip the opening bracket
+      argument_list = clean_lines.elided[func_line][start_col:end_col]
+
+  elif func_line > 1:
+    # template-argument-list one line before function name
+    match = Match(r'^(.*)>\s*$', clean_lines.elided[func_line - 1])
+    if match:
+      end_col = len(match.group(1))
+      _, start_line, start_col = ReverseCloseExpression(
+          clean_lines, func_line - 1, end_col)
+      if start_col > -1:
+        start_col += 1  # Skip the opening bracket
+        while start_line < func_line - 1:
+          argument_list += clean_lines.elided[start_line][start_col:]
+          start_col = 0
+          start_line += 1
+        argument_list += clean_lines.elided[func_line - 1][start_col:end_col]
+
+  if not argument_list:
+    return set()
+
+  # Extract type names
+  typenames = set()
+  while True:
+    match = Match(r'^[,\s]*(?:typename|class)(?:\.\.\.)?\s+(\w+)(.*)$',
+                  argument_list)
+    if not match:
+      break
+    typenames.add(match.group(1))
+    argument_list = match.group(2)
+  return typenames
+
+
+def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error):
+  """Check for rvalue references.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    error: The function to call with any errors found.
+  """
+  # Find lines missing spaces around &&.
+  # TODO(unknown): currently we don't check for rvalue references
+  # with spaces surrounding the && to avoid false positives with
+  # boolean expressions.
+  line = clean_lines.elided[linenum]
+  match = Match(r'^(.*\S)&&', line)
+  if not match:
+    match = Match(r'(.*)&&\S', line)
+  if (not match) or '(&&)' in line or Search(r'\boperator\s*$', match.group(1)):
+    return
+
+  # Either poorly formed && or an rvalue reference, check the context
+  # to get a more accurate error message.  Mostly we want to determine
+  # if what's to the left of "&&" is a type or not.
+  typenames = GetTemplateArgs(clean_lines, linenum)
+  and_pos = len(match.group(1))
+  if IsRValueType(typenames, clean_lines, nesting_state, linenum, and_pos):
+    if not IsRValueAllowed(clean_lines, linenum, typenames):
+      error(filename, linenum, 'build/c++11', 3,
+            'RValue references are an unapproved C++ feature.')
+  else:
+    error(filename, linenum, 'whitespace/operators', 3,
+          'Missing spaces around &&')
+
+
+def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error):
+  """Checks for additional blank line issues related to sections.
+
+  Currently the only thing checked here is blank line before protected/private.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    class_info: A _ClassInfo objects.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  # Skip checks if the class is small, where small means 25 lines or less.
+  # 25 lines seems like a good cutoff since that's the usual height of
+  # terminals, and any class that can't fit in one screen can't really
+  # be considered "small".
+  #
+  # Also skip checks if we are on the first line.  This accounts for
+  # classes that look like
+  #   class Foo { public: ... };
+  #
+  # If we didn't find the end of the class, last_line would be zero,
+  # and the check will be skipped by the first condition.
+  if (class_info.last_line - class_info.starting_linenum <= 24 or
+      linenum <= class_info.starting_linenum):
+    return
+
+  matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum])
+  if matched:
+    # Issue warning if the line before public/protected/private was
+    # not a blank line, but don't do this if the previous line contains
+    # "class" or "struct".  This can happen two ways:
+    #  - We are at the beginning of the class.
+    #  - We are forward-declaring an inner class that is semantically
+    #    private, but needed to be public for implementation reasons.
+    # Also ignores cases where the previous line ends with a backslash as can be
+    # common when defining classes in C macros.
+    prev_line = clean_lines.lines[linenum - 1]
+    if (not IsBlankLine(prev_line) and
+        not Search(r'\b(class|struct)\b', prev_line) and
+        not Search(r'\\$', prev_line)):
+      # Try a bit harder to find the beginning of the class.  This is to
+      # account for multi-line base-specifier lists, e.g.:
+      #   class Derived
+      #       : public Base {
+      end_class_head = class_info.starting_linenum
+      for i in range(class_info.starting_linenum, linenum):
+        if Search(r'\{\s*$', clean_lines.lines[i]):
+          end_class_head = i
+          break
+      if end_class_head < linenum - 1:
+        error(filename, linenum, 'whitespace/blank_line', 3,
+              '"%s:" should be preceded by a blank line' % matched.group(1))
+
+
+def GetPreviousNonBlankLine(clean_lines, linenum):
+  """Return the most recent non-blank line and its line number.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file contents.
+    linenum: The number of the line to check.
+
+  Returns:
+    A tuple with two elements.  The first element is the contents of the last
+    non-blank line before the current line, or the empty string if this is the
+    first non-blank line.  The second is the line number of that line, or -1
+    if this is the first non-blank line.
+  """
+
+  prevlinenum = linenum - 1
+  while prevlinenum >= 0:
+    prevline = clean_lines.elided[prevlinenum]
+    if not IsBlankLine(prevline):     # if not a blank line...
+      return (prevline, prevlinenum)
+    prevlinenum -= 1
+  return ('', -1)
+
+
+def CheckBraces(filename, clean_lines, linenum, error):
+  """Looks for misplaced braces (e.g. at the end of line).
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+
+  line = clean_lines.elided[linenum]        # get rid of comments and strings
+
+  if Match(r'\s*{\s*$', line):
+    # We allow an open brace to start a line in the case where someone is using
+    # braces in a block to explicitly create a new scope, which is commonly used
+    # to control the lifetime of stack-allocated variables.  Braces are also
+    # used for brace initializers inside function calls.  We don't detect this
+    # perfectly: we just don't complain if the last non-whitespace character on
+    # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the
+    # previous line starts a preprocessor block.
+    prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0]
+    if (not Search(r'[,;:}{(]\s*$', prevline) and
+        not Match(r'\s*#', prevline)):
+      error(filename, linenum, 'whitespace/braces', 4,
+            '{ should almost always be at the end of the previous line')
+
+  # An else clause should be on the same line as the preceding closing brace.
+  if Match(r'\s*else\b\s*(?:if\b|\{|$)', line):
+    prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0]
+    if Match(r'\s*}\s*$', prevline):
+      error(filename, linenum, 'whitespace/newline', 4,
+            'An else should appear on the same line as the preceding }')
+
+  # If braces come on one side of an else, they should be on both.
+  # However, we have to worry about "else if" that spans multiple lines!
+  if Search(r'else if\s*\(', line):       # could be multi-line if
+    brace_on_left = bool(Search(r'}\s*else if\s*\(', line))
+    # find the ( after the if
+    pos = line.find('else if')
+    pos = line.find('(', pos)
+    if pos > 0:
+      (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos)
+      brace_on_right = endline[endpos:].find('{') != -1
+      if brace_on_left != brace_on_right:    # must be brace after if
+        error(filename, linenum, 'readability/braces', 5,
+              'If an else has a brace on one side, it should have it on both')
+  elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line):
+    error(filename, linenum, 'readability/braces', 5,
+          'If an else has a brace on one side, it should have it on both')
+
+  # Likewise, an else should never have the else clause on the same line
+  if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line):
+    error(filename, linenum, 'whitespace/newline', 4,
+          'Else clause should never be on same line as else (use 2 lines)')
+
+  # In the same way, a do/while should never be on one line
+  if Match(r'\s*do [^\s{]', line):
+    error(filename, linenum, 'whitespace/newline', 4,
+          'do/while clauses should not be on a single line')
+
+  # Check single-line if/else bodies. The style guide says 'curly braces are not
+  # required for single-line statements'. We additionally allow multi-line,
+  # single statements, but we reject anything with more than one semicolon in
+  # it. This means that the first semicolon after the if should be at the end of
+  # its line, and the line after that should have an indent level equal to or
+  # lower than the if. We also check for ambiguous if/else nesting without
+  # braces.
+  if_else_match = Search(r'\b(if\s*\(|else\b)', line)
+  if if_else_match and not Match(r'\s*#', line):
+    if_indent = GetIndentLevel(line)
+    endline, endlinenum, endpos = line, linenum, if_else_match.end()
+    if_match = Search(r'\bif\s*\(', line)
+    if if_match:
+      # This could be a multiline if condition, so find the end first.
+      pos = if_match.end() - 1
+      (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos)
+    # Check for an opening brace, either directly after the if or on the next
+    # line. If found, this isn't a single-statement conditional.
+    if (not Match(r'\s*{', endline[endpos:])
+        and not (Match(r'\s*$', endline[endpos:])
+                 and endlinenum < (len(clean_lines.elided) - 1)
+                 and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))):
+      while (endlinenum < len(clean_lines.elided)
+             and ';' not in clean_lines.elided[endlinenum][endpos:]):
+        endlinenum += 1
+        endpos = 0
+      if endlinenum < len(clean_lines.elided):
+        endline = clean_lines.elided[endlinenum]
+        # We allow a mix of whitespace and closing braces (e.g. for one-liner
+        # methods) and a single \ after the semicolon (for macros)
+        endpos = endline.find(';')
+        if not Match(r';[\s}]*(\\?)$', endline[endpos:]):
+          # Semicolon isn't the last character, there's something trailing.
+          # Output a warning if the semicolon is not contained inside
+          # a lambda expression.
+          if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$',
+                       endline):
+            error(filename, linenum, 'readability/braces', 4,
+                  'If/else bodies with multiple statements require braces')
+        elif endlinenum < len(clean_lines.elided) - 1:
+          # Make sure the next line is dedented
+          next_line = clean_lines.elided[endlinenum + 1]
+          next_indent = GetIndentLevel(next_line)
+          # With ambiguous nested if statements, this will error out on the
+          # if that *doesn't* match the else, regardless of whether it's the
+          # inner one or outer one.
+          if (if_match and Match(r'\s*else\b', next_line)
+              and next_indent != if_indent):
+            error(filename, linenum, 'readability/braces', 4,
+                  'Else clause should be indented at the same level as if. '
+                  'Ambiguous nested if/else chains require braces.')
+          elif next_indent > if_indent:
+            error(filename, linenum, 'readability/braces', 4,
+                  'If/else bodies with multiple statements require braces')
+
+
+def CheckTrailingSemicolon(filename, clean_lines, linenum, error):
+  """Looks for redundant trailing semicolon.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+
+  line = clean_lines.elided[linenum]
+
+  # Block bodies should not be followed by a semicolon.  Due to C++11
+  # brace initialization, there are more places where semicolons are
+  # required than not, so we use a whitelist approach to check these
+  # rather than a blacklist.  These are the places where "};" should
+  # be replaced by just "}":
+  # 1. Some flavor of block following closing parenthesis:
+  #    for (;;) {};
+  #    while (...) {};
+  #    switch (...) {};
+  #    Function(...) {};
+  #    if (...) {};
+  #    if (...) else if (...) {};
+  #
+  # 2. else block:
+  #    if (...) else {};
+  #
+  # 3. const member function:
+  #    Function(...) const {};
+  #
+  # 4. Block following some statement:
+  #    x = 42;
+  #    {};
+  #
+  # 5. Block at the beginning of a function:
+  #    Function(...) {
+  #      {};
+  #    }
+  #
+  #    Note that naively checking for the preceding "{" will also match
+  #    braces inside multi-dimensional arrays, but this is fine since
+  #    that expression will not contain semicolons.
+  #
+  # 6. Block following another block:
+  #    while (true) {}
+  #    {};
+  #
+  # 7. End of namespaces:
+  #    namespace {};
+  #
+  #    These semicolons seems far more common than other kinds of
+  #    redundant semicolons, possibly due to people converting classes
+  #    to namespaces.  For now we do not warn for this case.
+  #
+  # Try matching case 1 first.
+  match = Match(r'^(.*\)\s*)\{', line)
+  if match:
+    # Matched closing parenthesis (case 1).  Check the token before the
+    # matching opening parenthesis, and don't warn if it looks like a
+    # macro.  This avoids these false positives:
+    #  - macro that defines a base class
+    #  - multi-line macro that defines a base class
+    #  - macro that defines the whole class-head
+    #
+    # But we still issue warnings for macros that we know are safe to
+    # warn, specifically:
+    #  - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P
+    #  - TYPED_TEST
+    #  - INTERFACE_DEF
+    #  - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED:
+    #
+    # We implement a whitelist of safe macros instead of a blacklist of
+    # unsafe macros, even though the latter appears less frequently in
+    # google code and would have been easier to implement.  This is because
+    # the downside for getting the whitelist wrong means some extra
+    # semicolons, while the downside for getting the blacklist wrong
+    # would result in compile errors.
+    #
+    # In addition to macros, we also don't want to warn on
+    #  - Compound literals
+    #  - Lambdas
+    #  - alignas specifier with anonymous structs:
+    closing_brace_pos = match.group(1).rfind(')')
+    opening_parenthesis = ReverseCloseExpression(
+        clean_lines, linenum, closing_brace_pos)
+    if opening_parenthesis[2] > -1:
+      line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]]
+      macro = Search(r'\b([A-Z_]+)\s*$', line_prefix)
+      func = Match(r'^(.*\])\s*$', line_prefix)
+      if ((macro and
+           macro.group(1) not in (
+               'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST',
+               'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED',
+               'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or
+          (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or
+          Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or
+          Search(r'\s+=\s*$', line_prefix)):
+        match = None
+    if (match and
+        opening_parenthesis[1] > 1 and
+        Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])):
+      # Multi-line lambda-expression
+      match = None
+
+  else:
+    # Try matching cases 2-3.
+    match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line)
+    if not match:
+      # Try matching cases 4-6.  These are always matched on separate lines.
+      #
+      # Note that we can't simply concatenate the previous line to the
+      # current line and do a single match, otherwise we may output
+      # duplicate warnings for the blank line case:
+      #   if (cond) {
+      #     // blank line
+      #   }
+      prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0]
+      if prevline and Search(r'[;{}]\s*$', prevline):
+        match = Match(r'^(\s*)\{', line)
+
+  # Check matching closing brace
+  if match:
+    (endline, endlinenum, endpos) = CloseExpression(
+        clean_lines, linenum, len(match.group(1)))
+    if endpos > -1 and Match(r'^\s*;', endline[endpos:]):
+      # Current {} pair is eligible for semicolon check, and we have found
+      # the redundant semicolon, output warning here.
+      #
+      # Note: because we are scanning forward for opening braces, and
+      # outputting warnings for the matching closing brace, if there are
+      # nested blocks with trailing semicolons, we will get the error
+      # messages in reversed order.
+      error(filename, endlinenum, 'readability/braces', 4,
+            "You don't need a ; after a }")
+
+
+def CheckEmptyBlockBody(filename, clean_lines, linenum, error):
+  """Look for empty loop/conditional body with only a single semicolon.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+
+  # Search for loop keywords at the beginning of the line.  Because only
+  # whitespaces are allowed before the keywords, this will also ignore most
+  # do-while-loops, since those lines should start with closing brace.
+  #
+  # We also check "if" blocks here, since an empty conditional block
+  # is likely an error.
+  line = clean_lines.elided[linenum]
+  matched = Match(r'\s*(for|while|if)\s*\(', line)
+  if matched:
+    # Find the end of the conditional expression
+    (end_line, end_linenum, end_pos) = CloseExpression(
+        clean_lines, linenum, line.find('('))
+
+    # Output warning if what follows the condition expression is a semicolon.
+    # No warning for all other cases, including whitespace or newline, since we
+    # have a separate check for semicolons preceded by whitespace.
+    if end_pos >= 0 and Match(r';', end_line[end_pos:]):
+      if matched.group(1) == 'if':
+        error(filename, end_linenum, 'whitespace/empty_conditional_body', 5,
+              'Empty conditional bodies should use {}')
+      else:
+        error(filename, end_linenum, 'whitespace/empty_loop_body', 5,
+              'Empty loop bodies should use {} or continue')
+
+
+def FindCheckMacro(line):
+  """Find a replaceable CHECK-like macro.
+
+  Args:
+    line: line to search on.
+  Returns:
+    (macro name, start position), or (None, -1) if no replaceable
+    macro is found.
+  """
+  for macro in _CHECK_MACROS:
+    i = line.find(macro)
+    if i >= 0:
+      # Find opening parenthesis.  Do a regular expression match here
+      # to make sure that we are matching the expected CHECK macro, as
+      # opposed to some other macro that happens to contain the CHECK
+      # substring.
+      matched = Match(r'^(.*\b' + macro + r'\s*)\(', line)
+      if not matched:
+        continue
+      return (macro, len(matched.group(1)))
+  return (None, -1)
+
+
+def CheckCheck(filename, clean_lines, linenum, error):
+  """Checks the use of CHECK and EXPECT macros.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+
+  # Decide the set of replacement macros that should be suggested
+  lines = clean_lines.elided
+  (check_macro, start_pos) = FindCheckMacro(lines[linenum])
+  if not check_macro:
+    return
+
+  # Find end of the boolean expression by matching parentheses
+  (last_line, end_line, end_pos) = CloseExpression(
+      clean_lines, linenum, start_pos)
+  if end_pos < 0:
+    return
+
+  # If the check macro is followed by something other than a
+  # semicolon, assume users will log their own custom error messages
+  # and don't suggest any replacements.
+  if not Match(r'\s*;', last_line[end_pos:]):
+    return
+
+  if linenum == end_line:
+    expression = lines[linenum][start_pos + 1:end_pos - 1]
+  else:
+    expression = lines[linenum][start_pos + 1:]
+    for i in xrange(linenum + 1, end_line):
+      expression += lines[i]
+    expression += last_line[0:end_pos - 1]
+
+  # Parse expression so that we can take parentheses into account.
+  # This avoids false positives for inputs like "CHECK((a < 4) == b)",
+  # which is not replaceable by CHECK_LE.
+  lhs = ''
+  rhs = ''
+  operator = None
+  while expression:
+    matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||'
+                    r'==|!=|>=|>|<=|<|\()(.*)$', expression)
+    if matched:
+      token = matched.group(1)
+      if token == '(':
+        # Parenthesized operand
+        expression = matched.group(2)
+        (end, _) = FindEndOfExpressionInLine(expression, 0, ['('])
+        if end < 0:
+          return  # Unmatched parenthesis
+        lhs += '(' + expression[0:end]
+        expression = expression[end:]
+      elif token in ('&&', '||'):
+        # Logical and/or operators.  This means the expression
+        # contains more than one term, for example:
+        #   CHECK(42 < a && a < b);
+        #
+        # These are not replaceable with CHECK_LE, so bail out early.
+        return
+      elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'):
+        # Non-relational operator
+        lhs += token
+        expression = matched.group(2)
+      else:
+        # Relational operator
+        operator = token
+        rhs = matched.group(2)
+        break
+    else:
+      # Unparenthesized operand.  Instead of appending to lhs one character
+      # at a time, we do another regular expression match to consume several
+      # characters at once if possible.  Trivial benchmark shows that this
+      # is more efficient when the operands are longer than a single
+      # character, which is generally the case.
+      matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression)
+      if not matched:
+        matched = Match(r'^(\s*\S)(.*)$', expression)
+        if not matched:
+          break
+      lhs += matched.group(1)
+      expression = matched.group(2)
+
+  # Only apply checks if we got all parts of the boolean expression
+  if not (lhs and operator and rhs):
+    return
+
+  # Check that rhs do not contain logical operators.  We already know
+  # that lhs is fine since the loop above parses out && and ||.
+  if rhs.find('&&') > -1 or rhs.find('||') > -1:
+    return
+
+  # At least one of the operands must be a constant literal.  This is
+  # to avoid suggesting replacements for unprintable things like
+  # CHECK(variable != iterator)
+  #
+  # The following pattern matches decimal, hex integers, strings, and
+  # characters (in that order).
+  lhs = lhs.strip()
+  rhs = rhs.strip()
+  match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$'
+  if Match(match_constant, lhs) or Match(match_constant, rhs):
+    # Note: since we know both lhs and rhs, we can provide a more
+    # descriptive error message like:
+    #   Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42)
+    # Instead of:
+    #   Consider using CHECK_EQ instead of CHECK(a == b)
+    #
+    # We are still keeping the less descriptive message because if lhs
+    # or rhs gets long, the error message might become unreadable.
+    error(filename, linenum, 'readability/check', 2,
+          'Consider using %s instead of %s(a %s b)' % (
+              _CHECK_REPLACEMENT[check_macro][operator],
+              check_macro, operator))
+
+
+def CheckAltTokens(filename, clean_lines, linenum, error):
+  """Check alternative keywords being used in boolean expressions.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # Avoid preprocessor lines
+  if Match(r'^\s*#', line):
+    return
+
+  # Last ditch effort to avoid multi-line comments.  This will not help
+  # if the comment started before the current line or ended after the
+  # current line, but it catches most of the false positives.  At least,
+  # it provides a way to workaround this warning for people who use
+  # multi-line comments in preprocessor macros.
+  #
+  # TODO(unknown): remove this once cpplint has better support for
+  # multi-line comments.
+  if line.find('/*') >= 0 or line.find('*/') >= 0:
+    return
+
+  for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line):
+    error(filename, linenum, 'readability/alt_tokens', 2,
+          'Use operator %s instead of %s' % (
+              _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1)))
+
+
+def GetLineWidth(line):
+  """Determines the width of the line in column positions.
+
+  Args:
+    line: A string, which may be a Unicode string.
+
+  Returns:
+    The width of the line in column positions, accounting for Unicode
+    combining characters and wide characters.
+  """
+  if isinstance(line, unicode):
+    width = 0
+    for uc in unicodedata.normalize('NFC', line):
+      if unicodedata.east_asian_width(uc) in ('W', 'F'):
+        width += 2
+      elif not unicodedata.combining(uc):
+        width += 1
+    return width
+  else:
+    return len(line)
+
+
+def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state,
+               error):
+  """Checks rules from the 'C++ style rules' section of cppguide.html.
+
+  Most of these rules are hard to test (naming, comment style), but we
+  do what we can.  In particular we check for 2-space indents, line lengths,
+  tab usage, spaces inside code, etc.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    file_extension: The extension (without the dot) of the filename.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    error: The function to call with any errors found.
+  """
+
+  # Don't use "elided" lines here, otherwise we can't check commented lines.
+  # Don't want to use "raw" either, because we don't want to check inside C++11
+  # raw strings,
+  raw_lines = clean_lines.lines_without_raw_strings
+  line = raw_lines[linenum]
+
+  if line.find('\t') != -1:
+    error(filename, linenum, 'whitespace/tab', 1,
+          'Tab found; better to use spaces')
+
+  # One or three blank spaces at the beginning of the line is weird; it's
+  # hard to reconcile that with 2-space indents.
+  # NOTE: here are the conditions rob pike used for his tests.  Mine aren't
+  # as sophisticated, but it may be worth becoming so:  RLENGTH==initial_spaces
+  # if(RLENGTH > 20) complain = 0;
+  # if(match($0, " +(error|private|public|protected):")) complain = 0;
+  # if(match(prev, "&& *$")) complain = 0;
+  # if(match(prev, "\\|\\| *$")) complain = 0;
+  # if(match(prev, "[\",=><] *$")) complain = 0;
+  # if(match($0, " <<")) complain = 0;
+  # if(match(prev, " +for \\(")) complain = 0;
+  # if(prevodd && match(prevprev, " +for \\(")) complain = 0;
+  scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$'
+  classinfo = nesting_state.InnermostClass()
+  initial_spaces = 0
+  cleansed_line = clean_lines.elided[linenum]
+  while initial_spaces < len(line) and line[initial_spaces] == ' ':
+    initial_spaces += 1
+  if line and line[-1].isspace():
+    error(filename, linenum, 'whitespace/end_of_line', 4,
+          'Line ends in whitespace.  Consider deleting these extra spaces.')
+  # There are certain situations we allow one space, notably for
+  # section labels, and also lines containing multi-line raw strings.
+  elif ((initial_spaces == 1 or initial_spaces == 3) and
+        not Match(scope_or_label_pattern, cleansed_line) and
+        not (clean_lines.raw_lines[linenum] != line and
+             Match(r'^\s*""', line))):
+    error(filename, linenum, 'whitespace/indent', 3,
+          'Weird number of spaces at line-start.  '
+          'Are you using a 2-space indent?')
+
+  # Check if the line is a header guard.
+  is_header_guard = False
+  if file_extension == 'h':
+    cppvar = GetHeaderGuardCPPVariable(filename)
+    if (line.startswith('#ifndef %s' % cppvar) or
+        line.startswith('#define %s' % cppvar) or
+        line.startswith('#endif  // %s' % cppvar)):
+      is_header_guard = True
+  # #include lines and header guards can be long, since there's no clean way to
+  # split them.
+  #
+  # URLs can be long too.  It's possible to split these, but it makes them
+  # harder to cut&paste.
+  #
+  # The "$Id:...$" comment may also get very long without it being the
+  # developers fault.
+  if (not line.startswith('#include') and not is_header_guard and
+      not Match(r'^\s*//.*http(s?)://\S*$', line) and
+      not Match(r'^// \$Id:.*#[0-9]+ \$$', line)):
+    line_width = GetLineWidth(line)
+    extended_length = int((_line_length * 1.25))
+    if line_width > extended_length:
+      error(filename, linenum, 'whitespace/line_length', 4,
+            'Lines should very rarely be longer than %i characters' %
+            extended_length)
+    elif line_width > _line_length:
+      error(filename, linenum, 'whitespace/line_length', 2,
+            'Lines should be <= %i characters long' % _line_length)
+
+  if (cleansed_line.count(';') > 1 and
+      # for loops are allowed two ;'s (and may run over two lines).
+      cleansed_line.find('for') == -1 and
+      (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or
+       GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and
+      # It's ok to have many commands in a switch case that fits in 1 line
+      not ((cleansed_line.find('case ') != -1 or
+            cleansed_line.find('default:') != -1) and
+           cleansed_line.find('break;') != -1)):
+    error(filename, linenum, 'whitespace/newline', 0,
+          'More than one command on the same line')
+
+  # Some more style checks
+  CheckBraces(filename, clean_lines, linenum, error)
+  CheckTrailingSemicolon(filename, clean_lines, linenum, error)
+  CheckEmptyBlockBody(filename, clean_lines, linenum, error)
+  CheckAccess(filename, clean_lines, linenum, nesting_state, error)
+  CheckSpacing(filename, clean_lines, linenum, nesting_state, error)
+  CheckOperatorSpacing(filename, clean_lines, linenum, error)
+  CheckParenthesisSpacing(filename, clean_lines, linenum, error)
+  CheckCommaSpacing(filename, clean_lines, linenum, error)
+  CheckBracesSpacing(filename, clean_lines, linenum, error)
+  CheckSpacingForFunctionCall(filename, clean_lines, linenum, error)
+  CheckRValueReference(filename, clean_lines, linenum, nesting_state, error)
+  CheckCheck(filename, clean_lines, linenum, error)
+  CheckAltTokens(filename, clean_lines, linenum, error)
+  classinfo = nesting_state.InnermostClass()
+  if classinfo:
+    CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error)
+
+
+_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$')
+# Matches the first component of a filename delimited by -s and _s. That is:
+#  _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo'
+#  _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo'
+#  _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo'
+#  _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo'
+_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+')
+
+
+def _DropCommonSuffixes(filename):
+  """Drops common suffixes like _test.cc or -inl.h from filename.
+
+  For example:
+    >>> _DropCommonSuffixes('foo/foo-inl.h')
+    'foo/foo'
+    >>> _DropCommonSuffixes('foo/bar/foo.cc')
+    'foo/bar/foo'
+    >>> _DropCommonSuffixes('foo/foo_internal.h')
+    'foo/foo'
+    >>> _DropCommonSuffixes('foo/foo_unusualinternal.h')
+    'foo/foo_unusualinternal'
+
+  Args:
+    filename: The input filename.
+
+  Returns:
+    The filename with the common suffix removed.
+  """
+  for suffix in ('test.cc', 'regtest.cc', 'unittest.cc',
+                 'inl.h', 'impl.h', 'internal.h'):
+    if (filename.endswith(suffix) and len(filename) > len(suffix) and
+        filename[-len(suffix) - 1] in ('-', '_')):
+      return filename[:-len(suffix) - 1]
+  return os.path.splitext(filename)[0]
+
+
+def _IsTestFilename(filename):
+  """Determines if the given filename has a suffix that identifies it as a test.
+
+  Args:
+    filename: The input filename.
+
+  Returns:
+    True if 'filename' looks like a test, False otherwise.
+  """
+  if (filename.endswith('_test.cc') or
+      filename.endswith('_unittest.cc') or
+      filename.endswith('_regtest.cc')):
+    return True
+  else:
+    return False
+
+
+def _ClassifyInclude(fileinfo, include, is_system):
+  """Figures out what kind of header 'include' is.
+
+  Args:
+    fileinfo: The current file cpplint is running over. A FileInfo instance.
+    include: The path to a #included file.
+    is_system: True if the #include used <> rather than "".
+
+  Returns:
+    One of the _XXX_HEADER constants.
+
+  For example:
+    >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True)
+    _C_SYS_HEADER
+    >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True)
+    _CPP_SYS_HEADER
+    >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False)
+    _LIKELY_MY_HEADER
+    >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'),
+    ...                  'bar/foo_other_ext.h', False)
+    _POSSIBLE_MY_HEADER
+    >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False)
+    _OTHER_HEADER
+  """
+  # This is a list of all standard c++ header files, except
+  # those already checked for above.
+  is_cpp_h = include in _CPP_HEADERS
+
+  if is_system:
+    if is_cpp_h:
+      return _CPP_SYS_HEADER
+    else:
+      return _C_SYS_HEADER
+
+  # If the target file and the include we're checking share a
+  # basename when we drop common extensions, and the include
+  # lives in . , then it's likely to be owned by the target file.
+  target_dir, target_base = (
+      os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName())))
+  include_dir, include_base = os.path.split(_DropCommonSuffixes(include))
+  if target_base == include_base and (
+      include_dir == target_dir or
+      include_dir == os.path.normpath(target_dir + '/../public')):
+    return _LIKELY_MY_HEADER
+
+  # If the target and include share some initial basename
+  # component, it's possible the target is implementing the
+  # include, so it's allowed to be first, but we'll never
+  # complain if it's not there.
+  target_first_component = _RE_FIRST_COMPONENT.match(target_base)
+  include_first_component = _RE_FIRST_COMPONENT.match(include_base)
+  if (target_first_component and include_first_component and
+      target_first_component.group(0) ==
+      include_first_component.group(0)):
+    return _POSSIBLE_MY_HEADER
+
+  return _OTHER_HEADER
+
+
+
+def CheckIncludeLine(filename, clean_lines, linenum, include_state, error):
+  """Check rules that are applicable to #include lines.
+
+  Strings on #include lines are NOT removed from elided line, to make
+  certain tasks easier. However, to prevent false positives, checks
+  applicable to #include lines in CheckLanguage must be put here.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    include_state: An _IncludeState instance in which the headers are inserted.
+    error: The function to call with any errors found.
+  """
+  fileinfo = FileInfo(filename)
+  line = clean_lines.lines[linenum]
+
+  # "include" should use the new style "foo/bar.h" instead of just "bar.h"
+  # Only do this check if the included header follows google naming
+  # conventions.  If not, assume that it's a 3rd party API that
+  # requires special include conventions.
+  #
+  # We also make an exception for Lua headers, which follow google
+  # naming convention but not the include convention.
+  match = Match(r'#include\s*"([^/]+\.h)"', line)
+  if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)):
+    error(filename, linenum, 'build/include', 4,
+          'Include the directory when naming .h files')
+
+  # we shouldn't include a file more than once. actually, there are a
+  # handful of instances where doing so is okay, but in general it's
+  # not.
+  match = _RE_PATTERN_INCLUDE.search(line)
+  if match:
+    include = match.group(2)
+    is_system = (match.group(1) == '<')
+    duplicate_line = include_state.FindHeader(include)
+    if duplicate_line >= 0:
+      error(filename, linenum, 'build/include', 4,
+            '"%s" already included at %s:%s' %
+            (include, filename, duplicate_line))
+    elif (include.endswith('.cc') and
+          os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)):
+      error(filename, linenum, 'build/include', 4,
+            'Do not include .cc files from other packages')
+    elif not _THIRD_PARTY_HEADERS_PATTERN.match(include):
+      include_state.include_list[-1].append((include, linenum))
+
+      # We want to ensure that headers appear in the right order:
+      # 1) for foo.cc, foo.h  (preferred location)
+      # 2) c system files
+      # 3) cpp system files
+      # 4) for foo.cc, foo.h  (deprecated location)
+      # 5) other google headers
+      #
+      # We classify each include statement as one of those 5 types
+      # using a number of techniques. The include_state object keeps
+      # track of the highest type seen, and complains if we see a
+      # lower type after that.
+      error_message = include_state.CheckNextIncludeOrder(
+          _ClassifyInclude(fileinfo, include, is_system))
+      if error_message:
+        error(filename, linenum, 'build/include_order', 4,
+              '%s. Should be: %s.h, c system, c++ system, other.' %
+              (error_message, fileinfo.BaseName()))
+      canonical_include = include_state.CanonicalizeAlphabeticalOrder(include)
+      if not include_state.IsInAlphabeticalOrder(
+          clean_lines, linenum, canonical_include):
+        error(filename, linenum, 'build/include_alpha', 4,
+              'Include "%s" not in alphabetical order' % include)
+      include_state.SetLastHeader(canonical_include)
+
+
+
+def _GetTextInside(text, start_pattern):
+  r"""Retrieves all the text between matching open and close parentheses.
+
+  Given a string of lines and a regular expression string, retrieve all the text
+  following the expression and between opening punctuation symbols like
+  (, [, or {, and the matching close-punctuation symbol. This properly nested
+  occurrences of the punctuations, so for the text like
+    printf(a(), b(c()));
+  a call to _GetTextInside(text, r'printf\(') will return 'a(), b(c())'.
+  start_pattern must match string having an open punctuation symbol at the end.
+
+  Args:
+    text: The lines to extract text. Its comments and strings must be elided.
+           It can be single line and can span multiple lines.
+    start_pattern: The regexp string indicating where to start extracting
+                   the text.
+  Returns:
+    The extracted text.
+    None if either the opening string or ending punctuation could not be found.
+  """
+  # TODO(unknown): Audit cpplint.py to see what places could be profitably
+  # rewritten to use _GetTextInside (and use inferior regexp matching today).
+
+  # Give opening punctuations to get the matching close-punctuations.
+  matching_punctuation = {'(': ')', '{': '}', '[': ']'}
+  closing_punctuation = set(matching_punctuation.itervalues())
+
+  # Find the position to start extracting text.
+  match = re.search(start_pattern, text, re.M)
+  if not match:  # start_pattern not found in text.
+    return None
+  start_position = match.end(0)
+
+  assert start_position > 0, (
+      'start_pattern must ends with an opening punctuation.')
+  assert text[start_position - 1] in matching_punctuation, (
+      'start_pattern must ends with an opening punctuation.')
+  # Stack of closing punctuations we expect to have in text after position.
+  punctuation_stack = [matching_punctuation[text[start_position - 1]]]
+  position = start_position
+  while punctuation_stack and position < len(text):
+    if text[position] == punctuation_stack[-1]:
+      punctuation_stack.pop()
+    elif text[position] in closing_punctuation:
+      # A closing punctuation without matching opening punctuations.
+      return None
+    elif text[position] in matching_punctuation:
+      punctuation_stack.append(matching_punctuation[text[position]])
+    position += 1
+  if punctuation_stack:
+    # Opening punctuations left without matching close-punctuations.
+    return None
+  # punctuations match.
+  return text[start_position:position - 1]
+
+
+# Patterns for matching call-by-reference parameters.
+#
+# Supports nested templates up to 2 levels deep using this messy pattern:
+#   < (?: < (?: < [^<>]*
+#               >
+#           |   [^<>] )*
+#         >
+#     |   [^<>] )*
+#   >
+_RE_PATTERN_IDENT = r'[_a-zA-Z]\w*'  # =~ [[:alpha:]][[:alnum:]]*
+_RE_PATTERN_TYPE = (
+    r'(?:const\s+)?(?:typename\s+|class\s+|struct\s+|union\s+|enum\s+)?'
+    r'(?:\w|'
+    r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|'
+    r'::)+')
+# A call-by-reference parameter ends with '& identifier'.
+_RE_PATTERN_REF_PARAM = re.compile(
+    r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*'
+    r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]')
+# A call-by-const-reference parameter either ends with 'const& identifier'
+# or looks like 'const type& identifier' when 'type' is atomic.
+_RE_PATTERN_CONST_REF_PARAM = (
+    r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT +
+    r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')')
+
+
+def CheckLanguage(filename, clean_lines, linenum, file_extension,
+                  include_state, nesting_state, error):
+  """Checks rules from the 'C++ language rules' section of cppguide.html.
+
+  Some of these rules are hard to test (function overloading, using
+  uint32 inappropriately), but we do the best we can.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    file_extension: The extension (without the dot) of the filename.
+    include_state: An _IncludeState instance in which the headers are inserted.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    error: The function to call with any errors found.
+  """
+  # If the line is empty or consists of entirely a comment, no need to
+  # check it.
+  line = clean_lines.elided[linenum]
+  if not line:
+    return
+
+  match = _RE_PATTERN_INCLUDE.search(line)
+  if match:
+    CheckIncludeLine(filename, clean_lines, linenum, include_state, error)
+    return
+
+  # Reset include state across preprocessor directives.  This is meant
+  # to silence warnings for conditional includes.
+  match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line)
+  if match:
+    include_state.ResetSection(match.group(1))
+
+  # Make Windows paths like Unix.
+  fullname = os.path.abspath(filename).replace('\\', '/')
+  
+  # Perform other checks now that we are sure that this is not an include line
+  CheckCasts(filename, clean_lines, linenum, error)
+  CheckGlobalStatic(filename, clean_lines, linenum, error)
+  CheckPrintf(filename, clean_lines, linenum, error)
+
+  if file_extension == 'h':
+    # TODO(unknown): check that 1-arg constructors are explicit.
+    #                How to tell it's a constructor?
+    #                (handled in CheckForNonStandardConstructs for now)
+    # TODO(unknown): check that classes declare or disable copy/assign
+    #                (level 1 error)
+    pass
+
+  # Check if people are using the verboten C basic types.  The only exception
+  # we regularly allow is "unsigned short port" for port.
+  if Search(r'\bshort port\b', line):
+    if not Search(r'\bunsigned short port\b', line):
+      error(filename, linenum, 'runtime/int', 4,
+            'Use "unsigned short" for ports, not "short"')
+  else:
+    match = Search(r'\b(short|long(?! +double)|long long)\b', line)
+    if match:
+      error(filename, linenum, 'runtime/int', 4,
+            'Use int16/int64/etc, rather than the C type %s' % match.group(1))
+
+  # Check if some verboten operator overloading is going on
+  # TODO(unknown): catch out-of-line unary operator&:
+  #   class X {};
+  #   int operator&(const X& x) { return 42; }  // unary operator&
+  # The trick is it's hard to tell apart from binary operator&:
+  #   class Y { int operator&(const Y& x) { return 23; } }; // binary operator&
+  if Search(r'\boperator\s*&\s*\(\s*\)', line):
+    error(filename, linenum, 'runtime/operator', 4,
+          'Unary operator& is dangerous.  Do not use it.')
+
+  # Check for suspicious usage of "if" like
+  # } if (a == b) {
+  if Search(r'\}\s*if\s*\(', line):
+    error(filename, linenum, 'readability/braces', 4,
+          'Did you mean "else if"? If not, start a new line for "if".')
+
+  # Check for potential format string bugs like printf(foo).
+  # We constrain the pattern not to pick things like DocidForPrintf(foo).
+  # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str())
+  # TODO(unknown): Catch the following case. Need to change the calling
+  # convention of the whole function to process multiple line to handle it.
+  #   printf(
+  #       boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line);
+  printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(')
+  if printf_args:
+    match = Match(r'([\w.\->()]+)$', printf_args)
+    if match and match.group(1) != '__VA_ARGS__':
+      function_name = re.search(r'\b((?:string)?printf)\s*\(',
+                                line, re.I).group(1)
+      error(filename, linenum, 'runtime/printf', 4,
+            'Potential format string bug. Do %s("%%s", %s) instead.'
+            % (function_name, match.group(1)))
+
+  # Check for potential memset bugs like memset(buf, sizeof(buf), 0).
+  match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line)
+  if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)):
+    error(filename, linenum, 'runtime/memset', 4,
+          'Did you mean "memset(%s, 0, %s)"?'
+          % (match.group(1), match.group(2)))
+
+  if Search(r'\busing namespace\b', line):
+    error(filename, linenum, 'build/namespaces', 5,
+          'Do not use namespace using-directives.  '
+          'Use using-declarations instead.')
+
+  # Detect variable-length arrays.
+  match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line)
+  if (match and match.group(2) != 'return' and match.group(2) != 'delete' and
+      match.group(3).find(']') == -1):
+    # Split the size using space and arithmetic operators as delimiters.
+    # If any of the resulting tokens are not compile time constants then
+    # report the error.
+    tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3))
+    is_const = True
+    skip_next = False
+    for tok in tokens:
+      if skip_next:
+        skip_next = False
+        continue
+
+      if Search(r'sizeof\(.+\)', tok): continue
+      if Search(r'arraysize\(\w+\)', tok): continue
+
+      tok = tok.lstrip('(')
+      tok = tok.rstrip(')')
+      if not tok: continue
+      if Match(r'\d+', tok): continue
+      if Match(r'0[xX][0-9a-fA-F]+', tok): continue
+      if Match(r'k[A-Z0-9]\w*', tok): continue
+      if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue
+      if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue
+      # A catch all for tricky sizeof cases, including 'sizeof expression',
+      # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)'
+      # requires skipping the next token because we split on ' ' and '*'.
+      if tok.startswith('sizeof'):
+        skip_next = True
+        continue
+      is_const = False
+      break
+    if not is_const:
+      error(filename, linenum, 'runtime/arrays', 1,
+            'Do not use variable-length arrays.  Use an appropriately named '
+            "('k' followed by CamelCase) compile-time constant for the size.")
+
+  # Check for use of unnamed namespaces in header files.  Registration
+  # macros are typically OK, so we allow use of "namespace {" on lines
+  # that end with backslashes.
+  if (file_extension == 'h'
+      and Search(r'\bnamespace\s*{', line)
+      and line[-1] != '\\'):
+    error(filename, linenum, 'build/namespaces', 4,
+          'Do not use unnamed namespaces in header files.  See '
+          'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces'
+          ' for more information.')
+
+
+def CheckGlobalStatic(filename, clean_lines, linenum, error):
+  """Check for unsafe global or static objects.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # Match two lines at a time to support multiline declarations
+  if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line):
+    line += clean_lines.elided[linenum + 1].strip()
+
+  # Check for people declaring static/global STL strings at the top level.
+  # This is dangerous because the C++ language does not guarantee that
+  # globals with constructors are initialized before the first access.
+  match = Match(
+      r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)',
+      line)
+
+  # Remove false positives:
+  # - String pointers (as opposed to values).
+  #    string *pointer
+  #    const string *pointer
+  #    string const *pointer
+  #    string *const pointer
+  #
+  # - Functions and template specializations.
+  #    string Function<Type>(...
+  #    string Class<Type>::Method(...
+  #
+  # - Operators.  These are matched separately because operator names
+  #   cross non-word boundaries, and trying to match both operators
+  #   and functions at the same time would decrease accuracy of
+  #   matching identifiers.
+  #    string Class::operator*()
+  if (match and
+      not Search(r'\bstring\b(\s+const)?\s*\*\s*(const\s+)?\w', line) and
+      not Search(r'\boperator\W', line) and
+      not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(3))):
+    error(filename, linenum, 'runtime/string', 4,
+          'For a static/global string constant, use a C style string instead: '
+          '"%schar %s[]".' %
+          (match.group(1), match.group(2)))
+
+  if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line):
+    error(filename, linenum, 'runtime/init', 4,
+          'You seem to be initializing a member variable with itself.')
+
+
+def CheckPrintf(filename, clean_lines, linenum, error):
+  """Check for printf related issues.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # When snprintf is used, the second argument shouldn't be a literal.
+  match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line)
+  if match and match.group(2) != '0':
+    # If 2nd arg is zero, snprintf is used to calculate size.
+    error(filename, linenum, 'runtime/printf', 3,
+          'If you can, use sizeof(%s) instead of %s as the 2nd arg '
+          'to snprintf.' % (match.group(1), match.group(2)))
+
+  # Check if some verboten C functions are being used.
+  if Search(r'\bsprintf\s*\(', line):
+    error(filename, linenum, 'runtime/printf', 5,
+          'Never use sprintf. Use snprintf instead.')
+  match = Search(r'\b(strcpy|strcat)\s*\(', line)
+  if match:
+    error(filename, linenum, 'runtime/printf', 4,
+          'Almost always, snprintf is better than %s' % match.group(1))
+
+
+def IsDerivedFunction(clean_lines, linenum):
+  """Check if current line contains an inherited function.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+  Returns:
+    True if current line contains a function with "override"
+    virt-specifier.
+  """
+  # Scan back a few lines for start of current function
+  for i in xrange(linenum, max(-1, linenum - 10), -1):
+    match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i])
+    if match:
+      # Look for "override" after the matching closing parenthesis
+      line, _, closing_paren = CloseExpression(
+          clean_lines, i, len(match.group(1)))
+      return (closing_paren >= 0 and
+              Search(r'\boverride\b', line[closing_paren:]))
+  return False
+
+
+def IsOutOfLineMethodDefinition(clean_lines, linenum):
+  """Check if current line contains an out-of-line method definition.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+  Returns:
+    True if current line contains an out-of-line method definition.
+  """
+  # Scan back a few lines for start of current function
+  for i in xrange(linenum, max(-1, linenum - 10), -1):
+    if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]):
+      return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None
+  return False
+
+
+def IsInitializerList(clean_lines, linenum):
+  """Check if current line is inside constructor initializer list.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+  Returns:
+    True if current line appears to be inside constructor initializer
+    list, False otherwise.
+  """
+  for i in xrange(linenum, 1, -1):
+    line = clean_lines.elided[i]
+    if i == linenum:
+      remove_function_body = Match(r'^(.*)\{\s*$', line)
+      if remove_function_body:
+        line = remove_function_body.group(1)
+
+    if Search(r'\s:\s*\w+[({]', line):
+      # A lone colon tend to indicate the start of a constructor
+      # initializer list.  It could also be a ternary operator, which
+      # also tend to appear in constructor initializer lists as
+      # opposed to parameter lists.
+      return True
+    if Search(r'\}\s*,\s*$', line):
+      # A closing brace followed by a comma is probably the end of a
+      # brace-initialized member in constructor initializer list.
+      return True
+    if Search(r'[{};]\s*$', line):
+      # Found one of the following:
+      # - A closing brace or semicolon, probably the end of the previous
+      #   function.
+      # - An opening brace, probably the start of current class or namespace.
+      #
+      # Current line is probably not inside an initializer list since
+      # we saw one of those things without seeing the starting colon.
+      return False
+
+  # Got to the beginning of the file without seeing the start of
+  # constructor initializer list.
+  return False
+
+
+def CheckForNonConstReference(filename, clean_lines, linenum,
+                              nesting_state, error):
+  """Check for non-const references.
+
+  Separate from CheckLanguage since it scans backwards from current
+  line, instead of scanning forward.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    error: The function to call with any errors found.
+  """
+  # Do nothing if there is no '&' on current line.
+  line = clean_lines.elided[linenum]
+  if '&' not in line:
+    return
+
+  # If a function is inherited, current function doesn't have much of
+  # a choice, so any non-const references should not be blamed on
+  # derived function.
+  if IsDerivedFunction(clean_lines, linenum):
+    return
+
+  # Don't warn on out-of-line method definitions, as we would warn on the
+  # in-line declaration, if it isn't marked with 'override'.
+  if IsOutOfLineMethodDefinition(clean_lines, linenum):
+    return
+
+  # Long type names may be broken across multiple lines, usually in one
+  # of these forms:
+  #   LongType
+  #       ::LongTypeContinued &identifier
+  #   LongType::
+  #       LongTypeContinued &identifier
+  #   LongType<
+  #       ...>::LongTypeContinued &identifier
+  #
+  # If we detected a type split across two lines, join the previous
+  # line to current line so that we can match const references
+  # accordingly.
+  #
+  # Note that this only scans back one line, since scanning back
+  # arbitrary number of lines would be expensive.  If you have a type
+  # that spans more than 2 lines, please use a typedef.
+  if linenum > 1:
+    previous = None
+    if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line):
+      # previous_line\n + ::current_line
+      previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$',
+                        clean_lines.elided[linenum - 1])
+    elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line):
+      # previous_line::\n + current_line
+      previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$',
+                        clean_lines.elided[linenum - 1])
+    if previous:
+      line = previous.group(1) + line.lstrip()
+    else:
+      # Check for templated parameter that is split across multiple lines
+      endpos = line.rfind('>')
+      if endpos > -1:
+        (_, startline, startpos) = ReverseCloseExpression(
+            clean_lines, linenum, endpos)
+        if startpos > -1 and startline < linenum:
+          # Found the matching < on an earlier line, collect all
+          # pieces up to current line.
+          line = ''
+          for i in xrange(startline, linenum + 1):
+            line += clean_lines.elided[i].strip()
+
+  # Check for non-const references in function parameters.  A single '&' may
+  # found in the following places:
+  #   inside expression: binary & for bitwise AND
+  #   inside expression: unary & for taking the address of something
+  #   inside declarators: reference parameter
+  # We will exclude the first two cases by checking that we are not inside a
+  # function body, including one that was just introduced by a trailing '{'.
+  # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare].
+  if (nesting_state.previous_stack_top and
+      not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or
+           isinstance(nesting_state.previous_stack_top, _NamespaceInfo))):
+    # Not at toplevel, not within a class, and not within a namespace
+    return
+
+  # Avoid initializer lists.  We only need to scan back from the
+  # current line for something that starts with ':'.
+  #
+  # We don't need to check the current line, since the '&' would
+  # appear inside the second set of parentheses on the current line as
+  # opposed to the first set.
+  if linenum > 0:
+    for i in xrange(linenum - 1, max(0, linenum - 10), -1):
+      previous_line = clean_lines.elided[i]
+      if not Search(r'[),]\s*$', previous_line):
+        break
+      if Match(r'^\s*:\s+\S', previous_line):
+        return
+
+  # Avoid preprocessors
+  if Search(r'\\\s*$', line):
+    return
+
+  # Avoid constructor initializer lists
+  if IsInitializerList(clean_lines, linenum):
+    return
+
+  # We allow non-const references in a few standard places, like functions
+  # called "swap()" or iostream operators like "<<" or ">>".  Do not check
+  # those function parameters.
+  #
+  # We also accept & in static_assert, which looks like a function but
+  # it's actually a declaration expression.
+  whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|'
+                           r'operator\s*[<>][<>]|'
+                           r'static_assert|COMPILE_ASSERT'
+                           r')\s*\(')
+  if Search(whitelisted_functions, line):
+    return
+  elif not Search(r'\S+\([^)]*$', line):
+    # Don't see a whitelisted function on this line.  Actually we
+    # didn't see any function name on this line, so this is likely a
+    # multi-line parameter list.  Try a bit harder to catch this case.
+    for i in xrange(2):
+      if (linenum > i and
+          Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])):
+        return
+
+  decls = ReplaceAll(r'{[^}]*}', ' ', line)  # exclude function body
+  for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls):
+    if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter):
+      error(filename, linenum, 'runtime/references', 2,
+            'Is this a non-const reference? '
+            'If so, make const or use a pointer: ' +
+            ReplaceAll(' *<', '<', parameter))
+
+
+def CheckCasts(filename, clean_lines, linenum, error):
+  """Various cast related checks.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # Check to see if they're using an conversion function cast.
+  # I just try to capture the most common basic types, though there are more.
+  # Parameterless conversion functions, such as bool(), are allowed as they are
+  # probably a member operator declaration or default constructor.
+  match = Search(
+      r'(\bnew\s+|\S<\s*(?:const\s+)?)?\b'
+      r'(int|float|double|bool|char|int32|uint32|int64|uint64)'
+      r'(\([^)].*)', line)
+  expecting_function = ExpectingFunctionArgs(clean_lines, linenum)
+  if match and not expecting_function:
+    matched_type = match.group(2)
+
+    # matched_new_or_template is used to silence two false positives:
+    # - New operators
+    # - Template arguments with function types
+    #
+    # For template arguments, we match on types immediately following
+    # an opening bracket without any spaces.  This is a fast way to
+    # silence the common case where the function type is the first
+    # template argument.  False negative with less-than comparison is
+    # avoided because those operators are usually followed by a space.
+    #
+    #   function<double(double)>   // bracket + no space = false positive
+    #   value < double(42)         // bracket + space = true positive
+    matched_new_or_template = match.group(1)
+
+    # Avoid arrays by looking for brackets that come after the closing
+    # parenthesis.
+    if Match(r'\([^()]+\)\s*\[', match.group(3)):
+      return
+
+    # Other things to ignore:
+    # - Function pointers
+    # - Casts to pointer types
+    # - Placement new
+    # - Alias declarations
+    matched_funcptr = match.group(3)
+    if (matched_new_or_template is None and
+        not (matched_funcptr and
+             (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(',
+                    matched_funcptr) or
+              matched_funcptr.startswith('(*)'))) and
+        not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and
+        not Search(r'new\(\S+\)\s*' + matched_type, line)):
+      error(filename, linenum, 'readability/casting', 4,
+            'Using deprecated casting style.  '
+            'Use static_cast<%s>(...) instead' %
+            matched_type)
+
+  if not expecting_function:
+    CheckCStyleCast(filename, clean_lines, linenum, 'static_cast',
+                    r'\((int|float|double|bool|char|u?int(16|32|64))\)', error)
+
+  # This doesn't catch all cases. Consider (const char * const)"hello".
+  #
+  # (char *) "foo" should always be a const_cast (reinterpret_cast won't
+  # compile).
+  if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast',
+                     r'\((char\s?\*+\s?)\)\s*"', error):
+    pass
+  else:
+    # Check pointer casts for other than string constants
+    CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast',
+                    r'\((\w+\s?\*+\s?)\)', error)
+
+  # In addition, we look for people taking the address of a cast.  This
+  # is dangerous -- casts can assign to temporaries, so the pointer doesn't
+  # point where you think.
+  #
+  # Some non-identifier character is required before the '&' for the
+  # expression to be recognized as a cast.  These are casts:
+  #   expression = &static_cast<int*>(temporary());
+  #   function(&(int*)(temporary()));
+  #
+  # This is not a cast:
+  #   reference_type&(int* function_param);
+  match = Search(
+      r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|'
+      r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line)
+  if match:
+    # Try a better error message when the & is bound to something
+    # dereferenced by the casted pointer, as opposed to the casted
+    # pointer itself.
+    parenthesis_error = False
+    match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line)
+    if match:
+      _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1)))
+      if x1 >= 0 and clean_lines.elided[y1][x1] == '(':
+        _, y2, x2 = CloseExpression(clean_lines, y1, x1)
+        if x2 >= 0:
+          extended_line = clean_lines.elided[y2][x2:]
+          if y2 < clean_lines.NumLines() - 1:
+            extended_line += clean_lines.elided[y2 + 1]
+          if Match(r'\s*(?:->|\[)', extended_line):
+            parenthesis_error = True
+
+    if parenthesis_error:
+      error(filename, linenum, 'readability/casting', 4,
+            ('Are you taking an address of something dereferenced '
+             'from a cast?  Wrapping the dereferenced expression in '
+             'parentheses will make the binding more obvious'))
+    else:
+      error(filename, linenum, 'runtime/casting', 4,
+            ('Are you taking an address of a cast?  '
+             'This is dangerous: could be a temp var.  '
+             'Take the address before doing the cast, rather than after'))
+
+
+def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error):
+  """Checks for a C-style cast by looking for the pattern.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    cast_type: The string for the C++ cast to recommend.  This is either
+      reinterpret_cast, static_cast, or const_cast, depending.
+    pattern: The regular expression used to find C-style casts.
+    error: The function to call with any errors found.
+
+  Returns:
+    True if an error was emitted.
+    False otherwise.
+  """
+  line = clean_lines.elided[linenum]
+  match = Search(pattern, line)
+  if not match:
+    return False
+
+  # Exclude lines with keywords that tend to look like casts
+  context = line[0:match.start(1) - 1]
+  if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context):
+    return False
+
+  # Try expanding current context to see if we one level of
+  # parentheses inside a macro.
+  if linenum > 0:
+    for i in xrange(linenum - 1, max(0, linenum - 5), -1):
+      context = clean_lines.elided[i] + context
+  if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context):
+    return False
+
+  # operator++(int) and operator--(int)
+  if context.endswith(' operator++') or context.endswith(' operator--'):
+    return False
+
+  # A single unnamed argument for a function tends to look like old
+  # style cast.  If we see those, don't issue warnings for deprecated
+  # casts, instead issue warnings for unnamed arguments where
+  # appropriate.
+  #
+  # These are things that we want warnings for, since the style guide
+  # explicitly require all parameters to be named:
+  #   Function(int);
+  #   Function(int) {
+  #   ConstMember(int) const;
+  #   ConstMember(int) const {
+  #   ExceptionMember(int) throw (...);
+  #   ExceptionMember(int) throw (...) {
+  #   PureVirtual(int) = 0;
+  #   [](int) -> bool {
+  #
+  # These are functions of some sort, where the compiler would be fine
+  # if they had named parameters, but people often omit those
+  # identifiers to reduce clutter:
+  #   (FunctionPointer)(int);
+  #   (FunctionPointer)(int) = value;
+  #   Function((function_pointer_arg)(int))
+  #   Function((function_pointer_arg)(int), int param)
+  #   <TemplateArgument(int)>;
+  #   <(FunctionPointerTemplateArgument)(int)>;
+  remainder = line[match.end(0):]
+  if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)',
+           remainder):
+    # Looks like an unnamed parameter.
+
+    # Don't warn on any kind of template arguments.
+    if Match(r'^\s*>', remainder):
+      return False
+
+    # Don't warn on assignments to function pointers, but keep warnings for
+    # unnamed parameters to pure virtual functions.  Note that this pattern
+    # will also pass on assignments of "0" to function pointers, but the
+    # preferred values for those would be "nullptr" or "NULL".
+    matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder)
+    if matched_zero and matched_zero.group(1) != '0':
+      return False
+
+    # Don't warn on function pointer declarations.  For this we need
+    # to check what came before the "(type)" string.
+    if Match(r'.*\)\s*$', line[0:match.start(0)]):
+      return False
+
+    # Don't warn if the parameter is named with block comments, e.g.:
+    #  Function(int /*unused_param*/);
+    raw_line = clean_lines.raw_lines[linenum]
+    if '/*' in raw_line:
+      return False
+
+    # Passed all filters, issue warning here.
+    error(filename, linenum, 'readability/function', 3,
+          'All parameters should be named in a function')
+    return True
+
+  # At this point, all that should be left is actual casts.
+  error(filename, linenum, 'readability/casting', 4,
+        'Using C-style cast.  Use %s<%s>(...) instead' %
+        (cast_type, match.group(1)))
+
+  return True
+
+
+def ExpectingFunctionArgs(clean_lines, linenum):
+  """Checks whether where function type arguments are expected.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+
+  Returns:
+    True if the line at 'linenum' is inside something that expects arguments
+    of function types.
+  """
+  line = clean_lines.elided[linenum]
+  return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or
+          (linenum >= 2 and
+           (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$',
+                  clean_lines.elided[linenum - 1]) or
+            Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$',
+                  clean_lines.elided[linenum - 2]) or
+            Search(r'\bstd::m?function\s*\<\s*$',
+                   clean_lines.elided[linenum - 1]))))
+
+
+_HEADERS_CONTAINING_TEMPLATES = (
+    ('<deque>', ('deque',)),
+    ('<functional>', ('unary_function', 'binary_function',
+                      'plus', 'minus', 'multiplies', 'divides', 'modulus',
+                      'negate',
+                      'equal_to', 'not_equal_to', 'greater', 'less',
+                      'greater_equal', 'less_equal',
+                      'logical_and', 'logical_or', 'logical_not',
+                      'unary_negate', 'not1', 'binary_negate', 'not2',
+                      'bind1st', 'bind2nd',
+                      'pointer_to_unary_function',
+                      'pointer_to_binary_function',
+                      'ptr_fun',
+                      'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t',
+                      'mem_fun_ref_t',
+                      'const_mem_fun_t', 'const_mem_fun1_t',
+                      'const_mem_fun_ref_t', 'const_mem_fun1_ref_t',
+                      'mem_fun_ref',
+                     )),
+    ('<limits>', ('numeric_limits',)),
+    ('<list>', ('list',)),
+    ('<map>', ('map', 'multimap',)),
+    ('<memory>', ('allocator',)),
+    ('<queue>', ('queue', 'priority_queue',)),
+    ('<set>', ('set', 'multiset',)),
+    ('<stack>', ('stack',)),
+    ('<string>', ('char_traits', 'basic_string',)),
+    ('<tuple>', ('tuple',)),
+    ('<utility>', ('pair',)),
+    ('<vector>', ('vector',)),
+
+    # gcc extensions.
+    # Note: std::hash is their hash, ::hash is our hash
+    ('<hash_map>', ('hash_map', 'hash_multimap',)),
+    ('<hash_set>', ('hash_set', 'hash_multiset',)),
+    ('<slist>', ('slist',)),
+    )
+
+_RE_PATTERN_STRING = re.compile(r'\bstring\b')
+
+_re_pattern_algorithm_header = []
+for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap',
+                  'transform'):
+  # Match max<type>(..., ...), max(..., ...), but not foo->max, foo.max or
+  # type::max().
+  _re_pattern_algorithm_header.append(
+      (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'),
+       _template,
+       '<algorithm>'))
+
+_re_pattern_templates = []
+for _header, _templates in _HEADERS_CONTAINING_TEMPLATES:
+  for _template in _templates:
+    _re_pattern_templates.append(
+        (re.compile(r'(\<|\b)' + _template + r'\s*\<'),
+         _template + '<>',
+         _header))
+
+
+def FilesBelongToSameModule(filename_cc, filename_h):
+  """Check if these two filenames belong to the same module.
+
+  The concept of a 'module' here is a as follows:
+  foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the
+  same 'module' if they are in the same directory.
+  some/path/public/xyzzy and some/path/internal/xyzzy are also considered
+  to belong to the same module here.
+
+  If the filename_cc contains a longer path than the filename_h, for example,
+  '/absolute/path/to/base/sysinfo.cc', and this file would include
+  'base/sysinfo.h', this function also produces the prefix needed to open the
+  header. This is used by the caller of this function to more robustly open the
+  header file. We don't have access to the real include paths in this context,
+  so we need this guesswork here.
+
+  Known bugs: tools/base/bar.cc and base/bar.h belong to the same module
+  according to this implementation. Because of this, this function gives
+  some false positives. This should be sufficiently rare in practice.
+
+  Args:
+    filename_cc: is the path for the .cc file
+    filename_h: is the path for the header path
+
+  Returns:
+    Tuple with a bool and a string:
+    bool: True if filename_cc and filename_h belong to the same module.
+    string: the additional prefix needed to open the header file.
+  """
+
+  if not filename_cc.endswith('.cc'):
+    return (False, '')
+  filename_cc = filename_cc[:-len('.cc')]
+  if filename_cc.endswith('_unittest'):
+    filename_cc = filename_cc[:-len('_unittest')]
+  elif filename_cc.endswith('_test'):
+    filename_cc = filename_cc[:-len('_test')]
+  filename_cc = filename_cc.replace('/public/', '/')
+  filename_cc = filename_cc.replace('/internal/', '/')
+
+  if not filename_h.endswith('.h'):
+    return (False, '')
+  filename_h = filename_h[:-len('.h')]
+  if filename_h.endswith('-inl'):
+    filename_h = filename_h[:-len('-inl')]
+  filename_h = filename_h.replace('/public/', '/')
+  filename_h = filename_h.replace('/internal/', '/')
+
+  files_belong_to_same_module = filename_cc.endswith(filename_h)
+  common_path = ''
+  if files_belong_to_same_module:
+    common_path = filename_cc[:-len(filename_h)]
+  return files_belong_to_same_module, common_path
+
+
+def UpdateIncludeState(filename, include_dict, io=codecs):
+  """Fill up the include_dict with new includes found from the file.
+
+  Args:
+    filename: the name of the header to read.
+    include_dict: a dictionary in which the headers are inserted.
+    io: The io factory to use to read the file. Provided for testability.
+
+  Returns:
+    True if a header was successfully added. False otherwise.
+  """
+  headerfile = None
+  try:
+    headerfile = io.open(filename, 'r', 'utf8', 'replace')
+  except IOError:
+    return False
+  linenum = 0
+  for line in headerfile:
+    linenum += 1
+    clean_line = CleanseComments(line)
+    match = _RE_PATTERN_INCLUDE.search(clean_line)
+    if match:
+      include = match.group(2)
+      include_dict.setdefault(include, linenum)
+  return True
+
+
+def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
+                              io=codecs):
+  """Reports for missing stl includes.
+
+  This function will output warnings to make sure you are including the headers
+  necessary for the stl containers and functions that you use. We only give one
+  reason to include a header. For example, if you use both equal_to<> and
+  less<> in a .h file, only one (the latter in the file) of these will be
+  reported as a reason to include the <functional>.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    include_state: An _IncludeState instance.
+    error: The function to call with any errors found.
+    io: The IO factory to use to read the header file. Provided for unittest
+        injection.
+  """
+  required = {}  # A map of header name to linenumber and the template entity.
+                 # Example of required: { '<functional>': (1219, 'less<>') }
+
+  for linenum in xrange(clean_lines.NumLines()):
+    line = clean_lines.elided[linenum]
+    if not line or line[0] == '#':
+      continue
+
+    # String is special -- it is a non-templatized type in STL.
+    matched = _RE_PATTERN_STRING.search(line)
+    if matched:
+      # Don't warn about strings in non-STL namespaces:
+      # (We check only the first match per line; good enough.)
+      prefix = line[:matched.start()]
+      if prefix.endswith('std::') or not prefix.endswith('::'):
+        required['<string>'] = (linenum, 'string')
+
+    for pattern, template, header in _re_pattern_algorithm_header:
+      if pattern.search(line):
+        required[header] = (linenum, template)
+
+    # The following function is just a speed up, no semantics are changed.
+    if not '<' in line:  # Reduces the cpu time usage by skipping lines.
+      continue
+
+    for pattern, template, header in _re_pattern_templates:
+      if pattern.search(line):
+        required[header] = (linenum, template)
+
+  # The policy is that if you #include something in foo.h you don't need to
+  # include it again in foo.cc. Here, we will look at possible includes.
+  # Let's flatten the include_state include_list and copy it into a dictionary.
+  include_dict = dict([item for sublist in include_state.include_list
+                       for item in sublist])
+
+  # Did we find the header for this file (if any) and successfully load it?
+  header_found = False
+
+  # Use the absolute path so that matching works properly.
+  abs_filename = FileInfo(filename).FullName()
+
+  # For Emacs's flymake.
+  # If cpplint is invoked from Emacs's flymake, a temporary file is generated
+  # by flymake and that file name might end with '_flymake.cc'. In that case,
+  # restore original file name here so that the corresponding header file can be
+  # found.
+  # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h'
+  # instead of 'foo_flymake.h'
+  abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename)
+
+  # include_dict is modified during iteration, so we iterate over a copy of
+  # the keys.
+  header_keys = include_dict.keys()
+  for header in header_keys:
+    (same_module, common_path) = FilesBelongToSameModule(abs_filename, header)
+    fullpath = common_path + header
+    if same_module and UpdateIncludeState(fullpath, include_dict, io):
+      header_found = True
+
+  # If we can't find the header file for a .cc, assume it's because we don't
+  # know where to look. In that case we'll give up as we're not sure they
+  # didn't include it in the .h file.
+  # TODO(unknown): Do a better job of finding .h files so we are confident that
+  # not having the .h file means there isn't one.
+  if filename.endswith('.cc') and not header_found:
+    return
+
+  # All the lines have been processed, report the errors found.
+  for required_header_unstripped in required:
+    template = required[required_header_unstripped][1]
+    if required_header_unstripped.strip('<>"') not in include_dict:
+      error(filename, required[required_header_unstripped][0],
+            'build/include_what_you_use', 4,
+            'Add #include ' + required_header_unstripped + ' for ' + template)
+
+
+_RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<')
+
+
+def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error):
+  """Check that make_pair's template arguments are deduced.
+
+  G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are
+  specified explicitly, and such use isn't intended in any case.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+  match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line)
+  if match:
+    error(filename, linenum, 'build/explicit_make_pair',
+          4,  # 4 = high confidence
+          'For C++11-compatibility, omit template arguments from make_pair'
+          ' OR use pair directly OR if appropriate, construct a pair directly')
+
+
+def CheckDefaultLambdaCaptures(filename, clean_lines, linenum, error):
+  """Check that default lambda captures are not used.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # A lambda introducer specifies a default capture if it starts with "[="
+  # or if it starts with "[&" _not_ followed by an identifier.
+  match = Match(r'^(.*)\[\s*(?:=|&[^\w])', line)
+  if match:
+    # Found a potential error, check what comes after the lambda-introducer.
+    # If it's not open parenthesis (for lambda-declarator) or open brace
+    # (for compound-statement), it's not a lambda.
+    line, _, pos = CloseExpression(clean_lines, linenum, len(match.group(1)))
+    if pos >= 0 and Match(r'^\s*[{(]', line[pos:]):
+      error(filename, linenum, 'build/c++11',
+            4,  # 4 = high confidence
+            'Default lambda captures are an unapproved C++ feature.')
+
+
+def CheckRedundantVirtual(filename, clean_lines, linenum, error):
+  """Check if line contains a redundant "virtual" function-specifier.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  # Look for "virtual" on current line.
+  line = clean_lines.elided[linenum]
+  virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line)
+  if not virtual: return
+
+  # Ignore "virtual" keywords that are near access-specifiers.  These
+  # are only used in class base-specifier and do not apply to member
+  # functions.
+  if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or
+      Match(r'^\s+(public|protected|private)\b', virtual.group(3))):
+    return
+
+  # Ignore the "virtual" keyword from virtual base classes.  Usually
+  # there is a column on the same line in these cases (virtual base
+  # classes are rare in google3 because multiple inheritance is rare).
+  if Match(r'^.*[^:]:[^:].*$', line): return
+
+  # Look for the next opening parenthesis.  This is the start of the
+  # parameter list (possibly on the next line shortly after virtual).
+  # TODO(unknown): doesn't work if there are virtual functions with
+  # decltype() or other things that use parentheses, but csearch suggests
+  # that this is rare.
+  end_col = -1
+  end_line = -1
+  start_col = len(virtual.group(2))
+  for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())):
+    line = clean_lines.elided[start_line][start_col:]
+    parameter_list = Match(r'^([^(]*)\(', line)
+    if parameter_list:
+      # Match parentheses to find the end of the parameter list
+      (_, end_line, end_col) = CloseExpression(
+          clean_lines, start_line, start_col + len(parameter_list.group(1)))
+      break
+    start_col = 0
+
+  if end_col < 0:
+    return  # Couldn't find end of parameter list, give up
+
+  # Look for "override" or "final" after the parameter list
+  # (possibly on the next few lines).
+  for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())):
+    line = clean_lines.elided[i][end_col:]
+    match = Search(r'\b(override|final)\b', line)
+    if match:
+      error(filename, linenum, 'readability/inheritance', 4,
+            ('"virtual" is redundant since function is '
+             'already declared as "%s"' % match.group(1)))
+
+    # Set end_col to check whole lines after we are done with the
+    # first line.
+    end_col = 0
+    if Search(r'[^\w]\s*$', line):
+      break
+
+
+def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error):
+  """Check if line contains a redundant "override" or "final" virt-specifier.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  # Look for closing parenthesis nearby.  We need one to confirm where
+  # the declarator ends and where the virt-specifier starts to avoid
+  # false positives.
+  line = clean_lines.elided[linenum]
+  declarator_end = line.rfind(')')
+  if declarator_end >= 0:
+    fragment = line[declarator_end:]
+  else:
+    if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0:
+      fragment = line
+    else:
+      return
+
+  # Check that at most one of "override" or "final" is present, not both
+  if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment):
+    error(filename, linenum, 'readability/inheritance', 4,
+          ('"override" is redundant since function is '
+           'already declared as "final"'))
+
+
+
+
+# Returns true if we are at a new block, and it is directly
+# inside of a namespace.
+def IsBlockInNameSpace(nesting_state, is_forward_declaration):
+  """Checks that the new block is directly in a namespace.
+
+  Args:
+    nesting_state: The _NestingState object that contains info about our state.
+    is_forward_declaration: If the class is a forward declared class.
+  Returns:
+    Whether or not the new block is directly in a namespace.
+  """
+  if is_forward_declaration:
+    if len(nesting_state.stack) >= 1 and (
+        isinstance(nesting_state.stack[-1], _NamespaceInfo)):
+      return True
+    else:
+      return False
+
+  return (len(nesting_state.stack) > 1 and
+          nesting_state.stack[-1].check_namespace_indentation and
+          isinstance(nesting_state.stack[-2], _NamespaceInfo))
+
+
+def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item,
+                                    raw_lines_no_comments, linenum):
+  """This method determines if we should apply our namespace indentation check.
+
+  Args:
+    nesting_state: The current nesting state.
+    is_namespace_indent_item: If we just put a new class on the stack, True.
+      If the top of the stack is not a class, or we did not recently
+      add the class, False.
+    raw_lines_no_comments: The lines without the comments.
+    linenum: The current line number we are processing.
+
+  Returns:
+    True if we should apply our namespace indentation check. Currently, it
+    only works for classes and namespaces inside of a namespace.
+  """
+
+  is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments,
+                                                     linenum)
+
+  if not (is_namespace_indent_item or is_forward_declaration):
+    return False
+
+  # If we are in a macro, we do not want to check the namespace indentation.
+  if IsMacroDefinition(raw_lines_no_comments, linenum):
+    return False
+
+  return IsBlockInNameSpace(nesting_state, is_forward_declaration)
+
+
+# Call this method if the line is directly inside of a namespace.
+# If the line above is blank (excluding comments) or the start of
+# an inner namespace, it cannot be indented.
+def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum,
+                                    error):
+  line = raw_lines_no_comments[linenum]
+  if Match(r'^\s+', line):
+    error(filename, linenum, 'runtime/indentation_namespace', 4,
+          'Do not indent within a namespace')
+
+
+def ProcessLine(filename, file_extension, clean_lines, line,
+                include_state, function_state, nesting_state, error,
+                extra_check_functions=[]):
+  """Processes a single line in the file.
+
+  Args:
+    filename: Filename of the file that is being processed.
+    file_extension: The extension (dot not included) of the file.
+    clean_lines: An array of strings, each representing a line of the file,
+                 with comments stripped.
+    line: Number of line being processed.
+    include_state: An _IncludeState instance in which the headers are inserted.
+    function_state: A _FunctionState instance which counts function lines, etc.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    error: A callable to which errors are reported, which takes 4 arguments:
+           filename, line number, error level, and message
+    extra_check_functions: An array of additional check functions that will be
+                           run on each source line. Each function takes 4
+                           arguments: filename, clean_lines, line, error
+  """
+  raw_lines = clean_lines.raw_lines
+  ParseNolintSuppressions(filename, raw_lines[line], line, error)
+  nesting_state.Update(filename, clean_lines, line, error)
+  CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line,
+                               error)
+  if nesting_state.InAsmBlock(): return
+  CheckForFunctionLengths(filename, clean_lines, line, function_state, error)
+  CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error)
+  CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error)
+  CheckLanguage(filename, clean_lines, line, file_extension, include_state,
+                nesting_state, error)
+  CheckForNonConstReference(filename, clean_lines, line, nesting_state, error)
+  CheckForNonStandardConstructs(filename, clean_lines, line,
+                                nesting_state, error)
+  CheckVlogArguments(filename, clean_lines, line, error)
+  CheckPosixThreading(filename, clean_lines, line, error)
+  CheckInvalidIncrement(filename, clean_lines, line, error)
+  CheckMakePairUsesDeduction(filename, clean_lines, line, error)
+  CheckDefaultLambdaCaptures(filename, clean_lines, line, error)
+  CheckRedundantVirtual(filename, clean_lines, line, error)
+  CheckRedundantOverrideOrFinal(filename, clean_lines, line, error)
+  for check_fn in extra_check_functions:
+    check_fn(filename, clean_lines, line, error)
+
+def FlagCxx11Features(filename, clean_lines, linenum, error):
+  """Flag those c++11 features that we only allow in certain places.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  # Flag unapproved C++11 headers.
+  include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line)
+  if include and include.group(1) in ('cfenv',
+                                      'condition_variable',
+                                      'fenv.h',
+                                      'future',
+                                      'mutex',
+                                      'thread',
+                                      'chrono',
+                                      'ratio',
+                                      'regex',
+                                      'system_error',
+                                     ):
+    error(filename, linenum, 'build/c++11', 5,
+          ('<%s> is an unapproved C++11 header.') % include.group(1))
+
+  # The only place where we need to worry about C++11 keywords and library
+  # features in preprocessor directives is in macro definitions.
+  if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return
+
+  # These are classes and free functions.  The classes are always
+  # mentioned as std::*, but we only catch the free functions if
+  # they're not found by ADL.  They're alphabetical by header.
+  for top_name in (
+      # type_traits
+      'alignment_of',
+      'aligned_union',
+      ):
+    if Search(r'\bstd::%s\b' % top_name, line):
+      error(filename, linenum, 'build/c++11', 5,
+            ('std::%s is an unapproved C++11 class or function.  Send c-style '
+             'an example of where it would make your code more readable, and '
+             'they may let you use it.') % top_name)
+
+
+def ProcessFileData(filename, file_extension, lines, error,
+                    extra_check_functions=[]):
+  """Performs lint checks and reports any errors to the given error function.
+
+  Args:
+    filename: Filename of the file that is being processed.
+    file_extension: The extension (dot not included) of the file.
+    lines: An array of strings, each representing a line of the file, with the
+           last element being empty if the file is terminated with a newline.
+    error: A callable to which errors are reported, which takes 4 arguments:
+           filename, line number, error level, and message
+    extra_check_functions: An array of additional check functions that will be
+                           run on each source line. Each function takes 4
+                           arguments: filename, clean_lines, line, error
+  """
+  lines = (['// marker so line numbers and indices both start at 1'] + lines +
+           ['// marker so line numbers end in a known way'])
+
+  include_state = _IncludeState()
+  function_state = _FunctionState()
+  nesting_state = NestingState()
+
+  ResetNolintSuppressions()
+
+  CheckForCopyright(filename, lines, error)
+
+  RemoveMultiLineComments(filename, lines, error)
+  clean_lines = CleansedLines(lines)
+
+  if file_extension == 'h':
+    CheckForHeaderGuard(filename, clean_lines, error)
+
+  for line in xrange(clean_lines.NumLines()):
+    ProcessLine(filename, file_extension, clean_lines, line,
+                include_state, function_state, nesting_state, error,
+                extra_check_functions)
+    FlagCxx11Features(filename, clean_lines, line, error)
+  nesting_state.CheckCompletedBlocks(filename, error)
+
+  CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error)
+  
+  # Check that the .cc file has included its header if it exists.
+  if file_extension == 'cc':
+    CheckHeaderFileIncluded(filename, include_state, error)
+
+  # We check here rather than inside ProcessLine so that we see raw
+  # lines rather than "cleaned" lines.
+  CheckForBadCharacters(filename, lines, error)
+
+  CheckForNewlineAtEOF(filename, lines, error)
+
+def ProcessConfigOverrides(filename):
+  """ Loads the configuration files and processes the config overrides.
+
+  Args:
+    filename: The name of the file being processed by the linter.
+
+  Returns:
+    False if the current |filename| should not be processed further.
+  """
+
+  abs_filename = os.path.abspath(filename)
+  cfg_filters = []
+  keep_looking = True
+  while keep_looking:
+    abs_path, base_name = os.path.split(abs_filename)
+    if not base_name:
+      break  # Reached the root directory.
+
+    cfg_file = os.path.join(abs_path, "CPPLINT.cfg")
+    abs_filename = abs_path
+    if not os.path.isfile(cfg_file):
+      continue
+
+    try:
+      with open(cfg_file) as file_handle:
+        for line in file_handle:
+          line, _, _ = line.partition('#')  # Remove comments.
+          if not line.strip():
+            continue
+
+          name, _, val = line.partition('=')
+          name = name.strip()
+          val = val.strip()
+          if name == 'set noparent':
+            keep_looking = False
+          elif name == 'filter':
+            cfg_filters.append(val)
+          elif name == 'exclude_files':
+            # When matching exclude_files pattern, use the base_name of
+            # the current file name or the directory name we are processing.
+            # For example, if we are checking for lint errors in /foo/bar/baz.cc
+            # and we found the .cfg file at /foo/CPPLINT.cfg, then the config
+            # file's "exclude_files" filter is meant to be checked against "bar"
+            # and not "baz" nor "bar/baz.cc".
+            if base_name:
+              pattern = re.compile(val)
+              if pattern.match(base_name):
+                sys.stderr.write('Ignoring "%s": file excluded by "%s". '
+                                 'File path component "%s" matches '
+                                 'pattern "%s"\n' %
+                                 (filename, cfg_file, base_name, val))
+                return False
+          elif name == 'linelength':
+            global _line_length
+            try:
+                _line_length = int(val)
+            except ValueError:
+                sys.stderr.write('Line length must be numeric.')
+          else:
+            sys.stderr.write(
+                'Invalid configuration option (%s) in file %s\n' %
+                (name, cfg_file))
+
+    except IOError:
+      sys.stderr.write(
+          "Skipping config file '%s': Can't open for reading\n" % cfg_file)
+      keep_looking = False
+
+  # Apply all the accumulated filters in reverse order (top-level directory
+  # config options having the least priority).
+  for filter in reversed(cfg_filters):
+     _AddFilters(filter)
+
+  return True
+
+
+def ProcessFile(filename, vlevel, extra_check_functions=[]):
+  """Does google-lint on a single file.
+
+  Args:
+    filename: The name of the file to parse.
+
+    vlevel: The level of errors to report.  Every error of confidence
+    >= verbose_level will be reported.  0 is a good default.
+
+    extra_check_functions: An array of additional check functions that will be
+                           run on each source line. Each function takes 4
+                           arguments: filename, clean_lines, line, error
+  """
+
+  _SetVerboseLevel(vlevel)
+  _BackupFilters()
+
+  if not ProcessConfigOverrides(filename):
+    _RestoreFilters()
+    return
+
+  lf_lines = []
+  crlf_lines = []
+  try:
+    # Support the UNIX convention of using "-" for stdin.  Note that
+    # we are not opening the file with universal newline support
+    # (which codecs doesn't support anyway), so the resulting lines do
+    # contain trailing '\r' characters if we are reading a file that
+    # has CRLF endings.
+    # If after the split a trailing '\r' is present, it is removed
+    # below.
+    if filename == '-':
+      lines = codecs.StreamReaderWriter(sys.stdin,
+                                        codecs.getreader('utf8'),
+                                        codecs.getwriter('utf8'),
+                                        'replace').read().split('\n')
+    else:
+      lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n')
+
+    # Remove trailing '\r'.
+    # The -1 accounts for the extra trailing blank line we get from split()
+    for linenum in range(len(lines) - 1):
+      if lines[linenum].endswith('\r'):
+        lines[linenum] = lines[linenum].rstrip('\r')
+        crlf_lines.append(linenum + 1)
+      else:
+        lf_lines.append(linenum + 1)
+
+  except IOError:
+    sys.stderr.write(
+        "Skipping input '%s': Can't open for reading\n" % filename)
+    _RestoreFilters()
+    return
+
+  # Note, if no dot is found, this will give the entire filename as the ext.
+  file_extension = filename[filename.rfind('.') + 1:]
+
+  # When reading from stdin, the extension is unknown, so no cpplint tests
+  # should rely on the extension.
+  if filename != '-' and file_extension not in _valid_extensions:
+    sys.stderr.write('Ignoring %s; not a valid file name '
+                     '(%s)\n' % (filename, ', '.join(_valid_extensions)))
+  else:
+    ProcessFileData(filename, file_extension, lines, Error,
+                    extra_check_functions)
+
+    # If end-of-line sequences are a mix of LF and CR-LF, issue
+    # warnings on the lines with CR.
+    #
+    # Don't issue any warnings if all lines are uniformly LF or CR-LF,
+    # since critique can handle these just fine, and the style guide
+    # doesn't dictate a particular end of line sequence.
+    #
+    # We can't depend on os.linesep to determine what the desired
+    # end-of-line sequence should be, since that will return the
+    # server-side end-of-line sequence.
+    if lf_lines and crlf_lines:
+      # Warn on every line with CR.  An alternative approach might be to
+      # check whether the file is mostly CRLF or just LF, and warn on the
+      # minority, we bias toward LF here since most tools prefer LF.
+      for linenum in crlf_lines:
+        Error(filename, linenum, 'whitespace/newline', 1,
+              'Unexpected \\r (^M) found; better to use only \\n')
+
+  sys.stderr.write('Done processing %s\n' % filename)
+  _RestoreFilters()
+
+
+def PrintUsage(message):
+  """Prints a brief usage string and exits, optionally with an error message.
+
+  Args:
+    message: The optional error message.
+  """
+  sys.stderr.write(_USAGE)
+  if message:
+    sys.exit('\nFATAL ERROR: ' + message)
+  else:
+    sys.exit(1)
+
+
+def PrintCategories():
+  """Prints a list of all the error-categories used by error messages.
+
+  These are the categories used to filter messages via --filter.
+  """
+  sys.stderr.write(''.join('  %s\n' % cat for cat in _ERROR_CATEGORIES))
+  sys.exit(0)
+
+
+def ParseArguments(args):
+  """Parses the command line arguments.
+
+  This may set the output format and verbosity level as side-effects.
+
+  Args:
+    args: The command line arguments:
+
+  Returns:
+    The list of filenames to lint.
+  """
+  try:
+    (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=',
+                                                 'counting=',
+                                                 'filter=',
+                                                 'root=',
+                                                 'linelength=',
+                                                 'extensions='])
+  except getopt.GetoptError:
+    PrintUsage('Invalid arguments.')
+
+  verbosity = _VerboseLevel()
+  output_format = _OutputFormat()
+  filters = ''
+  counting_style = ''
+
+  for (opt, val) in opts:
+    if opt == '--help':
+      PrintUsage(None)
+    elif opt == '--output':
+      if val not in ('emacs', 'vs7', 'eclipse'):
+        PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.')
+      output_format = val
+    elif opt == '--verbose':
+      verbosity = int(val)
+    elif opt == '--filter':
+      filters = val
+      if not filters:
+        PrintCategories()
+    elif opt == '--counting':
+      if val not in ('total', 'toplevel', 'detailed'):
+        PrintUsage('Valid counting options are total, toplevel, and detailed')
+      counting_style = val
+    elif opt == '--root':
+      global _root
+      _root = val
+    elif opt == '--linelength':
+      global _line_length
+      try:
+          _line_length = int(val)
+      except ValueError:
+          PrintUsage('Line length must be digits.')
+    elif opt == '--extensions':
+      global _valid_extensions
+      try:
+          _valid_extensions = set(val.split(','))
+      except ValueError:
+          PrintUsage('Extensions must be comma seperated list.')
+
+  if not filenames:
+    PrintUsage('No files were specified.')
+
+  _SetOutputFormat(output_format)
+  _SetVerboseLevel(verbosity)
+  _SetFilters(filters)
+  _SetCountingStyle(counting_style)
+
+  return filenames
+
+
+def main():
+  filenames = ParseArguments(sys.argv[1:])
+
+  # Change stderr to write with replacement characters so we don't die
+  # if we try to print something containing non-ASCII characters.
+  sys.stderr = codecs.StreamReaderWriter(sys.stderr,
+                                         codecs.getreader('utf8'),
+                                         codecs.getwriter('utf8'),
+                                         'replace')
+
+  _cpplint_state.ResetErrorCounts()
+  for filename in filenames:
+    ProcessFile(filename, _cpplint_state.verbose_level)
+  _cpplint_state.PrintErrorCounts()
+
+  sys.exit(_cpplint_state.error_count > 0)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh
new file mode 100755
index 0000000000000..b2039134d558d
--- /dev/null
+++ b/cpp/build-support/run-test.sh
@@ -0,0 +1,195 @@
+#!/bin/bash
+# Copyright 2014 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Script which wraps running a test and redirects its output to a
+# test log directory.
+#
+# If KUDU_COMPRESS_TEST_OUTPUT is non-empty, then the logs will be
+# gzip-compressed while they are written.
+#
+# If KUDU_FLAKY_TEST_ATTEMPTS is non-zero, and the test being run matches
+# one of the lines in the file KUDU_FLAKY_TEST_LIST, then the test will
+# be retried on failure up to the specified number of times. This can be
+# used in the gerrit workflow to prevent annoying false -1s caused by
+# tests that are known to be flaky in master.
+#
+# If KUDU_REPORT_TEST_RESULTS is non-zero, then tests are reported to the
+# central test server.
+
+ROOT=$(cd $(dirname $BASH_SOURCE)/..; pwd)
+
+TEST_LOGDIR=$ROOT/build/test-logs
+mkdir -p $TEST_LOGDIR
+
+TEST_DEBUGDIR=$ROOT/build/test-debug
+mkdir -p $TEST_DEBUGDIR
+
+TEST_DIRNAME=$(cd $(dirname $1); pwd)
+TEST_FILENAME=$(basename $1)
+shift
+TEST_EXECUTABLE="$TEST_DIRNAME/$TEST_FILENAME"
+TEST_NAME=$(echo $TEST_FILENAME | perl -pe 's/\..+?$//') # Remove path and extension (if any).
+
+# We run each test in its own subdir to avoid core file related races.
+TEST_WORKDIR=$ROOT/build/test-work/$TEST_NAME
+mkdir -p $TEST_WORKDIR
+pushd $TEST_WORKDIR >/dev/null || exit 1
+rm -f *
+
+set -o pipefail
+
+LOGFILE=$TEST_LOGDIR/$TEST_NAME.txt
+XMLFILE=$TEST_LOGDIR/$TEST_NAME.xml
+
+TEST_EXECUTION_ATTEMPTS=1
+
+# Remove both the uncompressed output, so the developer doesn't accidentally get confused
+# and read output from a prior test run.
+rm -f $LOGFILE $LOGFILE.gz
+
+pipe_cmd=cat
+
+# Configure TSAN (ignored if this isn't a TSAN build).
+#
+# Deadlock detection (new in clang 3.5) is disabled because:
+# 1. The clang 3.5 deadlock detector crashes in some unit tests. It
+#    needs compiler-rt commits c4c3dfd, 9a8efe3, and possibly others.
+# 2. Many unit tests report lock-order-inversion warnings; they should be
+#    fixed before reenabling the detector.
+TSAN_OPTIONS="$TSAN_OPTIONS detect_deadlocks=0"
+TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$ROOT/build-support/tsan-suppressions.txt"
+TSAN_OPTIONS="$TSAN_OPTIONS history_size=7"
+export TSAN_OPTIONS
+
+# Enable leak detection even under LLVM 3.4, where it was disabled by default.
+# This flag only takes effect when running an ASAN build.
+ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
+export ASAN_OPTIONS
+
+# Set up suppressions for LeakSanitizer
+LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt"
+export LSAN_OPTIONS
+
+# Suppressions require symbolization. We'll default to using the symbolizer in
+# thirdparty.
+if [ -z "$ASAN_SYMBOLIZER_PATH" ]; then
+  export ASAN_SYMBOLIZER_PATH=$(find $NATIVE_TOOLCHAIN/llvm-3.7.0/bin -name llvm-symbolizer)
+fi
+
+# Allow for collecting core dumps.
+ARROW_TEST_ULIMIT_CORE=${ARROW_TEST_ULIMIT_CORE:-0}
+ulimit -c $ARROW_TEST_ULIMIT_CORE
+
+# Run the actual test.
+for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
+  if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
+    # If the test fails, the test output may or may not be left behind,
+    # depending on whether the test cleaned up or exited immediately. Either
+    # way we need to clean it up. We do this by comparing the data directory
+    # contents before and after the test runs, and deleting anything new.
+    #
+    # The comm program requires that its two inputs be sorted.
+    TEST_TMPDIR_BEFORE=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
+  fi
+
+  # gtest won't overwrite old junit test files, resulting in a build failure
+  # even when retries are successful.
+  rm -f $XMLFILE
+
+  echo "Running $TEST_NAME, redirecting output into $LOGFILE" \
+    "(attempt ${ATTEMPT_NUMBER}/$TEST_EXECUTION_ATTEMPTS)"
+  $TEST_EXECUTABLE "$@" 2>&1 \
+    | $ROOT/build-support/asan_symbolize.py \
+    | c++filt \
+    | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \
+    | $pipe_cmd > $LOGFILE
+  STATUS=$?
+
+  # TSAN doesn't always exit with a non-zero exit code due to a bug:
+  # mutex errors don't get reported through the normal error reporting infrastructure.
+  # So we make sure to detect this and exit 1.
+  #
+  # Additionally, certain types of failures won't show up in the standard JUnit
+  # XML output from gtest. We assume that gtest knows better than us and our
+  # regexes in most cases, but for certain errors we delete the resulting xml
+  # file and let our own post-processing step regenerate it.
+  export GREP=$(which egrep)
+  if zgrep --silent "ThreadSanitizer|Leak check.*detected leaks" $LOGFILE ; then
+    echo ThreadSanitizer or leak check failures in $LOGFILE
+    STATUS=1
+    rm -f $XMLFILE
+  fi
+
+  if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
+    # Now delete any new test output.
+    TEST_TMPDIR_AFTER=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
+    DIFF=$(comm -13 <(echo "$TEST_TMPDIR_BEFORE") \
+                    <(echo "$TEST_TMPDIR_AFTER"))
+    for DIR in $DIFF; do
+      # Multiple tests may be running concurrently. To avoid deleting the
+      # wrong directories, constrain to only directories beginning with the
+      # test name.
+      #
+      # This may delete old test directories belonging to this test, but
+      # that's not typically a concern when rerunning flaky tests.
+      if [[ $DIR =~ ^$TEST_TMPDIR/$TEST_NAME ]]; then
+        echo Deleting leftover flaky test directory "$DIR"
+        rm -Rf "$DIR"
+      fi
+    done
+  fi
+
+  if [ "$STATUS" -eq "0" ]; then
+    break
+  elif [ "$ATTEMPT_NUMBER" -lt "$TEST_EXECUTION_ATTEMPTS" ]; then
+    echo Test failed attempt number $ATTEMPT_NUMBER
+    echo Will retry...
+  fi
+done
+
+# If we have a LeakSanitizer report, and XML reporting is configured, add a new test
+# case result to the XML file for the leak report. Otherwise Jenkins won't show
+# us which tests had LSAN errors.
+if zgrep --silent "ERROR: LeakSanitizer: detected memory leaks" $LOGFILE ; then
+    echo Test had memory leaks. Editing XML
+    perl -p -i -e '
+    if (m#</testsuite>#) {
+      print "<testcase name=\"LeakSanitizer\" status=\"run\" classname=\"LSAN\">\n";
+      print "  <failure message=\"LeakSanitizer failed\" type=\"\">\n";
+      print "    See txt log file for details\n";
+      print "  </failure>\n";
+      print "</testcase>\n";
+    }' $XMLFILE
+fi
+
+# Capture and compress core file and binary.
+COREFILES=$(ls | grep ^core)
+if [ -n "$COREFILES" ]; then
+  echo Found core dump. Saving executable and core files.
+  gzip < $TEST_EXECUTABLE > "$TEST_DEBUGDIR/$TEST_NAME.gz" || exit $?
+  for COREFILE in $COREFILES; do
+    gzip < $COREFILE > "$TEST_DEBUGDIR/$TEST_NAME.$COREFILE.gz" || exit $?
+  done
+  # Pull in any .so files as well.
+  for LIB in $(ldd $TEST_EXECUTABLE | grep $ROOT | awk '{print $3}'); do
+    LIB_NAME=$(basename $LIB)
+    gzip < $LIB > "$TEST_DEBUGDIR/$LIB_NAME.gz" || exit $?
+  done
+fi
+
+popd
+rm -Rf $TEST_WORKDIR
+
+exit $STATUS
diff --git a/cpp/build-support/stacktrace_addr2line.pl b/cpp/build-support/stacktrace_addr2line.pl
new file mode 100755
index 0000000000000..7664bab5af65b
--- /dev/null
+++ b/cpp/build-support/stacktrace_addr2line.pl
@@ -0,0 +1,92 @@
+#!/usr/bin/perl
+# Copyright 2014 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################
+# This script will convert a stack trace with addresses:
+#     @           0x5fb015 kudu::master::Master::Init()
+#     @           0x5c2d38 kudu::master::MiniMaster::StartOnPorts()
+#     @           0x5c31fa kudu::master::MiniMaster::Start()
+#     @           0x58270a kudu::MiniCluster::Start()
+#     @           0x57dc71 kudu::CreateTableStressTest::SetUp()
+# To one with line numbers:
+#     @           0x5fb015 kudu::master::Master::Init() at /home/mpercy/src/kudu/src/master/master.cc:54
+#     @           0x5c2d38 kudu::master::MiniMaster::StartOnPorts() at /home/mpercy/src/kudu/src/master/mini_master.cc:52
+#     @           0x5c31fa kudu::master::MiniMaster::Start() at /home/mpercy/src/kudu/src/master/mini_master.cc:33
+#     @           0x58270a kudu::MiniCluster::Start() at /home/mpercy/src/kudu/src/integration-tests/mini_cluster.cc:48
+#     @           0x57dc71 kudu::CreateTableStressTest::SetUp() at /home/mpercy/src/kudu/src/integration-tests/create-table-stress-test.cc:61
+#
+# If the script detects that the output is not symbolized, it will also attempt
+# to determine the function names, i.e. it will convert:
+#     @           0x5fb015
+#     @           0x5c2d38
+#     @           0x5c31fa
+# To:
+#     @           0x5fb015 kudu::master::Master::Init() at /home/mpercy/src/kudu/src/master/master.cc:54
+#     @           0x5c2d38 kudu::master::MiniMaster::StartOnPorts() at /home/mpercy/src/kudu/src/master/mini_master.cc:52
+#     @           0x5c31fa kudu::master::MiniMaster::Start() at /home/mpercy/src/kudu/src/master/mini_master.cc:33
+#######################################################################
+use strict;
+use warnings;
+
+if (!@ARGV) {
+  die <<EOF
+Usage: $0 executable [stack-trace-file]
+
+This script will read addresses from a file containing stack traces and
+will convert the addresses that conform to the pattern " @ 0x123456" to line
+numbers by calling addr2line on the provided executable.
+If no stack-trace-file is specified, it will take input from stdin.
+EOF
+}
+
+# el6 and other older systems don't support the -p flag,
+# so we do our own "pretty" parsing.
+sub parse_addr2line_output($$) {
+  defined(my $output = shift) or die;
+  defined(my $lookup_func_name = shift) or die;
+  my @lines = grep { $_ ne '' } split("\n", $output);
+  my $pretty_str = '';
+  if ($lookup_func_name) {
+    $pretty_str .= ' ' . $lines[0];
+  }
+  $pretty_str .= ' at ' . $lines[1];
+  return $pretty_str;
+}
+
+my $binary = shift @ARGV;
+if (! -x $binary || ! -r $binary) {
+  die "Error: Cannot access executable ($binary)";
+}
+
+# Cache lookups to speed processing of files with repeated trace addresses.
+my %addr2line_map = ();
+
+# Disable stdout buffering
+$| = 1;
+
+# Reading from <ARGV> is magical in Perl.
+while (defined(my $input = <ARGV>)) {
+  if ($input =~ /^\s+\@\s+(0x[[:xdigit:]]{6,})(?:\s+(\S+))?/) {
+    my $addr = $1;
+    my $lookup_func_name = (!defined $2);
+    if (!exists($addr2line_map{$addr})) {
+      $addr2line_map{$addr} = `addr2line -ifC -e $binary $addr`;
+    }
+    chomp $input;
+    $input .= parse_addr2line_output($addr2line_map{$addr}, $lookup_func_name) . "\n";
+  }
+  print $input;
+}
+
+exit 0;
diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
new file mode 100644
index 0000000000000..07860682f9b1b
--- /dev/null
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -0,0 +1,46 @@
+# Copyright 2013 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Sets COMPILER_FAMILY to 'clang' or 'gcc'
+# Sets COMPILER_VERSION to the version
+execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
+                ERROR_VARIABLE COMPILER_VERSION_FULL)
+message(INFO " ${COMPILER_VERSION_FULL}")
+
+# clang on Linux and Mac OS X before 10.9
+if("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
+  set(COMPILER_FAMILY "clang")
+  string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
+    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+# clang on Mac OS X 10.9 and later
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
+  set(COMPILER_FAMILY "clang")
+  string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
+    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+
+# clang on Mac OS X, XCode 7. No version replacement is done
+# because Apple no longer advertises the upstream LLVM version.
+elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-700\\..*")
+  set(COMPILER_FAMILY "clang")
+
+# gcc
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*gcc version.*")
+  set(COMPILER_FAMILY "gcc")
+  string(REGEX REPLACE ".*gcc version ([0-9\\.]+).*" "\\1"
+    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+else()
+  message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
+endif()
+message("Selected compiler ${COMPILER_FAMILY} ${COMPILER_VERSION}")
+
diff --git a/cpp/cmake_modules/FindGPerf.cmake b/cpp/cmake_modules/FindGPerf.cmake
new file mode 100644
index 0000000000000..e8310799c3671
--- /dev/null
+++ b/cpp/cmake_modules/FindGPerf.cmake
@@ -0,0 +1,69 @@
+# -*- cmake -*-
+
+# - Find Google perftools
+# Find the Google perftools includes and libraries
+# This module defines
+#  GOOGLE_PERFTOOLS_INCLUDE_DIR, where to find heap-profiler.h, etc.
+#  GOOGLE_PERFTOOLS_FOUND, If false, do not try to use Google perftools.
+# also defined for general use are
+#  TCMALLOC_LIBS, where to find the tcmalloc libraries.
+#  TCMALLOC_STATIC_LIB, path to libtcmalloc.a.
+#  TCMALLOC_SHARED_LIB, path to libtcmalloc's shared library
+#  PROFILER_LIBS, where to find the profiler libraries.
+#  PROFILER_STATIC_LIB, path to libprofiler.a.
+#  PROFILER_SHARED_LIB, path to libprofiler's shared library
+
+FIND_PATH(GOOGLE_PERFTOOLS_INCLUDE_DIR google/heap-profiler.h
+  $ENV{NATIVE_TOOLCHAIN}/gperftools-$ENV{GPERFTOOLS_VERSION}/include
+  NO_DEFAULT_PATH
+)
+
+SET(GPERF_LIB_SEARCH $ENV{NATIVE_TOOLCHAIN}/gperftools-$ENV{GPERFTOOLS_VERSION}/lib)
+
+FIND_LIBRARY(TCMALLOC_LIB_PATH
+  NAMES libtcmalloc.a
+  PATHS ${GPERF_LIB_SEARCH}
+  NO_DEFAULT_PATH
+)
+
+IF (TCMALLOC_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR)
+    SET(TCMALLOC_LIBS ${GPERF_LIB_SEARCH})
+    SET(TCMALLOC_LIB_NAME libtcmalloc)
+    SET(TCMALLOC_STATIC_LIB ${GPERF_LIB_SEARCH}/${TCMALLOC_LIB_NAME}.a)
+    SET(TCMALLOC_SHARED_LIB ${TCMALLOC_LIBS}/${TCMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    SET(GOOGLE_PERFTOOLS_FOUND "YES")
+ELSE (TCMALLOC_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR)
+  SET(GOOGLE_PERFTOOLS_FOUND "NO")
+ENDIF (TCMALLOC_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR)
+
+FIND_LIBRARY(PROFILER_LIB_PATH
+  NAMES libprofiler.a
+  PATHS ${GPERF_LIB_SEARCH}
+)
+
+IF (PROFILER_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR)
+  SET(PROFILER_LIBS ${GPERF_LIB_SEARCH})
+  SET(PROFILER_LIB_NAME libprofiler)
+  SET(PROFILER_STATIC_LIB ${GPERF_LIB_SEARCH}/${PROFILER_LIB_NAME}.a)
+  SET(PROFILER_SHARED_LIB ${PROFILER_LIBS}/${PROFILER_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ENDIF (PROFILER_LIB_PATH AND GOOGLE_PERFTOOLS_INCLUDE_DIR)
+
+IF (GOOGLE_PERFTOOLS_FOUND)
+  IF (NOT GPerf_FIND_QUIETLY)
+    MESSAGE(STATUS "Found the Google perftools library: ${TCMALLOC_LIBS}")
+  ENDIF (NOT GPerf_FIND_QUIETLY)
+ELSE (GOOGLE_PERFTOOLS_FOUND)
+  IF (GPerf_FIND_REQUIRED)
+    MESSAGE(FATAL_ERROR "Could not find the Google perftools library")
+  ENDIF (GPerf_FIND_REQUIRED)
+ENDIF (GOOGLE_PERFTOOLS_FOUND)
+
+MARK_AS_ADVANCED(
+  TCMALLOC_LIBS
+  TCMALLOC_STATIC_LIB
+  TCMALLOC_SHARED_LIB
+  PROFILER_LIBS
+  PROFILER_STATIC_LIB
+  PROFILER_SHARED_LIB
+  GOOGLE_PERFTOOLS_INCLUDE_DIR
+)
diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake
new file mode 100644
index 0000000000000..e47faf0dd89d2
--- /dev/null
+++ b/cpp/cmake_modules/FindGTest.cmake
@@ -0,0 +1,91 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find GTest headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(GTest)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  GTest_HOME - When set, this path is inspected instead of standard library
+#                locations as the root of the GTest installation.
+#                The environment variable GTEST_HOME overrides this veriable.
+#
+# This module defines
+#  GTEST_INCLUDE_DIR, directory containing headers
+#  GTEST_LIBS, directory containing gtest libraries
+#  GTEST_STATIC_LIB, path to libgtest.a
+#  GTEST_SHARED_LIB, path to libgtest's shared library
+#  GTEST_FOUND, whether gtest has been found
+
+if( NOT "$ENV{GTEST_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "$ENV{GTEST_HOME}" _native_path )
+    list( APPEND _gtest_roots ${_native_path} )
+elseif ( GTest_HOME )
+    list( APPEND _gtest_roots ${GTest_HOME} )
+endif()
+
+# Try the parameterized roots, if they exist
+if ( _gtest_roots )
+    find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h
+        PATHS ${_gtest_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "include" )
+    find_library( GTEST_LIBRARIES NAMES gtest
+        PATHS ${_gtest_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib" )
+else ()
+    find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h )
+    find_library( GTEST_LIBRARIES NAMES gtest )
+endif ()
+
+
+if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
+  set(GTEST_FOUND TRUE)
+  get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} DIRECTORY )
+  set(GTEST_LIB_NAME libgtest)
+  set(GTEST_STATIC_LIB ${GTEST_LIBS}/${GTEST_LIB_NAME}.a)
+  set(GTEST_SHARED_LIB ${GTEST_LIBS}/${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+else ()
+  set(GTEST_FOUND FALSE)
+endif ()
+
+if (GTEST_FOUND)
+  if (NOT GTest_FIND_QUIETLY)
+    message(STATUS "Found the GTest library: ${GTEST_LIBRARIES}")
+  endif ()
+else ()
+  if (NOT GTest_FIND_QUIETLY)
+    set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ")
+    if ( _gtest_roots )
+      set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.")
+    else ()
+      set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.")
+    endif ()
+    if (GTest_FIND_REQUIRED)
+      message(FATAL_ERROR "${GTEST_ERR_MSG}")
+    else (GTest_FIND_REQUIRED)
+      message(STATUS "${GTEST_ERR_MSG}")
+    endif (GTest_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  GTEST_INCLUDE_DIR
+  GTEST_LIBS
+  GTEST_LIBRARIES
+  GTEST_STATIC_LIB
+  GTEST_SHARED_LIB
+)
diff --git a/cpp/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
new file mode 100644
index 0000000000000..76c2d1dbee941
--- /dev/null
+++ b/cpp/cmake_modules/FindParquet.cmake
@@ -0,0 +1,80 @@
+# Copyright 2012 Cloudera Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# - Find PARQUET (parquet/parquet.h, libparquet.a, libparquet.so)
+# This module defines
+#  PARQUET_INCLUDE_DIR, directory containing headers
+#  PARQUET_LIBS, directory containing parquet libraries
+#  PARQUET_STATIC_LIB, path to libparquet.a
+#  PARQUET_SHARED_LIB, path to libparquet's shared library
+#  PARQUET_FOUND, whether parquet has been found
+
+if( NOT "$ENV{PARQUET_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "$ENV{PARQUET_HOME}" _native_path )
+    list( APPEND _parquet_roots ${_native_path} )
+elseif ( Parquet_HOME )
+    list( APPEND _parquet_roots ${Parquet_HOME} )
+endif()
+
+# Try the parameterized roots, if they exist
+if ( _parquet_roots )
+    find_path( PARQUET_INCLUDE_DIR NAMES parquet/parquet.h
+        PATHS ${_parquet_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "include" )
+    find_library( PARQUET_LIBRARIES NAMES parquet
+        PATHS ${_parquet_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib" )
+else ()
+    find_path( PARQUET_INCLUDE_DIR NAMES parquet/parquet.h )
+    find_library( PARQUET_LIBRARIES NAMES parquet )
+endif ()
+
+
+if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
+  set(PARQUET_FOUND TRUE)
+  get_filename_component( PARQUET_LIBS ${PARQUET_LIBRARIES} DIRECTORY )
+  set(PARQUET_LIB_NAME libparquet)
+  set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
+  set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+else ()
+  set(PARQUET_FOUND FALSE)
+endif ()
+
+if (PARQUET_FOUND)
+  if (NOT Parquet_FIND_QUIETLY)
+    message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}")
+  endif ()
+else ()
+  if (NOT Parquet_FIND_QUIETLY)
+    set(PARQUET_ERR_MSG "Could not find the Parquet library. Looked in ")
+    if ( _parquet_roots )
+      set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} in ${_parquet_roots}.")
+    else ()
+      set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} system search paths.")
+    endif ()
+    if (Parquet_FIND_REQUIRED)
+      message(FATAL_ERROR "${PARQUET_ERR_MSG}")
+    else (Parquet_FIND_REQUIRED)
+      message(STATUS "${PARQUET_ERR_MSG}")
+    endif (Parquet_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  PARQUET_INCLUDE_DIR
+  PARQUET_LIBS
+  PARQUET_LIBRARIES
+  PARQUET_STATIC_LIB
+  PARQUET_SHARED_LIB
+)
diff --git a/cpp/cmake_modules/san-config.cmake b/cpp/cmake_modules/san-config.cmake
new file mode 100644
index 0000000000000..b847c96657a4a
--- /dev/null
+++ b/cpp/cmake_modules/san-config.cmake
@@ -0,0 +1,92 @@
+# Clang does not support using ASAN and TSAN simultaneously.
+if ("${ARROW_USE_ASAN}" AND "${ARROW_USE_TSAN}")
+  message(SEND_ERROR "Can only enable one of ASAN or TSAN at a time")
+endif()
+
+# Flag to enable clang address sanitizer
+# This will only build if clang or a recent enough gcc is the chosen compiler
+if (${ARROW_USE_ASAN})
+  if(NOT (("${COMPILER_FAMILY}" STREQUAL "clang") OR
+          ("${COMPILER_FAMILY}" STREQUAL "gcc" AND "${COMPILER_VERSION}" VERSION_GREATER "4.8")))
+    message(SEND_ERROR "Cannot use ASAN without clang or gcc >= 4.8")
+  endif()
+
+  # If UBSAN is also enabled, and we're on clang < 3.5, ensure static linking is
+  # enabled. Otherwise, we run into https://llvm.org/bugs/show_bug.cgi?id=18211
+  if("${ARROW_USE_UBSAN}" AND
+      "${COMPILER_FAMILY}" STREQUAL "clang" AND
+      "${COMPILER_VERSION}" VERSION_LESS "3.5")
+    if("${ARROW_LINK}" STREQUAL "a")
+      message("Using static linking for ASAN+UBSAN build")
+      set(ARROW_LINK "s")
+    elseif("${ARROW_LINK}" STREQUAL "d")
+      message(SEND_ERROR "Cannot use dynamic linking when ASAN and UBSAN are both enabled")
+    endif()
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -DADDRESS_SANITIZER")
+endif()
+
+
+# Flag to enable clang undefined behavior sanitizer
+# We explicitly don't enable all of the sanitizer flags:
+# - disable 'vptr' because it currently crashes somewhere in boost::intrusive::list code
+# - disable 'alignment' because unaligned access is really OK on Nehalem and we do it
+#   all over the place.
+if (${ARROW_USE_UBSAN})
+  if(NOT (("${COMPILER_FAMILY}" STREQUAL "clang") OR
+          ("${COMPILER_FAMILY}" STREQUAL "gcc" AND "${COMPILER_VERSION}" VERSION_GREATER "4.9")))
+    message(SEND_ERROR "Cannot use UBSAN without clang or gcc >= 4.9")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-sanitize=alignment,vptr -fno-sanitize-recover")
+endif ()
+
+# Flag to enable thread sanitizer (clang or gcc 4.8)
+if (${ARROW_USE_TSAN})
+  if(NOT (("${COMPILER_FAMILY}" STREQUAL "clang") OR
+          ("${COMPILER_FAMILY}" STREQUAL "gcc" AND "${COMPILER_VERSION}" VERSION_GREATER "4.8")))
+    message(SEND_ERROR "Cannot use TSAN without clang or gcc >= 4.8")
+  endif()
+
+  add_definitions("-fsanitize=thread")
+
+  # Enables dynamic_annotations.h to actually generate code
+  add_definitions("-DDYNAMIC_ANNOTATIONS_ENABLED")
+
+  # changes atomicops to use the tsan implementations
+  add_definitions("-DTHREAD_SANITIZER")
+
+  # Disables using the precompiled template specializations for std::string, shared_ptr, etc
+  # so that the annotations in the header actually take effect.
+  add_definitions("-D_GLIBCXX_EXTERN_TEMPLATE=0")
+
+  # Some of the above also need to be passed to the linker.
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pie -fsanitize=thread")
+
+  # Strictly speaking, TSAN doesn't require dynamic linking. But it does
+  # require all code to be position independent, and the easiest way to
+  # guarantee that is via dynamic linking (not all 3rd party archives are
+  # compiled with -fPIC e.g. boost).
+  if("${ARROW_LINK}" STREQUAL "a")
+    message("Using dynamic linking for TSAN")
+    set(ARROW_LINK "d")
+  elseif("${ARROW_LINK}" STREQUAL "s")
+    message(SEND_ERROR "Cannot use TSAN with static linking")
+  endif()
+endif()
+
+
+if ("${ARROW_USE_UBSAN}" OR "${ARROW_USE_ASAN}" OR "${ARROW_USE_TSAN}")
+  # GCC 4.8 and 4.9 (latest as of this writing) don't allow you to specify a
+  # sanitizer blacklist.
+  if("${COMPILER_FAMILY}" STREQUAL "clang")
+    # Require clang 3.4 or newer; clang 3.3 has issues with TSAN and pthread
+    # symbol interception.
+    if("${COMPILER_VERSION}" VERSION_LESS "3.4")
+      message(SEND_ERROR "Must use clang 3.4 or newer to run a sanitizer build."
+        " Try using clang from $NATIVE_TOOLCHAIN/")
+    endif()
+    add_definitions("-fsanitize-blacklist=${BUILD_SUPPORT_DIR}/sanitize-blacklist.txt")
+  else()
+    message(WARNING "GCC does not support specifying a sanitizer blacklist. Known sanitizer check failures will not be suppressed.")
+  endif()
+endif()
diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
new file mode 100755
index 0000000000000..457b9717ebe81
--- /dev/null
+++ b/cpp/setup_build_env.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+SOURCE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+
+./thirdparty/download_thirdparty.sh
+./thirdparty/build_thirdparty.sh
+
+export GTEST_HOME=$SOURCE_DIR/thirdparty/$GTEST_BASEDIR
+
+echo "Build env initialized"
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
new file mode 100644
index 0000000000000..eeea2dbc517b4
--- /dev/null
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Headers: top level
+install(FILES
+  api.h
+  array.h
+  builder.h
+  type.h
+  DESTINATION include/arrow)
+
+#######################################
+# Unit tests
+#######################################
+
+set(ARROW_TEST_LINK_LIBS arrow_test_util ${ARROW_MIN_TEST_LIBS})
+
+ADD_ARROW_TEST(array-test)
+ADD_ARROW_TEST(field-test)
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
new file mode 100644
index 0000000000000..899e8aae19c0e
--- /dev/null
+++ b/cpp/src/arrow/api.h
@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_API_H
+#define ARROW_API_H
+
+#endif // ARROW_API_H
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
new file mode 100644
index 0000000000000..5ecf91624fe73
--- /dev/null
+++ b/cpp/src/arrow/array-test.cc
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/buffer.h"
+
+using std::string;
+using std::vector;
+
+namespace arrow {
+
+static TypePtr int32 = TypePtr(new Int32Type());
+static TypePtr int32_nn = TypePtr(new Int32Type(false));
+
+
+class TestArray : public ::testing::Test {
+ public:
+  void SetUp() {
+    auto data = std::make_shared<OwnedMutableBuffer>();
+    auto nulls = std::make_shared<OwnedMutableBuffer>();
+
+    ASSERT_OK(data->Resize(400));
+    ASSERT_OK(nulls->Resize(128));
+
+    arr_.reset(new Int32Array(100, data, nulls));
+  }
+
+ protected:
+  std::unique_ptr<Int32Array> arr_;
+};
+
+
+TEST_F(TestArray, TestNullable) {
+  std::shared_ptr<Buffer> tmp = arr_->data();
+  std::unique_ptr<Int32Array> arr_nn(new Int32Array(100, tmp));
+
+  ASSERT_TRUE(arr_->nullable());
+  ASSERT_FALSE(arr_nn->nullable());
+}
+
+
+TEST_F(TestArray, TestLength) {
+  ASSERT_EQ(arr_->length(), 100);
+}
+
+TEST_F(TestArray, TestIsNull) {
+  vector<uint8_t> nulls = {1, 0, 1, 1, 0, 1, 0, 0,
+                           1, 0, 1, 1, 0, 1, 0, 0,
+                           1, 0, 1, 1, 0, 1, 0, 0,
+                           1, 0, 1, 1, 0, 1, 0, 0,
+                           1, 0, 0, 1};
+
+  std::shared_ptr<Buffer> null_buf = bytes_to_null_buffer(nulls.data(), nulls.size());
+  std::unique_ptr<Array> arr;
+  arr.reset(new Array(int32, nulls.size(), null_buf));
+
+  ASSERT_EQ(null_buf->size(), 5);
+  for (size_t i = 0; i < nulls.size(); ++i) {
+    ASSERT_EQ(static_cast<bool>(nulls[i]), arr->IsNull(i));
+  }
+}
+
+
+TEST_F(TestArray, TestCopy) {
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
new file mode 100644
index 0000000000000..1726a2f27d82d
--- /dev/null
+++ b/cpp/src/arrow/array.cc
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/array.h"
+
+#include "arrow/util/buffer.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// Base array class
+
+Array::Array(const TypePtr& type, int64_t length,
+    const std::shared_ptr<Buffer>& nulls) {
+  Init(type, length, nulls);
+}
+
+void Array::Init(const TypePtr& type, int64_t length,
+    const std::shared_ptr<Buffer>& nulls) {
+  type_ = type;
+  length_ = length;
+  nulls_ = nulls;
+
+  nullable_ = type->nullable;
+  if (nulls_) {
+    null_bits_ = nulls_->data();
+  }
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
new file mode 100644
index 0000000000000..c95450d12a419
--- /dev/null
+++ b/cpp/src/arrow/array.h
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_ARRAY_H
+#define ARROW_ARRAY_H
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+
+#include "arrow/type.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/macros.h"
+
+namespace arrow {
+
+class Buffer;
+
+// Immutable data array with some logical type and some length. Any memory is
+// owned by the respective Buffer instance (or its parents). May or may not be
+// nullable.
+//
+// The base class only has a null array (if the data type is nullable)
+//
+// Any buffers used to initialize the array have their references "stolen". If
+// you wish to use the buffer beyond the lifetime of the array, you need to
+// explicitly increment its reference count
+class Array {
+ public:
+  Array() : length_(0), nulls_(nullptr), null_bits_(nullptr) {}
+  Array(const TypePtr& type, int64_t length,
+      const std::shared_ptr<Buffer>& nulls = nullptr);
+
+  virtual ~Array() {}
+
+  void Init(const TypePtr& type, int64_t length, const std::shared_ptr<Buffer>& nulls);
+
+  // Determine if a slot if null. For inner loops. Does *not* boundscheck
+  bool IsNull(int64_t i) const {
+    return nullable_ && util::get_bit(null_bits_, i);
+  }
+
+  int64_t length() const { return length_;}
+  bool nullable() const { return nullable_;}
+  const TypePtr& type() const { return type_;}
+  TypeEnum type_enum() const { return type_->type;}
+
+ protected:
+  TypePtr type_;
+  bool nullable_;
+  int64_t length_;
+
+  std::shared_ptr<Buffer> nulls_;
+  const uint8_t* null_bits_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(Array);
+};
+
+
+typedef std::shared_ptr<Array> ArrayPtr;
+
+} // namespace arrow
+
+#endif
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
new file mode 100644
index 0000000000000..1fd7471928367
--- /dev/null
+++ b/cpp/src/arrow/builder.cc
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/builder.h"
+
+#include <cstring>
+
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+Status ArrayBuilder::Init(int64_t capacity) {
+  capacity_ = capacity;
+
+  if (nullable_) {
+    int64_t to_alloc = util::ceil_byte(capacity) / 8;
+    nulls_ = std::make_shared<OwnedMutableBuffer>();
+    RETURN_NOT_OK(nulls_->Resize(to_alloc));
+    null_bits_ = nulls_->mutable_data();
+    memset(null_bits_, 0, to_alloc);
+  }
+  return Status::OK();
+}
+
+Status ArrayBuilder::Resize(int64_t new_bits) {
+  if (nullable_) {
+    int64_t new_bytes = util::ceil_byte(new_bits) / 8;
+    int64_t old_bytes = nulls_->size();
+    RETURN_NOT_OK(nulls_->Resize(new_bytes));
+    null_bits_ = nulls_->mutable_data();
+    if (old_bytes < new_bytes) {
+      memset(null_bits_ + old_bytes, 0, new_bytes - old_bytes);
+    }
+  }
+  return Status::OK();
+}
+
+Status ArrayBuilder::Advance(int64_t elements) {
+  if (nullable_ && length_ + elements > capacity_) {
+    return Status::Invalid("Builder must be expanded");
+  }
+  length_ += elements;
+  return Status::OK();
+}
+
+
+} // namespace arrow
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
new file mode 100644
index 0000000000000..b43668af77cbd
--- /dev/null
+++ b/cpp/src/arrow/builder.h
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_BUILDER_H
+#define ARROW_BUILDER_H
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "arrow/type.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+class Array;
+
+static constexpr int64_t MIN_BUILDER_CAPACITY = 1 << 8;
+
+// Base class for all data array builders
+class ArrayBuilder {
+ public:
+  explicit ArrayBuilder(const TypePtr& type)
+      : type_(type),
+        nullable_(type_->nullable),
+        nulls_(nullptr), null_bits_(nullptr),
+        length_(0),
+        capacity_(0) {}
+
+  virtual ~ArrayBuilder() {}
+
+  // For nested types. Since the objects are owned by this class instance, we
+  // skip shared pointers and just return a raw pointer
+  ArrayBuilder* child(int i) {
+    return children_[i].get();
+  }
+
+  int num_children() const {
+    return children_.size();
+  }
+
+  int64_t length() const { return length_;}
+  int64_t capacity() const { return capacity_;}
+  bool nullable() const { return nullable_;}
+
+  // Allocates requires memory at this level, but children need to be
+  // initialized independently
+  Status Init(int64_t capacity);
+
+  // Resizes the nulls array (if nullable)
+  Status Resize(int64_t new_bits);
+
+  // For cases where raw data was memcpy'd into the internal buffers, allows us
+  // to advance the length of the builder. It is your responsibility to use
+  // this function responsibly.
+  Status Advance(int64_t elements);
+
+  const std::shared_ptr<OwnedMutableBuffer>& nulls() const { return nulls_;}
+
+  // Creates new array object to hold the contents of the builder and transfers
+  // ownership of the data
+  virtual Status ToArray(Array** out) = 0;
+
+ protected:
+  TypePtr type_;
+  bool nullable_;
+
+  // If the type is not nullable, then null_ is nullptr after initialization
+  std::shared_ptr<OwnedMutableBuffer> nulls_;
+  uint8_t* null_bits_;
+
+  // Array length, so far. Also, the index of the next element to be added
+  int64_t length_;
+  int64_t capacity_;
+
+  // Child value array builders. These are owned by this class
+  std::vector<std::unique_ptr<ArrayBuilder> > children_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
+};
+
+} // namespace arrow
+
+#endif // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/field-test.cc b/cpp/src/arrow/field-test.cc
new file mode 100644
index 0000000000000..2bb8bad4054c3
--- /dev/null
+++ b/cpp/src/arrow/field-test.cc
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <memory>
+#include <string>
+
+#include "arrow/field.h"
+#include "arrow/type.h"
+#include "arrow/types/integer.h"
+
+using std::string;
+
+namespace arrow {
+
+TEST(TestField, Basics) {
+  TypePtr ftype = TypePtr(new Int32Type());
+  Field f0("f0", ftype);
+
+  ASSERT_EQ(f0.name, "f0");
+  ASSERT_EQ(f0.type->ToString(), ftype->ToString());
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/field.h b/cpp/src/arrow/field.h
new file mode 100644
index 0000000000000..664cae61a777a
--- /dev/null
+++ b/cpp/src/arrow/field.h
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_FIELD_H
+#define ARROW_FIELD_H
+
+#include <string>
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+// A field is a piece of metadata that includes (for now) a name and a data
+// type
+
+struct Field {
+  // Field name
+  std::string name;
+
+  // The field's data type
+  TypePtr type;
+
+  Field(const std::string& name, const TypePtr& type) :
+      name(name), type(type) {}
+
+  bool Equals(const Field& other) const {
+    return (this == &other) || (this->name == other.name &&
+        this->type->Equals(other.type.get()));
+  }
+};
+
+} // namespace arrow
+
+#endif  // ARROW_FIELD_H
diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
new file mode 100644
index 0000000000000..7b449affab025
--- /dev/null
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ----------------------------------------------------------------------
+# arrow_parquet : Arrow <-> Parquet adapter
+
+set(PARQUET_SRCS
+)
+
+set(PARQUET_LIBS
+)
+
+add_library(arrow_parquet STATIC
+  ${PARQUET_SRCS}
+)
+target_link_libraries(arrow_parquet ${PARQUET_LIBS})
+SET_TARGET_PROPERTIES(arrow_parquet PROPERTIES LINKER_LANGUAGE CXX)
+
+# Headers: top level
+install(FILES
+  DESTINATION include/arrow/parquet)
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
new file mode 100644
index 0000000000000..2233a4f832a8c
--- /dev/null
+++ b/cpp/src/arrow/test-util.h
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TEST_UTIL_H_
+#define ARROW_TEST_UTIL_H_
+
+#include <gtest/gtest.h>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/util/bit-util.h"
+#include "arrow/util/random.h"
+#include "arrow/util/status.h"
+
+#define ASSERT_RAISES(ENUM, expr)               \
+  do {                                          \
+    Status s = (expr);                          \
+    ASSERT_TRUE(s.Is##ENUM());                  \
+  } while (0)
+
+
+#define ASSERT_OK(expr)                         \
+  do {                                          \
+    Status s = (expr);                          \
+    ASSERT_TRUE(s.ok());                        \
+  } while (0)
+
+
+#define EXPECT_OK(expr)                         \
+  do {                                          \
+    Status s = (expr);                          \
+    EXPECT_TRUE(s.ok());                        \
+  } while (0)
+
+
+namespace arrow {
+
+template <typename T>
+void randint(int64_t N, T lower, T upper, std::vector<T>* out) {
+  Random rng(random_seed());
+  uint64_t draw;
+  uint64_t span = upper - lower;
+  T val;
+  for (int64_t i = 0; i < N; ++i) {
+    draw = rng.Uniform64(span);
+    val = lower + static_cast<T>(draw);
+    out->push_back(val);
+  }
+}
+
+
+template <typename T>
+std::shared_ptr<Buffer> to_buffer(const std::vector<T>& values) {
+  return std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(values.data()),
+      values.size() * sizeof(T));
+}
+
+void random_nulls(int64_t n, double pct_null, std::vector<uint8_t>* nulls) {
+  Random rng(random_seed());
+  for (int i = 0; i < n; ++i) {
+    nulls->push_back(static_cast<uint8_t>(rng.NextDoubleFraction() > pct_null));
+  }
+}
+
+void random_nulls(int64_t n, double pct_null, std::vector<bool>* nulls) {
+  Random rng(random_seed());
+  for (int i = 0; i < n; ++i) {
+    nulls->push_back(rng.NextDoubleFraction() > pct_null);
+  }
+}
+
+std::shared_ptr<Buffer> bytes_to_null_buffer(uint8_t* bytes, int length) {
+  std::shared_ptr<Buffer> out;
+
+  // TODO(wesm): error checking
+  util::bytes_to_bits(bytes, length, &out);
+  return out;
+}
+
+} // namespace arrow
+
+#endif // ARROW_TEST_UTIL_H_
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
new file mode 100644
index 0000000000000..492eee52b04b1
--- /dev/null
+++ b/cpp/src/arrow/type.cc
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+} // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
new file mode 100644
index 0000000000000..220f99f4e885a
--- /dev/null
+++ b/cpp/src/arrow/type.h
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPE_H
+#define ARROW_TYPE_H
+
+#include <memory>
+#include <string>
+
+namespace arrow {
+
+// Physical data type that describes the memory layout of values. See details
+// for each type
+enum class LayoutEnum: char {
+  // A physical type consisting of some non-negative number of bytes
+  BYTE = 0,
+
+  // A physical type consisting of some non-negative number of bits
+  BIT = 1,
+
+  // A parametric variable-length value type. Full specification requires a
+  // child logical type
+  LIST = 2,
+
+  // A collection of multiple equal-length child arrays. Parametric type taking
+  // 1 or more child logical types
+  STRUCT = 3,
+
+  // An array with heterogeneous value types. Parametric types taking 1 or more
+  // child logical types
+  DENSE_UNION = 4,
+  SPARSE_UNION = 5
+};
+
+
+struct LayoutType {
+  LayoutEnum type;
+  explicit LayoutType(LayoutEnum type) : type(type) {}
+};
+
+
+// Data types in this library are all *logical*. They can be expressed as
+// either a primitive physical type (bytes or bits of some fixed size), a
+// nested type consisting of other data types, or another data type (e.g. a
+// timestamp encoded as an int64)
+//
+// Any data type can be nullable
+
+enum class TypeEnum: char {
+  // A degerate NULL type represented as 0 bytes/bits
+  NA = 0,
+
+  // Little-endian integer types
+  UINT8 = 1,
+  INT8 = 2,
+  UINT16 = 3,
+  INT16 = 4,
+  UINT32 = 5,
+  INT32 = 6,
+  UINT64 = 7,
+  INT64 = 8,
+
+  // A boolean value represented as 1 byte
+  BOOL = 9,
+
+  // A boolean value represented as 1 bit
+  BIT = 10,
+
+  // 4-byte floating point value
+  FLOAT = 11,
+
+  // 8-byte floating point value
+  DOUBLE = 12,
+
+  // CHAR(N): fixed-length UTF8 string with length N
+  CHAR = 13,
+
+  // UTF8 variable-length string as List<Char>
+  STRING = 14,
+
+  // VARCHAR(N): Null-terminated string type embedded in a CHAR(N + 1)
+  VARCHAR = 15,
+
+  // Variable-length bytes (no guarantee of UTF8-ness)
+  BINARY = 16,
+
+  // By default, int32 days since the UNIX epoch
+  DATE = 17,
+
+  // Exact timestamp encoded with int64 since UNIX epoch
+  // Default unit millisecond
+  TIMESTAMP = 18,
+
+  // Timestamp as double seconds since the UNIX epoch
+  TIMESTAMP_DOUBLE = 19,
+
+  // Exact time encoded with int64, default unit millisecond
+  TIME = 20,
+
+  // Precision- and scale-based decimal type. Storage type depends on the
+  // parameters.
+  DECIMAL = 21,
+
+  // Decimal value encoded as a text string
+  DECIMAL_TEXT = 22,
+
+  // A list of some logical data type
+  LIST = 30,
+
+  // Struct of logical types
+  STRUCT = 31,
+
+  // Unions of logical types
+  DENSE_UNION = 32,
+  SPARSE_UNION = 33,
+
+  // Union<Null, Int32, Double, String, Bool>
+  JSON_SCALAR = 50,
+
+  // User-defined type
+  USER = 60
+};
+
+
+struct DataType {
+  TypeEnum type;
+  bool nullable;
+
+  explicit DataType(TypeEnum type, bool nullable = true)
+      : type(type), nullable(nullable) {}
+
+  virtual bool Equals(const DataType* other) {
+    return (this == other) || (this->type == other->type &&
+        this->nullable == other->nullable);
+  }
+
+  virtual std::string ToString() const = 0;
+};
+
+
+typedef std::shared_ptr<LayoutType> LayoutPtr;
+typedef std::shared_ptr<DataType> TypePtr;
+
+
+struct BytesType : public LayoutType {
+  int size;
+
+  explicit BytesType(int size)
+      : LayoutType(LayoutEnum::BYTE),
+        size(size) {}
+
+  BytesType(const BytesType& other)
+      : BytesType(other.size) {}
+};
+
+struct ListLayoutType : public LayoutType {
+  LayoutPtr value_type;
+
+  explicit ListLayoutType(const LayoutPtr& value_type)
+      : LayoutType(LayoutEnum::BYTE),
+        value_type(value_type) {}
+};
+
+} // namespace arrow
+
+#endif  // ARROW_TYPE_H
diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
new file mode 100644
index 0000000000000..e090aead1f8b9
--- /dev/null
+++ b/cpp/src/arrow/types/CMakeLists.txt
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######################################
+# arrow_types
+#######################################
+
+set(TYPES_SRCS
+  construct.cc
+  floating.cc
+  integer.cc
+  json.cc
+  list.cc
+  primitive.cc
+  string.cc
+  struct.cc
+  union.cc
+)
+
+set(TYPES_LIBS
+)
+
+add_library(arrow_types STATIC
+  ${TYPES_SRCS}
+)
+target_link_libraries(arrow_types ${TYPES_LIBS})
+SET_TARGET_PROPERTIES(arrow_types PROPERTIES LINKER_LANGUAGE CXX)
+
+# Headers: top level
+install(FILES
+  boolean.h
+  collection.h
+  datetime.h
+  decimal.h
+  floating.h
+  integer.h
+  json.h
+  list.h
+  primitive.h
+  string.h
+  struct.h
+  union.h
+  DESTINATION include/arrow/types)
+
+
+ADD_ARROW_TEST(list-test)
+ADD_ARROW_TEST(primitive-test)
+ADD_ARROW_TEST(string-test)
+ADD_ARROW_TEST(struct-test)
diff --git a/cpp/src/arrow/types/binary.h b/cpp/src/arrow/types/binary.h
new file mode 100644
index 0000000000000..a9f20046b582b
--- /dev/null
+++ b/cpp/src/arrow/types/binary.h
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_BINARY_H
+#define ARROW_TYPES_BINARY_H
+
+#include <string>
+#include <vector>
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+struct StringType : public DataType {
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_BINARY_H
diff --git a/cpp/src/arrow/types/boolean.h b/cpp/src/arrow/types/boolean.h
new file mode 100644
index 0000000000000..31388c8152d52
--- /dev/null
+++ b/cpp/src/arrow/types/boolean.h
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_BOOLEAN_H
+#define ARROW_TYPES_BOOLEAN_H
+
+#include "arrow/types/primitive.h"
+
+namespace arrow {
+
+struct BooleanType : public PrimitiveType<BooleanType> {
+  PRIMITIVE_DECL(BooleanType, uint8_t, BOOL, 1, "bool");
+};
+
+typedef PrimitiveArrayImpl<BooleanType> BooleanArray;
+
+// typedef PrimitiveBuilder<BooleanType, BooleanArray> BooleanBuilder;
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_BOOLEAN_H
diff --git a/cpp/src/arrow/types/collection.h b/cpp/src/arrow/types/collection.h
new file mode 100644
index 0000000000000..59ba61419417a
--- /dev/null
+++ b/cpp/src/arrow/types/collection.h
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_COLLECTION_H
+#define ARROW_TYPES_COLLECTION_H
+
+#include <string>
+#include <vector>
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+template <TypeEnum T>
+struct CollectionType : public DataType {
+  std::vector<TypePtr> child_types_;
+
+  explicit CollectionType(bool nullable = true) : DataType(T, nullable) {}
+
+  const TypePtr& child(int i) const {
+    return child_types_[i];
+  }
+
+  int num_children() const {
+    return child_types_.size();
+  }
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_COLLECTION_H
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
new file mode 100644
index 0000000000000..5176cafd3ba1c
--- /dev/null
+++ b/cpp/src/arrow/types/construct.cc
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/construct.h"
+
+#include <memory>
+
+#include "arrow/types/floating.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/list.h"
+#include "arrow/types/string.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+class ArrayBuilder;
+
+// Initially looked at doing this with vtables, but shared pointers makes it
+// difficult
+
+#define BUILDER_CASE(ENUM, BuilderType)                         \
+    case TypeEnum::ENUM:                                        \
+      *out = static_cast<ArrayBuilder*>(new BuilderType(type)); \
+      return Status::OK();
+
+Status make_builder(const TypePtr& type, ArrayBuilder** out) {
+  switch (type->type) {
+    BUILDER_CASE(UINT8, UInt8Builder);
+    BUILDER_CASE(INT8, Int8Builder);
+    BUILDER_CASE(UINT16, UInt16Builder);
+    BUILDER_CASE(INT16, Int16Builder);
+    BUILDER_CASE(UINT32, UInt32Builder);
+    BUILDER_CASE(INT32, Int32Builder);
+    BUILDER_CASE(UINT64, UInt64Builder);
+    BUILDER_CASE(INT64, Int64Builder);
+
+    // BUILDER_CASE(BOOL, BooleanBuilder);
+
+    BUILDER_CASE(FLOAT, FloatBuilder);
+    BUILDER_CASE(DOUBLE, DoubleBuilder);
+
+    BUILDER_CASE(STRING, StringBuilder);
+
+    case TypeEnum::LIST:
+      {
+        ListType* list_type = static_cast<ListType*>(type.get());
+        ArrayBuilder* value_builder;
+        RETURN_NOT_OK(make_builder(list_type->value_type, &value_builder));
+
+        // The ListBuilder takes ownership of the value_builder
+        ListBuilder* builder = new ListBuilder(type, value_builder);
+        *out = static_cast<ArrayBuilder*>(builder);
+        return Status::OK();
+      }
+    // BUILDER_CASE(CHAR, CharBuilder);
+
+    // BUILDER_CASE(VARCHAR, VarcharBuilder);
+    // BUILDER_CASE(BINARY, BinaryBuilder);
+
+    // BUILDER_CASE(DATE, DateBuilder);
+    // BUILDER_CASE(TIMESTAMP, TimestampBuilder);
+    // BUILDER_CASE(TIME, TimeBuilder);
+
+    // BUILDER_CASE(LIST, ListBuilder);
+    // BUILDER_CASE(STRUCT, StructBuilder);
+    // BUILDER_CASE(DENSE_UNION, DenseUnionBuilder);
+    // BUILDER_CASE(SPARSE_UNION, SparseUnionBuilder);
+
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
new file mode 100644
index 0000000000000..c0bfedd27d6ad
--- /dev/null
+++ b/cpp/src/arrow/types/construct.h
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_CONSTRUCT_H
+#define ARROW_TYPES_CONSTRUCT_H
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+class ArrayBuilder;
+class Status;
+
+Status make_builder(const TypePtr& type, ArrayBuilder** out);
+
+} // namespace arrow
+
+#endif // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
new file mode 100644
index 0000000000000..b4d62523c413a
--- /dev/null
+++ b/cpp/src/arrow/types/datetime.h
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_DATETIME_H
+#define ARROW_TYPES_DATETIME_H
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+struct DateType : public DataType {
+  enum class Unit: char {
+    DAY = 0,
+    MONTH = 1,
+    YEAR = 2
+  };
+
+  Unit unit;
+
+  explicit DateType(Unit unit = Unit::DAY, bool nullable = true)
+      : DataType(TypeEnum::DATE, nullable),
+        unit(unit) {}
+
+  DateType(const DateType& other)
+      : DateType(other.unit, other.nullable) {}
+
+  static char const *name() {
+    return "date";
+  }
+
+  // virtual std::string ToString() {
+  //   return name();
+  // }
+};
+
+
+struct TimestampType : public DataType {
+  enum class Unit: char {
+    SECOND = 0,
+    MILLI = 1,
+    MICRO = 2,
+    NANO = 3
+  };
+
+  Unit unit;
+
+  explicit TimestampType(Unit unit = Unit::MILLI, bool nullable = true)
+      : DataType(TypeEnum::TIMESTAMP, nullable),
+        unit(unit) {}
+
+  TimestampType(const TimestampType& other)
+      : TimestampType(other.unit, other.nullable) {}
+
+  static char const *name() {
+    return "timestamp";
+  }
+
+  // virtual std::string ToString() {
+  //   return name();
+  // }
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_DATETIME_H
diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h
new file mode 100644
index 0000000000000..464c3ff8da92b
--- /dev/null
+++ b/cpp/src/arrow/types/decimal.h
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_DECIMAL_H
+#define ARROW_TYPES_DECIMAL_H
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+struct DecimalType : public DataType {
+  int precision;
+  int scale;
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_DECIMAL_H
diff --git a/cpp/src/arrow/types/floating.cc b/cpp/src/arrow/types/floating.cc
new file mode 100644
index 0000000000000..bde28266e638c
--- /dev/null
+++ b/cpp/src/arrow/types/floating.cc
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/floating.h"
+
+namespace arrow {
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/floating.h b/cpp/src/arrow/types/floating.h
new file mode 100644
index 0000000000000..7551ce665a27b
--- /dev/null
+++ b/cpp/src/arrow/types/floating.h
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_FLOATING_H
+#define ARROW_TYPES_FLOATING_H
+
+#include <string>
+
+#include "arrow/types/primitive.h"
+
+namespace arrow {
+
+struct FloatType : public PrimitiveType<FloatType> {
+  PRIMITIVE_DECL(FloatType, float, FLOAT, 4, "float");
+};
+
+struct DoubleType : public PrimitiveType<DoubleType> {
+  PRIMITIVE_DECL(DoubleType, double, DOUBLE, 8, "double");
+};
+
+typedef PrimitiveArrayImpl<FloatType> FloatArray;
+typedef PrimitiveArrayImpl<DoubleType> DoubleArray;
+
+typedef PrimitiveBuilder<FloatType, FloatArray> FloatBuilder;
+typedef PrimitiveBuilder<DoubleType, DoubleArray> DoubleBuilder;
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_FLOATING_H
diff --git a/cpp/src/arrow/types/integer.cc b/cpp/src/arrow/types/integer.cc
new file mode 100644
index 0000000000000..4696536616971
--- /dev/null
+++ b/cpp/src/arrow/types/integer.cc
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/integer.h"
+
+namespace arrow {
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/integer.h b/cpp/src/arrow/types/integer.h
new file mode 100644
index 0000000000000..7e5eab55be0a9
--- /dev/null
+++ b/cpp/src/arrow/types/integer.h
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_INTEGER_H
+#define ARROW_TYPES_INTEGER_H
+
+#include <cstdint>
+#include <string>
+
+#include "arrow/types/primitive.h"
+
+namespace arrow {
+
+struct UInt8Type : public PrimitiveType<UInt8Type> {
+  PRIMITIVE_DECL(UInt8Type, uint8_t, UINT8, 1, "uint8");
+};
+
+struct Int8Type : public PrimitiveType<Int8Type> {
+  PRIMITIVE_DECL(Int8Type, int8_t, INT8, 1, "int8");
+};
+
+struct UInt16Type : public PrimitiveType<UInt16Type> {
+  PRIMITIVE_DECL(UInt16Type, uint16_t, UINT16, 2, "uint16");
+};
+
+struct Int16Type : public PrimitiveType<Int16Type> {
+  PRIMITIVE_DECL(Int16Type, int16_t, INT16, 2, "int16");
+};
+
+struct UInt32Type : public PrimitiveType<UInt32Type> {
+  PRIMITIVE_DECL(UInt32Type, uint32_t, UINT32, 4, "uint32");
+};
+
+struct Int32Type : public PrimitiveType<Int32Type> {
+  PRIMITIVE_DECL(Int32Type, int32_t, INT32, 4, "int32");
+};
+
+struct UInt64Type : public PrimitiveType<UInt64Type> {
+  PRIMITIVE_DECL(UInt64Type, uint64_t, UINT64, 8, "uint64");
+};
+
+struct Int64Type : public PrimitiveType<Int64Type> {
+  PRIMITIVE_DECL(Int64Type, int64_t, INT64, 8, "int64");
+};
+
+// Array containers
+
+typedef PrimitiveArrayImpl<UInt8Type> UInt8Array;
+typedef PrimitiveArrayImpl<Int8Type> Int8Array;
+
+typedef PrimitiveArrayImpl<UInt16Type> UInt16Array;
+typedef PrimitiveArrayImpl<Int16Type> Int16Array;
+
+typedef PrimitiveArrayImpl<UInt32Type> UInt32Array;
+typedef PrimitiveArrayImpl<Int32Type> Int32Array;
+
+typedef PrimitiveArrayImpl<UInt64Type> UInt64Array;
+typedef PrimitiveArrayImpl<Int64Type> Int64Array;
+
+// Builders
+
+typedef PrimitiveBuilder<UInt8Type, UInt8Array> UInt8Builder;
+typedef PrimitiveBuilder<UInt16Type, UInt16Array> UInt16Builder;
+typedef PrimitiveBuilder<UInt32Type, UInt32Array> UInt32Builder;
+typedef PrimitiveBuilder<UInt64Type, UInt64Array> UInt64Builder;
+
+typedef PrimitiveBuilder<Int8Type, Int8Array> Int8Builder;
+typedef PrimitiveBuilder<Int16Type, Int16Array> Int16Builder;
+typedef PrimitiveBuilder<Int32Type, Int32Array> Int32Builder;
+typedef PrimitiveBuilder<Int64Type, Int64Array> Int64Builder;
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_INTEGER_H
diff --git a/cpp/src/arrow/types/json.cc b/cpp/src/arrow/types/json.cc
new file mode 100644
index 0000000000000..b29b95715fef6
--- /dev/null
+++ b/cpp/src/arrow/types/json.cc
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/json.h"
+
+#include <vector>
+
+#include "arrow/types/boolean.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/floating.h"
+#include "arrow/types/null.h"
+#include "arrow/types/string.h"
+#include "arrow/types/union.h"
+
+namespace arrow {
+
+static const TypePtr Null(new NullType());
+static const TypePtr Int32(new Int32Type());
+static const TypePtr String(new StringType());
+static const TypePtr Double(new DoubleType());
+static const TypePtr Bool(new BooleanType());
+
+static const std::vector<TypePtr> json_types = {Null, Int32, String,
+                                                Double, Bool};
+TypePtr JSONScalar::dense_type = TypePtr(new DenseUnionType(json_types));
+TypePtr JSONScalar::sparse_type = TypePtr(new SparseUnionType(json_types));
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/json.h b/cpp/src/arrow/types/json.h
new file mode 100644
index 0000000000000..91fd132408fe6
--- /dev/null
+++ b/cpp/src/arrow/types/json.h
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_JSON_H
+#define ARROW_TYPES_JSON_H
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+struct JSONScalar : public DataType {
+  bool dense;
+
+  static TypePtr dense_type;
+  static TypePtr sparse_type;
+
+  explicit JSONScalar(bool dense = true, bool nullable = true)
+      : DataType(TypeEnum::JSON_SCALAR, nullable),
+        dense(dense) {}
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_JSON_H
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
new file mode 100644
index 0000000000000..47673ff898bbd
--- /dev/null
+++ b/cpp/src/arrow/types/list-test.cc
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <cstdlib>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/types/construct.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/list.h"
+#include "arrow/types/string.h"
+#include "arrow/types/test-common.h"
+#include "arrow/util/status.h"
+
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace arrow {
+
+class ArrayBuilder;
+
+TEST(TypesTest, TestListType) {
+  std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
+
+  ListType list_type(vt);
+  ListType list_type_nn(vt, false);
+
+  ASSERT_EQ(list_type.type, TypeEnum::LIST);
+  ASSERT_TRUE(list_type.nullable);
+  ASSERT_FALSE(list_type_nn.nullable);
+
+  ASSERT_EQ(list_type.name(), string("list"));
+  ASSERT_EQ(list_type.ToString(), string("list<uint8>"));
+
+  ASSERT_EQ(list_type.value_type->type, vt->type);
+  ASSERT_EQ(list_type.value_type->type, vt->type);
+
+  std::shared_ptr<DataType> st = std::make_shared<StringType>();
+  std::shared_ptr<DataType> lt = std::make_shared<ListType>(st);
+  ASSERT_EQ(lt->ToString(), string("list<string>"));
+
+  ListType lt2(lt);
+  ASSERT_EQ(lt2.ToString(), string("list<list<string>>"));
+}
+
+// ----------------------------------------------------------------------
+// List tests
+
+class TestListBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+
+    value_type_ = TypePtr(new Int32Type());
+    type_ = TypePtr(new ListType(value_type_));
+
+    ArrayBuilder* tmp;
+    ASSERT_OK(make_builder(type_, &tmp));
+    builder_.reset(static_cast<ListBuilder*>(tmp));
+  }
+
+  void Done() {
+    Array* out;
+    ASSERT_OK(builder_->ToArray(&out));
+    result_.reset(static_cast<ListArray*>(out));
+  }
+
+ protected:
+  TypePtr value_type_;
+  TypePtr type_;
+
+  unique_ptr<ListBuilder> builder_;
+  unique_ptr<ListArray> result_;
+};
+
+
+TEST_F(TestListBuilder, TestResize) {
+}
+
+TEST_F(TestListBuilder, TestAppendNull) {
+  ASSERT_OK(builder_->AppendNull());
+  ASSERT_OK(builder_->AppendNull());
+
+  Done();
+
+  ASSERT_TRUE(result_->IsNull(0));
+  ASSERT_TRUE(result_->IsNull(1));
+
+  ASSERT_EQ(0, result_->offsets()[0]);
+  ASSERT_EQ(0, result_->offset(1));
+  ASSERT_EQ(0, result_->offset(2));
+
+  Int32Array* values = static_cast<Int32Array*>(result_->values().get());
+  ASSERT_EQ(0, values->length());
+}
+
+TEST_F(TestListBuilder, TestBasics) {
+  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+  vector<int> lengths = {3, 0, 4};
+  vector<uint8_t> is_null = {0, 1, 0};
+
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
+
+  int pos = 0;
+  for (size_t i = 0; i < lengths.size(); ++i) {
+    ASSERT_OK(builder_->Append(is_null[i] > 0));
+    for (int j = 0; j < lengths[i]; ++j) {
+      ASSERT_OK(vb->Append(values[pos++]));
+    }
+  }
+
+  Done();
+
+  ASSERT_TRUE(result_->nullable());
+  ASSERT_TRUE(result_->values()->nullable());
+
+  ASSERT_EQ(3, result_->length());
+  vector<int32_t> ex_offsets = {0, 3, 3, 7};
+  for (size_t i = 0; i < ex_offsets.size(); ++i) {
+    ASSERT_EQ(ex_offsets[i], result_->offset(i));
+  }
+
+  for (int i = 0; i < result_->length(); ++i) {
+    ASSERT_EQ(static_cast<bool>(is_null[i]), result_->IsNull(i));
+  }
+
+  ASSERT_EQ(7, result_->values()->length());
+  Int32Array* varr = static_cast<Int32Array*>(result_->values().get());
+
+  for (size_t i = 0; i < values.size(); ++i) {
+    ASSERT_EQ(values[i], varr->Value(i));
+  }
+}
+
+TEST_F(TestListBuilder, TestBasicsNonNullable) {
+}
+
+
+TEST_F(TestListBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+}
+
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
new file mode 100644
index 0000000000000..f0ff5bf928a1a
--- /dev/null
+++ b/cpp/src/arrow/types/list.cc
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/list.h"
+
+#include <sstream>
+#include <string>
+
+namespace arrow {
+
+std::string ListType::ToString() const {
+  std::stringstream s;
+  s << "list<" << value_type->ToString() << ">";
+  return s.str();
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
new file mode 100644
index 0000000000000..0f1116257c507
--- /dev/null
+++ b/cpp/src/arrow/types/list.h
@@ -0,0 +1,206 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_LIST_H
+#define ARROW_TYPES_LIST_H
+
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <string>
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/type.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+struct ListType : public DataType {
+  // List can contain any other logical value type
+  TypePtr value_type;
+
+  explicit ListType(const TypePtr& value_type, bool nullable = true)
+      : DataType(TypeEnum::LIST, nullable),
+        value_type(value_type) {}
+
+  static char const *name() {
+    return "list";
+  }
+
+  virtual std::string ToString() const;
+};
+
+
+class ListArray : public Array {
+ public:
+  ListArray() : Array(), offset_buf_(nullptr), offsets_(nullptr) {}
+
+  ListArray(const TypePtr& type, int64_t length, std::shared_ptr<Buffer> offsets,
+      const ArrayPtr& values, std::shared_ptr<Buffer> nulls = nullptr) {
+    Init(type, length, offsets, values, nulls);
+  }
+
+  virtual ~ListArray() {}
+
+  void Init(const TypePtr& type, int64_t length, std::shared_ptr<Buffer> offsets,
+      const ArrayPtr& values, std::shared_ptr<Buffer> nulls = nullptr) {
+    offset_buf_ = offsets;
+    offsets_ = offsets == nullptr? nullptr :
+      reinterpret_cast<const int32_t*>(offset_buf_->data());
+
+    values_ = values;
+    Array::Init(type, length, nulls);
+  }
+
+  // Return a shared pointer in case the requestor desires to share ownership
+  // with this array.
+  const ArrayPtr& values() const {return values_;}
+
+  const int32_t* offsets() const { return offsets_;}
+
+  int32_t offset(int i) const { return offsets_[i];}
+
+  // Neither of these functions will perform boundschecking
+  int32_t value_offset(int i) { return offsets_[i];}
+  int32_t value_length(int i) { return offsets_[i + 1] - offsets_[i];}
+
+ protected:
+  std::shared_ptr<Buffer> offset_buf_;
+  const int32_t* offsets_;
+  ArrayPtr values_;
+};
+
+// ----------------------------------------------------------------------
+// Array builder
+
+
+// Builder class for variable-length list array value types
+//
+// To use this class, you must append values to the child array builder and use
+// the Append function to delimit each distinct list value (once the values
+// have been appended to the child array)
+class ListBuilder : public Int32Builder {
+ public:
+  ListBuilder(const TypePtr& type, ArrayBuilder* value_builder)
+      : Int32Builder(type) {
+    value_builder_.reset(value_builder);
+  }
+
+  Status Init(int64_t elements) {
+    // One more than requested.
+    //
+    // XXX: This is slightly imprecise, because we might trigger null mask
+    // resizes that are unnecessary when creating arrays with power-of-two size
+    return Int32Builder::Init(elements + 1);
+  }
+
+  Status Resize(int64_t capacity) {
+    // Need space for the end offset
+    RETURN_NOT_OK(Int32Builder::Resize(capacity + 1));
+
+    // Slight hack, as the "real" capacity is one less
+    --capacity_;
+    return Status::OK();
+  }
+
+  // Vector append
+  //
+  // If passed, null_bytes is of equal length to values, and any nonzero byte
+  // will be considered as a null for that slot
+  Status Append(T* values, int64_t length, uint8_t* null_bytes = nullptr) {
+    if (length_ + length > capacity_) {
+      int64_t new_capacity = util::next_power2(length_ + length);
+      RETURN_NOT_OK(Resize(new_capacity));
+    }
+    memcpy(raw_buffer() + length_, values, length * elsize_);
+
+    if (nullable_ && null_bytes != nullptr) {
+      // If null_bytes is all not null, then none of the values are null
+      for (int i = 0; i < length; ++i) {
+        util::set_bit(null_bits_, length_ + i, static_cast<bool>(null_bytes[i]));
+      }
+    }
+
+    length_ += length;
+    return Status::OK();
+  }
+
+  // Initialize an array type instance with the results of this builder
+  // Transfers ownership of all buffers
+  template <typename Container>
+  Status Transfer(Container* out) {
+    Array* child_values;
+    RETURN_NOT_OK(value_builder_->ToArray(&child_values));
+
+    // Add final offset if the length is non-zero
+    if (length_) {
+      raw_buffer()[length_] = child_values->length();
+    }
+
+    out->Init(type_, length_, values_, ArrayPtr(child_values), nulls_);
+    values_ = nulls_ = nullptr;
+    capacity_ = length_ = 0;
+    return Status::OK();
+  }
+
+  virtual Status ToArray(Array** out) {
+    ListArray* result = new ListArray();
+    RETURN_NOT_OK(Transfer(result));
+    *out = static_cast<Array*>(result);
+    return Status::OK();
+  }
+
+  // Start a new variable-length list slot
+  //
+  // This function should be called before beginning to append elements to the
+  // value builder
+  Status Append(bool is_null = false) {
+    if (length_ == capacity_) {
+      // If the capacity was not already a multiple of 2, do so here
+      RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
+    }
+    if (nullable_) {
+      util::set_bit(null_bits_, length_, is_null);
+    }
+
+    raw_buffer()[length_++] = value_builder_->length();
+    return Status::OK();
+  }
+
+  // Status Append(int32_t* offsets, int length, uint8_t* null_bytes) {
+  //   return Int32Builder::Append(offsets, length, null_bytes);
+  // }
+
+  Status AppendNull() {
+    return Append(true);
+  }
+
+  ArrayBuilder* value_builder() const { return value_builder_.get();}
+
+ protected:
+  std::unique_ptr<ArrayBuilder> value_builder_;
+};
+
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_LIST_H
diff --git a/cpp/src/arrow/types/null.h b/cpp/src/arrow/types/null.h
new file mode 100644
index 0000000000000..c67f752d40989
--- /dev/null
+++ b/cpp/src/arrow/types/null.h
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_NULL_H
+#define ARROW_TYPES_NULL_H
+
+#include <string>
+#include <vector>
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+struct NullType : public PrimitiveType<NullType> {
+  PRIMITIVE_DECL(NullType, void, NA, 0, "null");
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_NULL_H
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
new file mode 100644
index 0000000000000..12968608094d7
--- /dev/null
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -0,0 +1,345 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/types/boolean.h"
+#include "arrow/types/construct.h"
+#include "arrow/types/floating.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/primitive.h"
+#include "arrow/types/test-common.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace arrow {
+
+TEST(TypesTest, TestBytesType) {
+  BytesType t1(3);
+
+  ASSERT_EQ(t1.type, LayoutEnum::BYTE);
+  ASSERT_EQ(t1.size, 3);
+}
+
+
+#define PRIMITIVE_TEST(KLASS, ENUM, NAME)       \
+  TEST(TypesTest, TestPrimitive_##ENUM) {       \
+    KLASS tp;                                   \
+    KLASS tp_nn(false);                         \
+                                                \
+    ASSERT_EQ(tp.type, TypeEnum::ENUM);         \
+    ASSERT_EQ(tp.name(), string(NAME));         \
+    ASSERT_TRUE(tp.nullable);                   \
+    ASSERT_FALSE(tp_nn.nullable);               \
+                                                \
+    KLASS tp_copy = tp_nn;                      \
+    ASSERT_FALSE(tp_copy.nullable);             \
+  }
+
+PRIMITIVE_TEST(Int8Type, INT8, "int8");
+PRIMITIVE_TEST(Int16Type, INT16, "int16");
+PRIMITIVE_TEST(Int32Type, INT32, "int32");
+PRIMITIVE_TEST(Int64Type, INT64, "int64");
+PRIMITIVE_TEST(UInt8Type, UINT8, "uint8");
+PRIMITIVE_TEST(UInt16Type, UINT16, "uint16");
+PRIMITIVE_TEST(UInt32Type, UINT32, "uint32");
+PRIMITIVE_TEST(UInt64Type, UINT64, "uint64");
+
+PRIMITIVE_TEST(FloatType, FLOAT, "float");
+PRIMITIVE_TEST(DoubleType, DOUBLE, "double");
+
+PRIMITIVE_TEST(BooleanType, BOOL, "bool");
+
+// ----------------------------------------------------------------------
+// Primitive type tests
+
+TEST_F(TestBuilder, TestResize) {
+  builder_->Init(10);
+  ASSERT_EQ(2, builder_->nulls()->size());
+
+  builder_->Resize(30);
+  ASSERT_EQ(4, builder_->nulls()->size());
+}
+
+template <typename Attrs>
+class TestPrimitiveBuilder : public TestBuilder {
+ public:
+  typedef typename Attrs::ArrayType ArrayType;
+  typedef typename Attrs::BuilderType BuilderType;
+  typedef typename Attrs::T T;
+
+  void SetUp() {
+    TestBuilder::SetUp();
+
+    type_ = Attrs::type();
+    type_nn_ = Attrs::type(false);
+
+    ArrayBuilder* tmp;
+    ASSERT_OK(make_builder(type_, &tmp));
+    builder_.reset(static_cast<BuilderType*>(tmp));
+
+    ASSERT_OK(make_builder(type_nn_, &tmp));
+    builder_nn_.reset(static_cast<BuilderType*>(tmp));
+  }
+
+  void RandomData(int64_t N, double pct_null = 0.1) {
+    Attrs::draw(N, &draws_);
+    random_nulls(N, pct_null, &nulls_);
+  }
+
+  void CheckNullable() {
+    ArrayType result;
+    ArrayType expected;
+    int64_t size = builder_->length();
+
+    auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
+        size * sizeof(T));
+
+    auto ex_nulls = bytes_to_null_buffer(nulls_.data(), size);
+
+    expected.Init(size, ex_data, ex_nulls);
+    ASSERT_OK(builder_->Transfer(&result));
+
+    // Builder is now reset
+    ASSERT_EQ(0, builder_->length());
+    ASSERT_EQ(0, builder_->capacity());
+    ASSERT_EQ(nullptr, builder_->buffer());
+
+    ASSERT_TRUE(result.Equals(expected));
+  }
+
+  void CheckNonNullable() {
+    ArrayType result;
+    ArrayType expected;
+    int64_t size = builder_nn_->length();
+
+    auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
+        size * sizeof(T));
+
+    expected.Init(size, ex_data);
+    ASSERT_OK(builder_nn_->Transfer(&result));
+
+    // Builder is now reset
+    ASSERT_EQ(0, builder_nn_->length());
+    ASSERT_EQ(0, builder_nn_->capacity());
+    ASSERT_EQ(nullptr, builder_nn_->buffer());
+
+    ASSERT_TRUE(result.Equals(expected));
+  }
+
+ protected:
+  TypePtr type_;
+  TypePtr type_nn_;
+  unique_ptr<BuilderType> builder_;
+  unique_ptr<BuilderType> builder_nn_;
+
+  vector<T> draws_;
+  vector<uint8_t> nulls_;
+};
+
+#define PTYPE_DECL(CapType, c_type)             \
+  typedef CapType##Array ArrayType;             \
+  typedef CapType##Builder BuilderType;         \
+  typedef CapType##Type Type;                   \
+  typedef c_type T;                             \
+                                                \
+  static TypePtr type(bool nullable = true) {   \
+    return TypePtr(new Type(nullable));         \
+  }
+
+#define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
+  struct P##CapType {                               \
+    PTYPE_DECL(CapType, c_type);                    \
+    static void draw(int64_t N, vector<T>* draws) {  \
+      randint<T>(N, LOWER, UPPER, draws);           \
+    }                                               \
+  }
+
+PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX);
+PINT_DECL(UInt16, uint16_t, 0, UINT16_MAX);
+PINT_DECL(UInt32, uint32_t, 0, UINT32_MAX);
+PINT_DECL(UInt64, uint64_t, 0, UINT64_MAX);
+
+PINT_DECL(Int8, int8_t, INT8_MIN, INT8_MAX);
+PINT_DECL(Int16, int16_t, INT16_MIN, INT16_MAX);
+PINT_DECL(Int32, int32_t, INT32_MIN, INT32_MAX);
+PINT_DECL(Int64, int64_t, INT64_MIN, INT64_MAX);
+
+typedef ::testing::Types<PUInt8, PUInt16, PUInt32, PUInt64,
+                         PInt8, PInt16, PInt32, PInt64> Primitives;
+
+TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
+
+#define DECL_T()                                \
+  typedef typename TestFixture::T T;
+
+#define DECL_ARRAYTYPE()                                \
+  typedef typename TestFixture::ArrayType ArrayType;
+
+
+TYPED_TEST(TestPrimitiveBuilder, TestInit) {
+  DECL_T();
+
+  int64_t n = 1000;
+  ASSERT_OK(this->builder_->Init(n));
+  ASSERT_EQ(n, this->builder_->capacity());
+  ASSERT_EQ(n * sizeof(T), this->builder_->buffer()->size());
+
+  // unsure if this should go in all builder classes
+  ASSERT_EQ(0, this->builder_->num_children());
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
+  int size = 10000;
+  for (int i = 0; i < size; ++i) {
+    ASSERT_OK(this->builder_->AppendNull());
+  }
+
+  Array* result;
+  ASSERT_OK(this->builder_->ToArray(&result));
+  unique_ptr<Array> holder(result);
+
+  for (int i = 0; i < size; ++i) {
+    ASSERT_TRUE(result->IsNull(i));
+  }
+}
+
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
+  DECL_T();
+
+  int size = 10000;
+
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& nulls = this->nulls_;
+
+  this->RandomData(size);
+
+  int i;
+  // Append the first 1000
+  for (i = 0; i < 1000; ++i) {
+    ASSERT_OK(this->builder_->Append(draws[i], nulls[i] > 0));
+    ASSERT_OK(this->builder_nn_->Append(draws[i]));
+  }
+
+  ASSERT_EQ(1000, this->builder_->length());
+  ASSERT_EQ(1024, this->builder_->capacity());
+
+  ASSERT_EQ(1000, this->builder_nn_->length());
+  ASSERT_EQ(1024, this->builder_nn_->capacity());
+
+  // Append the next 9000
+  for (i = 1000; i < size; ++i) {
+    ASSERT_OK(this->builder_->Append(draws[i], nulls[i] > 0));
+    ASSERT_OK(this->builder_nn_->Append(draws[i]));
+  }
+
+  ASSERT_EQ(size, this->builder_->length());
+  ASSERT_EQ(util::next_power2(size), this->builder_->capacity());
+
+  ASSERT_EQ(size, this->builder_nn_->length());
+  ASSERT_EQ(util::next_power2(size), this->builder_nn_->capacity());
+
+  this->CheckNullable();
+  this->CheckNonNullable();
+}
+
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
+  DECL_T();
+
+  int size = 10000;
+  this->RandomData(size);
+
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& nulls = this->nulls_;
+
+  // first slug
+  int K = 1000;
+
+  ASSERT_OK(this->builder_->Append(draws.data(), K, nulls.data()));
+  ASSERT_OK(this->builder_nn_->Append(draws.data(), K));
+
+  ASSERT_EQ(1000, this->builder_->length());
+  ASSERT_EQ(1024, this->builder_->capacity());
+
+  ASSERT_EQ(1000, this->builder_nn_->length());
+  ASSERT_EQ(1024, this->builder_nn_->capacity());
+
+  // Append the next 9000
+  ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, nulls.data() + K));
+  ASSERT_OK(this->builder_nn_->Append(draws.data() + K, size - K));
+
+  ASSERT_EQ(size, this->builder_->length());
+  ASSERT_EQ(util::next_power2(size), this->builder_->capacity());
+
+  this->CheckNullable();
+  this->CheckNonNullable();
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
+  int n = 1000;
+  ASSERT_OK(this->builder_->Init(n));
+
+  ASSERT_OK(this->builder_->Advance(100));
+  ASSERT_EQ(100, this->builder_->length());
+
+  ASSERT_OK(this->builder_->Advance(900));
+  ASSERT_RAISES(Invalid, this->builder_->Advance(1));
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestResize) {
+  DECL_T();
+
+  int cap = MIN_BUILDER_CAPACITY * 2;
+
+  ASSERT_OK(this->builder_->Resize(cap));
+  ASSERT_EQ(cap, this->builder_->capacity());
+
+  ASSERT_EQ(cap * sizeof(T), this->builder_->buffer()->size());
+  ASSERT_EQ(util::ceil_byte(cap) / 8, this->builder_->nulls()->size());
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
+  int n = 100;
+  ASSERT_OK(this->builder_->Reserve(n));
+  ASSERT_EQ(0, this->builder_->length());
+  ASSERT_EQ(MIN_BUILDER_CAPACITY, this->builder_->capacity());
+
+  ASSERT_OK(this->builder_->Advance(100));
+  ASSERT_OK(this->builder_->Reserve(MIN_BUILDER_CAPACITY));
+
+  ASSERT_EQ(util::next_power2(MIN_BUILDER_CAPACITY + 100),
+      this->builder_->capacity());
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
new file mode 100644
index 0000000000000..2612e8ca7fd4a
--- /dev/null
+++ b/cpp/src/arrow/types/primitive.cc
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/primitive.h"
+
+#include <memory>
+
+#include "arrow/util/buffer.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// Primitive array base
+
+void PrimitiveArray::Init(const TypePtr& type, int64_t length,
+    const std::shared_ptr<Buffer>& data,
+    const std::shared_ptr<Buffer>& nulls) {
+  Array::Init(type, length, nulls);
+  data_ = data;
+  raw_data_ = data == nullptr? nullptr : data_->data();
+}
+
+bool PrimitiveArray::Equals(const PrimitiveArray& other) const {
+  if (this == &other) return true;
+  if (type_->nullable != other.type_->nullable) return false;
+
+  bool equal_data = data_->Equals(*other.data_, length_);
+  if (type_->nullable) {
+    return equal_data &&
+      nulls_->Equals(*other.nulls_, util::ceil_byte(length_) / 8);
+  } else {
+    return equal_data;
+  }
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
new file mode 100644
index 0000000000000..a41911224e05e
--- /dev/null
+++ b/cpp/src/arrow/types/primitive.h
@@ -0,0 +1,240 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_PRIMITIVE_H
+#define ARROW_TYPES_PRIMITIVE_H
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/type.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+template <typename Derived>
+struct PrimitiveType : public DataType {
+  explicit PrimitiveType(bool nullable = true)
+      : DataType(Derived::type_enum, nullable) {}
+
+  virtual std::string ToString() const {
+    return std::string(static_cast<const Derived*>(this)->name());
+  }
+};
+
+#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)          \
+  typedef C_TYPE c_type;                                            \
+  static constexpr TypeEnum type_enum = TypeEnum::ENUM;             \
+  static constexpr int size = SIZE;                              \
+                                                                    \
+  explicit TYPENAME(bool nullable = true)                           \
+      : PrimitiveType<TYPENAME>(nullable) {}                        \
+                                                                    \
+  static const char* name() {                                       \
+    return NAME;                                                    \
+  }
+
+
+// Base class for fixed-size logical types
+class PrimitiveArray : public Array {
+ public:
+  PrimitiveArray() : Array(), data_(nullptr), raw_data_(nullptr) {}
+
+  virtual ~PrimitiveArray() {}
+
+  void Init(const TypePtr& type, int64_t length, const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& nulls = nullptr);
+
+  const std::shared_ptr<Buffer>& data() const { return data_;}
+
+  bool Equals(const PrimitiveArray& other) const;
+
+ protected:
+  std::shared_ptr<Buffer> data_;
+  const uint8_t* raw_data_;
+};
+
+
+template <typename TypeClass>
+class PrimitiveArrayImpl : public PrimitiveArray {
+ public:
+  typedef typename TypeClass::c_type T;
+
+  PrimitiveArrayImpl() : PrimitiveArray() {}
+
+  PrimitiveArrayImpl(int64_t length, const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& nulls = nullptr) {
+    Init(length, data, nulls);
+  }
+
+  void Init(int64_t length, const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& nulls = nullptr) {
+    TypePtr type(new TypeClass(nulls != nullptr));
+    PrimitiveArray::Init(type, length, data, nulls);
+  }
+
+  bool Equals(const PrimitiveArrayImpl& other) const {
+    return PrimitiveArray::Equals(*static_cast<const PrimitiveArray*>(&other));
+  }
+
+  const T* raw_data() const { return reinterpret_cast<const T*>(raw_data_);}
+
+  T Value(int64_t i) const {
+    return raw_data()[i];
+  }
+
+  TypeClass* exact_type() const {
+    return static_cast<TypeClass*>(type_);
+  }
+};
+
+
+template <typename Type, typename ArrayType>
+class PrimitiveBuilder : public ArrayBuilder {
+ public:
+  typedef typename Type::c_type T;
+
+  explicit PrimitiveBuilder(const TypePtr& type)
+      : ArrayBuilder(type), values_(nullptr) {
+    elsize_ = sizeof(T);
+  }
+
+  virtual ~PrimitiveBuilder() {}
+
+  Status Resize(int64_t capacity) {
+    // XXX: Set floor size for now
+    if (capacity < MIN_BUILDER_CAPACITY) {
+      capacity = MIN_BUILDER_CAPACITY;
+    }
+
+    if (capacity_ == 0) {
+      RETURN_NOT_OK(Init(capacity));
+    } else {
+      RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
+      RETURN_NOT_OK(values_->Resize(capacity * elsize_));
+      capacity_ = capacity;
+    }
+    return Status::OK();
+  }
+
+  Status Init(int64_t capacity) {
+    RETURN_NOT_OK(ArrayBuilder::Init(capacity));
+
+    values_ = std::make_shared<OwnedMutableBuffer>();
+    return values_->Resize(capacity * elsize_);
+  }
+
+  Status Reserve(int64_t elements) {
+    if (length_ + elements > capacity_) {
+      int64_t new_capacity = util::next_power2(length_ + elements);
+      return Resize(new_capacity);
+    }
+    return Status::OK();
+  }
+
+  Status Advance(int64_t elements) {
+    return ArrayBuilder::Advance(elements);
+  }
+
+  // Scalar append
+  Status Append(T val, bool is_null = false) {
+    if (length_ == capacity_) {
+      // If the capacity was not already a multiple of 2, do so here
+      RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
+    }
+    if (nullable_) {
+      util::set_bit(null_bits_, length_, is_null);
+    }
+    raw_buffer()[length_++] = val;
+    return Status::OK();
+  }
+
+  // Vector append
+  //
+  // If passed, null_bytes is of equal length to values, and any nonzero byte
+  // will be considered as a null for that slot
+  Status Append(const T* values, int64_t length, uint8_t* null_bytes = nullptr) {
+    if (length_ + length > capacity_) {
+      int64_t new_capacity = util::next_power2(length_ + length);
+      RETURN_NOT_OK(Resize(new_capacity));
+    }
+    memcpy(raw_buffer() + length_, values, length * elsize_);
+
+    if (nullable_ && null_bytes != nullptr) {
+      // If null_bytes is all not null, then none of the values are null
+      for (int64_t i = 0; i < length; ++i) {
+        util::set_bit(null_bits_, length_ + i, static_cast<bool>(null_bytes[i]));
+      }
+    }
+
+    length_ += length;
+    return Status::OK();
+  }
+
+  Status AppendNull() {
+    if (!nullable_) {
+      return Status::Invalid("not nullable");
+    }
+    if (length_ == capacity_) {
+      // If the capacity was not already a multiple of 2, do so here
+      RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
+    }
+    util::set_bit(null_bits_, length_++, true);
+    return Status::OK();
+  }
+
+  // Initialize an array type instance with the results of this builder
+  // Transfers ownership of all buffers
+  Status Transfer(PrimitiveArray* out) {
+    out->Init(type_, length_, values_, nulls_);
+    values_ = nulls_ = nullptr;
+    capacity_ = length_ = 0;
+    return Status::OK();
+  }
+
+  Status Transfer(ArrayType* out) {
+    return Transfer(static_cast<PrimitiveArray*>(out));
+  }
+
+  virtual Status ToArray(Array** out) {
+    ArrayType* result = new ArrayType();
+    RETURN_NOT_OK(Transfer(result));
+    *out = static_cast<Array*>(result);
+    return Status::OK();
+  }
+
+  T* raw_buffer() {
+    return reinterpret_cast<T*>(values_->mutable_data());
+  }
+
+  std::shared_ptr<Buffer> buffer() const {
+    return values_;
+  }
+
+ protected:
+  std::shared_ptr<OwnedMutableBuffer> values_;
+  int64_t elsize_;
+};
+
+} // namespace arrow
+
+#endif  // ARROW_TYPES_PRIMITIVE_H
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
new file mode 100644
index 0000000000000..6dba3fdcbb6aa
--- /dev/null
+++ b/cpp/src/arrow/types/string-test.cc
@@ -0,0 +1,242 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/types/construct.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/string.h"
+#include "arrow/types/test-common.h"
+#include "arrow/util/status.h"
+
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace arrow {
+
+
+TEST(TypesTest, TestCharType) {
+  CharType t1(5);
+
+  ASSERT_EQ(t1.type, TypeEnum::CHAR);
+  ASSERT_TRUE(t1.nullable);
+  ASSERT_EQ(t1.size, 5);
+
+  ASSERT_EQ(t1.ToString(), string("char(5)"));
+
+  // Test copy constructor
+  CharType t2 = t1;
+  ASSERT_EQ(t2.type, TypeEnum::CHAR);
+  ASSERT_TRUE(t2.nullable);
+  ASSERT_EQ(t2.size, 5);
+}
+
+
+TEST(TypesTest, TestVarcharType) {
+  VarcharType t1(5);
+
+  ASSERT_EQ(t1.type, TypeEnum::VARCHAR);
+  ASSERT_TRUE(t1.nullable);
+  ASSERT_EQ(t1.size, 5);
+  ASSERT_EQ(t1.physical_type.size, 6);
+
+  ASSERT_EQ(t1.ToString(), string("varchar(5)"));
+
+  // Test copy constructor
+  VarcharType t2 = t1;
+  ASSERT_EQ(t2.type, TypeEnum::VARCHAR);
+  ASSERT_TRUE(t2.nullable);
+  ASSERT_EQ(t2.size, 5);
+  ASSERT_EQ(t2.physical_type.size, 6);
+}
+
+TEST(TypesTest, TestStringType) {
+  StringType str;
+  StringType str_nn(false);
+
+  ASSERT_EQ(str.type, TypeEnum::STRING);
+  ASSERT_EQ(str.name(), string("string"));
+  ASSERT_TRUE(str.nullable);
+  ASSERT_FALSE(str_nn.nullable);
+}
+
+// ----------------------------------------------------------------------
+// String container
+
+class TestStringContainer : public ::testing::Test  {
+ public:
+  void SetUp() {
+    chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
+    offsets_ = {0, 1, 1, 1, 3, 6};
+    nulls_ = {0, 0, 1, 0, 0};
+    expected_ = {"a", "", "", "bb", "ccc"};
+
+    MakeArray();
+  }
+
+  void MakeArray() {
+    length_ = offsets_.size() - 1;
+    int64_t nchars = chars_.size();
+
+    value_buf_ = to_buffer(chars_);
+    values_ = ArrayPtr(new UInt8Array(nchars, value_buf_));
+
+    offsets_buf_ = to_buffer(offsets_);
+
+    nulls_buf_ = bytes_to_null_buffer(nulls_.data(), nulls_.size());
+    strings_.Init(length_, offsets_buf_, values_, nulls_buf_);
+  }
+
+ protected:
+  vector<int32_t> offsets_;
+  vector<char> chars_;
+  vector<uint8_t> nulls_;
+
+  vector<string> expected_;
+
+  std::shared_ptr<Buffer> value_buf_;
+  std::shared_ptr<Buffer> offsets_buf_;
+  std::shared_ptr<Buffer> nulls_buf_;
+
+  int64_t length_;
+
+  ArrayPtr values_;
+  StringArray strings_;
+};
+
+
+TEST_F(TestStringContainer, TestArrayBasics) {
+  ASSERT_EQ(length_, strings_.length());
+  ASSERT_TRUE(strings_.nullable());
+}
+
+TEST_F(TestStringContainer, TestType) {
+  TypePtr type = strings_.type();
+
+  ASSERT_EQ(TypeEnum::STRING, type->type);
+  ASSERT_EQ(TypeEnum::STRING, strings_.type_enum());
+}
+
+
+TEST_F(TestStringContainer, TestListFunctions) {
+  int pos = 0;
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    ASSERT_EQ(pos, strings_.value_offset(i));
+    ASSERT_EQ(expected_[i].size(), strings_.value_length(i));
+    pos += expected_[i].size();
+  }
+}
+
+
+TEST_F(TestStringContainer, TestDestructor) {
+  auto arr = std::make_shared<StringArray>(length_, offsets_buf_, values_, nulls_buf_);
+}
+
+TEST_F(TestStringContainer, TestGetString) {
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    if (nulls_[i]) {
+      ASSERT_TRUE(strings_.IsNull(i));
+    } else {
+      ASSERT_EQ(expected_[i], strings_.GetString(i));
+    }
+  }
+}
+
+// ----------------------------------------------------------------------
+// String builder tests
+
+class TestStringBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+    type_ = TypePtr(new StringType());
+
+    ArrayBuilder* tmp;
+    ASSERT_OK(make_builder(type_, &tmp));
+    builder_.reset(static_cast<StringBuilder*>(tmp));
+  }
+
+  void Done() {
+    Array* out;
+    ASSERT_OK(builder_->ToArray(&out));
+    result_.reset(static_cast<StringArray*>(out));
+  }
+
+ protected:
+  TypePtr type_;
+
+  unique_ptr<StringBuilder> builder_;
+  unique_ptr<StringArray> result_;
+};
+
+TEST_F(TestStringBuilder, TestAttrs) {
+  ASSERT_FALSE(builder_->value_builder()->nullable());
+}
+
+TEST_F(TestStringBuilder, TestScalarAppend) {
+  vector<string> strings = {"a", "bb", "", "", "ccc"};
+  vector<uint8_t> is_null = {0, 0, 0, 1, 0};
+
+  int N = strings.size();
+  int reps = 1000;
+
+  for (int j = 0; j < reps; ++j) {
+    for (int i = 0; i < N; ++i) {
+      if (is_null[i]) {
+        builder_->AppendNull();
+      } else {
+        builder_->Append(strings[i]);
+      }
+    }
+  }
+  Done();
+
+  ASSERT_EQ(reps * N, result_->length());
+  ASSERT_EQ(reps * 6, result_->values()->length());
+
+  int64_t length;
+  int64_t pos = 0;
+  for (int i = 0; i < N * reps; ++i) {
+    if (is_null[i % N]) {
+      ASSERT_TRUE(result_->IsNull(i));
+    } else {
+      ASSERT_FALSE(result_->IsNull(i));
+      result_->GetValue(i, &length);
+      ASSERT_EQ(pos, result_->offset(i));
+      ASSERT_EQ(strings[i % N].size(), length);
+      ASSERT_EQ(strings[i % N], result_->GetString(i));
+
+      pos += length;
+    }
+  }
+}
+
+TEST_F(TestStringBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
new file mode 100644
index 0000000000000..f3dfbdc50f7a4
--- /dev/null
+++ b/cpp/src/arrow/types/string.cc
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/string.h"
+
+#include <sstream>
+#include <string>
+
+namespace arrow {
+
+std::string CharType::ToString() const {
+  std::stringstream s;
+  s << "char(" << size << ")";
+  return s.str();
+}
+
+
+std::string VarcharType::ToString() const {
+  std::stringstream s;
+  s << "varchar(" << size << ")";
+  return s.str();
+}
+
+TypePtr StringBuilder::value_type_ = TypePtr(new UInt8Type(false));
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
new file mode 100644
index 0000000000000..30d6e247db1ad
--- /dev/null
+++ b/cpp/src/arrow/types/string.h
@@ -0,0 +1,181 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_STRING_H
+#define ARROW_TYPES_STRING_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/type.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/list.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+class ArrayBuilder;
+
+struct CharType : public DataType {
+  int size;
+
+  BytesType physical_type;
+
+  explicit CharType(int size, bool nullable = true)
+      : DataType(TypeEnum::CHAR, nullable),
+        size(size),
+        physical_type(BytesType(size)) {}
+
+  CharType(const CharType& other)
+      : CharType(other.size, other.nullable) {}
+
+  virtual std::string ToString() const;
+};
+
+
+// Variable-length, null-terminated strings, up to a certain length
+struct VarcharType : public DataType {
+  int size;
+
+  BytesType physical_type;
+
+  explicit VarcharType(int size, bool nullable = true)
+      : DataType(TypeEnum::VARCHAR, nullable),
+        size(size),
+        physical_type(BytesType(size + 1)) {}
+  VarcharType(const VarcharType& other)
+      : VarcharType(other.size, other.nullable) {}
+
+  virtual std::string ToString() const;
+};
+
+static const LayoutPtr byte1(new BytesType(1));
+static const LayoutPtr physical_string = LayoutPtr(new ListLayoutType(byte1));
+
+// String is a logical type consisting of a physical list of 1-byte values
+struct StringType : public DataType {
+  explicit StringType(bool nullable = true)
+      : DataType(TypeEnum::STRING, nullable) {}
+
+  StringType(const StringType& other)
+      : StringType(other.nullable) {}
+
+  const LayoutPtr& physical_type() {
+    return physical_string;
+  }
+
+  static char const *name() {
+    return "string";
+  }
+
+  virtual std::string ToString() const {
+    return name();
+  }
+};
+
+
+// TODO: add a BinaryArray layer in between
+class StringArray : public ListArray {
+ public:
+  StringArray() : ListArray(), bytes_(nullptr), raw_bytes_(nullptr) {}
+
+  StringArray(int64_t length, const std::shared_ptr<Buffer>& offsets,
+      const ArrayPtr& values,
+      const std::shared_ptr<Buffer>& nulls = nullptr) {
+    Init(length, offsets, values, nulls);
+  }
+
+  void Init(const TypePtr& type, int64_t length,
+      const std::shared_ptr<Buffer>& offsets,
+      const ArrayPtr& values,
+      const std::shared_ptr<Buffer>& nulls = nullptr) {
+    ListArray::Init(type, length, offsets, values, nulls);
+
+    // TODO: type validation for values array
+
+    // For convenience
+    bytes_ = static_cast<UInt8Array*>(values.get());
+    raw_bytes_ = bytes_->raw_data();
+  }
+
+  void Init(int64_t length, const std::shared_ptr<Buffer>& offsets,
+      const ArrayPtr& values,
+      const std::shared_ptr<Buffer>& nulls = nullptr) {
+    TypePtr type(new StringType(nulls != nullptr));
+    Init(type, length, offsets, values, nulls);
+  }
+
+  // Compute the pointer t
+  const uint8_t* GetValue(int64_t i, int64_t* out_length) const {
+    int32_t pos = offsets_[i];
+    *out_length = offsets_[i + 1] - pos;
+    return raw_bytes_ + pos;
+  }
+
+  // Construct a std::string
+  std::string GetString(int64_t i) const {
+    int64_t nchars;
+    const uint8_t* str = GetValue(i, &nchars);
+    return std::string(reinterpret_cast<const char*>(str), nchars);
+  }
+
+ private:
+  UInt8Array* bytes_;
+  const uint8_t* raw_bytes_;
+};
+
+// Array builder
+
+
+
+class StringBuilder : public ListBuilder {
+ public:
+  explicit StringBuilder(const TypePtr& type) :
+      ListBuilder(type, static_cast<ArrayBuilder*>(new UInt8Builder(value_type_))) {
+    byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
+  }
+
+  Status Append(const std::string& value) {
+    RETURN_NOT_OK(ListBuilder::Append());
+    return byte_builder_->Append(reinterpret_cast<const uint8_t*>(value.c_str()),
+        value.size());
+  }
+
+  Status Append(const uint8_t* value, int64_t length);
+  Status Append(const std::vector<std::string>& values,
+                uint8_t* null_bytes);
+
+  virtual Status ToArray(Array** out) {
+    StringArray* result = new StringArray();
+    RETURN_NOT_OK(ListBuilder::Transfer(result));
+    *out = static_cast<Array*>(result);
+    return Status::OK();
+  }
+
+ protected:
+  UInt8Builder* byte_builder_;
+
+  static TypePtr value_type_;
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_STRING_H
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
new file mode 100644
index 0000000000000..644b5457d5851
--- /dev/null
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <string>
+#include <vector>
+
+#include "arrow/field.h"
+#include "arrow/type.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+
+using std::string;
+using std::vector;
+
+namespace arrow {
+
+TEST(TestStructType, Basics) {
+  TypePtr f0_type = TypePtr(new Int32Type());
+  Field f0("f0", f0_type);
+
+  TypePtr f1_type = TypePtr(new StringType());
+  Field f1("f1", f1_type);
+
+  TypePtr f2_type = TypePtr(new UInt8Type());
+  Field f2("f2", f2_type);
+
+  vector<Field> fields = {f0, f1, f2};
+
+  StructType struct_type(fields, true);
+  StructType struct_type_nn(fields, false);
+
+  ASSERT_TRUE(struct_type.nullable);
+  ASSERT_FALSE(struct_type_nn.nullable);
+
+  ASSERT_TRUE(struct_type.field(0).Equals(f0));
+  ASSERT_TRUE(struct_type.field(1).Equals(f1));
+  ASSERT_TRUE(struct_type.field(2).Equals(f2));
+
+  ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
+
+  // TODO: out of bounds for field(...)
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
new file mode 100644
index 0000000000000..b7be5d8245f1d
--- /dev/null
+++ b/cpp/src/arrow/types/struct.cc
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/struct.h"
+
+#include <memory>
+#include <sstream>
+#include <string>
+
+namespace arrow {
+
+std::string StructType::ToString() const {
+  std::stringstream s;
+  s << "struct<";
+  for (size_t i = 0; i < fields_.size(); ++i) {
+    if (i > 0) s << ", ";
+    const Field& field  = fields_[i];
+    s << field.name << ": " << field.type->ToString();
+  }
+  s << ">";
+  return s.str();
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
new file mode 100644
index 0000000000000..7d8885b830dba
--- /dev/null
+++ b/cpp/src/arrow/types/struct.h
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_STRUCT_H
+#define ARROW_TYPES_STRUCT_H
+
+#include <string>
+#include <vector>
+
+#include "arrow/field.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+struct StructType : public DataType {
+  std::vector<Field> fields_;
+
+  StructType(const std::vector<Field>& fields,
+      bool nullable = true)
+      : DataType(TypeEnum::STRUCT, nullable) {
+    fields_ = fields;
+  }
+
+  const Field& field(int i) const {
+    return fields_[i];
+  }
+
+  int num_children() const {
+    return fields_.size();
+  }
+
+  virtual std::string ToString() const;
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_STRUCT_H
diff --git a/cpp/src/arrow/types/test-common.h b/cpp/src/arrow/types/test-common.h
new file mode 100644
index 0000000000000..267e48a7f25c9
--- /dev/null
+++ b/cpp/src/arrow/types/test-common.h
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_TEST_COMMON_H
+#define ARROW_TYPES_TEST_COMMON_H
+
+#include <gtest/gtest.h>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+
+using std::unique_ptr;
+
+namespace arrow {
+
+class TestBuilder : public ::testing::Test {
+ public:
+  void SetUp() {
+    type_ = TypePtr(new UInt8Type());
+    type_nn_ = TypePtr(new UInt8Type(false));
+    builder_.reset(new UInt8Builder(type_));
+    builder_nn_.reset(new UInt8Builder(type_nn_));
+  }
+ protected:
+  TypePtr type_;
+  TypePtr type_nn_;
+  unique_ptr<ArrayBuilder> builder_;
+  unique_ptr<ArrayBuilder> builder_nn_;
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_TEST_COMMON_H
diff --git a/cpp/src/arrow/types/union.cc b/cpp/src/arrow/types/union.cc
new file mode 100644
index 0000000000000..54f41a7eef6be
--- /dev/null
+++ b/cpp/src/arrow/types/union.cc
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/union.h"
+
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+static inline std::string format_union(const std::vector<TypePtr>& child_types) {
+  std::stringstream s;
+  s << "union<";
+  for (size_t i = 0; i < child_types.size(); ++i) {
+    if (i) s << ", ";
+    s << child_types[i]->ToString();
+  }
+  s << ">";
+  return s.str();
+}
+
+std::string DenseUnionType::ToString() const {
+  return format_union(child_types_);
+}
+
+
+std::string SparseUnionType::ToString() const {
+  return format_union(child_types_);
+}
+
+
+} // namespace arrow
diff --git a/cpp/src/arrow/types/union.h b/cpp/src/arrow/types/union.h
new file mode 100644
index 0000000000000..7b66c3b88bf3c
--- /dev/null
+++ b/cpp/src/arrow/types/union.h
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPES_UNION_H
+#define ARROW_TYPES_UNION_H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/type.h"
+#include "arrow/types/collection.h"
+
+namespace arrow {
+
+class Buffer;
+
+struct DenseUnionType : public CollectionType<TypeEnum::DENSE_UNION> {
+  typedef CollectionType<TypeEnum::DENSE_UNION> Base;
+
+  DenseUnionType(const std::vector<TypePtr>& child_types,
+      bool nullable = true)
+      : Base(nullable) {
+    child_types_ = child_types;
+  }
+
+  virtual std::string ToString() const;
+};
+
+
+struct SparseUnionType : public CollectionType<TypeEnum::SPARSE_UNION> {
+  typedef CollectionType<TypeEnum::SPARSE_UNION> Base;
+
+  SparseUnionType(const std::vector<TypePtr>& child_types,
+      bool nullable = true)
+      : Base(nullable) {
+    child_types_ = child_types;
+  }
+
+  virtual std::string ToString() const;
+};
+
+
+class UnionArray : public Array {
+ public:
+  UnionArray() : Array() {}
+
+ protected:
+  // The data are types encoded as int16
+  Buffer* types_;
+  std::vector<std::shared_ptr<Array> > children_;
+};
+
+
+class DenseUnionArray : public UnionArray {
+ public:
+  DenseUnionArray() : UnionArray() {}
+
+ protected:
+  Buffer* offset_buf_;
+};
+
+
+class SparseUnionArray : public UnionArray {
+ public:
+  SparseUnionArray() : UnionArray() {}
+};
+
+} // namespace arrow
+
+#endif // ARROW_TYPES_UNION_H
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
new file mode 100644
index 0000000000000..88e3f7a656d90
--- /dev/null
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######################################
+# arrow_util
+#######################################
+
+set(UTIL_SRCS
+  bit-util.cc
+  buffer.cc
+  status.cc
+)
+
+set(UTIL_LIBS
+  rt)
+
+add_library(arrow_util STATIC
+  ${UTIL_SRCS}
+)
+target_link_libraries(arrow_util ${UTIL_LIBS})
+SET_TARGET_PROPERTIES(arrow_util PROPERTIES LINKER_LANGUAGE CXX)
+
+# Headers: top level
+install(FILES
+  bit-util.h
+  buffer.h
+  macros.h
+  status.h
+  DESTINATION include/arrow/util)
+
+#######################################
+# arrow_test_util
+#######################################
+
+add_library(arrow_test_util)
+target_link_libraries(arrow_test_util
+  arrow_util)
+
+SET_TARGET_PROPERTIES(arrow_test_util PROPERTIES LINKER_LANGUAGE CXX)
+
+#######################################
+# arrow_test_main
+#######################################
+
+add_library(arrow_test_main
+  test_main.cc)
+
+if (APPLE)
+  target_link_libraries(arrow_test_main
+    gtest
+	arrow_util
+	arrow_test_util
+    dl)
+  set_target_properties(arrow_test_main
+        PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+else()
+  target_link_libraries(arrow_test_main
+    gtest
+	arrow_util
+	arrow_test_util
+    pthread
+    dl
+  )
+endif()
+
+ADD_ARROW_TEST(bit-util-test)
+ADD_ARROW_TEST(buffer-test)
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
new file mode 100644
index 0000000000000..7506ca5b5531c
--- /dev/null
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include "arrow/util/bit-util.h"
+
+namespace arrow {
+
+TEST(UtilTests, TestNextPower2) {
+  using util::next_power2;
+
+  ASSERT_EQ(8, next_power2(6));
+  ASSERT_EQ(8, next_power2(8));
+
+  ASSERT_EQ(1, next_power2(1));
+  ASSERT_EQ(256, next_power2(131));
+
+  ASSERT_EQ(1024, next_power2(1000));
+
+  ASSERT_EQ(4096, next_power2(4000));
+
+  ASSERT_EQ(65536, next_power2(64000));
+
+  ASSERT_EQ(1LL << 32, next_power2((1LL << 32) - 1));
+  ASSERT_EQ(1LL << 31, next_power2((1LL << 31) - 1));
+  ASSERT_EQ(1LL << 62, next_power2((1LL << 62) - 1));
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
new file mode 100644
index 0000000000000..d2ddd6584a88c
--- /dev/null
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>
+
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+void util::bytes_to_bits(uint8_t* bytes, int length, uint8_t* bits) {
+  for (int i = 0; i < length; ++i) {
+    set_bit(bits, i, static_cast<bool>(bytes[i]));
+  }
+}
+
+Status util::bytes_to_bits(uint8_t* bytes, int length,
+    std::shared_ptr<Buffer>* out) {
+  int bit_length = ceil_byte(length) / 8;
+
+  auto buffer = std::make_shared<OwnedMutableBuffer>();
+  RETURN_NOT_OK(buffer->Resize(bit_length));
+  memset(buffer->mutable_data(), 0, bit_length);
+  bytes_to_bits(bytes, length, buffer->mutable_data());
+
+  *out = buffer;
+
+  return Status::OK();
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
new file mode 100644
index 0000000000000..61dffa30423b1
--- /dev/null
+++ b/cpp/src/arrow/util/bit-util.h
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_BIT_UTIL_H
+#define ARROW_UTIL_BIT_UTIL_H
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+
+#include "arrow/util/buffer.h"
+
+namespace arrow {
+
+class Status;
+
+namespace util {
+
+static inline int64_t ceil_byte(int64_t size) {
+  return (size + 7) & ~7;
+}
+
+static inline int64_t ceil_2bytes(int64_t size) {
+  return (size + 15) & ~15;
+}
+
+static inline bool get_bit(const uint8_t* bits, int i) {
+  return bits[i / 8] & (1 << (i % 8));
+}
+
+static inline void set_bit(uint8_t* bits, int i, bool is_set) {
+  bits[i / 8] |= (1 << (i % 8)) * is_set;
+}
+
+static inline int64_t next_power2(int64_t n) {
+  n--;
+  n |= n >> 1;
+  n |= n >> 2;
+  n |= n >> 4;
+  n |= n >> 8;
+  n |= n >> 16;
+  n |= n >> 32;
+  n++;
+  return n;
+}
+
+void bytes_to_bits(uint8_t* bytes, int length, uint8_t* bits);
+Status bytes_to_bits(uint8_t*, int, std::shared_ptr<Buffer>*);
+
+} // namespace util
+
+} // namespace arrow
+
+#endif // ARROW_UTIL_BIT_UTIL_H
diff --git a/cpp/src/arrow/util/buffer-test.cc b/cpp/src/arrow/util/buffer-test.cc
new file mode 100644
index 0000000000000..edfd08e850bd8
--- /dev/null
+++ b/cpp/src/arrow/util/buffer-test.cc
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <cstdlib>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <string>
+
+#include "arrow/test-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+using std::string;
+
+namespace arrow {
+
+class TestBuffer : public ::testing::Test {
+};
+
+TEST_F(TestBuffer, Resize) {
+  OwnedMutableBuffer buf;
+
+  ASSERT_EQ(0, buf.size());
+  ASSERT_OK(buf.Resize(100));
+  ASSERT_EQ(100, buf.size());
+  ASSERT_OK(buf.Resize(200));
+  ASSERT_EQ(200, buf.size());
+
+  // Make it smaller, too
+  ASSERT_OK(buf.Resize(50));
+  ASSERT_EQ(50, buf.size());
+}
+
+TEST_F(TestBuffer, ResizeOOM) {
+  // realloc fails, even though there may be no explicit limit
+  OwnedMutableBuffer buf;
+  ASSERT_OK(buf.Resize(100));
+  int64_t to_alloc = std::numeric_limits<int64_t>::max();
+  ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
new file mode 100644
index 0000000000000..2fb34d59e0b78
--- /dev/null
+++ b/cpp/src/arrow/util/buffer.cc
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/buffer.h"
+
+#include <cstdint>
+
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset,
+    int64_t size) {
+  data_ = parent->data() + offset;
+  size_ = size;
+  parent_ = parent;
+}
+
+std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
+  return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
+}
+
+OwnedMutableBuffer::OwnedMutableBuffer() :
+    MutableBuffer(nullptr, 0) {}
+
+Status OwnedMutableBuffer::Resize(int64_t new_size) {
+  size_ = new_size;
+  try {
+    buffer_owner_.resize(new_size);
+  } catch (const std::bad_alloc& e) {
+    return Status::OutOfMemory("resize failed");
+  }
+  data_ = buffer_owner_.data();
+  mutable_data_ = buffer_owner_.data();
+
+  return Status::OK();
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
new file mode 100644
index 0000000000000..3e4183936b33d
--- /dev/null
+++ b/cpp/src/arrow/util/buffer.h
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_BUFFER_H
+#define ARROW_UTIL_BUFFER_H
+
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <vector>
+
+#include "arrow/util/macros.h"
+
+namespace arrow {
+
+class Status;
+
+// ----------------------------------------------------------------------
+// Buffer classes
+
+// Immutable API for a chunk of bytes which may or may not be owned by the
+// class instance
+class Buffer : public std::enable_shared_from_this<Buffer> {
+ public:
+  Buffer(const uint8_t* data, int64_t size) :
+      data_(data),
+      size_(size) {}
+
+  // An offset into data that is owned by another buffer, but we want to be
+  // able to retain a valid pointer to it even after other shared_ptr's to the
+  // parent buffer have been destroyed
+  Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
+
+  std::shared_ptr<Buffer> get_shared_ptr() {
+    return shared_from_this();
+  }
+
+  // Return true if both buffers are the same size and contain the same bytes
+  // up to the number of compared bytes
+  bool Equals(const Buffer& other, int64_t nbytes) const {
+    return this == &other ||
+      (size_ >= nbytes && other.size_ >= nbytes &&
+          !memcmp(data_, other.data_, nbytes));
+  }
+
+  bool Equals(const Buffer& other) const {
+    return this == &other ||
+      (size_ == other.size_ && !memcmp(data_, other.data_, size_));
+  }
+
+  const uint8_t* data() const {
+    return data_;
+  }
+
+  int64_t size() const {
+    return size_;
+  }
+
+  // Returns true if this Buffer is referencing memory (possibly) owned by some
+  // other buffer
+  bool is_shared() const {
+    return static_cast<bool>(parent_);
+  }
+
+  const std::shared_ptr<Buffer> parent() const {
+    return parent_;
+  }
+
+ protected:
+  const uint8_t* data_;
+  int64_t size_;
+
+  // nullptr by default, but may be set
+  std::shared_ptr<Buffer> parent_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(Buffer);
+};
+
+// A Buffer whose contents can be mutated. May or may not own its data.
+class MutableBuffer : public Buffer {
+ public:
+  MutableBuffer(uint8_t* data, int64_t size) :
+      Buffer(data, size) {
+    mutable_data_ = data;
+  }
+
+  uint8_t* mutable_data() {
+    return mutable_data_;
+  }
+
+  // Get a read-only view of this buffer
+  std::shared_ptr<Buffer> GetImmutableView();
+
+ protected:
+  MutableBuffer() :
+      Buffer(nullptr, 0),
+      mutable_data_(nullptr) {}
+
+  uint8_t* mutable_data_;
+};
+
+// A MutableBuffer whose memory is owned by the class instance. For example,
+// for reading data out of files that you want to deallocate when this class is
+// garbage-collected
+class OwnedMutableBuffer : public MutableBuffer {
+ public:
+  OwnedMutableBuffer();
+  Status Resize(int64_t new_size);
+
+ private:
+  // TODO: aligned allocations
+  std::vector<uint8_t> buffer_owner_;
+};
+
+} // namespace arrow
+
+#endif // ARROW_UTIL_BUFFER_H
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
new file mode 100644
index 0000000000000..069e627c90eaa
--- /dev/null
+++ b/cpp/src/arrow/util/macros.h
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_MACROS_H
+#define ARROW_UTIL_MACROS_H
+
+// From Google gutil
+#define DISALLOW_COPY_AND_ASSIGN(TypeName)      \
+  TypeName(const TypeName&) = delete;           \
+  void operator=(const TypeName&) = delete
+
+#endif // ARROW_UTIL_MACROS_H
diff --git a/cpp/src/arrow/util/random.h b/cpp/src/arrow/util/random.h
new file mode 100644
index 0000000000000..64c197ef080fd
--- /dev/null
+++ b/cpp/src/arrow/util/random.h
@@ -0,0 +1,128 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+// Moved from Kudu http://github.com/cloudera/kudu
+
+#ifndef ARROW_UTIL_RANDOM_H_
+#define ARROW_UTIL_RANDOM_H_
+
+#include <stdint.h>
+
+#include <cmath>
+
+namespace arrow {
+
+namespace random_internal {
+
+static const uint32_t M = 2147483647L;   // 2^31-1
+const double kTwoPi = 6.283185307179586476925286;
+
+} // namespace random_internal
+
+// A very simple random number generator.  Not especially good at
+// generating truly random bits, but good enough for our needs in this
+// package. This implementation is not thread-safe.
+class Random {
+ public:
+  explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) {
+    // Avoid bad seeds.
+    if (seed_ == 0 || seed_ == random_internal::M) {
+      seed_ = 1;
+    }
+  }
+
+  // Next pseudo-random 32-bit unsigned integer.
+  // FIXME: This currently only generates 31 bits of randomness.
+  // The MSB will always be zero.
+  uint32_t Next() {
+    static const uint64_t A = 16807;  // bits 14, 8, 7, 5, 2, 1, 0
+    // We are computing
+    //       seed_ = (seed_ * A) % M,    where M = 2^31-1
+    //
+    // seed_ must not be zero or M, or else all subsequent computed values
+    // will be zero or M respectively.  For all other values, seed_ will end
+    // up cycling through every number in [1,M-1]
+    uint64_t product = seed_ * A;
+
+    // Compute (product % M) using the fact that ((x << 31) % M) == x.
+    seed_ = static_cast<uint32_t>((product >> 31) + (product & random_internal::M));
+    // The first reduction may overflow by 1 bit, so we may need to
+    // repeat.  mod == M is not possible; using > allows the faster
+    // sign-bit-based test.
+    if (seed_ > random_internal::M) {
+      seed_ -= random_internal::M;
+    }
+    return seed_;
+  }
+
+  // Alias for consistency with Next64
+  uint32_t Next32() { return Next(); }
+
+  // Next pseudo-random 64-bit unsigned integer.
+  // FIXME: This currently only generates 62 bits of randomness due to Next()
+  // only giving 31 bits of randomness. The 2 most significant bits will always
+  // be zero.
+  uint64_t Next64() {
+    uint64_t large = Next();
+    // Only shift by 31 bits so we end up with zeros in MSB and not scattered
+    // throughout the 64-bit word. This is due to the weakness in Next() noted
+    // above.
+    large <<= 31;
+    large |= Next();
+    return large;
+  }
+
+  // Returns a uniformly distributed value in the range [0..n-1]
+  // REQUIRES: n > 0
+  uint32_t Uniform(uint32_t n) { return Next() % n; }
+
+  // Alias for consistency with Uniform64
+  uint32_t Uniform32(uint32_t n) { return Uniform(n); }
+
+  // Returns a uniformly distributed 64-bit value in the range [0..n-1]
+  // REQUIRES: n > 0
+  uint64_t Uniform64(uint64_t n) { return Next64() % n; }
+
+  // Randomly returns true ~"1/n" of the time, and false otherwise.
+  // REQUIRES: n > 0
+  bool OneIn(int n) { return (Next() % n) == 0; }
+
+  // Skewed: pick "base" uniformly from range [0,max_log] and then
+  // return "base" random bits.  The effect is to pick a number in the
+  // range [0,2^max_log-1] with exponential bias towards smaller numbers.
+  uint32_t Skewed(int max_log) {
+    return Uniform(1 << Uniform(max_log + 1));
+  }
+
+  // Creates a normal distribution variable using the
+  // Box-Muller transform. See:
+  // http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform
+  // Adapted from WebRTC source code at:
+  // webrtc/trunk/modules/video_coding/main/test/test_util.cc
+  double Normal(double mean, double std_dev) {
+    double uniform1 = (Next() + 1.0) / (random_internal::M + 1.0);
+    double uniform2 = (Next() + 1.0) / (random_internal::M + 1.0);
+    return (mean + std_dev * sqrt(-2 * ::log(uniform1)) *
+        cos(random_internal::kTwoPi * uniform2));
+  }
+
+  // Return a random number between 0.0 and 1.0 inclusive.
+  double NextDoubleFraction() {
+    return Next() / static_cast<double>(random_internal::M + 1.0);
+  }
+
+ private:
+  uint32_t seed_;
+};
+
+
+uint32_t random_seed() {
+  // TODO: use system time to get a reasonably random seed
+  return 0;
+}
+
+
+} // namespace arrow
+
+#endif  // ARROW_UTIL_RANDOM_H_
diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/util/status.cc
new file mode 100644
index 0000000000000..c64b8a3d5f80a
--- /dev/null
+++ b/cpp/src/arrow/util/status.cc
@@ -0,0 +1,38 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// A Status encapsulates the result of an operation.  It may indicate success,
+// or it may indicate an error with an associated error message.
+//
+// Multiple threads can invoke const methods on a Status without
+// external synchronization, but if any of the threads may call a
+// non-const method, all threads accessing the same Status must use
+// external synchronization.
+
+#include "arrow/util/status.h"
+
+#include <assert.h>
+
+namespace arrow {
+
+Status::Status(StatusCode code, const std::string& msg, int16_t posix_code) {
+  assert(code != StatusCode::OK);
+  const uint32_t size = msg.size();
+  char* result = new char[size + 7];
+  memcpy(result, &size, sizeof(size));
+  result[4] = static_cast<char>(code);
+  memcpy(result + 5, &posix_code, sizeof(posix_code));
+  memcpy(result + 7, msg.c_str(), msg.size());
+  state_ = result;
+}
+
+const char* Status::CopyState(const char* state) {
+  uint32_t size;
+  memcpy(&size, state, sizeof(size));
+  char* result = new char[size + 7];
+  memcpy(result, state, size + 7);
+  return result;
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
new file mode 100644
index 0000000000000..47fda40db2596
--- /dev/null
+++ b/cpp/src/arrow/util/status.h
@@ -0,0 +1,152 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// A Status encapsulates the result of an operation.  It may indicate success,
+// or it may indicate an error with an associated error message.
+//
+// Multiple threads can invoke const methods on a Status without
+// external synchronization, but if any of the threads may call a
+// non-const method, all threads accessing the same Status must use
+// external synchronization.
+
+// Adapted from Kudu github.com/cloudera/kudu
+
+#ifndef ARROW_STATUS_H_
+#define ARROW_STATUS_H_
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+
+// Return the given status if it is not OK.
+#define ARROW_RETURN_NOT_OK(s) do {           \
+    ::arrow::Status _s = (s);                 \
+    if (!_s.ok()) return _s;                    \
+  } while (0);
+
+// Return the given status if it is not OK, but first clone it and
+// prepend the given message.
+#define ARROW_RETURN_NOT_OK_PREPEND(s, msg) do {                      \
+    ::arrow::Status _s = (s);                                         \
+    if (::gutil::PREDICT_FALSE(!_s.ok())) return _s.CloneAndPrepend(msg); \
+  } while (0);
+
+// Return 'to_return' if 'to_call' returns a bad status.
+// The substitution for 'to_return' may reference the variable
+// 's' for the bad status.
+#define ARROW_RETURN_NOT_OK_RET(to_call, to_return) do { \
+    ::arrow::Status s = (to_call); \
+    if (::gutil::PREDICT_FALSE(!s.ok())) return (to_return);    \
+  } while (0);
+
+// If 'to_call' returns a bad status, CHECK immediately with a logged message
+// of 'msg' followed by the status.
+#define ARROW_CHECK_OK_PREPEND(to_call, msg) do {         \
+::arrow::Status _s = (to_call);                           \
+ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString();   \
+} while (0);
+
+// If the status is bad, CHECK immediately, appending the status to the
+// logged message.
+#define ARROW_CHECK_OK(s) ARROW_CHECK_OK_PREPEND(s, "Bad status")
+
+namespace arrow {
+
+#define RETURN_NOT_OK(s) do {                   \
+    Status _s = (s);                            \
+    if (!_s.ok()) return _s;                    \
+  } while (0);
+
+enum class StatusCode: char {
+  OK = 0,
+  OutOfMemory = 1,
+  KeyError = 2,
+  Invalid = 3,
+
+  NotImplemented = 10,
+};
+
+class Status {
+ public:
+  // Create a success status.
+  Status() : state_(NULL) { }
+  ~Status() { delete[] state_; }
+
+  // Copy the specified status.
+  Status(const Status& s);
+  void operator=(const Status& s);
+
+  // Return a success status.
+  static Status OK() { return Status(); }
+
+  // Return error status of an appropriate type.
+  static Status OutOfMemory(const std::string& msg, int16_t posix_code = -1) {
+    return Status(StatusCode::OutOfMemory, msg, posix_code);
+  }
+
+  static Status KeyError(const std::string& msg) {
+    return Status(StatusCode::KeyError, msg, -1);
+  }
+
+  static Status NotImplemented(const std::string& msg) {
+    return Status(StatusCode::NotImplemented, msg, -1);
+  }
+
+  static Status Invalid(const std::string& msg) {
+    return Status(StatusCode::Invalid, msg, -1);
+  }
+
+  // Returns true iff the status indicates success.
+  bool ok() const { return (state_ == NULL); }
+
+  bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; }
+  bool IsKeyError() const { return code() == StatusCode::KeyError; }
+  bool IsInvalid() const { return code() == StatusCode::Invalid; }
+
+  // Return a string representation of this status suitable for printing.
+  // Returns the string "OK" for success.
+  std::string ToString() const;
+
+  // Return a string representation of the status code, without the message
+  // text or posix code information.
+  std::string CodeAsString() const;
+
+  // Get the POSIX code associated with this Status, or -1 if there is none.
+  int16_t posix_code() const;
+
+ private:
+  // OK status has a NULL state_.  Otherwise, state_ is a new[] array
+  // of the following form:
+  //    state_[0..3] == length of message
+  //    state_[4]    == code
+  //    state_[5..6] == posix_code
+  //    state_[7..]  == message
+  const char* state_;
+
+  StatusCode code() const {
+    return ((state_ == NULL) ?
+        StatusCode::OK : static_cast<StatusCode>(state_[4]));
+  }
+
+  Status(StatusCode code, const std::string& msg, int16_t posix_code);
+  static const char* CopyState(const char* s);
+};
+
+inline Status::Status(const Status& s) {
+  state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
+}
+
+inline void Status::operator=(const Status& s) {
+  // The following condition catches both aliasing (when this == &s),
+  // and the common case where both s and *this are ok.
+  if (state_ != s.state_) {
+    delete[] state_;
+    state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
+  }
+}
+
+}  // namespace arrow
+
+
+#endif // ARROW_STATUS_H_
diff --git a/cpp/src/arrow/util/test_main.cc b/cpp/src/arrow/util/test_main.cc
new file mode 100644
index 0000000000000..00139f36742ed
--- /dev/null
+++ b/cpp/src/arrow/util/test_main.cc
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+
+  int ret = RUN_ALL_TESTS();
+
+  return ret;
+}
diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
new file mode 100755
index 0000000000000..46794def400eb
--- /dev/null
+++ b/cpp/thirdparty/build_thirdparty.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -x
+set -e
+TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+
+source $TP_DIR/versions.sh
+PREFIX=$TP_DIR/installed
+
+################################################################################
+
+if [ "$#" = "0" ]; then
+  F_ALL=1
+else
+  # Allow passing specific libs to build on the command line
+  for arg in "$*"; do
+    case $arg in
+      "gtest")      F_GTEST=1 ;;
+      *)            echo "Unknown module: $arg"; exit 1 ;;
+    esac
+  done
+fi
+
+################################################################################
+
+# Determine how many parallel jobs to use for make based on the number of cores
+if [[ "$OSTYPE" =~ ^linux ]]; then
+  PARALLEL=$(grep -c processor /proc/cpuinfo)
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  PARALLEL=$(sysctl -n hw.ncpu)
+else
+  echo Unsupported platform $OSTYPE
+  exit 1
+fi
+
+mkdir -p "$PREFIX/include"
+mkdir -p "$PREFIX/lib"
+
+# On some systems, autotools installs libraries to lib64 rather than lib.  Fix
+# this by setting up lib64 as a symlink to lib.  We have to do this step first
+# to handle cases where one third-party library depends on another.
+ln -sf lib "$PREFIX/lib64"
+
+# use the compiled tools
+export PATH=$PREFIX/bin:$PATH
+
+
+# build googletest
+if [ -n "$F_ALL" -o -n "$F_GTEST" ]; then
+  cd $TP_DIR/$GTEST_BASEDIR
+
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    CXXFLAGS=-fPIC cmake -DCMAKE_CXX_FLAGS="-std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes"
+  else
+    CXXFLAGS=-fPIC cmake .
+  fi
+
+  make VERBOSE=1
+fi
+
+echo "---------------------"
+echo "Thirdparty dependencies built and installed into $PREFIX successfully"
diff --git a/cpp/thirdparty/download_thirdparty.sh b/cpp/thirdparty/download_thirdparty.sh
new file mode 100755
index 0000000000000..8ffb22a93f7e2
--- /dev/null
+++ b/cpp/thirdparty/download_thirdparty.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -x
+set -e
+
+TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+
+source $TP_DIR/versions.sh
+
+download_extract_and_cleanup() {
+	filename=$TP_DIR/$(basename "$1")
+	curl -#LC - "$1" -o $filename
+	tar xzf $filename -C $TP_DIR
+	rm $filename
+}
+
+if [ ! -d ${GTEST_BASEDIR} ]; then
+  echo "Fetching gtest"
+  download_extract_and_cleanup $GTEST_URL
+fi
diff --git a/cpp/thirdparty/versions.sh b/cpp/thirdparty/versions.sh
new file mode 100755
index 0000000000000..12ad56ef00103
--- /dev/null
+++ b/cpp/thirdparty/versions.sh
@@ -0,0 +1,3 @@
+GTEST_VERSION=1.7.0
+GTEST_URL="https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+GTEST_BASEDIR=googletest-release-$GTEST_VERSION

From 7e76e3aee92122f39702241db2d0eaea86fd3e8c Mon Sep 17 00:00:00 2001
From: proflin <proflin.me@gmail.com>
Date: Fri, 19 Feb 2016 23:07:17 +0800
Subject: [PATCH 0006/1644] ARROW-5: Update drill-fmpp-maven-plugin to 1.5.0

This closes #1.
---
 java/pom.xml        | 2 --
 java/vector/pom.xml | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/java/pom.xml b/java/pom.xml
index 8a3b192e13e40..4ee4ff4f7604e 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -36,8 +36,6 @@
     <forkCount>2</forkCount>
     <jackson.version>2.7.1</jackson.version>
     <hadoop.version>2.7.1</hadoop.version>
-    <fmpp.version>0.9.15</fmpp.version>
-    <freemarker.version>2.3.21</freemarker.version>
   </properties>
 
   <scm>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index e693344221b9a..1fef81b7eba2a 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -106,7 +106,7 @@
       <plugin> <!-- generate sources from fmpp -->
         <groupId>org.apache.drill.tools</groupId>
         <artifactId>drill-fmpp-maven-plugin</artifactId>
-        <version>1.4.0</version>
+        <version>1.5.0</version>
         <executions>
           <execution>
             <id>generate-fmpp</id>

From e9cc8ce390a1ab28bf71ce6eeb66c915140e2cb9 Mon Sep 17 00:00:00 2001
From: Jacques Nadeau <jacques@apache.org>
Date: Fri, 19 Feb 2016 18:42:35 -0800
Subject: [PATCH 0007/1644] ARROW-5: Correct Apache Maven repo for maven plugin
 use

---
 java/vector/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 1fef81b7eba2a..df5389261ba57 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -60,7 +60,7 @@
         <pluginRepository>
             <id>apache</id>
             <name>apache</name>
-            <url>https://repo.maven.apache.org/</url>
+            <url>https://repo.maven.apache.org/maven2/</url>
             <releases>
                 <enabled>true</enabled>
             </releases>

From e6905effbb9383afd2423a4f86cf9a33ca680b9d Mon Sep 17 00:00:00 2001
From: proflin <proflin.me@gmail.com>
Date: Sat, 20 Feb 2016 15:50:45 +0800
Subject: [PATCH 0008/1644] ARROW-9: Replace straggler references to Drill

- Renaming drill to arrow for TestBaseAllocator
- Fix ArrowBuffer as ArrowBuf
- Replace Drill with Arrow for ValueHolder

This closes #2.
---
 .../main/java/io/netty/buffer/ArrowBuf.java   |  36 +--
 .../io/netty/buffer/ExpandableByteBuf.java    |   2 +-
 .../netty/buffer/PooledByteBufAllocatorL.java |   6 +-
 .../buffer/UnsafeDirectLittleEndian.java      |   4 +-
 .../arrow/memory/AllocationManager.java       |  34 +--
 ...ocator.java => ArrowByteBufAllocator.java} |  10 +-
 .../apache/arrow/memory/BaseAllocator.java    |  22 +-
 .../apache/arrow/memory/BufferAllocator.java  |   4 +-
 .../apache/arrow/memory/BufferManager.java    |   2 +-
 .../org/apache/arrow/memory/package-info.java |   2 +-
 .../arrow/memory/TestBaseAllocator.java       | 232 +++++++++---------
 .../main/codegen/templates/ListWriters.java   |   2 +-
 .../complex/AbstractContainerVector.java      |   2 +-
 .../vector/complex/AbstractMapVector.java     |   2 +-
 .../arrow/vector/holders/ValueHolder.java     |   4 +-
 .../vector/util/ByteFunctionHelpers.java      |  16 +-
 .../arrow/vector/util/DecimalUtility.java     |  16 +-
 17 files changed, 198 insertions(+), 198 deletions(-)
 rename java/memory/src/main/java/org/apache/arrow/memory/{DrillByteBufAllocator.java => ArrowByteBufAllocator.java} (92%)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index f033ba6538e83..bbec26aa85c74 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -56,7 +56,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
   private final boolean isEmpty;
   private volatile int length;
   private final HistoricalLog historicalLog = BaseAllocator.DEBUG ?
-      new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "DrillBuf[%d]", id) : null;
+      new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", id) : null;
 
   public ArrowBuf(
       final AtomicInteger refCnt,
@@ -155,18 +155,18 @@ private void ensure(int width) {
   }
 
   /**
-   * Create a new DrillBuf that is associated with an alternative allocator for the purposes of memory ownership and
-   * accounting. This has no impact on the reference counting for the current DrillBuf except in the situation where the
+   * Create a new ArrowBuf that is associated with an alternative allocator for the purposes of memory ownership and
+   * accounting. This has no impact on the reference counting for the current ArrowBuf except in the situation where the
    * passed in Allocator is the same as the current buffer.
    *
-   * This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
+   * This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a
    * reference count of 1 (in the case that this is the first time this memory is being associated with the new
    * allocator) or the current value of the reference count + 1 for the other AllocationManager/BufferLedger combination
    * in the case that the provided allocator already had an association to this underlying memory.
    *
    * @param target
    *          The target allocator to create an association with.
-   * @return A new DrillBuf which shares the same underlying memory as this DrillBuf.
+   * @return A new ArrowBuf which shares the same underlying memory as this ArrowBuf.
    */
   public ArrowBuf retain(BufferAllocator target) {
 
@@ -178,17 +178,17 @@ public ArrowBuf retain(BufferAllocator target) {
       historicalLog.recordEvent("retain(%s)", target.getName());
     }
     final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
-    return otherLedger.newDrillBuf(offset, length, null);
+    return otherLedger.newArrowBuf(offset, length, null);
   }
 
   /**
-   * Transfer the memory accounting ownership of this DrillBuf to another allocator. This will generate a new DrillBuf
-   * that carries an association with the underlying memory of this DrillBuf. If this DrillBuf is connected to the
+   * Transfer the memory accounting ownership of this ArrowBuf to another allocator. This will generate a new ArrowBuf
+   * that carries an association with the underlying memory of this ArrowBuf. If this ArrowBuf is connected to the
    * owning BufferLedger of this memory, that memory ownership/accounting will be transferred to the taret allocator. If
-   * this DrillBuf does not currently own the memory underlying it (and is only associated with it), this does not
-   * transfer any ownership to the newly created DrillBuf.
+   * this ArrowBuf does not currently own the memory underlying it (and is only associated with it), this does not
+   * transfer any ownership to the newly created ArrowBuf.
    *
-   * This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
+   * This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a
    * reference count of 1 (in the case that this is the first time this memory is being associated with the new
    * allocator) or the current value of the reference count for the other AllocationManager/BufferLedger combination in
    * the case that the provided allocator already had an association to this underlying memory.
@@ -203,7 +203,7 @@ public ArrowBuf retain(BufferAllocator target) {
    * @param target
    *          The allocator to transfer ownership to.
    * @return A new transfer result with the impact of the transfer (whether it was overlimit) as well as the newly
-   *         created DrillBuf.
+   *         created ArrowBuf.
    */
   public TransferResult transferOwnership(BufferAllocator target) {
 
@@ -212,7 +212,7 @@ public TransferResult transferOwnership(BufferAllocator target) {
     }
 
     final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
-    final ArrowBuf newBuf = otherLedger.newDrillBuf(offset, length, null);
+    final ArrowBuf newBuf = otherLedger.newArrowBuf(offset, length, null);
     final boolean allocationFit = this.ledger.transferBalance(otherLedger);
     return new TransferResult(allocationFit, newBuf);
   }
@@ -267,7 +267,7 @@ public boolean release(int decrement) {
 
     if (refCnt < 0) {
       throw new IllegalStateException(
-          String.format("DrillBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString()));
+          String.format("ArrowBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString()));
     }
 
     return refCnt == 0;
@@ -370,7 +370,7 @@ public ArrowBuf slice(int index, int length) {
      * Re the behavior of reference counting, see http://netty.io/wiki/reference-counted-objects.html#wiki-h3-5, which
      * explains that derived buffers share their reference count with their parent
      */
-    final ArrowBuf newBuf = ledger.newDrillBuf(offset + index, length);
+    final ArrowBuf newBuf = ledger.newArrowBuf(offset + index, length);
     newBuf.writerIndex(length);
     return newBuf;
   }
@@ -437,7 +437,7 @@ public long memoryAddress() {
 
   @Override
   public String toString() {
-    return String.format("DrillBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity());
+    return String.format("ArrowBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity());
   }
 
   @Override
@@ -782,7 +782,7 @@ public void close() {
   }
 
   /**
-   * Returns the possible memory consumed by this DrillBuf in the worse case scenario. (not shared, connected to larger
+   * Returns the possible memory consumed by this ArrowBuf in the worse case scenario. (not shared, connected to larger
    * underlying buffer of allocated memory)
    *
    * @return Size in bytes.
@@ -833,7 +833,7 @@ public String toHexString(final int start, final int length) {
   }
 
   /**
-   * Get the integer id assigned to this DrillBuf for debugging purposes.
+   * Get the integer id assigned to this ArrowBuf for debugging purposes.
    *
    * @return integer id
    */
diff --git a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java b/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
index 59886474923f3..7fb884daa3952 100644
--- a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
@@ -20,7 +20,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 
 /**
- * Allows us to decorate DrillBuf to make it expandable so that we can use them in the context of the Netty framework
+ * Allows us to decorate ArrowBuf to make it expandable so that we can use them in the context of the Netty framework
  * (thus supporting RPC level memory accounting).
  */
 public class ExpandableByteBuf extends MutableWrappedByteBuf {
diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index 1610028df9de3..0b6e3f7f8392d 100644
--- a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -32,7 +32,7 @@
 import com.codahale.metrics.MetricRegistry;
 
 /**
- * The base allocator that we use for all of Drill's memory management. Returns UnsafeDirectLittleEndian buffers.
+ * The base allocator that we use for all of Arrow's memory management. Returns UnsafeDirectLittleEndian buffers.
  */
 public class PooledByteBufAllocatorL {
   private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
@@ -184,7 +184,7 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa
 
     private UnsupportedOperationException fail() {
       return new UnsupportedOperationException(
-          "Drill requries that the JVM used supports access sun.misc.Unsafe.  This platform didn't provide that functionality.");
+          "Arrow requries that the JVM used supports access sun.misc.Unsafe.  This platform didn't provide that functionality.");
     }
 
     public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
@@ -197,7 +197,7 @@ public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacit
 
     @Override
     public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
-      throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+      throw new UnsupportedOperationException("Arrow doesn't support using heap buffers.");
     }
 
 
diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
index 6495d5d371e76..a94c6d1988399 100644
--- a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -24,7 +24,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
- * The underlying class we use for little-endian access to memory. Is used underneath DrillBufs to abstract away the
+ * The underlying class we use for little-endian access to memory. Is used underneath ArrowBufs to abstract away the
  * Netty classes and underlying Netty memory management.
  */
 public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
@@ -55,7 +55,7 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
   private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake, AtomicLong bufferCount, AtomicLong bufferSize) {
     super(buf);
     if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
-      throw new IllegalStateException("Drill only runs on LittleEndian systems.");
+      throw new IllegalStateException("Arrow only runs on LittleEndian systems.");
     }
 
     this.bufferCount = bufferCount;
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index 0db61443266c6..37d1d34a62005 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -41,7 +41,7 @@
  * This class is also responsible for managing when memory is allocated and returned to the Netty-based
  * PooledByteBufAllocatorL.
  *
- * The only reason that this isn't package private is we're forced to put DrillBuf in Netty's package which need access
+ * The only reason that this isn't package private is we're forced to put ArrowBuf in Netty's package which need access
  * to these objects or methods.
  *
  * Threading: AllocationManager manages thread-safety internally. Operations within the context of a single BufferLedger
@@ -185,8 +185,8 @@ public void release() {
 
   /**
    * The reference manager that binds an allocator manager to a particular BaseAllocator. Also responsible for creating
-   * a set of DrillBufs that share a common fate and set of reference counts.
-   * As with AllocationManager, the only reason this is public is due to DrillBuf being in io.netty.buffer package.
+   * a set of ArrowBufs that share a common fate and set of reference counts.
+   * As with AllocationManager, the only reason this is public is due to ArrowBuf being in io.netty.buffer package.
    */
   public class BufferLedger {
 
@@ -322,7 +322,7 @@ public int decrement(int decrement) {
     /**
      * Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator doesn't currently have a
      * ledger associated with this AllocationManager, a new one is created. This is placed on BufferLedger rather than
-     * AllocationManager directly because DrillBufs don't have access to AllocationManager and they are the ones
+     * AllocationManager directly because ArrowBufs don't have access to AllocationManager and they are the ones
      * responsible for exposing the ability to associate multiple allocators with a particular piece of underlying
      * memory. Note that this will increment the reference count of this ledger by one to ensure the ledger isn't
      * destroyed before use.
@@ -335,32 +335,32 @@ public BufferLedger getLedgerForAllocator(BufferAllocator allocator) {
     }
 
     /**
-     * Create a new DrillBuf associated with this AllocationManager and memory. Does not impact reference count.
+     * Create a new ArrowBuf associated with this AllocationManager and memory. Does not impact reference count.
      * Typically used for slicing.
      * @param offset
-     *          The offset in bytes to start this new DrillBuf.
+     *          The offset in bytes to start this new ArrowBuf.
      * @param length
-     *          The length in bytes that this DrillBuf will provide access to.
-     * @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
+     *          The length in bytes that this ArrowBuf will provide access to.
+     * @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
      */
-    public ArrowBuf newDrillBuf(int offset, int length) {
+    public ArrowBuf newArrowBuf(int offset, int length) {
       allocator.assertOpen();
-      return newDrillBuf(offset, length, null);
+      return newArrowBuf(offset, length, null);
     }
 
     /**
-     * Create a new DrillBuf associated with this AllocationManager and memory.
+     * Create a new ArrowBuf associated with this AllocationManager and memory.
      * @param offset
-     *          The offset in bytes to start this new DrillBuf.
+     *          The offset in bytes to start this new ArrowBuf.
      * @param length
-     *          The length in bytes that this DrillBuf will provide access to.
+     *          The length in bytes that this ArrowBuf will provide access to.
      * @param manager
-     *          An optional BufferManager argument that can be used to manage expansion of this DrillBuf
+     *          An optional BufferManager argument that can be used to manage expansion of this ArrowBuf
      * @param retain
      *          Whether or not the newly created buffer should get an additional reference count added to it.
-     * @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
+     * @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
      */
-    public ArrowBuf newDrillBuf(int offset, int length, BufferManager manager) {
+    public ArrowBuf newArrowBuf(int offset, int length, BufferManager manager) {
       allocator.assertOpen();
 
       final ArrowBuf buf = new ArrowBuf(
@@ -375,7 +375,7 @@ public ArrowBuf newDrillBuf(int offset, int length, BufferManager manager) {
 
       if (BaseAllocator.DEBUG) {
         historicalLog.recordEvent(
-            "DrillBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == "
+            "ArrowBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == "
                 + "%d](%s)) => ledger hc == %d",
             allocator.name, System.identityHashCode(buf), buf.toString(),
             System.identityHashCode(this));
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
similarity index 92%
rename from java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java
rename to java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
index 23d644841e13f..f3f72fa57c33a 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
@@ -23,19 +23,19 @@
 import io.netty.buffer.ExpandableByteBuf;
 
 /**
- * An implementation of ByteBufAllocator that wraps a Drill BufferAllocator. This allows the RPC layer to be accounted
- * and managed using Drill's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is
+ * An implementation of ByteBufAllocator that wraps a Arrow BufferAllocator. This allows the RPC layer to be accounted
+ * and managed using Arrow's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is
  * the signature and the fact that this Allocator returns ExpandableByteBufs which enable otherwise non-expandable
- * DrillBufs to be expandable.
+ * ArrowBufs to be expandable.
  */
-public class DrillByteBufAllocator implements ByteBufAllocator {
+public class ArrowByteBufAllocator implements ByteBufAllocator {
 
   private static final int DEFAULT_BUFFER_SIZE = 4096;
   private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16;
 
   private final BufferAllocator allocator;
 
-  public DrillByteBufAllocator(BufferAllocator allocator) {
+  public ArrowByteBufAllocator(BufferAllocator allocator) {
     this.allocator = allocator;
   }
 
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 72f77ab0c7bc2..90257bb9ffbf7 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -82,7 +82,7 @@ protected BaseAllocator(
     this.parentAllocator = parentAllocator;
     this.name = name;
 
-    this.thisAsByteBufAllocator = new DrillByteBufAllocator(this);
+    this.thisAsByteBufAllocator = new ArrowByteBufAllocator(this);
 
     if (DEBUG) {
       childAllocators = new IdentityHashMap<>();
@@ -236,7 +236,7 @@ private ArrowBuf bufferWithoutReservation(final int size, BufferManager bufferMa
 
     final AllocationManager manager = new AllocationManager(this, size);
     final BufferLedger ledger = manager.associate(this); // +1 ref cnt (required)
-    final ArrowBuf buffer = ledger.newDrillBuf(0, size, bufferManager);
+    final ArrowBuf buffer = ledger.newArrowBuf(0, size, bufferManager);
 
     // make sure that our allocation is equal to what we expected.
     Preconditions.checkArgument(buffer.capacity() == size,
@@ -314,9 +314,9 @@ public ArrowBuf allocateBuffer() {
       Preconditions.checkState(!closed, "Attempt to allocate after closed");
       Preconditions.checkState(!used, "Attempt to allocate more than once");
 
-      final ArrowBuf drillBuf = allocate(nBytes);
+      final ArrowBuf arrowBuf = allocate(nBytes);
       used = true;
-      return drillBuf;
+      return arrowBuf;
     }
 
     public int getSize() {
@@ -397,13 +397,13 @@ private ArrowBuf allocate(int nBytes) {
        * as well, so we need to return the same number back to avoid double-counting them.
        */
       try {
-        final ArrowBuf drillBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);
+        final ArrowBuf arrowBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);
 
         if (DEBUG) {
-          historicalLog.recordEvent("allocate() => %s", String.format("DrillBuf[%d]", drillBuf.getId()));
+          historicalLog.recordEvent("allocate() => %s", String.format("ArrowBuf[%d]", arrowBuf.getId()));
         }
         success = true;
-        return drillBuf;
+        return arrowBuf;
       } finally {
         if (!success) {
           releaseBytes(nBytes);
@@ -565,7 +565,7 @@ void verifyAllocator() {
    * Verifies the accounting state of the allocator. Only works for DEBUG.
    *
    * <p>
-   * This overload is used for recursive calls, allowing for checking that DrillBufs are unique across all allocators
+   * This overload is used for recursive calls, allowing for checking that ArrowBufs are unique across all allocators
    * that are checked.
    * </p>
    *
@@ -594,7 +594,7 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas
        * Verify my relationships with my descendants.
        *
        * The sum of direct child allocators' owned memory must be <= my allocated memory; my allocated memory also
-       * includes DrillBuf's directly allocated by me.
+       * includes ArrowBuf's directly allocated by me.
        */
       long childTotal = 0;
       for (final BaseAllocator childAllocator : childSet) {
@@ -623,11 +623,11 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas
 
         final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
         /*
-         * Even when shared, DrillBufs are rewrapped, so we should never see the same instance twice.
+         * Even when shared, ArrowBufs are rewrapped, so we should never see the same instance twice.
          */
         final BaseAllocator otherOwner = buffersSeen.get(udle);
         if (otherOwner != null) {
-          throw new IllegalStateException("This allocator's drillBuf already owned by another allocator");
+          throw new IllegalStateException("This allocator's ArrowBuf already owned by another allocator");
         }
         buffersSeen.put(udle, this);
 
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
index 16a68128b704f..356a3416cbf85 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -30,7 +30,7 @@ public interface BufferAllocator extends AutoCloseable {
    *
    * @param size
    *          The size in bytes.
-   * @return a new DrillBuf, or null if the request can't be satisfied
+   * @return a new ArrowBuf, or null if the request can't be satisfied
    * @throws OutOfMemoryException
    *           if buffer cannot be allocated
    */
@@ -44,7 +44,7 @@ public interface BufferAllocator extends AutoCloseable {
    *          The size in bytes.
    * @param manager
    *          A buffer manager to manage reallocation.
-   * @return a new DrillBuf, or null if the request can't be satisfied
+   * @return a new ArrowBuf, or null if the request can't be satisfied
    * @throws OutOfMemoryException
    *           if buffer cannot be allocated
    */
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
index 0610ff09276bf..d6470fa51e7a2 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
@@ -27,7 +27,7 @@
  *
  * The current uses of these types of buffers are within the pluggable components of Drill.
  * In UDFs, memory management should not be a concern. We provide access to re-allocatable
- * DrillBufs to give UDF writers general purpose buffers we can account for. To prevent the need
+ * ArrowBufs to give UDF writers general purpose buffers we can account for. To prevent the need
  * for UDFs to contain boilerplate to close all of the buffers they request, this list
  * is tracked at a higher level and all of the buffers are freed once we are sure that
  * the code depending on them is done executing (currently {@link FragmentContext}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/package-info.java b/java/memory/src/main/java/org/apache/arrow/memory/package-info.java
index 712af3026e29c..40d25cada4519 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/package-info.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/package-info.java
@@ -18,7 +18,7 @@
 /**
  *  Memory Allocation, Account and Management
  *
- *  See the README.md file in this directory for detailed information about Drill's memory allocation subsystem.
+ *  See the README.md file in this directory for detailed information about Arrow's memory allocation subsystem.
  *
  */
 package org.apache.arrow.memory;
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index e13dabb9533da..aa6b70c5c74e2 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -64,17 +64,17 @@ public void checkBuffers() {
   public void test_privateMax() throws Exception {
     try(final RootAllocator rootAllocator =
         new RootAllocator(MAX_ALLOCATION)) {
-      final ArrowBuf drillBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2);
-      assertNotNull("allocation failed", drillBuf1);
+      final ArrowBuf arrowBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2);
+      assertNotNull("allocation failed", arrowBuf1);
 
       try(final BufferAllocator childAllocator =
           rootAllocator.newChildAllocator("noLimits", 0, MAX_ALLOCATION)) {
-        final ArrowBuf drillBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2);
-        assertNotNull("allocation failed", drillBuf2);
-        drillBuf2.release();
+        final ArrowBuf arrowBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2);
+        assertNotNull("allocation failed", arrowBuf2);
+        arrowBuf2.release();
       }
 
-      drillBuf1.release();
+      arrowBuf1.release();
     }
   }
 
@@ -83,8 +83,8 @@ public void testRootAllocator_closeWithOutstanding() throws Exception {
     try {
       try(final RootAllocator rootAllocator =
           new RootAllocator(MAX_ALLOCATION)) {
-        final ArrowBuf drillBuf = rootAllocator.buffer(512);
-        assertNotNull("allocation failed", drillBuf);
+        final ArrowBuf arrowBuf = rootAllocator.buffer(512);
+        assertNotNull("allocation failed", arrowBuf);
       }
     } finally {
       /*
@@ -105,10 +105,10 @@ public void testRootAllocator_closeWithOutstanding() throws Exception {
   public void testRootAllocator_getEmpty() throws Exception {
     try(final RootAllocator rootAllocator =
         new RootAllocator(MAX_ALLOCATION)) {
-      final ArrowBuf drillBuf = rootAllocator.buffer(0);
-      assertNotNull("allocation failed", drillBuf);
-      assertEquals("capacity was non-zero", 0, drillBuf.capacity());
-      drillBuf.release();
+      final ArrowBuf arrowBuf = rootAllocator.buffer(0);
+      assertNotNull("allocation failed", arrowBuf);
+      assertEquals("capacity was non-zero", 0, arrowBuf.capacity());
+      arrowBuf.release();
     }
   }
 
@@ -118,7 +118,7 @@ public void testAllocator_unreleasedEmpty() throws Exception {
     try(final RootAllocator rootAllocator =
         new RootAllocator(MAX_ALLOCATION)) {
       @SuppressWarnings("unused")
-      final ArrowBuf drillBuf = rootAllocator.buffer(0);
+      final ArrowBuf arrowBuf = rootAllocator.buffer(0);
     }
   }
 
@@ -131,14 +131,14 @@ public void testAllocator_transferOwnership() throws Exception {
       final BufferAllocator childAllocator2 =
           rootAllocator.newChildAllocator("changeOwnership2", 0, MAX_ALLOCATION);
 
-      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4);
+      final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4);
       rootAllocator.verify();
-      TransferResult transferOwnership = drillBuf1.transferOwnership(childAllocator2);
+      TransferResult transferOwnership = arrowBuf1.transferOwnership(childAllocator2);
       final boolean allocationFit = transferOwnership.allocationFit;
       rootAllocator.verify();
       assertTrue(allocationFit);
 
-      drillBuf1.release();
+      arrowBuf1.release();
       childAllocator1.close();
       rootAllocator.verify();
 
@@ -152,34 +152,34 @@ public void testAllocator_shareOwnership() throws Exception {
     try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
       final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("shareOwnership1", 0, MAX_ALLOCATION);
       final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("shareOwnership2", 0, MAX_ALLOCATION);
-      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4);
+      final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4);
       rootAllocator.verify();
 
       // share ownership of buffer.
-      final ArrowBuf drillBuf2 = drillBuf1.retain(childAllocator2);
+      final ArrowBuf arrowBuf2 = arrowBuf1.retain(childAllocator2);
       rootAllocator.verify();
-      assertNotNull(drillBuf2);
-      assertNotEquals(drillBuf2, drillBuf1);
+      assertNotNull(arrowBuf2);
+      assertNotEquals(arrowBuf2, arrowBuf1);
 
       // release original buffer (thus transferring ownership to allocator 2. (should leave allocator 1 in empty state)
-      drillBuf1.release();
+      arrowBuf1.release();
       rootAllocator.verify();
       childAllocator1.close();
       rootAllocator.verify();
 
       final BufferAllocator childAllocator3 = rootAllocator.newChildAllocator("shareOwnership3", 0, MAX_ALLOCATION);
-      final ArrowBuf drillBuf3 = drillBuf1.retain(childAllocator3);
-      assertNotNull(drillBuf3);
-      assertNotEquals(drillBuf3, drillBuf1);
-      assertNotEquals(drillBuf3, drillBuf2);
+      final ArrowBuf arrowBuf3 = arrowBuf1.retain(childAllocator3);
+      assertNotNull(arrowBuf3);
+      assertNotEquals(arrowBuf3, arrowBuf1);
+      assertNotEquals(arrowBuf3, arrowBuf2);
       rootAllocator.verify();
 
-      drillBuf2.release();
+      arrowBuf2.release();
       rootAllocator.verify();
       childAllocator2.close();
       rootAllocator.verify();
 
-      drillBuf3.release();
+      arrowBuf3.release();
       rootAllocator.verify();
       childAllocator3.close();
     }
@@ -190,9 +190,9 @@ public void testRootAllocator_createChildAndUse() throws Exception {
     try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
       try (final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createChildAndUse", 0,
           MAX_ALLOCATION)) {
-        final ArrowBuf drillBuf = childAllocator.buffer(512);
-        assertNotNull("allocation failed", drillBuf);
-        drillBuf.release();
+        final ArrowBuf arrowBuf = childAllocator.buffer(512);
+        assertNotNull("allocation failed", arrowBuf);
+        arrowBuf.release();
       }
     }
   }
@@ -203,8 +203,8 @@ public void testRootAllocator_createChildDontClose() throws Exception {
       try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
         final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createChildDontClose", 0,
             MAX_ALLOCATION);
-        final ArrowBuf drillBuf = childAllocator.buffer(512);
-        assertNotNull("allocation failed", drillBuf);
+        final ArrowBuf arrowBuf = childAllocator.buffer(512);
+        assertNotNull("allocation failed", arrowBuf);
       }
     } finally {
       /*
@@ -222,23 +222,23 @@ public void testRootAllocator_createChildDontClose() throws Exception {
   }
 
   private static void allocateAndFree(final BufferAllocator allocator) {
-    final ArrowBuf drillBuf = allocator.buffer(512);
-    assertNotNull("allocation failed", drillBuf);
-    drillBuf.release();
+    final ArrowBuf arrowBuf = allocator.buffer(512);
+    assertNotNull("allocation failed", arrowBuf);
+    arrowBuf.release();
 
-    final ArrowBuf drillBuf2 = allocator.buffer(MAX_ALLOCATION);
-    assertNotNull("allocation failed", drillBuf2);
-    drillBuf2.release();
+    final ArrowBuf arrowBuf2 = allocator.buffer(MAX_ALLOCATION);
+    assertNotNull("allocation failed", arrowBuf2);
+    arrowBuf2.release();
 
     final int nBufs = 8;
-    final ArrowBuf[] drillBufs = new ArrowBuf[nBufs];
-    for(int i = 0; i < drillBufs.length; ++i) {
-      ArrowBuf drillBufi = allocator.buffer(MAX_ALLOCATION / nBufs);
-      assertNotNull("allocation failed", drillBufi);
-      drillBufs[i] = drillBufi;
+    final ArrowBuf[] arrowBufs = new ArrowBuf[nBufs];
+    for(int i = 0; i < arrowBufs.length; ++i) {
+      ArrowBuf arrowBufi = allocator.buffer(MAX_ALLOCATION / nBufs);
+      assertNotNull("allocation failed", arrowBufi);
+      arrowBufs[i] = arrowBufi;
     }
-    for(ArrowBuf drillBufi : drillBufs) {
-      drillBufi.release();
+    for(ArrowBuf arrowBufi : arrowBufs) {
+      arrowBufi.release();
     }
   }
 
@@ -277,10 +277,10 @@ public void testAllocator_overAllocateParent() throws Exception {
         new RootAllocator(MAX_ALLOCATION)) {
       try(final BufferAllocator childAllocator =
           rootAllocator.newChildAllocator("overAllocateParent", 0, MAX_ALLOCATION)) {
-        final ArrowBuf drillBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2);
-        assertNotNull("allocation failed", drillBuf1);
-        final ArrowBuf drillBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2);
-        assertNotNull("allocation failed", drillBuf2);
+        final ArrowBuf arrowBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2);
+        assertNotNull("allocation failed", arrowBuf1);
+        final ArrowBuf arrowBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2);
+        assertNotNull("allocation failed", arrowBuf2);
 
         try {
           childAllocator.buffer(MAX_ALLOCATION / 4);
@@ -289,26 +289,26 @@ public void testAllocator_overAllocateParent() throws Exception {
           // expected
         }
 
-        drillBuf1.release();
-        drillBuf2.release();
+        arrowBuf1.release();
+        arrowBuf2.release();
       }
     }
   }
 
   private static void testAllocator_sliceUpBufferAndRelease(
       final RootAllocator rootAllocator, final BufferAllocator bufferAllocator) {
-    final ArrowBuf drillBuf1 = bufferAllocator.buffer(MAX_ALLOCATION / 2);
+    final ArrowBuf arrowBuf1 = bufferAllocator.buffer(MAX_ALLOCATION / 2);
     rootAllocator.verify();
 
-    final ArrowBuf drillBuf2 = drillBuf1.slice(16, drillBuf1.capacity() - 32);
+    final ArrowBuf arrowBuf2 = arrowBuf1.slice(16, arrowBuf1.capacity() - 32);
     rootAllocator.verify();
-    final ArrowBuf drillBuf3 = drillBuf2.slice(16, drillBuf2.capacity() - 32);
+    final ArrowBuf arrowBuf3 = arrowBuf2.slice(16, arrowBuf2.capacity() - 32);
     rootAllocator.verify();
     @SuppressWarnings("unused")
-    final ArrowBuf drillBuf4 = drillBuf3.slice(16, drillBuf3.capacity() - 32);
+    final ArrowBuf arrowBuf4 = arrowBuf3.slice(16, arrowBuf3.capacity() - 32);
     rootAllocator.verify();
 
-    drillBuf3.release(); // since they share refcounts, one is enough to release them all
+    arrowBuf3.release(); // since they share refcounts, one is enough to release them all
     rootAllocator.verify();
   }
 
@@ -327,11 +327,11 @@ public void testAllocator_createSlices() throws Exception {
       try (final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) {
         try (final BufferAllocator childAllocator2 =
             childAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) {
-          final ArrowBuf drillBuf1 = childAllocator2.buffer(MAX_ALLOCATION / 8);
+          final ArrowBuf arrowBuf1 = childAllocator2.buffer(MAX_ALLOCATION / 8);
           @SuppressWarnings("unused")
-          final ArrowBuf drillBuf2 = drillBuf1.slice(MAX_ALLOCATION / 16, MAX_ALLOCATION / 16);
+          final ArrowBuf arrowBuf2 = arrowBuf1.slice(MAX_ALLOCATION / 16, MAX_ALLOCATION / 16);
           testAllocator_sliceUpBufferAndRelease(rootAllocator, childAllocator);
-          drillBuf1.release();
+          arrowBuf1.release();
           rootAllocator.verify();
         }
         rootAllocator.verify();
@@ -348,14 +348,14 @@ public void testAllocator_sliceRanges() throws Exception {
     try(final RootAllocator rootAllocator =
         new RootAllocator(MAX_ALLOCATION)) {
       // Populate a buffer with byte values corresponding to their indices.
-      final ArrowBuf drillBuf = rootAllocator.buffer(256);
-      assertEquals(256, drillBuf.capacity());
-      assertEquals(0, drillBuf.readerIndex());
-      assertEquals(0, drillBuf.readableBytes());
-      assertEquals(0, drillBuf.writerIndex());
-      assertEquals(256, drillBuf.writableBytes());
-
-      final ArrowBuf slice3 = (ArrowBuf) drillBuf.slice();
+      final ArrowBuf arrowBuf = rootAllocator.buffer(256);
+      assertEquals(256, arrowBuf.capacity());
+      assertEquals(0, arrowBuf.readerIndex());
+      assertEquals(0, arrowBuf.readableBytes());
+      assertEquals(0, arrowBuf.writerIndex());
+      assertEquals(256, arrowBuf.writableBytes());
+
+      final ArrowBuf slice3 = (ArrowBuf) arrowBuf.slice();
       assertEquals(0, slice3.readerIndex());
       assertEquals(0, slice3.readableBytes());
       assertEquals(0, slice3.writerIndex());
@@ -363,14 +363,14 @@ public void testAllocator_sliceRanges() throws Exception {
 //      assertEquals(256, slice3.writableBytes());
 
       for(int i = 0; i < 256; ++i) {
-        drillBuf.writeByte(i);
+        arrowBuf.writeByte(i);
       }
-      assertEquals(0, drillBuf.readerIndex());
-      assertEquals(256, drillBuf.readableBytes());
-      assertEquals(256, drillBuf.writerIndex());
-      assertEquals(0, drillBuf.writableBytes());
+      assertEquals(0, arrowBuf.readerIndex());
+      assertEquals(256, arrowBuf.readableBytes());
+      assertEquals(256, arrowBuf.writerIndex());
+      assertEquals(0, arrowBuf.writableBytes());
 
-      final ArrowBuf slice1 = (ArrowBuf) drillBuf.slice();
+      final ArrowBuf slice1 = (ArrowBuf) arrowBuf.slice();
       assertEquals(0, slice1.readerIndex());
       assertEquals(256, slice1.readableBytes());
       for(int i = 0; i < 10; ++i) {
@@ -381,7 +381,7 @@ public void testAllocator_sliceRanges() throws Exception {
         assertEquals((byte) i, slice1.getByte(i));
       }
 
-      final ArrowBuf slice2 = (ArrowBuf) drillBuf.slice(25, 25);
+      final ArrowBuf slice2 = (ArrowBuf) arrowBuf.slice(25, 25);
       assertEquals(0, slice2.readerIndex());
       assertEquals(25, slice2.readableBytes());
       for(int i = 25; i < 50; ++i) {
@@ -397,7 +397,7 @@ public void testAllocator_sliceRanges() throws Exception {
       }
 */
 
-      drillBuf.release(); // all the derived buffers share this fate
+      arrowBuf.release(); // all the derived buffers share this fate
     }
   }
 
@@ -407,33 +407,33 @@ public void testAllocator_slicesOfSlices() throws Exception {
     try(final RootAllocator rootAllocator =
         new RootAllocator(MAX_ALLOCATION)) {
       // Populate a buffer with byte values corresponding to their indices.
-      final ArrowBuf drillBuf = rootAllocator.buffer(256);
+      final ArrowBuf arrowBuf = rootAllocator.buffer(256);
       for(int i = 0; i < 256; ++i) {
-        drillBuf.writeByte(i);
+        arrowBuf.writeByte(i);
       }
 
       // Slice it up.
-      final ArrowBuf slice0 = drillBuf.slice(0, drillBuf.capacity());
+      final ArrowBuf slice0 = arrowBuf.slice(0, arrowBuf.capacity());
       for(int i = 0; i < 256; ++i) {
-        assertEquals((byte) i, drillBuf.getByte(i));
+        assertEquals((byte) i, arrowBuf.getByte(i));
       }
 
-      final ArrowBuf slice10 = slice0.slice(10, drillBuf.capacity() - 10);
+      final ArrowBuf slice10 = slice0.slice(10, arrowBuf.capacity() - 10);
       for(int i = 10; i < 256; ++i) {
         assertEquals((byte) i, slice10.getByte(i - 10));
       }
 
-      final ArrowBuf slice20 = slice10.slice(10, drillBuf.capacity() - 20);
+      final ArrowBuf slice20 = slice10.slice(10, arrowBuf.capacity() - 20);
       for(int i = 20; i < 256; ++i) {
         assertEquals((byte) i, slice20.getByte(i - 20));
       }
 
-      final ArrowBuf slice30 = slice20.slice(10,  drillBuf.capacity() - 30);
+      final ArrowBuf slice30 = slice20.slice(10,  arrowBuf.capacity() - 30);
       for(int i = 30; i < 256; ++i) {
         assertEquals((byte) i, slice30.getByte(i - 30));
       }
 
-      drillBuf.release();
+      arrowBuf.release();
     }
   }
 
@@ -443,24 +443,24 @@ public void testAllocator_transferSliced() throws Exception {
       final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("transferSliced1", 0, MAX_ALLOCATION);
       final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferSliced2", 0, MAX_ALLOCATION);
 
-      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
-      final ArrowBuf drillBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8);
+      final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
+      final ArrowBuf arrowBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8);
 
-      final ArrowBuf drillBuf1s = drillBuf1.slice(0, drillBuf1.capacity() / 2);
-      final ArrowBuf drillBuf2s = drillBuf2.slice(0, drillBuf2.capacity() / 2);
+      final ArrowBuf arrowBuf1s = arrowBuf1.slice(0, arrowBuf1.capacity() / 2);
+      final ArrowBuf arrowBuf2s = arrowBuf2.slice(0, arrowBuf2.capacity() / 2);
 
       rootAllocator.verify();
 
-      TransferResult result1 = drillBuf2s.transferOwnership(childAllocator1);
+      TransferResult result1 = arrowBuf2s.transferOwnership(childAllocator1);
       rootAllocator.verify();
-      TransferResult result2 = drillBuf1s.transferOwnership(childAllocator2);
+      TransferResult result2 = arrowBuf1s.transferOwnership(childAllocator2);
       rootAllocator.verify();
 
       result1.buffer.release();
       result2.buffer.release();
 
-      drillBuf1s.release(); // releases drillBuf1
-      drillBuf2s.release(); // releases drillBuf2
+      arrowBuf1s.release(); // releases arrowBuf1
+      arrowBuf2s.release(); // releases arrowBuf2
 
       childAllocator1.close();
       childAllocator2.close();
@@ -473,24 +473,24 @@ public void testAllocator_shareSliced() throws Exception {
       final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("transferSliced", 0, MAX_ALLOCATION);
       final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferSliced", 0, MAX_ALLOCATION);
 
-      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
-      final ArrowBuf drillBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8);
+      final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
+      final ArrowBuf arrowBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8);
 
-      final ArrowBuf drillBuf1s = drillBuf1.slice(0, drillBuf1.capacity() / 2);
-      final ArrowBuf drillBuf2s = drillBuf2.slice(0, drillBuf2.capacity() / 2);
+      final ArrowBuf arrowBuf1s = arrowBuf1.slice(0, arrowBuf1.capacity() / 2);
+      final ArrowBuf arrowBuf2s = arrowBuf2.slice(0, arrowBuf2.capacity() / 2);
 
       rootAllocator.verify();
 
-      final ArrowBuf drillBuf2s1 = drillBuf2s.retain(childAllocator1);
-      final ArrowBuf drillBuf1s2 = drillBuf1s.retain(childAllocator2);
+      final ArrowBuf arrowBuf2s1 = arrowBuf2s.retain(childAllocator1);
+      final ArrowBuf arrowBuf1s2 = arrowBuf1s.retain(childAllocator2);
       rootAllocator.verify();
 
-      drillBuf1s.release(); // releases drillBuf1
-      drillBuf2s.release(); // releases drillBuf2
+      arrowBuf1s.release(); // releases arrowBuf1
+      arrowBuf2s.release(); // releases arrowBuf2
       rootAllocator.verify();
 
-      drillBuf2s1.release(); // releases the shared drillBuf2 slice
-      drillBuf1s2.release(); // releases the shared drillBuf1 slice
+      arrowBuf2s1.release(); // releases the shared arrowBuf2 slice
+      arrowBuf1s2.release(); // releases the shared arrowBuf1 slice
 
       childAllocator1.close();
       childAllocator2.close();
@@ -504,42 +504,42 @@ public void testAllocator_transferShared() throws Exception {
       final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferShared2", 0, MAX_ALLOCATION);
       final BufferAllocator childAllocator3 = rootAllocator.newChildAllocator("transferShared3", 0, MAX_ALLOCATION);
 
-      final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
+      final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8);
 
       boolean allocationFit;
 
-      ArrowBuf drillBuf2 = drillBuf1.retain(childAllocator2);
+      ArrowBuf arrowBuf2 = arrowBuf1.retain(childAllocator2);
       rootAllocator.verify();
-      assertNotNull(drillBuf2);
-      assertNotEquals(drillBuf2, drillBuf1);
+      assertNotNull(arrowBuf2);
+      assertNotEquals(arrowBuf2, arrowBuf1);
 
-      TransferResult result = drillBuf1.transferOwnership(childAllocator3);
+      TransferResult result = arrowBuf1.transferOwnership(childAllocator3);
       allocationFit = result.allocationFit;
-      final ArrowBuf drillBuf3 = result.buffer;
+      final ArrowBuf arrowBuf3 = result.buffer;
       assertTrue(allocationFit);
       rootAllocator.verify();
 
       // Since childAllocator3 now has childAllocator1's buffer, 1, can close
-      drillBuf1.release();
+      arrowBuf1.release();
       childAllocator1.close();
       rootAllocator.verify();
 
-      drillBuf2.release();
+      arrowBuf2.release();
       childAllocator2.close();
       rootAllocator.verify();
 
       final BufferAllocator childAllocator4 = rootAllocator.newChildAllocator("transferShared4", 0, MAX_ALLOCATION);
-      TransferResult result2 = drillBuf3.transferOwnership(childAllocator4);
+      TransferResult result2 = arrowBuf3.transferOwnership(childAllocator4);
       allocationFit = result.allocationFit;
-      final ArrowBuf drillBuf4 = result2.buffer;
+      final ArrowBuf arrowBuf4 = result2.buffer;
       assertTrue(allocationFit);
       rootAllocator.verify();
 
-      drillBuf3.release();
+      arrowBuf3.release();
       childAllocator3.close();
       rootAllocator.verify();
 
-      drillBuf4.release();
+      arrowBuf4.release();
       childAllocator4.close();
       rootAllocator.verify();
     }
@@ -569,11 +569,11 @@ public void testAllocator_claimedReservation() throws Exception {
           assertTrue(reservation.add(32));
           assertTrue(reservation.add(32));
 
-          final ArrowBuf drillBuf = reservation.allocateBuffer();
-          assertEquals(64, drillBuf.capacity());
+          final ArrowBuf arrowBuf = reservation.allocateBuffer();
+          assertEquals(64, arrowBuf.capacity());
           rootAllocator.verify();
 
-          drillBuf.release();
+          arrowBuf.release();
           rootAllocator.verify();
         }
         rootAllocator.verify();
diff --git a/java/vector/src/main/codegen/templates/ListWriters.java b/java/vector/src/main/codegen/templates/ListWriters.java
index cf9fa30fa4784..94b812b83dc96 100644
--- a/java/vector/src/main/codegen/templates/ListWriters.java
+++ b/java/vector/src/main/codegen/templates/ListWriters.java
@@ -226,7 +226,7 @@ public void endList() {
   private String getUnsupportedErrorMsg(String expected, String found) {
     final String f = found.substring(3);
     return String.format("In a list of type %s, encountered a value of type %s. "+
-      "Drill does not support lists of different types.",
+      "Arrow does not support lists of different types.",
        f, expected
     );
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index c671c9e0b3c55..9fae2382ecb24 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -106,7 +106,7 @@ protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
     if (clazz.isAssignableFrom(v.getClass())) {
       return (T) v;
     }
-    throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s].  Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
+    throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Arrow doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
   }
 
   MajorType getLastPathType() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index d4189b2314a6a..de6ae829b476d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -137,7 +137,7 @@ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T>
       }
       return vector;
     }
-    final String message = "Drill does not support schema change yet. Existing[%s] and desired[%s] vector types mismatch";
+    final String message = "Arrow does not support schema change yet. Existing[%s] and desired[%s] vector types mismatch";
     throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(), clazz.getSimpleName()));
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
index 88cbcd4a8c308..16777c806ec2d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
@@ -18,10 +18,10 @@
 package org.apache.arrow.vector.holders;
 
 /**
- * Wrapper object for an individual value in Drill.
+ * Wrapper object for an individual value in Arrow.
  *
  * ValueHolders are designed to be mutable wrapper objects for defining clean
- * APIs that access data in Drill. For performance, object creation is avoided
+ * APIs that access data in Arrow. For performance, object creation is avoided
  * at all costs throughout execution. For this reason, ValueHolders are
  * disallowed from implementing any methods, this allows for them to be
  * replaced by their java primitive inner members during optimization of
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
index 2bdfd70b22956..b6dd13a06a82d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
@@ -29,12 +29,12 @@ public class ByteFunctionHelpers {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
 
   /**
-   * Helper function to check for equality of bytes in two DrillBuffers
+   * Helper function to check for equality of bytes in two ArrowBufs
    *
-   * @param left Left DrillBuf for comparison
+   * @param left Left ArrowBuf for comparison
    * @param lStart start offset in the buffer
    * @param lEnd end offset in the buffer
-   * @param right Right DrillBuf for comparison
+   * @param right Right ArrowBuf for comparison
    * @param rStart start offset in the buffer
    * @param rEnd end offset in the buffer
    * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
@@ -81,14 +81,14 @@ private static final int memEqual(final long laddr, int lStart, int lEnd, final
   }
 
   /**
-   * Helper function to compare a set of bytes in two DrillBuffers.
+   * Helper function to compare a set of bytes in two ArrowBufs.
    *
    * Function will check data before completing in the case that
    *
-   * @param left Left DrillBuf to compare
+   * @param left Left ArrowBuf to compare
    * @param lStart start offset in the buffer
    * @param lEnd end offset in the buffer
-   * @param right Right DrillBuf to compare
+   * @param right Right ArrowBuf to compare
    * @param rStart start offset in the buffer
    * @param rEnd end offset in the buffer
    * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
@@ -138,9 +138,9 @@ private static final int memcmp(final long laddr, int lStart, int lEnd, final lo
   }
 
   /**
-   * Helper function to compare a set of bytes in DrillBuf to a ByteArray.
+   * Helper function to compare a set of bytes in ArrowBuf to a ByteArray.
    *
-   * @param left Left DrillBuf for comparison purposes
+   * @param left Left ArrowBuf for comparison purposes
    * @param lStart start offset in the buffer
    * @param lEnd end offset in the buffer
    * @param right second input to be compared
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index 576a5b6351ad1..a3763cd34f1a1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -145,16 +145,16 @@ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
   public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
 
         // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
-        return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false);
+        return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false);
     }
 
     public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
 
         // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
-        return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true);
+        return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true);
     }
 
-    public static BigDecimal getBigDecimalFromDrillBuf(ArrowBuf bytebuf, int start, int length, int scale) {
+    public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) {
       byte[] value = new byte[length];
       bytebuf.getBytes(start, value, 0, length);
       BigInteger unscaledValue = new BigInteger(value);
@@ -168,17 +168,17 @@ public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int sta
     return new BigDecimal(unscaledValue, scale);
   }
 
-    /* Create a BigDecimal object using the data in the DrillBuf.
+    /* Create a BigDecimal object using the data in the ArrowBuf.
      * This function assumes that data is provided in a non-dense format
      * It works on both sparse and intermediate representations.
      */
-  public static BigDecimal getBigDecimalFromDrillBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale,
+  public static BigDecimal getBigDecimalFromArrowBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale,
       boolean truncateScale) {
 
         // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
         int actualDigits;
 
-        // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out
+        // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out
         BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
 
         BigInteger base = BigInteger.valueOf(DIGITS_BASE);
@@ -208,7 +208,7 @@ public static BigDecimal getBigDecimalFromDrillBuf(ByteBuf data, int startIndex,
 
     /* This function returns a BigDecimal object from the dense decimal representation.
      * First step is to convert the dense representation into an intermediate representation
-     * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object
+     * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object
      */
     public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
 
@@ -340,7 +340,7 @@ public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int s
         destIndex = nDecimalDigits - 1;
 
         while (scale > 0) {
-            // Get next set of MAX_DIGITS (9) store it in the DrillBuf
+            // Get next set of MAX_DIGITS (9) store it in the ArrowBuf
             fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
             BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
 

From a3856222d78d58b51088769178715dcb1e5a8d2c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Tue, 1 Mar 2016 14:48:27 -0800
Subject: [PATCH 0009/1644] ARROW-8: Add .travis.yml and test script for Arrow
 C++. OS X build fixes

---
 .travis.yml                       | 27 ++++++++++++++++++++++
 README.md                         | 11 +++++++++
 ci/travis_script_cpp.sh           | 35 +++++++++++++++++++++++++++++
 cpp/CMakeLists.txt                | 37 ++++++++++++++++---------------
 cpp/setup_build_env.sh            |  3 +--
 cpp/src/arrow/util/CMakeLists.txt |  2 +-
 6 files changed, 94 insertions(+), 21 deletions(-)
 create mode 100644 .travis.yml
 create mode 100755 ci/travis_script_cpp.sh

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000000000..cb2d5cb1bad19
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,27 @@
+sudo: required
+dist: trusty
+addons:
+  apt:
+    sources:
+    - ubuntu-toolchain-r-test
+    - kalakris-cmake
+    packages:
+    - gcc-4.9   # Needed for C++11
+    - g++-4.9   # Needed for C++11
+    - gcov
+    - cmake
+    - valgrind
+
+matrix:
+  include:
+  - compiler: gcc
+    language: cpp
+    os: linux
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
+  - compiler: clang
+    language: cpp
+    os: osx
+    addons:
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
diff --git a/README.md b/README.md
index 4423a91351381..d948a996bc075 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,16 @@
 ## Apache Arrow
 
+<table>
+  <tr>
+    <td>Build Status</td>
+    <td>
+    <a href="https://travis-ci.org/apache/arrow">
+    <img src="https://travis-ci.org/apache/arrow.svg?branch=master" alt="travis build status" />
+    </a>
+    </td>
+  </tr>
+</table>
+
 #### Powering Columnar In-Memory Analytics
 
 Arrow is a set of technologies that enable big-data systems to process and move data fast.
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
new file mode 100755
index 0000000000000..28f16cc021fe3
--- /dev/null
+++ b/ci/travis_script_cpp.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+mkdir $TRAVIS_BUILD_DIR/cpp-build
+pushd $TRAVIS_BUILD_DIR/cpp-build
+
+CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+
+# Build an isolated thirdparty
+cp -r $CPP_DIR/thirdparty .
+cp $CPP_DIR/setup_build_env.sh .
+
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+  # Use a C++11 compiler on Linux
+  export CC="gcc-4.9"
+  export CXX="g++-4.9"
+fi
+
+source setup_build_env.sh
+
+echo $GTEST_HOME
+
+cmake -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+make lint
+make -j4
+
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+  valgrind --tool=memcheck --leak-check=yes --error-exitcode=1 ctest
+else
+  ctest
+fi
+
+popd
+rm -rf cpp-build
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 90e55dfddbf30..5ddd9dae3fe82 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -44,6 +44,11 @@ if (NOT "$ENV{ARROW_GCC_ROOT}" STREQUAL "")
   set(CMAKE_CXX_COMPILER ${GCC_ROOT}/bin/g++)
 endif()
 
+if(APPLE)
+  # In newer versions of CMake, this is the default setting
+  set(CMAKE_MACOSX_RPATH 1)
+endif()
+
 # ----------------------------------------------------------------------
 # cmake options
 
@@ -68,19 +73,15 @@ endif()
 ############################################################
 
 # compiler flags that are common across debug/release builds
-#  - msse4.2: Enable sse4.2 compiler intrinsics.
 #  - Wall: Enable all warnings.
-#  - Wno-sign-compare: suppress warnings for comparison between signed and unsigned
-#    integers
-#  -Wno-deprecated: some of the gutil code includes old things like ext/hash_set, ignore that
-#  - pthread: enable multithreaded malloc
-#  - -D__STDC_FORMAT_MACROS: for PRI* print format macros
-#  -fno-strict-aliasing
-#     Assume programs do not follow strict aliasing rules.
-#     GCC cannot always verify whether strict aliasing rules are indeed followed due to
-#     fundamental limitations in escape analysis, which can result in subtle bad code generation.
-#     This has a small perf hit but worth it to avoid hard to debug crashes.
-set(CXX_COMMON_FLAGS "-std=c++11 -fno-strict-aliasing -msse3 -Wall -Wno-deprecated -pthread -D__STDC_FORMAT_MACROS")
+set(CXX_COMMON_FLAGS "-std=c++11 -msse3 -Wall")
+
+if (APPLE)
+  # Depending on the default OSX_DEPLOYMENT_TARGET (< 10.9), libstdc++ may be
+  # the default standard library which does not support C++11. libc++ is the
+  # default from 10.9 onward.
+  set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++")
+endif()
 
 # compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
 # For all builds:
@@ -157,10 +158,6 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   else()
     message("Running without a controlling terminal or in a dumb terminal")
   endif()
-
-  # Use libstdc++ and not libc++. The latter lacks support for tr1 in OSX
-  # and since 10.9 is now the default.
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++")
 endif()
 
 # Sanity check linking option.
@@ -473,11 +470,15 @@ set(ARROW_SRCS
   src/arrow/type.cc
 )
 
-add_library(arrow SHARED
+set(LIBARROW_LINKAGE "SHARED")
+
+add_library(arrow
+  ${LIBARROW_LINKAGE}
   ${ARROW_SRCS}
 )
 target_link_libraries(arrow ${LINK_LIBS})
 set_target_properties(arrow PROPERTIES LINKER_LANGUAGE CXX)
 
 install(TARGETS arrow
-  LIBRARY DESTINATION lib)
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib)
diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
index 457b9717ebe81..e9901bdbecd42 100755
--- a/cpp/setup_build_env.sh
+++ b/cpp/setup_build_env.sh
@@ -1,11 +1,10 @@
 #!/bin/bash
 
-set -e
-
 SOURCE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
 
 ./thirdparty/download_thirdparty.sh
 ./thirdparty/build_thirdparty.sh
+source thirdparty/versions.sh
 
 export GTEST_HOME=$SOURCE_DIR/thirdparty/$GTEST_BASEDIR
 
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 88e3f7a656d90..ff8db6a04106d 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -26,7 +26,7 @@ set(UTIL_SRCS
 )
 
 set(UTIL_LIBS
-  rt)
+)
 
 add_library(arrow_util STATIC
   ${UTIL_SRCS}

From 8f2ca246b34daa49eed2a1eb2a747cab93bb2dbd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 3 Mar 2016 13:49:20 -0800
Subject: [PATCH 0010/1644] ARROW-13: Add PR merge tool from parquet-mr,
 suitably modified

Author: Wes McKinney <wesm@apache.org>

Closes #7 from wesm/ARROW-13 and squashes the following commits:

7a58712 [Wes McKinney] Add PR merge tool from parquet-mr, suitably modified
---
 dev/README.md         |  94 +++++++++++
 dev/merge_arrow_pr.py | 362 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 456 insertions(+)
 create mode 100644 dev/README.md
 create mode 100755 dev/merge_arrow_pr.py

diff --git a/dev/README.md b/dev/README.md
new file mode 100644
index 0000000000000..e986abef1913f
--- /dev/null
+++ b/dev/README.md
@@ -0,0 +1,94 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+# Arrow Developer Scripts
+
+This directory contains scripts useful to developers when packaging,
+testing, or committing to Arrow.
+
+Merging a pull request requires being a committer on the project.
+
+* How to merge a Pull request:
+have an apache and apache-github remote setup
+```
+git remote add apache-github https://github.com/apache/arrow.git
+git remote add apache https://git-wip-us.apache.org/repos/asf/arrow.git
+```
+run the following command
+```
+dev/merge_arrow_pr.py
+```
+
+Note:
+* The directory name of your Arrow git clone must be called arrow
+* Without jira-python installed you'll have to close the JIRA manually
+
+example output:
+```
+Which pull request would you like to merge? (e.g. 34):
+```
+Type the pull request number (from https://github.com/apache/arrow/pulls) and hit enter.
+```
+=== Pull Request #X ===
+title	Blah Blah Blah
+source	repo/branch
+target	master
+url	https://api.github.com/repos/apache/arrow/pulls/X
+
+Proceed with merging pull request #3? (y/n):
+```
+If this looks good, type y and hit enter.
+```
+From git-wip-us.apache.org:/repos/asf/arrow.git
+ * [new branch]      master     -> PR_TOOL_MERGE_PR_3_MASTER
+Switched to branch 'PR_TOOL_MERGE_PR_3_MASTER'
+
+Merge complete (local ref PR_TOOL_MERGE_PR_3_MASTER). Push to apache? (y/n):
+```
+A local branch with the merge has been created.
+type y and hit enter to push it to apache master
+```
+Counting objects: 67, done.
+Delta compression using up to 4 threads.
+Compressing objects: 100% (26/26), done.
+Writing objects: 100% (36/36), 5.32 KiB, done.
+Total 36 (delta 17), reused 0 (delta 0)
+To git-wip-us.apache.org:/repos/arrow-mr.git
+   b767ac4..485658a  PR_TOOL_MERGE_PR_X_MASTER -> master
+Restoring head pointer to b767ac4e
+Note: checking out 'b767ac4e'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by performing another checkout.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -b with the checkout command again. Example:
+
+  git checkout -b new_branch_name
+
+HEAD is now at b767ac4... Update README.md
+Deleting local branch PR_TOOL_MERGE_PR_X
+Deleting local branch PR_TOOL_MERGE_PR_X_MASTER
+Pull request #X merged!
+Merge hash: 485658a5
+
+Would you like to pick 485658a5 into another branch? (y/n):
+```
+For now just say n as we have 1 branch
diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
new file mode 100755
index 0000000000000..ef47dec88c124
--- /dev/null
+++ b/dev/merge_arrow_pr.py
@@ -0,0 +1,362 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Utility for creating well-formed pull request merges and pushing them to Apache.
+#   usage: ./apache-pr-merge.py    (see config env vars below)
+#
+# This utility assumes you already have a local Arrow git clone and that you
+# have added remotes corresponding to both (i) the Github Apache Arrow mirror
+# and (ii) the apache git repo.
+
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import urllib2
+import getpass
+
+try:
+    import jira.client
+    JIRA_IMPORTED = True
+except ImportError:
+    JIRA_IMPORTED = False
+
+# Location of your Arrow git clone
+ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
+PROJECT_NAME = ARROW_HOME.rsplit("/", 1)[1]
+print "ARROW_HOME = " + ARROW_HOME
+print "PROJECT_NAME = " + PROJECT_NAME
+
+# Remote name which points to the Gihub site
+PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github")
+# Remote name which points to Apache git
+PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache")
+# ASF JIRA username
+JIRA_USERNAME = os.environ.get("JIRA_USERNAME")
+# ASF JIRA password
+JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD")
+
+GITHUB_BASE = "https://github.com/apache/" + PROJECT_NAME + "/pull"
+GITHUB_API_BASE = "https://api.github.com/repos/apache/" + PROJECT_NAME
+JIRA_BASE = "https://issues.apache.org/jira/browse"
+JIRA_API_BASE = "https://issues.apache.org/jira"
+# Prefix added to temporary branches
+BRANCH_PREFIX = "PR_TOOL"
+
+os.chdir(ARROW_HOME)
+
+
+def get_json(url):
+    try:
+        return json.load(urllib2.urlopen(url))
+    except urllib2.HTTPError as e:
+        print "Unable to fetch URL, exiting: %s" % url
+        sys.exit(-1)
+
+
+def fail(msg):
+    print msg
+    clean_up()
+    sys.exit(-1)
+
+
+def run_cmd(cmd):
+    try:
+        if isinstance(cmd, list):
+            return subprocess.check_output(cmd)
+        else:
+            return subprocess.check_output(cmd.split(" "))
+    except subprocess.CalledProcessError as e:
+        # this avoids hiding the stdout / stderr of failed processes
+        print 'Command failed: %s' % cmd
+        print 'With output:'
+        print '--------------'
+        print e.output
+        print '--------------'
+        raise e
+
+def continue_maybe(prompt):
+    result = raw_input("\n%s (y/n): " % prompt)
+    if result.lower() != "y":
+        fail("Okay, exiting")
+
+
+original_head = run_cmd("git rev-parse HEAD")[:8]
+
+
+def clean_up():
+    print "Restoring head pointer to %s" % original_head
+    run_cmd("git checkout %s" % original_head)
+
+    branches = run_cmd("git branch").replace(" ", "").split("\n")
+
+    for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches):
+        print "Deleting local branch %s" % branch
+        run_cmd("git branch -D %s" % branch)
+
+
+# merge the requested PR and return the merge hash
+def merge_pr(pr_num, target_ref):
+    pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num)
+    target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper())
+    run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, pr_branch_name))
+    run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, target_branch_name))
+    run_cmd("git checkout %s" % target_branch_name)
+
+    had_conflicts = False
+    try:
+        run_cmd(['git', 'merge', pr_branch_name, '--squash'])
+    except Exception as e:
+        msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e
+        continue_maybe(msg)
+        msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
+        continue_maybe(msg)
+        had_conflicts = True
+
+    commit_authors = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,
+                             '--pretty=format:%an <%ae>']).split("\n")
+    distinct_authors = sorted(set(commit_authors),
+                              key=lambda x: commit_authors.count(x), reverse=True)
+    primary_author = distinct_authors[0]
+    commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,
+                      '--pretty=format:%h [%an] %s']).split("\n\n")
+
+    merge_message_flags = []
+
+    merge_message_flags += ["-m", title]
+    if body != None:
+        merge_message_flags += ["-m", body]
+
+    authors = "\n".join(["Author: %s" % a for a in distinct_authors])
+
+    merge_message_flags += ["-m", authors]
+
+    if had_conflicts:
+        committer_name = run_cmd("git config --get user.name").strip()
+        committer_email = run_cmd("git config --get user.email").strip()
+        message = "This patch had conflicts when merged, resolved by\nCommitter: %s <%s>" % (
+            committer_name, committer_email)
+        merge_message_flags += ["-m", message]
+
+    # The string "Closes #%s" string is required for GitHub to correctly close the PR
+    merge_message_flags += [
+        "-m",
+        "Closes #%s from %s and squashes the following commits:" % (pr_num, pr_repo_desc)]
+    for c in commits:
+        merge_message_flags += ["-m", c]
+
+    run_cmd(['git', 'commit', '--author="%s"' % primary_author] + merge_message_flags)
+
+    continue_maybe("Merge complete (local ref %s). Push to %s?" % (
+        target_branch_name, PUSH_REMOTE_NAME))
+
+    try:
+        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, target_ref))
+    except Exception as e:
+        clean_up()
+        fail("Exception while pushing: %s" % e)
+
+    merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8]
+    clean_up()
+    print("Pull request #%s merged!" % pr_num)
+    print("Merge hash: %s" % merge_hash)
+    return merge_hash
+
+
+def cherry_pick(pr_num, merge_hash, default_branch):
+    pick_ref = raw_input("Enter a branch name [%s]: " % default_branch)
+    if pick_ref == "":
+        pick_ref = default_branch
+
+    pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, pick_ref.upper())
+
+    run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))
+    run_cmd("git checkout %s" % pick_branch_name)
+    run_cmd("git cherry-pick -sx %s" % merge_hash)
+
+    continue_maybe("Pick complete (local ref %s). Push to %s?" % (
+        pick_branch_name, PUSH_REMOTE_NAME))
+
+    try:
+        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref))
+    except Exception as e:
+        clean_up()
+        fail("Exception while pushing: %s" % e)
+
+    pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8]
+    clean_up()
+
+    print("Pull request #%s picked into %s!" % (pr_num, pick_ref))
+    print("Pick hash: %s" % pick_hash)
+    return pick_ref
+
+
+def fix_version_from_branch(branch, versions):
+    # Note: Assumes this is a sorted (newest->oldest) list of un-released versions
+    if branch == "master":
+        return versions[0]
+    else:
+        branch_ver = branch.replace("branch-", "")
+        return filter(lambda x: x.name.startswith(branch_ver), versions)[-1]
+
+def exctract_jira_id(title):
+    m = re.search(r'^(ARROW-[0-9]+)\b.*$', title)
+    if m and m.groups > 0:
+        return m.group(1)
+    else:
+        fail("PR title should be prefixed by a jira id \"ARROW-XXX: ...\", found: \"%s\"" % title)
+
+def check_jira(title):
+    jira_id = exctract_jira_id(title)
+    asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
+                                basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
+    try:
+        issue = asf_jira.issue(jira_id)
+    except Exception as e:
+        fail("ASF JIRA could not find %s\n%s" % (jira_id, e))
+
+def resolve_jira(title, merge_branches, comment):
+    asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
+                                basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
+
+    default_jira_id = exctract_jira_id(title)
+
+    jira_id = raw_input("Enter a JIRA id [%s]: " % default_jira_id)
+    if jira_id == "":
+        jira_id = default_jira_id
+
+    try:
+        issue = asf_jira.issue(jira_id)
+    except Exception as e:
+        fail("ASF JIRA could not find %s\n%s" % (jira_id, e))
+
+    cur_status = issue.fields.status.name
+    cur_summary = issue.fields.summary
+    cur_assignee = issue.fields.assignee
+    if cur_assignee is None:
+        cur_assignee = "NOT ASSIGNED!!!"
+    else:
+        cur_assignee = cur_assignee.displayName
+
+    if cur_status == "Resolved" or cur_status == "Closed":
+        fail("JIRA issue %s already has status '%s'" % (jira_id, cur_status))
+    print ("=== JIRA %s ===" % jira_id)
+    print ("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n" % (
+        cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))
+
+    versions = asf_jira.project_versions("ARROW")
+    versions = sorted(versions, key=lambda x: x.name, reverse=True)
+    versions = filter(lambda x: x.raw['released'] is False, versions)
+
+    default_fix_versions = map(lambda x: fix_version_from_branch(x, versions).name, merge_branches)
+    for v in default_fix_versions:
+        # Handles the case where we have forked a release branch but not yet made the release.
+        # In this case, if the PR is committed to the master branch and the release branch, we
+        # only consider the release branch to be the fix version. E.g. it is not valid to have
+        # both 1.1.0 and 1.0.0 as fix versions.
+        (major, minor, patch) = v.split(".")
+        if patch == "0":
+            previous = "%s.%s.%s" % (major, int(minor) - 1, 0)
+            if previous in default_fix_versions:
+                default_fix_versions = filter(lambda x: x != v, default_fix_versions)
+    default_fix_versions = ",".join(default_fix_versions)
+
+    fix_versions = raw_input("Enter comma-separated fix version(s) [%s]: " % default_fix_versions)
+    if fix_versions == "":
+        fix_versions = default_fix_versions
+    fix_versions = fix_versions.replace(" ", "").split(",")
+
+    def get_version_json(version_str):
+        return filter(lambda v: v.name == version_str, versions)[0].raw
+
+    jira_fix_versions = map(lambda v: get_version_json(v), fix_versions)
+
+    resolve = filter(lambda a: a['name'] == "Resolve Issue", asf_jira.transitions(jira_id))[0]
+    asf_jira.transition_issue(
+        jira_id, resolve["id"], fixVersions=jira_fix_versions, comment=comment)
+
+    print "Succesfully resolved %s with fixVersions=%s!" % (jira_id, fix_versions)
+
+
+if not JIRA_USERNAME:
+    JIRA_USERNAME =  raw_input("Env JIRA_USERNAME not set, please enter your JIRA username:")
+
+if not JIRA_PASSWORD:
+    JIRA_PASSWORD =  getpass.getpass("Env JIRA_PASSWORD not set, please enter your JIRA password:")
+
+branches = get_json("%s/branches" % GITHUB_API_BASE)
+branch_names = filter(lambda x: x.startswith("branch-"), [x['name'] for x in branches])
+# Assumes branch names can be sorted lexicographically
+# Julien: I commented this out as we don't have any "branch-*" branch yet
+#latest_branch = sorted(branch_names, reverse=True)[0]
+
+pr_num = raw_input("Which pull request would you like to merge? (e.g. 34): ")
+pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num))
+
+url = pr["url"]
+title = pr["title"]
+check_jira(title)
+body = pr["body"]
+target_ref = pr["base"]["ref"]
+user_login = pr["user"]["login"]
+base_ref = pr["head"]["ref"]
+pr_repo_desc = "%s/%s" % (user_login, base_ref)
+
+if pr["merged"] is True:
+    print "Pull request %s has already been merged, assuming you want to backport" % pr_num
+    merge_commit_desc = run_cmd([
+        'git', 'log', '--merges', '--first-parent',
+        '--grep=pull request #%s' % pr_num, '--oneline']).split("\n")[0]
+    if merge_commit_desc == "":
+        fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num)
+
+    merge_hash = merge_commit_desc[:7]
+    message = merge_commit_desc[8:]
+
+    print "Found: %s" % message
+    maybe_cherry_pick(pr_num, merge_hash, latest_branch)
+    sys.exit(0)
+
+if not bool(pr["mergeable"]):
+    msg = "Pull request %s is not mergeable in its current form.\n" % pr_num + \
+        "Continue? (experts only!)"
+    continue_maybe(msg)
+
+print ("\n=== Pull Request #%s ===" % pr_num)
+print ("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" % (
+    title, pr_repo_desc, target_ref, url))
+continue_maybe("Proceed with merging pull request #%s?" % pr_num)
+
+merged_refs = [target_ref]
+
+merge_hash = merge_pr(pr_num, target_ref)
+
+pick_prompt = "Would you like to pick %s into another branch?" % merge_hash
+while raw_input("\n%s (y/n): " % pick_prompt).lower() == "y":
+    merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)]
+
+if JIRA_IMPORTED:
+    continue_maybe("Would you like to update the associated JIRA?")
+    jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % (pr_num, GITHUB_BASE, pr_num)
+    resolve_jira(title, merged_refs, jira_comment)
+else:
+    print "Could not find jira-python library. Run 'sudo pip install jira-python' to install."
+    print "Exiting without trying to close the associated JIRA."

From 1000d110cdc8a699cfb9caaee7772a0a5161538c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 3 Mar 2016 14:00:12 -0800
Subject: [PATCH 0011/1644] ARROW-36: Remove fixVersions from JIRA resolve code
 path

This one is tricky to test; sorry I missed this on the first go (the JIRA transition code executes after ARROW-13 was merged).

Author: Wes McKinney <wesm@apache.org>

Closes #11 from wesm/ARROW-36 and squashes the following commits:

432c17c [Wes McKinney] Remove fixVersions from JIRA resolve code path
---
 dev/merge_arrow_pr.py | 37 +++++--------------------------------
 1 file changed, 5 insertions(+), 32 deletions(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index ef47dec88c124..fe0bcd13dd8f1 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -262,38 +262,11 @@ def resolve_jira(title, merge_branches, comment):
     print ("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n" % (
         cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))
 
-    versions = asf_jira.project_versions("ARROW")
-    versions = sorted(versions, key=lambda x: x.name, reverse=True)
-    versions = filter(lambda x: x.raw['released'] is False, versions)
-
-    default_fix_versions = map(lambda x: fix_version_from_branch(x, versions).name, merge_branches)
-    for v in default_fix_versions:
-        # Handles the case where we have forked a release branch but not yet made the release.
-        # In this case, if the PR is committed to the master branch and the release branch, we
-        # only consider the release branch to be the fix version. E.g. it is not valid to have
-        # both 1.1.0 and 1.0.0 as fix versions.
-        (major, minor, patch) = v.split(".")
-        if patch == "0":
-            previous = "%s.%s.%s" % (major, int(minor) - 1, 0)
-            if previous in default_fix_versions:
-                default_fix_versions = filter(lambda x: x != v, default_fix_versions)
-    default_fix_versions = ",".join(default_fix_versions)
-
-    fix_versions = raw_input("Enter comma-separated fix version(s) [%s]: " % default_fix_versions)
-    if fix_versions == "":
-        fix_versions = default_fix_versions
-    fix_versions = fix_versions.replace(" ", "").split(",")
-
-    def get_version_json(version_str):
-        return filter(lambda v: v.name == version_str, versions)[0].raw
-
-    jira_fix_versions = map(lambda v: get_version_json(v), fix_versions)
-
-    resolve = filter(lambda a: a['name'] == "Resolve Issue", asf_jira.transitions(jira_id))[0]
-    asf_jira.transition_issue(
-        jira_id, resolve["id"], fixVersions=jira_fix_versions, comment=comment)
-
-    print "Succesfully resolved %s with fixVersions=%s!" % (jira_id, fix_versions)
+    resolve = filter(lambda a: a['name'] == "Resolve Issue",
+                     asf_jira.transitions(jira_id))[0]
+    asf_jira.transition_issue(jira_id, resolve["id"], comment=comment)
+
+    print "Succesfully resolved %s!" % (jira_id)
 
 
 if not JIRA_USERNAME:

From e418020852ad4fa148b07f21f5b4d47230fe4c5b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 3 Mar 2016 14:02:53 -0800
Subject: [PATCH 0012/1644] ARROW-19: Add an externalized MemoryPool interface
 for use in builder classes

Memory management will be an ongoing concern, but this is a stride in the right direction. Applications requiring custom memory management will be able to implement a subclass of MemoryPool; we can evolve its API as user needs evolve.

Author: Wes McKinney <wesm@apache.org>

Closes #8 from wesm/ARROW-19 and squashes the following commits:

08d3895 [Wes McKinney] Some include cleanup
e319a36 [Wes McKinney] cpplint fixes
abca6eb [Wes McKinney] Add a MemoryPool abstract interface, change builder instances to request memory from pool via Buffer subclass
---
 cpp/CMakeLists.txt                     |  2 +-
 cpp/src/arrow/array-test.cc            | 10 +++-
 cpp/src/arrow/array.h                  |  1 -
 cpp/src/arrow/builder.cc               |  2 +-
 cpp/src/arrow/builder.h                | 22 +++++---
 cpp/src/arrow/types/construct.cc       | 13 +++--
 cpp/src/arrow/types/construct.h        |  4 +-
 cpp/src/arrow/types/list-test.cc       |  2 +-
 cpp/src/arrow/types/list.h             |  7 ++-
 cpp/src/arrow/types/primitive-test.cc  |  5 +-
 cpp/src/arrow/types/primitive.h        | 12 ++--
 cpp/src/arrow/types/string-test.cc     | 29 +++++-----
 cpp/src/arrow/types/string.h           |  9 ++-
 cpp/src/arrow/types/struct.cc          |  1 +
 cpp/src/arrow/types/test-common.h      |  8 ++-
 cpp/src/arrow/types/union.cc           |  1 +
 cpp/src/arrow/util/CMakeLists.txt      |  3 +
 cpp/src/arrow/util/bit-util.cc         |  2 +-
 cpp/src/arrow/util/bit-util.h          |  3 +-
 cpp/src/arrow/util/buffer-test.cc      |  6 +-
 cpp/src/arrow/util/buffer.cc           | 36 ++++++++----
 cpp/src/arrow/util/buffer.h            | 36 ++++++++----
 cpp/src/arrow/util/memory-pool-test.cc | 47 ++++++++++++++++
 cpp/src/arrow/util/memory-pool.cc      | 78 ++++++++++++++++++++++++++
 cpp/src/arrow/util/memory-pool.h       | 41 ++++++++++++++
 25 files changed, 301 insertions(+), 79 deletions(-)
 create mode 100644 cpp/src/arrow/util/memory-pool-test.cc
 create mode 100644 cpp/src/arrow/util/memory-pool.cc
 create mode 100644 cpp/src/arrow/util/memory-pool.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5ddd9dae3fe82..d2c840abfe823 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -434,7 +434,7 @@ if (UNIX)
   add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
   --verbose=2
   --linelength=90
-  --filter=-whitespace/comments,-readability/todo,-build/header_guard
+  --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11
     `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h`)
 endif (UNIX)
 
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 5ecf91624fe73..16afb9bef348c 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -18,6 +18,7 @@
 #include <gtest/gtest.h>
 
 #include <cstdint>
+#include <cstdlib>
 #include <memory>
 #include <string>
 #include <vector>
@@ -28,6 +29,8 @@
 #include "arrow/types/integer.h"
 #include "arrow/types/primitive.h"
 #include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
 
 using std::string;
 using std::vector;
@@ -41,8 +44,10 @@ static TypePtr int32_nn = TypePtr(new Int32Type(false));
 class TestArray : public ::testing::Test {
  public:
   void SetUp() {
-    auto data = std::make_shared<OwnedMutableBuffer>();
-    auto nulls = std::make_shared<OwnedMutableBuffer>();
+    pool_ = GetDefaultMemoryPool();
+
+    auto data = std::make_shared<PoolBuffer>(pool_);
+    auto nulls = std::make_shared<PoolBuffer>(pool_);
 
     ASSERT_OK(data->Resize(400));
     ASSERT_OK(nulls->Resize(128));
@@ -51,6 +56,7 @@ class TestArray : public ::testing::Test {
   }
 
  protected:
+  MemoryPool* pool_;
   std::unique_ptr<Int32Array> arr_;
 };
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index c95450d12a419..0eaa28d528e37 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -19,7 +19,6 @@
 #define ARROW_ARRAY_H
 
 #include <cstdint>
-#include <cstdlib>
 #include <memory>
 
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 1fd7471928367..cb85067315099 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -30,7 +30,7 @@ Status ArrayBuilder::Init(int64_t capacity) {
 
   if (nullable_) {
     int64_t to_alloc = util::ceil_byte(capacity) / 8;
-    nulls_ = std::make_shared<OwnedMutableBuffer>();
+    nulls_ = std::make_shared<PoolBuffer>(pool_);
     RETURN_NOT_OK(nulls_->Resize(to_alloc));
     null_bits_ = nulls_->mutable_data();
     memset(null_bits_, 0, to_alloc);
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index b43668af77cbd..456bb04ae090a 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -23,25 +23,27 @@
 #include <vector>
 
 #include "arrow/type.h"
-#include "arrow/util/buffer.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
 
 class Array;
+class MemoryPool;
+class PoolBuffer;
 
 static constexpr int64_t MIN_BUILDER_CAPACITY = 1 << 8;
 
 // Base class for all data array builders
 class ArrayBuilder {
  public:
-  explicit ArrayBuilder(const TypePtr& type)
-      : type_(type),
-        nullable_(type_->nullable),
-        nulls_(nullptr), null_bits_(nullptr),
-        length_(0),
-        capacity_(0) {}
+  explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type) :
+      pool_(pool),
+      type_(type),
+      nullable_(type_->nullable),
+      nulls_(nullptr), null_bits_(nullptr),
+      length_(0),
+      capacity_(0) {}
 
   virtual ~ArrayBuilder() {}
 
@@ -71,18 +73,20 @@ class ArrayBuilder {
   // this function responsibly.
   Status Advance(int64_t elements);
 
-  const std::shared_ptr<OwnedMutableBuffer>& nulls() const { return nulls_;}
+  const std::shared_ptr<PoolBuffer>& nulls() const { return nulls_;}
 
   // Creates new array object to hold the contents of the builder and transfers
   // ownership of the data
   virtual Status ToArray(Array** out) = 0;
 
  protected:
+  MemoryPool* pool_;
+
   TypePtr type_;
   bool nullable_;
 
   // If the type is not nullable, then null_ is nullptr after initialization
-  std::shared_ptr<OwnedMutableBuffer> nulls_;
+  std::shared_ptr<PoolBuffer> nulls_;
   uint8_t* null_bits_;
 
   // Array length, so far. Also, the index of the next element to be added
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 5176cafd3ba1c..e1bb990063c1b 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -32,12 +32,13 @@ class ArrayBuilder;
 // Initially looked at doing this with vtables, but shared pointers makes it
 // difficult
 
-#define BUILDER_CASE(ENUM, BuilderType)                         \
-    case TypeEnum::ENUM:                                        \
-      *out = static_cast<ArrayBuilder*>(new BuilderType(type)); \
+#define BUILDER_CASE(ENUM, BuilderType)                                 \
+    case TypeEnum::ENUM:                                                \
+      *out = static_cast<ArrayBuilder*>(new BuilderType(pool, type));   \
       return Status::OK();
 
-Status make_builder(const TypePtr& type, ArrayBuilder** out) {
+Status make_builder(MemoryPool* pool, const TypePtr& type,
+    ArrayBuilder** out) {
   switch (type->type) {
     BUILDER_CASE(UINT8, UInt8Builder);
     BUILDER_CASE(INT8, Int8Builder);
@@ -59,10 +60,10 @@ Status make_builder(const TypePtr& type, ArrayBuilder** out) {
       {
         ListType* list_type = static_cast<ListType*>(type.get());
         ArrayBuilder* value_builder;
-        RETURN_NOT_OK(make_builder(list_type->value_type, &value_builder));
+        RETURN_NOT_OK(make_builder(pool, list_type->value_type, &value_builder));
 
         // The ListBuilder takes ownership of the value_builder
-        ListBuilder* builder = new ListBuilder(type, value_builder);
+        ListBuilder* builder = new ListBuilder(pool, type, value_builder);
         *out = static_cast<ArrayBuilder*>(builder);
         return Status::OK();
       }
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index c0bfedd27d6ad..b5ba436f787d9 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -23,9 +23,11 @@
 namespace arrow {
 
 class ArrayBuilder;
+class MemoryPool;
 class Status;
 
-Status make_builder(const TypePtr& type, ArrayBuilder** out);
+Status make_builder(MemoryPool* pool, const TypePtr& type,
+    ArrayBuilder** out);
 
 } // namespace arrow
 
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index 47673ff898bbd..abfc8a31b0daa 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -76,7 +76,7 @@ class TestListBuilder : public TestBuilder {
     type_ = TypePtr(new ListType(value_type_));
 
     ArrayBuilder* tmp;
-    ASSERT_OK(make_builder(type_, &tmp));
+    ASSERT_OK(make_builder(pool_, type_, &tmp));
     builder_.reset(static_cast<ListBuilder*>(tmp));
   }
 
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 0f1116257c507..4ca0f13d53c6f 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -34,6 +34,8 @@
 
 namespace arrow {
 
+class MemoryPool;
+
 struct ListType : public DataType {
   // List can contain any other logical value type
   TypePtr value_type;
@@ -100,8 +102,9 @@ class ListArray : public Array {
 // have been appended to the child array)
 class ListBuilder : public Int32Builder {
  public:
-  ListBuilder(const TypePtr& type, ArrayBuilder* value_builder)
-      : Int32Builder(type) {
+  ListBuilder(MemoryPool* pool, const TypePtr& type,
+      ArrayBuilder* value_builder)
+      : Int32Builder(pool, type) {
     value_builder_.reset(value_builder);
   }
 
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 12968608094d7..3484294a39f9a 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -18,7 +18,6 @@
 #include <gtest/gtest.h>
 
 #include <cstdint>
-#include <cstdlib>
 #include <memory>
 #include <string>
 #include <vector>
@@ -104,10 +103,10 @@ class TestPrimitiveBuilder : public TestBuilder {
     type_nn_ = Attrs::type(false);
 
     ArrayBuilder* tmp;
-    ASSERT_OK(make_builder(type_, &tmp));
+    ASSERT_OK(make_builder(pool_, type_, &tmp));
     builder_.reset(static_cast<BuilderType*>(tmp));
 
-    ASSERT_OK(make_builder(type_nn_, &tmp));
+    ASSERT_OK(make_builder(pool_, type_nn_, &tmp));
     builder_nn_.reset(static_cast<BuilderType*>(tmp));
   }
 
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index a41911224e05e..c5ae0f78a991b 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <cstring>
+#include <memory>
 #include <string>
 
 #include "arrow/array.h"
@@ -31,6 +32,8 @@
 
 namespace arrow {
 
+class MemoryPool;
+
 template <typename Derived>
 struct PrimitiveType : public DataType {
   explicit PrimitiveType(bool nullable = true)
@@ -113,8 +116,9 @@ class PrimitiveBuilder : public ArrayBuilder {
  public:
   typedef typename Type::c_type T;
 
-  explicit PrimitiveBuilder(const TypePtr& type)
-      : ArrayBuilder(type), values_(nullptr) {
+  explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type) :
+      ArrayBuilder(pool, type),
+      values_(nullptr) {
     elsize_ = sizeof(T);
   }
 
@@ -139,7 +143,7 @@ class PrimitiveBuilder : public ArrayBuilder {
   Status Init(int64_t capacity) {
     RETURN_NOT_OK(ArrayBuilder::Init(capacity));
 
-    values_ = std::make_shared<OwnedMutableBuffer>();
+    values_ = std::make_shared<PoolBuffer>(pool_);
     return values_->Resize(capacity * elsize_);
   }
 
@@ -231,7 +235,7 @@ class PrimitiveBuilder : public ArrayBuilder {
   }
 
  protected:
-  std::shared_ptr<OwnedMutableBuffer> values_;
+  std::shared_ptr<PoolBuffer> values_;
   int64_t elsize_;
 };
 
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index 6dba3fdcbb6aa..a2d87ead59c59 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -31,12 +31,9 @@
 #include "arrow/types/test-common.h"
 #include "arrow/util/status.h"
 
-using std::string;
-using std::unique_ptr;
-using std::vector;
-
 namespace arrow {
 
+class Buffer;
 
 TEST(TypesTest, TestCharType) {
   CharType t1(5);
@@ -45,7 +42,7 @@ TEST(TypesTest, TestCharType) {
   ASSERT_TRUE(t1.nullable);
   ASSERT_EQ(t1.size, 5);
 
-  ASSERT_EQ(t1.ToString(), string("char(5)"));
+  ASSERT_EQ(t1.ToString(), std::string("char(5)"));
 
   // Test copy constructor
   CharType t2 = t1;
@@ -63,7 +60,7 @@ TEST(TypesTest, TestVarcharType) {
   ASSERT_EQ(t1.size, 5);
   ASSERT_EQ(t1.physical_type.size, 6);
 
-  ASSERT_EQ(t1.ToString(), string("varchar(5)"));
+  ASSERT_EQ(t1.ToString(), std::string("varchar(5)"));
 
   // Test copy constructor
   VarcharType t2 = t1;
@@ -78,7 +75,7 @@ TEST(TypesTest, TestStringType) {
   StringType str_nn(false);
 
   ASSERT_EQ(str.type, TypeEnum::STRING);
-  ASSERT_EQ(str.name(), string("string"));
+  ASSERT_EQ(str.name(), std::string("string"));
   ASSERT_TRUE(str.nullable);
   ASSERT_FALSE(str_nn.nullable);
 }
@@ -111,11 +108,11 @@ class TestStringContainer : public ::testing::Test  {
   }
 
  protected:
-  vector<int32_t> offsets_;
-  vector<char> chars_;
-  vector<uint8_t> nulls_;
+  std::vector<int32_t> offsets_;
+  std::vector<char> chars_;
+  std::vector<uint8_t> nulls_;
 
-  vector<string> expected_;
+  std::vector<std::string> expected_;
 
   std::shared_ptr<Buffer> value_buf_;
   std::shared_ptr<Buffer> offsets_buf_;
@@ -175,7 +172,7 @@ class TestStringBuilder : public TestBuilder {
     type_ = TypePtr(new StringType());
 
     ArrayBuilder* tmp;
-    ASSERT_OK(make_builder(type_, &tmp));
+    ASSERT_OK(make_builder(pool_, type_, &tmp));
     builder_.reset(static_cast<StringBuilder*>(tmp));
   }
 
@@ -188,8 +185,8 @@ class TestStringBuilder : public TestBuilder {
  protected:
   TypePtr type_;
 
-  unique_ptr<StringBuilder> builder_;
-  unique_ptr<StringArray> result_;
+  std::unique_ptr<StringBuilder> builder_;
+  std::unique_ptr<StringArray> result_;
 };
 
 TEST_F(TestStringBuilder, TestAttrs) {
@@ -197,8 +194,8 @@ TEST_F(TestStringBuilder, TestAttrs) {
 }
 
 TEST_F(TestStringBuilder, TestScalarAppend) {
-  vector<string> strings = {"a", "bb", "", "", "ccc"};
-  vector<uint8_t> is_null = {0, 0, 0, 1, 0};
+  std::vector<std::string> strings = {"a", "bb", "", "", "ccc"};
+  std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
 
   int N = strings.size();
   int reps = 1000;
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index 30d6e247db1ad..d0690d9a7d2a4 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -27,12 +27,13 @@
 #include "arrow/type.h"
 #include "arrow/types/integer.h"
 #include "arrow/types/list.h"
-#include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
 
 class ArrayBuilder;
+class Buffer;
+class MemoryPool;
 
 struct CharType : public DataType {
   int size;
@@ -148,8 +149,9 @@ class StringArray : public ListArray {
 
 class StringBuilder : public ListBuilder {
  public:
-  explicit StringBuilder(const TypePtr& type) :
-      ListBuilder(type, static_cast<ArrayBuilder*>(new UInt8Builder(value_type_))) {
+  explicit StringBuilder(MemoryPool* pool, const TypePtr& type) :
+      ListBuilder(pool, type,
+          static_cast<ArrayBuilder*>(new UInt8Builder(pool, value_type_))) {
     byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
   }
 
@@ -171,6 +173,7 @@ class StringBuilder : public ListBuilder {
   }
 
  protected:
+  std::shared_ptr<ListBuilder> list_builder_;
   UInt8Builder* byte_builder_;
 
   static TypePtr value_type_;
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
index b7be5d8245f1d..a245656b516cc 100644
--- a/cpp/src/arrow/types/struct.cc
+++ b/cpp/src/arrow/types/struct.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/types/struct.h"
 
+#include <cstdlib>
 #include <memory>
 #include <sstream>
 #include <string>
diff --git a/cpp/src/arrow/types/test-common.h b/cpp/src/arrow/types/test-common.h
index 267e48a7f25c9..3ecb0dec7c04a 100644
--- a/cpp/src/arrow/types/test-common.h
+++ b/cpp/src/arrow/types/test-common.h
@@ -25,6 +25,7 @@
 
 #include "arrow/test-util.h"
 #include "arrow/type.h"
+#include "arrow/util/memory-pool.h"
 
 using std::unique_ptr;
 
@@ -33,12 +34,15 @@ namespace arrow {
 class TestBuilder : public ::testing::Test {
  public:
   void SetUp() {
+    pool_ = GetDefaultMemoryPool();
     type_ = TypePtr(new UInt8Type());
     type_nn_ = TypePtr(new UInt8Type(false));
-    builder_.reset(new UInt8Builder(type_));
-    builder_nn_.reset(new UInt8Builder(type_nn_));
+    builder_.reset(new UInt8Builder(pool_, type_));
+    builder_nn_.reset(new UInt8Builder(pool_, type_nn_));
   }
  protected:
+  MemoryPool* pool_;
+
   TypePtr type_;
   TypePtr type_nn_;
   unique_ptr<ArrayBuilder> builder_;
diff --git a/cpp/src/arrow/types/union.cc b/cpp/src/arrow/types/union.cc
index 54f41a7eef6be..db3f81795eae2 100644
--- a/cpp/src/arrow/types/union.cc
+++ b/cpp/src/arrow/types/union.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/types/union.h"
 
+#include <cstdlib>
 #include <sstream>
 #include <string>
 #include <vector>
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index ff8db6a04106d..c53f307c9f59a 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -22,6 +22,7 @@
 set(UTIL_SRCS
   bit-util.cc
   buffer.cc
+  memory-pool.cc
   status.cc
 )
 
@@ -39,6 +40,7 @@ install(FILES
   bit-util.h
   buffer.h
   macros.h
+  memory-pool.h
   status.h
   DESTINATION include/arrow/util)
 
@@ -79,3 +81,4 @@ endif()
 
 ADD_ARROW_TEST(bit-util-test)
 ADD_ARROW_TEST(buffer-test)
+ADD_ARROW_TEST(memory-pool-test)
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index d2ddd6584a88c..dbac0a42527be 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -33,7 +33,7 @@ Status util::bytes_to_bits(uint8_t* bytes, int length,
     std::shared_ptr<Buffer>* out) {
   int bit_length = ceil_byte(length) / 8;
 
-  auto buffer = std::make_shared<OwnedMutableBuffer>();
+  auto buffer = std::make_shared<PoolBuffer>();
   RETURN_NOT_OK(buffer->Resize(bit_length));
   memset(buffer->mutable_data(), 0, bit_length);
   bytes_to_bits(bytes, length, buffer->mutable_data());
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 61dffa30423b1..9ae6127c5ea9c 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -22,10 +22,9 @@
 #include <cstdlib>
 #include <memory>
 
-#include "arrow/util/buffer.h"
-
 namespace arrow {
 
+class Buffer;
 class Status;
 
 namespace util {
diff --git a/cpp/src/arrow/util/buffer-test.cc b/cpp/src/arrow/util/buffer-test.cc
index edfd08e850bd8..9f1fd91432b4d 100644
--- a/cpp/src/arrow/util/buffer-test.cc
+++ b/cpp/src/arrow/util/buffer-test.cc
@@ -16,10 +16,8 @@
 // under the License.
 
 #include <gtest/gtest.h>
-#include <cstdlib>
 #include <cstdint>
 #include <limits>
-#include <memory>
 #include <string>
 
 #include "arrow/test-util.h"
@@ -34,7 +32,7 @@ class TestBuffer : public ::testing::Test {
 };
 
 TEST_F(TestBuffer, Resize) {
-  OwnedMutableBuffer buf;
+  PoolBuffer buf;
 
   ASSERT_EQ(0, buf.size());
   ASSERT_OK(buf.Resize(100));
@@ -49,7 +47,7 @@ TEST_F(TestBuffer, Resize) {
 
 TEST_F(TestBuffer, ResizeOOM) {
   // realloc fails, even though there may be no explicit limit
-  OwnedMutableBuffer buf;
+  PoolBuffer buf;
   ASSERT_OK(buf.Resize(100));
   int64_t to_alloc = std::numeric_limits<int64_t>::max();
   ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
index 2fb34d59e0b78..3f3807d4e2094 100644
--- a/cpp/src/arrow/util/buffer.cc
+++ b/cpp/src/arrow/util/buffer.cc
@@ -19,6 +19,7 @@
 
 #include <cstdint>
 
+#include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
@@ -34,19 +35,34 @@ std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
   return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
 }
 
-OwnedMutableBuffer::OwnedMutableBuffer() :
-    MutableBuffer(nullptr, 0) {}
+PoolBuffer::PoolBuffer(MemoryPool* pool) :
+    ResizableBuffer(nullptr, 0) {
+  if (pool == nullptr) {
+    pool = GetDefaultMemoryPool();
+  }
+  pool_ = pool;
+}
 
-Status OwnedMutableBuffer::Resize(int64_t new_size) {
-  size_ = new_size;
-  try {
-    buffer_owner_.resize(new_size);
-  } catch (const std::bad_alloc& e) {
-    return Status::OutOfMemory("resize failed");
+Status PoolBuffer::Reserve(int64_t new_capacity) {
+  if (!mutable_data_ || new_capacity > capacity_) {
+    uint8_t* new_data;
+    if (mutable_data_) {
+      RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
+      memcpy(new_data, mutable_data_, size_);
+      pool_->Free(mutable_data_, capacity_);
+    } else {
+      RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
+    }
+    mutable_data_ = new_data;
+    data_ = mutable_data_;
+    capacity_ = new_capacity;
   }
-  data_ = buffer_owner_.data();
-  mutable_data_ = buffer_owner_.data();
+  return Status::OK();
+}
 
+Status PoolBuffer::Resize(int64_t new_size) {
+  RETURN_NOT_OK(Reserve(new_size));
+  size_ = new_size;
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 3e4183936b33d..8704723eb0a89 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -19,15 +19,14 @@
 #define ARROW_UTIL_BUFFER_H
 
 #include <cstdint>
-#include <cstdlib>
 #include <cstring>
 #include <memory>
-#include <vector>
 
 #include "arrow/util/macros.h"
 
 namespace arrow {
 
+class MemoryPool;
 class Status;
 
 // ----------------------------------------------------------------------
@@ -115,17 +114,34 @@ class MutableBuffer : public Buffer {
   uint8_t* mutable_data_;
 };
 
-// A MutableBuffer whose memory is owned by the class instance. For example,
-// for reading data out of files that you want to deallocate when this class is
-// garbage-collected
-class OwnedMutableBuffer : public MutableBuffer {
+class ResizableBuffer : public MutableBuffer {
  public:
-  OwnedMutableBuffer();
-  Status Resize(int64_t new_size);
+  // Change buffer reported size to indicated size, allocating memory if
+  // necessary
+  virtual Status Resize(int64_t new_size) = 0;
+
+  // Ensure that buffer has enough memory allocated to fit the indicated
+  // capacity. Does not change buffer's reported size
+  virtual Status Reserve(int64_t new_capacity) = 0;
+
+ protected:
+  ResizableBuffer(uint8_t* data, int64_t size) :
+      MutableBuffer(data, size),
+      capacity_(size) {}
+
+  int64_t capacity_;
+};
+
+// A Buffer whose lifetime is tied to a particular MemoryPool
+class PoolBuffer : public ResizableBuffer {
+ public:
+  explicit PoolBuffer(MemoryPool* pool = nullptr);
+
+  virtual Status Resize(int64_t new_size);
+  virtual Status Reserve(int64_t new_capacity);
 
  private:
-  // TODO: aligned allocations
-  std::vector<uint8_t> buffer_owner_;
+  MemoryPool* pool_;
 };
 
 } // namespace arrow
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
new file mode 100644
index 0000000000000..954b5f951b558
--- /dev/null
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <cstdint>
+#include <limits>
+
+#include "arrow/test-util.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+TEST(DefaultMemoryPool, MemoryTracking) {
+  MemoryPool* pool = GetDefaultMemoryPool();
+
+  uint8_t* data;
+  ASSERT_OK(pool->Allocate(100, &data));
+  ASSERT_EQ(100, pool->bytes_allocated());
+
+  pool->Free(data, 100);
+  ASSERT_EQ(0, pool->bytes_allocated());
+}
+
+TEST(DefaultMemoryPool, OOM) {
+  MemoryPool* pool = GetDefaultMemoryPool();
+
+  uint8_t* data;
+  int64_t to_alloc = std::numeric_limits<int64_t>::max();
+  ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
new file mode 100644
index 0000000000000..5820346e5a739
--- /dev/null
+++ b/cpp/src/arrow/util/memory-pool.cc
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/memory-pool.h"
+
+#include <cstdlib>
+#include <sstream>
+#include <mutex>
+
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+MemoryPool::~MemoryPool() {}
+
+class InternalMemoryPool : public MemoryPool {
+ public:
+  InternalMemoryPool() : bytes_allocated_(0) {}
+  virtual ~InternalMemoryPool();
+
+  Status Allocate(int64_t size, uint8_t** out) override;
+
+  void Free(uint8_t* buffer, int64_t size) override;
+
+  int64_t bytes_allocated() const override;
+
+ private:
+  mutable std::mutex pool_lock_;
+  int64_t bytes_allocated_;
+};
+
+Status InternalMemoryPool::Allocate(int64_t size, uint8_t** out) {
+  std::lock_guard<std::mutex> guard(pool_lock_);
+  *out = static_cast<uint8_t*>(std::malloc(size));
+  if (*out == nullptr) {
+    std::stringstream ss;
+    ss << "malloc of size " << size << " failed";
+    return Status::OutOfMemory(ss.str());
+  }
+
+  bytes_allocated_ += size;
+
+  return Status::OK();
+}
+
+int64_t InternalMemoryPool::bytes_allocated() const {
+  std::lock_guard<std::mutex> guard(pool_lock_);
+  return bytes_allocated_;
+}
+
+void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
+  std::lock_guard<std::mutex> guard(pool_lock_);
+  std::free(buffer);
+  bytes_allocated_ -= size;
+}
+
+InternalMemoryPool::~InternalMemoryPool() {}
+
+MemoryPool* GetDefaultMemoryPool() {
+  static InternalMemoryPool default_memory_pool;
+  return &default_memory_pool;
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/util/memory-pool.h b/cpp/src/arrow/util/memory-pool.h
new file mode 100644
index 0000000000000..a7cb10dae1703
--- /dev/null
+++ b/cpp/src/arrow/util/memory-pool.h
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_MEMORY_POOL_H
+#define ARROW_UTIL_MEMORY_POOL_H
+
+#include <cstdint>
+
+namespace arrow {
+
+class Status;
+
+class MemoryPool {
+ public:
+  virtual ~MemoryPool();
+
+  virtual Status Allocate(int64_t size, uint8_t** out) = 0;
+  virtual void Free(uint8_t* buffer, int64_t size) = 0;
+
+  virtual int64_t bytes_allocated() const = 0;
+};
+
+MemoryPool* GetDefaultMemoryPool();
+
+} // namespace arrow
+
+#endif // ARROW_UTIL_MEMORY_POOL_H

From b88b69e204b59fa8f19cd20dcb6c091fe9bde3a9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 3 Mar 2016 14:56:31 -0800
Subject: [PATCH 0013/1644] ARROW-20: Add null_count_ member to array
 containers, remove nullable_ member

Based off of ARROW-19.

After some contemplation / discussion, I believe it would be better to track nullability at the schema metadata level (if at all!) rather than making it a property of the data structures. This allows the data containers to be "plain ol' data" and thus both nullable data with `null_count == 0` and non-nullable data (implicitly `null_count == 0`) can be treated as semantically equivalent in algorithms code.

If it is deemed useful we can validate (cheaply) that physical data meets the metadata requirements (e.g. non-nullable type metadata cannot be associated with data containers having nulls).

Author: Wes McKinney <wesm@apache.org>

Closes #9 from wesm/ARROW-20 and squashes the following commits:

98be016 [Wes McKinney] ARROW-20: Add null_count_ member to Array containers, remove nullable member
---
 cpp/CMakeLists.txt                    |  2 +-
 cpp/src/arrow/array-test.cc           | 57 ++++++++--------
 cpp/src/arrow/array.cc                | 11 ++--
 cpp/src/arrow/array.h                 | 37 +++++++----
 cpp/src/arrow/builder.cc              | 35 +++++-----
 cpp/src/arrow/builder.h               | 29 ++++----
 cpp/src/arrow/test-util.h             | 10 +++
 cpp/src/arrow/type.h                  | 12 ++--
 cpp/src/arrow/types/collection.h      |  2 +-
 cpp/src/arrow/types/datetime.h        | 12 ++--
 cpp/src/arrow/types/json.h            |  4 +-
 cpp/src/arrow/types/list-test.cc      | 12 +---
 cpp/src/arrow/types/list.h            | 46 ++++++-------
 cpp/src/arrow/types/primitive-test.cc | 34 +++++-----
 cpp/src/arrow/types/primitive.cc      | 11 ++--
 cpp/src/arrow/types/primitive.h       | 95 +++++++++++++++------------
 cpp/src/arrow/types/string-test.cc    | 31 ++++-----
 cpp/src/arrow/types/string.cc         |  2 +-
 cpp/src/arrow/types/string.h          | 43 ++++++------
 cpp/src/arrow/types/struct-test.cc    |  6 +-
 cpp/src/arrow/types/struct.h          |  5 +-
 cpp/src/arrow/types/test-common.h     |  4 +-
 cpp/src/arrow/types/union.h           | 10 ++-
 cpp/src/arrow/util/bit-util.cc        |  4 +-
 cpp/src/arrow/util/bit-util.h         |  4 +-
 25 files changed, 265 insertions(+), 253 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d2c840abfe823..f0eb73dc41371 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -92,7 +92,7 @@ endif()
 # For CMAKE_BUILD_TYPE=Release
 #   -O3: Enable all compiler optimizations
 #   -g: Enable symbols for profiler tools (TODO: remove for shipping)
-set(CXX_FLAGS_DEBUG "-ggdb")
+set(CXX_FLAGS_DEBUG "-ggdb -O0")
 set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
 set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
 
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 16afb9bef348c..df827aaa113aa 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -20,7 +20,6 @@
 #include <cstdint>
 #include <cstdlib>
 #include <memory>
-#include <string>
 #include <vector>
 
 #include "arrow/array.h"
@@ -32,60 +31,60 @@
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
-using std::string;
-using std::vector;
-
 namespace arrow {
 
 static TypePtr int32 = TypePtr(new Int32Type());
-static TypePtr int32_nn = TypePtr(new Int32Type(false));
-
 
 class TestArray : public ::testing::Test {
  public:
   void SetUp() {
     pool_ = GetDefaultMemoryPool();
-
-    auto data = std::make_shared<PoolBuffer>(pool_);
-    auto nulls = std::make_shared<PoolBuffer>(pool_);
-
-    ASSERT_OK(data->Resize(400));
-    ASSERT_OK(nulls->Resize(128));
-
-    arr_.reset(new Int32Array(100, data, nulls));
   }
 
  protected:
   MemoryPool* pool_;
-  std::unique_ptr<Int32Array> arr_;
 };
 
 
-TEST_F(TestArray, TestNullable) {
-  std::shared_ptr<Buffer> tmp = arr_->data();
-  std::unique_ptr<Int32Array> arr_nn(new Int32Array(100, tmp));
+TEST_F(TestArray, TestNullCount) {
+  auto data = std::make_shared<PoolBuffer>(pool_);
+  auto nulls = std::make_shared<PoolBuffer>(pool_);
 
-  ASSERT_TRUE(arr_->nullable());
-  ASSERT_FALSE(arr_nn->nullable());
+  std::unique_ptr<Int32Array> arr(new Int32Array(100, data, 10, nulls));
+  ASSERT_EQ(10, arr->null_count());
+
+  std::unique_ptr<Int32Array> arr_no_nulls(new Int32Array(100, data));
+  ASSERT_EQ(0, arr_no_nulls->null_count());
 }
 
 
 TEST_F(TestArray, TestLength) {
-  ASSERT_EQ(arr_->length(), 100);
+  auto data = std::make_shared<PoolBuffer>(pool_);
+  std::unique_ptr<Int32Array> arr(new Int32Array(100, data));
+  ASSERT_EQ(arr->length(), 100);
 }
 
 TEST_F(TestArray, TestIsNull) {
-  vector<uint8_t> nulls = {1, 0, 1, 1, 0, 1, 0, 0,
-                           1, 0, 1, 1, 0, 1, 0, 0,
-                           1, 0, 1, 1, 0, 1, 0, 0,
-                           1, 0, 1, 1, 0, 1, 0, 0,
-                           1, 0, 0, 1};
+  std::vector<uint8_t> nulls = {1, 0, 1, 1, 0, 1, 0, 0,
+                                1, 0, 1, 1, 0, 1, 0, 0,
+                                1, 0, 1, 1, 0, 1, 0, 0,
+                                1, 0, 1, 1, 0, 1, 0, 0,
+                                1, 0, 0, 1};
+  int32_t null_count = 0;
+  for (uint8_t x : nulls) {
+    if (x > 0) ++null_count;
+  }
 
-  std::shared_ptr<Buffer> null_buf = bytes_to_null_buffer(nulls.data(), nulls.size());
+  std::shared_ptr<Buffer> null_buf = bytes_to_null_buffer(nulls.data(),
+      nulls.size());
   std::unique_ptr<Array> arr;
-  arr.reset(new Array(int32, nulls.size(), null_buf));
+  arr.reset(new Array(int32, nulls.size(), null_count, null_buf));
+
+  ASSERT_EQ(null_count, arr->null_count());
+  ASSERT_EQ(5, null_buf->size());
+
+  ASSERT_TRUE(arr->nulls()->Equals(*null_buf.get()));
 
-  ASSERT_EQ(null_buf->size(), 5);
   for (size_t i = 0; i < nulls.size(); ++i) {
     ASSERT_EQ(static_cast<bool>(nulls[i]), arr->IsNull(i));
   }
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 1726a2f27d82d..ee4ef66d11e26 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -17,6 +17,8 @@
 
 #include "arrow/array.h"
 
+#include <cstdint>
+
 #include "arrow/util/buffer.h"
 
 namespace arrow {
@@ -24,18 +26,17 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Base array class
 
-Array::Array(const TypePtr& type, int64_t length,
+Array::Array(const TypePtr& type, int32_t length, int32_t null_count,
     const std::shared_ptr<Buffer>& nulls) {
-  Init(type, length, nulls);
+  Init(type, length, null_count, nulls);
 }
 
-void Array::Init(const TypePtr& type, int64_t length,
+void Array::Init(const TypePtr& type, int32_t length, int32_t null_count,
     const std::shared_ptr<Buffer>& nulls) {
   type_ = type;
   length_ = length;
+  null_count_ = null_count;
   nulls_ = nulls;
-
-  nullable_ = type->nullable;
   if (nulls_) {
     null_bits_ = nulls_->data();
   }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 0eaa28d528e37..3d748c1bad6f8 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -30,38 +30,49 @@ namespace arrow {
 class Buffer;
 
 // Immutable data array with some logical type and some length. Any memory is
-// owned by the respective Buffer instance (or its parents). May or may not be
-// nullable.
+// owned by the respective Buffer instance (or its parents).
 //
-// The base class only has a null array (if the data type is nullable)
+// The base class is only required to have a nulls buffer if the null count is
+// greater than 0
 //
 // Any buffers used to initialize the array have their references "stolen". If
 // you wish to use the buffer beyond the lifetime of the array, you need to
 // explicitly increment its reference count
 class Array {
  public:
-  Array() : length_(0), nulls_(nullptr), null_bits_(nullptr) {}
-  Array(const TypePtr& type, int64_t length,
+  Array() :
+      null_count_(0),
+      length_(0),
+      nulls_(nullptr),
+      null_bits_(nullptr) {}
+
+  Array(const TypePtr& type, int32_t length, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr);
 
   virtual ~Array() {}
 
-  void Init(const TypePtr& type, int64_t length, const std::shared_ptr<Buffer>& nulls);
+  void Init(const TypePtr& type, int32_t length, int32_t null_count,
+      const std::shared_ptr<Buffer>& nulls);
 
-  // Determine if a slot if null. For inner loops. Does *not* boundscheck
-  bool IsNull(int64_t i) const {
-    return nullable_ && util::get_bit(null_bits_, i);
+  // Determine if a slot is null. For inner loops. Does *not* boundscheck
+  bool IsNull(int i) const {
+    return null_count_ > 0 && util::get_bit(null_bits_, i);
   }
 
-  int64_t length() const { return length_;}
-  bool nullable() const { return nullable_;}
+  int32_t length() const { return length_;}
+  int32_t null_count() const { return null_count_;}
+
   const TypePtr& type() const { return type_;}
   TypeEnum type_enum() const { return type_->type;}
 
+  const std::shared_ptr<Buffer>& nulls() const {
+    return nulls_;
+  }
+
  protected:
   TypePtr type_;
-  bool nullable_;
-  int64_t length_;
+  int32_t null_count_;
+  int32_t length_;
 
   std::shared_ptr<Buffer> nulls_;
   const uint8_t* null_bits_;
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index cb85067315099..ba70add155186 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -25,34 +25,29 @@
 
 namespace arrow {
 
-Status ArrayBuilder::Init(int64_t capacity) {
+Status ArrayBuilder::Init(int32_t capacity) {
   capacity_ = capacity;
-
-  if (nullable_) {
-    int64_t to_alloc = util::ceil_byte(capacity) / 8;
-    nulls_ = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(nulls_->Resize(to_alloc));
-    null_bits_ = nulls_->mutable_data();
-    memset(null_bits_, 0, to_alloc);
-  }
+  int32_t to_alloc = util::ceil_byte(capacity) / 8;
+  nulls_ = std::make_shared<PoolBuffer>(pool_);
+  RETURN_NOT_OK(nulls_->Resize(to_alloc));
+  null_bits_ = nulls_->mutable_data();
+  memset(null_bits_, 0, to_alloc);
   return Status::OK();
 }
 
-Status ArrayBuilder::Resize(int64_t new_bits) {
-  if (nullable_) {
-    int64_t new_bytes = util::ceil_byte(new_bits) / 8;
-    int64_t old_bytes = nulls_->size();
-    RETURN_NOT_OK(nulls_->Resize(new_bytes));
-    null_bits_ = nulls_->mutable_data();
-    if (old_bytes < new_bytes) {
-      memset(null_bits_ + old_bytes, 0, new_bytes - old_bytes);
-    }
+Status ArrayBuilder::Resize(int32_t new_bits) {
+  int32_t new_bytes = util::ceil_byte(new_bits) / 8;
+  int32_t old_bytes = nulls_->size();
+  RETURN_NOT_OK(nulls_->Resize(new_bytes));
+  null_bits_ = nulls_->mutable_data();
+  if (old_bytes < new_bytes) {
+    memset(null_bits_ + old_bytes, 0, new_bytes - old_bytes);
   }
   return Status::OK();
 }
 
-Status ArrayBuilder::Advance(int64_t elements) {
-  if (nullable_ && length_ + elements > capacity_) {
+Status ArrayBuilder::Advance(int32_t elements) {
+  if (length_ + elements > capacity_) {
     return Status::Invalid("Builder must be expanded");
   }
   length_ += elements;
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 456bb04ae090a..491b9133d2cca 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -32,7 +32,7 @@ class Array;
 class MemoryPool;
 class PoolBuffer;
 
-static constexpr int64_t MIN_BUILDER_CAPACITY = 1 << 8;
+static constexpr int32_t MIN_BUILDER_CAPACITY = 1 << 8;
 
 // Base class for all data array builders
 class ArrayBuilder {
@@ -40,8 +40,9 @@ class ArrayBuilder {
   explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type) :
       pool_(pool),
       type_(type),
-      nullable_(type_->nullable),
-      nulls_(nullptr), null_bits_(nullptr),
+      nulls_(nullptr),
+      null_count_(0),
+      null_bits_(nullptr),
       length_(0),
       capacity_(0) {}
 
@@ -57,21 +58,21 @@ class ArrayBuilder {
     return children_.size();
   }
 
-  int64_t length() const { return length_;}
-  int64_t capacity() const { return capacity_;}
-  bool nullable() const { return nullable_;}
+  int32_t length() const { return length_;}
+  int32_t null_count() const { return null_count_;}
+  int32_t capacity() const { return capacity_;}
 
   // Allocates requires memory at this level, but children need to be
   // initialized independently
-  Status Init(int64_t capacity);
+  Status Init(int32_t capacity);
 
-  // Resizes the nulls array (if nullable)
-  Status Resize(int64_t new_bits);
+  // Resizes the nulls array
+  Status Resize(int32_t new_bits);
 
   // For cases where raw data was memcpy'd into the internal buffers, allows us
   // to advance the length of the builder. It is your responsibility to use
   // this function responsibly.
-  Status Advance(int64_t elements);
+  Status Advance(int32_t elements);
 
   const std::shared_ptr<PoolBuffer>& nulls() const { return nulls_;}
 
@@ -83,15 +84,15 @@ class ArrayBuilder {
   MemoryPool* pool_;
 
   TypePtr type_;
-  bool nullable_;
 
-  // If the type is not nullable, then null_ is nullptr after initialization
+  // When nulls are first appended to the builder, the null bitmap is allocated
   std::shared_ptr<PoolBuffer> nulls_;
+  int32_t null_count_;
   uint8_t* null_bits_;
 
   // Array length, so far. Also, the index of the next element to be added
-  int64_t length_;
-  int64_t capacity_;
+  int32_t length_;
+  int32_t capacity_;
 
   // Child value array builders. These are owned by this class
   std::vector<std::unique_ptr<ArrayBuilder> > children_;
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 2233a4f832a8c..0898c8e3e3aa3 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -84,6 +84,16 @@ void random_nulls(int64_t n, double pct_null, std::vector<bool>* nulls) {
   }
 }
 
+static inline int null_count(const std::vector<uint8_t>& nulls) {
+  int result = 0;
+  for (size_t i = 0; i < nulls.size(); ++i) {
+    if (nulls[i] > 0) {
+      ++result;
+    }
+  }
+  return result;
+}
+
 std::shared_ptr<Buffer> bytes_to_null_buffer(uint8_t* bytes, int length) {
   std::shared_ptr<Buffer> out;
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 220f99f4e885a..12f19604c688d 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -57,11 +57,9 @@ struct LayoutType {
 // either a primitive physical type (bytes or bits of some fixed size), a
 // nested type consisting of other data types, or another data type (e.g. a
 // timestamp encoded as an int64)
-//
-// Any data type can be nullable
 
 enum class TypeEnum: char {
-  // A degerate NULL type represented as 0 bytes/bits
+  // A degenerate NULL type represented as 0 bytes/bits
   NA = 0,
 
   // Little-endian integer types
@@ -138,14 +136,12 @@ enum class TypeEnum: char {
 
 struct DataType {
   TypeEnum type;
-  bool nullable;
 
-  explicit DataType(TypeEnum type, bool nullable = true)
-      : type(type), nullable(nullable) {}
+  explicit DataType(TypeEnum type)
+      : type(type) {}
 
   virtual bool Equals(const DataType* other) {
-    return (this == other) || (this->type == other->type &&
-        this->nullable == other->nullable);
+    return this == other || this->type == other->type;
   }
 
   virtual std::string ToString() const = 0;
diff --git a/cpp/src/arrow/types/collection.h b/cpp/src/arrow/types/collection.h
index 59ba61419417a..094b63f28988a 100644
--- a/cpp/src/arrow/types/collection.h
+++ b/cpp/src/arrow/types/collection.h
@@ -29,7 +29,7 @@ template <TypeEnum T>
 struct CollectionType : public DataType {
   std::vector<TypePtr> child_types_;
 
-  explicit CollectionType(bool nullable = true) : DataType(T, nullable) {}
+  CollectionType() : DataType(T) {}
 
   const TypePtr& child(int i) const {
     return child_types_[i];
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
index b4d62523c413a..d90883cb01871 100644
--- a/cpp/src/arrow/types/datetime.h
+++ b/cpp/src/arrow/types/datetime.h
@@ -31,12 +31,12 @@ struct DateType : public DataType {
 
   Unit unit;
 
-  explicit DateType(Unit unit = Unit::DAY, bool nullable = true)
-      : DataType(TypeEnum::DATE, nullable),
+  explicit DateType(Unit unit = Unit::DAY)
+      : DataType(TypeEnum::DATE),
         unit(unit) {}
 
   DateType(const DateType& other)
-      : DateType(other.unit, other.nullable) {}
+      : DateType(other.unit) {}
 
   static char const *name() {
     return "date";
@@ -58,12 +58,12 @@ struct TimestampType : public DataType {
 
   Unit unit;
 
-  explicit TimestampType(Unit unit = Unit::MILLI, bool nullable = true)
-      : DataType(TypeEnum::TIMESTAMP, nullable),
+  explicit TimestampType(Unit unit = Unit::MILLI)
+      : DataType(TypeEnum::TIMESTAMP),
         unit(unit) {}
 
   TimestampType(const TimestampType& other)
-      : TimestampType(other.unit, other.nullable) {}
+      : TimestampType(other.unit) {}
 
   static char const *name() {
     return "timestamp";
diff --git a/cpp/src/arrow/types/json.h b/cpp/src/arrow/types/json.h
index 91fd132408fe6..6c2b097a737c7 100644
--- a/cpp/src/arrow/types/json.h
+++ b/cpp/src/arrow/types/json.h
@@ -28,8 +28,8 @@ struct JSONScalar : public DataType {
   static TypePtr dense_type;
   static TypePtr sparse_type;
 
-  explicit JSONScalar(bool dense = true, bool nullable = true)
-      : DataType(TypeEnum::JSON_SCALAR, nullable),
+  explicit JSONScalar(bool dense = true)
+      : DataType(TypeEnum::JSON_SCALAR),
         dense(dense) {}
 };
 
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index abfc8a31b0daa..1d9ddbe607a41 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -44,11 +44,7 @@ TEST(TypesTest, TestListType) {
   std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
 
   ListType list_type(vt);
-  ListType list_type_nn(vt, false);
-
   ASSERT_EQ(list_type.type, TypeEnum::LIST);
-  ASSERT_TRUE(list_type.nullable);
-  ASSERT_FALSE(list_type_nn.nullable);
 
   ASSERT_EQ(list_type.name(), string("list"));
   ASSERT_EQ(list_type.ToString(), string("list<uint8>"));
@@ -132,8 +128,8 @@ TEST_F(TestListBuilder, TestBasics) {
 
   Done();
 
-  ASSERT_TRUE(result_->nullable());
-  ASSERT_TRUE(result_->values()->nullable());
+  ASSERT_EQ(1, result_->null_count());
+  ASSERT_EQ(0, result_->values()->null_count());
 
   ASSERT_EQ(3, result_->length());
   vector<int32_t> ex_offsets = {0, 3, 3, 7};
@@ -153,10 +149,6 @@ TEST_F(TestListBuilder, TestBasics) {
   }
 }
 
-TEST_F(TestListBuilder, TestBasicsNonNullable) {
-}
-
-
 TEST_F(TestListBuilder, TestZeroLength) {
   // All buffers are null
   Done();
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 4ca0f13d53c6f..4190b53df01cd 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -40,8 +40,8 @@ struct ListType : public DataType {
   // List can contain any other logical value type
   TypePtr value_type;
 
-  explicit ListType(const TypePtr& value_type, bool nullable = true)
-      : DataType(TypeEnum::LIST, nullable),
+  explicit ListType(const TypePtr& value_type)
+      : DataType(TypeEnum::LIST),
         value_type(value_type) {}
 
   static char const *name() {
@@ -56,21 +56,25 @@ class ListArray : public Array {
  public:
   ListArray() : Array(), offset_buf_(nullptr), offsets_(nullptr) {}
 
-  ListArray(const TypePtr& type, int64_t length, std::shared_ptr<Buffer> offsets,
-      const ArrayPtr& values, std::shared_ptr<Buffer> nulls = nullptr) {
-    Init(type, length, offsets, values, nulls);
+  ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
+      const ArrayPtr& values,
+      int32_t null_count = 0,
+      std::shared_ptr<Buffer> nulls = nullptr) {
+    Init(type, length, offsets, values, null_count, nulls);
   }
 
   virtual ~ListArray() {}
 
-  void Init(const TypePtr& type, int64_t length, std::shared_ptr<Buffer> offsets,
-      const ArrayPtr& values, std::shared_ptr<Buffer> nulls = nullptr) {
+  void Init(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
+      const ArrayPtr& values,
+      int32_t null_count = 0,
+      std::shared_ptr<Buffer> nulls = nullptr) {
     offset_buf_ = offsets;
     offsets_ = offsets == nullptr? nullptr :
       reinterpret_cast<const int32_t*>(offset_buf_->data());
 
     values_ = values;
-    Array::Init(type, length, nulls);
+    Array::Init(type, length, null_count, nulls);
   }
 
   // Return a shared pointer in case the requestor desires to share ownership
@@ -108,7 +112,7 @@ class ListBuilder : public Int32Builder {
     value_builder_.reset(value_builder);
   }
 
-  Status Init(int64_t elements) {
+  Status Init(int32_t elements) {
     // One more than requested.
     //
     // XXX: This is slightly imprecise, because we might trigger null mask
@@ -116,7 +120,7 @@ class ListBuilder : public Int32Builder {
     return Int32Builder::Init(elements + 1);
   }
 
-  Status Resize(int64_t capacity) {
+  Status Resize(int32_t capacity) {
     // Need space for the end offset
     RETURN_NOT_OK(Int32Builder::Resize(capacity + 1));
 
@@ -129,18 +133,15 @@ class ListBuilder : public Int32Builder {
   //
   // If passed, null_bytes is of equal length to values, and any nonzero byte
   // will be considered as a null for that slot
-  Status Append(T* values, int64_t length, uint8_t* null_bytes = nullptr) {
+  Status Append(T* values, int32_t length, uint8_t* null_bytes = nullptr) {
     if (length_ + length > capacity_) {
-      int64_t new_capacity = util::next_power2(length_ + length);
+      int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
     }
     memcpy(raw_buffer() + length_, values, length * elsize_);
 
-    if (nullable_ && null_bytes != nullptr) {
-      // If null_bytes is all not null, then none of the values are null
-      for (int i = 0; i < length; ++i) {
-        util::set_bit(null_bits_, length_ + i, static_cast<bool>(null_bytes[i]));
-      }
+    if (null_bytes != nullptr) {
+      AppendNulls(null_bytes, length);
     }
 
     length_ += length;
@@ -159,9 +160,10 @@ class ListBuilder : public Int32Builder {
       raw_buffer()[length_] = child_values->length();
     }
 
-    out->Init(type_, length_, values_, ArrayPtr(child_values), nulls_);
+    out->Init(type_, length_, values_, ArrayPtr(child_values),
+        null_count_, nulls_);
     values_ = nulls_ = nullptr;
-    capacity_ = length_ = 0;
+    capacity_ = length_ = null_count_ = 0;
     return Status::OK();
   }
 
@@ -181,10 +183,10 @@ class ListBuilder : public Int32Builder {
       // If the capacity was not already a multiple of 2, do so here
       RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
     }
-    if (nullable_) {
-      util::set_bit(null_bits_, length_, is_null);
+    if (is_null) {
+      ++null_count_;
+      util::set_bit(null_bits_, length_);
     }
-
     raw_buffer()[length_++] = value_builder_->length();
     return Status::OK();
   }
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 3484294a39f9a..93634432d5ccb 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -53,15 +53,12 @@ TEST(TypesTest, TestBytesType) {
 #define PRIMITIVE_TEST(KLASS, ENUM, NAME)       \
   TEST(TypesTest, TestPrimitive_##ENUM) {       \
     KLASS tp;                                   \
-    KLASS tp_nn(false);                         \
                                                 \
     ASSERT_EQ(tp.type, TypeEnum::ENUM);         \
     ASSERT_EQ(tp.name(), string(NAME));         \
-    ASSERT_TRUE(tp.nullable);                   \
-    ASSERT_FALSE(tp_nn.nullable);               \
                                                 \
-    KLASS tp_copy = tp_nn;                      \
-    ASSERT_FALSE(tp_copy.nullable);             \
+    KLASS tp_copy = tp;                         \
+    ASSERT_EQ(tp_copy.type, TypeEnum::ENUM);    \
   }
 
 PRIMITIVE_TEST(Int8Type, INT8, "int8");
@@ -100,17 +97,16 @@ class TestPrimitiveBuilder : public TestBuilder {
     TestBuilder::SetUp();
 
     type_ = Attrs::type();
-    type_nn_ = Attrs::type(false);
 
     ArrayBuilder* tmp;
     ASSERT_OK(make_builder(pool_, type_, &tmp));
     builder_.reset(static_cast<BuilderType*>(tmp));
 
-    ASSERT_OK(make_builder(pool_, type_nn_, &tmp));
+    ASSERT_OK(make_builder(pool_, type_, &tmp));
     builder_nn_.reset(static_cast<BuilderType*>(tmp));
   }
 
-  void RandomData(int64_t N, double pct_null = 0.1) {
+  void RandomData(int N, double pct_null = 0.1) {
     Attrs::draw(N, &draws_);
     random_nulls(N, pct_null, &nulls_);
   }
@@ -118,28 +114,33 @@ class TestPrimitiveBuilder : public TestBuilder {
   void CheckNullable() {
     ArrayType result;
     ArrayType expected;
-    int64_t size = builder_->length();
+    int size = builder_->length();
 
-    auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
+    auto ex_data = std::make_shared<Buffer>(
+        reinterpret_cast<uint8_t*>(draws_.data()),
         size * sizeof(T));
 
     auto ex_nulls = bytes_to_null_buffer(nulls_.data(), size);
 
-    expected.Init(size, ex_data, ex_nulls);
+    int32_t ex_null_count = null_count(nulls_);
+
+    expected.Init(size, ex_data, ex_null_count, ex_nulls);
     ASSERT_OK(builder_->Transfer(&result));
 
     // Builder is now reset
     ASSERT_EQ(0, builder_->length());
     ASSERT_EQ(0, builder_->capacity());
+    ASSERT_EQ(0, builder_->null_count());
     ASSERT_EQ(nullptr, builder_->buffer());
 
     ASSERT_TRUE(result.Equals(expected));
+    ASSERT_EQ(ex_null_count, result.null_count());
   }
 
   void CheckNonNullable() {
     ArrayType result;
     ArrayType expected;
-    int64_t size = builder_nn_->length();
+    int size = builder_nn_->length();
 
     auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
         size * sizeof(T));
@@ -153,6 +154,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     ASSERT_EQ(nullptr, builder_nn_->buffer());
 
     ASSERT_TRUE(result.Equals(expected));
+    ASSERT_EQ(0, result.null_count());
   }
 
  protected:
@@ -171,14 +173,14 @@ class TestPrimitiveBuilder : public TestBuilder {
   typedef CapType##Type Type;                   \
   typedef c_type T;                             \
                                                 \
-  static TypePtr type(bool nullable = true) {   \
-    return TypePtr(new Type(nullable));         \
+  static TypePtr type() {                       \
+    return TypePtr(new Type());                 \
   }
 
 #define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
   struct P##CapType {                               \
     PTYPE_DECL(CapType, c_type);                    \
-    static void draw(int64_t N, vector<T>* draws) {  \
+    static void draw(int N, vector<T>* draws) {  \
       randint<T>(N, LOWER, UPPER, draws);           \
     }                                               \
   }
@@ -208,7 +210,7 @@ TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 TYPED_TEST(TestPrimitiveBuilder, TestInit) {
   DECL_T();
 
-  int64_t n = 1000;
+  int n = 1000;
   ASSERT_OK(this->builder_->Init(n));
   ASSERT_EQ(n, this->builder_->capacity());
   ASSERT_EQ(n * sizeof(T), this->builder_->buffer()->size());
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 2612e8ca7fd4a..c86260b0fc641 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -26,20 +26,23 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Primitive array base
 
-void PrimitiveArray::Init(const TypePtr& type, int64_t length,
+void PrimitiveArray::Init(const TypePtr& type, int32_t length,
     const std::shared_ptr<Buffer>& data,
+    int32_t null_count,
     const std::shared_ptr<Buffer>& nulls) {
-  Array::Init(type, length, nulls);
+  Array::Init(type, length, null_count, nulls);
   data_ = data;
   raw_data_ = data == nullptr? nullptr : data_->data();
 }
 
 bool PrimitiveArray::Equals(const PrimitiveArray& other) const {
   if (this == &other) return true;
-  if (type_->nullable != other.type_->nullable) return false;
+  if (null_count_ != other.null_count_) {
+    return false;
+  }
 
   bool equal_data = data_->Equals(*other.data_, length_);
-  if (type_->nullable) {
+  if (null_count_ > 0) {
     return equal_data &&
       nulls_->Equals(*other.nulls_, util::ceil_byte(length_) / 8);
   } else {
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index c5ae0f78a991b..aa8f351202a20 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -36,24 +36,24 @@ class MemoryPool;
 
 template <typename Derived>
 struct PrimitiveType : public DataType {
-  explicit PrimitiveType(bool nullable = true)
-      : DataType(Derived::type_enum, nullable) {}
+  PrimitiveType()
+      : DataType(Derived::type_enum) {}
 
   virtual std::string ToString() const {
     return std::string(static_cast<const Derived*>(this)->name());
   }
 };
 
-#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)          \
-  typedef C_TYPE c_type;                                            \
-  static constexpr TypeEnum type_enum = TypeEnum::ENUM;             \
-  static constexpr int size = SIZE;                              \
-                                                                    \
-  explicit TYPENAME(bool nullable = true)                           \
-      : PrimitiveType<TYPENAME>(nullable) {}                        \
-                                                                    \
-  static const char* name() {                                       \
-    return NAME;                                                    \
+#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)  \
+  typedef C_TYPE c_type;                                    \
+  static constexpr TypeEnum type_enum = TypeEnum::ENUM;     \
+  static constexpr int size = SIZE;                         \
+                                                            \
+  TYPENAME()                                                \
+      : PrimitiveType<TYPENAME>() {}                        \
+                                                            \
+  static const char* name() {                               \
+    return NAME;                                            \
   }
 
 
@@ -64,7 +64,9 @@ class PrimitiveArray : public Array {
 
   virtual ~PrimitiveArray() {}
 
-  void Init(const TypePtr& type, int64_t length, const std::shared_ptr<Buffer>& data,
+  void Init(const TypePtr& type, int32_t length,
+      const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr);
 
   const std::shared_ptr<Buffer>& data() const { return data_;}
@@ -84,15 +86,17 @@ class PrimitiveArrayImpl : public PrimitiveArray {
 
   PrimitiveArrayImpl() : PrimitiveArray() {}
 
-  PrimitiveArrayImpl(int64_t length, const std::shared_ptr<Buffer>& data,
+  PrimitiveArrayImpl(int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr) {
-    Init(length, data, nulls);
+    Init(length, data, null_count, nulls);
   }
 
-  void Init(int64_t length, const std::shared_ptr<Buffer>& data,
+  void Init(int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr) {
-    TypePtr type(new TypeClass(nulls != nullptr));
-    PrimitiveArray::Init(type, length, data, nulls);
+    TypePtr type(new TypeClass());
+    PrimitiveArray::Init(type, length, data, null_count, nulls);
   }
 
   bool Equals(const PrimitiveArrayImpl& other) const {
@@ -101,7 +105,7 @@ class PrimitiveArrayImpl : public PrimitiveArray {
 
   const T* raw_data() const { return reinterpret_cast<const T*>(raw_data_);}
 
-  T Value(int64_t i) const {
+  T Value(int i) const {
     return raw_data()[i];
   }
 
@@ -124,7 +128,7 @@ class PrimitiveBuilder : public ArrayBuilder {
 
   virtual ~PrimitiveBuilder() {}
 
-  Status Resize(int64_t capacity) {
+  Status Resize(int32_t capacity) {
     // XXX: Set floor size for now
     if (capacity < MIN_BUILDER_CAPACITY) {
       capacity = MIN_BUILDER_CAPACITY;
@@ -135,27 +139,26 @@ class PrimitiveBuilder : public ArrayBuilder {
     } else {
       RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
       RETURN_NOT_OK(values_->Resize(capacity * elsize_));
-      capacity_ = capacity;
     }
+    capacity_ = capacity;
     return Status::OK();
   }
 
-  Status Init(int64_t capacity) {
+  Status Init(int32_t capacity) {
     RETURN_NOT_OK(ArrayBuilder::Init(capacity));
-
     values_ = std::make_shared<PoolBuffer>(pool_);
     return values_->Resize(capacity * elsize_);
   }
 
-  Status Reserve(int64_t elements) {
+  Status Reserve(int32_t elements) {
     if (length_ + elements > capacity_) {
-      int64_t new_capacity = util::next_power2(length_ + elements);
+      int32_t new_capacity = util::next_power2(length_ + elements);
       return Resize(new_capacity);
     }
     return Status::OK();
   }
 
-  Status Advance(int64_t elements) {
+  Status Advance(int32_t elements) {
     return ArrayBuilder::Advance(elements);
   }
 
@@ -165,8 +168,9 @@ class PrimitiveBuilder : public ArrayBuilder {
       // If the capacity was not already a multiple of 2, do so here
       RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
     }
-    if (nullable_) {
-      util::set_bit(null_bits_, length_, is_null);
+    if (is_null) {
+      ++null_count_;
+      util::set_bit(null_bits_, length_);
     }
     raw_buffer()[length_++] = val;
     return Status::OK();
@@ -176,42 +180,49 @@ class PrimitiveBuilder : public ArrayBuilder {
   //
   // If passed, null_bytes is of equal length to values, and any nonzero byte
   // will be considered as a null for that slot
-  Status Append(const T* values, int64_t length, uint8_t* null_bytes = nullptr) {
+  Status Append(const T* values, int32_t length,
+      const uint8_t* null_bytes = nullptr) {
     if (length_ + length > capacity_) {
-      int64_t new_capacity = util::next_power2(length_ + length);
+      int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
     }
     memcpy(raw_buffer() + length_, values, length * elsize_);
 
-    if (nullable_ && null_bytes != nullptr) {
-      // If null_bytes is all not null, then none of the values are null
-      for (int64_t i = 0; i < length; ++i) {
-        util::set_bit(null_bits_, length_ + i, static_cast<bool>(null_bytes[i]));
-      }
+    if (null_bytes != nullptr) {
+      AppendNulls(null_bytes, length);
     }
 
     length_ += length;
     return Status::OK();
   }
 
-  Status AppendNull() {
-    if (!nullable_) {
-      return Status::Invalid("not nullable");
+  // Write nulls as uint8_t* into pre-allocated memory
+  void AppendNulls(const uint8_t* null_bytes, int32_t length) {
+    // If null_bytes is all not null, then none of the values are null
+    for (int i = 0; i < length; ++i) {
+      if (static_cast<bool>(null_bytes[i])) {
+        ++null_count_;
+        util::set_bit(null_bits_, length_ + i);
+      }
     }
+  }
+
+  Status AppendNull() {
     if (length_ == capacity_) {
       // If the capacity was not already a multiple of 2, do so here
       RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
     }
-    util::set_bit(null_bits_, length_++, true);
+    ++null_count_;
+    util::set_bit(null_bits_, length_++);
     return Status::OK();
   }
 
   // Initialize an array type instance with the results of this builder
   // Transfers ownership of all buffers
   Status Transfer(PrimitiveArray* out) {
-    out->Init(type_, length_, values_, nulls_);
+    out->Init(type_, length_, values_, null_count_, nulls_);
     values_ = nulls_ = nullptr;
-    capacity_ = length_ = 0;
+    capacity_ = length_ = null_count_ = 0;
     return Status::OK();
   }
 
@@ -236,7 +247,7 @@ class PrimitiveBuilder : public ArrayBuilder {
 
  protected:
   std::shared_ptr<PoolBuffer> values_;
-  int64_t elsize_;
+  int elsize_;
 };
 
 } // namespace arrow
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index a2d87ead59c59..e1dcebe97f013 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -39,7 +39,6 @@ TEST(TypesTest, TestCharType) {
   CharType t1(5);
 
   ASSERT_EQ(t1.type, TypeEnum::CHAR);
-  ASSERT_TRUE(t1.nullable);
   ASSERT_EQ(t1.size, 5);
 
   ASSERT_EQ(t1.ToString(), std::string("char(5)"));
@@ -47,7 +46,6 @@ TEST(TypesTest, TestCharType) {
   // Test copy constructor
   CharType t2 = t1;
   ASSERT_EQ(t2.type, TypeEnum::CHAR);
-  ASSERT_TRUE(t2.nullable);
   ASSERT_EQ(t2.size, 5);
 }
 
@@ -56,7 +54,6 @@ TEST(TypesTest, TestVarcharType) {
   VarcharType t1(5);
 
   ASSERT_EQ(t1.type, TypeEnum::VARCHAR);
-  ASSERT_TRUE(t1.nullable);
   ASSERT_EQ(t1.size, 5);
   ASSERT_EQ(t1.physical_type.size, 6);
 
@@ -65,19 +62,14 @@ TEST(TypesTest, TestVarcharType) {
   // Test copy constructor
   VarcharType t2 = t1;
   ASSERT_EQ(t2.type, TypeEnum::VARCHAR);
-  ASSERT_TRUE(t2.nullable);
   ASSERT_EQ(t2.size, 5);
   ASSERT_EQ(t2.physical_type.size, 6);
 }
 
 TEST(TypesTest, TestStringType) {
   StringType str;
-  StringType str_nn(false);
-
   ASSERT_EQ(str.type, TypeEnum::STRING);
   ASSERT_EQ(str.name(), std::string("string"));
-  ASSERT_TRUE(str.nullable);
-  ASSERT_FALSE(str_nn.nullable);
 }
 
 // ----------------------------------------------------------------------
@@ -96,7 +88,7 @@ class TestStringContainer : public ::testing::Test  {
 
   void MakeArray() {
     length_ = offsets_.size() - 1;
-    int64_t nchars = chars_.size();
+    int nchars = chars_.size();
 
     value_buf_ = to_buffer(chars_);
     values_ = ArrayPtr(new UInt8Array(nchars, value_buf_));
@@ -104,7 +96,9 @@ class TestStringContainer : public ::testing::Test  {
     offsets_buf_ = to_buffer(offsets_);
 
     nulls_buf_ = bytes_to_null_buffer(nulls_.data(), nulls_.size());
-    strings_.Init(length_, offsets_buf_, values_, nulls_buf_);
+    null_count_ = null_count(nulls_);
+
+    strings_.Init(length_, offsets_buf_, values_, null_count_, nulls_buf_);
   }
 
  protected:
@@ -118,7 +112,8 @@ class TestStringContainer : public ::testing::Test  {
   std::shared_ptr<Buffer> offsets_buf_;
   std::shared_ptr<Buffer> nulls_buf_;
 
-  int64_t length_;
+  int null_count_;
+  int length_;
 
   ArrayPtr values_;
   StringArray strings_;
@@ -127,7 +122,7 @@ class TestStringContainer : public ::testing::Test  {
 
 TEST_F(TestStringContainer, TestArrayBasics) {
   ASSERT_EQ(length_, strings_.length());
-  ASSERT_TRUE(strings_.nullable());
+  ASSERT_EQ(1, strings_.null_count());
 }
 
 TEST_F(TestStringContainer, TestType) {
@@ -149,7 +144,8 @@ TEST_F(TestStringContainer, TestListFunctions) {
 
 
 TEST_F(TestStringContainer, TestDestructor) {
-  auto arr = std::make_shared<StringArray>(length_, offsets_buf_, values_, nulls_buf_);
+  auto arr = std::make_shared<StringArray>(length_, offsets_buf_, values_,
+      null_count_, nulls_buf_);
 }
 
 TEST_F(TestStringContainer, TestGetString) {
@@ -189,10 +185,6 @@ class TestStringBuilder : public TestBuilder {
   std::unique_ptr<StringArray> result_;
 };
 
-TEST_F(TestStringBuilder, TestAttrs) {
-  ASSERT_FALSE(builder_->value_builder()->nullable());
-}
-
 TEST_F(TestStringBuilder, TestScalarAppend) {
   std::vector<std::string> strings = {"a", "bb", "", "", "ccc"};
   std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
@@ -212,10 +204,11 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
   Done();
 
   ASSERT_EQ(reps * N, result_->length());
+  ASSERT_EQ(reps * null_count(is_null), result_->null_count());
   ASSERT_EQ(reps * 6, result_->values()->length());
 
-  int64_t length;
-  int64_t pos = 0;
+  int32_t length;
+  int32_t pos = 0;
   for (int i = 0; i < N * reps; ++i) {
     if (is_null[i % N]) {
       ASSERT_TRUE(result_->IsNull(i));
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index f3dfbdc50f7a4..dea42e102b0d0 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -35,6 +35,6 @@ std::string VarcharType::ToString() const {
   return s.str();
 }
 
-TypePtr StringBuilder::value_type_ = TypePtr(new UInt8Type(false));
+TypePtr StringBuilder::value_type_ = TypePtr(new UInt8Type());
 
 } // namespace arrow
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index d0690d9a7d2a4..084562530a8fc 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -40,13 +40,13 @@ struct CharType : public DataType {
 
   BytesType physical_type;
 
-  explicit CharType(int size, bool nullable = true)
-      : DataType(TypeEnum::CHAR, nullable),
+  explicit CharType(int size)
+      : DataType(TypeEnum::CHAR),
         size(size),
         physical_type(BytesType(size)) {}
 
   CharType(const CharType& other)
-      : CharType(other.size, other.nullable) {}
+      : CharType(other.size) {}
 
   virtual std::string ToString() const;
 };
@@ -58,12 +58,12 @@ struct VarcharType : public DataType {
 
   BytesType physical_type;
 
-  explicit VarcharType(int size, bool nullable = true)
-      : DataType(TypeEnum::VARCHAR, nullable),
+  explicit VarcharType(int size)
+      : DataType(TypeEnum::VARCHAR),
         size(size),
         physical_type(BytesType(size + 1)) {}
   VarcharType(const VarcharType& other)
-      : VarcharType(other.size, other.nullable) {}
+      : VarcharType(other.size) {}
 
   virtual std::string ToString() const;
 };
@@ -73,11 +73,11 @@ static const LayoutPtr physical_string = LayoutPtr(new ListLayoutType(byte1));
 
 // String is a logical type consisting of a physical list of 1-byte values
 struct StringType : public DataType {
-  explicit StringType(bool nullable = true)
-      : DataType(TypeEnum::STRING, nullable) {}
+  StringType()
+      : DataType(TypeEnum::STRING) {}
 
   StringType(const StringType& other)
-      : StringType(other.nullable) {}
+      : StringType() {}
 
   const LayoutPtr& physical_type() {
     return physical_string;
@@ -98,17 +98,19 @@ class StringArray : public ListArray {
  public:
   StringArray() : ListArray(), bytes_(nullptr), raw_bytes_(nullptr) {}
 
-  StringArray(int64_t length, const std::shared_ptr<Buffer>& offsets,
+  StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values,
+      int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr) {
-    Init(length, offsets, values, nulls);
+    Init(length, offsets, values, null_count, nulls);
   }
 
-  void Init(const TypePtr& type, int64_t length,
+  void Init(const TypePtr& type, int32_t length,
       const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values,
+      int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr) {
-    ListArray::Init(type, length, offsets, values, nulls);
+    ListArray::Init(type, length, offsets, values, null_count, nulls);
 
     // TODO: type validation for values array
 
@@ -117,23 +119,24 @@ class StringArray : public ListArray {
     raw_bytes_ = bytes_->raw_data();
   }
 
-  void Init(int64_t length, const std::shared_ptr<Buffer>& offsets,
+  void Init(int32_t length, const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values,
+      int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr) {
-    TypePtr type(new StringType(nulls != nullptr));
-    Init(type, length, offsets, values, nulls);
+    TypePtr type(new StringType());
+    Init(type, length, offsets, values, null_count, nulls);
   }
 
   // Compute the pointer t
-  const uint8_t* GetValue(int64_t i, int64_t* out_length) const {
+  const uint8_t* GetValue(int i, int32_t* out_length) const {
     int32_t pos = offsets_[i];
     *out_length = offsets_[i + 1] - pos;
     return raw_bytes_ + pos;
   }
 
   // Construct a std::string
-  std::string GetString(int64_t i) const {
-    int64_t nchars;
+  std::string GetString(int i) const {
+    int32_t nchars;
     const uint8_t* str = GetValue(i, &nchars);
     return std::string(reinterpret_cast<const char*>(str), nchars);
   }
@@ -161,7 +164,7 @@ class StringBuilder : public ListBuilder {
         value.size());
   }
 
-  Status Append(const uint8_t* value, int64_t length);
+  Status Append(const uint8_t* value, int32_t length);
   Status Append(const std::vector<std::string>& values,
                 uint8_t* null_bytes);
 
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index 644b5457d5851..1a9fc6be4a5ce 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -43,11 +43,7 @@ TEST(TestStructType, Basics) {
 
   vector<Field> fields = {f0, f1, f2};
 
-  StructType struct_type(fields, true);
-  StructType struct_type_nn(fields, false);
-
-  ASSERT_TRUE(struct_type.nullable);
-  ASSERT_FALSE(struct_type_nn.nullable);
+  StructType struct_type(fields);
 
   ASSERT_TRUE(struct_type.field(0).Equals(f0));
   ASSERT_TRUE(struct_type.field(1).Equals(f1));
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index 7d8885b830dba..afba19a7e4699 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -29,9 +29,8 @@ namespace arrow {
 struct StructType : public DataType {
   std::vector<Field> fields_;
 
-  StructType(const std::vector<Field>& fields,
-      bool nullable = true)
-      : DataType(TypeEnum::STRUCT, nullable) {
+  explicit StructType(const std::vector<Field>& fields)
+      : DataType(TypeEnum::STRUCT) {
     fields_ = fields;
   }
 
diff --git a/cpp/src/arrow/types/test-common.h b/cpp/src/arrow/types/test-common.h
index 3ecb0dec7c04a..1744efce7d631 100644
--- a/cpp/src/arrow/types/test-common.h
+++ b/cpp/src/arrow/types/test-common.h
@@ -36,15 +36,13 @@ class TestBuilder : public ::testing::Test {
   void SetUp() {
     pool_ = GetDefaultMemoryPool();
     type_ = TypePtr(new UInt8Type());
-    type_nn_ = TypePtr(new UInt8Type(false));
     builder_.reset(new UInt8Builder(pool_, type_));
-    builder_nn_.reset(new UInt8Builder(pool_, type_nn_));
+    builder_nn_.reset(new UInt8Builder(pool_, type_));
   }
  protected:
   MemoryPool* pool_;
 
   TypePtr type_;
-  TypePtr type_nn_;
   unique_ptr<ArrayBuilder> builder_;
   unique_ptr<ArrayBuilder> builder_nn_;
 };
diff --git a/cpp/src/arrow/types/union.h b/cpp/src/arrow/types/union.h
index 7b66c3b88bf3c..62a3d1c10355d 100644
--- a/cpp/src/arrow/types/union.h
+++ b/cpp/src/arrow/types/union.h
@@ -33,9 +33,8 @@ class Buffer;
 struct DenseUnionType : public CollectionType<TypeEnum::DENSE_UNION> {
   typedef CollectionType<TypeEnum::DENSE_UNION> Base;
 
-  DenseUnionType(const std::vector<TypePtr>& child_types,
-      bool nullable = true)
-      : Base(nullable) {
+  explicit DenseUnionType(const std::vector<TypePtr>& child_types) :
+      Base() {
     child_types_ = child_types;
   }
 
@@ -46,9 +45,8 @@ struct DenseUnionType : public CollectionType<TypeEnum::DENSE_UNION> {
 struct SparseUnionType : public CollectionType<TypeEnum::SPARSE_UNION> {
   typedef CollectionType<TypeEnum::SPARSE_UNION> Base;
 
-  SparseUnionType(const std::vector<TypePtr>& child_types,
-      bool nullable = true)
-      : Base(nullable) {
+  explicit SparseUnionType(const std::vector<TypePtr>& child_types) :
+      Base() {
     child_types_ = child_types;
   }
 
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index dbac0a42527be..292cb33887ffa 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -25,7 +25,9 @@ namespace arrow {
 
 void util::bytes_to_bits(uint8_t* bytes, int length, uint8_t* bits) {
   for (int i = 0; i < length; ++i) {
-    set_bit(bits, i, static_cast<bool>(bytes[i]));
+    if (static_cast<bool>(bytes[i])) {
+      set_bit(bits, i);
+    }
   }
 }
 
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 9ae6127c5ea9c..841f617a3139c 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -41,8 +41,8 @@ static inline bool get_bit(const uint8_t* bits, int i) {
   return bits[i / 8] & (1 << (i % 8));
 }
 
-static inline void set_bit(uint8_t* bits, int i, bool is_set) {
-  bits[i / 8] |= (1 << (i % 8)) * is_set;
+static inline void set_bit(uint8_t* bits, int i) {
+  bits[i / 8] |= 1 << (i % 8);
 }
 
 static inline int64_t next_power2(int64_t n) {

From 89c6afd2026cab21fbe2b3c81f14335dffde6d08 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 3 Mar 2016 15:35:54 -0800
Subject: [PATCH 0014/1644] ARROW-21: Implement a simple in-memory Schema data
 structure

I also have restored the `nullable` bit to the type metadata only (for the moment mainly to facilitate schema testing / round-trips to Parquet and other media with required/optional distinction) and done some miscellaneous refactoring (`TypeEnum` is renamed to `LogicalType`).

Author: Wes McKinney <wesm@apache.org>

Closes #10 from wesm/ARROW-21 and squashes the following commits:

c770f7d [Wes McKinney] Add simple in-memory Schema data structure. Restore nullable bit to type metadata only. Add "?" to nullable type formatting.
---
 cpp/CMakeLists.txt                        |   2 +
 cpp/src/arrow/CMakeLists.txt              |   2 +-
 cpp/src/arrow/array.h                     |   4 +-
 cpp/src/arrow/{field-test.cc => field.cc} |  19 +--
 cpp/src/arrow/field.h                     |  17 +-
 cpp/src/arrow/schema-test.cc              | 110 ++++++++++++
 cpp/src/arrow/schema.cc                   |  58 +++++++
 cpp/src/arrow/schema.h                    |  56 +++++++
 cpp/src/arrow/type.h                      | 193 +++++++++++++++-------
 cpp/src/arrow/types/binary.h              |   3 -
 cpp/src/arrow/types/boolean.h             |   4 -
 cpp/src/arrow/types/collection.h          |   2 +-
 cpp/src/arrow/types/construct.cc          |   4 +-
 cpp/src/arrow/types/datetime.h            |   8 +-
 cpp/src/arrow/types/floating.h            |   9 +-
 cpp/src/arrow/types/integer.h             |  33 +---
 cpp/src/arrow/types/json.h                |   4 +-
 cpp/src/arrow/types/list-test.cc          |  10 +-
 cpp/src/arrow/types/list.cc               |   3 +
 cpp/src/arrow/types/list.h                |   5 +-
 cpp/src/arrow/types/primitive-test.cc     |   4 +-
 cpp/src/arrow/types/primitive.h           |  22 ---
 cpp/src/arrow/types/string-test.cc        |  14 +-
 cpp/src/arrow/types/string.h              |  24 +--
 cpp/src/arrow/types/struct-test.cc        |   2 +-
 cpp/src/arrow/types/struct.cc             |   1 +
 cpp/src/arrow/types/struct.h              |   4 +-
 cpp/src/arrow/types/union.h               |   8 +-
 28 files changed, 434 insertions(+), 191 deletions(-)
 rename cpp/src/arrow/{field-test.cc => field.cc} (74%)
 create mode 100644 cpp/src/arrow/schema-test.cc
 create mode 100644 cpp/src/arrow/schema.cc
 create mode 100644 cpp/src/arrow/schema.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f0eb73dc41371..5e4c204581369 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -467,6 +467,8 @@ set(LINK_LIBS
 set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/builder.cc
+  src/arrow/field.cc
+  src/arrow/schema.cc
   src/arrow/type.cc
 )
 
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index eeea2dbc517b4..04f8dd1f908cb 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -30,4 +30,4 @@ install(FILES
 set(ARROW_TEST_LINK_LIBS arrow_test_util ${ARROW_MIN_TEST_LIBS})
 
 ADD_ARROW_TEST(array-test)
-ADD_ARROW_TEST(field-test)
+ADD_ARROW_TEST(schema-test)
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 3d748c1bad6f8..0632146637e59 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -62,8 +62,8 @@ class Array {
   int32_t length() const { return length_;}
   int32_t null_count() const { return null_count_;}
 
-  const TypePtr& type() const { return type_;}
-  TypeEnum type_enum() const { return type_->type;}
+  const std::shared_ptr<DataType>& type() const { return type_;}
+  LogicalType::type logical_type() const { return type_->type;}
 
   const std::shared_ptr<Buffer>& nulls() const {
     return nulls_;
diff --git a/cpp/src/arrow/field-test.cc b/cpp/src/arrow/field.cc
similarity index 74%
rename from cpp/src/arrow/field-test.cc
rename to cpp/src/arrow/field.cc
index 2bb8bad4054c3..4568d905c2991 100644
--- a/cpp/src/arrow/field-test.cc
+++ b/cpp/src/arrow/field.cc
@@ -15,24 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <memory>
-#include <string>
-
 #include "arrow/field.h"
-#include "arrow/type.h"
-#include "arrow/types/integer.h"
 
-using std::string;
+#include <sstream>
+#include <string>
 
 namespace arrow {
 
-TEST(TestField, Basics) {
-  TypePtr ftype = TypePtr(new Int32Type());
-  Field f0("f0", ftype);
-
-  ASSERT_EQ(f0.name, "f0");
-  ASSERT_EQ(f0.type->ToString(), ftype->ToString());
+std::string Field::ToString() const {
+  std::stringstream ss;
+  ss << this->name << " " << this->type->ToString();
+  return ss.str();
 }
 
 } // namespace arrow
diff --git a/cpp/src/arrow/field.h b/cpp/src/arrow/field.h
index 664cae61a777a..89a450c66f256 100644
--- a/cpp/src/arrow/field.h
+++ b/cpp/src/arrow/field.h
@@ -35,12 +35,27 @@ struct Field {
   TypePtr type;
 
   Field(const std::string& name, const TypePtr& type) :
-      name(name), type(type) {}
+      name(name),
+      type(type) {}
+
+  bool operator==(const Field& other) const {
+    return this->Equals(other);
+  }
+
+  bool operator!=(const Field& other) const {
+    return !this->Equals(other);
+  }
 
   bool Equals(const Field& other) const {
     return (this == &other) || (this->name == other.name &&
         this->type->Equals(other.type.get()));
   }
+
+  bool nullable() const {
+    return this->type->nullable;
+  }
+
+  std::string ToString() const;
 };
 
 } // namespace arrow
diff --git a/cpp/src/arrow/schema-test.cc b/cpp/src/arrow/schema-test.cc
new file mode 100644
index 0000000000000..3debb9cec3c00
--- /dev/null
+++ b/cpp/src/arrow/schema-test.cc
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/field.h"
+#include "arrow/schema.h"
+#include "arrow/type.h"
+#include "arrow/types/string.h"
+
+using std::shared_ptr;
+using std::vector;
+
+namespace arrow {
+
+TEST(TestField, Basics) {
+  shared_ptr<DataType> ftype = std::make_shared<Int32Type>();
+  shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
+  Field f0("f0", ftype);
+  Field f0_nn("f0", ftype_nn);
+
+  ASSERT_EQ(f0.name, "f0");
+  ASSERT_EQ(f0.type->ToString(), ftype->ToString());
+
+  ASSERT_TRUE(f0.nullable());
+  ASSERT_FALSE(f0_nn.nullable());
+}
+
+TEST(TestField, Equals) {
+  shared_ptr<DataType> ftype = std::make_shared<Int32Type>();
+  shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
+
+  Field f0("f0", ftype);
+  Field f0_nn("f0", ftype_nn);
+  Field f0_other("f0", ftype);
+
+  ASSERT_EQ(f0, f0_other);
+  ASSERT_NE(f0, f0_nn);
+}
+
+class TestSchema : public ::testing::Test {
+ public:
+  void SetUp() {}
+};
+
+TEST_F(TestSchema, Basics) {
+  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int32Type>());
+
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(false));
+  auto f1_optional = std::make_shared<Field>("f1", std::make_shared<UInt8Type>());
+
+  auto f2 = std::make_shared<Field>("f2", std::make_shared<StringType>());
+
+  vector<shared_ptr<Field> > fields = {f0, f1, f2};
+  auto schema = std::make_shared<Schema>(fields);
+
+  ASSERT_EQ(3, schema->num_fields());
+  ASSERT_EQ(f0, schema->field(0));
+  ASSERT_EQ(f1, schema->field(1));
+  ASSERT_EQ(f2, schema->field(2));
+
+  auto schema2 = std::make_shared<Schema>(fields);
+
+  vector<shared_ptr<Field> > fields3 = {f0, f1_optional, f2};
+  auto schema3 = std::make_shared<Schema>(fields3);
+  ASSERT_TRUE(schema->Equals(schema2));
+  ASSERT_FALSE(schema->Equals(schema3));
+
+  ASSERT_TRUE(schema->Equals(*schema2.get()));
+  ASSERT_FALSE(schema->Equals(*schema3.get()));
+}
+
+TEST_F(TestSchema, ToString) {
+  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int32Type>());
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(false));
+  auto f2 = std::make_shared<Field>("f2", std::make_shared<StringType>());
+  auto f3 = std::make_shared<Field>("f3",
+      std::make_shared<ListType>(std::make_shared<Int16Type>()));
+
+  vector<shared_ptr<Field> > fields = {f0, f1, f2, f3};
+  auto schema = std::make_shared<Schema>(fields);
+
+  std::string result = schema->ToString();
+  std::string expected = R"(f0 ?int32
+f1 uint8
+f2 ?string
+f3 ?list<?int16>
+)";
+
+  ASSERT_EQ(expected, result);
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/schema.cc b/cpp/src/arrow/schema.cc
new file mode 100644
index 0000000000000..a735fd3d23075
--- /dev/null
+++ b/cpp/src/arrow/schema.cc
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/schema.h"
+
+#include <memory>
+#include <string>
+#include <sstream>
+#include <vector>
+
+#include "arrow/field.h"
+
+namespace arrow {
+
+Schema::Schema(const std::vector<std::shared_ptr<Field> >& fields) :
+    fields_(fields) {}
+
+bool Schema::Equals(const Schema& other) const {
+  if (this == &other) return true;
+  if (num_fields() != other.num_fields()) {
+    return false;
+  }
+  for (int i = 0; i < num_fields(); ++i) {
+    if (!field(i)->Equals(*other.field(i).get())) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool Schema::Equals(const std::shared_ptr<Schema>& other) const {
+  return Equals(*other.get());
+}
+
+std::string Schema::ToString() const {
+  std::stringstream buffer;
+
+  for (auto field : fields_) {
+    buffer << field->ToString() << std::endl;
+  }
+  return buffer.str();
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/schema.h b/cpp/src/arrow/schema.h
new file mode 100644
index 0000000000000..d04e3f628c1e3
--- /dev/null
+++ b/cpp/src/arrow/schema.h
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_SCHEMA_H
+#define ARROW_SCHEMA_H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/field.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+class Schema {
+ public:
+  explicit Schema(const std::vector<std::shared_ptr<Field> >& fields);
+
+  // Returns true if all of the schema fields are equal
+  bool Equals(const Schema& other) const;
+  bool Equals(const std::shared_ptr<Schema>& other) const;
+
+  // Return the ith schema element. Does not boundscheck
+  const std::shared_ptr<Field>& field(int i) const {
+    return fields_[i];
+  }
+
+  // Render a string representation of the schema suitable for debugging
+  std::string ToString() const;
+
+  int num_fields() const {
+    return fields_.size();
+  }
+
+ private:
+  std::vector<std::shared_ptr<Field> > fields_;
+};
+
+} // namespace arrow
+
+#endif  // ARROW_FIELD_H
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 12f19604c688d..04cdb52b535db 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -52,96 +52,98 @@ struct LayoutType {
   explicit LayoutType(LayoutEnum type) : type(type) {}
 };
 
-
 // Data types in this library are all *logical*. They can be expressed as
 // either a primitive physical type (bytes or bits of some fixed size), a
 // nested type consisting of other data types, or another data type (e.g. a
 // timestamp encoded as an int64)
+struct LogicalType {
+  enum type {
+    // A degenerate NULL type represented as 0 bytes/bits
+    NA = 0,
 
-enum class TypeEnum: char {
-  // A degenerate NULL type represented as 0 bytes/bits
-  NA = 0,
-
-  // Little-endian integer types
-  UINT8 = 1,
-  INT8 = 2,
-  UINT16 = 3,
-  INT16 = 4,
-  UINT32 = 5,
-  INT32 = 6,
-  UINT64 = 7,
-  INT64 = 8,
+    // Little-endian integer types
+    UINT8 = 1,
+    INT8 = 2,
+    UINT16 = 3,
+    INT16 = 4,
+    UINT32 = 5,
+    INT32 = 6,
+    UINT64 = 7,
+    INT64 = 8,
 
-  // A boolean value represented as 1 byte
-  BOOL = 9,
+    // A boolean value represented as 1 byte
+    BOOL = 9,
 
-  // A boolean value represented as 1 bit
-  BIT = 10,
+    // A boolean value represented as 1 bit
+    BIT = 10,
 
-  // 4-byte floating point value
-  FLOAT = 11,
+    // 4-byte floating point value
+    FLOAT = 11,
 
-  // 8-byte floating point value
-  DOUBLE = 12,
+    // 8-byte floating point value
+    DOUBLE = 12,
 
-  // CHAR(N): fixed-length UTF8 string with length N
-  CHAR = 13,
+    // CHAR(N): fixed-length UTF8 string with length N
+    CHAR = 13,
 
-  // UTF8 variable-length string as List<Char>
-  STRING = 14,
+    // UTF8 variable-length string as List<Char>
+    STRING = 14,
 
-  // VARCHAR(N): Null-terminated string type embedded in a CHAR(N + 1)
-  VARCHAR = 15,
+    // VARCHAR(N): Null-terminated string type embedded in a CHAR(N + 1)
+    VARCHAR = 15,
 
-  // Variable-length bytes (no guarantee of UTF8-ness)
-  BINARY = 16,
+    // Variable-length bytes (no guarantee of UTF8-ness)
+    BINARY = 16,
 
-  // By default, int32 days since the UNIX epoch
-  DATE = 17,
+    // By default, int32 days since the UNIX epoch
+    DATE = 17,
 
-  // Exact timestamp encoded with int64 since UNIX epoch
-  // Default unit millisecond
-  TIMESTAMP = 18,
+    // Exact timestamp encoded with int64 since UNIX epoch
+    // Default unit millisecond
+    TIMESTAMP = 18,
 
-  // Timestamp as double seconds since the UNIX epoch
-  TIMESTAMP_DOUBLE = 19,
+    // Timestamp as double seconds since the UNIX epoch
+    TIMESTAMP_DOUBLE = 19,
 
-  // Exact time encoded with int64, default unit millisecond
-  TIME = 20,
+    // Exact time encoded with int64, default unit millisecond
+    TIME = 20,
 
-  // Precision- and scale-based decimal type. Storage type depends on the
-  // parameters.
-  DECIMAL = 21,
+    // Precision- and scale-based decimal type. Storage type depends on the
+    // parameters.
+    DECIMAL = 21,
 
-  // Decimal value encoded as a text string
-  DECIMAL_TEXT = 22,
+    // Decimal value encoded as a text string
+    DECIMAL_TEXT = 22,
 
-  // A list of some logical data type
-  LIST = 30,
+    // A list of some logical data type
+    LIST = 30,
 
-  // Struct of logical types
-  STRUCT = 31,
+    // Struct of logical types
+    STRUCT = 31,
 
-  // Unions of logical types
-  DENSE_UNION = 32,
-  SPARSE_UNION = 33,
+    // Unions of logical types
+    DENSE_UNION = 32,
+    SPARSE_UNION = 33,
 
-  // Union<Null, Int32, Double, String, Bool>
-  JSON_SCALAR = 50,
+    // Union<Null, Int32, Double, String, Bool>
+    JSON_SCALAR = 50,
 
-  // User-defined type
-  USER = 60
+    // User-defined type
+    USER = 60
+  };
 };
 
-
 struct DataType {
-  TypeEnum type;
+  LogicalType::type type;
+  bool nullable;
 
-  explicit DataType(TypeEnum type)
-      : type(type) {}
+  explicit DataType(LogicalType::type type, bool nullable = true) :
+      type(type),
+      nullable(nullable) {}
 
   virtual bool Equals(const DataType* other) {
-    return this == other || this->type == other->type;
+    return this == other || (this->type == other->type &&
+        this->nullable == other->nullable);
   }
 
   virtual std::string ToString() const = 0;
@@ -171,6 +173,77 @@ struct ListLayoutType : public LayoutType {
         value_type(value_type) {}
 };
 
+template <typename Derived>
+struct PrimitiveType : public DataType {
+  explicit PrimitiveType(bool nullable = true)
+      : DataType(Derived::type_enum, nullable) {}
+
+  virtual std::string ToString() const {
+    std::string result;
+    if (nullable) {
+      result.append("?");
+    }
+    result.append(static_cast<const Derived*>(this)->name());
+    return result;
+  }
+};
+
+#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)          \
+  typedef C_TYPE c_type;                                            \
+  static constexpr LogicalType::type type_enum = LogicalType::ENUM; \
+  static constexpr int size = SIZE;                                 \
+                                                                    \
+  explicit TYPENAME(bool nullable = true)                           \
+      : PrimitiveType<TYPENAME>(nullable) {}                        \
+                                                                    \
+  static const char* name() {                                       \
+    return NAME;                                                    \
+  }
+
+struct BooleanType : public PrimitiveType<BooleanType> {
+  PRIMITIVE_DECL(BooleanType, uint8_t, BOOL, 1, "bool");
+};
+
+struct UInt8Type : public PrimitiveType<UInt8Type> {
+  PRIMITIVE_DECL(UInt8Type, uint8_t, UINT8, 1, "uint8");
+};
+
+struct Int8Type : public PrimitiveType<Int8Type> {
+  PRIMITIVE_DECL(Int8Type, int8_t, INT8, 1, "int8");
+};
+
+struct UInt16Type : public PrimitiveType<UInt16Type> {
+  PRIMITIVE_DECL(UInt16Type, uint16_t, UINT16, 2, "uint16");
+};
+
+struct Int16Type : public PrimitiveType<Int16Type> {
+  PRIMITIVE_DECL(Int16Type, int16_t, INT16, 2, "int16");
+};
+
+struct UInt32Type : public PrimitiveType<UInt32Type> {
+  PRIMITIVE_DECL(UInt32Type, uint32_t, UINT32, 4, "uint32");
+};
+
+struct Int32Type : public PrimitiveType<Int32Type> {
+  PRIMITIVE_DECL(Int32Type, int32_t, INT32, 4, "int32");
+};
+
+struct UInt64Type : public PrimitiveType<UInt64Type> {
+  PRIMITIVE_DECL(UInt64Type, uint64_t, UINT64, 8, "uint64");
+};
+
+struct Int64Type : public PrimitiveType<Int64Type> {
+  PRIMITIVE_DECL(Int64Type, int64_t, INT64, 8, "int64");
+};
+
+struct FloatType : public PrimitiveType<FloatType> {
+  PRIMITIVE_DECL(FloatType, float, FLOAT, 4, "float");
+};
+
+struct DoubleType : public PrimitiveType<DoubleType> {
+  PRIMITIVE_DECL(DoubleType, double, DOUBLE, 8, "double");
+};
+
 } // namespace arrow
 
 #endif  // ARROW_TYPE_H
diff --git a/cpp/src/arrow/types/binary.h b/cpp/src/arrow/types/binary.h
index a9f20046b582b..1fd675e5fdebf 100644
--- a/cpp/src/arrow/types/binary.h
+++ b/cpp/src/arrow/types/binary.h
@@ -25,9 +25,6 @@
 
 namespace arrow {
 
-struct StringType : public DataType {
-};
-
 } // namespace arrow
 
 #endif // ARROW_TYPES_BINARY_H
diff --git a/cpp/src/arrow/types/boolean.h b/cpp/src/arrow/types/boolean.h
index 31388c8152d52..8fc9cfd19c0d4 100644
--- a/cpp/src/arrow/types/boolean.h
+++ b/cpp/src/arrow/types/boolean.h
@@ -22,10 +22,6 @@
 
 namespace arrow {
 
-struct BooleanType : public PrimitiveType<BooleanType> {
-  PRIMITIVE_DECL(BooleanType, uint8_t, BOOL, 1, "bool");
-};
-
 typedef PrimitiveArrayImpl<BooleanType> BooleanArray;
 
 // typedef PrimitiveBuilder<BooleanType, BooleanArray> BooleanBuilder;
diff --git a/cpp/src/arrow/types/collection.h b/cpp/src/arrow/types/collection.h
index 094b63f28988a..42a9c926bb134 100644
--- a/cpp/src/arrow/types/collection.h
+++ b/cpp/src/arrow/types/collection.h
@@ -25,7 +25,7 @@
 
 namespace arrow {
 
-template <TypeEnum T>
+template <LogicalType::type T>
 struct CollectionType : public DataType {
   std::vector<TypePtr> child_types_;
 
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index e1bb990063c1b..05d6b270fc3fd 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -33,7 +33,7 @@ class ArrayBuilder;
 // difficult
 
 #define BUILDER_CASE(ENUM, BuilderType)                                 \
-    case TypeEnum::ENUM:                                                \
+    case LogicalType::ENUM:                                             \
       *out = static_cast<ArrayBuilder*>(new BuilderType(pool, type));   \
       return Status::OK();
 
@@ -56,7 +56,7 @@ Status make_builder(MemoryPool* pool, const TypePtr& type,
 
     BUILDER_CASE(STRING, StringBuilder);
 
-    case TypeEnum::LIST:
+    case LogicalType::LIST:
       {
         ListType* list_type = static_cast<ListType*>(type.get());
         ArrayBuilder* value_builder;
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
index d90883cb01871..765fc29dd57ae 100644
--- a/cpp/src/arrow/types/datetime.h
+++ b/cpp/src/arrow/types/datetime.h
@@ -31,8 +31,8 @@ struct DateType : public DataType {
 
   Unit unit;
 
-  explicit DateType(Unit unit = Unit::DAY)
-      : DataType(TypeEnum::DATE),
+  explicit DateType(Unit unit = Unit::DAY, bool nullable = true)
+      : DataType(LogicalType::DATE, nullable),
         unit(unit) {}
 
   DateType(const DateType& other)
@@ -58,8 +58,8 @@ struct TimestampType : public DataType {
 
   Unit unit;
 
-  explicit TimestampType(Unit unit = Unit::MILLI)
-      : DataType(TypeEnum::TIMESTAMP),
+  explicit TimestampType(Unit unit = Unit::MILLI, bool nullable = true)
+      : DataType(LogicalType::TIMESTAMP, nullable),
         unit(unit) {}
 
   TimestampType(const TimestampType& other)
diff --git a/cpp/src/arrow/types/floating.h b/cpp/src/arrow/types/floating.h
index 7551ce665a27b..e7522781d33e3 100644
--- a/cpp/src/arrow/types/floating.h
+++ b/cpp/src/arrow/types/floating.h
@@ -21,17 +21,10 @@
 #include <string>
 
 #include "arrow/types/primitive.h"
+#include "arrow/type.h"
 
 namespace arrow {
 
-struct FloatType : public PrimitiveType<FloatType> {
-  PRIMITIVE_DECL(FloatType, float, FLOAT, 4, "float");
-};
-
-struct DoubleType : public PrimitiveType<DoubleType> {
-  PRIMITIVE_DECL(DoubleType, double, DOUBLE, 8, "double");
-};
-
 typedef PrimitiveArrayImpl<FloatType> FloatArray;
 typedef PrimitiveArrayImpl<DoubleType> DoubleArray;
 
diff --git a/cpp/src/arrow/types/integer.h b/cpp/src/arrow/types/integer.h
index 7e5eab55be0a9..568419124941f 100644
--- a/cpp/src/arrow/types/integer.h
+++ b/cpp/src/arrow/types/integer.h
@@ -22,41 +22,10 @@
 #include <string>
 
 #include "arrow/types/primitive.h"
+#include "arrow/type.h"
 
 namespace arrow {
 
-struct UInt8Type : public PrimitiveType<UInt8Type> {
-  PRIMITIVE_DECL(UInt8Type, uint8_t, UINT8, 1, "uint8");
-};
-
-struct Int8Type : public PrimitiveType<Int8Type> {
-  PRIMITIVE_DECL(Int8Type, int8_t, INT8, 1, "int8");
-};
-
-struct UInt16Type : public PrimitiveType<UInt16Type> {
-  PRIMITIVE_DECL(UInt16Type, uint16_t, UINT16, 2, "uint16");
-};
-
-struct Int16Type : public PrimitiveType<Int16Type> {
-  PRIMITIVE_DECL(Int16Type, int16_t, INT16, 2, "int16");
-};
-
-struct UInt32Type : public PrimitiveType<UInt32Type> {
-  PRIMITIVE_DECL(UInt32Type, uint32_t, UINT32, 4, "uint32");
-};
-
-struct Int32Type : public PrimitiveType<Int32Type> {
-  PRIMITIVE_DECL(Int32Type, int32_t, INT32, 4, "int32");
-};
-
-struct UInt64Type : public PrimitiveType<UInt64Type> {
-  PRIMITIVE_DECL(UInt64Type, uint64_t, UINT64, 8, "uint64");
-};
-
-struct Int64Type : public PrimitiveType<Int64Type> {
-  PRIMITIVE_DECL(Int64Type, int64_t, INT64, 8, "int64");
-};
-
 // Array containers
 
 typedef PrimitiveArrayImpl<UInt8Type> UInt8Array;
diff --git a/cpp/src/arrow/types/json.h b/cpp/src/arrow/types/json.h
index 6c2b097a737c7..b67fb3807aded 100644
--- a/cpp/src/arrow/types/json.h
+++ b/cpp/src/arrow/types/json.h
@@ -28,8 +28,8 @@ struct JSONScalar : public DataType {
   static TypePtr dense_type;
   static TypePtr sparse_type;
 
-  explicit JSONScalar(bool dense = true)
-      : DataType(TypeEnum::JSON_SCALAR),
+  explicit JSONScalar(bool dense = true, bool nullable = true)
+      : DataType(LogicalType::JSON_SCALAR, nullable),
         dense(dense) {}
 };
 
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index 1d9ddbe607a41..b4bbd2841a89d 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -44,19 +44,19 @@ TEST(TypesTest, TestListType) {
   std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
 
   ListType list_type(vt);
-  ASSERT_EQ(list_type.type, TypeEnum::LIST);
+  ASSERT_EQ(list_type.type, LogicalType::LIST);
 
   ASSERT_EQ(list_type.name(), string("list"));
-  ASSERT_EQ(list_type.ToString(), string("list<uint8>"));
+  ASSERT_EQ(list_type.ToString(), string("?list<?uint8>"));
 
   ASSERT_EQ(list_type.value_type->type, vt->type);
   ASSERT_EQ(list_type.value_type->type, vt->type);
 
-  std::shared_ptr<DataType> st = std::make_shared<StringType>();
-  std::shared_ptr<DataType> lt = std::make_shared<ListType>(st);
+  std::shared_ptr<DataType> st = std::make_shared<StringType>(false);
+  std::shared_ptr<DataType> lt = std::make_shared<ListType>(st, false);
   ASSERT_EQ(lt->ToString(), string("list<string>"));
 
-  ListType lt2(lt);
+  ListType lt2(lt, false);
   ASSERT_EQ(lt2.ToString(), string("list<list<string>>"));
 }
 
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index f0ff5bf928a1a..577d71d0b2892 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -24,6 +24,9 @@ namespace arrow {
 
 std::string ListType::ToString() const {
   std::stringstream s;
+  if (this->nullable) {
+    s << "?";
+  }
   s << "list<" << value_type->ToString() << ">";
   return s.str();
 }
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 4190b53df01cd..1fc83536db8c6 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -40,8 +40,8 @@ struct ListType : public DataType {
   // List can contain any other logical value type
   TypePtr value_type;
 
-  explicit ListType(const TypePtr& value_type)
-      : DataType(TypeEnum::LIST),
+  explicit ListType(const TypePtr& value_type, bool nullable = true)
+      : DataType(LogicalType::LIST, nullable),
         value_type(value_type) {}
 
   static char const *name() {
@@ -51,7 +51,6 @@ struct ListType : public DataType {
   virtual std::string ToString() const;
 };
 
-
 class ListArray : public Array {
  public:
   ListArray() : Array(), offset_buf_(nullptr), offsets_(nullptr) {}
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 93634432d5ccb..02eaaa7542bf0 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -54,11 +54,11 @@ TEST(TypesTest, TestBytesType) {
   TEST(TypesTest, TestPrimitive_##ENUM) {       \
     KLASS tp;                                   \
                                                 \
-    ASSERT_EQ(tp.type, TypeEnum::ENUM);         \
+    ASSERT_EQ(tp.type, LogicalType::ENUM);      \
     ASSERT_EQ(tp.name(), string(NAME));         \
                                                 \
     KLASS tp_copy = tp;                         \
-    ASSERT_EQ(tp_copy.type, TypeEnum::ENUM);    \
+    ASSERT_EQ(tp_copy.type, LogicalType::ENUM); \
   }
 
 PRIMITIVE_TEST(Int8Type, INT8, "int8");
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index aa8f351202a20..49040fb66268f 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -34,28 +34,6 @@ namespace arrow {
 
 class MemoryPool;
 
-template <typename Derived>
-struct PrimitiveType : public DataType {
-  PrimitiveType()
-      : DataType(Derived::type_enum) {}
-
-  virtual std::string ToString() const {
-    return std::string(static_cast<const Derived*>(this)->name());
-  }
-};
-
-#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)  \
-  typedef C_TYPE c_type;                                    \
-  static constexpr TypeEnum type_enum = TypeEnum::ENUM;     \
-  static constexpr int size = SIZE;                         \
-                                                            \
-  TYPENAME()                                                \
-      : PrimitiveType<TYPENAME>() {}                        \
-                                                            \
-  static const char* name() {                               \
-    return NAME;                                            \
-  }
-
 
 // Base class for fixed-size logical types
 class PrimitiveArray : public Array {
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index e1dcebe97f013..9af667295026b 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -38,14 +38,14 @@ class Buffer;
 TEST(TypesTest, TestCharType) {
   CharType t1(5);
 
-  ASSERT_EQ(t1.type, TypeEnum::CHAR);
+  ASSERT_EQ(t1.type, LogicalType::CHAR);
   ASSERT_EQ(t1.size, 5);
 
   ASSERT_EQ(t1.ToString(), std::string("char(5)"));
 
   // Test copy constructor
   CharType t2 = t1;
-  ASSERT_EQ(t2.type, TypeEnum::CHAR);
+  ASSERT_EQ(t2.type, LogicalType::CHAR);
   ASSERT_EQ(t2.size, 5);
 }
 
@@ -53,7 +53,7 @@ TEST(TypesTest, TestCharType) {
 TEST(TypesTest, TestVarcharType) {
   VarcharType t1(5);
 
-  ASSERT_EQ(t1.type, TypeEnum::VARCHAR);
+  ASSERT_EQ(t1.type, LogicalType::VARCHAR);
   ASSERT_EQ(t1.size, 5);
   ASSERT_EQ(t1.physical_type.size, 6);
 
@@ -61,14 +61,14 @@ TEST(TypesTest, TestVarcharType) {
 
   // Test copy constructor
   VarcharType t2 = t1;
-  ASSERT_EQ(t2.type, TypeEnum::VARCHAR);
+  ASSERT_EQ(t2.type, LogicalType::VARCHAR);
   ASSERT_EQ(t2.size, 5);
   ASSERT_EQ(t2.physical_type.size, 6);
 }
 
 TEST(TypesTest, TestStringType) {
   StringType str;
-  ASSERT_EQ(str.type, TypeEnum::STRING);
+  ASSERT_EQ(str.type, LogicalType::STRING);
   ASSERT_EQ(str.name(), std::string("string"));
 }
 
@@ -128,8 +128,8 @@ TEST_F(TestStringContainer, TestArrayBasics) {
 TEST_F(TestStringContainer, TestType) {
   TypePtr type = strings_.type();
 
-  ASSERT_EQ(TypeEnum::STRING, type->type);
-  ASSERT_EQ(TypeEnum::STRING, strings_.type_enum());
+  ASSERT_EQ(LogicalType::STRING, type->type);
+  ASSERT_EQ(LogicalType::STRING, strings_.logical_type());
 }
 
 
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index 084562530a8fc..5795cfed577c5 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -40,8 +40,8 @@ struct CharType : public DataType {
 
   BytesType physical_type;
 
-  explicit CharType(int size)
-      : DataType(TypeEnum::CHAR),
+  explicit CharType(int size, bool nullable = true)
+      : DataType(LogicalType::CHAR, nullable),
         size(size),
         physical_type(BytesType(size)) {}
 
@@ -58,8 +58,8 @@ struct VarcharType : public DataType {
 
   BytesType physical_type;
 
-  explicit VarcharType(int size)
-      : DataType(TypeEnum::VARCHAR),
+  explicit VarcharType(int size, bool nullable = true)
+      : DataType(LogicalType::VARCHAR, nullable),
         size(size),
         physical_type(BytesType(size + 1)) {}
   VarcharType(const VarcharType& other)
@@ -73,26 +73,26 @@ static const LayoutPtr physical_string = LayoutPtr(new ListLayoutType(byte1));
 
 // String is a logical type consisting of a physical list of 1-byte values
 struct StringType : public DataType {
-  StringType()
-      : DataType(TypeEnum::STRING) {}
+  explicit StringType(bool nullable = true)
+      : DataType(LogicalType::STRING, nullable) {}
 
   StringType(const StringType& other)
       : StringType() {}
 
-  const LayoutPtr& physical_type() {
-    return physical_string;
-  }
-
   static char const *name() {
     return "string";
   }
 
   virtual std::string ToString() const {
-    return name();
+    std::string result;
+    if (nullable) {
+      result.append("?");
+    }
+    result.append(name());
+    return result;
   }
 };
 
-
 // TODO: add a BinaryArray layer in between
 class StringArray : public ListArray {
  public:
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index 1a9fc6be4a5ce..df6157104795e 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -49,7 +49,7 @@ TEST(TestStructType, Basics) {
   ASSERT_TRUE(struct_type.field(1).Equals(f1));
   ASSERT_TRUE(struct_type.field(2).Equals(f2));
 
-  ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
+  ASSERT_EQ(struct_type.ToString(), "?struct<f0: ?int32, f1: ?string, f2: ?uint8>");
 
   // TODO: out of bounds for field(...)
 }
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
index a245656b516cc..6b233bc372af1 100644
--- a/cpp/src/arrow/types/struct.cc
+++ b/cpp/src/arrow/types/struct.cc
@@ -26,6 +26,7 @@ namespace arrow {
 
 std::string StructType::ToString() const {
   std::stringstream s;
+  if (nullable) s << "?";
   s << "struct<";
   for (size_t i = 0; i < fields_.size(); ++i) {
     if (i > 0) s << ", ";
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index afba19a7e4699..e575c31287cb2 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -29,8 +29,8 @@ namespace arrow {
 struct StructType : public DataType {
   std::vector<Field> fields_;
 
-  explicit StructType(const std::vector<Field>& fields)
-      : DataType(TypeEnum::STRUCT) {
+  explicit StructType(const std::vector<Field>& fields, bool nullable = true)
+      : DataType(LogicalType::STRUCT, nullable) {
     fields_ = fields;
   }
 
diff --git a/cpp/src/arrow/types/union.h b/cpp/src/arrow/types/union.h
index 62a3d1c10355d..9aff780c6a392 100644
--- a/cpp/src/arrow/types/union.h
+++ b/cpp/src/arrow/types/union.h
@@ -30,8 +30,8 @@ namespace arrow {
 
 class Buffer;
 
-struct DenseUnionType : public CollectionType<TypeEnum::DENSE_UNION> {
-  typedef CollectionType<TypeEnum::DENSE_UNION> Base;
+struct DenseUnionType : public CollectionType<LogicalType::DENSE_UNION> {
+  typedef CollectionType<LogicalType::DENSE_UNION> Base;
 
   explicit DenseUnionType(const std::vector<TypePtr>& child_types) :
       Base() {
@@ -42,8 +42,8 @@ struct DenseUnionType : public CollectionType<TypeEnum::DENSE_UNION> {
 };
 
 
-struct SparseUnionType : public CollectionType<TypeEnum::SPARSE_UNION> {
-  typedef CollectionType<TypeEnum::SPARSE_UNION> Base;
+struct SparseUnionType : public CollectionType<LogicalType::SPARSE_UNION> {
+  typedef CollectionType<LogicalType::SPARSE_UNION> Base;
 
   explicit SparseUnionType(const std::vector<TypePtr>& child_types) :
       Base() {

From 307977e39eddf62f832a5f1a452963751c6b36a0 Mon Sep 17 00:00:00 2001
From: proflin <proflin.me@gmail.com>
Date: Thu, 3 Mar 2016 16:14:47 -0800
Subject: [PATCH 0015/1644] ARROW-15: Fix a naming typo for
 memory.AllocationManager.AllocationOutcome

Rename FORCED_SUCESS to FORCED_SUC**_C_**ESS in memory.AllocationManager.AllocationOutcome.

Author: proflin <proflin.me@gmail.com>

Closes #4 from proflin/ARROW-15--Fix-a-naming-typo-for-memory.AllocationManager.AllocationOutcome and squashes the following commits:

0e276fa [proflin] ARROW-15: Fix a naming typo for memory.AllocationManager.AllocationOutcome
---
 .../src/main/java/org/apache/arrow/memory/Accountant.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
index dc75e5d7231a8..37c598ad89ece 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -247,7 +247,7 @@ public static enum AllocationOutcome {
     /**
      * Allocation succeeded but only because the allocator was forced to move beyond a limit.
      */
-    FORCED_SUCESS(true),
+    FORCED_SUCCESS(true),
 
     /**
      * Allocation failed because the local allocator's limits were exceeded.

From 0c95d3cc6d954128bf400598878ad9c4228ccbce Mon Sep 17 00:00:00 2001
From: proflin <proflin.me@gmail.com>
Date: Thu, 3 Mar 2016 16:16:28 -0800
Subject: [PATCH 0016/1644] ARROW-10: Fix mismatch of javadoc names and method
 parameters

Author: proflin <proflin.me@gmail.com>
Author: Liwei Lin <proflin.me@gmail.com>

Closes #3 from proflin/ARROW-10--Fix-mismatch-of-javadoc-names-and-method-parameters and squashes the following commits:

99366ab [Liwei Lin] ARROW-10: Fix mismatch of javadoc names and method parameters
9186cb3 [proflin] ARROW-10: Fix mismatch of javadoc names and method parameters
2b1313e [proflin] Fix mismatch of javadoc names and method parameters
---
 .../main/java/org/apache/arrow/memory/AllocationManager.java | 5 ++---
 .../org/apache/arrow/memory/AllocatorClosedException.java    | 5 +++--
 .../src/main/java/org/apache/arrow/memory/BufferManager.java | 1 +
 .../main/java/org/apache/arrow/memory/ChildAllocator.java    | 5 +----
 .../java/org/apache/arrow/memory/util/HistoricalLog.java     | 2 +-
 .../main/java/org/apache/arrow/vector/AllocationHelper.java  | 2 +-
 .../org/apache/arrow/vector/complex/ContainerVectorLike.java | 2 +-
 7 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index 37d1d34a62005..43ee9c108d902 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -328,7 +328,8 @@ public int decrement(int decrement) {
      * destroyed before use.
      *
      * @param allocator
-     * @return
+     *          A BufferAllocator.
+     * @return The ledger associated with the BufferAllocator.
      */
     public BufferLedger getLedgerForAllocator(BufferAllocator allocator) {
       return associate((BaseAllocator) allocator);
@@ -356,8 +357,6 @@ public ArrowBuf newArrowBuf(int offset, int length) {
      *          The length in bytes that this ArrowBuf will provide access to.
      * @param manager
      *          An optional BufferManager argument that can be used to manage expansion of this ArrowBuf
-     * @param retain
-     *          Whether or not the newly created buffer should get an additional reference count added to it.
      * @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
      */
     public ArrowBuf newArrowBuf(int offset, int length, BufferManager manager) {
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
index 566457981c7ed..3274642dedd59 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
@@ -20,11 +20,12 @@
 /**
  * Exception thrown when a closed BufferAllocator is used. Note
  * this is an unchecked exception.
- *
- * @param message string associated with the cause
  */
 @SuppressWarnings("serial")
 public class AllocatorClosedException extends RuntimeException {
+  /**
+   * @param message string associated with the cause
+   */
   public AllocatorClosedException(String message) {
     super(message);
   }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
index d6470fa51e7a2..8969434791012 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
@@ -43,6 +43,7 @@ public interface BufferManager extends AutoCloseable {
    * @param newSize
    *          Size of new replacement buffer.
    * @return
+   *          A new version of the buffer.
    */
   public ArrowBuf replace(ArrowBuf old, int newSize);
 
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
index 6f120e5328bd4..11c9063fc9c69 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
@@ -31,15 +31,12 @@ class ChildAllocator extends BaseAllocator {
    * Constructor.
    *
    * @param parentAllocator parent allocator -- the one creating this child
-   * @param allocatorOwner a handle to the object making the request
-   * @param allocationPolicy the allocation policy to use; the policy for all
-   *   allocators must match for each invocation of a drillbit
+   * @param name the name of this child allocator
    * @param initReservation initial amount of space to reserve (obtained from the parent)
    * @param maxAllocation maximum amount of space that can be obtained from this allocator;
    *   note this includes direct allocations (via {@see BufferAllocator#buffer(int, int)}
    *   et al) and requests from descendant allocators. Depending on the allocation policy in
    *   force, even less memory may be available
-   * @param flags one or more of BaseAllocator.F_* flags
    */
   ChildAllocator(
       BaseAllocator parentAllocator,
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
index 38cb779343ab6..c9b5c5385c596 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
@@ -126,7 +126,7 @@ public void buildHistory(final StringBuilder sb, boolean includeStackTrace) {
   /**
    *
    * @param sb
-   * @param indexLevel
+   * @param indent
    * @param includeStackTrace
    */
   public synchronized void buildHistory(final StringBuilder sb, int indent, boolean includeStackTrace) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
index 54c3cd7331e0f..15c3a0227c656 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
@@ -49,7 +49,7 @@ public static void allocate(ValueVector v, int valueCount, int bytesPerValue, in
    * Allocates the exact amount if v is fixed width, otherwise falls back to dynamic allocation
    * @param v value vector we are trying to allocate
    * @param valueCount  size we are trying to allocate
-   * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the memory
+   * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the memory
    */
   public static void allocateNew(ValueVector v, int valueCount) {
     if (v instanceof  FixedWidthVector) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
index e50b0d0d0a5ea..655b55a6aa2c6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
@@ -31,7 +31,7 @@ public interface ContainerVectorLike {
    *
    * @param  descriptor vector descriptor
    * @return  result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
-   * @throws org.apache.drill.common.exceptions.DrillRuntimeException
+   * @throws org.apache.arrow.vector.util.SchemaChangeRuntimeException
    *    if schema change is not permissible between the given and existing data vector types.
    */
   <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);

From 3b777c7f43d75444f040351b8ae4b735250f2efc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 3 Mar 2016 16:18:28 -0800
Subject: [PATCH 0017/1644] ARROW-26: Add instructions for enabling Arrow C++
 Parquet adapter build

This patch documents the workflow for building the optional Arrow-Parquet C++ integration. I originally thought about adding an option to build it in Arrow's thirdparty, but it immediately results in a dependency-hell situation (Parquet requires Thrift, Boost, snappy, lz4, zlib)

Author: Wes McKinney <wesm@apache.org>

Closes #12 from wesm/ARROW-26 and squashes the following commits:

b28fd75 [Wes McKinney] Add instructions for enabling Arrow C++ Parquet adapter build
---
 cpp/CMakeLists.txt |  4 ++--
 cpp/doc/Parquet.md | 24 ++++++++++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)
 create mode 100644 cpp/doc/Parquet.md

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5e4c204581369..f425c5f310673 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -54,7 +54,7 @@ endif()
 
 # Top level cmake dir
 if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
-  option(ARROW_WITH_PARQUET
+  option(ARROW_PARQUET
     "Build the Parquet adapter and link to libparquet"
     OFF)
 
@@ -441,7 +441,7 @@ endif (UNIX)
 #----------------------------------------------------------------------
 # Parquet adapter
 
-if(ARROW_WITH_PARQUET)
+if(ARROW_PARQUET)
   find_package(Parquet REQUIRED)
   include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(parquet
diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md
new file mode 100644
index 0000000000000..370ac833388fc
--- /dev/null
+++ b/cpp/doc/Parquet.md
@@ -0,0 +1,24 @@
+## Building Arrow-Parquet integration
+
+To build the Arrow C++'s Parquet adapter library, you must first build [parquet-cpp][1]:
+
+```bash
+# Set this to your preferred install location
+export PARQUET_HOME=$HOME/local
+
+git clone https://github.com/apache/parquet-cpp.git
+cd parquet-cpp
+source setup_build_env.sh
+cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME
+make -j4
+make install
+```
+
+Make sure that `$PARQUET_HOME` is set to the installation location. Now, build
+Arrow with the Parquet adapter enabled:
+
+```bash
+cmake -DARROW_PARQUET=ON
+```
+
+[1]: https://github.com/apache/parquet-cpp
\ No newline at end of file

From 9c2b95446abe1ec4dd5c25215c9595a3d7b49f2b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Fri, 4 Mar 2016 15:02:10 -0800
Subject: [PATCH 0018/1644] ARROW-23: Add a logical Column data structure

I also added global const instances of common primitive types

Author: Wes McKinney <wesm@apache.org>

Closes #15 from wesm/ARROW-23 and squashes the following commits:

1835d33 [Wes McKinney] Don't use auto
988135c [Wes McKinney] Add Column chunk type validation function
8a2e40e [Wes McKinney] Remove unneeded operator()/shared_from_this experiment
de9ec70 [Wes McKinney] Aggregate null counts too
7049314 [Wes McKinney] cpplint
a565d26 [Wes McKinney] Add ChunkedArray / Column ctors, test passes
0648ed2 [Wes McKinney] Prototyping
---
 cpp/CMakeLists.txt                 |   2 +
 cpp/src/arrow/array.h              |   1 -
 cpp/src/arrow/schema-test.cc       |   7 +-
 cpp/src/arrow/table/CMakeLists.txt |  39 +++++++++++
 cpp/src/arrow/table/column-test.cc |  93 ++++++++++++++++++++++++++
 cpp/src/arrow/table/column.cc      |  62 +++++++++++++++++
 cpp/src/arrow/table/column.h       | 103 +++++++++++++++++++++++++++++
 cpp/src/arrow/type.cc              |  12 ++++
 cpp/src/arrow/type.h               |  17 +++++
 cpp/src/arrow/types/list.h         |   2 +-
 cpp/src/arrow/types/primitive.h    |  20 +++---
 cpp/src/arrow/util/bit-util.h      |   4 ++
 12 files changed, 347 insertions(+), 15 deletions(-)
 create mode 100644 cpp/src/arrow/table/CMakeLists.txt
 create mode 100644 cpp/src/arrow/table/column-test.cc
 create mode 100644 cpp/src/arrow/table/column.cc
 create mode 100644 cpp/src/arrow/table/column.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f425c5f310673..15afb1acf67cf 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -458,10 +458,12 @@ endif()
 
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/util)
+add_subdirectory(src/arrow/table)
 add_subdirectory(src/arrow/types)
 
 set(LINK_LIBS
   arrow_util
+  arrow_table
   arrow_types)
 
 set(ARROW_SRCS
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 0632146637e59..85e853e2ae5e2 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -81,7 +81,6 @@ class Array {
   DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
-
 typedef std::shared_ptr<Array> ArrayPtr;
 
 } // namespace arrow
diff --git a/cpp/src/arrow/schema-test.cc b/cpp/src/arrow/schema-test.cc
index 3debb9cec3c00..7c190d068c2a6 100644
--- a/cpp/src/arrow/schema-test.cc
+++ b/cpp/src/arrow/schema-test.cc
@@ -31,7 +31,7 @@ using std::vector;
 namespace arrow {
 
 TEST(TestField, Basics) {
-  shared_ptr<DataType> ftype = std::make_shared<Int32Type>();
+  shared_ptr<DataType> ftype = INT32;
   shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
   Field f0("f0", ftype);
   Field f0_nn("f0", ftype_nn);
@@ -44,7 +44,7 @@ TEST(TestField, Basics) {
 }
 
 TEST(TestField, Equals) {
-  shared_ptr<DataType> ftype = std::make_shared<Int32Type>();
+  shared_ptr<DataType> ftype = INT32;
   shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
 
   Field f0("f0", ftype);
@@ -61,8 +61,7 @@ class TestSchema : public ::testing::Test {
 };
 
 TEST_F(TestSchema, Basics) {
-  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int32Type>());
-
+  auto f0 = std::make_shared<Field>("f0", INT32);
   auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(false));
   auto f1_optional = std::make_shared<Field>("f1", std::make_shared<UInt8Type>());
 
diff --git a/cpp/src/arrow/table/CMakeLists.txt b/cpp/src/arrow/table/CMakeLists.txt
new file mode 100644
index 0000000000000..a401622d2e0d7
--- /dev/null
+++ b/cpp/src/arrow/table/CMakeLists.txt
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######################################
+# arrow_table
+#######################################
+
+set(TABLE_SRCS
+  column.cc
+)
+
+set(TABLE_LIBS
+)
+
+add_library(arrow_table STATIC
+  ${TABLE_SRCS}
+)
+target_link_libraries(arrow_table ${TABLE_LIBS})
+SET_TARGET_PROPERTIES(arrow_table PROPERTIES LINKER_LANGUAGE CXX)
+
+# Headers: top level
+install(FILES
+  DESTINATION include/arrow/table)
+
+ADD_ARROW_TEST(column-test)
diff --git a/cpp/src/arrow/table/column-test.cc b/cpp/src/arrow/table/column-test.cc
new file mode 100644
index 0000000000000..15f554f46325d
--- /dev/null
+++ b/cpp/src/arrow/table/column-test.cc
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/field.h"
+#include "arrow/schema.h"
+#include "arrow/table/column.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/types/integer.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+using std::shared_ptr;
+using std::vector;
+
+namespace arrow {
+
+class TestColumn : public ::testing::Test {
+ public:
+  void SetUp() {
+    pool_ = GetDefaultMemoryPool();
+  }
+
+  template <typename ArrayType>
+  std::shared_ptr<Array> MakeArray(int32_t length, int32_t null_count = 0) {
+    auto data = std::make_shared<PoolBuffer>(pool_);
+    auto nulls = std::make_shared<PoolBuffer>(pool_);
+    data->Resize(length * sizeof(typename ArrayType::value_type));
+    nulls->Resize(util::bytes_for_bits(length));
+    return std::make_shared<ArrayType>(length, data, 10, nulls);
+  }
+
+ protected:
+  MemoryPool* pool_;
+
+  std::shared_ptr<ChunkedArray> data_;
+  std::unique_ptr<Column> column_;
+};
+
+TEST_F(TestColumn, BasicAPI) {
+  ArrayVector arrays;
+  arrays.push_back(MakeArray<Int32Array>(100));
+  arrays.push_back(MakeArray<Int32Array>(100, 10));
+  arrays.push_back(MakeArray<Int32Array>(100, 20));
+
+  auto field = std::make_shared<Field>("c0", INT32);
+  column_.reset(new Column(field, arrays));
+
+  ASSERT_EQ("c0", column_->name());
+  ASSERT_TRUE(column_->type()->Equals(INT32));
+  ASSERT_EQ(300, column_->length());
+  ASSERT_EQ(30, column_->null_count());
+  ASSERT_EQ(3, column_->data()->num_chunks());
+}
+
+TEST_F(TestColumn, ChunksInhomogeneous) {
+  ArrayVector arrays;
+  arrays.push_back(MakeArray<Int32Array>(100));
+  arrays.push_back(MakeArray<Int32Array>(100, 10));
+
+  auto field = std::make_shared<Field>("c0", INT32);
+  column_.reset(new Column(field, arrays));
+
+  ASSERT_OK(column_->ValidateData());
+
+  arrays.push_back(MakeArray<Int16Array>(100, 10));
+  column_.reset(new Column(field, arrays));
+  ASSERT_RAISES(Invalid, column_->ValidateData());
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/table/column.cc b/cpp/src/arrow/table/column.cc
new file mode 100644
index 0000000000000..82750cf4d4306
--- /dev/null
+++ b/cpp/src/arrow/table/column.cc
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/table/column.h"
+
+#include <memory>
+#include <sstream>
+
+#include "arrow/field.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+ChunkedArray::ChunkedArray(const ArrayVector& chunks) :
+    chunks_(chunks) {
+  length_ = 0;
+  for (const std::shared_ptr<Array>& chunk : chunks) {
+    length_ += chunk->length();
+    null_count_ += chunk->null_count();
+  }
+}
+
+Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks) :
+    field_(field) {
+  data_ = std::make_shared<ChunkedArray>(chunks);
+}
+
+Column::Column(const std::shared_ptr<Field>& field,
+    const std::shared_ptr<ChunkedArray>& data) :
+    field_(field),
+    data_(data) {}
+
+Status Column::ValidateData() {
+  for (int i = 0; i < data_->num_chunks(); ++i) {
+    const std::shared_ptr<DataType>& type = data_->chunk(i)->type();
+    if (!this->type()->Equals(type)) {
+      std::stringstream ss;
+      ss << "In chunk " << i << " expected type "
+         << this->type()->ToString()
+         << " but saw "
+         << type->ToString();
+      return Status::Invalid(ss.str());
+    }
+  }
+  return Status::OK();
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/table/column.h b/cpp/src/arrow/table/column.h
new file mode 100644
index 0000000000000..9e9064e86545d
--- /dev/null
+++ b/cpp/src/arrow/table/column.h
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TABLE_COLUMN_H
+#define ARROW_TABLE_COLUMN_H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/field.h"
+
+namespace arrow {
+
+typedef std::vector<std::shared_ptr<Array> > ArrayVector;
+
+// A data structure managing a list of primitive Arrow arrays logically as one
+// large array
+class ChunkedArray {
+ public:
+  explicit ChunkedArray(const ArrayVector& chunks);
+
+  // @returns: the total length of the chunked array; computed on construction
+  int64_t length() const {
+    return length_;
+  }
+
+  int64_t null_count() const {
+    return null_count_;
+  }
+
+  int num_chunks() const {
+    return chunks_.size();
+  }
+
+  const std::shared_ptr<Array>& chunk(int i) const {
+    return chunks_[i];
+  }
+
+ protected:
+  ArrayVector chunks_;
+  int64_t length_;
+  int64_t null_count_;
+};
+
+// An immutable column data structure consisting of a field (type metadata) and
+// a logical chunked data array (which can be validated as all being the same
+// type).
+class Column {
+ public:
+  Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
+  Column(const std::shared_ptr<Field>& field,
+      const std::shared_ptr<ChunkedArray>& data);
+
+  int64_t length() const {
+    return data_->length();
+  }
+
+  int64_t null_count() const {
+    return data_->null_count();
+  }
+
+  // @returns: the column's name in the passed metadata
+  const std::string& name() const {
+    return field_->name;
+  }
+
+  // @returns: the column's type according to the metadata
+  const std::shared_ptr<DataType>& type() const {
+    return field_->type;
+  }
+
+  // @returns: the column's data as a chunked logical array
+  const std::shared_ptr<ChunkedArray>& data() const {
+    return data_;
+  }
+  // Verify that the column's array data is consistent with the passed field's
+  // metadata
+  Status ValidateData();
+
+ protected:
+  std::shared_ptr<Field> field_;
+  std::shared_ptr<ChunkedArray> data_;
+};
+
+} // namespace arrow
+
+#endif  // ARROW_TABLE_COLUMN_H
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 492eee52b04b1..ff145e2c1e3b4 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -19,4 +19,16 @@
 
 namespace arrow {
 
+const std::shared_ptr<BooleanType> BOOL = std::make_shared<BooleanType>();
+const std::shared_ptr<UInt8Type> UINT8 = std::make_shared<UInt8Type>();
+const std::shared_ptr<UInt16Type> UINT16 = std::make_shared<UInt16Type>();
+const std::shared_ptr<UInt32Type> UINT32 = std::make_shared<UInt32Type>();
+const std::shared_ptr<UInt64Type> UINT64 = std::make_shared<UInt64Type>();
+const std::shared_ptr<Int8Type> INT8 = std::make_shared<Int8Type>();
+const std::shared_ptr<Int16Type> INT16 = std::make_shared<Int16Type>();
+const std::shared_ptr<Int32Type> INT32 = std::make_shared<Int32Type>();
+const std::shared_ptr<Int64Type> INT64 = std::make_shared<Int64Type>();
+const std::shared_ptr<FloatType> FLOAT = std::make_shared<FloatType>();
+const std::shared_ptr<DoubleType> DOUBLE = std::make_shared<DoubleType>();
+
 } // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 04cdb52b535db..4193a0e8bc851 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -142,10 +142,15 @@ struct DataType {
       nullable(nullable) {}
 
   virtual bool Equals(const DataType* other) {
+    // Call with a pointer so more friendly to subclasses
     return this == other || (this->type == other->type &&
         this->nullable == other->nullable);
   }
 
+  bool Equals(const std::shared_ptr<DataType>& other) {
+    return Equals(other.get());
+  }
+
   virtual std::string ToString() const = 0;
 };
 
@@ -244,6 +249,18 @@ struct DoubleType : public PrimitiveType<DoubleType> {
   PRIMITIVE_DECL(DoubleType, double, DOUBLE, 8, "double");
 };
 
+extern const std::shared_ptr<BooleanType> BOOL;
+extern const std::shared_ptr<UInt8Type> UINT8;
+extern const std::shared_ptr<UInt16Type> UINT16;
+extern const std::shared_ptr<UInt32Type> UINT32;
+extern const std::shared_ptr<UInt64Type> UINT64;
+extern const std::shared_ptr<Int8Type> INT8;
+extern const std::shared_ptr<Int16Type> INT16;
+extern const std::shared_ptr<Int32Type> INT32;
+extern const std::shared_ptr<Int64Type> INT64;
+extern const std::shared_ptr<FloatType> FLOAT;
+extern const std::shared_ptr<DoubleType> DOUBLE;
+
 } // namespace arrow
 
 #endif  // ARROW_TYPE_H
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 1fc83536db8c6..f39fe5c4d811b 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -132,7 +132,7 @@ class ListBuilder : public Int32Builder {
   //
   // If passed, null_bytes is of equal length to values, and any nonzero byte
   // will be considered as a null for that slot
-  Status Append(T* values, int32_t length, uint8_t* null_bytes = nullptr) {
+  Status Append(value_type* values, int32_t length, uint8_t* null_bytes = nullptr) {
     if (length_ + length > capacity_) {
       int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 49040fb66268f..09d43e7ec8b80 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -60,7 +60,7 @@ class PrimitiveArray : public Array {
 template <typename TypeClass>
 class PrimitiveArrayImpl : public PrimitiveArray {
  public:
-  typedef typename TypeClass::c_type T;
+  typedef typename TypeClass::c_type value_type;
 
   PrimitiveArrayImpl() : PrimitiveArray() {}
 
@@ -81,9 +81,11 @@ class PrimitiveArrayImpl : public PrimitiveArray {
     return PrimitiveArray::Equals(*static_cast<const PrimitiveArray*>(&other));
   }
 
-  const T* raw_data() const { return reinterpret_cast<const T*>(raw_data_);}
+  const value_type* raw_data() const {
+    return reinterpret_cast<const value_type*>(raw_data_);
+  }
 
-  T Value(int i) const {
+  value_type Value(int i) const {
     return raw_data()[i];
   }
 
@@ -96,12 +98,12 @@ class PrimitiveArrayImpl : public PrimitiveArray {
 template <typename Type, typename ArrayType>
 class PrimitiveBuilder : public ArrayBuilder {
  public:
-  typedef typename Type::c_type T;
+  typedef typename Type::c_type value_type;
 
   explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type) :
       ArrayBuilder(pool, type),
       values_(nullptr) {
-    elsize_ = sizeof(T);
+    elsize_ = sizeof(value_type);
   }
 
   virtual ~PrimitiveBuilder() {}
@@ -141,7 +143,7 @@ class PrimitiveBuilder : public ArrayBuilder {
   }
 
   // Scalar append
-  Status Append(T val, bool is_null = false) {
+  Status Append(value_type val, bool is_null = false) {
     if (length_ == capacity_) {
       // If the capacity was not already a multiple of 2, do so here
       RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
@@ -158,7 +160,7 @@ class PrimitiveBuilder : public ArrayBuilder {
   //
   // If passed, null_bytes is of equal length to values, and any nonzero byte
   // will be considered as a null for that slot
-  Status Append(const T* values, int32_t length,
+  Status Append(const value_type* values, int32_t length,
       const uint8_t* null_bytes = nullptr) {
     if (length_ + length > capacity_) {
       int32_t new_capacity = util::next_power2(length_ + length);
@@ -215,8 +217,8 @@ class PrimitiveBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  T* raw_buffer() {
-    return reinterpret_cast<T*>(values_->mutable_data());
+  value_type* raw_buffer() {
+    return reinterpret_cast<value_type*>(values_->mutable_data());
   }
 
   std::shared_ptr<Buffer> buffer() const {
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 841f617a3139c..5e7197f901222 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -33,6 +33,10 @@ static inline int64_t ceil_byte(int64_t size) {
   return (size + 7) & ~7;
 }
 
+static inline int64_t bytes_for_bits(int64_t size) {
+  return ceil_byte(size) / 8;
+}
+
 static inline int64_t ceil_2bytes(int64_t size) {
   return (size + 15) & ~15;
 }

From 612fbc74ece160a52edbd260de8391aa07ad00ca Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Fri, 4 Mar 2016 17:59:58 -0800
Subject: [PATCH 0019/1644] ARROW-24: C++: Implement a logical Table container
 type

A table enables us to interpret a collection of Arrow arrays as a logical table or "data frame"-like structure.  Each column may consist of one or more "primitive" Arrow memory containers.

Note that this currently has the limitation that the table column names must be strings. At least, this is consistent with most storage media and up-stack table implementations (e.g. R's data.frame).

Currently this is somewhat limited in the arrangement of data (a vector of chunked columns -- the columns may contain only one data chunk) -- since a Table might be assembled from a vector of row batches (coming across the wire), "pivoting" the row batches might have performance implications that we can examine further on down the road.

Author: Wes McKinney <wesm@apache.org>

Closes #16 from wesm/ARROW-24 and squashes the following commits:

b701c76 [Wes McKinney] Test case for wrong number of columns passed
5faa5ac [Wes McKinney] cpplint
9a651cb [Wes McKinney] Basic table prototype. Move Schema code under arrow/table
---
 cpp/CMakeLists.txt                       |   1 -
 cpp/src/arrow/CMakeLists.txt             |   1 -
 cpp/src/arrow/table/CMakeLists.txt       |   4 +
 cpp/src/arrow/table/column-test.cc       |  37 ++-----
 cpp/src/arrow/table/column.cc            |   6 ++
 cpp/src/arrow/table/column.h             |   2 +
 cpp/src/arrow/{ => table}/schema-test.cc |   2 +-
 cpp/src/arrow/{ => table}/schema.cc      |   2 +-
 cpp/src/arrow/{ => table}/schema.h       |   0
 cpp/src/arrow/table/table-test.cc        | 125 +++++++++++++++++++++++
 cpp/src/arrow/table/table.cc             |  73 +++++++++++++
 cpp/src/arrow/table/table.h              |  82 +++++++++++++++
 cpp/src/arrow/table/test-common.h        |  55 ++++++++++
 13 files changed, 358 insertions(+), 32 deletions(-)
 rename cpp/src/arrow/{ => table}/schema-test.cc (99%)
 rename cpp/src/arrow/{ => table}/schema.cc (98%)
 rename cpp/src/arrow/{ => table}/schema.h (100%)
 create mode 100644 cpp/src/arrow/table/table-test.cc
 create mode 100644 cpp/src/arrow/table/table.cc
 create mode 100644 cpp/src/arrow/table/table.h
 create mode 100644 cpp/src/arrow/table/test-common.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 15afb1acf67cf..8042661533e1d 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -470,7 +470,6 @@ set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/builder.cc
   src/arrow/field.cc
-  src/arrow/schema.cc
   src/arrow/type.cc
 )
 
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 04f8dd1f908cb..77326ce38d754 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -30,4 +30,3 @@ install(FILES
 set(ARROW_TEST_LINK_LIBS arrow_test_util ${ARROW_MIN_TEST_LIBS})
 
 ADD_ARROW_TEST(array-test)
-ADD_ARROW_TEST(schema-test)
diff --git a/cpp/src/arrow/table/CMakeLists.txt b/cpp/src/arrow/table/CMakeLists.txt
index a401622d2e0d7..b51258ffd8b0d 100644
--- a/cpp/src/arrow/table/CMakeLists.txt
+++ b/cpp/src/arrow/table/CMakeLists.txt
@@ -21,6 +21,8 @@
 
 set(TABLE_SRCS
   column.cc
+  schema.cc
+  table.cc
 )
 
 set(TABLE_LIBS
@@ -37,3 +39,5 @@ install(FILES
   DESTINATION include/arrow/table)
 
 ADD_ARROW_TEST(column-test)
+ADD_ARROW_TEST(schema-test)
+ADD_ARROW_TEST(table-test)
diff --git a/cpp/src/arrow/table/column-test.cc b/cpp/src/arrow/table/column-test.cc
index 15f554f46325d..4959b82c6e2ae 100644
--- a/cpp/src/arrow/table/column-test.cc
+++ b/cpp/src/arrow/table/column-test.cc
@@ -22,48 +22,29 @@
 #include <vector>
 
 #include "arrow/field.h"
-#include "arrow/schema.h"
 #include "arrow/table/column.h"
+#include "arrow/table/schema.h"
+#include "arrow/table/test-common.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/types/integer.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
 
 using std::shared_ptr;
 using std::vector;
 
 namespace arrow {
 
-class TestColumn : public ::testing::Test {
- public:
-  void SetUp() {
-    pool_ = GetDefaultMemoryPool();
-  }
-
-  template <typename ArrayType>
-  std::shared_ptr<Array> MakeArray(int32_t length, int32_t null_count = 0) {
-    auto data = std::make_shared<PoolBuffer>(pool_);
-    auto nulls = std::make_shared<PoolBuffer>(pool_);
-    data->Resize(length * sizeof(typename ArrayType::value_type));
-    nulls->Resize(util::bytes_for_bits(length));
-    return std::make_shared<ArrayType>(length, data, 10, nulls);
-  }
-
+class TestColumn : public TestBase {
  protected:
-  MemoryPool* pool_;
-
   std::shared_ptr<ChunkedArray> data_;
   std::unique_ptr<Column> column_;
 };
 
 TEST_F(TestColumn, BasicAPI) {
   ArrayVector arrays;
-  arrays.push_back(MakeArray<Int32Array>(100));
-  arrays.push_back(MakeArray<Int32Array>(100, 10));
-  arrays.push_back(MakeArray<Int32Array>(100, 20));
+  arrays.push_back(MakePrimitive<Int32Array>(100));
+  arrays.push_back(MakePrimitive<Int32Array>(100, 10));
+  arrays.push_back(MakePrimitive<Int32Array>(100, 20));
 
   auto field = std::make_shared<Field>("c0", INT32);
   column_.reset(new Column(field, arrays));
@@ -77,15 +58,15 @@ TEST_F(TestColumn, BasicAPI) {
 
 TEST_F(TestColumn, ChunksInhomogeneous) {
   ArrayVector arrays;
-  arrays.push_back(MakeArray<Int32Array>(100));
-  arrays.push_back(MakeArray<Int32Array>(100, 10));
+  arrays.push_back(MakePrimitive<Int32Array>(100));
+  arrays.push_back(MakePrimitive<Int32Array>(100, 10));
 
   auto field = std::make_shared<Field>("c0", INT32);
   column_.reset(new Column(field, arrays));
 
   ASSERT_OK(column_->ValidateData());
 
-  arrays.push_back(MakeArray<Int16Array>(100, 10));
+  arrays.push_back(MakePrimitive<Int16Array>(100, 10));
   column_.reset(new Column(field, arrays));
   ASSERT_RAISES(Invalid, column_->ValidateData());
 }
diff --git a/cpp/src/arrow/table/column.cc b/cpp/src/arrow/table/column.cc
index 82750cf4d4306..d68b491fb99da 100644
--- a/cpp/src/arrow/table/column.cc
+++ b/cpp/src/arrow/table/column.cc
@@ -39,6 +39,12 @@ Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks) :
   data_ = std::make_shared<ChunkedArray>(chunks);
 }
 
+Column::Column(const std::shared_ptr<Field>& field,
+    const std::shared_ptr<Array>& data) :
+    field_(field) {
+  data_ = std::make_shared<ChunkedArray>(ArrayVector({data}));
+}
+
 Column::Column(const std::shared_ptr<Field>& field,
     const std::shared_ptr<ChunkedArray>& data) :
     field_(field),
diff --git a/cpp/src/arrow/table/column.h b/cpp/src/arrow/table/column.h
index 9e9064e86545d..64423bf956147 100644
--- a/cpp/src/arrow/table/column.h
+++ b/cpp/src/arrow/table/column.h
@@ -67,6 +67,8 @@ class Column {
   Column(const std::shared_ptr<Field>& field,
       const std::shared_ptr<ChunkedArray>& data);
 
+  Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data);
+
   int64_t length() const {
     return data_->length();
   }
diff --git a/cpp/src/arrow/schema-test.cc b/cpp/src/arrow/table/schema-test.cc
similarity index 99%
rename from cpp/src/arrow/schema-test.cc
rename to cpp/src/arrow/table/schema-test.cc
index 7c190d068c2a6..0cf1b3c5f9a8e 100644
--- a/cpp/src/arrow/schema-test.cc
+++ b/cpp/src/arrow/table/schema-test.cc
@@ -21,7 +21,7 @@
 #include <vector>
 
 #include "arrow/field.h"
-#include "arrow/schema.h"
+#include "arrow/table/schema.h"
 #include "arrow/type.h"
 #include "arrow/types/string.h"
 
diff --git a/cpp/src/arrow/schema.cc b/cpp/src/arrow/table/schema.cc
similarity index 98%
rename from cpp/src/arrow/schema.cc
rename to cpp/src/arrow/table/schema.cc
index a735fd3d23075..fb3b4d6f29268 100644
--- a/cpp/src/arrow/schema.cc
+++ b/cpp/src/arrow/table/schema.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/schema.h"
+#include "arrow/table/schema.h"
 
 #include <memory>
 #include <string>
diff --git a/cpp/src/arrow/schema.h b/cpp/src/arrow/table/schema.h
similarity index 100%
rename from cpp/src/arrow/schema.h
rename to cpp/src/arrow/table/schema.h
diff --git a/cpp/src/arrow/table/table-test.cc b/cpp/src/arrow/table/table-test.cc
new file mode 100644
index 0000000000000..dd4f74cd16f89
--- /dev/null
+++ b/cpp/src/arrow/table/table-test.cc
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/field.h"
+#include "arrow/table/column.h"
+#include "arrow/table/schema.h"
+#include "arrow/table/table.h"
+#include "arrow/table/test-common.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/types/integer.h"
+
+using std::shared_ptr;
+using std::vector;
+
+namespace arrow {
+
+class TestTable : public TestBase {
+ public:
+  void MakeExample1(int length) {
+    auto f0 = std::make_shared<Field>("f0", INT32);
+    auto f1 = std::make_shared<Field>("f1", UINT8);
+    auto f2 = std::make_shared<Field>("f2", INT16);
+
+    vector<shared_ptr<Field> > fields = {f0, f1, f2};
+    schema_ = std::make_shared<Schema>(fields);
+
+    columns_ = {
+      std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length)),
+      std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
+      std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length))
+    };
+  }
+
+ protected:
+  std::unique_ptr<Table> table_;
+  shared_ptr<Schema> schema_;
+  vector<std::shared_ptr<Column> > columns_;
+};
+
+TEST_F(TestTable, EmptySchema) {
+  auto empty_schema = shared_ptr<Schema>(new Schema({}));
+  table_.reset(new Table("data", empty_schema, columns_));
+  ASSERT_OK(table_->ValidateColumns());
+  ASSERT_EQ(0, table_->num_rows());
+  ASSERT_EQ(0, table_->num_columns());
+}
+
+TEST_F(TestTable, Ctors) {
+  int length = 100;
+  MakeExample1(length);
+
+  std::string name = "data";
+
+  table_.reset(new Table(name, schema_, columns_));
+  ASSERT_OK(table_->ValidateColumns());
+  ASSERT_EQ(name, table_->name());
+  ASSERT_EQ(length, table_->num_rows());
+  ASSERT_EQ(3, table_->num_columns());
+
+  table_.reset(new Table(name, schema_, columns_, length));
+  ASSERT_OK(table_->ValidateColumns());
+  ASSERT_EQ(name, table_->name());
+  ASSERT_EQ(length, table_->num_rows());
+}
+
+TEST_F(TestTable, Metadata) {
+  int length = 100;
+  MakeExample1(length);
+
+  std::string name = "data";
+  table_.reset(new Table(name, schema_, columns_));
+
+  ASSERT_TRUE(table_->schema()->Equals(schema_));
+
+  auto col = table_->column(0);
+  ASSERT_EQ(schema_->field(0)->name, col->name());
+  ASSERT_EQ(schema_->field(0)->type, col->type());
+}
+
+TEST_F(TestTable, InvalidColumns) {
+  // Check that columns are all the same length
+  int length = 100;
+  MakeExample1(length);
+
+  table_.reset(new Table("data", schema_, columns_, length - 1));
+  ASSERT_RAISES(Invalid, table_->ValidateColumns());
+
+  columns_.clear();
+
+  // Wrong number of columns
+  table_.reset(new Table("data", schema_, columns_, length));
+  ASSERT_RAISES(Invalid, table_->ValidateColumns());
+
+  columns_ = {
+    std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length)),
+    std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
+    std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length - 1))
+  };
+
+  table_.reset(new Table("data", schema_, columns_, length));
+  ASSERT_RAISES(Invalid, table_->ValidateColumns());
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/table/table.cc b/cpp/src/arrow/table/table.cc
new file mode 100644
index 0000000000000..4cefc924ed38f
--- /dev/null
+++ b/cpp/src/arrow/table/table.cc
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/table/table.h"
+
+#include <memory>
+#include <sstream>
+
+#include "arrow/field.h"
+#include "arrow/table/column.h"
+#include "arrow/table/schema.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
+    const std::vector<std::shared_ptr<Column> >& columns) :
+    name_(name),
+    schema_(schema),
+    columns_(columns) {
+  if (columns.size() == 0) {
+    num_rows_ = 0;
+  } else {
+    num_rows_ = columns[0]->length();
+  }
+}
+
+Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
+    const std::vector<std::shared_ptr<Column> >& columns, int64_t num_rows) :
+    name_(name),
+    schema_(schema),
+    columns_(columns),
+    num_rows_(num_rows) {}
+
+Status Table::ValidateColumns() const {
+  if (num_columns() != schema_->num_fields()) {
+    return Status::Invalid("Number of columns did not match schema");
+  }
+
+  if (columns_.size() == 0) {
+    return Status::OK();
+  }
+
+  // Make sure columns are all the same length
+  for (size_t i = 0; i < columns_.size(); ++i) {
+    const Column* col = columns_[i].get();
+    if (col->length() != num_rows_) {
+      std::stringstream ss;
+      ss << "Column " << i << " expected length "
+         << num_rows_
+         << " but got length "
+         << col->length();
+      return Status::Invalid(ss.str());
+    }
+  }
+  return Status::OK();
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/table/table.h b/cpp/src/arrow/table/table.h
new file mode 100644
index 0000000000000..b0129387b710c
--- /dev/null
+++ b/cpp/src/arrow/table/table.h
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TABLE_TABLE_H
+#define ARROW_TABLE_TABLE_H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace arrow {
+
+class Column;
+class Schema;
+class Status;
+
+// Immutable container of fixed-length columns conforming to a particular schema
+class Table {
+ public:
+  // If columns is zero-length, the table's number of rows is zero
+  Table(const std::string& name, const std::shared_ptr<Schema>& schema,
+      const std::vector<std::shared_ptr<Column> >& columns);
+
+  Table(const std::string& name, const std::shared_ptr<Schema>& schema,
+      const std::vector<std::shared_ptr<Column> >& columns, int64_t num_rows);
+
+  // @returns: the table's name, if any (may be length 0)
+  const std::string& name() const {
+    return name_;
+  }
+
+  // @returns: the table's schema
+  const std::shared_ptr<Schema>& schema() const {
+    return schema_;
+  }
+
+  // Note: Does not boundscheck
+  // @returns: the i-th column
+  const std::shared_ptr<Column>& column(int i) const {
+    return columns_[i];
+  }
+
+  // @returns: the number of columns in the table
+  int num_columns() const {
+    return columns_.size();
+  }
+
+  // @returns: the number of rows (the corresponding length of each column)
+  int64_t num_rows() const {
+    return num_rows_;
+  }
+
+  // After construction, perform any checks to validate the input arguments
+  Status ValidateColumns() const;
+
+ private:
+  // The table's name, optional
+  std::string name_;
+
+  std::shared_ptr<Schema> schema_;
+  std::vector<std::shared_ptr<Column> > columns_;
+
+  int64_t num_rows_;
+};
+
+} // namespace arrow
+
+#endif  // ARROW_TABLE_TABLE_H
diff --git a/cpp/src/arrow/table/test-common.h b/cpp/src/arrow/table/test-common.h
new file mode 100644
index 0000000000000..efe2f228cd0a3
--- /dev/null
+++ b/cpp/src/arrow/table/test-common.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/field.h"
+#include "arrow/table/column.h"
+#include "arrow/table/schema.h"
+#include "arrow/table/table.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
+
+namespace arrow {
+
+class TestBase : public ::testing::Test {
+ public:
+  void SetUp() {
+    pool_ = GetDefaultMemoryPool();
+  }
+
+  template <typename ArrayType>
+  std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
+    auto data = std::make_shared<PoolBuffer>(pool_);
+    auto nulls = std::make_shared<PoolBuffer>(pool_);
+    EXPECT_OK(data->Resize(length * sizeof(typename ArrayType::value_type)));
+    EXPECT_OK(nulls->Resize(util::bytes_for_bits(length)));
+    return std::make_shared<ArrayType>(length, data, 10, nulls);
+  }
+
+ protected:
+  MemoryPool* pool_;
+};
+
+} // namespace arrow

From 572cdf22e3595035966a05a5ec2398f9d29df669 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Mon, 7 Mar 2016 14:42:32 -0800
Subject: [PATCH 0020/1644] ARROW-7: Add barebones Python library build
 toolchain

This patch provides no actual functionality; it only builds an empty Cython extension that links to libarrow.so. I will hook this into Travis CI at some later time.

I have adapted a limited amount of BSD (2- or 3-clause) or Apache 2.0 3rd-party code (particularly the cmake/Cython integration) to bootstrap this Python package / build setup in accordance with http://www.apache.org/legal/resolved.html. I have noted the relevant copyright holders and licenses in `python/LICENSE.txt`. In particular, I expect to continue to refactor and reuse occasional utility code from pandas (https://github.com/pydata/pandas) as practical.

Since a significant amount of "glue code" will need to be written to marshal between Arrow data and pure Python / NumPy / pandas objects, to get started I've adopted the approach used by libdynd/dynd-python -- a C++ "glue library" that is then called from Cython to provide a Python user interface. This will allow us to build shims as necessary to abstract away complications that leak through (for example: enabling C++ code with no knowledge of Python to invoke Python functions). Let's see how this goes: there are other options, like Boost::Python, but Cython + shim code is a more lightweight and flexible solution for the moment.

Author: Wes McKinney <wesm@apache.org>

Closes #17 from wesm/ARROW-7 and squashes the following commits:

be059a2 [Wes McKinney] Nest arrow::py namespace
3ad3143 [Wes McKinney] Add preliminary Python development toolchain
---
 cpp/src/arrow/CMakeLists.txt                 |   1 +
 cpp/src/arrow/table/CMakeLists.txt           |   3 +
 python/.gitignore                            |  37 ++
 python/CMakeLists.txt                        | 464 +++++++++++++++++++
 python/LICENSE.txt                           |  88 ++++
 python/README.md                             |  14 +
 python/arrow/__init__.py                     |   0
 python/arrow/compat.py                       |  86 ++++
 python/arrow/config.pyx                      |   8 +
 python/arrow/includes/__init__.pxd           |   0
 python/arrow/includes/arrow.pxd              |  23 +
 python/arrow/includes/common.pxd             |  34 ++
 python/arrow/includes/parquet.pxd            |  51 ++
 python/arrow/includes/pyarrow.pxd            |  23 +
 python/arrow/parquet.pyx                     |  23 +
 python/arrow/tests/__init__.py               |   0
 python/cmake_modules/CompilerInfo.cmake      |  48 ++
 python/cmake_modules/FindArrow.cmake         |  77 +++
 python/cmake_modules/FindCython.cmake        |  30 ++
 python/cmake_modules/FindNumPy.cmake         | 100 ++++
 python/cmake_modules/FindPythonLibsNew.cmake | 236 ++++++++++
 python/cmake_modules/UseCython.cmake         | 164 +++++++
 python/setup.py                              | 244 ++++++++++
 python/src/pyarrow/CMakeLists.txt            |  20 +
 python/src/pyarrow/api.h                     |  21 +
 python/src/pyarrow/init.cc                   |  29 ++
 python/src/pyarrow/init.h                    |  31 ++
 python/src/pyarrow/util/CMakeLists.txt       |  53 +++
 python/src/pyarrow/util/test_main.cc         |  26 ++
 29 files changed, 1934 insertions(+)
 create mode 100644 python/.gitignore
 create mode 100644 python/CMakeLists.txt
 create mode 100644 python/LICENSE.txt
 create mode 100644 python/README.md
 create mode 100644 python/arrow/__init__.py
 create mode 100644 python/arrow/compat.py
 create mode 100644 python/arrow/config.pyx
 create mode 100644 python/arrow/includes/__init__.pxd
 create mode 100644 python/arrow/includes/arrow.pxd
 create mode 100644 python/arrow/includes/common.pxd
 create mode 100644 python/arrow/includes/parquet.pxd
 create mode 100644 python/arrow/includes/pyarrow.pxd
 create mode 100644 python/arrow/parquet.pyx
 create mode 100644 python/arrow/tests/__init__.py
 create mode 100644 python/cmake_modules/CompilerInfo.cmake
 create mode 100644 python/cmake_modules/FindArrow.cmake
 create mode 100644 python/cmake_modules/FindCython.cmake
 create mode 100644 python/cmake_modules/FindNumPy.cmake
 create mode 100644 python/cmake_modules/FindPythonLibsNew.cmake
 create mode 100644 python/cmake_modules/UseCython.cmake
 create mode 100644 python/setup.py
 create mode 100644 python/src/pyarrow/CMakeLists.txt
 create mode 100644 python/src/pyarrow/api.h
 create mode 100644 python/src/pyarrow/init.cc
 create mode 100644 python/src/pyarrow/init.h
 create mode 100644 python/src/pyarrow/util/CMakeLists.txt
 create mode 100644 python/src/pyarrow/util/test_main.cc

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 77326ce38d754..102a8a1853f3e 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -20,6 +20,7 @@ install(FILES
   api.h
   array.h
   builder.h
+  field.h
   type.h
   DESTINATION include/arrow)
 
diff --git a/cpp/src/arrow/table/CMakeLists.txt b/cpp/src/arrow/table/CMakeLists.txt
index b51258ffd8b0d..68bf3148a9889 100644
--- a/cpp/src/arrow/table/CMakeLists.txt
+++ b/cpp/src/arrow/table/CMakeLists.txt
@@ -36,6 +36,9 @@ SET_TARGET_PROPERTIES(arrow_table PROPERTIES LINKER_LANGUAGE CXX)
 
 # Headers: top level
 install(FILES
+  column.h
+  schema.h
+  table.h
   DESTINATION include/arrow/table)
 
 ADD_ARROW_TEST(column-test)
diff --git a/python/.gitignore b/python/.gitignore
new file mode 100644
index 0000000000000..80103a1a52942
--- /dev/null
+++ b/python/.gitignore
@@ -0,0 +1,37 @@
+thirdparty/
+CMakeFiles/
+CMakeCache.txt
+CTestTestfile.cmake
+Makefile
+cmake_install.cmake
+build/
+Testing/
+
+# Python stuff
+
+# Editor temporary/working/backup files
+*flymake*
+
+# Compiled source
+*.a
+*.dll
+*.o
+*.py[ocd]
+*.so
+.build_cache_dir
+MANIFEST
+
+# Generated sources
+*.c
+*.cpp
+# Python files
+
+# setup.py working directory
+build
+# setup.py dist directory
+dist
+# Egg metadata
+*.egg-info
+# coverage
+.coverage
+coverage.xml
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
new file mode 100644
index 0000000000000..df55bfac9eb4a
--- /dev/null
+++ b/python/CMakeLists.txt
@@ -0,0 +1,464 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Includes code assembled from BSD/MIT/Apache-licensed code from some 3rd-party
+# projects, including Kudu, Impala, and libdynd. See python/LICENSE.txt
+
+cmake_minimum_required(VERSION 2.7)
+project(pyarrow)
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
+
+# Use common cmake modules from Arrow C++ if available
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../cpp/cmake_modules")
+
+include(CMakeParseArguments)
+
+set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cpp/build-support)
+
+# Allow "make install" to not depend on all targets.
+#
+# Must be declared in the top-level CMakeLists.txt.
+set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
+
+set(CMAKE_MACOSX_RPATH 1)
+set(CMAKE_OSX_DEPLOYMENT_TARGET 10.9)
+
+# Generate a Clang compile_commands.json "compilation database" file for use
+# with various development tools, such as Vim's YouCompleteMe plugin.
+# See http://clang.llvm.org/docs/JSONCompilationDatabase.html
+if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
+  set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
+endif()
+
+############################################################
+# Compiler flags
+############################################################
+
+# compiler flags that are common across debug/release builds
+set(CXX_COMMON_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall")
+
+# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
+# For all builds:
+# For CMAKE_BUILD_TYPE=Debug
+#   -ggdb: Enable gdb debugging
+# For CMAKE_BUILD_TYPE=FastDebug
+#   Same as DEBUG, except with some optimizations on.
+# For CMAKE_BUILD_TYPE=Release
+#   -O3: Enable all compiler optimizations
+#   -g: Enable symbols for profiler tools (TODO: remove for shipping)
+#   -DNDEBUG: Turn off dchecks/asserts/debug only code.
+set(CXX_FLAGS_DEBUG "-ggdb -O0")
+set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
+set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
+
+# if no build build type is specified, default to debug builds
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Debug)
+endif(NOT CMAKE_BUILD_TYPE)
+
+string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
+
+# Set compile flags based on the build type.
+message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_DEBUG})
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
+  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_FASTDEBUG})
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
+  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE})
+else()
+  message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
+endif ()
+
+# Add common flags
+set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
+
+# Determine compiler version
+include(CompilerInfo)
+
+if ("${COMPILER_FAMILY}" STREQUAL "clang")
+  # Using Clang with ccache causes a bunch of spurious warnings that are
+  # purportedly fixed in the next version of ccache. See the following for details:
+  #
+  #   http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html
+  #   http://petereisentraut.blogspot.com/2011/09/ccache-and-clang-part-2.html
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
+endif()
+
+set(PYARROW_LINK "a")
+
+# For any C code, use the same flags.
+set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
+
+# Code coverage
+if ("${PYARROW_GENERATE_COVERAGE}")
+  if("${CMAKE_CXX_COMPILER}" MATCHES ".*clang.*")
+    # There appears to be some bugs in clang 3.3 which cause code coverage
+    # to have link errors, not locating the llvm_gcda_* symbols.
+    # This should be fixed in llvm 3.4 with http://llvm.org/viewvc/llvm-project?view=revision&revision=184666
+    message(SEND_ERROR "Cannot currently generate coverage with clang")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -DCOVERAGE_BUILD")
+
+  # For coverage to work properly, we need to use static linkage. Otherwise,
+  # __gcov_flush() doesn't properly flush coverage from every module.
+  # See http://stackoverflow.com/questions/28164543/using-gcov-flush-within-a-library-doesnt-force-the-other-modules-to-yield-gc
+  if("${PYARROW_LINK}" STREQUAL "a")
+    message("Using static linking for coverage build")
+    set(PYARROW_LINK "s")
+  elseif("${PYARROW_LINK}" STREQUAL "d")
+    message(SEND_ERROR "Cannot use coverage with static linking")
+  endif()
+endif()
+
+# If we still don't know what kind of linking to perform, choose based on
+# build type (developers like fast builds).
+if ("${PYARROW_LINK}" STREQUAL "a")
+  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
+      "${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
+    message("Using dynamic linking for ${CMAKE_BUILD_TYPE} builds")
+    set(PYARROW_LINK "d")
+  else()
+    message("Using static linking for ${CMAKE_BUILD_TYPE} builds")
+    set(PYARROW_LINK "s")
+  endif()
+endif()
+
+# Are we using the gold linker? It doesn't work with dynamic linking as
+# weak symbols aren't properly overridden, causing tcmalloc to be omitted.
+# Let's flag this as an error in RELEASE builds (we shouldn't release a
+# product like this).
+#
+# See https://sourceware.org/bugzilla/show_bug.cgi?id=16979 for details.
+#
+# The gold linker is only for ELF binaries, which OSX doesn't use. We can
+# just skip.
+if (NOT APPLE)
+  execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Wl,--version OUTPUT_VARIABLE LINKER_OUTPUT)
+endif ()
+if (LINKER_OUTPUT MATCHES "gold")
+  if ("${PYARROW_LINK}" STREQUAL "d" AND
+      "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
+    message(SEND_ERROR "Cannot use gold with dynamic linking in a RELEASE build "
+      "as it would cause tcmalloc symbols to get dropped")
+  else()
+    message("Using gold linker")
+  endif()
+  set(PYARROW_USING_GOLD 1)
+else()
+  message("Using ld linker")
+endif()
+
+# Having set PYARROW_LINK due to build type and/or sanitizer, it's now safe to
+# act on its value.
+if ("${PYARROW_LINK}" STREQUAL "d")
+  set(BUILD_SHARED_LIBS ON)
+
+  # Position independent code is only necessary when producing shared objects.
+  add_definitions(-fPIC)
+endif()
+
+# set compile output directory
+string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
+
+# If build in-source, create the latest symlink. If build out-of-source, which is
+# preferred, simply output the binaries in the build folder
+if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
+  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/${BUILD_SUBDIR_NAME}/")
+  # Link build/latest to the current build directory, to avoid developers
+  # accidentally running the latest debug build when in fact they're building
+  # release builds.
+  FILE(MAKE_DIRECTORY ${BUILD_OUTPUT_ROOT_DIRECTORY})
+  if (NOT APPLE)
+    set(MORE_ARGS "-T")
+  endif()
+EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
+  ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
+else()
+  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+  # set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
+endif()
+
+# where to put generated archives (.a files)
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
+# where to put generated libraries (.so files)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
+# where to put generated binaries
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
+## Python and libraries
+find_package(PythonLibsNew REQUIRED)
+include(UseCython)
+
+include_directories(SYSTEM
+  src)
+
+############################################################
+# Testing
+############################################################
+
+# Add a new test case, with or without an executable that should be built.
+#
+# REL_TEST_NAME is the name of the test. It may be a single component
+# (e.g. monotime-test) or contain additional components (e.g.
+# net/net_util-test). Either way, the last component must be a globally
+# unique name.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+function(ADD_PYARROW_TEST REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
+    # This test has a corresponding .cc file, set it up as an executable.
+    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+    add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
+    target_link_libraries(${TEST_NAME} ${PYARROW_TEST_LINK_LIBS})
+  else()
+    # No executable, just invoke the test (probably a script) directly.
+    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
+  endif()
+
+  add_test(${TEST_NAME}
+    ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
+  if(ARGN)
+    set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
+  endif()
+endfunction()
+
+# A wrapper for add_dependencies() that is compatible with NO_TESTS.
+function(ADD_PYARROW_TEST_DEPENDENCIES REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  add_dependencies(${TEST_NAME} ${ARGN})
+endfunction()
+
+enable_testing()
+
+############################################################
+# Dependencies
+############################################################
+function(ADD_THIRDPARTY_LIB LIB_NAME)
+  set(options)
+  set(one_value_args SHARED_LIB STATIC_LIB)
+  set(multi_value_args DEPS)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(("${PYARROW_LINK}" STREQUAL "s" AND ARG_STATIC_LIB) OR (NOT ARG_SHARED_LIB))
+    if(NOT ARG_STATIC_LIB)
+      message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
+    endif()
+    add_library(${LIB_NAME} STATIC IMPORTED)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
+  else()
+    add_library(${LIB_NAME} SHARED IMPORTED)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+  endif()
+
+  if(ARG_DEPS)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+  endif()
+
+  # Set up an "exported variant" for this thirdparty library (see "Visibility"
+  # above). It's the same as the real target, just with an "_exported" suffix.
+  # We prefer the static archive if it exists (as it's akin to an "internal"
+  # library), but we'll settle for the shared object if we must.
+  #
+  # A shared object exported variant will force any "leaf" library that
+  # transitively depends on it to also depend on it at runtime; this is
+  # desirable for some libraries (e.g. cyrus_sasl).
+  set(LIB_NAME_EXPORTED ${LIB_NAME}_exported)
+  if(ARG_STATIC_LIB)
+    add_library(${LIB_NAME_EXPORTED} STATIC IMPORTED)
+    set_target_properties(${LIB_NAME_EXPORTED}
+      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+  else()
+    add_library(${LIB_NAME_EXPORTED} SHARED IMPORTED)
+    set_target_properties(${LIB_NAME_EXPORTED}
+      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+  endif()
+  if(ARG_DEPS)
+    set_target_properties(${LIB_NAME_EXPORTED}
+      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+  endif()
+endfunction()
+
+## GMock
+find_package(GTest REQUIRED)
+include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(gtest
+  STATIC_LIB ${GTEST_STATIC_LIB})
+
+## Arrow
+find_package(Arrow REQUIRED)
+include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(arrow
+  SHARED_LIB ${ARROW_SHARED_LIB})
+
+############################################################
+# Linker setup
+############################################################
+
+set(PYARROW_MIN_TEST_LIBS
+  pyarrow_test_main
+  pyarrow)
+
+set(PYARROW_MIN_TEST_LIBS
+  pyarrow_test_main
+  pyarrow
+  ${PYARROW_BASE_LIBS})
+
+set(PYARROW_TEST_LINK_LIBS ${PYARROW_MIN_TEST_LIBS})
+
+############################################################
+# "make ctags" target
+############################################################
+if (UNIX)
+  add_custom_target(ctags ctags -R --languages=c++,c --exclude=thirdparty/installed)
+endif (UNIX)
+
+############################################################
+# "make etags" target
+############################################################
+if (UNIX)
+  add_custom_target(tags etags --members --declarations
+  `find ${CMAKE_CURRENT_SOURCE_DIR}/src
+   -name \\*.cc -or -name \\*.hh -or -name \\*.cpp -or -name \\*.h -or -name \\*.c -or
+   -name \\*.f`)
+  add_custom_target(etags DEPENDS tags)
+endif (UNIX)
+
+############################################################
+# "make cscope" target
+############################################################
+if (UNIX)
+  add_custom_target(cscope find ${CMAKE_CURRENT_SOURCE_DIR}
+  ( -name \\*.cc -or -name \\*.hh -or -name \\*.cpp -or
+    -name \\*.h -or -name \\*.c -or -name \\*.f )
+  -exec echo \"{}\" \; > cscope.files && cscope -q -b VERBATIM)
+endif (UNIX)
+
+############################################################
+# "make lint" target
+############################################################
+if (UNIX)
+  # Full lint
+  add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
+  --verbose=2
+  --filter=-whitespace/comments,-readability/todo,-build/header_guard
+    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h`)
+endif (UNIX)
+
+############################################################
+# Subdirectories
+############################################################
+
+add_subdirectory(src/pyarrow)
+add_subdirectory(src/pyarrow/util)
+
+set(PYARROW_SRCS
+  src/pyarrow/init.cc
+)
+
+set(LINK_LIBS
+  pyarrow_util
+  arrow
+)
+
+add_library(pyarrow SHARED
+  ${PYARROW_SRCS})
+target_link_libraries(pyarrow ${LINK_LIBS})
+set_target_properties(pyarrow PROPERTIES LINKER_LANGUAGE CXX)
+
+if(APPLE)
+  set_target_properties(pyarrow PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+endif()
+
+############################################################
+# Setup and build Cython modules
+############################################################
+
+foreach(pyx_api_file
+    arrow/config.pyx
+    arrow/parquet.pyx)
+    set_source_files_properties(${pyx_api_file} PROPERTIES CYTHON_API 1)
+endforeach(pyx_api_file)
+
+set(USE_RELATIVE_RPATH ON)
+set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+
+set(CYTHON_EXTENSIONS
+  config
+  parquet
+)
+
+foreach(module ${CYTHON_EXTENSIONS})
+    string(REPLACE "." ";" directories ${module})
+    list(GET directories -1 module_name)
+    list(REMOVE_AT directories -1)
+
+    string(REPLACE "." "/" module_root "${module}")
+    set(module_SRC arrow/${module_root}.pyx)
+    set_source_files_properties(${module_SRC} PROPERTIES CYTHON_IS_CXX 1)
+
+    cython_add_module(${module_name}
+      ${module_name}_pyx
+      ${module_name}_output
+      ${module_SRC})
+
+    if (directories)
+        string(REPLACE ";" "/" module_output_directory ${directories})
+        set_target_properties(${module_name} PROPERTIES
+          LIBRARY_OUTPUT_DIRECTORY ${module_output_directory})
+    endif()
+
+    if(APPLE)
+        set(module_install_rpath "@loader_path")
+    else()
+        set(module_install_rpath "$ORIGIN")
+    endif()
+    list(LENGTH directories i)
+    while(${i} GREATER 0)
+        set(module_install_rpath "${module_install_rpath}/..")
+        math(EXPR i "${i} - 1" )
+    endwhile(${i} GREATER 0)
+
+    # for inplace development for now
+    set(module_install_rpath "${CMAKE_SOURCE_DIR}/arrow/")
+
+    set_target_properties(${module_name} PROPERTIES
+      INSTALL_RPATH ${module_install_rpath})
+    target_link_libraries(${module_name} pyarrow)
+endforeach(module)
diff --git a/python/LICENSE.txt b/python/LICENSE.txt
new file mode 100644
index 0000000000000..078e144ded1c1
--- /dev/null
+++ b/python/LICENSE.txt
@@ -0,0 +1,88 @@
+## 3rd-party licenses for code that has been adapted for the Arrow Python
+   library
+
+-------------------------------------------------------------------------------
+Some code from pandas has been adapted for this codebase. pandas is available
+under the 3-clause BSD license, which follows:
+
+pandas license
+==============
+
+Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team
+All rights reserved.
+
+Copyright (c) 2008-2011 AQR Capital Management, LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+       copyright notice, this list of conditions and the following
+       disclaimer in the documentation and/or other materials provided
+       with the distribution.
+
+    * Neither the name of the copyright holder nor the names of any
+       contributors may be used to endorse or promote products derived
+       from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+-------------------------------------------------------------------------------
+
+Some bits from DyND, in particular aspects of the build system, have been
+adapted from libdynd and dynd-python under the terms of the BSD 2-clause
+license
+
+The BSD 2-Clause License
+
+    Copyright (C) 2011-12, Dynamic NDArray Developers
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+
+        * Redistributions of source code must retain the above copyright
+           notice, this list of conditions and the following disclaimer.
+
+        * Redistributions in binary form must reproduce the above
+           copyright notice, this list of conditions and the following
+           disclaimer in the documentation and/or other materials provided
+           with the distribution.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Dynamic NDArray Developers list:
+
+ * Mark Wiebe
+ * Continuum Analytics
+
+-------------------------------------------------------------------------------
+
+Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted
+for Arrow. Ibis is released under the Apache License, Version 2.0.
diff --git a/python/README.md b/python/README.md
new file mode 100644
index 0000000000000..c79fa9786f476
--- /dev/null
+++ b/python/README.md
@@ -0,0 +1,14 @@
+## Python library for Apache Arrow
+
+This library provides a Pythonic API wrapper for the reference Arrow C++
+implementation, along with tools for interoperability with pandas, NumPy, and
+other traditional Python scientific computing packages.
+
+#### Development details
+
+This project is layered in two pieces:
+
+* pyarrow, a C++ library for easier interoperability between Arrow C++, NumPy,
+  and pandas
+* Cython extensions and pure Python code under arrow/ which expose Arrow C++
+  and pyarrow to pure Python users
\ No newline at end of file
diff --git a/python/arrow/__init__.py b/python/arrow/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/python/arrow/compat.py b/python/arrow/compat.py
new file mode 100644
index 0000000000000..2ac41ac8abf89
--- /dev/null
+++ b/python/arrow/compat.py
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# flake8: noqa
+
+import itertools
+
+import numpy as np
+
+import sys
+import six
+from six import BytesIO, StringIO, string_types as py_string
+
+
+PY26 = sys.version_info[:2] == (2, 6)
+PY2 = sys.version_info[0] == 2
+
+
+if PY26:
+    import unittest2 as unittest
+else:
+    import unittest
+
+
+if PY2:
+    import cPickle
+
+    try:
+        from cdecimal import Decimal
+    except ImportError:
+        from decimal import Decimal
+
+    unicode_type = unicode
+    lzip = zip
+    zip = itertools.izip
+
+    def dict_values(x):
+        return x.values()
+
+    range = xrange
+    long = long
+
+    def tobytes(o):
+        if isinstance(o, unicode):
+            return o.encode('utf8')
+        else:
+            return o
+
+    def frombytes(o):
+        return o
+else:
+    unicode_type = str
+    def lzip(*x):
+        return list(zip(*x))
+    long = int
+    zip = zip
+    def dict_values(x):
+        return list(x.values())
+    from decimal import Decimal
+    range = range
+
+    def tobytes(o):
+        if isinstance(o, str):
+            return o.encode('utf8')
+        else:
+            return o
+
+    def frombytes(o):
+        return o.decode('utf8')
+
+
+integer_types = six.integer_types + (np.integer,)
diff --git a/python/arrow/config.pyx b/python/arrow/config.pyx
new file mode 100644
index 0000000000000..8f10beb3a2e72
--- /dev/null
+++ b/python/arrow/config.pyx
@@ -0,0 +1,8 @@
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+cdef extern from 'pyarrow/init.h' namespace 'arrow::py':
+    void pyarrow_init()
+
+pyarrow_init()
diff --git a/python/arrow/includes/__init__.pxd b/python/arrow/includes/__init__.pxd
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/python/arrow/includes/arrow.pxd b/python/arrow/includes/arrow.pxd
new file mode 100644
index 0000000000000..3635ceb868596
--- /dev/null
+++ b/python/arrow/includes/arrow.pxd
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from arrow.includes.common cimport *
+
+cdef extern from "arrow/api.h" namespace "arrow" nogil:
+    pass
diff --git a/python/arrow/includes/common.pxd b/python/arrow/includes/common.pxd
new file mode 100644
index 0000000000000..f2fc826625e45
--- /dev/null
+++ b/python/arrow/includes/common.pxd
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from libc.stdint cimport *
+from libcpp cimport bool as c_bool
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+
+# This must be included for cerr and other things to work
+cdef extern from "<iostream>":
+    pass
+
+cdef extern from "<memory>" namespace "std" nogil:
+
+    cdef cppclass shared_ptr[T]:
+        T* get()
+        void reset()
+        void reset(T* p)
diff --git a/python/arrow/includes/parquet.pxd b/python/arrow/includes/parquet.pxd
new file mode 100644
index 0000000000000..62342f3066969
--- /dev/null
+++ b/python/arrow/includes/parquet.pxd
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from arrow.includes.common cimport *
+
+cdef extern from "parquet/api/reader.h" namespace "parquet_cpp" nogil:
+    cdef cppclass ColumnReader:
+        pass
+
+    cdef cppclass BoolReader(ColumnReader):
+        pass
+
+    cdef cppclass Int32Reader(ColumnReader):
+        pass
+
+    cdef cppclass Int64Reader(ColumnReader):
+        pass
+
+    cdef cppclass Int96Reader(ColumnReader):
+        pass
+
+    cdef cppclass FloatReader(ColumnReader):
+        pass
+
+    cdef cppclass DoubleReader(ColumnReader):
+        pass
+
+    cdef cppclass ByteArrayReader(ColumnReader):
+        pass
+
+    cdef cppclass RowGroupReader:
+        pass
+
+    cdef cppclass ParquetFileReader:
+        pass
diff --git a/python/arrow/includes/pyarrow.pxd b/python/arrow/includes/pyarrow.pxd
new file mode 100644
index 0000000000000..dcef663f3894d
--- /dev/null
+++ b/python/arrow/includes/pyarrow.pxd
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from arrow.includes.common cimport *
+
+cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
+    pass
diff --git a/python/arrow/parquet.pyx b/python/arrow/parquet.pyx
new file mode 100644
index 0000000000000..23c3838bcad1f
--- /dev/null
+++ b/python/arrow/parquet.pyx
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from arrow.compat import frombytes, tobytes
+from arrow.includes.parquet cimport *
diff --git a/python/arrow/tests/__init__.py b/python/arrow/tests/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/python/cmake_modules/CompilerInfo.cmake b/python/cmake_modules/CompilerInfo.cmake
new file mode 100644
index 0000000000000..e66bc2693eead
--- /dev/null
+++ b/python/cmake_modules/CompilerInfo.cmake
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Sets COMPILER_FAMILY to 'clang' or 'gcc'
+# Sets COMPILER_VERSION to the version
+execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
+                ERROR_VARIABLE COMPILER_VERSION_FULL)
+message(INFO " ${COMPILER_VERSION_FULL}")
+
+# clang on Linux and Mac OS X before 10.9
+if("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
+  set(COMPILER_FAMILY "clang")
+  string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
+    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+# clang on Mac OS X 10.9 and later
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
+  set(COMPILER_FAMILY "clang")
+  string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
+    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+
+# clang on Mac OS X, XCode 7. No version replacement is done
+# because Apple no longer advertises the upstream LLVM version.
+elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-700\\..*")
+  set(COMPILER_FAMILY "clang")
+
+# gcc
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*gcc version.*")
+  set(COMPILER_FAMILY "gcc")
+  string(REGEX REPLACE ".*gcc version ([0-9\\.]+).*" "\\1"
+    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+else()
+  message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
+endif()
+message("Selected compiler ${COMPILER_FAMILY} ${COMPILER_VERSION}")
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
new file mode 100644
index 0000000000000..3d9983849ebb2
--- /dev/null
+++ b/python/cmake_modules/FindArrow.cmake
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find ARROW (arrow/api.h, libarrow.a, libarrow.so)
+# This module defines
+#  ARROW_INCLUDE_DIR, directory containing headers
+#  ARROW_LIBS, directory containing arrow libraries
+#  ARROW_STATIC_LIB, path to libarrow.a
+#  ARROW_SHARED_LIB, path to libarrow's shared library
+#  ARROW_FOUND, whether arrow has been found
+
+set(ARROW_SEARCH_HEADER_PATHS
+  $ENV{ARROW_HOME}/include
+)
+
+set(ARROW_SEARCH_LIB_PATH
+  $ENV{ARROW_HOME}/lib
+)
+
+find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
+  ${ARROW_SEARCH_HEADER_PATHS}
+  # make sure we don't accidentally pick up a different version
+  NO_DEFAULT_PATH
+)
+
+find_library(ARROW_LIB_PATH NAMES arrow
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+
+if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
+  set(ARROW_FOUND TRUE)
+  set(ARROW_LIB_NAME libarrow)
+  set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
+  set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
+  set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+else ()
+  set(ARROW_FOUND FALSE)
+endif ()
+
+if (ARROW_FOUND)
+  if (NOT Arrow_FIND_QUIETLY)
+    message(STATUS "Found the Arrow library: ${ARROW_LIB_PATH}")
+  endif ()
+else ()
+  if (NOT Arrow_FIND_QUIETLY)
+    set(ARROW_ERR_MSG "Could not find the Arrow library. Looked for headers")
+    set(ARROW_ERR_MSG "${ARROW_ERR_MSG} in ${ARROW_SEARCH_HEADER_PATHS}, and for libs")
+    set(ARROW_ERR_MSG "${ARROW_ERR_MSG} in ${ARROW_SEARCH_LIB_PATH}")
+    if (Arrow_FIND_REQUIRED)
+      message(FATAL_ERROR "${ARROW_ERR_MSG}")
+    else (Arrow_FIND_REQUIRED)
+      message(STATUS "${ARROW_ERR_MSG}")
+    endif (Arrow_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  ARROW_INCLUDE_DIR
+  ARROW_LIBS
+  ARROW_STATIC_LIB
+  ARROW_SHARED_LIB
+)
diff --git a/python/cmake_modules/FindCython.cmake b/python/cmake_modules/FindCython.cmake
new file mode 100644
index 0000000000000..9df3b5d59d274
--- /dev/null
+++ b/python/cmake_modules/FindCython.cmake
@@ -0,0 +1,30 @@
+# Find the Cython compiler.
+#
+# This code sets the following variables:
+#
+#  CYTHON_EXECUTABLE
+#
+# See also UseCython.cmake
+
+#=============================================================================
+# Copyright 2011 Kitware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+find_program( CYTHON_EXECUTABLE NAMES cython cython.bat )
+
+include( FindPackageHandleStandardArgs )
+FIND_PACKAGE_HANDLE_STANDARD_ARGS( Cython REQUIRED_VARS CYTHON_EXECUTABLE )
+
+mark_as_advanced( CYTHON_EXECUTABLE )
diff --git a/python/cmake_modules/FindNumPy.cmake b/python/cmake_modules/FindNumPy.cmake
new file mode 100644
index 0000000000000..58bb531f5324a
--- /dev/null
+++ b/python/cmake_modules/FindNumPy.cmake
@@ -0,0 +1,100 @@
+# - Find the NumPy libraries
+# This module finds if NumPy is installed, and sets the following variables
+# indicating where it is.
+#
+# TODO: Update to provide the libraries and paths for linking npymath lib.
+#
+#  NUMPY_FOUND               - was NumPy found
+#  NUMPY_VERSION             - the version of NumPy found as a string
+#  NUMPY_VERSION_MAJOR       - the major version number of NumPy
+#  NUMPY_VERSION_MINOR       - the minor version number of NumPy
+#  NUMPY_VERSION_PATCH       - the patch version number of NumPy
+#  NUMPY_VERSION_DECIMAL     - e.g. version 1.6.1 is 10601
+#  NUMPY_INCLUDE_DIRS        - path to the NumPy include files
+
+#============================================================================
+# Copyright 2012 Continuum Analytics, Inc.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+#============================================================================
+
+# Finding NumPy involves calling the Python interpreter
+if(NumPy_FIND_REQUIRED)
+    find_package(PythonInterp REQUIRED)
+else()
+    find_package(PythonInterp)
+endif()
+
+if(NOT PYTHONINTERP_FOUND)
+    set(NUMPY_FOUND FALSE)
+    return()
+endif()
+
+execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
+    "import numpy as n; print(n.__version__); print(n.get_include());"
+    RESULT_VARIABLE _NUMPY_SEARCH_SUCCESS
+    OUTPUT_VARIABLE _NUMPY_VALUES_OUTPUT
+    ERROR_VARIABLE _NUMPY_ERROR_VALUE
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if(NOT _NUMPY_SEARCH_SUCCESS MATCHES 0)
+    if(NumPy_FIND_REQUIRED)
+        message(FATAL_ERROR
+            "NumPy import failure:\n${_NUMPY_ERROR_VALUE}")
+    endif()
+    set(NUMPY_FOUND FALSE)
+    return()
+endif()
+
+# Convert the process output into a list
+string(REGEX REPLACE ";" "\\\\;" _NUMPY_VALUES ${_NUMPY_VALUES_OUTPUT})
+string(REGEX REPLACE "\n" ";" _NUMPY_VALUES ${_NUMPY_VALUES})
+list(GET _NUMPY_VALUES 0 NUMPY_VERSION)
+list(GET _NUMPY_VALUES 1 NUMPY_INCLUDE_DIRS)
+
+string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" _VER_CHECK "${NUMPY_VERSION}")
+if("${_VER_CHECK}" STREQUAL "")
+    # The output from Python was unexpected. Raise an error always
+    # here, because we found NumPy, but it appears to be corrupted somehow.
+    message(FATAL_ERROR
+        "Requested version and include path from NumPy, got instead:\n${_NUMPY_VALUES_OUTPUT}\n")
+    return()
+endif()
+
+# Make sure all directory separators are '/'
+string(REGEX REPLACE "\\\\" "/" NUMPY_INCLUDE_DIRS ${NUMPY_INCLUDE_DIRS})
+
+# Get the major and minor version numbers
+string(REGEX REPLACE "\\." ";" _NUMPY_VERSION_LIST ${NUMPY_VERSION})
+list(GET _NUMPY_VERSION_LIST 0 NUMPY_VERSION_MAJOR)
+list(GET _NUMPY_VERSION_LIST 1 NUMPY_VERSION_MINOR)
+list(GET _NUMPY_VERSION_LIST 2 NUMPY_VERSION_PATCH)
+string(REGEX MATCH "[0-9]*" NUMPY_VERSION_PATCH ${NUMPY_VERSION_PATCH})
+math(EXPR NUMPY_VERSION_DECIMAL
+    "(${NUMPY_VERSION_MAJOR} * 10000) + (${NUMPY_VERSION_MINOR} * 100) + ${NUMPY_VERSION_PATCH}")
+
+find_package_message(NUMPY
+    "Found NumPy: version \"${NUMPY_VERSION}\" ${NUMPY_INCLUDE_DIRS}"
+    "${NUMPY_INCLUDE_DIRS}${NUMPY_VERSION}")
+
+set(NUMPY_FOUND TRUE)
diff --git a/python/cmake_modules/FindPythonLibsNew.cmake b/python/cmake_modules/FindPythonLibsNew.cmake
new file mode 100644
index 0000000000000..c70e6bc26a719
--- /dev/null
+++ b/python/cmake_modules/FindPythonLibsNew.cmake
@@ -0,0 +1,236 @@
+# - Find python libraries
+# This module finds the libraries corresponding to the Python interpeter
+# FindPythonInterp provides.
+# This code sets the following variables:
+#
+#  PYTHONLIBS_FOUND           - have the Python libs been found
+#  PYTHON_PREFIX              - path to the Python installation
+#  PYTHON_LIBRARIES           - path to the python library
+#  PYTHON_INCLUDE_DIRS        - path to where Python.h is found
+#  PYTHON_SITE_PACKAGES       - path to installation site-packages
+#  PYTHON_IS_DEBUG            - whether the Python interpreter is a debug build
+#
+#  PYTHON_INCLUDE_PATH        - path to where Python.h is found (deprecated)
+#
+# A function PYTHON_ADD_MODULE(<name> src1 src2 ... srcN) is defined
+# to build modules for python.
+#
+# Thanks to talljimbo for the patch adding the 'LDVERSION' config
+# variable usage.
+
+#=============================================================================
+# Copyright 2001-2009 Kitware, Inc.
+# Copyright 2012-2014 Continuum Analytics, Inc.
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the names of Kitware, Inc., the Insight Software Consortium,
+# nor the names of their contributors may be used to endorse or promote
+# products derived from this software without specific prior written
+# permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+# Use the Python interpreter to find the libs.
+if(PythonLibsNew_FIND_REQUIRED)
+    find_package(PythonInterp REQUIRED)
+else()
+    find_package(PythonInterp)
+endif()
+
+if(NOT PYTHONINTERP_FOUND)
+    set(PYTHONLIBS_FOUND FALSE)
+    return()
+endif()
+
+# According to http://stackoverflow.com/questions/646518/python-how-to-detect-debug-interpreter
+# testing whether sys has the gettotalrefcount function is a reliable,
+# cross-platform way to detect a CPython debug interpreter.
+#
+# The library suffix is from the config var LDVERSION sometimes, otherwise
+# VERSION. VERSION will typically be like "2.7" on unix, and "27" on windows.
+#
+# The config var LIBPL is for Linux, and helps on Debian Jessie where the
+# addition of multi-arch support shuffled things around.
+execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
+    "from distutils import sysconfig as s;import sys;import struct;
+print('.'.join(str(v) for v in sys.version_info));
+print(sys.prefix);
+print(s.get_python_inc(plat_specific=True));
+print(s.get_python_lib(plat_specific=True));
+print(s.get_config_var('SO'));
+print(hasattr(sys, 'gettotalrefcount')+0);
+print(struct.calcsize('@P'));
+print(s.get_config_var('LDVERSION') or s.get_config_var('VERSION'));
+print(s.get_config_var('LIBPL'));
+"
+    RESULT_VARIABLE _PYTHON_SUCCESS
+    OUTPUT_VARIABLE _PYTHON_VALUES
+    ERROR_VARIABLE _PYTHON_ERROR_VALUE
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if(NOT _PYTHON_SUCCESS MATCHES 0)
+    if(PythonLibsNew_FIND_REQUIRED)
+        message(FATAL_ERROR
+            "Python config failure:\n${_PYTHON_ERROR_VALUE}")
+    endif()
+    set(PYTHONLIBS_FOUND FALSE)
+    return()
+endif()
+
+# Convert the process output into a list
+string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES})
+string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES})
+list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST)
+list(GET _PYTHON_VALUES 1 PYTHON_PREFIX)
+list(GET _PYTHON_VALUES 2 PYTHON_INCLUDE_DIR)
+list(GET _PYTHON_VALUES 3 PYTHON_SITE_PACKAGES)
+list(GET _PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION)
+list(GET _PYTHON_VALUES 5 PYTHON_IS_DEBUG)
+list(GET _PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P)
+list(GET _PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX)
+list(GET _PYTHON_VALUES 8 PYTHON_LIBRARY_PATH)
+
+# Make sure the Python has the same pointer-size as the chosen compiler
+# Skip the check on OS X, it doesn't consistently have CMAKE_SIZEOF_VOID_P defined
+if((NOT APPLE) AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}"))
+    if(PythonLibsNew_FIND_REQUIRED)
+        math(EXPR _PYTHON_BITS "${PYTHON_SIZEOF_VOID_P} * 8")
+        math(EXPR _CMAKE_BITS "${CMAKE_SIZEOF_VOID_P} * 8")
+        message(FATAL_ERROR
+            "Python config failure: Python is ${_PYTHON_BITS}-bit, "
+            "chosen compiler is  ${_CMAKE_BITS}-bit")
+    endif()
+    set(PYTHONLIBS_FOUND FALSE)
+    return()
+endif()
+
+# The built-in FindPython didn't always give the version numbers
+string(REGEX REPLACE "\\." ";" _PYTHON_VERSION_LIST ${_PYTHON_VERSION_LIST})
+list(GET _PYTHON_VERSION_LIST 0 PYTHON_VERSION_MAJOR)
+list(GET _PYTHON_VERSION_LIST 1 PYTHON_VERSION_MINOR)
+list(GET _PYTHON_VERSION_LIST 2 PYTHON_VERSION_PATCH)
+
+# Make sure all directory separators are '/'
+string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX ${PYTHON_PREFIX})
+string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR ${PYTHON_INCLUDE_DIR})
+string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES})
+
+if(CMAKE_HOST_WIN32)
+    if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
+        set(PYTHON_LIBRARY
+            "${PYTHON_PREFIX}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib")
+    else()
+        set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/libpython${PYTHON_LIBRARY_SUFFIX}.a")
+    endif()
+elseif(APPLE)
+     # Seems to require "-undefined dynamic_lookup" instead of linking
+     # against the .dylib, otherwise it crashes. This flag is added
+     # below
+    set(PYTHON_LIBRARY "")
+    #set(PYTHON_LIBRARY
+    #    "${PYTHON_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
+else()
+    if(${PYTHON_SIZEOF_VOID_P} MATCHES 8)
+        set(_PYTHON_LIBS_SEARCH "${PYTHON_PREFIX}/lib64" "${PYTHON_PREFIX}/lib" "${PYTHON_LIBRARY_PATH}")
+    else()
+        set(_PYTHON_LIBS_SEARCH "${PYTHON_PREFIX}/lib" "${PYTHON_LIBRARY_PATH}")
+    endif()
+    message(STATUS "Searching for Python libs in ${_PYTHON_LIBS_SEARCH}")
+    # Probably this needs to be more involved. It would be nice if the config
+    # information the python interpreter itself gave us were more complete.
+    find_library(PYTHON_LIBRARY
+        NAMES "python${PYTHON_LIBRARY_SUFFIX}"
+        PATHS ${_PYTHON_LIBS_SEARCH}
+        NO_DEFAULT_PATH)
+    message(STATUS "Found Python lib ${PYTHON_LIBRARY}")
+endif()
+
+# For backward compatibility, set PYTHON_INCLUDE_PATH, but make it internal.
+SET(PYTHON_INCLUDE_PATH "${PYTHON_INCLUDE_DIR}" CACHE INTERNAL
+          "Path to where Python.h is found (deprecated)")
+
+MARK_AS_ADVANCED(
+  PYTHON_LIBRARY
+  PYTHON_INCLUDE_DIR
+)
+
+# We use PYTHON_INCLUDE_DIR, PYTHON_LIBRARY and PYTHON_DEBUG_LIBRARY for the
+# cache entries because they are meant to specify the location of a single
+# library. We now set the variables listed by the documentation for this
+# module.
+SET(PYTHON_INCLUDE_DIRS "${PYTHON_INCLUDE_DIR}")
+SET(PYTHON_LIBRARIES "${PYTHON_LIBRARY}")
+SET(PYTHON_DEBUG_LIBRARIES "${PYTHON_DEBUG_LIBRARY}")
+
+
+# Don't know how to get to this directory, just doing something simple :P
+#INCLUDE(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake)
+#FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonLibs DEFAULT_MSG PYTHON_LIBRARIES PYTHON_INCLUDE_DIRS)
+find_package_message(PYTHON
+    "Found PythonLibs: ${PYTHON_LIBRARY}"
+    "${PYTHON_EXECUTABLE}${PYTHON_VERSION}")
+
+
+# PYTHON_ADD_MODULE(<name> src1 src2 ... srcN) is used to build modules for python.
+FUNCTION(PYTHON_ADD_MODULE _NAME )
+  GET_PROPERTY(_TARGET_SUPPORTS_SHARED_LIBS
+    GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS)
+  OPTION(PYTHON_ENABLE_MODULE_${_NAME} "Add module ${_NAME}" TRUE)
+  OPTION(PYTHON_MODULE_${_NAME}_BUILD_SHARED
+    "Add module ${_NAME} shared" ${_TARGET_SUPPORTS_SHARED_LIBS})
+
+  # Mark these options as advanced
+  MARK_AS_ADVANCED(PYTHON_ENABLE_MODULE_${_NAME}
+    PYTHON_MODULE_${_NAME}_BUILD_SHARED)
+
+  IF(PYTHON_ENABLE_MODULE_${_NAME})
+    IF(PYTHON_MODULE_${_NAME}_BUILD_SHARED)
+      SET(PY_MODULE_TYPE MODULE)
+    ELSE(PYTHON_MODULE_${_NAME}_BUILD_SHARED)
+      SET(PY_MODULE_TYPE STATIC)
+      SET_PROPERTY(GLOBAL  APPEND  PROPERTY  PY_STATIC_MODULES_LIST ${_NAME})
+    ENDIF(PYTHON_MODULE_${_NAME}_BUILD_SHARED)
+
+    SET_PROPERTY(GLOBAL  APPEND  PROPERTY  PY_MODULES_LIST ${_NAME})
+    ADD_LIBRARY(${_NAME} ${PY_MODULE_TYPE} ${ARGN})
+    IF(APPLE)
+      # On OS X, linking against the Python libraries causes
+      # segfaults, so do this dynamic lookup instead.
+      SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS
+                          "-undefined dynamic_lookup")
+    ELSE()
+      TARGET_LINK_LIBRARIES(${_NAME} ${PYTHON_LIBRARIES})
+    ENDIF()
+    IF(PYTHON_MODULE_${_NAME}_BUILD_SHARED)
+      SET_TARGET_PROPERTIES(${_NAME} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}")
+      SET_TARGET_PROPERTIES(${_NAME} PROPERTIES SUFFIX "${PYTHON_MODULE_EXTENSION}")
+    ELSE()
+    ENDIF()
+
+  ENDIF(PYTHON_ENABLE_MODULE_${_NAME})
+ENDFUNCTION(PYTHON_ADD_MODULE)
\ No newline at end of file
diff --git a/python/cmake_modules/UseCython.cmake b/python/cmake_modules/UseCython.cmake
new file mode 100644
index 0000000000000..e7034db52f335
--- /dev/null
+++ b/python/cmake_modules/UseCython.cmake
@@ -0,0 +1,164 @@
+# Define a function to create Cython modules.
+#
+# For more information on the Cython project, see http://cython.org/.
+# "Cython is a language that makes writing C extensions for the Python language
+# as easy as Python itself."
+#
+# This file defines a CMake function to build a Cython Python module.
+# To use it, first include this file.
+#
+#   include( UseCython )
+#
+# Then call cython_add_module to create a module.
+#
+#   cython_add_module( <target_name> <pyx_target_name> <output_files> <src1> <src2> ... <srcN> )
+#
+# Where <module_name> is the desired name of the target for the resulting Python module,
+# <pyx_target_name> is the desired name of the target that runs the Cython compiler
+# to generate the needed C or C++ files, <output_files> is a variable to hold the
+# files generated by Cython, and <src1> <src2> ... are source files
+# to be compiled into the module, e.g. *.pyx, *.c, *.cxx, etc.
+# only one .pyx file may be present for each target
+# (this is an inherent limitation of Cython).
+#
+# The sample paths set with the CMake include_directories() command will be used
+# for include directories to search for *.pxd when running the Cython complire.
+#
+# Cache variables that effect the behavior include:
+#
+#  CYTHON_ANNOTATE
+#  CYTHON_NO_DOCSTRINGS
+#  CYTHON_FLAGS
+#
+# Source file properties that effect the build process are
+#
+#  CYTHON_IS_CXX
+#  CYTHON_IS_PUBLIC
+#  CYTHON_IS_API
+#
+# If this is set of a *.pyx file with CMake set_source_files_properties()
+# command, the file will be compiled as a C++ file.
+#
+# See also FindCython.cmake
+
+#=============================================================================
+# Copyright 2011 Kitware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+# Configuration options.
+set( CYTHON_ANNOTATE OFF
+  CACHE BOOL "Create an annotated .html file when compiling *.pyx." )
+set( CYTHON_NO_DOCSTRINGS OFF
+  CACHE BOOL "Strip docstrings from the compiled module." )
+set( CYTHON_FLAGS "" CACHE STRING
+  "Extra flags to the cython compiler." )
+mark_as_advanced( CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS )
+
+find_package( Cython REQUIRED )
+find_package( PythonLibsNew REQUIRED )
+
+set( CYTHON_CXX_EXTENSION "cxx" )
+set( CYTHON_C_EXTENSION "c" )
+
+# Create a *.c or *.cxx file from a *.pyx file.
+# Input the generated file basename.  The generate files will put into the variable
+# placed in the "generated_files" argument. Finally all the *.py and *.pyx files.
+function( compile_pyx _name pyx_target_name generated_files pyx_file)
+  # Default to assuming all files are C.
+  set( cxx_arg "" )
+  set( extension ${CYTHON_C_EXTENSION} )
+  set( pyx_lang "C" )
+  set( comment "Compiling Cython C source for ${_name}..." )
+
+  get_filename_component( pyx_file_basename "${pyx_file}" NAME_WE )
+
+  # Determine if it is a C or C++ file.
+  get_source_file_property( property_is_cxx ${pyx_file} CYTHON_IS_CXX )
+  if( ${property_is_cxx} )
+    set( cxx_arg "--cplus" )
+    set( extension ${CYTHON_CXX_EXTENSION} )
+    set( pyx_lang "CXX" )
+    set( comment "Compiling Cython CXX source for ${_name}..." )
+  endif()
+  get_source_file_property( pyx_location ${pyx_file} LOCATION )
+
+  # Set additional flags.
+  if( CYTHON_ANNOTATE )
+    set( annotate_arg "--annotate" )
+  endif()
+
+  if( CYTHON_NO_DOCSTRINGS )
+    set( no_docstrings_arg "--no-docstrings" )
+  endif()
+
+  if(NOT WIN32)
+      if( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" OR
+            "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo" )
+          set( cython_debug_arg "--gdb" )
+      endif()
+  endif()
+
+  # Determining generated file names.
+  get_source_file_property( property_is_public ${pyx_file} CYTHON_PUBLIC )
+  get_source_file_property( property_is_api ${pyx_file} CYTHON_API )
+  if( ${property_is_api} )
+      set( _generated_files "${_name}.${extension}" "${_name}.h" "${name}_api.h")
+  elseif( ${property_is_public} )
+      set( _generated_files "${_name}.${extension}" "${_name}.h")
+  else()
+      set( _generated_files "${_name}.${extension}")
+  endif()
+  set_source_files_properties( ${_generated_files} PROPERTIES GENERATED TRUE )
+  set( ${generated_files} ${_generated_files} PARENT_SCOPE )
+
+  # Add the command to run the compiler.
+  add_custom_target(${pyx_target_name}
+    COMMAND ${CYTHON_EXECUTABLE} ${cxx_arg} ${include_directory_arg}
+    ${annotate_arg} ${no_docstrings_arg} ${cython_debug_arg} ${CYTHON_FLAGS}
+    --output-file "${_name}.${extension}" ${pyx_location}
+    DEPENDS ${pyx_location}
+    # do not specify byproducts for now since they don't work with the older
+    # version of cmake available in the apt repositories.
+    #BYPRODUCTS ${_generated_files}
+    COMMENT ${comment}
+    )
+
+  # Remove their visibility to the user.
+  set( corresponding_pxd_file "" CACHE INTERNAL "" )
+  set( header_location "" CACHE INTERNAL "" )
+  set( pxd_location "" CACHE INTERNAL "" )
+endfunction()
+
+# cython_add_module( <name> src1 src2 ... srcN )
+# Build the Cython Python module.
+function( cython_add_module _name pyx_target_name generated_files)
+  set( pyx_module_source "" )
+  set( other_module_sources "" )
+  foreach( _file ${ARGN} )
+    if( ${_file} MATCHES ".*\\.py[x]?$" )
+      list( APPEND pyx_module_source ${_file} )
+    else()
+      list( APPEND other_module_sources ${_file} )
+    endif()
+  endforeach()
+  compile_pyx( ${_name} ${pyx_target_name} _generated_files ${pyx_module_source} )
+  set( ${generated_files} ${_generated_files} PARENT_SCOPE )
+  include_directories( ${PYTHON_INCLUDE_DIRS} )
+  python_add_module( ${_name} ${_generated_files} ${other_module_sources} )
+  add_dependencies( ${_name} ${pyx_target_name})
+  target_link_libraries( ${_name} ${PYTHON_LIBRARIES} )
+endfunction()
+
+include( CMakeParseArguments )
diff --git a/python/setup.py b/python/setup.py
new file mode 100644
index 0000000000000..f6b0a4bee8316
--- /dev/null
+++ b/python/setup.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import glob
+import os.path as osp
+import re
+import shutil
+from Cython.Distutils import build_ext as _build_ext
+import Cython
+
+import sys
+
+import pkg_resources
+from setuptools import setup
+
+import os
+
+from os.path import join as pjoin
+
+from distutils.command.clean import clean as _clean
+from distutils import sysconfig
+
+# Check if we're running 64-bit Python
+is_64_bit = sys.maxsize > 2**32
+
+# Check if this is a debug build of Python.
+if hasattr(sys, 'gettotalrefcount'):
+    build_type = 'Debug'
+else:
+    build_type = 'Release'
+
+if Cython.__version__ < '0.19.1':
+    raise Exception('Please upgrade to Cython 0.19.1 or newer')
+
+MAJOR = 0
+MINOR = 1
+MICRO = 0
+VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)
+
+
+class clean(_clean):
+
+    def run(self):
+        _clean.run(self)
+        for x in []:
+            try:
+                os.remove(x)
+            except OSError:
+                pass
+
+
+class build_ext(_build_ext):
+
+    def build_extensions(self):
+        numpy_incl = pkg_resources.resource_filename('numpy', 'core/include')
+
+        for ext in self.extensions:
+            if (hasattr(ext, 'include_dirs') and
+                    numpy_incl not in ext.include_dirs):
+                ext.include_dirs.append(numpy_incl)
+        _build_ext.build_extensions(self)
+
+    def run(self):
+        self._run_cmake()
+        _build_ext.run(self)
+
+    # adapted from cmake_build_ext in dynd-python
+    # github.com/libdynd/dynd-python
+
+    description = "Build the C-extensions for arrow"
+    user_options = ([('extra-cmake-args=', None,
+                      'extra arguments for CMake')] +
+                    _build_ext.user_options)
+
+    def initialize_options(self):
+        _build_ext.initialize_options(self)
+        self.extra_cmake_args = ''
+
+    def _run_cmake(self):
+        # The directory containing this setup.py
+        source = osp.dirname(osp.abspath(__file__))
+
+        # The staging directory for the module being built
+        build_temp = pjoin(os.getcwd(), self.build_temp)
+
+        # Change to the build directory
+        saved_cwd = os.getcwd()
+        if not os.path.isdir(self.build_temp):
+            self.mkpath(self.build_temp)
+        os.chdir(self.build_temp)
+
+        # Detect if we built elsewhere
+        if os.path.isfile('CMakeCache.txt'):
+            cachefile = open('CMakeCache.txt', 'r')
+            cachedir = re.search('CMAKE_CACHEFILE_DIR:INTERNAL=(.*)',
+                                 cachefile.read()).group(1)
+            cachefile.close()
+            if (cachedir != build_temp):
+                return
+
+        pyexe_option = '-DPYTHON_EXECUTABLE=%s' % sys.executable
+        static_lib_option = ''
+        build_tests_option = ''
+
+        if sys.platform != 'win32':
+            cmake_command = ['cmake', self.extra_cmake_args, pyexe_option,
+                             build_tests_option,
+                             static_lib_option, source]
+
+            self.spawn(cmake_command)
+            self.spawn(['make'])
+        else:
+            import shlex
+            cmake_generator = 'Visual Studio 14 2015'
+            if is_64_bit:
+                cmake_generator += ' Win64'
+            # Generate the build files
+            extra_cmake_args = shlex.split(self.extra_cmake_args)
+            cmake_command = (['cmake'] + extra_cmake_args +
+                             [source, pyexe_option,
+                              static_lib_option,
+                              build_tests_option,
+                             '-G', cmake_generator])
+            if "-G" in self.extra_cmake_args:
+                cmake_command = cmake_command[:-2]
+
+            self.spawn(cmake_command)
+            # Do the build
+            self.spawn(['cmake', '--build', '.', '--config', build_type])
+
+        if self.inplace:
+            # a bit hacky
+            build_lib = saved_cwd
+        else:
+            build_lib = pjoin(os.getcwd(), self.build_lib)
+
+        # Move the built libpyarrow library to the place expected by the Python
+        # build
+        if sys.platform != 'win32':
+            name, = glob.glob('libpyarrow.*')
+            try:
+                os.makedirs(pjoin(build_lib, 'arrow'))
+            except OSError:
+                pass
+            shutil.move(name, pjoin(build_lib, 'arrow', name))
+        else:
+            shutil.move(pjoin(build_type, 'pyarrow.dll'),
+                        pjoin(build_lib, 'arrow', 'pyarrow.dll'))
+
+        # Move the built C-extension to the place expected by the Python build
+        self._found_names = []
+        for name in self.get_cmake_cython_names():
+            built_path = self.get_ext_built(name)
+            if not os.path.exists(built_path):
+                print(built_path)
+                raise RuntimeError('libpyarrow C-extension failed to build:',
+                                   os.path.abspath(built_path))
+
+            ext_path = pjoin(build_lib, self._get_cmake_ext_path(name))
+            if os.path.exists(ext_path):
+                os.remove(ext_path)
+            self.mkpath(os.path.dirname(ext_path))
+            print('Moving built libpyarrow C-extension', built_path,
+                  'to build path', ext_path)
+            shutil.move(self.get_ext_built(name), ext_path)
+            self._found_names.append(name)
+
+        os.chdir(saved_cwd)
+
+    def _get_inplace_dir(self):
+        pass
+
+    def _get_cmake_ext_path(self, name):
+        # Get the package directory from build_py
+        build_py = self.get_finalized_command('build_py')
+        package_dir = build_py.get_package_dir('arrow')
+        # This is the name of the arrow C-extension
+        suffix = sysconfig.get_config_var('EXT_SUFFIX')
+        if suffix is None:
+            suffix = sysconfig.get_config_var('SO')
+        filename = name + suffix
+        return pjoin(package_dir, filename)
+
+    def get_ext_built(self, name):
+        if sys.platform == 'win32':
+            head, tail = os.path.split(name)
+            suffix = sysconfig.get_config_var('SO')
+            return pjoin(head, build_type, tail + suffix)
+        else:
+            suffix = sysconfig.get_config_var('SO')
+            return name + suffix
+
+    def get_cmake_cython_names(self):
+        return ['config', 'parquet']
+
+    def get_names(self):
+        return self._found_names
+
+    def get_outputs(self):
+        # Just the C extensions
+        cmake_exts = [self._get_cmake_ext_path(name)
+                      for name in self.get_names()]
+        regular_exts = _build_ext.get_outputs(self)
+        return regular_exts + cmake_exts
+
+
+extensions = []
+
+DESC = """\
+Python library for Apache Arrow"""
+
+setup(
+    name="arrow",
+    packages=['arrow', 'arrow.tests'],
+    version=VERSION,
+    package_data={'arrow': ['*.pxd', '*.pyx']},
+    ext_modules=extensions,
+    cmdclass={
+        'clean': clean,
+        'build_ext': build_ext
+    },
+    install_requires=['cython >= 0.21'],
+    description=DESC,
+    license='Apache License, Version 2.0',
+    maintainer="Apache Arrow Developers",
+    maintainer_email="dev@arrow.apache.org",
+    test_suite="arrow.tests"
+)
diff --git a/python/src/pyarrow/CMakeLists.txt b/python/src/pyarrow/CMakeLists.txt
new file mode 100644
index 0000000000000..e20c3238b5f78
--- /dev/null
+++ b/python/src/pyarrow/CMakeLists.txt
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######################################
+# Unit tests
+#######################################
diff --git a/python/src/pyarrow/api.h b/python/src/pyarrow/api.h
new file mode 100644
index 0000000000000..c2285de77bf10
--- /dev/null
+++ b/python/src/pyarrow/api.h
@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_API_H
+#define PYARROW_API_H
+
+#endif // PYARROW_API_H
diff --git a/python/src/pyarrow/init.cc b/python/src/pyarrow/init.cc
new file mode 100644
index 0000000000000..c36f413725532
--- /dev/null
+++ b/python/src/pyarrow/init.cc
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "pyarrow/init.h"
+
+namespace arrow {
+
+namespace py {
+
+void pyarrow_init() {
+}
+
+} // namespace py
+
+} // namespace arrow
diff --git a/python/src/pyarrow/init.h b/python/src/pyarrow/init.h
new file mode 100644
index 0000000000000..1fc9f10102696
--- /dev/null
+++ b/python/src/pyarrow/init.h
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_INIT_H
+#define PYARROW_INIT_H
+
+namespace arrow {
+
+namespace py {
+
+void pyarrow_init();
+
+} // namespace py
+
+} // namespace arrow
+
+#endif // PYARROW_INIT_H
diff --git a/python/src/pyarrow/util/CMakeLists.txt b/python/src/pyarrow/util/CMakeLists.txt
new file mode 100644
index 0000000000000..60dc80eb38cb6
--- /dev/null
+++ b/python/src/pyarrow/util/CMakeLists.txt
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######################################
+# pyarrow_util
+#######################################
+
+set(UTIL_SRCS
+)
+
+set(UTIL_LIBS
+)
+
+add_library(pyarrow_util STATIC
+  ${UTIL_SRCS}
+)
+target_link_libraries(pyarrow_util ${UTIL_LIBS})
+SET_TARGET_PROPERTIES(pyarrow_util PROPERTIES LINKER_LANGUAGE CXX)
+
+#######################################
+# pyarrow_test_main
+#######################################
+
+add_library(pyarrow_test_main
+  test_main.cc)
+
+if (APPLE)
+  target_link_libraries(pyarrow_test_main
+    gmock
+    dl)
+  set_target_properties(pyarrow_test_main
+        PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+else()
+  target_link_libraries(pyarrow_test_main
+    gtest
+    pthread
+    dl
+  )
+endif()
diff --git a/python/src/pyarrow/util/test_main.cc b/python/src/pyarrow/util/test_main.cc
new file mode 100644
index 0000000000000..00139f36742ed
--- /dev/null
+++ b/python/src/pyarrow/util/test_main.cc
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+
+  int ret = RUN_ALL_TESTS();
+
+  return ret;
+}

From 8caa287263425c5b6c64c0e25fb8aa945e2f78d4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Mon, 7 Mar 2016 14:47:36 -0800
Subject: [PATCH 0021/1644] ARROW-35: Add a short call-to-action in the top
 level README.md

Author: Wes McKinney <wesm@apache.org>

Closes #13 from wesm/ARROW-35 and squashes the following commits:

e10bfc3 [Wes McKinney] Add a proper mailto link
c4428fe [Wes McKinney] Add a short 'how to get involved' blurb in top-level README
---
 README.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/README.md b/README.md
index d948a996bc075..84bae78cc7fbe 100644
--- a/README.md
+++ b/README.md
@@ -22,3 +22,24 @@ Initial implementations include:
  - [Arrow Structures and APIs in Java](https://github.com/apache/arrow/tree/master/java)
 
 Arrow is an [Apache Software Foundation](www.apache.org) project. More info can be found at [arrow.apache.org](http://arrow.apache.org).
+
+#### Getting involved
+
+Right now the primary audience for Apache Arrow are the designers and
+developers of data systems; most people will use Apache Arrow indirectly
+through systems that use it for internal data handling and interoperating with
+other Arrow-enabled systems.
+
+Even if you do not plan to contribute to Apache Arrow itself or Arrow
+integrations in other projects, we'd be happy to have you involved:
+
+- Join the mailing list: send an email to
+  [dev-subscribe@arrow.apache.org][1]. Share your ideas and use cases for the
+  project.
+- [Follow our activity on JIRA][3]
+- [Learn the format][2]
+- Contribute code to one of the reference implementations
+
+[1]: mailto:dev-subscribe@arrow.apache.org
+[2]: https://github.com/apache/arrow/tree/master/format
+[3]: https://issues.apache.org/jira/browse/ARROW
\ No newline at end of file

From 571343bbe36f99a11ed82e475b976bbe79dfb755 Mon Sep 17 00:00:00 2001
From: hyukjinkwon <gurwls223@gmail.com>
Date: Mon, 7 Mar 2016 14:49:27 -0800
Subject: [PATCH 0022/1644] ARROW-9: Rename some unchanged "Drill" to "Arrow"
 (follow-up)

https://issues.apache.org/jira/browse/ARROW-9

There is a unchanged one from "Drill" to "Arrow" at `ValueVector` and minor typos are fixed.

Author: hyukjinkwon <gurwls223@gmail.com>
Author: Hyukjin Kwon <gurwls223@gmail.com>

Closes #18 from HyukjinKwon/ARROW-9 and squashes the following commits:

54a5d9f [Hyukjin Kwon] Update typo
628f35d [hyukjinkwon] Replace straggler references to Drill (follow-up)
---
 .../main/java/org/apache/arrow/vector/ValueVector.java    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index c05f0e7c50fd2..a170c59abd7cc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -63,7 +63,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * Allocates new buffers. ValueVector implements logic to determine how much to allocate.
-   * @return Returns true if allocation was succesful.
+   * @return Returns true if allocation was successful.
    */
   boolean allocateNewSafe();
 
@@ -71,7 +71,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * Set the initial record capacity
-   * @param numRecords
+   * @param numRecords the initial record capacity.
    */
   void setInitialCapacity(int numRecords);
 
@@ -87,7 +87,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
   void close();
 
   /**
-   * Release the underlying DrillBuf and reset the ValueVector to empty.
+   * Release the underlying ArrowBuf and reset the ValueVector to empty.
    */
   void clear();
 
@@ -198,7 +198,7 @@ interface Accessor {
   }
 
   /**
-   * An abstractiong that is used to write into this vector instance.
+   * An abstraction that is used to write into this vector instance.
    */
   interface Mutator {
     /**

From 9afb667783b8cedbe6e9d6ee5eb02d35cf1d0f79 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Mon, 7 Mar 2016 15:02:56 -0800
Subject: [PATCH 0023/1644] ARROW-31: Python: prototype user object model, add
 PyList conversion path with type inference

Depends on ARROW-7. Pretty mundane stuff but got to start somewhere. I'm going to do a little more in this patch (handle normal lists of strings and lists of other supported Python types) before merging.

Author: Wes McKinney <wesm@apache.org>

Closes #19 from wesm/ARROW-31 and squashes the following commits:

2345541 [Wes McKinney] Test basic conversion of nested lists
1d4618b [Wes McKinney] Prototype string and double converters
b02b296 [Wes McKinney] Type inference for lists and lists-of-lists
8c3891c [Wes McKinney] Smoke test that array garbage collection deallocates memory
c28bf09 [Wes McKinney] Build array successfully, without validating contents
731544a [Wes McKinney] Move PrimitiveType::ToString template back to type.h
b5b5b82 [Wes McKinney] Failing test stubs, raise on null array
edb451c [Wes McKinney] Add a few data type smoke tests
47fd78e [Wes McKinney] Add unit test stub
07c1379 [Wes McKinney] Move some bits from arrow/type.h to type.cc
3a774fb [Wes McKinney] Add Status::ToString impls. Unit test stub
4e206fc [Wes McKinney] Add pandas converter placeholder
102ed36 [Wes McKinney] Cython array box scaffold builds
94f122f [Wes McKinney] Basic object model for sequence->arrow conversions
bdb02e7 [Wes McKinney] Use shared_ptr with dynamic make_builder too
d5655ba [Wes McKinney] Clean up array builder API to return shared_ptr<Array>
4132bda [Wes McKinney] Essential scaffolding -- error handling, memory pools, etc. -- to work toward converting Python lists to Arrow arrays
55e69a2 [Wes McKinney] Typed array stubs
ac8c796 [Wes McKinney] Cache primitive data type instances
8f7edaf [Wes McKinney] Consolidate Field and data type subclasses. Add more Python stubs
ea2f3ec [Wes McKinney] Bootstrap end-to-end exposure in Python, wrap DataType and Field types
---
 cpp/CMakeLists.txt                            |  83 ++--
 cpp/src/arrow/CMakeLists.txt                  |   1 -
 cpp/src/arrow/api.h                           |  21 +
 cpp/src/arrow/builder.h                       |  10 +-
 cpp/src/arrow/field.h                         |  63 ---
 cpp/src/arrow/table/CMakeLists.txt            |  15 -
 cpp/src/arrow/table/column-test.cc            |   1 -
 cpp/src/arrow/table/column.cc                 |   2 +-
 cpp/src/arrow/table/column.h                  |   2 +-
 cpp/src/arrow/table/schema-test.cc            |   9 +-
 cpp/src/arrow/table/schema.cc                 |   2 +-
 cpp/src/arrow/table/schema.h                  |   1 -
 cpp/src/arrow/table/table-test.cc             |   1 -
 cpp/src/arrow/table/table.cc                  |   2 +-
 cpp/src/arrow/table/test-common.h             |   1 -
 cpp/src/arrow/type.cc                         |  49 +++
 cpp/src/arrow/type.h                          | 143 ++++--
 cpp/src/arrow/types/CMakeLists.txt            |  22 +-
 cpp/src/arrow/types/boolean.h                 |   3 +-
 cpp/src/arrow/types/construct.cc              |  21 +-
 cpp/src/arrow/types/construct.h               |   6 +-
 cpp/src/arrow/types/json.cc                   |   5 +-
 cpp/src/arrow/types/list-test.cc              |  24 +-
 cpp/src/arrow/types/list.cc                   |  12 -
 cpp/src/arrow/types/list.h                    |  51 +--
 cpp/src/arrow/types/primitive-test.cc         |  64 ++-
 cpp/src/arrow/types/primitive.h               |  22 +-
 cpp/src/arrow/types/string-test.cc            |  11 +-
 cpp/src/arrow/types/string.h                  |  41 +-
 cpp/src/arrow/types/struct-test.cc            |  19 +-
 cpp/src/arrow/types/struct.cc                 |  18 -
 cpp/src/arrow/types/struct.h                  |  21 +-
 cpp/src/arrow/util/CMakeLists.txt             |  20 +-
 cpp/src/arrow/util/buffer.cc                  |   8 +
 cpp/src/arrow/util/buffer.h                   |   2 +
 cpp/src/arrow/util/status.cc                  |  40 ++
 python/CMakeLists.txt                         |  21 +-
 python/arrow/__init__.py                      |  34 ++
 python/arrow/array.pxd                        |  85 ++++
 python/arrow/array.pyx                        | 179 ++++++++
 python/arrow/config.pyx                       |   2 +-
 python/arrow/error.pxd                        |  20 +
 python/arrow/error.pyx                        |  30 ++
 python/arrow/includes/arrow.pxd               |  75 +++-
 python/arrow/includes/common.pxd              |   4 +-
 python/arrow/includes/pyarrow.pxd             |  24 +-
 python/arrow/scalar.pxd                       |  47 ++
 python/arrow/scalar.pyx                       |  28 ++
 python/arrow/schema.pxd                       |  39 ++
 python/arrow/schema.pyx                       | 150 +++++++
 python/arrow/tests/test_array.py              |  26 ++
 python/arrow/tests/test_convert_builtin.py    |  85 ++++
 python/arrow/tests/test_schema.py             |  51 +++
 python/setup.py                               |   7 +-
 python/src/pyarrow/adapters/builtin.cc        | 415 ++++++++++++++++++
 python/src/pyarrow/adapters/builtin.h         |  40 ++
 .../src/pyarrow/adapters/pandas.h             |  17 +-
 python/src/pyarrow/api.h                      |   7 +
 python/src/pyarrow/common.cc                  |  71 +++
 python/src/pyarrow/common.h                   |  95 ++++
 python/src/pyarrow/helpers.cc                 |  57 +++
 .../null.h => python/src/pyarrow/helpers.h    |  22 +-
 python/src/pyarrow/init.cc                    |   8 +-
 python/src/pyarrow/init.h                     |   8 +-
 python/src/pyarrow/status.cc                  |  92 ++++
 python/src/pyarrow/status.h                   | 144 ++++++
 66 files changed, 2246 insertions(+), 453 deletions(-)
 delete mode 100644 cpp/src/arrow/field.h
 create mode 100644 python/arrow/array.pxd
 create mode 100644 python/arrow/array.pyx
 create mode 100644 python/arrow/error.pxd
 create mode 100644 python/arrow/error.pyx
 create mode 100644 python/arrow/scalar.pxd
 create mode 100644 python/arrow/scalar.pyx
 create mode 100644 python/arrow/schema.pxd
 create mode 100644 python/arrow/schema.pyx
 create mode 100644 python/arrow/tests/test_array.py
 create mode 100644 python/arrow/tests/test_convert_builtin.py
 create mode 100644 python/arrow/tests/test_schema.py
 create mode 100644 python/src/pyarrow/adapters/builtin.cc
 create mode 100644 python/src/pyarrow/adapters/builtin.h
 rename cpp/src/arrow/field.cc => python/src/pyarrow/adapters/pandas.h (76%)
 create mode 100644 python/src/pyarrow/common.cc
 create mode 100644 python/src/pyarrow/common.h
 create mode 100644 python/src/pyarrow/helpers.cc
 rename cpp/src/arrow/types/null.h => python/src/pyarrow/helpers.h (72%)
 create mode 100644 python/src/pyarrow/status.cc
 create mode 100644 python/src/pyarrow/status.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 8042661533e1d..e8cb88c0b4d9b 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -37,18 +37,17 @@ if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
   set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 endif()
 
-# Enable using a custom GCC toolchain to build Arrow
-if (NOT "$ENV{ARROW_GCC_ROOT}" STREQUAL "")
-  set(GCC_ROOT $ENV{ARROW_GCC_ROOT})
-  set(CMAKE_C_COMPILER ${GCC_ROOT}/bin/gcc)
-  set(CMAKE_CXX_COMPILER ${GCC_ROOT}/bin/g++)
-endif()
-
 if(APPLE)
   # In newer versions of CMake, this is the default setting
   set(CMAKE_MACOSX_RPATH 1)
 endif()
 
+find_program(CCACHE_FOUND ccache)
+if(CCACHE_FOUND)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+endif(CCACHE_FOUND)
+
 # ----------------------------------------------------------------------
 # cmake options
 
@@ -126,38 +125,16 @@ endif ()
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
 
-# Required to avoid static linking errors with dependencies
-add_definitions(-fPIC)
-
 # Determine compiler version
 include(CompilerInfo)
 
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
-  # Clang helpfully provides a few extensions from C++11 such as the 'override'
-  # keyword on methods. This doesn't change behavior, and we selectively enable
-  # it in src/gutil/port.h only on clang. So, we can safely use it, and don't want
-  # to trigger warnings when we do so.
-  # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++11-extensions")
-
   # Using Clang with ccache causes a bunch of spurious warnings that are
   # purportedly fixed in the next version of ccache. See the following for details:
   #
   #   http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html
   #   http://petereisentraut.blogspot.com/2011/09/ccache-and-clang-part-2.html
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
-
-  # Only hardcode -fcolor-diagnostics if stderr is opened on a terminal. Otherwise
-  # the color codes show up as noisy artifacts.
-  #
-  # This test is imperfect because 'cmake' and 'make' can be run independently
-  # (with different terminal options), and we're testing during the former.
-  execute_process(COMMAND test -t 2 RESULT_VARIABLE ARROW_IS_TTY)
-  if ((${ARROW_IS_TTY} EQUAL 0) AND (NOT ("$ENV{TERM}" STREQUAL "dumb")))
-    message("Running in a controlling terminal")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcolor-diagnostics")
-  else()
-    message("Running without a controlling terminal or in a dumb terminal")
-  endif()
 endif()
 
 # Sanity check linking option.
@@ -278,12 +255,6 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 include_directories(src)
 
-############################################################
-# Visibility
-############################################################
-# For generate_export_header() and add_compiler_export_flags().
-include(GenerateExportHeader)
-
 ############################################################
 # Testing
 ############################################################
@@ -456,21 +427,32 @@ endif()
 # Subdirectories
 ############################################################
 
-add_subdirectory(src/arrow)
-add_subdirectory(src/arrow/util)
-add_subdirectory(src/arrow/table)
-add_subdirectory(src/arrow/types)
-
-set(LINK_LIBS
-  arrow_util
-  arrow_table
-  arrow_types)
+set(LIBARROW_LINK_LIBS
+)
 
 set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/builder.cc
-  src/arrow/field.cc
   src/arrow/type.cc
+
+  src/arrow/table/column.cc
+  src/arrow/table/schema.cc
+  src/arrow/table/table.cc
+
+  src/arrow/types/construct.cc
+  src/arrow/types/floating.cc
+  src/arrow/types/integer.cc
+  src/arrow/types/json.cc
+  src/arrow/types/list.cc
+  src/arrow/types/primitive.cc
+  src/arrow/types/string.cc
+  src/arrow/types/struct.cc
+  src/arrow/types/union.cc
+
+  src/arrow/util/bit-util.cc
+  src/arrow/util/buffer.cc
+  src/arrow/util/memory-pool.cc
+  src/arrow/util/status.cc
 )
 
 set(LIBARROW_LINKAGE "SHARED")
@@ -479,8 +461,15 @@ add_library(arrow
   ${LIBARROW_LINKAGE}
   ${ARROW_SRCS}
 )
-target_link_libraries(arrow ${LINK_LIBS})
-set_target_properties(arrow PROPERTIES LINKER_LANGUAGE CXX)
+set_target_properties(arrow
+  PROPERTIES
+  LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+target_link_libraries(arrow ${LIBARROW_LINK_LIBS})
+
+add_subdirectory(src/arrow)
+add_subdirectory(src/arrow/util)
+add_subdirectory(src/arrow/table)
+add_subdirectory(src/arrow/types)
 
 install(TARGETS arrow
   LIBRARY DESTINATION lib
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 102a8a1853f3e..77326ce38d754 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -20,7 +20,6 @@ install(FILES
   api.h
   array.h
   builder.h
-  field.h
   type.h
   DESTINATION include/arrow)
 
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 899e8aae19c0e..c73d4b386cf54 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -15,7 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
+// Coarse public API while the library is in development
+
 #ifndef ARROW_API_H
 #define ARROW_API_H
 
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/type.h"
+
+#include "arrow/table/column.h"
+#include "arrow/table/schema.h"
+#include "arrow/table/table.h"
+
+#include "arrow/types/boolean.h"
+#include "arrow/types/construct.h"
+#include "arrow/types/floating.h"
+#include "arrow/types/integer.h"
+#include "arrow/types/list.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
 #endif // ARROW_API_H
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 491b9133d2cca..8cc689c3e81ee 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -32,7 +32,7 @@ class Array;
 class MemoryPool;
 class PoolBuffer;
 
-static constexpr int32_t MIN_BUILDER_CAPACITY = 1 << 8;
+static constexpr int32_t MIN_BUILDER_CAPACITY = 1 << 5;
 
 // Base class for all data array builders
 class ArrayBuilder {
@@ -78,12 +78,16 @@ class ArrayBuilder {
 
   // Creates new array object to hold the contents of the builder and transfers
   // ownership of the data
-  virtual Status ToArray(Array** out) = 0;
+  virtual std::shared_ptr<Array> Finish() = 0;
+
+  const std::shared_ptr<DataType>& type() const {
+    return type_;
+  }
 
  protected:
   MemoryPool* pool_;
 
-  TypePtr type_;
+  std::shared_ptr<DataType> type_;
 
   // When nulls are first appended to the builder, the null bitmap is allocated
   std::shared_ptr<PoolBuffer> nulls_;
diff --git a/cpp/src/arrow/field.h b/cpp/src/arrow/field.h
deleted file mode 100644
index 89a450c66f256..0000000000000
--- a/cpp/src/arrow/field.h
+++ /dev/null
@@ -1,63 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_FIELD_H
-#define ARROW_FIELD_H
-
-#include <string>
-
-#include "arrow/type.h"
-
-namespace arrow {
-
-// A field is a piece of metadata that includes (for now) a name and a data
-// type
-
-struct Field {
-  // Field name
-  std::string name;
-
-  // The field's data type
-  TypePtr type;
-
-  Field(const std::string& name, const TypePtr& type) :
-      name(name),
-      type(type) {}
-
-  bool operator==(const Field& other) const {
-    return this->Equals(other);
-  }
-
-  bool operator!=(const Field& other) const {
-    return !this->Equals(other);
-  }
-
-  bool Equals(const Field& other) const {
-    return (this == &other) || (this->name == other.name &&
-        this->type->Equals(other.type.get()));
-  }
-
-  bool nullable() const {
-    return this->type->nullable;
-  }
-
-  std::string ToString() const;
-};
-
-} // namespace arrow
-
-#endif  // ARROW_FIELD_H
diff --git a/cpp/src/arrow/table/CMakeLists.txt b/cpp/src/arrow/table/CMakeLists.txt
index 68bf3148a9889..26d843d853bfb 100644
--- a/cpp/src/arrow/table/CMakeLists.txt
+++ b/cpp/src/arrow/table/CMakeLists.txt
@@ -19,21 +19,6 @@
 # arrow_table
 #######################################
 
-set(TABLE_SRCS
-  column.cc
-  schema.cc
-  table.cc
-)
-
-set(TABLE_LIBS
-)
-
-add_library(arrow_table STATIC
-  ${TABLE_SRCS}
-)
-target_link_libraries(arrow_table ${TABLE_LIBS})
-SET_TARGET_PROPERTIES(arrow_table PROPERTIES LINKER_LANGUAGE CXX)
-
 # Headers: top level
 install(FILES
   column.h
diff --git a/cpp/src/arrow/table/column-test.cc b/cpp/src/arrow/table/column-test.cc
index 4959b82c6e2ae..bf95932916cf4 100644
--- a/cpp/src/arrow/table/column-test.cc
+++ b/cpp/src/arrow/table/column-test.cc
@@ -21,7 +21,6 @@
 #include <string>
 #include <vector>
 
-#include "arrow/field.h"
 #include "arrow/table/column.h"
 #include "arrow/table/schema.h"
 #include "arrow/table/test-common.h"
diff --git a/cpp/src/arrow/table/column.cc b/cpp/src/arrow/table/column.cc
index d68b491fb99da..573e650875944 100644
--- a/cpp/src/arrow/table/column.cc
+++ b/cpp/src/arrow/table/column.cc
@@ -20,7 +20,7 @@
 #include <memory>
 #include <sstream>
 
-#include "arrow/field.h"
+#include "arrow/type.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/table/column.h b/cpp/src/arrow/table/column.h
index 64423bf956147..dfc7516e26aac 100644
--- a/cpp/src/arrow/table/column.h
+++ b/cpp/src/arrow/table/column.h
@@ -23,7 +23,7 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/field.h"
+#include "arrow/type.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/table/schema-test.cc b/cpp/src/arrow/table/schema-test.cc
index 0cf1b3c5f9a8e..d6725cc08c0c8 100644
--- a/cpp/src/arrow/table/schema-test.cc
+++ b/cpp/src/arrow/table/schema-test.cc
@@ -20,7 +20,6 @@
 #include <string>
 #include <vector>
 
-#include "arrow/field.h"
 #include "arrow/table/schema.h"
 #include "arrow/type.h"
 #include "arrow/types/string.h"
@@ -97,10 +96,10 @@ TEST_F(TestSchema, ToString) {
   auto schema = std::make_shared<Schema>(fields);
 
   std::string result = schema->ToString();
-  std::string expected = R"(f0 ?int32
-f1 uint8
-f2 ?string
-f3 ?list<?int16>
+  std::string expected = R"(f0 int32
+f1 uint8 not null
+f2 string
+f3 list<int16>
 )";
 
   ASSERT_EQ(expected, result);
diff --git a/cpp/src/arrow/table/schema.cc b/cpp/src/arrow/table/schema.cc
index fb3b4d6f29268..d49d0a713e7f4 100644
--- a/cpp/src/arrow/table/schema.cc
+++ b/cpp/src/arrow/table/schema.cc
@@ -22,7 +22,7 @@
 #include <sstream>
 #include <vector>
 
-#include "arrow/field.h"
+#include "arrow/type.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/table/schema.h b/cpp/src/arrow/table/schema.h
index d04e3f628c1e3..103f01b26e3ca 100644
--- a/cpp/src/arrow/table/schema.h
+++ b/cpp/src/arrow/table/schema.h
@@ -22,7 +22,6 @@
 #include <string>
 #include <vector>
 
-#include "arrow/field.h"
 #include "arrow/type.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/table/table-test.cc b/cpp/src/arrow/table/table-test.cc
index dd4f74cd16f89..c4fdb062db83a 100644
--- a/cpp/src/arrow/table/table-test.cc
+++ b/cpp/src/arrow/table/table-test.cc
@@ -21,7 +21,6 @@
 #include <string>
 #include <vector>
 
-#include "arrow/field.h"
 #include "arrow/table/column.h"
 #include "arrow/table/schema.h"
 #include "arrow/table/table.h"
diff --git a/cpp/src/arrow/table/table.cc b/cpp/src/arrow/table/table.cc
index 4cefc924ed38f..0c788b8fe3ff3 100644
--- a/cpp/src/arrow/table/table.cc
+++ b/cpp/src/arrow/table/table.cc
@@ -20,9 +20,9 @@
 #include <memory>
 #include <sstream>
 
-#include "arrow/field.h"
 #include "arrow/table/column.h"
 #include "arrow/table/schema.h"
+#include "arrow/type.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/table/test-common.h b/cpp/src/arrow/table/test-common.h
index efe2f228cd0a3..50a5f6a2f5018 100644
--- a/cpp/src/arrow/table/test-common.h
+++ b/cpp/src/arrow/table/test-common.h
@@ -21,7 +21,6 @@
 #include <string>
 #include <vector>
 
-#include "arrow/field.h"
 #include "arrow/table/column.h"
 #include "arrow/table/schema.h"
 #include "arrow/table/table.h"
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index ff145e2c1e3b4..265770822ce90 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -17,8 +17,56 @@
 
 #include "arrow/type.h"
 
+#include <sstream>
+#include <string>
+
 namespace arrow {
 
+std::string Field::ToString() const {
+  std::stringstream ss;
+  ss << this->name << " " << this->type->ToString();
+  return ss.str();
+}
+
+DataType::~DataType() {}
+
+StringType::StringType(bool nullable)
+    : DataType(LogicalType::STRING, nullable) {}
+
+StringType::StringType(const StringType& other)
+    : StringType(other.nullable) {}
+
+std::string StringType::ToString() const {
+  std::string result(name());
+  if (!nullable) {
+    result.append(" not null");
+  }
+  return result;
+}
+
+std::string ListType::ToString() const {
+  std::stringstream s;
+  s << "list<" << value_type->ToString() << ">";
+  if (!this->nullable) {
+    s << " not null";
+  }
+  return s.str();
+}
+
+std::string StructType::ToString() const {
+  std::stringstream s;
+  s << "struct<";
+  for (size_t i = 0; i < fields_.size(); ++i) {
+    if (i > 0) s << ", ";
+    const std::shared_ptr<Field>& field = fields_[i];
+    s << field->name << ": " << field->type->ToString();
+  }
+  s << ">";
+  if (!nullable) s << " not null";
+  return s.str();
+}
+
+const std::shared_ptr<NullType> NA = std::make_shared<NullType>();
 const std::shared_ptr<BooleanType> BOOL = std::make_shared<BooleanType>();
 const std::shared_ptr<UInt8Type> UINT8 = std::make_shared<UInt8Type>();
 const std::shared_ptr<UInt16Type> UINT16 = std::make_shared<UInt16Type>();
@@ -30,5 +78,6 @@ const std::shared_ptr<Int32Type> INT32 = std::make_shared<Int32Type>();
 const std::shared_ptr<Int64Type> INT64 = std::make_shared<Int64Type>();
 const std::shared_ptr<FloatType> FLOAT = std::make_shared<FloatType>();
 const std::shared_ptr<DoubleType> DOUBLE = std::make_shared<DoubleType>();
+const std::shared_ptr<StringType> STRING = std::make_shared<StringType>();
 
 } // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 4193a0e8bc851..e78e49491193e 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -20,6 +20,7 @@
 
 #include <memory>
 #include <string>
+#include <vector>
 
 namespace arrow {
 
@@ -71,49 +72,46 @@ struct LogicalType {
     UINT64 = 7,
     INT64 = 8,
 
-    // A boolean value represented as 1 byte
-    BOOL = 9,
-
     // A boolean value represented as 1 bit
-    BIT = 10,
+    BOOL = 9,
 
     // 4-byte floating point value
-    FLOAT = 11,
+    FLOAT = 10,
 
     // 8-byte floating point value
-    DOUBLE = 12,
+    DOUBLE = 11,
 
     // CHAR(N): fixed-length UTF8 string with length N
-    CHAR = 13,
+    CHAR = 12,
 
     // UTF8 variable-length string as List<Char>
-    STRING = 14,
+    STRING = 13,
 
     // VARCHAR(N): Null-terminated string type embedded in a CHAR(N + 1)
-    VARCHAR = 15,
+    VARCHAR = 14,
 
     // Variable-length bytes (no guarantee of UTF8-ness)
-    BINARY = 16,
+    BINARY = 15,
 
     // By default, int32 days since the UNIX epoch
-    DATE = 17,
+    DATE = 16,
 
     // Exact timestamp encoded with int64 since UNIX epoch
     // Default unit millisecond
-    TIMESTAMP = 18,
+    TIMESTAMP = 17,
 
     // Timestamp as double seconds since the UNIX epoch
-    TIMESTAMP_DOUBLE = 19,
+    TIMESTAMP_DOUBLE = 18,
 
     // Exact time encoded with int64, default unit millisecond
-    TIME = 20,
+    TIME = 19,
 
     // Precision- and scale-based decimal type. Storage type depends on the
     // parameters.
-    DECIMAL = 21,
+    DECIMAL = 20,
 
     // Decimal value encoded as a text string
-    DECIMAL_TEXT = 22,
+    DECIMAL_TEXT = 21,
 
     // A list of some logical data type
     LIST = 30,
@@ -141,7 +139,9 @@ struct DataType {
       type(type),
       nullable(nullable) {}
 
-  virtual bool Equals(const DataType* other) {
+  virtual ~DataType();
+
+  bool Equals(const DataType* other) {
     // Call with a pointer so more friendly to subclasses
     return this == other || (this->type == other->type &&
         this->nullable == other->nullable);
@@ -154,10 +154,45 @@ struct DataType {
   virtual std::string ToString() const = 0;
 };
 
-
 typedef std::shared_ptr<LayoutType> LayoutPtr;
 typedef std::shared_ptr<DataType> TypePtr;
 
+// A field is a piece of metadata that includes (for now) a name and a data
+// type
+struct Field {
+  // Field name
+  std::string name;
+
+  // The field's data type
+  TypePtr type;
+
+  Field(const std::string& name, const TypePtr& type) :
+      name(name),
+      type(type) {}
+
+  bool operator==(const Field& other) const {
+    return this->Equals(other);
+  }
+
+  bool operator!=(const Field& other) const {
+    return !this->Equals(other);
+  }
+
+  bool Equals(const Field& other) const {
+    return (this == &other) || (this->name == other.name &&
+        this->type->Equals(other.type.get()));
+  }
+
+  bool Equals(const std::shared_ptr<Field>& other) const {
+    return Equals(*other.get());
+  }
+
+  bool nullable() const {
+    return this->type->nullable;
+  }
+
+  std::string ToString() const;
+};
 
 struct BytesType : public LayoutType {
   int size;
@@ -183,16 +218,18 @@ struct PrimitiveType : public DataType {
   explicit PrimitiveType(bool nullable = true)
       : DataType(Derived::type_enum, nullable) {}
 
-  virtual std::string ToString() const {
-    std::string result;
-    if (nullable) {
-      result.append("?");
-    }
-    result.append(static_cast<const Derived*>(this)->name());
-    return result;
-  }
+  std::string ToString() const override;
 };
 
+template <typename Derived>
+inline std::string PrimitiveType<Derived>::ToString() const {
+  std::string result(static_cast<const Derived*>(this)->name());
+  if (!nullable) {
+    result.append(" not null");
+  }
+  return result;
+}
+
 #define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)          \
   typedef C_TYPE c_type;                                            \
   static constexpr LogicalType::type type_enum = LogicalType::ENUM; \
@@ -205,6 +242,10 @@ struct PrimitiveType : public DataType {
     return NAME;                                                    \
   }
 
+struct NullType : public PrimitiveType<NullType> {
+  PRIMITIVE_DECL(NullType, void, NA, 0, "null");
+};
+
 struct BooleanType : public PrimitiveType<BooleanType> {
   PRIMITIVE_DECL(BooleanType, uint8_t, BOOL, 1, "bool");
 };
@@ -249,6 +290,55 @@ struct DoubleType : public PrimitiveType<DoubleType> {
   PRIMITIVE_DECL(DoubleType, double, DOUBLE, 8, "double");
 };
 
+struct ListType : public DataType {
+  // List can contain any other logical value type
+  TypePtr value_type;
+
+  explicit ListType(const TypePtr& value_type, bool nullable = true)
+      : DataType(LogicalType::LIST, nullable),
+        value_type(value_type) {}
+
+  static char const *name() {
+    return "list";
+  }
+
+  std::string ToString() const override;
+};
+
+// String is a logical type consisting of a physical list of 1-byte values
+struct StringType : public DataType {
+  explicit StringType(bool nullable = true);
+
+  StringType(const StringType& other);
+
+  static char const *name() {
+    return "string";
+  }
+
+  std::string ToString() const override;
+};
+
+struct StructType : public DataType {
+  std::vector<std::shared_ptr<Field> > fields_;
+
+  explicit StructType(const std::vector<std::shared_ptr<Field> >& fields,
+      bool nullable = true)
+      : DataType(LogicalType::STRUCT, nullable) {
+    fields_ = fields;
+  }
+
+  const std::shared_ptr<Field>& field(int i) const {
+    return fields_[i];
+  }
+
+  int num_children() const {
+    return fields_.size();
+  }
+
+  std::string ToString() const override;
+};
+
+extern const std::shared_ptr<NullType> NA;
 extern const std::shared_ptr<BooleanType> BOOL;
 extern const std::shared_ptr<UInt8Type> UINT8;
 extern const std::shared_ptr<UInt16Type> UINT16;
@@ -260,6 +350,7 @@ extern const std::shared_ptr<Int32Type> INT32;
 extern const std::shared_ptr<Int64Type> INT64;
 extern const std::shared_ptr<FloatType> FLOAT;
 extern const std::shared_ptr<DoubleType> DOUBLE;
+extern const std::shared_ptr<StringType> STRING;
 
 } // namespace arrow
 
diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
index e090aead1f8b9..57cabdefd2525 100644
--- a/cpp/src/arrow/types/CMakeLists.txt
+++ b/cpp/src/arrow/types/CMakeLists.txt
@@ -19,31 +19,11 @@
 # arrow_types
 #######################################
 
-set(TYPES_SRCS
-  construct.cc
-  floating.cc
-  integer.cc
-  json.cc
-  list.cc
-  primitive.cc
-  string.cc
-  struct.cc
-  union.cc
-)
-
-set(TYPES_LIBS
-)
-
-add_library(arrow_types STATIC
-  ${TYPES_SRCS}
-)
-target_link_libraries(arrow_types ${TYPES_LIBS})
-SET_TARGET_PROPERTIES(arrow_types PROPERTIES LINKER_LANGUAGE CXX)
-
 # Headers: top level
 install(FILES
   boolean.h
   collection.h
+  construct.h
   datetime.h
   decimal.h
   floating.h
diff --git a/cpp/src/arrow/types/boolean.h b/cpp/src/arrow/types/boolean.h
index 8fc9cfd19c0d4..a5023d7b368d2 100644
--- a/cpp/src/arrow/types/boolean.h
+++ b/cpp/src/arrow/types/boolean.h
@@ -24,7 +24,8 @@ namespace arrow {
 
 typedef PrimitiveArrayImpl<BooleanType> BooleanArray;
 
-// typedef PrimitiveBuilder<BooleanType, BooleanArray> BooleanBuilder;
+class BooleanBuilder : public ArrayBuilder {
+};
 
 } // namespace arrow
 
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 05d6b270fc3fd..43f01a3051385 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -32,13 +32,13 @@ class ArrayBuilder;
 // Initially looked at doing this with vtables, but shared pointers makes it
 // difficult
 
-#define BUILDER_CASE(ENUM, BuilderType)                                 \
-    case LogicalType::ENUM:                                             \
-      *out = static_cast<ArrayBuilder*>(new BuilderType(pool, type));   \
+#define BUILDER_CASE(ENUM, BuilderType)         \
+    case LogicalType::ENUM:                     \
+      out->reset(new BuilderType(pool, type));  \
       return Status::OK();
 
-Status make_builder(MemoryPool* pool, const TypePtr& type,
-    ArrayBuilder** out) {
+Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+    std::shared_ptr<ArrayBuilder>* out) {
   switch (type->type) {
     BUILDER_CASE(UINT8, UInt8Builder);
     BUILDER_CASE(INT8, Int8Builder);
@@ -58,13 +58,12 @@ Status make_builder(MemoryPool* pool, const TypePtr& type,
 
     case LogicalType::LIST:
       {
-        ListType* list_type = static_cast<ListType*>(type.get());
-        ArrayBuilder* value_builder;
-        RETURN_NOT_OK(make_builder(pool, list_type->value_type, &value_builder));
+        std::shared_ptr<ArrayBuilder> value_builder;
 
-        // The ListBuilder takes ownership of the value_builder
-        ListBuilder* builder = new ListBuilder(pool, type, value_builder);
-        *out = static_cast<ArrayBuilder*>(builder);
+        const std::shared_ptr<DataType>& value_type = static_cast<ListType*>(
+            type.get())->value_type;
+        RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
+        out->reset(new ListBuilder(pool, type, value_builder));
         return Status::OK();
       }
     // BUILDER_CASE(CHAR, CharBuilder);
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index b5ba436f787d9..59ebe1acddc98 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -18,6 +18,8 @@
 #ifndef ARROW_TYPES_CONSTRUCT_H
 #define ARROW_TYPES_CONSTRUCT_H
 
+#include <memory>
+
 #include "arrow/type.h"
 
 namespace arrow {
@@ -26,8 +28,8 @@ class ArrayBuilder;
 class MemoryPool;
 class Status;
 
-Status make_builder(MemoryPool* pool, const TypePtr& type,
-    ArrayBuilder** out);
+Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+    std::shared_ptr<ArrayBuilder>* out);
 
 } // namespace arrow
 
diff --git a/cpp/src/arrow/types/json.cc b/cpp/src/arrow/types/json.cc
index b29b95715fef6..168e370d51a14 100644
--- a/cpp/src/arrow/types/json.cc
+++ b/cpp/src/arrow/types/json.cc
@@ -19,10 +19,7 @@
 
 #include <vector>
 
-#include "arrow/types/boolean.h"
-#include "arrow/types/integer.h"
-#include "arrow/types/floating.h"
-#include "arrow/types/null.h"
+#include "arrow/type.h"
 #include "arrow/types/string.h"
 #include "arrow/types/union.h"
 
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index b4bbd2841a89d..02991de2648e7 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -32,6 +32,7 @@
 #include "arrow/types/test-common.h"
 #include "arrow/util/status.h"
 
+using std::shared_ptr;
 using std::string;
 using std::unique_ptr;
 using std::vector;
@@ -47,17 +48,18 @@ TEST(TypesTest, TestListType) {
   ASSERT_EQ(list_type.type, LogicalType::LIST);
 
   ASSERT_EQ(list_type.name(), string("list"));
-  ASSERT_EQ(list_type.ToString(), string("?list<?uint8>"));
+  ASSERT_EQ(list_type.ToString(), string("list<uint8>"));
 
   ASSERT_EQ(list_type.value_type->type, vt->type);
   ASSERT_EQ(list_type.value_type->type, vt->type);
 
   std::shared_ptr<DataType> st = std::make_shared<StringType>(false);
   std::shared_ptr<DataType> lt = std::make_shared<ListType>(st, false);
-  ASSERT_EQ(lt->ToString(), string("list<string>"));
+  ASSERT_EQ(lt->ToString(), string("list<string not null> not null"));
 
   ListType lt2(lt, false);
-  ASSERT_EQ(lt2.ToString(), string("list<list<string>>"));
+  ASSERT_EQ(lt2.ToString(),
+      string("list<list<string not null> not null> not null"));
 }
 
 // ----------------------------------------------------------------------
@@ -71,23 +73,21 @@ class TestListBuilder : public TestBuilder {
     value_type_ = TypePtr(new Int32Type());
     type_ = TypePtr(new ListType(value_type_));
 
-    ArrayBuilder* tmp;
-    ASSERT_OK(make_builder(pool_, type_, &tmp));
-    builder_.reset(static_cast<ListBuilder*>(tmp));
+    std::shared_ptr<ArrayBuilder> tmp;
+    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+    builder_ = std::dynamic_pointer_cast<ListBuilder>(tmp);
   }
 
   void Done() {
-    Array* out;
-    ASSERT_OK(builder_->ToArray(&out));
-    result_.reset(static_cast<ListArray*>(out));
+    result_ = std::dynamic_pointer_cast<ListArray>(builder_->Finish());
   }
 
  protected:
   TypePtr value_type_;
   TypePtr type_;
 
-  unique_ptr<ListBuilder> builder_;
-  unique_ptr<ListArray> result_;
+  shared_ptr<ListBuilder> builder_;
+  shared_ptr<ListArray> result_;
 };
 
 
@@ -116,7 +116,7 @@ TEST_F(TestListBuilder, TestBasics) {
   vector<int> lengths = {3, 0, 4};
   vector<uint8_t> is_null = {0, 1, 0};
 
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
 
   int pos = 0;
   for (size_t i = 0; i < lengths.size(); ++i) {
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index 577d71d0b2892..69a79a77fabe0 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -17,18 +17,6 @@
 
 #include "arrow/types/list.h"
 
-#include <sstream>
-#include <string>
-
 namespace arrow {
 
-std::string ListType::ToString() const {
-  std::stringstream s;
-  if (this->nullable) {
-    s << "?";
-  }
-  s << "list<" << value_type->ToString() << ">";
-  return s.str();
-}
-
 } // namespace arrow
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index f39fe5c4d811b..f40a8245362b1 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -36,21 +36,6 @@ namespace arrow {
 
 class MemoryPool;
 
-struct ListType : public DataType {
-  // List can contain any other logical value type
-  TypePtr value_type;
-
-  explicit ListType(const TypePtr& value_type, bool nullable = true)
-      : DataType(LogicalType::LIST, nullable),
-        value_type(value_type) {}
-
-  static char const *name() {
-    return "list";
-  }
-
-  virtual std::string ToString() const;
-};
-
 class ListArray : public Array {
  public:
   ListArray() : Array(), offset_buf_(nullptr), offsets_(nullptr) {}
@@ -106,10 +91,9 @@ class ListArray : public Array {
 class ListBuilder : public Int32Builder {
  public:
   ListBuilder(MemoryPool* pool, const TypePtr& type,
-      ArrayBuilder* value_builder)
-      : Int32Builder(pool, type) {
-    value_builder_.reset(value_builder);
-  }
+      std::shared_ptr<ArrayBuilder> value_builder)
+      : Int32Builder(pool, type),
+        value_builder_(value_builder) {}
 
   Status Init(int32_t elements) {
     // One more than requested.
@@ -147,30 +131,27 @@ class ListBuilder : public Int32Builder {
     return Status::OK();
   }
 
-  // Initialize an array type instance with the results of this builder
-  // Transfers ownership of all buffers
   template <typename Container>
-  Status Transfer(Container* out) {
-    Array* child_values;
-    RETURN_NOT_OK(value_builder_->ToArray(&child_values));
+  std::shared_ptr<Array> Transfer() {
+    auto result = std::make_shared<Container>();
+
+    std::shared_ptr<Array> items = value_builder_->Finish();
 
     // Add final offset if the length is non-zero
     if (length_) {
-      raw_buffer()[length_] = child_values->length();
+      raw_buffer()[length_] = items->length();
     }
 
-    out->Init(type_, length_, values_, ArrayPtr(child_values),
+    result->Init(type_, length_, values_, items,
         null_count_, nulls_);
     values_ = nulls_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
-    return Status::OK();
+
+    return result;
   }
 
-  virtual Status ToArray(Array** out) {
-    ListArray* result = new ListArray();
-    RETURN_NOT_OK(Transfer(result));
-    *out = static_cast<Array*>(result);
-    return Status::OK();
+  std::shared_ptr<Array> Finish() override {
+    return Transfer<ListArray>();
   }
 
   // Start a new variable-length list slot
@@ -198,10 +179,12 @@ class ListBuilder : public Int32Builder {
     return Append(true);
   }
 
-  ArrayBuilder* value_builder() const { return value_builder_.get();}
+  const std::shared_ptr<ArrayBuilder>& value_builder() const {
+    return value_builder_;
+  }
 
  protected:
-  std::unique_ptr<ArrayBuilder> value_builder_;
+  std::shared_ptr<ArrayBuilder> value_builder_;
 };
 
 
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 02eaaa7542bf0..f35a258e2cb57 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -37,6 +37,7 @@
 #include "arrow/util/status.h"
 
 using std::string;
+using std::shared_ptr;
 using std::unique_ptr;
 using std::vector;
 
@@ -98,12 +99,12 @@ class TestPrimitiveBuilder : public TestBuilder {
 
     type_ = Attrs::type();
 
-    ArrayBuilder* tmp;
-    ASSERT_OK(make_builder(pool_, type_, &tmp));
-    builder_.reset(static_cast<BuilderType*>(tmp));
+    std::shared_ptr<ArrayBuilder> tmp;
+    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+    builder_ = std::dynamic_pointer_cast<BuilderType>(tmp);
 
-    ASSERT_OK(make_builder(pool_, type_, &tmp));
-    builder_nn_.reset(static_cast<BuilderType*>(tmp));
+    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+    builder_nn_ = std::dynamic_pointer_cast<BuilderType>(tmp);
   }
 
   void RandomData(int N, double pct_null = 0.1) {
@@ -112,7 +113,6 @@ class TestPrimitiveBuilder : public TestBuilder {
   }
 
   void CheckNullable() {
-    ArrayType result;
     ArrayType expected;
     int size = builder_->length();
 
@@ -125,7 +125,9 @@ class TestPrimitiveBuilder : public TestBuilder {
     int32_t ex_null_count = null_count(nulls_);
 
     expected.Init(size, ex_data, ex_null_count, ex_nulls);
-    ASSERT_OK(builder_->Transfer(&result));
+
+    std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(
+        builder_->Finish());
 
     // Builder is now reset
     ASSERT_EQ(0, builder_->length());
@@ -133,12 +135,11 @@ class TestPrimitiveBuilder : public TestBuilder {
     ASSERT_EQ(0, builder_->null_count());
     ASSERT_EQ(nullptr, builder_->buffer());
 
-    ASSERT_TRUE(result.Equals(expected));
-    ASSERT_EQ(ex_null_count, result.null_count());
+    ASSERT_TRUE(result->Equals(expected));
+    ASSERT_EQ(ex_null_count, result->null_count());
   }
 
   void CheckNonNullable() {
-    ArrayType result;
     ArrayType expected;
     int size = builder_nn_->length();
 
@@ -146,22 +147,24 @@ class TestPrimitiveBuilder : public TestBuilder {
         size * sizeof(T));
 
     expected.Init(size, ex_data);
-    ASSERT_OK(builder_nn_->Transfer(&result));
+
+    std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(
+        builder_nn_->Finish());
 
     // Builder is now reset
     ASSERT_EQ(0, builder_nn_->length());
     ASSERT_EQ(0, builder_nn_->capacity());
     ASSERT_EQ(nullptr, builder_nn_->buffer());
 
-    ASSERT_TRUE(result.Equals(expected));
-    ASSERT_EQ(0, result.null_count());
+    ASSERT_TRUE(result->Equals(expected));
+    ASSERT_EQ(0, result->null_count());
   }
 
  protected:
   TypePtr type_;
   TypePtr type_nn_;
-  unique_ptr<BuilderType> builder_;
-  unique_ptr<BuilderType> builder_nn_;
+  shared_ptr<BuilderType> builder_;
+  shared_ptr<BuilderType> builder_nn_;
 
   vector<T> draws_;
   vector<uint8_t> nulls_;
@@ -225,15 +228,36 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
     ASSERT_OK(this->builder_->AppendNull());
   }
 
-  Array* result;
-  ASSERT_OK(this->builder_->ToArray(&result));
-  unique_ptr<Array> holder(result);
+  auto result = this->builder_->Finish();
 
   for (int i = 0; i < size; ++i) {
     ASSERT_TRUE(result->IsNull(i));
   }
 }
 
+TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
+  DECL_T();
+
+  int size = 10000;
+
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& nulls = this->nulls_;
+
+  int64_t memory_before = this->pool_->bytes_allocated();
+
+  this->RandomData(size);
+
+  int i;
+  for (i = 0; i < size; ++i) {
+    ASSERT_OK(this->builder_->Append(draws[i], nulls[i] > 0));
+  }
+
+  do {
+    std::shared_ptr<Array> result = this->builder_->Finish();
+  } while (false);
+
+  ASSERT_EQ(memory_before, this->pool_->bytes_allocated());
+}
 
 TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   DECL_T();
@@ -331,11 +355,11 @@ TYPED_TEST(TestPrimitiveBuilder, TestResize) {
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
-  int n = 100;
-  ASSERT_OK(this->builder_->Reserve(n));
+  ASSERT_OK(this->builder_->Reserve(10));
   ASSERT_EQ(0, this->builder_->length());
   ASSERT_EQ(MIN_BUILDER_CAPACITY, this->builder_->capacity());
 
+  ASSERT_OK(this->builder_->Reserve(90));
   ASSERT_OK(this->builder_->Advance(100));
   ASSERT_OK(this->builder_->Reserve(MIN_BUILDER_CAPACITY));
 
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 09d43e7ec8b80..1073bb6e1c340 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -64,6 +64,8 @@ class PrimitiveArrayImpl : public PrimitiveArray {
 
   PrimitiveArrayImpl() : PrimitiveArray() {}
 
+  virtual ~PrimitiveArrayImpl() {}
+
   PrimitiveArrayImpl(int32_t length, const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr) {
@@ -197,24 +199,12 @@ class PrimitiveBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  // Initialize an array type instance with the results of this builder
-  // Transfers ownership of all buffers
-  Status Transfer(PrimitiveArray* out) {
-    out->Init(type_, length_, values_, null_count_, nulls_);
+  std::shared_ptr<Array> Finish() override {
+    std::shared_ptr<ArrayType> result = std::make_shared<ArrayType>();
+    result->PrimitiveArray::Init(type_, length_, values_, null_count_, nulls_);
     values_ = nulls_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
-    return Status::OK();
-  }
-
-  Status Transfer(ArrayType* out) {
-    return Transfer(static_cast<PrimitiveArray*>(out));
-  }
-
-  virtual Status ToArray(Array** out) {
-    ArrayType* result = new ArrayType();
-    RETURN_NOT_OK(Transfer(result));
-    *out = static_cast<Array*>(result);
-    return Status::OK();
+    return result;
   }
 
   value_type* raw_buffer() {
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index 9af667295026b..8e82fd95dd808 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -166,23 +166,18 @@ class TestStringBuilder : public TestBuilder {
   void SetUp() {
     TestBuilder::SetUp();
     type_ = TypePtr(new StringType());
-
-    ArrayBuilder* tmp;
-    ASSERT_OK(make_builder(pool_, type_, &tmp));
-    builder_.reset(static_cast<StringBuilder*>(tmp));
+    builder_.reset(new StringBuilder(pool_, type_));
   }
 
   void Done() {
-    Array* out;
-    ASSERT_OK(builder_->ToArray(&out));
-    result_.reset(static_cast<StringArray*>(out));
+    result_ = std::dynamic_pointer_cast<StringArray>(builder_->Finish());
   }
 
  protected:
   TypePtr type_;
 
   std::unique_ptr<StringBuilder> builder_;
-  std::unique_ptr<StringArray> result_;
+  std::shared_ptr<StringArray> result_;
 };
 
 TEST_F(TestStringBuilder, TestScalarAppend) {
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index 5795cfed577c5..8ccc0a9698a54 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -71,28 +71,6 @@ struct VarcharType : public DataType {
 static const LayoutPtr byte1(new BytesType(1));
 static const LayoutPtr physical_string = LayoutPtr(new ListLayoutType(byte1));
 
-// String is a logical type consisting of a physical list of 1-byte values
-struct StringType : public DataType {
-  explicit StringType(bool nullable = true)
-      : DataType(LogicalType::STRING, nullable) {}
-
-  StringType(const StringType& other)
-      : StringType() {}
-
-  static char const *name() {
-    return "string";
-  }
-
-  virtual std::string ToString() const {
-    std::string result;
-    if (nullable) {
-      result.append("?");
-    }
-    result.append(name());
-    return result;
-  }
-};
-
 // TODO: add a BinaryArray layer in between
 class StringArray : public ListArray {
  public:
@@ -153,26 +131,23 @@ class StringArray : public ListArray {
 class StringBuilder : public ListBuilder {
  public:
   explicit StringBuilder(MemoryPool* pool, const TypePtr& type) :
-      ListBuilder(pool, type,
-          static_cast<ArrayBuilder*>(new UInt8Builder(pool, value_type_))) {
+      ListBuilder(pool, type, std::make_shared<UInt8Builder>(pool, value_type_)) {
     byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
   }
 
   Status Append(const std::string& value) {
-    RETURN_NOT_OK(ListBuilder::Append());
-    return byte_builder_->Append(reinterpret_cast<const uint8_t*>(value.c_str()),
-        value.size());
+    return Append(value.c_str(), value.size());
   }
 
-  Status Append(const uint8_t* value, int32_t length);
+  Status Append(const char* value, int32_t length) {
+    RETURN_NOT_OK(ListBuilder::Append());
+    return byte_builder_->Append(reinterpret_cast<const uint8_t*>(value), length);
+  }
   Status Append(const std::vector<std::string>& values,
                 uint8_t* null_bytes);
 
-  virtual Status ToArray(Array** out) {
-    StringArray* result = new StringArray();
-    RETURN_NOT_OK(ListBuilder::Transfer(result));
-    *out = static_cast<Array*>(result);
-    return Status::OK();
+  std::shared_ptr<Array> Finish() override {
+    return ListBuilder::Transfer<StringArray>();
   }
 
  protected:
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index df6157104795e..9a4777e8b983d 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -17,15 +17,16 @@
 
 #include <gtest/gtest.h>
 
+#include <memory>
 #include <string>
 #include <vector>
 
-#include "arrow/field.h"
 #include "arrow/type.h"
 #include "arrow/types/integer.h"
 #include "arrow/types/string.h"
 #include "arrow/types/struct.h"
 
+using std::shared_ptr;
 using std::string;
 using std::vector;
 
@@ -33,23 +34,23 @@ namespace arrow {
 
 TEST(TestStructType, Basics) {
   TypePtr f0_type = TypePtr(new Int32Type());
-  Field f0("f0", f0_type);
+  auto f0 = std::make_shared<Field>("f0", f0_type);
 
   TypePtr f1_type = TypePtr(new StringType());
-  Field f1("f1", f1_type);
+  auto f1 = std::make_shared<Field>("f1", f1_type);
 
   TypePtr f2_type = TypePtr(new UInt8Type());
-  Field f2("f2", f2_type);
+  auto f2 = std::make_shared<Field>("f2", f2_type);
 
-  vector<Field> fields = {f0, f1, f2};
+  vector<shared_ptr<Field> > fields = {f0, f1, f2};
 
   StructType struct_type(fields);
 
-  ASSERT_TRUE(struct_type.field(0).Equals(f0));
-  ASSERT_TRUE(struct_type.field(1).Equals(f1));
-  ASSERT_TRUE(struct_type.field(2).Equals(f2));
+  ASSERT_TRUE(struct_type.field(0)->Equals(f0));
+  ASSERT_TRUE(struct_type.field(1)->Equals(f1));
+  ASSERT_TRUE(struct_type.field(2)->Equals(f2));
 
-  ASSERT_EQ(struct_type.ToString(), "?struct<f0: ?int32, f1: ?string, f2: ?uint8>");
+  ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
 
   // TODO: out of bounds for field(...)
 }
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
index 6b233bc372af1..02af600b017d0 100644
--- a/cpp/src/arrow/types/struct.cc
+++ b/cpp/src/arrow/types/struct.cc
@@ -17,24 +17,6 @@
 
 #include "arrow/types/struct.h"
 
-#include <cstdlib>
-#include <memory>
-#include <sstream>
-#include <string>
-
 namespace arrow {
 
-std::string StructType::ToString() const {
-  std::stringstream s;
-  if (nullable) s << "?";
-  s << "struct<";
-  for (size_t i = 0; i < fields_.size(); ++i) {
-    if (i > 0) s << ", ";
-    const Field& field  = fields_[i];
-    s << field.name << ": " << field.type->ToString();
-  }
-  s << ">";
-  return s.str();
-}
-
 } // namespace arrow
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index e575c31287cb2..5842534d35be1 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -18,33 +18,14 @@
 #ifndef ARROW_TYPES_STRUCT_H
 #define ARROW_TYPES_STRUCT_H
 
+#include <memory>
 #include <string>
 #include <vector>
 
-#include "arrow/field.h"
 #include "arrow/type.h"
 
 namespace arrow {
 
-struct StructType : public DataType {
-  std::vector<Field> fields_;
-
-  explicit StructType(const std::vector<Field>& fields, bool nullable = true)
-      : DataType(LogicalType::STRUCT, nullable) {
-    fields_ = fields;
-  }
-
-  const Field& field(int i) const {
-    return fields_[i];
-  }
-
-  int num_children() const {
-    return fields_.size();
-  }
-
-  virtual std::string ToString() const;
-};
-
 } // namespace arrow
 
 #endif // ARROW_TYPES_STRUCT_H
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index c53f307c9f59a..4272ce4285482 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -19,22 +19,6 @@
 # arrow_util
 #######################################
 
-set(UTIL_SRCS
-  bit-util.cc
-  buffer.cc
-  memory-pool.cc
-  status.cc
-)
-
-set(UTIL_LIBS
-)
-
-add_library(arrow_util STATIC
-  ${UTIL_SRCS}
-)
-target_link_libraries(arrow_util ${UTIL_LIBS})
-SET_TARGET_PROPERTIES(arrow_util PROPERTIES LINKER_LANGUAGE CXX)
-
 # Headers: top level
 install(FILES
   bit-util.h
@@ -50,7 +34,7 @@ install(FILES
 
 add_library(arrow_test_util)
 target_link_libraries(arrow_test_util
-  arrow_util)
+)
 
 SET_TARGET_PROPERTIES(arrow_test_util PROPERTIES LINKER_LANGUAGE CXX)
 
@@ -64,7 +48,6 @@ add_library(arrow_test_main
 if (APPLE)
   target_link_libraries(arrow_test_main
     gtest
-	arrow_util
 	arrow_test_util
     dl)
   set_target_properties(arrow_test_main
@@ -72,7 +55,6 @@ if (APPLE)
 else()
   target_link_libraries(arrow_test_main
     gtest
-	arrow_util
 	arrow_test_util
     pthread
     dl
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
index 3f3807d4e2094..50f4716769d70 100644
--- a/cpp/src/arrow/util/buffer.cc
+++ b/cpp/src/arrow/util/buffer.cc
@@ -31,6 +31,8 @@ Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset,
   parent_ = parent;
 }
 
+Buffer::~Buffer() {}
+
 std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
   return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
 }
@@ -43,6 +45,12 @@ PoolBuffer::PoolBuffer(MemoryPool* pool) :
   pool_ = pool;
 }
 
+PoolBuffer::~PoolBuffer() {
+  if (mutable_data_ != nullptr) {
+    pool_->Free(mutable_data_, capacity_);
+  }
+}
+
 Status PoolBuffer::Reserve(int64_t new_capacity) {
   if (!mutable_data_ || new_capacity > capacity_) {
     uint8_t* new_data;
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 8704723eb0a89..0c3e210abd910 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -39,6 +39,7 @@ class Buffer : public std::enable_shared_from_this<Buffer> {
   Buffer(const uint8_t* data, int64_t size) :
       data_(data),
       size_(size) {}
+  virtual ~Buffer();
 
   // An offset into data that is owned by another buffer, but we want to be
   // able to retain a valid pointer to it even after other shared_ptr's to the
@@ -136,6 +137,7 @@ class ResizableBuffer : public MutableBuffer {
 class PoolBuffer : public ResizableBuffer {
  public:
   explicit PoolBuffer(MemoryPool* pool = nullptr);
+  virtual ~PoolBuffer();
 
   virtual Status Resize(int64_t new_size);
   virtual Status Reserve(int64_t new_capacity);
diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/util/status.cc
index c64b8a3d5f80a..c6e113ebea590 100644
--- a/cpp/src/arrow/util/status.cc
+++ b/cpp/src/arrow/util/status.cc
@@ -35,4 +35,44 @@ const char* Status::CopyState(const char* state) {
   return result;
 }
 
+std::string Status::CodeAsString() const {
+  if (state_ == NULL) {
+    return "OK";
+  }
+
+  const char* type;
+  switch (code()) {
+    case StatusCode::OK:
+      type = "OK";
+      break;
+    case StatusCode::OutOfMemory:
+      type = "Out of memory";
+      break;
+    case StatusCode::KeyError:
+      type = "Key error";
+      break;
+    case StatusCode::Invalid:
+      type = "Invalid";
+      break;
+    case StatusCode::NotImplemented:
+      type = "NotImplemented";
+      break;
+  }
+  return std::string(type);
+}
+
+std::string Status::ToString() const {
+  std::string result(CodeAsString());
+  if (state_ == NULL) {
+    return result;
+  }
+
+  result.append(": ");
+
+  uint32_t length;
+  memcpy(&length, state_, sizeof(length));
+  result.append(reinterpret_cast<const char*>(state_ + 7), length);
+  return result;
+}
+
 } // namespace arrow
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index df55bfac9eb4a..8fdd829010eef 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -45,6 +45,12 @@ if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
   set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 endif()
 
+find_program(CCACHE_FOUND ccache)
+if(CCACHE_FOUND)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+endif(CCACHE_FOUND)
+
 ############################################################
 # Compiler flags
 ############################################################
@@ -389,7 +395,12 @@ add_subdirectory(src/pyarrow)
 add_subdirectory(src/pyarrow/util)
 
 set(PYARROW_SRCS
+  src/pyarrow/common.cc
+  src/pyarrow/helpers.cc
   src/pyarrow/init.cc
+  src/pyarrow/status.cc
+
+  src/pyarrow/adapters/builtin.cc
 )
 
 set(LINK_LIBS
@@ -410,18 +421,16 @@ endif()
 # Setup and build Cython modules
 ############################################################
 
-foreach(pyx_api_file
-    arrow/config.pyx
-    arrow/parquet.pyx)
-    set_source_files_properties(${pyx_api_file} PROPERTIES CYTHON_API 1)
-endforeach(pyx_api_file)
-
 set(USE_RELATIVE_RPATH ON)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 
 set(CYTHON_EXTENSIONS
+  array
   config
+  error
   parquet
+  scalar
+  schema
 )
 
 foreach(module ${CYTHON_EXTENSIONS})
diff --git a/python/arrow/__init__.py b/python/arrow/__init__.py
index e69de29bb2d1d..3c049b85e8c93 100644
--- a/python/arrow/__init__.py
+++ b/python/arrow/__init__.py
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# flake8: noqa
+
+from arrow.array import (Array, from_pylist, total_allocated_bytes,
+                         BooleanArray, NumericArray,
+                         Int8Array, UInt8Array,
+                         ListArray, StringArray)
+
+from arrow.error import ArrowException
+
+from arrow.scalar import ArrayValue, NA, Scalar
+
+from arrow.schema import (null, bool_,
+                          int8, int16, int32, int64,
+                          uint8, uint16, uint32, uint64,
+                          float_, double, string,
+                          list_, struct, field,
+                          DataType, Field, Schema)
diff --git a/python/arrow/array.pxd b/python/arrow/array.pxd
new file mode 100644
index 0000000000000..e32d27769b5e1
--- /dev/null
+++ b/python/arrow/array.pxd
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.includes.common cimport shared_ptr
+from arrow.includes.arrow cimport CArray, LogicalType
+
+from arrow.scalar import NA
+
+from arrow.schema cimport DataType
+
+cdef extern from "Python.h":
+    int PySlice_Check(object)
+
+cdef class Array:
+    cdef:
+        shared_ptr[CArray] sp_array
+        CArray* ap
+
+    cdef readonly:
+        DataType type
+
+    cdef init(self, const shared_ptr[CArray]& sp_array)
+    cdef _getitem(self, int i)
+
+
+cdef class BooleanArray(Array):
+    pass
+
+
+cdef class NumericArray(Array):
+    pass
+
+
+cdef class Int8Array(NumericArray):
+    pass
+
+
+cdef class UInt8Array(NumericArray):
+    pass
+
+
+cdef class Int16Array(NumericArray):
+    pass
+
+
+cdef class UInt16Array(NumericArray):
+    pass
+
+
+cdef class Int32Array(NumericArray):
+    pass
+
+
+cdef class UInt32Array(NumericArray):
+    pass
+
+
+cdef class Int64Array(NumericArray):
+    pass
+
+
+cdef class UInt64Array(NumericArray):
+    pass
+
+
+cdef class ListArray(Array):
+    pass
+
+
+cdef class StringArray(Array):
+    pass
diff --git a/python/arrow/array.pyx b/python/arrow/array.pyx
new file mode 100644
index 0000000000000..3a3210d6cc100
--- /dev/null
+++ b/python/arrow/array.pyx
@@ -0,0 +1,179 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from arrow.includes.arrow cimport *
+cimport arrow.includes.pyarrow as pyarrow
+
+from arrow.compat import frombytes, tobytes
+from arrow.error cimport check_status
+
+from arrow.scalar import NA
+
+def total_allocated_bytes():
+    cdef MemoryPool* pool = pyarrow.GetMemoryPool()
+    return pool.bytes_allocated()
+
+
+cdef class Array:
+
+    cdef init(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+        self.ap = sp_array.get()
+        self.type = DataType()
+        self.type.init(self.sp_array.get().type())
+
+    property null_count:
+
+        def __get__(self):
+            return self.sp_array.get().null_count()
+
+    def __len__(self):
+        return self.sp_array.get().length()
+
+    def isnull(self):
+        raise NotImplemented
+
+    def __getitem__(self, key):
+        cdef:
+            Py_ssize_t n = len(self)
+
+        if PySlice_Check(key):
+            start = key.start or 0
+            while start < 0:
+                start += n
+
+            stop = key.stop if key.stop is not None else n
+            while stop < 0:
+                stop += n
+
+            step = key.step or 1
+            if step != 1:
+                raise NotImplementedError
+            else:
+                return self.slice(start, stop)
+
+        while key < 0:
+            key += len(self)
+
+        if self.ap.IsNull(key):
+            return NA
+        else:
+            return self._getitem(key)
+
+    cdef _getitem(self, int i):
+        raise NotImplementedError
+
+    def slice(self, start, end):
+        pass
+
+
+cdef class NullArray(Array):
+    pass
+
+
+cdef class BooleanArray(Array):
+    pass
+
+
+cdef class NumericArray(Array):
+    pass
+
+
+cdef class Int8Array(NumericArray):
+    pass
+
+
+cdef class UInt8Array(NumericArray):
+    pass
+
+
+cdef class Int16Array(NumericArray):
+    pass
+
+
+cdef class UInt16Array(NumericArray):
+    pass
+
+
+cdef class Int32Array(NumericArray):
+    pass
+
+
+cdef class UInt32Array(NumericArray):
+    pass
+
+
+cdef class Int64Array(NumericArray):
+    pass
+
+
+cdef class UInt64Array(NumericArray):
+    pass
+
+
+cdef class FloatArray(NumericArray):
+    pass
+
+
+cdef class DoubleArray(NumericArray):
+    pass
+
+
+cdef class ListArray(Array):
+    pass
+
+
+cdef class StringArray(Array):
+    pass
+
+
+cdef dict _array_classes = {
+    LogicalType_NA: NullArray,
+    LogicalType_BOOL: BooleanArray,
+    LogicalType_INT64: Int64Array,
+    LogicalType_DOUBLE: DoubleArray,
+    LogicalType_LIST: ListArray,
+    LogicalType_STRING: StringArray,
+}
+
+cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
+    if sp_array.get() == NULL:
+        raise ValueError('Array was NULL')
+
+    cdef CDataType* data_type = sp_array.get().type().get()
+
+    if data_type == NULL:
+        raise ValueError('Array data type was NULL')
+
+    cdef Array arr = _array_classes[data_type.type]()
+    arr.init(sp_array)
+    return arr
+
+
+def from_pylist(object list_obj, type=None):
+    """
+    Convert Python list to Arrow array
+    """
+    cdef:
+        shared_ptr[CArray] sp_array
+
+    check_status(pyarrow.ConvertPySequence(list_obj, &sp_array))
+    return box_arrow_array(sp_array)
diff --git a/python/arrow/config.pyx b/python/arrow/config.pyx
index 8f10beb3a2e72..521bc066cd4a5 100644
--- a/python/arrow/config.pyx
+++ b/python/arrow/config.pyx
@@ -2,7 +2,7 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-cdef extern from 'pyarrow/init.h' namespace 'arrow::py':
+cdef extern from 'pyarrow/init.h' namespace 'pyarrow':
     void pyarrow_init()
 
 pyarrow_init()
diff --git a/python/arrow/error.pxd b/python/arrow/error.pxd
new file mode 100644
index 0000000000000..c18cb3efffca6
--- /dev/null
+++ b/python/arrow/error.pxd
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.includes.pyarrow cimport *
+
+cdef check_status(const Status& status)
diff --git a/python/arrow/error.pyx b/python/arrow/error.pyx
new file mode 100644
index 0000000000000..f1d516358819d
--- /dev/null
+++ b/python/arrow/error.pyx
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.includes.common cimport c_string
+
+from arrow.compat import frombytes
+
+class ArrowException(Exception):
+    pass
+
+cdef check_status(const Status& status):
+    if status.ok():
+        return
+
+    cdef c_string c_message = status.ToString()
+    raise ArrowException(frombytes(c_message))
diff --git a/python/arrow/includes/arrow.pxd b/python/arrow/includes/arrow.pxd
index 3635ceb868596..fde5de910915a 100644
--- a/python/arrow/includes/arrow.pxd
+++ b/python/arrow/includes/arrow.pxd
@@ -20,4 +20,77 @@
 from arrow.includes.common cimport *
 
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
-    pass
+
+    enum LogicalType" arrow::LogicalType::type":
+        LogicalType_NA" arrow::LogicalType::NA"
+
+        LogicalType_BOOL" arrow::LogicalType::BOOL"
+
+        LogicalType_UINT8" arrow::LogicalType::UINT8"
+        LogicalType_INT8" arrow::LogicalType::INT8"
+        LogicalType_UINT16" arrow::LogicalType::UINT16"
+        LogicalType_INT16" arrow::LogicalType::INT16"
+        LogicalType_UINT32" arrow::LogicalType::UINT32"
+        LogicalType_INT32" arrow::LogicalType::INT32"
+        LogicalType_UINT64" arrow::LogicalType::UINT64"
+        LogicalType_INT64" arrow::LogicalType::INT64"
+
+        LogicalType_FLOAT" arrow::LogicalType::FLOAT"
+        LogicalType_DOUBLE" arrow::LogicalType::DOUBLE"
+
+        LogicalType_STRING" arrow::LogicalType::STRING"
+
+        LogicalType_LIST" arrow::LogicalType::LIST"
+        LogicalType_STRUCT" arrow::LogicalType::STRUCT"
+
+    cdef cppclass CDataType" arrow::DataType":
+        LogicalType type
+        c_bool nullable
+
+        c_bool Equals(const CDataType* other)
+
+        c_string ToString()
+
+    cdef cppclass MemoryPool" arrow::MemoryPool":
+        int64_t bytes_allocated()
+
+    cdef cppclass CListType" arrow::ListType"(CDataType):
+        CListType(const shared_ptr[CDataType]& value_type,
+                  c_bool nullable)
+
+    cdef cppclass CStringType" arrow::StringType"(CDataType):
+        pass
+
+    cdef cppclass CField" arrow::Field":
+        c_string name
+        shared_ptr[CDataType] type
+
+        CField(const c_string& name, const shared_ptr[CDataType]& type)
+
+    cdef cppclass CStructType" arrow::StructType"(CDataType):
+        CStructType(const vector[shared_ptr[CField]]& fields,
+                    c_bool nullable)
+
+    cdef cppclass CSchema" arrow::Schema":
+        CSchema(const shared_ptr[CField]& fields)
+
+    cdef cppclass CArray" arrow::Array":
+        const shared_ptr[CDataType]& type()
+
+        int32_t length()
+        int32_t null_count()
+        LogicalType logical_type()
+
+        c_bool IsNull(int i)
+
+    cdef cppclass CUInt8Array" arrow::UInt8Array"(CArray):
+        pass
+
+    cdef cppclass CInt8Array" arrow::Int8Array"(CArray):
+        pass
+
+    cdef cppclass CListArray" arrow::ListArray"(CArray):
+        pass
+
+    cdef cppclass CStringArray" arrow::StringArray"(CListArray):
+        pass
diff --git a/python/arrow/includes/common.pxd b/python/arrow/includes/common.pxd
index f2fc826625e45..839427a699002 100644
--- a/python/arrow/includes/common.pxd
+++ b/python/arrow/includes/common.pxd
@@ -19,7 +19,7 @@
 
 from libc.stdint cimport *
 from libcpp cimport bool as c_bool
-from libcpp.string cimport string
+from libcpp.string cimport string as c_string
 from libcpp.vector cimport vector
 
 # This must be included for cerr and other things to work
@@ -29,6 +29,8 @@ cdef extern from "<iostream>":
 cdef extern from "<memory>" namespace "std" nogil:
 
     cdef cppclass shared_ptr[T]:
+        shared_ptr()
+        shared_ptr(T*)
         T* get()
         void reset()
         void reset(T* p)
diff --git a/python/arrow/includes/pyarrow.pxd b/python/arrow/includes/pyarrow.pxd
index dcef663f3894d..3eed5b8542493 100644
--- a/python/arrow/includes/pyarrow.pxd
+++ b/python/arrow/includes/pyarrow.pxd
@@ -18,6 +18,28 @@
 # distutils: language = c++
 
 from arrow.includes.common cimport *
+from arrow.includes.arrow cimport (CArray, CDataType, LogicalType,
+                                   MemoryPool)
 
 cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
-    pass
+    # We can later add more of the common status factory methods as needed
+    cdef Status Status_OK "Status::OK"()
+
+    cdef cppclass Status:
+        Status()
+
+        c_string ToString()
+
+        c_bool ok()
+        c_bool IsOutOfMemory()
+        c_bool IsKeyError()
+        c_bool IsTypeError()
+        c_bool IsIOError()
+        c_bool IsValueError()
+        c_bool IsNotImplemented()
+        c_bool IsArrowError()
+
+    shared_ptr[CDataType] GetPrimitiveType(LogicalType type, c_bool nullable)
+    Status ConvertPySequence(object obj, shared_ptr[CArray]* out)
+
+    MemoryPool* GetMemoryPool()
diff --git a/python/arrow/scalar.pxd b/python/arrow/scalar.pxd
new file mode 100644
index 0000000000000..e193c09cd69a2
--- /dev/null
+++ b/python/arrow/scalar.pxd
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.includes.common cimport *
+from arrow.includes.arrow cimport CArray, CListArray
+
+from arrow.schema cimport DataType
+
+cdef class Scalar:
+    cdef readonly:
+        DataType type
+
+
+cdef class NAType(Scalar):
+    pass
+
+
+cdef class ArrayValue(Scalar):
+    cdef:
+        shared_ptr[CArray] array
+        int index
+
+
+cdef class Int8Value(ArrayValue):
+    pass
+
+
+cdef class ListValue(ArrayValue):
+    pass
+
+
+cdef class StringValue(ArrayValue):
+    pass
diff --git a/python/arrow/scalar.pyx b/python/arrow/scalar.pyx
new file mode 100644
index 0000000000000..78dadecf9b422
--- /dev/null
+++ b/python/arrow/scalar.pyx
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import arrow.schema as schema
+
+cdef class NAType(Scalar):
+
+    def __cinit__(self):
+        self.type = schema.null()
+
+    def __repr__(self):
+        return 'NA'
+
+NA = NAType()
diff --git a/python/arrow/schema.pxd b/python/arrow/schema.pxd
new file mode 100644
index 0000000000000..487c246f44abf
--- /dev/null
+++ b/python/arrow/schema.pxd
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.includes.common cimport shared_ptr
+from arrow.includes.arrow cimport CDataType, CField, CSchema
+
+cdef class DataType:
+    cdef:
+        shared_ptr[CDataType] sp_type
+        CDataType* type
+
+    cdef init(self, const shared_ptr[CDataType]& type)
+
+cdef class Field:
+    cdef:
+        shared_ptr[CField] sp_field
+        CField* field
+
+    cdef readonly:
+        DataType type
+
+cdef class Schema:
+    cdef:
+        shared_ptr[CSchema] sp_schema
+        CSchema* schema
diff --git a/python/arrow/schema.pyx b/python/arrow/schema.pyx
new file mode 100644
index 0000000000000..63cd6e888abd0
--- /dev/null
+++ b/python/arrow/schema.pyx
@@ -0,0 +1,150 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+########################################
+# Data types, fields, schemas, and so forth
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from arrow.compat import frombytes, tobytes
+from arrow.includes.arrow cimport *
+cimport arrow.includes.pyarrow as pyarrow
+
+cimport cpython
+
+cdef class DataType:
+
+    def __cinit__(self):
+        pass
+
+    cdef init(self, const shared_ptr[CDataType]& type):
+        self.sp_type = type
+        self.type = type.get()
+
+    def __str__(self):
+        return frombytes(self.type.ToString())
+
+    def __repr__(self):
+        return 'DataType({0})'.format(str(self))
+
+    def __richcmp__(DataType self, DataType other, int op):
+        if op == cpython.Py_EQ:
+            return self.type.Equals(other.type)
+        elif op == cpython.Py_NE:
+            return not self.type.Equals(other.type)
+        else:
+            raise TypeError('Invalid comparison')
+
+
+cdef class Field:
+
+    def __cinit__(self, object name, DataType type):
+        self.type = type
+        self.sp_field.reset(new CField(tobytes(name), type.sp_type))
+        self.field = self.sp_field.get()
+
+    def __repr__(self):
+        return 'Field({0!r}, type={1})'.format(self.name, str(self.type))
+
+    property name:
+
+        def __get__(self):
+            return frombytes(self.field.name)
+
+cdef dict _type_cache = {}
+
+cdef DataType primitive_type(LogicalType type, bint nullable=True):
+    if (type, nullable) in _type_cache:
+        return _type_cache[type, nullable]
+
+    cdef DataType out = DataType()
+    out.init(pyarrow.GetPrimitiveType(type, nullable))
+
+    _type_cache[type, nullable] = out
+    return out
+
+#------------------------------------------------------------
+# Type factory functions
+
+def field(name, type):
+    return Field(name, type)
+
+def null():
+    return primitive_type(LogicalType_NA)
+
+def bool_(c_bool nullable=True):
+    return primitive_type(LogicalType_BOOL, nullable)
+
+def uint8(c_bool nullable=True):
+    return primitive_type(LogicalType_UINT8, nullable)
+
+def int8(c_bool nullable=True):
+    return primitive_type(LogicalType_INT8, nullable)
+
+def uint16(c_bool nullable=True):
+    return primitive_type(LogicalType_UINT16, nullable)
+
+def int16(c_bool nullable=True):
+    return primitive_type(LogicalType_INT16, nullable)
+
+def uint32(c_bool nullable=True):
+    return primitive_type(LogicalType_UINT32, nullable)
+
+def int32(c_bool nullable=True):
+    return primitive_type(LogicalType_INT32, nullable)
+
+def uint64(c_bool nullable=True):
+    return primitive_type(LogicalType_UINT64, nullable)
+
+def int64(c_bool nullable=True):
+    return primitive_type(LogicalType_INT64, nullable)
+
+def float_(c_bool nullable=True):
+    return primitive_type(LogicalType_FLOAT, nullable)
+
+def double(c_bool nullable=True):
+    return primitive_type(LogicalType_DOUBLE, nullable)
+
+def string(c_bool nullable=True):
+    """
+    UTF8 string
+    """
+    return primitive_type(LogicalType_STRING, nullable)
+
+def list_(DataType value_type, c_bool nullable=True):
+    cdef DataType out = DataType()
+    out.init(shared_ptr[CDataType](
+        new CListType(value_type.sp_type, nullable)))
+    return out
+
+def struct(fields, c_bool nullable=True):
+    """
+
+    """
+    cdef:
+        DataType out = DataType()
+        Field field
+        vector[shared_ptr[CField]] c_fields
+
+    for field in fields:
+        c_fields.push_back(field.sp_field)
+
+    out.init(shared_ptr[CDataType](
+        new CStructType(c_fields, nullable)))
+    return out
diff --git a/python/arrow/tests/test_array.py b/python/arrow/tests/test_array.py
new file mode 100644
index 0000000000000..8eaa53352061b
--- /dev/null
+++ b/python/arrow/tests/test_array.py
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.compat import unittest
+import arrow
+
+
+class TestArrayAPI(unittest.TestCase):
+
+    def test_getitem_NA(self):
+        arr = arrow.from_pylist([1, None, 2])
+        assert arr[1] is arrow.NA
diff --git a/python/arrow/tests/test_convert_builtin.py b/python/arrow/tests/test_convert_builtin.py
new file mode 100644
index 0000000000000..57e6ab9f0e7b5
--- /dev/null
+++ b/python/arrow/tests/test_convert_builtin.py
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.compat import unittest
+import arrow
+
+
+class TestConvertList(unittest.TestCase):
+
+    def test_boolean(self):
+        pass
+
+    def test_empty_list(self):
+        arr = arrow.from_pylist([])
+        assert len(arr) == 0
+        assert arr.null_count == 0
+        assert arr.type == arrow.null()
+
+    def test_all_none(self):
+        arr = arrow.from_pylist([None, None])
+        assert len(arr) == 2
+        assert arr.null_count == 2
+        assert arr.type == arrow.null()
+
+    def test_integer(self):
+        arr = arrow.from_pylist([1, None, 3, None])
+        assert len(arr) == 4
+        assert arr.null_count == 2
+        assert arr.type == arrow.int64()
+
+    def test_garbage_collection(self):
+        import gc
+        bytes_before = arrow.total_allocated_bytes()
+        arrow.from_pylist([1, None, 3, None])
+        gc.collect()
+        assert arrow.total_allocated_bytes() == bytes_before
+
+    def test_double(self):
+        data = [1.5, 1, None, 2.5, None, None]
+        arr = arrow.from_pylist(data)
+        assert len(arr) == 6
+        assert arr.null_count == 3
+        assert arr.type == arrow.double()
+
+    def test_string(self):
+        data = ['foo', b'bar', None, 'arrow']
+        arr = arrow.from_pylist(data)
+        assert len(arr) == 4
+        assert arr.null_count == 1
+        assert arr.type == arrow.string()
+
+    def test_mixed_nesting_levels(self):
+        arrow.from_pylist([1, 2, None])
+        arrow.from_pylist([[1], [2], None])
+        arrow.from_pylist([[1], [2], [None]])
+
+        with self.assertRaises(arrow.ArrowException):
+            arrow.from_pylist([1, 2, [1]])
+
+        with self.assertRaises(arrow.ArrowException):
+            arrow.from_pylist([1, 2, []])
+
+        with self.assertRaises(arrow.ArrowException):
+            arrow.from_pylist([[1], [2], [None, [1]]])
+
+    def test_list_of_int(self):
+        data = [[1, 2, 3], [], None, [1, 2]]
+        arr = arrow.from_pylist(data)
+        assert len(arr) == 4
+        assert arr.null_count == 1
+        assert arr.type == arrow.list_(arrow.int64())
diff --git a/python/arrow/tests/test_schema.py b/python/arrow/tests/test_schema.py
new file mode 100644
index 0000000000000..a89edd74a0adf
--- /dev/null
+++ b/python/arrow/tests/test_schema.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.compat import unittest
+import arrow
+
+
+class TestTypes(unittest.TestCase):
+
+    def test_integers(self):
+        dtypes = ['int8', 'int16', 'int32', 'int64',
+                  'uint8', 'uint16', 'uint32', 'uint64']
+
+        for name in dtypes:
+            factory = getattr(arrow, name)
+            t = factory()
+            t_required = factory(False)
+
+            assert str(t) == name
+            assert str(t_required) == '{0} not null'.format(name)
+
+    def test_list(self):
+        value_type = arrow.int32()
+        list_type = arrow.list_(value_type)
+        assert str(list_type) == 'list<int32>'
+
+    def test_string(self):
+        t = arrow.string()
+        assert str(t) == 'string'
+
+    def test_field(self):
+        t = arrow.string()
+        f = arrow.field('foo', t)
+
+        assert f.name == 'foo'
+        assert f.type is t
+        assert repr(f) == "Field('foo', type=string)"
diff --git a/python/setup.py b/python/setup.py
index f6b0a4bee8316..9a0de071a9c40 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -124,7 +124,10 @@ def _run_cmake(self):
                              static_lib_option, source]
 
             self.spawn(cmake_command)
-            self.spawn(['make'])
+            args = ['make']
+            if 'PYARROW_PARALLEL' in os.environ:
+                args.append('-j{0}'.format(os.environ['PYARROW_PARALLEL']))
+            self.spawn(args)
         else:
             import shlex
             cmake_generator = 'Visual Studio 14 2015'
@@ -207,7 +210,7 @@ def get_ext_built(self, name):
             return name + suffix
 
     def get_cmake_cython_names(self):
-        return ['config', 'parquet']
+        return ['array', 'config', 'error', 'parquet', 'scalar', 'schema']
 
     def get_names(self):
         return self._found_names
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
new file mode 100644
index 0000000000000..ae84fa12b0de6
--- /dev/null
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -0,0 +1,415 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <Python.h>
+#include <sstream>
+
+#include "pyarrow/adapters/builtin.h"
+
+#include <arrow/api.h>
+
+#include "pyarrow/status.h"
+
+using arrow::ArrayBuilder;
+using arrow::DataType;
+using arrow::LogicalType;
+
+namespace pyarrow {
+
+static inline bool IsPyInteger(PyObject* obj) {
+#if PYARROW_IS_PY2
+  return PyLong_Check(obj) || PyInt_Check(obj);
+#else
+  return PyLong_Check(obj);
+#endif
+}
+
+static inline bool IsPyBaseString(PyObject* obj) {
+#if PYARROW_IS_PY2
+  return PyString_Check(obj) || PyUnicode_Check(obj);
+#else
+  return PyUnicode_Check(obj);
+#endif
+}
+
+class ScalarVisitor {
+ public:
+  ScalarVisitor() :
+      total_count_(0),
+      none_count_(0),
+      bool_count_(0),
+      int_count_(0),
+      float_count_(0),
+      string_count_(0) {}
+
+  void Visit(PyObject* obj) {
+    ++total_count_;
+    if (obj == Py_None) {
+      ++none_count_;
+    } else if (PyFloat_Check(obj)) {
+      ++float_count_;
+    } else if (IsPyInteger(obj)) {
+      ++int_count_;
+    } else if (IsPyBaseString(obj)) {
+      ++string_count_;
+    } else {
+      // TODO(wesm): accumulate error information somewhere
+    }
+  }
+
+  std::shared_ptr<DataType> GetType() {
+    // TODO(wesm): handling mixed-type cases
+    if (float_count_) {
+      return arrow::DOUBLE;
+    } else if (int_count_) {
+      // TODO(wesm): tighter type later
+      return arrow::INT64;
+    } else if (bool_count_) {
+      return arrow::BOOL;
+    } else if (string_count_) {
+      return arrow::STRING;
+    } else {
+      return arrow::NA;
+    }
+  }
+
+  int64_t total_count() const {
+    return total_count_;
+  }
+
+ private:
+  int64_t total_count_;
+  int64_t none_count_;
+  int64_t bool_count_;
+  int64_t int_count_;
+  int64_t float_count_;
+  int64_t string_count_;
+
+  // Place to accumulate errors
+  // std::vector<Status> errors_;
+};
+
+static constexpr int MAX_NESTING_LEVELS = 32;
+
+class SeqVisitor {
+ public:
+  SeqVisitor() :
+      max_nesting_level_(0) {
+    memset(nesting_histogram_, 0, MAX_NESTING_LEVELS * sizeof(int));
+  }
+
+  Status Visit(PyObject* obj, int level=0) {
+    Py_ssize_t size = PySequence_Size(obj);
+
+    if (level > max_nesting_level_) {
+      max_nesting_level_ = level;
+    }
+
+    for (int64_t i = 0; i < size; ++i) {
+      // TODO(wesm): Error checking?
+      // TODO(wesm): Specialize for PyList_GET_ITEM?
+      OwnedRef item_ref(PySequence_GetItem(obj, i));
+      PyObject* item = item_ref.obj();
+
+      if (PyList_Check(item)) {
+        PY_RETURN_NOT_OK(Visit(item, level + 1));
+      } else if (PyDict_Check(item)) {
+        return Status::NotImplemented("No type inference for dicts");
+      } else {
+        // We permit nulls at any level of nesting
+        if (item == Py_None) {
+          // TODO
+        } else {
+          ++nesting_histogram_[level];
+          scalars_.Visit(item);
+        }
+      }
+    }
+    return Status::OK();
+  }
+
+  std::shared_ptr<DataType> GetType() {
+    if (scalars_.total_count() == 0) {
+      if (max_nesting_level_ == 0) {
+        return arrow::NA;
+      } else {
+        return nullptr;
+      }
+    } else {
+      std::shared_ptr<DataType> result = scalars_.GetType();
+      for (int i = 0; i < max_nesting_level_; ++i) {
+        result = std::make_shared<arrow::ListType>(result);
+      }
+      return result;
+    }
+  }
+
+  Status Validate() const {
+    if (scalars_.total_count() > 0) {
+      if (num_nesting_levels() > 1) {
+        return Status::ValueError("Mixed nesting levels not supported");
+      } else if (max_observed_level() < max_nesting_level_) {
+        return Status::ValueError("Mixed nesting levels not supported");
+      }
+    }
+    return Status::OK();
+  }
+
+  int max_observed_level() const {
+    int result = 0;
+    for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
+      if (nesting_histogram_[i] > 0) {
+        result = i;
+      }
+    }
+    return result;
+  }
+
+  int num_nesting_levels() const {
+    int result = 0;
+    for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
+      if (nesting_histogram_[i] > 0) {
+        ++result;
+      }
+    }
+    return result;
+  }
+
+ private:
+  ScalarVisitor scalars_;
+
+  // Track observed
+  int max_nesting_level_;
+  int nesting_histogram_[MAX_NESTING_LEVELS];
+};
+
+// Non-exhaustive type inference
+static Status InferArrowType(PyObject* obj, int64_t* size,
+    std::shared_ptr<DataType>* out_type) {
+  *size = PySequence_Size(obj);
+  if (PyErr_Occurred()) {
+    // Not a sequence
+    PyErr_Clear();
+    return Status::TypeError("Object is not a sequence");
+  }
+
+  // For 0-length sequences, refuse to guess
+  if (*size == 0) {
+    *out_type = arrow::NA;
+  }
+
+  SeqVisitor seq_visitor;
+  PY_RETURN_NOT_OK(seq_visitor.Visit(obj));
+  PY_RETURN_NOT_OK(seq_visitor.Validate());
+
+  *out_type = seq_visitor.GetType();
+  return Status::OK();
+}
+
+// Marshal Python sequence (list, tuple, etc.) to Arrow array
+class SeqConverter {
+ public:
+  virtual Status Init(const std::shared_ptr<ArrayBuilder>& builder) {
+    builder_ = builder;
+    return Status::OK();
+  }
+
+  virtual Status AppendData(PyObject* seq) = 0;
+
+ protected:
+  std::shared_ptr<ArrayBuilder> builder_;
+};
+
+template <typename BuilderType>
+class TypedConverter : public SeqConverter {
+ public:
+  Status Init(const std::shared_ptr<ArrayBuilder>& builder) override {
+    builder_ = builder;
+    typed_builder_ = static_cast<BuilderType*>(builder.get());
+    return Status::OK();
+  }
+
+ protected:
+  BuilderType* typed_builder_;
+};
+
+class BoolConverter : public TypedConverter<arrow::BooleanBuilder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    return Status::OK();
+  }
+};
+
+class Int64Converter : public TypedConverter<arrow::Int64Builder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    int64_t val;
+    Py_ssize_t size = PySequence_Size(seq);
+    for (int64_t i = 0; i < size; ++i) {
+      OwnedRef item(PySequence_GetItem(seq, i));
+      if (item.obj() == Py_None) {
+        RETURN_ARROW_NOT_OK(typed_builder_->AppendNull());
+      } else {
+        val = PyLong_AsLongLong(item.obj());
+        RETURN_IF_PYERROR();
+        RETURN_ARROW_NOT_OK(typed_builder_->Append(val));
+      }
+    }
+    return Status::OK();
+  }
+};
+
+class DoubleConverter : public TypedConverter<arrow::DoubleBuilder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    int64_t val;
+    Py_ssize_t size = PySequence_Size(seq);
+    for (int64_t i = 0; i < size; ++i) {
+      OwnedRef item(PySequence_GetItem(seq, i));
+      if (item.obj() == Py_None) {
+        RETURN_ARROW_NOT_OK(typed_builder_->AppendNull());
+      } else {
+        val = PyFloat_AsDouble(item.obj());
+        RETURN_IF_PYERROR();
+        RETURN_ARROW_NOT_OK(typed_builder_->Append(val));
+      }
+    }
+    return Status::OK();
+  }
+};
+
+class StringConverter : public TypedConverter<arrow::StringBuilder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    PyObject* item;
+    PyObject* bytes_obj;
+    OwnedRef tmp;
+    const char* bytes;
+    int32_t length;
+    Py_ssize_t size = PySequence_Size(seq);
+    for (int64_t i = 0; i < size; ++i) {
+      item = PySequence_GetItem(seq, i);
+      OwnedRef holder(item);
+
+      if (item == Py_None) {
+        RETURN_ARROW_NOT_OK(typed_builder_->AppendNull());
+        continue;
+      } else if (PyUnicode_Check(item)) {
+        tmp.reset(PyUnicode_AsUTF8String(item));
+        RETURN_IF_PYERROR();
+        bytes_obj = tmp.obj();
+      } else if (PyBytes_Check(item)) {
+        bytes_obj = item;
+      } else {
+        return Status::TypeError("Non-string value encountered");
+      }
+      // No error checking
+      length = PyBytes_GET_SIZE(bytes_obj);
+      bytes = PyBytes_AS_STRING(bytes_obj);
+      RETURN_ARROW_NOT_OK(typed_builder_->Append(bytes, length));
+    }
+    return Status::OK();
+  }
+};
+
+class ListConverter : public TypedConverter<arrow::ListBuilder> {
+ public:
+  Status Init(const std::shared_ptr<ArrayBuilder>& builder) override;
+
+  Status AppendData(PyObject* seq) override {
+    Py_ssize_t size = PySequence_Size(seq);
+    for (int64_t i = 0; i < size; ++i) {
+      OwnedRef item(PySequence_GetItem(seq, i));
+      if (item.obj() == Py_None) {
+        RETURN_ARROW_NOT_OK(typed_builder_->AppendNull());
+      } else {
+        typed_builder_->Append();
+        PY_RETURN_NOT_OK(value_converter_->AppendData(item.obj()));
+      }
+    }
+    return Status::OK();
+  }
+ protected:
+  std::shared_ptr<SeqConverter> value_converter_;
+};
+
+// Dynamic constructor for sequence converters
+std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type) {
+  switch (type->type) {
+    case LogicalType::BOOL:
+      return std::make_shared<BoolConverter>();
+    case LogicalType::INT64:
+      return std::make_shared<Int64Converter>();
+    case LogicalType::DOUBLE:
+      return std::make_shared<DoubleConverter>();
+    case LogicalType::STRING:
+      return std::make_shared<StringConverter>();
+    case LogicalType::LIST:
+      return std::make_shared<ListConverter>();
+    case LogicalType::STRUCT:
+    default:
+      return nullptr;
+      break;
+  }
+}
+
+Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
+  builder_ = builder;
+  typed_builder_ = static_cast<arrow::ListBuilder*>(builder.get());
+
+  value_converter_ = GetConverter(static_cast<arrow::ListType*>(
+          builder->type().get())->value_type);
+  if (value_converter_ == nullptr) {
+    return Status::NotImplemented("value type not implemented");
+  }
+
+  value_converter_->Init(typed_builder_->value_builder());
+  return Status::OK();
+}
+
+Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
+  std::shared_ptr<DataType> type;
+  int64_t size;
+  PY_RETURN_NOT_OK(InferArrowType(obj, &size, &type));
+
+  // Handle NA / NullType case
+  if (type->type == LogicalType::NA) {
+    out->reset(new arrow::Array(type, size, size));
+    return Status::OK();
+  }
+
+  std::shared_ptr<SeqConverter> converter = GetConverter(type);
+  if (converter == nullptr) {
+    std::stringstream ss;
+    ss << "No type converter implemented for "
+       << type->ToString();
+    return Status::NotImplemented(ss.str());
+  }
+
+  // Give the sequence converter an array builder
+  std::shared_ptr<ArrayBuilder> builder;
+  RETURN_ARROW_NOT_OK(arrow::MakeBuilder(GetMemoryPool(), type, &builder));
+  converter->Init(builder);
+
+  PY_RETURN_NOT_OK(converter->AppendData(obj));
+
+  *out = builder->Finish();
+
+  return Status::OK();
+}
+
+} // namespace pyarrow
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
new file mode 100644
index 0000000000000..24886f4970d50
--- /dev/null
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Functions for converting between CPython built-in data structures and Arrow
+// data structures
+
+#ifndef PYARROW_ADAPTERS_BUILTIN_H
+#define PYARROW_ADAPTERS_BUILTIN_H
+
+#include <Python.h>
+
+#include <memory>
+
+#include "pyarrow/common.h"
+
+namespace arrow { class Array; }
+
+namespace pyarrow {
+
+class Status;
+
+Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out);
+
+} // namespace pyarrow
+
+#endif // PYARROW_ADAPTERS_BUILTIN_H
diff --git a/cpp/src/arrow/field.cc b/python/src/pyarrow/adapters/pandas.h
similarity index 76%
rename from cpp/src/arrow/field.cc
rename to python/src/pyarrow/adapters/pandas.h
index 4568d905c2991..a4f4163808711 100644
--- a/cpp/src/arrow/field.cc
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -15,17 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/field.h"
+// Functions for converting between pandas's NumPy-based data representation
+// and Arrow data structures
 
-#include <sstream>
-#include <string>
+#ifndef PYARROW_ADAPTERS_PANDAS_H
+#define PYARROW_ADAPTERS_PANDAS_H
 
-namespace arrow {
+namespace pyarrow {
 
-std::string Field::ToString() const {
-  std::stringstream ss;
-  ss << this->name << " " << this->type->ToString();
-  return ss.str();
-}
+} // namespace pyarrow
 
-} // namespace arrow
+#endif // PYARROW_ADAPTERS_PANDAS_H
diff --git a/python/src/pyarrow/api.h b/python/src/pyarrow/api.h
index c2285de77bf10..72be6afe02c76 100644
--- a/python/src/pyarrow/api.h
+++ b/python/src/pyarrow/api.h
@@ -18,4 +18,11 @@
 #ifndef PYARROW_API_H
 #define PYARROW_API_H
 
+#include "pyarrow/status.h"
+
+#include "pyarrow/helpers.h"
+
+#include "pyarrow/adapters/builtin.h"
+#include "pyarrow/adapters/pandas.h"
+
 #endif // PYARROW_API_H
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
new file mode 100644
index 0000000000000..a2748f99b6733
--- /dev/null
+++ b/python/src/pyarrow/common.cc
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "pyarrow/common.h"
+
+#include <cstdlib>
+#include <mutex>
+#include <sstream>
+
+#include <arrow/util/memory-pool.h>
+#include <arrow/util/status.h>
+
+#include "pyarrow/status.h"
+
+namespace pyarrow {
+
+class PyArrowMemoryPool : public arrow::MemoryPool {
+ public:
+  PyArrowMemoryPool() : bytes_allocated_(0) {}
+  virtual ~PyArrowMemoryPool() {}
+
+  arrow::Status Allocate(int64_t size, uint8_t** out) override {
+    std::lock_guard<std::mutex> guard(pool_lock_);
+    *out = static_cast<uint8_t*>(std::malloc(size));
+    if (*out == nullptr) {
+      std::stringstream ss;
+      ss << "malloc of size " << size << " failed";
+      return arrow::Status::OutOfMemory(ss.str());
+    }
+
+    bytes_allocated_ += size;
+
+    return arrow::Status::OK();
+  }
+
+  int64_t bytes_allocated() const override {
+    std::lock_guard<std::mutex> guard(pool_lock_);
+    return bytes_allocated_;
+  }
+
+  void Free(uint8_t* buffer, int64_t size) override {
+    std::lock_guard<std::mutex> guard(pool_lock_);
+    std::free(buffer);
+    bytes_allocated_ -= size;
+  }
+
+ private:
+  mutable std::mutex pool_lock_;
+  int64_t bytes_allocated_;
+};
+
+arrow::MemoryPool* GetMemoryPool() {
+  static PyArrowMemoryPool memory_pool;
+  return &memory_pool;
+}
+
+} // namespace pyarrow
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
new file mode 100644
index 0000000000000..a43e4d28c899a
--- /dev/null
+++ b/python/src/pyarrow/common.h
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_COMMON_H
+#define PYARROW_COMMON_H
+
+#include <Python.h>
+
+namespace arrow { class MemoryPool; }
+
+namespace pyarrow {
+
+#define PYARROW_IS_PY2 PY_MAJOR_VERSION < 2
+
+#define RETURN_ARROW_NOT_OK(s) do {             \
+    arrow::Status _s = (s);                     \
+    if (!_s.ok()) {                             \
+      return Status::ArrowError(s.ToString());  \
+    }                                           \
+  } while (0);
+
+class OwnedRef {
+ public:
+  OwnedRef() : obj_(nullptr) {}
+
+  OwnedRef(PyObject* obj) :
+      obj_(obj) {}
+
+  ~OwnedRef() {
+    Py_XDECREF(obj_);
+  }
+
+  void reset(PyObject* obj) {
+    if (obj_ != nullptr) {
+      Py_XDECREF(obj_);
+    }
+    obj_ = obj;
+  }
+
+  PyObject* obj() const{
+    return obj_;
+  }
+
+ private:
+  PyObject* obj_;
+};
+
+struct PyObjectStringify {
+  OwnedRef tmp_obj;
+  const char* bytes;
+
+  PyObjectStringify(PyObject* obj) {
+    PyObject* bytes_obj;
+    if (PyUnicode_Check(obj)) {
+      bytes_obj = PyUnicode_AsUTF8String(obj);
+      tmp_obj.reset(bytes_obj);
+    } else {
+      bytes_obj = obj;
+    }
+    bytes = PyBytes_AsString(bytes_obj);
+  }
+};
+
+// TODO(wesm): We can just let errors pass through. To be explored later
+#define RETURN_IF_PYERROR()                         \
+  if (PyErr_Occurred()) {                           \
+    PyObject *exc_type, *exc_value, *traceback;     \
+    PyErr_Fetch(&exc_type, &exc_value, &traceback); \
+    PyObjectStringify stringified(exc_value);       \
+    std::string message(stringified.bytes);         \
+    Py_DECREF(exc_type);                            \
+    Py_DECREF(exc_value);                           \
+    Py_DECREF(traceback);                           \
+    return Status::UnknownError(message);           \
+  }
+
+arrow::MemoryPool* GetMemoryPool();
+
+} // namespace pyarrow
+
+#endif // PYARROW_COMMON_H
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
new file mode 100644
index 0000000000000..d0969dacc21e0
--- /dev/null
+++ b/python/src/pyarrow/helpers.cc
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "pyarrow/helpers.h"
+
+#include <arrow/api.h>
+
+using namespace arrow;
+
+namespace pyarrow {
+
+#define GET_PRIMITIVE_TYPE(NAME, Type)          \
+  case LogicalType::NAME:                       \
+    if (nullable) {                             \
+      return NAME;                              \
+    } else {                                    \
+      return std::make_shared<Type>(nullable);  \
+    }                                           \
+    break;
+
+std::shared_ptr<DataType> GetPrimitiveType(LogicalType::type type,
+    bool nullable) {
+  switch (type) {
+    case LogicalType::NA:
+      return NA;
+    GET_PRIMITIVE_TYPE(UINT8, UInt8Type);
+    GET_PRIMITIVE_TYPE(INT8, Int8Type);
+    GET_PRIMITIVE_TYPE(UINT16, UInt16Type);
+    GET_PRIMITIVE_TYPE(INT16, Int16Type);
+    GET_PRIMITIVE_TYPE(UINT32, UInt32Type);
+    GET_PRIMITIVE_TYPE(INT32, Int32Type);
+    GET_PRIMITIVE_TYPE(UINT64, UInt64Type);
+    GET_PRIMITIVE_TYPE(INT64, Int64Type);
+    GET_PRIMITIVE_TYPE(BOOL, BooleanType);
+    GET_PRIMITIVE_TYPE(FLOAT, FloatType);
+    GET_PRIMITIVE_TYPE(DOUBLE, DoubleType);
+    GET_PRIMITIVE_TYPE(STRING, StringType);
+    default:
+      return nullptr;
+  }
+}
+
+} // namespace pyarrow
diff --git a/cpp/src/arrow/types/null.h b/python/src/pyarrow/helpers.h
similarity index 72%
rename from cpp/src/arrow/types/null.h
rename to python/src/pyarrow/helpers.h
index c67f752d40989..1a24f056febe6 100644
--- a/cpp/src/arrow/types/null.h
+++ b/python/src/pyarrow/helpers.h
@@ -15,20 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef ARROW_TYPES_NULL_H
-#define ARROW_TYPES_NULL_H
+#ifndef PYARROW_HELPERS_H
+#define PYARROW_HELPERS_H
 
-#include <string>
-#include <vector>
+#include <arrow/api.h>
+#include <memory>
 
-#include "arrow/type.h"
+namespace pyarrow {
 
-namespace arrow {
+using arrow::DataType;
+using arrow::LogicalType;
 
-struct NullType : public PrimitiveType<NullType> {
-  PRIMITIVE_DECL(NullType, void, NA, 0, "null");
-};
+std::shared_ptr<DataType> GetPrimitiveType(LogicalType::type type,
+    bool nullable);
 
-} // namespace arrow
+} // namespace pyarrow
 
-#endif // ARROW_TYPES_NULL_H
+#endif // PYARROW_HELPERS_H
diff --git a/python/src/pyarrow/init.cc b/python/src/pyarrow/init.cc
index c36f413725532..acd851e168743 100644
--- a/python/src/pyarrow/init.cc
+++ b/python/src/pyarrow/init.cc
@@ -17,13 +17,9 @@
 
 #include "pyarrow/init.h"
 
-namespace arrow {
-
-namespace py {
+namespace pyarrow {
 
 void pyarrow_init() {
 }
 
-} // namespace py
-
-} // namespace arrow
+} // namespace pyarrow
diff --git a/python/src/pyarrow/init.h b/python/src/pyarrow/init.h
index 1fc9f10102696..71e67a20c1ca5 100644
--- a/python/src/pyarrow/init.h
+++ b/python/src/pyarrow/init.h
@@ -18,14 +18,10 @@
 #ifndef PYARROW_INIT_H
 #define PYARROW_INIT_H
 
-namespace arrow {
-
-namespace py {
+namespace pyarrow {
 
 void pyarrow_init();
 
-} // namespace py
-
-} // namespace arrow
+} // namespace pyarrow
 
 #endif // PYARROW_INIT_H
diff --git a/python/src/pyarrow/status.cc b/python/src/pyarrow/status.cc
new file mode 100644
index 0000000000000..1cd54f6a78560
--- /dev/null
+++ b/python/src/pyarrow/status.cc
@@ -0,0 +1,92 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// A Status encapsulates the result of an operation.  It may indicate success,
+// or it may indicate an error with an associated error message.
+//
+// Multiple threads can invoke const methods on a Status without
+// external synchronization, but if any of the threads may call a
+// non-const method, all threads accessing the same Status must use
+// external synchronization.
+
+#include "pyarrow/status.h"
+
+#include <assert.h>
+#include <cstdint>
+#include <cstring>
+
+namespace pyarrow {
+
+Status::Status(StatusCode code, const std::string& msg, int16_t posix_code) {
+  assert(code != StatusCode::OK);
+  const uint32_t size = msg.size();
+  char* result = new char[size + 7];
+  memcpy(result, &size, sizeof(size));
+  result[4] = static_cast<char>(code);
+  memcpy(result + 5, &posix_code, sizeof(posix_code));
+  memcpy(result + 7, msg.c_str(), msg.size());
+  state_ = result;
+}
+
+const char* Status::CopyState(const char* state) {
+  uint32_t size;
+  memcpy(&size, state, sizeof(size));
+  char* result = new char[size + 7];
+  memcpy(result, state, size + 7);
+  return result;
+}
+
+std::string Status::CodeAsString() const {
+  if (state_ == NULL) {
+    return "OK";
+  }
+
+  const char* type;
+  switch (code()) {
+    case StatusCode::OK:
+      type = "OK";
+      break;
+    case StatusCode::OutOfMemory:
+      type = "Out of memory";
+      break;
+    case StatusCode::KeyError:
+      type = "Key error";
+      break;
+    case StatusCode::TypeError:
+      type = "Value error";
+      break;
+    case StatusCode::ValueError:
+      type = "Value error";
+      break;
+    case StatusCode::IOError:
+      type = "IO error";
+      break;
+    case StatusCode::NotImplemented:
+      type = "Not implemented";
+      break;
+    case StatusCode::ArrowError:
+      type = "Arrow C++ error";
+      break;
+    case StatusCode::UnknownError:
+      type = "Unknown error";
+      break;
+  }
+  return std::string(type);
+}
+
+std::string Status::ToString() const {
+  std::string result(CodeAsString());
+  if (state_ == NULL) {
+    return result;
+  }
+
+  result.append(": ");
+
+  uint32_t length;
+  memcpy(&length, state_, sizeof(length));
+  result.append(reinterpret_cast<const char*>(state_ + 7), length);
+  return result;
+}
+
+} // namespace pyarrow
diff --git a/python/src/pyarrow/status.h b/python/src/pyarrow/status.h
new file mode 100644
index 0000000000000..cb8c8add210e4
--- /dev/null
+++ b/python/src/pyarrow/status.h
@@ -0,0 +1,144 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// A Status encapsulates the result of an operation.  It may indicate success,
+// or it may indicate an error with an associated error message.
+//
+// Multiple threads can invoke const methods on a Status without
+// external synchronization, but if any of the threads may call a
+// non-const method, all threads accessing the same Status must use
+// external synchronization.
+
+#ifndef PYARROW_STATUS_H_
+#define PYARROW_STATUS_H_
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+
+namespace pyarrow {
+
+#define PY_RETURN_NOT_OK(s) do {                \
+    Status _s = (s);                            \
+    if (!_s.ok()) return _s;                    \
+  } while (0);
+
+enum class StatusCode: char {
+  OK = 0,
+  OutOfMemory = 1,
+  KeyError = 2,
+  TypeError = 3,
+  ValueError = 4,
+  IOError = 5,
+  NotImplemented = 6,
+
+  ArrowError = 7,
+
+  UnknownError = 10
+};
+
+class Status {
+ public:
+  // Create a success status.
+  Status() : state_(NULL) { }
+  ~Status() { delete[] state_; }
+
+  // Copy the specified status.
+  Status(const Status& s);
+  void operator=(const Status& s);
+
+  // Return a success status.
+  static Status OK() { return Status(); }
+
+  // Return error status of an appropriate type.
+  static Status OutOfMemory(const std::string& msg, int16_t posix_code = -1) {
+    return Status(StatusCode::OutOfMemory, msg, posix_code);
+  }
+
+  static Status KeyError(const std::string& msg) {
+    return Status(StatusCode::KeyError, msg, -1);
+  }
+
+  static Status TypeError(const std::string& msg) {
+    return Status(StatusCode::TypeError, msg, -1);
+  }
+
+  static Status IOError(const std::string& msg) {
+    return Status(StatusCode::IOError, msg, -1);
+  }
+
+  static Status ValueError(const std::string& msg) {
+    return Status(StatusCode::ValueError, msg, -1);
+  }
+
+  static Status NotImplemented(const std::string& msg) {
+    return Status(StatusCode::NotImplemented, msg, -1);
+  }
+
+  static Status UnknownError(const std::string& msg) {
+    return Status(StatusCode::UnknownError, msg, -1);
+  }
+
+  static Status ArrowError(const std::string& msg) {
+    return Status(StatusCode::ArrowError, msg, -1);
+  }
+
+  // Returns true iff the status indicates success.
+  bool ok() const { return (state_ == NULL); }
+
+  bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; }
+  bool IsKeyError() const { return code() == StatusCode::KeyError; }
+  bool IsIOError() const { return code() == StatusCode::IOError; }
+  bool IsTypeError() const { return code() == StatusCode::TypeError; }
+  bool IsValueError() const { return code() == StatusCode::ValueError; }
+
+  bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
+
+  bool IsArrowError() const { return code() == StatusCode::ArrowError; }
+
+  // Return a string representation of this status suitable for printing.
+  // Returns the string "OK" for success.
+  std::string ToString() const;
+
+  // Return a string representation of the status code, without the message
+  // text or posix code information.
+  std::string CodeAsString() const;
+
+  // Get the POSIX code associated with this Status, or -1 if there is none.
+  int16_t posix_code() const;
+
+ private:
+  // OK status has a NULL state_.  Otherwise, state_ is a new[] array
+  // of the following form:
+  //    state_[0..3] == length of message
+  //    state_[4]    == code
+  //    state_[5..6] == posix_code
+  //    state_[7..]  == message
+  const char* state_;
+
+  StatusCode code() const {
+    return ((state_ == NULL) ?
+        StatusCode::OK : static_cast<StatusCode>(state_[4]));
+  }
+
+  Status(StatusCode code, const std::string& msg, int16_t posix_code);
+  static const char* CopyState(const char* s);
+};
+
+inline Status::Status(const Status& s) {
+  state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
+}
+
+inline void Status::operator=(const Status& s) {
+  // The following condition catches both aliasing (when this == &s),
+  // and the common case where both s and *this are ok.
+  if (state_ != s.state_) {
+    delete[] state_;
+    state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
+  }
+}
+
+}  // namespace pyarrow
+
+#endif // PYARROW_STATUS_H_

From ae95dbd189477442d39e55fb0a1aede206906cd9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Mon, 7 Mar 2016 22:39:07 -0800
Subject: [PATCH 0024/1644] ARROW-44: Python: prototype object model for array
 slot values ("scalars")

Non-exhaustive, but this will facilitate inspecting Arrow data while the library is in development.

```python
In [2]: arr = arrow.from_pylist([['foo', None], None, [], ['qux']])

In [3]: arr
Out[3]: <arrow.array.ListArray at 0x7f1970030f98>

In [4]: arr[0]
Out[4]: ['foo', None]

In [5]: type(arr[0])
Out[5]: arrow.scalar.ListValue

In [6]: arr[0][0]
Out[6]: 'foo'

In [7]: arr[0][1]
Out[7]: NA

In [8]: arr[1]
Out[8]: NA

In [9]: arr[2]
Out[9]: []

In [10]: len(arr[2])
Out[10]: 0

In [11]: arr.type
Out[11]: DataType(list<string>)
```

Author: Wes McKinney <wesm@apache.org>

Closes #20 from wesm/ARROW-44 and squashes the following commits:

df06ba1 [Wes McKinney] Add tests for scalars proxying implemented Python list type conversions, fix associated bugs
20fbdc1 [Wes McKinney] Draft scalar box types, no tests yet
---
 cpp/src/arrow/types/list.h             |   6 +-
 python/arrow/__init__.py               |   6 +-
 python/arrow/array.pxd                 |   1 -
 python/arrow/array.pyx                 |  17 ++-
 python/arrow/compat.py                 |   6 +
 python/arrow/includes/arrow.pxd        |  36 +++++-
 python/arrow/scalar.pxd                |  25 +++-
 python/arrow/scalar.pyx                | 165 +++++++++++++++++++++++++
 python/arrow/schema.pxd                |   2 +
 python/arrow/schema.pyx                |  14 +++
 python/arrow/tests/test_scalars.py     |  82 ++++++++++++
 python/src/pyarrow/adapters/builtin.cc |   2 +-
 12 files changed, 342 insertions(+), 20 deletions(-)
 create mode 100644 python/arrow/tests/test_scalars.py

diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index f40a8245362b1..210c76a046c21 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -63,7 +63,11 @@ class ListArray : public Array {
 
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
-  const ArrayPtr& values() const {return values_;}
+  const std::shared_ptr<Array>& values() const {return values_;}
+
+  const std::shared_ptr<DataType>& value_type() const {
+    return values_->type();
+  }
 
   const int32_t* offsets() const { return offsets_;}
 
diff --git a/python/arrow/__init__.py b/python/arrow/__init__.py
index 3c049b85e8c93..3507ea0235afe 100644
--- a/python/arrow/__init__.py
+++ b/python/arrow/__init__.py
@@ -24,7 +24,11 @@
 
 from arrow.error import ArrowException
 
-from arrow.scalar import ArrayValue, NA, Scalar
+from arrow.scalar import (ArrayValue, Scalar, NA, NAType,
+                          BooleanValue,
+                          Int8Value, Int16Value, Int32Value, Int64Value,
+                          UInt8Value, UInt16Value, UInt32Value, UInt64Value,
+                          FloatValue, DoubleValue, ListValue, StringValue)
 
 from arrow.schema import (null, bool_,
                           int8, int16, int32, int64,
diff --git a/python/arrow/array.pxd b/python/arrow/array.pxd
index e32d27769b5e1..04dd8d182bcf6 100644
--- a/python/arrow/array.pxd
+++ b/python/arrow/array.pxd
@@ -34,7 +34,6 @@ cdef class Array:
         DataType type
 
     cdef init(self, const shared_ptr[CArray]& sp_array)
-    cdef _getitem(self, int i)
 
 
 cdef class BooleanArray(Array):
diff --git a/python/arrow/array.pyx b/python/arrow/array.pyx
index 3a3210d6cc100..8ebd01d1dbe73 100644
--- a/python/arrow/array.pyx
+++ b/python/arrow/array.pyx
@@ -25,6 +25,7 @@ cimport arrow.includes.pyarrow as pyarrow
 from arrow.compat import frombytes, tobytes
 from arrow.error cimport check_status
 
+cimport arrow.scalar as scalar
 from arrow.scalar import NA
 
 def total_allocated_bytes():
@@ -73,13 +74,7 @@ cdef class Array:
         while key < 0:
             key += len(self)
 
-        if self.ap.IsNull(key):
-            return NA
-        else:
-            return self._getitem(key)
-
-    cdef _getitem(self, int i):
-        raise NotImplementedError
+        return scalar.box_arrow_scalar(self.type, self.sp_array, key)
 
     def slice(self, start, end):
         pass
@@ -168,12 +163,16 @@ cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
     return arr
 
 
-def from_pylist(object list_obj, type=None):
+def from_pylist(object list_obj, DataType type=None):
     """
     Convert Python list to Arrow array
     """
     cdef:
         shared_ptr[CArray] sp_array
 
-    check_status(pyarrow.ConvertPySequence(list_obj, &sp_array))
+    if type is None:
+        check_status(pyarrow.ConvertPySequence(list_obj, &sp_array))
+    else:
+        raise NotImplementedError
+
     return box_arrow_array(sp_array)
diff --git a/python/arrow/compat.py b/python/arrow/compat.py
index 2ac41ac8abf89..08f0f23796797 100644
--- a/python/arrow/compat.py
+++ b/python/arrow/compat.py
@@ -54,6 +54,9 @@ def dict_values(x):
     range = xrange
     long = long
 
+    def u(s):
+        return unicode(s, "unicode_escape")
+
     def tobytes(o):
         if isinstance(o, unicode):
             return o.encode('utf8')
@@ -73,6 +76,9 @@ def dict_values(x):
     from decimal import Decimal
     range = range
 
+    def u(s):
+        return s
+
     def tobytes(o):
         if isinstance(o, str):
             return o.encode('utf8')
diff --git a/python/arrow/includes/arrow.pxd b/python/arrow/includes/arrow.pxd
index fde5de910915a..0cc44c06cb607 100644
--- a/python/arrow/includes/arrow.pxd
+++ b/python/arrow/includes/arrow.pxd
@@ -84,13 +84,41 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsNull(int i)
 
     cdef cppclass CUInt8Array" arrow::UInt8Array"(CArray):
-        pass
+        uint8_t Value(int i)
 
     cdef cppclass CInt8Array" arrow::Int8Array"(CArray):
-        pass
+        int8_t Value(int i)
+
+    cdef cppclass CUInt16Array" arrow::UInt16Array"(CArray):
+        uint16_t Value(int i)
+
+    cdef cppclass CInt16Array" arrow::Int16Array"(CArray):
+        int16_t Value(int i)
+
+    cdef cppclass CUInt32Array" arrow::UInt32Array"(CArray):
+        uint32_t Value(int i)
+
+    cdef cppclass CInt32Array" arrow::Int32Array"(CArray):
+        int32_t Value(int i)
+
+    cdef cppclass CUInt64Array" arrow::UInt64Array"(CArray):
+        uint64_t Value(int i)
+
+    cdef cppclass CInt64Array" arrow::Int64Array"(CArray):
+        int64_t Value(int i)
+
+    cdef cppclass CFloatArray" arrow::FloatArray"(CArray):
+        float Value(int i)
+
+    cdef cppclass CDoubleArray" arrow::DoubleArray"(CArray):
+        double Value(int i)
 
     cdef cppclass CListArray" arrow::ListArray"(CArray):
-        pass
+        const int32_t* offsets()
+        int32_t offset(int i)
+        int32_t value_length(int i)
+        const shared_ptr[CArray]& values()
+        const shared_ptr[CDataType]& value_type()
 
     cdef cppclass CStringArray" arrow::StringArray"(CListArray):
-        pass
+        c_string GetString(int i)
diff --git a/python/arrow/scalar.pxd b/python/arrow/scalar.pxd
index e193c09cd69a2..15cdc956a2593 100644
--- a/python/arrow/scalar.pxd
+++ b/python/arrow/scalar.pxd
@@ -16,7 +16,7 @@
 # under the License.
 
 from arrow.includes.common cimport *
-from arrow.includes.arrow cimport CArray, CListArray
+from arrow.includes.arrow cimport *
 
 from arrow.schema cimport DataType
 
@@ -31,17 +31,36 @@ cdef class NAType(Scalar):
 
 cdef class ArrayValue(Scalar):
     cdef:
-        shared_ptr[CArray] array
+        shared_ptr[CArray] sp_array
         int index
 
+    cdef void init(self, DataType type,
+                   const shared_ptr[CArray]& sp_array, int index)
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array)
+
 
 cdef class Int8Value(ArrayValue):
     pass
 
 
-cdef class ListValue(ArrayValue):
+cdef class Int64Value(ArrayValue):
     pass
 
 
+cdef class ListValue(ArrayValue):
+    cdef readonly:
+        DataType value_type
+
+    cdef:
+        CListArray* ap
+
+    cdef _getitem(self, int i)
+
+
 cdef class StringValue(ArrayValue):
     pass
+
+cdef object box_arrow_scalar(DataType type,
+                             const shared_ptr[CArray]& sp_array,
+                             int index)
diff --git a/python/arrow/scalar.pyx b/python/arrow/scalar.pyx
index 78dadecf9b422..951ede2877690 100644
--- a/python/arrow/scalar.pyx
+++ b/python/arrow/scalar.pyx
@@ -15,14 +15,179 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from arrow.schema cimport DataType, box_data_type
+
+from arrow.compat import frombytes
 import arrow.schema as schema
 
+NA = None
+
 cdef class NAType(Scalar):
 
     def __cinit__(self):
+        global NA
+        if NA is not None:
+            raise Exception('Cannot create multiple NAType instances')
+
         self.type = schema.null()
 
     def __repr__(self):
         return 'NA'
 
+    def as_py(self):
+        return None
+
 NA = NAType()
+
+cdef class ArrayValue(Scalar):
+
+    cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
+                   int index):
+        self.type = type
+        self.index = index
+        self._set_array(sp_array)
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+
+    def __repr__(self):
+        if hasattr(self, 'as_py'):
+            return repr(self.as_py())
+        else:
+            return Scalar.__repr__(self)
+
+
+cdef class BooleanValue(ArrayValue):
+    pass
+
+
+cdef class Int8Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt8Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int16Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt16Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int32Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt32Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class FloatValue(ArrayValue):
+
+    def as_py(self):
+        cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class DoubleValue(ArrayValue):
+
+    def as_py(self):
+        cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class StringValue(ArrayValue):
+
+    def as_py(self):
+        cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
+        return frombytes(ap.GetString(self.index))
+
+
+cdef class ListValue(ArrayValue):
+
+    def __len__(self):
+        return self.ap.value_length(self.index)
+
+    def __getitem__(self, i):
+        return self._getitem(i)
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+        self.ap = <CListArray*> sp_array.get()
+        self.value_type = box_data_type(self.ap.value_type())
+
+    cdef _getitem(self, int i):
+        cdef int j = self.ap.offset(self.index) + i
+        return box_arrow_scalar(self.value_type, self.ap.values(), j)
+
+    def as_py(self):
+        cdef:
+            int j
+            list result = []
+
+        for j in range(len(self)):
+            result.append(self._getitem(j).as_py())
+
+        return result
+
+
+cdef dict _scalar_classes = {
+    LogicalType_UINT8: Int8Value,
+    LogicalType_UINT16: Int16Value,
+    LogicalType_UINT32: Int32Value,
+    LogicalType_UINT64: Int64Value,
+    LogicalType_INT8: Int8Value,
+    LogicalType_INT16: Int16Value,
+    LogicalType_INT32: Int32Value,
+    LogicalType_INT64: Int64Value,
+    LogicalType_FLOAT: FloatValue,
+    LogicalType_DOUBLE: DoubleValue,
+    LogicalType_LIST: ListValue,
+    LogicalType_STRING: StringValue
+}
+
+cdef object box_arrow_scalar(DataType type,
+                             const shared_ptr[CArray]& sp_array,
+                             int index):
+    cdef ArrayValue val
+    if sp_array.get().IsNull(index):
+        return NA
+    else:
+        val = _scalar_classes[type.type.type]()
+        val.init(type, sp_array, index)
+        return val
diff --git a/python/arrow/schema.pxd b/python/arrow/schema.pxd
index 487c246f44abf..8cc244aaba341 100644
--- a/python/arrow/schema.pxd
+++ b/python/arrow/schema.pxd
@@ -37,3 +37,5 @@ cdef class Schema:
     cdef:
         shared_ptr[CSchema] sp_schema
         CSchema* schema
+
+cdef DataType box_data_type(const shared_ptr[CDataType]& type)
diff --git a/python/arrow/schema.pyx b/python/arrow/schema.pyx
index 63cd6e888abd0..3001531eb607d 100644
--- a/python/arrow/schema.pyx
+++ b/python/arrow/schema.pyx
@@ -85,6 +85,14 @@ cdef DataType primitive_type(LogicalType type, bint nullable=True):
 def field(name, type):
     return Field(name, type)
 
+cdef set PRIMITIVE_TYPES = set([
+    LogicalType_NA, LogicalType_BOOL,
+    LogicalType_UINT8, LogicalType_INT8,
+    LogicalType_UINT16, LogicalType_INT16,
+    LogicalType_UINT32, LogicalType_INT32,
+    LogicalType_UINT64, LogicalType_INT64,
+    LogicalType_FLOAT, LogicalType_DOUBLE])
+
 def null():
     return primitive_type(LogicalType_NA)
 
@@ -148,3 +156,9 @@ def struct(fields, c_bool nullable=True):
     out.init(shared_ptr[CDataType](
         new CStructType(c_fields, nullable)))
     return out
+
+
+cdef DataType box_data_type(const shared_ptr[CDataType]& type):
+    cdef DataType out = DataType()
+    out.init(type)
+    return out
diff --git a/python/arrow/tests/test_scalars.py b/python/arrow/tests/test_scalars.py
new file mode 100644
index 0000000000000..951380bd981e4
--- /dev/null
+++ b/python/arrow/tests/test_scalars.py
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from arrow.compat import unittest, u
+import arrow
+
+
+class TestScalars(unittest.TestCase):
+
+    def test_null_singleton(self):
+        with self.assertRaises(Exception):
+            arrow.NAType()
+
+    def test_bool(self):
+        pass
+
+    def test_int64(self):
+        arr = arrow.from_pylist([1, 2, None])
+
+        v = arr[0]
+        assert isinstance(v, arrow.Int64Value)
+        assert repr(v) == "1"
+        assert v.as_py() == 1
+
+        assert arr[2] is arrow.NA
+
+    def test_double(self):
+        arr = arrow.from_pylist([1.5, None, 3])
+
+        v = arr[0]
+        assert isinstance(v, arrow.DoubleValue)
+        assert repr(v) == "1.5"
+        assert v.as_py() == 1.5
+
+        assert arr[1] is arrow.NA
+
+        v = arr[2]
+        assert v.as_py() == 3.0
+
+    def test_string(self):
+        arr = arrow.from_pylist(['foo', None, u('bar')])
+
+        v = arr[0]
+        assert isinstance(v, arrow.StringValue)
+        assert repr(v) == "'foo'"
+        assert v.as_py() == 'foo'
+
+        assert arr[1] is arrow.NA
+
+        v = arr[2].as_py()
+        assert v == 'bar'
+        assert isinstance(v, str)
+
+    def test_list(self):
+        arr = arrow.from_pylist([['foo', None], None, ['bar'], []])
+
+        v = arr[0]
+        assert len(v) == 2
+        assert isinstance(v, arrow.ListValue)
+        assert repr(v) == "['foo', None]"
+        assert v.as_py() == ['foo', None]
+        assert v[0].as_py() == 'foo'
+        assert v[1] is arrow.NA
+
+        assert arr[1] is arrow.NA
+
+        v = arr[3]
+        assert len(v) == 0
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index ae84fa12b0de6..60d6248842ec9 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -276,7 +276,7 @@ class Int64Converter : public TypedConverter<arrow::Int64Builder> {
 class DoubleConverter : public TypedConverter<arrow::DoubleBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
-    int64_t val;
+    double val;
     Py_ssize_t size = PySequence_Size(seq);
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));

From 45cd9fd8ddc75f5c8a558024c705ab8d37bbc5b5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Tue, 8 Mar 2016 12:48:42 -0800
Subject: [PATCH 0025/1644] ARROW-43: Python: format array values to in
 __repr__ for interactive computing

Author: Wes McKinney <wesm@apache.org>

Closes #21 from wesm/ARROW-43 and squashes the following commits:

dee6ba2 [Wes McKinney] Basic array formatter, not tweaking too much for now
---
 python/arrow/array.pxd           |  1 +
 python/arrow/array.pyx           | 16 +++++++++++++-
 python/arrow/scalar.pxd          |  2 +-
 python/arrow/scalar.pyx          | 11 +++++++---
 python/arrow/tests/test_array.py | 37 ++++++++++++++++++++++++++++++++
 5 files changed, 62 insertions(+), 5 deletions(-)

diff --git a/python/arrow/array.pxd b/python/arrow/array.pxd
index 04dd8d182bcf6..482f8f796dd26 100644
--- a/python/arrow/array.pxd
+++ b/python/arrow/array.pxd
@@ -34,6 +34,7 @@ cdef class Array:
         DataType type
 
     cdef init(self, const shared_ptr[CArray]& sp_array)
+    cdef getitem(self, int i)
 
 
 cdef class BooleanArray(Array):
diff --git a/python/arrow/array.pyx b/python/arrow/array.pyx
index 8ebd01d1dbe73..b367e3b84a8b3 100644
--- a/python/arrow/array.pyx
+++ b/python/arrow/array.pyx
@@ -46,6 +46,17 @@ cdef class Array:
         def __get__(self):
             return self.sp_array.get().null_count()
 
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self.getitem(i)
+        raise StopIteration
+
+    def __repr__(self):
+        from arrow.formatting import array_format
+        type_format = object.__repr__(self)
+        values = array_format(self, window=10)
+        return '{0}\n{1}'.format(type_format, values)
+
     def __len__(self):
         return self.sp_array.get().length()
 
@@ -74,7 +85,10 @@ cdef class Array:
         while key < 0:
             key += len(self)
 
-        return scalar.box_arrow_scalar(self.type, self.sp_array, key)
+        return self.getitem(key)
+
+    cdef getitem(self, int i):
+        return scalar.box_arrow_scalar(self.type, self.sp_array, i)
 
     def slice(self, start, end):
         pass
diff --git a/python/arrow/scalar.pxd b/python/arrow/scalar.pxd
index 15cdc956a2593..4e0a3647155a6 100644
--- a/python/arrow/scalar.pxd
+++ b/python/arrow/scalar.pxd
@@ -55,7 +55,7 @@ cdef class ListValue(ArrayValue):
     cdef:
         CListArray* ap
 
-    cdef _getitem(self, int i)
+    cdef getitem(self, int i)
 
 
 cdef class StringValue(ArrayValue):
diff --git a/python/arrow/scalar.pyx b/python/arrow/scalar.pyx
index 951ede2877690..72a280e334f4e 100644
--- a/python/arrow/scalar.pyx
+++ b/python/arrow/scalar.pyx
@@ -144,14 +144,19 @@ cdef class ListValue(ArrayValue):
         return self.ap.value_length(self.index)
 
     def __getitem__(self, i):
-        return self._getitem(i)
+        return self.getitem(i)
+
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self.getitem(i)
+        raise StopIteration
 
     cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
         self.sp_array = sp_array
         self.ap = <CListArray*> sp_array.get()
         self.value_type = box_data_type(self.ap.value_type())
 
-    cdef _getitem(self, int i):
+    cdef getitem(self, int i):
         cdef int j = self.ap.offset(self.index) + i
         return box_arrow_scalar(self.value_type, self.ap.values(), j)
 
@@ -161,7 +166,7 @@ cdef class ListValue(ArrayValue):
             list result = []
 
         for j in range(len(self)):
-            result.append(self._getitem(j).as_py())
+            result.append(self.getitem(j).as_py())
 
         return result
 
diff --git a/python/arrow/tests/test_array.py b/python/arrow/tests/test_array.py
index 8eaa53352061b..ebd872c744e44 100644
--- a/python/arrow/tests/test_array.py
+++ b/python/arrow/tests/test_array.py
@@ -17,6 +17,7 @@
 
 from arrow.compat import unittest
 import arrow
+import arrow.formatting as fmt
 
 
 class TestArrayAPI(unittest.TestCase):
@@ -24,3 +25,39 @@ class TestArrayAPI(unittest.TestCase):
     def test_getitem_NA(self):
         arr = arrow.from_pylist([1, None, 2])
         assert arr[1] is arrow.NA
+
+    def test_list_format(self):
+        arr = arrow.from_pylist([[1], None, [2, 3]])
+        result = fmt.array_format(arr)
+        expected = """\
+[
+  [1],
+  NA,
+  [2,
+   3]
+]"""
+        assert result == expected
+
+    def test_string_format(self):
+        arr = arrow.from_pylist(['foo', None, 'bar'])
+        result = fmt.array_format(arr)
+        expected = """\
+[
+  'foo',
+  NA,
+  'bar'
+]"""
+        assert result == expected
+
+    def test_long_array_format(self):
+        arr = arrow.from_pylist(range(100))
+        result = fmt.array_format(arr, window=2)
+        expected = """\
+[
+  0,
+  1,
+  ...
+  98,
+  99
+]"""
+        assert result == expected

From 1650026285bea52288c7f24720c3caf7cd3ce2a8 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Mon, 29 Feb 2016 19:32:12 -0800
Subject: [PATCH 0026/1644] ARROW-17: set some vector fields to package level
 access for Drill compatibility

---
 .../codegen/templates/BasicTypeHelper.java    |  1 +
 .../templates/NullableValueVectors.java       |  6 ++-
 .../templates/RepeatedValueVectors.java       |  2 +-
 .../main/codegen/templates/UnionVector.java   |  4 +-
 .../templates/VariableLengthVectors.java      |  2 +-
 .../org/apache/arrow/vector/BitVector.java    |  4 +-
 .../arrow/vector/complex/ListVector.java      |  4 +-
 .../arrow/vector/complex/MapVector.java       |  2 +-
 .../vector/complex/RepeatedListVector.java    |  3 +-
 .../vector/complex/RepeatedMapVector.java     |  2 +-
 .../org/apache/arrow/vector/types/Types.java  | 54 +++++++++++++++----
 11 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/BasicTypeHelper.java b/java/vector/src/main/codegen/templates/BasicTypeHelper.java
index bb6446e8d6b19..0bae715e35283 100644
--- a/java/vector/src/main/codegen/templates/BasicTypeHelper.java
+++ b/java/vector/src/main/codegen/templates/BasicTypeHelper.java
@@ -231,6 +231,7 @@ public static ValueVector getNewVector(MaterializedField field, BufferAllocator
     return getNewVector(field, allocator, null);
   }
   public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+    field = field.clone();
     MajorType type = field.getType();
 
     switch (type.getMinorType()) {
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 6893a25efbe18..b0029f7ad4c37 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -45,8 +45,10 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
 
   private final MaterializedField bitsField = MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED));
-  private final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
-  private final ${valuesName} values = new ${minor.class}Vector(field, allocator);
+  private final MaterializedField valuesField = MaterializedField.create("$values$", new MajorType(field.getType().getMinorType(), DataMode.REQUIRED, field.getPrecision(), field.getScale()));
+
+  final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
+  final ${valuesName} values = new ${minor.class}Vector(valuesField, allocator);
 
   private final Mutator mutator = new Mutator();
   private final Accessor accessor = new Accessor();
diff --git a/java/vector/src/main/codegen/templates/RepeatedValueVectors.java b/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
index 5ac80f57737ff..ceae53bbf58cf 100644
--- a/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
@@ -42,7 +42,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Repeated${minor.class}Vector.class);
 
   // we maintain local reference to concrete vector type for performance reasons.
-  private ${minor.class}Vector values;
+  ${minor.class}Vector values;
   private final FieldReader reader = new Repeated${minor.class}ReaderImpl(Repeated${minor.class}Vector.this);
   private final Mutator mutator = new Mutator();
   private final Accessor accessor = new Accessor();
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index ba94ac22a05f6..6042a5bf68352 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -51,9 +51,9 @@ public class UnionVector implements ValueVector {
   private BufferAllocator allocator;
   private Accessor accessor = new Accessor();
   private Mutator mutator = new Mutator();
-  private int valueCount;
+  int valueCount;
 
-  private MapVector internalMap;
+  MapVector internalMap;
   private UInt1Vector typeVector;
 
   private MapVector mapVector;
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index 13d53b8e846ab..84fb3eb55674f 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -57,7 +57,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
 
   public final static String OFFSETS_VECTOR_NAME = "$offsets$";
   private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
-  private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
+  final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
   private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
 
   private final Accessor accessor;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 952e9028e0668..c5bcb2decc43b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -41,7 +41,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
 
-  private int valueCount;
+  int valueCount;
   private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
   private int allocationMonitor = 0;
 
@@ -64,7 +64,7 @@ public int getBufferSizeFor(final int valueCount) {
     return getSizeFromCount(valueCount);
   }
 
-  private int getSizeFromCount(int valueCount) {
+  int getSizeFromCount(int valueCount) {
     return (int) Math.ceil(valueCount / 8.0);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 8387c9e5ba667..13610c4f03f61 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -47,8 +47,8 @@
 
 public class ListVector extends BaseRepeatedValueVector {
 
-  private UInt4Vector offsets;
-  private final UInt1Vector bits;
+  UInt4Vector offsets;
+  final UInt1Vector bits;
   private Mutator mutator = new Mutator();
   private Accessor accessor = new Accessor();
   private UnionListWriter writer;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 1bbce73d6ff82..cc0953a1af8ba 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -52,7 +52,7 @@ public class MapVector extends AbstractMapVector {
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
-  private int valueCount;
+  int valueCount;
 
   public MapVector(String path, BufferAllocator allocator, CallBack callBack){
     this(MaterializedField.create(path, TYPE), allocator, callBack);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
index 778fe81b5da6a..f337f9c4a60e0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
@@ -49,7 +49,7 @@ public class RepeatedListVector extends AbstractContainerVector
 
   public final static MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
   private final RepeatedListReaderImpl reader = new RepeatedListReaderImpl(null, this);
-  private final DelegateRepeatedVector delegate;
+  final DelegateRepeatedVector delegate;
 
   protected static class DelegateRepeatedVector extends BaseRepeatedValueVector {
 
@@ -313,7 +313,6 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor
     if (result.isCreated() && callBack != null) {
       callBack.doWork();
     }
-    this.field = delegate.getField();
     return result;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
index e7eacd3c67c40..686414e71cadf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
@@ -53,7 +53,7 @@ public class RepeatedMapVector extends AbstractMapVector
 
   public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.REPEATED);
 
-  private final UInt4Vector offsets;   // offsets to start of each record (considering record indices are 0-indexed)
+  final UInt4Vector offsets;   // offsets to start of each record (considering record indices are 0-indexed)
   private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
   private final RepeatedMapAccessor accessor = new RepeatedMapAccessor();
   private final Mutator mutator = new Mutator();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index cef892ce88030..88999cb8f5ab8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -20,6 +20,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 public class Types {
   public enum MinorType {
@@ -73,26 +74,35 @@ public enum DataMode {
   public static class MajorType {
     private MinorType minorType;
     private DataMode mode;
-    private Integer precision;
-    private Integer scale;
-    private Integer timezone;
+    private int precision;
+    private int scale;
+    private int timezone;
+    private int width;
     private List<MinorType> subTypes;
 
     public MajorType(MinorType minorType, DataMode mode) {
-      this(minorType, mode, null, null, null, null);
+      this(minorType, mode, 0, 0, 0, 0, null);
     }
 
-    public MajorType(MinorType minorType, DataMode mode, Integer precision, Integer scale) {
-      this(minorType, mode, precision, scale, null, null);
+    public MajorType(MinorType minorType, DataMode mode, int precision, int scale) {
+      this(minorType, mode, precision, scale, 0, 0, null);
     }
 
-    public MajorType(MinorType minorType, DataMode mode, Integer precision, Integer scale, Integer timezone, List<MinorType> subTypes) {
+    public MajorType(MinorType minorType, DataMode mode, int precision, int scale, int timezone, List<MinorType> subTypes) {
+      this(minorType, mode, precision, scale, timezone, 0, subTypes);
+    }
+
+    public MajorType(MinorType minorType, DataMode mode, int precision, int scale, int timezone, int width, List<MinorType> subTypes) {
       this.minorType = minorType;
       this.mode = mode;
       this.precision = precision;
       this.scale = scale;
       this.timezone = timezone;
+      this.width = width;
       this.subTypes = subTypes;
+      if (subTypes == null) {
+        this.subTypes = new ArrayList<>();
+      }
     }
 
     public MinorType getMinorType() {
@@ -103,21 +113,45 @@ public DataMode getMode() {
       return mode;
     }
 
-    public Integer getPrecision() {
+    public int getPrecision() {
       return precision;
     }
 
-    public Integer getScale() {
+    public int getScale() {
       return scale;
     }
 
-    public Integer getTimezone() {
+    public int getTimezone() {
       return timezone;
     }
 
     public List<MinorType> getSubTypes() {
       return subTypes;
     }
+
+    public int getWidth() {
+      return width;
+    }
+
+
+    @Override
+    public boolean equals(Object other) {
+      if (other == null) {
+        return false;
+      }
+      if (!(other instanceof MajorType)) {
+        return false;
+      }
+      MajorType that = (MajorType) other;
+      return this.minorType == that.minorType &&
+              this.mode == that.mode &&
+              this.precision == that.precision &&
+              this.scale == that.scale &&
+              this.timezone == that.timezone &&
+              this.width == that.width &&
+              Objects.equals(this.subTypes, that.subTypes);
+    }
+
   }
 
   public static MajorType required(MinorType minorType) {

From 243ed4e91d5ed922b205f7ac5fa8f9f821a07fbb Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Mon, 29 Feb 2016 19:33:44 -0800
Subject: [PATCH 0027/1644] ARROW-18: Fix decimal precision and scale in
 MapWriters

---
 java/vector/src/main/codegen/templates/MapWriters.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 7001367bb3774..42f39820393e5 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -206,7 +206,7 @@ public void end() {
   }
 
   public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) {
-    final MajorType ${upperName}_TYPE = new MajorType(MinorType.${upperName}, DataMode.OPTIONAL, scale, precision, null, null);
+    final MajorType ${upperName}_TYPE = new MajorType(MinorType.${upperName}, DataMode.OPTIONAL, precision, scale, 0, null);
   <#else>
   private static final MajorType ${upperName}_TYPE = Types.optional(MinorType.${upperName});
   @Override

From 31def7d81a094dd051d2f4bbead78edaae25755a Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Tue, 8 Mar 2016 14:11:29 -0800
Subject: [PATCH 0028/1644] ARROW-51: Add simple ValueVector tests

---
 .../apache/arrow/vector/TestValueVector.java  | 521 ++++++++++++++++++
 1 file changed, 521 insertions(+)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java

diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
new file mode 100644
index 0000000000000..4488d750284c7
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -0,0 +1,521 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.charset.Charset;
+
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.RepeatedListVector;
+import org.apache.arrow.vector.complex.RepeatedMapVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.holders.BitHolder;
+import org.apache.arrow.vector.holders.IntHolder;
+import org.apache.arrow.vector.holders.NullableFloat4Holder;
+import org.apache.arrow.vector.holders.NullableUInt4Holder;
+import org.apache.arrow.vector.holders.NullableVar16CharHolder;
+import org.apache.arrow.vector.holders.NullableVarCharHolder;
+import org.apache.arrow.vector.holders.RepeatedFloat4Holder;
+import org.apache.arrow.vector.holders.RepeatedIntHolder;
+import org.apache.arrow.vector.holders.RepeatedVarBinaryHolder;
+import org.apache.arrow.vector.holders.UInt4Holder;
+import org.apache.arrow.vector.holders.VarCharHolder;
+import org.apache.arrow.memory.BufferAllocator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestValueVector {
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestValueVector.class);
+
+  private final static String EMPTY_SCHEMA_PATH = "";
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  private final static Charset utf8Charset = Charset.forName("UTF-8");
+  private final static byte[] STR1 = new String("AAAAA1").getBytes(utf8Charset);
+  private final static byte[] STR2 = new String("BBBBBBBBB2").getBytes(utf8Charset);
+  private final static byte[] STR3 = new String("CCCC3").getBytes(utf8Charset);
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test(expected = OversizedAllocationException.class)
+  public void testFixedVectorReallocation() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    final UInt4Vector vector = new UInt4Vector(field, allocator);
+    // edge case 1: buffer size = max value capacity
+    final int expectedValueCapacity = BaseValueVector.MAX_ALLOCATION_SIZE / 4;
+    try {
+      vector.allocateNew(expectedValueCapacity);
+      assertEquals(expectedValueCapacity, vector.getValueCapacity());
+      vector.reAlloc();
+      assertEquals(expectedValueCapacity * 2, vector.getValueCapacity());
+    } finally {
+      vector.close();
+    }
+
+    // common case: value count < max value capacity
+    try {
+      vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 8);
+      vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION
+      vector.reAlloc(); // this should throw an IOOB
+    } finally {
+      vector.close();
+    }
+  }
+
+  @Test(expected = OversizedAllocationException.class)
+  public void testBitVectorReallocation() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    final BitVector vector = new BitVector(field, allocator);
+    // edge case 1: buffer size ~ max value capacity
+    final int expectedValueCapacity = 1 << 29;
+    try {
+      vector.allocateNew(expectedValueCapacity);
+      assertEquals(expectedValueCapacity, vector.getValueCapacity());
+      vector.reAlloc();
+      assertEquals(expectedValueCapacity * 2, vector.getValueCapacity());
+    } finally {
+      vector.close();
+    }
+
+    // common: value count < MAX_VALUE_ALLOCATION
+    try {
+      vector.allocateNew(expectedValueCapacity);
+      for (int i=0; i<3;i++) {
+        vector.reAlloc(); // expand buffer size
+      }
+      assertEquals(Integer.MAX_VALUE, vector.getValueCapacity());
+      vector.reAlloc(); // buffer size ~ max allocation
+      assertEquals(Integer.MAX_VALUE, vector.getValueCapacity());
+      vector.reAlloc(); // overflow
+    } finally {
+      vector.close();
+    }
+  }
+
+
+  @Test(expected = OversizedAllocationException.class)
+  public void testVariableVectorReallocation() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    final VarCharVector vector = new VarCharVector(field, allocator);
+    // edge case 1: value count = MAX_VALUE_ALLOCATION
+    final int expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE;
+    final int expectedOffsetSize = 10;
+    try {
+      vector.allocateNew(expectedAllocationInBytes, 10);
+      assertTrue(expectedOffsetSize <= vector.getValueCapacity());
+      assertTrue(expectedAllocationInBytes <= vector.getBuffer().capacity());
+      vector.reAlloc();
+      assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity());
+      assertTrue(expectedAllocationInBytes * 2 <= vector.getBuffer().capacity());
+    } finally {
+      vector.close();
+    }
+
+    // common: value count < MAX_VALUE_ALLOCATION
+    try {
+      vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 2, 0);
+      vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION
+      vector.reAlloc(); // this tests if it overflows
+    } finally {
+      vector.close();
+    }
+  }
+
+  @Test
+  public void testFixedType() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+
+    // Create a new value vector for 1024 integers.
+    try (final UInt4Vector vector = new UInt4Vector(field, allocator)) {
+      final UInt4Vector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024);
+
+      // Put and set a few values
+      m.setSafe(0, 100);
+      m.setSafe(1, 101);
+      m.setSafe(100, 102);
+      m.setSafe(1022, 103);
+      m.setSafe(1023, 104);
+
+      final UInt4Vector.Accessor accessor = vector.getAccessor();
+      assertEquals(100, accessor.get(0));
+      assertEquals(101, accessor.get(1));
+      assertEquals(102, accessor.get(100));
+      assertEquals(103, accessor.get(1022));
+      assertEquals(104, accessor.get(1023));
+    }
+  }
+
+  @Test
+  public void testNullableVarLen2() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+    // Create a new value vector for 1024 integers.
+    try (final NullableVarCharVector vector = new NullableVarCharVector(field, allocator)) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024 * 10, 1024);
+
+      m.set(0, STR1);
+      m.set(1, STR2);
+      m.set(2, STR3);
+
+      // Check the sample strings.
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+
+      // Ensure null value throws.
+      boolean b = false;
+      try {
+        vector.getAccessor().get(3);
+      } catch (IllegalStateException e) {
+        b = true;
+      } finally {
+        assertTrue(b);
+      }
+    }
+  }
+
+  @Test
+  public void testRepeatedIntVector() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+    // Create a new value vector.
+    try (final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator)) {
+
+      // Populate the vector.
+      final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+      final int nRecords = 7;
+      final int nElements = values.length;
+      vector1.allocateNew(nRecords, nRecords * nElements);
+      final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+      for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+        mutator.startNewValue(recordIndex);
+        for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+          mutator.add(recordIndex, recordIndex * values[elementIndex]);
+        }
+      }
+      mutator.setValueCount(nRecords);
+
+      // Verify the contents.
+      final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+      assertEquals(nRecords, accessor1.getValueCount());
+      for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+        for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+          final int value = accessor1.get(recordIndex, elementIndex);
+          assertEquals(recordIndex * values[elementIndex], value);
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testNullableFixedType() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableUInt4Holder.TYPE);
+
+    // Create a new value vector for 1024 integers.
+    try (final NullableUInt4Vector vector = new NullableUInt4Vector(field, allocator)) {
+      final NullableUInt4Vector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024);
+
+      // Put and set a few values
+      m.set(0, 100);
+      m.set(1, 101);
+      m.set(100, 102);
+      m.set(1022, 103);
+      m.set(1023, 104);
+
+      final NullableUInt4Vector.Accessor accessor = vector.getAccessor();
+      assertEquals(100, accessor.get(0));
+      assertEquals(101, accessor.get(1));
+      assertEquals(102, accessor.get(100));
+      assertEquals(103, accessor.get(1022));
+      assertEquals(104, accessor.get(1023));
+
+      // Ensure null values throw
+      {
+        boolean b = false;
+        try {
+          accessor.get(3);
+        } catch (IllegalStateException e) {
+          b = true;
+        } finally {
+          assertTrue(b);
+        }
+      }
+
+      vector.allocateNew(2048);
+      {
+        boolean b = false;
+        try {
+          accessor.get(0);
+        } catch (IllegalStateException e) {
+          b = true;
+        } finally {
+          assertTrue(b);
+        }
+      }
+
+      m.set(0, 100);
+      m.set(1, 101);
+      m.set(100, 102);
+      m.set(1022, 103);
+      m.set(1023, 104);
+      assertEquals(100, accessor.get(0));
+      assertEquals(101, accessor.get(1));
+      assertEquals(102, accessor.get(100));
+      assertEquals(103, accessor.get(1022));
+      assertEquals(104, accessor.get(1023));
+
+      // Ensure null values throw.
+      {
+        boolean b = false;
+        try {
+          vector.getAccessor().get(3);
+        } catch (IllegalStateException e) {
+          b = true;
+        } finally {
+          assertTrue(b);
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testNullableFloat() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
+
+    // Create a new value vector for 1024 integers
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) BasicTypeHelper.getNewVector(field, allocator)) {
+      final NullableFloat4Vector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024);
+
+      // Put and set a few values.
+      m.set(0, 100.1f);
+      m.set(1, 101.2f);
+      m.set(100, 102.3f);
+      m.set(1022, 103.4f);
+      m.set(1023, 104.5f);
+
+      final NullableFloat4Vector.Accessor accessor = vector.getAccessor();
+      assertEquals(100.1f, accessor.get(0), 0);
+      assertEquals(101.2f, accessor.get(1), 0);
+      assertEquals(102.3f, accessor.get(100), 0);
+      assertEquals(103.4f, accessor.get(1022), 0);
+      assertEquals(104.5f, accessor.get(1023), 0);
+
+      // Ensure null values throw.
+      {
+        boolean b = false;
+        try {
+          vector.getAccessor().get(3);
+        } catch (IllegalStateException e) {
+          b = true;
+        } finally {
+          assertTrue(b);
+        }
+      }
+
+      vector.allocateNew(2048);
+      {
+        boolean b = false;
+        try {
+          accessor.get(0);
+        } catch (IllegalStateException e) {
+          b = true;
+        } finally {
+          assertTrue(b);
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testBitVector() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
+
+    // Create a new value vector for 1024 integers
+    try (final BitVector vector = new BitVector(field, allocator)) {
+      final BitVector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024);
+
+      // Put and set a few values
+      m.set(0, 1);
+      m.set(1, 0);
+      m.set(100, 0);
+      m.set(1022, 1);
+
+      final BitVector.Accessor accessor = vector.getAccessor();
+      assertEquals(1, accessor.get(0));
+      assertEquals(0, accessor.get(1));
+      assertEquals(0, accessor.get(100));
+      assertEquals(1, accessor.get(1022));
+
+      // test setting the same value twice
+      m.set(0, 1);
+      m.set(0, 1);
+      m.set(1, 0);
+      m.set(1, 0);
+      assertEquals(1, accessor.get(0));
+      assertEquals(0, accessor.get(1));
+
+      // test toggling the values
+      m.set(0, 0);
+      m.set(1, 1);
+      assertEquals(0, accessor.get(0));
+      assertEquals(1, accessor.get(1));
+
+      // Ensure unallocated space returns 0
+      assertEquals(0, accessor.get(3));
+    }
+  }
+
+  @Test
+  public void testReAllocNullableFixedWidthVector() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
+
+    // Create a new value vector for 1024 integers
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) BasicTypeHelper.getNewVector(field, allocator)) {
+      final NullableFloat4Vector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024);
+
+      assertEquals(1024, vector.getValueCapacity());
+
+      // Put values in indexes that fall within the initial allocation
+      m.setSafe(0, 100.1f);
+      m.setSafe(100, 102.3f);
+      m.setSafe(1023, 104.5f);
+
+      // Now try to put values in space that falls beyond the initial allocation
+      m.setSafe(2000, 105.5f);
+
+      // Check valueCapacity is more than initial allocation
+      assertEquals(1024 * 2, vector.getValueCapacity());
+
+      final NullableFloat4Vector.Accessor accessor = vector.getAccessor();
+      assertEquals(100.1f, accessor.get(0), 0);
+      assertEquals(102.3f, accessor.get(100), 0);
+      assertEquals(104.5f, accessor.get(1023), 0);
+      assertEquals(105.5f, accessor.get(2000), 0);
+
+      // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors
+      // as we don't call setSafe for null values, but we do call setValueCount when all values are inserted into the
+      // vector
+      m.setValueCount(vector.getValueCapacity() + 200);
+    }
+  }
+
+  @Test
+  public void testReAllocNullableVariableWidthVector() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+    // Create a new value vector for 1024 integers
+    try (final NullableVarCharVector vector = (NullableVarCharVector) BasicTypeHelper.getNewVector(field, allocator)) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
+      vector.allocateNew();
+
+      int initialCapacity = vector.getValueCapacity();
+
+      // Put values in indexes that fall within the initial allocation
+      m.setSafe(0, STR1, 0, STR1.length);
+      m.setSafe(initialCapacity - 1, STR2, 0, STR2.length);
+
+      // Now try to put values in space that falls beyond the initial allocation
+      m.setSafe(initialCapacity + 200, STR3, 0, STR3.length);
+
+      // Check valueCapacity is more than initial allocation
+      assertEquals((initialCapacity + 1) * 2 - 1, vector.getValueCapacity());
+
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(initialCapacity - 1));
+      assertArrayEquals(STR3, accessor.get(initialCapacity + 200));
+
+      // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors
+      // as we don't call setSafe for null values, but we do call setValueCount when the current batch is processed.
+      m.setValueCount(vector.getValueCapacity() + 200);
+    }
+  }
+
+  @Test
+  public void testVVInitialCapacity() throws Exception {
+    final MaterializedField[] fields = new MaterializedField[9];
+    final ValueVector[] valueVectors = new ValueVector[9];
+
+    fields[0] = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
+    fields[1] = MaterializedField.create(EMPTY_SCHEMA_PATH, IntHolder.TYPE);
+    fields[2] = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+    fields[3] = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVar16CharHolder.TYPE);
+    fields[4] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedFloat4Holder.TYPE);
+    fields[5] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedVarBinaryHolder.TYPE);
+
+    fields[6] = MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE);
+    fields[6].addChild(fields[0] /*bit*/);
+    fields[6].addChild(fields[2] /*varchar*/);
+
+    fields[7] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE);
+    fields[7].addChild(fields[1] /*int*/);
+    fields[7].addChild(fields[3] /*optional var16char*/);
+
+    fields[8] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE);
+    fields[8].addChild(fields[1] /*int*/);
+
+    final int initialCapacity = 1024;
+
+    try {
+      for (int i = 0; i < valueVectors.length; i++) {
+        valueVectors[i] = BasicTypeHelper.getNewVector(fields[i], allocator);
+        valueVectors[i].setInitialCapacity(initialCapacity);
+        valueVectors[i].allocateNew();
+      }
+
+      for (int i = 0; i < valueVectors.length; i++) {
+        final ValueVector vv = valueVectors[i];
+        final int vvCapacity = vv.getValueCapacity();
+
+        // this can't be equality because Nullables will be allocated using power of two sized buffers (thus need 1025
+        // spots in one vector > power of two is 2048, available capacity will be 2048 => 2047)
+        assertTrue(String.format("Incorrect value capacity for %s [%d]", vv.getField(), vvCapacity),
+                initialCapacity <= vvCapacity);
+      }
+    } finally {
+      for (ValueVector v : valueVectors) {
+        v.close();
+      }
+    }
+  }
+
+}

From e822ea758dc18ade9d3386acfd1d38e7b05ba3dd Mon Sep 17 00:00:00 2001
From: Minji Kim <minji@dremio.com>
Date: Mon, 7 Mar 2016 15:23:33 -0800
Subject: [PATCH 0029/1644] ARROW-46: ListVector should initialize bits in
 allocateNew

---
 .../arrow/vector/complex/ListVector.java      |  1 +
 .../apache/arrow/vector/TestValueVector.java  | 20 +++++++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 13610c4f03f61..3e60c76802380 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -72,6 +72,7 @@ public UnionListWriter getWriter() {
   @Override
   public void allocateNew() throws OutOfMemoryException {
     super.allocateNewSafe();
+    bits.allocateNewSafe();
   }
 
   public void transferTo(ListVector target) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 4488d750284c7..ac3eebe98eab7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -24,10 +24,13 @@
 import java.nio.charset.Charset;
 
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.RepeatedListVector;
 import org.apache.arrow.vector.complex.RepeatedMapVector;
 import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.BasicTypeHelper;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.holders.BitHolder;
@@ -518,4 +521,21 @@ public void testVVInitialCapacity() throws Exception {
     }
   }
 
+  @Test
+  public void testListVectorShouldNotThrowOversizedAllocationException() throws Exception {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH,
+            Types.optional(MinorType.LIST));
+    ListVector vector = new ListVector(field, allocator, null);
+    ListVector vectorFrom = new ListVector(field, allocator, null);
+    vectorFrom.allocateNew();
+
+    for (int i = 0; i < 10000; i++) {
+      vector.allocateNew();
+      vector.copyFromSafe(0, 0, vectorFrom);
+      vector.clear();
+    }
+
+    vectorFrom.clear();
+    vector.clear();
+  }
 }

From 83675273bd2057552ae64b7d8632a54093a02ed9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Tue, 8 Mar 2016 20:28:58 -0800
Subject: [PATCH 0030/1644] ARROW-42: Add Python tests to Travis CI build

Author: Wes McKinney <wesm@apache.org>

Closes #22 from wesm/ARROW-42 and squashes the following commits:

3b056a1 [Wes McKinney] Modularize Travis CI build and add Python build script. Remove parquet.pyx from Cython build for now, suppress -Wunused-variable in Cython compilation. Add missing formatting.py file
---
 .travis.yml                            | 23 ++++++++++
 ci/travis_before_script_cpp.sh         | 26 ++++++++++++
 ci/travis_script_cpp.sh                | 22 +---------
 ci/travis_script_python.sh             | 59 ++++++++++++++++++++++++++
 cpp/src/arrow/table/column-test.cc     |  2 +
 cpp/src/arrow/table/schema-test.cc     |  2 +
 cpp/src/arrow/table/table-test.cc      |  4 ++
 cpp/src/arrow/type.cc                  | 14 ------
 cpp/src/arrow/type.h                   | 14 ------
 python/CMakeLists.txt                  |  2 -
 python/arrow/formatting.py             | 56 ++++++++++++++++++++++++
 python/cmake_modules/UseCython.cmake   |  5 +++
 python/requirements.txt                |  4 ++
 python/setup.py                        |  2 +-
 python/src/pyarrow/adapters/builtin.cc | 20 ++++++---
 python/src/pyarrow/adapters/builtin.h  |  2 +
 python/src/pyarrow/helpers.cc          | 14 ++++++
 python/src/pyarrow/helpers.h           | 14 ++++++
 python/src/pyarrow/util/CMakeLists.txt | 18 +-------
 19 files changed, 228 insertions(+), 75 deletions(-)
 create mode 100755 ci/travis_before_script_cpp.sh
 create mode 100755 ci/travis_script_python.sh
 create mode 100644 python/arrow/formatting.py
 create mode 100644 python/requirements.txt

diff --git a/.travis.yml b/.travis.yml
index cb2d5cb1bad19..9e858d7d98e48 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,9 @@ addons:
     packages:
     - gcc-4.9   # Needed for C++11
     - g++-4.9   # Needed for C++11
+    - gdb
     - gcov
+    - ccache
     - cmake
     - valgrind
 
@@ -17,11 +19,32 @@ matrix:
   - compiler: gcc
     language: cpp
     os: linux
+    before_script:
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
+    - export CC="gcc-4.9"
+    - export CXX="g++-4.9"
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
   - compiler: clang
     language: cpp
     os: osx
     addons:
+    before_script:
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
+
+before_install:
+- ulimit -c unlimited -S
+- export CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
+- export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
+- export LD_LIBRARY_PATH=$ARROW_CPP_INSTALL/lib:$LD_LIBRARY_PATH
+
+after_script:
+- rm -rf $CPP_BUILD_DIR
+
+after_failure:
+- COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1)
+- if [[ -f "$COREFILE" ]]; then gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch; fi
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
new file mode 100755
index 0000000000000..4d5bef8bbdf70
--- /dev/null
+++ b/ci/travis_before_script_cpp.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+
+mkdir $CPP_BUILD_DIR
+pushd $CPP_BUILD_DIR
+
+CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+
+# Build an isolated thirdparty
+cp -r $CPP_DIR/thirdparty .
+cp $CPP_DIR/setup_build_env.sh .
+
+source setup_build_env.sh
+
+echo $GTEST_HOME
+
+: ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
+
+cmake -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+make -j4
+make install
+
+popd
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index 28f16cc021fe3..3e843dd759ea1 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -2,28 +2,11 @@
 
 set -e
 
-mkdir $TRAVIS_BUILD_DIR/cpp-build
-pushd $TRAVIS_BUILD_DIR/cpp-build
+: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
 
-CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+pushd $CPP_BUILD_DIR
 
-# Build an isolated thirdparty
-cp -r $CPP_DIR/thirdparty .
-cp $CPP_DIR/setup_build_env.sh .
-
-if [ $TRAVIS_OS_NAME == "linux" ]; then
-  # Use a C++11 compiler on Linux
-  export CC="gcc-4.9"
-  export CXX="g++-4.9"
-fi
-
-source setup_build_env.sh
-
-echo $GTEST_HOME
-
-cmake -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
 make lint
-make -j4
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
   valgrind --tool=memcheck --leak-check=yes --error-exitcode=1 ctest
@@ -32,4 +15,3 @@ else
 fi
 
 popd
-rm -rf cpp-build
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
new file mode 100755
index 0000000000000..9b0bd4f54cbc9
--- /dev/null
+++ b/ci/travis_script_python.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+set -e
+
+PYTHON_DIR=$TRAVIS_BUILD_DIR/python
+
+# Share environment with C++
+pushd $CPP_BUILD_DIR
+source setup_build_env.sh
+popd
+
+pushd $PYTHON_DIR
+
+# Bootstrap a Conda Python environment
+
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
+else
+  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
+fi
+
+curl $MINICONDA_URL > miniconda.sh
+MINICONDA=$TRAVIS_BUILD_DIR/miniconda
+bash miniconda.sh -b -p $MINICONDA
+export PATH="$MINICONDA/bin:$PATH"
+conda update -y -q conda
+conda info -a
+
+PYTHON_VERSION=3.5
+CONDA_ENV_NAME=pyarrow-test
+
+conda create -y -q -n $CONDA_ENV_NAME python=$PYTHON_VERSION
+source activate $CONDA_ENV_NAME
+
+python --version
+which python
+
+# faster builds, please
+conda install -y nomkl
+
+# Expensive dependencies install from Continuum package repo
+conda install -y pip numpy pandas cython
+
+# Other stuff pip install
+pip install -r requirements.txt
+
+export ARROW_HOME=$ARROW_CPP_INSTALL
+
+python setup.py build_ext --inplace
+
+py.test -vv -r sxX arrow
+
+# if [ $TRAVIS_OS_NAME == "linux" ]; then
+#   valgrind --tool=memcheck py.test -vv -r sxX arrow
+# else
+#   py.test -vv -r sxX arrow
+# fi
+
+popd
diff --git a/cpp/src/arrow/table/column-test.cc b/cpp/src/arrow/table/column-test.cc
index bf95932916cf4..3b102e48c87cf 100644
--- a/cpp/src/arrow/table/column-test.cc
+++ b/cpp/src/arrow/table/column-test.cc
@@ -33,6 +33,8 @@ using std::vector;
 
 namespace arrow {
 
+const auto INT32 = std::make_shared<Int32Type>();
+
 class TestColumn : public TestBase {
  protected:
   std::shared_ptr<ChunkedArray> data_;
diff --git a/cpp/src/arrow/table/schema-test.cc b/cpp/src/arrow/table/schema-test.cc
index d6725cc08c0c8..9dfade2695311 100644
--- a/cpp/src/arrow/table/schema-test.cc
+++ b/cpp/src/arrow/table/schema-test.cc
@@ -29,6 +29,8 @@ using std::vector;
 
 namespace arrow {
 
+const auto INT32 = std::make_shared<Int32Type>();
+
 TEST(TestField, Basics) {
   shared_ptr<DataType> ftype = INT32;
   shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
diff --git a/cpp/src/arrow/table/table-test.cc b/cpp/src/arrow/table/table-test.cc
index c4fdb062db83a..8b354e8503c71 100644
--- a/cpp/src/arrow/table/table-test.cc
+++ b/cpp/src/arrow/table/table-test.cc
@@ -34,6 +34,10 @@ using std::vector;
 
 namespace arrow {
 
+const auto INT16 = std::make_shared<Int16Type>();
+const auto UINT8 = std::make_shared<UInt8Type>();
+const auto INT32 = std::make_shared<Int32Type>();
+
 class TestTable : public TestBase {
  public:
   void MakeExample1(int length) {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 265770822ce90..0a2e817ad30c6 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -66,18 +66,4 @@ std::string StructType::ToString() const {
   return s.str();
 }
 
-const std::shared_ptr<NullType> NA = std::make_shared<NullType>();
-const std::shared_ptr<BooleanType> BOOL = std::make_shared<BooleanType>();
-const std::shared_ptr<UInt8Type> UINT8 = std::make_shared<UInt8Type>();
-const std::shared_ptr<UInt16Type> UINT16 = std::make_shared<UInt16Type>();
-const std::shared_ptr<UInt32Type> UINT32 = std::make_shared<UInt32Type>();
-const std::shared_ptr<UInt64Type> UINT64 = std::make_shared<UInt64Type>();
-const std::shared_ptr<Int8Type> INT8 = std::make_shared<Int8Type>();
-const std::shared_ptr<Int16Type> INT16 = std::make_shared<Int16Type>();
-const std::shared_ptr<Int32Type> INT32 = std::make_shared<Int32Type>();
-const std::shared_ptr<Int64Type> INT64 = std::make_shared<Int64Type>();
-const std::shared_ptr<FloatType> FLOAT = std::make_shared<FloatType>();
-const std::shared_ptr<DoubleType> DOUBLE = std::make_shared<DoubleType>();
-const std::shared_ptr<StringType> STRING = std::make_shared<StringType>();
-
 } // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index e78e49491193e..00b01ea86e8a5 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -338,20 +338,6 @@ struct StructType : public DataType {
   std::string ToString() const override;
 };
 
-extern const std::shared_ptr<NullType> NA;
-extern const std::shared_ptr<BooleanType> BOOL;
-extern const std::shared_ptr<UInt8Type> UINT8;
-extern const std::shared_ptr<UInt16Type> UINT16;
-extern const std::shared_ptr<UInt32Type> UINT32;
-extern const std::shared_ptr<UInt64Type> UINT64;
-extern const std::shared_ptr<Int8Type> INT8;
-extern const std::shared_ptr<Int16Type> INT16;
-extern const std::shared_ptr<Int32Type> INT32;
-extern const std::shared_ptr<Int64Type> INT64;
-extern const std::shared_ptr<FloatType> FLOAT;
-extern const std::shared_ptr<DoubleType> DOUBLE;
-extern const std::shared_ptr<StringType> STRING;
-
 } // namespace arrow
 
 #endif  // ARROW_TYPE_H
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 8fdd829010eef..8f5c27b0f76d7 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -404,7 +404,6 @@ set(PYARROW_SRCS
 )
 
 set(LINK_LIBS
-  pyarrow_util
   arrow
 )
 
@@ -428,7 +427,6 @@ set(CYTHON_EXTENSIONS
   array
   config
   error
-  parquet
   scalar
   schema
 )
diff --git a/python/arrow/formatting.py b/python/arrow/formatting.py
new file mode 100644
index 0000000000000..a42d4e4bb5713
--- /dev/null
+++ b/python/arrow/formatting.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pretty-printing and other formatting utilities for Arrow data structures
+
+import arrow.scalar as scalar
+
+
+def array_format(arr, window=None):
+    values = []
+
+    if window is None or window * 2 >= len(arr):
+        for x in arr:
+            values.append(value_format(x, 0))
+        contents = _indent(',\n'.join(values), 2)
+    else:
+        for i in range(window):
+            values.append(value_format(arr[i], 0) + ',')
+        values.append('...')
+        for i in range(len(arr) - window, len(arr)):
+            formatted = value_format(arr[i], 0)
+            if i < len(arr) - 1:
+                formatted += ','
+            values.append(formatted)
+        contents = _indent('\n'.join(values), 2)
+
+    return '[\n{0}\n]'.format(contents)
+
+
+def value_format(x, indent_level=0):
+    if isinstance(x, scalar.ListValue):
+        contents = ',\n'.join(value_format(item) for item in x)
+        return '[{0}]'.format(_indent(contents, 1).strip())
+    else:
+        return repr(x)
+
+
+def _indent(text, spaces):
+    if spaces == 0:
+        return text
+    block = ' ' * spaces
+    return '\n'.join(block + x for x in text.split('\n'))
diff --git a/python/cmake_modules/UseCython.cmake b/python/cmake_modules/UseCython.cmake
index e7034db52f335..3b1c201edff5f 100644
--- a/python/cmake_modules/UseCython.cmake
+++ b/python/cmake_modules/UseCython.cmake
@@ -121,6 +121,11 @@ function( compile_pyx _name pyx_target_name generated_files pyx_file)
       set( _generated_files "${_name}.${extension}")
   endif()
   set_source_files_properties( ${_generated_files} PROPERTIES GENERATED TRUE )
+
+  # Cython creates a lot of compiler warning detritus on clang
+  set_source_files_properties(${_generated_files} PROPERTIES
+    COMPILE_FLAGS -Wno-unused-function)
+
   set( ${generated_files} ${_generated_files} PARENT_SCOPE )
 
   # Add the command to run the compiler.
diff --git a/python/requirements.txt b/python/requirements.txt
new file mode 100644
index 0000000000000..a82cb20aab86e
--- /dev/null
+++ b/python/requirements.txt
@@ -0,0 +1,4 @@
+pytest
+numpy>=1.7.0
+pandas>=0.12.0
+six
diff --git a/python/setup.py b/python/setup.py
index 9a0de071a9c40..eb3ff2a1547d6 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -210,7 +210,7 @@ def get_ext_built(self, name):
             return name + suffix
 
     def get_cmake_cython_names(self):
-        return ['array', 'config', 'error', 'parquet', 'scalar', 'schema']
+        return ['array', 'config', 'error', 'scalar', 'schema']
 
     def get_names(self):
         return self._found_names
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index 60d6248842ec9..bb7905236c59c 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -22,6 +22,7 @@
 
 #include <arrow/api.h>
 
+#include "pyarrow/helpers.h"
 #include "pyarrow/status.h"
 
 using arrow::ArrayBuilder;
@@ -74,16 +75,16 @@ class ScalarVisitor {
   std::shared_ptr<DataType> GetType() {
     // TODO(wesm): handling mixed-type cases
     if (float_count_) {
-      return arrow::DOUBLE;
+      return DOUBLE;
     } else if (int_count_) {
       // TODO(wesm): tighter type later
-      return arrow::INT64;
+      return INT64;
     } else if (bool_count_) {
-      return arrow::BOOL;
+      return BOOL;
     } else if (string_count_) {
-      return arrow::STRING;
+      return STRING;
     } else {
-      return arrow::NA;
+      return NA;
     }
   }
 
@@ -145,7 +146,7 @@ class SeqVisitor {
   std::shared_ptr<DataType> GetType() {
     if (scalars_.total_count() == 0) {
       if (max_nesting_level_ == 0) {
-        return arrow::NA;
+        return NA;
       } else {
         return nullptr;
       }
@@ -209,7 +210,7 @@ static Status InferArrowType(PyObject* obj, int64_t* size,
 
   // For 0-length sequences, refuse to guess
   if (*size == 0) {
-    *out_type = arrow::NA;
+    *out_type = NA;
   }
 
   SeqVisitor seq_visitor;
@@ -217,6 +218,11 @@ static Status InferArrowType(PyObject* obj, int64_t* size,
   PY_RETURN_NOT_OK(seq_visitor.Validate());
 
   *out_type = seq_visitor.GetType();
+
+  if (*out_type == nullptr) {
+    return Status::TypeError("Unable to determine data type");
+  }
+
   return Status::OK();
 }
 
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
index 24886f4970d50..88869c2048003 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -25,6 +25,8 @@
 
 #include <memory>
 
+#include <arrow/type.h>
+
 #include "pyarrow/common.h"
 
 namespace arrow { class Array; }
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
index d0969dacc21e0..0921fc4994599 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/python/src/pyarrow/helpers.cc
@@ -23,6 +23,20 @@ using namespace arrow;
 
 namespace pyarrow {
 
+const std::shared_ptr<NullType> NA = std::make_shared<NullType>();
+const std::shared_ptr<BooleanType> BOOL = std::make_shared<BooleanType>();
+const std::shared_ptr<UInt8Type> UINT8 = std::make_shared<UInt8Type>();
+const std::shared_ptr<UInt16Type> UINT16 = std::make_shared<UInt16Type>();
+const std::shared_ptr<UInt32Type> UINT32 = std::make_shared<UInt32Type>();
+const std::shared_ptr<UInt64Type> UINT64 = std::make_shared<UInt64Type>();
+const std::shared_ptr<Int8Type> INT8 = std::make_shared<Int8Type>();
+const std::shared_ptr<Int16Type> INT16 = std::make_shared<Int16Type>();
+const std::shared_ptr<Int32Type> INT32 = std::make_shared<Int32Type>();
+const std::shared_ptr<Int64Type> INT64 = std::make_shared<Int64Type>();
+const std::shared_ptr<FloatType> FLOAT = std::make_shared<FloatType>();
+const std::shared_ptr<DoubleType> DOUBLE = std::make_shared<DoubleType>();
+const std::shared_ptr<StringType> STRING = std::make_shared<StringType>();
+
 #define GET_PRIMITIVE_TYPE(NAME, Type)          \
   case LogicalType::NAME:                       \
     if (nullable) {                             \
diff --git a/python/src/pyarrow/helpers.h b/python/src/pyarrow/helpers.h
index 1a24f056febe6..e41568d5881d4 100644
--- a/python/src/pyarrow/helpers.h
+++ b/python/src/pyarrow/helpers.h
@@ -26,6 +26,20 @@ namespace pyarrow {
 using arrow::DataType;
 using arrow::LogicalType;
 
+extern const std::shared_ptr<arrow::NullType> NA;
+extern const std::shared_ptr<arrow::BooleanType> BOOL;
+extern const std::shared_ptr<arrow::UInt8Type> UINT8;
+extern const std::shared_ptr<arrow::UInt16Type> UINT16;
+extern const std::shared_ptr<arrow::UInt32Type> UINT32;
+extern const std::shared_ptr<arrow::UInt64Type> UINT64;
+extern const std::shared_ptr<arrow::Int8Type> INT8;
+extern const std::shared_ptr<arrow::Int16Type> INT16;
+extern const std::shared_ptr<arrow::Int32Type> INT32;
+extern const std::shared_ptr<arrow::Int64Type> INT64;
+extern const std::shared_ptr<arrow::FloatType> FLOAT;
+extern const std::shared_ptr<arrow::DoubleType> DOUBLE;
+extern const std::shared_ptr<arrow::StringType> STRING;
+
 std::shared_ptr<DataType> GetPrimitiveType(LogicalType::type type,
     bool nullable);
 
diff --git a/python/src/pyarrow/util/CMakeLists.txt b/python/src/pyarrow/util/CMakeLists.txt
index 60dc80eb38cb6..3fd8bac31506d 100644
--- a/python/src/pyarrow/util/CMakeLists.txt
+++ b/python/src/pyarrow/util/CMakeLists.txt
@@ -15,22 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-#######################################
-# pyarrow_util
-#######################################
-
-set(UTIL_SRCS
-)
-
-set(UTIL_LIBS
-)
-
-add_library(pyarrow_util STATIC
-  ${UTIL_SRCS}
-)
-target_link_libraries(pyarrow_util ${UTIL_LIBS})
-SET_TARGET_PROPERTIES(pyarrow_util PROPERTIES LINKER_LANGUAGE CXX)
-
 #######################################
 # pyarrow_test_main
 #######################################
@@ -40,7 +24,7 @@ add_library(pyarrow_test_main
 
 if (APPLE)
   target_link_libraries(pyarrow_test_main
-    gmock
+    gtest
     dl)
   set_target_properties(pyarrow_test_main
         PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")

From 6fdcd4943ff9a8cc66afbee380217cec40c0cda0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Wed, 9 Mar 2016 15:45:05 -0800
Subject: [PATCH 0031/1644] ARROW-54: [Python] Rename package to "pyarrow"

Also fixed rpath issues (at great cost) per ARROW-53

Author: Wes McKinney <wesm@apache.org>

Closes #23 from wesm/ARROW-54 and squashes the following commits:

b8ce0e8 [Wes McKinney] Update installation instructions
cae9b39 [Wes McKinney] Fix rpath issues per ARROW-53
7554539 [Wes McKinney] Twiddle rpath stuff, remove empty arrow_test_util module
8cca41a [Wes McKinney] Fix Travis CI script for renamed package
1d37c93 [Wes McKinney] Opt in to building unit tests
60088d0 [Wes McKinney] Rename package to pyarrow
e3d0caf [Wes McKinney] Note on other Python interpreters
80d3bac [Wes McKinney] Start installation document
---
 .travis.yml                                   |  4 +-
 ci/travis_script_python.sh                    |  2 +-
 cpp/CMakeLists.txt                            | 29 ++++---
 cpp/src/arrow/CMakeLists.txt                  |  2 +-
 cpp/src/arrow/util/CMakeLists.txt             | 44 ++++------
 python/CMakeLists.txt                         | 31 ++++---
 python/arrow/__init__.py                      | 38 --------
 python/doc/INSTALL.md                         | 87 +++++++++++++++++++
 python/pyarrow/__init__.py                    | 38 ++++++++
 python/{arrow => pyarrow}/array.pxd           |  8 +-
 python/{arrow => pyarrow}/array.pyx           | 14 +--
 python/{arrow => pyarrow}/compat.py           |  0
 python/{arrow => pyarrow}/config.pyx          |  0
 python/{arrow => pyarrow}/error.pxd           |  2 +-
 python/{arrow => pyarrow}/error.pyx           |  5 +-
 python/{arrow => pyarrow}/formatting.py       |  2 +-
 .../{arrow => pyarrow}/includes/__init__.pxd  |  0
 python/{arrow => pyarrow}/includes/common.pxd |  0
 .../includes/libarrow.pxd}                    |  2 +-
 .../{arrow => pyarrow}/includes/parquet.pxd   |  2 +-
 .../{arrow => pyarrow}/includes/pyarrow.pxd   |  6 +-
 python/{arrow => pyarrow}/parquet.pyx         |  4 +-
 python/{arrow => pyarrow}/scalar.pxd          |  6 +-
 python/{arrow => pyarrow}/scalar.pyx          |  6 +-
 python/{arrow => pyarrow}/schema.pxd          |  4 +-
 python/{arrow => pyarrow}/schema.pyx          |  6 +-
 python/{arrow => pyarrow}/tests/__init__.py   |  0
 python/{arrow => pyarrow}/tests/test_array.py | 16 ++--
 .../tests/test_convert_builtin.py             | 52 +++++------
 .../{arrow => pyarrow}/tests/test_scalars.py  |  4 +-
 .../{arrow => pyarrow}/tests/test_schema.py   |  4 +-
 python/requirements.txt                       |  1 -
 python/setup.py                               | 52 ++++++-----
 python/src/pyarrow/util/CMakeLists.txt        | 30 ++++---
 34 files changed, 300 insertions(+), 201 deletions(-)
 delete mode 100644 python/arrow/__init__.py
 create mode 100644 python/doc/INSTALL.md
 create mode 100644 python/pyarrow/__init__.py
 rename python/{arrow => pyarrow}/array.pxd (90%)
 rename python/{arrow => pyarrow}/array.pyx (93%)
 rename python/{arrow => pyarrow}/compat.py (100%)
 rename python/{arrow => pyarrow}/config.pyx (100%)
 rename python/{arrow => pyarrow}/error.pxd (95%)
 rename python/{arrow => pyarrow}/error.pyx (92%)
 rename python/{arrow => pyarrow}/formatting.py (98%)
 rename python/{arrow => pyarrow}/includes/__init__.pxd (100%)
 rename python/{arrow => pyarrow}/includes/common.pxd (100%)
 rename python/{arrow/includes/arrow.pxd => pyarrow/includes/libarrow.pxd} (99%)
 rename python/{arrow => pyarrow}/includes/parquet.pxd (97%)
 rename python/{arrow => pyarrow}/includes/pyarrow.pxd (90%)
 rename python/{arrow => pyarrow}/parquet.pyx (91%)
 rename python/{arrow => pyarrow}/scalar.pxd (93%)
 rename python/{arrow => pyarrow}/scalar.pyx (97%)
 rename python/{arrow => pyarrow}/schema.pxd (91%)
 rename python/{arrow => pyarrow}/schema.pyx (97%)
 rename python/{arrow => pyarrow}/tests/__init__.py (100%)
 rename python/{arrow => pyarrow}/tests/test_array.py (80%)
 rename python/{arrow => pyarrow}/tests/test_convert_builtin.py (58%)
 rename python/{arrow => pyarrow}/tests/test_scalars.py (97%)
 rename python/{arrow => pyarrow}/tests/test_schema.py (96%)

diff --git a/.travis.yml b/.travis.yml
index 9e858d7d98e48..49a956ead3dca 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,7 +27,8 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
   - compiler: clang
-    language: cpp
+    language: objective-c
+    osx_image: xcode6.4
     os: osx
     addons:
     before_script:
@@ -40,7 +41,6 @@ before_install:
 - ulimit -c unlimited -S
 - export CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
 - export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
-- export LD_LIBRARY_PATH=$ARROW_CPP_INSTALL/lib:$LD_LIBRARY_PATH
 
 after_script:
 - rm -rf $CPP_BUILD_DIR
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 9b0bd4f54cbc9..14d66b44ff8ee 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -48,7 +48,7 @@ export ARROW_HOME=$ARROW_CPP_INSTALL
 
 python setup.py build_ext --inplace
 
-py.test -vv -r sxX arrow
+py.test -vv -r sxX pyarrow
 
 # if [ $TRAVIS_OS_NAME == "linux" ]; then
 #   valgrind --tool=memcheck py.test -vv -r sxX arrow
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index e8cb88c0b4d9b..f5f6038031127 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -37,11 +37,6 @@ if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
   set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 endif()
 
-if(APPLE)
-  # In newer versions of CMake, this is the default setting
-  set(CMAKE_MACOSX_RPATH 1)
-endif()
-
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
   set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
@@ -339,10 +334,13 @@ endfunction()
 if ("$ENV{GTEST_HOME}" STREQUAL "")
   set(GTest_HOME ${THIRDPARTY_DIR}/googletest-release-1.7.0)
 endif()
-find_package(GTest REQUIRED)
-include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
-ADD_THIRDPARTY_LIB(gtest
-  STATIC_LIB ${GTEST_STATIC_LIB})
+
+if(ARROW_BUILD_TESTS)
+  find_package(GTest REQUIRED)
+  include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(gtest
+    STATIC_LIB ${GTEST_STATIC_LIB})
+endif()
 
 ## Google PerfTools
 ##
@@ -366,7 +364,7 @@ ADD_THIRDPARTY_LIB(gtest
 ############################################################
 # Linker setup
 ############################################################
-set(ARROW_MIN_TEST_LIBS arrow arrow_test_main arrow_test_util ${ARROW_BASE_LIBS})
+set(ARROW_MIN_TEST_LIBS arrow arrow_test_main ${ARROW_BASE_LIBS})
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 ############################################################
@@ -461,9 +459,18 @@ add_library(arrow
   ${LIBARROW_LINKAGE}
   ${ARROW_SRCS}
 )
+
+if (APPLE)
+  set_target_properties(arrow
+    PROPERTIES
+    BUILD_WITH_INSTALL_RPATH ON
+    INSTALL_NAME_DIR "@rpath")
+endif()
+
 set_target_properties(arrow
   PROPERTIES
-  LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+  LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+)
 target_link_libraries(arrow ${LIBARROW_LINK_LIBS})
 
 add_subdirectory(src/arrow)
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 77326ce38d754..73e6a9b22c94a 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -27,6 +27,6 @@ install(FILES
 # Unit tests
 #######################################
 
-set(ARROW_TEST_LINK_LIBS arrow_test_util ${ARROW_MIN_TEST_LIBS})
+set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 ADD_ARROW_TEST(array-test)
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 4272ce4285482..d8e2f98f2c85e 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -28,37 +28,27 @@ install(FILES
   status.h
   DESTINATION include/arrow/util)
 
-#######################################
-# arrow_test_util
-#######################################
-
-add_library(arrow_test_util)
-target_link_libraries(arrow_test_util
-)
-
-SET_TARGET_PROPERTIES(arrow_test_util PROPERTIES LINKER_LANGUAGE CXX)
-
 #######################################
 # arrow_test_main
 #######################################
 
-add_library(arrow_test_main
-  test_main.cc)
-
-if (APPLE)
-  target_link_libraries(arrow_test_main
-    gtest
-	arrow_test_util
-    dl)
-  set_target_properties(arrow_test_main
-        PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-else()
-  target_link_libraries(arrow_test_main
-    gtest
-	arrow_test_util
-    pthread
-    dl
-  )
+if (ARROW_BUILD_TESTS)
+  add_library(arrow_test_main
+	test_main.cc)
+
+  if (APPLE)
+	target_link_libraries(arrow_test_main
+      gtest
+      dl)
+	set_target_properties(arrow_test_main
+      PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  else()
+	target_link_libraries(arrow_test_main
+      gtest
+      pthread
+      dl
+	  )
+  endif()
 endif()
 
 ADD_ARROW_TEST(bit-util-test)
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 8f5c27b0f76d7..0ecafc7202e89 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -45,6 +45,13 @@ if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
   set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 endif()
 
+# Top level cmake dir
+if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
+  option(PYARROW_BUILD_TESTS
+    "Build the PyArrow C++ googletest unit tests"
+    OFF)
+endif()
+
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
   set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
@@ -322,10 +329,12 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
 endfunction()
 
 ## GMock
-find_package(GTest REQUIRED)
-include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
-ADD_THIRDPARTY_LIB(gtest
-  STATIC_LIB ${GTEST_STATIC_LIB})
+if (PYARROW_BUILD_TESTS)
+  find_package(GTest REQUIRED)
+  include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(gtest
+    STATIC_LIB ${GTEST_STATIC_LIB})
+endif()
 
 ## Arrow
 find_package(Arrow REQUIRED)
@@ -391,6 +400,10 @@ endif (UNIX)
 # Subdirectories
 ############################################################
 
+if (UNIX)
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+endif()
+
 add_subdirectory(src/pyarrow)
 add_subdirectory(src/pyarrow/util)
 
@@ -407,10 +420,11 @@ set(LINK_LIBS
   arrow
 )
 
+SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
 add_library(pyarrow SHARED
   ${PYARROW_SRCS})
 target_link_libraries(pyarrow ${LINK_LIBS})
-set_target_properties(pyarrow PROPERTIES LINKER_LANGUAGE CXX)
 
 if(APPLE)
   set_target_properties(pyarrow PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
@@ -420,9 +434,6 @@ endif()
 # Setup and build Cython modules
 ############################################################
 
-set(USE_RELATIVE_RPATH ON)
-set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-
 set(CYTHON_EXTENSIONS
   array
   config
@@ -437,7 +448,7 @@ foreach(module ${CYTHON_EXTENSIONS})
     list(REMOVE_AT directories -1)
 
     string(REPLACE "." "/" module_root "${module}")
-    set(module_SRC arrow/${module_root}.pyx)
+    set(module_SRC pyarrow/${module_root}.pyx)
     set_source_files_properties(${module_SRC} PROPERTIES CYTHON_IS_CXX 1)
 
     cython_add_module(${module_name}
@@ -463,7 +474,7 @@ foreach(module ${CYTHON_EXTENSIONS})
     endwhile(${i} GREATER 0)
 
     # for inplace development for now
-    set(module_install_rpath "${CMAKE_SOURCE_DIR}/arrow/")
+    #set(module_install_rpath "${CMAKE_SOURCE_DIR}/pyarrow/")
 
     set_target_properties(${module_name} PROPERTIES
       INSTALL_RPATH ${module_install_rpath})
diff --git a/python/arrow/__init__.py b/python/arrow/__init__.py
deleted file mode 100644
index 3507ea0235afe..0000000000000
--- a/python/arrow/__init__.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# flake8: noqa
-
-from arrow.array import (Array, from_pylist, total_allocated_bytes,
-                         BooleanArray, NumericArray,
-                         Int8Array, UInt8Array,
-                         ListArray, StringArray)
-
-from arrow.error import ArrowException
-
-from arrow.scalar import (ArrayValue, Scalar, NA, NAType,
-                          BooleanValue,
-                          Int8Value, Int16Value, Int32Value, Int64Value,
-                          UInt8Value, UInt16Value, UInt32Value, UInt64Value,
-                          FloatValue, DoubleValue, ListValue, StringValue)
-
-from arrow.schema import (null, bool_,
-                          int8, int16, int32, int64,
-                          uint8, uint16, uint32, uint64,
-                          float_, double, string,
-                          list_, struct, field,
-                          DataType, Field, Schema)
diff --git a/python/doc/INSTALL.md b/python/doc/INSTALL.md
new file mode 100644
index 0000000000000..d30a03046eda7
--- /dev/null
+++ b/python/doc/INSTALL.md
@@ -0,0 +1,87 @@
+## Building pyarrow (Apache Arrow Python library)
+
+First, clone the master git repository:
+
+```bash
+git clone https://github.com/apache/arrow.git arrow
+```
+
+#### System requirements
+
+Building pyarrow requires:
+
+* A C++11 compiler
+
+  * Linux: gcc >= 4.8 or clang >= 3.5
+  * OS X: XCode 6.4 or higher preferred
+
+* [cmake][1]
+
+#### Python requirements
+
+You will need Python (CPython) 2.7, 3.4, or 3.5 installed. Earlier releases and
+are not being targeted.
+
+> This library targets CPython only due to an emphasis on interoperability with
+> pandas and NumPy, which are only available for CPython.
+
+The build requires NumPy, Cython, and a few other Python dependencies:
+
+```bash
+pip install cython
+cd arrow/python
+pip install -r requirements.txt
+```
+
+#### Installing Arrow C++ library
+
+First, you should choose an installation location for Arrow C++. In the future
+using the default system install location will work, but for now we are being
+explicit:
+
+```bash
+export ARROW_HOME=$HOME/local
+```
+
+Now, we build Arrow:
+
+```bash
+cd arrow/cpp
+
+mkdir dev-build
+cd dev-build
+
+cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME ..
+
+make
+
+# Use sudo here if $ARROW_HOME requires it
+make install
+```
+
+#### Install `pyarrow`
+
+```bash
+cd arrow/python
+
+python setup.py install
+```
+
+> On XCode 6 and prior there are some known OS X `@rpath` issues. If you are
+> unable to import pyarrow, upgrading XCode may be the solution.
+
+
+```python
+In [1]: import pyarrow
+
+In [2]: pyarrow.from_pylist([1,2,3])
+Out[2]:
+<pyarrow.array.Int64Array object at 0x7f899f3e60e8>
+[
+  1,
+  2,
+  3
+]
+```
+
+[1]: https://cmake.org/
\ No newline at end of file
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
new file mode 100644
index 0000000000000..8d93a156bcc3d
--- /dev/null
+++ b/python/pyarrow/__init__.py
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# flake8: noqa
+
+from pyarrow.array import (Array, from_pylist, total_allocated_bytes,
+                           BooleanArray, NumericArray,
+                           Int8Array, UInt8Array,
+                           ListArray, StringArray)
+
+from pyarrow.error import ArrowException
+
+from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
+                            BooleanValue,
+                            Int8Value, Int16Value, Int32Value, Int64Value,
+                            UInt8Value, UInt16Value, UInt32Value, UInt64Value,
+                            FloatValue, DoubleValue, ListValue, StringValue)
+
+from pyarrow.schema import (null, bool_,
+                            int8, int16, int32, int64,
+                            uint8, uint16, uint32, uint64,
+                            float_, double, string,
+                            list_, struct, field,
+                            DataType, Field, Schema)
diff --git a/python/arrow/array.pxd b/python/pyarrow/array.pxd
similarity index 90%
rename from python/arrow/array.pxd
rename to python/pyarrow/array.pxd
index 482f8f796dd26..d0d3486c032fe 100644
--- a/python/arrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -15,12 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.includes.common cimport shared_ptr
-from arrow.includes.arrow cimport CArray, LogicalType
+from pyarrow.includes.common cimport shared_ptr
+from pyarrow.includes.libarrow cimport CArray, LogicalType
 
-from arrow.scalar import NA
+from pyarrow.scalar import NA
 
-from arrow.schema cimport DataType
+from pyarrow.schema cimport DataType
 
 cdef extern from "Python.h":
     int PySlice_Check(object)
diff --git a/python/arrow/array.pyx b/python/pyarrow/array.pyx
similarity index 93%
rename from python/arrow/array.pyx
rename to python/pyarrow/array.pyx
index b367e3b84a8b3..bceb333c94ea5 100644
--- a/python/arrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -19,14 +19,14 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from arrow.includes.arrow cimport *
-cimport arrow.includes.pyarrow as pyarrow
+from pyarrow.includes.libarrow cimport *
+cimport pyarrow.includes.pyarrow as pyarrow
 
-from arrow.compat import frombytes, tobytes
-from arrow.error cimport check_status
+from pyarrow.compat import frombytes, tobytes
+from pyarrow.error cimport check_status
 
-cimport arrow.scalar as scalar
-from arrow.scalar import NA
+cimport pyarrow.scalar as scalar
+from pyarrow.scalar import NA
 
 def total_allocated_bytes():
     cdef MemoryPool* pool = pyarrow.GetMemoryPool()
@@ -52,7 +52,7 @@ cdef class Array:
         raise StopIteration
 
     def __repr__(self):
-        from arrow.formatting import array_format
+        from pyarrow.formatting import array_format
         type_format = object.__repr__(self)
         values = array_format(self, window=10)
         return '{0}\n{1}'.format(type_format, values)
diff --git a/python/arrow/compat.py b/python/pyarrow/compat.py
similarity index 100%
rename from python/arrow/compat.py
rename to python/pyarrow/compat.py
diff --git a/python/arrow/config.pyx b/python/pyarrow/config.pyx
similarity index 100%
rename from python/arrow/config.pyx
rename to python/pyarrow/config.pyx
diff --git a/python/arrow/error.pxd b/python/pyarrow/error.pxd
similarity index 95%
rename from python/arrow/error.pxd
rename to python/pyarrow/error.pxd
index c18cb3efffca6..d226abeda04e0 100644
--- a/python/arrow/error.pxd
+++ b/python/pyarrow/error.pxd
@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.includes.pyarrow cimport *
+from pyarrow.includes.pyarrow cimport *
 
 cdef check_status(const Status& status)
diff --git a/python/arrow/error.pyx b/python/pyarrow/error.pyx
similarity index 92%
rename from python/arrow/error.pyx
rename to python/pyarrow/error.pyx
index f1d516358819d..3f8d7dd646091 100644
--- a/python/arrow/error.pyx
+++ b/python/pyarrow/error.pyx
@@ -15,9 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.includes.common cimport c_string
-
-from arrow.compat import frombytes
+from pyarrow.includes.common cimport c_string
+from pyarrow.compat import frombytes
 
 class ArrowException(Exception):
     pass
diff --git a/python/arrow/formatting.py b/python/pyarrow/formatting.py
similarity index 98%
rename from python/arrow/formatting.py
rename to python/pyarrow/formatting.py
index a42d4e4bb5713..5fe0611f8450b 100644
--- a/python/arrow/formatting.py
+++ b/python/pyarrow/formatting.py
@@ -17,7 +17,7 @@
 
 # Pretty-printing and other formatting utilities for Arrow data structures
 
-import arrow.scalar as scalar
+import pyarrow.scalar as scalar
 
 
 def array_format(arr, window=None):
diff --git a/python/arrow/includes/__init__.pxd b/python/pyarrow/includes/__init__.pxd
similarity index 100%
rename from python/arrow/includes/__init__.pxd
rename to python/pyarrow/includes/__init__.pxd
diff --git a/python/arrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
similarity index 100%
rename from python/arrow/includes/common.pxd
rename to python/pyarrow/includes/common.pxd
diff --git a/python/arrow/includes/arrow.pxd b/python/pyarrow/includes/libarrow.pxd
similarity index 99%
rename from python/arrow/includes/arrow.pxd
rename to python/pyarrow/includes/libarrow.pxd
index 0cc44c06cb607..baba112833e0d 100644
--- a/python/arrow/includes/arrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -17,7 +17,7 @@
 
 # distutils: language = c++
 
-from arrow.includes.common cimport *
+from pyarrow.includes.common cimport *
 
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
diff --git a/python/arrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
similarity index 97%
rename from python/arrow/includes/parquet.pxd
rename to python/pyarrow/includes/parquet.pxd
index 62342f3066969..99a2d423d9cba 100644
--- a/python/arrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -17,7 +17,7 @@
 
 # distutils: language = c++
 
-from arrow.includes.common cimport *
+from pyarrow.includes.common cimport *
 
 cdef extern from "parquet/api/reader.h" namespace "parquet_cpp" nogil:
     cdef cppclass ColumnReader:
diff --git a/python/arrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
similarity index 90%
rename from python/arrow/includes/pyarrow.pxd
rename to python/pyarrow/includes/pyarrow.pxd
index 3eed5b8542493..9a0c004b7684a 100644
--- a/python/arrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -17,9 +17,9 @@
 
 # distutils: language = c++
 
-from arrow.includes.common cimport *
-from arrow.includes.arrow cimport (CArray, CDataType, LogicalType,
-                                   MemoryPool)
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport (CArray, CDataType, LogicalType,
+                                        MemoryPool)
 
 cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     # We can later add more of the common status factory methods as needed
diff --git a/python/arrow/parquet.pyx b/python/pyarrow/parquet.pyx
similarity index 91%
rename from python/arrow/parquet.pyx
rename to python/pyarrow/parquet.pyx
index 23c3838bcad1f..622e7d0772456 100644
--- a/python/arrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -19,5 +19,5 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from arrow.compat import frombytes, tobytes
-from arrow.includes.parquet cimport *
+from pyarrow.compat import frombytes, tobytes
+from pyarrow.includes.parquet cimport *
diff --git a/python/arrow/scalar.pxd b/python/pyarrow/scalar.pxd
similarity index 93%
rename from python/arrow/scalar.pxd
rename to python/pyarrow/scalar.pxd
index 4e0a3647155a6..b06845718649b 100644
--- a/python/arrow/scalar.pxd
+++ b/python/pyarrow/scalar.pxd
@@ -15,10 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.includes.common cimport *
-from arrow.includes.arrow cimport *
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport *
 
-from arrow.schema cimport DataType
+from pyarrow.schema cimport DataType
 
 cdef class Scalar:
     cdef readonly:
diff --git a/python/arrow/scalar.pyx b/python/pyarrow/scalar.pyx
similarity index 97%
rename from python/arrow/scalar.pyx
rename to python/pyarrow/scalar.pyx
index 72a280e334f4e..261a38967c495 100644
--- a/python/arrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -15,10 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.schema cimport DataType, box_data_type
+from pyarrow.schema cimport DataType, box_data_type
 
-from arrow.compat import frombytes
-import arrow.schema as schema
+from pyarrow.compat import frombytes
+import pyarrow.schema as schema
 
 NA = None
 
diff --git a/python/arrow/schema.pxd b/python/pyarrow/schema.pxd
similarity index 91%
rename from python/arrow/schema.pxd
rename to python/pyarrow/schema.pxd
index 8cc244aaba341..07b9bd04da20e 100644
--- a/python/arrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.includes.common cimport shared_ptr
-from arrow.includes.arrow cimport CDataType, CField, CSchema
+from pyarrow.includes.common cimport shared_ptr
+from pyarrow.includes.libarrow cimport CDataType, CField, CSchema
 
 cdef class DataType:
     cdef:
diff --git a/python/arrow/schema.pyx b/python/pyarrow/schema.pyx
similarity index 97%
rename from python/arrow/schema.pyx
rename to python/pyarrow/schema.pyx
index 3001531eb607d..ea878720d5bb8 100644
--- a/python/arrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -22,9 +22,9 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from arrow.compat import frombytes, tobytes
-from arrow.includes.arrow cimport *
-cimport arrow.includes.pyarrow as pyarrow
+from pyarrow.compat import frombytes, tobytes
+from pyarrow.includes.libarrow cimport *
+cimport pyarrow.includes.pyarrow as pyarrow
 
 cimport cpython
 
diff --git a/python/arrow/tests/__init__.py b/python/pyarrow/tests/__init__.py
similarity index 100%
rename from python/arrow/tests/__init__.py
rename to python/pyarrow/tests/__init__.py
diff --git a/python/arrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
similarity index 80%
rename from python/arrow/tests/test_array.py
rename to python/pyarrow/tests/test_array.py
index ebd872c744e44..034c1576551d3 100644
--- a/python/arrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -15,19 +15,19 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.compat import unittest
-import arrow
-import arrow.formatting as fmt
+from pyarrow.compat import unittest
+import pyarrow
+import pyarrow.formatting as fmt
 
 
 class TestArrayAPI(unittest.TestCase):
 
     def test_getitem_NA(self):
-        arr = arrow.from_pylist([1, None, 2])
-        assert arr[1] is arrow.NA
+        arr = pyarrow.from_pylist([1, None, 2])
+        assert arr[1] is pyarrow.NA
 
     def test_list_format(self):
-        arr = arrow.from_pylist([[1], None, [2, 3]])
+        arr = pyarrow.from_pylist([[1], None, [2, 3]])
         result = fmt.array_format(arr)
         expected = """\
 [
@@ -39,7 +39,7 @@ def test_list_format(self):
         assert result == expected
 
     def test_string_format(self):
-        arr = arrow.from_pylist(['foo', None, 'bar'])
+        arr = pyarrow.from_pylist(['foo', None, 'bar'])
         result = fmt.array_format(arr)
         expected = """\
 [
@@ -50,7 +50,7 @@ def test_string_format(self):
         assert result == expected
 
     def test_long_array_format(self):
-        arr = arrow.from_pylist(range(100))
+        arr = pyarrow.from_pylist(range(100))
         result = fmt.array_format(arr, window=2)
         expected = """\
 [
diff --git a/python/arrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
similarity index 58%
rename from python/arrow/tests/test_convert_builtin.py
rename to python/pyarrow/tests/test_convert_builtin.py
index 57e6ab9f0e7b5..25f696912105d 100644
--- a/python/arrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.compat import unittest
-import arrow
+from pyarrow.compat import unittest
+import pyarrow
 
 
 class TestConvertList(unittest.TestCase):
@@ -25,61 +25,61 @@ def test_boolean(self):
         pass
 
     def test_empty_list(self):
-        arr = arrow.from_pylist([])
+        arr = pyarrow.from_pylist([])
         assert len(arr) == 0
         assert arr.null_count == 0
-        assert arr.type == arrow.null()
+        assert arr.type == pyarrow.null()
 
     def test_all_none(self):
-        arr = arrow.from_pylist([None, None])
+        arr = pyarrow.from_pylist([None, None])
         assert len(arr) == 2
         assert arr.null_count == 2
-        assert arr.type == arrow.null()
+        assert arr.type == pyarrow.null()
 
     def test_integer(self):
-        arr = arrow.from_pylist([1, None, 3, None])
+        arr = pyarrow.from_pylist([1, None, 3, None])
         assert len(arr) == 4
         assert arr.null_count == 2
-        assert arr.type == arrow.int64()
+        assert arr.type == pyarrow.int64()
 
     def test_garbage_collection(self):
         import gc
-        bytes_before = arrow.total_allocated_bytes()
-        arrow.from_pylist([1, None, 3, None])
+        bytes_before = pyarrow.total_allocated_bytes()
+        pyarrow.from_pylist([1, None, 3, None])
         gc.collect()
-        assert arrow.total_allocated_bytes() == bytes_before
+        assert pyarrow.total_allocated_bytes() == bytes_before
 
     def test_double(self):
         data = [1.5, 1, None, 2.5, None, None]
-        arr = arrow.from_pylist(data)
+        arr = pyarrow.from_pylist(data)
         assert len(arr) == 6
         assert arr.null_count == 3
-        assert arr.type == arrow.double()
+        assert arr.type == pyarrow.double()
 
     def test_string(self):
         data = ['foo', b'bar', None, 'arrow']
-        arr = arrow.from_pylist(data)
+        arr = pyarrow.from_pylist(data)
         assert len(arr) == 4
         assert arr.null_count == 1
-        assert arr.type == arrow.string()
+        assert arr.type == pyarrow.string()
 
     def test_mixed_nesting_levels(self):
-        arrow.from_pylist([1, 2, None])
-        arrow.from_pylist([[1], [2], None])
-        arrow.from_pylist([[1], [2], [None]])
+        pyarrow.from_pylist([1, 2, None])
+        pyarrow.from_pylist([[1], [2], None])
+        pyarrow.from_pylist([[1], [2], [None]])
 
-        with self.assertRaises(arrow.ArrowException):
-            arrow.from_pylist([1, 2, [1]])
+        with self.assertRaises(pyarrow.ArrowException):
+            pyarrow.from_pylist([1, 2, [1]])
 
-        with self.assertRaises(arrow.ArrowException):
-            arrow.from_pylist([1, 2, []])
+        with self.assertRaises(pyarrow.ArrowException):
+            pyarrow.from_pylist([1, 2, []])
 
-        with self.assertRaises(arrow.ArrowException):
-            arrow.from_pylist([[1], [2], [None, [1]]])
+        with self.assertRaises(pyarrow.ArrowException):
+            pyarrow.from_pylist([[1], [2], [None, [1]]])
 
     def test_list_of_int(self):
         data = [[1, 2, 3], [], None, [1, 2]]
-        arr = arrow.from_pylist(data)
+        arr = pyarrow.from_pylist(data)
         assert len(arr) == 4
         assert arr.null_count == 1
-        assert arr.type == arrow.list_(arrow.int64())
+        assert arr.type == pyarrow.list_(pyarrow.int64())
diff --git a/python/arrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
similarity index 97%
rename from python/arrow/tests/test_scalars.py
rename to python/pyarrow/tests/test_scalars.py
index 951380bd981e4..021737db6726e 100644
--- a/python/arrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.compat import unittest, u
-import arrow
+from pyarrow.compat import unittest, u
+import pyarrow as arrow
 
 
 class TestScalars(unittest.TestCase):
diff --git a/python/arrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
similarity index 96%
rename from python/arrow/tests/test_schema.py
rename to python/pyarrow/tests/test_schema.py
index a89edd74a0adf..0235526198f35 100644
--- a/python/arrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from arrow.compat import unittest
-import arrow
+from pyarrow.compat import unittest
+import pyarrow as arrow
 
 
 class TestTypes(unittest.TestCase):
diff --git a/python/requirements.txt b/python/requirements.txt
index a82cb20aab86e..f42c90c5c9b3f 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,4 +1,3 @@
 pytest
 numpy>=1.7.0
-pandas>=0.12.0
 six
diff --git a/python/setup.py b/python/setup.py
index eb3ff2a1547d6..5cc871aba9f81 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -27,7 +27,7 @@
 import sys
 
 import pkg_resources
-from setuptools import setup
+from setuptools import setup, Extension
 
 import os
 
@@ -40,10 +40,12 @@
 is_64_bit = sys.maxsize > 2**32
 
 # Check if this is a debug build of Python.
-if hasattr(sys, 'gettotalrefcount'):
-    build_type = 'Debug'
-else:
-    build_type = 'Release'
+# if hasattr(sys, 'gettotalrefcount'):
+#     build_type = 'Debug'
+# else:
+#     build_type = 'Release'
+
+build_type = 'Debug'
 
 if Cython.__version__ < '0.19.1':
     raise Exception('Please upgrade to Cython 0.19.1 or newer')
@@ -51,7 +53,7 @@
 MAJOR = 0
 MINOR = 1
 MICRO = 0
-VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)
+VERSION = '%d.%d.%ddev' % (MAJOR, MINOR, MICRO)
 
 
 class clean(_clean):
@@ -70,6 +72,9 @@ class build_ext(_build_ext):
     def build_extensions(self):
         numpy_incl = pkg_resources.resource_filename('numpy', 'core/include')
 
+        self.extensions = [ext for ext in self.extensions
+                           if ext.name != '__dummy__']
+
         for ext in self.extensions:
             if (hasattr(ext, 'include_dirs') and
                     numpy_incl not in ext.include_dirs):
@@ -98,6 +103,7 @@ def _run_cmake(self):
 
         # The staging directory for the module being built
         build_temp = pjoin(os.getcwd(), self.build_temp)
+        build_lib = os.path.join(os.getcwd(), self.build_lib)
 
         # Change to the build directory
         saved_cwd = os.getcwd()
@@ -124,7 +130,7 @@ def _run_cmake(self):
                              static_lib_option, source]
 
             self.spawn(cmake_command)
-            args = ['make']
+            args = ['make', 'VERBOSE=1']
             if 'PYARROW_PARALLEL' in os.environ:
                 args.append('-j{0}'.format(os.environ['PYARROW_PARALLEL']))
             self.spawn(args)
@@ -150,21 +156,19 @@ def _run_cmake(self):
         if self.inplace:
             # a bit hacky
             build_lib = saved_cwd
-        else:
-            build_lib = pjoin(os.getcwd(), self.build_lib)
 
         # Move the built libpyarrow library to the place expected by the Python
         # build
         if sys.platform != 'win32':
             name, = glob.glob('libpyarrow.*')
             try:
-                os.makedirs(pjoin(build_lib, 'arrow'))
+                os.makedirs(pjoin(build_lib, 'pyarrow'))
             except OSError:
                 pass
-            shutil.move(name, pjoin(build_lib, 'arrow', name))
+            shutil.move(name, pjoin(build_lib, 'pyarrow', name))
         else:
             shutil.move(pjoin(build_type, 'pyarrow.dll'),
-                        pjoin(build_lib, 'arrow', 'pyarrow.dll'))
+                        pjoin(build_lib, 'pyarrow', 'pyarrow.dll'))
 
         # Move the built C-extension to the place expected by the Python build
         self._found_names = []
@@ -192,7 +196,7 @@ def _get_inplace_dir(self):
     def _get_cmake_ext_path(self, name):
         # Get the package directory from build_py
         build_py = self.get_finalized_command('build_py')
-        package_dir = build_py.get_package_dir('arrow')
+        package_dir = build_py.get_package_dir('pyarrow')
         # This is the name of the arrow C-extension
         suffix = sysconfig.get_config_var('EXT_SUFFIX')
         if suffix is None:
@@ -217,23 +221,23 @@ def get_names(self):
 
     def get_outputs(self):
         # Just the C extensions
-        cmake_exts = [self._get_cmake_ext_path(name)
-                      for name in self.get_names()]
-        regular_exts = _build_ext.get_outputs(self)
-        return regular_exts + cmake_exts
+        # regular_exts = _build_ext.get_outputs(self)
+        return [self._get_cmake_ext_path(name)
+                for name in self.get_names()]
 
 
-extensions = []
-
 DESC = """\
 Python library for Apache Arrow"""
 
 setup(
-    name="arrow",
-    packages=['arrow', 'arrow.tests'],
+    name="pyarrow",
+    packages=['pyarrow', 'pyarrow.tests'],
     version=VERSION,
-    package_data={'arrow': ['*.pxd', '*.pyx']},
-    ext_modules=extensions,
+    zip_safe=False,
+    package_data={'pyarrow': ['*.pxd', '*.pyx']},
+    # Dummy extension to trigger build_ext
+    ext_modules=[Extension('__dummy__', sources=[])],
+
     cmdclass={
         'clean': clean,
         'build_ext': build_ext
@@ -243,5 +247,5 @@ def get_outputs(self):
     license='Apache License, Version 2.0',
     maintainer="Apache Arrow Developers",
     maintainer_email="dev@arrow.apache.org",
-    test_suite="arrow.tests"
+    test_suite="pyarrow.tests"
 )
diff --git a/python/src/pyarrow/util/CMakeLists.txt b/python/src/pyarrow/util/CMakeLists.txt
index 3fd8bac31506d..4afb4d0f912b1 100644
--- a/python/src/pyarrow/util/CMakeLists.txt
+++ b/python/src/pyarrow/util/CMakeLists.txt
@@ -19,19 +19,21 @@
 # pyarrow_test_main
 #######################################
 
-add_library(pyarrow_test_main
-  test_main.cc)
+if (PYARROW_BUILD_TESTS)
+  add_library(pyarrow_test_main
+	test_main.cc)
 
-if (APPLE)
-  target_link_libraries(pyarrow_test_main
-    gtest
-    dl)
-  set_target_properties(pyarrow_test_main
-        PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-else()
-  target_link_libraries(pyarrow_test_main
-    gtest
-    pthread
-    dl
-  )
+  if (APPLE)
+	target_link_libraries(pyarrow_test_main
+      gtest
+      dl)
+	set_target_properties(pyarrow_test_main
+      PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  else()
+	target_link_libraries(pyarrow_test_main
+      gtest
+      pthread
+      dl
+	  )
+  endif()
 endif()

From 883c62bddc534df2c0a4ee1e8bef38772aa4a7cd Mon Sep 17 00:00:00 2001
From: Dan Robinson <danrobinson010@gmail.com>
Date: Wed, 16 Mar 2016 15:11:56 -0700
Subject: [PATCH 0032/1644] ARROW-55: [Python] Fix unit tests in 2.7

Fixing the #define check for Python 2 makes all unit tests pass in Python 2.7.

Author: Dan Robinson <danrobinson010@gmail.com>

Closes #25 from danrobinson/ARROW-55 and squashes the following commits:

dda4396 [Dan Robinson] ARROW-55: Add Python 2.7 tests to travis-ci
b00524b [Dan Robinson] ARROW-55: [Python] Fix unit tests in 2.7
---
 ci/travis_script_python.sh  | 35 ++++++++++++++++++++---------------
 python/src/pyarrow/common.h |  2 +-
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 14d66b44ff8ee..af6b0085724fc 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -26,29 +26,34 @@ export PATH="$MINICONDA/bin:$PATH"
 conda update -y -q conda
 conda info -a
 
-PYTHON_VERSION=3.5
-CONDA_ENV_NAME=pyarrow-test
+python_version_tests() {
+  PYTHON_VERSION=$1
+  CONDA_ENV_NAME="pyarrow-test-${PYTHON_VERSION}"
+  conda create -y -q -n $CONDA_ENV_NAME python=$PYTHON_VERSION
+  source activate $CONDA_ENV_NAME
 
-conda create -y -q -n $CONDA_ENV_NAME python=$PYTHON_VERSION
-source activate $CONDA_ENV_NAME
+  python --version
+  which python
 
-python --version
-which python
+  # faster builds, please
+  conda install -y nomkl
 
-# faster builds, please
-conda install -y nomkl
+  # Expensive dependencies install from Continuum package repo
+  conda install -y pip numpy pandas cython
 
-# Expensive dependencies install from Continuum package repo
-conda install -y pip numpy pandas cython
+  # Other stuff pip install
+  pip install -r requirements.txt
 
-# Other stuff pip install
-pip install -r requirements.txt
+  export ARROW_HOME=$ARROW_CPP_INSTALL
 
-export ARROW_HOME=$ARROW_CPP_INSTALL
+  python setup.py build_ext --inplace
 
-python setup.py build_ext --inplace
+  py.test -vv -r sxX pyarrow
+}
 
-py.test -vv -r sxX pyarrow
+# run tests for python 2.7 and 3.5
+python_version_tests 2.7
+python_version_tests 3.5
 
 # if [ $TRAVIS_OS_NAME == "linux" ]; then
 #   valgrind --tool=memcheck py.test -vv -r sxX arrow
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index a43e4d28c899a..db6361384c10d 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -24,7 +24,7 @@ namespace arrow { class MemoryPool; }
 
 namespace pyarrow {
 
-#define PYARROW_IS_PY2 PY_MAJOR_VERSION < 2
+#define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
 
 #define RETURN_ARROW_NOT_OK(s) do {             \
     arrow::Status _s = (s);                     \

From 5881aacefc577ef8a2c39dc40d8f9cd978d50a88 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 16 Mar 2016 15:13:55 -0700
Subject: [PATCH 0033/1644] ARROW-64: Add zsh support to C++ build scripts

All scripts that have to be sourced during development currently only
support bash. This patch adds zsh support.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #24 from xhochy/zsh-support and squashes the following commits:

d3590aa [Uwe L. Korn] ARROW-64: Add zsh support to C++ build scripts
---
 cpp/setup_build_env.sh                | 2 +-
 cpp/thirdparty/build_thirdparty.sh    | 2 +-
 cpp/thirdparty/download_thirdparty.sh | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
index e9901bdbecd42..26a727c87e526 100755
--- a/cpp/setup_build_env.sh
+++ b/cpp/setup_build_env.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-SOURCE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
 ./thirdparty/download_thirdparty.sh
 ./thirdparty/build_thirdparty.sh
diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
index 46794def400eb..8de56a6d08678 100755
--- a/cpp/thirdparty/build_thirdparty.sh
+++ b/cpp/thirdparty/build_thirdparty.sh
@@ -2,7 +2,7 @@
 
 set -x
 set -e
-TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
 source $TP_DIR/versions.sh
 PREFIX=$TP_DIR/installed
diff --git a/cpp/thirdparty/download_thirdparty.sh b/cpp/thirdparty/download_thirdparty.sh
index 8ffb22a93f7e2..0c801179e8d30 100755
--- a/cpp/thirdparty/download_thirdparty.sh
+++ b/cpp/thirdparty/download_thirdparty.sh
@@ -3,7 +3,7 @@
 set -x
 set -e
 
-TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
 source $TP_DIR/versions.sh
 

From c99661069c2f1dbd29c3a86e1e0bd5fa3c6c809f Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Thu, 17 Mar 2016 15:05:24 -0700
Subject: [PATCH 0034/1644] ARROW-68: Better error handling for not fully setup
 systems

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #27 from emkornfield/emk_add_nice_errors_PR and squashes the following commits:

c0b9d78 [Micah Kornfield] ARROW-68: Better error handling for systems missing prerequistites
---
 cpp/setup_build_env.sh                | 4 ++--
 cpp/thirdparty/build_thirdparty.sh    | 9 ++++++---
 cpp/thirdparty/download_thirdparty.sh | 1 +
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
index 26a727c87e526..1a33fe386f103 100755
--- a/cpp/setup_build_env.sh
+++ b/cpp/setup_build_env.sh
@@ -2,8 +2,8 @@
 
 SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
-./thirdparty/download_thirdparty.sh
-./thirdparty/build_thirdparty.sh
+./thirdparty/download_thirdparty.sh || { echo "download_thirdparty.sh failed" ; return; } 
+./thirdparty/build_thirdparty.sh || { echo "build_thirdparty.sh failed" ; return; } 
 source thirdparty/versions.sh
 
 export GTEST_HOME=$SOURCE_DIR/thirdparty/$GTEST_BASEDIR
diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
index 8de56a6d08678..beb248803594c 100755
--- a/cpp/thirdparty/build_thirdparty.sh
+++ b/cpp/thirdparty/build_thirdparty.sh
@@ -44,18 +44,21 @@ ln -sf lib "$PREFIX/lib64"
 # use the compiled tools
 export PATH=$PREFIX/bin:$PATH
 
+type cmake >/dev/null 2>&1 || { echo >&2 "cmake not installed.  Aborting."; exit 1; }
+type make >/dev/null 2>&1 || { echo >&2 "make not installed.  Aborting."; exit 1; }
 
 # build googletest
+GOOGLETEST_ERROR="failed for googletest!"
 if [ -n "$F_ALL" -o -n "$F_GTEST" ]; then
   cd $TP_DIR/$GTEST_BASEDIR
 
   if [[ "$OSTYPE" == "darwin"* ]]; then
-    CXXFLAGS=-fPIC cmake -DCMAKE_CXX_FLAGS="-std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes"
+    CXXFLAGS=-fPIC cmake -DCMAKE_CXX_FLAGS="-std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes" || { echo "cmake $GOOGLETEST_ERROR" ; exit  1; }
   else
-    CXXFLAGS=-fPIC cmake .
+    CXXFLAGS=-fPIC cmake . || { echo "cmake $GOOGLETEST_ERROR"; exit  1; }
   fi
 
-  make VERBOSE=1
+  make VERBOSE=1 || { echo "Make $GOOGLETEST_ERROR" ; exit  1; }
 fi
 
 echo "---------------------"
diff --git a/cpp/thirdparty/download_thirdparty.sh b/cpp/thirdparty/download_thirdparty.sh
index 0c801179e8d30..c18dd4d8e80ab 100755
--- a/cpp/thirdparty/download_thirdparty.sh
+++ b/cpp/thirdparty/download_thirdparty.sh
@@ -8,6 +8,7 @@ TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 source $TP_DIR/versions.sh
 
 download_extract_and_cleanup() {
+	type curl >/dev/null 2>&1 || { echo >&2 "curl not installed.  Aborting."; exit 1; }
 	filename=$TP_DIR/$(basename "$1")
 	curl -#LC - "$1" -o $filename
 	tar xzf $filename -C $TP_DIR

From 3a99f39d64d4e0d6556582c0560140c7b06ee21d Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 21 Mar 2016 16:31:21 -0700
Subject: [PATCH 0035/1644] ARROW-73: Support older CMake versions

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #31 from xhochy/arrow-73 and squashes the following commits:

c92ce5c [Uwe L. Korn] ARROW-73: Support older CMake versions
---
 cpp/cmake_modules/FindGTest.cmake   | 2 +-
 cpp/cmake_modules/FindParquet.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake
index e47faf0dd89d2..3c5d2b67e4494 100644
--- a/cpp/cmake_modules/FindGTest.cmake
+++ b/cpp/cmake_modules/FindGTest.cmake
@@ -54,7 +54,7 @@ endif ()
 
 if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
   set(GTEST_FOUND TRUE)
-  get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} DIRECTORY )
+  get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH )
   set(GTEST_LIB_NAME libgtest)
   set(GTEST_STATIC_LIB ${GTEST_LIBS}/${GTEST_LIB_NAME}.a)
   set(GTEST_SHARED_LIB ${GTEST_LIBS}/${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
diff --git a/cpp/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
index 76c2d1dbee941..d16e6c98f8d1c 100644
--- a/cpp/cmake_modules/FindParquet.cmake
+++ b/cpp/cmake_modules/FindParquet.cmake
@@ -43,7 +43,7 @@ endif ()
 
 if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
   set(PARQUET_FOUND TRUE)
-  get_filename_component( PARQUET_LIBS ${PARQUET_LIBRARIES} DIRECTORY )
+  get_filename_component( PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
   set(PARQUET_LIB_NAME libparquet)
   set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
   set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})

From 016b92bccf60de480da07acbabe876fb695c45e5 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 21 Mar 2016 16:34:07 -0700
Subject: [PATCH 0036/1644] ARROW-72: Search for alternative parquet-cpp header

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #30 from xhochy/arrow-72 and squashes the following commits:

5b6b328 [Uwe L. Korn] ARROW-72: Search for alternative parquet-cpp header
---
 cpp/cmake_modules/FindParquet.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
index d16e6c98f8d1c..e3350d6e13da6 100644
--- a/cpp/cmake_modules/FindParquet.cmake
+++ b/cpp/cmake_modules/FindParquet.cmake
@@ -29,14 +29,14 @@ endif()
 
 # Try the parameterized roots, if they exist
 if ( _parquet_roots )
-    find_path( PARQUET_INCLUDE_DIR NAMES parquet/parquet.h
+    find_path( PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h
         PATHS ${_parquet_roots} NO_DEFAULT_PATH
         PATH_SUFFIXES "include" )
     find_library( PARQUET_LIBRARIES NAMES parquet
         PATHS ${_parquet_roots} NO_DEFAULT_PATH
         PATH_SUFFIXES "lib" )
 else ()
-    find_path( PARQUET_INCLUDE_DIR NAMES parquet/parquet.h )
+    find_path( PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
     find_library( PARQUET_LIBRARIES NAMES parquet )
 endif ()
 

From 4ec034bbe18bd961a4bac64f2e25dba0472c28c9 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Tue, 22 Mar 2016 08:51:23 -0700
Subject: [PATCH 0037/1644] ARROW-28: Adding google's benchmark library to the
 toolchain

This isn't yet complete, but before I go further I think its worth asking some questions on peoples' preferences:

1.  It seems that the build third-party script is setting up an install directory that it is not making use of.   Do we want to keep this functionality and start adding new libraries to be placed there?  The gtest component of the tool-chain assumes it is in its own location, and this how I patterned google benchmark integration.

2.  Do we want to couple unit test builds with benchmark builds?  I am currently aiming for having them decoupled and having benchmarks off by default.

3.  I am not familiar with the Darwin/mac build environment and it is not clear if the CXX flags are required universally.  (I need to fix it anyways to move -DGTEST_USE_OWN_TR1_TUPLE=1 back to be gtest only).  Travis-ci might provide the answer.

4.  Any other basic features in the benchmark toolchain people would like to see as part of this PR?  Wes mentioned starting to create benchmarking tools lib, but I think that likely belongs in a separate PR.

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #29 from emkornfield/emk_add_benchmark and squashes the following commits:

dbd4e71 [Micah Kornfield] only run unittests is travis
ab21150 [Micah Kornfield] Enable benchmarks in cpp toolchain
40847ee [Micah Kornfield] WIP-Adding google's benchmark library to the toolchain
---
 ci/travis_before_script_cpp.sh          |   2 +-
 ci/travis_script_cpp.sh                 |   4 +-
 cpp/CMakeLists.txt                      |  88 ++++++++++++-
 cpp/README.md                           |  23 +++-
 cpp/build-support/run-test.sh           | 160 ++++++++++++++----------
 cpp/cmake_modules/FindGBenchmark.cmake  |  88 +++++++++++++
 cpp/setup_build_env.sh                  |   1 +
 cpp/src/arrow/table/CMakeLists.txt      |   2 +
 cpp/src/arrow/table/column-benchmark.cc |  55 ++++++++
 cpp/src/arrow/util/CMakeLists.txt       |  14 +++
 cpp/src/arrow/util/benchmark_main.cc    |  24 ++++
 cpp/thirdparty/build_thirdparty.sh      |  20 ++-
 cpp/thirdparty/download_thirdparty.sh   |   6 +
 cpp/thirdparty/versions.sh              |   4 +
 14 files changed, 415 insertions(+), 76 deletions(-)
 create mode 100644 cpp/cmake_modules/FindGBenchmark.cmake
 create mode 100644 cpp/src/arrow/table/column-benchmark.cc
 create mode 100644 cpp/src/arrow/util/benchmark_main.cc

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 4d5bef8bbdf70..49dcc395fbc83 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -19,7 +19,7 @@ echo $GTEST_HOME
 
 : ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
 
-cmake -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+cmake -DARROW_BUILD_BENCHMARKS=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
 make -j4
 make install
 
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index 3e843dd759ea1..d96b98f8d37f5 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -9,9 +9,9 @@ pushd $CPP_BUILD_DIR
 make lint
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
-  valgrind --tool=memcheck --leak-check=yes --error-exitcode=1 ctest
+  valgrind --tool=memcheck --leak-check=yes --error-exitcode=1 ctest -L unittest
 else
-  ctest
+  ctest -L unittest
 fi
 
 popd
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f5f6038031127..268c1d11e1e8e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -55,12 +55,21 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(ARROW_BUILD_TESTS
     "Build the Arrow googletest unit tests"
     ON)
+
+  option(ARROW_BUILD_BENCHMARKS
+    "Build the Arrow micro benchmarks"
+    OFF)
+
 endif()
 
 if(NOT ARROW_BUILD_TESTS)
   set(NO_TESTS 1)
 endif()
 
+if(NOT ARROW_BUILD_BENCHMARKS)
+  set(NO_BENCHMARKS 1)
+endif()
+
 
 ############################################################
 # Compiler flags
@@ -251,9 +260,63 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 include_directories(src)
 
 ############################################################
-# Testing
+# Benchmarking 
 ############################################################
+# Add a new micro benchmark, with or without an executable that should be built.
+# If benchmarks are enabled then they will be run along side unit tests with ctest.
+# 'make runbenchmark' and 'make unittest' to build/run only benchmark or unittests, 
+# respectively.
+#
+# REL_BENCHMARK_NAME is the name of the benchmark app. It may be a single component
+# (e.g. monotime-benchmark) or contain additional components (e.g.
+# net/net_util-benchmark). Either way, the last component must be a globally
+# unique name.  
+
+# The benchmark will registered as unit test with ctest with a label
+# of 'benchmark'.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+function(ADD_ARROW_BENCHMARK REL_BENCHMARK_NAME)
+  if(NO_BENCHMARKS)
+    return()
+  endif()
+  get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
+   
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc)
+    # This benchmark has a corresponding .cc file, set it up as an executable.
+    set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}")
+    add_executable(${BENCHMARK_NAME} "${REL_BENCHMARK_NAME}.cc")
+    target_link_libraries(${BENCHMARK_NAME} ${ARROW_BENCHMARK_LINK_LIBS})
+    add_dependencies(runbenchmark ${BENCHMARK_NAME})
+    set(NO_COLOR "--color_print=false")
+  else()
+    # No executable, just invoke the benchmark (probably a script) directly.
+    set(BENCHMARK_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME})
+    set(NO_COLOR "")
+  endif()
+  
+  add_test(${BENCHMARK_NAME}
+    ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} benchmark ${BENCHMARK_PATH} ${NO_COLOR})
+  set_tests_properties(${BENCHMARK_NAME} PROPERTIES LABELS "benchmark")
+  if(ARGN)
+    set_tests_properties(${BENCHMARK_NAME} PROPERTIES ${ARGN})
+  endif()
+endfunction()
+
+# A wrapper for add_dependencies() that is compatible with NO_BENCHMARKS.
+function(ADD_ARROW_BENCHMARK_DEPENDENCIES REL_BENCHMARK_NAME)
+  if(NO_BENCHMARKS)
+    return()
+  endif()
+  get_filename_component(BENCMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
 
+  add_dependencies(${BENCHMARK_NAME} ${ARGN})
+endfunction()
+
+
+############################################################
+# Testing
+############################################################
 # Add a new test case, with or without an executable that should be built.
 #
 # REL_TEST_NAME is the name of the test. It may be a single component
@@ -261,6 +324,9 @@ include_directories(src)
 # net/net_util-test). Either way, the last component must be a globally
 # unique name.
 #
+# The unit test is added with a label of "unittest" to support filtering with
+# ctest.
+#
 # Arguments after the test name will be passed to set_tests_properties().
 function(ADD_ARROW_TEST REL_TEST_NAME)
   if(NO_TESTS)
@@ -273,13 +339,15 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
     set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
     add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
     target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+    add_dependencies(unittest ${TEST_NAME})
   else()
     # No executable, just invoke the test (probably a script) directly.
     set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
   endif()
 
   add_test(${TEST_NAME}
-    ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
+    ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
+  set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
   if(ARGN)
     set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
   endif()
@@ -335,13 +403,28 @@ if ("$ENV{GTEST_HOME}" STREQUAL "")
   set(GTest_HOME ${THIRDPARTY_DIR}/googletest-release-1.7.0)
 endif()
 
+## Google Benchmark 
+if ("$ENV{GBENCHMARK_HOME}" STREQUAL "")
+  set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed)
+endif()
+
+
 if(ARROW_BUILD_TESTS)
+  add_custom_target(unittest ctest -L unittest)
   find_package(GTest REQUIRED)
   include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(gtest
     STATIC_LIB ${GTEST_STATIC_LIB})
 endif()
 
+if(ARROW_BUILD_BENCHMARKS)
+  add_custom_target(runbenchmark ctest -L benchmark)
+  find_package(GBenchmark REQUIRED)
+  include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(benchmark
+    STATIC_LIB ${GBENCHMARK_STATIC_LIB})
+endif()
+
 ## Google PerfTools
 ##
 ## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment
@@ -366,6 +449,7 @@ endif()
 ############################################################
 set(ARROW_MIN_TEST_LIBS arrow arrow_test_main ${ARROW_BASE_LIBS})
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
+set(ARROW_BENCHMARK_LINK_LIBS arrow arrow_benchmark_main ${ARROW_BASE_LIBS})
 
 ############################################################
 # "make ctags" target
diff --git a/cpp/README.md b/cpp/README.md
index 378dc4e28de76..542cce43a1391 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -29,16 +29,29 @@ Simple debug build:
     mkdir debug
     cd debug
     cmake ..
-    make
-    ctest
+    make unittest
 
 Simple release build:
 
     mkdir release
     cd release
     cmake .. -DCMAKE_BUILD_TYPE=Release
-    make
-    ctest
+    make unittest
+
+Detailed unit test logs will be placed in the build directory under `build/test-logs`.
+
+### Building/Running benchmarks
+
+Follow the directions for simple build except run cmake 
+with the `--ARROW_BUILD_BENCHMARKS` parameter set correctly:
+
+    cmake -DARROW_BUILD_BENCHMARKS=ON ..
+
+and instead of make unittest run either `make; ctest` to run both unit tests 
+and benchmarks or `make runbenchmark` to run only the benchmark tests.
+
+Benchmark logs will be placed in the build directory under `build/benchmark-logs`.
+
 
 ### Third-party environment variables
 
@@ -46,3 +59,5 @@ To set up your own specific build toolchain, here are the relevant environment
 variables
 
 * Googletest: `GTEST_HOME` (only required to build the unit tests)
+* Google Benchmark: `GBENCHMARK_HOME` (only required if building benchmarks)
+
diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh
index b2039134d558d..0e628e26ecd52 100755
--- a/cpp/build-support/run-test.sh
+++ b/cpp/build-support/run-test.sh
@@ -16,24 +16,23 @@
 # Script which wraps running a test and redirects its output to a
 # test log directory.
 #
-# If KUDU_COMPRESS_TEST_OUTPUT is non-empty, then the logs will be
-# gzip-compressed while they are written.
+# Arguments:
+#    $1 - Base path for logs/artifacts.
+#    $2 - type of test (e.g. test or benchmark)
+#    $3 - path to executable
+#    $ARGN - arguments for executable
 #
-# If KUDU_FLAKY_TEST_ATTEMPTS is non-zero, and the test being run matches
-# one of the lines in the file KUDU_FLAKY_TEST_LIST, then the test will
-# be retried on failure up to the specified number of times. This can be
-# used in the gerrit workflow to prevent annoying false -1s caused by
-# tests that are known to be flaky in master.
-#
-# If KUDU_REPORT_TEST_RESULTS is non-zero, then tests are reported to the
-# central test server.
 
+OUTPUT_ROOT=$1
+shift
 ROOT=$(cd $(dirname $BASH_SOURCE)/..; pwd)
 
-TEST_LOGDIR=$ROOT/build/test-logs
+TEST_LOGDIR=$OUTPUT_ROOT/build/$1-logs
 mkdir -p $TEST_LOGDIR
 
-TEST_DEBUGDIR=$ROOT/build/test-debug
+RUN_TYPE=$1
+shift
+TEST_DEBUGDIR=$OUTPUT_ROOT/build/$RUN_TYPE-debug
 mkdir -p $TEST_DEBUGDIR
 
 TEST_DIRNAME=$(cd $(dirname $1); pwd)
@@ -43,7 +42,7 @@ TEST_EXECUTABLE="$TEST_DIRNAME/$TEST_FILENAME"
 TEST_NAME=$(echo $TEST_FILENAME | perl -pe 's/\..+?$//') # Remove path and extension (if any).
 
 # We run each test in its own subdir to avoid core file related races.
-TEST_WORKDIR=$ROOT/build/test-work/$TEST_NAME
+TEST_WORKDIR=$OUTPUT_ROOT/build/test-work/$TEST_NAME
 mkdir -p $TEST_WORKDIR
 pushd $TEST_WORKDIR >/dev/null || exit 1
 rm -f *
@@ -61,55 +60,49 @@ rm -f $LOGFILE $LOGFILE.gz
 
 pipe_cmd=cat
 
-# Configure TSAN (ignored if this isn't a TSAN build).
-#
-# Deadlock detection (new in clang 3.5) is disabled because:
-# 1. The clang 3.5 deadlock detector crashes in some unit tests. It
-#    needs compiler-rt commits c4c3dfd, 9a8efe3, and possibly others.
-# 2. Many unit tests report lock-order-inversion warnings; they should be
-#    fixed before reenabling the detector.
-TSAN_OPTIONS="$TSAN_OPTIONS detect_deadlocks=0"
-TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$ROOT/build-support/tsan-suppressions.txt"
-TSAN_OPTIONS="$TSAN_OPTIONS history_size=7"
-export TSAN_OPTIONS
-
-# Enable leak detection even under LLVM 3.4, where it was disabled by default.
-# This flag only takes effect when running an ASAN build.
-ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
-export ASAN_OPTIONS
-
-# Set up suppressions for LeakSanitizer
-LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt"
-export LSAN_OPTIONS
-
-# Suppressions require symbolization. We'll default to using the symbolizer in
-# thirdparty.
-if [ -z "$ASAN_SYMBOLIZER_PATH" ]; then
-  export ASAN_SYMBOLIZER_PATH=$(find $NATIVE_TOOLCHAIN/llvm-3.7.0/bin -name llvm-symbolizer)
-fi
-
 # Allow for collecting core dumps.
 ARROW_TEST_ULIMIT_CORE=${ARROW_TEST_ULIMIT_CORE:-0}
 ulimit -c $ARROW_TEST_ULIMIT_CORE
 
-# Run the actual test.
-for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
-  if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
-    # If the test fails, the test output may or may not be left behind,
-    # depending on whether the test cleaned up or exited immediately. Either
-    # way we need to clean it up. We do this by comparing the data directory
-    # contents before and after the test runs, and deleting anything new.
-    #
-    # The comm program requires that its two inputs be sorted.
-    TEST_TMPDIR_BEFORE=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
+
+function setup_sanitizers() {
+  # Sets environment variables for different sanitizers (it configures how) the run_tests. Function works.
+
+  # Configure TSAN (ignored if this isn't a TSAN build).
+  #
+  # Deadlock detection (new in clang 3.5) is disabled because:
+  # 1. The clang 3.5 deadlock detector crashes in some unit tests. It
+  #    needs compiler-rt commits c4c3dfd, 9a8efe3, and possibly others.
+  # 2. Many unit tests report lock-order-inversion warnings; they should be
+  #    fixed before reenabling the detector.
+  TSAN_OPTIONS="$TSAN_OPTIONS detect_deadlocks=0"
+  TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$ROOT/build-support/tsan-suppressions.txt"
+  TSAN_OPTIONS="$TSAN_OPTIONS history_size=7"
+  export TSAN_OPTIONS
+  
+  # Enable leak detection even under LLVM 3.4, where it was disabled by default.
+  # This flag only takes effect when running an ASAN build.
+  ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
+  export ASAN_OPTIONS
+  
+  # Set up suppressions for LeakSanitizer
+  LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt"
+  export LSAN_OPTIONS
+  
+  # Suppressions require symbolization. We'll default to using the symbolizer in
+  # thirdparty.
+  if [ -z "$ASAN_SYMBOLIZER_PATH" ]; then
+    export ASAN_SYMBOLIZER_PATH=$(find $NATIVE_TOOLCHAIN/llvm-3.7.0/bin -name llvm-symbolizer)
   fi
+}
+
+function run_test() {
+  # Run gtest style tests with sanitizers if they are setup appropriately.
 
   # gtest won't overwrite old junit test files, resulting in a build failure
   # even when retries are successful.
   rm -f $XMLFILE
 
-  echo "Running $TEST_NAME, redirecting output into $LOGFILE" \
-    "(attempt ${ATTEMPT_NUMBER}/$TEST_EXECUTION_ATTEMPTS)"
   $TEST_EXECUTABLE "$@" 2>&1 \
     | $ROOT/build-support/asan_symbolize.py \
     | c++filt \
@@ -131,6 +124,46 @@ for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
     STATUS=1
     rm -f $XMLFILE
   fi
+}
+
+function post_process_tests() {
+  # If we have a LeakSanitizer report, and XML reporting is configured, add a new test
+  # case result to the XML file for the leak report. Otherwise Jenkins won't show
+  # us which tests had LSAN errors.
+  if zgrep --silent "ERROR: LeakSanitizer: detected memory leaks" $LOGFILE ; then
+      echo Test had memory leaks. Editing XML
+      perl -p -i -e '
+      if (m#</testsuite>#) {
+        print "<testcase name=\"LeakSanitizer\" status=\"run\" classname=\"LSAN\">\n";
+        print "  <failure message=\"LeakSanitizer failed\" type=\"\">\n";
+        print "    See txt log file for details\n";
+        print "  </failure>\n";
+        print "</testcase>\n";
+      }' $XMLFILE
+  fi
+}
+
+function run_other() {
+  # Generic run function for test like executables that aren't actually gtest
+  $TEST_EXECUTABLE "$@" 2>&1 | $pipe_cmd > $LOGFILE
+  STATUS=$?
+}
+
+if [ $RUN_TYPE = "test" ]; then
+    setup_sanitizers
+fi
+
+# Run the actual test.
+for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
+  if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
+    # If the test fails, the test output may or may not be left behind,
+    # depending on whether the test cleaned up or exited immediately. Either
+    # way we need to clean it up. We do this by comparing the data directory
+    # contents before and after the test runs, and deleting anything new.
+    #
+    # The comm program requires that its two inputs be sorted.
+    TEST_TMPDIR_BEFORE=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
+  fi
 
   if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
     # Now delete any new test output.
@@ -150,7 +183,13 @@ for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
       fi
     done
   fi
-
+  echo "Running $TEST_NAME, redirecting output into $LOGFILE" \
+    "(attempt ${ATTEMPT_NUMBER}/$TEST_EXECUTION_ATTEMPTS)"
+  if [ $RUN_TYPE = "test" ]; then
+    run_test $*
+  else
+    run_other $*
+  fi
   if [ "$STATUS" -eq "0" ]; then
     break
   elif [ "$ATTEMPT_NUMBER" -lt "$TEST_EXECUTION_ATTEMPTS" ]; then
@@ -159,19 +198,8 @@ for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
   fi
 done
 
-# If we have a LeakSanitizer report, and XML reporting is configured, add a new test
-# case result to the XML file for the leak report. Otherwise Jenkins won't show
-# us which tests had LSAN errors.
-if zgrep --silent "ERROR: LeakSanitizer: detected memory leaks" $LOGFILE ; then
-    echo Test had memory leaks. Editing XML
-    perl -p -i -e '
-    if (m#</testsuite>#) {
-      print "<testcase name=\"LeakSanitizer\" status=\"run\" classname=\"LSAN\">\n";
-      print "  <failure message=\"LeakSanitizer failed\" type=\"\">\n";
-      print "    See txt log file for details\n";
-      print "  </failure>\n";
-      print "</testcase>\n";
-    }' $XMLFILE
+if [ $RUN_TYPE = "test" ]; then	
+  post_process_tests
 fi
 
 # Capture and compress core file and binary.
diff --git a/cpp/cmake_modules/FindGBenchmark.cmake b/cpp/cmake_modules/FindGBenchmark.cmake
new file mode 100644
index 0000000000000..3e46a60f5e68a
--- /dev/null
+++ b/cpp/cmake_modules/FindGBenchmark.cmake
@@ -0,0 +1,88 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find Google benchmark headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(GBenchark)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  GBenchmark_HOME - When set, this path is inspected instead of standard library
+#                    locations as the root of the benchark installation.
+#                    The environment variable GBENCHMARK_HOME overrides this veriable.
+#
+# This module defines
+#  GBENCHMARK_INCLUDE_DIR, directory containing benchmark header directory
+#  GBENCHMARK_LIBS, directory containing benchmark libraries
+#  GBENCHMARK_STATIC_LIB, path to libbenchmark.a
+#  GBENCHMARK_FOUND, whether gbenchmark has been found
+
+if( NOT "$ENV{GBENCHMARK_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "$ENV{GBENCHMARK_HOME}" _native_path )
+    list( APPEND _gbenchmark_roots ${_native_path} )
+elseif ( GBenchmark_HOME )
+    list( APPEND _gbenchmark_roots ${GBenchmark_HOME} )
+endif()
+
+# Try the parameterized roots, if they exist
+if ( _gbenchmark_roots )
+    find_path( GBENCHMARK_INCLUDE_DIR NAMES benchmark/benchmark.h 
+        PATHS ${_gbenchmark_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "include" )
+    find_library( GBENCHMARK_LIBRARIES NAMES benchmark
+        PATHS ${_gbenchmark_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib" )
+else ()
+    find_path( GBENCHMARK_INCLUDE_DIR NAMES benchmark/benchmark.hh )
+    find_library( GBENCHMARK_LIBRARIES NAMES benchmark )
+endif ()
+
+
+if (GBENCHMARK_INCLUDE_DIR AND GBENCHMARK_LIBRARIES)
+  set(GBENCHMARK_FOUND TRUE)
+  get_filename_component( GBENCHMARK_LIBS ${GBENCHMARK_LIBRARIES} PATH )
+  set(GBENCHMARK_LIB_NAME libbenchmark)
+  set(GBENCHMARK_STATIC_LIB ${GBENCHMARK_LIBS}/${GBENCHMARK_LIB_NAME}.a)
+else ()
+  set(GBENCHMARK_FOUND FALSE)
+endif ()
+
+if (GBENCHMARK_FOUND)
+  if (NOT GBenchmark_FIND_QUIETLY)
+    message(STATUS "Found the GBenchmark library: ${GBENCHMARK_LIBRARIES}")
+  endif ()
+else ()
+  if (NOT GBenchmark_FIND_QUIETLY)
+    set(GBENCHMARK_ERR_MSG "Could not find the GBenchmark library. Looked in ")
+    if ( _gbenchmark_roots )
+      set(GBENCHMARK_ERR_MSG "${GBENCHMARK_ERR_MSG} in ${_gbenchmark_roots}.")
+    else ()
+      set(GBENCHMARK_ERR_MSG "${GBENCHMARK_ERR_MSG} system search paths.")
+    endif ()
+    if (GBenchmark_FIND_REQUIRED)
+      message(FATAL_ERROR "${GBENCHMARK_ERR_MSG}")
+    else (GBenchmark_FIND_REQUIRED)
+      message(STATUS "${GBENCHMARK_ERR_MSG}")
+    endif (GBenchmark_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  GBENCHMARK_INCLUDE_DIR
+  GBENCHMARK_LIBS
+  GBENCHMARK_LIBRARIES
+  GBENCHMARK_STATIC_LIB
+)
diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
index 1a33fe386f103..04688e7d59400 100755
--- a/cpp/setup_build_env.sh
+++ b/cpp/setup_build_env.sh
@@ -7,5 +7,6 @@ SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 source thirdparty/versions.sh
 
 export GTEST_HOME=$SOURCE_DIR/thirdparty/$GTEST_BASEDIR
+export GBENCHMARK_HOME=$SOURCE_DIR/thirdparty/installed
 
 echo "Build env initialized"
diff --git a/cpp/src/arrow/table/CMakeLists.txt b/cpp/src/arrow/table/CMakeLists.txt
index 26d843d853bfb..d9f00e74a37db 100644
--- a/cpp/src/arrow/table/CMakeLists.txt
+++ b/cpp/src/arrow/table/CMakeLists.txt
@@ -29,3 +29,5 @@ install(FILES
 ADD_ARROW_TEST(column-test)
 ADD_ARROW_TEST(schema-test)
 ADD_ARROW_TEST(table-test)
+
+ADD_ARROW_BENCHMARK(column-benchmark)
diff --git a/cpp/src/arrow/table/column-benchmark.cc b/cpp/src/arrow/table/column-benchmark.cc
new file mode 100644
index 0000000000000..c01146d7b096f
--- /dev/null
+++ b/cpp/src/arrow/table/column-benchmark.cc
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include "benchmark/benchmark.h"
+
+#include "arrow/test-util.h"
+#include "arrow/table/test-common.h"
+#include "arrow/types/integer.h"
+#include "arrow/util/memory-pool.h"
+
+namespace arrow {
+namespace {
+  template <typename ArrayType>
+  std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
+    auto pool = GetDefaultMemoryPool();
+    auto data = std::make_shared<PoolBuffer>(pool);
+    auto nulls = std::make_shared<PoolBuffer>(pool);
+    data->Resize(length * sizeof(typename ArrayType::value_type));
+    nulls->Resize(util::bytes_for_bits(length));
+    return std::make_shared<ArrayType>(length, data, 10, nulls);
+  }
+}  // anonymous namespace
+
+
+static void BM_BuildInt32ColumnByChunk(benchmark::State& state) { //NOLINT non-const reference
+  ArrayVector arrays;
+  for (int chunk_n = 0; chunk_n < state.range_x(); ++chunk_n) {
+    arrays.push_back(MakePrimitive<Int32Array>(100, 10));
+  }
+  const auto INT32 = std::make_shared<Int32Type>();
+  const auto field = std::make_shared<Field>("c0", INT32);
+  std::unique_ptr<Column> column;
+  while (state.KeepRunning()) {
+    column.reset(new Column(field, arrays));
+  }
+}
+
+BENCHMARK(BM_BuildInt32ColumnByChunk)->Range(5, 50000);
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index d8e2f98f2c85e..fed05e3690c74 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -51,6 +51,20 @@ if (ARROW_BUILD_TESTS)
   endif()
 endif()
 
+if (ARROW_BUILD_BENCHMARKS)
+  add_library(arrow_benchmark_main benchmark_main.cc)
+  if (APPLE)
+    target_link_libraries(arrow_benchmark_main
+      benchmark
+    )
+  else()
+	  target_link_libraries(arrow_benchmark_main
+      benchmark 
+      pthread
+	  )
+  endif()
+endif()
+
 ADD_ARROW_TEST(bit-util-test)
 ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(memory-pool-test)
diff --git a/cpp/src/arrow/util/benchmark_main.cc b/cpp/src/arrow/util/benchmark_main.cc
new file mode 100644
index 0000000000000..c9739af03fb53
--- /dev/null
+++ b/cpp/src/arrow/util/benchmark_main.cc
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+int main(int argc, char** argv) {
+  benchmark::Initialize(&argc, argv);
+  benchmark::RunSpecifiedBenchmarks();
+  return 0;
+}
diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
index beb248803594c..294737cc50522 100755
--- a/cpp/thirdparty/build_thirdparty.sh
+++ b/cpp/thirdparty/build_thirdparty.sh
@@ -16,6 +16,7 @@ else
   for arg in "$*"; do
     case $arg in
       "gtest")      F_GTEST=1 ;;
+      "gbenchmark")      F_GBENCHMARK=1 ;;
       *)            echo "Unknown module: $arg"; exit 1 ;;
     esac
   done
@@ -47,13 +48,15 @@ export PATH=$PREFIX/bin:$PATH
 type cmake >/dev/null 2>&1 || { echo >&2 "cmake not installed.  Aborting."; exit 1; }
 type make >/dev/null 2>&1 || { echo >&2 "make not installed.  Aborting."; exit 1; }
 
+STANDARD_DARWIN_FLAGS="-std=c++11 -stdlib=libc++"
+
 # build googletest
 GOOGLETEST_ERROR="failed for googletest!"
 if [ -n "$F_ALL" -o -n "$F_GTEST" ]; then
   cd $TP_DIR/$GTEST_BASEDIR
 
   if [[ "$OSTYPE" == "darwin"* ]]; then
-    CXXFLAGS=-fPIC cmake -DCMAKE_CXX_FLAGS="-std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes" || { echo "cmake $GOOGLETEST_ERROR" ; exit  1; }
+    CXXFLAGS=-fPIC cmake -DCMAKE_CXX_FLAGS="$STANDARD_DARWIN_FLAGS -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes" || { echo "cmake $GOOGLETEST_ERROR" ; exit  1; }
   else
     CXXFLAGS=-fPIC cmake . || { echo "cmake $GOOGLETEST_ERROR"; exit  1; }
   fi
@@ -61,5 +64,20 @@ if [ -n "$F_ALL" -o -n "$F_GTEST" ]; then
   make VERBOSE=1 || { echo "Make $GOOGLETEST_ERROR" ; exit  1; }
 fi
 
+# build google benchmark
+GBENCHMARK_ERROR="failed for google benchmark"
+if [ -n "$F_ALL" -o -n "$F_GBENCHMARK" ]; then
+  cd $TP_DIR/$GBENCHMARK_BASEDIR
+
+  CMAKE_CXX_FLAGS="--std=c++11"
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    CMAKE_CXX_FLAGS=$STANDARD_DARWIN_FLAGS
+  fi
+  cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_CXX_FLAGS="-fPIC $CMAKE_CXX_FLAGS" . || { echo "cmake $GBENCHMARK_ERROR" ; exit 1; }
+
+  make VERBOSE=1 install || { echo "make $GBENCHMARK_ERROR" ; exit 1; }
+fi
+
+
 echo "---------------------"
 echo "Thirdparty dependencies built and installed into $PREFIX successfully"
diff --git a/cpp/thirdparty/download_thirdparty.sh b/cpp/thirdparty/download_thirdparty.sh
index c18dd4d8e80ab..d22c559b3e3ba 100755
--- a/cpp/thirdparty/download_thirdparty.sh
+++ b/cpp/thirdparty/download_thirdparty.sh
@@ -19,3 +19,9 @@ if [ ! -d ${GTEST_BASEDIR} ]; then
   echo "Fetching gtest"
   download_extract_and_cleanup $GTEST_URL
 fi
+
+echo ${GBENCHMARK_BASEDIR}
+if [ ! -d ${GBENCHMARK_BASEDIR} ]; then
+  echo "Fetching google benchmark"
+  download_extract_and_cleanup $GBENCHMARK_URL
+fi
diff --git a/cpp/thirdparty/versions.sh b/cpp/thirdparty/versions.sh
index 12ad56ef00103..9cfc7cd94b58c 100755
--- a/cpp/thirdparty/versions.sh
+++ b/cpp/thirdparty/versions.sh
@@ -1,3 +1,7 @@
 GTEST_VERSION=1.7.0
 GTEST_URL="https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
 GTEST_BASEDIR=googletest-release-$GTEST_VERSION
+
+GBENCHMARK_VERSION=1.0.0
+GBENCHMARK_URL="https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
+GBENCHMARK_BASEDIR=benchmark-$GBENCHMARK_VERSION

From 093f9bd8c30b1b77b3e6e7a4123cab9a6dd9daa1 Mon Sep 17 00:00:00 2001
From: Dan Robinson <danrobinson010@gmail.com>
Date: Tue, 22 Mar 2016 14:15:38 -0700
Subject: [PATCH 0038/1644] ARROW-75: Fix handling of empty strings

Fixes [ARROW-75](https://issues.apache.org/jira/browse/ARROW-75) (and changes Python tests to verify that behavior).

Author: Dan Robinson <danrobinson010@gmail.com>

Closes #32 from danrobinson/ARROW-75 and squashes the following commits:

cb8e527 [Dan Robinson] ARROW-75: remove whitespace
9604a21 [Dan Robinson] ARROW-75: Changed tests
722df19 [Dan Robinson] ARROW-75: Fixed braces
1ef3b75 [Dan Robinson] ARROW-75: Fix handling of empty strings
---
 cpp/src/arrow/types/primitive.h    | 4 +++-
 cpp/src/arrow/types/string-test.cc | 2 +-
 python/pyarrow/tests/test_array.py | 6 +++---
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 1073bb6e1c340..22ab59c309a1d 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -168,7 +168,9 @@ class PrimitiveBuilder : public ArrayBuilder {
       int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
     }
-    memcpy(raw_buffer() + length_, values, length * elsize_);
+    if (length > 0) {
+      memcpy(raw_buffer() + length_, values, length * elsize_);
+    }
 
     if (null_bytes != nullptr) {
       AppendNulls(null_bytes, length);
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index 8e82fd95dd808..6381093dcbb45 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -181,7 +181,7 @@ class TestStringBuilder : public TestBuilder {
 };
 
 TEST_F(TestStringBuilder, TestScalarAppend) {
-  std::vector<std::string> strings = {"a", "bb", "", "", "ccc"};
+  std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
   std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
 
   int N = strings.size();
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 034c1576551d3..36aaaa4f93d5d 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -39,13 +39,13 @@ def test_list_format(self):
         assert result == expected
 
     def test_string_format(self):
-        arr = pyarrow.from_pylist(['foo', None, 'bar'])
+        arr = pyarrow.from_pylist(['', None, 'foo'])
         result = fmt.array_format(arr)
         expected = """\
 [
-  'foo',
+  '',
   NA,
-  'bar'
+  'foo'
 ]"""
         assert result == expected
 

From 65db0da80b6a1fb6887b7ac1df24e2423d41dfb9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Tue, 22 Mar 2016 18:45:13 -0700
Subject: [PATCH 0039/1644] ARROW-67: C++ metadata flatbuffer serialization and
 data movement to memory maps

Several things here:

* Add Google flatbuffers dependency
* Flatbuffers IDL draft in collaboration with @jacques-n and @stevenmphillips
* Add Schema wrapper in Cython
* arrow::Schema conversion to/from flatbuffer representation
* Remove unneeded physical layout types from type.h
* Refactor ListType to be a nested type with a single child
* Implement shared memory round-trip for numeric row batches
* mmap-based shared memory interface and MemorySource abstract API

Quite a bit of judicious code cleaning and consolidation as part of this. For example, List types are now internally equivalent to a nested type with 1 named child field (versus a struct, which can have any number of child fields).

Associated JIRAs: ARROW-48, ARROW-57, ARROW-58

Author: Wes McKinney <wesm@apache.org>

Closes #28 from wesm/cpp-ipc-draft and squashes the following commits:

0cef7ea [Wes McKinney] Add NullArray type now that Array is virtual, fix pyarrow build
5e841f7 [Wes McKinney] Create explicit PrimitiveArray subclasses to avoid unwanted template instantiation
6fa6319 [Wes McKinney] ARROW-28: Draft C++ shared memory IPC workflow and related refactoring / scaffolding / cleaning.
---
 ci/travis_before_script_cpp.sh                |   9 +-
 ci/travis_script_cpp.sh                       |   6 +-
 cpp/CMakeLists.txt                            |  96 ++++--
 cpp/cmake_modules/FindFlatbuffers.cmake       |  95 ++++++
 cpp/setup_build_env.sh                        |   5 +-
 cpp/src/arrow/CMakeLists.txt                  |   8 +
 cpp/src/arrow/api.h                           |  11 +-
 cpp/src/arrow/array-test.cc                   |  14 +-
 cpp/src/arrow/array.cc                        |  26 +-
 cpp/src/arrow/array.h                         |  27 +-
 cpp/src/arrow/builder.h                       |   2 +-
 cpp/src/arrow/{table => }/column-benchmark.cc |   5 +-
 cpp/src/arrow/{table => }/column-test.cc      |  10 +-
 cpp/src/arrow/{table => }/column.cc           |   4 +-
 cpp/src/arrow/{table => }/column.h            |  13 +-
 cpp/src/arrow/ipc/.gitignore                  |   1 +
 cpp/src/arrow/ipc/CMakeLists.txt              |  51 +++
 cpp/src/arrow/ipc/adapter.cc                  | 305 +++++++++++++++++
 cpp/src/arrow/ipc/adapter.h                   |  86 +++++
 cpp/src/arrow/ipc/ipc-adapter-test.cc         | 112 +++++++
 cpp/src/arrow/ipc/ipc-memory-test.cc          |  82 +++++
 cpp/src/arrow/ipc/ipc-metadata-test.cc        |  99 ++++++
 cpp/src/arrow/ipc/memory.cc                   | 162 +++++++++
 cpp/src/arrow/ipc/memory.h                    | 131 ++++++++
 cpp/src/arrow/ipc/metadata-internal.cc        | 317 ++++++++++++++++++
 cpp/src/arrow/ipc/metadata-internal.h         |  69 ++++
 cpp/src/arrow/ipc/metadata.cc                 | 238 +++++++++++++
 cpp/src/arrow/ipc/metadata.h                  | 146 ++++++++
 .../{types/floating.h => ipc/test-common.h}   |  43 ++-
 cpp/src/arrow/{table => }/schema-test.cc      |  48 ++-
 cpp/src/arrow/{table => }/schema.cc           |  11 +-
 cpp/src/arrow/{table => }/schema.h            |   8 +-
 cpp/src/arrow/{table => }/table-test.cc       |  18 +-
 cpp/src/arrow/{table => }/table.cc            |  35 +-
 cpp/src/arrow/{table => }/table.h             |  58 +++-
 cpp/src/arrow/table/test-common.h             |  54 ---
 cpp/src/arrow/test-util.h                     |  68 +++-
 cpp/src/arrow/type.cc                         |  24 +-
 cpp/src/arrow/type.h                          | 177 ++++------
 cpp/src/arrow/types/CMakeLists.txt            |   2 -
 cpp/src/arrow/types/boolean.h                 |   2 +-
 cpp/src/arrow/types/collection.h              |   2 +-
 cpp/src/arrow/types/construct.cc              |  53 +--
 cpp/src/arrow/types/construct.h               |  11 +-
 cpp/src/arrow/types/datetime.h                |  16 +-
 cpp/src/arrow/types/floating.cc               |  22 --
 cpp/src/arrow/types/integer.cc                |  22 --
 cpp/src/arrow/types/integer.h                 |  57 ----
 cpp/src/arrow/types/json.cc                   |   1 -
 cpp/src/arrow/types/json.h                    |   4 +-
 cpp/src/arrow/types/list-test.cc              |  28 +-
 cpp/src/arrow/types/list.cc                   |  29 ++
 cpp/src/arrow/types/list.h                    |  28 +-
 cpp/src/arrow/types/primitive-test.cc         |  41 +--
 cpp/src/arrow/types/primitive.cc              |  16 +-
 cpp/src/arrow/types/primitive.h               | 102 +++---
 cpp/src/arrow/types/string-test.cc            |  54 ++-
 cpp/src/arrow/types/string.h                  |  55 +--
 cpp/src/arrow/types/struct-test.cc            |  15 +-
 cpp/src/arrow/types/test-common.h             |   5 +-
 cpp/src/arrow/types/union.h                   |  18 +-
 cpp/src/arrow/util/bit-util-test.cc           |   4 +-
 cpp/src/arrow/util/bit-util.h                 |   1 -
 cpp/src/arrow/util/buffer-test.cc             |   3 +-
 cpp/src/arrow/util/buffer.cc                  |   2 +-
 cpp/src/arrow/util/memory-pool-test.cc        |   7 +-
 cpp/src/arrow/util/memory-pool.cc             |   6 +-
 cpp/src/arrow/util/memory-pool.h              |   2 +-
 cpp/src/arrow/util/status.cc                  |   3 +
 cpp/src/arrow/util/status.h                   |   6 +
 cpp/src/arrow/util/test_main.cc               |   2 +-
 cpp/thirdparty/build_thirdparty.sh            |   9 +
 cpp/thirdparty/download_thirdparty.sh         |   5 +
 cpp/thirdparty/versions.sh                    |   4 +
 format/Message.fbs                            | 183 ++++++++++
 python/pyarrow/__init__.py                    |   4 +-
 python/pyarrow/array.pxd                      |   2 +-
 python/pyarrow/array.pyx                      |  47 ++-
 python/pyarrow/includes/libarrow.pxd          | 107 ++++--
 python/pyarrow/includes/pyarrow.pxd           |   5 +-
 python/pyarrow/scalar.pyx                     |  24 +-
 python/pyarrow/schema.pxd                     |   6 +-
 python/pyarrow/schema.pyx                     | 155 ++++++---
 python/pyarrow/tests/test_schema.py           |  28 +-
 .../pyarrow/tests/test_table.py               |  39 ++-
 python/src/pyarrow/adapters/builtin.cc        |  20 +-
 python/src/pyarrow/helpers.cc                 |  15 +-
 python/src/pyarrow/helpers.h                  |   5 +-
 88 files changed, 3113 insertions(+), 838 deletions(-)
 create mode 100644 cpp/cmake_modules/FindFlatbuffers.cmake
 rename cpp/src/arrow/{table => }/column-benchmark.cc (94%)
 rename cpp/src/arrow/{table => }/column-test.cc (93%)
 rename cpp/src/arrow/{table => }/column.cc (96%)
 rename cpp/src/arrow/{table => }/column.h (93%)
 create mode 100644 cpp/src/arrow/ipc/.gitignore
 create mode 100644 cpp/src/arrow/ipc/CMakeLists.txt
 create mode 100644 cpp/src/arrow/ipc/adapter.cc
 create mode 100644 cpp/src/arrow/ipc/adapter.h
 create mode 100644 cpp/src/arrow/ipc/ipc-adapter-test.cc
 create mode 100644 cpp/src/arrow/ipc/ipc-memory-test.cc
 create mode 100644 cpp/src/arrow/ipc/ipc-metadata-test.cc
 create mode 100644 cpp/src/arrow/ipc/memory.cc
 create mode 100644 cpp/src/arrow/ipc/memory.h
 create mode 100644 cpp/src/arrow/ipc/metadata-internal.cc
 create mode 100644 cpp/src/arrow/ipc/metadata-internal.h
 create mode 100644 cpp/src/arrow/ipc/metadata.cc
 create mode 100644 cpp/src/arrow/ipc/metadata.h
 rename cpp/src/arrow/{types/floating.h => ipc/test-common.h} (59%)
 rename cpp/src/arrow/{table => }/schema-test.cc (72%)
 rename cpp/src/arrow/{table => }/schema.cc (88%)
 rename cpp/src/arrow/{table => }/schema.h (91%)
 rename cpp/src/arrow/{table => }/table-test.cc (92%)
 rename cpp/src/arrow/{table => }/table.cc (69%)
 rename cpp/src/arrow/{table => }/table.h (55%)
 delete mode 100644 cpp/src/arrow/table/test-common.h
 delete mode 100644 cpp/src/arrow/types/floating.cc
 delete mode 100644 cpp/src/arrow/types/integer.cc
 delete mode 100644 cpp/src/arrow/types/integer.h
 create mode 100644 format/Message.fbs
 rename cpp/src/arrow/table/CMakeLists.txt => python/pyarrow/tests/test_table.py (58%)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 49dcc395fbc83..193c76feba1d7 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -19,7 +19,14 @@ echo $GTEST_HOME
 
 : ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
 
-cmake -DARROW_BUILD_BENCHMARKS=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+CMAKE_COMMON_FLAGS="-DARROW_BUILD_BENCHMARKS=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
+
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+  cmake -DARROW_TEST_MEMCHECK=on $CMAKE_COMMON_FLAGS -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+else
+  cmake $CMAKE_COMMON_FLAGS -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+fi
+
 make -j4
 make install
 
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index d96b98f8d37f5..997bdf35e83d2 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -8,10 +8,6 @@ pushd $CPP_BUILD_DIR
 
 make lint
 
-if [ $TRAVIS_OS_NAME == "linux" ]; then
-  valgrind --tool=memcheck --leak-check=yes --error-exitcode=1 ctest -L unittest
-else
-  ctest -L unittest
-fi
+ctest -L unittest
 
 popd
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 268c1d11e1e8e..6d701079b482c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -51,7 +51,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(ARROW_PARQUET
     "Build the Parquet adapter and link to libparquet"
     OFF)
-
+  option(ARROW_TEST_MEMCHECK
+	"Run the test suite using valgrind --tool=memcheck"
+	OFF)
   option(ARROW_BUILD_TESTS
     "Build the Arrow googletest unit tests"
     ON)
@@ -60,6 +62,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow micro benchmarks"
     OFF)
 
+  option(ARROW_IPC
+    "Build the Arrow IPC extensions"
+    ON)
+
 endif()
 
 if(NOT ARROW_BUILD_TESTS)
@@ -260,17 +266,17 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 include_directories(src)
 
 ############################################################
-# Benchmarking 
+# Benchmarking
 ############################################################
 # Add a new micro benchmark, with or without an executable that should be built.
 # If benchmarks are enabled then they will be run along side unit tests with ctest.
-# 'make runbenchmark' and 'make unittest' to build/run only benchmark or unittests, 
+# 'make runbenchmark' and 'make unittest' to build/run only benchmark or unittests,
 # respectively.
 #
 # REL_BENCHMARK_NAME is the name of the benchmark app. It may be a single component
 # (e.g. monotime-benchmark) or contain additional components (e.g.
 # net/net_util-benchmark). Either way, the last component must be a globally
-# unique name.  
+# unique name.
 
 # The benchmark will registered as unit test with ctest with a label
 # of 'benchmark'.
@@ -281,7 +287,7 @@ function(ADD_ARROW_BENCHMARK REL_BENCHMARK_NAME)
     return()
   endif()
   get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
-   
+
   if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc)
     # This benchmark has a corresponding .cc file, set it up as an executable.
     set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}")
@@ -294,7 +300,7 @@ function(ADD_ARROW_BENCHMARK REL_BENCHMARK_NAME)
     set(BENCHMARK_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME})
     set(NO_COLOR "")
   endif()
-  
+
   add_test(${BENCHMARK_NAME}
     ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} benchmark ${BENCHMARK_PATH} ${NO_COLOR})
   set_tests_properties(${BENCHMARK_NAME} PROPERTIES LABELS "benchmark")
@@ -345,9 +351,18 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
     set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
   endif()
 
-  add_test(${TEST_NAME}
-    ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
+  if (ARROW_TEST_MEMCHECK)
+	SET_PROPERTY(TARGET ${TEST_NAME}
+	  APPEND_STRING PROPERTY
+	  COMPILE_FLAGS " -DARROW_VALGRIND")
+	add_test(${TEST_NAME}
+	  valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
+  else()
+	add_test(${TEST_NAME}
+      ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
+  endif()
   set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
+
   if(ARGN)
     set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
   endif()
@@ -403,7 +418,7 @@ if ("$ENV{GTEST_HOME}" STREQUAL "")
   set(GTest_HOME ${THIRDPARTY_DIR}/googletest-release-1.7.0)
 endif()
 
-## Google Benchmark 
+## Google Benchmark
 if ("$ENV{GBENCHMARK_HOME}" STREQUAL "")
   set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed)
 endif()
@@ -487,24 +502,10 @@ if (UNIX)
   add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
   --verbose=2
   --linelength=90
-  --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11
-    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h`)
+  --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11,-runtime/references
+    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`)
 endif (UNIX)
 
-#----------------------------------------------------------------------
-# Parquet adapter
-
-if(ARROW_PARQUET)
-  find_package(Parquet REQUIRED)
-  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(parquet
-    STATIC_LIB ${PARQUET_STATIC_LIB}
-    SHARED_LIB ${PARQUET_SHARED_LIB})
-
-  add_subdirectory(src/arrow/parquet)
-  list(APPEND LINK_LIBS arrow_parquet parquet)
-endif()
-
 ############################################################
 # Subdirectories
 ############################################################
@@ -515,15 +516,18 @@ set(LIBARROW_LINK_LIBS
 set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/builder.cc
+  src/arrow/column.cc
+  src/arrow/schema.cc
+  src/arrow/table.cc
   src/arrow/type.cc
 
-  src/arrow/table/column.cc
-  src/arrow/table/schema.cc
-  src/arrow/table/table.cc
+  # IPC / Shared memory library; to be turned into an optional component
+  src/arrow/ipc/adapter.cc
+  src/arrow/ipc/memory.cc
+  src/arrow/ipc/metadata.cc
+  src/arrow/ipc/metadata-internal.cc
 
   src/arrow/types/construct.cc
-  src/arrow/types/floating.cc
-  src/arrow/types/integer.cc
   src/arrow/types/json.cc
   src/arrow/types/list.cc
   src/arrow/types/primitive.cc
@@ -559,9 +563,39 @@ target_link_libraries(arrow ${LIBARROW_LINK_LIBS})
 
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/util)
-add_subdirectory(src/arrow/table)
 add_subdirectory(src/arrow/types)
 
 install(TARGETS arrow
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
+
+#----------------------------------------------------------------------
+# Parquet adapter library
+
+if(ARROW_PARQUET)
+  find_package(Parquet REQUIRED)
+  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(parquet
+    STATIC_LIB ${PARQUET_STATIC_LIB}
+    SHARED_LIB ${PARQUET_SHARED_LIB})
+
+  add_subdirectory(src/arrow/parquet)
+  list(APPEND LINK_LIBS arrow_parquet parquet)
+endif()
+
+#----------------------------------------------------------------------
+# IPC library
+
+## Flatbuffers
+if(ARROW_IPC)
+  find_package(Flatbuffers REQUIRED)
+  message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
+  message(STATUS "Flatbuffers static library: ${FLATBUFFERS_STATIC_LIB}")
+  message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
+  include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
+  add_library(flatbuffers STATIC IMPORTED)
+  set_target_properties(flatbuffers PROPERTIES
+	IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB})
+
+  add_subdirectory(src/arrow/ipc)
+endif()
diff --git a/cpp/cmake_modules/FindFlatbuffers.cmake b/cpp/cmake_modules/FindFlatbuffers.cmake
new file mode 100644
index 0000000000000..ee472d1c8995f
--- /dev/null
+++ b/cpp/cmake_modules/FindFlatbuffers.cmake
@@ -0,0 +1,95 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find Flatbuffers headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(Flatbuffers)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  Flatbuffers_HOME -
+#   When set, this path is inspected instead of standard library locations as
+#   the root of the Flatbuffers installation.  The environment variable
+#   FLATBUFFERS_HOME overrides this veriable.
+#
+# This module defines
+#  FLATBUFFERS_INCLUDE_DIR, directory containing headers
+#  FLATBUFFERS_LIBS, directory containing flatbuffers libraries
+#  FLATBUFFERS_STATIC_LIB, path to libflatbuffers.a
+#  FLATBUFFERS_FOUND, whether flatbuffers has been found
+
+if( NOT "$ENV{FLATBUFFERS_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "$ENV{FLATBUFFERS_HOME}" _native_path )
+    list( APPEND _flatbuffers_roots ${_native_path} )
+elseif ( Flatbuffers_HOME )
+    list( APPEND _flatbuffers_roots ${Flatbuffers_HOME} )
+endif()
+
+# Try the parameterized roots, if they exist
+if ( _flatbuffers_roots )
+    find_path( FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h
+        PATHS ${_flatbuffers_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "include" )
+    find_library( FLATBUFFERS_LIBRARIES NAMES flatbuffers
+        PATHS ${_flatbuffers_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib" )
+else ()
+    find_path( FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h )
+    find_library( FLATBUFFERS_LIBRARIES NAMES flatbuffers )
+endif ()
+
+find_program(FLATBUFFERS_COMPILER flatc
+  $ENV{FLATBUFFERS_HOME}/bin
+  /usr/local/bin
+  /usr/bin
+  NO_DEFAULT_PATH
+)
+
+if (FLATBUFFERS_INCLUDE_DIR AND FLATBUFFERS_LIBRARIES)
+  set(FLATBUFFERS_FOUND TRUE)
+  get_filename_component( FLATBUFFERS_LIBS ${FLATBUFFERS_LIBRARIES} PATH )
+  set(FLATBUFFERS_LIB_NAME libflatbuffers)
+  set(FLATBUFFERS_STATIC_LIB ${FLATBUFFERS_LIBS}/${FLATBUFFERS_LIB_NAME}.a)
+else ()
+  set(FLATBUFFERS_FOUND FALSE)
+endif ()
+
+if (FLATBUFFERS_FOUND)
+  if (NOT Flatbuffers_FIND_QUIETLY)
+    message(STATUS "Found the Flatbuffers library: ${FLATBUFFERS_LIBRARIES}")
+  endif ()
+else ()
+  if (NOT Flatbuffers_FIND_QUIETLY)
+    set(FLATBUFFERS_ERR_MSG "Could not find the Flatbuffers library. Looked in ")
+    if ( _flatbuffers_roots )
+      set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} in ${_flatbuffers_roots}.")
+    else ()
+      set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} system search paths.")
+    endif ()
+    if (Flatbuffers_FIND_REQUIRED)
+      message(FATAL_ERROR "${FLATBUFFERS_ERR_MSG}")
+    else (Flatbuffers_FIND_REQUIRED)
+      message(STATUS "${FLATBUFFERS_ERR_MSG}")
+    endif (Flatbuffers_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  FLATBUFFERS_INCLUDE_DIR
+  FLATBUFFERS_LIBS
+  FLATBUFFERS_STATIC_LIB
+  FLATBUFFERS_COMPILER
+)
diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
index 04688e7d59400..6520dbd43f705 100755
--- a/cpp/setup_build_env.sh
+++ b/cpp/setup_build_env.sh
@@ -2,11 +2,12 @@
 
 SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
-./thirdparty/download_thirdparty.sh || { echo "download_thirdparty.sh failed" ; return; } 
-./thirdparty/build_thirdparty.sh || { echo "build_thirdparty.sh failed" ; return; } 
+./thirdparty/download_thirdparty.sh || { echo "download_thirdparty.sh failed" ; return; }
+./thirdparty/build_thirdparty.sh || { echo "build_thirdparty.sh failed" ; return; }
 source thirdparty/versions.sh
 
 export GTEST_HOME=$SOURCE_DIR/thirdparty/$GTEST_BASEDIR
 export GBENCHMARK_HOME=$SOURCE_DIR/thirdparty/installed
+export FLATBUFFERS_HOME=$SOURCE_DIR/thirdparty/installed
 
 echo "Build env initialized"
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 73e6a9b22c94a..2d42edcfbd499 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -19,7 +19,10 @@
 install(FILES
   api.h
   array.h
+  column.h
   builder.h
+  schema.h
+  table.h
   type.h
   DESTINATION include/arrow)
 
@@ -30,3 +33,8 @@ install(FILES
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 ADD_ARROW_TEST(array-test)
+ADD_ARROW_TEST(column-test)
+ADD_ARROW_TEST(schema-test)
+ADD_ARROW_TEST(table-test)
+
+ADD_ARROW_BENCHMARK(column-benchmark)
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index c73d4b386cf54..7be7f88c22eb6 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -22,20 +22,19 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/column.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
 #include "arrow/type.h"
 
-#include "arrow/table/column.h"
-#include "arrow/table/schema.h"
-#include "arrow/table/table.h"
-
 #include "arrow/types/boolean.h"
 #include "arrow/types/construct.h"
-#include "arrow/types/floating.h"
-#include "arrow/types/integer.h"
 #include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
 #include "arrow/types/struct.h"
 
+#include "arrow/util/buffer.h"
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index df827aaa113aa..eded5941e892e 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -15,30 +15,26 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-
 #include <cstdint>
 #include <cstdlib>
 #include <memory>
 #include <vector>
 
+#include "gtest/gtest.h"
+
 #include "arrow/array.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/integer.h"
 #include "arrow/types/primitive.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
-static TypePtr int32 = TypePtr(new Int32Type());
-
 class TestArray : public ::testing::Test {
  public:
   void SetUp() {
-    pool_ = GetDefaultMemoryPool();
+    pool_ = default_memory_pool();
   }
 
  protected:
@@ -75,10 +71,10 @@ TEST_F(TestArray, TestIsNull) {
     if (x > 0) ++null_count;
   }
 
-  std::shared_ptr<Buffer> null_buf = bytes_to_null_buffer(nulls.data(),
+  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(nulls.data(),
       nulls.size());
   std::unique_ptr<Array> arr;
-  arr.reset(new Array(int32, nulls.size(), null_count, null_buf));
+  arr.reset(new Int32Array(nulls.size(), nullptr, null_count, null_buf));
 
   ASSERT_EQ(null_count, arr->null_count());
   ASSERT_EQ(5, null_buf->size());
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index ee4ef66d11e26..5a5bc1069db13 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -28,11 +28,6 @@ namespace arrow {
 
 Array::Array(const TypePtr& type, int32_t length, int32_t null_count,
     const std::shared_ptr<Buffer>& nulls) {
-  Init(type, length, null_count, nulls);
-}
-
-void Array::Init(const TypePtr& type, int32_t length, int32_t null_count,
-    const std::shared_ptr<Buffer>& nulls) {
   type_ = type;
   length_ = length;
   null_count_ = null_count;
@@ -42,4 +37,25 @@ void Array::Init(const TypePtr& type, int32_t length, int32_t null_count,
   }
 }
 
+bool Array::EqualsExact(const Array& other) const {
+  if (this == &other) return true;
+  if (length_ != other.length_ || null_count_ != other.null_count_ ||
+      type_enum() != other.type_enum()) {
+    return false;
+  }
+  if (null_count_ > 0) {
+    return nulls_->Equals(*other.nulls_, util::bytes_for_bits(length_));
+  } else {
+    return true;
+  }
+}
+
+bool NullArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) return true;
+  if (Type::NA != arr->type_enum()) {
+    return false;
+  }
+  return arr->length() == length_;
+}
+
 } // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 85e853e2ae5e2..65fc0aaf583e9 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -40,20 +40,11 @@ class Buffer;
 // explicitly increment its reference count
 class Array {
  public:
-  Array() :
-      null_count_(0),
-      length_(0),
-      nulls_(nullptr),
-      null_bits_(nullptr) {}
-
   Array(const TypePtr& type, int32_t length, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr);
 
   virtual ~Array() {}
 
-  void Init(const TypePtr& type, int32_t length, int32_t null_count,
-      const std::shared_ptr<Buffer>& nulls);
-
   // Determine if a slot is null. For inner loops. Does *not* boundscheck
   bool IsNull(int i) const {
     return null_count_ > 0 && util::get_bit(null_bits_, i);
@@ -63,12 +54,15 @@ class Array {
   int32_t null_count() const { return null_count_;}
 
   const std::shared_ptr<DataType>& type() const { return type_;}
-  LogicalType::type logical_type() const { return type_->type;}
+  Type::type type_enum() const { return type_->type;}
 
   const std::shared_ptr<Buffer>& nulls() const {
     return nulls_;
   }
 
+  bool EqualsExact(const Array& arr) const;
+  virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
+
  protected:
   TypePtr type_;
   int32_t null_count_;
@@ -78,9 +72,22 @@ class Array {
   const uint8_t* null_bits_;
 
  private:
+  Array() {}
   DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
+// Degenerate null type Array
+class NullArray : public Array {
+ public:
+  NullArray(const std::shared_ptr<DataType>& type, int32_t length) :
+      Array(type, length, length, nullptr) {}
+
+  explicit NullArray(int32_t length) :
+      NullArray(std::make_shared<NullType>(), length) {}
+
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+};
+
 typedef std::shared_ptr<Array> ArrayPtr;
 
 } // namespace arrow
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 8cc689c3e81ee..d5d1fdf95af17 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -99,7 +99,7 @@ class ArrayBuilder {
   int32_t capacity_;
 
   // Child value array builders. These are owned by this class
-  std::vector<std::unique_ptr<ArrayBuilder> > children_;
+  std::vector<std::unique_ptr<ArrayBuilder>> children_;
 
  private:
   DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
diff --git a/cpp/src/arrow/table/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
similarity index 94%
rename from cpp/src/arrow/table/column-benchmark.cc
rename to cpp/src/arrow/column-benchmark.cc
index c01146d7b096f..69ee52c3e09ea 100644
--- a/cpp/src/arrow/table/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -19,15 +19,14 @@
 #include "benchmark/benchmark.h"
 
 #include "arrow/test-util.h"
-#include "arrow/table/test-common.h"
-#include "arrow/types/integer.h"
+#include "arrow/types/primitive.h"
 #include "arrow/util/memory-pool.h"
 
 namespace arrow {
 namespace {
   template <typename ArrayType>
   std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
-    auto pool = GetDefaultMemoryPool();
+    auto pool = default_memory_pool();
     auto data = std::make_shared<PoolBuffer>(pool);
     auto nulls = std::make_shared<PoolBuffer>(pool);
     data->Resize(length * sizeof(typename ArrayType::value_type));
diff --git a/cpp/src/arrow/table/column-test.cc b/cpp/src/arrow/column-test.cc
similarity index 93%
rename from cpp/src/arrow/table/column-test.cc
rename to cpp/src/arrow/column-test.cc
index 3b102e48c87cf..0630785630e81 100644
--- a/cpp/src/arrow/table/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -15,18 +15,18 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
 #include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
-#include "arrow/table/column.h"
-#include "arrow/table/schema.h"
-#include "arrow/table/test-common.h"
+#include "gtest/gtest.h"
+
+#include "arrow/column.h"
+#include "arrow/schema.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/integer.h"
+#include "arrow/types/primitive.h"
 
 using std::shared_ptr;
 using std::vector;
diff --git a/cpp/src/arrow/table/column.cc b/cpp/src/arrow/column.cc
similarity index 96%
rename from cpp/src/arrow/table/column.cc
rename to cpp/src/arrow/column.cc
index 573e650875944..46acf8df2ff57 100644
--- a/cpp/src/arrow/table/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -15,11 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/table/column.h"
+#include "arrow/column.h"
 
 #include <memory>
 #include <sstream>
 
+#include "arrow/array.h"
 #include "arrow/type.h"
 #include "arrow/util/status.h"
 
@@ -28,6 +29,7 @@ namespace arrow {
 ChunkedArray::ChunkedArray(const ArrayVector& chunks) :
     chunks_(chunks) {
   length_ = 0;
+  null_count_ = 0;
   for (const std::shared_ptr<Array>& chunk : chunks) {
     length_ += chunk->length();
     null_count_ += chunk->null_count();
diff --git a/cpp/src/arrow/table/column.h b/cpp/src/arrow/column.h
similarity index 93%
rename from cpp/src/arrow/table/column.h
rename to cpp/src/arrow/column.h
index dfc7516e26aac..1ad97b20863c8 100644
--- a/cpp/src/arrow/table/column.h
+++ b/cpp/src/arrow/column.h
@@ -15,19 +15,22 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef ARROW_TABLE_COLUMN_H
-#define ARROW_TABLE_COLUMN_H
+#ifndef ARROW_COLUMN_H
+#define ARROW_COLUMN_H
 
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
-#include "arrow/array.h"
 #include "arrow/type.h"
 
 namespace arrow {
 
-typedef std::vector<std::shared_ptr<Array> > ArrayVector;
+class Array;
+class Status;
+
+typedef std::vector<std::shared_ptr<Array>> ArrayVector;
 
 // A data structure managing a list of primitive Arrow arrays logically as one
 // large array
@@ -102,4 +105,4 @@ class Column {
 
 } // namespace arrow
 
-#endif  // ARROW_TABLE_COLUMN_H
+#endif  // ARROW_COLUMN_H
diff --git a/cpp/src/arrow/ipc/.gitignore b/cpp/src/arrow/ipc/.gitignore
new file mode 100644
index 0000000000000..8150d7efe33c4
--- /dev/null
+++ b/cpp/src/arrow/ipc/.gitignore
@@ -0,0 +1 @@
+*_generated.h
\ No newline at end of file
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
new file mode 100644
index 0000000000000..383684f42f952
--- /dev/null
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######################################
+# arrow_ipc
+#######################################
+
+# Headers: top level
+install(FILES
+  adapter.h
+  metadata.h
+  memory.h
+  DESTINATION include/arrow/ipc)
+
+ADD_ARROW_TEST(ipc-adapter-test)
+ADD_ARROW_TEST(ipc-memory-test)
+ADD_ARROW_TEST(ipc-metadata-test)
+
+# make clean will delete the generated file
+set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
+
+set(OUTPUT_DIR ${CMAKE_SOURCE_DIR}/src/arrow/ipc)
+set(FBS_OUTPUT_FILES "${OUTPUT_DIR}/Message_generated.h")
+
+set(FBS_SRC ${CMAKE_SOURCE_DIR}/../format/Message.fbs)
+get_filename_component(ABS_FBS_SRC ${FBS_SRC} ABSOLUTE)
+
+add_custom_command(
+  OUTPUT ${FBS_OUTPUT_FILES}
+  COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC}
+  DEPENDS ${ABS_FBS_SRC}
+  COMMENT "Running flatc compiler on ${FBS_SRC}"
+  VERBATIM
+)
+
+add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES})
+add_dependencies(arrow metadata_fbs)
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
new file mode 100644
index 0000000000000..7cdb965f5f45c
--- /dev/null
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -0,0 +1,305 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/adapter.h"
+
+#include <cstdint>
+#include <cstring>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/ipc/memory.h"
+#include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/metadata-internal.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
+#include "arrow/type.h"
+#include "arrow/types/construct.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+namespace flatbuf = apache::arrow::flatbuf;
+
+namespace ipc {
+
+static bool IsPrimitive(const DataType* type) {
+  switch (type->type) {
+    // NA is null type or "no type", considered primitive for now
+    case Type::NA:
+    case Type::BOOL:
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::UINT64:
+    case Type::INT64:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+      return true;
+    default:
+      return false;
+  }
+}
+
+// ----------------------------------------------------------------------
+// Row batch write path
+
+Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes,
+    std::vector<std::shared_ptr<Buffer>>* buffers) {
+  if (IsPrimitive(arr->type().get())) {
+    const PrimitiveArray* prim_arr = static_cast<const PrimitiveArray*>(arr);
+
+    field_nodes->push_back(
+        flatbuf::FieldNode(prim_arr->length(), prim_arr->null_count()));
+
+    if (prim_arr->null_count() > 0) {
+      buffers->push_back(prim_arr->nulls());
+    } else {
+      // Push a dummy zero-length buffer, not to be copied
+      buffers->push_back(std::make_shared<Buffer>(nullptr, 0));
+    }
+    buffers->push_back(prim_arr->data());
+  } else if (arr->type_enum() == Type::LIST) {
+    // TODO(wesm)
+    return Status::NotImplemented("List type");
+  } else if (arr->type_enum() == Type::STRUCT) {
+    // TODO(wesm)
+    return Status::NotImplemented("Struct type");
+  }
+
+  return Status::OK();
+}
+
+class RowBatchWriter {
+ public:
+  explicit RowBatchWriter(const RowBatch* batch) :
+      batch_(batch) {}
+
+  Status AssemblePayload() {
+    // Perform depth-first traversal of the row-batch
+    for (int i = 0; i < batch_->num_columns(); ++i) {
+      const Array* arr = batch_->column(i).get();
+      RETURN_NOT_OK(VisitArray(arr, &field_nodes_, &buffers_));
+    }
+    return Status::OK();
+  }
+
+  Status Write(MemorySource* dst, int64_t position, int64_t* data_header_offset) {
+    // Write out all the buffers contiguously and compute the total size of the
+    // memory payload
+    int64_t offset = 0;
+    for (size_t i = 0; i < buffers_.size(); ++i) {
+      const Buffer* buffer = buffers_[i].get();
+      int64_t size = buffer->size();
+
+      // TODO(wesm): We currently have no notion of shared memory page id's,
+      // but we've included it in the metadata IDL for when we have it in the
+      // future. Use page=0 for now
+      //
+      // Note that page ids are a bespoke notion for Arrow and not a feature we
+      // are using from any OS-level shared memory. The thought is that systems
+      // may (in the future) associate integer page id's with physical memory
+      // pages (according to whatever is the desired shared memory mechanism)
+      buffer_meta_.push_back(flatbuf::Buffer(0, position + offset, size));
+
+      if (size > 0) {
+        RETURN_NOT_OK(dst->Write(position + offset, buffer->data(), size));
+        offset += size;
+      }
+    }
+
+    // Now that we have computed the locations of all of the buffers in shared
+    // memory, the data header can be converted to a flatbuffer and written out
+    //
+    // Note: The memory written here is prefixed by the size of the flatbuffer
+    // itself as an int32_t. On reading from a MemorySource, you will have to
+    // determine the data header size then request a buffer such that you can
+    // construct the flatbuffer data accessor object (see arrow::ipc::Message)
+    std::shared_ptr<Buffer> data_header;
+    RETURN_NOT_OK(WriteDataHeader(batch_->num_rows(), offset,
+            field_nodes_, buffer_meta_, &data_header));
+
+    // Write the data header at the end
+    RETURN_NOT_OK(dst->Write(position + offset, data_header->data(),
+            data_header->size()));
+
+    *data_header_offset = position + offset;
+    return Status::OK();
+  }
+
+  // This must be called after invoking AssemblePayload
+  int64_t DataHeaderSize() {
+    // TODO(wesm): In case it is needed, compute the upper bound for the size
+    // of the buffer containing the flatbuffer data header.
+    return 0;
+  }
+
+  // Total footprint of buffers. This must be called after invoking
+  // AssemblePayload
+  int64_t TotalBytes() {
+    int64_t total = 0;
+    for (const std::shared_ptr<Buffer>& buffer : buffers_) {
+      total += buffer->size();
+    }
+    return total;
+  }
+
+ private:
+  const RowBatch* batch_;
+
+  std::vector<flatbuf::FieldNode> field_nodes_;
+  std::vector<flatbuf::Buffer> buffer_meta_;
+  std::vector<std::shared_ptr<Buffer>> buffers_;
+};
+
+Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
+    int64_t* header_offset) {
+  RowBatchWriter serializer(batch);
+  RETURN_NOT_OK(serializer.AssemblePayload());
+  return serializer.Write(dst, position, header_offset);
+}
+// ----------------------------------------------------------------------
+// Row batch read path
+
+static constexpr int64_t INIT_METADATA_SIZE = 4096;
+
+class RowBatchReader::Impl {
+ public:
+  Impl(MemorySource* source, const std::shared_ptr<RecordBatchMessage>& metadata) :
+      source_(source),
+      metadata_(metadata) {
+    num_buffers_ = metadata->num_buffers();
+    num_flattened_fields_ = metadata->num_fields();
+  }
+
+  Status AssembleBatch(const std::shared_ptr<Schema>& schema,
+      std::shared_ptr<RowBatch>* out) {
+    std::vector<std::shared_ptr<Array>> arrays(schema->num_fields());
+
+    // The field_index and buffer_index are incremented in NextArray based on
+    // how much of the batch is "consumed" (through nested data reconstruction,
+    // for example)
+    field_index_ = 0;
+    buffer_index_ = 0;
+    for (int i = 0; i < schema->num_fields(); ++i) {
+      const Field* field = schema->field(i).get();
+      RETURN_NOT_OK(NextArray(field, &arrays[i]));
+    }
+
+    *out = std::make_shared<RowBatch>(schema, metadata_->length(),
+        arrays);
+    return Status::OK();
+  }
+
+ private:
+  // Traverse the flattened record batch metadata and reassemble the
+  // corresponding array containers
+  Status NextArray(const Field* field, std::shared_ptr<Array>* out) {
+    const std::shared_ptr<DataType>& type = field->type;
+
+    // pop off a field
+    if (field_index_ >= num_flattened_fields_) {
+      return Status::Invalid("Ran out of field metadata, likely malformed");
+    }
+
+    // This only contains the length and null count, which we need to figure
+    // out what to do with the buffers. For example, if null_count == 0, then
+    // we can skip that buffer without reading from shared memory
+    FieldMetadata field_meta = metadata_->field(field_index_++);
+
+    if (IsPrimitive(type.get())) {
+      std::shared_ptr<Buffer> nulls;
+      std::shared_ptr<Buffer> data;
+      if (field_meta.null_count == 0) {
+        nulls = nullptr;
+        ++buffer_index_;
+      } else {
+        RETURN_NOT_OK(GetBuffer(buffer_index_++, &nulls));
+      }
+      if (field_meta.length > 0) {
+        RETURN_NOT_OK(GetBuffer(buffer_index_++, &data));
+      } else {
+        data.reset(new Buffer(nullptr, 0));
+      }
+      return MakePrimitiveArray(type, field_meta.length, data,
+          field_meta.null_count, nulls, out);
+    } else {
+      return Status::NotImplemented("Non-primitive types not complete yet");
+    }
+  }
+
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
+    BufferMetadata metadata = metadata_->buffer(buffer_index);
+    return source_->ReadAt(metadata.offset, metadata.length, out);
+  }
+
+  MemorySource* source_;
+  std::shared_ptr<RecordBatchMessage> metadata_;
+
+  int field_index_;
+  int buffer_index_;
+  int num_buffers_;
+  int num_flattened_fields_;
+};
+
+Status RowBatchReader::Open(MemorySource* source, int64_t position,
+    std::shared_ptr<RowBatchReader>* out) {
+  std::shared_ptr<Buffer> metadata;
+  RETURN_NOT_OK(source->ReadAt(position, INIT_METADATA_SIZE, &metadata));
+
+  int32_t metadata_size = *reinterpret_cast<const int32_t*>(metadata->data());
+
+  // We may not need to call source->ReadAt again
+  if (metadata_size > static_cast<int>(INIT_METADATA_SIZE - sizeof(int32_t))) {
+    // We don't have enough data, read the indicated metadata size.
+    RETURN_NOT_OK(source->ReadAt(position + sizeof(int32_t),
+            metadata_size, &metadata));
+  }
+
+  // TODO(wesm): buffer slicing here would be better in case ReadAt returns
+  // allocated memory
+
+  std::shared_ptr<Message> message;
+  RETURN_NOT_OK(Message::Open(metadata, &message));
+
+  if (message->type() != Message::RECORD_BATCH) {
+    return Status::Invalid("Metadata message is not a record batch");
+  }
+
+  std::shared_ptr<RecordBatchMessage> batch_meta = message->GetRecordBatch();
+
+  std::shared_ptr<RowBatchReader> result(new RowBatchReader());
+  result->impl_.reset(new Impl(source, batch_meta));
+  *out = result;
+
+  return Status::OK();
+}
+
+Status RowBatchReader::GetRowBatch(const std::shared_ptr<Schema>& schema,
+    std::shared_ptr<RowBatch>* out) {
+  return impl_->AssembleBatch(schema, out);
+}
+
+
+} // namespace ipc
+} // namespace arrow
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
new file mode 100644
index 0000000000000..26dea6d04b889
--- /dev/null
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Public API for writing and accessing (with zero copy, if possible) Arrow
+// data in shared memory
+
+#ifndef ARROW_IPC_ADAPTER_H
+#define ARROW_IPC_ADAPTER_H
+
+#include <cstdint>
+#include <memory>
+
+namespace arrow {
+
+class Array;
+class RowBatch;
+class Schema;
+class Status;
+
+namespace ipc {
+
+class MemorySource;
+class RecordBatchMessage;
+
+// ----------------------------------------------------------------------
+// Write path
+
+// Write the RowBatch (collection of equal-length Arrow arrays) to the memory
+// source at the indicated position
+//
+// First, each of the memory buffers are written out end-to-end in starting at
+// the indicated position.
+//
+// Then, this function writes the batch metadata as a flatbuffer (see
+// format/Message.fbs -- the RecordBatch message type) like so:
+//
+// <int32: metadata size> <uint8*: metadata>
+//
+// Finally, the memory offset to the start of the metadata / data header is
+// returned in an out-variable
+Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
+    int64_t* header_offset);
+
+// int64_t GetRowBatchMetadata(const RowBatch* batch);
+
+// Compute the precise number of bytes needed in a contiguous memory segment to
+// write the row batch. This involves generating the complete serialized
+// Flatbuffers metadata.
+int64_t GetRowBatchSize(const RowBatch* batch);
+
+// ----------------------------------------------------------------------
+// "Read" path; does not copy data if the MemorySource does not
+
+class RowBatchReader {
+ public:
+  static Status Open(MemorySource* source, int64_t position,
+      std::shared_ptr<RowBatchReader>* out);
+
+  // Reassemble the row batch. A Schema is required to be able to construct the
+  // right array containers
+  Status GetRowBatch(const std::shared_ptr<Schema>& schema,
+      std::shared_ptr<RowBatch>* out);
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+} // namespace ipc
+} // namespace arrow
+
+#endif // ARROW_IPC_MEMORY_H
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
new file mode 100644
index 0000000000000..d75998f0a5dd2
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/ipc/adapter.h"
+#include "arrow/ipc/memory.h"
+#include "arrow/ipc/test-common.h"
+
+#include "arrow/test-util.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+class TestWriteRowBatch : public ::testing::Test, public MemoryMapFixture {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+  }
+  void TearDown() {
+    MemoryMapFixture::TearDown();
+  }
+
+  void InitMemoryMap(int64_t size) {
+    std::string path = "test-write-row-batch";
+    MemoryMapFixture::CreateFile(path, size);
+    ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_WRITE, &mmap_));
+  }
+
+ protected:
+  MemoryPool* pool_;
+  std::shared_ptr<MemoryMappedSource> mmap_;
+};
+
+const auto INT32 = std::make_shared<Int32Type>();
+
+TEST_F(TestWriteRowBatch, IntegerRoundTrip) {
+  const int length = 1000;
+
+  // Make the schema
+  auto f0 = std::make_shared<Field>("f0", INT32);
+  auto f1 = std::make_shared<Field>("f1", INT32);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  // Example data
+
+  auto data = std::make_shared<PoolBuffer>(pool_);
+  ASSERT_OK(data->Resize(length * sizeof(int32_t)));
+  test::rand_uniform_int(length, 0, 0, std::numeric_limits<int32_t>::max(),
+      reinterpret_cast<int32_t*>(data->mutable_data()));
+
+  auto nulls = std::make_shared<PoolBuffer>(pool_);
+  int null_bytes = util::bytes_for_bits(length);
+  ASSERT_OK(nulls->Resize(null_bytes));
+  test::random_bytes(null_bytes, 0, nulls->mutable_data());
+
+  auto a0 = std::make_shared<Int32Array>(length, data);
+  auto a1 = std::make_shared<Int32Array>(length, data,
+      test::bitmap_popcount(nulls->data(), length), nulls);
+
+  RowBatch batch(schema, length, {a0, a1});
+
+  // TODO(wesm): computing memory requirements for a row batch
+  // 64k is plenty of space
+  InitMemoryMap(1 << 16);
+
+  int64_t header_location;
+  ASSERT_OK(WriteRowBatch(mmap_.get(), &batch, 0, &header_location));
+
+  std::shared_ptr<RowBatchReader> result;
+  ASSERT_OK(RowBatchReader::Open(mmap_.get(), header_location, &result));
+
+  std::shared_ptr<RowBatch> batch_result;
+  ASSERT_OK(result->GetRowBatch(schema, &batch_result));
+  EXPECT_EQ(batch.num_rows(), batch_result->num_rows());
+
+  for (int i = 0; i < batch.num_columns(); ++i) {
+    EXPECT_TRUE(batch.column(i)->Equals(batch_result->column(i)))
+      << i << batch.column_name(i);
+  }
+}
+
+} // namespace ipc
+} // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-memory-test.cc b/cpp/src/arrow/ipc/ipc-memory-test.cc
new file mode 100644
index 0000000000000..332ad2a2b809b
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-memory-test.cc
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/ipc/memory.h"
+#include "arrow/ipc/test-common.h"
+#include "arrow/test-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+class TestMemoryMappedSource : public ::testing::Test, public MemoryMapFixture {
+ public:
+  void TearDown() {
+    MemoryMapFixture::TearDown();
+  }
+};
+
+TEST_F(TestMemoryMappedSource, InvalidUsages) {
+}
+
+TEST_F(TestMemoryMappedSource, WriteRead) {
+  const int64_t buffer_size = 1024;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(1024, 0, buffer.data());
+
+  const int reps = 5;
+
+  std::string path = "ipc-write-read-test";
+  CreateFile(path, reps * buffer_size);
+
+  std::shared_ptr<MemoryMappedSource> result;
+  ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_WRITE, &result));
+
+  int64_t position = 0;
+
+  std::shared_ptr<Buffer> out_buffer;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(result->Write(position, buffer.data(), buffer_size));
+    ASSERT_OK(result->ReadAt(position, buffer_size, &out_buffer));
+
+    ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
+
+    position += buffer_size;
+  }
+}
+
+TEST_F(TestMemoryMappedSource, InvalidFile) {
+  std::string non_existent_path = "invalid-file-name-asfd";
+
+  std::shared_ptr<MemoryMappedSource> result;
+  ASSERT_RAISES(IOError, MemoryMappedSource::Open(non_existent_path,
+          MemorySource::READ_ONLY, &result));
+}
+
+} // namespace ipc
+} // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
new file mode 100644
index 0000000000000..ceabec0fa7c29
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include "arrow/ipc/metadata.h"
+#include "arrow/schema.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+class Buffer;
+
+static inline void assert_schema_equal(const Schema* lhs, const Schema* rhs) {
+  if (!lhs->Equals(*rhs)) {
+    std::stringstream ss;
+    ss << "left schema: " << lhs->ToString() << std::endl
+       << "right schema: " << rhs->ToString() << std::endl;
+    FAIL() << ss.str();
+  }
+}
+
+class TestSchemaMessage : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  void CheckRoundtrip(const Schema* schema) {
+    std::shared_ptr<Buffer> buffer;
+    ASSERT_OK(ipc::WriteSchema(schema, &buffer));
+
+    std::shared_ptr<ipc::Message> message;
+    ASSERT_OK(ipc::Message::Open(buffer, &message));
+
+    ASSERT_EQ(ipc::Message::SCHEMA, message->type());
+
+    std::shared_ptr<ipc::SchemaMessage> schema_msg = message->GetSchema();
+    ASSERT_EQ(schema->num_fields(), schema_msg->num_fields());
+
+    std::shared_ptr<Schema> schema2;
+    ASSERT_OK(schema_msg->GetSchema(&schema2));
+
+    assert_schema_equal(schema, schema2.get());
+  }
+};
+
+const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
+
+TEST_F(TestSchemaMessage, PrimitiveFields) {
+  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>());
+  auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
+  auto f3 = std::make_shared<Field>("f3", std::make_shared<Int64Type>());
+  auto f4 = std::make_shared<Field>("f4", std::make_shared<UInt8Type>());
+  auto f5 = std::make_shared<Field>("f5", std::make_shared<UInt16Type>());
+  auto f6 = std::make_shared<Field>("f6", std::make_shared<UInt32Type>());
+  auto f7 = std::make_shared<Field>("f7", std::make_shared<UInt64Type>());
+  auto f8 = std::make_shared<Field>("f8", std::make_shared<FloatType>());
+  auto f9 = std::make_shared<Field>("f9", std::make_shared<DoubleType>());
+  auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
+
+  Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
+  CheckRoundtrip(&schema);
+}
+
+TEST_F(TestSchemaMessage, NestedFields) {
+  auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
+  auto f0 = std::make_shared<Field>("f0", type);
+
+  std::shared_ptr<StructType> type2(new StructType({
+            std::make_shared<Field>("k1", INT32),
+              std::make_shared<Field>("k2", INT32),
+              std::make_shared<Field>("k3", INT32)}));
+  auto f1 = std::make_shared<Field>("f1", type2);
+
+  Schema schema({f0, f1});
+  CheckRoundtrip(&schema);
+}
+
+} // namespace arrow
diff --git a/cpp/src/arrow/ipc/memory.cc b/cpp/src/arrow/ipc/memory.cc
new file mode 100644
index 0000000000000..e630ccd109b77
--- /dev/null
+++ b/cpp/src/arrow/ipc/memory.cc
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/memory.h"
+
+#include <sys/mman.h> // For memory-mapping
+#include <algorithm>
+#include <cerrno>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <sstream>
+#include <string>
+
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+MemorySource::MemorySource(AccessMode access_mode) :
+    access_mode_(access_mode) {}
+
+MemorySource::~MemorySource() {}
+
+// Implement MemoryMappedSource
+
+class MemoryMappedSource::Impl {
+ public:
+  Impl() :
+      file_(nullptr),
+      is_open_(false),
+      data_(nullptr) {}
+
+  ~Impl() {
+    if (is_open_) {
+      munmap(data_, size_);
+      fclose(file_);
+    }
+  }
+
+  Status Open(const std::string& path, MemorySource::AccessMode mode) {
+    if (is_open_) {
+      return Status::IOError("A file is already open");
+    }
+
+    path_ = path;
+
+    if (mode == MemorySource::READ_WRITE) {
+      file_ = fopen(path.c_str(), "r+b");
+    } else {
+      file_ = fopen(path.c_str(), "rb");
+    }
+    if (file_ == nullptr) {
+      std::stringstream ss;
+      ss << "Unable to open file, errno: " << errno;
+      return Status::IOError(ss.str());
+    }
+
+    fseek(file_, 0L, SEEK_END);
+    if (ferror(file_)) {
+      return Status::IOError("Unable to seek to end of file");
+    }
+    size_ = ftell(file_);
+
+    fseek(file_, 0L, SEEK_SET);
+    is_open_ = true;
+
+    // TODO(wesm): Add read-only version of this
+    data_ = reinterpret_cast<uint8_t*>(mmap(nullptr, size_,
+            PROT_READ | PROT_WRITE,
+            MAP_SHARED, fileno(file_), 0));
+    if (data_ == nullptr) {
+      std::stringstream ss;
+      ss << "Memory mapping file failed, errno: " << errno;
+      return Status::IOError(ss.str());
+    }
+
+    return Status::OK();
+  }
+
+  int64_t size() const {
+    return size_;
+  }
+
+  uint8_t* data() {
+    return data_;
+  }
+
+ private:
+  std::string path_;
+  FILE* file_;
+  int64_t size_;
+  bool is_open_;
+
+  // The memory map
+  uint8_t* data_;
+};
+
+MemoryMappedSource::MemoryMappedSource(AccessMode access_mode) :
+    MemorySource(access_mode) {}
+
+Status MemoryMappedSource::Open(const std::string& path, AccessMode access_mode,
+    std::shared_ptr<MemoryMappedSource>* out) {
+  std::shared_ptr<MemoryMappedSource> result(new MemoryMappedSource(access_mode));
+
+  result->impl_.reset(new Impl());
+  RETURN_NOT_OK(result->impl_->Open(path, access_mode));
+
+  *out = result;
+  return Status::OK();
+}
+
+int64_t MemoryMappedSource::Size() const {
+  return impl_->size();
+}
+
+Status MemoryMappedSource::Close() {
+  // munmap handled in ::Impl dtor
+  return Status::OK();
+}
+
+Status MemoryMappedSource::ReadAt(int64_t position, int64_t nbytes,
+    std::shared_ptr<Buffer>* out) {
+  if (position < 0 || position >= impl_->size()) {
+    return Status::Invalid("position is out of bounds");
+  }
+
+  nbytes = std::min(nbytes, impl_->size() - position);
+  *out = std::make_shared<Buffer>(impl_->data() + position, nbytes);
+  return Status::OK();
+}
+
+Status MemoryMappedSource::Write(int64_t position, const uint8_t* data,
+    int64_t nbytes) {
+  if (position < 0 || position >= impl_->size()) {
+    return Status::Invalid("position is out of bounds");
+  }
+
+  // TODO(wesm): verify we are not writing past the end of the buffer
+  uint8_t* dst = impl_->data() + position;
+  memcpy(dst, data, nbytes);
+
+  return Status::OK();
+}
+
+} // namespace ipc
+} // namespace arrow
diff --git a/cpp/src/arrow/ipc/memory.h b/cpp/src/arrow/ipc/memory.h
new file mode 100644
index 0000000000000..0b4d8347c342f
--- /dev/null
+++ b/cpp/src/arrow/ipc/memory.h
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Public API for different interprocess memory sharing mechanisms
+
+#ifndef ARROW_IPC_MEMORY_H
+#define ARROW_IPC_MEMORY_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "arrow/util/macros.h"
+
+namespace arrow {
+
+class Buffer;
+class MutableBuffer;
+class Status;
+
+namespace ipc {
+
+// Abstract output stream
+class OutputStream {
+ public:
+  virtual ~OutputStream() {}
+  // Close the output stream
+  virtual Status Close() = 0;
+
+  // The current position in the output stream
+  virtual int64_t Tell() const = 0;
+
+  // Write bytes to the stream
+  virtual Status Write(const uint8_t* data, int64_t length) = 0;
+};
+
+// An output stream that writes to a MutableBuffer, such as one obtained from a
+// memory map
+class BufferOutputStream : public OutputStream {
+ public:
+  explicit BufferOutputStream(const std::shared_ptr<MutableBuffer>& buffer):
+      buffer_(buffer) {}
+
+  // Implement the OutputStream interface
+  Status Close() override;
+  int64_t Tell() const override;
+  Status Write(const uint8_t* data, int64_t length) override;
+
+  // Returns the number of bytes remaining in the buffer
+  int64_t bytes_remaining() const;
+
+ private:
+  std::shared_ptr<MutableBuffer> buffer_;
+  int64_t capacity_;
+  int64_t position_;
+};
+
+class MemorySource {
+ public:
+  // Indicates the access permissions of the memory source
+  enum AccessMode {
+    READ_ONLY,
+    READ_WRITE
+  };
+
+  virtual ~MemorySource();
+
+  // Retrieve a buffer of memory from the source of the indicates size and at
+  // the indicated location
+  // @returns: arrow::Status indicating success / failure. The buffer is set
+  // into the *out argument
+  virtual Status ReadAt(int64_t position, int64_t nbytes,
+      std::shared_ptr<Buffer>* out) = 0;
+
+  virtual Status Close() = 0;
+
+  virtual Status Write(int64_t position, const uint8_t* data, int64_t nbytes) = 0;
+
+  // @return: the size in bytes of the memory source
+  virtual int64_t Size() const = 0;
+
+ protected:
+  explicit MemorySource(AccessMode access_mode = AccessMode::READ_WRITE);
+
+  AccessMode access_mode_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(MemorySource);
+};
+
+// A memory source that uses memory-mapped files for memory interactions
+class MemoryMappedSource : public MemorySource {
+ public:
+  static Status Open(const std::string& path, AccessMode access_mode,
+      std::shared_ptr<MemoryMappedSource>* out);
+
+  Status Close() override;
+
+  Status ReadAt(int64_t position, int64_t nbytes,
+      std::shared_ptr<Buffer>* out) override;
+
+  Status Write(int64_t position, const uint8_t* data, int64_t nbytes) override;
+
+  // @return: the size in bytes of the memory source
+  int64_t Size() const override;
+
+ private:
+  explicit MemoryMappedSource(AccessMode access_mode);
+  // Hide the internal details of this class for now
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+} // namespace ipc
+} // namespace arrow
+
+#endif // ARROW_IPC_MEMORY_H
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
new file mode 100644
index 0000000000000..14b186906c3a0
--- /dev/null
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -0,0 +1,317 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/metadata-internal.h"
+
+#include <flatbuffers/flatbuffers.h>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include "arrow/ipc/Message_generated.h"
+#include "arrow/schema.h"
+#include "arrow/type.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+typedef flatbuffers::FlatBufferBuilder FBB;
+typedef flatbuffers::Offset<arrow::flatbuf::Field> FieldOffset;
+typedef flatbuffers::Offset<void> Offset;
+
+namespace arrow {
+
+namespace flatbuf = apache::arrow::flatbuf;
+
+namespace ipc {
+
+const std::shared_ptr<DataType> BOOL = std::make_shared<BooleanType>();
+const std::shared_ptr<DataType> INT8 = std::make_shared<Int8Type>();
+const std::shared_ptr<DataType> INT16 = std::make_shared<Int16Type>();
+const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
+const std::shared_ptr<DataType> INT64 = std::make_shared<Int64Type>();
+const std::shared_ptr<DataType> UINT8 = std::make_shared<UInt8Type>();
+const std::shared_ptr<DataType> UINT16 = std::make_shared<UInt16Type>();
+const std::shared_ptr<DataType> UINT32 = std::make_shared<UInt32Type>();
+const std::shared_ptr<DataType> UINT64 = std::make_shared<UInt64Type>();
+const std::shared_ptr<DataType> FLOAT = std::make_shared<FloatType>();
+const std::shared_ptr<DataType> DOUBLE = std::make_shared<DoubleType>();
+
+static Status IntFromFlatbuffer(const flatbuf::Int* int_data,
+    std::shared_ptr<DataType>* out) {
+  if (int_data->bitWidth() % 8 != 0) {
+    return Status::NotImplemented("Integers not in cstdint are not implemented");
+  } else if (int_data->bitWidth() > 64) {
+    return Status::NotImplemented("Integers with more than 64 bits not implemented");
+  }
+
+  switch (int_data->bitWidth()) {
+    case 8:
+      *out = int_data->is_signed() ? INT8 : UINT8;
+      break;
+    case 16:
+      *out = int_data->is_signed() ? INT16 : UINT16;
+      break;
+    case 32:
+      *out = int_data->is_signed() ? INT32 : UINT32;
+      break;
+    case 64:
+      *out = int_data->is_signed() ? INT64 : UINT64;
+      break;
+    default:
+      *out = nullptr;
+      break;
+  }
+  return Status::OK();
+}
+
+static Status FloatFromFlatuffer(const flatbuf::FloatingPoint* float_data,
+    std::shared_ptr<DataType>* out) {
+  if (float_data->precision() == flatbuf::Precision_SINGLE) {
+    *out = FLOAT;
+  } else {
+    *out = DOUBLE;
+  }
+  return Status::OK();
+}
+
+static Status TypeFromFlatbuffer(flatbuf::Type type,
+    const void* type_data, const std::vector<std::shared_ptr<Field>>& children,
+    std::shared_ptr<DataType>* out) {
+  switch (type) {
+    case flatbuf::Type_NONE:
+      return Status::Invalid("Type metadata cannot be none");
+    case flatbuf::Type_Int:
+      return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
+    case flatbuf::Type_Bit:
+      return Status::NotImplemented("Type is not implemented");
+    case flatbuf::Type_FloatingPoint:
+      return FloatFromFlatuffer(static_cast<const flatbuf::FloatingPoint*>(type_data),
+          out);
+    case flatbuf::Type_Binary:
+    case flatbuf::Type_Utf8:
+      return Status::NotImplemented("Type is not implemented");
+    case flatbuf::Type_Bool:
+      *out = BOOL;
+      return Status::OK();
+    case flatbuf::Type_Decimal:
+    case flatbuf::Type_Timestamp:
+    case flatbuf::Type_List:
+      if (children.size() != 1) {
+        return Status::Invalid("List must have exactly 1 child field");
+      }
+      *out = std::make_shared<ListType>(children[0]);
+      return Status::OK();
+    case flatbuf::Type_Tuple:
+      *out = std::make_shared<StructType>(children);
+      return Status::OK();
+    case flatbuf::Type_Union:
+      return Status::NotImplemented("Type is not implemented");
+    default:
+      return Status::Invalid("Unrecognized type");
+  }
+}
+
+// Forward declaration
+static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+    FieldOffset* offset);
+
+static Offset IntToFlatbuffer(FBB& fbb, int bitWidth,
+    bool is_signed) {
+  return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
+}
+
+static Offset FloatToFlatbuffer(FBB& fbb,
+    flatbuf::Precision precision) {
+  return flatbuf::CreateFloatingPoint(fbb, precision).Union();
+}
+
+static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, Offset* offset) {
+  FieldOffset field;
+  RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(0), &field));
+  out_children->push_back(field);
+  *offset = flatbuf::CreateList(fbb).Union();
+  return Status::OK();
+}
+
+static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, Offset* offset) {
+  FieldOffset field;
+  for (int i = 0; i < type->num_children(); ++i) {
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), &field));
+    out_children->push_back(field);
+  }
+  *offset = flatbuf::CreateTuple(fbb).Union();
+  return Status::OK();
+}
+
+#define INT_TO_FB_CASE(BIT_WIDTH, IS_SIGNED)            \
+  *out_type = flatbuf::Type_Int;                        \
+  *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
+  break;
+
+
+static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* children,
+    flatbuf::Type* out_type, Offset* offset) {
+  switch (type->type) {
+    case Type::BOOL:
+      *out_type = flatbuf::Type_Bool;
+      *offset = flatbuf::CreateBool(fbb).Union();
+      break;
+    case Type::UINT8:
+      INT_TO_FB_CASE(8, false);
+    case Type::INT8:
+      INT_TO_FB_CASE(8, true);
+    case Type::UINT16:
+      INT_TO_FB_CASE(16, false);
+    case Type::INT16:
+      INT_TO_FB_CASE(16, true);
+    case Type::UINT32:
+      INT_TO_FB_CASE(32, false);
+    case Type::INT32:
+      INT_TO_FB_CASE(32, true);
+    case Type::UINT64:
+      INT_TO_FB_CASE(64, false);
+    case Type::INT64:
+      INT_TO_FB_CASE(64, true);
+    case Type::FLOAT:
+      *out_type = flatbuf::Type_FloatingPoint;
+      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_SINGLE);
+      break;
+    case Type::DOUBLE:
+      *out_type = flatbuf::Type_FloatingPoint;
+      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
+      break;
+    case Type::LIST:
+      *out_type = flatbuf::Type_List;
+      return ListToFlatbuffer(fbb, type, children, offset);
+    case Type::STRUCT:
+      *out_type = flatbuf::Type_Tuple;
+      return StructToFlatbuffer(fbb, type, children, offset);
+    default:
+      std::stringstream ss;
+      ss << "Unable to convert type: " << type->ToString()
+         << std::endl;
+      return Status::NotImplemented(ss.str());
+  }
+  return Status::OK();
+}
+
+static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+    FieldOffset* offset) {
+  auto fb_name = fbb.CreateString(field->name);
+
+  flatbuf::Type type_enum;
+  Offset type_data;
+  std::vector<FieldOffset> children;
+
+  RETURN_NOT_OK(TypeToFlatbuffer(fbb, field->type, &children, &type_enum, &type_data));
+  auto fb_children = fbb.CreateVector(children);
+
+  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum,
+      type_data, fb_children);
+
+  return Status::OK();
+}
+
+Status FieldFromFlatbuffer(const flatbuf::Field* field,
+    std::shared_ptr<Field>* out) {
+  std::shared_ptr<DataType> type;
+
+  auto children = field->children();
+  std::vector<std::shared_ptr<Field>> child_fields(children->size());
+  for (size_t i = 0; i < children->size(); ++i) {
+    RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), &child_fields[i]));
+  }
+
+  RETURN_NOT_OK(TypeFromFlatbuffer(field->type_type(),
+          field->type(), child_fields, &type));
+
+  *out = std::make_shared<Field>(field->name()->str(), type);
+  return Status::OK();
+}
+
+// Implement MessageBuilder
+
+Status MessageBuilder::SetSchema(const Schema* schema) {
+  header_type_ = flatbuf::MessageHeader_Schema;
+
+  std::vector<FieldOffset> field_offsets;
+  for (int i = 0; i < schema->num_fields(); ++i) {
+    const std::shared_ptr<Field>& field = schema->field(i);
+    FieldOffset offset;
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb_, field, &offset));
+    field_offsets.push_back(offset);
+  }
+
+  header_ = flatbuf::CreateSchema(fbb_, fbb_.CreateVector(field_offsets)).Union();
+  body_length_ = 0;
+  return Status::OK();
+}
+
+Status MessageBuilder::SetRecordBatch(int32_t length, int64_t body_length,
+    const std::vector<flatbuf::FieldNode>& nodes,
+    const std::vector<flatbuf::Buffer>& buffers) {
+  header_type_ = flatbuf::MessageHeader_RecordBatch;
+  header_ = flatbuf::CreateRecordBatch(fbb_, length,
+      fbb_.CreateVectorOfStructs(nodes),
+      fbb_.CreateVectorOfStructs(buffers)).Union();
+  body_length_ = body_length;
+
+  return Status::OK();
+}
+
+
+Status WriteDataHeader(int32_t length, int64_t body_length,
+    const std::vector<flatbuf::FieldNode>& nodes,
+    const std::vector<flatbuf::Buffer>& buffers,
+    std::shared_ptr<Buffer>* out) {
+  MessageBuilder message;
+  RETURN_NOT_OK(message.SetRecordBatch(length, body_length, nodes, buffers));
+  RETURN_NOT_OK(message.Finish());
+  return message.GetBuffer(out);
+}
+
+Status MessageBuilder::Finish() {
+  auto message = flatbuf::CreateMessage(fbb_, header_type_, header_,
+      body_length_);
+  fbb_.Finish(message);
+  return Status::OK();
+}
+
+Status MessageBuilder::GetBuffer(std::shared_ptr<Buffer>* out) {
+  // The message buffer is prefixed by the size of the complete flatbuffer as
+  // int32_t
+  // <int32_t: flatbuffer size><uint8_t*: flatbuffer data>
+  int32_t size = fbb_.GetSize();
+
+  auto result = std::make_shared<PoolBuffer>();
+  RETURN_NOT_OK(result->Resize(size + sizeof(int32_t)));
+
+  uint8_t* dst = result->mutable_data();
+  memcpy(dst, reinterpret_cast<int32_t*>(&size), sizeof(int32_t));
+  memcpy(dst + sizeof(int32_t), fbb_.GetBufferPointer(), size);
+
+  *out = result;
+  return Status::OK();
+}
+
+} // namespace ipc
+} // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
new file mode 100644
index 0000000000000..f7365d2a49f95
--- /dev/null
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IPC_METADATA_INTERNAL_H
+#define ARROW_IPC_METADATA_INTERNAL_H
+
+#include <flatbuffers/flatbuffers.h>
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "arrow/ipc/Message_generated.h"
+
+namespace arrow {
+
+namespace flatbuf = apache::arrow::flatbuf;
+
+class Buffer;
+struct Field;
+class Schema;
+class Status;
+
+namespace ipc {
+
+Status FieldFromFlatbuffer(const flatbuf::Field* field,
+    std::shared_ptr<Field>* out);
+
+class MessageBuilder {
+ public:
+  Status SetSchema(const Schema* schema);
+
+  Status SetRecordBatch(int32_t length, int64_t body_length,
+      const std::vector<flatbuf::FieldNode>& nodes,
+      const std::vector<flatbuf::Buffer>& buffers);
+
+  Status Finish();
+
+  Status GetBuffer(std::shared_ptr<Buffer>* out);
+
+ private:
+  flatbuf::MessageHeader header_type_;
+  flatbuffers::Offset<void> header_;
+  int64_t body_length_;
+  flatbuffers::FlatBufferBuilder fbb_;
+};
+
+Status WriteDataHeader(int32_t length, int64_t body_length,
+    const std::vector<flatbuf::FieldNode>& nodes,
+    const std::vector<flatbuf::Buffer>& buffers,
+    std::shared_ptr<Buffer>* out);
+
+} // namespace ipc
+} // namespace arrow
+
+#endif // ARROW_IPC_METADATA_INTERNAL_H
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
new file mode 100644
index 0000000000000..642f21a41e640
--- /dev/null
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/metadata.h"
+
+#include <flatbuffers/flatbuffers.h>
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+// Generated C++ flatbuffer IDL
+#include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/metadata-internal.h"
+
+#include "arrow/schema.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+namespace flatbuf = apache::arrow::flatbuf;
+
+namespace ipc {
+
+Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out) {
+  MessageBuilder message;
+  RETURN_NOT_OK(message.SetSchema(schema));
+  RETURN_NOT_OK(message.Finish());
+  return message.GetBuffer(out);
+}
+
+//----------------------------------------------------------------------
+// Message reader
+
+class Message::Impl {
+ public:
+  explicit Impl(const std::shared_ptr<Buffer>& buffer,
+      const flatbuf::Message* message) :
+      buffer_(buffer),
+      message_(message) {}
+
+  Message::Type type() const {
+    switch (message_->header_type()) {
+      case flatbuf::MessageHeader_Schema:
+        return Message::SCHEMA;
+      case flatbuf::MessageHeader_DictionaryBatch:
+        return Message::DICTIONARY_BATCH;
+      case flatbuf::MessageHeader_RecordBatch:
+        return Message::RECORD_BATCH;
+      default:
+        return Message::NONE;
+    }
+  }
+
+  const void* header() const {
+    return message_->header();
+  }
+
+  int64_t body_length() const {
+    return message_->bodyLength();
+  }
+
+ private:
+  // Owns the memory this message accesses
+  std::shared_ptr<Buffer> buffer_;
+
+  const flatbuf::Message* message_;
+};
+
+class SchemaMessage::Impl {
+ public:
+  explicit Impl(const void* schema) :
+      schema_(static_cast<const flatbuf::Schema*>(schema)) {}
+
+  const flatbuf::Field* field(int i) const {
+    return schema_->fields()->Get(i);
+  }
+
+  int num_fields() const {
+    return schema_->fields()->size();
+  }
+
+ private:
+  const flatbuf::Schema* schema_;
+};
+
+Message::Message() {}
+
+Status Message::Open(const std::shared_ptr<Buffer>& buffer,
+    std::shared_ptr<Message>* out) {
+  std::shared_ptr<Message> result(new Message());
+
+  // The buffer is prefixed by its size as int32_t
+  const uint8_t* fb_head = buffer->data() + sizeof(int32_t);
+  const flatbuf::Message* message = flatbuf::GetMessage(fb_head);
+
+  // TODO(wesm): verify message
+  result->impl_.reset(new Impl(buffer, message));
+  *out = result;
+
+  return Status::OK();
+}
+
+Message::Type Message::type() const {
+  return impl_->type();
+}
+
+int64_t Message::body_length() const {
+  return impl_->body_length();
+}
+
+std::shared_ptr<Message> Message::get_shared_ptr() {
+  return this->shared_from_this();
+}
+
+std::shared_ptr<SchemaMessage> Message::GetSchema() {
+  return std::make_shared<SchemaMessage>(this->shared_from_this(),
+      impl_->header());
+}
+
+SchemaMessage::SchemaMessage(const std::shared_ptr<Message>& message,
+    const void* schema) {
+  message_ = message;
+  impl_.reset(new Impl(schema));
+}
+
+int SchemaMessage::num_fields() const {
+  return impl_->num_fields();
+}
+
+Status SchemaMessage::GetField(int i, std::shared_ptr<Field>* out) const {
+  const flatbuf::Field* field = impl_->field(i);
+  return FieldFromFlatbuffer(field, out);
+}
+
+Status SchemaMessage::GetSchema(std::shared_ptr<Schema>* out) const {
+  std::vector<std::shared_ptr<Field>> fields(num_fields());
+  for (int i = 0; i < this->num_fields(); ++i) {
+    RETURN_NOT_OK(GetField(i, &fields[i]));
+  }
+  *out = std::make_shared<Schema>(fields);
+  return Status::OK();
+}
+
+class RecordBatchMessage::Impl {
+ public:
+  explicit Impl(const void* batch) :
+      batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
+    nodes_ = batch_->nodes();
+    buffers_ = batch_->buffers();
+  }
+
+  const flatbuf::FieldNode* field(int i) const {
+    return nodes_->Get(i);
+  }
+
+  const flatbuf::Buffer* buffer(int i) const {
+    return buffers_->Get(i);
+  }
+
+  int32_t length() const {
+    return batch_->length();
+  }
+
+  int num_buffers() const {
+    return batch_->buffers()->size();
+  }
+
+  int num_fields() const {
+    return batch_->nodes()->size();
+  }
+
+ private:
+  const flatbuf::RecordBatch* batch_;
+  const flatbuffers::Vector<const flatbuf::FieldNode*>* nodes_;
+  const flatbuffers::Vector<const flatbuf::Buffer*>* buffers_;
+};
+
+std::shared_ptr<RecordBatchMessage> Message::GetRecordBatch() {
+  return std::make_shared<RecordBatchMessage>(this->shared_from_this(),
+      impl_->header());
+}
+
+RecordBatchMessage::RecordBatchMessage(const std::shared_ptr<Message>& message,
+    const void* batch) {
+  message_ = message;
+  impl_.reset(new Impl(batch));
+}
+
+// TODO(wesm): Copying the flatbuffer data isn't great, but this will do for
+// now
+FieldMetadata RecordBatchMessage::field(int i) const {
+  const flatbuf::FieldNode* node = impl_->field(i);
+
+  FieldMetadata result;
+  result.length = node->length();
+  result.null_count = node->null_count();
+  return result;
+}
+
+BufferMetadata RecordBatchMessage::buffer(int i) const {
+  const flatbuf::Buffer* buffer = impl_->buffer(i);
+
+  BufferMetadata result;
+  result.page = buffer->page();
+  result.offset = buffer->offset();
+  result.length = buffer->length();
+  return result;
+}
+
+int32_t RecordBatchMessage::length() const {
+  return impl_->length();
+}
+
+int RecordBatchMessage::num_buffers() const {
+  return impl_->num_buffers();
+}
+
+int RecordBatchMessage::num_fields() const {
+  return impl_->num_fields();
+}
+
+} // namespace ipc
+} // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
new file mode 100644
index 0000000000000..c7288529b9fbd
--- /dev/null
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// C++ object model and user API for interprocess schema messaging
+
+#ifndef ARROW_IPC_METADATA_H
+#define ARROW_IPC_METADATA_H
+
+#include <cstdint>
+#include <memory>
+
+namespace arrow {
+
+class Buffer;
+struct Field;
+class Schema;
+class Status;
+
+namespace ipc {
+
+//----------------------------------------------------------------------
+// Message read/write APIs
+
+// Serialize arrow::Schema as a Flatbuffer
+Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out);
+
+//----------------------------------------------------------------------
+
+// Read interface classes. We do not fully deserialize the flatbuffers so that
+// individual fields metadata can be retrieved from very large schema without
+//
+
+class Message;
+
+// Container for serialized Schema metadata contained in an IPC message
+class SchemaMessage {
+ public:
+  // Accepts an opaque flatbuffer pointer
+  SchemaMessage(const std::shared_ptr<Message>& message, const void* schema);
+
+  int num_fields() const;
+
+  // Construct an arrow::Field for the i-th value in the metadata
+  Status GetField(int i, std::shared_ptr<Field>* out) const;
+
+  // Construct a complete Schema from the message. May be expensive for very
+  // large schemas if you are only interested in a few fields
+  Status GetSchema(std::shared_ptr<Schema>* out) const;
+
+ private:
+  // Parent, owns the flatbuffer data
+  std::shared_ptr<Message> message_;
+
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+// Field metadata
+struct FieldMetadata {
+  int32_t length;
+  int32_t null_count;
+};
+
+struct BufferMetadata {
+  int32_t page;
+  int64_t offset;
+  int64_t length;
+};
+
+// Container for serialized record batch metadata contained in an IPC message
+class RecordBatchMessage {
+ public:
+  // Accepts an opaque flatbuffer pointer
+  RecordBatchMessage(const std::shared_ptr<Message>& message,
+      const void* batch_meta);
+
+  FieldMetadata field(int i) const;
+  BufferMetadata buffer(int i) const;
+
+  int32_t length() const;
+  int num_buffers() const;
+  int num_fields() const;
+
+ private:
+  // Parent, owns the flatbuffer data
+  std::shared_ptr<Message> message_;
+
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+class DictionaryBatchMessage {
+ public:
+  int64_t id() const;
+  std::unique_ptr<RecordBatchMessage> data() const;
+};
+
+class Message : public std::enable_shared_from_this<Message> {
+ public:
+  enum Type {
+    NONE,
+    SCHEMA,
+    DICTIONARY_BATCH,
+    RECORD_BATCH
+  };
+
+  static Status Open(const std::shared_ptr<Buffer>& buffer,
+    std::shared_ptr<Message>* out);
+
+  std::shared_ptr<Message> get_shared_ptr();
+
+  int64_t body_length() const;
+
+  Type type() const;
+
+  // These methods only to be invoked if you have checked the message type
+  std::shared_ptr<SchemaMessage> GetSchema();
+  std::shared_ptr<RecordBatchMessage> GetRecordBatch();
+  std::shared_ptr<DictionaryBatchMessage> GetDictionaryBatch();
+
+ private:
+  Message();
+
+  // Hide serialization details from user API
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+} // namespace ipc
+} // namespace arrow
+
+#endif // ARROW_IPC_METADATA_H
diff --git a/cpp/src/arrow/types/floating.h b/cpp/src/arrow/ipc/test-common.h
similarity index 59%
rename from cpp/src/arrow/types/floating.h
rename to cpp/src/arrow/ipc/test-common.h
index e7522781d33e3..0fccce941071b 100644
--- a/cpp/src/arrow/types/floating.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -15,22 +15,39 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef ARROW_TYPES_FLOATING_H
-#define ARROW_TYPES_FLOATING_H
+#ifndef ARROW_IPC_TEST_COMMON_H
+#define ARROW_IPC_TEST_COMMON_H
 
+#include <cstdint>
+#include <memory>
 #include <string>
-
-#include "arrow/types/primitive.h"
-#include "arrow/type.h"
+#include <vector>
 
 namespace arrow {
-
-typedef PrimitiveArrayImpl<FloatType> FloatArray;
-typedef PrimitiveArrayImpl<DoubleType> DoubleArray;
-
-typedef PrimitiveBuilder<FloatType, FloatArray> FloatBuilder;
-typedef PrimitiveBuilder<DoubleType, DoubleArray> DoubleBuilder;
-
+namespace ipc {
+
+class MemoryMapFixture {
+ public:
+  void TearDown() {
+    for (auto path : tmp_files_) {
+      std::remove(path.c_str());
+    }
+  }
+
+  void CreateFile(const std::string path, int64_t size) {
+    FILE* file = fopen(path.c_str(), "w");
+    if (file != nullptr) {
+      tmp_files_.push_back(path);
+    }
+    ftruncate(fileno(file), size);
+    fclose(file);
+  }
+
+ private:
+  std::vector<std::string> tmp_files_;
+};
+
+} // namespace ipc
 } // namespace arrow
 
-#endif // ARROW_TYPES_FLOATING_H
+#endif // ARROW_IPC_TEST_COMMON_H
diff --git a/cpp/src/arrow/table/schema-test.cc b/cpp/src/arrow/schema-test.cc
similarity index 72%
rename from cpp/src/arrow/table/schema-test.cc
rename to cpp/src/arrow/schema-test.cc
index 9dfade2695311..a1de1dc5ac8a4 100644
--- a/cpp/src/arrow/table/schema-test.cc
+++ b/cpp/src/arrow/schema-test.cc
@@ -15,14 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
 #include <memory>
 #include <string>
 #include <vector>
 
-#include "arrow/table/schema.h"
+#include "gtest/gtest.h"
+
+#include "arrow/schema.h"
 #include "arrow/type.h"
-#include "arrow/types/string.h"
 
 using std::shared_ptr;
 using std::vector;
@@ -32,25 +32,20 @@ namespace arrow {
 const auto INT32 = std::make_shared<Int32Type>();
 
 TEST(TestField, Basics) {
-  shared_ptr<DataType> ftype = INT32;
-  shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
-  Field f0("f0", ftype);
-  Field f0_nn("f0", ftype_nn);
+  Field f0("f0", INT32);
+  Field f0_nn("f0", INT32, false);
 
   ASSERT_EQ(f0.name, "f0");
-  ASSERT_EQ(f0.type->ToString(), ftype->ToString());
+  ASSERT_EQ(f0.type->ToString(), INT32->ToString());
 
-  ASSERT_TRUE(f0.nullable());
-  ASSERT_FALSE(f0_nn.nullable());
+  ASSERT_TRUE(f0.nullable);
+  ASSERT_FALSE(f0_nn.nullable);
 }
 
 TEST(TestField, Equals) {
-  shared_ptr<DataType> ftype = INT32;
-  shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
-
-  Field f0("f0", ftype);
-  Field f0_nn("f0", ftype_nn);
-  Field f0_other("f0", ftype);
+  Field f0("f0", INT32);
+  Field f0_nn("f0", INT32, false);
+  Field f0_other("f0", INT32);
 
   ASSERT_EQ(f0, f0_other);
   ASSERT_NE(f0, f0_nn);
@@ -63,12 +58,12 @@ class TestSchema : public ::testing::Test {
 
 TEST_F(TestSchema, Basics) {
   auto f0 = std::make_shared<Field>("f0", INT32);
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(false));
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(), false);
   auto f1_optional = std::make_shared<Field>("f1", std::make_shared<UInt8Type>());
 
   auto f2 = std::make_shared<Field>("f2", std::make_shared<StringType>());
 
-  vector<shared_ptr<Field> > fields = {f0, f1, f2};
+  vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto schema = std::make_shared<Schema>(fields);
 
   ASSERT_EQ(3, schema->num_fields());
@@ -78,7 +73,7 @@ TEST_F(TestSchema, Basics) {
 
   auto schema2 = std::make_shared<Schema>(fields);
 
-  vector<shared_ptr<Field> > fields3 = {f0, f1_optional, f2};
+  vector<shared_ptr<Field>> fields3 = {f0, f1_optional, f2};
   auto schema3 = std::make_shared<Schema>(fields3);
   ASSERT_TRUE(schema->Equals(schema2));
   ASSERT_FALSE(schema->Equals(schema3));
@@ -88,21 +83,20 @@ TEST_F(TestSchema, Basics) {
 }
 
 TEST_F(TestSchema, ToString) {
-  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int32Type>());
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(false));
+  auto f0 = std::make_shared<Field>("f0", INT32);
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(), false);
   auto f2 = std::make_shared<Field>("f2", std::make_shared<StringType>());
   auto f3 = std::make_shared<Field>("f3",
       std::make_shared<ListType>(std::make_shared<Int16Type>()));
 
-  vector<shared_ptr<Field> > fields = {f0, f1, f2, f3};
+  vector<shared_ptr<Field>> fields = {f0, f1, f2, f3};
   auto schema = std::make_shared<Schema>(fields);
 
   std::string result = schema->ToString();
-  std::string expected = R"(f0 int32
-f1 uint8 not null
-f2 string
-f3 list<int16>
-)";
+  std::string expected = R"(f0: int32
+f1: uint8 not null
+f2: string
+f3: list<item: int16>)";
 
   ASSERT_EQ(expected, result);
 }
diff --git a/cpp/src/arrow/table/schema.cc b/cpp/src/arrow/schema.cc
similarity index 88%
rename from cpp/src/arrow/table/schema.cc
rename to cpp/src/arrow/schema.cc
index d49d0a713e7f4..18aad0e806ff2 100644
--- a/cpp/src/arrow/table/schema.cc
+++ b/cpp/src/arrow/schema.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/table/schema.h"
+#include "arrow/schema.h"
 
 #include <memory>
 #include <string>
@@ -26,7 +26,7 @@
 
 namespace arrow {
 
-Schema::Schema(const std::vector<std::shared_ptr<Field> >& fields) :
+Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields) :
     fields_(fields) {}
 
 bool Schema::Equals(const Schema& other) const {
@@ -49,8 +49,13 @@ bool Schema::Equals(const std::shared_ptr<Schema>& other) const {
 std::string Schema::ToString() const {
   std::stringstream buffer;
 
+  int i = 0;
   for (auto field : fields_) {
-    buffer << field->ToString() << std::endl;
+    if (i > 0) {
+      buffer << std::endl;
+    }
+    buffer << field->ToString();
+    ++i;
   }
   return buffer.str();
 }
diff --git a/cpp/src/arrow/table/schema.h b/cpp/src/arrow/schema.h
similarity index 91%
rename from cpp/src/arrow/table/schema.h
rename to cpp/src/arrow/schema.h
index 103f01b26e3ca..52f3c1ceae46d 100644
--- a/cpp/src/arrow/table/schema.h
+++ b/cpp/src/arrow/schema.h
@@ -22,13 +22,13 @@
 #include <string>
 #include <vector>
 
-#include "arrow/type.h"
-
 namespace arrow {
 
+struct Field;
+
 class Schema {
  public:
-  explicit Schema(const std::vector<std::shared_ptr<Field> >& fields);
+  explicit Schema(const std::vector<std::shared_ptr<Field>>& fields);
 
   // Returns true if all of the schema fields are equal
   bool Equals(const Schema& other) const;
@@ -47,7 +47,7 @@ class Schema {
   }
 
  private:
-  std::vector<std::shared_ptr<Field> > fields_;
+  std::vector<std::shared_ptr<Field>> fields_;
 };
 
 } // namespace arrow
diff --git a/cpp/src/arrow/table/table-test.cc b/cpp/src/arrow/table-test.cc
similarity index 92%
rename from cpp/src/arrow/table/table-test.cc
rename to cpp/src/arrow/table-test.cc
index 8b354e8503c71..4c7b8f80486de 100644
--- a/cpp/src/arrow/table/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -15,19 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
-#include "arrow/table/column.h"
-#include "arrow/table/schema.h"
-#include "arrow/table/table.h"
-#include "arrow/table/test-common.h"
+#include "gtest/gtest.h"
+
+#include "arrow/column.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/integer.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/status.h"
 
 using std::shared_ptr;
 using std::vector;
@@ -45,7 +45,7 @@ class TestTable : public TestBase {
     auto f1 = std::make_shared<Field>("f1", UINT8);
     auto f2 = std::make_shared<Field>("f2", INT16);
 
-    vector<shared_ptr<Field> > fields = {f0, f1, f2};
+    vector<shared_ptr<Field>> fields = {f0, f1, f2};
     schema_ = std::make_shared<Schema>(fields);
 
     columns_ = {
@@ -58,7 +58,7 @@ class TestTable : public TestBase {
  protected:
   std::unique_ptr<Table> table_;
   shared_ptr<Schema> schema_;
-  vector<std::shared_ptr<Column> > columns_;
+  vector<std::shared_ptr<Column>> columns_;
 };
 
 TEST_F(TestTable, EmptySchema) {
diff --git a/cpp/src/arrow/table/table.cc b/cpp/src/arrow/table.cc
similarity index 69%
rename from cpp/src/arrow/table/table.cc
rename to cpp/src/arrow/table.cc
index 0c788b8fe3ff3..e405c1d508c22 100644
--- a/cpp/src/arrow/table/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -15,20 +15,30 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/table/table.h"
+#include "arrow/table.h"
 
+#include <cstdlib>
 #include <memory>
 #include <sstream>
 
-#include "arrow/table/column.h"
-#include "arrow/table/schema.h"
-#include "arrow/type.h"
+#include "arrow/column.h"
+#include "arrow/schema.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
 
+RowBatch::RowBatch(const std::shared_ptr<Schema>& schema, int num_rows,
+    const std::vector<std::shared_ptr<Array>>& columns) :
+    schema_(schema),
+    num_rows_(num_rows),
+    columns_(columns) {}
+
+const std::string& RowBatch::column_name(int i) const {
+  return schema_->field(i)->name;
+}
+
 Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
-    const std::vector<std::shared_ptr<Column> >& columns) :
+    const std::vector<std::shared_ptr<Column>>& columns) :
     name_(name),
     schema_(schema),
     columns_(columns) {
@@ -40,7 +50,7 @@ Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
 }
 
 Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
-    const std::vector<std::shared_ptr<Column> >& columns, int64_t num_rows) :
+    const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows) :
     name_(name),
     schema_(schema),
     columns_(columns),
@@ -51,16 +61,19 @@ Status Table::ValidateColumns() const {
     return Status::Invalid("Number of columns did not match schema");
   }
 
-  if (columns_.size() == 0) {
-    return Status::OK();
-  }
-
   // Make sure columns are all the same length
   for (size_t i = 0; i < columns_.size(); ++i) {
     const Column* col = columns_[i].get();
+    if (col == nullptr) {
+      std::stringstream ss;
+      ss << "Column " << i << " named " << col->name()
+         << " was null";
+      return Status::Invalid(ss.str());
+    }
     if (col->length() != num_rows_) {
       std::stringstream ss;
-      ss << "Column " << i << " expected length "
+      ss << "Column " << i << " named " << col->name()
+         << " expected length "
          << num_rows_
          << " but got length "
          << col->length();
diff --git a/cpp/src/arrow/table/table.h b/cpp/src/arrow/table.h
similarity index 55%
rename from cpp/src/arrow/table/table.h
rename to cpp/src/arrow/table.h
index b0129387b710c..e2f73a2eeddcb 100644
--- a/cpp/src/arrow/table/table.h
+++ b/cpp/src/arrow/table.h
@@ -15,28 +15,74 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef ARROW_TABLE_TABLE_H
-#define ARROW_TABLE_TABLE_H
+#ifndef ARROW_TABLE_H
+#define ARROW_TABLE_H
 
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
 namespace arrow {
 
+class Array;
 class Column;
 class Schema;
 class Status;
 
+// A row batch is a simpler and more rigid table data structure intended for
+// use primarily in shared memory IPC. It contains a schema (metadata) and a
+// corresponding vector of equal-length Arrow arrays
+class RowBatch {
+ public:
+  // num_rows is a parameter to allow for row batches of a particular size not
+  // having any materialized columns. Each array should have the same length as
+  // num_rows
+  RowBatch(const std::shared_ptr<Schema>& schema, int num_rows,
+      const std::vector<std::shared_ptr<Array>>& columns);
+
+  // @returns: the table's schema
+  const std::shared_ptr<Schema>& schema() const {
+    return schema_;
+  }
+
+  // @returns: the i-th column
+  // Note: Does not boundscheck
+  const std::shared_ptr<Array>& column(int i) const {
+    return columns_[i];
+  }
+
+  const std::string& column_name(int i) const;
+
+  // @returns: the number of columns in the table
+  int num_columns() const {
+    return columns_.size();
+  }
+
+  // @returns: the number of rows (the corresponding length of each column)
+  int64_t num_rows() const {
+    return num_rows_;
+  }
+
+ private:
+  std::shared_ptr<Schema> schema_;
+  int num_rows_;
+  std::vector<std::shared_ptr<Array>> columns_;
+};
+
 // Immutable container of fixed-length columns conforming to a particular schema
 class Table {
  public:
   // If columns is zero-length, the table's number of rows is zero
   Table(const std::string& name, const std::shared_ptr<Schema>& schema,
-      const std::vector<std::shared_ptr<Column> >& columns);
+      const std::vector<std::shared_ptr<Column>>& columns);
 
+  // num_rows is a parameter to allow for tables of a particular size not
+  // having any materialized columns. Each column should therefore have the
+  // same length as num_rows -- you can validate this using
+  // Table::ValidateColumns
   Table(const std::string& name, const std::shared_ptr<Schema>& schema,
-      const std::vector<std::shared_ptr<Column> >& columns, int64_t num_rows);
+      const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows);
 
   // @returns: the table's name, if any (may be length 0)
   const std::string& name() const {
@@ -72,11 +118,11 @@ class Table {
   std::string name_;
 
   std::shared_ptr<Schema> schema_;
-  std::vector<std::shared_ptr<Column> > columns_;
+  std::vector<std::shared_ptr<Column>> columns_;
 
   int64_t num_rows_;
 };
 
 } // namespace arrow
 
-#endif  // ARROW_TABLE_TABLE_H
+#endif  // ARROW_TABLE_H
diff --git a/cpp/src/arrow/table/test-common.h b/cpp/src/arrow/table/test-common.h
deleted file mode 100644
index 50a5f6a2f5018..0000000000000
--- a/cpp/src/arrow/table/test-common.h
+++ /dev/null
@@ -1,54 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/table/column.h"
-#include "arrow/table/schema.h"
-#include "arrow/table/table.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-
-namespace arrow {
-
-class TestBase : public ::testing::Test {
- public:
-  void SetUp() {
-    pool_ = GetDefaultMemoryPool();
-  }
-
-  template <typename ArrayType>
-  std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
-    auto data = std::make_shared<PoolBuffer>(pool_);
-    auto nulls = std::make_shared<PoolBuffer>(pool_);
-    EXPECT_OK(data->Resize(length * sizeof(typename ArrayType::value_type)));
-    EXPECT_OK(nulls->Resize(util::bytes_for_bits(length)));
-    return std::make_shared<ArrayType>(length, data, 10, nulls);
-  }
-
- protected:
-  MemoryPool* pool_;
-};
-
-} // namespace arrow
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 0898c8e3e3aa3..a9fb2a7644ab3 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -18,26 +18,39 @@
 #ifndef ARROW_TEST_UTIL_H_
 #define ARROW_TEST_UTIL_H_
 
-#include <gtest/gtest.h>
+#include <cstdint>
 #include <memory>
+#include <random>
 #include <string>
 #include <vector>
 
+#include "gtest/gtest.h"
+
+#include "arrow/type.h"
+#include "arrow/column.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
 #include "arrow/util/random.h"
 #include "arrow/util/status.h"
 
 #define ASSERT_RAISES(ENUM, expr)               \
   do {                                          \
     Status s = (expr);                          \
-    ASSERT_TRUE(s.Is##ENUM());                  \
+    if (!s.Is##ENUM()) {                        \
+      FAIL() << s.ToString();                   \
+    }                                           \
   } while (0)
 
 
 #define ASSERT_OK(expr)                         \
   do {                                          \
     Status s = (expr);                          \
-    ASSERT_TRUE(s.ok());                        \
+    if (!s.ok()) {                              \
+        FAIL() << s.ToString();                 \
+    }                                           \
   } while (0)
 
 
@@ -50,6 +63,27 @@
 
 namespace arrow {
 
+class TestBase : public ::testing::Test {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+  }
+
+  template <typename ArrayType>
+  std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
+    auto data = std::make_shared<PoolBuffer>(pool_);
+    auto nulls = std::make_shared<PoolBuffer>(pool_);
+    EXPECT_OK(data->Resize(length * sizeof(typename ArrayType::value_type)));
+    EXPECT_OK(nulls->Resize(util::bytes_for_bits(length)));
+    return std::make_shared<ArrayType>(length, data, 10, nulls);
+  }
+
+ protected:
+  MemoryPool* pool_;
+};
+
+namespace test {
+
 template <typename T>
 void randint(int64_t N, T lower, T upper, std::vector<T>* out) {
   Random rng(random_seed());
@@ -84,6 +118,33 @@ void random_nulls(int64_t n, double pct_null, std::vector<bool>* nulls) {
   }
 }
 
+static inline void random_bytes(int n, uint32_t seed, uint8_t* out) {
+  std::mt19937 gen(seed);
+  std::uniform_int_distribution<int> d(0, 255);
+
+  for (int i = 0; i < n; ++i) {
+    out[i] = d(gen) & 0xFF;
+  }
+}
+
+template <typename T>
+void rand_uniform_int(int n, uint32_t seed, T min_value, T max_value, T* out) {
+  std::mt19937 gen(seed);
+  std::uniform_int_distribution<T> d(min_value, max_value);
+  for (int i = 0; i < n; ++i) {
+    out[i] = d(gen);
+  }
+}
+
+static inline int bitmap_popcount(const uint8_t* data, int length) {
+  int count = 0;
+  for (int i = 0; i < length; ++i) {
+    // TODO: accelerate this
+    if (util::get_bit(data, i)) ++count;
+  }
+  return count;
+}
+
 static inline int null_count(const std::vector<uint8_t>& nulls) {
   int result = 0;
   for (size_t i = 0; i < nulls.size(); ++i) {
@@ -102,6 +163,7 @@ std::shared_ptr<Buffer> bytes_to_null_buffer(uint8_t* bytes, int length) {
   return out;
 }
 
+} // namespace test
 } // namespace arrow
 
 #endif // ARROW_TEST_UTIL_H_
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 0a2e817ad30c6..f7f835e96a729 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -24,45 +24,37 @@ namespace arrow {
 
 std::string Field::ToString() const {
   std::stringstream ss;
-  ss << this->name << " " << this->type->ToString();
+  ss << this->name << ": " << this->type->ToString();
+  if (!this->nullable) {
+    ss << " not null";
+  }
   return ss.str();
 }
 
 DataType::~DataType() {}
 
-StringType::StringType(bool nullable)
-    : DataType(LogicalType::STRING, nullable) {}
-
-StringType::StringType(const StringType& other)
-    : StringType(other.nullable) {}
+StringType::StringType() : DataType(Type::STRING) {}
 
 std::string StringType::ToString() const {
   std::string result(name());
-  if (!nullable) {
-    result.append(" not null");
-  }
   return result;
 }
 
 std::string ListType::ToString() const {
   std::stringstream s;
-  s << "list<" << value_type->ToString() << ">";
-  if (!this->nullable) {
-    s << " not null";
-  }
+  s << "list<" << value_field()->ToString() << ">";
   return s.str();
 }
 
 std::string StructType::ToString() const {
   std::stringstream s;
   s << "struct<";
-  for (size_t i = 0; i < fields_.size(); ++i) {
+  for (int i = 0; i < this->num_children(); ++i) {
     if (i > 0) s << ", ";
-    const std::shared_ptr<Field>& field = fields_[i];
+    const std::shared_ptr<Field>& field = this->child(i);
     s << field->name << ": " << field->type->ToString();
   }
   s << ">";
-  if (!nullable) s << " not null";
   return s.str();
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 00b01ea86e8a5..5984b6718ddbe 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -18,62 +18,34 @@
 #ifndef ARROW_TYPE_H
 #define ARROW_TYPE_H
 
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
 namespace arrow {
 
-// Physical data type that describes the memory layout of values. See details
-// for each type
-enum class LayoutEnum: char {
-  // A physical type consisting of some non-negative number of bytes
-  BYTE = 0,
-
-  // A physical type consisting of some non-negative number of bits
-  BIT = 1,
-
-  // A parametric variable-length value type. Full specification requires a
-  // child logical type
-  LIST = 2,
-
-  // A collection of multiple equal-length child arrays. Parametric type taking
-  // 1 or more child logical types
-  STRUCT = 3,
-
-  // An array with heterogeneous value types. Parametric types taking 1 or more
-  // child logical types
-  DENSE_UNION = 4,
-  SPARSE_UNION = 5
-};
-
-
-struct LayoutType {
-  LayoutEnum type;
-  explicit LayoutType(LayoutEnum type) : type(type) {}
-};
-
 // Data types in this library are all *logical*. They can be expressed as
 // either a primitive physical type (bytes or bits of some fixed size), a
 // nested type consisting of other data types, or another data type (e.g. a
 // timestamp encoded as an int64)
-struct LogicalType {
+struct Type {
   enum type {
     // A degenerate NULL type represented as 0 bytes/bits
     NA = 0,
 
-    // Little-endian integer types
-    UINT8 = 1,
-    INT8 = 2,
-    UINT16 = 3,
-    INT16 = 4,
-    UINT32 = 5,
-    INT32 = 6,
-    UINT64 = 7,
-    INT64 = 8,
-
     // A boolean value represented as 1 bit
-    BOOL = 9,
+    BOOL = 1,
+
+    // Little-endian integer types
+    UINT8 = 2,
+    INT8 = 3,
+    UINT16 = 4,
+    INT16 = 5,
+    UINT32 = 6,
+    INT32 = 7,
+    UINT64 = 8,
+    INT64 = 9,
 
     // 4-byte floating point value
     FLOAT = 10,
@@ -131,30 +103,38 @@ struct LogicalType {
   };
 };
 
+struct Field;
+
 struct DataType {
-  LogicalType::type type;
-  bool nullable;
+  Type::type type;
 
-  explicit DataType(LogicalType::type type, bool nullable = true) :
-      type(type),
-      nullable(nullable) {}
+  std::vector<std::shared_ptr<Field>> children_;
+
+  explicit DataType(Type::type type) :
+      type(type) {}
 
   virtual ~DataType();
 
   bool Equals(const DataType* other) {
     // Call with a pointer so more friendly to subclasses
-    return this == other || (this->type == other->type &&
-        this->nullable == other->nullable);
+    return this == other || (this->type == other->type);
   }
 
   bool Equals(const std::shared_ptr<DataType>& other) {
     return Equals(other.get());
   }
 
+  const std::shared_ptr<Field>& child(int i) const {
+    return children_[i];
+  }
+
+  int num_children() const {
+    return children_.size();
+  }
+
   virtual std::string ToString() const = 0;
 };
 
-typedef std::shared_ptr<LayoutType> LayoutPtr;
 typedef std::shared_ptr<DataType> TypePtr;
 
 // A field is a piece of metadata that includes (for now) a name and a data
@@ -166,9 +146,13 @@ struct Field {
   // The field's data type
   TypePtr type;
 
-  Field(const std::string& name, const TypePtr& type) :
+  // Fields can be nullable
+  bool nullable;
+
+  Field(const std::string& name, const TypePtr& type, bool nullable = true) :
       name(name),
-      type(type) {}
+      type(type),
+      nullable(nullable) {}
 
   bool operator==(const Field& other) const {
     return this->Equals(other);
@@ -180,6 +164,7 @@ struct Field {
 
   bool Equals(const Field& other) const {
     return (this == &other) || (this->name == other.name &&
+        this->nullable == other.nullable &&
         this->type->Equals(other.type.get()));
   }
 
@@ -187,36 +172,12 @@ struct Field {
     return Equals(*other.get());
   }
 
-  bool nullable() const {
-    return this->type->nullable;
-  }
-
   std::string ToString() const;
 };
 
-struct BytesType : public LayoutType {
-  int size;
-
-  explicit BytesType(int size)
-      : LayoutType(LayoutEnum::BYTE),
-        size(size) {}
-
-  BytesType(const BytesType& other)
-      : BytesType(other.size) {}
-};
-
-struct ListLayoutType : public LayoutType {
-  LayoutPtr value_type;
-
-  explicit ListLayoutType(const LayoutPtr& value_type)
-      : LayoutType(LayoutEnum::BYTE),
-        value_type(value_type) {}
-};
-
 template <typename Derived>
 struct PrimitiveType : public DataType {
-  explicit PrimitiveType(bool nullable = true)
-      : DataType(Derived::type_enum, nullable) {}
+  PrimitiveType() : DataType(Derived::type_enum) {}
 
   std::string ToString() const override;
 };
@@ -224,22 +185,19 @@ struct PrimitiveType : public DataType {
 template <typename Derived>
 inline std::string PrimitiveType<Derived>::ToString() const {
   std::string result(static_cast<const Derived*>(this)->name());
-  if (!nullable) {
-    result.append(" not null");
-  }
   return result;
 }
 
-#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)          \
-  typedef C_TYPE c_type;                                            \
-  static constexpr LogicalType::type type_enum = LogicalType::ENUM; \
-  static constexpr int size = SIZE;                                 \
-                                                                    \
-  explicit TYPENAME(bool nullable = true)                           \
-      : PrimitiveType<TYPENAME>(nullable) {}                        \
-                                                                    \
-  static const char* name() {                                       \
-    return NAME;                                                    \
+#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)  \
+  typedef C_TYPE c_type;                                    \
+  static constexpr Type::type type_enum = Type::ENUM;       \
+  static constexpr int size = SIZE;                         \
+                                                            \
+  TYPENAME()                                                \
+      : PrimitiveType<TYPENAME>() {}                        \
+                                                            \
+  static const char* name() {                               \
+    return NAME;                                            \
   }
 
 struct NullType : public PrimitiveType<NullType> {
@@ -292,11 +250,23 @@ struct DoubleType : public PrimitiveType<DoubleType> {
 
 struct ListType : public DataType {
   // List can contain any other logical value type
-  TypePtr value_type;
+  explicit ListType(const std::shared_ptr<DataType>& value_type)
+      : DataType(Type::LIST) {
+    children_ = {std::make_shared<Field>("item", value_type)};
+  }
+
+  explicit ListType(const std::shared_ptr<Field>& value_field)
+      : DataType(Type::LIST) {
+    children_ = {value_field};
+  }
 
-  explicit ListType(const TypePtr& value_type, bool nullable = true)
-      : DataType(LogicalType::LIST, nullable),
-        value_type(value_type) {}
+  const std::shared_ptr<Field>& value_field() const {
+    return children_[0];
+  }
+
+  const std::shared_ptr<DataType>& value_type() const {
+    return children_[0]->type;
+  }
 
   static char const *name() {
     return "list";
@@ -307,9 +277,7 @@ struct ListType : public DataType {
 
 // String is a logical type consisting of a physical list of 1-byte values
 struct StringType : public DataType {
-  explicit StringType(bool nullable = true);
-
-  StringType(const StringType& other);
+  StringType();
 
   static char const *name() {
     return "string";
@@ -319,20 +287,9 @@ struct StringType : public DataType {
 };
 
 struct StructType : public DataType {
-  std::vector<std::shared_ptr<Field> > fields_;
-
-  explicit StructType(const std::vector<std::shared_ptr<Field> >& fields,
-      bool nullable = true)
-      : DataType(LogicalType::STRUCT, nullable) {
-    fields_ = fields;
-  }
-
-  const std::shared_ptr<Field>& field(int i) const {
-    return fields_[i];
-  }
-
-  int num_children() const {
-    return fields_.size();
+  explicit StructType(const std::vector<std::shared_ptr<Field>>& fields)
+      : DataType(Type::STRUCT) {
+    children_ = fields;
   }
 
   std::string ToString() const override;
diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
index 57cabdefd2525..595b3be6e1661 100644
--- a/cpp/src/arrow/types/CMakeLists.txt
+++ b/cpp/src/arrow/types/CMakeLists.txt
@@ -26,8 +26,6 @@ install(FILES
   construct.h
   datetime.h
   decimal.h
-  floating.h
-  integer.h
   json.h
   list.h
   primitive.h
diff --git a/cpp/src/arrow/types/boolean.h b/cpp/src/arrow/types/boolean.h
index a5023d7b368d2..1cb91f9ba4966 100644
--- a/cpp/src/arrow/types/boolean.h
+++ b/cpp/src/arrow/types/boolean.h
@@ -22,7 +22,7 @@
 
 namespace arrow {
 
-typedef PrimitiveArrayImpl<BooleanType> BooleanArray;
+// typedef PrimitiveArrayImpl<BooleanType> BooleanArray;
 
 class BooleanBuilder : public ArrayBuilder {
 };
diff --git a/cpp/src/arrow/types/collection.h b/cpp/src/arrow/types/collection.h
index 42a9c926bb134..46d84f1f183c8 100644
--- a/cpp/src/arrow/types/collection.h
+++ b/cpp/src/arrow/types/collection.h
@@ -25,7 +25,7 @@
 
 namespace arrow {
 
-template <LogicalType::type T>
+template <Type::type T>
 struct CollectionType : public DataType {
   std::vector<TypePtr> child_types_;
 
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 43f01a3051385..290decd81ff42 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -19,24 +19,26 @@
 
 #include <memory>
 
-#include "arrow/types/floating.h"
-#include "arrow/types/integer.h"
+#include "arrow/type.h"
+#include "arrow/types/primitive.h"
 #include "arrow/types/list.h"
 #include "arrow/types/string.h"
+#include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
 
 class ArrayBuilder;
 
-// Initially looked at doing this with vtables, but shared pointers makes it
-// difficult
-
 #define BUILDER_CASE(ENUM, BuilderType)         \
-    case LogicalType::ENUM:                     \
+    case Type::ENUM:                            \
       out->reset(new BuilderType(pool, type));  \
       return Status::OK();
 
+// Initially looked at doing this with vtables, but shared pointers makes it
+// difficult
+//
+// TODO(wesm): come up with a less monolithic strategy
 Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     std::shared_ptr<ArrayBuilder>* out) {
   switch (type->type) {
@@ -56,30 +58,41 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 
     BUILDER_CASE(STRING, StringBuilder);
 
-    case LogicalType::LIST:
+    case Type::LIST:
       {
         std::shared_ptr<ArrayBuilder> value_builder;
 
         const std::shared_ptr<DataType>& value_type = static_cast<ListType*>(
-            type.get())->value_type;
+            type.get())->value_type();
         RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
         out->reset(new ListBuilder(pool, type, value_builder));
         return Status::OK();
       }
-    // BUILDER_CASE(CHAR, CharBuilder);
-
-    // BUILDER_CASE(VARCHAR, VarcharBuilder);
-    // BUILDER_CASE(BINARY, BinaryBuilder);
-
-    // BUILDER_CASE(DATE, DateBuilder);
-    // BUILDER_CASE(TIMESTAMP, TimestampBuilder);
-    // BUILDER_CASE(TIME, TimeBuilder);
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+}
 
-    // BUILDER_CASE(LIST, ListBuilder);
-    // BUILDER_CASE(STRUCT, StructBuilder);
-    // BUILDER_CASE(DENSE_UNION, DenseUnionBuilder);
-    // BUILDER_CASE(SPARSE_UNION, SparseUnionBuilder);
+#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                      \
+    case Type::ENUM:                                                    \
+      out->reset(new ArrayType(type, length, data, null_count, nulls)); \
+      return Status::OK();
 
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    int32_t length, const std::shared_ptr<Buffer>& data,
+    int32_t null_count, const std::shared_ptr<Buffer>& nulls,
+    std::shared_ptr<Array>* out) {
+  switch (type->type) {
+    MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(INT8, Int8Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(UINT16, UInt16Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(INT16, Int16Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(UINT32, UInt32Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(INT32, Int32Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
+    MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
     default:
       return Status::NotImplemented(type->ToString());
   }
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index 59ebe1acddc98..089c484c58bee 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -18,19 +18,26 @@
 #ifndef ARROW_TYPES_CONSTRUCT_H
 #define ARROW_TYPES_CONSTRUCT_H
 
+#include <cstdint>
 #include <memory>
 
-#include "arrow/type.h"
-
 namespace arrow {
 
+class Array;
 class ArrayBuilder;
+class Buffer;
+struct DataType;
 class MemoryPool;
 class Status;
 
 Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     std::shared_ptr<ArrayBuilder>* out);
 
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    int32_t length, const std::shared_ptr<Buffer>& data,
+    int32_t null_count, const std::shared_ptr<Buffer>& nulls,
+    std::shared_ptr<Array>* out);
+
 } // namespace arrow
 
 #endif // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
index 765fc29dd57ae..e57b66ab46adb 100644
--- a/cpp/src/arrow/types/datetime.h
+++ b/cpp/src/arrow/types/datetime.h
@@ -31,8 +31,8 @@ struct DateType : public DataType {
 
   Unit unit;
 
-  explicit DateType(Unit unit = Unit::DAY, bool nullable = true)
-      : DataType(LogicalType::DATE, nullable),
+  explicit DateType(Unit unit = Unit::DAY)
+      : DataType(Type::DATE),
         unit(unit) {}
 
   DateType(const DateType& other)
@@ -41,10 +41,6 @@ struct DateType : public DataType {
   static char const *name() {
     return "date";
   }
-
-  // virtual std::string ToString() {
-  //   return name();
-  // }
 };
 
 
@@ -58,8 +54,8 @@ struct TimestampType : public DataType {
 
   Unit unit;
 
-  explicit TimestampType(Unit unit = Unit::MILLI, bool nullable = true)
-      : DataType(LogicalType::TIMESTAMP, nullable),
+  explicit TimestampType(Unit unit = Unit::MILLI)
+      : DataType(Type::TIMESTAMP),
         unit(unit) {}
 
   TimestampType(const TimestampType& other)
@@ -68,10 +64,6 @@ struct TimestampType : public DataType {
   static char const *name() {
     return "timestamp";
   }
-
-  // virtual std::string ToString() {
-  //   return name();
-  // }
 };
 
 } // namespace arrow
diff --git a/cpp/src/arrow/types/floating.cc b/cpp/src/arrow/types/floating.cc
deleted file mode 100644
index bde28266e638c..0000000000000
--- a/cpp/src/arrow/types/floating.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/floating.h"
-
-namespace arrow {
-
-} // namespace arrow
diff --git a/cpp/src/arrow/types/integer.cc b/cpp/src/arrow/types/integer.cc
deleted file mode 100644
index 4696536616971..0000000000000
--- a/cpp/src/arrow/types/integer.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/integer.h"
-
-namespace arrow {
-
-} // namespace arrow
diff --git a/cpp/src/arrow/types/integer.h b/cpp/src/arrow/types/integer.h
deleted file mode 100644
index 568419124941f..0000000000000
--- a/cpp/src/arrow/types/integer.h
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_INTEGER_H
-#define ARROW_TYPES_INTEGER_H
-
-#include <cstdint>
-#include <string>
-
-#include "arrow/types/primitive.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-// Array containers
-
-typedef PrimitiveArrayImpl<UInt8Type> UInt8Array;
-typedef PrimitiveArrayImpl<Int8Type> Int8Array;
-
-typedef PrimitiveArrayImpl<UInt16Type> UInt16Array;
-typedef PrimitiveArrayImpl<Int16Type> Int16Array;
-
-typedef PrimitiveArrayImpl<UInt32Type> UInt32Array;
-typedef PrimitiveArrayImpl<Int32Type> Int32Array;
-
-typedef PrimitiveArrayImpl<UInt64Type> UInt64Array;
-typedef PrimitiveArrayImpl<Int64Type> Int64Array;
-
-// Builders
-
-typedef PrimitiveBuilder<UInt8Type, UInt8Array> UInt8Builder;
-typedef PrimitiveBuilder<UInt16Type, UInt16Array> UInt16Builder;
-typedef PrimitiveBuilder<UInt32Type, UInt32Array> UInt32Builder;
-typedef PrimitiveBuilder<UInt64Type, UInt64Array> UInt64Builder;
-
-typedef PrimitiveBuilder<Int8Type, Int8Array> Int8Builder;
-typedef PrimitiveBuilder<Int16Type, Int16Array> Int16Builder;
-typedef PrimitiveBuilder<Int32Type, Int32Array> Int32Builder;
-typedef PrimitiveBuilder<Int64Type, Int64Array> Int64Builder;
-
-} // namespace arrow
-
-#endif // ARROW_TYPES_INTEGER_H
diff --git a/cpp/src/arrow/types/json.cc b/cpp/src/arrow/types/json.cc
index 168e370d51a14..fb731edd6073f 100644
--- a/cpp/src/arrow/types/json.cc
+++ b/cpp/src/arrow/types/json.cc
@@ -20,7 +20,6 @@
 #include <vector>
 
 #include "arrow/type.h"
-#include "arrow/types/string.h"
 #include "arrow/types/union.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/types/json.h b/cpp/src/arrow/types/json.h
index b67fb3807aded..9c850afac0af4 100644
--- a/cpp/src/arrow/types/json.h
+++ b/cpp/src/arrow/types/json.h
@@ -28,8 +28,8 @@ struct JSONScalar : public DataType {
   static TypePtr dense_type;
   static TypePtr sparse_type;
 
-  explicit JSONScalar(bool dense = true, bool nullable = true)
-      : DataType(LogicalType::JSON_SCALAR, nullable),
+  explicit JSONScalar(bool dense = true)
+      : DataType(Type::JSON_SCALAR),
         dense(dense) {}
 };
 
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index 02991de2648e7..eb55ca868eeee 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -15,20 +15,21 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
 #include <cstdlib>
 #include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
+#include "gtest/gtest.h"
+
 #include "arrow/array.h"
+#include "arrow/builder.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/types/construct.h"
-#include "arrow/types/integer.h"
 #include "arrow/types/list.h"
-#include "arrow/types/string.h"
+#include "arrow/types/primitive.h"
 #include "arrow/types/test-common.h"
 #include "arrow/util/status.h"
 
@@ -39,27 +40,24 @@ using std::vector;
 
 namespace arrow {
 
-class ArrayBuilder;
-
 TEST(TypesTest, TestListType) {
   std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
 
   ListType list_type(vt);
-  ASSERT_EQ(list_type.type, LogicalType::LIST);
+  ASSERT_EQ(list_type.type, Type::LIST);
 
   ASSERT_EQ(list_type.name(), string("list"));
-  ASSERT_EQ(list_type.ToString(), string("list<uint8>"));
+  ASSERT_EQ(list_type.ToString(), string("list<item: uint8>"));
 
-  ASSERT_EQ(list_type.value_type->type, vt->type);
-  ASSERT_EQ(list_type.value_type->type, vt->type);
+  ASSERT_EQ(list_type.value_type()->type, vt->type);
+  ASSERT_EQ(list_type.value_type()->type, vt->type);
 
-  std::shared_ptr<DataType> st = std::make_shared<StringType>(false);
-  std::shared_ptr<DataType> lt = std::make_shared<ListType>(st, false);
-  ASSERT_EQ(lt->ToString(), string("list<string not null> not null"));
+  std::shared_ptr<DataType> st = std::make_shared<StringType>();
+  std::shared_ptr<DataType> lt = std::make_shared<ListType>(st);
+  ASSERT_EQ(lt->ToString(), string("list<item: string>"));
 
-  ListType lt2(lt, false);
-  ASSERT_EQ(lt2.ToString(),
-      string("list<list<string not null> not null> not null"));
+  ListType lt2(lt);
+  ASSERT_EQ(lt2.ToString(), string("list<item: list<item: string>>"));
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index 69a79a77fabe0..670ee4da11675 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -19,4 +19,33 @@
 
 namespace arrow {
 
+bool ListArray::EqualsExact(const ListArray& other) const {
+  if (this == &other) return true;
+  if (null_count_ != other.null_count_) {
+    return false;
+  }
+
+  bool equal_offsets = offset_buf_->Equals(*other.offset_buf_,
+      length_ + 1);
+  bool equal_nulls = true;
+  if (null_count_ > 0) {
+    equal_nulls = nulls_->Equals(*other.nulls_,
+        util::bytes_for_bits(length_));
+  }
+
+  if (!(equal_offsets && equal_nulls)) {
+    return false;
+  }
+
+  return values()->Equals(other.values());
+}
+
+bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) return true;
+  if (this->type_enum() != arr->type_enum()) {
+    return false;
+  }
+  return EqualsExact(*static_cast<const ListArray*>(arr.get()));
+}
+
 } // namespace arrow
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 210c76a046c21..141f762458b3b 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -21,12 +21,10 @@
 #include <cstdint>
 #include <cstring>
 #include <memory>
-#include <string>
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/type.h"
-#include "arrow/types/integer.h"
 #include "arrow/types/primitive.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
@@ -38,29 +36,19 @@ class MemoryPool;
 
 class ListArray : public Array {
  public:
-  ListArray() : Array(), offset_buf_(nullptr), offsets_(nullptr) {}
-
   ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
       const ArrayPtr& values,
       int32_t null_count = 0,
-      std::shared_ptr<Buffer> nulls = nullptr) {
-    Init(type, length, offsets, values, null_count, nulls);
-  }
-
-  virtual ~ListArray() {}
-
-  void Init(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
-      const ArrayPtr& values,
-      int32_t null_count = 0,
-      std::shared_ptr<Buffer> nulls = nullptr) {
+      std::shared_ptr<Buffer> nulls = nullptr) :
+      Array(type, length, null_count, nulls) {
     offset_buf_ = offsets;
     offsets_ = offsets == nullptr? nullptr :
       reinterpret_cast<const int32_t*>(offset_buf_->data());
-
     values_ = values;
-    Array::Init(type, length, null_count, nulls);
   }
 
+  virtual ~ListArray() {}
+
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   const std::shared_ptr<Array>& values() const {return values_;}
@@ -77,6 +65,9 @@ class ListArray : public Array {
   int32_t value_offset(int i) { return offsets_[i];}
   int32_t value_length(int i) { return offsets_[i + 1] - offsets_[i];}
 
+  bool EqualsExact(const ListArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+
  protected:
   std::shared_ptr<Buffer> offset_buf_;
   const int32_t* offsets_;
@@ -137,8 +128,6 @@ class ListBuilder : public Int32Builder {
 
   template <typename Container>
   std::shared_ptr<Array> Transfer() {
-    auto result = std::make_shared<Container>();
-
     std::shared_ptr<Array> items = value_builder_->Finish();
 
     // Add final offset if the length is non-zero
@@ -146,8 +135,9 @@ class ListBuilder : public Int32Builder {
       raw_buffer()[length_] = items->length();
     }
 
-    result->Init(type_, length_, values_, items,
+    auto result = std::make_shared<Container>(type_, length_, values_, items,
         null_count_, nulls_);
+
     values_ = nulls_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
 
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index f35a258e2cb57..7eae8cda8c488 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -15,21 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-
 #include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
-#include "arrow/array.h"
+#include "gtest/gtest.h"
+
 #include "arrow/builder.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/boolean.h"
 #include "arrow/types/construct.h"
-#include "arrow/types/floating.h"
-#include "arrow/types/integer.h"
 #include "arrow/types/primitive.h"
 #include "arrow/types/test-common.h"
 #include "arrow/util/bit-util.h"
@@ -43,23 +39,17 @@ using std::vector;
 
 namespace arrow {
 
-TEST(TypesTest, TestBytesType) {
-  BytesType t1(3);
-
-  ASSERT_EQ(t1.type, LayoutEnum::BYTE);
-  ASSERT_EQ(t1.size, 3);
-}
-
+class Array;
 
 #define PRIMITIVE_TEST(KLASS, ENUM, NAME)       \
   TEST(TypesTest, TestPrimitive_##ENUM) {       \
     KLASS tp;                                   \
                                                 \
-    ASSERT_EQ(tp.type, LogicalType::ENUM);      \
+    ASSERT_EQ(tp.type, Type::ENUM);             \
     ASSERT_EQ(tp.name(), string(NAME));         \
                                                 \
     KLASS tp_copy = tp;                         \
-    ASSERT_EQ(tp_copy.type, LogicalType::ENUM); \
+    ASSERT_EQ(tp_copy.type, Type::ENUM);        \
   }
 
 PRIMITIVE_TEST(Int8Type, INT8, "int8");
@@ -109,22 +99,20 @@ class TestPrimitiveBuilder : public TestBuilder {
 
   void RandomData(int N, double pct_null = 0.1) {
     Attrs::draw(N, &draws_);
-    random_nulls(N, pct_null, &nulls_);
+    test::random_nulls(N, pct_null, &nulls_);
   }
 
   void CheckNullable() {
-    ArrayType expected;
     int size = builder_->length();
 
     auto ex_data = std::make_shared<Buffer>(
         reinterpret_cast<uint8_t*>(draws_.data()),
         size * sizeof(T));
 
-    auto ex_nulls = bytes_to_null_buffer(nulls_.data(), size);
-
-    int32_t ex_null_count = null_count(nulls_);
+    auto ex_nulls = test::bytes_to_null_buffer(nulls_.data(), size);
+    int32_t ex_null_count = test::null_count(nulls_);
 
-    expected.Init(size, ex_data, ex_null_count, ex_nulls);
+    auto expected = std::make_shared<ArrayType>(size, ex_data, ex_null_count, ex_nulls);
 
     std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(
         builder_->Finish());
@@ -135,18 +123,17 @@ class TestPrimitiveBuilder : public TestBuilder {
     ASSERT_EQ(0, builder_->null_count());
     ASSERT_EQ(nullptr, builder_->buffer());
 
-    ASSERT_TRUE(result->Equals(expected));
+    ASSERT_TRUE(result->EqualsExact(*expected.get()));
     ASSERT_EQ(ex_null_count, result->null_count());
   }
 
   void CheckNonNullable() {
-    ArrayType expected;
     int size = builder_nn_->length();
 
     auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
         size * sizeof(T));
 
-    expected.Init(size, ex_data);
+    auto expected = std::make_shared<ArrayType>(size, ex_data);
 
     std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(
         builder_nn_->Finish());
@@ -156,7 +143,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     ASSERT_EQ(0, builder_nn_->capacity());
     ASSERT_EQ(nullptr, builder_nn_->buffer());
 
-    ASSERT_TRUE(result->Equals(expected));
+    ASSERT_TRUE(result->EqualsExact(*expected.get()));
     ASSERT_EQ(0, result->null_count());
   }
 
@@ -183,8 +170,8 @@ class TestPrimitiveBuilder : public TestBuilder {
 #define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
   struct P##CapType {                               \
     PTYPE_DECL(CapType, c_type);                    \
-    static void draw(int N, vector<T>* draws) {  \
-      randint<T>(N, LOWER, UPPER, draws);           \
+    static void draw(int N, vector<T>* draws) {     \
+      test::randint<T>(N, LOWER, UPPER, draws);     \
     }                                               \
   }
 
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index c86260b0fc641..32b8bfa7f1bd4 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -26,16 +26,16 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Primitive array base
 
-void PrimitiveArray::Init(const TypePtr& type, int32_t length,
+PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
     const std::shared_ptr<Buffer>& data,
     int32_t null_count,
-    const std::shared_ptr<Buffer>& nulls) {
-  Array::Init(type, length, null_count, nulls);
+    const std::shared_ptr<Buffer>& nulls) :
+    Array(type, length, null_count, nulls) {
   data_ = data;
   raw_data_ = data == nullptr? nullptr : data_->data();
 }
 
-bool PrimitiveArray::Equals(const PrimitiveArray& other) const {
+bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
   if (this == &other) return true;
   if (null_count_ != other.null_count_) {
     return false;
@@ -50,4 +50,12 @@ bool PrimitiveArray::Equals(const PrimitiveArray& other) const {
   }
 }
 
+bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) return true;
+  if (this->type_enum() != arr->type_enum()) {
+    return false;
+  }
+  return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
+}
+
 } // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 22ab59c309a1d..e01027cf55c39 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -21,7 +21,6 @@
 #include <cstdint>
 #include <cstring>
 #include <memory>
-#include <string>
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
@@ -38,64 +37,57 @@ class MemoryPool;
 // Base class for fixed-size logical types
 class PrimitiveArray : public Array {
  public:
-  PrimitiveArray() : Array(), data_(nullptr), raw_data_(nullptr) {}
-
-  virtual ~PrimitiveArray() {}
-
-  void Init(const TypePtr& type, int32_t length,
+  PrimitiveArray(const TypePtr& type, int32_t length,
       const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0,
       const std::shared_ptr<Buffer>& nulls = nullptr);
+  virtual ~PrimitiveArray() {}
 
   const std::shared_ptr<Buffer>& data() const { return data_;}
 
-  bool Equals(const PrimitiveArray& other) const;
+  bool EqualsExact(const PrimitiveArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
 
  protected:
   std::shared_ptr<Buffer> data_;
   const uint8_t* raw_data_;
 };
 
-
-template <typename TypeClass>
-class PrimitiveArrayImpl : public PrimitiveArray {
- public:
-  typedef typename TypeClass::c_type value_type;
-
-  PrimitiveArrayImpl() : PrimitiveArray() {}
-
-  virtual ~PrimitiveArrayImpl() {}
-
-  PrimitiveArrayImpl(int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr) {
-    Init(length, data, null_count, nulls);
-  }
-
-  void Init(int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr) {
-    TypePtr type(new TypeClass());
-    PrimitiveArray::Init(type, length, data, null_count, nulls);
-  }
-
-  bool Equals(const PrimitiveArrayImpl& other) const {
-    return PrimitiveArray::Equals(*static_cast<const PrimitiveArray*>(&other));
-  }
-
-  const value_type* raw_data() const {
-    return reinterpret_cast<const value_type*>(raw_data_);
-  }
-
-  value_type Value(int i) const {
-    return raw_data()[i];
-  }
-
-  TypeClass* exact_type() const {
-    return static_cast<TypeClass*>(type_);
-  }
+#define NUMERIC_ARRAY_DECL(NAME, TypeClass, T)                      \
+class NAME : public PrimitiveArray {                                \
+ public:                                                            \
+  using value_type = T;                                             \
+  using PrimitiveArray::PrimitiveArray;                             \
+  NAME(int32_t length, const std::shared_ptr<Buffer>& data,         \
+      int32_t null_count = 0,                                       \
+      const std::shared_ptr<Buffer>& nulls = nullptr) :             \
+      PrimitiveArray(std::make_shared<TypeClass>(), length, data,   \
+          null_count, nulls) {}                                     \
+                                                                    \
+  bool EqualsExact(const NAME& other) const {                       \
+    return PrimitiveArray::EqualsExact(                             \
+        *static_cast<const PrimitiveArray*>(&other));               \
+  }                                                                 \
+                                                                    \
+  const T* raw_data() const {                                       \
+    return reinterpret_cast<const T*>(raw_data_);                   \
+  }                                                                 \
+                                                                    \
+  T Value(int i) const {                                            \
+    return raw_data()[i];                                           \
+  }                                                                 \
 };
 
+NUMERIC_ARRAY_DECL(UInt8Array, UInt8Type, uint8_t);
+NUMERIC_ARRAY_DECL(Int8Array, Int8Type, int8_t);
+NUMERIC_ARRAY_DECL(UInt16Array, UInt16Type, uint16_t);
+NUMERIC_ARRAY_DECL(Int16Array, Int16Type, int16_t);
+NUMERIC_ARRAY_DECL(UInt32Array, UInt32Type, uint32_t);
+NUMERIC_ARRAY_DECL(Int32Array, Int32Type, int32_t);
+NUMERIC_ARRAY_DECL(UInt64Array, UInt64Type, uint64_t);
+NUMERIC_ARRAY_DECL(Int64Array, Int64Type, int64_t);
+NUMERIC_ARRAY_DECL(FloatArray, FloatType, float);
+NUMERIC_ARRAY_DECL(DoubleArray, DoubleType, double);
 
 template <typename Type, typename ArrayType>
 class PrimitiveBuilder : public ArrayBuilder {
@@ -202,8 +194,9 @@ class PrimitiveBuilder : public ArrayBuilder {
   }
 
   std::shared_ptr<Array> Finish() override {
-    std::shared_ptr<ArrayType> result = std::make_shared<ArrayType>();
-    result->PrimitiveArray::Init(type_, length_, values_, null_count_, nulls_);
+    std::shared_ptr<ArrayType> result = std::make_shared<ArrayType>(
+        type_, length_, values_, null_count_, nulls_);
+
     values_ = nulls_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
     return result;
@@ -222,6 +215,21 @@ class PrimitiveBuilder : public ArrayBuilder {
   int elsize_;
 };
 
+// Builders
+
+typedef PrimitiveBuilder<UInt8Type, UInt8Array> UInt8Builder;
+typedef PrimitiveBuilder<UInt16Type, UInt16Array> UInt16Builder;
+typedef PrimitiveBuilder<UInt32Type, UInt32Array> UInt32Builder;
+typedef PrimitiveBuilder<UInt64Type, UInt64Array> UInt64Builder;
+
+typedef PrimitiveBuilder<Int8Type, Int8Array> Int8Builder;
+typedef PrimitiveBuilder<Int16Type, Int16Array> Int16Builder;
+typedef PrimitiveBuilder<Int32Type, Int32Array> Int32Builder;
+typedef PrimitiveBuilder<Int64Type, Int64Array> Int64Builder;
+
+typedef PrimitiveBuilder<FloatType, FloatArray> FloatBuilder;
+typedef PrimitiveBuilder<DoubleType, DoubleArray> DoubleBuilder;
+
 } // namespace arrow
 
 #endif  // ARROW_TYPES_PRIMITIVE_H
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index 6381093dcbb45..7dc3d682cdc15 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -15,21 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
 #include <cstdint>
+#include <cstdlib>
 #include <memory>
 #include <string>
 #include <vector>
 
+#include "gtest/gtest.h"
+
 #include "arrow/array.h"
-#include "arrow/builder.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/integer.h"
+#include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
 #include "arrow/types/test-common.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
@@ -38,14 +37,14 @@ class Buffer;
 TEST(TypesTest, TestCharType) {
   CharType t1(5);
 
-  ASSERT_EQ(t1.type, LogicalType::CHAR);
+  ASSERT_EQ(t1.type, Type::CHAR);
   ASSERT_EQ(t1.size, 5);
 
   ASSERT_EQ(t1.ToString(), std::string("char(5)"));
 
   // Test copy constructor
   CharType t2 = t1;
-  ASSERT_EQ(t2.type, LogicalType::CHAR);
+  ASSERT_EQ(t2.type, Type::CHAR);
   ASSERT_EQ(t2.size, 5);
 }
 
@@ -53,22 +52,20 @@ TEST(TypesTest, TestCharType) {
 TEST(TypesTest, TestVarcharType) {
   VarcharType t1(5);
 
-  ASSERT_EQ(t1.type, LogicalType::VARCHAR);
+  ASSERT_EQ(t1.type, Type::VARCHAR);
   ASSERT_EQ(t1.size, 5);
-  ASSERT_EQ(t1.physical_type.size, 6);
 
   ASSERT_EQ(t1.ToString(), std::string("varchar(5)"));
 
   // Test copy constructor
   VarcharType t2 = t1;
-  ASSERT_EQ(t2.type, LogicalType::VARCHAR);
+  ASSERT_EQ(t2.type, Type::VARCHAR);
   ASSERT_EQ(t2.size, 5);
-  ASSERT_EQ(t2.physical_type.size, 6);
 }
 
 TEST(TypesTest, TestStringType) {
   StringType str;
-  ASSERT_EQ(str.type, LogicalType::STRING);
+  ASSERT_EQ(str.type, Type::STRING);
   ASSERT_EQ(str.name(), std::string("string"));
 }
 
@@ -90,15 +87,16 @@ class TestStringContainer : public ::testing::Test  {
     length_ = offsets_.size() - 1;
     int nchars = chars_.size();
 
-    value_buf_ = to_buffer(chars_);
+    value_buf_ = test::to_buffer(chars_);
     values_ = ArrayPtr(new UInt8Array(nchars, value_buf_));
 
-    offsets_buf_ = to_buffer(offsets_);
+    offsets_buf_ = test::to_buffer(offsets_);
 
-    nulls_buf_ = bytes_to_null_buffer(nulls_.data(), nulls_.size());
-    null_count_ = null_count(nulls_);
+    nulls_buf_ = test::bytes_to_null_buffer(nulls_.data(), nulls_.size());
+    null_count_ = test::null_count(nulls_);
 
-    strings_.Init(length_, offsets_buf_, values_, null_count_, nulls_buf_);
+    strings_ = std::make_shared<StringArray>(length_, offsets_buf_, values_,
+        null_count_, nulls_buf_);
   }
 
  protected:
@@ -116,28 +114,28 @@ class TestStringContainer : public ::testing::Test  {
   int length_;
 
   ArrayPtr values_;
-  StringArray strings_;
+  std::shared_ptr<StringArray> strings_;
 };
 
 
 TEST_F(TestStringContainer, TestArrayBasics) {
-  ASSERT_EQ(length_, strings_.length());
-  ASSERT_EQ(1, strings_.null_count());
+  ASSERT_EQ(length_, strings_->length());
+  ASSERT_EQ(1, strings_->null_count());
 }
 
 TEST_F(TestStringContainer, TestType) {
-  TypePtr type = strings_.type();
+  TypePtr type = strings_->type();
 
-  ASSERT_EQ(LogicalType::STRING, type->type);
-  ASSERT_EQ(LogicalType::STRING, strings_.logical_type());
+  ASSERT_EQ(Type::STRING, type->type);
+  ASSERT_EQ(Type::STRING, strings_->type_enum());
 }
 
 
 TEST_F(TestStringContainer, TestListFunctions) {
   int pos = 0;
   for (size_t i = 0; i < expected_.size(); ++i) {
-    ASSERT_EQ(pos, strings_.value_offset(i));
-    ASSERT_EQ(expected_[i].size(), strings_.value_length(i));
+    ASSERT_EQ(pos, strings_->value_offset(i));
+    ASSERT_EQ(expected_[i].size(), strings_->value_length(i));
     pos += expected_[i].size();
   }
 }
@@ -151,9 +149,9 @@ TEST_F(TestStringContainer, TestDestructor) {
 TEST_F(TestStringContainer, TestGetString) {
   for (size_t i = 0; i < expected_.size(); ++i) {
     if (nulls_[i]) {
-      ASSERT_TRUE(strings_.IsNull(i));
+      ASSERT_TRUE(strings_->IsNull(i));
     } else {
-      ASSERT_EQ(expected_[i], strings_.GetString(i));
+      ASSERT_EQ(expected_[i], strings_->GetString(i));
     }
   }
 }
@@ -199,7 +197,7 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
   Done();
 
   ASSERT_EQ(reps * N, result_->length());
-  ASSERT_EQ(reps * null_count(is_null), result_->null_count());
+  ASSERT_EQ(reps * test::null_count(is_null), result_->null_count());
   ASSERT_EQ(reps * 6, result_->values()->length());
 
   int32_t length;
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index 8ccc0a9698a54..2b3fba5ce0932 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -25,25 +25,21 @@
 
 #include "arrow/array.h"
 #include "arrow/type.h"
-#include "arrow/types/integer.h"
 #include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
 
-class ArrayBuilder;
 class Buffer;
 class MemoryPool;
 
 struct CharType : public DataType {
   int size;
 
-  BytesType physical_type;
-
-  explicit CharType(int size, bool nullable = true)
-      : DataType(LogicalType::CHAR, nullable),
-        size(size),
-        physical_type(BytesType(size)) {}
+  explicit CharType(int size)
+      : DataType(Type::CHAR),
+        size(size) {}
 
   CharType(const CharType& other)
       : CharType(other.size) {}
@@ -56,54 +52,36 @@ struct CharType : public DataType {
 struct VarcharType : public DataType {
   int size;
 
-  BytesType physical_type;
-
-  explicit VarcharType(int size, bool nullable = true)
-      : DataType(LogicalType::VARCHAR, nullable),
-        size(size),
-        physical_type(BytesType(size + 1)) {}
+  explicit VarcharType(int size)
+      : DataType(Type::VARCHAR),
+        size(size) {}
   VarcharType(const VarcharType& other)
       : VarcharType(other.size) {}
 
   virtual std::string ToString() const;
 };
 
-static const LayoutPtr byte1(new BytesType(1));
-static const LayoutPtr physical_string = LayoutPtr(new ListLayoutType(byte1));
-
 // TODO: add a BinaryArray layer in between
 class StringArray : public ListArray {
  public:
-  StringArray() : ListArray(), bytes_(nullptr), raw_bytes_(nullptr) {}
-
-  StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const ArrayPtr& values,
-      int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr) {
-    Init(length, offsets, values, null_count, nulls);
-  }
-
-  void Init(const TypePtr& type, int32_t length,
+  StringArray(const TypePtr& type, int32_t length,
       const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values,
       int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr) {
-    ListArray::Init(type, length, offsets, values, null_count, nulls);
-
-    // TODO: type validation for values array
-
+      const std::shared_ptr<Buffer>& nulls = nullptr) :
+      ListArray(type, length, offsets, values, null_count, nulls) {
     // For convenience
     bytes_ = static_cast<UInt8Array*>(values.get());
     raw_bytes_ = bytes_->raw_data();
   }
 
-  void Init(int32_t length, const std::shared_ptr<Buffer>& offsets,
+  StringArray(int32_t length,
+      const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values,
       int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr) {
-    TypePtr type(new StringType());
-    Init(type, length, offsets, values, null_count, nulls);
-  }
+      const std::shared_ptr<Buffer>& nulls = nullptr) :
+      StringArray(std::make_shared<StringType>(), length, offsets, values,
+          null_count, nulls) {}
 
   // Compute the pointer t
   const uint8_t* GetValue(int i, int32_t* out_length) const {
@@ -125,9 +103,6 @@ class StringArray : public ListArray {
 };
 
 // Array builder
-
-
-
 class StringBuilder : public ListBuilder {
  public:
   explicit StringBuilder(MemoryPool* pool, const TypePtr& type) :
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index 9a4777e8b983d..d94396f42c52a 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -15,16 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-
 #include <memory>
 #include <string>
 #include <vector>
 
+#include "gtest/gtest.h"
+
 #include "arrow/type.h"
-#include "arrow/types/integer.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
 
 using std::shared_ptr;
 using std::string;
@@ -42,13 +39,13 @@ TEST(TestStructType, Basics) {
   TypePtr f2_type = TypePtr(new UInt8Type());
   auto f2 = std::make_shared<Field>("f2", f2_type);
 
-  vector<shared_ptr<Field> > fields = {f0, f1, f2};
+  vector<shared_ptr<Field>> fields = {f0, f1, f2};
 
   StructType struct_type(fields);
 
-  ASSERT_TRUE(struct_type.field(0)->Equals(f0));
-  ASSERT_TRUE(struct_type.field(1)->Equals(f1));
-  ASSERT_TRUE(struct_type.field(2)->Equals(f2));
+  ASSERT_TRUE(struct_type.child(0)->Equals(f0));
+  ASSERT_TRUE(struct_type.child(1)->Equals(f1));
+  ASSERT_TRUE(struct_type.child(2)->Equals(f2));
 
   ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
 
diff --git a/cpp/src/arrow/types/test-common.h b/cpp/src/arrow/types/test-common.h
index 1744efce7d631..227aca632ef3c 100644
--- a/cpp/src/arrow/types/test-common.h
+++ b/cpp/src/arrow/types/test-common.h
@@ -18,11 +18,12 @@
 #ifndef ARROW_TYPES_TEST_COMMON_H
 #define ARROW_TYPES_TEST_COMMON_H
 
-#include <gtest/gtest.h>
 #include <memory>
 #include <string>
 #include <vector>
 
+#include "gtest/gtest.h"
+
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/util/memory-pool.h"
@@ -34,7 +35,7 @@ namespace arrow {
 class TestBuilder : public ::testing::Test {
  public:
   void SetUp() {
-    pool_ = GetDefaultMemoryPool();
+    pool_ = default_memory_pool();
     type_ = TypePtr(new UInt8Type());
     builder_.reset(new UInt8Builder(pool_, type_));
     builder_nn_.reset(new UInt8Builder(pool_, type_));
diff --git a/cpp/src/arrow/types/union.h b/cpp/src/arrow/types/union.h
index 9aff780c6a392..29cda90b972dd 100644
--- a/cpp/src/arrow/types/union.h
+++ b/cpp/src/arrow/types/union.h
@@ -30,8 +30,8 @@ namespace arrow {
 
 class Buffer;
 
-struct DenseUnionType : public CollectionType<LogicalType::DENSE_UNION> {
-  typedef CollectionType<LogicalType::DENSE_UNION> Base;
+struct DenseUnionType : public CollectionType<Type::DENSE_UNION> {
+  typedef CollectionType<Type::DENSE_UNION> Base;
 
   explicit DenseUnionType(const std::vector<TypePtr>& child_types) :
       Base() {
@@ -42,8 +42,8 @@ struct DenseUnionType : public CollectionType<LogicalType::DENSE_UNION> {
 };
 
 
-struct SparseUnionType : public CollectionType<LogicalType::SPARSE_UNION> {
-  typedef CollectionType<LogicalType::SPARSE_UNION> Base;
+struct SparseUnionType : public CollectionType<Type::SPARSE_UNION> {
+  typedef CollectionType<Type::SPARSE_UNION> Base;
 
   explicit SparseUnionType(const std::vector<TypePtr>& child_types) :
       Base() {
@@ -55,28 +55,20 @@ struct SparseUnionType : public CollectionType<LogicalType::SPARSE_UNION> {
 
 
 class UnionArray : public Array {
- public:
-  UnionArray() : Array() {}
-
  protected:
   // The data are types encoded as int16
   Buffer* types_;
-  std::vector<std::shared_ptr<Array> > children_;
+  std::vector<std::shared_ptr<Array>> children_;
 };
 
 
 class DenseUnionArray : public UnionArray {
- public:
-  DenseUnionArray() : UnionArray() {}
-
  protected:
   Buffer* offset_buf_;
 };
 
 
 class SparseUnionArray : public UnionArray {
- public:
-  SparseUnionArray() : UnionArray() {}
 };
 
 } // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index 7506ca5b5531c..220bff084fd6e 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -15,10 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-
 #include "arrow/util/bit-util.h"
 
+#include "gtest/gtest.h"
+
 namespace arrow {
 
 TEST(UtilTests, TestNextPower2) {
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 5e7197f901222..1d2d1d5f9d7e4 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -19,7 +19,6 @@
 #define ARROW_UTIL_BIT_UTIL_H
 
 #include <cstdint>
-#include <cstdlib>
 #include <memory>
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/buffer-test.cc b/cpp/src/arrow/util/buffer-test.cc
index 9f1fd91432b4d..1d58226d84a46 100644
--- a/cpp/src/arrow/util/buffer-test.cc
+++ b/cpp/src/arrow/util/buffer-test.cc
@@ -15,11 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
 #include <cstdint>
 #include <limits>
 #include <string>
 
+#include "gtest/gtest.h"
+
 #include "arrow/test-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
index 50f4716769d70..04cdcd75cd41a 100644
--- a/cpp/src/arrow/util/buffer.cc
+++ b/cpp/src/arrow/util/buffer.cc
@@ -40,7 +40,7 @@ std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
 PoolBuffer::PoolBuffer(MemoryPool* pool) :
     ResizableBuffer(nullptr, 0) {
   if (pool == nullptr) {
-    pool = GetDefaultMemoryPool();
+    pool = default_memory_pool();
   }
   pool_ = pool;
 }
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
index 954b5f951b558..6ef07a07ada3f 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -15,10 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
 #include <cstdint>
 #include <limits>
 
+#include "gtest/gtest.h"
+
 #include "arrow/test-util.h"
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
@@ -26,7 +27,7 @@
 namespace arrow {
 
 TEST(DefaultMemoryPool, MemoryTracking) {
-  MemoryPool* pool = GetDefaultMemoryPool();
+  MemoryPool* pool = default_memory_pool();
 
   uint8_t* data;
   ASSERT_OK(pool->Allocate(100, &data));
@@ -37,7 +38,7 @@ TEST(DefaultMemoryPool, MemoryTracking) {
 }
 
 TEST(DefaultMemoryPool, OOM) {
-  MemoryPool* pool = GetDefaultMemoryPool();
+  MemoryPool* pool = default_memory_pool();
 
   uint8_t* data;
   int64_t to_alloc = std::numeric_limits<int64_t>::max();
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
index 5820346e5a739..0b885e9376a62 100644
--- a/cpp/src/arrow/util/memory-pool.cc
+++ b/cpp/src/arrow/util/memory-pool.cc
@@ -70,9 +70,9 @@ void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
 
 InternalMemoryPool::~InternalMemoryPool() {}
 
-MemoryPool* GetDefaultMemoryPool() {
-  static InternalMemoryPool default_memory_pool;
-  return &default_memory_pool;
+MemoryPool* default_memory_pool() {
+  static InternalMemoryPool default_memory_pool_;
+  return &default_memory_pool_;
 }
 
 } // namespace arrow
diff --git a/cpp/src/arrow/util/memory-pool.h b/cpp/src/arrow/util/memory-pool.h
index a7cb10dae1703..0d2478686f5a4 100644
--- a/cpp/src/arrow/util/memory-pool.h
+++ b/cpp/src/arrow/util/memory-pool.h
@@ -34,7 +34,7 @@ class MemoryPool {
   virtual int64_t bytes_allocated() const = 0;
 };
 
-MemoryPool* GetDefaultMemoryPool();
+MemoryPool* default_memory_pool();
 
 } // namespace arrow
 
diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/util/status.cc
index c6e113ebea590..43cb87e1a8c56 100644
--- a/cpp/src/arrow/util/status.cc
+++ b/cpp/src/arrow/util/status.cc
@@ -54,6 +54,9 @@ std::string Status::CodeAsString() const {
     case StatusCode::Invalid:
       type = "Invalid";
       break;
+    case StatusCode::IOError:
+      type = "IOError";
+      break;
     case StatusCode::NotImplemented:
       type = "NotImplemented";
       break;
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
index 47fda40db2596..b5931232dbdcb 100644
--- a/cpp/src/arrow/util/status.h
+++ b/cpp/src/arrow/util/status.h
@@ -63,6 +63,7 @@ enum class StatusCode: char {
   OutOfMemory = 1,
   KeyError = 2,
   Invalid = 3,
+  IOError = 4,
 
   NotImplemented = 10,
 };
@@ -97,12 +98,17 @@ class Status {
     return Status(StatusCode::Invalid, msg, -1);
   }
 
+  static Status IOError(const std::string& msg) {
+    return Status(StatusCode::IOError, msg, -1);
+  }
+
   // Returns true iff the status indicates success.
   bool ok() const { return (state_ == NULL); }
 
   bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; }
   bool IsKeyError() const { return code() == StatusCode::KeyError; }
   bool IsInvalid() const { return code() == StatusCode::Invalid; }
+  bool IsIOError() const { return code() == StatusCode::IOError; }
 
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
diff --git a/cpp/src/arrow/util/test_main.cc b/cpp/src/arrow/util/test_main.cc
index 00139f36742ed..adc8466fb0be9 100644
--- a/cpp/src/arrow/util/test_main.cc
+++ b/cpp/src/arrow/util/test_main.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
+#include "gtest/gtest.h"
 
 int main(int argc, char **argv) {
   ::testing::InitGoogleTest(&argc, argv);
diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
index 294737cc50522..3d5f532b16309 100755
--- a/cpp/thirdparty/build_thirdparty.sh
+++ b/cpp/thirdparty/build_thirdparty.sh
@@ -17,6 +17,7 @@ else
     case $arg in
       "gtest")      F_GTEST=1 ;;
       "gbenchmark")      F_GBENCHMARK=1 ;;
+      "flatbuffers")      F_FLATBUFFERS=1 ;;
       *)            echo "Unknown module: $arg"; exit 1 ;;
     esac
   done
@@ -78,6 +79,14 @@ if [ -n "$F_ALL" -o -n "$F_GBENCHMARK" ]; then
   make VERBOSE=1 install || { echo "make $GBENCHMARK_ERROR" ; exit 1; }
 fi
 
+FLATBUFFERS_ERROR="failed for flatbuffers"
+if [ -n "$F_ALL" -o -n "$F_FLATBUFFERS" ]; then
+  cd $TP_DIR/$FLATBUFFERS_BASEDIR
+
+  CXXFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX -DFLATBUFFERS_BUILD_TESTS=OFF . || { echo "cmake $FLATBUFFERS_ERROR" ; exit 1; }
+  make -j$PARALLEL
+  make install
+fi
 
 echo "---------------------"
 echo "Thirdparty dependencies built and installed into $PREFIX successfully"
diff --git a/cpp/thirdparty/download_thirdparty.sh b/cpp/thirdparty/download_thirdparty.sh
index d22c559b3e3ba..d299afc15222b 100755
--- a/cpp/thirdparty/download_thirdparty.sh
+++ b/cpp/thirdparty/download_thirdparty.sh
@@ -25,3 +25,8 @@ if [ ! -d ${GBENCHMARK_BASEDIR} ]; then
   echo "Fetching google benchmark"
   download_extract_and_cleanup $GBENCHMARK_URL
 fi
+
+if [ ! -d ${FLATBUFFERS_BASEDIR} ]; then
+  echo "Fetching flatbuffers"
+  download_extract_and_cleanup $FLATBUFFERS_URL
+fi
diff --git a/cpp/thirdparty/versions.sh b/cpp/thirdparty/versions.sh
index 9cfc7cd94b58c..cb455b4eadd3b 100755
--- a/cpp/thirdparty/versions.sh
+++ b/cpp/thirdparty/versions.sh
@@ -5,3 +5,7 @@ GTEST_BASEDIR=googletest-release-$GTEST_VERSION
 GBENCHMARK_VERSION=1.0.0
 GBENCHMARK_URL="https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
 GBENCHMARK_BASEDIR=benchmark-$GBENCHMARK_VERSION
+
+FLATBUFFERS_VERSION=1.3.0
+FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
+FLATBUFFERS_BASEDIR=flatbuffers-$FLATBUFFERS_VERSION
diff --git a/format/Message.fbs b/format/Message.fbs
new file mode 100644
index 0000000000000..3ffd20332087a
--- /dev/null
+++ b/format/Message.fbs
@@ -0,0 +1,183 @@
+namespace apache.arrow.flatbuf;
+
+/// ----------------------------------------------------------------------
+/// Logical types and their metadata (if any)
+///
+/// These are stored in the flatbuffer in the Type union below
+
+/// A Tuple in the flatbuffer metadata is the same as an Arrow Struct
+/// (according to the physical memory layout). We used Tuple here as Struct is
+/// a reserved word in Flatbuffers
+table Tuple {
+}
+
+table List {
+}
+
+enum UnionMode:int { Sparse, Dense }
+
+table Union {
+  mode: UnionMode;
+}
+
+table Bit {
+}
+
+table Int {
+  bitWidth: int; // 1 to 64
+  is_signed: bool;
+}
+
+enum Precision:int {SINGLE, DOUBLE}
+
+table FloatingPoint {
+  precision: Precision;
+}
+
+table Utf8 {
+}
+
+table Binary {
+}
+
+table Bool {
+}
+
+table Decimal {
+  precision: int;
+  scale: int;
+}
+
+table Timestamp {
+  timezone: string;
+}
+
+table JSONScalar {
+  dense:bool=true;
+}
+
+/// ----------------------------------------------------------------------
+/// Top-level Type value, enabling extensible type-specific metadata. We can
+/// add new logical types to Type without breaking backwards compatibility
+
+union Type {
+  Int,
+  Bit,
+  FloatingPoint,
+  Binary,
+  Utf8,
+  Bool,
+  Decimal,
+  Timestamp,
+  List,
+  Tuple,
+  Union,
+  JSONScalar
+}
+
+/// ----------------------------------------------------------------------
+/// A field represents a named column in a record / row batch or child of a
+/// nested type.
+///
+/// - children is only for nested Arrow arrays
+/// - For primitive types, children will have length 0
+/// - nullable should default to true in general
+
+table Field {
+  // Name is not required, in i.e. a List
+  name: string;
+  nullable: bool;
+  type: Type;
+  children: [Field];
+}
+
+/// ----------------------------------------------------------------------
+/// A Schema describes the columns in a row batch
+
+table Schema {
+  fields: [Field];
+}
+
+/// ----------------------------------------------------------------------
+/// Data structures for describing a table row batch (a collection of
+/// equal-length Arrow arrays)
+
+/// A Buffer represents a single contiguous memory segment
+struct Buffer {
+  /// The shared memory page id where this buffer is located. Currently this is
+  /// not used
+  page: int;
+
+  /// The relative offset into the shared memory page where the bytes for this
+  /// buffer starts
+  offset: long;
+
+  /// The absolute length (in bytes) of the memory buffer. The memory is found
+  /// from offset (inclusive) to offset + length (non-inclusive).
+  length: long;
+}
+
+/// Metadata about a field at some level of a nested type tree (but not
+/// its children).
+///
+/// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
+/// would have {length: 5, null_count: 2} for its List node, and {length: 6,
+/// null_count: 0} for its Int16 node, as separate FieldNode structs
+struct FieldNode {
+  /// The number of value slots in the Arrow array at this level of a nested
+  /// tree
+  length: int;
+
+  /// The number of observed nulls. Fields with null_count == 0 may choose not
+  /// to write their physical null bitmap out as a materialized buffer, instead
+  /// setting the length of the null buffer to 0.
+  null_count: int;
+}
+
+/// A data header describing the shared memory layout of a "record" or "row"
+/// batch. Some systems call this a "row batch" internally and others a "record
+/// batch".
+table RecordBatch {
+  /// number of records / rows. The arrays in the batch should all have this
+  /// length
+  length: int;
+
+  /// Nodes correspond to the pre-ordered flattened logical schema
+  nodes: [FieldNode];
+
+  /// Buffers correspond to the pre-ordered flattened buffer tree
+  ///
+  /// The number of buffers appended to this list depends on the schema. For
+  /// example, most primitive arrays will have 2 buffers, 1 for the null bitmap
+  /// and 1 for the values. For struct arrays, there will only be a single
+  /// buffer for the null bitmap
+  buffers: [Buffer];
+}
+
+/// ----------------------------------------------------------------------
+/// For sending dictionary encoding information. Any Field can be
+/// dictionary-encoded, but in this case none of its children may be
+/// dictionary-encoded.
+///
+/// TODO(wesm): To be documented in more detail
+
+table DictionaryBatch {
+  id: long;
+  data: RecordBatch;
+}
+
+/// ----------------------------------------------------------------------
+/// The root Message type
+
+/// This union enables us to easily send different message types without
+/// redundant storage, and in the future we can easily add new message types.
+union MessageHeader {
+  Schema, DictionaryBatch, RecordBatch
+}
+
+table Message {
+  header: MessageHeader;
+  bodyLength: long;
+}
+
+root_type Message;
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 8d93a156bcc3d..9a080709bebda 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -35,4 +35,6 @@
                             uint8, uint16, uint32, uint64,
                             float_, double, string,
                             list_, struct, field,
-                            DataType, Field, Schema)
+                            DataType, Field, Schema, schema)
+
+from pyarrow.array import RowBatch
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index d0d3486c032fe..de3c77419623f 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -16,7 +16,7 @@
 # under the License.
 
 from pyarrow.includes.common cimport shared_ptr
-from pyarrow.includes.libarrow cimport CArray, LogicalType
+from pyarrow.includes.libarrow cimport CArray
 
 from pyarrow.scalar import NA
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index bceb333c94ea5..c5d40ddd7a481 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -28,6 +28,9 @@ from pyarrow.error cimport check_status
 cimport pyarrow.scalar as scalar
 from pyarrow.scalar import NA
 
+from pyarrow.schema cimport Schema
+import pyarrow.schema as schema
+
 def total_allocated_bytes():
     cdef MemoryPool* pool = pyarrow.GetMemoryPool()
     return pool.bytes_allocated()
@@ -155,12 +158,12 @@ cdef class StringArray(Array):
 
 
 cdef dict _array_classes = {
-    LogicalType_NA: NullArray,
-    LogicalType_BOOL: BooleanArray,
-    LogicalType_INT64: Int64Array,
-    LogicalType_DOUBLE: DoubleArray,
-    LogicalType_LIST: ListArray,
-    LogicalType_STRING: StringArray,
+    Type_NA: NullArray,
+    Type_BOOL: BooleanArray,
+    Type_INT64: Int64Array,
+    Type_DOUBLE: DoubleArray,
+    Type_LIST: ListArray,
+    Type_STRING: StringArray,
 }
 
 cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
@@ -190,3 +193,35 @@ def from_pylist(object list_obj, DataType type=None):
         raise NotImplementedError
 
     return box_arrow_array(sp_array)
+
+#----------------------------------------------------------------------
+# Table-like data structures
+
+cdef class RowBatch:
+    """
+
+    """
+    cdef readonly:
+        Schema schema
+        int num_rows
+        list arrays
+
+    def __cinit__(self, Schema schema, int num_rows, list arrays):
+        self.schema = schema
+        self.num_rows = num_rows
+        self.arrays = arrays
+
+        if len(self.schema) != len(arrays):
+            raise ValueError('Mismatch number of data arrays and '
+                             'schema fields')
+
+    def __len__(self):
+        return self.num_rows
+
+    property num_columns:
+
+        def __get__(self):
+            return len(self.arrays)
+
+    def __getitem__(self, i):
+        return self.arrays[i]
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index baba112833e0d..e6afcbd79b69f 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -21,31 +21,30 @@ from pyarrow.includes.common cimport *
 
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
-    enum LogicalType" arrow::LogicalType::type":
-        LogicalType_NA" arrow::LogicalType::NA"
+    enum Type" arrow::Type::type":
+        Type_NA" arrow::Type::NA"
 
-        LogicalType_BOOL" arrow::LogicalType::BOOL"
+        Type_BOOL" arrow::Type::BOOL"
 
-        LogicalType_UINT8" arrow::LogicalType::UINT8"
-        LogicalType_INT8" arrow::LogicalType::INT8"
-        LogicalType_UINT16" arrow::LogicalType::UINT16"
-        LogicalType_INT16" arrow::LogicalType::INT16"
-        LogicalType_UINT32" arrow::LogicalType::UINT32"
-        LogicalType_INT32" arrow::LogicalType::INT32"
-        LogicalType_UINT64" arrow::LogicalType::UINT64"
-        LogicalType_INT64" arrow::LogicalType::INT64"
+        Type_UINT8" arrow::Type::UINT8"
+        Type_INT8" arrow::Type::INT8"
+        Type_UINT16" arrow::Type::UINT16"
+        Type_INT16" arrow::Type::INT16"
+        Type_UINT32" arrow::Type::UINT32"
+        Type_INT32" arrow::Type::INT32"
+        Type_UINT64" arrow::Type::UINT64"
+        Type_INT64" arrow::Type::INT64"
 
-        LogicalType_FLOAT" arrow::LogicalType::FLOAT"
-        LogicalType_DOUBLE" arrow::LogicalType::DOUBLE"
+        Type_FLOAT" arrow::Type::FLOAT"
+        Type_DOUBLE" arrow::Type::DOUBLE"
 
-        LogicalType_STRING" arrow::LogicalType::STRING"
+        Type_STRING" arrow::Type::STRING"
 
-        LogicalType_LIST" arrow::LogicalType::LIST"
-        LogicalType_STRUCT" arrow::LogicalType::STRUCT"
+        Type_LIST" arrow::Type::LIST"
+        Type_STRUCT" arrow::Type::STRUCT"
 
     cdef cppclass CDataType" arrow::DataType":
-        LogicalType type
-        c_bool nullable
+        Type type
 
         c_bool Equals(const CDataType* other)
 
@@ -55,8 +54,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int64_t bytes_allocated()
 
     cdef cppclass CListType" arrow::ListType"(CDataType):
-        CListType(const shared_ptr[CDataType]& value_type,
-                  c_bool nullable)
+        CListType(const shared_ptr[CDataType]& value_type)
 
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
@@ -65,21 +63,26 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_string name
         shared_ptr[CDataType] type
 
-        CField(const c_string& name, const shared_ptr[CDataType]& type)
+        c_bool nullable
+
+        CField(const c_string& name, const shared_ptr[CDataType]& type,
+               c_bool nullable)
 
     cdef cppclass CStructType" arrow::StructType"(CDataType):
-        CStructType(const vector[shared_ptr[CField]]& fields,
-                    c_bool nullable)
+        CStructType(const vector[shared_ptr[CField]]& fields)
 
     cdef cppclass CSchema" arrow::Schema":
-        CSchema(const shared_ptr[CField]& fields)
+        CSchema(const vector[shared_ptr[CField]]& fields)
+        const shared_ptr[CField]& field(int i)
+        int num_fields()
+        c_string ToString()
 
     cdef cppclass CArray" arrow::Array":
         const shared_ptr[CDataType]& type()
 
         int32_t length()
         int32_t null_count()
-        LogicalType logical_type()
+        Type type_enum()
 
         c_bool IsNull(int i)
 
@@ -122,3 +125,57 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CStringArray" arrow::StringArray"(CListArray):
         c_string GetString(int i)
+
+
+cdef extern from "arrow/api.h" namespace "arrow" nogil:
+    # We can later add more of the common status factory methods as needed
+    cdef CStatus CStatus_OK "Status::OK"()
+
+    cdef cppclass CStatus "arrow::Status":
+        CStatus()
+
+        c_string ToString()
+
+        c_bool ok()
+        c_bool IsOutOfMemory()
+        c_bool IsKeyError()
+        c_bool IsNotImplemented()
+        c_bool IsInvalid()
+
+    cdef cppclass Buffer:
+        uint8_t* data()
+        int64_t size()
+
+
+cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
+    cdef cppclass SchemaMessage:
+        int num_fields()
+        CStatus GetField(int i, shared_ptr[CField]* out)
+        CStatus GetSchema(shared_ptr[CSchema]* out)
+
+    cdef cppclass FieldMetadata:
+        pass
+
+    cdef cppclass BufferMetadata:
+        pass
+
+    cdef cppclass RecordBatchMessage:
+        pass
+
+    cdef cppclass DictionaryBatchMessage:
+        pass
+
+    enum MessageType" arrow::ipc::Message::Type":
+        MessageType_SCHEMA" arrow::ipc::Message::SCHEMA"
+        MessageType_RECORD_BATCH" arrow::ipc::Message::RECORD_BATCH"
+        MessageType_DICTIONARY_BATCH" arrow::ipc::Message::DICTIONARY_BATCH"
+
+    cdef cppclass Message:
+        CStatus Open(const shared_ptr[Buffer]& buf,
+                     shared_ptr[Message]* out)
+        int64_t body_length()
+        MessageType type()
+
+        shared_ptr[SchemaMessage] GetSchema()
+        shared_ptr[RecordBatchMessage] GetRecordBatch()
+        shared_ptr[DictionaryBatchMessage] GetDictionaryBatch()
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 9a0c004b7684a..eedfc85446810 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -18,8 +18,7 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CDataType, LogicalType,
-                                        MemoryPool)
+from pyarrow.includes.libarrow cimport CArray, CDataType, Type, MemoryPool
 
 cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     # We can later add more of the common status factory methods as needed
@@ -39,7 +38,7 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
         c_bool IsNotImplemented()
         c_bool IsArrowError()
 
-    shared_ptr[CDataType] GetPrimitiveType(LogicalType type, c_bool nullable)
+    shared_ptr[CDataType] GetPrimitiveType(Type type)
     Status ConvertPySequence(object obj, shared_ptr[CArray]* out)
 
     MemoryPool* GetMemoryPool()
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 261a38967c495..04f013d6ca706 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -172,18 +172,18 @@ cdef class ListValue(ArrayValue):
 
 
 cdef dict _scalar_classes = {
-    LogicalType_UINT8: Int8Value,
-    LogicalType_UINT16: Int16Value,
-    LogicalType_UINT32: Int32Value,
-    LogicalType_UINT64: Int64Value,
-    LogicalType_INT8: Int8Value,
-    LogicalType_INT16: Int16Value,
-    LogicalType_INT32: Int32Value,
-    LogicalType_INT64: Int64Value,
-    LogicalType_FLOAT: FloatValue,
-    LogicalType_DOUBLE: DoubleValue,
-    LogicalType_LIST: ListValue,
-    LogicalType_STRING: StringValue
+    Type_UINT8: Int8Value,
+    Type_UINT16: Int16Value,
+    Type_UINT32: Int32Value,
+    Type_UINT64: Int64Value,
+    Type_INT8: Int8Value,
+    Type_INT16: Int16Value,
+    Type_INT32: Int32Value,
+    Type_INT64: Int64Value,
+    Type_FLOAT: FloatValue,
+    Type_DOUBLE: DoubleValue,
+    Type_LIST: ListValue,
+    Type_STRING: StringValue
 }
 
 cdef object box_arrow_scalar(DataType type,
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
index 07b9bd04da20e..61458b765c742 100644
--- a/python/pyarrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.includes.common cimport shared_ptr
+from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport CDataType, CField, CSchema
 
 cdef class DataType:
@@ -33,9 +33,13 @@ cdef class Field:
     cdef readonly:
         DataType type
 
+    cdef init(self, const shared_ptr[CField]& field)
+
 cdef class Schema:
     cdef:
         shared_ptr[CSchema] sp_schema
         CSchema* schema
 
+    cdef init(self, const vector[shared_ptr[CField]]& fields)
+
 cdef DataType box_data_type(const shared_ptr[CDataType]& type)
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index ea878720d5bb8..b3bf02aad76bb 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -54,94 +54,153 @@ cdef class DataType:
 
 cdef class Field:
 
-    def __cinit__(self, object name, DataType type):
-        self.type = type
-        self.sp_field.reset(new CField(tobytes(name), type.sp_type))
-        self.field = self.sp_field.get()
+    def __cinit__(self):
+        pass
+
+    cdef init(self, const shared_ptr[CField]& field):
+        self.sp_field = field
+        self.field = field.get()
+
+    @classmethod
+    def from_py(cls, object name, DataType type, bint nullable=True):
+        cdef Field result = Field()
+        result.type = type
+        result.sp_field.reset(new CField(tobytes(name), type.sp_type,
+                                         nullable))
+        result.field = result.sp_field.get()
+
+        return result
 
     def __repr__(self):
         return 'Field({0!r}, type={1})'.format(self.name, str(self.type))
 
+    property nullable:
+
+        def __get__(self):
+            return self.field.nullable
+
     property name:
 
         def __get__(self):
             return frombytes(self.field.name)
 
+cdef class Schema:
+
+    def __cinit__(self):
+        pass
+
+    def __len__(self):
+        return self.schema.num_fields()
+
+    def __getitem__(self, i):
+        if i < 0 or i >= len(self):
+            raise IndexError("{0} is out of bounds".format(i))
+
+        cdef Field result = Field()
+        result.init(self.schema.field(i))
+        result.type = box_data_type(result.field.type)
+
+        return result
+
+    cdef init(self, const vector[shared_ptr[CField]]& fields):
+        self.schema = new CSchema(fields)
+        self.sp_schema.reset(self.schema)
+
+    @classmethod
+    def from_fields(cls, fields):
+        cdef:
+            Schema result
+            Field field
+            vector[shared_ptr[CField]] c_fields
+
+        c_fields.resize(len(fields))
+
+        for i in range(len(fields)):
+            field = fields[i]
+            c_fields[i] = field.sp_field
+
+        result = Schema()
+        result.init(c_fields)
+
+        return result
+
+    def __repr__(self):
+        return frombytes(self.schema.ToString())
+
 cdef dict _type_cache = {}
 
-cdef DataType primitive_type(LogicalType type, bint nullable=True):
-    if (type, nullable) in _type_cache:
-        return _type_cache[type, nullable]
+cdef DataType primitive_type(Type type):
+    if type in _type_cache:
+        return _type_cache[type]
 
     cdef DataType out = DataType()
-    out.init(pyarrow.GetPrimitiveType(type, nullable))
+    out.init(pyarrow.GetPrimitiveType(type))
 
-    _type_cache[type, nullable] = out
+    _type_cache[type] = out
     return out
 
 #------------------------------------------------------------
 # Type factory functions
 
-def field(name, type):
-    return Field(name, type)
+def field(name, type, bint nullable=True):
+    return Field.from_py(name, type, nullable)
 
 cdef set PRIMITIVE_TYPES = set([
-    LogicalType_NA, LogicalType_BOOL,
-    LogicalType_UINT8, LogicalType_INT8,
-    LogicalType_UINT16, LogicalType_INT16,
-    LogicalType_UINT32, LogicalType_INT32,
-    LogicalType_UINT64, LogicalType_INT64,
-    LogicalType_FLOAT, LogicalType_DOUBLE])
+    Type_NA, Type_BOOL,
+    Type_UINT8, Type_INT8,
+    Type_UINT16, Type_INT16,
+    Type_UINT32, Type_INT32,
+    Type_UINT64, Type_INT64,
+    Type_FLOAT, Type_DOUBLE])
 
 def null():
-    return primitive_type(LogicalType_NA)
+    return primitive_type(Type_NA)
 
-def bool_(c_bool nullable=True):
-    return primitive_type(LogicalType_BOOL, nullable)
+def bool_():
+    return primitive_type(Type_BOOL)
 
-def uint8(c_bool nullable=True):
-    return primitive_type(LogicalType_UINT8, nullable)
+def uint8():
+    return primitive_type(Type_UINT8)
 
-def int8(c_bool nullable=True):
-    return primitive_type(LogicalType_INT8, nullable)
+def int8():
+    return primitive_type(Type_INT8)
 
-def uint16(c_bool nullable=True):
-    return primitive_type(LogicalType_UINT16, nullable)
+def uint16():
+    return primitive_type(Type_UINT16)
 
-def int16(c_bool nullable=True):
-    return primitive_type(LogicalType_INT16, nullable)
+def int16():
+    return primitive_type(Type_INT16)
 
-def uint32(c_bool nullable=True):
-    return primitive_type(LogicalType_UINT32, nullable)
+def uint32():
+    return primitive_type(Type_UINT32)
 
-def int32(c_bool nullable=True):
-    return primitive_type(LogicalType_INT32, nullable)
+def int32():
+    return primitive_type(Type_INT32)
 
-def uint64(c_bool nullable=True):
-    return primitive_type(LogicalType_UINT64, nullable)
+def uint64():
+    return primitive_type(Type_UINT64)
 
-def int64(c_bool nullable=True):
-    return primitive_type(LogicalType_INT64, nullable)
+def int64():
+    return primitive_type(Type_INT64)
 
-def float_(c_bool nullable=True):
-    return primitive_type(LogicalType_FLOAT, nullable)
+def float_():
+    return primitive_type(Type_FLOAT)
 
-def double(c_bool nullable=True):
-    return primitive_type(LogicalType_DOUBLE, nullable)
+def double():
+    return primitive_type(Type_DOUBLE)
 
-def string(c_bool nullable=True):
+def string():
     """
     UTF8 string
     """
-    return primitive_type(LogicalType_STRING, nullable)
+    return primitive_type(Type_STRING)
 
-def list_(DataType value_type, c_bool nullable=True):
+def list_(DataType value_type):
     cdef DataType out = DataType()
-    out.init(shared_ptr[CDataType](
-        new CListType(value_type.sp_type, nullable)))
+    out.init(shared_ptr[CDataType](new CListType(value_type.sp_type)))
     return out
 
-def struct(fields, c_bool nullable=True):
+def struct(fields):
     """
 
     """
@@ -154,9 +213,11 @@ def struct(fields, c_bool nullable=True):
         c_fields.push_back(field.sp_field)
 
     out.init(shared_ptr[CDataType](
-        new CStructType(c_fields, nullable)))
+        new CStructType(c_fields)))
     return out
 
+def schema(fields):
+    return Schema.from_fields(fields)
 
 cdef DataType box_data_type(const shared_ptr[CDataType]& type):
     cdef DataType out = DataType()
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 0235526198f35..2894ea8f84451 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -18,6 +18,8 @@
 from pyarrow.compat import unittest
 import pyarrow as arrow
 
+A = arrow
+
 
 class TestTypes(unittest.TestCase):
 
@@ -28,15 +30,12 @@ def test_integers(self):
         for name in dtypes:
             factory = getattr(arrow, name)
             t = factory()
-            t_required = factory(False)
-
             assert str(t) == name
-            assert str(t_required) == '{0} not null'.format(name)
 
     def test_list(self):
         value_type = arrow.int32()
         list_type = arrow.list_(value_type)
-        assert str(list_type) == 'list<int32>'
+        assert str(list_type) == 'list<item: int32>'
 
     def test_string(self):
         t = arrow.string()
@@ -47,5 +46,26 @@ def test_field(self):
         f = arrow.field('foo', t)
 
         assert f.name == 'foo'
+        assert f.nullable
         assert f.type is t
         assert repr(f) == "Field('foo', type=string)"
+
+        f = arrow.field('foo', t, False)
+        assert not f.nullable
+
+    def test_schema(self):
+        fields = [
+            A.field('foo', A.int32()),
+            A.field('bar', A.string()),
+            A.field('baz', A.list_(A.int8()))
+        ]
+        sch = A.schema(fields)
+
+        assert len(sch) == 3
+        assert sch[0].name == 'foo'
+        assert sch[0].type == fields[0].type
+
+        assert repr(sch) == """\
+foo: int32
+bar: string
+baz: list<item: int8>"""
diff --git a/cpp/src/arrow/table/CMakeLists.txt b/python/pyarrow/tests/test_table.py
similarity index 58%
rename from cpp/src/arrow/table/CMakeLists.txt
rename to python/pyarrow/tests/test_table.py
index d9f00e74a37db..2e24445bd0c22 100644
--- a/cpp/src/arrow/table/CMakeLists.txt
+++ b/python/pyarrow/tests/test_table.py
@@ -15,19 +15,26 @@
 # specific language governing permissions and limitations
 # under the License.
 
-#######################################
-# arrow_table
-#######################################
-
-# Headers: top level
-install(FILES
-  column.h
-  schema.h
-  table.h
-  DESTINATION include/arrow/table)
-
-ADD_ARROW_TEST(column-test)
-ADD_ARROW_TEST(schema-test)
-ADD_ARROW_TEST(table-test)
-
-ADD_ARROW_BENCHMARK(column-benchmark)
+from pyarrow.compat import unittest
+import pyarrow as arrow
+
+A = arrow
+
+
+class TestRowBatch(unittest.TestCase):
+
+    def test_basics(self):
+        data = [
+            A.from_pylist(range(5)),
+            A.from_pylist([-10, -5, 0, 5, 10])
+        ]
+        num_rows = 5
+
+        descr = A.schema([A.field('c0', data[0].type),
+                          A.field('c1', data[1].type)])
+
+        batch = A.RowBatch(descr, num_rows, data)
+
+        assert len(batch) == num_rows
+        assert batch.num_rows == num_rows
+        assert batch.num_columns == len(data)
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index bb7905236c59c..acb13acecaf33 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -27,7 +27,7 @@
 
 using arrow::ArrayBuilder;
 using arrow::DataType;
-using arrow::LogicalType;
+using arrow::Type;
 
 namespace pyarrow {
 
@@ -356,17 +356,17 @@ class ListConverter : public TypedConverter<arrow::ListBuilder> {
 // Dynamic constructor for sequence converters
 std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type) {
   switch (type->type) {
-    case LogicalType::BOOL:
+    case Type::BOOL:
       return std::make_shared<BoolConverter>();
-    case LogicalType::INT64:
+    case Type::INT64:
       return std::make_shared<Int64Converter>();
-    case LogicalType::DOUBLE:
+    case Type::DOUBLE:
       return std::make_shared<DoubleConverter>();
-    case LogicalType::STRING:
+    case Type::STRING:
       return std::make_shared<StringConverter>();
-    case LogicalType::LIST:
+    case Type::LIST:
       return std::make_shared<ListConverter>();
-    case LogicalType::STRUCT:
+    case Type::STRUCT:
     default:
       return nullptr;
       break;
@@ -378,7 +378,7 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
   typed_builder_ = static_cast<arrow::ListBuilder*>(builder.get());
 
   value_converter_ = GetConverter(static_cast<arrow::ListType*>(
-          builder->type().get())->value_type);
+          builder->type().get())->value_type());
   if (value_converter_ == nullptr) {
     return Status::NotImplemented("value type not implemented");
   }
@@ -393,8 +393,8 @@ Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
   PY_RETURN_NOT_OK(InferArrowType(obj, &size, &type));
 
   // Handle NA / NullType case
-  if (type->type == LogicalType::NA) {
-    out->reset(new arrow::Array(type, size, size));
+  if (type->type == Type::NA) {
+    out->reset(new arrow::NullArray(type, size));
     return Status::OK();
   }
 
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
index 0921fc4994599..08003aabf9f22 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/python/src/pyarrow/helpers.cc
@@ -37,19 +37,14 @@ const std::shared_ptr<FloatType> FLOAT = std::make_shared<FloatType>();
 const std::shared_ptr<DoubleType> DOUBLE = std::make_shared<DoubleType>();
 const std::shared_ptr<StringType> STRING = std::make_shared<StringType>();
 
-#define GET_PRIMITIVE_TYPE(NAME, Type)          \
-  case LogicalType::NAME:                       \
-    if (nullable) {                             \
-      return NAME;                              \
-    } else {                                    \
-      return std::make_shared<Type>(nullable);  \
-    }                                           \
+#define GET_PRIMITIVE_TYPE(NAME, Class)         \
+  case Type::NAME:                              \
+    return NAME;                                \
     break;
 
-std::shared_ptr<DataType> GetPrimitiveType(LogicalType::type type,
-    bool nullable) {
+std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
   switch (type) {
-    case LogicalType::NA:
+    case Type::NA:
       return NA;
     GET_PRIMITIVE_TYPE(UINT8, UInt8Type);
     GET_PRIMITIVE_TYPE(INT8, Int8Type);
diff --git a/python/src/pyarrow/helpers.h b/python/src/pyarrow/helpers.h
index e41568d5881d4..ec42bb31d3b9b 100644
--- a/python/src/pyarrow/helpers.h
+++ b/python/src/pyarrow/helpers.h
@@ -24,7 +24,7 @@
 namespace pyarrow {
 
 using arrow::DataType;
-using arrow::LogicalType;
+using arrow::Type;
 
 extern const std::shared_ptr<arrow::NullType> NA;
 extern const std::shared_ptr<arrow::BooleanType> BOOL;
@@ -40,8 +40,7 @@ extern const std::shared_ptr<arrow::FloatType> FLOAT;
 extern const std::shared_ptr<arrow::DoubleType> DOUBLE;
 extern const std::shared_ptr<arrow::StringType> STRING;
 
-std::shared_ptr<DataType> GetPrimitiveType(LogicalType::type type,
-    bool nullable);
+std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
 
 } // namespace pyarrow
 

From a4002c6e217bf1e74895dc11ab76f0c8befbfe2a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Wed, 23 Mar 2016 10:59:31 -0700
Subject: [PATCH 0040/1644] ARROW-70: Add adapt 'lite' DCHECK macros from Kudu
 as also used in Parquet

Also added a null pointer DCHECK to show that it works. cc @emkornfield

Author: Wes McKinney <wesm@apache.org>

Closes #33 from wesm/ARROW-70 and squashes the following commits:

258d77b [Wes McKinney] Add adapt 'lite' DCHECK macros from Kudu as also used in Parquet
---
 cpp/src/arrow/ipc/adapter.cc |   2 +
 cpp/src/arrow/util/logging.h | 109 +++++++++++++++++++++++++++++++++++
 2 files changed, 111 insertions(+)
 create mode 100644 cpp/src/arrow/util/logging.h

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 7cdb965f5f45c..8a7d818ceeedd 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -32,6 +32,7 @@
 #include "arrow/types/construct.h"
 #include "arrow/types/primitive.h"
 #include "arrow/util/buffer.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
@@ -41,6 +42,7 @@ namespace flatbuf = apache::arrow::flatbuf;
 namespace ipc {
 
 static bool IsPrimitive(const DataType* type) {
+  DCHECK(type != nullptr);
   switch (type->type) {
     // NA is null type or "no type", considered primitive for now
     case Type::NA:
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
new file mode 100644
index 0000000000000..3ce4ccc1e9c26
--- /dev/null
+++ b/cpp/src/arrow/util/logging.h
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_LOGGING_H
+#define ARROW_UTIL_LOGGING_H
+
+#include <iostream>
+
+namespace arrow {
+
+// Stubbed versions of macros defined in glog/logging.h, intended for
+// environments where glog headers aren't available.
+//
+// Add more as needed.
+
+// Log levels. LOG ignores them, so their values are abitrary.
+
+#define ARROW_INFO 0
+#define ARROW_WARNING 1
+#define ARROW_ERROR 2
+#define ARROW_FATAL 3
+
+#define ARROW_LOG_INTERNAL(level) arrow::internal::CerrLog(level)
+#define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level)
+
+#define ARROW_CHECK(condition) \
+  (condition) ? 0 : ARROW_LOG(FATAL) << "Check failed: " #condition " "
+
+#ifdef NDEBUG
+#define ARROW_DFATAL ARROW_WARNING
+
+#define DCHECK(condition) while (false) arrow::internal::NullLog()
+#define DCHECK_EQ(val1, val2) while (false) arrow::internal::NullLog()
+#define DCHECK_NE(val1, val2) while (false) arrow::internal::NullLog()
+#define DCHECK_LE(val1, val2) while (false) arrow::internal::NullLog()
+#define DCHECK_LT(val1, val2) while (false) arrow::internal::NullLog()
+#define DCHECK_GE(val1, val2) while (false) arrow::internal::NullLog()
+#define DCHECK_GT(val1, val2) while (false) arrow::internal::NullLog()
+
+#else
+#define ARROW_DFATAL ARROW_FATAL
+
+#define DCHECK(condition) ARROW_CHECK(condition)
+#define DCHECK_EQ(val1, val2) ARROW_CHECK((val1) == (val2))
+#define DCHECK_NE(val1, val2) ARROW_CHECK((val1) != (val2))
+#define DCHECK_LE(val1, val2) ARROW_CHECK((val1) <= (val2))
+#define DCHECK_LT(val1, val2) ARROW_CHECK((val1) < (val2))
+#define DCHECK_GE(val1, val2) ARROW_CHECK((val1) >= (val2))
+#define DCHECK_GT(val1, val2) ARROW_CHECK((val1) > (val2))
+
+#endif // NDEBUG
+
+namespace internal {
+
+class NullLog {
+ public:
+  template<class T>
+  NullLog& operator<<(const T& t) {
+    return *this;
+  }
+};
+
+class CerrLog {
+ public:
+  CerrLog(int severity) // NOLINT(runtime/explicit)
+    : severity_(severity),
+      has_logged_(false) {
+  }
+
+  ~CerrLog() {
+    if (has_logged_) {
+      std::cerr << std::endl;
+    }
+    if (severity_ == ARROW_FATAL) {
+      exit(1);
+    }
+  }
+
+  template<class T>
+  CerrLog& operator<<(const T& t) {
+    has_logged_ = true;
+    std::cerr << t;
+    return *this;
+  }
+
+ private:
+  const int severity_;
+  bool has_logged_;
+};
+
+} // namespace internal
+
+} // namespace arrow
+
+#endif // ARROW_UTIL_LOGGING_H

From fbbee3d2db5beb1ae6c623fc6392095cffdf74fe Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 24 Mar 2016 09:31:56 -0700
Subject: [PATCH 0041/1644] ARROW-77: [C++] Conform bitmap interpretation to
 ARROW-62; 1 for nulls, 0 for non-nulls

Author: Wes McKinney <wesm@apache.org>

Closes #35 from wesm/ARROW-77 and squashes the following commits:

848d1fe [Wes McKinney] Clean up variable names to indicate valid_bytes vs null_bytes and change nulls to null_bitmap to be more clear
8960c7d [Wes McKinney] Flip bit interpretation so that 1 is null and 0 is not-null. Do not compare null slots in primitive arrays
---
 cpp/src/arrow/array-test.cc           | 30 ++++-----
 cpp/src/arrow/array.cc                | 10 +--
 cpp/src/arrow/array.h                 | 16 ++---
 cpp/src/arrow/builder.cc              | 16 ++---
 cpp/src/arrow/builder.h               | 14 ++--
 cpp/src/arrow/column-benchmark.cc     |  6 +-
 cpp/src/arrow/ipc/adapter.cc          | 10 +--
 cpp/src/arrow/ipc/ipc-adapter-test.cc |  8 +--
 cpp/src/arrow/test-util.h             | 25 ++++---
 cpp/src/arrow/types/construct.cc      |  4 +-
 cpp/src/arrow/types/construct.h       |  2 +-
 cpp/src/arrow/types/list.cc           |  6 +-
 cpp/src/arrow/types/list.h            | 23 +++----
 cpp/src/arrow/types/primitive-test.cc | 55 ++++++++++------
 cpp/src/arrow/types/primitive.cc      | 29 +++++++--
 cpp/src/arrow/types/primitive.h       | 93 +++++++++++++++------------
 cpp/src/arrow/types/string-test.cc    | 18 +++---
 cpp/src/arrow/types/string.h          |  8 +--
 cpp/src/arrow/util/bit-util.h         |  8 ++-
 19 files changed, 213 insertions(+), 168 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index eded5941e892e..7c6eaf55c0d0f 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -44,9 +44,9 @@ class TestArray : public ::testing::Test {
 
 TEST_F(TestArray, TestNullCount) {
   auto data = std::make_shared<PoolBuffer>(pool_);
-  auto nulls = std::make_shared<PoolBuffer>(pool_);
+  auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
 
-  std::unique_ptr<Int32Array> arr(new Int32Array(100, data, 10, nulls));
+  std::unique_ptr<Int32Array> arr(new Int32Array(100, data, 10, null_bitmap));
   ASSERT_EQ(10, arr->null_count());
 
   std::unique_ptr<Int32Array> arr_no_nulls(new Int32Array(100, data));
@@ -61,28 +61,28 @@ TEST_F(TestArray, TestLength) {
 }
 
 TEST_F(TestArray, TestIsNull) {
-  std::vector<uint8_t> nulls = {1, 0, 1, 1, 0, 1, 0, 0,
-                                1, 0, 1, 1, 0, 1, 0, 0,
-                                1, 0, 1, 1, 0, 1, 0, 0,
-                                1, 0, 1, 1, 0, 1, 0, 0,
-                                1, 0, 0, 1};
+  std::vector<uint8_t> null_bitmap = {1, 0, 1, 1, 0, 1, 0, 0,
+                                      1, 0, 1, 1, 0, 1, 0, 0,
+                                      1, 0, 1, 1, 0, 1, 0, 0,
+                                      1, 0, 1, 1, 0, 1, 0, 0,
+                                      1, 0, 0, 1};
   int32_t null_count = 0;
-  for (uint8_t x : nulls) {
-    if (x > 0) ++null_count;
+  for (uint8_t x : null_bitmap) {
+    if (x == 0) ++null_count;
   }
 
-  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(nulls.data(),
-      nulls.size());
+  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(null_bitmap.data(),
+      null_bitmap.size());
   std::unique_ptr<Array> arr;
-  arr.reset(new Int32Array(nulls.size(), nullptr, null_count, null_buf));
+  arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_count, null_buf));
 
   ASSERT_EQ(null_count, arr->null_count());
   ASSERT_EQ(5, null_buf->size());
 
-  ASSERT_TRUE(arr->nulls()->Equals(*null_buf.get()));
+  ASSERT_TRUE(arr->null_bitmap()->Equals(*null_buf.get()));
 
-  for (size_t i = 0; i < nulls.size(); ++i) {
-    ASSERT_EQ(static_cast<bool>(nulls[i]), arr->IsNull(i));
+  for (size_t i = 0; i < null_bitmap.size(); ++i) {
+    EXPECT_EQ(static_cast<bool>(null_bitmap[i]), !arr->IsNull(i)) << i;
   }
 }
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 5a5bc1069db13..3736732740b5b 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -27,13 +27,13 @@ namespace arrow {
 // Base array class
 
 Array::Array(const TypePtr& type, int32_t length, int32_t null_count,
-    const std::shared_ptr<Buffer>& nulls) {
+    const std::shared_ptr<Buffer>& null_bitmap) {
   type_ = type;
   length_ = length;
   null_count_ = null_count;
-  nulls_ = nulls;
-  if (nulls_) {
-    null_bits_ = nulls_->data();
+  null_bitmap_ = null_bitmap;
+  if (null_bitmap_) {
+    null_bitmap_data_ = null_bitmap_->data();
   }
 }
 
@@ -44,7 +44,7 @@ bool Array::EqualsExact(const Array& other) const {
     return false;
   }
   if (null_count_ > 0) {
-    return nulls_->Equals(*other.nulls_, util::bytes_for_bits(length_));
+    return null_bitmap_->Equals(*other.null_bitmap_, util::bytes_for_bits(length_));
   } else {
     return true;
   }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 65fc0aaf583e9..133adf32cbd50 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -32,8 +32,8 @@ class Buffer;
 // Immutable data array with some logical type and some length. Any memory is
 // owned by the respective Buffer instance (or its parents).
 //
-// The base class is only required to have a nulls buffer if the null count is
-// greater than 0
+// The base class is only required to have a null bitmap buffer if the null
+// count is greater than 0
 //
 // Any buffers used to initialize the array have their references "stolen". If
 // you wish to use the buffer beyond the lifetime of the array, you need to
@@ -41,13 +41,13 @@ class Buffer;
 class Array {
  public:
   Array(const TypePtr& type, int32_t length, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr);
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   virtual ~Array() {}
 
   // Determine if a slot is null. For inner loops. Does *not* boundscheck
   bool IsNull(int i) const {
-    return null_count_ > 0 && util::get_bit(null_bits_, i);
+    return null_count_ > 0 && util::bit_not_set(null_bitmap_data_, i);
   }
 
   int32_t length() const { return length_;}
@@ -56,8 +56,8 @@ class Array {
   const std::shared_ptr<DataType>& type() const { return type_;}
   Type::type type_enum() const { return type_->type;}
 
-  const std::shared_ptr<Buffer>& nulls() const {
-    return nulls_;
+  const std::shared_ptr<Buffer>& null_bitmap() const {
+    return null_bitmap_;
   }
 
   bool EqualsExact(const Array& arr) const;
@@ -68,8 +68,8 @@ class Array {
   int32_t null_count_;
   int32_t length_;
 
-  std::shared_ptr<Buffer> nulls_;
-  const uint8_t* null_bits_;
+  std::shared_ptr<Buffer> null_bitmap_;
+  const uint8_t* null_bitmap_data_;
 
  private:
   Array() {}
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index ba70add155186..6a62dc3b0e08f 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -28,20 +28,20 @@ namespace arrow {
 Status ArrayBuilder::Init(int32_t capacity) {
   capacity_ = capacity;
   int32_t to_alloc = util::ceil_byte(capacity) / 8;
-  nulls_ = std::make_shared<PoolBuffer>(pool_);
-  RETURN_NOT_OK(nulls_->Resize(to_alloc));
-  null_bits_ = nulls_->mutable_data();
-  memset(null_bits_, 0, to_alloc);
+  null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
+  RETURN_NOT_OK(null_bitmap_->Resize(to_alloc));
+  null_bitmap_data_ = null_bitmap_->mutable_data();
+  memset(null_bitmap_data_, 0, to_alloc);
   return Status::OK();
 }
 
 Status ArrayBuilder::Resize(int32_t new_bits) {
   int32_t new_bytes = util::ceil_byte(new_bits) / 8;
-  int32_t old_bytes = nulls_->size();
-  RETURN_NOT_OK(nulls_->Resize(new_bytes));
-  null_bits_ = nulls_->mutable_data();
+  int32_t old_bytes = null_bitmap_->size();
+  RETURN_NOT_OK(null_bitmap_->Resize(new_bytes));
+  null_bitmap_data_ = null_bitmap_->mutable_data();
   if (old_bytes < new_bytes) {
-    memset(null_bits_ + old_bytes, 0, new_bytes - old_bytes);
+    memset(null_bitmap_data_ + old_bytes, 0, new_bytes - old_bytes);
   }
   return Status::OK();
 }
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index d5d1fdf95af17..308e54c80d794 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -40,9 +40,9 @@ class ArrayBuilder {
   explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type) :
       pool_(pool),
       type_(type),
-      nulls_(nullptr),
+      null_bitmap_(nullptr),
       null_count_(0),
-      null_bits_(nullptr),
+      null_bitmap_data_(nullptr),
       length_(0),
       capacity_(0) {}
 
@@ -66,7 +66,7 @@ class ArrayBuilder {
   // initialized independently
   Status Init(int32_t capacity);
 
-  // Resizes the nulls array
+  // Resizes the null_bitmap array
   Status Resize(int32_t new_bits);
 
   // For cases where raw data was memcpy'd into the internal buffers, allows us
@@ -74,7 +74,7 @@ class ArrayBuilder {
   // this function responsibly.
   Status Advance(int32_t elements);
 
-  const std::shared_ptr<PoolBuffer>& nulls() const { return nulls_;}
+  const std::shared_ptr<PoolBuffer>& null_bitmap() const { return null_bitmap_;}
 
   // Creates new array object to hold the contents of the builder and transfers
   // ownership of the data
@@ -89,10 +89,10 @@ class ArrayBuilder {
 
   std::shared_ptr<DataType> type_;
 
-  // When nulls are first appended to the builder, the null bitmap is allocated
-  std::shared_ptr<PoolBuffer> nulls_;
+  // When null_bitmap are first appended to the builder, the null bitmap is allocated
+  std::shared_ptr<PoolBuffer> null_bitmap_;
   int32_t null_count_;
-  uint8_t* null_bits_;
+  uint8_t* null_bitmap_data_;
 
   // Array length, so far. Also, the index of the next element to be added
   int32_t length_;
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index 69ee52c3e09ea..335d581782ac0 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -28,10 +28,10 @@ namespace {
   std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
     auto pool = default_memory_pool();
     auto data = std::make_shared<PoolBuffer>(pool);
-    auto nulls = std::make_shared<PoolBuffer>(pool);
+    auto null_bitmap = std::make_shared<PoolBuffer>(pool);
     data->Resize(length * sizeof(typename ArrayType::value_type));
-    nulls->Resize(util::bytes_for_bits(length));
-    return std::make_shared<ArrayType>(length, data, 10, nulls);
+    null_bitmap->Resize(util::bytes_for_bits(length));
+    return std::make_shared<ArrayType>(length, data, 10, null_bitmap);
   }
 }  // anonymous namespace
 
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 8a7d818ceeedd..c79e8469530f7 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -75,7 +75,7 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
         flatbuf::FieldNode(prim_arr->length(), prim_arr->null_count()));
 
     if (prim_arr->null_count() > 0) {
-      buffers->push_back(prim_arr->nulls());
+      buffers->push_back(prim_arr->null_bitmap());
     } else {
       // Push a dummy zero-length buffer, not to be copied
       buffers->push_back(std::make_shared<Buffer>(nullptr, 0));
@@ -230,13 +230,13 @@ class RowBatchReader::Impl {
     FieldMetadata field_meta = metadata_->field(field_index_++);
 
     if (IsPrimitive(type.get())) {
-      std::shared_ptr<Buffer> nulls;
+      std::shared_ptr<Buffer> null_bitmap;
       std::shared_ptr<Buffer> data;
       if (field_meta.null_count == 0) {
-        nulls = nullptr;
+        null_bitmap = nullptr;
         ++buffer_index_;
       } else {
-        RETURN_NOT_OK(GetBuffer(buffer_index_++, &nulls));
+        RETURN_NOT_OK(GetBuffer(buffer_index_++, &null_bitmap));
       }
       if (field_meta.length > 0) {
         RETURN_NOT_OK(GetBuffer(buffer_index_++, &data));
@@ -244,7 +244,7 @@ class RowBatchReader::Impl {
         data.reset(new Buffer(nullptr, 0));
       }
       return MakePrimitiveArray(type, field_meta.length, data,
-          field_meta.null_count, nulls, out);
+          field_meta.null_count, null_bitmap, out);
     } else {
       return Status::NotImplemented("Non-primitive types not complete yet");
     }
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index d75998f0a5dd2..79b4d710d282f 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -77,14 +77,14 @@ TEST_F(TestWriteRowBatch, IntegerRoundTrip) {
   test::rand_uniform_int(length, 0, 0, std::numeric_limits<int32_t>::max(),
       reinterpret_cast<int32_t*>(data->mutable_data()));
 
-  auto nulls = std::make_shared<PoolBuffer>(pool_);
+  auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
   int null_bytes = util::bytes_for_bits(length);
-  ASSERT_OK(nulls->Resize(null_bytes));
-  test::random_bytes(null_bytes, 0, nulls->mutable_data());
+  ASSERT_OK(null_bitmap->Resize(null_bytes));
+  test::random_bytes(null_bytes, 0, null_bitmap->mutable_data());
 
   auto a0 = std::make_shared<Int32Array>(length, data);
   auto a1 = std::make_shared<Int32Array>(length, data,
-      test::bitmap_popcount(nulls->data(), length), nulls);
+      test::bitmap_popcount(null_bitmap->data(), length), null_bitmap);
 
   RowBatch batch(schema, length, {a0, a1});
 
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index a9fb2a7644ab3..ea3ce5f7f53ff 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -72,10 +72,10 @@ class TestBase : public ::testing::Test {
   template <typename ArrayType>
   std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
     auto data = std::make_shared<PoolBuffer>(pool_);
-    auto nulls = std::make_shared<PoolBuffer>(pool_);
+    auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
     EXPECT_OK(data->Resize(length * sizeof(typename ArrayType::value_type)));
-    EXPECT_OK(nulls->Resize(util::bytes_for_bits(length)));
-    return std::make_shared<ArrayType>(length, data, 10, nulls);
+    EXPECT_OK(null_bitmap->Resize(util::bytes_for_bits(length)));
+    return std::make_shared<ArrayType>(length, data, 10, null_bitmap);
   }
 
  protected:
@@ -104,17 +104,22 @@ std::shared_ptr<Buffer> to_buffer(const std::vector<T>& values) {
       values.size() * sizeof(T));
 }
 
-void random_nulls(int64_t n, double pct_null, std::vector<uint8_t>* nulls) {
+void random_null_bitmap(int64_t n, double pct_null, std::vector<uint8_t>* null_bitmap) {
   Random rng(random_seed());
   for (int i = 0; i < n; ++i) {
-    nulls->push_back(static_cast<uint8_t>(rng.NextDoubleFraction() > pct_null));
+    if (rng.NextDoubleFraction() > pct_null) {
+      null_bitmap->push_back(1);
+    } else {
+      // null
+      null_bitmap->push_back(0);
+    }
   }
 }
 
-void random_nulls(int64_t n, double pct_null, std::vector<bool>* nulls) {
+void random_null_bitmap(int64_t n, double pct_null, std::vector<bool>* null_bitmap) {
   Random rng(random_seed());
   for (int i = 0; i < n; ++i) {
-    nulls->push_back(rng.NextDoubleFraction() > pct_null);
+    null_bitmap->push_back(rng.NextDoubleFraction() > pct_null);
   }
 }
 
@@ -145,10 +150,10 @@ static inline int bitmap_popcount(const uint8_t* data, int length) {
   return count;
 }
 
-static inline int null_count(const std::vector<uint8_t>& nulls) {
+static inline int null_count(const std::vector<uint8_t>& valid_bytes) {
   int result = 0;
-  for (size_t i = 0; i < nulls.size(); ++i) {
-    if (nulls[i] > 0) {
+  for (size_t i = 0; i < valid_bytes.size(); ++i) {
+    if (valid_bytes[i] == 0) {
       ++result;
     }
   }
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 290decd81ff42..df2317c340b2d 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -75,12 +75,12 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 
 #define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                      \
     case Type::ENUM:                                                    \
-      out->reset(new ArrayType(type, length, data, null_count, nulls)); \
+      out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
       return Status::OK();
 
 Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
     int32_t length, const std::shared_ptr<Buffer>& data,
-    int32_t null_count, const std::shared_ptr<Buffer>& nulls,
+    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
     std::shared_ptr<Array>* out) {
   switch (type->type) {
     MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index 089c484c58bee..228faeccc4e4d 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -35,7 +35,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 
 Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
     int32_t length, const std::shared_ptr<Buffer>& data,
-    int32_t null_count, const std::shared_ptr<Buffer>& nulls,
+    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
     std::shared_ptr<Array>* out);
 
 } // namespace arrow
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index 670ee4da11675..d64c06d90c174 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -27,13 +27,13 @@ bool ListArray::EqualsExact(const ListArray& other) const {
 
   bool equal_offsets = offset_buf_->Equals(*other.offset_buf_,
       length_ + 1);
-  bool equal_nulls = true;
+  bool equal_null_bitmap = true;
   if (null_count_ > 0) {
-    equal_nulls = nulls_->Equals(*other.nulls_,
+    equal_null_bitmap = null_bitmap_->Equals(*other.null_bitmap_,
         util::bytes_for_bits(length_));
   }
 
-  if (!(equal_offsets && equal_nulls)) {
+  if (!(equal_offsets && equal_null_bitmap)) {
     return false;
   }
 
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 141f762458b3b..72e20e943c347 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -39,8 +39,8 @@ class ListArray : public Array {
   ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
       const ArrayPtr& values,
       int32_t null_count = 0,
-      std::shared_ptr<Buffer> nulls = nullptr) :
-      Array(type, length, null_count, nulls) {
+      std::shared_ptr<Buffer> null_bitmap = nullptr) :
+      Array(type, length, null_count, null_bitmap) {
     offset_buf_ = offsets;
     offsets_ = offsets == nullptr? nullptr :
       reinterpret_cast<const int32_t*>(offset_buf_->data());
@@ -109,17 +109,17 @@ class ListBuilder : public Int32Builder {
 
   // Vector append
   //
-  // If passed, null_bytes is of equal length to values, and any nonzero byte
+  // If passed, valid_bytes is of equal length to values, and any zero byte
   // will be considered as a null for that slot
-  Status Append(value_type* values, int32_t length, uint8_t* null_bytes = nullptr) {
+  Status Append(value_type* values, int32_t length, uint8_t* valid_bytes = nullptr) {
     if (length_ + length > capacity_) {
       int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
     }
     memcpy(raw_buffer() + length_, values, length * elsize_);
 
-    if (null_bytes != nullptr) {
-      AppendNulls(null_bytes, length);
+    if (valid_bytes != nullptr) {
+      AppendNulls(valid_bytes, length);
     }
 
     length_ += length;
@@ -136,9 +136,9 @@ class ListBuilder : public Int32Builder {
     }
 
     auto result = std::make_shared<Container>(type_, length_, values_, items,
-        null_count_, nulls_);
+        null_count_, null_bitmap_);
 
-    values_ = nulls_ = nullptr;
+    values_ = null_bitmap_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
 
     return result;
@@ -159,16 +159,13 @@ class ListBuilder : public Int32Builder {
     }
     if (is_null) {
       ++null_count_;
-      util::set_bit(null_bits_, length_);
+    } else {
+      util::set_bit(null_bitmap_data_, length_);
     }
     raw_buffer()[length_++] = value_builder_->length();
     return Status::OK();
   }
 
-  // Status Append(int32_t* offsets, int length, uint8_t* null_bytes) {
-  //   return Int32Builder::Append(offsets, length, null_bytes);
-  // }
-
   Status AppendNull() {
     return Append(true);
   }
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 7eae8cda8c488..10ba113c5916c 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -71,10 +71,10 @@ PRIMITIVE_TEST(BooleanType, BOOL, "bool");
 
 TEST_F(TestBuilder, TestResize) {
   builder_->Init(10);
-  ASSERT_EQ(2, builder_->nulls()->size());
+  ASSERT_EQ(2, builder_->null_bitmap()->size());
 
   builder_->Resize(30);
-  ASSERT_EQ(4, builder_->nulls()->size());
+  ASSERT_EQ(4, builder_->null_bitmap()->size());
 }
 
 template <typename Attrs>
@@ -99,7 +99,7 @@ class TestPrimitiveBuilder : public TestBuilder {
 
   void RandomData(int N, double pct_null = 0.1) {
     Attrs::draw(N, &draws_);
-    test::random_nulls(N, pct_null, &nulls_);
+    test::random_null_bitmap(N, pct_null, &valid_bytes_);
   }
 
   void CheckNullable() {
@@ -109,10 +109,11 @@ class TestPrimitiveBuilder : public TestBuilder {
         reinterpret_cast<uint8_t*>(draws_.data()),
         size * sizeof(T));
 
-    auto ex_nulls = test::bytes_to_null_buffer(nulls_.data(), size);
-    int32_t ex_null_count = test::null_count(nulls_);
+    auto ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_.data(), size);
+    int32_t ex_null_count = test::null_count(valid_bytes_);
 
-    auto expected = std::make_shared<ArrayType>(size, ex_data, ex_null_count, ex_nulls);
+    auto expected = std::make_shared<ArrayType>(size, ex_data, ex_null_count,
+        ex_null_bitmap);
 
     std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(
         builder_->Finish());
@@ -123,8 +124,8 @@ class TestPrimitiveBuilder : public TestBuilder {
     ASSERT_EQ(0, builder_->null_count());
     ASSERT_EQ(nullptr, builder_->buffer());
 
-    ASSERT_TRUE(result->EqualsExact(*expected.get()));
     ASSERT_EQ(ex_null_count, result->null_count());
+    ASSERT_TRUE(result->EqualsExact(*expected.get()));
   }
 
   void CheckNonNullable() {
@@ -154,7 +155,7 @@ class TestPrimitiveBuilder : public TestBuilder {
   shared_ptr<BuilderType> builder_nn_;
 
   vector<T> draws_;
-  vector<uint8_t> nulls_;
+  vector<uint8_t> valid_bytes_;
 };
 
 #define PTYPE_DECL(CapType, c_type)             \
@@ -210,7 +211,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestInit) {
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
-  int size = 10000;
+  int size = 1000;
   for (int i = 0; i < size; ++i) {
     ASSERT_OK(this->builder_->AppendNull());
   }
@@ -218,17 +219,17 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
   auto result = this->builder_->Finish();
 
   for (int i = 0; i < size; ++i) {
-    ASSERT_TRUE(result->IsNull(i));
+    ASSERT_TRUE(result->IsNull(i)) << i;
   }
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
   DECL_T();
 
-  int size = 10000;
+  int size = 1000;
 
   vector<T>& draws = this->draws_;
-  vector<uint8_t>& nulls = this->nulls_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
 
   int64_t memory_before = this->pool_->bytes_allocated();
 
@@ -236,7 +237,11 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
 
   int i;
   for (i = 0; i < size; ++i) {
-    ASSERT_OK(this->builder_->Append(draws[i], nulls[i] > 0));
+    if (valid_bytes[i] > 0) {
+      ASSERT_OK(this->builder_->Append(draws[i]));
+    } else {
+      ASSERT_OK(this->builder_->AppendNull());
+    }
   }
 
   do {
@@ -249,17 +254,21 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
 TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   DECL_T();
 
-  int size = 10000;
+  const int size = 10000;
 
   vector<T>& draws = this->draws_;
-  vector<uint8_t>& nulls = this->nulls_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
 
   this->RandomData(size);
 
   int i;
   // Append the first 1000
   for (i = 0; i < 1000; ++i) {
-    ASSERT_OK(this->builder_->Append(draws[i], nulls[i] > 0));
+    if (valid_bytes[i] > 0) {
+      ASSERT_OK(this->builder_->Append(draws[i]));
+    } else {
+      ASSERT_OK(this->builder_->AppendNull());
+    }
     ASSERT_OK(this->builder_nn_->Append(draws[i]));
   }
 
@@ -271,7 +280,11 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
 
   // Append the next 9000
   for (i = 1000; i < size; ++i) {
-    ASSERT_OK(this->builder_->Append(draws[i], nulls[i] > 0));
+    if (valid_bytes[i] > 0) {
+      ASSERT_OK(this->builder_->Append(draws[i]));
+    } else {
+      ASSERT_OK(this->builder_->AppendNull());
+    }
     ASSERT_OK(this->builder_nn_->Append(draws[i]));
   }
 
@@ -293,12 +306,12 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
   this->RandomData(size);
 
   vector<T>& draws = this->draws_;
-  vector<uint8_t>& nulls = this->nulls_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
 
   // first slug
   int K = 1000;
 
-  ASSERT_OK(this->builder_->Append(draws.data(), K, nulls.data()));
+  ASSERT_OK(this->builder_->Append(draws.data(), K, valid_bytes.data()));
   ASSERT_OK(this->builder_nn_->Append(draws.data(), K));
 
   ASSERT_EQ(1000, this->builder_->length());
@@ -308,7 +321,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
   ASSERT_EQ(1024, this->builder_nn_->capacity());
 
   // Append the next 9000
-  ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, nulls.data() + K));
+  ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, valid_bytes.data() + K));
   ASSERT_OK(this->builder_nn_->Append(draws.data() + K, size - K));
 
   ASSERT_EQ(size, this->builder_->length());
@@ -338,7 +351,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestResize) {
   ASSERT_EQ(cap, this->builder_->capacity());
 
   ASSERT_EQ(cap * sizeof(T), this->builder_->buffer()->size());
-  ASSERT_EQ(util::ceil_byte(cap) / 8, this->builder_->nulls()->size());
+  ASSERT_EQ(util::ceil_byte(cap) / 8, this->builder_->null_bitmap()->size());
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 32b8bfa7f1bd4..ecd5d68ff45a8 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -26,13 +26,14 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Primitive array base
 
-PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
+PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length, int value_size,
     const std::shared_ptr<Buffer>& data,
     int32_t null_count,
-    const std::shared_ptr<Buffer>& nulls) :
-    Array(type, length, null_count, nulls) {
+    const std::shared_ptr<Buffer>& null_bitmap) :
+    Array(type, length, null_count, null_bitmap) {
   data_ = data;
   raw_data_ = data == nullptr? nullptr : data_->data();
+  value_size_ = value_size;
 }
 
 bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
@@ -41,12 +42,26 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
     return false;
   }
 
-  bool equal_data = data_->Equals(*other.data_, length_);
   if (null_count_ > 0) {
-    return equal_data &&
-      nulls_->Equals(*other.nulls_, util::ceil_byte(length_) / 8);
+    bool equal_bitmap = null_bitmap_->Equals(*other.null_bitmap_,
+        util::ceil_byte(length_) / 8);
+    if (!equal_bitmap) {
+      return false;
+    }
+
+    const uint8_t* this_data = raw_data_;
+    const uint8_t* other_data = other.raw_data_;
+
+    for (int i = 0; i < length_; ++i) {
+      if (!IsNull(i) && memcmp(this_data, other_data, value_size_)) {
+        return false;
+      }
+      this_data += value_size_;
+      other_data += value_size_;
+    }
+    return true;
   } else {
-    return equal_data;
+    return data_->Equals(*other.data_, length_);
   }
 }
 
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index e01027cf55c39..4eaff433229e0 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -37,10 +37,10 @@ class MemoryPool;
 // Base class for fixed-size logical types
 class PrimitiveArray : public Array {
  public:
-  PrimitiveArray(const TypePtr& type, int32_t length,
+  PrimitiveArray(const TypePtr& type, int32_t length, int value_size,
       const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr);
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
   virtual ~PrimitiveArray() {}
 
   const std::shared_ptr<Buffer>& data() const { return data_;}
@@ -51,31 +51,38 @@ class PrimitiveArray : public Array {
  protected:
   std::shared_ptr<Buffer> data_;
   const uint8_t* raw_data_;
+  int value_size_;
 };
 
-#define NUMERIC_ARRAY_DECL(NAME, TypeClass, T)                      \
-class NAME : public PrimitiveArray {                                \
- public:                                                            \
-  using value_type = T;                                             \
-  using PrimitiveArray::PrimitiveArray;                             \
-  NAME(int32_t length, const std::shared_ptr<Buffer>& data,         \
-      int32_t null_count = 0,                                       \
-      const std::shared_ptr<Buffer>& nulls = nullptr) :             \
-      PrimitiveArray(std::make_shared<TypeClass>(), length, data,   \
-          null_count, nulls) {}                                     \
-                                                                    \
-  bool EqualsExact(const NAME& other) const {                       \
-    return PrimitiveArray::EqualsExact(                             \
-        *static_cast<const PrimitiveArray*>(&other));               \
-  }                                                                 \
-                                                                    \
-  const T* raw_data() const {                                       \
-    return reinterpret_cast<const T*>(raw_data_);                   \
-  }                                                                 \
-                                                                    \
-  T Value(int i) const {                                            \
-    return raw_data()[i];                                           \
-  }                                                                 \
+#define NUMERIC_ARRAY_DECL(NAME, TypeClass, T)                  \
+class NAME : public PrimitiveArray {                            \
+ public:                                                        \
+  using value_type = T;                                         \
+  NAME(const TypePtr& type, int32_t length,                     \
+      const std::shared_ptr<Buffer>& data,                      \
+      int32_t null_count = 0,                                   \
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr) :   \
+      PrimitiveArray(std::make_shared<TypeClass>(), length,     \
+          sizeof(T), data, null_count, null_bitmap) {}          \
+                                                                \
+  NAME(int32_t length, const std::shared_ptr<Buffer>& data,     \
+      int32_t null_count = 0,                                   \
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr) :   \
+      PrimitiveArray(std::make_shared<TypeClass>(), length,     \
+          sizeof(T), data, null_count, null_bitmap) {}          \
+                                                                \
+  bool EqualsExact(const NAME& other) const {                   \
+    return PrimitiveArray::EqualsExact(                         \
+        *static_cast<const PrimitiveArray*>(&other));           \
+  }                                                             \
+                                                                \
+  const T* raw_data() const {                                   \
+    return reinterpret_cast<const T*>(raw_data_);               \
+  }                                                             \
+                                                                \
+  T Value(int i) const {                                        \
+    return raw_data()[i];                                       \
+  }                                                             \
 };
 
 NUMERIC_ARRAY_DECL(UInt8Array, UInt8Type, uint8_t);
@@ -137,25 +144,22 @@ class PrimitiveBuilder : public ArrayBuilder {
   }
 
   // Scalar append
-  Status Append(value_type val, bool is_null = false) {
+  Status Append(value_type val) {
     if (length_ == capacity_) {
       // If the capacity was not already a multiple of 2, do so here
       RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
     }
-    if (is_null) {
-      ++null_count_;
-      util::set_bit(null_bits_, length_);
-    }
+    util::set_bit(null_bitmap_data_, length_);
     raw_buffer()[length_++] = val;
     return Status::OK();
   }
 
   // Vector append
   //
-  // If passed, null_bytes is of equal length to values, and any nonzero byte
+  // If passed, valid_bytes is of equal length to values, and any zero byte
   // will be considered as a null for that slot
   Status Append(const value_type* values, int32_t length,
-      const uint8_t* null_bytes = nullptr) {
+      const uint8_t* valid_bytes = nullptr) {
     if (length_ + length > capacity_) {
       int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
@@ -164,21 +168,26 @@ class PrimitiveBuilder : public ArrayBuilder {
       memcpy(raw_buffer() + length_, values, length * elsize_);
     }
 
-    if (null_bytes != nullptr) {
-      AppendNulls(null_bytes, length);
+    if (valid_bytes != nullptr) {
+      AppendNulls(valid_bytes, length);
+    } else {
+      for (int i = 0; i < length; ++i) {
+        util::set_bit(null_bitmap_data_, length_ + i);
+      }
     }
 
     length_ += length;
     return Status::OK();
   }
 
-  // Write nulls as uint8_t* into pre-allocated memory
-  void AppendNulls(const uint8_t* null_bytes, int32_t length) {
-    // If null_bytes is all not null, then none of the values are null
+  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+  void AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+    // If valid_bytes is all not null, then none of the values are null
     for (int i = 0; i < length; ++i) {
-      if (static_cast<bool>(null_bytes[i])) {
+      if (valid_bytes[i] == 0) {
         ++null_count_;
-        util::set_bit(null_bits_, length_ + i);
+      } else {
+        util::set_bit(null_bitmap_data_, length_ + i);
       }
     }
   }
@@ -189,15 +198,15 @@ class PrimitiveBuilder : public ArrayBuilder {
       RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
     }
     ++null_count_;
-    util::set_bit(null_bits_, length_++);
+    ++length_;
     return Status::OK();
   }
 
   std::shared_ptr<Array> Finish() override {
     std::shared_ptr<ArrayType> result = std::make_shared<ArrayType>(
-        type_, length_, values_, null_count_, nulls_);
+        type_, length_, values_, null_count_, null_bitmap_);
 
-    values_ = nulls_ = nullptr;
+    values_ = null_bitmap_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
     return result;
   }
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index 7dc3d682cdc15..b329b4f0ef7e1 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -77,7 +77,7 @@ class TestStringContainer : public ::testing::Test  {
   void SetUp() {
     chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
     offsets_ = {0, 1, 1, 1, 3, 6};
-    nulls_ = {0, 0, 1, 0, 0};
+    valid_bytes_ = {1, 1, 0, 1, 1};
     expected_ = {"a", "", "", "bb", "ccc"};
 
     MakeArray();
@@ -92,23 +92,23 @@ class TestStringContainer : public ::testing::Test  {
 
     offsets_buf_ = test::to_buffer(offsets_);
 
-    nulls_buf_ = test::bytes_to_null_buffer(nulls_.data(), nulls_.size());
-    null_count_ = test::null_count(nulls_);
+    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_.data(), valid_bytes_.size());
+    null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<StringArray>(length_, offsets_buf_, values_,
-        null_count_, nulls_buf_);
+        null_count_, null_bitmap_);
   }
 
  protected:
   std::vector<int32_t> offsets_;
   std::vector<char> chars_;
-  std::vector<uint8_t> nulls_;
+  std::vector<uint8_t> valid_bytes_;
 
   std::vector<std::string> expected_;
 
   std::shared_ptr<Buffer> value_buf_;
   std::shared_ptr<Buffer> offsets_buf_;
-  std::shared_ptr<Buffer> nulls_buf_;
+  std::shared_ptr<Buffer> null_bitmap_;
 
   int null_count_;
   int length_;
@@ -143,12 +143,12 @@ TEST_F(TestStringContainer, TestListFunctions) {
 
 TEST_F(TestStringContainer, TestDestructor) {
   auto arr = std::make_shared<StringArray>(length_, offsets_buf_, values_,
-      null_count_, nulls_buf_);
+      null_count_, null_bitmap_);
 }
 
 TEST_F(TestStringContainer, TestGetString) {
   for (size_t i = 0; i < expected_.size(); ++i) {
-    if (nulls_[i]) {
+    if (valid_bytes_[i] == 0) {
       ASSERT_TRUE(strings_->IsNull(i));
     } else {
       ASSERT_EQ(expected_[i], strings_->GetString(i));
@@ -197,7 +197,7 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
   Done();
 
   ASSERT_EQ(reps * N, result_->length());
-  ASSERT_EQ(reps * test::null_count(is_null), result_->null_count());
+  ASSERT_EQ(reps, result_->null_count());
   ASSERT_EQ(reps * 6, result_->values()->length());
 
   int32_t length;
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index 2b3fba5ce0932..fda722ba6def2 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -68,8 +68,8 @@ class StringArray : public ListArray {
       const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values,
       int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr) :
-      ListArray(type, length, offsets, values, null_count, nulls) {
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr) :
+      ListArray(type, length, offsets, values, null_count, null_bitmap) {
     // For convenience
     bytes_ = static_cast<UInt8Array*>(values.get());
     raw_bytes_ = bytes_->raw_data();
@@ -79,9 +79,9 @@ class StringArray : public ListArray {
       const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values,
       int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& nulls = nullptr) :
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr) :
       StringArray(std::make_shared<StringType>(), length, offsets, values,
-          null_count, nulls) {}
+          null_count, null_bitmap) {}
 
   // Compute the pointer t
   const uint8_t* GetValue(int i, int32_t* out_length) const {
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 1d2d1d5f9d7e4..08222d5089474 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -40,8 +40,14 @@ static inline int64_t ceil_2bytes(int64_t size) {
   return (size + 15) & ~15;
 }
 
+static constexpr uint8_t BITMASK[] = {1, 2, 4, 8, 16, 32, 64, 128};
+
 static inline bool get_bit(const uint8_t* bits, int i) {
-  return bits[i / 8] & (1 << (i % 8));
+  return bits[i / 8] & BITMASK[i % 8];
+}
+
+static inline bool bit_not_set(const uint8_t* bits, int i) {
+  return (bits[i / 8] & BITMASK[i % 8]) == 0;
 }
 
 static inline void set_bit(uint8_t* bits, int i) {

From c06b7654bccfe8c461869a6e5922668896c27c45 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 24 Mar 2016 19:19:22 -0700
Subject: [PATCH 0042/1644] ARROW-62: Clarify null bitmap interpretation,
 indicate bit-endianness, add null count, remove non-nullable physical
 distinction

As the initial scribe for the Arrow format, I made a mistake in what the null bits mean (1 for not-null, 0 for null). I also addressed ARROW-56 (bit-numbering) here.

Database systems are split on this subject. PostgreSQL for example does it this way:

http://www.postgresql.org/docs/9.5/static/storage-page-layout.html

> In this list of bits, a 1 bit indicates not-null, a 0 bit is a null. When the bitmap is not present, all columns are assumed not-null.

Since the Drill implementation predates the Arrow project, I think it's safe to go with this.

This patch also includes ARROW-76 which adds a "null count" to the memory layout indicating the actual number of nulls in an array. This also strikes the "non-nullable" distinction from the memory layout as there is no semantic difference between arrays with null count 0 and a non-nullable array. Instead, users may choose to set `nullable=false` in the schema metadata and verify that Arrow memory conforms to the schema.

Author: Wes McKinney <wesm@apache.org>

Closes #34 from wesm/ARROW-62 and squashes the following commits:

8c92926 [Wes McKinney] Add to README about what the format documents are
1f6fe03 [Wes McKinney] Account for null count and non-nullable removal from ARROW-76
648fd47 [Wes McKinney] Indicate that bitmaps should be a multiple of 8 bytes
4333d82 [Wes McKinney] Use 'null bitmap' similar to PostgreSQL documentation
dac77d4 [Wes McKinney] Revise format document language re: null bitmaps per feedback
f7a3898 [Wes McKinney] Revise format to indicate LSB bit numbering and 0/1 null/not-null distinction
---
 format/Layout.md                          |  77 +++++++++++++++-------
 format/Message.fbs                        |  10 +--
 format/README.md                          |  17 +++++
 format/diagrams/layout-list-of-struct.png | Bin 60600 -> 54122 bytes
 4 files changed, 74 insertions(+), 30 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index c393163bf894b..2d46ece606ea7 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -42,7 +42,7 @@ Base requirements
 * Capable of representing fully-materialized and decoded / decompressed Parquet
   data
 * All leaf nodes (primitive value arrays) use contiguous memory regions
-* Each relative type can be nullable or non-nullable
+* Any relative type can be have null slots
 * Arrays are immutable once created. Implementations can provide APIs to mutate
   an array, but applying mutations will require a new array data structure to
   be built.
@@ -56,7 +56,7 @@ Base requirements
 * To describe relative types (physical value types and a preliminary set of
   nested types) sufficient for an unambiguous implementation
 * Memory layout and random access patterns for each relative type
-* Null representation for nullable types
+* Null value representation
 
 ## Non-goals (for this document
 
@@ -79,28 +79,55 @@ Base requirements
 Any array has a known and fixed length, stored as a 32-bit signed integer, so a
 maximum of 2^31 - 1 elements. We choose a signed int32 for a couple reasons:
 
-* Enhance compatibility with Java and client languages which may have varying quality of support for unsigned integers.
+* Enhance compatibility with Java and client languages which may have varying
+  quality of support for unsigned integers.
 * To encourage developers to compose smaller arrays (each of which contains
   contiguous memory in its leaf nodes) to create larger array structures
   possibly exceeding 2^31 - 1 elements, as opposed to allocating very large
   contiguous memory blocks.
 
-## Nullable and non-nullable arrays
+## Null count
 
-Any relative type can be nullable or non-nullable.
+The number of null value slots is a property of the physical array and
+considered part of the data structure. The null count is stored as a 32-bit
+signed integer, as it may be as large as the array length.
 
-Nullable arrays have a contiguous memory buffer, known as the null bitmask,
-whose length is large enough to have 1 bit for each array slot. Whether any
-array slot is null is encoded in the respective bits of this bitmask, i.e.:
+## Null bitmaps
+
+Any relative type can have null value slots, whether primitive or nested type.
+
+An array with nulls must have a contiguous memory buffer, known as the null (or
+validity) bitmap, whose length is a multiple of 8 bytes (to avoid
+word-alignment concerns) and large enough to have at least 1 bit for each array
+slot.
+
+Whether any array slot is valid (non-null) is encoded in the respective bits of
+this bitmap. A 1 (set bit) for index `j` indicates that the value is not null,
+while a 0 (bit not set) indicates that it is null. Bitmaps are to be
+initialized to be all unset at allocation time.
 
 ```
-is_null[j] -> bitmask[j / 8] & (1 << (j % 8))
+is_valid[j] -> bitmap[j / 8] & (1 << (j % 8))
 ```
 
-Physically, non-nullable (NN) arrays do not have a null bitmask.
+We use [least-significant bit (LSB) numbering][1] (also known as
+bit-endianness). This means that within a group of 8 bits, we read
+right-to-left:
 
-For nested types, if the top-level nested type is nullable, it has its own
-bitmask regardless of whether the child types are nullable.
+```
+values = [0, 1, null, 2, null, 3]
+
+bitmap
+j mod 8   7  6  5  4  3  2  1  0
+          0  0  1  0  1  0  1  1
+```
+
+Arrays having a 0 null count may choose to not allocate the null
+bitmap. Implementations may choose to always allocate one anyway as a matter of
+convenience, but this should be noted when memory is being shared.
+
+Nested type arrays have their own null bitmap and null count regardless of
+the null count and null bits of their child arrays.
 
 ## Primitive value arrays
 
@@ -112,9 +139,8 @@ Internally, the array contains a contiguous memory buffer whose total size is
 equal to the slot width multiplied by the array length. For bit-packed types,
 the size is rounded up to the nearest byte.
 
-The associated null bitmask (for nullable types) is contiguously allocated (as
-described above) but does not need to be adjacent in memory to the values
-buffer.
+The associated null bitmap is contiguously allocated (as described above) but
+does not need to be adjacent in memory to the values buffer.
 
 (diagram not to scale)
 
@@ -180,22 +206,22 @@ For example, the struct (field names shown here as strings for illustration
 purposes)
 
 ```
-Struct [nullable] <
-  name: String (= List<char>) [nullable],
-  age: Int32 [not-nullable]
+Struct <
+  name: String (= List<char>),
+  age: Int32
 >
 ```
 
-has two child arrays, one List<char> array (layout as above) and one
-non-nullable 4-byte physical value array having Int32 (not-null) logical
-type. Here is a diagram showing the full physical layout of this struct:
+has two child arrays, one List<char> array (layout as above) and one 4-byte
+physical value array having Int32 logical type. Here is a diagram showing the
+full physical layout of this struct:
 
 <img src="diagrams/layout-list-of-struct.png" width="400"/>
 
 While a struct does not have physical storage for each of its semantic slots
 (i.e. each scalar C-like struct), an entire struct slot can be set to null via
-the bitmask. Whether each of the child field arrays can have null values
-depends on whether or not the respective relative type is nullable.
+the null bitmap. Any of the child field arrays can have null values according
+to their respective independent null bitmaps.
 
 ## Dense union type
 
@@ -233,8 +259,7 @@ Here is a diagram of an example dense union:
 
 A sparse union has the same structure as a dense union, with the omission of
 the offsets array. In this case, the child arrays are each equal in length to
-the length of the union. This is analogous to a large struct in which all
-fields are nullable.
+the length of the union.
 
 While a sparse union may use significantly more space compared with a dense
 union, it has some advantages that may be desirable in certain use cases:
@@ -251,3 +276,5 @@ the correct value.
 ## References
 
 Drill docs https://drill.apache.org/docs/value-vectors/
+
+[1]: https://en.wikipedia.org/wiki/Bit_numbering
\ No newline at end of file
diff --git a/format/Message.fbs b/format/Message.fbs
index 3ffd20332087a..fc849eedf791a 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -129,8 +129,8 @@ struct FieldNode {
   length: int;
 
   /// The number of observed nulls. Fields with null_count == 0 may choose not
-  /// to write their physical null bitmap out as a materialized buffer, instead
-  /// setting the length of the null buffer to 0.
+  /// to write their physical validity bitmap out as a materialized buffer,
+  /// instead setting the length of the bitmap buffer to 0.
   null_count: int;
 }
 
@@ -148,9 +148,9 @@ table RecordBatch {
   /// Buffers correspond to the pre-ordered flattened buffer tree
   ///
   /// The number of buffers appended to this list depends on the schema. For
-  /// example, most primitive arrays will have 2 buffers, 1 for the null bitmap
-  /// and 1 for the values. For struct arrays, there will only be a single
-  /// buffer for the null bitmap
+  /// example, most primitive arrays will have 2 buffers, 1 for the validity
+  /// bitmap and 1 for the values. For struct arrays, there will only be a
+  /// single buffer for the validity (nulls) bitmap
   buffers: [Buffer];
 }
 
diff --git a/format/README.md b/format/README.md
index 1120e6282a50a..c84e00772c3d6 100644
--- a/format/README.md
+++ b/format/README.md
@@ -3,3 +3,20 @@
 > **Work-in-progress specification documents**. These are discussion documents
 > created by the Arrow developers during late 2015 and in no way represents a
 > finalized specification.
+
+Currently, the Arrow specification consists of these pieces:
+
+- Physical memory layout specification (see Layout.md)
+- Metadata serialized representation (see Message.fbs)
+
+The metadata currently uses Google's [flatbuffers library][1] for serializing a
+couple related pieces of information:
+
+- Schemas for tables or record (row) batches. This contains the logical types,
+  field names, and other metadata. Schemas do not contain any information about
+  actual data.
+- *Data headers* for record (row) batches. These must correspond to a known
+   schema, and enable a system to send and receive Arrow row batches in a form
+   that can be precisely disassembled or reconstructed.
+
+[1]: http://github.com/google/flatbuffers
\ No newline at end of file
diff --git a/format/diagrams/layout-list-of-struct.png b/format/diagrams/layout-list-of-struct.png
index 00d6c6fa441769a3c86044a52186d71c0bc23d54..fb6f2a27e07a766729d12ea33454db011ce6ae00 100644
GIT binary patch
literal 54122
zcmeGEWmJ{h_XZ3Df(j_zCDPs9N_RKX-Q9wKba#g|(kZ2Mce4p;Bo$B)Bz)FBN6-Iw
zp7DHrzq~)kaJaqqU2Cqn)|~UYuDQdN6eJ%b5g@_9z&w_g5>tVJfuDqdfjf8z2afm{
z35kM#U|m!spTks*6YYclAUaBEyTHKUP(uI0N~=(ufD`7d)U;f+<mLEG9PF5kOdX8P
zm^|$q!Pzh{0-k)}qn(+n5vix0t-TAMry$v1XYhf~(8J7Rq<@{_Y9mOdC9gy(>fmff
z%E`pR#6l*7L`q63;B0Enry?ft>vHfvK{88MS4Tc(W)BY!CJ%Nd2WJarR$g9SW)?PP
zHa13Z2BV9Yy{nNYqrD6H-#7VvA2Bl*6K5+&S1Sj5Qs{k+j2+xu1<A;u5BlfN-|y*a
zW&VFpvUmA4EHFT3=r_!)Of1a*+#6gf06oel>R{*SZ06zu?k~hC@Yk7t9Q(iL{C&QX
zrGu*ju!6IdiL|||nKQW9)d>1_LTtaT{{KGXzxPsfwlV{w{_ASiUswP8*st>in4tsz
zmm&VP@?S@RnT3!9nE#nFA*7luwpADy5g2K)XKJ3XN7-(#h}7rbt%*2#cxW>;^LGW+
zH}gl1Z04*Y;28KiAs=m$7&>`)JSJFq@l|l->f^P>TQl$0atrU`+&dHJ!}+<AxBTbj
zE}vUohCC*`5cZ=Wg+-JSfy0P<1^?@#I40hkz58~G|33ED*Cz&WND5!>{pWNHOC(qX
z3)PyX(%(1ueJ>iG_YeMi`#f5BtStF1>w@1W{eE`d`ZtpQdxyWj4Jbxaa^ylNkCyr4
z1;LkrGd}-41fTI-qzi&o!A1Aq0fCqP@1g&Hcxhhv5Y7vH7DEx2ezWrbntL)EKF%k4
zzt8Ap>Du6<AK&(8q}&ewYi5xV3`-@J3r17p$0+%6tczj)onb8TJhZKR8=lBkq9*eH
z+PDW@$q^;A=l|s|G!d@?>uD1Hcfnr<`=6Emwd((W;Tj@q`2=;@^G{Fua1Ek3rlm}x
zSS1c0m9<H**)G<+5e-GbXWKr0w?^X7ep;Mq-)dEyZr(5@aaeV_Sp9GJUy2Yxt#Q1&
z`8t8jP}BLu@273!oQ9p77au1C7wp58jq?N^D^AK9CNaFbwF4x}We4voPCn)O`}Lwr
z%5gzVRTX%5Fk~5OPx}c`@mLJ!wJpR$t3UiWh!wohV$O7UI>URlowyY_UzTaH>b6ic
z{~Vc7Z4j^ZeZ0$rAUCPNN$0&)p|mQZ&N)p3V$8Ld-&flz4E#QRY59}`?rgSf_ioku
zu<~kL-)kpX`1YIXIqb>thvTMY@HS~<T+2~YcVBaV`fi3(ztudg_&u&w@w^9GJk=d{
zKPXW3{cq1Z?#`=K<ygf%POTw+&}WF_zQ9^GC)ijp@Xwx3#)uWZyL4N4F#vhYwFtwi
zQKl6;(y<Xj$hG3w<M83C*mZkRprzo_wr+?DSKrepSLmg)Sq%T@W-v_a1ZgUW(*Utc
ziOxX{{?^TQq9S-x?Bc7_fhP+>kar)wBXzc7h5e4zhf{3(qlVqp1Tk4L(LP*%Zabj#
zfk2o$ZrG@V7W7=O*&0!p_mG*}I~PudsB=f%xaZVViCrh30Smlf5=MFOn4n=sf&Y|p
zEc=B?G-nLo0S$)>)0*h-Al8~Hf{K=9k^7P>N&?U5^_t)HF&LcS&zrBPz{a18<^$h9
zdHHSX4Oi{%n?|ZE$8PYD_8uo=oXZh|yXUMVtUl4}z9(G|z(u!8$Y5^1?`Ap(E!)<^
zZU><Xc^)?|G-v0$ywENlH97n?qbTgS-bqvhMn=NC;&=O_v3dQ7c58H_(s}XPT+OWR
z*zxTTg!i`>%bG9T=AQUn!eYI=zStogSW7kV!>7C&=l^2caeD!*zQxpblz`B;O!i?M
z_l2vn>E=Ke>VnUg_XG5(3NL()8ykj}r8zBdGOYd{r^q`@OSe>=90B{59UK<B`Z2CV
z1wL9tqK}?Ud+v^vT(2y0yh5HDRg1=bsp&!S<Yj7FG_K?tWWqyrpq$rp8B@vxhG*`_
z6!@&6Wfbi7DH}I+>4nGY`F=^R(D|&Y35C|km-esYOnB;FJ_lYOpY))9=T!RgVYqJ`
zMR6(H<0J3vuD;K4@Zk2&0(q>!*+%FS<D%_zvenzG34gzXvXZ)6eRqi28?FHY8wNf0
z8u2xcwU-r>y9*AdeYhnQ3XgBTz5IFm9imfA`72x5(}$~M;)z$3dWdIN+lSNqGQo2z
zn)`F}R?J^h7K%!tAz$In$kWb5h!L9uW)OY#4a?8F?;WWZkyHb&Ez9<8P7&O=6}t3{
zCJH4FZCiFy9uPTp=J<Sul<DBuTfkXlyUoiozlRSHyjTn>N~4WgC@qL-oe;V{c^jr0
zHtl$cs>E=#%|#wfWCr`my1X((oI_!Zb%3ZnHOnkTo2S?RsV+UzrRJ|3D!`s%ayk%G
z@RcAYANx4Z7Fi_2^OaOmY=x#(sY<gprWf&xb)t6Doi(0fBeF~_rpZhRZ47ts)FZrP
z;4m!j!(x$-kuiT?^Zz-0!M$WzFgVKerTws~&2w2wNX4ZK0e5tc05~Zs?{?gvsg?KV
ziqPNT-jzHcMZae#FpGQ57n2V@W&I1-uqHN%aIOrKe;n0M$OI*wAPL<3NYS-#!PeA^
zr|T6KxcxSRC?DXzZ&i{#Zv^|c_Y>G+D9@hmvIU;ja3xGu<DqJ5n<bym>&BLrD2K2T
zd2PoRBKmqS8ik6R?;CvFk)(+M9_$c^NJYb_lo2#dBH(#a5GxdsiO+nrPvEmE@O88&
zp(LC-he!ABY=n6LHJV6o>QZ(w_Wk#d{76LJdv7Y7hH(_Eh>{ig_9;+$&{KO6aP`op
zBVTn+mYe7c5a>G(QF8PlsYr~hJGAao^(*3{6=>(6<NKeF>4fdL08gn$LOy)JZl+D^
z6rTFR9l~*jf@$lIVfOYxYnIn;x^m&-bJ~j$<_<$T-cMOB-P;&lQXl<)-ijq;MiAzu
zWQIynGYDk5FIgd4tGJg6=+6){D{1AY(2IyOunW+$zqEHI8nm*w+DWyTTB0e4N<i}9
z8CR=Ife-4kiFeRo45m-s(BF*N3>SV!nu5I`=CE01xX&P9G+5aD*fBO##JbYPT#fuA
z$J?ZJAM`a8GCC)5EmS#2g3sG(q^YGlCFE~mK7DTcLNx|wtB<`wI$X|yL)MMtL;She
zRdb&{>#(6L^d{o0uMpY(i&H=TC$QtUfywyt{=5E%)m{gB1eFnlXt&v$97T>Pv7#XA
zW>?dY?kJA$S`^cn1A5|qrevW3MXFIv;w;5a(|^U@24dI-dGg%#IVoiI1B@S(OP>Nb
zBg9`5vi&@b^-t1}bJF%==|<;Xp(;Sg$oCCJpy?sM1i?{fNNrG*9W}4aMz1CRrP%Y~
zQK}DM>f^L5OKkm;Icn{mGfIv?IxK549X{~tz8e-|m7262vi$+p|2S8s{1%Dy5Z|u&
zqa4f7$!dm1WAaJc`uH4|abB;$(HM#%C&Ye7Wu0K63)yw+;)FTOqcXJUyyN(oIb@58
zs`j&CGaqX9=LxgUUtbHXA4@dt*m`frOBQ#5<wun%o#AHrb&T5p0(;U5`!r+tuNm?V
z+y@&t;*m^5u{>pB^uu@kNf0A4l5bwHNsQqF4{-;krB8Rdx!JJo52Er@mrhlL><~ee
zya^TWF!M*69BqFkr5Ywy;>TCoP0;sBw9m6TNvpOG{Y%MZ;|B9HCud^L{S&YYM#9!S
zVp|r6>t23mQ`JJnTUR`lFbKDQ5_m=xcr51`Ej0p(Q6V^+;yJ8UHTNzXn9u}vOBgp@
zD@a;9$5UOP`c*T%e<gAgyRt5(v(SA7Hrg}C9ty~@NeI4q805^JKPQ|O!SviTYwT`k
zf4dh`QH~cqn2b1~RPEJ68d@YW=JoxTU}N2tMs|V6K(V1lx0LG5Cx`p~RZ7MNnxANQ
zi7;|dBcM4$KGGMVSK%pk>q#3vDnxeD@N$>jA)BiiJY@+b-d_(wjlo%bPaY+%b>RuO
zs~xv+<zx*qn2K5C99v~%X&ibIR>5Gc{AbpjY}Nj^hXw-7XL8{x#y2za+{7CEqct={
z71?+*af;g(10h!E>{PWzLt;hiLeUMyEXCM+Mr0C>XYY;*{F{iUdr{utts#AWV#<CR
zP$Rpb5<hhxvllT)65n~X;B=b$NQPU{o`sWyO`c1s&rsnC<Wz)&q)E*jmuSXW50Ks6
zDsERIu=ZwoMo?beua(XLO9OqL$AfWLW8ao4NK)-<PFRGcj(#4CEgP#rsE%QqxN|qk
z$`QUBl7o9O2!Fd^`|J8{xe<&@6+(9wTWxU}cy`_Mp`f(F_DbQ_)s+5>7#<gAHfiV!
zcCj!M+F@Myc|;`b0mu9MGJUN)0m}&SewWTp6Kb&28wh6Zc+vEaNpxMkPvckdgK<@E
zNL7(Dts$*L20^t&aExKB(POXwPP+15JmjF!x~Ccxom3g~n4MsNnoAdk6XOb1@S74>
zu4~=1(@PQ@F8#T!NBu#}82Kr1RzqkL9K1bnu_FQ=5(_i|dj!nQN;#LWjEV=NLwgz*
z^fT#baE?-nEL?a8>9UjyB8yUV==fp!W8JtpC}M4**fqIHx*he$b~J^z<M~>9Skf4^
z2ig;Y6YryYdLo(=ZcpLM>U`@{+Hx+00|AhuWYTkSrLJF&!yP_%o54v*_38l8K5dWY
zj_gf1PoL~-Q|fB{>;W^nv(XIt`HfZaD?Wy-6|NzSu$pZv{&QqXhG<q~U)*|A8da;T
zGH<IuYG)$X#M2i4pLbGWwzc2#*7AbcA?a6C<`!ZGAnQ?2^NRltsgmY4k#Jj=zb`mg
zkb*u0aXPc6t$H}sQg@YoyPs!Y=NLl&fdBi6-)l`-U+sLekF)D}IUhapmQT&b+;wbQ
zMUkmH1YnHPU7OZqL}+Fp$iof{ewVdEi68DR-gTJqzs=coBSgV+va)YK?YG@v>Q6gf
zYWOuDC75A9bqTZj{O`GOoza@2nLR+O{_4fmcb3TLI80k2R+5HWcU723kq4t_)h_nM
zHlWa1TQs?R#Hd2K&<XB)N&RI=pC`*<(Wq2SK87xth4!~(e)S8M_vaQZ=Gs>oB*C>0
zc6Dep9H%`kJ|$yk@iyX%Qtp~QlzpYd6a91T=bZ<PX<|ZBItGg=t|wAn(FmI~Wj*3+
z*L@5Ff-c8lOo6OgTC)+0ndF=V{`NM@LX@s_CcY(xk{+wL+ZV<^9tnMt*0Rciv=nl1
zQ>R?5k80kF#J>yOx0S)7<e9k$XLzR&T_W~6)0`#kvno}%`DiIh`@U|-u2F!E`|Egx
zY6VZ`#PRvlnRs|=8ZG6~>9JWIt^i5yH+oz1`4mj3?Siy!dt(}9%r7F>D~KPOl->rO
zaXr50ypns<H=c_bO*|g<UOnHttiyZum@SGJW$t(WLn20c=VRyC$w<dDLnfNj?ZR$a
zYtVl73Gvb6IKj+h_`RZ|GAR$OSB|)QT0$);1>O-&v642{0>R>a?n!LqUDo~0Qt)s?
z*>paGd=oSiyK!d7h72+v`^0vJQfT!Y-QD%29gRzkp0pn=s_@x)54j)HV!Dl0#0Mtl
zyqR%f=|4~_?}7i&+@9G<^J2a+4Hq_yOrS`1x$Ilg%Ep+es@ucyW=FKn!)8~O?HW=X
zR0$yv!2)EqMpIyExt}&{edv}Z+{chrsqa&1rRhp<l6nT&eN_J~_?nxa@O0?CtaX3X
zL1!ks1;dSkqe`GzKv!aLDs6pCZVUxai#Q@@QU+b7$jYWZVc1^wcjhCi?!&aY;KLC)
zg9aHJ9)5(^th3I~OiEj{8k?o^4qOiB=;b2}`o6hGrAc_c(WI01!!@j!ZY%i}76fBN
z1io_c9N}PR^@fM<Z@kjv6V=%1)7DTin=q$@9f7MaAzL*Hn5LtB;L(ePz)H_Ui-y`L
zFlG`fvm^ZokL8t9odCv`K39ir#z1kL1cNt&E-72MAfbyzPnvCQHv)CEhc&QfT@x37
zjh;8CevNND-R@&XrJx5kej9uKHcTXf7IT!=Y(A&+8G2W2>>j9M<D0G{?@xsmu~kp2
zIx})BS|C`o?$u8S`5d!HBqaCaTcmTkn-e*i6l3)><oa^bpQ|UWi_wX7+=$cFV_-N8
z$z&JbbX%xB`AEUzbfRD~LZ$VJpq{+nma9Xhby`spBTH+Qer{}soxF)kA(FQie)=Jt
zkY>e)9ux!%ivYEbJjgS$Rj1+|AMX{Xfm$A}6Jw+C+ir_;F#^S1PZM$M6q}UB<<xm=
zy8h|Uqw#+Oo!|7}PIH}uwjBEMo-G!4k0pFxv874yoipyfk#7?_;i&^TRmn|KrykA*
zrHTU6lo4#rSuqjk^JXa_0k+~>E@ycsDbvVRk2T_FfruO_Ch_6tAqbijp4T}VM#=N?
zyr7EaBS|LGc;dxJs9RZhFeWw2UCq&5C%VbdPc%!#hGD~4zq=8efF~>(nNo?J&R|{X
z+%!!aMIu(YEeb1F%_Cc%`Y1lSxp&R?>tkkS63_L3hq!8G!Fqn7+=^Ac$)fJJ;gKIV
zR?0aN_=6!hMj}x~N`nj%;)#Y<a<bemJDE;{)}s2uq@!uXHl$aR{2%gq1&wEj1<FO!
z7Z%-C*{#tWN<K$$q=MKc#;6}$crb^_z}lG@h?+oRBu7JwMKCpW2{FRWH65<{D79*F
zcuz&vLu#)upEb5VC3#P5u^+;m%3}3Y&*k`}EdW!fTG`X8kJcC8^HVaBh;y(@=xQV@
zHy-M*dhLCV2tAU5@53$abjf$y?=>!D(75-mKIIPZCGlsbyx%#SF-nFL`@Z2tOiQRN
zs)`+jSW!=QU^^_0NI|(}x+ak0nXul=6<Lul7kz3)UP!%DF3etJ@j*@Kalwh$%dg##
z>R~AiuR<zzvz{TEsMBc0<cO?@SFLj+YQmAhiP1QhXc7;7Xs2GCPMQa4zy)YjrwL{j
z$Bk7mKYm@BCKrF+m{{4@<1H83tm;Xpv1ClGTZC_=u~XSFEd#B4l^x@<eQ_P(J_Ye@
z(ye!QF0$M$O#f{{hR!Op*UH6UrI%_A5F_XcEq-^b*f~aw_qlX0Toto^;!+~xjw4k<
zFo8qm%UVunTt;{Ov}CnL1>c^+PxI=OI#l`8J&3=u;;Zjl6|N~pypfn*xnDIqGu327
zAZD#5zY|gcs>{yE2lR=QGPPGVrR}tmua(kD*|khg7M^dROS<l+TTmHFXm{~`^xUef
zALlETudq1k??Q=FLnUXV%Ek)f;D0skisalwS1?^qOFd-BC!a1a1N`xzm30gS)ifl!
zk>&nGF`L+RsSvtt@*J|`m6nVozE0$2XYz3U^Qcg@3K}dCbagrnu8!$3)h0)#L=PHB
zmZ@8m2PQ<-tz^c8o6Aw)k(1JBh)G!+gpbbo>ly$fe92h`&=>LfEqGc@EX)mX;I``&
zc$zlKK6(Q8yZ=!}wDL;{xlZ|)7~=q@4k<<9TUU+*8g&Ar!|kt(Va#v}^~n;!nNGHf
z$y8}FIlgB@4B{vYFc&f#*!M@!rXk~0-c0@D<FjYKxPs4Qlg@#dKh^b!x&l@cB&TKc
zPMmg>gX#s6&}7MZAre=M9&c1$oY7+&QW}-Yue4fn3D#&>(J5@aj93@rGR!gK7m~9O
zFmeIKeA7XVJ(kF%M1E_vEuZeHOJlzRKZC|V`*gxcUGx?S{*}>Nd8dRlp{9&Fb?b@V
zdGy1>#eurk^+Ne7w}%R9`1)Rr&4L3`bgKfyF~Iv=y4hZQqnq^Ph^^GaH)5os(zrT{
zxu+;QI|%!L2nKCs-04bzu)&>Y+!*=fF8J8xYLdZkU(A^-gk{Q}WeQW9Lrnaf<S<8>
z{X~PndR3U1V-dn1@s4ZpcJc*$QxV^N6&!^Uw=c;Oi+oo+_Xhlg8Ti(V^dSM1sLPkn
z6wI1)0g@$x;uRgZPiS!J`nEniC#JUkYh(Mbgs1JHns3+2i+f1O*zfBVtPX~A)iqym
z>M7Rd0^o-;I0`41!<9iEogwtoxk_Vtg-g+DEfReLW?f>vnDhpM0P<WysGU`^9tt{h
zw`KVDu9JWV7xAVz(S8+rmfI-3MpZGa1tyH|QSHEBmvH)(1lcJ6;)goQdaGH|DeX&&
zpUX!O+l=CFZtO~Nn3AKiX$~V>GR&s~B@H6Pyl;rQFI02{-!n*+I0~aY9?}S?P2tRz
zL5zB>6?8@9zGy^SFB5p}o+!`N1xY~gtMC`CFot{nu^u1sy;_;pIxUXy7RFF^mQ5lv
z$D63;B!-DytZ==6x=^=?Qhd*mn40+9n7YRr!=~zVUr;&+%i(9KmM`iwG5gh1_Rq54
zs+G^3CC)sk4Wx@^WW*n=N;C9dP|CcyG`XtKwYxxxN}fz-{+si#<b<oF<T1g0fVV-?
zFY4l^RVGrR=UCnlUPM36^x<;TEHe4LM4poMenP(hzv*DyNr{<9fC7SKcqbPPyt{zn
zm^r3^iTH@rj7-pI3Nytj#A_8Gi4`?;%bbKhHPgwe=pEd{Vxvaz(*V}hXu{z?LpKjW
zoi4Wxur(LMkSHjAc~xgozuzD!TUTT{F`tUkGO}e0Ik3obl3;q0+AWTw{aP+6m!W4Q
zJ6iO)hXfgd1(R+CM?J{FFCPkgUvWu;D9@(qd((y(+@zR<_Dr84?AClhS#Z6t)Ff}b
z{F>VC=s>wi4l+7EURvf^Pi}isLcXk4#NlA=Y1SEePyfjYX4hEf<55C*J;g}x7b<0k
z-Cq+8?g>8>yHB_5w-jUz5UMD+8`s`cf=H%2^3u<B*yR=%R8eyl2im{(I%M?Amz?sq
z{>D`23r{{!g>+e>jf|dH3=^RraUhv6ZV(GoPWNN`-)ub@QQIpO@wVPZFLC`Ua#U*7
zhkj>RbZlbtc;X{K?S&rkJS?0xR(JjgK*}k}C)v_QH1w-333!zZ@=>&@pB0tXnrLim
z(H`$!c#fjf^?L<daqIL1qH9ttqaa@xqf<Fx)@9DGn-aDqi@_}0VozHpguO@9eN@;|
zUlcmHYMRkcF-RXZ&9+^W!&3Qh%#H=Kssy0AFRP40J#cqe+aCl>+Z}r8qI)19)cMSm
zU>Ax_ZUjHCQq76&v5t5yDsC~ML157&Cxbb$n-_!<G1vF#RU3jPT+X^`fD~3EBa;43
z7i=s<50?nzC-3;#$EQ9``0Q`*IV|BI=)QFKq$RdrmV-sMGoAlF6W=60^!Zb|cR0MT
z*a~mE9is7Kk5hJ^YZkqR_h1CfJi^ISeW!g$9WSwcqzZqWWydr^9|GynU=ONryDQ7h
zVt;C|gJAJEO&poehP19uk+;GsUOD&S?+K}<MpGnxtWe*0u|Z*OIEuut&xnb9h^FqD
zR_q&^kmn8CE3HF5kifpeMmw>EvcxQ4GNVGzz<61YeoetWs8x!iYIv*l*_JWXDpQ`|
zY9{yY^%UwdF1gXp@@q_mG##cn<#R-ZcBTUPUJ_XvZM=~EJlCwoQF_F2!>|E{$7F0%
z<!Q!Y?7aPWCSnG7E6y(p?be)#cAlkw@-!cl3cJm`Kpr}NnPKzPCykTB_>mDFjlU_Q
zW@1f|zz57+F1x{@56F<#!Wk0mQYsQVtLTVr%?_{p_;3C2m>k&Jbz$|CaUKveT#~$&
zzzyLn>@2lj3=-4H&M2H}D>&w-LaeS2I1Y@<qnM`GWVYlGWH7b*)08W6iX<tRVy|Dn
zk)cdn@5PBYG(ShBf|TNEnC0sixyRiaxo_>jxk#SZV#LZ@uE*X<cUgs_X1$HG?2oc+
zs`orZ%R0YxRm06{N9pt+$G_L&)y&6$GJ#Q^puqH<SKXjUW+SkE7@_YZ$4^1~PR93z
ze(7n!a4JHkoj2#b0FHIX$aSpYL}txoRAK%yjUYT#{@*~+u{3FpGGUevGNQs}@oU8R
zfzK+8n(jFQHTMzp@N0j%(jf9Nbc)JxX#K#24PUm{YvqaDvb0x&a9TYqS%K)FcwtU_
zwfDxNC%&K2ek5w~5+oh*75Y=S<8eh>6=u*{;D)oTclxw`<5`)`?$Rks_M^d~My2*5
zu20iW#YDE5`>W5aY-k@y8!R1l2HXR#>}WCV9LkMqX8BliTJM}^JA;g%M<)|AOUtP~
zK}a*3+E!n*u&hX|$f+1E8DlYZvEBlJ_>nP-+?2_3f$ZENb)iQYnAoxi)K^xl?vPf~
zd~?-~c&Hs){<7mYl>tRbx2_?&F_8i034o1_XKWKy9fVer@{}7W2_V0%;8|aQd9ZXu
zWHRSbv`m|s`!aHtwEqZ3PWErrz*3yZ4eIiGudd}|>d)B%)$H`6;f<xiH26zV@3w5;
z$GzMwWghbuCrL^~q#@XNW|3*vWYmJizhIIr+e$;JVqiOqKjDSv?n$J0)kv*`D-dFg
zPUdOi&W(H|e*5FRClO~Vp$VhxJw%3*wfBwSr0{61D_@yL$*#ES;-34eo91Y883GgL
zU1ykzT;tFVXEp-vL@{6s9}rqmh+1Zu1yGuc3a}?mk<YzN5P7pP6De#|)mR#sOM?7L
z+it|g*Q(xjFLZ>1kny#TOR~O<;uJxM_K&ggcuOYwiW)c#4Wd5T@;Fi0K|>bPx|VZK
z7+7<N<1@F|MC53BM0&J$_{&#G7&QEY<K}1b8-Z1|{RA!75ucma(j4~fCFC=dUwZQM
zuif|OlWE%;Ktp~yUUZrJ)tmydEi0=rF?3q3-@I5q$4v93>ES(z_1u-&OC&U70&bIv
zTPuNz$noHE`I5KP5#3tvaVLK336z3Gp1|8A&k*NCl+4{H+^<PxV(Mf^LKf?(;sgM^
z0-!@7u<PH&S#&QxF<|@IdOxE<o*$`AL6uywt^6@9V7}oK@`UOQfk}Yn%VQ3$@)WN(
zsbo<z!MOTi6Lf+qk&SkMw(YCNj#3|Of^5J(;Wj6!JAF!lW5#&RVP5`enrHTnz!lmM
zA|bkFXcr7S9NZ@)T+gsEobSq?$-KoR=I<tBuTm*vR(xjU*dK>%)&;h7I!$9ud1n%=
zV|_O)?)hQrN9kB>re9=`#gwUBe>#6Vqb!qmdI;zE!By9`7QueB==hxi%GUP9Iz6i3
zXRpt3D>kVTAxk{{65j8s{sER~#13hBBHJH_EyWIn;jj`Ki_z8X+uMQRBqNR66W3+S
z=B*+c+76_Rx@Xs}*|u#t-u=Vl<cK^CtPj?+=2y(yu||szX05$8h|pMa7Ok)mdGDt0
z)>V@L?+pXX^^#JLtLe~@dk>b}J?!_@-&Ej`#wC~;PNOa1ELi3X`{rYOj%p>{5e*86
zT~%*C@>Sb+TfM{BQ-QNxk~THxa*|*E>1y@eo$4P+$6cz(+E4jQhiv$VIM17PnobRW
zqGph;qw%^m&B@GpAGYg+*P6tO)IUTk!xoNgxF=uVTKnSf*bd6=NpzqPl6=j_r`U^Z
z%qT%iwxZ{EwWDr)>oKc^oc*QptM$`f6md9tauLJh;zqlk-|sDzhY8d|*{{@_Xr7Tm
zh7;aDhM)!Ss6gJ!4X1peP0!9=n3iRJ3FvWF{>|?fNxXat|2z`P6Xun{N2x5UwR183
zGmgJ7lsric78$$Pc3TvkKQ&GwnP}jPOg+mI+W%Zpj!47PmeG7G{io{45)&Aft7TF_
z^*=Y##8h&0=XAPG{6|&w*O(@$z<BJO-((m6?G(T77Enc%w|->3c|rZ3=gL2%;aT2(
z&M*JRz(n9)$OEm@T<`IJL|R|~N{&rK!R{IVnDFmMv}OR!O?mio?th-k^)zqYHWB%l
z<2SMTix6_<1T&_I`A+yh&$WYUtf+98tbQ-;-+Zh!FhmuVz5jonyYw17SDvY1Ch0$h
zm;;7*A-{6>AAMK@V!#JGwh5-p-PXgZ>rwWJD4-WW6~1vybz7Q^85jJ%QV_!z!32o#
zcA!OQD0?||scG=i?fa^S<`=WhSFq$UJjUm2Imv%Ktu-6G#JcRsyspFhvtfFuuAu9f
ztEFWth99Mfdx#?O2=K-&RYE^+T+GFG0ri~9KNr>0_PNz2nsdJN<Jr44KmH$k1)9uT
zzp|rWch6gfdc`e9LoZAW;N*dJ<l=ClTFue@^s)xk!ztGN&)MV%2d^P*XxF3w2Y6%#
z&iD^M0sT$K$qE=<bw0BXe@x9JA23j*3nS5xKjQ0P7ic=`IxS5}_J@CbvxJi<JDy6d
ze)st9`b&rZ`J86@+=|P@>hkWc@4qcO=?CB7=)TYC|0&y}3W!Vw{C?hCZpEs~bI!@o
zS2h5s3rZHYI1vHi1eTWNn_wVEA<jQvah;L>{?X?PP<}{S=0eOf?Z$aOt(e3L_7gg^
z>s7j50(vON|L#gr;I#L_6JM59*BQpzUJMX~x>aNuIYFfZq!-b~fc?87av6037q@?#
zmL-~I@CQ`(EtKvh-~~dK@J7P@BLkfe>w(CZ*Faw($A8@LC<ITqysFi@=@AI;e%_OL
z&*EDE<IOe$t>2j2Pr3oVHeA-Sn^A5H%l?P+2IPntYMJDwS)15>d%Fb0IWTfzD!P2-
zfXr~sa_Xmj&r6kN7)YJ#%cki!$~sWE&9!VRDyH$;*s7AYI1LUlqZ~UF493yQ%UMD2
zeAef43$+4JW2j?6r8FZc6p4;)Q%g|x$xKTv-1TUtM9~Vn{|XcVW)Hps3XC4x2hfmp
zC2tEMft1!SFCZ3wv-tl~IazWAc(OUv;4)@O<8X4<e#-I50kq_|E<X~${Zrty16d6o
z|3)P9hf6SaAoQqvx9YwOSo1`}Ua%Bi-NwD7jZf9$w@5cji52>>_v5hoCs15K5f^#x
zRjdcIA71|aLIzbPS#|(nOq{y}ffWUTp65m|Yn(AbUqTC*6gxnCK$xg<eb4YMvf_yt
za?@}mNxZhpN^<y+Of#di*e3+OM$lIo-jiFRC`G+cWN3VpSr1jo^b{P@Xn$YR>KwxB
zC-FIgs+hpCdilBK)1DI;YwW{TrlX3?O6-46$>d{r0;~$wO%VCz=SFmRpi&W-U^*aE
zS(x_qzdq(Rode>EVeO-=%f@0=qYWaX*FTx+R)NcFucRr|vhD^VGvPhXXH*pmhssvY
z&`T8s&($tw!48l#7pJN3Wp=?ZoEQ<-YHMDY7@P<PqWIYD_qAL`7a$*zjD^-`TpdEl
z7C-N9t?YnjX9JflfE*bd^#Ir6>*iUG|0R?8y{_@$H<jDFQht_O+;Aj!0<BTq2vgf+
zA7EL1fy!i@rRPx@|B7lYABj^h`nKVx(TE0`ATp@zMp5W$8?7il+=?}-h)4*?bmE5$
zlC-u>N_&^N-aT>#K!vn3hx)!6+&OgcOs#wQ5FJ9hhAAL}P#gzq3cRmDp(>$m4eVC9
zV%Ul6TN$zov95#2V^^S|T6XMttUZDhz+wx6XDa_uO%Et=UlbuI{uz`iMZp&R&4$(U
z#PH_%{_d-LD4kK-OKbLR$01&EzP=i=6C{3@8%60y1Q<57P*IobMTSjP7ki4RD?vb@
zik}Qr$W}BJJ`F*^1S`&7p;sdLYB!S^1JV^2BoeI{$|woDNw{GyM_{oIutuA`$v3Hb
z&O-)2e`rS4dZHAJ7A%4yzyK^3g5`#^KR2~#u&K$1J~_nLHwVs3yNv(VcE2cf7i(AM
z=rx3mC%(>}JrmRFQBEn_H2i|Jg_C<UM<BbIIc7l=+c0>^^s#-ljG|1lk@}BpXZ1m}
zx^>=E#fC@ey_95p%J#f8DE*ftnA)j0S_`c}$ffQuT>nQn5}`-C?C@l-WXaIyz2EdB
z%xi-{hpU&~LOTWD>>&cy{5Vfkm{9!ZlP_Sy!Z#bv;N};YGrG<ukPEF$#z}J?rmeL|
z*)7Qe@#79Sgb6Rbk%MY6;vW$*&kTfwOg}4qFhzS?=L-g3YS;rO%|}dA?ajZA6hj6n
z&ZyH(6Y3BK84hQ3%N&_HRD*`*7`Nlb%oq8;f%z0qsn>5f;(8P8YD)g+%RdFS3pT8n
ze^?v1$?+!0(YjLOoBy4~+p^-f`gy~U?IoF;H+6)5D&K+YS5H4b%>B{x<nGnFY1U%2
z%SNP*|4EF}zmt?>38<UUk5fGX(jfNLnzkVjZCBdw%rmTtLFkLTK>V0^J~DQZrc0h7
zICpFnc)vC(@xAU_+op7Q7fSKf<9bJ!KVOi{a5C?8zBVGf1;;JO%hV5o#(2Nn-F^l3
z(x96$5(;D=*7I0(GefTbaSHY<W}mm%Y!D-ad(6m6_!dQ{y@6U6FS};7%c8FSyl*5_
z1b+LxMxGKb-2SaC=}-(g{96!e<O@t+n&<#gp9GPU*Z~RwN0?GoQM8`xlsGM)@;=0V
zBAli<Dr;3{1SRB!NKZE}0J=;bGV1pK!q<N<b~lh>W%@A|XS**6-|j2A_B2u2zU2j4
zP`y-KJ;yFpML}_@<#aV9lgx6J3pD>##-_zURiMp#x}3WmRm8pOib?2~QYmHraMhvX
zwn-&me3!jI_~#c8X+M?~d4oNtU=fHO)8I&pB5X&7$QF47&IZX<Km5P^#**qFLbm53
z0cb?;Ku#mR2?Fmb0HXo8PZIc{Zi8Zkzq)0Y;65wFG?DU|`u8*x!EciwL?W^8GQL}>
zn|zL32_kzWV-wzT5y$}qHT9hO0H*%B1w?lTM!P2l8WFYKn6;7=T1TNg&2`siRzn0X
zCr`|A6KUGJ?i0>pJhB2RIFsu!z}sDeIs3d&y#~!HHY4bvJ02L^1+m9U*X)OzGg*Qa
zAh%28sB~h%`7^_94F)+5eGN35NaF^Y$d@UkWC&N=-r`FelEV^7EjQyN`G+8GuyWn_
z{_q!u;6U|+fJ767x8PYTt`8GTVhM%Dd($ewy+l78pLH(5fQIT3ZN!(~z;b$Wa!svC
zta}RLZ|2CiKO7pvQXZ~#)_uyY%Qg&a1;|nnBNC4jqE&#7i~1;FQK0_Htt4FW8l)^F
z067L5y6tOcvuhc-Y#Qly>lvY3UZ=J%Kwk<*A7uwtVs+bLKfUq!`>*5tAOM_co2OGQ
z6BHN+Q$Nqvj$-Y{gQ)%^M*J(Z2c%F=mbgkAmGl$Xpb**?stR5Nb~lF@zv>MO?Vus3
z?iu4f%wbyyrs3x~EG~NEf4#hhB6Mj;U|Oy4+H#fA<5?ew_i)59K8%}~O5{=NBk@dm
z_PGCE370?Nk*2(n#y`3UqpyKf&%jK`^F(85*cs$iz99D2`jUO}o*2yz|6|Wp;s|&S
zU!qZv&n)}BF;oz<LzRLuvm%MOZ@nu>IWtQKn$+O1{I`Ay0Gka$_t9>#1fd;A;3QiZ
z;K5=Yn}H!@-9>u4UpTx!A2fP^<!&*Q!^ZeWPAZj$5ARxLcYn^_<||<MsIeGzHP`|X
zsk7pw>U%3+0-0Kv=C{pA=6;oBQ50U&_0E7*E;0U;KN5CtxPa$+*4SBs8!8^=QKBJ_
zAVA9u(Jtgjm;qv5*(6OhON*uYe`L#l8EG;Gyz;T<oK4nO5J;Usc>Mqz|B3Gjtg^Mm
zkK6ja{K(c4sc0C+|0`-KNCUX2LH+ZUuyg+xF!VcVw#0#2?O9NUo0six5$CUAie!S?
zuIcLW`nP}QIG_&Vhy=c`ZEE+w^Z#cbs2}~$AO7m?`rmr_Rr>wUAO7bL|1K8(zx`ov
zdH1Ww4-88)xblj+VGyWwy<W%2|JKqqQL-3zz5+q6)~@7BuMNed#bS*SCk~@-g<`JI
zHus_wBxMZ1ZJXDW5B_=FWCpw%Y^B$W2m<w|I*P~^%Bel>6q5{acS+ol70189JSApW
zgUAKhs^f!}kose$7R*1xGQ}h(h_NqvAo{<<<S)+#@^J)<<rmJ>4F5?5Y(Q?^7E=HI
zca!HYw9t|tpoP2oXCUL*Tz2TNvwza`Ux=0xT1Po+>-(PeHO}kU09_X`D3ALAj0E7P
z6np#k=kvlhcP&SCBLw^BKrJ8Re*ZDSY?*si^JYY;Cy;)pR$g`^Gec1ikh*-K%CvO`
z4GJ$nSq73kdq{%$?=k-dFyme==evfL&LE&!MKZP0G>xf!J*@7a8e0SSLa^9{8l9>S
zT%uJ2G^bh6k4!M|yE0RYIyF7?1DP1Km<O;z@)(8SL1{t0059ie82PS(7tMdn!3WIb
zCUl-92+%}P6(#_c5!yFz078@p^m;AKC~3|;>lz2;SoQl09ao+yG0ZWJ899K&jDlld
z0dac-kVt4-5>WXrtM}AC`Dc2egUCw-brO#M(|Bu+Ve@8a6#@E<a{1B#1O)|9QE;^k
z1Ck&};h;Pep50Ybl@DG%2EfW-so(AS0u*WkrNbpKRaM((z7>ztdb{8$zWaYZq2#Y8
z6aiqvgrM(jx_ahzjo%pn7-&9p^y~N@R=xv(nc5N_=&1YL@zamPq9GLZ7~r`47mFs<
zAfMj`4JSgtZ&~SDYd==}nKm|{!v@{wJT(4}W#<mw=O<_&QCb9;MZ%Jtk_M<``fzlk
zdY~OJ@scFcSHGFh-|4ZdlnAm#XqYvlh}=S)DYQ6q3!AJ5(52*#D5_wRRwL-kS^-TY
zBIA6A74@K0N^Xk0=;^;)G>Lr=5J^k`U{=(Rx6WzkG6J}*h_ZaU#y82G<mETz`=Xa*
zk188#K+oJLsDjLFRzc+x-vJ7yB*soQHHki2nG|Aoh|Yig2~34MYR$qR=+4N@Ajhyh
z3VDt)jM;T{P+l#8^3=<D_&8Bf=$S4Vmg+eaVTE>;Z70Y!0iX}`2Ng*8>;u&5{l$_^
z2j~xkxTEC`Jn<J8k%Sab?v%juH^%|2E*{{9pOez%dlM9#^Gz?3{=~3E;MBp|EAQN2
zDU+s3Hw6;R!f~+FZozHOkC#1`tV*J(-hQDC6vQOoo_r%pD=@-P&rml=mc|_};0mmT
zCY(|!<LI^H5So!FQl~}U@$=4S>6vaFr)-Dc?V`@tH=!iJiDS{;i;YE}7Iui;e-3fx
z0j*#KysCa&=!s;Zb2$*-ncwsCYyH0knX|^Hneckao4Yq~Q<m{wP&|)nOy>aFc2+9?
z?W3|JaU<x>qBP<K;4!fI5va)c0Yw-ltR}dQcAW}E>fdmI<>2y`BAFi)>G!~T7$lEv
z2S#++dir4s)N0SEeBNy<6!m~D1NT&}7FFny)GXs)@GJD=<Wt~oZegV@2u#2Mpm>K6
zsK(Zzm1vo~GiYt$OOcl#%>~vXwlPh9lLd-MBXY`yBY;V!&QhscVlw)I?Ic;_C+Jrs
zPef}hvLFvV08LJORvdOkY`g!dCTJ-0))~yEi+Hs-_felL0nF`PgNZxk6|{v=Xv5#W
zb-#ETj`kD47o)y`2E<hL$}(sxT6iTJu>{=0np70a^G5!s<Hg;XLAhJMe+!^RLLdq7
zJ)~r~`k(FCwEsm}Aw5-+H^-U|h*o?0Y6|0%E>g|;pcHwuT9YY2BglUO?22CZ)hGbv
zLxKw^cGFB^z^LBY)&d^5oCC!!p?E_}w}~v>si~EwQf=WTh4SR<gfMeWC!8(nNNAcm
zBmcPl6TFQ&uq!iuT&Us5*fXHT(|U(m5%~b^9g=^dg;?mlX0YcL%;!A|;$v7LM>@GT
z62%2b4iZ+K2fv6lO9Z&8jid4M695Da6f6U*M0#)OKs0<$$ZTX%VO!I|6u=(i>QBs)
zl*vkdV47xCVyV+C6o(21!zjl?f~X}%7{4ua+_^>27VpspeW`QW)DyX3w9PD&gi|VU
zE$RjMg&*Jrol@Qj9610j80xSE?InG6JI1}3Y#$qZf%_Y!=g^&8<%3H4{Kn9*ul|WV
zz$}tOtpA~5ctMOJienP=1z>A|Fo{LIVuRU(XN8cuE7Ri3+BpPKh6iI@7?JEl;g+Hh
zRoZBL996u~GDkkMd`V26>!W&n;Sif|_cX5n4o}^z77OHi_x%B`FyN6$Iw2N7)sEw{
z?7lb!wjT8H6CkwH{DPK6tk6=1unqY?#?q1wL=4qa_emaE?t)(5uVe2<*+$fu1wjj;
z8eN?6s2?D{l857d0x)1_T!Ezvz95>5z?!|y&7w+<EjQ(0PEQ)u2Q9z@&_|%d&!`jh
zr9N)rd5%^e$q*lxnCga_A*&iXd|$i)6$hTTD?f8d^NS797gcu{bN!=>*=zGpQNJU7
zDHPbjkr|^)lzYO{M&dHc>gJIsV3o+ju&h?`bu>Ez;D(D2zk#MYv4EuHB(Q_%sl73+
zstOfMg*l@oB7e7NjBLB`^KwAgbKh}**rSHhb|PDiU1EIx8AqhLYNPgpkND<kZ+`eG
zxHVW|{|-CeI05M1^Nv0cOT_1dpS`3$$f%MVq6Tn7b%WCh3^YjB)b!bJn|DLR={1SB
zthd)2Quvcc6e&k=*y|sl8w!$K$lVv%iefkRUIN>;$;pQ4auC(DndscfvRUb|)bET+
zUjl>=Zi7j64ecEe2fWXQs2$pln|4xkt6Vv3`vA*ilc30YGb4j6HvZ)2r6dLX6a$nS
zs-r)JV(FnI-lYvcz|QUm@cTr?(KKir!Hu{8JG@coM5*}D^ug{oF*p#suK}_=G{_Ax
z#k?CA{&};a(>~P<E))jt0ydHkR~7CR$9oQfgBJ4S2a$Lzr871l6#xPIlX(BEsv@xm
z@%IzJ%1G4$QYv)J1hHEVe%BQ_UU4+pAGC8{mSIGNrjoz*+sQ^=3xuM6pkGDq_U?M^
z=k76L=*mq)+*v(h3R5mXvbRs@3OV3hnoPOm8Qt+V2?q#q%vk|Iis^whm;KcW;5;Kw
zytdKlprgC#LofueO^iDL9F>gc%$Mb?R^S!>)tY$%Ftr3yQWnP;fitOTTOVC_13iV)
zjZBgN_r=YJB_1K(LzJm;6G$o{(2&9AcOTlY2?62QaZX*cvM!m)4H3-^bZugiM|kLZ
zo^9zq)7xaKF`rol-I+ZwGS%c2;~)omK;rpIH(q@c8Zt<5KyD^Psp?yIAkboA*`k{J
zR5ZkJBG)uQ+PM)hl3soREjyQJ<k5n0P?ije>q5H`s!T89)^xc9`^kwH5<a}V9tYGP
zXgB->0Lf<@Teh`H<{9vWpLF}%AHqpuf3VS;aRY2oFt*0v-9_C4^<?H*XovaZ&LTy|
z%-@+W2dph|w6C#!bi?HJ!Pz1#P7Z>LYTLova<~O3v!-ymI7TOP=C%1rTVO{ra6m6G
z#xcdQPJl>~3WH;5V~Q*J(;LQ|X-_QBB<naF3m>ZuL2DHFRtV=BfAMMiiWyvk;5W4Z
zn*WpdcZ_h+9$p4}^qRbExLLBP7T-eO2$A!(8FByqN%$x4TSI4mfCAPBxGZu~AlM7;
z<prJ0t^&bmK8yc8Jwf5-uZB(eFc#St22Oq0Q*bJDX1f^o9-u_AmCme`$%KMda`fwa
zMYU#_mJa6ebQ%!q`-mA4I~eWrLrv-O4gem;GT+J{6uaIz=znVTLX<Bbk(}C&#u+rt
zosBWJ`D5@sh~2AuyooUyS$HI+;thK6cZ`!O)idx>7L`*QH;WL4cav^w^C`N0pud7Z
zK13ClZN)@<#;|M!k;~8l<iI}GMm9YlL#dOBhTqF1ub6JVxX1h~&KF8qx^Kr*IWGg&
zIP)-2RKJV#t?Y;-ivE4yXDXEk2BjeB3($e`r#kO0=+g+epF&e&8n{zr4K<slnGATO
zmf@c>>nlk3VnHUBjR)BnS&#gH&!I*MApN}k;y)1c`o$?oeY-!z<5-Y@j()3*<eUx9
zdXnQ4rC`qcZI}}o0_Fh!b^;t8eo4Y(3VI%^l+<s^PP59AedFPNBEiO*P?S=X<b*1x
zhXV%lg#>ApZ0)L(N1r>-(M7zDLb<=P7%h9IRG<rB%xI<5AEC}zUR+(qiK2wxTt>cN
zT6*@e&{w~jw37wfIhN7|^Si%y^FTM|6%DNY7BD7XiiipM<`uu0hO{XsMG8I2_az(V
zN0}=ZtSlew+cUy(E;TkCMyWq;onT+y?sfRXXqBZx>CK2DPeaS7!Hq+?${PP^;<fp@
zXg%-!B33+xA!Sm;@nlVXUNHjeCr=vy>Tj4Y_X8gV#1w-n`41!eb1+o=X<q;;-jL{r
zv~EyfOsnL$8I10-y=jaZ`sn*3W^xrhj<CM#Mij5zU0v_Nn%W*rBOdSV@d3812=px8
zbm^<gttL}x_DBU*UMhNe9L<O#x~Du|sDq?dm|iU)yHySLaM@Ro2s0j)Q<tj25PH9`
zm0)yOjHKyUR+J(WGS+c>>`*3i&WyE&BDcm$-ai`o9Im6N^{@2b6+hte=|SD+qf*_6
zSk9o8uFs%t97#dn_G^1G3IArR`OmX6{%a*IX~#3Q*DuQw5W*j{eX9W?L{h1gQQMCA
z-8X2cRhz63<Hzuu7_&+>nBp}?67|jP=H*2^`OW$kU&*uKEwm)3vIg(lCN%e{kP1Gj
z2BoMG!*leF{%{0Nl6fy9^z5_a&UBV#p{_*m3l|$aBR(mvNW>WFu1RdFWLFiRL8=8L
z>YISN44-$T>9pX~#pqZAzx&Xq-79*5wJ&4vNw0S0DFG<~XlJIcFPT){nCj%us}QG8
zHiA(TTf^YyVM|=6$aV0*$aliI1Hsi7bY|91hyCizRKPo}>v)=m1{!)#OU1Duvp&gU
z0fFq6<T>?RcCEV#dutv3Q4!h}4^+c}=$I+`evcQwmwVpirPru-i3umHV7^=ZcM|yk
z8oQ=+x7$H)RVXvBT{O_w_nx;umY0t|hjU(w{0u7ZSaJKbz@y+)tCEb;p`_J)rL7Q9
z;n@|?)iGV3K3nk@nbn68mRoJS+|Z~&t~98w0cufgdcij-Ze{YTUz>yaXthZ;4~*@*
z-?3a5N?pkWvS%0vpncXqRU(W&DGVkK3YMm&kSkk{GOrCVJ*QKA&k-E|^zpP9E+);k
zY!0D|ryJRGt*UfE?8B$~@$XO}H&WDH(y+c)TCM4fi8-;nBX&LAeMj2U(mTeU6WN<A
z^E4{ofvP~p0=?Xc+nlj>-!|f<PUJChq!I>+?_*VR+OEqU<X5cgKhn;m)qe0<j3gD8
zKP-g)G>ZP^4cJeI)O+hmM=SRdUFqm~T&&rL4dsxHq%*a%LrPA++}?P2xmz<*?$PI~
zY~AE0ij>K5DT9_atUj|osWdy`X<A<bVQomgx~F-hcm|UeS>`s`+bAEx;1qhqcaBdt
z%$|T2Eel?`wTHId`?eTaMPI0Bf;#M32FeB0VP8~4?q(~1p770GQzxt8qZ4Fo$`(fk
zF7x-IuEP^|DEvoCtsLZ>HqSx&B2o*#IZ(35Wmh@(_adlt(`Hp#&kIAvLuDH3RUSa|
zO4)ptCBkeQJY9D1x{H(!ahG&{|BW)g`a*Z`U#2Bi*TD*<Wy@5|7;tSM6Fid|`)dCg
zyc1rrOs-}9#K6P1dM{AIiGpHwRF~K+v?FlX$68|&bGt2z#9x-})EooaSb(-fhU}_r
zvrDT?tGCGrPmE<4+LgI^Nszb-lGb|S_<jbQl1l`s641u=Hmk9u2#(@2aJAAjb*Ij!
zn9I1{{JK0viOhmcdr8`~P($wZE5&;)rpUIL$ipsw-NrJE4oozZ75gDwy{u%sSDQ7*
zeV!&4dGG!ssdNZ@M?Qo9P7z_%h;$-#S9qIqlVf3v$kUSW&6M%8v53qkLDs;SVT4$6
z(@Oj|&&|pccNr0%t>)Wwo*gs2c$-na`35j%0sV-*4w_rXi_htSS-_m@USaSzxl=0*
zrV1dlG8Qt}?1|cl$Kv(WHv-Q~pEor}HqDJ?RWVa^OY7(5>RTQ~9H&5Z=4uUlGNNST
zdQl8X(=6NPe2l7&<DD7SMw=+M5y;+{{-T~hFy9xCdPFFm&+3u1qj|JBXwH3Ii{igq
z8YvQy+*;&-qk;{0aUW-1v3OJ_JRhF=m>SjE*1NgI99LcA;Js*9c$7u08H1dN$D0qs
z5}${faz7)9S#foE^TAUyH77^(Z7R;+*QARZv$$gYIPQdfsjJ~Qe;1s;?Y?Z6I!|Gu
z)K|dYn0~wfZFmlN{;+pK^Qk;UY6Jre`sYI4h9vmGkq2;|*ddveirb5U37Iw-K2lgH
z=vkK>#W%KOeFj73#T*wL`Dukc)9@W7i;YTAcwv{MTrAzGIFUpY@Q?jKk&_mpL{+Ax
z#ei?O^o%2$E>p^9Q3Ib)<#9Q!)?ytTzj|FlvRzG=oLUzYxsw@a$RxYJwE+c?rrD@A
z<36<LzkQUGkkxk<wIn71C7;Ctx<vzgqjG64w5qyX2j4CweGY^t<UvmkU4Dh9EH=Cm
z)Zgz!plLT>5Yk=!63_B2013G+e4pDqa{!e{#W)?ofCWDW@fj_#oY*6}u`etS?lIGA
zwj85}ieYT_jA;~Ugc_9EzA%@6==yPd{n4ntnCt5|$(gXEuLotGbLjT{ZVpF=;+ZgY
zFQoD)9~!beQl)sy9K21-&qlHaNv}DgdR==`ZOh13p4BH#WP+lCW0qVg1hB6H4#%J4
z<`M+xQnN}pOlPh!v6R<#2@~t0j>c`}4I4&PWfY8_m*d&PB+fWp($}6RDcL?vj^tKi
z=r^_YM(^bUF*Dhn<ITe<b(>C)go=s~pskLW(gc~M&wXV^EL`HgL3+$f5Sd5kC?r_#
ze_Gs4@(J4i`o=~e05QM6Ekq|Ipj4twC86yhGK;dWl4YY>p<KkMTp@&@ywErs+hIj*
zWn*RNh(Dtddxq4nqgPNNNcx2s*^7BSzhu#eH6!TUcuX_jq(*=$5A%AAY`=W=Ea`8G
zInN8=!DA*i;Ai3(;b^VZ7<=*^=ON7M(#y7mS+@Dp+7v`FNet!%EU;=?S>aWka$;F@
zWW@piCNe{y4nBny0^ozE{=gu9U}B%uk^V^85f?RtHK~}~6#1$bee8HYfpHYVQ=vRy
ze&|joe?3SsOZ7BK2>d?H;wD+tL@8Wal3iu1y<AXiv`WHfID*gFW`uI-R>BEMGe_`&
z(J;zKaD9G-BAvI%9z<egMFu#zU+G{A%+5Be*zj!6o2SS&JQJUb>&q2}YnEY`kY%y$
zPhJtHI26JVo)G!poHlg6Y9WL7+H4xA8}@m8KZ@RsvP*j(d2Yjy#q}P=H-TUi(QFhO
z?Oz(+$wvS$Fjs1nM{FR1z0s_#VqwT}dVorzXXVRtjZPC>4^6JUNqHf~XIB2*Ns01m
ze37poHimmV87sE9wOy^V5~KAFr(xS;O`@I{dpZgluL+*FBvq<p4E5ZCEM7wB{wUpD
zVph?kz{<}hH`7)0JL~F^>tY9n?o@R&-LH7&u&VO&XF!LdUC6^vYMws*R7FH=9?i*>
z)5qzh%qd^`k@QmtFz67~aRT%q(rHTXiM^|wJB&tl#s&crGQp25)DY6zliVO0j*z~G
zTth426oZH}8&|-<9gUFR0@p}{!7DJk3uHF~-cz0qDzf6kYCoQ<mnqCrZu51Gddc^s
zJqvJEjJGXU`}W&Cq~RqjRob}dVG$d!0<eOU*Gqf*%D!m>F=fvetP$O%wz{tH_1f^5
zBJ`9`(A>JnnqB+T|HIx}MpfB$QJ{b{DAFz69nuKW-QC?F0@96y(%m5`QWAnR(w&Nw
z64D~l(%k*%8~ToMf87}Oj`96r;PJ4}d3LP5V$Qh+-JkMM<wonoW%pz8psw82o}WVy
z)tl&hRbOKmHvreacPtqWvhgMl<W0q~r4-0%d)_0<BnmmmUp&XnDLPw)xvNfrA|!7v
zco&NzljA(OLX0*j^e&xm!i{550>U~5sX&s-fNxDuu??ADzZ6yX^|NdlOnC{TiXvkP
z2EjKqSZQw}6BRg79@d-o+atiqRogVmp>2H-?;ow90q}GDyE^@4yqfh+g9`X}3Y>J`
z=w4kbf74KQzS%nnjuz#}rEF2{=Ob{{cC{~JaW89F`kPUe2;q$2r>$tl#o&?O5$QYa
z>9h~fW@?b>GLAFgFEBbj*zd-E><&^0Z2D0fUuw?cpFQE0z%hE02FrnHMkwyARSKBi
z4#>$qK((#LfZc=V&Q!h@TviH;il-H_?eHkTNubn1`SwirPxRmJz=mZoj?jho#FZn{
zU_N##Q0OM<t>P?iF2dyqrcgRvn$YvEm*sycm{oS>e1FKjx)YU2!oS3Vz-i!Va;MgS
z;*ihnxLq(Bb08!($7J~UMDSS*a&?Fv6Dj(41F^^hcXE<s`Jwj->Q~p&K_zf+>a_F_
zIeaH&=_p!M=N2d=Oaw0Xx>T2Bc4vfN);;I{l?x84qy|WLxw#XXpk0a%smSGt!6{#w
z@9(&4sQEy8+Wp>wy8kDX9VHMhX6RJ9d|Q%{l^E%d>Uh-%-AOIfFVSI@_~+XLZh$%!
zk9PSWMLl!9bs^0B@Jek)eXxRN#5Y1r``4%|ap?zNm8g|vyvyGszX<6xC+yNj&Sbo<
zn?Tl-t3rN)A-Kbiy2nF-rJD9q2-6;4UK=$Nr{r`fSQa(T1Rc|ZF5Av77)EvCfnaub
z{)mCcOf?l<^6G2692|3232oXVCcTVI_yKPa#X6V0zD>)z))-?GP?W4i2`W#E)Hcqn
zDHvxU-7fsdPG)bAO1nWleCt~Ja)Z%q{gX{WbUwaPa_fz&W=*(_EzW&_&6gcE1Hybu
zXu;bLmI7~0Hmx~${gpe1X6ewFDwn{GibV$?OI}Lkz5K2%q%%d7QYYtUd6<ZCPphC`
z8PfoW%*{Fs$k6@h-|_!ZN-#FX@QRK&om`M17G4Z79LY8TI|AOodX}E@L$t)`2KE=7
zO40CecsRw_CFUZkKsr=-bA~}`rUp6Q?ea%tN_E}_f-PRUVgR6ijXm$MMP57Kv^1!%
z7nrrzSoFpuqB<4vv2f3IZwg&E6)C9q<g*>wRfa;PB_-25^DXAjwazBgX|9Rlwoc!t
z4A8dN%lcGa)7Nd2g-ILk^}?{fD=JR7U2EL#k4z0GXIc;$L_^+!_wEa?W@)@NBN+;F
zI5&aMunBJ8Dq6Jzp-^k0InU_o+rk=g^otb;ls<4R8*G_Xs+E>EpupcKu`oRA$G3lc
z8@U^4#V#L9wG&>p6OQmvz~mJME8n;Yh#wJ<DM!|n<jbFR_N2K|?L)vZ(OQ)5Sp8a`
z>Z@HE5U*T+zjrIz7o-QMZoRIA_vQ!^YctjnS}DyiDJ;^fWNx)^vFMBN+<heBJc&Tt
zohH9!{1h)GcV{>9Eka5RW2hcvzwEOlD>12^x=R>HZ}H1;c8gY`h69>oN{H0X?0fvo
z>?EETI0b~ePm2)xWrGvfs$^-3^%&kw<0D@fpcx3(6Ub&T;wqsW5y)|Bm+hEpITvqh
zV)e_+F);C=G(^5HSHmw+ao-xgQy`Ty%g7KN*lm|hxnC2*n@#x+IT4}8O@{eNFYkT*
zwma_ze3>NgNg+~Qvo)?Jyube{ZTA(L68Z~`Le!WQNYj$<js=n}M1B$dcsF9PvT|$4
zs3}sELR&X=?ue`v1?5Ny4t@{)O&~vqE>W3sa)RlTcT28dYiqL^4CziYVZEWpcf&C!
zZHsXj=%)=zvrk}^d*(2-6N?r8mVBBo(&5P@ym;pYP9}@;%`7Vc;#VHb)`6CkGMzpO
z;@qWzNSWe&2jj-#z3r$#DbKDI)=a@n!9s>%GiJr%O1f1}%y2~2L9y;h8wGAb=OxiB
zjaf$;La_qf1vV5qapsjc*2cCB?}!s|+b=tMO||qq9YCY+mSS4|POpsJs;>LYQLrMx
z>@zcI(e)&*GNWo&r`PWgmqxUZI5&|SHTsm&X5D4DjN2qUUxioPA9bk)*<IEHph_s?
z)Lw^ApUSM=dK71UXKn!yLH4Vx9C^MarEmbR_-ENj0RDW>SevTgS$Hxcv8@Hnbh0Rr
zX&7oV7W#_KgB{7#P5|<W)=u{sM(c#mYelSQD{wG1KuHnpV}^6NrMRE_HV;%|-9#Ic
zx;Ncyf}JF1-ldI=@-&v_I*X4Wl_l)Px%`!+p2=uF`Wt6Fni+IG<fs9PT8HB#6kmrH
zdU|(CNgr*FuSJX_N_Sr&T~H7=-R&a?w?@xQbFO_=z6Wb^&~rxt{$mYGA89|#>n197
zwA7Uu+s2!%P|G5n(OUeNvXD$}M*R6l;qa&DOHU1G+B9Zo)1jhFo-<tQ4nvXigVus~
zVwE+)wH^!Tqxl{nERg>!IKD@&#7=NTap!5PxzzB=AX8NoZef}@lVmXiZdR++xst@s
z`SIq4%~kV0S}S|J7iwT>XlGyO$N@&4M}AF(Y?ru8HRI3_U9cH5E1E#dQEa-*+~hd_
zFMO2cApj&}rM`OziiP}eo+1Yn*s<y-P2@pkjwSVr!}n(>!zF~wsDaoO$PdCR(u}<7
znOsX{;nWL<QhmR?Fe2*lhIi*$sZfOv9d3t7MnxA-r)KA_mlVDOteWxLJS+67U7lyT
z5(Gx>?IvID5>doq@jq#>lKguB>hTre(z5smk>tS7rNUgu?masz7_0}Gd!vrv2sT@N
zRmvX_cTFl_3nBFb!{4|Dm1ltuLm1;ZjNv{h=f#E2eG-XolG+1T0U)g3nDC-$=z_`{
zVAyXpW!N^#0XhdUtS!p;U>Lnbwx#~Wwz!2gk=dkL*lBCgMTADxKP2o!<hnz3xx<(B
zdP8775v<D$wfd-nZa{h^_3|czi{_u_Mx|U*mwQOA-;@rP?qP`DIgudWKacba;UJs}
z3OE}^g3|v*S`-1Kh5q>g-~TWEKy)!j(Cz0V<RM||SSD*CV%%E@LEKUCVG4r1g$Tit
zTc5a(%KzMsdeNe}rd?t8Ig20oovXCIlOiGvd_prGiE?_it%%QCY4s>0-gT=H{?fSl
z=fng}Q-EpwY7n^|T^LtZ9$M>#-=6}3JXcZNMne24ZiH9Ejh1KH7z4K?+&^;dbT2Xg
zr8RC76==^#{KeYwKa-C2QdIPvE^O57e+uE$;{>3vXMS1E62wCV2@(5(q^l*J;hM(m
z!S6_dBX1B?7zI4b&VxrK-vg-GN|1pBw6$IEoV*}-0)>m}(g{dhcjW#@g%XH`<~I13
zvK(|G0QQdEMkR9*M#&hC6x$Pg2)?dGJS3j`9P4Bq)KFrM0Fw^F#-<2=c-(wD)$?+{
zuL=M_GG<LeZd(Bj)Uf&5e|8zj7X*XNf`(}4inZwn9|O2YaY;bJxI2v4i4TIY0QL*V
zNIOWUKehuEABkf{P?3bxh*L2=fzNt<klH@}35bdmE$wjbxKsZ#t{nj3mS-o^(2N(u
z10g;_tI^Thf8$q*_5f(4w-k-_tqH&^jQj^}{L$GP1Drw9yQe@vm_A68tAuADN?qr^
zW+tiV(O*G3m<JnqFpc#sJ4$+R*>rb(rZ%>#a}u*^N?d4!f8!<Ehy(n1{j#LTprazp
zGd|C~C22ChB!oD=ZnK6gsDzqsodcv%o`6Rg&2RjxLgoTc1Glbgv|K@jY!r|D3Q#!q
zQ*WzImUi6{bO1Q<2(za+0*Av~WI(?OLh*p-$^k^ak_@|70M4HKvL5K+x>T0Bc>>T2
zM2Z9Q&oh9j)*Yvl)%X`2r4(ia`V*3{DmD}nZ-8iihy_VbgJMIf+%>45gPLI%O{&2G
zC<(RRTwgq$v_uEfwFGKgT%hz>Z1y`?CPN}mnSg%VNtl`VHjkRecdaDR@3bnXKfVHV
z(E>=-K{XyVCz)oz8*}c#)mA8}|Lpx0kO_E6r_)XWWAGWHgy{+jq^~T~Hu=1N6ch)1
z0D564yYnHw<FlpVHP^R5WEv7*goP(;fq*tC6xKs54GQM8Z+8L9Db|jKMOF~73T}V~
zsFC+8FaA*D5k~PhRgBJdLNr}O4|Ns5a_Pevo(ay@$2I${0K3PM^~s-<G!VWi0)|I?
zIL$`soY=m+8V?Y!0l0?Po}&I8?`O|j`fK1;5(4tdKz5lAksGi&pAzl?MUZR2p*uhM
zrr)G;Skdsg(=A;X0%xgR0U){fL_C3QVf^`z@39EDiR*C`;xj$)@(kIPuRBq7bUFB?
zj4ve#&3Z`$+~~Ie{y#KYXB=ohK@!^#83GiF&%yYoLl}!7&rEdd0u+EdyPtnb`1G)}
zKds=Q5=-qcCjk$C7O0Hhot$wQ=ypE`G`+)^od~KJa_f?8_vZ)KzY$eI8-;r<L}%S;
zM7Hmmz|p0FL-21`v41-%6(Y~_<~?Bdr3&;Q53EyC;79|dYKRq){k$8T4kby-TWbgW
z5`->xK=E@ld;4P**;(DQjFB9%gBK#^1R;pG2Xpj5&1XP56}mg4c(TRNA?TedLZ-za
zT~NEzD#X^B^_^VpE(P}GE5I5J=d^+vX6Fn14;dqm+y#24c8v1<F6^>u(TS~X(7?C*
zPCK65p4ggc!8BGGSLQID*Rg4ga^wOBWjP{w(y4oULHI-Z#VCWF61!|2z<MO_$UOqZ
zY`Kzc`6a>&Ku}Tj03nYKz~|LQo9OnO67qv1W-agp@WDO<@&*ZK0tCjn<XvITX#nd>
zW%QOL$NzlG($B5H;^a}C)F@!1rs%UhhlhA~{GnIa4f75xzXOhXumA>YXlj~zJRe?1
zm5@ph(DAy`(up?#qmr`=oh$AcNzWONN4<JDve4%bs-bxjU{1^`cge2zkno&81N0D}
z9-9l`y5Rpr?C@F^cLTUs^z9V!qh-LeOgbo|c3z9Fefr?}I}0>OBOE<Y`T3(HVf)`m
z<)8qGI+`yt7Xz=C3d6(>rWv<4{0+O1wtU}nxdn?vMM|<4I(m60BuAzMJk_ZXCV+s&
z^)@T;+L>P~u(lDf7<(=w%vtN&Vm)u%O`zJfOf^gaa*}cP^W&8x=4euM;4tPPrMcv8
zqd}O{C@@hBQ@`8)Wq5QV4L&vEB+(&E$i#GE<9;<uepgicLa8iiXAkFv82~PlTC4(J
z_eaB`NcXWWpijkTNW^42*3Z_kunVQ=CWIt@1BA*XVa*JyV_RD*+dHbkcbg;-%~s%1
zS3;6#WU$P2fULO6Crpc%b@_jV`9Ezt0@iq^F1bQxf@maGgh>TDYnByLG^e9t`k*%B
ztm%VHYltBUpbyHZ#6wm&5?@XIXelCII~oBrOldq&Jm;U>*^vF1{6LkqbF#Qe6U%cM
zYdIxQd|6C^+i+zi2iz$@s*9+t89+7kci8m)WI}R7kiZa_3f7=|q9)*0i&gc%KHJa?
zyO_1k?U@xLQJiggnw(xvYAH!C7yuA>tw3@0uBzQuu;^QfU4D?Yy#ZAq*NDqV{|zAM
zzF<IQxb>ayOO*PShV6qBg>#UER0aqtt4?Io4U0qE6gQ}tOb0mH0Ids^OsSvAmu3Fe
z5@VbkT=;)~9e{0$QPuj9tZ1G1$nlFS)#n|!_5d5y@6*77r@WLz7P$jq0zvKVRxRWD
z_-4do2yYYZkzsENM73v5e_)*LFn9xS$Ln0w`5-en8fvzEAJ?_t?=sLxf%T1Kd>tuD
z9c$1t+6?-rl<p`=8%-j5ynen9c}u{iZFaQ>)I9dKA^fOyLiO$?L34tMZTDKTZ!mff
z4@(*uQ}j2m1QhiwxvK&Cr539qfVGDHf?JAJKMcswx|?m_VtttZw{-w82~@yyjvlWC
z!83Xg03>Bs11MXA#2$m-Ca6Q718Gy<lDOh>A7=gHUIp|tiPANQo|{^cAl6TdzX+f=
zp$Tv2K7%5KW!Y<7I-sVzM5$jd%IXSk4+|KEddJ-z;5y*u)b)R#R8)5f{rzcQW&+>V
z$qig)%9MelB;YI(a@gZPdb=F9FkUSSVTE_h#I7mANjQN*{5q=JcAQF`>b<cKI(6w~
zqljtoiE{UV>Ntd*N(b%`S59T)0k;?|r!=7@PB37lIjmuAfwiW!lrjTlY08H>0N94O
zrt~;sjf)bIqN3})u3j(6zn&kcB+>KzG?SaK@Woj!a?=W&DkrPd3SU9#&Kb-Jwjxs3
zI-puV?}^Q;$+E;<Qxg34e#w{pUf6ByNSfzRWsI290{Nm~oRqbE=Y-J57E(yDWf}$i
z^kA6!qsTtybP3|ld7eRVk6t^h5AgBJ&&-z-SP700FwW=R4c?668kJ$K0pMcO#kL#6
zf;NDj-2tgQPqY94Z6feo1nwii!wBj=fcgyBylel>Jslw*FL~%u!+wW*Muv;lDidoH
z0si@w|GqGd$rzBU3tVSLvj&N-CP3C#8=ldy0!G{u9IW_qP&-x{jDp|y*YO<a><1kj
zn0!#RKL8W80#4kBmDQBP(6el=P_z{siqM&UczFi`z1mPJ#RW3BON-kHo~t#F=$g23
zdlOSwM8CwrTg|(@;tuK-25zO?VP=y0&WG{`G8eB#OHyh%%R>_^fiLR*^9uSl+~~7m
z)UP?+Iopaq@tJ1)3H<4R32_2M;Rnvl&>TuNJdkC;*y`b-D6*?<+pqRQ>6Jo{?SF58
z<_2hlKI~EDbA{*>p@?@#B2X>a5&^f5N*}Gq2e?10?`aWqoCZJV>=cS=6k<yC-()xf
za&(;(`N9GNp9+HZe5FR_>{i;pF!^ud!0KEW*jvvXBI6url}^?HpKk;pbz}BkLo;qq
z#e{3mPS#8#7~cXgl2Bs;aDr#LX$eOwHT8P?2jr9-N&p^w8J%l*d`T-E6!sAib%twa
zJPA7h7F<24vXV1&QeCk%qj7hK38^qfoNyX;gb|!Reay|l#esUbd9_5ApXJN#3;_ZZ
zbm_b}wlilI?{Po)rUSb<?Jpm7!=kzB5I|V&fern|XT_RiYwdo`QNb5)4xW-`{y`oD
zvCk1XxWewaRg=8>^M8QM0*KcWD^tvW^I?AGNJ2oO+h@GT<>ha?3i{O>3z$IVCcPNV
zU&)kTaVhxf7?k4;NYNi&K#=^+EqKZ~{g1<)ZA2Rj`h+}6y?B2?5br~`2^D=L#CErd
z=m><pMfB>CB?^v&NwTIE!%LdXzl2@g6gvF)1g$JkP~7l?#7-V;SH7G7r*lhdTA(sR
z>*-2yDXvTM8)w>_-<{*1ACxBpo|n>gF6Gb1g+-wTuRrbl?Q;D^S;UbA5gxVTe8R7P
z{+$sZ4+7JZu9Ef_PVC>8DS>f_EVR}C&*V_Yfmy>w#s2ThD%4=o;<+7vMYMmTF~O+N
zWQv(W7=S-M_&G?gGba=~{;3H7CYdsCNr7JBpYQQk@Ff}#jJD)t9m~Jw>(~FtfY(t%
z|0F4ciKkBhmR2UC{@>KYUtbAmn{d5g|2Gr!*UOSvoB9^!Ki{4gjRMFlj3G!EPUbL5
zhCjb}h9qFJzyhzuyjHKl5QRJOSHM2$4|>JC{A@xMjo#6Owb=uI>3>`vN&K<QI(l~E
zCvihmnPS{-Z2m#fQIcRu{9`<5F?n22%0a0~hFm%C!#(7KFNZomKf{{{*4XBxj%nR9
zoN9?^blKVS?~V7*#7TlqRd|w<?!Qg%&tFaWz?%Pm_7Av~mOv`;y6bx`xQ_0B&S3d%
zae=zApU>_%X(LF4WxzvC86X-<wI8t575rk-zbzl&^Z;p?+txmR9iP7k8FvRez|mtM
z`}Lm(A*d2a2JH}yp8P3ABa`<HY~cCb1VPjPY$ZeSj=YCINkijT@IWUd2c#Wx9y5au
zGoU+y)hsMTh5=1F2url|9p8XBmNWSb)FTJl0WNkC0MmgdRP|`F4FI~Y!M)fF5ja42
zMin)_Usm@10=NsjK<=X`MSTV8EVNae$+ut{PFf5cjwCXa^P(UBCrUR7rI#R@KkK-K
z(E_wB(EUNxs%Q|z(qr9_3`&jW@Nanm`kao}y$?9@dgor(!&9r|h*mFFfiV67Q5JxQ
z4*lSDm;vk}EK2~`Z`44nh%kw~oNR}*JV2Ty0X6Y0AaE4S$kJse9YC>te+vNSsso8F
zP&$%;b=MqNPizAWls*6!ovft$TbL%T3rY)sw=lCP->Er_=KV}xp(dkrq$8uw^Vht~
z!0R2}q)wA)23zIdh5H#M2TinNZC=5Hz^EPpPX~s09g>g%0V25m*myU<lrSjdrLh7P
zmlM871w+5%5ACNbXm>!*o$^pTJsTj>DPj5ml<Xki6Dq|4@2tQCq8l05OOiqhN0;Rc
zR}l@eJ_Lpv0MZ6l<n$d9nFSsgc?%%jd;r{Y1b~4`o2RageSX~YR2HW5U+A;iqEw~U
zm1AsHV|k`F-&5q<Hb^rmxu(`6_bkCpJO5jgq$730j0Lt|(FfGVC)G8mrUedzjQU9_
zj_vIwM(uvzeb<N6T}QLhT{AZ}V!wNg4*PsQ^cS%j-QcS7GG4z$#&<L>es&rmEo26{
zir9)74GS(qMIhS>pdaIEa*52yK#)jmt_J}DM@1AC`VkZckIqLNd|w}dR@DS#W2vU|
zEskQpq4PakKpvQB0@?tDli0FSbzUIl^bI)kvngGG6J`TkH#!<Y9vo*QP{GD@#M;D~
zg~anz+3(F^>a+o<Vmr9fjW1X5v6ZpV`-*`$N+iOdY}XFdLqunw9Rf3MDa-eL%Dr5r
zt9v)ep2^dg7YK(+|MSN<h^gbXZ<EeA;HsD$!2x*}@hB5&tsSBB5|JxgEzSq7P{yF5
z`w6#sP0NH7@i*Z5&Dta;S9y)tLsSgBN&r)rG5+3OSJ1Y`@`z%LPG}kBX#~TZ#mK?}
z$;6|h&Z6u?z=(?yzd8TdUd+Wl1ej1tJaZPeF&Q}Qs4qZrjs)$x?BDP6G7#3fpv9`P
z!vckUbKjE9htTqTBG%un>o%_inzY-G*)ji=`$v2<9g&LNE>+$i%G^N=2@DNeCLaqM
zSOR7!opn^9$n;fDr^WNJMC^C(kInR6*7cA4zUrHMQ>-IBuXs8IKfpX?o%W!i(Qw+x
z#}P4NvcC}kQCE4+RDZ4%^dk=i_SPt^#)x{*0B)YPBMz%ofg%D8dVvok89)qwtaAbG
zsKCaH-1qD0JRF)kz_C8h{_aaxQR=tsvv^^&+6cT~9IMftw@??F%*ys_?_xMJQ^4wo
z4O4_$0<FuIKw5_GV0d?kUd6>c&d$AP=&v;a3v(NVFRYeAr&iNM48iDpGWCEFbNv&=
zk`M?a)EJ<IK?=%@5y)(oB+nv*d2niCJY2#EI29+lcXkF)#kq<q7YC3ilh@>+9NApw
zWx!$PVz;-^G*(46a0*%3>`jg53>Oc_G1PaUUKCF=qE0N<3p;%cO8;Gmr(hwlU#*vR
ztfI1eFpVS+!*m0|S9R)j4?oa&%xj1y)JkW);NI~|hpnv_=TOBKe6q;*y5ZwYOG>g{
zh3lgdk&)u6$Lap<@%kd+ryr{Pai67grXr3(y|@a<t~LSvW?&GQ&o5jUSFaWPV)<4g
zoT@0INiG{k;1uF}JS=7{Qysa2pJ@L)K6z4LJ%~6)<bt;D=4rFQ(peV8bHORmx)qHi
z-k-g$Qzf(t76yby-Qwtd{xPpRhC1o=HXuK}+RA`_oRypVosKA~eDEf^I(3%YmYzW!
zN%$Z>cGzj#U>zaTmWJc-;`5I-$OQ%g%f$+Qdw+}c;`wyLF6eKBb=SQ8Ah~L_C=ZF$
zO+`+m3k^~Pw*3^{)V_qkmUre6#w4ilfna|~#ndYC`LhfFT$QN2H05*{Xfsgt{rGZ6
z`oRHaWnIeqCjI+_notM4th3=wg$WkLhGTH~d>HdTsVxy&V`wRTt55Wi2`s-lqe%yU
z5#6X6lZs&=d7?)?``h@y!`AP9Y=k|Rbs+sP2T=7HBdJzn5j>C(9c?|$(y&v7#~I)+
zj#<j+udSsTkIEG@Ki_h8MHv*qqB#7l>MtXO&8kkRnhG_{@%8N!{QPii865JhHU8la
zEu(U|_oTQ02Ml5ph(|Cg+PYoQ$>zV07F^_crMx;U@8*4Pr7%&6G6Blk%?lyn`prR^
zi1N*MpVb}evDv4WKyIQWhJ#~1FWC}Dpj9cY#c`aJsqn{SMc_IsDm{BtU?Sq}f{iv*
zg)&hE(r%VJt;$>U#8<Y)Kbk(Sg~=6Lf8#d8k;=41wp5PrBt;vDxI)D)xl?D$1Z&Og
zN|&m&j`vp5d_~tktvf<dqGHj1lMbJKPYfS1*a+Mym-3u?>bCDkEbk|hi5$JaP-U~y
zUNK%B1A%pV`FdP(exIjArjYaUCw`#_)5`T6uY=1KKtX%<qwPe&A$S&VY3K=n)GNx9
zwz`$9Dd;L8SV?sRx>_j};i{Uz1c9B(Z}e)Q^?nFf0HDv+9728YhA0)qngNt%m1jSG
zla*$3AwxfrXnK<;WxiXQNVc+R`_+PXQ;}qvoKW0+(gV)TB%`(@A+^cC0p>TCK;Z|0
zLD8K80qZJ!aQ0J){aO9IQ-L&x<$4d#a8@@QDY;-XtFm@c2EF5O`0x}S+yO*Dh<v`=
zr%2K-WUUrRIJEilrc~L_j@I6~2)bLA5zzU<dAX4gij@QRLMX(OGGl5ca#IKc1)3`+
z4cWJQ$NLOuY{e`<ZuW8+qxu06fvar|S?>A!l3wXv?=@Po^H1OXrAxX;+brDnKY@(U
zOeA{($`e5<nqq|WU5|G8U;s7Xn6pZM&J)v_$#1@7=BsdXaFVK{#4uGnt55EWx(%3d
z$KV3Fg<Gg7h%#*C$MlgNkzHg<R{8m}7^aOcG2bvv7&!%Aa@iz?KSn1n`R?tr?WBkc
zzWYP)-H~v)TH&}T@3z{3VgQ~Q*Q@ulEdzrDFaw>$pZ5^w(9@5SbHd$pg+XlrGT8z(
z#W6L*^)4R`J(9=wwueCOHWhvcYBdQu{=FF<bpe$se{f1cf5hV|0CS(o7+MAo$ot67
zlm@M3JoID+Fsk~v7yM4U$chP{hY9jjfGA;@5l0Y<LYkNslan_XzCj~y!ZIm(a?(1D
z#4I&E0GDpxO98W6<wG>jTm;WI^5;tR)jRFFK?VT_JG7~|{V44GcMlE0<Mo+lk9E2{
zyLt(%X`MlbY>1vetj^U5D!ctCDpR>A%)nP}FARygz%@SP)d(&^u>pDMi0K;rM!|p$
z!A03589}<Q_`*kBXc{%PHA96Z$q7H=*=M$D!u@OXxA5etHoz&*H5}lqLSOUWeN*Mf
z^8G`hhmG6akz!qt5$)d@Q1*Axla(2e5h|8g*@%ARYp$1zXY+}sE&5)M?b;5DO5bKH
z37u2OZJs4U*jtu_RX(6XC$jrEc1Sb5cN<iYqLSIXCnEEYGfgYWE|-wvO9B|Eh<#{f
z2oF>O<D-w%EjjP8va@m#VKj>JwQnh?hNZoJjvtqV?hkQh&6J2(>p@p54$^#!pyr#N
zNc=;Ikn7I|w@5O=(G-$;G+1pvf0@Z%zXE=E*qW!`9c9rpMMj=|JyO<+&Vs$yREW6J
z*aDv>ri@@F3k_Zv36Y(3(>-{%Y*SWJhjWc)d<wVLcb6mQ!U5#jlG-mVS-5NVElp$S
zpGzwe!i^l?bmF(YevP-rwDsB<B=tEvG8f)joeqwF3S`|CDq_rK%g{ca{2mzKWs23j
zH>dAEmZXqBHCBWrx#od37Nt`hBHXU9i8WS$Z3WkrYm8Qq3loK83W`3GDe>+upOvx<
z)xFXe2QF87a_s{jp?=wDgyBmD2x23iF=hXQ#6r{C?b^*x;ON-b&&X~2R{@WhsnP8$
z$BJvC@m@RATbR>j?>t-&@CW`=nV-AAuR3t1LMNaN{&ZewSee8;jbuR^7NH4+`|4^A
z6u`dRUBGc?y7dqZb4X48YO#wUUwzl-?SMJHcUSSd^g4YhT8Y4Tme?EDSr@wVC~G0A
z`U5uJm<7B;0aPa&vlU7$7yb14NZLUFk;+YXnTla{{xb9x3E9@=*;zl!93cPYnq}7E
zdn^N%8@XdncJuoh+B+KZT3yy+N%z)Fjf(oC37yPFfNwLG9*|-6tk)u+_8gU#w}#Sh
z48HJ%F}LDc)^yz?_aH$%)<CyS5SY-_yGUnhS>eAGiMbmYN!{FLkscF+$zmN#Je1`}
zD?GZZe=0<Yg&l0AZk1|l`+Znw+Syfk5cN1eTg2sbrtd*7Hg=m?cKFir9pU_18rh+z
z0)RZKxWQUJ3$WFEnGBX9Xuc0ot&ypC^F81u9B5li>0_Xr*&Ypx(D9MPlC7jWTMAN>
zhD`2f^X`~^5{eJ0+H<R%WJ7%fnWCkha%wMNFuA5mGm!{GNw3q-BM0Vd_)Dm;CaphO
zr$VZhfs_K@ytax-31iRG`h;jvN)Yay?>SS`xSA%xJ;syr>h_PhpI)?GvRb@loR?C2
zSu#r$epXs^w+pAk4>!#;XyD9$f+ks87Yo~^;h}Td{gR%9yYY1UEe9OXR7_*gMit!-
zcaTqb;*{SQWj-<ej%6Ki$;Z>lo5P6JoyKj8ztl$sMb^DPHtuwIa0TQU&&2rj@;&FJ
z83Vk{88KD$((!8csr?WNSohgRoT$I12Xa5<wUs8)*m|a@PEkQ=%f1I(E1P>{-4DNz
zVij&Mh%!pKQ;c+q6==EpHCPfM)*|IU40BgK$0~_We=08V$@T{%RnY9~pKr~lta<PJ
zs3ZVJVZa4VgDydmivOaxHBBh28+SVyPDw1dKV>{d=?f&e`b{q{l=0i7?_-y><z~?p
znT{!vVbBNk?t$#c3!Jw%X^G#p?XHS3W7nO~xBNu)ElGk?7o_MW>7`KljIXw}69YS^
z-uI5%pO`Jqga!zjzYb!>jR7CA&Zuu&l&m(&JdT*jML5)lbNxiC9<(6DRmEprBIxfB
zN1_$^QuYqUp-rx|&6jX$mO3gHEvH+^jb2D@_;`eW4c}@^p4Vk5@9|sEk1>o5GZj15
zijr(P5x4ytTI|}O6v9d&w|4|;y0TR%qei!(9K0R>5MmZhr<0n=Fx(s8fj)z99#Vgo
zQGCZVRf6+wt1*}meskeijGC{yO8eA@j#6mEuo$h}w&ZK+Vj?J>1eodASrVJccODkp
z_86;9-ik06Zlwe<Ac&k$Q%i0!Jl2cwn7NBI>6EVJuD`@86g~9Gp?0JOA@8y~9kYKz
z;r*-x_5_aN$|p`X12WTlRdpkd6)IIY&iM~Dvd~_8jf#0pTM(B}ktXeKI#BKkuO+Fg
z(RXU))E5pEnU7Y~$u}U3ifspq-fZj?(A<$bBDTy^7Z*@L<(uv9Hks8)DkvPte|A=j
zM0%uVx`^W}H6_de!3ktT`1xkD`raDIcMx_;cz~q2eLbtiJ!3qqajkS7%$Yt`o5EyU
zieh70*bu5oIlLyrCI>Tae~W%@mr@%&{+hqF09@g}@_%s~S$lj--*#h_ID2G#4HLQB
zi|}0#qCmGrid`OK+S?LQ7VFu-+LBrn9B=|zj1Q`l+x(DnvJP=u1yIB`4YPzXI5Hw#
zCWknhbJlR2Ymy0K<2^%YWmKKq8b-g}GshVanrCqa*Zgg60gLrXu6)j2C;4+9YmXNl
z@wi?0(Yii@o>5k5pW@fNZz*w-)EUhNa^X8lsoG}6b|eTS@FnNYP}z+{mD*bBb;P~!
zpe9dY<(6eIryx8Sr@)_12rS;}k48^FXL6T)C2^oYm^~C1%b-DfM_HfAfRSNob@PbN
za?@(C!Z&E0FSz<T0JZA#fq3_H6P!{UTC;y)1LdI#dA6ci=Vou#x1l(>A)C;*LM|A(
zO$b>AR{Lck*BhbHmn|J!oI;@BIHhVp5G~ptyRLIZX2~i4A>d4LHul%_kl9Vqv)>ES
zYg_%o;LQ{(K(P^NLM|sZ6DNpq)Ev3|b~r-(Y5J#(+kDSbYckCmt>LVuC}P&=QeAgO
zXWHpM08~Q`XGw@nxdl%~9VZ^a2tj5V0c}I+p858HG`6BC+}zh`9rNh^#L<RX?l(wT
zA=69<S=<X8rV8~Aw*2>nIf4Z$sGEwVo+4$3P*dH;zAv3@Tdw5uL?0Pxk}Flsl&Cj#
z(mW;iNQe?4i(|aOwdy^5U&m10o6e#HI3>MT>2~y%A6$cwNGQfP+PQar4A-(hG=$v%
zy>@ldM3`)wV)fXYD2ikaQw76MrL0|86O#Dwzzq3*q`c`g{k$_|9?_qu+SVv^HB%>=
zl~EkaX>*;Z`|6>+>A6dg-spi^eB=!yl&u&J6ZPJg-6(U4uufAC*Ue7*_6##2YKyVl
z#g2esVQ{b$9{p&g#2G*McE*wbFJ{x-RrZz>uG};9)&TXAhiX#uq=yBgV`Cn>Y5L-C
zaT8}bYc=dphKpXbrfyZ4-N7B~s%ve8zwcjez$TR?)$C+#`tl=4ILVeH;NLb7;#=Cy
zwN!I?+vycHQwKQuQd-^5X=-~C5XBM2Yxah%A8@r|Fq|mfU`6dGKCr@9OWoR$OZoCf
zig|n{hez|R#$?;karAPGt>PJ6MYDpwVGeEcec{IbN|H5~hDS7(o*+aYCfCQ?WEvW-
zQ)Sfh-;y^rOTRNbe2U9-mRiF2G>_TsLaR8*mg32~5yg&7tnrwUCo=WJF+=UPZf<f*
z!62ifLT}Qc+rJ_QvKJ<d2x+tT#qJBtVTKg7MV4}Iy}|1)3T>$8PPL?CkQFQr%|Y?h
zk|UR3B0QYk>oR1%H95K4h~Sip!M6lMBxwt$$n_N~O9Dx3POm_gr$Kv{>W~7{uezpX
z)z)63q)}n}N~gl8^l50QDHAN+)xGpPsndFPinS~ZwwKg+EDJOh7Ap37tOh0zw%K>m
zHJ5j9Ap|jm2eBs4*ynA=8+txcZ)sv6bxXiMbrC+Pj{T<9>E`83Sja7rv(uAV<^pRy
z+}QI&xHW|L+oU5!j`8c{<*5nv1}+-I0&^PeBv|W`SbLD6QRxV*jq6DUX_F2O)bTi6
zKjX$^<yJy8`_7h!5O`xmx3R)D%gD*^TqhxL)-SCoOoeAYt0#urm1DPZ?|#M8q@+X!
z?k%khI3*$>l(Lz4e(kBp@^}OTc>3xkvz^@x^m1Ehi63fe^jZZL*AiJ266oF+7Eh4N
z!qJW{(`DAUrUw?fTDmJ{bn~@t(l@`^aA^pebrRHit3g@zbev_Q)M)VrA?}sz7ap!z
z&ZAThhsiyHs=DQ`WMIm+<tb;9WSI+@7!A=I;b&uot6n;o0bek04GuAjr5&cc4ea+M
z(QV~<<q8=E-!yr`FU>AYwV;NV7fE`>NJuMZ_Bp#_oD(q?^9x7cfS!FtPZVb;ES*PJ
zi)x5X;(%TnQz#1itT2&nIueayMh1)MhKqNiw4MK!LRSV3;*S6{qcmj^rm_!ZLQV>A
zBJqpHD{*NkdN-$$IC0<Xy?JiSgjTBV^z?jI@eP}HvqnC_C)V~kH2xHHq$U_O6t6yo
zsLMhq|5A*$PTE7g*)GEJYznuhyuOr4LO{;Rw4Un5z!D08#8C;H@1Sh%LW@+0JRWdN
zym*RO^x~7t{z}iM=aOU}0pcU90>a%vVx7&k7Ukv*q~?AN@O4EJb!YVrt<etZFzPwP
z`dfZ-9kN;ChdUro><46S54Lfpe*{!1h!RUPd}qq6FRpx6tu20|^%CFDQXUnX0v;v5
zlqZ^}@VuLtM)0JTfiNX0$erulA(8kh3(0YEIOWAbkr*v_Du?T-W?zs*-iN~qlf%=Z
z@PIAkHoy>iyyVy=RJtr@VpQa5(G_WzmdH=@wXp;D2}t23Ishb={73PzRt3l|9pTVl
z7frkJV*kvkDrCSU21v!H9UU`#3P=;8ZOWO<_XNPi_iAY8AA6#x6yN}?j_!$6X6cj4
zL+7Z$$QcG&d&_!dL~PDjr)|^uMtP?ZfZsi}siR(9HzaSY*rIBUF!$nywGQ<z(l##S
zEJ1*Qz>*IpMbgiE#khY=q;k>>v9Pz^of_Ut^p;=8DvWhok0MBLO6TC_c9cUizL@)H
zW3aXX@sgR43<=iN2#<6MOKyt{E^M^>dN7g{RT_HkbPjB%=r;+eZR{l_ln^r?))RCp
zwLRdlWHoeb?y0ox+Y$4As_G&+L3;H9ZJX&6Yl1@<E0n9EFzMJz{%M_qVQQm#%Oq?T
zO$4S77>nR<J$c76kk~-lJFUnwTga|!8Zei{>6B`l(+bRI{^PJ&m1q3w!8vF^d5Oi=
zCctOR=d*g+DL{2bWT7ZF<JLI$nQGn9tE%!pE>mFD^OX23(J7<NV_EXeu+9e4-gC%o
zSbeUMiVKUYf?lNidA5G7eS`uK{GpoOgOl>Mvf?dTJDQ8)bJ7zT?~F?w6z!wrpdfMO
z{BhG|Z;L1_(YiI2Y2`9|fFJ?{NOT9%=K!+NRo|97rtfZ+paAl?VxVpSGN0?fpvb}G
zdmB+J37du_UyjDCG+3Yd%#3Eu#baP_6@dlH$n;?V&z}fV(s%{ZU*(Ms5&*2EsKuFK
z3F6m3+aJXKQTt>qSv$re>O<idODBA|WDMV>Th%qMO`kHj=UcNqQnWZFWVraQOgxKa
z%?j{)%usD3``tGH^cziTanquHhbc`0QKh<#x-N-%7bbe@^dI1E_poML!zbEktlEmz
zKiLO*R8bxS<485Gw1p6C4?gL(KH{J|lZ75&754QVfC5#@=eYR#$pbH-%e5EpI<hxc
z=RY1w!Iu0GqpxT*B<eb#N%%Fv(c9#yDZ&TY5lJX4AGlk4@qK}Cj)G!51FW|B(^Q4G
zxQ`hgb?bt>rCRWgSc81(5lFb;Gc<mNDq*8+oXWzd!HI#KZkfG2KIA`x+Z(nHwj@)3
z%2ZPqXJZ7MP1>a&Wtqbkm1N(~<f8A@eL4PuJ))bG?(rl3!Fv7L7f@W&e(U1}mtj1a
z_rR#D!;oPMB?vjBz}mqMWBEZ^Oh6sgT1<kU*R3yoF8UW|1n)Gn5Oy<dj|!?|67JIO
z!%gZakK)`<CH@pRE0;UDaDEWGbwv^-C=B4RAjj|UYN1hH<oJ18cM=7<?JG0t&j2)}
zm}_n1<pSC=Fay$rs{hK~0tF;<JtL8-8a6ulDPC`?w`ZY!3xO`2%-_?g%86{so+sTR
zf^!@t-PXZ|gcd!L^RCrh%%V@~t|YC(Q-eEV&$H=u%B$Q<q1G0vil1L(j0Cu(6yP(3
z3V{>}V-lnHz`^}zqA^JDzVTk#x3POCT`6w&P{V$gU>FwpXrEC$+G$bF+~~Homt~Ya
zDCm>Eof1FRG96KQ7l+}qYWE>?Z-)EO!trgC?&C$3_2k;@VKvRl7(qPL_bw@9T4_&i
znCuxOZo;{W;3}~8F76wa_(W|)J~J|VjrUm0npwL)ErZ1CHsUwBwS8*-BL&fJT)6BT
zto4a+Tq{j8a(xp9r8VLok+Y!bmAOT`ul(FWF!G^-l6Kc<`74AhfKPVYy_*#}I($*8
zjmk%Oosr*du<i>nR|E?zM@b#)X~d5sJma^Rb(V>wI;^Nx4G1GbBerV=wUGH{TRV#;
z69N-?_1%?7vPa>tyAF`azvk*ICE`wn86WuMT~5*zj`?d&O_&G8?7BT4CH)vnbr<;t
zlf))ZT*0ao@m26MALuc^sS!v0EF39RcYuN|{*|Pyk^)Qg+A)P?w^d7raB(rS@}eu#
z7u@tW^t{<9(N91{O|X>%QU}-DP%}uvQ>$-z$+dp);v@YhxvBGOeB^Ls`L?r>M+Z_V
z4kfx7mO01e){n`QrwqLJ#cyWa<-Pg=lnY3-lvTBPyRCgoame$2Cn-@4u+LKl>71+F
z`;}<HY8iHd*5<*Hr$MD=o9h`wV&;n=DH3kQyG|J+Rhwiw09T<;64A?X61`rYly=FW
zUoZY>j(ZcbPkG^)l}AV<&M^4fwYgV24%N0A3xI7Hag_6;WyA90Mv9k+Tsg(#f$u>F
z@~P@$M;ZoC4d!Prx6($aFxIhGo=DFg|Fp-#2<s8{KD3OocLRFEh6||!cp1j6VB3wS
zo}369XR)mqHC;;TFM=vZCb6QNwH_jpOV=FXDMuq3_E!jfs*o=)y|An-{0S36)N{o$
zu{|4TJ-0j^z8HIv>y5e8^HNVhnUfSPy%v80L0sAPeQ<T*ERR0c-c~PwdElGTQYsaI
zHk;?h#rKavkzz&T!2R}|q3eXh&6%6L?wQ(F%5qf21`R#rpRvsN8lU@K(8p4HhSJ`V
zh}st$rBJllFBZSwO8hn}LZt6hn5c%5>-B}jwO{(M+DOM%&PztH^tbuwwzKpI5b%)S
zAJ7=b6_91%?D^g<@ByXczHXjj@6M#mE{prB7bogfqk*pZuS`sfB?D7@@70upP?zJ)
zSRKy1%Lx^GdVn!)t;UsSMsWZLGinU&E|7SOpu(uAI^1xx`1J6CIGVM3AAxjRtzOk*
z9q(aI*w3Znb+XUXZ0ngz=b^0o1n_QRrsoTw3jp=9O0?xLt}+?@wS|C|76iS|?!vj@
zJ$rvCQg)n?fjyoPJh_}g_DbH^&FHcVF8c$L-<6epUHMBm)F?%h(rVJ#ThN7yTBjd4
zV4hZ)^3ike8iP8`IpRK-C?^;pNS(`E(<JY}HTS|m%9GPM6^ZwKOzLKm0zS9-hXw&q
zVqpQk<CIG=a|w6~c6&GlzEq=o{ph?9I-1QeEl6(EuGMPk{jd?Prz)d}DqD`*4ydo!
zYD?P}2Et9Tf|}De>+u8VJ4`W(F1CXt?mnoruZGeOk7oF#4%BIFy>P+MtAYqdpwoh-
zwlN>ahYi1@(0~=}Tu;zy6@pO;Rn58#u{6d>86(C*`R^kf=ajh|30f5s2e>!G=sfk4
z4c|L*s!p<9?aRS`M~Rtm-n4;pE%o@poLdxFhD5|STM5;b|JFhw&k-3^2&5yvz4*J7
z{;QtOgF@h3zvZB>4OQ3wDcqVo1+zfaeWLQm5Tco&_&)DeUs3jV=^L7AdUi2TRm3#&
zq5oruJVh{sqrm*M=0BrC;}g6Hgxx@{7EbzOh;dpl1W&f(*H_ShKSzarM1knU_36w#
zxj%-mNd#rp%+dNMn*V-b@S7Bf-1x;DFpB;dqD~gL7mC*NO+TaGe}<;QVP7Jls1H~B
zV~9CwlaATqT1P{WPyY8KgD{~TdUsy@=MMd62uCdTC5Nfm`v3Q)IjBg=llj;4u~)!F
z;<4zpc<$Gwe0CX&>$tqU1PF5;aoie4XtATvhFNrLEYcW9M@FhI;?mL8VSZ?ugdtq|
zD}Efv{&tA_CbN>u%+^}7qd7)bxOTcMv*VY6c*5o8cON>+ELk7+KFNbwuqo)s<30ay
zt<tD{GN)FcPSxCN39WPx3cfS?#?xH#$dW&1T%ioCa9s}nP{p4wzXdP<Klkn`$@D#J
zi~YYgv584gmG2r8lttVD;$zaRKo+l&8Gv=`KqIl_WM!YZsS`j_SeOR-p<e+xCN>!a
zFP}hftStE@>7Nhx0GyVKr=013ZAXRZ4nX>WFu2l>F$6aAMgtFl0`FTOgeChLK$qMR
zvw#PzJm&}6j<GLgWne#g2(Tar@H=XPUK`z5e{LKu;O_HdmrTh0vz2~s`9wPQ28)3N
zQ@Q)RynVakR-6j33)(@1c#!Ww=TSeV{bmlV0Fjot0nzN2rt%JMRE>*o6K;xhwHk^a
zn;g$Rx}-QRJIq^}`~7G4f)gHtJlFN-r#WJS-8)a%(DvJL1N$ZdoN#IQZ>GN+1JR&*
zl+$~o_1TZVLLd_j{D^^Dh@Tc)Ys~D>tKvuA{k<Drhq`tu$MvX#qeClZ;_vTIIcQ3u
z$M)K0B7d3~89L7jiGg69ifb)(=V?`jG~PRk6Px=98=H6k7>713I-0`zV(I|%2XtdZ
z^HM#K^u!yj&Jx7<(<ltLfsM42Yi<vj#(y6wN)p%#=AC;Q{~lcMTbds*EVCkg(<*<w
zOb`ji&;jZOzmGuAegRBYZ9q)Li$7n+17lRb_EGu6zL}hW9b$b$qA&jE%gkU5bJHJr
zzkN0EXgg>G&ZF1-^<jU$tOKUIuX7h{mf!8oEI!ynSFR4X|9rU`87ySzr;tA_SuGpb
zO&1$8M1Q_)jtLALS~Jq0Hqy!%Ow!3%^?i-sJ`Wb5mlBKtRFD5(oM`zGf83w*%q|wx
z*w~19Z{%O|ip^qzwjlv}rUJvFPR%YNA|hGz=H<_sQ>TWu69Z#~q$WAj&-1dg@i6s^
z?lmxR-YQc}<5<8=W8t6BQvbQ$L8aJxtd`qKx}k7Ck4HKAch)~vAP}w<ETa!`y4WB4
zLnSX5+%X8;9;$!L6V4_JxFF{HasD#~E?7pUyvE2swoRVsBd~8iOBMgVw%}7<l7kE*
z?1<qXM+_Fh8|;u19?sDJjDY~gK(*P;|6{MykL!b5ts$P_KVvY09ik=bivP!s6b<?e
zwq$Fx!izu0P<aT(fXit9_Xz#`?Ff&3X~)q5^*>{vf-!K%7bO2PO$}h09#N_PXAF(I
z>`Uc85i9>(<<L9+H`RYk2kQT$Q=PJ%-3|6ByB2i3s_!j^{Pc@|EtHCC2j1v&=iTBC
zWcc-s4QXc_jDI$V#$BB{8`aR(g(62dSNRlX!s5}16HW`-pViqvx3V4nOPcB0(Atwj
zvp?-xAY1@&j*bNWd^?Vf1lVzyT3mnHB;bi15&iF-{}`J8&C>sN$ZvN2f2ZS*E9ZZw
z<Nw0x(2;`!F0oFKB#0f2T0G4nt>69HX=(65XJ=<nXFV)AGeS@nR5x)NNTA8KzcA+F
z<&|Ec;M%%zF4oA?gxkNe(@d@?|K+9N!nff6M5Z3${JIH#LKJEP10?!<;e=YCH+8_H
zojSV*E5}HxFDb*tEdTFP`0Z@}-?o`wp<V9B23#HkT(Zc`p~#0?XXU^BHxqY+l#EZW
zbh&`%NxAm|!*3I6Rv`>EVzw5!nALPiKlZ!6Xur7tst5w#>ZSnvF$KVr_5fkv2modU
zzb3Q3Tok$fBGPu4{oSHV-uQeHG}e4WE_tgA`Jf%-di}rqEH*EP5hEXe1o#Nh;Km2A
zSk4Z?_edzXTIX$V1WFe-YsS}W#y7{tWIx{a34b9Rn*kkh=`Ii2KuaG$*p7u-U;`PB
z&!B0hoZUx9Y@e$gYZ0G8^_&Ccn<M2<lW&cF8i61w;H(p`-FF2DT?i`O4j4cM3!oY>
ziX?LSZmUfEkcM_pV5=~3^07bY(#Q%jAWlxK6$}O@_$!k@FWU<cXc!dOpD;m65O4LM
zuHt&Ai;0^c$?E;-tC93Vn&5)*Wsh;o1;DuiFhQ0+8JWoSek?=_&&EK2*aGd}{k~Hf
zu`Jl{Fad&%=)H55C?HPJq-T2lZSndWoyZx=nDnb}55|pz$3BTRrt3i8+r$cf{jDrf
zG3Vou`+qr70OmFk&<+tlHiW->#XSIc;*xJRpe%Z`#um^BT15=AyM(^F>I=B+!_GaG
z@Qs@VydItzlMs-wfI16or+;`=R_P6REh1la$MT<4TCbGKI0L1V*5%ue@?JvJlIvx%
zYtvKCRBS^4HfQsh9lGba1be6My_!75K)VPags4#ud`W0%S!0<s+uN`OWE#Pn%R}=u
zkb0DMxcq)|{aplfIsd|jZx6)Pn^2<o*;<d(R+i~`#{AA3J8E!a*Y1BbNDMr*1=$qk
zfb*%*isMQK5s%}0Lkl?ls8QFCEKfFBnb_w6Caj&Li+>*f*&B5e_8`Eq=-Sz{4mhy}
z%@^p=7W#~VZb69UMDTlvOo%?`dt7F$ctKFD^~NL*LF0*5d`{idADE61NoQl_a@3=(
ziz)#i(i3YtsIcz1k3U56`_a5Oaxa;kFJG}q7`t<#`UJ>%6jO0?+yHWy*x=&Tu1D58
zz+wL}+uGZvP_(Hacw(jl^Xqtofk>O=^Ee4k8YV1al;lk-e%oFsR6uZ))`+6W=2B}t
zfAtW}+M*uhc&1a!_>b@Ah5Ue6B3Kg>Nv6F}WAdH7m(s%9ppU|X)j07}85KE=bWW&$
z?_Q-xYh>)x0&reZ2{s6|@7qKDs{tDzGHTT#T{U~Qb#m9-U<P!Q2>?&5N!-BSYvyC~
z>uU*sZ7!&|Z2)KvXSoaAAJYrG7{=3J93MtWQePOl&sqZB%=1jcjF1)9_UbOB$GsC{
z7SEa3H6`iH3R|A0PF&_NU4Ud4j(ts+U{%vLBO*slj!Gg!uuJne)j7~GiAWx%OPUwx
zjfEIxKr#Ca(hdPCKm_I-LUq4P`x|6FTNRe5GX?*>5VAd;E70(UlgsXPbojm-jCq3S
zdC2}gIJcjGP{85E(bXA~@tK@L)ML<dg3C*4@j`fKD);&bR7<72i>H}$gM$glxAN;j
zKUyiIp=%7&!BV`4saDVdtPOBN3MmZ?Ar>K8x$@z(6_d=Ns(~tmou(83D983e6RzSX
zUhmM1_JS~c12e5ywcWjLE*6y?q#jAl$pZs6#>tx0TLh6X;)yHNx|@bQpymY}Z)14q
zX)!I&RRqf0R%zItRVEZdwCzPeQK=C{i51v>In!`xJonCwh<N>YnP^vGyfLiWHxi$%
z4dnE;@@36GCgvPb$6S95xcZ2W@7sx;6E}Hyxe1yWeW40LG^e?WrXqsK__tVSMp<D9
z2FzYkd<X!95KI)979cQ0HRsjCWE7<D-d8sq*aaFxg~X0|jk<#NwMclUVFv<uj57DP
z?sx;{UL4Ar^fPXFZEp2c<<$_Lb#&v^jrQxW?P+e|#%Pbcr!wyWkbTJ)Eu$s5(}j%(
zeY*;)hX7W)4tj$;XoqP(c~hk-CUt)G!T1Wd0DD>SEET%~uq+E#-|w-7u%OAm9b_z%
zaKbe0<$Mwq3UM&-Ty7rxXdH{akHg$xYe3fMy+xqVZ~}~d!SS8k-9v%1RXP#=afAW!
z4+%qpqlTR%N3W=^7H_T=1HKS1np7J4tFpUipdCY)dE=A=*=JLjhChQhf_4}Wy{=0M
zgs=hAlan<Ji(O@AjET~9Ady94Ak=!VcpCFo9PX4kdXn-s@xX}RS<GYUAwN?u&ITS9
z_QCJ?s%9iH>mx7mf5Z=$bjk&@nhlK(O1aXbATtt7olZBj-FI0s-bI!j^K&gXO&CUA
zRfD@XC=n(>yC6|~+=#FOf1(&(Ol(0-E+xlQi<Xq`oWvb)mKu<5Xj}u;-rc^wmFang
zIMoILH7h;0s2)|w3<%(RTFS_!>ysy}%Bqx8!+c%forWYR;=@eHrzPHwTCl@c2ORPS
zBo3v-!f(SDed4@)U-CebWTp`xlbPk|r;}}ioA_8?YEDxE3-_<U{4e&NLnK1bR9$wv
zKSP+ZRch*qrjo>Hi;opRU#?Lg^IKyirBs1Ncl}u^)D;RCw2)Z?1hh~bYtS*Re)jl_
z9&PdS6rCI@m9Qh5kPW^M&o!0>2#<nyymr+FkXpArJ0zB}TdgKJP3e^J)J9qB49Hod
zQmAzN5Qwl|u3cE7S)X5M&cj9X*FB9R{nG5d@L0NucG5&me|N;XyVh||I71{xI*mEV
z=5wI}e3mMwp0?5PD{a3Wb24*gLaGJ8plMFPa<x_-@{)_opfSJA?@--?SJ%IW#eQ*z
z_k7>aR0*H9f6XZqIdLy=nmt&DXcw8ZfHc|9b{^sk>Y50D^JHkK6V0@9Gm9-%pE`F!
zgRgw*r7$iAhbnsO3q;f9q9QtL%r2T)P{7`q$P9#+P<(tb%FXq(w48#{l5$aJY0*jK
z5;21GOHp>+@Sah(j6x3bUOvvCXc?QtW$)&2IA3Bi_a&C1WrLco9;a=lUpLh~*nZIL
zSb~c)luqSgtZDRcUWjJp*YES6PwIh&S*R~x4}2N-Q%=U@R!`Ipw`L2O4NW4?TBKJk
z3R4`yf~QOoEQx~$>~{_~NrtG{NBK<Y(o?bH1DT%VwLA6Xh4MU!8EGR7&8g$KVs?BL
zdQrR;%%E=?j9FE9gEmF3nsyZNA_WfJEgRkY;iS8vi&x13hWCdFNy>92S4T*c%OKA8
z@hRedb3~=Muk6-!M{=%DOXi&o2y6(vV-$44+tP|*!xp{nivZSavV14jcC%KqU<&hU
z|8zOs+x;qUXylEvOTifSrp^0&)OzPT-gb7%A~oOfZ!2~yk;Gj1XC$aeA_wK8yVWZ&
zO^Mkp7xFUjh87m4N5&+zgi&zHvSe|4)+DQJ1MCL%Yl_Y&g-LxfWUX43bcZ6+2X|V5
z4+YTsiai<}N4djAvD^~<JCZokqG>1F^`bITA7}YcjrF|`i~o=I&N3>>E$sU+Fd!w}
zCDIJtBGLi_LkZF#Eg)Uef(n8X%1}ciB?c%U4RR<cnIWVEq(ea|DN)`%c=R0o{J!t=
z;W>O@&D!@``@ZK|_l|2{zyE)dlaH}tuMt|6Y|DZx0?ZtS6GgdEfxBw553~`!#x!`j
zlROH=c`@|ih0nb2v4>D1hYmBOUnp^2&M@Sax~mSUKYVSCACgsCMbES|;dq-1@Xm#^
zlI{X#E90%P(}uZ+M(U8O5*!2x;uw`oE3}WyL(7Vt2R`AmBv)dLl@q>z8a}W4hq@NW
zSD<F24&QH@kO&(h+IRKp-n^_eYGEamW$^lYIg*&3OO|`Nto1T|A`VU@Hw?|l`#p|n
zGx#mg7g@k4Mpdsp@$8^=Hp)x=JlYm5XeSiKP(g~U4w2XL6gGcyVa9=lsfJ!m50Qo0
zuc;G2Oz>xpZN;s9O_3WA8<TyatWW>VsxBpQkIt#4I0ETIH9vrpGS%ZlEDKlCX0oTR
zB#QBJuCS+G_Zk1<f`S{4X~8Vb+aTWr%(}^~_$oh|6;E-_anFrS7PlsrcEoieTQ{V>
zkQ5DwWU8*?@<v{fR1v3>49PQ12r@49xvOfL3(-BK8x$O`qhHr0KX)N6*3@u)<ptwZ
zQc$zfHP>Zp;3ej>_Vz0JI2=I4W`J}CvP;?1?W(ueMw0TlGvgd{PB~cAIJ>e*Ixn~;
zierv)dK=$qu+eYn=N+@Njr;Wq)Z};?W*L4+k$wvxL%(dJ+B5vVHNGaZj|;jlG*e2x
z;*GDOP*p#7o%Eb8%*we^DfGcRL7ORBXI&7$G~BD-c=;gxVB6dM<?1*61i(d4@lc4*
zxUG=x=DGt?J^XZ<y6qI$?APryvUA^6E*YOJc58vNq`&KVP2)9)*aVcGY+6K!R=wet
zYKV!4VJPBF;<&oWM9)KctT~j2Nbqqd&k}c!yTFm=Exp2$gCbE}Rw-<zp{csulw7<U
zfy_b$@}C|)Cm!#lBuz<M0(pKPs33fM%Fx0Zo+P_P$k3-6JPAI)V9m*}*zwW4mmfw4
zrcD=2kR_H;BXMUqx^rfFZNx)aTUtkOdFe5?0!Pf7gt~RW-_G_G^U}vTI^y_F7&W&%
zAH!MOem{vBN$NW|d!HA{K$%nIAVrXcXT?ingCR+>s?Tm9HnHlfAv~2>PmDfY`QRH!
zf+X;il2S<A>0&<HXF{JgK;94yZ+$Zt@Y#Gn3w7poSR*@V4;1}wT^!4Y^|3uH71y(`
zdL+8XlRYk%sClIwD>Nnbn%fjwC!ZvHwHfbO=u#x%*v%vdOCpzfO%uH;SBrOs(8F;s
z%y+q2US(+)){qlsf5s9hp(_PioFA>oY45cTw~1a(bqP78yRYf|m%jR^<H6!J71yZV
zGDI(;Hw#i&mBd!v$Cn9+>)JCYZf>2Gq^Wj&rJPkS@<S9lG+MHBp3lWDk5i{&dwYA_
zia2dmmk~K$)_nM3XF>?kmM-R08av?Eev#Sjxw37Axv}_`sJb*&xCD`55Aw4QqM^t0
zR+V^z5~tp-eE>?{KGjBUv2zAJd_GOO$*rfV@8IGsP9GTh6=i7gGg`VFVBSUliPrwg
zczacS6j5;5!UaCbOkXyagn3Gw`EZxvCcp!thO63dySaEnkxO=|sEyQniWh=cg7kB^
zPVa2JRq*BEDhQEY-#&`Zt5ibU-H%YDzFi!}&&V-(j?*X>zU;43og&iJfA(_ihg$hs
zpXM*mJO{{cerBClJYY>?y=ozAV7#1fRe-IQh!R1<zS(BpzNs6|7+4VwtQ3lU2KSMN
z^OQkUUW2XF*G7P|6;LO>m-dL56M{UzYwNCMKLlDktUVE?Mo5mnsINf?Bj3SLU3)?2
zz^hG5=3Hlj0>HI}CSPn{_B^uUl@~^=^^8-4JxEzXFeR?K8|n#XwmGne3a=mbQN1FY
z4?}C%xu#AgtlFbP{May}m+k?TQ$gMC{w(cjm%T|yO>0%d7Onl^eFj!7$Xx1*npW>+
z#hNwJ7sXPEti9Sf)#`p3FnByW(J5PpjE9yj-OpVUwz@PFDmFUU)&|!@vY#=%3Vry<
zMc6Lf!0+ob74}ihKF|DFy#9VdvIhqn$AdDm^@K=TT%svURCWvun?vV!D44(V(U?^x
zsNBM6!P)JlDF;PQJzuc3^ti4-wYW7@7oIL9U$C$ze{RE~9*vXI9%o;e7(DBJ(DqlT
z|G3UAYu+`B^_zWcl0qg8ANE6f+RSzr)5ukQ4`$7%!upA%TzWo9lNiZ8A~`9vAvr<C
z>|LErPIv7~yWP`l^}f9g6RuA#eiHklS;OglM~8C(P@cLFbqlea_)NS{jX_rUZ{j3z
zy-7M=O?^6TSS`=F%){w1R7bc-F5%62<5`eqpOql^fsxGb$tfi3lrtZ+n$E61?9_u#
zx!mR_m1SfzuAtAX=guQMwcR8BLgMrIy*!!4HdRKpb_%*;DVNCmNvBaD`^OnjA(nO7
zh$F~VSS~3t6BHLrl)2QHJXm5k+|Xhqh8?^4GxU&W_rwTA6ezvKg$!+N;I}N`VpJh^
z2rJ2<jHIc9CWDHXXdSV~)#&y%u6F?cb6JoFY52jH5Rfj4QHiH9Cr;N&r!%0wB;Q>M
zcG?s=Ud-yv&rhtnFxZs!9$or1c%6`GM=l@dt#DUs5{PC@)4^GcVvO<o6`6K~P3W70
zS-u_7tJZUqg>RxXuAi`3En^_RymlV?T5(4;03Es{L9`u6muXJhllT|&QH=Cz<7?mR
z(+=qn?WG&bof(wZ8q046L~jY-12&_UiVt-<t6yl>KbXr&snOa{k(ib+u%@c1){hX5
zX*6g;-yM3`l`Ymp;k!y=fvj+;=e8M1O)rY>&7rv0pM{`NqsM7hlMl$h>Sk-HwsLWZ
zaDXS910`~Y|MEHGosz|KBzo=@cRp3KY}3Awl3JlHMzAW7%r~`K36vM2U<4BbU5vQA
zVQZ@ldeKvdL!Hcy^{ir-<7JxFn2hWri9B7T)~oHG>TS9PRSdYr5Ad@*u_*D7-fWML
zKtd0L7qq@yzp-~6=rnDrT(-+-oGuo_{Dt3`Ds1&CMlWucu}))<Fjc0B7W~%GuVX#s
zdx2f8^mmG1Ssw1Ee^hSkQ7%Vzm9#?5OmlinSkch^N=<ea^^6V2WjzM89;zs{0D~7(
zBI?Jah1WqszK&IpC`H0SC6r~`Ebm|r9M?RgkoQ1em|hBxVWvEc6KhX+;krmZALM9O
z^*~$$N@mRO%6&C8Hma@;@mY!8H~Yd77ZJvpu_K>0dvC%Y#-yVv89o@$-Y%bgJy2eF
z6?{_>L<>Ti(%q+20$)z$=~n2VaqwF(FBNvrTwoi=;849U@Q5^<5EE*rsSOo80sNa^
z^52=Dch+%q=I+NwtL4Z`+~vy{z3a!^`$mM&Ji4L7hVtp9{$XA1CzRR8W~!fuxb#CV
zW^~=h%z+aQo~v@YB?oRkgjyHef&{@LA<`0<0;`-D+^*hLcrk8&**Zb2Xeyw@R(iq-
z{8A=Tvl0lQ{j?|R!Z9DYd>(lz2e4&GsapFV5!nlKhj-o8AlO-)-6KHT?Q5c0iwkwr
zuFnowo!DoH21MRTY{C~3QR)lo!n2v{)+OfAjY+Vbl-(+;>&JH?BVV&el|_#`OK{u7
z8Sc_TT$;cdn-{c^q%Eu*v2=d=UYjN7RQSp|qrPO$C^wy|aCf|?vTYTLX$35+$88Mj
z>SNu;)*oB5`8bOV$=~%``CM}@iC6E%YPDUo`Yf3g1|L$vG$_1$&OltgJG!yif<5f$
z+lco<g<J~>E|CxK*WEyR)pL?dCJlJYq{)UAZi@zOAv3iFxfyy@Tx*?ni#?#4Q#mdi
zdM?7iuoaA|^E$oD_Xn2vOR`jay^j?oHH(vIgfTH1UBm0V7o2TeYcvhl#s110g$sPh
zqj#7PYtWy4Jb%E?T6oZ{8>N#YA7#5Q#(HBezM(2aVwWpK5X*%F(gusjW${Ka_oVAY
z#>y~?8k#2WM1S1jiSBpU>9CvO`xw-cJI(b#yOVWLNw3|v;j1?f)UJj3^85tfXPfTz
zDPaGjt<_rTY{9_BrK6I`<X#cv^BP6vFj}%$OY5}3n~g5uzP}V^*PooX+zRqTZKhx$
zadx<^-5jzDg{gXeKPrT*GRgYpfIZ6kW!3ngPw#dC%{u*377uT)d(k<$VlhrropJ@c
z>J(>pr{~VXVTY$U!!m<#490}Bli~w8ph9sgy<~ej+_AbPp}-HB%pRnjbB>vQ<ekW6
zetz$l(6ch(?{|+4#rH1LM3ha*@2u{sj;3f<gM;RaOmavQ*n!e0Tuv>DM0`DyPQcRt
z2q09v;rnL`Ln}y1&#QOam%0W2h*c++5L7?9Bc>oUK@muHW`2bD#vrwaQ{0;Ih-7~}
zOMQ`T*h`IhZpM0-3FswX#~@8rsaWZ{FL8&(*0rj}xk1%u@AUWImb`|cy4XcSM4UOn
zFw3GWPF6^X2ejeR72N#}5bIPu#czH~!go_vubj1Z@{JvfUWL1^o}u8dWJ*s<NTJiD
zSiXreIBU7^z%V0Rkh@n{(7G{QxL{mLP=W;Jer=#K`9VCtIVVhop6X0z)+0%Z#Q?6X
z`DYwY9m-pqsp@uK-dIy_m)$+Z8AOaZ_d>cOuf)-BXXddcdX*uLLN$5y&_KakH<xx+
z>YV)i%5llN$~Xmb6Z?wDGNj*Rq;Qu9HZ@9{4sO-<u){bBToNo?eRHYD5~6s54<QmF
z>}6jE2@g(&^{wbDl;RZ$qQsK+E~`ly1G~wjL&Px68SX{DA!q!Rvd(uJvUJ(BhWGTl
z^If*?et#la?RzzMy1=wT`+$eQ@gkB2PmM9TqeWf?+Dy~OqlJI5=F+DtoJ$L_q}zGz
z3rhTY8PU$``M9_D2wo&D^WTzIp^6kGh#PhDq2Wp5pyF)9x@@zp?Tp*^NenHh_xD9)
zB%JSqyjPxMx3+Ajxw4~SBYbV{gXI%J&o_=Zyc{wjH8ZA*29j!3b-;2WFS~W<tlBH0
zoA5dH;#+ptsTK%4x0BG@{pS-i1L@MEqK;4-$b@VYOK)h~c^7s2a;%N1SyXhX%VJmq
z<$$g>zApJ`UAixhbc~&!2K`gN>`>2goN^zaRFu$?cq7kJF3s(;go5U4!%MsF=J0S6
z*-;N&@z2D4vfONs(4Uoq7p))PTnqTr<%)TdP-0MJ>!agiyWQl}%4<Gl2EQlv>Lr60
zn^kzNeuMu9_~LctlRCi~fNbXFp)Q{k2%h_gkcS9dS-#ge#ABVy&^yH=6!$C&9c#!V
zT%~hH%4nCxiB23uJVb+~%P|1_j4RgWOZ6`0Ace4a`u#ink}QtCoGN*6SKzZ0PkH?w
z+u^F)?Lrgx58fth>5dX;;q5_aaHa%Q(w6k|;0$j=ilbZsRippM%^dB?ZBAu>1f5Q=
z1sBgT*xWAkdE(i*QDL4k*}tt{MD&qz*38H4#^L6UC*$MvbtV&db@J4&Q9TeJOOcI7
zfmHBQtN5aAeTDf1p6|K6?H9LW((hAOVHy8|6vmSMZb!|HZX5rJqM=|m>*t6wJ%?2l
zUXdC&-|(}dzUqGOAs&T4Haezv=&pO>^_KFIxKSoY!x2q*1By7OL;yJW8rjgFl+bu|
zBVM(G1IMtF1IKv!OWTABCNBd+80;jw+KVMG6E%(c#A+*avK%g`Dz<2*GAYUcAA}^<
zx|)RT4)(1i2e-T3Y%U@~dP^h4(2TF)0}<+Z7r##y4CRepy*PMO1=1PZhQb4R?3q|A
z%cWosMiOewq{bt->XqT-8$|2r>`{C%JMFZk138<iQRmt%6(;=#@v23Y6=6YB?0Y@q
zx|KAsqhsQ#q*kuyePc#wwX`wQbuS&rREbfpDP}<hFp+bZ3;7aZRqhoje2)Oj9V?oY
zP#juV<4~4;M+2|@gJ0=-^?5zAm`Oe&4Y*>J=uqmr<1b1}tK4OAM}2Itcm{{Lhiivb
zS@pOQ=MBueRrJmh(Cfbh{%LZwnb3av2j{kHoht3TJf(FGe2Dy|(Uc-3)}r$Hj{;UW
z%8|0piu7FkAvq1o<#N_q?f!!YYqzexly{gtGM4ZzAE%D-+*^Cw%ft^-AW*<rdIUen
zo!xpS)^)WD(L*lgZxKYQH-vkG6BVmq5t#E@!yXC*r`KDJKb6!O6l+`592hX%qn#fL
z%wXOdnY0|7ZfnNRDt*Vi*K@mEQpXFk7Uo-x!|YyVfT-(U7r%aIT&K>_<K=3#+Ia1k
zKKHG8KB)QDx4G)1w2{Gp9#fH=QOq|H4(iVX4Jl*k*Tk9A)FHu0aWc~KTg%Ys+^>_%
zgVrhZry}aMRwcS?WC>!7`(4vngI4WAU#`lU%ia({N!Vn<&4^@&C_I{<saYO%&g{XD
zrxXsF{(7XD!-^qG+4qfm$)*{nD0fs=5R?VOC!S^*3JM}kwhf^&sntEKJ3|OzS$SNL
zCR;gY;KufB@Ij7ZRF=2ip3;m&ZY}o>*wydy=6`k5fFsy1H^@kkKno{Ao6p&1z);+3
z$~j>w_fb1!31+=XJc4|>&_OCGCe{*5$I&U$aX!U(LmCH%k{GV0Y=q-uCT1hjG{aHT
zok$#K5|P=}Z}L{-fpdfqism*)o?2$1;F`Bd{w1}>Bcrhw(^hsy0zyqw58t%5ZQj3^
z+p5D2TYYlntYyV~pv>l;i1n<*v>jr80A|Sxas#TY$TGP@kqy4vxk;o`K&t?f+ipnz
zrSnUbsxqT1%$*@)L)r_MPz?Tz-)vk0_;Sf|%_Iw~Vh)(kzaN7)kQoLSyIJYmNaF`K
zs$8s%EROP(c2yG!EVn@{(F}F>BbuIeB@+iK>#oLDj$scr7qGDEfhETMO9bBOJc_ut
zwls7ifivY_t*%dX)D~k6GamTt1lYsoo9q3P<B2f^f;@wT50D4aaP(1l--{q%J0^O(
z43KQ#!YlJSgyyxSEqQs17i7~;Rqm2_i@4K>t)z(+@{rV|>O;s7c`EyE$zRe-jkd|=
z=<9Ed6mvoZ^aW<1W<B%jCmZp<uq<}E33Vy3ay0cw3$kKvdz6l<+@=7vL<vui70X<U
zHbK_g$&D6?lp>ABk-2U+fkoKePYnp_B&*d5h#_yw{?%e&=t7Kk<dgS(SV%cwXp&}R
zb9iPWiGjenGf!=V9aw!gKIY=~QjRnLp-8y&&K?VXONaoPbze^m&nSjBC|+v)ex16e
z)4j#Zgg~$xPC6ze0xOt`u|kogb$_cgl-=4@nLseCr}A_PqF+BFMpWQbc3@b~Jk*))
zeSJ;Qr<@kG(3+MUFS98=&LC7%$Z?zP`!fmd*FH$d1!&`$5GvFON0z6Dd$n|(+EuN)
zcFKW^_Zds0ITFrUms=KKBc~=t8n;Kd!J|6eVIoAm$JC%CBylUgiS7$s#OllR`>9T$
zOPC3uCFp6$7!6rj@+sgw0_HV6RZl_}fj6M`wD<dMQ~8JWpmHC0xBaU@2#<6m1dj-u
z8xr=sZzmpx&AjgkzVJpT@~M*0dl9^T!O~u0L@L&`qWf`D(GQ+B&lvaEc%;Y0iCB;V
zTeV1=fEI`UfdD?z2C$$oeESaq*A&pRbrEgkCkF%n{PPEp`v|v-!apr1pGW|hOYgOR
zvbz2P?DX@50aRPh^+w}2vPj=15<uaQ(~3VB$zQ1EFgG}TNGm+}qgfDSc9t!(gg+{_
zKcSAudT_^&Ns{b;G(*A4P9l5stJ5Ey>8AmF)+%Y~<gDqRYtNOyo*wrO`_E4Qe#{Sm
zseGzE82U#u9T;iCGaI*l)NOwTK*<W+(I1|X_<M5;1blTU%YVWC#X|qD4)${+D@GJ}
zMf#q%JOogfh_!qhZIx@=sXjFN3cw877G{3xLqwnaO#^%Q`ueUZ{DZLox2B75r}T-(
zWc)K%A~|rUtZ<<MB#8d^Jw2DbG@v}j*gAnw7IjFtP&$N~S2QE<Vd@q4u!*3j3i(<|
zQMPc1%-`}DrA#PG+xSwY=hW}I2Ldeg(=K)Gw;`4H2E5<y3SURS?fx|&oUwiu%9fSJ
z|BQC<X+jWS`~P)}onikA#jkbd|9>b_T}3^?s~I^?m7bZ&Y!)RyvN2pB$<kc4^mFo;
z;0K!pM2<16Na+m(lv|{hodYCJG%t*g<1hcc7DmcJmr@1I#1Y|}_+ed$X#R+=0-|k@
z@K;Xz3+7VT;Sgz#e`H}sQh)+}Hu}ALy>$I57AC(7gosID;56iE-q(NW0<cd(1H_JW
z6}6Xs8&>)@reNK$g-%`hz4>4EF~25M&`BHU<1}pL`_IMb?`u(D$VLg4{jNQtVK4@(
zovHHP4~yDg<CYKT0t6krPY$gA>AU4Q=%bCe{?+fxni5czUgfw4`)%UDg+|kY(^q56
zw?CSR!GuC$_AjH}ljw^+1<U{Qo1e=6xf}M=zYKwXJsSwHE$HJ8t>2A*Pd=a-D}_~|
z`!Cc#L-jup>gU7H8Ee7&At@*-BQv@|+4qATJ$Z2>)$!eC-EO}{&_22d6pGEMp?zvU
zE-=yo*~<XwX#Zn=(e=NVy3_v&Urr{={}&Pj)5ml-fiTH@ON{!DA%vo2sE+al@=+(j
zr!jvFb1JMh$-J!kMEU0$QlIww^#}+zq^;U3)nT3Ik4~k5x~}?Q&EooR?Rm;zxDt53
zG5>8=bFKuFNhZ4aZ`16buQC&uklZLnpZ+$PME=+D&)wJmolxxGSmar=`FP5a69@dk
MFY2n5so327KS``}od5s;

literal 60600
zcmeFZWn9%;)HR9-N+{ALozfsF-60*)-Q5k+-QArcAtfP5cZUL-kVa7&M4G$yIeH%T
z`Q7{Je!BPd3){W_v0|<{#~gF46|NvB@em0g2?hq{p_HVk5)2I71Pshw5(HTAjg1Kc
zCHQm4Nl8K&rfQ5}8~g{+UQ)ve1_m4afK7QvN{QkCT=3CCMcr9lR)*Wi&W6#z*v`;|
z(cQ)#Tnz)m>&^{6+L$;Skht4e+d6T(^O5~?1vmH%{hEo4<ey8Nt@y~)Wfe$7>>N!<
z*csUvnaTK(NJvO{9gR)7l|;pV-VXl9M`rHqY|qWa<mTqa=*G%u=V->n^6c3&Cg!J1
zPoFY?D;S(SY@H3<8El=%|9!}>=ZKm(897?mJ6qV<l0ctpU})#!%tuBBebIma{%xnT
zh3Wsi$=2y-w?GG(puaG&Ffud!_iS)0FZ5e(5jz`uM-wL}@O*xD-hZzA<GcTP&%f6z
znA<tqfl+X@Fp{!$HgN<uI~zb-$N%)_-T(Jj{QWF>M++0s>woTM`FZ#6-~GIvmkHYO
z-v;8}QU2#!FwFc&yiEU189!3>`qL#C7(o~*Q6UxgJ3HAfc?2IJx66Wg>JA8UB!V?7
zdvK}PGzyCr$e48K^>CVb6EI?$5!iMPj}#V{bOP_W{a7yLZU51ln7;JhYj*EbOYZgh
z)<}9`Zu-)QTh3?R(&a}2R|f(<6eM>LB?V#8<NV-$evGDK7B75U&-d5Y=y4ssh)=lQ
zKl=NQJQ_I6EZI)W!rz|p&&Rx#6Qcil?7zSIm!K-xbHJBJKmX&M;KzVzufM;7%Mcdq
zgkV;1(Ei;<B)R>+zd|=hx&v>fTs{BlS4V!e2Q>Qr{|_6zg?g6XHvYBz&8I8}$m_oc
z<%HtShVDzx!%&|OZ_MspUwkhwedj{?moenoAbz01z2r1DbU`T~HL6DY+a}C0{X=kZ
zbh;t0h7EXrjrT8eV#dD%|Jj_2_kY=dI2y2lh@#zJ7yf!{hdHpM!qtQSWe50Tzz&`y
zG?4#2>4o4)F(KV||9889Efh-5$Nb&xC~?8gXZR@p+lUpxtE0&*{!b_VnUw#p&qSVY
zA%1Q4De1v#C~*w?jQV=4fY0TFvQqJ*?neSq`0SP&1MzQKkA^4`Wu8vUu#R%AN3F;3
z*vd!u`~1E}CKTa3{ezD)?b<ZlSA%^|*Ui3FHtxL|o~|6^|NcSwTSeVi4~BeHZH9gT
zV#p)nPnBO^Rvg?X_O}0$A1cT9{ll~@=hBok)B8cH%!;;)uTCQ@_qnfd+Ru8ge=INe
zZ!N@a6Syrp9EyFL8Lhjy_@-<%DTZS+fa}>BKP|_@K(grZIh&6^O%-J&L7Fi<eE;3I
zDu!>Vxj(e*-hV$^k6pHH*=>NBK5nbS(Q%%f)$`Tyfbnj5(|&W&eznO28vU&Bx?kYt
za3%1OYSo&?uQlWuA;?s1uj|l@B}LSF(1q-KwN6}`?K+>c_Oao*?R4Hsn!X0EVU(>)
zp8uSpc$kvNeZ_y6sXeRiv27EnVTE>$ZmREQvYKuEq<F5+g@wb$9B*^snf1Hv60;l+
z({`V0N0S)dqk&j~8}|;sJ3KqpB6})Ygd9d$`)!wdb>sZQY|EfuU#TyThMAPDrlhDV
z+nx_5jHW3L#?YFPnZkB`&h^!8HYVn|oRlE$?fIPT_9;ZBav1qyBT@c{-5U(8<C3hv
z^~vRa`|TUw>u=gV1UWBHAv}9c<a$$!khe3M-wwJTNw9uC;!FA&^zzIQPhdMehG<GV
z%%fQP&ThcHmwPSK@&Yg0uTQ^eSlY~8qDc^-Sz=fbX<8i(Qs<gxSiB>kHuVpD#Ghu!
zYj}#+=`pM0cKubY9qBFeviGq~O)s|7*fVOuPu|C)pusr{+co_}Mnk*}W4y;B_X(X9
zVx|-&2!nC<RyqS0wCzt*9JUTrxpur*Uo%7GxL%wGe6sI;XjRp`m8$217;$rbc^tjI
z&pOT<nBe{Wqnht=<}&mz&u+akHl|(V8F#MBj7EGo@qGbpFu~k!c)464CaIHDcGMrU
zG|WqW8ZDn(-ja@`$59iQ#m@TOh6OqHk17=Q?(2|({?4}xFhnm;6hngEqX@NKpDwI#
zHf<zGH!P57F%x}yUh)2PKA$5Lg)rg}F196EP1nO89=oI16%4=6M!dw$X=8h@WOd52
z^Rz4j?u+HUfXm%D5tKuAh2^%P=xoBA`u5AEePNqDFbZKZx;{%@`|p93VyiyaWtv*r
zYuUd6b7FSMcYAd#Vsmp+D$vV*SL<ZAad`~M#^*Xk(~{=-+J_GLWgI>4_(~JapLoST
zhjL;s0OLF;$Zs-5%Qk$0IviYZ*!S3O2Rdh7XM%<wdn{)|x~(8_?kC=tyN%l@!RMa^
zZky+rG6<$O8C!O`Xg+CI+g7UgW!a+5S(aCxF1al)-`@OKdTp{QM`aJ@8ko{F>YPpE
zl6pk?ntL?s7?;=XTbwr$T)|HshTg*TcFEbYP7Nqx+uvM$7U(C#FWt)Vvi&4I#(%j(
zu+|<|i3NW}_cLw@K0-IQ>I^69$U)53Fikf(?8Twpla528ULY!(Y0$w@qiEPw$Yf)t
zCz?EZI!v#eAGyI8d217Fn3LcKmmvhxD_J)%3~ae{?gsRcARV%q&AIH&YWH|h!jiFe
zfUH?_J(}|q@y$W#fU$+=UPb7ykbFYo-=Nz)7?72!>xsw>m$I@ijLgupMu2^;pX_;i
z{UcK7xjQ_O&r7<hrZw8~3Sq1@;vD-f#GsOn_LF_zTd!33SLnH)bG%{(Leqv3`nu;O
zh1(5zwT+_LIp*~57UYQI_1VuEgvL`HY*!m}lhvK2x}Bp%8129Rkz-vu7y%#g?8z<~
zx%i^n^2?_5!>*jFn%Pl-+bepH496jwx=p6071*nZfljRG_Mq!Ir*Xc*4*&Z^1Nf!Z
zt4V!|=W|A}o6H0bQdtgtc%<(U5FWh8-|8(~ZT_4itz1E3Qvcuvpd8aH=Q}JJn|Seu
zYM&~C)%k(rykA~j?Zd2`l8m0}9*m_bNxuKI^ORb##4&FeVw(9-EP^4)aM|aQp(H_B
zo{wFI5iM#144))Lb6L}!*UcVb&Rtz~{&(-Ae<<2e@MtaJ(aozCWl-~rP&ai~HN9Be
z;EJ0bdrnu`Q}8zTYJ<AWK6PD&vRsN8w-x)`^o4ShHr^iP-jkd&CuBnn2iX&<J*1Se
zE{qVFhy|D*?g&LQnyiJihb4-|aLbWruBeLPbX}hM0n_%>rLfJS>;i;}@LY|Hj>s0+
z`p2&k?knmR*Px=Q&V94zEN*esTh%U=<=aq8VxrQq_mXFu6b0_|%muih1)IjhT`C_f
zG)Z<Gwi(4^FaBOkWz75X_S*vwg7GZ&Y9xy|W1Azc21HTrN|)l=Wa`U^JSF|tKEH6m
znrC|Qcb4a1(|({lk**)VYlRt!z-fH*{f#e!s*Hn%2Xhx1=OR@hd`5wHD*Tgf@-x#+
z8`UwD0TI?md1Y4G%>^$-g@;C{fbG}CshgKt`vkMq*a$O73`5$@tARVC(rWDDXOWz+
z9<;v5@!A{Y$dp|tk{ZO{5_^zi9y;-c{;eVTgmqnf4u}=*0V6RK#SVyl_KFq4D3_G>
z?8<{fh^H!H;uEKm{X@ZZW@=hCZxh}p;*m;$=@KS7a3x=G<lsA<Gn!rj!6SE+VwAfc
z9!I-+1IB&H4^?)vg}{tf<5{P34F83>E2Y?UNy4Cbke1QV&Qr8U7p*WuDWXI{kwy+m
z-pvQmGQpYLg<pz2Ta)Cqy1>f5HJG#;%!OrRWg?lvB%7?vMR{E<&a$Gw@`;X8)!L!w
zlE%m8Oj;pJTSAr_dO6-*eLUlaNVuRzJFw_19F1iQDdXX%HsLpA;TF;l@Z(eJ2#eV^
z3LR#QN=HtVVsqalrOQ%e54K6rw13A{o~8Pkvb^*Yyo%NSEIL2ib^gg~|Iogh0<vo~
zktf8W^~t86m8Hb5A}6X2($1Sr0Pq71b%8>W2>K=x>K#fkp$&N6_v6gn4|pvMVg$Pk
zb}3*VhO0Y^uJ{m1S?p#xjnUBHz6#zdf)Qq1@M>o3!djO~qeG1HcP1U99p8JUU4)PQ
zjA*NxoY21G&LnJ$TTl(cFp+!*FY*z$fL!#mt+Y+g&O`=F(m}&s@{&?iX*S79&n;jo
z*O8yaa%dEvNBNedn|4`-;$a5|zQr)YKMKFvnPX-!?4#hIPk)8ESQ<Zr*;fzCC)0x<
zca!3MGTtDPF-MBIuGM)RXideB>bh-7`*dDEfXm?EiAa?fo33{kqucSuJ>^Z(Y2&*u
z9&9D$0w+~-M>t$M@DXXBN436M|NMBRq`z%+-3E*~YUV~bbq=mUw9{;Xaq^z&Q+HAJ
z{qR>|C7bX_-BcLIF0bGlv?!=|I!1F88xKs)JOS3w0XWV(7F)}j#qbr`?U`o?)01|A
z9iE7xp_Kg@d5H$HWg&Y5A=&gRU4x5t*cy~{^*2pv1#;6?c`6mXh>MJ2C*7%$UL~+`
zj7(4Jy3Mt159SJ{BV%{Qp0viT-A~t?Q<0vLy}^%Sz%mz!vatR&$=Skra-H~TU#*Lb
zU98h4x%g`!u&?3=pdh5~yeRQuME{TK>ciJWX3Rw1@u(j$C8a~$O9t_<-L<!xS>K5x
ziq5)s1|Rj@f50kc%S%wzviD9!s7Ak$K4)~3q}*OSnlVMk6|w_^D=@P(5yxA_GN!%c
zj?nGFo~m~qC-L<QO1FN4@x$Zi*mC%yE31@gHgOVW^cIsiBGJR~Nc6x9WeKkvI1S^o
zPR^luC)D)_$!^k7HyIQZo<#;xtzj~uC1kfcx)7z74X|2Osi(T(Y+6;@J1$U2lXNAN
z?tWlOs%Gn>KhPOT$d_q$DEZn8C%?h<g=+mcj#MNL6=Jw8)Gmi?R?B)UJ2Dp5#@KeY
z#(Ye+U#F93+ZScok@>(gPJ)3vp>BQSqmnjSfs@H`%jLAtlJV*TH5Avq)oX+|DQC5!
z!>whVLVWM{1sm6gGj{81wiy|+u8v@WOr{_{!*}v!+`vrMoUQg>$Xr}Z=iVgkzH-c%
zwc%3eDQ^Dyy1X`6D~e^|t;N3}SkR6d^|o>5S>)}#uN9xC<Ajn31Y5^GLokQs8#)gA
ziHWlfbqBT*<r4g0d5aIx-?zXg44FK9QN<WqBOc#}xR^V(_85D;vm*5=d1u6P>+md-
zNwqzd!S0ip+|3NiO)F2#KB6?M{F;Eoz80Q06)PiUqqwvz)FSGzL0uMC0DZLICCN-G
z8cFgaZRt>kVbDpCqD_}2DT<W#wt(xVkw38SD@kI=l}+_}zIU`&5RVUM_U?;$T^35I
z(5YMaHtSgywL!Ck>F0#0#Vqx$Kvunn^rmOTC3Q5DPdd{Z<c*tMEO{x(7TE$jopYZ`
zu^EZ8ugESxWqX9IT}K|}?CH|wmG0aSk{d(8CDa*Vm||NpQ;W#^4sjE|5u4J%V`W!r
zGM`hcxWSn{pw5#dD*pm`Xo2m)fP?Q54v()Ko9i?3Q`PXVV!q#BEj*3VLHYO#t?7!A
z+;~+wb=c5yO_Pb_bUCpa*XXq#eMWrnFpe)X8SYE*PMM^ehMzso7j^#TltRylhFA$J
zOWt6yUe_crl$L!>l5ntt*|Z)5+~ZHix8f*~_358~>J{A_OtENM+1YDNSm{yd=68qG
z6g$tKB<#*BQ+>_QAcr636Dj?yAN8?JH5Jj@!GqIZ@m-M>3ll7=nEP{_1)3j8i<_e&
z%IpYZP4=iA-<r5^9o11I?~@bPD>n#&aUAwb%0!)a&|58vb`Ga3gBa2*LB~;22(tB{
z?j-n<ljrf_;CpGyf2<YGjQ&c_UMaxDzcVp7m8R~WVQdydWKYVVO&45T(|sKFCHn`{
z4prB;w1i-*VHv%8M=P!t_#hS)Cn2L(&FXKPB=f#Hefu~p8==>G;u@7E?w+4YC*_08
ztomh1`CmlmuPZ^4<MEZiH4{m0t*7V2zJqQo?uCa#O0o*1PD84rw&S0X%u}b_T!~35
zATfofhF{B1DfQS<EaMITsHT_gP|J(HuFKJGoY7wrCr<B4uR-!OobR!dS$CRsO$f3a
z{{9RYO=BAySN-M_kjD<49e#a4h08z=XLUF#{5ni9f(B!R#zZx@LkO)iHueihW3wAB
zBkxUyilwXV(KKgdmbL`2>YOxhWt3mwi48igN7aqs^RUJ=u_z-MrO3BsY+hI<t%%Z!
zwqJ?S)}f=@4LW9*T!oCk;%Jq!Ut7uUOOdUHVK<eCf>`@BJ2yj(vF2TBdFHU0xV3B+
zwv!=Z5gMF#mAXx+4${~jY?-%EU7+Z@b(!HGCj;cM#tzmiy<J(C!Q-moY&j!`eG#<u
zv-B2?Of#0WeTHsZKdC2ACjT?}=CQ*@qX`mg<yF%`FIx^wsWwX%gPRf^GIs(16K#}y
zp6fG49-z}ur+%r0QTI@e!6((GASS|__Icb~!bMTr3;b;aNhcisaLquEXLkS{dh1L1
z^6Cf|<uPQYRzC^K&}506u|8G0GEY|Yx1)5?)f;hKku3M$hDY^i4p(@b6;kw#PZUsE
ziBUyr(q%EYAr;<Nq&ruDnWh_?+MyINCTBb>&Gi`?j^x>IW$l7bwjxbk-W*M&(_4+x
z@m(u$d-ek<dPB~jK@1nwhz9d%q~aunI{x(3sP?%jLG658>Qe+((|&wO{*xG{Cl0Z5
z&iBW^Dr~)Q)j5v(+OtsELN;2g1>k0+8fls5CY<zxgQAtdA_-iK=Sacg&TKxebG_8*
zwRD_H$B=nHm1#nB?lSX{^$HV8ZnOr6(_u;(O@vu(o-5mxpm8fqT}=lhy?dWLDDTH<
zuPz#l8;(5e6Zz>>bEzj{0x*cX+*P+G89d%3x1#qv^jr(4u147$D>1QazH(O2hnalo
zt}8Jl{$WA|-U#+vMHqLcqY|vtAQyx)mO2NRz$jVy6J-<cd4z4L^PzE>?R2JO`EGOf
zf(DP(9W?`ra9*Xv^x9Qp%)Ya}QeiPUF~OxF6WZL~u0e5RZv^t7_etg*?&ftubpC>L
zY9l$a$^)h|Dffg`WJ(6+=Lcuj4ERb!7HN2~hHYvRE2pB^%8yiaTt4QDgrdXWhYY&i
zp|<CLMP3EV_#o=xx-YxVEN>zAVIS;{r8usM<=J*v1pA_5^=9D9NBx?ga!{K??Ih-9
zi!NB)%8lLpoy!lc4Z3jZUX!J~t|fMye@K_SD!%5HOzi83i)F{zH2s`=yp)y_gD;?F
zLlU_Df2hk71K8asTxeb&F-n;)nAO@g9v@n#JHjJG4rMxuA_RwX2eRMuWnd$65H-uR
z;ea(w!ps7c)u89M%Fy{9&pum%9T=(a%u|L_QY?~ncb}9f$30fX!x}Hiq6pk(GvT07
zTxg~J*2l)f8;8d+(}l~MIfS}fcqCf=LY1*&(};%9#GGO6ZYte45(|-1O@RW8m`auU
z;FTk%{9sqJ%%^-~r-Y9zpJa{YKW53rF3oiuGWQ6vJdP>eGPQa&FzO3UMxr|iB=pU8
zvR1#uWT(H>K!_XWUY!dmbzEm3e}(5h{P1l<Zj5<l15X6}yDKujqXF_s$pRJIpxwmr
zMv^uJuF;goShjsNd@nNeX+0Jl`fD(sY2Tfr3s%FW7m+k;$^7yjw}nS}So&Yi7;sk$
zh;mnI4i0elU8RYnh8;TllO-OG9VrOYM{gKE-ZWYcc61qJQu9j=%I9$SRscEa(9AF6
z3jRS8fC2IINZCDO&OQtpEvsz&^ltc_B8<@E@S(QaZEKV9kF-3ywdw#QNi#}w#CRER
zgqCsiN*P|vSw2@S<HyjFsCZ*UJUkST$k9AQ%B5)HzWHMPlz^PZ?b!L59Eauz;;{21
zo#6T;Z-b*r(8shi_03@(6kY~Ja)9xe1nqhHH+gI(<~O+>Yx(_Pvd$e2H9eH3PQYLZ
z3U>_*&lxBnrKSC$ejJtI-?*q?b+et<x%Wx2^=;=~=pI=$!$+35T6^wFFDA~e)4_f`
zuG33{$vaHnpQF%w%=JBTr0u?L1&OZ1fEA|9@>xWw@{%kFEKAvMHh&RAVtG5rTRHd&
z>aV_JyF6!2x~%|?d(|`IdILCin*sAfDUtpu%uYvlU8dKg#@TeP{j?PmaszT>=#;@Y
z2lOh^L=C;P%>-n+L6hi%qwDT_*4#ajTvL^3S?nWps#PU-%rIbDZ*MLKI|b6u#mPo^
z=Gtm0>nuKyOlq7_+%D`utTRfwII$|lU`lt&3^SNH;=|+CmFX2k<er39zsD>uO@-ga
z-0ROLh0rq3)RrjI_{XDKG3Od|hZK(>A?_^zA{En28vCr~=3z|)B7(=bDaDCWzTsW@
znlrpq$-NEjOz(U|lb&K@4~|mF4WuOA3J>wo4lT!^?^n)A*45fn<w_oIFygwDc^r_2
z)b|<8Rkbpy`zp<vyD7O@{9#xsw$>Jdg08)Amn|Xl8~wz?W}}vMcKY<8W8Oc#TML>w
zQzu>vR|momW}*zb?&t9DHUqTsW$ASsT+516Xf-N|1Z-%-S>o8XAo1i9Ap#67$j)?J
z&*2(`NUPU9EIBe)@aQzfGg2NKH+YLoEoILF8yrF^Ptd8WxlgC_4C~OVP^O1KoJ7a>
z`a7jn4LPxa=^4rU=hy?L;uHpKmh-7hUL2WnUiB1O&rhvww&PlO$5JYMfPdXOa-h-n
zKI|RrIU9W1x(MN-a5ke*vfoPRjYt;>dC)1b0w8VwNK66$e(N#w)mkF~4*~WtCv)g`
zm!=X{zXTif1u9+(BemEBljo+cZsM}NnS=(}0)&u~p<)l^Zo0d=1P2#>xd^uRZ1q2p
zta#49nx)))XY1~BWVLeNe?X9OLEcM&0nk&5ez`GcN=B2tzHlFPY7I9>+Jb71WcYrv
zW9H<r0#zkt*YYDprx%5@J<PZd4b$<iWx~k=vc(N5C|lP{MdzE=QroXg!jW{aB;})d
z*DyIB^~7YT60AN_wI<fGazDM<;%rB}ivnAwULibI{-%xlllwXg%>$EOKI-tvp{brU
zM+e`4`35nA$Y68k#(pE$xEEt|GA)z%*ilVn5LfkToJeMm%($icd(;bal@zdZrsZ+s
ziK08QbA}|N*$hNvZiyE7i6n2BBTNPi_zX1BG8XiywB-R?v8CP1xX(Cb<o*b@*XEmt
z7MdG8e681XDOQo_#A>i`m2ysOw`GK|h?v=U%44$zndcberhs{i{n-1!uLWKWHfKfI
zUy{vI3$t}7@GenZGb0zpjZ@a<dy!LXR#TO`pXwQq8ZLGYl(-ig(qXaZ2P$?;$!#JN
zmd07Xqa*r+H95I2p!ZaVu5(9jbE#Fh4@AJsz#ow6IviRLW-&I3l@UqS>+J=s!NwVH
zkcJW{5j+rRI5b(+iY;SBDjLvrpTaIz87spx+BE$&A775)))0%vEE;C{vw9vxX&CgE
zn;IGr79c&EPTE&OS7G$ylCs)RcMQ>@+Vofyx(qh6*0c;RhWF(#%-y($hy2kHse*Vo
zyK;-!rcR2y{d(7q<-*|7kfOpn)B9*pd033_MdnJ+x2N->J2gQG!3Jq45$hGEx>EM{
zM0iatP~*dOdoe|_27zq~Ty(9Zd?T~Dj9Gv3Xo$C;)k<7=XSrTa(sqAZ8yV7FL5gOC
z&@FgQZq=L-#ISjoeO7NFB6q4YFhiMr`bv=f&|KO|g`)jv6eGHblPWE*Y#>1_i^0DY
z6j2UgMzJ2;u2!rRk)<4v1l}{PcbCO*M?4JA=gCvV(K2z4zX7vFt01F~XCz+ZnL-rb
z#Jh(PLYpiTfT8uIDl=Ii&A8A|JvrPnd0yx=6cU|^JVN-f|2Z#5zQ4c)HGC;0zx`|B
z0zB3N$lh9|kp0Z*w4#Rn5jV_owDE(Dt<|s9T)fg1!raY^pH*H4(>}z>!eV?9^5ev5
z=tynt+mUcI@ryS4IqT_iJcOb<p;~aI3OU}*wBBbBskdKy(&kxyF?TYshf$iJ7Z)V$
z(K#t)!gyVA-f9ne9ChGh+MD#;OczrC==z9dQvZ+%7QJzN!o>l*e{%6vt<@B&Tyxz=
zf+MP&&r2syR0-~7v~Gx)*#uzJ#8>D};*7=hI;gU~q2SWE$xtzLxUzOvP#M<dAu36{
z)2xo%t5yFDr$61X+NQ+qe(#JDN<!P*j?jI=BqY%Qp2U=vR*xqlR!s?8ulq@7F(+|n
z=Cf9MA~KUtn!oZI9OIM8GEfdhvtm6-?P%-C;ge=;Hg@i5qbV~+&uV5#;x(SldZmt;
zwNctTNBJ!wJVW+c96F4f{|w`l^g{u{WO(Xww!4Jwywq9tuvQ69O|--ffh!f>LWlIP
zs@MGV?h=^SIYqr#-n0cg<SBzvHg4TrW`>_sry%U1fOR&8T_Jl`liN}2SOqh+Y9V%k
zu{rCfn7Z%6r|E?aY~Qdv6y&J#%!BS{1NuK@Q)x##)`fAqAW8CNjiMRvXM?#_2i{G!
zTtS@!+z7F-m1xKkeKpQQB3@j1O;PFj=S3H$!^u6Up{Ihk%>8T5N|m;wktNlWyRFA#
zX>`iS+EiEmXvJxkiNjXh$c#C%cCClv98;1Z6I{cv(jy`6rc{LzuV0Y*3wM{|_;zVq
zDZ&fzVZ+qTr+hv$th4?SI?VQ%A;`-qST~~7{9!l!wdrR`wp37=!K0&#S?i_UP=I-g
z`yk!gmXAo#Pvfb!!sz;ju#s{XQjblaE58^l<$cJKOdE~%1gMD7x#>+3{&;-MuxI+A
z2=8M^W6N>nacSWkp+JZ_f&*jkZ5Ai_AeA1y#xv^vcE{DeoJwyYk47<EX4u<}l0G#*
z)afQXqh0xR1082ZZ5!?v&nBNlC3_uK{3^d=A)JU*t8y)Embq(I*2Ob=SvcIuQB)_v
zcdhi;N;msjRyu|P)6NtCsTD9&JgeD;&!<;s>eu8aaHxB0DSez4ypQLT7GxO(U-0&=
zvRS3&7U(py9PqkS{N*Q-3^NVU`0PmSs?4g)3#Os>y%*Cw2Kqwd)RR8$Os7@mNUtk|
z9&LzFP+N;?6W+T!f`E`kx7Z<K-Ov>4h%-t4wDuy@XQZ9pVdGOKs4%c#recmfB%qc>
zt86U0Bj@nQl)FNHOAXdcbL+*`TD`USXrI=xyhOa<^(_2~_jbdgqx520<Zh!p%KFCX
z3f&{VJ&(P(MXOZL5KVzTPtV4xe+MD25yL!eM%kX@h=asZfQnqR$2ttHEME49xz@*#
z@+Vwg&I7KD{&LiQHH1b_7hL*$>at_H&wt`tiiwx757Dx0Hk1}#!rG_>hS5ibFWoBD
zPc=L?l)d}4{P;CuNHWC)GD7t>-D+)(PH2k|6K;*jfA!5+bOrkvFN1J9Q4FrY8Y+ec
z*F|8_wbEx_ck@EO{`lze9aXL+Sx_3W+JZVh9n!;Yb2~k?w=46XF6fHM|K>LHU>P3~
zl|IP+t@V264KbTg2dSf_b$Ryd?;kw?zuQmdZ;D2uMCXHTaX-V<#i-?1^+__14i;&w
z3c<xCCk&@YGfGLcsd*&;nYQO*|8)%q_p;jOkK$p35pWseejW3&Rtq8j)z}3&5uR|h
z+$lF!3=phMru2GTONumHT<evdM8B-{6mVyoDO!7g>Nj)k+23Vwa?bkimC$@ZP#F=o
zxS`D5M*F{h{FC<!zM;@hn%DJ!5Y+!N(4Vvl3Mu#jrFlY5`L8?5f%HjBcJbyf9g#Bv
z(3ym4o0ro3sZ7cH2;?M{RJOi<w8syQJIF7fPK2?3`q#|-l)6k<f&o&+XcYKWg#Y;-
zLZ|?W6ldPzFS(RCJ9uw-_(JagYJc9!P8{+s+kYeYZyH&j0p9zuXIK8OZiNChmg_*U
zYsO#g)RYD9wOl);{;OLeU^;j<gn4BDYG)-?-ikH5!)4-sDzbkbI6(@AfzUBOyX3ES
zs$wYEH)J$jOZ_SJ`GN|2&10Oykhyj7?al3%!dN-p!=4|DPV&RtHkJ}l5dzR_$l=_A
z`bw(0VR*@@*gCQA)geRUqEd4zV6|S3u=H1G_tpK;8^KucGLAF!)~lo8aZs2lN|c*l
zam`#@jF%t^pL)!!-+{uj6Zh>aDCO&m8~faxe{NPD<!U|O%AD1-svKhQJ%yNova0Up
z)12f#y62e%2DP6_NbAAMtHKx{8u<Wf`H=y2z3bili)%!?a(@iAIXalK#EyCEXgDIT
z9TrQnm$%2cw@FrpAgN_=8~T|>{Cq*)OEAz2pEbD>ZqX5tKf@x^Gk;W3mJ<EriC;d#
zMJX+mGwJC)zJiv$7oEmg%0aW=ycr!m|6N;^cZ^`ZYQ4?w3lsxP%^L}qzE@E30FXCa
zzb>zOH_EXf!_te{?6A{*dxNR&JSjG-Wye?WUHj$tkBv*N3!n-wXP*1)bCwf8xe(wt
zsM3uSv@4z6LAf#8=VJR2|Je#uO#*~64DU~-=zwshc9@aJabuwU_Ie+4!7!2ml*IQy
zjrIF>Y3^2>>G>tJLJ!KflLJ1NyAl;kl4iG4M|fsgoy(v=?6%V3{~!cc08|T28y;*+
z5P59u)Q!iC0G&sA`W8st0D7bbl@T7fQNV{$BB%Yi_D-l^1wQ-;FPoVa&wc$K=vTUf
zp1keV0C#&}$g(LOZl!^WG?C|*eC=~Ysx-fZ1=|(|G!3+fUQMW`2Axff4&@Qoil3o}
zBUIC{1eg?uuWOM^B?mwppju0j9<&fW1GGFuCh98T&O5Vj^lFgSuYkGM1)k5g-y}ys
zl?@POtSReG^tYdUZ?BPhc`5%`MQoXZyy>%$NbE6+$c)Uh>wRC}lSzr``<EW{bt6yJ
z4?cN(Cfo@EgY^MCsj&-*P8%eA$KwLNwy1hO=ai=+r^TYWhPzon2LN*VU1-T3WQv~G
z-)A`&tXT;ZE`Qv%N1lyxF6UhXt%=M$o>fJSxjoRMbPGmUU`lVr03{3%92vKy0TBxj
zAz(XJdmCyJc<t1fr5S`giL>Ewl;hqm0@P*PLuQM_JpMU>nh#H{$8tca{y}V2q4B7%
z(H&}|G*Ehj7M;HV_7h<A(tbb~(2;$4S6ye?p!&Oe6MTgFO1Zi(?3;BF(+ehx)z<Bj
zOjyj<JLjKs<&Up{)F%GLdK4>)S&h@$Y=&HLuu~MvP0PTkTl*zp<l|`?jy`SUc(DSZ
zdXph)hn<>!aemnEzzdA!a07W&E&fqKB;!zyn1>RYV(Ms=bFICSejlD$D^P3Inv4OP
z{o(-d#n{%Y=_<ifV}#04mVN?chI$3I%<IUPXF(Wpg1BZpPk(~KpF%~pKtbM!S;&#^
z_o&<w^t00iJGXfY>UU|N|8L%XeJvDtuXxX1zW1v;kQyZfX0y^w6+DP=UUHe`+iO}E
zP7<do>19rFJnOjoNR-M~(J($XAKh+Y@#z5IDA)!kRf@F`KkS>N>NNDAOB-9+v>x`+
zRo0_FV*+KbP-34nIh&@{kSJ<?(yr+;r0_Wm_;RL(>9W!s&%$U9Lk2pyuIh83d$Iw#
zBnbis(O~SqtgHSJa5K|ydc)Aj50Hy9U;bFaVKuF32jxHbW0Lc;3hZ$r#{tq_8P|v5
zZ<F4<Kh@;++AdCA!y6{$jusT65uj}N<oV6exxl$$xCdBuE=G-_Nhdr`l&CB728qJ@
zw8^ld-<fz5USc2K84z3%0j~!Z@+Hubp&8NVbU!(M;yNL8pAXdkUo=>-{1GPd?t>tj
zi6SsVIW-Dmwq7l(3dC`kp`T_Xc@DA8#GcG#$&`b73YS;WqzJJpacpe|-vZ7Is0n<I
z5_6&YBS@BQvBo=%THbFA<N`ct36ykZhDd!mQrct%7@H}9Yx+_9Eb%TD{w|eZ<+yE@
z@cz=>{0xG3{cj^Vr%?*G6OHuCW7BY|@+n1H9aBuw6+;vR4R)it!@gi_Gv2nHOi3%2
z6nWGQE9*2YS9Ur>G;2D5t|^qSE-lwFhWqO)uo`nWD}vt3rvED7{u^=p8z?~>yeNiZ
z+JaadUHAOqF$CdEhD4or>7z{PrsRnO#LW7FgRsUaC)*5aT&dvc8(%p#;zmv9cus))
zO(dmGw|;{o|Kaw4_#aTiKLo0qhBGP_55vMNMI;PTwkj^TJr;mU)HY$U#;9gh*B&J=
z9wfelW)Z>I^S2+yeNI{7sMmXJvyk;Z+yuG>{T>y+umQkPqUXSI-n5lr^(ACU7F0SK
zw!U+MMFo|t)yp5`ROs0@^e^I1?vxS9W3YvpWbG^|8PttZqz%)2CCUxH`=_oK{Z8OL
zO3cycBwU5S%lo|3Cbk~)7zVP2wu{%=>5Ru{S&Ca=)|oWYKE|qPJBSS`@5T+H(BZmk
z_ziIlM^j_>)b1mfj5(T&r2Y|Q=b?~k8z;bq7ERJw+2F2}D*>s(E@&+_rPbqU5TV((
z(u|aTfDl2IWK|PiH^xmTMopWudzcj#FgyUMtl!X|gJ+}9Iv(XD4w*<W`NgdN43P7o
z(8tdjf}jJXD7Hgvq~}zQ!9pBx@c@p5_v)bAxHem=nYp2vmh#cz>=LlpXWwVE>}sy1
z#?(DQ7_D118;mdxnYac`7{dBu%g;K0?JEGfZOQ&I^Z_1vuPabSVZzY7FxHa8?(ADN
z2qjE8lKnkC3&Pq+3$iNGF+N<7&H(`xU>cTi5SMI#nl#7v<{a^2C?t`7zZKXeUe5<*
z`E0lsqek2Z9e07xFdRvn)%Ds1UhpDP@5*qHGS#>x2`T{XLS}-7C6-2Aw<VX}(?gSD
zTsmGfFVTQrpRho>7Cmff)%3ty=9V<k)rXvm)fxJdAd-+R*g@Sq-~q<~miYlO%YC7$
zVFJV^P%$jfMh$%6zxYCqzi6B&o4{7-$c+6b;PZse71ms7njz9iX8Xm*H|halm>Y@T
zuq;A}X&!_5Z}AGNY5_Db<ls2|_S#rW-D5KuG;bC)Yt=h*jO<jQivX%Si-(rvXUMsf
zhnJp16|nSXlqZwax!$$=vANnV*1g@KkH$?rpZpP4>#6VPSZX=zFh3@P2eIBPV9mJ2
zxG;&(Z4u3jgk#*=XVhYC`V){yb%#9EG6=>ZOQKs)GyEzlwe}+O^n4_I0Tg-m-p=Yq
zK|OML^S28+!|(4wsA6nd2>|23w}4-07)&jktq}!+9z^+<r1HOoKk=ris9Fms;zC_G
zrYeap1r4-CSOg7wy7e%lXE(R9*yYo|2`3R|xlB(@2wZQ=-zTt#d04APEnL5qqNQ*M
zbjPBh>$;I=a364u&K!&2KkN$sj`nVI1x4y}0_ye`<$K&-?lpf-dqmW_*-)DWl3*yF
zq|Y5_Zew*aBl}}e*fIn~;7U~s#r0BhtR8t$`IJgN=g^>e76HU~c^uE)&Rdoz)9e3@
z&hLS2kptEA5hptcXy`#HLW=-r1a$*Vs9oJ%GVHy3kF5n!c=D=wq)(H6DklD0vZEw<
z(l)HGPSrEzc@N?8%^|f3Xfpig8=@JMw!7cnw9B;?fLKPGGJmuB$d~I&UZAISm@EAs
zjXt>t`_Q56h5c!W^inoU48b!;;F=7~;teTEp=!KMEUfwLQ~E!ByE7RGd(^t$X+K{9
zu%_z@F=1-IIs&LvDrsKB_lmxz2Yru5iBd4~_lzU28w4btt9$krzq>}@)-uHMNCT$x
zght=-HvIV>$^VCYK*~g4`~S-IUf&+t4>L4g?evcW)63pcc*>g1Km~wagn1C3N`x8R
zhI3;flI+{_vhTBc#G;Xm@60n}r&s^z+e5??yKa_eMV1KOAb|mi4?U&8*-D^F8`y-T
zCF6Z{*guI7z6L>hw#n_e;(wp9{mPNn7C;XmB9jQl_8g`;eF{Gb&6Tw7?Qs2`PEPQH
zq;A)V;i5~D$orrZs(?-(0Z9C%i2Z4B`5M9Race>ZT+0X7mg*Lnx(#UOE(&arsOa)z
zg);py<!q6_acCG{y0!$~dpHl0fXh<?t!FJjnQzn+hE};V+4hy9`r>Q3p3BD<E*yp~
z|KW;>&xp8eug3KZsppUpUU7JJis8Qu)2*HKaHsh11<x@ix->GRN1LIFaF8U5DNF+g
zz7K+UJ9wlW5XmptwP(NL1$ynzK%89%Jg@B2K?=wz@S>^(y$Q$P^lm+8G|WbGB2J+C
zP@bYj0KC)9a+=li6rcTAaw8_JK{*DXEZg?oHny(&su;G0@hJN&0Z2`qba;V$7;5q9
zRsd}Qo&Gt{s)7YuL!iX+JO|KC!2RJ^;6wBSA4(Qk1eh0uQ+b}PhJtJ70Zv~B$T}A!
zLgvQ6NoLzLKnl3efsmiC#y;2g#zP$ToesDiID?qF=60Yu9|f4&7lZ)jC~(nx-`ks>
zGYx%z_$D}LPKIl=vHaQ_G#6;MH=)|ubpRFm!2SeYkXo`#gS^f)MZ<KEU3c{Wgg^Zm
z&;)!FAmGCnpL?U%j<ngrD;B7eGKd|vSAiv?DNpXjAoGiz0vmXes5+)C`Et$cbFeSe
zmq2C(LnVo`)&vyRwPIiX&8&F7t#*e>)PRJ6F*A26l*rQ@Q_!z?WMeFJt1K&h3>(8j
zIvFV3F-AGSv}4_mlJo)kXeSq^_sc*L+#3;J_fb_#Y>7=46aShF+3AH~Xnl!&1<Gi)
zm>hx22gPOs`>+@>#x11MZdne<2AI;(8Q~<T3cE~@LkQ;#(F{3JoLTuEeCDvkTU*ds
zMZ$DV(Ih&(bLROn?GpDHO%1ADd2$}j>HnNT-ZkQfQ}5(VwVQka{O@+zaBYA>0BkAp
z%dpm-`S30eASh=sY6HfUw!*PsT?YixRj-=9?$qaey}ba*Ajt1Y3aURCSZ7(3q|dEv
z0Qu%Mz|72=jP(=3)~>=n&5QtIWK};j955-t6oCslB?9QMY6eWYFR+2z4QhdWp%oB|
zdD31$XBXt_&Y^^^evi%iL0AKk8QzxI=WG?EdQn2jkKLeX9-7Z-+jo9g0n>U2071ah
zjVqw0FJ^y`{6q$h`89NBg8<0n5mH)!4U1ugG=eS}&289&h7=)hNmJc>@p=M&J|_#j
zfW<}e>77@ATzR^20p9KgOd7LPqqZRZzX1a5%@=yuXLsh8y8S*bX})aoepO+=ezmmk
z^P?9>uV}E=fXeYnn1Q0q`QXWHAhw)%@XP%$@)*fNGXFR&i{fO%4z9NawNsF+VAjoC
z=P4=08-Qb;r^ic97r|83S=*)~t;|OBL`PipzDi@qT`P!s9aRUyxAa@dJUQ~)G^3b7
zji)SgePw_JWPD+u65l+={cXY;<cYv=Vu21gOYE81aU`}J?afhYdz34)AvDv59K9_r
ze7iUERy9Taa(W<U=6^^)b6?oEe0rN($$Fo#qc0leVCQDk<J|#4V{joYTcdga7^xVM
zolu%~S@bz9z)Owq=&M^u8DqjdPKR!WQJvU<WK35eKw(2|`3sg%szS1cQQzCkWsU1_
zl&fyy8`Py@_5uP&I-@>{&KY{(G><_i(g~*kP$Q-ju9+_Jjw+6a;%^b%5@ksRM!e8D
z0V54l?qpg0mJ(wGwV5}D3t0dl)xrP1_5hX)_X}XnH@7d?5Ednah?*xNC6siozp82m
zTUERPZ$_KayZ-|uq)}JUtv|i6>>F3m4I4J>Y7Pf0o$-OgRxHfuiot6;z&vtk+kSlV
zk`zrDHk6V}hlr|YA}$5p#9q})eh)xq6ntZfoY>bfOs2%&OCqJ1;L>byn*jKS3xGFA
z+7t%fZsq^M^{wEKl`P(!yC*GPB9xow(FV3LRoGh&A-sb;>2w*gpr+4KC)aR}h{tIB
zc#)cuqKzDJ&KWY2FuXR0_xPSvlZiZO0vOs2Yq{s^-#xn!sZnh?H$?;T-ii{D$%f5?
z%`xd=og39>tYdPw>(pwLglpWxGm!xQQ9p}hXi$gSmNokeYKwuyhb-~lhO?Dh^t}@`
zHw=MG$T)3%rv@B#ODxIMy@&WJ`Nr&}2_1AA?$F#koj7GRp>#m>VJ$K;N@Ea?VJ;Sb
zMGgZt)$@rv_Xz}zNoESspM*WS$bBUMHb(VC&{c)iqZFFgkRiE3_Z-Abh~jpkavKlC
z8A>_*9vVsfMFmsmg9_rq9cQ}4PbViD=DyVmd#T^7?EvaUAz%HQ{3^Ee&T?_8ZibDt
zDp0xufC_af#+-^DsL-sWE!Ux;H(Ld4jww?=4v#3|WeN2G7sx-`OafjP`Oq;Eqca`*
zZMX~PDQ5992p%ET?|Kt{N#B09<zK^D7Ik=Q(A|&E6PaX{a`w*Qb*)3K_V=yLeXv~z
zuKGPdenLI2G%a9<7En}&?Z)q#?SX6+0bgZ@(2Hs^<VklM75$ywaL$p^0Tm8*u{Ouo
z>XGK+sXDHBccPzkK!5<A2FIvOy_ev>zzsr%GZ#SYUY!q_EkC?6jYZdkAn6pLO4w}D
zdV>(w6Cfizj}G^wMZhe$xkx!tmIIGY?L6*yi0(}_&l9hAD;d&Bn$9~KX4%Ql6FQDl
zUOumhOfdbv1-xY>x7cD8V8oi{O{FtZ{zIlvuZ4bxAq!C~mU8NbfS#WISr`D%CB{B?
zSs}3X+E!_t03Y~_)!z1FD$6KVsee13PJ>Jo7tPCrK-f4MBn{3t0f(9FNXF9rfEBgi
z6zn)cRHhEsz9Z9;&oM|HgznQ!(N%MrD9=BLb*k_?ct=M7Mwio$Q34ojTsYl~-br3S
zkXZZ}Sc8afR#(IqM^S3=01|A@J3+a}oIxY6^r<rHJOz9mc_)1qQqfgq!+Zox-DbeG
zej@?1K1^_pld7u`RqHNUB)zZ{eRrCO^5tJd1U-PmDdOWrW?;|Emoz_4i~fNDQu<NA
zba$9iFyKPAGVS;QL8~p%<3Pw^jhh}c7cN>QxW}wvrG=_M-9vg8N^UuJcYJLEyQxZT
zRKi#u!vhVu26_;@;dj$2q)>H!azj+421f;L^DNhS3l<Fmhs4Uab0yFI2oA>o^!oyt
zzPbY-t8T7?Qwlhg6mfcMXmU^xM#uw2oLmbZ{LxiYABu(TaC@KGi&8BF8LD2d6W<Q-
zlRXHjYMe8S9MUT7IZ)!&Y9Pv|QGJHJr2LpYJ{U)rE2GQS1E4n4?a&DcW@C0Lk>ced
zA3Lzs5O}cIuE#d@iaX@*pfw_37I4^}0Q`gtUV;Pxn&b{10mjnG_1OUU6G^CAfEyp`
zT(O&w2LIbC<&N_c3iP}wFp}Zfh!dI7^|G0<*Zsau%|pcz+B1OH;$VFIIMWp7_N-rE
z&brQc<0A;0hagH+mGlEBmaEVEif1NUT=`XJHJz&Rdo2?%Qq;~=+~<H&{{$GNIv1gL
zprzy=YMc_<`(kqeOiO^NGuL-zyYt;az^C>t*?WMNY>{)^{eGPV0C>OND?^l)YHh&N
zgDw2bETIT-k!5x2eSfa4r@+!}9KN%Z)im}4Vj@R8(Acmg2GD^&=B?gPidqiYDFv7d
zrd={$m~irU3QIB*J*b>}P)^Vdw!Aum@Ci+y*tq&|N8h(lWtbNL40^8&Pkfl9n@b}?
z>IQHaA<bNsvC%r5G<5*~)Y%OICMNkYQ3oDJVx^MiH0H}y^~djX0aYJVQ+fQAImGs5
zL5uB+{OY&0Jzw(`KW{d?`!7s<frGuKy)UtiF2vhD1CdO9?<{x6IxubLV4Q+#6q<ot
zg!=;are~gX4P~rG9U)Q8@|%Rx%UCWLqs*6}GII}wfD1qx)BceR^(0<ePY}^Gbe^xD
zEu$ZO21TLGy==?ZW$IEqn@P&5%N9n`h5>{uhN4Cb^>0hiR}u_Z?JRLP82`%_JX8Kn
zw9S|b-v5Sgp?r<G1hfb{>f&iFVI3p&#{vhwMH5=9LmzSdGXs|d#TSrd?q~2A{qFm~
z;}}Q*SBPlJ6aR;YqL}aiDO0xFXm-ti1N0Bk=AQx>zy`Rw>_3?!^e?<%jg7Tzjj8|H
zAVF#HUabI#+;8R#{pd^tT*GG_C$m43Zpl1K@Lq;aYwF(@FnBX}EJ)3ctD0H=^rRFM
z=-|C6`VAO=bt_u}41>q)BHrJFhX~#)&Abr)SGUH<LCLb!YA@>d922y&9?B(ArtSV&
z^NJJv47U8ceqwj)`>UP*XZZgy;r}!I|Ev#S{U>%(Owh&nL0CVUeFhgSc?R~%|2aV-
zLq;Evx9xyv%vcN`{ox6F4N|F4PpZi`dIGO!A(DJ|tE=EHnElUuzON#3v;VSkPxd=6
z1mCoI>!7reYRV&<rDidZ_!CjB$a_4}gK7CiUFQjQ%&x&xm?lvH{5z#R@@L_gg2bO2
zFi+p$%>ReT02U8z_!Cfg*T{%${>!*26sW=K$%d)?A3ybTSr>x`Njq7n?17y943I_a
zuh|#=_9&6S6J$Ep<amB91G~h{fFv}>I6;akO+Nr6<kO>o6a7&~EC8qjAHMgW0KYvs
z1|GCj>B<PU|G#nvX<t$WwAKTvzv&{OC`-^%$iv63peob~_K{OcXZ%@!mCTcdwT+yj
zEU?89EMfq}+Z>1TD8TSRY`(cKPc_gX%4IU&5oJmy$G}Ml3NIO+`+%fOk)<!O@)9Jz
zP{avx7DtZ66L9E4;L*khXokeI`Da!^;y(nuRr2`<$5W6`B1km@YTOPSH~_HMIoq_1
zvb0F$>%agV`%ZYklbnCCkDw_@16?P6c`^x-=o4^U!6GOTL22$`#+Hf~-8IC&=J%gr
z&5MOrTW*BOKAI<u+C%REY|k?n=2>kgLQp*;fDQ~z9U(dZaq_Q5LAisdaRHD%L5d*0
zx<O0v02-bH008Q>n@#&`W(z2VE)V{gQb9YY_rFT^1pDXBG0RO=Z4$aK2Epx(hu0rx
z*FXcW8r!d<Ir=~?0PsL}NAY%7MOr}eB+s=P1ZwuQ&%h@1S*lFyfQYI$oK}DOT0+qd
zD0eycUMy1w9OJO;d&`IEbNx+^`fYk%MePtRklIm%Ldl=^KR#vtU5Y^elHi9u#)$Sp
z%P*~pbP!Yn8NQ<Cn~n4Rh#I<2<W5p!36)%cUGRZf&@u=(k%FNi^0d3p9f(<>TjiDI
zo^67is5RinH`QH8y$M*3>io}npfp(^sW)1Ps^*;Qg3W!9X4THTP}9LW2ggPj48qcj
zn5bkG7!MkxR+#+7M+3|6-n(;(|7^E@TFFT?AbLq*K*)oPE7xnUd4QBcUat8)(9oMp
z2N}Kw1N27VWI_ZmF?--GZot%4Etp<{0}5sUQwC;+WdUXF7^pQ2&!2eJ7hNUk-^ZCU
z0#pG=xl^Ckg{x0V$t8{C{8bQ^%!37OfahA`Vb|~Q0~_~)xuNwCz_Bg0``!XhS{x(!
z2WDqZIGW>!SBjX>(<Z>H97ZxliQa)>2qzv^rEF?FYLp(8BGIUb2lfHKo5ujjFab!#
zy3zv-4M24ZVY&rdeeP3wWOwtepJf?@;7w%F2xNvy#dlkdTetnqvnPXfPj|fC2NChT
zQt&sgA80Kr$v{dpW&aMyNv{u7p8McHg=J7?PN)V1`Ub%JfWvRh1ume6_%(rj*tImY
zETxdCWnBaRB4Ownl+*#e-`))}goXv%;posaaX=6;r0*;x@Fb&9=eQg8u=YZki6o0Z
z>JmU@l?wL&{{v;g7B&}HP_}L7rWZgzan}_vmPH(c>45rD=fLy#mg8E5tBA)6Ir`Cl
z%V%8_lyVI=NvD6HdSQ;iz-v8MZTKCO%xIq?Dr)gQex^bU%u}+^VmO&a?=M&r4`9JS
zQSmc%_U%W+^B^b2<}p=EfTqIUUAIulN-r$UXW%o!PXOJP{i3q01A2ggb@Dbddee=#
zVH6AP5~2eK))@O*K9FO<pfw*j?#B&s|Bs?+0u?sdSX_j=Ivxp*S#!->`c4B8R73GR
z1t<t4H=quM5;^W1tomB}q#i&5z3+fUPNF{rn@a0(Lph|T?JF2G;zzi&G?G%vt$Y^H
z&%odVTG*f=bW>#HbpxSU?4u}q4Nz!B*|$>1mpyrd`Jkfz{><8)p31J}4ZO)s7~AH}
zlvgf*ws+HLQo~%ipMnQ1O1(F%M^XBBG$v5sw7VgRYO5d#3X^Eew0##o@isXWee&h=
zl>F?0H;V(-b)zZeH;K7d6ppk4LVXZN#b8Gp&p9mwRIGG9cf<Nc;a}&ve0qA85{>a0
zcFoRUL52_AkNfvCU0OE{d<z|WJr((-JAwCx;M(sjYK^Y2AV^dK*-u`^(dJP7BYD-W
z;{ZG0W1n!U9RGrG0|2LO`PH8s0M{9Q<DNCjKbb-R7jG)Pm6*QPuZsPkhJbBBl75VW
zy5OOSd@-A;x{LuqN!NB#xUTF<lCqq^6>in!BkfI4;m%%BdhaPdihBh;gJoGZfENQk
z<3MxAX2C=PG(w-Nd;H(>gX^=+66`oL<-_@=e?ewm8Hf_Hy$fRN^h;6b$e@%5)g3gS
zh+@d9R0&3~Tmm1RN+)sy_|_~Q9*A*bH1M^rMt5eio1YGPYknwiP9QoZg9V296;uX?
z{QzsMW$h3|Vv3_dMoP`}O_wdT1SR=lggG%if@6Rf=i4`Jvs|A_cVI?Z=XT5XN5}Af
zm1Gni0uKVVfWDwg=eKvg9@e{2a^f6J<OuIhknQCjUn~PWaD23&0I05FN=iwcgX4Q7
zu%2&y^_=rUB+2b!0a3ZJ$MqVjFiJ(?{Nzrq)phzwirr|#nozV(o&7~7`wAhOxx(gk
zmZkYL?XOUC!U$`0Qnh+oGl?F=v3MNky;&mOK?G}eAa&E&n?U~r>5`f*o3?2yM2t?2
zaNTlar9Ooxd03uO8k@Ba2Th-k=v3w&?=XFxzUMrs_!=Ip8ZSgrP1WXGp4s48@_;i*
zenv~Qi7rStjx&H+T<05!#6VmHDt$h^SZee!A(dR9K5cVE6XL5`Be-z|h*(i!)X_CN
z-%G8*WOpcu3Qk<nski}T^%^*mfHXIaMd0>)+$rCD`Q;BC*Rv?iBzjfOQciTh6g<}T
z+@=(71N7>VTYfP1+RR#XA7!fc5n#3Dfn=#yN(a=FTfdX!vPZvSGVI8cV**4N7_C4Y
zJk}9rkwurq?(&LSq+VN4*x`t!L|{crn`RK@XHmO3jP+e9ea=jz8-se0?8fMg49+zP
zgjRpQUjp}$*+@;`cswmUG)|@d4q7@(>@JTpgxdnASY&AKF!ylAaMF($ww-U0?>%|V
z^Lg3_U?V8NQ1J#8{If{p4iJv;b_u{iNj<Me8`7tm5TSA_f)H}Xa1cn>fr5%mNKk);
z0J!8L$Z5kyTT|9rHzyzLZgnr3-2Q{_V2~(tv5Fpi2c(<P)q@;H67jl7`r3gEt11lx
zI-RfRBwvB5i$olAzu{n`zcHYxhoRbZdI_k8hF~IQ(Nw9kou_~RZm@EI_7EB-9!2Vb
zql%Jb4hLzEH*)efZC{GV5QOvjcKF?C)`U_s?H!Ab-eK{5L<8VJrb{qa3s7~J+EcHA
zeTGb;_4pVosoR@tkaUkhr7mDukY_kFs6S_w4TwHwFw%wdFGN~qui6IUxyGX*Ix}A{
zNZ{)(oUcvmG3)nMrb<(OakBFnFby%D{49;0nqRzJJa%A^#{-(nbiigM|8Jl&B~$9Y
zV~Rz2&U<)FpL0asX~ZZb!6nV5K@>xdfr!6*V4gX2Jb^85!HX1&%R;s(5zt>@9i~a-
zsdVR^<fYTmRVug#w*3qA*sRg)qiT3KSf6g5wHkg+sLHeOFw9<rs_iXh%L2e%g-BFC
z`iZ>gA9IE3h$yYhfD^cg<(GhLiW2EKwXHyi))pM4wJ8(47=%#w;X81<xC|u4N~Vks
zK<y-k_rYRMrMVkuPa?1Vr{-!A>pANrvFLU%0^wwP9T9cE883{KG|pSnA=No38W&tU
zx~#`lSCJA<2E`|GU_Lt@VLjhUSrkCb4wP^nyyt+pC8p#`LGvDHFCA*0&(NBN+;#n6
zQt4hJIGB|npqeD={(rIemSI_DZM?UX($Xc;H{IRc-Q5iW(j_h3b<-u?NQjgI(%ncZ
zpp+sif(QccbvZN7%shMV5BtM=yvKXIdp>Z?Tyt}C)mrOZ=lT2p&lF(3;Y7EJ+`=<e
zp;81BVarw00Ygr_kfz5A1-P3Wr0aGX)THOT^gP&tp{73sbjxa*Lf$A;<TGi+zz#rp
zT+N=9Lf(V<Bcj|G-m3|bh0j3wkF?G0%z%^64O-(R%TNb749kpL3%}FNrq*3a_)IrY
zKd$z|{G|WFBqF2#Gk7<`(-gL$;kuIclm&_oRs`mECXuiOsnl9Gc*q|vm))nReL8IE
zw_}tsQcs&A>IqQ)J>8aYTNH+=lXP+BWbpp42+iHC);wYc6*RR`q&I_JIkzoK?T~l6
zc-uve8&|MnHpR6koX$An8SU)xmD~Yo%ZySJzimY}B~JMZPz148KN8H?*QDN*2bnif
z^>Gq<rog=L&qD({Mqc4xFe13UJ5tmZ&XTmD-MJ94(^s_RqswB!K3}o9i<!Uios09O
zTTfmb(GUHdhm#C%-hCnb@%5lL^p;^`Hb775;w$_u9TP{lKdLr74V}OLb~!z%0F)QF
zNLa_TsL-ZCNs>*L%$bt94a{;@rK*>t*iih{sP=N=aOoDkNY35+!kNDF8VFZ)x~y0P
z1^!lp<eeQW1Jnaz0~4_X#8}o}LmE_yIxmRslJfid(_kv3Y+M*d>u^QzFz_(-`fOye
zI@!-L(T~{L4ecm5>%x#3J|mFQxzca|_|~lUiyQmq7^+rcZVft`{;R6<KvQm)P2FvD
zsO&oS^rA}Qnn|zbDr#pv{7}SxFegAuaqGG6>qszfhiBo&QkG`<I4FoomrxrR^{@VT
z@Y<|@EWx$t`MiyS>FXm4^AHD0^a(WeqWGsA@K&m?me~}(UJ>}nW9hk{1m;BvcKVn6
zRoe|w!aI=l2#(}&fpG~dd()~jQ!-=Y>(whohJMmzEnbEegnT-;W&*{T5fJjIlO$-$
zc6eAzCB^ud8_HG%iMZ2GwrhNOKSX50I(<?EE0m6rrN3V>nW-pV2f3S#*Qa)X&0QHE
zIr<l)sjR`zejsS}Ao`jrCfBv`TF8LaFZOoqZH}&k(jz8sU4Py$*o+V6l<i1v5zajy
z#7dr|6CB%LFN}07i24Prnh1iq7VCLS`+j4GZTRv%M-Nfy2P?9SnVoUv_|MF#=Op_{
z+i+)$^7+lY7%)k>>Ews%TOTux@R-nN%Rk=kDHIjabi5MXt6yl&3!|9pT78sKDVfQ?
zKG_vw`=TcFy!)*koPwD6e2+w8@8Ad&l^-;WN&rC3^L3%cJHeqLWxcK|X)CmS2O*S}
z9h5t_e|a|9Yq$PR-t%TjUIp>BtC>6yK*Z`CN1q<HSoYMYhM76*oQob?dMIbbbadt9
ze_@*xXVQLIu$5S8NEO*_Ai5||3;caXEUmQB(R2t3=2-bt>&<s-vMV{c%cb%hxx%A^
z#=LB>o781dC-_>cti|ZWPQFv;Qy$U@GKFzastB~&&41%^=yWyg$PrT!t}(^@+^Iu8
zOeXh%0%SxES*`hwhwEYdg=)<jb<HnqaFt&@ZnAQwgOP;#BXHg}rPL9R^R(D^$+hl^
zkX!>(z+u!(C@4dj%86pU+K3lds%nYgsp2og9B#wOYRg@6@+bSXG83BZP_?Wb@sfs_
zZ~B)ITIRSdeiB|mY-vY530mtX1>q=-cML3#+7V}5niUsNRBSU$W#JwV%Vs@c+eGYi
zJx4Z&_Q$L-hxfwO1Pj$7T7S4jaR2yt+pA|>hj4bN%kj-&`J@Ytn>tZ;<s^HrrdB$}
zh_%hKh-}r>(L*SMUO`%8gbzhYWGYl4L1gQd4bSd{N1NKAI6N}b$0QJB+(8<CtitWF
z$SGS?;g7>Gi(&G_yH41OGgdi5DAp&?anWt*nfm}5?|YS^`+CL%F-+X~8hIn$Z9u`U
zIQ7l#*afeEq%vNkZFS!U8S;gZO_mxtVQQZ=$|@aFZEbw_zKYY>Nz1}n$VCJ~yT@x%
z6+>$MIkQ-Znq5h5<0iJ)k+BPU1SO5R0dRXYS<!|lLFWpwTt~^>T5o!7K^p;2VAESi
zcd}M+D>d#D#*0F>g88v*3KQ>f)@(;$t8p-_nHI2;m9W`d8TKi_6aA@jztU3}&2{-}
zS4OT=!|4zCfvS~5AUiGw=G3eBmMeI+TL(LXS`jkC@%ZYiv^~VA0{VCp;Ryj2w-C%)
zCZ2pMH%(3@nNKGe@CNm~=XA(E35^I7&$~Nx`3#1BJ3O4>$G&a6TyKu@r3b4;{g%tV
z%F&8&p`^^q_eQZ!)OJcTl*TN9gCs=_a>8iSZS*`QR6m0f)0iLCPVi1QGrh4_cc)iM
zZJm?5GK@B^Xcd?m?uoB_%`2*Fbl^&JT&ao&#E~^qM|n#JxVxYV@jY%)$wo1P=K-&Z
zgAPri&kr62Gi2@hMYfklih8Ik_Y4gsa=9m9WH3P|i2zNTg@Gx)?&=oP&t~ZAwxS7a
zHdcsT7-xFio*M1f-Ud|4trzeGYC(^FHUn0o&=&p<({LB@iIlIt$#s2i(<53hIS63D
zOVXl>Z;V9u>w&tf{ZBrK8ZV(n-r29CR$EnE$8#LjyJ?adNI|BXMMM6=j47Z}G}?>;
z6L`&gp2|p0Bly)x^<q^vzBheJ)#=DdmP;vS2)Vav<(Xz8N$d|Qe$H3K33AwY5x2p<
z+f`MDF+-P?H7k+-6IGr%T(w-xg(VTICVGj0)x{{hklAQVncvWXzSa^f19BBMYihWU
zzq-FX?)vCRitnNL)|7+rwuaKQb%4Ap4K2vE$PzG$5{!=g$hA({B~y*5uT?uFqFd>U
zs}|k3Js3-)jZMkRpS;lF+)BYm5OOvkse?r+F8V!4`hnDEuw4(f(+8GR>uCq+8Xlau
zT-7;m?XYi~>o<C?(_+Wb4aVd*NH>5{w%RqBhjY+V*iV}CBK81PhD88L5;YAT>DV6s
zXZAu81FA{fT4kr3B|_#TiDtrE%9cLKBqvSze57jViT}#*hd~QbF5&bJO)}RVri7Mp
zeaPWUi>xEmfrmB;mCl(FgE&BVWF-BXf6U<vwP5omeEkdOYu$7r(V%^GySqXOTjjx`
z{wmXACLKk*rKmTC&!3vp_s30<b}F_Grn9a3`xehzhs#pbjLD`C)7Rn@TVJ?fr;9&Y
zIpW=p>dYHt9Nr>dFYtWG87%Id#K0-F=>J-CiW6ta(LmXg*b0F@{w{h$UVNrfT5cS#
zi5*<GzI!JrKVGCjBsW;0wcVO*@N;&@h!bZ`f@1kZ-b9l-DuS=j&g}uaJPM}(0PWcJ
z8qozu8AEoC(AJ3;s;vaLxEdjTlB1GKs<k&Hm=^w7!R(Zul$oKHDUuL5kd{8=LO>se
zp1?Clm=hPPBR7}{GQAz;TjiYMeW4c<7XIcVqoffTLrzEL6E#Qt469rO3%D&&K|8T~
z-@V6(_FTyIYiwG5y5W_<ws_7NILg43(cHxYci2G}K~=e3-$<ace?*>1$qxbRd^#Pg
za<Yw5Y23*dv0pxn(Kp;$LYr#qW?Tfjba9>|VI}|`Hpd;{VL*b}9=PF{=H8?9gocBD
zevoA`UI$k_*Zbg#ed4uF7?wYv<Qa`E`%G~oL_OARjCNn#MjD296>PDdM}6?*B<VuH
z3oLro{6szb8S9^!Z^zqUhs;yGR*AfpeCwvp(13(!rh`^Kkx@Xr;2Jga)hdXtF~m{}
zTy!qlB&^2A=*J7?q$Ae`yHo5JT0`31#&}B!UdR%Vo|24b?C;^&gTRvZiK((all_T1
z5YW8NNI__g_(kWy^w_fQf~j}Kr<1(L8Mi?3{^YbQ5pd3_%6mnq>uv05)G~#$U4Z^@
zW~Dq+W)J&$x?Z3t{@KvUreXhtwv06F$vDj{iwGKi?Q@kGy<-8F&K^=BkoF94=#N>}
z^y^6XWoo9v4@Tf0Vw~pck0A5R+^uYu<TWVClc9-;S%XM0Lk_afYPB_rxt<o)=u2^3
zk){keLQPI2JdCNwOzEys@&9=};`&I`oL}3+fO{uRYUxuva(IW}t~ex&`cY^dpz_FI
zSqN8@f3BJL)kzNsu3M`4Gkh9t6&!g^R%20C<5;cAsIWbrHJINjCk(E4W?4naWevl$
zu<%yw<i?tGU~+MM6^k}lo<oM<lpf>S++?_IG|6x$EC*d_%ZO|}JQZ~{ajwahkQ<FP
zQgA!t>fr?3#qy?Fs|*!B$#Bmx{`*{v3$OH1@k$&)v+VmN=S#WF79AFraHDQP%f4F?
zE(58r(UCK)=RBi`=Idfo3t=iiy~#oKG}XYZ)^;;9mz`XGGu^rM<hppRreN!4dl%(i
zxV5;Pe_5&r=U(I9va-pMzFYsFV!m*CkWb=AtDjOXNkYYm;A`L5Y7PPZ$qk|{x+Ny;
z9fWIuMZNRxNLz(XuZ3i01on*<|D$6pV~;Zzk~QWacl1`hCd?0g6^Hp8BnPcJEstHu
zE9DxH1p#fbol9NLC6Yoe#68&#B`(%{fbtm;lTUAwA!aDG9u<b`;Rbnc3^#GtcMrvr
z^y4rDeQ{(fa@EHEfW7Z5ZaS52Ru2-wX>Zv`qfNN2DxT_jE}WPnm4mbt4+hma$g6r9
z6kMy9(n=qdEXP0IE>v;UdV?KV=Gm;uCj|c-O5o3|srF!qJ9Zo7qKUSX=J&9v5f=b2
z8-b|cIdUvD#QYbvW2^+KtIBKNVc|v8eewz0bK?s^LZAD2Tj&HIc5L6K*w6`?LOW8|
z%6^t-*y#CbOHoO7YBaI0wiD5xFuI4X8WTZXNCd!0AZfi9uO~Q|HS|4^z%t(`NPj^;
zwFbRejMf@UUy7U?d`;YYG$pWFP<w0ov)cSXx6<h^<|lii{54+FZ{?{K!IrBuT`8wl
z0L?OcRwDsFlls?elner@eiE8vb>XE&WQ5YvU|-=lx%|%Kr#1@e%wtx3qD;f4v5l$+
z25r0oXz6FOGIw|Nt?sFsj(gpoy8%i><vrkqEjznq%1U_xIUIkfj7Dw}_4MiX$vF-k
z3~i*be*^ei!N6_Zv7ZtKut%|3@>*qeXgPzdkmc@6s_J8~kr!w%na&dUOXg3CDEe#Z
zTb$JEg$UqoAMS}vc6J?q=YjD+x_)DmOUOz8fHT1bQ^W+g?^$f5hwmwJH$RILz6%hR
zJF}(5w`JH(lN{apLZ#-3m=^Yt^`2aLSK8=BkTv-P;lW9&WXU0JL4`N+7uj%2R=Hf;
zU44(T>9z0MvUMZR2J*9EnGhH9bc*Os?&dOz!5eD?&sY0d%MsMW=469s1V8VY6yc<J
zgKSdD%h|e4B9QJa7K{o{u4wh2c@>^!Ijx3W(fLBdi>-X1o<gR-be1`kgu<u)9#S82
zcAhf-WdGwkw5h0!QAzB#v%<a@Bd5I8txzti38H&SK5u!%^;nNQgJZD8?#roq@~U9Q
zyYz8vuA>AmQ1fK?$$*ySQE6-yLC>@M9?yfv;e31WwEz_kWo%uXvMd@t9U6vh&$ZII
zM8}roV~`vmpo-<q7GZ0qM<=$KP8$3j?M>0+NVrWM4}E_Gc!|c7&~uV%$S84<=hjE%
zlDqhhqvse23Zi=WiEY)p%JVAp3ciW4`!|8e3}%+niJb($Xxg4$+4@<4M3rDIfwK=N
z)HHe$b;_r9;1*X3-u6tjjSV*?#D0KZz<MWXrL~5GUxbN28BrAlCvph!Jzf9U$?I}F
zorX-yj~J1`jVO)xO%mzU;OJGzftaw|I=vH47D1gebTnJKPrwl?a$U|@zdAvOvD@1|
zMhiCM0?(9}w0@?Z=5I8TubyTZIfiINj;B)iRc$J(Ny?r3t!G+HT7T>>fE^3?Br#WZ
zk4NCkFcQ{8ltFg4M9E02Qop&fSu(`0>q~hrk1DEAdEYaC)Xu*t6QB-%sT6jr=m`-g
z1HFpPZjCl~Jv&JdF!IhDkE#=z2$U-&+KouIt{-@v#zwkIhvTI-FUOSTntQFagF`&X
z?2rb+YfDA{V(Zr3Ly2{i0n1JLQi0spo`eqsZQtPsdE<`p7JXuQO<-P#JB#|zaKp~a
z!7GM0$|$iSZuKnElzwNHO4%sZFmYJ671)g9sMj8~r(-^uV?EA}%%CRnP|P;9P!p5=
z`J~Omm+hD}9?Ko1gk<n58N0dCnFF%O`CnM#%lnqD;HC=gPSzS66+o`;{;f;ln8MQR
zOHc9b+wad$W^>>ceXx3v)M{D}tlG1$K-DO5o#PYAIBY_XgW_aTDj(E|hkEm#t+wf~
z+Wgss0ce^}-}N>I%H!W<@!{8=xmC{+F0NXV65Q**%-}*#WDvpya*0J7&e{b|0jCb3
z)9EHmG}RC>)yTJPs1kj7wCwr?B<vT6i;_g^pIhQ*dfE&a;G;#Rz-|fzNwQg^f4SG!
z${AZumtsD1CxWeD8lxtTQMcNjWugVS^e{5|(#SnB;QUy@Jf)6Pd*>(hy)K*)k{Acf
z{2cez+WKvzzV`|Db5I*$CdneH2$ntymHejhHdVVfRm53vYoQr)!A5Mi`D#;_UNgL{
zkB{*^qkd?^92}rxWq36HfsF<e*8YM3nail2zWev2{C=ZTHAJk7^m8m;(O7)U+rhC&
zR&R4~TRRR)6Rc5~_GUfPoZ=%|XH?(D{Xr}=(QoM}1F}F5kOc-t*=x*BCcuB;fD7>m
zHDay&Co2EC%{xc$5f>_@43dSX?)+V@>wl}ST&2LeK$P7t1-?rN(O=?(8b3wndlyxB
zo$K@Vx08G;^f9uc)^r`3z~X%lSq(N-JG65$!B5&S@Eb1r4IlX69auO7*Fm*p5R82%
zq^oAdU^ViM6)nSPzg5F#s>u^=;FUk9L4(QBZSm9tH3>&Ct{~5(we*iN0Q75KfC=L7
zmBiLSEK~`*ru#8?rDfN9G4=5B76#4Ba#y*y4KI)yC%LY;YpP%MXW*AADTH_%i&<ef
z2<=ofJeqa!_`+$CHv+1<11n*cZljX%XiU<U4bO?CMb&k^E>Pn|Q&b7l=a;4az`7o<
zPeCKHfRmBMJ@}dJm{z;9(c#Yu)D-RPm<H2S*_s=hz<XvE!Szoq!W3v%lu$n8SCf#=
zUquqd5-H>$Whb1`cDG7ugVho#*Wjf41^>CGKj5o3D&YP;YAU4ti|_zf3;mzjWl&*q
z4Gtjx|HLnpgnqu?_4^WL6Hx1$hsi*EeISSATOADV1G1EmB|U?)rBxx0-!O!uz@JFW
z@Z!Ib7=&Jn;KQBar6Uy{PxcqtVZR=nzj?4f3IoC*a36~w%=M&!+3>-(%NvvyJLszd
z5E}w!AC89Jy?Bx>)ZWBx*_E~#7`;hSAou5^I>U9xatOYv|F;es{H7h)FHVGUnf!6H
z|9WA<kUs!utUjgq8En=HSsX|-XC!|0yf0@dU^aTA`ZviDF0KKd+CE1!ex@W6s_5Sx
zIYh?*<;dBO40MDVNq{cT0NNa1VIU$=sL=T*NGn_b@K5320<nX<KJcOTA3F(<hCnOx
zsQ`c*mC7wZ!5NliSo*%E5jbfMBHa~rzu3)Qn2f&XCOAUwwYXV?^?P*xx!=nH`@Mci
zS9QeTls>4KneGY1*KZullIX#=fvo|3CDffjNSzHh0aU*PDt)MJ#skuDK+1dpZ8|Eu
z!E|fTzsh~&`?tKQmI2(0A~*~-`ve(k;9x&JjRgd_my<j*1r=X_cm7Zf*bCV}C4hgc
z7PrWLcTp<AKm=SjJM5|R-XeXj2YOwlb1Oz{(cie=eJJKk%idH{Dfx+P{qWLMY}qSa
z=R~)${6GB#z%|}i1UNf(&w+Yp85$L?LL!L=sNXf;!izCH%7Xact8X*CfVTy7NTv}l
zG0><0_(WwIDD=iGmYsowsc03k9Ljv$v`{Mx^8p|Lz(5l5xY`GR0PO-27>mHeiiZF$
zs09TqppgwdtFype@&$$f(|mC3CLoQ2<Wisz7<zeFMT(ce0|4-kg!+cmge9v`a5wPf
z3>Gf)8Bic{)}U(s2I#G{$DocQ04Ns)RY<6;4^;{^yArL^<cL-QJ>rmSm!&%iCCmR{
zvr;89P){+C<fa;8?8>I)osT!;@ebCJ$!@NpI#)Zul%Y3d6wApEtYeLUvpHM@svxg5
zr>cqpW_-X#U~5G&s){~=N=st~^|41ifJ_D+43!&9<lb{xsMDt6F~Bi%6EZEAOznqM
zea)Cc0Q}b@<LUP?Fm{6JK4_-<1ZZCI7;;a=ue#qn$wK&8Fqd93_tl(aV%8zs0l|HV
z4O-iYUyBX(`H#VSp73Yx-+%ac-@<dmMR1AE;hHNrJqMtIHz4c_z}<XGWPJ}5+lZU>
z8r#>LLA(HI8gP)vfD9=I!2%RhK{Jsqpd%ZK$^%3acjd%U4KPs1_yB2Iw7ltt$sW{G
z%S2mB4V44NMA4V4Ut75fK+N3w`l`uu#GzW?FDfLF$pwPg)`6CYmy>@Ik|u)-5CMY@
zxQ#rzIT;WD70|&x@tYs-IGe4X+xlrTbMHB(>Hq{vG+GTC0POOSyzQpk&4q73Jk~=i
z0`Q)(RgaPTWj&#Pu?Ao!&XW$i(*!Dsm4U{9+oRJ=mPG2csOB?uDC-9h%y*!Z)F`x$
zwDn5EiMIhQUS#fyWva_rgRrcWCN13ag1i({h!0@>@6~~QS5DBH?3BF*^~G#4S0Z$>
zkDLLJK}%^C2!=k-nm0OE&!V=99HC1(Q!(;QS6V2jFWP-m%D}GzbX_5Yu{`<?Aam>(
zbKM90y{m^L8m*7wzJIC<ndYu9e!>O@OK`p<ODMxn{xRPoGnvK~smF+Ap|IHzv;qLB
zv98&sliY)LbBT|fUd5``=YfQepb+B=AZVg3tYr&#I0Eg8)HlJ9c?6Qf6Z{c&gzp^Q
zZvZUsa1qQ8et?zs8dZe&n=fx;Tm9dd1JR`awx>T(&t#s?C8e+S&XEa$ffwNRf)Nk$
zt=J{_uCd|P`s|;YBHk=8RW*ycsv){6hY`ZLTFR1znG<t6!;jk*)PYEo245)to9nU5
zb8KAyp1Lx0sW=EPG~Fz|K5}5)0<_9KKfo7xGrwmSZv(-gS}qgUb|k*_<6WF!f3OFs
zx<W@Y4S2`;B*mI@^%ZckKt|fi*tusn^H0HsBLPSW{qEwmmVJwb6QT!)W1ew}8YI}r
z!GPAXGA%`um8e?$PB2Nd+EE4Y^}vyzFntVw1xyUqj06t(2|yC2e>c+4i?nX+ejob?
zRDJi=;i9%lj_>hHeZkZ_JOIfxlwi%-+_%Jmajw!h@1Kr(zZZ9R5STa%YTwJ0T*n2C
z0f=AW6MnRg>aS>hqRx_ebVBgN8gk09eGkydn<`3)!80T4I9O_)Y;qyclD4vOie?$5
zKhF3B?6dR4^i*3f+j5L1sg-f0vqvRW3nZn2Qd^LftT7?oeXU3YCLv4S-*_tBtR$kb
z;62oZO1jxfWEw-3#~4+so@J^gI+Z88%D%ZXd%1T-zw`m=Km65jOD@A)pkc8RDve7%
z)NI1}Jp|qG{tTG+>D0)Kd)ijx7KLoyA=?4l((J&&SIh3`0E^QksMB}_`T2oHK`Rz*
z(infGouCD)Is-s}!U)D@h%dtWzzj@Qb_LcSX3TRid8M>sdGOkO!_&8K@y#TBa&#2{
z&01jVEn8E8fw+y=>G0#rHqZ)*#EkYH_<?wDaEfn!_eLFbDcMyg{MPI+{~7Y<09+}K
zJgNvoe$&(PZwVYP1wr4AUzCr|?=uSNNw{p()hQ~(>jJ#>ez_ukdAEvAU^We6f+DWF
zR99HM?jco4SQvp*cWJ~tE)JoUl+@vTIYu?XdGN;PpCWtqm#TZ}kB%w*c*ej#47uGG
zdu?d=^c!GOXQE@@+~h-NED1yW{eumi#JwH$;28ZhtPjZBNGj^z=<~a(xjrzF5E#xk
z^F6t1G}m<o^_N+sL6i_b2Gw0?N3>L{8#5fvxDRO6^-C!)p-DiUnrPfuiFw6C+LFnt
zSKcMM72I?j@6Pw;e`+D15NJw5KS~%8@)wC<PU7g^O?|-c2M?Z=NUBnzS5vYmIb2DE
zGz}C<dyQifs~&-)xs?r*3epwUlgiR+v})!?GgLBMY8Xo?@bT}UVG?&cn?%jEYei=B
z4fwB0)C!yT+;)sIi|HDZJ+GwCf;MXZyh3!w+4wL72bRF&I>{MSm#6)=GtYoz)E2q8
zCF&)#@^`++<HyaL|J>}7Cv2b{KF0X$3KEyt`6<baR?@$;8BU~zskwp)DY~VblaPV-
z`il(yY73et3mD;d&{K!or(DgzdtVA;+x9ywXX+db(fxg`&<{uBzOYs&VL#oHY~)y;
zsyt_AW!1MskObJk`vV>1t7Fe)bqz9I+|cBU$9tOg#F18v&7f?--p!A`FOiRF_el(e
zFpzGvcmhnbq~Za)SZ-<<Tt}4)<?-9f2Gw1>3CQupm=pt|Vlgl=yumo#<NunLH$%b|
zJ1bl(tg_G{|M&g(K|q>b#+s4}r@+YpiPQrgC8^wu3@c-(an8MmJ!x~^)hke>So76J
z)2Mwu!SiUPnXTw9JVq-<DG7$V2_{%T4y15NpvRP<v9tyco92NP;!$2aq#n1uWs<Ct
zV5HBzhST6md)?vjE@v{svjuMTu(h9a!AV{#_a>4$7U}@DDC-M|E^@vPy}^sUrxhHZ
zz--<|1PPvNw!jWeVFp+i>3-`3F^nAcjsD-aGp0{7VQQ?@vt&qh&BJ=HMv@Yh;OF(u
zqwz2?2+rklFe-p>?ih(HD>5Gb3M23w8Pp3DM=ybGNCmvP&tJm3r`+Z}PzR@N%IYiu
z4e>6BnPwU%*?HI)&I+75sQ9_caqx3B@$V5ftPVC$c1U1;$!JI|mt+6ti<ezF*j8?b
zKKo-7ajD%&=Ma(l7>%Jov+JeWgZLMDOBjdEwLceI^#%enRm;K<AG30%hcet*?QGpQ
zAJuD*Zfzx=D`VmGM7$caBufXmQ7<qBOfKVV<9C4A2{({SP0^`s)$}5#Cvse#nF3A6
zOUOMILX@p-%hcvM%V#0;H-d3fdZ5Dd@#Ua-1BD30uIXXNm6M;wRGAawPjAlmf8~7w
zn9=)T$&OAn!KWB%t_s|1lSRoBe+~J*EmXq+A;H&cJfFnErQa~?34a3k-xcULW$C{H
zLEJCX;OWmgiUL*aX#zkjY2m+(Qi)VyogqBtcw`K7U=}(YQ`$TP(X}XOR`y&_Zx7fE
zE2FS3wJv_u4g-&|ySk%moGH`I`Qy^${b`$B$-E!cR``y`rD2Fju-|tCy}eGoJ+;oI
zZ&mO0;wwf@)sj?>i>Kt*vqIr%J$V#$XCLdWK1T0>De9UH`h}k+E%`HzsdWVpBbTQb
zTvq>_Oa6K*0u{j+i($`_)BEel?+*kjfy3F2S$6+*`2X<Jmb!Q0OQp=<WL~i2l)Xrr
zH1e-sF8sOl)Q8mT&pAyU;ep`CNbX!Pqt2%H9n6)Pti3HHWK^`A%;7@WwdMcg)-Mvd
zQ%tYPS8A5%AVy6FFV?(zWcuf{cwmE2tbO@or&5RIJ+sL`uBh1`-~6BJVbliKqNw&w
z`#(p1e?SrOc0t@Ll)om|KmKPp4GN}U!Xhune;uZ??8{{|{r}q+%M<421~7rAhs4Th
zKbZqoK}^oDszzvSm}L*2D^p0Jlz3^@ncsx?iGU=J*GZe@;w~0e^P_~^Htk#IdL=0+
zS0QK1LrEW(Ozhg*em}_<G_$TV>*5us<An>#LKvOn%kJ%-Xd4D#k{!kHUPqF885Ht+
z$~xdfuyFjoM%dajfP6F{lsItruR-|d(j<pQ0ups#G5pVR{QKVqkTC^gJbp<1XV(7x
zV8POfG+jzJrSR{=mN{Vv`<&Pk|NaVe0+?l=QD*tYe;vle=iDM6-oEo+??x&3iwWtx
zl>c?u3cOzff>+?~HDGK2v;{-$rV}^{n#@d*|2+l+4J>s(6K$rmlt}uaL$;BSD%0)}
zXo1<)8kwd1pR={JMXui_Cv`{x6Tv?&7_$JJf1jDBhT(rc30*D;DY)7+kME5C_$XsZ
zUmv(**c-6(NWy=<$l+<Y?z1JoKkaS41;i(vK$ycc0)Qu=iMa=7n-x3X_y$<;NHtL2
zx^V!C_sl)OgIZl{gkD1CBp)Ec6BPFBfm#^o%>#kj+6d@N-u}8-YigXIgCax}B15X(
z0-_MG?FJFP#JhK(xhOv{rKmV-?(-13e8}+Y>L>X8%7}P7pb(%<0!U$~u-hqzf<4hD
z*scMuK*6YDph91<0~?h6L5b5Rcj22u@V~Eb2t(ER69f99F~xC17~(EU{B?+0fgl(6
z)nb*n-i?0-y=&_ff7%|HsTG}K3DH)pgAYGL&I5oSR*N$tkq8N*gSM3mm!E?_KWYY`
z@s*u%u9@KFK^ZMVYpw0;)*uk^tAn&+MNARMwS<5@>^|PNub>=w(RW8&Gp~p*vM>d7
zhC5`Bs*j^3N{8qE@$#f1cYXVneaxyS?ezpFT)MG^ub}_nA<CtNC4}DttD%|!sO#u4
z*v~=~o8L4*jC^&&wS78AX%YtYUoB$suWUg=)OX;nbOXN)4$x6cHV5*(f=<5}*qH*`
z3z`Tq;D+ScNqFV~HabFJvoz`oavi1fK5?+DAK>1GfUTV;5eIPC1iNmJ0W1O5XN%g0
z?vN`g2sD2D=)W7{4t7=!_<DJe_XXc;({DiPtEHn1XGMkR=e41~C(3Ys@E*0JExQmN
z$cF<la*<&|`kZ(>?-rdWEuGCrpa98N)}SMu+f!mF<n=vTahckN91B4^oV4`&>jdtQ
zcp29f?*MOgTH@E07N_|dt|6o_3I)k8;F`*|qZF~d;tf1So=OVU<XlfWh~u+dT&ILy
zJ^tRty8+?v()U>jTb(j6{(k(fc-5QVYSr%Wn}D}JAe^mzg(i>_k0p!Z8r(4l*fkFT
zXzNy#O%?d)Cs^vAz5+~jZW6yy=bs*3tQ%kllJ;`t?P3}9lEP+ELQcyfW0}@$r)Fny
z<{q}aswnyfcIC-3y&#Fld)qz$qJrh~>#1pA@=3P6pvR$2|J#3@4k7233rkZj%d8oq
zcc3XqTQDv_j|ytnVIo-C4Oaa;zBWNvUxM18Ct%l!0TsFrZPD`u*)({jcv*Xa^rx*s
zjU~Z2hto=0HU)~$U>sB5A6Wb}#^f0MA=wOf&!ft<Tf<z~<9|K@8gk0s(P9<?=11on
z^b*j5`^U+&$$+)*el3Xg&e3OU`O%5X<|-rbqUS{6DIjJwGluZtVzCQ`rGQE`Ou|Q?
zl6G_K*Q@dX-Qow((31p&9D5bB<O`L(gHk*$klvgWJvIuuYs~y;Vc9^=1c_%xdAd1X
zNRB0`*RI0ZS^oaxN`2x5t`U4DC|$Z+z<fefCyw(N8(|;xu7xWH0K)&NYF|45^DE)2
zC|!Qsx_70vif>LCm??n#4aj@cG=H3cZzl=Ro>0EG)^gDcXz=EuG2_zqZ-8mHYa#kx
z@#f4VBb3D24}2HhV)=6hS_b&k$UkO<PdMN?#MA~b&%QuR+*I~~(ca?n#rBFH{Lu(V
z7K&C=ZO1OVkRJn-(^*ACwE%|#YO0@Rdu@}?^6jW7FIA!5l~a`*?zz6bX(nPZ&^Dfi
zOJ+}dI{=JX&?!|S2xJ9}F2L!S30=h4O`n6dwKcq0PVcuqOA%Sdvc9B+jBBZA{g3~6
z90ZJDVWFxtVw$~&*j5YMDzDe|l<yl0Yg`Oyp<gfJSC3A!1aqB#R*enhyrYJ%e;eHx
zm@wrCL^T$H<+hQ5EU~`!SMtm3Ro)sFCYd{Gz<YO9tzts;X|6sVPw2kL+Y8{$a=rfc
z<~x)Nen)W)dRhbkf<;L(@j8m3c%A+)F9439a3v|A9($YlGKCnjjdk{Ww*ln|YeRqj
zur(3l3%r`w%b6>;F4==ppd!6+%saLg$BZu@6dZwl%t#?Wm4n$K@1foeVC^}JNRybD
zD{9(83VoIXKvf!ahECzp@u=W!Gyn@ype}d|6*vlp^h*K|B2@QepLo$C?1N;Wj69e4
zC&)HyoVm{STg4hr=@j_x>~L2!!B&$5vk;IA^(56cfG(glojSo<aKkj>17%!lu3si)
zx>nRgs$#(e)aQ2uNZ?Yk28w?SGSu$R-Z9BFgjb8GM;ahO2EagQuzF`-?}!O<(?V9^
zy4H9<Nhi#A*8_O-;Ev4zE7-@Pw+UU>fCkp8-m%|Yr@j||_VpGJt?rTAShmY+WSJkd
z30r+>m-lm_dGXxsk8r#JRnSrM_rACZeQ;JvbZ7Q5GrDE2$QSs{jrqV;Kd;pcGTw8o
zWY6TIwiYHz$NdRcD<GS#Y(Oq(p3E>c&NbP>O$v9ou)?9`$9?vO?}CAtlk*A2eJS86
z#vhWAxTeeO<7ITVI|BBGG2g7cVIj02%e;TS>(uVA-A0n)m_f`wSvZhDGVI)#%kY^a
zS#T#d=lq$yHU8=ITNaI=s#<R_J!kUVpfPh-x8xy+QGV`e2hyPPQauH8%Sd^isL6uR
zH-HV;?Sljif3_T1q%R0Nrd@Yv=ggEfyZD~d>($S>us)AV(M^Z)gPmu*DhZr$f2!w9
z!Q~4~4M9u+<>iPOTFw`ppem`IcO;EY_u($QwflVe1ibZf3D)D2gZzmFgNRsED*5c^
z%SzA6+PEmVADW<!f(^5P8*?7z^LLVatJ#Q&hVnAk0N-fin3>Li!gOO|Fm5+z$q5d`
z5MJHw!qjpj8<&4%EPspHwO!e(Ur?Y+d`~@{S@t$%AP^?n1nY9{&D%;$gGl9T+Dr4!
z{ut1?1&a)p)eb9*`wSE^<K~fRnAIj^OkAw9pu_PUJQt9Y)e@M{Z1W(n-%CtR_{w!{
z|2^AKooS)+#)K*e{R2nvp$a5#WKb;h#XIQz=6ab}iu4WSy>OB43lye^U?<;_Nt=@_
zRL}28EJG4<eSK$tLre#RfF)WW_L3h`ZIUcTm91L?vXI1;L>=aaH@0@(VYa~g!nb5P
zRm#FCKrv$p$X)G#6seT@fivTTBMVrbwF${8UqbzMl=_bEfpiPFqJaZed<JzO8pg;r
z^&A6f2Cnu!&mUk|kEycN+lHf=Ri{82)#{F)U=f$(BOvs1$pd0LQawgrI_YF#RT^6g
zWHdJ+slaS4er1(>E6-T7%MfPuoo^&Gv5mE0F+}uPlOjlRu~~g<6mI(H<cL0-a1@PG
z5a*rQf>Xw8tBTLBJmapb!HR;}5eAaCKd`nA?>62l+nWT<VUbf+tHwlew2-piQKiyO
zmt(*(l5raBo}YN%Bs~mz%$OFazMPVKHGs^#%DZYeucI;+MCa{GCUL>g4YzOB^@M~m
zt8i8xJH{C=kX07?<~%RVOBQPn?7*jZZTW#_x&j`Vc{*sLNEhM@#%r56n`XC4&^@;J
zh4CAcE%LH6Pty*&ETd>YWbe$1-eYK254>>p)N{u_JjND|g+sjCV#vbO+3A9YLeck%
zQ<wT&D!Wk?gh2jzb$GvjM}NmuwdniX+n1Fra-`OM%=A<>^A}K^B$>;pV->2fc*inh
zh+b>taE1$zwVm%#Y|ZwVT9&Xzdn&!77&+Gc4A$ID){#@89t11inw7ye;SFQElvfkU
z2fE~X2fPXzj}R2$p7{GiKhr1y(?g!z6{xJT9s0esSSth*_uY#bpeIncsMA=MhTXFu
z8jdaXFJLk&>HPKD&?c$HK`R)BMFc7hWWxR#_s%gbNb~N^4i-MwT^dQSuQgx0|0N$S
zV<pPz;f08f9Zq4O8=R3j)WqLmkQ?jh!bUp7-@K3$kL2dch~xsH&mZ@D=S(pBgrb;G
zE55^P-WQW^ZEwEp%!u^~MBRMbxMeyHs*yDL`#=>ZO}6~S@0m8OJEF~-yXn!YkUWR^
zIs4lMm%X0X?;fHE+>+S1ySM{}Ru>Rj;D=_kj+sK+whwq)zxMP)c1RlDZ@olk)Kn_j
zeB0C%w9DdwN`cM-4=;&|wOg$I^`d}Ac(*eqM%4ESpUZ~=r5AR#(#RyMCzlf0Y#OCd
zLu?zC-T?Ra%{hmR{v~nY35H`fc5WQCjF6wd{J5G(OW$LTg}XTO@A1B+DYTWPhzk$#
zJ8=}f0s+dy527l5Otxy|)v5KyGm(s9;{zSaY4>P12J=AXs+so}FG!q@$*Q#xs=C?A
zUFM-F%5*!{j(65Oy*)zIXxq4TI15$JTz2F58z$Mm1d&q{2`Kpn!|lj~<=YCx+*rNN
zNKXa3AFiI`rsd%9=5u$esy%YrYmSb2q|NVPzW7Z6TjFvq#oy+YLH#LC&&Cru?{+$;
zTuleCh;eL2@mT;rxz0Q$J28wPsqthSc{kb5$fc{QIaZ9xKOxgS4yRQW4He`!`nKoR
zoli`nPMs)deG|`74})^&>9YeA#O2>2D-W5Z)1%{X@g#Kxq-nMHOUQ(B`J3?Xd&^2+
z9$g-tfMgeirfe18!_79@Yb~gtOIJYTe>gCamN^-fMi5)anXXX3Z@6hx8>sA-cZq`D
zIsWT@Lo>Ws8huU~VZ$I?oamtFGqIX>?s$4mY86(*K)n!OR%C2XoWuCY(IxZ5VhE=t
ztB;_q3C!q_zIH;8$mhPO^$qa4WL8Tx8+gog_ds{ytD*deob!hd99I0rm)YI>fz;%D
zT9e<?#*6J$VKmAt3)1{W<qehvDp!?jNw$3WsnK8PKPg5rvo+T+m+RB{+%5wAPCziu
zog^VT#Tk<@-NYklvZyHKhNDa630+M4yRuQ3XQgd0buk{anAom`<9EYJ@0f~<9Z9E6
z;A6!%KQa0;S)cbE%C=O~IlOHnems)e5%*0Qfq`1eR>e`F>?<fo6bG`+O2Y%4UJ7=e
zd@-Z)pxbiA-h<qAFUmh?qML?Jj-9hCh3Ld7_y|O|iI+5TWB1cNpNr*<PeYsrPLYww
z1Vy#{UFNOCKZn_`*pvFny7UC~(FIBCTtMieF4QV{-(|X-mQv@3L14>SW=wai*Y@q?
zWXv9K%P%}H$-Mja>N2OZIc&P}BENbJWaMOH+oj;y1beAnJU05aoJ20@NbUyqaXTu5
zp9br4M`O-xhb(@!jCk6apzM^ko#Ck2zA3D5-jUs?h?fNTNmy(A(|m%1r;^kbC)mXU
zRQF##Y9#i_(IQzJv~RjqC}bnnbhqWBk@{D<cdqhS_Ym$j@YYc67D|hu`IF-2;upvs
zz_Fz&ury|x;oGq`VhuD^1)j?K^BCFBk(fM;V})E|k!qf+vaqmyAt<`sZ3KSytB9H*
zJCmsItLd2=O5+^knZGLMW}8u3QRuAgwS&>Xv?cqvT9DZCTT#?r{TM&Hf-vLM+^X!x
zBT!E)*NklFaaDS(GLmEBl&y?m8nt*U{QJzRfeLx%fxx(AS+sWvfqP3_cz;VXqk__7
zIs9)9ch?RYUcF@6O00ckJq;PEp?kTv2>H2priSrE#vK;{&#0r$)PNu(MM*QeHCao_
z?vgb9$|#bMTx9QLMHZuK?~g5UZ+@gv4~$j}<JRxTFYso+#&W4B%6RC_FO#hxN!2)k
zN?!d$#@nHWOc(|KH38#$=6mkT-u&T+_wgCutGsERzxyT}yv%ZS>Jd1`#12g|r+cLZ
zZJv{jK$cbD0vj%4FfT*X&b&9(;MJ`_DaSp-kFf#r9f(P{b30E@s}rl%nLoW~?bh5H
zjX%z_8+c7Xrsa^cnYL?<q|WVf+S7vIz`mAPsx3qr4%=7U(G9Uq5qu!8s>ZT_814On
z7NObOC?k`bD4m+)zNZx%T6JFX#%Su3UhC{~WxAQvxlYWB9BI3qFKy4_OkgMl<b_04
z-kYkOdKM+Z$(Hpu#Vvjh&twLdOA*ymC-|=wTWP*&Q<<q*zj7QdaGgk1R(TlJA?C?p
z#Eep~VgI%2@%Mc@mW$jz9&Qk%^DJnYl8}a{Qs&w$g=T`3#`Ug%ofu-2sU6aEV6|GA
zqysYgMY><*0aCysd#5=vk`~^p=nbEeO*8KiJ6Up?$)jbLKBb<_O@awNe9k;<4_1lJ
z0vo@}8@U>6&;=w@2us!mRyJ1-Z<c&GyfWCw*{NnHQAr=FJC~^@8N3M?5Xyl0AN={1
zU&31Fpm9v-he?vbW@4<ar*T@M&Jx!sMEmOTyn1?Cba>vo%$j=jM|#GncjkGrv{!jg
zIVy%&B@TmS(FM!O9v+(4peHA1J3YXEu8b~NQ)6HKBvWq+#oRqovaq*ir77286<H3Y
z00eI?t5-%!OTh&wFYamQEqZshB|PM%-=|lKvw5Q3H|EE7!m&P;9~^*%8!$g1kX=Q&
z&W4j+xvAj9`wQHlG6$>658wvvYNaLg{!Y4G%225VX$vo5eZ{o-z*&o5CN?LNE^cP0
zA*7d>xj8=Gf_88)q_Jv-%3mG9vMGBYohoy_206Pe_twl&F_j%Bf7Mb*p)l*Ral_iy
z$F!dq)vlrE%#OY^kM51<(q+6m@O=-D05rk+RyyLe_%Vwp`Wv)=jD5CM8j(PJNA$W?
zRT<mBw}6_z0r(Q65jI=H=d6_;Qx{Sj8G=q3^FfNu4=>d?hczy9%oNosTUm^fostYo
zTEtwPcl4Eh&MymeiB$3}5m#^8t$g_4)MB!bpKvsuX4-h#P>u;1M;5vml{B!@F38sy
z27olvQ&l8;`Kx&N)m{TSH6sT6C7j95SW(U03+YQG@}i9wWjb5gyAzSKWOcOP+4XTy
zQO&aRsDLue8HkBTWKYPjKE?KHuMc^1I~il_%(tYp>t@^CVNw#QjLJjvH&CXMWg~sJ
z`ei_e_5S?)aVH8zHZZB<V3w;vRHOZfS0IZd{lU0Qng6-LG3`5Q+~DSxz_EqpSaq*l
zf}hQu-;`mjCM4~6?(@fK+wL_DyY)5gFc`deuoM(9Sq>NTvL@I$DSSwlCu@(6u~5R9
zVxryPD|Jr}(#&$6GgZ1N-}$tM^3s&@VpM?{&bdjkahnM%jo}fOmmd38BG>aDFRO(7
z7uexMJJ%>ChvN6yd{lJid4=^o6-eE)E}_cezpq!<R&L#ZVpr$EaErsM4k-}!mkYBz
z=Q~@NXwaanqqFp;jBcdPoYHq_rd={U?Pd{CW1)ua+M=rE!7IZu&^2B$``S(}d2cXd
zoZ2Qd)Xcf84l*$+A5PG-*xGsR>j`ITv~wyIX=d;wp)28Wx?@YMW>!of(^asvKA5K(
zb}B79TF#q*GuyL#S{ZD>GlE4b3!R@{UOo)-&eL_{NoO+)&9%vhAIX`IZVE*%Xsc_&
z>#tO`rry*#v>%0yhSUcr*x`5AgK`18WuMTNKv-2%7-BFib!}nUOP`Kje6k|(#;g;*
zngYk09ZdFxd*j2nyclMmB~IH6`X_p{fVN)TrK+o88C6#Hi&Sg2Pl<+=t!}CMrYIW^
zODR3Bw8zlwDzHCMT;iy3GR%0jbxGMjv8OXLzBgSLxREo>POtpLoJCnWQL6^amf3mL
ziEu%mNV{wU^{h1?-NNlyKh9dW>l&TIR)O2);I<;nHaYj;LJ8)8T!2YDNj^kZ-7Cfx
zw{U-`y1j-;R#@5XNiXfMjj>Bme>Wxo+fm%vO)!MR-Ed2;KLDpjDW~e^1t2$9VLdnO
z14288k0`{ZUp_qy6*;$6vKUbo8NO6Tt5%6~^a2@K>UDxX-I=6@HgVV%-Sm?{U%*X6
zKuNA})Y+K?3;yh|R_$}Rnu*TB;7XlZZ@wVM+PFTGY9tO_Z0*|Odg=nho9Y%DvHWx^
zIJf3Q`FvN;vO=#iy}VU!-r%}arHXZq{Ls(r5}@>~Dk*I+)>4()ji60-eCLfI=Q`&@
z=MH=l119rTPAx1O?kZb8;*PS$-kZSmPXItT45{wSMgs+j@I1ubU)4(+Q^>zlRh#sE
zTg6&*w}Y?!cPVeD#8p-;GD0j+Q~}DTkJf{@w-JN^GN)WNr7L0(C>XdacSP{`8TT4P
zAsVTIz_X`po3Q8hz!D%%Sw-4y4{^*@jnYYue1aC=9RvkV9};x;+HP$@HXh>h;<?|F
zqN<O}*zA<EOf3DWqjR?zo@GtK6e*o1NxA5zxXT_mu4NKIwj0a_7eYr*2j<7+iH$q;
z$i#>aYX(xdUti>ZR>xOhawazKc5i0h&2mdw6cnv^|LOL`B$3R@*lUL9$%-1+iWR<3
zU{Y)1tliM!RVWUU*)%<*AG1Gb0doxwsMWkd0~anSfyz1Vs0x$5zCF;aFHw<M@`S!a
z@5Hu!V5JMBAG6i;+r?a!NmC|tXT$+=q9ruoSUp(4BsCQU^7LW6i@m`gP2Mu+{{U=Y
zLIIi@3t`Jqazi~>)`2Sz(QRV3N3!-bPpVk*TL!)-lk!CKO}3D{%gj0+jUB{f8Nb_X
zeYku?Mw0>1!*D*;KE;X2qsq2|%Bk3Qm1kINZT@w<-mG}|uej*Ft|e4e77nQ6)r#%&
z#_T4Dub_-5%)l9x1*Tjp<^|1V+&?)|N%A1bUsG>Q!s>3)lyu4vb19y{wNHlL`&uI(
z-84EkdS8O{T=*Wy4dr`zK_$<UZsZ(CF5|XF;2;l?31_b=!oGoiTXe&9MIA*AKt=a}
zuB?e3i^HoMry8<uZEkTHLX|m^oiMjoEB{H#^zos_y;VF1b-pECR>;7!qW=@9bq}QW
z@aog`|4C{Lnm1e}-#K=)HfAcOpH47SZbGHksu^96bosOccnvbIvDw=#-gM=_BC8NN
z6U#(+T%Xz27$Rp!PM`*%_M?g>H5Z})>-aJHtn!ho@!JFGgy+g-G4J&E<>ap=;O2(B
zfg6<z3nb!Zv6*P#R9*}d2LWP2RWFSR!Otc#8?%GZgNbJcD#J5L`97I%P`L!DUwG!m
zLxuZ3x`)KM&nbfXl_W`-v260iQlaX9-!I~=yUHvQX(}v29ZiVnx_DoH>M2!<y!QJZ
zSfPCP!`BaA<36yCPi_R4r!>1>K*qP(n=b+pYb~^^#4D|~YBEnFB_sRixcQf2zL2X9
zNPI~1R2Wl#Bl)X&V>wV<-ks1OeBYk=y1KrME>cNVlI-JHZt0$987ZfUlTMzA#kEad
z@#ByxsN4qVtTCqSW9^v$ITPz~oM>IWIsDI{=u$Stqf<ogF^CHh4w=P*qJFU+?}mry
z^AhKMvi6b-_<T$guk+~zg+L<*uSa^9IhCunsnp|=w6@XBPdsxtS;r-Fq+~OFHI_Wu
zzN@mdVlerZS~kDzG_eB-j-&-!>?N0*%~tXTL#FqT!=G$TDV)_J1NFVXR3C9}r!xs>
zRu)DL3q4v6euc|Tr~ZPjPnO#7I7k{`6)h~*Os-C7K0ghzH>S;#)-2B{i#<0w<!i|7
z{J2hsa~_({{ZM!v1WXH6?~?7H-RNhQe4lF;3?|v@7@i7%K5U)WJ!kj3(?pE)Le_z8
zCvx7fk=3Tw)0PbLgGTL0#(Pa31Y85u%p3a2vQ>vcpJsS;S*NyLcO$OgZu|1LG<4OS
zVuIByqQ^M}mC+SOe8ldMJwd08JE1a9(|(V3pVIkj0@xn+{=1cC8<jG<#``|)dR-rI
zztZR!KW%w%fRLY*>SaeoL{mF+ydv8-cC5C+jD+XT-D-E(H+RU)$;sMutL3J1m)DTX
z=ZD@6wr;Nv+F`53lJ}`$$rF*JQm>J-cc`e5zy7DK#d<kE$u%fgqMttP1M(EgPjWTt
zs(eXHGclRC`CQ?TFx8C2f~Oiz4Qv-nH{9uUm!bZzZNnfD2&*Yn@z63>T;<ksNZggF
z(*iNT*%=28Hg47i$e~Y;rn~a0Jevgl^M<`d^}L*RVlG=7GEc&guvb81IZzIK<2Y`@
zWF)hXLX0~E%3(@AcH^lNgQmpD6}n4V$-~IRUy)>nW+(adB8Ojsgb3r3TcR8}+@5KG
z8Dr7SgNk`%MWD7-+xn!+W%c$8C!I13wl~a46phd4KLfj~jcSn2^=@C$Wp;bEq%Ru?
z@^V4G!EU)1TGm)`byxSRLq2*Qk`~41r#=$Nm%wTb*zNb1R`ts?`wUITnmU%T9qsm|
z%xF8k%|tvk>Am>`B(^|0@WSWpg}Eumj(OgdoY`B*i}|CPvW>Cuos&LevL9R>7)T$*
zL5c{oR7GZEnL}&}m(qU=Vc5eiJ!fN)qrt&E6#J}z7{UCsw&E+@Ky#|lBV4oDvrJ_+
zk~-aWdlf|SsT?#kN9jq)!J6l`{A$+|i<Mww^4L__{&mS4KH{n6VvUMSwYSW0?xWT-
z)d+5n9a-n67cUouelj-E<w)b-p&8CeZI#1!?s3vo`*_=pVP^(AB*j=XO?>5r$gj}D
zY3;NA-Q!xF-+M~mCBA2z=$O!^Qk+#mqD`mo<5K)Fp@Atb(mLk{G#R({bUlMlCO|fQ
zyTHr`ov6|Pxq56LHi4(YiYxQ;RJobWK@4&8;+KkFRxO3b!Xk$z;G~pXueU0H6LgN*
zx85D6^3k4a4y-oXc+<QM5o=O644iaEoUD;)Q;H{XYB$z?4#6F?EFfBIc|)NFF$7z`
zkgca>^7QBX)3*(c06_I9+gInR_r2J+hrLAxGp1qTOpI7=7u()W3DbiTth0tM<_-n!
z$WP$wNroq*C4N{*6MG8P9{VgYIQ=mPmbjDdd<&)O50j}sL`nyzAl|_n?2GQL3zfbP
zx0OcB%s1S9RZDg3I!eSvf;9l25hw?WMZc>SAfI@t1^|co_u+}|UVtOuI+Ug?um$Hq
zX_|K<f<mvn%P^fFaNpu$+~D#vxfm@9!2yKmnfFFyS%8Z<&_h_enF4SnRO`bp<rIG4
z%bsB_Rx!5gcG?4(;)CRHv{|8hX;stDQQ{DgiWWH;r3E&JOmAM-gYD6VBCW;k6e#n(
zNZ_8imM2TESz?12m590gI|=Z@{ITDZm<_mTmUMqs`Ii(;Q>29DK`I6``a8_}YWi=|
z0vE)Q<WeqoEB-3a{$>O))4?@TnYqsB{Tt$t0+)h2_+9gNJoeX*J}L;}m|yz-f4};V
zI`jW4H-v;QV&vqH2lImYPc-V(`H;<z5-pKJEb!{b*I;kyqIhT*yN@W7L9gb#OAr$s
zz4Bs6)tD*lC-dwGVn}b`$DQxzXYs8c8(6HJe(LpAC5OCdDKdbCA7A~d5K++^n?0LC
zBk+VbY`b8;eF25&QUs2tQoWtOsyA33J*?5HO*4B3O_wE-*%&J^trHxnLpos0{DUsi
zgN9O5!S3G>C`A5rnS#Ur&wbiStLqBUNc7M1a5UTk7M3r8ddK=X5Kedl@_xe3V_u0t
z4B$5Pkd9<H`F{TT<-7{VeE_dEdqVAUf$^*t+7&eN7&WT^e_W_Z|L2v37vXk*NeOBm
z$_TZQFd=g{2b?OH^zWj9ZgcSYuy0`bQRxPvuB+PxVqV*9)rjcc#bIjB-JkOeY5s9@
zK(xyNRYiBW1M&VEUBAbr<T)CqS{`o=TpR0u0keJj{BdbxwbS4`g?^b1+3LHldPxp+
z-t*UGn+|Ir7kh4W*Z2mc#2yIV`PBY3=)cB49vJ`Bj&}jy`j5YenFq#yiJ8X?1i}4H
zW|2@s<3Hj%=RZCQ=G783!Mp$Yw4*uI?YoZNq37fAHFI6^{CBc{|3UD8$)?$RC5bek
zSr8o}_59?7SO@zs5=nXNx3G|*zXA|XC#}FkXQO2)d`4Tw4^lE8w=M!-2*-YY0_}4_
z1z`G@MOUkCaL_&P_lqDVTXy$-zZRhR&vVo)`2~2%pTEc+MgQw8B^AJ(6?t_m_3!Jy
z0cG8zUk{E*{{~c`8Sp+XIE>nT=ijX9r3H9Pe>F8b{C*Ao1C@Bog2S{&6uSSqSqGuu
zX5I7-7d`pson@TG1&5QLKKj2W{l{(j-`#0T-CxC}zi)J6N#E@3EGos63=|steyC<_
zmRJr378Dp&VTfXMoPnnY6wD2O@WKbw;aoPur<zV|wTMRHPn0s~?IZ-c$UmQ$OqS0*
z&NB~3!2R-*kBWG0;SZ|6WFM$J4?9(sWy61?pZwIH+5h$8gdsiyFVus3xzhi7yqYCw
z;6c*QZpi)D;dU@NxYFwU7pTcgZP~YqoY(zdhZl)Bx9VMO(f{=ZONP@!`(H^f;{Q6l
z266(=l2o7kYpgQPMuNZi2LbZW6W25WZ)JNL(?7;9G{*8E{s?i(?7t2>W01Hn^Z$?b
zzA`MUuIrXmQjii*q`Ol<N<m7xL%I<VkXA|%K|#6^Q94ybl#uR{R*(>oRuDuffipMi
zqdb4lb*}Hc=e+*#(gl0p_g;IhwdNXYj4}CcABoirDPs6U<&27Z|7v!GM@Oj7N2YcX
z+l%mt*aDvFN1O9FfnoH%jpXky2QN|?i`-c#aPGgI{kvcOV>-L{I2u?V_5j1L3=9DU
z-MG$8-sHcrb{(i_LS+_j7|*8hf3CP}5%_DS&B(=+{pPmFSq(`mD^hc8f0o-HaS2~m
zqHae{@j(xmGz(Sy7+=l+v33(Y&k+U`D6VceX>$7&x#qm~5>3$mJ<Jw}Ao#c{2+5=a
z&k_rR%<>_4wgOSLND{5*?7+)c|8!?01laes+oef3ZWjYu$^b_pku!!$siH17BlG_m
zLcQ-H#9#aFalA%zBuxT76Cy(^KikmtXPwy5tGf^cbbcTE-}gSeT8bI7XzEi|O{-1@
zJsL16B@x#~P=aj&VQ(CEmp>L&{e6JcgA`Fs&@&Ia;=K<9y1fs_B~52#p_Wg-BKxyC
zIi1qF&<j8OKv4?5I+A2tBEXupPBJ@2Rs%i*`@oYm2Gjj>;N^@6fRXa{Pzu}$JpYgn
zv|Jx~<L3b^;u{CM2fO9Vz;&(!rZffZcK}~@;e&)B3AqLeDMU*QMWEIu&)hkH-THaS
z?RHhaoq>lE0M7zddQvumQV=*QTtLP>V+BV1_wnwmu1RR5gM9PM9tf9hHi0Vp*@Ala
z9VAXYi#HdfsXG`;*4OHfQ;UHRn64f^uL@|Wx(c6|{+Y7^53mseG@m^u!KRK>;5r(n
zU#-Mim}=B}c;|8%@@<>7lBfqSW$L~^SiQ-tdw*@FwH1*siU|24b-`-I*AFDtnuwzj
z!i(Aap7X0+@5-&t`y-{WFdk+_YRvLIrdkD|T@rgc6!p9KD(P~v1VFOGcy+1oXMeGv
zqxvmRVV$QaKbKmwp){b=untg_h==5KD1HBqA}-Jud(L*wuAKT1qg|PD$O|ONV<7aO
zV0U!)^~X9)bU*Fe&|c1yGDB0^zo0w^<R@u&I?_~`pr&AiAPWpy8@f6$i2Xp|zl9$E
zP+D3#{)7mK5oO}GKvsopRtJgwe#}ruHff1iZ~A~RT*t_L2;m~a>E#=?!XiE>x}cfs
z11V<AT;*~JvfTr*)ERpkiuv#S54XEORJRV!LMOi=Rm%{|K5U)*cBKknl7l1+s(;1{
zBGo9}bG6!u%#d=x&^zpmxm^!;26OYwm8%d$6d@td+f+hvP;Q`Dlf?6xfKmx5huNB~
zL2@TAM)^iO*AHNC(YDO>VthmZJs{B-q%gg-ZV1M%5cK=xips5@XhL%jR$JIUyhr0a
zHtDj?X*~@TQZ=27x>enBuIP84pf+FE*)<?HKLo|#j5AQf6h^hOCnDQoH-hgWhxL>y
z<wUQ}f+@6}IIh;?7IpJw*4#9BZ$yORvJU`@jkYnU1@3K&G59-clrY80mOrxtsOIvb
zeuilg^SOq_Pd;p3VUiY!P*9G?k?lqf;u)ESJ3A%C>hkS&TeVM0fALDZ;G$oIgPjqw
zzc{9uQBgY-WH(f0W7MYY(VJnq@e(bFPU@<LsM7lR8x06^P22xj2w`+V=cnPE#M<vK
zo=}0?_sC&YsbuRE*5Oa3om>{J_q&I4O284T&oa@L-{)>jOl=>=V-7XyD$p#?8MGtX
zU2tQkF+>FTZ3&ZQZN5-~R!<Lh0r?Wj%la9&;yT{wA$3<pt5{$DfZ$GRKqSRYX~CIi
zSQ6_Z|I90bNUlMYEEQNoL}P7H^q^nU0A7|IYNrpK__1|X<=zF#(Nd&NZN=+6w(zZ}
zZ-}X^6FW8bNjRvcXhuWF^@Tc_S<#=NCaXF~0MxPD7i6>&WYO+lOzKz%dWq`g@iAj1
zt`43{NI`24gq@wU5gmBnwyASLabto3{3ZWpv46wGTfVM+-~<Rx;8{|Z({s%gYP0eT
zEqqkm6s<3siL<7hM%skjQyDHS(&m6~|Cr*%8bo6Ll23w<Yz2x6^khg>6kA?wW*X<l
zzJRc^fe3iV#U(S^F^l88gGR?kfXpU<j9^>UnF4)i3Ydy=>R+-ZTSVVQ%SFr%aYwzk
zASbyV9jPUHykOsa04(A(Se5sd?WzsL*R(X1wNq%`H<Y`v&+PRw$9g5%>rr;EOEX&9
z@eQXzX$_^s1~;a`n|@c<34+Fu+QY#qSBqZ^L!HMf9^5+py0Laz8dj_iRNjK-eJQ&j
zE`IsX3gDozES^z^g53ow1nnLn_fM8-@7UU9{fIwX*v$fptd-pVB&N?TnKYKyI!uV-
zS%RIG;ysyftU=R=iUK*KaIhL2Lk`VVvcTvZ>$zvYZMhrwaBqFQZESB`z590Cs|$E4
zhC<owe?HV64xWb@_yW=P=n47nX^B$j$cHHlvNN;K_U3#ufr_k)>Iy9KAKM+cfWyBE
z6Pv=@4p45VdpytAv|U*MQt6%ihe)wx3e`}3y`<xeFW&C$f&2Sy?weB}DeU}AM2*nP
z5%O0))#|yLHGHPe9I5?(an*+~9W)HbTZ?fv;!(b;YWTLs#aO*rf|q*#+lYu(N6wk{
zc%?RkTYbF=(UHPXq5t5puuv9~O>Di<Xt^<thziDFwesK_0TD@(GoxpzOt}(KW2hV+
zNt65~C9;l>*|0XzOOK8|EvNM+{)ecK73aSa2Zzx+8Xo`_4l@hvNT(X|c_wixC|+62
zoLw_NuY?1b%y&n$vVJLyfyXG>uSk}paU1noKI`l`>U{3ku=bW>>oN5(jE2Wg?oMpZ
zrCnD^xUjpwS^MzvqTHW_Dg{yi!7!x^DHCi$nRs#PoEpmvDk1d)h1C91JJC`;{EM9&
z53(_7%9XJDb=Y_)vJ)O6@H24ndaH#b5{?y)SVCEcJ*atb@@h-O>9>wXhR-j#|H)WX
zAY)mn=1Ncu^uSh1;+cXRsAsl^ga(ZC_CRFhIeZP$1S@8a_RkGh5{n*H*g5%5Q4i#S
zwzbPj1#=E?!Ap5(mXjG7$@+_!ZGg`XvZwU-_aZkAy0STxiLXN4n%5w`NwJcx8V5SB
zxz8KLFe5nV{xz*cfk><?a7#VVkBmk!i+&nTtja(y0mTQh5p(>;DV`w)?G%`=v29dt
zAuR2)x5)bwfQRq932RG4?+Uoo<=M2#4#;d?{!TZM*%A|Yi+m^a*mK}w;e377`%dtp
z2~ltNI;0=wf#iZqJYnkjr>x}PBShoS=;EwNa2ZS54Yow)g~p*$wY_;z;G2t5^pAnj
z2vQvBTa(kDjdmLju-J7PtuS}41lXz9z$CQ-%V_w$M%5LEFj|WE+@2F@8}1A!^mm}o
z#n)-=A3%cO>RrAcjYe$4w>AfhgH%ycchrYuTFX;Vwq{90U($O04J?hVe6)-Md1I2>
zcWC`fZ#44_xB@3TVq=M8JkF3G=)JuBIJBb6kpD@Y&|p^QqW&XVL1!_=pGPrvQ}xck
zObU(Bc~OxfzO)lMvVhlKmf*g`L&5a?3ZLsIWDc6=>SoNOpG6Eh@<i+h4~(ODOWpuL
zUcX~;UnIoE%EFd<T@{t#hI+EX%v68ATz(0|#~Mh<mhD8&SWO^hDbgO<>UGN3c0agx
z<DU(05EIWJ@TroRK`4C;fBdU<hDu<$Uu_bG)pQe7hPh&&pAD|u_eJjw)>|T_4q<9+
zs7^sr44B{24Ei+~=%iZGt#*15Mc3)X(y)9rAf_p(B;fQTl{)TrBYL>RoOiMID+K@~
zvANaCmX{ssW`bCI$0nER%a+EUb0NI5?<@GG?Ykx*@%297cORh;@*D!_kD|(i-p~Mn
zdIIZ7Y*(&*Gd1|OqJPRY{@|@lu_rA@1(G;8qOBdT?9-^w)p1}DEN%T~fZ&ikC9nxh
zvPLlI>T4SdeQX;EDCify=>&2X|BPXA2GiKlvz2;e?jJC4>{@u7?b8n-eYl{^q%Jq^
z`)SX8tV4+Fygw~x;P1dcgbR8TVRZX?>x;AgGvJY}M8;QWeo%WX;pk>{;WCWXn(CtT
zxT81mhUOj<^vk~yh@V4iBOA8fCdN9de`Jx!?G~z`dHzWt<IxTB4x;+y<}PGD3bwl$
zL-W_Sbwo$80bf=?3=$KE4%G*LcI%?t(7b)s6d_3b-V#Bt*8q<Hsk3uQ;h&H>{Duz@
zc<YDY)1M5A-)V-F2`nw&XRW?v9lc2pfB+u*lzM1Kdx{$Y)YeOVG=5)yII~F2L-Tom
ze2)LP?l#q$zlZc+MgagzYD1r${`fnnWn4x!b^W*&&q$<9ArwXbc`4(p2&%rqk(9l}
zc8Dvz(8$QUVaFS*gM;wOj|9eTToR&X{YL#QcV6iueJK@q0A24xee~CH%8&yl&LZnS
z9v6qVGp$<`KXP9fL}U<%<xf-ckA!DN0BY{$KlA<1OQZ||M0(#~pYyNp2DB3Ne>FJ*
z+UbB>n0yoiMZPG(|KJzRWPrFXAtmRx?h3{xt}OQ=Q#uUzKqo?Sai5K?wVA3%3<;pB
zR>TZpQclsTEZ-M|{TG81Z|n|%jUwhTMVcBDboY+*qMv=GjJ3rvia>B`ni)}gZzy|G
zdf@b|5pAD&bPD&w)M4UPsaKcy+~}qYM+({j=~IH-&7;Ac@gLu*|98Co-s(E=%RuC-
z`=AN5lfd2lX9(_q{QMw2#U>{Jts4t;av})h^^-SJ!WYnq?vMSeyVa2IB88wq%)9Yx
zlMQ*wXGn@BR+)|Fejvz(-^}Sr(?+=YHlnalx+_#f*H#P5ERrTpKxPxX19b`x5C{wW
z7ySXxC~z$U2x=CJtRYD<@DVoOK*2^t_mK$tbFSLgAP(>528VJ`&JM$VrMyRUSxcl!
zG4<Ut03^t5_dy0+szLeoXA;}1gACO5lnS)h%+BlW%#vX1r@jIy&@RY+R$pJY^$!ax
zg(^p{+|rQBWpE%zNlB@AwS(&^uFE2zYnAYjiORj#*-b#X{>l&mkS^kw#4Z5jG6wln
z=;`Zsc0rx~z5ANqbtp{UUK|;D%GhfPA&Z#cIDom$<aUo|sBY)V&{{lzQs=mRSy}*J
zcz?wVvCORfG0R;|ZRfNZ^1Z<OGi672)8UlhOV86Jt-;PrOrXbU-TzB=LB75E7Lvi&
zp!AmBL2~EHOvoNrfN_HqBHkgd2-DmJoJs=f-An~_!M^iZ7hh*J6gdGU&eiTBNV*Xh
zAZzgU%Je9)o=%r{2AbjO)5a4d?qGPm2S`-EPjVsgIOJmBJ2wH!W$qIK_Jb@8+ek@0
zkxMbvT-!^!wJDs(4#XV+`2w@O3q@&=^o2$ofU;B^FrqcdQ&;@}$MP`i@<x1^2;`o5
zuWG)*PY_?zGX}CTmhe0r_#v9;Y|wCO?v0Y(;JB_y!_KpQplcnuQVPZ-YHgdH;LPqD
z>*awX!1N`E`bPkj#Fs#V@YCn+W2Vm_mA6_-$pZ#~0UNH8n(9!B{mr7upc@k%T}t~S
zYky3_7m3H+VDPy-mIm<Os@J7;V258P^}5th{~dZy^2ZdxNG8!SxiW?4HW!5{hbO?i
zNg$YM4soSZ*akwIGbYp>^0(ex+TvrrWHEaGR3qBY$+x!>vI)dL1>}3ZW*nod#o%-a
zL2=4mE{)tsbob-N-Sc4UMgJV!Z?a!q>CO*9!LL35s4+S;B*%>(@o%!Y1MpPiF4Thu
zKfF4*TN;l$;`(6sfxr~&(asxS@0jt>W^AYkG1x(>oeadrTKTwJZz@)KI-3DA#}BaG
zl;xKr_2xHhe=cZ>u=-zCzI){>cuB1jnpobS%a(eNk%#-3n{EX3qUAtjyDs)ggNj6T
z+wr^ufz6rEN<Lc{J_QRh;c<yy_zEVkTTR**7Mm-V-<VfEH!j00J4|fPC!^+c%0Px1
zijqEySqpYGJioODIczmpFpz=8B4X=6#SwDn=h<-vA?k5_wqV|e?Qn?OboFpc9G+J%
zWZW`<5`F>Aj-aI#(qLDBM>|(FH<17kIo2W!<QWRF%$2<fPtf*LyY*hkva#)6l2+qE
zV^AJMphjC${R3L1kpVQ93Iqy+%`$EpmgieoZuCukDPIcRxTQtiO!Wdzk7eC_fWp^1
z5^iRsV_=t!?3M0VJL!6CBYJfQ*{iZkm%%*Ge&@DE#YULwO*+?1ox6I8YbpqxL-eis
z@n)LGc+`qPLvT8LK1DHReKyfV0<w6$j9`#t>U}cYN%eV3-V);C`|8eR;<%DW+>@{l
z0)lup#a3fP5I{!f4Jh?<cjXp>i)i{xP*Z{B=gq7E)1s``E;2LV!)Bh5-86!ULa*;8
z_;CaAZI?&+06P=vz8(OegAF}e%arakg5>jdB&ghTp)x%#*;WW26)tm|UH_R4#KnQQ
zBa+Att0XU{L?Vh{G-<#$`XOSLh*#P8@HRL#*a4wdq1DkR@ca~*Ggx^ZTTccvON8RI
zj{<mgaa>foAA;7sCZ1B>)@4qCGNAJ9*;D5paCRm*12hpiM%NH-)x%3x4!^C}wuf+V
zEbuWq9hU~>R*Wf+4HWGm-cQfQbC;`C!PzS&#7W+`K&RsM%qU}*?5b%0Lx89~$6StE
zd|tnEt7c#<4|&Zd2}?mlx%8EXiBPXMYI>Q$b{MfO<DLEYi3c>-51QAB9t>1Z-(|mb
z@8#R#H1{SH%jPfS__ML2vg^(!zO7!5XDgoi@VzWIG-aqYkqL7VE;u(+?RDQ$j8uon
z?WAEsx`Wo8n1GC15)mf5qi^Ys`+b#>_eoE+zz?#NrAZ|EqQS(+z&Az9SNw*PkooB~
zQGsq6dQ@Hwu<yv|3eL>h1_zLMBNMA_uvaP^bdDxwrc~NRd|NA~vm?6<qGRH@c&y|V
zlMQKNJ}RpfF`%%IpBp{{^uXv4Q<`C!S3RaY7|#%A!s{dJ(NYgG$}~d2<%HG6bkSJm
z=^&xbXrbF405YeNd-3_IzTKGIH7-JIM4x~<YA{OhhKxZ(&TfI|NfG|dE|`T{FB7`<
zoMquWm_Ah-+J$|ozhlE3kx?olgv*{Wa273HKL0{ViB#}ewOj_?;?S{GmlSpl;!TCj
zbC%s#6BQ>Gj9PJ-&{jW*$0lH(_kJGb>S0!gIo(3;jG<}*)*K}3d!-@#dk5O@(Bee~
zgQKYSs>;V)FQ+hRm}%&(e0~$^`F`+5*@5uz`xI3a6gE8tnG0HHA5~#I_KD>?=`7ox
zXS5ApZ7Y*{FSN#T!}&cd6oqj~%s1?NG;e8fQe17Dy8dwYjG$`;MtDtQkl#%c@`2l^
zuHK_e4Gdx#Lg6$Cch6sJ&6Be-S^AEMHOcf;#f>{SIn{D0Bi>mo43^dOqXn{?okcH;
z3P4k<fAulu@_J9GqTg8?Rw04`j2mT4G^pyDVm16-cxG%@L**&j&XH{yE#9d`d1j$B
zs?d%1l}^@tT_zw*g9ZIX4dL3UOaba?B^`~m%L4J^9qk&i3ge%SOKx3|($O=+EQn?m
z3&inpyK0F+RphGdzX{7Iv5Iiu%~wqrg<8S=?D4O?aO<3_Eahjk*}R?~xA#ddurToW
zQcF{g$JW^?+?DCV@ucq6bnpX9;|HM_W9oxj+HA)J%ko_uJgi9!`dhv{AIwh}KZk!x
zX{3XRd@8=o-;A6AC)SvkL?a+%C06#D{8CBb*hK!1hzWOr=+N?kvG;(|iJoIHIa71t
z2j}BOLKBS?N$2@_<JFwTik~Cs5*nv$qYD$H1U6Mj5Iy}8+{&{tqM6xu^7Go()B_(6
z-^|X>g;O6dJUrK{p5M%>*Eq6}4dw=+zO{L_bRN^rVw;-raVo)tEFDr7on?HSS(GXU
zvFmE`iUk$6^6dh6(Fz4qSq-tp!5T2<vIXLww6=_~_NCG59=0rv_t_HOfTbegzPtb_
z@IU?d5+(WxX`i1`KZ9yETG)?9OQCEt!*bKnt~*sjW1s%MFrB?Fmb+`bU%o@Il3k(?
z+4MN^lnfW|kiI(JP1n?|q+4`MKjaN(PD*~KW6J$5=x83OSRjz2Qa}=cA@D8oHP-i6
z8g|?Qwl6LdD3oiwQE0Xs)v|hIjnC|ZIOvN9K2Ih@m!rsg)LmjzL8Q*5zauY#Xs0g~
z)3ojkzpRND8oKhK9I7HWXEOXSDo(B$$KJAc?p)JW+%!}dOmn&aZIW4eEb4<+g@^K(
z`x*`kp<e0w6_@EZ!t^=1-A=CFrkK-y<nIPHn@DX7rD!LgY`)NI0u`(|MWy4WSNuec
zZSj|YSU+1a<0fHVmkTqVyDE>~bLqcLuxY9>$v;W;AmTYChI`*cyxexuOP(r0%(S>z
z@<^P#PdW*jbqZ+q{R>q|%2L77@l>gKiJc{?op>n?I`&ijlw`KL?TUW37n6<N@O?Xb
zlKW)qoKXB;l6_a680+bp9FIp@J66|O#6B>5xu+<<bIY25q?5OYEjMkNa5y(0w*0o%
z<%E?>uLmr1H+c!Kik>%ckbQ1B!L62RQ27>8Wa=A3BkQ&40kH<d{pbL^8@SLI(^<aM
z!8Ryu*PDxwD)+coFee)%yYig{eQ33!i{1C5J1sZn(!o@`a(0pH^0rX3;B0k&4jOt7
z#4(eTe8-i=Q9+lt>a%Aju?i+*Weq~PGOwJkJd+#Gg;y$dGy1rG-F4jzU$omdYrzex
zR_dg455V`$HsT_!4d^|J=vWla<%>AO4;?|8>T@E|#|%L{D>k|!iJCL|9Go<TJ}{i<
za-6jQHC~Pv*crHDd*x(_FiH-3&I&cRYV9QzyJeSfbUcT1x%uypzg=@Mzw&r2r<6MX
zOW1~1EeHgqyMaF~+28BM4P)U0gUvDHyXf)Xl`<)#lx3RpQ8=;!V%O42m!Eg`jo$LL
zW1wD+4$DmQnhqO@v!hrNK7;Qz(7}c)h~Bf%ZI~imAf0*rn#a}b48~NZgzFYVL=Op@
ztarx}R$_*otp-wYcS`#MMH~xiq+Y&5Df;SJa`kF_*EU@ga}fE-^lVGaV`=!~%p@6o
zr(=xRjH;McD#@=A5E{G3zZMOb{SF>jcTdo?cig@}7P>k^gA3j;WvUrbt8eq#nkS2K
z)l}stPp^mVoBr@mKK57)42$h@JA@<DGaKHe3Rts?(>b0*ETTk(*LD(ij;DvfdMsC<
zqo}E)FKuyYeQGMHf#=F`Rg{l)nMyuT;4x9RP0Y{g?elAH!JQ>;6(*Pq_@v4<cE(8t
z<T<fc@6O&LmLM|djgOwMEn}^UvY#qtAK+``%SB-=&brM>-;onJg6SkLtDt~hX?d=X
zG|E#{EC@Dk@mUJ^atg#$RQWMf;y!L3iKU$ax#9H=kh~fjd`?i7e#k4EA2#9EbP4=)
zl}0RY7iG<xZ8$LeuqMojW1gHP_lU1OW@qIQsF?n#^%I|W3roE~M?}xVe0MH)A9y3l
z%lD4SeN<1MJb5#xSefJI9REYsD%}lVE8@nb1ZB1Bj9)X<9DCfeMDOdOZ0u%hY(h$J
zsW!mIWO+G)JxS}-^R9rGV?P#$q`QOGnVvmr*In*PjtZX}8GQ5AA0q<Oo-7FC$GHo(
z211PD!I$NN3(Pfq9!5D1oU^oxPI<NbO8nJzKPVy7d*aqA)t=<`>6p(ph8`J3{nAy3
zW}q3TF_v~Bi%;W>?pb~4-hsR#wL@W6)jm@0qe7hmzG{&^?1EfTnH}GIq1@HSpgUd}
zyC${)eo9-gY2|Z#|JX&nqq-k$NOQyNUh6;{0YQ;KZ072MoWy!`>G=eeZ!HJQ!L6XA
zD?PvnW$-STsSZV<FYkIlA7L=*v&@+?xGrVlA*DMJ+0@HXK{e2t@@Anr<54adL5`U0
z_0_dTk`s2DV`z~LgH`LK7F$sZRElWR3AQqd-P+=jpWL412t`sACW<*mEC}evph*%D
zHnj0vstAa68aiWzl;J-A?0A*dj<JnugrtP}pa2=KKMUKQ_M?ca40BJ^mQigM?}fBg
zf151Bzdu)Rc}eObzpgCvw8{jSVB|lG94#MH_lmC16C2d%Z%I-P7H~r=TIb_>d7e7V
zb(_^_vV(F$ecdzt%oe+mO@N~L4sXBg>rXD*kwI%qrKyy!Ro}g=IL&vXw`^?8sP!PX
zI?c*tf2&t8k$Qr$Z!S$Ow&0Uurc`#&uUu0cK;_#;xGl-ya_q9JAF|I!Rs#sH`thtZ
z99j~ozSFuW0sm*ncuBf9;wAi{J#~>T&pHbyRF1_@*GE)oWc_~Hop3Lud5XA$q_ET5
zlPQT!V$DPw!#5XH^(%OU(Z-DOwPByKmG8Qu)ewg-ym(f}id#rqQ3{u^CEm)^W^UwO
z#<|(X%M`{<gk(8_c7ahb$CBZwur`zo6eYK9w6+FCBsho6qs_i~(Klaa^<|TNhKon~
zHJjTP8)u(Jt%}^@%QP`6b0c3oK1aTDp+<O9I=LoscUexy%8&89(ic6-eTC&cO~nkS
z&wFshr@>wv)!z@05<H)abFMQ(r#FwG(?3twP&azRQZsZQJnw2dAh&eZ&bViCe(>h&
zFX)0k57h$ZoZ3nmudS5Of1kT2wM%2~n>kKMAGjtse<!oIA<ZT1dXP|}c;(NdP*E;m
zkG*3Q_E{$w;Vf;V3-+$Qr~CqED6$^UPbO7hV&?UEeu|vBn6*jor#Y>ZU$xcVsJ(8d
zBbcmy3x-5Bj-O)AW_ej#>0mPnSfqz7F}f_H;PhfdQw%MOx!99<^W~75bNP;pW;vuN
zVw1mMl+B+iXKs4T%Z!fMhY`Si@`;fxg`p4^-NxgkAlulMsPM$bsDd->qXX;dw_gg4
zNb4HhU%UG*g>9oFM+sk|v*m`Br=1Vu(3QTv`-(kS!gdy_jb403gXu!nartgdZuVVU
zM0RQsn8&KKhs*l!Ka*%K9Ey$T8eVwN@Qh7x$(s1u7?v=|mE}sb;9zhQ=p(Cn1J)%Z
zvX^pL-MN%uJhn<!&ho!K!N2jbBdSt}CxB;CqI`W`+_>~h^P?s1L<xyGgWjj5eY*_n
zcGDpohUq!0q7%4ZCg~iV7wj4m9~?Az`FF6qICGm!s`~b$KD8wl<L|vChv?hFF+i+N
zm|zxg1D}Bjr+)eS*s^2#9WTO1-Q-4}77BItti&`EE>W>sXsMaSKh|-zPS#7h(<yM=
z=gymvM$cH3_yqMc_I=53OEmiOC?BuKR?;7vWA$yAaaYKuvhW?j2)88eV|poSltJHA
zkghtaSZIKzQXsHc-O7HNO=|jv7AwzXkWiDOB}NXt@5m|6cXaG{^aayibI~A-zn(vu
zc{QBwuC+n5qHo3aj8wOCHaW*F?T+LnyL`dNYm!%OeN!-SB%?hn<!zrG?C-|%<L3@|
z300tHla*JKwL47{m@3V7<dv6Ib5<)meAiRvmV4N*Sf$B(+Wk2k%)4fdfG&xRk;`au
z!Iwxxz(qRYY>FV~g!_+EEqaPSqDY_M*>*Ng;!iYxiB33=r|`gOwN%a%5rt<+wPH?8
zt`XsRp_+F|8>0}bCTQ?DUlhQbsgJIhQzXt3tbQ74l|pHq*&P=kp)$gX(eHEWn0~-g
zJ&?I@PzzR*PThGX%eL1$%Ab)Z#;wRvc_rrb<gL;^>w*keTR#(IzC3NlY?wTJn_1mQ
zF_)}7yXxbxn(FEJ1Oo95HA*7>#E{6ryrzzmSi&NNfdaLQsdDeyDVp5`Kj`AHVpjm1
zz+$=LNu;9sF;JF|a%GMvH&Wm%ld7MqMOZclmA*Xb+wcjke3bkVe!-8d-H6zykI6AN
zf7^j~R191j^SDY5?%=d*#gyV|O`qh`-cjJDBp>DM7uZb6fG8t<(RZ2;M69;cB{O@h
z_*vWSCvGdJLul>t;`^Jo7%jbCyb&Xw@YAl?2)yP-+V3@>TZQM;)up;6u^Pq+G`gpR
zJrbaBe6}Fm;bZ(IH<`h;-Rq+LLWGx-r{wo+DL1c~!B^)B&OQCO<^<KYnHZ<^2RX}5
z#2kC194|B?<~^;1UAjErDuy$#9osUmPAvaTg4ae-?YdbIz7P$5F&RZ<mV^0pY+G_~
znFB4rp&g0_*s}Mm42CppE_ZXTMrxFq2+a!!>pQUz_3?&N)Z3@*g!bt?E>G@xT(Pl8
zA717-hbc4Svl!<Pd`hGDVr0z~&q=KqTepr@pcioI^uMKHdY|RZlU}W=5SD64<2tkI
zxF@OIQgaurw=6YLBHquolBgZ2ULY@Ru|B6b(d!#?qOS=*`MWYMr2jvn&E%Q=w&=2M
zmg44%$i~0b7}@ygSLmh&XF@?+tC{`=gS0&OG<(Ax`&sA4!pZgXi}D|y7SU2{+?g|u
z&~5Kke$5qCz+QMqAdRVyDKg!qCZE*lUhW%V!)R$ci;D$<d0}*xcQLFAM;E5fT;L<k
zI*I;vGF_;YvBwg{I6+ln+p>l0Zv5i8liZmVG7gPj7;Xy?h7xkQcU^8(#AOJ*THnTU
zl54$#6<;zT#_AN4Qj%0`-zUlhirev(FKqP{-?WevN0UCn#U`{+(qapBJx|obdlid)
zq|m`I?8;7qqz>I<>Knn))tD?)da35U{d|-@&bnEfZf>%nTpa>?c@oYl511PSM5<@<
z4s!de!bJ&mO-muecN+_FYEyh$5?004?bE<;uFsC2l$EUii#4kiS0MGz(;7K&b9_)u
zX7@=#)-6vW4?Vkx&z#|_c9UHf_>F7#<Q9gY<ZGy$iwH;Q9!l~Jw&>I29m3KU5zsd7
z^Na8BNdGh-c30aYza~sYU!C#NadlCvKHf;RKyJ*;Fro{6MRD_QLq)vl8--bTh1EJ@
z3ux_c<e`{N(CDm7WcMk`(-H0j@OCrFnbA<t@$w-8NV|HKRPh}#ekG@mU08XGuZlUo
zP%-SAP1A*JJr+l^&?nZNx3@|bOgxbmJf`GS|Exf+k5GMHHdg(X2aak4;b$f+rm)i#
zC+?0bL@if$Gdu)3j|9OQ2M-SJHvb1^{PX_6?dp9UYcKHqr=MD0JinvQq@v8=08!w;
zt)P3&+kAKNd_C<%JAFpOz7PdhE`&Z@%ubd)dp}<}EX~{I!h%LNI67R8Wy3ZnZZy%R
zL@k`Fp;GE?@{Lh6Jiymq9F5k_^rX6$nb-I+{bS4AjqU)|!OG_`lotE;eN$p7Kb$jl
z><42zR&V{9_}~QC8@vCVubXM10KpA5lSP>cJK_Q67T4?xxE5DOHk(+9PaRXH(bgLN
zWKG7WRsPjW?o6Ab2b|{k&$64?sfR~}I7jycS1Uc5W=TDL5=~nJ2%TOVuTJxC=avxx
z(IUUu=vy5}s<%h7!YNzF9>U&W-mdI=RS)(~0a-?`!og|r#8-22x0c8`Qc_;ZEA2zp
z*p^&-tmz#?hJoNt0jv*66tZVq7e7-_Njzu8Mj?-2b^j2qEHJ`Im;IdGg_7QXJ$C%I
zRY#+<)$PIKbFL-x7Nm6M7rA;7{pmd&*|cfqMA~}h2SfRZD$Cy9^?ZRW%L5&yxYcNR
z%pW+au5CVXn3hmr)`rfTMRubnHTtzvxvGA|?RoH%h+Wg}_xhAy$8`wgyFEZTAbz<{
zJU^g#V(;ZOCHZduFBO{iMj8n-FER;bKT@=PopAZetVQ@ERf#h6aLfGl>rlX|xRn0Q
zu*>VM$ZADCp=Q&_!9*R^xnU37ven&7n3REJ+h4;~ZW}*fj%mp?vQD;o#~3mH43T<w
z(`L^{KXX7U*u@geU%1CVd+rg{OeP^_bowhg?a_hBQil_~2?PWW$^tl3Gt%RgHd~*&
zmiKX7GN&|@4Sl5GXm>)r>Y^R?=;fENjZ3^3Vqm2O^4!ZO1XNj+sohf@7;)IfDX-Fn
zXFX!2w&P4LoSzy(c_ylC&;Gf`5S{LvVuN}=fne9*?6ALNly^!~eEP!?i|n&Ocvs?#
z9wun=o60gPg7=Yv%siTkV+U)lQnPj7@+He2;gXS}g|Sp+P2UO}ccC{Yja7<Gy`=lb
z9vN1vPG@`@i^?j3^Qza&NpYdOSsUZWSM9yq8@@UAG`-$YrKuU;^E6J6MjNMr)10fJ
z{Q3Kn&fkjr+H;$g3aB5?m#<T%7ohJ)S~Bc7OnFmN$CF~D@lIeHdZ*7_NM>@3Z;^Y%
zt+Syz<LD%2_|;j#mtdF3Imh~%7&~!|hG6}L=Wn`4(7>n{hxa2>WTzOF^|{#u@4=iA
zBCDA`{+I2M^sGEIliX+7lgkn+HFtd0YC`7xv6H<bhl`t9#$rca2@v(B#F?2Uok+*V
z1NQA`neG5CM%~zUS@K82GgLCEJq7Vp$!@$gqXK6*&--yF516H1(8m_mJmpkVEOX^Y
z^M_T1y>5x^stZr#C#ZVy;<ug$*^Eb#A1r=(d4I`eg^(plBxmEhElXp)!dR29#vA+*
zLQ`o^s^TUJ(n#~K_VFPbKXltjH(k$ROl*c7|9H?*Q+q$%)Ds8b*&UPSi=OG0lZkhG
zqgPnmG#@=V>CTZof7y2q&Iy`)VziT!0@;{Eh1Kb04ziQbX%b1PzV~AgR4DstWjsd{
zlSxp~roQwTt-K?H#toHml)-9*%u_jy<RrJG@+ZAdTz%~EDHDo))+3E;G4~#prWO=s
z75kA+v7PtuAU-%~CXsl+Da(EKb=YoJ#pzNU75LJhPkE6xsK(=sXCbVH-fpz}8t<nH
zrTO35o^BEQWJgq?#pGiodtR@Asvt7t4NHnT7!XHf^LuDqYm7=~H9SSKK!9re#F<V^
zO0#i<+NMdL=ZPua97$vQ_ik@*I&roS_#>I8dd!id+HrbFN~W$}cs>O0<le<z-8;s5
zN?+;@R^YsuXfFl$d~&KH2361E{Y>I1J6f=NA9Eb*&x(m%6H<7y<rn@8Q8MuF&);P)
z+4|B*&?<J9O2UlbiPTjz-e<J<tc#`;^vk`eg?Sfz4cHC7-WWKMK>a<pL8ztrQKCV2
zg?)--`VBTU?Yh&YY>}rA7d1-arwWc1#-HU83(fUcU2cwUndX)~{n8F_@Z2?~Rw?l^
zL|qh%HQ6xoEVoi&ovKhhs3oMo8iioDm$>N%rW0oz@JmOo^WEO$BT45~*N9w5KcA>&
zI!eQ%fJeohsY$mSDLjpp!=?cKMy3py)wFUN?=2A8kS{l=s^+<QskB0wlUq`J+O9A%
zWH$2+LCKpdgj61diSZLoO(o&CJ3s7?(4I+t9VJTaQ<WXaTR1<XgITE`&}VLYg0PS2
z>pJgeIVXc?C$5|ErM$%4K@sZLFH>Gz7`Azk@5*tGRZSC~mMQutfoTbmF^fKuWub^n
z-T0W55R@-SArLdaICnxRh@~=+*B^zSg71{c_2gH&t(Z6z6h$*O>JF&n#`jR|@>^{1
zioqzh+(+}PIl*|%m@T$HxJC1QVl&wWd-4Y&D_76n)71hwC)2g)Bx=NmmZdjJiujzR
z^meLVpgHD`+EP7L<QNz^`wif~yi|%d&ZyRwSY4&k-v9ILRV^>zK$on;&iYUlziSmc
zYL?^oH}MJ>DJ+VRsIe$~{g>>CoC!4S|H(B1{w4{j#lac0dd%EWnJAR{s0+AGeuw%9
zjSLm@E{IAw6#H-ft2qSGiO+lq`bo$6eS0Z&z)UVFq(mNVuH%8-sM(qScJ@ECuK%w)
zo2MWzybG)tcJz=cEW}Jo3p2zUg9aymF~JPm>wOPrPhj<#9MZuCJ1YLAm9CuMC1qT+
zuvaa<gnDqw_Ru^l!!B@TyS8*u(^(F)QU@UKM8Pzp8@`-LXhzIDd>1a!`j0M^ebsZn
zsI5R2qD}Nl;TPrdm_F=0f^L2j_dG)H%ZPwjc5D*7dZg7f3b+!-x0i1GvRwZ;wPZ-0
zhUWL7SBHO(gu^#oM*}3x6YE2pW51r~C(AQX2<Uj_C*KwwZM^{c(K8<lS1bNu&L7^)
z5pf5jFVR2R8pHuJQk(ANb;JPu&)6JAcIC2Hp8sT3{QeO<VZimMCpGhjwN(z!0g=DT
z)M2uVcx8<k#Qb^=_!nL&;um4!`^M<+8<W^(!mM%HrQ6_0E7+!>_gtviNw50*7A(E8
z&@42a^5RG<PaE{;e>?hjZ~sqpbmnN%ONb;Xe{4K_{D^nh-bKUta{W)g^{4!v$$<tf
z_={qVLk1(#tzIO>2z)0f_;VtAS~Pzxj8Mo3;b{&k`mG&1;yUq1T6r65%ioGN-1@PU
ze&EF#lVRtC7w(4`+wbQGw5<>Vq8O*L_DGmyNa;iLlydw;GXE^lwus$?_Jz%(5l_Mb
z{xbRre-~>x4CZ2FjIk~3{}tItj9bAt?JRwzaU_b>GPL0nbuh;fTbDl{sFqO#F|SXi
z`!6nS85<0;f{fjxBN8VLs&a62PAeX{g;bype4-(#oBzL>OX1P2^soF8uO1G|-U9f<
z!dokU1RsaZYC7=f9QXbb6dis|1Eda!cJJPidE|)e<Kfjm|N4KYv%egSKDQtdxkZC=
z3vX=MWB(wX^rEwlOUqE!qt9Pt?s4T?bSNaCprU04d^n?jhKNQg0RDr4MF{_>(~)v^
z!W?J*s|h7gUCQ}B#|doY=f8i^l$}QS<*gOsznTK-SZIXh$nN~S(XU^8N=le!cwzJC
zwIqaT^!KEHed5nwe20V8Qw$}F55L5(Yk8n8x|#O(S37KVz{Hr>>GeHwEjhHcO-KE@
z>gRVXN4s^MeL>>Lwc*fa=_3EnGa&ykM6ItNb$fW^T3OV3ky2rXKMxy#5?CDYVGO5I
z^~kmVk=<8Qjy}dLMML(ggnj>1_@DP2g|&5?-%K#BjvnpF+vRAHki)kfNI>>#bvMXl
z-VS5mGjMs&@7lzd)t8C0QQiJ^Mn_(P*Od>^!(VadXC|Aaj{IN?BIPsckr%;$b_@Ba
zBb|<sOZ9gwp#;iEIWrZ790|^V4`t{_A|wh6t0#C!?a1d5rg<EBOBh6`^~p~u6^}ID
z8aVNHGzUo0(FhakNFNDPDd*86-<nV!1H-1lp5;g*mdg3R;{+8gEZ~EZ9zNrd#(KNs
zf5){H%}K&E?7~M!9tAq-NPl4cx0C+cN&n{}spc&0tCD|)`R|afYsGT#pSt<lnA&2B
z_DFbs;dGwd+<fT2c?TwegOwb^BR`e;a6Irccba7QcikiH&ZPk##I~1jod5j+NQhrV
zt*4l#RQ_u`2FAh+-flAa{_pWe2(v%oTid_Kb{Yqbu<HKKgv#I7s!2JAypsKUA^Kp6
zTJOF7cGKhUYlC6VX%bBNdyFyTV)ew<*1Y`tS{hiJXiNJ3ehOiIDd+Sb-(ElX``SkV
zAMjivnEo2(b?s=kzJ7eN`uXo`u`n@gLaz$^{rD<mglWwFzSsUty;1>EuyRUY$^85A
e|M$A?2dV*+bTK1SWh4sxQMjZcQ*zPt*8c)ByOAgW


From 0a8979d3ab80374eb4f84d08e060bcec70174c73 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Fri, 25 Mar 2016 16:52:11 -0700
Subject: [PATCH 0043/1644] ARROW-37: [C++ / Python] Implement BooleanArray and
 BooleanBuilder. Handle Python built-in bool

This patch implements `arrow::Type::BOOL` in bit-packed form.

ARROW-59 (Support for boolean in Python lists) is also addressed here.

Author: Wes McKinney <wesm@apache.org>

Closes #40 from wesm/ARROW-37 and squashes the following commits:

f6b60f1 [Wes McKinney] Clean up template instantiations to fix clang
2d3b855 [Wes McKinney] Use gcc-4.9 to build thirdparty, too
74f704b [Wes McKinney] Try to fix clang issues, make some PrimitiveBuilder methods protected
dc8d0a4 [Wes McKinney] Fix up pyarrow per C++ API changes. Add boolean builtin conversion / API support
2299a6c [Wes McKinney] Install logging.h
2dae07e [Wes McKinney] cpplint
0f55344 [Wes McKinney] Initialize memory to 0 after PoolBuffer::Resize to avoid boolean bit setting issues
83527a9 [Wes McKinney] Draft BooleanBuilder and arrange to share common code between normal numeric builders and boolean builder
---
 .travis.yml                                  |   4 +-
 cpp/src/arrow/api.h                          |   1 -
 cpp/src/arrow/array-test.cc                  |   5 +-
 cpp/src/arrow/builder.cc                     |   7 +
 cpp/src/arrow/builder.h                      |   2 +
 cpp/src/arrow/test-util.h                    |  31 +-
 cpp/src/arrow/type.h                         |  15 +-
 cpp/src/arrow/types/CMakeLists.txt           |   1 -
 cpp/src/arrow/types/boolean.h                |  32 --
 cpp/src/arrow/types/construct.cc             |   3 +-
 cpp/src/arrow/types/list-test.cc             |   5 +-
 cpp/src/arrow/types/list.h                   |  11 +-
 cpp/src/arrow/types/primitive-test.cc        | 214 ++++++++-----
 cpp/src/arrow/types/primitive.cc             | 179 ++++++++++-
 cpp/src/arrow/types/primitive.h              | 302 ++++++++++++-------
 cpp/src/arrow/types/string-test.cc           |   2 +-
 cpp/src/arrow/util/CMakeLists.txt            |   3 +-
 cpp/src/arrow/util/bit-util.cc               |  14 +-
 cpp/src/arrow/util/bit-util.h                |  13 +-
 python/pyarrow/includes/libarrow.pxd         |   3 +
 python/pyarrow/scalar.pyx                    |   6 +-
 python/pyarrow/tests/test_convert_builtin.py |   5 +-
 python/pyarrow/tests/test_scalars.py         |  39 ++-
 python/src/pyarrow/adapters/builtin.cc       |  26 +-
 24 files changed, 643 insertions(+), 280 deletions(-)
 delete mode 100644 cpp/src/arrow/types/boolean.h

diff --git a/.travis.yml b/.travis.yml
index 49a956ead3dca..d89a200b892e6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,10 +20,10 @@ matrix:
     language: cpp
     os: linux
     before_script:
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
-    script:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
   - compiler: clang
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 7be7f88c22eb6..2ae80f642f29d 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -27,7 +27,6 @@
 #include "arrow/table.h"
 #include "arrow/type.h"
 
-#include "arrow/types/boolean.h"
 #include "arrow/types/construct.h"
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 7c6eaf55c0d0f..121b802d994fa 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -71,8 +71,7 @@ TEST_F(TestArray, TestIsNull) {
     if (x == 0) ++null_count;
   }
 
-  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(null_bitmap.data(),
-      null_bitmap.size());
+  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(null_bitmap);
   std::unique_ptr<Array> arr;
   arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_count, null_buf));
 
@@ -82,7 +81,7 @@ TEST_F(TestArray, TestIsNull) {
   ASSERT_TRUE(arr->null_bitmap()->Equals(*null_buf.get()));
 
   for (size_t i = 0; i < null_bitmap.size(); ++i) {
-    EXPECT_EQ(static_cast<bool>(null_bitmap[i]), !arr->IsNull(i)) << i;
+    EXPECT_EQ(null_bitmap[i], !arr->IsNull(i)) << i;
   }
 }
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 6a62dc3b0e08f..4061f35fd5e53 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -54,5 +54,12 @@ Status ArrayBuilder::Advance(int32_t elements) {
   return Status::OK();
 }
 
+Status ArrayBuilder::Reserve(int32_t elements) {
+  if (length_ + elements > capacity_) {
+    int32_t new_capacity = util::next_power2(length_ + elements);
+    return Resize(new_capacity);
+  }
+  return Status::OK();
+}
 
 } // namespace arrow
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 308e54c80d794..d1a49dce79961 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -69,6 +69,8 @@ class ArrayBuilder {
   // Resizes the null_bitmap array
   Status Resize(int32_t new_bits);
 
+  Status Reserve(int32_t extra_bits);
+
   // For cases where raw data was memcpy'd into the internal buffers, allows us
   // to advance the length of the builder. It is your responsibility to use
   // this function responsibly.
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index ea3ce5f7f53ff..b2bce269992d0 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -98,28 +98,27 @@ void randint(int64_t N, T lower, T upper, std::vector<T>* out) {
 }
 
 
+template <typename T>
+void random_real(int n, uint32_t seed, T min_value, T max_value,
+    std::vector<T>* out) {
+  std::mt19937 gen(seed);
+  std::uniform_real_distribution<T> d(min_value, max_value);
+  for (int i = 0; i < n; ++i) {
+    out->push_back(d(gen));
+  }
+}
+
+
 template <typename T>
 std::shared_ptr<Buffer> to_buffer(const std::vector<T>& values) {
   return std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(values.data()),
       values.size() * sizeof(T));
 }
 
-void random_null_bitmap(int64_t n, double pct_null, std::vector<uint8_t>* null_bitmap) {
-  Random rng(random_seed());
-  for (int i = 0; i < n; ++i) {
-    if (rng.NextDoubleFraction() > pct_null) {
-      null_bitmap->push_back(1);
-    } else {
-      // null
-      null_bitmap->push_back(0);
-    }
-  }
-}
-
-void random_null_bitmap(int64_t n, double pct_null, std::vector<bool>* null_bitmap) {
+void random_null_bitmap(int64_t n, double pct_null, uint8_t* null_bitmap) {
   Random rng(random_seed());
   for (int i = 0; i < n; ++i) {
-    null_bitmap->push_back(rng.NextDoubleFraction() > pct_null);
+    null_bitmap[i] = rng.NextDoubleFraction() > pct_null;
   }
 }
 
@@ -160,11 +159,11 @@ static inline int null_count(const std::vector<uint8_t>& valid_bytes) {
   return result;
 }
 
-std::shared_ptr<Buffer> bytes_to_null_buffer(uint8_t* bytes, int length) {
+std::shared_ptr<Buffer> bytes_to_null_buffer(const std::vector<uint8_t>& bytes) {
   std::shared_ptr<Buffer> out;
 
   // TODO(wesm): error checking
-  util::bytes_to_bits(bytes, length, &out);
+  util::bytes_to_bits(bytes, &out);
   return out;
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 5984b6718ddbe..86e47791b7cea 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -132,6 +132,10 @@ struct DataType {
     return children_.size();
   }
 
+  virtual int value_size() const {
+    return -1;
+  }
+
   virtual std::string ToString() const = 0;
 };
 
@@ -191,11 +195,14 @@ inline std::string PrimitiveType<Derived>::ToString() const {
 #define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)  \
   typedef C_TYPE c_type;                                    \
   static constexpr Type::type type_enum = Type::ENUM;       \
-  static constexpr int size = SIZE;                         \
                                                             \
   TYPENAME()                                                \
       : PrimitiveType<TYPENAME>() {}                        \
                                                             \
+  virtual int value_size() const {                          \
+    return SIZE;                                            \
+  }                                                         \
+                                                            \
   static const char* name() {                               \
     return NAME;                                            \
   }
@@ -295,6 +302,12 @@ struct StructType : public DataType {
   std::string ToString() const override;
 };
 
+// These will be defined elsewhere
+template <typename T>
+struct type_traits {
+};
+
+
 } // namespace arrow
 
 #endif  // ARROW_TYPE_H
diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
index 595b3be6e1661..f3e41289bfe8d 100644
--- a/cpp/src/arrow/types/CMakeLists.txt
+++ b/cpp/src/arrow/types/CMakeLists.txt
@@ -21,7 +21,6 @@
 
 # Headers: top level
 install(FILES
-  boolean.h
   collection.h
   construct.h
   datetime.h
diff --git a/cpp/src/arrow/types/boolean.h b/cpp/src/arrow/types/boolean.h
deleted file mode 100644
index 1cb91f9ba4966..0000000000000
--- a/cpp/src/arrow/types/boolean.h
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_BOOLEAN_H
-#define ARROW_TYPES_BOOLEAN_H
-
-#include "arrow/types/primitive.h"
-
-namespace arrow {
-
-// typedef PrimitiveArrayImpl<BooleanType> BooleanArray;
-
-class BooleanBuilder : public ArrayBuilder {
-};
-
-} // namespace arrow
-
-#endif // ARROW_TYPES_BOOLEAN_H
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index df2317c340b2d..34647a5005b90 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -51,7 +51,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(UINT64, UInt64Builder);
     BUILDER_CASE(INT64, Int64Builder);
 
-    // BUILDER_CASE(BOOL, BooleanBuilder);
+    BUILDER_CASE(BOOL, BooleanBuilder);
 
     BUILDER_CASE(FLOAT, FloatBuilder);
     BUILDER_CASE(DOUBLE, DoubleBuilder);
@@ -83,6 +83,7 @@ Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
     int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
     std::shared_ptr<Array>* out) {
   switch (type->type) {
+    MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
     MAKE_PRIMITIVE_ARRAY_CASE(INT8, Int8Array);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT16, UInt16Array);
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index eb55ca868eeee..4eb560ea52256 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -116,11 +116,14 @@ TEST_F(TestListBuilder, TestBasics) {
 
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
 
+  EXPECT_OK(builder_->Reserve(lengths.size()));
+  EXPECT_OK(vb->Reserve(values.size()));
+
   int pos = 0;
   for (size_t i = 0; i < lengths.size(); ++i) {
     ASSERT_OK(builder_->Append(is_null[i] > 0));
     for (int j = 0; j < lengths[i]; ++j) {
-      ASSERT_OK(vb->Append(values[pos++]));
+      vb->Append(values[pos++]);
     }
   }
 
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 72e20e943c347..8073b5121764d 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -116,7 +116,8 @@ class ListBuilder : public Int32Builder {
       int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
     }
-    memcpy(raw_buffer() + length_, values, length * elsize_);
+    memcpy(raw_data_ + length_, values,
+        type_traits<Int32Type>::bytes_required(length));
 
     if (valid_bytes != nullptr) {
       AppendNulls(valid_bytes, length);
@@ -132,13 +133,13 @@ class ListBuilder : public Int32Builder {
 
     // Add final offset if the length is non-zero
     if (length_) {
-      raw_buffer()[length_] = items->length();
+      raw_data_[length_] = items->length();
     }
 
-    auto result = std::make_shared<Container>(type_, length_, values_, items,
+    auto result = std::make_shared<Container>(type_, length_, data_, items,
         null_count_, null_bitmap_);
 
-    values_ = null_bitmap_ = nullptr;
+    data_ = null_bitmap_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
 
     return result;
@@ -162,7 +163,7 @@ class ListBuilder : public Int32Builder {
     } else {
       util::set_bit(null_bitmap_data_, length_);
     }
-    raw_buffer()[length_++] = value_builder_->length();
+    raw_data_[length_++] = value_builder_->length();
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 10ba113c5916c..761845d93812a 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -69,11 +69,11 @@ PRIMITIVE_TEST(BooleanType, BOOL, "bool");
 // ----------------------------------------------------------------------
 // Primitive type tests
 
-TEST_F(TestBuilder, TestResize) {
+TEST_F(TestBuilder, TestReserve) {
   builder_->Init(10);
   ASSERT_EQ(2, builder_->null_bitmap()->size());
 
-  builder_->Resize(30);
+  builder_->Reserve(30);
   ASSERT_EQ(4, builder_->null_bitmap()->size());
 }
 
@@ -83,8 +83,9 @@ class TestPrimitiveBuilder : public TestBuilder {
   typedef typename Attrs::ArrayType ArrayType;
   typedef typename Attrs::BuilderType BuilderType;
   typedef typename Attrs::T T;
+  typedef typename Attrs::Type Type;
 
-  void SetUp() {
+  virtual void SetUp() {
     TestBuilder::SetUp();
 
     type_ = Attrs::type();
@@ -99,58 +100,44 @@ class TestPrimitiveBuilder : public TestBuilder {
 
   void RandomData(int N, double pct_null = 0.1) {
     Attrs::draw(N, &draws_);
-    test::random_null_bitmap(N, pct_null, &valid_bytes_);
+
+    valid_bytes_.resize(N);
+    test::random_null_bitmap(N, pct_null, valid_bytes_.data());
   }
 
-  void CheckNullable() {
-    int size = builder_->length();
+  void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
+    int size = builder->length();
 
-    auto ex_data = std::make_shared<Buffer>(
-        reinterpret_cast<uint8_t*>(draws_.data()),
+    auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
         size * sizeof(T));
 
-    auto ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_.data(), size);
-    int32_t ex_null_count = test::null_count(valid_bytes_);
+    std::shared_ptr<Buffer> ex_null_bitmap;
+    int32_t ex_null_count = 0;
+
+    if (nullable) {
+      ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
+      ex_null_count = test::null_count(valid_bytes_);
+    } else {
+      ex_null_bitmap = nullptr;
+    }
 
     auto expected = std::make_shared<ArrayType>(size, ex_data, ex_null_count,
         ex_null_bitmap);
-
     std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(
-        builder_->Finish());
+        builder->Finish());
 
     // Builder is now reset
-    ASSERT_EQ(0, builder_->length());
-    ASSERT_EQ(0, builder_->capacity());
-    ASSERT_EQ(0, builder_->null_count());
-    ASSERT_EQ(nullptr, builder_->buffer());
+    ASSERT_EQ(0, builder->length());
+    ASSERT_EQ(0, builder->capacity());
+    ASSERT_EQ(0, builder->null_count());
+    ASSERT_EQ(nullptr, builder->data());
 
     ASSERT_EQ(ex_null_count, result->null_count());
     ASSERT_TRUE(result->EqualsExact(*expected.get()));
   }
 
-  void CheckNonNullable() {
-    int size = builder_nn_->length();
-
-    auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
-        size * sizeof(T));
-
-    auto expected = std::make_shared<ArrayType>(size, ex_data);
-
-    std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(
-        builder_nn_->Finish());
-
-    // Builder is now reset
-    ASSERT_EQ(0, builder_nn_->length());
-    ASSERT_EQ(0, builder_nn_->capacity());
-    ASSERT_EQ(nullptr, builder_nn_->buffer());
-
-    ASSERT_TRUE(result->EqualsExact(*expected.get()));
-    ASSERT_EQ(0, result->null_count());
-  }
-
  protected:
-  TypePtr type_;
-  TypePtr type_nn_;
+  std::shared_ptr<DataType> type_;
   shared_ptr<BuilderType> builder_;
   shared_ptr<BuilderType> builder_nn_;
 
@@ -158,14 +145,14 @@ class TestPrimitiveBuilder : public TestBuilder {
   vector<uint8_t> valid_bytes_;
 };
 
-#define PTYPE_DECL(CapType, c_type)             \
-  typedef CapType##Array ArrayType;             \
-  typedef CapType##Builder BuilderType;         \
-  typedef CapType##Type Type;                   \
-  typedef c_type T;                             \
-                                                \
-  static TypePtr type() {                       \
-    return TypePtr(new Type());                 \
+#define PTYPE_DECL(CapType, c_type)                 \
+  typedef CapType##Array ArrayType;                 \
+  typedef CapType##Builder BuilderType;             \
+  typedef CapType##Type Type;                       \
+  typedef c_type T;                                 \
+                                                    \
+  static std::shared_ptr<DataType> type() {         \
+    return std::shared_ptr<DataType>(new Type());   \
   }
 
 #define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
@@ -176,6 +163,14 @@ class TestPrimitiveBuilder : public TestBuilder {
     }                                               \
   }
 
+#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)      \
+  struct P##CapType {                                   \
+    PTYPE_DECL(CapType, c_type);                        \
+    static void draw(int N, vector<T>* draws) {         \
+      test::random_real<T>(N, 0, LOWER, UPPER, draws);  \
+    }                                                   \
+  }
+
 PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX);
 PINT_DECL(UInt16, uint16_t, 0, UINT16_MAX);
 PINT_DECL(UInt32, uint32_t, 0, UINT32_MAX);
@@ -186,25 +181,89 @@ PINT_DECL(Int16, int16_t, INT16_MIN, INT16_MAX);
 PINT_DECL(Int32, int32_t, INT32_MIN, INT32_MAX);
 PINT_DECL(Int64, int64_t, INT64_MIN, INT64_MAX);
 
-typedef ::testing::Types<PUInt8, PUInt16, PUInt32, PUInt64,
-                         PInt8, PInt16, PInt32, PInt64> Primitives;
+PFLOAT_DECL(Float, float, -1000, 1000);
+PFLOAT_DECL(Double, double, -1000, 1000);
+
+struct PBoolean {
+  PTYPE_DECL(Boolean, uint8_t);
+};
+
+template <>
+void TestPrimitiveBuilder<PBoolean>::RandomData(int N, double pct_null) {
+  draws_.resize(N);
+  valid_bytes_.resize(N);
+
+  test::random_null_bitmap(N, 0.5, draws_.data());
+  test::random_null_bitmap(N, pct_null, valid_bytes_.data());
+}
+
+template <>
+void TestPrimitiveBuilder<PBoolean>::Check(
+    const std::shared_ptr<BooleanBuilder>& builder, bool nullable) {
+  int size = builder->length();
+
+  auto ex_data = test::bytes_to_null_buffer(draws_);
+
+  std::shared_ptr<Buffer> ex_null_bitmap;
+  int32_t ex_null_count = 0;
+
+  if (nullable) {
+    ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
+    ex_null_count = test::null_count(valid_bytes_);
+  } else {
+    ex_null_bitmap = nullptr;
+  }
+
+  auto expected = std::make_shared<BooleanArray>(size, ex_data, ex_null_count,
+      ex_null_bitmap);
+  std::shared_ptr<BooleanArray> result = std::dynamic_pointer_cast<BooleanArray>(
+      builder->Finish());
+
+  // Builder is now reset
+  ASSERT_EQ(0, builder->length());
+  ASSERT_EQ(0, builder->capacity());
+  ASSERT_EQ(0, builder->null_count());
+  ASSERT_EQ(nullptr, builder->data());
+
+  ASSERT_EQ(ex_null_count, result->null_count());
+
+  ASSERT_EQ(expected->length(), result->length());
+
+  for (int i = 0; i < result->length(); ++i) {
+    if (nullable) {
+      ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i;
+    }
+    bool actual = util::get_bit(result->raw_data(), i);
+    ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
+  }
+  ASSERT_TRUE(result->EqualsExact(*expected.get()));
+}
+
+typedef ::testing::Types<PBoolean,
+                         PUInt8, PUInt16, PUInt32, PUInt64,
+                         PInt8, PInt16, PInt32, PInt64,
+                         PFloat, PDouble> Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
 #define DECL_T()                                \
   typedef typename TestFixture::T T;
 
+#define DECL_TYPE()                             \
+  typedef typename TestFixture::Type Type;
+
 #define DECL_ARRAYTYPE()                                \
   typedef typename TestFixture::ArrayType ArrayType;
 
 
 TYPED_TEST(TestPrimitiveBuilder, TestInit) {
-  DECL_T();
+  DECL_TYPE();
 
   int n = 1000;
-  ASSERT_OK(this->builder_->Init(n));
-  ASSERT_EQ(n, this->builder_->capacity());
-  ASSERT_EQ(n * sizeof(T), this->builder_->buffer()->size());
+  ASSERT_OK(this->builder_->Reserve(n));
+  ASSERT_EQ(util::next_power2(n), this->builder_->capacity());
+  ASSERT_EQ(util::next_power2(type_traits<Type>::bytes_required(n)),
+      this->builder_->data()->size());
 
   // unsure if this should go in all builder classes
   ASSERT_EQ(0, this->builder_->num_children());
@@ -235,12 +294,14 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
 
   this->RandomData(size);
 
+  this->builder_->Reserve(size);
+
   int i;
   for (i = 0; i < size; ++i) {
     if (valid_bytes[i] > 0) {
-      ASSERT_OK(this->builder_->Append(draws[i]));
+      this->builder_->Append(draws[i]);
     } else {
-      ASSERT_OK(this->builder_->AppendNull());
+      this->builder_->AppendNull();
     }
   }
 
@@ -261,31 +322,41 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
 
   this->RandomData(size);
 
+  this->builder_->Reserve(1000);
+  this->builder_nn_->Reserve(1000);
+
   int i;
+  int null_count = 0;
   // Append the first 1000
   for (i = 0; i < 1000; ++i) {
     if (valid_bytes[i] > 0) {
-      ASSERT_OK(this->builder_->Append(draws[i]));
+      this->builder_->Append(draws[i]);
     } else {
-      ASSERT_OK(this->builder_->AppendNull());
+      this->builder_->AppendNull();
+      ++null_count;
     }
-    ASSERT_OK(this->builder_nn_->Append(draws[i]));
+    this->builder_nn_->Append(draws[i]);
   }
 
+  ASSERT_EQ(null_count, this->builder_->null_count());
+
   ASSERT_EQ(1000, this->builder_->length());
   ASSERT_EQ(1024, this->builder_->capacity());
 
   ASSERT_EQ(1000, this->builder_nn_->length());
   ASSERT_EQ(1024, this->builder_nn_->capacity());
 
+  this->builder_->Reserve(size - 1000);
+  this->builder_nn_->Reserve(size - 1000);
+
   // Append the next 9000
   for (i = 1000; i < size; ++i) {
     if (valid_bytes[i] > 0) {
-      ASSERT_OK(this->builder_->Append(draws[i]));
+      this->builder_->Append(draws[i]);
     } else {
-      ASSERT_OK(this->builder_->AppendNull());
+      this->builder_->AppendNull();
     }
-    ASSERT_OK(this->builder_nn_->Append(draws[i]));
+    this->builder_nn_->Append(draws[i]);
   }
 
   ASSERT_EQ(size, this->builder_->length());
@@ -294,8 +365,8 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   ASSERT_EQ(size, this->builder_nn_->length());
   ASSERT_EQ(util::next_power2(size), this->builder_nn_->capacity());
 
-  this->CheckNullable();
-  this->CheckNonNullable();
+  this->Check(this->builder_, true);
+  this->Check(this->builder_nn_, false);
 }
 
 
@@ -327,31 +398,34 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
   ASSERT_EQ(size, this->builder_->length());
   ASSERT_EQ(util::next_power2(size), this->builder_->capacity());
 
-  this->CheckNullable();
-  this->CheckNonNullable();
+  this->Check(this->builder_, true);
+  this->Check(this->builder_nn_, false);
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
   int n = 1000;
-  ASSERT_OK(this->builder_->Init(n));
+  ASSERT_OK(this->builder_->Reserve(n));
 
   ASSERT_OK(this->builder_->Advance(100));
   ASSERT_EQ(100, this->builder_->length());
 
   ASSERT_OK(this->builder_->Advance(900));
-  ASSERT_RAISES(Invalid, this->builder_->Advance(1));
+
+  int too_many = this->builder_->capacity() - 1000 + 1;
+  ASSERT_RAISES(Invalid, this->builder_->Advance(too_many));
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestResize) {
-  DECL_T();
+  DECL_TYPE();
 
   int cap = MIN_BUILDER_CAPACITY * 2;
 
-  ASSERT_OK(this->builder_->Resize(cap));
+  ASSERT_OK(this->builder_->Reserve(cap));
   ASSERT_EQ(cap, this->builder_->capacity());
 
-  ASSERT_EQ(cap * sizeof(T), this->builder_->buffer()->size());
-  ASSERT_EQ(util::ceil_byte(cap) / 8, this->builder_->null_bitmap()->size());
+  ASSERT_EQ(type_traits<Type>::bytes_required(cap), this->builder_->data()->size());
+  ASSERT_EQ(util::bytes_for_bits(cap),
+      this->builder_->null_bitmap()->size());
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index ecd5d68ff45a8..c54d0757c4789 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -20,20 +20,20 @@
 #include <memory>
 
 #include "arrow/util/buffer.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 
 // ----------------------------------------------------------------------
 // Primitive array base
 
-PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length, int value_size,
+PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
     const std::shared_ptr<Buffer>& data,
     int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap) :
     Array(type, length, null_count, null_bitmap) {
   data_ = data;
   raw_data_ = data == nullptr? nullptr : data_->data();
-  value_size_ = value_size;
 }
 
 bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
@@ -52,12 +52,15 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
     const uint8_t* this_data = raw_data_;
     const uint8_t* other_data = other.raw_data_;
 
+    int value_size = type_->value_size();
+    DCHECK_GT(value_size, 0);
+
     for (int i = 0; i < length_; ++i) {
-      if (!IsNull(i) && memcmp(this_data, other_data, value_size_)) {
+      if (!IsNull(i) && memcmp(this_data, other_data, value_size)) {
         return false;
       }
-      this_data += value_size_;
-      other_data += value_size_;
+      this_data += value_size;
+      other_data += value_size;
     }
     return true;
   } else {
@@ -73,4 +76,170 @@ bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
   return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
 }
 
+template <typename T>
+Status PrimitiveBuilder<T>::Init(int32_t capacity) {
+  RETURN_NOT_OK(ArrayBuilder::Init(capacity));
+  data_ = std::make_shared<PoolBuffer>(pool_);
+
+  int64_t nbytes = type_traits<T>::bytes_required(capacity);
+  RETURN_NOT_OK(data_->Resize(nbytes));
+  memset(data_->mutable_data(), 0, nbytes);
+
+  raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
+  return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
+  // XXX: Set floor size for now
+  if (capacity < MIN_BUILDER_CAPACITY) {
+    capacity = MIN_BUILDER_CAPACITY;
+  }
+
+  if (capacity_ == 0) {
+    RETURN_NOT_OK(Init(capacity));
+  } else {
+    RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
+
+    int64_t old_bytes = data_->size();
+    int64_t new_bytes = type_traits<T>::bytes_required(capacity);
+    RETURN_NOT_OK(data_->Resize(new_bytes));
+    raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
+
+    memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
+  }
+  capacity_ = capacity;
+  return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Reserve(int32_t elements) {
+  if (length_ + elements > capacity_) {
+    int32_t new_capacity = util::next_power2(length_ + elements);
+    return Resize(new_capacity);
+  }
+  return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Append(const value_type* values, int32_t length,
+    const uint8_t* valid_bytes) {
+  RETURN_NOT_OK(PrimitiveBuilder<T>::Reserve(length));
+
+  if (length > 0) {
+    memcpy(raw_data_ + length_, values, type_traits<T>::bytes_required(length));
+  }
+
+  if (valid_bytes != nullptr) {
+    PrimitiveBuilder<T>::AppendNulls(valid_bytes, length);
+  } else {
+    for (int i = 0; i < length; ++i) {
+      util::set_bit(null_bitmap_data_, length_ + i);
+    }
+  }
+
+  length_ += length;
+  return Status::OK();
+}
+
+template <typename T>
+void PrimitiveBuilder<T>::AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+  // If valid_bytes is all not null, then none of the values are null
+  for (int i = 0; i < length; ++i) {
+    if (valid_bytes[i] == 0) {
+      ++null_count_;
+    } else {
+      util::set_bit(null_bitmap_data_, length_ + i);
+    }
+  }
+}
+
+template <typename T>
+std::shared_ptr<Array> PrimitiveBuilder<T>::Finish() {
+  std::shared_ptr<Array> result = std::make_shared<
+    typename type_traits<T>::ArrayType>(
+        type_, length_, data_, null_count_, null_bitmap_);
+
+  data_ = null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
+  return result;
+}
+
+template <>
+Status PrimitiveBuilder<BooleanType>::Append(const uint8_t* values, int32_t length,
+    const uint8_t* valid_bytes) {
+  RETURN_NOT_OK(Reserve(length));
+
+  for (int i = 0; i < length; ++i) {
+    if (values[i] > 0) {
+      util::set_bit(raw_data_, length_ + i);
+    } else {
+      util::clear_bit(raw_data_, length_ + i);
+    }
+  }
+
+  if (valid_bytes != nullptr) {
+    PrimitiveBuilder<BooleanType>::AppendNulls(valid_bytes, length);
+  } else {
+    for (int i = 0; i < length; ++i) {
+      util::set_bit(null_bitmap_data_, length_ + i);
+    }
+  }
+  length_ += length;
+  return Status::OK();
+}
+
+template class PrimitiveBuilder<UInt8Type>;
+template class PrimitiveBuilder<UInt16Type>;
+template class PrimitiveBuilder<UInt32Type>;
+template class PrimitiveBuilder<UInt64Type>;
+template class PrimitiveBuilder<Int8Type>;
+template class PrimitiveBuilder<Int16Type>;
+template class PrimitiveBuilder<Int32Type>;
+template class PrimitiveBuilder<Int64Type>;
+template class PrimitiveBuilder<FloatType>;
+template class PrimitiveBuilder<DoubleType>;
+template class PrimitiveBuilder<BooleanType>;
+
+BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
+    int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap) :
+    PrimitiveArray(std::make_shared<BooleanType>(), length,
+        data, null_count, null_bitmap) {}
+
+bool BooleanArray::EqualsExact(const BooleanArray& other) const {
+  if (this == &other) return true;
+  if (null_count_ != other.null_count_) {
+    return false;
+  }
+
+  if (null_count_ > 0) {
+    bool equal_bitmap = null_bitmap_->Equals(*other.null_bitmap_,
+        util::bytes_for_bits(length_));
+    if (!equal_bitmap) {
+      return false;
+    }
+
+    const uint8_t* this_data = raw_data_;
+    const uint8_t* other_data = other.raw_data_;
+
+    for (int i = 0; i < length_; ++i) {
+      if (!IsNull(i) && util::get_bit(this_data, i) != util::get_bit(other_data, i)) {
+        return false;
+      }
+    }
+    return true;
+  } else {
+    return data_->Equals(*other.data_, util::bytes_for_bits(length_));
+  }
+}
+
+bool BooleanArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) return true;
+  if (Type::BOOL != arr->type_enum()) {
+    return false;
+  }
+  return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
+}
+
 } // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 4eaff433229e0..ec6fee35513ce 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <cstring>
 #include <memory>
+#include <vector>
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
@@ -37,7 +38,7 @@ class MemoryPool;
 // Base class for fixed-size logical types
 class PrimitiveArray : public Array {
  public:
-  PrimitiveArray(const TypePtr& type, int32_t length, int value_size,
+  PrimitiveArray(const TypePtr& type, int32_t length,
       const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
@@ -51,25 +52,19 @@ class PrimitiveArray : public Array {
  protected:
   std::shared_ptr<Buffer> data_;
   const uint8_t* raw_data_;
-  int value_size_;
 };
 
 #define NUMERIC_ARRAY_DECL(NAME, TypeClass, T)                  \
 class NAME : public PrimitiveArray {                            \
  public:                                                        \
   using value_type = T;                                         \
-  NAME(const TypePtr& type, int32_t length,                     \
-      const std::shared_ptr<Buffer>& data,                      \
-      int32_t null_count = 0,                                   \
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr) :   \
-      PrimitiveArray(std::make_shared<TypeClass>(), length,     \
-          sizeof(T), data, null_count, null_bitmap) {}          \
+  using PrimitiveArray::PrimitiveArray;                         \
                                                                 \
   NAME(int32_t length, const std::shared_ptr<Buffer>& data,     \
       int32_t null_count = 0,                                   \
       const std::shared_ptr<Buffer>& null_bitmap = nullptr) :   \
       PrimitiveArray(std::make_shared<TypeClass>(), length,     \
-          sizeof(T), data, null_count, null_bitmap) {}          \
+          data, null_count, null_bitmap) {}                     \
                                                                 \
   bool EqualsExact(const NAME& other) const {                   \
     return PrimitiveArray::EqualsExact(                         \
@@ -96,148 +91,241 @@ NUMERIC_ARRAY_DECL(Int64Array, Int64Type, int64_t);
 NUMERIC_ARRAY_DECL(FloatArray, FloatType, float);
 NUMERIC_ARRAY_DECL(DoubleArray, DoubleType, double);
 
-template <typename Type, typename ArrayType>
+template <typename Type>
 class PrimitiveBuilder : public ArrayBuilder {
  public:
   typedef typename Type::c_type value_type;
 
   explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type) :
       ArrayBuilder(pool, type),
-      values_(nullptr) {
-    elsize_ = sizeof(value_type);
-  }
+      data_(nullptr) {}
 
   virtual ~PrimitiveBuilder() {}
 
-  Status Resize(int32_t capacity) {
-    // XXX: Set floor size for now
-    if (capacity < MIN_BUILDER_CAPACITY) {
-      capacity = MIN_BUILDER_CAPACITY;
-    }
-
-    if (capacity_ == 0) {
-      RETURN_NOT_OK(Init(capacity));
-    } else {
-      RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
-      RETURN_NOT_OK(values_->Resize(capacity * elsize_));
-    }
-    capacity_ = capacity;
-    return Status::OK();
-  }
-
-  Status Init(int32_t capacity) {
-    RETURN_NOT_OK(ArrayBuilder::Init(capacity));
-    values_ = std::make_shared<PoolBuffer>(pool_);
-    return values_->Resize(capacity * elsize_);
-  }
-
-  Status Reserve(int32_t elements) {
-    if (length_ + elements > capacity_) {
-      int32_t new_capacity = util::next_power2(length_ + elements);
-      return Resize(new_capacity);
-    }
-    return Status::OK();
-  }
+  using ArrayBuilder::Advance;
 
-  Status Advance(int32_t elements) {
-    return ArrayBuilder::Advance(elements);
-  }
+  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+  void AppendNulls(const uint8_t* valid_bytes, int32_t length);
 
-  // Scalar append
-  Status Append(value_type val) {
+  Status AppendNull() {
     if (length_ == capacity_) {
       // If the capacity was not already a multiple of 2, do so here
       RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
     }
-    util::set_bit(null_bitmap_data_, length_);
-    raw_buffer()[length_++] = val;
+    ++null_count_;
+    ++length_;
     return Status::OK();
   }
 
+  std::shared_ptr<Buffer> data() const {
+    return data_;
+  }
+
   // Vector append
   //
   // If passed, valid_bytes is of equal length to values, and any zero byte
   // will be considered as a null for that slot
   Status Append(const value_type* values, int32_t length,
-      const uint8_t* valid_bytes = nullptr) {
-    if (length_ + length > capacity_) {
-      int32_t new_capacity = util::next_power2(length_ + length);
-      RETURN_NOT_OK(Resize(new_capacity));
-    }
-    if (length > 0) {
-      memcpy(raw_buffer() + length_, values, length * elsize_);
-    }
+      const uint8_t* valid_bytes = nullptr);
 
-    if (valid_bytes != nullptr) {
-      AppendNulls(valid_bytes, length);
-    } else {
-      for (int i = 0; i < length; ++i) {
-        util::set_bit(null_bitmap_data_, length_ + i);
-      }
-    }
+  // Ensure that builder can accommodate an additional number of
+  // elements. Resizes if the current capacity is not sufficient
+  Status Reserve(int32_t elements);
 
-    length_ += length;
-    return Status::OK();
+  std::shared_ptr<Array> Finish() override;
+
+ protected:
+  std::shared_ptr<PoolBuffer> data_;
+  value_type* raw_data_;
+
+  Status Init(int32_t capacity);
+
+  // Increase the capacity of the builder to accommodate at least the indicated
+  // number of elements
+  Status Resize(int32_t capacity);
+};
+
+template <typename T>
+class NumericBuilder : public PrimitiveBuilder<T> {
+ public:
+  using typename PrimitiveBuilder<T>::value_type;
+  using PrimitiveBuilder<T>::PrimitiveBuilder;
+
+  using PrimitiveBuilder<T>::Append;
+
+  // Scalar append. Does not capacity-check; make sure to call Reserve beforehand
+  void Append(value_type val) {
+    util::set_bit(null_bitmap_data_, length_);
+    raw_data_[length_++] = val;
   }
 
-  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
-  void AppendNulls(const uint8_t* valid_bytes, int32_t length) {
-    // If valid_bytes is all not null, then none of the values are null
-    for (int i = 0; i < length; ++i) {
-      if (valid_bytes[i] == 0) {
-        ++null_count_;
-      } else {
-        util::set_bit(null_bitmap_data_, length_ + i);
-      }
-    }
+ protected:
+  using PrimitiveBuilder<T>::length_;
+  using PrimitiveBuilder<T>::null_bitmap_data_;
+  using PrimitiveBuilder<T>::raw_data_;
+
+  using PrimitiveBuilder<T>::Init;
+  using PrimitiveBuilder<T>::Resize;
+};
+
+template <>
+struct type_traits<UInt8Type> {
+  typedef UInt8Array ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements;
   }
+};
 
-  Status AppendNull() {
-    if (length_ == capacity_) {
-      // If the capacity was not already a multiple of 2, do so here
-      RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
-    }
-    ++null_count_;
-    ++length_;
-    return Status::OK();
+template <>
+struct type_traits<Int8Type> {
+  typedef Int8Array ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements;
   }
+};
 
-  std::shared_ptr<Array> Finish() override {
-    std::shared_ptr<ArrayType> result = std::make_shared<ArrayType>(
-        type_, length_, values_, null_count_, null_bitmap_);
+template <>
+struct type_traits<UInt16Type> {
+  typedef UInt16Array ArrayType;
 
-    values_ = null_bitmap_ = nullptr;
-    capacity_ = length_ = null_count_ = 0;
-    return result;
+  static inline int bytes_required(int elements) {
+    return elements * sizeof(uint16_t);
   }
+};
+
+template <>
+struct type_traits<Int16Type> {
+  typedef Int16Array ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements * sizeof(int16_t);
+  }
+};
 
-  value_type* raw_buffer() {
-    return reinterpret_cast<value_type*>(values_->mutable_data());
+template <>
+struct type_traits<UInt32Type> {
+  typedef UInt32Array ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements * sizeof(uint32_t);
   }
+};
 
-  std::shared_ptr<Buffer> buffer() const {
-    return values_;
+template <>
+struct type_traits<Int32Type> {
+  typedef Int32Array ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements * sizeof(int32_t);
   }
+};
 
- protected:
-  std::shared_ptr<PoolBuffer> values_;
-  int elsize_;
+template <>
+struct type_traits<UInt64Type> {
+  typedef UInt64Array ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements * sizeof(uint64_t);
+  }
+};
+
+template <>
+struct type_traits<Int64Type> {
+  typedef Int64Array ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements * sizeof(int64_t);
+  }
+};
+template <>
+struct type_traits<FloatType> {
+  typedef FloatArray ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements * sizeof(float);
+  }
+};
+
+template <>
+struct type_traits<DoubleType> {
+  typedef DoubleArray ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return elements * sizeof(double);
+  }
 };
 
 // Builders
 
-typedef PrimitiveBuilder<UInt8Type, UInt8Array> UInt8Builder;
-typedef PrimitiveBuilder<UInt16Type, UInt16Array> UInt16Builder;
-typedef PrimitiveBuilder<UInt32Type, UInt32Array> UInt32Builder;
-typedef PrimitiveBuilder<UInt64Type, UInt64Array> UInt64Builder;
+typedef NumericBuilder<UInt8Type> UInt8Builder;
+typedef NumericBuilder<UInt16Type> UInt16Builder;
+typedef NumericBuilder<UInt32Type> UInt32Builder;
+typedef NumericBuilder<UInt64Type> UInt64Builder;
+
+typedef NumericBuilder<Int8Type> Int8Builder;
+typedef NumericBuilder<Int16Type> Int16Builder;
+typedef NumericBuilder<Int32Type> Int32Builder;
+typedef NumericBuilder<Int64Type> Int64Builder;
+
+typedef NumericBuilder<FloatType> FloatBuilder;
+typedef NumericBuilder<DoubleType> DoubleBuilder;
+
 
-typedef PrimitiveBuilder<Int8Type, Int8Array> Int8Builder;
-typedef PrimitiveBuilder<Int16Type, Int16Array> Int16Builder;
-typedef PrimitiveBuilder<Int32Type, Int32Array> Int32Builder;
-typedef PrimitiveBuilder<Int64Type, Int64Array> Int64Builder;
+class BooleanArray : public PrimitiveArray {
+ public:
+  using PrimitiveArray::PrimitiveArray;
+
+  BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+  bool EqualsExact(const BooleanArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+
+  const uint8_t* raw_data() const {
+    return reinterpret_cast<const uint8_t*>(raw_data_);
+  }
+
+  bool Value(int i) const {
+    return util::get_bit(raw_data(), i);
+  }
+};
+
+template <>
+struct type_traits<BooleanType> {
+  typedef BooleanArray ArrayType;
+
+  static inline int bytes_required(int elements) {
+    return util::bytes_for_bits(elements);
+  }
+};
+
+class BooleanBuilder : public PrimitiveBuilder<BooleanType> {
+ public:
+  explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type) :
+      PrimitiveBuilder<BooleanType>(pool, type) {}
+
+  virtual ~BooleanBuilder() {}
+
+  using PrimitiveBuilder<BooleanType>::Append;
+
+  // Scalar append
+  void Append(bool val) {
+    util::set_bit(null_bitmap_data_, length_);
+    if (val) {
+      util::set_bit(raw_data_, length_);
+    } else {
+      util::clear_bit(raw_data_, length_);
+    }
+    ++length_;
+  }
 
-typedef PrimitiveBuilder<FloatType, FloatArray> FloatBuilder;
-typedef PrimitiveBuilder<DoubleType, DoubleArray> DoubleBuilder;
+  void Append(uint8_t val) {
+    Append(static_cast<bool>(val));
+  }
+};
 
 } // namespace arrow
 
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index b329b4f0ef7e1..d3a4cc37f9c4c 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -92,7 +92,7 @@ class TestStringContainer : public ::testing::Test  {
 
     offsets_buf_ = test::to_buffer(offsets_);
 
-    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_.data(), valid_bytes_.size());
+    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<StringArray>(length_, offsets_buf_, values_,
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index fed05e3690c74..d2a4b091fada5 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -23,6 +23,7 @@
 install(FILES
   bit-util.h
   buffer.h
+  logging.h
   macros.h
   memory-pool.h
   status.h
@@ -59,7 +60,7 @@ if (ARROW_BUILD_BENCHMARKS)
     )
   else()
 	  target_link_libraries(arrow_benchmark_main
-      benchmark 
+      benchmark
       pthread
 	  )
   endif()
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 292cb33887ffa..6c6d5330eab0d 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include <cstring>
+#include <vector>
 
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
@@ -23,25 +24,24 @@
 
 namespace arrow {
 
-void util::bytes_to_bits(uint8_t* bytes, int length, uint8_t* bits) {
-  for (int i = 0; i < length; ++i) {
-    if (static_cast<bool>(bytes[i])) {
+void util::bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits) {
+  for (size_t i = 0; i < bytes.size(); ++i) {
+    if (bytes[i] > 0) {
       set_bit(bits, i);
     }
   }
 }
 
-Status util::bytes_to_bits(uint8_t* bytes, int length,
+Status util::bytes_to_bits(const std::vector<uint8_t>& bytes,
     std::shared_ptr<Buffer>* out) {
-  int bit_length = ceil_byte(length) / 8;
+  int bit_length = util::bytes_for_bits(bytes.size());
 
   auto buffer = std::make_shared<PoolBuffer>();
   RETURN_NOT_OK(buffer->Resize(bit_length));
   memset(buffer->mutable_data(), 0, bit_length);
-  bytes_to_bits(bytes, length, buffer->mutable_data());
+  bytes_to_bits(bytes, buffer->mutable_data());
 
   *out = buffer;
-
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 08222d5089474..8d6287130dd2b 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <vector>
 
 namespace arrow {
 
@@ -43,15 +44,19 @@ static inline int64_t ceil_2bytes(int64_t size) {
 static constexpr uint8_t BITMASK[] = {1, 2, 4, 8, 16, 32, 64, 128};
 
 static inline bool get_bit(const uint8_t* bits, int i) {
-  return bits[i / 8] & BITMASK[i % 8];
+  return static_cast<bool>(bits[i / 8] & BITMASK[i % 8]);
 }
 
 static inline bool bit_not_set(const uint8_t* bits, int i) {
   return (bits[i / 8] & BITMASK[i % 8]) == 0;
 }
 
+static inline void clear_bit(uint8_t* bits, int i) {
+  bits[i / 8] &= ~BITMASK[i % 8];
+}
+
 static inline void set_bit(uint8_t* bits, int i) {
-  bits[i / 8] |= 1 << (i % 8);
+  bits[i / 8] |= BITMASK[i % 8];
 }
 
 static inline int64_t next_power2(int64_t n) {
@@ -66,8 +71,8 @@ static inline int64_t next_power2(int64_t n) {
   return n;
 }
 
-void bytes_to_bits(uint8_t* bytes, int length, uint8_t* bits);
-Status bytes_to_bits(uint8_t*, int, std::shared_ptr<Buffer>*);
+void bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits);
+Status bytes_to_bits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
 } // namespace util
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index e6afcbd79b69f..943a08f84a055 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -86,6 +86,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_bool IsNull(int i)
 
+    cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray):
+        c_bool Value(int i)
+
     cdef cppclass CUInt8Array" arrow::UInt8Array"(CArray):
         uint8_t Value(int i)
 
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 04f013d6ca706..0d391e5f26b3e 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -58,7 +58,10 @@ cdef class ArrayValue(Scalar):
 
 
 cdef class BooleanValue(ArrayValue):
-    pass
+
+    def as_py(self):
+        cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
+        return ap.Value(self.index)
 
 
 cdef class Int8Value(ArrayValue):
@@ -172,6 +175,7 @@ cdef class ListValue(ArrayValue):
 
 
 cdef dict _scalar_classes = {
+    Type_BOOL: BooleanValue,
     Type_UINT8: Int8Value,
     Type_UINT16: Int16Value,
     Type_UINT32: Int32Value,
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 25f696912105d..2beb6b39d73ed 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -22,7 +22,10 @@
 class TestConvertList(unittest.TestCase):
 
     def test_boolean(self):
-        pass
+        arr = pyarrow.from_pylist([True, None, False, None])
+        assert len(arr) == 4
+        assert arr.null_count == 2
+        assert arr.type == pyarrow.bool_()
 
     def test_empty_list(self):
         arr = pyarrow.from_pylist([])
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 021737db6726e..4fb850a4d47bf 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -16,67 +16,74 @@
 # under the License.
 
 from pyarrow.compat import unittest, u
-import pyarrow as arrow
+import pyarrow as A
 
 
 class TestScalars(unittest.TestCase):
 
     def test_null_singleton(self):
         with self.assertRaises(Exception):
-            arrow.NAType()
+            A.NAType()
 
     def test_bool(self):
-        pass
+        arr = A.from_pylist([True, None, False, None])
+
+        v = arr[0]
+        assert isinstance(v, A.BooleanValue)
+        assert repr(v) == "True"
+        assert v.as_py() == True
+
+        assert arr[1] is A.NA
 
     def test_int64(self):
-        arr = arrow.from_pylist([1, 2, None])
+        arr = A.from_pylist([1, 2, None])
 
         v = arr[0]
-        assert isinstance(v, arrow.Int64Value)
+        assert isinstance(v, A.Int64Value)
         assert repr(v) == "1"
         assert v.as_py() == 1
 
-        assert arr[2] is arrow.NA
+        assert arr[2] is A.NA
 
     def test_double(self):
-        arr = arrow.from_pylist([1.5, None, 3])
+        arr = A.from_pylist([1.5, None, 3])
 
         v = arr[0]
-        assert isinstance(v, arrow.DoubleValue)
+        assert isinstance(v, A.DoubleValue)
         assert repr(v) == "1.5"
         assert v.as_py() == 1.5
 
-        assert arr[1] is arrow.NA
+        assert arr[1] is A.NA
 
         v = arr[2]
         assert v.as_py() == 3.0
 
     def test_string(self):
-        arr = arrow.from_pylist(['foo', None, u('bar')])
+        arr = A.from_pylist(['foo', None, u('bar')])
 
         v = arr[0]
-        assert isinstance(v, arrow.StringValue)
+        assert isinstance(v, A.StringValue)
         assert repr(v) == "'foo'"
         assert v.as_py() == 'foo'
 
-        assert arr[1] is arrow.NA
+        assert arr[1] is A.NA
 
         v = arr[2].as_py()
         assert v == 'bar'
         assert isinstance(v, str)
 
     def test_list(self):
-        arr = arrow.from_pylist([['foo', None], None, ['bar'], []])
+        arr = A.from_pylist([['foo', None], None, ['bar'], []])
 
         v = arr[0]
         assert len(v) == 2
-        assert isinstance(v, arrow.ListValue)
+        assert isinstance(v, A.ListValue)
         assert repr(v) == "['foo', None]"
         assert v.as_py() == ['foo', None]
         assert v[0].as_py() == 'foo'
-        assert v[1] is arrow.NA
+        assert v[1] is A.NA
 
-        assert arr[1] is arrow.NA
+        assert arr[1] is A.NA
 
         v = arr[3]
         assert len(v) == 0
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index acb13acecaf33..78ef1b31f34f1 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -61,6 +61,8 @@ class ScalarVisitor {
     ++total_count_;
     if (obj == Py_None) {
       ++none_count_;
+    } else if (PyBool_Check(obj)) {
+      ++bool_count_;
     } else if (PyFloat_Check(obj)) {
       ++float_count_;
     } else if (IsPyInteger(obj)) {
@@ -256,6 +258,20 @@ class TypedConverter : public SeqConverter {
 class BoolConverter : public TypedConverter<arrow::BooleanBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
+    Py_ssize_t size = PySequence_Size(seq);
+    RETURN_ARROW_NOT_OK(typed_builder_->Reserve(size));
+    for (int64_t i = 0; i < size; ++i) {
+      OwnedRef item(PySequence_GetItem(seq, i));
+      if (item.obj() == Py_None) {
+        typed_builder_->AppendNull();
+      } else {
+        if (item.obj() == Py_True) {
+          typed_builder_->Append(true);
+        } else {
+          typed_builder_->Append(false);
+        }
+      }
+    }
     return Status::OK();
   }
 };
@@ -265,14 +281,15 @@ class Int64Converter : public TypedConverter<arrow::Int64Builder> {
   Status AppendData(PyObject* seq) override {
     int64_t val;
     Py_ssize_t size = PySequence_Size(seq);
+    RETURN_ARROW_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
       if (item.obj() == Py_None) {
-        RETURN_ARROW_NOT_OK(typed_builder_->AppendNull());
+        typed_builder_->AppendNull();
       } else {
         val = PyLong_AsLongLong(item.obj());
         RETURN_IF_PYERROR();
-        RETURN_ARROW_NOT_OK(typed_builder_->Append(val));
+        typed_builder_->Append(val);
       }
     }
     return Status::OK();
@@ -284,14 +301,15 @@ class DoubleConverter : public TypedConverter<arrow::DoubleBuilder> {
   Status AppendData(PyObject* seq) override {
     double val;
     Py_ssize_t size = PySequence_Size(seq);
+    RETURN_ARROW_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
       if (item.obj() == Py_None) {
-        RETURN_ARROW_NOT_OK(typed_builder_->AppendNull());
+        typed_builder_->AppendNull();
       } else {
         val = PyFloat_AsDouble(item.obj());
         RETURN_IF_PYERROR();
-        RETURN_ARROW_NOT_OK(typed_builder_->Append(val));
+        typed_builder_->Append(val);
       }
     }
     return Status::OK();

From d3cb6b47fde2935522b73c7150d83e364f4e19c9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Sat, 26 Mar 2016 17:07:40 -0700
Subject: [PATCH 0044/1644] ARROW-22: [C++] Convert flat Parquet schemas to
 Arrow schemas

I'm going to limit the amount of nested data (especially repeated fields) cases in this patch as I haven't yet thought through the nested data reassembly from repetition / definition levels. Since the effective Arrow schemas may "collapse" multiple levels of nesting (for example: 3-level array encoding -- see https://github.com/apache/parquet-cpp/blob/master/src/parquet/schema/types.h), we'll need to track the logical correspondence between repetition and definition levels so that the right null bits can be set easily during reassembly.

Closes #37. Closes #38. Closes #39

Author: Wes McKinney <wesm@apache.org>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #41 from wesm/ARROW-22 and squashes the following commits:

f388210 [Wes McKinney] Correct typo in Layout.md (thanks @takahirox)
e5c429a [Wes McKinney] Test for some unsupported Parquet schema types, add unannotated FIXED_LEN_BYTE_ARRAY to List<UInt8>
54daa9b [Wes McKinney] Refactor tests to invoke FromParquetSchema
74d6bae [Wes McKinney] Convert BYTE_ARRAY to StringType or List<UInt8> depending on the logical type
b7b9ca9 [Uwe L. Korn] Add basic conversion for primitive types
0e2a7f1 [Uwe L. Korn] Add macro for adding dependencies to tests
0dd1109 [Uwe L. Korn] ARROW-78: Add constructor for DecimalType
---
 cpp/CMakeLists.txt                           |  11 ++
 cpp/src/arrow/parquet/CMakeLists.txt         |   8 +-
 cpp/src/arrow/parquet/parquet-schema-test.cc | 147 +++++++++++++++
 cpp/src/arrow/parquet/schema.cc              | 178 +++++++++++++++++++
 cpp/src/arrow/parquet/schema.h               |  44 +++++
 cpp/src/arrow/types/decimal.cc               |  32 ++++
 cpp/src/arrow/types/decimal.h                |  11 ++
 cpp/src/arrow/util/status.h                  |   1 +
 format/Layout.md                             |   2 +-
 9 files changed, 432 insertions(+), 2 deletions(-)
 create mode 100644 cpp/src/arrow/parquet/parquet-schema-test.cc
 create mode 100644 cpp/src/arrow/parquet/schema.cc
 create mode 100644 cpp/src/arrow/parquet/schema.h
 create mode 100644 cpp/src/arrow/types/decimal.cc

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6d701079b482c..6ed2768d13918 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -378,6 +378,16 @@ function(ADD_ARROW_TEST_DEPENDENCIES REL_TEST_NAME)
   add_dependencies(${TEST_NAME} ${ARGN})
 endfunction()
 
+# A wrapper for target_link_libraries() that is compatible with NO_TESTS.
+function(ARROW_TEST_LINK_LIBRARIES REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  target_link_libraries(${TEST_NAME} ${ARGN})
+endfunction()
+
 enable_testing()
 
 ############################################################
@@ -528,6 +538,7 @@ set(ARROW_SRCS
   src/arrow/ipc/metadata-internal.cc
 
   src/arrow/types/construct.cc
+  src/arrow/types/decimal.cc
   src/arrow/types/json.cc
   src/arrow/types/list.cc
   src/arrow/types/primitive.cc
diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
index 7b449affab025..0d5cf263ec3e2 100644
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -19,17 +19,23 @@
 # arrow_parquet : Arrow <-> Parquet adapter
 
 set(PARQUET_SRCS
+  schema.cc
 )
 
 set(PARQUET_LIBS
+  arrow
+  ${PARQUET_SHARED_LIB}
 )
 
-add_library(arrow_parquet STATIC
+add_library(arrow_parquet SHARED
   ${PARQUET_SRCS}
 )
 target_link_libraries(arrow_parquet ${PARQUET_LIBS})
 SET_TARGET_PROPERTIES(arrow_parquet PROPERTIES LINKER_LANGUAGE CXX)
 
+ADD_ARROW_TEST(parquet-schema-test)
+ARROW_TEST_LINK_LIBRARIES(parquet-schema-test arrow_parquet)
+
 # Headers: top level
 install(FILES
   DESTINATION include/arrow/parquet)
diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
new file mode 100644
index 0000000000000..9c3093d9ff7c9
--- /dev/null
+++ b/cpp/src/arrow/parquet/parquet-schema-test.cc
@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/util/status.h"
+
+#include "arrow/parquet/schema.h"
+
+namespace arrow {
+
+namespace parquet {
+
+using parquet_cpp::Repetition;
+using parquet_cpp::schema::NodePtr;
+using parquet_cpp::schema::GroupNode;
+using parquet_cpp::schema::PrimitiveNode;
+
+const auto BOOL = std::make_shared<BooleanType>();
+const auto UINT8 = std::make_shared<UInt8Type>();
+const auto INT32 = std::make_shared<Int32Type>();
+const auto INT64 = std::make_shared<Int64Type>();
+const auto FLOAT = std::make_shared<FloatType>();
+const auto DOUBLE = std::make_shared<DoubleType>();
+const auto UTF8 = std::make_shared<StringType>();
+const auto BINARY = std::make_shared<ListType>(
+    std::make_shared<Field>("", UINT8));
+
+class TestConvertParquetSchema : public ::testing::Test {
+ public:
+  virtual void SetUp() {}
+
+  void CheckFlatSchema(const std::shared_ptr<Schema>& expected_schema) {
+    ASSERT_EQ(expected_schema->num_fields(), result_schema_->num_fields());
+    for (int i = 0; i < expected_schema->num_fields(); ++i) {
+      auto lhs = result_schema_->field(i);
+      auto rhs = expected_schema->field(i);
+      EXPECT_TRUE(lhs->Equals(rhs))
+        << i << " " << lhs->ToString() << " != " << rhs->ToString();
+    }
+  }
+
+  Status ConvertSchema(const std::vector<NodePtr>& nodes) {
+    NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, nodes);
+    descr_.Init(schema);
+    return FromParquetSchema(&descr_, &result_schema_);
+  }
+
+ protected:
+  parquet_cpp::SchemaDescriptor descr_;
+  std::shared_ptr<Schema> result_schema_;
+};
+
+TEST_F(TestConvertParquetSchema, ParquetFlatPrimitives) {
+  std::vector<NodePtr> parquet_fields;
+  std::vector<std::shared_ptr<Field>> arrow_fields;
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("boolean", Repetition::REQUIRED, parquet_cpp::Type::BOOLEAN));
+  arrow_fields.push_back(std::make_shared<Field>("boolean", BOOL, false));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("int32", Repetition::REQUIRED, parquet_cpp::Type::INT32));
+  arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("int64", Repetition::REQUIRED, parquet_cpp::Type::INT64));
+  arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("float", Repetition::OPTIONAL, parquet_cpp::Type::FLOAT));
+  arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("double", Repetition::OPTIONAL, parquet_cpp::Type::DOUBLE));
+  arrow_fields.push_back(std::make_shared<Field>("double", DOUBLE));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("binary", Repetition::OPTIONAL,
+          parquet_cpp::Type::BYTE_ARRAY));
+  arrow_fields.push_back(std::make_shared<Field>("binary", BINARY));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("string", Repetition::OPTIONAL,
+          parquet_cpp::Type::BYTE_ARRAY,
+          parquet_cpp::LogicalType::UTF8));
+  arrow_fields.push_back(std::make_shared<Field>("string", UTF8));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("flba-binary", Repetition::OPTIONAL,
+          parquet_cpp::Type::FIXED_LEN_BYTE_ARRAY,
+          parquet_cpp::LogicalType::NONE, 12));
+  arrow_fields.push_back(std::make_shared<Field>("flba-binary", BINARY));
+
+  auto arrow_schema = std::make_shared<Schema>(arrow_fields);
+  ASSERT_OK(ConvertSchema(parquet_fields));
+
+  CheckFlatSchema(arrow_schema);
+}
+
+TEST_F(TestConvertParquetSchema, UnsupportedThings) {
+  std::vector<NodePtr> unsupported_nodes;
+
+  unsupported_nodes.push_back(
+      PrimitiveNode::Make("int96", Repetition::REQUIRED, parquet_cpp::Type::INT96));
+
+  unsupported_nodes.push_back(
+      GroupNode::Make("repeated-group", Repetition::REPEATED, {}));
+
+  unsupported_nodes.push_back(
+      PrimitiveNode::Make("int32", Repetition::OPTIONAL,
+          parquet_cpp::Type::INT32, parquet_cpp::LogicalType::DATE));
+
+  unsupported_nodes.push_back(
+      PrimitiveNode::Make("int64", Repetition::OPTIONAL,
+          parquet_cpp::Type::INT64, parquet_cpp::LogicalType::TIMESTAMP_MILLIS));
+
+  for (const NodePtr& node : unsupported_nodes) {
+    ASSERT_RAISES(NotImplemented, ConvertSchema({node}));
+  }
+}
+
+TEST(TestNodeConversion, DateAndTime) {
+}
+
+} // namespace parquet
+
+} // namespace arrow
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
new file mode 100644
index 0000000000000..6b1de572617b8
--- /dev/null
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/parquet/schema.h"
+
+#include <vector>
+
+#include "parquet/api/schema.h"
+
+#include "arrow/util/status.h"
+#include "arrow/types/decimal.h"
+
+using parquet_cpp::schema::Node;
+using parquet_cpp::schema::NodePtr;
+using parquet_cpp::schema::GroupNode;
+using parquet_cpp::schema::PrimitiveNode;
+
+using parquet_cpp::LogicalType;
+
+namespace arrow {
+
+namespace parquet {
+
+const auto BOOL = std::make_shared<BooleanType>();
+const auto UINT8 = std::make_shared<UInt8Type>();
+const auto INT32 = std::make_shared<Int32Type>();
+const auto INT64 = std::make_shared<Int64Type>();
+const auto FLOAT = std::make_shared<FloatType>();
+const auto DOUBLE = std::make_shared<DoubleType>();
+const auto UTF8 = std::make_shared<StringType>();
+const auto BINARY = std::make_shared<ListType>(
+    std::make_shared<Field>("", UINT8));
+
+TypePtr MakeDecimalType(const PrimitiveNode* node) {
+  int precision = node->decimal_metadata().precision;
+  int scale = node->decimal_metadata().scale;
+  return std::make_shared<DecimalType>(precision, scale);
+}
+
+static Status FromByteArray(const PrimitiveNode* node, TypePtr* out) {
+  switch (node->logical_type()) {
+    case LogicalType::UTF8:
+      *out = UTF8;
+      break;
+    default:
+      // BINARY
+      *out = BINARY;
+      break;
+  }
+  return Status::OK();
+}
+
+static Status FromFLBA(const PrimitiveNode* node, TypePtr* out) {
+  switch (node->logical_type()) {
+    case LogicalType::NONE:
+      *out = BINARY;
+      break;
+    case LogicalType::DECIMAL:
+      *out = MakeDecimalType(node);
+      break;
+    default:
+      return Status::NotImplemented("unhandled type");
+      break;
+  }
+
+  return Status::OK();
+}
+
+static Status FromInt32(const PrimitiveNode* node, TypePtr* out) {
+  switch (node->logical_type()) {
+    case LogicalType::NONE:
+      *out = INT32;
+      break;
+    default:
+      return Status::NotImplemented("Unhandled logical type for int32");
+      break;
+  }
+  return Status::OK();
+}
+
+static Status FromInt64(const PrimitiveNode* node, TypePtr* out) {
+  switch (node->logical_type()) {
+    case LogicalType::NONE:
+      *out = INT64;
+      break;
+    default:
+      return Status::NotImplemented("Unhandled logical type for int64");
+      break;
+  }
+  return Status::OK();
+}
+
+// TODO: Logical Type Handling
+Status NodeToField(const NodePtr& node, std::shared_ptr<Field>* out) {
+  std::shared_ptr<DataType> type;
+
+  if (node->is_repeated()) {
+    return Status::NotImplemented("No support yet for repeated node types");
+  }
+
+  if (node->is_group()) {
+    const GroupNode* group = static_cast<const GroupNode*>(node.get());
+    std::vector<std::shared_ptr<Field>> fields(group->field_count());
+    for (int i = 0; i < group->field_count(); i++) {
+      RETURN_NOT_OK(NodeToField(group->field(i), &fields[i]));
+    }
+    type = std::make_shared<StructType>(fields);
+  } else {
+    // Primitive (leaf) node
+    const PrimitiveNode* primitive = static_cast<const PrimitiveNode*>(node.get());
+
+    switch (primitive->physical_type()) {
+      case parquet_cpp::Type::BOOLEAN:
+        type = BOOL;
+        break;
+      case parquet_cpp::Type::INT32:
+        RETURN_NOT_OK(FromInt32(primitive, &type));
+        break;
+      case parquet_cpp::Type::INT64:
+        RETURN_NOT_OK(FromInt64(primitive, &type));
+        break;
+      case parquet_cpp::Type::INT96:
+        // TODO: Do we have that type in Arrow?
+        // type = TypePtr(new Int96Type());
+        return Status::NotImplemented("int96");
+      case parquet_cpp::Type::FLOAT:
+        type = FLOAT;
+        break;
+      case parquet_cpp::Type::DOUBLE:
+        type = DOUBLE;
+        break;
+      case parquet_cpp::Type::BYTE_ARRAY:
+        // TODO: Do we have that type in Arrow?
+        RETURN_NOT_OK(FromByteArray(primitive, &type));
+        break;
+      case parquet_cpp::Type::FIXED_LEN_BYTE_ARRAY:
+        RETURN_NOT_OK(FromFLBA(primitive, &type));
+        break;
+    }
+  }
+
+  *out = std::make_shared<Field>(node->name(), type, !node->is_required());
+  return Status::OK();
+}
+
+Status FromParquetSchema(const parquet_cpp::SchemaDescriptor* parquet_schema,
+    std::shared_ptr<Schema>* out) {
+  // TODO(wesm): Consider adding an arrow::Schema name attribute, which comes
+  // from the root Parquet node
+  const GroupNode* schema_node = static_cast<const GroupNode*>(
+      parquet_schema->schema().get());
+
+  std::vector<std::shared_ptr<Field>> fields(schema_node->field_count());
+  for (int i = 0; i < schema_node->field_count(); i++) {
+    RETURN_NOT_OK(NodeToField(schema_node->field(i), &fields[i]));
+  }
+
+  *out = std::make_shared<Schema>(fields);
+  return Status::OK();
+}
+
+} // namespace parquet
+
+} // namespace arrow
diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h
new file mode 100644
index 0000000000000..61de193a33877
--- /dev/null
+++ b/cpp/src/arrow/parquet/schema.h
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PARQUET_SCHEMA_H
+#define ARROW_PARQUET_SCHEMA_H
+
+#include <memory>
+
+#include "parquet/api/schema.h"
+
+#include "arrow/schema.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+class Status;
+
+namespace parquet {
+
+Status NodeToField(const parquet_cpp::schema::NodePtr& node,
+    std::shared_ptr<Field>* out);
+
+Status FromParquetSchema(const parquet_cpp::SchemaDescriptor* parquet_schema,
+    std::shared_ptr<Schema>* out);
+
+} // namespace parquet
+
+} // namespace arrow
+
+#endif
diff --git a/cpp/src/arrow/types/decimal.cc b/cpp/src/arrow/types/decimal.cc
new file mode 100644
index 0000000000000..f120c1a9dfde6
--- /dev/null
+++ b/cpp/src/arrow/types/decimal.cc
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/decimal.h"
+
+#include <sstream>
+#include <string>
+
+namespace arrow {
+
+std::string DecimalType::ToString() const {
+  std::stringstream s;
+  s << "decimal(" << precision << ", " << scale << ")";
+  return s.str();
+}
+
+} // namespace arrow
+
diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h
index 464c3ff8da92b..26243b42b0e7d 100644
--- a/cpp/src/arrow/types/decimal.h
+++ b/cpp/src/arrow/types/decimal.h
@@ -18,13 +18,24 @@
 #ifndef ARROW_TYPES_DECIMAL_H
 #define ARROW_TYPES_DECIMAL_H
 
+#include <string>
+
 #include "arrow/type.h"
 
 namespace arrow {
 
 struct DecimalType : public DataType {
+  explicit DecimalType(int precision_, int scale_)
+      : DataType(Type::DECIMAL), precision(precision_),
+        scale(scale_) { }
   int precision;
   int scale;
+
+  static char const *name() {
+    return "decimal";
+  }
+
+  std::string ToString() const override;
 };
 
 } // namespace arrow
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
index b5931232dbdcb..4e273edcb8f1f 100644
--- a/cpp/src/arrow/util/status.h
+++ b/cpp/src/arrow/util/status.h
@@ -109,6 +109,7 @@ class Status {
   bool IsKeyError() const { return code() == StatusCode::KeyError; }
   bool IsInvalid() const { return code() == StatusCode::Invalid; }
   bool IsIOError() const { return code() == StatusCode::IOError; }
+  bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
 
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
diff --git a/format/Layout.md b/format/Layout.md
index 2d46ece606ea7..1b532c6b3817c 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -58,7 +58,7 @@ Base requirements
 * Memory layout and random access patterns for each relative type
 * Null value representation
 
-## Non-goals (for this document
+## Non-goals (for this document)
 
 * To enumerate or specify logical types that can be implemented as primitive
   (fixed-width) value types. For example: signed and unsigned integers,

From d6d53b25ef4e8cd7d8c34df56661817366906bbf Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Sun, 27 Mar 2016 12:28:18 -0700
Subject: [PATCH 0045/1644] ARROW-63: [C++] Enable ctest to work on systems
 with Python 3 as the default Python

Author: Wes McKinney <wesm@apache.org>

Closes #42 from wesm/ARROW-63 and squashes the following commits:

9840308 [Wes McKinney] Make asan_symbolize.py work on both Python 2.7 and 3.x
---
 cpp/build-support/asan_symbolize.py | 36 ++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/cpp/build-support/asan_symbolize.py b/cpp/build-support/asan_symbolize.py
index 839a1984bd349..1108044d7d648 100755
--- a/cpp/build-support/asan_symbolize.py
+++ b/cpp/build-support/asan_symbolize.py
@@ -64,7 +64,7 @@ def open_llvm_symbolizer(self):
            '--functions=true',
            '--inlining=true']
     if DEBUG:
-      print ' '.join(cmd)
+      print(' '.join(cmd))
     return subprocess.Popen(cmd, stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE)
 
@@ -76,8 +76,9 @@ def symbolize(self, addr, binary, offset):
     try:
       symbolizer_input = '%s %s' % (binary, offset)
       if DEBUG:
-        print symbolizer_input
-      print >> self.pipe.stdin, symbolizer_input
+        print(symbolizer_input)
+      self.pipe.stdin.write(symbolizer_input)
+      self.pipe.stdin.write('\n')
       while True:
         function_name = self.pipe.stdout.readline().rstrip()
         if not function_name:
@@ -113,7 +114,7 @@ def __init__(self, binary):
   def open_addr2line(self):
     cmd = ['addr2line', '-f', '-e', self.binary]
     if DEBUG:
-      print ' '.join(cmd)
+      print(' '.join(cmd))
     return subprocess.Popen(cmd,
                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 
@@ -122,7 +123,8 @@ def symbolize(self, addr, binary, offset):
     if self.binary != binary:
       return None
     try:
-      print >> self.pipe.stdin, offset
+      self.pipe.stdin.write(offset)
+      self.pipe.stdin.write('\n')
       function_name = self.pipe.stdout.readline().rstrip()
       file_name = self.pipe.stdout.readline().rstrip()
     except Exception:
@@ -145,11 +147,12 @@ def __init__(self, addr, binary):
     self.pipe = None
 
   def write_addr_to_pipe(self, offset):
-    print >> self.pipe.stdin, '0x%x' % int(offset, 16)
+    self.pipe.stdin.write('0x%x' % int(offset, 16))
+    self.pipe.stdin.write('\n')
 
   def open_atos(self):
     if DEBUG:
-      print 'atos -o %s -arch %s' % (self.binary, self.arch)
+      print('atos -o %s -arch %s' % (self.binary, self.arch))
     cmdline = ['atos', '-o', self.binary, '-arch', self.arch]
     self.pipe = subprocess.Popen(cmdline,
                                  stdin=subprocess.PIPE,
@@ -168,7 +171,7 @@ def symbolize(self, addr, binary, offset):
     #   foo(type1, type2) (in object.name) (filename.cc:80)
     match = re.match('^(.*) \(in (.*)\) \((.*:\d*)\)$', atos_line)
     if DEBUG:
-      print 'atos_line: ', atos_line
+      print('atos_line: {0}'.format(atos_line))
     if match:
       function_name = match.group(1)
       function_name = re.sub('\(.*?\)', '', function_name)
@@ -282,7 +285,7 @@ def symbolize(self, addr, binary, offset):
       function_name, file_name, line_no = res
       result = ['%s in %s %s:%d' % (
           addr, function_name, file_name, line_no)]
-      print result
+      print(result)
       return result
     else:
       return None
@@ -318,15 +321,20 @@ def symbolize_address(self, addr, binary, offset):
 
   def print_symbolized_lines(self, symbolized_lines):
     if not symbolized_lines:
-      print self.current_line
+      print(self.current_line)
     else:
       for symbolized_frame in symbolized_lines:
-        print '    #' + str(self.frame_no) + ' ' + symbolized_frame.rstrip()
+        print('    #' + str(self.frame_no) + ' ' + symbolized_frame.rstrip())
         self.frame_no += 1
 
   def process_stdin(self):
     self.frame_no = 0
-    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+
+    if sys.version_info[0] == 2:
+      sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+    else:
+      # Unbuffered output is not supported in Python 3
+      sys.stdout = os.fdopen(sys.stdout.fileno(), 'w')
 
     while True:
       line = sys.stdin.readline()
@@ -337,10 +345,10 @@ def process_stdin(self):
           '^( *#([0-9]+) *)(0x[0-9a-f]+) *\((.*)\+(0x[0-9a-f]+)\)')
       match = re.match(stack_trace_line_format, line)
       if not match:
-        print self.current_line
+        print(self.current_line)
         continue
       if DEBUG:
-        print line
+        print(line)
       _, frameno_str, addr, binary, offset = match.groups()
       if frameno_str == '0':
         # Assume that frame #0 is the first frame of new stack trace.

From 017187749f3916e589015a4db2409258a0b3c03c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 27 Mar 2016 12:30:58 -0700
Subject: [PATCH 0046/1644] ARROW-65: Be less restrictive on PYTHON_LIBRARY
 search paths

Current CMake FindPythonLibs also uses this option instead of NO_DEFAULT_PATH.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #43 from xhochy/arrow-65 and squashes the following commits:

10eb9e0 [Uwe L. Korn] ARROW-65: Be less restrictive on PYTHON_LIBRARY search paths
---
 python/cmake_modules/FindPythonLibsNew.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/cmake_modules/FindPythonLibsNew.cmake b/python/cmake_modules/FindPythonLibsNew.cmake
index c70e6bc26a719..0f2295aa43bc1 100644
--- a/python/cmake_modules/FindPythonLibsNew.cmake
+++ b/python/cmake_modules/FindPythonLibsNew.cmake
@@ -166,7 +166,7 @@ else()
     find_library(PYTHON_LIBRARY
         NAMES "python${PYTHON_LIBRARY_SUFFIX}"
         PATHS ${_PYTHON_LIBS_SEARCH}
-        NO_DEFAULT_PATH)
+        NO_SYSTEM_ENVIRONMENT_PATH)
     message(STATUS "Found Python lib ${PYTHON_LIBRARY}")
 endif()
 

From 1fd0668a1330e72b1b137d90d00906bc188243e0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Mon, 28 Mar 2016 09:36:20 -0700
Subject: [PATCH 0047/1644] ARROW-30: [Python] Routines for converting between
 arrow::Array/Table and pandas.DataFrame

There is a lot to do here for maximum compatibility, but this gets things started.

Author: Wes McKinney <wesm@apache.org>

Closes #46 from wesm/ARROW-30 and squashes the following commits:

0a9e747 [Wes McKinney] Invoke py.test with python -m pytest
4c9f766 [Wes McKinney] More scaffolding. Table wrapper. Initial unit tests passing
8475a0e [Wes McKinney] More pandas conversion scaffolding, enable libpyarrow to use the NumPy C API globally
d1f05c5 [Wes McKinney] cpplint
f0cc451 [Wes McKinney] Give libpyarrow a reference to numpy.nan
5e09bfe [Wes McKinney] Compiling, but untested draft of pandas <-> arrow converters
---
 ci/travis_script_python.sh                    |   8 +-
 cpp/README.md                                 |   6 +-
 cpp/src/arrow/array.h                         |  13 +-
 cpp/src/arrow/types/string.cc                 |  10 +
 cpp/src/arrow/types/string.h                  |   4 +-
 cpp/src/arrow/util/buffer.h                   |  42 ++
 python/CMakeLists.txt                         |   6 +-
 python/pyarrow/__init__.py                    |   8 +-
 python/pyarrow/array.pyx                      | 135 ++++
 python/pyarrow/config.pyx                     |  13 +-
 python/pyarrow/includes/common.pxd            |   6 +
 python/pyarrow/includes/libarrow.pxd          |  52 +-
 python/pyarrow/includes/pyarrow.pxd           |   9 +-
 python/pyarrow/tests/test_convert_pandas.py   | 172 +++++
 python/src/pyarrow/adapters/pandas.cc         | 714 ++++++++++++++++++
 python/src/pyarrow/adapters/pandas.h          |  21 +
 python/src/pyarrow/common.h                   |  23 +-
 python/src/pyarrow/{init.cc => config.cc}     |  11 +-
 python/src/pyarrow/config.h                   |  39 +
 .../src/pyarrow/{init.h => do_import_numpy.h} |  12 +-
 python/src/pyarrow/numpy_interop.h            |  58 ++
 21 files changed, 1313 insertions(+), 49 deletions(-)
 create mode 100644 python/pyarrow/tests/test_convert_pandas.py
 create mode 100644 python/src/pyarrow/adapters/pandas.cc
 rename python/src/pyarrow/{init.cc => config.cc} (84%)
 create mode 100644 python/src/pyarrow/config.h
 rename python/src/pyarrow/{init.h => do_import_numpy.h} (83%)
 create mode 100644 python/src/pyarrow/numpy_interop.h

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index af6b0085724fc..d45b895d8cf38 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -48,17 +48,11 @@ python_version_tests() {
 
   python setup.py build_ext --inplace
 
-  py.test -vv -r sxX pyarrow
+  python -m pytest -vv -r sxX pyarrow
 }
 
 # run tests for python 2.7 and 3.5
 python_version_tests 2.7
 python_version_tests 3.5
 
-# if [ $TRAVIS_OS_NAME == "linux" ]; then
-#   valgrind --tool=memcheck py.test -vv -r sxX arrow
-# else
-#   py.test -vv -r sxX arrow
-# fi
-
 popd
diff --git a/cpp/README.md b/cpp/README.md
index 542cce43a1391..9026cf963f8ee 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -42,12 +42,12 @@ Detailed unit test logs will be placed in the build directory under `build/test-
 
 ### Building/Running benchmarks
 
-Follow the directions for simple build except run cmake 
+Follow the directions for simple build except run cmake
 with the `--ARROW_BUILD_BENCHMARKS` parameter set correctly:
 
     cmake -DARROW_BUILD_BENCHMARKS=ON ..
 
-and instead of make unittest run either `make; ctest` to run both unit tests 
+and instead of make unittest run either `make; ctest` to run both unit tests
 and benchmarks or `make runbenchmark` to run only the benchmark tests.
 
 Benchmark logs will be placed in the build directory under `build/benchmark-logs`.
@@ -60,4 +60,4 @@ variables
 
 * Googletest: `GTEST_HOME` (only required to build the unit tests)
 * Google Benchmark: `GBENCHMARK_HOME` (only required if building benchmarks)
-
+* Flatbuffers: `FLATBUFFERS_HOME` (only required for the IPC extensions)
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 133adf32cbd50..097634d74f890 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -34,13 +34,10 @@ class Buffer;
 //
 // The base class is only required to have a null bitmap buffer if the null
 // count is greater than 0
-//
-// Any buffers used to initialize the array have their references "stolen". If
-// you wish to use the buffer beyond the lifetime of the array, you need to
-// explicitly increment its reference count
 class Array {
  public:
-  Array(const TypePtr& type, int32_t length, int32_t null_count = 0,
+  Array(const std::shared_ptr<DataType>& type, int32_t length,
+      int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   virtual ~Array() {}
@@ -60,11 +57,15 @@ class Array {
     return null_bitmap_;
   }
 
+  const uint8_t* null_bitmap_data() const {
+    return null_bitmap_data_;
+  }
+
   bool EqualsExact(const Array& arr) const;
   virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
 
  protected:
-  TypePtr type_;
+  std::shared_ptr<DataType> type_;
   int32_t null_count_;
   int32_t length_;
 
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index dea42e102b0d0..80b075cdfbb23 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -20,8 +20,18 @@
 #include <sstream>
 #include <string>
 
+#include "arrow/type.h"
+
 namespace arrow {
 
+const std::shared_ptr<DataType> STRING(new StringType());
+
+StringArray::StringArray(int32_t length,
+    const std::shared_ptr<Buffer>& offsets,
+    const ArrayPtr& values, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap) :
+    StringArray(STRING, length, offsets, values, null_count, null_bitmap) {}
+
 std::string CharType::ToString() const {
   std::stringstream s;
   s << "char(" << size << ")";
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index fda722ba6def2..84cd0326ec850 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -79,9 +79,7 @@ class StringArray : public ListArray {
       const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values,
       int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr) :
-      StringArray(std::make_shared<StringType>(), length, offsets, values,
-          null_count, null_bitmap) {}
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   // Compute the pointer t
   const uint8_t* GetValue(int i, int32_t* out_length) const {
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 0c3e210abd910..c15f9b630cd97 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -18,11 +18,13 @@
 #ifndef ARROW_UTIL_BUFFER_H
 #define ARROW_UTIL_BUFFER_H
 
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <memory>
 
 #include "arrow/util/macros.h"
+#include "arrow/util/status.h"
 
 namespace arrow {
 
@@ -146,6 +148,46 @@ class PoolBuffer : public ResizableBuffer {
   MemoryPool* pool_;
 };
 
+static constexpr int64_t MIN_BUFFER_CAPACITY = 1024;
+
+class BufferBuilder {
+ public:
+  explicit BufferBuilder(MemoryPool* pool) :
+      pool_(pool),
+      capacity_(0),
+      size_(0) {}
+
+  Status Append(const uint8_t* data, int length) {
+    if (capacity_ < length + size_) {
+      if (capacity_ == 0) {
+        buffer_ = std::make_shared<PoolBuffer>(pool_);
+      }
+      capacity_ = std::max(MIN_BUFFER_CAPACITY, capacity_);
+      while (capacity_ < length + size_) {
+        capacity_ *= 2;
+      }
+      RETURN_NOT_OK(buffer_->Resize(capacity_));
+      data_ = buffer_->mutable_data();
+    }
+    memcpy(data_ + size_, data, length);
+    size_ += length;
+    return Status::OK();
+  }
+
+  std::shared_ptr<Buffer> Finish() {
+    auto result = buffer_;
+    buffer_ = nullptr;
+    return result;
+  }
+
+ private:
+  std::shared_ptr<PoolBuffer> buffer_;
+  MemoryPool* pool_;
+  uint8_t* data_;
+  int64_t capacity_;
+  int64_t size_;
+};
+
 } // namespace arrow
 
 #endif // ARROW_UTIL_BUFFER_H
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 0ecafc7202e89..ebe825f65c4da 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -220,9 +220,12 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 
 ## Python and libraries
 find_package(PythonLibsNew REQUIRED)
+find_package(NumPy REQUIRED)
 include(UseCython)
 
 include_directories(SYSTEM
+  ${NUMPY_INCLUDE_DIRS}
+  ${PYTHON_INCLUDE_DIRS}
   src)
 
 ############################################################
@@ -409,11 +412,12 @@ add_subdirectory(src/pyarrow/util)
 
 set(PYARROW_SRCS
   src/pyarrow/common.cc
+  src/pyarrow/config.cc
   src/pyarrow/helpers.cc
-  src/pyarrow/init.cc
   src/pyarrow/status.cc
 
   src/pyarrow/adapters/builtin.cc
+  src/pyarrow/adapters/pandas.cc
 )
 
 set(LINK_LIBS
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 9a080709bebda..c343f5ba5f129 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -17,7 +17,11 @@
 
 # flake8: noqa
 
-from pyarrow.array import (Array, from_pylist, total_allocated_bytes,
+import pyarrow.config
+
+from pyarrow.array import (Array,
+                           from_pandas_series, from_pylist,
+                           total_allocated_bytes,
                            BooleanArray, NumericArray,
                            Int8Array, UInt8Array,
                            ListArray, StringArray)
@@ -37,4 +41,4 @@
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
-from pyarrow.array import RowBatch
+from pyarrow.array import RowBatch, Table, from_pandas_dataframe
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index c5d40ddd7a481..88770cdaa966e 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -22,6 +22,8 @@
 from pyarrow.includes.libarrow cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 
+import pyarrow.config
+
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.error cimport check_status
 
@@ -44,6 +46,10 @@ cdef class Array:
         self.type = DataType()
         self.type.init(self.sp_array.get().type())
 
+    @staticmethod
+    def from_pandas(obj, mask=None):
+        return from_pandas_series(obj, mask)
+
     property null_count:
 
         def __get__(self):
@@ -160,7 +166,15 @@ cdef class StringArray(Array):
 cdef dict _array_classes = {
     Type_NA: NullArray,
     Type_BOOL: BooleanArray,
+    Type_UINT8: UInt8Array,
+    Type_UINT16: UInt16Array,
+    Type_UINT32: UInt32Array,
+    Type_UINT64: UInt64Array,
+    Type_INT8: Int8Array,
+    Type_INT16: Int16Array,
+    Type_INT32: Int32Array,
     Type_INT64: Int64Array,
+    Type_FLOAT: FloatArray,
     Type_DOUBLE: DoubleArray,
     Type_LIST: ListArray,
     Type_STRING: StringArray,
@@ -194,6 +208,49 @@ def from_pylist(object list_obj, DataType type=None):
 
     return box_arrow_array(sp_array)
 
+
+def from_pandas_series(object series, object mask=None):
+    cdef:
+        shared_ptr[CArray] out
+
+    series_values = series_as_ndarray(series)
+
+    if mask is None:
+        check_status(pyarrow.PandasToArrow(pyarrow.GetMemoryPool(),
+                                           series_values, &out))
+    else:
+        mask = series_as_ndarray(mask)
+        check_status(pyarrow.PandasMaskedToArrow(
+            pyarrow.GetMemoryPool(), series_values, mask, &out))
+
+    return box_arrow_array(out)
+
+
+def from_pandas_dataframe(object df, name=None):
+    cdef:
+        list names = []
+        list arrays = []
+
+    for name in df.columns:
+        col = df[name]
+        arr = from_pandas_series(col)
+
+        names.append(name)
+        arrays.append(arr)
+
+    return Table.from_arrays(names, arrays, name=name)
+
+
+cdef object series_as_ndarray(object obj):
+    import pandas as pd
+
+    if isinstance(obj, pd.Series):
+        result = obj.values
+    else:
+        result = obj
+
+    return result
+
 #----------------------------------------------------------------------
 # Table-like data structures
 
@@ -225,3 +282,81 @@ cdef class RowBatch:
 
     def __getitem__(self, i):
         return self.arrays[i]
+
+
+cdef class Table:
+    '''
+    Do not call this class's constructor directly.
+    '''
+    cdef:
+        shared_ptr[CTable] sp_table
+        CTable* table
+
+    def __cinit__(self):
+        pass
+
+    cdef init(self, const shared_ptr[CTable]& table):
+        self.sp_table = table
+        self.table = table.get()
+
+    @staticmethod
+    def from_pandas(df, name=None):
+        pass
+
+    @staticmethod
+    def from_arrays(names, arrays, name=None):
+        cdef:
+            Array arr
+            Table result
+            c_string c_name
+            vector[shared_ptr[CField]] fields
+            vector[shared_ptr[CColumn]] columns
+            shared_ptr[CSchema] schema
+            shared_ptr[CTable] table
+
+        cdef int K = len(arrays)
+
+        fields.resize(K)
+        columns.resize(K)
+        for i in range(K):
+            arr = arrays[i]
+            c_name = tobytes(names[i])
+
+            fields[i].reset(new CField(c_name, arr.type.sp_type, True))
+            columns[i].reset(new CColumn(fields[i], arr.sp_array))
+
+        if name is None:
+            c_name = ''
+        else:
+            c_name = tobytes(name)
+
+        schema.reset(new CSchema(fields))
+        table.reset(new CTable(c_name, schema, columns))
+
+        result = Table()
+        result.init(table)
+
+        return result
+
+    def to_pandas(self):
+        """
+        Convert the arrow::Table to a pandas DataFrame
+        """
+        cdef:
+            PyObject* arr
+            shared_ptr[CColumn] col
+
+        import pandas as pd
+
+        names = []
+        data = []
+        for i in range(self.table.num_columns()):
+            col = self.table.column(i)
+            check_status(pyarrow.ArrowToPandas(col, &arr))
+            names.append(frombytes(col.get().name()))
+            data.append(<object> arr)
+
+            # One ref count too many
+            Py_XDECREF(arr)
+
+        return pd.DataFrame(dict(zip(names, data)), columns=names)
diff --git a/python/pyarrow/config.pyx b/python/pyarrow/config.pyx
index 521bc066cd4a5..1047a472fe338 100644
--- a/python/pyarrow/config.pyx
+++ b/python/pyarrow/config.pyx
@@ -2,7 +2,18 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-cdef extern from 'pyarrow/init.h' namespace 'pyarrow':
+cdef extern from 'pyarrow/do_import_numpy.h':
+    pass
+
+cdef extern from 'pyarrow/numpy_interop.h' namespace 'pyarrow':
+    int import_numpy()
+
+cdef extern from 'pyarrow/config.h' namespace 'pyarrow':
     void pyarrow_init()
+    void pyarrow_set_numpy_nan(object o)
 
+import_numpy()
 pyarrow_init()
+
+import numpy as np
+pyarrow_set_numpy_nan(np.nan)
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 839427a699002..e86d5d77e8b10 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -22,10 +22,16 @@ from libcpp cimport bool as c_bool
 from libcpp.string cimport string as c_string
 from libcpp.vector cimport vector
 
+from cpython cimport PyObject
+cimport cpython
+
 # This must be included for cerr and other things to work
 cdef extern from "<iostream>":
     pass
 
+cdef extern from "<Python.h>":
+    void Py_XDECREF(PyObject* o)
+
 cdef extern from "<memory>" namespace "std" nogil:
 
     cdef cppclass shared_ptr[T]:
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 943a08f84a055..42f1f25073d1b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -19,6 +19,25 @@
 
 from pyarrow.includes.common cimport *
 
+cdef extern from "arrow/api.h" namespace "arrow" nogil:
+    # We can later add more of the common status factory methods as needed
+    cdef CStatus CStatus_OK "Status::OK"()
+
+    cdef cppclass CStatus "arrow::Status":
+        CStatus()
+
+        c_string ToString()
+
+        c_bool ok()
+        c_bool IsOutOfMemory()
+        c_bool IsKeyError()
+        c_bool IsNotImplemented()
+        c_bool IsInvalid()
+
+    cdef cppclass Buffer:
+        uint8_t* data()
+        int64_t size()
+
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     enum Type" arrow::Type::type":
@@ -129,25 +148,30 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CStringArray" arrow::StringArray"(CListArray):
         c_string GetString(int i)
 
+    cdef cppclass CChunkedArray" arrow::ChunkedArray":
+        pass
 
-cdef extern from "arrow/api.h" namespace "arrow" nogil:
-    # We can later add more of the common status factory methods as needed
-    cdef CStatus CStatus_OK "Status::OK"()
+    cdef cppclass CColumn" arrow::Column":
+        CColumn(const shared_ptr[CField]& field,
+                const shared_ptr[CArray]& data)
 
-    cdef cppclass CStatus "arrow::Status":
-        CStatus()
+        int64_t length()
+        int64_t null_count()
+        const c_string& name()
+        const shared_ptr[CDataType]& type()
+        const shared_ptr[CChunkedArray]& data()
 
-        c_string ToString()
+    cdef cppclass CTable" arrow::Table":
+        CTable(const c_string& name, const shared_ptr[CSchema]& schema,
+               const vector[shared_ptr[CColumn]]& columns)
 
-        c_bool ok()
-        c_bool IsOutOfMemory()
-        c_bool IsKeyError()
-        c_bool IsNotImplemented()
-        c_bool IsInvalid()
+        int num_columns()
+        int num_rows()
 
-    cdef cppclass Buffer:
-        uint8_t* data()
-        int64_t size()
+        const c_string& name()
+
+        const shared_ptr[CSchema]& schema()
+        const shared_ptr[CColumn]& column(int i)
 
 
 cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index eedfc85446810..1066b8034be70 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -18,7 +18,8 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport CArray, CDataType, Type, MemoryPool
+from pyarrow.includes.libarrow cimport (CArray, CColumn, CDataType,
+                                        Type, MemoryPool)
 
 cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     # We can later add more of the common status factory methods as needed
@@ -41,4 +42,10 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     shared_ptr[CDataType] GetPrimitiveType(Type type)
     Status ConvertPySequence(object obj, shared_ptr[CArray]* out)
 
+    Status PandasToArrow(MemoryPool* pool, object ao, shared_ptr[CArray]* out)
+    Status PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
+                               shared_ptr[CArray]* out)
+
+    Status ArrowToPandas(const shared_ptr[CColumn]& arr, PyObject** out)
+
     MemoryPool* GetMemoryPool()
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
new file mode 100644
index 0000000000000..6dc9c689e249b
--- /dev/null
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -0,0 +1,172 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+import numpy as np
+
+import pandas as pd
+import pandas.util.testing as tm
+
+import pyarrow as A
+
+
+class TestPandasConversion(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def _check_pandas_roundtrip(self, df, expected=None):
+        table = A.from_pandas_dataframe(df)
+        result = table.to_pandas()
+        if expected is None:
+            expected = df
+        tm.assert_frame_equal(result, expected)
+
+    def test_float_no_nulls(self):
+        data = {}
+        numpy_dtypes = ['f4', 'f8']
+        num_values = 100
+
+        for dtype in numpy_dtypes:
+            values = np.random.randn(num_values)
+            data[dtype] = values.astype(dtype)
+
+        df = pd.DataFrame(data)
+        self._check_pandas_roundtrip(df)
+
+    def test_float_nulls(self):
+        num_values = 100
+
+        null_mask = np.random.randint(0, 10, size=num_values) < 3
+        dtypes = ['f4', 'f8']
+        expected_cols = []
+
+        arrays = []
+        for name in dtypes:
+            values = np.random.randn(num_values).astype(name)
+
+            arr = A.from_pandas_series(values, null_mask)
+            arrays.append(arr)
+
+            values[null_mask] = np.nan
+
+            expected_cols.append(values)
+
+        ex_frame = pd.DataFrame(dict(zip(dtypes, expected_cols)),
+                                columns=dtypes)
+
+        table = A.Table.from_arrays(dtypes, arrays)
+        result = table.to_pandas()
+        tm.assert_frame_equal(result, ex_frame)
+
+    def test_integer_no_nulls(self):
+        data = {}
+
+        numpy_dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8']
+        num_values = 100
+
+        for dtype in numpy_dtypes:
+            info = np.iinfo(dtype)
+            values = np.random.randint(info.min,
+                                       min(info.max, np.iinfo('i8').max),
+                                       size=num_values)
+            data[dtype] = values.astype(dtype)
+
+        df = pd.DataFrame(data)
+        self._check_pandas_roundtrip(df)
+
+    def test_integer_with_nulls(self):
+        # pandas requires upcast to float dtype
+
+        int_dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8']
+        num_values = 100
+
+        null_mask = np.random.randint(0, 10, size=num_values) < 3
+
+        expected_cols = []
+        arrays = []
+        for name in int_dtypes:
+            values = np.random.randint(0, 100, size=num_values)
+
+            arr = A.from_pandas_series(values, null_mask)
+            arrays.append(arr)
+
+            expected = values.astype('f8')
+            expected[null_mask] = np.nan
+
+            expected_cols.append(expected)
+
+        ex_frame = pd.DataFrame(dict(zip(int_dtypes, expected_cols)),
+                                columns=int_dtypes)
+
+        table = A.Table.from_arrays(int_dtypes, arrays)
+        result = table.to_pandas()
+
+        tm.assert_frame_equal(result, ex_frame)
+
+    def test_boolean_no_nulls(self):
+        num_values = 100
+
+        np.random.seed(0)
+
+        df = pd.DataFrame({'bools': np.random.randn(num_values) > 0})
+        self._check_pandas_roundtrip(df)
+
+    def test_boolean_nulls(self):
+        # pandas requires upcast to object dtype
+        num_values = 100
+        np.random.seed(0)
+
+        mask = np.random.randint(0, 10, size=num_values) < 3
+        values = np.random.randint(0, 10, size=num_values) < 5
+
+        arr = A.from_pandas_series(values, mask)
+
+        expected = values.astype(object)
+        expected[mask] = None
+
+        ex_frame = pd.DataFrame({'bools': expected})
+
+        table = A.Table.from_arrays(['bools'], [arr])
+        result = table.to_pandas()
+
+        tm.assert_frame_equal(result, ex_frame)
+
+    def test_boolean_object_nulls(self):
+        arr = np.array([False, None, True] * 100, dtype=object)
+        df = pd.DataFrame({'bools': arr})
+        self._check_pandas_roundtrip(df)
+
+    def test_strings(self):
+        repeats = 1000
+        values = [b'foo', None, u'bar', 'qux', np.nan]
+        df = pd.DataFrame({'strings': values * repeats})
+
+        values = ['foo', None, u'bar', 'qux', None]
+        expected = pd.DataFrame({'strings': values * repeats})
+        self._check_pandas_roundtrip(df, expected)
+
+    # def test_category(self):
+    #     repeats = 1000
+    #     values = [b'foo', None, u'bar', 'qux', np.nan]
+    #     df = pd.DataFrame({'strings': values * repeats})
+    #     df['strings'] = df['strings'].astype('category')
+    #     self._check_pandas_roundtrip(df)
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
new file mode 100644
index 0000000000000..22f1d7575f8c5
--- /dev/null
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -0,0 +1,714 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Functions for pandas conversion via NumPy
+
+#include <Python.h>
+
+#include "pyarrow/numpy_interop.h"
+
+#include <cmath>
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include "arrow/api.h"
+#include "arrow/util/bit-util.h"
+
+#include "pyarrow/common.h"
+#include "pyarrow/config.h"
+#include "pyarrow/status.h"
+
+namespace pyarrow {
+
+using arrow::Array;
+using arrow::Column;
+namespace util = arrow::util;
+
+// ----------------------------------------------------------------------
+// Serialization
+
+template <int TYPE>
+struct npy_traits {
+};
+
+template <>
+struct npy_traits<NPY_BOOL> {
+  typedef uint8_t value_type;
+  using ArrayType = arrow::BooleanArray;
+
+  static constexpr bool supports_nulls = false;
+  static inline bool isnull(uint8_t v) {
+    return false;
+  }
+};
+
+#define NPY_INT_DECL(TYPE, CapType, T)              \
+  template <>                                       \
+  struct npy_traits<NPY_##TYPE> {                   \
+    typedef T value_type;                           \
+    using ArrayType = arrow::CapType##Array;        \
+                                                    \
+    static constexpr bool supports_nulls = false;   \
+    static inline bool isnull(T v) {                \
+      return false;                                 \
+    }                                               \
+  };
+
+NPY_INT_DECL(INT8, Int8, int8_t);
+NPY_INT_DECL(INT16, Int16, int16_t);
+NPY_INT_DECL(INT32, Int32, int32_t);
+NPY_INT_DECL(INT64, Int64, int64_t);
+NPY_INT_DECL(UINT8, UInt8, uint8_t);
+NPY_INT_DECL(UINT16, UInt16, uint16_t);
+NPY_INT_DECL(UINT32, UInt32, uint32_t);
+NPY_INT_DECL(UINT64, UInt64, uint64_t);
+
+template <>
+struct npy_traits<NPY_FLOAT32> {
+  typedef float value_type;
+  using ArrayType = arrow::FloatArray;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(float v) {
+    return v != v;
+  }
+};
+
+template <>
+struct npy_traits<NPY_FLOAT64> {
+  typedef double value_type;
+  using ArrayType = arrow::DoubleArray;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(double v) {
+    return v != v;
+  }
+};
+
+template <>
+struct npy_traits<NPY_OBJECT> {
+  typedef PyObject* value_type;
+  static constexpr bool supports_nulls = true;
+};
+
+template <int TYPE>
+class ArrowSerializer {
+ public:
+  ArrowSerializer(arrow::MemoryPool* pool, PyArrayObject* arr, PyArrayObject* mask) :
+      pool_(pool),
+      arr_(arr),
+      mask_(mask) {
+    length_ = PyArray_SIZE(arr_);
+  }
+
+  Status Convert(std::shared_ptr<Array>* out);
+
+  int stride() const {
+    return PyArray_STRIDES(arr_)[0];
+  }
+
+  Status InitNullBitmap() {
+    int null_bytes = util::bytes_for_bits(length_);
+
+    null_bitmap_ = std::make_shared<arrow::PoolBuffer>(pool_);
+    RETURN_ARROW_NOT_OK(null_bitmap_->Resize(null_bytes));
+
+    null_bitmap_data_ = null_bitmap_->mutable_data();
+    memset(null_bitmap_data_, 0, null_bytes);
+
+    return Status::OK();
+  }
+
+  bool is_strided() const {
+    npy_intp* astrides = PyArray_STRIDES(arr_);
+    return astrides[0] != PyArray_DESCR(arr_)->elsize;
+  }
+
+ private:
+  Status ConvertData();
+
+  Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
+    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+
+    auto offsets_buffer = std::make_shared<arrow::PoolBuffer>(pool_);
+    RETURN_ARROW_NOT_OK(offsets_buffer->Resize(sizeof(int32_t) * (length_ + 1)));
+    int32_t* offsets = reinterpret_cast<int32_t*>(offsets_buffer->mutable_data());
+
+    arrow::BufferBuilder data_builder(pool_);
+    arrow::Status s;
+    PyObject* obj;
+    int length;
+    int offset = 0;
+    int64_t null_count = 0;
+    for (int64_t i = 0; i < length_; ++i) {
+      obj = objects[i];
+      if (PyUnicode_Check(obj)) {
+        obj = PyUnicode_AsUTF8String(obj);
+        if (obj == NULL) {
+          PyErr_Clear();
+          return Status::TypeError("failed converting unicode to UTF8");
+        }
+        length = PyBytes_GET_SIZE(obj);
+        s = data_builder.Append(
+            reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)), length);
+        Py_DECREF(obj);
+        if (!s.ok()) {
+          return Status::ArrowError(s.ToString());
+        }
+        util::set_bit(null_bitmap_data_, i);
+      } else if (PyBytes_Check(obj)) {
+        length = PyBytes_GET_SIZE(obj);
+        RETURN_ARROW_NOT_OK(data_builder.Append(
+                reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)), length));
+        util::set_bit(null_bitmap_data_, i);
+      } else {
+        // NULL
+        // No change to offset
+        length = 0;
+        ++null_count;
+      }
+      offsets[i] = offset;
+      offset += length;
+    }
+    // End offset
+    offsets[length_] = offset;
+
+    std::shared_ptr<arrow::Buffer> data_buffer = data_builder.Finish();
+
+    auto values = std::make_shared<arrow::UInt8Array>(data_buffer->size(),
+        data_buffer);
+    *out = std::shared_ptr<arrow::Array>(
+        new arrow::StringArray(length_, offsets_buffer, values, null_count,
+            null_bitmap_));
+
+    return Status::OK();
+  }
+
+  Status ConvertBooleans(std::shared_ptr<Array>* out) {
+    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+
+    int nbytes = util::bytes_for_bits(length_);
+    auto data = std::make_shared<arrow::PoolBuffer>(pool_);
+    RETURN_ARROW_NOT_OK(data->Resize(nbytes));
+    uint8_t* bitmap = data->mutable_data();
+    memset(bitmap, 0, nbytes);
+
+    int64_t null_count = 0;
+    for (int64_t i = 0; i < length_; ++i) {
+      if (objects[i] == Py_True) {
+        util::set_bit(bitmap, i);
+        util::set_bit(null_bitmap_data_, i);
+      } else if (objects[i] != Py_False) {
+        ++null_count;
+      } else {
+        util::set_bit(null_bitmap_data_, i);
+      }
+    }
+
+    *out = std::make_shared<arrow::BooleanArray>(length_, data, null_count,
+        null_bitmap_);
+
+    return Status::OK();
+  }
+
+  arrow::MemoryPool* pool_;
+
+  PyArrayObject* arr_;
+  PyArrayObject* mask_;
+
+  int64_t length_;
+
+  std::shared_ptr<arrow::Buffer> data_;
+  std::shared_ptr<arrow::ResizableBuffer> null_bitmap_;
+  uint8_t* null_bitmap_data_;
+};
+
+// Returns null count
+static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
+  int64_t null_count = 0;
+  const uint8_t* mask_values = static_cast<const uint8_t*>(PyArray_DATA(mask));
+  // TODO(wesm): strided null mask
+  for (int i = 0; i < length; ++i) {
+    if (mask_values[i]) {
+      ++null_count;
+    } else {
+      util::set_bit(bitmap, i);
+    }
+  }
+  return null_count;
+}
+
+template <int TYPE>
+static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap) {
+  typedef npy_traits<TYPE> traits;
+  typedef typename traits::value_type T;
+
+  int64_t null_count = 0;
+  const T* values = reinterpret_cast<const T*>(data);
+
+  // TODO(wesm): striding
+  for (int i = 0; i < length; ++i) {
+    if (traits::isnull(values[i])) {
+      ++null_count;
+    } else {
+      util::set_bit(bitmap, i);
+    }
+  }
+
+  return null_count;
+}
+
+template <int TYPE>
+inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
+  typedef npy_traits<TYPE> traits;
+
+  if (mask_ != nullptr || traits::supports_nulls) {
+    RETURN_NOT_OK(InitNullBitmap());
+  }
+
+  int64_t null_count = 0;
+  if (mask_ != nullptr) {
+    null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
+  } else if (traits::supports_nulls) {
+    null_count = ValuesToBitmap<TYPE>(PyArray_DATA(arr_), length_, null_bitmap_data_);
+  }
+
+  RETURN_NOT_OK(ConvertData());
+  *out = std::make_shared<typename traits::ArrayType>(length_, data_, null_count,
+      null_bitmap_);
+
+  return Status::OK();
+}
+
+static inline bool PyObject_is_null(const PyObject* obj) {
+  return obj == Py_None || obj == numpy_nan;
+}
+
+static inline bool PyObject_is_string(const PyObject* obj) {
+#if PY_MAJOR_VERSION >= 3
+  return PyUnicode_Check(obj) || PyBytes_Check(obj);
+#else
+  return PyString_Check(obj) || PyUnicode_Check(obj);
+#endif
+}
+
+static inline bool PyObject_is_bool(const PyObject* obj) {
+#if PY_MAJOR_VERSION >= 3
+  return PyString_Check(obj) || PyBytes_Check(obj);
+#else
+  return PyString_Check(obj) || PyUnicode_Check(obj);
+#endif
+}
+
+template <>
+inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out) {
+  // Python object arrays are annoying, since we could have one of:
+  //
+  // * Strings
+  // * Booleans with nulls
+  // * Mixed type (not supported at the moment by arrow format)
+  //
+  // Additionally, nulls may be encoded either as np.nan or None. So we have to
+  // do some type inference and conversion
+
+  RETURN_NOT_OK(InitNullBitmap());
+
+  // TODO: mask not supported here
+  const PyObject** objects = reinterpret_cast<const PyObject**>(PyArray_DATA(arr_));
+
+  for (int64_t i = 0; i < length_; ++i) {
+    if (PyObject_is_null(objects[i])) {
+      continue;
+    } else if (PyObject_is_string(objects[i])) {
+      return ConvertObjectStrings(out);
+    } else if (PyBool_Check(objects[i])) {
+      return ConvertBooleans(out);
+    } else {
+      return Status::TypeError("unhandled python type");
+    }
+  }
+
+  return Status::TypeError("Unable to infer type of object array, were all null");
+}
+
+template <int TYPE>
+inline Status ArrowSerializer<TYPE>::ConvertData() {
+  // TODO(wesm): strided arrays
+  if (is_strided()) {
+    return Status::ValueError("no support for strided data yet");
+  }
+
+  data_ = std::make_shared<NumPyBuffer>(arr_);
+  return Status::OK();
+}
+
+template <>
+inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
+  if (is_strided()) {
+    return Status::ValueError("no support for strided data yet");
+  }
+
+  int nbytes = util::bytes_for_bits(length_);
+  auto buffer = std::make_shared<arrow::PoolBuffer>(pool_);
+  RETURN_ARROW_NOT_OK(buffer->Resize(nbytes));
+
+  const uint8_t* values = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
+
+  uint8_t* bitmap = buffer->mutable_data();
+
+  memset(bitmap, 0, nbytes);
+  for (int i = 0; i < length_; ++i) {
+    if (values[i] > 0) {
+      util::set_bit(bitmap, i);
+    }
+  }
+
+  data_ = buffer;
+
+  return Status::OK();
+}
+
+template <>
+inline Status ArrowSerializer<NPY_OBJECT>::ConvertData() {
+  return Status::TypeError("NYI");
+}
+
+
+#define TO_ARROW_CASE(TYPE)                                     \
+  case NPY_##TYPE:                                              \
+    {                                                           \
+      ArrowSerializer<NPY_##TYPE> converter(pool, arr, mask);   \
+      RETURN_NOT_OK(converter.Convert(out));                    \
+    }                                                           \
+    break;
+
+Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
+    std::shared_ptr<Array>* out) {
+  PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(ao);
+  PyArrayObject* mask = nullptr;
+
+  if (mo != nullptr) {
+    mask = reinterpret_cast<PyArrayObject*>(mo);
+  }
+
+  if (PyArray_NDIM(arr) != 1) {
+    return Status::ValueError("only handle 1-dimensional arrays");
+  }
+
+  switch(PyArray_DESCR(arr)->type_num) {
+    TO_ARROW_CASE(BOOL);
+    TO_ARROW_CASE(INT8);
+    TO_ARROW_CASE(INT16);
+    TO_ARROW_CASE(INT32);
+    TO_ARROW_CASE(INT64);
+    TO_ARROW_CASE(UINT8);
+    TO_ARROW_CASE(UINT16);
+    TO_ARROW_CASE(UINT32);
+    TO_ARROW_CASE(UINT64);
+    TO_ARROW_CASE(FLOAT32);
+    TO_ARROW_CASE(FLOAT64);
+    TO_ARROW_CASE(OBJECT);
+    default:
+      std::stringstream ss;
+      ss << "unsupported type " << PyArray_DESCR(arr)->type_num
+         << std::endl;
+      return Status::NotImplemented(ss.str());
+  }
+  return Status::OK();
+}
+
+Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
+    std::shared_ptr<Array>* out) {
+  return PandasMaskedToArrow(pool, ao, nullptr, out);
+}
+
+// ----------------------------------------------------------------------
+// Deserialization
+
+template <int TYPE>
+struct arrow_traits {
+};
+
+template <>
+struct arrow_traits<arrow::Type::BOOL> {
+  static constexpr int npy_type = NPY_BOOL;
+  static constexpr bool supports_nulls = false;
+  static constexpr bool is_boolean = true;
+  static constexpr bool is_integer = false;
+  static constexpr bool is_floating = false;
+};
+
+#define INT_DECL(TYPE)                                      \
+  template <>                                               \
+  struct arrow_traits<arrow::Type::TYPE> {              \
+    static constexpr int npy_type = NPY_##TYPE;             \
+    static constexpr bool supports_nulls = false;           \
+    static constexpr double na_value = NAN;                 \
+    static constexpr bool is_boolean = false;               \
+    static constexpr bool is_integer = true;                \
+    static constexpr bool is_floating = false;              \
+    typedef typename npy_traits<NPY_##TYPE>::value_type T;  \
+  };
+
+INT_DECL(INT8);
+INT_DECL(INT16);
+INT_DECL(INT32);
+INT_DECL(INT64);
+INT_DECL(UINT8);
+INT_DECL(UINT16);
+INT_DECL(UINT32);
+INT_DECL(UINT64);
+
+template <>
+struct arrow_traits<arrow::Type::FLOAT> {
+  static constexpr int npy_type = NPY_FLOAT32;
+  static constexpr bool supports_nulls = true;
+  static constexpr float na_value = NAN;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_integer = false;
+  static constexpr bool is_floating = true;
+  typedef typename npy_traits<NPY_FLOAT32>::value_type T;
+};
+
+template <>
+struct arrow_traits<arrow::Type::DOUBLE> {
+  static constexpr int npy_type = NPY_FLOAT64;
+  static constexpr bool supports_nulls = true;
+  static constexpr double na_value = NAN;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_integer = false;
+  static constexpr bool is_floating = true;
+  typedef typename npy_traits<NPY_FLOAT64>::value_type T;
+};
+
+template <>
+struct arrow_traits<arrow::Type::STRING> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_integer = false;
+  static constexpr bool is_floating = false;
+};
+
+
+static inline PyObject* make_pystring(const uint8_t* data, int32_t length) {
+#if PY_MAJOR_VERSION >= 3
+  return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+#else
+  return PyString_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+#endif
+}
+
+template <int TYPE>
+class ArrowDeserializer {
+ public:
+  ArrowDeserializer(const std::shared_ptr<Column>& col) :
+      col_(col) {}
+
+  Status Convert(PyObject** out) {
+    const std::shared_ptr<arrow::ChunkedArray> data = col_->data();
+    if (data->num_chunks() > 1) {
+      return Status::NotImplemented("Chunked column conversion NYI");
+    }
+
+    auto chunk = data->chunk(0);
+
+    RETURN_NOT_OK(ConvertValues<TYPE>(chunk));
+    *out = reinterpret_cast<PyObject*>(out_);
+    return Status::OK();
+  }
+
+  Status AllocateOutput(int type) {
+    npy_intp dims[1] = {col_->length()};
+    out_ = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNew(1, dims, type));
+
+    if (out_ == NULL) {
+      // Error occurred, trust that SimpleNew set the error state
+      return Status::OK();
+    }
+
+    return Status::OK();
+  }
+
+  template <int T2>
+  inline typename std::enable_if<
+    arrow_traits<T2>::is_floating, Status>::type
+  ConvertValues(const std::shared_ptr<Array>& arr) {
+    typedef typename arrow_traits<T2>::T T;
+
+    arrow::PrimitiveArray* prim_arr = static_cast<arrow::PrimitiveArray*>(
+        arr.get());
+
+    RETURN_NOT_OK(AllocateOutput(arrow_traits<T2>::npy_type));
+
+    if (arr->null_count() > 0) {
+      T* out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
+      const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        out_values[i] = arr->IsNull(i) ? NAN : in_values[i];
+      }
+    } else {
+      memcpy(PyArray_DATA(out_), prim_arr->data()->data(),
+          arr->length() * arr->type()->value_size());
+    }
+
+    return Status::OK();
+  }
+
+  // Integer specialization
+  template <int T2>
+  inline typename std::enable_if<
+    arrow_traits<T2>::is_integer, Status>::type
+  ConvertValues(const std::shared_ptr<Array>& arr) {
+    typedef typename arrow_traits<T2>::T T;
+
+    arrow::PrimitiveArray* prim_arr = static_cast<arrow::PrimitiveArray*>(
+        arr.get());
+
+    const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+
+    if (arr->null_count() > 0) {
+      RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
+
+      // Upcast to double, set NaN as appropriate
+      double* out_values = reinterpret_cast<double*>(PyArray_DATA(out_));
+      for (int i = 0; i < arr->length(); ++i) {
+        out_values[i] = prim_arr->IsNull(i) ? NAN : in_values[i];
+      }
+    } else {
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+
+      memcpy(PyArray_DATA(out_), in_values,
+          arr->length() * arr->type()->value_size());
+    }
+
+    return Status::OK();
+  }
+
+  // Boolean specialization
+  template <int T2>
+  inline typename std::enable_if<
+    arrow_traits<T2>::is_boolean, Status>::type
+  ConvertValues(const std::shared_ptr<Array>& arr) {
+    arrow::BooleanArray* bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
+
+    if (arr->null_count() > 0) {
+      RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+
+      PyObject** out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        if (bool_arr->IsNull(i)) {
+          Py_INCREF(Py_None);
+          out_values[i] = Py_None;
+        } else if (bool_arr->Value(i)) {
+          // True
+          Py_INCREF(Py_True);
+          out_values[i] = Py_True;
+        } else {
+          // False
+          Py_INCREF(Py_False);
+          out_values[i] = Py_False;
+        }
+      }
+    } else {
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+
+      uint8_t* out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(out_));
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        out_values[i] = static_cast<uint8_t>(bool_arr->Value(i));
+      }
+    }
+
+    return Status::OK();
+  }
+
+  // UTF8
+  template <int T2>
+  inline typename std::enable_if<
+    T2 == arrow::Type::STRING, Status>::type
+  ConvertValues(const std::shared_ptr<Array>& arr) {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+
+    PyObject** out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
+
+    arrow::StringArray* string_arr = static_cast<arrow::StringArray*>(arr.get());
+
+    const uint8_t* data;
+    int32_t length;
+    if (arr->null_count() > 0) {
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        if (string_arr->IsNull(i)) {
+          Py_INCREF(Py_None);
+          out_values[i] = Py_None;
+        } else {
+          data = string_arr->GetValue(i, &length);
+
+          out_values[i] = make_pystring(data, length);
+          if (out_values[i] == nullptr) {
+            return Status::OK();
+          }
+        }
+      }
+    } else {
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        data = string_arr->GetValue(i, &length);
+        out_values[i] = make_pystring(data, length);
+        if (out_values[i] == nullptr) {
+          return Status::OK();
+        }
+      }
+    }
+    return Status::OK();
+  }
+ private:
+  std::shared_ptr<Column> col_;
+  PyArrayObject* out_;
+};
+
+#define FROM_ARROW_CASE(TYPE)                               \
+  case arrow::Type::TYPE:                                   \
+    {                                                       \
+      ArrowDeserializer<arrow::Type::TYPE> converter(col);  \
+      return converter.Convert(out);                        \
+    }                                                       \
+    break;
+
+Status ArrowToPandas(const std::shared_ptr<Column>& col, PyObject** out) {
+  switch(col->type()->type) {
+    FROM_ARROW_CASE(BOOL);
+    FROM_ARROW_CASE(INT8);
+    FROM_ARROW_CASE(INT16);
+    FROM_ARROW_CASE(INT32);
+    FROM_ARROW_CASE(INT64);
+    FROM_ARROW_CASE(UINT8);
+    FROM_ARROW_CASE(UINT16);
+    FROM_ARROW_CASE(UINT32);
+    FROM_ARROW_CASE(UINT64);
+    FROM_ARROW_CASE(FLOAT);
+    FROM_ARROW_CASE(DOUBLE);
+    FROM_ARROW_CASE(STRING);
+    default:
+      return Status::NotImplemented("Arrow type reading not implemented");
+  }
+  return Status::OK();
+}
+
+} // namespace pyarrow
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index a4f4163808711..58eb3ca61cdf4 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -21,8 +21,29 @@
 #ifndef PYARROW_ADAPTERS_PANDAS_H
 #define PYARROW_ADAPTERS_PANDAS_H
 
+#include <Python.h>
+
+#include <memory>
+
+namespace arrow {
+
+class Array;
+class Column;
+
+} // namespace arrow
+
 namespace pyarrow {
 
+class Status;
+
+Status ArrowToPandas(const std::shared_ptr<arrow::Column>& col, PyObject** out);
+
+Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
+    std::shared_ptr<arrow::Array>* out);
+
+Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
+    std::shared_ptr<arrow::Array>* out);
+
 } // namespace pyarrow
 
 #endif // PYARROW_ADAPTERS_PANDAS_H
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index db6361384c10d..cc9ad9ec5bbea 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -18,7 +18,9 @@
 #ifndef PYARROW_COMMON_H
 #define PYARROW_COMMON_H
 
-#include <Python.h>
+#include "pyarrow/config.h"
+
+#include "arrow/util/buffer.h"
 
 namespace arrow { class MemoryPool; }
 
@@ -90,6 +92,25 @@ struct PyObjectStringify {
 
 arrow::MemoryPool* GetMemoryPool();
 
+class NumPyBuffer : public arrow::Buffer {
+ public:
+  NumPyBuffer(PyArrayObject* arr) :
+      Buffer(nullptr, 0) {
+    arr_ = arr;
+    Py_INCREF(arr);
+
+    data_ = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
+    size_ = PyArray_SIZE(arr_);
+  }
+
+  virtual ~NumPyBuffer() {
+    Py_XDECREF(arr_);
+  }
+
+ private:
+  PyArrayObject* arr_;
+};
+
 } // namespace pyarrow
 
 #endif // PYARROW_COMMON_H
diff --git a/python/src/pyarrow/init.cc b/python/src/pyarrow/config.cc
similarity index 84%
rename from python/src/pyarrow/init.cc
rename to python/src/pyarrow/config.cc
index acd851e168743..730d2db99a530 100644
--- a/python/src/pyarrow/init.cc
+++ b/python/src/pyarrow/config.cc
@@ -15,11 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "pyarrow/init.h"
+#include <Python.h>
+
+#include "pyarrow/config.h"
 
 namespace pyarrow {
 
 void pyarrow_init() {
 }
 
+PyObject* numpy_nan = nullptr;
+
+void pyarrow_set_numpy_nan(PyObject* obj) {
+  Py_INCREF(obj);
+  numpy_nan = obj;
+}
+
 } // namespace pyarrow
diff --git a/python/src/pyarrow/config.h b/python/src/pyarrow/config.h
new file mode 100644
index 0000000000000..48ae715d842b1
--- /dev/null
+++ b/python/src/pyarrow/config.h
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_CONFIG_H
+#define PYARROW_CONFIG_H
+
+#include <Python.h>
+
+#include "pyarrow/numpy_interop.h"
+
+#if PY_MAJOR_VERSION >= 3
+  #define PyString_Check PyUnicode_Check
+#endif
+
+namespace pyarrow {
+
+extern PyObject* numpy_nan;
+
+void pyarrow_init();
+
+void pyarrow_set_numpy_nan(PyObject* obj);
+
+} // namespace pyarrow
+
+#endif // PYARROW_CONFIG_H
diff --git a/python/src/pyarrow/init.h b/python/src/pyarrow/do_import_numpy.h
similarity index 83%
rename from python/src/pyarrow/init.h
rename to python/src/pyarrow/do_import_numpy.h
index 71e67a20c1ca5..bb4a382959102 100644
--- a/python/src/pyarrow/init.h
+++ b/python/src/pyarrow/do_import_numpy.h
@@ -15,13 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef PYARROW_INIT_H
-#define PYARROW_INIT_H
+// Trick borrowed from dynd-python for initializing the NumPy array API
 
-namespace pyarrow {
-
-void pyarrow_init();
-
-} // namespace pyarrow
-
-#endif // PYARROW_INIT_H
+// Trigger the array import (inversion of NO_IMPORT_ARRAY)
+#define NUMPY_IMPORT_ARRAY
diff --git a/python/src/pyarrow/numpy_interop.h b/python/src/pyarrow/numpy_interop.h
new file mode 100644
index 0000000000000..882d287c7c559
--- /dev/null
+++ b/python/src/pyarrow/numpy_interop.h
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_NUMPY_INTEROP_H
+#define PYARROW_NUMPY_INTEROP_H
+
+#include <Python.h>
+
+#include <numpy/numpyconfig.h>
+
+// Don't use the deprecated Numpy functions
+#ifdef NPY_1_7_API_VERSION
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+#else
+#define NPY_ARRAY_NOTSWAPPED NPY_NOTSWAPPED
+#define NPY_ARRAY_ALIGNED NPY_ALIGNED
+#define NPY_ARRAY_WRITEABLE NPY_WRITEABLE
+#define NPY_ARRAY_UPDATEIFCOPY NPY_UPDATEIFCOPY
+#endif
+
+// This is required to be able to access the NumPy C API properly in C++ files
+// other than this main one
+#define PY_ARRAY_UNIQUE_SYMBOL pyarrow_ARRAY_API
+#ifndef NUMPY_IMPORT_ARRAY
+#define NO_IMPORT_ARRAY
+#endif
+
+#include <numpy/arrayobject.h>
+#include <numpy/ufuncobject.h>
+
+namespace pyarrow {
+
+inline int import_numpy() {
+#ifdef NUMPY_IMPORT_ARRAY
+  import_array1(-1);
+  import_umath1(-1);
+#endif
+
+  return 0;
+}
+
+} // namespace pyarrow
+
+#endif // PYARROW_NUMPY_INTEROP_H

From ecadd0bcb9f022a5067826ed564f513ffd0c578e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 28 Mar 2016 09:38:13 -0700
Subject: [PATCH 0048/1644] ARROW-80: Handle len call for pre-init arrays

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #45 from xhochy/arrow-80 and squashes the following commits:

d9a1160 [Uwe L. Korn] Add unit test for repr on pre-init Array
6208d7d [Uwe L. Korn] ARROW-80: Handle len call for pre-init arrays
---
 python/pyarrow/array.pyx           | 5 ++++-
 python/pyarrow/tests/test_array.py | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 88770cdaa966e..155c965f3e8aa 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -67,7 +67,10 @@ cdef class Array:
         return '{0}\n{1}'.format(type_format, values)
 
     def __len__(self):
-        return self.sp_array.get().length()
+        if self.sp_array.get():
+            return self.sp_array.get().length()
+        else:
+            return 0
 
     def isnull(self):
         raise NotImplemented
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 36aaaa4f93d5d..d608f8167df65 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -22,6 +22,10 @@
 
 class TestArrayAPI(unittest.TestCase):
 
+    def test_repr_on_pre_init_array(self):
+        arr = pyarrow.array.Array()
+        assert len(repr(arr)) > 0
+
     def test_getitem_NA(self):
         arr = pyarrow.from_pylist([1, None, 2])
         assert arr[1] is pyarrow.NA

From 80ec2c17fccac484993868f951d95362cb75cea9 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 28 Mar 2016 09:39:55 -0700
Subject: [PATCH 0049/1644] ARROW-79: [Python] Add benchmarks

Run them using `asv run --python=same` or `asv dev`.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #44 from xhochy/arrow-79 and squashes the following commits:

d3c6401 [Uwe L. Korn] Move benchmarks to toplevel folder
2737f18 [Uwe L. Korn] ARROW-79: [Python] Add benchmarks
---
 python/.gitignore             |  3 ++
 python/asv.conf.json          | 73 +++++++++++++++++++++++++++++++++++
 python/benchmarks/__init__.py | 17 ++++++++
 python/benchmarks/array.py    | 38 ++++++++++++++++++
 python/doc/Benchmarks.md      | 11 ++++++
 5 files changed, 142 insertions(+)
 create mode 100644 python/asv.conf.json
 create mode 100644 python/benchmarks/__init__.py
 create mode 100644 python/benchmarks/array.py
 create mode 100644 python/doc/Benchmarks.md

diff --git a/python/.gitignore b/python/.gitignore
index 80103a1a52942..3cb591ea766d5 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -35,3 +35,6 @@ dist
 # coverage
 .coverage
 coverage.xml
+
+# benchmark working dir
+.asv
diff --git a/python/asv.conf.json b/python/asv.conf.json
new file mode 100644
index 0000000000000..96beba64c2e6e
--- /dev/null
+++ b/python/asv.conf.json
@@ -0,0 +1,73 @@
+{
+    // The version of the config file format.  Do not change, unless
+    // you know what you are doing.
+    "version": 1,
+
+    // The name of the project being benchmarked
+    "project": "pyarrow",
+
+    // The project's homepage
+    "project_url": "https://arrow.apache.org/",
+
+    // The URL or local path of the source code repository for the
+    // project being benchmarked
+    "repo": "https://github.com/apache/arrow/",
+
+    // List of branches to benchmark. If not provided, defaults to "master"
+    // (for git) or "tip" (for mercurial).
+    // "branches": ["master"], // for git
+    // "branches": ["tip"],    // for mercurial
+
+    // The DVCS being used.  If not set, it will be automatically
+    // determined from "repo" by looking at the protocol in the URL
+    // (if remote), or by looking for special directories, such as
+    // ".git" (if local).
+    "dvcs": "git",
+
+    // The tool to use to create environments.  May be "conda",
+    // "virtualenv" or other value depending on the plugins in use.
+    // If missing or the empty string, the tool will be automatically
+    // determined by looking for tools on the PATH environment
+    // variable.
+    "environment_type": "virtualenv",
+
+    // the base URL to show a commit for the project.
+    "show_commit_url": "https://github.com/apache/arrow/commit/",
+
+    // The Pythons you'd like to test against.  If not provided, defaults
+    // to the current version of Python used to run `asv`.
+    // "pythons": ["2.7", "3.3"],
+
+    // The matrix of dependencies to test.  Each key is the name of a
+    // package (in PyPI) and the values are version numbers.  An empty
+    // list indicates to just test against the default (latest)
+    // version.
+    // "matrix": {
+    //     "numpy": ["1.6", "1.7"]
+    // },
+
+    // The directory (relative to the current directory) that benchmarks are
+    // stored in.  If not provided, defaults to "benchmarks"
+    "benchmark_dir": "benchmarks",
+
+    // The directory (relative to the current directory) to cache the Python
+    // environments in.  If not provided, defaults to "env"
+    "env_dir": ".asv/env",
+
+
+    // The directory (relative to the current directory) that raw benchmark
+    // results are stored in.  If not provided, defaults to "results".
+    "results_dir": ".asv/results",
+
+    // The directory (relative to the current directory) that the html tree
+    // should be written to.  If not provided, defaults to "html".
+    "html_dir": "build/benchmarks/html",
+
+    // The number of characters to retain in the commit hashes.
+    // "hash_length": 8,
+
+    // `asv` will cache wheels of the recent builds in each
+    // environment, making them faster to install next time.  This is
+    // number of builds to keep, per environment.
+    // "wheel_cache_size": 0
+}
diff --git a/python/benchmarks/__init__.py b/python/benchmarks/__init__.py
new file mode 100644
index 0000000000000..245692337bc3f
--- /dev/null
+++ b/python/benchmarks/__init__.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
diff --git a/python/benchmarks/array.py b/python/benchmarks/array.py
new file mode 100644
index 0000000000000..6ab73d18d1f87
--- /dev/null
+++ b/python/benchmarks/array.py
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pyarrow
+
+class Conversions(object):
+    params = (1, 10 ** 5, 10 ** 6, 10 ** 7)
+
+    def time_from_pylist(self, n):
+        pyarrow.from_pylist(list(range(n)))
+
+    def peakmem_from_pylist(self, n):
+        pyarrow.from_pylist(list(range(n)))
+
+class ScalarAccess(object):
+    params = (1, 10 ** 5, 10 ** 6, 10 ** 7)
+
+    def setUp(self, n):
+        self._array = pyarrow.from_pylist(list(range(n)))
+
+    def time_as_py(self, n):
+        for i in range(n):
+            self._array[i].as_py()
+
diff --git a/python/doc/Benchmarks.md b/python/doc/Benchmarks.md
new file mode 100644
index 0000000000000..8edfb6209e4af
--- /dev/null
+++ b/python/doc/Benchmarks.md
@@ -0,0 +1,11 @@
+## Benchmark Requirements
+
+The benchmarks are run using [asv][1] which is also their only requirement.
+
+## Running the benchmarks
+
+To run the benchmarks, call `asv run --python=same`. You cannot use the
+plain `asv run` command at the moment as asv cannot handle python packages
+in subdirectories of a repository.
+
+[1]: https://asv.readthedocs.org/

From df7726d44ab59828aacc20a1786287ba7ade2562 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Mon, 28 Mar 2016 10:39:25 -0700
Subject: [PATCH 0050/1644] ARROW-88: [C++] Refactor usages of parquet_cpp
 namespace

I also removed an unneeded `Py_XDECREF` from ARROW-30; didn't want to create a separate patch for that.

Author: Wes McKinney <wesm@apache.org>

Closes #49 from wesm/ARROW-88 and squashes the following commits:

c4d81dc [Wes McKinney] Refactor usages of parquet_cpp namespace
---
 cpp/src/arrow/parquet/parquet-schema-test.cc | 40 ++++++++++----------
 cpp/src/arrow/parquet/schema.cc              | 29 +++++++-------
 cpp/src/arrow/parquet/schema.h               |  4 +-
 python/pyarrow/array.pyx                     |  3 --
 python/pyarrow/includes/parquet.pxd          |  2 +-
 5 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
index 9c3093d9ff7c9..02a8caf03c9bd 100644
--- a/cpp/src/arrow/parquet/parquet-schema-test.cc
+++ b/cpp/src/arrow/parquet/parquet-schema-test.cc
@@ -26,15 +26,17 @@
 
 #include "arrow/parquet/schema.h"
 
+using ParquetType = parquet::Type;
+using parquet::LogicalType;
+using parquet::Repetition;
+using parquet::schema::NodePtr;
+using parquet::schema::GroupNode;
+using parquet::schema::PrimitiveNode;
+
 namespace arrow {
 
 namespace parquet {
 
-using parquet_cpp::Repetition;
-using parquet_cpp::schema::NodePtr;
-using parquet_cpp::schema::GroupNode;
-using parquet_cpp::schema::PrimitiveNode;
-
 const auto BOOL = std::make_shared<BooleanType>();
 const auto UINT8 = std::make_shared<UInt8Type>();
 const auto INT32 = std::make_shared<Int32Type>();
@@ -66,7 +68,7 @@ class TestConvertParquetSchema : public ::testing::Test {
   }
 
  protected:
-  parquet_cpp::SchemaDescriptor descr_;
+  ::parquet::SchemaDescriptor descr_;
   std::shared_ptr<Schema> result_schema_;
 };
 
@@ -75,40 +77,40 @@ TEST_F(TestConvertParquetSchema, ParquetFlatPrimitives) {
   std::vector<std::shared_ptr<Field>> arrow_fields;
 
   parquet_fields.push_back(
-      PrimitiveNode::Make("boolean", Repetition::REQUIRED, parquet_cpp::Type::BOOLEAN));
+      PrimitiveNode::Make("boolean", Repetition::REQUIRED, ParquetType::BOOLEAN));
   arrow_fields.push_back(std::make_shared<Field>("boolean", BOOL, false));
 
   parquet_fields.push_back(
-      PrimitiveNode::Make("int32", Repetition::REQUIRED, parquet_cpp::Type::INT32));
+      PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
   arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
 
   parquet_fields.push_back(
-      PrimitiveNode::Make("int64", Repetition::REQUIRED, parquet_cpp::Type::INT64));
+      PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
   arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
 
   parquet_fields.push_back(
-      PrimitiveNode::Make("float", Repetition::OPTIONAL, parquet_cpp::Type::FLOAT));
+      PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
   arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
 
   parquet_fields.push_back(
-      PrimitiveNode::Make("double", Repetition::OPTIONAL, parquet_cpp::Type::DOUBLE));
+      PrimitiveNode::Make("double", Repetition::OPTIONAL, ParquetType::DOUBLE));
   arrow_fields.push_back(std::make_shared<Field>("double", DOUBLE));
 
   parquet_fields.push_back(
       PrimitiveNode::Make("binary", Repetition::OPTIONAL,
-          parquet_cpp::Type::BYTE_ARRAY));
+          ParquetType::BYTE_ARRAY));
   arrow_fields.push_back(std::make_shared<Field>("binary", BINARY));
 
   parquet_fields.push_back(
       PrimitiveNode::Make("string", Repetition::OPTIONAL,
-          parquet_cpp::Type::BYTE_ARRAY,
-          parquet_cpp::LogicalType::UTF8));
+          ParquetType::BYTE_ARRAY,
+          LogicalType::UTF8));
   arrow_fields.push_back(std::make_shared<Field>("string", UTF8));
 
   parquet_fields.push_back(
       PrimitiveNode::Make("flba-binary", Repetition::OPTIONAL,
-          parquet_cpp::Type::FIXED_LEN_BYTE_ARRAY,
-          parquet_cpp::LogicalType::NONE, 12));
+          ParquetType::FIXED_LEN_BYTE_ARRAY,
+          LogicalType::NONE, 12));
   arrow_fields.push_back(std::make_shared<Field>("flba-binary", BINARY));
 
   auto arrow_schema = std::make_shared<Schema>(arrow_fields);
@@ -121,18 +123,18 @@ TEST_F(TestConvertParquetSchema, UnsupportedThings) {
   std::vector<NodePtr> unsupported_nodes;
 
   unsupported_nodes.push_back(
-      PrimitiveNode::Make("int96", Repetition::REQUIRED, parquet_cpp::Type::INT96));
+      PrimitiveNode::Make("int96", Repetition::REQUIRED, ParquetType::INT96));
 
   unsupported_nodes.push_back(
       GroupNode::Make("repeated-group", Repetition::REPEATED, {}));
 
   unsupported_nodes.push_back(
       PrimitiveNode::Make("int32", Repetition::OPTIONAL,
-          parquet_cpp::Type::INT32, parquet_cpp::LogicalType::DATE));
+          ParquetType::INT32, LogicalType::DATE));
 
   unsupported_nodes.push_back(
       PrimitiveNode::Make("int64", Repetition::OPTIONAL,
-          parquet_cpp::Type::INT64, parquet_cpp::LogicalType::TIMESTAMP_MILLIS));
+          ParquetType::INT64, LogicalType::TIMESTAMP_MILLIS));
 
   for (const NodePtr& node : unsupported_nodes) {
     ASSERT_RAISES(NotImplemented, ConvertSchema({node}));
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index 6b1de572617b8..d8eb2addb0ada 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -24,12 +24,13 @@
 #include "arrow/util/status.h"
 #include "arrow/types/decimal.h"
 
-using parquet_cpp::schema::Node;
-using parquet_cpp::schema::NodePtr;
-using parquet_cpp::schema::GroupNode;
-using parquet_cpp::schema::PrimitiveNode;
+using parquet::schema::Node;
+using parquet::schema::NodePtr;
+using parquet::schema::GroupNode;
+using parquet::schema::PrimitiveNode;
 
-using parquet_cpp::LogicalType;
+using ParquetType = parquet::Type;
+using parquet::LogicalType;
 
 namespace arrow {
 
@@ -124,30 +125,30 @@ Status NodeToField(const NodePtr& node, std::shared_ptr<Field>* out) {
     const PrimitiveNode* primitive = static_cast<const PrimitiveNode*>(node.get());
 
     switch (primitive->physical_type()) {
-      case parquet_cpp::Type::BOOLEAN:
+      case ParquetType::BOOLEAN:
         type = BOOL;
         break;
-      case parquet_cpp::Type::INT32:
+      case ParquetType::INT32:
         RETURN_NOT_OK(FromInt32(primitive, &type));
         break;
-      case parquet_cpp::Type::INT64:
+      case ParquetType::INT64:
         RETURN_NOT_OK(FromInt64(primitive, &type));
         break;
-      case parquet_cpp::Type::INT96:
+      case ParquetType::INT96:
         // TODO: Do we have that type in Arrow?
         // type = TypePtr(new Int96Type());
         return Status::NotImplemented("int96");
-      case parquet_cpp::Type::FLOAT:
+      case ParquetType::FLOAT:
         type = FLOAT;
         break;
-      case parquet_cpp::Type::DOUBLE:
+      case ParquetType::DOUBLE:
         type = DOUBLE;
         break;
-      case parquet_cpp::Type::BYTE_ARRAY:
+      case ParquetType::BYTE_ARRAY:
         // TODO: Do we have that type in Arrow?
         RETURN_NOT_OK(FromByteArray(primitive, &type));
         break;
-      case parquet_cpp::Type::FIXED_LEN_BYTE_ARRAY:
+      case ParquetType::FIXED_LEN_BYTE_ARRAY:
         RETURN_NOT_OK(FromFLBA(primitive, &type));
         break;
     }
@@ -157,7 +158,7 @@ Status NodeToField(const NodePtr& node, std::shared_ptr<Field>* out) {
   return Status::OK();
 }
 
-Status FromParquetSchema(const parquet_cpp::SchemaDescriptor* parquet_schema,
+Status FromParquetSchema(const ::parquet::SchemaDescriptor* parquet_schema,
     std::shared_ptr<Schema>* out) {
   // TODO(wesm): Consider adding an arrow::Schema name attribute, which comes
   // from the root Parquet node
diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h
index 61de193a33877..a8408970ede48 100644
--- a/cpp/src/arrow/parquet/schema.h
+++ b/cpp/src/arrow/parquet/schema.h
@@ -31,10 +31,10 @@ class Status;
 
 namespace parquet {
 
-Status NodeToField(const parquet_cpp::schema::NodePtr& node,
+Status NodeToField(const ::parquet::schema::NodePtr& node,
     std::shared_ptr<Field>* out);
 
-Status FromParquetSchema(const parquet_cpp::SchemaDescriptor* parquet_schema,
+Status FromParquetSchema(const ::parquet::SchemaDescriptor* parquet_schema,
     std::shared_ptr<Schema>* out);
 
 } // namespace parquet
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 155c965f3e8aa..255efc268fe29 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -359,7 +359,4 @@ cdef class Table:
             names.append(frombytes(col.get().name()))
             data.append(<object> arr)
 
-            # One ref count too many
-            Py_XDECREF(arr)
-
         return pd.DataFrame(dict(zip(names, data)), columns=names)
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index 99a2d423d9cba..ffdc5d487068d 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -19,7 +19,7 @@
 
 from pyarrow.includes.common cimport *
 
-cdef extern from "parquet/api/reader.h" namespace "parquet_cpp" nogil:
+cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
     cdef cppclass ColumnReader:
         pass
 

From 38897ee29f85765f7646e90237fa85f98ccb55f5 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 28 Mar 2016 10:42:14 -0700
Subject: [PATCH 0051/1644] ARROW-83: [C++] Add basic test infrastructure for
 DecimalType

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #47 from xhochy/arrow-83 and squashes the following commits:

6eabd7a [Uwe L. Korn] Remove unused forward decl
e1854e9 [Uwe L. Korn] ARROW-83: [C++] Add basic test infrastructure for DecimalType
---
 cpp/src/arrow/types/CMakeLists.txt  |  1 +
 cpp/src/arrow/types/decimal-test.cc | 40 +++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 cpp/src/arrow/types/decimal-test.cc

diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
index f3e41289bfe8d..72a8e77664610 100644
--- a/cpp/src/arrow/types/CMakeLists.txt
+++ b/cpp/src/arrow/types/CMakeLists.txt
@@ -34,6 +34,7 @@ install(FILES
   DESTINATION include/arrow/types)
 
 
+ADD_ARROW_TEST(decimal-test)
 ADD_ARROW_TEST(list-test)
 ADD_ARROW_TEST(primitive-test)
 ADD_ARROW_TEST(string-test)
diff --git a/cpp/src/arrow/types/decimal-test.cc b/cpp/src/arrow/types/decimal-test.cc
new file mode 100644
index 0000000000000..89896c8b425d0
--- /dev/null
+++ b/cpp/src/arrow/types/decimal-test.cc
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/types/decimal.h"
+
+namespace arrow {
+
+TEST(TypesTest, TestDecimalType) {
+  DecimalType t1(8, 4);
+
+  ASSERT_EQ(t1.type, Type::DECIMAL);
+  ASSERT_EQ(t1.precision, 8);
+  ASSERT_EQ(t1.scale, 4);
+
+  ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
+
+  // Test copy constructor
+  DecimalType t2 = t1;
+  ASSERT_EQ(t2.type, Type::DECIMAL);
+  ASSERT_EQ(t2.precision, 8);
+  ASSERT_EQ(t2.scale, 4);
+}
+
+} // namespace arrow

From 2d8627cd81f83783b0ceb01d137a46b581ecba26 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 28 Mar 2016 10:49:08 -0700
Subject: [PATCH 0052/1644] ARROW-87: [C++] Add all four possible ways to
 encode Decimals in Parquet to schema conversion

See also: https://github.com/Parquet/parquet-format/blob/master/LogicalTypes.md#decimal

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #48 from xhochy/arrow-87 and squashes the following commits:

05ca3be [Uwe L. Korn] Use parquet:: namespace instead of parquet_cpp
6bafc5f [Uwe L. Korn] ARROW-87: [C++] Add all four possible ways to encode Decimals in Parquet to schema conversion
---
 cpp/src/arrow/parquet/parquet-schema-test.cc | 36 ++++++++++++++++++++
 cpp/src/arrow/parquet/schema.cc              |  9 +++++
 2 files changed, 45 insertions(+)

diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
index 02a8caf03c9bd..a289ddbfde6eb 100644
--- a/cpp/src/arrow/parquet/parquet-schema-test.cc
+++ b/cpp/src/arrow/parquet/parquet-schema-test.cc
@@ -22,6 +22,7 @@
 
 #include "arrow/test-util.h"
 #include "arrow/type.h"
+#include "arrow/types/decimal.h"
 #include "arrow/util/status.h"
 
 #include "arrow/parquet/schema.h"
@@ -46,6 +47,7 @@ const auto DOUBLE = std::make_shared<DoubleType>();
 const auto UTF8 = std::make_shared<StringType>();
 const auto BINARY = std::make_shared<ListType>(
     std::make_shared<Field>("", UINT8));
+const auto DECIMAL_8_4 = std::make_shared<DecimalType>(8, 4);
 
 class TestConvertParquetSchema : public ::testing::Test {
  public:
@@ -119,6 +121,40 @@ TEST_F(TestConvertParquetSchema, ParquetFlatPrimitives) {
   CheckFlatSchema(arrow_schema);
 }
 
+TEST_F(TestConvertParquetSchema, ParquetFlatDecimals) {
+  std::vector<NodePtr> parquet_fields;
+  std::vector<std::shared_ptr<Field>> arrow_fields;
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("flba-decimal", Repetition::OPTIONAL,
+          ParquetType::FIXED_LEN_BYTE_ARRAY,
+          LogicalType::DECIMAL, 4, 8, 4));
+  arrow_fields.push_back(std::make_shared<Field>("flba-decimal", DECIMAL_8_4));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("binary-decimal", Repetition::OPTIONAL,
+          ParquetType::BYTE_ARRAY,
+          LogicalType::DECIMAL, -1, 8, 4));
+  arrow_fields.push_back(std::make_shared<Field>("binary-decimal", DECIMAL_8_4));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("int32-decimal", Repetition::OPTIONAL,
+          ParquetType::INT32,
+          LogicalType::DECIMAL, -1, 8, 4));
+  arrow_fields.push_back(std::make_shared<Field>("int32-decimal", DECIMAL_8_4));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("int64-decimal", Repetition::OPTIONAL,
+          ParquetType::INT64,
+          LogicalType::DECIMAL, -1, 8, 4));
+  arrow_fields.push_back(std::make_shared<Field>("int64-decimal", DECIMAL_8_4));
+
+  auto arrow_schema = std::make_shared<Schema>(arrow_fields);
+  ASSERT_OK(ConvertSchema(parquet_fields));
+
+  CheckFlatSchema(arrow_schema);
+}
+
 TEST_F(TestConvertParquetSchema, UnsupportedThings) {
   std::vector<NodePtr> unsupported_nodes;
 
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index d8eb2addb0ada..14f4f5be53ce9 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -57,6 +57,9 @@ static Status FromByteArray(const PrimitiveNode* node, TypePtr* out) {
     case LogicalType::UTF8:
       *out = UTF8;
       break;
+    case LogicalType::DECIMAL:
+      *out = MakeDecimalType(node);
+      break;
     default:
       // BINARY
       *out = BINARY;
@@ -86,6 +89,9 @@ static Status FromInt32(const PrimitiveNode* node, TypePtr* out) {
     case LogicalType::NONE:
       *out = INT32;
       break;
+    case LogicalType::DECIMAL:
+      *out = MakeDecimalType(node);
+      break;
     default:
       return Status::NotImplemented("Unhandled logical type for int32");
       break;
@@ -98,6 +104,9 @@ static Status FromInt64(const PrimitiveNode* node, TypePtr* out) {
     case LogicalType::NONE:
       *out = INT64;
       break;
+    case LogicalType::DECIMAL:
+      *out = MakeDecimalType(node);
+      break;
     default:
       return Status::NotImplemented("Unhandled logical type for int64");
       break;

From 5a68f8d737aa94ff3d09dae4e5b29883e798e9c4 Mon Sep 17 00:00:00 2001
From: Dan Robinson <danrobinson010@gmail.com>
Date: Thu, 31 Mar 2016 10:02:54 -0700
Subject: [PATCH 0053/1644] ARROW-93: Fix builds when using XCode 7.3

Author: Dan Robinson <danrobinson010@gmail.com>

Closes #54 from danrobinson/ARROW-93 and squashes the following commits:

ddff5b0 [Dan Robinson] ARROW-93: Fix builds when using XCode 7.3
---
 cpp/cmake_modules/CompilerInfo.cmake    | 2 +-
 python/cmake_modules/CompilerInfo.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index 07860682f9b1b..e1c821cca5d45 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -31,7 +31,7 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
 
 # clang on Mac OS X, XCode 7. No version replacement is done
 # because Apple no longer advertises the upstream LLVM version.
-elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-700\\..*")
+elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-70[0-9]\\..*")
   set(COMPILER_FAMILY "clang")
 
 # gcc
diff --git a/python/cmake_modules/CompilerInfo.cmake b/python/cmake_modules/CompilerInfo.cmake
index e66bc2693eead..55f989a1a6c9d 100644
--- a/python/cmake_modules/CompilerInfo.cmake
+++ b/python/cmake_modules/CompilerInfo.cmake
@@ -34,7 +34,7 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
 
 # clang on Mac OS X, XCode 7. No version replacement is done
 # because Apple no longer advertises the upstream LLVM version.
-elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-700\\..*")
+elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-70[0-9]\\..*")
   set(COMPILER_FAMILY "clang")
 
 # gcc

From b3ebce1b3471abbdc4516ff86014aa26bcc99a24 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 31 Mar 2016 17:27:56 -0700
Subject: [PATCH 0054/1644] ARROW-89: [Python] Add benchmarks for
 Arrow<->Pandas conversion

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #51 from xhochy/arrow-89 and squashes the following commits:

bd6a7cb [Uwe L. Korn] Split benchmarks and add one for a float64 column with NaNs
8f74528 [Uwe L. Korn] ARROW-89: [Python] Add benchmarks for Arrow<->Pandas conversion
---
 python/benchmarks/array.py | 55 ++++++++++++++++++++++++++++++++++----
 1 file changed, 50 insertions(+), 5 deletions(-)

diff --git a/python/benchmarks/array.py b/python/benchmarks/array.py
index 6ab73d18d1f87..4268f0073f292 100644
--- a/python/benchmarks/array.py
+++ b/python/benchmarks/array.py
@@ -15,22 +15,67 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import pyarrow
+import numpy as np
+import pandas as pd
+import pyarrow as A
 
-class Conversions(object):
+
+class PyListConversions(object):
+    param_names = ('size',)
     params = (1, 10 ** 5, 10 ** 6, 10 ** 7)
 
+    def setup(self, n):
+        self.data = list(range(n))
+
     def time_from_pylist(self, n):
-        pyarrow.from_pylist(list(range(n)))
+        A.from_pylist(self.data)
 
     def peakmem_from_pylist(self, n):
-        pyarrow.from_pylist(list(range(n)))
+        A.from_pylist(self.data)
+
+
+class PandasConversionsBase(object):
+    def setup(self, n, dtype):
+        if dtype == 'float64_nans':
+            arr = np.arange(n).astype('float64')
+            arr[arr % 10 == 0] = np.nan
+        else:
+            arr = np.arange(n).astype(dtype)
+        self.data = pd.DataFrame({'column': arr})
+
+
+class PandasConversionsToArrow(PandasConversionsBase):
+    param_names = ('size', 'dtype')
+    params = ((1, 10 ** 5, 10 ** 6, 10 ** 7), ('int64', 'float64', 'float64_nans', 'str'))
+
+    def time_from_series(self, n, dtype):
+        A.from_pandas_dataframe(self.data)
+
+    def peakmem_from_series(self, n, dtype):
+        A.from_pandas_dataframe(self.data)
+
+
+class PandasConversionsFromArrow(PandasConversionsBase):
+    param_names = ('size', 'dtype')
+    params = ((1, 10 ** 5, 10 ** 6, 10 ** 7), ('int64', 'float64', 'float64_nans', 'str'))
+
+    def setup(self, n, dtype):
+        super(PandasConversionsFromArrow, self).setup(n, dtype)
+        self.arrow_data = A.from_pandas_dataframe(self.data)
+
+    def time_to_series(self, n, dtype):
+        self.arrow_data.to_pandas()
+
+    def peakmem_to_series(self, n, dtype):
+        self.arrow_data.to_pandas()
+
 
 class ScalarAccess(object):
+    param_names = ('size',)
     params = (1, 10 ** 5, 10 ** 6, 10 ** 7)
 
     def setUp(self, n):
-        self._array = pyarrow.from_pylist(list(range(n)))
+        self._array = A.from_pylist(list(range(n)))
 
     def time_as_py(self, n):
         for i in range(n):

From 6d31d5928f4ec5ced14a105b5b05d46a7dab5264 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 31 Mar 2016 17:47:42 -0700
Subject: [PATCH 0055/1644] ARROW-49: [Python] Add Column and Table wrapper
 interface

After https://github.com/apache/arrow/pull/52 is merged, I'd like to split Column and Table into separate .pyx files, array.pyx seems a bit overcrowded.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #53 from xhochy/arrow-49 and squashes the following commits:

b01b201 [Uwe L. Korn] Use correct number of chunks
e422faf [Uwe L. Korn] Incoportate PR feedback, Add ChunkedArray interface
e8f84a9 [Uwe L. Korn] ARROW-49: [Python] Add Column and Table wrapper interface
---
 python/CMakeLists.txt                |   1 +
 python/pyarrow/__init__.py           |   4 +-
 python/pyarrow/array.pxd             |   2 +
 python/pyarrow/array.pyx             |  75 +-------
 python/pyarrow/includes/libarrow.pxd |   5 +-
 python/pyarrow/schema.pxd            |   2 +
 python/pyarrow/schema.pyx            |   9 +
 python/pyarrow/table.pxd             |  46 +++++
 python/pyarrow/table.pyx             | 264 +++++++++++++++++++++++++++
 python/pyarrow/tests/test_column.py  |  49 +++++
 python/pyarrow/tests/test_table.py   |  39 ++++
 python/setup.py                      |   2 +-
 12 files changed, 422 insertions(+), 76 deletions(-)
 create mode 100644 python/pyarrow/table.pxd
 create mode 100644 python/pyarrow/table.pyx
 create mode 100644 python/pyarrow/tests/test_column.py

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index ebe825f65c4da..2173232d4eff5 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -444,6 +444,7 @@ set(CYTHON_EXTENSIONS
   error
   scalar
   schema
+  table
 )
 
 foreach(module ${CYTHON_EXTENSIONS})
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index c343f5ba5f129..40a09c2feaef0 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -41,4 +41,6 @@
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
-from pyarrow.array import RowBatch, Table, from_pandas_dataframe
+from pyarrow.array import RowBatch, from_pandas_dataframe
+
+from pyarrow.table import Column, Table
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index de3c77419623f..8cd15cd450219 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -36,6 +36,8 @@ cdef class Array:
     cdef init(self, const shared_ptr[CArray]& sp_array)
     cdef getitem(self, int i)
 
+cdef object box_arrow_array(const shared_ptr[CArray]& sp_array)
+
 
 cdef class BooleanArray(Array):
     pass
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 255efc268fe29..456bf6d1da848 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -33,6 +33,8 @@ from pyarrow.scalar import NA
 from pyarrow.schema cimport Schema
 import pyarrow.schema as schema
 
+from pyarrow.table cimport Table
+
 def total_allocated_bytes():
     cdef MemoryPool* pool = pyarrow.GetMemoryPool()
     return pool.bytes_allocated()
@@ -287,76 +289,3 @@ cdef class RowBatch:
         return self.arrays[i]
 
 
-cdef class Table:
-    '''
-    Do not call this class's constructor directly.
-    '''
-    cdef:
-        shared_ptr[CTable] sp_table
-        CTable* table
-
-    def __cinit__(self):
-        pass
-
-    cdef init(self, const shared_ptr[CTable]& table):
-        self.sp_table = table
-        self.table = table.get()
-
-    @staticmethod
-    def from_pandas(df, name=None):
-        pass
-
-    @staticmethod
-    def from_arrays(names, arrays, name=None):
-        cdef:
-            Array arr
-            Table result
-            c_string c_name
-            vector[shared_ptr[CField]] fields
-            vector[shared_ptr[CColumn]] columns
-            shared_ptr[CSchema] schema
-            shared_ptr[CTable] table
-
-        cdef int K = len(arrays)
-
-        fields.resize(K)
-        columns.resize(K)
-        for i in range(K):
-            arr = arrays[i]
-            c_name = tobytes(names[i])
-
-            fields[i].reset(new CField(c_name, arr.type.sp_type, True))
-            columns[i].reset(new CColumn(fields[i], arr.sp_array))
-
-        if name is None:
-            c_name = ''
-        else:
-            c_name = tobytes(name)
-
-        schema.reset(new CSchema(fields))
-        table.reset(new CTable(c_name, schema, columns))
-
-        result = Table()
-        result.init(table)
-
-        return result
-
-    def to_pandas(self):
-        """
-        Convert the arrow::Table to a pandas DataFrame
-        """
-        cdef:
-            PyObject* arr
-            shared_ptr[CColumn] col
-
-        import pandas as pd
-
-        names = []
-        data = []
-        for i in range(self.table.num_columns()):
-            col = self.table.column(i)
-            check_status(pyarrow.ArrowToPandas(col, &arr))
-            names.append(frombytes(col.get().name()))
-            data.append(<object> arr)
-
-        return pd.DataFrame(dict(zip(names, data)), columns=names)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 42f1f25073d1b..b2ef45a347bc0 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -149,7 +149,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_string GetString(int i)
 
     cdef cppclass CChunkedArray" arrow::ChunkedArray":
-        pass
+        int64_t length()
+        int64_t null_count()
+        int num_chunks()
+        const shared_ptr[CArray]& chunk(int i)
 
     cdef cppclass CColumn" arrow::Column":
         CColumn(const shared_ptr[CField]& field,
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
index 61458b765c742..f2cb776eb2e9f 100644
--- a/python/pyarrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -41,5 +41,7 @@ cdef class Schema:
         CSchema* schema
 
     cdef init(self, const vector[shared_ptr[CField]]& fields)
+    cdef init_schema(self, const shared_ptr[CSchema]& schema)
 
 cdef DataType box_data_type(const shared_ptr[CDataType]& type)
+cdef Schema box_schema(const shared_ptr[CSchema]& schema)
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index b3bf02aad76bb..22ddf0cf17e41 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -106,6 +106,10 @@ cdef class Schema:
         self.schema = new CSchema(fields)
         self.sp_schema.reset(self.schema)
 
+    cdef init_schema(self, const shared_ptr[CSchema]& schema):
+        self.schema = schema.get()
+        self.sp_schema = schema
+
     @classmethod
     def from_fields(cls, fields):
         cdef:
@@ -223,3 +227,8 @@ cdef DataType box_data_type(const shared_ptr[CDataType]& type):
     cdef DataType out = DataType()
     out.init(type)
     return out
+
+cdef Schema box_schema(const shared_ptr[CSchema]& type):
+    cdef Schema out = Schema()
+    out.init_schema(type)
+    return out
diff --git a/python/pyarrow/table.pxd b/python/pyarrow/table.pxd
new file mode 100644
index 0000000000000..0a5c122c95cff
--- /dev/null
+++ b/python/pyarrow/table.pxd
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyarrow.includes.common cimport shared_ptr
+from pyarrow.includes.libarrow cimport CChunkedArray, CColumn, CTable
+
+
+cdef class ChunkedArray:
+    cdef:
+        shared_ptr[CChunkedArray] sp_chunked_array
+        CChunkedArray* chunked_array
+
+    cdef init(self, const shared_ptr[CChunkedArray]& chunked_array)
+    cdef _check_nullptr(self)
+
+
+cdef class Column:
+    cdef:
+        shared_ptr[CColumn] sp_column
+        CColumn* column
+
+    cdef init(self, const shared_ptr[CColumn]& column)
+    cdef _check_nullptr(self)
+
+
+cdef class Table:
+    cdef:
+        shared_ptr[CTable] sp_table
+        CTable* table
+    
+    cdef init(self, const shared_ptr[CTable]& table)
+    cdef _check_nullptr(self)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
new file mode 100644
index 0000000000000..4c4816f0c7e69
--- /dev/null
+++ b/python/pyarrow/table.pyx
@@ -0,0 +1,264 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from pyarrow.includes.libarrow cimport *
+cimport pyarrow.includes.pyarrow as pyarrow
+
+import pyarrow.config
+
+from pyarrow.array cimport Array, box_arrow_array
+from pyarrow.compat import frombytes, tobytes
+from pyarrow.error cimport check_status
+from pyarrow.schema cimport box_data_type, box_schema
+
+cdef class ChunkedArray:
+    '''
+    Do not call this class's constructor directly.
+    '''
+
+    def __cinit__(self):
+        self.chunked_array = NULL
+
+    cdef init(self, const shared_ptr[CChunkedArray]& chunked_array):
+        self.sp_chunked_array = chunked_array
+        self.chunked_array = chunked_array.get()
+
+    cdef _check_nullptr(self):
+        if self.chunked_array == NULL:
+            raise ReferenceError("ChunkedArray object references a NULL pointer."
+                    "Not initialized.")
+
+    def length(self):
+        self._check_nullptr()
+        return self.chunked_array.length()
+
+    def __len__(self):
+        return self.length()
+
+    property null_count:
+
+        def __get__(self):
+            self._check_nullptr()
+            return self.chunked_array.null_count()
+
+    property num_chunks:
+
+        def __get__(self):
+            self._check_nullptr()
+            return self.chunked_array.num_chunks()
+
+    def chunk(self, i):
+        self._check_nullptr()
+        return box_arrow_array(self.chunked_array.chunk(i))
+
+
+    def iterchunks(self):
+        for i in range(self.num_chunks):
+            yield self.chunk(i)
+
+
+cdef class Column:
+    '''
+    Do not call this class's constructor directly.
+    '''
+
+    def __cinit__(self):
+        self.column = NULL
+
+    cdef init(self, const shared_ptr[CColumn]& column):
+        self.sp_column = column
+        self.column = column.get()
+
+    def to_pandas(self):
+        """
+        Convert the arrow::Column to a pandas Series
+        """
+        cdef:
+            PyObject* arr
+
+        import pandas as pd
+
+        check_status(pyarrow.ArrowToPandas(self.sp_column, &arr))
+        return pd.Series(<object>arr, name=self.name)
+
+    cdef _check_nullptr(self):
+        if self.column == NULL:
+            raise ReferenceError("Column object references a NULL pointer."
+                    "Not initialized.")
+
+    def __len__(self):
+        self._check_nullptr()
+        return self.column.length()
+
+    def length(self):
+        self._check_nullptr()
+        return self.column.length()
+
+    property shape:
+
+        def __get__(self):
+            self._check_nullptr()
+            return (self.length(),)
+
+    property null_count:
+
+        def __get__(self):
+            self._check_nullptr()
+            return self.column.null_count()
+
+    property name:
+
+        def __get__(self):
+            return frombytes(self.column.name())
+
+    property type:
+
+        def __get__(self):
+            return box_data_type(self.column.type())
+
+    property data:
+
+        def __get__(self):
+            cdef ChunkedArray chunked_array = ChunkedArray()
+            chunked_array.init(self.column.data())
+            return chunked_array
+
+
+cdef class Table:
+    '''
+    Do not call this class's constructor directly.
+    '''
+
+    def __cinit__(self):
+        self.table = NULL
+
+    cdef init(self, const shared_ptr[CTable]& table):
+        self.sp_table = table
+        self.table = table.get()
+
+    cdef _check_nullptr(self):
+        if self.table == NULL:
+            raise ReferenceError("Table object references a NULL pointer."
+                    "Not initialized.")
+
+    @staticmethod
+    def from_pandas(df, name=None):
+        pass
+
+    @staticmethod
+    def from_arrays(names, arrays, name=None):
+        cdef:
+            Array arr
+            Table result
+            c_string c_name
+            vector[shared_ptr[CField]] fields
+            vector[shared_ptr[CColumn]] columns
+            shared_ptr[CSchema] schema
+            shared_ptr[CTable] table
+
+        cdef int K = len(arrays)
+
+        fields.resize(K)
+        columns.resize(K)
+        for i in range(K):
+            arr = arrays[i]
+            c_name = tobytes(names[i])
+
+            fields[i].reset(new CField(c_name, arr.type.sp_type, True))
+            columns[i].reset(new CColumn(fields[i], arr.sp_array))
+
+        if name is None:
+            c_name = ''
+        else:
+            c_name = tobytes(name)
+
+        schema.reset(new CSchema(fields))
+        table.reset(new CTable(c_name, schema, columns))
+
+        result = Table()
+        result.init(table)
+
+        return result
+
+    def to_pandas(self):
+        """
+        Convert the arrow::Table to a pandas DataFrame
+        """
+        cdef:
+            PyObject* arr
+            shared_ptr[CColumn] col
+
+        import pandas as pd
+
+        names = []
+        data = []
+        for i in range(self.table.num_columns()):
+            col = self.table.column(i)
+            check_status(pyarrow.ArrowToPandas(col, &arr))
+            names.append(frombytes(col.get().name()))
+            data.append(<object> arr)
+
+        return pd.DataFrame(dict(zip(names, data)), columns=names)
+
+    property name:
+
+        def __get__(self):
+            self._check_nullptr()
+            return frombytes(self.table.name())
+
+    property schema:
+
+        def __get__(self):
+            raise box_schema(self.table.schema())
+
+    def column(self, index):
+        self._check_nullptr()
+        cdef Column column = Column()
+        column.init(self.table.column(index))
+        return column
+
+    def __getitem__(self, i):
+        return self.column(i)
+
+    def itercolumns(self):
+        for i in range(self.num_columns):
+            yield self.column(i)
+
+    property num_columns:
+
+        def __get__(self):
+            self._check_nullptr()
+            return self.table.num_columns()
+
+    property num_rows:
+
+        def __get__(self):
+            self._check_nullptr()
+            return self.table.num_rows()
+
+    def __len__(self):
+        return self.num_rows
+
+    property shape:
+
+        def __get__(self):
+            return (self.num_rows, self.num_columns)
+
diff --git a/python/pyarrow/tests/test_column.py b/python/pyarrow/tests/test_column.py
new file mode 100644
index 0000000000000..b62f58236e073
--- /dev/null
+++ b/python/pyarrow/tests/test_column.py
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyarrow.compat import unittest
+import pyarrow as arrow
+
+A = arrow
+
+import pandas as pd
+
+
+class TestColumn(unittest.TestCase):
+
+    def test_basics(self):
+        data = [
+            A.from_pylist([-10, -5, 0, 5, 10])
+        ]
+        table = A.Table.from_arrays(('a'), data, 'table_name')
+        column = table.column(0)
+        assert column.name == 'a'
+        assert column.length() == 5
+        assert len(column) == 5
+        assert column.shape == (5,)
+
+    def test_pandas(self):
+        data = [
+            A.from_pylist([-10, -5, 0, 5, 10])
+        ]
+        table = A.Table.from_arrays(('a'), data, 'table_name')
+        column = table.column(0)
+        series = column.to_pandas()
+        assert series.name == 'a'
+        assert series.shape == (5,)
+        assert series.iloc[0] == -10
+
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 2e24445bd0c22..83fcbb8faff5d 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -20,6 +20,8 @@
 
 A = arrow
 
+import pandas as pd
+
 
 class TestRowBatch(unittest.TestCase):
 
@@ -38,3 +40,40 @@ def test_basics(self):
         assert len(batch) == num_rows
         assert batch.num_rows == num_rows
         assert batch.num_columns == len(data)
+
+
+class TestTable(unittest.TestCase):
+
+    def test_basics(self):
+        data = [
+            A.from_pylist(range(5)),
+            A.from_pylist([-10, -5, 0, 5, 10])
+        ]
+        table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
+        assert table.name == 'table_name'
+        assert len(table) == 5
+        assert table.num_rows == 5
+        assert table.num_columns == 2
+        assert table.shape == (5, 2)
+
+        for col in table.itercolumns():
+            for chunk in col.data.iterchunks():
+                assert chunk is not None
+
+    def test_pandas(self):
+        data = [
+            A.from_pylist(range(5)),
+            A.from_pylist([-10, -5, 0, 5, 10])
+        ]
+        table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
+
+        # TODO: Use this part once from_pandas is implemented
+        # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}
+        # df = pd.DataFrame(data)
+        # A.Table.from_pandas(df)
+
+        df = table.to_pandas()
+        assert set(df.columns) == set(('a', 'b'))
+        assert df.shape == (5, 2)
+        assert df.ix[0, 'b'] == -10
+
diff --git a/python/setup.py b/python/setup.py
index 5cc871aba9f81..ebd80de46b4da 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -214,7 +214,7 @@ def get_ext_built(self, name):
             return name + suffix
 
     def get_cmake_cython_names(self):
-        return ['array', 'config', 'error', 'scalar', 'schema']
+        return ['array', 'config', 'error', 'scalar', 'schema', 'table']
 
     def get_names(self):
         return self._found_names

From 79fddd1138ff69953e943f5980533dc01eabbb97 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 31 Mar 2016 17:48:38 -0700
Subject: [PATCH 0056/1644] ARROW-90: [C++] Check for SIMD instruction set
 support

This also adds an option to disable the usage of a specific instruction
set, e.g. you compile on a machine that supports SSE3 but you want to
use the binary also on machines without SSE3. (Distribution packagers
will love that option!)

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #50 from xhochy/arrow-90 and squashes the following commits:

6fd80d3 [Uwe L. Korn] ARROW-90: Check for SIMD instruction set support
---
 cpp/CMakeLists.txt | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6ed2768d13918..26d12d2424796 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -66,6 +66,14 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow IPC extensions"
     ON)
 
+  option(ARROW_SSE3
+    "Build Arrow with SSE3"
+    ON)
+
+  option(ARROW_ALTIVEC
+    "Build Arrow with Altivec"
+    ON)
+
 endif()
 
 if(NOT ARROW_BUILD_TESTS)
@@ -81,9 +89,25 @@ endif()
 # Compiler flags
 ############################################################
 
+# Check if the target architecture and compiler supports some special
+# instruction sets that would boost performance.
+include(CheckCXXCompilerFlag)
+# x86/amd64 compiler flags
+CHECK_CXX_COMPILER_FLAG("-msse3" CXX_SUPPORTS_SSE3)
+# power compiler flags
+CHECK_CXX_COMPILER_FLAG("-maltivec" CXX_SUPPORTS_ALTIVEC)
+
 # compiler flags that are common across debug/release builds
 #  - Wall: Enable all warnings.
-set(CXX_COMMON_FLAGS "-std=c++11 -msse3 -Wall")
+set(CXX_COMMON_FLAGS "-std=c++11 -Wall")
+
+# Only enable additional instruction sets if they are supported
+if (CXX_SUPPORTS_SSE3 AND ARROW_SSE3)
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse3")
+endif()
+if (CXX_SUPPORTS_ALTIVEC AND ARROW_ALTIVEC)
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -maltivec")
+endif()
 
 if (APPLE)
   # Depending on the default OSX_DEPLOYMENT_TARGET (< 10.9), libstdc++ may be

From 5d129991b3369b0e45cb79d1efe6ba2fd8dd21d0 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Fri, 1 Apr 2016 21:40:20 -0700
Subject: [PATCH 0057/1644] ARROW-71: [C++] Add clang-tidy and clang-format to
 the the tool chain.

I changed the ubuntu flavor for building to precise because https://github.com/travis-ci/apt-source-whitelist/issues/199 is currently blocking using trusty.

I also expect there might be a couple of iterations on settings for clang-format and clang-tidy (or if we even want them as standard parts of the toolchain).  @wesm I noticed the lint target explicitly turns off some checks, I don't know if these were copy and pasted or you really don't like them.  If the latter I can do a first pass of turning the same ones off for clang-tidy.

In terms of reviewing: It is likely useful, to look at the PR commit by commit, since the last two commits are 99% driven by the first commit.   The main chunk of code that wasn't machine fixed is FatalLog in logging.

The good news is clang-tidy caught one potential corner case segfault when a column happened to be null :)

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #55 from emkornfield/emk_add_clang_tidy_PR and squashes the following commits:

2fafb10 [Micah Kornfield] adjust line length from 88 to 90, turn on bin packing of parameters. increase penality for before first call parameter
169352f [Micah Kornfield] add llvm tool chain as travis source
e7723d1 [Micah Kornfield] upgrade to precise to verify if build works.  address self comments
d3f76d8 [Micah Kornfield] clang format change
9c556ef [Micah Kornfield] cleanup from clang-tidy
26945e9 [Micah Kornfield] add more failure checks for build_thirdparty
4dd0b81 [Micah Kornfield] Add clang-format and clang-tidy targets to toolchain
---
 .travis.yml                                  |   5 +-
 ci/travis_script_cpp.sh                      |   4 +
 cpp/CMakeLists.txt                           |  39 +++++-
 cpp/README.md                                |  16 +++
 cpp/build-support/run-clang-format.sh        |  42 ++++++
 cpp/build-support/run-clang-tidy.sh          |  40 ++++++
 cpp/cmake_modules/FindClangTools.cmake       |  60 +++++++++
 cpp/src/.clang-format                        |  65 ++++++++++
 cpp/src/.clang-tidy                          |  14 ++
 cpp/src/arrow/api.h                          |   2 +-
 cpp/src/arrow/array-test.cc                  |  16 +--
 cpp/src/arrow/array.cc                       |  17 +--
 cpp/src/arrow/array.h                        |  28 ++--
 cpp/src/arrow/builder.cc                     |   2 +-
 cpp/src/arrow/builder.h                      |  40 +++---
 cpp/src/arrow/column-benchmark.cc            |  23 ++--
 cpp/src/arrow/column-test.cc                 |   2 +-
 cpp/src/arrow/column.cc                      |  27 ++--
 cpp/src/arrow/column.h                       |  41 ++----
 cpp/src/arrow/ipc/adapter.cc                 |  50 ++++----
 cpp/src/arrow/ipc/adapter.h                  |  18 +--
 cpp/src/arrow/ipc/ipc-adapter-test.cc        |  20 ++-
 cpp/src/arrow/ipc/ipc-memory-test.cc         |  15 +--
 cpp/src/arrow/ipc/ipc-metadata-test.cc       |   8 +-
 cpp/src/arrow/ipc/memory.cc                  |  46 +++----
 cpp/src/arrow/ipc/memory.h                   |  22 ++--
 cpp/src/arrow/ipc/metadata-internal.cc       |  70 +++++-----
 cpp/src/arrow/ipc/metadata-internal.h        |  12 +-
 cpp/src/arrow/ipc/metadata.cc                |  72 ++++-------
 cpp/src/arrow/ipc/metadata.h                 |  20 +--
 cpp/src/arrow/ipc/test-common.h              |  10 +-
 cpp/src/arrow/parquet/parquet-schema-test.cc |  63 ++++-----
 cpp/src/arrow/parquet/schema.cc              |  15 +--
 cpp/src/arrow/parquet/schema.h               |  11 +-
 cpp/src/arrow/schema-test.cc                 |   6 +-
 cpp/src/arrow/schema.cc                      |  20 +--
 cpp/src/arrow/schema.h                       |  10 +-
 cpp/src/arrow/table-test.cc                  |  16 +--
 cpp/src/arrow/table.cc                       |  31 ++---
 cpp/src/arrow/table.h                        |  38 ++----
 cpp/src/arrow/test-util.h                    |  58 ++++-----
 cpp/src/arrow/type.cc                        |   8 +-
 cpp/src/arrow/type.h                         |  94 +++++---------
 cpp/src/arrow/types/binary.h                 |   6 +-
 cpp/src/arrow/types/collection.h             |  12 +-
 cpp/src/arrow/types/construct.cc             |  42 +++---
 cpp/src/arrow/types/construct.h              |  11 +-
 cpp/src/arrow/types/datetime.h               |  39 ++----
 cpp/src/arrow/types/decimal-test.cc          |   2 +-
 cpp/src/arrow/types/decimal.cc               |   3 +-
 cpp/src/arrow/types/decimal.h                |  11 +-
 cpp/src/arrow/types/json.cc                  |   5 +-
 cpp/src/arrow/types/json.h                   |   8 +-
 cpp/src/arrow/types/list-test.cc             |  11 +-
 cpp/src/arrow/types/list.cc                  |  25 ++--
 cpp/src/arrow/types/list.h                   |  65 ++++------
 cpp/src/arrow/types/primitive-test.cc        | 107 +++++++---------
 cpp/src/arrow/types/primitive.cc             |  75 ++++-------
 cpp/src/arrow/types/primitive.h              | 128 +++++++------------
 cpp/src/arrow/types/string-test.cc           |  20 +--
 cpp/src/arrow/types/string.cc                |  10 +-
 cpp/src/arrow/types/string.h                 |  48 +++----
 cpp/src/arrow/types/struct-test.cc           |   4 +-
 cpp/src/arrow/types/struct.cc                |   4 +-
 cpp/src/arrow/types/struct.h                 |   6 +-
 cpp/src/arrow/types/test-common.h            |   9 +-
 cpp/src/arrow/types/union.cc                 |   6 +-
 cpp/src/arrow/types/union.h                  |  17 +--
 cpp/src/arrow/util/bit-util-test.cc          |   2 +-
 cpp/src/arrow/util/bit-util.cc               |  10 +-
 cpp/src/arrow/util/bit-util.h                |   6 +-
 cpp/src/arrow/util/buffer-test.cc            |   5 +-
 cpp/src/arrow/util/buffer.cc                 |  16 +--
 cpp/src/arrow/util/buffer.h                  |  61 +++------
 cpp/src/arrow/util/logging.h                 |  78 +++++++----
 cpp/src/arrow/util/macros.h                  |   6 +-
 cpp/src/arrow/util/memory-pool-test.cc       |   2 +-
 cpp/src/arrow/util/memory-pool.cc            |   2 +-
 cpp/src/arrow/util/memory-pool.h             |   4 +-
 cpp/src/arrow/util/random.h                  |  27 ++--
 cpp/src/arrow/util/status.cc                 |  10 +-
 cpp/src/arrow/util/status.h                  |  45 ++++---
 cpp/src/arrow/util/test_main.cc              |   2 +-
 cpp/thirdparty/build_thirdparty.sh           |   4 +-
 84 files changed, 1015 insertions(+), 1155 deletions(-)
 create mode 100755 cpp/build-support/run-clang-format.sh
 create mode 100755 cpp/build-support/run-clang-tidy.sh
 create mode 100644 cpp/cmake_modules/FindClangTools.cmake
 create mode 100644 cpp/src/.clang-format
 create mode 100644 cpp/src/.clang-tidy

diff --git a/.travis.yml b/.travis.yml
index d89a200b892e6..a0138a79598a1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,11 +1,14 @@
 sudo: required
-dist: trusty
+dist: precise 
 addons:
   apt:
     sources:
     - ubuntu-toolchain-r-test
     - kalakris-cmake
+    - llvm-toolchain-precise-3.7
     packages:
+    - clang-format-3.7
+    - clang-tidy-3.7
     - gcc-4.9   # Needed for C++11
     - g++-4.9   # Needed for C++11
     - gdb
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index 997bdf35e83d2..c9b3b5f1442a1 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -7,6 +7,10 @@ set -e
 pushd $CPP_BUILD_DIR
 
 make lint
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+  make check-format
+  make check-clang-tidy
+fi
 
 ctest -L unittest
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 26d12d2424796..f803c0fb3e428 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -30,10 +30,11 @@ set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 # Must be declared in the top-level CMakeLists.txt.
 set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
 
-# Generate a Clang compile_commands.json "compilation database" file for use
-# with various development tools, such as Vim's YouCompleteMe plugin.
-# See http://clang.llvm.org/docs/JSONCompilationDatabase.html
-if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
+find_package(ClangTools)
+if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
+  # Generate a Clang compile_commands.json "compilation database" file for use
+  # with various development tools, such as Vim's YouCompleteMe plugin.
+  # See http://clang.llvm.org/docs/JSONCompilationDatabase.html
   set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 endif()
 
@@ -540,6 +541,36 @@ if (UNIX)
     `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`)
 endif (UNIX)
 
+
+############################################################
+# "make format" and "make check-format" targets
+############################################################
+if (${CLANG_FORMAT_FOUND})
+  # runs clang format and updates files in place.
+  add_custom_target(format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1
+  `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`)
+
+  # runs clang format and exits with a non-zero exit code if any files need to be reformatted
+  add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0
+  `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`)
+endif()
+
+
+############################################################
+# "make clang-tidy" and "make check-clang-tidy" targets
+############################################################
+if (${CLANG_TIDY_FOUND})
+  # runs clang-tidy and attempts to fix any warning automatically
+  add_custom_target(clang-tidy ${BUILD_SUPPORT_DIR}/run-clang-tidy.sh ${CLANG_TIDY_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 1
+  `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc | sed -e '/_generated/g'`)
+  # runs clang-tidy and exits with a non-zero exit code if any errors are found.
+  add_custom_target(check-clang-tidy ${BUILD_SUPPORT_DIR}/run-clang-tidy.sh ${CLANG_TIDY_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 
+  0 `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc | sed -e '/_generated/g'`)
+
+endif()
+
+
+
 ############################################################
 # Subdirectories
 ############################################################
diff --git a/cpp/README.md b/cpp/README.md
index 9026cf963f8ee..3f5da21b7d417 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -61,3 +61,19 @@ variables
 * Googletest: `GTEST_HOME` (only required to build the unit tests)
 * Google Benchmark: `GBENCHMARK_HOME` (only required if building benchmarks)
 * Flatbuffers: `FLATBUFFERS_HOME` (only required for the IPC extensions)
+
+## Continuous Integration
+
+Pull requests are run through travis-ci for continuous integration.  You can avoid
+build failures by running the following checks before submitting your pull request:
+
+    make unittest
+    make lint
+    # The next two commands may change your code.  It is recommended you commit
+    # before running them.
+    make clang-tidy # requires clang-tidy is installed
+    make format # requires clang-format is installed
+
+Note that the clang-tidy target may take a while to run.  You might consider
+running clang-tidy separately on the files you have added/changed before
+invoking the make target to reduce iteration time.
diff --git a/cpp/build-support/run-clang-format.sh b/cpp/build-support/run-clang-format.sh
new file mode 100755
index 0000000000000..ba525dfc33c69
--- /dev/null
+++ b/cpp/build-support/run-clang-format.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Runs clang format in the given directory
+# Arguments:
+#   $1 - Path to the source tree
+#   $2 - Path to the clang format binary
+#   $3 - Apply fixes (will raise an error if false and not there where changes)
+#   $ARGN - Files to run clang format on
+#
+SOURCE_DIR=$1
+shift
+CLANG_FORMAT=$1
+shift
+APPLY_FIXES=$1
+shift
+
+# clang format will only find its configuration if we are in 
+# the source tree or in a path relative to the source tree
+pushd $SOURCE_DIR
+if [ "$APPLY_FIXES" == "1" ]; then
+  $CLANG_FORMAT -i $@
+else
+
+  NUM_CORRECTIONS=`$CLANG_FORMAT -output-replacements-xml  $@ | grep offset | wc -l`
+  if [ "$NUM_CORRECTIONS" -gt "0" ]; then
+    echo "clang-format suggested changes, please run 'make format'!!!!"
+    exit 1
+  fi
+fi 
+popd
diff --git a/cpp/build-support/run-clang-tidy.sh b/cpp/build-support/run-clang-tidy.sh
new file mode 100755
index 0000000000000..4ba8ab8cd766d
--- /dev/null
+++ b/cpp/build-support/run-clang-tidy.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# Runs clang format in the given directory
+# Arguments:
+#   $1 - Path to the clang tidy binary
+#   $2 - Path to the compile_commands.json to use
+#   $3 - Apply fixes (will raise an error if false and not there where changes)
+#   $ARGN - Files to run clang-tidy on
+#
+CLANG_TIDY=$1
+shift
+COMPILE_COMMANDS=$1
+shift
+APPLY_FIXES=$1
+shift
+
+# clang format will only find its configuration if we are in 
+# the source tree or in a path relative to the source tree
+if [ "$APPLY_FIXES" == "1" ]; then
+  $CLANG_TIDY -p $COMPILE_COMMANDS -fix  $@
+else
+  NUM_CORRECTIONS=`$CLANG_TIDY -p $COMPILE_COMMANDS $@ 2>&1 | grep -v Skipping | grep "warnings* generated" | wc -l`
+  if [ "$NUM_CORRECTIONS" -gt "0" ]; then
+    echo "clang-tidy had suggested fixes.  Please fix these!!!"
+    exit 1
+  fi
+fi 
diff --git a/cpp/cmake_modules/FindClangTools.cmake b/cpp/cmake_modules/FindClangTools.cmake
new file mode 100644
index 0000000000000..c07c7d244493e
--- /dev/null
+++ b/cpp/cmake_modules/FindClangTools.cmake
@@ -0,0 +1,60 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find the clang-tidy and clang-format modules
+#
+# Usage of this module as follows:
+#
+#  find_package(ClangTools)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  ClangToolsBin_HOME -
+#   When set, this path is inspected instead of standard library binary locations
+#   to find clang-tidy and clang-format
+#
+# This module defines
+#  CLANG_TIDY_BIN, The  path to the clang tidy binary
+#  CLANG_TIDY_FOUND, Whether clang tidy was found
+#  CLANG_FORMAT_BIN, The path to the clang format binary 
+#  CLANG_TIDY_FOUND, Whether clang format was found
+
+find_program(CLANG_TIDY_BIN 
+  NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6  clang-tidy
+  PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin 
+        NO_DEFAULT_PATH
+)
+
+if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" ) 
+  set(CLANG_TIDY_FOUND 0)
+  message("clang-tidy not found")
+else()
+  set(CLANG_TIDY_FOUND 1)
+  message("clang-tidy found at ${CLANG_TIDY_BIN}")
+endif()
+
+find_program(CLANG_FORMAT_BIN 
+  NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6  clang-format
+  PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin 
+        NO_DEFAULT_PATH
+)
+
+if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" ) 
+  set(CLANG_FORMAT_FOUND 0)
+  message("clang-format not found")
+else()
+  set(CLANG_FORMAT_FOUND 1)
+  message("clang-format found at ${CLANG_FORMAT_BIN}")
+endif()
+
diff --git a/cpp/src/.clang-format b/cpp/src/.clang-format
new file mode 100644
index 0000000000000..7d5b3cf30ef51
--- /dev/null
+++ b/cpp/src/.clang-format
@@ -0,0 +1,65 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: false 
+AlignConsecutiveAssignments: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: true 
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: false 
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true 
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit: 90 
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false 
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1000
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
diff --git a/cpp/src/.clang-tidy b/cpp/src/.clang-tidy
new file mode 100644
index 0000000000000..deaa9bdf97fa1
--- /dev/null
+++ b/cpp/src/.clang-tidy
@@ -0,0 +1,14 @@
+---
+Checks:          'clang-diagnostic-*,clang-analyzer-*,-clang-analyzer-alpha*,google-.*,modernize-.*,readablity-.*'
+HeaderFilterRegex: 'arrow/.*'
+AnalyzeTemporaryDtors: true
+CheckOptions:    
+  - key:             google-readability-braces-around-statements.ShortStatementLines
+    value:           '1'
+  - key:             google-readability-function-size.StatementThreshold
+    value:           '800'
+  - key:             google-readability-namespace-comments.ShortNamespaceLines
+    value:           '10'
+  - key:             google-readability-namespace-comments.SpacesBeforeComments
+    value:           '2'
+
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 2ae80f642f29d..2d317b49cb7b6 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -37,4 +37,4 @@
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
-#endif // ARROW_API_H
+#endif  // ARROW_API_H
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 121b802d994fa..b4c727997ee7e 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -33,15 +33,12 @@ namespace arrow {
 
 class TestArray : public ::testing::Test {
  public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-  }
+  void SetUp() { pool_ = default_memory_pool(); }
 
  protected:
   MemoryPool* pool_;
 };
 
-
 TEST_F(TestArray, TestNullCount) {
   auto data = std::make_shared<PoolBuffer>(pool_);
   auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
@@ -53,7 +50,6 @@ TEST_F(TestArray, TestNullCount) {
   ASSERT_EQ(0, arr_no_nulls->null_count());
 }
 
-
 TEST_F(TestArray, TestLength) {
   auto data = std::make_shared<PoolBuffer>(pool_);
   std::unique_ptr<Int32Array> arr(new Int32Array(100, data));
@@ -61,14 +57,16 @@ TEST_F(TestArray, TestLength) {
 }
 
 TEST_F(TestArray, TestIsNull) {
+  // clang-format off
   std::vector<uint8_t> null_bitmap = {1, 0, 1, 1, 0, 1, 0, 0,
                                       1, 0, 1, 1, 0, 1, 0, 0,
                                       1, 0, 1, 1, 0, 1, 0, 0,
                                       1, 0, 1, 1, 0, 1, 0, 0,
                                       1, 0, 0, 1};
+  // clang-format on
   int32_t null_count = 0;
   for (uint8_t x : null_bitmap) {
-    if (x == 0) ++null_count;
+    if (x == 0) { ++null_count; }
   }
 
   std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(null_bitmap);
@@ -85,8 +83,6 @@ TEST_F(TestArray, TestIsNull) {
   }
 }
 
+TEST_F(TestArray, TestCopy) {}
 
-TEST_F(TestArray, TestCopy) {
-}
-
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 3736732740b5b..a1536861a20be 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -32,30 +32,25 @@ Array::Array(const TypePtr& type, int32_t length, int32_t null_count,
   length_ = length;
   null_count_ = null_count;
   null_bitmap_ = null_bitmap;
-  if (null_bitmap_) {
-    null_bitmap_data_ = null_bitmap_->data();
-  }
+  if (null_bitmap_) { null_bitmap_data_ = null_bitmap_->data(); }
 }
 
 bool Array::EqualsExact(const Array& other) const {
-  if (this == &other) return true;
+  if (this == &other) { return true; }
   if (length_ != other.length_ || null_count_ != other.null_count_ ||
       type_enum() != other.type_enum()) {
     return false;
   }
   if (null_count_ > 0) {
     return null_bitmap_->Equals(*other.null_bitmap_, util::bytes_for_bits(length_));
-  } else {
-    return true;
   }
+  return true;
 }
 
 bool NullArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) return true;
-  if (Type::NA != arr->type_enum()) {
-    return false;
-  }
+  if (this == arr.get()) { return true; }
+  if (Type::NA != arr->type_enum()) { return false; }
   return arr->length() == length_;
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 097634d74f890..c6735f87d8f42 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -36,8 +36,7 @@ class Buffer;
 // count is greater than 0
 class Array {
  public:
-  Array(const std::shared_ptr<DataType>& type, int32_t length,
-      int32_t null_count = 0,
+  Array(const std::shared_ptr<DataType>& type, int32_t length, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   virtual ~Array() {}
@@ -47,19 +46,15 @@ class Array {
     return null_count_ > 0 && util::bit_not_set(null_bitmap_data_, i);
   }
 
-  int32_t length() const { return length_;}
-  int32_t null_count() const { return null_count_;}
+  int32_t length() const { return length_; }
+  int32_t null_count() const { return null_count_; }
 
-  const std::shared_ptr<DataType>& type() const { return type_;}
-  Type::type type_enum() const { return type_->type;}
+  const std::shared_ptr<DataType>& type() const { return type_; }
+  Type::type type_enum() const { return type_->type; }
 
-  const std::shared_ptr<Buffer>& null_bitmap() const {
-    return null_bitmap_;
-  }
+  const std::shared_ptr<Buffer>& null_bitmap() const { return null_bitmap_; }
 
-  const uint8_t* null_bitmap_data() const {
-    return null_bitmap_data_;
-  }
+  const uint8_t* null_bitmap_data() const { return null_bitmap_data_; }
 
   bool EqualsExact(const Array& arr) const;
   virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
@@ -80,17 +75,16 @@ class Array {
 // Degenerate null type Array
 class NullArray : public Array {
  public:
-  NullArray(const std::shared_ptr<DataType>& type, int32_t length) :
-      Array(type, length, length, nullptr) {}
+  NullArray(const std::shared_ptr<DataType>& type, int32_t length)
+      : Array(type, length, length, nullptr) {}
 
-  explicit NullArray(int32_t length) :
-      NullArray(std::make_shared<NullType>(), length) {}
+  explicit NullArray(int32_t length) : NullArray(std::make_shared<NullType>(), length) {}
 
   bool Equals(const std::shared_ptr<Array>& arr) const override;
 };
 
 typedef std::shared_ptr<Array> ArrayPtr;
 
-} // namespace arrow
+}  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 4061f35fd5e53..1447078f76028 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -62,4 +62,4 @@ Status ArrayBuilder::Reserve(int32_t elements) {
   return Status::OK();
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index d1a49dce79961..21a6341ef5086 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -37,30 +37,26 @@ static constexpr int32_t MIN_BUILDER_CAPACITY = 1 << 5;
 // Base class for all data array builders
 class ArrayBuilder {
  public:
-  explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type) :
-      pool_(pool),
-      type_(type),
-      null_bitmap_(nullptr),
-      null_count_(0),
-      null_bitmap_data_(nullptr),
-      length_(0),
-      capacity_(0) {}
+  explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type)
+      : pool_(pool),
+        type_(type),
+        null_bitmap_(nullptr),
+        null_count_(0),
+        null_bitmap_data_(nullptr),
+        length_(0),
+        capacity_(0) {}
 
   virtual ~ArrayBuilder() {}
 
   // For nested types. Since the objects are owned by this class instance, we
   // skip shared pointers and just return a raw pointer
-  ArrayBuilder* child(int i) {
-    return children_[i].get();
-  }
+  ArrayBuilder* child(int i) { return children_[i].get(); }
 
-  int num_children() const {
-    return children_.size();
-  }
+  int num_children() const { return children_.size(); }
 
-  int32_t length() const { return length_;}
-  int32_t null_count() const { return null_count_;}
-  int32_t capacity() const { return capacity_;}
+  int32_t length() const { return length_; }
+  int32_t null_count() const { return null_count_; }
+  int32_t capacity() const { return capacity_; }
 
   // Allocates requires memory at this level, but children need to be
   // initialized independently
@@ -76,15 +72,13 @@ class ArrayBuilder {
   // this function responsibly.
   Status Advance(int32_t elements);
 
-  const std::shared_ptr<PoolBuffer>& null_bitmap() const { return null_bitmap_;}
+  const std::shared_ptr<PoolBuffer>& null_bitmap() const { return null_bitmap_; }
 
   // Creates new array object to hold the contents of the builder and transfers
   // ownership of the data
   virtual std::shared_ptr<Array> Finish() = 0;
 
-  const std::shared_ptr<DataType>& type() const {
-    return type_;
-  }
+  const std::shared_ptr<DataType>& type() const { return type_; }
 
  protected:
   MemoryPool* pool_;
@@ -107,6 +101,6 @@ class ArrayBuilder {
   DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
 };
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_BUILDER_H_
+#endif  // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index 335d581782ac0..edea0948860de 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
 #include "benchmark/benchmark.h"
 
 #include "arrow/test-util.h"
@@ -24,19 +23,19 @@
 
 namespace arrow {
 namespace {
-  template <typename ArrayType>
-  std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
-    auto pool = default_memory_pool();
-    auto data = std::make_shared<PoolBuffer>(pool);
-    auto null_bitmap = std::make_shared<PoolBuffer>(pool);
-    data->Resize(length * sizeof(typename ArrayType::value_type));
-    null_bitmap->Resize(util::bytes_for_bits(length));
-    return std::make_shared<ArrayType>(length, data, 10, null_bitmap);
-  }
+template <typename ArrayType>
+std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
+  auto pool = default_memory_pool();
+  auto data = std::make_shared<PoolBuffer>(pool);
+  auto null_bitmap = std::make_shared<PoolBuffer>(pool);
+  data->Resize(length * sizeof(typename ArrayType::value_type));
+  null_bitmap->Resize(util::bytes_for_bits(length));
+  return std::make_shared<ArrayType>(length, data, 10, null_bitmap);
+}
 }  // anonymous namespace
 
-
-static void BM_BuildInt32ColumnByChunk(benchmark::State& state) { //NOLINT non-const reference
+static void BM_BuildInt32ColumnByChunk(
+    benchmark::State& state) {  // NOLINT non-const reference
   ArrayVector arrays;
   for (int chunk_n = 0; chunk_n < state.range_x(); ++chunk_n) {
     arrays.push_back(MakePrimitive<Int32Array>(100, 10));
diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
index 0630785630e81..1edf313d49bf6 100644
--- a/cpp/src/arrow/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -72,4 +72,4 @@ TEST_F(TestColumn, ChunksInhomogeneous) {
   ASSERT_RAISES(Invalid, column_->ValidateData());
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index 46acf8df2ff57..52e4c58e1dc3d 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -26,8 +26,7 @@
 
 namespace arrow {
 
-ChunkedArray::ChunkedArray(const ArrayVector& chunks) :
-    chunks_(chunks) {
+ChunkedArray::ChunkedArray(const ArrayVector& chunks) : chunks_(chunks) {
   length_ = 0;
   null_count_ = 0;
   for (const std::shared_ptr<Array>& chunk : chunks) {
@@ -36,35 +35,31 @@ ChunkedArray::ChunkedArray(const ArrayVector& chunks) :
   }
 }
 
-Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks) :
-    field_(field) {
+Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks)
+    : field_(field) {
   data_ = std::make_shared<ChunkedArray>(chunks);
 }
 
-Column::Column(const std::shared_ptr<Field>& field,
-    const std::shared_ptr<Array>& data) :
-    field_(field) {
+Column::Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data)
+    : field_(field) {
   data_ = std::make_shared<ChunkedArray>(ArrayVector({data}));
 }
 
-Column::Column(const std::shared_ptr<Field>& field,
-    const std::shared_ptr<ChunkedArray>& data) :
-    field_(field),
-    data_(data) {}
+Column::Column(
+    const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data)
+    : field_(field), data_(data) {}
 
 Status Column::ValidateData() {
   for (int i = 0; i < data_->num_chunks(); ++i) {
     const std::shared_ptr<DataType>& type = data_->chunk(i)->type();
     if (!this->type()->Equals(type)) {
       std::stringstream ss;
-      ss << "In chunk " << i << " expected type "
-         << this->type()->ToString()
-         << " but saw "
-         << type->ToString();
+      ss << "In chunk " << i << " expected type " << this->type()->ToString()
+         << " but saw " << type->ToString();
       return Status::Invalid(ss.str());
     }
   }
   return Status::OK();
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
index 1ad97b20863c8..22becc3454780 100644
--- a/cpp/src/arrow/column.h
+++ b/cpp/src/arrow/column.h
@@ -39,21 +39,13 @@ class ChunkedArray {
   explicit ChunkedArray(const ArrayVector& chunks);
 
   // @returns: the total length of the chunked array; computed on construction
-  int64_t length() const {
-    return length_;
-  }
+  int64_t length() const { return length_; }
 
-  int64_t null_count() const {
-    return null_count_;
-  }
+  int64_t null_count() const { return null_count_; }
 
-  int num_chunks() const {
-    return chunks_.size();
-  }
+  int num_chunks() const { return chunks_.size(); }
 
-  const std::shared_ptr<Array>& chunk(int i) const {
-    return chunks_[i];
-  }
+  const std::shared_ptr<Array>& chunk(int i) const { return chunks_[i]; }
 
  protected:
   ArrayVector chunks_;
@@ -67,33 +59,22 @@ class ChunkedArray {
 class Column {
  public:
   Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
-  Column(const std::shared_ptr<Field>& field,
-      const std::shared_ptr<ChunkedArray>& data);
+  Column(const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data);
 
   Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data);
 
-  int64_t length() const {
-    return data_->length();
-  }
+  int64_t length() const { return data_->length(); }
 
-  int64_t null_count() const {
-    return data_->null_count();
-  }
+  int64_t null_count() const { return data_->null_count(); }
 
   // @returns: the column's name in the passed metadata
-  const std::string& name() const {
-    return field_->name;
-  }
+  const std::string& name() const { return field_->name; }
 
   // @returns: the column's type according to the metadata
-  const std::shared_ptr<DataType>& type() const {
-    return field_->type;
-  }
+  const std::shared_ptr<DataType>& type() const { return field_->type; }
 
   // @returns: the column's data as a chunked logical array
-  const std::shared_ptr<ChunkedArray>& data() const {
-    return data_;
-  }
+  const std::shared_ptr<ChunkedArray>& data() const { return data_; }
   // Verify that the column's array data is consistent with the passed field's
   // metadata
   Status ValidateData();
@@ -103,6 +84,6 @@ class Column {
   std::shared_ptr<ChunkedArray> data_;
 };
 
-} // namespace arrow
+}  // namespace arrow
 
 #endif  // ARROW_COLUMN_H
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index c79e8469530f7..2f72c3aa8467a 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -94,8 +94,7 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
 
 class RowBatchWriter {
  public:
-  explicit RowBatchWriter(const RowBatch* batch) :
-      batch_(batch) {}
+  explicit RowBatchWriter(const RowBatch* batch) : batch_(batch) {}
 
   Status AssemblePayload() {
     // Perform depth-first traversal of the row-batch
@@ -138,12 +137,12 @@ class RowBatchWriter {
     // determine the data header size then request a buffer such that you can
     // construct the flatbuffer data accessor object (see arrow::ipc::Message)
     std::shared_ptr<Buffer> data_header;
-    RETURN_NOT_OK(WriteDataHeader(batch_->num_rows(), offset,
-            field_nodes_, buffer_meta_, &data_header));
+    RETURN_NOT_OK(WriteDataHeader(
+        batch_->num_rows(), offset, field_nodes_, buffer_meta_, &data_header));
 
     // Write the data header at the end
-    RETURN_NOT_OK(dst->Write(position + offset, data_header->data(),
-            data_header->size()));
+    RETURN_NOT_OK(
+        dst->Write(position + offset, data_header->data(), data_header->size()));
 
     *data_header_offset = position + offset;
     return Status::OK();
@@ -174,8 +173,8 @@ class RowBatchWriter {
   std::vector<std::shared_ptr<Buffer>> buffers_;
 };
 
-Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
-    int64_t* header_offset) {
+Status WriteRowBatch(
+    MemorySource* dst, const RowBatch* batch, int64_t position, int64_t* header_offset) {
   RowBatchWriter serializer(batch);
   RETURN_NOT_OK(serializer.AssemblePayload());
   return serializer.Write(dst, position, header_offset);
@@ -187,15 +186,14 @@ static constexpr int64_t INIT_METADATA_SIZE = 4096;
 
 class RowBatchReader::Impl {
  public:
-  Impl(MemorySource* source, const std::shared_ptr<RecordBatchMessage>& metadata) :
-      source_(source),
-      metadata_(metadata) {
+  Impl(MemorySource* source, const std::shared_ptr<RecordBatchMessage>& metadata)
+      : source_(source), metadata_(metadata) {
     num_buffers_ = metadata->num_buffers();
     num_flattened_fields_ = metadata->num_fields();
   }
 
-  Status AssembleBatch(const std::shared_ptr<Schema>& schema,
-      std::shared_ptr<RowBatch>* out) {
+  Status AssembleBatch(
+      const std::shared_ptr<Schema>& schema, std::shared_ptr<RowBatch>* out) {
     std::vector<std::shared_ptr<Array>> arrays(schema->num_fields());
 
     // The field_index and buffer_index are incremented in NextArray based on
@@ -208,8 +206,7 @@ class RowBatchReader::Impl {
       RETURN_NOT_OK(NextArray(field, &arrays[i]));
     }
 
-    *out = std::make_shared<RowBatch>(schema, metadata_->length(),
-        arrays);
+    *out = std::make_shared<RowBatch>(schema, metadata_->length(), arrays);
     return Status::OK();
   }
 
@@ -243,11 +240,10 @@ class RowBatchReader::Impl {
       } else {
         data.reset(new Buffer(nullptr, 0));
       }
-      return MakePrimitiveArray(type, field_meta.length, data,
-          field_meta.null_count, null_bitmap, out);
-    } else {
-      return Status::NotImplemented("Non-primitive types not complete yet");
+      return MakePrimitiveArray(
+          type, field_meta.length, data, field_meta.null_count, null_bitmap, out);
     }
+    return Status::NotImplemented("Non-primitive types not complete yet");
   }
 
   Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
@@ -264,8 +260,8 @@ class RowBatchReader::Impl {
   int num_flattened_fields_;
 };
 
-Status RowBatchReader::Open(MemorySource* source, int64_t position,
-    std::shared_ptr<RowBatchReader>* out) {
+Status RowBatchReader::Open(
+    MemorySource* source, int64_t position, std::shared_ptr<RowBatchReader>* out) {
   std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(source->ReadAt(position, INIT_METADATA_SIZE, &metadata));
 
@@ -274,8 +270,7 @@ Status RowBatchReader::Open(MemorySource* source, int64_t position,
   // We may not need to call source->ReadAt again
   if (metadata_size > static_cast<int>(INIT_METADATA_SIZE - sizeof(int32_t))) {
     // We don't have enough data, read the indicated metadata size.
-    RETURN_NOT_OK(source->ReadAt(position + sizeof(int32_t),
-            metadata_size, &metadata));
+    RETURN_NOT_OK(source->ReadAt(position + sizeof(int32_t), metadata_size, &metadata));
   }
 
   // TODO(wesm): buffer slicing here would be better in case ReadAt returns
@@ -297,11 +292,10 @@ Status RowBatchReader::Open(MemorySource* source, int64_t position,
   return Status::OK();
 }
 
-Status RowBatchReader::GetRowBatch(const std::shared_ptr<Schema>& schema,
-    std::shared_ptr<RowBatch>* out) {
+Status RowBatchReader::GetRowBatch(
+    const std::shared_ptr<Schema>& schema, std::shared_ptr<RowBatch>* out) {
   return impl_->AssembleBatch(schema, out);
 }
 
-
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 26dea6d04b889..d453fa05f4982 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -52,8 +52,8 @@ class RecordBatchMessage;
 //
 // Finally, the memory offset to the start of the metadata / data header is
 // returned in an out-variable
-Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
-    int64_t* header_offset);
+Status WriteRowBatch(
+    MemorySource* dst, const RowBatch* batch, int64_t position, int64_t* header_offset);
 
 // int64_t GetRowBatchMetadata(const RowBatch* batch);
 
@@ -67,20 +67,20 @@ int64_t GetRowBatchSize(const RowBatch* batch);
 
 class RowBatchReader {
  public:
-  static Status Open(MemorySource* source, int64_t position,
-      std::shared_ptr<RowBatchReader>* out);
+  static Status Open(
+      MemorySource* source, int64_t position, std::shared_ptr<RowBatchReader>* out);
 
   // Reassemble the row batch. A Schema is required to be able to construct the
   // right array containers
-  Status GetRowBatch(const std::shared_ptr<Schema>& schema,
-      std::shared_ptr<RowBatch>* out);
+  Status GetRowBatch(
+      const std::shared_ptr<Schema>& schema, std::shared_ptr<RowBatch>* out);
 
  private:
   class Impl;
   std::unique_ptr<Impl> impl_;
 };
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
 
-#endif // ARROW_IPC_MEMORY_H
+#endif  // ARROW_IPC_MEMORY_H
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 79b4d710d282f..fbdda77e4919c 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -42,12 +42,8 @@ namespace ipc {
 
 class TestWriteRowBatch : public ::testing::Test, public MemoryMapFixture {
  public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-  }
-  void TearDown() {
-    MemoryMapFixture::TearDown();
-  }
+  void SetUp() { pool_ = default_memory_pool(); }
+  void TearDown() { MemoryMapFixture::TearDown(); }
 
   void InitMemoryMap(int64_t size) {
     std::string path = "test-write-row-batch";
@@ -83,8 +79,8 @@ TEST_F(TestWriteRowBatch, IntegerRoundTrip) {
   test::random_bytes(null_bytes, 0, null_bitmap->mutable_data());
 
   auto a0 = std::make_shared<Int32Array>(length, data);
-  auto a1 = std::make_shared<Int32Array>(length, data,
-      test::bitmap_popcount(null_bitmap->data(), length), null_bitmap);
+  auto a1 = std::make_shared<Int32Array>(
+      length, data, test::bitmap_popcount(null_bitmap->data(), length), null_bitmap);
 
   RowBatch batch(schema, length, {a0, a1});
 
@@ -103,10 +99,10 @@ TEST_F(TestWriteRowBatch, IntegerRoundTrip) {
   EXPECT_EQ(batch.num_rows(), batch_result->num_rows());
 
   for (int i = 0; i < batch.num_columns(); ++i) {
-    EXPECT_TRUE(batch.column(i)->Equals(batch_result->column(i)))
-      << i << batch.column_name(i);
+    EXPECT_TRUE(batch.column(i)->Equals(batch_result->column(i))) << i
+                                                                  << batch.column_name(i);
   }
 }
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-memory-test.cc b/cpp/src/arrow/ipc/ipc-memory-test.cc
index 332ad2a2b809b..1933921222595 100644
--- a/cpp/src/arrow/ipc/ipc-memory-test.cc
+++ b/cpp/src/arrow/ipc/ipc-memory-test.cc
@@ -35,13 +35,10 @@ namespace ipc {
 
 class TestMemoryMappedSource : public ::testing::Test, public MemoryMapFixture {
  public:
-  void TearDown() {
-    MemoryMapFixture::TearDown();
-  }
+  void TearDown() { MemoryMapFixture::TearDown(); }
 };
 
-TEST_F(TestMemoryMappedSource, InvalidUsages) {
-}
+TEST_F(TestMemoryMappedSource, InvalidUsages) {}
 
 TEST_F(TestMemoryMappedSource, WriteRead) {
   const int64_t buffer_size = 1024;
@@ -74,9 +71,9 @@ TEST_F(TestMemoryMappedSource, InvalidFile) {
   std::string non_existent_path = "invalid-file-name-asfd";
 
   std::shared_ptr<MemoryMappedSource> result;
-  ASSERT_RAISES(IOError, MemoryMappedSource::Open(non_existent_path,
-          MemorySource::READ_ONLY, &result));
+  ASSERT_RAISES(IOError,
+      MemoryMappedSource::Open(non_existent_path, MemorySource::READ_ONLY, &result));
 }
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
index ceabec0fa7c29..51d79cfb4c4bb 100644
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -86,14 +86,12 @@ TEST_F(TestSchemaMessage, NestedFields) {
   auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
   auto f0 = std::make_shared<Field>("f0", type);
 
-  std::shared_ptr<StructType> type2(new StructType({
-            std::make_shared<Field>("k1", INT32),
-              std::make_shared<Field>("k2", INT32),
-              std::make_shared<Field>("k3", INT32)}));
+  std::shared_ptr<StructType> type2(new StructType({std::make_shared<Field>("k1", INT32),
+      std::make_shared<Field>("k2", INT32), std::make_shared<Field>("k3", INT32)}));
   auto f1 = std::make_shared<Field>("f1", type2);
 
   Schema schema({f0, f1});
   CheckRoundtrip(&schema);
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/memory.cc b/cpp/src/arrow/ipc/memory.cc
index e630ccd109b77..2b077e9792925 100644
--- a/cpp/src/arrow/ipc/memory.cc
+++ b/cpp/src/arrow/ipc/memory.cc
@@ -17,7 +17,7 @@
 
 #include "arrow/ipc/memory.h"
 
-#include <sys/mman.h> // For memory-mapping
+#include <sys/mman.h>  // For memory-mapping
 #include <algorithm>
 #include <cerrno>
 #include <cstdint>
@@ -32,8 +32,7 @@
 namespace arrow {
 namespace ipc {
 
-MemorySource::MemorySource(AccessMode access_mode) :
-    access_mode_(access_mode) {}
+MemorySource::MemorySource(AccessMode access_mode) : access_mode_(access_mode) {}
 
 MemorySource::~MemorySource() {}
 
@@ -41,10 +40,7 @@ MemorySource::~MemorySource() {}
 
 class MemoryMappedSource::Impl {
  public:
-  Impl() :
-      file_(nullptr),
-      is_open_(false),
-      data_(nullptr) {}
+  Impl() : file_(nullptr), is_open_(false), data_(nullptr) {}
 
   ~Impl() {
     if (is_open_) {
@@ -54,9 +50,7 @@ class MemoryMappedSource::Impl {
   }
 
   Status Open(const std::string& path, MemorySource::AccessMode mode) {
-    if (is_open_) {
-      return Status::IOError("A file is already open");
-    }
+    if (is_open_) { return Status::IOError("A file is already open"); }
 
     path_ = path;
 
@@ -72,18 +66,15 @@ class MemoryMappedSource::Impl {
     }
 
     fseek(file_, 0L, SEEK_END);
-    if (ferror(file_)) {
-      return Status::IOError("Unable to seek to end of file");
-    }
+    if (ferror(file_)) { return Status::IOError("Unable to seek to end of file"); }
     size_ = ftell(file_);
 
     fseek(file_, 0L, SEEK_SET);
     is_open_ = true;
 
     // TODO(wesm): Add read-only version of this
-    data_ = reinterpret_cast<uint8_t*>(mmap(nullptr, size_,
-            PROT_READ | PROT_WRITE,
-            MAP_SHARED, fileno(file_), 0));
+    data_ = reinterpret_cast<uint8_t*>(
+        mmap(nullptr, size_, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(file_), 0));
     if (data_ == nullptr) {
       std::stringstream ss;
       ss << "Memory mapping file failed, errno: " << errno;
@@ -93,13 +84,9 @@ class MemoryMappedSource::Impl {
     return Status::OK();
   }
 
-  int64_t size() const {
-    return size_;
-  }
+  int64_t size() const { return size_; }
 
-  uint8_t* data() {
-    return data_;
-  }
+  uint8_t* data() { return data_; }
 
  private:
   std::string path_;
@@ -111,8 +98,8 @@ class MemoryMappedSource::Impl {
   uint8_t* data_;
 };
 
-MemoryMappedSource::MemoryMappedSource(AccessMode access_mode) :
-    MemorySource(access_mode) {}
+MemoryMappedSource::MemoryMappedSource(AccessMode access_mode)
+    : MemorySource(access_mode) {}
 
 Status MemoryMappedSource::Open(const std::string& path, AccessMode access_mode,
     std::shared_ptr<MemoryMappedSource>* out) {
@@ -134,8 +121,8 @@ Status MemoryMappedSource::Close() {
   return Status::OK();
 }
 
-Status MemoryMappedSource::ReadAt(int64_t position, int64_t nbytes,
-    std::shared_ptr<Buffer>* out) {
+Status MemoryMappedSource::ReadAt(
+    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
   if (position < 0 || position >= impl_->size()) {
     return Status::Invalid("position is out of bounds");
   }
@@ -145,8 +132,7 @@ Status MemoryMappedSource::ReadAt(int64_t position, int64_t nbytes,
   return Status::OK();
 }
 
-Status MemoryMappedSource::Write(int64_t position, const uint8_t* data,
-    int64_t nbytes) {
+Status MemoryMappedSource::Write(int64_t position, const uint8_t* data, int64_t nbytes) {
   if (position < 0 || position >= impl_->size()) {
     return Status::Invalid("position is out of bounds");
   }
@@ -158,5 +144,5 @@ Status MemoryMappedSource::Write(int64_t position, const uint8_t* data,
   return Status::OK();
 }
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/memory.h b/cpp/src/arrow/ipc/memory.h
index 0b4d8347c342f..e529603dc6e2a 100644
--- a/cpp/src/arrow/ipc/memory.h
+++ b/cpp/src/arrow/ipc/memory.h
@@ -52,8 +52,8 @@ class OutputStream {
 // memory map
 class BufferOutputStream : public OutputStream {
  public:
-  explicit BufferOutputStream(const std::shared_ptr<MutableBuffer>& buffer):
-      buffer_(buffer) {}
+  explicit BufferOutputStream(const std::shared_ptr<MutableBuffer>& buffer)
+      : buffer_(buffer) {}
 
   // Implement the OutputStream interface
   Status Close() override;
@@ -72,10 +72,7 @@ class BufferOutputStream : public OutputStream {
 class MemorySource {
  public:
   // Indicates the access permissions of the memory source
-  enum AccessMode {
-    READ_ONLY,
-    READ_WRITE
-  };
+  enum AccessMode { READ_ONLY, READ_WRITE };
 
   virtual ~MemorySource();
 
@@ -83,8 +80,8 @@ class MemorySource {
   // the indicated location
   // @returns: arrow::Status indicating success / failure. The buffer is set
   // into the *out argument
-  virtual Status ReadAt(int64_t position, int64_t nbytes,
-      std::shared_ptr<Buffer>* out) = 0;
+  virtual Status ReadAt(
+      int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
 
   virtual Status Close() = 0;
 
@@ -110,8 +107,7 @@ class MemoryMappedSource : public MemorySource {
 
   Status Close() override;
 
-  Status ReadAt(int64_t position, int64_t nbytes,
-      std::shared_ptr<Buffer>* out) override;
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   Status Write(int64_t position, const uint8_t* data, int64_t nbytes) override;
 
@@ -125,7 +121,7 @@ class MemoryMappedSource : public MemorySource {
   std::unique_ptr<Impl> impl_;
 };
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
 
-#endif // ARROW_IPC_MEMORY_H
+#endif  // ARROW_IPC_MEMORY_H
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 14b186906c3a0..ad5951d17e2c0 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -52,11 +52,12 @@ const std::shared_ptr<DataType> UINT64 = std::make_shared<UInt64Type>();
 const std::shared_ptr<DataType> FLOAT = std::make_shared<FloatType>();
 const std::shared_ptr<DataType> DOUBLE = std::make_shared<DoubleType>();
 
-static Status IntFromFlatbuffer(const flatbuf::Int* int_data,
-    std::shared_ptr<DataType>* out) {
+static Status IntFromFlatbuffer(
+    const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
   if (int_data->bitWidth() % 8 != 0) {
     return Status::NotImplemented("Integers not in cstdint are not implemented");
-  } else if (int_data->bitWidth() > 64) {
+  }
+  if (int_data->bitWidth() > 64) {
     return Status::NotImplemented("Integers with more than 64 bits not implemented");
   }
 
@@ -80,8 +81,8 @@ static Status IntFromFlatbuffer(const flatbuf::Int* int_data,
   return Status::OK();
 }
 
-static Status FloatFromFlatuffer(const flatbuf::FloatingPoint* float_data,
-    std::shared_ptr<DataType>* out) {
+static Status FloatFromFlatuffer(
+    const flatbuf::FloatingPoint* float_data, std::shared_ptr<DataType>* out) {
   if (float_data->precision() == flatbuf::Precision_SINGLE) {
     *out = FLOAT;
   } else {
@@ -90,9 +91,8 @@ static Status FloatFromFlatuffer(const flatbuf::FloatingPoint* float_data,
   return Status::OK();
 }
 
-static Status TypeFromFlatbuffer(flatbuf::Type type,
-    const void* type_data, const std::vector<std::shared_ptr<Field>>& children,
-    std::shared_ptr<DataType>* out) {
+static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
+    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
   switch (type) {
     case flatbuf::Type_NONE:
       return Status::Invalid("Type metadata cannot be none");
@@ -101,8 +101,8 @@ static Status TypeFromFlatbuffer(flatbuf::Type type,
     case flatbuf::Type_Bit:
       return Status::NotImplemented("Type is not implemented");
     case flatbuf::Type_FloatingPoint:
-      return FloatFromFlatuffer(static_cast<const flatbuf::FloatingPoint*>(type_data),
-          out);
+      return FloatFromFlatuffer(
+          static_cast<const flatbuf::FloatingPoint*>(type_data), out);
     case flatbuf::Type_Binary:
     case flatbuf::Type_Utf8:
       return Status::NotImplemented("Type is not implemented");
@@ -128,16 +128,14 @@ static Status TypeFromFlatbuffer(flatbuf::Type type,
 }
 
 // Forward declaration
-static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
-    FieldOffset* offset);
+static Status FieldToFlatbuffer(
+    FBB& fbb, const std::shared_ptr<Field>& field, FieldOffset* offset);
 
-static Offset IntToFlatbuffer(FBB& fbb, int bitWidth,
-    bool is_signed) {
+static Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
   return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
 }
 
-static Offset FloatToFlatbuffer(FBB& fbb,
-    flatbuf::Precision precision) {
+static Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
   return flatbuf::CreateFloatingPoint(fbb, precision).Union();
 }
 
@@ -166,10 +164,8 @@ static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type
   *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
   break;
 
-
 static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* children,
-    flatbuf::Type* out_type, Offset* offset) {
+    std::vector<FieldOffset>* children, flatbuf::Type* out_type, Offset* offset) {
   switch (type->type) {
     case Type::BOOL:
       *out_type = flatbuf::Type_Bool;
@@ -206,16 +202,16 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *out_type = flatbuf::Type_Tuple;
       return StructToFlatbuffer(fbb, type, children, offset);
     default:
+      *out_type = flatbuf::Type_NONE;  // Make clang-tidy happy
       std::stringstream ss;
-      ss << "Unable to convert type: " << type->ToString()
-         << std::endl;
+      ss << "Unable to convert type: " << type->ToString() << std::endl;
       return Status::NotImplemented(ss.str());
   }
   return Status::OK();
 }
 
-static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
-    FieldOffset* offset) {
+static Status FieldToFlatbuffer(
+    FBB& fbb, const std::shared_ptr<Field>& field, FieldOffset* offset) {
   auto fb_name = fbb.CreateString(field->name);
 
   flatbuf::Type type_enum;
@@ -225,14 +221,13 @@ static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
   RETURN_NOT_OK(TypeToFlatbuffer(fbb, field->type, &children, &type_enum, &type_data));
   auto fb_children = fbb.CreateVector(children);
 
-  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum,
-      type_data, fb_children);
+  *offset = flatbuf::CreateField(
+      fbb, fb_name, field->nullable, type_enum, type_data, fb_children);
 
   return Status::OK();
 }
 
-Status FieldFromFlatbuffer(const flatbuf::Field* field,
-    std::shared_ptr<Field>* out) {
+Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out) {
   std::shared_ptr<DataType> type;
 
   auto children = field->children();
@@ -241,8 +236,8 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field,
     RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), &child_fields[i]));
   }
 
-  RETURN_NOT_OK(TypeFromFlatbuffer(field->type_type(),
-          field->type(), child_fields, &type));
+  RETURN_NOT_OK(
+      TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
 
   *out = std::make_shared<Field>(field->name()->str(), type);
   return Status::OK();
@@ -270,19 +265,17 @@ Status MessageBuilder::SetRecordBatch(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
     const std::vector<flatbuf::Buffer>& buffers) {
   header_type_ = flatbuf::MessageHeader_RecordBatch;
-  header_ = flatbuf::CreateRecordBatch(fbb_, length,
-      fbb_.CreateVectorOfStructs(nodes),
-      fbb_.CreateVectorOfStructs(buffers)).Union();
+  header_ = flatbuf::CreateRecordBatch(fbb_, length, fbb_.CreateVectorOfStructs(nodes),
+                fbb_.CreateVectorOfStructs(buffers))
+                .Union();
   body_length_ = body_length;
 
   return Status::OK();
 }
 
-
 Status WriteDataHeader(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
-    const std::vector<flatbuf::Buffer>& buffers,
-    std::shared_ptr<Buffer>* out) {
+    const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
   MessageBuilder message;
   RETURN_NOT_OK(message.SetRecordBatch(length, body_length, nodes, buffers));
   RETURN_NOT_OK(message.Finish());
@@ -290,8 +283,7 @@ Status WriteDataHeader(int32_t length, int64_t body_length,
 }
 
 Status MessageBuilder::Finish() {
-  auto message = flatbuf::CreateMessage(fbb_, header_type_, header_,
-      body_length_);
+  auto message = flatbuf::CreateMessage(fbb_, header_type_, header_, body_length_);
   fbb_.Finish(message);
   return Status::OK();
 }
@@ -313,5 +305,5 @@ Status MessageBuilder::GetBuffer(std::shared_ptr<Buffer>* out) {
   return Status::OK();
 }
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index f7365d2a49f95..779c5a30a044a 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -36,8 +36,7 @@ class Status;
 
 namespace ipc {
 
-Status FieldFromFlatbuffer(const flatbuf::Field* field,
-    std::shared_ptr<Field>* out);
+Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out);
 
 class MessageBuilder {
  public:
@@ -60,10 +59,9 @@ class MessageBuilder {
 
 Status WriteDataHeader(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
-    const std::vector<flatbuf::Buffer>& buffers,
-    std::shared_ptr<Buffer>* out);
+    const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
 
-#endif // ARROW_IPC_METADATA_INTERNAL_H
+#endif  // ARROW_IPC_METADATA_INTERNAL_H
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 642f21a41e640..bcf104f0b8ba6 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -48,10 +48,8 @@ Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out) {
 
 class Message::Impl {
  public:
-  explicit Impl(const std::shared_ptr<Buffer>& buffer,
-      const flatbuf::Message* message) :
-      buffer_(buffer),
-      message_(message) {}
+  explicit Impl(const std::shared_ptr<Buffer>& buffer, const flatbuf::Message* message)
+      : buffer_(buffer), message_(message) {}
 
   Message::Type type() const {
     switch (message_->header_type()) {
@@ -66,13 +64,9 @@ class Message::Impl {
     }
   }
 
-  const void* header() const {
-    return message_->header();
-  }
+  const void* header() const { return message_->header(); }
 
-  int64_t body_length() const {
-    return message_->bodyLength();
-  }
+  int64_t body_length() const { return message_->bodyLength(); }
 
  private:
   // Owns the memory this message accesses
@@ -83,16 +77,12 @@ class Message::Impl {
 
 class SchemaMessage::Impl {
  public:
-  explicit Impl(const void* schema) :
-      schema_(static_cast<const flatbuf::Schema*>(schema)) {}
+  explicit Impl(const void* schema)
+      : schema_(static_cast<const flatbuf::Schema*>(schema)) {}
 
-  const flatbuf::Field* field(int i) const {
-    return schema_->fields()->Get(i);
-  }
+  const flatbuf::Field* field(int i) const { return schema_->fields()->Get(i); }
 
-  int num_fields() const {
-    return schema_->fields()->size();
-  }
+  int num_fields() const { return schema_->fields()->size(); }
 
  private:
   const flatbuf::Schema* schema_;
@@ -100,8 +90,8 @@ class SchemaMessage::Impl {
 
 Message::Message() {}
 
-Status Message::Open(const std::shared_ptr<Buffer>& buffer,
-    std::shared_ptr<Message>* out) {
+Status Message::Open(
+    const std::shared_ptr<Buffer>& buffer, std::shared_ptr<Message>* out) {
   std::shared_ptr<Message> result(new Message());
 
   // The buffer is prefixed by its size as int32_t
@@ -128,12 +118,11 @@ std::shared_ptr<Message> Message::get_shared_ptr() {
 }
 
 std::shared_ptr<SchemaMessage> Message::GetSchema() {
-  return std::make_shared<SchemaMessage>(this->shared_from_this(),
-      impl_->header());
+  return std::make_shared<SchemaMessage>(this->shared_from_this(), impl_->header());
 }
 
-SchemaMessage::SchemaMessage(const std::shared_ptr<Message>& message,
-    const void* schema) {
+SchemaMessage::SchemaMessage(
+    const std::shared_ptr<Message>& message, const void* schema) {
   message_ = message;
   impl_.reset(new Impl(schema));
 }
@@ -158,31 +147,21 @@ Status SchemaMessage::GetSchema(std::shared_ptr<Schema>* out) const {
 
 class RecordBatchMessage::Impl {
  public:
-  explicit Impl(const void* batch) :
-      batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
+  explicit Impl(const void* batch)
+      : batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
     nodes_ = batch_->nodes();
     buffers_ = batch_->buffers();
   }
 
-  const flatbuf::FieldNode* field(int i) const {
-    return nodes_->Get(i);
-  }
+  const flatbuf::FieldNode* field(int i) const { return nodes_->Get(i); }
 
-  const flatbuf::Buffer* buffer(int i) const {
-    return buffers_->Get(i);
-  }
+  const flatbuf::Buffer* buffer(int i) const { return buffers_->Get(i); }
 
-  int32_t length() const {
-    return batch_->length();
-  }
+  int32_t length() const { return batch_->length(); }
 
-  int num_buffers() const {
-    return batch_->buffers()->size();
-  }
+  int num_buffers() const { return batch_->buffers()->size(); }
 
-  int num_fields() const {
-    return batch_->nodes()->size();
-  }
+  int num_fields() const { return batch_->nodes()->size(); }
 
  private:
   const flatbuf::RecordBatch* batch_;
@@ -191,12 +170,11 @@ class RecordBatchMessage::Impl {
 };
 
 std::shared_ptr<RecordBatchMessage> Message::GetRecordBatch() {
-  return std::make_shared<RecordBatchMessage>(this->shared_from_this(),
-      impl_->header());
+  return std::make_shared<RecordBatchMessage>(this->shared_from_this(), impl_->header());
 }
 
-RecordBatchMessage::RecordBatchMessage(const std::shared_ptr<Message>& message,
-    const void* batch) {
+RecordBatchMessage::RecordBatchMessage(
+    const std::shared_ptr<Message>& message, const void* batch) {
   message_ = message;
   impl_.reset(new Impl(batch));
 }
@@ -234,5 +212,5 @@ int RecordBatchMessage::num_fields() const {
   return impl_->num_fields();
 }
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index c7288529b9fbd..838a4a676ea35 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -85,8 +85,7 @@ struct BufferMetadata {
 class RecordBatchMessage {
  public:
   // Accepts an opaque flatbuffer pointer
-  RecordBatchMessage(const std::shared_ptr<Message>& message,
-      const void* batch_meta);
+  RecordBatchMessage(const std::shared_ptr<Message>& message, const void* batch_meta);
 
   FieldMetadata field(int i) const;
   BufferMetadata buffer(int i) const;
@@ -111,15 +110,10 @@ class DictionaryBatchMessage {
 
 class Message : public std::enable_shared_from_this<Message> {
  public:
-  enum Type {
-    NONE,
-    SCHEMA,
-    DICTIONARY_BATCH,
-    RECORD_BATCH
-  };
+  enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH };
 
-  static Status Open(const std::shared_ptr<Buffer>& buffer,
-    std::shared_ptr<Message>* out);
+  static Status Open(
+      const std::shared_ptr<Buffer>& buffer, std::shared_ptr<Message>* out);
 
   std::shared_ptr<Message> get_shared_ptr();
 
@@ -140,7 +134,7 @@ class Message : public std::enable_shared_from_this<Message> {
   std::unique_ptr<Impl> impl_;
 };
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
 
-#endif // ARROW_IPC_METADATA_H
+#endif  // ARROW_IPC_METADATA_H
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 0fccce941071b..65c837dc8b141 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -36,9 +36,7 @@ class MemoryMapFixture {
 
   void CreateFile(const std::string path, int64_t size) {
     FILE* file = fopen(path.c_str(), "w");
-    if (file != nullptr) {
-      tmp_files_.push_back(path);
-    }
+    if (file != nullptr) { tmp_files_.push_back(path); }
     ftruncate(fileno(file), size);
     fclose(file);
   }
@@ -47,7 +45,7 @@ class MemoryMapFixture {
   std::vector<std::string> tmp_files_;
 };
 
-} // namespace ipc
-} // namespace arrow
+}  // namespace ipc
+}  // namespace arrow
 
-#endif // ARROW_IPC_TEST_COMMON_H
+#endif  // ARROW_IPC_TEST_COMMON_H
diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
index a289ddbfde6eb..e2280f41189ef 100644
--- a/cpp/src/arrow/parquet/parquet-schema-test.cc
+++ b/cpp/src/arrow/parquet/parquet-schema-test.cc
@@ -45,8 +45,7 @@ const auto INT64 = std::make_shared<Int64Type>();
 const auto FLOAT = std::make_shared<FloatType>();
 const auto DOUBLE = std::make_shared<DoubleType>();
 const auto UTF8 = std::make_shared<StringType>();
-const auto BINARY = std::make_shared<ListType>(
-    std::make_shared<Field>("", UINT8));
+const auto BINARY = std::make_shared<ListType>(std::make_shared<Field>("", UINT8));
 const auto DECIMAL_8_4 = std::make_shared<DecimalType>(8, 4);
 
 class TestConvertParquetSchema : public ::testing::Test {
@@ -58,8 +57,8 @@ class TestConvertParquetSchema : public ::testing::Test {
     for (int i = 0; i < expected_schema->num_fields(); ++i) {
       auto lhs = result_schema_->field(i);
       auto rhs = expected_schema->field(i);
-      EXPECT_TRUE(lhs->Equals(rhs))
-        << i << " " << lhs->ToString() << " != " << rhs->ToString();
+      EXPECT_TRUE(lhs->Equals(rhs)) << i << " " << lhs->ToString()
+                                    << " != " << rhs->ToString();
     }
   }
 
@@ -99,20 +98,15 @@ TEST_F(TestConvertParquetSchema, ParquetFlatPrimitives) {
   arrow_fields.push_back(std::make_shared<Field>("double", DOUBLE));
 
   parquet_fields.push_back(
-      PrimitiveNode::Make("binary", Repetition::OPTIONAL,
-          ParquetType::BYTE_ARRAY));
+      PrimitiveNode::Make("binary", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY));
   arrow_fields.push_back(std::make_shared<Field>("binary", BINARY));
 
-  parquet_fields.push_back(
-      PrimitiveNode::Make("string", Repetition::OPTIONAL,
-          ParquetType::BYTE_ARRAY,
-          LogicalType::UTF8));
+  parquet_fields.push_back(PrimitiveNode::Make(
+      "string", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::UTF8));
   arrow_fields.push_back(std::make_shared<Field>("string", UTF8));
 
-  parquet_fields.push_back(
-      PrimitiveNode::Make("flba-binary", Repetition::OPTIONAL,
-          ParquetType::FIXED_LEN_BYTE_ARRAY,
-          LogicalType::NONE, 12));
+  parquet_fields.push_back(PrimitiveNode::Make("flba-binary", Repetition::OPTIONAL,
+      ParquetType::FIXED_LEN_BYTE_ARRAY, LogicalType::NONE, 12));
   arrow_fields.push_back(std::make_shared<Field>("flba-binary", BINARY));
 
   auto arrow_schema = std::make_shared<Schema>(arrow_fields);
@@ -125,28 +119,20 @@ TEST_F(TestConvertParquetSchema, ParquetFlatDecimals) {
   std::vector<NodePtr> parquet_fields;
   std::vector<std::shared_ptr<Field>> arrow_fields;
 
-  parquet_fields.push_back(
-      PrimitiveNode::Make("flba-decimal", Repetition::OPTIONAL,
-          ParquetType::FIXED_LEN_BYTE_ARRAY,
-          LogicalType::DECIMAL, 4, 8, 4));
+  parquet_fields.push_back(PrimitiveNode::Make("flba-decimal", Repetition::OPTIONAL,
+      ParquetType::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 4, 8, 4));
   arrow_fields.push_back(std::make_shared<Field>("flba-decimal", DECIMAL_8_4));
 
-  parquet_fields.push_back(
-      PrimitiveNode::Make("binary-decimal", Repetition::OPTIONAL,
-          ParquetType::BYTE_ARRAY,
-          LogicalType::DECIMAL, -1, 8, 4));
+  parquet_fields.push_back(PrimitiveNode::Make("binary-decimal", Repetition::OPTIONAL,
+      ParquetType::BYTE_ARRAY, LogicalType::DECIMAL, -1, 8, 4));
   arrow_fields.push_back(std::make_shared<Field>("binary-decimal", DECIMAL_8_4));
 
-  parquet_fields.push_back(
-      PrimitiveNode::Make("int32-decimal", Repetition::OPTIONAL,
-          ParquetType::INT32,
-          LogicalType::DECIMAL, -1, 8, 4));
+  parquet_fields.push_back(PrimitiveNode::Make("int32-decimal", Repetition::OPTIONAL,
+      ParquetType::INT32, LogicalType::DECIMAL, -1, 8, 4));
   arrow_fields.push_back(std::make_shared<Field>("int32-decimal", DECIMAL_8_4));
 
-  parquet_fields.push_back(
-      PrimitiveNode::Make("int64-decimal", Repetition::OPTIONAL,
-          ParquetType::INT64,
-          LogicalType::DECIMAL, -1, 8, 4));
+  parquet_fields.push_back(PrimitiveNode::Make("int64-decimal", Repetition::OPTIONAL,
+      ParquetType::INT64, LogicalType::DECIMAL, -1, 8, 4));
   arrow_fields.push_back(std::make_shared<Field>("int64-decimal", DECIMAL_8_4));
 
   auto arrow_schema = std::make_shared<Schema>(arrow_fields);
@@ -164,22 +150,19 @@ TEST_F(TestConvertParquetSchema, UnsupportedThings) {
   unsupported_nodes.push_back(
       GroupNode::Make("repeated-group", Repetition::REPEATED, {}));
 
-  unsupported_nodes.push_back(
-      PrimitiveNode::Make("int32", Repetition::OPTIONAL,
-          ParquetType::INT32, LogicalType::DATE));
+  unsupported_nodes.push_back(PrimitiveNode::Make(
+      "int32", Repetition::OPTIONAL, ParquetType::INT32, LogicalType::DATE));
 
-  unsupported_nodes.push_back(
-      PrimitiveNode::Make("int64", Repetition::OPTIONAL,
-          ParquetType::INT64, LogicalType::TIMESTAMP_MILLIS));
+  unsupported_nodes.push_back(PrimitiveNode::Make(
+      "int64", Repetition::OPTIONAL, ParquetType::INT64, LogicalType::TIMESTAMP_MILLIS));
 
   for (const NodePtr& node : unsupported_nodes) {
     ASSERT_RAISES(NotImplemented, ConvertSchema({node}));
   }
 }
 
-TEST(TestNodeConversion, DateAndTime) {
-}
+TEST(TestNodeConversion, DateAndTime) {}
 
-} // namespace parquet
+}  // namespace parquet
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index 14f4f5be53ce9..066388b4d0e23 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -43,8 +43,7 @@ const auto INT64 = std::make_shared<Int64Type>();
 const auto FLOAT = std::make_shared<FloatType>();
 const auto DOUBLE = std::make_shared<DoubleType>();
 const auto UTF8 = std::make_shared<StringType>();
-const auto BINARY = std::make_shared<ListType>(
-    std::make_shared<Field>("", UINT8));
+const auto BINARY = std::make_shared<ListType>(std::make_shared<Field>("", UINT8));
 
 TypePtr MakeDecimalType(const PrimitiveNode* node) {
   int precision = node->decimal_metadata().precision;
@@ -167,12 +166,12 @@ Status NodeToField(const NodePtr& node, std::shared_ptr<Field>* out) {
   return Status::OK();
 }
 
-Status FromParquetSchema(const ::parquet::SchemaDescriptor* parquet_schema,
-    std::shared_ptr<Schema>* out) {
+Status FromParquetSchema(
+    const ::parquet::SchemaDescriptor* parquet_schema, std::shared_ptr<Schema>* out) {
   // TODO(wesm): Consider adding an arrow::Schema name attribute, which comes
   // from the root Parquet node
-  const GroupNode* schema_node = static_cast<const GroupNode*>(
-      parquet_schema->schema().get());
+  const GroupNode* schema_node =
+      static_cast<const GroupNode*>(parquet_schema->schema().get());
 
   std::vector<std::shared_ptr<Field>> fields(schema_node->field_count());
   for (int i = 0; i < schema_node->field_count(); i++) {
@@ -183,6 +182,6 @@ Status FromParquetSchema(const ::parquet::SchemaDescriptor* parquet_schema,
   return Status::OK();
 }
 
-} // namespace parquet
+}  // namespace parquet
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h
index a8408970ede48..a44a9a4b6a892 100644
--- a/cpp/src/arrow/parquet/schema.h
+++ b/cpp/src/arrow/parquet/schema.h
@@ -31,14 +31,13 @@ class Status;
 
 namespace parquet {
 
-Status NodeToField(const ::parquet::schema::NodePtr& node,
-    std::shared_ptr<Field>* out);
+Status NodeToField(const ::parquet::schema::NodePtr& node, std::shared_ptr<Field>* out);
 
-Status FromParquetSchema(const ::parquet::SchemaDescriptor* parquet_schema,
-    std::shared_ptr<Schema>* out);
+Status FromParquetSchema(
+    const ::parquet::SchemaDescriptor* parquet_schema, std::shared_ptr<Schema>* out);
 
-} // namespace parquet
+}  // namespace parquet
 
-} // namespace arrow
+}  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/schema-test.cc b/cpp/src/arrow/schema-test.cc
index a1de1dc5ac8a4..8cc80be120a44 100644
--- a/cpp/src/arrow/schema-test.cc
+++ b/cpp/src/arrow/schema-test.cc
@@ -86,8 +86,8 @@ TEST_F(TestSchema, ToString) {
   auto f0 = std::make_shared<Field>("f0", INT32);
   auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(), false);
   auto f2 = std::make_shared<Field>("f2", std::make_shared<StringType>());
-  auto f3 = std::make_shared<Field>("f3",
-      std::make_shared<ListType>(std::make_shared<Int16Type>()));
+  auto f3 = std::make_shared<Field>(
+      "f3", std::make_shared<ListType>(std::make_shared<Int16Type>()));
 
   vector<shared_ptr<Field>> fields = {f0, f1, f2, f3};
   auto schema = std::make_shared<Schema>(fields);
@@ -101,4 +101,4 @@ f3: list<item: int16>)";
   ASSERT_EQ(expected, result);
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/schema.cc b/cpp/src/arrow/schema.cc
index 18aad0e806ff2..a38acaa94ba56 100644
--- a/cpp/src/arrow/schema.cc
+++ b/cpp/src/arrow/schema.cc
@@ -26,18 +26,14 @@
 
 namespace arrow {
 
-Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields) :
-    fields_(fields) {}
+Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields) : fields_(fields) {}
 
 bool Schema::Equals(const Schema& other) const {
-  if (this == &other) return true;
-  if (num_fields() != other.num_fields()) {
-    return false;
-  }
+  if (this == &other) { return true; }
+
+  if (num_fields() != other.num_fields()) { return false; }
   for (int i = 0; i < num_fields(); ++i) {
-    if (!field(i)->Equals(*other.field(i).get())) {
-      return false;
-    }
+    if (!field(i)->Equals(*other.field(i).get())) { return false; }
   }
   return true;
 }
@@ -51,13 +47,11 @@ std::string Schema::ToString() const {
 
   int i = 0;
   for (auto field : fields_) {
-    if (i > 0) {
-      buffer << std::endl;
-    }
+    if (i > 0) { buffer << std::endl; }
     buffer << field->ToString();
     ++i;
   }
   return buffer.str();
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/schema.h b/cpp/src/arrow/schema.h
index 52f3c1ceae46d..a8b0d8444ac92 100644
--- a/cpp/src/arrow/schema.h
+++ b/cpp/src/arrow/schema.h
@@ -35,21 +35,17 @@ class Schema {
   bool Equals(const std::shared_ptr<Schema>& other) const;
 
   // Return the ith schema element. Does not boundscheck
-  const std::shared_ptr<Field>& field(int i) const {
-    return fields_[i];
-  }
+  const std::shared_ptr<Field>& field(int i) const { return fields_[i]; }
 
   // Render a string representation of the schema suitable for debugging
   std::string ToString() const;
 
-  int num_fields() const {
-    return fields_.size();
-  }
+  int num_fields() const { return fields_.size(); }
 
  private:
   std::vector<std::shared_ptr<Field>> fields_;
 };
 
-} // namespace arrow
+}  // namespace arrow
 
 #endif  // ARROW_FIELD_H
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 4c7b8f80486de..385e7d831500a 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -49,10 +49,9 @@ class TestTable : public TestBase {
     schema_ = std::make_shared<Schema>(fields);
 
     columns_ = {
-      std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length)),
-      std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
-      std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length))
-    };
+        std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length)),
+        std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
+        std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length))};
   }
 
  protected:
@@ -116,13 +115,12 @@ TEST_F(TestTable, InvalidColumns) {
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
 
   columns_ = {
-    std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length)),
-    std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
-    std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length - 1))
-  };
+      std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length)),
+      std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
+      std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length - 1))};
 
   table_.reset(new Table("data", schema_, columns_, length));
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index e405c1d508c22..d9573eae74ddd 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -28,20 +28,16 @@
 namespace arrow {
 
 RowBatch::RowBatch(const std::shared_ptr<Schema>& schema, int num_rows,
-    const std::vector<std::shared_ptr<Array>>& columns) :
-    schema_(schema),
-    num_rows_(num_rows),
-    columns_(columns) {}
+    const std::vector<std::shared_ptr<Array>>& columns)
+    : schema_(schema), num_rows_(num_rows), columns_(columns) {}
 
 const std::string& RowBatch::column_name(int i) const {
   return schema_->field(i)->name;
 }
 
 Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
-    const std::vector<std::shared_ptr<Column>>& columns) :
-    name_(name),
-    schema_(schema),
-    columns_(columns) {
+    const std::vector<std::shared_ptr<Column>>& columns)
+    : name_(name), schema_(schema), columns_(columns) {
   if (columns.size() == 0) {
     num_rows_ = 0;
   } else {
@@ -50,11 +46,8 @@ Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
 }
 
 Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
-    const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows) :
-    name_(name),
-    schema_(schema),
-    columns_(columns),
-    num_rows_(num_rows) {}
+    const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
+    : name_(name), schema_(schema), columns_(columns), num_rows_(num_rows) {}
 
 Status Table::ValidateColumns() const {
   if (num_columns() != schema_->num_fields()) {
@@ -66,21 +59,17 @@ Status Table::ValidateColumns() const {
     const Column* col = columns_[i].get();
     if (col == nullptr) {
       std::stringstream ss;
-      ss << "Column " << i << " named " << col->name()
-         << " was null";
+      ss << "Column " << i << " was null";
       return Status::Invalid(ss.str());
     }
     if (col->length() != num_rows_) {
       std::stringstream ss;
-      ss << "Column " << i << " named " << col->name()
-         << " expected length "
-         << num_rows_
-         << " but got length "
-         << col->length();
+      ss << "Column " << i << " named " << col->name() << " expected length " << num_rows_
+         << " but got length " << col->length();
       return Status::Invalid(ss.str());
     }
   }
   return Status::OK();
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index e2f73a2eeddcb..756b2a19593f4 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -42,27 +42,19 @@ class RowBatch {
       const std::vector<std::shared_ptr<Array>>& columns);
 
   // @returns: the table's schema
-  const std::shared_ptr<Schema>& schema() const {
-    return schema_;
-  }
+  const std::shared_ptr<Schema>& schema() const { return schema_; }
 
   // @returns: the i-th column
   // Note: Does not boundscheck
-  const std::shared_ptr<Array>& column(int i) const {
-    return columns_[i];
-  }
+  const std::shared_ptr<Array>& column(int i) const { return columns_[i]; }
 
   const std::string& column_name(int i) const;
 
   // @returns: the number of columns in the table
-  int num_columns() const {
-    return columns_.size();
-  }
+  int num_columns() const { return columns_.size(); }
 
   // @returns: the number of rows (the corresponding length of each column)
-  int64_t num_rows() const {
-    return num_rows_;
-  }
+  int64_t num_rows() const { return num_rows_; }
 
  private:
   std::shared_ptr<Schema> schema_;
@@ -85,30 +77,20 @@ class Table {
       const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows);
 
   // @returns: the table's name, if any (may be length 0)
-  const std::string& name() const {
-    return name_;
-  }
+  const std::string& name() const { return name_; }
 
   // @returns: the table's schema
-  const std::shared_ptr<Schema>& schema() const {
-    return schema_;
-  }
+  const std::shared_ptr<Schema>& schema() const { return schema_; }
 
   // Note: Does not boundscheck
   // @returns: the i-th column
-  const std::shared_ptr<Column>& column(int i) const {
-    return columns_[i];
-  }
+  const std::shared_ptr<Column>& column(int i) const { return columns_[i]; }
 
   // @returns: the number of columns in the table
-  int num_columns() const {
-    return columns_.size();
-  }
+  int num_columns() const { return columns_.size(); }
 
   // @returns: the number of rows (the corresponding length of each column)
-  int64_t num_rows() const {
-    return num_rows_;
-  }
+  int64_t num_rows() const { return num_rows_; }
 
   // After construction, perform any checks to validate the input arguments
   Status ValidateColumns() const;
@@ -123,6 +105,6 @@ class Table {
   int64_t num_rows_;
 };
 
-} // namespace arrow
+}  // namespace arrow
 
 #endif  // ARROW_TABLE_H
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index b2bce269992d0..538d9b233d990 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -36,38 +36,29 @@
 #include "arrow/util/random.h"
 #include "arrow/util/status.h"
 
-#define ASSERT_RAISES(ENUM, expr)               \
-  do {                                          \
-    Status s = (expr);                          \
-    if (!s.Is##ENUM()) {                        \
-      FAIL() << s.ToString();                   \
-    }                                           \
+#define ASSERT_RAISES(ENUM, expr)                  \
+  do {                                             \
+    Status s = (expr);                             \
+    if (!s.Is##ENUM()) { FAIL() << s.ToString(); } \
   } while (0)
 
-
-#define ASSERT_OK(expr)                         \
-  do {                                          \
-    Status s = (expr);                          \
-    if (!s.ok()) {                              \
-        FAIL() << s.ToString();                 \
-    }                                           \
+#define ASSERT_OK(expr)                      \
+  do {                                       \
+    Status s = (expr);                       \
+    if (!s.ok()) { FAIL() << s.ToString(); } \
   } while (0)
 
-
-#define EXPECT_OK(expr)                         \
-  do {                                          \
-    Status s = (expr);                          \
-    EXPECT_TRUE(s.ok());                        \
+#define EXPECT_OK(expr)  \
+  do {                   \
+    Status s = (expr);   \
+    EXPECT_TRUE(s.ok()); \
   } while (0)
 
-
 namespace arrow {
 
 class TestBase : public ::testing::Test {
  public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-  }
+  void SetUp() { pool_ = default_memory_pool(); }
 
   template <typename ArrayType>
   std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
@@ -97,10 +88,8 @@ void randint(int64_t N, T lower, T upper, std::vector<T>* out) {
   }
 }
 
-
 template <typename T>
-void random_real(int n, uint32_t seed, T min_value, T max_value,
-    std::vector<T>* out) {
+void random_real(int n, uint32_t seed, T min_value, T max_value, std::vector<T>* out) {
   std::mt19937 gen(seed);
   std::uniform_real_distribution<T> d(min_value, max_value);
   for (int i = 0; i < n; ++i) {
@@ -108,11 +97,10 @@ void random_real(int n, uint32_t seed, T min_value, T max_value,
   }
 }
 
-
 template <typename T>
 std::shared_ptr<Buffer> to_buffer(const std::vector<T>& values) {
-  return std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(values.data()),
-      values.size() * sizeof(T));
+  return std::make_shared<Buffer>(
+      reinterpret_cast<const uint8_t*>(values.data()), values.size() * sizeof(T));
 }
 
 void random_null_bitmap(int64_t n, double pct_null, uint8_t* null_bitmap) {
@@ -143,8 +131,8 @@ void rand_uniform_int(int n, uint32_t seed, T min_value, T max_value, T* out) {
 static inline int bitmap_popcount(const uint8_t* data, int length) {
   int count = 0;
   for (int i = 0; i < length; ++i) {
-    // TODO: accelerate this
-    if (util::get_bit(data, i)) ++count;
+    // TODO(wesm): accelerate this
+    if (util::get_bit(data, i)) { ++count; }
   }
   return count;
 }
@@ -152,9 +140,7 @@ static inline int bitmap_popcount(const uint8_t* data, int length) {
 static inline int null_count(const std::vector<uint8_t>& valid_bytes) {
   int result = 0;
   for (size_t i = 0; i < valid_bytes.size(); ++i) {
-    if (valid_bytes[i] == 0) {
-      ++result;
-    }
+    if (valid_bytes[i] == 0) { ++result; }
   }
   return result;
 }
@@ -167,7 +153,7 @@ std::shared_ptr<Buffer> bytes_to_null_buffer(const std::vector<uint8_t>& bytes)
   return out;
 }
 
-} // namespace test
-} // namespace arrow
+}  // namespace test
+}  // namespace arrow
 
-#endif // ARROW_TEST_UTIL_H_
+#endif  // ARROW_TEST_UTIL_H_
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index f7f835e96a729..4e686d9cf4a6f 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -25,9 +25,7 @@ namespace arrow {
 std::string Field::ToString() const {
   std::stringstream ss;
   ss << this->name << ": " << this->type->ToString();
-  if (!this->nullable) {
-    ss << " not null";
-  }
+  if (!this->nullable) { ss << " not null"; }
   return ss.str();
 }
 
@@ -50,7 +48,7 @@ std::string StructType::ToString() const {
   std::stringstream s;
   s << "struct<";
   for (int i = 0; i < this->num_children(); ++i) {
-    if (i > 0) s << ", ";
+    if (i > 0) { s << ", "; }
     const std::shared_ptr<Field>& field = this->child(i);
     s << field->name << ": " << field->type->ToString();
   }
@@ -58,4 +56,4 @@ std::string StructType::ToString() const {
   return s.str();
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 86e47791b7cea..051ab46b199f9 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -110,8 +110,7 @@ struct DataType {
 
   std::vector<std::shared_ptr<Field>> children_;
 
-  explicit DataType(Type::type type) :
-      type(type) {}
+  explicit DataType(Type::type type) : type(type) {}
 
   virtual ~DataType();
 
@@ -120,21 +119,13 @@ struct DataType {
     return this == other || (this->type == other->type);
   }
 
-  bool Equals(const std::shared_ptr<DataType>& other) {
-    return Equals(other.get());
-  }
+  bool Equals(const std::shared_ptr<DataType>& other) { return Equals(other.get()); }
 
-  const std::shared_ptr<Field>& child(int i) const {
-    return children_[i];
-  }
+  const std::shared_ptr<Field>& child(int i) const { return children_[i]; }
 
-  int num_children() const {
-    return children_.size();
-  }
+  int num_children() const { return children_.size(); }
 
-  virtual int value_size() const {
-    return -1;
-  }
+  virtual int value_size() const { return -1; }
 
   virtual std::string ToString() const = 0;
 };
@@ -153,28 +144,20 @@ struct Field {
   // Fields can be nullable
   bool nullable;
 
-  Field(const std::string& name, const TypePtr& type, bool nullable = true) :
-      name(name),
-      type(type),
-      nullable(nullable) {}
+  Field(const std::string& name, const TypePtr& type, bool nullable = true)
+      : name(name), type(type), nullable(nullable) {}
 
-  bool operator==(const Field& other) const {
-    return this->Equals(other);
-  }
+  bool operator==(const Field& other) const { return this->Equals(other); }
 
-  bool operator!=(const Field& other) const {
-    return !this->Equals(other);
-  }
+  bool operator!=(const Field& other) const { return !this->Equals(other); }
 
   bool Equals(const Field& other) const {
-    return (this == &other) || (this->name == other.name &&
-        this->nullable == other.nullable &&
-        this->type->Equals(other.type.get()));
+    return (this == &other) ||
+           (this->name == other.name && this->nullable == other.nullable &&
+               this->type->Equals(other.type.get()));
   }
 
-  bool Equals(const std::shared_ptr<Field>& other) const {
-    return Equals(*other.get());
-  }
+  bool Equals(const std::shared_ptr<Field>& other) const { return Equals(*other.get()); }
 
   std::string ToString() const;
 };
@@ -192,20 +175,15 @@ inline std::string PrimitiveType<Derived>::ToString() const {
   return result;
 }
 
-#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)  \
-  typedef C_TYPE c_type;                                    \
-  static constexpr Type::type type_enum = Type::ENUM;       \
-                                                            \
-  TYPENAME()                                                \
-      : PrimitiveType<TYPENAME>() {}                        \
-                                                            \
-  virtual int value_size() const {                          \
-    return SIZE;                                            \
-  }                                                         \
-                                                            \
-  static const char* name() {                               \
-    return NAME;                                            \
-  }
+#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME) \
+  typedef C_TYPE c_type;                                   \
+  static constexpr Type::type type_enum = Type::ENUM;      \
+                                                           \
+  TYPENAME() : PrimitiveType<TYPENAME>() {}                \
+                                                           \
+  virtual int value_size() const { return SIZE; }          \
+                                                           \
+  static const char* name() { return NAME; }
 
 struct NullType : public PrimitiveType<NullType> {
   PRIMITIVE_DECL(NullType, void, NA, 0, "null");
@@ -257,27 +235,19 @@ struct DoubleType : public PrimitiveType<DoubleType> {
 
 struct ListType : public DataType {
   // List can contain any other logical value type
-  explicit ListType(const std::shared_ptr<DataType>& value_type)
-      : DataType(Type::LIST) {
+  explicit ListType(const std::shared_ptr<DataType>& value_type) : DataType(Type::LIST) {
     children_ = {std::make_shared<Field>("item", value_type)};
   }
 
-  explicit ListType(const std::shared_ptr<Field>& value_field)
-      : DataType(Type::LIST) {
+  explicit ListType(const std::shared_ptr<Field>& value_field) : DataType(Type::LIST) {
     children_ = {value_field};
   }
 
-  const std::shared_ptr<Field>& value_field() const {
-    return children_[0];
-  }
+  const std::shared_ptr<Field>& value_field() const { return children_[0]; }
 
-  const std::shared_ptr<DataType>& value_type() const {
-    return children_[0]->type;
-  }
+  const std::shared_ptr<DataType>& value_type() const { return children_[0]->type; }
 
-  static char const *name() {
-    return "list";
-  }
+  static char const* name() { return "list"; }
 
   std::string ToString() const override;
 };
@@ -286,9 +256,7 @@ struct ListType : public DataType {
 struct StringType : public DataType {
   StringType();
 
-  static char const *name() {
-    return "string";
-  }
+  static char const* name() { return "string"; }
 
   std::string ToString() const override;
 };
@@ -304,10 +272,8 @@ struct StructType : public DataType {
 
 // These will be defined elsewhere
 template <typename T>
-struct type_traits {
-};
-
+struct type_traits {};
 
-} // namespace arrow
+}  // namespace arrow
 
 #endif  // ARROW_TYPE_H
diff --git a/cpp/src/arrow/types/binary.h b/cpp/src/arrow/types/binary.h
index 1fd675e5fdebf..201fbb6e79536 100644
--- a/cpp/src/arrow/types/binary.h
+++ b/cpp/src/arrow/types/binary.h
@@ -23,8 +23,6 @@
 
 #include "arrow/type.h"
 
-namespace arrow {
+namespace arrow {}  // namespace arrow
 
-} // namespace arrow
-
-#endif // ARROW_TYPES_BINARY_H
+#endif  // ARROW_TYPES_BINARY_H
diff --git a/cpp/src/arrow/types/collection.h b/cpp/src/arrow/types/collection.h
index 46d84f1f183c8..1712030203fa2 100644
--- a/cpp/src/arrow/types/collection.h
+++ b/cpp/src/arrow/types/collection.h
@@ -31,15 +31,11 @@ struct CollectionType : public DataType {
 
   CollectionType() : DataType(T) {}
 
-  const TypePtr& child(int i) const {
-    return child_types_[i];
-  }
+  const TypePtr& child(int i) const { return child_types_[i]; }
 
-  int num_children() const {
-    return child_types_.size();
-  }
+  int num_children() const { return child_types_.size(); }
 };
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_TYPES_COLLECTION_H
+#endif  // ARROW_TYPES_COLLECTION_H
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 34647a5005b90..0a30929b97c51 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -30,10 +30,10 @@ namespace arrow {
 
 class ArrayBuilder;
 
-#define BUILDER_CASE(ENUM, BuilderType)         \
-    case Type::ENUM:                            \
-      out->reset(new BuilderType(pool, type));  \
-      return Status::OK();
+#define BUILDER_CASE(ENUM, BuilderType)      \
+  case Type::ENUM:                           \
+    out->reset(new BuilderType(pool, type)); \
+    return Status::OK();
 
 // Initially looked at doing this with vtables, but shared pointers makes it
 // difficult
@@ -58,30 +58,28 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 
     BUILDER_CASE(STRING, StringBuilder);
 
-    case Type::LIST:
-      {
-        std::shared_ptr<ArrayBuilder> value_builder;
+    case Type::LIST: {
+      std::shared_ptr<ArrayBuilder> value_builder;
 
-        const std::shared_ptr<DataType>& value_type = static_cast<ListType*>(
-            type.get())->value_type();
-        RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
-        out->reset(new ListBuilder(pool, type, value_builder));
-        return Status::OK();
-      }
+      const std::shared_ptr<DataType>& value_type =
+          static_cast<ListType*>(type.get())->value_type();
+      RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
+      out->reset(new ListBuilder(pool, type, value_builder));
+      return Status::OK();
+    }
     default:
       return Status::NotImplemented(type->ToString());
   }
 }
 
-#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                      \
-    case Type::ENUM:                                                    \
-      out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
-      return Status::OK();
+#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                          \
+  case Type::ENUM:                                                          \
+    out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
+    return Status::OK();
 
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    int32_t length, const std::shared_ptr<Buffer>& data,
-    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
-    std::shared_ptr<Array>* out) {
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out) {
   switch (type->type) {
     MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
@@ -99,4 +97,4 @@ Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
   }
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index 228faeccc4e4d..27fb7bd2149cf 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -33,11 +33,10 @@ class Status;
 Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     std::shared_ptr<ArrayBuilder>* out);
 
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    int32_t length, const std::shared_ptr<Buffer>& data,
-    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
-    std::shared_ptr<Array>* out);
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_BUILDER_H_
+#endif  // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
index e57b66ab46adb..b782455546c33 100644
--- a/cpp/src/arrow/types/datetime.h
+++ b/cpp/src/arrow/types/datetime.h
@@ -23,49 +23,30 @@
 namespace arrow {
 
 struct DateType : public DataType {
-  enum class Unit: char {
-    DAY = 0,
-    MONTH = 1,
-    YEAR = 2
-  };
+  enum class Unit : char { DAY = 0, MONTH = 1, YEAR = 2 };
 
   Unit unit;
 
-  explicit DateType(Unit unit = Unit::DAY)
-      : DataType(Type::DATE),
-        unit(unit) {}
+  explicit DateType(Unit unit = Unit::DAY) : DataType(Type::DATE), unit(unit) {}
 
-  DateType(const DateType& other)
-      : DateType(other.unit) {}
+  DateType(const DateType& other) : DateType(other.unit) {}
 
-  static char const *name() {
-    return "date";
-  }
+  static char const* name() { return "date"; }
 };
 
-
 struct TimestampType : public DataType {
-  enum class Unit: char {
-    SECOND = 0,
-    MILLI = 1,
-    MICRO = 2,
-    NANO = 3
-  };
+  enum class Unit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
 
   Unit unit;
 
   explicit TimestampType(Unit unit = Unit::MILLI)
-      : DataType(Type::TIMESTAMP),
-        unit(unit) {}
+      : DataType(Type::TIMESTAMP), unit(unit) {}
 
-  TimestampType(const TimestampType& other)
-      : TimestampType(other.unit) {}
+  TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
 
-  static char const *name() {
-    return "timestamp";
-  }
+  static char const* name() { return "timestamp"; }
 };
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_TYPES_DATETIME_H
+#endif  // ARROW_TYPES_DATETIME_H
diff --git a/cpp/src/arrow/types/decimal-test.cc b/cpp/src/arrow/types/decimal-test.cc
index 89896c8b425d0..7296ff8176113 100644
--- a/cpp/src/arrow/types/decimal-test.cc
+++ b/cpp/src/arrow/types/decimal-test.cc
@@ -37,4 +37,4 @@ TEST(TypesTest, TestDecimalType) {
   ASSERT_EQ(t2.scale, 4);
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/decimal.cc b/cpp/src/arrow/types/decimal.cc
index f120c1a9dfde6..1d9a5e50e460b 100644
--- a/cpp/src/arrow/types/decimal.cc
+++ b/cpp/src/arrow/types/decimal.cc
@@ -28,5 +28,4 @@ std::string DecimalType::ToString() const {
   return s.str();
 }
 
-} // namespace arrow
-
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h
index 26243b42b0e7d..1be489d4f51b6 100644
--- a/cpp/src/arrow/types/decimal.h
+++ b/cpp/src/arrow/types/decimal.h
@@ -26,18 +26,15 @@ namespace arrow {
 
 struct DecimalType : public DataType {
   explicit DecimalType(int precision_, int scale_)
-      : DataType(Type::DECIMAL), precision(precision_),
-        scale(scale_) { }
+      : DataType(Type::DECIMAL), precision(precision_), scale(scale_) {}
   int precision;
   int scale;
 
-  static char const *name() {
-    return "decimal";
-  }
+  static char const* name() { return "decimal"; }
 
   std::string ToString() const override;
 };
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_TYPES_DECIMAL_H
+#endif  // ARROW_TYPES_DECIMAL_H
diff --git a/cpp/src/arrow/types/json.cc b/cpp/src/arrow/types/json.cc
index fb731edd6073f..a4e0d085620a0 100644
--- a/cpp/src/arrow/types/json.cc
+++ b/cpp/src/arrow/types/json.cc
@@ -30,9 +30,8 @@ static const TypePtr String(new StringType());
 static const TypePtr Double(new DoubleType());
 static const TypePtr Bool(new BooleanType());
 
-static const std::vector<TypePtr> json_types = {Null, Int32, String,
-                                                Double, Bool};
+static const std::vector<TypePtr> json_types = {Null, Int32, String, Double, Bool};
 TypePtr JSONScalar::dense_type = TypePtr(new DenseUnionType(json_types));
 TypePtr JSONScalar::sparse_type = TypePtr(new SparseUnionType(json_types));
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/json.h b/cpp/src/arrow/types/json.h
index 9c850afac0af4..9de961f79a60a 100644
--- a/cpp/src/arrow/types/json.h
+++ b/cpp/src/arrow/types/json.h
@@ -28,11 +28,9 @@ struct JSONScalar : public DataType {
   static TypePtr dense_type;
   static TypePtr sparse_type;
 
-  explicit JSONScalar(bool dense = true)
-      : DataType(Type::JSON_SCALAR),
-        dense(dense) {}
+  explicit JSONScalar(bool dense = true) : DataType(Type::JSON_SCALAR), dense(dense) {}
 };
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_TYPES_JSON_H
+#endif  // ARROW_TYPES_JSON_H
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index 4eb560ea52256..aa34f23cc0230 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -76,9 +76,7 @@ class TestListBuilder : public TestBuilder {
     builder_ = std::dynamic_pointer_cast<ListBuilder>(tmp);
   }
 
-  void Done() {
-    result_ = std::dynamic_pointer_cast<ListArray>(builder_->Finish());
-  }
+  void Done() { result_ = std::dynamic_pointer_cast<ListArray>(builder_->Finish()); }
 
  protected:
   TypePtr value_type_;
@@ -88,9 +86,7 @@ class TestListBuilder : public TestBuilder {
   shared_ptr<ListArray> result_;
 };
 
-
-TEST_F(TestListBuilder, TestResize) {
-}
+TEST_F(TestListBuilder, TestResize) {}
 
 TEST_F(TestListBuilder, TestAppendNull) {
   ASSERT_OK(builder_->AppendNull());
@@ -155,5 +151,4 @@ TEST_F(TestListBuilder, TestZeroLength) {
   Done();
 }
 
-
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index d64c06d90c174..23f12ddc4ecd7 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -20,32 +20,25 @@
 namespace arrow {
 
 bool ListArray::EqualsExact(const ListArray& other) const {
-  if (this == &other) return true;
-  if (null_count_ != other.null_count_) {
-    return false;
-  }
+  if (this == &other) { return true; }
+  if (null_count_ != other.null_count_) { return false; }
 
-  bool equal_offsets = offset_buf_->Equals(*other.offset_buf_,
-      length_ + 1);
+  bool equal_offsets = offset_buf_->Equals(*other.offset_buf_, length_ + 1);
   bool equal_null_bitmap = true;
   if (null_count_ > 0) {
-    equal_null_bitmap = null_bitmap_->Equals(*other.null_bitmap_,
-        util::bytes_for_bits(length_));
+    equal_null_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, util::bytes_for_bits(length_));
   }
 
-  if (!(equal_offsets && equal_null_bitmap)) {
-    return false;
-  }
+  if (!(equal_offsets && equal_null_bitmap)) { return false; }
 
   return values()->Equals(other.values());
 }
 
 bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) return true;
-  if (this->type_enum() != arr->type_enum()) {
-    return false;
-  }
+  if (this == arr.get()) { return true; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
   return EqualsExact(*static_cast<const ListArray*>(arr.get()));
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 8073b5121764d..6b815460ecb1e 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -37,13 +37,12 @@ class MemoryPool;
 class ListArray : public Array {
  public:
   ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
-      const ArrayPtr& values,
-      int32_t null_count = 0,
-      std::shared_ptr<Buffer> null_bitmap = nullptr) :
-      Array(type, length, null_count, null_bitmap) {
+      const ArrayPtr& values, int32_t null_count = 0,
+      std::shared_ptr<Buffer> null_bitmap = nullptr)
+      : Array(type, length, null_count, null_bitmap) {
     offset_buf_ = offsets;
-    offsets_ = offsets == nullptr? nullptr :
-      reinterpret_cast<const int32_t*>(offset_buf_->data());
+    offsets_ = offsets == nullptr ? nullptr
+                                  : reinterpret_cast<const int32_t*>(offset_buf_->data());
     values_ = values;
   }
 
@@ -51,19 +50,17 @@ class ListArray : public Array {
 
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
-  const std::shared_ptr<Array>& values() const {return values_;}
+  const std::shared_ptr<Array>& values() const { return values_; }
 
-  const std::shared_ptr<DataType>& value_type() const {
-    return values_->type();
-  }
+  const std::shared_ptr<DataType>& value_type() const { return values_->type(); }
 
-  const int32_t* offsets() const { return offsets_;}
+  const int32_t* offsets() const { return offsets_; }
 
-  int32_t offset(int i) const { return offsets_[i];}
+  int32_t offset(int i) const { return offsets_[i]; }
 
   // Neither of these functions will perform boundschecking
-  int32_t value_offset(int i) { return offsets_[i];}
-  int32_t value_length(int i) { return offsets_[i + 1] - offsets_[i];}
+  int32_t value_offset(int i) { return offsets_[i]; }
+  int32_t value_length(int i) { return offsets_[i + 1] - offsets_[i]; }
 
   bool EqualsExact(const ListArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
@@ -77,7 +74,6 @@ class ListArray : public Array {
 // ----------------------------------------------------------------------
 // Array builder
 
-
 // Builder class for variable-length list array value types
 //
 // To use this class, you must append values to the child array builder and use
@@ -85,10 +81,9 @@ class ListArray : public Array {
 // have been appended to the child array)
 class ListBuilder : public Int32Builder {
  public:
-  ListBuilder(MemoryPool* pool, const TypePtr& type,
-      std::shared_ptr<ArrayBuilder> value_builder)
-      : Int32Builder(pool, type),
-        value_builder_(value_builder) {}
+  ListBuilder(
+      MemoryPool* pool, const TypePtr& type, std::shared_ptr<ArrayBuilder> value_builder)
+      : Int32Builder(pool, type), value_builder_(value_builder) {}
 
   Status Init(int32_t elements) {
     // One more than requested.
@@ -116,12 +111,9 @@ class ListBuilder : public Int32Builder {
       int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
     }
-    memcpy(raw_data_ + length_, values,
-        type_traits<Int32Type>::bytes_required(length));
+    memcpy(raw_data_ + length_, values, type_traits<Int32Type>::bytes_required(length));
 
-    if (valid_bytes != nullptr) {
-      AppendNulls(valid_bytes, length);
-    }
+    if (valid_bytes != nullptr) { AppendNulls(valid_bytes, length); }
 
     length_ += length;
     return Status::OK();
@@ -132,12 +124,10 @@ class ListBuilder : public Int32Builder {
     std::shared_ptr<Array> items = value_builder_->Finish();
 
     // Add final offset if the length is non-zero
-    if (length_) {
-      raw_data_[length_] = items->length();
-    }
+    if (length_) { raw_data_[length_] = items->length(); }
 
-    auto result = std::make_shared<Container>(type_, length_, data_, items,
-        null_count_, null_bitmap_);
+    auto result = std::make_shared<Container>(
+        type_, length_, data_, items, null_count_, null_bitmap_);
 
     data_ = null_bitmap_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
@@ -145,9 +135,7 @@ class ListBuilder : public Int32Builder {
     return result;
   }
 
-  std::shared_ptr<Array> Finish() override {
-    return Transfer<ListArray>();
-  }
+  std::shared_ptr<Array> Finish() override { return Transfer<ListArray>(); }
 
   // Start a new variable-length list slot
   //
@@ -167,19 +155,14 @@ class ListBuilder : public Int32Builder {
     return Status::OK();
   }
 
-  Status AppendNull() {
-    return Append(true);
-  }
+  Status AppendNull() { return Append(true); }
 
-  const std::shared_ptr<ArrayBuilder>& value_builder() const {
-    return value_builder_;
-  }
+  const std::shared_ptr<ArrayBuilder>& value_builder() const { return value_builder_; }
 
  protected:
   std::shared_ptr<ArrayBuilder> value_builder_;
 };
 
+}  // namespace arrow
 
-} // namespace arrow
-
-#endif // ARROW_TYPES_LIST_H
+#endif  // ARROW_TYPES_LIST_H
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 761845d93812a..6bd9e73eb46ac 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -41,15 +41,15 @@ namespace arrow {
 
 class Array;
 
-#define PRIMITIVE_TEST(KLASS, ENUM, NAME)       \
-  TEST(TypesTest, TestPrimitive_##ENUM) {       \
-    KLASS tp;                                   \
-                                                \
-    ASSERT_EQ(tp.type, Type::ENUM);             \
-    ASSERT_EQ(tp.name(), string(NAME));         \
-                                                \
-    KLASS tp_copy = tp;                         \
-    ASSERT_EQ(tp_copy.type, Type::ENUM);        \
+#define PRIMITIVE_TEST(KLASS, ENUM, NAME) \
+  TEST(TypesTest, TestPrimitive_##ENUM) { \
+    KLASS tp;                             \
+                                          \
+    ASSERT_EQ(tp.type, Type::ENUM);       \
+    ASSERT_EQ(tp.name(), string(NAME));   \
+                                          \
+    KLASS tp_copy = tp;                   \
+    ASSERT_EQ(tp_copy.type, Type::ENUM);  \
   }
 
 PRIMITIVE_TEST(Int8Type, INT8, "int8");
@@ -108,8 +108,8 @@ class TestPrimitiveBuilder : public TestBuilder {
   void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
     int size = builder->length();
 
-    auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
-        size * sizeof(T));
+    auto ex_data = std::make_shared<Buffer>(
+        reinterpret_cast<uint8_t*>(draws_.data()), size * sizeof(T));
 
     std::shared_ptr<Buffer> ex_null_bitmap;
     int32_t ex_null_count = 0;
@@ -121,10 +121,10 @@ class TestPrimitiveBuilder : public TestBuilder {
       ex_null_bitmap = nullptr;
     }
 
-    auto expected = std::make_shared<ArrayType>(size, ex_data, ex_null_count,
-        ex_null_bitmap);
-    std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(
-        builder->Finish());
+    auto expected =
+        std::make_shared<ArrayType>(size, ex_data, ex_null_count, ex_null_bitmap);
+    std::shared_ptr<ArrayType> result =
+        std::dynamic_pointer_cast<ArrayType>(builder->Finish());
 
     // Builder is now reset
     ASSERT_EQ(0, builder->length());
@@ -145,30 +145,30 @@ class TestPrimitiveBuilder : public TestBuilder {
   vector<uint8_t> valid_bytes_;
 };
 
-#define PTYPE_DECL(CapType, c_type)                 \
-  typedef CapType##Array ArrayType;                 \
-  typedef CapType##Builder BuilderType;             \
-  typedef CapType##Type Type;                       \
-  typedef c_type T;                                 \
-                                                    \
-  static std::shared_ptr<DataType> type() {         \
-    return std::shared_ptr<DataType>(new Type());   \
+#define PTYPE_DECL(CapType, c_type)               \
+  typedef CapType##Array ArrayType;               \
+  typedef CapType##Builder BuilderType;           \
+  typedef CapType##Type Type;                     \
+  typedef c_type T;                               \
+                                                  \
+  static std::shared_ptr<DataType> type() {       \
+    return std::shared_ptr<DataType>(new Type()); \
   }
 
-#define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
-  struct P##CapType {                               \
-    PTYPE_DECL(CapType, c_type);                    \
-    static void draw(int N, vector<T>* draws) {     \
-      test::randint<T>(N, LOWER, UPPER, draws);     \
-    }                                               \
+#define PINT_DECL(CapType, c_type, LOWER, UPPER) \
+  struct P##CapType {                            \
+    PTYPE_DECL(CapType, c_type);                 \
+    static void draw(int N, vector<T>* draws) {  \
+      test::randint<T>(N, LOWER, UPPER, draws);  \
+    }                                            \
   }
 
-#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)      \
-  struct P##CapType {                                   \
-    PTYPE_DECL(CapType, c_type);                        \
-    static void draw(int N, vector<T>* draws) {         \
-      test::random_real<T>(N, 0, LOWER, UPPER, draws);  \
-    }                                                   \
+#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)     \
+  struct P##CapType {                                  \
+    PTYPE_DECL(CapType, c_type);                       \
+    static void draw(int N, vector<T>* draws) {        \
+      test::random_real<T>(N, 0, LOWER, UPPER, draws); \
+    }                                                  \
   }
 
 PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX);
@@ -214,10 +214,10 @@ void TestPrimitiveBuilder<PBoolean>::Check(
     ex_null_bitmap = nullptr;
   }
 
-  auto expected = std::make_shared<BooleanArray>(size, ex_data, ex_null_count,
-      ex_null_bitmap);
-  std::shared_ptr<BooleanArray> result = std::dynamic_pointer_cast<BooleanArray>(
-      builder->Finish());
+  auto expected =
+      std::make_shared<BooleanArray>(size, ex_data, ex_null_count, ex_null_bitmap);
+  std::shared_ptr<BooleanArray> result =
+      std::dynamic_pointer_cast<BooleanArray>(builder->Finish());
 
   // Builder is now reset
   ASSERT_EQ(0, builder->length());
@@ -230,31 +230,23 @@ void TestPrimitiveBuilder<PBoolean>::Check(
   ASSERT_EQ(expected->length(), result->length());
 
   for (int i = 0; i < result->length(); ++i) {
-    if (nullable) {
-      ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i;
-    }
+    if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
     bool actual = util::get_bit(result->raw_data(), i);
     ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
   }
   ASSERT_TRUE(result->EqualsExact(*expected.get()));
 }
 
-typedef ::testing::Types<PBoolean,
-                         PUInt8, PUInt16, PUInt32, PUInt64,
-                         PInt8, PInt16, PInt32, PInt64,
-                         PFloat, PDouble> Primitives;
+typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
+    PInt32, PInt64, PFloat, PDouble> Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
-#define DECL_T()                                \
-  typedef typename TestFixture::T T;
+#define DECL_T() typedef typename TestFixture::T T;
 
-#define DECL_TYPE()                             \
-  typedef typename TestFixture::Type Type;
-
-#define DECL_ARRAYTYPE()                                \
-  typedef typename TestFixture::ArrayType ArrayType;
+#define DECL_TYPE() typedef typename TestFixture::Type Type;
 
+#define DECL_ARRAYTYPE() typedef typename TestFixture::ArrayType ArrayType;
 
 TYPED_TEST(TestPrimitiveBuilder, TestInit) {
   DECL_TYPE();
@@ -369,7 +361,6 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   this->Check(this->builder_nn_, false);
 }
 
-
 TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
   DECL_T();
 
@@ -424,8 +415,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestResize) {
   ASSERT_EQ(cap, this->builder_->capacity());
 
   ASSERT_EQ(type_traits<Type>::bytes_required(cap), this->builder_->data()->size());
-  ASSERT_EQ(util::bytes_for_bits(cap),
-      this->builder_->null_bitmap()->size());
+  ASSERT_EQ(util::bytes_for_bits(cap), this->builder_->null_bitmap()->size());
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
@@ -437,8 +427,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
   ASSERT_OK(this->builder_->Advance(100));
   ASSERT_OK(this->builder_->Reserve(MIN_BUILDER_CAPACITY));
 
-  ASSERT_EQ(util::next_power2(MIN_BUILDER_CAPACITY + 100),
-      this->builder_->capacity());
+  ASSERT_EQ(util::next_power2(MIN_BUILDER_CAPACITY + 100), this->builder_->capacity());
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index c54d0757c4789..9549c47b41157 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -28,26 +28,21 @@ namespace arrow {
 // Primitive array base
 
 PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
-    const std::shared_ptr<Buffer>& data,
-    int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap) :
-    Array(type, length, null_count, null_bitmap) {
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : Array(type, length, null_count, null_bitmap) {
   data_ = data;
-  raw_data_ = data == nullptr? nullptr : data_->data();
+  raw_data_ = data == nullptr ? nullptr : data_->data();
 }
 
 bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
-  if (this == &other) return true;
-  if (null_count_ != other.null_count_) {
-    return false;
-  }
+  if (this == &other) { return true; }
+  if (null_count_ != other.null_count_) { return false; }
 
   if (null_count_ > 0) {
-    bool equal_bitmap = null_bitmap_->Equals(*other.null_bitmap_,
-        util::ceil_byte(length_) / 8);
-    if (!equal_bitmap) {
-      return false;
-    }
+    bool equal_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, util::ceil_byte(length_) / 8);
+    if (!equal_bitmap) { return false; }
 
     const uint8_t* this_data = raw_data_;
     const uint8_t* other_data = other.raw_data_;
@@ -56,9 +51,7 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
     DCHECK_GT(value_size, 0);
 
     for (int i = 0; i < length_; ++i) {
-      if (!IsNull(i) && memcmp(this_data, other_data, value_size)) {
-        return false;
-      }
+      if (!IsNull(i) && memcmp(this_data, other_data, value_size)) { return false; }
       this_data += value_size;
       other_data += value_size;
     }
@@ -69,10 +62,8 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
 }
 
 bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) return true;
-  if (this->type_enum() != arr->type_enum()) {
-    return false;
-  }
+  if (this == arr.get()) { return true; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
   return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
 }
 
@@ -92,9 +83,7 @@ Status PrimitiveBuilder<T>::Init(int32_t capacity) {
 template <typename T>
 Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
   // XXX: Set floor size for now
-  if (capacity < MIN_BUILDER_CAPACITY) {
-    capacity = MIN_BUILDER_CAPACITY;
-  }
+  if (capacity < MIN_BUILDER_CAPACITY) { capacity = MIN_BUILDER_CAPACITY; }
 
   if (capacity_ == 0) {
     RETURN_NOT_OK(Init(capacity));
@@ -122,8 +111,8 @@ Status PrimitiveBuilder<T>::Reserve(int32_t elements) {
 }
 
 template <typename T>
-Status PrimitiveBuilder<T>::Append(const value_type* values, int32_t length,
-    const uint8_t* valid_bytes) {
+Status PrimitiveBuilder<T>::Append(
+    const value_type* values, int32_t length, const uint8_t* valid_bytes) {
   RETURN_NOT_OK(PrimitiveBuilder<T>::Reserve(length));
 
   if (length > 0) {
@@ -156,9 +145,8 @@ void PrimitiveBuilder<T>::AppendNulls(const uint8_t* valid_bytes, int32_t length
 
 template <typename T>
 std::shared_ptr<Array> PrimitiveBuilder<T>::Finish() {
-  std::shared_ptr<Array> result = std::make_shared<
-    typename type_traits<T>::ArrayType>(
-        type_, length_, data_, null_count_, null_bitmap_);
+  std::shared_ptr<Array> result = std::make_shared<typename type_traits<T>::ArrayType>(
+      type_, length_, data_, null_count_, null_bitmap_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -166,8 +154,8 @@ std::shared_ptr<Array> PrimitiveBuilder<T>::Finish() {
 }
 
 template <>
-Status PrimitiveBuilder<BooleanType>::Append(const uint8_t* values, int32_t length,
-    const uint8_t* valid_bytes) {
+Status PrimitiveBuilder<BooleanType>::Append(
+    const uint8_t* values, int32_t length, const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
 
   for (int i = 0; i < length; ++i) {
@@ -202,23 +190,18 @@ template class PrimitiveBuilder<DoubleType>;
 template class PrimitiveBuilder<BooleanType>;
 
 BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
-    int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap) :
-    PrimitiveArray(std::make_shared<BooleanType>(), length,
-        data, null_count, null_bitmap) {}
+    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
+    : PrimitiveArray(
+          std::make_shared<BooleanType>(), length, data, null_count, null_bitmap) {}
 
 bool BooleanArray::EqualsExact(const BooleanArray& other) const {
   if (this == &other) return true;
-  if (null_count_ != other.null_count_) {
-    return false;
-  }
+  if (null_count_ != other.null_count_) { return false; }
 
   if (null_count_ > 0) {
-    bool equal_bitmap = null_bitmap_->Equals(*other.null_bitmap_,
-        util::bytes_for_bits(length_));
-    if (!equal_bitmap) {
-      return false;
-    }
+    bool equal_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, util::bytes_for_bits(length_));
+    if (!equal_bitmap) { return false; }
 
     const uint8_t* this_data = raw_data_;
     const uint8_t* other_data = other.raw_data_;
@@ -236,10 +219,8 @@ bool BooleanArray::EqualsExact(const BooleanArray& other) const {
 
 bool BooleanArray::Equals(const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) return true;
-  if (Type::BOOL != arr->type_enum()) {
-    return false;
-  }
+  if (Type::BOOL != arr->type_enum()) { return false; }
   return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index ec6fee35513ce..fcd3db4e96e53 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -34,17 +34,14 @@ namespace arrow {
 
 class MemoryPool;
 
-
 // Base class for fixed-size logical types
 class PrimitiveArray : public Array {
  public:
-  PrimitiveArray(const TypePtr& type, int32_t length,
-      const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  PrimitiveArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
   virtual ~PrimitiveArray() {}
 
-  const std::shared_ptr<Buffer>& data() const { return data_;}
+  const std::shared_ptr<Buffer>& data() const { return data_; }
 
   bool EqualsExact(const PrimitiveArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
@@ -54,31 +51,25 @@ class PrimitiveArray : public Array {
   const uint8_t* raw_data_;
 };
 
-#define NUMERIC_ARRAY_DECL(NAME, TypeClass, T)                  \
-class NAME : public PrimitiveArray {                            \
- public:                                                        \
-  using value_type = T;                                         \
-  using PrimitiveArray::PrimitiveArray;                         \
-                                                                \
-  NAME(int32_t length, const std::shared_ptr<Buffer>& data,     \
-      int32_t null_count = 0,                                   \
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr) :   \
-      PrimitiveArray(std::make_shared<TypeClass>(), length,     \
-          data, null_count, null_bitmap) {}                     \
-                                                                \
-  bool EqualsExact(const NAME& other) const {                   \
-    return PrimitiveArray::EqualsExact(                         \
-        *static_cast<const PrimitiveArray*>(&other));           \
-  }                                                             \
-                                                                \
-  const T* raw_data() const {                                   \
-    return reinterpret_cast<const T*>(raw_data_);               \
-  }                                                             \
-                                                                \
-  T Value(int i) const {                                        \
-    return raw_data()[i];                                       \
-  }                                                             \
-};
+#define NUMERIC_ARRAY_DECL(NAME, TypeClass, T)                                         \
+  class NAME : public PrimitiveArray {                                                 \
+   public:                                                                             \
+    using value_type = T;                                                              \
+    using PrimitiveArray::PrimitiveArray;                                              \
+                                                                                       \
+    NAME(int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count = 0,  \
+        const std::shared_ptr<Buffer>& null_bitmap = nullptr)                          \
+        : PrimitiveArray(                                                              \
+              std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {} \
+                                                                                       \
+    bool EqualsExact(const NAME& other) const {                                        \
+      return PrimitiveArray::EqualsExact(*static_cast<const PrimitiveArray*>(&other)); \
+    }                                                                                  \
+                                                                                       \
+    const T* raw_data() const { return reinterpret_cast<const T*>(raw_data_); }        \
+                                                                                       \
+    T Value(int i) const { return raw_data()[i]; }                                     \
+  };
 
 NUMERIC_ARRAY_DECL(UInt8Array, UInt8Type, uint8_t);
 NUMERIC_ARRAY_DECL(Int8Array, Int8Type, int8_t);
@@ -96,9 +87,8 @@ class PrimitiveBuilder : public ArrayBuilder {
  public:
   typedef typename Type::c_type value_type;
 
-  explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type) :
-      ArrayBuilder(pool, type),
-      data_(nullptr) {}
+  explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type)
+      : ArrayBuilder(pool, type), data_(nullptr) {}
 
   virtual ~PrimitiveBuilder() {}
 
@@ -117,16 +107,14 @@ class PrimitiveBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  std::shared_ptr<Buffer> data() const {
-    return data_;
-  }
+  std::shared_ptr<Buffer> data() const { return data_; }
 
   // Vector append
   //
   // If passed, valid_bytes is of equal length to values, and any zero byte
   // will be considered as a null for that slot
-  Status Append(const value_type* values, int32_t length,
-      const uint8_t* valid_bytes = nullptr);
+  Status Append(
+      const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
 
   // Ensure that builder can accommodate an additional number of
   // elements. Resizes if the current capacity is not sufficient
@@ -172,89 +160,69 @@ template <>
 struct type_traits<UInt8Type> {
   typedef UInt8Array ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements;
-  }
+  static inline int bytes_required(int elements) { return elements; }
 };
 
 template <>
 struct type_traits<Int8Type> {
   typedef Int8Array ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements;
-  }
+  static inline int bytes_required(int elements) { return elements; }
 };
 
 template <>
 struct type_traits<UInt16Type> {
   typedef UInt16Array ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements * sizeof(uint16_t);
-  }
+  static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
 };
 
 template <>
 struct type_traits<Int16Type> {
   typedef Int16Array ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements * sizeof(int16_t);
-  }
+  static inline int bytes_required(int elements) { return elements * sizeof(int16_t); }
 };
 
 template <>
 struct type_traits<UInt32Type> {
   typedef UInt32Array ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements * sizeof(uint32_t);
-  }
+  static inline int bytes_required(int elements) { return elements * sizeof(uint32_t); }
 };
 
 template <>
 struct type_traits<Int32Type> {
   typedef Int32Array ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements * sizeof(int32_t);
-  }
+  static inline int bytes_required(int elements) { return elements * sizeof(int32_t); }
 };
 
 template <>
 struct type_traits<UInt64Type> {
   typedef UInt64Array ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements * sizeof(uint64_t);
-  }
+  static inline int bytes_required(int elements) { return elements * sizeof(uint64_t); }
 };
 
 template <>
 struct type_traits<Int64Type> {
   typedef Int64Array ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements * sizeof(int64_t);
-  }
+  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
 };
 template <>
 struct type_traits<FloatType> {
   typedef FloatArray ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements * sizeof(float);
-  }
+  static inline int bytes_required(int elements) { return elements * sizeof(float); }
 };
 
 template <>
 struct type_traits<DoubleType> {
   typedef DoubleArray ArrayType;
 
-  static inline int bytes_required(int elements) {
-    return elements * sizeof(double);
-  }
+  static inline int bytes_required(int elements) { return elements * sizeof(double); }
 };
 
 // Builders
@@ -272,25 +240,19 @@ typedef NumericBuilder<Int64Type> Int64Builder;
 typedef NumericBuilder<FloatType> FloatBuilder;
 typedef NumericBuilder<DoubleType> DoubleBuilder;
 
-
 class BooleanArray : public PrimitiveArray {
  public:
   using PrimitiveArray::PrimitiveArray;
 
   BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   bool EqualsExact(const BooleanArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
 
-  const uint8_t* raw_data() const {
-    return reinterpret_cast<const uint8_t*>(raw_data_);
-  }
+  const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
 
-  bool Value(int i) const {
-    return util::get_bit(raw_data(), i);
-  }
+  bool Value(int i) const { return util::get_bit(raw_data(), i); }
 };
 
 template <>
@@ -304,8 +266,8 @@ struct type_traits<BooleanType> {
 
 class BooleanBuilder : public PrimitiveBuilder<BooleanType> {
  public:
-  explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type) :
-      PrimitiveBuilder<BooleanType>(pool, type) {}
+  explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type)
+      : PrimitiveBuilder<BooleanType>(pool, type) {}
 
   virtual ~BooleanBuilder() {}
 
@@ -322,11 +284,9 @@ class BooleanBuilder : public PrimitiveBuilder<BooleanType> {
     ++length_;
   }
 
-  void Append(uint8_t val) {
-    Append(static_cast<bool>(val));
-  }
+  void Append(uint8_t val) { Append(static_cast<bool>(val)); }
 };
 
-} // namespace arrow
+}  // namespace arrow
 
 #endif  // ARROW_TYPES_PRIMITIVE_H
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index d3a4cc37f9c4c..ee4307c4d168a 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -48,7 +48,6 @@ TEST(TypesTest, TestCharType) {
   ASSERT_EQ(t2.size, 5);
 }
 
-
 TEST(TypesTest, TestVarcharType) {
   VarcharType t1(5);
 
@@ -72,7 +71,7 @@ TEST(TypesTest, TestStringType) {
 // ----------------------------------------------------------------------
 // String container
 
-class TestStringContainer : public ::testing::Test  {
+class TestStringContainer : public ::testing::Test {
  public:
   void SetUp() {
     chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
@@ -95,8 +94,8 @@ class TestStringContainer : public ::testing::Test  {
     null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
     null_count_ = test::null_count(valid_bytes_);
 
-    strings_ = std::make_shared<StringArray>(length_, offsets_buf_, values_,
-        null_count_, null_bitmap_);
+    strings_ = std::make_shared<StringArray>(
+        length_, offsets_buf_, values_, null_count_, null_bitmap_);
   }
 
  protected:
@@ -117,7 +116,6 @@ class TestStringContainer : public ::testing::Test  {
   std::shared_ptr<StringArray> strings_;
 };
 
-
 TEST_F(TestStringContainer, TestArrayBasics) {
   ASSERT_EQ(length_, strings_->length());
   ASSERT_EQ(1, strings_->null_count());
@@ -130,7 +128,6 @@ TEST_F(TestStringContainer, TestType) {
   ASSERT_EQ(Type::STRING, strings_->type_enum());
 }
 
-
 TEST_F(TestStringContainer, TestListFunctions) {
   int pos = 0;
   for (size_t i = 0; i < expected_.size(); ++i) {
@@ -140,10 +137,9 @@ TEST_F(TestStringContainer, TestListFunctions) {
   }
 }
 
-
 TEST_F(TestStringContainer, TestDestructor) {
-  auto arr = std::make_shared<StringArray>(length_, offsets_buf_, values_,
-      null_count_, null_bitmap_);
+  auto arr = std::make_shared<StringArray>(
+      length_, offsets_buf_, values_, null_count_, null_bitmap_);
 }
 
 TEST_F(TestStringContainer, TestGetString) {
@@ -167,9 +163,7 @@ class TestStringBuilder : public TestBuilder {
     builder_.reset(new StringBuilder(pool_, type_));
   }
 
-  void Done() {
-    result_ = std::dynamic_pointer_cast<StringArray>(builder_->Finish());
-  }
+  void Done() { result_ = std::dynamic_pointer_cast<StringArray>(builder_->Finish()); }
 
  protected:
   TypePtr type_;
@@ -222,4 +216,4 @@ TEST_F(TestStringBuilder, TestZeroLength) {
   Done();
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index 80b075cdfbb23..29d97d039477c 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -26,11 +26,10 @@ namespace arrow {
 
 const std::shared_ptr<DataType> STRING(new StringType());
 
-StringArray::StringArray(int32_t length,
-    const std::shared_ptr<Buffer>& offsets,
+StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
     const ArrayPtr& values, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap) :
-    StringArray(STRING, length, offsets, values, null_count, null_bitmap) {}
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : StringArray(STRING, length, offsets, values, null_count, null_bitmap) {}
 
 std::string CharType::ToString() const {
   std::stringstream s;
@@ -38,7 +37,6 @@ std::string CharType::ToString() const {
   return s.str();
 }
 
-
 std::string VarcharType::ToString() const {
   std::stringstream s;
   s << "varchar(" << size << ")";
@@ -47,4 +45,4 @@ std::string VarcharType::ToString() const {
 
 TypePtr StringBuilder::value_type_ = TypePtr(new UInt8Type());
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index 84cd0326ec850..c5cbe1058c7cf 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -37,48 +37,37 @@ class MemoryPool;
 struct CharType : public DataType {
   int size;
 
-  explicit CharType(int size)
-      : DataType(Type::CHAR),
-        size(size) {}
+  explicit CharType(int size) : DataType(Type::CHAR), size(size) {}
 
-  CharType(const CharType& other)
-      : CharType(other.size) {}
+  CharType(const CharType& other) : CharType(other.size) {}
 
   virtual std::string ToString() const;
 };
 
-
 // Variable-length, null-terminated strings, up to a certain length
 struct VarcharType : public DataType {
   int size;
 
-  explicit VarcharType(int size)
-      : DataType(Type::VARCHAR),
-        size(size) {}
-  VarcharType(const VarcharType& other)
-      : VarcharType(other.size) {}
+  explicit VarcharType(int size) : DataType(Type::VARCHAR), size(size) {}
+  VarcharType(const VarcharType& other) : VarcharType(other.size) {}
 
   virtual std::string ToString() const;
 };
 
-// TODO: add a BinaryArray layer in between
+// TODO(wesm): add a BinaryArray layer in between
 class StringArray : public ListArray {
  public:
-  StringArray(const TypePtr& type, int32_t length,
-      const std::shared_ptr<Buffer>& offsets,
-      const ArrayPtr& values,
-      int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr) :
-      ListArray(type, length, offsets, values, null_count, null_bitmap) {
+  StringArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const ArrayPtr& values, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+      : ListArray(type, length, offsets, values, null_count, null_bitmap) {
     // For convenience
     bytes_ = static_cast<UInt8Array*>(values.get());
     raw_bytes_ = bytes_->raw_data();
   }
 
-  StringArray(int32_t length,
-      const std::shared_ptr<Buffer>& offsets,
-      const ArrayPtr& values,
-      int32_t null_count = 0,
+  StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const ArrayPtr& values, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   // Compute the pointer t
@@ -103,21 +92,18 @@ class StringArray : public ListArray {
 // Array builder
 class StringBuilder : public ListBuilder {
  public:
-  explicit StringBuilder(MemoryPool* pool, const TypePtr& type) :
-      ListBuilder(pool, type, std::make_shared<UInt8Builder>(pool, value_type_)) {
+  explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
+      : ListBuilder(pool, type, std::make_shared<UInt8Builder>(pool, value_type_)) {
     byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
   }
 
-  Status Append(const std::string& value) {
-    return Append(value.c_str(), value.size());
-  }
+  Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
 
   Status Append(const char* value, int32_t length) {
     RETURN_NOT_OK(ListBuilder::Append());
     return byte_builder_->Append(reinterpret_cast<const uint8_t*>(value), length);
   }
-  Status Append(const std::vector<std::string>& values,
-                uint8_t* null_bytes);
+  Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
 
   std::shared_ptr<Array> Finish() override {
     return ListBuilder::Transfer<StringArray>();
@@ -130,6 +116,6 @@ class StringBuilder : public ListBuilder {
   static TypePtr value_type_;
 };
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_TYPES_STRING_H
+#endif  // ARROW_TYPES_STRING_H
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index d94396f42c52a..79d560e19bcc0 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -49,7 +49,7 @@ TEST(TestStructType, Basics) {
 
   ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
 
-  // TODO: out of bounds for field(...)
+  // TODO(wesm): out of bounds for field(...)
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
index 02af600b017d0..04a277a86fa58 100644
--- a/cpp/src/arrow/types/struct.cc
+++ b/cpp/src/arrow/types/struct.cc
@@ -17,6 +17,4 @@
 
 #include "arrow/types/struct.h"
 
-namespace arrow {
-
-} // namespace arrow
+namespace arrow {}  // namespace arrow
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index 5842534d35be1..17e32993bf975 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -24,8 +24,6 @@
 
 #include "arrow/type.h"
 
-namespace arrow {
+namespace arrow {}  // namespace arrow
 
-} // namespace arrow
-
-#endif // ARROW_TYPES_STRUCT_H
+#endif  // ARROW_TYPES_STRUCT_H
diff --git a/cpp/src/arrow/types/test-common.h b/cpp/src/arrow/types/test-common.h
index 227aca632ef3c..1957636b141fd 100644
--- a/cpp/src/arrow/types/test-common.h
+++ b/cpp/src/arrow/types/test-common.h
@@ -28,10 +28,10 @@
 #include "arrow/type.h"
 #include "arrow/util/memory-pool.h"
 
-using std::unique_ptr;
-
 namespace arrow {
 
+using std::unique_ptr;
+
 class TestBuilder : public ::testing::Test {
  public:
   void SetUp() {
@@ -40,6 +40,7 @@ class TestBuilder : public ::testing::Test {
     builder_.reset(new UInt8Builder(pool_, type_));
     builder_nn_.reset(new UInt8Builder(pool_, type_));
   }
+
  protected:
   MemoryPool* pool_;
 
@@ -48,6 +49,6 @@ class TestBuilder : public ::testing::Test {
   unique_ptr<ArrayBuilder> builder_nn_;
 };
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_TYPES_TEST_COMMON_H
+#endif  // ARROW_TYPES_TEST_COMMON_H
diff --git a/cpp/src/arrow/types/union.cc b/cpp/src/arrow/types/union.cc
index db3f81795eae2..c891b4a5357ef 100644
--- a/cpp/src/arrow/types/union.cc
+++ b/cpp/src/arrow/types/union.cc
@@ -30,7 +30,7 @@ static inline std::string format_union(const std::vector<TypePtr>& child_types)
   std::stringstream s;
   s << "union<";
   for (size_t i = 0; i < child_types.size(); ++i) {
-    if (i) s << ", ";
+    if (i) { s << ", "; }
     s << child_types[i]->ToString();
   }
   s << ">";
@@ -41,10 +41,8 @@ std::string DenseUnionType::ToString() const {
   return format_union(child_types_);
 }
 
-
 std::string SparseUnionType::ToString() const {
   return format_union(child_types_);
 }
 
-
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/union.h b/cpp/src/arrow/types/union.h
index 29cda90b972dd..d2ee9bde04d0d 100644
--- a/cpp/src/arrow/types/union.h
+++ b/cpp/src/arrow/types/union.h
@@ -33,27 +33,23 @@ class Buffer;
 struct DenseUnionType : public CollectionType<Type::DENSE_UNION> {
   typedef CollectionType<Type::DENSE_UNION> Base;
 
-  explicit DenseUnionType(const std::vector<TypePtr>& child_types) :
-      Base() {
+  explicit DenseUnionType(const std::vector<TypePtr>& child_types) : Base() {
     child_types_ = child_types;
   }
 
   virtual std::string ToString() const;
 };
 
-
 struct SparseUnionType : public CollectionType<Type::SPARSE_UNION> {
   typedef CollectionType<Type::SPARSE_UNION> Base;
 
-  explicit SparseUnionType(const std::vector<TypePtr>& child_types) :
-      Base() {
+  explicit SparseUnionType(const std::vector<TypePtr>& child_types) : Base() {
     child_types_ = child_types;
   }
 
   virtual std::string ToString() const;
 };
 
-
 class UnionArray : public Array {
  protected:
   // The data are types encoded as int16
@@ -61,16 +57,13 @@ class UnionArray : public Array {
   std::vector<std::shared_ptr<Array>> children_;
 };
 
-
 class DenseUnionArray : public UnionArray {
  protected:
   Buffer* offset_buf_;
 };
 
+class SparseUnionArray : public UnionArray {};
 
-class SparseUnionArray : public UnionArray {
-};
-
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_TYPES_UNION_H
+#endif  // ARROW_TYPES_UNION_H
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index 220bff084fd6e..26554d2c9069c 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -41,4 +41,4 @@ TEST(UtilTests, TestNextPower2) {
   ASSERT_EQ(1LL << 62, next_power2((1LL << 62) - 1));
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 6c6d5330eab0d..475576e87cadd 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -26,14 +26,12 @@ namespace arrow {
 
 void util::bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits) {
   for (size_t i = 0; i < bytes.size(); ++i) {
-    if (bytes[i] > 0) {
-      set_bit(bits, i);
-    }
+    if (bytes[i] > 0) { set_bit(bits, i); }
   }
 }
 
-Status util::bytes_to_bits(const std::vector<uint8_t>& bytes,
-    std::shared_ptr<Buffer>* out) {
+Status util::bytes_to_bits(
+    const std::vector<uint8_t>& bytes, std::shared_ptr<Buffer>* out) {
   int bit_length = util::bytes_for_bits(bytes.size());
 
   auto buffer = std::make_shared<PoolBuffer>();
@@ -45,4 +43,4 @@ Status util::bytes_to_bits(const std::vector<uint8_t>& bytes,
   return Status::OK();
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 8d6287130dd2b..1f0f08c4d88ef 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -74,8 +74,8 @@ static inline int64_t next_power2(int64_t n) {
 void bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits);
 Status bytes_to_bits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
-} // namespace util
+}  // namespace util
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_UTIL_BIT_UTIL_H
+#endif  // ARROW_UTIL_BIT_UTIL_H
diff --git a/cpp/src/arrow/util/buffer-test.cc b/cpp/src/arrow/util/buffer-test.cc
index 1d58226d84a46..dad0f7461d914 100644
--- a/cpp/src/arrow/util/buffer-test.cc
+++ b/cpp/src/arrow/util/buffer-test.cc
@@ -29,8 +29,7 @@ using std::string;
 
 namespace arrow {
 
-class TestBuffer : public ::testing::Test {
-};
+class TestBuffer : public ::testing::Test {};
 
 TEST_F(TestBuffer, Resize) {
   PoolBuffer buf;
@@ -54,4 +53,4 @@ TEST_F(TestBuffer, ResizeOOM) {
   ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
index 04cdcd75cd41a..bc9c22c10de44 100644
--- a/cpp/src/arrow/util/buffer.cc
+++ b/cpp/src/arrow/util/buffer.cc
@@ -24,8 +24,7 @@
 
 namespace arrow {
 
-Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset,
-    int64_t size) {
+Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size) {
   data_ = parent->data() + offset;
   size_ = size;
   parent_ = parent;
@@ -37,18 +36,13 @@ std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
   return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
 }
 
-PoolBuffer::PoolBuffer(MemoryPool* pool) :
-    ResizableBuffer(nullptr, 0) {
-  if (pool == nullptr) {
-    pool = default_memory_pool();
-  }
+PoolBuffer::PoolBuffer(MemoryPool* pool) : ResizableBuffer(nullptr, 0) {
+  if (pool == nullptr) { pool = default_memory_pool(); }
   pool_ = pool;
 }
 
 PoolBuffer::~PoolBuffer() {
-  if (mutable_data_ != nullptr) {
-    pool_->Free(mutable_data_, capacity_);
-  }
+  if (mutable_data_ != nullptr) { pool_->Free(mutable_data_, capacity_); }
 }
 
 Status PoolBuffer::Reserve(int64_t new_capacity) {
@@ -74,4 +68,4 @@ Status PoolBuffer::Resize(int64_t new_size) {
   return Status::OK();
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index c15f9b630cd97..94e53b61f2e83 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -38,9 +38,7 @@ class Status;
 // class instance
 class Buffer : public std::enable_shared_from_this<Buffer> {
  public:
-  Buffer(const uint8_t* data, int64_t size) :
-      data_(data),
-      size_(size) {}
+  Buffer(const uint8_t* data, int64_t size) : data_(data), size_(size) {}
   virtual ~Buffer();
 
   // An offset into data that is owned by another buffer, but we want to be
@@ -48,40 +46,28 @@ class Buffer : public std::enable_shared_from_this<Buffer> {
   // parent buffer have been destroyed
   Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
 
-  std::shared_ptr<Buffer> get_shared_ptr() {
-    return shared_from_this();
-  }
+  std::shared_ptr<Buffer> get_shared_ptr() { return shared_from_this(); }
 
   // Return true if both buffers are the same size and contain the same bytes
   // up to the number of compared bytes
   bool Equals(const Buffer& other, int64_t nbytes) const {
-    return this == &other ||
-      (size_ >= nbytes && other.size_ >= nbytes &&
-          !memcmp(data_, other.data_, nbytes));
+    return this == &other || (size_ >= nbytes && other.size_ >= nbytes &&
+                                 !memcmp(data_, other.data_, nbytes));
   }
 
   bool Equals(const Buffer& other) const {
-    return this == &other ||
-      (size_ == other.size_ && !memcmp(data_, other.data_, size_));
+    return this == &other || (size_ == other.size_ && !memcmp(data_, other.data_, size_));
   }
 
-  const uint8_t* data() const {
-    return data_;
-  }
+  const uint8_t* data() const { return data_; }
 
-  int64_t size() const {
-    return size_;
-  }
+  int64_t size() const { return size_; }
 
   // Returns true if this Buffer is referencing memory (possibly) owned by some
   // other buffer
-  bool is_shared() const {
-    return static_cast<bool>(parent_);
-  }
+  bool is_shared() const { return static_cast<bool>(parent_); }
 
-  const std::shared_ptr<Buffer> parent() const {
-    return parent_;
-  }
+  const std::shared_ptr<Buffer> parent() const { return parent_; }
 
  protected:
   const uint8_t* data_;
@@ -97,22 +83,17 @@ class Buffer : public std::enable_shared_from_this<Buffer> {
 // A Buffer whose contents can be mutated. May or may not own its data.
 class MutableBuffer : public Buffer {
  public:
-  MutableBuffer(uint8_t* data, int64_t size) :
-      Buffer(data, size) {
+  MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
     mutable_data_ = data;
   }
 
-  uint8_t* mutable_data() {
-    return mutable_data_;
-  }
+  uint8_t* mutable_data() { return mutable_data_; }
 
   // Get a read-only view of this buffer
   std::shared_ptr<Buffer> GetImmutableView();
 
  protected:
-  MutableBuffer() :
-      Buffer(nullptr, 0),
-      mutable_data_(nullptr) {}
+  MutableBuffer() : Buffer(nullptr, 0), mutable_data_(nullptr) {}
 
   uint8_t* mutable_data_;
 };
@@ -128,9 +109,8 @@ class ResizableBuffer : public MutableBuffer {
   virtual Status Reserve(int64_t new_capacity) = 0;
 
  protected:
-  ResizableBuffer(uint8_t* data, int64_t size) :
-      MutableBuffer(data, size),
-      capacity_(size) {}
+  ResizableBuffer(uint8_t* data, int64_t size)
+      : MutableBuffer(data, size), capacity_(size) {}
 
   int64_t capacity_;
 };
@@ -152,16 +132,11 @@ static constexpr int64_t MIN_BUFFER_CAPACITY = 1024;
 
 class BufferBuilder {
  public:
-  explicit BufferBuilder(MemoryPool* pool) :
-      pool_(pool),
-      capacity_(0),
-      size_(0) {}
+  explicit BufferBuilder(MemoryPool* pool) : pool_(pool), capacity_(0), size_(0) {}
 
   Status Append(const uint8_t* data, int length) {
     if (capacity_ < length + size_) {
-      if (capacity_ == 0) {
-        buffer_ = std::make_shared<PoolBuffer>(pool_);
-      }
+      if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
       capacity_ = std::max(MIN_BUFFER_CAPACITY, capacity_);
       while (capacity_ < length + size_) {
         capacity_ *= 2;
@@ -188,6 +163,6 @@ class BufferBuilder {
   int64_t size_;
 };
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_UTIL_BUFFER_H
+#endif  // ARROW_UTIL_BUFFER_H
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 3ce4ccc1e9c26..527ce423e7751 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -19,6 +19,7 @@
 #define ARROW_UTIL_LOGGING_H
 
 #include <iostream>
+#include <cstdlib>
 
 namespace arrow {
 
@@ -37,19 +38,34 @@ namespace arrow {
 #define ARROW_LOG_INTERNAL(level) arrow::internal::CerrLog(level)
 #define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level)
 
-#define ARROW_CHECK(condition) \
-  (condition) ? 0 : ARROW_LOG(FATAL) << "Check failed: " #condition " "
+#define ARROW_CHECK(condition)                               \
+  (condition) ? 0 : ::arrow::internal::FatalLog(ARROW_FATAL) \
+                        << __FILE__ << __LINE__ << "Check failed: " #condition " "
 
 #ifdef NDEBUG
 #define ARROW_DFATAL ARROW_WARNING
 
-#define DCHECK(condition) while (false) arrow::internal::NullLog()
-#define DCHECK_EQ(val1, val2) while (false) arrow::internal::NullLog()
-#define DCHECK_NE(val1, val2) while (false) arrow::internal::NullLog()
-#define DCHECK_LE(val1, val2) while (false) arrow::internal::NullLog()
-#define DCHECK_LT(val1, val2) while (false) arrow::internal::NullLog()
-#define DCHECK_GE(val1, val2) while (false) arrow::internal::NullLog()
-#define DCHECK_GT(val1, val2) while (false) arrow::internal::NullLog()
+#define DCHECK(condition) \
+  while (false)           \
+  arrow::internal::NullLog()
+#define DCHECK_EQ(val1, val2) \
+  while (false)               \
+  arrow::internal::NullLog()
+#define DCHECK_NE(val1, val2) \
+  while (false)               \
+  arrow::internal::NullLog()
+#define DCHECK_LE(val1, val2) \
+  while (false)               \
+  arrow::internal::NullLog()
+#define DCHECK_LT(val1, val2) \
+  while (false)               \
+  arrow::internal::NullLog()
+#define DCHECK_GE(val1, val2) \
+  while (false)               \
+  arrow::internal::NullLog()
+#define DCHECK_GT(val1, val2) \
+  while (false)               \
+  arrow::internal::NullLog()
 
 #else
 #define ARROW_DFATAL ARROW_FATAL
@@ -62,13 +78,13 @@ namespace arrow {
 #define DCHECK_GE(val1, val2) ARROW_CHECK((val1) >= (val2))
 #define DCHECK_GT(val1, val2) ARROW_CHECK((val1) > (val2))
 
-#endif // NDEBUG
+#endif  // NDEBUG
 
 namespace internal {
 
 class NullLog {
  public:
-  template<class T>
+  template <class T>
   NullLog& operator<<(const T& t) {
     return *this;
   }
@@ -76,34 +92,42 @@ class NullLog {
 
 class CerrLog {
  public:
-  CerrLog(int severity) // NOLINT(runtime/explicit)
-    : severity_(severity),
-      has_logged_(false) {
-  }
+  CerrLog(int severity)  // NOLINT(runtime/explicit)
+      : severity_(severity),
+        has_logged_(false) {}
 
-  ~CerrLog() {
-    if (has_logged_) {
-      std::cerr << std::endl;
-    }
-    if (severity_ == ARROW_FATAL) {
-      exit(1);
-    }
+  virtual ~CerrLog() {
+    if (has_logged_) { std::cerr << std::endl; }
+    if (severity_ == ARROW_FATAL) { std::exit(1); }
   }
 
-  template<class T>
+  template <class T>
   CerrLog& operator<<(const T& t) {
     has_logged_ = true;
     std::cerr << t;
     return *this;
   }
 
- private:
+ protected:
   const int severity_;
   bool has_logged_;
 };
 
-} // namespace internal
+// Clang-tidy isn't smart enough to determine that DCHECK using CerrLog doesn't
+// return so we create a new class to give it a hint.
+class FatalLog : public CerrLog {
+ public:
+  FatalLog(int /* severity */)  // NOLINT
+      : CerrLog(ARROW_FATAL) {}
+
+  [[noreturn]] ~FatalLog() {
+    if (has_logged_) { std::cerr << std::endl; }
+    std::exit(1);
+  }
+};
+
+}  // namespace internal
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_UTIL_LOGGING_H
+#endif  // ARROW_UTIL_LOGGING_H
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index 069e627c90eaa..51e605ee50ac4 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -19,8 +19,8 @@
 #define ARROW_UTIL_MACROS_H
 
 // From Google gutil
-#define DISALLOW_COPY_AND_ASSIGN(TypeName)      \
-  TypeName(const TypeName&) = delete;           \
+#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
+  TypeName(const TypeName&) = delete;      \
   void operator=(const TypeName&) = delete
 
-#endif // ARROW_UTIL_MACROS_H
+#endif  // ARROW_UTIL_MACROS_H
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
index 6ef07a07ada3f..e4600a9bd9b27 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -45,4 +45,4 @@ TEST(DefaultMemoryPool, OOM) {
   ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
index 0b885e9376a62..fb417e74daf53 100644
--- a/cpp/src/arrow/util/memory-pool.cc
+++ b/cpp/src/arrow/util/memory-pool.cc
@@ -75,4 +75,4 @@ MemoryPool* default_memory_pool() {
   return &default_memory_pool_;
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/memory-pool.h b/cpp/src/arrow/util/memory-pool.h
index 0d2478686f5a4..824c7248e2e86 100644
--- a/cpp/src/arrow/util/memory-pool.h
+++ b/cpp/src/arrow/util/memory-pool.h
@@ -36,6 +36,6 @@ class MemoryPool {
 
 MemoryPool* default_memory_pool();
 
-} // namespace arrow
+}  // namespace arrow
 
-#endif // ARROW_UTIL_MEMORY_POOL_H
+#endif  // ARROW_UTIL_MEMORY_POOL_H
diff --git a/cpp/src/arrow/util/random.h b/cpp/src/arrow/util/random.h
index 64c197ef080fd..31f2b0680fe0a 100644
--- a/cpp/src/arrow/util/random.h
+++ b/cpp/src/arrow/util/random.h
@@ -15,10 +15,10 @@ namespace arrow {
 
 namespace random_internal {
 
-static const uint32_t M = 2147483647L;   // 2^31-1
+static const uint32_t M = 2147483647L;  // 2^31-1
 const double kTwoPi = 6.283185307179586476925286;
 
-} // namespace random_internal
+}  // namespace random_internal
 
 // A very simple random number generator.  Not especially good at
 // generating truly random bits, but good enough for our needs in this
@@ -27,9 +27,7 @@ class Random {
  public:
   explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) {
     // Avoid bad seeds.
-    if (seed_ == 0 || seed_ == random_internal::M) {
-      seed_ = 1;
-    }
+    if (seed_ == 0 || seed_ == random_internal::M) { seed_ = 1; }
   }
 
   // Next pseudo-random 32-bit unsigned integer.
@@ -50,9 +48,7 @@ class Random {
     // The first reduction may overflow by 1 bit, so we may need to
     // repeat.  mod == M is not possible; using > allows the faster
     // sign-bit-based test.
-    if (seed_ > random_internal::M) {
-      seed_ -= random_internal::M;
-    }
+    if (seed_ > random_internal::M) { seed_ -= random_internal::M; }
     return seed_;
   }
 
@@ -91,9 +87,7 @@ class Random {
   // Skewed: pick "base" uniformly from range [0,max_log] and then
   // return "base" random bits.  The effect is to pick a number in the
   // range [0,2^max_log-1] with exponential bias towards smaller numbers.
-  uint32_t Skewed(int max_log) {
-    return Uniform(1 << Uniform(max_log + 1));
-  }
+  uint32_t Skewed(int max_log) { return Uniform(1 << Uniform(max_log + 1)); }
 
   // Creates a normal distribution variable using the
   // Box-Muller transform. See:
@@ -103,8 +97,9 @@ class Random {
   double Normal(double mean, double std_dev) {
     double uniform1 = (Next() + 1.0) / (random_internal::M + 1.0);
     double uniform2 = (Next() + 1.0) / (random_internal::M + 1.0);
-    return (mean + std_dev * sqrt(-2 * ::log(uniform1)) *
-        cos(random_internal::kTwoPi * uniform2));
+    return (
+        mean +
+        std_dev * sqrt(-2 * ::log(uniform1)) * cos(random_internal::kTwoPi * uniform2));
   }
 
   // Return a random number between 0.0 and 1.0 inclusive.
@@ -116,13 +111,11 @@ class Random {
   uint32_t seed_;
 };
 
-
 uint32_t random_seed() {
-  // TODO: use system time to get a reasonably random seed
+  // TODO(wesm): use system time to get a reasonably random seed
   return 0;
 }
 
-
-} // namespace arrow
+}  // namespace arrow
 
 #endif  // ARROW_UTIL_RANDOM_H_
diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/util/status.cc
index 43cb87e1a8c56..d194ed5572f52 100644
--- a/cpp/src/arrow/util/status.cc
+++ b/cpp/src/arrow/util/status.cc
@@ -36,9 +36,7 @@ const char* Status::CopyState(const char* state) {
 }
 
 std::string Status::CodeAsString() const {
-  if (state_ == NULL) {
-    return "OK";
-  }
+  if (state_ == NULL) { return "OK"; }
 
   const char* type;
   switch (code()) {
@@ -66,9 +64,7 @@ std::string Status::CodeAsString() const {
 
 std::string Status::ToString() const {
   std::string result(CodeAsString());
-  if (state_ == NULL) {
-    return result;
-  }
+  if (state_ == NULL) { return result; }
 
   result.append(": ");
 
@@ -78,4 +74,4 @@ std::string Status::ToString() const {
   return result;
 }
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
index 4e273edcb8f1f..6ddc177a9a50d 100644
--- a/cpp/src/arrow/util/status.h
+++ b/cpp/src/arrow/util/status.h
@@ -20,32 +20,36 @@
 #include <string>
 
 // Return the given status if it is not OK.
-#define ARROW_RETURN_NOT_OK(s) do {           \
-    ::arrow::Status _s = (s);                 \
-    if (!_s.ok()) return _s;                    \
+#define ARROW_RETURN_NOT_OK(s)   \
+  do {                           \
+    ::arrow::Status _s = (s);    \
+    if (!_s.ok()) { return _s; } \
   } while (0);
 
 // Return the given status if it is not OK, but first clone it and
 // prepend the given message.
-#define ARROW_RETURN_NOT_OK_PREPEND(s, msg) do {                      \
-    ::arrow::Status _s = (s);                                         \
+#define ARROW_RETURN_NOT_OK_PREPEND(s, msg)                               \
+  do {                                                                    \
+    ::arrow::Status _s = (s);                                             \
     if (::gutil::PREDICT_FALSE(!_s.ok())) return _s.CloneAndPrepend(msg); \
   } while (0);
 
 // Return 'to_return' if 'to_call' returns a bad status.
 // The substitution for 'to_return' may reference the variable
 // 's' for the bad status.
-#define ARROW_RETURN_NOT_OK_RET(to_call, to_return) do { \
-    ::arrow::Status s = (to_call); \
-    if (::gutil::PREDICT_FALSE(!s.ok())) return (to_return);    \
+#define ARROW_RETURN_NOT_OK_RET(to_call, to_return)          \
+  do {                                                       \
+    ::arrow::Status s = (to_call);                           \
+    if (::gutil::PREDICT_FALSE(!s.ok())) return (to_return); \
   } while (0);
 
 // If 'to_call' returns a bad status, CHECK immediately with a logged message
 // of 'msg' followed by the status.
-#define ARROW_CHECK_OK_PREPEND(to_call, msg) do {         \
-::arrow::Status _s = (to_call);                           \
-ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString();   \
-} while (0);
+#define ARROW_CHECK_OK_PREPEND(to_call, msg)                \
+  do {                                                      \
+    ::arrow::Status _s = (to_call);                         \
+    ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString(); \
+  } while (0);
 
 // If the status is bad, CHECK immediately, appending the status to the
 // logged message.
@@ -53,12 +57,13 @@ ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString();   \
 
 namespace arrow {
 
-#define RETURN_NOT_OK(s) do {                   \
-    Status _s = (s);                            \
-    if (!_s.ok()) return _s;                    \
+#define RETURN_NOT_OK(s)         \
+  do {                           \
+    Status _s = (s);             \
+    if (!_s.ok()) { return _s; } \
   } while (0);
 
-enum class StatusCode: char {
+enum class StatusCode : char {
   OK = 0,
   OutOfMemory = 1,
   KeyError = 2,
@@ -71,7 +76,7 @@ enum class StatusCode: char {
 class Status {
  public:
   // Create a success status.
-  Status() : state_(NULL) { }
+  Status() : state_(NULL) {}
   ~Status() { delete[] state_; }
 
   // Copy the specified status.
@@ -132,8 +137,7 @@ class Status {
   const char* state_;
 
   StatusCode code() const {
-    return ((state_ == NULL) ?
-        StatusCode::OK : static_cast<StatusCode>(state_[4]));
+    return ((state_ == NULL) ? StatusCode::OK : static_cast<StatusCode>(state_[4]));
   }
 
   Status(StatusCode code, const std::string& msg, int16_t posix_code);
@@ -155,5 +159,4 @@ inline void Status::operator=(const Status& s) {
 
 }  // namespace arrow
 
-
-#endif // ARROW_STATUS_H_
+#endif  // ARROW_STATUS_H_
diff --git a/cpp/src/arrow/util/test_main.cc b/cpp/src/arrow/util/test_main.cc
index adc8466fb0be9..f928047023966 100644
--- a/cpp/src/arrow/util/test_main.cc
+++ b/cpp/src/arrow/util/test_main.cc
@@ -17,7 +17,7 @@
 
 #include "gtest/gtest.h"
 
-int main(int argc, char **argv) {
+int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
 
   int ret = RUN_ALL_TESTS();
diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
index 3d5f532b16309..f1738ff748299 100755
--- a/cpp/thirdparty/build_thirdparty.sh
+++ b/cpp/thirdparty/build_thirdparty.sh
@@ -84,8 +84,8 @@ if [ -n "$F_ALL" -o -n "$F_FLATBUFFERS" ]; then
   cd $TP_DIR/$FLATBUFFERS_BASEDIR
 
   CXXFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX -DFLATBUFFERS_BUILD_TESTS=OFF . || { echo "cmake $FLATBUFFERS_ERROR" ; exit 1; }
-  make -j$PARALLEL
-  make install
+  make VERBOSE=1 -j$PARALLEL || { echo "make $FLATBUFFERS_ERROR" ; exit 1; }
+  make install || { echo "install $FLATBUFFERS_ERROR" ; exit 1; }
 fi
 
 echo "---------------------"

From 9d88a50c55d18860c5543dfa6ddc8f4f162dd5e5 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 3 Apr 2016 13:10:17 -0700
Subject: [PATCH 0058/1644] ARROW-86: [Python] Implement zero-copy
 Arrow-to-Pandas conversion

We can create zero-copy NumPy arrays for floats and ints if we have no
nulls. Each numpy-arrow-view has a reference to the underlying column to
ensure that the Arrow structure lives at least as long as the newly
created NumPy array.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #52 from xhochy/arrow-86 and squashes the following commits:

ee29e90 [Uwe L. Korn] Remove duplicate ref counting
2cb4c7d [Uwe L. Korn] Release instead of reset reference
9d35528 [Uwe L. Korn] Handle reference counting with OwnedRef
327b368 [Uwe L. Korn] ARROW-86: [Python] Implement zero-copy Arrow-to-Pandas conversion
---
 python/pyarrow/array.pyx              |  1 -
 python/pyarrow/includes/pyarrow.pxd   |  2 +-
 python/pyarrow/table.pyx              |  6 ++-
 python/src/pyarrow/adapters/pandas.cc | 67 ++++++++++++++++++++-------
 python/src/pyarrow/adapters/pandas.h  |  3 +-
 python/src/pyarrow/common.h           |  4 ++
 6 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 456bf6d1da848..a80b3ce83980e 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -288,4 +288,3 @@ cdef class RowBatch:
     def __getitem__(self, i):
         return self.arrays[i]
 
-
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 1066b8034be70..92c814706fdd6 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -46,6 +46,6 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     Status PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
                                shared_ptr[CArray]* out)
 
-    Status ArrowToPandas(const shared_ptr[CColumn]& arr, PyObject** out)
+    Status ArrowToPandas(const shared_ptr[CColumn]& arr, object py_ref, PyObject** out)
 
     MemoryPool* GetMemoryPool()
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 4c4816f0c7e69..f02d36f520be6 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -96,7 +96,7 @@ cdef class Column:
 
         import pandas as pd
 
-        check_status(pyarrow.ArrowToPandas(self.sp_column, &arr))
+        check_status(pyarrow.ArrowToPandas(self.sp_column, self, &arr))
         return pd.Series(<object>arr, name=self.name)
 
     cdef _check_nullptr(self):
@@ -205,6 +205,7 @@ cdef class Table:
         cdef:
             PyObject* arr
             shared_ptr[CColumn] col
+            Column column
 
         import pandas as pd
 
@@ -212,7 +213,8 @@ cdef class Table:
         data = []
         for i in range(self.table.num_columns()):
             col = self.table.column(i)
-            check_status(pyarrow.ArrowToPandas(col, &arr))
+            column = self.column(i)
+            check_status(pyarrow.ArrowToPandas(col, column, &arr))
             names.append(frombytes(col.get().name()))
             data.append(<object> arr)
 
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 22f1d7575f8c5..b39fde92034aa 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -520,8 +520,8 @@ static inline PyObject* make_pystring(const uint8_t* data, int32_t length) {
 template <int TYPE>
 class ArrowDeserializer {
  public:
-  ArrowDeserializer(const std::shared_ptr<Column>& col) :
-      col_(col) {}
+  ArrowDeserializer(const std::shared_ptr<Column>& col, PyObject* py_ref) :
+      col_(col), py_ref_(py_ref) {}
 
   Status Convert(PyObject** out) {
     const std::shared_ptr<arrow::ChunkedArray> data = col_->data();
@@ -548,6 +548,33 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
+  Status OutputFromData(int type, void* data) {
+    // Zero-Copy. We can pass the data pointer directly to NumPy.
+    Py_INCREF(py_ref_);
+    OwnedRef py_ref(py_ref);
+    npy_intp dims[1] = {col_->length()};
+    out_ = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNewFromData(1, dims,
+                type, data));
+
+    if (out_ == NULL) {
+      // Error occurred, trust that SimpleNew set the error state
+      return Status::OK();
+    }
+
+    if (PyArray_SetBaseObject(out_, py_ref_) == -1) {
+      // Error occurred, trust that SetBaseObject set the error state
+      return Status::OK();
+    } else {
+      // PyArray_SetBaseObject steals our reference to py_ref_
+      py_ref.release();
+    }
+
+    // Arrow data is immutable.
+    PyArray_CLEARFLAGS(out_, NPY_ARRAY_WRITEABLE);
+
+    return Status::OK();
+  }
+
   template <int T2>
   inline typename std::enable_if<
     arrow_traits<T2>::is_floating, Status>::type
@@ -556,18 +583,20 @@ class ArrowDeserializer {
 
     arrow::PrimitiveArray* prim_arr = static_cast<arrow::PrimitiveArray*>(
         arr.get());
-
-    RETURN_NOT_OK(AllocateOutput(arrow_traits<T2>::npy_type));
+    const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
 
     if (arr->null_count() > 0) {
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<T2>::npy_type));
+
       T* out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
-      const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
       for (int64_t i = 0; i < arr->length(); ++i) {
         out_values[i] = arr->IsNull(i) ? NAN : in_values[i];
       }
     } else {
-      memcpy(PyArray_DATA(out_), prim_arr->data()->data(),
-          arr->length() * arr->type()->value_size());
+      // Zero-Copy. We can pass the data pointer directly to NumPy.
+      void* data = const_cast<T*>(in_values);
+      int type = arrow_traits<TYPE>::npy_type;
+      RETURN_NOT_OK(OutputFromData(type, data));
     }
 
     return Status::OK();
@@ -594,10 +623,10 @@ class ArrowDeserializer {
         out_values[i] = prim_arr->IsNull(i) ? NAN : in_values[i];
       }
     } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
-
-      memcpy(PyArray_DATA(out_), in_values,
-          arr->length() * arr->type()->value_size());
+      // Zero-Copy. We can pass the data pointer directly to NumPy.
+      void* data = const_cast<T*>(in_values);
+      int type = arrow_traits<TYPE>::npy_type;
+      RETURN_NOT_OK(OutputFromData(type, data));
     }
 
     return Status::OK();
@@ -680,18 +709,20 @@ class ArrowDeserializer {
   }
  private:
   std::shared_ptr<Column> col_;
+  PyObject* py_ref_;
   PyArrayObject* out_;
 };
 
-#define FROM_ARROW_CASE(TYPE)                               \
-  case arrow::Type::TYPE:                                   \
-    {                                                       \
-      ArrowDeserializer<arrow::Type::TYPE> converter(col);  \
-      return converter.Convert(out);                        \
-    }                                                       \
+#define FROM_ARROW_CASE(TYPE)                                       \
+  case arrow::Type::TYPE:                                           \
+    {                                                               \
+      ArrowDeserializer<arrow::Type::TYPE> converter(col, py_ref);  \
+      return converter.Convert(out);                                \
+    }                                                               \
     break;
 
-Status ArrowToPandas(const std::shared_ptr<Column>& col, PyObject** out) {
+Status ArrowToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
+        PyObject** out) {
   switch(col->type()->type) {
     FROM_ARROW_CASE(BOOL);
     FROM_ARROW_CASE(INT8);
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index 58eb3ca61cdf4..17922349de6c1 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -36,7 +36,8 @@ namespace pyarrow {
 
 class Status;
 
-Status ArrowToPandas(const std::shared_ptr<arrow::Column>& col, PyObject** out);
+Status ArrowToPandas(const std::shared_ptr<arrow::Column>& col, PyObject* py_ref,
+    PyObject** out);
 
 Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     std::shared_ptr<arrow::Array>* out);
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index cc9ad9ec5bbea..0211e8948f2f7 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -53,6 +53,10 @@ class OwnedRef {
     obj_ = obj;
   }
 
+  void release() {
+    obj_ = nullptr;
+  }
+
   PyObject* obj() const{
     return obj_;
   }

From 7b2153b0430b825730a6e986993bb290ef29d22a Mon Sep 17 00:00:00 2001
From: Kai Zheng <kai.zheng@intel.com>
Date: Mon, 11 Apr 2016 10:35:50 +0200
Subject: [PATCH 0059/1644] =?UTF-8?q?ARROW-85:=20memcmp=20can=20be=20avoid?=
 =?UTF-8?q?ed=20in=20Equal=20when=20comparing=20with=20the=20same=20?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Avoid memcmp when possible, if the two underlying buffers are the same.

Author: Kai Zheng <kai.zheng@intel.com>

Closes #57 from drankye/upstream and squashes the following commits:

2a70944 [Kai Zheng] Free test buffers afterwards
6a8bef5 [Kai Zheng] Fixed some comments
b83f989 [Kai Zheng] ARROW-85. Corrected another format issue by clang-format
0ddcd01 [Kai Zheng] ARROW-85. Fixed another format issue
1b48663 [Kai Zheng] ARROW-85. Fixed checking styles
9f239a3 [Kai Zheng] ARROW-85. Added tests for Buffer and the new behavior
4d04c27 [Kai Zheng] ARROW-85 memcmp can be avoided in Equal when comparing with the same Buffer
---
 cpp/src/arrow/util/buffer-test.cc | 43 +++++++++++++++++++++++++++++++
 cpp/src/arrow/util/buffer.h       |  9 ++++---
 2 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/util/buffer-test.cc b/cpp/src/arrow/util/buffer-test.cc
index dad0f7461d914..cc4ec98e4fb29 100644
--- a/cpp/src/arrow/util/buffer-test.cc
+++ b/cpp/src/arrow/util/buffer-test.cc
@@ -53,4 +53,47 @@ TEST_F(TestBuffer, ResizeOOM) {
   ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));
 }
 
+TEST_F(TestBuffer, EqualsWithSameContent) {
+  MemoryPool* pool = default_memory_pool();
+  const int32_t bufferSize = 128 * 1024;
+  uint8_t* rawBuffer1;
+  ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer1));
+  memset(rawBuffer1, 12, bufferSize);
+  uint8_t* rawBuffer2;
+  ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer2));
+  memset(rawBuffer2, 12, bufferSize);
+  uint8_t* rawBuffer3;
+  ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer3));
+  memset(rawBuffer3, 3, bufferSize);
+
+  Buffer buffer1(rawBuffer1, bufferSize);
+  Buffer buffer2(rawBuffer2, bufferSize);
+  Buffer buffer3(rawBuffer3, bufferSize);
+  ASSERT_TRUE(buffer1.Equals(buffer2));
+  ASSERT_FALSE(buffer1.Equals(buffer3));
+
+  pool->Free(rawBuffer1, bufferSize);
+  pool->Free(rawBuffer2, bufferSize);
+  pool->Free(rawBuffer3, bufferSize);
+}
+
+TEST_F(TestBuffer, EqualsWithSameBuffer) {
+  MemoryPool* pool = default_memory_pool();
+  const int32_t bufferSize = 128 * 1024;
+  uint8_t* rawBuffer;
+  ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer));
+  memset(rawBuffer, 111, bufferSize);
+
+  Buffer buffer1(rawBuffer, bufferSize);
+  Buffer buffer2(rawBuffer, bufferSize);
+  ASSERT_TRUE(buffer1.Equals(buffer2));
+
+  const int64_t nbytes = bufferSize / 2;
+  Buffer buffer3(rawBuffer, nbytes);
+  ASSERT_TRUE(buffer1.Equals(buffer3, nbytes));
+  ASSERT_FALSE(buffer1.Equals(buffer3, nbytes + 1));
+
+  pool->Free(rawBuffer, bufferSize);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 94e53b61f2e83..56532be8070ae 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -51,12 +51,15 @@ class Buffer : public std::enable_shared_from_this<Buffer> {
   // Return true if both buffers are the same size and contain the same bytes
   // up to the number of compared bytes
   bool Equals(const Buffer& other, int64_t nbytes) const {
-    return this == &other || (size_ >= nbytes && other.size_ >= nbytes &&
-                                 !memcmp(data_, other.data_, nbytes));
+    return this == &other ||
+           (size_ >= nbytes && other.size_ >= nbytes &&
+               (data_ == other.data_ || !memcmp(data_, other.data_, nbytes)));
   }
 
   bool Equals(const Buffer& other) const {
-    return this == &other || (size_ == other.size_ && !memcmp(data_, other.data_, size_));
+    return this == &other ||
+           (size_ == other.size_ &&
+               (data_ == other.data_ || !memcmp(data_, other.data_, size_)));
   }
 
   const uint8_t* data() const { return data_; }

From 37f72716822f5b7ec3055b2dd0fabbc992e46c08 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Thu, 14 Apr 2016 19:24:19 +0200
Subject: [PATCH 0060/1644] ARROW-94: [Format] Expand list example to clarify
 null vs empty list

WIP to make sure what I've done so far looks good.  Per discussion on the JIRA item I started conversion of examples images to "text diagrams", but I wanted to get feedback to see if this is actually desirable (and if the way I'm approaching it is desirable).  The remaining diagrams are for unions which I can convert if the existing changes look OK to others (although I think the Union diagrams are are pretty reasonable/compact).

This change also includes some other minor cleanup, as well as including a statement about endianness per the discussion on the mailing list.

Rendered markdown can be seen at: https://github.com/emkornfield/arrow/blob/emk_doc_fixes_PR3/format/Layout.md

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #58 from emkornfield/emk_doc_fixes_PR3 and squashes the following commits:

00b99ef [Micah Kornfield] remove png diagrams that are no longer used
cab6f87 [Micah Kornfield] a few more consistency fixes
5550a78 [Micah Kornfield] fix some off by one bugs
69e1a78 [Micah Kornfield] fix some alignment, and one last offset array to buffer conversion
b7aa7ea [Micah Kornfield] change list offset array to offset buffer
7dda5d5 [Micah Kornfield] clarify requirements of child types, finish replacing diagrams, fix some typos
0f23052 [Micah Kornfield] replace diagrams with physical layouts, clarify memory requirements for struct
590e4a7 [Micah Kornfield] cleanup magic quotes and clarify/fix some minor points
---
 format/Layout.md                           | 343 +++++++++++++++++++--
 format/diagrams/layout-dense-union.png     | Bin 47999 -> 0 bytes
 format/diagrams/layout-list-of-list.png    | Bin 40105 -> 0 bytes
 format/diagrams/layout-list-of-struct.png  | Bin 54122 -> 0 bytes
 format/diagrams/layout-list.png            | Bin 15906 -> 0 bytes
 format/diagrams/layout-primitive-array.png | Bin 10907 -> 0 bytes
 format/diagrams/layout-sparse-union.png    | Bin 43020 -> 0 bytes
 7 files changed, 311 insertions(+), 32 deletions(-)
 delete mode 100644 format/diagrams/layout-dense-union.png
 delete mode 100644 format/diagrams/layout-list-of-list.png
 delete mode 100644 format/diagrams/layout-list-of-struct.png
 delete mode 100644 format/diagrams/layout-list.png
 delete mode 100644 format/diagrams/layout-primitive-array.png
 delete mode 100644 format/diagrams/layout-sparse-union.png

diff --git a/format/Layout.md b/format/Layout.md
index 1b532c6b3817c..92553d944c2d1 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -9,7 +9,7 @@ concepts, here is a small glossary to help disambiguate.
 * Slot or array slot: a single logical value in an array of some particular data type
 * Contiguous memory region: a sequential virtual address space with a given
   length. Any byte can be reached via a single pointer offset less than the
-  region’s length.
+  region's length.
 * Primitive type: a data type that occupies a fixed-size memory slot specified
   in bit width or byte width
 * Nested or parametric type: a data type whose full structure depends on one or
@@ -42,7 +42,7 @@ Base requirements
 * Capable of representing fully-materialized and decoded / decompressed Parquet
   data
 * All leaf nodes (primitive value arrays) use contiguous memory regions
-* Any relative type can be have null slots
+* Any relative type can have null slots
 * Arrays are immutable once created. Implementations can provide APIs to mutate
   an array, but applying mutations will require a new array data structure to
   be built.
@@ -69,11 +69,15 @@ Base requirements
 * To define a selection or masking vector construct
 * Implementation-specific details
 * Details of a user or developer C/C++/Java API.
-* Any “table” structure composed of named arrays each having their own type or
+* Any "table" structure composed of named arrays each having their own type or
   any other structure that composes arrays.
 * Any memory management or reference counting subsystem
 * To enumerate or specify types of encodings or compression support
 
+## Byte Order (Endianness)
+
+The Arrow format is little endian.
+
 ## Array lengths
 
 Any array has a known and fixed length, stored as a 32-bit signed integer, so a
@@ -142,9 +146,59 @@ the size is rounded up to the nearest byte.
 The associated null bitmap is contiguously allocated (as described above) but
 does not need to be adjacent in memory to the values buffer.
 
-(diagram not to scale)
 
-<img src="diagrams/layout-primitive-array.png" width="400"/>
+### Example Layout: Int32 Array
+For example a primitive array of int32s:
+
+[1, 2, null, 4, 8]
+
+Would look like:
+
+```
+* Length: 5, Null count: 1
+* Null bitmap buffer:
+
+  |Byte 0 (validity bitmap) | Bytes 1-7             |
+  |-------------------------|-----------------------|
+  |00011011                 | 0 (padding)           |
+
+* Value Buffer:
+
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 |
+  |------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | unspecified | 4           | 8           |
+```
+
+### Example Layout: Non-null int32 Array
+
+[1, 2, 3, 4, 8] has two possible layouts:
+
+```
+* Length: 5, Null count: 0
+* Null bitmap buffer:
+
+  | Byte 0 (validity bitmap) | Bytes 1-7             |
+  |--------------------------|-----------------------|
+  | 00011111                 | 0 (padding)           |
+
+* Value Buffer:
+
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | bytes 12-15 | bytes 16-19 |
+  |------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | 3           | 4           | 8           |
+```
+
+or with the bitmap elided:
+
+```
+* Length 5, Null count: 0
+* Null bitmap buffer: Not required
+* Value Buffer:
+
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | bytes 12-15 | bytes 16-19 |
+  |------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | 3           | 4           | 8           |
+```
 
 ## List type
 
@@ -158,7 +212,7 @@ A list type is specified like `List<T>`, where `T` is any relative type
 A list-array is represented by the combination of the following:
 
 * A values array, a child array of type T. T may also be a nested type.
-* An offsets array containing 32-bit signed integers with length equal to the
+* An offsets buffer containing 32-bit signed integers with length equal to the
   length of the top-level array plus one. Note that this limits the size of the
   values array to 2^31 -1.
 
@@ -175,20 +229,76 @@ slot_length = offsets[j + 1] - offsets[j]  // (for 0 <= j < length)
 The first value in the offsets array is 0, and the last element is the length
 of the values array.
 
-Let’s consider an example, the type `List<Char>`, where Char is a 1-byte
+### Example Layout: `List<Char>` Array
+Let's consider an example, the type `List<Char>`, where Char is a 1-byte
 logical type.
 
-For an array of length 3 with respective values:
+For an array of length 4 with respective values:
 
-[[‘j’, ‘o’, ‘e’], null, [‘m’, ‘a’, ‘r’, ‘k’]]
+[['j', 'o', 'e'], null, ['m', 'a', 'r', 'k'], []]
 
-We have the following offsets and values arrays
+will have the following representation:
 
-<img src="diagrams/layout-list.png" width="400"/>
+```
+* Length: 4, Null count: 1
+* Null bitmap buffer:
+
+  | Byte 0 (validity bitmap) | Bytes 1-7             |
+  |--------------------------|-----------------------|
+  | 00001101                 | 0 (padding)           |
+
+* Offsets buffer (int32)
+
+  | Bytes 0-3  | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 |
+  |------------|-------------|-------------|-------------|-------------|
+  | 0          | 3           | 3           | 7           | 7           |
+
+* Values array (char array):
+  * Length: 7,  Null count: 0
+  * Null bitmap buffer: Not required
+
+    | Bytes 0-7  |
+    |------------|
+    | joemark    |
+```
+
+### Example Layout: `List<List<byte>>`
+[[[1, 2], [3, 4]], [[5, 6, 7], null, [8]], [[9, 10]]]
+
+will be be represented as follows:
+
+```
+* Length 3
+* Nulls count: 0
+* Null bitmap buffer: Not required
+* Offsets buffer (int32)
+
+  | Bytes 0-3  | Bytes 4-7  | Bytes 8-11 | Bytes 12-15 |
+  |------------|------------|------------|-------------|
+  | 0          |  2         |  6         |  7          |
 
-Let’s consider an array of a nested type, `List<List<byte>>`
+* Values array (`List<byte>`)
+  * Length: 6, Null count: 1
+  * Null bitmap buffer:
 
-<img src="diagrams/layout-list-of-list.png" width="400"/>
+    | Byte 0 (validity bitmap) | Bytes 1-7   |
+    |--------------------------|-------------|
+    | 00110111                 | 0 (padding) |
+
+  * Offsets buffer (int32)
+
+    | Bytes 0-28           |
+    |----------------------|
+    | 0, 2, 4, 7, 7, 8, 10 |
+
+  * Values array (bytes):
+    * Length: 10, Null count: 0
+    * Null bitmap buffer: Not required
+
+      | Bytes 0-9                     |
+      |-------------------------------|
+      | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 |
+```
 
 ## Struct type
 
@@ -198,7 +308,8 @@ types (which can all be distinct), called its fields.
 Typically the fields have names, but the names and their types are part of the
 type metadata, not the physical memory layout.
 
-A struct does not have any additional allocated physical storage.
+A struct array does not have any additional allocated physical storage for its values.
+A struct array must still have an allocated null bitmap, if it has one or more null values.
 
 Physically, a struct type has one child array for each field.
 
@@ -213,15 +324,67 @@ Struct <
 ```
 
 has two child arrays, one List<char> array (layout as above) and one 4-byte
-physical value array having Int32 logical type. Here is a diagram showing the
-full physical layout of this struct:
+primitive value array having Int32 logical type.
+
+### Example Layout: `Struct<List<char>, Int32>`:
+The layout for [{'joe', 1}, {null, 2}, null, {'mark', 4}] would be:
+
+```
+* Length: 4, Null count: 1
+* Null bitmap buffer:
+
+  | Byte 0 (validity bitmap) | Bytes 1-7   |
+  |--------------------------|-------------|
+  | 00001011                 | 0 (padding) |
+
+* Children arrays:
+  * field-0 array (`List<char>`):
+    * Length: 4, Null count: 1
+    * Null bitmap buffer:
+
+      | Byte 0 (validity bitmap) | Bytes 1-7             |
+      |--------------------------|-----------------------|
+      | 00011101                 | 0 (padding)           |
 
-<img src="diagrams/layout-list-of-struct.png" width="400"/>
+    * Offsets buffer:
+
+      | Bytes 0-19     |
+      |----------------|
+      | 0, 3, 3, 6, 10 |
+
+     * Values array:
+        * Length: 10, Null count: 0
+        * Null bitmap buffer: Not required
+
+        * Value buffer:
+
+          | Bytes 0-9      |
+          |----------------|
+          | joebobmark     |
+
+  * field-1 array (int32 array):
+    * Length: 4, Null count: 0
+    * Null bitmap buffer: Not required
+    * Value Buffer:
+
+      | Bytes 0-15     |
+      |----------------|
+      | 1, 2, 3, 4     |
+
+```
 
 While a struct does not have physical storage for each of its semantic slots
 (i.e. each scalar C-like struct), an entire struct slot can be set to null via
 the null bitmap. Any of the child field arrays can have null values according
 to their respective independent null bitmaps.
+This implies that for a particular struct slot the null bitmap for the struct
+array might indicate a null slot when one or more of its child arrays has a
+non-null value in their corresponding slot.  When reading the struct array the
+parent null bitmap is authoritative.
+This is illustrated in the example above, the child arrays have valid entries
+for the null struct but are 'hidden' from the consumer by the parent array's
+null bitmap.  However, when treated independently corresponding
+values of the children array will be non-null.
 
 ## Dense union type
 
@@ -237,23 +400,64 @@ cases. This first, the dense union, represents a mixed-type array with 6 bytes
 of overhead for each value. Its physical layout is as follows:
 
 * One child array for each relative type
-* Types array: An array of unsigned integers, enumerated from 0 corresponding
+* Types buffer: A buffer of unsigned integers, enumerated from 0 corresponding
   to each type, with the smallest byte width capable of representing the number
   of types in the union.
-* Offsets array: An array of signed int32 values indicating the relative offset
+* Offsets buffer: A buffer of signed int32 values indicating the relative offset
   into the respective child array for the type in a given slot. The respective
   offsets for each child value array must be in order / increasing.
 
-Alternate proposal (TBD): the types and offset values may be packed into an
-int48 with 2 bytes for the type and 4 bytes for the offset.
-
 Critically, the dense union allows for minimal overhead in the ubiquitous
-union-of-structs with non-overlapping-fields use case (Union<s1: Struct1, s2:
-Struct2, s3: Struct3, …>)
+union-of-structs with non-overlapping-fields use case (`Union<s1: Struct1, s2:
+Struct2, s3: Struct3, ...>`)
 
-Here is a diagram of an example dense union:
+### Example Layout: Dense union
+
+An example layout for logical union of:
+`Union<f: float, i: int32>` having the values:
+[{f=1.2}, null, {f=3.4}, {i=5}]
+
+```
+* Length: 4, Null count: 1
+* Null bitmap buffer:
+  |Byte 0 (validity bitmap) | Bytes 1-7             |
+  |-------------------------|-----------------------|
+  |00001101                 | 0 (padding)           |
 
-<img src="diagrams/layout-dense-union.png" width="400"/>
+* Types buffer:
+
+  |Byte 0-1 | Byte 2-3    | Byte 4-5 | Byte 6-7 |
+  |---------|-------------|----------|----------|
+  | 0       | unspecified | 0        | 1        |
+
+* Offset buffer:
+
+  |Byte 0-3 | Byte 4-7    | Byte 8-11 | Byte 12-15 |
+  |---------|-------------|-----------|------------|
+  | 0       | unspecified | 1         | 0          |
+
+* Children arrays:
+  * Field-0 array (f: float):
+    * Length: 2, nulls: 0
+    * Null bitmap buffer: Not required
+
+    * Value Buffer:
+
+      | Bytes 0-7 |
+      |-----------|
+      | 1.2, 3.4  |
+
+
+  * Field-1 array (f: float):
+    * Length: 1, nulls: 0
+    * Null bitmap buffer: Not required
+
+    * Value Buffer:
+
+      | Bytes 0-3 |
+      |-----------|
+      | 5         |
+```
 
 ## Sparse union type
 
@@ -264,17 +468,92 @@ the length of the union.
 While a sparse union may use significantly more space compared with a dense
 union, it has some advantages that may be desirable in certain use cases:
 
-<img src="diagrams/layout-sparse-union.png" width="400"/>
+* A sparse union is more amenable to vectorized expression evaluation in some use cases.
+* Equal-length arrays can be interpreted as a union by only defining the types array.
 
-More amenable to vectorized expression evaluation in some use cases.
-Equal-length arrays can be interpreted as a union by only defining the types array
+### Example layout: `SparseUnion<u0: Int32, u1: Float, u2: List<Char>>`
+
+For the union array:
+
+[{u0=5}, {u1=1.2}, {u2='joe'}, {u1=3.4}, {u0=4}, 'mark']
+
+will have the following layout:
+```
+* Length: 6, Null count: 0
+* Null bitmap buffer: Not required
+
+* Types buffer:
+
+ | Bytes 0-1  | Bytes 2-3   | Bytes 4-5   | Bytes 6-7   | Bytes 8-9   | Bytes 10-11  |
+ |------------|-------------|-------------|-------------|-------------|--------------|
+ | 0          | 1           | 2           | 1           | 0           | 2            |
+
+* Children arrays:
+
+  * u0 (Int32):
+    * Length: 6, Null count: 4
+    * Null bitmap buffer:
+
+      |Byte 0 (validity bitmap) | Bytes 1-7             |
+      |-------------------------|-----------------------|
+      |00010001                 | 0 (padding)           |
+
+    * Value buffer:
+
+      |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  |
+      |------------|-------------|-------------|-------------|-------------|--------------|
+      | 1          | unspecified | unspecified | unspecified | 4           |  unspecified |
+
+  * u1 (float):
+    * Length: 6, Null count: 4
+    * Null bitmap buffer:
+
+      |Byte 0 (validity bitmap) | Bytes 1-7             |
+      |-------------------------|-----------------------|
+      |00001010                 | 0 (padding)           |
+
+    * Value buffer:
+
+      |Bytes 0-3    | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  |
+      |-------------|-------------|-------------|-------------|-------------|--------------|
+      | unspecified |  1.2        | unspecified | 3.4         | unspecified |  unspecified |
+
+  * u2 (`List<char>`)
+    * Length: 6, Null count: 4
+    * Null bitmap buffer:
+
+      | Byte 0 (validity bitmap) | Bytes 1-7             |
+      |--------------------------|-----------------------|
+      | 00100100                 | 0 (padding)           |
+
+    * Offsets buffer (int32)
+
+      | Bytes 0-3  | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-27 |
+      |------------|-------------|-------------|-------------|-------------|-------------|-------------|
+      | 0          | 0           | 0           | 3           | 3           | 3           | 7           |
+
+    * Values array (char array):
+      * Length: 7,  Null count: 0
+      * Null bitmap buffer: Not required
+
+        | Bytes 0-7  |
+        |------------|
+        | joemark    |
+```
 
 Note that nested types in a sparse union must be internally consistent
-(e.g. see the List in the diagram), i.e. random access at any index j yields
-the correct value.
+(e.g. see the List in the diagram), i.e. random access at any index j
+on any child array will not cause an error.
+In other words, the array for the nested type must be valid if it is
+reinterpreted as a non-nested array.
+
+Similar to structs, a particular child array may have a non-null slot
+even if the null bitmap of the parent union array indicates the slot is
+null.  Additionally, a child array may have a non-null slot even if
+the the types array indicates that a slot contains a different type at the index.
 
 ## References
 
 Drill docs https://drill.apache.org/docs/value-vectors/
 
-[1]: https://en.wikipedia.org/wiki/Bit_numbering
\ No newline at end of file
+[1]: https://en.wikipedia.org/wiki/Bit_numbering
diff --git a/format/diagrams/layout-dense-union.png b/format/diagrams/layout-dense-union.png
deleted file mode 100644
index 5f1f3811bf0056defe19abf494afcaba1bedbb77..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 47999
zcmeFZbx>W~*DVMH2n2TxF2OB8a0%|gHMj+L2n2V6yF-8wT!RG-8k``(-Q6Wvuan&S
z``&x?y1J^XySo4QZk^nO&E9M8z1CcFjycAhCrn8}66G1;Gbku16lp0jWhf|^Z78TG
zIS4S|$g&u=Blrv2Nm)`9s(hGu2mF9&FQw%K1%*ou`2#JjOmz%Sn6y;Ybk>xY<2AOk
zVKy|eGcsj%x3LFjLqYMo^MZfcm^vGhx!YLVI`O&-Q2aTA7yKJ?n1zDu&neDU0u-9^
zN@TC?98JkMnO`!qQV2dHBO~K?G%@2<7L)kra`2M?g@v=TJueH3o0}W68wazUqd5y3
z4-XFuD?1B2I}<pA$;rdk+0dQI)`{}(ApeddX6j_@Xld_kX=h6Y8Q0Lr&c#`Pf&y}*
z|NQs&Ih`%d{_9S*PXD|XctIA(Z&=uvSy}!wHn@}@a+LSAosGStsgn~JUyzgk&zb+z
zvH!Zy-}99$?40euEI3*kOWQh|I)aOx4Ixh_$o|jO|L14?uVX1XTAG5l{&O|kKUe>c
z$No8=p9S*b|HDN5o#j7A!88j#<7fHLk_kSmTw`B=f)a+3786l*hu+WhSjHKDd@Nx6
zfr%L=6;bv{oa$DcJ}z4Eh1HK%FO#L3{JvIdMCmEUTsRDI@$|lOWvV#zT&cF+Uwc<P
z*=!tFuE~v80!~L~_mY`Bw;mad92Fj!zdSN0)yK0DLVf=q`Urbqe5rNZ=szW_l3;+L
zAp7%)(S#|Xt=rmd`1dBlGZ-<;%C0Kq&=G%cEbNFVWiNd=U$phlB}%TaR1FCW30Cd@
zj-d;KsT;@Vw%hp6APLW*5t&W7HwY{LIY5={3!A(?ocW&lpZl6h3j52>$Tz|>{&V05
zHCYFBqf=qlzjyub$N0ZF8#toNooGV*XM-F^O?%}+57&93yg3r$!mwgFFA-GfA`LCD
zDPn@#^?mL+ZRcMNRL<FcD{Hx&%}o`$UJf{Fy^YHMET3*$*#V#H;TXk#+DB<VoFRSU
z@=Nb35KE!2kpu@D8Ux*rot7+8HgyTgl+7!F`1axIXwI%po^$hU%X=;6cvfZ80rH2!
zZ13N9XCr0H6GbXcUqr;kz#|C5e)NUKTR@~xvWMBW8y0%J>rZ5`YrWYt&;H=rF{knb
zVp2*?5Cc1$e`n%AVSn)Tm-pEa_u1x<)o6}L#_xhw$c7O9j?mXbZHV>xkS19;l8`fQ
z&f`KCGQ5$fFky0vTwoLVzZPEk4p^u9kQC3~`~N*hR-7#r!G-cO?As3)lRM*u<BhM;
z1ZoXCgJIQ6HBYYBqS2o7p65jiYC8>a>eO2JtuXO@`S1i^$omew=Rw4JwbifkukR+G
z^@Q{crg2(zg<_0<md`_Ld-%;zc3l7Xa9eLZ!=Cb3zvTVx_V8PnxqSaYLl%emFjM_J
zyXDXPo3kyuuaoMM&s`>d&}Qp`kqjKV|GXDFLg_>{LISVflF#G9Jz?ARYJ_U1z-lNZ
z<6OF3>(1#i=P*SG3XufW_x{T{yQ6nC?b=ploQB_FHM(H%^u4NB()B1_Upszo^15CL
zDN!rIEj78A)M(qU9(cGrQ`GMekHFR1l!zpZ4ZKQYsA#U1!FlYJ?u~jbeVRCQce>!V
zbTCnbWn=nQj0a*dWTG(I^v_!Huqk0yV9_m(^+WbW*}d15yvbgpzpb+-I67r%yZfnw
z90slEb~xhro+BUe4C!dG<<_D1dA;+Fa`i#OTGT?53*TO8jZ&==laVC&<rSDhQlC3E
zHR|<_jRnqMx&dCVN$S{nI7g0GI>C6Ss&M07BPnca$DL^A6C;ARi~@4;D$;)}Ll_3*
znMkn=?5Z>wX5_#@{nFbSGkNcDBAePVuz)7dqAOZ|7Xu@_yxpIvF;80|b)7a$V#10<
zYQi1($Q)8xKO2TctIgf#D~oRzr?IvwhNa#h_d_h?_F~U`QoUFeG24BydZ+WVX>J0J
zzFo^jOzKa(H^mSyh8^1e4z5^ZGF;dTL5!jy$7tniOE<2L0psfPSy`4AlM1Ki!}n_+
zRtDklY8k&;{i@>y=5MJlpzD1(V{TPi1y3;An)Bd5kg;4n%)6Ho<JZ$&plxuq;8vfu
z<i3WYue@!lWMY`OdwbM|HdHEwfewxF0tT81XU16fF}$4~;qG*RWm_oAW0RnH>3vO|
z?ZPax1M^no2#?pCO+7i$8?UWQH#18bO~w1)7h2*JB-H%{$!`e<eF<1bQ2TjXA_mrA
zlhrP#>RD?ae($Le^{4NZe?!fTFHr)!8Qh8;cx-X_@=Z*+<t)#`8PjAvuP5!^Q|F|i
zNUizv%vJeS{=>|#G|?`Zad+5nPxmW3Q3IZ$xkm6YyL6&(*V?Bo{uaX>W@uOqXF%7o
ze_)91j#Y)YoKFbXZp5mpLKEc(T^Uf{LaSm>4ybS&GS*Tguo0LN=}*_9h3fJT>gJf$
zvTW)XaC2e5KI_$DdD}cqt5{habYEI#bMr2z1Q%;oM=YrXtZw60R-RckdG+1aQQPih
z$?fh~0c3epv7s$>>BF06BcDUNuIBHIi#&avK%Y2{C2eH)rWiT3*Eov9?NE21?P5x&
zkUsRTYUgvh_AYI<*Xn?Th!f9R#MaLynf6~g*-3EHK{eH5)yEb2Nn{d`O{q!-85+Gc
zjTsaAs_wJEICUV`egb+2PSWUIP^d{wbbK}|gY-CTQO2wk{7YubYSXxv_RH-m3OwDh
zGiK>FJyFkf3KcTt`st(p7*eY5%Rf_1?+f+KZrZ{ag>Q#y8ZCKU#{+SbQ5X3=HW!i3
zy<jhEtHey04cqkY@3`ug{jt2K>9uHr*9Hu)JTI;H{n@_jRhvlHsnNCl88Z#US5YkW
z`zo^KDL(Iqn=RGnj?%+#WQsH3B742NcipTcIyPBt*w~Q18Ei?>q%fV~>&`_I^TcsH
zlk>V+2_a?eE7PfG&N?Jc8njYT_+t{pz$Ai<OVW(Rp!ihwBAF^+J$n$v%oxA(2-@z+
zv+7_NWyl|<uo)|JZpFoDkJ1+OC)qSBdKqGbGmUBvp6`r@EG*AVeZKp%aS?zMekP*k
z(h~*Md!?EEO7%sp6S0W}??*11S=F3Kqz}d`J?~!`v_t8(z<EZnJ&OzMuK7imLOzm`
zbM0S`-<90@;a0}$M|@d000`@f06jb)tG$LnsI{4!dOy?V<70xYAFYML^KiY!RPD4i
z+&V#_fB5y+Ti1C?X6IPhW?P>!+I>y$$NMY98kZN570LYH4zTpmbD100F>#}98&=@?
zMCe9-tNjTgj+6k5aNE)+vceG0)SUDXja!;v2jaU9EqiqL_%0<d>orY}g?&40vdV=^
zF3ftr%RHj5#GI?az35KKpbl$4@eO#bk0b(VYDCzgYrWAzzqj)+bW69mm%M6jT5ASG
z@Ak0tU#a=3RAA_vTq`5Y7OMbXiv<%~7G?f3<up`S9OKKIJoqdVKG*$FCZwb?;<7=(
zJX|Ji`@<1`IHXdI{cQqvlTf!KXY%(EwJT+~B3(Q|PA5|EjF#l0V*=)^vc22Q8y!>E
z9ZyO3M{@%hlt+ETSdN;{P;K>q?}rnfZ*N#ogtO63{DZclfk6tAH8E<z9L&{A8n5uz
z<hJ8;+h)XTzjCxVUH^eX_%igz^-K)E%kG46w7`@q%Xxo3uAIYIhg`PcLguVyEvn~N
z791;h)mrirasUqSF|UOxR@nBFny6slDB4k?*#pYj+@(8gjpOCHI>re_g=@@)o=q5K
zO;S`;ehkX@&!4Qd9Pf!F;(J3glJ<8E;-tVDW17o<!2RQ(0OFv|Dfq;m>5ifUiSyv|
zzd3`fqd<=KZRfw2P+`U_$8~9`WJ6*Z#2d*0C`&gU_N(%r17u+UX;3*#O_)P09^wa8
zfSvX`uX0-db3lp$>;mZw&I_%7?%JOKw%Y#(kKsozy<hvA5yn*1V6oW)jsLhEm`E##
z3L68-c8wcJp8Hk3(zKbi;|hG;f$g8fBg}>~Ql-u79&TCUGlB-re(C$P++WU7N_<>i
z0&a8{pot<7C^LCopyt5jgUf6(U8ba$ekahWK%z{r+zCJ_6#yAVK0a817#@Q9{AaQn
zh=sy0VbymPfx#xh!lMwX0`RhY)T~?}gQw>_2I4wK$XlVSd8ZDN;GXl{NrqEY%E9}~
ztEF%p2KJ(6o4LB&4>f&5U?&_ecjR6$zHM$WAIZ|O`Kp$-`t7Kd&q&w8K|I_SK}-tZ
zxR+=Y)M3Iw7!6m8o^PNu&n)1sA;B_I-}B(>)sj#4(jyA@e0#}x>+NAl<wJMgitR#^
z06?tgrh_SLAdo)N^Id?y;?0LRf6arC2I58i5Z4?elmY`J<Z)cDHXUdM@el-|R<ofr
zksJ8QUjU*RUF2n1P8Q=lva<kJ@&UM0JZ4>~pMi*2yl#hgyCoIn)kOUOsK%fH<G2}B
z(ZYCv;0K=ZXuoEpu6TrJE5rN#cUe^rq3aLNv}6WZk|$45=K;cP0+1+%#6))oJTQQT
zNAGL6?G}mepR#oZqm<~kwaU&~*9_@4e1a+6b>5vQ0vJ=uZ&7eCm1C=qGCG6bgFUvi
z3-UCj&>kG<ize~%@6WfP?}AZ?-p3y{tf6j(?vP>rh!YnuA+&=J0wzvDgPwaSp%w<F
zQ`Fb3@P4?;0+AU}&%OgLX|`0WMk+0$E{<O9;c*K+m~3o{xbz^!IEoG1Lf5}Yt9vj@
zfJpx1B$vagnEnh%780rH^z*-oDOvr@4@(sHcQ|<l0Y(14t~fK}5|81+x(JL~)$3Kg
zBzfGBFb%Nt?mE;jA!Is6ew*3aB9+1>ceqn>rFvUETCZzoDb<LoH%d>}(u_Z&vmH40
zd?a9g3pSviC^|?!5sHC7bfopItOoPdpH3cBL4iw2;&m*7N5_E|Lh&fP(Vtir_inP-
zRA&y%s+ssSU!slH1|Bp<U%0QeZuR$qL5dhUzg|+GjOR7$W(!Xlh7H(WGSIkIrgmEu
zpN_sp)7p%B&i7lUO55S|2QAhQhN*q27{|lrGllw|l>s4;7Rkx90T9nI!jDCSs{gM0
ztx*^6hu_Y>?YDD7(k2loVP;2jUjwgBDKP<d+S!OuwqAet4(<2~HdD%u$>a;u-e^)q
z3@)#)c1waK)%_sJjp6hYEfp$)MAmv&-vaX&ku0s(-@VJf9n__Bcfu1{8=R1OU*~dX
z9S7mPwUCDFR=8`tHDrCRZzgRYOTw-3?}E)D_<xq!6x2J2Bvq3ebjp&VtJf`gdvd=(
zD*Za8Z9OSH-ictN8zpe5X?5=V1XXI>$Re&r+UQKf0{)*ZB`e-4S->owLxET+bv~{r
zq=i=5bnFWE!S|p~;JE$C$=Y|>1?8Epy`jzT;PF0{giASdFw@!kf$fU5gEA8oRR)5d
zg;pgY`5>+-7j%_k@0?*R;o-=oXnc6>3PlU_yW=Dj6qUy@Nm2omK6GUVi7mY_q@kmT
zuGh==S4)cP5=1r=AXw`cGegXi7I^f&*kVzPzD(bEQFkBDG7&9_m70eUp~o(ngnG&&
z7i&7fn{_O0M{%z*NHl*T^up*QwSnhkQYl0BZY0b5_fDT?^L6Z?(x*yviFL38BSRAH
z+U_swKvsLRnNGQn9N}OQAl$D_@|*sp&gRlaP5>$)t3f-ij&-buj&X04z{P~hkCBgL
zCnjoKXX*(c4!6q+`$r@Pf2UCDfX%@6Cwgh8;qEa9|B0cz0cOOKYK5PvWm&mIy-dfh
z1uSoCDZ$DYraJ!t;9exqJT}&XNu5ZETn=Wx9s#5Zz>U4-XV$15J-3C8@2{FTw-~^7
zs9$hpUWA)(N{k?9^}0UJ>C|86cc!h~8%h=Yhzdd0#%;Vahpo4^A6<TKX+S(K6$n#e
z{%;Z?zBgl8IZpybgYs~!y!ljy`6N5IrMSE1iy^c|*M=MP+pX;<#}RG)l<vB*t50LJ
zH7{48$Y{T6lD@j{g>f&txZM57F>Ie4Pb1IX0S8Ut^aD8;ed~VP-TUrDQUfP=kgWUX
z#^-d~)9-EsUeI#Mr~q*eI)?SeR_*9{c{tO7<QxH}+JO#MZITX#2NPj|v`ZG_sRg4%
zCDM<^QR<#q1g6zA$qjV#qVC^ej&P)R<s-NTiq$AnkrT<}Te4FDl^-7f+WbOr^Te18
zH|z_FPuCZF%}><_+m^e9uN*PWU&e)Sq-2aar%}jYLSux6gJAnbs+a^L)Yo^MH)Rrz
z)9B@I;J&ehy*m6Szpl)WHc~LtX+d+j*Tmt>n%A6$>DoKk7_xRX5fP>6zu%$8Ew;wO
zrD55bq}A-%2kH3dr%ji$RuxUV^cuMkhCrMQU<QPk$v^}Xnf4h?FOZNWtSmW=2qZ)O
zl+G6X(EL>sUiP$ymf3nXYScDevO&%jJ_vV&9t4ggG@(0IzPK0NX4G5b-5Q7`vZd%%
z?vnf*jyD_0x9r=>0C0?gg~Lv(+{N=_M6k!qZQ;`jZohs~1b5|-&^s{yd#~cCEc64n
z>yF8DyW;OW=<Q9eg_*9^{wkm45T|?*g4j^_;-s23)2x_VuU8F7AlV4^zfj*)rky{Q
z)qS2h2p}w5R+0dm8sq}(x108qvLtW6gphhSaC4w;;xTF(uu1FFpGK8@?`+unDi}xt
z<I+!8*bGpuN&6Af+&B8>`dwdy1X>WvHh7K*T{;jAgbZyei)g3u6~r?3G_y_C232ir
z5JH$2QQFmULdbQXT##ha4DL-a*A(y^58s^<zp5<@UUm>??R8F84NMmafZv#j`GeEt
z0EQdhR#k%iiDB>Qk&#-)&0TKCQ`$?k_FdrpxxFFGFR(F^bZmDb$?y6ZDm3M;4i_@t
z^(Ge*Gjq?|zb5+ZX6z*R#O*Va!iCXS@()cm(;&)L#LJvn%9;WP@-I%OhAAOimQ(h!
zm3nbGZ5&<T05{;lZF}+bU_i?}bHfU;mnlyok^(apuA@77_ge;^Te&B}ZJf;VPafSH
z;0Dh~QIYKTnX2D=`45C*P?jR9>JFuG4iqwF92gNBbEGH=&@;f80=FPLqtF8XRuhKr
zupy7rS5fWAXf$3?(9-<WVO_}r?YMch?y2!pW9H#z9*=`m+(H0$hEX!P1Ca8XJ35#p
zANPSr$YoGxHWz&Y+-S)~hB$G(&^#Zm9o6PwZW2QhX{U|?5o>4Oa3<fc3jHOW5rE6~
zF{^%-7Pb_nhG4v&k%{bVay^(`fucF{?+1Hv8KKXq{B<q}@6x_kScDi}X@*38Ok0Sw
z8SN6ev{~*(=i6g1$D2LQzyo&<zea6E&`QLAWlz?7Ug@_H^BT27>=!I{U<#lwCZe8n
z@#wt2Rl19vUh#R^OBYQ1eZD|8`Gw@Qo1QhY1kp-a68Ri;FR$aiudGoUo6jM8TsuH9
z1H6QVRF1_MQERqs_pZ*Wj6WS<#;#A+X2RhFre;f-ntL2fxQi12x<v6bFDxTZH04W~
z!*~|T*nZ_QV(Dx@;n{^Dd9C~*Ip?x_ntFUoprgv|Xfaeq5RtAnhXO3|gNU!4$UQ(X
z*#1z%YmX<!w?}Xlih-DmdANQ$jRt$}Iaegr(_h|D>MnoVpnc{zVN{vG9gOAiFt_}8
z?*phWPnEo9PtopAxOP$U434>Wt>Zn0Jl9%Q02WkvLfjnN%E}&owGfTHEsz}=5Tomz
z&u!;}uA367UZ$P8#3TEibm1cbhb{)5W<lj@*-YXo-1JiG;`==hr*cQk51f!q4h1-}
zQ$Y!AT)!8H8E%UnTNzGKb2p-s&6EyWJy$J9$lV4djEDB<(M2Xw;uoh=I(9SFV>Id3
zr+^$SygHP@z@?lw>~0>=utdSye@$VuJ5`26KzuxR1I0gK4VL|Ix{A_PWB$doclXb@
z8dEbI#5ZQEU17x9VkBbE)a{j2I4ys6;e>IuD^O=|a4NXx*@%7(6Xrqx(sk5K+*8CB
z4$?^jwjk4p?g+d*{AC2AiYb#sz16L-gaC5d67meuk7WLFzR7HbnB~=s1fzSEyM9P~
zxRQii)+y4l>zf@?a2zJYy}8cN*m2o&PV3Jnz6T*`%^7uv5Zv5sW&dbASn}z6x!6r<
z7i-bE{Y8^i^gLf%{a%wPyv}=)28J4vtN&1}f`0)+fzAR$+&^8RFZ4T5=^G$odP|&k
zBN4i?e3QbBgv57qK%}$8sFeS;Uz#+btc_JNh8+g{KZTz<2ZTd_g1+oD_K^3LSA?Me
zfa(cwSUkci7?4D*FAa)D5})WfNP-f&TpR)0Qe7?z2U0$D9eFP1CZs>fA3p4HTnHbj
z_V<qE4_6CY3%2SXHz$J;5}rZx<S-<rPP_tVt;$2Wc?o!4c2Oe_8?P>?TqwXA$%9Np
zQ!$#C_FqPMnVO6Uj|BySupt8j(pet7V`-seCf46v6&m9hWS*-9x&P`Pp~9sg^9;K<
z|KbFBlRuxgAk%DqiTzKm3Su26c%iB+5*LMkM*S}&^6w>8G=P_w8qXUB`vHUk@Nq{3
z9smMX$Y7{t4KDX*knot1QSBny9{o?kF~Fd-&>q8)Tg_iID2~x!07tD;ZG!klKJDd<
zMPB4Xta>h91TIq=kJIP9?;x#tTc-UL5dFzM+;i47^lC=umJ@lB(IA6ba+!o<2F$Eg
zMaz^PJ$M{+Cm6Q(rCk{bO|&pLq6Rmo8^rcbAa?5P0Z;pGuFfXtoT_%L9&8i<XOQGY
zT`gy-H{kGe?T<U2hL|bkL!NzISeWbY3ckSf<8j&q2`q`iJ38eO49*(oou5_4y(I6q
zb93KEg%bYwlqSpEXcD&-J;3;kp#<28HG<R{W_pp5q*(iFD65ZxAAX-7E&0q;n~G_a
z>w0vD<7Ayr!$I!9mE{>_dk-S5Gf6<)QxbPywS1sD#B-*}b1$3&A6PW4V0U(vy%*pX
zG$8)x)qKFhtO`l)8qQ$2pN~l@@@ypNd*1}bR)B==m#)(g2(7OGxMgNgE5=qN22*mW
zz<02IFbBN*0R2zEVTTd(IGO|@F)y~#L6E;L@DDN&{S2%SU{h$mzO-As1JpGL=@Daq
zeW=yYbqI2~z4)cCX!JM<m<Yh1#4KZWO20|&&xG=Hnys}AC^za)pp#GMl4p+keX*yr
z1=eC<JxWvn<UFA`fF<sJ$2*bwll&Mc7Z9iSaVtMw0m6gZ8(7=50#PS)$?I(BoAb`<
zkgv-?npOEu<PRo(+xZ3^+r|yBzCu7PiUwNGJpGTtiTOsSvTIbn{g~n`pl2Yh;U~zA
z7b=>ITbGp8iK<&omCj&D5_i<?sunPMPnBu`M!}%sm1$L3E}0KxsBeHWSA~Gq|LBHQ
zH0|m+Accr$cHNzzQ?-ZijnnjU!3(Z)L$nQI6kjWgLY@l<-JfI0k79|103HaBV5Tb!
ztE?MPUV&VZ1V?PW#Ag5>G^jdr^ppY6ttfay<!z-K$KpFSVp*8k!>V_;mF@E|p`w^u
zVbnW1siB`adcny}3~>RXh!#aJup=nM$xdM3dYy>JYg<#{-{E>lS-dX?FpyBsJky1B
z9I-Cy2KBO;=BqVOzMejJMvt2ZPx-b)or&EC<cJ_%?}3yr01jB!OpE8$S`^=Ke$sQd
z`6``yTQC4_IIiwu?)ROAZ&%*82UmbkfFGcj0368<Y`%l^Z{*@f3UmbnU@EIj!?q@}
zX3*69`i4erfCK=+-skq9UUj7W3$M%W_3RC}HVUYAKokqU7Py)}OEu3bdXpEtL5`@_
zpCb|gp)djH@W}pfS)})^uKt=bn*aM-E2Al(1Nl99N&;LUGma$Fb3RwG*iLo=qj6wq
zhjG{tZVLtGXG1~{|2u4oWm@0GRS<|a5&<=r`6CEdmvx5Va!H6P7?26o@d%VSsx&}P
z$ZB&cDRUhWxXMu>&&pIW?)m7TjKsDN)oZ>GMw=ytKY){67mUBfhsqhJ@s3<*&-T?>
z0sQxuCnvq6SffC>7N&z}$uc!XRiW)d2nO-D-9Vzczxb81jXMjCTy}Tc=JQ3<Iu*Vw
zY{aYgcP}Oy*9+XuFF8u-SwNcl8O1Alp2}t%Of(HLCN7@`FJ=&2&8HG|n~^FL!_=sl
z$GG&~?-cwc4t_##{L9TZdFX!jFwM7YY>0{sUi)u=!UneRw-)j84nUqm&6zN3i5i*q
zc6&Z)SUniceZr<$n<M5RZPqkEZIl{u4p-?`g`Yf=ERqZy>VHYn!XFWGwlpgZpl{!Q
zR><tX+<d&-5+br~AZLF%_Z=9(5F*rKYZDH!dcw~y{vh?_W7s%#+WF2rsZ$dKl0BUg
zT$OYXe4lc>fS5!hgmZK|oN&Sj#ei1jOSh@hYI%QQ-{Fr5cSfjN^YYz9D06jnXB4@d
zt)AOZQ;_TgJu^GLeG#+JW|M-?nsNXfkB{$oEaPG#ikZqsFw2-Awn(p*7L8EEK*1oe
z?VrpTMKEnMWUT^YVEbiR2&d(9f1zi>x~#0{4HBif8ONpx0}&(LVg{l#`B?C#VGzFC
zLFF?lZ2*+*XlWi}a^U4PNS2TySql09&yt$?EZ{wmu@*+z`Jc5v$6h|olhgw0SSd;J
zZt(^DAkY=cOS#nyf$Lg_P_FZhDkH}7IpE5vHb754<!dtPerB$dCR2@=Bx5uP(sY}J
z2f&r-K4yA^>s&cVFZ#;5+(>tOcb_(>+F>9W&wekr{JtL{ae_6Nga1E|+J3On!ALnh
z7Qx)K4fZ)oPx9MAfN-sq8sZ)Doc#Mdk6J=%Uf~^~R8pOTL{IDEz(@ho+I@HW5|OL4
z6jNQ{Y<e*W%fg=^lXiYybaq0$Y|S{IK6im*6%9zy89m&WzS^rm31zatvZU9qvz`HR
zx-~&_FAllla$L51KJTFWYpeF;avQQ$beZ}^v0)N{_nHsTPvGr~g)12e85N3Z-HDIj
zIm9qIT=zBk*MK}tA2+KS25!0umfmDohDV$X35GbwxRJ+Z>>GYphhGMs{7=89Bq`!9
zJf<7Z+W<#tZ@?~i@mizQv6ypQR0`+SMXxCmT;(+npetIaV>Jemx68jBI;P<yLmR(M
zP<HhIyULhhE$&snZ$MsSR|HUhlyZZtzCl=0<o&O%dk*F9RM2-_WbO<0ohX2v*qbcT
zpmHh31e1%5sCp`>%)yvv4NReg%9jso_^yVDRg%PG9g;~j@+<P6bZw92(M4o?qR!6%
z|Ax3$u$cMDdUBO(sE~%7LlQIY{qLPZ9lC)ZZ4EtFt-ti{K#bfDq@_3!BHHbm+8AMi
z@Eo0}d>^Y*hE|{aqF)K6Ag<4eWMy$;oI)nye>Khf6mtQr9CimLoaK|R&m=YTa^9~+
za)SZ^ukp`7yk<t8-`X=B&fq~FDoy7J#TbL0u=We)v3)opHePUF3BqeGCX8@?T%G(y
z)dxry_nI-C?2jAan!cSVN-&KoML!^v)DPHcN$F$?C1Dr@=HCFSy@2&gw{4<3FZ|X^
z;(7e8m!PeHqTyP**Hg!$V^pec<T`wgnj(l>C{Dzpj-T>`bIXPrzb8{DLNK>%c?Rxl
zxW<c<nLI$$B9774?gPDx=%vN7O*yErIjnwvv1@JgjA@&K2ldCs6+F7HZq{f!(6(1y
zxwJYS?r%djRRIPPCP%O+>bvbz)`a~;)xP2n$sDeN!u{yqAr3l%KucgMYv@uHZYZcK
zB}TTr*y5>KzP*Mn7DEJ5gt=n15<p2oa6unw?$Brn5XhqN+59gU2bPDcHwVRssYF+E
zi$|-8RRg_)uy7X+nYiA?#`E&X6IS({3+I}ieH3fk*tagFufO2^E4ly$N}IkvFb)V%
zhp?KzK(zzE?Wm@FyI?25!RA}z9_q0@Wa3vboOrmVtk-=Fx^O1>@lr&Lb~d1t;d*hw
zae#?qANKSa180KiBgYhfBoVjkC%ItG^&}Lv@c`g{LB6Bn)fYqFOY9_Pd+{X!Oh6rE
zL&(bAK%n&UBZ4kiN}yj;`dX)L-a&Uj7@#DO9DLd+ngx-(Kgc&#_E)){j<Rf{Tw@Ob
z8U(1Zq3s4iwlhLy*@yWC2j`n%z9Tu+-bpPCDSiN`!?HzDAj(e)Y$fq`GlXvjgr*v*
z^y6zruF^JP!de(Hk+NFy_3)q`DuxBx>*sP}h<209p6k+wD-XBG4V|z3Kp`^`&XIxl
zND%*n%a<RqzNnR`k7Nm)EDm_vi2#uOLJ$bV;Ul_CK)3h{%h#nK(AHmt?H?5h-ES2O
zL_)94cW?d$_hgdb=^bGUjlib+`{||yA?$i^^3y+qgbGK2*;z+W=>D5!fd7X>lBtu_
z67m0xOAOrD4~&fP|6~N%*&t<F<<N-u&$v98F<bM*9YrZ%ngU@3?~Xe?4;#$^@yxf8
zpfaUt9Eo71V4<e(b<zzm+UqSMF6)Ld2_kA|?A<yD!4HIGCACZnbZ9>y?wo*(kCej<
z;&C@evfC`iJ^?t&4Juk3)_NWvuG)ZP@mz%Hjq1sadG?sVyHBz}JM1g#2N=b1b1*dl
zustpZR5|mzVLnBc<DXH8U!jW}0Qy|N$+<|x3Zy5umj`<jMQgu(K*`S%NJVr}iMaOl
zAAcuDXn<8T_?igWx1j{k>xp$XbD!VvtCpxAJlq~dOIM8WpUHW~VfK~DhFSnj?wXZo
zGI@TwG2nB1YX{N+Eu2BxGXPJ0+M%E3d+fP0xT#|3HlI8L%PX}Ox5@U23gZht3`AA~
zp3c$}2u9W*V*x0*641zxAbkLooCl~y3f+>uUyoNT*Qi(&yxmt})B9LEiNZC#3~c*o
zyl@0ir;tD?sqN-&O1ysuWSf{^Q7Zu1&_=K=_ZuA6c!8oDDp3X`65dE4I1H;5tBUAz
z8uuVSMAmYu(ZCcyC^Nc|@}4AJ7?7<j7Xf)0<V+f6+LL_YAS9=A1_52`p`z`9kljSe
z5`h`{7(_V%;5oSF?K?p+m1nOG;R=usp5x2{juKx16iyuhVGXET9wA(=`_rKKW`QUV
z5&`Lf0V>D7X+}`R_XC_=L3QsunDL{QtGCv=+w*`FQZ>|L6z=4FH}OKa3`d`OK>16r
z+&jyOFBrU8kjjguuOQ=UmC^%NABZ{Xx~;?*j_nst6(jSmO7NiNY7w`c3U^Xg+*Q@a
zTzDBwOm!mJzB@qR-8mC+BG3Cl4u32J9^s)^lpI3}7TO%G3Z!c`fS3&1cA9ZLjTU-n
z03@Z&_5S(dyOOeg+#q;nm^T=s%5PMu)S?mY0@?vmegF^v!{w(mZIHJKi&izP&Qo>+
z_c=gZ`Cuz}vKe(D$o5WC+w@GbBD8EI>5TSQiU-~I>~P@mxZGWzOz*)Gflz1#vR?m_
zT(&;@``fE6eI*vnN~lEmIAWrYI$z%b(ZTbp?E-JDDjQ{$?afYMwgpDOJOCdrvOmv2
zb<1-)Z;wU`{<er<s%y8{1j+6kD193LB~3TMZ*7qVm;^u)R|1967<#Cf_&%46v$A(l
z=Lv%r$jQpN!w9oJTx!jlK3=YNN3bC-xz5={^K9Z5-kjlTXu^hz2osfnP5(^9AX?zE
zl9V<2cJ%iNs~G_}t$q|aHx?8a?eOlEzV47oq?ga)=Xn_|{s6e~{i!ON>npGTvhh6G
zRP_$rB?qIB(y+aBZdR~U<#+>V)<B+EUtdw*fK&mLBg@^D7AqbP#ljy+uQBcJ6*75O
zoO}DT9C`?*(0in{CO5y#x>^J#f<$65au^2To#w^oB3B^a_7SuyFUnG3>O07NXX*r;
zx#Z8Mq+Y&H`2mE}cbQZxC&Sf9(P@;0gYe+Bs@QkZASE)AfubG)snu|+#qZvS6_DZO
z@4Q%=xh%8uAo*}U_A)ghyGy}f@!KIwBm(9vcP(CX;*7R9{cX4+-$5-1c>c)(t+Uet
zB-*fVM($_cT2!xSc8kA%VkS$ZtynDO%E`e8(qvA{_Uf5Sm+uWJvTF8;Cm@%i`N>p`
z#N#k3i50vDK#|b-=<5?xJnc^EWL<x2ZmTIqS@B<@)Iig5+57^j#|kH7P<bNC`kpcT
z>0H_?2a3GcZ5JQ|g;l|!gc-wnQEQU_TRev!QLNlSq_=(7R{+L6vr>avU>dS9SK0;G
z-HY-^vYm>@QA`RPPB?qhf?PO3720p)fsfIxU`zLF;3NZ5f!ZjNWihvp%2#fgA#3wz
zEn}a6I!B%4wN=%f!j9>)k1rFi<A~Mjq<@ua#iy~&cmw$nQpV?4{c=;44j9*Y;>Ic9
zZCpFWGDI6h`f9BEOhCqyQ1E%IVY%K76kCwb0d^h2aqBqb8!}9nXt4bH;%0RPv}cH=
ze4_$OgAV&}rR(I@2#G>$RK8^&JqkKY{jMYl+nT047{zT{E~Wsqs5T31`?}ho#em8^
z)mKVB$Qbo%HPj$4#h2J^-l5g~EXOW3lv5q?6#6OEjicjE;*Eb1GdvR#g+C&-7)AB6
zFq-uDb)0<C!gKbOPMkr0dr)D<7Vr@vo@_9;l+2Z%_`pOvLqq?=qy|w4tuBZXxHEtz
zOmJF6cIOncF=kVkJ~gXkweqMPz`!b%C|KGIF1=X}K)XC#NUx!zkxQ{36?sbDTA3t9
zrdehrMh2m9PR+j%isdZBGB+&4EPj1i13+~S-w%5Z3_DXf+|i*FEx8U)tx|Y&gi<<W
z1+7;D_Rhoefz{1c^7bO*@##`qSX!E)A|^bbl%eJ5ItU4mm+EtieRAs47?R3g9-FD=
z852gPnLAaP-P>j88s6Jmp*!=9HHz;}%w$PF>m~5cTQ6b+;fe(Jyg{LrBX89ODHu&F
znX0rGVs}B`9U`b+^$9K(fqf?H8?E8xHwupnKP_j{x)V%cfy(Yxf#-@GI~8@NH6BXV
z0Fezu5?|++-7uWZROje+G{dV{Wjy{~Jb7D|B8~ab#Qsg95kTZ`*Qbte6DLPfWf!TE
zevHPVzos_|<fbcO{mk0eFDQ-?`6*|WWt$0I9QC%vCL6Crwl?Up+ge|fzrWbh6xZ(@
zfQ|gEJXMS@1~3-O6^)yOyl;S7|B-qF0T#O%kEQuoG#wvRr>7k^x)vn0=^CAHLM@TP
z8T%4K8RgJJaA0-T%Z?Zgf%rOS5G-e;WLEhTFAZzlY_;nB$!DPA#%toBJN1!=yyC;R
z3r6mI5tE_Zi?b4nXdQTSR$QS=58Z{A;I*IsdfLb%0zZu9t>80BQ4kBM@YTW)umhna
zg~5K=P>rlX$xi}A33h%eta><q2p;hkS%}^@#&Lt24w2vng)lE+!Pp?zDg|+q75!f!
z26-9Z)^RWVk$5}qa@i~y#rJh-=d?HatDl(}HkUQ@5yRXN2LxSz7i8&522oOHRTlMF
zjY<Fj$DNxOYOwSlp`tbii{PKZ=i{ohX=vK5{<4oI5nwHB0I1+OfioIVs`+~#Dl7e8
zV5ucPMJ3)|SpS%c;fUOn^pYaNYW9HCd#|is*F@yWtJRv{AF?){&01J&q-<sY&z<dc
zipXI^Ab0CHrDYzpsgaLh0x5Y4&PkAmt)^U_LXfi~W!&eGKQ>)%i7=&BkgZZ85cFyo
zVI5Cyv}T#<?!?Ukq^U20(D$etP_v_5XT9U~W&OufL_|(_S{EGw{HbN0U>m-MNiEZ%
z@(EBZq@KzXhin~C|9(`cpcuV#PE{JC$LY$^mxn!jW?qSds4%sN@2HvMySl0x0m$N&
zgX7D|@dAV=i`B{qxGrGl)>03NU0|t+j7Gm!>IW|rHykdHk(P?EML{vpTpJokyrO_h
zA~3sw7T}60WDol)j{EMIAhdz^19VqTETv>M`3x+E*OqZi#3TlTI-p={SZc0*-V}CV
z%!AaaMH=qex3XaXyY(!vq+DLLRpl?Y;AvCMNqsSzjzD2a|2gE78OEqms9^fJx=j8?
zyd5+UM4*K-i<1$;*s?^=ecLO;>#eZ7{C+&zt3`h^MzO58prf?%61GyeM~`TGPyjIv
ze~M1@j88|KzdOhbA@V$og?K&y4l_aoP{+~*ox<BDmOtKglF`)8T2NvXmWob|`E%xN
z@tt&G#ij>H$99obSJDy21}%(DHz*Z~B}ugxW3m7{c}w^#Lf8b>e&5gqZiy5GYWySu
zOYQFt83mh<9-v%Emyd%*-ibP?%gnMc&r#6zgR%2B4|&pMS$Q7<Y<=dIbLB3)vkN>%
z*=BJgpEX>0|CvwW4*s&oF;BfV9iuQ0cXV}jelh6P$-#s&2Ee0yr$`Y{avk;+OOk7#
zQ$&R%2vG9kVg{42^HnX@rN{K^GiCjl@EZ^$JE}c8I=#d31%=mn8$FG-Q|cqf_52o7
zR89OSP6_f}m<|{An<wzdl^p5*A6{b^u_!<peZ(9TMwTSe=I<|4TA_LQQzuc;YX!k}
z7+%z(VI`O-iCJ%VC~SCeh*OfTfLHs)WN}h`vE29JY!Wl&eHkZlkQ0m?bXkT@WApjF
z!s2+Y@wymFppbmnXuZemYm_ynmU5M{7ZvO}4>#t=Vlmh|SWv?*WGy~Z+<((4N&4Ys
zniE%-AeiyZI?}Rsz^)o@-On}!GmOB55WKCVGfuo^VO`!BNxXn}7iqe}Eukf52O=e&
zU2tFLbapWBa(%FrqvU$NCE*#;)>yW6Gt!V*w@zTLp@V<UXp5(t347fLS$1rs3u&Ex
z)lx6<!P*k_WDn_OV<jG<zD?+>45D@zETUM=g|!ww?>qNWNeBPc=1tG^%OH6#4?u@%
z!W&(O^11g*W=ed;NBuS!sx^;ho!#>)7<H`}Ro`ZK7#@$s`rvS>t*xfS1<na!)M@xf
zb%BIe`|=h%T3yPo98_o|ZX$J+?Np^lB@cB5(<0u^A6m6WMWKwxZvJnIDZ=smnh}nf
zz6b$UMiiaZt-fQ4_6TQDt+|IWY^$~2rGiFCa|*BX_NUGik=bo9H;3e|y9y;TAA_--
zQwQrNnI)l%L}{ykC`^a4ZFJSZ;4MDSh#sbaB1%kvsU9+WxLYSHr~3%)Y9LuMN2f2x
zLV*ji@rQ!xf_=3|vMKv%cF{oq(|4N=m{z|~IJ{qLF~k~(nQo2^jJ<0%iZ3B;s}CJE
zvF~Zr`{sI)`1ScU_K!PNsj~_HK7ZEr5_v$ZAOQZrQRp3<7?}|am@ksTo(43lcfA7q
zM|OjbAF?JWZLg1Z0pK>b!!CEH&dYcq#(<+td!)Kv31HwWQyer&<X5E})+H+zv|N?>
zA%oSZt!kQ@Y`r%V2I0(_E8WbAWQk_YKx+KOmh1xp`s?I?XuUlg&sZ9T45^Q2U-qos
z^<p!g#E7_dq*s>sJ>-Om)Uw~}uJuGR{qQJp;&DPq>4ULP>&tBjhfojnCr(3kA5S3}
zzJhN7>7pY3+v5*y0PyqVEeu-q@zAn}XA~NG`w^+gp*xn-*1?^ejC?|#B4VsQwv|v?
zL%SU<C8InsXd9$4EP-YUopVwR^)v1aWdAtH%(Y;E5@7ZPq*WO^#(TjSb$wIh#2e(L
zIUhtyw$xa}r^Aae|F@g~AYlG4v-#yX%2zo5ks1N}fXLi!UXJ@8Int2|(7BTK^fdnv
z3lvIT01y4I1ckqB)UvAFzjQ>-HE<=^*^Sze?w&uNzF3g@|D_H7(i*}ifNP)=bovUE
z+`pVf3?;br|Bq;n|5qo5k;@NgH4x#g+g9!1FW-Xg3+4z$SS^u_lGS0g3owthwhO-#
zHQt74{hE0GXCnRf5dcFImF!3xD-5bib}m5`DJt(aSs06ShRENlVo0GqY7j)g{h2Qf
z9P$^HLR3oU)d<{Z8P`@$R^SZcXp}5L(Id5hFp2`~gODE})gj$lAkDuAObp{YpuD#2
z7MCb>hQ|IOkA#)MO{M$gin7Phg}pqM{hp%hgIZzPk~gceFHO`UASYo5foK0@piap~
z>3hv?F-q2z4C*%fn`lm`%~bLaAw-t&EAV71y4X6Xut`ih!Ph?v6-mQbHdq%3^T01H
zA+?%>XJ6i|D=dR20dh*`bd|9ljyWhF&U9PcVaEAuNTF!Z2lT-DzR-5lR2p@MlfMpr
zX)`+^%><@{2q2pLB%_hQS%^LReMD6K@v4wLTzCil8qh+Z<oNx9%P&aDD;K>Ot}Fg?
zAP5(5NJWZ4%vJ_spvoC)k#wBUVz{-8K$gqmv`X5D>x)ZEQU)i8kwH2YyKH49a$zup
zO$hkiS^2`uMzU<=rA+A1->Lm+f(!!MM}fVynv*WV9nh_4szJGJ?{BJDYG$WA??01}
z1Y2A3Le`)^OBiM-ojaS$Ce1~cLKcQrNe`mk`#oaB^lbPrV$-X^0P)?#B9sxRIlSvJ
zuVq|P2$LZ*od79S*kB#l5R?#00^FM<g>2)}Q$o<WAuNp!_zY>eJotax)iy+gQMdbz
zvp8k`*@iC=!QPcIBIHE)_W*?U{O>dVdp`c|E?^&OGDg@QD64>Seg4v7l)eN~E?+-w
z6rpCO63k9}40s#BqFRGCt^J<`vUM*&dnebsi7#NYzgzhp1n9l#yQ8*EVA_BMnJa#l
z-~*~T6oDEK7{FREQnveQ7~DJR(|1U)xP%kLvvWYUXL|WIR+@^{Y!Kf&1(Za{gB}3?
z`$w>SfN*WR3Y3dRkZr?qbn$Q2j0mnBNYFyY1DZ(CC6}$8G$M1!P?mrvXqwEe*@)By
z+^-(k;n{$t`OHnH<_^jXih-zFtlFJS;s*$&<9Vl%dqBJmgSz@s*kr95v*&<_2J)kN
z4B`ozz_}7@tEiCz$S^wso{qnKR7hpt0Q^RIjIGI{g-C}t@Qr}rdv<`V7zXIk5E?p6
z7zXvyWx6IH)ByJI0}$oInXm-OL1kU|vM5>wXki=mKCOYM!=RtS`DeS|lh<cmTydm*
zfOoqBj&AIA76%rM+%)(xpk?TiiJ+8x79WxMacw3?<cV11quwJ&Eokx6gvA_$)cKEe
z9hVFM=?BoXD2fd48An1v>F5}vVmRo0tl2(D_6`6wHk&`XDSeY?fL@(=;|sE&g(s-5
zmH}N|PvJEQ>7j(=zM%7my&@lh2ao0L9pLZ21^>P}^4Tpa<g%Tg?(SlW!LD>zQxANY
zcOI7qbRQc^8KCw6RqTnOJ(!D_KjQOvy9AIBptvrEU#)!vR4;dkjL}<@c3)_kucM0I
z%gr^Qlc~rc8xZiF9Q62hY#`gZ2TYxE0WQT`|AEIZS&sd50M;!&mdi2(_UX2Hf9R<N
z%GpO!q1IY>jg!1o4s%p}GkzGQdb5HMgT@s|H$hRsn>>lv!N?-mmC-J(m+lu77V3A$
z4~Nj0I6gEFn0_zrdWh`0)a~hDrscV2Oh5o!oZ$h^Q>oM47#kJS9njIB%(>sdYu*S#
z<mBV7<9G*4>ptlZq#e*%v<4O27jK(Do0iYvPrPN>2T(+FC60Tp>@R!h&OK|92O2^O
z=$3$@^Rh^Wo*D)ei3lz+%wvK^Eo1ey?<VU&mdfb^EdMcJ(cWiM2BPt9=ajx<urL#h
z(n2_+A8Z48zJFbc;)Xa>#HJ&tR=se=6kC)H3V=m_5zTk_>H&pj@sXwE-|qtGrr!i*
z?ror*ar-A|ztk>PM#?j${tY-dhkiX0BVe|@gjS^(C5;HK68^F=(x2IyHGqJUY-zi}
zU<u=u%GR3cI*XVorvAj#@5ia}@2y1b>f(Hh)G?|&wpP%v8I1Z|CJ$6^(P@v*a3wHS
z&O|Xc-axu}2sq5T+8!?>K4%x)--k`v1F^-6-Owr;2lkTrBw0Hpp}`_|&4;ukgd!%q
z9W-V)S&UIA-)MS(O67$#hvK`%>mTv7a~nY0-HMF1Nx5fNXb(T*5O6Rs=3B#X{h0U_
zR6F=R1D()sz*a}FifX1er;ASEbUU2y+ly8V+71IfM8@9~SMMS&G2{PMzO9S;a(XiK
zNFb6xXY#oXH{)`=TVn`$K9QPSJe^@Jkn?1F1gJ%NRRYrE?=d?*6DS~3LCUlkF{`pe
z;W%9jQpq(6WD-|-US8z^rKLuz=L0B{S(_P=dcToh`|9ZUG)h%g9Aw{*0Z?dZAgYGE
zXsz^E)tm*kFR~7*hY$^{qu<3IIDz!ufqKT>pB9sO#z>{*y$jwEjBFBPBVgbgy5yzP
z1UlXE45VRdUJZutKpo0m&O0gYYZ_nYgA%x*`E3LPJ&~=M{LM%?oVqKk7hhDxL(Q=$
zJ+B=o*g-?c=c1h@NYRteKoZk<LVsaSNfJsnf0xHzm)DlLtpNdz-4u8n46LWN{Pd6_
z4DJPImHFM2#B;-t=VlUzs8U(bb>)?Z!B!cd*Xpcur8AjjIg)!3U)IL<`Ux68x~X>>
z4dyvV<MyYBZ6;&G3hGb7X(HX^VWeLPvd0L#jX*(6;Yb$U2U;P}B_U_xt^nBK5$tsl
zbDhuQ$=`c?*85`T-M8HvmKyf9gKaD>1Ov<B=CejRQ0)g+TK(Kf$Sg_vI{a}6I;VWu
zAB~N(<e~iJl$LG#PS1viseI&4y?|eyvT?;Rou)Q_-*`66rxN(|H|TW0-Oc>2Lw=$2
zx^_F#&5(*@p#4)o#5w#l$q#?4ZQHcVgSi^$-gF*RLov3uTwE`1lUY*acza|APV<`j
zsM<C>KyUhMCQGs&*f-|I#g5DtIH#(h?>rtc>cySPSIm&>`SzbqFQJRP>}TXw!Wf32
zGNb!n)+{qU03TZ5&><Y*h3S6g+4<vlDsT-9u3MG(9r1yF(}$}-t(oKdavfOl?T|O!
z;T8{(U2O^7i3u(H-FnC2b!pA;GjIL-EulwpIyLGBp;w?-yhK&HNT%I;Sc9v6tEK`o
zgYXzy`f=uXF!+`JyWnJCchMN_82Z8Dr-PS1o`eH;Ean}c9SND9)O{J6G5tDjnFo_Q
z4gkLDTe^53hmz&JwyThifD}9Vk`O#9QKD=NTeP5hgDiif#kuRz+F(s#%egwHbj%x6
z`=v|^_UYI?8#YwKdGZEHwPTF6y<E?hf^58aycW}{tP@+4&IP1oe(E>14U#1NZ_i;D
zl>?oax669(mf7f%bsfXF1qD|!I%d%aYeLhpotgSTL08LHjo>IHdl7W!Q|7oAbMlRo
zJ5PuNDt#p~r4^MCnZb-5SJeY0jGZ{)DL;LrE9h2#N&QyC+H!M&p#jntcX-n6J~r7n
z7uF73Dd}%x5rKgDWye#h73hdI>qB8Y0<YrrORq~8-??-er6a@sHQD|<;B7ZcDF#w&
zmRD`*knHPElDcsPs<Srq-URw1dBPCT7_S$FYk2Mv;<|FV+9{r4>&{0IK*P03l$&`M
z0M_*zXGk749%(j|LdOoIFml+K_d|uFciSg+Hki<lfh}b@39ei<rTw1q2wS?NI%sco
z+@nupE+l&BGdMIQI&ATb2iicv$u;;BT5F#7TFI3JZoIj@Z=W|NtOqV^T)PWJ;y7z%
zk9Y~cv_A+_VQQp{37FGrFqTtCkoPIH{iK@43R4<s@}4W}GHmkU!nZkPq!H<6{|}(5
z#>7nR*A1^IdIwK@1w2WiLaX7AYfwvav4}i8LDRAvcXKxLi}w~5u({H%IM=H+O$p0!
zIOP`XlGF_ig(R5VJ!fOi%x&7BEd2SP*nYHr&INjU|6|YnnhzuMhcce9tPv6K@H@lw
z)gi3LW&h22-Y}R{I5Rt)KrPSeviHp-MqidrFSv%NX`kS~&nG5gK`Zr<>si9nMDk!T
zq?;w^G?G}8uV!7U?g8BnfI?emU)LR-j7bm%-sd?kQ*hKyd#{Lnb1BTU94IDq>QKdt
zPh4KLrWw3xDKSa(Gr5<8G<&}8ZRMdwV(+;|VlK%*7+8kI8-M0R$!21wMTK9OVFuTq
zE_S`07tJNu$1TOvs#=3b^;+pMZSHJF!-pyvQ<tJ6MUQ&s8MBx7g3sf<CXF)sBr_?N
zHPuO~`6&tg>^uG7yrJHnZf}i`zkaE&`y<g{PvMfHx7O`r65$QBpXyo!n~3wOAis4l
z+4lfn5Mi_5A(_H3z@5F}mme|Mm{UUT;8)tt&RdU{`P?u!fGKeKJ9n)-uH<|WR~kLL
zh!EBU;w7*72GSbWQK*cfy&;#Dnwa0p^UpyGVfpyg?kbt}_|GT=JmVQN!j_HP-UN?q
z8DQP?V(Z(Fxmu*3Gd^_d(NsJoG08ylO;;_wv3KFLTjXzzUZ8Pz{t!a){>7u&8@uKj
ziVV1s*X`voUo{^ncK8}9ztk7cYlK#8f`UnHIZIFV<H%X&8|M0UP_9Xk<|kGEdzl?;
z=lcw<+@wrcw=te&*6@vb&$C>MJSSQy4l9d&8=?t%9mT1UKpZT-C(u`=J3}!VJjY@t
z2A&FEth@^KjC*6;$eIQBPr<jYw2yYBC#W?bjBJ(uR@ok;vq?c(un(IYp*-qqdt_xX
zc%D4Al7Nhb<wTfy-f{XhFpln3IQT9TI7+9ej!)gg9fqdilOG=Z)pBLNN>OziytC}h
z>8yeN9E;6HYa$N5rDze?6&70Dky&$=e_?wcD^*s3P^PV*4=0SO9hVh1YgF36&{utb
zGu>`8GvuTnl<y5ueSSv)CVsIWImWNn>ep~lsd8iq>?+%U)2#fG{XF5PtB&qwV|G)X
z8K-rIxEoW4tdZ2KLIfV_1~Fs;X}CS{`9kH2vt29)BJql-!p7SboN@<2uQEq#_KZz3
zHK)ZTpJM(3>}dID74{hk-_bEcqQKm2CT}NFM+KK*xYv6uKcw80KDpsZV@UK%5X~P~
zPVs|e1ay<WGKv@CO0nPPYI;_`)i;-#e<%0qp{F^(z#%AnR>alVQ01jz>Dy*$3sX>?
zxN%Cw;M>(G<JMj$Y__(#mQ;cCy|0L%>BBQP<%}&Si*Xzrx(;fiQ%1GZArpTjqCq9{
zS2d-fI`oaZ(<_$7-w!Z}K8#0?c%kKwsK^wxuVC0wp4!GCN5!gQ>F%edICU5d6%ei|
zq(n^B=+ZQ}ORp^lmc3@=&C95ylX%hFgT$b~Usn~lzVJ~#Q~D<LSiP>?$nLzt=bixv
zg-enA?Kf3qPJh{-%$$wT=0@8e>>G)qM??jA;`o|YuXO!o&j&LLkkW*?+zez0J`k~$
zgT{ZYN`!+=(Eo@_cvLrNutVIWfUR9DM=_R{gWgjQyS9<e7rFMve8{|FB?Bo5Q8W0*
zXEdVHZ%+b@dxa!f*^BX!$);(7o7uOIgmkx+IlSM@Ack_m;Qd%kr2Hj)FZXFxKk$9V
z@~T?0`gNp6pda2BL`3GXfQxdY<$UJf@Q%e}9?dR9%#OuolR+8P33f#Ww&$J0ViUA8
z4cRY=a;t<*%+)L1d3q~j*i8Ge5h~3RWb8X?gYZZ^SeqO-33>1sv1S!K?acB!-UQr(
z!k23IC3&L33@2m;vFo@-5b0;+A|hEs^ptRh{p=lfxFRs(6S5H>#+$8#MHt;HR)~UI
zP;JdoJes|y{lzf}I|VoZi@(R8&Ncr+-D52lt1NwkWj=E2cYF@$S{4`5BIYwntd?3A
zjQ(UtGeEdN;2pHMXF_W=$~Uzi@L}DpPhr!5pN)TKFfKUz1xdE(S%Oo3%@5A_P$g_;
z#G>w-pF@oX7FY#%T0kWwO1O&eM2Syk@YEV2>lDXjZhs0-(-bOWJHk;NQQruxPzn8$
zw9J`#w(<*$7LPcq<<(3|{cF(i&}r=}CD>!HysNwr84|^G4)3`hD;bPNdU(pP3`jhO
z7oQNxa6thkmO369S-mZ$_ATf*K6dRUJ;Z*^A%@@B0Z(*{qY%WO`C2^u%d8_Ve#b8;
zA%i-F{C5cd2YYWFRn;2Bivohu(uhH~bV*2ugrIavmw-r@AR!@*G)Si)-5^rZ9nv6O
zBB-Q*z?=K%Iq$vu?;GQed*8TYoH5Q2*lVw~_S)Y!=WqT(BPG+a;Ks(rSdX!vB5z9f
z%8P?dBWH(3q-%U=+Yc5U^Pep5j+u2M7T9Pc-H4wS-(TQ7P;pF)t2@4?q~N7)lpTn~
z>A3!oG~Ft5%<MI7i^9D7#iIGqgm#T+%+wdrFVh%-+Fy8UYiNy%!vcHO?C+5e3D$n>
z&NXJ-nh#M=xiwTxevEYe1J~uJ+x_%Rn=K5n*tU1=uaPJF|6XFz(TY#I@9tp97urtJ
zN@iN#jCAgu-2aAFVC9x4GwR*dlg&eGxvLvVoZ(%ShQf(tHy;h^{?vGV-=g<M#NA44
z&!;CLSB6vN{AZn;Bs|ILOo#BVy~_RBsP9?Us$_m+C4T2IubIl~YnliSr<Nu0^}EX{
zG%Guut|_{MXkLn|o?8>wwnWeS>(j8Y%+Zbjez;foJJdvO67?RzF~=KEMj;$;4-0$S
z{X2K;4*jp<E!)qxmShd9QFHcH>!(7y(R4reCwEUT@{>2;DJXs%Bn&#TIRvsZh^(b-
zGZCeuGd5vw4%x)pFZ=O+R9MsF&~<dpr!yQ~Rbh=nY@2el2_3Ft`I01>=6CB-yf04X
ztWvr^ZO6nZkW)7r*q8Y5Ebc0E**U#9eF2U-C1tu37i-uV2PgZF))eNeyZf$uGV&N~
z5V7v}%-9$DnX2_>_WBjp+A;2>nLu`%+vP%NPl5OKbg}{8Ly$)!q$CERXb8eytO%Y&
zns^4Id=w$v{-ZT07!}cSei^DBn#TRq{%BBuLp11*UVgpioqp76qPS}!?g7%8wcv(d
zlZV3xQ`g{Qe(Q;{Ik^}0>UF&;(|*p^Y<_;UpLv>J&QK2~4vb8`a@c2HDg0k*pD*wf
z3$+G(y8UBg^`bG~R@+{vJo;3Ch>$_<zTA|F1OWlVnf+tM>B!gKNppswbU(Oz^z4`}
z&IwV|=sPjkz6D6F1q%?dQMlU~C9O^jOfv<KsFV@Z*7T+NEZw}<50k;ARiu|Io4>!+
ze$p%{b+a9^Fra&**MLhpV9*Ft3pCPQ?qdrDj@@X6JC8o9f5A&(*$KfjJnmu+T}^MD
zd{?^h2A$1mw8!rYGQs+x&j;4~-nyRe7A5S{w)6v-lTzygiB}hRD4Lo$Nv_%0s>bY`
z#cBMKIj;A#GoLgRFI314?X5SomE4avR=_2>Zo7+4dGLZ;ihPiSrB~VDdU0(d_b}0@
zutmx&7(wxTboL8%E%Sf196_kpC{P}<vb7YBLH=w-=h5j9ts_uAk0W2D$hBaweF@$V
zEayK;yR-g&?5t;rwZr08I-RLY6P4$36Dlc1&t8Mm^J(makY6_)&v+Y;lL7?1X+N1b
zO@02_FJ4si=%Yc#(?!)5*9wB-m17Li<}Yz~bcQy9TCGZs1A4pe7Cvtxf9DeChvhyM
z=@Z>~jF+E&a66OZV76nakg=@fO479ouV$SAXpNWJ3G+S5-(H)J_>Q0$<UAGL@O%2T
zpCeei^5sl3fA~O|q3yH-QeRir<c8oed4yBBZ8<Lg_fNl%#+;ZOThYd@;bJUt7+LUs
zUz1EM-N{(*pLELdrcc6R3v1{;j*N6X?Njr^s>Nf&;AiKpE8H%Q%4b_HIDQ%&z{O5e
z`*@f3mcH;8zmxmv9}ib4*qS=mE*oE+z2sPP39f%vd|_8=x_gG+ryAB_rxYU2@4(5f
zR>QQH<9IFK%n$zq$;6~GR%JlkwXE6gbh+vK`(-bOrbik#%9xhaREo6xR3y?A9}8gG
zk=ULbW?es)dzz#c@yfgCXF*T`PoT~3Oq(AUZK~=jzJ{vKOnYu=)}Ec250=)S&QvkQ
zY!aHORKupOMr?j#$PX(t(EM>5%@mojg;pc_H07uBKq3Ar_IFo8S0ZuA`p9XHXqS#(
zAAGvk@TfH2{Fm7tckt!;D|EkHN_n#;yr>86`ewgkOdYf<(m9M!uCqNf`qA>1r8f7Q
z&AlPr?qg{O(bVRuxPzM&<8@L(`XY@FjT!yI6{>_I-JhY{`h>B}<NN|Se-gW+Y$JCj
z7pi!t%l3T{mZnbW*~~G|9wM3^7xIy4?&~Zv@6Cn-TRQEcmCc9mrrZ)>x8nR}s_o3A
zElM~qYUb>iM^<!y<KjAQP<sOo`(^0)^VeqG8Be>t2_1h^m}ngds?-5msl~dg6&iwu
zj&6e_*YV)Tnei*^bHi<^j$;{Nt!LH$yt~?>aHuNTul%n_)~N;2n8Xy1##K3Ati^|n
z(nNS>PnH>d>bDnqKcm%8<k+1%AK%jX>}5IiYMsd}Q{eV$opb5^vzUzZ;N5i0X6d_M
zS(B_5N9&3L=N20~bKg9nu8jxU!#VGb<)skJYfev-HB>ZEQTgubNq67(IIyfK9mBhp
zzd~4#&RQ6(P9QrxBy~K*Y*9H(#*8?U-i&%|$ivo^A^sI?9=55~nVJ^AT1`G2bep-n
zo5I}J+?GDp0-@V}TX*`10cu*fEiV%CZq}hDsF@P6|2yi$6GE9$C;Gso9E2Dw{qvs+
zh?}BA!j}J=yBZ@*Ex+YS-2vYEA8=k)8h*me6#_E7zq2reGiLc)0%zI32a%#AhMyF4
zGwT2MgMWusRDWNC=c@)|tQ_iWxBq@H1#eC8e@^EA+?l@*<9~zB|DQiI8S;gw9id`_
zs57<CMIBbVQ$pWoz!cRJ9ePk(m?!<*iu9mHiPI?2W5T4yWAz1Mw6t!f70j03iJ^EX
zs!@9K-k}BK(x_nLEGoQ>pwwWEuSO!=tbr%Pl@)SEiI=kKuMpN#d~En_YU=+fC<4Ho
zHn+@E{lkCC41f4WN};PFtd$TmVwk4K!!WvSPkraV7vUpByi^a>>Hb|p9&5qi7PC*Y
z8VDzUUIg#tr{K?_9{1mIjSv(I7K8DBeR8Sss9<^cpL8-A^8U^R%=oY45lW>!X!l?K
zI9>qAbsorrP;%LW5LD0WkRNjn)CPl~5{4Bu3{Ywg9o&=8M<9}W2xelZ2U`ez0{Hw?
z!JM!fL=-TuJNc|#SMk&I#^o7@cBNk3*#q7LEMJgv_`buKiewnlc@Rnd1T_1L-|b++
zA|2gRxhLzfxr1PYnL6X%<{!S5?#Lfajtic50LWeE`ZiU*f-rY-TqrW66+SqkO2dZP
z2EBFsNA8j`^i>(d>ba0L4J#O9HlLr@0^?xL>T!V-UiW333yhN*CZ7S7;w^AB%8VpQ
zE5TevDTSYtg6QOUc=i-T#O~04>7;Vv>j7)O6k&IVuM2!+7)SyUT354Bt%o|eN7aSN
zJ@BD@|MySr0~N=bAg5S;nDNf94|K{V*FSKYbgs^s5QmBS-Zyy4QujlrTHfR6+X)aF
zW}^@TC6KP@{6<Ji;O?X-uTD!o|6l+%v{XU2jfwZ3KSWJ{i<@hx0yTv2<?nqM8U-CD
zp=ApnO?`^KdC&Iuc4@OmSG4*Rb5EgcVgJ9kHO1<Pm>p#rA-`nhTr=QByf?VERo)yF
zIkPb^Y4j<#FIQ^{^eRX)SfBochD8EOc3!9Jyn$mzTB-JUdU#tk)8@BMUvUm29Bw}$
zthZR66xBAKc)}~<`6T(<e~Ub9XR&xG#FP(wDR^+#i(<J|QBE31wPfbNi|ie;wVoPF
zE4_jH8u5FI9R&{FPl=u$vV>qYRB_YqvQym7eJeMi$YiR*AH^L+xvZbS4*)NjDn|A_
zRZL2zyYV2$#jz2!xq#DGVkQ)Kg5i>^#rF?5iRBQYc^HY_5uS2g@PY}-n4u;NM}2Pc
zTCwO<S;E7h^-#T7hc_{Qm{Y59RQUPb19upIeM!W`_J;A+ga-_nBtMh`KU(WI7^+Bc
z6H#%k|MS2Y$A{X{6a;etT`CuwL3l#aMOR^}`fXu0EjVKmsGop^>@;O6v+mL|2yf1S
z7dD`K38q<IU|Hg3!psrY^SZieTthd&<w_-0Xek(mSO!u{b*32o{{Bg--vQs2mRC6t
z)tI?%k-zoZ2`42#Xg9}1^?~}jk1!?Z0VlN{a4D5qeDi?ZD9#tjY@j~|M^Q2e*9)8w
zrsP`3ZVO?=uEOBW=#=Ra)UqH!FM&IdKxzuanYZ_zD~Uf0^D@V2l4l$TORbBw9zRZ0
z1@c?Z8|^{V@>jZIM7-2js=$uSLcLP~WVA1C&+EXN&P8q=6Vy+s!A3VsmOxw`A7RvX
zdj=FEFNT)Ohj0cmx_>PMC7aFXnki7``C*M-)url;q<Bu~2q<=LQYOd&aKRE1o2c9d
zabtjdj7|a44Yh!Y<AuwM+7|?w#vR3E68)-=gvSA_7*0M(7Tu{DyLz3xKTxvIhsaO=
z4pt!8(=5YqxidktjB?iak>D2X9CMvYd{7?KSsNBR!q5ycyDrNxzKtv;j8#d#O6`Tr
z?MF4*t_tR1KiLyHJ8oYctetZRRi1%ydBELRvESb%2OKurnJR9h6Y1Gus4E%f;bA5u
z2$s4iG=PNE=KD5SJ~)9;HJ{7u7jPBu#Tos`bG^=vTCLtF*;)EhN)RO9I?_%|`yI;O
zWC=pHndFS15;}!=kodH==kbD64uo7ApPVj!pCkqt_i`AwzlJpq=hfB5;9tofU`7|^
z%;Eq8a+!{{S<?LY`=}%Ly<yVT^t%C@pe%`6Bs_V5P7V_rMbE4shQn4jN&B6CVH9zB
zyPM0NdcD(T8#=3bC<E?%bQTdnBon{mx9nePb#U887^fI>?OGF~tg$-dCdNxJ%Yy~t
z{t#t*G7L}N3UZp8A6_HIMb5#i;stub$I^9>;!Ng{XxyD*|0<%7Amqd1?Ahjz!EyzG
z&%ajv;s@^1AHBM%x67<)EPwh8c3Rw6Vl64dH&h-%X`Oa@try+*Et*JH5=jqinid%(
z(?H_pV<45>Z<e-d%j#N*=37UbAdr~<i!*r*p*57Ie+;IAOszN=&rsj<>A^@LX)wlL
zf$ni`ujuKk(Oh5>v?Gig^HUr`R6E^56Pd8(=0CU|ck2EEc-wHL2PV9qxWf0J2I_Hv
z-8j(DWA1~-U>5ip;qR7uD>T<Y9lb~}OZ3p9Q3J)sVJS0GnX2~)Tt9}hY8Ci+Fn8hM
z6)y<*#ac7940H)c940*ZmV!+!SF8cC90iBx5o!X&-_Z>V155~TS*Zi7`S`{h7(S?>
zd7=_{P*}x#?Z4}bru}hUJQI&*wO>G!HSN~7otINaLFE_A%c^KUBz(8%u8oyFe8u5A
z63o5<E3j?nB}}zlT&{n)zp17*@l&2PK15W5mN}N~HE&{LVlA5jmv114d!tHd?!<e|
z>t_ZUp^wuWVF!UBc6!Htscub6G%4LM$hTIaiMmsKV$f1cM6@R8&@{LGl8CRAAZ$ef
zZ`pGAMjD__Y|r&j#{`Nk@4^F~Jgue@jISj;xU!TT6sNOIWi7ZV;Q7&PKceH8#xp&>
zZ<=W^ipsaP|9N|6+0&oWqRx%Qe9FQ);l@#e+w+z?$ty)(tZ~8*@;I|3_$WS}etrK%
z1=B*d1;evDwjc!Yv8pR|>=(48PJeKHo<kD*8nx_U!sm+KTc752cFnO!?B&=TIUd;?
z^YnEr6=@htWGvoHxZl+??X3C4nV7x`U?-E@U}2#W2^#~1xQ(q`e2Hch{1!l4y8n;U
zm%R~*t<57$-Vfbi3OO@*e?9%3<1a@!HV&X4VZFd2kd!3Vcgxi7FAGAyV)|LysZJq*
z%_i5SgIM;Fk9#?Ggd<0@f;?}1EA<-LtJA0e<&+jKPnV&J*%IJ-tOZlgDl=?~d}gr_
zs_#vALa`^+<0nfj7Stj_xmihIqmgn4TQ@`T+RBsgy@4)-J8me3zQ-ZwAQ|08eTwsh
zc5DBa8|jM@1nHHV4tNH`b9Oz{q@_5wJIh>OA?5LXS;h<^K+5RIx0l+ERuRy`Dz2ZS
zev?^FVDcH{^j)~5$PIJotK465=7N&fJk7Pu2<0hv{jjt+E!V_^q%h+R?WnuhsHjg~
zHv;(peNl>SJXz_KNj%hqjBmlyF<~Ew85JC}iJDu}{kp^BK4H$El^>MjU{)FszIr^e
zVi+jOCnpTTwf996>gs986TCKy1&t#nq6gq1A9bzvCCEAFY2EgC3j)-sjw;3rR2kr-
z3?h(ihs6aqRY*Es#P|M?U~eJ6hy~6j7&liiyLH`h)e`@v5Oswy^tK^q;_%h?4z_C?
znK*f1KXcF4N;#$OKEGG_FE9<R@18?BaQyXSsV@HKuBQ>}?b(Dm=7*1&OcEV~=50xl
zuHLPO`El_}+8INuKu-4~j;UT~`iqoICk(A{IZqDoX0cRnl*I-fq)2Wj5%uH;Wgd2g
zPzi^fcqI<Q*?CBZh-sjPSf%teL<)a%E6}C<<Xuk0$^8bHV?<rmv0RmUMXs-O!?_9^
zLFFA^4C-NhA&h(q^SG|~7hDCbJl9_#ZVjlIrHQ2ZGh5lBZ7i{FC=B=VR|QstJ2CT2
zOwQ@EX8SrTG`~D&`Bo9f`cCc2#-y*ev<f&~8e6=7!(6$nif?yM>po%|DGXNi>>*_Q
zs`5tY7Nc$wQf<AZh8rd~1=Oc^EWU+<wKE6q+FUtr(j(%&hk4TZ=zW4;{H9L!GVTcv
z0QZEO^UY6^>o}DuzRI*;kt0^6-?ax|-@Rk$B*A@`07VO}oF1LdR3v4&ETkE%(WqL>
zE^nVvRi<^SMhIw-lv5(}Bui0GeCpLud*#j&7@%+`Ee=zu|KIc4BJ7qIjj=E{#k~?T
zAYY@t6u8hM+}K$}e!I$lBIacQI%ivm39WB7sR0f>`5WJBg1uS>Rw4<5SJ*{%_X-#;
z%y6EIoFnYwVFG4$h<dqpwI$SE<Zxp_(qOw>XC3|UDN(yD_DfXHD?Ps}s>p5z@h=2n
zq^|Wp=U5{GC?cdmtqQ0}GoK>D2lVd}9?Rl{SAB+sZOHw+>y79emTl7)qnH*H##BGk
zFMoG^ZI|@^K`W{zbR@NS)&zu~rWJx>%DW<N->PW}SOdK|Eyy?wk>uDqZ)r8Ul|Nfv
z6G}7*Ym~#hhLMrFY{}qB_`xKY<+=q3vv5j@XN<`G%{f^Js3P*^)jc&(jbFgpU77sA
zjExtifHqm8-DGANTi#B-FDami%)X)%w#hI#6DUBnTII&rf{@*7*;b<2)XnqlmtGMV
zp}f%YoL{~k*cJ6wwB|C0Iik3!FECv%cBe>t*qx6$b7d)o<|rY02~hIG3Bx1KcL4}H
ziX!>)XwCqfAQtLne%2A#mNr?{qm8mjdxr6DyUVN)<M9%|bfqqck*EG<r1oTQJe&N~
zXVs33N^(9(;D;tYJ|mv*IHub3LaR4OjT%hvC{hea(zv$JNWYqnPm{|`HC)wn*UMJ2
zEAMahF6c7D#VLC}y`l7F_R)L1+|b%0?r+WmoPrztbJTDvQ#Wn~)pkq;J}WEVA-|(~
zeLwvUC()<h81Js)cl(MdCxp0-F4e`*-N0q8aKzav-{Fv8Su~{{<fzCuD)hF0U@iBp
z2)AS}hk0$8bFYJ#eIv@=DmxS#XOT59^+BRHwWrqAT0$$3S(pd}%kv+P=#o@pZtzlb
z*HecLvdHxWmY_ey7;4m6`5~dI^%uOVpne>jV6B0c)!Tn7)%6l8L$`s-H$k7gQg)st
z(}N7DCF{Dh)@yrx62B8|f-Ljp=_Ef{|1IofeV1cONB#n8EoPpppNJk>O=0>Ezxv!^
z0E(x%rrlv`U{>c4U6XtX{U4v;)jA;9IoPRKUW|tFbQ3#Uo3*J5_xr1y-ZxH6dqiJg
zoQOfUhL23)i%F>GdEUJm{b?!cdG`qnR{NeUqqA<JlJfCY3KAwicoRUA(Pe*O)A1O4
zpCQT{=V$ZveTr-=+1tLx?8Ix>=(qG|ar|pu(fB%)JV5&o=8IOKB=f^kQK+?@=fFee
zIih3vK$}qaU$hN%6t{1d$KXI?LJ_9D|1d(VZP@B37eg-$Y@!?0k8fXR<S*@OBwzba
zX~9T`!bz<6qj41ExfGTxns(X7K^MQG5pJu0bQX_s5p9Iqh{+v|Qhi~)#xm(?mX5}P
zOd}-}uG+gI1}vFL&@%li${<4FoLBcB(&j^5Vv1vJw-MC`gQ{p0wR3;5->%f}f=(9R
z88Zf+5I)8M6FuKVOg4cqDd}ZNvC^vF8xq(Z9Ns7=lX%doUvd@(w&TlomSnXXsR#Xc
z`IzF+`%%3s{3j#;fol5tAxP}vW%2I0dHQUQ=r^0tW3hetDHfEuqIjeFF~DCz;O<&~
ztk?&a9j<{wPdLw?^uy3ZxVb+5cvlEvO7;H^6B_+%kMx(|EQH{@+xGGr{tK$I0VL9s
zD)cAt1rV^_2*6${-a6p?`xr(eygD5=lo0l4|D0h3g5>`#Pw&dV&hY=)6#c!d41`L9
z=>yZG&)*;XKU|vr=bnZd(+7O(mH-CC4FcIDNz<KbcnBaa2b#KDoW@eEZ<Vi5M{)Yf
z-_<PX=dT2<gbXUPdSR@xh#A1nvM9(O>7zLm%sxMOs@H2-5qkeKzv#DzRVvr_>*?A#
zC_FUyn6URL3as#a(GePCi9`rf`3oXulmVzk6klrff1}k=aAVPp3Yq@{p4$+@P_3rI
zC;z?(5@LY=<6ltt_aI<bfVlZaCxtHmy$BsZtzf*o!vA1#Mi%%41J>Tp{P$2%BP2`z
zlTJo0xU6JZjTc%hpe6(Eq6v#jk#ctDy7wh8e;NU=+7c=Q142Zw1K5~neFNHfG36zA
zXOzQ^xz)#9E%8qp3nI4Yn)f9lJP9nGX2ACW2v$SmnvtO`#pE(COaLSk0Q-31l`w8x
zP-9wJM-=iMaxh01J^*v`mb>?vyb);A_C&E{kOvTh^C_ekn!N5!8NM2>a9r<~GtelQ
zCg2#Hj^A6O5cY5aY|T%#1~y*mo?l!RFAB&z{WD#cf4wVnC*!xvxF@n<M_c*r*%RS|
z(Z^b-%;3k~r1$DOgF!#Eh<&#}g#4xA7o0r;kOQEi$tE)a!EFaWw%-UFa4TRix<GRe
zlyq($M``H`Xpftr*@bpeU5#RSY}W_*%1q5OAs|LIsMR~HAtVGgQ^O6%px<;908@#O
zonnt0te3=hH4vWvGlJR0io#*>`BsF!a~Nu|?Xl_~fFWxV-g33aqZQ=70D(MuzXU*&
z7eGS#?rU*s#JG0^KaXrpmM#Edz_J0xEf(P6@AhNypQNB^<4y|4e$I&@fYyjA=OwDl
zuJqKKlVD9eaeeYW2WHO*J+b;Bc*al;Xd$4rmM3o<3BOXIGRJ4S?rMD$21dD!%CCEY
zH%SkNg$zKk+d%+E0dI%O4gqBROm21G>Wvx~i`n#C!YO#v#HqJZTf{Fe`?zjT4CKMG
zJ>>F+M2tNkJTw=`Pam5;_p=uV_V1rL6b2#n#4G_ZzXas+d5MP$061$uI|2-k^pN9H
z`{>*b7V$TDvVn^ZbM{@+gwLkRjI?oHfKR6ep0J_@T4=hqUnu7Ts^%X0w;a}5D_O=E
zN~kluZ&j<<TR&r)6a`{)pH&nDU8SW6{2XOo&Q9+a>e0=!CeYWk5CPI>0+gO9bsqOU
z)3bAgw>3%kkA)O0bFJnIX(b;fM8Lp1gpm;-W{R&4DePL1ztNlYJcq{^AOYQ<uKoo<
zt#Un7=CG$)gnUFxItKo}hY2%Z5xEA7cQqSI0XnVZKVFSDT7a}(4)Wi;n{t+o#I#HY
z#5Z)L-ViIaJezCzC8+*L7NQhr`PeK!r~Fhc?0?hsKIjeQJaEY&r-ApmIzsyaj*&~@
z1yf}nTIIjT8_;v8AKz~^`(qy2f`ye=jR?-VCHZR>;EA!ubiI7Vi+B1T8$x>!e9WR=
z<C9Q1cE0BQjiD~a>(@9CX-}XU==UJJO5&DSRvo71^$R|uvV5akM5W^p$F*z{s8WP_
zLb9hyJj5q5^u>HuFLU1{$yyQQF_h}Mhuf32v|OYHcFNaP2SWdu-({eqhS1hwlX0y=
zfQBTBon#vm*6!jDT9+O$2!S=i#@$)235g!fQ<pI5(^d0H<8?D_AAVrv%=h&OzLMrZ
zPRTUFn*n(opXQn1(}aT<qYBlYXHR8z#WhR@F2F3&4S*ng9|^4W+ewd}JK9!dpq0O_
zI@P`|KORMRo6}D&F5m9wSEHq=LUY8jsYU1OhsCC~yeNYJKFUgdc!dd(G?p!fzq%(l
z;|E4ykrbjYp%)X$?S`JZM|VL8>3~8!7mZx|@se95BCmzyBd7n1-{%{lp1buMc5k=C
ze=mo!Kh}U?r9@f5>5?ZOU`4o@8RAM@JZ4d|?g4CS82oR6=Ea06TAejk4$Qmqh&ppv
zzGOd}0fbPi?KtFd>(XPATe#1w*}`HH0H5o7LeQNL7OTd@d{l94t)l8(dIS$u)|m7f
zJ+%013jYaA(U8kn^y>rrCu0>X_9sNwK`v6XO(=8LjpMdUanE*69Z&;6L_Xw%V9tSm
z7=5)NAL64tP1&g+bp^z#phz4sb%@c?T5*FlavmSm(i&Ivw;g7MzZ0xa2nQVHaJKX>
zGUGwr+pWzJ<4K*R%?b`lG~Z>CF`s0Y+-mJ->oLP!XVMF^{_IRM6%W6A<qmOXP=s*O
zKGteQ(<RaHvzhv3ymY=J?Q<37QC*$6IAt9Gv|6UqwjXx%0s)DcAo0**UiQRWR_;M~
zld!!)^TWafQBhlayNZF^652y`bl0s>Gf)~G$326540eS6vHXV4u^elA_Je6yGQ6a}
zMbq7i8JcPjr4ACWx0yn#l=aV~_+*+?@Zxw^$$t4%&YUFC=K5!#k8Z6kel1Yx5Y-kq
zo){<l<TECb8LlucoUr<(slWlGIKnLdxbMRikP-Z7#^Pp)H$uc6gjW-ARc5Tj=oDQo
zV);4Jwv1bv1rn|UIHEmox?LLvi3_mM)dai-%w&nhEbSbw@Zr(dl||kRrc%cz{T@%Q
zH74iG2EZTA9Eja7OFzE2-e@&=EGn{a8XwY|gT)lW7Av1?OvHQRgk+hBxoa(hi=Jlu
zc0v@*gRPuw)iBOat9;oU4X<efUfVLxD`+w`$iL}5J{A<&s5PIh{qDL`eWa6KeY}5m
z?S=P*g_p?pA(Ne^g=78gN|C<CarsiE?@DOGri=HQPr^$`ww4ndcsKHPr<OK<$H?h{
zfG4l-;b7BKebd{|lKCVTR!yA;iIdu+QWVQ4=Lz1M75Jl9qAFCPQ~HQKMXfk1q8>2k
z72jl$D6|BT)N?+8IMs3N3_S+lhwLF%;6OL1D7W=tLh_1}Ye~UarNvNATq>I9O5Qnr
zv0iUme-^9Rijn$zwSwp^9qX!`kk;u{H6?ZKTWd=BN;iT(^1n^#4R)^3Tz2~E|7I$o
zvZo+LD~JFUPpn14_gk91cDZo?nd7K6g}Exot(<qCIJ)t^Qr}nbZ2sO!d<wpj#aP10
zaanzW$-MVLh6m=6mR_z;s2tevX!+{wmLwc;#bRVEE)Qa~9=N`GE3v=W(vC?MYfAF$
z>K(Lt1t<d^x<+Uje;=eDSX7!z-XWmwui@CC@j*g?3s0=uln8Wm5kFqf!IX%R;K6F2
zn*^Pk{J>*5b#5PrbE?n6@(muj40j5JaK!I9Y;OSix{2_bZ}N&+SBIGxSNT(w2uBnq
z^(qm1wj}Ad%p3~6sgd%@(rT%S2t)ZFMubK`)(>8T&nQ2NcB5LA#SpoL03mDIj#JXF
zwrQ)~O}QdG(;ldW@^2;bu1Li=q|~SP^6{WO2=*h#+Q9A&mZ#w@<yF<%Fdx1AT=Hag
zdgLq=JL~YvQw`l=53fA0dDiqQnt@*$e(x1^2smQf1JGm0Nip%okbDhME8IGzH<Xz*
zOKeC)Mg)S}ncfegrylC05G$uD8hso#=GO7iI|-r3-R^qGIg$QwzNR=uX3WN<^fp2-
zNIwX(JPpkdzsPCzs@9;5!UtYgovy{lk57(%ieY%vv>L6j>@u0Ey^i|@;=OXTYdLJ$
zDHaGaBNKT7;A-oLyO+sG1F}x;(o=3ED;s+z&t&<Ftu;(*xh~LZpqEQ&zOE`P^#d}G
zpKTb|n?@KO4tjXxwb#2qDXF6Pd4R=m<9V55Y3ZzsJX!Iq`K!|J5A7yK`3F-~>9GfF
z$pjp7+;`My+N6-Lb?JO+Ph=8DLPnLRiJEWkNQ$w}3!}y9p3W(vvoDCw`GBhTE>=R-
z38EbkHcCk)aUa?{Gpg77{S_5C(}Y}YzvYcKyQEPh*?%=TW#*h24I<JQ0Bz*kSk3D%
zMxAS6oBygkEXbPQMZc-!z<pf@>o(}PlF(H#rk*D<JM<!M_5sSYqF&Wt(-vay0#)||
zW5!d3Qtr#V#z*NQ!NlK~lqE7OZX~%KxbpCEC<csh6DbDoN{0x^;puV@SX<D21V&A6
zfkv<5Q}{ArQE7~MK2v->ad4+2L;C0d8&x<(mM{$AzRX=+4~dlUJ)vpiK{>jWhmod;
z_zvIn{YKA*aKm(0`e<9LtRMZ0^a-hPJd1YLud05z<ZAs<?3OM?oqU6W;5>~St1<0@
z3GKm!f&%A_{+P?ECOr@l5HkEI%qw}lMHb#LKQn#zC}6!Hu~tXo9vdguntgzNR`|~3
z;3V__+BG&w6WC7V3Z~1SpFGj|tabNrt9*7zM#+EabW8{ZeYMnQ(B1U#fyG&B(Annm
zO16s4&!KxagwtgWGVZt*52Q0=TMP!Vb>$Ss6zYX`C$SwwWvDr)>@{B;mgZILv5rr^
z3OcZnem1W5I2CP#Fmpgr=Cst$^~gcrlB<;ac5%#EzUaA}iRKRv+!~pCc8g;BL_H(f
z4}?CnH$7HNRpVf_q^{hx8ReyMF0Qsa2xAaV6?<RPK5=~GnT={wGhg<PT<u&RkNBL4
z<1tBMr=dpnXNCCG2W(r}na76FHDdm9+Iqw@&o{03)dq2V4Ny6-i^H-uaFe}})|(rm
zmKw*?9fb!u6mPULuqoMiWFF3&Jf~C%PD_Yo)}bS`4;^M0oEeczQ2TIqB=}>g{CVeY
z8nb;5|BRZE8~Z&f_vZ{RSn)Dlb+I`Q*u=2VO(~=16{OcS-|lve%v5-=RXKi@j-(i+
zKUaqggY40p!l_-Kv5%WC_t-D9w%I*yXLCQ&_-V=fPD{~H&?Lu^L7q~j17WiG)ijJg
zT(CYO#(7KW=@X?<XU1aV1gm3EN^Z-?kgMg~DpGrJo;;71+hy6s<Q~bVej^sK0HRYG
z7PnlMR8wf#^=ej$1aT1$99e=TJY;>PsL_|*kRN+9ipFZ6SFL-HjisC1nDt(*VVZxp
zQ8_u!X>u2#iUgxSoX1<QUsgT;=AOrMW))LGm&@M}DTfi&unYFzHf+STrN@>lHxDi*
zR-cK#teWe5=x^IWux*lPxm<%PS!?|Hff$M}J=^PzV5H`Vj}(@O8?*Qk?MyW2OP4Ix
z8$$GD73I~(=r@pIyMQnFHzG>=^8GqIVmNoi1<>MC+ROP#;Q#&S6qDGp@rFGW^Pj)Q
zlcg40jz$fj{`=1|RiE~b!4(O$KQE??lJaS%!@!jO`%lAsq~Orhu7G#{ycM1pWiV2(
z0s&FT-+#6!A!`g)?*qM%xpKz5s!Qwuo$|*en$1hkuFJ?bmm6=8EcEs^4X^Qi&dSPa
zInAO$#p^B<OJI9aF(VRAv#8<2&1T#2E)sFu;nXuR>^kXqJOpLnst|8bi$eSV{&iHH
zzs#eqzo!PTZtp0*aR^Hd;?F60kO>loH@E)1EpwPrb#=tqB7bj#KfTy8!A;>ie?DU$
zA6;YuQpBzP^H#pYsIqzz+4&}a?~2`BvE^lru+zV9g_jIhZF%4n_V+|>(PZ`JLX;`~
zegYDNl=c|mX%T<Vq@@_{nJ(wozi%aG373&MGRFD$OzMa!?d|W%c>R4VbVaz~|L3Cr
z&nKOU@xROCEtWZZ=N$zG!pqkltfRp-w;xO)0Ls>Zeg@(k^$y1#f$W-^6V<Q3rUeXd
zz-^#Nzfi4U6#S-bTpygF=a7OEek&OC?}6dOvoDP&z?OLTI8&lB{UuP1zCml5w~$cm
zPWSk+o?eRPmBFA^@GeLIfC=1CU6u{OiGts#%^y%FVS7S4ce)=y0@1hcAco>|^d0c!
z^K0FCJ+E3vw!ncvTXzJS)aO%WRO)KYr{ksckJ3E*Ibp9AbJvGkzB;q?rPAUN#EpPI
zuG!TIXcMs5>JVupu;iZtflW;dz$&mRP6g+JdOn9vRY7DgP0-5sFSfqAPJZ9=h|-=Z
z^>%dy^$+lu5H!dIU+ov*#<$fBs8H434Z6F)%Zl4NoP^CBY+))1ki%sZeH5gXg^cMs
zW^11t#8hwY4W|e=DhVH|RYHn(wT^T)5|Y4OIVo{9Aw*_Zz2gSRNsr+o3?Qx-_96wV
zu$_8(4oFf38=TEY0ax>`w=8`Ma(w0=#^8qnW*l`8Ok4txB8*%+0!y*aV7okryoZ7|
z0C;p6(BsD1H5Q#;VsZ%F%a1sNs0;?Bly_9bL`3%id{mmqB1knuU}^`#GNsNa<`5ks
z!0T{THM#{nPfPcA<n<LNvQ+KCgJ0>iDz@t0EPi*qPc-rQGQwkcrGn&8v)s5t^x|hE
zxPjdBf>KXdSA5tu2Nqar=Q?@Jqap8$%DlD035eaa8<8gMP`?BK5K=6JRDVGg!(@B+
zxufnP7-k4A<Z*tDYz4(>_GG#HGuz12k!hBU>pAO7c_C#Flei#f_MWcdU}_=#+6hfd
zZq}T@gy7C)Li6vAunllo4ZD?S&Id5z1MiOyPk?MuQg8-0Eckur>}?W$NV){;`v!!M
z1!R2<EE#cC-6quf=uu6k3=f%ujU_e%H*6>Ua}{+7gUMezDV&FM<YHhC%Mtzve!>k;
zAHQvI{l5F#Bn`ksur*TC0!ss6%-pkmkQ#=tc~i4A=VR=12ODQd%pI(=YOL0#Tp0?y
zn)@QH`h^kI<?1t$Hs$bvJ_hddL}!Io=<f3c8#1QBbod}{I+-2ilzK2y9R*G={^Z9K
zz>W$~#*WQZACQHqT0kCG-s9SO!GQGuf}7kQq&2U7E-G*Vv*~vAmEtuNGH<@<2x~Av
zUyrc^hf(@sF)EZ?q94aGWV2gf3t%adtJ6zYlQMb2(sG-0Ka!gC!dj`;k{0l+$Xkyt
zFU}!4tGJW}?E3w=D_aXFG?Qv!6ZFABc_e%ot3-U4V4mg#j6-Je2so!GTii`<Mfizw
zS{BQVMeVYNJ^Kk{oAG5O<`LHfz+(BwW%Z2X+`^GiG%zf;3V<}mL_09#1?;J$D2wqC
zjC(f*>-@LBTYOidR)Gli{3lPSF6O_#%&&!;=>b^R;W8x9aOtL=Dw*jc>H|J>Pl!<g
zLL#>{ZN#m6yU_}4<y6eZWl*woL|<8wyaC4nZ-)GOL{wRdF@hjCz>`3z#NTQ&-a)gm
zYnd;?4t0gE9U7p9+2gnS<rg4$hGXMN8&2?-Vge|mzS_BA=6Rm%*`EP%>`bRgF5r}P
zagOq%_vK4fMOFQ}E?T89xF2{WKI>uWV4UE=Q`B|?Mv@d4KZ5L5GF`EtFkrdQNEWE}
z!R1+H{)Fs!C}_&X-&<O**OOm6hi6S7+;*X9zHve=(P6DuV672RytEMyFROUazp*XR
zxPkCo1h|#d_WprwK2nV1&V9BdDQAGK-p)5&oLYYzCYaHgF<{?qzI<s>O!|>m_pnXQ
z=^&W`Q32^8KWUt(Uvl1_U<w1#v1~&9r|_UJ#l(wYq18d1&W1|VS$Eq3Lq`~9B|}lW
z28g`ztivv|13~IkaU-Ja@*^G2`jQ-OJu&_}^77a9iulFr;{G11VWG~~6pk;F5}o20
z)a~RL#s!1>z%-2KgLFbjm263x9MLh6W#!8f9ha~9UjrOwQMoQ`ruguddpME5r#}>)
z`10H8>vUy)T5g<)2zHkN)L!1gUY3fHx4DZM_ho11U|~E=v8ZI?s|QX-IFpkK;XJxm
zCz(Y%U)Jf&d~Ex(km62jAvDBM1#V~ck|9T?KV)r07%V>BOo-lZsZMD%qHMylE9iDr
zDaEm3D}LJ6Umk9)q`K?u9hJsIq*+ocuLnfWrP#@0T~`3m2X*(2d!p%J)gRl=ju4bz
zuvxCzn{e<A6TKB58TuoOZE-E8;JGD&8;C#rBuWXF$eD0?C=JTMBO7iT?dzYoUnCbC
zmKSegj)0X#4OqxbwM|WWTvYCByys&Qvj37yI#`?ZzNDcLK4yDeck6negrUGSNt#Ak
zEVmQWCdiM3_|MTFyxS$v<-T#D{ACCWq@TUZNFOxImIM(H_3}@$Pnw`=mW8;rxZJaG
z)t@PQQ=}d~S^MK%EdF>O)VW=F*E(X`^Z3i1-JVq<-e6#by5=3?4cK(5y0Hx+l|MtT
zkWYDmgqpZQfK>5;?ckB-HanQV$n7hGl@c$^M(7ojEh~0_$pfK)xitri<?so3!I#hD
z&hdA>g%_}$=Lnq4IDwZwg?&4I1Gy=QwKDE!;B`5k$Td@@U~umGWxKpX(HFTTNcBCZ
z*c?yDb(e=db_K$5`CLq4#sj7tsx}y<__8!`PoFB)zC&a}Z-3BE*01VG`jXIeQ`b>X
z%=^wsxuTdtF|EjwHicAi`1>O15q*G3*Bws~-acrhTkO3GDUXUvu00jcM}&OSA!vXE
z8Fe4sZV@9sH67YdC)lySB!*VPFwyOGsH1)|RJNPa9j#dP7-G=qA+r&8Tx6H6Rg?4O
zaI=-a?_}Z<p3=5ZEP<iHf%4ZTDPi1G2z}t`JNXV<E%-<1*E{4+Z-Z=dxZvjgaMr<2
z*|-uidpeHL11Yt7sul+Bdl4qIGRhf^Oa#8`Mo~c!Z2iQqSD;X%xX^jk`1=j!@WA!<
z#vg51z&sld@PV-zujQ~9p~Ed+E45^=0+V{5E_fcYX&N^%fAoaymWMuGrDekx)bbDc
zr^gUOJj{#>qR3%?nhcn<M%77e8sanm=^{?ppo{ySj@0?5cY;g?*j}B=(24(PNqjv?
zDeZfXA0e&$>2F}b`ac{T>Yg(OpPzqP6wn6{B%t46k&yn=NiD!Cws%h0^6O7S^EwJ1
zdw*P#OzMC7<EZa&u#*c6!{2|pBPm&EU?i@EKK}F9-L7yj&r4K}mwy@@YFcQ#GG9r*
z`}5bUAK+k3-Y7Rj|6B+@EW5$Ot<-9N{(A2T9PEF%Y=8GMMCi5Qd88lq7^A`2dgYtz
z57M%4PbkZXu(%tXvNE!<9pzHp>=6@_Q@Hv9tr?3VdU4>y2K70)XXNd*#R0|0?jXAF
z3e5pcRT4D%s5VMiS$jjXq0Iwld(&kno4<%}J(y;ftKFgeuF)TTs>fWhDu#FUZ?_i9
zBDSnjfzSJI!%F~bmjW)T(Lb#=piDh*Gw5z9U-{b#vOx1){=ewU8N*u;b>z;hW^(f9
z9<2<H72)ZTtG<X4G>jVLDCCrtm7%&En>4Z(LH`Xw91B3cgCui@Ry5MpPj34bVL;U0
zfl4+0;y^W1A^=!t2nyXDK_^Dlz|uTqFU)v$gO|W=18TW7WSe{iam^bb-(bG`{(C#H
zoo=Sf6NZ)v`W{bDIT_DO)e2Nj1#Jf0ePQ4W@t887zJaUj;B~&ztNdWM1hib6xjIhU
z&p=st0S0BsJWmNU2Xz<F_#pW1y}`NshE7U6B)x7ot_L6<$flU9GmDUbaAhZQFpJ3!
z=31GOK{DJfSxO8)+6f;WY)=k-9u6J{i@NR5UT8zq_P~GzCfO<$rilK<NwI98vIQv4
zHWehPYp_>Y9DW@Kdelz6`b}9tP@<#h32q^<L;knC)<@74<YCr<D5QW-Iqf<MBK8Ev
z#`n#7V?A%!=!Yv?Xs2xBBWUel<mC<gOF2W>XyC@o1F`-mya}Xi;2@eOXfpQY=@(m*
z*`v_V(7eyL4S>U*<MU$Z)2^&ume=rrXd(>S5s8!|<QOAo%w@2(Go*VuK*9;|4Ss`p
zeMo4i*R!<FBD}|WS<|eT2sQQ{UaMF_Rp`WG^Dj;(n_q%l1uE1d@b8C@esa1OYXJ>W
zNmLh9vY4_q&C)-MD$teF&Qkt`7%S+1)G0?KVvoE|WPC+1Ogjgasb-AM7w=22iujDr
zS9Ugl565FM!gEV=Ok(AfjfLg&wH6{U4|@-r+*=&vRQn#8%Bm#B7P17XnC{rk53%cP
zjC0`3D@AXKf+Ljyvu&6@gzU&0V2Z*_e2PH-fsP??;nzc(aI2KcX(QIad}`7ZE7f_d
znC&$B6WxjwbtD}jG${9;r`;T>7C6^DCd47XKUL)pQ|Y8W7xE8=$2su%Bd%bxzke-E
zfd^O1M;YNl2ObtKHbEdma)9T=M8WOOQ{;JX>*eS0SrqzOj)9H?bU{BrwZDCJOKqHS
z=%HBUYo~?y%F<oh)6&aR_I(X9?BGzEl`|{?(gnF!*2A=H3@LVgP5O}#j0XhzQ~6>c
ztb<$fW6s`ZvkP25>FQpIKSD>noAKT^?-R70d0*>6E@2+S98;I0(p#s;&=6K;E@w?%
z#Wn;?-=V6`QEjsK@#YT*5=I%db-HNTAT2A$)>W^}(3dcmzG<_tX8j&Eg@_*3{WA18
znon4$4l;{i0?1=Ba+sm^su1%0rgM5`3cfy0D{t|>ILMxlBl_C*v57HRE{N=2cSJ7W
zf=qO=;p6=uhu6b?!Z>9T;xM5N)V_S_L>m0^(U#wPTfgzID4@y;3LhaPL_v4M8%Cq7
ztZiz@vFe;YfQL7~P*MQp4ZvH$b5v)|K5@y31@0hxf(Nq8w(+No1#N?*N>49D(x_$K
zqDKh6pyL$Kb@xR=5unVlCNw8{s#G8dqb>msyFedU_|-(Oz)lINGq2OsonY>;rKj+X
zgNNy%s(x$}SODrTRK>O<?6}H_5nY2ashw@pc&VS7i9MeH*~7x*Sen0ERZ)6?4v4)F
z;Wif1qTc(Ek@GN-L%dcT#1}7{-nc@E{>+yqg1-&BrQST2pWK4*?gstxV}d8hBpIQ*
z<#(y}5G)+3^1>YA1(Wm(^)_1@86;mubpA(5;|VcpKN06#V5JiA@4l)RG*f49QW<If
zt>H`39hlM?Bd0p@jub!nz^9%#n1>iG{<%|&0L|h~U!oQD)HnDP$6Xm^nOL6V(1wQQ
zT|`VBz8GbaPo=4BGTNIzqp%7-MoG$Olob2mRU_45Xgx=3-vL4`Wbumdfw5B6=3b5C
zSGsasI=CKb--%+~w`HWPI<Y(u{&ifcSFiFm0+L<`5^|)7WYl2a%(}Kqs^g@Txt*5@
zLL=6#8&KPs4%Th@A_;QFRN)4|QIwI3rwgwMG5&(|OH<H91!ucK9!UR_;U^7FMpTnh
z6!KXBeG-hD<XcR<9j>lwuKZQYoye+N1IefhjpN_h^HWICDWkYiAUK4_1ZcNna|Djb
z^IQ(AvX$cn@!VG|DBYnnAsD~zLw;^Wk~+ra<iYi(f-seH!m3_J=@=GW7;g6f_lKtj
zC<sDKN@_v!b8y6mVvqK?HGL1UJD`nr1?`mw$pI|Z2rl;d`+LU{+davkl~Ag=XNE4N
z?sFGiS=b@u%===)8(KN;7e_Y2aR(*oFPE>5LFC8A(_H(vZlQ<=2tcJaun&C!aK_iH
zvUuy!grqwBYRi$fqKrGRouE>)!b*isA#y$gd)5T7t`SMCQ%05pR|kBOU=M<LpxygL
zv^6ND@Y&r%1uP_6;1?oRe6HDvbtMOld1ENu0Dz*o8%)_^E%=O3inx6FL2Za!X<5r2
zB&N_`tW_?N5FXpUsT}&U_{WZkKk}^1N1?N|oyg&fA)m;=Q{D>u6&X5C*aSc4%<EKJ
zop5<l&>YZ_H{UGx);oHzIa%JFafcuQd)RZf<03Ne9RJjXgT=c0rTZC8)wd{mjINiG
zXEYDnbV8`ly3D-&a}5VIh}nqE6yja}g`=S$gZ1kv-ro|Xf)2JbR_&QT#U2teqT2Ih
z$QA#KM|a1=_q4|JDe2!oN0gF3Q)93GJsPDhEVk3pxljK7`41@AI$Tw5O8o`WZJ<sy
z{r~XhVn#JL^X65s5`&(`H3$;#%lZt-h}^b`aL{@{c2c$~eAGuq8tL;b;x7{4iWnL2
zZ>tFtlal7o5P1kJ$hHZidh)d$Ue9xlaNrmW{z#c_Nf1mhSSPj2flvFls{37v6pWRA
z<?l7T4ueZJ#U=LrQ{civg_ZvI>hb>9`kk1ODK*`&3f4CuLx9cxOtbeT6ds<CT^R2n
z%DkS@0!4=qh@Wyr&(+>NUIHGHH9WGg<o3ocDJbmT-UUj|OMpc}0g(lS0%|K*2Uej~
zL+qFJ+!8W=Vu*WzEmP8Xo5QeW;o|6CcpmT%>$&JV-vqY+oOX7wmECM*FH~zixeNA6
zR$#WOC-9tx0a0<`)5u`o5j+4~w7m5$=gvrW7vc+=<}@l-zSby1c@6c~MBJ5YKP@B^
z@4N58Ms!k1{be<`h49h+%>HpxRxpWp2K6U|KZp4@rEc$l6<=ZkM?LrE^71l3Z$Ltv
zHD!fw>3taL#ug9}5g{CGL3ptY8qhLez|4_x(1<TGK>Lw$N@+7&qevR;U?q!9PlLR6
z7s^ns4_}~sJ>7<He^%UGO;Gxbrv#k=a%@|-boKmR_WE=WrY|A^81ASVBj91_P;y+H
zG=pPhgHDy*QXBUmzvLSHUZ5GTh+hqL={<~jN>hyPxj@L|OQ3-ETVtH@`!4{cXrXkP
zHDu*aEe7S^9W!I7I46L~&eJ9RJmF6<7DRCu-;_3^k_EB-x(M%e(}14t92CX;1Dj6C
zgN@|CRb(Dniue$+s<wiLf)7)p;7wlzf*Lj)&`7H%TmgTlG*fPAcbbeo)KkaYB`dyN
zs4$7|?Y6bGLDe?-Z&e2UF<8BO|Lh<Qse#yIQ0wUFO)-!B)*~k8w-Y0?n$!3~jrSCp
zenaN;$2s>efFmGM@3ATws5K1=RC2v-7u7E?`qM=(0Ysgs6LS8VqskJD1UKp>l*r)H
zBLY^<h$J#7hl8CN%k>NaSD}}1ykOzfVR$=&mJP)ptwu(Bw$J(vKm!yn=bY|n1W6t{
z!k#Ff{{bA63r}RoZ&@7kz{2v>j=B;EWRjOqfE9kofGp#F?jbb-!lYpg9xRkoHd~!%
zi0W;<TCUmg&{@xW-0785p;fHz-3+IFaNpH%D~KjI=jCm6G*RZm%`zu?@?xjD!G7qe
zC*YxA0`2>S*%}eMRA>6V``3jZEL@{_H$B1Go`<&-D7(qicoN4>Pm@}pczY;?bBpgG
zu?{>q{q*Ye_HwzVjwfrP=Q$ydNs%`OQ{NXi915$O$5<KkDiFai!&m7&qpR`&(0n0O
zc^c-xr?AJFmOMvEwsZrOd#m;*0NJX>s%qV0uv1^*`QS35SPXfVhA>=+T}fa!_)@a?
znjYI1il|Yi_uJwHr=|ZCcxPcM<C*szP?EpoJUjsCd>^6(ov9=h4iQ-Wn_B^E$XFWt
zArB`UfTGs~a6^Nvsd0Ss%UgHNMotBF^frL8ry*f-VwZJ;JXnD!az@l-B9*2=HpiP5
zm}V505*J(13#CrT-AJ1+W7G%+^FVknqJF2BhBiYth$mfc+f3*|Gw9ZbM9zvFCAIxK
zjosg=@Fpzrq?*AVovgiyTI77`6fa$gU46qh`o{S&Py!ul_S)EycbWGTxKdoIWOrd_
zbz)eB7L!Z~^}s6Kr&3TM+4($GQOWGAQ(TG-)#;`o$g5%QAUsg7MUvs@tJP3A`0OoY
ziObqdEPtB^VY0BZ`181l&8%qEL9AwgI(|oEXQqN2Vk@Q0)_puxs-cV5AANIF9+@@8
zYI(G1M-3S&IBa@##a<e!j0L32UtOYA<f#+uBg?|4qotufr-Uznsk}2+&rKltS}IEZ
zU|{~b&cV=}UD}Nrw&}O-V>7FBzvHyRqr+fx<jTWxfAGYJ*V_1NH1i`>tP?}6iE|*M
z8c~OwU)&+ht_*fpw&%k4dXK6$8CcbM39i`3V02jmy;S8PFUgb7cJVvGo5Mwzx>$tU
zqm2?)AU;cONhzLNNs?h&QKV%f8w+U6p0J+Oxuesz1T;J>0iuT5hb`LO4}dv_Iq(m2
zEM|`p)3RX`f*&4g6kfARHjB;X$D1{t$aN=-uD=n>e1D@vIR7P-$QG0>hXL5bZ40oR
zvVs6Kd3pFd&3qgwX%vYQCmjHl<Gl+W#fMiBj@U4l7*O|xB&@{Dty!Z&PQkY@yn0)R
zyT*rQxtrfMW+uMTYp7R+fZFK4S!bfs6$m0{C*{}^yS?=lgYR#@ezJ0`x$DN>9J}{P
zAJ6zyY!d7q=C(E84uSlgk72g;NVE_kLO}e=1O?U$Zkkh|W~$%}Ogk+HGeFT?UyYI+
zqMIvE>?J*dNsZvF6K_BodtSS=?tpx6L7}p~2l58~_EssffNrX_OzwBTWICAa$I6<T
zpec5LJQ!6_mChUENFFFD)^Y3ojpKMxo;wq;r$=Bp-%w2!QR!kex9K!iltxr6AsiIh
zPKV1t@Z33G+8f{5b8P6`sYJD`Gs7TA9gA{{HU#D%USFZGRRbr0r+p2%eX%ihw1+=+
z-u0)Zm!bBz?tgUMzb6+C{`4#4w}w>w$+LTd=7oEO_N$$@wW%kgs3aerA)M@&Csg9y
z!pdSluDbBD2STI---l*6QMouL@#%+#qn-&l#PX_}bZw(m6dHHfS4QJpD?h|PmAiQf
zy}hyR1UHk?s30AQM{rqtXSTj?PyPBzt2cf(lP~z)C;zm}O|PR{=lNg;;EMTsRo48g
z=@hT(A-JPo>~KR>Kz=TRw=@^bFa)df_F&hnD{efb=CGOKh{2D5q2<@l1@?+tTAPCy
z+*jm|9q)kT9zM<OS7M*JOJ%#BN2Gt3glWO|oYs4rKPxkvbq58kgdZvQxmv})O3U3t
z%n^a@#}N)gqWQkM`!buk+bPqN*+!-MA0J{RbDqM0<7wwxHvP@zH%irlgjKxK@RuLS
zLfykSR$f+q%<i&(n(Hv;cQFf9`uW*t<%DyV^syxW(DX2x2kU9{PqF(liT2HJrpm>X
z1+V##B4Z`aB72IWCX<?9SZ*CBTqwJfsl`(t5U8Ym+kWyAaD!y@wfnV<gNpm<Ra!~p
zCbu;8Piw!>uy`5N>x^YDL1A-{(}8e&h>IkV8`}vQbpE6^J{tQXj6<<+$###Q?R<2i
zq8@N&&BB0u0IrLi#ZQjoa<lt$vshY3Sx~63HpTG}zTDc@8|6~;lwLxUEA4R;ZYRYz
z_og;47?1~wU&6F64b1o2d8M5a&Y%@`+8bAxKyhS7UK74@4fk_p2IRZ_o(dJ$Y1Z1l
z*uqGFgxZw_O=jWmYbo|WZ4=$G$SW`Uh1d?$jvi=z&6Fz2irhPOf#dvb3eq`+LxxDP
z5IsQ}Bj0j%<}Tgm0*yPCiX9*X$Ta8+`gJ2y{aW{eeX&8a{x+7ZzUy;=ZH*VVOPldM
zusdt!<@DO_Qy+Qr>_l?(uOtjK?IhHSw%L$Vc+b=<3}2vg3lSVe_QC&Sj?LKFh)>#N
zY5oAUlz9H|UB>G@fM))mpERLI)&2Tk_18qFqnP1t_#D^A_C`_8)(er+!}WwK%2$>d
zyR1-8Zk|w9dwmP#W)m@Puq}M|uKQ5L_xmsmTU*X@J{kO&UX=WYI7N(T-;^4BB@lgq
zZxLqV^WS`z1T~^<{kT!*D;4nnRj+9{N)yK@`rk$Zl?s|&e=e`TaIg<M6fyt1{`wzy
z{GX4S3eyIrtU#TIYWD=3W~?Sl5dJ-oSi1>de{@dT1{q2PxKeGuTy3~ll<2-tqWfOC
z2H&?MMD>5J{_e5SaE3UPD31W+H0g5>7yFEcd5{@-dgC=>pz%6OOkCWz?dX?+8$?H3
zo-bS$yY$Ah&H~^JEpRau96;y1x(dVq3YeF4@;6B(+JB#(9l5ZdrZ75+Pzq+{?#h*-
zW=7ohcuZ(IKAW-vuBZdW6hiegl=cy<i<wmO0ArRGhp<HiWxATjqfLU62HJzLsVNLN
z33OdVsR{^fbJjb9<_ln>IKX&W2;X^U=FqkIEcMG*r$ImjQXTXuqprZ!3fSS1p#YgP
zy9JBAF~cZ<Np&5BM}nuD*(>P4@~;7GvmlC_&7P|<yaCBkFY6_0FkEXE^P(B~E2Z2N
zXoI#VOGUSfDn%f6DF+p}AD7U5eC`8eE}=>yI6$A8rTiz}-Cta6cl-cXA4nJV9?6mx
zJj;Ds4>x5Tcps^|h1!*QT_Yz1nO!%E=2Jg}6bbX72ErFKhbU3VwWNaN85=XRym|r5
z_5hmRRTisF1^rC*SpFl|nUN(9vYd)|M6wc?qd4*XZ2L2ZeTsv)7ZPFy2a5Md_3V7z
zGeuJnRxAMY7L{I{e({ERy=IA?!0X3*rIFzIvso45K#KFE+aleort03anH(Dxg)L9$
ze{`fCj72)Jdt6MTavBpPP{wW{rXoKyVYsLGO2^C=AXD&8iLs>**;xj_?lw?pn$KsO
zJ4p6sKKPX29RfOS$_k>167!qTDGk6b8wmw+hD!XWpzGU#r9ctV!U#Vme@dm~(^GFx
z*@yrzs3{FE?5ybho7T1q{d;tHq=ALhKz|B`Ec1Y|Lo$mq>Q%x5JgCXF6|I!J1&evX
zi<ZG!IhbC4o2BnHKFxuSTo?&b*ynC)LYDs<1c>kvSjm#!fa#z|>J@~iy(a8%|5tn0
z{torF#$%6AXlBSglm^*um0Y_>N|GpLw<Ri)TWJ)jkV{0i$gVwZg%ZhT=d#P4N}|x@
zI*O7@SG%H;qS@#3{m#rEaGw3kIp;Z^C(kp@n6<uZt@mB;`}us{wS0T@bXVkMyg)S#
zTVq=ot~L0qHDjN?Yp|R=2z759`J}SAiMf9QNLgQ6QQ11V^8}|1zAOI78fSv%kCb&d
zb4hOwQe>o<(-SxALLq8WGW6ON$sT4y%0q<oFaCl75#ocap$Ct+hgsdo91TqhFLVIA
z&AHsUR5n9;XXO(u?|HLo;Ez$x>qZ!$vQVi((YGR}!n7CL7S3eiEP5Oox@p8rv=n)3
z!>n^2YS)AQa&C2dbOr#xrqZg?E)P?0pOQsCz#1gj91&s#{5(M1wL;5)tRb%8VX7UW
zv(U$7jfw%~746><#J4!?=~=W%Ipfg8gZ#|xr}(k4yEV}PLm^Ejv$jt&W^kIhvv|+i
zL8lE%sO+!0rL2mrtZUJL?$Ye)rbqN0k<7RgCit2|BO0}<Z~h4=e3G`9U6VmN*wZvb
zpG6d$%jEGUuWshBU!c^l1+G^!T#+=@%-KPJP)Z_t6~pvW?P*+1zUA&qKOfNf*UZjU
z(cz(DOye$z!c&xLcGW_7k=<)Gn0&g))-*0&Zp8$#UT)La<MWBC&1cZo5Lz#K6#Ige
zbFk}(vr9uuyWQ_=Y-S2?Q23MeTeE_Z-}d%?)SY!8>UsY-M<x%e%I2-ZtF=J9;#H;y
zuqa#H1%-!icCbzZwau%$T+H;^%fiVshj4_6_Go|A&85AJ8&ZEPm3FAzqf%8E9S6`_
zv8M@qOxe3kr%cDMIC3W4=jq`-*Xlq<eMk0A=4J)LX6)*Rq{Bb=6eEs<M((t5J45_?
zYNx4uzeZtt=Dh@^JW;uVYRl+Z#JX;bi@zrp`s;Yr{KTR50IqY-Gs>0Ui-CtgC*nN_
zx~|M-&PcaLYs}dYxW^^&{q84I0~OELfME*AiIo<j1lZM2scP<=(|+u4<naf#q``Z5
z@2&*%#OQZUvwTncEtfU$HdvXp$;-R953;ETsa$2_lhE~{g3R?tp~mwpZs9#K+h?0)
zd;{{ePFp9WV4foWp34?Lj~ii#8_A2mNWS*J*iCyhG&k_gU4%zWZ6WbFfEn!L%uvVa
zu0M0GtgMl5w;#k!X8j`!Uz;xdk#$f9htNsE4kcvTi4frw2Hx=Q(6Fm>D50m{3sy)^
zq#YB#^Ck5AydI2y;I{Q0S0DjAhO@OM>4kN)Z}j992}$LgLpre(A<DRIJ$Kcn3G0+z
z%|>z`@f@!my?bSWo4hyPCHdV*Dule2sVfC!uN;um|J1i5EZv;5X+*yxS|}|;pO!pf
z)l`eAy<e-}5NbL^U$Angdbow_Y?--pdslH$jlxk~NPBVKVl|V=bd^lGMM9m!72&2m
zdp>5ano^e-Q*TgUQO8~@^2eeNKyLZROGf-a=cE<_9r8=ba4_5)LvzHnlP}edpdwGU
zY{>KFjU)A-CiK&7@s9<e<RyfXv6F|v>5%{P4k>)!(alQyrYyl!iPJKR1c3DgxJ+|>
zAAx>47JKjf!H#A8o|J=*V!m9vpWl?&X1E5ckV=7Q{0Kh%uKKE=srg72PsZB!knPf>
zQ<m)AFq9bvVnn8xeTdO_+eJZah%X?2D~#iXTH$Xy6I?xNmYS{C05{;FeQrJ}dk7Cd
zk8gHTk&U$EL){*D-Mm$64ezDUj~IgAm+Mf7N*^LFX{MxPq;HH~!t3XT`35Jr)2M|G
z#hwoMwFt*qVXdMdnmFGX#f>F0#~m5caE%)~IilpiM8rvJ;&SpSYzQ%X;Eh4#UUW4E
z{SA>Fg8h|Ny;!czG1~iib6HNnwKs_)l0H1YgYP6RJUXAN<5GU`Y4uUKgRWi{5KMQF
zKH3kN(FQTrgY+1)Q1WlBTr7Tyoi%HgO4#Ic6#1C10sn1}u*tI}r5NZhco3bROL8_U
zyht`%&aIx&scP*^lonB&j3a>gHhAhPhIL)rmS>mVjVhlSS5(8a-5*Z3nK8s*iqA;Z
zBh4wmE(?AI#Ff4Bw%xm*L*usZzV#X9)a?=UL=2>gQ13kAsnt|P40Uh9wb@t~E!6`C
z*Qs&gu<7Af4((<#p3q_>FP>-4d)`07FP5cve()?fU_zY2nC9bmp9if4@%nZ|sPJ)e
zOcvq+&$mz(bZm8ZoQZ}MvWjOD69k3L=+hc=$y?9}w{!!y%gb-fmB7MWV&N`F%o#*9
z>rY5}!IRG<?qL0@H}YlD#${Ue$_nz`<y7__-TZa=9lEn+b=ciFJ^}Y*t8ZI&e<{{y
zb4x-E`PAdi0|-64qE08%!EbVL%~MY<vhtj{TAxsdF}OA{lw9=VKMkb$7CrhHH`s`+
z|2;y4t<k`5mgx^_f9SqE!<KuT;&Fu6DH-+xJ={Hqv*r2;)nO@aiR*N<?@~YgDQT^4
zUKQ}zI2VL`ul-Zote357+&DORYQcUoJsD1|$5my3!njNzG0Xz4Qp}D66?@@wkFKM$
z4_0iIp&It{KA9Zu*^jkw3YEFOqmmDSAN%)LekKI2IKcplwvqPWzZ7iYONv)KqKfm@
zd#^0p@j_R$+X-&j@^@I{H~e}LLZun|7xmpeKFwYCYK2OHGpVtpN{|!KEi!69do<DD
zt>~gaQ5WqE`|rpSkBC)peIvHmk{>KhNJ9cU=nNbGQ*sCIU5ol9a6S*D#GaF##Z>yh
zY&wnn$a%*XJisjKn5&(P;#*=qEOX0~>qL}8e7z1PsCYD-D#h!!SdJ_TX8u9r5w|<j
zIgw;r7HzXqL=|3)eh4_SD}&kG@BW~P*XJfvp|<kjzWIAC$Va+DMW(5@w(#J!gGO2j
z=KEaQ2d)z-3DLb#LWue{m?|mg9#PK|bw@)mHntd>qM_A>g0WeLv5}NY7mSS_#-{3w
z=p(_{C}M03P5k&yOQ;HCGis({B^VoTy#B9kJj<Zf$cXQrEw)5NiO$)SV(?rQa4*1j
z5JT~xLWlz^@DC2K?rW<*c(7`VEk>%dW)sR8uNV#9u*Jp_`RIbr<1gW4pH|_zgc}F?
z(+)QPyHi;e76Ml4*8{qC*<ZwLJ8Z4qLW%dOo<{DB7BRvq9g#p1K&^y9X7Z*M)&W}0
z_sNaFAV6CA?Ug#i5P%_&LP@0{K<@kGpXrA^Uwr$Qsf30z3-!*LY^1~yOJ^WG{9aiW
z5?`D&wZ$+Ie+cCj{kZ4e!G<_9>C1MIoMbHXb!wymyBZ2pFIJDUT_`2JnskMe8UqYn
zwn*-qu7dY{6(4a$**8do=<WcT$FQBjHI;$*X(+s+Ge+ONs4DirGLSuAI5bnz4B5iw
zL8C;D)8&ikG<hNvj8%RJl;D+~4Kn>XRw}bihX!d${iLH>ppkUtqk^cEGEqq{Dj~ni
zL`rpK26Hg^ikNy(#+H*06j(9_GQ6ET$!Ql<VAwx)RtJxrp8Mm<sn;(-;@MV-DurU_
z8WTFp8~t%>iw>9EO&7j>xHHH1Onk+uCWWeZ29E(7V;nEgT_+(Z7&Kob$#eT9^;xT<
zRE`eQ5**|XJKsD|s1%EAp%cgf<{V;x(yLcv8yE$rGV!b0kmQMz9C4>Oa!YV}J*^Kn
zxCHnFZ<d$vrCUGOqNx*O5I#>B$*4iZFY=2Qcz2f|(KBDr+N-tRC18TR#DFxsXCmOS
zJRWxrR>URv_m<qvPbxo8cZ2bS?9SmuVb#>D+X{YDAOup%#sgS66K}fSg2OZzsWdk$
z67~;;9gjtyT}Q6Camy@_VV)TX2)^S?-@5%zj8>Gbw_l`a9`t*6ODHhK`z(EbZLj7N
zg-$Fvty@ybBCjGp=Nc1ziMTFOt@9ogh;7S1Zq)5dHW6Dk_5N?}d(>O>9F;5m6N=MH
zrdhFP9D_s@uKqOei*0A38c*y=(R6@0;O@#H@MU^B(=Yc|K+xm+=u-53Ra}g0i`)u%
zx3SROl_wULQxDZ>!fkz{d4`bQW|iEBk&u{+rRCVkK~$Bv)*OSDm{~;4|FLj`F?eTe
zqr?rcMs;70%DjI1mA<=D>QZ$=1LMARB8OZ*eX}{4vK-ay<AKj62DY5_KWBK7@1%R-
zg`4?R=){!xW3R5=+)m|tL6049b5CxU32Fsv5uQ%2vsokXg8Sj-{APCu%Kau$?l-8!
zkQ6wF8`uI$bT>}%KR%F-5c8ijKK(JjuXdTB`R)3<R6M*wQz-KK2pNNEPx+y1_@9vW
z6i=|J@}#B7ioofVaw>1;nkt=ZOk4`OlOuTiAUqEzaCz)Z{w~mt>^|q#Ziuo-N6ulO
zinC`-O_9F~tXD@UH1iZu=uC{?E*73==e;N|{w|=FfPh<3S>PMPcVaMu=FPdp-ytJ-
zT<iZp<G{T@Ukzl2VPv;?gGN-hKI8YS_B&3qtoCe8J6BbFE8QadW`8d)*N9j?Hw6R_
zkOIR=R7~mPLUUkYeI<M2i%k$?UcBUSf&+Prajqz7Xc@ATuo5&-7CIHQ@7&(|2Up*@
z_2H{X$;zm9WpYYu#U<noEPl2{K!vp27&Y)XlGfO#Q|lR3SV@1Mi}2-K)V#uG`(%Q_
zz&|v7sr>y5QF7Eudu*uPR?azW7)!<_YLUVS_`W}I<63spTe8)e;*F`U<ZK2w4M2wO
z1vs@$>S0O5+4Ai{7Gh+EpyneC@olVg6YUJEfpJ?jkxkl|ANA^vn?;hlhixPl>KzTp
zQqUl~dV>NO_d;Q=xAAJ2Qp8DN-csrlI|uUFF7VAU#2sS|_uBYNikgEZl=S$MAf34g
zU$=a$cqxBDQXq<@=NN?wk|W42<m^L61r@TZk#<q??c4<RW|NgwUe0m+f&T%E917A6
znLk=ja1j7D{LuD4l6pqJyXMGWX&F=l`(15u*~2|*CPT??_tGDc`xlXqCxUvAn~IWP
z+>Qip?qHVd$hT?>ln(Fve`PV^^M9wDdh)~!ZevZ!=c4Kv__0~L$*Rb5SLD9{2x%!u

diff --git a/format/diagrams/layout-list-of-list.png b/format/diagrams/layout-list-of-list.png
deleted file mode 100644
index 5bc00784641ab3cb3c3275ce1e52e7ccd3b2ec37..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 40105
zcmd?RWmH_-7Bz?l4Z+=m26rd8LvVL@ch}(V?hqij1`F;Iyl{7yAOVs-<lgsQ^7ZK7
zJw`X<P!y;3*|N5+HRs$FE-xqk8XgxO3=HhGq=bkf7#L&^7#PG8ECg_;C?+!z_yg{w
zC@ut6HGzK!`~zn%q2UAuhDrYN3ofZhb^#=qvrtxdR+p9GHnOv!H!!v{G@*C5u?JFv
zf$_R?18;3ioDGQGZLDpbxZU|kf2ZIE-oM;tASM2t#Mz3ER9#k{SlG_dgqV$<m7bB5
zAD)<)nAg$Rlv`0m?BB<M|M*DFot^Eu85rE$-00m{=<OWM7?`-YxEL6j8JL;rfE092
z9=6T~?sT?JZ~w{UKlzB5I2k!w*gIR;*%H6xYhY;S;><@%`clw;fBvbbvxVvZO0sqO
zw^={~8D2hNV4`Pa_;238qr5M7xrOa)>>W*<oPhlK*?51a{OjKT%K0a~yt$pT9bg4V
z3nNKeXA?)@VP}Jv+VM00`}F^FivKgO4~`ZlK&yX0&Ghfnf8P5yJukyc!~ZbEKUV&I
z7cetFJTJq4yNn;cW}A5x3`_t_QbbVM9sDF4`U$=Naqt;lDm2iaI6p8T8roGK%+SrP
z8MGRnotB18RS>os5MgMjPaGj3S=#q&{Cw(x!RvChW3_F$?Qtb?A$4&v+-!}->u!u?
za!i#@yTu;PM??VX&#MPnoms}P7y}#y`HxFoM8L9PAOJSZ|9@971u+{pE3?U;>5#$8
zrf?xii6Q=XA!qo{9()^068x)}A~7q(KQjvxFo#eq6n?Ap>aTL+a4VVq%#0W!A5J0~
zDOv{S&pQ9E+#fXqGXDQW<#M`u%yZ9H+d`|Axc+KyUK*6VVWCAgLd6hKF|>+`ib5St
z_5XSXXg_+0fCyWQDXow=Iw>ZOa@b!VLXjXxlBjo13Chz%|MdZ_n1JPtA*VCRAN@xR
zAqWeYTu9R5uPW#>0sV#5n~(5Uf58d~SUMw1Nxl6mgGiJ(+zxCcoxeLA8OR_=UJ>`N
z4Ai55(b8{!AoSO02}F*=B?wNK{wsqnL#QG#hL6Hn{~K$-P(cm`G9XS$68kFy%(ujB
zo=o_ye~r}uVju&U{36!BGN{1@#&G23w~)Uq14RN1c3ARals_f~azH;|8C}Fs`oAng
z3=3o+NJahsbJJSBk}P7~mD0bz*nI!<e!KeXPt9uGjubkrCaVS8akd58g#pZF9ixFL
zkG*uWc}=Uz%BH3AmV}+gTfv}L&$r8VkLO?gwg;lGH5G1p4&zCLKgv(Y@tiiyDoTqo
z$dmsw6qIoT%0m#a#4MatR4^9bVxo$I?T$xx-L3oijZl}i|2V2xJE5s+*<$$mv=u!}
z$Ln^sQk<&G-P~@Hew^#~Q^O=#MJxti@>tjVvY(L8vu}AU80_oanpZ6U5A%$s&5wA#
zSHqN-P4UxPHhb5T@4p^bwJDwyNhV&E-%Mt5*bV>kF@!0A5dFP`I#M7<zS8oPBk=q6
z?Dlwjs`g`jzr3Pq1V@kDEQ{SnbD>ISD&K1Ub2x>=yoMQp%dEn(P4nsqO*K9W0$<$#
zu2q)rk3*DEN$cAMU9a(hwaNFt@U@5D|9YBnHm~!(nG)}2QO#uj(B)*c7fR%74)KP7
z8$G#Mb2Er?NQor~Ax=w*q9Uy|RmXKP9F1~MHPA8q_mW7KE^x@@f{4x7Wx@D{@8+Aw
zes-#H=Wx>LOUZnmtlT$V=N$`Qoc6IpywASu+oLaf9#<OCvM7j-r!heiVT<&1;cM5h
z+bmhPoi^FEUy232dVRa%G`WFOA)8LQ@rCW$9}>y^>2gpZ53a8bKfcsdHIl;+qO<J6
zF(0|!Lxfg#@gMsnh~?eP9+c+#;)dBQH}yk1^@XA6`Fx*SvZ~Hz{S247|9rPD-qUfr
zM59Z<y;(_L;BKx^-F`K)pX(R1EHhIg8%ZT;Id{E3S*=hgX6cQ@MC-sfcz;~|bzaM^
z1HW#cO*(3PtRkx$VNfkN-M;H49~FjHY51S1DK%CG$si`<>8y3W)^R*rChTvcb<|%2
z1yYs3Jf%P!rIYehR^k1KRxCW3!<X6NuO6eoRLDA>nHpW5&FvzUbje0Ee#8ES${=c`
z0vM+?2EBo^NC5<HE0?oMAEIjugJvs!uOjldP=UBCs1OjbNSHHt?AD-qvq=J1P=NI8
z!6<#(7$qUak#2`)t^e9rKR?^}s8f-H_3y_EgFih_8yD!CR~*-9&2l{UQuTa20E2E*
zMKwcql7f)@?l(hY2za(wqsljBW$wQ(=uvh9cKk8(;k|4+vq`#1N;h|mM3MjoJyYYl
z(<GnvntktU3)ww!e22dCufHbfP+PPOAYr`+3EVD%e)MDM$@7d{Vrp4CY=u*ra(Q?0
zvTdR@RW%$|x1DkOgXomL!64zAgQxmtyUc(2`Q!M@PN?^J$K3$ea(@deru*Y*^Z0J|
zO%cge%XW;B!&46e!;1Iyc=ykf`pGR_UXQCCS&r44U^F?9b^W+EIcOJ#$opa^@q;5e
z<n5G-noe@T1t{k1DZ8mkCLZz4$Kdlbwgv5=g2PDXRK(Iu+t%=BkIxPENl=uZSnbxb
zPQ#s<z@;(aTWA)sx=h;vE6St%c#an*3@2GkCBEMUT65xbKkz<B)tH(@VSGo@z}I&{
z$BoI9|M7(76&xWambQJswKUv!;u)(@e8;dxpPpbiO#R)#JKmCKIap3}Cc0dw2`*lA
zUX)ngt0CLA(-dg7X8Z8X<=8jt7SBJvTg`q_xV>qm`MskdCxL}pi#({aDM-O4H)}Q9
z&bFL)J|<Myt#_(EHn000)9h3>uWezzgEdW4R-09=)Ql$Jna?yCrE4bR^TcaA1HCcs
zZ~dyK?GVCo`B|fWg-$Qe<#>KA!GXshh~MJdZ9BPo+NpTTGccxQJMb7k65T5YP^YN4
zcqg%77%GaEEofZ@zz`m6$9WD@+tDLP;lZiLif!7*)7Px~K006}$3G;9R<)hA_N}<B
z`;=z8W>oJiHCnL<I(q1U5%hDt<1`7o6d<9OwFPHIP26ld@2qHoa%Rr$(ysGjP!h|s
z@kZlK&7RtF41qplZUs9dr}<n15#l5S&4Pw?4w0WR3^hG$0dC4!^=Vc>C}a2p*McdF
z<#-_M%nIT<;&7%k^CF+ncu%ggU|N@1=O!MG%{(2>u6~@I;=LAuw`j30|79N<FKyAw
zw2x<G3ov|{oq5x+pj4+mRR-<~`VmS@5(xM%90Jp+?hy%0Ux_g`WUGd1D5b(LR_`xp
zOWqLjVpr?{%ciLYDD7a}(315y=V^MjT;;eRl3vJavjN<RV4zQWj-JnwwaH8_a(kx^
z*myjj?l0H9ucMa_TOQBa+LEAJ-OiCn$L6SR*4%s2)V_oygbt6m(QFjggWJ^&Q>gEX
zN8^SB;L~JS#HmBoYHBEiMQCpD&s%}ab)ulD$n@vdj0FVP8s4}n!LI|$>95H9J3swa
z+A}KF6lm=t=o-a^O087X3C?Xqx6xViBJvz+<Z{N2UTk&Whuv@kOu^WeJ}6sYCEYqa
zsHpDrTxoImv<*6x-S<ZvMOy1K=*_>8&bU7FBf7yZ+JcO4NS?^zO0>X+V_!0V>zU#p
z+QzvBiLiUxygpGpih<cOY7SnaJ4W>hwi*ocwYS?TvoZ_LuV0N<t3ZmJg!)fY!xklF
zQ&DJ{3N;6Pl5cqh*~bep)-G|L1WN*JFZ6&qCH0095YSIUtW~3@?K7I=fM{pcAKdqf
ze0%RbQ}!oJ5o9x1B~`Y3!d<9eW`W6vuFm&w+3S;L4}hs1$6mN6m(4wwF}_5@#3_1D
zm4<8Mf4gXw%jLA!0`<8HHCArV5H<~0mUlG!y!|?+EyPzpBoFe-kPJ9a4n$}oGy-M|
zD@ckZSs&@u&^Br)d>-jF{TujF-CV*4U~$zbc3bl(P4>83b<>i${rZ#=nn1#K`3lck
z0K=m4e66nu`Ze1Go+PkB60yT|&>)tGp~V_~T<|}QQS;gefYIjS^h)!{X@tg-1k>I_
znE2wmc3-n8|D(9s-mP7Rf-NJ!%w~}DXgrM%AGw7FYP6V(zP<*M2J0MQXJsr1i91Zy
zSfMa+Mn<VH9+UWf%pGonQH=Ld@KUhueudDuq<X$|O(U+E6hR!PLBh5KeD8_#s0i~c
zx*L4#72+){(!=eVqWX&``Yzhv=e@Y~chb}##Rm`$0Z2AmshSJAz;#0+qX`2vK<IO$
z(ERyBMl{D-+SeA{qLTemXkmhw{xi&1I@d08nQP?rT|g7%z%X*x-?>F%+2{-GPVa%y
z6Xxp7R3P!3Fk<1nyO@?{Fi0jth=`|KXg&F~r_TDN&u*Huc7&zTo=B-kj{A@d%LqKv
zsKxOLt0vW1U-|Z7mj4!Hq#SYKUi@e|s3@JN)nLB4)#iK{NIJ1Q%JB7s^zr5}DEp1q
z@-SuY2`QX@s3~{=7@cVG?V6MXsA-H8vFX}7tj`$LE)MXHa;p)ep3U7wFXLtPSu`fD
zzSJx$4gbKH=$prS@QR~;KawZvIPm^h$GiAqqXv{<a-kn>x(YT5kz(!YQ={e2<(G2P
zO8$G9jms7o{xpSc&A2^<eeR3v3Y8(2cG!+IR$*H1fNnqCp9w*LVA3P1=}(|$x^_FT
zy&?Fc0S}4}r*x<8eR!;r)HgM4ICHxa-FlhWnUF{hAC%Y2nB6+w7ygIeK8qL1mh3T8
zI_{SYPg1^iBMO?&7lr}sqO~eVM|X>j`E9%!Ot_9F>r2F!GHfO+JMeZBMFMq)h0QY}
zzo$*qOr#OxVIH@-Hq)Qtp<;ZFhlyxi^D*_vSKqu#a4eq7ol-EHoS6z^`te=AZGE)*
zdC7&9<}?7sp7`9-xd&<?oqk>U#5XjH>hSHV^84i=QGL$<2jVf<g#DRSk`8mUK`IFg
zOeYlR)d*B(y5CY#1p4oQ<5S9uL7v$41vi_CzG^MJo0Wg;9xW1kbT((M`ug+7Q?Rz@
zVR4C6YXNK#1x11|uf(}mF6>!sOGQCNEgF(Eqb`(pd!R?;E+<moz|r=tNWt*tl5|GU
z8~(IhGrO$~gZjkut>lafoi<mIx}>pq*fO_B!k}Y>M-dcAr1rz!9g;Cz?)TTOG+}YB
zzW7Le^tA??@({=9=Xze(EYyokd()JZ{%f(B;9_ncVGzz}I}7hvB1aphQMDaEZQaE-
zCeQ-TO(c&vsXH7~32nG(s}o#@FwKWI%+`f(G)wxdS46CHC@8Wf)~S^S4RH0%9a1p}
zqh`_*sC9pbal||U2?~Ydg^<NK9f>MlcinTzn2uKh-7-<wxAt6JZ+nmsj7TsREcw$G
zr|ZOK*wmyy`<p@xevS#0-pb<R9*XqH`HGD-qk5Dwp%i^)F@wcF{lpCl-Y*d_%@>EW
zcYU@jHZ&E*O>#_~ma#%a9lGbY*6pJVjy*MW)<{6>Ldb#?0O43ny!-leh2S*{yB*W>
zsxE6c_*HD|=~Q2K5E`Tt{kxQ?VY?D7dB>kdth_+O$i;~oyom~4*K6QRvdWzqO*-7>
z7}moKQY5_6@}FF%^%7hC2V=QFrFzYVn^^AQT;m$PTyYm<R?`U6Y--ngg*A{C%7r!;
zJ#)#ZYOrIp%#)jG*U6<+Jc#dv*!(R*LaggX6p44>V-mG=<0VaH1F8ixjw)A7MjQ4h
z*VN~wf`|nF!_ZR#)~`s`Rjz^JpUOfWhV4J5T0t!Lz_QpGuo#wnXq5U$Q+WaIt;30m
zZIThx#q~4f`doc*!@LJ@BRL`%PnVm`@|gGgG>1{*&dS0SO_SQ;>?mT(8z=S%tBN1o
z<EVMJ@^;Oob4obYy%QU;Vi+3W$`co{^mU{Ccy(1yr1$J4JLA*Te@q`J{?n7h0n_q@
zmj_+YTwaM;3M^iAVQx%HmrkN2>fgtZ35x`7Dbw{Bd<hPiTtx(^e3%+BmAa?ij@O(@
z>;Gu|0C!P5JXq{O=Eb35TGsD-zIfdy6(~!0AzGVwShwTY%qUt9-)&Nkdm5we5H3yM
zg%4?h<ukFyu+1Cj{TdD_3l3wDe+wpxw7)EjN>PD!g`df>Q^u5KTmy%Of98$dOb`Fi
zJWo>Gb2uX#ceCK=G2$XYx37sRT@Ca7%4B;F+8&wpZOVn`TkR$FgO`xUr!CRv@SjbQ
z6j-w?K27aSb+@hGvq?SYO-)N!r<(?juWf|6m965#dAD7vr|dd^kqOy$v02_rdy~e&
z2;+aj{8i>KsIT5AD%T^pwi(j1zUC1*Rosx&@@S12@}mtN(UB%{t~F8{+b{SM0sK!w
z80tTieCAnHq7xB$y43q`I}HY(%xY{3NsUL;g)XBKvt5_q0xo?ey}Bf*HW0C6<beJo
z)|ekegA4LOH76@iAEQ6v%fMd?)r(900D@Wc3`GUa?b8inO?Z6UO6yKR?DPhejpcLj
z7q6=!B&wj>b_Tj1aH1uaB{2lh{}adwp>?rh>>7O+rOsb!=LFAG2wiV#2p=a4C$r{l
zhQhZr4fI%lPGPSWE%*?(rW=z+j?*l!b()HSNAgt{Gic&y^(gh0+~Dcs;v#I5*Wowe
zH{-erac!pK#p;OC>c_)9rp!yWc0ES(n{plBlT8!)zGDj8ZJy|}T*=@aV6}x<v#w{3
zh1j?n$jIl-P_V4OnNV;zFB8n)p*uE}=w0qkHU;;xK+>&*754Ku-&%<Icp43d=L++#
zJ~Gk;Y<*BNP1Cw@UyS8VALFE92nPcs=&4A-zx|RPAG5_|M+7}aqOdgWwIf*=%{a89
zttpU{vNZi{#oWAw5}JWJW;Pg6SIZ*S0O>n1i@c5Xbvf~#i+3`Pfhoi7>85`uLBqk1
zGn16hjPmg`?`-^csS|^37A?GZZRkZKfN)#@2aVB%+z?WF<2%h+GX9C#<g6x7T2IFf
ziJv7aM3&F(;>R2nM{QIaG*tGfNQO?()=c@7@lFGjbeCS!oSCKcg?o~aS1cSf^UQ_~
zt*sNGFVY%E+Wp8_E^hQp<lT^&Z+*HxF<z5XE~2Z&pg1<>Y-)prOf<#o0p&3-?bg>x
zpnNX2{?tee4=r*^gmU&!KNeYtq%xEa&+k{oZ(I&<q!SnKQjgk;^AGgDHoWQ4GKBcm
zKPS@Ycz9ZwveNS_6cQ6hvkK!=yRF>Tv=%X~eQI{;ur61)&Gh34i8OM1dOOS%Tc|%B
z#l?as^K1KLR0%njY+Tu|7ql!wV76D=^kW8Ny0smf@^9*R+Ojp>kuU1&9Kv*u=~!m-
zOdQPzwILQe42QKc7-a0OphLi{r%ZW5&#QiqyF6w%P^@ELQ>s6%4a3bIhO<V-h!<2>
z_lLY^i5-<_F$X~-*HS+HtO4fe0GOZ@7fA}XrR+GM2c)%xR(rmUcB|;VL7P=Ya+Oe_
z@95wA5M6?q<fRK|Pz@l=%BlU+=My^<MJ;K#<0Y(Unat^Qpof+<F^}FB0sADWOm-U?
z8b`yt=|vCfwZ8nG?(fnO{l7rv6E|3Dm})$sD~reD9pSa%hBY6oirjE|A>44d)&j~9
zN|8!qQZ~j#qHp1|a0h0(<|K%@#*GibwD`}EU4yw2Sb7bDYlad{R01f1ed4ATaCU5O
z6j314nq$0bk&)QU7t==NtWFL(g{F0<r({q$N8UCT&XlMXy+D)75a5s95uoXJSEWC%
zpktZX$PG!lE2QP5CO%+uIg)+##?~93R;siHu&`f1xGc%+jPCZ)q45g@g?<NiGkDB-
zP5%%7FIO3ep)jnx&gj-&6TT-E@fKhoo!F-xwDV4nugEu0U;Ar~z2=0Afyj@J=V8-W
z!uBTnc6~sL#AH7IIbA7jnnF&940p1{VH*Vh3TG_kAW!!@O<S;_|D#kF4%m!o$=W4(
z%q6qM_>SO5!ecPOrj^}W;+cH+D8|KstL$g{DE}bL^p5>YlXC(w^q_T~im@DBx0gjG
z?*y)|pj#@M{u^gE2C5H=5trX$0+BYyyEt#r@)Ec6HFphpE}k{!4wwG2DUBF(+JyV^
zrY+b5!KxN|wICpekt6|HOJ|F$S%o9MFb|7ZzFUFN0Z1r4$-{5+`B`)QIK#mKiISo8
z->Xuec0~l<IC6b2St5OLKBwSjqjZC%hD7|8ia`mJI36ct+YG}m;o6#54z&+?A0$+$
zemgBl+uW=DLaT<OMp@{3Za|SN59SNHmn?JS%Pa>6aN&u!#=6wZ_8S5f{#fviUD|#1
z1=mzo`4M}_we=I+qjLjIluaHU>wBlR)Wi^QBzZ7gBm^W(F99*J1%ONM*NU^RdppIn
zY(=q8Xm69_4lh2X4zjDMv1&bY79(}wP-Lm2?sTjGAv_O^%OafK97gpcDe`-8zM)GN
zmGBoxTOJdtp}&T<RU!IM*c(<xKnoGFP*LbF+&wJWU+yRjqZA(P4}kj<A^_3T6z0xJ
z{gF069bjN${{N7YjixZ&(u53x&-2FpusD?zBY*6_Ou-?n^ZA<3w>-haGb;K+sB%nt
zZEVqj^740e;<RKDU?W;!5c8wZpCnZNr7Q%XQR4_z?&V@dgOb@=F9_7i*^WpU=5i8O
z{>f63oLE>z(TgHZ`zjhKSTg({*z(`kE@^;$Oa0jM-_l>+Acp}&hd^4A#9u^5J1}&-
zEU0LIhfzSP-&ep3f%E@rrn;{PVG~iq1<KIUm`z3ieDmQ2yUgMBc-T#~n=608fLDQ4
zMA1$SVwZ@;6rd-0>j6%{rg2Uc4t<gawEzi+IXDgQZ7j-)Q?CAvsYUWMf$>=(S_DNZ
zzXY{6nd=8YlX^fLlt>|a)l>vvasWJ0!$A>xvGT<!RoC+npE<=U#qZ~XS)wdE4zNbP
z22g)2ubTrDJLzrcpD!V9=i`~g5NPNw>z(fKc5(Ui@9D?^;dnF@o#}`X*afVmtDQcN
z$v^MD9F<qVE0}Q2I~-Rw=Xf5K9Y|Jut{Y(jobv%3vy;XJT|C<s0{}lPWKAX1Fp6fA
zd;fex6+*^!ZdKi3YIF{u+DP=k%TEh#Rt^3i2Zb+qx<+f4wAz&=k0=~a@VhMk&r=xu
zV5qB4;E=C4U+?9I5^@6w3M!o~X-FocL60bwPOv#PL9Y8&I5yqjB;Vc2#b$5wn&+`d
z6VI!B)3LNm0326DezmvJ-)J#o3a|}6+tD1jtmZZQIbN?O)on(mp><mm)Rw8GfMFuP
z31u8zbpalCum{w1W@w?wM(bw`Hq#aWQAstlgrlDU{zw|X4|jnMfZj13eG9j?A`LDt
zFZbL{R<lpJ-%Zt<*Ku1F8M?_uZn=BHqD}m@Kc1;ZL8rh1G)5{B7_HMM&G2!!lu$CB
ztPQ{#?s7Cs($!Q6Z|*iG3pmq6)U^O!+A`Ru-4HP<kS7+6dq2i%v&7o_OezAP@?O85
z@7{0tLykx$60(|qv#h8D;Po0(FqPz&L4NTb=%jA~5>(*4-0CuRpc(_<5jT(l481}j
zL)%H6vcn|rwai2wCqPBHt-35s6^d{F4B)x|n3Mxh?Y4qALCEdKc%WpPTD2~1%iZ~U
zu6u;f&xie~1qbxAl`q#of3~eSj(W@h!S->|?O|!I;{bH14sHDyv-%)P_)&Xjf08TE
z1#y=j@oYDs5bfYF`M1K}`T|stZ9MUJ+lw^c=N}|*%6kD2p%xfAj`)2Jg9ObJ+{ec6
z47U2hSIX6@*Ak=$7|lA*T93gDUD5;jfxQEP6C8Q($8q&OYP1ai!RN5sWJEex&|U!a
zSmr3tVI>@q@BJous`hl1P8+|^560Nm>j|Fl2LOpz)%7spC8uxM2yA%uU8CM~6)3K#
z@0E75txg)F{(&cAZzt}x+wg_H#Qn(Ey$maEm!qA?8{N)_z05bf7hp7WoD2r)HnX2l
z)<BOycC~|qR>MNzcPS^Ye*ydhU{M77Z|5(FIZ(3axn`OXH`ojA?sL%%V-XN~I-W%?
z?oXNPop3FD@yQ^Vh4JF68Q8t?s|Bw8EXd|@j`b(NCs_o5TyHq%-=Ja$aSh|rph%SJ
zM@PzR0R_)5FW@?`xjhPnvXIMswEOzZjxnF2Wjm~+T&t$%<FaUy+L&~9j-jT#IerD8
z<qimqsE^v^$L|KaULf~k@%K)<_Yjp@&9@~_01ztkfp0SyP1_ag{^#R49Z%b{wlN3{
zzLubTBM}Pv;A)g%QaMAGw=02K@Y6ly8{ThfUPt7m?O&J`mL%lsO=Xc(o&JT`n?0`g
z#!_h$G+r~+^dDz*USP!h3=N7HwFR#}4^|r<u;8&N?mW?;zdT?A2Ue3d+Tjz-^3o5Q
z^ovDl)Z2m32rmfx>Oi{*A_8w?E5Px2hhV7Y*&}V~L+nZGcct4gc33ag6(@E>@NeXM
zvTzKi8y6gFomO#pXr8y7^XKkpUx@&`3OkwDPnzm><`)b-j|>*%&j6<|Dl?=$#DIb#
zMJptdB)E7jn|IMMhC|>QLFY=XIe1Q9j%}ESUcx><djJquwXUa2?hG}BNR2KB-}2Q{
zhXcpcrnTvn3aMX5GP`Mb6Y;t;x>hgSwvCtTzb%xG!i_$CNK%s9{67HoFvVj<)PuDS
z9%#cVF~Pe5w9bnjgfAhQT_4}CS8k-LuTu3TnbJm5oq?um+fj!@PlKwcth=ZLZkbpJ
zWgi)~)_*a5Xuau%{EesY`^ukB@SH#C^?m&%yoM2$^()a9ZtMv_rq#=8@tsl=g;(fi
zjt<1xj~oz0(ePNL!ArVOwrOB0GgxZE2N52&W8aGrl*XQQKHAcxLrQ87b3qA}O|kB;
z{^GxZCcO-hDDmV9jDz|Vb>OF7w>S~Tltzx8!uDxl7an27qyelrKS&W};tehC{h6ah
z7Y<>Gr5OxiIfql+(*T~Gh;ndutRDc?-%#W{mry}QV3aC9NcLJHe!1VMOZah6Y^o3i
zOBU__dZS2E1@5X(NY~Pt95MNmO_a%}5Y(-17@{IY=dCiOQcbb>oA7RG(nfrM)ELdU
z=Q(fRM$Nm**wMw9nKh3z`|b_E{q%gASeKvQwx#_8lA~Ei(7%?lv2<}=a9*U+TvhtO
zXA)9)(}oK-mc`}t{C!~`^N3&S>#F5cK?wQ871n0Q?hf*re$Z>G_m$sQwUz)zSDbm4
z(~b;)kbOaY*Hw&Zs^Yx2qED>t8W%IfzWo?#eQjO+nU&eMtqz9GX+9d8?yxO=29Wu)
z=60KB?&-9a4X;VoWyZ85EIEnoQ&FH-S2Wc@o7p_{Yk05-yvla15rGmh!A(hhbqS*q
zVEfSkrI)z20<dZAmurl|{d-+wS(o~kFB^9Pz#e=~U@&`-#720)ULlu7dd3LAh66Bh
zY&i$RCd2W&_+m4B$r;t)bj;;tfTi3#b1bj@!m#bLNvfj{?C^d76~5ZTbctuz&g3^k
z;rfNv9zg%m1YCMCmRGk#@%BRXoxreSQ%et3LJ3^cA`$VrMpI~qqNY|eQ^!-?osoka
zaE(5#213?DKdKk&KfMgX#hhI_O25s*SpJM24WbKzf+vtsEXvm@F}%&598I1dwUcuo
z+@4l&3N7i_K$Dm~&cU`9JJqInV2H9eUXb;Hsx+1o`W>juxYF^rlpKJa<GMbyHn$Po
zG;L8D0U`_4@p$}1Wm%_vQ-jTNx@h>t*^rGT$k-(|#BRB1xl2J{QP%Z+Fu@FTpT|}J
zRj$%O|MoqqUqS(lYWT?zFjwjs-}dE?Ab#_bMwxw=jbeRFE4yS)_YL`Y*GVFpA=7Th
zLx&@BrmtI6MD){m8*J8}+pis@;C@6%cb#{XEm-&9u~{blM#J&QL82ew^{)ri(rDeS
zicI^Y#OM09@$}t1A_;A3-|T6TrW-wBU7r8Sv}w|?RCoyM{?PRV50c&=F{?7v!iY|b
zbPGWHz&jN}=l>WD4brjzSGF`|4jy0aCp~2l4(ad=_E{$Cu!z7jZ6O6qaX)DrYXf!Q
zrt;Pl$Ktrq?$xZOq>II^77O~nHL&f#DYCe?i;F7z>}G0(W5WbPkU==9j6TV)%STIo
zd0^9eSYm5IrR3TFJF=n$>@gUq5Yi{<pDUnE_;5OSKA#+kYZ;*og!`pvT4E<j00!Ji
zO2)>whC;SiyjW7L1?MiiF)&ii!?Ss@XtJ6ftHwx40v@w1VBMhGOqsCm>%_RxxLk7L
zIRk@eWHE4<*ynCrWSUQ6>M?Fy`e9sg2)kVz#3CMT-vtROr--t9g{jeoAwuZpi(d`b
za5F(W{?zxQ|8XH|Jr6<AeRD>Co^3o?l`$2Fka5sNQ&7^%MNwXre^%2^Dvr~#{jwV;
zj5PeEL3slmL(Q&4%A{jBooh|5fzg=O=XUMnTYfJfdx-@hvb39Alz*rcW%}*vGXRRB
z%PVCICS=A~q$JZDxp|LOn0lIGrz-)>LS@7`uZXUi(9v#5%53A%kXqd+B6cF4?D52l
zbz;k^FBgW;tcmQVF6<o+JQ3nBdma^FkA{}TeQBWr6mLD3XhWh{KY+1sexZ)jIGA$D
z()p_Ck*1-6%gTk2MxDWUD-EFGEifA_(Vm(%!~IfSRjS&}4&Tf?Ki-a=#1|S}K#F0F
zX^;`(5E4bzCHeCV)=%<hIw{x|_wO^JN&jwP_*19vPQvR#XL6!tu8fQY!}|g-iZ*u*
zugx9)d1K~c<@OgmqfZo&j%kyr$nip{ud>G?rI=3zw-YtVXb@nZZk2s*Rb<>V028*`
zrs9jr!+zjAN}B|{h%D-3z~k<vlQVdW13rUwnb)P9@`8r%<qFiv%yY^W?a?pam(!I~
z!KL|v9-qdlAuGOQKOkFLZax{S0$zkw$U_CPX_tMI&e8{TzB;ha^4~dOvy`Z@DcQjx
zk>qZxYPp8Gyj-zAjd3xJ6r{G5QdlcQ8H^fAeHkT`wric%?(#%~wRoH&tv4TV=Oe-1
zw>RqA(|$kmU1t;iCGr!<c|iiu)G;#Of?GSzRax;An#Q<5jecZ{(KDGgoMRDokMlkd
zaQ#XD8}ddK`VD!@G~Fs<Pb5fpNEdNQ-_K^xCS4Asy{T|ZG)cLVtu#O*qk&^@+#sNF
zP0>ivUel7PXzBZQu-3OZYwkt{%x}05rn@btF$yjoJgBzMJOaJT5gT1xq#0;B_P7QC
zfo}8iEnLtZ(iQC16C-J%O6z_&O&mcIwjJqOYIS-|(y8>;J0=vvoRY_VDmGrp|7e?t
z0r?w@`AC8|#BYTA+oa-q^M=`Ix|x$|k0hgT@=pN5J$4P63rXo+{=QPo;9pv2lWD}S
zzt)w7nc!-b$6$0Tz05Z(O;w})L(wsbnMu8EQDe6)c|lsqOkp9=I(Az|Pb=OGz7L!l
z_G9t-aZrwqGIvkLFbcl;bV{0v?{(FtO4yuSM_p=eT-yl`tCY2oECw~Y{j6S=81^Sb
z2_)}60O7W^OiwqIZ##ufgduQ7w#Dc)Ht=Ez+nK=rb-x|^SU&1Ie2ltS(WMj~$%}MZ
zCgpc~@{g_~9#+39?5ei4Fe$Hnm{Pt*cK{BAsP+_G+8=1t0OvF#%kc&36Re2m`tzP`
z@heXDQK=2srrEYNY-J85V*B9@@eHlL_M2(xH6LI6n9XrR1H>v<_CT9y?wfDIN;ARH
z1|h4#-ENqX*(QTHvmL2Wk0wjZi>#7f8i#UT98zRFpeVALS3mkAgtChtEFv2hLzn~u
z<TA%J3>Gc-CUyw99E-C11LEi0-Z4tG4|gqwb-KMKg|ovnjx^IW2oz~`DqQ^hF{tf4
zaaWdmE(cF`MjrD}sT$k7%Jnk|os`ciF?BCQa}O6vNa6-n;v%s?I~%HNm*gJO&VU<i
zj&XPPFwBa{o9Rq?2Z7;JA4$-j<YfI`9h!bB#2Nh>b`P@Cxm4c`VR5w&`LQAtyku8r
zAG}845i>+>+`_RMYgRcW@~C`8C+;K-7pN(hnr{MZC}?+%d?*oD_Q5RIr>T(u8fj8`
z*(!wXW-F4R>At~|6{n+=r`~8R9GAR{ha!sv^DD}H^>{|)$V87%Ny2rh*!Im9-&J~{
za+*FP!d63LLl5W@W#Ba(oegg3Ts-1jZtsRWQ;SJ=Wk}=WtTz6;lg~?_Qit5#AjxU<
z;!a-P#$XvHxoCE9Z@)#XgYhu_(A5Qg^*}nNfL~jd1srWRRG(A8+z3QW%rDuc@O1i&
zO5LbC0&2Q+baN=YcF9U?&mY4jHlQbnVK^r+2GG*((_4hR1Bg2g>!6bGIZ*dMLF!KU
z>`#f2WqN);nj6KC0OP|GN3s{hE^NC?=^>TZggRNyYcQsNV_`(Xz}Cw9O4wHmWmF!4
zw4fsoFUTsT572ILUtLH~=De3qdC!)S>&I&9#t(L@Pdk%#lZBfk8STom%vfV$Aq~>%
z{rI|<-fG_w47^(&$0s7bdhDreN;x<>jzbT`V{09kH)E!OX%FLWHa(f8y6-FnFEYgo
zQ2HJu?bQ}%s93IYWDPTAvWPzm^0&OAyQj$!@L-F@w*5}Z<?y<a`{BCoy9xUDX}jZ0
z-%^-Kv>BJZm;sJfo6)74EesN0hYZ~MD9zU))c!{PIX!>#IjeugmRxx`b1ecwYQljY
zKdQH4LpGTsAqrfiZpN<Nq|PXrZU{8h5f2|JFpN^;pzCOu8qif6R}CSu#PIANe`|tO
z8|Uvrtx*fB?9NaR6{`pS{sLPsn~&2=Zw1gz?%4*s+mq`U&mq6p*%9;+$LQ@lK+Nr~
z-+xPv^yWbFlVKWTcFK7LECT}4Vpk&g>*=QI2UPoT;V+v)r*HYjm>kXCeM`-@BjMdk
z*WQfqC(m^1>?luTa6qMZ5ZX{-zDF=%9=so6Mlggyr-ilGQ&0|t=0K1!SE5BAfZZO=
z^u#IRP;x>+lXr9-zdFMI$SuR!_|4*>lR18Iyv4g2h*XP(@_PIXB0;jXajCdEeT1~>
z<wiJ`<TFb{fd$z43<Z;ySno}(E#f0-Lm@ox&c<7L=?n(7N}8)U*)sTC@3E7F>qi@2
z3wqYeWLBAm3yeYCAmFil>+0_p(tUps;6alQ3q|aYVe=cWHXZ&61j+G8He>cy{QFr>
zu#l9ANCf^>31or(o-8P<#3X==h$@h03Sk@vODcZ_1;9|$Na)*9ysinCskbZc+tKYh
z?K^mw`QTvPBxx=lO@-+*EV>i7uTo%(-&r}pe&UGi)oBEHDY8reoiV<PEuukzXje4e
z=n9~q)%^lMNme)Y1=2-_IQ=`H8(vWB)jiSi6H}+^E7wc-Bq;FjrjpKy!GJ`8iV^To
zpkGxTjU<A>nA^|eRi|$xRRMV2u-p;dWXUUAjLcql95zoD8P3{ZVbSvOk$4Yr0vGh*
zllu$H85+TE*qv{g(224bFPc?>T3GW)gW91~#g5vR%~Zu8@```8*yzFjz6J<^T(acA
z+Jo84YK-xZUgS5*33y7205m-@A^w{}_`Pra_Mu+`b+gE$lKKlMtwW2$&CZTJdtv4N
zdkNqIfZ~jRLed}i{9%nLfz6MH=wH76Uo1U<74Sq$0}81AfX2a5hU5Sz#+CnHwcvlM
zWdS$=a*(15|Ddt|zQ$z(Hb3!pR{N`3(}2=F`&C52OU3_t5%T~oSbR9B`m0(+FOu}v
z=%W9~eE)+py8<n!zn;?ht6CDmd8Qehzf%f)K|Yu&9F%70P^Pa`;1S6yfcqE8#)uY(
z{($|Do*JdBygcFr1tFZ&{|eLJ-}4JtoQh};Ea+fdY4e>bz|V9+jeJ=D#$X)~8dpJ&
zc~vLIlk?*6B+&(2Js#36S1I<25JQ5?hO_u9I5WWwakQr0s5JjkeSvV)I9#0q0?OKd
zIOTsD8zBjpkLj?&_K(^Jppp}_wTke_8NLJn|GCg#@I=FDwbeguN-hOF9`2gehxrGI
zK}7)=ufM3e_0NK&l7PoiPdUTs{s7A5Wgtv`{z%^J4SjJJ0HDtbphY#CtQP@@3y5b~
z(9Y$<SuhIR|E!!8v3!%|&7XR6U2TVc48Mn6EW4->VRE@~LZh*EkL#4Db;<!|l|KiT
zzzNidQZxYRy(q)QV+nQYO~+pl?qn6!bXJS$q4>8moyeTZTOgh;jIej>6>8Ohr1K7d
zjCFx1Lc1Mem<-wSlvH`kxJgYhTp$XnL?-<T0AYXL%zQ|s`~U=#yIPHwjkily9JVW*
z1Mf!4J~0^$AObi#=A}~;prTJ!nwiB!p>l!PEfdr8*i9}Geo?+>^SDi{N#>)PF}xem
zY1jv}_zFV60h1)hB#w0`p-P=#sVueO#&7Xz?eBPr10JsT0o~~-0IpM)ytYdLpm9R)
zi|(~F{GMIzq4y6viK}kw1jGWPcg98liV6%mXQYc{QK)x`0v_d4fHawZ^$Q?&SE&$A
zq*dM;!6&zDeEYjmhH9cG;B?M?fj2;+HS@w~Oj>mT3voh^9YAl1C0(1WIt1u!J%1dQ
z&RAl?y<pxyz8}ZpvPBdNK`6+Z$BE~gc~rL^l@AhnMadcDpmne&6GLDK<?+s$H(^Zs
zX}F?!dT}Tl+KGkG#0h+AjFb<*hEYNwsI4Jj)CVK0{1i&)DPif1q)aK3W<Mp06_2uf
z1jM^2g>y<lyrE|oCLS+J`S-%nxBY-iBtnc3fGS_;FbU|Pc|N|zy9G4K`n3XFYle|@
za=e#)wDg8(lyXJmuW?z2rXAgA+zp=|Y+Da0<`=}GumkFGOZ@<$Gkzxg7K%M|YuCSO
z>8+Uo1(J%Y)+6rY&p|k*0LJbLN3{b4kG0ai`n3Kis|yVP!2muem>MadF4^vZ5hFxA
zyCnLlPoYS04FcI6Filx8ptJw_e2cdZ1_*AG&&z225sP)9*EeTMKa^s;fT&U2&SggN
zPrw{1mL@*5ya8f2uUQ50+N>kwdK&2ZZ*OCHd#+}VDyq-6t_cJv*8seBAeQL-T?R0c
z0J^3Vm?BXzWtPAK(lN>ZXpIjpIr-+*;={%-H$d<Vh|f_}hsM%b^t?{l)bgO$d>?J}
zq9()R9EBveXk=3rV1qT=V3Ro`lt(h{Ab5j-vjwjjFSLoAzX<{}!4o(YQY04IZ~H-{
zQnPWX*^Y?4dipm600Bp)8NKIb1?-XjOB9*bfdp8CeYHk|{J);A)tMRPj2ca<-J)^X
zB)az60m0PbCEG92s6I#-B<Faj7`|I|c*JB--F3wmVBqEpzlB6H1|`=<UXMnk<49((
zT5MH@Fp62ltZrl>i-=n2nx>m{KHq9(T<m4q5ju@A0XENn26IYkR4QcB)N|hq+37#n
zT^8ATZbYI9|G=o@31^-!j6kq<8Am=g`}{)h{NJdE*%ysJ#k%w$Km`C&?Gdov=j~)Q
z-3I`h<90m>6V(TZjyu!tqH!$fW7FryJIWlx2j6Y4*lMg>7j3W8#!g``fP-M5MabH}
z214MPlz!R{*-Z8%?<I>;oA%3rD&s9+9@}YOKa{ir33b{L7}fT@qNL9f2Kqx^-yAQ*
z&R-JT1Etq&GyxnNfCW2fz&8}XpTFdsW3kpLI~1<m+wI!vee=G3CqdeMSs<6+=f3Id
ze~1Njg%#Ij8;0M+f=g$?!yyUzaSEVe4}pXHSQTq>&WVsL;RrAR1~QT~k|0xEfNcRJ
z*sZmVl1T3}Fzvf$VhY;XXH+VI%>oR3!awq=7E_FxL`nBn`x;v0p~dpD&ohzl`>c{G
z!@3owA0YdMF?gY^68M!3X?|t9t<_#VEM7|`u#ch00fzx-4<Ev>DDp)2sybF(mp!)W
zo`4ub6NBow%uPpzN>aNj3>=NtkVJAcX3KEZxYEgQLCcO0ABsiN>isY4#F3;fJR(M>
zP9lDrVYuD_YP$eTvAf2_x5F!MxXOop?pF6v>&>^}hfxKx9W`Ksllw9`Y7MM>MJ`)i
zj>WhDbf00jro`BUnM01w6&Ue5^ppId^QA~Nz>%Y#%w1rGB4zU7@VY$#sJWR47eB67
zNWB3Zd<fcSRSf=`C$hs75~z%c;tMweTG)kPZW_WywsD>u5{U4%*#w(tkbLLAd5OUP
z@DepaytHVhYz<X_wY^EcVRfcBmb^oH#~<Lq(A|1mWktPOKTQ7B(1NEYm2ak4643Cz
zK^{mwNeH&wdpR>eL-*YgIJ|NNmr`o_!&0(?by0C!ny*BAIr+&qJmd)#Wf><D*}}yB
zqjAaiGPoLeC6`osofAw|BHy@`!M$h7Z0Cf?+Ad%{mir+eUg>=TKR*qi$($<KdWhFv
zs73`jEM_t1^Bs3<qjbK@ihdms<x}}&mvcGS#f^OXCvam2h>Wz(`l>sh{Xe{aM!!xd
zE4BJu`^t$BEz@`NZrTSC`XdJ*!-XgD`P8S%AH(+~r$j4^Tb3rmWbp)ui)-0y_4r-n
z`jx+ZL`EREM`wz=;8-=yv<VY&Kgp|7$K*Z<(lj!x|8}Xj^7Yp?y~*JNo+@f8(<Zo;
z7dlD4@iGSJNPx>cj<*LrQZY^QbJ-NnS&Px>QsbqrzBqwXe!g(%Qkh^{ddJv<ZUMej
z?~ts&OL=z?O#Ofhz?*7dSR3D{b_ZgV$VSD>xZ6(+yc%1|kexX6`P@Ta`&iIbbV^F+
zpNvaGJ(uk;fYYhf%>d{joun9x(_4{W@wDRbhnYR*(ZIL0c&S^P!OA)@VS-Fjb+mtM
zfICGX@DKGeGwmk(_U(KqRF~Ws3}b+wQ3}7T6|LAqwBDL30*=oa<;dpfeP$QS+@0Bw
zy)I^z+-T~7r%|Sn{A9Y7bkK<g+85pgh=lscTCl<enh!rS4lBuRurKe6I#j0PcbBSm
z+@xrC3S)-UgC$0BLyh3~wcehtzVNwipo{6X*XGa-?SiO<$n}k3xDTX89us$sTZ!HC
zHf?U_XOa7;rATiF%G1wn1=zTg$U5IScCw~BP2@C0X&y){zZKOukf4!lkDn4h2js*O
zBnN#;rE+YG>NAwD>?)KMWTW?N3&o-Zj!-u~8YH-@07`l~-&~*DOS$%`)-bc4J*gpd
zdjdc~$npC4T~f`GWrfFEI3t7_+!@UopYR#o*6#vIzIG5dO91Pb+w;)mEFDIt*-&#-
z_tdX7pe2X2>A#+vpO$tN0Rov^%v?zI8-H}yjf_MCWZzmmr!L!lwoH$8O)MO);lj{8
zQmEzWatW{5L10`TI5(lN64~RvnwC^f;8|dIFzuqyFSwF_bu%Q%)n;3Y*;;p=k)62>
zey#}sdlNAWCCF-&N~3Q`%V)~QdiJ8WP03fbNi#eR4JOjjF7P4prK9Dk_onoRhWLpX
z$Jz4RP5}3_tL=4y1NcjrWh{o!8jg}$Ld<UqV|7dV(5DN<-z*5Y)z}_!_?JzkS!ih4
zAJX|#XE=iTH@C8JJAgBgkE);SS4o(b$+Wj<!Ix_X*%9q<qvq#b*8_9P{6G7;qlLn|
z9nFR(#B;GN0M3<h@)nDY+Z2ICiO<WCfU_-xpD$F}b<XG60-=7Wr=N)Zebm)uw(mGY
zv{<qsu~*IX&O3(@EA^{mG~zECkmjgTIg`kC;OAu2q><UUtH+%ajN!4bW!$7WI$xFv
z!tfAp$2s$q6myb_ISa2IpUB#EQZ_+>aS-#X4`YUyG>7VLh<tv-UcjMAp0-E~5TLBL
zt&I!SAf2I{qeZze>new(l6;rGV)(h^%dV_GJ)LvWX;J~Lb5Szc+E7?wsvY+A!9{D%
zwyZ&p5r3>P83!A#`2MWw###SLCONem0^M|9uPQLrx)QL$FX05?u2&sX@x8+%VTrsG
zsbJo#xkZAxUA9sXQbJ)(Qixt__bhh3@Em^qlL3Yu0@`q&G*mk$y{+K3zDK(zqr%=l
z<3xlj=_>h|tiWx#cV%MRY7x#TRO@5z#i@qX%BOqMNC0~X6{9jsHkjb-0kCaJCQyS3
z^QWJs*m{3yqIuv$1t_d0=}i3EU;Qi6=`Ab8bT|xVTZ^RIJhP-WTP>syvfM!64)DM{
zC*Ak7TZ%ml_-*48e0SP*7@uz(!iPio$kX|6>-Wbo-r>t9$eXCj8OF%w)$SE1E?Khg
zp@l2GeJ@((W)y2c#QA*G@SpDjK>Zp|>%3tZc+C6N_&&CP*MZr=Qip<zlB-5?^lnod
zRyy}EUXpGc3rz}DZ;iRLZx?GW4z6DIq_WtWvo|)Kk^j<S**0gqQwx79fH3{`J7(FT
z#w;ll!#KOoDn@_mn<Hi?7mB0%Xb@x9+6ih4*nq;Vh-0`Cc&U>nuffd*AQb@7rE<Fa
zgAaU^m@f4R%=%@!1diXH!dn~_DG40Zloe`XQt!%<?ZH0unWRW1^0`52cJyVhjA-yv
zA}7(%9+#=CRUem07GWZdLxvbuDI6~uU1LDcP*!(sVU7tZYlcVeMM9;`I8j{#sP0b_
zrz_3nveB+pap1cS%dLyxIE)Cxb<(KqSrJm<Z;?;?k11>_rWk2+(w&!}YQ?9+$HrHL
zV74PfjWPM;xOUNJ)U>1TwNVF_LN7XFwVrF9W$2|p3D`>tVQ5W<pH|XPjq7vFR6p;w
z6D)my*w3whTciKl_yZmi27Y=hSJnd|yIj<|_qWoS5qp4nM4QYJDm_a~#YS2jN+P%6
zT9fed4l{uA+|~i*<OTBaSFQO6sHZX3*By<;2i5RpLfuo`@nve)ZD%D9n~rBdqcgiR
zW5(0gp1T;;va+Z|8M3jcp!SCeD8h;~v}&=CVkWTjEbCc2Afv@FfQSA}3wia%Vx6uH
z@vKx3*miy9bezW7kh{Qoo6Fn!yQ@Bi=+R<n*mPB*8eUYfJ6HxbBotAM(oy(zWM%TA
zIAsI6qz2eRyCuhyEFDa4V5Mi%6e4drS`i9m5?8Lq#%>MU-{n~wvg`$4X!F@MPVGzO
z98OgbV|Z7Xxvt4o=I67oSumS05HnDL8R>K+;Z>K<z>}kY+u_y)4r8wwc2}ix<aAU6
zB@-P76Ot+2OqzStd`d^>1uC0rhorHQx+d%9<M?hnmCY?v<S$d#qR8k|7vBX18gWK|
zIBUq7-P^}c6h+nL)K4h-0S#?klK3u6NnAJckZc&eCdIT>tM&3_@4Azq-AS~H-phtK
zNz0Ev0kv#J&{*HbqCvZwHx1A5she2i6$A_&>mhL&q9Hzh>u9WL?{C8h>sH*=NfaHe
z?X7fQMxZC6o%eEbkkDA#meU|<+~u-fKG}66guZ7slyM{vPUzcJ1XOki1iHBCJx{O9
zG#rPMD#o@h57KztY;VQ5gzavPr_Ps3_^dmCSpU5*1ULd~1(}_7x20`~`54Gm$gm65
zdzjH^nQ7nM?$OpkvWP1V-M#GFxNG3&JyEkB4_8*qQV*V<*cv(^RM<8o$rLK0i8eAG
ziqcsu;g2>YOk+qj>f=#?5+0(&)ldl)Vl&n$O<XN7bLoJ}$#LJE+L!l&wr@OW!QTjh
z$x2TbTw{}={Q_bS-}n7R_)fB1J0s$PP0DI5i327~riC|ijZbJb-mKpIu7=gNzM5Q(
z`1$@m49h;?hi*$xlCKA~KgNNb&7O!DJj7l}P6`%HTy?M3BowI@VgG<wk{KJjIDzh=
zE_a>vC!pTwP#Iz@d^t;6AoelN4cF9+cEjZd0Fc0>4n9w}MStnOHccRNha=CP=H+&G
zoT5{FpVcOF3{o`JV!K*Zye2fLJH0N^@2e9?EpwGw*xL_k{dG_nn{WcevX>89W+R!@
zYZH$~ktAZ^dblfcVFo3DlDt#PqXUs`=q>YRMQ_G%esI%sQpsk$LA9Onf<>cJb&LGQ
zjfq|14)pg<1wjmhObbM=zHc}Vio3K@EK2?IfDj{0K%}Crk<5@pqwd<L&y|LUZs@4T
zGSW7}fK$3kKJriywf=5&V(#u1U)SNqi~VY2K`scs^~{j9eE!^DK%Zn}4YH_59^N9T
zhd#dnTO~kfkj@Pvqqmy~@V;}L2;9w@I$^apEg!%x#J|<4?XfjwpQSi=s9DntjKWOi
zmoQ9Xqz%Eo(58ownZ%~lcVobIbhF71`m#vZyt3s&P6}?<F5K%%tpg^yg2|2Dd$3MB
zvP?%|y#{I9E44N9!JT-2>X?^XF?PF#qV0hWO&^BDPmUl;#R|dJNR=Riuk=Zi%x{L<
zB`%?w`fLe%<CtnjEyYTx5q#xN)}mKn!yC>X$L*Ya(~H)I<iY6(mUAduSul-L4W9jb
z>$M{Al?N8pIR<V#(jip>5L9{dSnWy!TwdxBoyP8N96|sQ{)R{LV!%O9)>^%Y)N&_v
zh%%gpOM*qmuv1aS*O94-j&RrqlEUw|e~{+>*LL{HY_N}11bH5xtIv^SeO3^4pZfLB
zFmJAc*{z~5@z6)^QC))2nakwhi>F)Ui5$>bofQ;=*pbkbWF|j{<TNG*S)G0dzDSz-
zbe_W!9=tiObb7nb_-Tt_{@YP2>Ok`WxgO$h4GY0nS%L0@3@lV~*Dj+aw9YL=ENT>1
zE%e~~F_%4}H|$G>Qi{OWO=wHm`4{6Ii&!xroTaQiGGEjrxQ~8t1mWfc1)s5z^PZ&F
zEq|e3+dc(Ir)!%HB#NH>=m1Muqj|IQ-H9SGo;!`{LU$8O=}+z{8(AbN`J<mFPoSeb
z)?Q_leFBJ>S#0Atp6)<KhY^|;b=X~Mr)nhq&hf{$Iu`W);e(5^j4O_$XhbV?Q7T>M
zerg#A?&JoD#Bh2Lswj=^PgkS##SF+r>GTL+d7^0Wl%GrvF$yvCV#^pHCzV*T$<iSx
zv`X5F@${-rrF>)iggT994|eXvaip<F<)v|n=$y$6@(aNVN%yC$7!C$DGDL>}O>Op-
zf;s2?Si&xFzQNmcTc+W&=#%Ia7XXdm46{Rz6p1Cq<7DiP6gZk#m@a8q3-!|<%Qbbo
zKqXH-ZVk+8d#DtaA-S+TTIQ$~S*COD&JhH5lYwWa$ue}(<*XTUVc}Q5Up71Rj4*rc
z=j_ecP8}2qKxF=9X#@jR>MPS}tu^HXl_$H(%QAMZCkOz>(B45g6kIxNaW}eGn64{1
z{ZY24p;_wxYVSRuqFTN^VQEppL>2`J5=8_=lH^til9PaB5tSr4v}9UDP?Cs<fCNDV
zM3Ur;WF$5@HBF|;Ifwr0fO7rMn)S`hn^`mO&Ahu@cj@Umb?WTe;kQHWBlH7;lZpUN
zQp{z)u-!2y8}aoX)5x)<@P|8X+%(^~9P%PdxC`^*%9?uB%`=u?FQzEJ&LCS+v-t4L
z;3kGSjsClK#(;nFS!3r7vKhIFk}s}ZFSPFc{OV29;L=c0AoEUJ(e}|VtJRUhU%rw)
zPva$to9%_Znrl>A59ZLA-nKm<U%6=1!pmp|f~~jn&U|D~um46f)<OG{kXnB3mLu)y
zw$a2>b^;c4gZS=bOCODGy=zYdv5Gz!z2TBZk09|v-~N8AN6z|(F=1O{HnZ}HYkUT5
zu5yn}ZTBO>HNGvod`JxkmMc5zV-hII>z2PC!sJgt9)0X|LP=T4qIwb4*5~n2&f_OH
zgYDAT@Uk)_%uj_rmW_#I3YS%L6(e}qR~5|oygm=@NG-%WB`DAHJk66WeBm?vl)Uc8
z$DQYGs|c!JVzgHuiv1jCJwt7@iQhtcgZZnCF-<zZT+&l#xk_{IAORMOTOLjFI#CRg
z^hkVCH&tnvlME9SyS{n_mcJOWUJQbxW4jX-q-IvcD(o+1ym@)<1%ZY@V?m(qc*`i|
z9TYH+CBX0Tuj_Tp*U6}CyzFX2<_r#1zoD0X^zAz_3q@OF)me5z2AJHvfG&C{F@pTT
zP`+er9OdxAybUy9s65HH(Pu_!pn^PYsCuR8d(!8W8kSEW^!^g@whB6FfV9T9wuV|(
zpdtl{g1u@4;f%Wbw(?QO$pT%3r<3A6ky#GO^!1;^6WVy|*+9rUHbm%MFIWA$Yah_x
z>&@PMNxlf9DxqlaG&2b&yV`AB0P)rE#WPM8-16?ya(|z3Kvsb+L{a&i!SKIR2K7*G
zXUmO=@bByZsWiwVj1)@1{!Xnh#ej8>Z3HtNp2$5+1Kqm}k_)eANC}z#$|JmW1v!`y
zjPyI@zjHV?AYa6qCdc&lpCpDad>JIH6y51${*NtmJY8x7<b$p)(Y=)4e-Qjq3eG6Y
zcpLu(acc<aVZ){OyuVxQ96NpOpu>)**-FIAP8N3O7h^o~KVEvziUlOyX$#cXTrP``
z%i|7b#>&V%EqkAKe|-CTn0e7V@wP`-40yKY{3R={kiDu^RTMitHIQv2_C`J2iTTFk
zs;c4-bP0>iG!YW@X~hi0jL6JNpPC2S#gcpr`L~NteB%XdBFm_}uSgTx6>s);o2!F!
zHk`*EtN-1vO4$++I!q_%o9N&Bd;%1(6oMqy-(8*H1tdxDIbLP1_RofAy&~Fqx;bgU
zXWC`Iw$^2q_e(aDJmt={59PC3JU$+L&o(t<1hmq6r_EElEyvZbUGS#j{JrJO5Vy2)
zx@;r}aEcUmf9kc<eHDFb)@J=2Wv8wg8)dvF0Y2;E{0x`g^Jx{kiAF~)=Xr;r-mt+~
zl5x+g^f0N1jXgGb=<bb>ma<P`b~T0O_9A=vjG9_bl0Tt#Q^+`ws-#Ieqwh3PEf*wi
zTu;S!UT_WTQ1o5=Dd@o9g4wGu+Dnv=a79<<ow#=AwM6;dp-^OtuJeSApHxR?Vrr3L
z7`KPN^CCD1E3$`uFXLf==w^(5dHM}JfoyZCJJ^NJX=IE|Ee~V1BIIC3ba8jue4!ZE
zqbGnEd{`nY9ls93?&Oj1gkpE*Uj?X@iKc!FblND6%BQASa>F9EK`Ll>FE-UNM$oN>
z(Mr7Eu&Fa{H9m?uYAh_W%2vXOT;laFbKba5)Fmbo7CG~qnMSQ2^RRt;7!p3SvD-z@
z`0Tc`rB9Z&8?N!i^Y%6eC@%ZHX)DZd;&ZMf^<ysy4sy2yO^R4!JeFqfA`sdxD7z)J
zZ%pBw9d<e;n{Hs?vu|8q0m@mG;nDR6>$Ywm8glebP+t_(B!{%X_h?){SLv(sCF@c%
zf;15#seL(v<FTTIA~u?sX+P!DB#Xq)cRDFipS$O?gWEqB^nOz)(pwjLTTVtaI4+YC
zOK_hTPad`4dh2l>t?<Y5=lnW_vllYaT`?YO=Wy4k0;srP3~!GCc~4n*iO_fUlIUN^
z3=x=0TyR~L#oexQ-J7BueC{m?PO2;v=bDT?!`PvC8{NeA7Na!vu`T1>F&-lf6vW8W
ztcyQcV{XSvcc%taf+BIt;+az8QzaGS-IzV)Ve1apLpHFoU(>r`xoW^<);)}&;Q+mm
z$~GeUcLoP)KOC<KxTNDoB6|f!3mdKRpZJFUMRVN}<y$x&QHsE<F8ERj*Mw-?CP^}Z
zdF<z4&*Y9Tb8HG*?!FvtDP1hN5@Xr>WE3H`{n5>bB0wsD5|8BGF+45kt!z9vRpTRh
zt$rC))GoNz-4#Grj?cA-KFD(~5zKm;h054geSZgkpYg_gx;P=fX`g>tQb#Ra>&oXM
z%V$4ySNiE_mGj)a1hb44!bPUavbPQJuSJyS6Ox&Fv_GLL+xz)8713{nr8L0Uj#uXy
z*s%qV)^HUj1VLJoJQ%3l>0zZPkx+?qRD8}0<9X)?Z1cFp_lsBfBSn|j?V|Y6)>GPY
zcIMmfO*_Nl$Qc&tkjZKnnu@4+&2|T=x#;p&J{Y*K+e|j)85$cV1_<amPvHWRlGXO*
zQgD(W_swh8<GF6&(vRM`?C_}lNtoN(l=M0}b@IJw`|D>NCMb@B{)n-@_+uR^qPt%?
zK16a`Y;#cR?et}Fbtb8dZk|yR$Br>R@G$0Z|Ap~ey)sy%>c$3k0Y_re_r2PGs@<|+
zyP{j))e+@8?SQ&19kaXO8?h>o{G7VM!*s0-@E7t!Q)K(Tbj&5|a@rz&C--!r9CJHC
z$6fsWzSK8g<7*D24<dw1Z0|CMJM|K~ko}5AgMs_Pxy!wVPAp2Bks<G>_$}`jSw>CE
zshlR+41#A};PBXaf)56c>*%?!xPgHP7M!Rd%GazT-!ZBXm(Yo0P)6ZvnDs}jFXZhs
z3+THekN@nYdD*S)rVgnri2UBrQ!@pua!e2nR8d`}OZ7$69PK?%Hl49yFEt-z2KcGK
zfvJGIu=G;ReD@f2Q{p!(_HS5T%TOXrZ*^pb=DB|hnaV#mV77JPmXuNye~F>jcI!p)
zr3LqvlDbTU^Pa4C&^bN>{C9A8DXMd$;)7`6JCo`6Y+$=rztVPC9SajB>OoT{XAs<%
zd)P4N{PrZ#)X8(yPGm&RxpZNo8zSWgzuiPe@R>*33<O{PmD%u9T2yeT+9&$?)4aQZ
z(IQWNc^iMaU;m)P5eU&sP+y_vCjBh8Qp2JzZ)a~3GhZ;R5#|iXBo0#VKHYnDu5hj-
z%kMt)UOmquY5KnR>~92%W9-`1Irlu>-7qdj)_W(<70ngqL4zcQJVg;DGu!OXFKyOO
z>e<eRc4wa3crBJ^QB?hv;h+nmBx`IK^3?Q;Ak6mA>|MoZQM<0`nbd+lA#?xDi5;s~
z?yu!m`H0XJ$u!6}b)W75@ZZSt8Zm1>d6nyUqB>pK#zZgc<(n>Djh)@vRylm#mzK~V
zx4=yvhBcqZHYC%KM5cxzUGTI$ro3D9S-c6Ux3FBt5pA}cdT61XneLRyM+_%dZ!Y>C
zJM*bTH)h<{LKEpDeZCVE9GTfGFEf_z4M((d?(e4zuS_<js3)$2jTTxO(WenkrQr3L
zuvd-e$&wkNaam;ehT?rzwSm#M`)P{t*cp!_Q7>wyGDiq4&B_U-q1l`x6Qo=&)htU+
zaVQUa;`_V$o7tB;d$ER4(R@-TRsArNr3)OHijVc2cdw7-RNoLD!<F?L2B<H1<X{E_
z?P~q0&}P+L&u8~=qTUkpBFOV8sWv#C^yVlY<3_AXYTx%H5(?VSw!bSt;M|1wD9=jA
zY@}xIEu;<&+g{5d_`;ZBpQ>%4xxJA(G&Rqz_y>K_e+K7ZZ+xa<%>1XM29Ezu9+>TS
zF48~#1>`|>Auk>Mt4RN<Ah~xFfIOLbntGPML7p;D`ec3Fq;h{A1Gs^{9s+p<2&Txt
zLEaq*<eAjoyZ(3K2oqF)G&UIg>~EAp2ufXq{%00?2-Skq|5We5bDR_^%t9zcZ!ceB
z(W<!P7$bH6im7xu`xZt8)&H@PboC2UmVvD1w?n&GPiG41+5jM?;rK5AaqQC(re8tu
zyDVp0lzwu*?<+8^y(p_f;(Y+%Dl{&}<pKcr@?Po}hPO&EBOjtB<E7wVmV-84_qIZ{
zfEzJFSVyI|bQgZ??=7f+fK7DxpFKTLI`>5Yu_?sy@sFyXAi6>qhr`)?{w8ZR*#Ua9
zUyX^EBEs#aU_7=`)%&|<EPve28jq}1{eC+60kC!4W&pb3#Fi&DQu`q&^prze<N9en
z-QU|L8whqBuE#&$QpPE=-N@zRI-%PAO>HNCMG3vp<E7^hPTGDd&*zSZuoIWhc^fUW
z&NRD$smTSG9ve<6h)tx2Hc&>E`B^Pt7*|A>Os(TWVW^+x!TR6a15CEIXx#MCsmIYp
z>9X)=V7MFx9kZT9@;l)=^9<(1u@h&~O#2P1Ir}a3f7|b@r_HyX1<kPp()By7SO{5V
z&izpM;DSrl)OY)gYtqzfbsSeOe3qHYn)-r4c?!QqX?JJBg%-Or#)I4CA-WKfD4%T4
zU{(ey=^lZ~v$Rt7>5dea5lQ35Nb$1C{UV*|xeYAnp&0N}>746q;Hd;DO(kWv3#`hC
zmL+5@h0h0zoq%E;!jN>@fsz`t^R$a&MjL2OK4%;4!fz%po~O;zZ5A7*?THJ6Mzx@w
z<~NKT*L~jee4tkky&({QQ`2uPcV<WAx*hT?lThMBI22{N90|dl*#5Mg&36uSnbc;h
zE#gcYqPNA6pn1v(dgXM>D78Q_`xFJC^iG?cnFXwxC)FWT$A|p0TPfA5U1@qQndW-F
za9$Xd3Tl%L(Mx)3s_t$>u7%l0@S?BDT9F7r$Du_-gAlv%z?89%2E7swZD5sUR8m#H
zQl^_0{p?VozuAW=6zsXK^V7cf$x7&*i-H?Mk8rxn>_G(A6(9C>uqlb!a@n}h*pH1M
z?2qaT-$L?!7tsj78K>}4@sd%EW#Qfj9sBOjr$6Ie^9m}DBXQ^=9K~(hwWsD`;>-~k
zJ-wsaLMEtM&3f;L^>9UsVhOh7r&X?g!5lL}PY1Lu<t$V6D<8qX5RkN@<hBs-6R@dk
z{W*lW1V^B>=5OurRu??Ia;@pyr0&wlXiM(1q6(T8(Vke9O6%}YZG2UN0qdz=F0qvd
z4P6!s&o>KziA)&KeECOQ;CBeIUjk!kygC<OWpORw;AP|e*@Z^sUQOD$1QBXe67)1k
zNyh26tZni2@@4kyg3;i~NDj<3y=&D++K~xTjc^*T8as42FubiHshVc&G(yBTnSaSX
z1UNIl+fGTOqi{|ZA#xKgWUrxJsD;9xcO44G>|Z?hf~!=^{@XSW*V+B${sn@|2rP20
z@4GU}qytpbAs+!fpyx&uyQJGc(}w33Gt!Ach-@ggUuaN|k!F*i=YbX%$8&OZrr$B)
zJhoLK()0-<&vkV2a~N0Yc8#%t+vp9~#d_s^n|haX2~xt>p;E-(=!M7)K>&g%HU3m3
zh)K)Rvva~}eUk~2KA0&+7#`u-J*=L+32D%lQO97v)oeqjduO0q^yOK<js$}H*O6h{
zqW3c`G%G=HfE23iT%)OSdi=p;{!_v3%9D?Lb|1K;!a(Dn@)uhx96#fhcoBhIpsOQ~
z8+YMRkl-FYP+gsfUo|>bLXRsLVno~|=r6#itxB8Tl8_m($$I}bXkfv9p683%FF17`
zwwgRF9#yV@yjkl>8T-GD$0HER(7V3;?eE&r*q6Y}a1J+zVE?u#)eOMzeK4l}FK#m-
zMDL0yR@C1?k2lcyo6Nct{l6bX3VM7Oz7$r9gDS87`~eFEwU@z~C4WDN88WAYh-eAm
zI{Yq?1(Bu;;N`0s(w+Z)knBfrJL$a0>pu=OG>!6gnEDw?PJdOzpGLm_mM{aCx&5-R
zHtCVvfh98q!kemO-qRsp$zP0)d0})l$F4n)`+Xe4X}O1WzOTJgZ%9}~3G)&LYs*N|
z$KcnTr;BJAVGKnVkDd%(X3qkurJC$5U*U+4=dmLKfWN$i^pXhA_|dC^za%{eLdxz=
zK=6l9&<$|tVkM8-`G2TFjC{kY%%*de(Qy?~S$b(_(jeLUI;L>^o4L8y!)q6-@82CN
zO7{t%;Su6PEd$_FSksrh^yaAZ^gSv-RHXA4j3`mra-(0e3_{SP#m=hmI-fqsr$)ic
zeCFMs4DlqJI`;0w+NBo@Lf4h+@uTT>`(nE>yCabcX5HF_4JV_Tg{FE~8hFzl)0-86
zmEM8cM4(6j;Er$lWl?geFFo_yhN>Y6fn@BY{;6EkydMH`JDpvN`qw~v3G3+-VWLa3
z<?HOZGsXLbWxMM;clV;5b}Fw}iFN~h-uL1F{Z>PW1|J9N-13*YB45?T0)P+-@3t&@
zR=mZp6@vXra9;@7Jd>X1Slf9X<6(8&rYk{n!|jJh+2qN>kFuw}90VVN0LP4EYBn7*
zTqj8nA}@N72^3bnqS~9=5O-#zfGU2Gam1}MG?G8lWDnWR_R0;9eb|rA6di%E#$N}^
z$s~OXmeU+lMy-;lkFIIpZ2#ia?tE_Ar!E;REU>(Vu{2#R-F@`Q2aC3Bt&)@H+L={8
z5%_{u<2~keYZvtRwXbU&^cz=(TBF=mE?bM6(VccEdX~{0ykk|=5C|ax0=c2C6fS$M
zE_6nN{wbOqi#$DD03+yOVlX<7_3FJK(VV628Ar4&4~FBwM!<)G=$@}~ZJ6)RH7kP2
zOg8TskMt~6ie28WmiF5ZpyRQ`7!jSn|C02zr%(9@y`nrbbf7P2*(`rq+_Hz2&UI-@
zdY9T`kYb0{@e}Lk7vNDi(oM@MrBgA_w`~<ZfZo04_>vIdc?XX*8?NwH8B6;1jlrer
zP8@-<g?@;2-OJ(5uV<ql{(NzQXu!JLts8xNI32faBXKIIS%I>ooO$bP$aTZTQorbB
z5Vd*qXf;lk5X6?|x-p1;2k;l;G70-d?7(M2=9IFt>1$yo`9C!>JcH?kT}`g@?QgE6
z7tuMNl@#sHQ}PMzt5$uw5=gBaSR-})#u?>%1!4^#R1Q1)+E*PMn$|gMH{}f)8Z6gk
zDVk(IGb;5c+kX6jQ}SvM!E=J^R4S}7Xp`_KlmSVGG0yW>ufiBIs))q*e%$j(`!<am
zLfcHWR$PNfjRF*HjvBtHK-s(dwCQ8G3&}4w5E*IU)e20};`!<_LQam9WnJ9!jR8Hm
zJCc@mE8R3K%2X3Zm3IL1ypsUlsNj0d%d(oF?J3Qh=~*_3m;IFaLA2;2h;yiv`O`m&
zIw#~3M?sE!!*auX{h2-XbqI^qkKygSRj1a5kyf{j+3fX*!HeDyE5?=sB|Zt;O$ULC
zp*o&LW3=VQyg~%m5Bh0Fm<qw}+9Vz2C)=txlAQK=g%9Sm30KD{khTmNCYl>x>a($x
zO5!??Y!HbS{QVMU_JV6Upixl*1T16qg)L%?sCX?Olo@($txN>$&k-q3(`P77u5snB
z{eo|W-owAZYTZZTB4E;y@9y;qY}G>9xt-NYNk%iTW=dVV{t-e4W*CJ601_u{EL6Xn
z1e$@vm~eMU6I8A3aFs>z4qM4-+a)Jx=$BL?S!puSmgr}Z#<2{b?~Dk3DQ==c#zQH?
z7t9yhNhC5oHs?J1m1G?GCH4>0g1-I$N4I-m*~w}ZK5|<~?Y&wHvuc0S#EJaD7EJJ%
zz7^Nh!lHOlz-T*rdySg2><)E6(%Dbse+u*KAqAq0=}0vQ@<6`377WhEDCg<B?Vq@p
z#B`_*!2b!EV~;P8{(E!AfSUea?=>cA=UW?+Z-Krmb+nmpufpy}b~AnF*Fupo#TdS0
zzW#W_RD$B2<7(kM91TTDK>hXGO0qY166_PM{Z5VqUpTY1oF@g^uZV8B^~OsFjh$ip
z7hmr(68v2B_1aq=<b$Wr1iL|<pOg`>lrm(E_3Z_B$ttu(Z_rbfSUx1b^>D2AJe~HC
zvzKN!FqceSA*7!k*r*+P`q$=72vuFKk2PSZXz&yM^~(+LtB~>>R(<rBO~fPl@BYRJ
zGUCo5Vkp~RtqiIxqutx7+dfB?YlgOrSIE;gn@aFw_Mblbf(ZcxY_96cUWY9af8=-g
z<v<y=)E7HhT+PuPH@>BbZUa76F?3+fo9y{s)W(8aHpbULsyUn|@rmC=q-F8XB_EoO
z?^AK3CaVYG_z~#jithfmk~@UDHq9~%#r!6nf#wVO4f-kRXw$J4@zRxPKf02g8Yfn|
zL90cdChhF+%A97o1Yd|Zt+A89ed=>=Z$5K@b9b@5W!`xarHOG~=uD(bGf3B$)s{Q}
z4Yvug1K1ZMoTy)q1a&ec`lGMqqI_*FS3&UH*)o@9Tx7rP6nm~g=eLid<`bMcX*^Hb
zFku7M0)-A{B|nYqEwjTyrP9Nsu2f2it=q+K2u@e!iC=6>+~_Q_@8^7EXK!gQ;d!E2
zKdr^KAUCDFlF?`S(Uo+LSAuUfFOc|3JZ!5X9JBhNYnOP}POC@}9V+}?@u6vfv~*lb
z;}gY)dGo@p6;nJ<-#ZC+iU6~#3guSFon>x*pafBy^w-p@AZUD!cJYh(rT%l`xG^v7
z95Y1kkCcOeS-JF(pVcd}c87*0*QEoPj09l-EBoQ_lp0yq@;hzOr!L!QE;Mk7u54PB
z?e1ofHE%AaYFd>}!}2e?KS~#J7+=zwVw9NMAh{@DII>75B_Aoa7eyrQKH>dSK+oN9
zWjx|fx6#u|VCTWaQX~&X-Wn;5KPz&vESwIPUT|4oIt5OYl!qfge7$F)fBvcv&>~{6
zyRXg*Pp{ftx9wbK=X<7%34@10I!2isG#9#h+EM))Ozv6F6ThxG4H&)<uLmw%7y%+;
zjWOZ|CcwD5;#&`I&aP2nT!yE7EQ<_#YXX99t<CCei(uvA<se?aDuNf)q4InpLDT4Z
zk(OL~?<FRFDiy`)*dYM}+ULWj(oHb}RxQVa1{^SVquujc$`k4I@~wv_`W=hbn|;D`
zZ?^4fusS?1QjG7Gi+^j#NVg+H16=-exjZzBV)Sl%tyOoD3Ztx_y{a_1vv-Tm;r=c%
zGCd2(d>gcCPNVTC0pN4b@=t<UT;e+t_E<R=O#94;d4GD}(g3?v>2P$ddB^vyL255l
zqNEJ)=gE)>AHMbGI;4PZ-upaO`d@k9KmZ?E0mV@-Hh+`E-;XJ;T`1!-pYV4OnEg@N
z>{8gHkT>WaRzftgyeDAb7z{aMn*{!s*J>vXJc<S*Xw196jTa>x666M+1^4wE*6OS<
zIafh@0`d<QeT}tok_}x|jZr+cP~0HtF^ouT$=(Yb@aanCap=pYbKf6-y?^{OYUhU}
zYd`h~W)|k}s=F|gK1i-K<J}qE7~FNJ+F;-3Y8+oRRWrD>$&1Vpwz&UVx#KBG9en6(
z(E^>YJBr$G^Ll%dN?w{yl;bRkg?3K%#XU%IAQ?zoFH|yAaDn5UHkDlAT^g)ODBmB|
z#2Bp6`rHzDuMS<aWKJD>i-J3m^&u?otIacNiSji{+tI8iJIz}&gxG$dyD!}3q6yIF
z3b^wsz;%`56_9)?aT^BVGdjHHZ?uGJSimVqe*l5*|0@JCv#-)m@}12*oAcw!6W+Ka
zRP4u_Qwu&j12<z0FqhgBFNMDfpt5^hyYCaz`n;EgL`x=M9oB(eYrFcmr%qZkeuXuw
z1G}5qBwc@)fL-Q3Il{qnVOcePbngA~b&e08HTX^Zr6l<JP#<}1Dyj%aBX})x<wJS~
ze}Ntv!_2X$nFN)(j9_rS_1?s|p621|#)4`d)HMhkga#YJkM-I4pSBkaQSAqO5B&jw
zANwJ_3L%LRNPqqL^Bx9lU_z_hJC(nKq)?2A_4{3>z<&n%!r35(I$gqa>2Ft&6nu}!
z{@>rEhc;L+<&O=%YK2FT6jt&!;FSPNxS?oh7Z+7YHC2qW>Kn;Y-5HMGme&T9C1`=V
z0Mvh37huMs$N&lm65x<S`kErk&WGi7sKN_%Dku}ql3U*-Z29&<FB25zk|74ZC{v-#
zoqut0A%Fbwn$MRb7ZFl5guuqi#YHk5Gz<N`Bk&*4NmLDwdw+j)3Hgpbmg5yON`dr;
z2%$gK*2;!~2lw_yFnhM1X+Vx}&NFnbNPn`%??4b-5Hf#(CExg&qs#qG)vSuvNAQ%y
zW|>k|;%iWe(TG+QD_6mnF5k-(CnSb&gg`msdU#rJ<^wN*t5`C%d<KVmb>~2<GJzHP
z;p=y5@FQ0g#db$GFxo~@Yea(c*&s%{zd5IwXW%iVSHhKE(4h*9!ks5sDl?MwqDYr*
zF|oz*?g9bxuGv*K@`G@RH#2>arlC^`KgokfuQd&@LFdth=wArY^@<Dqjye_r4vvF6
zvHd$@ry`}d2X2~AoV^Vq{EH+P1*e0|t%_W_E&BzYDdRl2TGr6r2@qeBBLPtv;F_Tn
zipOZEV8U~mxV9WKLGl!PUGu0lf#}_hDue;uEJ)tsk=%e8>$*y!g&fjEaPoKuAYlwT
z$NQ=DEmP>7hX4|^bzI=kT;#h*NA#%22I~hN0rFzCJL0(g+Pmo@Wtt%Km~2gMRtmN_
z4%(voh<hL3&}ZnbjGIn=M8&(N>V2)Z#rfgTzcu1$jZ*$*VPq~y*`r-%AJktyNN(YS
zt%)jE488}9bV&gJXYV~amxmSkWx8s{@-gcTz6%APcx{Rg^H!hO({%E!U(axgX<j`L
zU{icxHQJmdmfwpbMc2i5w^-?<Ob65ZS*WdY9YV$<aNN_~Ew75jUAp9{T7J@u2f;bW
zkq_}tmh5-Mb`Q?F^`2-Du)1*FM-^fe_%L`2&dcVVOfV@=wYhn>>G2|GgQaG3FMSWw
zW5Esl7F{{DmVD+NLrkXn)eOpjABaElAm#<b^!#8Y@}pAN_L{L6j|qivp@0VFly8L=
zMWOJ6iAf0Wb3D$uUXeKM!Ac}~o$1crFKq(Y^b@Wp@!XHAB&;4c4^Op!jZ+6_YJEsK
zryOP*TEyMZ=pftE`pqZaWdhmS?Nl>8%jX>LpK#SFV7F)Zt{vy&kuUW$0&@{;E=>`9
z(jI(F?P&TXAa?MZqHMPxFkewP)PYR?;j=e^liG!yee0l!zbqFEd@7Bfv%Og$p^t`8
zj;?vhu10d3WUK@MuVv3rD&C%J#e6nym_vtm#%`2PNVHEHL=5R+-XMHbxY(#3Pw?f6
z^<K5H;$`u5tXhjj-*?G~{x_t^UyRBiJ|CEpy^&KS6~H{xkrqX@AoumqW*!4d^@Qj5
zYZCN~NGD+aXWMb+3yS6AR~uZB?e&C&k6Dp$&kVvlob+<7o>K1>L{IfH(i<K_B7fe3
zG58BT?`*hANy2sMXTfab@vDBsS|@JjX1vt%A|`$;&UEDpq09{?w=+9p!%R%9dgxAd
z1NFU$;qXL7gd1kVy)3V9B@x@cS1vjEX=}5R=)1t9cpGUqBkx4#7sv2m_yoiRGI0O%
z$<$_S0p&8at`6HeQK-2|EYE_ORg-^d$D)BENJ?%{eK=VuMOa1rak~}G-RyV`+^Nk*
zFO{d-J%F!4zc~9oF}}rzwsi2Mu)AZogY>i2>XO>7(hBPni7p?1KSmKeCXM;@HLvn2
zml)1U{<+|TAIKk+=Uu#zUyv*1c{q^1^%@>`q56*rX87+bS2H{>>J3fwQ5~$89_|TG
z!LPz<)JKH<z5^ax4tr#ej5Id~?*o1Zms3fJ<8_$L^`SR}t{`D&d{HMX(^5$ef5#`t
zmWBIUhJJS<{Cx-h>Fe}Tl>hS!2{p^<Yh1q;D$843FB8F5-b0RWU%+mw5I@6cNb92B
zwLLQ3G`+nlTQ6IhG;?7sN4LYx+k=LMVZqXVGeG0Y(K}khEPkqgp@%sUKewLtf7o!j
zWa{lhbF~M(_(F;D=N%g=r2<dq@_UoezdeJ8h_^$rNRs|}wU^BFQa+tToqyhpVC)Xu
zzgaSpT;>mri9dK=q|p$+fA`M?Fp0;bkAD1YKj>jxC5r=UL1Mg*n+#+2cWKejruMwg
zEEJ4R6xcmoSS>*Nq~$mY7(2f?q<$TsW3%jF;W+8@l{HPSkBenirTgEi{U|*i?I@*Q
zm6C256kF<BVg0n<*T5wS$2m<oJhCs~Brn?YEoau$x0_SQ<6K#+9}T+1<LvW<c9juO
zK=lG~yxPe;-6{ORXzsapb}fbTzx?7|8qrq!gThvO=~;>{!v)5L;!RojvdoLiGwI^H
zJ!COk9Z9*lL>`7YYqMI{@FSVw^zuj|oF-f2_I*kE1y~}j$7(#$V1GK!(_?!AG0_<9
zu;oOiu~m!k;LJ*Ar43)5Ga`CTdpT#+xHe|lt8$~4S*W>Tqnsm)Qk?Xj?oeEohmbWK
zSCz;IZBhgYaeO3-=R^6g6YJt*=8+be<HNOX=0mYRQ9D8!yhCL1QVYuoB^y-2ma89$
zVA(jvLw1Hy62~96db@1PmC-=C@n&CZia(CJMLgOXC97lyJKTNHDflAIJ;wL9q1|C8
zzcuk?VSS++(RAgm<K>(=O0lw)Zz=2A!^zQL!{OS1^|jl&u0Tw_31jS{*-xI!3m5R%
ze{Y=Vs0fZ~rl$=)_aZ4PnJqRnKOZ0-P69;Yo+{j5QMFwcl8(3E%ZE?V5bvo)uH*;|
z7LR>1XZhq&k5CjIjmv7e=XfOxhzxz2rxmw9`-CebPjylc(GFevgM_0#XK8t-f@^;y
zmwyA-u?$0&_LOIRph<~~;X$AUti#sXsi7HldOVmn1#osVQggV$4CG1=M=TJ)4IEvG
z8O(gHlf?+bw`B!4&uf2u2!4NmZ>U>fH|>XP0%!@$W84(MyR|f9S?G|Zz`y;ozBbIa
zT!WC9KpiNQ9G+<g=*87F3PvI@Vyjp(zSr1xLB(6^bw)%xW;pk~9v}qtc1YPAZHuqI
zePI*=f-*cOE=Ioa<U~lOnTp(Ukm^2FBv?A!9Tv0U;jSudxL=HzED_F2)l5fvbtQ=|
z`^lyEo3uBX*Pe1WTpbY1GOy+N>L7LM_{nqgo#vg`hqY0NosAgzB+;BUoVauQ6_lWU
zDBo4h1zE0-7owCy7Czl9sNbS2-WbfDvT8BNco=19&Jr2z6L6WLb0W0uv6m{fen+>h
z%0_6PH)`?D>$)OpbVeBxf11=c&W?kOOK=AWUzo`9ew?HnZ4j+4X}md6b2ML;<BiEE
zEjdLyc<g<coAZXg2iie4=Ly90T(Ehm$pkg6uJF3D%qIm%iS9Jh4i5n#6zG21h{mSL
zrwyi2nbLNTMV3*c3nS10mQfb$PdTT}3zBxss`=i|etKQE^Cj?c<Rktg?Ws!Oi%Kxb
zQaDmEOziYhtTU$nAx0=CYos(aK1jYFH>{x8CXZM1l|HsXFW8()kB)GqC*?;MrKXie
zccP}O#<HYdUQ+sa+&ug4%ncGJM0}-H_UTGe6DupCl-P0gb`<D|>LdEwQZcarj(zn0
zMD*2Q4<q)Bpj7YcL(@&)g!#6b*+R0eJxrK6Pc3*&sMaG-i|sx=EBS${KVjQzuO`K?
zJTHY{oMrn%Up!a?PzzxTBP#z~K?9Pbho1k|vOv<ymrqn-$BUkWQ~19}!3BWq3*3r^
zMBwmRhAT36TzvL$8e)I=suLArzz$auLhC-hCh)ZJlT)jh!T$8V%PF;}I|Rg6lYwPz
ztLf(lcP}$6wy~$vIu8)Glq^2NbZTO(tDNt4KZw_iG|+dNiWVXy*DA-6QWO-e0|Fq6
z&oq8oYt|nP)YeUncBwJlad7>-u9EJAdB06aeDxKW*I%sf1eo^?+l}EDa$lRC@4RYg
z-WhZ3C%5x)l=OsKp4feDXsMHE4PNFwe5mhrp8ue`&ZR>9)Lco}7MZnxkcX%mH<&<{
zVKGnBVCECQTNt<S-fkJE>msTaGy5sGxzBuVIXo&*WQ<+ZW&Jwg3ky<u;_seCsGqq~
zhCm5jErL(#v@^V5PcK?}W1eP!n9Hf1&A+>PGDuAc*Be|Lqv}Yt!wRQA2JGi?lNY2A
zQh4YDTbbbUrKLrbi;3_3M7XSO4wbfyRQrKXF;;LO2KN??l%6NnE;ouudBUB6q$L_#
zFhkqn%Y4H=;3sNpZ+kuU2^M=xsK=v)OY)SG!j|0k<WKT|VpU-Ks$>-IL3>qCaMr;1
z0k6txD7#)_%6+lTWDwugqqB#`t%7l@O{oXt90<R2o3wT@3Y4ux@5Czw>E9g66(1m@
zS;%PWlB2bORG&P+Lh?Nvve2{+grBTuRnCBB5@nS^1^W~I`_>D`S)-l%tgH(5e7$Nd
zOQ%>I0uZEts@g0EVj-`;b0?XF)%5EzBQ@gIaq_nuC<Z)DE5C=DLKX_8cE83#qYK)|
zSOy(ylM`dsq$YJx;i#g7S~`awaXM$~6}smF&&HBW?v#~jOw-j7O1g1SIv5~z`O)u#
zeTy%m12Pi#%is)QR0^<gf_!Hl!=MYP^{haJ>3rn2>xz@~>!K23N~U>@l9Bn!qoiWc
zAc@u#(v;*$VPaS~z7S!G%!F*o+jr$}e|o8E*WU#HN>VC!iJ!5vy#U?;+Au%?P9O!p
z&Ca19Op_7+M;`*A$nn}9pHNw5s`fFaOiey|Jv=?d?tU|w)A`OJ9sWF;3S!N^&zjeE
z*r_)*Qx1V-^@4RDBWW-~Kxi&kE9j4&h&=|gIML(90Z|M75f-J!;-mH8erHB-{EJit
zASh#Z%F$TeY?<KS1jU0H68fSH*Y5HiT__+X{x?8OrD=i`x^D-_gc*hqUmR*_E_qlV
zq(|g#|BJBSa!tHYYy@%-*Z?+X^U?`~7CcSy++%phZn9Z#Vm9w<%g6oo#$LM*Zm!|&
zyFen48enMZ1%7d8_j#{o^f$-6-q(7%IDg9Dk9KR!G3LR9jW{q;^iS{>0u(O|*g|j6
znqNML!0dFqFS~5lb0uYMs(tgi6f}qsk6VaP((ZsR*I>rcc7vY#dx5t$J@0E|5r8KY
z0zN8gU%CuE>?NahE!~34a!bo)r#j@`&f1y8YoWyS%G$tL$xh9)0Yk>*hZ^l#4BT?W
zq=9TSPit`E!A?*R$DiqY&<|8JJQIJucyhlsW`E_g(FXO+=gOImPX)kj0LV0(Pj)DP
z*Ha~w$cxjQ>zv6?r_uhV*MA)kb_Klg6@;_}^d^hQa=i@&=h<X&n*5e%^Yk1R67XDn
z270MeZ3IuC_b5MoB8~_PH4NK~Kd&=<=>Q^50|A*HyTAkq%PklJgBu!SjTy{y3p&?r
zz}0+f63`+x2aGXyYf^;Nh{EaJ)lVyNm-g4Nj1iw|Aq~yMKGaYivc^jd(Sqr^b-Rbb
z`$hk+0Mx=sffi)_Y;qp_a43)O0QWGP=g<8E7OnuW@PEySLpY)KuW*8mUMEfxmNt0d
zXPSLq#ICGT!!CYV*-fk4Q73@mUwNh(ee4*Ox{So_`@rAp?Cm<MBquS5#-0^6F+HMN
z@#*aG9BT(MNK6+YF+HU-`oC33pWx5Cm@MN;RyAAp)~d^JZo|H7uRNZl2YRjV*@DkA
zgb6kUfla=r1@y)HpB&N<uW;k170=h~1>IWjrPRN?l1$@BlhqEYr5r{!l!WId%Y3=L
z)qsoLN`;4c)I-(%s41dOV2|9d%F>n*LQCp=TNB~}&M7VKHv2D51fjQz@b-2jUBX3;
z=W~nfB|?<kuxWjy=d3c;3s9xi8HVNWPaUed1Xm~C3|$CF3D8?X!U_U2i{MiMEpCW;
zkOWk1Xhpq4{oJnX1|PZ2+l-Vt{{6NmJP$B=o=Eh8Ak5<T_4PUK%^K4>?bgesm*#3z
z=adz#g(tV<Ev#?Ritg_JO5@gGfe+Ea_N!XG4=l^@>D|S#Q3IFJrVx+K5X_+K&Rf2d
zO2u=5SMSC+k8E1AYh1#EC6Fu*V%xF{Uk9)(?|)|L5_k_E1)Nbc7eTERVzpm_QlYSF
ziQs$MlG%=lzC=Srb!%vP#Ur!n17l>$yccSa8eMD{!E*=~6Pzq~b2S*2_v0TJZAwA0
z3()aUE!w4YbSNEkRoPrsQ`*`N#_@XWzgYJQ%AfzL)Bd2J3hWV91MHEdlgJhkQer<K
zfEmC9-IDMOX}>NA_Lr#b?1SV+@!WI+|K&`5-Z?Ulh#w2XwGyX7Jj8A4hHf{_Hlwjm
zSKH`x4t<k-3Q|P1#27jtuwnkd2fQaU#H9w>X(g1=ERL^hzY5d{7;GkS+xrDt=f&){
zzkmNG<1E+|4SIU1^kX%ZkZO3FQ!bn}DZyO4|J1nq4#1)md$*pY@)e0;AsaAtX7fwn
zKd|RnR%e}pz~JJ9@cj06nVUyYRILP#WWCvCAxP}I(Q1-KUXOi9;aBTJC^J-o;)v&H
zt$JPTkp!pBea8wQj0XB!y#Q_ytEulY>Jh~^{1t<taHEjR4(rr3m`mB2b6egtI(Z(8
z8S5$*D3mFG??ZV2(Esq&Bm3ja3DocE5;TPPsL$Um?~GKZGh1$|#cJ>QUEpxuYszz8
zGo#L(XzgokT9EU%9K6wV>CLt_=X7aXF+Y_KJBgFkw8>e_RY<z;Da9JMM9-ubDLT(+
zy#_DqK(I+xDfIT!9Y}&8+Q1*0;&D5h5{Vy$r-+m8!u`{Z*Rvh`d@$|<h>C=*FOvPf
z1t@bNfc^emPXqbgdfUOz#nrEcn!rOss&v|Vlh0snfi;qs2Q<C-r17=cGWV$`)`b{@
zOR)mkj}JlMT?Iu4Wk|Ra`*{40XhMEBHuYkE#u<oY1j?}hogPd17Rn0{C<u&Q#Df(O
z1FwP7LY3(ch2Rf>z(3eiK6iA16Hx4b80rj#2vtn`m>ulXC8RX1$MS$5CvGK64Sa03
z1l^_MzOk8}&SzhDb8-_3YEGTWr;aKupnxKavHD1$Th<#!1D%(~H8<uqE3k8UCBq;3
z5xpBj-I&?$h~gD6Z|hA+J>KktDSU#|V_l`jG83d8b@;Ye#09oNTxKBI4aXsXnSIsJ
zVr9NFBpxG$__`r7R#$m`0Oya}iekyr*cmVZlQ0>XRr(z;qH56%D*daF8%w<|@D}1U
zZFXy1We60E_-xRd{^4_cwPVjZS-(}uxO%q|xrdINl_Nl40PD@bZBIiqdwgY4YBJlT
z7kPXeq37z?B(^??!3_#pICr0-0d|Vs4In5tJ9fy@!}VZv{U%=6VwoV3uC>|Q$4^O#
zmo9HpA9*#ds?ysN8V9<!5Z@8w)W||@X}wRO$&tnxxL>-6c3N2|#r>qVDq^H6)&?IN
zQW$0eE6A2FkGYuZp9?2rUFP3~IrYd!15Kvokivo)i@7LOElu}psRXw<g$|*O5U|<@
ztq(7FPl4#8wBfe5^nOpe73Qhqq|(_)l$^)2L*Clfyf;8W27zSGq7`V5K!$4AoTzA1
zyv2V%*1CBBacXselEZZfHR_YbonL=8YVhQD2~W7q37{zQOLBO?s>mhj3$5-qv~ra!
zPu~p^LuJ;cN%n@j0VQVyC{o*`CIqMlLX(XhlR=v`G5~X~psi>%zXd>u&hYeeHR+FT
zDijcfUFjDisjb|n#x>)VN_S>3`q8{wnItpk#TUNEoTa+uc^(Y934#MWR!G<Z1A4<W
z^*lxygT(nBl?+p-l?^7dx{oH^C7K@P5z<H+Cj`+`ibWZkWZOymOo{ynzC;&I$H5a2
z$ud|lyQ~h`Y3?x{D4~9hM%@G5-ucFyzE_Yecw-Rze^1Egza5<VuLgzw|54n(W8?-H
z-T={VLAgIYz3;ZOzg=M`c6vFI+HxDcfLmPUX$gFQ*rbi2Ug47VZ)m8!?_tkHdj(Ek
z2DGEQOQJ2r(8@zy;=4<$Ssq(}XX8uG8MyRFBR1n>6mCscG~^D9j2a9)nS^{7q85>z
zVAkkD>?{w{RFJmQ%VWq75JSs3DI7Qt-1yDg<0pIqa9zT76Y`r&L?48&%QmXK8r+d9
z!tD!$;Wk|`rCY!JYUev`R%k%8Nz9k==!M&msDl)dWmIbTflNljYv+qxaXL7msSh*f
z2aH?B1B(~mq{P>j;Zm*Eu}eP}(y`}PWol_?Xg+yVt2c;Zynt0OiSiV1$Zqaxgsdj9
z($FYf|E=xBNbU_$5vNEF=`;b?xjsQP+;$s^byvs!a*0x5qzV*1gR|og1Xy~16Fhjf
z6Vm$(CWhaw=AD4Nfx}*vXiZ=wzfnzHlh_4OTG7becTa&9D}MvxG!*^MUqOoX4yiAx
z-TmfPzLRA;^Tp{gw$lP-8<lgpgUe|%>6dewx=^F-7<b_>k2>#o!k11IC&zD!=H7+q
ze)r*!Cmu>F+GEM8h~~z8f~Nl7`lgYh$a)=Cp$!$oBo)*h%~7K`_u?2*7eoXLOhOo-
zH9ib2O-bjE^(Z?<>tx>(aW}nWGfa3c!|hyxUEeCu*6F3%bW(r__teTGh1xoD55gt9
z%o$538RM1&_BN{JZm!M>+QmF~{-vSgTAb{3js;M`6)3OLg9HKn+u-v2%pECqrQ*G2
z<{1O)YIP7+S^zO5PNyn7@WyQ<i(donKcyZ(7<S6%^WK#43&#v>2fz1bDQ%&qP(h#)
zZ&+pY{G0PN^PTR^l3{zbEgnl<t5f|JIZr?g&M$qpDN$T-TCT|5vi|BRXsyz*s;L^0
zTFL9<DgOj&|5YYkC*Rkzi)PIbqQXdMS6eXH?R#_28?iKL1=z5h(eSLn*JAOELMsbr
zDATx2ly@v+c9zOw23$JT#f84@NWTJ6_o$%_%?;UMkwaul(Hxo;CrcVk8LwG2vqW@)
zw4g)mQ!@#Y9B53oM_f7g6~rw4_08T`diaVl3vD-4Fk>p)<_a1-RQ>f!<&uJaC-9tU
z+#YB$EDuueT}cAT43H!7gQx3TG(E-^KU0eW(F2fM^ckSZIf63y*MQI7ck#*5h5yha
z4wUJ8jsFps_`jx1$u(k2N<?={9PI618hkkoay*b1^H7IO+3@u)wGMnoMK&B5|0mPA
zB^R_}e+TN#x&H*}A-OmJ$}d3N#>@Qm3&>s`T&MsBv~nRB%=n)pAa(WsJUvX#q%oWT
zvPW-<ludZI`hEh+&LV#R;I6mMUcOIAt^+u+q=!0N7&cCV#bU3vJe}jEgh5eR`Oi5n
zBLAYE4hkt`uMP=3(@;=?qB`IEjP)QT05Kc#GKk*^T1K}W@!k(mnvnG_4+IAfhrk1Y
zA-OD|H1+!qJeU>;plQ|X9F;fF*8)Qw7@25)H0gKX#(W=3mHXpA{tY`1o(tCiw`eSP
z9Tid3WqMwW8l7N^g7zO8LeCDAJ-gG)QIu}%0f<A|th_(4;?R({q2QL#Vuz!Z+X5{%
zneQaadMH&u%l*#%AFW&vxTtmmA_-6o>)=6;K`DYUv(<l+<3GuP2m4>c9euXCMQ!aA
zN%yZ!ahmY`5u$t$h4$%>w7zBzlC_B!L1-QlJ_bU2dwX(I;e<Z91npG*tl>VIUx(SQ
zBc>*(BE84-vyhsmVW9qqkMYHT6RZb;lHZK{2uN{e{n<ZkO$_2%MJJDbC8z>?ZxnLr
zk6h5h;r@>2{@e?agB<iywJbb;q!sSzd2x>FpSg&2;QoOMPya|J7{=Yl_s?APxu6^*
zvhmm->BoaT$~OOJF8DJb$o%1W|17r3r2C(_kUc;XJiB@PpT+v3(*EckFf0>jpPt7j
z|5+?}@#Fu@^`9^PhtmJg?s)45g4+qIGLLCP{$Y|jZ+I3WzQ4cUE>r2B!ZYFr46UF>
zVk|$?3o5vI_C@6%Mv`T3oke5YDtd4IVLo^`V7;3N%;1qMQjrtKQ@`Z#|HHkpjevK<
zi>wxora5=o7j>@U(owOX@>7t1YeeTE{s;&1B$)K%@Y&zxR-g(X3)CV^%=Zf)J=P0M
z>Sg`v2ztNQg8OsYZOk2+#@O3n(%ZKxk3{AdFnnPJJL^ZLu{GB7BAWgaRAq1|#!r9(
zXc%yFIWmnV%D|Z=ZoYA}uA4xEJee;oIWmpBSLvk$2#i78@^F@a8##8xL(>Y>xfkcp
R9s~c$NGeLC-Ff)p{{zlJ;^+VX

diff --git a/format/diagrams/layout-list-of-struct.png b/format/diagrams/layout-list-of-struct.png
deleted file mode 100644
index fb6f2a27e07a766729d12ea33454db011ce6ae00..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 54122
zcmeGEWmJ{h_XZ3Df(j_zCDPs9N_RKX-Q9wKba#g|(kZ2Mce4p;Bo$B)Bz)FBN6-Iw
zp7DHrzq~)kaJaqqU2Cqn)|~UYuDQdN6eJ%b5g@_9z&w_g5>tVJfuDqdfjf8z2afm{
z35kM#U|m!spTks*6YYclAUaBEyTHKUP(uI0N~=(ufD`7d)U;f+<mLEG9PF5kOdX8P
zm^|$q!Pzh{0-k)}qn(+n5vix0t-TAMry$v1XYhf~(8J7Rq<@{_Y9mOdC9gy(>fmff
z%E`pR#6l*7L`q63;B0Enry?ft>vHfvK{88MS4Tc(W)BY!CJ%Nd2WJarR$g9SW)?PP
zHa13Z2BV9Yy{nNYqrD6H-#7VvA2Bl*6K5+&S1Sj5Qs{k+j2+xu1<A;u5BlfN-|y*a
zW&VFpvUmA4EHFT3=r_!)Of1a*+#6gf06oel>R{*SZ06zu?k~hC@Yk7t9Q(iL{C&QX
zrGu*ju!6IdiL|||nKQW9)d>1_LTtaT{{KGXzxPsfwlV{w{_ASiUswP8*st>in4tsz
zmm&VP@?S@RnT3!9nE#nFA*7luwpADy5g2K)XKJ3XN7-(#h}7rbt%*2#cxW>;^LGW+
zH}gl1Z04*Y;28KiAs=m$7&>`)JSJFq@l|l->f^P>TQl$0atrU`+&dHJ!}+<AxBTbj
zE}vUohCC*`5cZ=Wg+-JSfy0P<1^?@#I40hkz58~G|33ED*Cz&WND5!>{pWNHOC(qX
z3)PyX(%(1ueJ>iG_YeMi`#f5BtStF1>w@1W{eE`d`ZtpQdxyWj4Jbxaa^ylNkCyr4
z1;LkrGd}-41fTI-qzi&o!A1Aq0fCqP@1g&Hcxhhv5Y7vH7DEx2ezWrbntL)EKF%k4
zzt8Ap>Du6<AK&(8q}&ewYi5xV3`-@J3r17p$0+%6tczj)onb8TJhZKR8=lBkq9*eH
z+PDW@$q^;A=l|s|G!d@?>uD1Hcfnr<`=6Emwd((W;Tj@q`2=;@^G{Fua1Ek3rlm}x
zSS1c0m9<H**)G<+5e-GbXWKr0w?^X7ep;Mq-)dEyZr(5@aaeV_Sp9GJUy2Yxt#Q1&
z`8t8jP}BLu@273!oQ9p77au1C7wp58jq?N^D^AK9CNaFbwF4x}We4voPCn)O`}Lwr
z%5gzVRTX%5Fk~5OPx}c`@mLJ!wJpR$t3UiWh!wohV$O7UI>URlowyY_UzTaH>b6ic
z{~Vc7Z4j^ZeZ0$rAUCPNN$0&)p|mQZ&N)p3V$8Ld-&flz4E#QRY59}`?rgSf_ioku
zu<~kL-)kpX`1YIXIqb>thvTMY@HS~<T+2~YcVBaV`fi3(ztudg_&u&w@w^9GJk=d{
zKPXW3{cq1Z?#`=K<ygf%POTw+&}WF_zQ9^GC)ijp@Xwx3#)uWZyL4N4F#vhYwFtwi
zQKl6;(y<Xj$hG3w<M83C*mZkRprzo_wr+?DSKrepSLmg)Sq%T@W-v_a1ZgUW(*Utc
ziOxX{{?^TQq9S-x?Bc7_fhP+>kar)wBXzc7h5e4zhf{3(qlVqp1Tk4L(LP*%Zabj#
zfk2o$ZrG@V7W7=O*&0!p_mG*}I~PudsB=f%xaZVViCrh30Smlf5=MFOn4n=sf&Y|p
zEc=B?G-nLo0S$)>)0*h-Al8~Hf{K=9k^7P>N&?U5^_t)HF&LcS&zrBPz{a18<^$h9
zdHHSX4Oi{%n?|ZE$8PYD_8uo=oXZh|yXUMVtUl4}z9(G|z(u!8$Y5^1?`Ap(E!)<^
zZU><Xc^)?|G-v0$ywENlH97n?qbTgS-bqvhMn=NC;&=O_v3dQ7c58H_(s}XPT+OWR
z*zxTTg!i`>%bG9T=AQUn!eYI=zStogSW7kV!>7C&=l^2caeD!*zQxpblz`B;O!i?M
z_l2vn>E=Ke>VnUg_XG5(3NL()8ykj}r8zBdGOYd{r^q`@OSe>=90B{59UK<B`Z2CV
z1wL9tqK}?Ud+v^vT(2y0yh5HDRg1=bsp&!S<Yj7FG_K?tWWqyrpq$rp8B@vxhG*`_
z6!@&6Wfbi7DH}I+>4nGY`F=^R(D|&Y35C|km-esYOnB;FJ_lYOpY))9=T!RgVYqJ`
zMR6(H<0J3vuD;K4@Zk2&0(q>!*+%FS<D%_zvenzG34gzXvXZ)6eRqi28?FHY8wNf0
z8u2xcwU-r>y9*AdeYhnQ3XgBTz5IFm9imfA`72x5(}$~M;)z$3dWdIN+lSNqGQo2z
zn)`F}R?J^h7K%!tAz$In$kWb5h!L9uW)OY#4a?8F?;WWZkyHb&Ez9<8P7&O=6}t3{
zCJH4FZCiFy9uPTp=J<Sul<DBuTfkXlyUoiozlRSHyjTn>N~4WgC@qL-oe;V{c^jr0
zHtl$cs>E=#%|#wfWCr`my1X((oI_!Zb%3ZnHOnkTo2S?RsV+UzrRJ|3D!`s%ayk%G
z@RcAYANx4Z7Fi_2^OaOmY=x#(sY<gprWf&xb)t6Doi(0fBeF~_rpZhRZ47ts)FZrP
z;4m!j!(x$-kuiT?^Zz-0!M$WzFgVKerTws~&2w2wNX4ZK0e5tc05~Zs?{?gvsg?KV
ziqPNT-jzHcMZae#FpGQ57n2V@W&I1-uqHN%aIOrKe;n0M$OI*wAPL<3NYS-#!PeA^
zr|T6KxcxSRC?DXzZ&i{#Zv^|c_Y>G+D9@hmvIU;ja3xGu<DqJ5n<bym>&BLrD2K2T
zd2PoRBKmqS8ik6R?;CvFk)(+M9_$c^NJYb_lo2#dBH(#a5GxdsiO+nrPvEmE@O88&
zp(LC-he!ABY=n6LHJV6o>QZ(w_Wk#d{76LJdv7Y7hH(_Eh>{ig_9;+$&{KO6aP`op
zBVTn+mYe7c5a>G(QF8PlsYr~hJGAao^(*3{6=>(6<NKeF>4fdL08gn$LOy)JZl+D^
z6rTFR9l~*jf@$lIVfOYxYnIn;x^m&-bJ~j$<_<$T-cMOB-P;&lQXl<)-ijq;MiAzu
zWQIynGYDk5FIgd4tGJg6=+6){D{1AY(2IyOunW+$zqEHI8nm*w+DWyTTB0e4N<i}9
z8CR=Ife-4kiFeRo45m-s(BF*N3>SV!nu5I`=CE01xX&P9G+5aD*fBO##JbYPT#fuA
z$J?ZJAM`a8GCC)5EmS#2g3sG(q^YGlCFE~mK7DTcLNx|wtB<`wI$X|yL)MMtL;She
zRdb&{>#(6L^d{o0uMpY(i&H=TC$QtUfywyt{=5E%)m{gB1eFnlXt&v$97T>Pv7#XA
zW>?dY?kJA$S`^cn1A5|qrevW3MXFIv;w;5a(|^U@24dI-dGg%#IVoiI1B@S(OP>Nb
zBg9`5vi&@b^-t1}bJF%==|<;Xp(;Sg$oCCJpy?sM1i?{fNNrG*9W}4aMz1CRrP%Y~
zQK}DM>f^L5OKkm;Icn{mGfIv?IxK549X{~tz8e-|m7262vi$+p|2S8s{1%Dy5Z|u&
zqa4f7$!dm1WAaJc`uH4|abB;$(HM#%C&Ye7Wu0K63)yw+;)FTOqcXJUyyN(oIb@58
zs`j&CGaqX9=LxgUUtbHXA4@dt*m`frOBQ#5<wun%o#AHrb&T5p0(;U5`!r+tuNm?V
z+y@&t;*m^5u{>pB^uu@kNf0A4l5bwHNsQqF4{-;krB8Rdx!JJo52Er@mrhlL><~ee
zya^TWF!M*69BqFkr5Ywy;>TCoP0;sBw9m6TNvpOG{Y%MZ;|B9HCud^L{S&YYM#9!S
zVp|r6>t23mQ`JJnTUR`lFbKDQ5_m=xcr51`Ej0p(Q6V^+;yJ8UHTNzXn9u}vOBgp@
zD@a;9$5UOP`c*T%e<gAgyRt5(v(SA7Hrg}C9ty~@NeI4q805^JKPQ|O!SviTYwT`k
zf4dh`QH~cqn2b1~RPEJ68d@YW=JoxTU}N2tMs|V6K(V1lx0LG5Cx`p~RZ7MNnxANQ
zi7;|dBcM4$KGGMVSK%pk>q#3vDnxeD@N$>jA)BiiJY@+b-d_(wjlo%bPaY+%b>RuO
zs~xv+<zx*qn2K5C99v~%X&ibIR>5Gc{AbpjY}Nj^hXw-7XL8{x#y2za+{7CEqct={
z71?+*af;g(10h!E>{PWzLt;hiLeUMyEXCM+Mr0C>XYY;*{F{iUdr{utts#AWV#<CR
zP$Rpb5<hhxvllT)65n~X;B=b$NQPU{o`sWyO`c1s&rsnC<Wz)&q)E*jmuSXW50Ks6
zDsERIu=ZwoMo?beua(XLO9OqL$AfWLW8ao4NK)-<PFRGcj(#4CEgP#rsE%QqxN|qk
z$`QUBl7o9O2!Fd^`|J8{xe<&@6+(9wTWxU}cy`_Mp`f(F_DbQ_)s+5>7#<gAHfiV!
zcCj!M+F@Myc|;`b0mu9MGJUN)0m}&SewWTp6Kb&28wh6Zc+vEaNpxMkPvckdgK<@E
zNL7(Dts$*L20^t&aExKB(POXwPP+15JmjF!x~Ccxom3g~n4MsNnoAdk6XOb1@S74>
zu4~=1(@PQ@F8#T!NBu#}82Kr1RzqkL9K1bnu_FQ=5(_i|dj!nQN;#LWjEV=NLwgz*
z^fT#baE?-nEL?a8>9UjyB8yUV==fp!W8JtpC}M4**fqIHx*he$b~J^z<M~>9Skf4^
z2ig;Y6YryYdLo(=ZcpLM>U`@{+Hx+00|AhuWYTkSrLJF&!yP_%o54v*_38l8K5dWY
zj_gf1PoL~-Q|fB{>;W^nv(XIt`HfZaD?Wy-6|NzSu$pZv{&QqXhG<q~U)*|A8da;T
zGH<IuYG)$X#M2i4pLbGWwzc2#*7AbcA?a6C<`!ZGAnQ?2^NRltsgmY4k#Jj=zb`mg
zkb*u0aXPc6t$H}sQg@YoyPs!Y=NLl&fdBi6-)l`-U+sLekF)D}IUhapmQT&b+;wbQ
zMUkmH1YnHPU7OZqL}+Fp$iof{ewVdEi68DR-gTJqzs=coBSgV+va)YK?YG@v>Q6gf
zYWOuDC75A9bqTZj{O`GOoza@2nLR+O{_4fmcb3TLI80k2R+5HWcU723kq4t_)h_nM
zHlWa1TQs?R#Hd2K&<XB)N&RI=pC`*<(Wq2SK87xth4!~(e)S8M_vaQZ=Gs>oB*C>0
zc6Dep9H%`kJ|$yk@iyX%Qtp~QlzpYd6a91T=bZ<PX<|ZBItGg=t|wAn(FmI~Wj*3+
z*L@5Ff-c8lOo6OgTC)+0ndF=V{`NM@LX@s_CcY(xk{+wL+ZV<^9tnMt*0Rciv=nl1
zQ>R?5k80kF#J>yOx0S)7<e9k$XLzR&T_W~6)0`#kvno}%`DiIh`@U|-u2F!E`|Egx
zY6VZ`#PRvlnRs|=8ZG6~>9JWIt^i5yH+oz1`4mj3?Siy!dt(}9%r7F>D~KPOl->rO
zaXr50ypns<H=c_bO*|g<UOnHttiyZum@SGJW$t(WLn20c=VRyC$w<dDLnfNj?ZR$a
zYtVl73Gvb6IKj+h_`RZ|GAR$OSB|)QT0$);1>O-&v642{0>R>a?n!LqUDo~0Qt)s?
z*>paGd=oSiyK!d7h72+v`^0vJQfT!Y-QD%29gRzkp0pn=s_@x)54j)HV!Dl0#0Mtl
zyqR%f=|4~_?}7i&+@9G<^J2a+4Hq_yOrS`1x$Ilg%Ep+es@ucyW=FKn!)8~O?HW=X
zR0$yv!2)EqMpIyExt}&{edv}Z+{chrsqa&1rRhp<l6nT&eN_J~_?nxa@O0?CtaX3X
zL1!ks1;dSkqe`GzKv!aLDs6pCZVUxai#Q@@QU+b7$jYWZVc1^wcjhCi?!&aY;KLC)
zg9aHJ9)5(^th3I~OiEj{8k?o^4qOiB=;b2}`o6hGrAc_c(WI01!!@j!ZY%i}76fBN
z1io_c9N}PR^@fM<Z@kjv6V=%1)7DTin=q$@9f7MaAzL*Hn5LtB;L(ePz)H_Ui-y`L
zFlG`fvm^ZokL8t9odCv`K39ir#z1kL1cNt&E-72MAfbyzPnvCQHv)CEhc&QfT@x37
zjh;8CevNND-R@&XrJx5kej9uKHcTXf7IT!=Y(A&+8G2W2>>j9M<D0G{?@xsmu~kp2
zIx})BS|C`o?$u8S`5d!HBqaCaTcmTkn-e*i6l3)><oa^bpQ|UWi_wX7+=$cFV_-N8
z$z&JbbX%xB`AEUzbfRD~LZ$VJpq{+nma9Xhby`spBTH+Qer{}soxF)kA(FQie)=Jt
zkY>e)9ux!%ivYEbJjgS$Rj1+|AMX{Xfm$A}6Jw+C+ir_;F#^S1PZM$M6q}UB<<xm=
zy8h|Uqw#+Oo!|7}PIH}uwjBEMo-G!4k0pFxv874yoipyfk#7?_;i&^TRmn|KrykA*
zrHTU6lo4#rSuqjk^JXa_0k+~>E@ycsDbvVRk2T_FfruO_Ch_6tAqbijp4T}VM#=N?
zyr7EaBS|LGc;dxJs9RZhFeWw2UCq&5C%VbdPc%!#hGD~4zq=8efF~>(nNo?J&R|{X
z+%!!aMIu(YEeb1F%_Cc%`Y1lSxp&R?>tkkS63_L3hq!8G!Fqn7+=^Ac$)fJJ;gKIV
zR?0aN_=6!hMj}x~N`nj%;)#Y<a<bemJDE;{)}s2uq@!uXHl$aR{2%gq1&wEj1<FO!
z7Z%-C*{#tWN<K$$q=MKc#;6}$crb^_z}lG@h?+oRBu7JwMKCpW2{FRWH65<{D79*F
zcuz&vLu#)upEb5VC3#P5u^+;m%3}3Y&*k`}EdW!fTG`X8kJcC8^HVaBh;y(@=xQV@
zHy-M*dhLCV2tAU5@53$abjf$y?=>!D(75-mKIIPZCGlsbyx%#SF-nFL`@Z2tOiQRN
zs)`+jSW!=QU^^_0NI|(}x+ak0nXul=6<Lul7kz3)UP!%DF3etJ@j*@Kalwh$%dg##
z>R~AiuR<zzvz{TEsMBc0<cO?@SFLj+YQmAhiP1QhXc7;7Xs2GCPMQa4zy)YjrwL{j
z$Bk7mKYm@BCKrF+m{{4@<1H83tm;Xpv1ClGTZC_=u~XSFEd#B4l^x@<eQ_P(J_Ye@
z(ye!QF0$M$O#f{{hR!Op*UH6UrI%_A5F_XcEq-^b*f~aw_qlX0Toto^;!+~xjw4k<
zFo8qm%UVunTt;{Ov}CnL1>c^+PxI=OI#l`8J&3=u;;Zjl6|N~pypfn*xnDIqGu327
zAZD#5zY|gcs>{yE2lR=QGPPGVrR}tmua(kD*|khg7M^dROS<l+TTmHFXm{~`^xUef
zALlETudq1k??Q=FLnUXV%Ek)f;D0skisalwS1?^qOFd-BC!a1a1N`xzm30gS)ifl!
zk>&nGF`L+RsSvtt@*J|`m6nVozE0$2XYz3U^Qcg@3K}dCbagrnu8!$3)h0)#L=PHB
zmZ@8m2PQ<-tz^c8o6Aw)k(1JBh)G!+gpbbo>ly$fe92h`&=>LfEqGc@EX)mX;I``&
zc$zlKK6(Q8yZ=!}wDL;{xlZ|)7~=q@4k<<9TUU+*8g&Ar!|kt(Va#v}^~n;!nNGHf
z$y8}FIlgB@4B{vYFc&f#*!M@!rXk~0-c0@D<FjYKxPs4Qlg@#dKh^b!x&l@cB&TKc
zPMmg>gX#s6&}7MZAre=M9&c1$oY7+&QW}-Yue4fn3D#&>(J5@aj93@rGR!gK7m~9O
zFmeIKeA7XVJ(kF%M1E_vEuZeHOJlzRKZC|V`*gxcUGx?S{*}>Nd8dRlp{9&Fb?b@V
zdGy1>#eurk^+Ne7w}%R9`1)Rr&4L3`bgKfyF~Iv=y4hZQqnq^Ph^^GaH)5os(zrT{
zxu+;QI|%!L2nKCs-04bzu)&>Y+!*=fF8J8xYLdZkU(A^-gk{Q}WeQW9Lrnaf<S<8>
z{X~PndR3U1V-dn1@s4ZpcJc*$QxV^N6&!^Uw=c;Oi+oo+_Xhlg8Ti(V^dSM1sLPkn
z6wI1)0g@$x;uRgZPiS!J`nEniC#JUkYh(Mbgs1JHns3+2i+f1O*zfBVtPX~A)iqym
z>M7Rd0^o-;I0`41!<9iEogwtoxk_Vtg-g+DEfReLW?f>vnDhpM0P<WysGU`^9tt{h
zw`KVDu9JWV7xAVz(S8+rmfI-3MpZGa1tyH|QSHEBmvH)(1lcJ6;)goQdaGH|DeX&&
zpUX!O+l=CFZtO~Nn3AKiX$~V>GR&s~B@H6Pyl;rQFI02{-!n*+I0~aY9?}S?P2tRz
zL5zB>6?8@9zGy^SFB5p}o+!`N1xY~gtMC`CFot{nu^u1sy;_;pIxUXy7RFF^mQ5lv
z$D63;B!-DytZ==6x=^=?Qhd*mn40+9n7YRr!=~zVUr;&+%i(9KmM`iwG5gh1_Rq54
zs+G^3CC)sk4Wx@^WW*n=N;C9dP|CcyG`XtKwYxxxN}fz-{+si#<b<oF<T1g0fVV-?
zFY4l^RVGrR=UCnlUPM36^x<;TEHe4LM4poMenP(hzv*DyNr{<9fC7SKcqbPPyt{zn
zm^r3^iTH@rj7-pI3Nytj#A_8Gi4`?;%bbKhHPgwe=pEd{Vxvaz(*V}hXu{z?LpKjW
zoi4Wxur(LMkSHjAc~xgozuzD!TUTT{F`tUkGO}e0Ik3obl3;q0+AWTw{aP+6m!W4Q
zJ6iO)hXfgd1(R+CM?J{FFCPkgUvWu;D9@(qd((y(+@zR<_Dr84?AClhS#Z6t)Ff}b
z{F>VC=s>wi4l+7EURvf^Pi}isLcXk4#NlA=Y1SEePyfjYX4hEf<55C*J;g}x7b<0k
z-Cq+8?g>8>yHB_5w-jUz5UMD+8`s`cf=H%2^3u<B*yR=%R8eyl2im{(I%M?Amz?sq
z{>D`23r{{!g>+e>jf|dH3=^RraUhv6ZV(GoPWNN`-)ub@QQIpO@wVPZFLC`Ua#U*7
zhkj>RbZlbtc;X{K?S&rkJS?0xR(JjgK*}k}C)v_QH1w-333!zZ@=>&@pB0tXnrLim
z(H`$!c#fjf^?L<daqIL1qH9ttqaa@xqf<Fx)@9DGn-aDqi@_}0VozHpguO@9eN@;|
zUlcmHYMRkcF-RXZ&9+^W!&3Qh%#H=Kssy0AFRP40J#cqe+aCl>+Z}r8qI)19)cMSm
zU>Ax_ZUjHCQq76&v5t5yDsC~ML157&Cxbb$n-_!<G1vF#RU3jPT+X^`fD~3EBa;43
z7i=s<50?nzC-3;#$EQ9``0Q`*IV|BI=)QFKq$RdrmV-sMGoAlF6W=60^!Zb|cR0MT
z*a~mE9is7Kk5hJ^YZkqR_h1CfJi^ISeW!g$9WSwcqzZqWWydr^9|GynU=ONryDQ7h
zVt;C|gJAJEO&poehP19uk+;GsUOD&S?+K}<MpGnxtWe*0u|Z*OIEuut&xnb9h^FqD
zR_q&^kmn8CE3HF5kifpeMmw>EvcxQ4GNVGzz<61YeoetWs8x!iYIv*l*_JWXDpQ`|
zY9{yY^%UwdF1gXp@@q_mG##cn<#R-ZcBTUPUJ_XvZM=~EJlCwoQF_F2!>|E{$7F0%
z<!Q!Y?7aPWCSnG7E6y(p?be)#cAlkw@-!cl3cJm`Kpr}NnPKzPCykTB_>mDFjlU_Q
zW@1f|zz57+F1x{@56F<#!Wk0mQYsQVtLTVr%?_{p_;3C2m>k&Jbz$|CaUKveT#~$&
zzzyLn>@2lj3=-4H&M2H}D>&w-LaeS2I1Y@<qnM`GWVYlGWH7b*)08W6iX<tRVy|Dn
zk)cdn@5PBYG(ShBf|TNEnC0sixyRiaxo_>jxk#SZV#LZ@uE*X<cUgs_X1$HG?2oc+
zs`orZ%R0YxRm06{N9pt+$G_L&)y&6$GJ#Q^puqH<SKXjUW+SkE7@_YZ$4^1~PR93z
ze(7n!a4JHkoj2#b0FHIX$aSpYL}txoRAK%yjUYT#{@*~+u{3FpGGUevGNQs}@oU8R
zfzK+8n(jFQHTMzp@N0j%(jf9Nbc)JxX#K#24PUm{YvqaDvb0x&a9TYqS%K)FcwtU_
zwfDxNC%&K2ek5w~5+oh*75Y=S<8eh>6=u*{;D)oTclxw`<5`)`?$Rks_M^d~My2*5
zu20iW#YDE5`>W5aY-k@y8!R1l2HXR#>}WCV9LkMqX8BliTJM}^JA;g%M<)|AOUtP~
zK}a*3+E!n*u&hX|$f+1E8DlYZvEBlJ_>nP-+?2_3f$ZENb)iQYnAoxi)K^xl?vPf~
zd~?-~c&Hs){<7mYl>tRbx2_?&F_8i034o1_XKWKy9fVer@{}7W2_V0%;8|aQd9ZXu
zWHRSbv`m|s`!aHtwEqZ3PWErrz*3yZ4eIiGudd}|>d)B%)$H`6;f<xiH26zV@3w5;
z$GzMwWghbuCrL^~q#@XNW|3*vWYmJizhIIr+e$;JVqiOqKjDSv?n$J0)kv*`D-dFg
zPUdOi&W(H|e*5FRClO~Vp$VhxJw%3*wfBwSr0{61D_@yL$*#ES;-34eo91Y883GgL
zU1ykzT;tFVXEp-vL@{6s9}rqmh+1Zu1yGuc3a}?mk<YzN5P7pP6De#|)mR#sOM?7L
z+it|g*Q(xjFLZ>1kny#TOR~O<;uJxM_K&ggcuOYwiW)c#4Wd5T@;Fi0K|>bPx|VZK
z7+7<N<1@F|MC53BM0&J$_{&#G7&QEY<K}1b8-Z1|{RA!75ucma(j4~fCFC=dUwZQM
zuif|OlWE%;Ktp~yUUZrJ)tmydEi0=rF?3q3-@I5q$4v93>ES(z_1u-&OC&U70&bIv
zTPuNz$noHE`I5KP5#3tvaVLK336z3Gp1|8A&k*NCl+4{H+^<PxV(Mf^LKf?(;sgM^
z0-!@7u<PH&S#&QxF<|@IdOxE<o*$`AL6uywt^6@9V7}oK@`UOQfk}Yn%VQ3$@)WN(
zsbo<z!MOTi6Lf+qk&SkMw(YCNj#3|Of^5J(;Wj6!JAF!lW5#&RVP5`enrHTnz!lmM
zA|bkFXcr7S9NZ@)T+gsEobSq?$-KoR=I<tBuTm*vR(xjU*dK>%)&;h7I!$9ud1n%=
zV|_O)?)hQrN9kB>re9=`#gwUBe>#6Vqb!qmdI;zE!By9`7QueB==hxi%GUP9Iz6i3
zXRpt3D>kVTAxk{{65j8s{sER~#13hBBHJH_EyWIn;jj`Ki_z8X+uMQRBqNR66W3+S
z=B*+c+76_Rx@Xs}*|u#t-u=Vl<cK^CtPj?+=2y(yu||szX05$8h|pMa7Ok)mdGDt0
z)>V@L?+pXX^^#JLtLe~@dk>b}J?!_@-&Ej`#wC~;PNOa1ELi3X`{rYOj%p>{5e*86
zT~%*C@>Sb+TfM{BQ-QNxk~THxa*|*E>1y@eo$4P+$6cz(+E4jQhiv$VIM17PnobRW
zqGph;qw%^m&B@GpAGYg+*P6tO)IUTk!xoNgxF=uVTKnSf*bd6=NpzqPl6=j_r`U^Z
z%qT%iwxZ{EwWDr)>oKc^oc*QptM$`f6md9tauLJh;zqlk-|sDzhY8d|*{{@_Xr7Tm
zh7;aDhM)!Ss6gJ!4X1peP0!9=n3iRJ3FvWF{>|?fNxXat|2z`P6Xun{N2x5UwR183
zGmgJ7lsric78$$Pc3TvkKQ&GwnP}jPOg+mI+W%Zpj!47PmeG7G{io{45)&Aft7TF_
z^*=Y##8h&0=XAPG{6|&w*O(@$z<BJO-((m6?G(T77Enc%w|->3c|rZ3=gL2%;aT2(
z&M*JRz(n9)$OEm@T<`IJL|R|~N{&rK!R{IVnDFmMv}OR!O?mio?th-k^)zqYHWB%l
z<2SMTix6_<1T&_I`A+yh&$WYUtf+98tbQ-;-+Zh!FhmuVz5jonyYw17SDvY1Ch0$h
zm;;7*A-{6>AAMK@V!#JGwh5-p-PXgZ>rwWJD4-WW6~1vybz7Q^85jJ%QV_!z!32o#
zcA!OQD0?||scG=i?fa^S<`=WhSFq$UJjUm2Imv%Ktu-6G#JcRsyspFhvtfFuuAu9f
ztEFWth99Mfdx#?O2=K-&RYE^+T+GFG0ri~9KNr>0_PNz2nsdJN<Jr44KmH$k1)9uT
zzp|rWch6gfdc`e9LoZAW;N*dJ<l=ClTFue@^s)xk!ztGN&)MV%2d^P*XxF3w2Y6%#
z&iD^M0sT$K$qE=<bw0BXe@x9JA23j*3nS5xKjQ0P7ic=`IxS5}_J@CbvxJi<JDy6d
ze)st9`b&rZ`J86@+=|P@>hkWc@4qcO=?CB7=)TYC|0&y}3W!Vw{C?hCZpEs~bI!@o
zS2h5s3rZHYI1vHi1eTWNn_wVEA<jQvah;L>{?X?PP<}{S=0eOf?Z$aOt(e3L_7gg^
z>s7j50(vON|L#gr;I#L_6JM59*BQpzUJMX~x>aNuIYFfZq!-b~fc?87av6037q@?#
zmL-~I@CQ`(EtKvh-~~dK@J7P@BLkfe>w(CZ*Faw($A8@LC<ITqysFi@=@AI;e%_OL
z&*EDE<IOe$t>2j2Pr3oVHeA-Sn^A5H%l?P+2IPntYMJDwS)15>d%Fb0IWTfzD!P2-
zfXr~sa_Xmj&r6kN7)YJ#%cki!$~sWE&9!VRDyH$;*s7AYI1LUlqZ~UF493yQ%UMD2
zeAef43$+4JW2j?6r8FZc6p4;)Q%g|x$xKTv-1TUtM9~Vn{|XcVW)Hps3XC4x2hfmp
zC2tEMft1!SFCZ3wv-tl~IazWAc(OUv;4)@O<8X4<e#-I50kq_|E<X~${Zrty16d6o
z|3)P9hf6SaAoQqvx9YwOSo1`}Ua%Bi-NwD7jZf9$w@5cji52>>_v5hoCs15K5f^#x
zRjdcIA71|aLIzbPS#|(nOq{y}ffWUTp65m|Yn(AbUqTC*6gxnCK$xg<eb4YMvf_yt
za?@}mNxZhpN^<y+Of#di*e3+OM$lIo-jiFRC`G+cWN3VpSr1jo^b{P@Xn$YR>KwxB
zC-FIgs+hpCdilBK)1DI;YwW{TrlX3?O6-46$>d{r0;~$wO%VCz=SFmRpi&W-U^*aE
zS(x_qzdq(Rode>EVeO-=%f@0=qYWaX*FTx+R)NcFucRr|vhD^VGvPhXXH*pmhssvY
z&`T8s&($tw!48l#7pJN3Wp=?ZoEQ<-YHMDY7@P<PqWIYD_qAL`7a$*zjD^-`TpdEl
z7C-N9t?YnjX9JflfE*bd^#Ir6>*iUG|0R?8y{_@$H<jDFQht_O+;Aj!0<BTq2vgf+
zA7EL1fy!i@rRPx@|B7lYABj^h`nKVx(TE0`ATp@zMp5W$8?7il+=?}-h)4*?bmE5$
zlC-u>N_&^N-aT>#K!vn3hx)!6+&OgcOs#wQ5FJ9hhAAL}P#gzq3cRmDp(>$m4eVC9
zV%Ul6TN$zov95#2V^^S|T6XMttUZDhz+wx6XDa_uO%Et=UlbuI{uz`iMZp&R&4$(U
z#PH_%{_d-LD4kK-OKbLR$01&EzP=i=6C{3@8%60y1Q<57P*IobMTSjP7ki4RD?vb@
zik}Qr$W}BJJ`F*^1S`&7p;sdLYB!S^1JV^2BoeI{$|woDNw{GyM_{oIutuA`$v3Hb
z&O-)2e`rS4dZHAJ7A%4yzyK^3g5`#^KR2~#u&K$1J~_nLHwVs3yNv(VcE2cf7i(AM
z=rx3mC%(>}JrmRFQBEn_H2i|Jg_C<UM<BbIIc7l=+c0>^^s#-ljG|1lk@}BpXZ1m}
zx^>=E#fC@ey_95p%J#f8DE*ftnA)j0S_`c}$ffQuT>nQn5}`-C?C@l-WXaIyz2EdB
z%xi-{hpU&~LOTWD>>&cy{5Vfkm{9!ZlP_Sy!Z#bv;N};YGrG<ukPEF$#z}J?rmeL|
z*)7Qe@#79Sgb6Rbk%MY6;vW$*&kTfwOg}4qFhzS?=L-g3YS;rO%|}dA?ajZA6hj6n
z&ZyH(6Y3BK84hQ3%N&_HRD*`*7`Nlb%oq8;f%z0qsn>5f;(8P8YD)g+%RdFS3pT8n
ze^?v1$?+!0(YjLOoBy4~+p^-f`gy~U?IoF;H+6)5D&K+YS5H4b%>B{x<nGnFY1U%2
z%SNP*|4EF}zmt?>38<UUk5fGX(jfNLnzkVjZCBdw%rmTtLFkLTK>V0^J~DQZrc0h7
zICpFnc)vC(@xAU_+op7Q7fSKf<9bJ!KVOi{a5C?8zBVGf1;;JO%hV5o#(2Nn-F^l3
z(x96$5(;D=*7I0(GefTbaSHY<W}mm%Y!D-ad(6m6_!dQ{y@6U6FS};7%c8FSyl*5_
z1b+LxMxGKb-2SaC=}-(g{96!e<O@t+n&<#gp9GPU*Z~RwN0?GoQM8`xlsGM)@;=0V
zBAli<Dr;3{1SRB!NKZE}0J=;bGV1pK!q<N<b~lh>W%@A|XS**6-|j2A_B2u2zU2j4
zP`y-KJ;yFpML}_@<#aV9lgx6J3pD>##-_zURiMp#x}3WmRm8pOib?2~QYmHraMhvX
zwn-&me3!jI_~#c8X+M?~d4oNtU=fHO)8I&pB5X&7$QF47&IZX<Km5P^#**qFLbm53
z0cb?;Ku#mR2?Fmb0HXo8PZIc{Zi8Zkzq)0Y;65wFG?DU|`u8*x!EciwL?W^8GQL}>
zn|zL32_kzWV-wzT5y$}qHT9hO0H*%B1w?lTM!P2l8WFYKn6;7=T1TNg&2`siRzn0X
zCr`|A6KUGJ?i0>pJhB2RIFsu!z}sDeIs3d&y#~!HHY4bvJ02L^1+m9U*X)OzGg*Qa
zAh%28sB~h%`7^_94F)+5eGN35NaF^Y$d@UkWC&N=-r`FelEV^7EjQyN`G+8GuyWn_
z{_q!u;6U|+fJ767x8PYTt`8GTVhM%Dd($ewy+l78pLH(5fQIT3ZN!(~z;b$Wa!svC
zta}RLZ|2CiKO7pvQXZ~#)_uyY%Qg&a1;|nnBNC4jqE&#7i~1;FQK0_Htt4FW8l)^F
z067L5y6tOcvuhc-Y#Qly>lvY3UZ=J%Kwk<*A7uwtVs+bLKfUq!`>*5tAOM_co2OGQ
z6BHN+Q$Nqvj$-Y{gQ)%^M*J(Z2c%F=mbgkAmGl$Xpb**?stR5Nb~lF@zv>MO?Vus3
z?iu4f%wbyyrs3x~EG~NEf4#hhB6Mj;U|Oy4+H#fA<5?ew_i)59K8%}~O5{=NBk@dm
z_PGCE370?Nk*2(n#y`3UqpyKf&%jK`^F(85*cs$iz99D2`jUO}o*2yz|6|Wp;s|&S
zU!qZv&n)}BF;oz<LzRLuvm%MOZ@nu>IWtQKn$+O1{I`Ay0Gka$_t9>#1fd;A;3QiZ
z;K5=Yn}H!@-9>u4UpTx!A2fP^<!&*Q!^ZeWPAZj$5ARxLcYn^_<||<MsIeGzHP`|X
zsk7pw>U%3+0-0Kv=C{pA=6;oBQ50U&_0E7*E;0U;KN5CtxPa$+*4SBs8!8^=QKBJ_
zAVA9u(Jtgjm;qv5*(6OhON*uYe`L#l8EG;Gyz;T<oK4nO5J;Usc>Mqz|B3Gjtg^Mm
zkK6ja{K(c4sc0C+|0`-KNCUX2LH+ZUuyg+xF!VcVw#0#2?O9NUo0six5$CUAie!S?
zuIcLW`nP}QIG_&Vhy=c`ZEE+w^Z#cbs2}~$AO7m?`rmr_Rr>wUAO7bL|1K8(zx`ov
zdH1Ww4-88)xblj+VGyWwy<W%2|JKqqQL-3zz5+q6)~@7BuMNed#bS*SCk~@-g<`JI
zHus_wBxMZ1ZJXDW5B_=FWCpw%Y^B$W2m<w|I*P~^%Bel>6q5{acS+ol70189JSApW
zgUAKhs^f!}kose$7R*1xGQ}h(h_NqvAo{<<<S)+#@^J)<<rmJ>4F5?5Y(Q?^7E=HI
zca!HYw9t|tpoP2oXCUL*Tz2TNvwza`Ux=0xT1Po+>-(PeHO}kU09_X`D3ALAj0E7P
z6np#k=kvlhcP&SCBLw^BKrJ8Re*ZDSY?*si^JYY;Cy;)pR$g`^Gec1ikh*-K%CvO`
z4GJ$nSq73kdq{%$?=k-dFyme==evfL&LE&!MKZP0G>xf!J*@7a8e0SSLa^9{8l9>S
zT%uJ2G^bh6k4!M|yE0RYIyF7?1DP1Km<O;z@)(8SL1{t0059ie82PS(7tMdn!3WIb
zCUl-92+%}P6(#_c5!yFz078@p^m;AKC~3|;>lz2;SoQl09ao+yG0ZWJ899K&jDlld
z0dac-kVt4-5>WXrtM}AC`Dc2egUCw-brO#M(|Bu+Ve@8a6#@E<a{1B#1O)|9QE;^k
z1Ck&};h;Pep50Ybl@DG%2EfW-so(AS0u*WkrNbpKRaM((z7>ztdb{8$zWaYZq2#Y8
z6aiqvgrM(jx_ahzjo%pn7-&9p^y~N@R=xv(nc5N_=&1YL@zamPq9GLZ7~r`47mFs<
zAfMj`4JSgtZ&~SDYd==}nKm|{!v@{wJT(4}W#<mw=O<_&QCb9;MZ%Jtk_M<``fzlk
zdY~OJ@scFcSHGFh-|4ZdlnAm#XqYvlh}=S)DYQ6q3!AJ5(52*#D5_wRRwL-kS^-TY
zBIA6A74@K0N^Xk0=;^;)G>Lr=5J^k`U{=(Rx6WzkG6J}*h_ZaU#y82G<mETz`=Xa*
zk188#K+oJLsDjLFRzc+x-vJ7yB*soQHHki2nG|Aoh|Yig2~34MYR$qR=+4N@Ajhyh
z3VDt)jM;T{P+l#8^3=<D_&8Bf=$S4Vmg+eaVTE>;Z70Y!0iX}`2Ng*8>;u&5{l$_^
z2j~xkxTEC`Jn<J8k%Sab?v%juH^%|2E*{{9pOez%dlM9#^Gz?3{=~3E;MBp|EAQN2
zDU+s3Hw6;R!f~+FZozHOkC#1`tV*J(-hQDC6vQOoo_r%pD=@-P&rml=mc|_};0mmT
zCY(|!<LI^H5So!FQl~}U@$=4S>6vaFr)-Dc?V`@tH=!iJiDS{;i;YE}7Iui;e-3fx
z0j*#KysCa&=!s;Zb2$*-ncwsCYyH0knX|^Hneckao4Yq~Q<m{wP&|)nOy>aFc2+9?
z?W3|JaU<x>qBP<K;4!fI5va)c0Yw-ltR}dQcAW}E>fdmI<>2y`BAFi)>G!~T7$lEv
z2S#++dir4s)N0SEeBNy<6!m~D1NT&}7FFny)GXs)@GJD=<Wt~oZegV@2u#2Mpm>K6
zsK(Zzm1vo~GiYt$OOcl#%>~vXwlPh9lLd-MBXY`yBY;V!&QhscVlw)I?Ic;_C+Jrs
zPef}hvLFvV08LJORvdOkY`g!dCTJ-0))~yEi+Hs-_felL0nF`PgNZxk6|{v=Xv5#W
zb-#ETj`kD47o)y`2E<hL$}(sxT6iTJu>{=0np70a^G5!s<Hg;XLAhJMe+!^RLLdq7
zJ)~r~`k(FCwEsm}Aw5-+H^-U|h*o?0Y6|0%E>g|;pcHwuT9YY2BglUO?22CZ)hGbv
zLxKw^cGFB^z^LBY)&d^5oCC!!p?E_}w}~v>si~EwQf=WTh4SR<gfMeWC!8(nNNAcm
zBmcPl6TFQ&uq!iuT&Us5*fXHT(|U(m5%~b^9g=^dg;?mlX0YcL%;!A|;$v7LM>@GT
z62%2b4iZ+K2fv6lO9Z&8jid4M695Da6f6U*M0#)OKs0<$$ZTX%VO!I|6u=(i>QBs)
zl*vkdV47xCVyV+C6o(21!zjl?f~X}%7{4ua+_^>27VpspeW`QW)DyX3w9PD&gi|VU
zE$RjMg&*Jrol@Qj9610j80xSE?InG6JI1}3Y#$qZf%_Y!=g^&8<%3H4{Kn9*ul|WV
zz$}tOtpA~5ctMOJienP=1z>A|Fo{LIVuRU(XN8cuE7Ri3+BpPKh6iI@7?JEl;g+Hh
zRoZBL996u~GDkkMd`V26>!W&n;Sif|_cX5n4o}^z77OHi_x%B`FyN6$Iw2N7)sEw{
z?7lb!wjT8H6CkwH{DPK6tk6=1unqY?#?q1wL=4qa_emaE?t)(5uVe2<*+$fu1wjj;
z8eN?6s2?D{l857d0x)1_T!Ezvz95>5z?!|y&7w+<EjQ(0PEQ)u2Q9z@&_|%d&!`jh
zr9N)rd5%^e$q*lxnCga_A*&iXd|$i)6$hTTD?f8d^NS797gcu{bN!=>*=zGpQNJU7
zDHPbjkr|^)lzYO{M&dHc>gJIsV3o+ju&h?`bu>Ez;D(D2zk#MYv4EuHB(Q_%sl73+
zstOfMg*l@oB7e7NjBLB`^KwAgbKh}**rSHhb|PDiU1EIx8AqhLYNPgpkND<kZ+`eG
zxHVW|{|-CeI05M1^Nv0cOT_1dpS`3$$f%MVq6Tn7b%WCh3^YjB)b!bJn|DLR={1SB
zthd)2Quvcc6e&k=*y|sl8w!$K$lVv%iefkRUIN>;$;pQ4auC(DndscfvRUb|)bET+
zUjl>=Zi7j64ecEe2fWXQs2$pln|4xkt6Vv3`vA*ilc30YGb4j6HvZ)2r6dLX6a$nS
zs-r)JV(FnI-lYvcz|QUm@cTr?(KKir!Hu{8JG@coM5*}D^ug{oF*p#suK}_=G{_Ax
z#k?CA{&};a(>~P<E))jt0ydHkR~7CR$9oQfgBJ4S2a$Lzr871l6#xPIlX(BEsv@xm
z@%IzJ%1G4$QYv)J1hHEVe%BQ_UU4+pAGC8{mSIGNrjoz*+sQ^=3xuM6pkGDq_U?M^
z=k76L=*mq)+*v(h3R5mXvbRs@3OV3hnoPOm8Qt+V2?q#q%vk|Iis^whm;KcW;5;Kw
zytdKlprgC#LofueO^iDL9F>gc%$Mb?R^S!>)tY$%Ftr3yQWnP;fitOTTOVC_13iV)
zjZBgN_r=YJB_1K(LzJm;6G$o{(2&9AcOTlY2?62QaZX*cvM!m)4H3-^bZugiM|kLZ
zo^9zq)7xaKF`rol-I+ZwGS%c2;~)omK;rpIH(q@c8Zt<5KyD^Psp?yIAkboA*`k{J
zR5ZkJBG)uQ+PM)hl3soREjyQJ<k5n0P?ije>q5H`s!T89)^xc9`^kwH5<a}V9tYGP
zXgB->0Lf<@Teh`H<{9vWpLF}%AHqpuf3VS;aRY2oFt*0v-9_C4^<?H*XovaZ&LTy|
z%-@+W2dph|w6C#!bi?HJ!Pz1#P7Z>LYTLova<~O3v!-ymI7TOP=C%1rTVO{ra6m6G
z#xcdQPJl>~3WH;5V~Q*J(;LQ|X-_QBB<naF3m>ZuL2DHFRtV=BfAMMiiWyvk;5W4Z
zn*WpdcZ_h+9$p4}^qRbExLLBP7T-eO2$A!(8FByqN%$x4TSI4mfCAPBxGZu~AlM7;
z<prJ0t^&bmK8yc8Jwf5-uZB(eFc#St22Oq0Q*bJDX1f^o9-u_AmCme`$%KMda`fwa
zMYU#_mJa6ebQ%!q`-mA4I~eWrLrv-O4gem;GT+J{6uaIz=znVTLX<Bbk(}C&#u+rt
zosBWJ`D5@sh~2AuyooUyS$HI+;thK6cZ`!O)idx>7L`*QH;WL4cav^w^C`N0pud7Z
zK13ClZN)@<#;|M!k;~8l<iI}GMm9YlL#dOBhTqF1ub6JVxX1h~&KF8qx^Kr*IWGg&
zIP)-2RKJV#t?Y;-ivE4yXDXEk2BjeB3($e`r#kO0=+g+epF&e&8n{zr4K<slnGATO
zmf@c>>nlk3VnHUBjR)BnS&#gH&!I*MApN}k;y)1c`o$?oeY-!z<5-Y@j()3*<eUx9
zdXnQ4rC`qcZI}}o0_Fh!b^;t8eo4Y(3VI%^l+<s^PP59AedFPNBEiO*P?S=X<b*1x
zhXV%lg#>ApZ0)L(N1r>-(M7zDLb<=P7%h9IRG<rB%xI<5AEC}zUR+(qiK2wxTt>cN
zT6*@e&{w~jw37wfIhN7|^Si%y^FTM|6%DNY7BD7XiiipM<`uu0hO{XsMG8I2_az(V
zN0}=ZtSlew+cUy(E;TkCMyWq;onT+y?sfRXXqBZx>CK2DPeaS7!Hq+?${PP^;<fp@
zXg%-!B33+xA!Sm;@nlVXUNHjeCr=vy>Tj4Y_X8gV#1w-n`41!eb1+o=X<q;;-jL{r
zv~EyfOsnL$8I10-y=jaZ`sn*3W^xrhj<CM#Mij5zU0v_Nn%W*rBOdSV@d3812=px8
zbm^<gttL}x_DBU*UMhNe9L<O#x~Du|sDq?dm|iU)yHySLaM@Ro2s0j)Q<tj25PH9`
zm0)yOjHKyUR+J(WGS+c>>`*3i&WyE&BDcm$-ai`o9Im6N^{@2b6+hte=|SD+qf*_6
zSk9o8uFs%t97#dn_G^1G3IArR`OmX6{%a*IX~#3Q*DuQw5W*j{eX9W?L{h1gQQMCA
z-8X2cRhz63<Hzuu7_&+>nBp}?67|jP=H*2^`OW$kU&*uKEwm)3vIg(lCN%e{kP1Gj
z2BoMG!*leF{%{0Nl6fy9^z5_a&UBV#p{_*m3l|$aBR(mvNW>WFu1RdFWLFiRL8=8L
z>YISN44-$T>9pX~#pqZAzx&Xq-79*5wJ&4vNw0S0DFG<~XlJIcFPT){nCj%us}QG8
zHiA(TTf^YyVM|=6$aV0*$aliI1Hsi7bY|91hyCizRKPo}>v)=m1{!)#OU1Duvp&gU
z0fFq6<T>?RcCEV#dutv3Q4!h}4^+c}=$I+`evcQwmwVpirPru-i3umHV7^=ZcM|yk
z8oQ=+x7$H)RVXvBT{O_w_nx;umY0t|hjU(w{0u7ZSaJKbz@y+)tCEb;p`_J)rL7Q9
z;n@|?)iGV3K3nk@nbn68mRoJS+|Z~&t~98w0cufgdcij-Ze{YTUz>yaXthZ;4~*@*
z-?3a5N?pkWvS%0vpncXqRU(W&DGVkK3YMm&kSkk{GOrCVJ*QKA&k-E|^zpP9E+);k
zY!0D|ryJRGt*UfE?8B$~@$XO}H&WDH(y+c)TCM4fi8-;nBX&LAeMj2U(mTeU6WN<A
z^E4{ofvP~p0=?Xc+nlj>-!|f<PUJChq!I>+?_*VR+OEqU<X5cgKhn;m)qe0<j3gD8
zKP-g)G>ZP^4cJeI)O+hmM=SRdUFqm~T&&rL4dsxHq%*a%LrPA++}?P2xmz<*?$PI~
zY~AE0ij>K5DT9_atUj|osWdy`X<A<bVQomgx~F-hcm|UeS>`s`+bAEx;1qhqcaBdt
z%$|T2Eel?`wTHId`?eTaMPI0Bf;#M32FeB0VP8~4?q(~1p770GQzxt8qZ4Fo$`(fk
zF7x-IuEP^|DEvoCtsLZ>HqSx&B2o*#IZ(35Wmh@(_adlt(`Hp#&kIAvLuDH3RUSa|
zO4)ptCBkeQJY9D1x{H(!ahG&{|BW)g`a*Z`U#2Bi*TD*<Wy@5|7;tSM6Fid|`)dCg
zyc1rrOs-}9#K6P1dM{AIiGpHwRF~K+v?FlX$68|&bGt2z#9x-})EooaSb(-fhU}_r
zvrDT?tGCGrPmE<4+LgI^Nszb-lGb|S_<jbQl1l`s641u=Hmk9u2#(@2aJAAjb*Ij!
zn9I1{{JK0viOhmcdr8`~P($wZE5&;)rpUIL$ipsw-NrJE4oozZ75gDwy{u%sSDQ7*
zeV!&4dGG!ssdNZ@M?Qo9P7z_%h;$-#S9qIqlVf3v$kUSW&6M%8v53qkLDs;SVT4$6
z(@Oj|&&|pccNr0%t>)Wwo*gs2c$-na`35j%0sV-*4w_rXi_htSS-_m@USaSzxl=0*
zrV1dlG8Qt}?1|cl$Kv(WHv-Q~pEor}HqDJ?RWVa^OY7(5>RTQ~9H&5Z=4uUlGNNST
zdQl8X(=6NPe2l7&<DD7SMw=+M5y;+{{-T~hFy9xCdPFFm&+3u1qj|JBXwH3Ii{igq
z8YvQy+*;&-qk;{0aUW-1v3OJ_JRhF=m>SjE*1NgI99LcA;Js*9c$7u08H1dN$D0qs
z5}${faz7)9S#foE^TAUyH77^(Z7R;+*QARZv$$gYIPQdfsjJ~Qe;1s;?Y?Z6I!|Gu
z)K|dYn0~wfZFmlN{;+pK^Qk;UY6Jre`sYI4h9vmGkq2;|*ddveirb5U37Iw-K2lgH
z=vkK>#W%KOeFj73#T*wL`Dukc)9@W7i;YTAcwv{MTrAzGIFUpY@Q?jKk&_mpL{+Ax
z#ei?O^o%2$E>p^9Q3Ib)<#9Q!)?ytTzj|FlvRzG=oLUzYxsw@a$RxYJwE+c?rrD@A
z<36<LzkQUGkkxk<wIn71C7;Ctx<vzgqjG64w5qyX2j4CweGY^t<UvmkU4Dh9EH=Cm
z)Zgz!plLT>5Yk=!63_B2013G+e4pDqa{!e{#W)?ofCWDW@fj_#oY*6}u`etS?lIGA
zwj85}ieYT_jA;~Ugc_9EzA%@6==yPd{n4ntnCt5|$(gXEuLotGbLjT{ZVpF=;+ZgY
zFQoD)9~!beQl)sy9K21-&qlHaNv}DgdR==`ZOh13p4BH#WP+lCW0qVg1hB6H4#%J4
z<`M+xQnN}pOlPh!v6R<#2@~t0j>c`}4I4&PWfY8_m*d&PB+fWp($}6RDcL?vj^tKi
z=r^_YM(^bUF*Dhn<ITe<b(>C)go=s~pskLW(gc~M&wXV^EL`HgL3+$f5Sd5kC?r_#
ze_Gs4@(J4i`o=~e05QM6Ekq|Ipj4twC86yhGK;dWl4YY>p<KkMTp@&@ywErs+hIj*
zWn*RNh(Dtddxq4nqgPNNNcx2s*^7BSzhu#eH6!TUcuX_jq(*=$5A%AAY`=W=Ea`8G
zInN8=!DA*i;Ai3(;b^VZ7<=*^=ON7M(#y7mS+@Dp+7v`FNet!%EU;=?S>aWka$;F@
zWW@piCNe{y4nBny0^ozE{=gu9U}B%uk^V^85f?RtHK~}~6#1$bee8HYfpHYVQ=vRy
ze&|joe?3SsOZ7BK2>d?H;wD+tL@8Wal3iu1y<AXiv`WHfID*gFW`uI-R>BEMGe_`&
z(J;zKaD9G-BAvI%9z<egMFu#zU+G{A%+5Be*zj!6o2SS&JQJUb>&q2}YnEY`kY%y$
zPhJtHI26JVo)G!poHlg6Y9WL7+H4xA8}@m8KZ@RsvP*j(d2Yjy#q}P=H-TUi(QFhO
z?Oz(+$wvS$Fjs1nM{FR1z0s_#VqwT}dVorzXXVRtjZPC>4^6JUNqHf~XIB2*Ns01m
ze37poHimmV87sE9wOy^V5~KAFr(xS;O`@I{dpZgluL+*FBvq<p4E5ZCEM7wB{wUpD
zVph?kz{<}hH`7)0JL~F^>tY9n?o@R&-LH7&u&VO&XF!LdUC6^vYMws*R7FH=9?i*>
z)5qzh%qd^`k@QmtFz67~aRT%q(rHTXiM^|wJB&tl#s&crGQp25)DY6zliVO0j*z~G
zTth426oZH}8&|-<9gUFR0@p}{!7DJk3uHF~-cz0qDzf6kYCoQ<mnqCrZu51Gddc^s
zJqvJEjJGXU`}W&Cq~RqjRob}dVG$d!0<eOU*Gqf*%D!m>F=fvetP$O%wz{tH_1f^5
zBJ`9`(A>JnnqB+T|HIx}MpfB$QJ{b{DAFz69nuKW-QC?F0@96y(%m5`QWAnR(w&Nw
z64D~l(%k*%8~ToMf87}Oj`96r;PJ4}d3LP5V$Qh+-JkMM<wonoW%pz8psw82o}WVy
z)tl&hRbOKmHvreacPtqWvhgMl<W0q~r4-0%d)_0<BnmmmUp&XnDLPw)xvNfrA|!7v
zco&NzljA(OLX0*j^e&xm!i{550>U~5sX&s-fNxDuu??ADzZ6yX^|NdlOnC{TiXvkP
z2EjKqSZQw}6BRg79@d-o+atiqRogVmp>2H-?;ow90q}GDyE^@4yqfh+g9`X}3Y>J`
z=w4kbf74KQzS%nnjuz#}rEF2{=Ob{{cC{~JaW89F`kPUe2;q$2r>$tl#o&?O5$QYa
z>9h~fW@?b>GLAFgFEBbj*zd-E><&^0Z2D0fUuw?cpFQE0z%hE02FrnHMkwyARSKBi
z4#>$qK((#LfZc=V&Q!h@TviH;il-H_?eHkTNubn1`SwirPxRmJz=mZoj?jho#FZn{
zU_N##Q0OM<t>P?iF2dyqrcgRvn$YvEm*sycm{oS>e1FKjx)YU2!oS3Vz-i!Va;MgS
z;*ihnxLq(Bb08!($7J~UMDSS*a&?Fv6Dj(41F^^hcXE<s`Jwj->Q~p&K_zf+>a_F_
zIeaH&=_p!M=N2d=Oaw0Xx>T2Bc4vfN);;I{l?x84qy|WLxw#XXpk0a%smSGt!6{#w
z@9(&4sQEy8+Wp>wy8kDX9VHMhX6RJ9d|Q%{l^E%d>Uh-%-AOIfFVSI@_~+XLZh$%!
zk9PSWMLl!9bs^0B@Jek)eXxRN#5Y1r``4%|ap?zNm8g|vyvyGszX<6xC+yNj&Sbo<
zn?Tl-t3rN)A-Kbiy2nF-rJD9q2-6;4UK=$Nr{r`fSQa(T1Rc|ZF5Av77)EvCfnaub
z{)mCcOf?l<^6G2692|3232oXVCcTVI_yKPa#X6V0zD>)z))-?GP?W4i2`W#E)Hcqn
zDHvxU-7fsdPG)bAO1nWleCt~Ja)Z%q{gX{WbUwaPa_fz&W=*(_EzW&_&6gcE1Hybu
zXu;bLmI7~0Hmx~${gpe1X6ewFDwn{GibV$?OI}Lkz5K2%q%%d7QYYtUd6<ZCPphC`
z8PfoW%*{Fs$k6@h-|_!ZN-#FX@QRK&om`M17G4Z79LY8TI|AOodX}E@L$t)`2KE=7
zO40CecsRw_CFUZkKsr=-bA~}`rUp6Q?ea%tN_E}_f-PRUVgR6ijXm$MMP57Kv^1!%
z7nrrzSoFpuqB<4vv2f3IZwg&E6)C9q<g*>wRfa;PB_-25^DXAjwazBgX|9Rlwoc!t
z4A8dN%lcGa)7Nd2g-ILk^}?{fD=JR7U2EL#k4z0GXIc;$L_^+!_wEa?W@)@NBN+;F
zI5&aMunBJ8Dq6Jzp-^k0InU_o+rk=g^otb;ls<4R8*G_Xs+E>EpupcKu`oRA$G3lc
z8@U^4#V#L9wG&>p6OQmvz~mJME8n;Yh#wJ<DM!|n<jbFR_N2K|?L)vZ(OQ)5Sp8a`
z>Z@HE5U*T+zjrIz7o-QMZoRIA_vQ!^YctjnS}DyiDJ;^fWNx)^vFMBN+<heBJc&Tt
zohH9!{1h)GcV{>9Eka5RW2hcvzwEOlD>12^x=R>HZ}H1;c8gY`h69>oN{H0X?0fvo
z>?EETI0b~ePm2)xWrGvfs$^-3^%&kw<0D@fpcx3(6Ub&T;wqsW5y)|Bm+hEpITvqh
zV)e_+F);C=G(^5HSHmw+ao-xgQy`Ty%g7KN*lm|hxnC2*n@#x+IT4}8O@{eNFYkT*
zwma_ze3>NgNg+~Qvo)?Jyube{ZTA(L68Z~`Le!WQNYj$<js=n}M1B$dcsF9PvT|$4
zs3}sELR&X=?ue`v1?5Ny4t@{)O&~vqE>W3sa)RlTcT28dYiqL^4CziYVZEWpcf&C!
zZHsXj=%)=zvrk}^d*(2-6N?r8mVBBo(&5P@ym;pYP9}@;%`7Vc;#VHb)`6CkGMzpO
z;@qWzNSWe&2jj-#z3r$#DbKDI)=a@n!9s>%GiJr%O1f1}%y2~2L9y;h8wGAb=OxiB
zjaf$;La_qf1vV5qapsjc*2cCB?}!s|+b=tMO||qq9YCY+mSS4|POpsJs;>LYQLrMx
z>@zcI(e)&*GNWo&r`PWgmqxUZI5&|SHTsm&X5D4DjN2qUUxioPA9bk)*<IEHph_s?
z)Lw^ApUSM=dK71UXKn!yLH4Vx9C^MarEmbR_-ENj0RDW>SevTgS$Hxcv8@Hnbh0Rr
zX&7oV7W#_KgB{7#P5|<W)=u{sM(c#mYelSQD{wG1KuHnpV}^6NrMRE_HV;%|-9#Ic
zx;Ncyf}JF1-ldI=@-&v_I*X4Wl_l)Px%`!+p2=uF`Wt6Fni+IG<fs9PT8HB#6kmrH
zdU|(CNgr*FuSJX_N_Sr&T~H7=-R&a?w?@xQbFO_=z6Wb^&~rxt{$mYGA89|#>n197
zwA7Uu+s2!%P|G5n(OUeNvXD$}M*R6l;qa&DOHU1G+B9Zo)1jhFo-<tQ4nvXigVus~
zVwE+)wH^!Tqxl{nERg>!IKD@&#7=NTap!5PxzzB=AX8NoZef}@lVmXiZdR++xst@s
z`SIq4%~kV0S}S|J7iwT>XlGyO$N@&4M}AF(Y?ru8HRI3_U9cH5E1E#dQEa-*+~hd_
zFMO2cApj&}rM`OziiP}eo+1Yn*s<y-P2@pkjwSVr!}n(>!zF~wsDaoO$PdCR(u}<7
znOsX{;nWL<QhmR?Fe2*lhIi*$sZfOv9d3t7MnxA-r)KA_mlVDOteWxLJS+67U7lyT
z5(Gx>?IvID5>doq@jq#>lKguB>hTre(z5smk>tS7rNUgu?masz7_0}Gd!vrv2sT@N
zRmvX_cTFl_3nBFb!{4|Dm1ltuLm1;ZjNv{h=f#E2eG-XolG+1T0U)g3nDC-$=z_`{
zVAyXpW!N^#0XhdUtS!p;U>Lnbwx#~Wwz!2gk=dkL*lBCgMTADxKP2o!<hnz3xx<(B
zdP8775v<D$wfd-nZa{h^_3|czi{_u_Mx|U*mwQOA-;@rP?qP`DIgudWKacba;UJs}
z3OE}^g3|v*S`-1Kh5q>g-~TWEKy)!j(Cz0V<RM||SSD*CV%%E@LEKUCVG4r1g$Tit
zTc5a(%KzMsdeNe}rd?t8Ig20oovXCIlOiGvd_prGiE?_it%%QCY4s>0-gT=H{?fSl
z=fng}Q-EpwY7n^|T^LtZ9$M>#-=6}3JXcZNMne24ZiH9Ejh1KH7z4K?+&^;dbT2Xg
zr8RC76==^#{KeYwKa-C2QdIPvE^O57e+uE$;{>3vXMS1E62wCV2@(5(q^l*J;hM(m
z!S6_dBX1B?7zI4b&VxrK-vg-GN|1pBw6$IEoV*}-0)>m}(g{dhcjW#@g%XH`<~I13
zvK(|G0QQdEMkR9*M#&hC6x$Pg2)?dGJS3j`9P4Bq)KFrM0Fw^F#-<2=c-(wD)$?+{
zuL=M_GG<LeZd(Bj)Uf&5e|8zj7X*XNf`(}4inZwn9|O2YaY;bJxI2v4i4TIY0QL*V
zNIOWUKehuEABkf{P?3bxh*L2=fzNt<klH@}35bdmE$wjbxKsZ#t{nj3mS-o^(2N(u
z10g;_tI^Thf8$q*_5f(4w-k-_tqH&^jQj^}{L$GP1Drw9yQe@vm_A68tAuADN?qr^
zW+tiV(O*G3m<JnqFpc#sJ4$+R*>rb(rZ%>#a}u*^N?d4!f8!<Ehy(n1{j#LTprazp
zGd|C~C22ChB!oD=ZnK6gsDzqsodcv%o`6Rg&2RjxLgoTc1Glbgv|K@jY!r|D3Q#!q
zQ*WzImUi6{bO1Q<2(za+0*Av~WI(?OLh*p-$^k^ak_@|70M4HKvL5K+x>T0Bc>>T2
zM2Z9Q&oh9j)*Yvl)%X`2r4(ia`V*3{DmD}nZ-8iihy_VbgJMIf+%>45gPLI%O{&2G
zC<(RRTwgq$v_uEfwFGKgT%hz>Z1y`?CPN}mnSg%VNtl`VHjkRecdaDR@3bnXKfVHV
z(E>=-K{XyVCz)oz8*}c#)mA8}|Lpx0kO_E6r_)XWWAGWHgy{+jq^~T~Hu=1N6ch)1
z0D564yYnHw<FlpVHP^R5WEv7*goP(;fq*tC6xKs54GQM8Z+8L9Db|jKMOF~73T}V~
zsFC+8FaA*D5k~PhRgBJdLNr}O4|Ns5a_Pevo(ay@$2I${0K3PM^~s-<G!VWi0)|I?
zIL$`soY=m+8V?Y!0l0?Po}&I8?`O|j`fK1;5(4tdKz5lAksGi&pAzl?MUZR2p*uhM
zrr)G;Skdsg(=A;X0%xgR0U){fL_C3QVf^`z@39EDiR*C`;xj$)@(kIPuRBq7bUFB?
zj4ve#&3Z`$+~~Ie{y#KYXB=ohK@!^#83GiF&%yYoLl}!7&rEdd0u+EdyPtnb`1G)}
zKds=Q5=-qcCjk$C7O0Hhot$wQ=ypE`G`+)^od~KJa_f?8_vZ)KzY$eI8-;r<L}%S;
zM7Hmmz|p0FL-21`v41-%6(Y~_<~?Bdr3&;Q53EyC;79|dYKRq){k$8T4kby-TWbgW
z5`->xK=E@ld;4P**;(DQjFB9%gBK#^1R;pG2Xpj5&1XP56}mg4c(TRNA?TedLZ-za
zT~NEzD#X^B^_^VpE(P}GE5I5J=d^+vX6Fn14;dqm+y#24c8v1<F6^>u(TS~X(7?C*
zPCK65p4ggc!8BGGSLQID*Rg4ga^wOBWjP{w(y4oULHI-Z#VCWF61!|2z<MO_$UOqZ
zY`Kzc`6a>&Ku}Tj03nYKz~|LQo9OnO67qv1W-agp@WDO<@&*ZK0tCjn<XvITX#nd>
zW%QOL$NzlG($B5H;^a}C)F@!1rs%UhhlhA~{GnIa4f75xzXOhXumA>YXlj~zJRe?1
zm5@ph(DAy`(up?#qmr`=oh$AcNzWONN4<JDve4%bs-bxjU{1^`cge2zkno&81N0D}
z9-9l`y5Rpr?C@F^cLTUs^z9V!qh-LeOgbo|c3z9Fefr?}I}0>OBOE<Y`T3(HVf)`m
z<)8qGI+`yt7Xz=C3d6(>rWv<4{0+O1wtU}nxdn?vMM|<4I(m60BuAzMJk_ZXCV+s&
z^)@T;+L>P~u(lDf7<(=w%vtN&Vm)u%O`zJfOf^gaa*}cP^W&8x=4euM;4tPPrMcv8
zqd}O{C@@hBQ@`8)Wq5QV4L&vEB+(&E$i#GE<9;<uepgicLa8iiXAkFv82~PlTC4(J
z_eaB`NcXWWpijkTNW^42*3Z_kunVQ=CWIt@1BA*XVa*JyV_RD*+dHbkcbg;-%~s%1
zS3;6#WU$P2fULO6Crpc%b@_jV`9Ezt0@iq^F1bQxf@maGgh>TDYnByLG^e9t`k*%B
ztm%VHYltBUpbyHZ#6wm&5?@XIXelCII~oBrOldq&Jm;U>*^vF1{6LkqbF#Qe6U%cM
zYdIxQd|6C^+i+zi2iz$@s*9+t89+7kci8m)WI}R7kiZa_3f7=|q9)*0i&gc%KHJa?
zyO_1k?U@xLQJiggnw(xvYAH!C7yuA>tw3@0uBzQuu;^QfU4D?Yy#ZAq*NDqV{|zAM
zzF<IQxb>ayOO*PShV6qBg>#UER0aqtt4?Io4U0qE6gQ}tOb0mH0Ids^OsSvAmu3Fe
z5@VbkT=;)~9e{0$QPuj9tZ1G1$nlFS)#n|!_5d5y@6*77r@WLz7P$jq0zvKVRxRWD
z_-4do2yYYZkzsENM73v5e_)*LFn9xS$Ln0w`5-en8fvzEAJ?_t?=sLxf%T1Kd>tuD
z9c$1t+6?-rl<p`=8%-j5ynen9c}u{iZFaQ>)I9dKA^fOyLiO$?L34tMZTDKTZ!mff
z4@(*uQ}j2m1QhiwxvK&Cr539qfVGDHf?JAJKMcswx|?m_VtttZw{-w82~@yyjvlWC
z!83Xg03>Bs11MXA#2$m-Ca6Q718Gy<lDOh>A7=gHUIp|tiPANQo|{^cAl6TdzX+f=
zp$Tv2K7%5KW!Y<7I-sVzM5$jd%IXSk4+|KEddJ-z;5y*u)b)R#R8)5f{rzcQW&+>V
z$qig)%9MelB;YI(a@gZPdb=F9FkUSSVTE_h#I7mANjQN*{5q=JcAQF`>b<cKI(6w~
zqljtoiE{UV>Ntd*N(b%`S59T)0k;?|r!=7@PB37lIjmuAfwiW!lrjTlY08H>0N94O
zrt~;sjf)bIqN3})u3j(6zn&kcB+>KzG?SaK@Woj!a?=W&DkrPd3SU9#&Kb-Jwjxs3
zI-puV?}^Q;$+E;<Qxg34e#w{pUf6ByNSfzRWsI290{Nm~oRqbE=Y-J57E(yDWf}$i
z^kA6!qsTtybP3|ld7eRVk6t^h5AgBJ&&-z-SP700FwW=R4c?668kJ$K0pMcO#kL#6
zf;NDj-2tgQPqY94Z6feo1nwii!wBj=fcgyBylel>Jslw*FL~%u!+wW*Muv;lDidoH
z0si@w|GqGd$rzBU3tVSLvj&N-CP3C#8=ldy0!G{u9IW_qP&-x{jDp|y*YO<a><1kj
zn0!#RKL8W80#4kBmDQBP(6el=P_z{siqM&UczFi`z1mPJ#RW3BON-kHo~t#F=$g23
zdlOSwM8CwrTg|(@;tuK-25zO?VP=y0&WG{`G8eB#OHyh%%R>_^fiLR*^9uSl+~~7m
z)UP?+Iopaq@tJ1)3H<4R32_2M;Rnvl&>TuNJdkC;*y`b-D6*?<+pqRQ>6Jo{?SF58
z<_2hlKI~EDbA{*>p@?@#B2X>a5&^f5N*}Gq2e?10?`aWqoCZJV>=cS=6k<yC-()xf
za&(;(`N9GNp9+HZe5FR_>{i;pF!^ud!0KEW*jvvXBI6url}^?HpKk;pbz}BkLo;qq
z#e{3mPS#8#7~cXgl2Bs;aDr#LX$eOwHT8P?2jr9-N&p^w8J%l*d`T-E6!sAib%twa
zJPA7h7F<24vXV1&QeCk%qj7hK38^qfoNyX;gb|!Reay|l#esUbd9_5ApXJN#3;_ZZ
zbm_b}wlilI?{Po)rUSb<?Jpm7!=kzB5I|V&fern|XT_RiYwdo`QNb5)4xW-`{y`oD
zvCk1XxWewaRg=8>^M8QM0*KcWD^tvW^I?AGNJ2oO+h@GT<>ha?3i{O>3z$IVCcPNV
zU&)kTaVhxf7?k4;NYNi&K#=^+EqKZ~{g1<)ZA2Rj`h+}6y?B2?5br~`2^D=L#CErd
z=m><pMfB>CB?^v&NwTIE!%LdXzl2@g6gvF)1g$JkP~7l?#7-V;SH7G7r*lhdTA(sR
z>*-2yDXvTM8)w>_-<{*1ACxBpo|n>gF6Gb1g+-wTuRrbl?Q;D^S;UbA5gxVTe8R7P
z{+$sZ4+7JZu9Ef_PVC>8DS>f_EVR}C&*V_Yfmy>w#s2ThD%4=o;<+7vMYMmTF~O+N
zWQv(W7=S-M_&G?gGba=~{;3H7CYdsCNr7JBpYQQk@Ff}#jJD)t9m~Jw>(~FtfY(t%
z|0F4ciKkBhmR2UC{@>KYUtbAmn{d5g|2Gr!*UOSvoB9^!Ki{4gjRMFlj3G!EPUbL5
zhCjb}h9qFJzyhzuyjHKl5QRJOSHM2$4|>JC{A@xMjo#6Owb=uI>3>`vN&K<QI(l~E
zCvihmnPS{-Z2m#fQIcRu{9`<5F?n22%0a0~hFm%C!#(7KFNZomKf{{{*4XBxj%nR9
zoN9?^blKVS?~V7*#7TlqRd|w<?!Qg%&tFaWz?%Pm_7Av~mOv`;y6bx`xQ_0B&S3d%
zae=zApU>_%X(LF4WxzvC86X-<wI8t575rk-zbzl&^Z;p?+txmR9iP7k8FvRez|mtM
z`}Lm(A*d2a2JH}yp8P3ABa`<HY~cCb1VPjPY$ZeSj=YCINkijT@IWUd2c#Wx9y5au
zGoU+y)hsMTh5=1F2url|9p8XBmNWSb)FTJl0WNkC0MmgdRP|`F4FI~Y!M)fF5ja42
zMin)_Usm@10=NsjK<=X`MSTV8EVNae$+ut{PFf5cjwCXa^P(UBCrUR7rI#R@KkK-K
z(E_wB(EUNxs%Q|z(qr9_3`&jW@Nanm`kao}y$?9@dgor(!&9r|h*mFFfiV67Q5JxQ
z4*lSDm;vk}EK2~`Z`44nh%kw~oNR}*JV2Ty0X6Y0AaE4S$kJse9YC>te+vNSsso8F
zP&$%;b=MqNPizAWls*6!ovft$TbL%T3rY)sw=lCP->Er_=KV}xp(dkrq$8uw^Vht~
z!0R2}q)wA)23zIdh5H#M2TinNZC=5Hz^EPpPX~s09g>g%0V25m*myU<lrSjdrLh7P
zmlM871w+5%5ACNbXm>!*o$^pTJsTj>DPj5ml<Xki6Dq|4@2tQCq8l05OOiqhN0;Rc
zR}l@eJ_Lpv0MZ6l<n$d9nFSsgc?%%jd;r{Y1b~4`o2RageSX~YR2HW5U+A;iqEw~U
zm1AsHV|k`F-&5q<Hb^rmxu(`6_bkCpJO5jgq$730j0Lt|(FfGVC)G8mrUedzjQU9_
zj_vIwM(uvzeb<N6T}QLhT{AZ}V!wNg4*PsQ^cS%j-QcS7GG4z$#&<L>es&rmEo26{
zir9)74GS(qMIhS>pdaIEa*52yK#)jmt_J}DM@1AC`VkZckIqLNd|w}dR@DS#W2vU|
zEskQpq4PakKpvQB0@?tDli0FSbzUIl^bI)kvngGG6J`TkH#!<Y9vo*QP{GD@#M;D~
zg~anz+3(F^>a+o<Vmr9fjW1X5v6ZpV`-*`$N+iOdY}XFdLqunw9Rf3MDa-eL%Dr5r
zt9v)ep2^dg7YK(+|MSN<h^gbXZ<EeA;HsD$!2x*}@hB5&tsSBB5|JxgEzSq7P{yF5
z`w6#sP0NH7@i*Z5&Dta;S9y)tLsSgBN&r)rG5+3OSJ1Y`@`z%LPG}kBX#~TZ#mK?}
z$;6|h&Z6u?z=(?yzd8TdUd+Wl1ej1tJaZPeF&Q}Qs4qZrjs)$x?BDP6G7#3fpv9`P
z!vckUbKjE9htTqTBG%un>o%_inzY-G*)ji=`$v2<9g&LNE>+$i%G^N=2@DNeCLaqM
zSOR7!opn^9$n;fDr^WNJMC^C(kInR6*7cA4zUrHMQ>-IBuXs8IKfpX?o%W!i(Qw+x
z#}P4NvcC}kQCE4+RDZ4%^dk=i_SPt^#)x{*0B)YPBMz%ofg%D8dVvok89)qwtaAbG
zsKCaH-1qD0JRF)kz_C8h{_aaxQR=tsvv^^&+6cT~9IMftw@??F%*ys_?_xMJQ^4wo
z4O4_$0<FuIKw5_GV0d?kUd6>c&d$AP=&v;a3v(NVFRYeAr&iNM48iDpGWCEFbNv&=
zk`M?a)EJ<IK?=%@5y)(oB+nv*d2niCJY2#EI29+lcXkF)#kq<q7YC3ilh@>+9NApw
zWx!$PVz;-^G*(46a0*%3>`jg53>Oc_G1PaUUKCF=qE0N<3p;%cO8;Gmr(hwlU#*vR
ztfI1eFpVS+!*m0|S9R)j4?oa&%xj1y)JkW);NI~|hpnv_=TOBKe6q;*y5ZwYOG>g{
zh3lgdk&)u6$Lap<@%kd+ryr{Pai67grXr3(y|@a<t~LSvW?&GQ&o5jUSFaWPV)<4g
zoT@0INiG{k;1uF}JS=7{Qysa2pJ@L)K6z4LJ%~6)<bt;D=4rFQ(peV8bHORmx)qHi
z-k-g$Qzf(t76yby-Qwtd{xPpRhC1o=HXuK}+RA`_oRypVosKA~eDEf^I(3%YmYzW!
zN%$Z>cGzj#U>zaTmWJc-;`5I-$OQ%g%f$+Qdw+}c;`wyLF6eKBb=SQ8Ah~L_C=ZF$
zO+`+m3k^~Pw*3^{)V_qkmUre6#w4ilfna|~#ndYC`LhfFT$QN2H05*{Xfsgt{rGZ6
z`oRHaWnIeqCjI+_notM4th3=wg$WkLhGTH~d>HdTsVxy&V`wRTt55Wi2`s-lqe%yU
z5#6X6lZs&=d7?)?``h@y!`AP9Y=k|Rbs+sP2T=7HBdJzn5j>C(9c?|$(y&v7#~I)+
zj#<j+udSsTkIEG@Ki_h8MHv*qqB#7l>MtXO&8kkRnhG_{@%8N!{QPii865JhHU8la
zEu(U|_oTQ02Ml5ph(|Cg+PYoQ$>zV07F^_crMx;U@8*4Pr7%&6G6Blk%?lyn`prR^
zi1N*MpVb}evDv4WKyIQWhJ#~1FWC}Dpj9cY#c`aJsqn{SMc_IsDm{BtU?Sq}f{iv*
zg)&hE(r%VJt;$>U#8<Y)Kbk(Sg~=6Lf8#d8k;=41wp5PrBt;vDxI)D)xl?D$1Z&Og
zN|&m&j`vp5d_~tktvf<dqGHj1lMbJKPYfS1*a+Mym-3u?>bCDkEbk|hi5$JaP-U~y
zUNK%B1A%pV`FdP(exIjArjYaUCw`#_)5`T6uY=1KKtX%<qwPe&A$S&VY3K=n)GNx9
zwz`$9Dd;L8SV?sRx>_j};i{Uz1c9B(Z}e)Q^?nFf0HDv+9728YhA0)qngNt%m1jSG
zla*$3AwxfrXnK<;WxiXQNVc+R`_+PXQ;}qvoKW0+(gV)TB%`(@A+^cC0p>TCK;Z|0
zLD8K80qZJ!aQ0J){aO9IQ-L&x<$4d#a8@@QDY;-XtFm@c2EF5O`0x}S+yO*Dh<v`=
zr%2K-WUUrRIJEilrc~L_j@I6~2)bLA5zzU<dAX4gij@QRLMX(OGGl5ca#IKc1)3`+
z4cWJQ$NLOuY{e`<ZuW8+qxu06fvar|S?>A!l3wXv?=@Po^H1OXrAxX;+brDnKY@(U
zOeA{($`e5<nqq|WU5|G8U;s7Xn6pZM&J)v_$#1@7=BsdXaFVK{#4uGnt55EWx(%3d
z$KV3Fg<Gg7h%#*C$MlgNkzHg<R{8m}7^aOcG2bvv7&!%Aa@iz?KSn1n`R?tr?WBkc
zzWYP)-H~v)TH&}T@3z{3VgQ~Q*Q@ulEdzrDFaw>$pZ5^w(9@5SbHd$pg+XlrGT8z(
z#W6L*^)4R`J(9=wwueCOHWhvcYBdQu{=FF<bpe$se{f1cf5hV|0CS(o7+MAo$ot67
zlm@M3JoID+Fsk~v7yM4U$chP{hY9jjfGA;@5l0Y<LYkNslan_XzCj~y!ZIm(a?(1D
z#4I&E0GDpxO98W6<wG>jTm;WI^5;tR)jRFFK?VT_JG7~|{V44GcMlE0<Mo+lk9E2{
zyLt(%X`MlbY>1vetj^U5D!ctCDpR>A%)nP}FARygz%@SP)d(&^u>pDMi0K;rM!|p$
z!A03589}<Q_`*kBXc{%PHA96Z$q7H=*=M$D!u@OXxA5etHoz&*H5}lqLSOUWeN*Mf
z^8G`hhmG6akz!qt5$)d@Q1*Axla(2e5h|8g*@%ARYp$1zXY+}sE&5)M?b;5DO5bKH
z37u2OZJs4U*jtu_RX(6XC$jrEc1Sb5cN<iYqLSIXCnEEYGfgYWE|-wvO9B|Eh<#{f
z2oF>O<D-w%EjjP8va@m#VKj>JwQnh?hNZoJjvtqV?hkQh&6J2(>p@p54$^#!pyr#N
zNc=;Ikn7I|w@5O=(G-$;G+1pvf0@Z%zXE=E*qW!`9c9rpMMj=|JyO<+&Vs$yREW6J
z*aDv>ri@@F3k_Zv36Y(3(>-{%Y*SWJhjWc)d<wVLcb6mQ!U5#jlG-mVS-5NVElp$S
zpGzwe!i^l?bmF(YevP-rwDsB<B=tEvG8f)joeqwF3S`|CDq_rK%g{ca{2mzKWs23j
zH>dAEmZXqBHCBWrx#od37Nt`hBHXU9i8WS$Z3WkrYm8Qq3loK83W`3GDe>+upOvx<
z)xFXe2QF87a_s{jp?=wDgyBmD2x23iF=hXQ#6r{C?b^*x;ON-b&&X~2R{@WhsnP8$
z$BJvC@m@RATbR>j?>t-&@CW`=nV-AAuR3t1LMNaN{&ZewSee8;jbuR^7NH4+`|4^A
z6u`dRUBGc?y7dqZb4X48YO#wUUwzl-?SMJHcUSSd^g4YhT8Y4Tme?EDSr@wVC~G0A
z`U5uJm<7B;0aPa&vlU7$7yb14NZLUFk;+YXnTla{{xb9x3E9@=*;zl!93cPYnq}7E
zdn^N%8@XdncJuoh+B+KZT3yy+N%z)Fjf(oC37yPFfNwLG9*|-6tk)u+_8gU#w}#Sh
z48HJ%F}LDc)^yz?_aH$%)<CyS5SY-_yGUnhS>eAGiMbmYN!{FLkscF+$zmN#Je1`}
zD?GZZe=0<Yg&l0AZk1|l`+Znw+Syfk5cN1eTg2sbrtd*7Hg=m?cKFir9pU_18rh+z
z0)RZKxWQUJ3$WFEnGBX9Xuc0ot&ypC^F81u9B5li>0_Xr*&Ypx(D9MPlC7jWTMAN>
zhD`2f^X`~^5{eJ0+H<R%WJ7%fnWCkha%wMNFuA5mGm!{GNw3q-BM0Vd_)Dm;CaphO
zr$VZhfs_K@ytax-31iRG`h;jvN)Yay?>SS`xSA%xJ;syr>h_PhpI)?GvRb@loR?C2
zSu#r$epXs^w+pAk4>!#;XyD9$f+ks87Yo~^;h}Td{gR%9yYY1UEe9OXR7_*gMit!-
zcaTqb;*{SQWj-<ej%6Ki$;Z>lo5P6JoyKj8ztl$sMb^DPHtuwIa0TQU&&2rj@;&FJ
z83Vk{88KD$((!8csr?WNSohgRoT$I12Xa5<wUs8)*m|a@PEkQ=%f1I(E1P>{-4DNz
zVij&Mh%!pKQ;c+q6==EpHCPfM)*|IU40BgK$0~_We=08V$@T{%RnY9~pKr~lta<PJ
zs3ZVJVZa4VgDydmivOaxHBBh28+SVyPDw1dKV>{d=?f&e`b{q{l=0i7?_-y><z~?p
znT{!vVbBNk?t$#c3!Jw%X^G#p?XHS3W7nO~xBNu)ElGk?7o_MW>7`KljIXw}69YS^
z-uI5%pO`Jqga!zjzYb!>jR7CA&Zuu&l&m(&JdT*jML5)lbNxiC9<(6DRmEprBIxfB
zN1_$^QuYqUp-rx|&6jX$mO3gHEvH+^jb2D@_;`eW4c}@^p4Vk5@9|sEk1>o5GZj15
zijr(P5x4ytTI|}O6v9d&w|4|;y0TR%qei!(9K0R>5MmZhr<0n=Fx(s8fj)z99#Vgo
zQGCZVRf6+wt1*}meskeijGC{yO8eA@j#6mEuo$h}w&ZK+Vj?J>1eodASrVJccODkp
z_86;9-ik06Zlwe<Ac&k$Q%i0!Jl2cwn7NBI>6EVJuD`@86g~9Gp?0JOA@8y~9kYKz
z;r*-x_5_aN$|p`X12WTlRdpkd6)IIY&iM~Dvd~_8jf#0pTM(B}ktXeKI#BKkuO+Fg
z(RXU))E5pEnU7Y~$u}U3ifspq-fZj?(A<$bBDTy^7Z*@L<(uv9Hks8)DkvPte|A=j
zM0%uVx`^W}H6_de!3ktT`1xkD`raDIcMx_;cz~q2eLbtiJ!3qqajkS7%$Yt`o5EyU
zieh70*bu5oIlLyrCI>Tae~W%@mr@%&{+hqF09@g}@_%s~S$lj--*#h_ID2G#4HLQB
zi|}0#qCmGrid`OK+S?LQ7VFu-+LBrn9B=|zj1Q`l+x(DnvJP=u1yIB`4YPzXI5Hw#
zCWknhbJlR2Ymy0K<2^%YWmKKq8b-g}GshVanrCqa*Zgg60gLrXu6)j2C;4+9YmXNl
z@wi?0(Yii@o>5k5pW@fNZz*w-)EUhNa^X8lsoG}6b|eTS@FnNYP}z+{mD*bBb;P~!
zpe9dY<(6eIryx8Sr@)_12rS;}k48^FXL6T)C2^oYm^~C1%b-DfM_HfAfRSNob@PbN
za?@(C!Z&E0FSz<T0JZA#fq3_H6P!{UTC;y)1LdI#dA6ci=Vou#x1l(>A)C;*LM|A(
zO$b>AR{Lck*BhbHmn|J!oI;@BIHhVp5G~ptyRLIZX2~i4A>d4LHul%_kl9Vqv)>ES
zYg_%o;LQ{(K(P^NLM|sZ6DNpq)Ev3|b~r-(Y5J#(+kDSbYckCmt>LVuC}P&=QeAgO
zXWHpM08~Q`XGw@nxdl%~9VZ^a2tj5V0c}I+p858HG`6BC+}zh`9rNh^#L<RX?l(wT
zA=69<S=<X8rV8~Aw*2>nIf4Z$sGEwVo+4$3P*dH;zAv3@Tdw5uL?0Pxk}Flsl&Cj#
z(mW;iNQe?4i(|aOwdy^5U&m10o6e#HI3>MT>2~y%A6$cwNGQfP+PQar4A-(hG=$v%
zy>@ldM3`)wV)fXYD2ikaQw76MrL0|86O#Dwzzq3*q`c`g{k$_|9?_qu+SVv^HB%>=
zl~EkaX>*;Z`|6>+>A6dg-spi^eB=!yl&u&J6ZPJg-6(U4uufAC*Ue7*_6##2YKyVl
z#g2esVQ{b$9{p&g#2G*McE*wbFJ{x-RrZz>uG};9)&TXAhiX#uq=yBgV`Cn>Y5L-C
zaT8}bYc=dphKpXbrfyZ4-N7B~s%ve8zwcjez$TR?)$C+#`tl=4ILVeH;NLb7;#=Cy
zwN!I?+vycHQwKQuQd-^5X=-~C5XBM2Yxah%A8@r|Fq|mfU`6dGKCr@9OWoR$OZoCf
zig|n{hez|R#$?;karAPGt>PJ6MYDpwVGeEcec{IbN|H5~hDS7(o*+aYCfCQ?WEvW-
zQ)Sfh-;y^rOTRNbe2U9-mRiF2G>_TsLaR8*mg32~5yg&7tnrwUCo=WJF+=UPZf<f*
z!62ifLT}Qc+rJ_QvKJ<d2x+tT#qJBtVTKg7MV4}Iy}|1)3T>$8PPL?CkQFQr%|Y?h
zk|UR3B0QYk>oR1%H95K4h~Sip!M6lMBxwt$$n_N~O9Dx3POm_gr$Kv{>W~7{uezpX
z)z)63q)}n}N~gl8^l50QDHAN+)xGpPsndFPinS~ZwwKg+EDJOh7Ap37tOh0zw%K>m
zHJ5j9Ap|jm2eBs4*ynA=8+txcZ)sv6bxXiMbrC+Pj{T<9>E`83Sja7rv(uAV<^pRy
z+}QI&xHW|L+oU5!j`8c{<*5nv1}+-I0&^PeBv|W`SbLD6QRxV*jq6DUX_F2O)bTi6
zKjX$^<yJy8`_7h!5O`xmx3R)D%gD*^TqhxL)-SCoOoeAYt0#urm1DPZ?|#M8q@+X!
z?k%khI3*$>l(Lz4e(kBp@^}OTc>3xkvz^@x^m1Ehi63fe^jZZL*AiJ266oF+7Eh4N
z!qJW{(`DAUrUw?fTDmJ{bn~@t(l@`^aA^pebrRHit3g@zbev_Q)M)VrA?}sz7ap!z
z&ZAThhsiyHs=DQ`WMIm+<tb;9WSI+@7!A=I;b&uot6n;o0bek04GuAjr5&cc4ea+M
z(QV~<<q8=E-!yr`FU>AYwV;NV7fE`>NJuMZ_Bp#_oD(q?^9x7cfS!FtPZVb;ES*PJ
zi)x5X;(%TnQz#1itT2&nIueayMh1)MhKqNiw4MK!LRSV3;*S6{qcmj^rm_!ZLQV>A
zBJqpHD{*NkdN-$$IC0<Xy?JiSgjTBV^z?jI@eP}HvqnC_C)V~kH2xHHq$U_O6t6yo
zsLMhq|5A*$PTE7g*)GEJYznuhyuOr4LO{;Rw4Un5z!D08#8C;H@1Sh%LW@+0JRWdN
zym*RO^x~7t{z}iM=aOU}0pcU90>a%vVx7&k7Ukv*q~?AN@O4EJb!YVrt<etZFzPwP
z`dfZ-9kN;ChdUro><46S54Lfpe*{!1h!RUPd}qq6FRpx6tu20|^%CFDQXUnX0v;v5
zlqZ^}@VuLtM)0JTfiNX0$erulA(8kh3(0YEIOWAbkr*v_Du?T-W?zs*-iN~qlf%=Z
z@PIAkHoy>iyyVy=RJtr@VpQa5(G_WzmdH=@wXp;D2}t23Ishb={73PzRt3l|9pTVl
z7frkJV*kvkDrCSU21v!H9UU`#3P=;8ZOWO<_XNPi_iAY8AA6#x6yN}?j_!$6X6cj4
zL+7Z$$QcG&d&_!dL~PDjr)|^uMtP?ZfZsi}siR(9HzaSY*rIBUF!$nywGQ<z(l##S
zEJ1*Qz>*IpMbgiE#khY=q;k>>v9Pz^of_Ut^p;=8DvWhok0MBLO6TC_c9cUizL@)H
zW3aXX@sgR43<=iN2#<6MOKyt{E^M^>dN7g{RT_HkbPjB%=r;+eZR{l_ln^r?))RCp
zwLRdlWHoeb?y0ox+Y$4As_G&+L3;H9ZJX&6Yl1@<E0n9EFzMJz{%M_qVQQm#%Oq?T
zO$4S77>nR<J$c76kk~-lJFUnwTga|!8Zei{>6B`l(+bRI{^PJ&m1q3w!8vF^d5Oi=
zCctOR=d*g+DL{2bWT7ZF<JLI$nQGn9tE%!pE>mFD^OX23(J7<NV_EXeu+9e4-gC%o
zSbeUMiVKUYf?lNidA5G7eS`uK{GpoOgOl>Mvf?dTJDQ8)bJ7zT?~F?w6z!wrpdfMO
z{BhG|Z;L1_(YiI2Y2`9|fFJ?{NOT9%=K!+NRo|97rtfZ+paAl?VxVpSGN0?fpvb}G
zdmB+J37du_UyjDCG+3Yd%#3Eu#baP_6@dlH$n;?V&z}fV(s%{ZU*(Ms5&*2EsKuFK
z3F6m3+aJXKQTt>qSv$re>O<idODBA|WDMV>Th%qMO`kHj=UcNqQnWZFWVraQOgxKa
z%?j{)%usD3``tGH^cziTanquHhbc`0QKh<#x-N-%7bbe@^dI1E_poML!zbEktlEmz
zKiLO*R8bxS<485Gw1p6C4?gL(KH{J|lZ75&754QVfC5#@=eYR#$pbH-%e5EpI<hxc
z=RY1w!Iu0GqpxT*B<eb#N%%Fv(c9#yDZ&TY5lJX4AGlk4@qK}Cj)G!51FW|B(^Q4G
zxQ`hgb?bt>rCRWgSc81(5lFb;Gc<mNDq*8+oXWzd!HI#KZkfG2KIA`x+Z(nHwj@)3
z%2ZPqXJZ7MP1>a&Wtqbkm1N(~<f8A@eL4PuJ))bG?(rl3!Fv7L7f@W&e(U1}mtj1a
z_rR#D!;oPMB?vjBz}mqMWBEZ^Oh6sgT1<kU*R3yoF8UW|1n)Gn5Oy<dj|!?|67JIO
z!%gZakK)`<CH@pRE0;UDaDEWGbwv^-C=B4RAjj|UYN1hH<oJ18cM=7<?JG0t&j2)}
zm}_n1<pSC=Fay$rs{hK~0tF;<JtL8-8a6ulDPC`?w`ZY!3xO`2%-_?g%86{so+sTR
zf^!@t-PXZ|gcd!L^RCrh%%V@~t|YC(Q-eEV&$H=u%B$Q<q1G0vil1L(j0Cu(6yP(3
z3V{>}V-lnHz`^}zqA^JDzVTk#x3POCT`6w&P{V$gU>FwpXrEC$+G$bF+~~Homt~Ya
zDCm>Eof1FRG96KQ7l+}qYWE>?Z-)EO!trgC?&C$3_2k;@VKvRl7(qPL_bw@9T4_&i
znCuxOZo;{W;3}~8F76wa_(W|)J~J|VjrUm0npwL)ErZ1CHsUwBwS8*-BL&fJT)6BT
zto4a+Tq{j8a(xp9r8VLok+Y!bmAOT`ul(FWF!G^-l6Kc<`74AhfKPVYy_*#}I($*8
zjmk%Oosr*du<i>nR|E?zM@b#)X~d5sJma^Rb(V>wI;^Nx4G1GbBerV=wUGH{TRV#;
z69N-?_1%?7vPa>tyAF`azvk*ICE`wn86WuMT~5*zj`?d&O_&G8?7BT4CH)vnbr<;t
zlf))ZT*0ao@m26MALuc^sS!v0EF39RcYuN|{*|Pyk^)Qg+A)P?w^d7raB(rS@}eu#
z7u@tW^t{<9(N91{O|X>%QU}-DP%}uvQ>$-z$+dp);v@YhxvBGOeB^Ls`L?r>M+Z_V
z4kfx7mO01e){n`QrwqLJ#cyWa<-Pg=lnY3-lvTBPyRCgoame$2Cn-@4u+LKl>71+F
z`;}<HY8iHd*5<*Hr$MD=o9h`wV&;n=DH3kQyG|J+Rhwiw09T<;64A?X61`rYly=FW
zUoZY>j(ZcbPkG^)l}AV<&M^4fwYgV24%N0A3xI7Hag_6;WyA90Mv9k+Tsg(#f$u>F
z@~P@$M;ZoC4d!Prx6($aFxIhGo=DFg|Fp-#2<s8{KD3OocLRFEh6||!cp1j6VB3wS
zo}369XR)mqHC;;TFM=vZCb6QNwH_jpOV=FXDMuq3_E!jfs*o=)y|An-{0S36)N{o$
zu{|4TJ-0j^z8HIv>y5e8^HNVhnUfSPy%v80L0sAPeQ<T*ERR0c-c~PwdElGTQYsaI
zHk;?h#rKavkzz&T!2R}|q3eXh&6%6L?wQ(F%5qf21`R#rpRvsN8lU@K(8p4HhSJ`V
zh}st$rBJllFBZSwO8hn}LZt6hn5c%5>-B}jwO{(M+DOM%&PztH^tbuwwzKpI5b%)S
zAJ7=b6_91%?D^g<@ByXczHXjj@6M#mE{prB7bogfqk*pZuS`sfB?D7@@70upP?zJ)
zSRKy1%Lx^GdVn!)t;UsSMsWZLGinU&E|7SOpu(uAI^1xx`1J6CIGVM3AAxjRtzOk*
z9q(aI*w3Znb+XUXZ0ngz=b^0o1n_QRrsoTw3jp=9O0?xLt}+?@wS|C|76iS|?!vj@
zJ$rvCQg)n?fjyoPJh_}g_DbH^&FHcVF8c$L-<6epUHMBm)F?%h(rVJ#ThN7yTBjd4
zV4hZ)^3ike8iP8`IpRK-C?^;pNS(`E(<JY}HTS|m%9GPM6^ZwKOzLKm0zS9-hXw&q
zVqpQk<CIG=a|w6~c6&GlzEq=o{ph?9I-1QeEl6(EuGMPk{jd?Prz)d}DqD`*4ydo!
zYD?P}2Et9Tf|}De>+u8VJ4`W(F1CXt?mnoruZGeOk7oF#4%BIFy>P+MtAYqdpwoh-
zwlN>ahYi1@(0~=}Tu;zy6@pO;Rn58#u{6d>86(C*`R^kf=ajh|30f5s2e>!G=sfk4
z4c|L*s!p<9?aRS`M~Rtm-n4;pE%o@poLdxFhD5|STM5;b|JFhw&k-3^2&5yvz4*J7
z{;QtOgF@h3zvZB>4OQ3wDcqVo1+zfaeWLQm5Tco&_&)DeUs3jV=^L7AdUi2TRm3#&
zq5oruJVh{sqrm*M=0BrC;}g6Hgxx@{7EbzOh;dpl1W&f(*H_ShKSzarM1knU_36w#
zxj%-mNd#rp%+dNMn*V-b@S7Bf-1x;DFpB;dqD~gL7mC*NO+TaGe}<;QVP7Jls1H~B
zV~9CwlaATqT1P{WPyY8KgD{~TdUsy@=MMd62uCdTC5Nfm`v3Q)IjBg=llj;4u~)!F
z;<4zpc<$Gwe0CX&>$tqU1PF5;aoie4XtATvhFNrLEYcW9M@FhI;?mL8VSZ?ugdtq|
zD}Efv{&tA_CbN>u%+^}7qd7)bxOTcMv*VY6c*5o8cON>+ELk7+KFNbwuqo)s<30ay
zt<tD{GN)FcPSxCN39WPx3cfS?#?xH#$dW&1T%ioCa9s}nP{p4wzXdP<Klkn`$@D#J
zi~YYgv584gmG2r8lttVD;$zaRKo+l&8Gv=`KqIl_WM!YZsS`j_SeOR-p<e+xCN>!a
zFP}hftStE@>7Nhx0GyVKr=013ZAXRZ4nX>WFu2l>F$6aAMgtFl0`FTOgeChLK$qMR
zvw#PzJm&}6j<GLgWne#g2(Tar@H=XPUK`z5e{LKu;O_HdmrTh0vz2~s`9wPQ28)3N
zQ@Q)RynVakR-6j33)(@1c#!Ww=TSeV{bmlV0Fjot0nzN2rt%JMRE>*o6K;xhwHk^a
zn;g$Rx}-QRJIq^}`~7G4f)gHtJlFN-r#WJS-8)a%(DvJL1N$ZdoN#IQZ>GN+1JR&*
zl+$~o_1TZVLLd_j{D^^Dh@Tc)Ys~D>tKvuA{k<Drhq`tu$MvX#qeClZ;_vTIIcQ3u
z$M)K0B7d3~89L7jiGg69ifb)(=V?`jG~PRk6Px=98=H6k7>713I-0`zV(I|%2XtdZ
z^HM#K^u!yj&Jx7<(<ltLfsM42Yi<vj#(y6wN)p%#=AC;Q{~lcMTbds*EVCkg(<*<w
zOb`ji&;jZOzmGuAegRBYZ9q)Li$7n+17lRb_EGu6zL}hW9b$b$qA&jE%gkU5bJHJr
zzkN0EXgg>G&ZF1-^<jU$tOKUIuX7h{mf!8oEI!ynSFR4X|9rU`87ySzr;tA_SuGpb
zO&1$8M1Q_)jtLALS~Jq0Hqy!%Ow!3%^?i-sJ`Wb5mlBKtRFD5(oM`zGf83w*%q|wx
z*w~19Z{%O|ip^qzwjlv}rUJvFPR%YNA|hGz=H<_sQ>TWu69Z#~q$WAj&-1dg@i6s^
z?lmxR-YQc}<5<8=W8t6BQvbQ$L8aJxtd`qKx}k7Ck4HKAch)~vAP}w<ETa!`y4WB4
zLnSX5+%X8;9;$!L6V4_JxFF{HasD#~E?7pUyvE2swoRVsBd~8iOBMgVw%}7<l7kE*
z?1<qXM+_Fh8|;u19?sDJjDY~gK(*P;|6{MykL!b5ts$P_KVvY09ik=bivP!s6b<?e
zwq$Fx!izu0P<aT(fXit9_Xz#`?Ff&3X~)q5^*>{vf-!K%7bO2PO$}h09#N_PXAF(I
z>`Uc85i9>(<<L9+H`RYk2kQT$Q=PJ%-3|6ByB2i3s_!j^{Pc@|EtHCC2j1v&=iTBC
zWcc-s4QXc_jDI$V#$BB{8`aR(g(62dSNRlX!s5}16HW`-pViqvx3V4nOPcB0(Atwj
zvp?-xAY1@&j*bNWd^?Vf1lVzyT3mnHB;bi15&iF-{}`J8&C>sN$ZvN2f2ZS*E9ZZw
z<Nw0x(2;`!F0oFKB#0f2T0G4nt>69HX=(65XJ=<nXFV)AGeS@nR5x)NNTA8KzcA+F
z<&|Ec;M%%zF4oA?gxkNe(@d@?|K+9N!nff6M5Z3${JIH#LKJEP10?!<;e=YCH+8_H
zojSV*E5}HxFDb*tEdTFP`0Z@}-?o`wp<V9B23#HkT(Zc`p~#0?XXU^BHxqY+l#EZW
zbh&`%NxAm|!*3I6Rv`>EVzw5!nALPiKlZ!6Xur7tst5w#>ZSnvF$KVr_5fkv2modU
zzb3Q3Tok$fBGPu4{oSHV-uQeHG}e4WE_tgA`Jf%-di}rqEH*EP5hEXe1o#Nh;Km2A
zSk4Z?_edzXTIX$V1WFe-YsS}W#y7{tWIx{a34b9Rn*kkh=`Ii2KuaG$*p7u-U;`PB
z&!B0hoZUx9Y@e$gYZ0G8^_&Ccn<M2<lW&cF8i61w;H(p`-FF2DT?i`O4j4cM3!oY>
ziX?LSZmUfEkcM_pV5=~3^07bY(#Q%jAWlxK6$}O@_$!k@FWU<cXc!dOpD;m65O4LM
zuHt&Ai;0^c$?E;-tC93Vn&5)*Wsh;o1;DuiFhQ0+8JWoSek?=_&&EK2*aGd}{k~Hf
zu`Jl{Fad&%=)H55C?HPJq-T2lZSndWoyZx=nDnb}55|pz$3BTRrt3i8+r$cf{jDrf
zG3Vou`+qr70OmFk&<+tlHiW->#XSIc;*xJRpe%Z`#um^BT15=AyM(^F>I=B+!_GaG
z@Qs@VydItzlMs-wfI16or+;`=R_P6REh1la$MT<4TCbGKI0L1V*5%ue@?JvJlIvx%
zYtvKCRBS^4HfQsh9lGba1be6My_!75K)VPags4#ud`W0%S!0<s+uN`OWE#Pn%R}=u
zkb0DMxcq)|{aplfIsd|jZx6)Pn^2<o*;<d(R+i~`#{AA3J8E!a*Y1BbNDMr*1=$qk
zfb*%*isMQK5s%}0Lkl?ls8QFCEKfFBnb_w6Caj&Li+>*f*&B5e_8`Eq=-Sz{4mhy}
z%@^p=7W#~VZb69UMDTlvOo%?`dt7F$ctKFD^~NL*LF0*5d`{idADE61NoQl_a@3=(
ziz)#i(i3YtsIcz1k3U56`_a5Oaxa;kFJG}q7`t<#`UJ>%6jO0?+yHWy*x=&Tu1D58
zz+wL}+uGZvP_(Hacw(jl^Xqtofk>O=^Ee4k8YV1al;lk-e%oFsR6uZ))`+6W=2B}t
zfAtW}+M*uhc&1a!_>b@Ah5Ue6B3Kg>Nv6F}WAdH7m(s%9ppU|X)j07}85KE=bWW&$
z?_Q-xYh>)x0&reZ2{s6|@7qKDs{tDzGHTT#T{U~Qb#m9-U<P!Q2>?&5N!-BSYvyC~
z>uU*sZ7!&|Z2)KvXSoaAAJYrG7{=3J93MtWQePOl&sqZB%=1jcjF1)9_UbOB$GsC{
z7SEa3H6`iH3R|A0PF&_NU4Ud4j(ts+U{%vLBO*slj!Gg!uuJne)j7~GiAWx%OPUwx
zjfEIxKr#Ca(hdPCKm_I-LUq4P`x|6FTNRe5GX?*>5VAd;E70(UlgsXPbojm-jCq3S
zdC2}gIJcjGP{85E(bXA~@tK@L)ML<dg3C*4@j`fKD);&bR7<72i>H}$gM$glxAN;j
zKUyiIp=%7&!BV`4saDVdtPOBN3MmZ?Ar>K8x$@z(6_d=Ns(~tmou(83D983e6RzSX
zUhmM1_JS~c12e5ywcWjLE*6y?q#jAl$pZs6#>tx0TLh6X;)yHNx|@bQpymY}Z)14q
zX)!I&RRqf0R%zItRVEZdwCzPeQK=C{i51v>In!`xJonCwh<N>YnP^vGyfLiWHxi$%
z4dnE;@@36GCgvPb$6S95xcZ2W@7sx;6E}Hyxe1yWeW40LG^e?WrXqsK__tVSMp<D9
z2FzYkd<X!95KI)979cQ0HRsjCWE7<D-d8sq*aaFxg~X0|jk<#NwMclUVFv<uj57DP
z?sx;{UL4Ar^fPXFZEp2c<<$_Lb#&v^jrQxW?P+e|#%Pbcr!wyWkbTJ)Eu$s5(}j%(
zeY*;)hX7W)4tj$;XoqP(c~hk-CUt)G!T1Wd0DD>SEET%~uq+E#-|w-7u%OAm9b_z%
zaKbe0<$Mwq3UM&-Ty7rxXdH{akHg$xYe3fMy+xqVZ~}~d!SS8k-9v%1RXP#=afAW!
z4+%qpqlTR%N3W=^7H_T=1HKS1np7J4tFpUipdCY)dE=A=*=JLjhChQhf_4}Wy{=0M
zgs=hAlan<Ji(O@AjET~9Ady94Ak=!VcpCFo9PX4kdXn-s@xX}RS<GYUAwN?u&ITS9
z_QCJ?s%9iH>mx7mf5Z=$bjk&@nhlK(O1aXbATtt7olZBj-FI0s-bI!j^K&gXO&CUA
zRfD@XC=n(>yC6|~+=#FOf1(&(Ol(0-E+xlQi<Xq`oWvb)mKu<5Xj}u;-rc^wmFang
zIMoILH7h;0s2)|w3<%(RTFS_!>ysy}%Bqx8!+c%forWYR;=@eHrzPHwTCl@c2ORPS
zBo3v-!f(SDed4@)U-CebWTp`xlbPk|r;}}ioA_8?YEDxE3-_<U{4e&NLnK1bR9$wv
zKSP+ZRch*qrjo>Hi;opRU#?Lg^IKyirBs1Ncl}u^)D;RCw2)Z?1hh~bYtS*Re)jl_
z9&PdS6rCI@m9Qh5kPW^M&o!0>2#<nyymr+FkXpArJ0zB}TdgKJP3e^J)J9qB49Hod
zQmAzN5Qwl|u3cE7S)X5M&cj9X*FB9R{nG5d@L0NucG5&me|N;XyVh||I71{xI*mEV
z=5wI}e3mMwp0?5PD{a3Wb24*gLaGJ8plMFPa<x_-@{)_opfSJA?@--?SJ%IW#eQ*z
z_k7>aR0*H9f6XZqIdLy=nmt&DXcw8ZfHc|9b{^sk>Y50D^JHkK6V0@9Gm9-%pE`F!
zgRgw*r7$iAhbnsO3q;f9q9QtL%r2T)P{7`q$P9#+P<(tb%FXq(w48#{l5$aJY0*jK
z5;21GOHp>+@Sah(j6x3bUOvvCXc?QtW$)&2IA3Bi_a&C1WrLco9;a=lUpLh~*nZIL
zSb~c)luqSgtZDRcUWjJp*YES6PwIh&S*R~x4}2N-Q%=U@R!`Ipw`L2O4NW4?TBKJk
z3R4`yf~QOoEQx~$>~{_~NrtG{NBK<Y(o?bH1DT%VwLA6Xh4MU!8EGR7&8g$KVs?BL
zdQrR;%%E=?j9FE9gEmF3nsyZNA_WfJEgRkY;iS8vi&x13hWCdFNy>92S4T*c%OKA8
z@hRedb3~=Muk6-!M{=%DOXi&o2y6(vV-$44+tP|*!xp{nivZSavV14jcC%KqU<&hU
z|8zOs+x;qUXylEvOTifSrp^0&)OzPT-gb7%A~oOfZ!2~yk;Gj1XC$aeA_wK8yVWZ&
zO^Mkp7xFUjh87m4N5&+zgi&zHvSe|4)+DQJ1MCL%Yl_Y&g-LxfWUX43bcZ6+2X|V5
z4+YTsiai<}N4djAvD^~<JCZokqG>1F^`bITA7}YcjrF|`i~o=I&N3>>E$sU+Fd!w}
zCDIJtBGLi_LkZF#Eg)Uef(n8X%1}ciB?c%U4RR<cnIWVEq(ea|DN)`%c=R0o{J!t=
z;W>O@&D!@``@ZK|_l|2{zyE)dlaH}tuMt|6Y|DZx0?ZtS6GgdEfxBw553~`!#x!`j
zlROH=c`@|ih0nb2v4>D1hYmBOUnp^2&M@Sax~mSUKYVSCACgsCMbES|;dq-1@Xm#^
zlI{X#E90%P(}uZ+M(U8O5*!2x;uw`oE3}WyL(7Vt2R`AmBv)dLl@q>z8a}W4hq@NW
zSD<F24&QH@kO&(h+IRKp-n^_eYGEamW$^lYIg*&3OO|`Nto1T|A`VU@Hw?|l`#p|n
zGx#mg7g@k4Mpdsp@$8^=Hp)x=JlYm5XeSiKP(g~U4w2XL6gGcyVa9=lsfJ!m50Qo0
zuc;G2Oz>xpZN;s9O_3WA8<TyatWW>VsxBpQkIt#4I0ETIH9vrpGS%ZlEDKlCX0oTR
zB#QBJuCS+G_Zk1<f`S{4X~8Vb+aTWr%(}^~_$oh|6;E-_anFrS7PlsrcEoieTQ{V>
zkQ5DwWU8*?@<v{fR1v3>49PQ12r@49xvOfL3(-BK8x$O`qhHr0KX)N6*3@u)<ptwZ
zQc$zfHP>Zp;3ej>_Vz0JI2=I4W`J}CvP;?1?W(ueMw0TlGvgd{PB~cAIJ>e*Ixn~;
zierv)dK=$qu+eYn=N+@Njr;Wq)Z};?W*L4+k$wvxL%(dJ+B5vVHNGaZj|;jlG*e2x
z;*GDOP*p#7o%Eb8%*we^DfGcRL7ORBXI&7$G~BD-c=;gxVB6dM<?1*61i(d4@lc4*
zxUG=x=DGt?J^XZ<y6qI$?APryvUA^6E*YOJc58vNq`&KVP2)9)*aVcGY+6K!R=wet
zYKV!4VJPBF;<&oWM9)KctT~j2Nbqqd&k}c!yTFm=Exp2$gCbE}Rw-<zp{csulw7<U
zfy_b$@}C|)Cm!#lBuz<M0(pKPs33fM%Fx0Zo+P_P$k3-6JPAI)V9m*}*zwW4mmfw4
zrcD=2kR_H;BXMUqx^rfFZNx)aTUtkOdFe5?0!Pf7gt~RW-_G_G^U}vTI^y_F7&W&%
zAH!MOem{vBN$NW|d!HA{K$%nIAVrXcXT?ingCR+>s?Tm9HnHlfAv~2>PmDfY`QRH!
zf+X;il2S<A>0&<HXF{JgK;94yZ+$Zt@Y#Gn3w7poSR*@V4;1}wT^!4Y^|3uH71y(`
zdL+8XlRYk%sClIwD>Nnbn%fjwC!ZvHwHfbO=u#x%*v%vdOCpzfO%uH;SBrOs(8F;s
z%y+q2US(+)){qlsf5s9hp(_PioFA>oY45cTw~1a(bqP78yRYf|m%jR^<H6!J71yZV
zGDI(;Hw#i&mBd!v$Cn9+>)JCYZf>2Gq^Wj&rJPkS@<S9lG+MHBp3lWDk5i{&dwYA_
zia2dmmk~K$)_nM3XF>?kmM-R08av?Eev#Sjxw37Axv}_`sJb*&xCD`55Aw4QqM^t0
zR+V^z5~tp-eE>?{KGjBUv2zAJd_GOO$*rfV@8IGsP9GTh6=i7gGg`VFVBSUliPrwg
zczacS6j5;5!UaCbOkXyagn3Gw`EZxvCcp!thO63dySaEnkxO=|sEyQniWh=cg7kB^
zPVa2JRq*BEDhQEY-#&`Zt5ibU-H%YDzFi!}&&V-(j?*X>zU;43og&iJfA(_ihg$hs
zpXM*mJO{{cerBClJYY>?y=ozAV7#1fRe-IQh!R1<zS(BpzNs6|7+4VwtQ3lU2KSMN
z^OQkUUW2XF*G7P|6;LO>m-dL56M{UzYwNCMKLlDktUVE?Mo5mnsINf?Bj3SLU3)?2
zz^hG5=3Hlj0>HI}CSPn{_B^uUl@~^=^^8-4JxEzXFeR?K8|n#XwmGne3a=mbQN1FY
z4?}C%xu#AgtlFbP{May}m+k?TQ$gMC{w(cjm%T|yO>0%d7Onl^eFj!7$Xx1*npW>+
z#hNwJ7sXPEti9Sf)#`p3FnByW(J5PpjE9yj-OpVUwz@PFDmFUU)&|!@vY#=%3Vry<
zMc6Lf!0+ob74}ihKF|DFy#9VdvIhqn$AdDm^@K=TT%svURCWvun?vV!D44(V(U?^x
zsNBM6!P)JlDF;PQJzuc3^ti4-wYW7@7oIL9U$C$ze{RE~9*vXI9%o;e7(DBJ(DqlT
z|G3UAYu+`B^_zWcl0qg8ANE6f+RSzr)5ukQ4`$7%!upA%TzWo9lNiZ8A~`9vAvr<C
z>|LErPIv7~yWP`l^}f9g6RuA#eiHklS;OglM~8C(P@cLFbqlea_)NS{jX_rUZ{j3z
zy-7M=O?^6TSS`=F%){w1R7bc-F5%62<5`eqpOql^fsxGb$tfi3lrtZ+n$E61?9_u#
zx!mR_m1SfzuAtAX=guQMwcR8BLgMrIy*!!4HdRKpb_%*;DVNCmNvBaD`^OnjA(nO7
zh$F~VSS~3t6BHLrl)2QHJXm5k+|Xhqh8?^4GxU&W_rwTA6ezvKg$!+N;I}N`VpJh^
z2rJ2<jHIc9CWDHXXdSV~)#&y%u6F?cb6JoFY52jH5Rfj4QHiH9Cr;N&r!%0wB;Q>M
zcG?s=Ud-yv&rhtnFxZs!9$or1c%6`GM=l@dt#DUs5{PC@)4^GcVvO<o6`6K~P3W70
zS-u_7tJZUqg>RxXuAi`3En^_RymlV?T5(4;03Es{L9`u6muXJhllT|&QH=Cz<7?mR
z(+=qn?WG&bof(wZ8q046L~jY-12&_UiVt-<t6yl>KbXr&snOa{k(ib+u%@c1){hX5
zX*6g;-yM3`l`Ymp;k!y=fvj+;=e8M1O)rY>&7rv0pM{`NqsM7hlMl$h>Sk-HwsLWZ
zaDXS910`~Y|MEHGosz|KBzo=@cRp3KY}3Awl3JlHMzAW7%r~`K36vM2U<4BbU5vQA
zVQZ@ldeKvdL!Hcy^{ir-<7JxFn2hWri9B7T)~oHG>TS9PRSdYr5Ad@*u_*D7-fWML
zKtd0L7qq@yzp-~6=rnDrT(-+-oGuo_{Dt3`Ds1&CMlWucu}))<Fjc0B7W~%GuVX#s
zdx2f8^mmG1Ssw1Ee^hSkQ7%Vzm9#?5OmlinSkch^N=<ea^^6V2WjzM89;zs{0D~7(
zBI?Jah1WqszK&IpC`H0SC6r~`Ebm|r9M?RgkoQ1em|hBxVWvEc6KhX+;krmZALM9O
z^*~$$N@mRO%6&C8Hma@;@mY!8H~Yd77ZJvpu_K>0dvC%Y#-yVv89o@$-Y%bgJy2eF
z6?{_>L<>Ti(%q+20$)z$=~n2VaqwF(FBNvrTwoi=;849U@Q5^<5EE*rsSOo80sNa^
z^52=Dch+%q=I+NwtL4Z`+~vy{z3a!^`$mM&Ji4L7hVtp9{$XA1CzRR8W~!fuxb#CV
zW^~=h%z+aQo~v@YB?oRkgjyHef&{@LA<`0<0;`-D+^*hLcrk8&**Zb2Xeyw@R(iq-
z{8A=Tvl0lQ{j?|R!Z9DYd>(lz2e4&GsapFV5!nlKhj-o8AlO-)-6KHT?Q5c0iwkwr
zuFnowo!DoH21MRTY{C~3QR)lo!n2v{)+OfAjY+Vbl-(+;>&JH?BVV&el|_#`OK{u7
z8Sc_TT$;cdn-{c^q%Eu*v2=d=UYjN7RQSp|qrPO$C^wy|aCf|?vTYTLX$35+$88Mj
z>SNu;)*oB5`8bOV$=~%``CM}@iC6E%YPDUo`Yf3g1|L$vG$_1$&OltgJG!yif<5f$
z+lco<g<J~>E|CxK*WEyR)pL?dCJlJYq{)UAZi@zOAv3iFxfyy@Tx*?ni#?#4Q#mdi
zdM?7iuoaA|^E$oD_Xn2vOR`jay^j?oHH(vIgfTH1UBm0V7o2TeYcvhl#s110g$sPh
zqj#7PYtWy4Jb%E?T6oZ{8>N#YA7#5Q#(HBezM(2aVwWpK5X*%F(gusjW${Ka_oVAY
z#>y~?8k#2WM1S1jiSBpU>9CvO`xw-cJI(b#yOVWLNw3|v;j1?f)UJj3^85tfXPfTz
zDPaGjt<_rTY{9_BrK6I`<X#cv^BP6vFj}%$OY5}3n~g5uzP}V^*PooX+zRqTZKhx$
zadx<^-5jzDg{gXeKPrT*GRgYpfIZ6kW!3ngPw#dC%{u*377uT)d(k<$VlhrropJ@c
z>J(>pr{~VXVTY$U!!m<#490}Bli~w8ph9sgy<~ej+_AbPp}-HB%pRnjbB>vQ<ekW6
zetz$l(6ch(?{|+4#rH1LM3ha*@2u{sj;3f<gM;RaOmavQ*n!e0Tuv>DM0`DyPQcRt
z2q09v;rnL`Ln}y1&#QOam%0W2h*c++5L7?9Bc>oUK@muHW`2bD#vrwaQ{0;Ih-7~}
zOMQ`T*h`IhZpM0-3FswX#~@8rsaWZ{FL8&(*0rj}xk1%u@AUWImb`|cy4XcSM4UOn
zFw3GWPF6^X2ejeR72N#}5bIPu#czH~!go_vubj1Z@{JvfUWL1^o}u8dWJ*s<NTJiD
zSiXreIBU7^z%V0Rkh@n{(7G{QxL{mLP=W;Jer=#K`9VCtIVVhop6X0z)+0%Z#Q?6X
z`DYwY9m-pqsp@uK-dIy_m)$+Z8AOaZ_d>cOuf)-BXXddcdX*uLLN$5y&_KakH<xx+
z>YV)i%5llN$~Xmb6Z?wDGNj*Rq;Qu9HZ@9{4sO-<u){bBToNo?eRHYD5~6s54<QmF
z>}6jE2@g(&^{wbDl;RZ$qQsK+E~`ly1G~wjL&Px68SX{DA!q!Rvd(uJvUJ(BhWGTl
z^If*?et#la?RzzMy1=wT`+$eQ@gkB2PmM9TqeWf?+Dy~OqlJI5=F+DtoJ$L_q}zGz
z3rhTY8PU$``M9_D2wo&D^WTzIp^6kGh#PhDq2Wp5pyF)9x@@zp?Tp*^NenHh_xD9)
zB%JSqyjPxMx3+Ajxw4~SBYbV{gXI%J&o_=Zyc{wjH8ZA*29j!3b-;2WFS~W<tlBH0
zoA5dH;#+ptsTK%4x0BG@{pS-i1L@MEqK;4-$b@VYOK)h~c^7s2a;%N1SyXhX%VJmq
z<$$g>zApJ`UAixhbc~&!2K`gN>`>2goN^zaRFu$?cq7kJF3s(;go5U4!%MsF=J0S6
z*-;N&@z2D4vfONs(4Uoq7p))PTnqTr<%)TdP-0MJ>!agiyWQl}%4<Gl2EQlv>Lr60
zn^kzNeuMu9_~LctlRCi~fNbXFp)Q{k2%h_gkcS9dS-#ge#ABVy&^yH=6!$C&9c#!V
zT%~hH%4nCxiB23uJVb+~%P|1_j4RgWOZ6`0Ace4a`u#ink}QtCoGN*6SKzZ0PkH?w
z+u^F)?Lrgx58fth>5dX;;q5_aaHa%Q(w6k|;0$j=ilbZsRippM%^dB?ZBAu>1f5Q=
z1sBgT*xWAkdE(i*QDL4k*}tt{MD&qz*38H4#^L6UC*$MvbtV&db@J4&Q9TeJOOcI7
zfmHBQtN5aAeTDf1p6|K6?H9LW((hAOVHy8|6vmSMZb!|HZX5rJqM=|m>*t6wJ%?2l
zUXdC&-|(}dzUqGOAs&T4Haezv=&pO>^_KFIxKSoY!x2q*1By7OL;yJW8rjgFl+bu|
zBVM(G1IMtF1IKv!OWTABCNBd+80;jw+KVMG6E%(c#A+*avK%g`Dz<2*GAYUcAA}^<
zx|)RT4)(1i2e-T3Y%U@~dP^h4(2TF)0}<+Z7r##y4CRepy*PMO1=1PZhQb4R?3q|A
z%cWosMiOewq{bt->XqT-8$|2r>`{C%JMFZk138<iQRmt%6(;=#@v23Y6=6YB?0Y@q
zx|KAsqhsQ#q*kuyePc#wwX`wQbuS&rREbfpDP}<hFp+bZ3;7aZRqhoje2)Oj9V?oY
zP#juV<4~4;M+2|@gJ0=-^?5zAm`Oe&4Y*>J=uqmr<1b1}tK4OAM}2Itcm{{Lhiivb
zS@pOQ=MBueRrJmh(Cfbh{%LZwnb3av2j{kHoht3TJf(FGe2Dy|(Uc-3)}r$Hj{;UW
z%8|0piu7FkAvq1o<#N_q?f!!YYqzexly{gtGM4ZzAE%D-+*^Cw%ft^-AW*<rdIUen
zo!xpS)^)WD(L*lgZxKYQH-vkG6BVmq5t#E@!yXC*r`KDJKb6!O6l+`592hX%qn#fL
z%wXOdnY0|7ZfnNRDt*Vi*K@mEQpXFk7Uo-x!|YyVfT-(U7r%aIT&K>_<K=3#+Ia1k
zKKHG8KB)QDx4G)1w2{Gp9#fH=QOq|H4(iVX4Jl*k*Tk9A)FHu0aWc~KTg%Ys+^>_%
zgVrhZry}aMRwcS?WC>!7`(4vngI4WAU#`lU%ia({N!Vn<&4^@&C_I{<saYO%&g{XD
zrxXsF{(7XD!-^qG+4qfm$)*{nD0fs=5R?VOC!S^*3JM}kwhf^&sntEKJ3|OzS$SNL
zCR;gY;KufB@Ij7ZRF=2ip3;m&ZY}o>*wydy=6`k5fFsy1H^@kkKno{Ao6p&1z);+3
z$~j>w_fb1!31+=XJc4|>&_OCGCe{*5$I&U$aX!U(LmCH%k{GV0Y=q-uCT1hjG{aHT
zok$#K5|P=}Z}L{-fpdfqism*)o?2$1;F`Bd{w1}>Bcrhw(^hsy0zyqw58t%5ZQj3^
z+p5D2TYYlntYyV~pv>l;i1n<*v>jr80A|Sxas#TY$TGP@kqy4vxk;o`K&t?f+ipnz
zrSnUbsxqT1%$*@)L)r_MPz?Tz-)vk0_;Sf|%_Iw~Vh)(kzaN7)kQoLSyIJYmNaF`K
zs$8s%EROP(c2yG!EVn@{(F}F>BbuIeB@+iK>#oLDj$scr7qGDEfhETMO9bBOJc_ut
zwls7ifivY_t*%dX)D~k6GamTt1lYsoo9q3P<B2f^f;@wT50D4aaP(1l--{q%J0^O(
z43KQ#!YlJSgyyxSEqQs17i7~;Rqm2_i@4K>t)z(+@{rV|>O;s7c`EyE$zRe-jkd|=
z=<9Ed6mvoZ^aW<1W<B%jCmZp<uq<}E33Vy3ay0cw3$kKvdz6l<+@=7vL<vui70X<U
zHbK_g$&D6?lp>ABk-2U+fkoKePYnp_B&*d5h#_yw{?%e&=t7Kk<dgS(SV%cwXp&}R
zb9iPWiGjenGf!=V9aw!gKIY=~QjRnLp-8y&&K?VXONaoPbze^m&nSjBC|+v)ex16e
z)4j#Zgg~$xPC6ze0xOt`u|kogb$_cgl-=4@nLseCr}A_PqF+BFMpWQbc3@b~Jk*))
zeSJ;Qr<@kG(3+MUFS98=&LC7%$Z?zP`!fmd*FH$d1!&`$5GvFON0z6Dd$n|(+EuN)
zcFKW^_Zds0ITFrUms=KKBc~=t8n;Kd!J|6eVIoAm$JC%CBylUgiS7$s#OllR`>9T$
zOPC3uCFp6$7!6rj@+sgw0_HV6RZl_}fj6M`wD<dMQ~8JWpmHC0xBaU@2#<6m1dj-u
z8xr=sZzmpx&AjgkzVJpT@~M*0dl9^T!O~u0L@L&`qWf`D(GQ+B&lvaEc%;Y0iCB;V
zTeV1=fEI`UfdD?z2C$$oeESaq*A&pRbrEgkCkF%n{PPEp`v|v-!apr1pGW|hOYgOR
zvbz2P?DX@50aRPh^+w}2vPj=15<uaQ(~3VB$zQ1EFgG}TNGm+}qgfDSc9t!(gg+{_
zKcSAudT_^&Ns{b;G(*A4P9l5stJ5Ey>8AmF)+%Y~<gDqRYtNOyo*wrO`_E4Qe#{Sm
zseGzE82U#u9T;iCGaI*l)NOwTK*<W+(I1|X_<M5;1blTU%YVWC#X|qD4)${+D@GJ}
zMf#q%JOogfh_!qhZIx@=sXjFN3cw877G{3xLqwnaO#^%Q`ueUZ{DZLox2B75r}T-(
zWc)K%A~|rUtZ<<MB#8d^Jw2DbG@v}j*gAnw7IjFtP&$N~S2QE<Vd@q4u!*3j3i(<|
zQMPc1%-`}DrA#PG+xSwY=hW}I2Ldeg(=K)Gw;`4H2E5<y3SURS?fx|&oUwiu%9fSJ
z|BQC<X+jWS`~P)}onikA#jkbd|9>b_T}3^?s~I^?m7bZ&Y!)RyvN2pB$<kc4^mFo;
z;0K!pM2<16Na+m(lv|{hodYCJG%t*g<1hcc7DmcJmr@1I#1Y|}_+ed$X#R+=0-|k@
z@K;Xz3+7VT;Sgz#e`H}sQh)+}Hu}ALy>$I57AC(7gosID;56iE-q(NW0<cd(1H_JW
z6}6Xs8&>)@reNK$g-%`hz4>4EF~25M&`BHU<1}pL`_IMb?`u(D$VLg4{jNQtVK4@(
zovHHP4~yDg<CYKT0t6krPY$gA>AU4Q=%bCe{?+fxni5czUgfw4`)%UDg+|kY(^q56
zw?CSR!GuC$_AjH}ljw^+1<U{Qo1e=6xf}M=zYKwXJsSwHE$HJ8t>2A*Pd=a-D}_~|
z`!Cc#L-jup>gU7H8Ee7&At@*-BQv@|+4qATJ$Z2>)$!eC-EO}{&_22d6pGEMp?zvU
zE-=yo*~<XwX#Zn=(e=NVy3_v&Urr{={}&Pj)5ml-fiTH@ON{!DA%vo2sE+al@=+(j
zr!jvFb1JMh$-J!kMEU0$QlIww^#}+zq^;U3)nT3Ik4~k5x~}?Q&EooR?Rm;zxDt53
zG5>8=bFKuFNhZ4aZ`16buQC&uklZLnpZ+$PME=+D&)wJmolxxGSmar=`FP5a69@dk
MFY2n5so327KS``}od5s;

diff --git a/format/diagrams/layout-list.png b/format/diagrams/layout-list.png
deleted file mode 100644
index 167b10b11e37e761de81de8fa9fc8c5c9a30e4f8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 15906
zcmd_R1yEhVwk?VU34~z5-7UC#aCZp7U4mO+<H6l6c!GrxJh;0<2niAh8r<FC_vV~?
z-o3x-)<6H(t6TNzr$Vy#THU?6d-a@ij4>w>s>(9Yo)SKVfq{7@Co8EA0|UDceqxc~
zz?Fsf9(v$ESXXr!ahUQk;w|t6#Yy&!D+~-S4fF?APM!J?H27tsspF=jq$ptKXwPC|
z?r3Vk;%V;$TEoByc?y7!_7-j?<ev6+4z2>8!jz9K1i)wLWmZb^$0lyJ!jw8ns^k)m
zE*9k6EL<#Xlp;^b$;pLW%q<1fC8ht_9efj}w03iI5@2QZ@bF;q;AC-hv0`QC=jUf-
z<6!0BcnMm(bcHy$nRvc*aHaakApbFrq=l=Qi;a_;jiUoObX*ftM|U@2N=oR1{`Kb{
z^K`SZ{9jLUaQ)}9zyevJcUak3*jWEHHs~q@y(%E#Xz%1=;pz&;7vUCqZ24cV{jcZz
zqrIxNqnjhx1s5ALIR`fj7tq<w1Ufqrj(_(4?_2!e$5L^zu>h-n?9Kkq-v9a9Kidnj
zLKps@HsT+<{CE{?v&d5+)_=V+k*Afb9DiV71lQ#xUuk;6?qwtS;7C36Q3R_deiuK`
zDnOP!$ZOO2{G*xt$15pJ*eAX()EFq$@2JShpVDhJxiPTPB!8z!R>c;R6-PmUyCoww
z=x-jj;BEZV{Kt2{n{H%G`Nr4Zb$o1WWWUhw_R^WrPc{Gn{2fv37$1qTnv;9{mV=R(
zTtgNQgAyOGg`zs7bRbSB21hLhm+W#5i$xBL#U>0Z-?(KU3;)<zeN5!>ZV{ar18*y;
z4d?~~dd(t1?{=Ib(61GFP&@<O$YJIAZyxWa=fD)spRyFdf<dXp7_7)1?@nUpC1);#
za6Qg}er*>Xde_(sJa)1<9uYbnn8GaI$GeIu0qPE3ct2pE(*cux{(sP9^(vpAZcT3V
z;u+01xii`6HNUg3Fl-%4VR~I@+<Cq?=QvmQl0B&2aq)gi&rz#Pms=*5xYl-hf4X9E
zYqB^Sn@q?PBc$GLRy|9%$!%+=qCK`tA&uovnO?o^bUDA(tC7AKB0=|U%`)BE-Psze
z!Nfw<kK*IP7xVRtemfjTw^t`>mLX4H#5ZV{>9%@bprfL?Dl?2o);PaCTIoI8nSPFf
z!l2*aG+CtdoS1JZ%TObFs%yH!NYwf}`8Nvx``gaWP8juK<?M-knZ-6=y>bumZyr+j
z>gsB<$3YI$>+*7g<{V*fTpPdZtzs-1`GAqjqt!KLN}r4UKUHS^b=DJbYocdAHI_u}
ze}C(bBNHq(k5)p-<2^KEhr@Vvx(*iEeljMA{zt&~`gAz;rE47?M*^)9VFDjHxflW*
zJiJ=C#EMcT&oZ;}pJtB=tx|3DwT<Dl1F(ZGsWNe-?wj8`DL(MoO_yh9VJ+`WS4fm)
zVHYc9zUv9cM2`sh;JQB2Svy35s@Rb-LeW|zp%fxvMM9WIj7zU7hFI<JM<8o^wuY`e
zSv~`7m^YY!pWj1t>G|#~RsH$M2hLBz1D`yRO!U#IsHoPG(dJUc80T~yOuBog+sp?O
zFY6@qDEh&wj4F*^<p@DAl3wcZ$i~^-H#o0i92^`xfY+|y>UGu`goOV~<l*)tJh;tF
zl9>)yJ+?<FgX>iwqt=(NhOLCFLlM|?lvwO!K8NqK{C*V3eacFplF1Qp-yX{roS2xP
zQ_j-)W<8NeqwqHw*^D}>N4&w~V8M5Po?qWH2eM8exx(qVAjIFwhTW}`C7uB`iA3<7
z-9S7AJ`NQkFRqJenTb+J5M2hWlwp-=??;I+CtH`xLlY;@ADfybPR{CY@zl|+VQ$va
zj40W$l3(i8xvUK}kOi`!!gYr})2KGqXC)4!-TPgOy71}>f7)s!9TTEg?@(4?-Z2IY
z9{02@p2BB*IE|Gx?)SUhnfJ0jmSb5Z=_J_L@Xj)S)7j0EWf8Eju*liN9lz(jCuS|U
zm18JTueG1U#eOpMX5cBMGm4riM#lhVLUq=l)!N~>=)-NncQ7;i_|?^w6|hOco&$wG
znCc^6_3HOWGkH^(<7KIpvju9ibofg-mJ+%`_C9b}bd^PXel5$N5f(v(8L>r6EIu1Y
z%pb^g(00Ru&aPGX?PO9#R7!DwzR`ln8F$*UXElg{mWCa^j^=P&IC~TK)|LRSp|)S_
z1y-JpX1;VxajqX>nC;Y8HA?n1uhqV1BOGR*^dlr;Rsv0E;l4WzH2KH}vu_A`?CW3o
zk!dXRg<K)<JBOkXIE9<=1#886prfKlSFyI=R{P-67{7bMj$iO_e{*om5bV6x;nRvA
z5v8Fjhj5r8?G@Ie`&>hZ4~KwTJdo9(>8~m`b5}27LZHgVSa#J+K)2i0kL3!EZuH~y
z8*wz+6$>KFAXK{bQA|~q=g22!dcb22H<7|Gw?Iw=(!=z;d-Pj8)gthif>ll4B=^pm
z_C`{`aAWSc$m3#me2Y$3Q(LLp<n_69(#Imtmp8Am9L4C1Kbf`4y#uo|Gc!xkaDkEd
znJ*JZSp>|^r3HC)3?>aXm|bes3AZVi+g6(&D~-lE`~xw+2*L8-7k&@-zM3&A>mwhe
z3L88UZ2Rrw@hhp+kp=c0CE=VuQzJt3QJm;$4U;_);;XGEJ|SlbLDXZdW^xd66djJu
z$lzwM;mlZ#+P<boA8~XM@z`NZGRq6WJtq^v3BzA7bq(=(YJx-}gI4saM=?ALOTi}{
zvoD&Tv5Wr2oSGqI?FE`R`!5zFA=M{SY6)s1zQZ)+(M@b{QW=MbPGK5jVd!r+v&d=K
zBVNS5Ay?G9JM4ariHVs$oXToB_A6pIk(jjtGIZ1>7~VM=>H5p`Fy}b)d82fwk~|1F
z>isk|vpQT7$|DPmSG{@9S129!Qamv+Y;9L6x*5*GbTE5G22ikRjh|tA#k5~8{f61=
zUMZkQ(m1DFswUv0p{1NxD7dDD@X)L$JLyqTzfefl>F80W(1gv`cgWW?YCc5&v0A;s
zd=aQMmc{=IZy;HlMvS5Ui<QBg`~58>i(czXvP+X}0ww--+vyL5nT#??FCo>F@p0h$
zQ(28-_kw2jV&#dr-v+}9)*0suu)hv~L70GWz#ieT=yw>gPsatQ_=1=bXt@g4=(H?8
z@`S;Dt_4!Vk?=|K?b8lHN@3cbBd36(7DYQIxGbjngl2Z?ekw+`nO8%YL8GpS0Ucc=
z=jmn>FXk}RNJ6u8s&LLsbqV5VDToOPx971WP4TE!deId&5OmF)P-Yul>1a1=E!~vk
zAWlLsA(!vrk%k2Jvp5LYBa?|%^US}+zwrqMI6#KjM3U2TG$Z7R%I@z}vb|1EVt2FX
z=;CKF);+oI#xFG$e;eDYmYjy*8H(XIFICkWMBuzv%a^7I;3udO>|OpkC1H2jjf{5#
zA>^@3cj4~e$6>dpIZvXG=?r;0l*}+>att>U5KK}yaC5bhYsSqD31LekK~^1lxnuFQ
zeR)~VgcO&pYD?9Q*XN-10(8hv{{t*%`&a2I25MsZ*t;1IWG(jvu1|lJI08!Sg{D_4
zVG+A^geGF$#m&QV7(>L$*sh``c)3Uf4JC!XwSf?n-q1nuu5Fm9P+nRI_N$!V-PtrH
z>N<oC=@Ub<tEnPJ1i$U>cqbYbRscm^-|E|=Vb*rO;%P#vb89%{_58!nX;}vImJFqn
z#|)wNGB8Fwe$3Khkd`M=A7)g;C8{jL$6}87xPEpT?#gI7{H-!X`dIP0g7h3ca*rk6
zkH7Ue9K&2H)`LJ>6Y+$uHwv%ci=1>=^IXO(a+YHH?_iSiBzar#j9kPA=PG-6Cck6&
zc*-<3?94@yr=6U+pMoFm4*b!Josb#Q>I21wvM0%<HI{>rp5t<_K*(n3(Sj++{BNtm
z``1)|Ql1BxG0K%_+Tr&~N%e$VNmfy5$OXXIY5tCOZqo6DQR#eoqw}{1Z7vBW_xxMl
zVhGwYH^Wd45<No^ZKWeMyDfW`&&en+dxs`9JYn!IhKAzsK;t)g{&gf`p2EDR-!RN{
z8g*z|{2%<7y&0!TN4mut4<oT?-eY|)sFP<wsJt?uXEv0lA#F?^kE|@}a?XYQrG0+Y
zVYoXo3xI(Ojl##|e7W@4ep-S{&2Rd*A$muCPwbw$IEb3<N|SS=r3xZ-<B|u!awGo6
zA6m|4oS_S=B3kJA^n+!@aVY^CnYiwQSz2b$yU)x$6a=)yY%+0;X3=Eq+x^3+*%y_9
z=C9z%{o}=3o_vSk)k&<F6PtFU>?{9WTQtX6iGO0>SAfeB>FJ=+Snnj8)Ft7gSCxbm
z_r$m!4S%IK^0<&UB!siM1jUK-Q^}t+$yN5c5C__X&Bfx6e;TlE1z8GZJUe7;Ih`b6
zFjZ`NBQ`v)$kkT{wX$Djro2q~!eOm~u-uo8E%LjDBdiz2X?Ai2y$Ax}@;h?iFM`r+
zr~q{D1t9$8zHx>O0MSTLbIwaUQkHCiRa>aGe??`31)4|(2<lMdqa^}p9l(#`6xf^P
zXI2X30AiG43~9z}vQX5&4iGwT@#4%|DAtY#c0=>89;!p5TfB7{PzFH_f({B|&?975
z2A^H8W6Efum_GuhP(QGf97+n(lYqJZ|K<^w3MRRzeSLkQAtBiU?u@jwv?@75SI27r
zA?v3yztQK>)G$q>VM|2t`2f@QB$LP9#o7cQD%ZaQ%NT<af>@rrNx8wn!AGn8R-+le
zI%#UG#@{(q6!nbF3KL<IksZv!y&*(FhV#DIcfS-^a{ANc{#OEyd?`Qw7=)mfoBig^
z5*6XIpRVJ=csMw2_qSK!;m8PC-ZhUSyf^{A@ZoYb&U&vvE=fp>c9;S-M6AwkR={JQ
z8Qt+uQ?1*U`Wg;Ab^H(#tMzob0e0e_Pf}6TNR1g-J8$jQhEgVSug`XhZC^5HA&4yj
zLL=V7$7tn`f#Bmjd(!b99`F}wmAjQFRya7gc_T+|ns-Jx<BtbT4(0Mwa;^{ri)*q_
z;dP7W5p~sD`7CoA#YYxtNo^dQJf181a5imJ0f-C0-L}8J{A%`a+*NRf-vQ6$Bf`hu
z+(>gf;i6e+a5e)><cn5mXsR)t;z#iiw%WVB`o$)@5kL+4V@aaqasb~k8)~x{P6dRf
zyEQt?^K^59_5KbMFU83J?(87E{ho}=IzCIv*7NTAjQ*qF-L<u}lao_t;FF!7B|%=j
zF;NmWvJtwy%-e+uY0|O8Z|WWX=r_7_j^z4X4^kWfAfJH$F2eI~V5X|y){nww_npuW
zOkt;LYHAuqiZ)>9W~(hE#KmDSh$5XDY^PcH*T6mouhARDzV32tIYPv0xmfU)kLHF$
z#bdz<B@=KPD3r?Ewrik$XkwnS_!~<iC~1~;gM#bsv>o(Z#A)nv(fDv9m+!~F(fi#Q
z>6POTPc)`K&~7ppe4n+qnJgk<H<dKi@(0vwAW$l&2W#Mn5nI&j^s_`5`q;}?Xg#tl
zs=?z~{6V&HvaBk{V%UJiq?|UeH85+$20vxmszp5fl^B)S{p2Z5gUec^#QU@DUlJMY
zW_`EkbB-|d#WI&Q<046uQQ7tl<_rI1UlCPjlOJ!vMhF(Dp?^e|9|R~Bd)hc{7!mLw
zyv9nMGV?0wCBQ{XG>Svr9oz5fYNW!Rli`K}dw9cN`m0^Ok9uP$MQUPm)0!p?`5RyX
zFCklnX~Zj9=a2-29lW8BEWBa5A1cRD)2~^)O>e`Mse-|=|8tqhs#mxB!72yS*K45#
z5-BtH&E_PxU9>myg#?_x|3f>~{dN1B8cWL7q8`eq8LdRXBGELf)b0R>+FSt?ssqDi
zFp)-{QxXQAz|KsDC~qC#Ma1(EMnV&9fzxs%d|;|fk7<kjzN;ze()e#|4S>IQf-4md
z5)u+Ar@HW)Qw{C`Twisn;M_kFs3KShg$x%+y#J)vetSF&SZqJqFBvjlge^(<FRv?N
z>rB|M05g}zrC3{<FOb8|DpM~U0wCj~3^tvzB(<q(LL&bA<?gW68&UV|pZ0Tgohf*%
zhTx>pNN>ubRNh=1yvE|So54t!QHYER#$te@M<{l3zg>+JC02g3gpqHc6Be^@aj@vY
zwsCi{*k6K1z+vp&;38k@pAt=)=5pq1`0dpsp1e(6Mu0Z$?n&U6SoQmpanbyP75Mk@
z2Kab*uS?WvdL#;#c$^tM_UHYJ+%M?B2_BSXt~Tb1`IKF&;*gGoOXbV8wLY3DPYs8P
z%Z#x%*XG+y%-SWrZ;ZJPgN!KSq99o>O~q+7)?zV?yOzvBoy?khd_s#~GME|S)F@6t
zqmZg8FUuUDi;z^&b}cU#gjyG#T#9Yg>#o~I%-G0`p?WQZJg_e0bIIno&|-Z2+qPox
zC?W*=q#|))RlU4+-1b_WVN_X;oE^G^w?C35Rs=spw;TCg(YTmo1-|-}MFNjKQcjLY
z)x7BcV35_di9bXsl3pc6y&m!qd%5Eu4V#ppWB#fT*%&`IDlA-Xmz9*S3x|Kq6dm0@
zKIv5en@J3Qbp4v=YInLjpf5j+#pfH}%_F|2vm8v(uoz0lyb)x#`XtE+dua`5<FOD{
zDz?*<g(*0;dAL;qzNedNYfz@hCK}Xz(EeZ;A!!VZK|ClpYE5Gbd9@F7n61(RVEbyR
zO|~>vBWcArQsKXsJze{EnL6d>{dD=`r4QmX&06RLe>hEYr*|+ZqH%jYh>|GKSvpOE
z5T7|I#D09bF3=-vGn&DjVh$*`6n<MLa}5>)5*H0F!4ZjG?b}~ap}_b2&B&FhXRLFn
zei2q58ay>tnCUy&2pJg8k5W;%y*+_C%gHJP*nRebMWL!`KRnS2vr9v(SX&MQC4^}8
z4)sxR8A(>0`5fl?*z-SS!9hlXI}G&?dv7aT<WT~!gu=VC_+7i=Tojh{eU6+u`>tQ)
z>~*Ps83>X2j1e$|TFfN%hBjcBqVLaAqtBJghy}y&3-MXtFl~ga4*GOb?%g%SkCkcu
zvDykbd6pE#nT@jy<4&wdC&WG~mV+;B&4CE;h-gkCr7koY=9;+;{tx%^4;`=3+xk=C
zA=<BS!dXAchLRAb@3~(t!3EXKgssloWz*{9{Yh^r*R5s5nS7)5@uQ1E&uGzbh(7-Z
z<W)-<oyaS<jT#m+ps`I?naQAa_vR@sN5E(YtTB6Xy=M07dPPa9fWBf+0eoeOhA#8h
z`E2^(Oi#i1j*w<yD>Lq^*Ms}BD5-zXASp)lg`?Z>Q1^>k=@m0nRY%pfq2rU(ekiZw
z=D;N$F(C-^%5Jeum`B-|6nohDGD;-^iv~y_3~?zv@x(Fj=LHW!an{AO&K%0<$ux-5
zcRCbt(5tXeU-f&@c6IFdNfeWg0v`7>o<c&E<F{&oEaql)!?z`dm!DH4Vj48fVsM73
z0Z#{n2VZ^V6AKTYuKPPF48RX@%1?EaI&T86&-bEaqw(3w=n!|&lYM&H2X{^w+f~tN
z=^&d_ijsn(KR13EF>)7b7TkuB`8j>S^j?BNoukMa|3YR>1wh6p7B~I*EeY%3`btma
zGR=(l&l)L7bxS0_plq@;;>Ad!RyC?R1aai?*lD8MV|B%-@t>y;#b9<*hN<ym(cdgr
z-r$D~AXhfL9fG_h(AGu<VGI$z_tTR)Zvi5x$4AK_Nug^Vk2-I75NEL0Cip%i*{;^u
zc-@j4|1$2O*Sx$Tvrn0CqYIN^-KR>X>+0(QyUs`t=?S;(lra(Iwz!O@BvKf@V<aMd
zvxsILQa4M1TO)c&HA0wq`t5sUN|&XM_c{hne71+nziVxew0c^9MiX!*GZ<gkcdQi8
z)V)7qn14yG=Pk<;VC;+h+9<dxJ-QIC>U_0jsRTGof=UpK2OPS%UVC~r5C=I`-?Y&8
z-$CQhMOYJuvr+BI8`m>PYXPx-&(IveK0v?S4nV1Y8Q%F9(LsttnE(>L<x-_Z>^+j=
zRfSaLb9KzuW5%lEqPCKR7H<D8UEL|iUY4U`7@>i2DJo3EbS^p(ahg=fy@Z$$yJHBS
zvFjWyjDVAwmhN1WE>*p^9A44fgpBSdakI5POAwJ+ph<0>h6&^IUw6kFNNF8iA*id%
z$YF?^+eQ{8ZyMiA>{-i!8C5PVe@`^OfP}f|`+X$-(IEOWCF15w_IO6=_zwMAytvQa
z(UZ_3vz>7AClft~2sL4RJrWV7)H`SsHSC+6GfT|G`N`2cqNM))v>rV_KdVVZ$cWkH
zOJ=f{Yvw59pyZWZ(uQXfx!?x(BA`b0?m&)Oq7z9`1JD_?NLS|Q-LQFgXM6(nXShNY
zp1k`gMyZLrQewUWOOJNcQ-&7VyDz}(1DoJO;r1DQeABEIw#I%%PEC;`TsQ%J@T<p6
ztG5S!NVYP7XxwlO1^gAwU~&3FJv?-7Kw3V}dzOyd{RA<LrQI##x@C$r+3R6@NYa;6
zQa<Sn;wjbs9QUX<^?6=rD-VxZ#Fe(2EprHsc16*aBX`7$HK~oheN<-Bl?V-sanCds
zgDky3BxWg%376<Mm3NqAe_yOGP4I0nV>q|8N^Y^txP%po`q--<FSQ+i6{?%eoBcFu
z-Q7>-&&<eSS592PBt|4z$c7!|O{rgHDz&>SIRh!zCr`{}>@fQ@Hvg3{zvU8EkJA~&
z&8CR<A&||%tp~~C={@o-$wu7t?m(^2@gN=LvwOzOST<)?(uUsTZ&~;zXTBl?M)YA#
z^TcmW16@utnRs94PTO*RT49V<sd}aNp|QJ_j!VrCLrty(;pb-n=r<whoX+V40qU#6
zp|2v<lO<qIe$EI}rdp*`Ag~|;@wDaSWRsE=cd^Rm6{+XSr$QIoJc+#(j5MUrb?HyU
ziHzY=pB>q940#N)G01pVcD)~t2!xyd@?7*5!&OzB5G&~z$Ouu)o{#_qqB6pgTR0R4
zeXJU_B!6u0i1-yu#nIsx%jj6}v=(Tw4iRZcc{_f9)(@;OuMLz|kJBBX($tSC>0o0g
z=jSHpJ7J$)(TxYt8@y_V$-xiY%tKnrks?ste4Ra<$S(N4*qd!4nw`6Dav5E7^qAz*
zST<x3f!Ulwc!A?{^pwFAC1uqPdUfnDjP&V{#fXyEMX}iP8m@G+PA8qlVdq?__8J|O
zj&qbvKGn}9PlgLj>tH!K0o3@C!Te4kztq21fbJAi4=tJfy>}Pmjn8qL>VB#A7&ts7
zQWhOJz6<0GuOp%dnWbUyloflI#KYTXF#ie}*|B@uprd#K!)e%+EB>7Yqup}E;kL=Q
z9&+dj>ps7J*NMLFT{XUQ_N|y?UYf`Nt78tf$}rf#LMNo#sGw6&uBVyJpva0EB|STU
z^%SC9ncj+brGHb^h%Z`5oe^^7*22(<?mg%`_RKE+J%Z22aaT=;yzV`<7d*@+(+H&L
zU77AgybjS93A5b}x;92|ZQnQe8S5P%^49?0PNnFuP#JDW3<V5(CJiv`wzlGbTnAb}
zG?NW7AospN3)tK7)>Q^t00C4U{-1Adti;gon6=uU?0Xv|ty@Gifsus8cA42F0#Zv=
z>o-FbE7ue!#)FqOuFq5ZKFNwP!LGJ5?4IcQ7I3tt_)A$^UAm{>M5}1GN`@b$R7+zy
z$t?{{sHPp|#EfV)Ra>E&+i<|zDpwBRawtMoi&T(}tZ#|30m4JT8CZSUftfClGmIh7
z)i<XjCI)Pv>@IlZjpvz&vg$2geR^YBjjRKi<$NPX%=xF8$m4YBJ|F2K<nO?n?GIL{
zc-}+oPvYb{{1Cia1i7}iYkQfde}2|45o;4VhhK8V)foEI`1j(hH1zo<CYisA+|L5{
z;?CUYGnGv32$KK=J6UptBE|S)H&|Fe3~D}JiCA+K3aJ%;y>nlt3%b56l$$KP5Pb3F
zXLyU}{SAZB%LfYwyVIBn4S!iKB9-`nXUMmgd%VNi58C<iy~Gr=cE;r>ir<c@#2IE^
z=fuZ<Z}AJUeNFlCxBJ7Ss^{^KtMIYW?fyS)XWbQg^2v;uSQnALYd-pvG-w}q*ARNB
z#HII{ta4Qvr}a}i)EX6-OL!jY-~<fIO|_)@6IhHoyGC_AGA`~kZ*Qzm&8PVZ6yUU}
z54UZFu}MYTD!+;?2x;{DtOJpv0wju?``Rr2$Y*(0$uV7>=8))3u^0uXOWeF0K0-?h
zSVPGYAUZl+rs^F*LP4o_Sd5L0&@#n8WC00e+xyp4tjm(irZhcUfDZayjHcDJ-|5cB
z8@6I?2@sjdb$miVcf3=ovvZtT4&7@WVn_2%gSkj)UiatT=1>kcACG8{KWih&9LW%1
zp_VC^b4|5Gdm7vB!OKJY+g-WuQEXTS*{Je_#psyMh^3J?bq+d{B{r><t<7JiA*7|*
zFG_zsMMmx@nd61nZV{500qHB6t^);kdZcD!#CPMo%&`r>&7pUN%jU<a??L{U@M72Q
zQ%l`S0!+wZPy9|+G!0bY>PB(W+$V&L%|o2w1ag>E+Db&rJj<r%OjCu{`^aXLQW#uQ
zPYF1h#s+nu5*k|;j0~8QM5ps7#Vh&MBG!Q{n$pD0@AtyU8?2g|(BlI1v=-%ne3xmK
z&(lkT6}2XPLVOyQ{JlfHC(bhkS~C2JCeV@oxsA2C{w2l<D|nR34~x^eObn@GxvlQq
z+j&ch6)nG|Bh3F}JZ$E2gMbf5@4MUa=POXp6Au>JuC}Vzy5mShe8_P${^t5}24Sz2
zm$0XTyuAVfAEOG6Pp!bX|HTQRVp^`yJL|!gH_QETnqPHohvqG2QqrwSbI4)6F+8&(
z+u8Q@a@If3XN$@U8CdppE$;XC%NPpeRv2UY`>(i!MQ!X#EPbC?W#cj0*FR^U?SEOw
zL$j9`;I{Sj#oZ3deSrq&6O@^Nj*J*RV~Yq=DLdm*e#L_1%x)Ds8Whijj<N4|etsoP
zujw~7F3jcM{YkZ-^9Wm+XheUR38~Whs!I#xN*gv{_)CFeo9~0D`jke}DSX?68XYD3
z_2XhY+X74uPFXb)JxW=eAT*Kc`!9CCCL@eUE;R~pk7qSwygQvCn|2f%%`G*^m)SUf
zF_g-a^i124$zN(fkaWEQ84@OFd&diVB=Gb;@_Tzp`f+$BrqUJ<&!hBd4qVCy#O_Dw
zQyzTQXo`t>ls-QI>9eSl?NRzn1dch`!7Tq#UH#W|#o&sd>T2QKj5YLy#-r#!j2i8V
z9Yc@iT-s*Y>V{|$dfHs5Vk~l}BY0%i1m0UzJwApQmPxdso%6QInVfryI_Landc>fd
z7#^Io$X-blEHjrDW9-l@>c3flVZ3HBj^l+Vx&^ru+uTZ$$**|8OlEM+@p-~H{h26>
z!S!jB#k<Gz#s@18`6Mgq+kSAZrLZ<h5ESU$CR|`Jz2HLqxcnt|xa4C=1Jz2|BlJJl
zB3e!QQ+5=^%A|8Oz9(a4ME!BMm14`LwcEUpUJa@`FC%=t{Z`z6k>X*ed2>2w_AIj=
z_?*8uUXksT$>ISc`<QUIFDT-SPea5Tc1t=@F`Nt9UJ&I|JWomt&_bSXbKN*OO44)N
zGx)OZ|Camhue)rDhq;<@#=Z{==*pGWPRq5WYO)4NHB5x*>?Kmj!~$NoF|a}71}(`v
zW&RPLq*$b+vvrB8R_dP}ocpcb@iuz>^<{O{ET2M}<E_xUT^$j+LY?xiO870LC<B*D
z9uIu&#+}LcBNltToZqb$lZg;I4+p}V!La*#o3;!AU&k2}1sxj)Dn3S!jrJ4F;(i~Y
zO(5zd;EhWeHmTNX-;K9@NMLZq^?-QBA~4Gs?{^h0TTwEmFbQ6-evEX^*5ef=_4B%(
zzcMRU%)Ic&PWD7>5eJ^YW9DF?(e>|A8;D7-`V$-$_xJKe2|ZV@x&LFNn|;-ppC#_D
zNW%i(>_XGK`;`THPQH7>5vMep8CEMc#@tZFuQ<$;2(fVau2}DKiF$AMjP&jR4fcSA
zZag6%1UXB<8+{6gNq?ZTsO##4Vtt#;e?Fp_D4jj1`wc0f9GHdHH%@c3uj(}m!Ed5m
zHb&#_!smF(wNhdtWIM1B9R4&t6Rcd?GFB231Ifl)%oj*pp=9bq+~obTEpAux0~mlJ
zgxV>}!_m^IHW>TqI4!Fe1d*hD%Guo-N@B7yHkJ<45aLP)Mg#A3oBC_^aCJcSljfUe
zv+2nQipqJO30;-OdqpqN-!bd8qO-5bi|%2JORR#}{);0nuALV$LYGe84}Z7Dq@|G=
zvnAVci_qk5okXjY0b+Ie>}6cWfuN`7@d&@Cp_^)Z&N}bJDK4L~--iYA=LAkWmb*8;
z0&7D_qHOe=CgNScm=a{V&ikYn2eP>5J~O!fuv*$H2otNp=fv>*qLa{8;gQ4VtDSZa
zzcck&x7t&UIe34jmG}Ps&coZoKQ!Nk%!zWbbLBUR>R#HHRMoMotE;uK*LiZKvG#^L
z1o${#d*}o^JSwS)NI1RX1ByH<Z<{Rd29A;_$BZQ5*<@MYd>ueKICWI5s2e)}%V<Er
zz^+ijA)I6b(lRPIc!=xM6$TOQ9<20y8s#llcXz@NBnbwUWI+{tS+S(#sCR!Fu|3#1
zzNo(U*_tez{HZSANUN`l?X8~kZEA0I>xc>a_9F_}1!ks|5UibcCqmTGS+(>9Lc{J>
z&hvk0^e7<BqqIcLlK>4x^Pn!Pd851L(euCo2ZzlUp+pUhd>0_P?q3};KYF+T;NcLI
zO|hXKt^-;j<}D(m0EJ?-ASuQBK50f0>O-7B<<|bc^B8Tv?9_wR4ZRhgMVTpwkDl47
z2|U%~JCd3CF4-N`|A?fq0V#{o4gcW=4P8pvv6GSlSP;sIpZzMF_s3&2bh$a;BZWkR
zh)XIl82IQA^}wVTD961X=V=DM{C9W0w(Ncom%UhBeNKif?0al|dL|mkVKJ6eA+|>7
zeueGKP%yL$mgfaP5S{$l<i$<7QLUy(HamU8#+<}&_q~@g|A3-~0E(*oeDWj;=iqzU
zfkxr5#VD@J_TMVuIo^Z$Cg<CM=cqU=+NFL$k~K#N(5=K*?if<>_VO`;^h?*O<O)qy
zLYm{GRg^y#gJ2+Xl`0-Rk|PecVxvNJky0~W{M{nLVx!v1)Lg$uq6gEPiKnlZUpgV+
z7mASAsrBzH$ewqrR?)kl-!Xb;#C)eq6RX<1zq!uUhPWHHfB&K{DBmKg0S00~B;o@V
z_la&dkYI@^)q{kArG7z~Z*eE06uBL)KqnDYYqPU?;JP2yuJA52m?rak^Qj4q_GbbN
zozBm$KW&gd-~9~4UDh+I4(shG_<vd|P#Zj@)4^pmoClecxq3%I95NgnF7NYQ+?Tq1
zJty%9`dF%>ogC&ppAR<fezYtrtjhQ$ix>zb*LiiX^CF@t<Hf)0|I!O_S#x{SM$AAu
zzkv`n(i3D7JM^bhh|KwVl;!31ySGyS`@R?ct@T;m?EHrk!?d%9))zh`635x<!kSnr
zCQ9e0D4ICk)YN&Y9ajoz?APdn85X;AWXL2kW3LTAFP{JEbM`p6JLJ5{)J?oD%ujAM
z^S&6%rb4<P5pG~T{_&(>uHO4*GY)Z~VSiK(P4T0#(0=(ui8=+WmF_&%${Qtwx`U@m
zva+8wTHmS~=VevNhq&+wL1@Ke&1aYp8VDvx&J!($?B~he<9Mv@FFAkQ!j>^BzM9pb
zvt%*MWijyb;CC0~HR(b~TXk*}rxvTmq>>&;KuZ#<2GH3X9a4WVJC~<L*BC#er&4su
zqTj?U!?F7Uc=gh*911hzl#Dal5#3}#N55T}`8wyaFLnadx@|s<8S1aU>U^Xnn2Zk~
zg!5QgiD!<<H4^*%?YNzIwpyFzL2)aQ<|j@UR}<FKXMK3X=pKc%3LN(<)4#nZgw+tP
zEx=NKC3`G&z^?j}ODlEzP$+@AFOaz0LVd}uK=gEdJgjJ{LhCLwfh@Cd^qno8%o?Fk
zZ0|+QvI@_SRC+CK2_D)rcDPdQvg0b;D=*i`7cAkl`@cPwI)f0597_%6WEfuWTwPvz
zC-wUTMz{!lH)>PouALRiF4w6teH-~z^&=dM&@CUfszjU(F>;G)v1L_P$XHJE;UfF$
z!J_|?Q)*b7SO5UDt^bDS9@rGfysrY)bttH)3e`Cv2}AcG6h&5y1{UOq#zWC}uh+dN
zH^z+WA+|A$FHY8n+z<S2*9MqmR6<pVE~HAZdbMzS%>^)*sKm=V#cPeeT#uW^=H?ph
z`#j0`wmq|6oAdrvXZeYBgwi>c_>2Xp%Tv3^)H%%i{(;lV<*)3PtD-I%hmvkpaKJ>9
zjqX<~iC?0W!P-z_)M;gQ9!wG))yzZ~av(KmDz%|V@S4qNfjV<gCOA<tIS~QCsVx&@
zzq{Wlm);kFft+hYnPl)ul1Xnx6YLmdv{o=Fc->iazlA^dq3sJ(XGxl%UX$+|R(t))
z%+V4ZeT+0w3t}qEpQ#g|o`X*i*P}5c9H{0>!I0ph*@tw&jr|}9UwG??rE;&5lRQlG
z`MJi>uNrx2c@}S4_>7Ah<L@yH$tK;QCJ^jNhD-7Q2}u{Becy}9C)CD#u(3nWWbUm(
z7>HB6iziY)r{ufL#t^3rb19Ml9`JMDY!&KXmr$TJzp#(-;eTt*Z{_i1PTPS&Tkg0g
z*^!OPhsHdL{!J1~0F&AN3rtpNYxvcmU0P}b<O6q<)_D1rpkCA$HS$<alum2bpEa^J
zsCqIng^gGgZj(Te*hPK}a{tdY$Cby`R`M<0x3W4v3sECd;<>41uH=7V2@ZQggB%+G
zG4tGqxmGHAwdQR*t|jVPWm^7SU!@|lErcFDydsE#gq|rG?f_7^iFmBY&Md=!-1=eV
zB+`rj2(@vc4EY!kINXP<ad!vZ=Lr>cJ1ahs?9d4=y@b<0|0#A41c$CT5*D>Dc4gPk
zxXjJeu~cP@=j!Z18t`@6)OSt$ri+K`IRZRZ8Sbs0t=mzfmaLu}%?-v|k;i97C9{=l
z<*i6)_RpAHrNDF|<td#xX=6J-l>06y{T0fWBbP!zM8l7#Q^~o%y9o;=Ji08ruG@bm
z==N9A^WYVkVQge%tNTv7S>FM_lT$~fN==x#Ac?cXce>nC)<_0-fK^_jzS{}qri&$I
z)+xAjW#v}O4BL_wqk0d-EX5q5M)?#*ke7!RrfO;?_raaYw^{U$g`?t4v)(!z=aM=*
zdw92ByxQP-$-{hIr5OP-w%Luo8?lBjYV_Sf+}`FZl;<{Ev8MI~WZTa#xCobUk|whQ
zq`wfK5hjBQLSX5v`VHTAuvvK>|B&8V9(;YnVfB3QX};6ta)2Llu=#Lc`*glpqCEqr
zS1^p0+>5@wOuugBlX=KV`~0`sFKCLAa5?0Mn-lq<IJ*v127|olLbm51pq9<AbLZoV
zjue90%QAU}vKa7T*{dP9n^klp76gMJ-oNg3xXzV0Ru*131rm@JA}u9s99DR2?D>9=
zf|wX^#1D3!s82^W)oyfj<N<_DqHXX!X)+5tw?ZA90NT+bA>Nehn5H#6{gz^3OMW<R
zp)En1N%NxS?_TqnQW{I^sAB!s#=ZxySSz*22)vP2^U;i2khwBwepja7SZ!5mY_m?;
zp$5Uazg|2biY1-L157Roy@D<wxp-78516&3HSL@%+;3zj%^U<)c7=s&s3TLsg<P@=
z!bY3>P9a2p1xfbOB8ZNi&sxfBHhjn5-xw^bewmvP+Cm}W$pPSWam2{I!#cm)rX~#p
za;RYQk&_uT5T7oTXwdE2uzyjWGJ^s}59JII6C7n;$7veZDZBk2N5m+biUF=G0`NS(
zM<ryUbxWbDRiD>xL?`X!E{h(NJ;z}-P;gu6)O&b)2XbwIS(o@bYnEf{;3~i|P<BKl
z<1E)0S;{xw`6x&ErbEkCe%CvWkA{fsQj*ydf|5d#Zp+#D4p4UvBG3>C1CN{UkWx`{
zaTbLFIqW#bXM^TCtEZ+6=Tsds!Mhu-x<EKE<_Gf}ZcIi@04i!6Fke5B;$C2DsZ>Bm
z%u=wy0A%$JpyIv%H_dgfKwkSmsZDp_!q<Ecm+-+>e)mM+jNUeONa;v{J%*6`$6yIF
zRBmxV_zD!=OgtM_6zVt_BbQ7e2a)vQ3_v6hMrpy6;sgA^iNCJ07<_b47{EKc%6sz+
z>Pt3d0@TaN^im&XPejbTWFmx&U*P(`-(|_sn-JP3paexStDVnru%h__)GT@LPwmZ7
z%vR&DsyE0Kzk4X@U!*!i>37lG$96ELlI`e6`KsG)bNetl-zv&%AolxH2&!RhQxoK7
zEY=rMF_Uw=U(X?F{ixG7lP|+EF#r{An{0lu%F1zfRPW4IYdl^=_-PX+Hs^Bl8;$l{
zqIjOYE#Y<~gaUW9>-Kz?l=5igHszf~O`&&^3>WzFxRP*1RC$I&*-G793XNW@{eN2P
zO(x<j6jI!jmoOw4JYZI4C6p^YQ|T0_V4E?J(Vu#K(NWX!rlOr<;kl{I);~Wr*{HOk
zUQ$5liY31BTVMseo3cW`)#HFP1K0Hg_nz?KhpW-26(;+S$v2U7Gw4`{7H~;cBUh)T
zDApz+)rCx{Oqo*;5pH(B&)YX=bd>#X0$1tR+LoTw0m(k2W7ZW<Iy!Un;Glh}S+2HJ
z_)RNLZra?t2oaoNwW~#;mMhVXp;ozE_oM&@<ls<}uJ6r0Tg4V6{-9)F{!ep}bvp^6
z+&^pg3c_LD5mWmAtQ92{lErvp(10`BVq1Y-Jhl{%&CC$E^Km50_y|K5R|0N|MzvIT
zZRA5BtCTdi-D0hB%on<@al1v5$kxAw@;IWnenU;~G6y+gherRttV=;?5hwSq`|3&?
z)17pU6i3>By-{c2y{%bM4cWTgPRTu!_tq)lgcYIuI}*%JZHxz}ONM#9D(&W`XqOOp
zRfX^+1(OW(hrFVO*XaFsO#$pA7!;idn9z+&E1D=~GgLYq=2dw@abQeFz?g2Xb5fxV
zLV+=HlowEdgL@7wXRbJ~;RlD)0$PFlzqxtGw9V}F#kVYk=m%Kxwt4+x!+y0!W#c!G
z1QlN<z)_kjmfMyRc99wy{9&JAvo><)QF?24sx=DVDMKqCnL#9UVDR_>4Z_~wH;d*b
z5557BuR|SJU#bgT3<WG!B^@xn|1ZtOh~8IewAs!d#h5J@lL~ur`R?Cp|G7^7a+NDh
z(o{BAt)%x#vFte*T}1y4BZ^K`$A@rN@AEJzX@k9^{qN63$%y=DSC;d`J|dyN+vYSr
zInSP|{*^ohsQRN6*&@DF_FVK$J!B_%{K4CJ@{2E#_BY!Fgwb>^v%1%W-SQUTt^7l%
z+((w|?-|P$Xxl4|Ig;<q>*%QWa+BINqf==d?sg5uSlao9);S^%m30tLeOeu2zC<!K
zx~_jUDs9RolL>x`Rq=#=%}`=IQm(@dp*qYwX^uFZy*u!Lb`Xg4Q828<{}#di@2U5H
zxuY)XcRQo!USkN}$pKbJl8ALcg6O1T=fvPDamo%tYVV`c?+o5hUzMd2)Ia87gH8NT
z?_J%2*mg=1sznF@|M)-A1#F-j+_0%!2LeLR&bMegdZXe(Z2Q<Dcj!f^K*9%i->21^
z0Tc<J#DE0W#xpq>ZEGBvHSI%6dw?J-`N0ZC1z|?%z-Kt{`9+H?5om0u9N@afa{$AY
zi=c^#&+G2xrnsB=r}^(lf&l;-9H{`q_8nW=QyVM8ouo16o_Dtpu~)6zwVmlHG1Rxk
z*-!w>TLXCG<$)<7G#%y(j>OJQIxg5kogA1n;eT9j!f3$uxX5141^!&O{B$z+8Oi2G
zzw<|yEG=C5YsOdX1Ss(xq6jPd3p+opI;wx|>@T)I_&pf@Jo5kU$pHK{mZ!D?ToH{E
ztH^z=-@)!oO}%Mjb^AzqB8@BUWQJ*WsB2)aKwAL~wK2^5%bSblQ?ds4jhvyi+z$_b
zdY1CY;TbJn$N_39Bqs?i3q8bpfAgHQ<$fw}bk0M&Ovj}2RPLwq;uZmM%1tcm3>Ma*
zJDh|}^wVd8nxDn)uO?{`Ek~AiMr$>0_!#Sx>TQ@%3Zlj8W9{KNErmRmS6JK272n8>
z?3sYYKZmz)rZzT#vye|szZ%^HnI1F(_U&ozotJhcTZ;L~nPV1qY$%>dVt2Z~fS8NZ
zzBA=<Sou=$9i)cgtKPrLr1JvB`n4<2H-GJQ1_hTpw$?v<$9%z`F6jXt@dvX84y%&%
zJ%!_$&BW$fLqju(S1XAhZI>5f#uz2|O-lVf#H4@=lmg}KYe1-{KkFY!D=KF0-vMzN
z(ynyw@apXdWHu9~j1Kk&D}b`wlghv7Ltws!<LYO9x=JW~0BI`#Q+OYJ8!(b=;9CFF
zK}&X7)v2L{vWoA}V(kBkF3{5s1#7hA<c7r-<`P!|FSD(Itp3l7-z!ZdsPtw(t5H#v
z)@T_Jr_2rg;ex`T43KrIFEX-u0gh=IScUzX10m25jg`PQ+Fqw<L%~@jh$q_nMbJdZ
h6iA-@pWFQK0gXQMMp4Xh2l|WKa#G5Y<>JPH{}-IO`fUIJ

diff --git a/format/diagrams/layout-primitive-array.png b/format/diagrams/layout-primitive-array.png
deleted file mode 100644
index bd212f081151234c01f5814be4a4e4e1e4841835..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 10907
zcmeHtWmJ^yzb*_zBOpk3Bi#%=gp{;|bk|EaNDU<*l1d6PD2f4!fRfVE2+|1BEdqjc
zog0|<egAu}b3X31&WG(<>fF!V&z--xuIo4PI$A2%@Tl?7(9o`_sVeEAp+PFZ=MEeQ
z_&0-Pkskbq?xUxofcEYy?J8*Cda4@xprMhoUHnB?(_`HN9fqCt@A}@=)R46GaO1PK
z^RTh!3v}}Yz0uI510}&nH+x@e#y~e$cOS_>8Rp9#lHl{jV}54F%PzhyGR${1br=;r
zyzLo9`9$~xm}T)885yO$?HnZal$24ggO&`lldrF*BtL&ZKmcEWFrSCFBfp@8gap5U
z5WkQRFX+MR6Xfn|9mwnM!*Vsq-{UCR``CIrd-^(ixHDdiYi;A<=PSd^d@<2Kf3DW)
z>+JCFOzu9YZGjE)Uwp$a$S1)6&)DEq>5He5iXLvB-u6B|V0>9o>C2w~<=MaUT=my+
z^6>Qlr{L{utLE-&?+sq|wZ2%LtPtw$|96l7K9;t(vpv}L<=cX&xBv4Qs=qY<#m4{R
zAg)gN@+mlGSv+a}f38dxuXJAM2O1iurJ9nwejxg<Jo_rzTO$ZusKVBa7S6^-h82@M
zgvTAdd84e!m^RXtJkTnY&3d+pB9f=R%2d^}67CsUtJuG3zE)6Q^TTJkt7mdmDrmX=
z#nI!v3C*=XukRGqoE-gpPkQ(W0cB-G$5n+x8c4Ga_M^|!|9NoEg7lsVzWS0R1x^a#
z!q)T*WV-B3l7i@p*!-ufCeABbk*3+0zsKUjuEzjfE}J4uNbi1VEXCjLSF{I&V7kj@
ze>9>?o|M+&?>3lF>VGG^JmLSLBQ<|wX0U71&086=l<oNPiF-{fLs-?DYojF}cKYPb
zqe-=p$M}3zR>KBHCA+ZtlkM)~joSAi{HMqJ`F3e)U73PLsGDPli0Ttgh3xcxux;0p
zJN@HEt@8svKV-YJr>ne2sC%X<N;v9py}YlAC3-sS*p2-ydFK99@FoHsqpR|ySA1GS
z@XL+G?kw&rb8;e$pytAMqgL_4%lGcOxmka_L2{cX)re3sFlEeKvPz{EWvFoYBC_gz
zQ+c0k_l(XR_lDEGCFSwY!TU>Pk8fWY)pr%pC3I5qnPDj{PwjeH`BzC7l|POQSN0b%
zjZ<Y>faGrum=4|F?&cS+dXZ^SgH-grdu5PF=$eBBrMUa#$5D&pF^`7d^X+}2qh>U&
zrA<irD2E*9mjr7pNw7!=cS*$nxv6Ue#|&@fr1$yheloQvUS-oQ3Ul91w#-S7hSPh0
zR#IP+@iT;MHHDuaPiss6Zav;?klU==dH11>`se$GGZUARUEqcJUTsu2XkeZ*a|!%m
zQ$fq059Sig26)1!gEv285J?M&cpuEhmM=3G4mMPT%*Ie`@6RX8ZH$+5L^XwbK2H==
z?@kFjS}o$QKbT{j1n*Q2YDA=E^4D!OgDce-(YwuSP=-*r@NwYAB{(N0<<ofF#f`}E
zl9bI^a0dDQ-<@|VMOp}ir4JV#WpCfn6eVK|m~a!9-pL-?`M8v2OE?sxQQ|3K*O3wu
z#Cm=-+Mt|7OI7x+dWjlu|L5p?JGE%nTAY$R-vPA(0xdL+9rGIBP9y7FxsxqL$*gX^
zS|0M?g;bS+Vj<^Gn)gle!;YM@Z5-6D?(2;fk}xCcbbtDM=#IAik}c6xj?25w%-<>u
zp{@J-S%S8*U{QV}#>-!&Hq9Ke?dPJYNv6ZkWu_O`k5$MTACzu2<A;L-5})*J!3Kv7
zx~*5uCsEGU`k$Up9R6-+$S&fqSz&&oExB4KQnlU1Z5!CQfQYe5mf0H%JN`Au=a3(|
zcetF-fJ=@kJI?XMwXAyisniYCC)d(|nH|4*`;NI0&23H~&Kw;dr-glg(x=Zp(3ejA
z)3Jvq%!*)8L%4UfC_k*7E<hT5_or&iCLt#8h9q;vlM*tvi)e<#gcul!+eESPVuKFa
zl*Ga--0$m1@Xx{7YeYQKmX)X;BP#vXUTh_i6*zZZt+Ealz|+>w10gD|ntM<n{|CXz
zA?P|*%BYeZxc0hep!hnAa^cSDu#sin(la)lR9qO@ecuXN9jm87OH?xVd`*_pZu8;F
z?5z!Rr&soQm+pOc8T`(iz*7eNIo4sYVa)ui^p3{ktO}MvlvlhV&U(dF^4OL2J?)lg
z-<s^B$sXap9?8&pazU)-8-XLnj$bB`w2afh-=7PxXB8->hwhDY;DsNpa!lf<*rEmg
z`Z)Sbnmw{o{?!=aBAO=KbI<6EQB#l4{p=>U*At-Sfd?;|w^xVuZ9a`VO!!Z@{r-%A
z|0MuBJDBelw|RG!H;{DDvwevv>xL&D^;8qEA9Vg>)>+uwr$S9T<B7Djp|bm9w#mKt
z<Es=SXCS8h$~vAtKg%3O=gYnml|fvtU^8kSpy@cR;hvDGbf0gPqiWK-JEmsUDg~(#
zZJEFu3pvm#8&b%@O!fIH5sTEww79q0U!eHS_O6VzChH9;VCM?0ZH8hlf+$yU!SQ|N
zO{{pfyE<63Fq0)8yuYD20$lGaMYsu;qGTA>Yt};Jcq=Tn#CeOJ#3ktCC{7L1A2ogx
zzJr=~)Se7fQ9tHM4u_`{Mcp(mt0V5l_J@Sf{|@P?;<r+<K8>LxuRt)gF~!pA)!+2F
zG!!_*jcx0kKE$nLJZamt5C5Y;a;EPt2Sro}XVeqxQs(IH8D7c(-m}p8L~(`eG>0hi
zaue4E%_!#&A44J}mUBs^<SppgiW|(lzK~T}mYy=<X`(Xg8wI3ddKB34O@bp}Hf~;j
zT$NS3p_X6t_1JrtT2i_X)q2UetKWSyn|)ZP8Pz#d(#1xj^m{7s&o>p_0@?GEoiuC8
zHv_4Om*X`;Rl#r=C&U<YpZC7ww}b^vmJpEZm!&#QIeOT2BJhTodvrT%@@`f2E8rK<
z3Ye@5(s_pYIyTM0t{lP}LidKYRB&^|)Wg1iUDeF936q;4)Q6X+v=NdUpuv|EvHD%f
zM0S2K#bH<`QIXjld%aR)pWRNGEd9w+MZQK5KkB?nSo;eI<2kXd?=tClbuy8hAXHUO
zaI($H;4aae*ac&|qn4Is(r<U6I$YTX&{<%3v9r)v+_v$_z(@@2U{OIP5hLa5f6Px(
zUxJYU?Ge>;5UWX_&~Vp6$4lX@Ejne@Hj9g`daPND_OKze%(D>dImTOk?IG?z*sR5&
zEDftKpQsZq)%<*ipMjBNuZ|&^&Qq-OPoe~S^_2X!QM>u|VC*l|en_*BU0augGL%B0
zgS>U=ld#>J$8~nSHp_Vd><KenuEsr-Ot6}G1RfhVGl4qpGEBq>S>nC1^1k65=Ke->
zSTRsZoseD{CTZ06AX=u;nZ=9*`)1dF-gGD-rQpQA1WW2l2P=#>-byL=9%4t1=EzHh
zG4Q%Q4h|(pga7n`t_t0kc=ar7_u5}#3Zfomo$PN*K2e*}N_fQEUWd`(3S(iDFHE#Q
z)fSW{$;ukP-WVXEd`9^~(6as(t_!E&_Pp4H>mapCx8JCF<`x$1Ae@sGBEfKPr6BZp
zFVCa?aO&RAlG@GsC#J_BudQUxGd-jJHYdP?!go$=5lM!Gakt~ft$vnNI&|hbKe2Q`
zYP5_EjIZRUR_>-rWVNK=ylRZ^oXz~y__`P15Dy)~W{6Ucy^XTVY+4I`WJ-A<w=SVg
zx0^ZvgDmYn{v=n!(Hb{IeY;z~9qZB4-IUH<J=7(*L+EOXiIqE-VS{3`=R(&Xj7_Rq
zS=E>Cs0e-9+#4%uv`z;3MubBKD>JMa!%D$w#L=FqO}B%S`68vrW7D;c!9>bJPYqv?
zkK7=9T?4X+uon{*89gqi!Q_|6ceISt>s^(uCQBh%NwYlAE_camlm6-`!_Yp~lbviz
z5Z1r(+w)1_)(5NW=2T~A`wc`?LeW_9F^GZwjSMLyvvpE`_FWseF4_(Gmr}?(wU_WY
z0yhJmmmWp!)!vtn@xUq9;Ng-2CQtQ38c0k$<d`p~sIRp+pnM@O6>b|AWy8_9_w`R-
zTSuC9%ns8xf{wM(_m-UyJX9xAo`^Pm!kU21uMSZ9wO6{&$H?U?n*N}dbJ&2}S7?6s
zC9NG^6r+g(Q|uy;#OmV4J0WL>%j0QqRkQ)&uP_!V!m)-%!CMCkNmRn@fhM#;3#sk2
zK0VQU9iJ^xu`>vA?Jce61OPcl05^?Y%L=Grr6rv@b0Sz={TuXUq2Y>@c@xuf%s#{s
z)5w+A61J&?wDWDvQ!4JGYa!m(R`Oh;!xE<cgD!a&&b!K22h13uiWH<Hng5lIq8O^k
z5~x1mclDlaR64`r0GwlnXlw$78Y|KP&oGTQwm_M~4CqdMf*u@<>i>zv3A#7~K&R<S
z<@6T{OWMtbLV=|vQh+HGouWgN4=07vKKbjvd>!d{J%L&YU_ZU4_SLOR=7dy$m~$6f
zMz38Y$FUE#RGp6bq2%p%^eBCH;`tj+ghUjW^G%fp%v6%)&JI98(*EKGl3?mB@|%MS
zj-AYg&^WxXNPEYK)yv>`BOA`biT!2axRq4<<d;Xdt?O;xkNUP3kJevZO}GYD&vom;
z3z^)hiY~6V`rk8lQMZf9i1gN3Rm=k3M<}&ks&4w>Y!fO5HbFRacG`9B4}kz6e-W)D
zo8WZMpsf9WK@w+D#@3#EwBleyI`rex>kTp;b>wVssY4Rs$om;l0PEVE9Q=k=dVKgo
zevx$l1MMa^{5Q0#M=RmYT8ou(vZ@_E7g#95T?l}@`*)_(%!ZQ}M(-^SA;W3XM=Q^L
zqo*5%`=o`JbA8^GHsf)>tC~}$aeFNHZWWxpGyupe0fU<2=58N{+Y)KSUB?9YUQ=TE
z0OTusFcVeiYzHir`ML+G(x8^quRGc@eiLq$iS#l@05zGqe)!^Kx6><b2~vup-$KAt
z5JxbxMbILdGNdMS|3|>@FBC-8*$v<(=E;oAyZ~$wBKTUg@%<zBS=0qkO-5i8b3BE;
z#XmdQ=@Xv{oCowkeSY)v{uD^+;y;Vay^CspeVPDY<~!H*uq|77j{>MKW)Co=zw+3N
z+nGX(rafH@g~s*jD_>0_^x6kG&wYCUsg`6wKa(l>=JDqBO9=KL!P$vR$wx!_Uf?zn
zH7R>x18N*GHUX1fDqP8{^GQ;CH>}DYHaiX2g>msetUgql#0vU$i$btL3t-qo^ymGa
zZMHAnU+x3)2D21Y^&hv-)DfRzFTZ&z6yx{?K`0v6-1LO$QMcS_QLeYT1EtwBAAygK
zc<tT4DyPo?R#kT?fb#%;@ByBK#2--~XfD)s=ZyN4n)=-O<acs7!LaVl?Kq*uZp-`r
zU74NBCfxRVA=Il*7f}I|H6<^jS&rM_<QQOfB1!>7HLU>}H>CU<SWH~dOcasq%%DF2
z!y$)@nLFjSWZ3;9FZlouoA7AJ!%gP5lAQyI&O(e$cDsYcGIW=$4Y8PM`SjX@jwdeu
z+npRl)032{Rd*f;T4T#NXBazl4oRN=8PtZ&M!Wy0`Nrred$hvRLq^Mc3OtF3UYgQ<
z!GCk=v&VG3=P|exa8VWnWOd5ebiVz7MpOd?_ysFJ{OnLuY&=>>_2tuhf0?wPA;b`6
zkyv}({0(N_oouc-u#>G8MY8*&McNDmSY}^=Y5F#uGPml;lhwe96wBJ_gIoF3+0j1k
ze3il;{%iF`X8rDn=}x6_*|RHF#mN`({U(pKGQrDrnf=Kfkn|ElucshhbHitE%XH0i
z_|frVKYXycM5-+<nDhYpXai(M3ap3L`BZ{S4m{YpVSD3_vD-$Lp?){+E7fV@m4?8#
z5O>igWY<(*by&)d3=(-|IQOF|Y<yE;pmRxuKf>7trq653)JRNHc5i=!?ybNZ#!1)m
zUMI0GH8Y=H(MOtMRMk!rucbXCkaG-YTe1CwZ3TcIGV@N?`#P1B_?4uSaLXq8R=i|m
zjDV1I@F~QG8@@Tn^e{B+k0oOG%Hijl@$tg7M9sF%=@&88wmyf=L}0&+Y7$0!&&_TW
z4ag;ABu1q2hqE;02JpL~T8OaX$=!B`t^D&7j$yK_@?`1lj(+n$&NFEg_&aF0pYCh`
zA9|!Aq$bJAOoK&)wa%cMbe*{{;Jb4X60enK2$P;FbBrvKM!rX#bS#>a>M_9yZz&bv
zRwB;mS1hGp4R^>)0&%-D#;0x}aza0QwTMr{G1sdNU{LO9Q5OakIzRoC^Ld8z4hpAO
zJK*f?zWJ}KT1jq0aUIU<2HEHnnDz&?<r2x{4i`vY`b<id_A6QE;26?QDtsG`l$JCw
z3tCKHA$PVOpw#lbH<KCANWk`+B|exi;D_Jn+jliw24_MgK{qdRQeFbQH<a*TiEnkc
z&8+$4PYaP;;Ulx_W3yGGlhUz^i1#76tOE2_HSZ@4o)vCF9MT&L?96VpwZ+O#>KSs1
zfXJuC74OV*?)Wr_eS+^9E9+Pt!idHB6BjW>b{>nWry)+Y&*@&betaemL+QuU2yA+I
z82}xzz0`uE(U*AD*B>HkMEJde<sFVleBfON%7tgNrM4-!>02uH>Yv!Twc6P5wSmaO
zw%9bMCxp)=EH1@i?Az#<`Z@JvYp;W%4a2J=g(;A3PA-C!S<Y8f8qdhwZy10%3Q<l<
znr*UgVu1DKRZ&LoGh^KcYKWAjUi{U>dJs`dL#GYyy3)EhHXR4o5Z%~GV7wnSJ|y+o
zI3r{jU5Y<onO-~I{TZ;;gnbc-m0aM$R?7-%GWVY|jGa~;t-Zx&z!ahHn(wRn-A2<V
zz0-5NUQTq(zAfGy3CTe>d_-Ix5r%z@x^?eCh#K3KGIM$RulO=g)&l(|iawUolvX>;
zHzhC19X)w#A$5q3P1~r9Wup-h$D0&>`qQ#c@+VLI>bTHRzar6>6!mO$D~`rTsY2_w
zTJ0lZ9$IC<UobjB-(VSMbD7QE>}(o=-PS{NgMiO6-;n+O*%g$NydIIm&#j;N1$eaS
z53HS%4;CpD%4)v(JVV~NOxL3@&`oJP1vKUvT2nLCdIVYs0!_A__W;&=>ksChb(wUN
zJEm{a1y}bo4rL^S?qNj-(s(Yh=0G3d$Wt1-tKQDZxX?%=wA7JzGoq@Bl47>f;FkAR
zU!-kw;o*MhH*?9lMjyN`=sjL84rB&VqvH8RuO8QCp4U`LQ7W`F6VzT5l)?@gkE2_G
zDA8+(8EJ?~3W?W1*I<_yh9?C(z%nFx_=q{U?u^~O#APcFp$}fr65kSpx9a5;sg)wD
zq~!8Vd2ze9cs^&kCf*?#o75(+KVrilx=kt0(&{Wlhr5(<m$55;u}3*-i5B+*b~HSe
z>78zIx^`(VQ)_wiTx2$rB^+fJl8`TYgmLpUICB~l0Snk_ahyF^&4h3bGvugcxXj@7
zvlA$rQrxkHsvVvk-*d7;jNDzD);W&+5FufL`lZSV_u8u6)X!K2aM&+MW()Bl><)f{
z!j`hq1ZRT;gnBtYJgmx?oaNaymO*6Zl}*Is+_){lC`1xV6G4<9zTp?rw6oat8ePL`
zy{z`<yIKWz&BL<Kk(!9W1JVV`7=!>p7WaKZx~?fXEX7d_Y(k!fAA$t@8!lG0jG7;~
z^2NUy{Zv-4vNdU;Z=}Ktygp}$t5}Ck$#tr%8H0GJr15Ow(}X+CoQS6oM4rsc<m6yJ
zc|0CovE<VyBW`j77FYH~o+LBgJhgz*B9V`;Lbc}-Z9dRK49`)YHY*3sorsKvWqpXl
z$Rf9rvtU2+VC`5s4%9#+wJ0{&_KiwwzRQ*$$IuTB#GASgwntCWDz?NeT{2~+;zR~J
zdNI6HB$Oeo5UD0JWYeZ8^}w0%uV-i?&!7fKyTY3fe*XyXV2UMP<xGn1xL;tC(3AiR
zlJJ*j4*#Uk6NCHRbS09M`gt8)e7~wGw+T0!;lKG@ho>9;{MGN*U14>;Z*1?`CeU%<
zOarD|)fF3l@Yg<twxaCw!z?Xv?q}^YE3uHK%0TY+{^u%0S^^5Y1x=nu`oG#VDAkk1
z#;jxM29SvlNk`Ea+8M&79kN#XuKV#f)0dB?l0Fe?!GN_aIm{nY@FwNf-!;SeO!_wU
z7Xh!~O-qDe#VUn~%N;U2Q?M}H#BJ?CGoYFcUSEl9*6WeCJ#DK?PNWVOhaCH!Fyi7x
zGcZhr@DP{gUgzDRlS>=fkE@PEOQQWXR_>^+m8{YJkO<p^f||%tyigLF2(-RD&&evU
zI)dDj=w)pA6q9LZzuRprG;bz~ONNuslGod4Jc`YaKLM)YD;+V%g-zGjOpk)oBnbFy
z23ZK#6-z+J9m)hWBJ^;g79|*X0w9jJuS%POYS)AITKDu6>h%~<K+&mEnM0r|EW2!=
zZF#oGcbP#*IBC(-qnc$%uvxF#ulPaxXuC21MWJRvQRT))DvS};4#j;%JCxb*8AZ+T
zE_80s<W_W)j=FXmwChiPnO|`n9T=$B#00Yvs-oJKpuM~3v5BHQ&DV7#I44=W;HdTy
ziH^i8{_x<;tE$b7B(T+4`gn}AtM>nH{r}fnN4kX80-LeJajlxYFFqaOqrCFs(jF_G
z!q3l6J!)28j%4;_u5W{i!_o_NRvmr_kmp|;jqY94UXa!}`mkyj38yUaf&%x#=FuAK
zyHEo)>e&c&me3C`m=YhubKZ4vWXNLQ)I*8I=(rj~)#*_eM6e)++{F1sdPv&e9-^NB
zU$!s|1O%)-iB1XtcO(4RUw>Ek`m1@dW#z7{83BEx1?+WA)$MPctQwfN@5b`-3d(y7
z0EgG{D-`3r@{W~B=wEXwFz<y98uu!60YCx2gYuP0TL1H8*?V8ej8Z-VB=Np_Dd$5E
z;Ms8}-(B9*fijNNX5HXN7o(GFndbiEyr0Hw0B^_-oXhx_jbgSk!F!Xo$uc(O&T{}M
zoB}0(3GfBE^TWQ^(uJ#;Qw+zyRHOi;9>t<@6>#cDW;^8Ev^)UGs2<G6|M|~=5nz`9
z+JaKrszaXN$OWsc{NexnG$iN>xPbHyM;ve&oXY?eb@nTTv0dsC^)!f;3fap~gZVP#
z-&%RQC4JwU@aQX~$A5I!I);q%;a&<}QO;jn(JO4P1vItA#_evb3>Il~5v&5v5&|G|
zFOk)KDu+P8mxLE}S;#E{XG5%6TamE!(iWk`&#u=Z8p}o5IHH9HRP)P33R6i`2kU7J
zigZc4o1P``byXy~S@mQI0u5;$RN1aSH9I*5_!T+`VUPGiZpzc4dlSs~2o#TUyI&VQ
z%~`<=iF|%d8jz%YP<R271|UtDNH^NBgV_&xYwUaAA)pHBVntatcn_(7TrmW+v_F6e
zdQ)5!1SNkK`vA(v2g-s4gWzqFT}Tvq=1z~W_H=zH&`SKHpQ|K0G)fl%-b<+FGm@~D
zG&zC_nZv9pZ)glOK*ea)`Z49<PH!6qQ`SS<4Nyi@NWbDhIQYa1dmBHbGB|W^bvFV2
zMS4%=rgc~HXXIl~epGQK>r8P3$li2a9zc01mTlc;1+}Xjt6VF4rt~N7*COB_|1YQY
zZJQ(vF*VXK;aaYl#rVE%yDQ`eU;%hcXHXzGn0G9;2@s`N*ryxpfx4y|dp2vyE!I{{
z3NzwPmSpL)uD9vR6yq-MKRdaBN~^M0Hqc~jx_OAVc9W^laQ8072D=}iTF3MO>?20%
zTIR_u^D!VHsUhv^PWL8s-4#$`)lHlsb%`04U{L9UG~EU!2k3ASPgeZ<+6_ro7^M(Z
zEYFfTyZ*_+9sq$Ao*LNNkwB8j&_3iWHjEUG^a@3a(qd8u)sKUcO%YIGIxg-#MQXM1
z(S7K5q8!baUGW@G1NS`|ZWiSVJzBg99;(P0=wB~@f(Kv{fj9#NW;5HPr3>XC$76$W
zcJra|*TAnH4NkY?j46KsUkR55g1N5VZY$u<q~BW;k#_UzX;!zQiUANf0px)t!hux1
z%)mNPbi^us_U-$6Jd(4r_K24UyDQ&V{;5((Vn*b^Sno@1j=pD@INw@Tlz{Dx*|H>+
zn#lEv3@UL%Ws#9shWy5KM`(;)uvvybCqT2S)yQ76s0UTR0{LQ!UONB7amV}^A=lE@
z2MFxB6eh*H?pER~T5%_(Y&^hS!40kfp+15Ax(3}aT~P@tdrC&a*S_fNii{k15{2x?
zwi(~SLC1)rO<7IajRb)+Y<JkG4_mlVP2dP!fU5n$u(zt+pwds{d4cQL2(3nhFg=OL
z@aNzFX^yxM;@deu+}>g|F|L|TP-6*g9Y|4DXVg4)*UwYF#)~b@Ye+tJ4-T1^AA_p0
zVz}a}4v5BVnGwNe5O=gBVnn7lKy{v$VH(E72dW|KeyySf;pb=X>yQ58G@i9&bTEl~
zM^we;|8s=*(UWSsPDb@o(ixTbme!KWsRs$nHINa)@9jgNS}0&frJP7D+Ranmy3LMq
zr#uAwjr$NYT_mklEK%Zxe8L&c->Q}3-@ey?<^oF3Y%uE_vH>2wXb71JVdP&5cVN&~
z3lN$b`e7T7bjRwUMXC&eGZBJmBp;rBAN>k7<(7`a-P;ou=S#d1_aA{qF}8K9BGQyu
zE2L_>fS5R&QsRz-gY$Wk)CSHX&~l(XSWJ0miMP*Qm)c3X@+ie}W=r|6kJaUlsmpf+
zgv*Mnj%5Z;d&tqUrk1`=vr=hQ>3W<Wyiq;=2HxZu#VUkZ_ZS6Ja^bz{7&i2=Y(a+P
z^no+z8M91fAdWaxwf5rh%sQ4;g#aDEHX_7bSqMnfj6tAsCbaDt1gP!eBB(o9=-kX2
z-KmyWD8@9DqzvM$1{z>7@O0p}{+-)OMg(thVMfpT(|Mkw(v*(I1?z@!qqm2-KMh1<
z+Pgj#!}Ls7ot3Z0Ct9A1qnBKMPn^U{qN61?_@%}@H33^b5FSG_PVPuJLueBz1({hE
z((IFd1We~awJ=<ZJG7T6K=cOj^Hzjg<qxqe=!l66FW>4SsF|>*9$M=%;xpA~wJE3F
zT>aZIJ)r|J52#;A^m0y$$)2#rDd%oFt@IZZ5Y$I#RQ41Xvq=BxS8RzT$_FxY#{*rQ
z?3oL-vtOa}2cA4;0iGaMDQO+huNXl1HV^m_Q_i#_f=&k8MobPZBStt|Ho63`DyZ6e
zAW*+PnSC?DXx1zi<A>2L849C;hFDDg*G(KY*`^3A!%sym9<{o}wX5WT-!B|KG0?jY
z4}{v|Y3LXas4`UkYYc+A-)#~HE=u-}RI#xt0tnd0q90yUdfB9eL=p?cHrHav4{B%c
z)7-CNOUIC=5jbdXc7|NT@PUYhM^h~#uq0R>usk2wcFw87^ip(KIj!6IdK-UX@S-4e
zFZrdZBBPb7W+!gJLy>_Zo@>s9(1D6g<zeTc5aS5Pwxvek7LwA^{;U}fwI2q{)uMap
z6Xeeq2;@LXCAL8=_bCQVlrXySVHKnx<RBpkVID*KL3Nk2oeQp^_=&E~oJf$Ho!m0S
zOH7?OA3;Z|s~Y*+rw&s8;J0d9$_2$F`IV}pmEA}{fYys~tOA=CbnB`>v4I0<4buI_
zUyqdx!+;d}+q9Jkv3XFH)5s`@ODu*M%pIrCUL)>J%tzd&JakT^gKC^)->j=Pkxyb_
zQc;-MfN!z+#%wnc2?Uo$2^Pajk%&P>f?S<?3ZETlWP<WHi|!MclrnST<MMxxS{Js+
z_*rp~lo9=$Lz|)I?=J>aaSK=9)X~s=C{6*2(bf8ZlzV}&wy+nj>KJecohCqJ&*LZZ
zQ5(3ZlHi>oXrH3m&o2;~h}SWysB$A0P=qFilonLmR|N2vBzE_36r+eF75ECi?}v)l
zl_~~A*1CDz1sM%)fU8fbuGKAXkbghXKm&m`K~`yf`C~D5CrB<_fA-Wx;uSG+7=;28
zmEqm;&t-HsRZujB2~2{AZh~St{Z3$w)W72I$)P6218daB>0d)Nl>r6ochVj@PQ8j3
z`l?`y3J60K%MqalqbUbJn@UD)Q2~%7GDt-Ys`>5WcWvKEcf(Nelk9>d{g=fpBELO-
zwa<cLaNt*Q@-b*7D1zrC0!B;RQ~Jz+VsJ5Fv0P9mKUDKMpkPiF9BZd2fXu}T?t=;b
m7De!=6TxWzgIkX{$KX1Ei4}Yz&%5}=sM;+prFRNek^cvN5e8%c

diff --git a/format/diagrams/layout-sparse-union.png b/format/diagrams/layout-sparse-union.png
deleted file mode 100644
index 450ea29fd84f83578567160de5878b8101ba9785..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 43020
zcmeFZbySt#+BQme=c2oF(cLK>(n@!CNJ!VBkq{&mP(n&V5F{4e-JKFr((yg~_I};_
zoH5RS-+$+I42LkC&+NG8bzS!ztEsMtiAIVB0|SGptR$xm0|Rdc0|R%A0tcihif{Y|
z{(<$>R+NFM8X?~YenE9pGW3LjA!2y`3#+V6e*$Efw%0ZAGEh?$v2=CjG`DiKu;%o0
zb^~(5z=-*Y06#igdzn-FIXk&{iuj4s{*yxl`1v`Ri<bJIEMAV{v<7OL)UvJ~*3|r*
ze4O005@^)a)M6f1HX_<`3jh8b_)VPF&dbY9gp149*O$|mm($h5mJ2K_EX>8t!^OkH
z0p#HD^mp+x_v3K!r2AJP|5J{fwWp<ry_=W4s|)pWx#kwG-d^Igw9gg&{qwJOdfD6j
zrzRKAe|HP$AlLK%aDh3wx&D?7d@A;wDkAIZ?B-$Z=?RoC!7uht&i_Z+f9m;HeoZ@9
zFIQj`JnSu%UA(M4fRDY*pIaxv^Y5?!zjOS*%hK?$w+4Fs&)498fBoNS|IRPQ_1y9Q
z8i;?5@;|A-FiW6`as8b#3AE}>o@E#qDHvrrX<a|q!yn#j9P&5UPe0uDkB^Uc;KJ&=
z84z$;5Mh;JSr9+Uk|HXn=%wS>?`IJ0kEr<iF0G>P>;_&Q=XoCc=jV#=dO!NL+cM?*
z4<8=a@s18*kV*tzKb<qdqAE$j;U&LCpr;Of!Ub)IU_Gh)PXh3NYsPS_F_-ZFl?fN6
zj5_gc7y5saU<u{!p1=QhQ9n@-ba51au1EeS=`C7O(0>(bh5!^ww{{})?~0#4L>!m+
zp8|lgxZz+4yK&w8X#Uf@f-pO(f13HPBv9yEv<(!Y*8fV<l&;7A+Zk9?h?EqfI|BLZ
z|4J&OUO@ia&bJ5()CFNea9CXbm86Sv1plwm_@^2=98GC**l6PaN&<%CKaHXe{r~%L
zguelG6emdpY{W8+2=C?^yZ-oNSDY}@#sAF#FIkET&+l^HcGYh)!2(+PaIAkZTPlap
zcC(Ws`LM5nEpbi#`zs=mL4jOqqw>_(w%cDTZ+<hJ*Yr}pdA!+m)nT#8Vc@pAK3ShX
z&}-gbYFzSA>2r={$N+f`f!TKM`{<HdfuayN>c?d8Rk@|1LYQwx1XlyGCGWZWM};;a
z!vc#H>wy7WG?kw7)|nrIF6U1|j=s$@=J{<H+J5j3{~f&aCU)wovT+4%SWw&F=Kff4
z!p=DXW}@Zz`#~IA)!J_rt{(`~s7$$TxBI2}F5ma}78hMwoSmzG|F~NDl=(Vf*?Sg~
zOyF=k)3oHc9tB(a_9Q9fe$&1ryQH$ym*M>GxaD^^wnTy57iH|jI1EDcVeeKg1rqCA
z_Z|}8CHEWA?Q|V1%G@!<KU~45@sdwBqfnzmplD<YFU|4xLka6<wY-2|EXNjhp1UIg
zi%tpV?I)e+%~#5z=R>@bH(P1Ni6Rb`Hy`uk6gxhb<_)|ZnbeScIPIlEwkxlw9T+Q=
z^^Z`vSq;8BQfB&)KIp&u;i@p2GVW1iK?w%2hN68`;*K5d;r@J7pUvXOpBeOVmzJXj
z2Y1r*T`jzC6Q+-6nF-btX`f&FZ@=5lFy?GfpXsPOY=3%irTx}+y%AqmJMH$n{pqpt
z9f|Yr$=|{GA&>qaguj0O^@X24H{jRjQIUg+IKx8X)P^1u!2r}3QqX#aKM}t!7hS2`
z9}Y~PUVNeD7ol{nxj!2;1*(WeX$5LgfEXg9S<vD&ynuKtGq$?Rc=Db9nzu|6mthP{
zJL#BbO`Sfw*@)-rl>L41wY`d{bo+77pRKZ9yWwDGX&&#!1HF(r#xP|0h%R|-)~(8x
z&%fVq3+R{6USYI47)l^|++>;7{73cdYBhxFDEareX0bexq1E;~{ouozUYFKWg(%PK
zlZVHv_K>S(UtX?N-j^-~v_nhaw~C#>l;6%6yH+wqpUCsw?iF)2%vsb8@t{*ZPz7C#
z^Nxyz^Mp!7Xt(W`<fu#BY{`qCT}*1U|N2tS99AFC)pot)Ihr7F)6bO8o$b+uDY!Pu
z^heLwSw3%i>NcQTg=xfdRQxI}|9&g)&9SoL$6JO%jwYQwJf;s`+WgxUk4x{9ns@Ww
z*g%`MQdRsm`175^gfAAIW$>LEmwiHj0VRoDGLNS4i~rW5C9oQJ-hRK$mEI}V1FYn^
z<Bh(2;ht}^uX75?mmrfmdtvCDF#<!5d!fjbYp%#v)iDbQ0eaFO7gTEx>nGLm)%diq
zu}5ZImMxff*ri3Sw3D<Jn|I!`b(>!V5A)BqZYGLK+@ExL&%VmUZmE*BEZcd!`#q`$
z>z(n{I4~;^70m?2dV`CslFel&Y!=vT34$wrsmhG~Xvx#jewn<{c*qTwd|nsM4f8hN
zruE1iD{6|Eux!e+tqScy<6^tHo&#0X797#HVGfZ!sz-Zfh5U6d)dv!`Wk70jriy+N
zWSTnMZ+;TRHNRS!U*cIN%=(QJtQ94q#$$2sA$+?f5StL@jh5!R^q8h5V0dvz(yJ!8
z+&$~zM{r`+yWp`NNjx%>$W;TZDuILI7IcIKKB4(PEIR_z<x}%*z!nQE8}CQLB-PDj
z+|(hgYaT<)9#gI^8NZJwHaCXKCZLmpq@CKR3D6agWuwa0nlb!gw*qT3iA~(3m1170
z&$N5F^vG~;qI;W5P^g`jN<P>(S;6ps{?*N5(=#3)2HwH#Rw(6)@|k5?pu1cl@N9NT
z%TNXTHgP%E_A?e*@`lYF7mcGA+cTa+f4AMah%<;lO|U=B^xWvZI+<lc@C)H4lDBT0
zvq0}D_xNV>f`BZ2t^dk|b80XOb%<|L)qVJti)y-<<JVCGpLuK9=Ca?dbYvNY2dLRP
zCE+wk#fT7yh3WJknqNIxhjpV8G&Hi*3EO60kWeoAohM+jc^HL%l@)hl@JzB~ng;um
zXE9s83NW@I>|SwMeJZDz^e_G3W7O1xa`*(-=6O0FT4)S9=?(XWXNL|$PixKLv`8%8
zb-m?@jS|{S5Pa=&79e%wCC(!|%dXO`gtho0!`t^&UKC<X117K|EV{C@s=pxw><HL%
zWbk?3oegp&32nqslQC-W4spxA_h2PN(5{zmeY`y|REc+Uto$|uG&vD|d*^q^lW3hT
zd#PE2%eXwTHSCL1{_s<waD1o_l*j4I=MvUi{R?w**qnFOCzK@B;zb0(bECCy3qbia
zjri(9upt?%y&t@|MMZbAERS2y)fIyF9u6tPkKgffvDv(A$7T+@j8Zd(x^iXYM9@fO
zh&t7eE70sZe*Ka039rep4Zru3MIi>~O5pjh3DO|<Tg!_m)IdnGr<tiL1<9lZ&!;k%
z9a`V(+V#gtMq&FzA-BHk`L{0$*`lsNbXd&hqO^iDr(yI%EWQ4hHdY4#y<O^beWM;l
z4us~pdHxPaYW`8x{>a#{c{EuDHTJT(wqG-Rq(<`{vOn&xsy=;wRebB3R&|w6O1S>{
z?r!y|4HU`}L-nwyDE|~Eu{V%)bC&7bUH-Lg0EdDq160P|yp_r>b-!}erVoC}Bu)}w
zTS?DwHh!J6dSB9Cw7HmpYFANzgZ8kF9b!;_K9T`v8^uvm*Lxj3e+1kmRBtR+);b>J
zwod7|KU$24i~Y335xzgqf0Co=0PTP9T`@;oC-)rcAP+i)zUOF6Kb{#e=v{hWBMm_!
zuaKvfP48^Vprxg>gn(VzA1)0{+`5BcE$Y{#^ar68KK(Oq348taMA{#YT*2z23`2(U
zCl+(Qt3kz|O$gvy7{Wa4586Na*209<%HUya%8Xo$#ATZ;kkeMfTSIWD#DOay_yPlq
zptr!TQ`o5J{?Jx(P(D6$h|or`(u!>sG^MAAV7!P_v;K%5=~{_t!Pj6Z8*Lybj+KEQ
zxr|>!BEMM1q-?y&H*@+WmO!%XSEKZyYHzu7xZhbnQ*=)J4-PiHqd&8zw!-o;X?KIh
z5!=~>I*6m85+NRj3@+hw3y=5*ub-SJvLD0EPH`jX286X0FF$1;3A68j?2Tx-d3yCw
zr3d>NlU!cE=Rb?pc+6O&-dQnqk1wZ@jhad>1?lcC`Xunrz{rFBPt04u7tMVi_oU7T
zKS39S<Tm@IaB`5N-OEiD9xfNT7_Hz3fjJd<J@ea6$oxSbgW2Yn&QL|Y{GhbrNAUIa
zj;z=cjA~R7@3xJ<CiS&fOgm}giz@r(2+|CHJNZMaj%~5?p|0AHOJcLNWGAiPSi)P$
zgn9-v75!X{z0pFDPZWM@p?fJN77=989NC6lPOTPS7V=uv4Rlm1WxrNZH2c_rB<QhL
zi<`4_(*g=3b3F!Fw1|^VJyV*u&e;Otq6&cPs}aP6n}2&Z3X4n*6M7PFnoR$czl!qY
z2&uGTQ-4}=>%t_7ho(`O&R2_*87&Ss(W#As1ja#Jwo|P>_rWMf0lk)exw9~3T1#xf
zi9F#Qg~X5<{OF`Is{1wMmfvlnM<7Z>4Y5-ME_o!%xP<BZ^U^F^$!m-~f&G9d68re>
zOyb1Q#3o+S=*uq080|T(C~VN7^|&TUP=&z4hoG`ZaTldQU#A(#@5QhaN{+a6x`82)
z%PlcocBT+vO~r&pj>@4SZzzVpOR+4k5|s>zs9cL#UD)kQ#GQF1JRi7qBwLe=EoiL)
zMCl0h6*DQ!BEmrelc^%ZyzUWRo9Q4a_W?w@r!aqd9{=*#i9!*xMpFVX)w*CP>2^=i
z%`#v4e5S#J?+jHJr;7<}iS7%uSs$@c$w36i{LF}(FE|ub(}m3@!G5W*MwcYUV>Rd;
zrTtFLc#)#*yAollWmpszaVp;>4dw28FwO{3!Y`6Ivt6wFr_09nNR>R6c{@YF@0R34
zbUas50`)=126i9lV&H#C9yU<0z8*DA509IcLtFx`?2td@rr_+pbXtv@WU@Fo9U8(k
z-HR=Bq2sK+^B8d_Hid&~TT6<s?~^eh)IKGfFUh2>N^_bfHIEt5ygz1x=rB0ihIj_u
z6m%;@r2|8vsMA#_Ex^n|q;DR#c=j7{yF*tPKkQ5Ly$FoNv_k;TzBe~Ylh7GOu?WPk
zk%uv++N#}1O`6_#{ZJ@7bCc;C0A6&E;0HFvB@0I1%fZ4us)N4H-33?U6@sDr9S<`U
z>X~#`0Vl~Z9$9iPQ}zxSy9C_s%_K<)C*f`^xu@%1Ng0oe@nr5duVsA(^FC!5{d(_M
z)sL!kRJ$HA&Equjw3t%ob8l2cjr4{hkr0~2#YP|ASg1V5%5{+MIs275YBNYCk1vpu
ziW9t#V*6_B95e#KwaDBqq)Hc7z_GM*8<Uh72@Ko{V|`8J%3;%3*CNJunV0yxq2e}v
zB+egg!#o1_k~BskS5-uxfXnakG%19hrtx+}^f$R{9qFkt9-k(nyVOJUQAhJr7Z(IC
zG*WHnaaXYOsmNcp%<Ryk5ZYy`8WM*l?MxlG6IBnfn(TE@1Sao!2K+FYW2~k_6Nw4s
zRQ>#Cqo_*gIxxc3p*UvH7thBvN@dh1man-Mx%p4`yEJf<{_|r|`I7UhZdW;Bb3dIY
zLLX&iy<SUO8Jh1I!Pi>$OH9qJOK!bAF?`zNG&0!!ZrW7w@EAvCt*Q5|)DG%&N<OwI
zP`YdG4$>&Mm?|-lTXJhFYv^IwIgv3}i90-KDhlgErkQk~_k`1e|CR0KB4l2pz;UW<
zB>fQ^oY$Kpsy|(Aid2ZfuoG6BdFm`afh$*%)b*pIFHj)5LKF*z`y0iBWhQ3-4JV6(
z$`E#xW(lcfXI$b^B@{+zRe8P*q|FbEtPTD=1x|JpO0BshB~Nda@byXN&2XFtl7_^p
zf6a8v*jYg+Ym9DT&EsS>CT)3&K?)9EYz!{44ybjgR`vz|S~pgBz(AH_LKX#<L&fGx
z{8}(#%F@E}X;D~~bH|><@78nHnV4w2H)qjy&6zroQD%?5u0U0fI>jeo?7-1$du)g`
z>x&<kAEF!fTEjq0VSaXemj*5?NHuv^`X-yu51rL2Rq(TIz+h#(6PaJ)#zw!g8Ap(m
zD_4}JN7luHK<b3`b#Y`7+VH@JE>dbvLgsqa8BIEk(qgCG_?ww(c4m*YiS&V(2<Vl2
ztJUy2b;$xNbI=OzDfB$B2Pc+iEo8k{c^yP*O2+?jc;11VC;A5Ee2N@N=M8+d!GIEl
ze0)a8)7@&vMxHqC=8G+6dcs&VL+B7HrV2A_6ywHx_9H&qpgvt?j{!SqoX6jXyIGYv
zAM*lprs?!BF@;S8)M|B`O08<lJriqFK}dI1YU?XXsiQUb&SurKkdXUT>9YH|L`;tT
zE@_MaG>lRsZ+R&Q%qOA!-v6<C>9%8h$a!M<ak6B_d`v{F$}l_LA6`_r{&PXx{LIs1
zT$kTgc*$#UV}O|AZNo6uOP(Ji&Foex`|Wqjzkl_&TadS%`$H7p)ZrjIM^OB^slgJ-
zjPvE3Nj85aTsINHV@K+QpY1gwKk`~2mpeyUWl5PW2GU!t{n0`^+*^OjhPCJO>N#MQ
zV5ivcIDD?=L>r*ml$sOVDgs{*37UYyMiAv?>*FSXothWKOrrWyahNC5+%j`0Gcv;b
zGQaT9%qu1}nuzcmVrmaEwsPG870M+h{>i7s_u=0#PWxe>rDrxHk+p302hWx*PWi-~
zKGvtVLxvUu4+N<Isf~`{m{sDz|6;xc-s&MBz3%#SW4z_Wb)TV8e|%eL%qULOuXxi~
zuuWiR_cemNFGj*?aq;CjlfnUVbfa?3yK%{2;GKpXa%pE-L?SEhM9H1Yq2N+DDz$2A
zr|&M0BI}-@tUbCYz)pyH1H7hhmKvIb$#^-<><aAks0pr6JAKg0d4t`(eK>9F#VPN^
z6#PalxD@Rk94stRJ>mEOm;*5-<sIon^mt|*Zs$lBq+1VzBAE$Iha7tizum~_qSenr
zq#W0CTnV)6<|*x-wv}7oY#i^eTw$Vxd1IVgJhVOk)4H0Uo(7FG#J_K%)ZOfRPNg>r
zRFXBlN#*uH^@)UqoT$_CiX<H=1C0k)2aI|i-f>3y1L25U6s!eq%R6HHnT*jJc&^al
zkt;1N5NA<Gv(l0t|B~+qn<xTaKI(kSXJ&I?Sd7o^S%{{}B~}=xjWQHP7FT<Gk!2S^
zG3Rro2MwE3`8_M9t(copPhWpr8h!gdb@_GIIp-xuH5i?whu`Yk`Bx&+qw8|{vh}3*
zG19~iv}8=!x@u1B=9uQcupKhPK;LNPPzT<~XKxf3CAc2Pp&IY?%OuQYbKjMxcw%rQ
zQqL&81n*7hXbii&4qr#PBD0J>y{yXmLNZ!*GOkE6AaYpU)rU#q7h_eoJxt75ER&5n
zy$pgSYN)$YRvOfzg{sc&3u18c;Jx($?a;-IU+i<jrpvz;Sic#uA#v}Q)P~y11isKE
z;E(FLe&zqDo6ZrW;hMJK-2HK^+Po`K;&$&E#+>?SLWRqkpDDwr4Du~)mTF<>jxCtZ
zx|01D*Z0t@@oI@<(-|p74&^Yld!I$;-d-9GJRXeV*zvcOMlAo>93LK!v~lWuD7Dbp
z2MEHAtnKyLEj~&>&}(B=V=@*Ez?m7}$~sDKf_pv;<uFpl40#bASYk<Bn>G9yAT(-)
z$X-nI{jRu?SEVGBBYMyj(HUYs4Rp78=}%kyA%Y$pJS45ii?E>(HKf0Dw%(Dj&)0I!
zyF8ka-Jj}OvANA^e5s<E7)C)u7|RCl)@v(sWUB<sYs=aF{tGs~`H%sR9Ir@c_H<v8
zmi8L*I3`p^&J^L*Ppa14;wYB)J><i!?Hk5iRjn?nQMWoZQ(SUY;9t1<Z8Q1K@L>PG
zI=y%9*=G34SVQV#+U!+dKNxqRI@*d(jQOy{5RfBxJ%bMp9^D1G>$cynozIf!M%}BL
z4RAcaN`HXzd?gW$%HnNgLyrS%q6=aW9kZ);Ttl+MVi*)gL^eJLG5Z@nt(7pz>s(bJ
zk<8u}z@1IYSQW`vGJIqRu6G$hBt-vZo?pgii)r}2!nIywm|_~<vj=P62&1TOo@q3Z
z5#7P`tViVa-PXR|$f*>^&}cEncwlu8Z$uxMucx<;Z8V?Rzn6UYA|G#=kz&)Zm^#mF
z$f#N=u@ISkx2Rvj(YwwEcb**ScFghmfzX%O977`aa}HrK#qsrF0+MdC_E_HhFsq-9
z>HB88VUsgf^H6kO@o4vmpoVaVK0jEFmvp;P_%nTVj|Bp(G47H>4kbusJRLaE$cm1{
zR#T==IJ^=S!q#O*3!n%e&`kWxcwy?NCFh=Q{3tt&PEQ}_Z?5l~e1ux&t=Q8aE{ao$
z#oY_2dkob%v%}4P@{qv_=Ex&<nIAJ}IPA#W?ebjqkRtD5q$}`=581DX@psbdNn8or
z+850G@B{-UN*bO|3A@dyjPxmPuXSLQ+nLq3cd%Kt-88u%$s7R*t2n|~>B^u-0(6Ib
z4c>=RxklwX7=dJ7^kpHgw@j+s8x~ez()Jw>$HT(epLg-DW~yIg_CZU#-FCg!Vq)wo
z&GXt>ZkFNN^(hs$=*xC~<OM$;MtWY)`KEY2d@;8~HD(SZBkVRCPN*<%{ld%_U)h;O
zj^5s47an7Vk%fFZR}v9~9A9+yGPH;2-jK`~GWJp9RA3Q!#%to&5U_uV9fy3LTL#B<
z!hCwqZP*Wfm$3R0{AgeLfoSP)8@F`X(r%=~KuI^iL)yB13reNuCfig*Br%4@GezJ_
zJ<1Yu3OQU@P5G1FSC6RJTJO41vYx1)bn>Tn-=){`v!U1@`o;7;XY;T8BC1Z4CZ9OJ
z?-f;>k+6jcX{hB^Nrr^wvRviI)ZZ+AY<XasWf1Y~XB2XGpgq<B?&#wQyq-pn!f#y2
z7mu{%X0S={cYQ+hMlCTmGR0vAVuTrs*$D~4@7Z=t$EDb!u=+wB`q<vxzzSHR?%qv*
zB^AhUliHXjB$+N%YcF#yaNtgS&xUB(;&;e3oPn9Z`;qvypZSdB0oM5}B-d8l^H?Zd
zd?KM%xo_SslsM<rq`Ze=O(1<lt!{SgGKkbwpniQX+q3(eu!+q>nviSrqbGC9oj_SE
z_zYK(95-xk@GUrYZ<MHQoh%5zQL%76N((Kqs3bRPBp|5a@c8*nbQ<oSXC_u9tYO~h
zIZU2O+cJw@B*-oYVh&#cW2KQ-2tZ-trVE$rw!rGvsYM`;>jLlz3?@hFV#&KaB33A}
zcG??@-j=Aay{hR5`<`}Sm}|y*1O|?!dgIBg&&TuAa($9Sqb8mV58oyH6sdF5IoXFY
z2MeGGh9kSLAsC05*jbL|n=lSLrGM~vvS0tlGA@?O67q1?eX_p4ui7p9F|BX<LCv|`
zIc2y^5p=X8jITb#8i%WaxuX;PdCnptE)Qx)7Zegnf64&uWB8h9A#rVG`prb(zCj)n
zGs*O^wv{mh>Y3xqoL7c_+64~*%eX~shs&{1)~3djaHZsi6<NGkC?M@YnI`||@J@RS
z$Nixu8B<sAp(<9M;uZNHL~B}%p%ao}l1uX(!gSEed*@Udt9XCdJDqM?KLt58=76&^
z%UE&^+O{*p;gK;xF7Kkkh1wuo(fr6P$mq8>zCTa+qvE14AV}`|-0X7AfngC^;z+E1
z*o6h$2i<bqTt<wi9dD*z>eJ9u#rIP9nrOCpZq`Cz-0!94?33P|vqNb8%SJdFmugRY
z$fC!2dIX9(EVk0TPhLa5(lT<5b!|wG&fBSd&&Dj2ug%uncIzjW_d&LgIUVeAT1oh%
zWMCBy6^!GT{SruK1|wB!T~3OTaMKx!#+HLW-bkVm#=0yK98ZT72X|)~X-_f3a5B|5
zpg?gm&>h_X+LsY<NptW;>*OZE_3VMX^TgfYiP>BISiGL7*nE+8-HC1yu}CGD^t%E~
zB{6ip#m+CAI^xx;NfhG$&SS)&srv%Xlk>Q@noK}#^rP*4H>SH#c)WDLYi`KxdAB7z
ziwlF#g*7>8m#NDSZqbfm`QLG$5ELErCB=@5wziV$PZ=`V1Wg!MLX5m~yqcbVW?<pJ
ze4_Oa+ra$^9oy?lwr$|wx5-^0kqr<3Vm{smpzt0W#AMc<zt8zQ@6hQ;ZYfP6V%;&Z
z5(KkITOMYwID8b@i#FuL-*Mnidg&q&>|Hjl1hADe&&gHWy-OdAP@#k>X>UFdrg9th
zo<}e-2@=40(j&Zip(v{4A{fOjgvn6F#X_Xpj-7fQh?OC>RH*vO<Ah=-31S`^*+@_2
z1j#`XuVt{2S7EO~_=JF}g}F{eIJRIRm|N}BkPt2K*=$BVF~!NrL+)eacwOCBAH9=t
z;1RH~tIta(wMC+c<LRZ@?7<MF5-{XcH=L1>OPF`}#V<^+Yn6`>kvYJoy{DGSZM{07
zqO4=s`DZ2c6*1?&bF)}Yecy<4p}R+>0-3)y*g^XUpVtQZr93(0dN7_F>Dw4CvPZ4;
zORr*O+d=1-PpI#`*uhukZ!3M-`NRS&R0-rrE&3TUhZf3CUyY#XsN#{&e@@71eNU%9
z?4)N$FP!#vNm*yUZ@7u<*zb_%Rpo2>CkI7+ris(J3CL()9B7{q1Zyifw4pCo#&~Wx
zef$+EPg0m<!`2v!B~FIZ=9uU7aq`RJN8ShIhEv>h$CxFCot@^WzSlbvWflu@#i{PU
zvp@;zeYUo(-5sQ_SFq-3JzTsx4)qy?t={~04Y^x1xlehm_a-DV;!vV%g+fav&rFP0
zi)b^r3&A?xZpV3$N#_`qWMEsCk35fr_%pX=DhGk@ebbi^1n^-*q(jU^_F(0u^|P6>
z=(dM~Ip)U6k7x0=Z%pM7!4SF5FVL8HvZtlI3yn7QgyCT_|B<h%0e!+Fjk}~z-$`rA
z=DQQP(LrKeU~vDN=uy3nUWh#Lx;wtj8%O3Cf{tWDs`vAvk^*S(x`C1}@#${Qi7+0r
zBDQpnyeT&YclN%A#moglYhQY7;gin=6_;C=W@U?8QH!Z^?a+Gmhwq<sVSg3EUHu|R
zRm3xq&G9|7)c?cqltHg!Y}j07VsB3BZ&^l&UuexV%%G-2odp8J2xG6pdL67|+5#%_
z?S23DOm&!&1aMpFEk8p$KOtYM=WYopICOEw8E-Ug@#yZ{@z9|=p)8_|?>#6={9=)s
zH{#fKB%%OIZc#u)4;(BmLAYv4LsJ(Y7_Gn96_LZ76O!g?yu-2i9i<N@E=B&k9y4v8
zO?PHsgXR(ddO(P=bontNfj1zNXE`Twiq@fmvxg>v11+{%)YojQbqqoESTjHDhwo|(
z0@)m0PD3H5(xarLPH%Q|T-$O7=ByqoUnGtUtVy2d1zv*r$Yyb-(j>9r7#+ZmYo9w9
z)3<PJv<{S2$WwWZUpVr7HadbY7^#>@ib5KKpw{ny_d;q5V3UY6Y#GQEO42flK&BGm
zldZcSqo_+8IV$I3<;Nd3FN#ohv&ml6)#Ii3s+{s->a<DJFN*uBml$#Mdz0^}5~|n1
zjU@m|PJW~cEyeNd&h`Dd#h%2HC$lciSI`J=T4$F#_cEFsG#&4lyC@Aa$%y3JtaCll
z14&I$FF7;#{+@T;uFdY4y~JJ9;Kt52fJ4+|A#g`LTb7BSI}gG36MZr@?1-iKSsxuP
z21S+7L3+W@xO4*8I}PpOLT<c0&QJjt&&lOD!|w>U$uLyCQ|~q0zI91BJ?xY_J=y0K
z3M&=Z0#c4kRUM<0U?w+!aIYh!K%*0H`n;-a`1AtdQ1V^b9#iNeLqXVFyR+-?n<$tJ
ze^7XBTO4UkT*jwbaUFiP30Szxd@QL36x8&@w*oPr9C;8kr1D_ioJ9te&e!I@o1Nc2
zx+#0DQXuv7QAesM8iD@gX`xB0!Ha=96nIIyYBjK!ufDo8-&c*zm{0+#{4w=@Mjoz`
zb`8a%T<Lw%Uj@k?F5Ck3rIM@>=3m*#7%Lzmsid`+_^Zt*P<-ys|7QB%A^5+x3NVgS
z0{C<jLLSfa0n&3pDJ?JXtbbBn^hl|gq;M4VnU(}B>2*1LHGm!@!z6$PT`gaI1-STU
zaR|VEGJf!y_}qTSQQJq0&eeJ%Rp+2{53rsaQ~+!Z*qc8NYX=|icO?O-3&&`KnpvUr
z2;rw!7O@Ph0NtqoH~Y$$LU0FQ=!HgvHVL*v0M!X)$i4eLQ-0uCwryTBh{ip?G%p~o
zkxa$~kN^Nm1QmJvAQipt38uWDN@|I_!#aHvcc_x_EdZBexdZx|@@M(V$J1EuE)3ph
zrua`cpSmj!AeG_uS@GWm6cHgnH%XJju7`L>oxbmvJOMI@c=8iTfWwG?e04hSP+mDA
za=;$9^DKaQ(}~LDB@ekpcV?RdWH+9`<zB&zp%JU4Nb$`gIS=Rh_Ta0392S*CV7ivV
z4Ny`(JHEuUz>1$GfgJskZGI5zK)-WK52+(gJ)rvlgf5>6H|Y=hF?j&`KlprDP^r&z
z2hfV-TP6y}O<-Mc7!FD2pW*E}B7y-aOx<wQ3t}$ML_jPgdR@T6*?xc4JQkPv256Wx
zGKDU<>^x5lP}JOe4s6GY0a6X$*KhN<PQR2_ZD4j36MZrd#}p`QZv%S#^n$JO0;ndM
z9-<BOTI_jx0GiDxgpnfV1_1HzAAW}<Ap-NL{9Gx(_Ywzbd!{2gIV_1#k3M%5P%CMU
zfh-}X^S1fk1>Hz-QiD$b2M{>O(Fg%l4~wqt4^5kii&BiSOu~TPC(I`%cWJG)j<abk
ztY`?(>xhq*6@{DtaN^@RAQ#d@pKd<{1a$4r9e^$=09XPXZ;bWQ@-=<}DvEbw|D!-5
zQhVw_5xZLQgz%k|S--cD+H_FWaRJl=(xSP(EB>RxyX3l6+}GWEh^$y>L6)@N#0cNB
zwVz~5?*w`*0iEb)jiG=MowrI`21G%IBaT(gj9qbtRLk;kfGTUP?e5t4LSxt#5Wpyk
z%5iN8ux~E8i;HCwjViPNZ-ctb@r0rwuGc%{kgO%INi|^&t~XI{(JbiHgaLrt70`F_
zYX|>+vs#Yppvi?h=iIn_{aB(x?%s_H|Am+_C`s%LMR^C%xB%p>#9eF8LpasFnN;&P
zex7h~)dE|lh1YM6zBOdbsqRp~3@Y6x)5zA!gfYMQ+@b^neXy0hBMFI@R=yFt7?TaT
z?xMo62Oz><FOQNJ%sV`Wz`fo5Xp|PF!8hB03@t;~m|4{qwTxPzGym!;MUiq|>BZy)
zIjjO{*(4k~`%lo~Th`ugub3p%Ks`}%O-73NAq)auwO*=_d()>|eY6h%1S_g{A%)IL
zJQq4s)q098wDuO6(vLdmri&sW_s7GBQReF|Y{{Wg7RQn5yiQ83BkqThd2>{q?5<yK
z-#FCaIA~6`(lpxs%otlid$0IsP4Jxqe)Zo2oktzlGcb<~*zgs{I`Ax}%{gnS^D=8$
zsinYqOL7C0nHxs*%4@u)hqN4?LBK@L1RN!D*NM)>m#cA$ntf$p{qAKa>rnddy$j{b
zFA=OUKNf0d%JC7ZKchjbxziMnfW2oL^0}ia=c|uT&u7V(zLvF9xUgBiIX3U@zq2VP
zObntlD0l}@@1<!-n0sXZ8C31ALz4h4V3G1M4}XE?1ON}j?)%NEmg5&XBN<AB-`3Oh
z(;GjQ*i68{mS&N?ow-x?E6DfTh@DEqrOP`k>>QPR63YWbZOB+eS73&Y29#*Z0K4uE
z8GWC#%~XaTfab#dA}R5Cb!mj$;%7RY>sxYxe0D8Z@z32dW~pHoVL5>VtYTN`B!LIm
zCOr%5KW{PD4GHr9RP0~{sLE?jr>>3e5F8G7GX_2Zd(yrnvhA+2X(J90R{1694%0*C
zYg3T9R922-fA5Wh48Fu7dERViT81g~(xm}}K}Z}+iJ9?0wb5R*<h{AUAz>)rzS(E?
z^FK};H-PHIbCWx{D`tc~O|>(QLF9gb?duS|Spw|~ublfG!FFk*U!5yhWhVTr?__WU
zy}7XzKrS_s0M#e_W^zGB0$BA~+-JD}CW)=w7tZMvsUu40xLnnG_CkcxGUGxpfWjgI
zldl$gJz3dubhik`OSoaXGap$g0oemwDDfck7BM#H&ax|VY)I%==+A~S8j0^(;-~nr
z=YV_+xyBG`QmW*R+6hRul<;2XK1(Ioj*l~jlka7v)3g30x$q6*g!T{YWqVg2AtHeZ
z^?zegh@>O7&-bx3Y5Dtqj(!o@9+)yq2O5x$8PzlqKb6e|R6i|6pXj>nsLv`gUUs3%
zMz#K!0S=gg0$weH%5;KzGWhr{h||kOHlca~QKZ>2vn7>w1vaa+Re-_gZ<r~PllW14
zT6YX6^k!ebao!VFwK$_Mq&1(-0{4>h8h`d$LZ%9IvpYQ@R{QbZDNsE^C=d@n`j4RK
zVYTe3=%s%9Eur)@kGXMq{^LbyII=!PMMs-(W<$-cCnvr@`GzdT&3s=J;&t<w2v?@D
zOS9Bl%&r4fi+ViVYoFe@wg>(S>a%yFongS~Nr7(K2JwGg+T-^0aD{#4EFLnS!Oj$N
z3EPdHRFjsB9fvXi&WE}W+X$Gaq0zRe^19)nS!{R%D2R0(YMbMx5E`B5VSOqG7*TLB
z7OF3Es+E_<{RQ{iBE#PHe7%974e?0bE85J~1dtHqD-mNX=xU&(bJ?xaNy5C{h141l
z><!(PZn}V+%4FuUxdhNGqXWB!fD3ir^&Jsw4D-@y3tU3Jv)taVzMq}idT?N|B>HKB
zZeZL3bu|Q+y>z2HxGCR-`fQ-l(uK@(6g$?x(UZR4D^A+jjKAf%ByYh_Tzxq9C3Gc<
zA{!sIP|c$kGZU7Kr38t`YrN~p8x@A#TOsIh<1fi*F1Y1-iF084vL}l*W_2YBo<jOw
zKy9b@Rf0g*LZerJ6gr{YWk5mIR7xXpX@8`Mg*_#VRr%7#Fc2X%f%K(mg77(?iNohl
zGVV@%pod9!HxO|r)tkebRxC%A>1xU@95d>-4vC`1XlaJ-M<{wa^VCb2OhY?eRndEF
z0P&iq+)Rd~!eQ`=Z0^h#ki`OeNX79lvPaj{v7-62nglWAh-J12xWf|ZeF7d(i7q?*
zH-q3D(-b#A`({_rDd$|F4vc}vS>ZQ8uIe6uBdAZ%vKNE4L1gTNTVm(0UR!?5_O(^U
zszQ3>bs=-0lgs<T&|do~IxxR5a0}Lbpm{Gdqla@4BC-3z)->#VR5E|sTlfj~)fZ|k
z6#aeLa_Zo_0vBy*jM0F{GsJ$29|R&D%f71#B&OG=AJOq@8`eUC>F#Z}rPX(?BL_~9
zlEi}NuxvK#O{A@QI8ge#@sXtFYu$KlDH_by_;c6%oYfadGjsch$BNaWZqF=0h2XZ9
zpYM5dP)lUowNPvl=L^<soET5c`Az`?B_5Yl9puRzUkt9E!iw*Do48~4`-??Twm1fd
z_bHT^ShjfdEkU>4BoWq!_-x3OR@LHGQ9LKwo<S*<x67}rwlJZIC=}eHFp!P#o|3L`
z4v`vIy@A{F17_!s^(ZnViM<rNwB|9hQMHJTKpEhGF?r-@)|GZb?Xlw3lD@yhs??FT
z5?Tig3|^w!Zrc|zNtQ%aI~gn|Z5qR*#Uy0iC-<yp_RV22vBu2t*c)-a=mIf-mf)6<
zE`5;q{Xn4sME6V<OPGG@$Mu&Ody*~`cWM%GvgaOsu#__U{A$TEhH#INZ4nDG1zb30
z09dCUP?>W7j6DH!KlU=Up)AY5+)AzAEHU+rGbk8+*5xEipqJ4NJ!9Ntq`t=VG^Q-T
zXjLgnV+haQ1acCMw<e)koMY&WbU3?oa_G&PCTgtl521&9-Utc6m-{qqQ3yDJHpSiX
zS$0rnj2t;0XWm(o&=w*PkJ5CvQ5us3b9{zvGUy<#jWeBY&IQcO%eBcx(X?#`3QiJR
zw8UpsGa=m?i7khvm00nc0@U(%6PAw^*eFa8@MZWrgI<T|P)p``J<Ad<_7+O}{q65+
zS@xK=*d8W|N83Vlm3}O=)#ObZDPHdNS%d@gaVNJZnK{l4BvEhY3t*c?PXZ~E#T1Si
z;9lyh|IubxG$MY@A$fBgeM9UckPhAH&w`2SdcR=gh0L>ikS{%HA^nWRkJp(~v14@!
zw-4D_20_1O5$bSv;3u?#78WHS%F$>uV}0QAUc9$SS}OzVdy=Vqxz&ySJqwXERGh-f
z&6jvPSsxj&GIDv%he2dSW{@*VVDs$}frvR<4r}==zCB-J1rM6ithK-cpqFOO;m<GF
zkfD-IyPztO@k{EvwW4LPJto+2Ggn3X$=Wt=^v|H>5G5V(?yp6Xr|XW(-{k89A4@FM
zF14iNx(;8zb<9V>*oqjc$&RNppN<=puq}t~GlV#CZWh=Z_!-pr%(+;cP(sr7qE$GX
zt-4Uzc2?8!3^1Ytto18>3QE`yt;tlJbOFmkB(C_=$jxzY*bJZpj;d*F>JesBHFUvG
zyuxfbFvlHCSBW4$=(G`Rs(CSl1RfBY*42RuZF>~VCtdhuao14(f>Ohox;NcWmU%8t
z#Yu^r2UZ!>=RE*^A2rsM5EfKR7-?y%O_-?eB7cv;fxU6aTr=Gi<j8#f=O-VIu9a8c
zHZQc9-TdA~ky;D3GtZ=q6;Ma{YILE-3txS~{-RoD<mmge3YmBShYX=sm&B3~X;B9M
z{4^Ysvz%nk@crlALq^dp!;D3QBc_-*@x7$ri(&^pgNk8WRSalymE8cI?BGER<Jvs3
z{-IxB@VUMI$^whOE54?*ll7f~ikuCOf$BhNnBGoa;Oo<%9ywcibIFH`$+$=NSWdQJ
z))6j!)oG?Y2s%$n_lnb0Cio*MvX)%=Wx+8p{Te^D)rhjA26`wBDn#!V3B`@K!$V02
zC&tk=?v8s?Sht3RNe;z+?vX0)SlQ+lC$BJ8Zmfm|0ty(4$KL_H2U1XuB4hw9X{g1m
ztIKyPSstj!>0`@#i%0AykYZT*(LFV?Ce%V~=iP>OY%UTB)dwfHa0Ussj`;q&`DoNJ
z;FU!TH^i^6g98?qtD(kHs2R_#T8{{l5goaW+$Mk;g+Bns3d&=%x-n#_8k=|U5ZVL?
znQN8N9Ib_{QY+?6(kibvH@pG?lDK-*)91J;CUTkTM$c;4)TCb}_b_-IO3(18aC9l)
zpi<h8h^xm4Qoo6trnQCB9z?-4P%H~ah}sm_Oq+ouF09A@(aJ+O_*gJY{~%S3Z(NS1
z&gW|-7E`n~k-$s&Q~#A3S+#L0sukvri4;W|(MdhX<Y=C|^9ydn@)%mALW-Wcm!#?j
zH8Csaxq#n>%%|FXpPKi~;6p7ujGZFeJMzH38ePP9e8<+|iCG*~?2V}KD-g)cb)<$K
zBYCa+faBXN=i=Nu06!+%QSdt(A2de;^ZAJNK-he$p9T1tCLYR=ud;BjXn_61b694&
z`@#2hUm|i)u4|>iPrzLUTGe_#_qyF8f+#?py8{8jnnL5+5%<Nod?1d<13Jrr{UXCb
zj}1jH$Ej8s#TsnJE8~Ob^zB#u(i3s}wR~Ti<%3Z2_bkacex?ryA&DU||J*fpvEh|g
zLW-QF$My8PjJ&bG?Xhb5tRpeOaSOoHg8ZSJM5!Bv!@WQZ`r&+UXkP5@Qb4)k$;!BN
zuZNwU+*1ap+y?|saJT5)G$hXC?bXDXuTyCkojkSJ7ok51LGb%_afDsB2UFBEZYR%3
zZut{zLKr@FUIeAFkp*sJ4X9lQzHyCR)=+@L4yL1=i?SwPqLoIiMO15a?JhYE&GD3}
zrR|P|MAjm5UM^G4FwPHQ3{H>3pqQV1(wxFDo7scpzCHThoN$>H?mKU9NSX4yDV?<X
zM1nTFBxOtg&>I3?MsOl)C!Ku^vC-Sr#`NcRfbb39hS$D_;s`C7N;s$gOI|%Z+?)MR
znMhymt>N#pBq?GgAWZO*?l#qzB{S|dl5pLDtRmM276+nwV0{c|+BS>y7eLG`Ok67H
zk_PLXzP$dJ2LJkwC+}mJL}q;d%B=f=9@3c>f+sEr8>@1NF{XkwzA*O<rIYstOJr7V
z&)AyX;Z5$~M}|z7^kqCqzs-g+>*~r!Z`K&Vr-m#=*d4@=IEY6%vj=ukkD4LON0bxV
zD+qUJyaHgFbLY=bckVszN7uvAJZBf}6Kz4tb#IaSYg#&yVqlr_6r1sd$=<+#MTvUV
z=1Eg=>9C~Y@!8_pYf-k5z;nz^1}qhSijyU;JFtb8k}9qJmo2t~0FS4dPVgM#BLKW`
zm0(JIKsHuOO}WKZN}>_el*8AUa-8EykLgiG@97FFL@qNK2dug6$vK8{!}&!X0XEOo
zT(mg@J?|1y<~hH&LaW{Ya>UUx3;<S~7~kmtg#lM$zIZf+j{znLAPS_3JB%g`nb(2J
zdhor4*#ac#5;>4FdUG{4X}imcC+jl)CAs@g?(qzlKb^JP!{%-tbr4>6F|o!R{`iR_
zvGc`AZ8RKxaNN$dYXk%Ij)*r_fW`|6BYvOznWmg-Z0aTH!n(pLMs40o5YP9_uHWK8
z%U!rJ=lJ{~y9}`GDkzh~K$OSpu36s`BwDrnS|IiM&uV5cDwbYOBA@a(u1Dk!y=!b(
zOtQPCfU)#aYt%eZ7Jg$`h&*D-R;y?oK3Fbs$BTw0#1Ug_>(?Afbprpe)F03b^C6Z-
zYa^3j1fu#~2g`CyCZ`IAd0lqc9$JnD3M`4LT3n-JN@fG3J5M|$dw)URvjji)lz|>H
zR}<NVGHM&P_?9v~1w=3zez|oXrD6WY_>Y>4+EvCsY%w2xOX7NnCjyzsI1g;L)h?Yb
zO!DS6TAWxVl5XzlJeJPct_(kgWzp?h=Ku$F@EgmS&(u8Mab}}!e($dB1Wt=2q1x1?
z+Eo!x?35rbPR#`_?5HP}cwLrmFJ&G=xIYP=!*nh%mN#*Oy3$EZyR1aD5mSUJLafHX
zSG98L^oloC*Bq4Y-snyQ1;M>if3lL#fhXI~*eZ8suotD5UE5~<o8xBEPXMK;kmrpw
z?jSvKk|`_qOLfkm9_c5~b&B`_q^-Au)qn~KcEg@;>5gCSi>Y)sUqY9#;DTLoQvmxQ
zvS6_yG-b_X{8O9SFMXWP-qCj&iJnJ;4rsKmVk%1#B_FSg_>}}?^S%cnS><O`Os!iJ
zU^*2c^_Mf{RA=vc3gquBCsGTHa&MEjiRZ9g`TfdQ$xw?hv^?&=y|S}LL{gX}*Xw}e
zYdhC8nF|gRL_@Q9*sB)pevI9xZ(v-9;$IDXk&KSAv@rDsfUIH-vvUBYJU6#CnCc-N
z5u>N~WE|nHSZC=MGg2q3V)DeAvHU}IQ-EPUHk<^wxuw+`F}^}DR{NN7cm8N*5auo|
zGAe`4AxF2P+vC6>kXvmO<$cSm6<;IGeZBdTn3#S#yA^(%S~kRT(Ln*9WaW6MZGNwI
z9gLSpzm#N`4H4e%iFMR%*KkR@xWoQJnX#k_@PJV(qvu~nj8l7{Q@a2Lx!V!>nU<+G
z{lE{=haF9H_KaeBeKW>ki5Te{L~3BcvBf(6ywE$5x<QW;2`|iM29~UQ=0%>LOPE@9
zsLyK9)ydEo@rmq$tskt~%`W(N8t)&*aJ%^yKfTm$9#$LFJqooU&=W~xeczAGhrFpz
zPdyiB@cp2FjU298*ly{lx#i3t;Q6~GaXq*XOl!dT#RDLy=bLbg7G!3O+I-7w3S`OH
z@iKE=Ve|m%BBW;x|H5&L6am2)UcE7_sofx1WuJZeFaskahe5MyvHLO2xdp3bJ{|XQ
z8V#yYWR(2)L#p7VY>M>|lR9|Tr7weP?b)BPE7abA!hGC7!OMPteQs(2(eW0r2N^3=
zcbPv-bsuQ>1NQ4&$oCcbmOmMX0Z>91xFGScm|!B`#;QWPh9!rrci@U$DD?*p&w1t+
zg>fXzn73xg5#{hHfF~kr14B{4u;6MguZM_ckvt?_Di-MAsFHErGVb?38BV@GhT19N
zf_}s`Q%W7h)B%1LykUI{>D9}!qH%kpVu(3LExeLY*zMe%TEOY9gwc3)K&J@k*<|C7
zskl<W(ga89X(Ih4rIK-A^}$S2pg7vUKEm37lW~>|8*}(yGD|52z)EEj3Yq^hSJa3A
zm&H;a#`D)#IWz*;^o;*o(EpCnzt+|@S&qT~!~%GW@V{&7|Jrs5*TIn?35fa$_)33m
zJsZgToJg1=C5dkk^k@1j04@hJnk_SojUpgnQc1MtAGj_XtC)=yNLLPzEdc=&IPp#p
z$K|hyI6{x{+z@J7?%&j1q?@uzN51T|3CP39DI}iAh%`Uv=SN0WD)@*{FU@Y>#nSNi
zAh06_q8#|7@SFZUKQy(0gMh(V2%7OXpu!O?@NnR)-C_HCqP%zkJa9i^md^b>KMGy}
zjM1X}Od!Mm#88k@23U`b`gH>1zcCi-<AGzRrSJXMzx6Y-0QIK|<XZfV_z>!Y3p`JW
z4bR{Gt)Br902oX}={Wsu<QON=$Rk?Ew*S=s5vX5f!7YyCZzC7vfkwWv8rJ_?e|q}Q
zwbX&ixgTn4K)`_SM-7XVH^+j2@DCu_S^<J_lxL8gcYv8Cc;eXu8DL90fg7>?Y=EtF
zk`s759zT<?&wM1d_{CT^zQ(hYFekv(0lcc42%Zgta+>+=L$yg|uCx0x5P9GiLeyQF
z)B#Gw@rqTj{wwck07^G)^^C$G3G11&+TlpyX6pjne~He-{!HoAJz};P)2;*Y3wR%|
zH;GBAeq1gjNj~~@)ka1<=ahYqG5I@s+hN-&{s5Sn38t~a(=0B4a;*Cu444#Zr=LD0
zA<%~Es0&*I!16ObB>{Nm{{qYfK7fOb9s}YD#18_!<W)HSz6r2iD<Mx0Eu0E^Fq9&a
zck`pcx0Ct7CmqPwNq}?u>fOqf4G<teIvmN}AD~p(0nsSn@g^sr?7NK0gP^qWQ0-N@
zv@Bt>5A>1_L8Q3rAKI_!psJ;cm%-?TK1zh1OOqX=gO;5PBLDymMpFfYY6F0{2V<n+
zPk;yd!_98Kpov@>p3Pto@IIO`@f=R33Bf0kz6Z>9N;y*3)>F<O7us#ZVMD5XTt6-*
zXzTz+?+&1`YQ0I4MXh)W=%;B*`?(*24XX62W^QkR)x&~+9r6KJ9kGTtx8?PVv@cyd
zIj)o*bMCzqp5D^q9Z2AaXX3nh?3ODh{RAMs#ZP|1<0hFmb2Ae2Y`LPL(SCVe01$4F
zBhT9Vt}}qHv5W;FD0KBj&V-P7lt~72+H7E?2%H8R#OaOjf6oNEzK`N%7bo1Rb<Y+`
zioh+|1oSlK=a+O~=GcfmPu4ns*bA;Ftw59l&q9h8D<@QNA~3kmZZk_q6l{$nVL&v8
z5&#^`=)-WXrDw==0G$65F$~*N(U0b=H=d{alihZrHxqj5TZ9F<pD%};;T#*_8QlQD
z|2gOcy)nQrmS$PRS=_$UGqRshW(qo}Yy_eQ^Z)@EyeevXSbOXnvDqYXxQx+wBmrW`
zFrV|gY2YbT^Y;05pap>v8M9xn0*<bPbY4Gs-J|xey0q&(kVR7E0UN;iS#+b<Y9j#y
z=g#8jzQ?vBGxkpZbw2^wrj=75{sZ8&3^2AXP3uuk?#~WzKum%k6&l3LSr?HLwpr#;
zaTaDXB=@}}wz+Y*B5T<~T^D!eEJn@n6#cZosnvj|qz>R>VD_OhUZmFmtF3l4JNHbb
z*pTf4PAAW<mLG2iO&>eMkjOt`4pIP{EWZ8m&sRVqw>q~R-pJ_V1h`_}0ip+8vI89r
zaqs)nUVuk`V5SAQ#2eapb{QFV7Ec@7W>qC`U|>!&nS`SBRNtxd^wUqO@(!n}aNYp?
z9aZ@^lR&OIge-rI%(`I#V?d3Z=RITC#gNicL~RqPJT^RZ2DlZ6nz3(xW?liJXn^hw
zOSQpadHYwg#`CXPVNt!_N_n8s8d0cl0dT;K=FKbL<!?xlU09;e=La2}vP?a&!7qW3
zBG6};Je9Cc{+Y4c#Lxds(qO7!*t!;mK6hAbF#=I{2A=3ZKoGs96lCY58=(y9XAT`$
z90G5!i1CB}OzQ(26vJXFRF?)zK8<MdJGciF9uyph`X?4a^K*EP{}!N*Ru!1H>R<{)
z+LHNUMRiDOY+g6En`N1Znx0|gz~cUrJ~>#j{vDe@ZXU399Z~DGq$`BxqDTy(Xa!=0
zGLZ$VkC%<+8vg|J02)(n<M7<@!e1#7?j41)(L{Cf=vIP41QKDOSgTjIS*FhcQuIxO
zycMY8MN#X#mq;BeaHl2#4T>|AP~WOx8;D5KcDdjP=5GOXco_y3s2C#(K#U3#_x77%
z*D;pxT>zi!OQtcxiY7m)HShT~u1GZuy{c8Bt|xfpk4msiUhu(PQwlP;#aF-q7jF@P
z?C&2TnwP<LY*)7J-)8XaghK)61I)qi1FYrTt-Vsz#d46vnomm53eb=xmta_y8fV_m
z%b)Pr&ngnJ11%~Ip>>2at6`S99t`CTsYT0+Q{VQxSDXMEwjyj7J>*e~twav-fA}*C
zC?1x9bN|Es=5suhvdZ}gKtSOG%91X9mOI1~*+ZmvFlKA}y^-NS@Cb@E{B%Vi0=iym
z3VxKg2LL37*ofiIm}Lfc97CFNC&uhkIX53CWuXZ%flT^S88@E~acIE-GJ^y(+64oI
z-aXQ~A0_}J&<+u;fohUFS)~53!g2rLJ=3upLzg%Q;E$VY76R(rFX`>#IU4$Ik*{oW
zoO@(u0l`Y*{&Le5&+Y1S(2ico#}uJwH3F8<8jSAutc-6prTM`knR8ec3t8VbEVMYd
z^dSe?A4LJt0t=W;yNnbS@CBYP@`-r@*6QFoQ4KW(A!;{`?al$4!cSf#sG!%ktXoC6
ztclths%W0gF7gN9)Kt?*EEMrTc(H)(ylo;H@PFES>!>WZXm3<rUyzdSknZl31_?o0
zT0mMrN)YLg?rsSYq)S3jP!VZC0Rd4^DG`tmR7AP+ac}XQbH8!NH^v=zjC;Ou_8;uc
zyLs2Mo)vTb=5PMyTwEA9q8irddo=@HUIT3qHd}qN?W7og0C}PPU2b3Mmlt|XvKT$K
z_+NH%x=~>++cuG!n7FosX6amKiz{6pb}qrFOEgo)uGE+XwTenmFRRcxH(hx6`<Q%5
z(+$DMQ8<umB5bm9-zn_(nA_YW50?-5;KRXVMm1sA)<Jc<EEuJ?lH?uuv_+^=a_CY*
z4PiTh^lC8>jIx|lbHp|8OF6XeV1>QdQ#S8VyF$YX-967Zp4jv~CfS@#gmSpuvM!33
zf<Ufg<}KCfACDnsC!a*G7D7;-7_eUQTs4H3W=5XPqN#`Nz>$7iccFcaEwZUkDj_tD
zDZa42``3Xh_nT`ZF>zw|1Maa3ajMl(7=*|Ttma13d122oMZA|ZDJl1re33-lX}s?G
z6!{DOzK;pXwR6;?S<GQ_tYdw|z3k4v7CGc+V1=1z3cF4VI#imn(24YlZ$myUNA@(f
zFek~+<|L%P5iH}O0~HC<hc>5qrOsU-$igl&;Ek}po^}a8F3GXTpS`^qXS<{rWFxO0
zm8Oaf@%Gw$BMk#8gr&7VcHr6syFzCqcB;2fFn-lk`%~7RAD?(gVv%18`6)K(c#+Kb
z>vK?J{fxCOzQ?ZzcH-1*dtY`adW|EcE?g3daWsii4c*|5m|+mVIj)mh>-IRu)|W!&
z;>x74Ip2yw*Km?R-xD0kdS4*qIMqfE7Rgk?sfuh|NyJi9)%kPjR`!OY;%msChH=@6
zcWlO?Pf#6Rp*v3!QRRlV6j}-1T{Oq!v2r>4E|RqA%uT|~V3x7-K|?QKl{uKrw2-9q
zsN>wOW@MJelOd4ZQZSUb<36*VXEbnKm~#heo8X+vx4f8-S7+&_#-<zzWr(EvK%nt8
zB7ehK?fyw`ahqWbOJ9&%S<i?KT?)THFmZ-fbmy|(J}B6vUbtR1{`t{bPSmh#mdj(>
z4^dL|+Whb&?{z7ROt%d8?ROQvvI1M_z(r`UUcH#?=a_NbwC@c~1yn6-k5?kWp=1Or
zb`VmAsS-q1GW@K)zX*fkTUN~<MVGTat$DxyikFQm>irXqhr|pg!-NR(*|Cm5@g#9W
zd|6=>!c&p8)Xh9@jYf~f8yoSpGPng@O%~ekQ-nS}kJ#T7`AY;($E1VuG9d5vCZ!eC
z&f@E0s@c#BA|$r7lvq7E;VV4lVJ|*uyhw;7-@OpWM8f`-P-}LOJm2a9jyj_$ir(z@
z<3P{rcZ+2)&lQF=F5)=^!V$@ODmR6j_ks<%mZ!7#S^u|QLwroL02VXuU5?+?943f%
zet*NOnkss*?%tFclwgrMdCT>E|G|k;p#xRRxN0}<<*~LRPxT!C@9pBUV?763Pzeiv
zldATrqEWS^->Dxxr1wFTrec2IC3UY}jnTJ3^hZgpMeVyUu%yh?i<J$Ax7R;}ty$5P
zJyvJBDzg>{*b|Yx0Zb-$7k`-aY-cJt;^RlbA8I6bBARhbKR?D3OOSaBXl^e@&>{Vz
zkI3o4_a~w{AvP~0M?4*#O0BQP$)h$!iZXGt)jGlEb45I}chDsalS0A3?h@0w%oA>A
z-#q+N8|pRcp~N?2zXygJHS}`!;mA?RUYEQsCK(GydWYGQj{+DtEolmled6OA;bYTe
z1T88_-SG!(YfAmmrwMqL4Fck*uko=yi%Co*@s>i1Wngzn=JpQU6!)gSU}qz+>9C_A
z@7D7=p#WgjS3?7B@*S>U<tII=v_-y42-wtpxqasx&nAvpZw%oN<t1*2bzT+0r_<kz
zW^-FQkjW7OQ8F0<D*fSLd{_QBO3aj}=i6G<Dz(xx+N)jHXV08keK+8<rEnqbC&1eV
zYsj+-Cf`}Klcgx@eGw5+GPt#9-{pd+J~be6F-+zij<-@{7a=Gi3wkE(!5eB{!37`{
zceJsHzMNOqofL<ra$8z;8Lf(a#vknj9*t0dT=S=VnNvoKQ5@+*f%}GdlOYP|w2kQ2
zozU6wegF<n&{!qq;sx!$B8NAcPMe#&9%5okSlm9ee}g3wTM#ttZsj^jagI8Eru4Fq
z<qtU2Z!)%yI-Dx76%FjqQZDcmRCw$}`f}{Nt}s;(`}8|b%5W2R3%C6ZkQ*F~CkK;x
zEzglTh|El+Tq~B%kZQj<LOf#=ZMriLewW@WR$(I+)RTfrG1m<361uVQk4!?K^>gs`
z#=o%vdZ|FD3l%vWuqU7?XmmxRf|=BqxP;f%p{F#g{ege_su`zEdYYkDUF2l{#OHaX
zsQo7gjkE?Rl2tC0>@VqEWy)c*(>v(oA2cYkmp!w#*5qXEW!K3aqc6`&KDDS#?dMRd
zTeX(Yn-g+3l~QiSXLfkmhsocl>q1Gmj#~t{QajzdY=$KLTJ#`0&zW#8o84L#Woq#Y
zJ#>J0dvRb|b0u(}xSoIGMYIdbz&<vM$8io50y<X93CmkkclWbXyNG&|q@K>abGch8
zU#3ep^4Nd>9G$1imzq`UHdSNkW_)IRB_fWxt|F*=k{iXI4)RRt{4DB~lLV3{-iWf7
z_}Zvqj!!4i)@G!e70`|-z5L@0D>R{{MV}Vb%IA?vQ;<HYG~lIk&=Idw98eqTcx7@q
zk*BUboYhsC)Q@ZE={sQ^p=J__Nz#{U=W%k2)Y{@_&4r9q8Sqc#(l~q+8f2U0>z;AQ
z=i7W`AT$80Qn3o`!*}ovaPF0bLs`62r<5-y%m)-<2p@Ph85ulY&R2^Pb}b5(0<R*L
zbTvGD`a~*3a!#L3i=@KtQ!OLkjm2fAof51z7fLGHU@2Z3lQpLvZ16{otN>tjeVDWi
zjKT!tSO?cKjnV|c0aAD^j!5mZ<W9S<o$@i=5Q@%YO@Sm7-*)hgF87=}+jw;Oe(L~k
zzQwWouU;%3Mo@TL-#o^Zvw6VP9Z~rriey0AD}eMaBoM^*5QMun6(-*O%UP%w)1%W2
zh>+9%=hu1pi$=V7QqpX6E(Lfy`v0Njd&nO(Pd?GSk!9*&6R*-*ghTOiW-QKKN1EU@
z#qz#z4V5Vv<Ub`S!<fSsjfK3ih4E4@1#F15M%kEC>zH%l)?R2rzqs_hGIGT4%<n-!
zUY=ElQ3_MY(<<&_erT1lo7`UHV8a>Bdms86#BbeDdg*9#fiHpOH?-L^8a{n-IP#bk
zQ9vakY)V4@-Qxn23eQPA;yN7jTY1RwUg}$N%_@P>$s$$%K0TauaS9)P;l(&d)GX>F
zp=Awj$5nB>m*8DH6H!U~IMIZ2LP^arOjHu1FUEaUI0FB9QdkC#WO$C(C(it1VrLa8
zpUJP#rhfd+DZ=nchWWXfzg?jOhWWoXRLOW}PI{32n8mq|Xesc<`2G&|?eoBSFHPx*
zA2$s>;*PNlbx%EhaYlQD&TdO;(m3uTKNQkjNBP8O9P>(CAcQ#IaV<mpcpiPAq^B)q
zvH9)e_fXG;PX=qtT#uVB#sj9Q^TZqG<E5uC2%i|Z)7z&XH*E=&`!wb%1pwrK=Uks1
zNcjI5qW>A9f0ow&tfK#&jfUk{az5UbBMcGrL)c#is7wa`#Z*M_3)Nl`vPR<Dro{gA
z<+f(KUH;Qyh$Vk?ia})#QCmZaMgGJ&+}SepCZvkQL+blGAR>Wpv;yJ`M8e(=fjR}M
za|#Z?Gqo*~X~5VbB)jZ2>dbm~6jBTp$P7p>kRUf0hI|i%nF7wj1ph@vJ@y<xc%V^#
zK@d{CHX5J$vHc~Z`N2EN3n1w{`)wft$-e;p7;o~n87NwT&3vK3o_pI2{SyqVIAW7Y
zr}^!EK*dS~^#UNb;|xfJBA`%%6j8jwvAPX>Q;#>Wd7VHb!a+64kL{R>L$q*Z3rXdG
zBp={@hZvJ^ZBZjHx|wBx&D>7E1}>tx*8tmh8StY@1xm>tLaGGD2rf^J2hbW{1=z3f
zhw&4G`PW)_t-B7C9es=Poy@Di_8^5d$##J4u802u3H>t|sdgwqD+g==QsI?$P-5sd
zjs*-H2(KjS?QY*b83;{k#8-R#RJ4Yqj~`D3f*Yae=qPAOW@vWVnjuB05b|lRzXiEA
z{`E2$2bNXft!K4+RJk|U=SmKvPt*E7!lcxlE>?^{*bkKrA_fOX{;n;hheMts#n&N=
z-&dVIUL&FhSg<}>dIlV*0qR6;FBGG8RYO^=eDxfP^QLS&QbMl47dsxZ6g~kzTm<P7
zC$k}W<~{?4sUwb0CRp!_+ii7Q2$REmfvKAh>ZL-t&msFmfTlrEQ}@v>Bm)>g;mWt{
zTR=;T1fQq(|F{Y2^!Da?!UvXqSF(jMq**`dRcS~La+dKTWVC4DYs2`v-Ea~Tpmc=V
zB3vqV?e1>@i-K2<`0@kf$xAa(RP|;S(cuwrmW_rUv(6PGs=H0mIOoMC4MD0o15B_a
zK=OSLv0Afa;_GX7E0huu%Kc1r-eju60h}kKE-Vh_CQb<o2)VOXU2mn=R9$yJZiErr
z47kSk%Db~y>XA|hTLsqPI9b5tSSVppB4AiHVu^Zo5dsO#kR!fa>Ypnd7WD%w(^h~s
zjquF+NRby_wo$9Y4TlZs=9W!Z18f9QM9R?wwe&N(5F%Pa2RgRUIe(zu=@`Z}cySe`
z7R2Ja5Fi0|oR#2pq$=h%Qv0a9u4$0FEK%IdBf8r)a2HDH(nvJNQQfjBijnDi-ZxSB
z&r{$3tn$tA`jBPz`-yR%?|-Vxc}{X_W9$K?OS7;cYJ;)(w5?2~uk>@lK=r#KBccX%
z>-4_U?_k3!t0E&*x@4MgD}KEktR}cd1;Ga&&j&H$1PB*EHWXLPZG4hd2|y$JO7Y(N
z3YAy^YIdKzLC@Tb!zTxX=@QUKq-A#*9k@Qi+Dl{ZnW_b@`80E*&QgNN=$N(i-6_4C
zUrdndDbRAw@EbcMwV5#RUJw30bjeHVDVYJ1FC^(u<iAWr^Wj;)Gy+_1D=fgKJpzJ@
zYTrv9wi)PuYsqfwW8r*Z0(@iT^rdo3Tl(YeMmgBV=ct5b!8ko(j{$Bw2;2UJ)aDJ+
z51){N;bWF4py@?BeQu`TA^>(?FlpK{mDds#qy=z?GA|gk>(7lofY&4OdiedoOL6zW
zT`1Zc=~QRd{|*73tI}a$kAi_}4TQK#iXD=AQ2psXe<n)3_$mb6_LW{<IJ-AdLSr*N
zPkAZu(MKcK5J~6@k}uO31uMwd<8oDw`8wT))!*ImN4bs;<_@{$DUU<x=P{_){K}u2
zR}V$fPvViPnWbj>261qReu9vp_tCf!-QDO9p(hDq%-?xXD|xEiBx#dJfX*AX&~ZF(
zdRVb^wsbdEOLCt1Ts)%%K0hNe#y%01TQ-k&A+_L)@3hT?OUX@&Sumr`lkwF#zv~+z
z?n?14T5>Vgh|A{a=p>=6?nc9|wskB?rFsu>FGxi`3=|QRSUW^lYv_7<Y8VnBwP|Yz
zb8VN$$VM%euZrLDD8iK!3Klo?8${D=(sEtYoVgLIuzS`ULL&spcTP*pr8GYd2#qyL
zQMd|t@O&k`)O|Vih^UKtHHbBb>?V9gQ#P;0rxbWadYE<*`KpwbqWL)68fOB%6la|7
zzhoWrm{)I>)|?bU=dI@=OflrFV4gmsxvlWMDD1K-tF8;!1dfTo8ZMj{k@wi8229>R
zyJ4rJKl<G}HoUzDg{N`^W^!{Of-ouj(e{`!;R9ep+Yyo3!wK&aD|VgYcMnsqKOn1L
zN9TRa4a5Ora6jxsC%=ki^ZNOWOQ;8mfPPDS=|T>-BSdYuE*T3uWFh5^fqj+vlzZ(e
z7`N@TH}>PG5B46t^}DCVr>)@-8M=8RS2E1uK<&MeL6<(M(;Y`E&!^-F5j6RX`v!7S
zik-Tdt?(I6rt_8BNC2mJ8ac+0g6A0tsoju0mV6Qm8%Yc;4Vh{jV9)Qp@Vz0`L_8yt
zCx;d5Rt<M<VwsFRYUE2|#;0;udycX5&%0o9l=i^AJT30XCxg3`t&mJSh9y*jwM0B1
z*9fXqoc(ZGx0Z28Ig~>ZC{`EYWD5YD6miDMbtDpwa20-={)2vpP&~VjMj^cQjqhtD
z@ZrDMFvE0z8Nyg}cWV8K&P6<pbgAQH0nTvVnpq(DOr#g!fWn0Z^Y`CABtNH+doH`n
z3&$lS7m`#e7UvdGM-l!`fO|*VuEvE{-TgE>jt}8xGqn?zy_clX(^VdKaviu^`5nuP
z>6ur6Zl_QJAx~AobPh-9(bJY;mgens2n-PiEf;v}kd0Vy3k)g2v^`??ZX`^YXar_H
z3p2)x9X>0P(k~Z$ZAp5!##X&J1Y7cHK@j@d{D{q90;7Xpo7x@wEYm2x9-w+x?k!#J
zjs-Mv9oamx-(H8kgCg=<QMB%`VL6nlpaSfzl4l;A-W!4V+UW`+K`bLQTt-)k8XWi4
zWX$kz7P0Y_C`bMMXRe_x1?`b{$``-Z1}*QpU!i4*MY8KOFXa1mAZd@JF+lTd*EEj~
zKF)x2(Fa8>eHsM{=|L>pvs_mGIMX6SuYEQ7!{Xow;;|hEcI4md4Za9K(%p}?joOZ5
z->4GqOlV}LIQ!QTIwAodYreH@`Oml1i<!YH#+iKi>vjD^EL_Ozw|Rd<{{O3ok@x;`
z3%J66>_2fs(={=we|_k&Zs_8Fiw<SA20%KTkM53qq^k@z9w$-5RkD;X#nH8MwQ8Ax
zZT;r%AF+|kFs^@$+{6geBhb>r>pZuTKk#tef4NX#+;~%VSGRPrQ}wNv);VjBkF1+)
zVFt$CgF93k<-byD<AYeV$}YZX^=dOS-g(1(wS$d$fy_IWMoaJSyl^0nc6gKhU;cpv
zb~>-#Wv;5@*<3}#OzG&h;{WkjS#_BaDToWN>R%~FA>!BK3}*ek6+=@6rc))cyz=jq
zlfuNvD9#A~b@XH5`B{~XWJbr!$tN6;83(6%)baB<iQ)MqVp*ksX$2j`Wk=f&{ksIA
z_iP|VF)8iSH$3j-7!Gu@u-KLQ_w&bIqk%Mk0LqE>wogtQ2XQ?M53Ae+jP?OdBgD{^
zS581`FkG9^yYfuP5Jkc%5S9$3@LFHH2P}5ZL$Fi|gdn4kgFVFCJj_}4ZF|1NI0)kM
z5Xh`YKsiuA6+m^%dO@=LC*2temM~vfm#!Rde3yg3QK*I3|2?e*Z(sOI9sM%|P~Aq?
zfg@4dI_omsU?cS%Tm>&cov_L0q(TJDbr!%KK|BO1nFOhjZ||bVYn4&uKoyuwEe8N6
zKE!?B3-*aNDT0NdQhk2PX;MriOvMn17;iZw&kbn%Ui>E{TD;>vQVzQDJxD>U7G*Uz
z38-}pmoE4KxU9?U5yZ)eh9UQ#by3k^xUtG43k32*2;n>QrW|!b?6&dOXQgbwB|K;<
zv4q_mM79AHlm$22Kj?S7x)c%w0rF0`MgWP#1BQMNS1^2rgZwQ|rcUFTcgjJaNP-a5
zGJwQ_(wITkFf}CQS+nl-$Hx)QaJ(RlWhY<!61)pNTc}#h9+N7xoCFZuhc^JyViD~G
zl8lU8-T}b;ER+LieS6~*RaMy%8Y-M?@7PTM6j*W{Y50T|wF49-%G`s*WT1Xxsu#Es
zuB;E70U^F1`3A2wh9bUY#KoK3KY95-t1zruBTlfyZ1hjbvH|`A5-U3-qFh9L3yf>}
zb%+%>3F4&#VYI}GXe=Vbsv(43#Pr0L4dfMU3Gbjkd|6XFLU-?K5bLGOL8*Nte@~lz
z_~UN&-Vo>!0{Xq-u*Q4&Q=7uZ3&*DrhWk~1F)_w_T=H{7w2vHVa58dF{HQ{210dUW
z*<)M<%K8wlzJUBNQ}Ks)e<##iFAYd<=VWCLouj{I*sfw=sF+~qlaU8aapeneA~^CV
zoM~|QKLg(I9&*nJ54AsOgR4C_aW4fMGd8NmNGc<GK$1jc3zv_$PL?89U@*n>bH_|_
zimgBIabp~IVVXKAR4Wf-95pw0qt&L0{d^)kz=cmgTUJq#&auFAy6-(~qeN%H^Sx`U
zNK^}oQBE_CjCiq7)03=HBfhaqCEjlTB)1qO@ZT&3hrHH|?k6q#R_Etn5O()1L=)~<
zlfCtbV$V<rN4y!=jMR9t{dOD?Z3AtwNeDMdzz*9gxDRqye+Fx!X3%0a$-Fufur9RI
zl>pWyBklz0Ges*o;%b~T5l9&c7f&R1#Ht$sJv}WMi&cWcBHaVE!3GGi=AgZ@;`P8V
z{p(N=V&RDz6-bdbp-TT5<!fLx&MJgTjp)YH7@#2nyBQW|jtToR9WhcGonrPNv%82m
zwT?pd&7;x@^bR6VC27)CCVJtw_|%gbPk}k$q*e60MRXlT9r3#3GN1Zf3&&xvq`K|M
zh25)dbe+tQR1!;7yV2}C`1cq%hODrGiM3u7A$2;CS`f;;U|E)7svQI38;F$?bFlWf
zF<Lc~vRpc(k%PGaCu{dhlM1>Iia0R36<p!RWr&c~D91j>06jf}a?h|NEKhry3s3;b
z-H8M@JiED~Kf8t|4m5d&%z^;yPhFKm@!SmgL>Xbb2$9O)W0e}p5<}jY@zxt2ZY^P1
z^=L2?ic?p8*=t0dWbNVHfjmilp>e2gfE4?nKn*jY^=|nu=eh_#D+Y{O2Q!Mi%>>wd
zlao57ioE)ddSVLOTwHyrDk$%XYqgNS=AFW#bsbwK8Gd*L{XQ#TJ=jb;hDsDldl<%-
zfrHEl!l(k<It-5S;zaS;7`?Duhdi!PW^zpX1)>YX$yA!WQER|lXJq$)W``uQ8xSM+
zRQRJ097Wf|i-9mv0ery>_#!M7tSlW@=^}H3l(=xC`ggccq>^)vE!%=bjYujcIjk?v
z6M-0-nxP`H%XwG$BfF1&{XmpswtNI2)@$e-0d?is>9YQRN0d8ePgj$N0M9T&f$lCP
z;hQ+9XgA1%<c7B|o+bYId(|j2)+x^ej`TLSxg2zTG4)(jUZ2xh15X}MR{JEtGJK}n
zR#ObxD=ych5#O1Dt^|ty^Q09O9$kY-Nf3av-C!)qT7iQ(XVheGpP!0mO*s*U5JV=)
zvaqCwAUhIF&k>@X`tm{xruK|q(DVlfkP?|gdN?VOF)@swJ_(g*;I}P(Tr01VC&q23
zpEwR%1IX`$8LG0G25<^8zsl=##`m1W4)Ze_)JCx;*;C}ce_^Z9mb^K(Ox>jY2o5j$
z?6M{9d*FV6)=msWQvRdezlN6CYPxH5`dr-}JT@J!;UT=W#CKQy3Y%-Jy2FcGXgWn+
z`_5#k<s2v>@&S|5LJ(Le2cDiHM-xsf*fhwUFe0-y;ujFDV>lX##7>pEf#=jAQ|P8U
zqt39`NWCHW=zu#7P8A!(+ERTxx@3XZO+15}?>(90u4Y;Q@Yn2GlQACKI>GxFS$RmW
z&v$Q|_ou|h*gT0U;Y|{oGU2t#wT<I4)Bj1KIZfhkmZ@>Dr+MkI@x7bhS#<z<$ehmj
z#ljeb)A`bCLqIHO=cCs2FJqV1$S>4)1Z*z5r362Ck=-D)bBCKA@T=A<J6zslNI8oj
z(FcJemdB_BYDnSQe#|=`_7^$@z(LlUUElE+t-e5@lVv(r!!dMm5&-D3za!>%j7oH+
z0hRnNMs*w#*gzC@Ru7l|LR!sp@O&TbJ3W7)(*v+duZLdE{eAvN1Ul6XkeM8JvLzcj
z8Sp)?^Z5Ck^kI+6)7!Jr1kp`j^%HBfs<Edf%v6P!0$rEf?<M^M9aX@C{;%)~6vLz$
z)tN%DzFZV7zkbE*?9{fNU*$g<&2Z=Oi}G$H8=%ZxWeKB$kdU`R*M9)<$k5LRg#NZO
zU?afFO2HdTF<SjaZ^0b!!k^FfAnMG2-VoX$@1-LW`1^L<5Q16%ddB*<q4|wY6_6<)
zhyi`&zav*XkI)$bgblxrKVpP|6FSwabv)c-pJCj7JUC?%a{N&WWDn<@e2)-E|4jI*
zA9V2N+mkhjUHa#vewfK&jP1Xnr7tqj!QjsrQ<siM8yu@q%K(Vue8KvEhKi=sw0wtd
ze6CAxowcqohxNj|u}?9^v4j~DwL_4xZPuk4*7TAvt*~7`T;+whyeo$fcv&#aI}&Q|
z43lu;g(*IXCKQOTdBC~?q-dqi;MU9RhU^37R{oK6`9Jt?r4sm9NQprp0Kyw@QA4UV
z!jH3MnJ}BLhR%&>Nfy7qA?&q_@3zz@t+u;J%hUGD@S~PVh-HE)kQ3sW_rH@_q5M$^
zG@F2XANhd{%pYnNfgAo1?CyKGp$V}*+eZ*+1+o<H*RsJ4%QL7}FaKFh4K`h47)5(G
zsiiChRn#duj}yMm=#77n+!y#4ZK*a~a0M+=i^o_1ur32Iq4sp~Mpk%etwo%JzB&$I
z8H3?B&JVwE5W~il;d}K<KcDFcXr=Q|nh4&5y;>hgT>p&61ziZc0uU*IqbwFQP2FB4
z*Q~@@YzkGs1hzO(%?^(EBs=s=7`Xp*hGF`3pQ?ER7D4CBz>o5%zb%JABbXl^gGvbz
z2-I+{+ofyogTYM%9sy39P2m2Ggmm1VJFwOidLKoKKN&N2zjI4eSuQL7yX4nb0MjFW
z_9+vd6$OnpA_&KWtKI-D3&G82&zusa%$RL1tLAH>Oi^{Vqc6tz!n{bY1NK+Kq(C5^
z7*mYjYb0y{P^4*F`<~MQhgOgAru$2zYFEDdeJ-v3GOC0r<aRWRQu-pe9S-hH)5L$l
z6t+V$hSk{RR<TIZPkJ5jbNDNyN+#it`91L5X$pjPj+72B5uh61xh{=LOaoim38o#r
z;Q>_76GKFEIt>@z0U&pX^anKvRhK*@dP)oAuxSFpOQ+~6r7x19@Cv&N)@8{$_y{d|
zMCXN~8K=Z9#+QW*q7kb>ATk8Uk{C-2b^~B@1f-njKV>6`e|9lN*+ljWoLk7fYv4dE
zLri3vPm1p%R2l-7PfIW_7M;e})9aaX83^sW?7H{|@cLCx$K~x|5w)2iv5ZHnDKCU1
zkWnb}jUhZmwgG%QL>wgDGoH3j8n&t2v%nF(02anSYMC+25WDxy=_>^l2(mG}eKF|(
zyEiIc+w?s0yDeQ1mgYjk69PO9z&KQ`#RBZ}Z~>V^u06$}9z$*dzuMS2a2kI*#5@9P
zP=xFkfcoX2?#?2hk~wm$!{Zch><fl9{7S_|RJM~<)DfiwdVF_&Ig>PGcD~@RoP8%*
z3aG`oV=1ee^W-x!<bJI4qNIas%Rmjb*ovNI)5S$>z0;jRrMia4xZ^ZjlqWyr0MjVA
z^3nb|sr7E2?lVa|(;6@Q_x8s4lN>=X3KwTrL3{S-VM-|?(4|53j$LoJPwdCifVj1e
zPz4eI4+|rv0t#6<91wqt*&ql^FUH>u$wS*dd3uL;kow<uS6Kj4La_aW)$3l1XLcx6
zH-bP@B*9#hUM)6zZ*vM^jSUlo?D|vUe%;s?4}-$e;)W^XNiJfEir}zX-jh72I1g)5
z^V4~_8luF)H>aMd2O-{+IP#oIX~yx>en%r-UM;I4ac_8pW%LApRSP9DDtWo`E66M@
zPAlN&*+fzQncj<Lj&R!|;nGx*Oi~HdL+5#6R*BBzfbrkOA5lczB{H!~RSoB~5lJdF
z!IzQ;t=+c^Aj;;BQI#v-YJ>ql@5)Gng()?#44!s9`xDUSQimH7RWrTl=7rro^g@4-
zD=d|?=a(}qJm$CXM<Ec<qaa1dW}EqR&?y_wRg~qZ+?v3i9(?1H3hK4^QpocBfcXEV
z_=FHA15Vv9T$c^1<9$Rxqk<`Nz?PS17@)2Z$5g*ULWpz5(<TgOAkfH^O;uZWJ~4wT
zY&x*ji^Cl2fgtaJ6nL&ku{Voxgro%A^N4@u+h?$zNP#K-@N5!JB&o*hUiJ%3YgROp
z6e-xY2V204hKhC{sl$cfQoxnZ9;(|U=OfIjh~2rg)JHg^$cmHXB@CBU6~TXQa|6kD
z{`v}($Rj=P;0DMZ>JjA12kfV2o<BC)KF38;aEDQlQ1!k`;~O5WSz{$?y{}Cjd_Xc4
zd*5d~k0S(p*c#v?CinXn#e6{-bsp{mJe}@r^*AbiGdFTXq$QK~6TWuZO3BDRG5C@f
z1t&QNdnynvdFiJSYuSa5*plHU_t_1saS|eXY~%U%!QU5tO`dE?fHPh3p*Vs-snKb9
zLoBW7^d=b`Vg|R*ai03=1@`s<IE>&jm#?$#EIf)h>fYa!m5yHvT#yV4#fquHN(_%`
zuw&vysU_==8{lE#nmh{EcP2D4tg59+ZfT7x$5szBARs%-eU`I$ftXXbvba`nUmL&w
zVWg;QQyf|)>r1NhGr`Y~JBPcxKM%i>5AOEcoa^}7aZ6@NHg`$K%Ia;q=LZU2lb)l8
z{7jdX%`xg2?~7dX{6(H0T0EI7h=E<_V^h{;vBAm5muV-&#-?gz<vdZD%E!_qq-6en
zypfppbna56&I#J3^7*th5>pB8vuDprNW40HtZlfBHg1)8+_TT6$f$%t(u-Z<T6gD{
zJjU8Nr`-(BmjwqqYv(G5?V|8if|U1I@MCFxqYpml%Rg@DS^tuBqAllvNOl4m15KoG
z0)8~8hN8zQ)MyluUyx6_SQNrn3p?ysu^B^8px{R+ABr`mRqzEi@(c3G0zC$a%I9G=
zmgDcZ5;H2c+)P#br%7n#Gj<B&4M%=zoIl^Ojz57@%9kJ-{<}%|as3=TSH^ol{rEfP
zXmuhto7=?yc1)p12A=yiS4-{qJJZYx;ZArgiT`%&1nMoZ`u?b{C|&Hfp6}_C9Cptz
zB_-%MjEhe?%+zWqyyf#hm{bqtiJ{S^C}IBRWlo@Ci5B2i%?#x|Jv}{Qm+VhVhc`Di
z$voWKKi$fCo4eF2d1^mAuo#C8+XUSn?u21O`e&rJI^h-OpP%SU{__qd&|^ME*W;#x
zDH#>N{GjgnM*Mqd;Ro{8n33AYuk;ySDOmm_mHhFzAR`yr5c{XeZ<a7DpB+zpIQM5h
zc;PD)MB>LycH^Zm9{j|&K0KZuWCl@>+5a@T;0M#weG4Oj;ZM&SU^dJYrd9qldB_h_
z{8c1$o8`}-P=whHiRb@r@jiBetiRVl8rvm{!_NR#TqmoH+}+(PbPB7Cmy$y)ms2E(
zbqzz&dgHN7yzpX-T|JO5k0@_5vcRifT3NYJBrC<i@l1ad=@L<gg1)M%>f*h-&e;n3
z`e~x&!VeFKXT>P1?v*|4Iv}nRuL#@jz4{HD!?OU?)P~@3SKi#e=I{T#)$Mgc8jY9>
z89DhIpjN^x&zX>fL=yXEEV!&CYyHF~m9_U-$vV--G?!L!3F(O%Mu&@uiHQ^4A>_C*
z$Oi{PJLYZfXFV1aUe#HaAA${|m-Hw;oTrjm5JVGgJ%HKyd;#MGk%EFm7+7MBBDsu8
zh`tU8UGAHxvwTgZSf-Y^Jd)4xoN;jZj&?p|rgVQ47k;mL0>cR1{&aoxd}Ro!;!o^1
zxl$_uMwhI9Vi(>jza9|tZ0qq&yjS4j!a=ZDn*rNb1D9_+5O`J4<F=6vPWIl!qcq!B
zYp1>duk%h*Z0}ilaAe(ix<YoVP_)qY^88-q@BO?wIxIC`jqK;uL*A#am1U!NF4l>j
z;qQW&I@lH>(G1WS9BjRNf>bwcy!=dGUq5^b=MvS0$am(xe)W7G(w;q24c=Ru^l}V&
z5_S6H$B$5H&7jkF<G$bLqO2^6l@o5Uf}>QDbQr`7d=5G%VGtEK!bb8XXx||hI+nc8
zvP^qaAm!}<!`HN>UFnkX4ALkT9$}os>gQQpZuFw}n`yFd;+lB}AZ@Z-i_)7?o@nf1
zs3Xx{p$s!DVR_v3ndWCyzPrmK%8A{k%?^G1&0dvx8Rys<97giQ-Das~Y+PKb=6V^x
z9f8Afy2aVRd1XkTJ7efRbU40Bo13Nh&WVdelLeHeO0a&z@z&?N=cmWv@|Xp&D#KLs
z;XZ!<dMQKt!7}m`nV|@*(`h_0UJLYob3#D}J{=vdqRfO*P<hqD>Wp;AuS1|;;96pa
zHn-Q}v-@|GL-kP>iiR%I*jTVt+7m+OB*k4Os*HFHjl5q$i3HLrf~9fXE+7YY#!Bee
zl4+56xPvbz+s?;4`NMaFusYJ_(|GgZiBw->?h)&PNz*IrU?hQIT-ZS^uSMBBtZ>{E
z=DGCH(#1l@da*xMEfZ#G@cm-gBoYB19zM^N{YB4R?UZpb8Y2;wRL!`Zsq&j9`=Oa;
z$J%nrwtP`%eb{-$ot$4HPOL;lJI-7mGQzqJCynkCQNP?=TVyOb8Sl1km*gcRB#aiy
zoI0y1XZ^A}B&1`UH(+UF8A3kjS~Vt=8Q1_9A?-&IT|i0>SETZaExyYM{&`@EJM$$O
zX?n$sqFzYN@$D4?x|$9x{j8n`8NG}R-YYu(4|NG#VVZNEj67I&Bz$7%K#AhU^N2TH
znqd-$SMA}}ZFbNuTmAC!<y6qoytrR(=23U_5kpx|61_|UOJaV1iJ8zmgT$xkDh(-A
z9G(ZJ-Zl;{Ws&SD@3o1$#bP_VyAfFitE3pCrwK10i2iDVPMz~jnmZlUo*(*3)^yKg
zD*QayOFU@Wzn*h4`OI0D_1cT|Ryi)~N(w<>>h@MR?IMGI?4RyPrvF8;Ft@dJh}^T?
zw<_*FN9B^UTu1BP&V(!w5~uOfn|ktx4=#Jx!<wnGs!Rx#M$Oq8JMzaM=V5!7!jJup
zM9sc5p0HE>i8pJ_+cf=QMX3{=46@}(EoV!N58q+ShaJal$iib>Bj}sT7{y#_K?$9(
zbcmA#6CL&$y~x$JzV7<H5yrN|dj+dBflBDH$(uW1hZHYf#*vLoQ%B$UvdT0$I5c$h
zr9vLF_6YPo-Jq^&($XIQ3w73s9+!9wY`R`{F0&4@ltNI;G?*Zx$gt;f^*`|VvO2DI
zwwcDG6<giPsq2eK@Zq=byDR3ib8T1Wd%{lIXS#ABqO-NB+ADW^*ikodQW)37E~&)B
z5o72NusBgB2P{C^)rSL3!_kBc{5(AQLYJdzJnU*bUv~Ok{jM))B97btNs(FL^dP3$
z_nf!j$g%I7OQK7~izemJ$+#+7Z`o<QKlm~(;uK+P<E1>M9`Z2O@9npF=znw>T)u0b
zxjmUYR$Y<u8;AVrAnty*Hd!68zfSGKW~gCI!Dj{s1%-&}A#vZ_y|Pn%QvBb~nP*{t
zd-FNiajusT-zaT|%is<p1E+GVFKo{V_d#qbn4QWOuywTa#Rtx6?=}x%#nto)%V3&W
zpE$wRuA!o62suj{3?BycwobD#eqVynS##j`9fn_9(V177IXF00mX;D}CEOo{WA;bC
zD6<gw4aYA1CpcDu97p$sJ@u}+l$BV#<o-{<OwTYivdZ;xWXXMx^0~L3nHd@VaY|rM
zJ32a+Dc#@GWj`S9qsEW^VG3t);mpOOTPOBdd>oexJ{tx3!+!;5z28to&{%>S!30D0
z0dXJubG=y{*hHONEY>i|zmMh~Q&>rkLC*Al4(>4>SO))1{({l8Gv)zV10XSh#F)lu
zww?}~?AwDIf<x{j1Uv2nFYDEi$#iPMCjawx>XBgBNlWOelH=mybUQ5sRs>xADnh-V
zSRVC$8Rg~ZlWL$`JgLp8*u&gIu7ZdC4?aN<={%-7(a&Hk!QkVOLy+PBAy|#oymbbk
z+Gm_YV4L1HJ3a^R1rZp1P8@yC0;|%p!^_^&a~o{^%a<=jL_}o#HVX<1HylY<`sA*@
z?RxxJ$@vKKAU@7CS9sHp3!}Kf@>dDs^@fH<pv+rcspW&ae8wIG_?_x3`PhqFARiJu
zQK<*gyR!lNu1iJL#<jB*7j8i^MThgtJDEe@>AQPkNjU(Ev2XMze={$`M3Vu`H4F^L
zCJbvwvnHm{G1sw6nw$-TNemAUSB<BjE-5cBx3{;?%*@<)```y)ArXgRlLt5UX7643
zG*w$|wXPC*0(Ap>3%~Ttu1W?r95a5Ok$CyV7dNSrmWVPj0c$zfMN?B#LvN(7Z&~Z<
zrOw%B-EM$hl6<eT>^u|v%_cKi!4;daxE<CI`7OrB%8b%sCr=R|WDXd$EDIYrvAphb
z-T!d6wkveb;Eb?cOr$9^SFV+fRMmFq-<Z9ZFXQJ9j#?_qxy3N4yQU9rT=N!?hrJlq
z59^Dit&|-_gWW&G34~iXRv3wG4Ju#aYSxt}USEJ0<#vo#3(#l&e76fiJdUFUGk{$3
z#ocT-qCdcq_zFP#!=hlU&u7CxF1ZG}#TwZ<=)K!qd!rXl_dN=T!bcE?z|kH~&^t`C
zq$Yx6tIKSTyQKIWpkbYPo6cKiNcH$6<GX$x3cwFlTYK5R%G!5EWB5VW-t1NQp<O9b
z*mRI&bR%~|e)hf1H=Vy5R@cz@IN#lk&t$m6!4)s0U}fj)UOEqh3m|<g@TP?o;Q@f-
z^3JJR^XIuD=Za2WtGi;R#wY|Gj`h4liJa$W!fsg{3K*KF74lwsXtBS%vJySWs7$Zr
zABh54LQ)qis}{O{HXXDF`L`o>`s=&+kW+hTQpNv*dJ#Hv2iv@i2IYbat4>Nl<t7|E
zDol*kulqk#fO4S^yyC@?+XV#$cMU4kJY|i_hyk`~XlOK9>;wCD<;+(|?;XI+h^G{g
zgJTjj4$K!&{dpP7R>6$Lg(oWNeGSxlzy$PsMR{d6!x<}fWh#Squt{$J<I9SGy*0L5
z7oDeSX1->Avu_Kn(YKP4$yok0akts}5z1d#J$UGoqF3hlrkpcjSR{0iJtKb<vAqG9
zi#OiE@v>NG-#kOkV~o!rcld4lg9DEII=Rzy!#mI_5l=gKdF_BQ$|(aNh;##r4cm#e
zhtDCAeTb$iP-!GkWwxnaBb|L`rKUUu9KNlhP*yhByjc>C_8a-n<$j#s6dp3mVL6wS
z5xuzRTVY}G<!y>3Ln>rZfqWM0v~nGz_|><806@Hmx92*%g=G}0^<qK~Pz9Z9&-MDT
z5zXq)A74+sIU98F1>PI;jVcp((HaV8e9{93`(lEjD2P0e`Ieo&nWq*OEgejWy&p&7
zSkcrlUv@^&o4Q>+Ys}88PN;vMn@fF%y^~1SG}L^U`#C`)6S#rnIQ;?lL>xzJ^va^>
z#Qk>`(>X4v`N`g6AxbeKd@|tz3%Bv2Trkxg8u+3GIMs0%blbQ3lIR0|gE+gJ@x@DI
z%%P#1sGY}6_=;~!GmpZP*m!uV-Cnmgt;KaUFlRR{{s@4VqH0$7wj#5Xs#~@+F)@+T
zxuAfy0E^-iglkJ!e0Ba|Bg1eiZ0wTIt~6$bAua)dZ2SEzUQ-embgkLFG_>CSpr4P@
zWQnW;Mu#8W@^n*^@MQskv}irs>;nYMg(Srg!3p#2R;Y!T=$TLv`F6?-hi8ya?GIk;
z7%o{K3**(JNNEdxhC0(`wSCPqAwalBSE)&%*1)r_l0T;s14}kgfbo(+Mk3&y{&QsA
z0<~YX-dc)yk@=t5hIy!NiN3Xu4SpiW4%2e6n|7}75~x^GEio}M6?t@HLiu4c-5)0N
zM2LZToSf%bWE2#euiKu7Ge4&v{3@GJkv^cuD=F@d$l7;)+G|B<W%GkFtCGyP+IZxf
z4T9Qe0JXV@nnuPUsO=+IQBa{W60xzd`TP5)vum4m2S2dTorBE;4&ZF5DxQXcU42&-
z^Q|{ehHx-U(Rx`WnSL0c=aY|b4K)7@g!373SCedh%f*O*7au|~PCgd6`9a07B{uW{
zs_V9GJx49F29od+804+qMyAjfrsc^3T^aV%-p+0~PpsN^V>YalGxm~#oFPj`dl!@Y
z_t~OxgjUlS`Ec$I<}1wh@YMq{BIqn5QK150W;hJMh*Pu^H1j@|@O<A3Seho8TkRG9
zJr>fGwhM~^QApHi>87jQ#=!o@>kEa_Mid<NRu692@#*1VT*FMW8%JL&+jY0=LpgAN
zsAed+_?%#GX);;&r}q;zWmDXM*R5`F%Is*y1uoXY*IM!C`>@?&pVg_Y(`N_rt3EVg
zxe%^t_4fX?E0=?IB$e+;JeG`j=OGVMd|%|p+7W4SaOCE2!R~`9f%@Rc(dYVIXDhE(
z21hoIg#G5qrWd6&WiZ@@u1MIU6Oh+Jm=Z}I;7w`-YL97?S!9;}oBRdq70SkJl(^jB
zW;wACMJHNPVm-~UgC3Z&8O8r$%KinC|GP|?8D2(qHd4(7PVgHLX}j<9v2@P_`wRw}
zpGVis?7r91;QROQ3uXN7R8%-RIQVYPNy*6kXgZ~j0j3FCKp~v!?YZU$h%_F894v}{
z8?#2uV>px;=u%=ypeZWB2ouoX*k7L+d}ahz@4_uPD6-`B@{Tr7_AV_n5u*t_5jTH1
z&+|f=uW<NYlvW8>K7k>F=H}+c#l;05%T6>4{>jWVGB$1v__~Ql$v>PUbQ!3iE3fKc
zGuF{R7{@|kf71i5=B)6iQ-o(S@`uh)<39oP(}02pi8MU{Lcyroc;M(TJ}z#NznS$W
z*r*g`=<8E?Y#W>z;4HI&z3jqI;qMHATm?i9B<oi>6iCd062j|0yTqg%)&K6E%cR+T
zA7?PYBX<D*arlwYh&nN{0U^{p7bIT2eG=ZLz2*eYd;3_?$}M04M{db?1Bir!a)Qx9
zN##VCBpMFJF2B!K&Pc&AMB`P?#+CZj@b=blt_VP`rgeLSqm9vpS7C|NZ<__I3L3y%
zm&q!y%;mr{tzl`}VKZ7s_e7nhO>mIDwno-+R^&<s?TFiK3yxXz<Ll_f7wgcN<MX1Q
zAc8)Zwzs&rc(C_r9<r_;1bqDk3E`}tah|@Ee8yZH1Dqi<De+~hmC-_^Vih|e8@>w2
z@=#5CN=5q~XvHxVMmDzckfUEDu9b^dfr9MQy`DM~%LOH^FCf86K&giAfzON1g5Yij
z&W-ON!XKPp+o23tdT^+wa1MMxddH0)Igpyfo?HozE27Go&)&fUZk}f=J^tt>%{*{&
zSpt{%19o?JmF-hXlFAR8P!Q@>1!$7QAkQymU|@g=I@7CJ@)P!Ani_OV<X-+E>Qg0w
zP%TH`H5oZ_grk4cFmL0?d7=mzz=ZC>Ej!g}i8Sx3ng}t_vV4qpV=a1-yPNtFX*uE-
zU`WmvN>9FI@pbe_`%XRc4yhmi2(GMnsQ(SX`#`2D&L;@Pu`2pF-dLS9hPN=6BAuQd
zNat8uZsN`QfyD8Fb_+ZEk{pH2$;$ZriI~fT`omkSSIenvU|oLNXqyLU->Rf{gPClg
zYe~2Se+eSwL2%0QT~6B@SkhMXA3e9ZX5!lz8=3?stn}!tpU}`b1K;&&CLW7J;ASVU
z;rAv&r6@f0DX;bE#=#M)uM`k6mZBJ}8zwpW3`?qiBglCL&^Z2VUTTI`6F;SozY!ZJ
z@y&ma+xh5&uiZzMn{1Yl0k<RC(xveACqXciho1FN7GKF!&y^RqH43G#dF&*F9^x_9
zzZ}eq*^#d8;$S)w?*BpB^ggrZ_Jix&w*H+{A}rT}7*ErR<L}+$?g*4|e!;rC$M{N_
z?F9NgrFvSo+=r_-DN4!JH8s<eZJ%V<v-g{GMHA4|fTIN3O3RKb9i8J%_F7!2Ayr5X
z`@zoApY#_Z$M;o#4x(E7D=I2J1IG7iUj7WI_v@!u&*u`}Me`B&jn!upc#e9^KMdO8
z^1aMO=dbP#7_hNJiqgh)d-__2<p&70*Du-^zkhH0{j%$ar$;)s<lL~h2#<OLF~{q$
zN36evDY&960^>$TM#``V=s0Iec*FMd=MVcvBv--kOpvB@pSFvY^AGtBR8^s@f0Kpg
zqKyizqsFhQ?0OOiMDoB$rT^&9G|!UABV>@{Wst40=)a2oLcV}oD&tMLJ0IiazW^92
zy?80#3^?*cwi~8I+aLu?eLOd3(UHz6kA+XQ1-C*@=<}tG<gg^%_FRi3ZczONg0;+3
zCE@$bw`O9z{bkLW@oDlya6br%?vOY?a$@DoW98|qCh>Y{@T(Tvj3cmQ{JN{3)Z@1G
z&9{=z*e<bA$@tdbE6%27HZFx|@7QXB0b05>*V57gJ}bCfgH{lqlNtKr9uNMcn~hHi
zvHvvZ`T8c4>_ZN6chJ)cPEIcwg*^=VapRhk97DF9E7S<dJmk9m+O3Vxw23uC4SdT4
zwBDw8WR<di3u>pPB9-33&aS^yeB*Unax3N<OK~cg4z=_>%Sw1WY=31;zAN*Az$iCe
zbQgRBjOC8<uluKsa=pNW=qT`840!i>S&khpSdB6DlbllPu$aL$3wun3UMy$$00)E`
z<_G~_efqO8%xK2qBxqXNn^=!^N2fH9i^<SvmyO(CnQb)NQ0bbY6bieYhmm0|1-;!H
zONy8v$lIi(O6a=P;sEIz*(=0}9QvqFx+D>y-_yLtWJGojlTDeT`lQepljv`E+FeAH
zWB~vvNMh`M4+DBZLzCBN$<oZUappS8=uB+$y1mF54AYc3N!cx&TS$chxTr0KG9m$Y
zxpZmU=UF2>9dJ{f`~w6xjpO)yerxGPEOqDhZ589LG_l&L`bz?oWcIb0P{zxsh}-nO
z&)xNr!NIiWo2suw?>)iokH$&(+5PRQz+(PT)BBu2-wO{P^IpLj2s$b)W!_D%KuRIn
z8*MZsHj-jYVqTA*DWs0u)xNK*nBSITdP;~a^<ashkXD?);wQEnU5&E#Q)j2Z7f(!@
z7@Z<_D4s$kG{O#QdY`MTH$By;G$`vPVjV4-_TRznzsz_HSZe$ugB4fI+T7h1H}y+^
z#BE~Ku^t)9>Lq<=Pg{XPr2?HT<i+EHKz&VYQKvD5x{Cv<HNfm^DFESEB<cAcJnaty
z1bl8y&e;7NF~1;oz7M;1=WBJ<#d>}LOwI2x#4O<I+Q@w|tu+&yBN{UU?`re>t%DGj
zDuxmnk(I#uwIM*jyUk=*3B_p!XED7#6}Pk)T4|(Weg|fi@~Q`AavKsgZmcFY@4E*4
zFf6&DA%!Lr6&;-<e@D~s3?pv)?<Erxh8;gHNhK%92AeY{bji3d6Cp4nBcXZeA3!`h
zni5c3jR4}!5v$v0s}Vq)YIe!&!cWh;4DZTIUHy&0V=O8(!;1O?iF?@r%jDCZL)?eo
zAi0PfVxa32!f+7&LuxXLLx_5MP7)W+=-;1g!D(OWSdkTW{GCRC(S6H(djFto;P)JX
z<TxvH4<G-}cVe`mqv8@tNC@b6E3(Lvnz?Il{pUMiwpe2tJ3Pt$#p;X;2$}j^hwmid
zqJO?)OUkI2Fp^mIZ^uqxJjZ1X@U@f1AO6PKNsvl%xfH_3Bz&7{A;V3yYx8@tUFJJq
zs#C!rh~@DrLD27?W=NQN$a_^dqu0t{{Y~({8v2g+`VH5@oX0D9p$8&Ev7Q9$nBNsh
zHHjLqJuv@Y2WH_V{(CT=16gkW1gU+9LDTTRCMj#R4toJM4g&SUEhC0TR#7C`e*wIi
zRsWTNgd-hWg)PMp;`NglhOMfMTwEHndeoO-Kj&g&(&{0BSMma5Ep<<tL3w&0KrCCF
zQpa7*%*>39jZIBwfEr5-F=Cam==601IL7x=?IE>eWU61rLf(IjBR^-n&vv03F?0$4
z2>{c}UT>(ZRNoZV(w{C-KIQBK!BEI}@JOijrh>($TaJ5?=#3QK#1XCD{~{EuyUC?x
z)eiio**zDx`G-FN;nvcP{QCK0cV{PwPRa{OorM(k$?A*sT@pL<+bUH40<raXf3;5}
z3vM^7CD$Q~h}>5iS$?);(_Q|}cF|s2KtS@&l#pr0c2zLTj}}NZJHKTA{wM_LJU;9L
zp<+)&t+9~z8}UAr0mLhc{TuNDKS9;qbNGGt)`M@iw#Qb0^ioSc69h`0q4-5$8)LeI
z%oKXAVZ62V4Z_7nmVAuTzi`$|0l;?nnO65V(q#pdxe*7``rjZ2JDFNUEssXZ>$b(q
zjThxZa6%?zfmh<FMJXB{+6kT6Ja{PsogpghxxoL6axF*7d-=cciufqRc>s}>|F<oh
zT@zF1{=*SM1Tx4@`EAC~K_|Z5>VZSrE-@dGF(SuHevy=vltG1#DhCD%360qSS{5&m
z^enx-{4AN_>UgOt>YyACT|fb_<K^QPM6w~t$tWNu;WDv0ao1qSIBr}U=!UnTc?7RS
zW|~L)=#q6`0yT1V#Nv?^sMC)GJ{++uU_zkih7ASvN${1773wGlX-OaeOaRe8D!IUR
zi@}{*4dzo;;G#jeIDK6(b@@vs6?}!gZzk2M)pf@9`(W0LHV`&Il3jM+d2$t1K>I6q
z8?ZD4c!%<0n)XMKdzp|FQb`Ymw*@JgEtCRQ7Tv)?N4o98FMfRPMuO&bKH3%7lH5H?
zGf6v9u5(%Cy1@L~>&+Dhoxdgb;8J(6qMxdXlU5}0zUn;$A9}kRvKwu+@io)=SnTlE
z2Ht#bg6jxjds6oE*k&6|H%C>k&(zV^TEA+?#luT%%RywVa*(_s-sgHTQann<;1}5f
zMYH@$p&m<Us08NX)d=v7rXWZVCGih2c*DD3Frf;fvxB`#jM&p$z=O(&k<Lv`H(p*E
z_)=Urh)bzB#PuZ>i7RuMS(#*w4_ssapl`6C-rUp#{u$4_lU&e2OJF=i5`Dw2aW-lx
z2zzzrmC~a#NFcW0B3+7i2bjQddIh(6Y!CPA*Q%B>Gk{31>gqZKv9NiQokpcitIMP&
ziONjC>R5^HHO@BGo_3QfS_ZJ_^bs2H-=ID=2jj)l5cS{^8z$VutaS_C;FFM$aKgkz
zg0)+)I^P9NBazWcwv@s`K8!**uR+a6NR!*u*$UujhRsjD9*PzhU*km7Gmn8jiu!6J
z0L!^JEA;zeXqWnx;G<XgdPOqpFDsI_eY_ME!%RmB;h<pThGVa?y%Hgh+If+9%lD!p
zv>qs6va=<eU)<jN^lAw6DL@3I^wUmZ8^k3Bt^AGbfz#uEruh{fP<uk)u9M6)bkYe_
zFgX>dS-_MbSBk;;4rN7TQ<zv-E>G2%2D%C8MuFr2{9m3JVkN!Puw8Uj!9ZYv>;@e9
zq2b|{Ue=)<ZP*<Ued#NU%X;uzvP6u##g>U^#~gUA*?Eko=jwt+?lP1FT{fy_`%Sgg
zQlF+sOpIzG+Xu%@Q%eix0K_n_sxJzdMC)bGP}xib$@qsNbeR&zN^~nZOr4LT@Wc^^
z!vD<<>dx!_#J-$=^0&~Xalkvn7v^Z{-^lK7RN!F@kyF!4X2-gx7C5c{#D4$({uk;~
z{I&v^u7MeEv{-C;eQ`Y8r`hg$$qYA{H%A<Uh&Q=4;o&Gh>7+Za+~3?+H7LBl4{Lk+
zgE|j3nsg!L(j!AoWyYA_YN-_4xZfa!j#RMD`tITL`U7HVaVb$)OY=A$n5nyepa_Tq
zNMM<1NlyQRwUEyl6&LCL{?@J%iryw65I@E|AX$?-_4hY^gu#i7IQ<W_K|Uu@8HHB<
zZ6Qnx{$AjnHNw*T^HC@|)%*X$$A{Tr|9+4t$#k>h1y4`UNQ{-0m5TCmRg+v7INn0J
zAkeG7h4g1k0f-YhDp4oh$S{7Bh6V8SV;h7gk*^y7Ol8MJLV5r8`yr`%Rb{0D3i*{W
z6$SC87nZI^hd|0-g*{<Y`oQDe<NDyeYCiMU7N>Dr(4bvgo4_%Uny9<Zj6cb~1lutc
zl#_pOVAbtQhXVM$5Dnr)^h;|%$Jh=LJP{q`hrmxP0LCEUaeN@9$Vfak59UAu1-e{0
z>a`lv<{Nu!<hwbA@q!?ugD?dNSC2?c?8F@b=vR+o7?=YjCg8P#&=5hgGbIsBFGNw;
zPDuP)n>|jDSfNK5rDJjH2T(gZJ3GL9j04Ml?bDP|EzPU>BOR615*#~D?CX~p>p~Q}
zo<G4xRD;@RIM-NaO3*U4y)lx_%Gzq@a{wC@LpSMr%bk%CsSMqCwX!(r;P2Q078a_)
ze1q)H2wxhbk_}?DNtsM|?lnX)A%}@ZaDFe3pj|)F06}>sgiiZjQ#?Z)yPAl8Ddc~$
zTJ^mB<PAEtF3=P&y~#l0x>i=L_LBQ*PFm`S05p~iQDENt2!s|<!=(jCBQ0xO_O-Do
z`C0pt8O~XKAo7BwHt=aM&=%wk+Xr(H*sTM%a(Cr9a_efR-)C{Nq!15rHL4K77I<VV
zCA9Z`<RdZCp0@SDmyc=mn%<xYy2m5a!Y4@sPJ<gb6!!|HQHXV^GRzgWk9#y$e@R{z
zSs|UU#5dAz(<1{hjskq)@9Ek2B@d2+PnS9F2ONO^@EHK>4L^5kYKXX5E<m1xNYpe)
zW$KeNx0yl}l%R3aL2XB5c_yN~Fvp#0A?PJ8u4Mq9oWNmTfl*;BuI_<+TbszD^z^b%
zR58G%WRO|s(y~?DEQYCniE8C#X1S%Kp{<=sPIP6WlBqjAJ>5;TnN5lILkSm}NWB53
zixgkS4|&L0BoHk=_00WkSI2{ylC#_6xG3yZUWM?=-F0HXTHj1Ue+<bL1x6)>egHe`
zTaZJ5yl8yrfrJN0bCd=_m^sY@YbxNZGjd$~-X>!pJ=FWbWp@JUG;}x=LPM-E`!`<D
zAiw;61w=2oZ_p2V$$$4%0KECe-fgjen@~^)0Q0d?k%IAW6DLsnFmkh>G_D*sx(|)c
zUSK-;rwQ~rIE*c?(x;9a-9YB*mJ-%Ky?_T#gRkfGZ;0x5JF{>MyU5?f|FgS^8ZN@&
zVRgam*wt&HRG$h)|Net91H@l=-V+PUsY?J}`tx%k7C6i&(Lj_N;&BNUfi=dY(1(R7
zM~qmBtpQ!?V=-;00xQT?5>#TPkKZ5})=QA+4F0i>B?fG)5DDAb@k`6WN%&>lPQ>uf
zxG*ACT0NsO;`hM9k8kSG;dALDx_@N7Vd~H-i(V<gpTS;WfuUs37FRy@JDI^${~y$m
bup<;phmJF=)LWku@Q;S7j!K=9b@=}Ulqxs$


From 5843e6872f495fb0ad2f16e3805794eaf90a461b Mon Sep 17 00:00:00 2001
From: Dan Robinson <danrobinson010@gmail.com>
Date: Sun, 17 Apr 2016 15:25:39 +0200
Subject: [PATCH 0061/1644] ARROW-103: Add files to gitignore

Patches [ARROW-103](https://issues.apache.org/jira/browse/ARROW-103), though perhaps it would make sense to leave that issue open to cover any future .gitignore-related pull requests.

Author: Dan Robinson <danrobinson010@gmail.com>

Closes #62 from danrobinson/ARROW-103 and squashes the following commits:

7c1c7d8 [Dan Robinson] ARROW-103: Added '*-build' to cpp/.gitignore
633bacf [Dan Robinson] ARROW-103: Added '.cache' to python/.gitignore
59f58ba [Dan Robinson] ARROW-103: Add '*.dylib to python/.gitignore'
52572ab [Dan Robinson] ARROW-103: Add 'dev-build/' to cpp/.gitignore
---
 cpp/.gitignore    | 1 +
 python/.gitignore | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/cpp/.gitignore b/cpp/.gitignore
index ab30247d49378..4910544ec87cd 100644
--- a/cpp/.gitignore
+++ b/cpp/.gitignore
@@ -5,6 +5,7 @@ CTestTestfile.cmake
 Makefile
 cmake_install.cmake
 build/
+*-build/
 Testing/
 
 #########################################
diff --git a/python/.gitignore b/python/.gitignore
index 3cb591ea766d5..7e2e360557ad8 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -18,6 +18,7 @@ Testing/
 *.o
 *.py[ocd]
 *.so
+*.dylib
 .build_cache_dir
 MANIFEST
 
@@ -35,6 +36,8 @@ dist
 # coverage
 .coverage
 coverage.xml
+# cache
+.cache
 
 # benchmark working dir
 .asv

From 0b472d860260f7063aee742939be23b921382741 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Mon, 18 Apr 2016 19:44:29 +0200
Subject: [PATCH 0062/1644] ARROW-82: Initial IPC support for ListArray

This is a work in progress because I can't get clang-tidy to shut-up on parameterized test files (see last commit which I need to revert before merge).   I'd like to make sure this is a clean build and make sure people are ok with these change.  This PR also has a lot of collateral damage for small/large things I cleaned up my way to make this work.  I tried to split the commits up logically but if people would prefer separate pull requests I can try to do that as well.

Open questions:
1.  For supporting strings, binary, etc.  I was thinking of changing thei4 type definitions to inherit from ListType, and to hard-code the child type.  This would allow for simpler IPC code (all of the instantiation of types would happen in construct.h/.cc?) vs handling each of there types separately for IPC.
2.  There are some TODOs I left sprinkled in the code and would like peoples thoughts on if they are urgent/worthwhile for following up on.

Open issues:
1.  Supporting the rest of the List backed logical types
2.  More unit tests for added functionality.

As part of this commit I also refactored the Builder interfaces a little bit for the following reasons:
1.  It seems that if ArrayBuilder owns the bitmap it should be responsible for having methods to manipulate it.
2.  This allows ListBuilder to use the parent class + a BufferBuilder instead of inheriting Int32Builder, which means it doesn't need to do strange length/capacity hacks.

Other misc things here:
1.  Native popcount in test-util.h
2.  Ability to build a new list on top of an existing by incrementally add offsets/sizes
3.  Added missing types primitive types in construct.h for primitive.

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #59 from emkornfield/emk_list_ipc_PR and squashes the following commits:

0c5162d [Micah Kornfield] another format fix
0af558b [Micah Kornfield] remove a now unnecessary NOLINT, but mostly to trigger another travis-ci job that failed due to apt get issue
7789205 [Micah Kornfield] make clang-format-3.7 happy
6e57728 [Micah Kornfield] make format fixes
5e15815 [Micah Kornfield] fix make lint
8982723 [Micah Kornfield] remaining style cleanup
be04b3e [Micah Kornfield] add unit tests for zero length row batches and non-null batches.  fix bugs
10e6651 [Micah Kornfield] add in maximum recursion depth, surfaced possible recursion issue with flatbuffers
3b219a1 [Micah Kornfield] Make append is_null parameter is_valid for api consistency
2e6c477 [Micah Kornfield] add missing RETURN_NOT_OK
e71810b [Micah Kornfield] make Resize and Init virtual on builder
8ab5315 [Micah Kornfield] make clang tidy ignore a little bit less hacky
53d37bc [Micah Kornfield] filter out ipc-adapter-test from tidy
8e464b5 [Micah Kornfield] Fixes per tidy and lint
aa0602c [Micah Kornfield] add potentially useful pool factories to test utils
39c57ed [Micah Kornfield] add potentially useful methods for generative arrays to ipc test-common
a2e1e52 [Micah Kornfield] native popcount
61b0481 [Micah Kornfield] small fixes to naming/style for c++ and potential bugs
5f87aef [Micah Kornfield] Refactor ipc-adapter-test to make it paramaterizable.  add unit test for lists.  make unit test pass and and construction method for list arrays
45e41c0 [Micah Kornfield] Make BufferBuilder more useable for appending primitives
1374485 [Micah Kornfield] augment python unittest to have null element in list
20f984b [Micah Kornfield] refactor primitive builders to use parent builders bitmap
3895d34 [Micah Kornfield] Refactor list builder to use ArrayBuilders bitmap methods and a separate buffer builder
01c50be [Micah Kornfield] Add utility methods for managing null bitmap directly to ArrayBuilder
cc7f851 [Micah Kornfield] add Validate method to array and implementation for ListArray
---
 cpp/CMakeLists.txt                     |   2 +-
 cpp/README.md                          |   9 +-
 cpp/src/.clang-tidy-ignore             |   1 +
 cpp/src/arrow/array.cc                 |   5 +
 cpp/src/arrow/array.h                  |   6 +-
 cpp/src/arrow/builder.cc               |  56 +++++++
 cpp/src/arrow/builder.h                |  46 ++++--
 cpp/src/arrow/ipc/adapter.cc           | 136 +++++++++++-----
 cpp/src/arrow/ipc/adapter.h            |  11 +-
 cpp/src/arrow/ipc/ipc-adapter-test.cc  | 216 +++++++++++++++++++++----
 cpp/src/arrow/ipc/memory.cc            |   1 +
 cpp/src/arrow/ipc/metadata-internal.cc |   3 +-
 cpp/src/arrow/ipc/metadata-internal.h  |   3 +-
 cpp/src/arrow/ipc/metadata.cc          |   3 +-
 cpp/src/arrow/ipc/test-common.h        |  67 ++++++++
 cpp/src/arrow/parquet/schema.cc        |   2 +-
 cpp/src/arrow/schema.cc                |   2 +-
 cpp/src/arrow/test-util.h              |  49 +++++-
 cpp/src/arrow/type.h                   |   2 +-
 cpp/src/arrow/types/construct.cc       |  43 ++++-
 cpp/src/arrow/types/construct.h        |   9 ++
 cpp/src/arrow/types/list-test.cc       |  80 ++++++---
 cpp/src/arrow/types/list.cc            |  60 ++++++-
 cpp/src/arrow/types/list.h             | 112 +++++++------
 cpp/src/arrow/types/primitive-test.cc  |   6 +-
 cpp/src/arrow/types/primitive.cc       |  45 +-----
 cpp/src/arrow/types/primitive.h        |  41 +++--
 cpp/src/arrow/types/string.h           |   5 +-
 cpp/src/arrow/util/buffer.h            |  59 +++++--
 cpp/src/arrow/util/logging.h           |   2 +-
 cpp/src/arrow/util/memory-pool.cc      |   2 +-
 python/pyarrow/tests/test_array.py     |   5 +-
 32 files changed, 839 insertions(+), 250 deletions(-)
 create mode 100644 cpp/src/.clang-tidy-ignore

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f803c0fb3e428..b38f91e5d687c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -565,7 +565,7 @@ if (${CLANG_TIDY_FOUND})
   `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc | sed -e '/_generated/g'`)
   # runs clang-tidy and exits with a non-zero exit code if any errors are found.
   add_custom_target(check-clang-tidy ${BUILD_SUPPORT_DIR}/run-clang-tidy.sh ${CLANG_TIDY_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 
-  0 `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc | sed -e '/_generated/g'`)
+  0 `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc |grep -v -F -f ${CMAKE_CURRENT_SOURCE_DIR}/src/.clang-tidy-ignore | sed -e '/_generated/g'`)
 
 endif()
 
diff --git a/cpp/README.md b/cpp/README.md
index 3f5da21b7d417..c8cd86fedc6fe 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -76,4 +76,11 @@ build failures by running the following checks before submitting your pull reque
 
 Note that the clang-tidy target may take a while to run.  You might consider
 running clang-tidy separately on the files you have added/changed before
-invoking the make target to reduce iteration time.
+invoking the make target to reduce iteration time.  Also, it might generate warnings
+that aren't valid.  To avoid these you can use add a line comment `// NOLINT`. If  
+NOLINT doesn't suppress the warnings, you add the file in question to 
+the .clang-tidy-ignore file.  This will allow `make check-clang-tidy` to pass in 
+travis-CI (but still surface the potential warnings in `make clang-tidy`).   Ideally,
+both of these options would be used rarely.  Current known uses-cases whent hey are required:
+
+*  Parameterized tests in google test.
diff --git a/cpp/src/.clang-tidy-ignore b/cpp/src/.clang-tidy-ignore
new file mode 100644
index 0000000000000..a128c38889672
--- /dev/null
+++ b/cpp/src/.clang-tidy-ignore
@@ -0,0 +1 @@
+ipc-adapter-test.cc
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index a1536861a20be..c6b9b1599cdd2 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -20,6 +20,7 @@
 #include <cstdint>
 
 #include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
 
 namespace arrow {
 
@@ -47,6 +48,10 @@ bool Array::EqualsExact(const Array& other) const {
   return true;
 }
 
+Status Array::Validate() const {
+  return Status::OK();
+}
+
 bool NullArray::Equals(const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) { return true; }
   if (Type::NA != arr->type_enum()) { return false; }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index c6735f87d8f42..f98c4c28310f8 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -28,6 +28,7 @@
 namespace arrow {
 
 class Buffer;
+class Status;
 
 // Immutable data array with some logical type and some length. Any memory is
 // owned by the respective Buffer instance (or its parents).
@@ -39,7 +40,7 @@ class Array {
   Array(const std::shared_ptr<DataType>& type, int32_t length, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
-  virtual ~Array() {}
+  virtual ~Array() = default;
 
   // Determine if a slot is null. For inner loops. Does *not* boundscheck
   bool IsNull(int i) const {
@@ -58,6 +59,9 @@ class Array {
 
   bool EqualsExact(const Array& arr) const;
   virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
+  // Determines if the array is internally consistent.  Defaults to always
+  // returning Status::OK.  This can be an expensive check.
+  virtual Status Validate() const;
 
  protected:
   std::shared_ptr<DataType> type_;
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 1447078f76028..87c1219025d37 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -25,6 +25,25 @@
 
 namespace arrow {
 
+Status ArrayBuilder::AppendToBitmap(bool is_valid) {
+  if (length_ == capacity_) {
+    // If the capacity was not already a multiple of 2, do so here
+    // TODO(emkornfield) doubling isn't great default allocation practice
+    // see https://github.com/facebook/folly/blob/master/folly/docs/FBVector.md
+    // fo discussion
+    RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
+  }
+  UnsafeAppendToBitmap(is_valid);
+  return Status::OK();
+}
+
+Status ArrayBuilder::AppendToBitmap(const uint8_t* valid_bytes, int32_t length) {
+  RETURN_NOT_OK(Reserve(length));
+
+  UnsafeAppendToBitmap(valid_bytes, length);
+  return Status::OK();
+}
+
 Status ArrayBuilder::Init(int32_t capacity) {
   capacity_ = capacity;
   int32_t to_alloc = util::ceil_byte(capacity) / 8;
@@ -36,6 +55,7 @@ Status ArrayBuilder::Init(int32_t capacity) {
 }
 
 Status ArrayBuilder::Resize(int32_t new_bits) {
+  if (!null_bitmap_) { return Init(new_bits); }
   int32_t new_bytes = util::ceil_byte(new_bits) / 8;
   int32_t old_bytes = null_bitmap_->size();
   RETURN_NOT_OK(null_bitmap_->Resize(new_bytes));
@@ -56,10 +76,46 @@ Status ArrayBuilder::Advance(int32_t elements) {
 
 Status ArrayBuilder::Reserve(int32_t elements) {
   if (length_ + elements > capacity_) {
+    // TODO(emkornfield) power of 2 growth is potentially suboptimal
     int32_t new_capacity = util::next_power2(length_ + elements);
     return Resize(new_capacity);
   }
   return Status::OK();
 }
 
+Status ArrayBuilder::SetNotNull(int32_t length) {
+  RETURN_NOT_OK(Reserve(length));
+  UnsafeSetNotNull(length);
+  return Status::OK();
+}
+
+void ArrayBuilder::UnsafeAppendToBitmap(bool is_valid) {
+  if (is_valid) {
+    util::set_bit(null_bitmap_data_, length_);
+  } else {
+    ++null_count_;
+  }
+  ++length_;
+}
+
+void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int32_t length) {
+  if (valid_bytes == nullptr) {
+    UnsafeSetNotNull(length);
+    return;
+  }
+  for (int32_t i = 0; i < length; ++i) {
+    // TODO(emkornfield) Optimize for large values of length?
+    UnsafeAppendToBitmap(valid_bytes[i] > 0);
+  }
+}
+
+void ArrayBuilder::UnsafeSetNotNull(int32_t length) {
+  const int32_t new_length = length + length_;
+  // TODO(emkornfield) Optimize for large values of length?
+  for (int32_t i = length_; i < new_length; ++i) {
+    util::set_bit(null_bitmap_data_, i);
+  }
+  length_ = new_length;
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 21a6341ef5086..7d3f4398d73e3 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -34,7 +34,10 @@ class PoolBuffer;
 
 static constexpr int32_t MIN_BUILDER_CAPACITY = 1 << 5;
 
-// Base class for all data array builders
+// Base class for all data array builders.
+// This class provides a facilities for incrementally building the null bitmap
+// (see Append methods) and as a side effect the current number of slots and
+// the null count.
 class ArrayBuilder {
  public:
   explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type)
@@ -46,7 +49,7 @@ class ArrayBuilder {
         length_(0),
         capacity_(0) {}
 
-  virtual ~ArrayBuilder() {}
+  virtual ~ArrayBuilder() = default;
 
   // For nested types. Since the objects are owned by this class instance, we
   // skip shared pointers and just return a raw pointer
@@ -58,14 +61,27 @@ class ArrayBuilder {
   int32_t null_count() const { return null_count_; }
   int32_t capacity() const { return capacity_; }
 
-  // Allocates requires memory at this level, but children need to be
-  // initialized independently
-  Status Init(int32_t capacity);
+  // Append to null bitmap
+  Status AppendToBitmap(bool is_valid);
+  // Vector append. Treat each zero byte as a null.   If valid_bytes is null
+  // assume all of length bits are valid.
+  Status AppendToBitmap(const uint8_t* valid_bytes, int32_t length);
+  // Set the next length bits to not null (i.e. valid).
+  Status SetNotNull(int32_t length);
 
-  // Resizes the null_bitmap array
-  Status Resize(int32_t new_bits);
+  // Allocates initial capacity requirements for the builder.  In most
+  // cases subclasses should override and call there parent classes
+  // method as well.
+  virtual Status Init(int32_t capacity);
 
-  Status Reserve(int32_t extra_bits);
+  // Resizes the null_bitmap array.  In most
+  // cases subclasses should override and call there parent classes
+  // method as well.
+  virtual Status Resize(int32_t new_bits);
+
+  // Ensures there is enough space for adding the number of elements by checking
+  // capacity and calling Resize if necessary.
+  Status Reserve(int32_t elements);
 
   // For cases where raw data was memcpy'd into the internal buffers, allows us
   // to advance the length of the builder. It is your responsibility to use
@@ -75,7 +91,7 @@ class ArrayBuilder {
   const std::shared_ptr<PoolBuffer>& null_bitmap() const { return null_bitmap_; }
 
   // Creates new array object to hold the contents of the builder and transfers
-  // ownership of the data
+  // ownership of the data.  This resets all variables on the builder.
   virtual std::shared_ptr<Array> Finish() = 0;
 
   const std::shared_ptr<DataType>& type() const { return type_; }
@@ -97,6 +113,18 @@ class ArrayBuilder {
   // Child value array builders. These are owned by this class
   std::vector<std::unique_ptr<ArrayBuilder>> children_;
 
+  //
+  // Unsafe operations (don't check capacity/don't resize)
+  //
+
+  // Append to null bitmap.
+  void UnsafeAppendToBitmap(bool is_valid);
+  // Vector append. Treat each zero byte as a nullzero. If valid_bytes is null
+  // assume all of length bits are valid.
+  void UnsafeAppendToBitmap(const uint8_t* valid_bytes, int32_t length);
+  // Set the next length bits to not null (i.e. valid).
+  void UnsafeSetNotNull(int32_t length);
+
  private:
   DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
 };
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 2f72c3aa8467a..bf6fa94dea7a4 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -19,17 +19,19 @@
 
 #include <cstdint>
 #include <cstring>
+#include <sstream>
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/ipc/memory.h"
 #include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/memory.h"
 #include "arrow/ipc/metadata-internal.h"
+#include "arrow/ipc/metadata.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/types/construct.h"
+#include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
@@ -63,44 +65,70 @@ static bool IsPrimitive(const DataType* type) {
   }
 }
 
+static bool IsListType(const DataType* type) {
+  DCHECK(type != nullptr);
+  switch (type->type) {
+    // TODO(emkornfield) grouping like this are used in a few places in the
+    // code consider using pattern like:
+    // http://stackoverflow.com/questions/26784685/c-macro-for-calling-function-based-on-enum-type
+    //
+    // TODO(emkornfield) Fix type systems so these are all considered lists and
+    // the types behave the same way?
+    // case Type::BINARY:
+    // case Type::CHAR:
+    case Type::LIST:
+      // see todo on common types
+      // case Type::STRING:
+      // case Type::VARCHAR:
+      return true;
+    default:
+      return false;
+  }
+}
+
 // ----------------------------------------------------------------------
 // Row batch write path
 
 Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes,
-    std::vector<std::shared_ptr<Buffer>>* buffers) {
-  if (IsPrimitive(arr->type().get())) {
-    const PrimitiveArray* prim_arr = static_cast<const PrimitiveArray*>(arr);
-
-    field_nodes->push_back(
-        flatbuf::FieldNode(prim_arr->length(), prim_arr->null_count()));
+    std::vector<std::shared_ptr<Buffer>>* buffers, int max_recursion_depth) {
+  if (max_recursion_depth <= 0) { return Status::Invalid("Max recursion depth reached"); }
+  DCHECK(arr);
+  DCHECK(field_nodes);
+  // push back all common elements
+  field_nodes->push_back(flatbuf::FieldNode(arr->length(), arr->null_count()));
+  if (arr->null_count() > 0) {
+    buffers->push_back(arr->null_bitmap());
+  } else {
+    // Push a dummy zero-length buffer, not to be copied
+    buffers->push_back(std::make_shared<Buffer>(nullptr, 0));
+  }
 
-    if (prim_arr->null_count() > 0) {
-      buffers->push_back(prim_arr->null_bitmap());
-    } else {
-      // Push a dummy zero-length buffer, not to be copied
-      buffers->push_back(std::make_shared<Buffer>(nullptr, 0));
-    }
+  const DataType* arr_type = arr->type().get();
+  if (IsPrimitive(arr_type)) {
+    const auto prim_arr = static_cast<const PrimitiveArray*>(arr);
     buffers->push_back(prim_arr->data());
-  } else if (arr->type_enum() == Type::LIST) {
-    // TODO(wesm)
-    return Status::NotImplemented("List type");
+  } else if (IsListType(arr_type)) {
+    const auto list_arr = static_cast<const ListArray*>(arr);
+    buffers->push_back(list_arr->offset_buffer());
+    RETURN_NOT_OK(VisitArray(
+        list_arr->values().get(), field_nodes, buffers, max_recursion_depth - 1));
   } else if (arr->type_enum() == Type::STRUCT) {
     // TODO(wesm)
     return Status::NotImplemented("Struct type");
   }
-
   return Status::OK();
 }
 
 class RowBatchWriter {
  public:
-  explicit RowBatchWriter(const RowBatch* batch) : batch_(batch) {}
+  RowBatchWriter(const RowBatch* batch, int max_recursion_depth)
+      : batch_(batch), max_recursion_depth_(max_recursion_depth) {}
 
   Status AssemblePayload() {
     // Perform depth-first traversal of the row-batch
     for (int i = 0; i < batch_->num_columns(); ++i) {
       const Array* arr = batch_->column(i).get();
-      RETURN_NOT_OK(VisitArray(arr, &field_nodes_, &buffers_));
+      RETURN_NOT_OK(VisitArray(arr, &field_nodes_, &buffers_, max_recursion_depth_));
     }
     return Status::OK();
   }
@@ -111,8 +139,10 @@ class RowBatchWriter {
     int64_t offset = 0;
     for (size_t i = 0; i < buffers_.size(); ++i) {
       const Buffer* buffer = buffers_[i].get();
-      int64_t size = buffer->size();
+      int64_t size = 0;
 
+      // The buffer might be null if we are handling zero row lengths.
+      if (buffer) { size = buffer->size(); }
       // TODO(wesm): We currently have no notion of shared memory page id's,
       // but we've included it in the metadata IDL for when we have it in the
       // future. Use page=0 for now
@@ -171,11 +201,13 @@ class RowBatchWriter {
   std::vector<flatbuf::FieldNode> field_nodes_;
   std::vector<flatbuf::Buffer> buffer_meta_;
   std::vector<std::shared_ptr<Buffer>> buffers_;
+  int max_recursion_depth_;
 };
 
-Status WriteRowBatch(
-    MemorySource* dst, const RowBatch* batch, int64_t position, int64_t* header_offset) {
-  RowBatchWriter serializer(batch);
+Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
+    int64_t* header_offset, int max_recursion_depth) {
+  DCHECK_GT(max_recursion_depth, 0);
+  RowBatchWriter serializer(batch, max_recursion_depth);
   RETURN_NOT_OK(serializer.AssemblePayload());
   return serializer.Write(dst, position, header_offset);
 }
@@ -186,8 +218,9 @@ static constexpr int64_t INIT_METADATA_SIZE = 4096;
 
 class RowBatchReader::Impl {
  public:
-  Impl(MemorySource* source, const std::shared_ptr<RecordBatchMessage>& metadata)
-      : source_(source), metadata_(metadata) {
+  Impl(MemorySource* source, const std::shared_ptr<RecordBatchMessage>& metadata,
+      int max_recursion_depth)
+      : source_(source), metadata_(metadata), max_recursion_depth_(max_recursion_depth) {
     num_buffers_ = metadata->num_buffers();
     num_flattened_fields_ = metadata->num_fields();
   }
@@ -203,7 +236,7 @@ class RowBatchReader::Impl {
     buffer_index_ = 0;
     for (int i = 0; i < schema->num_fields(); ++i) {
       const Field* field = schema->field(i).get();
-      RETURN_NOT_OK(NextArray(field, &arrays[i]));
+      RETURN_NOT_OK(NextArray(field, max_recursion_depth_, &arrays[i]));
     }
 
     *out = std::make_shared<RowBatch>(schema, metadata_->length(), arrays);
@@ -213,8 +246,12 @@ class RowBatchReader::Impl {
  private:
   // Traverse the flattened record batch metadata and reassemble the
   // corresponding array containers
-  Status NextArray(const Field* field, std::shared_ptr<Array>* out) {
-    const std::shared_ptr<DataType>& type = field->type;
+  Status NextArray(
+      const Field* field, int max_recursion_depth, std::shared_ptr<Array>* out) {
+    const TypePtr& type = field->type;
+    if (max_recursion_depth <= 0) {
+      return Status::Invalid("Max recursion depth reached");
+    }
 
     // pop off a field
     if (field_index_ >= num_flattened_fields_) {
@@ -226,23 +263,42 @@ class RowBatchReader::Impl {
     // we can skip that buffer without reading from shared memory
     FieldMetadata field_meta = metadata_->field(field_index_++);
 
+    // extract null_bitmap which is common to all arrays
+    std::shared_ptr<Buffer> null_bitmap;
+    if (field_meta.null_count == 0) {
+      ++buffer_index_;
+    } else {
+      RETURN_NOT_OK(GetBuffer(buffer_index_++, &null_bitmap));
+    }
+
     if (IsPrimitive(type.get())) {
-      std::shared_ptr<Buffer> null_bitmap;
       std::shared_ptr<Buffer> data;
-      if (field_meta.null_count == 0) {
-        null_bitmap = nullptr;
-        ++buffer_index_;
-      } else {
-        RETURN_NOT_OK(GetBuffer(buffer_index_++, &null_bitmap));
-      }
       if (field_meta.length > 0) {
         RETURN_NOT_OK(GetBuffer(buffer_index_++, &data));
       } else {
+        buffer_index_++;
         data.reset(new Buffer(nullptr, 0));
       }
       return MakePrimitiveArray(
           type, field_meta.length, data, field_meta.null_count, null_bitmap, out);
     }
+
+    if (IsListType(type.get())) {
+      std::shared_ptr<Buffer> offsets;
+      RETURN_NOT_OK(GetBuffer(buffer_index_++, &offsets));
+      const int num_children = type->num_children();
+      if (num_children != 1) {
+        std::stringstream ss;
+        ss << "Field: " << field->ToString()
+           << " has wrong number of children:" << num_children;
+        return Status::Invalid(ss.str());
+      }
+      std::shared_ptr<Array> values_array;
+      RETURN_NOT_OK(
+          NextArray(type->child(0).get(), max_recursion_depth - 1, &values_array));
+      return MakeListArray(type, field_meta.length, offsets, values_array,
+          field_meta.null_count, null_bitmap, out);
+    }
     return Status::NotImplemented("Non-primitive types not complete yet");
   }
 
@@ -256,12 +312,18 @@ class RowBatchReader::Impl {
 
   int field_index_;
   int buffer_index_;
+  int max_recursion_depth_;
   int num_buffers_;
   int num_flattened_fields_;
 };
 
 Status RowBatchReader::Open(
     MemorySource* source, int64_t position, std::shared_ptr<RowBatchReader>* out) {
+  return Open(source, position, kMaxIpcRecursionDepth, out);
+}
+
+Status RowBatchReader::Open(MemorySource* source, int64_t position,
+    int max_recursion_depth, std::shared_ptr<RowBatchReader>* out) {
   std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(source->ReadAt(position, INIT_METADATA_SIZE, &metadata));
 
@@ -286,7 +348,7 @@ Status RowBatchReader::Open(
   std::shared_ptr<RecordBatchMessage> batch_meta = message->GetRecordBatch();
 
   std::shared_ptr<RowBatchReader> result(new RowBatchReader());
-  result->impl_.reset(new Impl(source, batch_meta));
+  result->impl_.reset(new Impl(source, batch_meta, max_recursion_depth));
   *out = result;
 
   return Status::OK();
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index d453fa05f4982..4c9a8a9d8ee39 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -38,7 +38,9 @@ class RecordBatchMessage;
 
 // ----------------------------------------------------------------------
 // Write path
-
+// We have trouble decoding flatbuffers if the size i > 70, so 64 is a nice round number
+// TODO(emkornfield) investigate this more
+constexpr int kMaxIpcRecursionDepth = 64;
 // Write the RowBatch (collection of equal-length Arrow arrays) to the memory
 // source at the indicated position
 //
@@ -52,8 +54,8 @@ class RecordBatchMessage;
 //
 // Finally, the memory offset to the start of the metadata / data header is
 // returned in an out-variable
-Status WriteRowBatch(
-    MemorySource* dst, const RowBatch* batch, int64_t position, int64_t* header_offset);
+Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
+    int64_t* header_offset, int max_recursion_depth = kMaxIpcRecursionDepth);
 
 // int64_t GetRowBatchMetadata(const RowBatch* batch);
 
@@ -70,6 +72,9 @@ class RowBatchReader {
   static Status Open(
       MemorySource* source, int64_t position, std::shared_ptr<RowBatchReader>* out);
 
+  static Status Open(MemorySource* source, int64_t position, int max_recursion_depth,
+      std::shared_ptr<RowBatchReader>* out);
+
   // Reassemble the row batch. A Schema is required to be able to construct the
   // right array containers
   Status GetRowBatch(
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index fbdda77e4919c..c243cfba820cc 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -18,9 +18,7 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
-#include <limits>
 #include <memory>
-#include <random>
 #include <string>
 #include <vector>
 
@@ -31,6 +29,7 @@
 #include "arrow/ipc/test-common.h"
 
 #include "arrow/test-util.h"
+#include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
@@ -40,25 +39,56 @@
 namespace arrow {
 namespace ipc {
 
-class TestWriteRowBatch : public ::testing::Test, public MemoryMapFixture {
+// TODO(emkornfield) convert to google style kInt32, etc?
+const auto INT32 = std::make_shared<Int32Type>();
+const auto LIST_INT32 = std::make_shared<ListType>(INT32);
+const auto LIST_LIST_INT32 = std::make_shared<ListType>(LIST_INT32);
+
+typedef Status MakeRowBatch(std::shared_ptr<RowBatch>* out);
+
+class TestWriteRowBatch : public ::testing::TestWithParam<MakeRowBatch*>,
+                          public MemoryMapFixture {
  public:
   void SetUp() { pool_ = default_memory_pool(); }
   void TearDown() { MemoryMapFixture::TearDown(); }
 
-  void InitMemoryMap(int64_t size) {
+  Status RoundTripHelper(const RowBatch& batch, int memory_map_size,
+      std::shared_ptr<RowBatch>* batch_result) {
     std::string path = "test-write-row-batch";
-    MemoryMapFixture::CreateFile(path, size);
-    ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_WRITE, &mmap_));
+    MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
+    int64_t header_location;
+    RETURN_NOT_OK(WriteRowBatch(mmap_.get(), &batch, 0, &header_location));
+
+    std::shared_ptr<RowBatchReader> reader;
+    RETURN_NOT_OK(RowBatchReader::Open(mmap_.get(), header_location, &reader));
+
+    RETURN_NOT_OK(reader->GetRowBatch(batch.schema(), batch_result));
+    return Status::OK();
   }
 
  protected:
-  MemoryPool* pool_;
   std::shared_ptr<MemoryMappedSource> mmap_;
+  MemoryPool* pool_;
 };
 
-const auto INT32 = std::make_shared<Int32Type>();
+TEST_P(TestWriteRowBatch, RoundTrip) {
+  std::shared_ptr<RowBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
+  std::shared_ptr<RowBatch> batch_result;
+  ASSERT_OK(RoundTripHelper(*batch, 1 << 16, &batch_result));
+
+  // do checks
+  ASSERT_TRUE(batch->schema()->Equals(batch_result->schema()));
+  ASSERT_EQ(batch->num_columns(), batch_result->num_columns())
+      << batch->schema()->ToString() << " result: " << batch_result->schema()->ToString();
+  EXPECT_EQ(batch->num_rows(), batch_result->num_rows());
+  for (int i = 0; i < batch->num_columns(); ++i) {
+    EXPECT_TRUE(batch->column(i)->Equals(batch_result->column(i)))
+        << "Idx: " << i << " Name: " << batch->column_name(i);
+  }
+}
 
-TEST_F(TestWriteRowBatch, IntegerRoundTrip) {
+Status MakeIntRowBatch(std::shared_ptr<RowBatch>* out) {
   const int length = 1000;
 
   // Make the schema
@@ -67,41 +97,159 @@ TEST_F(TestWriteRowBatch, IntegerRoundTrip) {
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
   // Example data
+  std::shared_ptr<Array> a0, a1;
+  MemoryPool* pool = default_memory_pool();
+  RETURN_NOT_OK(MakeRandomInt32Array(length, false, pool, &a0));
+  RETURN_NOT_OK(MakeRandomInt32Array(length, true, pool, &a1));
+  out->reset(new RowBatch(schema, length, {a0, a1}));
+  return Status::OK();
+}
 
-  auto data = std::make_shared<PoolBuffer>(pool_);
-  ASSERT_OK(data->Resize(length * sizeof(int32_t)));
-  test::rand_uniform_int(length, 0, 0, std::numeric_limits<int32_t>::max(),
-      reinterpret_cast<int32_t*>(data->mutable_data()));
+Status MakeListRowBatch(std::shared_ptr<RowBatch>* out) {
+  // Make the schema
+  auto f0 = std::make_shared<Field>("f0", LIST_INT32);
+  auto f1 = std::make_shared<Field>("f1", LIST_LIST_INT32);
+  auto f2 = std::make_shared<Field>("f2", INT32);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
-  auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
-  int null_bytes = util::bytes_for_bits(length);
-  ASSERT_OK(null_bitmap->Resize(null_bytes));
-  test::random_bytes(null_bytes, 0, null_bitmap->mutable_data());
+  // Example data
 
-  auto a0 = std::make_shared<Int32Array>(length, data);
-  auto a1 = std::make_shared<Int32Array>(
-      length, data, test::bitmap_popcount(null_bitmap->data(), length), null_bitmap);
+  MemoryPool* pool = default_memory_pool();
+  const int length = 200;
+  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
+  const bool include_nulls = true;
+  RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool, &leaf_values));
+  RETURN_NOT_OK(
+      MakeRandomListArray(leaf_values, length, include_nulls, pool, &list_array));
+  RETURN_NOT_OK(
+      MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array));
+  RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array));
+  out->reset(new RowBatch(schema, length, {list_array, list_list_array, flat_array}));
+  return Status::OK();
+}
 
-  RowBatch batch(schema, length, {a0, a1});
+Status MakeZeroLengthRowBatch(std::shared_ptr<RowBatch>* out) {
+  // Make the schema
+  auto f0 = std::make_shared<Field>("f0", LIST_INT32);
+  auto f1 = std::make_shared<Field>("f1", LIST_LIST_INT32);
+  auto f2 = std::make_shared<Field>("f2", INT32);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
-  // TODO(wesm): computing memory requirements for a row batch
-  // 64k is plenty of space
-  InitMemoryMap(1 << 16);
+  // Example data
+  MemoryPool* pool = default_memory_pool();
+  const int length = 200;
+  const bool include_nulls = true;
+  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
+  RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &leaf_values));
+  RETURN_NOT_OK(MakeRandomListArray(leaf_values, 0, include_nulls, pool, &list_array));
+  RETURN_NOT_OK(
+      MakeRandomListArray(list_array, 0, include_nulls, pool, &list_list_array));
+  RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &flat_array));
+  out->reset(new RowBatch(schema, length, {list_array, list_list_array, flat_array}));
+  return Status::OK();
+}
 
-  int64_t header_location;
-  ASSERT_OK(WriteRowBatch(mmap_.get(), &batch, 0, &header_location));
+Status MakeNonNullRowBatch(std::shared_ptr<RowBatch>* out) {
+  // Make the schema
+  auto f0 = std::make_shared<Field>("f0", LIST_INT32);
+  auto f1 = std::make_shared<Field>("f1", LIST_LIST_INT32);
+  auto f2 = std::make_shared<Field>("f2", INT32);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
-  std::shared_ptr<RowBatchReader> result;
-  ASSERT_OK(RowBatchReader::Open(mmap_.get(), header_location, &result));
+  // Example data
+  MemoryPool* pool = default_memory_pool();
+  const int length = 200;
+  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
 
-  std::shared_ptr<RowBatch> batch_result;
-  ASSERT_OK(result->GetRowBatch(schema, &batch_result));
-  EXPECT_EQ(batch.num_rows(), batch_result->num_rows());
+  RETURN_NOT_OK(MakeRandomInt32Array(1000, true, pool, &leaf_values));
+  bool include_nulls = false;
+  RETURN_NOT_OK(MakeRandomListArray(leaf_values, 50, include_nulls, pool, &list_array));
+  RETURN_NOT_OK(
+      MakeRandomListArray(list_array, 50, include_nulls, pool, &list_list_array));
+  RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &flat_array));
+  out->reset(new RowBatch(schema, length, {list_array, list_list_array, flat_array}));
+  return Status::OK();
+}
 
-  for (int i = 0; i < batch.num_columns(); ++i) {
-    EXPECT_TRUE(batch.column(i)->Equals(batch_result->column(i))) << i
-                                                                  << batch.column_name(i);
+Status MakeDeeplyNestedList(std::shared_ptr<RowBatch>* out) {
+  const int batch_length = 5;
+  TypePtr type = INT32;
+
+  MemoryPool* pool = default_memory_pool();
+  ArrayPtr array;
+  const bool include_nulls = true;
+  RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool, &array));
+  for (int i = 0; i < 63; ++i) {
+    type = std::static_pointer_cast<DataType>(std::make_shared<ListType>(type));
+    RETURN_NOT_OK(MakeRandomListArray(array, batch_length, include_nulls, pool, &array));
+  }
+
+  auto f0 = std::make_shared<Field>("f0", type);
+  std::shared_ptr<Schema> schema(new Schema({f0}));
+  std::vector<ArrayPtr> arrays = {array};
+  out->reset(new RowBatch(schema, batch_length, arrays));
+  return Status::OK();
+}
+
+INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRowBatch,
+    ::testing::Values(&MakeIntRowBatch, &MakeListRowBatch, &MakeNonNullRowBatch,
+                            &MakeZeroLengthRowBatch, &MakeDeeplyNestedList));
+
+class RecursionLimits : public ::testing::Test, public MemoryMapFixture {
+ public:
+  void SetUp() { pool_ = default_memory_pool(); }
+  void TearDown() { MemoryMapFixture::TearDown(); }
+
+  Status WriteToMmap(int recursion_level, bool override_level,
+      int64_t* header_out = nullptr, std::shared_ptr<Schema>* schema_out = nullptr) {
+    const int batch_length = 5;
+    TypePtr type = INT32;
+    ArrayPtr array;
+    const bool include_nulls = true;
+    RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
+    for (int i = 0; i < recursion_level; ++i) {
+      type = std::static_pointer_cast<DataType>(std::make_shared<ListType>(type));
+      RETURN_NOT_OK(
+          MakeRandomListArray(array, batch_length, include_nulls, pool_, &array));
+    }
+
+    auto f0 = std::make_shared<Field>("f0", type);
+    std::shared_ptr<Schema> schema(new Schema({f0}));
+    if (schema_out != nullptr) { *schema_out = schema; }
+    std::vector<ArrayPtr> arrays = {array};
+    auto batch = std::make_shared<RowBatch>(schema, batch_length, arrays);
+
+    std::string path = "test-write-past-max-recursion";
+    const int memory_map_size = 1 << 16;
+    MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
+    int64_t header_location;
+    int64_t* header_out_param = header_out == nullptr ? &header_location : header_out;
+    if (override_level) {
+      return WriteRowBatch(
+          mmap_.get(), batch.get(), 0, header_out_param, recursion_level + 1);
+    } else {
+      return WriteRowBatch(mmap_.get(), batch.get(), 0, header_out_param);
+    }
   }
+
+ protected:
+  std::shared_ptr<MemoryMappedSource> mmap_;
+  MemoryPool* pool_;
+};
+
+TEST_F(RecursionLimits, WriteLimit) {
+  ASSERT_RAISES(Invalid, WriteToMmap((1 << 8) + 1, false));
+}
+
+TEST_F(RecursionLimits, ReadLimit) {
+  int64_t header_location;
+  std::shared_ptr<Schema> schema;
+  ASSERT_OK(WriteToMmap(64, true, &header_location, &schema));
+
+  std::shared_ptr<RowBatchReader> reader;
+  ASSERT_OK(RowBatchReader::Open(mmap_.get(), header_location, &reader));
+  std::shared_ptr<RowBatch> batch_result;
+  ASSERT_RAISES(Invalid, reader->GetRowBatch(schema, &batch_result));
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/memory.cc b/cpp/src/arrow/ipc/memory.cc
index 2b077e9792925..84cbc182cd26f 100644
--- a/cpp/src/arrow/ipc/memory.cc
+++ b/cpp/src/arrow/ipc/memory.cc
@@ -18,6 +18,7 @@
 #include "arrow/ipc/memory.h"
 
 #include <sys/mman.h>  // For memory-mapping
+
 #include <algorithm>
 #include <cerrno>
 #include <cstdint>
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index ad5951d17e2c0..1b1d50f96eaf5 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -17,13 +17,14 @@
 
 #include "arrow/ipc/metadata-internal.h"
 
-#include <flatbuffers/flatbuffers.h>
 #include <cstdint>
 #include <cstring>
 #include <memory>
 #include <sstream>
 #include <string>
 
+#include "flatbuffers/flatbuffers.h"
+
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/schema.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index 779c5a30a044a..871b5bc4bf606 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -18,11 +18,12 @@
 #ifndef ARROW_IPC_METADATA_INTERNAL_H
 #define ARROW_IPC_METADATA_INTERNAL_H
 
-#include <flatbuffers/flatbuffers.h>
 #include <cstdint>
 #include <memory>
 #include <vector>
 
+#include "flatbuffers/flatbuffers.h"
+
 #include "arrow/ipc/Message_generated.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index bcf104f0b8ba6..4fc8ec50eb716 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -17,11 +17,12 @@
 
 #include "arrow/ipc/metadata.h"
 
-#include <flatbuffers/flatbuffers.h>
 #include <cstdint>
 #include <memory>
 #include <vector>
 
+#include "flatbuffers/flatbuffers.h"
+
 // Generated C++ flatbuffer IDL
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/ipc/metadata-internal.h"
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 65c837dc8b141..e7dbb84d790a1 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -18,11 +18,19 @@
 #ifndef ARROW_IPC_TEST_COMMON_H
 #define ARROW_IPC_TEST_COMMON_H
 
+#include <algorithm>
 #include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
+#include "arrow/array.h"
+#include "arrow/test-util.h"
+#include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
+
 namespace arrow {
 namespace ipc {
 
@@ -41,10 +49,69 @@ class MemoryMapFixture {
     fclose(file);
   }
 
+  Status InitMemoryMap(
+      int64_t size, const std::string& path, std::shared_ptr<MemoryMappedSource>* mmap) {
+    CreateFile(path, size);
+    return MemoryMappedSource::Open(path, MemorySource::READ_WRITE, mmap);
+  }
+
  private:
   std::vector<std::string> tmp_files_;
 };
 
+Status MakeRandomInt32Array(
+    int32_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* array) {
+  std::shared_ptr<PoolBuffer> data;
+  test::MakeRandomInt32PoolBuffer(length, pool, &data);
+  const auto INT32 = std::make_shared<Int32Type>();
+  Int32Builder builder(pool, INT32);
+  if (include_nulls) {
+    std::shared_ptr<PoolBuffer> valid_bytes;
+    test::MakeRandomBytePoolBuffer(length, pool, &valid_bytes);
+    RETURN_NOT_OK(builder.Append(
+        reinterpret_cast<const int32_t*>(data->data()), length, valid_bytes->data()));
+    *array = builder.Finish();
+    return Status::OK();
+  }
+  RETURN_NOT_OK(builder.Append(reinterpret_cast<const int32_t*>(data->data()), length));
+  *array = builder.Finish();
+  return Status::OK();
+}
+
+Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_lists,
+    bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* array) {
+  // Create the null list values
+  std::vector<uint8_t> valid_lists(num_lists);
+  const double null_percent = include_nulls ? 0.1 : 0;
+  test::random_null_bytes(num_lists, null_percent, valid_lists.data());
+
+  // Create list offsets
+  const int max_list_size = 10;
+
+  std::vector<int32_t> list_sizes(num_lists, 0);
+  std::vector<int32_t> offsets(
+      num_lists + 1, 0);  // +1 so we can shift for nulls. See partial sum below.
+  const int seed = child_array->length();
+  if (num_lists > 0) {
+    test::rand_uniform_int(num_lists, seed, 0, max_list_size, list_sizes.data());
+    // make sure sizes are consistent with null
+    std::transform(list_sizes.begin(), list_sizes.end(), valid_lists.begin(),
+        list_sizes.begin(),
+        [](int32_t size, int32_t valid) { return valid == 0 ? 0 : size; });
+    std::partial_sum(list_sizes.begin(), list_sizes.end(), ++offsets.begin());
+
+    // Force invariants
+    const int child_length = child_array->length();
+    offsets[0] = 0;
+    std::replace_if(offsets.begin(), offsets.end(),
+        [child_length](int32_t offset) { return offset > child_length; }, child_length);
+  }
+  ListBuilder builder(pool, child_array);
+  RETURN_NOT_OK(builder.Append(offsets.data(), num_lists, valid_lists.data()));
+  *array = builder.Finish();
+  return (*array)->Validate();
+}
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index 066388b4d0e23..560e28374066b 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -21,8 +21,8 @@
 
 #include "parquet/api/schema.h"
 
-#include "arrow/util/status.h"
 #include "arrow/types/decimal.h"
+#include "arrow/util/status.h"
 
 using parquet::schema::Node;
 using parquet::schema::NodePtr;
diff --git a/cpp/src/arrow/schema.cc b/cpp/src/arrow/schema.cc
index a38acaa94ba56..ff3ea1990e551 100644
--- a/cpp/src/arrow/schema.cc
+++ b/cpp/src/arrow/schema.cc
@@ -18,8 +18,8 @@
 #include "arrow/schema.h"
 
 #include <memory>
-#include <string>
 #include <sstream>
+#include <string>
 #include <vector>
 
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 538d9b233d990..2f81161d1d6d1 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -19,6 +19,7 @@
 #define ARROW_TEST_UTIL_H_
 
 #include <cstdint>
+#include <limits>
 #include <memory>
 #include <random>
 #include <string>
@@ -26,12 +27,13 @@
 
 #include "gtest/gtest.h"
 
-#include "arrow/type.h"
 #include "arrow/column.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
+#include "arrow/type.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/random.h"
 #include "arrow/util/status.h"
@@ -103,10 +105,12 @@ std::shared_ptr<Buffer> to_buffer(const std::vector<T>& values) {
       reinterpret_cast<const uint8_t*>(values.data()), values.size() * sizeof(T));
 }
 
-void random_null_bitmap(int64_t n, double pct_null, uint8_t* null_bitmap) {
+// Sets approximately pct_null of the first n bytes in null_bytes to zero
+// and the rest to non-zero (true) values.
+void random_null_bytes(int64_t n, double pct_null, uint8_t* null_bytes) {
   Random rng(random_seed());
   for (int i = 0; i < n; ++i) {
-    null_bitmap[i] = rng.NextDoubleFraction() > pct_null;
+    null_bytes[i] = rng.NextDoubleFraction() > pct_null;
   }
 }
 
@@ -121,6 +125,7 @@ static inline void random_bytes(int n, uint32_t seed, uint8_t* out) {
 
 template <typename T>
 void rand_uniform_int(int n, uint32_t seed, T min_value, T max_value, T* out) {
+  DCHECK(out);
   std::mt19937 gen(seed);
   std::uniform_int_distribution<T> d(min_value, max_value);
   for (int i = 0; i < n; ++i) {
@@ -129,11 +134,25 @@ void rand_uniform_int(int n, uint32_t seed, T min_value, T max_value, T* out) {
 }
 
 static inline int bitmap_popcount(const uint8_t* data, int length) {
+  // book keeping
+  constexpr int pop_len = sizeof(uint64_t);
+  const uint64_t* i64_data = reinterpret_cast<const uint64_t*>(data);
+  const int fast_counts = length / pop_len;
+  const uint64_t* end = i64_data + fast_counts;
+
   int count = 0;
-  for (int i = 0; i < length; ++i) {
-    // TODO(wesm): accelerate this
+  // popcount as much as possible with the widest possible count
+  for (auto iter = i64_data; iter < end; ++iter) {
+    count += __builtin_popcountll(*iter);
+  }
+
+  // Account for left over bytes (in theory we could fall back to smaller
+  // versions of popcount but the code complexity is likely not worth it)
+  const int loop_tail_index = fast_counts * pop_len;
+  for (int i = loop_tail_index; i < length; ++i) {
     if (util::get_bit(data, i)) { ++count; }
   }
+
   return count;
 }
 
@@ -153,6 +172,26 @@ std::shared_ptr<Buffer> bytes_to_null_buffer(const std::vector<uint8_t>& bytes)
   return out;
 }
 
+Status MakeRandomInt32PoolBuffer(int32_t length, MemoryPool* pool,
+    std::shared_ptr<PoolBuffer>* pool_buffer, uint32_t seed = 0) {
+  DCHECK(pool);
+  auto data = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(data->Resize(length * sizeof(int32_t)));
+  test::rand_uniform_int(length, seed, 0, std::numeric_limits<int32_t>::max(),
+      reinterpret_cast<int32_t*>(data->mutable_data()));
+  *pool_buffer = data;
+  return Status::OK();
+}
+
+Status MakeRandomBytePoolBuffer(int32_t length, MemoryPool* pool,
+    std::shared_ptr<PoolBuffer>* pool_buffer, uint32_t seed = 0) {
+  auto bytes = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(bytes->Resize(length));
+  test::random_bytes(length, seed, bytes->mutable_data());
+  *pool_buffer = bytes;
+  return Status::OK();
+}
+
 }  // namespace test
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 051ab46b199f9..77404cd702524 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -116,7 +116,7 @@ struct DataType {
 
   bool Equals(const DataType* other) {
     // Call with a pointer so more friendly to subclasses
-    return this == other || (this->type == other->type);
+    return other && ((this == other) || (this->type == other->type));
   }
 
   bool Equals(const std::shared_ptr<DataType>& other) { return Equals(other.get()); }
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 0a30929b97c51..78036d4bf5711 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -20,8 +20,8 @@
 #include <memory>
 
 #include "arrow/type.h"
-#include "arrow/types/primitive.h"
 #include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
@@ -60,11 +60,10 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
-
       const std::shared_ptr<DataType>& value_type =
           static_cast<ListType*>(type.get())->value_type();
       RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
-      out->reset(new ListBuilder(pool, type, value_builder));
+      out->reset(new ListBuilder(pool, value_builder));
       return Status::OK();
     }
     default:
@@ -75,11 +74,11 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 #define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                          \
   case Type::ENUM:                                                          \
     out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
-    return Status::OK();
+    break;
 
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
+Status MakePrimitiveArray(const TypePtr& type, int32_t length,
     const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out) {
+    const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
   switch (type->type) {
     MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
@@ -90,11 +89,43 @@ Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
     MAKE_PRIMITIVE_ARRAY_CASE(INT32, Int32Array);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
     MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, Int64Array);
     MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
     MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP_DOUBLE, DoubleArray);
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+#ifdef NDEBUG
+  return Status::OK();
+#else
+  return (*out)->Validate();
+#endif
+}
+
+Status MakeListArray(const TypePtr& type, int32_t length,
+    const std::shared_ptr<Buffer>& offsets, const ArrayPtr& values, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
+  switch (type->type) {
+    case Type::BINARY:
+    case Type::LIST:
+      out->reset(new ListArray(type, length, offsets, values, null_count, null_bitmap));
+      break;
+    case Type::CHAR:
+    case Type::DECIMAL_TEXT:
+    case Type::STRING:
+    case Type::VARCHAR:
+      out->reset(new StringArray(type, length, offsets, values, null_count, null_bitmap));
+      break;
     default:
       return Status::NotImplemented(type->ToString());
   }
+#ifdef NDEBUG
+  return Status::OK();
+#else
+  return (*out)->Validate();
+#endif
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index 27fb7bd2149cf..43c0018c67e41 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -33,10 +33,19 @@ class Status;
 Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     std::shared_ptr<ArrayBuilder>* out);
 
+// Create new arrays for logical types that are backed by primitive arrays.
 Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
 
+// Create new list arrays for logical types that are backed by ListArrays (e.g. list of
+// primitives and strings)
+// TODO(emkornfield) split up string vs list?
+Status MakeListArray(const std::shared_ptr<DataType>& type, int32_t length,
+    const std::shared_ptr<Buffer>& offests, const std::shared_ptr<Array>& values,
+    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
+    std::shared_ptr<Array>* out);
+
 }  // namespace arrow
 
 #endif  // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index aa34f23cc0230..6a8ad9aa59ead 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <cstdlib>
 #include <cstdint>
+#include <cstdlib>
 #include <memory>
 #include <string>
 #include <vector>
@@ -94,6 +94,7 @@ TEST_F(TestListBuilder, TestAppendNull) {
 
   Done();
 
+  ASSERT_OK(result_->Validate());
   ASSERT_TRUE(result_->IsNull(0));
   ASSERT_TRUE(result_->IsNull(1));
 
@@ -105,50 +106,93 @@ TEST_F(TestListBuilder, TestAppendNull) {
   ASSERT_EQ(0, values->length());
 }
 
+void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& values,
+    const vector<uint8_t>& is_valid) {
+  ASSERT_OK(result->Validate());
+  ASSERT_EQ(1, result->null_count());
+  ASSERT_EQ(0, result->values()->null_count());
+
+  ASSERT_EQ(3, result->length());
+  vector<int32_t> ex_offsets = {0, 3, 3, 7};
+  for (size_t i = 0; i < ex_offsets.size(); ++i) {
+    ASSERT_EQ(ex_offsets[i], result->offset(i));
+  }
+
+  for (int i = 0; i < result->length(); ++i) {
+    ASSERT_EQ(!static_cast<bool>(is_valid[i]), result->IsNull(i));
+  }
+
+  ASSERT_EQ(7, result->values()->length());
+  Int32Array* varr = static_cast<Int32Array*>(result->values().get());
+
+  for (size_t i = 0; i < values.size(); ++i) {
+    ASSERT_EQ(values[i], varr->Value(i));
+  }
+}
+
 TEST_F(TestListBuilder, TestBasics) {
   vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
   vector<int> lengths = {3, 0, 4};
-  vector<uint8_t> is_null = {0, 1, 0};
+  vector<uint8_t> is_valid = {1, 0, 1};
 
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
 
-  EXPECT_OK(builder_->Reserve(lengths.size()));
-  EXPECT_OK(vb->Reserve(values.size()));
+  ASSERT_OK(builder_->Reserve(lengths.size()));
+  ASSERT_OK(vb->Reserve(values.size()));
 
   int pos = 0;
   for (size_t i = 0; i < lengths.size(); ++i) {
-    ASSERT_OK(builder_->Append(is_null[i] > 0));
+    ASSERT_OK(builder_->Append(is_valid[i] > 0));
     for (int j = 0; j < lengths[i]; ++j) {
       vb->Append(values[pos++]);
     }
   }
 
   Done();
+  ValidateBasicListArray(result_.get(), values, is_valid);
+}
 
-  ASSERT_EQ(1, result_->null_count());
-  ASSERT_EQ(0, result_->values()->null_count());
+TEST_F(TestListBuilder, BulkAppend) {
+  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+  vector<int> lengths = {3, 0, 4};
+  vector<uint8_t> is_valid = {1, 0, 1};
+  vector<int32_t> offsets = {0, 3, 3};
 
-  ASSERT_EQ(3, result_->length());
-  vector<int32_t> ex_offsets = {0, 3, 3, 7};
-  for (size_t i = 0; i < ex_offsets.size(); ++i) {
-    ASSERT_EQ(ex_offsets[i], result_->offset(i));
-  }
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+  ASSERT_OK(vb->Reserve(values.size()));
 
-  for (int i = 0; i < result_->length(); ++i) {
-    ASSERT_EQ(static_cast<bool>(is_null[i]), result_->IsNull(i));
+  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+  for (int32_t value : values) {
+    vb->Append(value);
   }
+  Done();
+  ValidateBasicListArray(result_.get(), values, is_valid);
+}
 
-  ASSERT_EQ(7, result_->values()->length());
-  Int32Array* varr = static_cast<Int32Array*>(result_->values().get());
+TEST_F(TestListBuilder, BulkAppendInvalid) {
+  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+  vector<int> lengths = {3, 0, 4};
+  vector<uint8_t> is_null = {0, 1, 0};
+  vector<uint8_t> is_valid = {1, 0, 1};
+  vector<int32_t> offsets = {0, 2, 4};  // should be 0, 3, 3 given the is_null array
 
-  for (size_t i = 0; i < values.size(); ++i) {
-    ASSERT_EQ(values[i], varr->Value(i));
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+  ASSERT_OK(vb->Reserve(values.size()));
+
+  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+  for (int32_t value : values) {
+    vb->Append(value);
   }
+
+  Done();
+  ASSERT_RAISES(Invalid, result_->Validate());
 }
 
 TEST_F(TestListBuilder, TestZeroLength) {
   // All buffers are null
   Done();
+  ASSERT_OK(result_->Validate());
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index 23f12ddc4ecd7..fc3331139c6d8 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -14,23 +14,26 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-
 #include "arrow/types/list.h"
 
+#include <sstream>
+
 namespace arrow {
 
 bool ListArray::EqualsExact(const ListArray& other) const {
   if (this == &other) { return true; }
   if (null_count_ != other.null_count_) { return false; }
 
-  bool equal_offsets = offset_buf_->Equals(*other.offset_buf_, length_ + 1);
+  bool equal_offsets =
+      offset_buf_->Equals(*other.offset_buf_, (length_ + 1) * sizeof(int32_t));
+  if (!equal_offsets) { return false; }
   bool equal_null_bitmap = true;
   if (null_count_ > 0) {
     equal_null_bitmap =
         null_bitmap_->Equals(*other.null_bitmap_, util::bytes_for_bits(length_));
   }
 
-  if (!(equal_offsets && equal_null_bitmap)) { return false; }
+  if (!equal_null_bitmap) { return false; }
 
   return values()->Equals(other.values());
 }
@@ -41,4 +44,55 @@ bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
   return EqualsExact(*static_cast<const ListArray*>(arr.get()));
 }
 
+Status ListArray::Validate() const {
+  if (length_ < 0) { return Status::Invalid("Length was negative"); }
+  if (!offset_buf_) { return Status::Invalid("offset_buf_ was null"); }
+  if (offset_buf_->size() / sizeof(int32_t) < length_) {
+    std::stringstream ss;
+    ss << "offset buffer size (bytes): " << offset_buf_->size()
+       << " isn't large enough for length: " << length_;
+    return Status::Invalid(ss.str());
+  }
+  const int32_t last_offset = offset(length_);
+  if (last_offset > 0) {
+    if (!values_) {
+      return Status::Invalid("last offset was non-zero and values was null");
+    }
+    if (values_->length() != last_offset) {
+      std::stringstream ss;
+      ss << "Final offset invariant not equal to values length: " << last_offset
+         << "!=" << values_->length();
+      return Status::Invalid(ss.str());
+    }
+
+    const Status child_valid = values_->Validate();
+    if (!child_valid.ok()) {
+      std::stringstream ss;
+      ss << "Child array invalid: " << child_valid.ToString();
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  int32_t prev_offset = offset(0);
+  if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
+  for (int32_t i = 1; i <= length_; ++i) {
+    int32_t current_offset = offset(i);
+    if (IsNull(i - 1) && current_offset != prev_offset) {
+      std::stringstream ss;
+      ss << "Offset invariant failure at: " << i << " inconsistent offsets for null slot"
+         << current_offset << "!=" << prev_offset;
+      return Status::Invalid(ss.str());
+    }
+    if (current_offset < prev_offset) {
+      std::stringstream ss;
+      ss << "Offset invariant failure: " << i
+         << " inconsistent offset for non-null slot: " << current_offset << "<"
+         << prev_offset;
+      return Status::Invalid(ss.str());
+    }
+    prev_offset = current_offset;
+  }
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 6b815460ecb1e..e2302d917b8f6 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -28,6 +28,7 @@
 #include "arrow/types/primitive.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
@@ -46,11 +47,16 @@ class ListArray : public Array {
     values_ = values;
   }
 
-  virtual ~ListArray() {}
+  Status Validate() const override;
+
+  virtual ~ListArray() = default;
 
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   const std::shared_ptr<Array>& values() const { return values_; }
+  const std::shared_ptr<Buffer> offset_buffer() const {
+    return std::static_pointer_cast<Buffer>(offset_buf_);
+  }
 
   const std::shared_ptr<DataType>& value_type() const { return values_->type(); }
 
@@ -78,59 +84,73 @@ class ListArray : public Array {
 //
 // To use this class, you must append values to the child array builder and use
 // the Append function to delimit each distinct list value (once the values
-// have been appended to the child array)
-class ListBuilder : public Int32Builder {
+// have been appended to the child array) or use the bulk API to append
+// a sequence of offests and null values.
+//
+// A note on types.  Per arrow/type.h all types in the c++ implementation are
+// logical so even though this class always builds an Array of lists, this can
+// represent multiple different logical types.  If no logical type is provided
+// at construction time, the class defaults to List<T> where t is take from the
+// value_builder/values that the object is constructed with.
+class ListBuilder : public ArrayBuilder {
  public:
+  // Use this constructor to incrementally build the value array along with offsets and
+  // null bitmap.
+  ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
+      const TypePtr& type = nullptr)
+      : ArrayBuilder(
+            pool, type ? type : std::static_pointer_cast<DataType>(
+                                    std::make_shared<ListType>(value_builder->type()))),
+        offset_builder_(pool),
+        value_builder_(value_builder) {}
+
+  // Use this constructor to build the list with a pre-existing values array
   ListBuilder(
-      MemoryPool* pool, const TypePtr& type, std::shared_ptr<ArrayBuilder> value_builder)
-      : Int32Builder(pool, type), value_builder_(value_builder) {}
-
-  Status Init(int32_t elements) {
-    // One more than requested.
-    //
-    // XXX: This is slightly imprecise, because we might trigger null mask
-    // resizes that are unnecessary when creating arrays with power-of-two size
-    return Int32Builder::Init(elements + 1);
+      MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr)
+      : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
+                                             std::make_shared<ListType>(values->type()))),
+        offset_builder_(pool),
+        values_(values) {}
+
+  Status Init(int32_t elements) override {
+    RETURN_NOT_OK(ArrayBuilder::Init(elements));
+    // one more then requested for offsets
+    return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
   }
 
-  Status Resize(int32_t capacity) {
-    // Need space for the end offset
-    RETURN_NOT_OK(Int32Builder::Resize(capacity + 1));
-
-    // Slight hack, as the "real" capacity is one less
-    --capacity_;
-    return Status::OK();
+  Status Resize(int32_t capacity) override {
+    // one more then requested for offsets
+    RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
+    return ArrayBuilder::Resize(capacity);
   }
 
   // Vector append
   //
   // If passed, valid_bytes is of equal length to values, and any zero byte
   // will be considered as a null for that slot
-  Status Append(value_type* values, int32_t length, uint8_t* valid_bytes = nullptr) {
-    if (length_ + length > capacity_) {
-      int32_t new_capacity = util::next_power2(length_ + length);
-      RETURN_NOT_OK(Resize(new_capacity));
-    }
-    memcpy(raw_data_ + length_, values, type_traits<Int32Type>::bytes_required(length));
-
-    if (valid_bytes != nullptr) { AppendNulls(valid_bytes, length); }
-
-    length_ += length;
+  Status Append(
+      const int32_t* offsets, int32_t length, const uint8_t* valid_bytes = nullptr) {
+    RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(valid_bytes, length);
+    offset_builder_.UnsafeAppend<int32_t>(offsets, length);
     return Status::OK();
   }
 
+  // The same as Finalize but allows for overridding the c++ type
   template <typename Container>
   std::shared_ptr<Array> Transfer() {
-    std::shared_ptr<Array> items = value_builder_->Finish();
+    std::shared_ptr<Array> items = values_;
+    if (!items) { items = value_builder_->Finish(); }
 
-    // Add final offset if the length is non-zero
-    if (length_) { raw_data_[length_] = items->length(); }
+    offset_builder_.Append<int32_t>(items->length());
 
+    const auto offsets_buffer = offset_builder_.Finish();
     auto result = std::make_shared<Container>(
-        type_, length_, data_, items, null_count_, null_bitmap_);
+        type_, length_, offsets_buffer, items, null_count_, null_bitmap_);
 
-    data_ = null_bitmap_ = nullptr;
+    // TODO(emkornfield) make a reset method
     capacity_ = length_ = null_count_ = 0;
+    null_bitmap_ = nullptr;
 
     return result;
   }
@@ -141,26 +161,24 @@ class ListBuilder : public Int32Builder {
   //
   // This function should be called before beginning to append elements to the
   // value builder
-  Status Append(bool is_null = false) {
-    if (length_ == capacity_) {
-      // If the capacity was not already a multiple of 2, do so here
-      RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
-    }
-    if (is_null) {
-      ++null_count_;
-    } else {
-      util::set_bit(null_bitmap_data_, length_);
-    }
-    raw_data_[length_++] = value_builder_->length();
+  Status Append(bool is_valid = true) {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(is_valid);
+    RETURN_NOT_OK(offset_builder_.Append<int32_t>(value_builder_->length()));
     return Status::OK();
   }
 
-  Status AppendNull() { return Append(true); }
+  Status AppendNull() { return Append(false); }
 
-  const std::shared_ptr<ArrayBuilder>& value_builder() const { return value_builder_; }
+  const std::shared_ptr<ArrayBuilder>& value_builder() const {
+    DCHECK(!values_) << "Using value builder is pointless when values_ is set";
+    return value_builder_;
+  }
 
  protected:
+  BufferBuilder offset_builder_;
   std::shared_ptr<ArrayBuilder> value_builder_;
+  std::shared_ptr<Array> values_;
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 6bd9e73eb46ac..2b4c0879a28f4 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -102,7 +102,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     Attrs::draw(N, &draws_);
 
     valid_bytes_.resize(N);
-    test::random_null_bitmap(N, pct_null, valid_bytes_.data());
+    test::random_null_bytes(N, pct_null, valid_bytes_.data());
   }
 
   void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
@@ -193,8 +193,8 @@ void TestPrimitiveBuilder<PBoolean>::RandomData(int N, double pct_null) {
   draws_.resize(N);
   valid_bytes_.resize(N);
 
-  test::random_null_bitmap(N, 0.5, draws_.data());
-  test::random_null_bitmap(N, pct_null, valid_bytes_.data());
+  test::random_null_bytes(N, 0.5, draws_.data());
+  test::random_null_bytes(N, pct_null, valid_bytes_.data());
 }
 
 template <>
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 9549c47b41157..9102c530e25da 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -57,12 +57,14 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
     }
     return true;
   } else {
+    if (length_ == 0 && other.length_ == 0) { return true; }
     return data_->Equals(*other.data_, length_);
   }
 }
 
 bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
   if (this->type_enum() != arr->type_enum()) { return false; }
   return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
 }
@@ -101,48 +103,21 @@ Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
   return Status::OK();
 }
 
-template <typename T>
-Status PrimitiveBuilder<T>::Reserve(int32_t elements) {
-  if (length_ + elements > capacity_) {
-    int32_t new_capacity = util::next_power2(length_ + elements);
-    return Resize(new_capacity);
-  }
-  return Status::OK();
-}
-
 template <typename T>
 Status PrimitiveBuilder<T>::Append(
     const value_type* values, int32_t length, const uint8_t* valid_bytes) {
-  RETURN_NOT_OK(PrimitiveBuilder<T>::Reserve(length));
+  RETURN_NOT_OK(Reserve(length));
 
   if (length > 0) {
     memcpy(raw_data_ + length_, values, type_traits<T>::bytes_required(length));
   }
 
-  if (valid_bytes != nullptr) {
-    PrimitiveBuilder<T>::AppendNulls(valid_bytes, length);
-  } else {
-    for (int i = 0; i < length; ++i) {
-      util::set_bit(null_bitmap_data_, length_ + i);
-    }
-  }
+  // length_ is update by these
+  ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
 
-  length_ += length;
   return Status::OK();
 }
 
-template <typename T>
-void PrimitiveBuilder<T>::AppendNulls(const uint8_t* valid_bytes, int32_t length) {
-  // If valid_bytes is all not null, then none of the values are null
-  for (int i = 0; i < length; ++i) {
-    if (valid_bytes[i] == 0) {
-      ++null_count_;
-    } else {
-      util::set_bit(null_bitmap_data_, length_ + i);
-    }
-  }
-}
-
 template <typename T>
 std::shared_ptr<Array> PrimitiveBuilder<T>::Finish() {
   std::shared_ptr<Array> result = std::make_shared<typename type_traits<T>::ArrayType>(
@@ -166,14 +141,8 @@ Status PrimitiveBuilder<BooleanType>::Append(
     }
   }
 
-  if (valid_bytes != nullptr) {
-    PrimitiveBuilder<BooleanType>::AppendNulls(valid_bytes, length);
-  } else {
-    for (int i = 0; i < length; ++i) {
-      util::set_bit(null_bitmap_data_, length_ + i);
-    }
-  }
-  length_ += length;
+  // this updates length_
+  ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index fcd3db4e96e53..6f6b2fed5a320 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -95,15 +95,13 @@ class PrimitiveBuilder : public ArrayBuilder {
   using ArrayBuilder::Advance;
 
   // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
-  void AppendNulls(const uint8_t* valid_bytes, int32_t length);
+  void AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+    UnsafeAppendToBitmap(valid_bytes, length);
+  }
 
   Status AppendNull() {
-    if (length_ == capacity_) {
-      // If the capacity was not already a multiple of 2, do so here
-      RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
-    }
-    ++null_count_;
-    ++length_;
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(false);
     return Status::OK();
   }
 
@@ -116,21 +114,17 @@ class PrimitiveBuilder : public ArrayBuilder {
   Status Append(
       const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
 
-  // Ensure that builder can accommodate an additional number of
-  // elements. Resizes if the current capacity is not sufficient
-  Status Reserve(int32_t elements);
-
   std::shared_ptr<Array> Finish() override;
 
- protected:
-  std::shared_ptr<PoolBuffer> data_;
-  value_type* raw_data_;
-
-  Status Init(int32_t capacity);
+  Status Init(int32_t capacity) override;
 
   // Increase the capacity of the builder to accommodate at least the indicated
   // number of elements
-  Status Resize(int32_t capacity);
+  Status Resize(int32_t capacity) override;
+
+ protected:
+  std::shared_ptr<PoolBuffer> data_;
+  value_type* raw_data_;
 };
 
 template <typename T>
@@ -140,9 +134,17 @@ class NumericBuilder : public PrimitiveBuilder<T> {
   using PrimitiveBuilder<T>::PrimitiveBuilder;
 
   using PrimitiveBuilder<T>::Append;
+  using PrimitiveBuilder<T>::Init;
+  using PrimitiveBuilder<T>::Resize;
 
-  // Scalar append. Does not capacity-check; make sure to call Reserve beforehand
+  // Scalar append.
   void Append(value_type val) {
+    ArrayBuilder::Reserve(1);
+    UnsafeAppend(val);
+  }
+
+  // Does not capacity-check; make sure to call Reserve beforehand
+  void UnsafeAppend(value_type val) {
     util::set_bit(null_bitmap_data_, length_);
     raw_data_[length_++] = val;
   }
@@ -151,9 +153,6 @@ class NumericBuilder : public PrimitiveBuilder<T> {
   using PrimitiveBuilder<T>::length_;
   using PrimitiveBuilder<T>::null_bitmap_data_;
   using PrimitiveBuilder<T>::raw_data_;
-
-  using PrimitiveBuilder<T>::Init;
-  using PrimitiveBuilder<T>::Resize;
 };
 
 template <>
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index c5cbe1058c7cf..d2d3c5b6b5a83 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -89,11 +89,11 @@ class StringArray : public ListArray {
   const uint8_t* raw_bytes_;
 };
 
-// Array builder
+// String builder
 class StringBuilder : public ListBuilder {
  public:
   explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
-      : ListBuilder(pool, type, std::make_shared<UInt8Builder>(pool, value_type_)) {
+      : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, value_type_), type) {
     byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
   }
 
@@ -110,7 +110,6 @@ class StringBuilder : public ListBuilder {
   }
 
  protected:
-  std::shared_ptr<ListBuilder> list_builder_;
   UInt8Builder* byte_builder_;
 
   static TypePtr value_type_;
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 56532be8070ae..5ef0076953cea 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -23,6 +23,7 @@
 #include <cstring>
 #include <memory>
 
+#include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/status.h"
 
@@ -137,26 +138,64 @@ class BufferBuilder {
  public:
   explicit BufferBuilder(MemoryPool* pool) : pool_(pool), capacity_(0), size_(0) {}
 
+  Status Resize(int32_t elements) {
+    if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
+    capacity_ = elements;
+    RETURN_NOT_OK(buffer_->Resize(capacity_));
+    data_ = buffer_->mutable_data();
+    return Status::OK();
+  }
+
   Status Append(const uint8_t* data, int length) {
-    if (capacity_ < length + size_) {
-      if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
-      capacity_ = std::max(MIN_BUFFER_CAPACITY, capacity_);
-      while (capacity_ < length + size_) {
-        capacity_ *= 2;
-      }
-      RETURN_NOT_OK(buffer_->Resize(capacity_));
-      data_ = buffer_->mutable_data();
-    }
+    if (capacity_ < length + size_) { RETURN_NOT_OK(Resize(length + size_)); }
+    UnsafeAppend(data, length);
+    return Status::OK();
+  }
+
+  template <typename T>
+  Status Append(T arithmetic_value) {
+    static_assert(std::is_arithmetic<T>::value,
+        "Convenience buffer append only supports arithmetic types");
+    return Append(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
+  }
+
+  template <typename T>
+  Status Append(const T* arithmetic_values, int num_elements) {
+    static_assert(std::is_arithmetic<T>::value,
+        "Convenience buffer append only supports arithmetic types");
+    return Append(
+        reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
+  }
+
+  // Unsafe methods don't check existing size
+  void UnsafeAppend(const uint8_t* data, int length) {
     memcpy(data_ + size_, data, length);
     size_ += length;
-    return Status::OK();
+  }
+
+  template <typename T>
+  void UnsafeAppend(T arithmetic_value) {
+    static_assert(std::is_arithmetic<T>::value,
+        "Convenience buffer append only supports arithmetic types");
+    UnsafeAppend(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
+  }
+
+  template <typename T>
+  void UnsafeAppend(const T* arithmetic_values, int num_elements) {
+    static_assert(std::is_arithmetic<T>::value,
+        "Convenience buffer append only supports arithmetic types");
+    UnsafeAppend(
+        reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
   }
 
   std::shared_ptr<Buffer> Finish() {
     auto result = buffer_;
     buffer_ = nullptr;
+    capacity_ = size_ = 0;
     return result;
   }
+  int capacity() { return capacity_; }
+  int length() { return size_; }
 
  private:
   std::shared_ptr<PoolBuffer> buffer_;
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 527ce423e7751..fccc5e3085de5 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -18,8 +18,8 @@
 #ifndef ARROW_UTIL_LOGGING_H
 #define ARROW_UTIL_LOGGING_H
 
-#include <iostream>
 #include <cstdlib>
+#include <iostream>
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
index fb417e74daf53..961554fe06bcc 100644
--- a/cpp/src/arrow/util/memory-pool.cc
+++ b/cpp/src/arrow/util/memory-pool.cc
@@ -18,8 +18,8 @@
 #include "arrow/util/memory-pool.h"
 
 #include <cstdlib>
-#include <sstream>
 #include <mutex>
+#include <sstream>
 
 #include "arrow/util/status.h"
 
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index d608f8167df65..bf5a22089cdba 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -31,14 +31,15 @@ def test_getitem_NA(self):
         assert arr[1] is pyarrow.NA
 
     def test_list_format(self):
-        arr = pyarrow.from_pylist([[1], None, [2, 3]])
+        arr = pyarrow.from_pylist([[1], None, [2, 3, None]])
         result = fmt.array_format(arr)
         expected = """\
 [
   [1],
   NA,
   [2,
-   3]
+   3,
+   NA]
 ]"""
         assert result == expected
 

From a541644721ba4cb4723931b2a5eff1ac58c8aedd Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 23 Apr 2016 11:11:05 -0400
Subject: [PATCH 0063/1644] ARROW-100: [C++] Computing RowBatch size

Implement RowBatchWriter::DataHeaderSize and arrow::ipc::GetRowBatchSize. To achieve this, the Flatbuffer metadata is written to a temporary buffer and its size is determined. This commit also adds MockMemorySource, a new MemorySource that tracks the amount of memory written.

Author: Philipp Moritz <pcmoritz@gmail.com>

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #61 from pcmoritz/rowbatchsize and squashes the following commits:

e95fc5c [Philipp Moritz] fix formating
253c9f0 [Philipp Moritz] rename MockMemorySource methods to reflect better what they are doing
3484458 [Philipp Moritz] add tests for more datatypes
6b798f8 [Philipp Moritz] fix maximum recursion depth
67af8e1 [Philipp Moritz] merge GetRowBatchSize
9b69f12 [Philipp Moritz] factor out GetRowBatchSize test, use MockMemorySource to implement GetRowBatchSize, unify DataHeaderSize and TotalBytes into GetTotalSize
aa48cdf [Philipp Moritz] ARROW-100: [C++] Computing RowBatch size
---
 cpp/src/arrow/ipc/adapter.cc          | 29 ++++++++++++++-------------
 cpp/src/arrow/ipc/adapter.h           |  2 +-
 cpp/src/arrow/ipc/ipc-adapter-test.cc | 28 ++++++++++++++++++++++++++
 cpp/src/arrow/ipc/memory.cc           | 25 +++++++++++++++++++++++
 cpp/src/arrow/ipc/memory.h            | 22 ++++++++++++++++++++
 5 files changed, 91 insertions(+), 15 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index bf6fa94dea7a4..34700080746e7 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -179,20 +179,13 @@ class RowBatchWriter {
   }
 
   // This must be called after invoking AssemblePayload
-  int64_t DataHeaderSize() {
-    // TODO(wesm): In case it is needed, compute the upper bound for the size
-    // of the buffer containing the flatbuffer data header.
-    return 0;
-  }
-
-  // Total footprint of buffers. This must be called after invoking
-  // AssemblePayload
-  int64_t TotalBytes() {
-    int64_t total = 0;
-    for (const std::shared_ptr<Buffer>& buffer : buffers_) {
-      total += buffer->size();
-    }
-    return total;
+  Status GetTotalSize(int64_t* size) {
+    // emulates the behavior of Write without actually writing
+    int64_t data_header_offset;
+    MockMemorySource source(0);
+    RETURN_NOT_OK(Write(&source, 0, &data_header_offset));
+    *size = source.GetExtentBytesWritten();
+    return Status::OK();
   }
 
  private:
@@ -211,6 +204,14 @@ Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
   RETURN_NOT_OK(serializer.AssemblePayload());
   return serializer.Write(dst, position, header_offset);
 }
+
+Status GetRowBatchSize(const RowBatch* batch, int64_t* size) {
+  RowBatchWriter serializer(batch, kMaxIpcRecursionDepth);
+  RETURN_NOT_OK(serializer.AssemblePayload());
+  RETURN_NOT_OK(serializer.GetTotalSize(size));
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // Row batch read path
 
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 4c9a8a9d8ee39..0d2b77f5acefe 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -62,7 +62,7 @@ Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the row batch. This involves generating the complete serialized
 // Flatbuffers metadata.
-int64_t GetRowBatchSize(const RowBatch* batch);
+Status GetRowBatchSize(const RowBatch* batch, int64_t* size);
 
 // ----------------------------------------------------------------------
 // "Read" path; does not copy data if the MemorySource does not
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index c243cfba820cc..3b147343f772a 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -195,6 +195,34 @@ INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRowBatch,
     ::testing::Values(&MakeIntRowBatch, &MakeListRowBatch, &MakeNonNullRowBatch,
                             &MakeZeroLengthRowBatch, &MakeDeeplyNestedList));
 
+void TestGetRowBatchSize(std::shared_ptr<RowBatch> batch) {
+  MockMemorySource mock_source(1 << 16);
+  int64_t mock_header_location;
+  int64_t size;
+  ASSERT_OK(WriteRowBatch(&mock_source, batch.get(), 0, &mock_header_location));
+  ASSERT_OK(GetRowBatchSize(batch.get(), &size));
+  ASSERT_EQ(mock_source.GetExtentBytesWritten(), size);
+}
+
+TEST_F(TestWriteRowBatch, IntegerGetRowBatchSize) {
+  std::shared_ptr<RowBatch> batch;
+
+  ASSERT_OK(MakeIntRowBatch(&batch));
+  TestGetRowBatchSize(batch);
+
+  ASSERT_OK(MakeListRowBatch(&batch));
+  TestGetRowBatchSize(batch);
+
+  ASSERT_OK(MakeZeroLengthRowBatch(&batch));
+  TestGetRowBatchSize(batch);
+
+  ASSERT_OK(MakeNonNullRowBatch(&batch));
+  TestGetRowBatchSize(batch);
+
+  ASSERT_OK(MakeDeeplyNestedList(&batch));
+  TestGetRowBatchSize(batch);
+}
+
 class RecursionLimits : public ::testing::Test, public MemoryMapFixture {
  public:
   void SetUp() { pool_ = default_memory_pool(); }
diff --git a/cpp/src/arrow/ipc/memory.cc b/cpp/src/arrow/ipc/memory.cc
index 84cbc182cd26f..caff2c610b907 100644
--- a/cpp/src/arrow/ipc/memory.cc
+++ b/cpp/src/arrow/ipc/memory.cc
@@ -145,5 +145,30 @@ Status MemoryMappedSource::Write(int64_t position, const uint8_t* data, int64_t
   return Status::OK();
 }
 
+MockMemorySource::MockMemorySource(int64_t size)
+    : size_(size), extent_bytes_written_(0) {}
+
+Status MockMemorySource::Close() {
+  return Status::OK();
+}
+
+Status MockMemorySource::ReadAt(
+    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  return Status::OK();
+}
+
+Status MockMemorySource::Write(int64_t position, const uint8_t* data, int64_t nbytes) {
+  extent_bytes_written_ = std::max(extent_bytes_written_, position + nbytes);
+  return Status::OK();
+}
+
+int64_t MockMemorySource::Size() const {
+  return size_;
+}
+
+int64_t MockMemorySource::GetExtentBytesWritten() const {
+  return extent_bytes_written_;
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/memory.h b/cpp/src/arrow/ipc/memory.h
index e529603dc6e2a..c6fd7a718991b 100644
--- a/cpp/src/arrow/ipc/memory.h
+++ b/cpp/src/arrow/ipc/memory.h
@@ -121,6 +121,28 @@ class MemoryMappedSource : public MemorySource {
   std::unique_ptr<Impl> impl_;
 };
 
+// A MemorySource that tracks the size of allocations from a memory source
+class MockMemorySource : public MemorySource {
+ public:
+  explicit MockMemorySource(int64_t size);
+
+  Status Close() override;
+
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
+  Status Write(int64_t position, const uint8_t* data, int64_t nbytes) override;
+
+  int64_t Size() const override;
+
+  // @return: the smallest number of bytes containing the modified region of the
+  // MockMemorySource
+  int64_t GetExtentBytesWritten() const;
+
+ private:
+  int64_t size_;
+  int64_t extent_bytes_written_;
+};
+
 }  // namespace ipc
 }  // namespace arrow
 

From 56514d93a2d1c5ad9419c807f23127eb07d9ccfe Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Fri, 29 Apr 2016 19:31:25 -0700
Subject: [PATCH 0064/1644] ARROW-104: [FORMAT] Add alignment and padding
 requirements + union clarification

I believe this change captures the discussion we had on the mailing list about alignment and padding for arrays.  It also captures the update to UnionArrays.   The rendered version should be viewable here: https://github.com/emkornfield/arrow/blob/emk_format_changes/format/Layout.md

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #67 from emkornfield/emk_format_changes and squashes the following commits:

c91421e [Micah Kornfield] fixes per code review
b33d4c2 [Micah Kornfield] Add alignment and padding requirements.  update union types buffer to reflect using only 1 type buffer
---
 format/Layout.md | 165 +++++++++++++++++++++++++++--------------------
 1 file changed, 95 insertions(+), 70 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index 92553d944c2d1..34eade313415a 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -10,6 +10,8 @@ concepts, here is a small glossary to help disambiguate.
 * Contiguous memory region: a sequential virtual address space with a given
   length. Any byte can be reached via a single pointer offset less than the
   region's length.
+* Contiguous memory buffer: A contiguous memory region that stores
+  a multi-value component of an Array.  Sometimes referred to as just "buffer".
 * Primitive type: a data type that occupies a fixed-size memory slot specified
   in bit width or byte width
 * Nested or parametric type: a data type whose full structure depends on one or
@@ -41,7 +43,7 @@ Base requirements
   linearly in the nesting level
 * Capable of representing fully-materialized and decoded / decompressed Parquet
   data
-* All leaf nodes (primitive value arrays) use contiguous memory regions
+* All contiguous memory buffers are aligned at 64-byte boundaries and padded to a multiple of 64 bytes.
 * Any relative type can have null slots
 * Arrays are immutable once created. Implementations can provide APIs to mutate
   an array, but applying mutations will require a new array data structure to
@@ -78,6 +80,28 @@ Base requirements
 
 The Arrow format is little endian.
 
+## Alignment and Padding
+
+As noted above, all buffers are intended to be aligned in memory at 64 byte
+boundaries and padded to a length that is a multiple of 64 bytes.  The alignment
+requirement follows best practices for optimized memory access:
+
+* Elements in numeric arrays will be guaranteed to be retrieved via aligned access.
+* On some architectures alignment can help limit partially used cache lines.
+* 64 byte alignment is recommended by the [Intel performance guide][2] for
+data-structures over 64 bytes (which will be a common case for Arrow Arrays).
+
+Requiring padding to a multiple of 64 bytes allows for using SIMD instructions
+consistently in loops without additional conditional checks.
+This should allow for simpler and more efficient code.  
+The specific padding length was chosen because it matches the largest known
+SIMD instruction registers available as of April 2016 (Intel AVX-512).
+Guaranteed padding can also allow certain compilers
+to generate more optimized code directly (e.g. One can safely use Intel's
+`-qopt-assume-safe-padding`).
+
+Unless otherwise noted, padded bytes do not need to have a specific value.
+
 ## Array lengths
 
 Any array has a known and fixed length, stored as a 32-bit signed integer, so a
@@ -101,14 +125,14 @@ signed integer, as it may be as large as the array length.
 Any relative type can have null value slots, whether primitive or nested type.
 
 An array with nulls must have a contiguous memory buffer, known as the null (or
-validity) bitmap, whose length is a multiple of 8 bytes (to avoid
-word-alignment concerns) and large enough to have at least 1 bit for each array
+validity) bitmap, whose length is a multiple of 64 bytes (as discussed above)  
+and large enough to have at least 1 bit for each array
 slot.
 
 Whether any array slot is valid (non-null) is encoded in the respective bits of
 this bitmap. A 1 (set bit) for index `j` indicates that the value is not null,
 while a 0 (bit not set) indicates that it is null. Bitmaps are to be
-initialized to be all unset at allocation time.
+initialized to be all unset at allocation time (this includes padding).
 
 ```
 is_valid[j] -> bitmap[j / 8] & (1 << (j % 8))
@@ -158,15 +182,15 @@ Would look like:
 * Length: 5, Null count: 1
 * Null bitmap buffer:
 
-  |Byte 0 (validity bitmap) | Bytes 1-7             |
+  |Byte 0 (validity bitmap) | Bytes 1-63            |
   |-------------------------|-----------------------|
   |00011011                 | 0 (padding)           |
 
 * Value Buffer:
 
-  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 |
-  |------------|-------------|-------------|-------------|-------------|
-  | 1          | 2           | unspecified | 4           | 8           |
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-63 |
+  |------------|-------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | unspecified | 4           | 8           | unspecified |
 ```
 
 ### Example Layout: Non-null int32 Array
@@ -177,15 +201,15 @@ Would look like:
 * Length: 5, Null count: 0
 * Null bitmap buffer:
 
-  | Byte 0 (validity bitmap) | Bytes 1-7             |
+  | Byte 0 (validity bitmap) | Bytes 1-63            |
   |--------------------------|-----------------------|
   | 00011111                 | 0 (padding)           |
 
 * Value Buffer:
 
-  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | bytes 12-15 | bytes 16-19 |
-  |------------|-------------|-------------|-------------|-------------|
-  | 1          | 2           | 3           | 4           | 8           |
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | bytes 12-15 | bytes 16-19 | Bytes 20-63 |
+  |------------|-------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | 3           | 4           | 8           | unspecified |
 ```
 
 or with the bitmap elided:
@@ -195,9 +219,9 @@ or with the bitmap elided:
 * Null bitmap buffer: Not required
 * Value Buffer:
 
-  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | bytes 12-15 | bytes 16-19 |
-  |------------|-------------|-------------|-------------|-------------|
-  | 1          | 2           | 3           | 4           | 8           |
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | bytes 12-15 | bytes 16-19 | Bytes 20-63 |
+  |------------|-------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | 3           | 4           | 8           | unspecified |
 ```
 
 ## List type
@@ -243,23 +267,23 @@ will have the following representation:
 * Length: 4, Null count: 1
 * Null bitmap buffer:
 
-  | Byte 0 (validity bitmap) | Bytes 1-7             |
+  | Byte 0 (validity bitmap) | Bytes 1-63            |
   |--------------------------|-----------------------|
   | 00001101                 | 0 (padding)           |
 
 * Offsets buffer (int32)
 
-  | Bytes 0-3  | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 |
-  |------------|-------------|-------------|-------------|-------------|
-  | 0          | 3           | 3           | 7           | 7           |
+  | Bytes 0-3  | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-63 |
+  |------------|-------------|-------------|-------------|-------------|-------------|
+  | 0          | 3           | 3           | 7           | 7           | unspecified |
 
 * Values array (char array):
   * Length: 7,  Null count: 0
   * Null bitmap buffer: Not required
 
-    | Bytes 0-7  |
-    |------------|
-    | joemark    |
+    | Bytes 0-7  | Bytes 8-63  |
+    |------------|-------------|
+    | joemark    | unspecified |
 ```
 
 ### Example Layout: `List<List<byte>>`
@@ -273,31 +297,31 @@ will be be represented as follows:
 * Null bitmap buffer: Not required
 * Offsets buffer (int32)
 
-  | Bytes 0-3  | Bytes 4-7  | Bytes 8-11 | Bytes 12-15 |
-  |------------|------------|------------|-------------|
-  | 0          |  2         |  6         |  7          |
+  | Bytes 0-3  | Bytes 4-7  | Bytes 8-11 | Bytes 12-15 | Bytes 16-63 |
+  |------------|------------|------------|-------------|-------------|
+  | 0          |  2         |  6         |  7          | unspecified |
 
 * Values array (`List<byte>`)
   * Length: 6, Null count: 1
   * Null bitmap buffer:
 
-    | Byte 0 (validity bitmap) | Bytes 1-7   |
+    | Byte 0 (validity bitmap) | Bytes 1-63  |
     |--------------------------|-------------|
     | 00110111                 | 0 (padding) |
 
   * Offsets buffer (int32)
 
-    | Bytes 0-28           |
-    |----------------------|
-    | 0, 2, 4, 7, 7, 8, 10 |
+    | Bytes 0-28           | Bytes 29-63 |
+    |----------------------|-------------|
+    | 0, 2, 4, 7, 7, 8, 10 | unspecified |
 
   * Values array (bytes):
     * Length: 10, Null count: 0
     * Null bitmap buffer: Not required
 
-      | Bytes 0-9                     |
-      |-------------------------------|
-      | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 |
+      | Bytes 0-9                     | Bytes 10-63 |
+      |-------------------------------|-------------|
+      | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 | unspecified |
 ```
 
 ## Struct type
@@ -333,9 +357,9 @@ The layout for [{'joe', 1}, {null, 2}, null, {'mark', 4}] would be:
 * Length: 4, Null count: 1
 * Null bitmap buffer:
 
-  | Byte 0 (validity bitmap) | Bytes 1-7   |
-  |--------------------------|-------------|
-  | 00001011                 | 0 (padding) |
+  | Byte 0 (validity bitmap) | Bytes 1-7   | Bytes 8-63  |
+  |--------------------------|-------------|-------------|
+  | 00001011                 | 0 (padding) | unspecified |
 
 * Children arrays:
   * field-0 array (`List<char>`):
@@ -396,13 +420,13 @@ The union types may be named, but like structs this will be a matter of the
 metadata and will not affect the physical memory layout.
 
 We define two distinct union types that are optimized for different use
-cases. This first, the dense union, represents a mixed-type array with 6 bytes
+cases. This first, the dense union, represents a mixed-type array with 5 bytes
 of overhead for each value. Its physical layout is as follows:
 
 * One child array for each relative type
-* Types buffer: A buffer of unsigned integers, enumerated from 0 corresponding
-  to each type, with the smallest byte width capable of representing the number
-  of types in the union.
+* Types buffer: A buffer of 8-bit signed integers, enumerated from 0 corresponding
+  to each type.  A union with more then 127 possible types can be modeled as a
+  union of unions. 
 * Offsets buffer: A buffer of signed int32 values indicating the relative offset
   into the respective child array for the type in a given slot. The respective
   offsets for each child value array must be in order / increasing.
@@ -420,21 +444,21 @@ An example layout for logical union of:
 ```
 * Length: 4, Null count: 1
 * Null bitmap buffer:
-  |Byte 0 (validity bitmap) | Bytes 1-7             |
+  |Byte 0 (validity bitmap) | Bytes 1-63            |
   |-------------------------|-----------------------|
   |00001101                 | 0 (padding)           |
 
 * Types buffer:
 
-  |Byte 0-1 | Byte 2-3    | Byte 4-5 | Byte 6-7 |
-  |---------|-------------|----------|----------|
-  | 0       | unspecified | 0        | 1        |
+  |Byte 0   | Byte 1      | Byte 2   | Byte 3   | Bytes 4-63  |
+  |---------|-------------|----------|----------|-------------|
+  | 0       | unspecified | 0        | 1        | unspecified |
 
 * Offset buffer:
 
-  |Byte 0-3 | Byte 4-7    | Byte 8-11 | Byte 12-15 |
-  |---------|-------------|-----------|------------|
-  | 0       | unspecified | 1         | 0          |
+  |Byte 0-3 | Byte 4-7    | Byte 8-11 | Byte 12-15 | Bytes 16-63 |
+  |---------|-------------|-----------|------------|-------------|
+  | 0       | unspecified | 1         | 0          | unspecified |
 
 * Children arrays:
   * Field-0 array (f: float):
@@ -443,9 +467,9 @@ An example layout for logical union of:
 
     * Value Buffer:
 
-      | Bytes 0-7 |
-      |-----------|
-      | 1.2, 3.4  |
+      | Bytes 0-7 | Bytes 8-63  |
+      |-----------|-------------|
+      | 1.2, 3.4  | unspecified |
 
 
   * Field-1 array (f: float):
@@ -454,9 +478,9 @@ An example layout for logical union of:
 
     * Value Buffer:
 
-      | Bytes 0-3 |
-      |-----------|
-      | 5         |
+      | Bytes 0-3 | Bytes 4-63  |
+      |-----------|-------------|
+      | 5         | unspecified |
 ```
 
 ## Sparse union type
@@ -484,9 +508,9 @@ will have the following layout:
 
 * Types buffer:
 
- | Bytes 0-1  | Bytes 2-3   | Bytes 4-5   | Bytes 6-7   | Bytes 8-9   | Bytes 10-11  |
- |------------|-------------|-------------|-------------|-------------|--------------|
- | 0          | 1           | 2           | 1           | 0           | 2            |
+ | Byte 0     | Byte 1      | Byte 2      | Byte 3      | Byte 4      | Byte 5       | Bytes  6-63           |
+ |------------|-------------|-------------|-------------|-------------|--------------|-----------------------|
+ | 0          | 1           | 2           | 1           | 0           | 2            | unspecified (padding) |
 
 * Children arrays:
 
@@ -494,51 +518,51 @@ will have the following layout:
     * Length: 6, Null count: 4
     * Null bitmap buffer:
 
-      |Byte 0 (validity bitmap) | Bytes 1-7             |
+      |Byte 0 (validity bitmap) | Bytes 1-63            |
       |-------------------------|-----------------------|
       |00010001                 | 0 (padding)           |
 
     * Value buffer:
 
-      |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  |
-      |------------|-------------|-------------|-------------|-------------|--------------|
-      | 1          | unspecified | unspecified | unspecified | 4           |  unspecified |
+      |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  | Bytes 24-63           |
+      |------------|-------------|-------------|-------------|-------------|--------------|-----------------------|
+      | 1          | unspecified | unspecified | unspecified | 4           |  unspecified | unspecified (padding) |
 
   * u1 (float):
     * Length: 6, Null count: 4
     * Null bitmap buffer:
 
-      |Byte 0 (validity bitmap) | Bytes 1-7             |
+      |Byte 0 (validity bitmap) | Bytes 1-63            |
       |-------------------------|-----------------------|
       |00001010                 | 0 (padding)           |
 
     * Value buffer:
 
-      |Bytes 0-3    | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  |
-      |-------------|-------------|-------------|-------------|-------------|--------------|
-      | unspecified |  1.2        | unspecified | 3.4         | unspecified |  unspecified |
+      |Bytes 0-3    | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  | Bytes 24-63           |
+      |-------------|-------------|-------------|-------------|-------------|--------------|-----------------------|
+      | unspecified |  1.2        | unspecified | 3.4         | unspecified |  unspecified | unspecified (padding) |
 
   * u2 (`List<char>`)
     * Length: 6, Null count: 4
     * Null bitmap buffer:
 
-      | Byte 0 (validity bitmap) | Bytes 1-7             |
+      | Byte 0 (validity bitmap) | Bytes 1-63            |
       |--------------------------|-----------------------|
       | 00100100                 | 0 (padding)           |
 
     * Offsets buffer (int32)
 
-      | Bytes 0-3  | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-27 |
-      |------------|-------------|-------------|-------------|-------------|-------------|-------------|
-      | 0          | 0           | 0           | 3           | 3           | 3           | 7           |
+      | Bytes 0-3  | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-27 | Bytes 28-63 |
+      |------------|-------------|-------------|-------------|-------------|-------------|-------------|-------------|
+      | 0          | 0           | 0           | 3           | 3           | 3           | 7           | unspecified |
 
     * Values array (char array):
       * Length: 7,  Null count: 0
       * Null bitmap buffer: Not required
 
-        | Bytes 0-7  |
-        |------------|
-        | joemark    |
+        | Bytes 0-7  | Bytes 8-63            |
+        |------------|-----------------------|
+        | joemark    | unspecified (padding) |
 ```
 
 Note that nested types in a sparse union must be internally consistent
@@ -557,3 +581,4 @@ the the types array indicates that a slot contains a different type at the index
 Drill docs https://drill.apache.org/docs/value-vectors/
 
 [1]: https://en.wikipedia.org/wiki/Bit_numbering
+[2]: https://software.intel.com/en-us/articles/practical-intel-avx-optimization-on-2nd-generation-intel-core-processors

From 355f7c96a194c65bad523466586f51a9ae0e8627 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 1 May 2016 15:53:37 -0700
Subject: [PATCH 0065/1644] ARROW-92: Arrow to Parquet Schema conversion

My current WIP state. To make the actual schema conversion complete, we probably need the physical structure too as Arrow schemas only care about logical types whereas Parquet schema is about logical and physical types.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #68 from xhochy/arrow-92 and squashes the following commits:

e3aa261 [Uwe L. Korn] Add macro to convert ParquetException to Status
9c5b085 [Uwe L. Korn] Include string
42ed0ea [Uwe L. Korn] Add struct conversion
38e68e5 [Uwe L. Korn] make format
9a6c876 [Uwe L. Korn] Add more types
8a0293e [Uwe L. Korn] ARROW-92: Arrow to Parquet Schema conversion
---
 cpp/src/arrow/parquet/parquet-schema-test.cc |  75 +++++++++++
 cpp/src/arrow/parquet/schema.cc              | 130 +++++++++++++++++++
 cpp/src/arrow/parquet/schema.h               |   5 +
 3 files changed, 210 insertions(+)

diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
index e2280f41189ef..8de739491b56f 100644
--- a/cpp/src/arrow/parquet/parquet-schema-test.cc
+++ b/cpp/src/arrow/parquet/parquet-schema-test.cc
@@ -161,6 +161,81 @@ TEST_F(TestConvertParquetSchema, UnsupportedThings) {
   }
 }
 
+class TestConvertArrowSchema : public ::testing::Test {
+ public:
+  virtual void SetUp() {}
+
+  void CheckFlatSchema(const std::vector<NodePtr>& nodes) {
+    NodePtr schema_node = GroupNode::Make("schema", Repetition::REPEATED, nodes);
+    const GroupNode* expected_schema_node =
+        static_cast<const GroupNode*>(schema_node.get());
+    const GroupNode* result_schema_node =
+        static_cast<const GroupNode*>(result_schema_->schema().get());
+
+    ASSERT_EQ(expected_schema_node->field_count(), result_schema_node->field_count());
+
+    for (int i = 0; i < expected_schema_node->field_count(); i++) {
+      auto lhs = result_schema_node->field(i);
+      auto rhs = expected_schema_node->field(i);
+      EXPECT_TRUE(lhs->Equals(rhs.get()));
+    }
+  }
+
+  Status ConvertSchema(const std::vector<std::shared_ptr<Field>>& fields) {
+    arrow_schema_ = std::make_shared<Schema>(fields);
+    return ToParquetSchema(arrow_schema_.get(), &result_schema_);
+  }
+
+ protected:
+  std::shared_ptr<Schema> arrow_schema_;
+  std::shared_ptr<::parquet::SchemaDescriptor> result_schema_;
+};
+
+TEST_F(TestConvertArrowSchema, ParquetFlatPrimitives) {
+  std::vector<NodePtr> parquet_fields;
+  std::vector<std::shared_ptr<Field>> arrow_fields;
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("boolean", Repetition::REQUIRED, ParquetType::BOOLEAN));
+  arrow_fields.push_back(std::make_shared<Field>("boolean", BOOL, false));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
+  arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
+  arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
+  arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
+
+  parquet_fields.push_back(
+      PrimitiveNode::Make("double", Repetition::OPTIONAL, ParquetType::DOUBLE));
+  arrow_fields.push_back(std::make_shared<Field>("double", DOUBLE));
+
+  // TODO: String types need to be clarified a bit more in the Arrow spec
+  parquet_fields.push_back(PrimitiveNode::Make(
+      "string", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::UTF8));
+  arrow_fields.push_back(std::make_shared<Field>("string", UTF8));
+
+  ASSERT_OK(ConvertSchema(arrow_fields));
+
+  CheckFlatSchema(parquet_fields);
+}
+
+TEST_F(TestConvertArrowSchema, ParquetFlatDecimals) {
+  std::vector<NodePtr> parquet_fields;
+  std::vector<std::shared_ptr<Field>> arrow_fields;
+
+  // TODO: Test Decimal Arrow -> Parquet conversion
+
+  ASSERT_OK(ConvertSchema(arrow_fields));
+
+  CheckFlatSchema(parquet_fields);
+}
+
 TEST(TestNodeConversion, DateAndTime) {}
 
 }  // namespace parquet
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index 560e28374066b..214c764f08b6e 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -17,13 +17,18 @@
 
 #include "arrow/parquet/schema.h"
 
+#include <string>
 #include <vector>
 
 #include "parquet/api/schema.h"
+#include "parquet/exception.h"
 
 #include "arrow/types/decimal.h"
+#include "arrow/types/string.h"
 #include "arrow/util/status.h"
 
+using parquet::ParquetException;
+using parquet::Repetition;
 using parquet::schema::Node;
 using parquet::schema::NodePtr;
 using parquet::schema::GroupNode;
@@ -36,6 +41,11 @@ namespace arrow {
 
 namespace parquet {
 
+#define PARQUET_CATCH_NOT_OK(s) \
+  try {                         \
+    (s);                        \
+  } catch (const ParquetException& e) { return Status::Invalid(e.what()); }
+
 const auto BOOL = std::make_shared<BooleanType>();
 const auto UINT8 = std::make_shared<UInt8Type>();
 const auto INT32 = std::make_shared<Int32Type>();
@@ -182,6 +192,126 @@ Status FromParquetSchema(
   return Status::OK();
 }
 
+Status StructToNode(const std::shared_ptr<StructType>& type, const std::string& name,
+    bool nullable, NodePtr* out) {
+  Repetition::type repetition = Repetition::REQUIRED;
+  if (nullable) { repetition = Repetition::OPTIONAL; }
+
+  std::vector<NodePtr> children(type->num_children());
+  for (int i = 0; i < type->num_children(); i++) {
+    RETURN_NOT_OK(FieldToNode(type->child(i), &children[i]));
+  }
+
+  *out = GroupNode::Make(name, repetition, children);
+  return Status::OK();
+}
+
+Status FieldToNode(const std::shared_ptr<Field>& field, NodePtr* out) {
+  LogicalType::type logical_type = LogicalType::NONE;
+  ParquetType::type type;
+  Repetition::type repetition = Repetition::REQUIRED;
+  if (field->nullable) { repetition = Repetition::OPTIONAL; }
+  int length = -1;
+
+  switch (field->type->type) {
+    // TODO:
+    // case Type::NA:
+    // break;
+    case Type::BOOL:
+      type = ParquetType::BOOLEAN;
+      break;
+    case Type::UINT8:
+      type = ParquetType::INT32;
+      logical_type = LogicalType::UINT_8;
+      break;
+    case Type::INT8:
+      type = ParquetType::INT32;
+      logical_type = LogicalType::INT_8;
+      break;
+    case Type::UINT16:
+      type = ParquetType::INT32;
+      logical_type = LogicalType::UINT_16;
+      break;
+    case Type::INT16:
+      type = ParquetType::INT32;
+      logical_type = LogicalType::INT_16;
+      break;
+    case Type::UINT32:
+      type = ParquetType::INT32;
+      logical_type = LogicalType::UINT_32;
+      break;
+    case Type::INT32:
+      type = ParquetType::INT32;
+      break;
+    case Type::UINT64:
+      type = ParquetType::INT64;
+      logical_type = LogicalType::UINT_64;
+      break;
+    case Type::INT64:
+      type = ParquetType::INT64;
+      break;
+    case Type::FLOAT:
+      type = ParquetType::FLOAT;
+      break;
+    case Type::DOUBLE:
+      type = ParquetType::DOUBLE;
+      break;
+    case Type::CHAR:
+      type = ParquetType::FIXED_LEN_BYTE_ARRAY;
+      logical_type = LogicalType::UTF8;
+      length = static_cast<CharType*>(field->type.get())->size;
+      break;
+    case Type::STRING:
+      type = ParquetType::BYTE_ARRAY;
+      logical_type = LogicalType::UTF8;
+      break;
+    case Type::BINARY:
+      type = ParquetType::BYTE_ARRAY;
+      break;
+    case Type::DATE:
+      type = ParquetType::INT32;
+      logical_type = LogicalType::DATE;
+      break;
+    case Type::TIMESTAMP:
+      type = ParquetType::INT64;
+      logical_type = LogicalType::TIMESTAMP_MILLIS;
+      break;
+    case Type::TIMESTAMP_DOUBLE:
+      type = ParquetType::INT64;
+      // This is specified as seconds since the UNIX epoch
+      // TODO: Converted type in Parquet?
+      // logical_type = LogicalType::TIMESTAMP_MILLIS;
+      break;
+    case Type::TIME:
+      type = ParquetType::INT64;
+      logical_type = LogicalType::TIME_MILLIS;
+      break;
+    case Type::STRUCT: {
+      auto struct_type = std::static_pointer_cast<StructType>(field->type);
+      return StructToNode(struct_type, field->name, field->nullable, out);
+    } break;
+    default:
+      // TODO: LIST, DENSE_UNION, SPARE_UNION, JSON_SCALAR, DECIMAL, DECIMAL_TEXT, VARCHAR
+      return Status::NotImplemented("unhandled type");
+  }
+  *out = PrimitiveNode::Make(field->name, repetition, type, logical_type, length);
+  return Status::OK();
+}
+
+Status ToParquetSchema(
+    const Schema* arrow_schema, std::shared_ptr<::parquet::SchemaDescriptor>* out) {
+  std::vector<NodePtr> nodes(arrow_schema->num_fields());
+  for (int i = 0; i < arrow_schema->num_fields(); i++) {
+    RETURN_NOT_OK(FieldToNode(arrow_schema->field(i), &nodes[i]));
+  }
+
+  NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, nodes);
+  *out = std::make_shared<::parquet::SchemaDescriptor>();
+  PARQUET_CATCH_NOT_OK((*out)->Init(schema));
+
+  return Status::OK();
+}
+
 }  // namespace parquet
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h
index a44a9a4b6a892..bfc7d21138154 100644
--- a/cpp/src/arrow/parquet/schema.h
+++ b/cpp/src/arrow/parquet/schema.h
@@ -36,6 +36,11 @@ Status NodeToField(const ::parquet::schema::NodePtr& node, std::shared_ptr<Field
 Status FromParquetSchema(
     const ::parquet::SchemaDescriptor* parquet_schema, std::shared_ptr<Schema>* out);
 
+Status FieldToNode(const std::shared_ptr<Field>& field, ::parquet::schema::NodePtr* out);
+
+Status ToParquetSchema(
+    const Schema* arrow_schema, std::shared_ptr<::parquet::SchemaDescriptor>* out);
+
 }  // namespace parquet
 
 }  // namespace arrow

From ad3d01dd5c47f6d21771a53d437772cf71bee10f Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 3 May 2016 18:23:43 -0700
Subject: [PATCH 0066/1644] ARROW-188: Add numpy as install requirement

Successfully tested with NumPy 1.9 which should be a recent but still old version that we can support for now.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #69 from xhochy/arrow-188 and squashes the following commits:

651a9aa [Uwe L. Korn] ARROW-188: Add numpy as install requirement
---
 python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index ebd80de46b4da..5f228ed0af245 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -242,7 +242,7 @@ def get_outputs(self):
         'clean': clean,
         'build_ext': build_ext
     },
-    install_requires=['cython >= 0.21'],
+    install_requires=['cython >= 0.21', 'numpy >= 1.9'],
     description=DESC,
     license='Apache License, Version 2.0',
     maintainer="Apache Arrow Developers",

From 33022579e31b2448ed227ddf51160d08edd625e3 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 8 May 2016 18:03:28 -0700
Subject: [PATCH 0067/1644] ARROW-190: Python: Provide installable sdist builds

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #71 from xhochy/arrow-190 and squashes the following commits:

e28db45 [Uwe L. Korn] Add LICENSE and README to MANIFEST
f9943f5 [Uwe L. Korn] ARROW-190: Python: Provide standalone installable sdist builds
---
 python/MANIFEST.in | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 python/MANIFEST.in

diff --git a/python/MANIFEST.in b/python/MANIFEST.in
new file mode 100644
index 0000000000000..756879a0bb033
--- /dev/null
+++ b/python/MANIFEST.in
@@ -0,0 +1,14 @@
+include README.md
+include LICENSE.txt
+
+global-include CMakeLists.txt
+graft cmake_modules
+recursive-include src/pyarrow *.cc *.h
+recursive-include pyarrow *.pxd
+
+global-exclude *.so
+global-exclude *.pyc
+global-exclude *~
+global-exclude \#*
+global-exclude .git*
+global-exclude .DS_Store

From c9ffe546b8ddb81851bcff78e4db051942dcc546 Mon Sep 17 00:00:00 2001
From: Jihoon Son <jihoonson@apache.org>
Date: Sun, 8 May 2016 22:15:40 -0700
Subject: [PATCH 0068/1644] ARROW-194: C++: Allow read-only memory mapped
 source

A simple patch to allow read-only mode. A test is also included.

Author: Jihoon Son <jihoonson@apache.org>

Closes #72 from jihoonson/ARROW-194 and squashes the following commits:

f55dd22 [Jihoon Son] Change the type of protection flag from int8_t to int
b928031 [Jihoon Son] Add missing initialization
63b99c5 [Jihoon Son] Remove unintended whitespace
22e6128 [Jihoon Son] Simplify error check
5559b8d [Jihoon Son] - Fixed a wrong protection flag in a test - Added a routine to check the protection flag before writing - Added a unit test to check the error status for protection mode - Improved failure check for mmap()
d8939fa [Jihoon Son] Allow read-only memory mapped source.
---
 cpp/src/arrow/ipc/ipc-memory-test.cc | 54 ++++++++++++++++++++++++++--
 cpp/src/arrow/ipc/memory.cc          | 22 ++++++++----
 2 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-memory-test.cc b/cpp/src/arrow/ipc/ipc-memory-test.cc
index 1933921222595..a2dbd35728c49 100644
--- a/cpp/src/arrow/ipc/ipc-memory-test.cc
+++ b/cpp/src/arrow/ipc/ipc-memory-test.cc
@@ -26,9 +26,6 @@
 
 #include "arrow/ipc/memory.h"
 #include "arrow/ipc/test-common.h"
-#include "arrow/test-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 namespace ipc {
@@ -67,6 +64,57 @@ TEST_F(TestMemoryMappedSource, WriteRead) {
   }
 }
 
+TEST_F(TestMemoryMappedSource, ReadOnly) {
+  const int64_t buffer_size = 1024;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(1024, 0, buffer.data());
+
+  const int reps = 5;
+
+  std::string path = "ipc-read-only-test";
+  CreateFile(path, reps * buffer_size);
+
+  std::shared_ptr<MemoryMappedSource> rwmmap;
+  ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_WRITE, &rwmmap));
+
+  int64_t position = 0;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(rwmmap->Write(position, buffer.data(), buffer_size));
+
+    position += buffer_size;
+  }
+  rwmmap->Close();
+
+  std::shared_ptr<MemoryMappedSource> rommap;
+  ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_ONLY, &rommap));
+
+  position = 0;
+  std::shared_ptr<Buffer> out_buffer;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(rommap->ReadAt(position, buffer_size, &out_buffer));
+
+    ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
+    position += buffer_size;
+  }
+  rommap->Close();
+}
+
+TEST_F(TestMemoryMappedSource, InvalidMode) {
+  const int64_t buffer_size = 1024;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(1024, 0, buffer.data());
+
+  std::string path = "ipc-invalid-mode-test";
+  CreateFile(path, buffer_size);
+
+  std::shared_ptr<MemoryMappedSource> rommap;
+  ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_ONLY, &rommap));
+
+  ASSERT_RAISES(IOError, rommap->Write(0, buffer.data(), buffer_size));
+}
+
 TEST_F(TestMemoryMappedSource, InvalidFile) {
   std::string non_existent_path = "invalid-file-name-asfd";
 
diff --git a/cpp/src/arrow/ipc/memory.cc b/cpp/src/arrow/ipc/memory.cc
index caff2c610b907..a6c56d64f4aed 100644
--- a/cpp/src/arrow/ipc/memory.cc
+++ b/cpp/src/arrow/ipc/memory.cc
@@ -41,7 +41,7 @@ MemorySource::~MemorySource() {}
 
 class MemoryMappedSource::Impl {
  public:
-  Impl() : file_(nullptr), is_open_(false), data_(nullptr) {}
+  Impl() : file_(nullptr), is_open_(false), is_writable_(false), data_(nullptr) {}
 
   ~Impl() {
     if (is_open_) {
@@ -53,10 +53,12 @@ class MemoryMappedSource::Impl {
   Status Open(const std::string& path, MemorySource::AccessMode mode) {
     if (is_open_) { return Status::IOError("A file is already open"); }
 
-    path_ = path;
+    int prot_flags = PROT_READ;
 
     if (mode == MemorySource::READ_WRITE) {
       file_ = fopen(path.c_str(), "r+b");
+      prot_flags |= PROT_WRITE;
+      is_writable_ = true;
     } else {
       file_ = fopen(path.c_str(), "rb");
     }
@@ -73,14 +75,13 @@ class MemoryMappedSource::Impl {
     fseek(file_, 0L, SEEK_SET);
     is_open_ = true;
 
-    // TODO(wesm): Add read-only version of this
-    data_ = reinterpret_cast<uint8_t*>(
-        mmap(nullptr, size_, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(file_), 0));
-    if (data_ == nullptr) {
+    void* result = mmap(nullptr, size_, prot_flags, MAP_SHARED, fileno(file_), 0);
+    if (result == MAP_FAILED) {
       std::stringstream ss;
       ss << "Memory mapping file failed, errno: " << errno;
       return Status::IOError(ss.str());
     }
+    data_ = reinterpret_cast<uint8_t*>(result);
 
     return Status::OK();
   }
@@ -89,11 +90,15 @@ class MemoryMappedSource::Impl {
 
   uint8_t* data() { return data_; }
 
+  bool writable() { return is_writable_; }
+
+  bool opened() { return is_open_; }
+
  private:
-  std::string path_;
   FILE* file_;
   int64_t size_;
   bool is_open_;
+  bool is_writable_;
 
   // The memory map
   uint8_t* data_;
@@ -134,6 +139,9 @@ Status MemoryMappedSource::ReadAt(
 }
 
 Status MemoryMappedSource::Write(int64_t position, const uint8_t* data, int64_t nbytes) {
+  if (!impl_->opened() || !impl_->writable()) {
+    return Status::IOError("Unable to write");
+  }
   if (position < 0 || position >= impl_->size()) {
     return Status::Invalid("position is out of bounds");
   }

From 1f04f7ff90c43efd72b57cc09ba21da1597682d6 Mon Sep 17 00:00:00 2001
From: lfzCarlosC <lfz.carlos@gmail.com>
Date: Thu, 5 May 2016 21:58:31 +0200
Subject: [PATCH 0069/1644] ARROW-193: typos "int his" fix to "in this"

---
 .../main/java/org/apache/arrow/vector/VariableWidthVector.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
index e227bb4c4176c..971a241adafc2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
@@ -30,7 +30,7 @@ public interface VariableWidthVector extends ValueVector{
   void allocateNew(int totalBytes, int valueCount);
 
   /**
-   * Provide the maximum amount of variable width bytes that can be stored int his vector.
+   * Provide the maximum amount of variable width bytes that can be stored in this vector.
    * @return
    */
   int getByteCapacity();

From 4bd13b852d376065fdb16c36fa821ab0e167f0fc Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 10 May 2016 15:58:04 -0700
Subject: [PATCH 0070/1644] ARROW-91: Basic Parquet read support

Depends on (mainly one line fixes):

- [x] https://github.com/apache/parquet-cpp/pull/99
- [x] https://github.com/apache/parquet-cpp/pull/98
- [x] https://github.com/apache/parquet-cpp/pull/97

Author: Uwe L. Korn <uwelk@xhochy.com>
Author: Wes McKinney <wesm@apache.org>

Closes #73 from xhochy/arrow-91 and squashes the following commits:

7579fed [Uwe L. Korn] Mark single argument constructor as explicit
47441a1 [Uwe L. Korn] Assert that no exception was thrown
5fa1026 [Uwe L. Korn] Incorporate review comments
8d2db22 [Uwe L. Korn] ARROW-91: Basic Parquet read support
d9940d8 [Wes McKinney] Public API draft
---
 cpp/src/arrow/parquet/CMakeLists.txt         |   4 +
 cpp/src/arrow/parquet/parquet-reader-test.cc | 116 +++++++++++
 cpp/src/arrow/parquet/reader.cc              | 194 +++++++++++++++++++
 cpp/src/arrow/parquet/reader.h               | 134 +++++++++++++
 cpp/src/arrow/parquet/schema.cc              |   8 +-
 cpp/src/arrow/parquet/schema.h               |   2 +-
 cpp/src/arrow/parquet/utils.h                |  38 ++++
 7 files changed, 488 insertions(+), 8 deletions(-)
 create mode 100644 cpp/src/arrow/parquet/parquet-reader-test.cc
 create mode 100644 cpp/src/arrow/parquet/reader.cc
 create mode 100644 cpp/src/arrow/parquet/reader.h
 create mode 100644 cpp/src/arrow/parquet/utils.h

diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
index 0d5cf263ec3e2..1ae6709652ea5 100644
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -19,6 +19,7 @@
 # arrow_parquet : Arrow <-> Parquet adapter
 
 set(PARQUET_SRCS
+  reader.cc
   schema.cc
 )
 
@@ -36,6 +37,9 @@ SET_TARGET_PROPERTIES(arrow_parquet PROPERTIES LINKER_LANGUAGE CXX)
 ADD_ARROW_TEST(parquet-schema-test)
 ARROW_TEST_LINK_LIBRARIES(parquet-schema-test arrow_parquet)
 
+ADD_ARROW_TEST(parquet-reader-test)
+ARROW_TEST_LINK_LIBRARIES(parquet-reader-test arrow_parquet)
+
 # Headers: top level
 install(FILES
   DESTINATION include/arrow/parquet)
diff --git a/cpp/src/arrow/parquet/parquet-reader-test.cc b/cpp/src/arrow/parquet/parquet-reader-test.cc
new file mode 100644
index 0000000000000..a7fc2a89f5f45
--- /dev/null
+++ b/cpp/src/arrow/parquet/parquet-reader-test.cc
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/test-util.h"
+#include "arrow/parquet/reader.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+#include "parquet/api/reader.h"
+#include "parquet/api/writer.h"
+
+using ParquetBuffer = parquet::Buffer;
+using parquet::BufferReader;
+using parquet::InMemoryOutputStream;
+using parquet::Int64Writer;
+using parquet::ParquetFileReader;
+using parquet::ParquetFileWriter;
+using parquet::RandomAccessSource;
+using parquet::Repetition;
+using parquet::SchemaDescriptor;
+using ParquetType = parquet::Type;
+using parquet::schema::GroupNode;
+using parquet::schema::NodePtr;
+using parquet::schema::PrimitiveNode;
+
+namespace arrow {
+
+namespace parquet {
+
+class TestReadParquet : public ::testing::Test {
+ public:
+  virtual void SetUp() {}
+
+  std::shared_ptr<GroupNode> Int64Schema() {
+    auto pnode = PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64);
+    NodePtr node_ =
+        GroupNode::Make("schema", Repetition::REQUIRED, std::vector<NodePtr>({pnode}));
+    return std::static_pointer_cast<GroupNode>(node_);
+  }
+
+  std::unique_ptr<ParquetFileReader> Int64File(
+      std::vector<int64_t>& values, int num_chunks) {
+    std::shared_ptr<GroupNode> schema = Int64Schema();
+    std::shared_ptr<InMemoryOutputStream> sink(new InMemoryOutputStream());
+    auto file_writer = ParquetFileWriter::Open(sink, schema);
+    size_t chunk_size = values.size() / num_chunks;
+    for (int i = 0; i < num_chunks; i++) {
+      auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
+      auto column_writer = static_cast<Int64Writer*>(row_group_writer->NextColumn());
+      int64_t* data = values.data() + i * chunk_size;
+      column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
+      column_writer->Close();
+      row_group_writer->Close();
+    }
+    file_writer->Close();
+
+    std::shared_ptr<ParquetBuffer> buffer = sink->GetBuffer();
+    std::unique_ptr<RandomAccessSource> source(new BufferReader(buffer));
+    return ParquetFileReader::Open(std::move(source));
+  }
+
+ private:
+};
+
+TEST_F(TestReadParquet, SingleColumnInt64) {
+  std::vector<int64_t> values(100, 128);
+  std::unique_ptr<ParquetFileReader> file_reader = Int64File(values, 1);
+  arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
+  std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
+  ASSERT_NO_THROW(ASSERT_OK(reader.GetFlatColumn(0, &column_reader)));
+  ASSERT_NE(nullptr, column_reader.get());
+  std::shared_ptr<Array> out;
+  ASSERT_OK(column_reader->NextBatch(100, &out));
+  ASSERT_NE(nullptr, out.get());
+  Int64Array* out_array = static_cast<Int64Array*>(out.get());
+  for (size_t i = 0; i < values.size(); i++) {
+    EXPECT_EQ(values[i], out_array->raw_data()[i]);
+  }
+}
+
+TEST_F(TestReadParquet, SingleColumnInt64Chunked) {
+  std::vector<int64_t> values(100, 128);
+  std::unique_ptr<ParquetFileReader> file_reader = Int64File(values, 4);
+  arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
+  std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
+  ASSERT_NO_THROW(ASSERT_OK(reader.GetFlatColumn(0, &column_reader)));
+  ASSERT_NE(nullptr, column_reader.get());
+  std::shared_ptr<Array> out;
+  ASSERT_OK(column_reader->NextBatch(100, &out));
+  ASSERT_NE(nullptr, out.get());
+  Int64Array* out_array = static_cast<Int64Array*>(out.get());
+  for (size_t i = 0; i < values.size(); i++) {
+    EXPECT_EQ(values[i], out_array->raw_data()[i]);
+  }
+}
+
+}  // namespace parquet
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
new file mode 100644
index 0000000000000..481ded5789a71
--- /dev/null
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -0,0 +1,194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/parquet/reader.h"
+
+#include <queue>
+
+#include "arrow/parquet/schema.h"
+#include "arrow/parquet/utils.h"
+#include "arrow/schema.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/status.h"
+
+using parquet::ColumnReader;
+using parquet::TypedColumnReader;
+
+namespace arrow {
+namespace parquet {
+
+class FileReader::Impl {
+ public:
+  Impl(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader);
+  virtual ~Impl() {}
+
+  Status GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out);
+  Status ReadFlatColumn(int i, std::shared_ptr<Array>* out);
+
+ private:
+  MemoryPool* pool_;
+  std::unique_ptr<::parquet::ParquetFileReader> reader_;
+};
+
+class FlatColumnReader::Impl {
+ public:
+  Impl(MemoryPool* pool, const ::parquet::ColumnDescriptor* descr,
+      ::parquet::ParquetFileReader* reader, int column_index);
+  virtual ~Impl() {}
+
+  Status NextBatch(int batch_size, std::shared_ptr<Array>* out);
+  template <typename ArrowType, typename ParquetType, typename CType>
+  Status TypedReadBatch(int batch_size, std::shared_ptr<Array>* out);
+
+ private:
+  void NextRowGroup();
+
+  MemoryPool* pool_;
+  const ::parquet::ColumnDescriptor* descr_;
+  ::parquet::ParquetFileReader* reader_;
+  int column_index_;
+  int next_row_group_;
+  std::shared_ptr<ColumnReader> column_reader_;
+  std::shared_ptr<Field> field_;
+
+  PoolBuffer values_buffer_;
+  PoolBuffer def_levels_buffer_;
+  PoolBuffer rep_levels_buffer_;
+};
+
+FileReader::Impl::Impl(
+    MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader)
+    : pool_(pool), reader_(std::move(reader)) {}
+
+Status FileReader::Impl::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out) {
+  std::unique_ptr<FlatColumnReader::Impl> impl(
+      new FlatColumnReader::Impl(pool_, reader_->descr()->Column(i), reader_.get(), i));
+  *out = std::unique_ptr<FlatColumnReader>(new FlatColumnReader(std::move(impl)));
+  return Status::OK();
+}
+
+Status FileReader::Impl::ReadFlatColumn(int i, std::shared_ptr<Array>* out) {
+  std::unique_ptr<FlatColumnReader> flat_column_reader;
+  RETURN_NOT_OK(GetFlatColumn(i, &flat_column_reader));
+  return flat_column_reader->NextBatch(reader_->num_rows(), out);
+}
+
+FileReader::FileReader(
+    MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader)
+    : impl_(new FileReader::Impl(pool, std::move(reader))) {}
+
+FileReader::~FileReader() {}
+
+Status FileReader::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out) {
+  return impl_->GetFlatColumn(i, out);
+}
+
+Status FileReader::ReadFlatColumn(int i, std::shared_ptr<Array>* out) {
+  return impl_->ReadFlatColumn(i, out);
+}
+
+FlatColumnReader::Impl::Impl(MemoryPool* pool, const ::parquet::ColumnDescriptor* descr,
+    ::parquet::ParquetFileReader* reader, int column_index)
+    : pool_(pool),
+      descr_(descr),
+      reader_(reader),
+      column_index_(column_index),
+      next_row_group_(0),
+      values_buffer_(pool),
+      def_levels_buffer_(pool),
+      rep_levels_buffer_(pool) {
+  NodeToField(descr_->schema_node(), &field_);
+  NextRowGroup();
+}
+
+template <typename ArrowType, typename ParquetType, typename CType>
+Status FlatColumnReader::Impl::TypedReadBatch(
+    int batch_size, std::shared_ptr<Array>* out) {
+  int values_to_read = batch_size;
+  NumericBuilder<ArrowType> builder(pool_, field_->type);
+  while ((values_to_read > 0) && column_reader_) {
+    values_buffer_.Resize(values_to_read * sizeof(CType));
+    if (descr_->max_definition_level() > 0) {
+      def_levels_buffer_.Resize(values_to_read * sizeof(int16_t));
+    }
+    if (descr_->max_repetition_level() > 0) {
+      rep_levels_buffer_.Resize(values_to_read * sizeof(int16_t));
+    }
+    auto reader = dynamic_cast<TypedColumnReader<ParquetType>*>(column_reader_.get());
+    int64_t values_read;
+    CType* values = reinterpret_cast<CType*>(values_buffer_.mutable_data());
+    PARQUET_CATCH_NOT_OK(
+        values_to_read -= reader->ReadBatch(values_to_read,
+            reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data()),
+            reinterpret_cast<int16_t*>(rep_levels_buffer_.mutable_data()), values,
+            &values_read));
+    if (descr_->max_definition_level() == 0) {
+      RETURN_NOT_OK(builder.Append(values, values_read));
+    } else {
+      return Status::NotImplemented("no support for definition levels yet");
+    }
+    if (!column_reader_->HasNext()) { NextRowGroup(); }
+  }
+  *out = builder.Finish();
+  return Status::OK();
+}
+
+#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType, CType)              \
+  case Type::ENUM:                                                         \
+    return TypedReadBatch<ArrowType, ParquetType, CType>(batch_size, out); \
+    break;
+
+Status FlatColumnReader::Impl::NextBatch(int batch_size, std::shared_ptr<Array>* out) {
+  if (!column_reader_) {
+    // Exhausted all row groups.
+    *out = nullptr;
+    return Status::OK();
+  }
+
+  if (descr_->max_repetition_level() > 0) {
+    return Status::NotImplemented("no support for repetition yet");
+  }
+
+  switch (field_->type->type) {
+    TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type, int32_t)
+    TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type, int64_t)
+    TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType, float)
+    TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType, double)
+    default:
+      return Status::NotImplemented(field_->type->ToString());
+  }
+}
+
+void FlatColumnReader::Impl::NextRowGroup() {
+  if (next_row_group_ < reader_->num_row_groups()) {
+    column_reader_ = reader_->RowGroup(next_row_group_)->Column(column_index_);
+    next_row_group_++;
+  } else {
+    column_reader_ = nullptr;
+  }
+}
+
+FlatColumnReader::FlatColumnReader(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
+
+FlatColumnReader::~FlatColumnReader() {}
+
+Status FlatColumnReader::NextBatch(int batch_size, std::shared_ptr<Array>* out) {
+  return impl_->NextBatch(batch_size, out);
+}
+
+}  // namespace parquet
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/reader.h b/cpp/src/arrow/parquet/reader.h
new file mode 100644
index 0000000000000..41ca7eb35b9f0
--- /dev/null
+++ b/cpp/src/arrow/parquet/reader.h
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PARQUET_READER_H
+#define ARROW_PARQUET_READER_H
+
+#include <memory>
+
+#include "parquet/api/reader.h"
+#include "parquet/api/schema.h"
+
+namespace arrow {
+
+class Array;
+class MemoryPool;
+class RowBatch;
+class Status;
+
+namespace parquet {
+
+class FlatColumnReader;
+
+// Arrow read adapter class for deserializing Parquet files as Arrow row
+// batches.
+//
+// TODO(wesm): nested data does not always make sense with this user
+// interface unless you are only reading a single leaf node from a branch of
+// a table. For example:
+//
+// repeated group data {
+//   optional group record {
+//     optional int32 val1;
+//     optional byte_array val2;
+//     optional bool val3;
+//   }
+//   optional int32 val4;
+// }
+//
+// In the Parquet file, there are 3 leaf nodes:
+//
+// * data.record.val1
+// * data.record.val2
+// * data.record.val3
+// * data.val4
+//
+// When materializing this data in an Arrow array, we would have:
+//
+// data: list<struct<
+//   record: struct<
+//    val1: int32,
+//    val2: string (= list<uint8>),
+//    val3: bool,
+//   >,
+//   val4: int32
+// >>
+//
+// However, in the Parquet format, each leaf node has its own repetition and
+// definition levels describing the structure of the intermediate nodes in
+// this array structure. Thus, we will need to scan the leaf data for a group
+// of leaf nodes part of the same type tree to create a single result Arrow
+// nested array structure.
+//
+// This is additionally complicated "chunky" repeated fields or very large byte
+// arrays
+class FileReader {
+ public:
+  FileReader(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader);
+
+  // Since the distribution of columns amongst a Parquet file's row groups may
+  // be uneven (the number of values in each column chunk can be different), we
+  // provide a column-oriented read interface. The ColumnReader hides the
+  // details of paging through the file's row groups and yielding
+  // fully-materialized arrow::Array instances
+  //
+  // Returns error status if the column of interest is not flat.
+  Status GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out);
+  // Read column as a whole into an Array.
+  Status ReadFlatColumn(int i, std::shared_ptr<Array>* out);
+
+  virtual ~FileReader();
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+// At this point, the column reader is a stream iterator. It only knows how to
+// read the next batch of values for a particular column from the file until it
+// runs out.
+//
+// We also do not expose any internal Parquet details, such as row groups. This
+// might change in the future.
+class FlatColumnReader {
+ public:
+  virtual ~FlatColumnReader();
+
+  // Scan the next array of the indicated size. The actual size of the
+  // returned array may be less than the passed size depending how much data is
+  // available in the file.
+  //
+  // When all the data in the file has been exhausted, the result is set to
+  // nullptr.
+  //
+  // Returns Status::OK on a successful read, including if you have exhausted
+  // the data available in the file.
+  Status NextBatch(int batch_size, std::shared_ptr<Array>* out);
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+  explicit FlatColumnReader(std::unique_ptr<Impl> impl);
+
+  friend class FileReader;
+};
+
+}  // namespace parquet
+
+}  // namespace arrow
+
+#endif  // ARROW_PARQUET_READER_H
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index 214c764f08b6e..fd758940c9f3a 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -21,13 +21,12 @@
 #include <vector>
 
 #include "parquet/api/schema.h"
-#include "parquet/exception.h"
 
+#include "arrow/parquet/utils.h"
 #include "arrow/types/decimal.h"
 #include "arrow/types/string.h"
 #include "arrow/util/status.h"
 
-using parquet::ParquetException;
 using parquet::Repetition;
 using parquet::schema::Node;
 using parquet::schema::NodePtr;
@@ -41,11 +40,6 @@ namespace arrow {
 
 namespace parquet {
 
-#define PARQUET_CATCH_NOT_OK(s) \
-  try {                         \
-    (s);                        \
-  } catch (const ParquetException& e) { return Status::Invalid(e.what()); }
-
 const auto BOOL = std::make_shared<BooleanType>();
 const auto UINT8 = std::make_shared<UInt8Type>();
 const auto INT32 = std::make_shared<Int32Type>();
diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h
index bfc7d21138154..ec5f96062e89f 100644
--- a/cpp/src/arrow/parquet/schema.h
+++ b/cpp/src/arrow/parquet/schema.h
@@ -45,4 +45,4 @@ Status ToParquetSchema(
 
 }  // namespace arrow
 
-#endif
+#endif  // ARROW_PARQUET_SCHEMA_H
diff --git a/cpp/src/arrow/parquet/utils.h b/cpp/src/arrow/parquet/utils.h
new file mode 100644
index 0000000000000..b32792fdf7030
--- /dev/null
+++ b/cpp/src/arrow/parquet/utils.h
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PARQUET_UTILS_H
+#define ARROW_PARQUET_UTILS_H
+
+#include "arrow/util/status.h"
+
+#include "parquet/exception.h"
+
+namespace arrow {
+
+namespace parquet {
+
+#define PARQUET_CATCH_NOT_OK(s) \
+  try {                         \
+    (s);                        \
+  } catch (const ::parquet::ParquetException& e) { return Status::Invalid(e.what()); }
+
+}  // namespace parquet
+
+}  // namespace arrow
+
+#endif  // ARROW_PARQUET_UTILS_H

From 68b80a83876b1306f80d3914eef98f51100a8009 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Sat, 14 May 2016 18:53:22 -0700
Subject: [PATCH 0071/1644] ARROW-197: Working first draft of a conda recipe
 for pyarrow

Includes ARROW-196. I will close that PR and merge these together as I had to make some additional changes. Requires PARQUET-617. Closes #76

Author: Wes McKinney <wesm@apache.org>

Closes #77 from wesm/ARROW-197 and squashes the following commits:

4bf3d2c [Wes McKinney] Finagle toolchain environment variables to get pyarrow conda package working
c2d3684 [Wes McKinney] Add conda recipe and ensure that libarrow_parquet is installed as well
---
 cpp/conda.recipe/build.sh            | 45 ++++++++++++++++++++++++++++
 cpp/conda.recipe/meta.yaml           | 32 ++++++++++++++++++++
 cpp/src/arrow/parquet/CMakeLists.txt |  7 +++++
 cpp/src/arrow/types/primitive.h      |  1 +
 python/conda.recipe/build.sh         | 18 +++++++++++
 python/conda.recipe/meta.yaml        | 41 +++++++++++++++++++++++++
 6 files changed, 144 insertions(+)
 create mode 100644 cpp/conda.recipe/build.sh
 create mode 100644 cpp/conda.recipe/meta.yaml
 create mode 100644 python/conda.recipe/build.sh
 create mode 100644 python/conda.recipe/meta.yaml

diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh
new file mode 100644
index 0000000000000..ac1f9c89cc9ed
--- /dev/null
+++ b/cpp/conda.recipe/build.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+set -x
+
+cd $RECIPE_DIR
+
+# Build dependencies
+export FLATBUFFERS_HOME=$PREFIX
+export PARQUET_HOME=$PREFIX
+
+cd ..
+
+rm -rf conda-build
+mkdir conda-build
+
+cp -r thirdparty conda-build/
+
+cd conda-build
+pwd
+
+# Build googletest for running unit tests
+./thirdparty/download_thirdparty.sh
+./thirdparty/build_thirdparty.sh gtest
+
+source thirdparty/versions.sh
+export GTEST_HOME=`pwd`/thirdparty/$GTEST_BASEDIR
+
+if [ `uname` == Linux ]; then
+    SHARED_LINKER_FLAGS='-static-libstdc++'
+elif [ `uname` == Darwin ]; then
+    SHARED_LINKER_FLAGS=''
+fi
+
+cmake \
+    -DCMAKE_BUILD_TYPE=debug \
+    -DCMAKE_INSTALL_PREFIX=$PREFIX \
+    -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \
+    -DARROW_IPC=on \
+    -DARROW_PARQUET=on \
+    ..
+
+make
+ctest -L unittest
+make install
diff --git a/cpp/conda.recipe/meta.yaml b/cpp/conda.recipe/meta.yaml
new file mode 100644
index 0000000000000..2e834d5cbf86c
--- /dev/null
+++ b/cpp/conda.recipe/meta.yaml
@@ -0,0 +1,32 @@
+package:
+  name: arrow-cpp
+  version: "0.1"
+
+build:
+  number: {{environ.get('TRAVIS_BUILD_NUMBER', 0)}}    # [unix]
+  skip: true  # [win]
+  script_env:
+    - CC [linux]
+    - CXX [linux]
+    - LD_LIBRARY_PATH [linux]
+
+requirements:
+  build:
+    - cmake
+    - flatbuffers
+    - parquet-cpp
+    - thrift-cpp
+
+  run:
+    - parquet-cpp
+
+test:
+  commands:
+    - test -f $PREFIX/lib/libarrow.so
+    - test -f $PREFIX/lib/libarrow_parquet.so
+    - test -f $PREFIX/include/arrow/api.h
+
+about:
+  home: http://github.com/apache/arrow
+  license: Apache 2.0
+  summary: 'C++ libraries for the reference Apache Arrow implementation'
diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
index 1ae6709652ea5..cd6f05d6b5f8a 100644
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -42,4 +42,11 @@ ARROW_TEST_LINK_LIBRARIES(parquet-reader-test arrow_parquet)
 
 # Headers: top level
 install(FILES
+  reader.h
+  schema.h
+  utils.h
   DESTINATION include/arrow/parquet)
+
+install(TARGETS arrow_parquet
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib)
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 6f6b2fed5a320..fc45f6c5b0568 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -136,6 +136,7 @@ class NumericBuilder : public PrimitiveBuilder<T> {
   using PrimitiveBuilder<T>::Append;
   using PrimitiveBuilder<T>::Init;
   using PrimitiveBuilder<T>::Resize;
+  using PrimitiveBuilder<T>::Reserve;
 
   // Scalar append.
   void Append(value_type val) {
diff --git a/python/conda.recipe/build.sh b/python/conda.recipe/build.sh
new file mode 100644
index 0000000000000..a9d9aedead399
--- /dev/null
+++ b/python/conda.recipe/build.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -ex
+
+# Build dependency
+export ARROW_HOME=$PREFIX
+
+cd $RECIPE_DIR
+
+echo Setting the compiler...
+if [ `uname` == Linux ]; then
+  EXTRA_CMAKE_ARGS=-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++
+elif [ `uname` == Darwin ]; then
+  EXTRA_CMAKE_ARGS=
+fi
+
+cd ..
+$PYTHON setup.py build_ext --extra-cmake-args=$EXTRA_CMAKE_ARGS || exit 1
+$PYTHON setup.py install || exit 1
diff --git a/python/conda.recipe/meta.yaml b/python/conda.recipe/meta.yaml
new file mode 100644
index 0000000000000..85d24b6bc322e
--- /dev/null
+++ b/python/conda.recipe/meta.yaml
@@ -0,0 +1,41 @@
+package:
+  name: pyarrow
+  version: "0.1"
+
+build:
+  number: {{environ.get('TRAVIS_BUILD_NUMBER', 0)}}    # [unix]
+  rpaths:
+    - lib                                                        # [unix]
+    - lib/python{{environ.get('PY_VER')}}/site-packages/pyarrow  # [unix]
+  script_env:
+    - CC [linux]
+    - CXX [linux]
+    - LD_LIBRARY_PATH [linux]
+  skip: true  # [win]
+
+requirements:
+  build:
+    - cmake
+    - python
+    - setuptools
+    - cython
+    - numpy
+    - pandas
+    - arrow-cpp
+    - pytest
+
+  run:
+    - arrow-cpp
+    - python
+    - numpy
+    - pandas
+    - six
+
+test:
+  imports:
+    - pyarrow
+
+about:
+  home: http://github.com/apache/arrow
+  license: Apache 2.0
+  summary: 'Python bindings for Arrow C++ and interoperability tool for pandas and NumPy'

From 6968ec01d722584e9561dc3c0438bce29c664b5a Mon Sep 17 00:00:00 2001
From: hzhang2 <zhanghuan929@163.com>
Date: Sat, 14 May 2016 19:07:44 -0700
Subject: [PATCH 0072/1644] ARROW-199: [C++] Refine third party dependency

  To generate makefile, run download_thirdparty.sh and build_thirdparty.sh is not enough
  source setup_build_env.sh is necessary since FLATBUFFERS_HOME must be set .

Author: hzhang2 <zhanghuan929@163.com>

Closes #75 from zhangh43/arrow2 and squashes the following commits:

ea3101b [hzhang2] remove CMAKE_SKIP_INSTALL_ALL_DEPENDENCY for target install and fix typo
8c02a38 [hzhang2] ARROW-199: [C++] Refine third party dependency
b2312e0 [hzhang2] ARROW-199: [C++] Refine third party dependency
fefc314 [hzhang2] FLATBUFFERS_HOME must be set before cmake
---
 cpp/CMakeLists.txt                   |  5 -----
 cpp/README.md                        |  1 +
 cpp/setup_build_env.sh               |  6 +-----
 cpp/thirdparty/set_thirdparty_env.sh | 12 ++++++++++++
 4 files changed, 14 insertions(+), 10 deletions(-)
 create mode 100755 cpp/thirdparty/set_thirdparty_env.sh

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index b38f91e5d687c..a3fb01076d44e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -25,11 +25,6 @@ include(CMakeParseArguments)
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 
-# Allow "make install" to not depend on all targets.
-#
-# Must be declared in the top-level CMakeLists.txt.
-set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
-
 find_package(ClangTools)
 if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
   # Generate a Clang compile_commands.json "compilation database" file for use
diff --git a/cpp/README.md b/cpp/README.md
index c8cd86fedc6fe..129c5f15b150c 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -13,6 +13,7 @@ To build the thirdparty build dependencies, run:
 ```
 ./thirdparty/download_thirdparty.sh
 ./thirdparty/build_thirdparty.sh
+source ./thirdparty/set_thirdparty_env.sh
 ```
 
 You can also run from the root of the C++ tree
diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
index 6520dbd43f705..fa779fdd5c2a3 100755
--- a/cpp/setup_build_env.sh
+++ b/cpp/setup_build_env.sh
@@ -4,10 +4,6 @@ SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
 ./thirdparty/download_thirdparty.sh || { echo "download_thirdparty.sh failed" ; return; }
 ./thirdparty/build_thirdparty.sh || { echo "build_thirdparty.sh failed" ; return; }
-source thirdparty/versions.sh
-
-export GTEST_HOME=$SOURCE_DIR/thirdparty/$GTEST_BASEDIR
-export GBENCHMARK_HOME=$SOURCE_DIR/thirdparty/installed
-export FLATBUFFERS_HOME=$SOURCE_DIR/thirdparty/installed
+source ./thirdparty/set_thirdparty_env.sh || { echo "source set_thirdparty_env.sh failed" ; return; }
 
 echo "Build env initialized"
diff --git a/cpp/thirdparty/set_thirdparty_env.sh b/cpp/thirdparty/set_thirdparty_env.sh
new file mode 100755
index 0000000000000..7e9531cd50864
--- /dev/null
+++ b/cpp/thirdparty/set_thirdparty_env.sh
@@ -0,0 +1,12 @@
+#!/usr/bash
+
+SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+source $SOURCE_DIR/versions.sh
+
+if [ -z "$THIRDPARTY_DIR" ]; then
+	THIRDPARTY_DIR=$SOURCE_DIR
+fi
+
+export GTEST_HOME=$THIRDPARTY_DIR/$GTEST_BASEDIR
+export GBENCHMARK_HOME=$THIRDPARTY_DIR/installed
+export FLATBUFFERS_HOME=$THIRDPARTY_DIR/installed

From 9c59158b4dc84e4de8e9271430befb840e523a4c Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Tue, 17 May 2016 16:46:40 -0700
Subject: [PATCH 0073/1644] ARROW-185: Make padding and alignment for all
 buffers be 64 bytes

+ some small cleanup/removal of unnecessary code.  I think there is likely a good opportunity to factor this code better generally, but this seems to work for now.

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #74 from emkornfield/emk_fix_allocations_PR and squashes the following commits:

e3cca14 [Micah Kornfield] fix cast style
1d006d8 [Micah Kornfield] fix warning
c140e04 [Micah Kornfield] fix lint
7543267 [Micah Kornfield] cleanup
11b3fd7 [Micah Kornfield] replace cython string conversion with string builder
05653cb [Micah Kornfield] add back in memsets because they make valgrind happy
6ff3048 [Micah Kornfield] ARROW-185: Make padding and alignment for all buffers be 64 bytes
---
 cpp/src/arrow/builder.cc               | 11 +++++--
 cpp/src/arrow/ipc/adapter.cc           | 20 ++++++++++++-
 cpp/src/arrow/ipc/ipc-adapter-test.cc  |  6 ++--
 cpp/src/arrow/types/list.cc            |  2 +-
 cpp/src/arrow/types/list.h             |  3 ++
 cpp/src/arrow/types/primitive.cc       |  7 ++---
 cpp/src/arrow/util/bit-util-test.cc    | 10 +++++++
 cpp/src/arrow/util/bit-util.h          |  4 +++
 cpp/src/arrow/util/buffer.cc           | 17 +++++++++++
 cpp/src/arrow/util/buffer.h            | 34 +++++++++++++--------
 cpp/src/arrow/util/memory-pool-test.cc |  1 +
 cpp/src/arrow/util/memory-pool.cc      | 31 ++++++++++++++-----
 python/src/pyarrow/adapters/pandas.cc  | 41 ++++++--------------------
 13 files changed, 124 insertions(+), 63 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 87c1219025d37..1fba96169228f 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -45,12 +45,14 @@ Status ArrayBuilder::AppendToBitmap(const uint8_t* valid_bytes, int32_t length)
 }
 
 Status ArrayBuilder::Init(int32_t capacity) {
-  capacity_ = capacity;
   int32_t to_alloc = util::ceil_byte(capacity) / 8;
   null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
   RETURN_NOT_OK(null_bitmap_->Resize(to_alloc));
+  // Buffers might allocate more then necessary to satisfy padding requirements
+  const int byte_capacity = null_bitmap_->capacity();
+  capacity_ = capacity;
   null_bitmap_data_ = null_bitmap_->mutable_data();
-  memset(null_bitmap_data_, 0, to_alloc);
+  memset(null_bitmap_data_, 0, byte_capacity);
   return Status::OK();
 }
 
@@ -60,8 +62,11 @@ Status ArrayBuilder::Resize(int32_t new_bits) {
   int32_t old_bytes = null_bitmap_->size();
   RETURN_NOT_OK(null_bitmap_->Resize(new_bytes));
   null_bitmap_data_ = null_bitmap_->mutable_data();
+  // The buffer might be overpadded to deal with padding according to the spec
+  const int32_t byte_capacity = null_bitmap_->capacity();
+  capacity_ = new_bits;
   if (old_bytes < new_bytes) {
-    memset(null_bitmap_data_ + old_bytes, 0, new_bytes - old_bytes);
+    memset(null_bitmap_data_ + old_bytes, 0, byte_capacity - old_bytes);
   }
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 34700080746e7..45cc288cd6b9e 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -43,6 +43,15 @@ namespace flatbuf = apache::arrow::flatbuf;
 
 namespace ipc {
 
+namespace {
+Status CheckMultipleOf64(int64_t size) {
+  if (util::is_multiple_of_64(size)) { return Status::OK(); }
+  return Status::Invalid(
+      "Attempted to write a buffer that "
+      "wasn't a multiple of 64 bytes");
+}
+}
+
 static bool IsPrimitive(const DataType* type) {
   DCHECK(type != nullptr);
   switch (type->type) {
@@ -115,6 +124,8 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
   } else if (arr->type_enum() == Type::STRUCT) {
     // TODO(wesm)
     return Status::NotImplemented("Struct type");
+  } else {
+    return Status::NotImplemented("Unrecognized type");
   }
   return Status::OK();
 }
@@ -142,7 +153,13 @@ class RowBatchWriter {
       int64_t size = 0;
 
       // The buffer might be null if we are handling zero row lengths.
-      if (buffer) { size = buffer->size(); }
+      if (buffer) {
+        // We use capacity here, because size might not reflect the padding
+        // requirements of buffers but capacity always should.
+        size = buffer->capacity();
+        // check that padding is appropriate
+        RETURN_NOT_OK(CheckMultipleOf64(size));
+      }
       // TODO(wesm): We currently have no notion of shared memory page id's,
       // but we've included it in the metadata IDL for when we have it in the
       // future. Use page=0 for now
@@ -305,6 +322,7 @@ class RowBatchReader::Impl {
 
   Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
     BufferMetadata metadata = metadata_->buffer(buffer_index);
+    RETURN_NOT_OK(CheckMultipleOf64(metadata.length));
     return source_->ReadAt(metadata.offset, metadata.length, out);
   }
 
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 3b147343f772a..eb47ac6fee8a1 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -197,8 +197,8 @@ INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRowBatch,
 
 void TestGetRowBatchSize(std::shared_ptr<RowBatch> batch) {
   MockMemorySource mock_source(1 << 16);
-  int64_t mock_header_location;
-  int64_t size;
+  int64_t mock_header_location = -1;
+  int64_t size = -1;
   ASSERT_OK(WriteRowBatch(&mock_source, batch.get(), 0, &mock_header_location));
   ASSERT_OK(GetRowBatchSize(batch.get(), &size));
   ASSERT_EQ(mock_source.GetExtentBytesWritten(), size);
@@ -270,7 +270,7 @@ TEST_F(RecursionLimits, WriteLimit) {
 }
 
 TEST_F(RecursionLimits, ReadLimit) {
-  int64_t header_location;
+  int64_t header_location = -1;
   std::shared_ptr<Schema> schema;
   ASSERT_OK(WriteToMmap(64, true, &header_location, &schema));
 
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index fc3331139c6d8..76e7fe5f4d429 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -47,7 +47,7 @@ bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
 Status ListArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
   if (!offset_buf_) { return Status::Invalid("offset_buf_ was null"); }
-  if (offset_buf_->size() / sizeof(int32_t) < length_) {
+  if (offset_buf_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
     std::stringstream ss;
     ss << "offset buffer size (bytes): " << offset_buf_->size()
        << " isn't large enough for length: " << length_;
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index e2302d917b8f6..a020b8ad22668 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <cstring>
+#include <limits>
 #include <memory>
 
 #include "arrow/array.h"
@@ -113,12 +114,14 @@ class ListBuilder : public ArrayBuilder {
         values_(values) {}
 
   Status Init(int32_t elements) override {
+    DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
     RETURN_NOT_OK(ArrayBuilder::Init(elements));
     // one more then requested for offsets
     return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
   }
 
   Status Resize(int32_t capacity) override {
+    DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
     // one more then requested for offsets
     RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
     return ArrayBuilder::Resize(capacity);
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 9102c530e25da..57a3f1e4e150b 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -76,6 +76,7 @@ Status PrimitiveBuilder<T>::Init(int32_t capacity) {
 
   int64_t nbytes = type_traits<T>::bytes_required(capacity);
   RETURN_NOT_OK(data_->Resize(nbytes));
+  // TODO(emkornfield) valgrind complains without this
   memset(data_->mutable_data(), 0, nbytes);
 
   raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
@@ -91,15 +92,13 @@ Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
     RETURN_NOT_OK(Init(capacity));
   } else {
     RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
-
-    int64_t old_bytes = data_->size();
-    int64_t new_bytes = type_traits<T>::bytes_required(capacity);
+    const int64_t old_bytes = data_->size();
+    const int64_t new_bytes = type_traits<T>::bytes_required(capacity);
     RETURN_NOT_OK(data_->Resize(new_bytes));
     raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
 
     memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
   }
-  capacity_ = capacity;
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index 26554d2c9069c..e1d8a0808b41a 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -21,6 +21,16 @@
 
 namespace arrow {
 
+TEST(UtilTests, TestIsMultipleOf64) {
+  using util::is_multiple_of_64;
+  EXPECT_TRUE(is_multiple_of_64(64));
+  EXPECT_TRUE(is_multiple_of_64(0));
+  EXPECT_TRUE(is_multiple_of_64(128));
+  EXPECT_TRUE(is_multiple_of_64(192));
+  EXPECT_FALSE(is_multiple_of_64(23));
+  EXPECT_FALSE(is_multiple_of_64(32));
+}
+
 TEST(UtilTests, TestNextPower2) {
   using util::next_power2;
 
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 1f0f08c4d88ef..a6c8dd904d8e0 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -71,6 +71,10 @@ static inline int64_t next_power2(int64_t n) {
   return n;
 }
 
+static inline bool is_multiple_of_64(int64_t n) {
+  return (n & 63) == 0;
+}
+
 void bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits);
 Status bytes_to_bits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
index bc9c22c10de44..703ef8384ac07 100644
--- a/cpp/src/arrow/util/buffer.cc
+++ b/cpp/src/arrow/util/buffer.cc
@@ -18,16 +18,32 @@
 #include "arrow/util/buffer.h"
 
 #include <cstdint>
+#include <limits>
 
+#include "arrow/util/logging.h"
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
 
+namespace {
+int64_t RoundUpToMultipleOf64(int64_t num) {
+  DCHECK_GE(num, 0);
+  constexpr int64_t round_to = 64;
+  constexpr int64_t force_carry_addend = round_to - 1;
+  constexpr int64_t truncate_bitmask = ~(round_to - 1);
+  constexpr int64_t max_roundable_num = std::numeric_limits<int64_t>::max() - round_to;
+  if (num <= max_roundable_num) { return (num + force_carry_addend) & truncate_bitmask; }
+  // handle overflow case.  This should result in a malloc error upstream
+  return num;
+}
+}  // namespace
+
 Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size) {
   data_ = parent->data() + offset;
   size_ = size;
   parent_ = parent;
+  capacity_ = size;
 }
 
 Buffer::~Buffer() {}
@@ -48,6 +64,7 @@ PoolBuffer::~PoolBuffer() {
 Status PoolBuffer::Reserve(int64_t new_capacity) {
   if (!mutable_data_ || new_capacity > capacity_) {
     uint8_t* new_data;
+    new_capacity = RoundUpToMultipleOf64(new_capacity);
     if (mutable_data_) {
       RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
       memcpy(new_data, mutable_data_, size_);
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 5ef0076953cea..f845d67761fe4 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -36,15 +36,23 @@ class Status;
 // Buffer classes
 
 // Immutable API for a chunk of bytes which may or may not be owned by the
-// class instance
+// class instance.  Buffers have two related notions of length: size and
+// capacity.  Size is the number of bytes that might have valid data.
+// Capacity is the number of bytes that where allocated for the buffer in
+// total.
+// The following invariant is always true: Size < Capacity
 class Buffer : public std::enable_shared_from_this<Buffer> {
  public:
-  Buffer(const uint8_t* data, int64_t size) : data_(data), size_(size) {}
+  Buffer(const uint8_t* data, int64_t size) : data_(data), size_(size), capacity_(size) {}
   virtual ~Buffer();
 
   // An offset into data that is owned by another buffer, but we want to be
   // able to retain a valid pointer to it even after other shared_ptr's to the
   // parent buffer have been destroyed
+  //
+  // This method makes no assertions about alignment or padding of the buffer but
+  // in general we expected buffers to be aligned and padded to 64 bytes.  In the future
+  // we might add utility methods to help determine if a buffer satisfies this contract.
   Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
 
   std::shared_ptr<Buffer> get_shared_ptr() { return shared_from_this(); }
@@ -63,6 +71,7 @@ class Buffer : public std::enable_shared_from_this<Buffer> {
                (data_ == other.data_ || !memcmp(data_, other.data_, size_)));
   }
 
+  int64_t capacity() const { return capacity_; }
   const uint8_t* data() const { return data_; }
 
   int64_t size() const { return size_; }
@@ -76,6 +85,7 @@ class Buffer : public std::enable_shared_from_this<Buffer> {
  protected:
   const uint8_t* data_;
   int64_t size_;
+  int64_t capacity_;
 
   // nullptr by default, but may be set
   std::shared_ptr<Buffer> parent_;
@@ -105,18 +115,17 @@ class MutableBuffer : public Buffer {
 class ResizableBuffer : public MutableBuffer {
  public:
   // Change buffer reported size to indicated size, allocating memory if
-  // necessary
+  // necessary.  This will ensure that the capacity of the buffer is a multiple
+  // of 64 bytes as defined in Layout.md.
   virtual Status Resize(int64_t new_size) = 0;
 
   // Ensure that buffer has enough memory allocated to fit the indicated
-  // capacity. Does not change buffer's reported size
+  // capacity (and meets the 64 byte padding requirement in Layout.md).
+  // It does not change buffer's reported size.
   virtual Status Reserve(int64_t new_capacity) = 0;
 
  protected:
-  ResizableBuffer(uint8_t* data, int64_t size)
-      : MutableBuffer(data, size), capacity_(size) {}
-
-  int64_t capacity_;
+  ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {}
 };
 
 // A Buffer whose lifetime is tied to a particular MemoryPool
@@ -125,8 +134,8 @@ class PoolBuffer : public ResizableBuffer {
   explicit PoolBuffer(MemoryPool* pool = nullptr);
   virtual ~PoolBuffer();
 
-  virtual Status Resize(int64_t new_size);
-  virtual Status Reserve(int64_t new_capacity);
+  Status Resize(int64_t new_size) override;
+  Status Reserve(int64_t new_capacity) override;
 
  private:
   MemoryPool* pool_;
@@ -138,10 +147,11 @@ class BufferBuilder {
  public:
   explicit BufferBuilder(MemoryPool* pool) : pool_(pool), capacity_(0), size_(0) {}
 
+  // Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
   Status Resize(int32_t elements) {
     if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
-    capacity_ = elements;
-    RETURN_NOT_OK(buffer_->Resize(capacity_));
+    RETURN_NOT_OK(buffer_->Resize(elements));
+    capacity_ = buffer_->capacity();
     data_ = buffer_->mutable_data();
     return Status::OK();
   }
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
index e4600a9bd9b27..4ab9736c2b468 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -31,6 +31,7 @@ TEST(DefaultMemoryPool, MemoryTracking) {
 
   uint8_t* data;
   ASSERT_OK(pool->Allocate(100, &data));
+  EXPECT_EQ(0, reinterpret_cast<uint64_t>(data) % 64);
   ASSERT_EQ(100, pool->bytes_allocated());
 
   pool->Free(data, 100);
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
index 961554fe06bcc..0a58e5aa21f72 100644
--- a/cpp/src/arrow/util/memory-pool.cc
+++ b/cpp/src/arrow/util/memory-pool.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/util/memory-pool.h"
 
+#include <stdlib.h>
 #include <cstdlib>
 #include <mutex>
 #include <sstream>
@@ -25,6 +26,28 @@
 
 namespace arrow {
 
+namespace {
+// Allocate memory according to the alignment requirements for Arrow
+// (as of May 2016 64 bytes)
+Status AllocateAligned(int64_t size, uint8_t** out) {
+  // TODO(emkornfield) find something compatible with windows
+  constexpr size_t kAlignment = 64;
+  const int result = posix_memalign(reinterpret_cast<void**>(out), kAlignment, size);
+  if (result == ENOMEM) {
+    std::stringstream ss;
+    ss << "malloc of size " << size << " failed";
+    return Status::OutOfMemory(ss.str());
+  }
+
+  if (result == EINVAL) {
+    std::stringstream ss;
+    ss << "invalid alignment parameter: " << kAlignment;
+    return Status::Invalid(ss.str());
+  }
+  return Status::OK();
+}
+}  // namespace
+
 MemoryPool::~MemoryPool() {}
 
 class InternalMemoryPool : public MemoryPool {
@@ -45,13 +68,7 @@ class InternalMemoryPool : public MemoryPool {
 
 Status InternalMemoryPool::Allocate(int64_t size, uint8_t** out) {
   std::lock_guard<std::mutex> guard(pool_lock_);
-  *out = static_cast<uint8_t*>(std::malloc(size));
-  if (*out == nullptr) {
-    std::stringstream ss;
-    ss << "malloc of size " << size << " failed";
-    return Status::OutOfMemory(ss.str());
-  }
-
+  RETURN_NOT_OK(AllocateAligned(size, out));
   bytes_allocated_ += size;
 
   return Status::OK();
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index b39fde92034aa..5159d86865caa 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -147,17 +147,12 @@ class ArrowSerializer {
 
   Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
     PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    arrow::TypePtr string_type(new arrow::StringType());
+    arrow::StringBuilder string_builder(pool_, string_type);
+    RETURN_ARROW_NOT_OK(string_builder.Resize(length_));
 
-    auto offsets_buffer = std::make_shared<arrow::PoolBuffer>(pool_);
-    RETURN_ARROW_NOT_OK(offsets_buffer->Resize(sizeof(int32_t) * (length_ + 1)));
-    int32_t* offsets = reinterpret_cast<int32_t*>(offsets_buffer->mutable_data());
-
-    arrow::BufferBuilder data_builder(pool_);
     arrow::Status s;
     PyObject* obj;
-    int length;
-    int offset = 0;
-    int64_t null_count = 0;
     for (int64_t i = 0; i < length_; ++i) {
       obj = objects[i];
       if (PyUnicode_Check(obj)) {
@@ -166,38 +161,20 @@ class ArrowSerializer {
           PyErr_Clear();
           return Status::TypeError("failed converting unicode to UTF8");
         }
-        length = PyBytes_GET_SIZE(obj);
-        s = data_builder.Append(
-            reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)), length);
+        const int32_t length = PyBytes_GET_SIZE(obj);
+        s = string_builder.Append(PyBytes_AS_STRING(obj), length);
         Py_DECREF(obj);
         if (!s.ok()) {
           return Status::ArrowError(s.ToString());
         }
-        util::set_bit(null_bitmap_data_, i);
       } else if (PyBytes_Check(obj)) {
-        length = PyBytes_GET_SIZE(obj);
-        RETURN_ARROW_NOT_OK(data_builder.Append(
-                reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)), length));
-        util::set_bit(null_bitmap_data_, i);
+        const int32_t length = PyBytes_GET_SIZE(obj);
+        RETURN_ARROW_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
       } else {
-        // NULL
-        // No change to offset
-        length = 0;
-        ++null_count;
+        string_builder.AppendNull();
       }
-      offsets[i] = offset;
-      offset += length;
     }
-    // End offset
-    offsets[length_] = offset;
-
-    std::shared_ptr<arrow::Buffer> data_buffer = data_builder.Finish();
-
-    auto values = std::make_shared<arrow::UInt8Array>(data_buffer->size(),
-        data_buffer);
-    *out = std::shared_ptr<arrow::Array>(
-        new arrow::StringArray(length_, offsets_buffer, values, null_count,
-            null_bitmap_));
+    *out = std::shared_ptr<arrow::Array>(string_builder.Finish());
 
     return Status::OK();
   }

From 978de1a94dd451b3142aca0eb95ce410064a2330 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Wed, 18 May 2016 10:15:14 -0700
Subject: [PATCH 0074/1644] ARROW-204: Add Travis CI builds that post conda
 artifacts for Linux and OS X

I tested this on my fork of Arrow, but ultimately we'll see if it works when the commit hits master. I've arranged so that packaging issues won't fail the build.

Author: Wes McKinney <wesm@apache.org>

Closes #79 from wesm/ARROW-204 and squashes the following commits:

afd0582 [Wes McKinney] Change encrypted token to apache/arrow, only upload on commits to master
58955e5 [Wes McKinney] Draft of automated conda builds for libarrow, pyarrow. Remove unneeded thrift-cpp build dependency
---
 .travis.yml                | 27 ++++++++++++++++++-
 ci/travis_conda_build.sh   | 53 ++++++++++++++++++++++++++++++++++++++
 cpp/conda.recipe/build.sh  | 15 ++++++++++-
 cpp/conda.recipe/meta.yaml |  5 ++--
 4 files changed, 95 insertions(+), 5 deletions(-)
 create mode 100755 ci/travis_conda_build.sh

diff --git a/.travis.yml b/.travis.yml
index a0138a79598a1..646f80fee7b3b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 sudo: required
-dist: precise 
+dist: precise
 addons:
   apt:
     sources:
@@ -18,6 +18,9 @@ addons:
     - valgrind
 
 matrix:
+  fast_finish: true
+  allow_failures:
+  - env: ARROW_TEST_GROUP=packaging
   include:
   - compiler: gcc
     language: cpp
@@ -39,6 +42,24 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
+  - compiler: gcc
+    env: ARROW_TEST_GROUP=packaging
+    os: linux
+    before_script:
+    - export CC="gcc-4.9"
+    - export CXX="g++-4.9"
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_conda_build.sh
+  - os: osx
+    env: ARROW_TEST_GROUP=packaging
+    language: objective-c
+    osx_image: xcode6.4
+    compiler: clang
+    addons:
+    before_script:
+    before_install:
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_conda_build.sh
 
 before_install:
 - ulimit -c unlimited -S
@@ -51,3 +72,7 @@ after_script:
 after_failure:
 - COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1)
 - if [[ -f "$COREFILE" ]]; then gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch; fi
+
+env:
+  global:
+  - secure: "GcrPtsKUCgNY7HKYjWlHQo8SiFrShDvdZSU8t1m1FJrE+UfK0Dgh9zXmAausM8GmhqSwkF0q4UbLQf2uCnSITWKeEPAL8Mo9eu4ib+ikJx/b3Sk81frgW5ADoHfW1Eyqd8xJNIMwMegJOtRLSDqiXh1CvMlKnY8PyTOGM2DgN9ona/v6p9OFH9Qs0JhBRVXAn0S4ztjumck8E56+01hqRfxbZ88pTfpKghBxYp9PJaMjtGdomjVWlqPaWaWJj+KptT8inV9NK+TVYKx0dXWD+S1Vgr1PytQnLdILOYV23gsOBYqn33ByF/yADl4m3hUjU/qeT0Fi7aWxmVpj+oTJISOSH5N8nIsuNH8mQk2ZzzXHfV7btFvP+cOPRczadoKkT6D6cHA8nQ7b0dphC6bl6SAeSfc/cbhRT+fYnIjg8jFXC8jlyWBr7LR6GXVpc0bND7i300ITo0FuRJhy2OxqPtGo3dKLE7eAcv78tuO0OYJ/kol1PEqFdFkbYbNVbg/cFpbGqiCXDsOtPDbAGBv69YnXdVowSxxs8cRGjSkDydv6ZSytb/Zd4lH/KAomcFNk8adx12O1Lk4sbmVav1cGig5P6OcQKS0jC5IiRb4THcQzVzAkXXbaafKm5sru/NoYxhzmkyhkOc11nTYHKVng+XKWzLCNn7pTTSLitp5+xa4="
diff --git a/ci/travis_conda_build.sh b/ci/travis_conda_build.sh
new file mode 100755
index 0000000000000..afa531dbd6b5f
--- /dev/null
+++ b/ci/travis_conda_build.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
+else
+  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
+fi
+
+wget -O miniconda.sh $MINICONDA_URL
+MINICONDA=$TRAVIS_BUILD_DIR/miniconda
+bash miniconda.sh -b -p $MINICONDA
+export PATH="$MINICONDA/bin:$PATH"
+conda update -y -q conda
+conda info -a
+
+conda config --set show_channel_urls yes
+conda config --add channels conda-forge
+conda config --add channels apache
+
+conda install --yes conda-build jinja2 anaconda-client
+
+# faster builds, please
+conda install -y nomkl
+
+# Build libarrow
+
+cd $TRAVIS_BUILD_DIR/cpp
+
+conda build conda.recipe --channel apache/channel/dev
+CONDA_PACKAGE=`conda build --output conda.recipe | grep bz2`
+
+if [ $TRAVIS_BRANCH == "master" ] && [ $TRAVIS_PULL_REQUEST == "false" ]; then
+  anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev;
+fi
+
+# Build pyarrow
+
+cd $TRAVIS_BUILD_DIR/python
+
+build_for_python_version() {
+  PY_VERSION=$1
+  conda build conda.recipe --python $PY_VERSION --channel apache/channel/dev
+  CONDA_PACKAGE=`conda build --python $PY_VERSION --output conda.recipe | grep bz2`
+
+  if [ $TRAVIS_BRANCH == "master" ] && [ $TRAVIS_PULL_REQUEST == "false" ]; then
+	anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev;
+  fi
+}
+
+build_for_python_version 2.7
+build_for_python_version 3.5
diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh
index ac1f9c89cc9ed..b10dd03349bd3 100644
--- a/cpp/conda.recipe/build.sh
+++ b/cpp/conda.recipe/build.sh
@@ -9,6 +9,19 @@ cd $RECIPE_DIR
 export FLATBUFFERS_HOME=$PREFIX
 export PARQUET_HOME=$PREFIX
 
+if [ "$(uname)" == "Darwin" ]; then
+  # C++11 finagling for Mac OSX
+  export CC=clang
+  export CXX=clang++
+  export MACOSX_VERSION_MIN="10.7"
+  CXXFLAGS="${CXXFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}"
+  CXXFLAGS="${CXXFLAGS} -stdlib=libc++ -std=c++11"
+  export LDFLAGS="${LDFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}"
+  export LDFLAGS="${LDFLAGS} -stdlib=libc++ -std=c++11"
+  export LINKFLAGS="${LDFLAGS}"
+  export MACOSX_DEPLOYMENT_TARGET=10.7
+fi
+
 cd ..
 
 rm -rf conda-build
@@ -33,7 +46,7 @@ elif [ `uname` == Darwin ]; then
 fi
 
 cmake \
-    -DCMAKE_BUILD_TYPE=debug \
+    -DCMAKE_BUILD_TYPE=release \
     -DCMAKE_INSTALL_PREFIX=$PREFIX \
     -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \
     -DARROW_IPC=on \
diff --git a/cpp/conda.recipe/meta.yaml b/cpp/conda.recipe/meta.yaml
index 2e834d5cbf86c..75f3a8ba3d98f 100644
--- a/cpp/conda.recipe/meta.yaml
+++ b/cpp/conda.recipe/meta.yaml
@@ -15,15 +15,14 @@ requirements:
     - cmake
     - flatbuffers
     - parquet-cpp
-    - thrift-cpp
 
   run:
     - parquet-cpp
 
 test:
   commands:
-    - test -f $PREFIX/lib/libarrow.so
-    - test -f $PREFIX/lib/libarrow_parquet.so
+    - test -f $PREFIX/lib/libarrow.so            # [linux]
+    - test -f $PREFIX/lib/libarrow_parquet.so    # [linux]
     - test -f $PREFIX/include/arrow/api.h
 
 about:

From e0fb3698e5602bccaee232d4c259b3df089886e6 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 18 May 2016 10:49:04 -0700
Subject: [PATCH 0075/1644] ARROW-201: [C++] Initial ParquetWriter
 implementation

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #78 from xhochy/arrow-201 and squashes the following commits:

5d95099 [Uwe L. Korn] Add check for flat column
88ae3ca [Uwe L. Korn] Install arrow_parquet headers
f81021b [Uwe L. Korn] Incorporate reader comments
ba240e8 [Uwe L. Korn] Incorporate writer comments
2179c0e [Uwe L. Korn] Infer c-type from ArrowType
efd46fb [Uwe L. Korn] Infer c-type from ArrowType
77386ea [Uwe L. Korn] Templatize test functions
1aa7698 [Uwe L. Korn] Add comment to helper function
8fdd4c8 [Uwe L. Korn] Parameterize schema creation
8e8d7d7 [Uwe L. Korn] ARROW-201: [C++] Initial ParquetWriter implementation
---
 cpp/src/arrow/parquet/CMakeLists.txt         |   6 +-
 cpp/src/arrow/parquet/parquet-io-test.cc     | 222 +++++++++++++++++++
 cpp/src/arrow/parquet/parquet-reader-test.cc | 116 ----------
 cpp/src/arrow/parquet/reader.cc              |  79 ++++---
 cpp/src/arrow/parquet/writer.cc              | 148 +++++++++++++
 cpp/src/arrow/parquet/writer.h               |  59 +++++
 6 files changed, 485 insertions(+), 145 deletions(-)
 create mode 100644 cpp/src/arrow/parquet/parquet-io-test.cc
 delete mode 100644 cpp/src/arrow/parquet/parquet-reader-test.cc
 create mode 100644 cpp/src/arrow/parquet/writer.cc
 create mode 100644 cpp/src/arrow/parquet/writer.h

diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
index cd6f05d6b5f8a..c00cc9f0f25d0 100644
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -21,6 +21,7 @@
 set(PARQUET_SRCS
   reader.cc
   schema.cc
+  writer.cc
 )
 
 set(PARQUET_LIBS
@@ -37,14 +38,15 @@ SET_TARGET_PROPERTIES(arrow_parquet PROPERTIES LINKER_LANGUAGE CXX)
 ADD_ARROW_TEST(parquet-schema-test)
 ARROW_TEST_LINK_LIBRARIES(parquet-schema-test arrow_parquet)
 
-ADD_ARROW_TEST(parquet-reader-test)
-ARROW_TEST_LINK_LIBRARIES(parquet-reader-test arrow_parquet)
+ADD_ARROW_TEST(parquet-io-test)
+ARROW_TEST_LINK_LIBRARIES(parquet-io-test arrow_parquet)
 
 # Headers: top level
 install(FILES
   reader.h
   schema.h
   utils.h
+  writer.h
   DESTINATION include/arrow/parquet)
 
 install(TARGETS arrow_parquet
diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
new file mode 100644
index 0000000000000..845574d2c53b9
--- /dev/null
+++ b/cpp/src/arrow/parquet/parquet-io-test.cc
@@ -0,0 +1,222 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/test-util.h"
+#include "arrow/parquet/reader.h"
+#include "arrow/parquet/writer.h"
+#include "arrow/types/primitive.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+#include "parquet/api/reader.h"
+#include "parquet/api/writer.h"
+
+using ParquetBuffer = parquet::Buffer;
+using parquet::BufferReader;
+using parquet::InMemoryOutputStream;
+using parquet::ParquetFileReader;
+using parquet::ParquetFileWriter;
+using parquet::RandomAccessSource;
+using parquet::Repetition;
+using parquet::SchemaDescriptor;
+using ParquetType = parquet::Type;
+using parquet::schema::GroupNode;
+using parquet::schema::NodePtr;
+using parquet::schema::PrimitiveNode;
+
+namespace arrow {
+
+namespace parquet {
+
+template <typename ArrowType>
+std::shared_ptr<PrimitiveArray> NonNullArray(
+    size_t size, typename ArrowType::c_type value) {
+  std::vector<typename ArrowType::c_type> values(size, value);
+  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
+  builder.Append(values.data(), values.size());
+  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
+}
+
+// This helper function only supports (size/2) nulls yet.
+template <typename ArrowType>
+std::shared_ptr<PrimitiveArray> NullableArray(
+    size_t size, typename ArrowType::c_type value, size_t num_nulls) {
+  std::vector<typename ArrowType::c_type> values(size, value);
+  std::vector<uint8_t> valid_bytes(size, 1);
+
+  for (size_t i = 0; i < num_nulls; i++) {
+    valid_bytes[i * 2] = 0;
+  }
+
+  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
+  builder.Append(values.data(), values.size(), valid_bytes.data());
+  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
+}
+
+class TestParquetIO : public ::testing::Test {
+ public:
+  virtual void SetUp() {}
+
+  std::shared_ptr<GroupNode> Schema(
+      ParquetType::type parquet_type, Repetition::type repetition) {
+    auto pnode = PrimitiveNode::Make("column1", repetition, parquet_type);
+    NodePtr node_ =
+        GroupNode::Make("schema", Repetition::REQUIRED, std::vector<NodePtr>({pnode}));
+    return std::static_pointer_cast<GroupNode>(node_);
+  }
+
+  std::unique_ptr<ParquetFileWriter> MakeWriter(std::shared_ptr<GroupNode>& schema) {
+    sink_ = std::make_shared<InMemoryOutputStream>();
+    return ParquetFileWriter::Open(sink_, schema);
+  }
+
+  std::unique_ptr<ParquetFileReader> ReaderFromSink() {
+    std::shared_ptr<ParquetBuffer> buffer = sink_->GetBuffer();
+    std::unique_ptr<RandomAccessSource> source(new BufferReader(buffer));
+    return ParquetFileReader::Open(std::move(source));
+  }
+
+  void ReadSingleColumnFile(
+      std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Array>* out) {
+    arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
+    std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
+    ASSERT_NO_THROW(ASSERT_OK(reader.GetFlatColumn(0, &column_reader)));
+    ASSERT_NE(nullptr, column_reader.get());
+    ASSERT_OK(column_reader->NextBatch(100, out));
+    ASSERT_NE(nullptr, out->get());
+  }
+
+  std::unique_ptr<ParquetFileReader> Int64File(
+      std::vector<int64_t>& values, int num_chunks) {
+    std::shared_ptr<GroupNode> schema = Schema(ParquetType::INT64, Repetition::REQUIRED);
+    std::unique_ptr<ParquetFileWriter> file_writer = MakeWriter(schema);
+    size_t chunk_size = values.size() / num_chunks;
+    for (int i = 0; i < num_chunks; i++) {
+      auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
+      auto column_writer =
+          static_cast<::parquet::Int64Writer*>(row_group_writer->NextColumn());
+      int64_t* data = values.data() + i * chunk_size;
+      column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
+      column_writer->Close();
+      row_group_writer->Close();
+    }
+    file_writer->Close();
+    return ReaderFromSink();
+  }
+
+ private:
+  std::shared_ptr<InMemoryOutputStream> sink_;
+};
+
+TEST_F(TestParquetIO, SingleColumnInt64Read) {
+  std::vector<int64_t> values(100, 128);
+  std::unique_ptr<ParquetFileReader> file_reader = Int64File(values, 1);
+
+  std::shared_ptr<Array> out;
+  ReadSingleColumnFile(std::move(file_reader), &out);
+
+  Int64Array* out_array = static_cast<Int64Array*>(out.get());
+  for (size_t i = 0; i < values.size(); i++) {
+    EXPECT_EQ(values[i], out_array->raw_data()[i]);
+  }
+}
+
+TEST_F(TestParquetIO, SingleColumnInt64ChunkedRead) {
+  std::vector<int64_t> values(100, 128);
+  std::unique_ptr<ParquetFileReader> file_reader = Int64File(values, 4);
+
+  std::shared_ptr<Array> out;
+  ReadSingleColumnFile(std::move(file_reader), &out);
+
+  Int64Array* out_array = static_cast<Int64Array*>(out.get());
+  for (size_t i = 0; i < values.size(); i++) {
+    EXPECT_EQ(values[i], out_array->raw_data()[i]);
+  }
+}
+
+TEST_F(TestParquetIO, SingleColumnInt64Write) {
+  std::shared_ptr<PrimitiveArray> values = NonNullArray<Int64Type>(100, 128);
+
+  std::shared_ptr<GroupNode> schema = Schema(ParquetType::INT64, Repetition::REQUIRED);
+  FileWriter writer(default_memory_pool(), MakeWriter(schema));
+  ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values->length())));
+  ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values.get())));
+  ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
+
+  std::shared_ptr<Array> out;
+  ReadSingleColumnFile(ReaderFromSink(), &out);
+  ASSERT_TRUE(values->Equals(out));
+}
+
+TEST_F(TestParquetIO, SingleColumnDoubleReadWrite) {
+  // This also tests max_definition_level = 1
+  std::shared_ptr<PrimitiveArray> values = NullableArray<DoubleType>(100, 128, 10);
+
+  std::shared_ptr<GroupNode> schema = Schema(ParquetType::DOUBLE, Repetition::OPTIONAL);
+  FileWriter writer(default_memory_pool(), MakeWriter(schema));
+  ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values->length())));
+  ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values.get())));
+  ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
+
+  std::shared_ptr<Array> out;
+  ReadSingleColumnFile(ReaderFromSink(), &out);
+  ASSERT_TRUE(values->Equals(out));
+}
+
+TEST_F(TestParquetIO, SingleColumnInt64ChunkedWrite) {
+  std::shared_ptr<PrimitiveArray> values = NonNullArray<Int64Type>(100, 128);
+  std::shared_ptr<PrimitiveArray> values_chunk = NonNullArray<Int64Type>(25, 128);
+
+  std::shared_ptr<GroupNode> schema = Schema(ParquetType::INT64, Repetition::REQUIRED);
+  FileWriter writer(default_memory_pool(), MakeWriter(schema));
+  for (int i = 0; i < 4; i++) {
+    ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values_chunk->length())));
+    ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values_chunk.get())));
+  }
+  ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
+
+  std::shared_ptr<Array> out;
+  ReadSingleColumnFile(ReaderFromSink(), &out);
+  ASSERT_TRUE(values->Equals(out));
+}
+
+TEST_F(TestParquetIO, SingleColumnDoubleChunkedWrite) {
+  std::shared_ptr<PrimitiveArray> values = NullableArray<DoubleType>(100, 128, 10);
+  std::shared_ptr<PrimitiveArray> values_chunk_nulls =
+      NullableArray<DoubleType>(25, 128, 10);
+  std::shared_ptr<PrimitiveArray> values_chunk = NullableArray<DoubleType>(25, 128, 0);
+
+  std::shared_ptr<GroupNode> schema = Schema(ParquetType::DOUBLE, Repetition::OPTIONAL);
+  FileWriter writer(default_memory_pool(), MakeWriter(schema));
+  ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values_chunk_nulls->length())));
+  ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values_chunk_nulls.get())));
+  for (int i = 0; i < 3; i++) {
+    ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values_chunk->length())));
+    ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values_chunk.get())));
+  }
+  ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
+
+  std::shared_ptr<Array> out;
+  ReadSingleColumnFile(ReaderFromSink(), &out);
+  ASSERT_TRUE(values->Equals(out));
+}
+
+}  // namespace parquet
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/parquet-reader-test.cc b/cpp/src/arrow/parquet/parquet-reader-test.cc
deleted file mode 100644
index a7fc2a89f5f45..0000000000000
--- a/cpp/src/arrow/parquet/parquet-reader-test.cc
+++ /dev/null
@@ -1,116 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-#include "arrow/test-util.h"
-#include "arrow/parquet/reader.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
-
-#include "parquet/api/reader.h"
-#include "parquet/api/writer.h"
-
-using ParquetBuffer = parquet::Buffer;
-using parquet::BufferReader;
-using parquet::InMemoryOutputStream;
-using parquet::Int64Writer;
-using parquet::ParquetFileReader;
-using parquet::ParquetFileWriter;
-using parquet::RandomAccessSource;
-using parquet::Repetition;
-using parquet::SchemaDescriptor;
-using ParquetType = parquet::Type;
-using parquet::schema::GroupNode;
-using parquet::schema::NodePtr;
-using parquet::schema::PrimitiveNode;
-
-namespace arrow {
-
-namespace parquet {
-
-class TestReadParquet : public ::testing::Test {
- public:
-  virtual void SetUp() {}
-
-  std::shared_ptr<GroupNode> Int64Schema() {
-    auto pnode = PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64);
-    NodePtr node_ =
-        GroupNode::Make("schema", Repetition::REQUIRED, std::vector<NodePtr>({pnode}));
-    return std::static_pointer_cast<GroupNode>(node_);
-  }
-
-  std::unique_ptr<ParquetFileReader> Int64File(
-      std::vector<int64_t>& values, int num_chunks) {
-    std::shared_ptr<GroupNode> schema = Int64Schema();
-    std::shared_ptr<InMemoryOutputStream> sink(new InMemoryOutputStream());
-    auto file_writer = ParquetFileWriter::Open(sink, schema);
-    size_t chunk_size = values.size() / num_chunks;
-    for (int i = 0; i < num_chunks; i++) {
-      auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
-      auto column_writer = static_cast<Int64Writer*>(row_group_writer->NextColumn());
-      int64_t* data = values.data() + i * chunk_size;
-      column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
-      column_writer->Close();
-      row_group_writer->Close();
-    }
-    file_writer->Close();
-
-    std::shared_ptr<ParquetBuffer> buffer = sink->GetBuffer();
-    std::unique_ptr<RandomAccessSource> source(new BufferReader(buffer));
-    return ParquetFileReader::Open(std::move(source));
-  }
-
- private:
-};
-
-TEST_F(TestReadParquet, SingleColumnInt64) {
-  std::vector<int64_t> values(100, 128);
-  std::unique_ptr<ParquetFileReader> file_reader = Int64File(values, 1);
-  arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
-  std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
-  ASSERT_NO_THROW(ASSERT_OK(reader.GetFlatColumn(0, &column_reader)));
-  ASSERT_NE(nullptr, column_reader.get());
-  std::shared_ptr<Array> out;
-  ASSERT_OK(column_reader->NextBatch(100, &out));
-  ASSERT_NE(nullptr, out.get());
-  Int64Array* out_array = static_cast<Int64Array*>(out.get());
-  for (size_t i = 0; i < values.size(); i++) {
-    EXPECT_EQ(values[i], out_array->raw_data()[i]);
-  }
-}
-
-TEST_F(TestReadParquet, SingleColumnInt64Chunked) {
-  std::vector<int64_t> values(100, 128);
-  std::unique_ptr<ParquetFileReader> file_reader = Int64File(values, 4);
-  arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
-  std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
-  ASSERT_NO_THROW(ASSERT_OK(reader.GetFlatColumn(0, &column_reader)));
-  ASSERT_NE(nullptr, column_reader.get());
-  std::shared_ptr<Array> out;
-  ASSERT_OK(column_reader->NextBatch(100, &out));
-  ASSERT_NE(nullptr, out.get());
-  Int64Array* out_array = static_cast<Int64Array*>(out.get());
-  for (size_t i = 0; i < values.size(); i++) {
-    EXPECT_EQ(values[i], out_array->raw_data()[i]);
-  }
-}
-
-}  // namespace parquet
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
index 481ded5789a71..346de25360649 100644
--- a/cpp/src/arrow/parquet/reader.cc
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -26,6 +26,7 @@
 #include "arrow/util/status.h"
 
 using parquet::ColumnReader;
+using parquet::Repetition;
 using parquet::TypedColumnReader;
 
 namespace arrow {
@@ -36,6 +37,7 @@ class FileReader::Impl {
   Impl(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader);
   virtual ~Impl() {}
 
+  bool CheckForFlatColumn(const ::parquet::ColumnDescriptor* descr);
   Status GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out);
   Status ReadFlatColumn(int i, std::shared_ptr<Array>* out);
 
@@ -51,7 +53,7 @@ class FlatColumnReader::Impl {
   virtual ~Impl() {}
 
   Status NextBatch(int batch_size, std::shared_ptr<Array>* out);
-  template <typename ArrowType, typename ParquetType, typename CType>
+  template <typename ArrowType, typename ParquetType>
   Status TypedReadBatch(int batch_size, std::shared_ptr<Array>* out);
 
  private:
@@ -67,14 +69,28 @@ class FlatColumnReader::Impl {
 
   PoolBuffer values_buffer_;
   PoolBuffer def_levels_buffer_;
-  PoolBuffer rep_levels_buffer_;
+  PoolBuffer values_builder_buffer_;
+  PoolBuffer valid_bytes_buffer_;
 };
 
 FileReader::Impl::Impl(
     MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader)
     : pool_(pool), reader_(std::move(reader)) {}
 
+bool FileReader::Impl::CheckForFlatColumn(const ::parquet::ColumnDescriptor* descr) {
+  if ((descr->max_repetition_level() > 0) || (descr->max_definition_level() > 1)) {
+    return false;
+  } else if ((descr->max_definition_level() == 1) &&
+             (descr->schema_node()->repetition() != Repetition::OPTIONAL)) {
+    return false;
+  }
+  return true;
+}
+
 Status FileReader::Impl::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out) {
+  if (!CheckForFlatColumn(reader_->descr()->Column(i))) {
+    return Status::Invalid("The requested column is not flat");
+  }
   std::unique_ptr<FlatColumnReader::Impl> impl(
       new FlatColumnReader::Impl(pool_, reader_->descr()->Column(i), reader_.get(), i));
   *out = std::unique_ptr<FlatColumnReader>(new FlatColumnReader(std::move(impl)));
@@ -109,37 +125,50 @@ FlatColumnReader::Impl::Impl(MemoryPool* pool, const ::parquet::ColumnDescriptor
       column_index_(column_index),
       next_row_group_(0),
       values_buffer_(pool),
-      def_levels_buffer_(pool),
-      rep_levels_buffer_(pool) {
+      def_levels_buffer_(pool) {
   NodeToField(descr_->schema_node(), &field_);
   NextRowGroup();
 }
 
-template <typename ArrowType, typename ParquetType, typename CType>
+template <typename ArrowType, typename ParquetType>
 Status FlatColumnReader::Impl::TypedReadBatch(
     int batch_size, std::shared_ptr<Array>* out) {
   int values_to_read = batch_size;
   NumericBuilder<ArrowType> builder(pool_, field_->type);
   while ((values_to_read > 0) && column_reader_) {
-    values_buffer_.Resize(values_to_read * sizeof(CType));
+    values_buffer_.Resize(values_to_read * sizeof(typename ParquetType::c_type));
     if (descr_->max_definition_level() > 0) {
       def_levels_buffer_.Resize(values_to_read * sizeof(int16_t));
     }
-    if (descr_->max_repetition_level() > 0) {
-      rep_levels_buffer_.Resize(values_to_read * sizeof(int16_t));
-    }
     auto reader = dynamic_cast<TypedColumnReader<ParquetType>*>(column_reader_.get());
     int64_t values_read;
-    CType* values = reinterpret_cast<CType*>(values_buffer_.mutable_data());
-    PARQUET_CATCH_NOT_OK(
-        values_to_read -= reader->ReadBatch(values_to_read,
-            reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data()),
-            reinterpret_cast<int16_t*>(rep_levels_buffer_.mutable_data()), values,
-            &values_read));
+    int64_t levels_read;
+    int16_t* def_levels = reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
+    auto values =
+        reinterpret_cast<typename ParquetType::c_type*>(values_buffer_.mutable_data());
+    PARQUET_CATCH_NOT_OK(levels_read = reader->ReadBatch(
+                             values_to_read, def_levels, nullptr, values, &values_read));
+    values_to_read -= levels_read;
     if (descr_->max_definition_level() == 0) {
       RETURN_NOT_OK(builder.Append(values, values_read));
     } else {
-      return Status::NotImplemented("no support for definition levels yet");
+      // descr_->max_definition_level() == 1
+      RETURN_NOT_OK(values_builder_buffer_.Resize(
+          levels_read * sizeof(typename ParquetType::c_type)));
+      RETURN_NOT_OK(valid_bytes_buffer_.Resize(levels_read * sizeof(uint8_t)));
+      auto values_ptr = reinterpret_cast<typename ParquetType::c_type*>(
+          values_builder_buffer_.mutable_data());
+      uint8_t* valid_bytes = valid_bytes_buffer_.mutable_data();
+      int values_idx = 0;
+      for (int64_t i = 0; i < levels_read; i++) {
+        if (def_levels[i] < descr_->max_definition_level()) {
+          valid_bytes[i] = 0;
+        } else {
+          valid_bytes[i] = 1;
+          values_ptr[i] = values[values_idx++];
+        }
+      }
+      builder.Append(values_ptr, levels_read, valid_bytes);
     }
     if (!column_reader_->HasNext()) { NextRowGroup(); }
   }
@@ -147,9 +176,9 @@ Status FlatColumnReader::Impl::TypedReadBatch(
   return Status::OK();
 }
 
-#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType, CType)              \
-  case Type::ENUM:                                                         \
-    return TypedReadBatch<ArrowType, ParquetType, CType>(batch_size, out); \
+#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType)              \
+  case Type::ENUM:                                                  \
+    return TypedReadBatch<ArrowType, ParquetType>(batch_size, out); \
     break;
 
 Status FlatColumnReader::Impl::NextBatch(int batch_size, std::shared_ptr<Array>* out) {
@@ -159,15 +188,11 @@ Status FlatColumnReader::Impl::NextBatch(int batch_size, std::shared_ptr<Array>*
     return Status::OK();
   }
 
-  if (descr_->max_repetition_level() > 0) {
-    return Status::NotImplemented("no support for repetition yet");
-  }
-
   switch (field_->type->type) {
-    TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type, int32_t)
-    TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type, int64_t)
-    TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType, float)
-    TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType, double)
+    TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type)
+    TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
+    TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
     default:
       return Status::NotImplemented(field_->type->ToString());
   }
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
new file mode 100644
index 0000000000000..3ad2c5b073501
--- /dev/null
+++ b/cpp/src/arrow/parquet/writer.cc
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/parquet/writer.h"
+
+#include "arrow/array.h"
+#include "arrow/types/primitive.h"
+#include "arrow/parquet/utils.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+namespace parquet {
+
+class FileWriter::Impl {
+ public:
+  Impl(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer);
+
+  Status NewRowGroup(int64_t chunk_size);
+  template <typename ParquetType>
+  Status TypedWriteBatch(::parquet::ColumnWriter* writer, const PrimitiveArray* data);
+  Status WriteFlatColumnChunk(const PrimitiveArray* data);
+  Status Close();
+
+  virtual ~Impl() {}
+
+ private:
+  MemoryPool* pool_;
+  PoolBuffer data_buffer_;
+  PoolBuffer def_levels_buffer_;
+  std::unique_ptr<::parquet::ParquetFileWriter> writer_;
+  ::parquet::RowGroupWriter* row_group_writer_;
+};
+
+FileWriter::Impl::Impl(
+    MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer)
+    : pool_(pool),
+      data_buffer_(pool),
+      writer_(std::move(writer)),
+      row_group_writer_(nullptr) {}
+
+Status FileWriter::Impl::NewRowGroup(int64_t chunk_size) {
+  if (row_group_writer_ != nullptr) { PARQUET_CATCH_NOT_OK(row_group_writer_->Close()); }
+  PARQUET_CATCH_NOT_OK(row_group_writer_ = writer_->AppendRowGroup(chunk_size));
+  return Status::OK();
+}
+
+template <typename ParquetType>
+Status FileWriter::Impl::TypedWriteBatch(
+    ::parquet::ColumnWriter* column_writer, const PrimitiveArray* data) {
+  auto data_ptr =
+      reinterpret_cast<const typename ParquetType::c_type*>(data->data()->data());
+  auto writer =
+      reinterpret_cast<::parquet::TypedColumnWriter<ParquetType>*>(column_writer);
+  if (writer->descr()->max_definition_level() == 0) {
+    // no nulls, just dump the data
+    PARQUET_CATCH_NOT_OK(writer->WriteBatch(data->length(), nullptr, nullptr, data_ptr));
+  } else if (writer->descr()->max_definition_level() == 1) {
+    RETURN_NOT_OK(def_levels_buffer_.Resize(data->length() * sizeof(int16_t)));
+    int16_t* def_levels_ptr =
+        reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
+    if (data->null_count() == 0) {
+      std::fill(def_levels_ptr, def_levels_ptr + data->length(), 1);
+      PARQUET_CATCH_NOT_OK(
+          writer->WriteBatch(data->length(), def_levels_ptr, nullptr, data_ptr));
+    } else {
+      RETURN_NOT_OK(data_buffer_.Resize(
+          (data->length() - data->null_count()) * sizeof(typename ParquetType::c_type)));
+      auto buffer_ptr =
+          reinterpret_cast<typename ParquetType::c_type*>(data_buffer_.mutable_data());
+      int buffer_idx = 0;
+      for (size_t i = 0; i < data->length(); i++) {
+        if (data->IsNull(i)) {
+          def_levels_ptr[i] = 0;
+        } else {
+          def_levels_ptr[i] = 1;
+          buffer_ptr[buffer_idx++] = data_ptr[i];
+        }
+      }
+      PARQUET_CATCH_NOT_OK(
+          writer->WriteBatch(data->length(), def_levels_ptr, nullptr, buffer_ptr));
+    }
+  } else {
+    return Status::NotImplemented("no support for max definition level > 1 yet");
+  }
+  PARQUET_CATCH_NOT_OK(writer->Close());
+  return Status::OK();
+}
+
+Status FileWriter::Impl::Close() {
+  if (row_group_writer_ != nullptr) { PARQUET_CATCH_NOT_OK(row_group_writer_->Close()); }
+  PARQUET_CATCH_NOT_OK(writer_->Close());
+  return Status::OK();
+}
+
+#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType) \
+  case Type::ENUM:                                     \
+    return TypedWriteBatch<ParquetType>(writer, data); \
+    break;
+
+Status FileWriter::Impl::WriteFlatColumnChunk(const PrimitiveArray* data) {
+  ::parquet::ColumnWriter* writer;
+  PARQUET_CATCH_NOT_OK(writer = row_group_writer_->NextColumn());
+  switch (data->type_enum()) {
+    TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type)
+    TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
+    TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
+    default:
+      return Status::NotImplemented(data->type()->ToString());
+  }
+}
+
+FileWriter::FileWriter(
+    MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer)
+    : impl_(new FileWriter::Impl(pool, std::move(writer))) {}
+
+Status FileWriter::NewRowGroup(int64_t chunk_size) {
+  return impl_->NewRowGroup(chunk_size);
+}
+
+Status FileWriter::WriteFlatColumnChunk(const PrimitiveArray* data) {
+  return impl_->WriteFlatColumnChunk(data);
+}
+
+Status FileWriter::Close() {
+  return impl_->Close();
+}
+
+FileWriter::~FileWriter() {}
+
+}  // namespace parquet
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h
new file mode 100644
index 0000000000000..38f7d0b3a89d5
--- /dev/null
+++ b/cpp/src/arrow/parquet/writer.h
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PARQUET_WRITER_H
+#define ARROW_PARQUET_WRITER_H
+
+#include <memory>
+
+#include "parquet/api/schema.h"
+#include "parquet/api/writer.h"
+
+namespace arrow {
+
+class MemoryPool;
+class PrimitiveArray;
+class RowBatch;
+class Status;
+
+namespace parquet {
+
+/**
+ * Iterative API:
+ *  Start a new RowGroup/Chunk with NewRowGroup
+ *  Write column-by-column the whole column chunk
+ */
+class FileWriter {
+ public:
+  FileWriter(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer);
+
+  Status NewRowGroup(int64_t chunk_size);
+  Status WriteFlatColumnChunk(const PrimitiveArray* data);
+  Status Close();
+
+  virtual ~FileWriter();
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace parquet
+
+}  // namespace arrow
+
+#endif  // ARROW_PARQUET_WRITER_H

From c0985a47665f8ce8847a6a0215e6e3c0f1db28f4 Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent@dremio.com>
Date: Mon, 18 Apr 2016 11:07:22 -0700
Subject: [PATCH 0076/1644] Make BaseValueVector#MAX_ALLOCATION_SIZE
 configurable

This closes #65

Some of the tests are based on the assumption that the JVM can allocate at least
2GB of memory, which is not a common occurence (JVM usually defaults at 512MB).
Current Travis CI VM only have 3GB of memory total, which would have make challenging
to run some of the tests on them

Add a system property to change BaseValueVector.MAX_ALLOCATION_SIZE to allow to use
a much smaller value during tests.
---
 .../apache/arrow/vector/BaseValueVector.java  | 14 ++++----
 .../apache/arrow/vector/TestValueVector.java  | 36 +++++++++++++++----
 2 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 8bca3c005370e..932e6f13caf2b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -17,23 +17,24 @@
  */
 package org.apache.arrow.vector;
 
-import io.netty.buffer.ArrowBuf;
-
 import java.util.Iterator;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.types.MaterializedField;
 import org.apache.arrow.vector.util.TransferPair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+
+import io.netty.buffer.ArrowBuf;
+
 public abstract class BaseValueVector implements ValueVector {
   private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
 
-  public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
+  public static final String MAX_ALLOCATION_SIZE_PROPERTY = "arrow.vector.max_allocation_bytes";
+  public static final int MAX_ALLOCATION_SIZE = Integer.getInteger(MAX_ALLOCATION_SIZE_PROPERTY, Integer.MAX_VALUE);
   public static final int INITIAL_VALUE_ALLOCATION = 4096;
 
   protected final BufferAllocator allocator;
@@ -99,6 +100,7 @@ protected BaseMutator() { }
     public void generateTestData(int values) {}
 
     //TODO: consider making mutator stateless(if possible) on another issue.
+    @Override
     public void reset() {}
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index ac3eebe98eab7..b5c4509c8b540 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -23,16 +23,12 @@
 
 import java.nio.charset.Charset;
 
+import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.RepeatedListVector;
 import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
-import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.holders.BitHolder;
 import org.apache.arrow.vector.holders.IntHolder;
 import org.apache.arrow.vector.holders.NullableFloat4Holder;
@@ -44,10 +40,16 @@
 import org.apache.arrow.vector.holders.RepeatedVarBinaryHolder;
 import org.apache.arrow.vector.holders.UInt4Holder;
 import org.apache.arrow.vector.holders.VarCharHolder;
-import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExternalResource;
 
 
 public class TestValueVector {
@@ -57,6 +59,28 @@ public class TestValueVector {
 
   private BufferAllocator allocator;
 
+  // Rule to adjust MAX_ALLOCATION_SIZE and restore it back after the tests
+  @Rule
+  public final ExternalResource rule = new ExternalResource() {
+    private final String systemValue = System.getProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY);
+    private final String testValue = Long.toString(32*1024*1024);
+
+    @Override
+    protected void before() throws Throwable {
+      System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, testValue);
+    }
+
+    @Override
+    protected void after() {
+      if (systemValue != null) {
+        System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, systemValue);
+      }
+      else {
+        System.clearProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY);
+      }
+    }
+  };
+
   @Before
   public void init() {
     allocator = new RootAllocator(Long.MAX_VALUE);

From e316b3f765167fa1f45197061624e73332b095f4 Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent@dremio.com>
Date: Fri, 15 Apr 2016 14:00:19 -0700
Subject: [PATCH 0077/1644] Fix BaseAllocator.java NPE when assertions are
 disabled

This closes #64

When verifying memory using verifyAllocator() method, BaseAllocator throws NPE
if assertions are disabled.

Fixing this issue by checking first if assertion are disabled
---
 .../apache/arrow/memory/BaseAllocator.java    | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 90257bb9ffbf7..f1503c902d0be 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -99,6 +99,7 @@ protected BaseAllocator(
 
   }
 
+  @Override
   public void assertOpen() {
     if (AssertionUtil.ASSERT_ENABLED) {
       if (isClosed) {
@@ -287,6 +288,7 @@ public Reservation() {
       }
     }
 
+    @Override
     public boolean add(final int nBytes) {
       assertOpen();
 
@@ -308,6 +310,7 @@ public boolean add(final int nBytes) {
       return true;
     }
 
+    @Override
     public ArrowBuf allocateBuffer() {
       assertOpen();
 
@@ -319,14 +322,17 @@ public ArrowBuf allocateBuffer() {
       return arrowBuf;
     }
 
+    @Override
     public int getSize() {
       return nBytes;
     }
 
+    @Override
     public boolean isUsed() {
       return used;
     }
 
+    @Override
     public boolean isClosed() {
       return closed;
     }
@@ -364,6 +370,7 @@ public void close() {
       closed = true;
     }
 
+    @Override
     public boolean reserve(int nBytes) {
       assertOpen();
 
@@ -509,6 +516,7 @@ public synchronized void close() {
 
   }
 
+  @Override
   public String toString() {
     final Verbosity verbosity = logger.isTraceEnabled() ? Verbosity.LOG_WITH_STACKTRACE
         : Verbosity.BASIC;
@@ -523,6 +531,7 @@ public String toString() {
    *
    * @return A Verbose string of current allocator state.
    */
+  @Override
   public String toVerboseString() {
     final StringBuilder sb = new StringBuilder();
     print(sb, 0, Verbosity.LOG_WITH_STACKTRACE);
@@ -575,13 +584,12 @@ void verifyAllocator() {
    *           when any problems are found
    */
   private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, BaseAllocator> buffersSeen) {
-    synchronized (DEBUG_LOCK) {
-
-      // The remaining tests can only be performed if we're in debug mode.
-      if (!DEBUG) {
-        return;
-      }
+    // The remaining tests can only be performed if we're in debug mode.
+    if (!DEBUG) {
+      return;
+    }
 
+    synchronized (DEBUG_LOCK) {
       final long allocated = getAllocatedMemory();
 
       // verify my direct descendants

From 703546787e049f1abbc96082f60fe4d08731a5ce Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent.goujon@online.fr>
Date: Wed, 13 Apr 2016 22:36:38 -0700
Subject: [PATCH 0078/1644] Add java support to Travis CI

Add java support to Travis CI using oracle JDK7 on a Linux host.
---
 .travis.yml              |  6 +++++-
 ci/travis_script_java.sh | 11 +++++++++++
 java/pom.xml             |  2 +-
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100755 ci/travis_script_java.sh

diff --git a/.travis.yml b/.travis.yml
index 646f80fee7b3b..7c4183700ca10 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,6 @@ addons:
     - gcc-4.9   # Needed for C++11
     - g++-4.9   # Needed for C++11
     - gdb
-    - gcov
     - ccache
     - cmake
     - valgrind
@@ -60,6 +59,11 @@ matrix:
     before_install:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_conda_build.sh
+  - language: java
+    os: linux
+    jdk: oraclejdk7
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
 
 before_install:
 - ulimit -c unlimited -S
diff --git a/ci/travis_script_java.sh b/ci/travis_script_java.sh
new file mode 100755
index 0000000000000..2d11eaeb4c57d
--- /dev/null
+++ b/ci/travis_script_java.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+
+pushd $JAVA_DIR
+
+mvn -B test
+
+popd
diff --git a/java/pom.xml b/java/pom.xml
index 4ee4ff4f7604e..ea42894fda22e 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -297,7 +297,7 @@
           <artifactId>maven-surefire-plugin</artifactId>
           <version>2.17</version>
           <configuration>
-            <argLine>-ea</argLine>
+            <enableAssertions>true</enableAssertions>
             <forkCount>${forkCount}</forkCount>
             <reuseForks>true</reuseForks>
             <systemPropertyVariables>

From cd1d770ede57f08b8be2f2b42f2f629eb5106098 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Mon, 23 May 2016 13:55:51 -0700
Subject: [PATCH 0079/1644] ARROW-206: Expose a C++ api to compare ranges of
 slots between two arrays

@wesm the need for this grew out of @fengguangyuan PR to add struct type (#66) and struct builder.  I considered a different APIs before settling on this:
1.  Add an API that took the parent bitmask (this potentially has possibility of being the most performant, but would have a more awkward contract then provided here)
2.  Add an equality comparison for a single slot (leaves the least amount of room for optimization but it would be the simplest to implement).
3.  This API which potentially leaves some room for optimization but I think places the least requirements on the caller.

Let me know if you would prefer a different API.

WIP because I need to add more unit tests (I also need to think about if it is worth mirroring the EqualsExact in addition to the Equals method).  Which I should get to by the end of the weekend.

@fengguangyuan let me know if this makes sense to you as a way forward on your PR

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #80 from emkornfield/emk_add_equality and squashes the following commits:

d5ae777 [Micah Kornfield] remove todo, its handled by type_traits
f963639 [Micah Kornfield] add in check for null arrays
f5c6bd5 [Micah Kornfield] make format/lint check
dcbaad4 [Micah Kornfield] unittests passing
318855d [Micah Kornfield] working primitive tests
dadb244 [Micah Kornfield] wip expose range equality to to allow for nested comparisons
---
 cpp/src/arrow/array-test.cc           | 29 +++++++++++++++
 cpp/src/arrow/array.cc                |  7 ++++
 cpp/src/arrow/array.h                 |  9 ++++-
 cpp/src/arrow/types/list-test.cc      | 36 +++++++++++++++++++
 cpp/src/arrow/types/list.cc           | 26 ++++++++++++++
 cpp/src/arrow/types/list.h            |  3 ++
 cpp/src/arrow/types/primitive-test.cc | 51 +++++++++++++++++++++++++++
 cpp/src/arrow/types/primitive.cc      | 17 ++++++++-
 cpp/src/arrow/types/primitive.h       | 35 ++++++++++++++----
 9 files changed, 205 insertions(+), 8 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index b4c727997ee7e..3b4736327b47c 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -56,6 +56,35 @@ TEST_F(TestArray, TestLength) {
   ASSERT_EQ(arr->length(), 100);
 }
 
+ArrayPtr MakeArrayFromValidBytes(const std::vector<uint8_t>& v, MemoryPool* pool) {
+  int32_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
+  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(v);
+
+  BufferBuilder value_builder(pool);
+  for (size_t i = 0; i < v.size(); ++i) {
+    value_builder.Append<int32_t>(0);
+  }
+
+  ArrayPtr arr(new Int32Array(v.size(), value_builder.Finish(), null_count, null_buf));
+  return arr;
+}
+
+TEST_F(TestArray, TestEquality) {
+  auto array = MakeArrayFromValidBytes({1, 0, 1, 1, 0, 1, 0, 0}, pool_);
+  auto equal_array = MakeArrayFromValidBytes({1, 0, 1, 1, 0, 1, 0, 0}, pool_);
+  auto unequal_array = MakeArrayFromValidBytes({1, 1, 1, 1, 0, 1, 0, 0}, pool_);
+
+  EXPECT_TRUE(array->Equals(array));
+  EXPECT_TRUE(array->Equals(equal_array));
+  EXPECT_TRUE(equal_array->Equals(array));
+  EXPECT_FALSE(equal_array->Equals(unequal_array));
+  EXPECT_FALSE(unequal_array->Equals(equal_array));
+  EXPECT_TRUE(array->RangeEquals(4, 8, 4, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(0, 4, 0, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(0, 8, 0, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
+}
+
 TEST_F(TestArray, TestIsNull) {
   // clang-format off
   std::vector<uint8_t> null_bitmap = {1, 0, 1, 1, 0, 1, 0, 0,
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index c6b9b1599cdd2..d6b081f315532 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -58,4 +58,11 @@ bool NullArray::Equals(const std::shared_ptr<Array>& arr) const {
   return arr->length() == length_;
 }
 
+bool NullArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_index,
+    const std::shared_ptr<Array>& arr) const {
+  if (!arr) { return false; }
+  if (Type::NA != arr->type_enum()) { return false; }
+  return true;
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index f98c4c28310f8..76dc0f598141f 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -59,6 +59,12 @@ class Array {
 
   bool EqualsExact(const Array& arr) const;
   virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
+
+  // Compare if the range of slots specified are equal for the given array and
+  // this array.  end_idx exclusive.  This methods does not bounds check.
+  virtual bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const std::shared_ptr<Array>& arr) const = 0;
+
   // Determines if the array is internally consistent.  Defaults to always
   // returning Status::OK.  This can be an expensive check.
   virtual Status Validate() const;
@@ -85,10 +91,11 @@ class NullArray : public Array {
   explicit NullArray(int32_t length) : NullArray(std::make_shared<NullType>(), length) {}
 
   bool Equals(const std::shared_ptr<Array>& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_index,
+      const std::shared_ptr<Array>& arr) const override;
 };
 
 typedef std::shared_ptr<Array> ArrayPtr;
-
 }  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index 6a8ad9aa59ead..2e41b4a61caf2 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -86,6 +86,42 @@ class TestListBuilder : public TestBuilder {
   shared_ptr<ListArray> result_;
 };
 
+TEST_F(TestListBuilder, Equality) {
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+
+  ArrayPtr array, equal_array, unequal_array;
+  vector<int32_t> equal_offsets = {0, 1, 2, 5};
+  vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2};
+  vector<int32_t> unequal_offsets = {0, 1, 4};
+  vector<int32_t> unequal_values = {1, 2, 2, 2, 3, 4, 5};
+
+  // setup two equal arrays
+  ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
+  ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
+  array = builder_->Finish();
+  ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
+  ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
+  equal_array = builder_->Finish();
+  // now an unequal one
+  ASSERT_OK(builder_->Append(unequal_offsets.data(), unequal_offsets.size()));
+  ASSERT_OK(vb->Append(unequal_values.data(), unequal_values.size()));
+  unequal_array = builder_->Finish();
+
+  // Test array equality
+  EXPECT_TRUE(array->Equals(array));
+  EXPECT_TRUE(array->Equals(equal_array));
+  EXPECT_TRUE(equal_array->Equals(array));
+  EXPECT_FALSE(equal_array->Equals(unequal_array));
+  EXPECT_FALSE(unequal_array->Equals(equal_array));
+
+  // Test range equality
+  EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
+  EXPECT_TRUE(array->RangeEquals(2, 3, 2, unequal_array));
+  EXPECT_TRUE(array->RangeEquals(3, 4, 1, unequal_array));
+}
+
 TEST_F(TestListBuilder, TestResize) {}
 
 TEST_F(TestListBuilder, TestAppendNull) {
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index 76e7fe5f4d429..6334054caf84a 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -44,6 +44,32 @@ bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
   return EqualsExact(*static_cast<const ListArray*>(arr.get()));
 }
 
+bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  const auto other = static_cast<ListArray*>(arr.get());
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    const bool is_null = IsNull(i);
+    if (is_null != arr->IsNull(o_i)) { return false; }
+    if (is_null) continue;
+    const int32_t begin_offset = offset(i);
+    const int32_t end_offset = offset(i + 1);
+    const int32_t other_begin_offset = other->offset(o_i);
+    const int32_t other_end_offset = other->offset(o_i + 1);
+    // Underlying can't be equal if the size isn't equal
+    if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
+      return false;
+    }
+    if (!values_->RangeEquals(
+            begin_offset, end_offset, other_begin_offset, other->values())) {
+      return false;
+    }
+  }
+  return true;
+}
+
 Status ListArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
   if (!offset_buf_) { return Status::Invalid("offset_buf_ was null"); }
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index a020b8ad22668..0a3941633eb83 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -72,6 +72,9 @@ class ListArray : public Array {
   bool EqualsExact(const ListArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
 
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const ArrayPtr& arr) const override;
+
  protected:
   std::shared_ptr<Buffer> offset_buf_;
   const int32_t* offsets_;
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 2b4c0879a28f4..87eb0fe3a8bf7 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -304,6 +304,57 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
   ASSERT_EQ(memory_before, this->pool_->bytes_allocated());
 }
 
+template <class T, class Builder>
+Status MakeArray(const vector<uint8_t>& valid_bytes, const vector<T>& draws, int size,
+    Builder* builder, ArrayPtr* out) {
+  // Append the first 1000
+  for (int i = 0; i < size; ++i) {
+    if (valid_bytes[i] > 0) {
+      RETURN_NOT_OK(builder->Append(draws[i]));
+    } else {
+      RETURN_NOT_OK(builder->AppendNull());
+    }
+  }
+  *out = builder->Finish();
+  return Status::OK();
+}
+
+TYPED_TEST(TestPrimitiveBuilder, Equality) {
+  DECL_T();
+
+  const int size = 1000;
+  this->RandomData(size);
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
+  ArrayPtr array, equal_array, unequal_array;
+  auto builder = this->builder_.get();
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &equal_array));
+
+  // Make the not equal array by negating the first valid element with itself.
+  const auto first_valid = std::find_if(
+      valid_bytes.begin(), valid_bytes.end(), [](uint8_t valid) { return valid > 0; });
+  const int first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
+  // This should be true with a very high probability, but might introduce flakiness
+  ASSERT_LT(first_valid_idx, size - 1);
+  draws[first_valid_idx] = ~*reinterpret_cast<int64_t*>(&draws[first_valid_idx]);
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &unequal_array));
+
+  // test normal equality
+  EXPECT_TRUE(array->Equals(array));
+  EXPECT_TRUE(array->Equals(equal_array));
+  EXPECT_TRUE(equal_array->Equals(array));
+  EXPECT_FALSE(equal_array->Equals(unequal_array));
+  EXPECT_FALSE(unequal_array->Equals(equal_array));
+
+  // Test range equality
+  EXPECT_FALSE(array->RangeEquals(0, first_valid_idx + 1, 0, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(first_valid_idx, size, first_valid_idx, unequal_array));
+  EXPECT_TRUE(array->RangeEquals(0, first_valid_idx, 0, unequal_array));
+  EXPECT_TRUE(
+      array->RangeEquals(first_valid_idx + 1, size, first_valid_idx + 1, unequal_array));
+}
+
 TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   DECL_T();
 
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 57a3f1e4e150b..8e6c0f809ca44 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -185,10 +185,25 @@ bool BooleanArray::EqualsExact(const BooleanArray& other) const {
   }
 }
 
-bool BooleanArray::Equals(const std::shared_ptr<Array>& arr) const {
+bool BooleanArray::Equals(const ArrayPtr& arr) const {
   if (this == arr.get()) return true;
   if (Type::BOOL != arr->type_enum()) { return false; }
   return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
 }
 
+bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
+    int32_t other_start_idx, const ArrayPtr& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  const auto other = static_cast<BooleanArray*>(arr.get());
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    const bool is_null = IsNull(i);
+    if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
+      return false;
+    }
+  }
+  return true;
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index fc45f6c5b0568..9597fc8363138 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -66,6 +66,22 @@ class PrimitiveArray : public Array {
       return PrimitiveArray::EqualsExact(*static_cast<const PrimitiveArray*>(&other)); \
     }                                                                                  \
                                                                                        \
+    bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,      \
+        const ArrayPtr& arr) const override {                                          \
+      if (this == arr.get()) { return true; }                                          \
+      if (!arr) { return false; }                                                      \
+      if (this->type_enum() != arr->type_enum()) { return false; }                     \
+      const auto other = static_cast<NAME*>(arr.get());                                \
+      for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {    \
+        const bool is_null = IsNull(i);                                                \
+        if (is_null != arr->IsNull(o_i) ||                                             \
+            (!is_null && Value(i) != other->Value(o_i))) {                             \
+          return false;                                                                \
+        }                                                                              \
+      }                                                                                \
+      return true;                                                                     \
+    }                                                                                  \
+                                                                                       \
     const T* raw_data() const { return reinterpret_cast<const T*>(raw_data_); }        \
                                                                                        \
     T Value(int i) const { return raw_data()[i]; }                                     \
@@ -95,8 +111,10 @@ class PrimitiveBuilder : public ArrayBuilder {
   using ArrayBuilder::Advance;
 
   // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
-  void AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+  Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+    RETURN_NOT_OK(Reserve(length));
     UnsafeAppendToBitmap(valid_bytes, length);
+    return Status::OK();
   }
 
   Status AppendNull() {
@@ -139,9 +157,10 @@ class NumericBuilder : public PrimitiveBuilder<T> {
   using PrimitiveBuilder<T>::Reserve;
 
   // Scalar append.
-  void Append(value_type val) {
-    ArrayBuilder::Reserve(1);
+  Status Append(value_type val) {
+    RETURN_NOT_OK(ArrayBuilder::Reserve(1));
     UnsafeAppend(val);
+    return Status::OK();
   }
 
   // Does not capacity-check; make sure to call Reserve beforehand
@@ -248,7 +267,9 @@ class BooleanArray : public PrimitiveArray {
       int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   bool EqualsExact(const BooleanArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
+  bool Equals(const ArrayPtr& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const ArrayPtr& arr) const override;
 
   const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
 
@@ -274,7 +295,8 @@ class BooleanBuilder : public PrimitiveBuilder<BooleanType> {
   using PrimitiveBuilder<BooleanType>::Append;
 
   // Scalar append
-  void Append(bool val) {
+  Status Append(bool val) {
+    Reserve(1);
     util::set_bit(null_bitmap_data_, length_);
     if (val) {
       util::set_bit(raw_data_, length_);
@@ -282,9 +304,10 @@ class BooleanBuilder : public PrimitiveBuilder<BooleanType> {
       util::clear_bit(raw_data_, length_);
     }
     ++length_;
+    return Status::OK();
   }
 
-  void Append(uint8_t val) { Append(static_cast<bool>(val)); }
+  Status Append(uint8_t val) { return Append(static_cast<bool>(val)); }
 };
 
 }  // namespace arrow

From c8b8078810be1d703c0261859b0862d574384600 Mon Sep 17 00:00:00 2001
From: Edmon Begoli <ebegoli@gmail.com>
Date: Sat, 28 May 2016 19:11:47 -0400
Subject: [PATCH 0080/1644] [Doc] Update Layout.md

For clarity, added references to official SIMD documentation, the description
of Endiandness, Parquet.  Used Markdown syntax for the exponent to document the
size of the arrays.

Closes PR #82.
---
 format/Layout.md | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index 34eade313415a..9de0479738ac5 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -41,7 +41,7 @@ Base requirements
   proprietary systems that utilize the open source components.
 * All array slots are accessible in constant time, with complexity growing
   linearly in the nesting level
-* Capable of representing fully-materialized and decoded / decompressed Parquet
+* Capable of representing fully-materialized and decoded / decompressed [Parquet][5]
   data
 * All contiguous memory buffers are aligned at 64-byte boundaries and padded to a multiple of 64 bytes.
 * Any relative type can have null slots
@@ -76,7 +76,7 @@ Base requirements
 * Any memory management or reference counting subsystem
 * To enumerate or specify types of encodings or compression support
 
-## Byte Order (Endianness)
+## Byte Order ([Endianness][3])
 
 The Arrow format is little endian.
 
@@ -91,7 +91,7 @@ requirement follows best practices for optimized memory access:
 * 64 byte alignment is recommended by the [Intel performance guide][2] for
 data-structures over 64 bytes (which will be a common case for Arrow Arrays).
 
-Requiring padding to a multiple of 64 bytes allows for using SIMD instructions
+Requiring padding to a multiple of 64 bytes allows for using [SIMD][4] instructions
 consistently in loops without additional conditional checks.
 This should allow for simpler and more efficient code.  
 The specific padding length was chosen because it matches the largest known
@@ -105,13 +105,13 @@ Unless otherwise noted, padded bytes do not need to have a specific value.
 ## Array lengths
 
 Any array has a known and fixed length, stored as a 32-bit signed integer, so a
-maximum of 2^31 - 1 elements. We choose a signed int32 for a couple reasons:
+maximum of 2<sup>31</sup> - 1 elements. We choose a signed int32 for a couple reasons:
 
 * Enhance compatibility with Java and client languages which may have varying
   quality of support for unsigned integers.
 * To encourage developers to compose smaller arrays (each of which contains
   contiguous memory in its leaf nodes) to create larger array structures
-  possibly exceeding 2^31 - 1 elements, as opposed to allocating very large
+  possibly exceeding 2<sup>31</sup> - 1 elements, as opposed to allocating very large
   contiguous memory blocks.
 
 ## Null count
@@ -238,7 +238,7 @@ A list-array is represented by the combination of the following:
 * A values array, a child array of type T. T may also be a nested type.
 * An offsets buffer containing 32-bit signed integers with length equal to the
   length of the top-level array plus one. Note that this limits the size of the
-  values array to 2^31 -1.
+  values array to 2<sup>31</sup>-1.
 
 The offsets array encodes a start position in the values array, and the length
 of the value in each slot is computed using the first difference with the next
@@ -578,7 +578,11 @@ the the types array indicates that a slot contains a different type at the index
 
 ## References
 
-Drill docs https://drill.apache.org/docs/value-vectors/
+Apache Drill Documentation - [Value Vectors][6] 
 
 [1]: https://en.wikipedia.org/wiki/Bit_numbering
 [2]: https://software.intel.com/en-us/articles/practical-intel-avx-optimization-on-2nd-generation-intel-core-processors
+[3]: https://en.wikipedia.org/wiki/Endianness
+[4]: https://software.intel.com/en-us/node/600110
+[5]: https://parquet.apache.org/documentation/latest/
+[6]: https://drill.apache.org/docs/value-vectors/

From 65740950c852b82c475ca84e970e147d25d27398 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 2 Jun 2016 18:36:43 -0700
Subject: [PATCH 0081/1644] ARROW-209: [C++] Triage builds due to unavailable
 LLVM apt repo

For now, this unblocks builds until we can resolve the LLVM apt issue.

Author: Wes McKinney <wesm@apache.org>

Closes #84 from wesm/ARROW-209 and squashes the following commits:

c6bf166 [Wes McKinney] Remove clang-* packages from apt list
30d8c5c [Wes McKinney] Temporarily disable clang-format and clang-tidy checks in Travis CI build
---
 .travis.yml             |  3 ---
 ci/travis_script_cpp.sh | 12 ++++++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7c4183700ca10..ac2b0d457cb8e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,10 +5,7 @@ addons:
     sources:
     - ubuntu-toolchain-r-test
     - kalakris-cmake
-    - llvm-toolchain-precise-3.7
     packages:
-    - clang-format-3.7
-    - clang-tidy-3.7
     - gcc-4.9   # Needed for C++11
     - g++-4.9   # Needed for C++11
     - gdb
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index c9b3b5f1442a1..9cf4f8e352109 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -7,10 +7,14 @@ set -e
 pushd $CPP_BUILD_DIR
 
 make lint
-if [ $TRAVIS_OS_NAME == "linux" ]; then
-  make check-format
-  make check-clang-tidy
-fi
+
+# ARROW-209: checks depending on the LLVM toolchain are disabled temporarily
+# until we are able to install the full LLVM toolchain in Travis CI again
+
+# if [ $TRAVIS_OS_NAME == "linux" ]; then
+#   make check-format
+#   make check-clang-tidy
+# fi
 
 ctest -L unittest
 

From ce2fe7a782c9c1f84a6ccdc2b7b00768d535d8fc Mon Sep 17 00:00:00 2001
From: Smyatkin Maxim <smyatkinmaxim@gmail.com>
Date: Mon, 6 Jun 2016 23:25:31 -0700
Subject: [PATCH 0082/1644] ARROW-211: [Format] Fixed typos in layout examples

Just a few typo fixes according to the ticket.

Author: Smyatkin Maxim <smyatkinmaxim@gmail.com>

Closes #86 from Smyatkin-Maxim/ARROW-211 and squashes the following commits:

6cefba6 [Smyatkin Maxim] ARROW-211: [Format] Fixed typos in layout examples
---
 format/Layout.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index 9de0479738ac5..815c47f2c934b 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -299,7 +299,7 @@ will be be represented as follows:
 
   | Bytes 0-3  | Bytes 4-7  | Bytes 8-11 | Bytes 12-15 | Bytes 16-63 |
   |------------|------------|------------|-------------|-------------|
-  | 0          |  2         |  6         |  7          | unspecified |
+  | 0          |  2         |  5         |  6          | unspecified |
 
 * Values array (`List<byte>`)
   * Length: 6, Null count: 1
@@ -368,7 +368,7 @@ The layout for [{'joe', 1}, {null, 2}, null, {'mark', 4}] would be:
 
       | Byte 0 (validity bitmap) | Bytes 1-7             |
       |--------------------------|-----------------------|
-      | 00011101                 | 0 (padding)           |
+      | 00001101                 | 0 (padding)           |
 
     * Offsets buffer:
 
@@ -472,7 +472,7 @@ An example layout for logical union of:
       | 1.2, 3.4  | unspecified |
 
 
-  * Field-1 array (f: float):
+  * Field-1 array (i: int32):
     * Length: 1, nulls: 0
     * Null bitmap buffer: Not required
 
@@ -499,7 +499,7 @@ union, it has some advantages that may be desirable in certain use cases:
 
 For the union array:
 
-[{u0=5}, {u1=1.2}, {u2='joe'}, {u1=3.4}, {u0=4}, 'mark']
+[{u0=5}, {u1=1.2}, {u2='joe'}, {u1=3.4}, {u0=4}, {u2='mark'}]
 
 will have the following layout:
 ```

From 9ce13a06726874c04433100127f74e6ea4afa855 Mon Sep 17 00:00:00 2001
From: fengguangyuan <root@node20.(none)>
Date: Mon, 6 Jun 2016 23:32:38 -0700
Subject: [PATCH 0083/1644] ARROW-60: [C++] Struct type builder API

Implement the basic classes, `StructArray` and `StructBuilder,` meanwhile,

add the perspective test cases for them.

Other necessary methods will be added subsequently.

Author: fengguangyuan <root@node20.(none)>

Closes #66 from fengguangyuan/ARROW-60 and squashes the following commits:

190967f [fengguangyuan] ARROW-60: [C++] Struct type builder API Add field index and TODO comment.
ae74c80 [fengguangyuan] ARROW-60: Struct type builder API Add RangeEquals method to implement Equals method.
fa856fd [fengguangyuan] ARROW-60:[C++] Struct typebuilder API Modify Validate() refered to the specification.
bfabdc1 [fengguangyuan] ARROW-60: Struct type builder API Refine the previous committed patch. Add validate methods for testing StructArray and StructBuilder. TODO, Equals methods also need to be tested, but now it's not convient to do it.
5733de7 [fengguangyuan] ARROW-60: Struct type builder API
---
 cpp/src/arrow/type.h               |   1 +
 cpp/src/arrow/types/construct.cc   |  15 ++
 cpp/src/arrow/types/construct.h    |   3 +-
 cpp/src/arrow/types/struct-test.cc | 332 +++++++++++++++++++++++++++++
 cpp/src/arrow/types/struct.cc      |  72 ++++++-
 cpp/src/arrow/types/struct.h       |  97 ++++++++-
 6 files changed, 517 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 77404cd702524..f366645cd5cf9 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -161,6 +161,7 @@ struct Field {
 
   std::string ToString() const;
 };
+typedef std::shared_ptr<Field> FieldPtr;
 
 template <typename Derived>
 struct PrimitiveType : public DataType {
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 78036d4bf5711..bcb0ec490901f 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -23,6 +23,7 @@
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
+#include "arrow/types/struct.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
 
@@ -66,6 +67,20 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
       out->reset(new ListBuilder(pool, value_builder));
       return Status::OK();
     }
+
+    case Type::STRUCT: {
+      std::vector<FieldPtr>& fields = type->children_;
+      std::vector<std::shared_ptr<ArrayBuilder>> values_builder;
+
+      for (auto it : fields) {
+        std::shared_ptr<ArrayBuilder> builder;
+        RETURN_NOT_OK(MakeBuilder(pool, it->type, &builder));
+        values_builder.push_back(builder);
+      }
+      out->reset(new StructBuilder(pool, type, values_builder));
+      return Status::OK();
+    }
+
     default:
       return Status::NotImplemented(type->ToString());
   }
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index 43c0018c67e41..d0370840ca108 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -20,13 +20,14 @@
 
 #include <cstdint>
 #include <memory>
-
+#include <vector>
 namespace arrow {
 
 class Array;
 class ArrayBuilder;
 class Buffer;
 struct DataType;
+struct Field;
 class MemoryPool;
 class Status;
 
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index 79d560e19bcc0..d2bd2971d0438 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -21,7 +21,16 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/test-util.h"
 #include "arrow/type.h"
+#include "arrow/types/construct.h"
+#include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
+#include "arrow/types/struct.h"
+#include "arrow/types/test-common.h"
+#include "arrow/util/status.h"
 
 using std::shared_ptr;
 using std::string;
@@ -52,4 +61,327 @@ TEST(TestStructType, Basics) {
   // TODO(wesm): out of bounds for field(...)
 }
 
+void ValidateBasicStructArray(const StructArray* result,
+    const vector<uint8_t>& struct_is_valid, const vector<char>& list_values,
+    const vector<uint8_t>& list_is_valid, const vector<int>& list_lengths,
+    const vector<int>& list_offsets, const vector<int32_t>& int_values) {
+  ASSERT_EQ(4, result->length());
+  ASSERT_OK(result->Validate());
+
+  auto list_char_arr = static_cast<ListArray*>(result->field(0).get());
+  auto char_arr = static_cast<Int8Array*>(list_char_arr->values().get());
+  auto int32_arr = static_cast<Int32Array*>(result->field(1).get());
+
+  ASSERT_EQ(0, result->null_count());
+  ASSERT_EQ(1, list_char_arr->null_count());
+  ASSERT_EQ(0, int32_arr->null_count());
+
+  // List<char>
+  ASSERT_EQ(4, list_char_arr->length());
+  ASSERT_EQ(10, list_char_arr->values()->length());
+  for (size_t i = 0; i < list_offsets.size(); ++i) {
+    ASSERT_EQ(list_offsets[i], list_char_arr->offsets()[i]);
+  }
+  for (size_t i = 0; i < list_values.size(); ++i) {
+    ASSERT_EQ(list_values[i], char_arr->Value(i));
+  }
+
+  // Int32
+  ASSERT_EQ(4, int32_arr->length());
+  for (size_t i = 0; i < int_values.size(); ++i) {
+    ASSERT_EQ(int_values[i], int32_arr->Value(i));
+  }
+}
+
+// ----------------------------------------------------------------------------------
+// Struct test
+class TestStructBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+
+    auto int32_type = TypePtr(new Int32Type());
+    auto char_type = TypePtr(new Int8Type());
+    auto list_type = TypePtr(new ListType(char_type));
+
+    std::vector<TypePtr> types = {list_type, int32_type};
+    std::vector<FieldPtr> fields;
+    fields.push_back(FieldPtr(new Field("list", list_type)));
+    fields.push_back(FieldPtr(new Field("int", int32_type)));
+
+    type_ = TypePtr(new StructType(fields));
+    value_fields_ = fields;
+
+    std::shared_ptr<ArrayBuilder> tmp;
+    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+
+    builder_ = std::dynamic_pointer_cast<StructBuilder>(tmp);
+    ASSERT_EQ(2, builder_->field_builders().size());
+  }
+
+  void Done() { result_ = std::dynamic_pointer_cast<StructArray>(builder_->Finish()); }
+
+ protected:
+  std::vector<FieldPtr> value_fields_;
+  TypePtr type_;
+
+  std::shared_ptr<StructBuilder> builder_;
+  std::shared_ptr<StructArray> result_;
+};
+
+TEST_F(TestStructBuilder, TestAppendNull) {
+  ASSERT_OK(builder_->AppendNull());
+  ASSERT_OK(builder_->AppendNull());
+  ASSERT_EQ(2, builder_->field_builders().size());
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  ASSERT_OK(list_vb->AppendNull());
+  ASSERT_OK(list_vb->AppendNull());
+  ASSERT_EQ(2, list_vb->length());
+
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ASSERT_OK(int_vb->AppendNull());
+  ASSERT_OK(int_vb->AppendNull());
+  ASSERT_EQ(2, int_vb->length());
+
+  Done();
+
+  ASSERT_OK(result_->Validate());
+
+  ASSERT_EQ(2, result_->fields().size());
+  ASSERT_EQ(2, result_->length());
+  ASSERT_EQ(2, result_->field(0)->length());
+  ASSERT_EQ(2, result_->field(1)->length());
+  ASSERT_TRUE(result_->IsNull(0));
+  ASSERT_TRUE(result_->IsNull(1));
+  ASSERT_TRUE(result_->field(0)->IsNull(0));
+  ASSERT_TRUE(result_->field(0)->IsNull(1));
+  ASSERT_TRUE(result_->field(1)->IsNull(0));
+  ASSERT_TRUE(result_->field(1)->IsNull(1));
+
+  ASSERT_EQ(Type::LIST, result_->field(0)->type_enum());
+  ASSERT_EQ(Type::INT32, result_->field(1)->type_enum());
+}
+
+TEST_F(TestStructBuilder, TestBasics) {
+  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6, 10};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ASSERT_EQ(2, builder_->field_builders().size());
+
+  EXPECT_OK(builder_->Resize(list_lengths.size()));
+  EXPECT_OK(char_vb->Resize(list_values.size()));
+  EXPECT_OK(int_vb->Resize(int_values.size()));
+
+  int pos = 0;
+  for (size_t i = 0; i < list_lengths.size(); ++i) {
+    ASSERT_OK(list_vb->Append(list_is_valid[i] > 0));
+    int_vb->UnsafeAppend(int_values[i]);
+    for (int j = 0; j < list_lengths[i]; ++j) {
+      char_vb->UnsafeAppend(list_values[pos++]);
+    }
+  }
+
+  for (size_t i = 0; i < struct_is_valid.size(); ++i) {
+    ASSERT_OK(builder_->Append(struct_is_valid[i] > 0));
+  }
+
+  Done();
+
+  ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
+      list_lengths, list_offsets, int_values);
+}
+
+TEST_F(TestStructBuilder, BulkAppend) {
+  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  Done();
+  ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
+      list_lengths, list_offsets, int_values);
+}
+
+TEST_F(TestStructBuilder, BulkAppendInvalid) {
+  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 0, 1, 1};  // should be 1, 1, 1, 1
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+
+  ASSERT_OK(builder_->Reserve(list_lengths.size()));
+  ASSERT_OK(char_vb->Reserve(list_values.size()));
+  ASSERT_OK(int_vb->Reserve(int_values.size()));
+
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  Done();
+  // Even null bitmap of the parent Struct is not valid, Validate() will ignore it.
+  ASSERT_OK(result_->Validate());
+}
+
+TEST_F(TestStructBuilder, TestEquality) {
+  ArrayPtr array, equal_array;
+  ArrayPtr unequal_bitmap_array, unequal_offsets_array, unequal_values_array;
+
+  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+  vector<int32_t> unequal_int_values = {4, 2, 3, 1};
+  vector<char> unequal_list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'l', 'u', 'c', 'y'};
+  vector<int> unequal_list_offsets = {0, 3, 4, 6};
+  vector<uint8_t> unequal_list_is_valid = {1, 1, 1, 1};
+  vector<uint8_t> unequal_struct_is_valid = {1, 0, 0, 1};
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ASSERT_OK(builder_->Reserve(list_lengths.size()));
+  ASSERT_OK(char_vb->Reserve(list_values.size()));
+  ASSERT_OK(int_vb->Reserve(int_values.size()));
+
+  // setup two equal arrays, one of which takes an unequal bitmap
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+  array = builder_->Finish();
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+  equal_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  // setup an unequal one with the unequal bitmap
+  builder_->Append(unequal_struct_is_valid.size(), unequal_struct_is_valid.data());
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+  unequal_bitmap_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  // setup an unequal one with unequal offsets
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  list_vb->Append(unequal_list_offsets.data(), unequal_list_offsets.size(),
+      unequal_list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+  unequal_offsets_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  // setup anunequal one with unequal values
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : unequal_list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : unequal_int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+  unequal_values_array = builder_->Finish();
+
+  // Test array equality
+  EXPECT_TRUE(array->Equals(array));
+  EXPECT_TRUE(array->Equals(equal_array));
+  EXPECT_TRUE(equal_array->Equals(array));
+  EXPECT_FALSE(equal_array->Equals(unequal_bitmap_array));
+  EXPECT_FALSE(unequal_bitmap_array->Equals(equal_array));
+  EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_values_array));
+  EXPECT_FALSE(unequal_values_array->Equals(unequal_bitmap_array));
+  EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_offsets_array));
+  EXPECT_FALSE(unequal_offsets_array->Equals(unequal_bitmap_array));
+
+  // Test range equality
+  EXPECT_TRUE(array->RangeEquals(0, 4, 0, equal_array));
+  EXPECT_TRUE(array->RangeEquals(3, 4, 3, unequal_bitmap_array));
+  EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_offsets_array));
+  EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_offsets_array));
+  EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_offsets_array));
+  EXPECT_FALSE(array->RangeEquals(0, 1, 0, unequal_values_array));
+  EXPECT_TRUE(array->RangeEquals(1, 3, 1, unequal_values_array));
+  EXPECT_FALSE(array->RangeEquals(3, 4, 3, unequal_values_array));
+}
+
+TEST_F(TestStructBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+  ASSERT_OK(result_->Validate());
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
index 04a277a86fa58..e8176f08268b4 100644
--- a/cpp/src/arrow/types/struct.cc
+++ b/cpp/src/arrow/types/struct.cc
@@ -17,4 +17,74 @@
 
 #include "arrow/types/struct.h"
 
-namespace arrow {}  // namespace arrow
+#include <sstream>
+
+namespace arrow {
+
+bool StructArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  if (null_count_ != arr->null_count()) { return false; }
+  return RangeEquals(0, length_, 0, arr);
+}
+
+bool StructArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (Type::STRUCT != arr->type_enum()) { return false; }
+  const auto other = static_cast<StructArray*>(arr.get());
+
+  bool equal_fields = true;
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    if (IsNull(i) != arr->IsNull(o_i)) { return false; }
+    if (IsNull(i)) continue;
+    for (size_t j = 0; j < field_arrays_.size(); ++j) {
+      // TODO: really we should be comparing stretches of non-null data rather
+      // than looking at one value at a time.
+      equal_fields = field(j)->RangeEquals(i, i + 1, o_i, other->field(j));
+      if (!equal_fields) { return false; }
+    }
+  }
+
+  return true;
+}
+
+Status StructArray::Validate() const {
+  if (length_ < 0) { return Status::Invalid("Length was negative"); }
+
+  if (null_count() > length_) {
+    return Status::Invalid("Null count exceeds the length of this struct");
+  }
+
+  if (field_arrays_.size() > 0) {
+    // Validate fields
+    int32_t array_length = field_arrays_[0]->length();
+    size_t idx = 0;
+    for (auto it : field_arrays_) {
+      if (it->length() != array_length) {
+        std::stringstream ss;
+        ss << "Length is not equal from field " << it->type()->ToString()
+           << " at position {" << idx << "}";
+        return Status::Invalid(ss.str());
+      }
+
+      const Status child_valid = it->Validate();
+      if (!child_valid.ok()) {
+        std::stringstream ss;
+        ss << "Child array invalid: " << child_valid.ToString() << " at position {" << idx
+           << "}";
+        return Status::Invalid(ss.str());
+      }
+      ++idx;
+    }
+
+    if (array_length > 0 && array_length != length_) {
+      return Status::Invalid("Struct's length is not equal to its child arrays");
+    }
+  }
+  return Status::OK();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index 17e32993bf975..78afd29eb8df5 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -23,7 +23,102 @@
 #include <vector>
 
 #include "arrow/type.h"
+#include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
 
-namespace arrow {}  // namespace arrow
+namespace arrow {
+
+class StructArray : public Array {
+ public:
+  StructArray(const TypePtr& type, int32_t length, std::vector<ArrayPtr>& field_arrays,
+      int32_t null_count = 0, std::shared_ptr<Buffer> null_bitmap = nullptr)
+      : Array(type, length, null_count, null_bitmap) {
+    type_ = type;
+    field_arrays_ = field_arrays;
+  }
+
+  Status Validate() const override;
+
+  virtual ~StructArray() {}
+
+  // Return a shared pointer in case the requestor desires to share ownership
+  // with this array.
+  const std::shared_ptr<Array>& field(int32_t pos) const {
+    DCHECK_GT(field_arrays_.size(), 0);
+    return field_arrays_[pos];
+  }
+  const std::vector<ArrayPtr>& fields() const { return field_arrays_; }
+
+  bool EqualsExact(const StructArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const std::shared_ptr<Array>& arr) const override;
+
+ protected:
+  // The child arrays corresponding to each field of the struct data type.
+  std::vector<ArrayPtr> field_arrays_;
+};
+
+// ---------------------------------------------------------------------------------
+// StructArray builder
+// Append, Resize and Reserve methods are acting on StructBuilder.
+// Please make sure all these methods of all child-builders' are consistently
+// called to maintain data-structure consistency.
+class StructBuilder : public ArrayBuilder {
+ public:
+  StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+      const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders)
+      : ArrayBuilder(pool, type) {
+    field_builders_ = field_builders;
+  }
+
+  // Null bitmap is of equal length to every child field, and any zero byte
+  // will be considered as a null for that field, but users must using app-
+  // end methods or advance methods of the child builders' independently to
+  // insert data.
+  Status Append(int32_t length, const uint8_t* valid_bytes) {
+    RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(valid_bytes, length);
+    return Status::OK();
+  }
+
+  std::shared_ptr<Array> Finish() override {
+    std::vector<ArrayPtr> fields;
+    for (auto it : field_builders_) {
+      fields.push_back(it->Finish());
+    }
+
+    auto result =
+        std::make_shared<StructArray>(type_, length_, fields, null_count_, null_bitmap_);
+
+    null_bitmap_ = nullptr;
+    capacity_ = length_ = null_count_ = 0;
+
+    return result;
+  }
+
+  // Append an element to the Struct. All child-builders' Append method must
+  // be called independently to maintain data-structure consistency.
+  Status Append(bool is_valid = true) {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(is_valid);
+    return Status::OK();
+  }
+
+  Status AppendNull() { return Append(false); }
+
+  const std::shared_ptr<ArrayBuilder> field_builder(int pos) const {
+    DCHECK_GT(field_builders_.size(), 0);
+    return field_builders_[pos];
+  }
+  const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders() const {
+    return field_builders_;
+  }
+
+ protected:
+  std::vector<std::shared_ptr<ArrayBuilder>> field_builders_;
+};
+
+}  // namespace arrow
 
 #endif  // ARROW_TYPES_STRUCT_H

From bc6c4c88fb4bfd1d99e71c8043f0ba0ca5544ae2 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Wed, 8 Jun 2016 11:23:07 -0700
Subject: [PATCH 0084/1644] ARROW-200: [C++/Python] Return error status on
 string initialization failure

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #88 from emkornfield/emk_arrow_200 and squashes the following commits:

37e23be [Micah Kornfield] ARROW-200: Return error status on string initialization failure
---
 python/src/pyarrow/adapters/pandas.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 5159d86865caa..8dcc2b1c92e11 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -669,7 +669,7 @@ class ArrowDeserializer {
 
           out_values[i] = make_pystring(data, length);
           if (out_values[i] == nullptr) {
-            return Status::OK();
+            return Status::UnknownError("String initialization failed");
           }
         }
       }
@@ -678,7 +678,7 @@ class ArrowDeserializer {
         data = string_arr->GetValue(i, &length);
         out_values[i] = make_pystring(data, length);
         if (out_values[i] == nullptr) {
-          return Status::OK();
+          return Status::UnknownError("String initialization failed");
         }
       }
     }

From 8197f246de934db14b3af26a0899d95bffbdc6b2 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Wed, 8 Jun 2016 11:24:04 -0700
Subject: [PATCH 0085/1644] ARROW-212: Change contract of PrimitiveArray to
 reflect its abstractness

Follow-up based on #80

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #87 from emkornfield/emk_clarify_primitive and squashes the following commits:

14bd5b2 [Micah Kornfield] ARROW-212: Make the fact that PrimitiveArray is a abstract class more apparent fromt the contract
---
 cpp/src/arrow/types/primitive.cc |  5 +++++
 cpp/src/arrow/types/primitive.h  | 15 +++++++++------
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 8e6c0f809ca44..08fc8478e6de5 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -162,6 +162,11 @@ BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
     : PrimitiveArray(
           std::make_shared<BooleanType>(), length, data, null_count, null_bitmap) {}
 
+BooleanArray::BooleanArray(const TypePtr& type, int32_t length,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
+
 bool BooleanArray::EqualsExact(const BooleanArray& other) const {
   if (this == &other) return true;
   if (null_count_ != other.null_count_) { return false; }
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 9597fc8363138..f1ec417d51014 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -34,11 +34,10 @@ namespace arrow {
 
 class MemoryPool;
 
-// Base class for fixed-size logical types
+// Base class for fixed-size logical types.  See MakePrimitiveArray
+// (types/construct.h) for constructing a specific subclass.
 class PrimitiveArray : public Array {
  public:
-  PrimitiveArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
   virtual ~PrimitiveArray() {}
 
   const std::shared_ptr<Buffer>& data() const { return data_; }
@@ -47,6 +46,8 @@ class PrimitiveArray : public Array {
   bool Equals(const std::shared_ptr<Array>& arr) const override;
 
  protected:
+  PrimitiveArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
   std::shared_ptr<Buffer> data_;
   const uint8_t* raw_data_;
 };
@@ -55,12 +56,14 @@ class PrimitiveArray : public Array {
   class NAME : public PrimitiveArray {                                                 \
    public:                                                                             \
     using value_type = T;                                                              \
-    using PrimitiveArray::PrimitiveArray;                                              \
                                                                                        \
     NAME(int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count = 0,  \
         const std::shared_ptr<Buffer>& null_bitmap = nullptr)                          \
         : PrimitiveArray(                                                              \
               std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {} \
+    NAME(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,     \
+        int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)  \
+        : PrimitiveArray(type, length, data, null_count, null_bitmap) {}               \
                                                                                        \
     bool EqualsExact(const NAME& other) const {                                        \
       return PrimitiveArray::EqualsExact(*static_cast<const PrimitiveArray*>(&other)); \
@@ -261,10 +264,10 @@ typedef NumericBuilder<DoubleType> DoubleBuilder;
 
 class BooleanArray : public PrimitiveArray {
  public:
-  using PrimitiveArray::PrimitiveArray;
-
   BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  BooleanArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   bool EqualsExact(const BooleanArray& other) const;
   bool Equals(const ArrayPtr& arr) const override;

From ec66ddd1fd4954b78967bfa1893480473e4d380c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 10 Jun 2016 15:08:23 -0700
Subject: [PATCH 0086/1644] ARROW-203: Python: Basic filename based Parquet
 read/write

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #83 from xhochy/arrow-203 and squashes the following commits:

405f85d [Uwe L. Korn] Remove FindParquet duplication
38d786c [Uwe L. Korn] Make code more readable by using using
ec07768 [Uwe L. Korn] Set LD_LIBRARY_PATH in python build
8d90d3f [Uwe L. Korn] Do not set LD_LIBRARY_PATH in python build
000e1e3 [Uwe L. Korn] Use unique_ptr and shared_ptr from Cython
8f6010a [Uwe L. Korn] Linter fixes
0514d01 [Uwe L. Korn] Handle exceptions on RowGroupWriter::Close better
77bd21a [Uwe L. Korn] Add pandas roundtrip to tests
f583b61 [Uwe L. Korn] Fix rpath for libarrow_parquet
00c1461 [Uwe L. Korn] Also ensure correct OSX compiler flags in PyArrow
4a80116 [Uwe L. Korn] Handle Python3 strings correctly
066c08a [Uwe L. Korn] Add missing functions to smart pointers
5706db2 [Uwe L. Korn] Use length and offset instead of slicing
443de8b [Uwe L. Korn] Add miniconda to the LD_LIBRARY_PATH
2dffc14 [Uwe L. Korn] Fix min mistake, use equals instead of ==
2006e70 [Uwe L. Korn] Rewrite test py.test style
9520c39 [Uwe L. Korn] Use PARQUET from miniconda path
cd3b9a9 [Uwe L. Korn] Also search for Parquet in PyArrow
6a41d23 [Uwe L. Korn] Re-use conda installation from C++
81f501e [Uwe L. Korn] No need to install conda in travis_script_python anymore
b505feb [Uwe L. Korn] Install parquet-cpp via conda
5d4929a [Uwe L. Korn] Add test-util.h
9b06e41 [Uwe L. Korn] Make tests templated
be6415c [Uwe L. Korn] Incorportate review comments
0fbed3f [Uwe L. Korn] Remove obsolete parquet files
081db5f [Uwe L. Korn] Limit and document chunk_size
7192cfb [Uwe L. Korn] Add const to slicing parameters
0463995 [Uwe L. Korn] ARROW-203: Python: Basic filename based Parquet read/write
---
 ci/travis_before_script_cpp.sh           |   6 +-
 ci/travis_conda_build.sh                 |  22 +-
 ci/travis_install_conda.sh               |  26 +++
 ci/travis_script_python.sh               |  21 +-
 cpp/src/arrow/column.h                   |   2 +
 cpp/src/arrow/parquet/CMakeLists.txt     |   7 +
 cpp/src/arrow/parquet/parquet-io-test.cc | 256 +++++++++++++++++------
 cpp/src/arrow/parquet/reader.cc          |  25 +++
 cpp/src/arrow/parquet/reader.h           |   3 +
 cpp/src/arrow/parquet/test-util.h        |  77 +++++++
 cpp/src/arrow/parquet/utils.h            |   5 +
 cpp/src/arrow/parquet/writer.cc          |  99 +++++++--
 cpp/src/arrow/parquet/writer.h           |  12 +-
 cpp/src/arrow/util/status.h              |   9 +
 python/CMakeLists.txt                    |   8 +
 python/cmake_modules/FindArrow.cmake     |  14 +-
 python/conda.recipe/build.sh             |  13 ++
 python/pyarrow/array.pyx                 |   3 +
 python/pyarrow/error.pxd                 |   2 +
 python/pyarrow/error.pyx                 |   8 +
 python/pyarrow/includes/common.pxd       |   9 +-
 python/pyarrow/includes/libarrow.pxd     |   3 +
 python/pyarrow/includes/parquet.pxd      |  46 ++++
 python/pyarrow/parquet.pyx               |  50 ++++-
 python/pyarrow/schema.pyx                |   9 +-
 python/pyarrow/tests/test_parquet.py     |  59 ++++++
 python/setup.py                          |   4 +-
 27 files changed, 654 insertions(+), 144 deletions(-)
 create mode 100644 ci/travis_install_conda.sh
 create mode 100644 cpp/src/arrow/parquet/test-util.h
 create mode 100644 python/pyarrow/tests/test_parquet.py

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 193c76feba1d7..6159f67e3613b 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -2,6 +2,10 @@
 
 set -e
 
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+conda install -y --channel apache/channel/dev parquet-cpp
+export PARQUET_HOME=$MINICONDA
+
 : ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
 
 mkdir $CPP_BUILD_DIR
@@ -19,7 +23,7 @@ echo $GTEST_HOME
 
 : ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
 
-CMAKE_COMMON_FLAGS="-DARROW_BUILD_BENCHMARKS=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
+CMAKE_COMMON_FLAGS="-DARROW_BUILD_BENCHMARKS=ON -DARROW_PARQUET=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
   cmake -DARROW_TEST_MEMCHECK=on $CMAKE_COMMON_FLAGS -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
diff --git a/ci/travis_conda_build.sh b/ci/travis_conda_build.sh
index afa531dbd6b5f..c43a85170b094 100755
--- a/ci/travis_conda_build.sh
+++ b/ci/travis_conda_build.sh
@@ -2,27 +2,7 @@
 
 set -e
 
-if [ $TRAVIS_OS_NAME == "linux" ]; then
-  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
-else
-  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
-fi
-
-wget -O miniconda.sh $MINICONDA_URL
-MINICONDA=$TRAVIS_BUILD_DIR/miniconda
-bash miniconda.sh -b -p $MINICONDA
-export PATH="$MINICONDA/bin:$PATH"
-conda update -y -q conda
-conda info -a
-
-conda config --set show_channel_urls yes
-conda config --add channels conda-forge
-conda config --add channels apache
-
-conda install --yes conda-build jinja2 anaconda-client
-
-# faster builds, please
-conda install -y nomkl
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 
 # Build libarrow
 
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
new file mode 100644
index 0000000000000..bef667dff7cc1
--- /dev/null
+++ b/ci/travis_install_conda.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
+else
+  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
+fi
+
+wget -O miniconda.sh $MINICONDA_URL
+export MINICONDA=$TRAVIS_BUILD_DIR/miniconda
+bash miniconda.sh -b -p $MINICONDA
+export PATH="$MINICONDA/bin:$PATH"
+conda update -y -q conda
+conda info -a
+
+conda config --set show_channel_urls yes
+conda config --add channels conda-forge
+conda config --add channels apache
+
+conda install --yes conda-build jinja2 anaconda-client
+
+# faster builds, please
+conda install -y nomkl
+
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index d45b895d8cf38..6d35785356ab4 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -4,6 +4,12 @@ set -e
 
 PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 
+# Re-use conda installation from C++
+export MINICONDA=$TRAVIS_BUILD_DIR/miniconda
+export PATH="$MINICONDA/bin:$PATH"
+export LD_LIBRARY_PATH="$MINICONDA/lib:$LD_LIBRARY_PATH"
+export PARQUET_HOME=$MINICONDA
+
 # Share environment with C++
 pushd $CPP_BUILD_DIR
 source setup_build_env.sh
@@ -11,21 +17,6 @@ popd
 
 pushd $PYTHON_DIR
 
-# Bootstrap a Conda Python environment
-
-if [ $TRAVIS_OS_NAME == "linux" ]; then
-  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
-else
-  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
-fi
-
-curl $MINICONDA_URL > miniconda.sh
-MINICONDA=$TRAVIS_BUILD_DIR/miniconda
-bash miniconda.sh -b -p $MINICONDA
-export PATH="$MINICONDA/bin:$PATH"
-conda update -y -q conda
-conda info -a
-
 python_version_tests() {
   PYTHON_VERSION=$1
   CONDA_ENV_NAME="pyarrow-test-${PYTHON_VERSION}"
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
index 22becc3454780..e409566e1f139 100644
--- a/cpp/src/arrow/column.h
+++ b/cpp/src/arrow/column.h
@@ -67,6 +67,8 @@ class Column {
 
   int64_t null_count() const { return data_->null_count(); }
 
+  const std::shared_ptr<Field>& field() const { return field_; }
+
   // @returns: the column's name in the passed metadata
   const std::string& name() const { return field_->name; }
 
diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
index c00cc9f0f25d0..f00bb53c0848f 100644
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -35,6 +35,13 @@ add_library(arrow_parquet SHARED
 target_link_libraries(arrow_parquet ${PARQUET_LIBS})
 SET_TARGET_PROPERTIES(arrow_parquet PROPERTIES LINKER_LANGUAGE CXX)
 
+if (APPLE)
+  set_target_properties(arrow_parquet
+    PROPERTIES
+    BUILD_WITH_INSTALL_RPATH ON
+    INSTALL_NAME_DIR "@rpath")
+endif()
+
 ADD_ARROW_TEST(parquet-schema-test)
 ARROW_TEST_LINK_LIBRARIES(parquet-schema-test arrow_parquet)
 
diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
index 845574d2c53b9..db779d8309cf6 100644
--- a/cpp/src/arrow/parquet/parquet-io-test.cc
+++ b/cpp/src/arrow/parquet/parquet-io-test.cc
@@ -18,6 +18,7 @@
 #include "gtest/gtest.h"
 
 #include "arrow/test-util.h"
+#include "arrow/parquet/test-util.h"
 #include "arrow/parquet/reader.h"
 #include "arrow/parquet/writer.h"
 #include "arrow/types/primitive.h"
@@ -44,36 +45,45 @@ namespace arrow {
 
 namespace parquet {
 
-template <typename ArrowType>
-std::shared_ptr<PrimitiveArray> NonNullArray(
-    size_t size, typename ArrowType::c_type value) {
-  std::vector<typename ArrowType::c_type> values(size, value);
-  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
-  builder.Append(values.data(), values.size());
-  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
-}
+const int SMALL_SIZE = 100;
+const int LARGE_SIZE = 10000;
 
-// This helper function only supports (size/2) nulls yet.
-template <typename ArrowType>
-std::shared_ptr<PrimitiveArray> NullableArray(
-    size_t size, typename ArrowType::c_type value, size_t num_nulls) {
-  std::vector<typename ArrowType::c_type> values(size, value);
-  std::vector<uint8_t> valid_bytes(size, 1);
+template <typename TestType>
+struct test_traits {};
 
-  for (size_t i = 0; i < num_nulls; i++) {
-    valid_bytes[i * 2] = 0;
-  }
+template <>
+struct test_traits<Int32Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+};
 
-  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
-  builder.Append(values.data(), values.size(), valid_bytes.data());
-  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
-}
+template <>
+struct test_traits<Int64Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
+};
+
+template <>
+struct test_traits<FloatType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::FLOAT;
+};
+
+template <>
+struct test_traits<DoubleType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::DOUBLE;
+};
+
+template <typename T>
+using ParquetDataType = ::parquet::DataType<test_traits<T>::parquet_enum>;
 
+template <typename T>
+using ParquetWriter = ::parquet::TypedColumnWriter<ParquetDataType<T>>;
+
+template <typename TestType>
 class TestParquetIO : public ::testing::Test {
  public:
+  typedef typename TestType::c_type T;
   virtual void SetUp() {}
 
-  std::shared_ptr<GroupNode> Schema(
+  std::shared_ptr<GroupNode> MakeSchema(
       ParquetType::type parquet_type, Repetition::type repetition) {
     auto pnode = PrimitiveNode::Make("column1", repetition, parquet_type);
     NodePtr node_ =
@@ -98,20 +108,27 @@ class TestParquetIO : public ::testing::Test {
     std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
     ASSERT_NO_THROW(ASSERT_OK(reader.GetFlatColumn(0, &column_reader)));
     ASSERT_NE(nullptr, column_reader.get());
-    ASSERT_OK(column_reader->NextBatch(100, out));
+    ASSERT_OK(column_reader->NextBatch(SMALL_SIZE, out));
+    ASSERT_NE(nullptr, out->get());
+  }
+
+  void ReadTableFromFile(
+      std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Table>* out) {
+    arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
+    ASSERT_NO_THROW(ASSERT_OK(reader.ReadFlatTable(out)));
     ASSERT_NE(nullptr, out->get());
   }
 
-  std::unique_ptr<ParquetFileReader> Int64File(
-      std::vector<int64_t>& values, int num_chunks) {
-    std::shared_ptr<GroupNode> schema = Schema(ParquetType::INT64, Repetition::REQUIRED);
+  std::unique_ptr<ParquetFileReader> TestFile(std::vector<T>& values, int num_chunks) {
+    std::shared_ptr<GroupNode> schema =
+        MakeSchema(test_traits<TestType>::parquet_enum, Repetition::REQUIRED);
     std::unique_ptr<ParquetFileWriter> file_writer = MakeWriter(schema);
     size_t chunk_size = values.size() / num_chunks;
     for (int i = 0; i < num_chunks; i++) {
       auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
-      auto column_writer =
-          static_cast<::parquet::Int64Writer*>(row_group_writer->NextColumn());
-      int64_t* data = values.data() + i * chunk_size;
+      auto column_writer = static_cast<ParquetWriter<TestType>*>(
+          row_group_writer->NextColumn());
+      T* data = values.data() + i * chunk_size;
       column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
       column_writer->Close();
       row_group_writer->Close();
@@ -120,71 +137,135 @@ class TestParquetIO : public ::testing::Test {
     return ReaderFromSink();
   }
 
- private:
   std::shared_ptr<InMemoryOutputStream> sink_;
 };
 
-TEST_F(TestParquetIO, SingleColumnInt64Read) {
-  std::vector<int64_t> values(100, 128);
-  std::unique_ptr<ParquetFileReader> file_reader = Int64File(values, 1);
+typedef ::testing::Types<Int32Type, Int64Type, FloatType, DoubleType> TestTypes;
+
+TYPED_TEST_CASE(TestParquetIO, TestTypes);
+
+TYPED_TEST(TestParquetIO, SingleColumnRequiredRead) {
+  std::vector<typename TypeParam::c_type> values(SMALL_SIZE, 128);
+  std::unique_ptr<ParquetFileReader> file_reader = this->TestFile(values, 1);
 
   std::shared_ptr<Array> out;
-  ReadSingleColumnFile(std::move(file_reader), &out);
+  this->ReadSingleColumnFile(std::move(file_reader), &out);
 
-  Int64Array* out_array = static_cast<Int64Array*>(out.get());
-  for (size_t i = 0; i < values.size(); i++) {
-    EXPECT_EQ(values[i], out_array->raw_data()[i]);
-  }
+  ExpectArray<typename TypeParam::c_type>(values.data(), out.get());
 }
 
-TEST_F(TestParquetIO, SingleColumnInt64ChunkedRead) {
-  std::vector<int64_t> values(100, 128);
-  std::unique_ptr<ParquetFileReader> file_reader = Int64File(values, 4);
+TYPED_TEST(TestParquetIO, SingleColumnRequiredTableRead) {
+  std::vector<typename TypeParam::c_type> values(SMALL_SIZE, 128);
+  std::unique_ptr<ParquetFileReader> file_reader = this->TestFile(values, 1);
+
+  std::shared_ptr<Table> out;
+  this->ReadTableFromFile(std::move(file_reader), &out);
+  ASSERT_EQ(1, out->num_columns());
+  ASSERT_EQ(SMALL_SIZE, out->num_rows());
+
+  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+  ASSERT_EQ(1, chunked_array->num_chunks());
+  ExpectArray<typename TypeParam::c_type>(values.data(), chunked_array->chunk(0).get());
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedRead) {
+  std::vector<typename TypeParam::c_type> values(SMALL_SIZE, 128);
+  std::unique_ptr<ParquetFileReader> file_reader = this->TestFile(values, 4);
 
   std::shared_ptr<Array> out;
-  ReadSingleColumnFile(std::move(file_reader), &out);
+  this->ReadSingleColumnFile(std::move(file_reader), &out);
 
-  Int64Array* out_array = static_cast<Int64Array*>(out.get());
-  for (size_t i = 0; i < values.size(); i++) {
-    EXPECT_EQ(values[i], out_array->raw_data()[i]);
-  }
+  ExpectArray<typename TypeParam::c_type>(values.data(), out.get());
 }
 
-TEST_F(TestParquetIO, SingleColumnInt64Write) {
-  std::shared_ptr<PrimitiveArray> values = NonNullArray<Int64Type>(100, 128);
+TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedTableRead) {
+  std::vector<typename TypeParam::c_type> values(SMALL_SIZE, 128);
+  std::unique_ptr<ParquetFileReader> file_reader = this->TestFile(values, 4);
+
+  std::shared_ptr<Table> out;
+  this->ReadTableFromFile(std::move(file_reader), &out);
+  ASSERT_EQ(1, out->num_columns());
+  ASSERT_EQ(SMALL_SIZE, out->num_rows());
 
-  std::shared_ptr<GroupNode> schema = Schema(ParquetType::INT64, Repetition::REQUIRED);
-  FileWriter writer(default_memory_pool(), MakeWriter(schema));
+  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+  ASSERT_EQ(1, chunked_array->num_chunks());
+  ExpectArray<typename TypeParam::c_type>(values.data(), chunked_array->chunk(0).get());
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnRequiredWrite) {
+  std::shared_ptr<PrimitiveArray> values = NonNullArray<TypeParam>(SMALL_SIZE, 128);
+
+  std::shared_ptr<GroupNode> schema =
+      this->MakeSchema(test_traits<TypeParam>::parquet_enum, Repetition::REQUIRED);
+  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
   ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values->length())));
   ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values.get())));
   ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
 
   std::shared_ptr<Array> out;
-  ReadSingleColumnFile(ReaderFromSink(), &out);
+  this->ReadSingleColumnFile(this->ReaderFromSink(), &out);
   ASSERT_TRUE(values->Equals(out));
 }
 
-TEST_F(TestParquetIO, SingleColumnDoubleReadWrite) {
+TYPED_TEST(TestParquetIO, SingleColumnTableRequiredWrite) {
+  std::shared_ptr<PrimitiveArray> values = NonNullArray<TypeParam>(SMALL_SIZE, 128);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, false);
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  ASSERT_NO_THROW(ASSERT_OK(
+      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, values->length())));
+
+  std::shared_ptr<Table> out;
+  this->ReadTableFromFile(this->ReaderFromSink(), &out);
+  ASSERT_EQ(1, out->num_columns());
+  ASSERT_EQ(100, out->num_rows());
+
+  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+  ASSERT_EQ(1, chunked_array->num_chunks());
+  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnOptionalReadWrite) {
   // This also tests max_definition_level = 1
-  std::shared_ptr<PrimitiveArray> values = NullableArray<DoubleType>(100, 128, 10);
+  std::shared_ptr<PrimitiveArray> values = NullableArray<TypeParam>(SMALL_SIZE, 128, 10);
 
-  std::shared_ptr<GroupNode> schema = Schema(ParquetType::DOUBLE, Repetition::OPTIONAL);
-  FileWriter writer(default_memory_pool(), MakeWriter(schema));
+  std::shared_ptr<GroupNode> schema =
+      this->MakeSchema(test_traits<TypeParam>::parquet_enum, Repetition::OPTIONAL);
+  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
   ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values->length())));
   ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values.get())));
   ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
 
   std::shared_ptr<Array> out;
-  ReadSingleColumnFile(ReaderFromSink(), &out);
+  this->ReadSingleColumnFile(this->ReaderFromSink(), &out);
   ASSERT_TRUE(values->Equals(out));
 }
 
-TEST_F(TestParquetIO, SingleColumnInt64ChunkedWrite) {
-  std::shared_ptr<PrimitiveArray> values = NonNullArray<Int64Type>(100, 128);
-  std::shared_ptr<PrimitiveArray> values_chunk = NonNullArray<Int64Type>(25, 128);
+TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {
+  // This also tests max_definition_level = 1
+  std::shared_ptr<PrimitiveArray> values = NullableArray<TypeParam>(SMALL_SIZE, 128, 10);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  ASSERT_NO_THROW(ASSERT_OK(
+      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, values->length())));
+
+  std::shared_ptr<Table> out;
+  this->ReadTableFromFile(this->ReaderFromSink(), &out);
+  ASSERT_EQ(1, out->num_columns());
+  ASSERT_EQ(SMALL_SIZE, out->num_rows());
+
+  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+  ASSERT_EQ(1, chunked_array->num_chunks());
+  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+}
 
-  std::shared_ptr<GroupNode> schema = Schema(ParquetType::INT64, Repetition::REQUIRED);
-  FileWriter writer(default_memory_pool(), MakeWriter(schema));
+TYPED_TEST(TestParquetIO, SingleColumnIntRequiredChunkedWrite) {
+  std::shared_ptr<PrimitiveArray> values = NonNullArray<TypeParam>(SMALL_SIZE, 128);
+  std::shared_ptr<PrimitiveArray> values_chunk =
+      NonNullArray<TypeParam>(SMALL_SIZE / 4, 128);
+
+  std::shared_ptr<GroupNode> schema =
+      this->MakeSchema(test_traits<TypeParam>::parquet_enum, Repetition::REQUIRED);
+  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
   for (int i = 0; i < 4; i++) {
     ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values_chunk->length())));
     ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values_chunk.get())));
@@ -192,18 +273,37 @@ TEST_F(TestParquetIO, SingleColumnInt64ChunkedWrite) {
   ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
 
   std::shared_ptr<Array> out;
-  ReadSingleColumnFile(ReaderFromSink(), &out);
+  this->ReadSingleColumnFile(this->ReaderFromSink(), &out);
   ASSERT_TRUE(values->Equals(out));
 }
 
-TEST_F(TestParquetIO, SingleColumnDoubleChunkedWrite) {
-  std::shared_ptr<PrimitiveArray> values = NullableArray<DoubleType>(100, 128, 10);
+TYPED_TEST(TestParquetIO, SingleColumnTableRequiredChunkedWrite) {
+  std::shared_ptr<PrimitiveArray> values = NonNullArray<TypeParam>(LARGE_SIZE, 128);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, false);
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  ASSERT_NO_THROW(
+      ASSERT_OK(WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512)));
+
+  std::shared_ptr<Table> out;
+  this->ReadTableFromFile(this->ReaderFromSink(), &out);
+  ASSERT_EQ(1, out->num_columns());
+  ASSERT_EQ(LARGE_SIZE, out->num_rows());
+
+  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+  ASSERT_EQ(1, chunked_array->num_chunks());
+  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnOptionalChunkedWrite) {
+  std::shared_ptr<PrimitiveArray> values = NullableArray<TypeParam>(SMALL_SIZE, 128, 10);
   std::shared_ptr<PrimitiveArray> values_chunk_nulls =
-      NullableArray<DoubleType>(25, 128, 10);
-  std::shared_ptr<PrimitiveArray> values_chunk = NullableArray<DoubleType>(25, 128, 0);
+      NullableArray<TypeParam>(SMALL_SIZE / 4, 128, 10);
+  std::shared_ptr<PrimitiveArray> values_chunk =
+      NullableArray<TypeParam>(SMALL_SIZE / 4, 128, 0);
 
-  std::shared_ptr<GroupNode> schema = Schema(ParquetType::DOUBLE, Repetition::OPTIONAL);
-  FileWriter writer(default_memory_pool(), MakeWriter(schema));
+  std::shared_ptr<GroupNode> schema =
+      this->MakeSchema(test_traits<TypeParam>::parquet_enum, Repetition::OPTIONAL);
+  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
   ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values_chunk_nulls->length())));
   ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values_chunk_nulls.get())));
   for (int i = 0; i < 3; i++) {
@@ -213,10 +313,28 @@ TEST_F(TestParquetIO, SingleColumnDoubleChunkedWrite) {
   ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
 
   std::shared_ptr<Array> out;
-  ReadSingleColumnFile(ReaderFromSink(), &out);
+  this->ReadSingleColumnFile(this->ReaderFromSink(), &out);
   ASSERT_TRUE(values->Equals(out));
 }
 
+TYPED_TEST(TestParquetIO, SingleColumnTableOptionalChunkedWrite) {
+  // This also tests max_definition_level = 1
+  std::shared_ptr<PrimitiveArray> values = NullableArray<TypeParam>(LARGE_SIZE, 128, 100);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  ASSERT_NO_THROW(
+      ASSERT_OK(WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512)));
+
+  std::shared_ptr<Table> out;
+  this->ReadTableFromFile(this->ReaderFromSink(), &out);
+  ASSERT_EQ(1, out->num_columns());
+  ASSERT_EQ(LARGE_SIZE, out->num_rows());
+
+  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+  ASSERT_EQ(1, chunked_array->num_chunks());
+  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+}
+
 }  // namespace parquet
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
index 346de25360649..3b4882d4439d5 100644
--- a/cpp/src/arrow/parquet/reader.cc
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -18,10 +18,14 @@
 #include "arrow/parquet/reader.h"
 
 #include <queue>
+#include <string>
+#include <vector>
 
+#include "arrow/column.h"
 #include "arrow/parquet/schema.h"
 #include "arrow/parquet/utils.h"
 #include "arrow/schema.h"
+#include "arrow/table.h"
 #include "arrow/types/primitive.h"
 #include "arrow/util/status.h"
 
@@ -40,6 +44,7 @@ class FileReader::Impl {
   bool CheckForFlatColumn(const ::parquet::ColumnDescriptor* descr);
   Status GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out);
   Status ReadFlatColumn(int i, std::shared_ptr<Array>* out);
+  Status ReadFlatTable(std::shared_ptr<Table>* out);
 
  private:
   MemoryPool* pool_;
@@ -103,6 +108,22 @@ Status FileReader::Impl::ReadFlatColumn(int i, std::shared_ptr<Array>* out) {
   return flat_column_reader->NextBatch(reader_->num_rows(), out);
 }
 
+Status FileReader::Impl::ReadFlatTable(std::shared_ptr<Table>* table) {
+  const std::string& name = reader_->descr()->schema()->name();
+  std::shared_ptr<Schema> schema;
+  RETURN_NOT_OK(FromParquetSchema(reader_->descr(), &schema));
+
+  std::vector<std::shared_ptr<Column>> columns(reader_->num_columns());
+  for (int i = 0; i < reader_->num_columns(); i++) {
+    std::shared_ptr<Array> array;
+    RETURN_NOT_OK(ReadFlatColumn(i, &array));
+    columns[i] = std::make_shared<Column>(schema->field(i), array);
+  }
+
+  *table = std::make_shared<Table>(name, schema, columns);
+  return Status::OK();
+}
+
 FileReader::FileReader(
     MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader)
     : impl_(new FileReader::Impl(pool, std::move(reader))) {}
@@ -117,6 +138,10 @@ Status FileReader::ReadFlatColumn(int i, std::shared_ptr<Array>* out) {
   return impl_->ReadFlatColumn(i, out);
 }
 
+Status FileReader::ReadFlatTable(std::shared_ptr<Table>* out) {
+  return impl_->ReadFlatTable(out);
+}
+
 FlatColumnReader::Impl::Impl(MemoryPool* pool, const ::parquet::ColumnDescriptor* descr,
     ::parquet::ParquetFileReader* reader, int column_index)
     : pool_(pool),
diff --git a/cpp/src/arrow/parquet/reader.h b/cpp/src/arrow/parquet/reader.h
index 41ca7eb35b9f0..db7a15753d8e8 100644
--- a/cpp/src/arrow/parquet/reader.h
+++ b/cpp/src/arrow/parquet/reader.h
@@ -29,6 +29,7 @@ class Array;
 class MemoryPool;
 class RowBatch;
 class Status;
+class Table;
 
 namespace parquet {
 
@@ -90,6 +91,8 @@ class FileReader {
   Status GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out);
   // Read column as a whole into an Array.
   Status ReadFlatColumn(int i, std::shared_ptr<Array>* out);
+  // Read a table of flat columns into a Table.
+  Status ReadFlatTable(std::shared_ptr<Table>* out);
 
   virtual ~FileReader();
 
diff --git a/cpp/src/arrow/parquet/test-util.h b/cpp/src/arrow/parquet/test-util.h
new file mode 100644
index 0000000000000..1496082d5c661
--- /dev/null
+++ b/cpp/src/arrow/parquet/test-util.h
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <string>
+#include <vector>
+
+#include "arrow/types/primitive.h"
+
+namespace arrow {
+
+namespace parquet {
+
+template <typename ArrowType>
+std::shared_ptr<PrimitiveArray> NonNullArray(
+    size_t size, typename ArrowType::c_type value) {
+  std::vector<typename ArrowType::c_type> values(size, value);
+  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
+  builder.Append(values.data(), values.size());
+  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
+}
+
+// This helper function only supports (size/2) nulls yet.
+template <typename ArrowType>
+std::shared_ptr<PrimitiveArray> NullableArray(
+    size_t size, typename ArrowType::c_type value, size_t num_nulls) {
+  std::vector<typename ArrowType::c_type> values(size, value);
+  std::vector<uint8_t> valid_bytes(size, 1);
+
+  for (size_t i = 0; i < num_nulls; i++) {
+    valid_bytes[i * 2] = 0;
+  }
+
+  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
+  builder.Append(values.data(), values.size(), valid_bytes.data());
+  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
+}
+
+std::shared_ptr<Column> MakeColumn(const std::string& name,
+    const std::shared_ptr<PrimitiveArray>& array, bool nullable) {
+  auto field = std::make_shared<Field>(name, array->type(), nullable);
+  return std::make_shared<Column>(field, array);
+}
+
+std::shared_ptr<Table> MakeSimpleTable(
+    const std::shared_ptr<PrimitiveArray>& values, bool nullable) {
+  std::shared_ptr<Column> column = MakeColumn("col", values, nullable);
+  std::vector<std::shared_ptr<Column>> columns({column});
+  std::vector<std::shared_ptr<Field>> fields({column->field()});
+  auto schema = std::make_shared<Schema>(fields);
+  return std::make_shared<Table>("table", schema, columns);
+}
+
+template <typename T>
+void ExpectArray(T* expected, Array* result) {
+  PrimitiveArray* p_array = static_cast<PrimitiveArray*>(result);
+  for (size_t i = 0; i < result->length(); i++) {
+    EXPECT_EQ(expected[i], reinterpret_cast<const T*>(p_array->data()->data())[i]);
+  }
+}
+
+}  // namespace parquet
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/utils.h b/cpp/src/arrow/parquet/utils.h
index b32792fdf7030..409bcd9065cda 100644
--- a/cpp/src/arrow/parquet/utils.h
+++ b/cpp/src/arrow/parquet/utils.h
@@ -31,6 +31,11 @@ namespace parquet {
     (s);                        \
   } catch (const ::parquet::ParquetException& e) { return Status::Invalid(e.what()); }
 
+#define PARQUET_IGNORE_NOT_OK(s) \
+  try {                          \
+    (s);                         \
+  } catch (const ::parquet::ParquetException& e) {}
+
 }  // namespace parquet
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
index 3ad2c5b073501..1223901d5505a 100644
--- a/cpp/src/arrow/parquet/writer.cc
+++ b/cpp/src/arrow/parquet/writer.cc
@@ -17,11 +17,21 @@
 
 #include "arrow/parquet/writer.h"
 
+#include <algorithm>
+#include <vector>
+
 #include "arrow/array.h"
+#include "arrow/column.h"
+#include "arrow/table.h"
+#include "arrow/types/construct.h"
 #include "arrow/types/primitive.h"
+#include "arrow/parquet/schema.h"
 #include "arrow/parquet/utils.h"
 #include "arrow/util/status.h"
 
+using parquet::ParquetFileWriter;
+using parquet::schema::GroupNode;
+
 namespace arrow {
 
 namespace parquet {
@@ -32,8 +42,9 @@ class FileWriter::Impl {
 
   Status NewRowGroup(int64_t chunk_size);
   template <typename ParquetType>
-  Status TypedWriteBatch(::parquet::ColumnWriter* writer, const PrimitiveArray* data);
-  Status WriteFlatColumnChunk(const PrimitiveArray* data);
+  Status TypedWriteBatch(::parquet::ColumnWriter* writer, const PrimitiveArray* data,
+      int64_t offset, int64_t length);
+  Status WriteFlatColumnChunk(const PrimitiveArray* data, int64_t offset, int64_t length);
   Status Close();
 
   virtual ~Impl() {}
@@ -60,31 +71,31 @@ Status FileWriter::Impl::NewRowGroup(int64_t chunk_size) {
 }
 
 template <typename ParquetType>
-Status FileWriter::Impl::TypedWriteBatch(
-    ::parquet::ColumnWriter* column_writer, const PrimitiveArray* data) {
+Status FileWriter::Impl::TypedWriteBatch(::parquet::ColumnWriter* column_writer,
+    const PrimitiveArray* data, int64_t offset, int64_t length) {
+  // TODO: DCHECK((offset + length) <= data->length());
   auto data_ptr =
-      reinterpret_cast<const typename ParquetType::c_type*>(data->data()->data());
+      reinterpret_cast<const typename ParquetType::c_type*>(data->data()->data()) +
+      offset;
   auto writer =
       reinterpret_cast<::parquet::TypedColumnWriter<ParquetType>*>(column_writer);
   if (writer->descr()->max_definition_level() == 0) {
     // no nulls, just dump the data
-    PARQUET_CATCH_NOT_OK(writer->WriteBatch(data->length(), nullptr, nullptr, data_ptr));
+    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, nullptr, nullptr, data_ptr));
   } else if (writer->descr()->max_definition_level() == 1) {
-    RETURN_NOT_OK(def_levels_buffer_.Resize(data->length() * sizeof(int16_t)));
+    RETURN_NOT_OK(def_levels_buffer_.Resize(length * sizeof(int16_t)));
     int16_t* def_levels_ptr =
         reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
     if (data->null_count() == 0) {
-      std::fill(def_levels_ptr, def_levels_ptr + data->length(), 1);
-      PARQUET_CATCH_NOT_OK(
-          writer->WriteBatch(data->length(), def_levels_ptr, nullptr, data_ptr));
+      std::fill(def_levels_ptr, def_levels_ptr + length, 1);
+      PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, def_levels_ptr, nullptr, data_ptr));
     } else {
-      RETURN_NOT_OK(data_buffer_.Resize(
-          (data->length() - data->null_count()) * sizeof(typename ParquetType::c_type)));
+      RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(typename ParquetType::c_type)));
       auto buffer_ptr =
           reinterpret_cast<typename ParquetType::c_type*>(data_buffer_.mutable_data());
       int buffer_idx = 0;
-      for (size_t i = 0; i < data->length(); i++) {
-        if (data->IsNull(i)) {
+      for (size_t i = 0; i < length; i++) {
+        if (data->IsNull(offset + i)) {
           def_levels_ptr[i] = 0;
         } else {
           def_levels_ptr[i] = 1;
@@ -92,7 +103,7 @@ Status FileWriter::Impl::TypedWriteBatch(
         }
       }
       PARQUET_CATCH_NOT_OK(
-          writer->WriteBatch(data->length(), def_levels_ptr, nullptr, buffer_ptr));
+          writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
     }
   } else {
     return Status::NotImplemented("no support for max definition level > 1 yet");
@@ -107,12 +118,13 @@ Status FileWriter::Impl::Close() {
   return Status::OK();
 }
 
-#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType) \
-  case Type::ENUM:                                     \
-    return TypedWriteBatch<ParquetType>(writer, data); \
+#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType)                 \
+  case Type::ENUM:                                                     \
+    return TypedWriteBatch<ParquetType>(writer, data, offset, length); \
     break;
 
-Status FileWriter::Impl::WriteFlatColumnChunk(const PrimitiveArray* data) {
+Status FileWriter::Impl::WriteFlatColumnChunk(
+    const PrimitiveArray* data, int64_t offset, int64_t length) {
   ::parquet::ColumnWriter* writer;
   PARQUET_CATCH_NOT_OK(writer = row_group_writer_->NextColumn());
   switch (data->type_enum()) {
@@ -133,8 +145,11 @@ Status FileWriter::NewRowGroup(int64_t chunk_size) {
   return impl_->NewRowGroup(chunk_size);
 }
 
-Status FileWriter::WriteFlatColumnChunk(const PrimitiveArray* data) {
-  return impl_->WriteFlatColumnChunk(data);
+Status FileWriter::WriteFlatColumnChunk(
+    const PrimitiveArray* data, int64_t offset, int64_t length) {
+  int64_t real_length = length;
+  if (length == -1) { real_length = data->length(); }
+  return impl_->WriteFlatColumnChunk(data, offset, real_length);
 }
 
 Status FileWriter::Close() {
@@ -143,6 +158,48 @@ Status FileWriter::Close() {
 
 FileWriter::~FileWriter() {}
 
+Status WriteFlatTable(const Table* table, MemoryPool* pool,
+    std::shared_ptr<::parquet::OutputStream> sink, int64_t chunk_size) {
+  std::shared_ptr<::parquet::SchemaDescriptor> parquet_schema;
+  RETURN_NOT_OK(ToParquetSchema(table->schema().get(), &parquet_schema));
+  auto schema_node = std::static_pointer_cast<GroupNode>(parquet_schema->schema());
+  std::unique_ptr<ParquetFileWriter> parquet_writer =
+      ParquetFileWriter::Open(sink, schema_node);
+  FileWriter writer(pool, std::move(parquet_writer));
+
+  // TODO: Support writing chunked arrays.
+  for (int i = 0; i < table->num_columns(); i++) {
+    if (table->column(i)->data()->num_chunks() != 1) {
+      return Status::NotImplemented("No support for writing chunked arrays yet.");
+    }
+  }
+
+  // Cast to PrimitiveArray instances as we work with them.
+  std::vector<std::shared_ptr<PrimitiveArray>> arrays(table->num_columns());
+  for (int i = 0; i < table->num_columns(); i++) {
+    // num_chunks == 1 as per above loop
+    std::shared_ptr<Array> array = table->column(i)->data()->chunk(0);
+    auto primitive_array = std::dynamic_pointer_cast<PrimitiveArray>(array);
+    if (!primitive_array) {
+      PARQUET_IGNORE_NOT_OK(writer.Close());
+      return Status::NotImplemented("Table must consist of PrimitiveArray instances");
+    }
+    arrays[i] = primitive_array;
+  }
+
+  for (int chunk = 0; chunk * chunk_size < table->num_rows(); chunk++) {
+    int64_t offset = chunk * chunk_size;
+    int64_t size = std::min(chunk_size, table->num_rows() - offset);
+    RETURN_NOT_OK_ELSE(writer.NewRowGroup(size), PARQUET_IGNORE_NOT_OK(writer.Close()));
+    for (int i = 0; i < table->num_columns(); i++) {
+      RETURN_NOT_OK_ELSE(writer.WriteFlatColumnChunk(arrays[i].get(), offset, size),
+          PARQUET_IGNORE_NOT_OK(writer.Close()));
+    }
+  }
+
+  return writer.Close();
+}
+
 }  // namespace parquet
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h
index 38f7d0b3a89d5..83e799f7ed1ed 100644
--- a/cpp/src/arrow/parquet/writer.h
+++ b/cpp/src/arrow/parquet/writer.h
@@ -29,6 +29,7 @@ class MemoryPool;
 class PrimitiveArray;
 class RowBatch;
 class Status;
+class Table;
 
 namespace parquet {
 
@@ -42,7 +43,8 @@ class FileWriter {
   FileWriter(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer);
 
   Status NewRowGroup(int64_t chunk_size);
-  Status WriteFlatColumnChunk(const PrimitiveArray* data);
+  Status WriteFlatColumnChunk(
+      const PrimitiveArray* data, int64_t offset = 0, int64_t length = -1);
   Status Close();
 
   virtual ~FileWriter();
@@ -52,6 +54,14 @@ class FileWriter {
   std::unique_ptr<Impl> impl_;
 };
 
+/**
+ * Write a flat Table to Parquet.
+ *
+ * The table shall only consist of nullable, non-repeated columns of primitive type.
+ */
+Status WriteFlatTable(const Table* table, MemoryPool* pool,
+    std::shared_ptr<::parquet::OutputStream> sink, int64_t chunk_size);
+
 }  // namespace parquet
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
index 6ddc177a9a50d..d1a742500084c 100644
--- a/cpp/src/arrow/util/status.h
+++ b/cpp/src/arrow/util/status.h
@@ -63,6 +63,15 @@ namespace arrow {
     if (!_s.ok()) { return _s; } \
   } while (0);
 
+#define RETURN_NOT_OK_ELSE(s, else_) \
+  do {                               \
+    Status _s = (s);                 \
+    if (!_s.ok()) {                  \
+      else_;                         \
+      return _s;                     \
+    }                                \
+  } while (0);
+
 enum class StatusCode : char {
   OK = 0,
   OutOfMemory = 1,
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 2173232d4eff5..f1becfcf44964 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -339,11 +339,17 @@ if (PYARROW_BUILD_TESTS)
     STATIC_LIB ${GTEST_STATIC_LIB})
 endif()
 
+## Parquet
+find_package(Parquet REQUIRED)
+include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
+
 ## Arrow
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 ADD_THIRDPARTY_LIB(arrow
   SHARED_LIB ${ARROW_SHARED_LIB})
+ADD_THIRDPARTY_LIB(arrow_parquet
+    SHARED_LIB ${ARROW_PARQUET_SHARED_LIB})
 
 ############################################################
 # Linker setup
@@ -422,6 +428,7 @@ set(PYARROW_SRCS
 
 set(LINK_LIBS
   arrow
+  arrow_parquet
 )
 
 SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
@@ -442,6 +449,7 @@ set(CYTHON_EXTENSIONS
   array
   config
   error
+  parquet
   scalar
   schema
   table
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 3d9983849ebb2..f0b258ed027b0 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -42,19 +42,27 @@ find_library(ARROW_LIB_PATH NAMES arrow
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
-if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
+find_library(ARROW_PARQUET_LIB_PATH NAMES arrow_parquet
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+
+if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH AND ARROW_PARQUET_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
+  set(ARROW_PARQUET_LIB_NAME libarrow_parquet)
   set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
   set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  set(ARROW_PARQUET_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PARQUET_LIB_NAME}.a)
+  set(ARROW_PARQUET_SHARED_LIB ${ARROW_LIBS}/${ARROW_PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 else ()
   set(ARROW_FOUND FALSE)
 endif ()
 
 if (ARROW_FOUND)
   if (NOT Arrow_FIND_QUIETLY)
-    message(STATUS "Found the Arrow library: ${ARROW_LIB_PATH}")
+    message(STATUS "Found the Arrow library: ${ARROW_LIB_PATH}, ${ARROW_PARQUET_LIB_PATH}")
   endif ()
 else ()
   if (NOT Arrow_FIND_QUIETLY)
@@ -74,4 +82,6 @@ mark_as_advanced(
   ARROW_LIBS
   ARROW_STATIC_LIB
   ARROW_SHARED_LIB
+  ARROW_PARQUET_STATIC_LIB
+  ARROW_PARQUET_SHARED_LIB
 )
diff --git a/python/conda.recipe/build.sh b/python/conda.recipe/build.sh
index a9d9aedead399..a164c1af51833 100644
--- a/python/conda.recipe/build.sh
+++ b/python/conda.recipe/build.sh
@@ -6,6 +6,19 @@ export ARROW_HOME=$PREFIX
 
 cd $RECIPE_DIR
 
+if [ "$(uname)" == "Darwin" ]; then
+  # C++11 finagling for Mac OSX
+  export CC=clang
+  export CXX=clang++
+  export MACOSX_VERSION_MIN="10.7"
+  CXXFLAGS="${CXXFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}"
+  CXXFLAGS="${CXXFLAGS} -stdlib=libc++ -std=c++11"
+  export LDFLAGS="${LDFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}"
+  export LDFLAGS="${LDFLAGS} -stdlib=libc++ -std=c++11"
+  export LINKFLAGS="${LDFLAGS}"
+  export MACOSX_DEPLOYMENT_TARGET=10.7
+fi
+
 echo Setting the compiler...
 if [ `uname` == Linux ]; then
   EXTRA_CMAKE_ARGS=-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index a80b3ce83980e..619e5ef7e3943 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -68,6 +68,9 @@ cdef class Array:
         values = array_format(self, window=10)
         return '{0}\n{1}'.format(type_format, values)
 
+    def equals(Array self, Array other):
+        return self.ap.Equals(other.sp_array)
+
     def __len__(self):
         if self.sp_array.get():
             return self.sp_array.get().length()
diff --git a/python/pyarrow/error.pxd b/python/pyarrow/error.pxd
index d226abeda04e0..97ba0ef2e9fcb 100644
--- a/python/pyarrow/error.pxd
+++ b/python/pyarrow/error.pxd
@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from pyarrow.includes.libarrow cimport CStatus
 from pyarrow.includes.pyarrow cimport *
 
+cdef check_cstatus(const CStatus& status)
 cdef check_status(const Status& status)
diff --git a/python/pyarrow/error.pyx b/python/pyarrow/error.pyx
index 3f8d7dd646091..5a6a038a92e43 100644
--- a/python/pyarrow/error.pyx
+++ b/python/pyarrow/error.pyx
@@ -15,12 +15,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from pyarrow.includes.libarrow cimport CStatus
 from pyarrow.includes.common cimport c_string
 from pyarrow.compat import frombytes
 
 class ArrowException(Exception):
     pass
 
+cdef check_cstatus(const CStatus& status):
+    if status.ok():
+        return
+
+    cdef c_string c_message = status.ToString()
+    raise ArrowException(frombytes(c_message))
+
 cdef check_status(const Status& status):
     if status.ok():
         return
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index e86d5d77e8b10..1f6ecee510521 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -19,6 +19,7 @@
 
 from libc.stdint cimport *
 from libcpp cimport bool as c_bool
+from libcpp.memory cimport shared_ptr, unique_ptr
 from libcpp.string cimport string as c_string
 from libcpp.vector cimport vector
 
@@ -32,11 +33,3 @@ cdef extern from "<iostream>":
 cdef extern from "<Python.h>":
     void Py_XDECREF(PyObject* o)
 
-cdef extern from "<memory>" namespace "std" nogil:
-
-    cdef cppclass shared_ptr[T]:
-        shared_ptr()
-        shared_ptr(T*)
-        T* get()
-        void reset()
-        void reset(T* p)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index b2ef45a347bc0..90414e3d542db 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -72,6 +72,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass MemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
 
+    cdef MemoryPool* default_memory_pool()
+
     cdef cppclass CListType" arrow::ListType"(CDataType):
         CListType(const shared_ptr[CDataType]& value_type)
 
@@ -103,6 +105,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int32_t null_count()
         Type type_enum()
 
+        c_bool Equals(const shared_ptr[CArray]& arr)
         c_bool IsNull(int i)
 
     cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray):
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index ffdc5d487068d..0918344070eb0 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -18,6 +18,26 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport CSchema, CStatus, CTable, MemoryPool
+
+
+cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
+  cdef cppclass Node:
+    pass
+
+  cdef cppclass GroupNode(Node):
+    pass
+
+  cdef cppclass PrimitiveNode(Node):
+    pass
+
+cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
+  cdef cppclass SchemaDescriptor:
+    shared_ptr[Node] schema()
+    GroupNode* group()
+
+  cdef cppclass ColumnDescriptor:
+    pass
 
 cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
     cdef cppclass ColumnReader:
@@ -48,4 +68,30 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         pass
 
     cdef cppclass ParquetFileReader:
+        # TODO: Some default arguments are missing
+        @staticmethod
+        unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
+
+cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
+    cdef cppclass OutputStream:
         pass
+
+    cdef cppclass LocalFileOutputStream(OutputStream):
+        LocalFileOutputStream(const c_string& path)
+        void Close()
+
+
+cdef extern from "arrow/parquet/reader.h" namespace "arrow::parquet" nogil:
+    cdef cppclass FileReader:
+        FileReader(MemoryPool* pool, unique_ptr[ParquetFileReader] reader)
+        CStatus ReadFlatTable(shared_ptr[CTable]* out);
+
+
+cdef extern from "arrow/parquet/schema.h" namespace "arrow::parquet" nogil:
+    CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema, shared_ptr[CSchema]* out)
+    CStatus ToParquetSchema(const CSchema* arrow_schema, shared_ptr[SchemaDescriptor]* out)
+
+
+cdef extern from "arrow/parquet/writer.h" namespace "arrow::parquet" nogil:
+    cdef CStatus WriteFlatTable(const CTable* table, MemoryPool* pool, shared_ptr[OutputStream] sink, int64_t chunk_size)
+
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 622e7d0772456..3d5355ebe433a 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -19,5 +19,53 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from pyarrow.compat import frombytes, tobytes
+from pyarrow.includes.libarrow cimport *
+cimport pyarrow.includes.pyarrow as pyarrow
 from pyarrow.includes.parquet cimport *
+
+from pyarrow.compat import tobytes
+from pyarrow.error cimport check_cstatus
+from pyarrow.table cimport Table
+
+def read_table(filename, columns=None):
+    """
+    Read a Table from Parquet format
+    Returns
+    -------
+    table: pyarrow.Table
+    """
+    cdef unique_ptr[FileReader] reader
+    cdef Table table = Table()
+    cdef shared_ptr[CTable] ctable
+
+    # Must be in one expression to avoid calling std::move which is not possible
+    # in Cython (due to missing rvalue support)
+    reader = unique_ptr[FileReader](new FileReader(default_memory_pool(),
+        ParquetFileReader.OpenFile(tobytes(filename))))
+    check_cstatus(reader.get().ReadFlatTable(&ctable))
+    table.init(ctable)
+    return table
+
+def write_table(table, filename, chunk_size=None):
+    """
+    Write a Table to Parquet format
+
+    Parameters
+    ----------
+    table : pyarrow.Table
+    filename : string
+    chunk_size : int
+        The maximum number of rows in each Parquet RowGroup
+    """
+    cdef Table table_ = table
+    cdef CTable* ctable_ = table_.table
+    cdef shared_ptr[OutputStream] sink
+    cdef int64_t chunk_size_ = 0
+    if chunk_size is None:
+        chunk_size_ = min(ctable_.num_rows(), int(2**16))
+    else:
+        chunk_size_ = chunk_size
+
+    sink.reset(new LocalFileOutputStream(tobytes(filename)))
+    check_cstatus(WriteFlatTable(ctable_, default_memory_pool(), sink, chunk_size_))
+
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 22ddf0cf17e41..084c304aed2a2 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -201,7 +201,9 @@ def string():
 
 def list_(DataType value_type):
     cdef DataType out = DataType()
-    out.init(shared_ptr[CDataType](new CListType(value_type.sp_type)))
+    cdef shared_ptr[CDataType] list_type
+    list_type.reset(new CListType(value_type.sp_type))
+    out.init(list_type)
     return out
 
 def struct(fields):
@@ -212,12 +214,13 @@ def struct(fields):
         DataType out = DataType()
         Field field
         vector[shared_ptr[CField]] c_fields
+        cdef shared_ptr[CDataType] struct_type
 
     for field in fields:
         c_fields.push_back(field.sp_field)
 
-    out.init(shared_ptr[CDataType](
-        new CStructType(c_fields)))
+    struct_type.reset(new CStructType(c_fields))
+    out.init(struct_type)
     return out
 
 def schema(fields):
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
new file mode 100644
index 0000000000000..d92cf4ca6563e
--- /dev/null
+++ b/python/pyarrow/tests/test_parquet.py
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyarrow.compat import unittest
+import pyarrow as arrow
+import pyarrow.parquet
+
+A = arrow
+
+import numpy as np
+import os.path
+import pandas as pd
+
+import pandas.util.testing as pdt
+
+
+def test_single_pylist_column_roundtrip(tmpdir):
+    for dtype in [int, float]:
+        filename = tmpdir.join('single_{}_column.parquet'.format(dtype.__name__))
+        data = [A.from_pylist(list(map(dtype, range(5))))]
+        table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
+        A.parquet.write_table(table, filename.strpath)
+        table_read = pyarrow.parquet.read_table(filename.strpath)
+        for col_written, col_read in zip(table.itercolumns(), table_read.itercolumns()):
+            assert col_written.name == col_read.name
+            assert col_read.data.num_chunks == 1
+            data_written = col_written.data.chunk(0)
+            data_read = col_read.data.chunk(0)
+            assert data_written.equals(data_read)
+
+def test_pandas_rountrip(tmpdir):
+    size = 10000
+    df = pd.DataFrame({
+        'int32': np.arange(size, dtype=np.int32),
+        'int64': np.arange(size, dtype=np.int64),
+        'float32': np.arange(size, dtype=np.float32),
+        'float64': np.arange(size, dtype=np.float64)
+    })
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = A.from_pandas_dataframe(df)
+    A.parquet.write_table(arrow_table, filename.strpath)
+    table_read = pyarrow.parquet.read_table(filename.strpath)
+    df_read = table_read.to_pandas()
+    pdt.assert_frame_equal(df, df_read)
+
diff --git a/python/setup.py b/python/setup.py
index 5f228ed0af245..7edeb9143319b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -214,7 +214,7 @@ def get_ext_built(self, name):
             return name + suffix
 
     def get_cmake_cython_names(self):
-        return ['array', 'config', 'error', 'scalar', 'schema', 'table']
+        return ['array', 'config', 'error', 'parquet', 'scalar', 'schema', 'table']
 
     def get_names(self):
         return self._found_names
@@ -242,7 +242,7 @@ def get_outputs(self):
         'clean': clean,
         'build_ext': build_ext
     },
-    install_requires=['cython >= 0.21', 'numpy >= 1.9'],
+    install_requires=['cython >= 0.23', 'numpy >= 1.9'],
     description=DESC,
     license='Apache License, Version 2.0',
     maintainer="Apache Arrow Developers",

From b4e0e93d580b8e0344c0caa1cf51cbe088bd25ac Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 15 Jun 2016 13:28:10 -0700
Subject: [PATCH 0087/1644] ARROW-217: Fix Travis w.r.t conda 4.1.0 changes

Travis is happy, fixes the problems we see with Travis in #85

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #90 from xhochy/fix-conda-show-channel-urls and squashes the following commits:

82e9840 [Uwe L. Korn] ARROW-217: Fix Travis w.r.t. conda 4.1.0 changes
---
 ci/travis_before_script_cpp.sh | 2 +-
 ci/travis_conda_build.sh       | 2 +-
 ci/travis_install_conda.sh     | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 6159f67e3613b..9060cc9b5ef22 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 conda install -y --channel apache/channel/dev parquet-cpp
diff --git a/ci/travis_conda_build.sh b/ci/travis_conda_build.sh
index c43a85170b094..a787df79a5574 100755
--- a/ci/travis_conda_build.sh
+++ b/ci/travis_conda_build.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index bef667dff7cc1..be7f59a4733bd 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -15,9 +15,11 @@ export PATH="$MINICONDA/bin:$PATH"
 conda update -y -q conda
 conda info -a
 
-conda config --set show_channel_urls yes
+conda config --set show_channel_urls True
+conda config --add channels https://repo.continuum.io/pkgs/free
 conda config --add channels conda-forge
 conda config --add channels apache
+conda info -a
 
 conda install --yes conda-build jinja2 anaconda-client
 

From 790d5412da67f807159f236179a8a7df37b270d2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 16 Jun 2016 10:50:40 -0700
Subject: [PATCH 0088/1644] ARROW-218: Add optional API token authentication
 option to PR merge tool

You can use an API token with extremely limited privileges (i.e., only access to public GitHub repos), but this helps avoid rate limiting issues on shared outbound IP addresses.

Author: Wes McKinney <wesm@apache.org>

Closes #91 from wesm/ARROW-218 and squashes the following commits:

f45808c [Wes McKinney] Add optional GitHub API token to patch tool (to avoid rate limiting issues with unauthenticated requests)
---
 dev/merge_arrow_pr.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index fe0bcd13dd8f1..981779ffb4c76 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -66,7 +66,17 @@
 
 def get_json(url):
     try:
-        return json.load(urllib2.urlopen(url))
+        from urllib2 import urlopen, Request
+        env_var = 'ARROW_GITHUB_API_TOKEN'
+
+        if env_var in os.environ:
+            token = os.environ[env_var]
+            request = Request(url)
+            request.add_header('Authorization', 'token %s' % token)
+            response = urlopen(request)
+        else:
+            response = urlopen(url)
+        return json.load(response)
     except urllib2.HTTPError as e:
         print "Unable to fetch URL, exiting: %s" % url
         sys.exit(-1)

From 27edd25eb4f714ff1cc2770ed5a1fbc695eb8a08 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Thu, 16 Jun 2016 10:58:18 -0700
Subject: [PATCH 0089/1644] ARROW-210:  Cleanup of the string related types in
 C++ code base

One thing that is worth discussing is if char types should also be removed (if they aren't i'll add the missing unit tests).

I also moved CharType to type.h which seems more consistent with existing code.  I can clean it up either way in a follow-up review if we decide with want to push types into their corresponding Array headers.

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #85 from emkornfield/emk_string_types_wip and squashes the following commits:

4414816 [Micah Kornfield] remove CHAR from parquet
6f0634c [Micah Kornfield] remove char type and add dcheck
58bfcc9 [Micah Kornfield] fix style of char_type_
1e0152d [Micah Kornfield] wip
---
 cpp/src/arrow/parquet/schema.cc    |   5 -
 cpp/src/arrow/type.cc              |  17 ++-
 cpp/src/arrow/type.h               |  55 ++++++---
 cpp/src/arrow/types/construct.cc   |   2 -
 cpp/src/arrow/types/decimal.h      |   1 -
 cpp/src/arrow/types/list.h         |   8 +-
 cpp/src/arrow/types/string-test.cc | 188 ++++++++++++++++++++++++-----
 cpp/src/arrow/types/string.cc      |  40 ++++--
 cpp/src/arrow/types/string.h       | 104 +++++++++-------
 cpp/src/arrow/util/macros.h        |   2 +-
 10 files changed, 307 insertions(+), 115 deletions(-)

diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index fd758940c9f3a..c7979db349453 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -250,11 +250,6 @@ Status FieldToNode(const std::shared_ptr<Field>& field, NodePtr* out) {
     case Type::DOUBLE:
       type = ParquetType::DOUBLE;
       break;
-    case Type::CHAR:
-      type = ParquetType::FIXED_LEN_BYTE_ARRAY;
-      logical_type = LogicalType::UTF8;
-      length = static_cast<CharType*>(field->type.get())->size;
-      break;
     case Type::STRING:
       type = ParquetType::BYTE_ARRAY;
       logical_type = LogicalType::UTF8;
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 4e686d9cf4a6f..4fd50b7c19365 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -31,7 +31,18 @@ std::string Field::ToString() const {
 
 DataType::~DataType() {}
 
-StringType::StringType() : DataType(Type::STRING) {}
+bool DataType::Equals(const DataType* other) const {
+  bool equals = other && ((this == other) ||
+                             ((this->type == other->type) &&
+                                 ((this->num_children() == other->num_children()))));
+  if (equals) {
+    for (int i = 0; i < num_children(); ++i) {
+      // TODO(emkornfield) limit recursion
+      if (!children_[i]->Equals(other->children_[i])) { return false; }
+    }
+  }
+  return equals;
+}
 
 std::string StringType::ToString() const {
   std::string result(name());
@@ -44,6 +55,10 @@ std::string ListType::ToString() const {
   return s.str();
 }
 
+std::string BinaryType::ToString() const {
+  return std::string(name());
+}
+
 std::string StructType::ToString() const {
   std::stringstream s;
   s << "struct<";
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index f366645cd5cf9..8fb41211ba945 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -23,6 +23,8 @@
 #include <string>
 #include <vector>
 
+#include "arrow/util/macros.h"
+
 namespace arrow {
 
 // Data types in this library are all *logical*. They can be expressed as
@@ -53,15 +55,9 @@ struct Type {
     // 8-byte floating point value
     DOUBLE = 11,
 
-    // CHAR(N): fixed-length UTF8 string with length N
-    CHAR = 12,
-
     // UTF8 variable-length string as List<Char>
     STRING = 13,
 
-    // VARCHAR(N): Null-terminated string type embedded in a CHAR(N + 1)
-    VARCHAR = 14,
-
     // Variable-length bytes (no guarantee of UTF8-ness)
     BINARY = 15,
 
@@ -114,12 +110,15 @@ struct DataType {
 
   virtual ~DataType();
 
-  bool Equals(const DataType* other) {
-    // Call with a pointer so more friendly to subclasses
-    return other && ((this == other) || (this->type == other->type));
-  }
+  // Return whether the types are equal
+  //
+  // Types that are logically convertable from one to another e.g. List<UInt8>
+  // and Binary are NOT equal).
+  virtual bool Equals(const DataType* other) const;
 
-  bool Equals(const std::shared_ptr<DataType>& other) { return Equals(other.get()); }
+  bool Equals(const std::shared_ptr<DataType>& other) const {
+    return Equals(other.get());
+  }
 
   const std::shared_ptr<Field>& child(int i) const { return children_[i]; }
 
@@ -236,9 +235,8 @@ struct DoubleType : public PrimitiveType<DoubleType> {
 
 struct ListType : public DataType {
   // List can contain any other logical value type
-  explicit ListType(const std::shared_ptr<DataType>& value_type) : DataType(Type::LIST) {
-    children_ = {std::make_shared<Field>("item", value_type)};
-  }
+  explicit ListType(const std::shared_ptr<DataType>& value_type)
+      : ListType(value_type, Type::LIST) {}
 
   explicit ListType(const std::shared_ptr<Field>& value_field) : DataType(Type::LIST) {
     children_ = {value_field};
@@ -251,15 +249,38 @@ struct ListType : public DataType {
   static char const* name() { return "list"; }
 
   std::string ToString() const override;
+
+ protected:
+  // Constructor for classes that are implemented as List Arrays.
+  ListType(const std::shared_ptr<DataType>& value_type, Type::type logical_type)
+      : DataType(logical_type) {
+    // TODO ARROW-187 this can technically fail, make a constructor method ?
+    children_ = {std::make_shared<Field>("item", value_type)};
+  }
 };
 
-// String is a logical type consisting of a physical list of 1-byte values
-struct StringType : public DataType {
-  StringType();
+// BinaryType type is reprsents lists of 1-byte values.
+struct BinaryType : public ListType {
+  BinaryType() : BinaryType(Type::BINARY) {}
+  static char const* name() { return "binary"; }
+  std::string ToString() const override;
+
+ protected:
+  // Allow subclasses to change the logical type.
+  explicit BinaryType(Type::type logical_type)
+      : ListType(std::shared_ptr<DataType>(new UInt8Type()), logical_type) {}
+};
+
+// UTF encoded strings
+struct StringType : public BinaryType {
+  StringType() : BinaryType(Type::STRING) {}
 
   static char const* name() { return "string"; }
 
   std::string ToString() const override;
+
+ protected:
+  explicit StringType(Type::type logical_type) : BinaryType(logical_type) {}
 };
 
 struct StructType : public DataType {
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index bcb0ec490901f..2d913a737486f 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -127,10 +127,8 @@ Status MakeListArray(const TypePtr& type, int32_t length,
     case Type::LIST:
       out->reset(new ListArray(type, length, offsets, values, null_count, null_bitmap));
       break;
-    case Type::CHAR:
     case Type::DECIMAL_TEXT:
     case Type::STRING:
-    case Type::VARCHAR:
       out->reset(new StringArray(type, length, offsets, values, null_count, null_bitmap));
       break;
     default:
diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h
index 1be489d4f51b6..598df3ef70d2d 100644
--- a/cpp/src/arrow/types/decimal.h
+++ b/cpp/src/arrow/types/decimal.h
@@ -29,7 +29,6 @@ struct DecimalType : public DataType {
       : DataType(Type::DECIMAL), precision(precision_), scale(scale_) {}
   int precision;
   int scale;
-
   static char const* name() { return "decimal"; }
 
   std::string ToString() const override;
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 0a3941633eb83..2f6f85d66ca60 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -66,8 +66,8 @@ class ListArray : public Array {
   int32_t offset(int i) const { return offsets_[i]; }
 
   // Neither of these functions will perform boundschecking
-  int32_t value_offset(int i) { return offsets_[i]; }
-  int32_t value_length(int i) { return offsets_[i + 1] - offsets_[i]; }
+  int32_t value_offset(int i) const { return offsets_[i]; }
+  int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
 
   bool EqualsExact(const ListArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
@@ -92,9 +92,9 @@ class ListArray : public Array {
 // a sequence of offests and null values.
 //
 // A note on types.  Per arrow/type.h all types in the c++ implementation are
-// logical so even though this class always builds an Array of lists, this can
+// logical so even though this class always builds list array, this can
 // represent multiple different logical types.  If no logical type is provided
-// at construction time, the class defaults to List<T> where t is take from the
+// at construction time, the class defaults to List<T> where t is taken from the
 // value_builder/values that the object is constructed with.
 class ListBuilder : public ArrayBuilder {
  public:
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index ee4307c4d168a..a141fc113211a 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -34,32 +34,14 @@ namespace arrow {
 
 class Buffer;
 
-TEST(TypesTest, TestCharType) {
-  CharType t1(5);
-
-  ASSERT_EQ(t1.type, Type::CHAR);
-  ASSERT_EQ(t1.size, 5);
-
-  ASSERT_EQ(t1.ToString(), std::string("char(5)"));
-
-  // Test copy constructor
-  CharType t2 = t1;
-  ASSERT_EQ(t2.type, Type::CHAR);
-  ASSERT_EQ(t2.size, 5);
-}
-
-TEST(TypesTest, TestVarcharType) {
-  VarcharType t1(5);
-
-  ASSERT_EQ(t1.type, Type::VARCHAR);
-  ASSERT_EQ(t1.size, 5);
-
-  ASSERT_EQ(t1.ToString(), std::string("varchar(5)"));
-
-  // Test copy constructor
-  VarcharType t2 = t1;
-  ASSERT_EQ(t2.type, Type::VARCHAR);
-  ASSERT_EQ(t2.size, 5);
+TEST(TypesTest, BinaryType) {
+  BinaryType t1;
+  BinaryType e1;
+  StringType t2;
+  EXPECT_TRUE(t1.Equals(&e1));
+  EXPECT_FALSE(t1.Equals(&t2));
+  ASSERT_EQ(t1.type, Type::BINARY);
+  ASSERT_EQ(t1.ToString(), std::string("binary"));
 }
 
 TEST(TypesTest, TestStringType) {
@@ -119,6 +101,7 @@ class TestStringContainer : public ::testing::Test {
 TEST_F(TestStringContainer, TestArrayBasics) {
   ASSERT_EQ(length_, strings_->length());
   ASSERT_EQ(1, strings_->null_count());
+  ASSERT_OK(strings_->Validate());
 }
 
 TEST_F(TestStringContainer, TestType) {
@@ -163,7 +146,10 @@ class TestStringBuilder : public TestBuilder {
     builder_.reset(new StringBuilder(pool_, type_));
   }
 
-  void Done() { result_ = std::dynamic_pointer_cast<StringArray>(builder_->Finish()); }
+  void Done() {
+    result_ = std::dynamic_pointer_cast<StringArray>(builder_->Finish());
+    result_->Validate();
+  }
 
  protected:
   TypePtr type_;
@@ -216,4 +202,152 @@ TEST_F(TestStringBuilder, TestZeroLength) {
   Done();
 }
 
+// Binary container type
+// TODO(emkornfield) there should be some way to refactor these to avoid code duplicating
+// with String
+class TestBinaryContainer : public ::testing::Test {
+ public:
+  void SetUp() {
+    chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
+    offsets_ = {0, 1, 1, 1, 3, 6};
+    valid_bytes_ = {1, 1, 0, 1, 1};
+    expected_ = {"a", "", "", "bb", "ccc"};
+
+    MakeArray();
+  }
+
+  void MakeArray() {
+    length_ = offsets_.size() - 1;
+    int nchars = chars_.size();
+
+    value_buf_ = test::to_buffer(chars_);
+    values_ = ArrayPtr(new UInt8Array(nchars, value_buf_));
+
+    offsets_buf_ = test::to_buffer(offsets_);
+
+    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
+    null_count_ = test::null_count(valid_bytes_);
+
+    strings_ = std::make_shared<BinaryArray>(
+        length_, offsets_buf_, values_, null_count_, null_bitmap_);
+  }
+
+ protected:
+  std::vector<int32_t> offsets_;
+  std::vector<char> chars_;
+  std::vector<uint8_t> valid_bytes_;
+
+  std::vector<std::string> expected_;
+
+  std::shared_ptr<Buffer> value_buf_;
+  std::shared_ptr<Buffer> offsets_buf_;
+  std::shared_ptr<Buffer> null_bitmap_;
+
+  int null_count_;
+  int length_;
+
+  ArrayPtr values_;
+  std::shared_ptr<BinaryArray> strings_;
+};
+
+TEST_F(TestBinaryContainer, TestArrayBasics) {
+  ASSERT_EQ(length_, strings_->length());
+  ASSERT_EQ(1, strings_->null_count());
+  ASSERT_OK(strings_->Validate());
+}
+
+TEST_F(TestBinaryContainer, TestType) {
+  TypePtr type = strings_->type();
+
+  ASSERT_EQ(Type::BINARY, type->type);
+  ASSERT_EQ(Type::BINARY, strings_->type_enum());
+}
+
+TEST_F(TestBinaryContainer, TestListFunctions) {
+  int pos = 0;
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    ASSERT_EQ(pos, strings_->value_offset(i));
+    ASSERT_EQ(expected_[i].size(), strings_->value_length(i));
+    pos += expected_[i].size();
+  }
+}
+
+TEST_F(TestBinaryContainer, TestDestructor) {
+  auto arr = std::make_shared<BinaryArray>(
+      length_, offsets_buf_, values_, null_count_, null_bitmap_);
+}
+
+TEST_F(TestBinaryContainer, TestGetValue) {
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    if (valid_bytes_[i] == 0) {
+      ASSERT_TRUE(strings_->IsNull(i));
+    } else {
+      int32_t len = -1;
+      const uint8_t* bytes = strings_->GetValue(i, &len);
+      ASSERT_EQ(0, std::memcmp(expected_[i].data(), bytes, len));
+    }
+  }
+}
+
+class TestBinaryBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+    type_ = TypePtr(new BinaryType());
+    builder_.reset(new BinaryBuilder(pool_, type_));
+  }
+
+  void Done() {
+    result_ = std::dynamic_pointer_cast<BinaryArray>(builder_->Finish());
+    result_->Validate();
+  }
+
+ protected:
+  TypePtr type_;
+
+  std::unique_ptr<BinaryBuilder> builder_;
+  std::shared_ptr<BinaryArray> result_;
+};
+
+TEST_F(TestBinaryBuilder, TestScalarAppend) {
+  std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
+  std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
+
+  int N = strings.size();
+  int reps = 1000;
+
+  for (int j = 0; j < reps; ++j) {
+    for (int i = 0; i < N; ++i) {
+      if (is_null[i]) {
+        builder_->AppendNull();
+      } else {
+        builder_->Append(
+            reinterpret_cast<const uint8_t*>(strings[i].data()), strings[i].size());
+      }
+    }
+  }
+  Done();
+  ASSERT_OK(result_->Validate());
+  ASSERT_EQ(reps * N, result_->length());
+  ASSERT_EQ(reps, result_->null_count());
+  ASSERT_EQ(reps * 6, result_->values()->length());
+
+  int32_t length;
+  for (int i = 0; i < N * reps; ++i) {
+    if (is_null[i % N]) {
+      ASSERT_TRUE(result_->IsNull(i));
+    } else {
+      ASSERT_FALSE(result_->IsNull(i));
+      const uint8_t* vals = result_->GetValue(i, &length);
+      ASSERT_EQ(strings[i % N].size(), length);
+      ASSERT_EQ(0, std::memcmp(vals, strings[i % N].data(), length));
+    }
+  }
+}
+
+TEST_F(TestBinaryBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index 29d97d039477c..da02c7d1d8a9e 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -24,25 +24,43 @@
 
 namespace arrow {
 
+const std::shared_ptr<DataType> BINARY(new BinaryType());
 const std::shared_ptr<DataType> STRING(new StringType());
 
-StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
     const ArrayPtr& values, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap)
-    : StringArray(STRING, length, offsets, values, null_count, null_bitmap) {}
+    : BinaryArray(BINARY, length, offsets, values, null_count, null_bitmap) {}
+
+BinaryArray::BinaryArray(const TypePtr& type, int32_t length,
+    const std::shared_ptr<Buffer>& offsets, const ArrayPtr& values, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : ListArray(type, length, offsets, values, null_count, null_bitmap),
+      bytes_(std::dynamic_pointer_cast<UInt8Array>(values).get()),
+      raw_bytes_(bytes_->raw_data()) {
+  // Check in case the dynamic cast fails.
+  DCHECK(bytes_);
+}
 
-std::string CharType::ToString() const {
-  std::stringstream s;
-  s << "char(" << size << ")";
-  return s.str();
+Status BinaryArray::Validate() const {
+  if (values()->null_count() > 0) {
+    std::stringstream ss;
+    ss << type()->ToString() << " can have null values in the value array";
+    Status::Invalid(ss.str());
+  }
+  return ListArray::Validate();
 }
 
-std::string VarcharType::ToString() const {
-  std::stringstream s;
-  s << "varchar(" << size << ")";
-  return s.str();
+StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+    const ArrayPtr& values, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : StringArray(STRING, length, offsets, values, null_count, null_bitmap) {}
+
+Status StringArray::Validate() const {
+  // TODO(emkornfield) Validate proper UTF8 code points?
+  return BinaryArray::Validate();
 }
 
-TypePtr StringBuilder::value_type_ = TypePtr(new UInt8Type());
+TypePtr BinaryBuilder::value_type_ = TypePtr(new UInt8Type());
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index d2d3c5b6b5a83..b3c00d298b35c 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -34,87 +34,99 @@ namespace arrow {
 class Buffer;
 class MemoryPool;
 
-struct CharType : public DataType {
-  int size;
-
-  explicit CharType(int size) : DataType(Type::CHAR), size(size) {}
-
-  CharType(const CharType& other) : CharType(other.size) {}
-
-  virtual std::string ToString() const;
-};
-
-// Variable-length, null-terminated strings, up to a certain length
-struct VarcharType : public DataType {
-  int size;
-
-  explicit VarcharType(int size) : DataType(Type::VARCHAR), size(size) {}
-  VarcharType(const VarcharType& other) : VarcharType(other.size) {}
-
-  virtual std::string ToString() const;
-};
-
-// TODO(wesm): add a BinaryArray layer in between
-class StringArray : public ListArray {
+class BinaryArray : public ListArray {
  public:
-  StringArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
+  BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr)
-      : ListArray(type, length, offsets, values, null_count, null_bitmap) {
-    // For convenience
-    bytes_ = static_cast<UInt8Array*>(values.get());
-    raw_bytes_ = bytes_->raw_data();
-  }
-
-  StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  // Constructor that allows sub-classes/builders to propagate there logical type up the
+  // class hierarchy.
+  BinaryArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
-  // Compute the pointer t
+  // Return the pointer to the given elements bytes
+  // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
+  // pointer + offset
   const uint8_t* GetValue(int i, int32_t* out_length) const {
-    int32_t pos = offsets_[i];
+    DCHECK(out_length);
+    const int32_t pos = offsets_[i];
     *out_length = offsets_[i + 1] - pos;
     return raw_bytes_ + pos;
   }
 
+  Status Validate() const override;
+
+ private:
+  UInt8Array* bytes_;
+  const uint8_t* raw_bytes_;
+};
+
+class StringArray : public BinaryArray {
+ public:
+  StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const ArrayPtr& values, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  // Constructor that allows overriding the logical type, so subclasses can propagate
+  // there
+  // up the class hierarchy.
+  StringArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const ArrayPtr& values, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+      : BinaryArray(type, length, offsets, values, null_count, null_bitmap) {}
+
   // Construct a std::string
+  // TODO: std::bad_alloc possibility
   std::string GetString(int i) const {
     int32_t nchars;
     const uint8_t* str = GetValue(i, &nchars);
     return std::string(reinterpret_cast<const char*>(str), nchars);
   }
 
- private:
-  UInt8Array* bytes_;
-  const uint8_t* raw_bytes_;
+  Status Validate() const override;
 };
 
-// String builder
-class StringBuilder : public ListBuilder {
+// BinaryBuilder : public ListBuilder
+class BinaryBuilder : public ListBuilder {
  public:
-  explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
+  explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type)
       : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, value_type_), type) {
     byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
   }
 
-  Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
-
-  Status Append(const char* value, int32_t length) {
+  Status Append(const uint8_t* value, int32_t length) {
     RETURN_NOT_OK(ListBuilder::Append());
-    return byte_builder_->Append(reinterpret_cast<const uint8_t*>(value), length);
+    return byte_builder_->Append(value, length);
   }
-  Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
 
   std::shared_ptr<Array> Finish() override {
-    return ListBuilder::Transfer<StringArray>();
+    return ListBuilder::Transfer<BinaryArray>();
   }
 
  protected:
   UInt8Builder* byte_builder_;
-
   static TypePtr value_type_;
 };
 
+// String builder
+class StringBuilder : public BinaryBuilder {
+ public:
+  explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
+      : BinaryBuilder(pool, type) {}
+
+  Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
+
+  Status Append(const char* value, int32_t length) {
+    return BinaryBuilder::Append(reinterpret_cast<const uint8_t*>(value), length);
+  }
+
+  Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
+
+  std::shared_ptr<Array> Finish() override {
+    return ListBuilder::Transfer<StringArray>();
+  }
+};
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPES_STRING_H
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index 51e605ee50ac4..69ecda16ceba5 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -21,6 +21,6 @@
 // From Google gutil
 #define DISALLOW_COPY_AND_ASSIGN(TypeName) \
   TypeName(const TypeName&) = delete;      \
-  void operator=(const TypeName&) = delete
+  TypeName& operator=(const TypeName&) = delete
 
 #endif  // ARROW_UTIL_MACROS_H

From a3e3849cde60f611ea47271f510a96c2f36606a7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Thu, 16 Jun 2016 22:07:30 -0700
Subject: [PATCH 0090/1644] ARROW-219: Preserve CMAKE_CXX_FLAGS, fix compiler
 warnings

Some undesired compiler warnings had crept into our build; future warnings should fail the build now.

Author: Wes McKinney <wesm@apache.org>

Closes #92 from wesm/ARROW-219 and squashes the following commits:

fd68a74 [Wes McKinney] Buglet
6507351 [Wes McKinney] Fix clang warning
0f9e3ca [Wes McKinney] Preserve CMAKE_CXX_FLAGS, fix compiler warnings
---
 cpp/CMakeLists.txt                | 13 +++++++------
 cpp/src/arrow/parquet/test-util.h |  2 +-
 cpp/src/arrow/parquet/writer.cc   |  8 +++++++-
 cpp/src/arrow/parquet/writer.h    |  2 ++
 cpp/src/arrow/util/macros.h       |  2 ++
 5 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index a3fb01076d44e..bdf757238cc6b 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -139,15 +139,15 @@ string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
 # Set compile flags based on the build type.
 message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_DEBUG})
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_DEBUG}")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
-  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_FASTDEBUG})
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_FASTDEBUG}")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
-  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE})
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_RELEASE}")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN")
-  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_PROFILE_GEN})
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_GEN}")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD")
-  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_PROFILE_BUILD})
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_BUILD}")
 else()
   message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
 endif ()
@@ -165,6 +165,7 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   #   http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html
   #   http://petereisentraut.blogspot.com/2011/09/ccache-and-clang-part-2.html
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CLANG_OPTIONS}")
 endif()
 
 # Sanity check linking option.
@@ -559,7 +560,7 @@ if (${CLANG_TIDY_FOUND})
   add_custom_target(clang-tidy ${BUILD_SUPPORT_DIR}/run-clang-tidy.sh ${CLANG_TIDY_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 1
   `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc | sed -e '/_generated/g'`)
   # runs clang-tidy and exits with a non-zero exit code if any errors are found.
-  add_custom_target(check-clang-tidy ${BUILD_SUPPORT_DIR}/run-clang-tidy.sh ${CLANG_TIDY_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 
+  add_custom_target(check-clang-tidy ${BUILD_SUPPORT_DIR}/run-clang-tidy.sh ${CLANG_TIDY_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json
   0 `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc |grep -v -F -f ${CMAKE_CURRENT_SOURCE_DIR}/src/.clang-tidy-ignore | sed -e '/_generated/g'`)
 
 endif()
diff --git a/cpp/src/arrow/parquet/test-util.h b/cpp/src/arrow/parquet/test-util.h
index 1496082d5c661..cc8723bf6ecab 100644
--- a/cpp/src/arrow/parquet/test-util.h
+++ b/cpp/src/arrow/parquet/test-util.h
@@ -67,7 +67,7 @@ std::shared_ptr<Table> MakeSimpleTable(
 template <typename T>
 void ExpectArray(T* expected, Array* result) {
   PrimitiveArray* p_array = static_cast<PrimitiveArray*>(result);
-  for (size_t i = 0; i < result->length(); i++) {
+  for (int i = 0; i < result->length(); i++) {
     EXPECT_EQ(expected[i], reinterpret_cast<const T*>(p_array->data()->data())[i]);
   }
 }
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
index 1223901d5505a..4005e3b2b0c1b 100644
--- a/cpp/src/arrow/parquet/writer.cc
+++ b/cpp/src/arrow/parquet/writer.cc
@@ -50,6 +50,8 @@ class FileWriter::Impl {
   virtual ~Impl() {}
 
  private:
+  friend class FileWriter;
+
   MemoryPool* pool_;
   PoolBuffer data_buffer_;
   PoolBuffer def_levels_buffer_;
@@ -94,7 +96,7 @@ Status FileWriter::Impl::TypedWriteBatch(::parquet::ColumnWriter* column_writer,
       auto buffer_ptr =
           reinterpret_cast<typename ParquetType::c_type*>(data_buffer_.mutable_data());
       int buffer_idx = 0;
-      for (size_t i = 0; i < length; i++) {
+      for (int i = 0; i < length; i++) {
         if (data->IsNull(offset + i)) {
           def_levels_ptr[i] = 0;
         } else {
@@ -156,6 +158,10 @@ Status FileWriter::Close() {
   return impl_->Close();
 }
 
+MemoryPool* FileWriter::memory_pool() const {
+  return impl_->pool_;
+}
+
 FileWriter::~FileWriter() {}
 
 Status WriteFlatTable(const Table* table, MemoryPool* pool,
diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h
index 83e799f7ed1ed..93693f511846b 100644
--- a/cpp/src/arrow/parquet/writer.h
+++ b/cpp/src/arrow/parquet/writer.h
@@ -49,6 +49,8 @@ class FileWriter {
 
   virtual ~FileWriter();
 
+  MemoryPool* memory_pool() const;
+
  private:
   class Impl;
   std::unique_ptr<Impl> impl_;
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index 69ecda16ceba5..e2bb355115b42 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -19,8 +19,10 @@
 #define ARROW_UTIL_MACROS_H
 
 // From Google gutil
+#ifndef DISALLOW_COPY_AND_ASSIGN
 #define DISALLOW_COPY_AND_ASSIGN(TypeName) \
   TypeName(const TypeName&) = delete;      \
   TypeName& operator=(const TypeName&) = delete
+#endif
 
 #endif  // ARROW_UTIL_MACROS_H

From f7ade7bfeaa7e0d7fb3dd9d5a93e29a413cc142a Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 21 Jun 2016 15:11:26 -0700
Subject: [PATCH 0091/1644] ARROW-223: Do not link against libpython

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #95 from xhochy/arrow-223 and squashes the following commits:

4fdf1e7 [Uwe L. Korn] ARROW-223: Do not link against libpython

Change-Id: I1238a48aaf94ab175b367551f74c335c6455d78a
---
 python/cmake_modules/FindPythonLibsNew.cmake | 6 +++++-
 python/cmake_modules/UseCython.cmake         | 1 -
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/python/cmake_modules/FindPythonLibsNew.cmake b/python/cmake_modules/FindPythonLibsNew.cmake
index 0f2295aa43bc1..5cb65c9f1a484 100644
--- a/python/cmake_modules/FindPythonLibsNew.cmake
+++ b/python/cmake_modules/FindPythonLibsNew.cmake
@@ -224,7 +224,11 @@ FUNCTION(PYTHON_ADD_MODULE _NAME )
       SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS
                           "-undefined dynamic_lookup")
     ELSE()
-      TARGET_LINK_LIBRARIES(${_NAME} ${PYTHON_LIBRARIES})
+     # In general, we should not link against libpython as we do not embed
+     # the Python interpreter. The python binary itself can then define where
+     # the symbols should loaded from.
+     SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS
+         "-Wl,-undefined,dynamic_lookup")
     ENDIF()
     IF(PYTHON_MODULE_${_NAME}_BUILD_SHARED)
       SET_TARGET_PROPERTIES(${_NAME} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}")
diff --git a/python/cmake_modules/UseCython.cmake b/python/cmake_modules/UseCython.cmake
index 3b1c201edff5f..cee6066d31de0 100644
--- a/python/cmake_modules/UseCython.cmake
+++ b/python/cmake_modules/UseCython.cmake
@@ -163,7 +163,6 @@ function( cython_add_module _name pyx_target_name generated_files)
   include_directories( ${PYTHON_INCLUDE_DIRS} )
   python_add_module( ${_name} ${_generated_files} ${other_module_sources} )
   add_dependencies( ${_name} ${pyx_target_name})
-  target_link_libraries( ${_name} ${PYTHON_LIBRARIES} )
 endfunction()
 
 include( CMakeParseArguments )

From ef90830290491294d2fccfc5dcb16d3c0f96a70a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Fri, 24 Jun 2016 16:41:08 -0700
Subject: [PATCH 0092/1644] ARROW-222: Prototyping an IO interface for Arrow,
 with initial HDFS target

- Switch Travis CI back to Ubuntu trusty (old Boost in precise has issues with
  C++11)
- Adapt SFrame libhdfs shim for arrow
- Create C++ public API within arrow:io to libhdfs
- Implement and test many functions in libhdfs
- Start Cython wrapper interface to arrow_io. Begin Python file-like interface,
  unit tests
- Add thirdparty hdfs.h so builds are possible without a local Hadoop distro
  (e.g. in Travis CI).

Change-Id: I4a46e50f6c1c22787baa3749d8a542216341e630
---
 .travis.yml                              |    5 +-
 NOTICE.txt                               |    9 +
 ci/travis_before_script_cpp.sh           |   15 +-
 cpp/CMakeLists.txt                       |   60 +-
 cpp/doc/HDFS.md                          |   39 +
 cpp/src/arrow/io/CMakeLists.txt          |   97 ++
 cpp/src/arrow/io/hdfs-io-test.cc         |  315 +++++++
 cpp/src/arrow/io/hdfs.cc                 |  458 ++++++++++
 cpp/src/arrow/io/hdfs.h                  |  213 +++++
 cpp/src/arrow/io/interfaces.h            |   71 ++
 cpp/src/arrow/io/libhdfs_shim.cc         |  544 ++++++++++++
 cpp/src/arrow/parquet/parquet-io-test.cc |    4 +-
 cpp/thirdparty/hadoop/include/hdfs.h     | 1024 ++++++++++++++++++++++
 dev/merge_arrow_pr.py                    |    5 +-
 python/CMakeLists.txt                    |    6 +-
 python/cmake_modules/FindArrow.cmake     |   17 +-
 python/conda.recipe/meta.yaml            |    1 +
 python/pyarrow/error.pxd                 |    4 +-
 python/pyarrow/error.pyx                 |   14 +-
 python/pyarrow/includes/common.pxd       |   18 +
 python/pyarrow/includes/libarrow.pxd     |   19 -
 python/pyarrow/includes/libarrow_io.pxd  |   93 ++
 python/pyarrow/io.pyx                    |  504 +++++++++++
 python/pyarrow/tests/test_array.py       |   47 +-
 python/pyarrow/tests/test_io.py          |  126 +++
 python/setup.py                          |    9 +-
 26 files changed, 3656 insertions(+), 61 deletions(-)
 create mode 100644 NOTICE.txt
 create mode 100644 cpp/doc/HDFS.md
 create mode 100644 cpp/src/arrow/io/CMakeLists.txt
 create mode 100644 cpp/src/arrow/io/hdfs-io-test.cc
 create mode 100644 cpp/src/arrow/io/hdfs.cc
 create mode 100644 cpp/src/arrow/io/hdfs.h
 create mode 100644 cpp/src/arrow/io/interfaces.h
 create mode 100644 cpp/src/arrow/io/libhdfs_shim.cc
 create mode 100644 cpp/thirdparty/hadoop/include/hdfs.h
 create mode 100644 python/pyarrow/includes/libarrow_io.pxd
 create mode 100644 python/pyarrow/io.pyx
 create mode 100644 python/pyarrow/tests/test_io.py

diff --git a/.travis.yml b/.travis.yml
index ac2b0d457cb8e..97229b1ceb3bc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 sudo: required
-dist: precise
+dist: trusty
 addons:
   apt:
     sources:
@@ -12,6 +12,9 @@ addons:
     - ccache
     - cmake
     - valgrind
+    - libboost-dev
+    - libboost-filesystem-dev
+    - libboost-system-dev
 
 matrix:
   fast_finish: true
diff --git a/NOTICE.txt b/NOTICE.txt
new file mode 100644
index 0000000000000..0310c897cd743
--- /dev/null
+++ b/NOTICE.txt
@@ -0,0 +1,9 @@
+Apache Arrow
+Copyright 2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This product includes software from the SFrame project (BSD, 3-clause).
+* Copyright (C) 2015 Dato, Inc.
+* Copyright (c) 2009 Carnegie Mellon University.
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 9060cc9b5ef22..08551f3b009a8 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -23,12 +23,21 @@ echo $GTEST_HOME
 
 : ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
 
-CMAKE_COMMON_FLAGS="-DARROW_BUILD_BENCHMARKS=ON -DARROW_PARQUET=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
+CMAKE_COMMON_FLAGS="\
+-DARROW_BUILD_BENCHMARKS=ON \
+-DARROW_PARQUET=ON \
+-DARROW_HDFS=on \
+-DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
-  cmake -DARROW_TEST_MEMCHECK=on $CMAKE_COMMON_FLAGS -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+    cmake -DARROW_TEST_MEMCHECK=on \
+          $CMAKE_COMMON_FLAGS \
+          -DCMAKE_CXX_FLAGS="-Werror" \
+          $CPP_DIR
 else
-  cmake $CMAKE_COMMON_FLAGS -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+    cmake $CMAKE_COMMON_FLAGS \
+          -DCMAKE_CXX_FLAGS="-Werror" \
+          $CPP_DIR
 fi
 
 make -j4
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index bdf757238cc6b..18b47599b93d0 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -62,6 +62,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow IPC extensions"
     ON)
 
+  option(ARROW_HDFS
+    "Build the Arrow IO extensions for the Hadoop file system"
+    OFF)
+
   option(ARROW_SSE3
     "Build Arrow with SSE3"
     ON)
@@ -454,6 +458,47 @@ if ("$ENV{GBENCHMARK_HOME}" STREQUAL "")
   set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed)
 endif()
 
+# ----------------------------------------------------------------------
+# Add Boost dependencies (code adapted from Apache Kudu (incubating))
+
+# find boost headers and libs
+set(Boost_DEBUG TRUE)
+set(Boost_USE_MULTITHREADED ON)
+set(Boost_USE_STATIC_LIBS ON)
+find_package(Boost COMPONENTS system filesystem REQUIRED)
+include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
+set(BOOST_STATIC_LIBS ${Boost_LIBRARIES})
+list(LENGTH BOOST_STATIC_LIBS BOOST_STATIC_LIBS_LEN)
+
+# Find Boost shared libraries.
+set(Boost_USE_STATIC_LIBS OFF)
+find_package(Boost COMPONENTS system filesystem REQUIRED)
+set(BOOST_SHARED_LIBS ${Boost_LIBRARIES})
+list(LENGTH BOOST_SHARED_LIBS BOOST_SHARED_LIBS_LEN)
+list(SORT BOOST_SHARED_LIBS)
+
+message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
+message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
+
+math(EXPR LAST_IDX "${BOOST_STATIC_LIBS_LEN} - 1")
+foreach(IDX RANGE ${LAST_IDX})
+  list(GET BOOST_STATIC_LIBS ${IDX} BOOST_STATIC_LIB)
+  list(GET BOOST_SHARED_LIBS ${IDX} BOOST_SHARED_LIB)
+
+  # Remove the prefix/suffix from the library name.
+  #
+  # e.g. libboost_system-mt --> boost_system
+  get_filename_component(LIB_NAME ${BOOST_STATIC_LIB} NAME_WE)
+  string(REGEX REPLACE "lib([^-]*)(-mt)?" "\\1" LIB_NAME_NO_PREFIX_SUFFIX ${LIB_NAME})
+  ADD_THIRDPARTY_LIB(${LIB_NAME_NO_PREFIX_SUFFIX}
+    STATIC_LIB "${BOOST_STATIC_LIB}"
+    SHARED_LIB "${BOOST_SHARED_LIB}")
+  list(APPEND ARROW_BOOST_LIBS ${LIB_NAME_NO_PREFIX_SUFFIX})
+endforeach()
+include_directories(SYSTEM ${Boost_INCLUDE_DIR})
+
+# ----------------------------------------------------------------------
+# Enable / disable tests and benchmarks
 
 if(ARROW_BUILD_TESTS)
   add_custom_target(unittest ctest -L unittest)
@@ -529,12 +574,24 @@ endif (UNIX)
 # "make lint" target
 ############################################################
 if (UNIX)
+
+  file(GLOB_RECURSE LINT_FILES
+	"${CMAKE_CURRENT_SOURCE_DIR}/src/*.h"
+	"${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc"
+	)
+
+  FOREACH(item ${LINT_FILES})
+	IF(NOT (item MATCHES "_generated.h"))
+      LIST(APPEND FILTERED_LINT_FILES ${item})
+	ENDIF()
+  ENDFOREACH(item ${LINT_FILES})
+
   # Full lint
   add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
   --verbose=2
   --linelength=90
   --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11,-runtime/references
-    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`)
+  ${FILTERED_LINT_FILES})
 endif (UNIX)
 
 
@@ -624,6 +681,7 @@ set_target_properties(arrow
 target_link_libraries(arrow ${LIBARROW_LINK_LIBS})
 
 add_subdirectory(src/arrow)
+add_subdirectory(src/arrow/io)
 add_subdirectory(src/arrow/util)
 add_subdirectory(src/arrow/types)
 
diff --git a/cpp/doc/HDFS.md b/cpp/doc/HDFS.md
new file mode 100644
index 0000000000000..e0d5dfda21d93
--- /dev/null
+++ b/cpp/doc/HDFS.md
@@ -0,0 +1,39 @@
+## Using Arrow's HDFS (Apache Hadoop Distributed File System) interface
+
+### Build requirements
+
+To build the integration, pass the following option to CMake
+
+```shell
+-DARROW_HDFS=on
+```
+
+For convenience, we have bundled `hdfs.h` for libhdfs from Apache Hadoop in
+Arrow's thirdparty. If you wish to build against the `hdfs.h` in your installed
+Hadoop distribution, set the `$HADOOP_HOME` environment variable.
+
+### Runtime requirements
+
+By default, the HDFS client C++ class in `libarrow_io` uses the libhdfs JNI
+interface to the Java Hadoop client. This library is loaded **at runtime**
+(rather than at link / library load time, since the library may not be in your
+LD_LIBRARY_PATH), and relies on some environment variables.
+
+* `HADOOP_HOME`: the root of your installed Hadoop distribution. Check in the
+  `lib/native` directory to look for `libhdfs.so` if you have any questions
+  about which directory you're after.
+* `JAVA_HOME`: the location of your Java SDK installation
+* `CLASSPATH`: must contain the Hadoop jars. You can set these using:
+
+```shell
+export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
+```
+
+#### Setting $JAVA_HOME  automatically on OS X
+
+The installed location of Java on OS X can vary, however the following snippet
+will set it automatically for you:
+
+```shell
+export JAVA_HOME=$(/usr/libexec/java_home)
+```
\ No newline at end of file
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
new file mode 100644
index 0000000000000..33b654f81903f
--- /dev/null
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ----------------------------------------------------------------------
+# arrow_io : Arrow IO interfaces
+
+set(ARROW_IO_LINK_LIBS
+  arrow
+)
+
+set(ARROW_IO_PRIVATE_LINK_LIBS
+  boost_system
+  boost_filesystem
+)
+
+set(ARROW_IO_TEST_LINK_LIBS
+  arrow_io
+  ${ARROW_IO_PRIVATE_LINK_LIBS})
+
+set(ARROW_IO_SRCS
+)
+
+if(ARROW_HDFS)
+  if(NOT THIRDPARTY_DIR)
+	message(FATAL_ERROR "THIRDPARTY_DIR not set")
+  endif()
+
+  if (DEFINED ENV{HADOOP_HOME})
+	set(HADOOP_HOME $ENV{HADOOP_HOME})
+  else()
+	set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+  endif()
+
+  set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
+  if (NOT EXISTS ${HDFS_H_PATH})
+	message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
+  endif()
+  message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
+  message(STATUS "Building libhdfs shim component")
+
+  include_directories(SYSTEM "${HADOOP_HOME}/include")
+
+  set(ARROW_HDFS_SRCS
+	hdfs.cc
+	libhdfs_shim.cc)
+
+  set_property(SOURCE ${ARROW_HDFS_SRCS}
+	APPEND_STRING PROPERTY
+	COMPILE_FLAGS "-DHAS_HADOOP")
+
+  set(ARROW_IO_SRCS
+	${ARROW_HDFS_SRCS}
+	${ARROW_IO_SRCS})
+
+  ADD_ARROW_TEST(hdfs-io-test)
+  ARROW_TEST_LINK_LIBRARIES(hdfs-io-test
+	${ARROW_IO_TEST_LINK_LIBS})
+endif()
+
+add_library(arrow_io SHARED
+  ${ARROW_IO_SRCS}
+)
+target_link_libraries(arrow_io LINK_PUBLIC ${ARROW_IO_LINK_LIBS})
+target_link_libraries(arrow_io LINK_PRIVATE ${ARROW_IO_PRIVATE_LINK_LIBS})
+
+SET_TARGET_PROPERTIES(arrow_io PROPERTIES LINKER_LANGUAGE CXX)
+
+if (APPLE)
+  set_target_properties(arrow_io
+    PROPERTIES
+    BUILD_WITH_INSTALL_RPATH ON
+    INSTALL_NAME_DIR "@rpath")
+endif()
+
+# Headers: top level
+install(FILES
+  hdfs.h
+  interfaces.h
+  DESTINATION include/arrow/io)
+
+install(TARGETS arrow_io
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib)
diff --git a/cpp/src/arrow/io/hdfs-io-test.cc b/cpp/src/arrow/io/hdfs-io-test.cc
new file mode 100644
index 0000000000000..11d67aeba2026
--- /dev/null
+++ b/cpp/src/arrow/io/hdfs-io-test.cc
@@ -0,0 +1,315 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdlib>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include <boost/filesystem.hpp>  // NOLINT
+
+#include "arrow/io/hdfs.h"
+#include "arrow/test-util.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace io {
+
+std::vector<uint8_t> RandomData(int64_t size) {
+  std::vector<uint8_t> buffer(size);
+  test::random_bytes(size, 0, buffer.data());
+  return buffer;
+}
+
+class TestHdfsClient : public ::testing::Test {
+ public:
+  Status MakeScratchDir() {
+    if (client_->Exists(scratch_dir_)) {
+      RETURN_NOT_OK((client_->Delete(scratch_dir_, true)));
+    }
+    return client_->CreateDirectory(scratch_dir_);
+  }
+
+  Status WriteDummyFile(const std::string& path, const uint8_t* buffer, int64_t size,
+      bool append = false, int buffer_size = 0, int replication = 0,
+      int default_block_size = 0) {
+    std::shared_ptr<HdfsWriteableFile> file;
+    RETURN_NOT_OK(client_->OpenWriteable(
+        path, append, buffer_size, replication, default_block_size, &file));
+
+    RETURN_NOT_OK(file->Write(buffer, size));
+    RETURN_NOT_OK(file->Close());
+
+    return Status::OK();
+  }
+
+  std::string ScratchPath(const std::string& name) {
+    std::stringstream ss;
+    ss << scratch_dir_ << "/" << name;
+    return ss.str();
+  }
+
+  std::string HdfsAbsPath(const std::string& relpath) {
+    std::stringstream ss;
+    ss << "hdfs://" << conf_.host << ":" << conf_.port << relpath;
+    return ss.str();
+  }
+
+ protected:
+  // Set up shared state between unit tests
+  static void SetUpTestCase() {
+    if (!ConnectLibHdfs().ok()) {
+      std::cout << "Loading libhdfs failed, skipping tests gracefully" << std::endl;
+      return;
+    }
+
+    loaded_libhdfs_ = true;
+
+    const char* host = std::getenv("ARROW_HDFS_TEST_HOST");
+    const char* port = std::getenv("ARROW_HDFS_TEST_PORT");
+    const char* user = std::getenv("ARROW_HDFS_TEST_USER");
+
+    ASSERT_TRUE(user) << "Set ARROW_HDFS_TEST_USER";
+
+    conf_.host = host == nullptr ? "localhost" : host;
+    conf_.user = user;
+    conf_.port = port == nullptr ? 20500 : atoi(port);
+
+    ASSERT_OK(HdfsClient::Connect(&conf_, &client_));
+  }
+
+  static void TearDownTestCase() {
+    if (client_) {
+      EXPECT_OK(client_->Delete(scratch_dir_, true));
+      EXPECT_OK(client_->Disconnect());
+    }
+  }
+
+  static bool loaded_libhdfs_;
+
+  // Resources shared amongst unit tests
+  static HdfsConnectionConfig conf_;
+  static std::string scratch_dir_;
+  static std::shared_ptr<HdfsClient> client_;
+};
+
+bool TestHdfsClient::loaded_libhdfs_ = false;
+HdfsConnectionConfig TestHdfsClient::conf_ = HdfsConnectionConfig();
+
+std::string TestHdfsClient::scratch_dir_ =
+    boost::filesystem::unique_path("/tmp/arrow-hdfs/scratch-%%%%").native();
+
+std::shared_ptr<HdfsClient> TestHdfsClient::client_ = nullptr;
+
+#define SKIP_IF_NO_LIBHDFS()                          \
+  if (!loaded_libhdfs_) {                             \
+    std::cout << "No libhdfs, skipping" << std::endl; \
+    return;                                           \
+  }
+
+TEST_F(TestHdfsClient, ConnectsAgain) {
+  SKIP_IF_NO_LIBHDFS();
+
+  std::shared_ptr<HdfsClient> client;
+  ASSERT_OK(HdfsClient::Connect(&conf_, &client));
+  ASSERT_OK(client->Disconnect());
+}
+
+TEST_F(TestHdfsClient, CreateDirectory) {
+  SKIP_IF_NO_LIBHDFS();
+
+  std::string path = ScratchPath("create-directory");
+
+  if (client_->Exists(path)) { ASSERT_OK(client_->Delete(path, true)); }
+
+  ASSERT_OK(client_->CreateDirectory(path));
+  ASSERT_TRUE(client_->Exists(path));
+  EXPECT_OK(client_->Delete(path, true));
+  ASSERT_FALSE(client_->Exists(path));
+}
+
+TEST_F(TestHdfsClient, GetCapacityUsed) {
+  SKIP_IF_NO_LIBHDFS();
+
+  // Who knows what is actually in your DFS cluster, but expect it to have
+  // positive used bytes and capacity
+  int64_t nbytes = 0;
+  ASSERT_OK(client_->GetCapacity(&nbytes));
+  ASSERT_LT(0, nbytes);
+
+  ASSERT_OK(client_->GetUsed(&nbytes));
+  ASSERT_LT(0, nbytes);
+}
+
+TEST_F(TestHdfsClient, GetPathInfo) {
+  SKIP_IF_NO_LIBHDFS();
+
+  HdfsPathInfo info;
+
+  ASSERT_OK(MakeScratchDir());
+
+  // Directory info
+  ASSERT_OK(client_->GetPathInfo(scratch_dir_, &info));
+  ASSERT_EQ(ObjectType::DIRECTORY, info.kind);
+  ASSERT_EQ(HdfsAbsPath(scratch_dir_), info.name);
+  ASSERT_EQ(conf_.user, info.owner);
+
+  // TODO(wesm): test group, other attrs
+
+  auto path = ScratchPath("test-file");
+
+  const int size = 100;
+
+  std::vector<uint8_t> buffer = RandomData(size);
+
+  ASSERT_OK(WriteDummyFile(path, buffer.data(), size));
+  ASSERT_OK(client_->GetPathInfo(path, &info));
+
+  ASSERT_EQ(ObjectType::FILE, info.kind);
+  ASSERT_EQ(HdfsAbsPath(path), info.name);
+  ASSERT_EQ(conf_.user, info.owner);
+  ASSERT_EQ(size, info.size);
+}
+
+TEST_F(TestHdfsClient, AppendToFile) {
+  SKIP_IF_NO_LIBHDFS();
+
+  ASSERT_OK(MakeScratchDir());
+
+  auto path = ScratchPath("test-file");
+  const int size = 100;
+
+  std::vector<uint8_t> buffer = RandomData(size);
+  ASSERT_OK(WriteDummyFile(path, buffer.data(), size));
+
+  // now append
+  ASSERT_OK(WriteDummyFile(path, buffer.data(), size, true));
+
+  HdfsPathInfo info;
+  ASSERT_OK(client_->GetPathInfo(path, &info));
+  ASSERT_EQ(size * 2, info.size);
+}
+
+TEST_F(TestHdfsClient, ListDirectory) {
+  SKIP_IF_NO_LIBHDFS();
+
+  const int size = 100;
+  std::vector<uint8_t> data = RandomData(size);
+
+  auto p1 = ScratchPath("test-file-1");
+  auto p2 = ScratchPath("test-file-2");
+  auto d1 = ScratchPath("test-dir-1");
+
+  ASSERT_OK(MakeScratchDir());
+  ASSERT_OK(WriteDummyFile(p1, data.data(), size));
+  ASSERT_OK(WriteDummyFile(p2, data.data(), size / 2));
+  ASSERT_OK(client_->CreateDirectory(d1));
+
+  std::vector<HdfsPathInfo> listing;
+  ASSERT_OK(client_->ListDirectory(scratch_dir_, &listing));
+
+  // Do it again, appends!
+  ASSERT_OK(client_->ListDirectory(scratch_dir_, &listing));
+
+  ASSERT_EQ(6, listing.size());
+
+  // Argh, well, shouldn't expect the listing to be in any particular order
+  for (size_t i = 0; i < listing.size(); ++i) {
+    const HdfsPathInfo& info = listing[i];
+    if (info.name == HdfsAbsPath(p1)) {
+      ASSERT_EQ(ObjectType::FILE, info.kind);
+      ASSERT_EQ(size, info.size);
+    } else if (info.name == HdfsAbsPath(p2)) {
+      ASSERT_EQ(ObjectType::FILE, info.kind);
+      ASSERT_EQ(size / 2, info.size);
+    } else if (info.name == HdfsAbsPath(d1)) {
+      ASSERT_EQ(ObjectType::DIRECTORY, info.kind);
+    } else {
+      FAIL() << "Unexpected path: " << info.name;
+    }
+  }
+}
+
+TEST_F(TestHdfsClient, ReadableMethods) {
+  SKIP_IF_NO_LIBHDFS();
+
+  ASSERT_OK(MakeScratchDir());
+
+  auto path = ScratchPath("test-file");
+  const int size = 100;
+
+  std::vector<uint8_t> data = RandomData(size);
+  ASSERT_OK(WriteDummyFile(path, data.data(), size));
+
+  std::shared_ptr<HdfsReadableFile> file;
+  ASSERT_OK(client_->OpenReadable(path, &file));
+
+  // Test GetSize -- move this into its own unit test if ever needed
+  int64_t file_size;
+  ASSERT_OK(file->GetSize(&file_size));
+  ASSERT_EQ(size, file_size);
+
+  uint8_t buffer[50];
+  int32_t bytes_read = 0;
+
+  ASSERT_OK(file->Read(50, &bytes_read, buffer));
+  ASSERT_EQ(0, std::memcmp(buffer, data.data(), 50));
+  ASSERT_EQ(50, bytes_read);
+
+  ASSERT_OK(file->Read(50, &bytes_read, buffer));
+  ASSERT_EQ(0, std::memcmp(buffer, data.data() + 50, 50));
+  ASSERT_EQ(50, bytes_read);
+
+  // EOF
+  ASSERT_OK(file->Read(1, &bytes_read, buffer));
+  ASSERT_EQ(0, bytes_read);
+
+  // ReadAt to EOF
+  ASSERT_OK(file->ReadAt(60, 100, &bytes_read, buffer));
+  ASSERT_EQ(40, bytes_read);
+  ASSERT_EQ(0, std::memcmp(buffer, data.data() + 60, bytes_read));
+
+  // Seek, Tell
+  ASSERT_OK(file->Seek(60));
+
+  int64_t position;
+  ASSERT_OK(file->Tell(&position));
+  ASSERT_EQ(60, position);
+}
+
+TEST_F(TestHdfsClient, RenameFile) {
+  SKIP_IF_NO_LIBHDFS();
+
+  ASSERT_OK(MakeScratchDir());
+
+  auto src_path = ScratchPath("src-file");
+  auto dst_path = ScratchPath("dst-file");
+  const int size = 100;
+
+  std::vector<uint8_t> data = RandomData(size);
+  ASSERT_OK(WriteDummyFile(src_path, data.data(), size));
+
+  ASSERT_OK(client_->Rename(src_path, dst_path));
+
+  ASSERT_FALSE(client_->Exists(src_path));
+  ASSERT_TRUE(client_->Exists(dst_path));
+}
+
+}  // namespace io
+}  // namespace arrow
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
new file mode 100644
index 0000000000000..6da6ea4e71bd8
--- /dev/null
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -0,0 +1,458 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <hdfs.h>
+
+#include <cstdint>
+#include <sstream>
+#include <string>
+
+#include "arrow/io/hdfs.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace io {
+
+#define CHECK_FAILURE(RETURN_VALUE, WHAT)  \
+  do {                                     \
+    if (RETURN_VALUE == -1) {              \
+      std::stringstream ss;                \
+      ss << "HDFS: " << WHAT << " failed"; \
+      return Status::IOError(ss.str());    \
+    }                                      \
+  } while (0)
+
+static Status CheckReadResult(int ret) {
+  // Check for error on -1 (possibly errno set)
+
+  // ret == 0 at end of file, which is OK
+  if (ret == -1) {
+    // EOF
+    std::stringstream ss;
+    ss << "HDFS read failed, errno: " << errno;
+    return Status::IOError(ss.str());
+  }
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// File reading
+
+class HdfsAnyFileImpl {
+ public:
+  void set_members(const std::string& path, hdfsFS fs, hdfsFile handle) {
+    path_ = path;
+    fs_ = fs;
+    file_ = handle;
+    is_open_ = true;
+  }
+
+  Status Seek(int64_t position) {
+    int ret = hdfsSeek(fs_, file_, position);
+    CHECK_FAILURE(ret, "seek");
+    return Status::OK();
+  }
+
+  Status Tell(int64_t* offset) {
+    int64_t ret = hdfsTell(fs_, file_);
+    CHECK_FAILURE(ret, "tell");
+    *offset = ret;
+    return Status::OK();
+  }
+
+  bool is_open() const { return is_open_; }
+
+ protected:
+  std::string path_;
+
+  // These are pointers in libhdfs, so OK to copy
+  hdfsFS fs_;
+  hdfsFile file_;
+
+  bool is_open_;
+};
+
+// Private implementation for read-only files
+class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
+ public:
+  HdfsReadableFileImpl() {}
+
+  Status Close() {
+    if (is_open_) {
+      int ret = hdfsCloseFile(fs_, file_);
+      CHECK_FAILURE(ret, "CloseFile");
+      is_open_ = false;
+    }
+    return Status::OK();
+  }
+
+  Status ReadAt(int64_t position, int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) {
+    tSize ret = hdfsPread(fs_, file_, static_cast<tOffset>(position),
+        reinterpret_cast<void*>(buffer), nbytes);
+    RETURN_NOT_OK(CheckReadResult(ret));
+    *bytes_read = ret;
+    return Status::OK();
+  }
+
+  Status Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) {
+    tSize ret = hdfsRead(fs_, file_, reinterpret_cast<void*>(buffer), nbytes);
+    RETURN_NOT_OK(CheckReadResult(ret));
+    *bytes_read = ret;
+    return Status::OK();
+  }
+
+  Status GetSize(int64_t* size) {
+    hdfsFileInfo* entry = hdfsGetPathInfo(fs_, path_.c_str());
+    if (entry == nullptr) { return Status::IOError("HDFS: GetPathInfo failed"); }
+
+    *size = entry->mSize;
+    hdfsFreeFileInfo(entry, 1);
+    return Status::OK();
+  }
+};
+
+HdfsReadableFile::HdfsReadableFile() {
+  impl_.reset(new HdfsReadableFileImpl());
+}
+
+HdfsReadableFile::~HdfsReadableFile() {
+  impl_->Close();
+}
+
+Status HdfsReadableFile::Close() {
+  return impl_->Close();
+}
+
+Status HdfsReadableFile::ReadAt(
+    int64_t position, int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) {
+  return impl_->ReadAt(position, nbytes, bytes_read, buffer);
+}
+
+Status HdfsReadableFile::Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) {
+  return impl_->Read(nbytes, bytes_read, buffer);
+}
+
+Status HdfsReadableFile::GetSize(int64_t* size) {
+  return impl_->GetSize(size);
+}
+
+Status HdfsReadableFile::Seek(int64_t position) {
+  return impl_->Seek(position);
+}
+
+Status HdfsReadableFile::Tell(int64_t* position) {
+  return impl_->Tell(position);
+}
+
+// ----------------------------------------------------------------------
+// File writing
+
+// Private implementation for writeable-only files
+class HdfsWriteableFile::HdfsWriteableFileImpl : public HdfsAnyFileImpl {
+ public:
+  HdfsWriteableFileImpl() {}
+
+  Status Close() {
+    if (is_open_) {
+      int ret = hdfsFlush(fs_, file_);
+      CHECK_FAILURE(ret, "Flush");
+      ret = hdfsCloseFile(fs_, file_);
+      CHECK_FAILURE(ret, "CloseFile");
+      is_open_ = false;
+    }
+    return Status::OK();
+  }
+
+  Status Write(const uint8_t* buffer, int32_t nbytes, int32_t* bytes_written) {
+    tSize ret = hdfsWrite(fs_, file_, reinterpret_cast<const void*>(buffer), nbytes);
+    CHECK_FAILURE(ret, "Write");
+    *bytes_written = ret;
+    return Status::OK();
+  }
+};
+
+HdfsWriteableFile::HdfsWriteableFile() {
+  impl_.reset(new HdfsWriteableFileImpl());
+}
+
+HdfsWriteableFile::~HdfsWriteableFile() {
+  impl_->Close();
+}
+
+Status HdfsWriteableFile::Close() {
+  return impl_->Close();
+}
+
+Status HdfsWriteableFile::Write(
+    const uint8_t* buffer, int32_t nbytes, int32_t* bytes_read) {
+  return impl_->Write(buffer, nbytes, bytes_read);
+}
+
+Status HdfsWriteableFile::Write(const uint8_t* buffer, int32_t nbytes) {
+  int32_t bytes_written_dummy = 0;
+  return Write(buffer, nbytes, &bytes_written_dummy);
+}
+
+Status HdfsWriteableFile::Tell(int64_t* position) {
+  return impl_->Tell(position);
+}
+
+// ----------------------------------------------------------------------
+// HDFS client
+
+// TODO(wesm): this could throw std::bad_alloc in the course of copying strings
+// into the path info object
+static void SetPathInfo(const hdfsFileInfo* input, HdfsPathInfo* out) {
+  out->kind = input->mKind == kObjectKindFile ? ObjectType::FILE : ObjectType::DIRECTORY;
+  out->name = std::string(input->mName);
+  out->owner = std::string(input->mOwner);
+  out->group = std::string(input->mGroup);
+
+  out->last_access_time = static_cast<int32_t>(input->mLastAccess);
+  out->last_modified_time = static_cast<int32_t>(input->mLastMod);
+  out->size = static_cast<int64_t>(input->mSize);
+
+  out->replication = input->mReplication;
+  out->block_size = input->mBlockSize;
+
+  out->permissions = input->mPermissions;
+}
+
+// Private implementation
+class HdfsClient::HdfsClientImpl {
+ public:
+  HdfsClientImpl() {}
+
+  Status Connect(const HdfsConnectionConfig* config) {
+    RETURN_NOT_OK(ConnectLibHdfs());
+
+    fs_ = hdfsConnectAsUser(config->host.c_str(), config->port, config->user.c_str());
+
+    if (fs_ == nullptr) { return Status::IOError("HDFS connection failed"); }
+    namenode_host_ = config->host;
+    port_ = config->port;
+    user_ = config->user;
+
+    return Status::OK();
+  }
+
+  Status CreateDirectory(const std::string& path) {
+    int ret = hdfsCreateDirectory(fs_, path.c_str());
+    CHECK_FAILURE(ret, "create directory");
+    return Status::OK();
+  }
+
+  Status Delete(const std::string& path, bool recursive) {
+    int ret = hdfsDelete(fs_, path.c_str(), static_cast<int>(recursive));
+    CHECK_FAILURE(ret, "delete");
+    return Status::OK();
+  }
+
+  Status Disconnect() {
+    int ret = hdfsDisconnect(fs_);
+    CHECK_FAILURE(ret, "hdfsFS::Disconnect");
+    return Status::OK();
+  }
+
+  bool Exists(const std::string& path) {
+    // hdfsExists does not distinguish between RPC failure and the file not
+    // existing
+    int ret = hdfsExists(fs_, path.c_str());
+    return ret == 0;
+  }
+
+  Status GetCapacity(int64_t* nbytes) {
+    tOffset ret = hdfsGetCapacity(fs_);
+    CHECK_FAILURE(ret, "GetCapacity");
+    *nbytes = ret;
+    return Status::OK();
+  }
+
+  Status GetUsed(int64_t* nbytes) {
+    tOffset ret = hdfsGetUsed(fs_);
+    CHECK_FAILURE(ret, "GetUsed");
+    *nbytes = ret;
+    return Status::OK();
+  }
+
+  Status GetPathInfo(const std::string& path, HdfsPathInfo* info) {
+    hdfsFileInfo* entry = hdfsGetPathInfo(fs_, path.c_str());
+
+    if (entry == nullptr) { return Status::IOError("HDFS: GetPathInfo failed"); }
+
+    SetPathInfo(entry, info);
+    hdfsFreeFileInfo(entry, 1);
+
+    return Status::OK();
+  }
+
+  Status ListDirectory(const std::string& path, std::vector<HdfsPathInfo>* listing) {
+    int num_entries = 0;
+    hdfsFileInfo* entries = hdfsListDirectory(fs_, path.c_str(), &num_entries);
+
+    if (entries == nullptr) {
+      // If the directory is empty, entries is NULL but errno is 0. Non-zero
+      // errno indicates error
+      //
+      // Note: errno is thread-locala
+      if (errno == 0) { num_entries = 0; }
+      { return Status::IOError("HDFS: list directory failed"); }
+    }
+
+    // Allocate additional space for elements
+
+    int vec_offset = listing->size();
+    listing->resize(vec_offset + num_entries);
+
+    for (int i = 0; i < num_entries; ++i) {
+      SetPathInfo(entries + i, &(*listing)[vec_offset + i]);
+    }
+
+    // Free libhdfs file info
+    hdfsFreeFileInfo(entries, num_entries);
+
+    return Status::OK();
+  }
+
+  Status OpenReadable(const std::string& path, std::shared_ptr<HdfsReadableFile>* file) {
+    hdfsFile handle = hdfsOpenFile(fs_, path.c_str(), O_RDONLY, 0, 0, 0);
+
+    if (handle == nullptr) {
+      // TODO(wesm): determine cause of failure
+      std::stringstream ss;
+      ss << "Unable to open file " << path;
+      return Status::IOError(ss.str());
+    }
+
+    // std::make_shared does not work with private ctors
+    *file = std::shared_ptr<HdfsReadableFile>(new HdfsReadableFile());
+    (*file)->impl_->set_members(path, fs_, handle);
+
+    return Status::OK();
+  }
+
+  Status OpenWriteable(const std::string& path, bool append, int32_t buffer_size,
+      int16_t replication, int64_t default_block_size,
+      std::shared_ptr<HdfsWriteableFile>* file) {
+    int flags = O_WRONLY;
+    if (append) flags |= O_APPEND;
+
+    hdfsFile handle = hdfsOpenFile(
+        fs_, path.c_str(), flags, buffer_size, replication, default_block_size);
+
+    if (handle == nullptr) {
+      // TODO(wesm): determine cause of failure
+      std::stringstream ss;
+      ss << "Unable to open file " << path;
+      return Status::IOError(ss.str());
+    }
+
+    // std::make_shared does not work with private ctors
+    *file = std::shared_ptr<HdfsWriteableFile>(new HdfsWriteableFile());
+    (*file)->impl_->set_members(path, fs_, handle);
+
+    return Status::OK();
+  }
+
+  Status Rename(const std::string& src, const std::string& dst) {
+    int ret = hdfsRename(fs_, src.c_str(), dst.c_str());
+    CHECK_FAILURE(ret, "Rename");
+    return Status::OK();
+  }
+
+ private:
+  std::string namenode_host_;
+  std::string user_;
+  int port_;
+
+  hdfsFS fs_;
+};
+
+// ----------------------------------------------------------------------
+// Public API for HDFSClient
+
+HdfsClient::HdfsClient() {
+  impl_.reset(new HdfsClientImpl());
+}
+
+HdfsClient::~HdfsClient() {}
+
+Status HdfsClient::Connect(
+    const HdfsConnectionConfig* config, std::shared_ptr<HdfsClient>* fs) {
+  // ctor is private, make_shared will not work
+  *fs = std::shared_ptr<HdfsClient>(new HdfsClient());
+
+  RETURN_NOT_OK((*fs)->impl_->Connect(config));
+  return Status::OK();
+}
+
+Status HdfsClient::CreateDirectory(const std::string& path) {
+  return impl_->CreateDirectory(path);
+}
+
+Status HdfsClient::Delete(const std::string& path, bool recursive) {
+  return impl_->Delete(path, recursive);
+}
+
+Status HdfsClient::Disconnect() {
+  return impl_->Disconnect();
+}
+
+bool HdfsClient::Exists(const std::string& path) {
+  return impl_->Exists(path);
+}
+
+Status HdfsClient::GetPathInfo(const std::string& path, HdfsPathInfo* info) {
+  return impl_->GetPathInfo(path, info);
+}
+
+Status HdfsClient::GetCapacity(int64_t* nbytes) {
+  return impl_->GetCapacity(nbytes);
+}
+
+Status HdfsClient::GetUsed(int64_t* nbytes) {
+  return impl_->GetUsed(nbytes);
+}
+
+Status HdfsClient::ListDirectory(
+    const std::string& path, std::vector<HdfsPathInfo>* listing) {
+  return impl_->ListDirectory(path, listing);
+}
+
+Status HdfsClient::OpenReadable(
+    const std::string& path, std::shared_ptr<HdfsReadableFile>* file) {
+  return impl_->OpenReadable(path, file);
+}
+
+Status HdfsClient::OpenWriteable(const std::string& path, bool append,
+    int32_t buffer_size, int16_t replication, int64_t default_block_size,
+    std::shared_ptr<HdfsWriteableFile>* file) {
+  return impl_->OpenWriteable(
+      path, append, buffer_size, replication, default_block_size, file);
+}
+
+Status HdfsClient::OpenWriteable(
+    const std::string& path, bool append, std::shared_ptr<HdfsWriteableFile>* file) {
+  return OpenWriteable(path, append, 0, 0, 0, file);
+}
+
+Status HdfsClient::Rename(const std::string& src, const std::string& dst) {
+  return impl_->Rename(src, dst);
+}
+
+}  // namespace io
+}  // namespace arrow
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
new file mode 100644
index 0000000000000..a1972db96157a
--- /dev/null
+++ b/cpp/src/arrow/io/hdfs.h
@@ -0,0 +1,213 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IO_HDFS
+#define ARROW_IO_HDFS
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/io/interfaces.h"
+#include "arrow/util/macros.h"
+
+namespace arrow {
+
+class Status;
+
+namespace io {
+
+Status ConnectLibHdfs();
+
+class HdfsClient;
+class HdfsReadableFile;
+class HdfsWriteableFile;
+
+struct HdfsPathInfo {
+  ObjectType::type kind;
+
+  std::string name;
+  std::string owner;
+  std::string group;
+
+  // Access times in UNIX timestamps (seconds)
+  int64_t size;
+  int64_t block_size;
+
+  int32_t last_modified_time;
+  int32_t last_access_time;
+
+  int16_t replication;
+  int16_t permissions;
+};
+
+struct HdfsConnectionConfig {
+  std::string host;
+  int port;
+  std::string user;
+
+  // TODO: Kerberos, etc.
+};
+
+class HdfsClient : public FileSystemClient {
+ public:
+  ~HdfsClient();
+
+  // Connect to an HDFS cluster at indicated host, port, and as user
+  //
+  // @param host (in)
+  // @param port (in)
+  // @param user (in): user to identify as
+  // @param fs (out): the created client
+  // @returns Status
+  static Status Connect(
+      const HdfsConnectionConfig* config, std::shared_ptr<HdfsClient>* fs);
+
+  // Create directory and all parents
+  //
+  // @param path (in): absolute HDFS path
+  // @returns Status
+  Status CreateDirectory(const std::string& path);
+
+  // Delete file or directory
+  // @param path: absolute path to data
+  // @param recursive: if path is a directory, delete contents as well
+  // @returns error status on failure
+  Status Delete(const std::string& path, bool recursive = false);
+
+  // Disconnect from cluster
+  //
+  // @returns Status
+  Status Disconnect();
+
+  // @param path (in): absolute HDFS path
+  // @returns bool, true if the path exists, false if not (or on error)
+  bool Exists(const std::string& path);
+
+  // @param path (in): absolute HDFS path
+  // @param info (out)
+  // @returns Status
+  Status GetPathInfo(const std::string& path, HdfsPathInfo* info);
+
+  // @param nbytes (out): total capacity of the filesystem
+  // @returns Status
+  Status GetCapacity(int64_t* nbytes);
+
+  // @param nbytes (out): total bytes used of the filesystem
+  // @returns Status
+  Status GetUsed(int64_t* nbytes);
+
+  Status ListDirectory(const std::string& path, std::vector<HdfsPathInfo>* listing);
+
+  // @param path file path to change
+  // @param owner pass nullptr for no change
+  // @param group pass nullptr for no change
+  Status Chown(const std::string& path, const char* owner, const char* group);
+
+  Status Chmod(const std::string& path, int mode);
+
+  // Move file or directory from source path to destination path within the
+  // current filesystem
+  Status Rename(const std::string& src, const std::string& dst);
+
+  // TODO(wesm): GetWorkingDirectory, SetWorkingDirectory
+
+  // Open an HDFS file in READ mode. Returns error
+  // status if the file is not found.
+  //
+  // @param path complete file path
+  Status OpenReadable(const std::string& path, std::shared_ptr<HdfsReadableFile>* file);
+
+  // FileMode::WRITE options
+  // @param path complete file path
+  // @param buffer_size, 0 for default
+  // @param replication, 0 for default
+  // @param default_block_size, 0 for default
+  Status OpenWriteable(const std::string& path, bool append, int32_t buffer_size,
+      int16_t replication, int64_t default_block_size,
+      std::shared_ptr<HdfsWriteableFile>* file);
+
+  Status OpenWriteable(
+      const std::string& path, bool append, std::shared_ptr<HdfsWriteableFile>* file);
+
+ private:
+  friend class HdfsReadableFile;
+  friend class HdfsWriteableFile;
+
+  class HdfsClientImpl;
+  std::unique_ptr<HdfsClientImpl> impl_;
+
+  HdfsClient();
+  DISALLOW_COPY_AND_ASSIGN(HdfsClient);
+};
+
+class HdfsReadableFile : public RandomAccessFile {
+ public:
+  ~HdfsReadableFile();
+
+  Status Close() override;
+
+  Status GetSize(int64_t* size) override;
+
+  Status ReadAt(
+      int64_t position, int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) override;
+
+  Status Seek(int64_t position) override;
+  Status Tell(int64_t* position) override;
+
+  // NOTE: If you wish to read a particular range of a file in a multithreaded
+  // context, you may prefer to use ReadAt to avoid locking issues
+  Status Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) override;
+
+ private:
+  class HdfsReadableFileImpl;
+  std::unique_ptr<HdfsReadableFileImpl> impl_;
+
+  friend class HdfsClient::HdfsClientImpl;
+
+  HdfsReadableFile();
+  DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
+};
+
+class HdfsWriteableFile : public WriteableFile {
+ public:
+  ~HdfsWriteableFile();
+
+  Status Close() override;
+
+  Status Write(const uint8_t* buffer, int32_t nbytes) override;
+
+  Status Write(const uint8_t* buffer, int32_t nbytes, int32_t* bytes_written);
+
+  Status Tell(int64_t* position) override;
+
+ private:
+  class HdfsWriteableFileImpl;
+  std::unique_ptr<HdfsWriteableFileImpl> impl_;
+
+  friend class HdfsClient::HdfsClientImpl;
+
+  HdfsWriteableFile();
+
+  DISALLOW_COPY_AND_ASSIGN(HdfsWriteableFile);
+};
+
+}  // namespace io
+}  // namespace arrow
+
+#endif  // ARROW_IO_HDFS
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
new file mode 100644
index 0000000000000..4bd8a8ffc2f9d
--- /dev/null
+++ b/cpp/src/arrow/io/interfaces.h
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IO_INTERFACES
+#define ARROW_IO_INTERFACES
+
+#include <cstdint>
+
+namespace arrow {
+
+class Status;
+
+namespace io {
+
+struct FileMode {
+  enum type { READ, WRITE, READWRITE };
+};
+
+struct ObjectType {
+  enum type { FILE, DIRECTORY };
+};
+
+class FileSystemClient {
+ public:
+  virtual ~FileSystemClient() {}
+};
+
+class FileBase {
+  virtual Status Close() = 0;
+
+  virtual Status Tell(int64_t* position) = 0;
+};
+
+class ReadableFile : public FileBase {
+ public:
+  virtual Status ReadAt(
+      int64_t position, int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) = 0;
+
+  virtual Status Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) = 0;
+
+  virtual Status GetSize(int64_t* size) = 0;
+};
+
+class RandomAccessFile : public ReadableFile {
+ public:
+  virtual Status Seek(int64_t position) = 0;
+};
+
+class WriteableFile : public FileBase {
+ public:
+  virtual Status Write(const uint8_t* buffer, int32_t nbytes) = 0;
+};
+
+}  // namespace io
+}  // namespace arrow
+
+#endif  // ARROW_IO_INTERFACES
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
new file mode 100644
index 0000000000000..f75266536e5b3
--- /dev/null
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -0,0 +1,544 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// This shim interface to libhdfs (for runtime shared library loading) has been
+// adapted from the SFrame project, released under the ASF-compatible 3-clause
+// BSD license
+//
+// Using this required having the $JAVA_HOME and $HADOOP_HOME environment
+// variables set, so that libjvm and libhdfs can be located easily
+
+// Copyright (C) 2015 Dato, Inc.
+// All rights reserved.
+//
+// This software may be modified and distributed under the terms
+// of the BSD license. See the LICENSE file for details.
+
+#ifdef HAS_HADOOP
+
+#ifndef _WIN32
+#include <dlfcn.h>
+#else
+#include <winsock2.h>
+#include <windows.h>
+
+// TODO(wesm): address when/if we add windows support
+// #include <util/syserr_reporting.hpp>
+#endif
+
+extern "C" {
+#include <hdfs.h>
+}
+
+#include <iostream>
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include <boost/filesystem.hpp>        // NOLINT
+#include <boost/algorithm/string.hpp>  // NOLINT
+
+#include "arrow/util/status.h"
+
+namespace fs = boost::filesystem;
+
+extern "C" {
+
+#ifndef _WIN32
+static void* libhdfs_handle = NULL;
+static void* libjvm_handle = NULL;
+#else
+static HINSTANCE libhdfs_handle = NULL;
+static HINSTANCE libjvm_handle = NULL;
+#endif
+/*
+ * All the shim pointers
+ */
+
+// NOTE(wesm): cpplint does not like use of short and other imprecise C types
+
+static hdfsFS (*ptr_hdfsConnectAsUser)(
+    const char* host, tPort port, const char* user) = NULL;
+static hdfsFS (*ptr_hdfsConnect)(const char* host, tPort port) = NULL;
+static int (*ptr_hdfsDisconnect)(hdfsFS fs) = NULL;
+
+static hdfsFile (*ptr_hdfsOpenFile)(hdfsFS fs, const char* path, int flags,
+    int bufferSize, short replication, tSize blocksize) = NULL;  // NOLINT
+
+static int (*ptr_hdfsCloseFile)(hdfsFS fs, hdfsFile file) = NULL;
+static int (*ptr_hdfsExists)(hdfsFS fs, const char* path) = NULL;
+static int (*ptr_hdfsSeek)(hdfsFS fs, hdfsFile file, tOffset desiredPos) = NULL;
+static tOffset (*ptr_hdfsTell)(hdfsFS fs, hdfsFile file) = NULL;
+static tSize (*ptr_hdfsRead)(hdfsFS fs, hdfsFile file, void* buffer, tSize length) = NULL;
+static tSize (*ptr_hdfsPread)(
+    hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length) = NULL;
+static tSize (*ptr_hdfsWrite)(
+    hdfsFS fs, hdfsFile file, const void* buffer, tSize length) = NULL;
+static int (*ptr_hdfsFlush)(hdfsFS fs, hdfsFile file) = NULL;
+static int (*ptr_hdfsAvailable)(hdfsFS fs, hdfsFile file) = NULL;
+static int (*ptr_hdfsCopy)(
+    hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) = NULL;
+static int (*ptr_hdfsMove)(
+    hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) = NULL;
+static int (*ptr_hdfsDelete)(hdfsFS fs, const char* path, int recursive) = NULL;
+static int (*ptr_hdfsRename)(hdfsFS fs, const char* oldPath, const char* newPath) = NULL;
+static char* (*ptr_hdfsGetWorkingDirectory)(
+    hdfsFS fs, char* buffer, size_t bufferSize) = NULL;
+static int (*ptr_hdfsSetWorkingDirectory)(hdfsFS fs, const char* path) = NULL;
+static int (*ptr_hdfsCreateDirectory)(hdfsFS fs, const char* path) = NULL;
+static int (*ptr_hdfsSetReplication)(
+    hdfsFS fs, const char* path, int16_t replication) = NULL;
+static hdfsFileInfo* (*ptr_hdfsListDirectory)(
+    hdfsFS fs, const char* path, int* numEntries) = NULL;
+static hdfsFileInfo* (*ptr_hdfsGetPathInfo)(hdfsFS fs, const char* path) = NULL;
+static void (*ptr_hdfsFreeFileInfo)(hdfsFileInfo* hdfsFileInfo, int numEntries) = NULL;
+static char*** (*ptr_hdfsGetHosts)(
+    hdfsFS fs, const char* path, tOffset start, tOffset length) = NULL;
+static void (*ptr_hdfsFreeHosts)(char*** blockHosts) = NULL;
+static tOffset (*ptr_hdfsGetDefaultBlockSize)(hdfsFS fs) = NULL;
+static tOffset (*ptr_hdfsGetCapacity)(hdfsFS fs) = NULL;
+static tOffset (*ptr_hdfsGetUsed)(hdfsFS fs) = NULL;
+static int (*ptr_hdfsChown)(
+    hdfsFS fs, const char* path, const char* owner, const char* group) = NULL;
+static int (*ptr_hdfsChmod)(hdfsFS fs, const char* path, short mode) = NULL;  // NOLINT
+static int (*ptr_hdfsUtime)(hdfsFS fs, const char* path, tTime mtime, tTime atime) = NULL;
+
+// Helper functions for dlopens
+static std::vector<fs::path> get_potential_libjvm_paths();
+static std::vector<fs::path> get_potential_libhdfs_paths();
+static arrow::Status try_dlopen(std::vector<fs::path> potential_paths, const char* name,
+#ifndef _WIN32
+    void*& out_handle);
+#else
+    HINSTANCE& out_handle);
+#endif
+
+#define GET_SYMBOL(SYMBOL_NAME)                                                  \
+  if (!ptr_##SYMBOL_NAME) {                                                      \
+    *reinterpret_cast<void**>(&ptr_##SYMBOL_NAME) = get_symbol("" #SYMBOL_NAME); \
+  }
+
+static void* get_symbol(const char* symbol) {
+  if (libhdfs_handle == NULL) return NULL;
+#ifndef _WIN32
+  return dlsym(libhdfs_handle, symbol);
+#else
+
+  void* ret = reinterpret_cast<void*>(GetProcAddress(libhdfs_handle, symbol));
+  if (ret == NULL) {
+    // logstream(LOG_INFO) << "GetProcAddress error: "
+    //                     << get_last_err_str(GetLastError()) << std::endl;
+  }
+  return ret;
+#endif
+}
+
+hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char* user) {
+  return ptr_hdfsConnectAsUser(host, port, user);
+}
+
+// Returns NULL on failure
+hdfsFS hdfsConnect(const char* host, tPort port) {
+  if (ptr_hdfsConnect) {
+    return ptr_hdfsConnect(host, port);
+  } else {
+    // TODO: error reporting when shim setup fails
+    return NULL;
+  }
+}
+
+int hdfsDisconnect(hdfsFS fs) {
+  return ptr_hdfsDisconnect(fs);
+}
+
+hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, int bufferSize,
+    short replication, tSize blocksize) {  // NOLINT
+  return ptr_hdfsOpenFile(fs, path, flags, bufferSize, replication, blocksize);
+}
+
+int hdfsCloseFile(hdfsFS fs, hdfsFile file) {
+  return ptr_hdfsCloseFile(fs, file);
+}
+
+int hdfsExists(hdfsFS fs, const char* path) {
+  return ptr_hdfsExists(fs, path);
+}
+
+int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
+  return ptr_hdfsSeek(fs, file, desiredPos);
+}
+
+tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
+  return ptr_hdfsTell(fs, file);
+}
+
+tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length) {
+  return ptr_hdfsRead(fs, file, buffer, length);
+}
+
+tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length) {
+  return ptr_hdfsPread(fs, file, position, buffer, length);
+}
+
+tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer, tSize length) {
+  return ptr_hdfsWrite(fs, file, buffer, length);
+}
+
+int hdfsFlush(hdfsFS fs, hdfsFile file) {
+  return ptr_hdfsFlush(fs, file);
+}
+
+int hdfsAvailable(hdfsFS fs, hdfsFile file) {
+  GET_SYMBOL(hdfsAvailable);
+  if (ptr_hdfsAvailable)
+    return ptr_hdfsAvailable(fs, file);
+  else
+    return 0;
+}
+
+int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) {
+  GET_SYMBOL(hdfsCopy);
+  if (ptr_hdfsCopy)
+    return ptr_hdfsCopy(srcFS, src, dstFS, dst);
+  else
+    return 0;
+}
+
+int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) {
+  GET_SYMBOL(hdfsMove);
+  if (ptr_hdfsMove)
+    return ptr_hdfsMove(srcFS, src, dstFS, dst);
+  else
+    return 0;
+}
+
+int hdfsDelete(hdfsFS fs, const char* path, int recursive) {
+  return ptr_hdfsDelete(fs, path, recursive);
+}
+
+int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
+  GET_SYMBOL(hdfsRename);
+  if (ptr_hdfsRename)
+    return ptr_hdfsRename(fs, oldPath, newPath);
+  else
+    return 0;
+}
+
+char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) {
+  GET_SYMBOL(hdfsGetWorkingDirectory);
+  if (ptr_hdfsGetWorkingDirectory) {
+    return ptr_hdfsGetWorkingDirectory(fs, buffer, bufferSize);
+  } else {
+    return NULL;
+  }
+}
+
+int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
+  GET_SYMBOL(hdfsSetWorkingDirectory);
+  if (ptr_hdfsSetWorkingDirectory) {
+    return ptr_hdfsSetWorkingDirectory(fs, path);
+  } else {
+    return 0;
+  }
+}
+
+int hdfsCreateDirectory(hdfsFS fs, const char* path) {
+  return ptr_hdfsCreateDirectory(fs, path);
+}
+
+int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) {
+  GET_SYMBOL(hdfsSetReplication);
+  if (ptr_hdfsSetReplication) {
+    return ptr_hdfsSetReplication(fs, path, replication);
+  } else {
+    return 0;
+  }
+}
+
+hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char* path, int* numEntries) {
+  return ptr_hdfsListDirectory(fs, path, numEntries);
+}
+
+hdfsFileInfo* hdfsGetPathInfo(hdfsFS fs, const char* path) {
+  return ptr_hdfsGetPathInfo(fs, path);
+}
+
+void hdfsFreeFileInfo(hdfsFileInfo* hdfsFileInfo, int numEntries) {
+  ptr_hdfsFreeFileInfo(hdfsFileInfo, numEntries);
+}
+
+char*** hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) {
+  GET_SYMBOL(hdfsGetHosts);
+  if (ptr_hdfsGetHosts) {
+    return ptr_hdfsGetHosts(fs, path, start, length);
+  } else {
+    return NULL;
+  }
+}
+
+void hdfsFreeHosts(char*** blockHosts) {
+  GET_SYMBOL(hdfsFreeHosts);
+  if (ptr_hdfsFreeHosts) { ptr_hdfsFreeHosts(blockHosts); }
+}
+
+tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
+  GET_SYMBOL(hdfsGetDefaultBlockSize);
+  if (ptr_hdfsGetDefaultBlockSize) {
+    return ptr_hdfsGetDefaultBlockSize(fs);
+  } else {
+    return 0;
+  }
+}
+
+tOffset hdfsGetCapacity(hdfsFS fs) {
+  return ptr_hdfsGetCapacity(fs);
+}
+
+tOffset hdfsGetUsed(hdfsFS fs) {
+  return ptr_hdfsGetUsed(fs);
+}
+
+int hdfsChown(hdfsFS fs, const char* path, const char* owner, const char* group) {
+  GET_SYMBOL(hdfsChown);
+  if (ptr_hdfsChown) {
+    return ptr_hdfsChown(fs, path, owner, group);
+  } else {
+    return 0;
+  }
+}
+
+int hdfsChmod(hdfsFS fs, const char* path, short mode) {  // NOLINT
+  GET_SYMBOL(hdfsChmod);
+  if (ptr_hdfsChmod) {
+    return ptr_hdfsChmod(fs, path, mode);
+  } else {
+    return 0;
+  }
+}
+
+int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
+  GET_SYMBOL(hdfsUtime);
+  if (ptr_hdfsUtime) {
+    return ptr_hdfsUtime(fs, path, mtime, atime);
+  } else {
+    return 0;
+  }
+}
+
+static std::vector<fs::path> get_potential_libhdfs_paths() {
+  std::vector<fs::path> libhdfs_potential_paths = {
+      // find one in the local directory
+      fs::path("./libhdfs.so"), fs::path("./hdfs.dll"),
+      // find a global libhdfs.so
+      fs::path("libhdfs.so"), fs::path("hdfs.dll"),
+  };
+
+  const char* hadoop_home = std::getenv("HADOOP_HOME");
+  if (hadoop_home != nullptr) {
+    auto path = fs::path(hadoop_home) / "lib/native/libhdfs.so";
+    libhdfs_potential_paths.push_back(path);
+  }
+  return libhdfs_potential_paths;
+}
+
+static std::vector<fs::path> get_potential_libjvm_paths() {
+  std::vector<fs::path> libjvm_potential_paths;
+
+  std::vector<fs::path> search_prefixes;
+  std::vector<fs::path> search_suffixes;
+  std::string file_name;
+
+// From heuristics
+#ifdef __WIN32
+  search_prefixes = {""};
+  search_suffixes = {"/jre/bin/server", "/bin/server"};
+  file_name = "jvm.dll";
+#elif __APPLE__
+  search_prefixes = {""};
+  search_suffixes = {""};
+  file_name = "libjvm.dylib";
+
+// SFrame uses /usr/libexec/java_home to find JAVA_HOME; for now we are
+// expecting users to set an environment variable
+#else
+  search_prefixes = {
+      "/usr/lib/jvm/default-java",                // ubuntu / debian distros
+      "/usr/lib/jvm/java",                        // rhel6
+      "/usr/lib/jvm",                             // centos6
+      "/usr/lib64/jvm",                           // opensuse 13
+      "/usr/local/lib/jvm/default-java",          // alt ubuntu / debian distros
+      "/usr/local/lib/jvm/java",                  // alt rhel6
+      "/usr/local/lib/jvm",                       // alt centos6
+      "/usr/local/lib64/jvm",                     // alt opensuse 13
+      "/usr/local/lib/jvm/java-7-openjdk-amd64",  // alt ubuntu / debian distros
+      "/usr/lib/jvm/java-7-openjdk-amd64",        // alt ubuntu / debian distros
+      "/usr/local/lib/jvm/java-6-openjdk-amd64",  // alt ubuntu / debian distros
+      "/usr/lib/jvm/java-6-openjdk-amd64",        // alt ubuntu / debian distros
+      "/usr/lib/jvm/java-7-oracle",               // alt ubuntu
+      "/usr/lib/jvm/java-8-oracle",               // alt ubuntu
+      "/usr/lib/jvm/java-6-oracle",               // alt ubuntu
+      "/usr/local/lib/jvm/java-7-oracle",         // alt ubuntu
+      "/usr/local/lib/jvm/java-8-oracle",         // alt ubuntu
+      "/usr/local/lib/jvm/java-6-oracle",         // alt ubuntu
+      "/usr/lib/jvm/default",                     // alt centos
+      "/usr/java/latest",                         // alt centos
+  };
+  search_suffixes = {"/jre/lib/amd64/server"};
+  file_name = "libjvm.so";
+#endif
+  // From direct environment variable
+  char* env_value = NULL;
+  if ((env_value = getenv("JAVA_HOME")) != NULL) {
+    // logstream(LOG_INFO) << "Found environment variable " << env_name << ": " <<
+    // env_value << std::endl;
+    search_prefixes.insert(search_prefixes.begin(), env_value);
+  }
+
+  // Generate cross product between search_prefixes, search_suffixes, and file_name
+  for (auto& prefix : search_prefixes) {
+    for (auto& suffix : search_suffixes) {
+      auto path = (fs::path(prefix) / fs::path(suffix) / fs::path(file_name));
+      libjvm_potential_paths.push_back(path);
+    }
+  }
+
+  return libjvm_potential_paths;
+}
+
+#ifndef _WIN32
+static arrow::Status try_dlopen(
+    std::vector<fs::path> potential_paths, const char* name, void*& out_handle) {
+  std::vector<std::string> error_messages;
+
+  for (auto& i : potential_paths) {
+    i.make_preferred();
+    // logstream(LOG_INFO) << "Trying " << i.string().c_str() << std::endl;
+    out_handle = dlopen(i.native().c_str(), RTLD_NOW | RTLD_LOCAL);
+
+    if (out_handle != NULL) {
+      // logstream(LOG_INFO) << "Success!" << std::endl;
+      break;
+    } else {
+      const char* err_msg = dlerror();
+      if (err_msg != NULL) {
+        error_messages.push_back(std::string(err_msg));
+      } else {
+        error_messages.push_back(std::string(" returned NULL"));
+      }
+    }
+  }
+
+  if (out_handle == NULL) {
+    std::stringstream ss;
+    ss << "Unable to load " << name;
+    return arrow::Status::IOError(ss.str());
+  }
+
+  return arrow::Status::OK();
+}
+
+#else
+static arrow::Status try_dlopen(
+    std::vector<fs::path> potential_paths, const char* name, HINSTANCE& out_handle) {
+  std::vector<std::string> error_messages;
+
+  for (auto& i : potential_paths) {
+    i.make_preferred();
+    // logstream(LOG_INFO) << "Trying " << i.string().c_str() << std::endl;
+
+    out_handle = LoadLibrary(i.string().c_str());
+
+    if (out_handle != NULL) {
+      // logstream(LOG_INFO) << "Success!" << std::endl;
+      break;
+    } else {
+      // error_messages.push_back(get_last_err_str(GetLastError()));
+    }
+  }
+
+  if (out_handle == NULL) {
+    std::stringstream ss;
+    ss << "Unable to load " << name;
+    return arrow::Status::IOError(ss.str());
+  }
+
+  return arrow::Status::OK();
+}
+#endif  // _WIN32
+
+}  // extern "C"
+
+#define GET_SYMBOL_REQUIRED(SYMBOL_NAME)                                           \
+  do {                                                                             \
+    if (!ptr_##SYMBOL_NAME) {                                                      \
+      *reinterpret_cast<void**>(&ptr_##SYMBOL_NAME) = get_symbol("" #SYMBOL_NAME); \
+    }                                                                              \
+    if (!ptr_##SYMBOL_NAME)                                                        \
+      return Status::IOError("Getting symbol " #SYMBOL_NAME "failed");             \
+  } while (0)
+
+namespace arrow {
+namespace io {
+
+Status ConnectLibHdfs() {
+  static std::mutex lock;
+  std::lock_guard<std::mutex> guard(lock);
+
+  static bool shim_attempted = false;
+  if (!shim_attempted) {
+    shim_attempted = true;
+
+    std::vector<fs::path> libjvm_potential_paths = get_potential_libjvm_paths();
+    RETURN_NOT_OK(try_dlopen(libjvm_potential_paths, "libjvm", libjvm_handle));
+
+    std::vector<fs::path> libhdfs_potential_paths = get_potential_libhdfs_paths();
+    RETURN_NOT_OK(try_dlopen(libhdfs_potential_paths, "libhdfs", libhdfs_handle));
+  } else if (libhdfs_handle == nullptr) {
+    return Status::IOError("Prior attempt to load libhdfs failed");
+  }
+
+  GET_SYMBOL_REQUIRED(hdfsConnect);
+  GET_SYMBOL_REQUIRED(hdfsConnectAsUser);
+  GET_SYMBOL_REQUIRED(hdfsCreateDirectory);
+  GET_SYMBOL_REQUIRED(hdfsDelete);
+  GET_SYMBOL_REQUIRED(hdfsDisconnect);
+  GET_SYMBOL_REQUIRED(hdfsExists);
+  GET_SYMBOL_REQUIRED(hdfsFreeFileInfo);
+  GET_SYMBOL_REQUIRED(hdfsGetCapacity);
+  GET_SYMBOL_REQUIRED(hdfsGetUsed);
+  GET_SYMBOL_REQUIRED(hdfsGetPathInfo);
+  GET_SYMBOL_REQUIRED(hdfsListDirectory);
+
+  // File methods
+  GET_SYMBOL_REQUIRED(hdfsCloseFile);
+  GET_SYMBOL_REQUIRED(hdfsFlush);
+  GET_SYMBOL_REQUIRED(hdfsOpenFile);
+  GET_SYMBOL_REQUIRED(hdfsRead);
+  GET_SYMBOL_REQUIRED(hdfsPread);
+  GET_SYMBOL_REQUIRED(hdfsSeek);
+  GET_SYMBOL_REQUIRED(hdfsTell);
+  GET_SYMBOL_REQUIRED(hdfsWrite);
+
+  return Status::OK();
+}
+
+}  // namespace io
+}  // namespace arrow
+
+#endif  // HAS_HADOOP
diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
index db779d8309cf6..edcac88705668 100644
--- a/cpp/src/arrow/parquet/parquet-io-test.cc
+++ b/cpp/src/arrow/parquet/parquet-io-test.cc
@@ -126,8 +126,8 @@ class TestParquetIO : public ::testing::Test {
     size_t chunk_size = values.size() / num_chunks;
     for (int i = 0; i < num_chunks; i++) {
       auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
-      auto column_writer = static_cast<ParquetWriter<TestType>*>(
-          row_group_writer->NextColumn());
+      auto column_writer =
+          static_cast<ParquetWriter<TestType>*>(row_group_writer->NextColumn());
       T* data = values.data() + i * chunk_size;
       column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
       column_writer->Close();
diff --git a/cpp/thirdparty/hadoop/include/hdfs.h b/cpp/thirdparty/hadoop/include/hdfs.h
new file mode 100644
index 0000000000000..a4df6ae3b2be7
--- /dev/null
+++ b/cpp/thirdparty/hadoop/include/hdfs.h
@@ -0,0 +1,1024 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_HDFS_H
+#define LIBHDFS_HDFS_H
+
+#include <errno.h> /* for EINTERNAL, etc. */
+#include <fcntl.h> /* for O_RDONLY, O_WRONLY */
+#include <stdint.h> /* for uint64_t, etc. */
+#include <time.h> /* for time_t */
+
+/*
+ * Support export of DLL symbols during libhdfs build, and import of DLL symbols
+ * during client application build.  A client application may optionally define
+ * symbol LIBHDFS_DLL_IMPORT in its build.  This is not strictly required, but
+ * the compiler can produce more efficient code with it.
+ */
+#ifdef WIN32
+    #ifdef LIBHDFS_DLL_EXPORT
+        #define LIBHDFS_EXTERNAL __declspec(dllexport)
+    #elif LIBHDFS_DLL_IMPORT
+        #define LIBHDFS_EXTERNAL __declspec(dllimport)
+    #else
+        #define LIBHDFS_EXTERNAL
+    #endif
+#else
+    #ifdef LIBHDFS_DLL_EXPORT
+        #define LIBHDFS_EXTERNAL __attribute__((visibility("default")))
+    #elif LIBHDFS_DLL_IMPORT
+        #define LIBHDFS_EXTERNAL __attribute__((visibility("default")))
+    #else
+        #define LIBHDFS_EXTERNAL
+    #endif
+#endif
+
+#ifndef O_RDONLY
+#define O_RDONLY 1
+#endif
+
+#ifndef O_WRONLY
+#define O_WRONLY 2
+#endif
+
+#ifndef EINTERNAL
+#define EINTERNAL 255
+#endif
+
+#define ELASTIC_BYTE_BUFFER_POOL_CLASS \
+  "org/apache/hadoop/io/ElasticByteBufferPool"
+
+/** All APIs set errno to meaningful values */
+
+#ifdef __cplusplus
+extern  "C" {
+#endif
+    /**
+     * Some utility decls used in libhdfs.
+     */
+    struct hdfsBuilder;
+    typedef int32_t   tSize; /// size of data for read/write io ops
+    typedef time_t    tTime; /// time type in seconds
+    typedef int64_t   tOffset;/// offset within the file
+    typedef uint16_t  tPort; /// port
+    typedef enum tObjectKind {
+        kObjectKindFile = 'F',
+        kObjectKindDirectory = 'D',
+    } tObjectKind;
+    struct hdfsStreamBuilder;
+
+
+    /**
+     * The C reflection of org.apache.org.hadoop.FileSystem .
+     */
+    struct hdfs_internal;
+    typedef struct hdfs_internal* hdfsFS;
+
+    struct hdfsFile_internal;
+    typedef struct hdfsFile_internal* hdfsFile;
+
+    struct hadoopRzOptions;
+
+    struct hadoopRzBuffer;
+
+    /**
+     * Determine if a file is open for read.
+     *
+     * @param file     The HDFS file
+     * @return         1 if the file is open for read; 0 otherwise
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsFileIsOpenForRead(hdfsFile file);
+
+    /**
+     * Determine if a file is open for write.
+     *
+     * @param file     The HDFS file
+     * @return         1 if the file is open for write; 0 otherwise
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsFileIsOpenForWrite(hdfsFile file);
+
+    struct hdfsReadStatistics {
+      uint64_t totalBytesRead;
+      uint64_t totalLocalBytesRead;
+      uint64_t totalShortCircuitBytesRead;
+      uint64_t totalZeroCopyBytesRead;
+    };
+
+    /**
+     * Get read statistics about a file.  This is only applicable to files
+     * opened for reading.
+     *
+     * @param file     The HDFS file
+     * @param stats    (out parameter) on a successful return, the read
+     *                 statistics.  Unchanged otherwise.  You must free the
+     *                 returned statistics with hdfsFileFreeReadStatistics.
+     * @return         0 if the statistics were successfully returned,
+     *                 -1 otherwise.  On a failure, please check errno against
+     *                 ENOTSUP.  webhdfs, LocalFilesystem, and so forth may
+     *                 not support read statistics.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsFileGetReadStatistics(hdfsFile file,
+                                  struct hdfsReadStatistics **stats);
+
+    /**
+     * @param stats    HDFS read statistics for a file.
+     *
+     * @return the number of remote bytes read.
+     */
+    LIBHDFS_EXTERNAL
+    int64_t hdfsReadStatisticsGetRemoteBytesRead(
+                            const struct hdfsReadStatistics *stats);
+
+    /**
+     * Clear the read statistics for a file.
+     *
+     * @param file      The file to clear the read statistics of.
+     *
+     * @return          0 on success; the error code otherwise.
+     *                  EINVAL: the file is not open for reading.
+     *                  ENOTSUP: the file does not support clearing the read
+     *                  statistics.
+     *                  Errno will also be set to this code on failure.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsFileClearReadStatistics(hdfsFile file);
+
+    /**
+     * Free some HDFS read statistics.
+     *
+     * @param stats    The HDFS read statistics to free.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats);
+
+    /**
+     * hdfsConnectAsUser - Connect to a hdfs file system as a specific user
+     * Connect to the hdfs.
+     * @param nn   The NameNode.  See hdfsBuilderSetNameNode for details.
+     * @param port The port on which the server is listening.
+     * @param user the user name (this is hadoop domain user). Or NULL is equivelant to hhdfsConnect(host, port)
+     * @return Returns a handle to the filesystem or NULL on error.
+     * @deprecated Use hdfsBuilderConnect instead.
+     */
+     LIBHDFS_EXTERNAL
+     hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user);
+
+    /**
+     * hdfsConnect - Connect to a hdfs file system.
+     * Connect to the hdfs.
+     * @param nn   The NameNode.  See hdfsBuilderSetNameNode for details.
+     * @param port The port on which the server is listening.
+     * @return Returns a handle to the filesystem or NULL on error.
+     * @deprecated Use hdfsBuilderConnect instead.
+     */
+     LIBHDFS_EXTERNAL
+     hdfsFS hdfsConnect(const char* nn, tPort port);
+
+    /**
+     * hdfsConnect - Connect to an hdfs file system.
+     *
+     * Forces a new instance to be created
+     *
+     * @param nn     The NameNode.  See hdfsBuilderSetNameNode for details.
+     * @param port   The port on which the server is listening.
+     * @param user   The user name to use when connecting
+     * @return       Returns a handle to the filesystem or NULL on error.
+     * @deprecated   Use hdfsBuilderConnect instead.
+     */
+     LIBHDFS_EXTERNAL
+     hdfsFS hdfsConnectAsUserNewInstance(const char* nn, tPort port, const char *user );
+
+    /**
+     * hdfsConnect - Connect to an hdfs file system.
+     *
+     * Forces a new instance to be created
+     *
+     * @param nn     The NameNode.  See hdfsBuilderSetNameNode for details.
+     * @param port   The port on which the server is listening.
+     * @return       Returns a handle to the filesystem or NULL on error.
+     * @deprecated   Use hdfsBuilderConnect instead.
+     */
+     LIBHDFS_EXTERNAL
+     hdfsFS hdfsConnectNewInstance(const char* nn, tPort port);
+
+    /**
+     * Connect to HDFS using the parameters defined by the builder.
+     *
+     * The HDFS builder will be freed, whether or not the connection was
+     * successful.
+     *
+     * Every successful call to hdfsBuilderConnect should be matched with a call
+     * to hdfsDisconnect, when the hdfsFS is no longer needed.
+     *
+     * @param bld    The HDFS builder
+     * @return       Returns a handle to the filesystem, or NULL on error.
+     */
+     LIBHDFS_EXTERNAL
+     hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld);
+
+    /**
+     * Create an HDFS builder.
+     *
+     * @return The HDFS builder, or NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    struct hdfsBuilder *hdfsNewBuilder(void);
+
+    /**
+     * Force the builder to always create a new instance of the FileSystem,
+     * rather than possibly finding one in the cache.
+     *
+     * @param bld The HDFS builder
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld);
+
+    /**
+     * Set the HDFS NameNode to connect to.
+     *
+     * @param bld  The HDFS builder
+     * @param nn   The NameNode to use.
+     *
+     *             If the string given is 'default', the default NameNode
+     *             configuration will be used (from the XML configuration files)
+     *
+     *             If NULL is given, a LocalFileSystem will be created.
+     *
+     *             If the string starts with a protocol type such as file:// or
+     *             hdfs://, this protocol type will be used.  If not, the
+     *             hdfs:// protocol type will be used.
+     *
+     *             You may specify a NameNode port in the usual way by
+     *             passing a string of the format hdfs://<hostname>:<port>.
+     *             Alternately, you may set the port with
+     *             hdfsBuilderSetNameNodePort.  However, you must not pass the
+     *             port in two different ways.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn);
+
+    /**
+     * Set the port of the HDFS NameNode to connect to.
+     *
+     * @param bld The HDFS builder
+     * @param port The port.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port);
+
+    /**
+     * Set the username to use when connecting to the HDFS cluster.
+     *
+     * @param bld The HDFS builder
+     * @param userName The user name.  The string will be shallow-copied.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName);
+
+    /**
+     * Set the path to the Kerberos ticket cache to use when connecting to
+     * the HDFS cluster.
+     *
+     * @param bld The HDFS builder
+     * @param kerbTicketCachePath The Kerberos ticket cache path.  The string
+     *                            will be shallow-copied.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld,
+                                   const char *kerbTicketCachePath);
+
+    /**
+     * Free an HDFS builder.
+     *
+     * It is normally not necessary to call this function since
+     * hdfsBuilderConnect frees the builder.
+     *
+     * @param bld The HDFS builder
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsFreeBuilder(struct hdfsBuilder *bld);
+
+    /**
+     * Set a configuration string for an HdfsBuilder.
+     *
+     * @param key      The key to set.
+     * @param val      The value, or NULL to set no value.
+     *                 This will be shallow-copied.  You are responsible for
+     *                 ensuring that it remains valid until the builder is
+     *                 freed.
+     *
+     * @return         0 on success; nonzero error code otherwise.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
+                              const char *val);
+
+    /**
+     * Get a configuration string.
+     *
+     * @param key      The key to find
+     * @param val      (out param) The value.  This will be set to NULL if the
+     *                 key isn't found.  You must free this string with
+     *                 hdfsConfStrFree.
+     *
+     * @return         0 on success; nonzero error code otherwise.
+     *                 Failure to find the key is not an error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsConfGetStr(const char *key, char **val);
+
+    /**
+     * Get a configuration integer.
+     *
+     * @param key      The key to find
+     * @param val      (out param) The value.  This will NOT be changed if the
+     *                 key isn't found.
+     *
+     * @return         0 on success; nonzero error code otherwise.
+     *                 Failure to find the key is not an error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsConfGetInt(const char *key, int32_t *val);
+
+    /**
+     * Free a configuration string found with hdfsConfGetStr.
+     *
+     * @param val      A configuration string obtained from hdfsConfGetStr
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsConfStrFree(char *val);
+
+    /**
+     * hdfsDisconnect - Disconnect from the hdfs file system.
+     * Disconnect from hdfs.
+     * @param fs The configured filesystem handle.
+     * @return Returns 0 on success, -1 on error.
+     *         Even if there is an error, the resources associated with the
+     *         hdfsFS will be freed.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsDisconnect(hdfsFS fs);
+
+    /**
+     * hdfsOpenFile - Open a hdfs file in given mode.
+     * @deprecated    Use the hdfsStreamBuilder functions instead.
+     * This function does not support setting block sizes bigger than 2 GB.
+     *
+     * @param fs The configured filesystem handle.
+     * @param path The full path to the file.
+     * @param flags - an | of bits/fcntl.h file flags - supported flags are O_RDONLY, O_WRONLY (meaning create or overwrite i.e., implies O_TRUNCAT),
+     * O_WRONLY|O_APPEND. Other flags are generally ignored other than (O_RDWR || (O_EXCL & O_CREAT)) which return NULL and set errno equal ENOTSUP.
+     * @param bufferSize Size of buffer for read/write - pass 0 if you want
+     * to use the default configured values.
+     * @param replication Block replication - pass 0 if you want to use
+     * the default configured values.
+     * @param blocksize Size of block - pass 0 if you want to use the
+     * default configured values.  Note that if you want a block size bigger
+     * than 2 GB, you must use the hdfsStreamBuilder API rather than this
+     * deprecated function.
+     * @return Returns the handle to the open file or NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags,
+                          int bufferSize, short replication, tSize blocksize);
+
+    /**
+     * hdfsStreamBuilderAlloc - Allocate an HDFS stream builder.
+     *
+     * @param fs The configured filesystem handle.
+     * @param path The full path to the file.  Will be deep-copied.
+     * @param flags The open flags, as in hdfsOpenFile.
+     * @return Returns the hdfsStreamBuilder, or NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs,
+                                      const char *path, int flags);
+
+    /**
+     * hdfsStreamBuilderFree - Free an HDFS file builder.
+     *
+     * It is normally not necessary to call this function since
+     * hdfsStreamBuilderBuild frees the builder.
+     *
+     * @param bld The hdfsStreamBuilder to free.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld);
+
+    /**
+     * hdfsStreamBuilderSetBufferSize - Set the stream buffer size.
+     *
+     * @param bld The hdfs stream builder.
+     * @param bufferSize The buffer size to set.
+     *
+     * @return 0 on success, or -1 on error.  Errno will be set on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld,
+                                       int32_t bufferSize);
+
+    /**
+     * hdfsStreamBuilderSetReplication - Set the replication for the stream.
+     * This is only relevant for output streams, which will create new blocks.
+     *
+     * @param bld The hdfs stream builder.
+     * @param replication The replication to set.
+     *
+     * @return 0 on success, or -1 on error.  Errno will be set on error.
+     *              If you call this on an input stream builder, you will get
+     *              EINVAL, because this configuration is not relevant to input
+     *              streams.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld,
+                                        int16_t replication);
+
+    /**
+     * hdfsStreamBuilderSetDefaultBlockSize - Set the default block size for
+     * the stream.  This is only relevant for output streams, which will create
+     * new blocks.
+     *
+     * @param bld The hdfs stream builder.
+     * @param defaultBlockSize The default block size to set.
+     *
+     * @return 0 on success, or -1 on error.  Errno will be set on error.
+     *              If you call this on an input stream builder, you will get
+     *              EINVAL, because this configuration is not relevant to input
+     *              streams.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld,
+                                       int64_t defaultBlockSize);
+
+    /**
+     * hdfsStreamBuilderBuild - Build the stream by calling open or create.
+     *
+     * @param bld The hdfs stream builder.  This pointer will be freed, whether
+     *            or not the open succeeds.
+     *
+     * @return the stream pointer on success, or NULL on error.  Errno will be
+     * set on error.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld);
+
+    /**
+     * hdfsTruncateFile - Truncate a hdfs file to given lenght.
+     * @param fs The configured filesystem handle.
+     * @param path The full path to the file.
+     * @param newlength The size the file is to be truncated to
+     * @return 1 if the file has been truncated to the desired newlength
+     *         and is immediately available to be reused for write operations
+     *         such as append.
+     *         0 if a background process of adjusting the length of the last
+     *         block has been started, and clients should wait for it to
+     *         complete before proceeding with further file updates.
+     *         -1 on error.
+     */
+    int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength);
+
+    /**
+     * hdfsUnbufferFile - Reduce the buffering done on a file.
+     *
+     * @param file  The file to unbuffer.
+     * @return      0 on success
+     *              ENOTSUP if the file does not support unbuffering
+     *              Errno will also be set to this value.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsUnbufferFile(hdfsFile file);
+
+    /**
+     * hdfsCloseFile - Close an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns 0 on success, -1 on error.
+     *         On error, errno will be set appropriately.
+     *         If the hdfs file was valid, the memory associated with it will
+     *         be freed at the end of this call, even if there was an I/O
+     *         error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsCloseFile(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsExists - Checks if a given path exsits on the filesystem
+     * @param fs The configured filesystem handle.
+     * @param path The path to look for
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsExists(hdfsFS fs, const char *path);
+
+
+    /**
+     * hdfsSeek - Seek to given offset in file.
+     * This works only for files opened in read-only mode.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param desiredPos Offset into the file to seek into.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos);
+
+
+    /**
+     * hdfsTell - Get the current offset in the file, in bytes.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Current offset, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    tOffset hdfsTell(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsRead - Read data from an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param buffer The buffer to copy read bytes into.
+     * @param length The length of the buffer.
+     * @return      On success, a positive number indicating how many bytes
+     *              were read.
+     *              On end-of-file, 0.
+     *              On error, -1.  Errno will be set to the error code.
+     *              Just like the POSIX read function, hdfsRead will return -1
+     *              and set errno to EINTR if data is temporarily unavailable,
+     *              but we are not yet at the end of the file.
+     */
+    LIBHDFS_EXTERNAL
+    tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
+
+    /**
+     * hdfsPread - Positional read of data from an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param position Position from which to read
+     * @param buffer The buffer to copy read bytes into.
+     * @param length The length of the buffer.
+     * @return      See hdfsRead
+     */
+    LIBHDFS_EXTERNAL
+    tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position,
+                    void* buffer, tSize length);
+
+
+    /**
+     * hdfsWrite - Write data into an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param buffer The data.
+     * @param length The no. of bytes to write.
+     * @return Returns the number of bytes written, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer,
+                    tSize length);
+
+
+    /**
+     * hdfsWrite - Flush the data.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsFlush(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsHFlush - Flush out the data in client's user buffer. After the
+     * return of this call, new readers will see the data.
+     * @param fs configured filesystem handle
+     * @param file file handle
+     * @return 0 on success, -1 on error and sets errno
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsHFlush(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsHSync - Similar to posix fsync, Flush out the data in client's
+     * user buffer. all the way to the disk device (but the disk may have
+     * it in its cache).
+     * @param fs configured filesystem handle
+     * @param file file handle
+     * @return 0 on success, -1 on error and sets errno
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsHSync(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsAvailable - Number of bytes that can be read from this
+     * input stream without blocking.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns available bytes; -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsAvailable(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsCopy - Copy file from one filesystem to another.
+     * @param srcFS The handle to source filesystem.
+     * @param src The path of source file.
+     * @param dstFS The handle to destination filesystem.
+     * @param dst The path of destination file.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+
+
+    /**
+     * hdfsMove - Move file from one filesystem to another.
+     * @param srcFS The handle to source filesystem.
+     * @param src The path of source file.
+     * @param dstFS The handle to destination filesystem.
+     * @param dst The path of destination file.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+
+
+    /**
+     * hdfsDelete - Delete file.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file.
+     * @param recursive if path is a directory and set to
+     * non-zero, the directory is deleted else throws an exception. In
+     * case of a file the recursive argument is irrelevant.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsDelete(hdfsFS fs, const char* path, int recursive);
+
+    /**
+     * hdfsRename - Rename file.
+     * @param fs The configured filesystem handle.
+     * @param oldPath The path of the source file.
+     * @param newPath The path of the destination file.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath);
+
+
+    /**
+     * hdfsGetWorkingDirectory - Get the current working directory for
+     * the given filesystem.
+     * @param fs The configured filesystem handle.
+     * @param buffer The user-buffer to copy path of cwd into.
+     * @param bufferSize The length of user-buffer.
+     * @return Returns buffer, NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize);
+
+
+    /**
+     * hdfsSetWorkingDirectory - Set the working directory. All relative
+     * paths will be resolved relative to it.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the new 'cwd'.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsSetWorkingDirectory(hdfsFS fs, const char* path);
+
+
+    /**
+     * hdfsCreateDirectory - Make the given file and all non-existent
+     * parents into directories.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the directory.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsCreateDirectory(hdfsFS fs, const char* path);
+
+
+    /**
+     * hdfsSetReplication - Set the replication of the specified
+     * file to the supplied value
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file.
+     * @return Returns 0 on success, -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication);
+
+
+    /**
+     * hdfsFileInfo - Information about a file/directory.
+     */
+    typedef struct  {
+        tObjectKind mKind;   /* file or directory */
+        char *mName;         /* the name of the file */
+        tTime mLastMod;      /* the last modification time for the file in seconds */
+        tOffset mSize;       /* the size of the file in bytes */
+        short mReplication;    /* the count of replicas */
+        tOffset mBlockSize;  /* the block size for the file */
+        char *mOwner;        /* the owner of the file */
+        char *mGroup;        /* the group associated with the file */
+        short mPermissions;  /* the permissions associated with the file */
+        tTime mLastAccess;    /* the last access time for the file in seconds */
+    } hdfsFileInfo;
+
+
+    /**
+     * hdfsListDirectory - Get list of files/directories for a given
+     * directory-path. hdfsFreeFileInfo should be called to deallocate memory.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the directory.
+     * @param numEntries Set to the number of files/directories in path.
+     * @return Returns a dynamically-allocated array of hdfsFileInfo
+     * objects; NULL on error or empty directory.
+     * errno is set to non-zero on error or zero on success.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path,
+                                    int *numEntries);
+
+
+    /**
+     * hdfsGetPathInfo - Get information about a path as a (dynamically
+     * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be
+     * called when the pointer is no longer needed.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file.
+     * @return Returns a dynamically-allocated hdfsFileInfo object;
+     * NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path);
+
+
+    /**
+     * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields)
+     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo
+     * objects.
+     * @param numEntries The size of the array.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries);
+
+    /**
+     * hdfsFileIsEncrypted: determine if a file is encrypted based on its
+     * hdfsFileInfo.
+     * @return -1 if there was an error (errno will be set), 0 if the file is
+     *         not encrypted, 1 if the file is encrypted.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsFileIsEncrypted(hdfsFileInfo *hdfsFileInfo);
+
+
+    /**
+     * hdfsGetHosts - Get hostnames where a particular block (determined by
+     * pos & blocksize) of a file is stored. The last element in the array
+     * is NULL. Due to replication, a single block could be present on
+     * multiple hosts.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file.
+     * @param start The start of the block.
+     * @param length The length of the block.
+     * @return Returns a dynamically-allocated 2-d array of blocks-hosts;
+     * NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    char*** hdfsGetHosts(hdfsFS fs, const char* path,
+            tOffset start, tOffset length);
+
+
+    /**
+     * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts
+     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo
+     * objects.
+     * @param numEntries The size of the array.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsFreeHosts(char ***blockHosts);
+
+
+    /**
+     * hdfsGetDefaultBlockSize - Get the default blocksize.
+     *
+     * @param fs            The configured filesystem handle.
+     * @deprecated          Use hdfsGetDefaultBlockSizeAtPath instead.
+     *
+     * @return              Returns the default blocksize, or -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    tOffset hdfsGetDefaultBlockSize(hdfsFS fs);
+
+
+    /**
+     * hdfsGetDefaultBlockSizeAtPath - Get the default blocksize at the
+     * filesystem indicated by a given path.
+     *
+     * @param fs            The configured filesystem handle.
+     * @param path          The given path will be used to locate the actual
+     *                      filesystem.  The full path does not have to exist.
+     *
+     * @return              Returns the default blocksize, or -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path);
+
+
+    /**
+     * hdfsGetCapacity - Return the raw capacity of the filesystem.
+     * @param fs The configured filesystem handle.
+     * @return Returns the raw-capacity; -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    tOffset hdfsGetCapacity(hdfsFS fs);
+
+
+    /**
+     * hdfsGetUsed - Return the total raw size of all files in the filesystem.
+     * @param fs The configured filesystem handle.
+     * @return Returns the total-size; -1 on error.
+     */
+    LIBHDFS_EXTERNAL
+    tOffset hdfsGetUsed(hdfsFS fs);
+
+    /**
+     * Change the user and/or group of a file or directory.
+     *
+     * @param fs            The configured filesystem handle.
+     * @param path          the path to the file or directory
+     * @param owner         User string.  Set to NULL for 'no change'
+     * @param group         Group string.  Set to NULL for 'no change'
+     * @return              0 on success else -1
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsChown(hdfsFS fs, const char* path, const char *owner,
+                  const char *group);
+
+    /**
+     * hdfsChmod
+     * @param fs The configured filesystem handle.
+     * @param path the path to the file or directory
+     * @param mode the bitmask to set it to
+     * @return 0 on success else -1
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsChmod(hdfsFS fs, const char* path, short mode);
+
+    /**
+     * hdfsUtime
+     * @param fs The configured filesystem handle.
+     * @param path the path to the file or directory
+     * @param mtime new modification time or -1 for no change
+     * @param atime new access time or -1 for no change
+     * @return 0 on success else -1
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime);
+
+    /**
+     * Allocate a zero-copy options structure.
+     *
+     * You must free all options structures allocated with this function using
+     * hadoopRzOptionsFree.
+     *
+     * @return            A zero-copy options structure, or NULL if one could
+     *                    not be allocated.  If NULL is returned, errno will
+     *                    contain the error number.
+     */
+    LIBHDFS_EXTERNAL
+    struct hadoopRzOptions *hadoopRzOptionsAlloc(void);
+
+    /**
+     * Determine whether we should skip checksums in read0.
+     *
+     * @param opts        The options structure.
+     * @param skip        Nonzero to skip checksums sometimes; zero to always
+     *                    check them.
+     *
+     * @return            0 on success; -1 plus errno on failure.
+     */
+    LIBHDFS_EXTERNAL
+    int hadoopRzOptionsSetSkipChecksum(
+            struct hadoopRzOptions *opts, int skip);
+
+    /**
+     * Set the ByteBufferPool to use with read0.
+     *
+     * @param opts        The options structure.
+     * @param className   If this is NULL, we will not use any
+     *                    ByteBufferPool.  If this is non-NULL, it will be
+     *                    treated as the name of the pool class to use.
+     *                    For example, you can use
+     *                    ELASTIC_BYTE_BUFFER_POOL_CLASS.
+     *
+     * @return            0 if the ByteBufferPool class was found and
+     *                    instantiated;
+     *                    -1 plus errno otherwise.
+     */
+    LIBHDFS_EXTERNAL
+    int hadoopRzOptionsSetByteBufferPool(
+            struct hadoopRzOptions *opts, const char *className);
+
+    /**
+     * Free a hadoopRzOptionsFree structure.
+     *
+     * @param opts        The options structure to free.
+     *                    Any associated ByteBufferPool will also be freed.
+     */
+    LIBHDFS_EXTERNAL
+    void hadoopRzOptionsFree(struct hadoopRzOptions *opts);
+
+    /**
+     * Perform a byte buffer read.
+     * If possible, this will be a zero-copy (mmap) read.
+     *
+     * @param file       The file to read from.
+     * @param opts       An options structure created by hadoopRzOptionsAlloc.
+     * @param maxLength  The maximum length to read.  We may read fewer bytes
+     *                   than this length.
+     *
+     * @return           On success, we will return a new hadoopRzBuffer.
+     *                   This buffer will continue to be valid and readable
+     *                   until it is released by readZeroBufferFree.  Failure to
+     *                   release a buffer will lead to a memory leak.
+     *                   You can access the data within the hadoopRzBuffer with
+     *                   hadoopRzBufferGet.  If you have reached EOF, the data
+     *                   within the hadoopRzBuffer will be NULL.  You must still
+     *                   free hadoopRzBuffer instances containing NULL.
+     *
+     *                   On failure, we will return NULL plus an errno code.
+     *                   errno = EOPNOTSUPP indicates that we could not do a
+     *                   zero-copy read, and there was no ByteBufferPool
+     *                   supplied.
+     */
+    LIBHDFS_EXTERNAL
+    struct hadoopRzBuffer* hadoopReadZero(hdfsFile file,
+            struct hadoopRzOptions *opts, int32_t maxLength);
+
+    /**
+     * Determine the length of the buffer returned from readZero.
+     *
+     * @param buffer     a buffer returned from readZero.
+     * @return           the length of the buffer.
+     */
+    LIBHDFS_EXTERNAL
+    int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer);
+
+    /**
+     * Get a pointer to the raw buffer returned from readZero.
+     *
+     * To find out how many bytes this buffer contains, call
+     * hadoopRzBufferLength.
+     *
+     * @param buffer     a buffer returned from readZero.
+     * @return           a pointer to the start of the buffer.  This will be
+     *                   NULL when end-of-file has been reached.
+     */
+    LIBHDFS_EXTERNAL
+    const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer);
+
+    /**
+     * Release a buffer obtained through readZero.
+     *
+     * @param file       The hdfs stream that created this buffer.  This must be
+     *                   the same stream you called hadoopReadZero on.
+     * @param buffer     The buffer to release.
+     */
+    LIBHDFS_EXTERNAL
+    void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#undef LIBHDFS_EXTERNAL
+#endif /*LIBHDFS_HDFS_H*/
+
+/**
+ * vim: ts=4: sw=4: et
+ */
diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index 981779ffb4c76..8f47f93b26dd1 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -173,7 +173,10 @@ def merge_pr(pr_num, target_ref):
     for c in commits:
         merge_message_flags += ["-m", c]
 
-    run_cmd(['git', 'commit', '--author="%s"' % primary_author] + merge_message_flags)
+    run_cmd(['git', 'commit',
+             '--no-verify',  # do not run commit hooks
+             '--author="%s"' % primary_author] +
+            merge_message_flags)
 
     continue_maybe("Merge complete (local ref %s). Push to %s?" % (
         target_branch_name, PUSH_REMOTE_NAME))
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index f1becfcf44964..fdbfce99656ca 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -348,8 +348,10 @@ find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 ADD_THIRDPARTY_LIB(arrow
   SHARED_LIB ${ARROW_SHARED_LIB})
+ADD_THIRDPARTY_LIB(arrow_io
+  SHARED_LIB ${ARROW_IO_SHARED_LIB})
 ADD_THIRDPARTY_LIB(arrow_parquet
-    SHARED_LIB ${ARROW_PARQUET_SHARED_LIB})
+  SHARED_LIB ${ARROW_PARQUET_SHARED_LIB})
 
 ############################################################
 # Linker setup
@@ -428,6 +430,7 @@ set(PYARROW_SRCS
 
 set(LINK_LIBS
   arrow
+  arrow_io
   arrow_parquet
 )
 
@@ -449,6 +452,7 @@ set(CYTHON_EXTENSIONS
   array
   config
   error
+  io
   parquet
   scalar
   schema
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index f0b258ed027b0..6bd305615fce2 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -47,13 +47,24 @@ find_library(ARROW_PARQUET_LIB_PATH NAMES arrow_parquet
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
+find_library(ARROW_IO_LIB_PATH NAMES arrow_io
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+
 if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH AND ARROW_PARQUET_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
+  set(ARROW_IO_LIB_NAME libarrow_io)
   set(ARROW_PARQUET_LIB_NAME libarrow_parquet)
+
   set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
   set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+  set(ARROW_IO_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IO_LIB_NAME}.a)
+  set(ARROW_IO_SHARED_LIB ${ARROW_LIBS}/${ARROW_IO_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
   set(ARROW_PARQUET_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PARQUET_LIB_NAME}.a)
   set(ARROW_PARQUET_SHARED_LIB ${ARROW_LIBS}/${ARROW_PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 else ()
@@ -62,7 +73,9 @@ endif ()
 
 if (ARROW_FOUND)
   if (NOT Arrow_FIND_QUIETLY)
-    message(STATUS "Found the Arrow library: ${ARROW_LIB_PATH}, ${ARROW_PARQUET_LIB_PATH}")
+    message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
+    message(STATUS "Found the Arrow IO library: ${ARROW_IO_LIB_PATH}")
+    message(STATUS "Found the Arrow Parquet library: ${ARROW_PARQUET_LIB_PATH}")
   endif ()
 else ()
   if (NOT Arrow_FIND_QUIETLY)
@@ -82,6 +95,8 @@ mark_as_advanced(
   ARROW_LIBS
   ARROW_STATIC_LIB
   ARROW_SHARED_LIB
+  ARROW_IO_STATIC_LIB
+  ARROW_IO_SHARED_LIB
   ARROW_PARQUET_STATIC_LIB
   ARROW_PARQUET_SHARED_LIB
 )
diff --git a/python/conda.recipe/meta.yaml b/python/conda.recipe/meta.yaml
index 85d24b6bc322e..98ae4141e3bd7 100644
--- a/python/conda.recipe/meta.yaml
+++ b/python/conda.recipe/meta.yaml
@@ -26,6 +26,7 @@ requirements:
 
   run:
     - arrow-cpp
+    - parquet-cpp
     - python
     - numpy
     - pandas
diff --git a/python/pyarrow/error.pxd b/python/pyarrow/error.pxd
index 97ba0ef2e9fcb..1fb6fad396a8b 100644
--- a/python/pyarrow/error.pxd
+++ b/python/pyarrow/error.pxd
@@ -18,5 +18,5 @@
 from pyarrow.includes.libarrow cimport CStatus
 from pyarrow.includes.pyarrow cimport *
 
-cdef check_cstatus(const CStatus& status)
-cdef check_status(const Status& status)
+cdef int check_cstatus(const CStatus& status) nogil except -1
+cdef int check_status(const Status& status) nogil except -1
diff --git a/python/pyarrow/error.pyx b/python/pyarrow/error.pyx
index 5a6a038a92e43..244019321a7fd 100644
--- a/python/pyarrow/error.pyx
+++ b/python/pyarrow/error.pyx
@@ -22,16 +22,18 @@ from pyarrow.compat import frombytes
 class ArrowException(Exception):
     pass
 
-cdef check_cstatus(const CStatus& status):
+cdef int check_cstatus(const CStatus& status) nogil except -1:
     if status.ok():
-        return
+        return 0
 
     cdef c_string c_message = status.ToString()
-    raise ArrowException(frombytes(c_message))
+    with gil:
+        raise ArrowException(frombytes(c_message))
 
-cdef check_status(const Status& status):
+cdef int check_status(const Status& status) nogil except -1:
     if status.ok():
-        return
+        return 0
 
     cdef c_string c_message = status.ToString()
-    raise ArrowException(frombytes(c_message))
+    with gil:
+        raise ArrowException(frombytes(c_message))
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 1f6ecee510521..133797bc37b5c 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -33,3 +33,21 @@ cdef extern from "<iostream>":
 cdef extern from "<Python.h>":
     void Py_XDECREF(PyObject* o)
 
+cdef extern from "arrow/api.h" namespace "arrow" nogil:
+    # We can later add more of the common status factory methods as needed
+    cdef CStatus CStatus_OK "Status::OK"()
+
+    cdef cppclass CStatus "arrow::Status":
+        CStatus()
+
+        c_string ToString()
+
+        c_bool ok()
+        c_bool IsOutOfMemory()
+        c_bool IsKeyError()
+        c_bool IsNotImplemented()
+        c_bool IsInvalid()
+
+    cdef cppclass Buffer:
+        uint8_t* data()
+        int64_t size()
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 90414e3d542db..91ce069df8f42 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -19,25 +19,6 @@
 
 from pyarrow.includes.common cimport *
 
-cdef extern from "arrow/api.h" namespace "arrow" nogil:
-    # We can later add more of the common status factory methods as needed
-    cdef CStatus CStatus_OK "Status::OK"()
-
-    cdef cppclass CStatus "arrow::Status":
-        CStatus()
-
-        c_string ToString()
-
-        c_bool ok()
-        c_bool IsOutOfMemory()
-        c_bool IsKeyError()
-        c_bool IsNotImplemented()
-        c_bool IsInvalid()
-
-    cdef cppclass Buffer:
-        uint8_t* data()
-        int64_t size()
-
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     enum Type" arrow::Type::type":
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
new file mode 100644
index 0000000000000..d874ba3091237
--- /dev/null
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from pyarrow.includes.common cimport *
+
+cdef extern from "arrow/io/interfaces.h" nogil:
+    enum ObjectType" arrow::io::ObjectType::type":
+        ObjectType_FILE" arrow::io::ObjectType::FILE"
+        ObjectType_DIRECTORY" arrow::io::ObjectType::DIRECTORY"
+
+cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
+    CStatus ConnectLibHdfs()
+
+    cdef cppclass HdfsConnectionConfig:
+        c_string host
+        int port
+        c_string user
+
+    cdef cppclass HdfsPathInfo:
+        ObjectType kind;
+        c_string name
+        c_string owner
+        c_string group
+        int32_t last_modified_time
+        int32_t last_access_time
+        int64_t size
+        int16_t replication
+        int64_t block_size
+        int16_t permissions
+
+    cdef cppclass CHdfsFile:
+        CStatus Close()
+        CStatus Seek(int64_t position)
+        CStatus Tell(int64_t* position)
+
+    cdef cppclass HdfsReadableFile(CHdfsFile):
+        CStatus GetSize(int64_t* size)
+        CStatus Read(int32_t nbytes, int32_t* bytes_read,
+                     uint8_t* buffer)
+
+        CStatus ReadAt(int64_t position, int32_t nbytes,
+                       int32_t* bytes_read, uint8_t* buffer)
+
+    cdef cppclass HdfsWriteableFile(CHdfsFile):
+        CStatus Write(const uint8_t* buffer, int32_t nbytes)
+
+        CStatus Write(const uint8_t* buffer, int32_t nbytes,
+                      int32_t* bytes_written)
+
+    cdef cppclass CHdfsClient" arrow::io::HdfsClient":
+        @staticmethod
+        CStatus Connect(const HdfsConnectionConfig* config,
+                        shared_ptr[CHdfsClient]* client)
+
+        CStatus CreateDirectory(const c_string& path)
+
+        CStatus Delete(const c_string& path, c_bool recursive)
+
+        CStatus Disconnect()
+
+        c_bool Exists(const c_string& path)
+
+        CStatus GetCapacity(int64_t* nbytes)
+        CStatus GetUsed(int64_t* nbytes)
+
+        CStatus ListDirectory(const c_string& path,
+                              vector[HdfsPathInfo]* listing)
+
+        CStatus Rename(const c_string& src, const c_string& dst)
+
+        CStatus OpenReadable(const c_string& path,
+                             shared_ptr[HdfsReadableFile]* handle)
+
+        CStatus OpenWriteable(const c_string& path, c_bool append,
+                              int32_t buffer_size, int16_t replication,
+                              int64_t default_block_size,
+                              shared_ptr[HdfsWriteableFile]* handle)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
new file mode 100644
index 0000000000000..8b97671e45373
--- /dev/null
+++ b/python/pyarrow/io.pyx
@@ -0,0 +1,504 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cython wrappers for IO interfaces defined in arrow/io
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from libc.stdlib cimport malloc, free
+
+from pyarrow.includes.libarrow cimport *
+cimport pyarrow.includes.pyarrow as pyarrow
+from pyarrow.includes.libarrow_io cimport *
+
+from pyarrow.compat import frombytes, tobytes
+from pyarrow.error cimport check_cstatus
+
+cimport cpython as cp
+
+import re
+import sys
+import threading
+
+_HDFS_PATH_RE = re.compile('hdfs://(.*):(\d+)(.*)')
+
+try:
+    # Python 3
+    from queue import Queue, Empty as QueueEmpty, Full as QueueFull
+except ImportError:
+    from Queue import Queue, Empty as QueueEmpty, Full as QueueFull
+
+
+def have_libhdfs():
+    try:
+        check_cstatus(ConnectLibHdfs())
+        return True
+    except:
+        return False
+
+
+def strip_hdfs_abspath(path):
+    m = _HDFS_PATH_RE.match(path)
+    if m:
+        return m.group(3)
+    else:
+        return path
+
+
+cdef class HdfsClient:
+    cdef:
+        shared_ptr[CHdfsClient] client
+
+    cdef readonly:
+        object host
+        int port
+        object user
+        bint is_open
+
+    def __cinit__(self):
+        self.is_open = False
+
+    def __dealloc__(self):
+        if self.is_open:
+            self.close()
+
+    def close(self):
+        self._ensure_client()
+        with nogil:
+            check_cstatus(self.client.get().Disconnect())
+        self.is_open = False
+
+    cdef _ensure_client(self):
+        if self.client.get() == NULL:
+            raise IOError('HDFS client improperly initialized')
+        elif not self.is_open:
+            raise IOError('HDFS client is closed')
+
+    @classmethod
+    def connect(cls, host, port, user):
+        """
+
+        Parameters
+        ----------
+        host :
+        port :
+        user :
+
+        Notes
+        -----
+        The first time you call this method, it will take longer than usual due
+        to JNI spin-up time.
+
+        Returns
+        -------
+        client : HDFSClient
+        """
+        cdef:
+            HdfsClient out = HdfsClient()
+            HdfsConnectionConfig conf
+
+        conf.host = tobytes(host)
+        conf.port = port
+        conf.user = tobytes(user)
+
+        with nogil:
+            check_cstatus(
+                CHdfsClient.Connect(&conf, &out.client))
+        out.is_open = True
+
+        return out
+
+    def exists(self, path):
+        """
+        Returns True if the path is known to the cluster, False if it does not
+        (or there is an RPC error)
+        """
+        self._ensure_client()
+
+        cdef c_string c_path = tobytes(path)
+        cdef c_bool result
+        with nogil:
+            result = self.client.get().Exists(c_path)
+        return result
+
+    def ls(self, path, bint full_info=True):
+        """
+        Retrieve directory contents and metadata, if requested.
+
+        Parameters
+        ----------
+        path : HDFS path
+        full_info : boolean, default True
+            If False, only return list of paths
+
+        Returns
+        -------
+        result : list of dicts (full_info=True) or strings (full_info=False)
+        """
+        cdef:
+            c_string c_path = tobytes(path)
+            vector[HdfsPathInfo] listing
+            list results = []
+            int i
+
+        self._ensure_client()
+
+        with nogil:
+            check_cstatus(self.client.get()
+                          .ListDirectory(c_path, &listing))
+
+        cdef const HdfsPathInfo* info
+        for i in range(listing.size()):
+            info = &listing[i]
+
+            # Try to trim off the hdfs://HOST:PORT piece
+            name = strip_hdfs_abspath(frombytes(info.name))
+
+            if full_info:
+                kind = ('file' if info.kind == ObjectType_FILE
+                        else 'directory')
+
+                results.append({
+                    'kind': kind,
+                    'name': name,
+                    'owner': frombytes(info.owner),
+                    'group': frombytes(info.group),
+                    'list_modified_time': info.last_modified_time,
+                    'list_access_time': info.last_access_time,
+                    'size': info.size,
+                    'replication': info.replication,
+                    'block_size': info.block_size,
+                    'permissions': info.permissions
+                })
+            else:
+                results.append(name)
+
+        return results
+
+    def mkdir(self, path):
+        """
+        Create indicated directory and any necessary parent directories
+        """
+        self._ensure_client()
+
+        cdef c_string c_path = tobytes(path)
+        with nogil:
+            check_cstatus(self.client.get()
+                          .CreateDirectory(c_path))
+
+    def delete(self, path, bint recursive=False):
+        """
+        Delete the indicated file or directory
+
+        Parameters
+        ----------
+        path : string
+        recursive : boolean, default False
+            If True, also delete child paths for directories
+        """
+        self._ensure_client()
+
+        cdef c_string c_path = tobytes(path)
+        with nogil:
+            check_cstatus(self.client.get()
+                          .Delete(c_path, recursive))
+
+    def open(self, path, mode='rb', buffer_size=None, replication=None,
+             default_block_size=None):
+        """
+        Parameters
+        ----------
+        mode : string, 'rb', 'wb', 'ab'
+        """
+        self._ensure_client()
+
+        cdef HdfsFile out = HdfsFile()
+
+        if mode not in ('rb', 'wb', 'ab'):
+            raise Exception("Mode must be 'rb' (read), "
+                            "'wb' (write, new file), or 'ab' (append)")
+
+        cdef c_string c_path = tobytes(path)
+        cdef c_bool append = False
+
+        # 0 in libhdfs means "use the default"
+        cdef int32_t c_buffer_size = buffer_size or 0
+        cdef int16_t c_replication = replication or 0
+        cdef int64_t c_default_block_size = default_block_size or 0
+
+        if mode in ('wb', 'ab'):
+            if mode == 'ab':
+                append = True
+
+            with nogil:
+                check_cstatus(
+                    self.client.get()
+                    .OpenWriteable(c_path, append, c_buffer_size,
+                                   c_replication, c_default_block_size,
+                                   &out.wr_file))
+
+            out.is_readonly = False
+        else:
+            with nogil:
+                check_cstatus(self.client.get()
+                              .OpenReadable(c_path, &out.rd_file))
+            out.is_readonly = True
+
+        if c_buffer_size == 0:
+            c_buffer_size = 2 ** 16
+
+        out.mode = mode
+        out.buffer_size = c_buffer_size
+        out.parent = self
+        out.is_open = True
+
+        return out
+
+    def upload(self, path, stream, buffer_size=2**16):
+        """
+        Upload file-like object to HDFS path
+        """
+        write_queue = Queue(50)
+
+        f = self.open(path, 'wb')
+
+        done = False
+        exc_info = None
+        def bg_write():
+            try:
+                while not done or write_queue.qsize() > 0:
+                    try:
+                        buf = write_queue.get(timeout=0.01)
+                    except QueueEmpty:
+                        continue
+
+                    f.write(buf)
+
+            except Exception as e:
+                exc_info = sys.exc_info()
+
+        writer_thread = threading.Thread(target=bg_write)
+        writer_thread.start()
+
+        try:
+            while True:
+                buf = stream.read(buffer_size)
+                if not buf:
+                    break
+
+                write_queue.put_nowait(buf)
+        finally:
+            done = True
+
+        writer_thread.join()
+        if exc_info is not None:
+            raise exc_info[0], exc_info[1], exc_info[2]
+
+    def download(self, path, stream, buffer_size=None):
+        f = self.open(path, 'rb', buffer_size=buffer_size)
+        f.download(stream)
+
+
+cdef class HdfsFile:
+    cdef:
+        shared_ptr[HdfsReadableFile] rd_file
+        shared_ptr[HdfsWriteableFile] wr_file
+        bint is_readonly
+        bint is_open
+        object parent
+
+    cdef readonly:
+        int32_t buffer_size
+        object mode
+
+    def __cinit__(self):
+        self.is_open = False
+
+    def __dealloc__(self):
+        if self.is_open:
+            self.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close()
+
+    def close(self):
+        if self.is_open:
+            with nogil:
+                if self.is_readonly:
+                    check_cstatus(self.rd_file.get().Close())
+                else:
+                    check_cstatus(self.wr_file.get().Close())
+        self.is_open = False
+
+    cdef _assert_readable(self):
+        if not self.is_readonly:
+            raise IOError("only valid on readonly files")
+
+    cdef _assert_writeable(self):
+        if self.is_readonly:
+            raise IOError("only valid on writeonly files")
+
+    def size(self):
+        cdef int64_t size
+        self._assert_readable()
+        with nogil:
+            check_cstatus(self.rd_file.get().GetSize(&size))
+        return size
+
+    def tell(self):
+        cdef int64_t position
+        with nogil:
+            if self.is_readonly:
+                check_cstatus(self.rd_file.get().Tell(&position))
+            else:
+                check_cstatus(self.wr_file.get().Tell(&position))
+        return position
+
+    def seek(self, int64_t position):
+        self._assert_readable()
+        with nogil:
+            check_cstatus(self.rd_file.get().Seek(position))
+
+    def read(self, int nbytes):
+        """
+        Read indicated number of bytes from the file, up to EOF
+        """
+        cdef:
+            int32_t bytes_read = 0
+            uint8_t* buf
+
+        self._assert_readable()
+
+        # This isn't ideal -- PyBytes_FromStringAndSize copies the data from
+        # the passed buffer, so it's hard for us to avoid doubling the memory
+        buf = <uint8_t*> malloc(nbytes)
+        if buf == NULL:
+            raise MemoryError("Failed to allocate {0} bytes".format(nbytes))
+
+        cdef int32_t total_bytes = 0
+
+        cdef int rpc_chunksize = min(self.buffer_size, nbytes)
+
+        try:
+            with nogil:
+                while total_bytes < nbytes:
+                    check_cstatus(self.rd_file.get()
+                                  .Read(rpc_chunksize, &bytes_read,
+                                        buf + total_bytes))
+
+                    total_bytes += bytes_read
+
+                    # EOF
+                    if bytes_read == 0:
+                        break
+            result = cp.PyBytes_FromStringAndSize(<const char*>buf,
+                                                  total_bytes)
+        finally:
+            free(buf)
+
+        return result
+
+    def download(self, stream_or_path):
+        """
+        Read file completely to local path (rather than reading completely into
+        memory). First seeks to the beginning of the file.
+        """
+        cdef:
+            int32_t bytes_read = 0
+            uint8_t* buf
+        self._assert_readable()
+
+        write_queue = Queue(50)
+
+        if not hasattr(stream_or_path, 'read'):
+            stream = open(stream_or_path, 'wb')
+            cleanup = lambda: stream.close()
+        else:
+            stream = stream_or_path
+            cleanup = lambda: None
+
+        done = False
+        exc_info = None
+        def bg_write():
+            try:
+                while not done or write_queue.qsize() > 0:
+                    try:
+                        buf = write_queue.get(timeout=0.01)
+                    except QueueEmpty:
+                        continue
+                    stream.write(buf)
+            except Exception as e:
+                exc_info = sys.exc_info()
+            finally:
+                cleanup()
+
+        self.seek(0)
+
+        writer_thread = threading.Thread(target=bg_write)
+
+        # This isn't ideal -- PyBytes_FromStringAndSize copies the data from
+        # the passed buffer, so it's hard for us to avoid doubling the memory
+        buf = <uint8_t*> malloc(self.buffer_size)
+        if buf == NULL:
+            raise MemoryError("Failed to allocate {0} bytes"
+                              .format(self.buffer_size))
+
+        writer_thread.start()
+
+        cdef int64_t total_bytes = 0
+
+        try:
+            while True:
+                with nogil:
+                    check_cstatus(self.rd_file.get()
+                                  .Read(self.buffer_size, &bytes_read, buf))
+
+                total_bytes += bytes_read
+
+                # EOF
+                if bytes_read == 0:
+                    break
+
+                pybuf = cp.PyBytes_FromStringAndSize(<const char*>buf,
+                                                     bytes_read)
+
+                write_queue.put_nowait(pybuf)
+        finally:
+            free(buf)
+            done = True
+
+        writer_thread.join()
+        if exc_info is not None:
+            raise exc_info[0], exc_info[1], exc_info[2]
+
+    def write(self, data):
+        """
+        Write bytes-like (unicode, encoded to UTF-8) to file
+        """
+        self._assert_writeable()
+
+        data = tobytes(data)
+
+        cdef const uint8_t* buf = <const uint8_t*> cp.PyBytes_AS_STRING(data)
+        cdef int32_t bufsize = len(data)
+        with nogil:
+            check_cstatus(self.wr_file.get().Write(buf, bufsize))
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index bf5a22089cdba..86147f8df5a11 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -15,25 +15,24 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import unittest
 import pyarrow
 import pyarrow.formatting as fmt
 
 
-class TestArrayAPI(unittest.TestCase):
+def test_repr_on_pre_init_array():
+    arr = pyarrow.array.Array()
+    assert len(repr(arr)) > 0
 
-    def test_repr_on_pre_init_array(self):
-        arr = pyarrow.array.Array()
-        assert len(repr(arr)) > 0
 
-    def test_getitem_NA(self):
-        arr = pyarrow.from_pylist([1, None, 2])
-        assert arr[1] is pyarrow.NA
+def test_getitem_NA():
+    arr = pyarrow.from_pylist([1, None, 2])
+    assert arr[1] is pyarrow.NA
 
-    def test_list_format(self):
-        arr = pyarrow.from_pylist([[1], None, [2, 3, None]])
-        result = fmt.array_format(arr)
-        expected = """\
+
+def test_list_format():
+    arr = pyarrow.from_pylist([[1], None, [2, 3, None]])
+    result = fmt.array_format(arr)
+    expected = """\
 [
   [1],
   NA,
@@ -41,23 +40,25 @@ def test_list_format(self):
    3,
    NA]
 ]"""
-        assert result == expected
+    assert result == expected
+
 
-    def test_string_format(self):
-        arr = pyarrow.from_pylist(['', None, 'foo'])
-        result = fmt.array_format(arr)
-        expected = """\
+def test_string_format():
+    arr = pyarrow.from_pylist(['', None, 'foo'])
+    result = fmt.array_format(arr)
+    expected = """\
 [
   '',
   NA,
   'foo'
 ]"""
-        assert result == expected
+    assert result == expected
+
 
-    def test_long_array_format(self):
-        arr = pyarrow.from_pylist(range(100))
-        result = fmt.array_format(arr, window=2)
-        expected = """\
+def test_long_array_format():
+    arr = pyarrow.from_pylist(range(100))
+    result = fmt.array_format(arr, window=2)
+    expected = """\
 [
   0,
   1,
@@ -65,4 +66,4 @@ def test_long_array_format(self):
   98,
   99
 ]"""
-        assert result == expected
+    assert result == expected
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
new file mode 100644
index 0000000000000..328e923b941a4
--- /dev/null
+++ b/python/pyarrow/tests/test_io.py
@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from io import BytesIO
+from os.path import join as pjoin
+import os
+import random
+
+import pytest
+
+import pyarrow.io as io
+
+#----------------------------------------------------------------------
+# HDFS tests
+
+
+def hdfs_test_client():
+    host = os.environ.get('ARROW_HDFS_TEST_HOST', 'localhost')
+    user = os.environ['ARROW_HDFS_TEST_USER']
+    try:
+        port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 20500))
+    except ValueError:
+        raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
+                         'an integer')
+
+    return io.HdfsClient.connect(host, port, user)
+
+
+libhdfs = pytest.mark.skipif(not io.have_libhdfs(),
+                             reason='No libhdfs available on system')
+
+
+HDFS_TMP_PATH = '/tmp/pyarrow-test-{0}'.format(random.randint(0, 1000))
+
+@pytest.fixture(scope='session')
+def hdfs(request):
+    fixture = hdfs_test_client()
+    def teardown():
+        fixture.delete(HDFS_TMP_PATH, recursive=True)
+        fixture.close()
+    request.addfinalizer(teardown)
+    return fixture
+
+
+@libhdfs
+def test_hdfs_close():
+    client = hdfs_test_client()
+    assert client.is_open
+    client.close()
+    assert not client.is_open
+
+    with pytest.raises(Exception):
+        client.ls('/')
+
+
+@libhdfs
+def test_hdfs_mkdir(hdfs):
+    path = pjoin(HDFS_TMP_PATH, 'test-dir/test-dir')
+    parent_path = pjoin(HDFS_TMP_PATH, 'test-dir')
+
+    hdfs.mkdir(path)
+    assert hdfs.exists(path)
+
+    hdfs.delete(parent_path, recursive=True)
+    assert not hdfs.exists(path)
+
+
+@libhdfs
+def test_hdfs_ls(hdfs):
+    base_path = pjoin(HDFS_TMP_PATH, 'ls-test')
+    hdfs.mkdir(base_path)
+
+    dir_path = pjoin(base_path, 'a-dir')
+    f1_path = pjoin(base_path, 'a-file-1')
+
+    hdfs.mkdir(dir_path)
+
+    f = hdfs.open(f1_path, 'wb')
+    f.write('a' * 10)
+
+    contents = sorted(hdfs.ls(base_path, False))
+    assert contents == [dir_path, f1_path]
+
+
+@libhdfs
+def test_hdfs_download_upload(hdfs):
+    base_path = pjoin(HDFS_TMP_PATH, 'upload-test')
+
+    data = b'foobarbaz'
+    buf = BytesIO(data)
+    buf.seek(0)
+
+    hdfs.upload(base_path, buf)
+
+    out_buf = BytesIO()
+    hdfs.download(base_path, out_buf)
+    out_buf.seek(0)
+    assert out_buf.getvalue() == data
+
+
+@libhdfs
+def test_hdfs_file_context_manager(hdfs):
+    path = pjoin(HDFS_TMP_PATH, 'ctx-manager')
+
+    data = b'foo'
+    with hdfs.open(path, 'wb') as f:
+        f.write(data)
+
+    with hdfs.open(path, 'rb') as f:
+        assert f.size() == 3
+        result = f.read(10)
+        assert result == data
diff --git a/python/setup.py b/python/setup.py
index 7edeb9143319b..59410d75a61e2 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -214,7 +214,14 @@ def get_ext_built(self, name):
             return name + suffix
 
     def get_cmake_cython_names(self):
-        return ['array', 'config', 'error', 'parquet', 'scalar', 'schema', 'table']
+        return ['array',
+                'config',
+                'error',
+                'io',
+                'parquet',
+                'scalar',
+                'schema',
+                'table']
 
     def get_names(self):
         return self._found_names

From 2f52cf4eed1033d1bf1f043d9063e462e60d6605 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 12 Jun 2016 11:48:10 +0200
Subject: [PATCH 0093/1644] ARROW-215: Support other integer types and strings
 in Parquet I/O

Change-Id: I72c6c82bc38c895a04172531bebbc78d4fb08732
---
 cpp/src/arrow/parquet/parquet-io-test.cc     | 461 ++++++++++++-------
 cpp/src/arrow/parquet/parquet-schema-test.cc |   4 +-
 cpp/src/arrow/parquet/reader.cc              | 160 ++++++-
 cpp/src/arrow/parquet/schema.cc              |  47 +-
 cpp/src/arrow/parquet/schema.h               |   9 +-
 cpp/src/arrow/parquet/test-util.h            | 136 +++++-
 cpp/src/arrow/parquet/writer.cc              | 234 ++++++++--
 cpp/src/arrow/parquet/writer.h               |   9 +-
 cpp/src/arrow/test-util.h                    |   2 +
 cpp/src/arrow/types/primitive.cc             |   5 +
 python/pyarrow/includes/parquet.pxd          |  13 +-
 python/pyarrow/parquet.pyx                   |  22 +-
 python/pyarrow/tests/test_parquet.py         |  43 +-
 13 files changed, 901 insertions(+), 244 deletions(-)

diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
index edcac88705668..572cae16e58c0 100644
--- a/cpp/src/arrow/parquet/parquet-io-test.cc
+++ b/cpp/src/arrow/parquet/parquet-io-test.cc
@@ -21,7 +21,9 @@
 #include "arrow/parquet/test-util.h"
 #include "arrow/parquet/reader.h"
 #include "arrow/parquet/writer.h"
+#include "arrow/types/construct.h"
 #include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
@@ -30,12 +32,15 @@
 
 using ParquetBuffer = parquet::Buffer;
 using parquet::BufferReader;
+using parquet::default_writer_properties;
 using parquet::InMemoryOutputStream;
+using parquet::LogicalType;
 using parquet::ParquetFileReader;
 using parquet::ParquetFileWriter;
 using parquet::RandomAccessSource;
 using parquet::Repetition;
 using parquet::SchemaDescriptor;
+using parquet::ParquetVersion;
 using ParquetType = parquet::Type;
 using parquet::schema::GroupNode;
 using parquet::schema::NodePtr;
@@ -51,26 +56,114 @@ const int LARGE_SIZE = 10000;
 template <typename TestType>
 struct test_traits {};
 
+template <>
+struct test_traits<BooleanType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::BOOLEAN;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static uint8_t const value;
+};
+
+const uint8_t test_traits<BooleanType>::value(1);
+
+template <>
+struct test_traits<UInt8Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::UINT_8;
+  static uint8_t const value;
+};
+
+const uint8_t test_traits<UInt8Type>::value(64);
+
+template <>
+struct test_traits<Int8Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::INT_8;
+  static int8_t const value;
+};
+
+const int8_t test_traits<Int8Type>::value(-64);
+
+template <>
+struct test_traits<UInt16Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::UINT_16;
+  static uint16_t const value;
+};
+
+const uint16_t test_traits<UInt16Type>::value(1024);
+
+template <>
+struct test_traits<Int16Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::INT_16;
+  static int16_t const value;
+};
+
+const int16_t test_traits<Int16Type>::value(-1024);
+
+template <>
+struct test_traits<UInt32Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::UINT_32;
+  static uint32_t const value;
+};
+
+const uint32_t test_traits<UInt32Type>::value(1024);
+
 template <>
 struct test_traits<Int32Type> {
   static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static int32_t const value;
+};
+
+const int32_t test_traits<Int32Type>::value(-1024);
+
+template <>
+struct test_traits<UInt64Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
+  static constexpr LogicalType::type logical_enum = LogicalType::UINT_64;
+  static uint64_t const value;
 };
 
+const uint64_t test_traits<UInt64Type>::value(1024);
+
 template <>
 struct test_traits<Int64Type> {
   static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static int64_t const value;
 };
 
+const int64_t test_traits<Int64Type>::value(-1024);
+
 template <>
 struct test_traits<FloatType> {
   static constexpr ParquetType::type parquet_enum = ParquetType::FLOAT;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static float const value;
 };
 
+const float test_traits<FloatType>::value(2.1f);
+
 template <>
 struct test_traits<DoubleType> {
   static constexpr ParquetType::type parquet_enum = ParquetType::DOUBLE;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static double const value;
+};
+
+const double test_traits<DoubleType>::value(4.2);
+
+template <>
+struct test_traits<StringType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::BYTE_ARRAY;
+  static constexpr LogicalType::type logical_enum = LogicalType::UTF8;
+  static std::string const value;
 };
 
+const std::string test_traits<StringType>::value("Test");
+
 template <typename T>
 using ParquetDataType = ::parquet::DataType<test_traits<T>::parquet_enum>;
 
@@ -80,18 +173,18 @@ using ParquetWriter = ::parquet::TypedColumnWriter<ParquetDataType<T>>;
 template <typename TestType>
 class TestParquetIO : public ::testing::Test {
  public:
-  typedef typename TestType::c_type T;
   virtual void SetUp() {}
 
-  std::shared_ptr<GroupNode> MakeSchema(
-      ParquetType::type parquet_type, Repetition::type repetition) {
-    auto pnode = PrimitiveNode::Make("column1", repetition, parquet_type);
+  std::shared_ptr<GroupNode> MakeSchema(Repetition::type repetition) {
+    auto pnode = PrimitiveNode::Make("column1", repetition,
+        test_traits<TestType>::parquet_enum, test_traits<TestType>::logical_enum);
     NodePtr node_ =
         GroupNode::Make("schema", Repetition::REQUIRED, std::vector<NodePtr>({pnode}));
     return std::static_pointer_cast<GroupNode>(node_);
   }
 
-  std::unique_ptr<ParquetFileWriter> MakeWriter(std::shared_ptr<GroupNode>& schema) {
+  std::unique_ptr<ParquetFileWriter> MakeWriter(
+      const std::shared_ptr<GroupNode>& schema) {
     sink_ = std::make_shared<InMemoryOutputStream>();
     return ParquetFileWriter::Open(sink_, schema);
   }
@@ -106,113 +199,74 @@ class TestParquetIO : public ::testing::Test {
       std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Array>* out) {
     arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
     std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
-    ASSERT_NO_THROW(ASSERT_OK(reader.GetFlatColumn(0, &column_reader)));
+    ASSERT_OK_NO_THROW(reader.GetFlatColumn(0, &column_reader));
     ASSERT_NE(nullptr, column_reader.get());
+
     ASSERT_OK(column_reader->NextBatch(SMALL_SIZE, out));
     ASSERT_NE(nullptr, out->get());
   }
 
+  void ReadAndCheckSingleColumnFile(Array* values) {
+    std::shared_ptr<Array> out;
+    ReadSingleColumnFile(ReaderFromSink(), &out);
+    ASSERT_TRUE(values->Equals(out));
+  }
+
   void ReadTableFromFile(
       std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Table>* out) {
     arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
-    ASSERT_NO_THROW(ASSERT_OK(reader.ReadFlatTable(out)));
+    ASSERT_OK_NO_THROW(reader.ReadFlatTable(out));
     ASSERT_NE(nullptr, out->get());
   }
 
-  std::unique_ptr<ParquetFileReader> TestFile(std::vector<T>& values, int num_chunks) {
-    std::shared_ptr<GroupNode> schema =
-        MakeSchema(test_traits<TestType>::parquet_enum, Repetition::REQUIRED);
-    std::unique_ptr<ParquetFileWriter> file_writer = MakeWriter(schema);
-    size_t chunk_size = values.size() / num_chunks;
-    for (int i = 0; i < num_chunks; i++) {
-      auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
-      auto column_writer =
-          static_cast<ParquetWriter<TestType>*>(row_group_writer->NextColumn());
-      T* data = values.data() + i * chunk_size;
-      column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
-      column_writer->Close();
-      row_group_writer->Close();
-    }
-    file_writer->Close();
-    return ReaderFromSink();
+  void ReadAndCheckSingleColumnTable(const std::shared_ptr<Array>& values) {
+    std::shared_ptr<Table> out;
+    ReadTableFromFile(ReaderFromSink(), &out);
+    ASSERT_EQ(1, out->num_columns());
+    ASSERT_EQ(values->length(), out->num_rows());
+
+    std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+    ASSERT_EQ(1, chunked_array->num_chunks());
+    ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+  }
+
+  template <typename ArrayType>
+  void WriteFlatColumn(const std::shared_ptr<GroupNode>& schema,
+      const std::shared_ptr<ArrayType>& values) {
+    FileWriter writer(default_memory_pool(), MakeWriter(schema));
+    ASSERT_OK_NO_THROW(writer.NewRowGroup(values->length()));
+    ASSERT_OK_NO_THROW(writer.WriteFlatColumnChunk(values.get()));
+    ASSERT_OK_NO_THROW(writer.Close());
   }
 
   std::shared_ptr<InMemoryOutputStream> sink_;
 };
 
-typedef ::testing::Types<Int32Type, Int64Type, FloatType, DoubleType> TestTypes;
-
-TYPED_TEST_CASE(TestParquetIO, TestTypes);
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredRead) {
-  std::vector<typename TypeParam::c_type> values(SMALL_SIZE, 128);
-  std::unique_ptr<ParquetFileReader> file_reader = this->TestFile(values, 1);
-
-  std::shared_ptr<Array> out;
-  this->ReadSingleColumnFile(std::move(file_reader), &out);
-
-  ExpectArray<typename TypeParam::c_type>(values.data(), out.get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredTableRead) {
-  std::vector<typename TypeParam::c_type> values(SMALL_SIZE, 128);
-  std::unique_ptr<ParquetFileReader> file_reader = this->TestFile(values, 1);
-
-  std::shared_ptr<Table> out;
-  this->ReadTableFromFile(std::move(file_reader), &out);
-  ASSERT_EQ(1, out->num_columns());
-  ASSERT_EQ(SMALL_SIZE, out->num_rows());
-
-  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-  ASSERT_EQ(1, chunked_array->num_chunks());
-  ExpectArray<typename TypeParam::c_type>(values.data(), chunked_array->chunk(0).get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedRead) {
-  std::vector<typename TypeParam::c_type> values(SMALL_SIZE, 128);
-  std::unique_ptr<ParquetFileReader> file_reader = this->TestFile(values, 4);
-
-  std::shared_ptr<Array> out;
-  this->ReadSingleColumnFile(std::move(file_reader), &out);
+// We habe separate tests for UInt32Type as this is currently the only type
+// where a roundtrip does not yield the identical Array structure.
+// There we write an UInt32 Array but receive an Int64 Array as result for
+// Parquet version 1.0.
 
-  ExpectArray<typename TypeParam::c_type>(values.data(), out.get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedTableRead) {
-  std::vector<typename TypeParam::c_type> values(SMALL_SIZE, 128);
-  std::unique_ptr<ParquetFileReader> file_reader = this->TestFile(values, 4);
-
-  std::shared_ptr<Table> out;
-  this->ReadTableFromFile(std::move(file_reader), &out);
-  ASSERT_EQ(1, out->num_columns());
-  ASSERT_EQ(SMALL_SIZE, out->num_rows());
+typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
+    Int32Type, UInt64Type, Int64Type, FloatType, DoubleType, StringType> TestTypes;
 
-  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-  ASSERT_EQ(1, chunked_array->num_chunks());
-  ExpectArray<typename TypeParam::c_type>(values.data(), chunked_array->chunk(0).get());
-}
+TYPED_TEST_CASE(TestParquetIO, TestTypes);
 
 TYPED_TEST(TestParquetIO, SingleColumnRequiredWrite) {
-  std::shared_ptr<PrimitiveArray> values = NonNullArray<TypeParam>(SMALL_SIZE, 128);
+  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
 
-  std::shared_ptr<GroupNode> schema =
-      this->MakeSchema(test_traits<TypeParam>::parquet_enum, Repetition::REQUIRED);
-  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
-  ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values->length())));
-  ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values.get())));
-  ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
+  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
+  this->WriteFlatColumn(schema, values);
 
-  std::shared_ptr<Array> out;
-  this->ReadSingleColumnFile(this->ReaderFromSink(), &out);
-  ASSERT_TRUE(values->Equals(out));
+  this->ReadAndCheckSingleColumnFile(values.get());
 }
 
 TYPED_TEST(TestParquetIO, SingleColumnTableRequiredWrite) {
-  std::shared_ptr<PrimitiveArray> values = NonNullArray<TypeParam>(SMALL_SIZE, 128);
+  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
   std::shared_ptr<Table> table = MakeSimpleTable(values, false);
   this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_NO_THROW(ASSERT_OK(
-      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, values->length())));
+  ASSERT_OK_NO_THROW(WriteFlatTable(table.get(), default_memory_pool(), this->sink_,
+      values->length(), default_writer_properties()));
 
   std::shared_ptr<Table> out;
   this->ReadTableFromFile(this->ReaderFromSink(), &out);
@@ -226,113 +280,208 @@ TYPED_TEST(TestParquetIO, SingleColumnTableRequiredWrite) {
 
 TYPED_TEST(TestParquetIO, SingleColumnOptionalReadWrite) {
   // This also tests max_definition_level = 1
-  std::shared_ptr<PrimitiveArray> values = NullableArray<TypeParam>(SMALL_SIZE, 128, 10);
+  auto values = NullableArray<TypeParam>(SMALL_SIZE, 10);
 
-  std::shared_ptr<GroupNode> schema =
-      this->MakeSchema(test_traits<TypeParam>::parquet_enum, Repetition::OPTIONAL);
-  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
-  ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values->length())));
-  ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values.get())));
-  ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
+  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::OPTIONAL);
+  this->WriteFlatColumn(schema, values);
 
-  std::shared_ptr<Array> out;
-  this->ReadSingleColumnFile(this->ReaderFromSink(), &out);
-  ASSERT_TRUE(values->Equals(out));
+  this->ReadAndCheckSingleColumnFile(values.get());
 }
 
 TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {
   // This also tests max_definition_level = 1
-  std::shared_ptr<PrimitiveArray> values = NullableArray<TypeParam>(SMALL_SIZE, 128, 10);
+  std::shared_ptr<Array> values = NullableArray<TypeParam>(SMALL_SIZE, 10);
   std::shared_ptr<Table> table = MakeSimpleTable(values, true);
   this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_NO_THROW(ASSERT_OK(
-      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, values->length())));
-
-  std::shared_ptr<Table> out;
-  this->ReadTableFromFile(this->ReaderFromSink(), &out);
-  ASSERT_EQ(1, out->num_columns());
-  ASSERT_EQ(SMALL_SIZE, out->num_rows());
+  ASSERT_OK_NO_THROW(WriteFlatTable(table.get(), default_memory_pool(), this->sink_,
+      values->length(), default_writer_properties()));
 
-  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-  ASSERT_EQ(1, chunked_array->num_chunks());
-  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+  this->ReadAndCheckSingleColumnTable(values);
 }
 
-TYPED_TEST(TestParquetIO, SingleColumnIntRequiredChunkedWrite) {
-  std::shared_ptr<PrimitiveArray> values = NonNullArray<TypeParam>(SMALL_SIZE, 128);
-  std::shared_ptr<PrimitiveArray> values_chunk =
-      NonNullArray<TypeParam>(SMALL_SIZE / 4, 128);
+TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
+  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
+  int64_t chunk_size = values->length() / 4;
 
-  std::shared_ptr<GroupNode> schema =
-      this->MakeSchema(test_traits<TypeParam>::parquet_enum, Repetition::REQUIRED);
+  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
   FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
   for (int i = 0; i < 4; i++) {
-    ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values_chunk->length())));
-    ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values_chunk.get())));
+    ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
+    ASSERT_OK_NO_THROW(
+        writer.WriteFlatColumnChunk(values.get(), i * chunk_size, chunk_size));
   }
-  ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
+  ASSERT_OK_NO_THROW(writer.Close());
 
-  std::shared_ptr<Array> out;
-  this->ReadSingleColumnFile(this->ReaderFromSink(), &out);
-  ASSERT_TRUE(values->Equals(out));
+  this->ReadAndCheckSingleColumnFile(values.get());
 }
 
 TYPED_TEST(TestParquetIO, SingleColumnTableRequiredChunkedWrite) {
-  std::shared_ptr<PrimitiveArray> values = NonNullArray<TypeParam>(LARGE_SIZE, 128);
+  auto values = NonNullArray<TypeParam>(LARGE_SIZE);
   std::shared_ptr<Table> table = MakeSimpleTable(values, false);
   this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_NO_THROW(
-      ASSERT_OK(WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512)));
-
-  std::shared_ptr<Table> out;
-  this->ReadTableFromFile(this->ReaderFromSink(), &out);
-  ASSERT_EQ(1, out->num_columns());
-  ASSERT_EQ(LARGE_SIZE, out->num_rows());
+  ASSERT_OK_NO_THROW(WriteFlatTable(
+      table.get(), default_memory_pool(), this->sink_, 512, default_writer_properties()));
 
-  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-  ASSERT_EQ(1, chunked_array->num_chunks());
-  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+  this->ReadAndCheckSingleColumnTable(values);
 }
 
 TYPED_TEST(TestParquetIO, SingleColumnOptionalChunkedWrite) {
-  std::shared_ptr<PrimitiveArray> values = NullableArray<TypeParam>(SMALL_SIZE, 128, 10);
-  std::shared_ptr<PrimitiveArray> values_chunk_nulls =
-      NullableArray<TypeParam>(SMALL_SIZE / 4, 128, 10);
-  std::shared_ptr<PrimitiveArray> values_chunk =
-      NullableArray<TypeParam>(SMALL_SIZE / 4, 128, 0);
-
-  std::shared_ptr<GroupNode> schema =
-      this->MakeSchema(test_traits<TypeParam>::parquet_enum, Repetition::OPTIONAL);
+  int64_t chunk_size = SMALL_SIZE / 4;
+  auto values = NullableArray<TypeParam>(SMALL_SIZE, 10);
+
+  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::OPTIONAL);
   FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
-  ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values_chunk_nulls->length())));
-  ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values_chunk_nulls.get())));
-  for (int i = 0; i < 3; i++) {
-    ASSERT_NO_THROW(ASSERT_OK(writer.NewRowGroup(values_chunk->length())));
-    ASSERT_NO_THROW(ASSERT_OK(writer.WriteFlatColumnChunk(values_chunk.get())));
+  for (int i = 0; i < 4; i++) {
+    ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
+    ASSERT_OK_NO_THROW(
+        writer.WriteFlatColumnChunk(values.get(), i * chunk_size, chunk_size));
   }
-  ASSERT_NO_THROW(ASSERT_OK(writer.Close()));
+  ASSERT_OK_NO_THROW(writer.Close());
 
-  std::shared_ptr<Array> out;
-  this->ReadSingleColumnFile(this->ReaderFromSink(), &out);
-  ASSERT_TRUE(values->Equals(out));
+  this->ReadAndCheckSingleColumnFile(values.get());
 }
 
 TYPED_TEST(TestParquetIO, SingleColumnTableOptionalChunkedWrite) {
   // This also tests max_definition_level = 1
-  std::shared_ptr<PrimitiveArray> values = NullableArray<TypeParam>(LARGE_SIZE, 128, 100);
+  auto values = NullableArray<TypeParam>(LARGE_SIZE, 100);
   std::shared_ptr<Table> table = MakeSimpleTable(values, true);
   this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_NO_THROW(
-      ASSERT_OK(WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512)));
+  ASSERT_OK_NO_THROW(WriteFlatTable(
+      table.get(), default_memory_pool(), this->sink_, 512, default_writer_properties()));
 
-  std::shared_ptr<Table> out;
-  this->ReadTableFromFile(this->ReaderFromSink(), &out);
-  ASSERT_EQ(1, out->num_columns());
-  ASSERT_EQ(LARGE_SIZE, out->num_rows());
+  this->ReadAndCheckSingleColumnTable(values);
+}
 
-  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-  ASSERT_EQ(1, chunked_array->num_chunks());
-  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+using TestUInt32ParquetIO = TestParquetIO<UInt32Type>;
+
+TEST_F(TestUInt32ParquetIO, Parquet_2_0_Compability) {
+  // This also tests max_definition_level = 1
+  std::shared_ptr<PrimitiveArray> values = NullableArray<UInt32Type>(LARGE_SIZE, 100);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
+
+  // Parquet 2.0 roundtrip should yield an uint32_t column again
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  std::shared_ptr<::parquet::WriterProperties> properties =
+      ::parquet::WriterProperties::Builder()
+          .version(ParquetVersion::PARQUET_2_0)
+          ->build();
+  ASSERT_OK_NO_THROW(
+      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512, properties));
+  this->ReadAndCheckSingleColumnTable(values);
+}
+
+TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compability) {
+  // This also tests max_definition_level = 1
+  std::shared_ptr<PrimitiveArray> values = NullableArray<UInt32Type>(LARGE_SIZE, 100);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
+
+  // Parquet 1.0 returns an int64_t column as there is no way to tell a Parquet 1.0
+  // reader that a column is unsigned.
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  std::shared_ptr<::parquet::WriterProperties> properties =
+      ::parquet::WriterProperties::Builder()
+          .version(ParquetVersion::PARQUET_1_0)
+          ->build();
+  ASSERT_OK_NO_THROW(
+      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512, properties));
+
+  std::shared_ptr<Array> expected_values;
+  std::shared_ptr<PoolBuffer> int64_data =
+      std::make_shared<PoolBuffer>(default_memory_pool());
+  {
+    ASSERT_OK(int64_data->Resize(sizeof(int64_t) * values->length()));
+    int64_t* int64_data_ptr = reinterpret_cast<int64_t*>(int64_data->mutable_data());
+    const uint32_t* uint32_data_ptr =
+        reinterpret_cast<const uint32_t*>(values->data()->data());
+    // std::copy might be faster but this is explicit on the casts)
+    for (int64_t i = 0; i < values->length(); i++) {
+      int64_data_ptr[i] = static_cast<int64_t>(uint32_data_ptr[i]);
+    }
+  }
+  ASSERT_OK(MakePrimitiveArray(std::make_shared<Int64Type>(), values->length(),
+      int64_data, values->null_count(), values->null_bitmap(), &expected_values));
+  this->ReadAndCheckSingleColumnTable(expected_values);
+}
+
+template <typename T>
+using ParquetCDataType = typename ParquetDataType<T>::c_type;
+
+template <typename TestType>
+class TestPrimitiveParquetIO : public TestParquetIO<TestType> {
+ public:
+  typedef typename TestType::c_type T;
+
+  void TestFile(std::vector<T>& values, int num_chunks,
+      std::unique_ptr<ParquetFileReader>* file_reader) {
+    std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
+    std::unique_ptr<ParquetFileWriter> file_writer = this->MakeWriter(schema);
+    size_t chunk_size = values.size() / num_chunks;
+    // Convert to Parquet's expected physical type
+    std::vector<uint8_t> values_buffer(
+        sizeof(ParquetCDataType<TestType>) * values.size());
+    auto values_parquet =
+        reinterpret_cast<ParquetCDataType<TestType>*>(values_buffer.data());
+    std::copy(values.cbegin(), values.cend(), values_parquet);
+    for (int i = 0; i < num_chunks; i++) {
+      auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
+      auto column_writer =
+          static_cast<ParquetWriter<TestType>*>(row_group_writer->NextColumn());
+      ParquetCDataType<TestType>* data = values_parquet + i * chunk_size;
+      column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
+      column_writer->Close();
+      row_group_writer->Close();
+    }
+    file_writer->Close();
+    *file_reader = this->ReaderFromSink();
+  }
+
+  void TestSingleColumnRequiredTableRead(int num_chunks) {
+    std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
+    std::unique_ptr<ParquetFileReader> file_reader;
+    ASSERT_NO_THROW(TestFile(values, num_chunks, &file_reader));
+
+    std::shared_ptr<Table> out;
+    this->ReadTableFromFile(std::move(file_reader), &out);
+    ASSERT_EQ(1, out->num_columns());
+    ASSERT_EQ(SMALL_SIZE, out->num_rows());
+
+    std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+    ASSERT_EQ(1, chunked_array->num_chunks());
+    ExpectArray<TestType>(values.data(), chunked_array->chunk(0).get());
+  }
+
+  void TestSingleColumnRequiredRead(int num_chunks) {
+    std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
+    std::unique_ptr<ParquetFileReader> file_reader;
+    ASSERT_NO_THROW(TestFile(values, num_chunks, &file_reader));
+
+    std::shared_ptr<Array> out;
+    this->ReadSingleColumnFile(std::move(file_reader), &out);
+
+    ExpectArray<TestType>(values.data(), out.get());
+  }
+};
+
+typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
+    UInt32Type, Int32Type, UInt64Type, Int64Type, FloatType,
+    DoubleType> PrimitiveTestTypes;
+
+TYPED_TEST_CASE(TestPrimitiveParquetIO, PrimitiveTestTypes);
+
+TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredRead) {
+  this->TestSingleColumnRequiredRead(1);
+}
+
+TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredTableRead) {
+  this->TestSingleColumnRequiredTableRead(1);
+}
+
+TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedRead) {
+  this->TestSingleColumnRequiredRead(4);
+}
+
+TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
+  this->TestSingleColumnRequiredTableRead(4);
 }
 
 }  // namespace parquet
diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
index 8de739491b56f..819cdd3ec4394 100644
--- a/cpp/src/arrow/parquet/parquet-schema-test.cc
+++ b/cpp/src/arrow/parquet/parquet-schema-test.cc
@@ -183,7 +183,9 @@ class TestConvertArrowSchema : public ::testing::Test {
 
   Status ConvertSchema(const std::vector<std::shared_ptr<Field>>& fields) {
     arrow_schema_ = std::make_shared<Schema>(fields);
-    return ToParquetSchema(arrow_schema_.get(), &result_schema_);
+    std::shared_ptr<::parquet::WriterProperties> properties =
+        ::parquet::default_writer_properties();
+    return ToParquetSchema(arrow_schema_.get(), *properties.get(), &result_schema_);
   }
 
  protected:
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
index 3b4882d4439d5..7b05665b230f0 100644
--- a/cpp/src/arrow/parquet/reader.cc
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/parquet/reader.h"
 
+#include <algorithm>
 #include <queue>
 #include <string>
 #include <vector>
@@ -27,6 +28,7 @@
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
 #include "arrow/util/status.h"
 
 using parquet::ColumnReader;
@@ -36,6 +38,19 @@ using parquet::TypedColumnReader;
 namespace arrow {
 namespace parquet {
 
+template <typename ArrowType>
+struct ArrowTypeTraits {
+  typedef NumericBuilder<ArrowType> builder_type;
+};
+
+template <>
+struct ArrowTypeTraits<BooleanType> {
+  typedef BooleanBuilder builder_type;
+};
+
+template <typename ArrowType>
+using BuilderType = typename ArrowTypeTraits<ArrowType>::builder_type;
+
 class FileReader::Impl {
  public:
   Impl(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader);
@@ -61,9 +76,45 @@ class FlatColumnReader::Impl {
   template <typename ArrowType, typename ParquetType>
   Status TypedReadBatch(int batch_size, std::shared_ptr<Array>* out);
 
+  template <typename ArrowType, typename ParquetType>
+  Status ReadNullableFlatBatch(const int16_t* def_levels,
+      typename ParquetType::c_type* values, int64_t values_read, int64_t levels_read,
+      BuilderType<ArrowType>* builder);
+  template <typename ArrowType, typename ParquetType>
+  Status ReadNonNullableBatch(typename ParquetType::c_type* values, int64_t values_read,
+      BuilderType<ArrowType>* builder);
+
  private:
   void NextRowGroup();
 
+  template <typename InType, typename OutType>
+  struct can_copy_ptr {
+    static constexpr bool value =
+        std::is_same<InType, OutType>::value ||
+        (std::is_integral<InType>{} && std::is_integral<OutType>{} &&
+            (sizeof(InType) == sizeof(OutType)));
+  };
+
+  template <typename InType, typename OutType,
+      typename std::enable_if<can_copy_ptr<InType, OutType>::value>::type* = nullptr>
+  Status ConvertPhysicalType(
+      const InType* in_ptr, int64_t length, const OutType** out_ptr) {
+    *out_ptr = reinterpret_cast<const OutType*>(in_ptr);
+    return Status::OK();
+  }
+
+  template <typename InType, typename OutType,
+      typename std::enable_if<not can_copy_ptr<InType, OutType>::value>::type* = nullptr>
+  Status ConvertPhysicalType(
+      const InType* in_ptr, int64_t length, const OutType** out_ptr) {
+    RETURN_NOT_OK(values_builder_buffer_.Resize(length * sizeof(OutType)));
+    OutType* mutable_out_ptr =
+        reinterpret_cast<OutType*>(values_builder_buffer_.mutable_data());
+    std::copy(in_ptr, in_ptr + length, mutable_out_ptr);
+    *out_ptr = mutable_out_ptr;
+    return Status::OK();
+  }
+
   MemoryPool* pool_;
   const ::parquet::ColumnDescriptor* descr_;
   ::parquet::ParquetFileReader* reader_;
@@ -155,13 +206,53 @@ FlatColumnReader::Impl::Impl(MemoryPool* pool, const ::parquet::ColumnDescriptor
   NextRowGroup();
 }
 
+template <typename ArrowType, typename ParquetType>
+Status FlatColumnReader::Impl::ReadNonNullableBatch(typename ParquetType::c_type* values,
+    int64_t values_read, BuilderType<ArrowType>* builder) {
+  using ArrowCType = typename ArrowType::c_type;
+  using ParquetCType = typename ParquetType::c_type;
+
+  DCHECK(builder);
+  const ArrowCType* values_ptr;
+  RETURN_NOT_OK(
+      (ConvertPhysicalType<ParquetCType, ArrowCType>(values, values_read, &values_ptr)));
+  RETURN_NOT_OK(builder->Append(values_ptr, values_read));
+  return Status::OK();
+}
+
+template <typename ArrowType, typename ParquetType>
+Status FlatColumnReader::Impl::ReadNullableFlatBatch(const int16_t* def_levels,
+    typename ParquetType::c_type* values, int64_t values_read, int64_t levels_read,
+    BuilderType<ArrowType>* builder) {
+  using ArrowCType = typename ArrowType::c_type;
+
+  DCHECK(builder);
+  RETURN_NOT_OK(values_builder_buffer_.Resize(levels_read * sizeof(ArrowCType)));
+  RETURN_NOT_OK(valid_bytes_buffer_.Resize(levels_read * sizeof(uint8_t)));
+  auto values_ptr = reinterpret_cast<ArrowCType*>(values_builder_buffer_.mutable_data());
+  uint8_t* valid_bytes = valid_bytes_buffer_.mutable_data();
+  int values_idx = 0;
+  for (int64_t i = 0; i < levels_read; i++) {
+    if (def_levels[i] < descr_->max_definition_level()) {
+      valid_bytes[i] = 0;
+    } else {
+      valid_bytes[i] = 1;
+      values_ptr[i] = values[values_idx++];
+    }
+  }
+  RETURN_NOT_OK(builder->Append(values_ptr, levels_read, valid_bytes));
+  return Status::OK();
+}
+
 template <typename ArrowType, typename ParquetType>
 Status FlatColumnReader::Impl::TypedReadBatch(
     int batch_size, std::shared_ptr<Array>* out) {
+  using ParquetCType = typename ParquetType::c_type;
+
   int values_to_read = batch_size;
-  NumericBuilder<ArrowType> builder(pool_, field_->type);
+  BuilderType<ArrowType> builder(pool_, field_->type);
   while ((values_to_read > 0) && column_reader_) {
-    values_buffer_.Resize(values_to_read * sizeof(typename ParquetType::c_type));
+    values_buffer_.Resize(values_to_read * sizeof(ParquetCType));
     if (descr_->max_definition_level() > 0) {
       def_levels_buffer_.Resize(values_to_read * sizeof(int16_t));
     }
@@ -169,31 +260,62 @@ Status FlatColumnReader::Impl::TypedReadBatch(
     int64_t values_read;
     int64_t levels_read;
     int16_t* def_levels = reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
-    auto values =
-        reinterpret_cast<typename ParquetType::c_type*>(values_buffer_.mutable_data());
+    auto values = reinterpret_cast<ParquetCType*>(values_buffer_.mutable_data());
     PARQUET_CATCH_NOT_OK(levels_read = reader->ReadBatch(
                              values_to_read, def_levels, nullptr, values, &values_read));
     values_to_read -= levels_read;
     if (descr_->max_definition_level() == 0) {
-      RETURN_NOT_OK(builder.Append(values, values_read));
+      RETURN_NOT_OK(
+          (ReadNonNullableBatch<ArrowType, ParquetType>(values, values_read, &builder)));
+    } else {
+      // As per the defintion and checks for flat columns:
+      // descr_->max_definition_level() == 1
+      RETURN_NOT_OK((ReadNullableFlatBatch<ArrowType, ParquetType>(
+          def_levels, values, values_read, levels_read, &builder)));
+    }
+    if (!column_reader_->HasNext()) { NextRowGroup(); }
+  }
+  *out = builder.Finish();
+  return Status::OK();
+}
+
+template <>
+Status FlatColumnReader::Impl::TypedReadBatch<StringType, ::parquet::ByteArrayType>(
+    int batch_size, std::shared_ptr<Array>* out) {
+  int values_to_read = batch_size;
+  StringBuilder builder(pool_, field_->type);
+  while ((values_to_read > 0) && column_reader_) {
+    values_buffer_.Resize(values_to_read * sizeof(::parquet::ByteArray));
+    if (descr_->max_definition_level() > 0) {
+      def_levels_buffer_.Resize(values_to_read * sizeof(int16_t));
+    }
+    auto reader =
+        dynamic_cast<TypedColumnReader<::parquet::ByteArrayType>*>(column_reader_.get());
+    int64_t values_read;
+    int64_t levels_read;
+    int16_t* def_levels = reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
+    auto values = reinterpret_cast<::parquet::ByteArray*>(values_buffer_.mutable_data());
+    PARQUET_CATCH_NOT_OK(levels_read = reader->ReadBatch(
+                             values_to_read, def_levels, nullptr, values, &values_read));
+    values_to_read -= levels_read;
+    if (descr_->max_definition_level() == 0) {
+      for (int64_t i = 0; i < levels_read; i++) {
+        RETURN_NOT_OK(
+            builder.Append(reinterpret_cast<const char*>(values[i].ptr), values[i].len));
+      }
     } else {
       // descr_->max_definition_level() == 1
-      RETURN_NOT_OK(values_builder_buffer_.Resize(
-          levels_read * sizeof(typename ParquetType::c_type)));
-      RETURN_NOT_OK(valid_bytes_buffer_.Resize(levels_read * sizeof(uint8_t)));
-      auto values_ptr = reinterpret_cast<typename ParquetType::c_type*>(
-          values_builder_buffer_.mutable_data());
-      uint8_t* valid_bytes = valid_bytes_buffer_.mutable_data();
       int values_idx = 0;
       for (int64_t i = 0; i < levels_read; i++) {
         if (def_levels[i] < descr_->max_definition_level()) {
-          valid_bytes[i] = 0;
+          RETURN_NOT_OK(builder.AppendNull());
         } else {
-          valid_bytes[i] = 1;
-          values_ptr[i] = values[values_idx++];
+          RETURN_NOT_OK(
+              builder.Append(reinterpret_cast<const char*>(values[values_idx].ptr),
+                  values[values_idx].len));
+          values_idx++;
         }
       }
-      builder.Append(values_ptr, levels_read, valid_bytes);
     }
     if (!column_reader_->HasNext()) { NextRowGroup(); }
   }
@@ -214,10 +336,18 @@ Status FlatColumnReader::Impl::NextBatch(int batch_size, std::shared_ptr<Array>*
   }
 
   switch (field_->type->type) {
+    TYPED_BATCH_CASE(BOOL, BooleanType, ::parquet::BooleanType)
+    TYPED_BATCH_CASE(UINT8, UInt8Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(INT8, Int8Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(UINT16, UInt16Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(INT16, Int16Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(UINT32, UInt32Type, ::parquet::Int32Type)
     TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(UINT64, UInt64Type, ::parquet::Int64Type)
     TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type)
     TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
     TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
+    TYPED_BATCH_CASE(STRING, StringType, ::parquet::ByteArrayType)
     default:
       return Status::NotImplemented(field_->type->ToString());
   }
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index c7979db349453..a79342afe2f9d 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -42,7 +42,12 @@ namespace parquet {
 
 const auto BOOL = std::make_shared<BooleanType>();
 const auto UINT8 = std::make_shared<UInt8Type>();
+const auto INT8 = std::make_shared<Int8Type>();
+const auto UINT16 = std::make_shared<UInt16Type>();
+const auto INT16 = std::make_shared<Int16Type>();
+const auto UINT32 = std::make_shared<UInt32Type>();
 const auto INT32 = std::make_shared<Int32Type>();
+const auto UINT64 = std::make_shared<UInt64Type>();
 const auto INT64 = std::make_shared<Int64Type>();
 const auto FLOAT = std::make_shared<FloatType>();
 const auto DOUBLE = std::make_shared<DoubleType>();
@@ -92,6 +97,21 @@ static Status FromInt32(const PrimitiveNode* node, TypePtr* out) {
     case LogicalType::NONE:
       *out = INT32;
       break;
+    case LogicalType::UINT_8:
+      *out = UINT8;
+      break;
+    case LogicalType::INT_8:
+      *out = INT8;
+      break;
+    case LogicalType::UINT_16:
+      *out = UINT16;
+      break;
+    case LogicalType::INT_16:
+      *out = INT16;
+      break;
+    case LogicalType::UINT_32:
+      *out = UINT32;
+      break;
     case LogicalType::DECIMAL:
       *out = MakeDecimalType(node);
       break;
@@ -107,6 +127,9 @@ static Status FromInt64(const PrimitiveNode* node, TypePtr* out) {
     case LogicalType::NONE:
       *out = INT64;
       break;
+    case LogicalType::UINT_64:
+      *out = UINT64;
+      break;
     case LogicalType::DECIMAL:
       *out = MakeDecimalType(node);
       break;
@@ -187,20 +210,21 @@ Status FromParquetSchema(
 }
 
 Status StructToNode(const std::shared_ptr<StructType>& type, const std::string& name,
-    bool nullable, NodePtr* out) {
+    bool nullable, const ::parquet::WriterProperties& properties, NodePtr* out) {
   Repetition::type repetition = Repetition::REQUIRED;
   if (nullable) { repetition = Repetition::OPTIONAL; }
 
   std::vector<NodePtr> children(type->num_children());
   for (int i = 0; i < type->num_children(); i++) {
-    RETURN_NOT_OK(FieldToNode(type->child(i), &children[i]));
+    RETURN_NOT_OK(FieldToNode(type->child(i), properties, &children[i]));
   }
 
   *out = GroupNode::Make(name, repetition, children);
   return Status::OK();
 }
 
-Status FieldToNode(const std::shared_ptr<Field>& field, NodePtr* out) {
+Status FieldToNode(const std::shared_ptr<Field>& field,
+    const ::parquet::WriterProperties& properties, NodePtr* out) {
   LogicalType::type logical_type = LogicalType::NONE;
   ParquetType::type type;
   Repetition::type repetition = Repetition::REQUIRED;
@@ -231,8 +255,12 @@ Status FieldToNode(const std::shared_ptr<Field>& field, NodePtr* out) {
       logical_type = LogicalType::INT_16;
       break;
     case Type::UINT32:
-      type = ParquetType::INT32;
-      logical_type = LogicalType::UINT_32;
+      if (properties.version() == ::parquet::ParquetVersion::PARQUET_1_0) {
+        type = ParquetType::INT64;
+      } else {
+        type = ParquetType::INT32;
+        logical_type = LogicalType::UINT_32;
+      }
       break;
     case Type::INT32:
       type = ParquetType::INT32;
@@ -277,7 +305,7 @@ Status FieldToNode(const std::shared_ptr<Field>& field, NodePtr* out) {
       break;
     case Type::STRUCT: {
       auto struct_type = std::static_pointer_cast<StructType>(field->type);
-      return StructToNode(struct_type, field->name, field->nullable, out);
+      return StructToNode(struct_type, field->name, field->nullable, properties, out);
     } break;
     default:
       // TODO: LIST, DENSE_UNION, SPARE_UNION, JSON_SCALAR, DECIMAL, DECIMAL_TEXT, VARCHAR
@@ -287,11 +315,12 @@ Status FieldToNode(const std::shared_ptr<Field>& field, NodePtr* out) {
   return Status::OK();
 }
 
-Status ToParquetSchema(
-    const Schema* arrow_schema, std::shared_ptr<::parquet::SchemaDescriptor>* out) {
+Status ToParquetSchema(const Schema* arrow_schema,
+    const ::parquet::WriterProperties& properties,
+    std::shared_ptr<::parquet::SchemaDescriptor>* out) {
   std::vector<NodePtr> nodes(arrow_schema->num_fields());
   for (int i = 0; i < arrow_schema->num_fields(); i++) {
-    RETURN_NOT_OK(FieldToNode(arrow_schema->field(i), &nodes[i]));
+    RETURN_NOT_OK(FieldToNode(arrow_schema->field(i), properties, &nodes[i]));
   }
 
   NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, nodes);
diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h
index ec5f96062e89f..39bee059522a3 100644
--- a/cpp/src/arrow/parquet/schema.h
+++ b/cpp/src/arrow/parquet/schema.h
@@ -21,6 +21,7 @@
 #include <memory>
 
 #include "parquet/api/schema.h"
+#include "parquet/api/writer.h"
 
 #include "arrow/schema.h"
 #include "arrow/type.h"
@@ -36,10 +37,12 @@ Status NodeToField(const ::parquet::schema::NodePtr& node, std::shared_ptr<Field
 Status FromParquetSchema(
     const ::parquet::SchemaDescriptor* parquet_schema, std::shared_ptr<Schema>* out);
 
-Status FieldToNode(const std::shared_ptr<Field>& field, ::parquet::schema::NodePtr* out);
+Status FieldToNode(const std::shared_ptr<Field>& field,
+    const ::parquet::WriterProperties& properties, ::parquet::schema::NodePtr* out);
 
-Status ToParquetSchema(
-    const Schema* arrow_schema, std::shared_ptr<::parquet::SchemaDescriptor>* out);
+Status ToParquetSchema(const Schema* arrow_schema,
+    const ::parquet::WriterProperties& properties,
+    std::shared_ptr<::parquet::SchemaDescriptor>* out);
 
 }  // namespace parquet
 
diff --git a/cpp/src/arrow/parquet/test-util.h b/cpp/src/arrow/parquet/test-util.h
index cc8723bf6ecab..68a7fb94c2aed 100644
--- a/cpp/src/arrow/parquet/test-util.h
+++ b/cpp/src/arrow/parquet/test-util.h
@@ -18,26 +18,90 @@
 #include <string>
 #include <vector>
 
+#include "arrow/test-util.h"
 #include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
 
 namespace arrow {
 
 namespace parquet {
 
 template <typename ArrowType>
-std::shared_ptr<PrimitiveArray> NonNullArray(
-    size_t size, typename ArrowType::c_type value) {
-  std::vector<typename ArrowType::c_type> values(size, value);
+using is_arrow_float = std::is_floating_point<typename ArrowType::c_type>;
+
+template <typename ArrowType>
+using is_arrow_int = std::is_integral<typename ArrowType::c_type>;
+
+template <typename ArrowType>
+using is_arrow_string = std::is_same<ArrowType, StringType>;
+
+template <class ArrowType>
+typename std::enable_if<is_arrow_float<ArrowType>::value,
+    std::shared_ptr<PrimitiveArray>>::type
+NonNullArray(size_t size) {
+  std::vector<typename ArrowType::c_type> values;
+  ::arrow::test::random_real<typename ArrowType::c_type>(size, 0, 0, 1, &values);
   NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
   builder.Append(values.data(), values.size());
   return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
 }
 
-// This helper function only supports (size/2) nulls yet.
+template <class ArrowType>
+typename std::enable_if<is_arrow_int<ArrowType>::value,
+    std::shared_ptr<PrimitiveArray>>::type
+NonNullArray(size_t size) {
+  std::vector<typename ArrowType::c_type> values;
+  ::arrow::test::randint<typename ArrowType::c_type>(size, 0, 64, &values);
+  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
+  builder.Append(values.data(), values.size());
+  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
+}
+
+template <class ArrowType>
+typename std::enable_if<is_arrow_string<ArrowType>::value,
+    std::shared_ptr<StringArray>>::type
+NonNullArray(size_t size) {
+  StringBuilder builder(default_memory_pool(), std::make_shared<StringType>());
+  for (size_t i = 0; i < size; i++) {
+    builder.Append("test-string");
+  }
+  return std::static_pointer_cast<StringArray>(builder.Finish());
+}
+
+template <>
+std::shared_ptr<PrimitiveArray> NonNullArray<BooleanType>(size_t size) {
+  std::vector<uint8_t> values;
+  ::arrow::test::randint<uint8_t>(size, 0, 1, &values);
+  BooleanBuilder builder(default_memory_pool(), std::make_shared<BooleanType>());
+  builder.Append(values.data(), values.size());
+  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
+}
+
+// This helper function only supports (size/2) nulls.
+template <typename ArrowType>
+typename std::enable_if<is_arrow_float<ArrowType>::value,
+    std::shared_ptr<PrimitiveArray>>::type
+NullableArray(size_t size, size_t num_nulls) {
+  std::vector<typename ArrowType::c_type> values;
+  ::arrow::test::random_real<typename ArrowType::c_type>(size, 0, 0, 1, &values);
+  std::vector<uint8_t> valid_bytes(size, 1);
+
+  for (size_t i = 0; i < num_nulls; i++) {
+    valid_bytes[i * 2] = 0;
+  }
+
+  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
+  builder.Append(values.data(), values.size(), valid_bytes.data());
+  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
+}
+
+// This helper function only supports (size/2) nulls.
 template <typename ArrowType>
-std::shared_ptr<PrimitiveArray> NullableArray(
-    size_t size, typename ArrowType::c_type value, size_t num_nulls) {
-  std::vector<typename ArrowType::c_type> values(size, value);
+typename std::enable_if<is_arrow_int<ArrowType>::value,
+    std::shared_ptr<PrimitiveArray>>::type
+NullableArray(size_t size, size_t num_nulls) {
+  std::vector<typename ArrowType::c_type> values;
+  ::arrow::test::randint<typename ArrowType::c_type>(size, 0, 64, &values);
   std::vector<uint8_t> valid_bytes(size, 1);
 
   for (size_t i = 0; i < num_nulls; i++) {
@@ -49,14 +113,49 @@ std::shared_ptr<PrimitiveArray> NullableArray(
   return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
 }
 
-std::shared_ptr<Column> MakeColumn(const std::string& name,
-    const std::shared_ptr<PrimitiveArray>& array, bool nullable) {
+// This helper function only supports (size/2) nulls yet.
+template <typename ArrowType>
+typename std::enable_if<is_arrow_string<ArrowType>::value,
+    std::shared_ptr<StringArray>>::type
+NullableArray(size_t size, size_t num_nulls) {
+  std::vector<uint8_t> valid_bytes(size, 1);
+
+  for (size_t i = 0; i < num_nulls; i++) {
+    valid_bytes[i * 2] = 0;
+  }
+
+  StringBuilder builder(default_memory_pool(), std::make_shared<StringType>());
+  for (size_t i = 0; i < size; i++) {
+    builder.Append("test-string");
+  }
+  return std::static_pointer_cast<StringArray>(builder.Finish());
+}
+
+// This helper function only supports (size/2) nulls yet.
+template <>
+std::shared_ptr<PrimitiveArray> NullableArray<BooleanType>(
+    size_t size, size_t num_nulls) {
+  std::vector<uint8_t> values;
+  ::arrow::test::randint<uint8_t>(size, 0, 1, &values);
+  std::vector<uint8_t> valid_bytes(size, 1);
+
+  for (size_t i = 0; i < num_nulls; i++) {
+    valid_bytes[i * 2] = 0;
+  }
+
+  BooleanBuilder builder(default_memory_pool(), std::make_shared<BooleanType>());
+  builder.Append(values.data(), values.size(), valid_bytes.data());
+  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
+}
+
+std::shared_ptr<Column> MakeColumn(
+    const std::string& name, const std::shared_ptr<Array>& array, bool nullable) {
   auto field = std::make_shared<Field>(name, array->type(), nullable);
   return std::make_shared<Column>(field, array);
 }
 
 std::shared_ptr<Table> MakeSimpleTable(
-    const std::shared_ptr<PrimitiveArray>& values, bool nullable) {
+    const std::shared_ptr<Array>& values, bool nullable) {
   std::shared_ptr<Column> column = MakeColumn("col", values, nullable);
   std::vector<std::shared_ptr<Column>> columns({column});
   std::vector<std::shared_ptr<Field>> fields({column->field()});
@@ -72,6 +171,23 @@ void ExpectArray(T* expected, Array* result) {
   }
 }
 
+template <typename ArrowType>
+void ExpectArray(typename ArrowType::c_type* expected, Array* result) {
+  PrimitiveArray* p_array = static_cast<PrimitiveArray*>(result);
+  for (int64_t i = 0; i < result->length(); i++) {
+    EXPECT_EQ(expected[i],
+        reinterpret_cast<const typename ArrowType::c_type*>(p_array->data()->data())[i]);
+  }
+}
+
+template <>
+void ExpectArray<BooleanType>(uint8_t* expected, Array* result) {
+  BooleanBuilder builder(default_memory_pool(), std::make_shared<BooleanType>());
+  builder.Append(expected, result->length());
+  std::shared_ptr<Array> expected_array = builder.Finish();
+  EXPECT_TRUE(result->Equals(expected_array));
+}
+
 }  // namespace parquet
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
index 4005e3b2b0c1b..63449bb20b1a1 100644
--- a/cpp/src/arrow/parquet/writer.cc
+++ b/cpp/src/arrow/parquet/writer.cc
@@ -25,11 +25,13 @@
 #include "arrow/table.h"
 #include "arrow/types/construct.h"
 #include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
 #include "arrow/parquet/schema.h"
 #include "arrow/parquet/utils.h"
 #include "arrow/util/status.h"
 
 using parquet::ParquetFileWriter;
+using parquet::ParquetVersion;
 using parquet::schema::GroupNode;
 
 namespace arrow {
@@ -41,10 +43,40 @@ class FileWriter::Impl {
   Impl(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer);
 
   Status NewRowGroup(int64_t chunk_size);
-  template <typename ParquetType>
+  template <typename ParquetType, typename ArrowType>
   Status TypedWriteBatch(::parquet::ColumnWriter* writer, const PrimitiveArray* data,
       int64_t offset, int64_t length);
+
+  // TODO(uwe): Same code as in reader.cc the only difference is the name of the temporary
+  // buffer
+  template <typename InType, typename OutType>
+  struct can_copy_ptr {
+    static constexpr bool value =
+        std::is_same<InType, OutType>::value ||
+        (std::is_integral<InType>{} && std::is_integral<OutType>{} &&
+            (sizeof(InType) == sizeof(OutType)));
+  };
+
+  template <typename InType, typename OutType,
+      typename std::enable_if<can_copy_ptr<InType, OutType>::value>::type* = nullptr>
+  Status ConvertPhysicalType(const InType* in_ptr, int64_t, const OutType** out_ptr) {
+    *out_ptr = reinterpret_cast<const OutType*>(in_ptr);
+    return Status::OK();
+  }
+
+  template <typename InType, typename OutType,
+      typename std::enable_if<not can_copy_ptr<InType, OutType>::value>::type* = nullptr>
+  Status ConvertPhysicalType(
+      const InType* in_ptr, int64_t length, const OutType** out_ptr) {
+    RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(OutType)));
+    OutType* mutable_out_ptr = reinterpret_cast<OutType*>(data_buffer_.mutable_data());
+    std::copy(in_ptr, in_ptr + length, mutable_out_ptr);
+    *out_ptr = mutable_out_ptr;
+    return Status::OK();
+  }
+
   Status WriteFlatColumnChunk(const PrimitiveArray* data, int64_t offset, int64_t length);
+  Status WriteFlatColumnChunk(const StringArray* data, int64_t offset, int64_t length);
   Status Close();
 
   virtual ~Impl() {}
@@ -53,6 +85,8 @@ class FileWriter::Impl {
   friend class FileWriter;
 
   MemoryPool* pool_;
+  // Buffer used for storing the data of an array converted to the physical type
+  // as expected by parquet-cpp.
   PoolBuffer data_buffer_;
   PoolBuffer def_levels_buffer_;
   std::unique_ptr<::parquet::ParquetFileWriter> writer_;
@@ -72,36 +106,95 @@ Status FileWriter::Impl::NewRowGroup(int64_t chunk_size) {
   return Status::OK();
 }
 
-template <typename ParquetType>
+template <typename ParquetType, typename ArrowType>
 Status FileWriter::Impl::TypedWriteBatch(::parquet::ColumnWriter* column_writer,
     const PrimitiveArray* data, int64_t offset, int64_t length) {
-  // TODO: DCHECK((offset + length) <= data->length());
-  auto data_ptr =
-      reinterpret_cast<const typename ParquetType::c_type*>(data->data()->data()) +
-      offset;
+  using ArrowCType = typename ArrowType::c_type;
+  using ParquetCType = typename ParquetType::c_type;
+
+  DCHECK((offset + length) <= data->length());
+  auto data_ptr = reinterpret_cast<const ArrowCType*>(data->data()->data()) + offset;
   auto writer =
       reinterpret_cast<::parquet::TypedColumnWriter<ParquetType>*>(column_writer);
   if (writer->descr()->max_definition_level() == 0) {
     // no nulls, just dump the data
-    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, nullptr, nullptr, data_ptr));
+    const ParquetCType* data_writer_ptr;
+    RETURN_NOT_OK((ConvertPhysicalType<ArrowCType, ParquetCType>(
+        data_ptr, length, &data_writer_ptr)));
+    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, nullptr, nullptr, data_writer_ptr));
   } else if (writer->descr()->max_definition_level() == 1) {
     RETURN_NOT_OK(def_levels_buffer_.Resize(length * sizeof(int16_t)));
     int16_t* def_levels_ptr =
         reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
     if (data->null_count() == 0) {
       std::fill(def_levels_ptr, def_levels_ptr + length, 1);
-      PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, def_levels_ptr, nullptr, data_ptr));
+      const ParquetCType* data_writer_ptr;
+      RETURN_NOT_OK((ConvertPhysicalType<ArrowCType, ParquetCType>(
+          data_ptr, length, &data_writer_ptr)));
+      PARQUET_CATCH_NOT_OK(
+          writer->WriteBatch(length, def_levels_ptr, nullptr, data_writer_ptr));
     } else {
-      RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(typename ParquetType::c_type)));
-      auto buffer_ptr =
-          reinterpret_cast<typename ParquetType::c_type*>(data_buffer_.mutable_data());
+      RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(ParquetCType)));
+      auto buffer_ptr = reinterpret_cast<ParquetCType*>(data_buffer_.mutable_data());
       int buffer_idx = 0;
       for (int i = 0; i < length; i++) {
         if (data->IsNull(offset + i)) {
           def_levels_ptr[i] = 0;
         } else {
           def_levels_ptr[i] = 1;
-          buffer_ptr[buffer_idx++] = data_ptr[i];
+          buffer_ptr[buffer_idx++] = static_cast<ParquetCType>(data_ptr[i]);
+        }
+      }
+      PARQUET_CATCH_NOT_OK(
+          writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
+    }
+  } else {
+    return Status::NotImplemented("no support for max definition level > 1 yet");
+  }
+  PARQUET_CATCH_NOT_OK(writer->Close());
+  return Status::OK();
+}
+
+// This specialization seems quite similar but it significantly differs in two points:
+// * offset is added at the most latest time to the pointer as we have sub-byte access
+// * Arrow data is stored bitwise thus we cannot use std::copy to transform from
+//   ArrowType::c_type to ParquetType::c_type
+template <>
+Status FileWriter::Impl::TypedWriteBatch<::parquet::BooleanType, BooleanType>(
+    ::parquet::ColumnWriter* column_writer, const PrimitiveArray* data, int64_t offset,
+    int64_t length) {
+  DCHECK((offset + length) <= data->length());
+  RETURN_NOT_OK(data_buffer_.Resize(length));
+  auto data_ptr = reinterpret_cast<const uint8_t*>(data->data()->data());
+  auto buffer_ptr = reinterpret_cast<bool*>(data_buffer_.mutable_data());
+  auto writer = reinterpret_cast<::parquet::TypedColumnWriter<::parquet::BooleanType>*>(
+      column_writer);
+  if (writer->descr()->max_definition_level() == 0) {
+    // no nulls, just dump the data
+    for (int64_t i = 0; i < length; i++) {
+      buffer_ptr[i] = util::get_bit(data_ptr, offset + i);
+    }
+    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, nullptr, nullptr, buffer_ptr));
+  } else if (writer->descr()->max_definition_level() == 1) {
+    RETURN_NOT_OK(def_levels_buffer_.Resize(length * sizeof(int16_t)));
+    int16_t* def_levels_ptr =
+        reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
+    if (data->null_count() == 0) {
+      std::fill(def_levels_ptr, def_levels_ptr + length, 1);
+      for (int64_t i = 0; i < length; i++) {
+        buffer_ptr[i] = util::get_bit(data_ptr, offset + i);
+      }
+      // TODO(PARQUET-644): write boolean values as a packed bitmap
+      PARQUET_CATCH_NOT_OK(
+          writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
+    } else {
+      int buffer_idx = 0;
+      for (int i = 0; i < length; i++) {
+        if (data->IsNull(offset + i)) {
+          def_levels_ptr[i] = 0;
+        } else {
+          def_levels_ptr[i] = 1;
+          buffer_ptr[buffer_idx++] = util::get_bit(data_ptr, offset + i);
         }
       }
       PARQUET_CATCH_NOT_OK(
@@ -120,9 +213,9 @@ Status FileWriter::Impl::Close() {
   return Status::OK();
 }
 
-#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType)                 \
-  case Type::ENUM:                                                     \
-    return TypedWriteBatch<ParquetType>(writer, data, offset, length); \
+#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType)                            \
+  case Type::ENUM:                                                                \
+    return TypedWriteBatch<ParquetType, ArrowType>(writer, data, offset, length); \
     break;
 
 Status FileWriter::Impl::WriteFlatColumnChunk(
@@ -130,15 +223,76 @@ Status FileWriter::Impl::WriteFlatColumnChunk(
   ::parquet::ColumnWriter* writer;
   PARQUET_CATCH_NOT_OK(writer = row_group_writer_->NextColumn());
   switch (data->type_enum()) {
-    TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type)
-    TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
-    TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
+    TYPED_BATCH_CASE(BOOL, BooleanType, ::parquet::BooleanType)
+    TYPED_BATCH_CASE(UINT8, UInt8Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(INT8, Int8Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(UINT16, UInt16Type, ::parquet::Int32Type)
+    TYPED_BATCH_CASE(INT16, Int16Type, ::parquet::Int32Type)
+    case Type::UINT32:
+      if (writer_->properties()->version() == ParquetVersion::PARQUET_1_0) {
+        // Parquet 1.0 reader cannot read the UINT_32 logical type. Thus we need
+        // to use the larger Int64Type to store them lossless.
+        return TypedWriteBatch<::parquet::Int64Type, UInt32Type>(
+            writer, data, offset, length);
+      } else {
+        return TypedWriteBatch<::parquet::Int32Type, UInt32Type>(
+            writer, data, offset, length);
+      }
+      TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type)
+      TYPED_BATCH_CASE(UINT64, UInt64Type, ::parquet::Int64Type)
+      TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type)
+      TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
+      TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
     default:
       return Status::NotImplemented(data->type()->ToString());
   }
 }
 
+Status FileWriter::Impl::WriteFlatColumnChunk(
+    const StringArray* data, int64_t offset, int64_t length) {
+  ::parquet::ColumnWriter* column_writer;
+  PARQUET_CATCH_NOT_OK(column_writer = row_group_writer_->NextColumn());
+  DCHECK((offset + length) <= data->length());
+  RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(::parquet::ByteArray)));
+  auto buffer_ptr = reinterpret_cast<::parquet::ByteArray*>(data_buffer_.mutable_data());
+  auto values = std::dynamic_pointer_cast<PrimitiveArray>(data->values());
+  auto data_ptr = reinterpret_cast<const uint8_t*>(values->data()->data());
+  DCHECK(values != nullptr);
+  auto writer = reinterpret_cast<::parquet::TypedColumnWriter<::parquet::ByteArrayType>*>(
+      column_writer);
+  if (writer->descr()->max_definition_level() > 0) {
+    RETURN_NOT_OK(def_levels_buffer_.Resize(length * sizeof(int16_t)));
+  }
+  int16_t* def_levels_ptr = reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
+  if (writer->descr()->max_definition_level() == 0 || data->null_count() == 0) {
+    // no nulls, just dump the data
+    for (int64_t i = 0; i < length; i++) {
+      buffer_ptr[i] = ::parquet::ByteArray(
+          data->value_length(i + offset), data_ptr + data->value_offset(i));
+    }
+    if (writer->descr()->max_definition_level() > 0) {
+      std::fill(def_levels_ptr, def_levels_ptr + length, 1);
+    }
+    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
+  } else if (writer->descr()->max_definition_level() == 1) {
+    int buffer_idx = 0;
+    for (int64_t i = 0; i < length; i++) {
+      if (data->IsNull(offset + i)) {
+        def_levels_ptr[i] = 0;
+      } else {
+        def_levels_ptr[i] = 1;
+        buffer_ptr[buffer_idx++] = ::parquet::ByteArray(
+            data->value_length(i + offset), data_ptr + data->value_offset(i + offset));
+      }
+    }
+    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
+  } else {
+    return Status::NotImplemented("no support for max definition level > 1 yet");
+  }
+  PARQUET_CATCH_NOT_OK(writer->Close());
+  return Status::OK();
+}
+
 FileWriter::FileWriter(
     MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer)
     : impl_(new FileWriter::Impl(pool, std::move(writer))) {}
@@ -148,10 +302,20 @@ Status FileWriter::NewRowGroup(int64_t chunk_size) {
 }
 
 Status FileWriter::WriteFlatColumnChunk(
-    const PrimitiveArray* data, int64_t offset, int64_t length) {
+    const Array* array, int64_t offset, int64_t length) {
   int64_t real_length = length;
-  if (length == -1) { real_length = data->length(); }
-  return impl_->WriteFlatColumnChunk(data, offset, real_length);
+  if (length == -1) { real_length = array->length(); }
+  if (array->type_enum() == Type::STRING) {
+    auto string_array = dynamic_cast<const StringArray*>(array);
+    DCHECK(string_array);
+    return impl_->WriteFlatColumnChunk(string_array, offset, real_length);
+  } else {
+    auto primitive_array = dynamic_cast<const PrimitiveArray*>(array);
+    if (!primitive_array) {
+      return Status::NotImplemented("Table must consist of PrimitiveArray instances");
+    }
+    return impl_->WriteFlatColumnChunk(primitive_array, offset, real_length);
+  }
 }
 
 Status FileWriter::Close() {
@@ -165,40 +329,30 @@ MemoryPool* FileWriter::memory_pool() const {
 FileWriter::~FileWriter() {}
 
 Status WriteFlatTable(const Table* table, MemoryPool* pool,
-    std::shared_ptr<::parquet::OutputStream> sink, int64_t chunk_size) {
+    const std::shared_ptr<::parquet::OutputStream>& sink, int64_t chunk_size,
+    const std::shared_ptr<::parquet::WriterProperties>& properties) {
   std::shared_ptr<::parquet::SchemaDescriptor> parquet_schema;
-  RETURN_NOT_OK(ToParquetSchema(table->schema().get(), &parquet_schema));
+  RETURN_NOT_OK(
+      ToParquetSchema(table->schema().get(), *properties.get(), &parquet_schema));
   auto schema_node = std::static_pointer_cast<GroupNode>(parquet_schema->schema());
   std::unique_ptr<ParquetFileWriter> parquet_writer =
-      ParquetFileWriter::Open(sink, schema_node);
+      ParquetFileWriter::Open(sink, schema_node, properties);
   FileWriter writer(pool, std::move(parquet_writer));
 
-  // TODO: Support writing chunked arrays.
+  // TODO(ARROW-232) Support writing chunked arrays.
   for (int i = 0; i < table->num_columns(); i++) {
     if (table->column(i)->data()->num_chunks() != 1) {
       return Status::NotImplemented("No support for writing chunked arrays yet.");
     }
   }
 
-  // Cast to PrimitiveArray instances as we work with them.
-  std::vector<std::shared_ptr<PrimitiveArray>> arrays(table->num_columns());
-  for (int i = 0; i < table->num_columns(); i++) {
-    // num_chunks == 1 as per above loop
-    std::shared_ptr<Array> array = table->column(i)->data()->chunk(0);
-    auto primitive_array = std::dynamic_pointer_cast<PrimitiveArray>(array);
-    if (!primitive_array) {
-      PARQUET_IGNORE_NOT_OK(writer.Close());
-      return Status::NotImplemented("Table must consist of PrimitiveArray instances");
-    }
-    arrays[i] = primitive_array;
-  }
-
   for (int chunk = 0; chunk * chunk_size < table->num_rows(); chunk++) {
     int64_t offset = chunk * chunk_size;
     int64_t size = std::min(chunk_size, table->num_rows() - offset);
     RETURN_NOT_OK_ELSE(writer.NewRowGroup(size), PARQUET_IGNORE_NOT_OK(writer.Close()));
     for (int i = 0; i < table->num_columns(); i++) {
-      RETURN_NOT_OK_ELSE(writer.WriteFlatColumnChunk(arrays[i].get(), offset, size),
+      std::shared_ptr<Array> array = table->column(i)->data()->chunk(0);
+      RETURN_NOT_OK_ELSE(writer.WriteFlatColumnChunk(array.get(), offset, size),
           PARQUET_IGNORE_NOT_OK(writer.Close()));
     }
   }
diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h
index 93693f511846b..cfd80d80b7997 100644
--- a/cpp/src/arrow/parquet/writer.h
+++ b/cpp/src/arrow/parquet/writer.h
@@ -25,10 +25,12 @@
 
 namespace arrow {
 
+class Array;
 class MemoryPool;
 class PrimitiveArray;
 class RowBatch;
 class Status;
+class StringArray;
 class Table;
 
 namespace parquet {
@@ -43,8 +45,7 @@ class FileWriter {
   FileWriter(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer);
 
   Status NewRowGroup(int64_t chunk_size);
-  Status WriteFlatColumnChunk(
-      const PrimitiveArray* data, int64_t offset = 0, int64_t length = -1);
+  Status WriteFlatColumnChunk(const Array* data, int64_t offset = 0, int64_t length = -1);
   Status Close();
 
   virtual ~FileWriter();
@@ -62,7 +63,9 @@ class FileWriter {
  * The table shall only consist of nullable, non-repeated columns of primitive type.
  */
 Status WriteFlatTable(const Table* table, MemoryPool* pool,
-    std::shared_ptr<::parquet::OutputStream> sink, int64_t chunk_size);
+    const std::shared_ptr<::parquet::OutputStream>& sink, int64_t chunk_size,
+    const std::shared_ptr<::parquet::WriterProperties>& properties =
+        ::parquet::default_writer_properties());
 
 }  // namespace parquet
 
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 2f81161d1d6d1..055dac7444488 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -50,6 +50,8 @@
     if (!s.ok()) { FAIL() << s.ToString(); } \
   } while (0)
 
+#define ASSERT_OK_NO_THROW(expr) ASSERT_NO_THROW(ASSERT_OK(expr))
+
 #define EXPECT_OK(expr)  \
   do {                   \
     Status s = (expr);   \
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 08fc8478e6de5..f4b47f9d2f503 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -133,6 +133,11 @@ Status PrimitiveBuilder<BooleanType>::Append(
   RETURN_NOT_OK(Reserve(length));
 
   for (int i = 0; i < length; ++i) {
+    // Skip reading from unitialised memory
+    // TODO: This actually is only to keep valgrind happy but may or may not
+    // have a performance impact.
+    if ((valid_bytes != nullptr) && !valid_bytes[i]) continue;
+
     if (values[i] > 0) {
       util::set_bit(raw_data_, length_ + i);
     } else {
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index 0918344070eb0..a2f83ea5ea566 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -32,6 +32,10 @@ cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
     pass
 
 cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
+  enum ParquetVersion" parquet::ParquetVersion::type":
+      PARQUET_1_0" parquet::ParquetVersion::PARQUET_1_0"
+      PARQUET_2_0" parquet::ParquetVersion::PARQUET_2_0"
+
   cdef cppclass SchemaDescriptor:
     shared_ptr[Node] schema()
     GroupNode* group()
@@ -80,6 +84,11 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
         LocalFileOutputStream(const c_string& path)
         void Close()
 
+    cdef cppclass WriterProperties:
+        cppclass Builder:
+            Builder* version(ParquetVersion version)
+            shared_ptr[WriterProperties] build()
+
 
 cdef extern from "arrow/parquet/reader.h" namespace "arrow::parquet" nogil:
     cdef cppclass FileReader:
@@ -93,5 +102,7 @@ cdef extern from "arrow/parquet/schema.h" namespace "arrow::parquet" nogil:
 
 
 cdef extern from "arrow/parquet/writer.h" namespace "arrow::parquet" nogil:
-    cdef CStatus WriteFlatTable(const CTable* table, MemoryPool* pool, shared_ptr[OutputStream] sink, int64_t chunk_size)
+    cdef CStatus WriteFlatTable(const CTable* table, MemoryPool* pool,
+            const shared_ptr[OutputStream]& sink, int64_t chunk_size,
+            const shared_ptr[WriterProperties]& properties)
 
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 3d5355ebe433a..0b2b20880332b 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -24,6 +24,7 @@ cimport pyarrow.includes.pyarrow as pyarrow
 from pyarrow.includes.parquet cimport *
 
 from pyarrow.compat import tobytes
+from pyarrow.error import ArrowException
 from pyarrow.error cimport check_cstatus
 from pyarrow.table cimport Table
 
@@ -42,11 +43,13 @@ def read_table(filename, columns=None):
     # in Cython (due to missing rvalue support)
     reader = unique_ptr[FileReader](new FileReader(default_memory_pool(),
         ParquetFileReader.OpenFile(tobytes(filename))))
-    check_cstatus(reader.get().ReadFlatTable(&ctable))
+    with nogil:
+        check_cstatus(reader.get().ReadFlatTable(&ctable))
+
     table.init(ctable)
     return table
 
-def write_table(table, filename, chunk_size=None):
+def write_table(table, filename, chunk_size=None, version=None):
     """
     Write a Table to Parquet format
 
@@ -56,16 +59,29 @@ def write_table(table, filename, chunk_size=None):
     filename : string
     chunk_size : int
         The maximum number of rows in each Parquet RowGroup
+    version : {"1.0", "2.0"}, default "1.0"
+        The Parquet format version, defaults to 1.0
     """
     cdef Table table_ = table
     cdef CTable* ctable_ = table_.table
     cdef shared_ptr[OutputStream] sink
+    cdef WriterProperties.Builder properties_builder
     cdef int64_t chunk_size_ = 0
     if chunk_size is None:
         chunk_size_ = min(ctable_.num_rows(), int(2**16))
     else:
         chunk_size_ = chunk_size
 
+    if version is not None:
+        if version == "1.0":
+            properties_builder.version(PARQUET_1_0)
+        elif version == "2.0":
+            properties_builder.version(PARQUET_2_0)
+        else:
+            raise ArrowException("Unsupported Parquet format version")
+
     sink.reset(new LocalFileOutputStream(tobytes(filename)))
-    check_cstatus(WriteFlatTable(ctable_, default_memory_pool(), sink, chunk_size_))
+    with nogil:
+        check_cstatus(WriteFlatTable(ctable_, default_memory_pool(), sink,
+            chunk_size_, properties_builder.build()))
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index d92cf4ca6563e..de9cfbb46e1a2 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -42,18 +42,55 @@ def test_single_pylist_column_roundtrip(tmpdir):
             data_read = col_read.data.chunk(0)
             assert data_written.equals(data_read)
 
-def test_pandas_rountrip(tmpdir):
+def test_pandas_parquet_2_0_rountrip(tmpdir):
     size = 10000
+    np.random.seed(0)
     df = pd.DataFrame({
+        'uint8': np.arange(size, dtype=np.uint8),
+        'uint16': np.arange(size, dtype=np.uint16),
+        'uint32': np.arange(size, dtype=np.uint32),
+        'uint64': np.arange(size, dtype=np.uint64),
+        'int8': np.arange(size, dtype=np.int16),
+        'int16': np.arange(size, dtype=np.int16),
         'int32': np.arange(size, dtype=np.int32),
         'int64': np.arange(size, dtype=np.int64),
         'float32': np.arange(size, dtype=np.float32),
-        'float64': np.arange(size, dtype=np.float64)
+        'float64': np.arange(size, dtype=np.float64),
+        'bool': np.random.randn(size) > 0,
+        'str': [str(x) for x in range(size)],
+        'str_with_nulls': [None] + [str(x) for x in range(size - 2)] + [None]
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = A.from_pandas_dataframe(df)
-    A.parquet.write_table(arrow_table, filename.strpath)
+    A.parquet.write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pyarrow.parquet.read_table(filename.strpath)
     df_read = table_read.to_pandas()
     pdt.assert_frame_equal(df, df_read)
 
+def test_pandas_parquet_1_0_rountrip(tmpdir):
+    size = 10000
+    np.random.seed(0)
+    df = pd.DataFrame({
+        'uint8': np.arange(size, dtype=np.uint8),
+        'uint16': np.arange(size, dtype=np.uint16),
+        'uint32': np.arange(size, dtype=np.uint32),
+        'uint64': np.arange(size, dtype=np.uint64),
+        'int8': np.arange(size, dtype=np.int16),
+        'int16': np.arange(size, dtype=np.int16),
+        'int32': np.arange(size, dtype=np.int32),
+        'int64': np.arange(size, dtype=np.int64),
+        'float32': np.arange(size, dtype=np.float32),
+        'float64': np.arange(size, dtype=np.float64),
+        'bool': np.random.randn(size) > 0
+    })
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = A.from_pandas_dataframe(df)
+    A.parquet.write_table(arrow_table, filename.strpath, version="1.0")
+    table_read = pyarrow.parquet.read_table(filename.strpath)
+    df_read = table_read.to_pandas()
+
+    # We pass uint32_t as int64_t if we write Parquet version 1.0
+    df['uint32'] = df['uint32'].values.astype(np.int64)
+
+    pdt.assert_frame_equal(df, df_read)
+

From fab4c82d2668e4f8c450053c34dd70ea99365fac Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Fri, 1 Jul 2016 14:25:46 -0700
Subject: [PATCH 0094/1644] ARROW-234: Build libhdfs IO extension in conda
 artifacts

Author: Wes McKinney <wesm@apache.org>

Closes #97 from wesm/ARROW-234 and squashes the following commits:

3edb8d1 [Wes McKinney] Enable ARROW_HDFS extension in conda artifact
---
 cpp/conda.recipe/build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh
index b10dd03349bd3..7e60ccc911faa 100644
--- a/cpp/conda.recipe/build.sh
+++ b/cpp/conda.recipe/build.sh
@@ -49,6 +49,7 @@ cmake \
     -DCMAKE_BUILD_TYPE=release \
     -DCMAKE_INSTALL_PREFIX=$PREFIX \
     -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \
+    -DARROW_HDFS=on \
     -DARROW_IPC=on \
     -DARROW_PARQUET=on \
     ..

From 77598fa59a92c07dedf7d93307e5c72c5b2724d0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Sun, 10 Jul 2016 13:17:50 -0700
Subject: [PATCH 0095/1644] ARROW-233: Add visibility macros, add static build
 option

This also resolves ARROW-213. Builds off work done in PARQUET-489.

I inserted a hack to deal with the fast the boost libs in apt won't statically link properly. We'll deal with that some other time.

Author: Wes McKinney <wesm@apache.org>

Closes #100 from wesm/ARROW-233 and squashes the following commits:

0253827 [Wes McKinney] Remove -Wno-unused-local-typedef
69b03b0 [Wes McKinney] - Add visibility macros. Hide boost symbols in arrow_io - Hack around Travis CI inability to use its boost static libraries - Use parquet_shared name - More informative verbose test logs - Fix some gtest-1.7.0 crankiness - Fix a valgrind shared_ptr possible memory leak stemming from static variable   referenced at compile-time in libarrow_parquet - Fix a bunch of compiler warnings in release builds
---
 ci/travis_install_conda.sh               |   1 -
 ci/travis_script_cpp.sh                  |   2 +-
 ci/travis_script_python.sh               |   6 +-
 cpp/CMakeLists.txt                       | 217 ++++++++++++-----------
 cpp/build-support/run-test.sh            |  10 +-
 cpp/conda.recipe/build.sh                |  13 +-
 cpp/src/arrow/array.h                    |   5 +-
 cpp/src/arrow/builder.h                  |   3 +-
 cpp/src/arrow/column.h                   |   5 +-
 cpp/src/arrow/io/CMakeLists.txt          |  53 ++++--
 cpp/src/arrow/io/hdfs-io-test.cc         |   2 +-
 cpp/src/arrow/io/hdfs.h                  |  17 +-
 cpp/src/arrow/io/libhdfs_shim.cc         |   3 +-
 cpp/src/arrow/io/symbols.map             |  18 ++
 cpp/src/arrow/ipc/CMakeLists.txt         |   2 +-
 cpp/src/arrow/parquet/CMakeLists.txt     |   4 +-
 cpp/src/arrow/parquet/parquet-io-test.cc |  18 +-
 cpp/src/arrow/parquet/reader.cc          |   2 +-
 cpp/src/arrow/parquet/reader.h           |   6 +-
 cpp/src/arrow/parquet/schema.h           |  10 +-
 cpp/src/arrow/parquet/writer.cc          |   4 +-
 cpp/src/arrow/parquet/writer.h           |   6 +-
 cpp/src/arrow/schema.h                   |   4 +-
 cpp/src/arrow/symbols.map                |  15 ++
 cpp/src/arrow/table.h                    |   6 +-
 cpp/src/arrow/type.h                     |  39 ++--
 cpp/src/arrow/types/construct.h          |  11 +-
 cpp/src/arrow/types/decimal.h            |   3 +-
 cpp/src/arrow/types/list.h               |   7 +-
 cpp/src/arrow/types/primitive.h          |  13 +-
 cpp/src/arrow/types/string-test.cc       |   8 +-
 cpp/src/arrow/types/string.cc            |  11 +-
 cpp/src/arrow/types/string.h             |  16 +-
 cpp/src/arrow/types/struct-test.cc       |   8 +-
 cpp/src/arrow/types/struct.h             |   5 +-
 cpp/src/arrow/util/CMakeLists.txt        |   1 +
 cpp/src/arrow/util/buffer.h              |  12 +-
 cpp/src/arrow/util/memory-pool-test.cc   |   2 +-
 cpp/src/arrow/util/memory-pool.h         |   6 +-
 cpp/src/arrow/util/status.cc             |   3 +
 cpp/src/arrow/util/status.h              |   4 +-
 cpp/src/arrow/util/visibility.h          |  32 ++++
 python/conda.recipe/build.sh             |  15 +-
 python/src/pyarrow/adapters/builtin.h    |   2 +
 python/src/pyarrow/adapters/pandas.h     |   5 +
 python/src/pyarrow/common.h              |   6 +-
 python/src/pyarrow/config.h              |   4 +
 python/src/pyarrow/helpers.h             |   3 +
 python/src/pyarrow/status.h              |   4 +-
 python/src/pyarrow/visibility.h          |  32 ++++
 50 files changed, 439 insertions(+), 245 deletions(-)
 create mode 100644 cpp/src/arrow/io/symbols.map
 create mode 100644 cpp/src/arrow/symbols.map
 create mode 100644 cpp/src/arrow/util/visibility.h
 create mode 100644 python/src/pyarrow/visibility.h

diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index be7f59a4733bd..3a8f57bf8f1bf 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -25,4 +25,3 @@ conda install --yes conda-build jinja2 anaconda-client
 
 # faster builds, please
 conda install -y nomkl
-
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index 9cf4f8e352109..a3585507f0a6d 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -16,6 +16,6 @@ make lint
 #   make check-clang-tidy
 # fi
 
-ctest -L unittest
+ctest -VV -L unittest
 
 popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 6d35785356ab4..4a377428ae43a 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -7,7 +7,6 @@ PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 # Re-use conda installation from C++
 export MINICONDA=$TRAVIS_BUILD_DIR/miniconda
 export PATH="$MINICONDA/bin:$PATH"
-export LD_LIBRARY_PATH="$MINICONDA/lib:$LD_LIBRARY_PATH"
 export PARQUET_HOME=$MINICONDA
 
 # Share environment with C++
@@ -32,12 +31,15 @@ python_version_tests() {
   # Expensive dependencies install from Continuum package repo
   conda install -y pip numpy pandas cython
 
+  conda install -y parquet-cpp arrow-cpp -c apache/channel/dev
+
   # Other stuff pip install
   pip install -r requirements.txt
 
   export ARROW_HOME=$ARROW_CPP_INSTALL
 
-  python setup.py build_ext --inplace
+  python setup.py build_ext \
+		 --inplace
 
   python -m pytest -vv -r sxX pyarrow
 }
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 18b47599b93d0..a39a752123155 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -44,12 +44,22 @@ endif(CCACHE_FOUND)
 
 # Top level cmake dir
 if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
+  option(ARROW_BUILD_STATIC
+    "Build the libarrow static libraries"
+    ON)
+
+  option(ARROW_BUILD_SHARED
+    "Build the libarrow shared libraries"
+    ON)
+
   option(ARROW_PARQUET
     "Build the Parquet adapter and link to libparquet"
     OFF)
+
   option(ARROW_TEST_MEMCHECK
-	"Run the test suite using valgrind --tool=memcheck"
-	OFF)
+    "Run the test suite using valgrind --tool=memcheck"
+    OFF)
+
   option(ARROW_BUILD_TESTS
     "Build the Arrow googletest unit tests"
     ON)
@@ -66,6 +76,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow IO extensions for the Hadoop file system"
     OFF)
 
+  option(ARROW_BOOST_USE_SHARED
+    "Rely on boost shared libraries where relevant"
+    ON)
+
   option(ARROW_SSE3
     "Build Arrow with SSE3"
     ON)
@@ -172,18 +186,6 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CLANG_OPTIONS}")
 endif()
 
-# Sanity check linking option.
-if (NOT ARROW_LINK)
-  set(ARROW_LINK "d")
-elseif(NOT ("auto" MATCHES "^${ARROW_LINK}" OR
-            "dynamic" MATCHES "^${ARROW_LINK}" OR
-            "static" MATCHES "^${ARROW_LINK}"))
-  message(FATAL_ERROR "Unknown value for ARROW_LINK, must be auto|dynamic|static")
-else()
-  # Remove all but the first letter.
-  string(SUBSTRING "${ARROW_LINK}" 0 1 ARROW_LINK)
-endif()
-
 # ASAN / TSAN / UBSAN
 include(san-config)
 
@@ -203,61 +205,11 @@ if ("${ARROW_GENERATE_COVERAGE}")
   # For coverage to work properly, we need to use static linkage. Otherwise,
   # __gcov_flush() doesn't properly flush coverage from every module.
   # See http://stackoverflow.com/questions/28164543/using-gcov-flush-within-a-library-doesnt-force-the-other-modules-to-yield-gc
-  if("${ARROW_LINK}" STREQUAL "a")
-    message("Using static linking for coverage build")
-    set(ARROW_LINK "s")
-  elseif("${ARROW_LINK}" STREQUAL "d")
-    message(SEND_ERROR "Cannot use coverage with dynamic linking")
-  endif()
-endif()
-
-# If we still don't know what kind of linking to perform, choose based on
-# build type (developers like fast builds).
-if ("${ARROW_LINK}" STREQUAL "a")
-  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
-      "${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
-    message("Using dynamic linking for ${CMAKE_BUILD_TYPE} builds")
-    set(ARROW_LINK "d")
-  else()
-    message("Using static linking for ${CMAKE_BUILD_TYPE} builds")
-    set(ARROW_LINK "s")
+  if(NOT ARROW_BUILD_STATIC)
+    message(SEND_ERROR "Coverage requires the static lib to be built")
   endif()
 endif()
 
-# Are we using the gold linker? It doesn't work with dynamic linking as
-# weak symbols aren't properly overridden, causing tcmalloc to be omitted.
-# Let's flag this as an error in RELEASE builds (we shouldn't release a
-# product like this).
-#
-# See https://sourceware.org/bugzilla/show_bug.cgi?id=16979 for details.
-#
-# The gold linker is only for ELF binaries, which OSX doesn't use. We can
-# just skip.
-if (NOT APPLE)
-  execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Wl,--version OUTPUT_VARIABLE LINKER_OUTPUT)
-endif ()
-if (LINKER_OUTPUT MATCHES "gold")
-  if ("${ARROW_LINK}" STREQUAL "d" AND
-      "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
-    message(SEND_ERROR "Cannot use gold with dynamic linking in a RELEASE build "
-      "as it would cause tcmalloc symbols to get dropped")
-  else()
-    message("Using gold linker")
-  endif()
-  set(ARROW_USING_GOLD 1)
-else()
-  message("Using ld linker")
-endif()
-
-# Having set ARROW_LINK due to build type and/or sanitizer, it's now safe to
-# act on its value.
-if ("${ARROW_LINK}" STREQUAL "d")
-  set(BUILD_SHARED_LIBS ON)
-
-  # Position independent code is only necessary when producing shared objects.
-  add_definitions(-fPIC)
-endif()
-
 # set compile output directory
 string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
 
@@ -290,6 +242,15 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 include_directories(src)
 
+############################################################
+# Visibility
+############################################################
+# For generate_export_header() and add_compiler_export_flags().
+include(GenerateExportHeader)
+
+# Sets -fvisibility=hidden for gcc
+add_compiler_export_flags()
+
 ############################################################
 # Benchmarking
 ############################################################
@@ -360,7 +321,7 @@ endfunction()
 #
 # Arguments after the test name will be passed to set_tests_properties().
 function(ADD_ARROW_TEST REL_TEST_NAME)
-  if(NO_TESTS)
+  if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
     return()
   endif()
   get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
@@ -377,13 +338,13 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
   endif()
 
   if (ARROW_TEST_MEMCHECK)
-	SET_PROPERTY(TARGET ${TEST_NAME}
-	  APPEND_STRING PROPERTY
-	  COMPILE_FLAGS " -DARROW_VALGRIND")
-	add_test(${TEST_NAME}
-	  valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
+    SET_PROPERTY(TARGET ${TEST_NAME}
+      APPEND_STRING PROPERTY
+      COMPILE_FLAGS " -DARROW_VALGRIND")
+    add_test(${TEST_NAME}
+      valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
   else()
-	add_test(${TEST_NAME}
+    add_test(${TEST_NAME}
       ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
   endif()
   set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
@@ -427,19 +388,34 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
   endif()
 
-  if(("${ARROW_LINK}" STREQUAL "s" AND ARG_STATIC_LIB) OR (NOT ARG_SHARED_LIB))
+  if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
     if(NOT ARG_STATIC_LIB)
       message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
     endif()
+
+    SET(AUG_LIB_NAME "${LIB_NAME}_static")
+    add_library(${AUG_LIB_NAME} STATIC IMPORTED)
+    set_target_properties(${AUG_LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
+
+    SET(AUG_LIB_NAME "${LIB_NAME}_shared")
+    add_library(${AUG_LIB_NAME} SHARED IMPORTED)
+    set_target_properties(${AUG_LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+  elseif(ARG_STATIC_LIB)
     add_library(${LIB_NAME} STATIC IMPORTED)
     set_target_properties(${LIB_NAME}
       PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
     message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
-  else()
+  elseif(ARG_SHARED_LIB)
     add_library(${LIB_NAME} SHARED IMPORTED)
     set_target_properties(${LIB_NAME}
       PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+  else()
+    message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
   endif()
 
   if(ARG_DEPS)
@@ -538,9 +514,17 @@ endif()
 ############################################################
 # Linker setup
 ############################################################
-set(ARROW_MIN_TEST_LIBS arrow arrow_test_main ${ARROW_BASE_LIBS})
+set(ARROW_MIN_TEST_LIBS
+  arrow_static
+  arrow_test_main
+  ${ARROW_BASE_LIBS})
+
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
-set(ARROW_BENCHMARK_LINK_LIBS arrow arrow_benchmark_main ${ARROW_BASE_LIBS})
+
+set(ARROW_BENCHMARK_LINK_LIBS
+  arrow_static
+  arrow_benchmark_main
+  ${ARROW_BASE_LIBS})
 
 ############################################################
 # "make ctags" target
@@ -576,14 +560,14 @@ endif (UNIX)
 if (UNIX)
 
   file(GLOB_RECURSE LINT_FILES
-	"${CMAKE_CURRENT_SOURCE_DIR}/src/*.h"
-	"${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc"
-	)
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/*.h"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc"
+    )
 
   FOREACH(item ${LINT_FILES})
-	IF(NOT (item MATCHES "_generated.h"))
+    IF(NOT (item MATCHES "_generated.h"))
       LIST(APPEND FILTERED_LINT_FILES ${item})
-	ENDIF()
+    ENDIF()
   ENDFOREACH(item ${LINT_FILES})
 
   # Full lint
@@ -628,7 +612,10 @@ endif()
 # Subdirectories
 ############################################################
 
-set(LIBARROW_LINK_LIBS
+set(ARROW_LINK_LIBS
+)
+
+set(ARROW_PRIVATE_LINK_LIBS
 )
 
 set(ARROW_SRCS
@@ -660,35 +647,67 @@ set(ARROW_SRCS
   src/arrow/util/status.cc
 )
 
-set(LIBARROW_LINKAGE "SHARED")
-
-add_library(arrow
-  ${LIBARROW_LINKAGE}
+add_library(arrow_objlib OBJECT
   ${ARROW_SRCS}
 )
 
+# Necessary to make static linking into other shared libraries work properly
+set_property(TARGET arrow_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
+
+if(NOT APPLE)
+  # Localize thirdparty symbols using a linker version script. This hides them
+  # from the client application. The OS X linker does not support the
+  # version-script option.
+  set(SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map")
+endif()
+
+if (ARROW_BUILD_SHARED)
+  add_library(arrow_shared SHARED $<TARGET_OBJECTS:arrow_objlib>)
+  if(APPLE)
+    set_target_properties(arrow_shared PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  endif()
+  set_target_properties(arrow_shared
+    PROPERTIES
+    LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+    LINK_FLAGS "${SHARED_LINK_FLAGS}"
+    OUTPUT_NAME "arrow")
+  target_link_libraries(arrow_shared
+    LINK_PUBLIC ${ARROW_LINK_LIBS}
+    LINK_PRIVATE ${ARROW_PRIVATE_LINK_LIBS})
+
+  install(TARGETS arrow_shared
+    LIBRARY DESTINATION lib
+    ARCHIVE DESTINATION lib)
+endif()
+
+if (ARROW_BUILD_STATIC)
+  add_library(arrow_static STATIC $<TARGET_OBJECTS:arrow_objlib>)
+  set_target_properties(arrow_static
+    PROPERTIES
+    LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+    OUTPUT_NAME "arrow")
+
+  target_link_libraries(arrow_static
+    LINK_PUBLIC ${ARROW_LINK_LIBS}
+    LINK_PRIVATE ${ARROW_PRIVATE_LINK_LIBS})
+
+  install(TARGETS arrow_static
+    LIBRARY DESTINATION lib
+    ARCHIVE DESTINATION lib)
+endif()
+
 if (APPLE)
-  set_target_properties(arrow
+  set_target_properties(arrow_shared
     PROPERTIES
     BUILD_WITH_INSTALL_RPATH ON
     INSTALL_NAME_DIR "@rpath")
 endif()
 
-set_target_properties(arrow
-  PROPERTIES
-  LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
-)
-target_link_libraries(arrow ${LIBARROW_LINK_LIBS})
-
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
 add_subdirectory(src/arrow/util)
 add_subdirectory(src/arrow/types)
 
-install(TARGETS arrow
-  LIBRARY DESTINATION lib
-  ARCHIVE DESTINATION lib)
-
 #----------------------------------------------------------------------
 # Parquet adapter library
 
@@ -715,7 +734,7 @@ if(ARROW_IPC)
   include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
   add_library(flatbuffers STATIC IMPORTED)
   set_target_properties(flatbuffers PROPERTIES
-	IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB})
+    IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB})
 
   add_subdirectory(src/arrow/ipc)
 endif()
diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh
index 0e628e26ecd52..f563da53679be 100755
--- a/cpp/build-support/run-test.sh
+++ b/cpp/build-support/run-test.sh
@@ -79,16 +79,16 @@ function setup_sanitizers() {
   TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$ROOT/build-support/tsan-suppressions.txt"
   TSAN_OPTIONS="$TSAN_OPTIONS history_size=7"
   export TSAN_OPTIONS
-  
+
   # Enable leak detection even under LLVM 3.4, where it was disabled by default.
   # This flag only takes effect when running an ASAN build.
   ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
   export ASAN_OPTIONS
-  
+
   # Set up suppressions for LeakSanitizer
   LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt"
   export LSAN_OPTIONS
-  
+
   # Suppressions require symbolization. We'll default to using the symbolizer in
   # thirdparty.
   if [ -z "$ASAN_SYMBOLIZER_PATH" ]; then
@@ -107,7 +107,7 @@ function run_test() {
     | $ROOT/build-support/asan_symbolize.py \
     | c++filt \
     | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \
-    | $pipe_cmd > $LOGFILE
+    | $pipe_cmd 2>&1 | tee $LOGFILE
   STATUS=$?
 
   # TSAN doesn't always exit with a non-zero exit code due to a bug:
@@ -198,7 +198,7 @@ for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
   fi
 done
 
-if [ $RUN_TYPE = "test" ]; then	
+if [ $RUN_TYPE = "test" ]; then
   post_process_tests
 fi
 
diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh
index 7e60ccc911faa..2f2b748266747 100644
--- a/cpp/conda.recipe/build.sh
+++ b/cpp/conda.recipe/build.sh
@@ -39,16 +39,17 @@ pwd
 source thirdparty/versions.sh
 export GTEST_HOME=`pwd`/thirdparty/$GTEST_BASEDIR
 
-if [ `uname` == Linux ]; then
-    SHARED_LINKER_FLAGS='-static-libstdc++'
-elif [ `uname` == Darwin ]; then
-    SHARED_LINKER_FLAGS=''
-fi
+# if [ `uname` == Linux ]; then
+#     SHARED_LINKER_FLAGS='-static-libstdc++'
+# elif [ `uname` == Darwin ]; then
+#     SHARED_LINKER_FLAGS=''
+# fi
+
+# -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \
 
 cmake \
     -DCMAKE_BUILD_TYPE=release \
     -DCMAKE_INSTALL_PREFIX=$PREFIX \
-    -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \
     -DARROW_HDFS=on \
     -DARROW_IPC=on \
     -DARROW_PARQUET=on \
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 76dc0f598141f..c7ffb23ca18a1 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -24,6 +24,7 @@
 #include "arrow/type.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -35,7 +36,7 @@ class Status;
 //
 // The base class is only required to have a null bitmap buffer if the null
 // count is greater than 0
-class Array {
+class ARROW_EXPORT Array {
  public:
   Array(const std::shared_ptr<DataType>& type, int32_t length, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
@@ -83,7 +84,7 @@ class Array {
 };
 
 // Degenerate null type Array
-class NullArray : public Array {
+class ARROW_EXPORT NullArray : public Array {
  public:
   NullArray(const std::shared_ptr<DataType>& type, int32_t length)
       : Array(type, length, length, nullptr) {}
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 7d3f4398d73e3..5d9fb992ff0b5 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -25,6 +25,7 @@
 #include "arrow/type.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/status.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -38,7 +39,7 @@ static constexpr int32_t MIN_BUILDER_CAPACITY = 1 << 5;
 // This class provides a facilities for incrementally building the null bitmap
 // (see Append methods) and as a side effect the current number of slots and
 // the null count.
-class ArrayBuilder {
+class ARROW_EXPORT ArrayBuilder {
  public:
   explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type)
       : pool_(pool),
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
index e409566e1f139..d5168cb032ba5 100644
--- a/cpp/src/arrow/column.h
+++ b/cpp/src/arrow/column.h
@@ -24,6 +24,7 @@
 #include <vector>
 
 #include "arrow/type.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -34,7 +35,7 @@ typedef std::vector<std::shared_ptr<Array>> ArrayVector;
 
 // A data structure managing a list of primitive Arrow arrays logically as one
 // large array
-class ChunkedArray {
+class ARROW_EXPORT ChunkedArray {
  public:
   explicit ChunkedArray(const ArrayVector& chunks);
 
@@ -56,7 +57,7 @@ class ChunkedArray {
 // An immutable column data structure consisting of a field (type metadata) and
 // a logical chunked data array (which can be validated as all being the same
 // type).
-class Column {
+class ARROW_EXPORT Column {
  public:
   Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
   Column(const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data);
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 33b654f81903f..b8c0e138afb06 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -19,13 +19,18 @@
 # arrow_io : Arrow IO interfaces
 
 set(ARROW_IO_LINK_LIBS
-  arrow
+  arrow_shared
 )
 
-set(ARROW_IO_PRIVATE_LINK_LIBS
-  boost_system
-  boost_filesystem
-)
+if (ARROW_BOOST_USE_SHARED)
+  set(ARROW_IO_PRIVATE_LINK_LIBS
+    boost_system_shared
+    boost_filesystem_shared)
+else()
+  set(ARROW_IO_PRIVATE_LINK_LIBS
+    boost_system_static
+    boost_filesystem_static)
+endif()
 
 set(ARROW_IO_TEST_LINK_LIBS
   arrow_io
@@ -36,18 +41,18 @@ set(ARROW_IO_SRCS
 
 if(ARROW_HDFS)
   if(NOT THIRDPARTY_DIR)
-	message(FATAL_ERROR "THIRDPARTY_DIR not set")
+    message(FATAL_ERROR "THIRDPARTY_DIR not set")
   endif()
 
   if (DEFINED ENV{HADOOP_HOME})
-	set(HADOOP_HOME $ENV{HADOOP_HOME})
+    set(HADOOP_HOME $ENV{HADOOP_HOME})
   else()
-	set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+    set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
   endif()
 
   set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
   if (NOT EXISTS ${HDFS_H_PATH})
-	message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
+    message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
   endif()
   message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
   message(STATUS "Building libhdfs shim component")
@@ -55,29 +60,39 @@ if(ARROW_HDFS)
   include_directories(SYSTEM "${HADOOP_HOME}/include")
 
   set(ARROW_HDFS_SRCS
-	hdfs.cc
-	libhdfs_shim.cc)
+    hdfs.cc
+    libhdfs_shim.cc)
 
   set_property(SOURCE ${ARROW_HDFS_SRCS}
-	APPEND_STRING PROPERTY
-	COMPILE_FLAGS "-DHAS_HADOOP")
+    APPEND_STRING PROPERTY
+    COMPILE_FLAGS "-DHAS_HADOOP")
 
   set(ARROW_IO_SRCS
-	${ARROW_HDFS_SRCS}
-	${ARROW_IO_SRCS})
+    ${ARROW_HDFS_SRCS}
+    ${ARROW_IO_SRCS})
 
   ADD_ARROW_TEST(hdfs-io-test)
   ARROW_TEST_LINK_LIBRARIES(hdfs-io-test
-	${ARROW_IO_TEST_LINK_LIBS})
+    ${ARROW_IO_TEST_LINK_LIBS})
 endif()
 
 add_library(arrow_io SHARED
   ${ARROW_IO_SRCS}
 )
-target_link_libraries(arrow_io LINK_PUBLIC ${ARROW_IO_LINK_LIBS})
-target_link_libraries(arrow_io LINK_PRIVATE ${ARROW_IO_PRIVATE_LINK_LIBS})
+target_link_libraries(arrow_io
+  LINK_PUBLIC ${ARROW_IO_LINK_LIBS}
+  LINK_PRIVATE ${ARROW_IO_PRIVATE_LINK_LIBS})
+
+if(NOT APPLE)
+  # Localize thirdparty symbols using a linker version script. This hides them
+  # from the client application. The OS X linker does not support the
+  # version-script option.
+  set(ARROW_IO_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
+endif()
 
-SET_TARGET_PROPERTIES(arrow_io PROPERTIES LINKER_LANGUAGE CXX)
+SET_TARGET_PROPERTIES(arrow_io PROPERTIES
+  LINKER_LANGUAGE CXX
+  LINK_FLAGS "${ARROW_IO_LINK_FLAGS}")
 
 if (APPLE)
   set_target_properties(arrow_io
diff --git a/cpp/src/arrow/io/hdfs-io-test.cc b/cpp/src/arrow/io/hdfs-io-test.cc
index 11d67aeba2026..d1bf140ae68e2 100644
--- a/cpp/src/arrow/io/hdfs-io-test.cc
+++ b/cpp/src/arrow/io/hdfs-io-test.cc
@@ -227,7 +227,7 @@ TEST_F(TestHdfsClient, ListDirectory) {
   // Do it again, appends!
   ASSERT_OK(client_->ListDirectory(scratch_dir_, &listing));
 
-  ASSERT_EQ(6, listing.size());
+  ASSERT_EQ(6, static_cast<int>(listing.size()));
 
   // Argh, well, shouldn't expect the listing to be in any particular order
   for (size_t i = 0; i < listing.size(); ++i) {
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index a1972db96157a..532e3c536a188 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -25,6 +25,7 @@
 
 #include "arrow/io/interfaces.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -32,8 +33,6 @@ class Status;
 
 namespace io {
 
-Status ConnectLibHdfs();
-
 class HdfsClient;
 class HdfsReadableFile;
 class HdfsWriteableFile;
@@ -64,7 +63,7 @@ struct HdfsConnectionConfig {
   // TODO: Kerberos, etc.
 };
 
-class HdfsClient : public FileSystemClient {
+class ARROW_EXPORT HdfsClient : public FileSystemClient {
  public:
   ~HdfsClient();
 
@@ -149,14 +148,14 @@ class HdfsClient : public FileSystemClient {
   friend class HdfsReadableFile;
   friend class HdfsWriteableFile;
 
-  class HdfsClientImpl;
+  class ARROW_NO_EXPORT HdfsClientImpl;
   std::unique_ptr<HdfsClientImpl> impl_;
 
   HdfsClient();
   DISALLOW_COPY_AND_ASSIGN(HdfsClient);
 };
 
-class HdfsReadableFile : public RandomAccessFile {
+class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
  public:
   ~HdfsReadableFile();
 
@@ -175,7 +174,7 @@ class HdfsReadableFile : public RandomAccessFile {
   Status Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) override;
 
  private:
-  class HdfsReadableFileImpl;
+  class ARROW_NO_EXPORT HdfsReadableFileImpl;
   std::unique_ptr<HdfsReadableFileImpl> impl_;
 
   friend class HdfsClient::HdfsClientImpl;
@@ -184,7 +183,7 @@ class HdfsReadableFile : public RandomAccessFile {
   DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
 };
 
-class HdfsWriteableFile : public WriteableFile {
+class ARROW_EXPORT HdfsWriteableFile : public WriteableFile {
  public:
   ~HdfsWriteableFile();
 
@@ -197,7 +196,7 @@ class HdfsWriteableFile : public WriteableFile {
   Status Tell(int64_t* position) override;
 
  private:
-  class HdfsWriteableFileImpl;
+  class ARROW_NO_EXPORT HdfsWriteableFileImpl;
   std::unique_ptr<HdfsWriteableFileImpl> impl_;
 
   friend class HdfsClient::HdfsClientImpl;
@@ -207,6 +206,8 @@ class HdfsWriteableFile : public WriteableFile {
   DISALLOW_COPY_AND_ASSIGN(HdfsWriteableFile);
 };
 
+Status ARROW_EXPORT ConnectLibHdfs();
+
 }  // namespace io
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
index f75266536e5b3..003570d4fdee6 100644
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -55,6 +55,7 @@ extern "C" {
 #include <boost/algorithm/string.hpp>  // NOLINT
 
 #include "arrow/util/status.h"
+#include "arrow/util/visibility.h"
 
 namespace fs = boost::filesystem;
 
@@ -496,7 +497,7 @@ static arrow::Status try_dlopen(
 namespace arrow {
 namespace io {
 
-Status ConnectLibHdfs() {
+Status ARROW_EXPORT ConnectLibHdfs() {
   static std::mutex lock;
   std::lock_guard<std::mutex> guard(lock);
 
diff --git a/cpp/src/arrow/io/symbols.map b/cpp/src/arrow/io/symbols.map
new file mode 100644
index 0000000000000..b4ad98cd7f2d0
--- /dev/null
+++ b/cpp/src/arrow/io/symbols.map
@@ -0,0 +1,18 @@
+{
+  # Symbols marked as 'local' are not exported by the DSO and thus may not
+  # be used by client applications.
+  local:
+    # devtoolset / static-libstdc++ symbols
+    __cxa_*;
+
+    extern "C++" {
+      # boost
+      boost::*;
+
+      # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
+      # links c++11 symbols into binaries so that the result may be executed on
+      # a system with an older libstdc++ which doesn't include the necessary
+      # c++11 symbols.
+      std::*;
+    };
+};
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 383684f42f952..82634169ed925 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -48,4 +48,4 @@ add_custom_command(
 )
 
 add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES})
-add_dependencies(arrow metadata_fbs)
+add_dependencies(arrow_objlib metadata_fbs)
diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
index f00bb53c0848f..00f19b354e379 100644
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -25,8 +25,8 @@ set(PARQUET_SRCS
 )
 
 set(PARQUET_LIBS
-  arrow
-  ${PARQUET_SHARED_LIB}
+  arrow_shared
+  parquet_shared
 )
 
 add_library(arrow_parquet SHARED
diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
index 572cae16e58c0..bfc27d26d63a1 100644
--- a/cpp/src/arrow/parquet/parquet-io-test.cc
+++ b/cpp/src/arrow/parquet/parquet-io-test.cc
@@ -411,7 +411,7 @@ class TestPrimitiveParquetIO : public TestParquetIO<TestType> {
  public:
   typedef typename TestType::c_type T;
 
-  void TestFile(std::vector<T>& values, int num_chunks,
+  void MakeTestFile(std::vector<T>& values, int num_chunks,
       std::unique_ptr<ParquetFileReader>* file_reader) {
     std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
     std::unique_ptr<ParquetFileWriter> file_writer = this->MakeWriter(schema);
@@ -435,10 +435,10 @@ class TestPrimitiveParquetIO : public TestParquetIO<TestType> {
     *file_reader = this->ReaderFromSink();
   }
 
-  void TestSingleColumnRequiredTableRead(int num_chunks) {
+  void CheckSingleColumnRequiredTableRead(int num_chunks) {
     std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
     std::unique_ptr<ParquetFileReader> file_reader;
-    ASSERT_NO_THROW(TestFile(values, num_chunks, &file_reader));
+    ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader));
 
     std::shared_ptr<Table> out;
     this->ReadTableFromFile(std::move(file_reader), &out);
@@ -450,10 +450,10 @@ class TestPrimitiveParquetIO : public TestParquetIO<TestType> {
     ExpectArray<TestType>(values.data(), chunked_array->chunk(0).get());
   }
 
-  void TestSingleColumnRequiredRead(int num_chunks) {
+  void CheckSingleColumnRequiredRead(int num_chunks) {
     std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
     std::unique_ptr<ParquetFileReader> file_reader;
-    ASSERT_NO_THROW(TestFile(values, num_chunks, &file_reader));
+    ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader));
 
     std::shared_ptr<Array> out;
     this->ReadSingleColumnFile(std::move(file_reader), &out);
@@ -469,19 +469,19 @@ typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type
 TYPED_TEST_CASE(TestPrimitiveParquetIO, PrimitiveTestTypes);
 
 TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredRead) {
-  this->TestSingleColumnRequiredRead(1);
+  this->CheckSingleColumnRequiredRead(1);
 }
 
 TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredTableRead) {
-  this->TestSingleColumnRequiredTableRead(1);
+  this->CheckSingleColumnRequiredTableRead(1);
 }
 
 TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedRead) {
-  this->TestSingleColumnRequiredRead(4);
+  this->CheckSingleColumnRequiredRead(4);
 }
 
 TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
-  this->TestSingleColumnRequiredTableRead(4);
+  this->CheckSingleColumnRequiredTableRead(4);
 }
 
 }  // namespace parquet
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
index 7b05665b230f0..c7c400e957343 100644
--- a/cpp/src/arrow/parquet/reader.cc
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -213,7 +213,7 @@ Status FlatColumnReader::Impl::ReadNonNullableBatch(typename ParquetType::c_type
   using ParquetCType = typename ParquetType::c_type;
 
   DCHECK(builder);
-  const ArrowCType* values_ptr;
+  const ArrowCType* values_ptr = nullptr;
   RETURN_NOT_OK(
       (ConvertPhysicalType<ParquetCType, ArrowCType>(values, values_read, &values_ptr)));
   RETURN_NOT_OK(builder->Append(values_ptr, values_read));
diff --git a/cpp/src/arrow/parquet/reader.h b/cpp/src/arrow/parquet/reader.h
index db7a15753d8e8..2c8a9dfd025f0 100644
--- a/cpp/src/arrow/parquet/reader.h
+++ b/cpp/src/arrow/parquet/reader.h
@@ -23,6 +23,8 @@
 #include "parquet/api/reader.h"
 #include "parquet/api/schema.h"
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Array;
@@ -77,7 +79,7 @@ class FlatColumnReader;
 //
 // This is additionally complicated "chunky" repeated fields or very large byte
 // arrays
-class FileReader {
+class ARROW_EXPORT FileReader {
  public:
   FileReader(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader);
 
@@ -107,7 +109,7 @@ class FileReader {
 //
 // We also do not expose any internal Parquet details, such as row groups. This
 // might change in the future.
-class FlatColumnReader {
+class ARROW_EXPORT FlatColumnReader {
  public:
   virtual ~FlatColumnReader();
 
diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h
index 39bee059522a3..88b5977d223a4 100644
--- a/cpp/src/arrow/parquet/schema.h
+++ b/cpp/src/arrow/parquet/schema.h
@@ -25,6 +25,7 @@
 
 #include "arrow/schema.h"
 #include "arrow/type.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -32,15 +33,16 @@ class Status;
 
 namespace parquet {
 
-Status NodeToField(const ::parquet::schema::NodePtr& node, std::shared_ptr<Field>* out);
+Status ARROW_EXPORT NodeToField(
+    const ::parquet::schema::NodePtr& node, std::shared_ptr<Field>* out);
 
-Status FromParquetSchema(
+Status ARROW_EXPORT FromParquetSchema(
     const ::parquet::SchemaDescriptor* parquet_schema, std::shared_ptr<Schema>* out);
 
-Status FieldToNode(const std::shared_ptr<Field>& field,
+Status ARROW_EXPORT FieldToNode(const std::shared_ptr<Field>& field,
     const ::parquet::WriterProperties& properties, ::parquet::schema::NodePtr* out);
 
-Status ToParquetSchema(const Schema* arrow_schema,
+Status ARROW_EXPORT ToParquetSchema(const Schema* arrow_schema,
     const ::parquet::WriterProperties& properties,
     std::shared_ptr<::parquet::SchemaDescriptor>* out);
 
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
index 63449bb20b1a1..0139edd3bb8d9 100644
--- a/cpp/src/arrow/parquet/writer.cc
+++ b/cpp/src/arrow/parquet/writer.cc
@@ -118,7 +118,7 @@ Status FileWriter::Impl::TypedWriteBatch(::parquet::ColumnWriter* column_writer,
       reinterpret_cast<::parquet::TypedColumnWriter<ParquetType>*>(column_writer);
   if (writer->descr()->max_definition_level() == 0) {
     // no nulls, just dump the data
-    const ParquetCType* data_writer_ptr;
+    const ParquetCType* data_writer_ptr = nullptr;
     RETURN_NOT_OK((ConvertPhysicalType<ArrowCType, ParquetCType>(
         data_ptr, length, &data_writer_ptr)));
     PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, nullptr, nullptr, data_writer_ptr));
@@ -128,7 +128,7 @@ Status FileWriter::Impl::TypedWriteBatch(::parquet::ColumnWriter* column_writer,
         reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
     if (data->null_count() == 0) {
       std::fill(def_levels_ptr, def_levels_ptr + length, 1);
-      const ParquetCType* data_writer_ptr;
+      const ParquetCType* data_writer_ptr = nullptr;
       RETURN_NOT_OK((ConvertPhysicalType<ArrowCType, ParquetCType>(
           data_ptr, length, &data_writer_ptr)));
       PARQUET_CATCH_NOT_OK(
diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h
index cfd80d80b7997..45d0fd59868e5 100644
--- a/cpp/src/arrow/parquet/writer.h
+++ b/cpp/src/arrow/parquet/writer.h
@@ -23,6 +23,8 @@
 #include "parquet/api/schema.h"
 #include "parquet/api/writer.h"
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Array;
@@ -40,7 +42,7 @@ namespace parquet {
  *  Start a new RowGroup/Chunk with NewRowGroup
  *  Write column-by-column the whole column chunk
  */
-class FileWriter {
+class ARROW_EXPORT FileWriter {
  public:
   FileWriter(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer);
 
@@ -62,7 +64,7 @@ class FileWriter {
  *
  * The table shall only consist of nullable, non-repeated columns of primitive type.
  */
-Status WriteFlatTable(const Table* table, MemoryPool* pool,
+Status ARROW_EXPORT WriteFlatTable(const Table* table, MemoryPool* pool,
     const std::shared_ptr<::parquet::OutputStream>& sink, int64_t chunk_size,
     const std::shared_ptr<::parquet::WriterProperties>& properties =
         ::parquet::default_writer_properties());
diff --git a/cpp/src/arrow/schema.h b/cpp/src/arrow/schema.h
index a8b0d8444ac92..4301968e01578 100644
--- a/cpp/src/arrow/schema.h
+++ b/cpp/src/arrow/schema.h
@@ -22,11 +22,13 @@
 #include <string>
 #include <vector>
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 struct Field;
 
-class Schema {
+class ARROW_EXPORT Schema {
  public:
   explicit Schema(const std::vector<std::shared_ptr<Field>>& fields);
 
diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map
new file mode 100644
index 0000000000000..2ca8d7306105f
--- /dev/null
+++ b/cpp/src/arrow/symbols.map
@@ -0,0 +1,15 @@
+{
+  # Symbols marked as 'local' are not exported by the DSO and thus may not
+  # be used by client applications.
+  local:
+    # devtoolset / static-libstdc++ symbols
+    __cxa_*;
+
+    extern "C++" {
+      # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
+      # links c++11 symbols into binaries so that the result may be executed on
+      # a system with an older libstdc++ which doesn't include the necessary
+      # c++11 symbols.
+      std::*;
+    };
+};
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 756b2a19593f4..2088fdf0b6415 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -23,6 +23,8 @@
 #include <string>
 #include <vector>
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Array;
@@ -33,7 +35,7 @@ class Status;
 // A row batch is a simpler and more rigid table data structure intended for
 // use primarily in shared memory IPC. It contains a schema (metadata) and a
 // corresponding vector of equal-length Arrow arrays
-class RowBatch {
+class ARROW_EXPORT RowBatch {
  public:
   // num_rows is a parameter to allow for row batches of a particular size not
   // having any materialized columns. Each array should have the same length as
@@ -63,7 +65,7 @@ class RowBatch {
 };
 
 // Immutable container of fixed-length columns conforming to a particular schema
-class Table {
+class ARROW_EXPORT Table {
  public:
   // If columns is zero-length, the table's number of rows is zero
   Table(const std::string& name, const std::shared_ptr<Schema>& schema,
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 8fb41211ba945..4cb37fd1dead8 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -24,6 +24,7 @@
 #include <vector>
 
 #include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -101,7 +102,7 @@ struct Type {
 
 struct Field;
 
-struct DataType {
+struct ARROW_EXPORT DataType {
   Type::type type;
 
   std::vector<std::shared_ptr<Field>> children_;
@@ -133,7 +134,7 @@ typedef std::shared_ptr<DataType> TypePtr;
 
 // A field is a piece of metadata that includes (for now) a name and a data
 // type
-struct Field {
+struct ARROW_EXPORT Field {
   // Field name
   std::string name;
 
@@ -163,7 +164,7 @@ struct Field {
 typedef std::shared_ptr<Field> FieldPtr;
 
 template <typename Derived>
-struct PrimitiveType : public DataType {
+struct ARROW_EXPORT PrimitiveType : public DataType {
   PrimitiveType() : DataType(Derived::type_enum) {}
 
   std::string ToString() const override;
@@ -185,55 +186,55 @@ inline std::string PrimitiveType<Derived>::ToString() const {
                                                            \
   static const char* name() { return NAME; }
 
-struct NullType : public PrimitiveType<NullType> {
+struct ARROW_EXPORT NullType : public PrimitiveType<NullType> {
   PRIMITIVE_DECL(NullType, void, NA, 0, "null");
 };
 
-struct BooleanType : public PrimitiveType<BooleanType> {
+struct ARROW_EXPORT BooleanType : public PrimitiveType<BooleanType> {
   PRIMITIVE_DECL(BooleanType, uint8_t, BOOL, 1, "bool");
 };
 
-struct UInt8Type : public PrimitiveType<UInt8Type> {
+struct ARROW_EXPORT UInt8Type : public PrimitiveType<UInt8Type> {
   PRIMITIVE_DECL(UInt8Type, uint8_t, UINT8, 1, "uint8");
 };
 
-struct Int8Type : public PrimitiveType<Int8Type> {
+struct ARROW_EXPORT Int8Type : public PrimitiveType<Int8Type> {
   PRIMITIVE_DECL(Int8Type, int8_t, INT8, 1, "int8");
 };
 
-struct UInt16Type : public PrimitiveType<UInt16Type> {
+struct ARROW_EXPORT UInt16Type : public PrimitiveType<UInt16Type> {
   PRIMITIVE_DECL(UInt16Type, uint16_t, UINT16, 2, "uint16");
 };
 
-struct Int16Type : public PrimitiveType<Int16Type> {
+struct ARROW_EXPORT Int16Type : public PrimitiveType<Int16Type> {
   PRIMITIVE_DECL(Int16Type, int16_t, INT16, 2, "int16");
 };
 
-struct UInt32Type : public PrimitiveType<UInt32Type> {
+struct ARROW_EXPORT UInt32Type : public PrimitiveType<UInt32Type> {
   PRIMITIVE_DECL(UInt32Type, uint32_t, UINT32, 4, "uint32");
 };
 
-struct Int32Type : public PrimitiveType<Int32Type> {
+struct ARROW_EXPORT Int32Type : public PrimitiveType<Int32Type> {
   PRIMITIVE_DECL(Int32Type, int32_t, INT32, 4, "int32");
 };
 
-struct UInt64Type : public PrimitiveType<UInt64Type> {
+struct ARROW_EXPORT UInt64Type : public PrimitiveType<UInt64Type> {
   PRIMITIVE_DECL(UInt64Type, uint64_t, UINT64, 8, "uint64");
 };
 
-struct Int64Type : public PrimitiveType<Int64Type> {
+struct ARROW_EXPORT Int64Type : public PrimitiveType<Int64Type> {
   PRIMITIVE_DECL(Int64Type, int64_t, INT64, 8, "int64");
 };
 
-struct FloatType : public PrimitiveType<FloatType> {
+struct ARROW_EXPORT FloatType : public PrimitiveType<FloatType> {
   PRIMITIVE_DECL(FloatType, float, FLOAT, 4, "float");
 };
 
-struct DoubleType : public PrimitiveType<DoubleType> {
+struct ARROW_EXPORT DoubleType : public PrimitiveType<DoubleType> {
   PRIMITIVE_DECL(DoubleType, double, DOUBLE, 8, "double");
 };
 
-struct ListType : public DataType {
+struct ARROW_EXPORT ListType : public DataType {
   // List can contain any other logical value type
   explicit ListType(const std::shared_ptr<DataType>& value_type)
       : ListType(value_type, Type::LIST) {}
@@ -260,7 +261,7 @@ struct ListType : public DataType {
 };
 
 // BinaryType type is reprsents lists of 1-byte values.
-struct BinaryType : public ListType {
+struct ARROW_EXPORT BinaryType : public ListType {
   BinaryType() : BinaryType(Type::BINARY) {}
   static char const* name() { return "binary"; }
   std::string ToString() const override;
@@ -272,7 +273,7 @@ struct BinaryType : public ListType {
 };
 
 // UTF encoded strings
-struct StringType : public BinaryType {
+struct ARROW_EXPORT StringType : public BinaryType {
   StringType() : BinaryType(Type::STRING) {}
 
   static char const* name() { return "string"; }
@@ -283,7 +284,7 @@ struct StringType : public BinaryType {
   explicit StringType(Type::type logical_type) : BinaryType(logical_type) {}
 };
 
-struct StructType : public DataType {
+struct ARROW_EXPORT StructType : public DataType {
   explicit StructType(const std::vector<std::shared_ptr<Field>>& fields)
       : DataType(Type::STRUCT) {
     children_ = fields;
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index d0370840ca108..afdadbe079013 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -21,6 +21,9 @@
 #include <cstdint>
 #include <memory>
 #include <vector>
+
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Array;
@@ -31,18 +34,18 @@ struct Field;
 class MemoryPool;
 class Status;
 
-Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     std::shared_ptr<ArrayBuilder>* out);
 
 // Create new arrays for logical types that are backed by primitive arrays.
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
+Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
 
 // Create new list arrays for logical types that are backed by ListArrays (e.g. list of
 // primitives and strings)
 // TODO(emkornfield) split up string vs list?
-Status MakeListArray(const std::shared_ptr<DataType>& type, int32_t length,
+Status ARROW_EXPORT MakeListArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::shared_ptr<Buffer>& offests, const std::shared_ptr<Array>& values,
     int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
     std::shared_ptr<Array>* out);
diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h
index 598df3ef70d2d..6c497c597d987 100644
--- a/cpp/src/arrow/types/decimal.h
+++ b/cpp/src/arrow/types/decimal.h
@@ -21,10 +21,11 @@
 #include <string>
 
 #include "arrow/type.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
-struct DecimalType : public DataType {
+struct ARROW_EXPORT DecimalType : public DataType {
   explicit DecimalType(int precision_, int scale_)
       : DataType(Type::DECIMAL), precision(precision_), scale(scale_) {}
   int precision;
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 2f6f85d66ca60..f3894510d091a 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -31,12 +31,13 @@
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/status.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class MemoryPool;
 
-class ListArray : public Array {
+class ARROW_EXPORT ListArray : public Array {
  public:
   ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
       const ArrayPtr& values, int32_t null_count = 0,
@@ -96,7 +97,7 @@ class ListArray : public Array {
 // represent multiple different logical types.  If no logical type is provided
 // at construction time, the class defaults to List<T> where t is taken from the
 // value_builder/values that the object is constructed with.
-class ListBuilder : public ArrayBuilder {
+class ARROW_EXPORT ListBuilder : public ArrayBuilder {
  public:
   // Use this constructor to incrementally build the value array along with offsets and
   // null bitmap.
@@ -116,6 +117,8 @@ class ListBuilder : public ArrayBuilder {
         offset_builder_(pool),
         values_(values) {}
 
+  virtual ~ListBuilder() {}
+
   Status Init(int32_t elements) override {
     DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
     RETURN_NOT_OK(ArrayBuilder::Init(elements));
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index f1ec417d51014..18f954adc0894 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -29,6 +29,7 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -36,7 +37,7 @@ class MemoryPool;
 
 // Base class for fixed-size logical types.  See MakePrimitiveArray
 // (types/construct.h) for constructing a specific subclass.
-class PrimitiveArray : public Array {
+class ARROW_EXPORT PrimitiveArray : public Array {
  public:
   virtual ~PrimitiveArray() {}
 
@@ -53,7 +54,7 @@ class PrimitiveArray : public Array {
 };
 
 #define NUMERIC_ARRAY_DECL(NAME, TypeClass, T)                                         \
-  class NAME : public PrimitiveArray {                                                 \
+  class ARROW_EXPORT NAME : public PrimitiveArray {                                    \
    public:                                                                             \
     using value_type = T;                                                              \
                                                                                        \
@@ -102,7 +103,7 @@ NUMERIC_ARRAY_DECL(FloatArray, FloatType, float);
 NUMERIC_ARRAY_DECL(DoubleArray, DoubleType, double);
 
 template <typename Type>
-class PrimitiveBuilder : public ArrayBuilder {
+class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
  public:
   typedef typename Type::c_type value_type;
 
@@ -149,7 +150,7 @@ class PrimitiveBuilder : public ArrayBuilder {
 };
 
 template <typename T>
-class NumericBuilder : public PrimitiveBuilder<T> {
+class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
  public:
   using typename PrimitiveBuilder<T>::value_type;
   using PrimitiveBuilder<T>::PrimitiveBuilder;
@@ -262,7 +263,7 @@ typedef NumericBuilder<Int64Type> Int64Builder;
 typedef NumericBuilder<FloatType> FloatBuilder;
 typedef NumericBuilder<DoubleType> DoubleBuilder;
 
-class BooleanArray : public PrimitiveArray {
+class ARROW_EXPORT BooleanArray : public PrimitiveArray {
  public:
   BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
@@ -288,7 +289,7 @@ struct type_traits<BooleanType> {
   }
 };
 
-class BooleanBuilder : public PrimitiveBuilder<BooleanType> {
+class ARROW_EXPORT BooleanBuilder : public PrimitiveBuilder<BooleanType> {
  public:
   explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type)
       : PrimitiveBuilder<BooleanType>(pool, type) {}
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index a141fc113211a..6807b00e8ca99 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -115,7 +115,7 @@ TEST_F(TestStringContainer, TestListFunctions) {
   int pos = 0;
   for (size_t i = 0; i < expected_.size(); ++i) {
     ASSERT_EQ(pos, strings_->value_offset(i));
-    ASSERT_EQ(expected_[i].size(), strings_->value_length(i));
+    ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
     pos += expected_[i].size();
   }
 }
@@ -189,7 +189,7 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
       ASSERT_FALSE(result_->IsNull(i));
       result_->GetValue(i, &length);
       ASSERT_EQ(pos, result_->offset(i));
-      ASSERT_EQ(strings[i % N].size(), length);
+      ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
       ASSERT_EQ(strings[i % N], result_->GetString(i));
 
       pos += length;
@@ -267,7 +267,7 @@ TEST_F(TestBinaryContainer, TestListFunctions) {
   int pos = 0;
   for (size_t i = 0; i < expected_.size(); ++i) {
     ASSERT_EQ(pos, strings_->value_offset(i));
-    ASSERT_EQ(expected_[i].size(), strings_->value_length(i));
+    ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
     pos += expected_[i].size();
   }
 }
@@ -339,7 +339,7 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
     } else {
       ASSERT_FALSE(result_->IsNull(i));
       const uint8_t* vals = result_->GetValue(i, &length);
-      ASSERT_EQ(strings[i % N].size(), length);
+      ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
       ASSERT_EQ(0, std::memcmp(vals, strings[i % N].data(), length));
     }
   }
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index da02c7d1d8a9e..2f0037024c78d 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -61,6 +61,15 @@ Status StringArray::Validate() const {
   return BinaryArray::Validate();
 }
 
-TypePtr BinaryBuilder::value_type_ = TypePtr(new UInt8Type());
+// This used to be a static member variable of BinaryBuilder, but it can cause
+// valgrind to report a (spurious?) memory leak when needed in other shared
+// libraries. The problem came up while adding explicit visibility to libarrow
+// and libarrow_parquet
+static TypePtr kBinaryValueType = TypePtr(new UInt8Type());
+
+BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
+    : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, kBinaryValueType), type) {
+  byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
+}
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index b3c00d298b35c..bab0c58f617b2 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -28,13 +28,14 @@
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
 #include "arrow/util/status.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class Buffer;
 class MemoryPool;
 
-class BinaryArray : public ListArray {
+class ARROW_EXPORT BinaryArray : public ListArray {
  public:
   BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values, int32_t null_count = 0,
@@ -62,7 +63,7 @@ class BinaryArray : public ListArray {
   const uint8_t* raw_bytes_;
 };
 
-class StringArray : public BinaryArray {
+class ARROW_EXPORT StringArray : public BinaryArray {
  public:
   StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
       const ArrayPtr& values, int32_t null_count = 0,
@@ -87,12 +88,10 @@ class StringArray : public BinaryArray {
 };
 
 // BinaryBuilder : public ListBuilder
-class BinaryBuilder : public ListBuilder {
+class ARROW_EXPORT BinaryBuilder : public ListBuilder {
  public:
-  explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type)
-      : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, value_type_), type) {
-    byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
-  }
+  explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type);
+  virtual ~BinaryBuilder() {}
 
   Status Append(const uint8_t* value, int32_t length) {
     RETURN_NOT_OK(ListBuilder::Append());
@@ -105,11 +104,10 @@ class BinaryBuilder : public ListBuilder {
 
  protected:
   UInt8Builder* byte_builder_;
-  static TypePtr value_type_;
 };
 
 // String builder
-class StringBuilder : public BinaryBuilder {
+class ARROW_EXPORT StringBuilder : public BinaryBuilder {
  public:
   explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
       : BinaryBuilder(pool, type) {}
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index d2bd2971d0438..ccf5a52dc831c 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -116,7 +116,7 @@ class TestStructBuilder : public TestBuilder {
     ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
 
     builder_ = std::dynamic_pointer_cast<StructBuilder>(tmp);
-    ASSERT_EQ(2, builder_->field_builders().size());
+    ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
   }
 
   void Done() { result_ = std::dynamic_pointer_cast<StructArray>(builder_->Finish()); }
@@ -132,7 +132,7 @@ class TestStructBuilder : public TestBuilder {
 TEST_F(TestStructBuilder, TestAppendNull) {
   ASSERT_OK(builder_->AppendNull());
   ASSERT_OK(builder_->AppendNull());
-  ASSERT_EQ(2, builder_->field_builders().size());
+  ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
 
   ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
   ASSERT_OK(list_vb->AppendNull());
@@ -148,7 +148,7 @@ TEST_F(TestStructBuilder, TestAppendNull) {
 
   ASSERT_OK(result_->Validate());
 
-  ASSERT_EQ(2, result_->fields().size());
+  ASSERT_EQ(2, static_cast<int>(result_->fields().size()));
   ASSERT_EQ(2, result_->length());
   ASSERT_EQ(2, result_->field(0)->length());
   ASSERT_EQ(2, result_->field(1)->length());
@@ -174,7 +174,7 @@ TEST_F(TestStructBuilder, TestBasics) {
   ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
   Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
   Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-  ASSERT_EQ(2, builder_->field_builders().size());
+  ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
 
   EXPECT_OK(builder_->Resize(list_lengths.size()));
   EXPECT_OK(char_vb->Resize(list_values.size()));
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index 78afd29eb8df5..63955eb31bb7d 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -25,10 +25,11 @@
 #include "arrow/type.h"
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
-class StructArray : public Array {
+class ARROW_EXPORT StructArray : public Array {
  public:
   StructArray(const TypePtr& type, int32_t length, std::vector<ArrayPtr>& field_arrays,
       int32_t null_count = 0, std::shared_ptr<Buffer> null_bitmap = nullptr)
@@ -64,7 +65,7 @@ class StructArray : public Array {
 // Append, Resize and Reserve methods are acting on StructBuilder.
 // Please make sure all these methods of all child-builders' are consistently
 // called to maintain data-structure consistency.
-class StructBuilder : public ArrayBuilder {
+class ARROW_EXPORT StructBuilder : public ArrayBuilder {
  public:
   StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
       const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders)
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index d2a4b091fada5..4e941fb5f5cec 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -27,6 +27,7 @@ install(FILES
   macros.h
   memory-pool.h
   status.h
+  visibility.h
   DESTINATION include/arrow/util)
 
 #######################################
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index f845d67761fe4..1aeebc69b4e14 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -26,6 +26,7 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/status.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -41,7 +42,7 @@ class Status;
 // Capacity is the number of bytes that where allocated for the buffer in
 // total.
 // The following invariant is always true: Size < Capacity
-class Buffer : public std::enable_shared_from_this<Buffer> {
+class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
  public:
   Buffer(const uint8_t* data, int64_t size) : data_(data), size_(size), capacity_(size) {}
   virtual ~Buffer();
@@ -95,7 +96,7 @@ class Buffer : public std::enable_shared_from_this<Buffer> {
 };
 
 // A Buffer whose contents can be mutated. May or may not own its data.
-class MutableBuffer : public Buffer {
+class ARROW_EXPORT MutableBuffer : public Buffer {
  public:
   MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
     mutable_data_ = data;
@@ -112,7 +113,7 @@ class MutableBuffer : public Buffer {
   uint8_t* mutable_data_;
 };
 
-class ResizableBuffer : public MutableBuffer {
+class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
  public:
   // Change buffer reported size to indicated size, allocating memory if
   // necessary.  This will ensure that the capacity of the buffer is a multiple
@@ -129,7 +130,7 @@ class ResizableBuffer : public MutableBuffer {
 };
 
 // A Buffer whose lifetime is tied to a particular MemoryPool
-class PoolBuffer : public ResizableBuffer {
+class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
  public:
   explicit PoolBuffer(MemoryPool* pool = nullptr);
   virtual ~PoolBuffer();
@@ -145,7 +146,8 @@ static constexpr int64_t MIN_BUFFER_CAPACITY = 1024;
 
 class BufferBuilder {
  public:
-  explicit BufferBuilder(MemoryPool* pool) : pool_(pool), capacity_(0), size_(0) {}
+  explicit BufferBuilder(MemoryPool* pool)
+      : pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
 
   // Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
   Status Resize(int32_t elements) {
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
index 4ab9736c2b468..8e7dfd60baa62 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -31,7 +31,7 @@ TEST(DefaultMemoryPool, MemoryTracking) {
 
   uint8_t* data;
   ASSERT_OK(pool->Allocate(100, &data));
-  EXPECT_EQ(0, reinterpret_cast<uint64_t>(data) % 64);
+  EXPECT_EQ(static_cast<uint64_t>(0), reinterpret_cast<uint64_t>(data) % 64);
   ASSERT_EQ(100, pool->bytes_allocated());
 
   pool->Free(data, 100);
diff --git a/cpp/src/arrow/util/memory-pool.h b/cpp/src/arrow/util/memory-pool.h
index 824c7248e2e86..4c1d699addd50 100644
--- a/cpp/src/arrow/util/memory-pool.h
+++ b/cpp/src/arrow/util/memory-pool.h
@@ -20,11 +20,13 @@
 
 #include <cstdint>
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Status;
 
-class MemoryPool {
+class ARROW_EXPORT MemoryPool {
  public:
   virtual ~MemoryPool();
 
@@ -34,7 +36,7 @@ class MemoryPool {
   virtual int64_t bytes_allocated() const = 0;
 };
 
-MemoryPool* default_memory_pool();
+ARROW_EXPORT MemoryPool* default_memory_pool();
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/util/status.cc
index d194ed5572f52..8dd07d0d064e7 100644
--- a/cpp/src/arrow/util/status.cc
+++ b/cpp/src/arrow/util/status.cc
@@ -58,6 +58,9 @@ std::string Status::CodeAsString() const {
     case StatusCode::NotImplemented:
       type = "NotImplemented";
       break;
+    default:
+      type = "Unknown";
+      break;
   }
   return std::string(type);
 }
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
index d1a742500084c..6ba2035bcd3a4 100644
--- a/cpp/src/arrow/util/status.h
+++ b/cpp/src/arrow/util/status.h
@@ -19,6 +19,8 @@
 #include <cstring>
 #include <string>
 
+#include "arrow/util/visibility.h"
+
 // Return the given status if it is not OK.
 #define ARROW_RETURN_NOT_OK(s)   \
   do {                           \
@@ -82,7 +84,7 @@ enum class StatusCode : char {
   NotImplemented = 10,
 };
 
-class Status {
+class ARROW_EXPORT Status {
  public:
   // Create a success status.
   Status() : state_(NULL) {}
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
new file mode 100644
index 0000000000000..b197c198297c8
--- /dev/null
+++ b/cpp/src/arrow/util/visibility.h
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_VISIBILITY_H
+#define ARROW_UTIL_VISIBILITY_H
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#define ARROW_EXPORT __declspec(dllexport)
+#else  // Not Windows
+#ifndef ARROW_EXPORT
+#define ARROW_EXPORT __attribute__((visibility("default")))
+#endif
+#ifndef ARROW_NO_EXPORT
+#define ARROW_NO_EXPORT __attribute__((visibility("hidden")))
+#endif
+#endif  // Non-Windows
+
+#endif  // ARROW_UTIL_VISIBILITY_H
diff --git a/python/conda.recipe/build.sh b/python/conda.recipe/build.sh
index a164c1af51833..f32710073c7c2 100644
--- a/python/conda.recipe/build.sh
+++ b/python/conda.recipe/build.sh
@@ -19,13 +19,14 @@ if [ "$(uname)" == "Darwin" ]; then
   export MACOSX_DEPLOYMENT_TARGET=10.7
 fi
 
-echo Setting the compiler...
-if [ `uname` == Linux ]; then
-  EXTRA_CMAKE_ARGS=-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++
-elif [ `uname` == Darwin ]; then
-  EXTRA_CMAKE_ARGS=
-fi
+# echo Setting the compiler...
+# if [ `uname` == Linux ]; then
+#   EXTRA_CMAKE_ARGS=-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++
+# elif [ `uname` == Darwin ]; then
+#   EXTRA_CMAKE_ARGS=
+# fi
 
 cd ..
-$PYTHON setup.py build_ext --extra-cmake-args=$EXTRA_CMAKE_ARGS || exit 1
+# $PYTHON setup.py build_ext --extra-cmake-args=$EXTRA_CMAKE_ARGS || exit 1
+$PYTHON setup.py build_ext || exit 1
 $PYTHON setup.py install || exit 1
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
index 88869c2048003..4e997e31dd690 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -28,6 +28,7 @@
 #include <arrow/type.h>
 
 #include "pyarrow/common.h"
+#include "pyarrow/visibility.h"
 
 namespace arrow { class Array; }
 
@@ -35,6 +36,7 @@ namespace pyarrow {
 
 class Status;
 
+PYARROW_EXPORT
 Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out);
 
 } // namespace pyarrow
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index 17922349de6c1..c3377685bcce9 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -25,6 +25,8 @@
 
 #include <memory>
 
+#include "pyarrow/visibility.h"
+
 namespace arrow {
 
 class Array;
@@ -36,12 +38,15 @@ namespace pyarrow {
 
 class Status;
 
+PYARROW_EXPORT
 Status ArrowToPandas(const std::shared_ptr<arrow::Column>& col, PyObject* py_ref,
     PyObject** out);
 
+PYARROW_EXPORT
 Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     std::shared_ptr<arrow::Array>* out);
 
+PYARROW_EXPORT
 Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
     std::shared_ptr<arrow::Array>* out);
 
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 0211e8948f2f7..fb0ba3e482296 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -22,6 +22,8 @@
 
 #include "arrow/util/buffer.h"
 
+#include "pyarrow/visibility.h"
+
 namespace arrow { class MemoryPool; }
 
 namespace pyarrow {
@@ -94,9 +96,9 @@ struct PyObjectStringify {
     return Status::UnknownError(message);           \
   }
 
-arrow::MemoryPool* GetMemoryPool();
+PYARROW_EXPORT arrow::MemoryPool* GetMemoryPool();
 
-class NumPyBuffer : public arrow::Buffer {
+class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
  public:
   NumPyBuffer(PyArrayObject* arr) :
       Buffer(nullptr, 0) {
diff --git a/python/src/pyarrow/config.h b/python/src/pyarrow/config.h
index 48ae715d842b1..82936b1a5f317 100644
--- a/python/src/pyarrow/config.h
+++ b/python/src/pyarrow/config.h
@@ -21,6 +21,7 @@
 #include <Python.h>
 
 #include "pyarrow/numpy_interop.h"
+#include "pyarrow/visibility.h"
 
 #if PY_MAJOR_VERSION >= 3
   #define PyString_Check PyUnicode_Check
@@ -28,10 +29,13 @@
 
 namespace pyarrow {
 
+PYARROW_EXPORT
 extern PyObject* numpy_nan;
 
+PYARROW_EXPORT
 void pyarrow_init();
 
+PYARROW_EXPORT
 void pyarrow_set_numpy_nan(PyObject* obj);
 
 } // namespace pyarrow
diff --git a/python/src/pyarrow/helpers.h b/python/src/pyarrow/helpers.h
index ec42bb31d3b9b..fa9c713b0c22c 100644
--- a/python/src/pyarrow/helpers.h
+++ b/python/src/pyarrow/helpers.h
@@ -21,6 +21,8 @@
 #include <arrow/api.h>
 #include <memory>
 
+#include "pyarrow/visibility.h"
+
 namespace pyarrow {
 
 using arrow::DataType;
@@ -40,6 +42,7 @@ extern const std::shared_ptr<arrow::FloatType> FLOAT;
 extern const std::shared_ptr<arrow::DoubleType> DOUBLE;
 extern const std::shared_ptr<arrow::StringType> STRING;
 
+PYARROW_EXPORT
 std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
 
 } // namespace pyarrow
diff --git a/python/src/pyarrow/status.h b/python/src/pyarrow/status.h
index cb8c8add210e4..67cd66c58eeb3 100644
--- a/python/src/pyarrow/status.h
+++ b/python/src/pyarrow/status.h
@@ -17,6 +17,8 @@
 #include <cstring>
 #include <string>
 
+#include "pyarrow/visibility.h"
+
 namespace pyarrow {
 
 #define PY_RETURN_NOT_OK(s) do {                \
@@ -38,7 +40,7 @@ enum class StatusCode: char {
   UnknownError = 10
 };
 
-class Status {
+class PYARROW_EXPORT Status {
  public:
   // Create a success status.
   Status() : state_(NULL) { }
diff --git a/python/src/pyarrow/visibility.h b/python/src/pyarrow/visibility.h
new file mode 100644
index 0000000000000..9f0c13b4b2083
--- /dev/null
+++ b/python/src/pyarrow/visibility.h
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_VISIBILITY_H
+#define PYARROW_VISIBILITY_H
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#define PYARROW_EXPORT __declspec(dllexport)
+#else  // Not Windows
+#ifndef PYARROW_EXPORT
+#define PYARROW_EXPORT __attribute__((visibility("default")))
+#endif
+#ifndef PYARROW_NO_EXPORT
+#define PYARROW_NO_EXPORT __attribute__((visibility("hidden")))
+#endif
+#endif  // Non-Windows
+
+#endif  // PYARROW_VISIBILITY_H

From ff6132f8a1c2a98cf7c94ae327342c8b38aecb18 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Mon, 11 Jul 2016 22:58:57 -0700
Subject: [PATCH 0096/1644] ARROW-237: Implement parquet-cpp's abstract IO
 interfaces for memory allocation and file reading

Part of ARROW-227 and ARROW-236

Author: Wes McKinney <wesm@apache.org>

Closes #101 from wesm/ARROW-237 and squashes the following commits:

00c8211 [Wes McKinney] Draft implementations of parquet-cpp allocator and read-only file interfaces
---
 cpp/src/arrow/io/hdfs-io-test.cc              |   2 +-
 cpp/src/arrow/io/hdfs.cc                      |  16 +-
 cpp/src/arrow/io/hdfs.h                       |   8 +-
 cpp/src/arrow/io/interfaces.h                 |  14 +-
 cpp/src/arrow/parquet/CMakeLists.txt          |   5 +
 cpp/src/arrow/parquet/io.cc                   |  94 ++++
 cpp/src/arrow/parquet/io.h                    |  80 +++
 cpp/src/arrow/parquet/parquet-io-test.cc      | 511 ++++--------------
 .../parquet/parquet-reader-writer-test.cc     | 489 +++++++++++++++++
 cpp/src/arrow/parquet/utils.h                 |  15 +-
 python/pyarrow/includes/libarrow_io.pxd       |  12 +-
 python/pyarrow/io.pyx                         |   8 +-
 12 files changed, 810 insertions(+), 444 deletions(-)
 create mode 100644 cpp/src/arrow/parquet/io.cc
 create mode 100644 cpp/src/arrow/parquet/io.h
 create mode 100644 cpp/src/arrow/parquet/parquet-reader-writer-test.cc

diff --git a/cpp/src/arrow/io/hdfs-io-test.cc b/cpp/src/arrow/io/hdfs-io-test.cc
index d1bf140ae68e2..e48a28142fa46 100644
--- a/cpp/src/arrow/io/hdfs-io-test.cc
+++ b/cpp/src/arrow/io/hdfs-io-test.cc
@@ -266,7 +266,7 @@ TEST_F(TestHdfsClient, ReadableMethods) {
   ASSERT_EQ(size, file_size);
 
   uint8_t buffer[50];
-  int32_t bytes_read = 0;
+  int64_t bytes_read = 0;
 
   ASSERT_OK(file->Read(50, &bytes_read, buffer));
   ASSERT_EQ(0, std::memcmp(buffer, data.data(), 50));
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 6da6ea4e71bd8..800c3edf4f31a 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -100,7 +100,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status ReadAt(int64_t position, int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) {
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
     tSize ret = hdfsPread(fs_, file_, static_cast<tOffset>(position),
         reinterpret_cast<void*>(buffer), nbytes);
     RETURN_NOT_OK(CheckReadResult(ret));
@@ -108,7 +108,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) {
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
     tSize ret = hdfsRead(fs_, file_, reinterpret_cast<void*>(buffer), nbytes);
     RETURN_NOT_OK(CheckReadResult(ret));
     *bytes_read = ret;
@@ -138,11 +138,11 @@ Status HdfsReadableFile::Close() {
 }
 
 Status HdfsReadableFile::ReadAt(
-    int64_t position, int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) {
+    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
   return impl_->ReadAt(position, nbytes, bytes_read, buffer);
 }
 
-Status HdfsReadableFile::Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) {
+Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
   return impl_->Read(nbytes, bytes_read, buffer);
 }
 
@@ -177,7 +177,7 @@ class HdfsWriteableFile::HdfsWriteableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* buffer, int32_t nbytes, int32_t* bytes_written) {
+  Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
     tSize ret = hdfsWrite(fs_, file_, reinterpret_cast<const void*>(buffer), nbytes);
     CHECK_FAILURE(ret, "Write");
     *bytes_written = ret;
@@ -198,12 +198,12 @@ Status HdfsWriteableFile::Close() {
 }
 
 Status HdfsWriteableFile::Write(
-    const uint8_t* buffer, int32_t nbytes, int32_t* bytes_read) {
+    const uint8_t* buffer, int64_t nbytes, int64_t* bytes_read) {
   return impl_->Write(buffer, nbytes, bytes_read);
 }
 
-Status HdfsWriteableFile::Write(const uint8_t* buffer, int32_t nbytes) {
-  int32_t bytes_written_dummy = 0;
+Status HdfsWriteableFile::Write(const uint8_t* buffer, int64_t nbytes) {
+  int64_t bytes_written_dummy = 0;
   return Write(buffer, nbytes, &bytes_written_dummy);
 }
 
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 532e3c536a188..b6449fcb88a75 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -164,14 +164,14 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
   Status GetSize(int64_t* size) override;
 
   Status ReadAt(
-      int64_t position, int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) override;
+      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
 
   Status Seek(int64_t position) override;
   Status Tell(int64_t* position) override;
 
   // NOTE: If you wish to read a particular range of a file in a multithreaded
   // context, you may prefer to use ReadAt to avoid locking issues
-  Status Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
 
  private:
   class ARROW_NO_EXPORT HdfsReadableFileImpl;
@@ -189,9 +189,9 @@ class ARROW_EXPORT HdfsWriteableFile : public WriteableFile {
 
   Status Close() override;
 
-  Status Write(const uint8_t* buffer, int32_t nbytes) override;
+  Status Write(const uint8_t* buffer, int64_t nbytes) override;
 
-  Status Write(const uint8_t* buffer, int32_t nbytes, int32_t* bytes_written);
+  Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written);
 
   Status Tell(int64_t* position) override;
 
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 4bd8a8ffc2f9d..25361d5633d12 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef ARROW_IO_INTERFACES
-#define ARROW_IO_INTERFACES
+#ifndef ARROW_IO_INTERFACES_H
+#define ARROW_IO_INTERFACES_H
 
 #include <cstdint>
 
@@ -40,17 +40,17 @@ class FileSystemClient {
 };
 
 class FileBase {
+ public:
   virtual Status Close() = 0;
-
   virtual Status Tell(int64_t* position) = 0;
 };
 
 class ReadableFile : public FileBase {
  public:
   virtual Status ReadAt(
-      int64_t position, int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) = 0;
+      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) = 0;
 
-  virtual Status Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) = 0;
+  virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) = 0;
 
   virtual Status GetSize(int64_t* size) = 0;
 };
@@ -62,10 +62,10 @@ class RandomAccessFile : public ReadableFile {
 
 class WriteableFile : public FileBase {
  public:
-  virtual Status Write(const uint8_t* buffer, int32_t nbytes) = 0;
+  virtual Status Write(const uint8_t* buffer, int64_t nbytes) = 0;
 };
 
 }  // namespace io
 }  // namespace arrow
 
-#endif  // ARROW_IO_INTERFACES
+#endif  // ARROW_IO_INTERFACES_H
diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
index 00f19b354e379..f2a90b71a4968 100644
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -19,6 +19,7 @@
 # arrow_parquet : Arrow <-> Parquet adapter
 
 set(PARQUET_SRCS
+  io.cc
   reader.cc
   schema.cc
   writer.cc
@@ -48,8 +49,12 @@ ARROW_TEST_LINK_LIBRARIES(parquet-schema-test arrow_parquet)
 ADD_ARROW_TEST(parquet-io-test)
 ARROW_TEST_LINK_LIBRARIES(parquet-io-test arrow_parquet)
 
+ADD_ARROW_TEST(parquet-reader-writer-test)
+ARROW_TEST_LINK_LIBRARIES(parquet-reader-writer-test arrow_parquet)
+
 # Headers: top level
 install(FILES
+  io.h
   reader.h
   schema.h
   utils.h
diff --git a/cpp/src/arrow/parquet/io.cc b/cpp/src/arrow/parquet/io.cc
new file mode 100644
index 0000000000000..c81aa8c4da9ca
--- /dev/null
+++ b/cpp/src/arrow/parquet/io.cc
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/parquet/io.h"
+
+#include <cstdint>
+#include <memory>
+
+#include "parquet/api/io.h"
+
+#include "arrow/parquet/utils.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+// To assist with readability
+using ArrowROFile = arrow::io::RandomAccessFile;
+
+namespace arrow {
+namespace parquet {
+
+// ----------------------------------------------------------------------
+// ParquetAllocator
+
+ParquetAllocator::ParquetAllocator() : pool_(default_memory_pool()) {}
+
+ParquetAllocator::ParquetAllocator(MemoryPool* pool) : pool_(pool) {}
+
+ParquetAllocator::~ParquetAllocator() {}
+
+uint8_t* ParquetAllocator::Malloc(int64_t size) {
+  uint8_t* result;
+  PARQUET_THROW_NOT_OK(pool_->Allocate(size, &result));
+  return result;
+}
+
+void ParquetAllocator::Free(uint8_t* buffer, int64_t size) {
+  // Does not report Status
+  pool_->Free(buffer, size);
+}
+
+// ----------------------------------------------------------------------
+// ParquetReadSource
+
+ParquetReadSource::ParquetReadSource(
+    const std::shared_ptr<ArrowROFile>& file, ParquetAllocator* allocator)
+    : file_(file), allocator_(allocator) {}
+
+void ParquetReadSource::Close() {
+  PARQUET_THROW_NOT_OK(file_->Close());
+}
+
+int64_t ParquetReadSource::Tell() const {
+  int64_t position;
+  PARQUET_THROW_NOT_OK(file_->Tell(&position));
+  return position;
+}
+
+void ParquetReadSource::Seek(int64_t position) {
+  PARQUET_THROW_NOT_OK(file_->Seek(position));
+}
+
+int64_t ParquetReadSource::Read(int64_t nbytes, uint8_t* out) {
+  int64_t bytes_read;
+  PARQUET_THROW_NOT_OK(file_->Read(nbytes, &bytes_read, out));
+  return bytes_read;
+}
+
+std::shared_ptr<::parquet::Buffer> ParquetReadSource::Read(int64_t nbytes) {
+  // TODO(wesm): This code is duplicated from parquet/util/input.cc; suggests
+  // that there should be more code sharing amongst file-like sources
+  auto result = std::make_shared<::parquet::OwnedMutableBuffer>(0, allocator_);
+  result->Resize(nbytes);
+
+  int64_t bytes_read = Read(nbytes, result->mutable_data());
+  if (bytes_read < nbytes) { result->Resize(bytes_read); }
+  return result;
+}
+
+}  // namespace parquet
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/io.h b/cpp/src/arrow/parquet/io.h
new file mode 100644
index 0000000000000..ef8871da4df61
--- /dev/null
+++ b/cpp/src/arrow/parquet/io.h
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Bridges Arrow's IO interfaces and Parquet-cpp's IO interfaces
+
+#ifndef ARROW_PARQUET_IO_H
+#define ARROW_PARQUET_IO_H
+
+#include <cstdint>
+#include <memory>
+
+#include "parquet/api/io.h"
+
+#include "arrow/io/interfaces.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class MemoryPool;
+
+namespace parquet {
+
+// An implementation of the Parquet MemoryAllocator API that plugs into an
+// existing Arrow memory pool. This way we can direct all allocations to a
+// single place rather than tracking allocations in different locations (for
+// example: without utilizing parquet-cpp's default allocator)
+class ARROW_EXPORT ParquetAllocator : public ::parquet::MemoryAllocator {
+ public:
+  // Uses the default memory pool
+  ParquetAllocator();
+
+  explicit ParquetAllocator(MemoryPool* pool);
+  virtual ~ParquetAllocator();
+
+  uint8_t* Malloc(int64_t size) override;
+  void Free(uint8_t* buffer, int64_t size) override;
+
+  MemoryPool* pool() { return pool_; }
+
+ private:
+  MemoryPool* pool_;
+};
+
+class ARROW_EXPORT ParquetReadSource : public ::parquet::RandomAccessSource {
+ public:
+  ParquetReadSource(
+      const std::shared_ptr<io::RandomAccessFile>& file, ParquetAllocator* allocator);
+
+  void Close() override;
+  int64_t Tell() const override;
+  void Seek(int64_t pos) override;
+  int64_t Read(int64_t nbytes, uint8_t* out) override;
+  std::shared_ptr<::parquet::Buffer> Read(int64_t nbytes) override;
+
+ private:
+  // An Arrow readable file of some kind
+  std::shared_ptr<io::RandomAccessFile> file_;
+
+  // The allocator is required for creating managed buffers
+  ParquetAllocator* allocator_;
+};
+
+}  // namespace parquet
+}  // namespace arrow
+
+#endif  // ARROW_PARQUET_IO_H
diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
index bfc27d26d63a1..7e724b31e3801 100644
--- a/cpp/src/arrow/parquet/parquet-io-test.cc
+++ b/cpp/src/arrow/parquet/parquet-io-test.cc
@@ -15,475 +15,164 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <string>
+
 #include "gtest/gtest.h"
 
-#include "arrow/test-util.h"
-#include "arrow/parquet/test-util.h"
-#include "arrow/parquet/reader.h"
-#include "arrow/parquet/writer.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
+#include "arrow/parquet/io.h"
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
-#include "parquet/api/reader.h"
-#include "parquet/api/writer.h"
-
-using ParquetBuffer = parquet::Buffer;
-using parquet::BufferReader;
-using parquet::default_writer_properties;
-using parquet::InMemoryOutputStream;
-using parquet::LogicalType;
-using parquet::ParquetFileReader;
-using parquet::ParquetFileWriter;
-using parquet::RandomAccessSource;
-using parquet::Repetition;
-using parquet::SchemaDescriptor;
-using parquet::ParquetVersion;
-using ParquetType = parquet::Type;
-using parquet::schema::GroupNode;
-using parquet::schema::NodePtr;
-using parquet::schema::PrimitiveNode;
+#include "parquet/api/io.h"
 
 namespace arrow {
-
 namespace parquet {
 
-const int SMALL_SIZE = 100;
-const int LARGE_SIZE = 10000;
-
-template <typename TestType>
-struct test_traits {};
+// Allocator tests
 
-template <>
-struct test_traits<BooleanType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::BOOLEAN;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static uint8_t const value;
-};
-
-const uint8_t test_traits<BooleanType>::value(1);
+TEST(TestParquetAllocator, DefaultCtor) {
+  ParquetAllocator allocator;
 
-template <>
-struct test_traits<UInt8Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::UINT_8;
-  static uint8_t const value;
-};
+  const int buffer_size = 10;
 
-const uint8_t test_traits<UInt8Type>::value(64);
-
-template <>
-struct test_traits<Int8Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::INT_8;
-  static int8_t const value;
-};
-
-const int8_t test_traits<Int8Type>::value(-64);
-
-template <>
-struct test_traits<UInt16Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::UINT_16;
-  static uint16_t const value;
-};
+  uint8_t* buffer = nullptr;
+  ASSERT_NO_THROW(buffer = allocator.Malloc(buffer_size););
 
-const uint16_t test_traits<UInt16Type>::value(1024);
+  // valgrind will complain if we write into nullptr
+  memset(buffer, 0, buffer_size);
 
-template <>
-struct test_traits<Int16Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::INT_16;
-  static int16_t const value;
-};
-
-const int16_t test_traits<Int16Type>::value(-1024);
-
-template <>
-struct test_traits<UInt32Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::UINT_32;
-  static uint32_t const value;
-};
-
-const uint32_t test_traits<UInt32Type>::value(1024);
-
-template <>
-struct test_traits<Int32Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static int32_t const value;
-};
-
-const int32_t test_traits<Int32Type>::value(-1024);
-
-template <>
-struct test_traits<UInt64Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
-  static constexpr LogicalType::type logical_enum = LogicalType::UINT_64;
-  static uint64_t const value;
-};
-
-const uint64_t test_traits<UInt64Type>::value(1024);
-
-template <>
-struct test_traits<Int64Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static int64_t const value;
-};
-
-const int64_t test_traits<Int64Type>::value(-1024);
-
-template <>
-struct test_traits<FloatType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::FLOAT;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static float const value;
-};
-
-const float test_traits<FloatType>::value(2.1f);
-
-template <>
-struct test_traits<DoubleType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::DOUBLE;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static double const value;
-};
-
-const double test_traits<DoubleType>::value(4.2);
-
-template <>
-struct test_traits<StringType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::BYTE_ARRAY;
-  static constexpr LogicalType::type logical_enum = LogicalType::UTF8;
-  static std::string const value;
-};
-
-const std::string test_traits<StringType>::value("Test");
-
-template <typename T>
-using ParquetDataType = ::parquet::DataType<test_traits<T>::parquet_enum>;
-
-template <typename T>
-using ParquetWriter = ::parquet::TypedColumnWriter<ParquetDataType<T>>;
+  allocator.Free(buffer, buffer_size);
+}
 
-template <typename TestType>
-class TestParquetIO : public ::testing::Test {
+// Pass through to the default memory pool
+class TrackingPool : public MemoryPool {
  public:
-  virtual void SetUp() {}
-
-  std::shared_ptr<GroupNode> MakeSchema(Repetition::type repetition) {
-    auto pnode = PrimitiveNode::Make("column1", repetition,
-        test_traits<TestType>::parquet_enum, test_traits<TestType>::logical_enum);
-    NodePtr node_ =
-        GroupNode::Make("schema", Repetition::REQUIRED, std::vector<NodePtr>({pnode}));
-    return std::static_pointer_cast<GroupNode>(node_);
-  }
-
-  std::unique_ptr<ParquetFileWriter> MakeWriter(
-      const std::shared_ptr<GroupNode>& schema) {
-    sink_ = std::make_shared<InMemoryOutputStream>();
-    return ParquetFileWriter::Open(sink_, schema);
-  }
-
-  std::unique_ptr<ParquetFileReader> ReaderFromSink() {
-    std::shared_ptr<ParquetBuffer> buffer = sink_->GetBuffer();
-    std::unique_ptr<RandomAccessSource> source(new BufferReader(buffer));
-    return ParquetFileReader::Open(std::move(source));
-  }
-
-  void ReadSingleColumnFile(
-      std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Array>* out) {
-    arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
-    std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
-    ASSERT_OK_NO_THROW(reader.GetFlatColumn(0, &column_reader));
-    ASSERT_NE(nullptr, column_reader.get());
-
-    ASSERT_OK(column_reader->NextBatch(SMALL_SIZE, out));
-    ASSERT_NE(nullptr, out->get());
-  }
+  TrackingPool() : pool_(default_memory_pool()), bytes_allocated_(0) {}
 
-  void ReadAndCheckSingleColumnFile(Array* values) {
-    std::shared_ptr<Array> out;
-    ReadSingleColumnFile(ReaderFromSink(), &out);
-    ASSERT_TRUE(values->Equals(out));
+  Status Allocate(int64_t size, uint8_t** out) override {
+    RETURN_NOT_OK(pool_->Allocate(size, out));
+    bytes_allocated_ += size;
+    return Status::OK();
   }
 
-  void ReadTableFromFile(
-      std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Table>* out) {
-    arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
-    ASSERT_OK_NO_THROW(reader.ReadFlatTable(out));
-    ASSERT_NE(nullptr, out->get());
+  void Free(uint8_t* buffer, int64_t size) override {
+    pool_->Free(buffer, size);
+    bytes_allocated_ -= size;
   }
 
-  void ReadAndCheckSingleColumnTable(const std::shared_ptr<Array>& values) {
-    std::shared_ptr<Table> out;
-    ReadTableFromFile(ReaderFromSink(), &out);
-    ASSERT_EQ(1, out->num_columns());
-    ASSERT_EQ(values->length(), out->num_rows());
-
-    std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-    ASSERT_EQ(1, chunked_array->num_chunks());
-    ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
-  }
+  int64_t bytes_allocated() const override { return bytes_allocated_; }
 
-  template <typename ArrayType>
-  void WriteFlatColumn(const std::shared_ptr<GroupNode>& schema,
-      const std::shared_ptr<ArrayType>& values) {
-    FileWriter writer(default_memory_pool(), MakeWriter(schema));
-    ASSERT_OK_NO_THROW(writer.NewRowGroup(values->length()));
-    ASSERT_OK_NO_THROW(writer.WriteFlatColumnChunk(values.get()));
-    ASSERT_OK_NO_THROW(writer.Close());
-  }
-
-  std::shared_ptr<InMemoryOutputStream> sink_;
+ private:
+  MemoryPool* pool_;
+  int64_t bytes_allocated_;
 };
 
-// We habe separate tests for UInt32Type as this is currently the only type
-// where a roundtrip does not yield the identical Array structure.
-// There we write an UInt32 Array but receive an Int64 Array as result for
-// Parquet version 1.0.
+TEST(TestParquetAllocator, CustomPool) {
+  TrackingPool pool;
 
-typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
-    Int32Type, UInt64Type, Int64Type, FloatType, DoubleType, StringType> TestTypes;
+  ParquetAllocator allocator(&pool);
 
-TYPED_TEST_CASE(TestParquetIO, TestTypes);
+  ASSERT_EQ(&pool, allocator.pool());
 
-TYPED_TEST(TestParquetIO, SingleColumnRequiredWrite) {
-  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
+  const int buffer_size = 10;
 
-  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
-  this->WriteFlatColumn(schema, values);
+  uint8_t* buffer = nullptr;
+  ASSERT_NO_THROW(buffer = allocator.Malloc(buffer_size););
 
-  this->ReadAndCheckSingleColumnFile(values.get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableRequiredWrite) {
-  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, false);
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_OK_NO_THROW(WriteFlatTable(table.get(), default_memory_pool(), this->sink_,
-      values->length(), default_writer_properties()));
-
-  std::shared_ptr<Table> out;
-  this->ReadTableFromFile(this->ReaderFromSink(), &out);
-  ASSERT_EQ(1, out->num_columns());
-  ASSERT_EQ(100, out->num_rows());
-
-  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-  ASSERT_EQ(1, chunked_array->num_chunks());
-  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
-}
+  ASSERT_EQ(buffer_size, pool.bytes_allocated());
 
-TYPED_TEST(TestParquetIO, SingleColumnOptionalReadWrite) {
-  // This also tests max_definition_level = 1
-  auto values = NullableArray<TypeParam>(SMALL_SIZE, 10);
+  // valgrind will complain if we write into nullptr
+  memset(buffer, 0, buffer_size);
 
-  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::OPTIONAL);
-  this->WriteFlatColumn(schema, values);
+  allocator.Free(buffer, buffer_size);
 
-  this->ReadAndCheckSingleColumnFile(values.get());
+  ASSERT_EQ(0, pool.bytes_allocated());
 }
 
-TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {
-  // This also tests max_definition_level = 1
-  std::shared_ptr<Array> values = NullableArray<TypeParam>(SMALL_SIZE, 10);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_OK_NO_THROW(WriteFlatTable(table.get(), default_memory_pool(), this->sink_,
-      values->length(), default_writer_properties()));
+// ----------------------------------------------------------------------
+// Read source tests
 
-  this->ReadAndCheckSingleColumnTable(values);
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
-  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
-  int64_t chunk_size = values->length() / 4;
+class BufferReader : public io::RandomAccessFile {
+ public:
+  BufferReader(const uint8_t* buffer, int buffer_size)
+      : buffer_(buffer), buffer_size_(buffer_size), position_(0) {}
 
-  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
-  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
-  for (int i = 0; i < 4; i++) {
-    ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
-    ASSERT_OK_NO_THROW(
-        writer.WriteFlatColumnChunk(values.get(), i * chunk_size, chunk_size));
+  Status Close() override {
+    // no-op
+    return Status::OK();
   }
-  ASSERT_OK_NO_THROW(writer.Close());
-
-  this->ReadAndCheckSingleColumnFile(values.get());
-}
 
-TYPED_TEST(TestParquetIO, SingleColumnTableRequiredChunkedWrite) {
-  auto values = NonNullArray<TypeParam>(LARGE_SIZE);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, false);
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_OK_NO_THROW(WriteFlatTable(
-      table.get(), default_memory_pool(), this->sink_, 512, default_writer_properties()));
-
-  this->ReadAndCheckSingleColumnTable(values);
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnOptionalChunkedWrite) {
-  int64_t chunk_size = SMALL_SIZE / 4;
-  auto values = NullableArray<TypeParam>(SMALL_SIZE, 10);
-
-  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::OPTIONAL);
-  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
-  for (int i = 0; i < 4; i++) {
-    ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
-    ASSERT_OK_NO_THROW(
-        writer.WriteFlatColumnChunk(values.get(), i * chunk_size, chunk_size));
+  Status Tell(int64_t* position) override {
+    *position = position_;
+    return Status::OK();
   }
-  ASSERT_OK_NO_THROW(writer.Close());
 
-  this->ReadAndCheckSingleColumnFile(values.get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableOptionalChunkedWrite) {
-  // This also tests max_definition_level = 1
-  auto values = NullableArray<TypeParam>(LARGE_SIZE, 100);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_OK_NO_THROW(WriteFlatTable(
-      table.get(), default_memory_pool(), this->sink_, 512, default_writer_properties()));
-
-  this->ReadAndCheckSingleColumnTable(values);
-}
-
-using TestUInt32ParquetIO = TestParquetIO<UInt32Type>;
-
-TEST_F(TestUInt32ParquetIO, Parquet_2_0_Compability) {
-  // This also tests max_definition_level = 1
-  std::shared_ptr<PrimitiveArray> values = NullableArray<UInt32Type>(LARGE_SIZE, 100);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-
-  // Parquet 2.0 roundtrip should yield an uint32_t column again
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  std::shared_ptr<::parquet::WriterProperties> properties =
-      ::parquet::WriterProperties::Builder()
-          .version(ParquetVersion::PARQUET_2_0)
-          ->build();
-  ASSERT_OK_NO_THROW(
-      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512, properties));
-  this->ReadAndCheckSingleColumnTable(values);
-}
+  Status ReadAt(
+      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override {
+    RETURN_NOT_OK(Seek(position));
+    return Read(nbytes, bytes_read, buffer);
+  }
 
-TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compability) {
-  // This also tests max_definition_level = 1
-  std::shared_ptr<PrimitiveArray> values = NullableArray<UInt32Type>(LARGE_SIZE, 100);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-
-  // Parquet 1.0 returns an int64_t column as there is no way to tell a Parquet 1.0
-  // reader that a column is unsigned.
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  std::shared_ptr<::parquet::WriterProperties> properties =
-      ::parquet::WriterProperties::Builder()
-          .version(ParquetVersion::PARQUET_1_0)
-          ->build();
-  ASSERT_OK_NO_THROW(
-      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512, properties));
-
-  std::shared_ptr<Array> expected_values;
-  std::shared_ptr<PoolBuffer> int64_data =
-      std::make_shared<PoolBuffer>(default_memory_pool());
-  {
-    ASSERT_OK(int64_data->Resize(sizeof(int64_t) * values->length()));
-    int64_t* int64_data_ptr = reinterpret_cast<int64_t*>(int64_data->mutable_data());
-    const uint32_t* uint32_data_ptr =
-        reinterpret_cast<const uint32_t*>(values->data()->data());
-    // std::copy might be faster but this is explicit on the casts)
-    for (int64_t i = 0; i < values->length(); i++) {
-      int64_data_ptr[i] = static_cast<int64_t>(uint32_data_ptr[i]);
-    }
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override {
+    memcpy(buffer, buffer_ + position_, nbytes);
+    *bytes_read = std::min(nbytes, buffer_size_ - position_);
+    position_ += *bytes_read;
+    return Status::OK();
   }
-  ASSERT_OK(MakePrimitiveArray(std::make_shared<Int64Type>(), values->length(),
-      int64_data, values->null_count(), values->null_bitmap(), &expected_values));
-  this->ReadAndCheckSingleColumnTable(expected_values);
-}
 
-template <typename T>
-using ParquetCDataType = typename ParquetDataType<T>::c_type;
+  Status GetSize(int64_t* size) override {
+    *size = buffer_size_;
+    return Status::OK();
+  }
 
-template <typename TestType>
-class TestPrimitiveParquetIO : public TestParquetIO<TestType> {
- public:
-  typedef typename TestType::c_type T;
-
-  void MakeTestFile(std::vector<T>& values, int num_chunks,
-      std::unique_ptr<ParquetFileReader>* file_reader) {
-    std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
-    std::unique_ptr<ParquetFileWriter> file_writer = this->MakeWriter(schema);
-    size_t chunk_size = values.size() / num_chunks;
-    // Convert to Parquet's expected physical type
-    std::vector<uint8_t> values_buffer(
-        sizeof(ParquetCDataType<TestType>) * values.size());
-    auto values_parquet =
-        reinterpret_cast<ParquetCDataType<TestType>*>(values_buffer.data());
-    std::copy(values.cbegin(), values.cend(), values_parquet);
-    for (int i = 0; i < num_chunks; i++) {
-      auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
-      auto column_writer =
-          static_cast<ParquetWriter<TestType>*>(row_group_writer->NextColumn());
-      ParquetCDataType<TestType>* data = values_parquet + i * chunk_size;
-      column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
-      column_writer->Close();
-      row_group_writer->Close();
+  Status Seek(int64_t position) override {
+    if (position < 0 || position >= buffer_size_) {
+      return Status::IOError("position out of bounds");
     }
-    file_writer->Close();
-    *file_reader = this->ReaderFromSink();
+
+    position_ = position;
+    return Status::OK();
   }
 
-  void CheckSingleColumnRequiredTableRead(int num_chunks) {
-    std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
-    std::unique_ptr<ParquetFileReader> file_reader;
-    ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader));
+ private:
+  const uint8_t* buffer_;
+  int buffer_size_;
+  int64_t position_;
+};
 
-    std::shared_ptr<Table> out;
-    this->ReadTableFromFile(std::move(file_reader), &out);
-    ASSERT_EQ(1, out->num_columns());
-    ASSERT_EQ(SMALL_SIZE, out->num_rows());
+TEST(TestParquetReadSource, Basics) {
+  std::string data = "this is the data";
+  auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str());
 
-    std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-    ASSERT_EQ(1, chunked_array->num_chunks());
-    ExpectArray<TestType>(values.data(), chunked_array->chunk(0).get());
-  }
+  ParquetAllocator allocator;
+  auto file = std::make_shared<BufferReader>(data_buffer, data.size());
+  auto source = std::make_shared<ParquetReadSource>(file, &allocator);
 
-  void CheckSingleColumnRequiredRead(int num_chunks) {
-    std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
-    std::unique_ptr<ParquetFileReader> file_reader;
-    ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader));
+  ASSERT_EQ(0, source->Tell());
+  ASSERT_NO_THROW(source->Seek(5));
+  ASSERT_EQ(5, source->Tell());
+  ASSERT_NO_THROW(source->Seek(0));
 
-    std::shared_ptr<Array> out;
-    this->ReadSingleColumnFile(std::move(file_reader), &out);
-
-    ExpectArray<TestType>(values.data(), out.get());
-  }
-};
+  // Seek out of bounds
+  ASSERT_THROW(source->Seek(100), ::parquet::ParquetException);
 
-typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
-    UInt32Type, Int32Type, UInt64Type, Int64Type, FloatType,
-    DoubleType> PrimitiveTestTypes;
+  uint8_t buffer[50];
 
-TYPED_TEST_CASE(TestPrimitiveParquetIO, PrimitiveTestTypes);
+  ASSERT_NO_THROW(source->Read(4, buffer));
+  ASSERT_EQ(0, std::memcmp(buffer, "this", 4));
+  ASSERT_EQ(4, source->Tell());
 
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredRead) {
-  this->CheckSingleColumnRequiredRead(1);
-}
+  std::shared_ptr<::parquet::Buffer> pq_buffer;
 
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredTableRead) {
-  this->CheckSingleColumnRequiredTableRead(1);
-}
+  ASSERT_NO_THROW(pq_buffer = source->Read(7));
 
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedRead) {
-  this->CheckSingleColumnRequiredRead(4);
-}
+  auto expected_buffer = std::make_shared<::parquet::Buffer>(data_buffer + 4, 7);
 
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
-  this->CheckSingleColumnRequiredTableRead(4);
+  ASSERT_TRUE(expected_buffer->Equals(*pq_buffer.get()));
 }
 
 }  // namespace parquet
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/parquet/parquet-reader-writer-test.cc b/cpp/src/arrow/parquet/parquet-reader-writer-test.cc
new file mode 100644
index 0000000000000..bfc27d26d63a1
--- /dev/null
+++ b/cpp/src/arrow/parquet/parquet-reader-writer-test.cc
@@ -0,0 +1,489 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/test-util.h"
+#include "arrow/parquet/test-util.h"
+#include "arrow/parquet/reader.h"
+#include "arrow/parquet/writer.h"
+#include "arrow/types/construct.h"
+#include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+#include "parquet/api/reader.h"
+#include "parquet/api/writer.h"
+
+using ParquetBuffer = parquet::Buffer;
+using parquet::BufferReader;
+using parquet::default_writer_properties;
+using parquet::InMemoryOutputStream;
+using parquet::LogicalType;
+using parquet::ParquetFileReader;
+using parquet::ParquetFileWriter;
+using parquet::RandomAccessSource;
+using parquet::Repetition;
+using parquet::SchemaDescriptor;
+using parquet::ParquetVersion;
+using ParquetType = parquet::Type;
+using parquet::schema::GroupNode;
+using parquet::schema::NodePtr;
+using parquet::schema::PrimitiveNode;
+
+namespace arrow {
+
+namespace parquet {
+
+const int SMALL_SIZE = 100;
+const int LARGE_SIZE = 10000;
+
+template <typename TestType>
+struct test_traits {};
+
+template <>
+struct test_traits<BooleanType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::BOOLEAN;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static uint8_t const value;
+};
+
+const uint8_t test_traits<BooleanType>::value(1);
+
+template <>
+struct test_traits<UInt8Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::UINT_8;
+  static uint8_t const value;
+};
+
+const uint8_t test_traits<UInt8Type>::value(64);
+
+template <>
+struct test_traits<Int8Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::INT_8;
+  static int8_t const value;
+};
+
+const int8_t test_traits<Int8Type>::value(-64);
+
+template <>
+struct test_traits<UInt16Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::UINT_16;
+  static uint16_t const value;
+};
+
+const uint16_t test_traits<UInt16Type>::value(1024);
+
+template <>
+struct test_traits<Int16Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::INT_16;
+  static int16_t const value;
+};
+
+const int16_t test_traits<Int16Type>::value(-1024);
+
+template <>
+struct test_traits<UInt32Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::UINT_32;
+  static uint32_t const value;
+};
+
+const uint32_t test_traits<UInt32Type>::value(1024);
+
+template <>
+struct test_traits<Int32Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static int32_t const value;
+};
+
+const int32_t test_traits<Int32Type>::value(-1024);
+
+template <>
+struct test_traits<UInt64Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
+  static constexpr LogicalType::type logical_enum = LogicalType::UINT_64;
+  static uint64_t const value;
+};
+
+const uint64_t test_traits<UInt64Type>::value(1024);
+
+template <>
+struct test_traits<Int64Type> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static int64_t const value;
+};
+
+const int64_t test_traits<Int64Type>::value(-1024);
+
+template <>
+struct test_traits<FloatType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::FLOAT;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static float const value;
+};
+
+const float test_traits<FloatType>::value(2.1f);
+
+template <>
+struct test_traits<DoubleType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::DOUBLE;
+  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
+  static double const value;
+};
+
+const double test_traits<DoubleType>::value(4.2);
+
+template <>
+struct test_traits<StringType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::BYTE_ARRAY;
+  static constexpr LogicalType::type logical_enum = LogicalType::UTF8;
+  static std::string const value;
+};
+
+const std::string test_traits<StringType>::value("Test");
+
+template <typename T>
+using ParquetDataType = ::parquet::DataType<test_traits<T>::parquet_enum>;
+
+template <typename T>
+using ParquetWriter = ::parquet::TypedColumnWriter<ParquetDataType<T>>;
+
+template <typename TestType>
+class TestParquetIO : public ::testing::Test {
+ public:
+  virtual void SetUp() {}
+
+  std::shared_ptr<GroupNode> MakeSchema(Repetition::type repetition) {
+    auto pnode = PrimitiveNode::Make("column1", repetition,
+        test_traits<TestType>::parquet_enum, test_traits<TestType>::logical_enum);
+    NodePtr node_ =
+        GroupNode::Make("schema", Repetition::REQUIRED, std::vector<NodePtr>({pnode}));
+    return std::static_pointer_cast<GroupNode>(node_);
+  }
+
+  std::unique_ptr<ParquetFileWriter> MakeWriter(
+      const std::shared_ptr<GroupNode>& schema) {
+    sink_ = std::make_shared<InMemoryOutputStream>();
+    return ParquetFileWriter::Open(sink_, schema);
+  }
+
+  std::unique_ptr<ParquetFileReader> ReaderFromSink() {
+    std::shared_ptr<ParquetBuffer> buffer = sink_->GetBuffer();
+    std::unique_ptr<RandomAccessSource> source(new BufferReader(buffer));
+    return ParquetFileReader::Open(std::move(source));
+  }
+
+  void ReadSingleColumnFile(
+      std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Array>* out) {
+    arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
+    std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
+    ASSERT_OK_NO_THROW(reader.GetFlatColumn(0, &column_reader));
+    ASSERT_NE(nullptr, column_reader.get());
+
+    ASSERT_OK(column_reader->NextBatch(SMALL_SIZE, out));
+    ASSERT_NE(nullptr, out->get());
+  }
+
+  void ReadAndCheckSingleColumnFile(Array* values) {
+    std::shared_ptr<Array> out;
+    ReadSingleColumnFile(ReaderFromSink(), &out);
+    ASSERT_TRUE(values->Equals(out));
+  }
+
+  void ReadTableFromFile(
+      std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Table>* out) {
+    arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
+    ASSERT_OK_NO_THROW(reader.ReadFlatTable(out));
+    ASSERT_NE(nullptr, out->get());
+  }
+
+  void ReadAndCheckSingleColumnTable(const std::shared_ptr<Array>& values) {
+    std::shared_ptr<Table> out;
+    ReadTableFromFile(ReaderFromSink(), &out);
+    ASSERT_EQ(1, out->num_columns());
+    ASSERT_EQ(values->length(), out->num_rows());
+
+    std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+    ASSERT_EQ(1, chunked_array->num_chunks());
+    ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+  }
+
+  template <typename ArrayType>
+  void WriteFlatColumn(const std::shared_ptr<GroupNode>& schema,
+      const std::shared_ptr<ArrayType>& values) {
+    FileWriter writer(default_memory_pool(), MakeWriter(schema));
+    ASSERT_OK_NO_THROW(writer.NewRowGroup(values->length()));
+    ASSERT_OK_NO_THROW(writer.WriteFlatColumnChunk(values.get()));
+    ASSERT_OK_NO_THROW(writer.Close());
+  }
+
+  std::shared_ptr<InMemoryOutputStream> sink_;
+};
+
+// We habe separate tests for UInt32Type as this is currently the only type
+// where a roundtrip does not yield the identical Array structure.
+// There we write an UInt32 Array but receive an Int64 Array as result for
+// Parquet version 1.0.
+
+typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
+    Int32Type, UInt64Type, Int64Type, FloatType, DoubleType, StringType> TestTypes;
+
+TYPED_TEST_CASE(TestParquetIO, TestTypes);
+
+TYPED_TEST(TestParquetIO, SingleColumnRequiredWrite) {
+  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
+
+  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
+  this->WriteFlatColumn(schema, values);
+
+  this->ReadAndCheckSingleColumnFile(values.get());
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnTableRequiredWrite) {
+  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, false);
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  ASSERT_OK_NO_THROW(WriteFlatTable(table.get(), default_memory_pool(), this->sink_,
+      values->length(), default_writer_properties()));
+
+  std::shared_ptr<Table> out;
+  this->ReadTableFromFile(this->ReaderFromSink(), &out);
+  ASSERT_EQ(1, out->num_columns());
+  ASSERT_EQ(100, out->num_rows());
+
+  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+  ASSERT_EQ(1, chunked_array->num_chunks());
+  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnOptionalReadWrite) {
+  // This also tests max_definition_level = 1
+  auto values = NullableArray<TypeParam>(SMALL_SIZE, 10);
+
+  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::OPTIONAL);
+  this->WriteFlatColumn(schema, values);
+
+  this->ReadAndCheckSingleColumnFile(values.get());
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {
+  // This also tests max_definition_level = 1
+  std::shared_ptr<Array> values = NullableArray<TypeParam>(SMALL_SIZE, 10);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  ASSERT_OK_NO_THROW(WriteFlatTable(table.get(), default_memory_pool(), this->sink_,
+      values->length(), default_writer_properties()));
+
+  this->ReadAndCheckSingleColumnTable(values);
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
+  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
+  int64_t chunk_size = values->length() / 4;
+
+  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
+  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
+  for (int i = 0; i < 4; i++) {
+    ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
+    ASSERT_OK_NO_THROW(
+        writer.WriteFlatColumnChunk(values.get(), i * chunk_size, chunk_size));
+  }
+  ASSERT_OK_NO_THROW(writer.Close());
+
+  this->ReadAndCheckSingleColumnFile(values.get());
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnTableRequiredChunkedWrite) {
+  auto values = NonNullArray<TypeParam>(LARGE_SIZE);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, false);
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  ASSERT_OK_NO_THROW(WriteFlatTable(
+      table.get(), default_memory_pool(), this->sink_, 512, default_writer_properties()));
+
+  this->ReadAndCheckSingleColumnTable(values);
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnOptionalChunkedWrite) {
+  int64_t chunk_size = SMALL_SIZE / 4;
+  auto values = NullableArray<TypeParam>(SMALL_SIZE, 10);
+
+  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::OPTIONAL);
+  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
+  for (int i = 0; i < 4; i++) {
+    ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
+    ASSERT_OK_NO_THROW(
+        writer.WriteFlatColumnChunk(values.get(), i * chunk_size, chunk_size));
+  }
+  ASSERT_OK_NO_THROW(writer.Close());
+
+  this->ReadAndCheckSingleColumnFile(values.get());
+}
+
+TYPED_TEST(TestParquetIO, SingleColumnTableOptionalChunkedWrite) {
+  // This also tests max_definition_level = 1
+  auto values = NullableArray<TypeParam>(LARGE_SIZE, 100);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  ASSERT_OK_NO_THROW(WriteFlatTable(
+      table.get(), default_memory_pool(), this->sink_, 512, default_writer_properties()));
+
+  this->ReadAndCheckSingleColumnTable(values);
+}
+
+using TestUInt32ParquetIO = TestParquetIO<UInt32Type>;
+
+TEST_F(TestUInt32ParquetIO, Parquet_2_0_Compability) {
+  // This also tests max_definition_level = 1
+  std::shared_ptr<PrimitiveArray> values = NullableArray<UInt32Type>(LARGE_SIZE, 100);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
+
+  // Parquet 2.0 roundtrip should yield an uint32_t column again
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  std::shared_ptr<::parquet::WriterProperties> properties =
+      ::parquet::WriterProperties::Builder()
+          .version(ParquetVersion::PARQUET_2_0)
+          ->build();
+  ASSERT_OK_NO_THROW(
+      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512, properties));
+  this->ReadAndCheckSingleColumnTable(values);
+}
+
+TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compability) {
+  // This also tests max_definition_level = 1
+  std::shared_ptr<PrimitiveArray> values = NullableArray<UInt32Type>(LARGE_SIZE, 100);
+  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
+
+  // Parquet 1.0 returns an int64_t column as there is no way to tell a Parquet 1.0
+  // reader that a column is unsigned.
+  this->sink_ = std::make_shared<InMemoryOutputStream>();
+  std::shared_ptr<::parquet::WriterProperties> properties =
+      ::parquet::WriterProperties::Builder()
+          .version(ParquetVersion::PARQUET_1_0)
+          ->build();
+  ASSERT_OK_NO_THROW(
+      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512, properties));
+
+  std::shared_ptr<Array> expected_values;
+  std::shared_ptr<PoolBuffer> int64_data =
+      std::make_shared<PoolBuffer>(default_memory_pool());
+  {
+    ASSERT_OK(int64_data->Resize(sizeof(int64_t) * values->length()));
+    int64_t* int64_data_ptr = reinterpret_cast<int64_t*>(int64_data->mutable_data());
+    const uint32_t* uint32_data_ptr =
+        reinterpret_cast<const uint32_t*>(values->data()->data());
+    // std::copy might be faster but this is explicit on the casts)
+    for (int64_t i = 0; i < values->length(); i++) {
+      int64_data_ptr[i] = static_cast<int64_t>(uint32_data_ptr[i]);
+    }
+  }
+  ASSERT_OK(MakePrimitiveArray(std::make_shared<Int64Type>(), values->length(),
+      int64_data, values->null_count(), values->null_bitmap(), &expected_values));
+  this->ReadAndCheckSingleColumnTable(expected_values);
+}
+
+template <typename T>
+using ParquetCDataType = typename ParquetDataType<T>::c_type;
+
+template <typename TestType>
+class TestPrimitiveParquetIO : public TestParquetIO<TestType> {
+ public:
+  typedef typename TestType::c_type T;
+
+  void MakeTestFile(std::vector<T>& values, int num_chunks,
+      std::unique_ptr<ParquetFileReader>* file_reader) {
+    std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
+    std::unique_ptr<ParquetFileWriter> file_writer = this->MakeWriter(schema);
+    size_t chunk_size = values.size() / num_chunks;
+    // Convert to Parquet's expected physical type
+    std::vector<uint8_t> values_buffer(
+        sizeof(ParquetCDataType<TestType>) * values.size());
+    auto values_parquet =
+        reinterpret_cast<ParquetCDataType<TestType>*>(values_buffer.data());
+    std::copy(values.cbegin(), values.cend(), values_parquet);
+    for (int i = 0; i < num_chunks; i++) {
+      auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
+      auto column_writer =
+          static_cast<ParquetWriter<TestType>*>(row_group_writer->NextColumn());
+      ParquetCDataType<TestType>* data = values_parquet + i * chunk_size;
+      column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
+      column_writer->Close();
+      row_group_writer->Close();
+    }
+    file_writer->Close();
+    *file_reader = this->ReaderFromSink();
+  }
+
+  void CheckSingleColumnRequiredTableRead(int num_chunks) {
+    std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
+    std::unique_ptr<ParquetFileReader> file_reader;
+    ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader));
+
+    std::shared_ptr<Table> out;
+    this->ReadTableFromFile(std::move(file_reader), &out);
+    ASSERT_EQ(1, out->num_columns());
+    ASSERT_EQ(SMALL_SIZE, out->num_rows());
+
+    std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
+    ASSERT_EQ(1, chunked_array->num_chunks());
+    ExpectArray<TestType>(values.data(), chunked_array->chunk(0).get());
+  }
+
+  void CheckSingleColumnRequiredRead(int num_chunks) {
+    std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
+    std::unique_ptr<ParquetFileReader> file_reader;
+    ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader));
+
+    std::shared_ptr<Array> out;
+    this->ReadSingleColumnFile(std::move(file_reader), &out);
+
+    ExpectArray<TestType>(values.data(), out.get());
+  }
+};
+
+typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
+    UInt32Type, Int32Type, UInt64Type, Int64Type, FloatType,
+    DoubleType> PrimitiveTestTypes;
+
+TYPED_TEST_CASE(TestPrimitiveParquetIO, PrimitiveTestTypes);
+
+TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredRead) {
+  this->CheckSingleColumnRequiredRead(1);
+}
+
+TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredTableRead) {
+  this->CheckSingleColumnRequiredTableRead(1);
+}
+
+TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedRead) {
+  this->CheckSingleColumnRequiredRead(4);
+}
+
+TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
+  this->CheckSingleColumnRequiredTableRead(4);
+}
+
+}  // namespace parquet
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/utils.h b/cpp/src/arrow/parquet/utils.h
index 409bcd9065cda..bcc46be60e6ec 100644
--- a/cpp/src/arrow/parquet/utils.h
+++ b/cpp/src/arrow/parquet/utils.h
@@ -18,12 +18,12 @@
 #ifndef ARROW_PARQUET_UTILS_H
 #define ARROW_PARQUET_UTILS_H
 
-#include "arrow/util/status.h"
+#include <sstream>
 
+#include "arrow/util/status.h"
 #include "parquet/exception.h"
 
 namespace arrow {
-
 namespace parquet {
 
 #define PARQUET_CATCH_NOT_OK(s) \
@@ -36,8 +36,17 @@ namespace parquet {
     (s);                         \
   } catch (const ::parquet::ParquetException& e) {}
 
-}  // namespace parquet
+#define PARQUET_THROW_NOT_OK(s)                    \
+  do {                                             \
+    ::arrow::Status _s = (s);                      \
+    if (!_s.ok()) {                                \
+      std::stringstream ss;                        \
+      ss << "Arrow error: " << _s.ToString();      \
+      throw ::parquet::ParquetException(ss.str()); \
+    }                                              \
+  } while (0);
 
+}  // namespace parquet
 }  // namespace arrow
 
 #endif  // ARROW_PARQUET_UTILS_H
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index d874ba3091237..d0fb8f9f000b9 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -51,17 +51,17 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
 
     cdef cppclass HdfsReadableFile(CHdfsFile):
         CStatus GetSize(int64_t* size)
-        CStatus Read(int32_t nbytes, int32_t* bytes_read,
+        CStatus Read(int64_t nbytes, int64_t* bytes_read,
                      uint8_t* buffer)
 
-        CStatus ReadAt(int64_t position, int32_t nbytes,
-                       int32_t* bytes_read, uint8_t* buffer)
+        CStatus ReadAt(int64_t position, int64_t nbytes,
+                       int64_t* bytes_read, uint8_t* buffer)
 
     cdef cppclass HdfsWriteableFile(CHdfsFile):
-        CStatus Write(const uint8_t* buffer, int32_t nbytes)
+        CStatus Write(const uint8_t* buffer, int64_t nbytes)
 
-        CStatus Write(const uint8_t* buffer, int32_t nbytes,
-                      int32_t* bytes_written)
+        CStatus Write(const uint8_t* buffer, int64_t nbytes,
+                      int64_t* bytes_written)
 
     cdef cppclass CHdfsClient" arrow::io::HdfsClient":
         @staticmethod
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 8b97671e45373..071eea5ba6e60 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -383,7 +383,7 @@ cdef class HdfsFile:
         Read indicated number of bytes from the file, up to EOF
         """
         cdef:
-            int32_t bytes_read = 0
+            int64_t bytes_read = 0
             uint8_t* buf
 
         self._assert_readable()
@@ -394,7 +394,7 @@ cdef class HdfsFile:
         if buf == NULL:
             raise MemoryError("Failed to allocate {0} bytes".format(nbytes))
 
-        cdef int32_t total_bytes = 0
+        cdef int64_t total_bytes = 0
 
         cdef int rpc_chunksize = min(self.buffer_size, nbytes)
 
@@ -423,7 +423,7 @@ cdef class HdfsFile:
         memory). First seeks to the beginning of the file.
         """
         cdef:
-            int32_t bytes_read = 0
+            int64_t bytes_read = 0
             uint8_t* buf
         self._assert_readable()
 
@@ -499,6 +499,6 @@ cdef class HdfsFile:
         data = tobytes(data)
 
         cdef const uint8_t* buf = <const uint8_t*> cp.PyBytes_AS_STRING(data)
-        cdef int32_t bufsize = len(data)
+        cdef int64_t bufsize = len(data)
         with nogil:
             check_cstatus(self.wr_file.get().Write(buf, bufsize))

From 62390d8427445b033ba7f7cf3150184222d2c2c1 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Tue, 12 Jul 2016 17:34:36 -0700
Subject: [PATCH 0097/1644] ARROW-106: [C++] Add IPC to binary/string types

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #103 from emkornfield/emk_add_string_rpc and squashes the following commits:

9c563fe [Micah Kornfield] ARROW-106: [C++] Add IPC to binary/string types
---
 cpp/src/arrow/ipc/adapter.cc          | 10 ++----
 cpp/src/arrow/ipc/ipc-adapter-test.cc | 52 +++++++++++++++++++++++++--
 cpp/src/arrow/types/construct.cc      |  4 +++
 3 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 45cc288cd6b9e..bac1172700615 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -33,6 +33,7 @@
 #include "arrow/types/construct.h"
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/status.h"
@@ -81,14 +82,9 @@ static bool IsListType(const DataType* type) {
     // code consider using pattern like:
     // http://stackoverflow.com/questions/26784685/c-macro-for-calling-function-based-on-enum-type
     //
-    // TODO(emkornfield) Fix type systems so these are all considered lists and
-    // the types behave the same way?
-    // case Type::BINARY:
-    // case Type::CHAR:
+    case Type::BINARY:
     case Type::LIST:
-      // see todo on common types
-      // case Type::STRING:
-      // case Type::VARCHAR:
+    case Type::STRING:
       return true;
     default:
       return false;
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index eb47ac6fee8a1..2bfb459d6e06a 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -31,6 +31,7 @@
 #include "arrow/test-util.h"
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/memory-pool.h"
@@ -105,6 +106,52 @@ Status MakeIntRowBatch(std::shared_ptr<RowBatch>* out) {
   return Status::OK();
 }
 
+template <class Builder, class RawType>
+Status MakeRandomBinaryArray(
+    const TypePtr& type, int32_t length, MemoryPool* pool, ArrayPtr* array) {
+  const std::vector<std::string> values = {
+      "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
+  Builder builder(pool, type);
+  const auto values_len = values.size();
+  for (int32_t i = 0; i < length; ++i) {
+    int values_index = i % values_len;
+    if (values_index == 0) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else {
+      const std::string& value = values[values_index];
+      RETURN_NOT_OK(
+          builder.Append(reinterpret_cast<const RawType*>(value.data()), value.size()));
+    }
+  }
+  *array = builder.Finish();
+  return Status::OK();
+}
+
+Status MakeStringTypesRowBatch(std::shared_ptr<RowBatch>* out) {
+  const int32_t length = 500;
+  auto string_type = std::make_shared<StringType>();
+  auto binary_type = std::make_shared<BinaryType>();
+  auto f0 = std::make_shared<Field>("f0", string_type);
+  auto f1 = std::make_shared<Field>("f1", binary_type);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  std::shared_ptr<Array> a0, a1;
+  MemoryPool* pool = default_memory_pool();
+
+  {
+    auto status =
+        MakeRandomBinaryArray<StringBuilder, char>(string_type, length, pool, &a0);
+    RETURN_NOT_OK(status);
+  }
+  {
+    auto status =
+        MakeRandomBinaryArray<BinaryBuilder, uint8_t>(binary_type, length, pool, &a1);
+    RETURN_NOT_OK(status);
+  }
+  out->reset(new RowBatch(schema, length, {a0, a1}));
+  return Status::OK();
+}
+
 Status MakeListRowBatch(std::shared_ptr<RowBatch>* out) {
   // Make the schema
   auto f0 = std::make_shared<Field>("f0", LIST_INT32);
@@ -191,9 +238,10 @@ Status MakeDeeplyNestedList(std::shared_ptr<RowBatch>* out) {
   return Status::OK();
 }
 
-INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRowBatch,
+INSTANTIATE_TEST_CASE_P(
+    RoundTripTests, TestWriteRowBatch,
     ::testing::Values(&MakeIntRowBatch, &MakeListRowBatch, &MakeNonNullRowBatch,
-                            &MakeZeroLengthRowBatch, &MakeDeeplyNestedList));
+        &MakeZeroLengthRowBatch, &MakeDeeplyNestedList, &MakeStringTypesRowBatch));
 
 void TestGetRowBatchSize(std::shared_ptr<RowBatch> batch) {
   MockMemorySource mock_source(1 << 16);
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 2d913a737486f..5ae9c5ab6d4f9 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -124,9 +124,13 @@ Status MakeListArray(const TypePtr& type, int32_t length,
     const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
   switch (type->type) {
     case Type::BINARY:
+      out->reset(new BinaryArray(type, length, offsets, values, null_count, null_bitmap));
+      break;
+
     case Type::LIST:
       out->reset(new ListArray(type, length, offsets, values, null_count, null_bitmap));
       break;
+
     case Type::DECIMAL_TEXT:
     case Type::STRING:
       out->reset(new StringArray(type, length, offsets, values, null_count, null_bitmap));

From 55bfa834312685991d615301ac0b4fcc7c11640b Mon Sep 17 00:00:00 2001
From: Jihoon Son <jihoonson@apache.org>
Date: Mon, 18 Jul 2016 15:07:48 -0700
Subject: [PATCH 0098/1644] =?UTF-8?q?ARROW-238:=20Change=20InternalMemoryP?=
 =?UTF-8?q?ool::Free()=20to=20return=20Status::Invalid=20when=20ther?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…e is insufficient memory.

Author: Jihoon Son <jihoonson@apache.org>

Closes #102 from jihoonson/ARROW-238 and squashes the following commits:

cb9e7b1 [Jihoon Son] Disable FreeLargeMemory test for release builds
f903130 [Jihoon Son] Free allocated memory after death
0077a70 [Jihoon Son] Adjust the amount of memory allocation
b1af59b [Jihoon Son] Change to ASSERT_EXIT
b4159f0 [Jihoon Son] Reflect comments
e89a1f9 [Jihoon Son] Change python implementation as well.
7651570 [Jihoon Son] Change InternalMemoryPool::Free() to return Status::Invalid when there is insufficient memory.
---
 cpp/src/arrow/util/logging.h           |  2 +-
 cpp/src/arrow/util/memory-pool-test.cc | 14 ++++++++++++++
 cpp/src/arrow/util/memory-pool.cc      |  2 ++
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index fccc5e3085de5..54f67593bec5e 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -40,7 +40,7 @@ namespace arrow {
 
 #define ARROW_CHECK(condition)                               \
   (condition) ? 0 : ::arrow::internal::FatalLog(ARROW_FATAL) \
-                        << __FILE__ << __LINE__ << "Check failed: " #condition " "
+                        << __FILE__ << __LINE__ << " Check failed: " #condition " "
 
 #ifdef NDEBUG
 #define ARROW_DFATAL ARROW_WARNING
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
index 8e7dfd60baa62..919f3740982cf 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -46,4 +46,18 @@ TEST(DefaultMemoryPool, OOM) {
   ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
 }
 
+TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
+  MemoryPool* pool = default_memory_pool();
+
+  uint8_t* data;
+  ASSERT_OK(pool->Allocate(100, &data));
+
+#ifndef NDEBUG
+  EXPECT_EXIT(pool->Free(data, 120), ::testing::ExitedWithCode(1),
+               ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
+#endif
+
+  pool->Free(data, 100);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
index 0a58e5aa21f72..fed149bc3598c 100644
--- a/cpp/src/arrow/util/memory-pool.cc
+++ b/cpp/src/arrow/util/memory-pool.cc
@@ -23,6 +23,7 @@
 #include <sstream>
 
 #include "arrow/util/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 
@@ -81,6 +82,7 @@ int64_t InternalMemoryPool::bytes_allocated() const {
 
 void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
   std::lock_guard<std::mutex> guard(pool_lock_);
+  DCHECK_GE(bytes_allocated_, size);
   std::free(buffer);
   bytes_allocated_ -= size;
 }

From 59e5f9806515e8a5360870c93082316f74d7ec7c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Mon, 18 Jul 2016 15:37:27 -0700
Subject: [PATCH 0099/1644] ARROW-236: Bridging IO interfaces under the hood in
 pyarrow

Author: Wes McKinney <wesm@apache.org>

Closes #104 from wesm/ARROW-236 and squashes the following commits:

73648e0 [Wes McKinney] cpplint
f2cd77f [Wes McKinney] Check in io.pxd
94bcd30 [Wes McKinney] Do not let Parquet close an Arrow file
9b9d94d [Wes McKinney] Barely working direct HDFS-Parquet reads
06ddd06 [Wes McKinney] Slight refactoring of read table to be able to also handle classes wrapping C++ file interfaces
c7a913e [Wes McKinney] Provide a means to expose abstract native file handles
e6724de [Wes McKinney] Implement alternate ctor to construct parquet::FileReader from an arrow::io::RandomAccessFile
---
 cpp/src/arrow/io/interfaces.h            |  1 +
 cpp/src/arrow/parquet/io.cc              | 19 +++++--
 cpp/src/arrow/parquet/io.h               | 10 +++-
 cpp/src/arrow/parquet/parquet-io-test.cc |  8 ++-
 cpp/src/arrow/parquet/reader.cc          | 20 +++++++
 cpp/src/arrow/parquet/reader.h           | 13 ++++-
 cpp/src/arrow/parquet/writer.cc          |  1 -
 cpp/src/arrow/parquet/writer.h           |  2 +-
 python/pyarrow/includes/libarrow_io.pxd  | 49 +++++++++++------
 python/pyarrow/includes/parquet.pxd      | 24 +++++++-
 python/pyarrow/io.pxd                    | 32 +++++++++++
 python/pyarrow/io.pyx                    | 19 ++++++-
 python/pyarrow/parquet.pyx               | 70 +++++++++++++++++++-----
 13 files changed, 216 insertions(+), 52 deletions(-)
 create mode 100644 python/pyarrow/io.pxd

diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 25361d5633d12..c21285253714e 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -19,6 +19,7 @@
 #define ARROW_IO_INTERFACES_H
 
 #include <cstdint>
+#include <memory>
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/parquet/io.cc b/cpp/src/arrow/parquet/io.cc
index c81aa8c4da9ca..b6fdd67d15b6c 100644
--- a/cpp/src/arrow/parquet/io.cc
+++ b/cpp/src/arrow/parquet/io.cc
@@ -55,12 +55,23 @@ void ParquetAllocator::Free(uint8_t* buffer, int64_t size) {
 // ----------------------------------------------------------------------
 // ParquetReadSource
 
-ParquetReadSource::ParquetReadSource(
-    const std::shared_ptr<ArrowROFile>& file, ParquetAllocator* allocator)
-    : file_(file), allocator_(allocator) {}
+ParquetReadSource::ParquetReadSource(ParquetAllocator* allocator)
+    : file_(nullptr), allocator_(allocator) {}
+
+Status ParquetReadSource::Open(const std::shared_ptr<io::RandomAccessFile>& file) {
+  int64_t file_size;
+  RETURN_NOT_OK(file->GetSize(&file_size));
+
+  file_ = file;
+  size_ = file_size;
+  return Status::OK();
+}
 
 void ParquetReadSource::Close() {
-  PARQUET_THROW_NOT_OK(file_->Close());
+  // TODO(wesm): Make this a no-op for now. This leaves Python wrappers for
+  // these classes in a borked state. Probably better to explicitly close.
+
+  // PARQUET_THROW_NOT_OK(file_->Close());
 }
 
 int64_t ParquetReadSource::Tell() const {
diff --git a/cpp/src/arrow/parquet/io.h b/cpp/src/arrow/parquet/io.h
index ef8871da4df61..1c59695c6c151 100644
--- a/cpp/src/arrow/parquet/io.h
+++ b/cpp/src/arrow/parquet/io.h
@@ -49,7 +49,9 @@ class ARROW_EXPORT ParquetAllocator : public ::parquet::MemoryAllocator {
   uint8_t* Malloc(int64_t size) override;
   void Free(uint8_t* buffer, int64_t size) override;
 
-  MemoryPool* pool() { return pool_; }
+  void set_pool(MemoryPool* pool) { pool_ = pool; }
+
+  MemoryPool* pool() const { return pool_; }
 
  private:
   MemoryPool* pool_;
@@ -57,8 +59,10 @@ class ARROW_EXPORT ParquetAllocator : public ::parquet::MemoryAllocator {
 
 class ARROW_EXPORT ParquetReadSource : public ::parquet::RandomAccessSource {
  public:
-  ParquetReadSource(
-      const std::shared_ptr<io::RandomAccessFile>& file, ParquetAllocator* allocator);
+  explicit ParquetReadSource(ParquetAllocator* allocator);
+
+  // We need to ask for the file size on opening the file, and this can fail
+  Status Open(const std::shared_ptr<io::RandomAccessFile>& file);
 
   void Close() override;
   int64_t Tell() const override;
diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
index 7e724b31e3801..6615457c483f5 100644
--- a/cpp/src/arrow/parquet/parquet-io-test.cc
+++ b/cpp/src/arrow/parquet/parquet-io-test.cc
@@ -23,6 +23,7 @@
 #include "gtest/gtest.h"
 
 #include "arrow/parquet/io.h"
+#include "arrow/test-util.h"
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
@@ -147,9 +148,12 @@ TEST(TestParquetReadSource, Basics) {
   std::string data = "this is the data";
   auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str());
 
-  ParquetAllocator allocator;
+  ParquetAllocator allocator(default_memory_pool());
+
   auto file = std::make_shared<BufferReader>(data_buffer, data.size());
-  auto source = std::make_shared<ParquetReadSource>(file, &allocator);
+  auto source = std::make_shared<ParquetReadSource>(&allocator);
+
+  ASSERT_OK(source->Open(file));
 
   ASSERT_EQ(0, source->Tell());
   ASSERT_NO_THROW(source->Seek(5));
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
index c7c400e957343..e92967e5363d2 100644
--- a/cpp/src/arrow/parquet/reader.cc
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -23,6 +23,7 @@
 #include <vector>
 
 #include "arrow/column.h"
+#include "arrow/parquet/io.h"
 #include "arrow/parquet/schema.h"
 #include "arrow/parquet/utils.h"
 #include "arrow/schema.h"
@@ -35,6 +36,10 @@ using parquet::ColumnReader;
 using parquet::Repetition;
 using parquet::TypedColumnReader;
 
+// Help reduce verbosity
+using ParquetRAS = parquet::RandomAccessSource;
+using ParquetReader = parquet::ParquetFileReader;
+
 namespace arrow {
 namespace parquet {
 
@@ -181,6 +186,21 @@ FileReader::FileReader(
 
 FileReader::~FileReader() {}
 
+// Static ctor
+Status OpenFile(const std::shared_ptr<io::RandomAccessFile>& file,
+    ParquetAllocator* allocator, std::unique_ptr<FileReader>* reader) {
+  std::unique_ptr<ParquetReadSource> source(new ParquetReadSource(allocator));
+  RETURN_NOT_OK(source->Open(file));
+
+  // TODO(wesm): reader properties
+  std::unique_ptr<ParquetReader> pq_reader;
+  PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(std::move(source)));
+
+  // Use the same memory pool as the ParquetAllocator
+  reader->reset(new FileReader(allocator->pool(), std::move(pq_reader)));
+  return Status::OK();
+}
+
 Status FileReader::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out) {
   return impl_->GetFlatColumn(i, out);
 }
diff --git a/cpp/src/arrow/parquet/reader.h b/cpp/src/arrow/parquet/reader.h
index 2c8a9dfd025f0..f1492f64521cb 100644
--- a/cpp/src/arrow/parquet/reader.h
+++ b/cpp/src/arrow/parquet/reader.h
@@ -23,6 +23,8 @@
 #include "parquet/api/reader.h"
 #include "parquet/api/schema.h"
 
+#include "arrow/io/interfaces.h"
+#include "arrow/parquet/io.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -99,7 +101,7 @@ class ARROW_EXPORT FileReader {
   virtual ~FileReader();
 
  private:
-  class Impl;
+  class ARROW_NO_EXPORT Impl;
   std::unique_ptr<Impl> impl_;
 };
 
@@ -125,15 +127,20 @@ class ARROW_EXPORT FlatColumnReader {
   Status NextBatch(int batch_size, std::shared_ptr<Array>* out);
 
  private:
-  class Impl;
+  class ARROW_NO_EXPORT Impl;
   std::unique_ptr<Impl> impl_;
   explicit FlatColumnReader(std::unique_ptr<Impl> impl);
 
   friend class FileReader;
 };
 
-}  // namespace parquet
+// Helper function to create a file reader from an implementation of an Arrow
+// readable file
+ARROW_EXPORT
+Status OpenFile(const std::shared_ptr<io::RandomAccessFile>& file,
+    ParquetAllocator* allocator, std::unique_ptr<FileReader>* reader);
 
+}  // namespace parquet
 }  // namespace arrow
 
 #endif  // ARROW_PARQUET_READER_H
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
index 0139edd3bb8d9..f9514aa2ad2ff 100644
--- a/cpp/src/arrow/parquet/writer.cc
+++ b/cpp/src/arrow/parquet/writer.cc
@@ -35,7 +35,6 @@ using parquet::ParquetVersion;
 using parquet::schema::GroupNode;
 
 namespace arrow {
-
 namespace parquet {
 
 class FileWriter::Impl {
diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h
index 45d0fd59868e5..5aa1ba587176a 100644
--- a/cpp/src/arrow/parquet/writer.h
+++ b/cpp/src/arrow/parquet/writer.h
@@ -55,7 +55,7 @@ class ARROW_EXPORT FileWriter {
   MemoryPool* memory_pool() const;
 
  private:
-  class Impl;
+  class ARROW_NO_EXPORT Impl;
   std::unique_ptr<Impl> impl_;
 };
 
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index d0fb8f9f000b9..734ace6c923b4 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -19,11 +19,37 @@
 
 from pyarrow.includes.common cimport *
 
-cdef extern from "arrow/io/interfaces.h" nogil:
+cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
+    enum FileMode" arrow::io::FileMode::type":
+        FileMode_READ" arrow::io::FileMode::READ"
+        FileMode_WRITE" arrow::io::FileMode::WRITE"
+        FileMode_READWRITE" arrow::io::FileMode::READWRITE"
+
     enum ObjectType" arrow::io::ObjectType::type":
         ObjectType_FILE" arrow::io::ObjectType::FILE"
         ObjectType_DIRECTORY" arrow::io::ObjectType::DIRECTORY"
 
+    cdef cppclass FileBase:
+        CStatus Close()
+        CStatus Tell(int64_t* position)
+
+    cdef cppclass ReadableFile(FileBase):
+        CStatus GetSize(int64_t* size)
+        CStatus Read(int64_t nbytes, int64_t* bytes_read,
+                     uint8_t* buffer)
+
+        CStatus ReadAt(int64_t position, int64_t nbytes,
+                       int64_t* bytes_read, uint8_t* buffer)
+
+    cdef cppclass RandomAccessFile(ReadableFile):
+        CStatus Seek(int64_t position)
+
+    cdef cppclass WriteableFile(FileBase):
+        CStatus Write(const uint8_t* buffer, int64_t nbytes)
+        # CStatus Write(const uint8_t* buffer, int64_t nbytes,
+        #               int64_t* bytes_written)
+
+
 cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
     CStatus ConnectLibHdfs()
 
@@ -44,24 +70,11 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
         int64_t block_size
         int16_t permissions
 
-    cdef cppclass CHdfsFile:
-        CStatus Close()
-        CStatus Seek(int64_t position)
-        CStatus Tell(int64_t* position)
-
-    cdef cppclass HdfsReadableFile(CHdfsFile):
-        CStatus GetSize(int64_t* size)
-        CStatus Read(int64_t nbytes, int64_t* bytes_read,
-                     uint8_t* buffer)
-
-        CStatus ReadAt(int64_t position, int64_t nbytes,
-                       int64_t* bytes_read, uint8_t* buffer)
-
-    cdef cppclass HdfsWriteableFile(CHdfsFile):
-        CStatus Write(const uint8_t* buffer, int64_t nbytes)
+    cdef cppclass HdfsReadableFile(RandomAccessFile):
+        pass
 
-        CStatus Write(const uint8_t* buffer, int64_t nbytes,
-                      int64_t* bytes_written)
+    cdef cppclass HdfsWriteableFile(WriteableFile):
+        pass
 
     cdef cppclass CHdfsClient" arrow::io::HdfsClient":
         @staticmethod
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index a2f83ea5ea566..fe24f593e3294 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -19,6 +19,7 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport CSchema, CStatus, CTable, MemoryPool
+from pyarrow.includes.libarrow_io cimport RandomAccessFile
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
@@ -90,19 +91,36 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
             shared_ptr[WriterProperties] build()
 
 
+cdef extern from "arrow/parquet/io.h" namespace "arrow::parquet" nogil:
+    cdef cppclass ParquetAllocator:
+        ParquetAllocator()
+        ParquetAllocator(MemoryPool* pool)
+        MemoryPool* pool()
+        void set_pool(MemoryPool* pool)
+
+    cdef cppclass ParquetReadSource:
+        ParquetReadSource(ParquetAllocator* allocator)
+        Open(const shared_ptr[RandomAccessFile]& file)
+
+
 cdef extern from "arrow/parquet/reader.h" namespace "arrow::parquet" nogil:
+    CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
+                     ParquetAllocator* allocator,
+                     unique_ptr[FileReader]* reader)
+
     cdef cppclass FileReader:
         FileReader(MemoryPool* pool, unique_ptr[ParquetFileReader] reader)
         CStatus ReadFlatTable(shared_ptr[CTable]* out);
 
 
 cdef extern from "arrow/parquet/schema.h" namespace "arrow::parquet" nogil:
-    CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema, shared_ptr[CSchema]* out)
-    CStatus ToParquetSchema(const CSchema* arrow_schema, shared_ptr[SchemaDescriptor]* out)
+    CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema,
+                              shared_ptr[CSchema]* out)
+    CStatus ToParquetSchema(const CSchema* arrow_schema,
+                            shared_ptr[SchemaDescriptor]* out)
 
 
 cdef extern from "arrow/parquet/writer.h" namespace "arrow::parquet" nogil:
     cdef CStatus WriteFlatTable(const CTable* table, MemoryPool* pool,
             const shared_ptr[OutputStream]& sink, int64_t chunk_size,
             const shared_ptr[WriterProperties]& properties)
-
diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
new file mode 100644
index 0000000000000..b92af72704ae8
--- /dev/null
+++ b/python/pyarrow/io.pxd
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.libarrow_io cimport RandomAccessFile, WriteableFile
+
+
+cdef class NativeFileInterface:
+
+    # By implementing these "virtual" functions (all functions in Cython
+    # extension classes are technically virtual in the C++ sense)m we can
+    # expose the arrow::io abstract file interfaces to other components
+    # throughout the suite of Arrow C++ libraries
+    cdef read_handle(self, shared_ptr[RandomAccessFile]* file)
+    cdef write_handle(self, shared_ptr[WriteableFile]* file)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 071eea5ba6e60..b8bf883562060 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -164,7 +164,7 @@ cdef class HdfsClient:
                           .ListDirectory(c_path, &listing))
 
         cdef const HdfsPathInfo* info
-        for i in range(listing.size()):
+        for i in range(<int> listing.size()):
             info = &listing[i]
 
             # Try to trim off the hdfs://HOST:PORT piece
@@ -314,8 +314,15 @@ cdef class HdfsClient:
         f = self.open(path, 'rb', buffer_size=buffer_size)
         f.download(stream)
 
+cdef class NativeFileInterface:
 
-cdef class HdfsFile:
+    cdef read_handle(self, shared_ptr[RandomAccessFile]* file):
+        raise NotImplementedError
+
+    cdef write_handle(self, shared_ptr[WriteableFile]* file):
+        raise NotImplementedError
+
+cdef class HdfsFile(NativeFileInterface):
     cdef:
         shared_ptr[HdfsReadableFile] rd_file
         shared_ptr[HdfsWriteableFile] wr_file
@@ -357,6 +364,14 @@ cdef class HdfsFile:
         if self.is_readonly:
             raise IOError("only valid on writeonly files")
 
+    cdef read_handle(self, shared_ptr[RandomAccessFile]* file):
+        self._assert_readable()
+        file[0] = <shared_ptr[RandomAccessFile]> self.rd_file
+
+    cdef write_handle(self, shared_ptr[WriteableFile]* file):
+        self._assert_writeable()
+        file[0] = <shared_ptr[WriteableFile]> self.wr_file
+
     def size(self):
         cdef int64_t size
         self._assert_readable()
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 0b2b20880332b..ebba1a17ac742 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -20,34 +20,75 @@
 # cython: embedsignature = True
 
 from pyarrow.includes.libarrow cimport *
-cimport pyarrow.includes.pyarrow as pyarrow
 from pyarrow.includes.parquet cimport *
+from pyarrow.includes.libarrow_io cimport RandomAccessFile, WriteableFile
+cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.compat import tobytes
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_cstatus
+from pyarrow.io import NativeFileInterface
 from pyarrow.table cimport Table
 
-def read_table(filename, columns=None):
+from pyarrow.io cimport NativeFileInterface
+
+import six
+
+
+cdef class ParquetReader:
+    cdef:
+        ParquetAllocator allocator
+        unique_ptr[FileReader] reader
+
+    def __cinit__(self):
+        self.allocator.set_pool(default_memory_pool())
+
+    cdef open_local_file(self, file_path):
+        cdef c_string path = tobytes(file_path)
+
+        # Must be in one expression to avoid calling std::move which is not
+        # possible in Cython (due to missing rvalue support)
+
+        # TODO(wesm): ParquetFileReader::OpenFIle can throw?
+        self.reader = unique_ptr[FileReader](
+            new FileReader(default_memory_pool(),
+                           ParquetFileReader.OpenFile(path)))
+
+    cdef open_native_file(self, NativeFileInterface file):
+        cdef shared_ptr[RandomAccessFile] cpp_handle
+        file.read_handle(&cpp_handle)
+
+        check_cstatus(OpenFile(cpp_handle, &self.allocator, &self.reader))
+
+    def read_all(self):
+        cdef:
+            Table table = Table()
+            shared_ptr[CTable] ctable
+
+        with nogil:
+            check_cstatus(self.reader.get()
+                          .ReadFlatTable(&ctable))
+
+        table.init(ctable)
+        return table
+
+
+def read_table(source, columns=None):
     """
     Read a Table from Parquet format
     Returns
     -------
     table: pyarrow.Table
     """
-    cdef unique_ptr[FileReader] reader
-    cdef Table table = Table()
-    cdef shared_ptr[CTable] ctable
-
-    # Must be in one expression to avoid calling std::move which is not possible
-    # in Cython (due to missing rvalue support)
-    reader = unique_ptr[FileReader](new FileReader(default_memory_pool(),
-        ParquetFileReader.OpenFile(tobytes(filename))))
-    with nogil:
-        check_cstatus(reader.get().ReadFlatTable(&ctable))
+    cdef ParquetReader reader = ParquetReader()
+
+    if isinstance(source, six.string_types):
+        reader.open_local_file(source)
+    elif isinstance(source, NativeFileInterface):
+        reader.open_native_file(source)
+
+    return reader.read_all()
 
-    table.init(ctable)
-    return table
 
 def write_table(table, filename, chunk_size=None, version=None):
     """
@@ -84,4 +125,3 @@ def write_table(table, filename, chunk_size=None, version=None):
     with nogil:
         check_cstatus(WriteFlatTable(ctable_, default_memory_pool(), sink,
             chunk_size_, properties_builder.build()))
-

From a2fb756a43441a72e10ae74fa0e483e01bc5917e Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Tue, 19 Jul 2016 13:39:48 -0700
Subject: [PATCH 0100/1644] ARROW-241: Add missing implementation for
 splitAndTransfer in UnionVector

Use simple implementation that actually just copies
---
 java/vector/src/main/codegen/templates/UnionVector.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 6042a5bf68352..482944828ade1 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -264,7 +264,11 @@ public void transfer() {
 
     @Override
     public void splitAndTransfer(int startIndex, int length) {
-
+      to.allocateNew();
+      for (int i = 0; i < length; i++) {
+        to.copyFromSafe(startIndex + i, i, org.apache.arrow.vector.complex.UnionVector.this);
+      }
+      to.getMutator().setValueCount(length);
     }
 
     @Override

From dc79ceb05c05e626e2324863cfc3f386ecccce90 Mon Sep 17 00:00:00 2001
From: Jihoon Son <jihoonson@apache.org>
Date: Mon, 1 Aug 2016 11:29:02 -0700
Subject: [PATCH 0101/1644] ARROW-244: Some global APIs of IPC module should be
 visible to the outside

Author: Jihoon Son <jihoonson@apache.org>

Closes #109 from jihoonson/ARROW-244 and squashes the following commits:

51d9a87 [Jihoon Son] Make line length shorter than 90
2da5466 [Jihoon Son] Make some APIs of IPC module visible
---
 cpp/src/arrow/ipc/adapter.h | 11 +++++++----
 cpp/src/arrow/ipc/memory.h  |  5 +++--
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 0d2b77f5acefe..a34a5c4fcc99f 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -24,6 +24,8 @@
 #include <cstdint>
 #include <memory>
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Array;
@@ -54,20 +56,21 @@ constexpr int kMaxIpcRecursionDepth = 64;
 //
 // Finally, the memory offset to the start of the metadata / data header is
 // returned in an out-variable
-Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
-    int64_t* header_offset, int max_recursion_depth = kMaxIpcRecursionDepth);
+ARROW_EXPORT Status WriteRowBatch(MemorySource* dst, const RowBatch* batch,
+    int64_t position, int64_t* header_offset,
+    int max_recursion_depth = kMaxIpcRecursionDepth);
 
 // int64_t GetRowBatchMetadata(const RowBatch* batch);
 
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the row batch. This involves generating the complete serialized
 // Flatbuffers metadata.
-Status GetRowBatchSize(const RowBatch* batch, int64_t* size);
+ARROW_EXPORT Status GetRowBatchSize(const RowBatch* batch, int64_t* size);
 
 // ----------------------------------------------------------------------
 // "Read" path; does not copy data if the MemorySource does not
 
-class RowBatchReader {
+class ARROW_EXPORT RowBatchReader {
  public:
   static Status Open(
       MemorySource* source, int64_t position, std::shared_ptr<RowBatchReader>* out);
diff --git a/cpp/src/arrow/ipc/memory.h b/cpp/src/arrow/ipc/memory.h
index c6fd7a718991b..377401d85c00a 100644
--- a/cpp/src/arrow/ipc/memory.h
+++ b/cpp/src/arrow/ipc/memory.h
@@ -25,6 +25,7 @@
 #include <string>
 
 #include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -69,7 +70,7 @@ class BufferOutputStream : public OutputStream {
   int64_t position_;
 };
 
-class MemorySource {
+class ARROW_EXPORT MemorySource {
  public:
   // Indicates the access permissions of the memory source
   enum AccessMode { READ_ONLY, READ_WRITE };
@@ -100,7 +101,7 @@ class MemorySource {
 };
 
 // A memory source that uses memory-mapped files for memory interactions
-class MemoryMappedSource : public MemorySource {
+class ARROW_EXPORT MemoryMappedSource : public MemorySource {
  public:
   static Status Open(const std::string& path, AccessMode access_mode,
       std::shared_ptr<MemoryMappedSource>* out);

From 356d015bb7de3a12167ac8ea02dbda9bbdc8c27f Mon Sep 17 00:00:00 2001
From: MechCoder <mks542@nyu.edu>
Date: Wed, 13 Jul 2016 17:24:26 -0700
Subject: [PATCH 0102/1644] ARROW-240: Provide more detailed installation
 instructions for pyarrow. Closes

---
 python/README.md | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/python/README.md b/python/README.md
index c79fa9786f476..bafe71b05ec22 100644
--- a/python/README.md
+++ b/python/README.md
@@ -4,11 +4,40 @@ This library provides a Pythonic API wrapper for the reference Arrow C++
 implementation, along with tools for interoperability with pandas, NumPy, and
 other traditional Python scientific computing packages.
 
-#### Development details
+### Development details
 
 This project is layered in two pieces:
 
 * pyarrow, a C++ library for easier interoperability between Arrow C++, NumPy,
   and pandas
 * Cython extensions and pure Python code under arrow/ which expose Arrow C++
-  and pyarrow to pure Python users
\ No newline at end of file
+  and pyarrow to pure Python users
+
+#### PyArrow Dependencies:
+These are the various projects that PyArrow depends on.
+
+1. **g++ and gcc Version >= 4.8**
+2. **cmake > 2.8.6**
+3. **boost**
+4. **Parquet-cpp**
+
+  The preferred way to install parquet-cpp is to use conda.
+  You need to set the ``PARQUET_HOME`` environment variable to where parquet-cpp is installed.
+  ```bash
+  conda install -y --channel apache/channel/dev parquet-cpp
+  ```
+5. **Arrow-cpp and its dependencies***
+
+   The Arrow C++ library must be built with all options enabled and installed with ``ARROW_HOME`` environment variable set to
+   the installation location. Look at (https://github.com/apache/arrow/blob/master/cpp/README.md) for
+   instructions. Alternatively you could just install arrow-cpp
+   from conda.
+  ```bash
+  conda install arrow-cpp -c apache/channel/dev
+  ```
+6. **Python dependencies: numpy, pandas, cython, pytest**
+
+#### Install pyarrow
+  ```bash
+  python setup.py build_ext --inplace
+  ```

From 3a2dfba59a2482226cc3c49a11a779dd9ce3dfd7 Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent.goujon@online.fr>
Date: Mon, 1 Aug 2016 16:31:54 -0700
Subject: [PATCH 0103/1644] ARROW-101: Fix java compiler warnings

Fixes several warnings emitted by java compiler regarding the use of raw types and unclosed resources.

Author: Laurent Goujon <laurent.goujon@online.fr>

Closes #60 from laurentgo/laurent/fix-generic-warnings and squashes the following commits:

96ccc67 [Laurent Goujon] [ARROW-101] Fix java compiler resources warnings
61bde83 [Laurent Goujon] [ARROW-101] Fix java compiler rawtypes warnings
---
 .../src/main/java/org/apache/arrow/vector/ZeroVector.java | 5 +++--
 .../arrow/vector/complex/impl/PromotableWriter.java       | 8 ++++----
 .../org/apache/arrow/vector/util/JsonStringArrayList.java | 2 +-
 .../org/apache/arrow/vector/util/JsonStringHashMap.java   | 2 +-
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 78de8706fb7d4..c94e8d1db090c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -19,6 +19,7 @@
 
 import io.netty.buffer.ArrowBuf;
 
+import java.util.Collections;
 import java.util.Iterator;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -109,8 +110,8 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
 //  }
 
   @Override
-  public Iterator iterator() {
-    return Iterators.emptyIterator();
+  public Iterator<ValueVector> iterator() {
+    return Collections.emptyIterator();
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index ea62e02360802..45509f688ba88 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -85,16 +85,16 @@ private void setWriter(ValueVector v) {
     state = State.SINGLE;
     vector = v;
     type = v.getField().getType().getMinorType();
-    Class writerClass = BasicTypeHelper
+    Class<?> writerClass = BasicTypeHelper
         .getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
     if (writerClass.equals(SingleListWriter.class)) {
       writerClass = UnionListWriter.class;
     }
-    Class vectorClass = BasicTypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField()
+    Class<?> vectorClass = BasicTypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField()
         .getDataMode());
     try {
-      Constructor constructor = null;
-      for (Constructor c : writerClass.getConstructors()) {
+      Constructor<?> constructor = null;
+      for (Constructor<?> c : writerClass.getConstructors()) {
         if (c.getParameterTypes().length == 3) {
           constructor = c;
         }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
index 7aeaa12ef9fcf..6291bfeaee666 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
@@ -42,7 +42,7 @@ public boolean equals(Object obj) {
     if (!(obj instanceof List)) {
       return false;
     }
-    List other = (List) obj;
+    List<?> other = (List<?>) obj;
     return this.size() == other.size() && this.containsAll(other);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
index 750dd592aa49c..e8ce5221eebd9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
@@ -46,7 +46,7 @@ public boolean equals(Object obj) {
     if (!(obj instanceof Map)) {
       return false;
     }
-    Map other = (Map) obj;
+    Map<?, ?> other = (Map<?, ?>) obj;
     if (this.size() != other.size()) {
       return false;
     }

From 56835c338f01aebcace01312e82431306e7fd578 Mon Sep 17 00:00:00 2001
From: adeneche <adeneche@apache.org>
Date: Mon, 1 Aug 2016 15:28:08 -0700
Subject: [PATCH 0104/1644] ARROW-246: [Java] UnionVector doesn't call
 allocateNew() when creating it's vectorType

---
 .../main/codegen/templates/UnionVector.java   |   2 +
 .../arrow/vector/DirtyBufferAllocator.java    | 120 ++++++++++++++++++
 .../apache/arrow/vector/TestUnionVector.java  |  88 +++++++++++++
 3 files changed, 210 insertions(+)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 482944828ade1..692436d12854d 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -73,6 +73,8 @@ public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack
     this.allocator = allocator;
     this.internalMap = new MapVector("internal", allocator, callBack);
     this.typeVector = internalMap.addOrGet("types", new MajorType(MinorType.UINT1, DataMode.REQUIRED), UInt1Vector.class);
+    this.typeVector.allocateNew();
+    this.typeVector.zeroVector();
     this.field.addChild(internalMap.getField().clone());
     this.majorType = field.getType();
     this.callBack = callBack;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java b/java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java
new file mode 100644
index 0000000000000..cc6b9ec51d61c
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.AllocationReservation;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.BufferManager;
+
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.ByteBufAllocator;
+
+/**
+ * Wrapper around a buffer delegate that populates any allocated buffer with a constant
+ * value. Useful for testing if value vectors are properly resetting their buffers.
+ */
+public class DirtyBufferAllocator implements BufferAllocator {
+
+  private final BufferAllocator delegate;
+  private final byte fillValue;
+
+  DirtyBufferAllocator(final BufferAllocator delegate, final byte fillValue) {
+    this.delegate = delegate;
+    this.fillValue = fillValue;
+  }
+
+  @Override
+  public ArrowBuf buffer(int size) {
+    return buffer(size, null);
+  }
+
+  @Override
+  public ArrowBuf buffer(int size, BufferManager manager) {
+    ArrowBuf buffer = delegate.buffer(size, manager);
+    // contaminate the buffer
+    for (int i = 0; i < buffer.capacity(); i++) {
+      buffer.setByte(i, fillValue);
+    }
+
+    return buffer;
+  }
+
+  @Override
+  public ByteBufAllocator getAsByteBufAllocator() {
+    return delegate.getAsByteBufAllocator();
+  }
+
+  @Override
+  public BufferAllocator newChildAllocator(String name, long initReservation, long maxAllocation) {
+    return delegate.newChildAllocator(name, initReservation, maxAllocation);
+  }
+
+  @Override
+  public void close() {
+    delegate.close();
+  }
+
+  @Override
+  public long getAllocatedMemory() {
+    return delegate.getAllocatedMemory();
+  }
+
+  @Override
+  public void setLimit(long newLimit) {
+    delegate.setLimit(newLimit);
+  }
+
+  @Override
+  public long getLimit() {
+    return delegate.getLimit();
+  }
+
+  @Override
+  public long getPeakMemoryAllocation() {
+    return delegate.getPeakMemoryAllocation();
+  }
+
+  @Override
+  public AllocationReservation newReservation() {
+    return delegate.newReservation();
+  }
+
+  @Override
+  public ArrowBuf getEmpty() {
+    return delegate.getEmpty();
+  }
+
+  @Override
+  public String getName() {
+    return delegate.getName();
+  }
+
+  @Override
+  public boolean isOverLimit() {
+    return delegate.isOverLimit();
+  }
+
+  @Override
+  public String toVerboseString() {
+    return delegate.toVerboseString();
+  }
+
+  @Override
+  public void assertOpen() {
+    delegate.assertOpen();
+  }}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
new file mode 100644
index 0000000000000..8f19b3191ba15
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.holders.NullableUInt4Holder;
+import org.apache.arrow.vector.holders.UInt4Holder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestUnionVector {
+  private final static String EMPTY_SCHEMA_PATH = "";
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testUnionVector() throws Exception {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+
+    final BufferAllocator alloc = new DirtyBufferAllocator(allocator, (byte) 100);
+
+    UnionVector unionVector = new UnionVector(field, alloc, null);
+
+    final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder();
+    uInt4Holder.value = 100;
+    uInt4Holder.isSet = 1;
+
+    try {
+      // write some data
+      final UnionVector.Mutator mutator = unionVector.getMutator();
+      mutator.setType(0, Types.MinorType.UINT4);
+      mutator.setSafe(0, uInt4Holder);
+      mutator.setType(2, Types.MinorType.UINT4);
+      mutator.setSafe(2, uInt4Holder);
+      mutator.setValueCount(4);
+
+      // check that what we wrote is correct
+      final UnionVector.Accessor accessor = unionVector.getAccessor();
+      assertEquals(4, accessor.getValueCount());
+
+      assertEquals(false, accessor.isNull(0));
+      assertEquals(100, accessor.getObject(0));
+
+      assertEquals(true, accessor.isNull(1));
+
+      assertEquals(false, accessor.isNull(2));
+      assertEquals(100, accessor.getObject(2));
+
+      assertEquals(true, accessor.isNull(3));
+
+    } finally {
+      unionVector.clear();
+    }
+  }
+
+}

From 5df7d4dee5fd57e91d9bb83f44f2269f61b79fb3 Mon Sep 17 00:00:00 2001
From: Jihoon Son <jihoonson@apache.org>
Date: Thu, 4 Aug 2016 15:29:01 -0700
Subject: [PATCH 0105/1644] ARROW-247: Missing explicit destructor in
 RowBatchReader causes an incomplete type error

Author: Jihoon Son <jihoonson@apache.org>

Closes #111 from jihoonson/ARROW-247 and squashes the following commits:

cc7281c [Jihoon Son] Make destructor virtual
795d3d3 [Jihoon Son] Merge branch 'master' of https://github.com/apache/arrow into ARROW-247
df297ef [Jihoon Son] Trigger travis
65d64c8 [Jihoon Son] Make the comment into two line
9555260 [Jihoon Son] Add a comment
f671a32 [Jihoon Son] Add explicit destructor for RowBatchReader
---
 cpp/src/arrow/ipc/adapter.cc | 4 ++++
 cpp/src/arrow/ipc/adapter.h  | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index bac1172700615..84f7830092cf4 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -369,6 +369,10 @@ Status RowBatchReader::Open(MemorySource* source, int64_t position,
   return Status::OK();
 }
 
+// Here the explicit destructor is required for compilers to be aware of
+// the complete information of RowBatchReader::Impl class
+RowBatchReader::~RowBatchReader() {}
+
 Status RowBatchReader::GetRowBatch(
     const std::shared_ptr<Schema>& schema, std::shared_ptr<RowBatch>* out) {
   return impl_->AssembleBatch(schema, out);
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index a34a5c4fcc99f..6231af66aa180 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -78,6 +78,8 @@ class ARROW_EXPORT RowBatchReader {
   static Status Open(MemorySource* source, int64_t position, int max_recursion_depth,
       std::shared_ptr<RowBatchReader>* out);
 
+  virtual ~RowBatchReader();
+
   // Reassemble the row batch. A Schema is required to be able to construct the
   // right array containers
   Status GetRowBatch(

From 34e7f48cb71428c4d78cf00d8fdf0045532d6607 Mon Sep 17 00:00:00 2001
From: adeneche <adeneche@apache.org>
Date: Fri, 5 Aug 2016 10:26:47 -0700
Subject: [PATCH 0106/1644] ARROW-250: Fix for ARROW-246 may cause memory leaks

this closes #112
---
 .../main/codegen/templates/UnionVector.java   |   3 +-
 .../vector/complex/impl/PromotableWriter.java |   1 +
 .../arrow/vector/DirtyBufferAllocator.java    | 120 ------------------
 .../arrow/vector/DirtyRootAllocator.java      |  53 ++++++++
 .../apache/arrow/vector/TestUnionVector.java  |  14 +-
 .../complex/impl/TestPromotableWriter.java    |  98 ++++++++++++++
 6 files changed, 157 insertions(+), 132 deletions(-)
 delete mode 100644 java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/DirtyRootAllocator.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 692436d12854d..0f089b7e91537 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -73,8 +73,6 @@ public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack
     this.allocator = allocator;
     this.internalMap = new MapVector("internal", allocator, callBack);
     this.typeVector = internalMap.addOrGet("types", new MajorType(MinorType.UINT1, DataMode.REQUIRED), UInt1Vector.class);
-    this.typeVector.allocateNew();
-    this.typeVector.zeroVector();
     this.field.addChild(internalMap.getField().clone());
     this.majorType = field.getType();
     this.callBack = callBack;
@@ -193,6 +191,7 @@ public int getValueCapacity() {
 
   @Override
   public void close() {
+    clear();
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 45509f688ba88..462ec9dd86a9b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -155,6 +155,7 @@ private FieldWriter promoteToUnion() {
     tp.transfer();
     if (parentContainer != null) {
       unionVector = parentContainer.addOrGet(name, new MajorType(MinorType.UNION, DataMode.OPTIONAL), UnionVector.class);
+      unionVector.allocateNew();
     } else if (listVector != null) {
       unionVector = listVector.promoteToUnion();
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java b/java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java
deleted file mode 100644
index cc6b9ec51d61c..0000000000000
--- a/java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector;
-
-import org.apache.arrow.memory.AllocationReservation;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.BufferManager;
-
-import io.netty.buffer.ArrowBuf;
-import io.netty.buffer.ByteBufAllocator;
-
-/**
- * Wrapper around a buffer delegate that populates any allocated buffer with a constant
- * value. Useful for testing if value vectors are properly resetting their buffers.
- */
-public class DirtyBufferAllocator implements BufferAllocator {
-
-  private final BufferAllocator delegate;
-  private final byte fillValue;
-
-  DirtyBufferAllocator(final BufferAllocator delegate, final byte fillValue) {
-    this.delegate = delegate;
-    this.fillValue = fillValue;
-  }
-
-  @Override
-  public ArrowBuf buffer(int size) {
-    return buffer(size, null);
-  }
-
-  @Override
-  public ArrowBuf buffer(int size, BufferManager manager) {
-    ArrowBuf buffer = delegate.buffer(size, manager);
-    // contaminate the buffer
-    for (int i = 0; i < buffer.capacity(); i++) {
-      buffer.setByte(i, fillValue);
-    }
-
-    return buffer;
-  }
-
-  @Override
-  public ByteBufAllocator getAsByteBufAllocator() {
-    return delegate.getAsByteBufAllocator();
-  }
-
-  @Override
-  public BufferAllocator newChildAllocator(String name, long initReservation, long maxAllocation) {
-    return delegate.newChildAllocator(name, initReservation, maxAllocation);
-  }
-
-  @Override
-  public void close() {
-    delegate.close();
-  }
-
-  @Override
-  public long getAllocatedMemory() {
-    return delegate.getAllocatedMemory();
-  }
-
-  @Override
-  public void setLimit(long newLimit) {
-    delegate.setLimit(newLimit);
-  }
-
-  @Override
-  public long getLimit() {
-    return delegate.getLimit();
-  }
-
-  @Override
-  public long getPeakMemoryAllocation() {
-    return delegate.getPeakMemoryAllocation();
-  }
-
-  @Override
-  public AllocationReservation newReservation() {
-    return delegate.newReservation();
-  }
-
-  @Override
-  public ArrowBuf getEmpty() {
-    return delegate.getEmpty();
-  }
-
-  @Override
-  public String getName() {
-    return delegate.getName();
-  }
-
-  @Override
-  public boolean isOverLimit() {
-    return delegate.isOverLimit();
-  }
-
-  @Override
-  public String toVerboseString() {
-    return delegate.toVerboseString();
-  }
-
-  @Override
-  public void assertOpen() {
-    delegate.assertOpen();
-  }}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/DirtyRootAllocator.java b/java/vector/src/test/java/org/apache/arrow/vector/DirtyRootAllocator.java
new file mode 100644
index 0000000000000..f775f1d2d67af
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/DirtyRootAllocator.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferManager;
+import org.apache.arrow.memory.RootAllocator;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Root allocator that returns buffers pre-filled with a given value.<br>
+ * Useful for testing if value vectors are properly zeroing their buffers.
+ */
+public class DirtyRootAllocator extends RootAllocator {
+
+  private final byte fillValue;
+
+  public DirtyRootAllocator(final long limit, final byte fillValue) {
+    super(limit);
+    this.fillValue = fillValue;
+  }
+
+  @Override
+  public ArrowBuf buffer(int size) {
+    return buffer(size, null);
+  }
+
+  @Override
+  public ArrowBuf buffer(int size, BufferManager manager) {
+    ArrowBuf buffer = super.buffer(size, manager);
+    // contaminate the buffer
+    for (int i = 0; i < buffer.capacity(); i++) {
+      buffer.setByte(i, fillValue);
+    }
+
+    return buffer;
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index 8f19b3191ba15..e4d28c3f88ca6 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -20,7 +20,6 @@
 import static org.junit.Assert.assertEquals;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.holders.NullableUInt4Holder;
 import org.apache.arrow.vector.holders.UInt4Holder;
@@ -37,7 +36,7 @@ public class TestUnionVector {
 
   @Before
   public void init() {
-    allocator = new RootAllocator(Long.MAX_VALUE);
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
   }
 
   @After
@@ -49,15 +48,13 @@ public void terminate() throws Exception {
   public void testUnionVector() throws Exception {
     final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
 
-    final BufferAllocator alloc = new DirtyBufferAllocator(allocator, (byte) 100);
-
-    UnionVector unionVector = new UnionVector(field, alloc, null);
-
     final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder();
     uInt4Holder.value = 100;
     uInt4Holder.isSet = 1;
 
-    try {
+    try (UnionVector unionVector = new UnionVector(field, allocator, null)) {
+      unionVector.allocateNew();
+
       // write some data
       final UnionVector.Mutator mutator = unionVector.getMutator();
       mutator.setType(0, Types.MinorType.UINT4);
@@ -79,9 +76,6 @@ public void testUnionVector() throws Exception {
       assertEquals(100, accessor.getObject(2));
 
       assertEquals(true, accessor.isNull(3));
-
-    } finally {
-      unionVector.clear();
     }
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
new file mode 100644
index 0000000000000..4c24444d81d18
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.DirtyRootAllocator;
+import org.apache.arrow.vector.complex.AbstractMapVector;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.holders.UInt4Holder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestPromotableWriter {
+  private final static String EMPTY_SCHEMA_PATH = "";
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testPromoteToUnion() throws Exception {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+
+    try (final AbstractMapVector container = new MapVector(field, allocator, null);
+         final MapVector v = container.addOrGet("test", MapVector.TYPE, MapVector.class);
+         final PromotableWriter writer = new PromotableWriter(v, container)) {
+
+      container.allocateNew();
+
+      writer.start();
+
+      writer.setPosition(0);
+      writer.bit("A").writeBit(0);
+
+      writer.setPosition(1);
+      writer.bit("A").writeBit(1);
+
+      writer.setPosition(2);
+      writer.integer("A").writeInt(10);
+
+      // we don't write anything in 3
+
+      writer.setPosition(4);
+      writer.integer("A").writeInt(100);
+
+      writer.end();
+
+      container.getMutator().setValueCount(5);
+
+      final UnionVector uv = v.getChild("A", UnionVector.class);
+      final UnionVector.Accessor accessor = uv.getAccessor();
+
+      assertFalse("0 shouldn't be null", accessor.isNull(0));
+      assertEquals(false, accessor.getObject(0));
+
+      assertFalse("1 shouldn't be null", accessor.isNull(1));
+      assertEquals(true, accessor.getObject(1));
+
+      assertFalse("2 shouldn't be null", accessor.isNull(2));
+      assertEquals(10, accessor.getObject(2));
+
+      assertTrue("3 should be null", accessor.isNull(3));
+
+      assertFalse("4 shouldn't be null", accessor.isNull(4));
+      assertEquals(100, accessor.getObject(4));
+    }
+  }
+}

From 2742d37cc3f890ffd68ba46920240c18ae5528ae Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 12 Aug 2016 15:58:20 -0700
Subject: [PATCH 0107/1644] ARROW-254: remove Bit type as it is redundant with
 Boolean

The only use of Bit is for the nullability (or validity) vector which is best understood as a boolean type.
We should remove it as it is not used.

Author: Julien Le Dem <julien@dremio.com>

Closes #116 from julienledem/arrow_254_remove_bit_type and squashes the following commits:

1cada12 [Julien Le Dem] ARROW-254: remove Bit type
---
 cpp/src/arrow/ipc/metadata-internal.cc | 2 --
 format/Message.fbs                     | 4 ----
 2 files changed, 6 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 1b1d50f96eaf5..5c439120b173a 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -99,8 +99,6 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       return Status::Invalid("Type metadata cannot be none");
     case flatbuf::Type_Int:
       return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
-    case flatbuf::Type_Bit:
-      return Status::NotImplemented("Type is not implemented");
     case flatbuf::Type_FloatingPoint:
       return FloatFromFlatuffer(
           static_cast<const flatbuf::FloatingPoint*>(type_data), out);
diff --git a/format/Message.fbs b/format/Message.fbs
index fc849eedf791a..e0a956c3b257a 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -20,9 +20,6 @@ table Union {
   mode: UnionMode;
 }
 
-table Bit {
-}
-
 table Int {
   bitWidth: int; // 1 to 64
   is_signed: bool;
@@ -62,7 +59,6 @@ table JSONScalar {
 
 union Type {
   Int,
-  Bit,
   FloatingPoint,
   Binary,
   Utf8,

From dc01f099d966b92f4de7679b4a1caf97c363e08e Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 12 Aug 2016 16:00:18 -0700
Subject: [PATCH 0108/1644] ARROW-253: restrict ints to 8, 16, 32, or 64 bits
 in V1

Author: Julien Le Dem <julien@dremio.com>

Closes #115 from julienledem/arrow_253_int_8_16_32_64 and squashes the following commits:

d8df119 [Julien Le Dem] ARROW-253: restrict ints to 8, 16, 32, or 64 bits in V1
---
 cpp/src/arrow/ipc/metadata-internal.cc | 9 ++++-----
 format/Message.fbs                     | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 5c439120b173a..e6b47de70ed70 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -55,12 +55,12 @@ const std::shared_ptr<DataType> DOUBLE = std::make_shared<DoubleType>();
 
 static Status IntFromFlatbuffer(
     const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
-  if (int_data->bitWidth() % 8 != 0) {
-    return Status::NotImplemented("Integers not in cstdint are not implemented");
-  }
   if (int_data->bitWidth() > 64) {
     return Status::NotImplemented("Integers with more than 64 bits not implemented");
   }
+  if (int_data->bitWidth() < 8) {
+    return Status::NotImplemented("Integers with less than 8 bits not implemented");
+  }
 
   switch (int_data->bitWidth()) {
     case 8:
@@ -76,8 +76,7 @@ static Status IntFromFlatbuffer(
       *out = int_data->is_signed() ? INT64 : UINT64;
       break;
     default:
-      *out = nullptr;
-      break;
+      return Status::NotImplemented("Integers not in cstdint are not implemented");
   }
   return Status::OK();
 }
diff --git a/format/Message.fbs b/format/Message.fbs
index e0a956c3b257a..6a351b9dbf0a6 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -21,7 +21,7 @@ table Union {
 }
 
 table Int {
-  bitWidth: int; // 1 to 64
+  bitWidth: int; // restricted to 8, 16, 32, and 64 in v1
   is_signed: bool;
 }
 

From e8724f8379324c59d285d2380005577a49290c42 Mon Sep 17 00:00:00 2001
From: Jihoon Son <jihoonson@apache.org>
Date: Sat, 13 Aug 2016 13:50:02 +0900
Subject: [PATCH 0109/1644] ARROW-260: Fix flaky oversized tests

- Limit max allocation bytes for a vector as 1 KB
- Remove System.setProperty() in TestValueVector
- Move tests which test OversizedAllocationException for ValueVector into a separate class and add a disclaimer
- Add a comment for the new test

This closes #118.
---
 java/pom.xml                                  |   3 +
 ...TestOversizedAllocationForValueVector.java | 137 ++++++++++++++++++
 .../apache/arrow/vector/TestValueVector.java  | 131 +----------------
 3 files changed, 145 insertions(+), 126 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java

diff --git a/java/pom.xml b/java/pom.xml
index ea42894fda22e..71f59caf2798e 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -303,6 +303,9 @@
             <systemPropertyVariables>
               <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
             </systemPropertyVariables>
+            <!-- Note: changing the below configuration might increase the max allocation size for a vector
+            which in turn can cause OOM. -->
+            <argLine>-Darrow.vector.max_allocation_bytes=1048576</argLine>
           </configuration>
         </plugin>
         <plugin>
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
new file mode 100644
index 0000000000000..4dee86c9d595a
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.holders.UInt4Holder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This class tests that OversizedAllocationException occurs when a large memory is allocated for a vector.
+ * Typically, arrow allows the allocation of the size of at most Integer.MAX_VALUE, but this might cause OOM in tests.
+ * Thus, the max allocation size is limited to 1 KB in this class. Please see the surefire option in pom.xml.
+ */
+public class TestOversizedAllocationForValueVector {
+
+  private final static String EMPTY_SCHEMA_PATH = "";
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test(expected = OversizedAllocationException.class)
+  public void testFixedVectorReallocation() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    final UInt4Vector vector = new UInt4Vector(field, allocator);
+    // edge case 1: buffer size = max value capacity
+    final int expectedValueCapacity = BaseValueVector.MAX_ALLOCATION_SIZE / 4;
+    try {
+      vector.allocateNew(expectedValueCapacity);
+      assertEquals(expectedValueCapacity, vector.getValueCapacity());
+      vector.reAlloc();
+      assertEquals(expectedValueCapacity * 2, vector.getValueCapacity());
+    } finally {
+      vector.close();
+    }
+
+    // common case: value count < max value capacity
+    try {
+      vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 8);
+      vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION
+      vector.reAlloc(); // this should throw an IOOB
+    } finally {
+      vector.close();
+    }
+  }
+
+  @Test(expected = OversizedAllocationException.class)
+  public void testBitVectorReallocation() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    final BitVector vector = new BitVector(field, allocator);
+    // edge case 1: buffer size ~ max value capacity
+    final int expectedValueCapacity = 1 << 29;
+    try {
+      vector.allocateNew(expectedValueCapacity);
+      assertEquals(expectedValueCapacity, vector.getValueCapacity());
+      vector.reAlloc();
+      assertEquals(expectedValueCapacity * 2, vector.getValueCapacity());
+    } finally {
+      vector.close();
+    }
+
+    // common: value count < MAX_VALUE_ALLOCATION
+    try {
+      vector.allocateNew(expectedValueCapacity);
+      for (int i=0; i<3;i++) {
+        vector.reAlloc(); // expand buffer size
+      }
+      assertEquals(Integer.MAX_VALUE, vector.getValueCapacity());
+      vector.reAlloc(); // buffer size ~ max allocation
+      assertEquals(Integer.MAX_VALUE, vector.getValueCapacity());
+      vector.reAlloc(); // overflow
+    } finally {
+      vector.close();
+    }
+  }
+
+
+  @Test(expected = OversizedAllocationException.class)
+  public void testVariableVectorReallocation() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    final VarCharVector vector = new VarCharVector(field, allocator);
+    // edge case 1: value count = MAX_VALUE_ALLOCATION
+    final int expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE;
+    final int expectedOffsetSize = 10;
+    try {
+      vector.allocateNew(expectedAllocationInBytes, 10);
+      assertTrue(expectedOffsetSize <= vector.getValueCapacity());
+      assertTrue(expectedAllocationInBytes <= vector.getBuffer().capacity());
+      vector.reAlloc();
+      assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity());
+      assertTrue(expectedAllocationInBytes * 2 <= vector.getBuffer().capacity());
+    } finally {
+      vector.close();
+    }
+
+    // common: value count < MAX_VALUE_ALLOCATION
+    try {
+      vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 2, 0);
+      vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION
+      vector.reAlloc(); // this tests if it overflows
+    } finally {
+      vector.close();
+    }
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index b5c4509c8b540..ce091ab1ed06b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -17,29 +17,13 @@
  */
 package org.apache.arrow.vector;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.charset.Charset;
-
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.RepeatedListVector;
 import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.holders.BitHolder;
-import org.apache.arrow.vector.holders.IntHolder;
-import org.apache.arrow.vector.holders.NullableFloat4Holder;
-import org.apache.arrow.vector.holders.NullableUInt4Holder;
-import org.apache.arrow.vector.holders.NullableVar16CharHolder;
-import org.apache.arrow.vector.holders.NullableVarCharHolder;
-import org.apache.arrow.vector.holders.RepeatedFloat4Holder;
-import org.apache.arrow.vector.holders.RepeatedIntHolder;
-import org.apache.arrow.vector.holders.RepeatedVarBinaryHolder;
-import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.holders.VarCharHolder;
+import org.apache.arrow.vector.holders.*;
 import org.apache.arrow.vector.types.MaterializedField;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -47,40 +31,19 @@
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.ExternalResource;
+
+import java.nio.charset.Charset;
+
+import static org.junit.Assert.*;
 
 
 public class TestValueVector {
-  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestValueVector.class);
 
   private final static String EMPTY_SCHEMA_PATH = "";
 
   private BufferAllocator allocator;
 
-  // Rule to adjust MAX_ALLOCATION_SIZE and restore it back after the tests
-  @Rule
-  public final ExternalResource rule = new ExternalResource() {
-    private final String systemValue = System.getProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY);
-    private final String testValue = Long.toString(32*1024*1024);
-
-    @Override
-    protected void before() throws Throwable {
-      System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, testValue);
-    }
-
-    @Override
-    protected void after() {
-      if (systemValue != null) {
-        System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, systemValue);
-      }
-      else {
-        System.clearProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY);
-      }
-    }
-  };
-
   @Before
   public void init() {
     allocator = new RootAllocator(Long.MAX_VALUE);
@@ -96,90 +59,6 @@ public void terminate() throws Exception {
     allocator.close();
   }
 
-  @Test(expected = OversizedAllocationException.class)
-  public void testFixedVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final UInt4Vector vector = new UInt4Vector(field, allocator);
-    // edge case 1: buffer size = max value capacity
-    final int expectedValueCapacity = BaseValueVector.MAX_ALLOCATION_SIZE / 4;
-    try {
-      vector.allocateNew(expectedValueCapacity);
-      assertEquals(expectedValueCapacity, vector.getValueCapacity());
-      vector.reAlloc();
-      assertEquals(expectedValueCapacity * 2, vector.getValueCapacity());
-    } finally {
-      vector.close();
-    }
-
-    // common case: value count < max value capacity
-    try {
-      vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 8);
-      vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION
-      vector.reAlloc(); // this should throw an IOOB
-    } finally {
-      vector.close();
-    }
-  }
-
-  @Test(expected = OversizedAllocationException.class)
-  public void testBitVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final BitVector vector = new BitVector(field, allocator);
-    // edge case 1: buffer size ~ max value capacity
-    final int expectedValueCapacity = 1 << 29;
-    try {
-      vector.allocateNew(expectedValueCapacity);
-      assertEquals(expectedValueCapacity, vector.getValueCapacity());
-      vector.reAlloc();
-      assertEquals(expectedValueCapacity * 2, vector.getValueCapacity());
-    } finally {
-      vector.close();
-    }
-
-    // common: value count < MAX_VALUE_ALLOCATION
-    try {
-      vector.allocateNew(expectedValueCapacity);
-      for (int i=0; i<3;i++) {
-        vector.reAlloc(); // expand buffer size
-      }
-      assertEquals(Integer.MAX_VALUE, vector.getValueCapacity());
-      vector.reAlloc(); // buffer size ~ max allocation
-      assertEquals(Integer.MAX_VALUE, vector.getValueCapacity());
-      vector.reAlloc(); // overflow
-    } finally {
-      vector.close();
-    }
-  }
-
-
-  @Test(expected = OversizedAllocationException.class)
-  public void testVariableVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final VarCharVector vector = new VarCharVector(field, allocator);
-    // edge case 1: value count = MAX_VALUE_ALLOCATION
-    final int expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE;
-    final int expectedOffsetSize = 10;
-    try {
-      vector.allocateNew(expectedAllocationInBytes, 10);
-      assertTrue(expectedOffsetSize <= vector.getValueCapacity());
-      assertTrue(expectedAllocationInBytes <= vector.getBuffer().capacity());
-      vector.reAlloc();
-      assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity());
-      assertTrue(expectedAllocationInBytes * 2 <= vector.getBuffer().capacity());
-    } finally {
-      vector.close();
-    }
-
-    // common: value count < MAX_VALUE_ALLOCATION
-    try {
-      vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 2, 0);
-      vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION
-      vector.reAlloc(); // this tests if it overflows
-    } finally {
-      vector.close();
-    }
-  }
-
   @Test
   public void testFixedType() {
     final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);

From 689cd270e923d4f3f15913843c2569b36e87c4db Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Mon, 15 Aug 2016 09:25:51 -0700
Subject: [PATCH 0110/1644] ARROW-245: add endianness to RecordBatch

Author: Julien Le Dem <julien@dremio.com>

Closes #113 from julienledem/arrow_245_endianness and squashes the following commits:

e4cd749 [Julien Le Dem] fix linter error
c727844 [Julien Le Dem] Fix NOTICE; typo; doc wording
88aaee3 [Julien Le Dem] move endianness to Schema
e5f7355 [Julien Le Dem] clarifying big endian support
36caf3c [Julien Le Dem] autodetect endianness
7477de1 [Julien Le Dem] update Layout.md endianness; add image source file
eea3edd [Julien Le Dem] update cpp to use the new field
9b56874 [Julien Le Dem] ARROW-245: add endianness to RecordBatch
---
 NOTICE.txt                             |   5 +++++
 cpp/src/arrow/ipc/metadata-internal.cc |  20 ++++++++++++++++++--
 format/Arrow.graffle                   | Bin 0 -> 3646 bytes
 format/Arrow.png                       | Bin 0 -> 86598 bytes
 format/Layout.md                       |   9 ++++++++-
 format/Message.fbs                     |  11 +++++++++++
 6 files changed, 42 insertions(+), 3 deletions(-)
 create mode 100644 format/Arrow.graffle
 create mode 100644 format/Arrow.png

diff --git a/NOTICE.txt b/NOTICE.txt
index 0310c897cd743..a85101617cec8 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -7,3 +7,8 @@ The Apache Software Foundation (http://www.apache.org/).
 This product includes software from the SFrame project (BSD, 3-clause).
 * Copyright (C) 2015 Dato, Inc.
 * Copyright (c) 2009 Carnegie Mellon University.
+
+This product includes software from the Numpy project (BSD-new)
+ https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
+ * Copyright (c) 1995, 1996, 1997 Jim Hugunin, hugunin@mit.edu
+ * Copyright (c) 2005 Travis E. Oliphant oliphant@ee.byu.edu Brigham Young University
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index e6b47de70ed70..1d3edf0117f91 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -243,6 +243,17 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>*
 
 // Implement MessageBuilder
 
+// will return the endianness of the system we are running on
+// based the NUMPY_API function. See NOTICE.txt
+flatbuf::Endianness endianness() {
+  union {
+    uint32_t i;
+    char c[4];
+  } bint = {0x01020304};
+
+  return bint.c[0] == 1 ? flatbuf::Endianness_Big : flatbuf::Endianness_Little;
+}
+
 Status MessageBuilder::SetSchema(const Schema* schema) {
   header_type_ = flatbuf::MessageHeader_Schema;
 
@@ -254,7 +265,11 @@ Status MessageBuilder::SetSchema(const Schema* schema) {
     field_offsets.push_back(offset);
   }
 
-  header_ = flatbuf::CreateSchema(fbb_, fbb_.CreateVector(field_offsets)).Union();
+  header_ = flatbuf::CreateSchema(
+                fbb_,
+                endianness(),
+                fbb_.CreateVector(field_offsets))
+                .Union();
   body_length_ = 0;
   return Status::OK();
 }
@@ -263,7 +278,8 @@ Status MessageBuilder::SetRecordBatch(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
     const std::vector<flatbuf::Buffer>& buffers) {
   header_type_ = flatbuf::MessageHeader_RecordBatch;
-  header_ = flatbuf::CreateRecordBatch(fbb_, length, fbb_.CreateVectorOfStructs(nodes),
+  header_ = flatbuf::CreateRecordBatch(fbb_, length,
+                fbb_.CreateVectorOfStructs(nodes),
                 fbb_.CreateVectorOfStructs(buffers))
                 .Union();
   body_length_ = body_length;
diff --git a/format/Arrow.graffle b/format/Arrow.graffle
new file mode 100644
index 0000000000000000000000000000000000000000..453e85025d8d324310e27daa3692a98e3bceb332
GIT binary patch
literal 3646
zcmV-E4#DvsiwFP!000030PS7>Q`<Ne|9SZ<eEe{8Hw%d*e`ne50HK8*1(wkEwv(A1
z#a0p>j+NR@Ded-uKgmvFCw3BGNdmUmoyLiuq-W`UK6;X#<o(~i_C4hb2|~*JU$<cm
zY%9cfm`nZcuiNjhU+-GmzrEVp|3~BE==zV#V`bpcFj6kxAD*2aDcigC`oUn}k$Syx
z-B2#iPMg;X&!b*HKHpZhdr>sltJiODZ);dc)g0Crf?@qKU;`3FcV|4aU7n!kM(#E*
zCdqwD9sa9Jo#@rpPy09I?$v=4(J$l--;v<dcgffNy7-mfq<%!YBzOh(>*LFD5?(fS
zaKy9xkmNi`go6N!i=XzxDBzF$ifdS7eV=v%HXPJ0_|=;LcRC(1_Uj{&j5IXt8Z^23
z$WT=SWJLDs<E#@Eh7mh(|2GVyNzqX-B=zzAL+sp$B7K(^H|SAkA~t&Vk(iJO$M+^`
zeE$x16!2q5`3_an+)->@`|)E+Uy<O*!(l#clX)~PbSBAZBb6UlD$3SxBG!!VJd#R|
zRZed5EZI$ZH*-6=-w5z+s#dDl`9)S8UZInk7+<9qNfn0V6Jz~*7{n3wzu@p9pk11p
z!niKus?4@znxaX+k(_*5<+()68}|JJk9Pf>QZTPnEZrPnhi5U74+!>NsoePDi?JyG
zBCNTAqnthnKw}qJyFgc<v8P&lKvRAO{4b}S$b<#6>E8}LN_^#vC=Jq|P1B&^z{7V<
z2Yb1Kp+bG-S4DixCVNdia(y?*CBYmOv$C-%HkdQ)6F-`T>!kKjuKGqM912D$fcLab
zPD5d$&l&Ye0Tf0L!Tjjd@38rnEF=mH`Gk01NW=+fD$fyeG8QF%i(6Sd-Ke><cAum;
zA=sU-LI$uI$g1iTW^;LFQ%w>XU?!1}xCo*i>oOmEhd2<V*5`)kWKxe=tw{co(QE~!
z2m-*`QJ^&iyW$K&8fENZTw%K~TVdi}TvWD&Czg;YR|i&2-%(5**7P4UjWd=pRL%#c
zW-h$1A$3OedRkrqL*|+->|vMPW}7Y*C}zodG?{JNV*btNEllbl2&NR6<UhKREh0&E
zM!ML!n@ZBbUN~y~u@Fa87Wq?VIo*?tlU!r!&D93Xeb`x>7TG)$C)!+`WUeHWJd{av
zyX;KD31Ualt;H;&n}%eODzJztS!5L~VpW4hG;NVLVhaf%d87(FVn`lY1&^53;1Osp
z8P({}mON4g9?>O_tb#|3ip-B#`eGhgc3<*HNqEGPJhBQN(W}BEKwsjKSO7{UsREN|
zl1Wy<BwAHgY=E(B#ReeBBvoJ%B$;FtOrlnVNet6o!X$<%M>myV5-6Eu6-<Jv!X!Fc
zz$6HA3C)sBQUxXfl1Wy<B=9jzk};<HQ%TbC1NE^?!l|fcfXpBITg(FGSf&cRq1uu+
zR>2!piDT8xWpS*!A>&vp!6b5}W))1L*;Qc@71n@(mXL`m7i%g(CJzs1o!O?A*{7yK
zZAY<?74KEE;$X%f1vWm1jha0?Ansf~Dh6#)*yu(qFV`64YcyYn>APtawt}bwTi6e2
z{Esu}LZqXX!yJaiEmgCUFX6B)zMEj{d#l5FIBI*X4rq0J%0HbR4nh(EWkSX0WJrGb
z!DSuhv49Kz(h7)+biLIL2=T>5+Zz)8$)#EY9JsCMt8M`PFN7%mFf8$bkQRSfKzxX8
zk>bOSMKmoI(LpT2Q;QD+#v+EvMMS<d@NqN@u-E$RY;6gh+pSKhBBkYYfbzC^aW3|s
z;h?o9HMkCKkQD9`KO}ofn=y}<zV&0y4#l(uFFouKv6HR9GN=90M}WyN3=SYY0h58&
z0VY#|sZ3yc?{nW_132n4m&2$z5cT#~LxAepz=?Y`V>lsm)o?N-oXP~J>*2s7iIuMl
zQ*pV^1Uf)lBXo2Loiag3Y}>CQa^eAzX{nO|k*2R2IhsUHsX86nCt-y9gAMp`5ik22
z^Mx-Lx5hK@;u1h1FaeaIt{PCP1XP&-)gTV-iw<J#z0n}xMT-M(_7rYVt*ua;v$Gdp
zJ_l48cG^v?-oZl>Rb@ieA!A-;9XvihS|f6xL{6zD>7tHRUF4|t8j%Aea!S?bP#*_(
zmBo%YkZNt~tg0=sQzq=Z_85+MO~i2{8<<+ijD?z94UhcQV8gaYp34-&c-?g^9g`}9
z7)sVLRd=#`yiAnzN;Tf_s|Jq@S|}BGPB+pYwQ2N`x=}Blnp6G-mEC`KqhAz5`}f*<
zBpIW-JVsY73aPkP?5k?n1wpG73Tkcg*1q|Bk*y2ER9{oiBW{D@KJ(qxhOB}o3|U3M
ztO2MS$gmB=)>UM2GqB=W3B%k`RPc1&*J~QhbO|GgKaQIP(q6Wn$D&sxjL~_&uCXYJ
zLh#9hYkX!=d5pW+==2d+fSP6?&9FJ<ps51Qw$%)IB#EzB6u;zPCQ^D1Y3A(aS>sST
z59@RuQrE%hQO^yCzqq^p$nlv6)^yXzPI?*DE~1^a9aMxY!Pa)}kdKw`q*>-oAfzWl
zp*i=9)0fEGSl|t8a4XFb|2=$7Nc3|26J3Se3o(<~D4UzR4OTs%mH{v2iXh_vs^WB%
zs&NihkzUhP+qNLE4ckI?yi%`OFWs|8PvzM|>Df!q{@Hu>9ZEbmlz#nF`SnP9j9(8|
zHTf3b=jJTk;m6Ovy+Gd<nPA<%tqcJwR~Uon7dGJB`nXT_lyI~w!XJ7bZhNDx;{^N9
zAd11Z)S7OZ8q`b^iT}7Wj8rrBe$fl}ev$n|-Y>Eq>iud^g&MR}9dOGM^Xt<2l{B|v
z=hwv%magxUO=&;E^F`KUJzq;R3Ow7)QBcs(#hFZHc7DsV@{HsggqPG1v>yYn(!aBB
zF3gvzHbW)0El*u5!>MbMM^?cjYkOy3j74bbYhQR?=1)kHC=W?UZ(}tqv9>qz#pfm=
zU~VEfWdTRTYDq_<8XTfM^?bds=e5n(ql%qoqya5Ek)^%ti7fF$M7zC+_#Ny-ECBf#
z{HD_*ecalxQsZqPYttjI#PbN6iC-?iUWm(ZvvQrU#H)$>GB~(9#F5k6z+Nuu?kOHi
zjw{<ZpmfWwaXi_`guoP0XdGWwyhYLgFMSyfKGiZDc;GUejts*#05Vh#Eu>pA2u*U^
z>eu0Vd{99Gxe~Y8(K~o_5SnS_hqcui^_FK94q8t*ri~q5U0a{O3*|{VrSf;*kubzv
zauRnARorKROnc1;lE>wg5kVWkPMf)RXT<M{b3c+1<A$*WdE?eJ6U2uRXqIYmpG37_
zP1kf?GgSWF(34|&U;OA1nol%(gxVuVkCqB-+fWS|#VL7i^`po3u+t&ID)%rgpcP=L
zH3M4we@nH2rHhkIM1Z)hBL0hJSP;SZ%oAOI>9k@z6|Lxx+Zm~wrf#aXZA)5_w6c0y
z@tI5H#>kCEheQ3bQ&JGb?{%F2s@*DthnHq=q<k&+k_vjvreml*8eI2?xkjp5fWH!f
zYmVi=z(fC@aL(7Z#aWM5^3o1XTf>x<oUtn3w&Qc|d98QO${S5p#YeV_O~-#a+{UB$
zhmB8M8F0;*k}o<e*P-e0{K$F%HNagmf)nd>c)g9MfxXP6@TQRktW4Cc3D<E-h77tb
zchKz>aaiES@p)?n+`O>YOvknow51MxoXeYly4fMr#E=q8?{Sw|h)nXIGiRAd`*;vu
zvwO&=+Qd2xsY}8$=G>4;!a_u3q(9J*wo??&DJC8hTykccpsg6#3I6+B@JM(OB*=-w
zocMnT8C<ibcyZ-KdNO24#uWID1Zl~Wfz%A&i3eXcCzVK^<31@EB8y@am>0hc3isj^
zPB%9_>X7h$YJ8yNc4@Zr432BdhS7YJJMd{AM<i!JGjnv}u0sQdy`z4EhUsID#_Lu(
zUQcD4go6%G%~_9Utig=D7pr;fceB&wV_jSp<&^OWj+mIONET6Z1t5_8A=?5-MPfI{
zRDp5&2PhcbvH`u9{506OiZ}DnlLy;7-1SNH5c!a!r`hA9>>0zKpyC~KX@_?iLhDAR
zrN@~f0F7N>?Lwd+xQC2ARa1TzHJ2+Z&M1S`(e1ZGj}l)wBT9qxXA_;Xh}{2iE&`rY
zG*!XH`<uqn++6ORx!|Qk-bOd!J!A!x=Y!_FdB~!O_21!uYmnMPX6|N`Uq`&`Y8uj`
zYYk4rH>InjgcBCfZ(^4o*PHq?Mwc(h;X^{Vu%N?(=$m`;*8Afl1mwSm-lx+OcA))!
z`1@(|xOdRIcn@^g{`l5&x<~xGd+{EA>a|ZkcnAFc(XoDgjoyC#^zj@VoPIn$Xny80
zufKY?M{gFZe?~gd->DwKD2wvH<i2GvHFsE$eSYz7vJnx&n$7=A71<_)cKmK{dwRC6
zpip5JSL6%tW=MgpzJzwU5<bu>8|In~R+0TQtHup+;S+)BIo`k(!S03c-7N?E6}vs~
zXxG1o6T(HC1MKj`6XBTk#miyF^q)9#oRP0-l;@=*?0>=Gl>3mR=yU%xRzvugiPLr9
zMD)vje6BdL=$5hXkO!uH@6*pik`BfeJpKES9QzRs7FGL)V)rl4-XE#Yhi&sY_GlOc
zye8tBOigKyw=HfTe;r_-gYD0xiZk9-+9QruDT1tHmyUP&VAs$`vq=2ff@pfQJMqz{
z7D3Ye*^~QMLoe+oa{|11+IZRR_?1xk=SHC5$vNbj9&Voyf$bU31nIM$%&5b>BHbzK
z%GLs`#noUVL)3^?Pz#?J>(BatGvx@8515Za!$Kw~ST`xa#!m&g7yXpwp&VzVp97DB
Q<ki;y0j5-&GMvf)041Ut%K!iX

literal 0
HcmV?d00001

diff --git a/format/Arrow.png b/format/Arrow.png
new file mode 100644
index 0000000000000000000000000000000000000000..361dc82ed5a453f25914675b95d428f9ba2f9b81
GIT binary patch
literal 86598
zcmeFZbyQVt*FFjxkVaZcT1vXROS(h4OL~(M8>CyMI|W2akVcRaknWUDrBOf-e)sl$
zp7(pccZ~DjIpdsh#_JfuX6?1^b+0?-yyi8pxuP}H6tFSKFc1(Bu%9T(Y9Sz?SRf!E
zwV*-35qDb1XYdQrQ%gY_aeja`82o|mrfBGifPhH=|Br}}nN0#tn0C-HfEuW(2wS;2
zvs+lZTH3JtI=g|h5fDUug~3N>8>j`fuk$k(Phnp%+COIqgU|589JJJbPJud!(Hf{~
zP|LV_*iiGa^Rjc&iepeyQ;T|7+X`#R%Kvpa_)Cn|9tw36=HT%0@nQGjVR!Yg<KPk!
z65`<G=HTXL181;#`nf<YeA!$)>HgKo-|fiScv^WlxIrCUU8v#hT3EVzLB(ik;WzsC
zzkl@;>R|gncXILk>#@KCa=^dg;9}?G_;=gjQc?I(VHsCvHxC<6Ptd+NpXi@6|JSkq
zxzE4O*RXemx`I*gaIkvf0=4k~7eg)Jy%XpD>+1jOGybzJbq@y{@YH{<=KAaEe;)hm
zd{GYgga2b7{x!;fj)Gwp#}MWC_mqia)NF7sBOpj3Jdu^s@kRW;?2|vGb6evssUe9{
zTVSdG{T-1Mik#*<rl0HvK7j`&-z|M+TYkn^D(Dc35Nl{?K;@pz;g-x1$+4mxc?wC?
z%{Ti*WIQk3$&ff4re|bi%p0<P<*Mc&`uyj2`HzGel87k({6T#JQN`z=hCwix^8eS5
zDKrwgW6%G641Nk9qz~<e;y+Gi%9o6XaX}<d|Fab)4Lb9<&e4$nHviwj`R~dB8|=T^
z3Ydcb*Ng&hiWDLW8Piztlh^Zgb^@FP`uf!P_DfAC?+oh-8`Xdr%Yo!R|7PuO+m0f7
zF_SeyM5K{Fh#ZHfGmq%yqeLwU>m-pvU#jqVZCt8%G}3H`v8meW>H3vv*ggTaV=93L
z?4KlVO=U<P^8d23aHCs>$0@(M^B?9H{OJgqzZCjUyDS+(hG>Nxg!4~Z{pZ?z`A(9M
zJf#2In$I7v{Y?3#X>9Gs0ArpkEsx!ZvF}=V_s!vQtHN0hyj!mrI6XG|JJepsb2M+h
zx;&%`IQ}$V=-Tm{|6|#+(xPR*{qoX6>7c8vGz*2Q_MZ#(w%v?}tw-YTO{`zak>#cc
z{c#qAQef!tl9t0DDfFE7WBFxG933~O7c0S3?OxJwpM(%{>E|)?&QK-->!sWuCl&G*
z44DRB<itZwY6-}Fv2%^&@uEa=GF6cRxU0fNHhq5ZVN~KyR3(v&Z@;4Yqn7-CUYB10
zseJl@7PHESQ8knB6oU(rNnUI{EK}#Q+`?8rCjG6l(X=Dzw{ILVe%wP5@6Qi?ilrT5
ziP`kb0)Kqr=n#E+k|1(OeR6X)I{MV;S#9s}T6dA0&q|=r&PdL@|Cdy@S<r`XRjuq#
z&$}==pnDS~Uf*VJ9tyZDz40_1FY#-!l&$#F+f5VdY?dN7$EuwHv!JW+ZIdr-jY|P%
z!;PD-lqb~IzEyYNgb%r447=r8{{Um?bkh64%=yE#%g3v2o>9@)JX<hCB<WlNMm6ib
z31yD<@9)1HHm(G{W>g<Rr|;eelkwA~^<bvKdC_ktC++6$_M$^Qi=WL}9C-b+QHkJq
zzL|U-)Q8ZqI8vT<^4}YY5+mZ*-#dOExZY5l?~ZSPxyMqTS5OLCa46LbcV70IJs2yH
z31BNFg0FxW#!kPbGzbQ-EUel|Mvo)XD^X~h<Y%GP=Y)gBM!WTuyIX;h_1tWS!=KF`
ztwYJLHKxjP_VJZjKLwaxAGUJDHVGR;$L0K52N<%RH7<KE9rOII4*nf0mneFQHfgMI
z*z&E4+@*<T()#-L`lxPq#>jQcdGd|wKwhle8$Yq*P9#?UAJh6-d`^9fxQ0>u7oSk6
zYIse!40QH5yqhO{KAk?+Kn|e|)|QVW`9$pJ-nZd8TV>KM?!h>;h;(1H#@4V%ekI@x
z`-~?$zia;sYuyl7pCe{^dsF2E71n~`wu0)n;KpJ1`hj6!kfpHyIjSn++APO7db07k
z(P1*8@#YczC*)f6E@aji@12@%oYwW~2V7<o$1U5LPTbYE^LBaR+4I68()4K`(#_dS
zk)^h+C+%x{i8pxFSYeG}S%$JI8FgZ=S8`~G_9J)q#V@I2*;{sLw_R&K-NR(9K_GZs
z-R7ZcDi=^YPU|wP$_#+k*5lRrnt;UkM-_x2EKP5tERjV!Equ!85N^XCnD(EKh#U=B
zu`@`=!6++<joqZ!<@17X{U+K@`pAuUYR`Ynno$WunF!8?c<!qY81!ts<C``1JZitN
z&D>FD1+$!!zTN9ZAN%X#Xq4B4hTrvTHe|?9>nt@nk}62h)o0EE^-Z*3(i&)VpquII
zDXn}`0SQfB%<UI(J?Ir%8>APH=~UoROL@@cTI4)$>8En-Qus#|Py7w`G<4G*H153o
zu6VA|A<axq3yAs`Au5lR+%>|?HPI(TZ2U@7DZ*&hmzNufqs+Z`y=YyA=lyhvn<LpH
zgyl)S6WVG*;mPBNVdu=jGI7-1`^Zvr--z@KLX;3Uy}Pjml8`Jc-;_ra2R&eKKiB#S
zk^aD)8bKn(#OYaf+u5>{<3ky%_tfE$+9{2Er_nkN*eD`D`VHFCIrvS6hMU=dvB`gi
zlIbCRUIOe0G9pU?KAG=cNm`RXCXs!Hj3D*tTH!%yUeNjp&i(4#pkEH#+GfcJh|7A~
zu_=wlJL7J!F@%tNRff%Rwzj(?nONltl0-=C6lrEbe%oglX}!iBr#dkK(p%vgTq*l+
z=&1ubw>St&bN%<78<+L!)QaRuV`^`IeT(G1)diaigGQ3icO--=@Enh?_>Fqr2t&iX
zZPxLw$TRjc^q8|XU~N7JisX<z1Eyf9g230NcdT40r*sT!XTYsubU2_aV-!7=*vPg0
z2QjHfDvlMgn_j?nhF!W0ANxqf#n<OYz>-kJ44RGf6(Fd1PO4#7*D#&!-?@&&CWldi
zuiM8?X|r3dP%KOwNhy4m-097O=jZ7X1Zk~Be2>i;E;C4YY_SNCF{4<6Z%#gE-^oCN
z_@bcin$~01Due%J9yrY7@-MetG(%2_Ru|HCgp8|}eK+EvDw4}O>|z^S5XUMlfA)lC
zhJ>TOWM=IqPPq9R5xX#!Mrt!q;#)9sX$~hwUi#{}JkTvrMb}+I{{g>At)!eK1)?T?
zwYih~oZZ-Q@*TDH^&&8I$KCf;6jZV7kI0<4WVi}>f!BsXQ1GdJ2c7xtct&)dh3?t~
z$YiK%oD+_fs#B>&>yP-kXNs+b5wJI{hGLbDL3f_;E1BH8mEU>tMo2e;1QBVTJBZD*
z&>L6f45K2<;np<x_mA<bY%*L;y$pRu47Xo<r7NDJVg)Z=>`tjh6nr5e;~R^je)b3h
z<BWf*Ek6^Xx`y{=OMzbL8H}2JqaF@$0)X@G74V70NdAB@`w{E+Fv{cUyoLqGiZKy=
z9ZRw_?&T_zRt+;=o7<DmRHNO67TpnICp`o{U;P;MW1b&2GZLx#DfVhuh(0QEefyd6
zd7Q&KPT=*=l9y~;XtX#zc=jEGkGN+Cb)HZgRb}~M_4daaYfK!b3DGOh8v~y+kPSE6
z7&N$+$OQvLKlXtacyh-)9>JZ*xEBaRJ%V0-p7f9|k<}@0?=?&y{{j6oDh-NM$T!#m
zUkebB{kmnD?oIL{=da78liqsxCw5eN3ZLzVz1VZHEiXx3*UQ^SKl*sh9^Wr5jvSn^
zIz(kd4*9s|+@IQFTR>!#9C7bK%`rdJ6nEI?k8{tzgz!zn0^xS;B^VHuyI<d`Z?GrF
zX&FeeU>&>yTcmiZ)_(3K%A=5a)%>DQ@d97dMS@)}{Y3jzv^*Rxfe)LcLX0glSZwui
zeR>eV-um%!9g~tGTqIata?X3vxp`@GfXFF));7zz4;c}Ya_}X$juCb&H{r&e-p@Sv
zlyD+C9!(PFm7eor&v|@dlSvF5pW)8VZ}Qx>Rup(VHCGr)#CIp7%{mgfr&f?n&y&qa
ziUvCXUa}4e6{nbngsr)-53)HFoms-`j8^Kj;OuVuCV|9+T)sqA;HBRWPX6J7NJ^y~
zG=Jw}t8gZ&{UWYm%JFU5A$oQSHf%vcZo=p&ba#u;k%Fv6rVN5Vf$}558cHejljVBI
z7L#P&#1!UMXyQm2X+LL4lqkn~G^rP&y=u!B_Et_-;&%Mwxk%GmbPBh<f!MdGPL&_@
zqVUFy5cNmT>jEcL1vGnM!d1LtB_CMkJsiI9G5tU(d#6kdd*<LN;(u;WJkF5Xp<``>
zgBYr-Z<5kjaOTF~Yr`b3xM~JBhWU^%jk~MS;P=rBwNqG(O#VNL{0V*fpMS4K^_e|t
zJHv5x2vMJcEnIu$N~6nL3#y-<=k8p&e!OBJ)k+^rTm`n0>}Sq6_Rp2zySfqKSaakZ
z3o?ZpVBL9f`?VOl_rstAVNLgbV@@nlL&Zqcme1oPm%{1<I`Cw}uK1$;wX9QKs|iO_
zIx}+)>?VXH;M+eLuh6ef7IrdB9kc#3R48~%i(d)Z(jh6dodR`QF_~4_<4yNyK1DO;
z_NzI!|Cm{G4lqXAc8m7wr-=-6sqkZwt3<4Fl2Kw8BMlW1);(#8BdcLu?|OM#S@>Dj
zxL2zVmcd%sEH#ysa;_t3<HaBy0ku#hNA_PUq&pnzODUNuwsDBC4D)(?!EtTS9NxHz
zL8zMFzmjZEjo{)xuQ2kQ5XNu;UHH5>CPM95#@x9G#*GCf?58&Uwp>;@XCI<diAcDu
zxcWo}W@R^tV1(z7{+D#X6Fq-l(u&NzDObvAq4RVfXDs(Imh+bzdmnoUKcmHPURgyb
zXBDF7MCC|)Aj#u2=kcc1A)e89^BF=Ph72$Xf~2JLKJ+k@G4~aM)0FSd^6ogN4zoA;
zoF9?YMYvOCQ1$WU7a~qR^3yDS^=vkt1r8(_#?m3-ll8z@=LbQIeIFe^?|CSG04a0t
zM5<~crz6zpO=h?U4Ip}2BCou6n(2>9Po03<i()W{!alo1pOD!Rp<5O$d8WhecrkBB
zwe5zk_8fT~)0)y5K(NAiy>(e`3C$GWVpdNFl)<~KS8K>}Od$+9=hM<FRZ~?l^piQ#
zlR1o}u%7^8?Iib`2&Z(|Uw1|)x1jTiMd`ay_YDs4Y*COqL#qh9szw^hY9!Yj#&Nq4
znXlY-)CG9!->xc_u&%RN5R#RbK$A5@47qla?-<{vles8^;Dj})=Xn-AmFq=2F2q!q
zP&!3)&pB%WRrHP35-YomL(QNr-!z8(tVTXt>XU|`!of{RgE;O7$5~y7>xVfr2D0XR
zr!~rH>|ysvreNFXymn|G+6Ql_Q*}>nWK*WsYB+)AkOs#5QIF50F72wW+!_u&pAM<|
z%f6}}w6a;HGn0k)GgqB&-E5eGMuG}sTn~vKYj(OS2|AwhNg~yb{S0|~z1-r}y8pIf
z-^3;#UjqG#r7o{5;7Mj81Vg6<#uch05t`a@4)6Ofk}+lhXM)0?DB)j?2hJ`CbWdf>
z5m}@`b2prtDX?3!hk~3gufEmil^OS{RKp<CEWq(TO?-E)D~edn5@VMW$95I1)aUNa
zLC&%&P*M<Db=Lt2D!iTT_Q7CEG7e!FSoXRw{u3kp^M@s%^ejo#;A`cd{9Teccy4s@
z%R~Q9$i#-$FF}CSvNkXA`+QO-&>C8XWjHQI6liFtnDkkWqwN^7@iY<(^**J_d=T)L
z*{60oq1gO$)TO#A@BZ1W903UFcpU|bC3`W?4;C*{ja<qvm)%hR8Rt?1E-{psb3_mL
zu>W=30(2DLyK{Nj5`8F>?^_swnt??)QL2Gu^fEw66U9%)Q$HtCtyJi9Y;!a&d0ZcL
z-07PI60!YSi=@I4-p#j3Gid<v87hSvg4J!foB~xK^yq7Qk+n+Fm8QZ&qCD-La}X|!
zm1{G~cQ5#VtIP{H9Z1@K2TU690sx?mkK~9nI!-%o#0w-m#Z6lMFl+j4zK(JD(TrK}
z@6S}hH=RiLiw_ZO4(Dn&n%p<;;ZXGV%+*@S=)vPN;F)IL+kE+XuxRG9>}9bH;;Sc~
zD}fi`jik457?$<yJ-Uqn_==-tw-B%skv)VCb2S!l+!*t?s_Ak)_F`|6@oQNKw&p3n
z^TIRGB-{co0(dPQQQ4&U&#`(>$SI%nD0zUeUv>4BGCQewz@>wyX?4e~@J0R7-RoXd
zssP+W2QX`0uhqpfL@Zo-y9+aT>|9#*-YlgO;5C8O`bH%KUk-5wgtx7SEl#JK0|mHP
zp>R**wL3=Rnc>oQB5#Si-0IV@7C}x7LgO5tuU7zRTBoRsUxgV3HjGSQvNxIUPnFAZ
zp*hw>nIrB61BL)RNN5H!S_Bqpmv?hKqg$u*)#(?G(H;;yQ}DNsieEpgAD1VzElJT6
zUXP;dizDq0kS70r3ZOOVMDew98uq)9n0Fti^^EYE1visqq%BVib|BLDzkmIp)01Wj
z@raOnU>0!lQh{<nzPkPAQ*TEQ(OVmJ-2QTEsT4H(=MZ_|u71klKq0h`LRe``o^B2|
zwau{cJ%6|z{Bz#!Y*=9BYA0`WQS4%=&ZN~_;_k+?zRd4z+b-9S!4(yYL{hIvJ{}_$
zjr3SBM@WY^o!^;kbicVACzkU0cP{Q5VCnq6SOrk_MqZtCyu)u%i?clhizUnTqmb(b
zSgW?fnSlUITa1Ix!+PSs&$Yj5wSAa*`VH_Q_hZo>)$A^Lj$*r(zmlY;R*^XEr(<ma
zkh#l<|8s(||6WPkqaIH8^&a~Rnb6~lbgKj!hOYBOiCe{k2$}n0yifgy`DVw^*Fi|o
zn+>M}m?Z0?93PxR`;AWNpW-!jX+cN{H^80a@%29;Ki5HdzQ)d^VUO6s98RqE^fsvQ
zm3PlFzy*qo=3<Z_ya48;;E;9@NQ9seqg_pSo3|b`JO=@WFW3{K6OQJ$S38Ns2OuUM
z+GM+bHEUoMk1epo$(jj<3P=2Mm9u>{6!Y-?7xuM;93jBr;5xY{K1`jtx&RRG^5>$f
z1EiDZ>Ue!STQy7=Q%5DAEZE@Xl}b{2hq~wq@*~DjTw={1usPZEsxuDst^lR}1`r03
ze{HZI55;{_f9+MJ7<*4ix`*beQd_>m<ePeJG|wnZ*;v3Jv}?41Ws_l!%!gShcYV-k
z#(NO<Py5EJk?(*s>*EEOIc?a-d+TTAh3a4YC|_vbmuL~XkM8@Wl4BAcQXhOMRIz`(
zQlxO7jCxEwuL{`(pbdM?90DBz^V^0+m#p{66zO%lg|Tmj7!&#r1RfbXmX{RD^)Q!>
zQngo#X^P}6IP7Q3EMsg>n#t5Bw;%!f2AV!P<U2COk?%W%wJhADtk{U7iSs9r`P%?X
zBB8|L@#2n66O|dkA`nhc+Df=25@*Vjs3;p0l>NvLyPL1>c8d}VuT*+?<}G8{n!+i0
z?Z+~mss?CT7<#m4a$Oc1*m(ACsY_a<rcxvibeSjx_Sd2V4Swz91^2b;ylYo#-wDG-
zxK>*XVIuh2CxbM8V;?$0prYMX<}HKfCyV|khWOk5hD4yv&E(VgS5-(pMQr&Mo9bSw
z>I8XvaV8p&<24h_QFtLmJYFFYeWhFkE3=Z`9t*ZeM*pVuId}UWFK_-VtV~EUnc60F
zSQ8UQf>;oFTv+f2x(y{Kk1sN)W@@xu%9CPPfDNx1DypU-%KxA=|61OW3poZ#j^Wg+
z93|h<Got?we-L5}#%Gho)*t0PK8cwBIS8ZF&k!TFidusXZxTfg1!qJz;&|*0JNv^$
z5lXWpGbih!q;3slxu5<x0+WWt)KB2$&t%Q|;<C@)h10SJt&h-JH1smyExG9Lj_Q|m
z3R-(GO(gqcb*#|TyBnKg1YrpM46d1g&97u_HNUm+2WrB5^n;xT7r?bpR{KZ_V4UM9
z>O&?=)zuBTZc}p~MDc}H;clF5OxC|OUm^VhD?|PLfcqY@6%#JCxnQ^N`(%d&hVOuS
zCBzTo+(*1ldDYkW_uRw%rG_iwBH0w2fbOT4B!pf2`9G$|Yy1``d)(OZJKo_igmfUW
z1gvi%SA9d96c8f20D{ae^XbJBi|5_wq(|mTU6dvg3F=F^wO-DUrA?kU)O5_ZeAU|$
z7YB>a>Qn=8gg9f?082C~2#Le#AA><@n*fhF!C@|Hc;MhKZ|xg?HX@R!IvEL}qvkF{
zJfM?S8kApsVM;XrVd0y+C)yKEQACRbd$k;bxPC>IuujwsY7Mr~DgroTIYfhX+Wh<H
zA~RBv^n0Q{!={Y{k=-{Trug5M!gz?lcDp2G(<@`p-kFFPrT?lBNgcNMZ=M()&C3yA
zcI0-(wCM_zO7fw*8Ik)>RkUksm#W!}18-)yL)*tu{5o_Yw&~jDT#5{B%CzUq&t=2!
z#`L*ni%>l@|1E?o6p9XZ^sCNQ$WqHBTIjm_g&G%NNcnTWetx~>?hklotYk%jFladE
z?6W5-yCIasiWEwht1s?zzoC=DjS-uje*8!(LS3iiyL3j{B#Bf8RX%I%j=&>jrq?#i
ztBfq*H*>z^G$z;-`}b=_43rT4m#cXsB{K|k>|ijtC8SbhG&J7h!XM(g_u$YY+dCQu
z?L8K5(!pM7-bl9}EebN)+<Jyx?!Z1YEq#*MQ%R*7S;wHCK@#^ppk9$wX_&|#(VMz`
zLV3W6WzC0z<}txj`w#PVMFc#&nYyQTJ~d>io<MMSzHSGQdh%bwL!Vixy@956CL#6^
z+E0xEp5gZQwRWS#nE_JxW0iw!bSBQ-4dAVx(BWhPtb-U&Sk-nqxWHUtH3ATfR2nXp
zrW6(KbghEThX*=IhJITqq$5@U@w{<X)8UVJP!Nto_5AI@*kJ@r$Drr9gm=Fn5@(?S
z`hD&Mc+5mSQa;Bi0P5yjyqqAW1*DqwQPHTBzOXTJv$e{`ufmS2lBk7g#vbU4%b>eB
zBHwF#{Y&W(v!it)UK$r={A~jzrBEhjUb64%4I76z;xkDf$Sz>I$#mgHFVz!<Bz#5Q
zb$n^0-Dg<5EQQyQKFMHjcntV|ZnQ3dloa)Gka0Yow%-s$suM=S#K}Mi&l8cFuZ-rR
z*IdDR*QGCu?%*UTLA|s;$%BS%u<JvI1?jSj&XwI0@dWs$1p5nPUO<<#XuGf5qWT?5
zMP&n7VCdM3d$kC9q!K}*JccJ=+Z-)?cr5jp7;nT1FES2;qd+2*+_mkcsoWf{*eea9
zs2MJGu^&2(F3bKu_T>=K*fplhe2%_Gpzq&owgFcP)s{Z0tf*>XY2)>e$$J^&c34@b
zGxAOkQ9^Ut-+CFql72%NBRxph=hHv}zLC9e`Non&+sPPHIvhufcRyI8;|1e1R^-c9
zbB#Z6=K%IC59ylRt8pO#6cvK2VGnk2GGozmT$Fq(Jcg*Lw!7PF3p~1Lk8~2F<VGFg
zRh-l@T?1J@WU>xil+}ZJhe-hM@%ot1V=t9=Edchh3*ARrO2Q3{A{Kh72E*WcfqMTd
z020w7xGSW9RQ6Q--Yk%le)f%VmGNzmOBG;Irx`WG+5_BeFW`)-P9y>!nwEFiEu9$v
zE6`=5V)~!X#xS$P%SF!NIjSic^huEK5GT;V$S*;l^PzO>M6~LQLyxt0mnn{Le?syW
zwtzUad6*ozSnr2EdGpA$v;Pe=2~V$}O6uZpc~ARwqifsAz4->$74g9&=Ntgi&y@$Y
z5^MujU)A_uDqm9H^4Cs=^<8d$_Q?}QX+tA`QRE%LX*qa+v#AX7P#@`s6#y9Q&qA0U
zC#`surRCTasnTq?VEtN^B;yr7wcnyjD9_vFFZT+<00g)Ky|rCcH+3hMzqiI#c@GVH
z>;-}t<~sn`Xb|~Oe8>6ZMa63Gy(Qsf<Snj#cv2vartH8h^C<i>W*nO19&hh3!4UOD
z6!3e{MCmhZN9VC!SReHh8MlAYr-V$IAl@m|Y^4Fw4z&@Ya3i^Zi?LELGKeSw8op&!
znL_H1R=q}qb~I9`a}g2AAjk?2y$A!bMQ<-R&atG6VJ?`8GH1RE)MMCEqJd=$o!Pi!
zG^UaUG`X;Z`+iw2)~|gB!_;|RVgAMguu`l*0`lu|{Dnf28->rmAaMu;UB}4G48@Y$
z2u=s}LquaKMZ8(t=ecmVLB<Okm+(^_+3nI*cY>eqxCzrqD9DnG%gBNpSfb!M8U#6+
ze%~QaclSwhADZTgy^W;m<#8%8c7Z@o$Ei_v1p2vHM%6FL<Y)pZ7z+|IihKgafIiH^
z3>)|I6P%_5cSHk$T@36y13()gztG02zk(fhx0Pnb4KWU!@yS_2Q4`uC%zZjJgW&zv
zgY)Mm2<%2=p5<O?KU@Wt%t8|T7pr+T<P=Bs-mV!;<nCkO^#uJ|_kD>ayS%_cN0d07
zK_gmB5M8jQU|s@N>oa#v*)q1ZpS64<<#M%|M2l}fcl*F!P$z#4bcW2u9_P<`(WM2*
zR%8hfhH0k$dmyDH)r;t|efRq)xX!gS$NTASM80l?9<Q)D6LJgBl4C`s?MU{M;OlqN
z`Y1yMv53MP)^1@@FPPL5DaGvN)RWhMhk28Qeos3tRrpU56TT#qvm~uV<XIYJEd5y2
zV6A8!3Pe6rgUy|QtytUDV1}6@?K)AjV3`3D^e+1&x^!^lIg?P}uXEp(NY*W9kgY7#
zEks{nO<Lpr!b&jgUbqGc(uO~PJ1{5i*dT>RJplQ-NPo54dUS<%{FYWCVf!=<mRHt6
ze^V{V0C0Z>)Ht;j99SAQmO(^$tKHdfZ`#jz`7xN9e?Tro6c4!5sYu`Z!}|Z@yD~Mv
z6#Vy!|Blt4HSyn7^8f7qu+RL4@kU=VgfOJf)0CMfutKsO<x+C`xrx3GwGfc6y!Lz-
zTUVO<Ja*%|9<Z%9C}g9#Vy!pI4(R{WFAt(ifY#u&6d=6ZKxE(VWit5~Zj9*RyMI9R
z@bJT54pXP26Oin_=i>?h9-d5hR2&w-zmd#Eey5Pcs92}@_p%>UF$ZWh8~Dp|)y|5c
z7+d#0%VAKb(;wN!-<&n|uLhaMa6>>|oq&vl)4cP1Z!*J0QLm~3+?pwg0`VIj{f<Fv
z>0hBx5_Jf>NlP0@mcl8Vv+dzoK(yh}$<y2KEbGFpQ^+q2gT(xg9#y6A>;D+j+D<r8
z35>QzXRg@u1Twz2amtS4|LTe&@cpC@v0@AVK6_JcwW{^-7qF}#f2!wUWsmeG<8@m0
zu<ReV)PnNNNn`o*Y`gZqo-u?TgSGYh`+Dw6fHu2E8l4xtL9~-4=x&M6wq*CGKgK}K
zvsq~K^l#IdkB>|S)F6QP1Dwxrwon})DWIl`x}kvK)0duSEBZ5*%qW?u9vd!3@VoxG
zRXzd)a9ZTt)@Y=_bNGXtw>9m#ss>CoqvX#8y6E7s-{O>t6#e}qJt$=w=08FNqbC21
zLk7zCI?lR8<^P%Tf3>U_1F%Lhg~NsRzwh<evkd{a$9>(T^Pk6yfe=)2@TIrJe>T<6
zY92@Z-&_1OW0E2NzaOYE-=E*-GqnFKk$%&B00MJ$-|uk$S&{H%s8K4p(HCzwoT<A*
zYsaFY5%LCQczFoG4+jV`S{xqzL4IooIBNv5Wb6BseFw>U9&n7EX#&f%JC8Lq0N+8F
zJPS_$+n)SuV9Q7VwjlB>V=en@ga6s(@E<4ujc90?^8Aw)`m=2<;ex%Pg7^7<SrNEd
z_;VGZ4%q)ve#8Hn0ttzM3(dG92Sfie2Liffg^aSr=lS=>{JV23K!zvPOTmKq_V333
z>KBMy<M96bApdK;^8dd&BO!ftcB574T~9Solo)|3l%RvPWTU!^4OnbnKoRLSSslKD
z^x_}a8?){eKLeTxQc8!8Z5v;W7(@Dwl-{WX*9wQ5cwljp)L;l9fOHZZsg6XDc>crS
zi!O-ez=af=f-fH+0%F->$dw^z`0SgZd-GK#>oiSWrBwOma@Bt>>a9m0rPr1aS@LhU
z`j>sl*8%of>Cut*e{K*a4zNVGyhK^!|Ff+Ue6Xrn{2KrG)_-pT;JW?LN$8lWH!K|f
zwq1YQe5Mqb6;-Oz*8>&}{}6cO3(`YZKrEPQXqy3&J8kDE?~(txsWh~J!%j|KiGQp?
z8$!xsO93Qcv1|=<3r^MTpYD;@J1;gozc>W0b;dZh4tQ}My#vsU=xj0re3!pfG2iTe
znT=4S|C&Kn&(v?r#Ao@3#NCy8eQCDx6E`3$mG_)b75y=#{o)=rtp+*1b%$jO)|uel
zoK<4HH;)P+E^Gl|6A1E+b@PKD;WqbStvhm*kE?@WbClzYv390ph&1NZ?VNz~0u(49
zn}Dv;ZZy|qx6-cv?&J00hSU3g+Snw9E!LhJVz#I6TK3;Q07^Rk@S)q*fpX@6iy}{*
z2>h0^UvPV}!sV4nnBLln2(pL39c#RRIf<JNXYxG<`H#l0{@(^(GmZw@X#cSSypBCY
zSMYq`n9O}IK#T!C44uBmbn%1JY>U^oW{ih^n=e5Y#Rxi^^E<|KSSS&FJLim+>=Z<-
zZRZopA_w)8a24L{4UpKlcpqeYj$}9~5in^)K-R3(VXGhv3;;^L%eNi_IxbtqzR9mU
z06hA3d9pDoxDf~RBG0PA&s&A`X$g?Q(g?yQRLt%K>7jl(w$?zP?8MGGBNOxI9bX3m
zT#x=Fn&=I;NU*3vC{RFVL(e6}3{ZX`PiF}Sb3&fmfLWLD+Shr%7QF&;YvYZsD-?C_
z?qZr<n6A%(bg|Rb(-b+m`GdLKS3Kr^aa3&<4R^@+=Ub4;+7bCUCXHE(DX<dY*?iD%
zl_=JC@$zUJq+RR5Q);&X*1Bp5!@-Y5AK20@N)hEMNimeBpd3CY;>X$&Y}U0S!ryLy
z!hs&`VUTV572pIw2GkDZ1|6s0PWO?!iuxX}CdX?^LFYkY7bIN(&r1D^FDez^W_PBN
zyrcXtac@_|8`Ugg7FHlzMiq^C3e?TzhJ)eTfEt(qnPSSO%J+jTK=-DQJOJcn!#qq+
zC2{FW6@%}tR|aUF?!$9N8cz|`0_(YVmzsG}UV}WaCdS#Mx`ZZ8Ml>L{6DF5IDvLzP
zV5b?7ptZ#rybkyd>|ki$2=$h{yPVcHd+56wk{wpQ*#UHY;gw;c_k}ib^$H+e%%#~~
zq}*Gg*@rhz__$`)<wjbMK8grPgM{N2n^b%fTWe=TI*;v8Iw#{VWwB789^`K`0Xo`l
z*%s`BwzCnCD}Lbc)prBi15<*^OIY@b=L95EwI9C7LsCjn?QXnmQG2X`hM1dk{P1YJ
z2sJ}Hk>OeB^V2#&gt!J@lg|pV`kxLkZe(rOe$g{>mMeli0>ms>SPA+qPcGust}O9A
zH?!m(mH>2|XNz-A)T>K;AhTigBx9Sk7D(B+czET_Eq|TwWmBZDwE20eQBUD-Y2u!J
zyK#BXR)yf;Eqr%#=E>EUVl=ot=fgfQw*tnZ9qXHRvHc~#D<ptBfH}X@80B(sm`}BY
ze)R!+Pw!5K9VPp<;6^am(kESK%JL?L-Wf*cWqqvoUn+LzNSx^#yUX3a7Y_D2L124F
z$^J5<t_NRv$Z*0QB+_lhfxL$NtA~&X(E5fu1P$pKSfIQeIN=$hsFDJ4gBL!5;3{Lh
z(9O0x4LL@q?40cMjSXsrt=SSDiW05H4aayC9^D)hhYQwv>re>~&t5t#10=RlCXMrX
zat@7u0izaPN5$NoaSVh7Aj^|2=YS?pj{NEb*o}%Vd7kV@o@@yRV!YFNpzH4L0iv|e
z$QMF1iy+ya)Y#+ZC+IiC^_g?|y^XTVtltT53Jk?FDo%qS{+yJ*9wgBt&iF4FzcX|^
z+@t8<DU4;OHTwC|sAyNfrg<y1D8YwGm+MaJm8Hvwvkjfb&<lL=pJAixC`IKGHgmw(
zrVm&p9*b@5N4a90HAk$)5=`*miFdPFiYxm~!Fl<*<Z8(lSLgu;!zaGb#CIdH)%O@f
z+=k^Ftch~M%P<CgUXN&i>^}p)!pmI0?F@}q&JFVsUQg6DsIzmL?snYO4vrZ1n$@!E
z%H=gzrr-76l$jf!%B`qsdGMndF^`xv7-VBNrw}48YfA=HNG?Nky<e?=Qqzg1zTw9u
z+!4~9y8^=f%#52VH&=C&)))81(;J<tni-V4KQK!?h04n<hPhQfS<R4Z+oDZ03T)wD
zWs~r@v8<)_xz&t=IoVf<%ZO&liSn)Qk#^+MFi9>W>(P(lR<TCg9P=Yr`g!z1`P%1x
zJ1#uDkPBJIB)n55G|4ILLGuT)M+;%-E`s!po<w(Za<b@_6&w3Gl1YFZi+XL;<SATR
ztF=#doNDwyt*&j@jdH~8Hm>SY=+kM`DX@FNt0AcUsRUk42S>}T0<QN(zH{>9xup7%
z2jh)y4N)m&S;)|%ned+?;o;l4p>aVPJb;Q=>t#sLY!`9!75X>gGmb3w?d>H!en_t<
z)0)+&n9vHuRJ7A|<)awN>WRE>0~L#?hnH+S3A-lHbBm^)$}tr8ei(*JA{L7hgku`%
z&3l->PCQ0!%~&HhxsJU8y7h=Jj2{W#T?2(e*!4{^31Y@)vw$z5J8)rb=Y|vI$I5IO
z>GuyzeM69$S5)_pYLamg4hRjDuEUekxn)r{EbwO?jGOW7&G$eeijrR^yW?4{Ai2k5
zW=KmZ>{2hJ1?BA#eXllk5Q{WS^oHK;IaSx1=#>S|%RO@`x<y2!dm28;X#P$7*oj{Z
z20hudPWz+wG%Qj|RiHBuO?6DH_5fKx(u-C`L4Jz0=Exq<y$A{&UM1g%MmDZ-bi~#x
zpiSRg+Qe|Y4R!e0i1D+_L?b^8(tC$r5{m~U4(x=sz(dK=1y$s&87mE9afQ0oxr5|A
zs_8EgG`<_Epi7HwYr*@G=Qovs@R{n4M|Ef#Ku;kQeD0}F@CIl<<cPzw9#N`knPooU
zYqlTn6bDK_T+jLxVS%2Z<m)6X9H0p|A^lxdYdPiWaxTmJu~J(7!5x9_uGrhqn^-@M
zeXMv*7=fA~jbenWu$N<(oSKfi`Ba&vC=Sh|Sxw{Pu%4mKh3Fa7w&COWs^7g>`r4u*
z=;}%aVmcEl(RgFrOaWp=7E)18VXBcGpuh#Q0DA#PgVPkQW;oOSm<3Wx+_Cmlc$JE~
zNtWvAoHQ&XTj<KI&^_sOxOPLyI1IhPAgr<}dI7@6X8|mvBm`ww6&A~58t3Q<l<lS-
z1CNXg$M0r9U5cRlx~yd3F=EOV&{1><;?XfgG+K8dOIY2M7@6Eo_^6d_65}7IC3*8c
z<V(fmIxVA-v4?e%#w1S}Q!53Iid}@o_Tn}sc(}nOx=aFyK-0*)gsxz-(IYtP>5wca
z{wb!%ErTEwP31T_asOiJXC!ubk5_JFolr1$zh!?2E`q?Fj`eCo-i_bsz?qgmE}HVy
zmm)>A-Zi!zJIB}KpG6uU->6vE71UVr<J>HTnIS%P*JiP#U9)?{5={W~l(59W-GbMH
zSF&4_ew%oGn5*6D_SEMblE(@9ZKB3IZYaGJ5a*?VNI9MFJH60tSKjGx>0i6C12WII
zk5e{Vaf!omWzrLFJsPP@T!0F}P{+|6O0;i<Z%iWSMy>Co)3-K`<KAwcvOqwCi)(W9
z66PtbNQ~`AlH({njMkC$fwaOSfh+W_=)1#YNCtA*mHZvA=s+nsWMX_GPDtl;vs_%v
z;Y2i&tWrm8i^!wkj2yD25~=V@C5eGG<_>!_f486d>oqlc*NyMs&?G=JhaI;+IC2e7
z$cK&|6MZc-dpGpB1<t21eMC^l^fUPET~z8<fR!;?GCm!A0I8$Zzna2LQ0pp2pFEj#
z$k|zYHj-E1>^XM`GL|xNFw{6%)bwtvfiWqS11+vR46`0I)4(6PskU_mNl$DelNe;<
zom{7cFbK_HN48}gop1akyM7A^8~~|4UKj_p+3T@;JS^mR$n1H(UrGoYx271(GovU+
z)yfciQdIa?c6Vb*S)Sq(#GNMktHf;=tGM~e$H7!5XVHzY+8MLG7aYK%65~S4{j8zM
zWRr578p?$yF+KJRCZAuQ9W$X|HVS2oWGW$;rq?*5`3drV(f9^HmRuGhFZX`YrIqcw
z;C*M1HGKGH&k)v8K>OHzw<h_ySCB1^#LY3*gAe0R&!<$6v)lwZs<N&@5lUTol6q>e
zgOq|2OC47YB<Gjb2&+DC1{{4+ARdx_4Zi#B7Gvc`oLkF-jKw&RTxsXYUU{~kXNze~
z^b6B`9oR!GJ~ZhFW~1ni;U6~xHGJ8%%`#1(KBqr2NlLWwlDB9ASIJzhY&mH-qR`|^
zAI(Ql-NZzIc=(f)5Y`oDcfIy{$Vb9>6S_YY?SCoyV9ijhaow2C-Ca(ds^GW>ugM!D
zO-m@SNX9ZEzYsMfX*yr(1vl(i&Wd*?305k;v`N64kBQXsaNfQs-u_xcL^A1gx%eTI
zJEF%beu9fElXu%a`v}$6vKW+i3L3qrvnJa4E+G%S=R&CXa|2J1<gtZW6647Q*yFmq
zCY~_X;&eQ&7cP<_BRVL9DqLn~C{Vb|F3<nwOH&kj=rg{CZH0pOqVyG8hj34?L&vfH
zt9Q4Ndez1hJXMIp*wT5DGtOTIM`Xd^5h5=9?rW40^E<5VCTPCA8QRgxeWEp<#_|cC
z5Brg!tyg@euhzBH?s*>!@kL#--HVgR=O^dm(5fAEN~~9G&ch@Jap|lZtMZ0ipVTGM
zunCaYs_0lT#H#GRYWOZGC4O5f1WHbUI+=i2fHz$rYeggdbcAtE#X#<sk^XQ+_ON7+
z@5LOcrnGce+iqnNW-bzrAk2W=VfQZYBT_evIsj!Nu&9XT_jtl<AusO;oQH!YgfDQ8
zi|J!TWF691)x<^T5~UH0m&X}(m^g=Nce!GmYTyavcXHR%3|b$v2ot|as>#CjlMgA}
zXx9ty(xT28w+J$uyzxtgj;^&NLz*$%ywjaBF_a{C=8QKYe&((9D16T_MK3ZT;kH(s
zYDh&8QeAG6w01x%^wcERia16@?=oVyBW)7VQQjzhIp``&Ye4wJkVIa<S`CPPH$~p{
zt3T{TYd<}WCHKd-#o;>~5j`8ykS-*lare5hDSd0Du!M=C^ay}T%_%zzLFSXVm)>nl
zR*9rM4AM~-hW$g(9W7#aAxaja*W^E9wDmR%0;i}DcN`b8nJE5#7l2x#junp4z`sC=
zKeZJt3mZW}HQ!*TvkGsA_l~oyWQ+H~FA!pe^OEw*Lb_V9j0lpO<Sf*LQG-+!`zYnI
za5T|fVhV-ivhwNW7E{Rwwer(Z9|e5Z?i0tF_Pcz~6GUUfa7q-|uqXba_P#HK>SuTX
z_6P_e%YFy%b_p#ELsxc%JJOB$G=pjC4fEB@INv`}$_rn+?ll{^FRESODjn+oEFtSg
z?B^UEYsur+bcv$e*-wC)L@9OV)&(Ck-|R&V+swMfeO(eo!&dWxTs$w)q8yekWI2{D
zy(L-akiH=dD!MzCe93!=p&zC%Y}P*EbztALjV~oo6)d!lM%`n5WT0W!J-e{iO_6&=
z@PKjD7mL<pU3iCpu(UdCOjZB9Ro8BCFvl<@uWV3^BDD=4g=JSc`OtW96l2F=#A_*a
z$`8%Hq8CYp{p5+ZUFZT&okZ}fSI^2ZA09F_J)$-+7U_n1Znw|`S?>17H;j!th~~(&
zs1o`Gp#&DjuxfV=r|d|rMWd~#79xDG@y+kjk3LL|rq;MjMh1pJNC-E5^_!VUUh*@i
z4iNZ&lCYk_bU9H~fR)A6$8}joUCVIs)D2JfPHzyBvrEuW%DNVFY0H|UkuiK<DXmIz
z?6O}+voKX>FiLfDXd)$YkM5HeQ1jB}W0Lf>I-5tNd~C@p;}#92Oy(A4>ScXJ1+#Sg
z+-MVw{3ogV7i}+##tO=s;>^Az)q^Vwq-cN6=jgz{>z4+37end{*%0}7EbCv){na4D
zd*h#J1VOpMe{`Jx0Gjq&mMFO~DC4=t{*E4>(-k7qbOH2xx7I+b$^WA7Kx>0=)r8-}
z`#{|X$w1i^NKbgrUFu%`PsV9V9h~xe$ry<8mFULWpI^Y^eBMRxihmGBI;|l4;3&~`
zXy%7W;n7b67sT372zk&uSCJf7w|&WKfP#WrwyivF8>IungM|;RfGFs`fuq^l@FL!m
z#%7E#w)f-5mWs~uWbm3S(w9t|!62dU0_rHo8-QXl<n4o~JZCL$e&P8yLy5o>uo9Zh
zIsg}(AQ@5ZhlF_ymx7aW(+`H7WVr$kX4g5DV4g6s02LLR>>T(I>x|L?;ICmDWP~eA
zY%Cg)2x7`dpga|;%;@ubjJyv()m9Do0t?1pKM%eHUW1}a?VUyc@4G-wi~aC1yowZF
zh-cz?Hc8Gx23L;Xbk5SL>uXAeU=V?hu|OZNHh_%3(khVtwSwB;k2j}ZA_%Fo>Tt(H
zmO;n}6yI4cEe<jbTP+xbYG6dyo(&s@P$27>cuj*qk~5Nsh0#Ik^U@pQKDe-(6NETd
z2aPN6lC^U~Noc-$o_NX&>nDJ8zYjqXTih%Numw7UY4%mavV}x9t)HU}_nrt%9NN(!
zQvy~(Ek_t$1r0AA1WX?(zx#}#T~6T7dC|ecrDo3xy-I#SFZdq((kzm924QHWQKKoS
z^M)60m+8QZe}T?@bKrG1#oI=p3NMh0C3aq@r<oh7bWK-xLCysds3{_S<CkxLEZw_1
zpH%lW=@B~v;X%{s;A3r+Ll_cHRD*N_SShBUvbwCJb0d&;m!<-G7K!sviRK7!AxW%X
zIu-%#0W=;$K{viL@Ve^G5TuMw#@p0o^??HrQ!coFe)zxys`(Zv8x+q0HdpP5#*R@7
zPfq~A9yZKv-=8Ia2h+C^wfZFP4nDu%2pCkN{)jJhsc}d6Efgklo*51@C-ZcI++vPj
znvm0v{b{G@N2vxb&0Fw-rs|KsSc<8Df*#Z^b305F=Ky2`^q(#uhAT~DssQUX-uKau
ztbddMv~8st6_&u?CP0ZYVZvhFWMKKt%k;iR+S%aa>KhQk2?`mz_YxH(0k!DeXNlWS
zv!YuukGqBFg)vK7$*jKtQ5@10mG_*5anhz{_dCM=BCu;jEi-lBBg19)Rb+e8M-k*M
ztNhPFKoA5f#c3`#U#mA^gIdYghW>!IQ^^tb0y6VFK%~XOR|Y6{{|&;~8=QxrhT#TH
z3<%1l4uA~UmRaDetwj10D5q9AUjsqJ(QNe!pog9V01|EuG+6~1Vm=E)JNGU@fr1SI
zn7`_A5T(eb!?K)R5*q;r1hSd6#1E>IuhwHZJ_0h#mrR{Aw9l7x`)u(Ejb~+q0s#VU
zlh;m5h7mNBPKCAoeBz|>1|hN?EqE`D2KvYih*q^(T6hFOb>=#{qdL4S5U`s!Kz5HL
zB1V{GzTOiJ=Rg$)#6g7clH9o-ubsrLCQ7`9?wyth)eacqEsU@_?5$C^_cqKIMD(jV
z*%s;&K*x#u1Qa)53gr=GZ{^+rGSxf<XoiA7seAS(P}}b{qWQS8egaQ{;?qEe94JE5
zyaZ|^!FJE>9SY^MZ6?+%3r}`{8;nu7jp6gB<Rd2ep@(nA;Du6Dz=sU9@MfU*hF2hd
z+-BA{CLN&}H)L9#UIp@>zB<}d#w0ujk7f7C#~N6Oxg%G9@W-)pT<IER)@Rw9XR<sO
z3r-wC=DuKsU-LW7d<O(OPOLrWXv@tjKox5*5V|>$5ucFDs6Us*_7wrz<X0!LUA#hW
zkrWfhX%(ZOkU*qu3tdQ{Wt=wGU5<H@jTURX&F@SDL2pw$FY+v`He-7v+i4|<{NSq9
z`#>15q*!Oo$6I|_nV1$Smn&nv2x(m>85aJakh3tvES=%_)1j=xx~n!alS*<N!e@YF
zWb#9VL_cn@5m>~A{q!6B92<!21~y{+jPQ24RnwatHWUVFcTfW$AKT>3$b}bXuXe)7
zBzYpRPfS$C5VBxaxclt&nsqEfe27pANP6>tfS$MzNr%N0uN~U~uE$j4UgO2LY)v!q
zHZ<rNMT|wWIZXv}Iwv-@QVYBjOANom=ADSfTQ<*-1>gcX-0)-cuk&TiJlRvhhAM8Q
z9SjibOv{XoQRFjWpjY(4vCwSu-K)AMbXQ|0OIhR$WQ+MfQK?Ooc6RhTZ_zHlUujX>
zg$R80eSvYM!-YMXV34<{dDz&k&ET@<fQid(BThiSeI<1N04T?s%IL&$#ZP*eryfS7
z!PNm6u6u=fjulhKt`nObx<R14AbQrwRg89Kd{Ov;7$~quW?(|7PI~pBsZZ?YxVw2}
zyBYpw2*a0euyyjW(I<vW54z#XEaxgWg5-?!K<h6l!Vuo@e7n=+c&#-!&2G)W$7e6%
z2H^J*pg?Z6B88OTWig(3RIWZXD?!)cP8Uq^pk!1-6_TOeAnWUA^VN*SP4lzhzJfR4
zzDuNudCI$q^okD+(pL$4T%~pkZ-AN2$M=BV(qJ>r4Uu~QPd%kheO4l!qc7WRdVeYd
zy<~WsLi#;a|HY3*vsnNPL3Syo{Gj2j0?ZNe9ZS7Z+nX8d_VbHh$yYr;n^d>tT4ye4
zA*fUK@uL=*S$<?FW0rZ0`XDOm*(}#dX2=yv*&h%(qupn~RY23dC`uOHv&?jX*ux}q
z<~&D)BFQe7<myZyh<n=#6%U}Hs6O=s4B4ZPKNbjo`$p+CrKy=0PQP6a`5=VmjT;a>
zQ!>vv8TPYCXi6mN4R{Ah2n#{~t#y{mXTLeclnSOPa<EUx<Lpx|%LRw5Ti=mLOlu&<
zXgk%%fH6wq2x&HD>FFpK@71cBMe?$J^_#s%|Grai_5AzM&&eQK22S;4c-fK<4Wy@1
zmsfx0;$UuEzn-WP{}^SbI;BuT;}l?9=7XEp&HF(&8fR>n$a&ge3%IXf@OlI73G2Jm
zLM>7bn=#}n<AIQ_I-+}!{02zM12kfd@-^@x44;;(dDnUxW<R&ju{Clx0}`aRXM#9f
zjuaZf9F%lu_pzeeWQD>s)MPOBTEL7pK<4wFO&)YXFj+gB1$3~ZCXV^fuqB){g@oSL
z_QsqSlKp^^Y;Axn1g~*^P#T8meO}|Q6!^5qD;PuQ^A<kZA#G?H!EuYvERlIlFge~f
zoT`W~J)dnx@0x>-Q^h}&By;I^P)K@}{uRU2cMWA>!WF!g!npW7@06k9lhofvPl?Bl
zbIm3@lv)J6Y#b)keM51)9C)QZ7`RmK6~M+xK~^pWdRDh$iKGqr0N&HVJt3uIUR4+U
zAnZ7Jd+4NiT&m3$aZktq8E4mA^l931;@|UiYMo_KErSCUASqdcs>&d#sg7qa5M<j4
z4t!SUi}T?D-erL~2f#^e!m^c;&{-sGYo8yRaIMKmt+)Cavt;89C@{LeeB?geugVup
z>H_<Ud^J8;Ln_Qlojlg)YW8-RClo=bSm&|1iT~qO_X~gq6;$>8_liD>=^~VM4I+q=
zdW_WU=D)xx)B)vn0(jWm*0P-d>ZIu5>k#{b%@*y%`SC$o2lVrAK)$-6WV?uG<sVS#
zcNRS86K+(5ZtUF!jt_$p`~nOJv5pa1>+4f~CBH%DQch{a>k$$RtaZs*^!kQ;{ct8J
zUOeZM^3vQwtC|j=4V5LHn`c5Kc>J!W3)F*2<;E?LM1-yS_7VtI!XUZ~Nmtfi6uGho
zxf26@`$+A^-ox;8%e-ats-c!$epw&yam%>pt`*DP0(lmHt>TCYDNCBo$Y2Rr+lJKF
z%Hd3TL$gT6f#dfHXC2~fa4WTTWchcJg!Qp_nlCLvy<dJn4H~IQjU{Ay17L%tV{7GG
z{QQ_8AS2op%R0H)6-s>R1?sTIhMp_l=}sN&5a*~}lJLg5Asp;z>#YUluHi@FY0xDo
z_JDfjXgoLP!K5H&=DGmoANoPZQ8+bnWP=lVE2;v^vLo*(9SPiyRKrn(Syu})s+xI*
z7|En(?=k;y-RL#<X}DU^AH2`@T#C~Y<V&$k*xt0ozb|y<%xA)ShzdPiA|-e#DXY9e
zFisrppA<oYn)V}1Xx$;*mhg$zgdZY$6qIv@j%Ux0=23<#C|njs=FWT8yt6}??Dt*f
ztsz$vYlxg$*12qFiD;?KNMT0cE{|6y4Dp<ydt-2H5wjwa%pyY{S`OJmMr%8xOBX8c
zZ9y)*^ixfHqB}Brt3~tinzRbFO!di8alNj}9uD(I|MZb<3a#gr_BaJHefsd!RU=@r
z?B5VzB&T%VL4Eb~9@TMz&j=l3UO#zd_<UVfJr54<KyIv*F$?RWtX?qRQLG@$&HYL2
z%uJ<`0ut?%dIU~T*^j5KVr=@9I(GR%K_t}FgQCkfOj39ZHCv^;LE+0K$`5kC%ZbI<
zY2SNFg*!{_CaG=VsM4u87hcOEW&3^LnFo_>CN!^1`q>vzqB5nBXCGxf#pXwJBM=@l
zVq1RlRDA*&DaqCqOpDOQCMqwRhXOCHnZlekt*ZSr<e9i;F9!`5#PZ1a-qh_?!h;B3
z;o4hH7Po+)5_LOI4M#JU;>B;LZIp`oPj~2&hEXNUq|tz`4EODJ1~sBEbw8gZ45z{x
z4L1qaYAMefbr|0~_{<yz6D`-!*de;d1Ft*+??u_09JbROH@)&l{@kB1g+)JzM0XMj
zia%jH_a9;55@L^GGbg!)H0M6s*_UyC!-|A#E%%@fAXHkd&hiL?5qJ%ZD{{`4A+N_k
z6%gW&xd|;_{UO<&W5T5goD#lQ)3PDTD<(+=%lad8NripfiJ_1+#=SQu08FbChF^p7
z;&8(J2S}N`1nK$D$yh>v9IfQV41f46@z@$~NaQ}GUZ#KpT^^<IB3slt&5D@!TM->i
zP)(VHVsJZDjnJlZHIp~9=mYii`Xw9P*I`;=qW!sAy7mJs3c;l4v7X-`Xej=PNOa;$
z{b>kbX7Rz|B?czS@%nq-quE<(eqw*>Y|C1I)w6>QN^5RYN|>b89=Q2Z!G2e{QKp#~
zmWl!oAs!g;vc1vddVtZqowRZuRCxN$!3CX=X^{gj=b=hEXNbrUh$ZX1i7JsP%a!K0
zj-%J-#y?@x;Z6!CE(E||X=9>H===bv%LnXb&u5jF1qzj+_(RL$NRAN=_pstsI!NLI
zdK6+ud|aE>B7j~qG{}3`c6U3=b=zI|w>_>IHG2)f>zd@XAVNeYBbiGVgk)Hg_?O#@
z7(AB&NngrvI|N&Qg~CqWq3B>Ij1xYN#&01D3?qQ7I6;i!_Mv?NtO=GNyVhz4B)9n1
zX-!8Q`9^k2W?U4XZ8eQ)JhB5pWRx*xjm2ltS)dS!*Z96sz9m{W(j?KNN;Qm{;VIT;
z-8TX&Ad=Ml7H@OzSHJfrYvQ^4ZfX=i3@e^6zFLZ6AJj9x3CCpK_>CtDfD~4??1Kmk
z(R}c36i~DsUmI_)pWf2`-cQ{v%okPHwd{1P&Q3iPCCj;iR2A|&Gi{iLMO6wgEq(2p
zMX!F$>b`yG-TDviVTw5ap46a<aBcud((TR&*aUt2`UV;~Bq;0ju2vomw+21;hQ|$`
z3-ZRntg}DBq80wKtvJg6!``3xL;1FE;J6thV;^gFW6P54S+b2KBv~UQ357^PQB*Vb
zoscC&S(05ONtz_tLkg8b_LQwfvi#1w?Oo63_x%gL-`C6YyoK5B`?}8SJdg7@j<bH?
zh~vmh(Q5|}H}Q7f=*5`MxS!Zs=y79wOX-(Rp{Ihh$@0eXBo+tV2*#t4B%!S>UC^_^
ziJv-gkSXLZcw_><&~IrfcjkP99Jha^dYfwio!2DYqS26t{#b4vX{8hD56yGn)iLdG
z|D2pL(>@x`=Cm`I-s>q*h_Hx2`TZ7Dm4);=yr#C18tpru&OEzOP5Y4XCCe@UIkue=
zZWh`a!2#yk)&!ChEwFnEYhPLXM2L|7gCz;HS68|>i;@Le_-WUf7n-lytv{BKd&DUx
ztsiu-_uwq!f_-*6)%o|b>2kMu5VilV?Pwf6*W=Nf`+*9P@^NgyXjlpepHNzd`jsE(
z5`N7Akb3sE6UAxH(<^XCIx}5<b?KIR6ia29_I1WPYX6ar60M0-U+5VKYMT-K=iGpB
z@~J*Y$sNVfgBo4``c~`glFKeU{g4)jS&jtfQe1<;Lyw}fV>P>Xty`$41U<~kd#aiF
zi2JE<`l4!)q0apGsBp(z4XSLZW65t6A%(&K2Z;?ewC@)I5BPZd!ztdqLkeGdX|r((
z)UF6#AME|rXD7#?qp{DQ5<CHR?pO}7%Tzj%&Z!<CTlRJ4C{d2Fs9+N)CeL08EW2Bi
zDbH^4aM>m?b8P_;n1~BVYG-*j(TJo2Fx7hF^+q4@`r!u`wqV;o)Y)7OrGu<xg_=D4
z9}`iWj$~bC#~`4Y_3#_#M4CB07~|O;ABjlnWpskgwgFYYwn{E28Wjf+d3Ob9>`9c=
zNkpYJdKZEKhW`Fpu;}V_eQ4)Vw%ZHTB@~a(e0T|daMprX%s9Tg3j(jZfr|HgWkW(j
z!FC=L^<jxjU2{>YMOG_O>b_y|Y~tbTeM1MPQaff@KLe|%Cr)6PHQ(kUfC6BvmUmd!
zs6%K~czi?4_=#m79iHL)OoL<Z?yW~P8CZ)B;|kxkwO_0+Wi<NcYjv1rog!s6om*J^
za>J`KS(K>-`x$b3V;yKyHgL9NpREgf_kT1f>T918E2d9Kr!JspGLnlKr_N1(xp-pe
z1-^XrxA5xE?P)_)B(I(4um0eh|M@W(`fZif#|?Pf!HWHx1WFo*1#S4T%Kz#AQr{xs
z!bf@_N!4Hb954YNpkRu89lSU7&yV4+n&B7CJG1_evBX_CXp#cwhW`)Ea}0hl#%=Py
z?=;aKh*fPGUq0Ub_mzq0Q}Bzw;Mw0F#jqTr=Qw}h|8d_S)xj_7!he#s|9ZP32q(%X
z;eKHG|9&`o_=Wu=fHeQUMqq@p9oX1+pS;odZxR0d1H3f+;{Ur1KQ6}qOK-!KZT<tW
zwtbl0)Tb_Ya2HzPlPLs>jAITJ64Ab_3pMJ>f@xFRM%-1EE3W5%@o-?9TpEsi+it;^
ziST2kpVuNh)%_V{Aidr1QhKEhIOAH`twX;4zt<q?92PXh+cg5etO1I65eOR~7rn!;
z+JCnlOh@X01z2`>@(MH@=fLR0-jy5J;aZ5k{b|>2pvhV}Higna1l;b6<PL?FTfaN}
z**8gKaI9p6as~WIS@CW_`~%;V<+`t6@3I#{*^@XPgY*$_r|59*K2|{dj9sHgU{?$r
zCJBI(1+J|u0*iGDAgyO03aoN|0(hGglWk_$?|p0Lj)tc>=$d^R3{RRUFSeW9{-$|;
z^)*28XNQ=M2g_7<$^$!<R`rfs?-=m>Y)(J7T28u~Ar9UcT9lCSQIim6Q#~aH+ZyvA
z=lmBg-=)L~+dol_U3>xPSH4UFXbfA<z14>y)KeZ>V1Kd%pkMgw)^8x;w}rUAl3B6V
zB{qtF@#`P~zf<7Ou7k{E?CcHz8o@L4Xi)s0C_(VSAf79T|4ZY8Ywm;~ll8)z@`Q;l
zw?oC=UZCnZ8>_9Ss&>g3(bFu~-5a1Cw<QLg0Rc^(amF!|PcLYGhY#T9-#_;``rIVK
z_MNGWYNT8WUI6SV`ACZDA;GKg!?SHe#%u&IofQDn+13!klX$*wr;ZQZ5<R>46-bEA
zCj3h~>3%<5t~?c%7D%CJzo*UXYe-JxJwgupK)sU<q?Y@#yJXFp;4{6srW+A-)Jv0g
z@CErP#JBN%xdsXRk0O{5RrRBV@g0^5f07qTGsnr!3=ky?4D3eS4590h@l|65vJlP_
ztH><|ROV)N;@7n<*(&rt)_YSQ>);6wIYKeGSE>qgartUB*B@5@Wzhw&a(z}e0xV=H
zvnuE-jUrYLQN$?pi6+lv3CDvWF8uyq?MT7c9$&$4EZf8lKOXzT@ZTR#hGgLE^tA|%
znudLuN(Ea`w^ius*%eF?0dGFvsjmlquUjPA&~9IV4Fk<lX8<xxk0?JY>MhLlIv$(b
z)4HeQEvI&0eBJl|vLxVVhq32IGy!|s>C$cviMeCoY1>!O0mh(9P<C*f)1A!WMNS>n
z7a#xWyRqqx02gQ)tazAf`+?~S9hXy`>2kXVQ!@=C;eCbjy7~Vj_K6mhJ<=2kAH)^*
zsZ2|Yr<WP;gvg1<oZ3n8wcr1IT4V`Rapba~G?*1VUX+qweS0v+<*-<g<{;T!``QR}
zO&}{%qxn9g2@Tv+pmO?BYQgJ%=eM-#sAOdm?RI)@MK@r7Z#Vf?Xp&vu!tD_RuW297
z?i%A&P>&EH{!&-{xOSv5ngv5-%gWdhiG0q3GrOU?^SbWT<5zp`?0*iYaq;Rxz3*QK
zrMQ_M>H24D&bML)gjf%iVFFwu28<ury8Jx;ek8%U`i@{NQX@YIkV<stk=WeUk>~eS
zL9=>3Xjtv<zZ6d*BeLPUzEi-3y=D07yN5yxb|Y!wB0r+*LiI}h{j3C1lA<E<JlGnY
zd|5LC?apyQvwzjYzusq}9|`E;fWXursBjP^4SH60g){J2C+)k}D~b4F9i{Fc{+sVl
zV!_gbI1aM4lW!W%($o$YnS-A#m&G-a%rkjb+dv`E&RBHskMs+<Q=+)(w+}AnXr4E!
z997M%*&A%pcHr%qxL>^Wzb{1*8k~_{0y(`3tYWvGE~$zuYh0a0Y`uWq&73)vkN%9P
zVFY##{QZ_tR6-;*umVJyc5hN$nu;-^M=BcQxP^-vooD$GU&r6~gs_My()NZV7jnj{
zbq6K$VGZicUb*w<58zMt-y<gq>LPgm7QEDlc)Nt$nhmS!)6N(xU-4mPs&*YmP#PUI
zXQcb1k!IWbdzDw!rk+s)%Ig>K>P6IdH4^r@1P)jkw+Ql4f4_*hSsKOHam1P|kN?H(
z%A(~fitc>{d(%K803X|{bQYoY4-1}-`1_vb+MvIJNFH`n9T`o><`I%n`wVFNApVxH
zq)n5MRfgr|%j)xQ-j}GKYs}3Z+BYPseI;*vju*rIDnNyLNAjPz^W)lFq=9g$hV)vS
zqgZcFZ4NY8Y`BJoftQ!@RrR}|A}^4NcvbE~y0s#HKtA{A-+3G<l=c&#8iGt6A_TIW
z+l$g6=>MrWHAjZ>Gx@6tz=vPD=Enc~4j}g;QI>?5RzVa5p}0iJdhK0pTfTTBsaP<a
zQ$L<+`*)x+l3D<dK`AKkN=k1TMRhDb$W+!}c);J-VGFrtw@Z00;W`Fr=!*b$i5*V^
zIOf+C!}E}C&6deZF6i@|_ykY`h!84zdBxaT_rT*!?6+Ctwb6cV^#~gbx^d9fd;~!3
z00=eYoLgK5HzD*UO+tt)Q|J<m;JpR+a^P-j#_vx-cua#hNP+-m98AcpyhKm-UVpsn
zeJRpJO*!~b%CrE$;-VDcqpIxUwdBepkG3#yHz&M>{_qU9|G**tg3Q-WTbn-oS(-;B
zkOs@w>E1l`ig<)M13uc9MKi;SDM*FvX8JKjM05cz0QMXWgWvlXJ3QB*D=M{H;&tKg
zHDwl$R#cpRb?2mNYN@$!y$CqqZf~#lOf6nR0@=osRu)M!Gzka-$QgA_EI6J|4fuM`
zq2KH{28Axu?z2EA-gSBw8t!y*cv*IaS*f>0=6SVXE&zbxv$UshT=JVfO0fR^<wK_O
z$wv5jHhAe(AM1*iIsK<7689w~Og11E!AnxEY0`)4SiVCl(Ow;pTHFcMP0BaatpM5-
zed2;Iw9z@8?ttJ0^su{z-ykYlXgJ?cgf`ea;3>R?#ygm9AAx=VVj2$A*~cjsN$IrM
zHTgpT`a?KV)Fv75pB2eJ5}6D3obIrt8=+%)?(4_%fS?rW;Vf<~mHoLmUT8Q`rXa(6
zR^y&w-99PU1@|fIuGmc<;oIsH!e)g6@0U=wL_l9W0*(rqs=nKH41vBw3Ui$s1Hp&Y
zbYt^##YH~o1$6>1S%1sNfr`WZB7)w?iQCWYNXUidk)Vo5W1!2*!4bzP=LGn}QT8`q
zW}n9xVGyS-n{mX=^k=|dqnaMb2;Y*Rx$366b!RMb5JI|-bpFxprifx`>QvQ!+LW7Q
zfL0cPJ<(@DZ1oE$0})NXFr*U#W1`0xM?=7<X~SrHncn@7L}L5N*z;ryA?$u*6+nh7
zfboxrD}G142`R#Z?wUG#u%qv|n<%Jn)-77&d33v@M(`~SMGj+<4;$iiE%5@3r!p<-
zBrFT{d<(nInkV3|Ji^>(L3Am`J<m`msxkDZE$R?t`vhgX7~HyXrgMLaXJ{1d7s|cw
zGQ=Un1}5i7$3CoU+rA)4`^hK8bJ0^`g>#HiR2I2eR4;2!>|LyTwYhZhaKBV!G-AKj
zCoqZ_?ROm!a_AiapY1j8?)T8JFPuWxE9F1bQLp+hu{=u)gcKRXr|qdaEGqMx(~8I2
zMALZH&Uh+5p`rR?ENGUiPdt7aaTk{he!O|Ew{)bL)zsw}rXO}UFmb1V-V!s36a#>Z
zF`P67`;IL}j#tY4kiZJo=Wbvq2>4no%TFtR<U*#Dk%3VO8?*hi*-)x1@S^ii-kdID
zrFD^j0`{lhQXCmJ?LL`CAVW3-PhszKiVbM6R3W^8QzVrhaIrU@BWj1--MrC4QKI*p
z*6=L2ru1+<gc<_!k5y>mft&lbf+ITW+kZOS!*U@lF&9OKZy|L^)3i@HTiS}s>6_U;
z>F7fwaKJLWYJxcf;-HTo+^1^6AdCOp^figcPK0JKt)2jvxf28BeCHW@&W?hfjHeF}
zGqn&{P>n}BQumVrL4dS{iQjYYT{gnoUuOY<ixxK8)=zIct<1Pm%g$ViW$srxWuk<R
z<Ej_V^eaTpTv!Hp0w3vUs_iz->Mx4#xWOv^`VPAqL)+%aEu|(;sUWe{YZv<cl;H3x
z;SG!wpA(Okla{0$K3Md+E1p(50F0rRF<E(pU2tU0xMYD}=<m=9->n^RyliF7CF@AI
zVUoR}R?={TQDubUoM_RZiRy;ECuWSKAQxs4piutjVG}Zl&r)BR1r-$*fBUZXw~OWN
zm7y)z_Jv#<k>7g{*?{6QBw9Ebf)(^}Ab`A2Ji&15&VNE-<aUdg6G>S!9%n%^6}!0K
z_WN6yT>#nq3E0$SrA`N5vLM3#&=s#gfe+c(l7tbkAvgp_sH8N3SZYNhIr7fx*ltCA
zvt!iFV}FV+_?(0cn*C0?Aq2PB+)hm+b~D~r-22Jew<7<#1mb3rBtYhi!B^X|(Aa(G
z*3p~oX7e8!KfgE#4%|~9@;N-uYcJ=H6lE`&{ePMD{ERPnY7($F5dKYhGX+X*2nrg+
z0Dt}Sfa+Z3p5z^{Vuzl*y?+4z79C5IGB4F-`uo5YFCa`PDH6&+Jq<c9J|#~Z398{U
zyFrmzRP^ZRk-s6X+(>ln(FI_Z0sLJS`Koida4re|H23+{?4Kk$#Mb(E2%_yjkb7}+
z17dpl^p&I*>}=oyYyH*p9NRvT@4cdBGbA4F0mp%@;=a|0=lqW-87(2}JJ|`iA`(j{
zE+3GdJhOan#}}t-VH}VtS-@*9XI9^}LaOTGrOJBCzg|qd$BYEVq|wua-PBmT+shhv
z@B)Gwbt2rS?t(nzRU+?H?xp?xfW$n)?yr!gAIRhZ0IvK0k97b(3Jq+lb(nTh4ODvT
zhuu8FMZqC9HHRP>MGRyAUcT(Y-XQoo0sOt^z2P~vTVO%FfITj$|28-kA<fOq4d43v
z;-r)io+~O+)PaL1?8!IK+3a9{<#YKGq^9z_I5~cf8~Jrv+O-`4HGoqaG6e%wwrw&;
zEAMJMa3&{4$pwq>9{p<m`%WS&!3RmZH#atO?kh0W+??Iql+oR7ew|)WH<U$GcXy<a
zrJ$&Mzi<l69?3ltmn`~<66K6SBI%h0qjb3BFJEJ6Pr06Sp`zlm`um}$s)&H#{`aiv
zubT7APwpZaF1Lfn+v=~YrBA`d4H9An@3=cLM*1TkzAc)~v$eg}O}<thNqkE)@^UIx
zp?eUV(Hn!lt!Z+6K8(>L{P@BjrGdy@8n8G9IF2TqT$GffKy%x%zo-XH;o$CbrE9z6
z7x8%L9hg~V3ngC1$i8wOZymPuTjQ?9MiPH~5%Ac~9IXeA4}-N?aI!njE{#hZD1W^)
z(LpJ<n_G{U#!w$;P*-e+v4|ssf{t?u@{KFUDF*^}|9le!Ot2o_@C_|UTzx6U7z&;V
z;kmT!TvBAvLWy!{urJyL?611;lYLzGYzqE7Mvu&cVdUGT2#i9=kOT76yuRqjqm|va
z+>vg>2CLoS&cKSKQp{3jtN9mpfk_BwSa5S~#fx7b9A24Bg@$VH3^W>P*~CY54)K><
z2S|Qzs?@_*!B18TJ_f`PgE-E4X!i3kFG!%)*ixkrA+()u%Tr>9Zn?C<V>vd%H2LD|
zcc6vvKfV%gVP;>H1ww9l+pSk95#xPJ_QKi<5Z_c1^Pc6}3*t4<2V{#`m<>;(!3V*k
zFF%Tz7T1s1l8)!^VvGB8SI8$Z5bVA{vtil#-JSR+FArz%G#g2Vb?nhkY)Y=U7DEig
z$V!u&$uGowDPweKm~p6|UjodI$$1bApuKlJVKwVoxsiUNp#4o-8~Nb=EdMw=%xe+H
z26hmxKeINnyGpl%LG;h%Skocmx?#f2r@bc?dQID~@~9opr7y+epC7Y-$Bz0+V{<aB
zyLj<%5kArcZxr+Mz7w$lSri*<;B=Y4SNd-AyY)_^ugKUK;F^^mexSMi^7Knb*2Gx+
z;BrZC6hR0(vex46=6GceK%8$;ZVo4YEg$5`crY#Te6OzQQY#-{^5e4*UHJ4VAlEnA
zV}!KFZ0i~Uf3VgQR%e31xnj-L5Q<vPjX3rEi_{liyKKx9RBZCCh1hDaqnHgN$WSxT
zZk29;s3o*JI^VK?>INW38c-%jv5Kt*@y=doz>Y^AcwBq{6TF)OMRLJBywWF<5Wd?y
zDKIf?^7DKmRY1KW8g+P6O^-9-7F?dSJ%~y1^I|Ky1|qRdI{}HJ8(E^CcQIpJJQA_(
z$8~>irk|^s+<~!Z?jcI3K$(bC^rzoj%a}&2rs|=Sm_?WyuW<ajSHEtWp(Y7d=mqc;
z?#fg)y>Nh~FXZKI4;lM9S;2ibnKzjewa}Yg8RxRt&_G#vftMH$d0A93FMilxv&cWC
z;z@y8`@pjs-7y4NETV(wQS#(e^>L96&ohrCSYr(n@Y_H6mHpnQzwX*R4U+BvJ-1OH
zd<!1!h3k1y#KUG~K#S{9IzVgeqZ2Xx9-cPabXu{bY7jdeA03x$6!mMX3n5R+obo*U
zf1VWiXFv|o1@k6H__Nu+m*mHVVbLbY2kV(x^9YR?puv!pfKpjg0O}!}NLSWm8dro?
zduE@P7_`B$s?`9d*iVc_m#3aGX_fFYFR-D4Y{NEX(zEtQ=6|?(*Ua_LvK=$R8`ygs
zn1CHf5=q=~uokku%xl>JV^JjD+%2NNcjk}%d2<iEgP;4rNz@y7I;D2E+iiDU+*pJ8
z1gnq$&v)*(!vyaefizq}yE=ha)uPI2_m)5q!o9xm{e#FY*Qfk*RUUr7b~_#|v71o4
zMPlbLcPZ8-o#JKx)W*En;l>~^Id2$SafK~j*aDM$a)CtM$|wyWEr=r>4RBQjP;?Z*
z0%dzQi!}b7xQ<2>scF2Qz1*aHrV)B{96<s2K$Y585c5Id7NyrbyuEWkbQR%825aRR
zq{$rKfO=f79Xa}klAz!`2@dkT*LTNkmAL>w=qA(>Us70py-nOnbOe)trr&hpi3z|7
zH{gGB*H~^E!?oCd^?ckRjFZqp=DSDRBnNmk&iVB&J_SI&eH1*1`@udTjBe{>`(=-b
z_RH(vzk(|De86+?3@z#fl_CN<mfKbr0(W@yXo@g;L&)r!4%G&`ejImLGUK^!tyYNj
zt*WqV{pSX?$3WWs0j4Rrg1fBp>Dhk7;}{^Tl-D)xmhkK#$nJu}{XEz=V%em@LJ9W7
zs>jOg&!tOXp&99ox}RmNu9iq;4{cl5tW){`x=(L*W+-%|NQ6jbrqzfr8te;<Ko(#I
z-Df>agCPZK`DQ%REn=d>t%ir9g8Z}rW{WASP`sS7AP0-R&K?C^eiaV8s_N+6E2cn%
z)hX_~hFys#fSE+VlT}!UuAS-HQ{QA9yPkTyfUy`yTj0SD@Y@K|m5?Vih0!I0Rlru2
zuzHk52tZ%SXg6Gcsjcs;+7qy1vgWZxkAB{QMqzR-!#QuzrjCLLcX|z&^if=V*FS(u
z!8b3dS#NbL9FR=K#g-V0NHU0uW0kaTS`_p!rfUQ$35C$fG$byXUR593gPI{Sn`ptX
zD=1a<3wywq&=pm%o1B12jJ)kRlynvHjPF6f-pHOuW@Q<~U%dDIVLXKfT7GYec<)_b
zSC+NRg{Sh+-{`X{-Vbi>-Ecy8c?<ze*4l6IGaa~uF;V-Vs3^ob<ka`d?vB?XfETM+
zb{Fk)RdJ!Cex=N&ll(^cG?@+~BD~NfRm3DTatJ+*%QbS;Iz}@-SAJ>_ncw}<W{`EM
zjW!9_rOY_wA^HG@o=y7j?^V(Bvo5H?u3REOTYXw-`$c+&SS3=MoU?Y&F>0$?Puspa
zLB#J{Zcy+%%sxxs%mjn*@JFyS+c>OsegzaKK_br#2*bA&D0RN(2eI#aR$A6tI6#NE
zWn=DveTQl@C1W%tXOi{$Q`gOnb@gx+`E3P|{aFKP|5S5GIdA=z!Yz8QidxMnF5aI!
z<kglWdi(hEu|tg18c;wHc)Du~Lf_Jemy_PQnyqgo)v-o4CMmPH=?DT?^J0iS`J^u-
z3yytm*3QfNSIYL++&Y00)g#})3<i#FYyvV6@KDFR+NCY2A$6!~VeIHLmG2^5;g{Fv
z10a9)4sxSQB<r)ivv7^?`@K8E2!P)tGCQ``f)lKE$QD6)@&W0Is@3M2XAZ4iai_L+
zImfuN)}DMpWYa*8(*<-2S}-}Bt_QPN7=Qx)Wn-7PPu?@6?ok<8XUMV4x}xS^Ud6M!
zB~JLb!=`L@;8>KH-R<L_p~duSGG$>WYKD2`+-m<>W<yOBnyZI~!*D9g*W~`Y9N6^d
zpeX=zfKHuZ{2uHx(SAw6>$OVqCUjKV&X==Z>eCb@j?$0POs@f&)cqCsedD}X85cNk
z@DcacArOg@y2K7juRTa?NxSx~Li>L5<`pP%_Yg+%Z`2@%gPtIfz;W+fme92-sQ+ET
z_N2GYVv?oyp6vvSWw@j9TBp<2nNsR08Nl<IwHivJ0lS)24|5RN>!qaRz5P3Qx(sb)
zmzmFab|e&CRC$e0)^kvn;{!yFIA1eCclx&rg6Bb&aLvzAvmyVQJn&kfZWC2x^0>f%
z0+MxtKA}O<lJcq4r;!)shxW3n51Yv&B&>z=h5?Sd$j=h95^MdYE#Z(}@f+(u!Ms3=
znk$Oqt%q=?ei|pq^U$}z=(0||p{6RRTjamI5-;Ww{*WlYiw&j4BHKcRxpaNmM3<7W
z_Z!mZ4E!`riry9|GoCp43d<HC=m8TXp{!_s&t2g(jrTSZ_;L7?Y_OH4-7=Dwl5Cn;
z%xKc2VUEza4#yXSezM`T7~lDESG@BI)v>*=`}cejEw2aQ#rDP0Y*Rwzx6hMOqwX+m
zk%a}CVy-i4F0D8W1Me-Kr|%i;982Ti1nNAqD{!(<NhYXT@6Pbbz=Ms)cIRO;D~ajS
zsGIVfHyymfbitQH%Er-ejH1Jtj)Xp)l#oK2ejxjBS^8lyLcyosuh5@G8<<M;WKdVz
zIh&4M38(Pj;lD19ZGeqtIB`~1_}rZqsxzF1I5W?8qwQXj{KMjp6jYi|kam=YVpk$5
zUMfn>pS|0BL~Qm-_tk3F#~d6yYAf%^waI~3aa-8PyvR%f$L}$bmeY*hf)>>{K8&lz
zuiP?}b)@W()W$b+u)~P4<ypR>J#DMrbOomn>2>M(8?a@Fr0j`?0d~{S+Wij1DRZ6&
z^_(%EKMud@j|JvdTp5GbgQ-?Dn2$W_zB{C#%!2W3_9h$08>BX`G~aeX#)bt`(H(oa
zlY;&xZ3orCYkLOo?A$+U%*etTWUkA86T?&o3`GhmOrF(NpEAaEL&7=Cg{+;QZ80p`
zL@Nfgb4l%#C(8qstg#Tbj3E1WLVP1fqP%!a4`<yO8dOy;lL)sb+jl?v3TQK1o}IM;
z_Nr<}@+;fBeL^Eg(G3^Y-<EAY)k2bk8L>Ci%on((xS^fPFr)rmRs4j{t9Qe!JNCJ@
zn!YKKZF?9dm{wtTJ6a`DaA5XlVujyD0;H*%gJ9pmuM#1CV)R=p7OSGi81rRp_4E6x
z7?F7T#??xr$^#jZ3X`}hw2bWLkoG2e5%pj+z~jrxRaK^~YP>9S-<GdVcFMVL;lIk2
z6cR+@H69O;Rn=-qX1h^2T$x6uWvAqK2{qSqA(E+{F$G%F9EMz;{3F{WMqtOG=L(66
z?xNJWoF05GCtca}V?S#Qlo4C=;AmWy4vRso8TICr;<e5`jGp$$wW)G?5q6~6<Fdrh
zVu=HFqfh8T71mqgF679r##}y~g5Oq_-}MB9%ALOT^oVu8=jqDVyE`b3qLv@t(Ylm5
z^i(!*a?M)!)oyRH!#P8utEJVg#xgD{KAVOBxmy&hVOI>8vhUn)nuyzX<1*Qd?M_%L
zc46L8(-38L53exJ!Y$e(lVQk!Jx&&s38+$Jwn$*S4=EuUCf%L;nJJDip+ku}^K1<0
z>Pt>)*<z+wp@?u>KS>f7MBa0hAepJ+S&mlR6<rMNq%v2w!7J8=7S79itows1wY@*)
zBsW5hE<QJ*u?*;Vidz_cbS2tQPI7;dV=Pr7+lUl}GU}fHe2XRJD)X6schP8$9ntRk
zU;^Irm>@tp8KgWiBxm0e=e-69VGAp!p2tuWP|0X(zRA<>&r5?YGQ;&hyj<`m`bRDv
z5JS<JT_sH39wB$XD*GKD-F<5v{UP38B^?Wam`)XH<MVR7x+6}S;w5G-*;#Ysb-)t>
z%Td(1$MWfP-}Y=iWn8sQA9_c7>99}2d<JVbF)`Flw=&&8sVlNGi9Z|5`!?M;RthKD
zj23v?r;;jUjiyB#@-8TR_!Q;lF>>iEfUO#k;2AnBbPx|8Z^qfX(TR_Gpf_K%Eq>-i
zP8kYR2|1Nm+pi~yTlma3@xJ>;)9}PSx!L%o;gp+?ghr^)>B_fqMHORb=og)k%<;QT
z>6V^_rd$fwJ%hgYp#eFK#uYqWxpX9l*R?7%v&`~FH&h9i(0~etEKmc@kopz{FK^{|
z3#HH*YJ|0p5GbDJJY@`mitQ$9RHh{fuse_@9<+%LblnrHgrSt#UxX{hb>Dorf?lSz
z{?2u}A~Fc0Mm4SKL1&t$Mx+n3mVHQ`RBRyn1R24u*UVNqeemCU2A@Ll{>T_NZV4w7
z4sd|$P16Qlb<s>8VS{)iG@hz_(w<>RJy-6`eavL1dtYdEsJctlJ{!H}w7F#8_V0m{
z;s$sQsM>pc!^yj}^3Z7MW|Ve48XY|vLSai!Vi&;^^%FJwgt2{|$E8k*<en&3Y1-FT
zeEz(_xB6v?v-i%P@2Ssg(A1sH!LB4zFuN}3nKc9~E_~#14V4P9Jrt31l#cKDI&QQ5
zr2-RmK~^pc)Zj%5C8zEhlOI%6LN%a&qa5lFleZ62yOG)g8q^3SN6-IGhLH%7cyV5Q
zWTk$Y287+0%+d9E@<zrJzhD>q1&okk_t-%CaVTRA)lnkTh~X;KxI#*CjQnrhnl3*R
z;;=6Kw>BhH(u`xjEGX_`#*wBmh>wlVfSycV^{*7@Cw2nWlnz{N3He^`!tWTQ1c2IO
z3>{&%E3|E9uqrRAV(;GV2g*GMD+N<M@A^zMF-!BG+PBQ4wI=J}`{w-DL-MK}UpAon
zZ%Gg^Rk3~DW)D3o$TV;9iT0m><j)j(NYDTbg7(cefu~jWuV9GN!Om@wrU>YO-n`D{
z4P6XlNS-Z23m&4FqPnxy-la!0!|@XH=neE<M=K+Yoz~0*&-iD_55x0vUpTik+0{WI
zq2KroxAO+1pwoOKcU^%+5b)`7#nX!>vlahy6>v!9jIAz`6A7arc>|GGyEF|d!phW^
z<sA;*$dPTAZKD#9b{=vzYzs0x2N_~xtlsSOC1G7lJ{Sm4bjqmak(&qae1xG5SS@mq
z*1O+c039zIp*(9I&QSB0e#9jQV;#K$VX{Djt<a((jBXWwkC_?mk7nQQ{hUYBh2ul$
z!Zk|8U1#QOFgT!Y09t1cZnDgn%8?42Dxm`>-ZW&0@9aNZ=Y1r9OEdL#ou(<D=#M{m
z3_vIv7LH`pI0-~6peT4xJ)(I3yd*-AWQ?J$39HeOrf+i9R{iJBj(8wR6=a>?L|6&?
zh*av?`1p^hI}T2EWg4tPEp%kGOOkKr1;!XQiKdrVt08Hez(rA%_`R6QH&@Q({J!k@
z-M97StI$Dh0o2)t&<^gh%U*+qC~E=79?a*FArEMWVldmZ#v$)!`9z0PPE3fW*b4{Y
zfHHhG;|%$y>dW-U8G-c@b{8W-271^fP4CFgCR?WS-=Qbn6N_iPOai>Ek5}*nP6PL9
zi)vWZ310yCg*3e#ihks-{{dA53Zz&g?&-%R>5qVJxcUTZNZN@Y#-U{c10v?gX`2EV
zHQk=SuMLVjE)*&vk_m$RaSVnxDPaahqqN+s=6E8ZIQTB@=M5Nqd=g5ObpCwy^3(+k
zKp;*as-9t&q>Pm8ygm9>F*HdhKvVUyq?J4x-KW^Dtf8$>aKV&HdM+GnG1O)DF#Bhp
z3DAs;aX+#+Q@b{J_Vdtg^pe4dkg?TeA52y+CcElh&$s~x@i4`wHK-B;#kRVZ9+E~V
zC(EUKGIJ)8ifWPc?bF9$p5YBBy~DTGssenW`;OuuMFWrHXX0CwaB(G-=k=g(OPrd6
zx~@&CLdmB0V9n~?Mn#41Xq(_P7}Wg_OhjNalU$uJ+BrimWvdGUF{?T3un1hju9mZt
zuoxW9QJDCh(76FsOx)Frp-?i>a!9qF<wZHBkF4d|;wE<Or}q}B1H2`c(OaKbN%1@+
za$<R+W2>B2nem#$7z!1JK1lJ3I{X;Z<=@yRq|{GY0-)itK>uOr<Jn?oU}dz3-MLYp
z-z{<#V1X#tat=xpA}6Mq3f%uPhuoirHgeEzdiDzo<L!gLN!e6V+$Yv3ZK9%kwo-?3
zN;onS19h6Xa12LaXm@#Ki>qKgO|yLEZMl}36gE$;zdkIK!`_?ClW|+mjjd~rG)hL_
zFaG+yg1=q|a*H2$?}`lS%#7`uibMA}*4Mx>5tm8y?d`&B1THSJnb1OWBqW8zNe$WV
zHewUdr;c}W*_6K{%8s+7U>9~q@*aU(fsp{xJ6|2Bip9g*DBrHVW*x?l7Y{Oh|1`4<
z%5CSR1we~g(-4ie*cs=!w37V)q7TFy$yCqSJg1D57v~4uVBR^HS&rZHDtYt~P6qO!
ztHt{;8v@J>wpE8=oD4u4kEan`b+bVO_}0bqYjCS&BZIOiqV+@xV}zn$*t>6l{y6EQ
zF6{NPMxsG7^&^%ITS)AnY~aKjfnf^vu}vpy$OyczlVFUsVkI|;Ydi%#6ChIRK0)ol
zgwxlFd9PYWR`Ij5{y{@YdMqRR=>5Bob%<R0uv)Xj0S~y*XpQXW8AAVuQk)7Z!o~Kk
z!$3448s}XvV(j0#c5rbydg|gkH175qXkd4|=5wt7?D!?&ReBAYORs_V$TwhgM%v?g
zd7PT}I<@6@gO7xnm=Ud>!y2$H<bsi@P%w!M*D)c<YJ%7G-@h`^*0p$gDmlia%s?bp
zO;U&hfH`cP4#jOF9nB5_?K#;~nTR0E^;7v^g;kp)E|FhbcG5gAax3X^%+@;|16~Uw
z54nyLzYdQ*jyZUZP;P(MbC>St5h7j}jh1{uqXQkpVRoF^3Bb{yc#qGgK%rO?FRZWj
zDr<NtC}v7Z{e~+O9nlcFj(pDtaE-=mV4$l_lvHu!>E-Ud`XjXZRxFBWa7~5QAScSP
zXQ9HPpdIROz8dCZ-s(tW{x~yM$~%TQ)0{5njEEdMX*A^reHLO6^}>ojKK(J^KKB(~
z7F(4F(X@v?**8zGt-<{3KD@D6ux_1<Gix>hd~nc~>UO>-L-4L@?gQIsa#aWmO-RWC
zok@rhEvI5`8WvZ2li8af7~6IsRM7GgNf)1{pumi~c_UH!NXJ&z@Zp0;VC)%wk6I?y
z9S#|UTNX;gy5LPpFpf3KQYgPjEz@b9xC+<Xu=-|IB+(!1Sl!mL#Vgai>XS)D!`oz_
zDY0I(34Ob5_4e1m68@GD*vyN$#k<xGC?1ney=%F@I$I*%=N$YTZa411`nj;rZ}ZRH
zd-B>mG_ko$zK;YzcSiU1QtfG@*SZ{%`&;RTpY#G_RADGIL+j!NSd+{gI3=#9Y^(kY
zO>>x=O!axeikaLo*nkL%1liDeudv`R-E}NItfdN(?$3cngsCamrfvv(o5>`){*x94
ztlLxlo?Zb^)3(x4yYJqjmc8GIFIp;ED4rsLNhLfsK=M+C;r0b%t~X;)7VNKZ&?uX3
zY7-aRj`1@%iiAW+rDDjy9d5OQMO5z2Zg#;A19R6Cc8s%{s$a53O;BeN*O)PZ7eZur
zSx?fejKn1bT$t!clQ~e%OC!{%!m|#D?;-7PVA!`Lxdv;&{2+85BF#AO+g&~WVo)$Y
zzq&ZlQjUM}Y0kx!g;6<<_yU_3_t^Gs*OpWOI8aAUkg7Vw>1aIj2xLj)4eLos+gPYS
zTtA@#wyx?tIpAlZGM}Y<lK&j0_Kwl-v(S}>SH+v+{C&9jfwd71A35i9pjEfz!}R=4
z9m~r6J$L&|^9iAmQ)tLAcg1{xuGEo`b)eOzNLtOpvHKz+?}=;I{IEw)E@~rHE=gpG
z%2{p$3pXa@Nkl$V+-4YnTM(yG0?ci=a2>49yh65)Zi$~7Z&LosE)t~?(qx|GQ#yxw
zQ7e2Fg=R72F=xRsc)_xL4;7p@`At$Dv0Qkw>SOy#9}dkvsZfij>5(FoQN%3L@4t3k
z2&X_|5j6?|kBK^a&G)6%-o<`^+%^Zf2hok4qYw+54(}-eI^Oe{O4&w+k~6<LHr^p*
zXiuDNs5iHid_lVUj%urYrRUJC{k#n=YnvN6E|_{cJv<uKaF=Dir-&V|e;!haCc4ps
zUZYK6&}7guqLx~)v0Mi#9GmmLu%L(T9mBxN1vFi=VN0eiYh?XmQtJizfiZ#Zt3rza
zzcF8AK6<jI2f*l2^`p;#_~rGYl<OfDlHu4u{+i}4iF$)Dy)Ct(*);YOkm!~G`WVpa
z0r?r^#*R<DUe_@*_%1>Bm^Y;p&f62Fg8L^W^To`X6g76}#@5th3LvqJVVFghT01o(
zuiJ6JsUzerK#pq{$GsDhTskEG9nz=FYRyoX0t7tvtK+eDwMT2fl!i%{X7zsV^L~4N
zM-UPxjpxgnsD=AJcee_cM?Pjl$qA*hwWY-3cW<+M)G31Mr!}iOR={^bV;fDnSnl?2
zi)a-rVEOVLhwOPuZK{I3)xNN_2LTiBi>wG~J_LHj=O9Z`3FiV!Vn#Z<8n+`dVi-O_
zPUr}J1F=A58jz|6SkmP3x76^*dQh&>jfh<Y4O!{vJ}!<~zbzFCf@$fGpkKif)e`bV
zi_Q6ziL%OVAOpTA37yE>3v@n#>$p(1Nwz-fb9*VU7;3+$(;LVP>}%4nqiL+C-EuL5
zz`_G*8rD~v8`yg%Zy0Cj`1aH9@myj)XcMu?{Jp~X*V2Go1S>{lLFwlKmx4*FAd3^}
zeG3;ka=eQ>EWo@VkJ?yl=()e(RjJ-$)A1$lkUz&P@7TMzCdi~}Zb({}w)Ukx65X5X
z9sO9={S%u_Kr+HNl+)XU4P$&34F=CvyCS6W_dLy38_3e}CI-GSx-5BWy0jR;qq<bN
zqbr3v{LwA*!(frpaQ^!lp~0bQCDKcsDyPST&y`i7G4UvG1A%V^FA$@|f?!T-U^M@i
z_X7!!z!*1E`iR0wld5pBY06zxB2F_b{NH<-IThU!?`ECcFKe6^8_ah)e`B&AxT)^g
z#fTkTm6*Kc`N*K*vzty#-cL^(PzCtxR78rxH+#$uG^fZc?xViBTj{<D%sKm#5v(X4
zC#sL`6CUZ(6JTx?L5-uov_of{GmavVGw?bj9V==oz-(J#pFFE{ykEThw%R5qy1(M^
z+x~FMH6Vy0HJnM;MUyJ03hOfNa}1M@maZlNU2%%-Asr5rFKX}r^r9=Dl`n6VamYG!
zzIY6G6SFNS{rZsQzAK7?&$1*f3+d}AC)w<n;Ipc=Gn%MTnG`%1#cW^UQaZ<lm4TsC
zQEZe`-wKtQ1vtIZ713-g8M?=h_uToNM-gaQ$h0i)_nvXH@Sc-q<nm8Q=Uwueamp@k
zm%)r*dnuz(aN*n~3RC>6veM?S{7`~jM|IABQ*F_WElb95eDBt^e|@U;>VqzqI|BZc
zDmhcm3VRU#YWJhanT7VnSmLpVAf^0ZH*DWJ3i-nxXCm%G`}ix)3%&P3uc>XLFWQUe
zmc78?`8G>^-?k<bz~lZ~bcSif2KK@j@fLK;>s;s~$fX{rA|vUEW1$7*qBQEpX48U$
z5S|<%@coPD<2*b<=T+&bwD;$~{`IloF|<<h@C8k9V`4Lz&&sh;AJ2U0X~jDhj_X28
z(G9-2^G!qr1TqSH7Wghi2Td($8m0B{k-LsSwW0+$u^x?Hn%*<iVs!P6zt|agUkvn2
zl}k0{96Mi6)VJ@Uu(YN^`=j?{*SKJADhzPD2V*%uoWB>BaF}AM_F{k-jIXaIL%x%-
zTV>0)Zuu28KL<@XoY1_ETOB(p$&HZu%36>s-v)UfBr-k5O<X+qCc}G`dt9=VrsvBI
zonT=V(@{O5WlhuVAjo+Jx~0vNq*X7|UH6L;W4MlL+LOBd0ONhiSgDbBb4kCuF91Cx
zx}#N0iVxkZJ$s|wPqe-VM@S$lD@&X%99ahz3av?fIPYP!%0kB_Vd@JjXXXY;@3k8~
zRiEH5*i-vaqV~)OyOcG#cglh^XFk4mmk}&NZKDupgZBF5<pT~gZ|<u#uFu)B0K@t$
z3`nfpNl2i`Kr^Dv-aUDUx&b`du3)XQ!HA;uJTHP)6(0YZ!Y2++Js<X}EtOKaVt$kr
zq<}4xDW`-8iB$ALkt)Q4n0h@oN+;ftOUwq8Ren(qE#@GVHMoP8*`i{g_r~<=Lh*Ei
zz59O)qZNbV;&wsX*`={b*(9!uDMj>2=C1t#ghb;AmxOjlYOX+hx%J8v^AKjxWROGM
zPe-<fv@oC;H|2JbbF@sTPtm%ib`^2rE;8?x^;?2%YI0o=QD6&Njhf-6E-Km;nF+R6
z#{m@hG*iBG!M5`tQ>5TlAu5|%Pp<W51tjeW&3uG(GUdU+a+|tf6%7|1Em~ZnGgFyq
zi26oP65)ER9)&=hAC0>XMEgh1x`odR0InGkuY!zdP*_%6WSxE%Y9pc9BZhDN#D6Es
zcr=#~xt97(_7$ZQSFV-pSD{7iCd8t}$Tv(9zW_q4%&F`A_<h(t#k^rhv7Gz<sc&B-
zA5NKZB&t6*Q<k+}cN?j`_pxTdKudrRWxdqvk+<8PePkcr=-}uL<CJE=u?^IB6ZERT
z3HWgArr-s84t^cl^hc40bWo(h!~o+^$;XXQPB#l}<Eu&{96R<Yeu)ip*#C{C$a5}T
zf`7_K4BUzXSa1r!9o+kc-~Q&l%LIfxHYW&tw1d|#Szy$wR*fXaxIOi00>+1Xg@;?k
zH7<XZLLGi<-uSV>^<>okkLe$ae)j3@MF;`MKFQHwKR9JeIIv3>nXRRKE!uyN0ca@4
z70MTXqQbw@GeQqm>QKER^z&G7b!ruE=bet!1GCk?#OegU7@mvtM9gsbzuN0Z@^2V{
z1QAN{>Klvvx4lQ3IvvX12O;?QVUx^Ve>hG*(K<&g?gm%p+2B1ik?%18b%Z@QK2Ty6
z=CBQ(I?dJK{Qr4siX)dx_TRLqaqIPKpYdy<5wFp1$@sQG+lCF&3giAoO!*aR^W|aD
zHlO^`ae2Ez!ww<u#}zcs=XOFq7jQzq|7Yv$zpBxO1}AV1y13B2jr@$h+<`~kWO2cz
z&&NQi^j)H#;tx{<`3C52V5NXUX+!9;OeHi>OLJ*A&;*uUXu~!r3C||{j`hE|E~OZN
z^PT|}#b$4beG44riKmWjGy==);?taSz)t&(T*5aJdmVws0F3&kuNT@`49R3<b~-W-
z3-US>!olNB|Fz8jdK5g6Ya8)S6U=<i0rfanqs7){V}$Euc!WdRo^zR9a{43!o&|37
z2mZIIA<Hn_hXz_xHVl@6m$dL*iQ?TU9}M<-qPKS6zWo2dG-?$gLDQclqXO<jMuS7z
zV`G9ejgvlmXmAoU<_O2mE&sm4PBbUCVH932`2ql=p3uM?c^DGqZc-+a2K86lM8%8$
zn0P<<J@UqmP}llSb>9cCdikIN=~fjIjEAtUa5%p>_L%Fz{sSus@W_o!h0iws!08rQ
zWS}6x5G*J$-mpz2R*rVO#vdt4ev{~e*^`4l{gYPp=l}7}7#FuhIR{(m9bDR}%&UdL
zte^%Hc<X1)2WBUj>cyr*vzNT`$9Ld5wZE^%FP)wWIji+8Xn%cPkS1xi5i3F>18@nH
zu4M?FAb!xG>AK;P44}BQSuPnFpa_X|3MNq}B$Z#Ac3*9{<UjWrg_i}P3xMZ)2#KvX
z^%TtjiU!Hhwp-@6FSEg|5}~@s4fWh#pcHc}JyL=~+$%>a(YrPSE+oQF3??Ov*6*ek
z0E#0CXDwv@=X&C0fwTq@=s=N~wjv91uUP2SZM%R1D7AF_z&}?|5B~@=UMBmf)xJI#
zA~=u~zXLJdzS#76Knh%P{%p?Ed;jMFp=9Xz?}CE~xRasd?`2V#NJZOP&ms2h)*`t(
zc3%2_T0;~hP$wYJI`loBl6OkZ@}n|%>Ynu7+qp-7WcaT6(|`6G0h0wcYb5{D3LN@T
zu2)w$G?>q7E<Uo3(G(iFR5!#w6P~cm<WQUmc>u_4Y69mS{FEf1Di-~56}4Cfp|gPK
z>|j;FQd+IrL`{RlXw$ylZw8wCM>`6;CMh~y#|7Pv3;UXv|GRsMH?X;SL@_iCHc3T-
zik3|U5Oq-3WjAXU%>*E<#5bW<5B}#NbiX#d!B$BNc3Zp|7+k`pI(iL#X!bM`f6j;Z
zYyER?ai|vsH?n-%5xFQ^n<zdFc18~Ig{ZJ0h0D$#J;VQkM4~8q$vRy$=;I+NE>fVB
zhMgs2!k4|nEdQ<+9O)e<o>z4V>WqrR4;<8!OpwN@xE~Otlp(_GJ?Gj_3c$hJ=ihJT
zN<lMnCexIHx=>Iwidb>-#=A|1Re!_2GS=o}=hx?P%{0B~dgK@k6p{D2cOGYxwjYei
z-g{q@%L)w?*<H)9?@u4wDV>IJfIBC+Lu`iUhN^+lfk<4I_?j2pjW@UDUW~Bu3WEZ0
zCyYFTaCmw9f%t-k_MgX$o*-T~MBJO@A}!ZLATA-Y$Y$j-4pzcQD4;=NsIDR!=2k`2
zmo&e$V^<u6WJFvRp;}%5myK%|YGI{9^(}Ji^leeJgRnPrJvIQK!b~H>_U^w>{``ZI
zw6@z3nB#zTYy!9eU3-^{pVf=*4+CoqRLL;lm%krXxKcM>E&KQ+1;b&?YeBLd6l?l}
zv=)4C2kd<rR6p)7v(qEzU^*WVH<;H{17~l;J->sE70_yg55<BDaJ6m+1{o#@s-rhE
zN$19dP;FTI(3{12_|r6g^KbYxf&nq-!2~UL;>Av%avPU0bsQm+WZeL_c)Klo$q@51
z`k6WceFpfmook=#<ry(-!+1us`xXT?b@%02XASHSt|$kW>BLqk8*d=FbxR|5^uRgu
znH1lFzNt%A8GU5ZzRkjQj{7FEChPV(K8|KbgjfK!zAm%Aee@C;(x*zu?Lh;vBj)q8
z4--owKZ#mw_aPS4CNnaaJd2x{-?sQxw%?iAJArtK(WRqssx;8pDj6F|Fk(r(qjS>a
z-Y0B>s!aJOqj~9a$Ni#0iL;nu>#|j-Lf|6v;DWS3i~NI%tYt+rViR*s1`gGEXiNm3
zOinIVSM);GeycjFq2Bn4;!W6r85KGQS|SJlkXI=TS02#~Dn3xwCV~f%bl-tjRk*}5
zZK)J>i%F1#I6#9EQG5G;v`_HcGmIlOvtp}cu&TyE7yGXxpLivWa8>;*5V}K4gl`@_
zdxE9vhO<R9o5U^WCyZV31INT-)-UmEbD(B@po_T4LjK@=O($$+<7BFZP+~C#$B=GX
zjP9vJVko`@1F7{s<D)Nf?1GQ6ALT9I1SI<AXP{!}hSNvselC{J0REccw)3}lpi#oa
zsU{GuG0JI^TM9-a3E|i*(9{V|Jo43{o&H6)2t}GkpeHz_RDv`ie2gL-@Rg8F3E^S%
zVoHX5_-Tjf%c~=_$jW(GWb`Wpj=aUdx&CDD4uSZ6!Mwdc5!T3TGN7M8<i88exU1lN
zxb2qD+(5C56BaLA2lFiuUjkr(-S!$r2&tyV_l}FAkT?5(!wHU+1R%p#5VxIt&2xID
zZvp~i?2J6Skrc`W1Cc`QU919Z5C@=_ff7Zn>O;wG6{Hay?`?%_fxo^8bC^__!*2ks
zg_q0`orWeVxCTpbY0rIPvF#h=-Vdl6aK_>J*MQj#jD#z#j74QDz(&Ad@U*J_>zUwL
z38fe%uh)Zcq60MhyvRHxy&XF7YTyV_6`=g$VAYH3J~V=kE|{(>I}ZZ$0X*@HBHN+a
zIuEmx1mj9Bt+1HQJz80^`d}+YboSQZycI8ZWT#srOSm1w*&N~qUxSvx>@|2vf9!}D
zRI^Z7)PRC)nsFRxst}$Sc$D;vD=%(dXuSW->T_t?GQ^}!plx=tnTUR!8?wwLkf7%(
z1uznvDC!%btpk{RgOT?$mQ(*6zj!PL2!FN5pG#se^x@v;TCq4>qUKC~vUlv(Q06Ue
z^lE43T646toY3g6<6W8l;2TR1l=lm8$~!>Ul(^-}b=|g0*WbOU{O4&27<JuTx8Ccu
zGC%|2SMp3(MTUM)B$1OA4V}Pz0oWtuGscicAwI?2pGmO10`enf^?}J9qXtaw6TrT<
zRc1~Z_Rw9P;NSavJP-}2^*Y;e3ZX=AG_)6zqmDM+bZykWFlBzRb?x(!1G8JOx??l+
z11BjGVy(|%_%pDl|K_XXpVQ#+&;v41M(x$}4=mw<ZqQ<R*JmF58@~9e3*UqznNy;F
zQNRCLGGqkUTA*PhFNo}8B8wcy{&&v($Zs3*PzlHs?#X8RpW_Yw39L$3Iv*pS|NV@=
zzh4Q4jHNMs+i8n`GnOAwb`0rC)CpVvhJgQ0rAO$HOxfv5|DOLjp^<+^s=?kIi??C;
zXW_#lf*Ti7_i;1_Xd?-na=~k$|2CNa06<LF1Trfopba2qUnQW3PxcSF=A4*C*pHs0
zKx|61y<x(a$r^Ya(0_YmaND4ad7Clo3P=dAeOQCl_#W6%JSyIfvAs{A)9nwCM2=6p
z#D`aRZt%S3XW&*;fz`wh1%mNcqpdBM;X|K#T+;r}Y9T9QY+9@Gfv=MIUSk6if}C@+
z5=favBb*3S69llGhyNnE`Lzgs?p>gyE(6^3E!d(X!ZQdk;dAbg;PQ3@ZVRP3hzh*x
z(@F>y*l|OnlH>dTy}<Y=42~LM#1Em)Y(SXvhlE2-;gB&Cw_*|Ouu}FzrIuErvB*GP
zND&Nup-Rz4jRQ;CKzjrdjJP*Q6AanKP;(Y#d3PXf8K_ZqI5ez4_mFE{Cwrd|&`%Kw
zgh?_P9+#m1402Fzho)(@^MG<@>2zzjv?eadk9S_x$mGUZ>8qFgg{vB#LN`cce}5o=
z!9dIA9!RT!o_u0t4N?Z6rau5LXN2JaXz?oeGsI#UvVA|bY~Nqx4OG+NbMMO_zh{)u
z_eGlD6%O~COfUJ)4azAMKuwwiEH6M&fi5r)5w1oQF!FtA(2llg3_4pX=-If(^KD~x
zHIfESL=oRXmP0R*tLsmb%V<_^uR5zZDMB%Ib=0|yiBp!StEcl?9pNO2-qkI?JIO6y
z<vO|mA6`(ychb1eeb^Zkt@)iW+$u6<6b5(|1J@$#^B#KSs)SeDor<H-(&xM;817f5
zZzb|y)giHG1R%@mcgJpg1sCsQ9sa=d2QI2m&S}2yEs%v9JX!++By{9HBHVE~)ss$N
zB$MkPlt7uIzY0iS7T9?}2DsINsB1kBg60-rp%*QH`kG#%r}o6b<J(x*m*GyV!o{--
zSPOAMb$tPpKM3Mc{)~th7H2cq0Rs~EL&60|zTgO9%fqH5(Fxco1y8(~?_PF{i9b79
zz5)vjk{j|Nk~Q|Sg7#~LZAV_dtUN-g8p#siY`{y{cHo*nVGPdTAmObeiQl4q62Kfj
z#3)f2A&@~4tCMw9a=;&raAZ)4&qFgY-j|RZSh(O!wZ-$lbQ}`;#M4{1U;OS4ZF4LU
z#za?Nb$!192Hu0P2a!f&Y5<sA3@-bpx_n(h03+%TCQ56U<n`=x@?yP#<z!h3z0>}8
zr6g+{4k0lD1Lc|$$MqZ8XSm<%?jlqHp@|QLTBWkl_vTpt5<<=NvV++akFNt!hHZ=w
zlC?sni0D5(QKGMfi24+g{E!m^prD`-Svgr;QXcK_4N^7YDb&#gvq)W9O{C`&@p^y$
zDVOw*8nsT3X%+SAsKK$|_VkIF(ik}0y_pM(+v!Pg><X<<eA^FI7_`i|-LIEXN%+8C
zhT`W?p)uF;+$dW@95ij0=qn!^I{`{IU3<;BMVIQv&AL}RzpaIuUwylkyVyLhrT?a7
z+eMHuy0Hl=I#`vx;7t(C?^mZnI<E@dDvyTT3=aZr`FE5gky*)KlI50GLq%1$`@T5s
z`OZ0}+vBqDoYZ6$og>c!D3QnWmfE6+ZOMh^+A>figFeZ2*Gpp-$SlS(sa&C!HM9rf
z0A#CqT{qnO;NLp!1T-JUGFI`!6%RQJGQ%6_IG0$J=x;z;u=3)`A#_9PTfa5R6UX^A
zgH#?$+#1xY{|Z>)5R}%v0Jn68kz1X<T72<o!&;n~)sXXk{R$x*q;uW}ZuQ!jaK;5F
z7CfKA3h)J^0ipAHR=|n!+E&LXfAXqv_n>Nu0gm{P<pJKX@Di^uxe>Us#u}0J5(%>?
zr(;>4QpL^D3QB7?VYp+0es845n+DZc3pRah+$@`pA3terR+VR`)y!X%KeR59kK3VS
zjyl0lP**oDhqu<dN@e6xxs2v)+vx5NTb68ZMx?){{w6cX;Z7EnC%0-ajsI8B(dVW+
zE}OIByGqi!<lK`_&iY^7VS6iIqRpYkd`}&45{<=j&Q5ZffVGNgtt7IVXenL@1qIx0
zIf8k9i}J~L-+=XQjRy<W-C2I~avw#Vzl}8*A;qg!Z~&|Ix;qQOls<}LuRp6Y)%Kk}
zHg`Z<GhqKHIHx*)Xz)oLg@&SVEqw*^x?&0=R1wEQpF~pNxJHJ1QuAMe(-!@KT`St9
zAk(9z(~l%Z@LWA(cstHB=?VRCWR>~ByT9c_aKwUMtCvr`>T*-=py~!N$tQUE%-?54
zw4gr$xK*3s=Ks>qFt|(bjE8l(eUz_WV0-^a15ltEXY`v_yCh~u_sKLnWFM8>pz~xk
zs@qFJojQ26l0D+%mijw|KJPq;HK(Nc_r{oitb!118S?C_o)mj=>~>^dP~}Os(W5R{
z#*&El1Ay_VB2yJHw~(pG;kLG_lEY9NyC)Y*UVV$jwrVrSM`T#)6Z?tiETL}N$>aUU
zYFJveKO!pNp(hqnakH14D-G7ZV0jgvIi(#j;Ej~GOr-EX8F{HJ0Y829Yp=`E`jW=R
z-3l7V52fp&!RSU#O%bF83@_9CDB~!h#W2xqL7sb)*?0|Sgx=~GpjdkFj`R#zPdv$@
z-JE<v3qNA;)-2q|fGKdgCl1Js4UUuDFWFSNhaEZGY*U2#$yZatjk^YS<~|@&qUHCA
zom~$E;$<(*&WKSyGm{rI3oq57@ny-BxmFaBu6T05n2X1!mFVOt+OEOj%pl3Z1o=&K
zNbziJ6ouAw%Dc4T)RqGbZ!mL$93D!18YGr0)%>&(;&+b>9y?BsjK@4Ypd(ph=Ub&X
z8&n+DpQoFEbBtHq$DctbaQcxjOL-^rGmRWouZ{ZlfjdPH%u4Qr6XxICKgIUsTtcO<
z)P~&3lf&z$2M#@vf3xetbrDzdzBMpkg|t~aAQ+~6+{#aA!hJPm6Q&^T2q~bjD=5l#
zc8PAIU}OY3z?1D6<{z4$3BI`5pzM<xz94e+YspR2JF41XM6m;uDqd!Dv}<p~oy*%t
zIY=#WJeh|Doqcp}L!(Y^md7S|AwJGhw_i6Mdk{ZdnD^Rw#-RkgD2%^|S#&PB@=VaN
z8P&>tLbXpl&}q}QYU}lA4kre7A2+Hjb@jSv>&jr?3#F+kWtM-p$AEHwnE4{^E$FyP
zk1XfxyRfF40F$(`+F-s+lz<y;!^&6;X9>>@iqXfOjLf)&a7U)7>k*OH07;mZ^4_s?
z<Z@zR*LNOs7VN#q!^i3Of{mMP!)sy7$^j?QXe)K@84y$PDjcJ~r{~B19ea>Ka|fGy
z{K>&x0>Nx4&~`d_h_yUvwRf023MS()e1dXZWraDWs&{a{igWvZIe%<=Hud=!!Wa$b
zcHucX&u4473ws|jB}QqIlUuIp=*8`(hyYpg=!+-28D@Zxj@avoBx>Y^5Dd`=->9o&
z$<7TtOVjc<dQ`|PD6Shj+9aBv_VU~gySzY$@?v@5XSM!`5jAj<XJC75ljNlnyJx~p
z9$zLNQ!NkBu(+%2$~Ekx^zGU1<}I3!18x}Iaj~PSR27{?{r@O?3#h8vu3ebD*}$eZ
z9n#&>-7V6nfPr)<NQ$7u29Z!CB&3y4L{TXb6r@81B&3w?Qb3eYgfo{u@B6*q_n-5h
zamILt!+N%BueI*`p7WYlWb4zA-#2P}a<0|N83{9aH(K@Vo#UI;t@*1rY}xGB?mC4v
zuQ~M}`NiLJM{2^U=XhPx_Iik3v7j2-_0+vNoF+kB53PuPR24D{xARXvsHpxBCE)^g
z5*xPo*XJ-@HR4<%P0$VVB#@2DOw;1@xsnz-V%MSRx&JCXPM>$^zs=Q|Q2`n{s%U*S
zE_p2d`*$iQfnMdqHGe|NM?Oj<;<DvZm%_4R+f6;CBX)L5Uuo^>5=ITOx%>y)gA!ey
z`JGF%faT+yn(^-SF@no-XtF7Gf*+{;v5Kn(Ucz*g8^%AOwgTMEvc!^~gL<hp(}tAh
ziS`k)^QF+J>O@cuM}xMsXgQYuAN9h2bRNQbQ<#qwmKPdXOG!wl>KV`1KytABkcsUY
z$79sT=vMCWtCA0Y0t7*$RD8ygXx|WTLh-w1&v)j|XSaVIis1izD6XTtbTm#1XYU<_
zXXeW#lCFf-$~{`I@AfsU_awAhc>=hP_dLyXcET6WHh7(VE=e}4Tg~v`h%$pUI%~dW
z=7-^YmP?|sZv41U=o|Rs_J~I|3s}tMHvmJ`{?fAk-?enT2(z_{WiSZAF9^(Xu+_eE
zEaW$BD_!fM_Gu-Hdr<Z+Ruhf2zuHb2-;l9U4P29s5=5@MpDGih=ruODTjJ{~^6yCz
zjvRU}$NVB`q&39ZAof3kQd+Jb^4LdcF4&`Z@YbjKur&<{>JLGh?PrbH-1gmCF2(Lf
z671}Q``1k0NuEKP?unuJGmk*^`niRft~_9UW4a~wOROq`Q?AH@92(St!rw*#zmd4w
zq5bvI(v8bQ@8RUw__p@Q``;buA-^Pe2#1WRgLLw++tLUiqDss|vwIvC4uU#AXgXd2
zd#CTHmNI?>#?P0LYj7i-`*81`raeHrM&CJJ`5EYWYrE0uK=aw^b4nEX;3PJ5>v|2S
ze4!WwmW(nS;lRf_grd4P8+wqNG7w@&xQr+mze8)8GI0Eb2)dkRX|z&MVFh%t#+y*)
z*n!bYUA3PRlBeBXb00Vmpi{fw2o1wA#EwU!`qfF~T4gYZOB56_5;E2+tcN&+CNTlC
zbsKcdo}2NChb+9<x`l=Q$Yo7kK^r8n*#_*lkX-<P1L!=+XcsYjgcH9jiXAAALQZ2Z
zy*WQuf=%Prub}#&@KhPPt<6u3vmZGzXU2N;C)3)(;m<&vd#zh;c;f(ha}_*Cd|IL@
zJKMm(#9Da+`V1fIL#WPuSK=Qt0F*ZZ>CR9Mf;1ADhjegU>(bw=2Q&+YSg}tMEFTz^
zC(WLRn63<tH;~sQcj05_-J_h>4q(y(4JTM8oEc>VG-L$YZlw@V!kZjBI1fHQS0Vc#
z>V$ST{b%s;-c3qYhdn!CA4>h2@WTUW0t=r49M%5-VBBQ0RfOe%h+LI%cGvw1z@@n1
z0L+{EfM<D!|2WY89>HJJukn7?tXc%S1YD(Kn8gqh>f`A}vpAj;3ia}1;A9580Q3!%
zy7_PC0v*#;F0VLWEF%rOrquRaRLNu9lV)8ATyGA*pdzAS;dnhV>4`9=wWn!2J`J_d
zr2wLGt$GOU@;Mm<zCz}_>sdv?=Q^Fgyb_~C*ZJ`;r0D<B>XBc6fyIuDpN84^dUt68
zE-YD}tiAJS{ddc+D=>}0WJb298)1?XHeYwgLfhJN0VhaQdV~@WI4s)xMbIi6c=7<{
zT@b#p&y=SB1t6X7=1xye!wjOekc1~1MmSvR&!8Z_LB_D5)eBHL?ZXXlPD(1Z2sh;K
z6sTE&W>2~;XA<ts0if!g)018RfkTyY9sp_gT;l^#8nwFs^PiL;vN@9#wdMC<fFExY
z6Q^&s(lpB{fEMloZ;;xg^BSdkHeZlndE?)JvO!nnC1QMWLg|(Wqkj~+O)6O}e{cQi
zGW&?yoA1D@%e%j49dHC+vL?~YzjxA&-{@DIrsKr9k2etvk?#&cLvx&G&qc3sD4B4I
zU}vyA0)=ar45`kgfe%@77w`B(dO>Cjbe{g5m#z=%C+{9KHjV3_doF3TLpypZ^-_)B
zW5Z+~ibe2UqI^zync4Km@<5DLop_4ANg;Ib0G2pIv_^I#NnP2e9DBY<+4oHu$wBcv
zA?bUGK@fHB@dSuepa-tF4Q@@Z;oNc)Htm6=oLKtCJncPB4|I$>UD<<4P+niVd0E9|
ziWQ#&#y$l{?adyXBqj0@Dg)Bh_a0A&Y@fiRZaq1w0bRmpWBQZ!I_6IIyDN`8L8oBv
zJr|=Auhagh4Dd|*iL9)0Hr<LX0Db*C#T^7b<sgdzntMZo%q#I^z7lxm&a=%|z-i~p
zbAB0Ua3!w26G#`c)$YI%CezKUlhzil&7(}CK3Gc<iz3JFpXoSO9W`;Jiz|4Li^f+)
zXqR5LnSe$UhY!G`$}P2(7{$+4w*|sr<39HVA)XBlOc@i5xmaJ<Ea+%Pz)m>L-$U=G
zOw?!=)2DPEgmo9SQ_-G!3X>s~CNY9Mr?jO`rKzgmjzyiH>6+HMuAKN5wCZG}#VfBX
zi~3o3d<IMuxOWup(jT>B`$ct$m};}WB4p2xmYFszVE;M;cZ|gXyMef|QKq!Zfj?(x
zlX1R*Jw08<yiE>O(dBy?94jgW7(a|X1#9FVSXuKG2DhIrz4rRy85<(q#$On)1=Eq0
zi$XKAml^Y@g&%}ybHsz-;#aOUwH{X?#(cY-o#0nhNX{@Tov)vdrTx}kuA5AXj#JI0
z0RJvqicenU8@jDV&9@$;&DM>jN{t&nsr%-{XJ3l;nucMNJ-(6qNvJW=CMv=YbvgpZ
zS@P16LR*)r$jkdHUSzfcPVMwkA;7e%epFlJWBzjhLH~^%H?pH$37c|Xow$Xq(ZXM&
zJ<RIK>ZSML2n1*zrm<harXgmQGti*Vt3CBodfz!O3n3Z;bj_XVEoc-K)6mRJ@0*1R
zoeu9+mhyQ?rAJ!4TTN9Uow6Pom7>}-H|zm^aBKj9lW2%qPRO^n`y6k;>~_u|a|dLs
zmIOP`xM@*htT`q^xiiS9X+{Abjo9a`U^c8V==~E8%fK@#2~!C2*tfyqiU;;cX;?NM
zLaMaonzscTGnB5IR7>hjRD1GQ<kSbtdtGC?kGEK!*X2sVj6AIe>XpyYM3I_3ggD<x
zo+I(K<%#|t=d~yn8P^f3eYepn+AwZ>w!`O>3+$6oU1AA>af^{8JRH!OfekWFFY}72
zgG^-nl*Ue<Ei>yFU{TAgcSfj$lF5+K#tqvc=09^A6EySXtpa>Fubtdn5toUv^0XQn
zbNV%;#`V5wXK8|%@W?(?7Ez5%QSr!E+K1%)Zp~~qCZ6lJHPvV%;%TO~1&BGSiG$L3
zqfnbF=3;`bCaI@;-_!yfp$7Nzg}@5x{qZv#^B^6xg$xg$HCs2d_nz*Vc;;Wdwqj3Z
zS#dedG8)neRquo}$WV^7<B%5wC_k4vO+(Or>5sKhx(_$Hrr^ZE+Q+RLU7~1a<60HO
zyQBugkW8r0F6o({;C;_F1sW+_F>4CSHM23(;wtU$6Khvae7_nXzlVvOzG_kUhZi7-
z3Okb<V`Lzx=_z7IU;1Hg=>}oQ(2vjUKgbFj&799zyh`4+0;E#qCy}B@$7z+K%ch$v
zU3q5=--J_J1*ZR2amic?-WB`Yi2A4>t5WnT)nV}Vq+<H-hoq`K_syNMTlCpeF8c@V
zZa5T?GAAu|CeOkEMB&X>;~Lc1$Jy6ZdW5Iq=o2iD0Z+h#K9}fol*!1~X{QE=(U6}M
zI+(KfFo!dXkP{DM^op?OYhDz+HB0s!CraQfYeVOAlloT3M>1p4LW9J18S+eb)Km+D
z4q^JwR_Qs<)ZoS8JF#*4KY%(N)$SaFKOV$#Uxby6Q`_+D^{PwNNh%_Bc~kS6VnNp(
zbq}}Qn>W5AL$L^8Dw5<j8oTh2%>4A4y?VpKov-4l(0FeNu<pe=7@ix5B)X)J>bs7+
z4+ZtT?tYwDcEcvG@<6ag?MXm&R@~*7EM=bSOT#OkVi?EF=Qb<lF3qB?wVHsC!Ib6F
z%vq>HMi>Q}PVl_e0*Mug>|?~^BGUY!%6g|+ZL;wrUbh=YZn2uXAudaP9~jf0Mz|I0
zv;Q{+Q-nF%bG(CxOY-i?Qk&#+?()EuDZ2#xcY9zi^KoHaR$`nSEu*m5CAlkT8p^fb
z_$*R7ldxA?2`dp8OjySD+;<D<$T=VDC;u)=k)PnK(IYhX8P$QK4UA9gC#P=XuC7T^
zACVV(cKh+SBD&r>!BDdO3DhX-ZYc3!INT~MCi3x528T!Q{1Wl2FK@3h=(Akp`FyjV
zGWVt^s@w1E=GCHaL0!@-OS|i>%f_%X$1*en>GnFMCiD`|H^_it6R#4NWPYRk{y<%a
zw(u<%<oPmOodc*%Rhni6arMc@ME*?qBEUPyppXCT3I?(AM+VPk2MRcT2>;$^Fd(v4
z+M8DOHP~A6Y8`ENMTcN#+E<VCb4?j#Mt|BBW9FM5J%t`^@q6=^1Z!OUh<wqF@Pq^a
zhmN)KQdcx~NayGIH04gcp)u^L?z}$vFgGrKLFM`Fi^@+E+Fn?kl=uUEC!Iw{Nj9RU
zVL*7{Rm3pq7~4u*(%7-h&{a?0C0^vOpm-b2{5xZPP`|;C_l9#;3ye8m1ClRL50gho
zKgpJLc+jJ^b9RwmSMZ^0v(;zg$R`e;_mJJT1r5Y#t)zS~Tg_+zCQGHX!ZO(NS#tNq
zd%d-x?fYCIr`X{rlN-ma6FmmZiypSplap6h#oZEadGN^7^}YcVxau2pjI4)U+&|1R
zrHO7E*<I;GrVs!)jVGaf)hHUgDn@eP0WF*Tj|b#u?K--b;iEo#M_r#I3v+4&s@qp;
zapj+T3@*I7++K(m-I`ysi5B6zMC2gsYh&{|&`Ts!%87Vg{9~DY6WT3T>fu1Bm*fw{
z^=Hf%seH;Qy6+|`c5=jJ4gzC>>j5c@Yil7Rh8TrL4{ZsAwlP@SV`5b&`uCv9*d*+F
z+0Fe;MrFd<E|_2XTHXMrw7R(8NEKQmsE&+RJ;nB<s^{5`2fe@w*gQH?LgE$|*M`n}
zdX0AaMrB^hfogs>6rNFYH_FZ+J{Z$AI*;d{Zh~nifg-hq>1L$i4k8${G>-ZEqO0>a
z9z{kp4bWKy{8|r@H4x7EVQ?Y!U48fkkV8Nx(*yN<Y*g;VHygp6CYZS<wYe2+^_@Q=
zZILgrc@%l|uYCRJ4`B5bYygaG9Q6$6-<}7_0o4D;*1*#oXyGtZGf?$@&uj7lfOz;d
zsap3$Y<J?t8Tj<~SI;2w&m%t&SCMmitj)bitgX*K*S!h(xm*l&PqP!_Q-El|$jBA=
z&nIBC;#CMu;PqN|B>e^KBd&`yfk@2${a*(Fo+L6%|5e%I<`E}O-u`qr4GJE)OH7x6
z_O^%Q;@ZiQBCH~*VPfMk9U$RGaM5Y<KSvzBkEioB`@p{j$_f&h)2H5-?jI|}YU&C(
zqo6B+I9)f<laE^qnOGu(qpJ{WC%AAG7`Zq<*VD&$tD+;3j`B&^{-%Ld5Da+%C)EI(
zTccDVt=|H4;_B~8Z7ZXRWJo2X_CSm+cX1Y~@w>uS3o!kV&Rlz7^$=oY7JB!-c&<Wt
zWT6w%T3+-n;Muagv3P+jCcZ6L$-`XI@`+T_%s%uGY#L!9hUEd&ah2fA^)=v@1)iHO
z`7`)S)GcIPloUc}R9{}i5|8}6FDOqk{}xySt`>{ULl2{nf^9cd^K`=&m_v~!l2+|}
zslh!(Oa-?kw=vcBk9_kV_E-UX|Fo%vqM5|^hM-Hd3gzH&xvijKwCer<V!6Tr(ueNw
zzV^>Vd`8a(epXAf0LC8(XLu8QJ*A>x8XI$EP(8~dc2O*G94rtSE46;zH(pI<wL3}R
z*V{~pmP}rQiDjqzX9w}v83KP2ovqP^aE&p;S??uite1Q93D?El747`q@-Ja+sFn<Q
z9(GhvyWB>4){l$_pBkSKoy?zcI~t!6>ldS&qi98XA;YurK2uz|pa3moO5Ebuq1hA=
zwUUl>PPTpxFL|6mV_~ZGZIZ-rFg<E|O>SnTTn=48;f@Wk8^{<$asv8~slGI{KZW0{
zOQW%O#G9ZJ7ePjwxg*H&WbL*;LmHcMK&<lZ0(H`;pCeplRDi;&%n;9B!56E@`71>d
z($=jqW9kN{6*s^_OB^`8x8|S|Y@W0F01{`DPw-f8H4A;=z0%07afix|*XE^_qO(}f
z`(rY5U?-C*er9NHGHCX8-GvQO;q<T5&|U-=1Iw$JA))9p=_!gjfp5w|Fcssy&jO<u
z{-*}~c3RAB$c?2<CchmABVCo>*oP+@zWu4t9%1@_=1u~9kVu*{I2y4fwhKPdUmW5|
zi`tpTxZmoDbQ9h>@7h|Uf3qGGHmBnDKWYHK0Xj=g9*<K23w<mPet>Oa`X`}8;|K%V
z3Af$RVg}jHq`U!94ibR~^E__E8nEukcAv41s#cIY?Tci$m{9YFKMX0Kzgt&|&)UHd
z_grfRb<PcJ`x@tc!dMqA!0hl&>T5PRKsYN3j&K<4e*F%Zs(@ge0V0O6>c})f8`#Cf
zbmB3CcjDZ5s?8KB!o4(88`n0Z>4@yVtMJXVS{#w)y`<E7W(=4#0Y6|qYq+DwW26x9
zE)-oiyYjEY#@nSc)+UKb^i;fBU`m76XL)B7_}3`Ln06RKU+bbUVfVg;0i`OiIfqQc
z#2k;66*g0;l1{X{W2I0Y#pIl=TVj`zXHq`I6pJPLtXAS$Fc{QAYBGE8BPHF>_qzQ#
zMi!D3_$G3U|2$MCaC_Po3!NSHrqXG5e4ms7Va;mPJFJERpD;almLIaP%R)2MGIMXI
zv+CU}%`I%yzq|lOX;M&u6hSVnQYcMUc~5upt(zOYI@G?M<U*lS^-Vc54<dBzq&?r(
z2)Mr=|M_3t^U{g4r0(gnH-~xCVm|o6I-a)kww0=ocH`>lh}vRd@{MnACXc95x`oLU
z6lgwAmCRMRal`&o&P?B1oyS0aVbk|uXOIlrdg(dp_$*7&m*={p$2;y1a{c=sv#s_&
zr8seR3k-yOzGM(?uFP4|^PVo93c6u=c1!L7U)YT&VZ>Mc2_iR8N>q%IbMSOaTl2K#
z@F!m?In#D`H*yyQe$<%&leqpaDvxWV#GzRFu3ws+7YTcKGuXGtoZqMTOq^Su;HH`M
zskwPY<f_YKy@J+9CRO%sI&TbPy6ztR$#z`ufk9&z@<fwpF_&H~3mjp21m33r;AP3U
znSM`pLgN`ZKMT8CgTKO1jl6Dl;}XqAMv?Zmjkd_N*rmNBt&9-o!-eJ~$8E#{PN3W9
zh`7-zKTSR{n*5B|Q;Z+#+6wq)>(}>kJWNB7x1bNV9{8Oh9IRk+(D-O?Dy96LQ0nef
zl7ad+bP+L(>8He#Zeu4*&B>S*F0oOi3m9kMbI?Du$u_Q%IXGjut$kxJh)%*}pF-mS
z8#1spWJhQ2<|eZDz!pDRVDPf%Y;#sh8n?(2n;%)|<<*3{27>Kq=Z9yM+OH1PPz(~j
zGYFBN?u`-54V5)0k$+0!l6pMSmn=7gULhwssFOUYvhXfbD1K>=SId6-wk%4R;A*O;
zy$`9kdXTKHQd>w8D!*y5O1Er~XUHwEW@sTwu)ki1$8-$Tv%&m2NzNig6btFoX`+I0
zNF&D+A_w4ak~!M7$s=2LdeYN<BxeKTEwB2rm)?H#!~CxFeF|xGwSi=wfj%C7c(J7&
zfZC0EIWI<cM}LF!PD;Mc;{2L87l-MNA92sgQ-d1g-{>_B2;4}v>AnRv2T93-@lPP#
zry_3kCk|NtQZpr&Pma_mQX(EgdXwNEO<}a$=;0ag0p^PzBxWpQtGMO(tpAUGt4JWi
zc)zEEitGykM{LhGEYWUR{RgwEb1T_swr`3)j0qWmfg|*>^P}|TA}K6(4ynyr%57_s
zY+FAa+HcsA+L4-sr6%(vyS+xgK<7|xNP;WKwczG&L2WgO^`R%Ix=HjX@)|M+wi4&5
zNbf05B-fMy3{|}q%e|3pPo1`b1|Ymnl6>Vl(BpWp_5(|v(K=@d75$qBu(f@22808O
zyplwc+fJdkJ`%7*`07v9;m#3MFX?=e9?I@<d}~r^66J~+$}hh{=o@FHL^*kzd-&T>
z>=$C13`wy7^<O4Ze%f~QcxuKF_CC5+5}(=YDD-^OXR!OFZ9hq?V$JQfw}jI&c2rk)
zbbM`Of2(RLYEpaq@e*me>_gM)&VsuiZE3I^sMKm{DWS2|3QvE#&YLt!il2X}_LN@8
z7H)Jn{l2#A#aB=3Wgj%`;p*hvi61w}%gr_4Cw2&}i|n#%+TfQXk`FdM-{LRaaw6o8
z8K&bGEW8q1a`Oq!U-5_0Mu|hHgT9ApF)iZ2soQ_*3>=(q`RwCAn@|UF<#WYIJHKLY
z9wT6B+TYvC-A2a}Z&1a`Yt_W#e~?OFEB?fBW`_Iq*JXLub+o6jJxT2Ed{N5s9s>z{
zCgD%CqHXI%!si0qa<i^+Hq`PdFBv_p`Y!2BMWTy-GbfI@YdgjnIaHcyOI^!fr!{Rd
zIy>-&w~z`PY@L(w7JV;O8uyEeU%zB4*9(hDF5x~e<2HIypNaj7>Voe=NWeG6**=fR
zg;#@DTP1`CXs7gu&JZk$O?5|v2FZ|D+Zdv0RGW985We%2N>#UDY52Ytp`B}nl|<_s
zYTLTK4dtj7rz1~=i}IFY`qq=Gp1CO239vZnSVfN$73yUS@xL=ug;#_2!&z|<@9@PC
zJX=pntyE}EP*9Hue%>r-WoT_PUoe`GfWJ(6d(!@_Y$Zi|Ya^pTH}CEGyRp<Bx<5Wb
z-A>($xfJDVuy#UJ^Jb!mIwOR*mN7&N+L(RA>Alq0KM}7q?NrY6tzA>u-#ew6mNv+%
zwFq~e{3DmI4*q9>Oiq3!U-GjuT|C7RCPNZQpEioL9ohhrnz1m6x()4TJUN(4REG9+
z5wB3|Qaz7TsVGpT7xqY|o%U@dB@T2<sE(YEV0lQ)gZAaY@6*dj(ANY#o3`UfEZbjm
zD|ia~s3rBYwcYhuhmHpB8RD=9C1hq&%r~ZYI2t7P1LzK-7g@PC71L^mg3*5BehzdB
zs5nV8<=KytbZU!tm^n)_pNx@PJG>(abYz(tN!+39<xX83Z^0xQKZ!A130`nK-x$R+
z%AG)<p4N5bI1qn;F-@5_teOxUJL{dHM#v^^brMbpve2#si>2dF@`bJ#p1zx8#%cbc
zC@gH>H_MRIv#40ediewoqV6&LbOCe{T$z}l_YR^*+AcFzd9GCsdY>xlQRj9qCJ-T1
zRX0(8cr}QVkG`P%wnC<UD8Z(d?s@f~Nv$Tt=|U_c_|}ZyM7^hIUHfL0>s?|6ZFw2T
zNOWDfo+`s}*Ik>nlZs5bcU2ti_09;SV~$E}Xhk=#9{*0FIy?U$Di<eNaj-~~nw#Wv
z`FfImaN9HyKc7`h1a~8)=9=-A)b(b)bfs2O?BRKVH+PlVG}x7pG_jbul!ZVZ3veEo
zxJ08-zQ>YG(H1MBedf_C9FMGyfIN-piHF#rC<2Yy)}~;f^aXoPt1%@8*krgBc#lxs
zXUE@<z}M_6nB=MML7%p0kp0*uwm{Z^%4<3ff8#eaIPsUP7fvr8)8aZZ$67=AFs%A@
zaQ_XVem8e)xrU|8iKoS64ZzA#^%|9#x)Ps}toyui`RKLXK9#qhj?@_o5%Z2Qs~Xpy
zslhx?sVhiY@Qp7tT9IN6$sr5&(iCZ3EBY~T;dCOD+RGf%x3LA_<}F*rhjX?kY5Jn8
zQFyk%ic|oXa^(qT=U6u~v(#iH-RN@SRCnYrHC56por-lE@19E|zojMLHU4xc?bC+s
z?NrS8&C-w2368qL$5NHcl`z;9LW_49R_+|%W{zVp6Q}#d*Nh^BUYabdT&Zss8aMm{
zxYb%=$m9+MqAbH9Y@-CD<|gZFGkE4D+P#Z5lKjVsbVe{vi%Accs4BFbC_B>5FQz&H
zcz_>H*2mDs`)0nw#r=pa`L09tJ?vUdiQ(?%?5gcH??%ygrNYWC0m*B>_D>fy=eAK*
z7C}+=Y;9S`@Z|n4u63#_GqaAX+~PmyB@4IGzSi_V#_b8bx?K7|YQgL!?HK}b0$G;e
zMbhv!zt3D1z878chS(ji%^3IcpRit{Ri+xV4yw>}@Bt2Mt6?DHmY(v{y>EJpg#pq}
z8U!pJ4+$d`!v+^Boh*(9CmA?VRVU1K-}}*ae)dgTk5damU_AC|MvZIVgx`^K<86GJ
zXEO#v1Kre7x~cB?YZ**dl?w(m!|q2vtMevl3uoP+JJ9V-)h}&)EqA*6)K~lCBHZW$
z{>F1ACn5ugFzz3Ie@^~=cpw!`=(_7^#d~o|Lo|mtvBw96YIF2dtJ9u$P9?DF(seV`
z)TA@Cu%M3H9l3mERsAUx;F2)Tx<$)#2aUmae*O-6^kLFsJ-5~;shNi8L%IslWXdPv
zf9Prve6;q|jD)C2D#!hMt(S@hWsgz{aTcs+fk(zI0_ApQfkxCI0X?oPx2bv$hCnZz
z%dWIlGP#Vb<bap%UItVCdHjs52yt}=Ez-&NS}ZP@-AKL=KOu-`BK*iaYS45EJ#6Ss
zw^3uJV0nF~gQ1ASNFP@;`q2iNiE|NRV%dGbZNPOZ62sVb6ZO<@wpKEt`?0VFjn4$f
z$QTc=Ce5%CLC$60SOpV0tARZ2;>L&n7!(;wqE0X;*-WnyaQ5$}YPm@kJk*K2_Fy3g
zAv*P&pGL^Imazmwc-JFC*PR}E_b}Wix%Nl`cp80y)l#41WFloElNif$tCb!)gO8^d
z6)9aC`*1!;$|yfvK|~}7-~zreiehsqQ~HBv<1urIT>9oB0{<eCCUKz!jN4(JW;|xz
z9C||`e)tlvqUKS&B9>m$YtWYBZVVgkPWF@oMs!*y=P3j*=~B<nmudly@rNpCs0BEp
zVbihUq<=WdjRP2CtiqTjE-*D?LXlPT1)pvj!SHomO}@ZkU?9fD<DbjTcueCPsp6-+
zAPkQ`jKoFp21tEKH{L}*AwQ!}`p+lhB5?50SX6<|zkdUa!X$d3ei5U>p24qbHD8|{
z7b!AR{3!WGoOI^4j5fNCj#H|Lf}0uuyr`)uNcq!YnFf`KzA@Ja8}3v@9}G5|%*K=~
z(tI3h;4wWw!Wm+JQQ#c`3JdLO-0tP0?NpC!z<4<k(NPuunKdEXODP81+kHW!NOrvy
zSjBEzoFC|;emRzKTH?*m9KS2U->SgfWSCvaEUfsXSqIf00kL1ppl0ItWcF_4PW!7D
zu}mkpa0*GKLJq%lJaHZ)obmlUg(3g`dd{?p^Ql(vU<O`KvfGhhhp${>!tHX6JBq+F
z{QDmPAYV)Je{Vgqis?{jMlM$KQZbG0%%31hoFYa=)<RY##r2YRX`+a3=akn9z&p_N
zBB9CWAD<|x>`{IKuZQWP*plURIzJebBkF`&KNrwrkSdRWZ4Ds)U&deG)meQo4-Df~
z=)K8oxzo&L+WIf=8nuoB<$$V9Lba_LoQ*)U<l38Y^&CWa1SRp$NT&h76fI%P5s;=&
zS`}r&LU|hmRp9ZmT}pclat<Pk?5zv``6^uKOg>4(iOsI$J+h1%Ut%`uBQ>l;UQltW
zv;7k8!o==>1JFy7E2L(7sgH1&fRLEN2Bxr}1N)#I3ghjZnO~4+)EaRdu441>W^9Ce
z#cnXL&?IO<>arWyrc#<O?cU49(`^1{zzsmmCa`$*B3=Rrq1p!X(r+-sRSemc9Df3{
z%0A}}`r{f%_!(=nurZ)=8}tM4VZdgwnp$r-1R2Kx;E<3Xsm|BXNhM&K)5&ka+FzoJ
z(>y5pqY%c#6Ex(ZjwDB?5TKt;%0IJaCer?&7;krlfdEhYr5g3~(+majZCA4os$jGX
z*jKevYj`*0A|{-B3(hZy+m~5+W}<9_PW6)S`~E+ZWqxH7U>7F*=V&?XV9%|t1$VZc
zv6}RAJrEmcxZ7az+*JY}I(@NS*5o@2>jNeK6+k|6nwKeJ0A-Oc2I;MrDYuEHCcqI)
z89M9XF(2;k{9-#!?Q-#N6a4SD?axmp`0<PcM}9n_$e<?oQ>~u4()*pc7XL;06#)dl
zk;j0!ysOs4o6u|55c{6$Wn_*jlnxF`6ewgG?C6h`7yv89{mu-wnLc@Ny)-_9P+A0T
zzady4?qpX40jEG9UF$=sVjRvVtzI|InFjP>6`fDc0aCh4%XQ&&$Px1|JX2EWSBq-$
z+$#ylqO8%!{|+8Ql`(wiuoW&p*Of2xSTvDv`{vsYB7#ZN$!GFp@s+gM2{v(RNBAR}
z(XQk{IzH*sIPM$WzNaq9NQL+WChjk(RWXUsSR?F7hUX$o&_uJMwgQY#Z(SIP!R)_j
zA5vRy$(C?BxBW{ztGw?qV)%ER;MZ0NNAMg?u_eJG0FYmV7_yuX?}-JFd6pygc#M9t
zyTG|`3u$}=K&WDCTd7)n#{HPxQ$Be&t*CQh3M&EYx9FB35%R+S=YL!t7#xjVK%d_E
z&>svOm@X3U@J{mhhnlCyVa3vJV`W^s%7```?FS?TmN*0mIYTxgo^Aom)Yywx|C|#`
z+QlQ83rzRBO1COO=dds7c+5rM4v0tMik?>gw|@vZuUaqwU~jb_@wUI%c=IX{*<h9b
zLW0mBh)u5~R)|nx%!iQ4G*312c<j}w036=02|~s}`haf^Pc>2;gdJFypbyHe@4Y%U
zjNexV$@K`dC8u@?RIrXL`p@?uC!ysDPXu!<%(x%yiZxz#{C+$@{{Ns?gKa<%FVg@l
zcUmffTfae%T6Wniz9rzvHIT-T#`-{N3M3;8w*WE9?--0Qb}xa#GLHz{C7EVsx#~=Z
z_M<prIewMFa0iBN+RPE*C{>#K-w>5Fu*+kH7rK-H6jui#7WllKg%n<pf11E`!H_$0
zG_o?jKTdGNJBBgN+XiX2Vm0@%8&j!Xf%_Z8FI@l9@U+$ck2JjJHX_d92N0qY>%tuh
zV~89yP7L22oO)=cyv*lFSq*YM$P5yD@5JV$e~Ts|IT|jB>HQ66rl|y)mk@;&n~itD
zNYO8EgY?3&R(07029-rur0LkfqebBUGVQ;PhCfFw?DvLfl>CYl4K5=O297kxQSzoX
zFL{qLEBbK##BS2zdW9}fNs(cN_Cb?0Z8r@PhPRnAg6K%FOhsq`J-GN|Q%?jACl3cs
zWq(D(`XZpkKDJT7Tg^Y<t_}NKW9LABI?huRAWo5I7}@u}ZL7)e<lb{+#xlR2{uxqh
z#~p?NNt3}?1nAGfwQw7>9f+U+CJr>S1heNT(hy}@9r%$mh@4OaCsW|?0!XzFc1(@j
z6DF@M8-OHg^t{3@&KYCt_J|c;OjQG}M;fDfgzTkM|6IVt|L+A5HdWxi{Dg4^g91ts
zYyVoky^p_?d58`f*>FXE&4r-}=|zQA$~LDr6%$WNho@@K@U7}tQvRue-eER=x6r-c
zYm%c0OSCvDtIoRpb^Kxvtryn(*UozQ;x5TfXGvFg{P<h(RgcYh?fWnw!Rb2<*@FH+
ziujsgji(z+3gY0yzj=uv%;+CGQ&Tjv02DTe)vFLqkh!L&bugpsg^!QG!0&(-r^rnr
z#(va|0o7784I5W1ejRJ?)%k?rbciR{{W=449^j7E@7PIq%Zx-ZHQR1Nv-M#~(hA8<
zT90176f=JL@>`yv@vkNF$EQ-QOsD1BR|XgU89KrC4C2ydnwm(9v_GG{hy|xwcENT<
zGAIT8VQ96?X%V~I-XN8nv^Uk!>7RQlMwd$-tvj*6VOQaN%(D$468_zWxK1G(p^?!%
z@@oh#=<SdI7r9)83uy45g4hz0`ACoseLa8p^DYZhLc->exdExipCkD2YUr0MFgO6i
z$?G}fE+4@cEltRlA`m7vIXsbf*>@p+1pEBleN9wzD4p^}pe+5diW1%fWipQ(uc~5s
z4S162-S@th|6kGqQ*?#gzlz^T9Rglt&_y=C2Rf{frbr?i#(<^Xi0NsCj6r(RF(!5x
zw<BdpR}e+>7?<Mgu;IQLV!DCEim_iAi@1iKaYf{Bh_M0pi)bqLSN@Bpmz?&!Gc9P3
z9SKII%e7KWB2?tpZrxdJxru^s$_JX?36BA_O&0n@1pvV?_efW+_qe-y7(9Ued&Blk
zz?JwQPveIjatHf)8s9wzJ|heo1>$jj5m4e)yl1sU1oN)`gU%Jq*^&<0hA8%%n*F3)
zT|5Eu(xqTw(9bdxbR&Q?@d)IfJO566O*wZqXugQ=r!+{aD)^<08@&WMl=lRX<PfNn
z%H96U)3gk_gEFC1qz;VGq<I8O(WFBI(mNdNkuYo^!rj+@)Ko-_5+f?3QUnLPXRp61
z>(_y0PLglsCnS%;Ty|;rIcFFT>0E>h%0>O>5-*12p$||)ff`{NwE-c8ZF5n>be2!4
z{weFZ<PanzYwMW>KxCMEjc`K`+c!S|Tg^sAY6^C5ATk}@<sHWU*2L?njO%Vqj*h~$
zfN@<wOOZv46e$~JQX^%tQq^S8Jl}1>NI(#>jOplT?&UJGX0XLm#ja|-0&|(DeJ&ci
zMBx(ee|A2Q_kmyEF6I4c_-BhS5uioO@~tR-1(UK=<}%{tV03#LK_IG~B!3x+!!e3%
zQ~gT%1sgBh#1Spnqq=1*Ep@EyG@;@rE|Od{e8oDeC4>n2&r7d~5ku{XRsC}#?P(i{
zuK`@+>N(VLX4l+8N-~1F*-Th)0Ause_2(!1^X}T-Gs1*7m_l$oOe4@dK<UCPS1dhe
zlJxBn@ji^afbP$yN+LH^nY0dNbROtBo7x}s5hW66u>Kt1=&O3AZribVo|<ZM3<^X7
z4whOl;rN&G_N+i}jjz+zQpL)c*gb@w5O^t9eDwCI07_tl?m<<d^g$$758ay9=}0MM
z_x1H~D(Lv^fFcZ1zi>!~G>B)`Qbp4_UXO78F3@@s<)A`(rgwXn#A6u545Tk1`in>Z
zDztqQ-1dVGbp!lA_>wUJgnW0ZKS8vDC*{8ykqrgZgiI3EhC@vC*;+rSxHD9P{dSY!
zIm3uhLAIQf%E0~l2KpUMgd)vUbZer32!T5tGYX{u&M8)_A4^jD7~@X$HiU<-1cQpD
zq#DA1K;7sGdy)RCMex7Qvj{rc>Nf77KfAId&<IyL#C57{&pPznb_^8B_Imy2EJ{@}
z`Gdj4Ss|to--K1MsbaSML5!Wc{0p>mjwkfa5*DjmHy&PSBLA)Oo+GM6;xB!@NKf;5
zpGUIA8`y0VNtaAMb)?*_d-nG;>E-|1@$xTaEd|n6nf|-2`p=*2YQP(~r2e13_3xkm
zzx<h@2O3npBs7|qtP#ZjG6_25PMXyq{r!h8tVz>YYkj}Huqjp(kA;CJ;$MwK=2;Z9
zLOyBE_vAq!<^pQ~$pgD;_VM2(fij{K4JAW<BccY!wVkidlf;eot);nsav3(}<Tqb{
zT|UqP8#0(n{vd~=`RS)axAh*vr>r}~3Fo_0h_g<x3hQBoEsTawL>Fo6Ixkq#xL~=Z
zT^M;RjFKNwuvi*qp0pMsD~azejNbWmb>j6z=m#aQ;sN!=waT3<hvn;)Lsy3?^M2T#
z*->ryK|unv@T69+m*eTP{j^~`FWV{g?~Mn3ehQohh){BjXpcJo9{o43($Ew#M&It8
zElvBIx>zDm28<90V7dx?1|%2>y02oB?Di+Wz>0bSvSE)uK2r~Dn)AO_{X5vN04x{)
zL?px$9XyqiPoz~z|ILn^y8{W0XCq(O{F5yTPRrVa<cwm>szH*J$o>ek$Zx<LcxqSw
z6P&|;pWpj-%?X5%Jn2ovz>tQ)@D9wfn8Z)(it#qA&j0v(wVmW}wZZvoa(`kl={ubw
zz=#y#WFVv&r-Qv->Ef1s@TIW?sUr}Ml4D($5hrm-Bhh?$2yr`oA_qV$FL*3*{>^@l
ze@_XJOMq|=yHpp&dVdF4N&#U53dDh&Ft8qNv5jBn!uWuoRR4x<92_in>>zclsu)~f
zfMoX-ln)~<l2ITjNiRXc+S3VDhDr;*i(V$sEZ#!pbyv)`0<5+n$M7d?)=IEn2}tY!
zX`h;1xe6Mv*^WfFLtD4pHZ|I4#ije#VaA9{lLSROovC6zP~kzqaoDXeX?(I!$CoyM
zXKA7y$&2p!80|cJWKeB;<oXh^0Q7!Ti`(myhma~Bge-S`cx;=9-CIk3l(n#_m)hU-
z&Bg>W&tV%A86^0aNE;u{LawX!)Y+1hZ{ICt13l~G@`kZ{9n#cNs%4*E`m`5cUi{_L
ztIHQZ^~?IcHKPtE+XoECz4t4$DP~}d0E74%pkWV#iNrJk8)6INkpd2{dypW&2HwY%
zyXUTqr_qt}_ymC8=h*_7Ko{!YMwA$Ex#}+O=(oK`D{iP(B^`6yMS_MQQh;1M#I7eC
z_y{Zs6>lxH31!zxJ_BFNz@7JT7kIh~r@N!?zJ?U7k}W8#yM^<|kq0D02x`;3UHBb#
zSlAf38WU5cuQjue__T0-etJCYZ0t&Ye5goimv_)1iGG}hQ9C<KbC+?PwNQ(<94!5+
zUCf(Zfii{vnh#ghCJ@lL^mp%rwoF%Tgc>a)ChOiypnt)TV+M<HR7|}tqIWmJ#5`ui
z<e$*xxoG^MTgs8g=_Ppdp4im+{=IzyvW0oNgUVw;5x!?-&fluuv@l?2FwK8qR+x3_
z;tBNAQOK^rcUze%-PoD#qQX&yLR};#>jRBe<sKj{=3;7JTDT`h2EMsbQ+iy*;Evws
zXQSE-T`4JsEG+@g3RjdUlWvS79cSown&-cTqo_Ay;qGgQ-1BV3j?`WleC$J(#{Jy~
zV!FzXk>`wPouL2C6??Ia#StTWaQGX5hC+J3!IRcEWX}mQ_igkaKU#i_w5TQQ4JjFT
z5CuU~R&*$@-}sUA#4gw*=HElB$7f*Yd&?v3NRgg4JM}#H#32v{8xL)hFt;1kV|uc;
z8zvDf+cj_xVC8Ez6R}-vJe>ZCq#@Ira}g0SYvH~k-2d<Oi*nZjQo2SsPY;9Comg#a
zIh~VglQ{3n`kHe}K3N@~8!FN#mhrNz&=f492ZyS^>DBi>EW#L#Ps53>(^%!+1j%$M
z#bni1W*VO9TL2xGnqw<gUleyIwjvjtkn5(Yvrjw()>4X~nQKZ;E0f_y|3P<ydih04
zN!`fD(UtBS8qh8)j0X&Wic~AtV?yQ)UMIDse!uw5JsgYhiAUyZhEYCO(<hs5S}@RS
zo1wm{rRq#Z5mG7Q8?_W~mFWTfOotUmJ2$7I@R<JD%*($!CMlE9oef%X^MAr+?`Yxg
zZdk}MU(~dY95LyyD_=AT`dCV7Fd70~h9Qyw*kz8()sE~rbVzwQ&zArTxr`<BsBJ}5
zbU;<@Q%pVc?%BXmm+uDlh8k)_;HTAGL;TI{4NnWnh?!G13LU>)?5_poDQs^HM|kgl
z2oDf<)pV95+B!N0Y$9EKp?h@8TOVFU$NNsv&GXT83*cVM__LA)(pGW_X)Z-9>3ila
zt$Y`b9XHFmq7t&GpC)~_)8y|aP0Meu5vhCZbOL;_jq_;ma7{dAn`jdC@)m$uE^gp{
za=4oFWo#<<3wd~`wo_cYT@B7#@nvPv;kya&MHwO6BFpla01nj5#?9S^3fE8eSyy`=
zkltN3Y-eW?xwl#NIhd#M$hE6IGA5d9gCj*A-5fvBl2Tf0`R<kQb=D5Sd6~ru_bcB`
z%kzN3;}tg>oImAIMM0iBbqZDV?>_zllZB&P-5IgqbT0M25tJtx=kFul2Rge{!P;3|
zM5D3TI@l(RJ2cDi`6L0UQ!TAz<uCYwj<`PJSDp5-m_Q)l%~U%}SyDbu+1zurwM+w^
z<>5_^zuC%M9q4wfpYdWG@+zy7OMc80z;R9xTQMk`4uob=6ZX+r%DkIyA-Bnj?CKo?
z$-le6zE_%=Ogx$7m$7PT-Cr^%)IWLi93{doNwyq<A#dTW{*t^1Vo(G*P*1jIJ5B;S
zkXmW^gqcEE7;`I4b4;ML_xdOcLz<FSnFqM!<Wp-0$m$9eaFCU!P4DQt)C>WaEEBg0
zJaLVgSYGS}pG(o&1^uyVH0t(37%M`_YiJpwvV~?q(HnF9R6y$W(=1ffO7R?p<FH9A
z+1sS4XC(|*I>H^}#q9568nzzgfXR;J3$17?M+N^vqw@LeQ-2p~Iec;2RSys3i|eBV
zH108N#YIcFb-ir)T=&G8;fi|xhi$6#jw7yFw?ZJM4T;jV_yomG<}dNEa!7q7W|;cz
zweQJc!hs}UB_tDxDRfGr&<d(rk?8yw@>Qk^8v-pM4mI&$S0QHr?fuh_ECH-dN75f%
zOxdL{Zbqdvhs&Dz{!H*|qhmzodU6^zmQzYQ;*gA4Mbi_Bes)~Z)!+5M)cL`asP1#I
z_aPu4B?M^>KTyObk~R3QfOLm+t#)!%IQ!zQ?we^vKiN2s5|!)CD}Ih@usg-+EYX7d
zX8(mSJ!tCeRTR_~i(vFoKuC4B#~huVy7BWo=QHge5SYu?Od0TC+T4aa@)%ugqIn7!
zO2iyC$|Tmtipy}HyeZyW^>^9ViY9H?;{GiA&MT;c@va?Z-7n5Vw?CX^v5T=h#wtFI
zyUqCNL_A@!Qq$^BNy=-Atx;RwQx|aEd<oK4p&2B3^P}4DWBhCe60A$R=Jk)dt5?|t
zsj&9TR=GDyuBtruStiE~MwKit*O}~id@?MRG0;igkIxby)UyzfiWHK0GJ3;fF35#7
zP@4Lz4Qb-&GV?JG0q;eKykRpEU@xZ16y-c8%1K9hILcSosHm9vT#pl%^}J8Gv}L#<
z{Lnq6Phx+rS>OrjDPXtL?t-E7Jr0zG{HOl?F893{ohVYnjahEO2IF?BsFy$YDI`Dz
zcgoB@Jm&?q64W_o8POC(mh``=+#~3$$*i3Z4%I_{Mh(&_8u3Lk?N0Xs@O7+_C#N4U
zL)dJSwH5Db5~$c_OFGYH*JysfM?5d%z@2=<CzXLM_|Pm&87-O5dxdG+&1M)+F|AXa
z!JDXhr;jW^IriP6Si0-fN=ODqYTvrWVn|MLA;YP_6T>PL70fuA0!Q+EtALPX;K0A0
z3is6E=v?$;7ZGIr^0l1rw}{8LQ#>tMqH}k2!c^60FNRz-i1D5Jc+YlH$MiJ2XMbRU
zQmd?Yk;jBN>p=AY-XVqBs-#byROV6T+upB_1m&A<jC`F8c=gc6?jEf)mtbfyi=&B^
zAjMJYTU}txFR80aPdjv*6NEZ~e#$S%Omy`jX{lfi_~~W$Zrw?+is?$DR>*Fc{HR*a
zU<hTw<Cn9;1)N$R(|dfl!w90vrPnC3#bP^+Vr)5XR;D~7p&*&kd0RfZt9RqPdRjh-
z)*5#|KsMPL;p&qP8Ydkow4+t~1P*O!#0sBlRS6p5>Ti;fsuVJ7r7n8Y1w>-Y?=*C?
zxQsj#(Q|7P2;3L$NOwK`=VXcy`;U{^&jMeqc67SXELwQ9Mv}at&m$NL?g!mEqLCBt
z*N~{JW0m1h(^j=CXLn~T;;!5empdvLEXXGQS^tda3MOybbV)$d8vWCo&UVV2d9C}J
zQ`UDNQ_uYn5SI${Tc!}~-@dl;oZ3r%LTQU5%^aOidD`Xs?}c+x<F(7-rcWwh>{!*h
zRGOC`<u2pTUSYfU2tyalrFmZ@aG_yDv_FEsE-Z1Jr5}^Hq?;5RzONSGIek)t)2K&U
zFpc|a+BEKvTaw)sd)zlxq)lQhiCbn`{-<u#GZGf6J`WxdzLxwo?{DV===<I;*w&tV
z;CZUW^KVzu$|R|eo9rk=cDZNdk{g*9dpD@WWf`NKh~%!4l}V$;`5zKa<qBSW+SjrN
zlP!uD4rcOCkW|b*hwNv4RUL{qY6vH}hkV9-J8oo>+eGGhOiR-nisAcaLzvx6DIyCg
zg1IyXr{u)Mb+nJ@QetJ#`ghB*xwQT4eXFXY<mqY~(=|F)_@`7ten?tJ>wT*y{<8-g
zdZF4`C5x0^b!@0>d~(JrtkzeuJY_Q6%I2R5)-!u~VLOpsWgC!vr(=Pqr;I-{=4N#3
ze%z%#daagCuwN}=U9yooRfjF&cx(DGIqes=zmgINx~@0IP~@M|m0~`R`xcWWBc@u)
z&3VSq(CzPa5cqp-47|HWqTHt2&|lZ^%KF6L!=MBcPs2AjTL!&cQY8BOg&rJJZ2hqX
zuKT|T)V4EyB|mZYe>YwrIsggCJ9DXt!ODUovouRZCNB?#7AFNVY9!{SbHh`WzR$Rp
z-*7jRy`otaLxt|DX7<vL>utzt8fg`4F2&?mr#3ttI+uPqmP|HZldrM$riJoRx~}sX
zckgbU{`<K2D)CN~clMTT6o}{z$s;7m@Non!<47d;^f@)r@=76g6W@m-&hlD}F;1tN
z#RxH(i;A(-9sQ)v_Vvoj!o8mwK%&;nwzm0!%z)*R<_9!HjRGac!R(QeQ^RbO_DpJB
zeOjVbYRunu(j2~hpm`&66BYXED|k9(h3KloVA4axsR?qcjrDPi(r^o&ZkF+|AKlMX
z^nu8ZQfX$1_GS!U!OnI(+P<x3ZMYD|QWycOf;pEC@uiySXqbdMZv_b4bc#kxSKdL3
zwpW@XF@ik{V7XLjtKFGC`_6m#WLN3GUy8DGa28kJdf$b-r87iX3f}4-{e1dAVA%Jb
zT6Kfq+)f<5D%cU8jgzBsZ<mQ9==K{ba9e4T+`adJOz|B(;oY--q-uNM`GkAO(B(SM
zJf%r}13`r!VcOn6OahQ<XU-~Por^K1e5H-eB#EluI+ywiGl%UVp+n5}Uao(ks?xKN
z6Op9<^zP=G5mDr8t&+o0Ocx@Mi#j0NIQ@p}V0Ihill_}{LIhhx?dS?g?m~(S!_RW(
zb8b`(vK-aaICLKgU#qo!NGhZXk?W;LIzC}q$(PI<#R$)4<&!{L{O((7mkJ9;%QMo0
zT-z<Ci>0DckR<JcX*oo4Zrn3zCXe?&9~}Htad5-@8QE^Cf7)>K9~>Z3Gh$!nQ%o`3
zvR0;zqszPMVw#PXL}K?lYoc4IRN{LHZm~*^;cZDPINf8z<06A(r!}&pn3xUZjy`87
zX61Ny5_Qk-*|*a*$Wg9|lhmI&s)Zu5G;eGuI^52e9>V{au(^<sBj(==J@7+S>_x`?
zFSR9E-2Z%v`v3J)D)-Enjmlt(4`70uO!x)2<GUXOrilQqeSfj7Bs#|LWMgs8lRCy{
z$V11$YTG>yH<_Luz6*WGquTJ}|D4*y|J!emUu8x9K5XkF7AV))qT|g8=`(wOmy8fJ
z_<Q0>PhFmUdws|5JhJsqtrOWCOd&>G!Hc=JsC?Zx+VF7g89%TgPkljQTg2t7w_tXx
zuDJUi`TEAmlHjJN`&5&mujr?EP45bR2^B#;Q3N`Jkolb%*br!u#$eI=3L`q+UjYy?
zWR1-@k0k4WE!nR{jYC@?2p|9fe0#o2ufJ43YGkhq2>dGXiZKqNI+nuEJP8lAvGIjG
z7gcE11<dl;vIR+&0a?iA0l?(We631QqD5vvtl^ZPh&(l|m|kXM)|nH2WE-hlQ!nTE
z|18!1vyIGJS%(C5p|COF8v$Iu3L<gHB~~`I1<?`s^M>V6D5>iT9{zujtU!n&J_nm}
zqT05Nkj#z*dvOowt3rT01dNdfV6T>Ah?1)VVeQP^P7?Bd{8^Wp(pbeIE|$E=%kZ2c
z#d|@ukL1D}sOgGt?~eNUr1lb=W6ry}QUZZ7epf-inJss58|J&J0D2&Q@tWUyzwd?5
z4^57wpP*oYVyYk941U0+!hj2!tK0tR|FH*svu+H-?5(a3GMIBEVwP?m0=j1B+r|K~
zC;6gbU<exyq<o(*qR;E@Fs{L;Bc@x?`agi8Yo<6I1wipv(5QkgT&6yu@H7~=rZNVB
zs(XnZndIKkfgYhC^i9yFV?{l5N>%;KR+@{EFmVY3C}MgH47B^)sh1JQCpuyx=!qlb
zw_r3{@a;^MyA-4jWeotA<S78tFz~T{db-U*qLm-8@-ZQ|RGV5q{$=nu(7gtiAP>Vw
z7)3^@^*va2^jXS}@tYl6hVaN{Aw0Z^ips0-V3c7KE7^`9;ZXGGFRJ{Yo}GO=K|fAW
z(IT%a4s>#WR(?Uw;FxNDP0HGC5ZUYV2YmVT1OA6tc|ZmO*2+69_ki*J8eAuA1M6Mo
z4DyyauB|l@M9F+ml~KJ5HO0T4A0F)stE{zY6CP<%LL-27`jt5XLb)4zA^V#h0O$e<
zN9l@T;avjMmpz!pf4P61Y9?28at-d}ui*7=K6DN<*H=Iq0c(5?%_sQPDnUQ??9I;=
z7|i>{avabb4y4j1C@o|_LN_I+I)ZC07GQXrYr(COi0)VN!x!NnEetj<H2}I<mFe+;
zGk7i;aCwSR;3K6#wggx~x&e|}<!x}A^)mE3KtxAU2f5jQnb^QH0G#kLEE!#Y9=5;{
z2ib4HO6hk{MMn@c(~X@2I6wj-r=J<hV`5MBDslqGdU@gO<J8*A>?DHKMwtU+3*0em
zXt|gfthvr5e^{s%q~>s@%g2rPs-YAX6zzL%h+P+81FLs72lR}JDmqAqnO~$e>fi5w
zqdd{>neIgRb&?}`KjrRpg1er~ruzC^OPJj{1iS2~pSk3aa(Bfz+JI1n4T1WWvAA5*
zi<K}@l3fqBfZMwVCL#tgpzy6i7zW2CQO|ZE5DST0;0xmB{3hAWkxSLA7ktaqn`k@6
zg0yRz>Dz`t9lUL_pN^FYRy}Cpexm|7ZX?fsp<rjFC}gKr#suoQU3+p^Tw*2E!uErt
z#{f8-#)-S7Yj9zk7ley*D!^jJbF4;+ja4O3OvxM6CGv%@dF#d53P^=+`0Kc#wxhb2
ze!*as>G#o*cwXEE!?uD-xECRSl$Eo1*1lN6F@aAB&X;bxb%IwxPmI?YL1LGvwXei2
zl`4{6QtIhrV>1ehP<nK|_@4~}kNym33i=>w3#vU2Zq1ZD`3p6J09f=BYGirdvmUs6
zF3?u7)$|m})R(Kfz2|eIm%wnzJ!W8i3V`XjvL&&`Ghf|PF!kR_)i#Xo>B!z;Nzrg>
zk6$X_7L+0CeS7cjXNQyxOv+wg@9lPVc$V)}r|^Wx8ZJ9Fd#{hAFMkB%i*e=O1>|5I
z7*qYc`te-yz5q0KEX4J9xij~d-Z3;;`qj7q+ejpJ3VF27oG{EU1B_Bf7)?{TO(Vk?
zzn5{fOp>(<M}@KVSrA)oxNEO*eL6;&$$_rg*u{?7Giy^@aQw#-`%LT8)9m%O22LGI
zllwraDt~zbG_uMCF=}<~i*kw{`jq<s@zi-N(q(57Yjj&|w=!zqQkK11nBys(6R|TP
zdwyB+L-rmFd$(cSc3V2hrpAZcpR)BUY4%#NH5wKoJ3D`*=9t2#_<K_lySTL}HPu=A
zCH_9h{+noVQxvV<_leyP2%I_@l1g_QASJzQZW<CnogOiTWfCp-XdPcng{oAbpqcom
zq6SUK?<`>9Yz~V=$3Mju>J{Xu#tM09|7n(xAGi|mzp*n@L5A>cjPT`O%XeA?tcau4
zEFx=(_axT|>VG<EP)K2?@sii=6>oh<+$mLp@_v8ULZU9QvXC-Cp40q3c!4XvUczgA
zV@3TFi@c1&cp=3mT`MmYxDbt~f<rC8W+O-YcE{JrQ(1Xb@xHOjVoA5Nf5MPLdM@#L
z3b)&WNtN#iAg+p4=g{R_OnwqhB#z|8ZX|_<jypIx`$vXrmAu__aotXTyFBE=?av_f
zBmZm|oQ-l3?sl4&;^igPCUjEdw$+2ZKcq&LT4_E%*x(jipJTx`>i(RYD-{ZF2>T85
z5r&^kQMqR$t#^t@l_g1evh{A#J6ymo(bJ}CfO+i5%SmV6tfYK@tu^7z0r1IW;$=k&
z$`McQuee`BYTNG7vM(FSs7hUMU%~;OAAWj<$tzly4US?NZ|Un?DeO_BoU_;{UzYEm
zu>6vgx?=PyPMkB~I`I-@+puaCw*3I7>LQwsKU8ge*LBcYL|YzgjLa<l?&F@vp*XL0
z9(#^Qmk2b*hQ1<1a+<zhc2f^wEfDLxyC-1#6n~PI<znM6w}(gUD+pL$Qur1N12^s$
zz!Q=Ld6&>&mUIo!^o+xM>Ha6MTdb1YoVP6}-VTJgFz>_J?H*>pVko)F;m=tSNN{c@
zShT10TB&d224-Te!}WcVYb$ELjNUQ&x2Gk^DSu}@O^J@n%j24qM{B0gD_xf8%q_Rt
znAfhP`d<@viNr5_Rc`sU=QC98BYpRn2V&%kyxFaEX1HTbD^MIbt7-eS)7Y0{EEtrv
zHZqn9oBF9)9_rb%wgJhvS2Uvd7|BhD-u9g>w{~mB=3&Pb6mxNeRu+~K6v2*L{~&ir
z8Mu2i;!D<a@1KSt0ylN9mf}ZI+i}7S7j#PlK7N1kc0gy#E8<dL=Indh8s7PD->$m5
z;`l6a3H|8eY|V_3M4De+_c~NV*ly<T9VQ!a#OeR9_P#tE>vav6&(LScoGI~{hZG?h
zGDV?`4VnvytW?6P%sxXXp;8etB$^XViu%x?GHb1*l1heB2^lh-`^~u4KIb~uxvq1r
z^UvOY>}zlG8{XghKEwUo&;88g<8u5|;<;PvhhehTRx-Z3nwc)isukUSg)fg^Io*18
zZ9u-CR3NiO?q}wo86(8ptdh*jbP~FfzS~N#>=<3z)~8+P5_^1ilG~XJ5VMRxbvp9T
znWCjq!p>1e`gz-<O&Jk~4?d6hb7rWfgVrE;xh~o6RP)mP63iQZnj|XETAzM&p~AM8
z3qwxTunNUWelg)&PGwuxSEf*Pc#Y&8enalz1BipRsm)QezP>lMrOB>@tK4MH;-wQO
zxPCtUYvRngL&dEGsgEC`Xz>i>sNXt+sd`_~InD?OD%{w@r#7@L&NV6f?Ftuq^bZY*
za~qi}_WZ5y$PvaJS)vd)Lo}JOrcChkz4fl!vJ7M?C0ar2UF#cDFISeu1g0#NP3wHP
zFSRn+cS}>9s^aJ3K>J0Ra^y`!^^O*t=hlHZpwlb1-HW{|b^LG1hp2y&71WQFRf+WE
zSTVcy&rqt(4NdP*F=4U!M}1o!mFu20ND9`j$w-kG*y31u@%{RV=*75~O=CnXby%~4
zR9WP%7`NP7FC=Txpx3(R(R@rNR_HzZoJmue@2%sn5Rgwbo|wE)lnMIjB`h1Y)wNNo
zVkrMru^v#sIhW&dCQnoF+@^-e;1w?H#Mb>}`%O^kuM-KmDUhTP-^d!HT;>`=`}t>~
zDSkf&3+zG&l~r-JciAk9J<P%)xsPVIDl!fP2`_gMU#j5a!Z13lqyCL%Gr}F2xaK`5
zE|s};<CO(mflg6uS{Fv?E4!Ra;T$Ro2`n6KER#rL75H5(OS;&>GkKq4%GJO`N4vwT
ze&2TLztTfEMP-%Jc`N^ze2MWDig!d^n1lVM&crn){)qED%FvWwO_y(tn~+KO%pt74
zEdKxFO@x%wf8J`q^BV`yQNB_RVvi%4*5oH=#~X<mzgdm6kRQ`lb{={Yef#KIkZrn_
zH+-4MO1e0amH4I|S@`dzqN%aP@Hd~k^>XFJN=$r2U;IMud;WFTi91c)v-upBs_;ec
zIQBBRP@Hnt@QY>$Ct6LMtg-!IOfM9|8lGy{vf|&rXr6^%6jnVxtvv0O$c^Gp!8diS
ze7%$WLRaKGs{b|%(jc9=8vH>-#u!w}83E;jlOIvlsTf#TJ5ny%#41;JW););r&T&Y
zS<1QP#FgvhYL6+9Qj^hRVvJSWHJl$t?YTeTidm%{7r@%x8;)T7+=8mu#$Vvh?7tC}
z*v+B&&?9LK=EBNwhA?cRhyBkpy}7;yE3L(szU!pl=$)Do=xhDZZB6NhvF_vs<s%S8
z?Zsd@yy-wjPtrSfUhVyR7W@VT67TixX`fh4hSA9Tbu7Vb<ipa1lvZV1AzixQ2f>%+
zCsi#oaVby}{2<q2nTE<j7I>6D!Kk_mGas{@cE@-f-829b9ZE+qPvnG>eJ1Dw7^KjH
zl&LJpnJ?v0E0IcgQ-CbwyTmg&(H}2;M+XTTSmO;OVQZ!bW6H8PZVA2#jZw&WA{bz2
zEWpsd-6!0sk_VcY1yB(q@<<%L4^YU*h%VYN%IJZxr-18!fPJwR?FvYXAbAtx$v~W5
zQDc*;2c7DjDAuol8XXd!OVE5E=@AI~-KbJ2)46ky0V7$ocyV#f7bwrC!s&v9vMfl0
zFN}z#AjZ)7Y(BYHXpdMs%;TMTU%a-*Qi+Hf1q}rS2R7W^Lt+h-MYkc#etowUX&}(z
zBaO_v8g58ep{6#l1yxaalXeqhdko-~)nnNB2?t<F?RCM*CTdg~$R|Mn6cDF9G(LTn
z)T@~(3Whq+u{>L@Fn$p5^;a%Ko~_)iRK))qUIZTyeH$?wH$(r=dKls}5c){iKZ2n6
zX}~Bq=Z;r-@`EG;k@Vw?zq;Y3%GUq^g3N<*M#g2@xM<4tOF{t<y^bX<;0uw^<N-t#
zFK0mIopm3W*z%5F-@TsD0w_-w{j?}!;_xZOi&J`KEsnIdZN|pAOI4r2+%44QOIRpg
z>;M#JTKAQ+`sh&gK|{3s02(HDGCD{M5b`J1q=R@3_{6_OaQ(6c4VG&uwN|?r#$naT
zw2o+v#PPF=Rksj>Ntt+EedZ1QKAcUrFueBYr;12lP7Mj?GnDMvL@hpPRY){@;+c33
zE{$eoWNtSiE2*j%7y*$$@v1<1Uk9aF^-m@V#Z&pA@iVHZtVNP}`w=|89~m_34#uui
zKhs<xwB^4;hzFHcEwGZrW}wg)|1NkmwVnM#$`*_#bmnxS8NGwb`-YTq$tiQCR1HhP
z+4C{K3+H?vMb(zEi75cU!oFw9IUaBCo)BZbeW}~%3oTO{Jag>fJpf~^=JW9_f@aga
z3cH}hT*rG?Z(gCiMX)%2IO9`r$z$CG>*m6i&~`GT|FgF>zs^P<(@-#zrPA@{5(lMn
zz5NdUggpL%Qf^(1vNli*^-oT12Xm3iUd|i+Zm6K<7{wfoaCJW_`i&0SeN)~@$cLWV
z@139iF6`%8Y%mRKbx^O=ZEgiIZLutdQWz*@r&QLclxA=zbN@k8tdsKM{Gz%AjCe7P
zta8_qU~LXH_3h`$cKrR-EH*HY#@IBNZ+nEtZKd3*@Y;Q<y&T*r&1WHrTt%}`Xg>@C
zxBb~M8T~Ogs!aL1Yr;1Oa0yv{xVwHKxfp#&#8HlNNm3H$gK!7lGntH-Ze$t1j~$JJ
zoJ3aMVikP7OtygF<Q+9Pl+?zon{Zrl^0>jI2h2i-H--cL)3D1Ofse@Y6>>0}Jl<oD
zhy01*1a!EQ;?rGuGjHCyseeR&L43&Rs=w~x)4w$AS(IA@A9rhQ`Ow5i^P|Krv?y}>
zJm)`ri6Hi~u6?(}#A}S*5$D<#=-_9ARo{`iO`UeK3@O;9WznZwr<~i^|BCm+&RF?9
z$<KWJu9I)qY=&#uyjVPCw#MJ<g&>~Wuj+;V)Rr9&lN50+ugn%*T*|nMngTNq%wrW$
zQRhoNNNcYjuY!GjGJ?@r4?17E!GcM@hIjf7E)ue!S@SmZp!dkR6^J}0O5ca2e|ce5
zD|Y9)8$0Yh=<dgOc0~WO5!BkVYW(F^TBcj5yp8G0{Ow=QF(5mLlEcFcpjf*A_{lGT
z-JE;%PA0Z5SOT8lyGaK2)R2*%xAsD-iGZeF&1JV6{dNLw!b4T%F;>5|Y?GtkKIleS
zk*_qdgfw2VNoJI7At|zj9v!9Kl{tV5t<wyPfYM<8JS`caFD5r=Le=)QDX&3-MKKh0
zk6mqanEDX3MI<oEzPg<p8cUid%8ozVYGhY8-}xB(1-hOH1D3>uL@~7!KaPGlmmRfP
z9Pf~ax})hN8b%~Me`t)dK)!<o*#hoe=|zY$Ew<7DKB7jV_I?D2RJkWhi#WW_JpX=$
z3OTkpdOnWb03Xu8^o9>Xugw5l@*Mw;>+P3XQYTNayK++itNTEGkvlG^NQay%85KE0
z!joyyFceu24p8AG9@`PT6g_e&!{uB1K~~}yPgHlJBqJ7aHVg$As+pQnxOQUQjOoGk
zaSSOsu|f<Z(GbSn8S={$(oKu`%_WMJ!eD-m_|7~x#+Um2PIJjS{k&7O9*<zcEUeai
z6(&x?e-E4uS2Bi=9e$g({q!1M;>NRka=tU*eaiZv4^^HL0VQX*lLK$ZxLCjU!$q>O
ze?Ilw3M)FFg75UnfpCz3+ne{5|G}08MjnYscC_9S$aug2UjH}R1t3i*8z=Vzop!sw
z2u(9@Sy?#c2?3WI6A)7zHOAI0cn#kS9D6_V>a11-4A65HkZ^XTuR9EI<I(Pe0Bmns
zs$Y}lk_b1D*0aNim)!Po;uJX_r>R3P-^IFnRW)Xlcy2l6iG(+|Ce_m%DW@GkYxG{N
zd=L~UK17a?*1B}iBHy2k)BAiLp@qwxVH}vR0FGlE;GDNZp9w8R8V5QZyNJ$@nAhED
z>(D^fVQ|22Kkgz-0e>^wU~85Izn@_EcAmb5o!2n01hP<iZ$|ObWUi+ABIZ@Rxi?JM
zg|b!19C~!ol;=g6&3GI;=Q%Y3)qNss^zU2fFR)zm$GeREC``F4G3NGY4L#X~MG9v^
z<D)&CCJ?FZLEp^y0FA9d471DH<qqBeEL$f)Mpw5K=^ad3Ve!>qtoZYyx9{7y^`eQf
z*8Zq!bMd^*gU7!_Ji&=YYq)HG7nQOv7WSPR_i;~2(AVrM{P%yD;UQvX(|&E)_-~Wg
z%#U9zt|~eGA2u-!OV+jO-OkCwF>V{<7gLrV%Tf7#4~ulBx<~T4T8F1uLE0RWCzh`2
zm7HZ^PS@Hrs6I!^d&&1#cc5`QZz6}_%tC3Gvw2&7?f6wEEv_7??UYzf66z)_IGv*G
z7z0*^_S`Fz9)0YOKsrQDlk#Gm(mWOMD8=Iy{vX_gUMToK%Ul|YmoNU+c2;N;cKICD
z9LH42ZAl?6#dr7byfpE|G&?gkVi@S5^X(A(WU{<@^dG4U+wlFa`y)t(teci{Xv$4r
z+%988tiB-~^5hb4W)Vv8!3T!0nV-R>5*frG2#7f*-CKLpXUh5{5J%+|_hR-+Bsjw3
z840rz32TqZhdqQgqkr>JbL`OT*o$OjCWuv~BOMY)D%__qMcOAUan_J*FFR^dKiSAI
zy!F9>r1@cn<bjac*={E2FR+Z6qkr;zF}fWr)kuffp9le=VD^qcqjn#ptOIqvPp~9W
zm3)9J4&9F2NX!7U5TgIZ^MVag5hE5FRh$Lr8~(Npe0jtx4d~?nA9Vl?-TcdU)Md}y
z#&`A3heHnxHZX}@&YKAD(m`_WNT?r3+IK_m42k;WmV^|1sBM;7^!LJbg$MS6zKN>b
z!8f=VM&;cF%^m$4Hc1+vpqfqsR>&Sk6J55G+&)T<Z%i;77gFnku=<a6$soq92i`{{
zkjlNnha{m8jj@1>jj2XuJZ(2-7V+F1Mj2tcavx9B85gX6x(W1Tsz8ryJsdvDcKXNz
zvK1W_HTlDtnA>wtN_Wo|Fib?FZpzsVtp{Tacd~hFmou)jPk4YpGKT||wiAOX3$`sI
z3e>sv%%w9+Qm?JM9?JUtHiyJRBk``msWTDpfK6Zfd9QyyY{dr(#1fqyO5pF32D6?s
z{Rl;_t$sPAfV(u+JHZUgi{|+`qdN|*@b#ZMi0e53E6knMM<On^<&O2JWJmJmJnIpm
z3NENNPe&IM*zn(8<+p<8{U?}nu5%B7C10iu)${uwrRLc&IXFz38QKCoj=;wq%H%s6
za`K5ll3sI`63Qw0q8bLpNLwMVI<NN)MuGW=-9fm6!Wh5y{@1$}En!C+828O1!CK8f
zrpy*CkhOOE=kGnfnT;1HoV2EZF0ml?9<_DQS;2nBEF*LoC;y0ZtxUV<KlY6-sC*9`
zj~x+@Ax$~s=46eC+6i`4U|Et4oBVGQd|kWyfotPy5N|}OT%Tz4ZOfX;_ZScRT*zti
zuzmRJKg^{OF_%x?UYd^LkYgY3fnOZ^8of?x@=T6jw9A}vlehc#pBV4vQs2C3_tuiG
zscj%+v4A9Oy~agz0Pz~pfsg!=!Pv(ky0&5d{NGG|vZTui1SBtYY0-oanD~s0eO#il
z^}97}r|jW)fGSVUa`U#I*%PL9;w9^M<A$%TTR0}JF=4i*JN|#Z`0II&F^>11E(U+%
zA6KI%5QlRt?yux&p5gZzI^M(WJr*S?k_#2gBPmv@vmwZ1q=1F_phfWUU<{z;PWcYh
zJN4JubhH<sC2SyjYH7Nx;3Ll1`%B}dv`EUAm)A&*oSj$l4Q9-uwc=B5!UG#J)*Pav
z7*>8;HsZk5Ni$)5kyFU>ui4%A$L%mYG3jdrs)PO%9C8sd{15tyq<!&PVb}85J-Z;}
zqHFk4aOwe8ZXwXYXjG92n~>6Z3O8pxCY*hKd5sA163quPr)Yl{nkFitDDb8h5hv{V
zW(=-dCc}++bUT*#Tq5NF6w%Q}y%qzuNFNKpg7+jsLLufSOnKe_V%g)aRT$01W7yxc
zdEcu~fTK`)|AdII9-S(w`x^@oe%0IrV&sh&WtC&JfIJG7#wQGiI!oKUrb8X3c-{*?
zB#O8YR0h%y&%fISfVRbuaEf4Gj^D>@gnEp8P?6>DCj*U9Ep+5%4pp!|J%kZh2s*dT
z2dASy9eRpjOQ1LE3+~^D!K<0<E*tY)INC{d1G5fs_i9+xC~-pF_HHS6@<#jvF<Bc#
zKP{E1&lRFECHIKVlK()EG*jm&JtOtyahLuFwLXrv@XXo$fJiQ=8K;GM9=EdW1U*Cj
zJI%U}utl;*NV}m<h!R6FEnvwRG)zV?nIH_+H7iWVc*71N0)29s&_cATf;v!Z9l{;6
z`wjpsRHe`!rLG;VAAo8p%pZtIDhAZYaZ*OR-{@qrNMyPd19MB*p7d=JClpdj$06oy
z3(E&$3p4`l+zLvJUB?sATLKn1W&=EbJpwnEx^D#Y37%)Dza_(DEi|J-UXEbA0V#50
zo>&qZl6SqmyD(oojGuHyF3%1#*71Yo%(HwMmittW>a&IxW!%&R?m_0MVyw3)&viv_
z`nRdvK<N&WVWm4&m8a(tm_^;&DRFN)fVslP)|S?)wBYguPadjN?`^_<9>Vvne+8b3
z=Xr#kU}!J%xK;0iFIlQ!z`sQ1vY#Cw&Rj7WF5pN-UvIqzn85Yf4)J!qz_f~6Rt$;w
z$5F&POm)Iukyb8c+z}Y5qFAAF5N;u^XxB;YtdGVB)*Xg^azu&>F)WyC&FB2N+M}f-
zN9?&Ls%-+QcOX#<WY0E@?E@8QZDkNg%e2MiBa8cRTi!7&ZY)H(-1ZrNOX)StXCjL9
zSp<x6(4#J(D9j<VLK1UA&RQ*FY+!Gej$$~nHV%;t%|j!f@nGcVH5Ax2@|lu#`wP+G
zvpO5CJmo%&_MTPh#_=r6Y$q4;*jCt&E+g5U=Mkx^Foc-O^+o-=!&%s8ia(S*sggBZ
zRmg&5=6n5h*(3X0<sHEs37g@aaB|2ga?WE3R<r@SW}5u1;o`QARr=PAjvTn1Wv(sD
z3o-BJ&NGmg1uC~|hj)lL+e{iodb0?iljh)Me<K8{-5m0hE1Iq6+uP(bl$y`%s@`SI
zq2-(U5oOo{>fFm3&Qa1#QQc+HAL7mx#e+Ethd8KckbjqBn<k-O>zjdHTkZqPY3pFQ
z1Ca|tB&A&lUv%f!O|wzQ>>P7ca|&!q4sDPpc5;xy-C%n>_|1;nq|pjzA~stzW{q=2
z37olrs^hWH`eEG_q;(9SuKvsAf9Z0Kbu#Jtd!CteE-qVXn@3_Q??>q(5M*QZNIu6f
zEg#%y@jDf1DT*GakQ&o{1U)aC+%?$yeeJ(J!)4fim#-hTOs>uNKd80FNBKYei*$>-
zZ*%*`!fVDY*2L|e8zV`1c+8Gug`tFB`(D{U4jbzp;jQKDDsU_g)j4$mg$J~z^DCOh
z@@{YaEMy1}5yOsJ`c>S$WLCH;bRJjO@jE3Y<|vEwGOqSJ-FTS7*&H{WZC3dS!y`a8
z8`cr?3Q*Eto72f5r~qdn`(*78v&r|P`|#CAizu{vrZt|$)lP(bsdWo_L%>uVPYB4g
zmPfu394~|!Do^|w?KT_mme(aG@j8~$omIgjr1z}|^HoWsFW@8ul%r&*tt)AMJkm<e
zJp2P`fOw_*A6<kJvsVBD*#YtwCd!!RgrEpIR|2QS0T2*DOA63z7z7vCzKZc>c!CY4
zLI&2NCSdA^K=DNu7=7qn-SPU?SAhH_LHEZfVJ}c!2Yz)MNF<61TR|ciEZ{q(gV#Ho
zVlUJCnGAu&q=zl=rWm(-3h{OA9W}4oiub4x41-3a<N_g~nu2N0N+=5DBfd*cNIzCO
zG$Fq~yWS8W*3OY%gsgQV^OV}w%9(=v>C)EzBk%<J=$%Y-2-(vMRCyT2+*yFxhI>As
zkXG;=U&4DLTbJ1Tkbf<dUBJ23z2W!s56(B3fGRKHUA_Q11+xtU`UxX<DaW#K?*{-?
z&#gFhY>cl($`HsK{GmN}7}zrsrDd9I=<ZZRl9B@CKZKcubdSQ$kXg1Ftn(M_|Mn(6
zb~yHGWhIQkZp=npyZO&q5#BOuvnUwMOVrWP3wcOH4Wi*sjz<~19dBIty<tby7+Ws|
zk+4Kau<6|$?)^8OgE9w?Qh1zq(6`7oY~VM|>KjWBN6y?J9Upv}I)0j7;F~*p;gPbh
zlh6tEDMS_edOi*AIb_PW^)DXJr46lnihc@5#~J=oYW77+*6zbMf5Vjx3;pKijt;b@
zDONQ<J#H!~-igncZOhEj%cr5!9B*1O$+#NnEai<6isD%s-S)X7qYEw#2;ZL48mJC<
z&C8VZKqT<i$BH^QT)C3zl4@V;0XNj*_gXKupE&zv**J;)f|c;_SrJUuevhf)8?UU-
zS4bUoKOb3v-sG>4$l`L}M*!-=Dqg9-Wl4r0I8xJ<^R5_$I6<eH^&ui7H44!zyo=+j
zOF@TwBz;h_vo{D$Q#DIXD>37A8~AZq-;L|#7VoWDDpF%Y6CX6*L{sSwVd3dV|81$l
z0v_>!nv7lYcyy!3%Hd~UM4GKGxn;BW!DQGGLNQ^8(jTxQND`qV413i02abTRrc`gI
zcv>YTtNr#UD9>HbQg8RYH#HBzi9607oVcELi?6X>^D^z0m9|)=9R}4o@X_u-OyDDW
zgrB(5iU7TypDD!?Vci>P-C{!JxbLmrzPY14L&Ev7J^ddWU)C{(#uE!JDD5086yb#Z
zjXX5AJJt)zC+>{r?dGYzWYk;A7skg!)ubNGw?OgmCA?|uypx7h1X(=vgAi$^uWBn~
zNQ7S(i=wITGU+&zn^dq%D*4IuedbHUZFu90P`{CjFxFC|-OO;ZO$)yJ$?d$@_peFE
z;!jZ8{*p3mcKK$^PU%3A!=>eNTLPjNSe-*hsWp=G3FAt8k|UI)>+PCAXZ?hc*7_U1
zhZsz<gr<e;LS?h*qoZ}&d#<*wAv=6XpdXv6ou?Bfp!QPQQdTj;YWUUi6HYfN4wf!=
zn1`u}yF;E*$jo|}Nt7Nhp(GsmtH;TC$z2Za+1efGKF~u+mSe72g!(n+RRVc(%&V9X
zmbwo0{~%c}^CKk|j+1L$zL}n!?RbtwSk}dz<!>EFd5Y`Yn2M9q6$hA~IVnHn9?XBR
z{^Dxo?9EwmrNBP(f=|M=jb(ekQ*lv~4F<dSIM8nhJ93zAY1iE71RK*B&~2S9{RVlx
z=3xm8mL@vmPfHOB<{8RJhX50yxe&v-KCB6e!MVfChiO2{Pk0)8aFp|t_dcFRXR%si
z!5zv*UEi**%{vGje6tLt-24u@=e8iLh5(c)8ZNhWyCf-O6DcKC-tBI!Xqc}kx{dO9
zr}@6F<f)njZ6@13ulbj>0zNn9QIyq;OASkncHemHeklI{^CeNQD3Oui!^Y+;4pyr6
znPvx5eC9T-S&Rh>H#unPbo@R`jCGSrv(uJR)$Uwx?Dujr<9mS{#onYnp@5H~zm6U*
zawqKPk4a3aX%E|>p<O(hBv!v5XZ$_HqtOOCBd^?2B&6SahB}&-x@N2wF0s)YolUD3
zjvbs@&#t9_HU>(*EuH!iHUq4Lg&y-Dzua5d>r3FOXH&|}nm&XUFz-y7NC?QC;`f&A
zR(YFDDjn`~n{+yF1or5AeH=qe=M?ByjrDNiYIy_(?NCaZTA;D-Nq>R<pZH%@_4+9G
zKjh5az&uyh^+7pCl5`vTLf7Vpt<&aGgaMDMKH5*XVf-c7`l2S|?E;;=CpVKZp9D5}
z=Ufubfx&%IX}>#)CZdk!iy_{^Eh^U{2yi;OqdL79E!t{RwbBsEQ#H`($74YOS|@Z%
zpoVbl)Ii~1h&VrD420AyNETEBk2|5fDjVkx8{#E=Y-t8kojjC#ku6__t_PAT!uV`U
zP`VD^3?io&SnZBXC7@gtKs3kmJo0BGh_Dh{U!_2$zTNXYOgAX!pLW2hVYtQ{Op-uI
zeKazb9r5yaMA9aeh;+?AB~P9Jn4&2z34Ms4c*=j$oJs19mOzP%UO=(W!0)2lRZPRB
z4E2q@$Vz!7bmC152Xb1Cu7}|%XNsyfVPaMrg0YqcDI6Unh_Y6m$T?SafO>}Pc6nY!
zn$alC!U)#Q68Z3Hts2%^XewC3zO<GnnI=6H0$+lVr1KsSYJm7c70j}Su!3!<0r8%b
zPRF15iFk8S0b){mp>UJ6;mn^9==(va-3^7k-yy%_kIZ|CSvn@m<0LN{tkU_&y`i6#
z>nf}b3IfNI=Cfy9B@_*$A*V7LF$qvWA2{Ada2a?~F+QTlOjOsad8x?e9;EBFbJCG$
ziP<<EkL7UMy<~nL7V16{zYdbuqbhBYkP$pOl4QGgD$5O__!5J^C#C2RBFh{GnC)eA
ze?cDd3#OHSSYxF273F!8h#hHaV|o?QD(2>42;B2@L3GLWMluI(NDc7T_2=y#$F3!f
z?^l^0)67cwcyyYWLb)DNMQa%7q5!ANG;+uBAy7M(-f*9yxWWn(qRxYnEt{Fyx|W34
zvm-o_HD+|uN%Diht4D|o`zyW7N({e>Jt~kI`Ih-<?^(!*8DB_)tcB%l>^6V3<O6Wz
zvk_9i6wD%(7B6>8IuDhM8)*yq&$jbsrcQTpWoq=8Ymd4vNp2yDLD?EQ!4-gHI0+OJ
z=cUAO&q;m!hAu6^=Q@*ntztzk<pY(V(um_kcjn?qpfH#XPmpx!i5;cIWN6@l(I$Cs
zo4#qVlpPHP2%)GaF7mVN93xppN8iC)9deAI2rqQZhZF(gHj-GrdKwUJ-(J3|`r?|o
zce$+q1Osnz6E4NLInRV>mdwTWp+_l19Ac+@?h=!JbwTM>qRYgrrr#)++~{)a_FYJZ
z6e(zSnf*nmI>e2Yd6<%@;fTc~8XlM^zjOw(u&Sdo<*sOMc=0M*gFn&2CMe?Q$*jLV
z#uKUw<vYsSt=!5AuAg#~VPWU0mZx8x;c^dhUuV#F24v~v;snb9r>~{;t3G5E$aqUt
z&vXLbzBkWzs<4WB0D&g`Z=FOZ*;5%tN(2-eIuR{_H?jR$LOay@=JH7X`NICicR<<z
zzZcH**Q4Z8odI%l$X9m}CQmfK`gcbhCyZdViE#t~4l8lENd8c!rL9jo&MxY)sAjhk
zbiV7j8w{q^8os{`V+`bNQ1*x;qk4eamCA7K_41qkV4DOy?_M6$f+@45tF}_IzB3^C
z@=$_~4dk}Fr2f}}J0Z9GI@|`rc8r2a^t7r{{Y?{?PEI4C_1yM0C+#-@JOlKdX`sNF
zX?)Tsc}~Gm_$1jAKQ>L6X_`JTguJsZlEC8}0qk<tv#f~ytneGzJzn~B!vwCR!UeeW
zMB3H9><M!{@ta<fmcDzIGUai`lgK=h{RQ1^Or91d{{Gb~C_fBe{taYs#xt1~bJ^{>
za&g)}dP>5q@t&cFT)9CeAGmHlOb}MTnot|}GI^Y`NB1CL`$j9*f8(UnOD>Zp?YD6&
zJBcUqSde5nVd%z99zB7$x30MQ*;Clc*so}W5}x<SMPXvOCN2mZQ`F>{k@KFZ!pzvO
z@K~Tq2p><HW>0@#%fgp{(qqV{g$x~e$G00_t8koN-rBV>7K82=?NKqnh)o=3CqNUv
z-n!R*GZTK}dhyM8j$=d2>kZ)X)#$L7Yhd32Y5ID<Cb;5@3w7`fiHlBkt4w_c%lUS4
zPrtbR#FNlh3jpAK4C%6xIjJbXf?nrdpZgxLphT~y4-)SSJ`YHiKJiV91#+cKiv70y
zjd}#<E#spyOMCmPa(vor-&kRHkLGWS!(oRw4DcHVk{m@w&58ft1Fk@<W6dBx3tHwE
zq+3#+42UXlL$`N@&JBclXy$=5#NnL))pjE~=!f^!g|kpRBDx^-Au!8l5$H6NuNAVW
z3RPKiQ3zve5+AXNY+}rM^<eV}UF6Nm@xS8J&jX7hVwreL#ZUc{$d!=AAM`7~Q^W+p
zrnI-s4|33c7wbnlzOJt^Kuc0RA&upGLwF|(4t^6%_UXQ?f%r6ImYMyz>j723v>zV;
zRLkjq5z|El@IZb_Nv!_+5P<?w=(ax1#}XN(A8s=9p#p%0(mP==VfVp9bg2h>y-XtZ
zSbgI&zN~a2NQUXXQKHQ5h8Zb#=bDGIk40ZuSHflG@l(}$VrQ1ABGMThe*PD6dhs_X
z1aGd+Fm2~DZVFjWpbx;oU$Kn9j32k4WzDqo%wj8Vi!f4WgFqt3Foh~p-7<=SZe`s-
z)4KwZyvB)Xwt$hi>&=}X_;OFR0W$;7jxvpu@G65Fe|dA~+Qy+(8b2t0g7LoK5K3$}
zW_|bmc<Qqu#qUOm0Y8wzfkEirc13+3CQjzNE;YBFvgvXeI9qZ7N)F_3^#{|5iX4#J
z8>5ij6zQ#{r&XY#F1!Nefw~xA9cy4A;oJMcEU7#84tLiTcU@GEqL$@WbQ5xK1el;k
zCgYscbgVYfG3Xt{|5rd4vLk%AYMwd(=h!^OXLvTVq<bKisB>z!moaW|$&t3mISZ%J
z1H;L|>C9M1ki4c*wzlh1+L=GtoT$cHzd|IPdlYC@X#b0YUJl|WSKd^%{-Bm5H&Oju
z%#9_%i`dL{dh|2YjB@UG;iAx<EiK{PbOQ_xGeHDuHrq2-VI@V*B^?Xtf!E;sy5Jvo
zzPd9$ocECmsiD1AG|o>G-0_-M%5ZUSd2ZUP83fhuWNtB(Sonv~oDs~mwxEU_SmNPY
zON4e%A6x^Nfn7s=V=?UcQ#p!QmrJnsK-VYx7WE&}Z~0)+rwq(7I}|A_`|U0t!)D11
zq&USgFh)H35m&>=PidJ>!kr2#2Ah*mR0zH3_OJ5w3~H!)k0YF6&BDybLy*|2CmOnc
zf(rsnhD&6AOjk3l*e(KL;$E4zV#Kk9Uf}bxv)OW{M1qrJsCj!u(a8yW5zj|*knox%
z#L^52HKmRsrZjRY2?uVqzoY)>{NeKD44J5~8N9AOOlP?X1$O@CyVYHzeGuwWs1xH`
znEp89EOR$aW0w;STAK;O#&xcX2c=Gej4JeKIZ(00)yjp<?4nlCP(L~>UtV5+{O%{k
zRoYi87)LyspY0Ydnfa>X55ox~P17ZeD~YP&-`-1KfziNRzP+qG0+F%KTjp53&q^h%
zPQREO<~q7Cv&@7B?Z+dFG&V|W3WZy;+JZ2HewJGGGpHI<IXl2E<F;}%IxM~kINa((
zmYmSDYS^nMFPMYMDYLiI{U48KtA0of^P1|Pu%lJ6?pxlWe|+(!+akm}eV}%ZXd8^~
z{g`TWN#MoR&HATn)K@xJGY5_ku#GhH0qCd+61cpH;UAQ}BrCWW3+G;YobJTs@!0iw
z&@X5?xePygz5Cvgnwbwplyaa~^~%WLf!COu?q2cu%S0{x7iqT*JacpZMN<A@9R8IX
zYTUB%oUKWlL;Qf<(+7r5Mm6+YN2eAK-PeMT)tWmtRZaO%$ihk7mwonegP$&0N;*Z3
zp*16%y)EA^<g>?`smznlqVC%rLw~rUUCe_Aa(cTf&fObp)NZLgvhni7w&3GHDJoNa
z>PY-{W$DHnP+zt9^5lEPzX>vY96D}JW=<r|R;s6fvFl{da#GvTG8D(*rsKPqn}i!W
zw(ZM!vE`a0@4iHevREV$Q$SHFJo_91;lmi+7^ThS=Zq*4N5>a9B~bgVf6sAw#z4hq
z7#8RK{dEnU5h7b0zm}@*@y4m<PIfCQRgJqlK@CLic7<?p{`%PKIt%=`ad7(XXz$n8
z82oM2pxPFVu1x6B^<bxOMKS~v><w0{&1tO*sPezRAH~C`L0Sm&3$6Vm$arHwqm^Cf
z^`{nbI#~flYK-esJ)sJn+5+!NT}4(lXz4#d9E<VvFG)O#k?$BLcDsLn127%Z7#$=z
zt?eNzn&9=#CjQBsSvQ%!KlQyi0&WJ;7z9MT?#9lS-w?Vg)Ih#w_U=1QKs_ZIwx8O3
z;N5+s$ZltRiO8(ls_b$?tfNJg?FRh>(%NVFgHV)xfWzuEy71+u(t_)07$Cez;=_rc
zNS-<HfzMc(VZ{g0b5}9~e0GjsTZR!DSi#*WTk2k0cP8$t#uS|X;yv^P8_|!U^8(2K
zRe>Y4s4k#0NlDa`YFj10O7BIcZm-bV6$pip>#ZoJ<n*ydlgu2B2qX;MwQj9|sFe00
z{E*-jM;hgYZd~B*uQQi&<y|)lP&zaWj6FRCEvqs4n2+0^7B+HiVkf2>s}TM|L{6Il
zD*pj=45cSJ$4K?DOp7@*hzcoy3Hf5VU^7;#U<!_9KwX!jW;7i8JG?Xk7s44feNSSY
zuS4jw;1SaZXC75VGOPc+uvGf-=qiu?Y7av1(R~)w*c|E51H}B=-Om>xuY64NMza`y
zC+dn`%aO+;EFfJ9n=!BvxjBd10yT!oj5%hZ!>p=_TJI0dz3O<J_c{xw+wv0Vn~C5}
zfP~F~bn|Us=~8?eO?+=>pbDJ{8$f#RM~7OCUYUj&$thu&%ahnSVquJ%I0e?%XKQ=V
zc*RkqJa_9&J7l0Z`FVh&zpqAs5Y&F|l%!XgG`NC%i!6k~@}qJ*%cinx<ImL2lTUvK
z9_U$~o%%lZ8xZ;Ip2PcU543F7OD2&I`Z5k2mgGr=#7oK`(6tEbP`1KW#Uh{(kH6OK
z<{ca`L;@VGV3sVGsM87H4$6h{6$i9!bw3HYF3Vz*>>@>E@Hy{{{QB|=6FL{K%~<}P
zFTy*@lPUQ{qyVZU?M=qikXw50r+Y{~wl+F;8OL&+?sCn;LIdyKZ}SOG|HxD%mzk3C
zC@54&+UV_!#H@I#MLoH!?sA_H>#JKPB=^LjW*AA`h${V|-E0|M@A<B?@GN*@Qg3x+
z&o`y(i;i_t1UDM2nCt}7;Hr-cF?v6Qow9*T0=RljdZ6<R4DqDy=`wK&KCos4hiL@Z
z6E|nmuHgP^<&YrO>jpz&K4hHbf`!IAE<J!xFq6nEU?NQib*ZUG7ou|P2aqSbIrkd_
zC4&o9!SlN~><=P3T<~5-mt!TVRfKbA3S_d%bAujhSAuB05}=BoC7*`lCMNzvO12)M
zgLgafo;&xMwWEe`4T-4JaeukuBM$&TNptgW|EOZ$Fol(&uK>36+2ZAUz-aVRV(1UG
zS#8O!YmC)C%X;l^%er$$4T1tzoMk<%S?%JCWZw<_(r*}t_h%bLiNBWCOxC(}Rv>XN
zZFwZCwO{t^^kC++dQ+#oka`1%lE_rwqx1f7T((U8?~wZ(2;j?7k{0l+e=J^>RN#>C
z_yvcAOo+g&M{->MdY-91hxwdfm_*bjIn&HoGv_W4plitvL1-?h&U(>;Uo?r#QlpH>
z>V>pt<*tbrX${AvO>$TaQKyhCVshqPATF~{)70<7!{``($Im9N%eW(>d^Ul70wXWA
ziruA5LOW@{`t#w}4VFE0Qy-<MKbCm9Y)qQg&0yyVK_@%UwA6)b0dEPW&oESLL;`;S
zNJVuB8>PMgbk@JEbvklOky4ASiE_}pBv9=a*Q>5fW0`qc%U$5>)08Gvjp~Ncs--HL
z%+fzXJH?H>M{>~y)ayiJfFRD7Og}a1;{tK(eI1n&IR$|S7h9dyQdXfy3uNAyCJvK8
zD5M&3wX@O3bAVFFwDr0(+sr{AjgOw+KC;bfCHNO>9i-Azt0Y(h`Z``6w;XZ*tL|Ld
z`r-L7e=VxiZfbnGC=}XUm@f%*K1=21ZgMMQ`XaLXBj@PtK}7vT@JDQCC(yvEB+bYY
z#M`I{u<-P?*G1kjZMn(I!OCY&TT)!hVr6SlPDB$n#r`FDUQ@Al5q*#FCb?go!KYqT
zj>3Z3fb4$$HJJ4mW_=WKR&=PZEGhpj!7`Sd7Lpjd@6|d@L<H#!g0)|!U*44HbUwIO
zjfjIv06n=pU~WE)mXPZGAvf7|Q=g6|1%{ZG`-leC1FiRW<;$|KP!DpINe>*sl%oaw
z#zS*8uDwUblVl#kJZhz9!TkXsA2o+>hoeSG`A#XXgioUWKR-W`In2F?CSLL@_C_-j
z2YJ6mo(atSs-{H96K-$*$X_2N=0I6hRva9M;F>zRyGWwU(Q?4RG2Vx!DTi>TFl>6v
z=XsvERe~JYIDq-|-vvE!vr|Xgc^JPaKj!lJCwyW@@$r7qDkjfa4ZYX9l^>r>+J0Jc
z20~?*DCSGj9&2ajtIpfkisDnE6}><dS>EGxj@_-O61sVr2?tUS>pXOwf2Tcs)B{bb
z#W!!Zj3?W6cC21ss@g)mM*l|mv2rEj)_P~DxeF+N@1!YtBBXF$QP_kv%&MYX(v|C7
zGO%&OSiehfU?I_3B3}UI`URmr2cDry?yl(gTJ?Qg%76$-i~MG?a*aT}l=Ry`1HtkI
z=51WtRmo{J#yaS@zj(3(vFtvs(XlZJ&%hZIvMc}1fz>gpi5Sv-Tur@*A~-j<)@1i4
zd8y~)e<~Y$YP_3(u0?jS0&m0X)wCkQYk1jhQ~h=<G8ObWD?BOM<*_QgOgruX3E`B=
zC|XsVUO%tgoAl7o$|0zaRCQ4nAYHog{3|=-XDsX!7?5||r2L?{Xo7|&7iWC&h=R4`
z`khng$aq{?ETF#3F89o-o0Ij7u~^ZLb{oN5^U2Gf{!KBon}$P9hL&Sc(LA6v1@b!!
zCLoas)JsSezhL{~yME(DwlVRCUMs*#^j`6|7h*j6R{_v9yU|htNtD|_D_$UYL6Dzh
z`wpoV3hH1HUqH7>^MQ=}!14&M1b9U&IP@52^r?L9G_2dWffHf$E~%nLIB^3CouDdI
zm$Z_L*+rvo@Q~{$f=1sm>Btx8)I=HM0E+C}zVxARD}`eR1k)Yy9*8aoDBqnm=iM-9
ze+D`0Dq5trM?K%y!lqEJ`d0uFa@`QiVWhY{;6l`SAB6GfMLC)QmLzq3rC=Tbgg!g&
z?B=QbDKz=w^l79{;|7ix;0q{ej36W?odFQ4IlX`IArF-qJRFsE2r9!IhEYc_8GL}`
zn(zdndt(w#sB#FDuW#|;BP`Z!eGjuS-@F{$gX*yKzqY0!XC+z*oAx5|yZ&lA#p()n
zEJ|68P@|GN%mCN+{4*3nAm+UFKIm$!)F^6h<Tv(!J#X*+ght6roJg?jIuVA>M<9e$
zTI;7+43Uin2<39;2+6KIO%JAetSmtR+F~)7fU8D-1z->YM!J$(zfDS_?gRhvErpL>
z3&k@NaYLUy>zUb(tq9r6y867!(_<MM<N!eVAe#MgFwFoo$Y%&Ktv&!Za^7K_u_OW$
zM~Z2oE9Mu3r~3RPW8Za-?r9i8&Oit%nFjXGNXhYdPMUYC5n|Zpu*_c#-b4IXkmSB%
zjy8*M+m&i83C46O>jn09ydu1f152(uKPt0jYkST&=ZFA;jsgVj%Jk&|FPem-%sdmW
za%OZ>=18H|L?pjgIr^NO=7i+<!LO3@<#RB_E3G91j?gw4h*j(ZYlfuhSF>$RsC*&V
zR%x9Tiz84;^SW%0o)180L9Po>9&-S^U7DeZcm;m}>al7VHCE?CRQ=FQaV<w-mR~gW
zt8{yABIAE5>WBg~X*)}RVw(wt=#a)L*!W1$A%Oq=BmzfR_VyCS4ypi(GC>e3s&`2+
z>yWn)G*6%V5S)%$WY5`a<yui6uw5GgEP%A(x{zW^j#bi<(9>=@FQ>K62Z#X1yl+Gy
ziDvtJPaMU%5Oex<b`ng~32R(wf}8=y?=0I#(yV}dc4q6ob;>OZxs#<$Eg=4KzkIpH
zFYqg#S5wtiW|eimFLfWv4=F+4u^-s38Y^VN#EJHJWznaT8>xKg()0wA`XN^$AAJ~;
z%A1pmspb1Eg0_qZilisA=$o!GN|}AXy@o+!_v4F^GCH$&rWv$}t3pXq@oZTeoeTB9
z=-Pi&eU6bXx?_u}mykra&t=F(sLt<%Gk^^_=*vn+wqcaGBV_kzRw)znK~Wc`L%reL
zHTE`WN%-&ps?}Is^#(Bk#S~A<_g-l7&t4+Xk;O|%W!rM#JzK5xI`9V6{Fv>IK;!Nw
zN}ug0><P+j3i$EpAvKZjsO3ylHjBUz@YhJN-Os*EFE?$eAUeba*U+>T|GI}G*1p|1
z=OA~^y=m=2-zj3FZRLHOI1@;sf3k|I{T_qFcz<j6k#n2}u2(V9Ne}C{;h0uMMOn&$
zb5af6<?H$G&N7oPPhO;!)jlfqRjyc-sqnm_$thO7fhUqOoAX_*6&A{^>_p~LyN14h
zBwQ4P<#Oj4+)6Z4Aw$Q{%**Dd1pR6+-^)0MSuUkrhs>lfctQ+HeM-6>8s+c7W1;TU
zsx^}FmdsvVR#;~+Wz#KX;Bzb`!~cTGUM-wx7^t_aquQs{@T@&kIQxO)TYp{S{obnk
z&?P8lRMCo0N5`*;KI2%+@ZyT^xtalyQ7p1l2t?jX>j7cvNjask+buS%jZd28s^HJg
zYnbNd-$pZLzq;fJvv_Zo?%b3TQReIw&8EAEl9IDThZ_gGpOVMi`X9(AG~P0h)!Te$
z*`3tRu|7aLK{)Vr;B@e0I=;HnnpT*y74VZmXZzi%^uTZNOw89liH+LIrtp!TbBwA#
z=-H~3Z<>kx9>WGC_O!fmkq#p~+gGe*R*Qm8s=KrT*t!Rs7g1ibsFaWN#Az2e+&Ip`
zuX{9N7SZ9)J^rP(NwH#0KhN4|8UAn<uu!Wy*u$j(jxA$f!pj}J<HdRYh}@(B)5NeX
z9B=q*_*~bd+p^xQNU4~<z;m1KvRe-BxSWVECDs@;x*bn7JwraRw<|L~P~M>I7yfES
zTXz*-V9UTZ8dEA|$A(};tuBs7LOL7LesYO2s%DRDi`6}NdpKM`RNR>Tpj@>`h*?V&
z4C*s=`gtRYSkywhBhUcT5R?8vRWOlrES8Iq$c$ror`5r$-=W?@TUX?_a2wwilyLVQ
zoFz;cr>9?WioIm090X{inVi8RmuXw}m=*;?5;sCA&*kIUE-KIej~;U$O5ye{63ApF
zGL61acs(Z&ZlUlg@2He9?+^~*8SUWJmg0XQM~oVg3O&I?%s$AW%f#Bk7%L-u45+@(
zPR?7g%1GOEN7_5=l^W$6qotk94s*Hnlg=~&?1<ycP+BGtBf3CFFXAUxW-;{zsd@Ks
zH7Tu7BoO5L+Vb23uu+j(r$jYeER-TWe5@S5KK6|ACq;5-bt4H>tCI>JmdkuDzcEx?
z;ns4rL|^|aUu(iy)AAWM9h||p2S~$9Rt!^Jhx~g`U2eO7R@-<h=G(6u49y+hK5#xa
z?k1_m^RV@FR>jk;#oxmjeA?8|`(|4UH^$<quy700?%Sb`fAz$G&`I@XrQ*=)3TYR|
z7fdH|K5a!q)eg&u^47da3)aojN{hP<T$7A^_l;K*pFErBl|v_i9K^OR{P7hHCdOwt
z>b0M1eqMdoZY;7raid6N{IM}*qgB<aveo&@q~D!=LtLGSiMP_y%w+ZWzbe~_n9O(H
zH&W@E00_rwZvPt6F?H~L(SAX@_NeI0v;g$?Z=Wj1KPr6nbiMt~G3qX9FnR7sQ9D{+
zE3j_CRJ-0SqWM0q*|jMhl{lbuo*HkO0V|@Q_59)AS-^O}(lB>^ZWURpp~hb*>%Xo_
zV@`FU`NX@)<Ebz7XidKC<imT-$HTSmJ2Pch$0J-7)D|ty<+K+eM_EkAh9#1Eqyq$`
zLv%WX)nCR)9i#W*f+`M|@1QRI4)D2;8n5~Pf^fP<N9#am6Z&rRA#Dk!!d=+(vs*p^
zIl2S2*S`8~yncg(eve*Po&j<#oc&ly1HIn6*V~EQ1%ahm&G69?!wJpV0O)?!+D$bG
z#5iZEU^wJHFlK9{1;_{s%ntxo_xuV7i@WSAwsj{?wIbWml1Yt~W4rcm)R*wLKv!|M
zF#=f#B{wi}h7q2Y%$v6^mk|7A$mECppI<~f4c&NJnt{gY@O(HWO-BZk_>fi^JedQJ
z0Z<|k-pKU49o=aJqVfX8F?`iXPSHJ%R||GFF*WvY;9hmQ>sb8L64&L*9;!+SD-+C0
zRm=LzUR!C09Z~hj+8<Gy?xA$lN;Sa4<z#}L*6ab*cVRKjvfqtf4sLmCz+2_-t$pp|
zel?%1FLhh}u6FC&)@>SXP4TXPKso}2XpUnGQ_!J?N3%^cv<9Cs4p`!TFif*I#06j5
zRFDVCDWo#U>w$-ogf>~y7p_<}M5O2mMhBX!5CSMA5syPFPtP6!dPo``Idva^_=9fq
zzB_rQ6jmx9%{YXKnIa`^mpA+PE&TufQK7P6R{cF4?@J?5aH;h6_xB=7D)~SHQ`V!K
z?-sYm;jZv^z=CnlW~(z#HQUqYK`VDlZMX!66m}u(*;(Z#SqxfqN-1i~F0Lj^BD0kW
zj9V03+#X68B4EaYBps|4`g0`t(kEA7>VKsK0F3QvuT!*2j4uVH?_3)|E^B+&xmL;A
z^Sn8`K0)yWKPH{v4J+i$%9i~Cu346Zuy6OPymRxq&dsJvKG1%vWgm%mAunZ@6ynP|
z5WGZ+!m1*o16dOiG@PL9GU_%T?DapxP_#XhqRdjExRVta&3@FcvzK_$Z?&KXAh@X6
z6XjP93L1kFZWVN)l9C@U-RiA_($Nd|`G2CZNz2kRJ`<u|s4>S&n>JkfGsyCTI@047
zt02EKL*3_tQ3PY1vA!l=mtt)%6GFHawZ&+#th?|ePU_nF{Olr|4Pax@4vRNv-XH$}
zWaX9U%u;JMZpRA*#8MC-KilQ@F{HO8KV%g7RK<IM$lu;Q7`?VX?UvTohCo&g{&a&h
z^Pc91x|=Z3?o8EjE_B||yl3V~(4NoiY8?)z_}<#%F#pDt`#8zHgOT@r8aIAMr>OSm
z7MwIlV!Yza$Y0}~*sWYK5YDyB{hJFWfK?3iW~ohX3)Z_L@%%n}&M7cc(k_t<$?FS9
zS!)N~hRXlivrnpAJUjqUc*0k9y?cGAw(pnh9T}<GpXRYIQEUbWRfCJRg5w~s)S2yE
z6%_g{OFYXvqaOK70(ytNF6mkwI;vlFnO>QRDSOT8PnXuoMLyMU;MDElUf9q*ZCTpT
zauT*X;f1tuQ@3zg!-!SWp4-1xxY?xWkScV*(-$Fy{7oMo9roR2(*1NF(^D>uqz7?^
zBzhWT%U;sBH5Zvdp&hN8=NP}s_G0XOLA{|q=g~6P$X8A2aVr)seI<Oo`}-(9O%)dp
zNX^In(agTm&ED^5X1PM!+Y5!E(#7VdhpMD<C}VpI2Zr9qSw?Mq{y~nJvWCM&fs8~T
zlb^0I=PjdE)xw}H*Q@r?;2lEiNYx)<g+8>Z&+9u!L0RyaTRp?1!S%DrKB=<j{CW4B
z1R@I))s42Ji(dKJAkPR~c}mrD9^_@3*tDBePINmzBJA6?Jl_i@jbZVxnEhb_3cTI4
zO-P$nuXWkua5T@A2^<vKzP6;WqzN%vMPBI89aZ15d)z#dixY>rs#0|G!MtE)_1#_?
z85G_<^>u#<=l-3(N9r6ab+t?`F1&cCwS5?g^?-h`<gH+&aM^o&H##jpF4X*JhVnCZ
zI<omM_g-^Qccbo-^8<yf9M^p^gyOBaMtY~1O`2h<+s~Az1Mn%IjvD&2Hv8vAFK%5l
zm(6eM;%nQiK7mr{vd{FJbl=Iy(bdn{h#Z0$Z&F75PyQs6_)|I<Gkm{bNoIJV(gObl
zE$28mganqM;q6j|q2H*G+k~ZJrS{D0OdG!a>uOd44tv@u<1@#TE7<(s>^sBASE79v
zDeDf1%aq7g+Q`)WDWd3bqrbu)EsNue&&7!@$TW}eh)7gsuP~$<9QvhWbqQ5n?eL;K
zYRbl45tj|y-Y$M+a#FW>dt}n4>U%GezOfnh?5rD}^+N8qjl{A1M)^MauC1zQ?YA^0
zy4og2X`{#pSEu{izMb9fyEpKU+0um^g{^At3;(p;I`z!IPdR-)n+xmaB^Q<vi#zod
zRB*;t8#3;i|MSOpg>dvZ&WTU|r&PZ4n~ay)4KRI?{O6C`RvIs>mQVO&>Z;`4>0ZoY
zCZAX?`~U05EAZn}b{EH=cl?+1XL!03yt&{0_2d655AA6DX}HdS!g}9=eyuU^e%aM8
zJvm&>w_ECF@3b6$?ej~TF5TYq&Y^}sb!;7{9{LdL<%LQg>JtC4Y}D1VG}-g6N`IoK
zd)jJJg*eKnhx)Y?#=qIPef^7CM!hJue2(-#f4r2U5h`PNgk#DI<KD&#@zXyJt~CGW
z_cJ~k5|bWfyKVZ-;AxKFr>8Ss@c-+_ax~fhKJxgo{&$t9cC_7pH^$US`R~T~@5cDQ
eV3!#Wm;;tBmOrnYo5h6xST40OySl_N^nU;(ZqPgc

literal 0
HcmV?d00001

diff --git a/format/Layout.md b/format/Layout.md
index 815c47f2c934b..5eaefeebf210a 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -78,7 +78,14 @@ Base requirements
 
 ## Byte Order ([Endianness][3])
 
-The Arrow format is little endian.
+The Arrow format is little endian by default.
+The Schema metadata has an endianness field indicating endianness of RecordBatches.
+Typically this is the endianness of the system where the RecordBatch was generated.
+The main use case is exchanging RecordBatches between systems with the same Endianness.
+At first we will return an error when trying to read a Schema with an endianness
+that does not match the underlying system. The reference implementation is focused on
+Little Endian and provides tests for it. Eventually we may provide automatic conversion
+via byte swapping.
 
 ## Alignment and Padding
 
diff --git a/format/Message.fbs b/format/Message.fbs
index 6a351b9dbf0a6..3f688c156e3ea 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -87,10 +87,21 @@ table Field {
   children: [Field];
 }
 
+/// ----------------------------------------------------------------------
+/// Endianness of the platform that produces the RecordBatch
+
+enum Endianness:int { Little, Big }
+
 /// ----------------------------------------------------------------------
 /// A Schema describes the columns in a row batch
 
 table Schema {
+
+  /// endianness of the buffer
+  /// it is Little Endian by default
+  /// if endianness doesn't match the underlying system then the vectors need to be converted
+  endianness: Endianness=Little;
+
   fields: [Field];
 }
 

From 268e108c2d9101eccf2624fccf1fddf6f7f97b8b Mon Sep 17 00:00:00 2001
From: Jihoon Son <jihoonson@apache.org>
Date: Mon, 15 Aug 2016 22:08:56 -0700
Subject: [PATCH 0111/1644] ARROW-251: Expose APIs for getting code and message
 of the status

Author: Jihoon Son <jihoonson@apache.org>

Closes #114 from jihoonson/ARROW-251 and squashes the following commits:

d1186bf [Jihoon Son] Fix compilation failure
4275c70 [Jihoon Son] Add tests for status
1162084 [Jihoon Son] Merge branch 'master' of https://github.com/apache/arrow into ARROW-251
a76b888 [Jihoon Son] Make code() public and add message()
---
 cpp/src/arrow/util/CMakeLists.txt |  1 +
 cpp/src/arrow/util/status-test.cc | 38 +++++++++++++++++++++++++++++++
 cpp/src/arrow/util/status.h       | 16 +++++++++----
 3 files changed, 51 insertions(+), 4 deletions(-)
 create mode 100644 cpp/src/arrow/util/status-test.cc

diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 4e941fb5f5cec..13c0d7514feef 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -70,3 +70,4 @@ endif()
 ADD_ARROW_TEST(bit-util-test)
 ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(memory-pool-test)
+ADD_ARROW_TEST(status-test)
\ No newline at end of file
diff --git a/cpp/src/arrow/util/status-test.cc b/cpp/src/arrow/util/status-test.cc
new file mode 100644
index 0000000000000..45e0ff361ac22
--- /dev/null
+++ b/cpp/src/arrow/util/status-test.cc
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/util/status.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+TEST(StatusTest, TestCodeAndMessage) {
+  Status ok = Status::OK();
+  ASSERT_EQ(StatusCode::OK, ok.code());
+  Status file_error = Status::IOError("file error");
+  ASSERT_EQ(StatusCode::IOError, file_error.code());
+  ASSERT_EQ("file error", file_error.message());
+}
+
+TEST(StatusTest, TestToString) {
+  Status file_error = Status::IOError("file error");
+  ASSERT_EQ("IOError: file error", file_error.ToString());
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
index 6ba2035bcd3a4..d5585313c728b 100644
--- a/cpp/src/arrow/util/status.h
+++ b/cpp/src/arrow/util/status.h
@@ -138,6 +138,18 @@ class ARROW_EXPORT Status {
   // Get the POSIX code associated with this Status, or -1 if there is none.
   int16_t posix_code() const;
 
+  StatusCode code() const {
+    return ((state_ == NULL) ? StatusCode::OK : static_cast<StatusCode>(state_[4]));
+  }
+
+  std::string message() const {
+    uint32_t length;
+    memcpy(&length, state_, sizeof(length));
+    std::string msg;
+    msg.append((state_ + 7), length);
+    return msg;
+  }
+
  private:
   // OK status has a NULL state_.  Otherwise, state_ is a new[] array
   // of the following form:
@@ -147,10 +159,6 @@ class ARROW_EXPORT Status {
   //    state_[7..]  == message
   const char* state_;
 
-  StatusCode code() const {
-    return ((state_ == NULL) ? StatusCode::OK : static_cast<StatusCode>(state_[4]));
-  }
-
   Status(StatusCode code, const std::string& msg, int16_t posix_code);
   static const char* CopyState(const char* s);
 };

From 246a126b23dc20bca7b665ec76d75ca4a68cd1f1 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <emkornfield@gmail.com>
Date: Mon, 15 Aug 2016 23:04:46 -0700
Subject: [PATCH 0112/1644] ARROW-107: [C++] Implement IPC for structs

Some other changes (I tried to isolate each in there own commit):
1.  Changed NumericTypes to be its own tempated type instead of separate macros (this made debugging easier)
2.  Fix an existing unit test for IPC that row counts inconsistent with row batch size.
3.  Some minor make-format changes.

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #117 from emkornfield/emk_struct_ipc and squashes the following commits:

777e338 [Micah Kornfield] fix formatting
9008046 [Micah Kornfield] use TypeClass::c_type
e46b0d8 [Micah Kornfield] add skip for memory pool test
fc63bff [Micah Kornfield] make lint and make format
9aa972b [Micah Kornfield] change macro to templates instead (makes debugging easier)
3e01e7f [Micah Kornfield] Implement struct round-trip.  Fix unit test for non null to have consistent batch sizes
8eaf1e7 [Micah Kornfield] fix formatting
---
 cpp/src/.clang-tidy-ignore             |  1 +
 cpp/src/arrow/ipc/adapter.cc           | 24 ++++++-
 cpp/src/arrow/ipc/ipc-adapter-test.cc  | 46 ++++++++++--
 cpp/src/arrow/ipc/metadata-internal.cc | 10 +--
 cpp/src/arrow/types/primitive.h        | 97 +++++++++++++-------------
 cpp/src/arrow/util/memory-pool-test.cc |  2 +-
 6 files changed, 115 insertions(+), 65 deletions(-)

diff --git a/cpp/src/.clang-tidy-ignore b/cpp/src/.clang-tidy-ignore
index a128c38889672..5ab4d20d61942 100644
--- a/cpp/src/.clang-tidy-ignore
+++ b/cpp/src/.clang-tidy-ignore
@@ -1 +1,2 @@
 ipc-adapter-test.cc
+memory-pool-test.cc
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 84f7830092cf4..3259980058b8e 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -34,6 +34,7 @@
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
+#include "arrow/types/struct.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/status.h"
@@ -118,8 +119,11 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
     RETURN_NOT_OK(VisitArray(
         list_arr->values().get(), field_nodes, buffers, max_recursion_depth - 1));
   } else if (arr->type_enum() == Type::STRUCT) {
-    // TODO(wesm)
-    return Status::NotImplemented("Struct type");
+    const auto struct_arr = static_cast<const StructArray*>(arr);
+    for (auto& field : struct_arr->fields()) {
+      RETURN_NOT_OK(
+          VisitArray(field.get(), field_nodes, buffers, max_recursion_depth - 1));
+    }
   } else {
     return Status::NotImplemented("Unrecognized type");
   }
@@ -313,6 +317,22 @@ class RowBatchReader::Impl {
       return MakeListArray(type, field_meta.length, offsets, values_array,
           field_meta.null_count, null_bitmap, out);
     }
+
+    if (type->type == Type::STRUCT) {
+      const int num_children = type->num_children();
+      std::vector<ArrayPtr> fields;
+      fields.reserve(num_children);
+      for (int child_idx = 0; child_idx < num_children; ++child_idx) {
+        std::shared_ptr<Array> field_array;
+        RETURN_NOT_OK(NextArray(
+            type->child(child_idx).get(), max_recursion_depth - 1, &field_array));
+        fields.push_back(field_array);
+      }
+      out->reset(new StructArray(
+          type, field_meta.length, fields, field_meta.null_count, null_bitmap));
+      return Status::OK();
+    }
+
     return Status::NotImplemented("Non-primitive types not complete yet");
   }
 
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 2bfb459d6e06a..6740e0fc5acc2 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -32,6 +32,7 @@
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
+#include "arrow/types/struct.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/memory-pool.h"
@@ -205,15 +206,16 @@ Status MakeNonNullRowBatch(std::shared_ptr<RowBatch>* out) {
 
   // Example data
   MemoryPool* pool = default_memory_pool();
-  const int length = 200;
+  const int length = 50;
   std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
 
   RETURN_NOT_OK(MakeRandomInt32Array(1000, true, pool, &leaf_values));
   bool include_nulls = false;
-  RETURN_NOT_OK(MakeRandomListArray(leaf_values, 50, include_nulls, pool, &list_array));
   RETURN_NOT_OK(
-      MakeRandomListArray(list_array, 50, include_nulls, pool, &list_list_array));
-  RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &flat_array));
+      MakeRandomListArray(leaf_values, length, include_nulls, pool, &list_array));
+  RETURN_NOT_OK(
+      MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array));
+  RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array));
   out->reset(new RowBatch(schema, length, {list_array, list_list_array, flat_array}));
   return Status::OK();
 }
@@ -238,10 +240,40 @@ Status MakeDeeplyNestedList(std::shared_ptr<RowBatch>* out) {
   return Status::OK();
 }
 
-INSTANTIATE_TEST_CASE_P(
-    RoundTripTests, TestWriteRowBatch,
+Status MakeStruct(std::shared_ptr<RowBatch>* out) {
+  // reuse constructed list columns
+  std::shared_ptr<RowBatch> list_batch;
+  RETURN_NOT_OK(MakeListRowBatch(&list_batch));
+  std::vector<ArrayPtr> columns = {
+      list_batch->column(0), list_batch->column(1), list_batch->column(2)};
+  auto list_schema = list_batch->schema();
+
+  // Define schema
+  std::shared_ptr<DataType> type(new StructType(
+      {list_schema->field(0), list_schema->field(1), list_schema->field(2)}));
+  auto f0 = std::make_shared<Field>("non_null_struct", type);
+  auto f1 = std::make_shared<Field>("null_struct", type);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  // construct individual nullable/non-nullable struct arrays
+  ArrayPtr no_nulls(new StructArray(type, list_batch->num_rows(), columns));
+  std::vector<uint8_t> null_bytes(list_batch->num_rows(), 1);
+  null_bytes[0] = 0;
+  std::shared_ptr<Buffer> null_bitmask;
+  RETURN_NOT_OK(util::bytes_to_bits(null_bytes, &null_bitmask));
+  ArrayPtr with_nulls(
+      new StructArray(type, list_batch->num_rows(), columns, 1, null_bitmask));
+
+  // construct batch
+  std::vector<ArrayPtr> arrays = {no_nulls, with_nulls};
+  out->reset(new RowBatch(schema, list_batch->num_rows(), arrays));
+  return Status::OK();
+}
+
+INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRowBatch,
     ::testing::Values(&MakeIntRowBatch, &MakeListRowBatch, &MakeNonNullRowBatch,
-        &MakeZeroLengthRowBatch, &MakeDeeplyNestedList, &MakeStringTypesRowBatch));
+                            &MakeZeroLengthRowBatch, &MakeDeeplyNestedList,
+                            &MakeStringTypesRowBatch, &MakeStruct));
 
 void TestGetRowBatchSize(std::shared_ptr<RowBatch> batch) {
   MockMemorySource mock_source(1 << 16);
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 1d3edf0117f91..8cd416ff5853f 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -265,11 +265,8 @@ Status MessageBuilder::SetSchema(const Schema* schema) {
     field_offsets.push_back(offset);
   }
 
-  header_ = flatbuf::CreateSchema(
-                fbb_,
-                endianness(),
-                fbb_.CreateVector(field_offsets))
-                .Union();
+  header_ =
+      flatbuf::CreateSchema(fbb_, endianness(), fbb_.CreateVector(field_offsets)).Union();
   body_length_ = 0;
   return Status::OK();
 }
@@ -278,8 +275,7 @@ Status MessageBuilder::SetRecordBatch(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
     const std::vector<flatbuf::Buffer>& buffers) {
   header_type_ = flatbuf::MessageHeader_RecordBatch;
-  header_ = flatbuf::CreateRecordBatch(fbb_, length,
-                fbb_.CreateVectorOfStructs(nodes),
+  header_ = flatbuf::CreateRecordBatch(fbb_, length, fbb_.CreateVectorOfStructs(nodes),
                 fbb_.CreateVectorOfStructs(buffers))
                 .Union();
   body_length_ = body_length;
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 18f954adc0894..770de765f1fcc 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -53,54 +53,55 @@ class ARROW_EXPORT PrimitiveArray : public Array {
   const uint8_t* raw_data_;
 };
 
-#define NUMERIC_ARRAY_DECL(NAME, TypeClass, T)                                         \
-  class ARROW_EXPORT NAME : public PrimitiveArray {                                    \
-   public:                                                                             \
-    using value_type = T;                                                              \
-                                                                                       \
-    NAME(int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count = 0,  \
-        const std::shared_ptr<Buffer>& null_bitmap = nullptr)                          \
-        : PrimitiveArray(                                                              \
-              std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {} \
-    NAME(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,     \
-        int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)  \
-        : PrimitiveArray(type, length, data, null_count, null_bitmap) {}               \
-                                                                                       \
-    bool EqualsExact(const NAME& other) const {                                        \
-      return PrimitiveArray::EqualsExact(*static_cast<const PrimitiveArray*>(&other)); \
-    }                                                                                  \
-                                                                                       \
-    bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,      \
-        const ArrayPtr& arr) const override {                                          \
-      if (this == arr.get()) { return true; }                                          \
-      if (!arr) { return false; }                                                      \
-      if (this->type_enum() != arr->type_enum()) { return false; }                     \
-      const auto other = static_cast<NAME*>(arr.get());                                \
-      for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {    \
-        const bool is_null = IsNull(i);                                                \
-        if (is_null != arr->IsNull(o_i) ||                                             \
-            (!is_null && Value(i) != other->Value(o_i))) {                             \
-          return false;                                                                \
-        }                                                                              \
-      }                                                                                \
-      return true;                                                                     \
-    }                                                                                  \
-                                                                                       \
-    const T* raw_data() const { return reinterpret_cast<const T*>(raw_data_); }        \
-                                                                                       \
-    T Value(int i) const { return raw_data()[i]; }                                     \
-  };
-
-NUMERIC_ARRAY_DECL(UInt8Array, UInt8Type, uint8_t);
-NUMERIC_ARRAY_DECL(Int8Array, Int8Type, int8_t);
-NUMERIC_ARRAY_DECL(UInt16Array, UInt16Type, uint16_t);
-NUMERIC_ARRAY_DECL(Int16Array, Int16Type, int16_t);
-NUMERIC_ARRAY_DECL(UInt32Array, UInt32Type, uint32_t);
-NUMERIC_ARRAY_DECL(Int32Array, Int32Type, int32_t);
-NUMERIC_ARRAY_DECL(UInt64Array, UInt64Type, uint64_t);
-NUMERIC_ARRAY_DECL(Int64Array, Int64Type, int64_t);
-NUMERIC_ARRAY_DECL(FloatArray, FloatType, float);
-NUMERIC_ARRAY_DECL(DoubleArray, DoubleType, double);
+template <class TypeClass>
+class ARROW_EXPORT NumericArray : public PrimitiveArray {
+ public:
+  using value_type = typename TypeClass::c_type;
+  NumericArray(int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+      : PrimitiveArray(
+            std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {}
+  NumericArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+      : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
+
+  bool EqualsExact(const NumericArray<TypeClass>& other) const {
+    return PrimitiveArray::EqualsExact(*static_cast<const PrimitiveArray*>(&other));
+  }
+
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const ArrayPtr& arr) const override {
+    if (this == arr.get()) { return true; }
+    if (!arr) { return false; }
+    if (this->type_enum() != arr->type_enum()) { return false; }
+    const auto other = static_cast<NumericArray<TypeClass>*>(arr.get());
+    for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+      const bool is_null = IsNull(i);
+      if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
+        return false;
+      }
+    }
+    return true;
+  }
+  const value_type* raw_data() const {
+    return reinterpret_cast<const value_type*>(raw_data_);
+  }
+
+  value_type Value(int i) const { return raw_data()[i]; }
+};
+
+#define NUMERIC_ARRAY_DECL(NAME, TypeClass) using NAME = NumericArray<TypeClass>;
+
+NUMERIC_ARRAY_DECL(UInt8Array, UInt8Type);
+NUMERIC_ARRAY_DECL(Int8Array, Int8Type);
+NUMERIC_ARRAY_DECL(UInt16Array, UInt16Type);
+NUMERIC_ARRAY_DECL(Int16Array, Int16Type);
+NUMERIC_ARRAY_DECL(UInt32Array, UInt32Type);
+NUMERIC_ARRAY_DECL(Int32Array, Int32Type);
+NUMERIC_ARRAY_DECL(UInt64Array, UInt64Type);
+NUMERIC_ARRAY_DECL(Int64Array, Int64Type);
+NUMERIC_ARRAY_DECL(FloatArray, FloatType);
+NUMERIC_ARRAY_DECL(DoubleArray, DoubleType);
 
 template <typename Type>
 class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
index 919f3740982cf..deb7ffd03ba75 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -54,7 +54,7 @@ TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
 
 #ifndef NDEBUG
   EXPECT_EXIT(pool->Free(data, 120), ::testing::ExitedWithCode(1),
-               ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
+      ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
 #endif
 
   pool->Free(data, 100);

From e7e399db5fc6913e67426514279f81766a0778d2 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Tue, 24 May 2016 13:38:09 -0700
Subject: [PATCH 0113/1644] ARROW-259: Use Flatbuffer Field type instead of
 MaterializedField

Remove MaterializedField, MajorType, RepeatedTypes

Add code to convert from FlatBuf representation to Pojo

also adds tests to test the conversion
---
 format/Message.fbs                            |  22 +-
 header                                        |  16 +
 java/format/pom.xml                           | 163 ++++
 java/memory/pom.xml                           |   2 +-
 java/pom.xml                                  |   3 +-
 java/vector/pom.xml                           |   7 +-
 java/vector/src/main/codegen/config.fmpp      |   1 +
 .../src/main/codegen/data/ArrowTypes.tdd      |  80 ++
 .../main/codegen/data/ValueVectorTypes.tdd    |  59 +-
 .../src/main/codegen/includes/vv_imports.ftl  |   4 +
 .../templates/AbstractFieldReader.java        |   8 +-
 .../templates/AbstractFieldWriter.java        |  10 +-
 .../AbstractPromotableFieldWriter.java        |   4 -
 .../src/main/codegen/templates/ArrowType.java | 129 +++
 .../main/codegen/templates/BaseReader.java    |   5 +-
 .../main/codegen/templates/BaseWriter.java    |   3 +-
 .../codegen/templates/BasicTypeHelper.java    | 539 ------------
 .../main/codegen/templates/ComplexCopier.java |  18 +-
 .../codegen/templates/ComplexReaders.java     |  72 +-
 .../codegen/templates/ComplexWriters.java     |  30 +-
 .../codegen/templates/FixedValueVectors.java  |  94 +-
 .../codegen/templates/HolderReaderImpl.java   |  98 +--
 .../main/codegen/templates/ListWriters.java   | 234 -----
 .../main/codegen/templates/MapWriters.java    |  42 +-
 .../main/codegen/templates/NullReader.java    |  23 +-
 .../templates/NullableValueVectors.java       | 104 ++-
 .../templates/RepeatedValueVectors.java       | 421 ---------
 .../codegen/templates/UnionListWriter.java    |  23 +-
 .../main/codegen/templates/UnionReader.java   |  28 +-
 .../main/codegen/templates/UnionVector.java   | 105 ++-
 .../main/codegen/templates/UnionWriter.java   |  16 +-
 .../main/codegen/templates/ValueHolders.java  |  43 +-
 .../templates/VariableLengthVectors.java      |  73 +-
 .../arrow/vector/BaseDataValueVector.java     |   5 +-
 .../apache/arrow/vector/BaseValueVector.java  |  31 +-
 .../org/apache/arrow/vector/BitVector.java    |  43 +-
 .../org/apache/arrow/vector/ObjectVector.java | 220 -----
 .../arrow/vector/ValueHolderHelper.java       | 203 -----
 .../org/apache/arrow/vector/ValueVector.java  |  10 +-
 .../apache/arrow/vector/VectorDescriptor.java |  83 --
 .../org/apache/arrow/vector/ZeroVector.java   |  30 +-
 .../complex/AbstractContainerVector.java      |  49 +-
 .../vector/complex/AbstractMapVector.java     |  47 +-
 .../complex/BaseRepeatedValueVector.java      |  63 +-
 .../vector/complex/ContainerVectorLike.java   |  43 -
 .../arrow/vector/complex/ListVector.java      |  89 +-
 .../arrow/vector/complex/MapVector.java       |  97 +--
 .../vector/complex/RepeatedListVector.java    | 427 ----------
 .../vector/complex/RepeatedMapVector.java     | 584 -------------
 .../vector/complex/RepeatedValueVector.java   |   2 +-
 .../complex/impl/AbstractBaseReader.java      |  19 +-
 .../complex/impl/AbstractBaseWriter.java      |  16 +-
 .../complex/impl/ComplexWriterImpl.java       |  22 +-
 .../vector/complex/impl/PromotableWriter.java |  48 +-
 .../complex/impl/RepeatedListReaderImpl.java  | 145 ----
 .../complex/impl/RepeatedMapReaderImpl.java   | 192 -----
 .../impl/SingleLikeRepeatedMapReaderImpl.java |  89 --
 .../complex/impl/SingleListReaderImpl.java    |  14 +-
 .../complex/impl/SingleMapReaderImpl.java     |  10 +-
 .../vector/complex/impl/UnionListReader.java  |  19 +-
 .../arrow/vector/holders/ObjectHolder.java    |  38 -
 .../arrow/vector/holders/UnionHolder.java     |   7 +-
 .../arrow/vector/types/MaterializedField.java | 217 -----
 .../org/apache/arrow/vector/types/Types.java  | 596 ++++++++++---
 .../apache/arrow/vector/types/pojo/Field.java | 105 +++
 .../arrow/vector/types/pojo/Schema.java       |  74 ++
 .../vector/util/ByteFunctionHelpers.java      |  50 --
 .../arrow/vector/util/CoreDecimalUtility.java |  91 --
 .../arrow/vector/util/DecimalUtility.java     | 802 +++++++++---------
 .../arrow/vector/util/MapWithOrdinal.java     |  12 +
 .../arrow/vector/TestDecimalVector.java       |  63 ++
 ...TestOversizedAllocationForValueVector.java |  11 +-
 .../apache/arrow/vector/TestUnionVector.java  |   5 +-
 .../apache/arrow/vector/TestValueVector.java  | 137 +--
 .../complex/impl/TestPromotableWriter.java    |   7 +-
 .../complex/writer/TestComplexWriter.java     | 270 ++++++
 .../apache/arrow/vector/pojo/TestConvert.java |  80 ++
 77 files changed, 2464 insertions(+), 5180 deletions(-)
 create mode 100644 header
 create mode 100644 java/format/pom.xml
 create mode 100644 java/vector/src/main/codegen/data/ArrowTypes.tdd
 create mode 100644 java/vector/src/main/codegen/templates/ArrowType.java
 delete mode 100644 java/vector/src/main/codegen/templates/BasicTypeHelper.java
 delete mode 100644 java/vector/src/main/codegen/templates/ListWriters.java
 delete mode 100644 java/vector/src/main/codegen/templates/RepeatedValueVectors.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java

diff --git a/format/Message.fbs b/format/Message.fbs
index 3f688c156e3ea..2928207db8cc0 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -1,10 +1,13 @@
-namespace apache.arrow.flatbuf;
+namespace org.apache.arrow.flatbuf;
 
 /// ----------------------------------------------------------------------
 /// Logical types and their metadata (if any)
 ///
 /// These are stored in the flatbuffer in the Type union below
 
+table Null {
+}
+
 /// A Tuple in the flatbuffer metadata is the same as an Arrow Struct
 /// (according to the physical memory layout). We used Tuple here as Struct is
 /// a reserved word in Flatbuffers
@@ -45,10 +48,22 @@ table Decimal {
   scale: int;
 }
 
+table Date {
+}
+
+table Time {
+}
+
 table Timestamp {
   timezone: string;
 }
 
+table IntervalDay {
+}
+
+table IntervalYear {
+}
+
 table JSONScalar {
   dense:bool=true;
 }
@@ -58,13 +73,18 @@ table JSONScalar {
 /// add new logical types to Type without breaking backwards compatibility
 
 union Type {
+  Null,
   Int,
   FloatingPoint,
   Binary,
   Utf8,
   Bool,
   Decimal,
+  Date,
+  Time,
   Timestamp,
+  IntervalDay,
+  IntervalYear,
   List,
   Tuple,
   Union,
diff --git a/header b/header
new file mode 100644
index 0000000000000..70665d1a26295
--- /dev/null
+++ b/header
@@ -0,0 +1,16 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
diff --git a/java/format/pom.xml b/java/format/pom.xml
new file mode 100644
index 0000000000000..ea27a3072bc9e
--- /dev/null
+++ b/java/format/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+  license agreements. See the NOTICE file distributed with this work for additional 
+  information regarding copyright ownership. The ASF licenses this file to 
+  You under the Apache License, Version 2.0 (the "License"); you may not use 
+  this file except in compliance with the License. You may obtain a copy of 
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+  by applicable law or agreed to in writing, software distributed under the 
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+  OF ANY KIND, either express or implied. See the License for the specific 
+  language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<modelVersion>4.0.0</modelVersion>
+
+<parent>
+  <artifactId>arrow-java-root</artifactId>
+  <groupId>org.apache.arrow</groupId>
+  <version>0.1-decimal</version>
+</parent>
+
+<artifactId>arrow-format</artifactId>
+<packaging>jar</packaging>
+<name>Arrow Format</name>
+
+  <properties>
+    <fbs.version>1.2.0-3f79e055</fbs.version>
+    <maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
+    <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
+    <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.vlkan</groupId>
+      <artifactId>flatbuffers</artifactId>
+      <version>${fbs.version}</version>
+    </dependency>
+  </dependencies>
+
+ <build>
+  <extensions>
+    <!-- provides os.detected.classifier (i.e. linux-x86_64, osx-x86_64) property -->
+    <extension>
+      <groupId>kr.motd.maven</groupId>
+      <artifactId>os-maven-plugin</artifactId>
+      <version>${os-maven-plugin.version}</version>
+    </extension>
+  </extensions>
+
+  <plugins>
+    <plugin>
+      <groupId>org.apache.maven.plugins</groupId>
+      <artifactId>maven-dependency-plugin</artifactId>
+      <version>${maven-dependency-plugin.version}</version>
+      <executions>
+        <execution>
+          <id>copy-flatc</id>
+          <phase>generate-sources</phase>
+          <goals>
+            <goal>copy</goal>
+          </goals>
+          <configuration>
+            <artifactItems>
+              <artifactItem>
+                <groupId>com.vlkan</groupId>
+                <artifactId>flatc-${os.detected.classifier}</artifactId>
+                <version>${fbs.version}</version>
+                <type>exe</type>
+                <overWrite>true</overWrite>
+                <outputDirectory>${project.build.directory}</outputDirectory>
+              </artifactItem>
+            </artifactItems>
+          </configuration>
+        </execution>
+      </executions>
+    </plugin>
+    <plugin>
+      <groupId>org.codehaus.mojo</groupId>
+      <artifactId>exec-maven-plugin</artifactId>
+      <version>1.4.0</version>
+      <executions>
+        <execution>
+          <id>script-chmod</id>
+          <goals>
+            <goal>exec</goal>
+          </goals>
+          <phase>generate-sources</phase>
+          <configuration>
+            <executable>chmod</executable>
+            <arguments>
+              <argument>+x</argument>
+              <argument>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</argument>
+            </arguments>
+          </configuration>
+        </execution>
+        <execution>
+          <goals>
+            <goal>exec</goal>
+          </goals>
+          <phase>generate-sources</phase>
+          <configuration>
+            <executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</executable>
+            <arguments>
+              <argument>-j</argument>
+              <argument>-o</argument>
+              <argument>target/generated-sources/</argument>
+              <argument>../../format/Message.fbs</argument>
+            </arguments>
+          </configuration>
+        </execution>
+      </executions>
+    </plugin>
+    <plugin>
+      <groupId>com.mycila</groupId>
+      <artifactId>license-maven-plugin</artifactId>
+      <version>2.3</version>
+      <configuration>
+        <header>${basedir}/../../header</header>
+        <includes>
+          <include>**/*.java</include>
+        </includes>
+      </configuration>
+      <executions>
+        <execution>
+          <phase>process-sources</phase>
+          <goals>
+            <goal>format</goal>
+          </goals>
+        </execution>
+      </executions>
+    </plugin>
+    <plugin>
+      <groupId>org.codehaus.mojo</groupId>
+      <artifactId>build-helper-maven-plugin</artifactId>
+      <version>1.9.1</version>
+      <executions>
+        <execution>
+          <id>add-sources-as-resources</id>
+          <phase>generate-sources</phase>
+          <goals>
+            <goal>add-source</goal>
+          </goals>
+          <configuration>
+            <sources>
+              <source>${project.build.directory}/generated-sources</source>
+            </sources>
+          </configuration>
+        </execution>
+      </executions>
+    </plugin>
+    <plugin>
+      <groupId>org.apache.maven.plugins</groupId>
+      <artifactId>maven-checkstyle-plugin</artifactId>
+      <configuration>
+        <skip>true</skip>
+      </configuration>
+    </plugin>
+  </plugins>
+
+</build>
+</project>
+
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 44332f5ed14a8..12ff4c81d86c0 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -15,7 +15,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1-SNAPSHOT</version>
+    <version>0.1-decimal</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>arrow-memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 71f59caf2798e..92ab109f939e1 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -21,7 +21,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.1-SNAPSHOT</version>
+  <version>0.1-decimal</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -465,6 +465,7 @@
   </dependencies>
 
   <modules>
+    <module>format</module>
     <module>memory</module>
     <module>vector</module>
   </modules>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index df5389261ba57..fac788cef14d9 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -15,13 +15,18 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1-SNAPSHOT</version>
+    <version>0.1-decimal</version>
   </parent>
   <artifactId>vector</artifactId>
   <name>vectors</name>
 
   <dependencies>
 
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-format</artifactId>
+      <version>0.1-decimal</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory</artifactId>
diff --git a/java/vector/src/main/codegen/config.fmpp b/java/vector/src/main/codegen/config.fmpp
index 663677cbb5a76..6d92ba830ee2c 100644
--- a/java/vector/src/main/codegen/config.fmpp
+++ b/java/vector/src/main/codegen/config.fmpp
@@ -17,6 +17,7 @@
 data: {
     # TODO:  Rename to ~valueVectorModesAndTypes for clarity.
     vv:                       tdd(../data/ValueVectorTypes.tdd),
+    arrowTypes:               tdd(../data/ArrowTypes.tdd)
 
 }
 freemarkerLinks: {
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
new file mode 100644
index 0000000000000..4ab7f8562f907
--- /dev/null
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  types: [
+    {
+      name: "Null",
+      fields: []
+    },
+    {
+      name: "Tuple",
+      fields: []
+    },
+    {
+      name: "List",
+      fields: []
+    },
+    {
+      name: "Union",
+      fields: []
+    },
+    {
+      name: "Int",
+      fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
+    },
+    {
+      name: "FloatingPoint",
+      fields: [{name: precision, type: int}]
+    },
+    {
+      name: "Utf8",
+      fields: []
+    },
+    {
+      name: "Binary",
+      fields: []
+    },
+    {
+      name: "Bool",
+      fields: []
+    },
+    {
+      name: "Decimal",
+      fields: [{name: "precision", type: int}, {name: "scale", type: int}]
+    },
+    {
+      name: "Date",
+      fields: []
+    },
+    {
+      name: "Time",
+      fields: []
+    },
+    {
+      name: "Timestamp",
+      fields: [{name: "timezone", type: "String"}]
+    },
+    {
+      name: "IntervalDay",
+      fields: []
+    },
+    {
+      name: "IntervalYear",
+      fields: []
+    }
+  ]
+}
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index e747c30c5d1cb..421dd7ef92e63 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -17,8 +17,7 @@
 {
   modes: [
     {name: "Optional", prefix: "Nullable"},
-    {name: "Required", prefix: ""},
-    {name: "Repeated", prefix: "Repeated"}
+    {name: "Required", prefix: ""}
     ],
   types: [
     {
@@ -61,9 +60,8 @@
         { class: "Int", valueHolder: "IntHolder"},
         { class: "UInt4", valueHolder: "UInt4Holder" },
         { class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]},
-        { class: "Time", javaType: "int", friendlyType: "DateTime" },
         { class: "IntervalYear", javaType: "int", friendlyType: "Period" }
-        { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal", fields: [{name:"value", type:"int"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
+        { class: "Time", javaType: "int", friendlyType: "DateTime" }
       ]
     },
     {
@@ -78,15 +76,11 @@
         { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
         { class: "Date", javaType: "long", friendlyType: "DateTime" },
         { class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
-        { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal", fields: [{name:"value", type:"long"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
-        <#--
-        { class: "Money", maxPrecisionDigits: 2, scale: 1, },
-        -->
       ]
     },
     {
       major: "Fixed",
-      width: 12,
+      width: 8,
       javaType: "ArrowBuf",
       boxedType: "ArrowBuf",
       minor: [
@@ -96,51 +90,11 @@
     {
       major: "Fixed",
       width: 16,
-      javaType: "ArrowBuf"
-      boxedType: "ArrowBuf",      
-      minor: [
-        { class: "Interval", daysOffset: 4, millisecondsOffset: 8, friendlyType: "Period", fields: [ {name: "months", type: "int"}, {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
-      ]
-    },
-    {
-      major: "Fixed",
-      width: 12,
-      javaType: "ArrowBuf",
-      boxedType: "ArrowBuf",
-      minor: [
-        <#--
-        { class: "TimeTZ" },
-        { class: "Interval" }
-        -->
-        { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
-      ]
-    },
-    {
-      major: "Fixed",
-      width: 16,
-      javaType: "ArrowBuf",
-      boxedType: "ArrowBuf",
-      
-      minor: [
-        { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
-      ]
-    },
-    {
-      major: "Fixed",
-      width: 24,
-      javaType: "ArrowBuf",
-      boxedType: "ArrowBuf",
-      minor: [
-        { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
-      ]
-    },
-    {
-      major: "Fixed",
-      width: 20,
       javaType: "ArrowBuf",
       boxedType: "ArrowBuf",
+
       minor: [
-        { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+        { class: "Decimal", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
       ]
     },
     {
@@ -151,8 +105,7 @@
       fields: [{name: "start", type: "int"}, {name: "end", type: "int"}, {name: "buffer", type: "ArrowBuf"}],
       minor: [
         { class: "VarBinary" , friendlyType: "byte[]" },
-        { class: "VarChar" , friendlyType: "Text" },
-        { class: "Var16Char" , friendlyType: "String" }
+        { class: "VarChar" , friendlyType: "Text" }
       ]
     },
     {
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
index 2d808b1b3cb3f..9b4b79bfd7b90 100644
--- a/java/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -17,6 +17,8 @@ import com.google.common.collect.ObjectArrays;
 import com.google.common.base.Charsets;
 import com.google.common.collect.ObjectArrays;
 
+import com.google.flatbuffers.FlatBufferBuilder;
+
 import com.google.common.base.Preconditions;
 import io.netty.buffer.*;
 
@@ -25,6 +27,8 @@ import org.apache.commons.lang3.ArrayUtils;
 import org.apache.arrow.memory.*;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.*;
+import org.apache.arrow.vector.types.pojo.*;
+import org.apache.arrow.vector.types.pojo.ArrowType.*;
 import org.apache.arrow.vector.types.*;
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.holders.*;
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
index b83dba2879190..e0d0fc9715ba2 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -41,7 +41,13 @@ public boolean isSet() {
     return true;
   }
 
-  <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
+  @Override
+  public Field getField() {
+    fail("getField");
+    return null;
+  }
+
+  <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
           "Character", "DateTime", "Period", "Double", "Float",
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 6ee9dad44e929..de076fc46ffb2 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -31,10 +31,6 @@
  */
 @SuppressWarnings("unused")
 abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter {
-  AbstractFieldWriter(FieldWriter parent) {
-    super(parent);
-  }
-
   @Override
   public void start() {
     throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
@@ -62,9 +58,15 @@ public void write(${name}Holder holder) {
     fail("${name}");
   }
 
+  <#if minor.class == "Decimal">
+  public void writeDecimal(int start, ArrowBuf buffer) {
+    fail("${name}");
+  }
+  <#else>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
     fail("${name}");
   }
+  </#if>
 
   </#list></#list>
 
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 549dbf107ea67..7e60320cfb8ac 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -37,10 +37,6 @@
  */
 @SuppressWarnings("unused")
 abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
-  AbstractPromotableFieldWriter(FieldWriter parent) {
-    super(parent);
-  }
-
   /**
    * Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type
    * @param type
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
new file mode 100644
index 0000000000000..6dfaf216ad042
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.arrow.flatbuf.Field;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+
+import java.util.Objects;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/types/pojo/ArrowType.java" />
+
+
+<#include "/@includes/license.ftl" />
+package org.apache.arrow.vector.types.pojo;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Type;
+
+import java.util.Objects;
+
+public abstract class ArrowType {
+
+  public abstract byte getTypeType();
+  public abstract int getType(FlatBufferBuilder builder);
+
+
+  <#list arrowTypes.types as type>
+  <#assign name = type.name>
+  <#assign fields = type.fields>
+  public static class ${name} extends ArrowType {
+    public static final byte TYPE_TYPE = Type.${name};
+    <#if type.fields?size == 0>
+    public static final ${name} INSTANCE = new ${name}();
+    </#if>
+
+    <#list fields as field>
+    <#assign fieldName = field.name>
+    <#assign fieldType = field.type>
+    ${fieldType} ${fieldName};
+    </#list>
+
+    <#if type.fields?size != 0>
+    public ${type.name}(<#list type.fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+      <#list type.fields as field>
+      this.${field.name} = ${field.name};
+      </#list>
+    }
+    </#if>
+
+    @Override
+    public byte getTypeType() {
+      return TYPE_TYPE;
+    }
+
+    @Override
+    public int getType(FlatBufferBuilder builder) {
+      org.apache.arrow.flatbuf.${type.name}.start${type.name}(builder);
+      <#list type.fields as field>
+      org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, <#if field.type == "String">builder.createString(${field.name})<#else>${field.name}</#if>);
+      </#list>
+      return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
+    }
+
+    <#list fields as field>
+    public ${field.type} get${field.name?cap_first}() {
+      return ${field.name};
+    }
+    </#list>
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (!(obj instanceof ${type.name})) {
+        return false;
+      }
+      <#if type.fields?size == 0>
+      return true;
+      <#else>
+      ${type.name} that = (${type.name}) obj;
+      return
+      <#list type.fields as field>Objects.equals(this.${field.name}, that.${field.name}) <#if field_has_next>&&<#else>;</#if>
+      </#list>
+      </#if>
+    }
+  }
+  </#list>
+
+  public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.apache.arrow.flatbuf.Field field) {
+    switch(field.typeType()) {
+    <#list arrowTypes.types as type>
+    <#assign name = type.name>
+    <#assign nameLower = type.name?lower_case>
+    <#assign fields = type.fields>
+    case Type.${type.name}:
+      org.apache.arrow.flatbuf.${type.name} ${nameLower}Type = (org.apache.arrow.flatbuf.${type.name}) field.type(new org.apache.arrow.flatbuf.${type.name}());
+      return new ${type.name}(<#list type.fields as field>${nameLower}Type.${field.name}()<#if field_has_next>, </#if></#list>);
+    </#list>
+    default:
+      throw new UnsupportedOperationException("Unsupported type: " + field.typeType());
+    }
+  }
+
+  public static Int getInt(org.apache.arrow.flatbuf.Field field) {
+    org.apache.arrow.flatbuf.Int intType = (org.apache.arrow.flatbuf.Int) field.type(new org.apache.arrow.flatbuf.Int());
+    return new Int(intType.bitWidth(), intType.isSigned());
+  }
+}
+
+
diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java
index 8f12b1da80424..72fea58d0bc9e 100644
--- a/java/vector/src/main/codegen/templates/BaseReader.java
+++ b/java/vector/src/main/codegen/templates/BaseReader.java
@@ -30,8 +30,8 @@
 
 @SuppressWarnings("unused")
 public interface BaseReader extends Positionable{
-  MajorType getType();
-  MaterializedField getField();
+  Field getField();
+  MinorType getMinorType();
   void reset();
   void read(UnionHolder holder);
   void read(int index, UnionHolder holder);
@@ -60,7 +60,6 @@ public interface RepeatedListReader extends ListReader{
   
   public interface ScalarReader extends  
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, </#list></#list> 
-  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> Repeated${name}Reader, </#list></#list>
   BaseReader {}
   
   interface ComplexReader{
diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java
index 299b2389bb35c..08bd39eae2358 100644
--- a/java/vector/src/main/codegen/templates/BaseWriter.java
+++ b/java/vector/src/main/codegen/templates/BaseWriter.java
@@ -31,12 +31,11 @@
  */
 @SuppressWarnings("unused")
   public interface BaseWriter extends AutoCloseable, Positionable {
-  FieldWriter getParent();
   int getValueCapacity();
 
   public interface MapWriter extends BaseWriter {
 
-    MaterializedField getField();
+    Field getField();
 
     /**
      * Whether this writer is a map writer and is empty (has no children).
diff --git a/java/vector/src/main/codegen/templates/BasicTypeHelper.java b/java/vector/src/main/codegen/templates/BasicTypeHelper.java
deleted file mode 100644
index 0bae715e35283..0000000000000
--- a/java/vector/src/main/codegen/templates/BasicTypeHelper.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/arrow/vector/util/BasicTypeHelper.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector.util;
-
-<#include "/@includes/vv_imports.ftl" />
-import org.apache.arrow.vector.complex.UnionVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.util.CallBack;
-
-public class BasicTypeHelper {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicTypeHelper.class);
-
-  private static final int WIDTH_ESTIMATE = 50;
-
-  // Default length when casting to varchar : 65536 = 2^16
-  // This only defines an absolute maximum for values, setting
-  // a high value like this will not inflate the size for small values
-  public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
-
-  protected static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
-    return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
-  }
-
-  protected static String buildErrorMessage(final String operation, final MajorType type) {
-    return buildErrorMessage(operation, type.getMinorType(), type.getMode());
-  }
-
-  public static int getSize(MajorType major) {
-    switch (major.getMinorType()) {
-<#list vv.types as type>
-  <#list type.minor as minor>
-    case ${minor.class?upper_case}:
-      return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
-                               minor.class?substring(0, 3) == "PRO" ||
-                               minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
-  </#list>
-</#list>
-//      case FIXEDCHAR: return major.getWidth();
-//      case FIXED16CHAR: return major.getWidth();
-//      case FIXEDBINARY: return major.getWidth();
-    }
-    throw new UnsupportedOperationException(buildErrorMessage("get size", major));
-  }
-
-  public static ValueVector getNewVector(String name, BufferAllocator allocator, MajorType type, CallBack callback){
-    MaterializedField field = MaterializedField.create(name, type);
-    return getNewVector(field, allocator, callback);
-  }
-  
-  
-  public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
-    switch (type) {
-    case UNION:
-      return UnionVector.class;
-    case MAP:
-      switch (mode) {
-      case OPTIONAL:
-      case REQUIRED:
-        return MapVector.class;
-      case REPEATED:
-        return RepeatedMapVector.class;
-      }
-      
-    case LIST:
-      switch (mode) {
-      case REPEATED:
-        return RepeatedListVector.class;
-      case REQUIRED:
-      case OPTIONAL:
-        return ListVector.class;
-      }
-    
-<#list vv.types as type>
-  <#list type.minor as minor>
-      case ${minor.class?upper_case}:
-        switch (mode) {
-          case REQUIRED:
-            return ${minor.class}Vector.class;
-          case OPTIONAL:
-            return Nullable${minor.class}Vector.class;
-          case REPEATED:
-            return Repeated${minor.class}Vector.class;
-        }
-  </#list>
-</#list>
-    case GENERIC_OBJECT      :
-      return ObjectVector.class  ;
-    default:
-      break;
-    }
-    throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
-  }
-  public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
-    switch (type) {
-    case MAP:
-      switch (mode) {
-      case REQUIRED:
-        if (!isSingularRepeated)
-          return SingleMapReaderImpl.class;
-        else
-          return SingleLikeRepeatedMapReaderImpl.class;
-      case REPEATED: 
-          return RepeatedMapReaderImpl.class;
-      }
-    case LIST:
-      switch (mode) {
-      case REQUIRED:
-        return SingleListReaderImpl.class;
-      case REPEATED:
-        return RepeatedListReaderImpl.class;
-      }
-      
-<#list vv.types as type>
-  <#list type.minor as minor>
-      case ${minor.class?upper_case}:
-        switch (mode) {
-          case REQUIRED:
-            return ${minor.class}ReaderImpl.class;
-          case OPTIONAL:
-            return Nullable${minor.class}ReaderImpl.class;
-          case REPEATED:
-            return Repeated${minor.class}ReaderImpl.class;
-        }
-  </#list>
-</#list>
-      default:
-        break;
-      }
-      throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
-  }
-  
-  public static Class<?> getWriterInterface( MinorType type, DataMode mode){
-    switch (type) {
-    case UNION: return UnionWriter.class;
-    case MAP: return MapWriter.class;
-    case LIST: return ListWriter.class;
-<#list vv.types as type>
-  <#list type.minor as minor>
-      case ${minor.class?upper_case}: return ${minor.class}Writer.class;
-  </#list>
-</#list>
-      default:
-        break;
-      }
-      throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
-  }
-  
-  public static Class<?> getWriterImpl( MinorType type, DataMode mode){
-    switch (type) {
-    case UNION:
-      return UnionWriter.class;
-    case MAP:
-      switch (mode) {
-      case REQUIRED:
-      case OPTIONAL:
-        return SingleMapWriter.class;
-      case REPEATED:
-        return RepeatedMapWriter.class;
-      }
-    case LIST:
-      switch (mode) {
-      case REQUIRED:
-      case OPTIONAL:
-        return UnionListWriter.class;
-      case REPEATED:
-        return RepeatedListWriter.class;
-      }
-      
-<#list vv.types as type>
-  <#list type.minor as minor>
-      case ${minor.class?upper_case}:
-        switch (mode) {
-          case REQUIRED:
-            return ${minor.class}WriterImpl.class;
-          case OPTIONAL:
-            return Nullable${minor.class}WriterImpl.class;
-          case REPEATED:
-            return Repeated${minor.class}WriterImpl.class;
-        }
-  </#list>
-</#list>
-      default:
-        break;
-      }
-      throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
-  }
-
-  public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
-    switch (type) {      
-<#list vv.types as type>
-  <#list type.minor as minor>
-      case ${minor.class?upper_case}:
-        switch (mode) {
-          case REQUIRED:
-            return ${minor.class}HolderReaderImpl.class;
-          case OPTIONAL:
-            return Nullable${minor.class}HolderReaderImpl.class;
-          case REPEATED:
-            return Repeated${minor.class}HolderReaderImpl.class;
-        }
-  </#list>
-</#list>
-      default:
-        break;
-      }
-      throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
-  }
-  
-  public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
-    return getNewVector(field, allocator, null);
-  }
-  public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
-    field = field.clone();
-    MajorType type = field.getType();
-
-    switch (type.getMinorType()) {
-    
-    case UNION:
-      return new UnionVector(field, allocator, callBack);
-
-    case MAP:
-      switch (type.getMode()) {
-      case REQUIRED:
-      case OPTIONAL:
-        return new MapVector(field, allocator, callBack);
-      case REPEATED:
-        return new RepeatedMapVector(field, allocator, callBack);
-      }
-    case LIST:
-      switch (type.getMode()) {
-      case REPEATED:
-        return new RepeatedListVector(field, allocator, callBack);
-      case OPTIONAL:
-      case REQUIRED:
-        return new ListVector(field, allocator, callBack);
-      }
-<#list vv.  types as type>
-  <#list type.minor as minor>
-    case ${minor.class?upper_case}:
-      switch (type.getMode()) {
-        case REQUIRED:
-          return new ${minor.class}Vector(field, allocator);
-        case OPTIONAL:
-          return new Nullable${minor.class}Vector(field, allocator);
-        case REPEATED:
-          return new Repeated${minor.class}Vector(field, allocator);
-      }
-  </#list>
-</#list>
-    case GENERIC_OBJECT:
-      return new ObjectVector(field, allocator)        ;
-    default:
-      break;
-    }
-    // All ValueVector types have been handled.
-    throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
-  }
-
-  public static ValueHolder getValue(ValueVector vector, int index) {
-    MajorType type = vector.getField().getType();
-    ValueHolder holder;
-    switch(type.getMinorType()) {
-<#list vv.types as type>
-  <#list type.minor as minor>
-    case ${minor.class?upper_case} :
-      <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
-        minor.class?starts_with("Decimal28") ||  minor.class?starts_with("Decimal38")>
-        switch (type.getMode()) {
-          case REQUIRED:
-            holder = new ${minor.class}Holder();
-            ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
-            return holder;
-          case OPTIONAL:
-            holder = new Nullable${minor.class}Holder();
-            ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
-            if (((Nullable${minor.class}Holder)holder).isSet == 1) {
-              ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
-            }
-            return holder;
-        }
-      <#else>
-      switch (type.getMode()) {
-        case REQUIRED:
-          holder = new ${minor.class}Holder();
-          ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
-          return holder;
-        case OPTIONAL:
-          holder = new Nullable${minor.class}Holder();
-          ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
-          if (((Nullable${minor.class}Holder)holder).isSet == 1) {
-            ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
-          }
-          return holder;
-      }
-      </#if>
-  </#list>
-</#list>
-    case GENERIC_OBJECT:
-      holder = new ObjectHolder();
-      ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index)         ;
-      break;
-    }
-
-    throw new UnsupportedOperationException(buildErrorMessage("get value", type));
-  }
-
-  public static void setValue(ValueVector vector, int index, ValueHolder holder) {
-    MajorType type = vector.getField().getType();
-
-    switch(type.getMinorType()) {
-<#list vv.types as type>
-  <#list type.minor as minor>
-    case ${minor.class?upper_case} :
-      switch (type.getMode()) {
-        case REQUIRED:
-          ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
-          return;
-        case OPTIONAL:
-          if (((Nullable${minor.class}Holder) holder).isSet == 1) {
-            ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
-          }
-          return;
-      }
-  </#list>
-</#list>
-    case GENERIC_OBJECT:
-      ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
-      return;
-    default:
-      throw new UnsupportedOperationException(buildErrorMessage("set value", type));
-    }
-  }
-
-  public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
-    MajorType type = vector.getField().getType();
-
-    switch(type.getMinorType()) {
-      <#list vv.types as type>
-      <#list type.minor as minor>
-      case ${minor.class?upper_case} :
-      switch (type.getMode()) {
-        case REQUIRED:
-          ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
-          return;
-        case OPTIONAL:
-          if (((Nullable${minor.class}Holder) holder).isSet == 1) {
-            ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
-          } else {
-            ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
-          }
-          return;
-      }
-      </#list>
-      </#list>
-      case GENERIC_OBJECT:
-        ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
-      default:
-        throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
-    }
-  }
-
-  public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
-    MajorType type1 = v1.getField().getType();
-    MajorType type2 = v2.getField().getType();
-
-    if (type1.getMinorType() != type2.getMinorType()) {
-      return false;
-    }
-
-    switch(type1.getMinorType()) {
-<#list vv.types as type>
-  <#list type.minor as minor>
-    case ${minor.class?upper_case} :
-      if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) == 
-           ((${minor.class}Vector) v2).getAccessor().get(v2index) ) 
-        return true;
-      break;
-  </#list>
-</#list>
-    default:
-      break;
-    }
-    return false;
-  }
-
-  /**
-   *  Create a ValueHolder of MajorType.
-   * @param type
-   * @return
-   */
-  public static ValueHolder createValueHolder(MajorType type) {
-    switch(type.getMinorType()) {
-      <#list vv.types as type>
-      <#list type.minor as minor>
-      case ${minor.class?upper_case} :
-
-        switch (type.getMode()) {
-          case REQUIRED:
-            return new ${minor.class}Holder();
-          case OPTIONAL:
-            return new Nullable${minor.class}Holder();
-          case REPEATED:
-            return new Repeated${minor.class}Holder();
-        }
-      </#list>
-      </#list>
-      case GENERIC_OBJECT:
-        return new ObjectHolder();
-      default:
-        throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
-    }
-  }
-
-  public static boolean isNull(ValueHolder holder) {
-    MajorType type = getValueHolderType(holder);
-
-    switch(type.getMinorType()) {
-      <#list vv.types as type>
-      <#list type.minor as minor>
-      case ${minor.class?upper_case} :
-
-      switch (type.getMode()) {
-        case REQUIRED:
-          return true;
-        case OPTIONAL:
-          return ((Nullable${minor.class}Holder) holder).isSet == 0;
-        case REPEATED:
-          return true;
-      }
-      </#list>
-      </#list>
-      default:
-        throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
-    }
-  }
-
-  public static ValueHolder deNullify(ValueHolder holder) {
-    MajorType type = getValueHolderType(holder);
-
-    switch(type.getMinorType()) {
-      <#list vv.types as type>
-      <#list type.minor as minor>
-      case ${minor.class?upper_case} :
-
-        switch (type.getMode()) {
-          case REQUIRED:
-            return holder;
-          case OPTIONAL:
-            if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
-              ${minor.class}Holder newHolder = new ${minor.class}Holder();
-
-              <#assign fields = minor.fields!type.fields />
-              <#list fields as field>
-              newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
-              </#list>
-
-              return newHolder;
-            } else {
-              throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
-            }
-          case REPEATED:
-            return holder;
-        }
-      </#list>
-      </#list>
-      default:
-        throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
-    }
-  }
-
-  public static ValueHolder nullify(ValueHolder holder) {
-    MajorType type = getValueHolderType(holder);
-
-    switch(type.getMinorType()) {
-      <#list vv.types as type>
-      <#list type.minor as minor>
-      case ${minor.class?upper_case} :
-        switch (type.getMode()) {
-          case REQUIRED:
-            Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
-            newHolder.isSet = 1;
-            <#assign fields = minor.fields!type.fields />
-            <#list fields as field>
-            newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
-            </#list>
-            return newHolder;
-          case OPTIONAL:
-            return holder;
-          case REPEATED:
-            throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
-        }
-      </#list>
-      </#list>
-      default:
-        throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
-    }
-  }
-
-  public static MajorType getValueHolderType(ValueHolder holder) {
-
-    if (0 == 1) {
-      return null;
-    }
-    <#list vv.types as type>
-    <#list type.minor as minor>
-      else if (holder instanceof ${minor.class}Holder) {
-        return ((${minor.class}Holder) holder).TYPE;
-      } else if (holder instanceof Nullable${minor.class}Holder) {
-      return ((Nullable${minor.class}Holder) holder).TYPE;
-      }
-    </#list>
-    </#list>
-
-    throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
-
-  }
-
-}
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index 3614231c8342e..a5756a47ad785 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -42,13 +42,7 @@ public static void copy(FieldReader input, FieldWriter output) {
   }
 
   private static void writeValue(FieldReader reader, FieldWriter writer) {
-    final DataMode m = reader.getType().getMode();
-    final MinorType mt = reader.getType().getMinorType();
-
-    switch(m){
-    case OPTIONAL:
-    case REQUIRED:
-
+    final MinorType mt = reader.getMinorType();
 
       switch (mt) {
 
@@ -89,12 +83,10 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
   </#if>
   </#list></#list>
       }
-              break;
-    }
  }
 
   private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
-    switch (reader.getType().getMinorType()) {
+    switch (reader.getMinorType()) {
     <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
     <#assign fields = minor.fields!type.fields />
     <#assign uncappedName = name?uncap_first/>
@@ -108,12 +100,12 @@ private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter w
     case LIST:
       return (FieldWriter) writer.list(name);
     default:
-      throw new UnsupportedOperationException(reader.getType().toString());
+      throw new UnsupportedOperationException(reader.getMinorType().toString());
     }
   }
 
   private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) {
-    switch (reader.getType().getMinorType()) {
+    switch (reader.getMinorType()) {
     <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
     <#assign fields = minor.fields!type.fields />
     <#assign uncappedName = name?uncap_first/>
@@ -127,7 +119,7 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter
     case LIST:
       return (FieldWriter) writer.list();
     default:
-      throw new UnsupportedOperationException(reader.getType().toString());
+      throw new UnsupportedOperationException(reader.getMinorType().toString());
     }
   }
 }
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 34c657126015e..74a19a605e21e 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -27,10 +27,10 @@
 <@pp.dropOutputFile />
 <#list vv.types as type>
 <#list type.minor as minor>
-<#list ["", "Repeated"] as mode>
+<#list [""] as mode>
 <#assign lowerName = minor.class?uncap_first />
 <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-<#assign name = mode + minor.class?cap_first />
+<#assign name = minor.class?cap_first />
 <#assign javaType = (minor.javaType!type.javaType) />
 <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
 <#assign safeType=friendlyType />
@@ -38,9 +38,9 @@
 
 <#assign hasFriendly = minor.friendlyType!"no" == "no" />
 
-<#list ["", "Nullable"] as nullMode>
-<#if (mode == "Repeated" && nullMode  == "") || mode == "" >
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${nullMode}${name}ReaderImpl.java" />
+<#list ["Nullable"] as nullMode>
+<#if mode == "" >
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${name}ReaderImpl.java" />
 <#include "/@includes/license.ftl" />
 
 package org.apache.arrow.vector.complex.impl;
@@ -48,20 +48,20 @@
 <#include "/@includes/vv_imports.ftl" />
 
 @SuppressWarnings("unused")
-public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
+public class ${name}ReaderImpl extends AbstractFieldReader {
   
   private final ${nullMode}${name}Vector vector;
   
-  public ${nullMode}${name}ReaderImpl(${nullMode}${name}Vector vector){
+  public ${name}ReaderImpl(${nullMode}${name}Vector vector){
     super();
     this.vector = vector;
   }
 
-  public MajorType getType(){
-    return vector.getField().getType();
+  public MinorType getMinorType(){
+    return vector.getMinorType();
   }
 
-  public MaterializedField getField(){
+  public Field getField(){
     return vector.getField();
   }
   
@@ -73,50 +73,13 @@ public boolean isSet(){
     </#if>
   }
 
-
-  
-  
-  <#if mode == "Repeated">
-
-  public void copyAsValue(${minor.class?cap_first}Writer writer){
-    Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer;
-    impl.vector.copyFromSafe(idx(), impl.idx(), vector);
-  }
-  
-  public void copyAsField(String name, MapWriter writer){
-    Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl)  writer.list(name).${lowerName}();
-    impl.vector.copyFromSafe(idx(), impl.idx(), vector);
-  }
-  
-  public int size(){
-    return vector.getAccessor().getInnerValueCountAt(idx());
-  }
-  
-  public void read(int arrayIndex, ${minor.class?cap_first}Holder h){
-    vector.getAccessor().get(idx(), arrayIndex, h);
-  }
-  public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h){
-    vector.getAccessor().get(idx(), arrayIndex, h);
-  }
-  
-  public ${friendlyType} read${safeType}(int arrayIndex){
-    return vector.getAccessor().getSingleObject(idx(), arrayIndex);
-  }
-
-  
-  public List<Object> readObject(){
-    return (List<Object>) (Object) vector.getAccessor().getObject(idx());
-  }
-  
-  <#else>
-  
   public void copyAsValue(${minor.class?cap_first}Writer writer){
-    ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer;
+    ${minor.class?cap_first}WriterImpl impl = (${minor.class?cap_first}WriterImpl) writer;
     impl.vector.copyFromSafe(idx(), impl.idx(), vector);
   }
   
   public void copyAsField(String name, MapWriter writer){
-    ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
+    ${minor.class?cap_first}WriterImpl impl = (${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
     impl.vector.copyFromSafe(idx(), impl.idx(), vector);
   }
 
@@ -141,9 +104,6 @@ public void copyValue(FieldWriter w){
   public Object readObject(){
     return vector.getAccessor().getObject(idx());
   }
-
-  
-  </#if>
 }
 </#if>
 </#list>
@@ -156,18 +116,10 @@ public Object readObject(){
 @SuppressWarnings("unused")
 public interface ${name}Reader extends BaseReader{
   
-  <#if mode == "Repeated">
-  public int size();
-  public void read(int arrayIndex, ${minor.class?cap_first}Holder h);
-  public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h);
-  public Object readObject(int arrayIndex);
-  public ${friendlyType} read${safeType}(int arrayIndex);
-  <#else>
   public void read(${minor.class?cap_first}Holder h);
   public void read(Nullable${minor.class?cap_first}Holder h);
   public Object readObject();
   public ${friendlyType} read${safeType}();
-  </#if>  
   public boolean isSet();
   public void copyAsValue(${minor.class}Writer writer);
   public void copyAsField(String name, ${minor.class}Writer writer);
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 8f9a6e7b97117..3457545cea5d7 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -19,8 +19,8 @@
 <@pp.dropOutputFile />
 <#list vv.types as type>
 <#list type.minor as minor>
-<#list ["", "Nullable", "Repeated"] as mode>
-<#assign name = mode + minor.class?cap_first />
+<#list ["Nullable"] as mode>
+<#assign name = minor.class?cap_first />
 <#assign eName = name />
 <#assign javaType = (minor.javaType!type.javaType) />
 <#assign fields = minor.fields!type.fields />
@@ -38,17 +38,16 @@
 @SuppressWarnings("unused")
 public class ${eName}WriterImpl extends AbstractFieldWriter {
 
-  private final ${name}Vector.Mutator mutator;
-  final ${name}Vector vector;
+  private final Nullable${name}Vector.Mutator mutator;
+  final Nullable${name}Vector vector;
 
-  public ${eName}WriterImpl(${name}Vector vector, AbstractFieldWriter parent) {
-    super(parent);
+  public ${eName}WriterImpl(Nullable${name}Vector vector) {
     this.mutator = vector.getMutator();
     this.vector = vector;
   }
 
   @Override
-  public MaterializedField getField() {
+  public Field getField() {
     return vector.getField();
   }
 
@@ -89,12 +88,10 @@ public void write(Nullable${minor.class?cap_first}Holder h) {
     vector.getMutator().setValueCount(idx()+1);
   }
 
-  <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
     mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
     vector.getMutator().setValueCount(idx()+1);
   }
-  </#if>
 
   public void setPosition(int idx) {
     super.setPosition(idx);
@@ -114,11 +111,17 @@ public void write(Nullable${minor.class}Holder h) {
     vector.getMutator().setValueCount(idx()+1);
   }
 
-  <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+  <#if minor.class == "Decimal">
+  public void writeDecimal(int start, ArrowBuf buffer) {
+    mutator.setSafe(idx(), 1, start, buffer);
+    vector.getMutator().setValueCount(idx()+1);
+  }
+  <#else>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-    mutator.setSafe(idx(), <#if mode == "Nullable">1, </#if><#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+    mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
     vector.getMutator().setValueCount(idx()+1);
   }
+  </#if>
 
   <#if mode == "Nullable">
 
@@ -128,7 +131,6 @@ public void writeNull() {
   }
   </#if>
   </#if>
-  </#if>
 }
 
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/writer/${eName}Writer.java" />
@@ -141,7 +143,9 @@ public void writeNull() {
 public interface ${eName}Writer extends BaseWriter {
   public void write(${minor.class}Holder h);
 
-  <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+  <#if minor.class == "Decimal">
+  public void writeDecimal(int start, ArrowBuf buffer);
+  <#else>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
   </#if>
 }
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 18fcac93bb6f0..fe2b5c5b5bc92 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -43,20 +43,42 @@
 public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
 
-  private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
 
   private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
   private int allocationMonitor = 0;
 
-  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
+  <#if minor.class == "Decimal">
+
+  private int precision;
+  private int scale;
+
+  public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) {
+    super(name, allocator);
+    this.precision = precision;
+    this.scale = scale;
+  }
+  <#else>
+  public ${minor.class}Vector(String name, BufferAllocator allocator) {
+    super(name, allocator);
+  }
+  </#if>
+
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.${minor.class?upper_case};
+  }
+
+  @Override
+  public Field getField() {
+        throw new UnsupportedOperationException("internal vector");
   }
 
   @Override
   public FieldReader getReader(){
-    return reader;
+        throw new UnsupportedOperationException("non-nullable vectors cannot be used in readers");
   }
 
   @Override
@@ -162,7 +184,7 @@ public void reAlloc() {
       throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
     }
 
-    logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize);
+    logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", name, allocationSizeInBytes, newAllocationSize);
     final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
     newBuf.setBytes(0, data, 0, data.capacity());
     final int halfNewCapacity = newBuf.capacity() / 2;
@@ -181,30 +203,13 @@ public void zeroVector() {
     data.setZero(0, data.capacity());
   }
 
-//  @Override
-//  public void load(SerializedField metadata, ArrowBuf buffer) {
-//    Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
-//    final int actualLength = metadata.getBufferLength();
-//    final int valueCount = metadata.getValueCount();
-//    final int expectedLength = valueCount * ${type.width};
-//    assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength);
-//
-//    clear();
-//    if (data != null) {
-//      data.release(1);
-//    }
-//    data = buffer.slice(0, actualLength);
-//    data.retain(1);
-//    data.writerIndex(actualLength);
-//    }
-
   public TransferPair getTransferPair(BufferAllocator allocator){
-    return new TransferImpl(getField(), allocator);
+    return new TransferImpl(name, allocator);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    return new TransferImpl(getField().withPath(ref), allocator);
+    return new TransferImpl(ref, allocator);
   }
 
   @Override
@@ -230,8 +235,12 @@ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector
   private class TransferImpl implements TransferPair{
     private ${minor.class}Vector to;
 
-    public TransferImpl(MaterializedField field, BufferAllocator allocator){
-      to = new ${minor.class}Vector(field, allocator);
+    public TransferImpl(String name, BufferAllocator allocator){
+      <#if minor.class == "Decimal">
+      to = new ${minor.class}Vector(name, allocator, precision, scale);
+      <#else>
+      to = new ${minor.class}Vector(name, allocator);
+      </#if>
     }
 
     public TransferImpl(${minor.class}Vector to) {
@@ -260,7 +269,7 @@ public void copyValueSafe(int fromIndex, int toIndex) {
   }
 
   public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
-    <#if (type.width > 8)>
+    <#if (type.width > 8 || minor.class == "IntervalDay")>
     from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
     <#else> <#-- type.width <= 8 -->
     data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width},
@@ -298,7 +307,7 @@ public boolean isNull(int index){
       return false;
     }
 
-    <#if (type.width > 8)>
+    <#if (type.width > 8 || minor.class == "IntervalDay")>
 
     public ${minor.javaType!type.javaType} get(int index) {
       return data.slice(index * ${type.width}, ${type.width});
@@ -416,31 +425,30 @@ public StringBuilder getAsStringBuilder(int index) {
               append(millis));
     }
 
-    <#elseif (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+    <#elseif minor.class == "Decimal">
 
     public void get(int index, ${minor.class}Holder holder) {
         holder.start = index * ${type.width};
         holder.buffer = data;
-        holder.scale = getField().getScale();
-        holder.precision = getField().getPrecision();
+        holder.scale = scale;
+        holder.precision = precision;
     }
 
     public void get(int index, Nullable${minor.class}Holder holder) {
         holder.isSet = 1;
         holder.start = index * ${type.width};
         holder.buffer = data;
-        holder.scale = getField().getScale();
-        holder.precision = getField().getPrecision();
+        holder.scale = scale;
+        holder.precision = precision;
     }
 
-      @Override
-      public ${friendlyType} getObject(int index) {
-      <#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")>
-      // Get the BigDecimal object
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
-      <#else>
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
-      </#if>
+    @Override
+    public ${friendlyType} getObject(int index) {
+      byte[] bytes = new byte[${type.width}];
+      int start = ${type.width} * index;
+      data.getBytes(start, bytes, 0, ${type.width});
+      ${friendlyType} value = new BigDecimal(new BigInteger(bytes), scale);
+      return value;
     }
 
     <#else>
@@ -581,7 +589,7 @@ public final class Mutator extends BaseDataValueVector.BaseMutator {
     * @param index   position of the bit to set
     * @param value   value to set
     */
-  <#if (type.width > 8)>
+  <#if (type.width > 8) || minor.class == "IntervalDay">
    public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
      data.setBytes(index * ${type.width}, value, 0, ${type.width});
    }
@@ -653,7 +661,7 @@ public void setSafe(int index, Nullable${minor.class}Holder holder){
      setSafe(index, holder.days, holder.milliseconds);
    }
 
-   <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+   <#elseif minor.class == "Decimal">
 
    public void set(int index, ${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index 3005fca0385aa..1ed9287b00eec 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -19,9 +19,8 @@
 <@pp.dropOutputFile />
 <#list vv.types as type>
 <#list type.minor as minor>
-<#list ["", "Nullable", "Repeated"] as holderMode>
+<#list ["", "Nullable"] as holderMode>
 <#assign nullMode = holderMode />
-<#if holderMode == "Repeated"><#assign nullMode = "Nullable" /></#if>
 
 <#assign lowerName = minor.class?uncap_first />
 <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
@@ -50,42 +49,18 @@
 public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
 
   private ${nullMode}${name}Holder holder;
-<#if holderMode == "Repeated" >
-  private int index = -1;
-  private ${holderMode}${name}Holder repeatedHolder;
-</#if>
-
   public ${holderMode}${name}HolderReaderImpl(${holderMode}${name}Holder holder) {
-<#if holderMode == "Repeated" >
-    this.holder = new ${nullMode}${name}Holder();
-    this.repeatedHolder = holder;
-<#else>
     this.holder = holder;
-</#if>
   }
 
   @Override
   public int size() {
-<#if holderMode == "Repeated">
-    return repeatedHolder.end - repeatedHolder.start;
-<#else>
     throw new UnsupportedOperationException("You can't call size on a Holder value reader.");
-</#if>
   }
 
   @Override
   public boolean next() {
-<#if holderMode == "Repeated">
-    if(index + 1 < repeatedHolder.end) {
-      index++;
-      repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
-      return true;
-    } else {
-      return false;
-    }
-<#else>
     throw new UnsupportedOperationException("You can't call next on a single value reader.");
-</#if>
 
   }
 
@@ -95,19 +70,13 @@ public void setPosition(int index) {
   }
 
   @Override
-  public MajorType getType() {
-<#if holderMode == "Repeated">
-    return this.repeatedHolder.TYPE;
-<#else>
-    return this.holder.TYPE;
-</#if>
+  public MinorType getMinorType() {
+        return MinorType.${name?upper_case};
   }
 
   @Override
   public boolean isSet() {
-    <#if holderMode == "Repeated">
-    return this.repeatedHolder.end!=this.repeatedHolder.start;
-    <#elseif nullMode == "Nullable">
+    <#if holderMode == "Nullable">
     return this.holder.isSet == 1;
     <#else>
     return true;
@@ -115,7 +84,6 @@ public boolean isSet() {
     
   }
 
-<#if holderMode != "Repeated">
 @Override
   public void read(${name}Holder h) {
   <#list fields as field>
@@ -130,19 +98,7 @@ public void read(Nullable${name}Holder h) {
   </#list>
     h.isSet = isSet() ? 1 : 0;
   }
-</#if>
 
-<#if holderMode == "Repeated">
-  @Override
-  public ${friendlyType} read${safeType}(int index){
-    repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
-    ${friendlyType} value = read${safeType}();
-    if (this.index > -1) {
-      repeatedHolder.vector.getAccessor().get(repeatedHolder.start + this.index, holder);
-    }
-    return value;
-  }
-</#if>
 
   @Override
   public ${friendlyType} read${safeType}(){
@@ -176,29 +132,10 @@ public void read(Nullable${name}Holder h) {
       Period p = new Period();
       return p.plusDays(holder.days).plusMillis(holder.milliseconds);
 
-<#elseif minor.class == "Decimal9" ||
-         minor.class == "Decimal18" >
-      BigInteger value = BigInteger.valueOf(holder.value);
-      return new BigDecimal(value, holder.scale);
-
-<#elseif minor.class == "Decimal28Dense" ||
-         minor.class == "Decimal38Dense">
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
-                                                                                holder.start,
-                                                                                holder.nDecimalDigits,
-                                                                                holder.scale,
-                                                                                holder.maxPrecision,
-                                                                                holder.WIDTH);
-
-<#elseif minor.class == "Decimal28Sparse" ||
-         minor.class == "Decimal38Sparse">
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
-                                                                                 holder.start,
-                                                                                 holder.nDecimalDigits,
-                                                                                 holder.scale);
-
 <#elseif minor.class == "Bit" >
       return new Boolean(holder.value != 0);
+<#elseif minor.class == "Decimal" >
+        return (BigDecimal) readSingleObject();
 <#else>
       ${friendlyType} value = new ${friendlyType}(this.holder.value);
       return value;
@@ -208,15 +145,7 @@ public void read(Nullable${name}Holder h) {
 
   @Override
   public Object readObject() {
-<#if holderMode == "Repeated" >
-    List<Object> valList = Lists.newArrayList();
-    for (int i = repeatedHolder.start; i < repeatedHolder.end; i++) {
-      valList.add(repeatedHolder.vector.getAccessor().getObject(i));
-    }
-    return valList;
-<#else>
     return readSingleObject();
-</#if>
   }
 
   private Object readSingleObject() {
@@ -239,6 +168,9 @@ private Object readSingleObject() {
       Text text = new Text();
       text.set(value);
       return text;
+<#elseif minor.class == "Decimal" >
+        return new BigDecimal(new BigInteger(value), holder.scale);
+
 </#if>
 
 <#elseif minor.class == "Interval">
@@ -249,11 +181,6 @@ private Object readSingleObject() {
       Period p = new Period();
       return p.plusDays(holder.days).plusMillis(holder.milliseconds);
 
-<#elseif minor.class == "Decimal9" ||
-         minor.class == "Decimal18" >
-      BigInteger value = BigInteger.valueOf(holder.value);
-      return new BigDecimal(value, holder.scale);
-
 <#elseif minor.class == "Decimal28Dense" ||
          minor.class == "Decimal38Dense">
       return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
@@ -272,13 +199,18 @@ private Object readSingleObject() {
 
 <#elseif minor.class == "Bit" >
       return new Boolean(holder.value != 0);
+<#elseif minor.class == "Decimal">
+        byte[] bytes = new byte[${type.width}];
+        holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
+        ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
+        return value;
 <#else>
       ${friendlyType} value = new ${friendlyType}(this.holder.value);
       return value;
 </#if>
   }
 
-<#if holderMode != "Repeated" && nullMode != "Nullable">
+<#if nullMode != "Nullable">
   public void copyAsValue(${minor.class?cap_first}Writer writer){
     writer.write(holder);
   }
diff --git a/java/vector/src/main/codegen/templates/ListWriters.java b/java/vector/src/main/codegen/templates/ListWriters.java
deleted file mode 100644
index 94b812b83dc96..0000000000000
--- a/java/vector/src/main/codegen/templates/ListWriters.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-
-<#list ["Single", "Repeated"] as mode>
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}ListWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector.complex.impl;
-<#if mode == "Single">
-  <#assign containerClass = "AbstractContainerVector" />
-  <#assign index = "idx()">
-<#else>
-  <#assign containerClass = "RepeatedListVector" />
-  <#assign index = "currentChildIndex">
-</#if>
-
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using FreeMarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-public class ${mode}ListWriter extends AbstractFieldWriter {
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${mode}ListWriter.class);
-
-  static enum Mode { INIT, IN_MAP, IN_LIST <#list vv.types as type><#list type.minor as minor>, IN_${minor.class?upper_case}</#list></#list> }
-
-  private final String name;
-  protected final ${containerClass} container;
-  private Mode mode = Mode.INIT;
-  private FieldWriter writer;
-  protected RepeatedValueVector innerVector;
-
-  <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
-  public ${mode}ListWriter(String name, ${containerClass} container, FieldWriter parent){
-    super(parent);
-    this.name = name;
-    this.container = container;
-  }
-
-  public ${mode}ListWriter(${containerClass} container, FieldWriter parent){
-    super(parent);
-    this.name = null;
-    this.container = container;
-  }
-
-  @Override
-  public void allocate() {
-    if(writer != null) {
-      writer.allocate();
-    }
-
-    <#if mode == "Repeated">
-    container.allocateNew();
-    </#if>
-  }
-
-  @Override
-  public void clear() {
-    if (writer != null) {
-      writer.clear();
-    }
-  }
-
-  @Override
-  public void close() {
-    clear();
-    container.close();
-    if (innerVector != null) {
-      innerVector.close();
-    }
-  }
-
-  @Override
-  public int getValueCapacity() {
-    return innerVector == null ? 0 : innerVector.getValueCapacity();
-  }
-
-  public void setValueCount(int count){
-    if(innerVector != null) innerVector.getMutator().setValueCount(count);
-  }
-
-  @Override
-  public MapWriter map() {
-    switch(mode) {
-    case INIT:
-      int vectorCount = container.size();
-      final RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class);
-      innerVector = vector;
-      writer = new RepeatedMapWriter(vector, this);
-      if(vectorCount != container.size()) {
-        writer.allocate();
-      }
-      writer.setPosition(${index});
-      mode = Mode.IN_MAP;
-      return writer;
-    case IN_MAP:
-      return writer;
-    }
-
-    throw new RuntimeException(getUnsupportedErrorMsg("MAP", mode.name()));
-
-  }
-
-  @Override
-  public ListWriter list() {
-    switch(mode) {
-    case INIT:
-      final int vectorCount = container.size();
-      final RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class);
-      innerVector = vector;
-      writer = new RepeatedListWriter(null, vector, this);
-      if(vectorCount != container.size()) {
-        writer.allocate();
-      }
-      writer.setPosition(${index});
-      mode = Mode.IN_LIST;
-      return writer;
-    case IN_LIST:
-      return writer;
-    }
-
-    throw new RuntimeException(getUnsupportedErrorMsg("LIST", mode.name()));
-
-  }
-
-  <#list vv.types as type><#list type.minor as minor>
-  <#assign lowerName = minor.class?uncap_first />
-  <#assign upperName = minor.class?upper_case />
-  <#assign capName = minor.class?cap_first />
-  <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-
-  private static final MajorType ${upperName}_TYPE = Types.repeated(MinorType.${upperName});
-
-  @Override
-  public ${capName}Writer ${lowerName}() {
-    switch(mode) {
-    case INIT:
-      final int vectorCount = container.size();
-      final Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class);
-      innerVector = vector;
-      writer = new Repeated${capName}WriterImpl(vector, this);
-      if(vectorCount != container.size()) {
-        writer.allocate();
-      }
-      writer.setPosition(${index});
-      mode = Mode.IN_${upperName};
-      return writer;
-    case IN_${upperName}:
-      return writer;
-    }
-
-    throw new RuntimeException(getUnsupportedErrorMsg("${upperName}", mode.name()));
-
-  }
-  </#list></#list>
-
-  public MaterializedField getField() {
-    return container.getField();
-  }
-
-  <#if mode == "Repeated">
-
-  public void startList() {
-    final RepeatedListVector list = (RepeatedListVector) container;
-    final RepeatedListVector.RepeatedMutator mutator = list.getMutator();
-
-    // make sure that the current vector can support the end position of this list.
-    if(container.getValueCapacity() <= idx()) {
-      mutator.setValueCount(idx()+1);
-    }
-
-    // update the repeated vector to state that there is current+1 objects.
-    final RepeatedListHolder h = new RepeatedListHolder();
-    list.getAccessor().get(idx(), h);
-    if (h.start >= h.end) {
-      mutator.startNewValue(idx());
-    }
-    currentChildIndex = container.getMutator().add(idx());
-    if(writer != null) {
-      writer.setPosition(currentChildIndex);
-    }
-  }
-
-  public void endList() {
-    // noop, we initialize state at start rather than end.
-  }
-  <#else>
-
-  public void setPosition(int index) {
-    super.setPosition(index);
-    if(writer != null) {
-      writer.setPosition(index);
-    }
-  }
-
-  public void startList() {
-    // noop
-  }
-
-  public void endList() {
-    // noop
-  }
-  </#if>
-
-  private String getUnsupportedErrorMsg(String expected, String found) {
-    final String f = found.substring(3);
-    return String.format("In a list of type %s, encountered a value of type %s. "+
-      "Arrow does not support lists of different types.",
-       f, expected
-    );
-  }
-}
-</#list>
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 42f39820393e5..af2922826ec4d 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -17,7 +17,7 @@
  */
 
 <@pp.dropOutputFile />
-<#list ["Single", "Repeated"] as mode>
+<#list ["Single"] as mode>
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" />
 <#if mode == "Single">
 <#assign containerClass = "MapVector" />
@@ -51,16 +51,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
   private final Map<String, FieldWriter> fields = Maps.newHashMap();
   <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
 
-  private final boolean unionEnabled;
-
-  public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled) {
-    super(parent);
+  public ${mode}MapWriter(${containerClass} container) {
     this.container = container;
-    this.unionEnabled = unionEnabled;
-  }
-
-  public ${mode}MapWriter(${containerClass} container, FieldWriter parent) {
-    this(container, parent, false);
   }
 
   @Override
@@ -74,7 +66,7 @@ public boolean isEmptyMap() {
   }
 
   @Override
-  public MaterializedField getField() {
+  public Field getField() {
       return container.getField();
   }
 
@@ -83,12 +75,8 @@ public MapWriter map(String name) {
       FieldWriter writer = fields.get(name.toLowerCase());
     if(writer == null){
       int vectorCount=container.size();
-        MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class);
-      if(!unionEnabled){
-        writer = new SingleMapWriter(vector, this);
-      } else {
-        writer = new PromotableWriter(vector, container);
-      }
+      MapVector vector = container.addOrGet(name, MinorType.MAP, MapVector.class);
+      writer = new PromotableWriter(vector, container);
       if(vectorCount != container.size()) {
         writer.allocate();
       }
@@ -125,11 +113,7 @@ public ListWriter list(String name) {
     FieldWriter writer = fields.get(name.toLowerCase());
     int vectorCount = container.size();
     if(writer == null) {
-      if (!unionEnabled){
-        writer = new SingleListWriter(name,container,this);
-      } else{
-        writer = new PromotableWriter(container.addOrGet(name, Types.optional(MinorType.LIST), ListVector.class), container);
-      }
+      writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class), container);
       if (container.size() > vectorCount) {
         writer.allocate();
       }
@@ -206,9 +190,7 @@ public void end() {
   }
 
   public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) {
-    final MajorType ${upperName}_TYPE = new MajorType(MinorType.${upperName}, DataMode.OPTIONAL, precision, scale, 0, null);
   <#else>
-  private static final MajorType ${upperName}_TYPE = Types.optional(MinorType.${upperName});
   @Override
   public ${minor.class}Writer ${lowerName}(String name) {
   </#if>
@@ -216,15 +198,9 @@ public void end() {
     if(writer == null) {
       ValueVector vector;
       ValueVector currentVector = container.getChild(name);
-      if (unionEnabled){
-        ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
-        writer = new PromotableWriter(v, container);
-        vector = v;
-      } else {
-        ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
-        writer = new ${vectName}WriterImpl(v, this);
-        vector = v;
-      }
+      ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class);
+      writer = new PromotableWriter(v, container);
+      vector = v;
       if (currentVector == null || currentVector != vector) {
         vector.allocateNewSafe();
       } 
diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java
index 3ef6c7dcc49a6..ba0c088add7c9 100644
--- a/java/vector/src/main/codegen/templates/NullReader.java
+++ b/java/vector/src/main/codegen/templates/NullReader.java
@@ -16,6 +16,9 @@
  * limitations under the License.
  */
 
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.Field;
+
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/NullReader.java" />
 
@@ -31,25 +34,31 @@
 public class NullReader extends AbstractBaseReader implements FieldReader{
   
   public static final NullReader INSTANCE = new NullReader();
-  public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(Types.repeated(MinorType.NULL));
-  public static final NullReader EMPTY_MAP_INSTANCE = new NullReader(Types.required(MinorType.MAP));
-  private MajorType type;
+  public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(MinorType.NULL);
+  public static final NullReader EMPTY_MAP_INSTANCE = new NullReader(MinorType.MAP);
+  private MinorType type;
   
   private NullReader(){
     super();
-    type = Types.required(MinorType.NULL);
+    type = MinorType.NULL;
   }
 
-  private NullReader(MajorType type){
+  private NullReader(MinorType type){
     super();
     this.type = type;
   }
 
   @Override
-  public MajorType getType() {
+  public MinorType getMinorType() {
     return type;
   }
-  
+
+
+  @Override
+  public Field getField() {
+    return new Field("", true, new Null(), null);
+  }
+
   public void copyAsValue(MapWriter writer) {}
 
   public void copyAsValue(ListWriter writer) {}
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index b0029f7ad4c37..df508979c48b5 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -42,19 +42,79 @@
 public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
-  private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
+  private final FieldReader reader = new ${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
 
-  private final MaterializedField bitsField = MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED));
-  private final MaterializedField valuesField = MaterializedField.create("$values$", new MajorType(field.getType().getMinorType(), DataMode.REQUIRED, field.getPrecision(), field.getScale()));
+  private final String bitsField = "$bits$";
+  private final String valuesField = "$values$";
+  private final Field field;
 
   final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
-  final ${valuesName} values = new ${minor.class}Vector(valuesField, allocator);
+  final ${valuesName} values;
 
-  private final Mutator mutator = new Mutator();
-  private final Accessor accessor = new Accessor();
+  private final Mutator mutator;
+  private final Accessor accessor;
 
-  public ${className}(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
+  <#if minor.class == "Decimal">
+  private final int precision;
+  private final int scale;
+
+  public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
+    super(name, allocator);
+    values = new ${minor.class}Vector(valuesField, allocator, precision, scale);
+    this.precision = precision;
+    this.scale = scale;
+    mutator = new Mutator();
+    accessor = new Accessor();
+    field = new Field(name, true, new Decimal(precision, scale), null);
+  }
+  <#else>
+  public ${className}(String name, BufferAllocator allocator) {
+    super(name, allocator);
+    values = new ${minor.class}Vector(valuesField, allocator);
+    mutator = new Mutator();
+    accessor = new Accessor();
+  <#if minor.class == "TinyInt" ||
+        minor.class == "SmallInt" ||
+        minor.class == "Int" ||
+        minor.class == "BigInt">
+    field = new Field(name, true, new Int(${type.width} * 8, true), null);
+  <#elseif minor.class == "UInt1" ||
+        minor.class == "UInt2" ||
+        minor.class == "UInt4" ||
+        minor.class == "UInt8">
+    field = new Field(name, true, new Int(${type.width} * 8, false), null);
+  <#elseif minor.class == "Date">
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Date(), null);
+  <#elseif minor.class == "Time">
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), null);
+  <#elseif minor.class == "Float4">
+    field = new Field(name, true, new FloatingPoint(0), null);
+  <#elseif minor.class == "Float8">
+    field = new Field(name, true, new FloatingPoint(1), null);
+  <#elseif minor.class == "TimeStamp">
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(""), null);
+  <#elseif minor.class == "IntervalDay">
+    field = new Field(name, true, new IntervalDay(), null);
+  <#elseif minor.class == "IntervalYear">
+    field = new Field(name, true, new IntervalYear(), null);
+  <#elseif minor.class == "VarChar">
+    field = new Field(name, true, new Utf8(), null);
+  <#elseif minor.class == "VarBinary">
+    field = new Field(name, true, new Binary(), null);
+  <#elseif minor.class == "Bit">
+    field = new Field(name, true, new Bool(), null);
+  </#if>
+  }
+  </#if>
+
+  @Override
+  public Field getField() {
+    return field;
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.${minor.class?upper_case};
   }
 
   @Override
@@ -240,12 +300,13 @@ public void zeroVector() {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator){
-    return new TransferImpl(getField(), allocator);
+    return new TransferImpl(name, allocator);
+
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    return new TransferImpl(getField().withPath(ref), allocator);
+    return new TransferImpl(ref, allocator);
   }
 
   @Override
@@ -273,8 +334,12 @@ public void splitAndTransferTo(int startIndex, int length, Nullable${minor.class
   private class TransferImpl implements TransferPair {
     Nullable${minor.class}Vector to;
 
-    public TransferImpl(MaterializedField field, BufferAllocator allocator){
-      to = new Nullable${minor.class}Vector(field, allocator);
+    public TransferImpl(String name, BufferAllocator allocator){
+      <#if minor.class == "Decimal">
+      to = new Nullable${minor.class}Vector(name, allocator, precision, scale);
+      <#else>
+      to = new Nullable${minor.class}Vector(name, allocator);
+      </#if>
     }
 
     public TransferImpl(Nullable${minor.class}Vector to){
@@ -312,17 +377,6 @@ public Mutator getMutator(){
     return mutator;
   }
 
-  public ${minor.class}Vector convertToRequiredVector(){
-    ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
-    if (v.data != null) {
-      v.data.release(1);
-    }
-    v.data = values.data;
-    v.data.retain(1);
-    clear();
-    return v;
-  }
-
   public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
     final Accessor fromAccessor = from.getAccessor();
     if (!fromAccessor.isNull(fromIndex)) {
@@ -389,8 +443,8 @@ public void get(int index, Nullable${minor.class}Holder holder){
       holder.isSet = bAccessor.get(index);
 
       <#if minor.class.startsWith("Decimal")>
-      holder.scale = getField().getScale();
-      holder.precision = getField().getPrecision();
+      holder.scale = scale;
+      holder.precision = precision;
       </#if>
     }
 
diff --git a/java/vector/src/main/codegen/templates/RepeatedValueVectors.java b/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
deleted file mode 100644
index ceae53bbf58cf..0000000000000
--- a/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-<#assign fields = minor.fields!type.fields />
-
-<@pp.changeOutputFile name="/org/apache/arrow/vector/Repeated${minor.class}Vector.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/**
- * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
- * repeated protobuf field).  The implementation uses two additional value vectors; one to convert
- * the index offset to the underlying element offset, and another to store the number of values
- * in the vector.
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-
-public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>VectorLike {
-  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Repeated${minor.class}Vector.class);
-
-  // we maintain local reference to concrete vector type for performance reasons.
-  ${minor.class}Vector values;
-  private final FieldReader reader = new Repeated${minor.class}ReaderImpl(Repeated${minor.class}Vector.this);
-  private final Mutator mutator = new Mutator();
-  private final Accessor accessor = new Accessor();
-
-  public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
-    addOrGetVector(VectorDescriptor.create(new MajorType(field.getType().getMinorType(), DataMode.REQUIRED)));
-  }
-
-  @Override
-  public Mutator getMutator() {
-    return mutator;
-  }
-
-  @Override
-  public Accessor getAccessor() {
-    return accessor;
-  }
-
-  @Override
-  public FieldReader getReader() {
-    return reader;
-  }
-
-  @Override
-  public ${minor.class}Vector getDataVector() {
-    return values;
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new TransferImpl(getField(), allocator);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    return new TransferImpl(getField().withPath(ref), allocator);
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((Repeated${minor.class}Vector) to);
-  }
-
-  @Override
-  public AddOrGetResult<${minor.class}Vector> addOrGetVector(VectorDescriptor descriptor) {
-    final AddOrGetResult<${minor.class}Vector> result = super.addOrGetVector(descriptor);
-    if (result.isCreated()) {
-      values = result.getVector();
-    }
-    return result;
-  }
-
-  public void transferTo(Repeated${minor.class}Vector target) {
-    target.clear();
-    offsets.transferTo(target.offsets);
-    values.transferTo(target.values);
-    clear();
-  }
-
-  public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) {
-    final UInt4Vector.Accessor a = offsets.getAccessor();
-    final UInt4Vector.Mutator m = to.offsets.getMutator();
-
-    final int startPos = a.get(startIndex);
-    final int endPos = a.get(startIndex + groups);
-    final int valuesToCopy = endPos - startPos;
-
-    values.splitAndTransferTo(startPos, valuesToCopy, to.values);
-    to.offsets.clear();
-    to.offsets.allocateNew(groups + 1);
-    int normalizedPos = 0;
-    for (int i=0; i < groups + 1;i++ ) {
-      normalizedPos = a.get(startIndex+i) - startPos;
-      m.set(i, normalizedPos);
-    }
-    m.setValueCount(groups == 0 ? 0 : groups + 1);
-  }
-
-  private class TransferImpl implements TransferPair {
-    final Repeated${minor.class}Vector to;
-
-    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
-      this.to = new Repeated${minor.class}Vector(field, allocator);
-    }
-
-    public TransferImpl(Repeated${minor.class}Vector to) {
-      this.to = to;
-    }
-
-    @Override
-    public Repeated${minor.class}Vector getTo() {
-      return to;
-    }
-
-    @Override
-    public void transfer() {
-      transferTo(to);
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      splitAndTransferTo(startIndex, length, to);
-    }
-
-    @Override
-    public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, Repeated${minor.class}Vector.this);
-    }
-  }
-
-    public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
-      final Accessor vAccessor = v.getAccessor();
-      final int count = vAccessor.getInnerValueCountAt(inIndex);
-      mutator.startNewValue(outIndex);
-      for (int i = 0; i < count; i++) {
-        mutator.add(outIndex, vAccessor.get(inIndex, i));
-      }
-    }
-
-    public void copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
-      final Accessor vAccessor = v.getAccessor();
-      final int count = vAccessor.getInnerValueCountAt(inIndex);
-      mutator.startNewValue(outIndex);
-      for (int i = 0; i < count; i++) {
-        mutator.addSafe(outIndex, vAccessor.get(inIndex, i));
-      }
-    }
-
-  public boolean allocateNewSafe() {
-    /* boolean to keep track if all the memory allocation were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      if(!offsets.allocateNewSafe()) return false;
-      if(!values.allocateNewSafe()) return false;
-      success = true;
-    } finally {
-      if (!success) {
-        clear();
-      }
-    }
-    offsets.zeroVector();
-    mutator.reset();
-    return true;
-  }
-
-  @Override
-  public void allocateNew() {
-    try {
-      offsets.allocateNew();
-      values.allocateNew();
-    } catch (OutOfMemoryException e) {
-      clear();
-      throw e;
-    }
-    offsets.zeroVector();
-    mutator.reset();
-  }
-
-  <#if type.major == "VarLen">
-//  @Override
-//  protected SerializedField.Builder getMetadataBuilder() {
-//    return super.getMetadataBuilder()
-//            .setVarByteLength(values.getVarByteLength());
-//  }
-
-  public void allocateNew(int totalBytes, int valueCount, int innerValueCount) {
-    try {
-      offsets.allocateNew(valueCount + 1);
-      values.allocateNew(totalBytes, innerValueCount);
-    } catch (OutOfMemoryException e) {
-      clear();
-      throw e;
-    }
-    offsets.zeroVector();
-    mutator.reset();
-  }
-
-  public int getByteCapacity(){
-    return values.getByteCapacity();
-  }
-
-  <#else>
-
-  @Override
-  public void allocateNew(int valueCount, int innerValueCount) {
-    clear();
-    /* boolean to keep track if all the memory allocation were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to//
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      offsets.allocateNew(valueCount + 1);
-      values.allocateNew(innerValueCount);
-    } catch(OutOfMemoryException e){
-      clear();
-      throw e;
-    }
-    offsets.zeroVector();
-    mutator.reset();
-  }
-
-  </#if>
-
-  // This is declared a subclass of the accessor declared inside of FixedWidthVector, this is also used for
-  // variable length vectors, as they should ahve consistent interface as much as possible, if they need to diverge
-  // in the future, the interface shold be declared in the respective value vector superclasses for fixed and variable
-  // and we should refer to each in the generation template
-  public final class Accessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
-    @Override
-    public List<${friendlyType}> getObject(int index) {
-      final List<${friendlyType}> vals = new JsonStringArrayList<>();
-      final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
-      final int start = offsetsAccessor.get(index);
-      final int end = offsetsAccessor.get(index + 1);
-      final ${minor.class}Vector.Accessor valuesAccessor = values.getAccessor();
-      for(int i = start; i < end; i++) {
-        vals.add(valuesAccessor.getObject(i));
-      }
-      return vals;
-    }
-
-    public ${friendlyType} getSingleObject(int index, int arrayIndex) {
-      final int start = offsets.getAccessor().get(index);
-      return values.getAccessor().getObject(start + arrayIndex);
-    }
-
-    /**
-     * Get a value for the given record.  Each element in the repeated field is accessed by
-     * the positionIndex param.
-     *
-     * @param  index           record containing the repeated field
-     * @param  positionIndex   position within the repeated field
-     * @return element at the given position in the given record
-     */
-    public <#if type.major == "VarLen">byte[]
-           <#else>${minor.javaType!type.javaType}
-           </#if> get(int index, int positionIndex) {
-      return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
-    }
-
-    public void get(int index, Repeated${minor.class}Holder holder) {
-      holder.start = offsets.getAccessor().get(index);
-      holder.end =  offsets.getAccessor().get(index+1);
-      holder.vector = values;
-    }
-
-    public void get(int index, int positionIndex, ${minor.class}Holder holder) {
-      final int offset = offsets.getAccessor().get(index);
-      assert offset >= 0;
-      assert positionIndex < getInnerValueCountAt(index);
-      values.getAccessor().get(offset + positionIndex, holder);
-    }
-
-    public void get(int index, int positionIndex, Nullable${minor.class}Holder holder) {
-      final int offset = offsets.getAccessor().get(index);
-      assert offset >= 0;
-      if (positionIndex >= getInnerValueCountAt(index)) {
-        holder.isSet = 0;
-        return;
-      }
-      values.getAccessor().get(offset + positionIndex, holder);
-    }
-  }
-
-  public final class Mutator extends BaseRepeatedValueVector.BaseRepeatedMutator implements RepeatedMutator {
-    private Mutator() {}
-
-    /**
-     * Add an element to the given record index.  This is similar to the set() method in other
-     * value vectors, except that it permits setting multiple values for a single record.
-     *
-     * @param index   record of the element to add
-     * @param value   value to add to the given row
-     */
-    public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
-      int nextOffset = offsets.getAccessor().get(index+1);
-      values.getMutator().set(nextOffset, value);
-      offsets.getMutator().set(index+1, nextOffset+1);
-    }
-
-    <#if type.major == "VarLen">
-    public void addSafe(int index, byte[] bytes) {
-      addSafe(index, bytes, 0, bytes.length);
-    }
-
-    public void addSafe(int index, byte[] bytes, int start, int length) {
-      final int nextOffset = offsets.getAccessor().get(index+1);
-      values.getMutator().setSafe(nextOffset, bytes, start, length);
-      offsets.getMutator().setSafe(index+1, nextOffset+1);
-    }
-
-    <#else>
-
-    public void addSafe(int index, ${minor.javaType!type.javaType} srcValue) {
-      final int nextOffset = offsets.getAccessor().get(index+1);
-      values.getMutator().setSafe(nextOffset, srcValue);
-      offsets.getMutator().setSafe(index+1, nextOffset+1);
-    }
-
-    </#if>
-
-    public void setSafe(int index, Repeated${minor.class}Holder h) {
-      final ${minor.class}Holder ih = new ${minor.class}Holder();
-      final ${minor.class}Vector.Accessor hVectorAccessor = h.vector.getAccessor();
-      mutator.startNewValue(index);
-      for(int i = h.start; i < h.end; i++){
-        hVectorAccessor.get(i, ih);
-        mutator.addSafe(index, ih);
-      }
-    }
-
-    public void addSafe(int index, ${minor.class}Holder holder) {
-      int nextOffset = offsets.getAccessor().get(index+1);
-      values.getMutator().setSafe(nextOffset, holder);
-      offsets.getMutator().setSafe(index+1, nextOffset+1);
-    }
-
-    public void addSafe(int index, Nullable${minor.class}Holder holder) {
-      final int nextOffset = offsets.getAccessor().get(index+1);
-      values.getMutator().setSafe(nextOffset, holder);
-      offsets.getMutator().setSafe(index+1, nextOffset+1);
-    }
-
-    <#if (fields?size > 1) && !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
-    public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-      int nextOffset = offsets.getAccessor().get(arrayIndex+1);
-      values.getMutator().setSafe(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
-      offsets.getMutator().setSafe(arrayIndex+1, nextOffset+1);
-    }
-    </#if>
-
-    protected void add(int index, ${minor.class}Holder holder) {
-      int nextOffset = offsets.getAccessor().get(index+1);
-      values.getMutator().set(nextOffset, holder);
-      offsets.getMutator().set(index+1, nextOffset+1);
-    }
-
-    public void add(int index, Repeated${minor.class}Holder holder) {
-
-      ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
-      ${minor.class}Holder innerHolder = new ${minor.class}Holder();
-
-      for(int i = holder.start; i < holder.end; i++) {
-        accessor.get(i, innerHolder);
-        add(index, innerHolder);
-      }
-    }
-
-    @Override
-    public void generateTestData(final int valCount) {
-      final int[] sizes = {1, 2, 0, 6};
-      int size = 0;
-      int runningOffset = 0;
-      final UInt4Vector.Mutator offsetsMutator = offsets.getMutator();
-      for(int i = 1; i < valCount + 1; i++, size++) {
-        runningOffset += sizes[size % sizes.length];
-        offsetsMutator.set(i, runningOffset);
-      }
-      values.getMutator().generateTestData(valCount * 9);
-      setValueCount(size);
-    }
-
-    @Override
-    public void reset() {
-    }
-  }
-}
-</#list>
-</#list>
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index 9a6b08fc561f9..49d57e716bc8a 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -43,7 +43,6 @@ public class UnionListWriter extends AbstractFieldWriter {
   private int lastIndex = 0;
 
   public UnionListWriter(ListVector vector) {
-    super(null);
     this.vector = vector;
     this.writer = new PromotableWriter(vector.getDataVector(), vector);
     this.offsets = vector.getOffsetVector();
@@ -64,10 +63,14 @@ public void clear() {
   }
 
   @Override
-  public MaterializedField getField() {
+  public Field getField() {
     return null;
   }
 
+  public void setValueCount(int count) {
+    vector.getMutator().setValueCount(count);
+  }
+
   @Override
   public int getValueCapacity() {
     return vector.getValueCapacity();
@@ -78,6 +81,12 @@ public void close() throws Exception {
 
   }
 
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    startList();
+  }
+
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
   <#assign uncappedName = name?uncap_first/>
@@ -91,7 +100,7 @@ public void close() throws Exception {
 
   @Override
   public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
-    assert inMap;
+//    assert inMap;
     mapName = name;
     final int nextOffset = offsets.getAccessor().get(idx() + 1);
     vector.getMutator().setNotNull(idx());
@@ -146,7 +155,7 @@ public void endList() {
 
   @Override
   public void start() {
-    assert inMap;
+//    assert inMap;
     final int nextOffset = offsets.getAccessor().get(idx() + 1);
     vector.getMutator().setNotNull(idx());
     offsets.getMutator().setSafe(idx() + 1, nextOffset);
@@ -155,11 +164,11 @@ public void start() {
 
   @Override
   public void end() {
-    if (inMap) {
+//    if (inMap) {
       inMap = false;
       final int nextOffset = offsets.getAccessor().get(idx() + 1);
       offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
-    }
+//    }
   }
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
@@ -170,7 +179,7 @@ public void end() {
 
   @Override
   public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-    assert !inMap;
+//    assert !inMap;
     final int nextOffset = offsets.getAccessor().get(idx() + 1);
     vector.getMutator().setNotNull(idx());
     writer.setPosition(nextOffset);
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index 44c3e55dcc6f1..7351ae3776f57 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -17,6 +17,8 @@
  */
 
 
+import org.apache.arrow.vector.types.Types.MinorType;
+
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionReader.java" />
 
@@ -37,18 +39,18 @@ public UnionReader(UnionVector data) {
     this.data = data;
   }
 
-  private static MajorType[] TYPES = new MajorType[43];
+  public MinorType getMinorType() {
+    return TYPES[data.getTypeValue(idx())];
+  }
+
+  private static MinorType[] TYPES = new MinorType[43];
 
   static {
     for (MinorType minorType : MinorType.values()) {
-      TYPES[minorType.ordinal()] = new MajorType(minorType, DataMode.OPTIONAL);
+      TYPES[minorType.ordinal()] = minorType;
     }
   }
 
-  public MajorType getType() {
-    return TYPES[data.getTypeValue(idx())];
-  }
-
   public boolean isSet(){
     return !data.getAccessor().isNull(idx());
   }
@@ -69,7 +71,7 @@ private FieldReader getReaderForIndex(int index) {
       return reader;
     }
     switch (MinorType.values()[typeValue]) {
-    case LATE:
+    case NULL:
       return NullReader.INSTANCE;
     case MAP:
       return (FieldReader) getMap();
@@ -119,9 +121,9 @@ public void copyAsValue(UnionWriter writer) {
     writer.data.copyFrom(idx(), writer.idx(), data);
   }
 
-  <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
-          "Character", "DateTime", "Period", "Double", "Float",
-          "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
+  <#list ["Object", "Integer", "Long", "Boolean",
+          "Character", "DateTime", "Double", "Float",
+          "Text", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
 
@@ -141,11 +143,11 @@ public void copyAsValue(UnionWriter writer) {
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
   <#if !minor.class?starts_with("Decimal")>
 
-  private Nullable${name}ReaderImpl ${uncappedName}Reader;
+  private ${name}ReaderImpl ${uncappedName}Reader;
 
-  private Nullable${name}ReaderImpl get${name}() {
+  private ${name}ReaderImpl get${name}() {
     if (${uncappedName}Reader == null) {
-      ${uncappedName}Reader = new Nullable${name}ReaderImpl(data.get${name}Vector());
+      ${uncappedName}Reader = new ${name}ReaderImpl(data.get${name}Vector());
       ${uncappedName}Reader.setPosition(idx());
       readers[MinorType.${name?upper_case}.ordinal()] = ${uncappedName}Reader;
     }
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 0f089b7e91537..e2f19f4b33ba5 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -16,6 +16,16 @@
  * limitations under the License.
  */
 
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Field;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.flatbuf.Union;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+
+import java.util.ArrayList;
+import java.util.List;
+
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/UnionVector.java" />
 
@@ -29,7 +39,6 @@
 import java.util.Iterator;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.BasicTypeHelper;
 
 /*
  * This class is generated using freemarker and the ${.template_name} template.
@@ -47,34 +56,30 @@
  */
 public class UnionVector implements ValueVector {
 
-  private MaterializedField field;
+  private String name;
   private BufferAllocator allocator;
   private Accessor accessor = new Accessor();
   private Mutator mutator = new Mutator();
   int valueCount;
 
   MapVector internalMap;
-  private UInt1Vector typeVector;
+  UInt1Vector typeVector;
 
   private MapVector mapVector;
   private ListVector listVector;
 
   private FieldReader reader;
-  private NullableBitVector bit;
 
   private int singleType = 0;
   private ValueVector singleVector;
-  private MajorType majorType;
 
   private final CallBack callBack;
 
-  public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    this.field = field.clone();
+  public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
+    this.name = name;
     this.allocator = allocator;
     this.internalMap = new MapVector("internal", allocator, callBack);
-    this.typeVector = internalMap.addOrGet("types", new MajorType(MinorType.UINT1, DataMode.REQUIRED), UInt1Vector.class);
-    this.field.addChild(internalMap.getField().clone());
-    this.majorType = field.getType();
+    this.typeVector = new UInt1Vector("types", allocator);
     this.callBack = callBack;
   }
 
@@ -82,34 +87,20 @@ public BufferAllocator getAllocator() {
     return allocator;
   }
 
-  public List<MinorType> getSubTypes() {
-    return majorType.getSubTypes();
-  }
-
-  public void addSubType(MinorType type) {
-    if (majorType.getSubTypes().contains(type)) {
-      return;
-    }
-    List<MinorType> subTypes = this.majorType.getSubTypes();
-    List<MinorType> newSubTypes = new ArrayList<>(subTypes);
-    newSubTypes.add(type);
-    majorType =  new MajorType(this.majorType.getMinorType(), this.majorType.getMode(), this.majorType.getPrecision(),
-            this.majorType.getScale(), this.majorType.getTimezone(), newSubTypes);
-    field = MaterializedField.create(field.getName(), majorType);
-    if (callBack != null) {
-      callBack.doWork();
-    }
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.UNION;
   }
 
-  private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
-
   public MapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
-      mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class);
-      addSubType(MinorType.MAP);
+      mapVector = internalMap.addOrGet("map", MinorType.MAP, MapVector.class);
       if (internalMap.size() > vectorCount) {
         mapVector.allocateNew();
+        if (callBack != null) {
+          callBack.doWork();
+        }
       }
     }
     return mapVector;
@@ -121,15 +112,16 @@ public MapVector getMap() {
   <#if !minor.class?starts_with("Decimal")>
 
   private Nullable${name}Vector ${uncappedName}Vector;
-  private static final MajorType ${name?upper_case}_TYPE = new MajorType(MinorType.${name?upper_case}, DataMode.OPTIONAL);
 
   public Nullable${name}Vector get${name}Vector() {
     if (${uncappedName}Vector == null) {
       int vectorCount = internalMap.size();
-      ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class);
-      addSubType(MinorType.${name?upper_case});
+      ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", MinorType.${name?upper_case}, Nullable${name}Vector.class);
       if (internalMap.size() > vectorCount) {
         ${uncappedName}Vector.allocateNew();
+        if (callBack != null) {
+          callBack.doWork();
+        }
       }
     }
     return ${uncappedName}Vector;
@@ -139,15 +131,15 @@ public MapVector getMap() {
 
   </#list></#list>
 
-  private static final MajorType LIST_TYPE = new MajorType(MinorType.LIST, DataMode.OPTIONAL);
-
   public ListVector getList() {
     if (listVector == null) {
       int vectorCount = internalMap.size();
-      listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class);
-      addSubType(MinorType.LIST);
+      listVector = internalMap.addOrGet("list", MinorType.LIST, ListVector.class);
       if (internalMap.size() > vectorCount) {
         listVector.allocateNew();
+        if (callBack != null) {
+          callBack.doWork();
+        }
       }
     }
     return listVector;
@@ -164,6 +156,7 @@ public UInt1Vector getTypeVector() {
   @Override
   public void allocateNew() throws OutOfMemoryException {
     internalMap.allocateNew();
+    typeVector.allocateNew();
     if (typeVector != null) {
       typeVector.zeroVector();
     }
@@ -172,6 +165,7 @@ public void allocateNew() throws OutOfMemoryException {
   @Override
   public boolean allocateNewSafe() {
     boolean safe = internalMap.allocateNewSafe();
+    safe = safe && typeVector.allocateNewSafe();
     if (safe) {
       if (typeVector != null) {
         typeVector.zeroVector();
@@ -196,22 +190,27 @@ public void close() {
 
   @Override
   public void clear() {
+    typeVector.clear();
     internalMap.clear();
   }
 
   @Override
-  public MaterializedField getField() {
-    return field;
+  public Field getField() {
+    List<org.apache.arrow.vector.types.pojo.Field> childFields = new ArrayList<>();
+    for (ValueVector v : internalMap.getChildren()) {
+      childFields.add(v.getField());
+    }
+    return new Field(name, true, new ArrowType.Union(), childFields);
   }
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new TransferImpl(field, allocator);
+    return new TransferImpl(name, allocator);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new TransferImpl(field.withPath(ref), allocator);
+    return new TransferImpl(ref, allocator);
   }
 
   @Override
@@ -219,10 +218,9 @@ public TransferPair makeTransferPair(ValueVector target) {
     return new TransferImpl((UnionVector) target);
   }
 
-  public void transferTo(UnionVector target) {
+  public void transferTo(org.apache.arrow.vector.complex.UnionVector target) {
     internalMap.makeTransferPair(target.internalMap).transfer();
     target.valueCount = valueCount;
-    target.majorType = majorType;
   }
 
   public void copyFrom(int inIndex, int outIndex, UnionVector from) {
@@ -236,13 +234,14 @@ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
   }
 
   public ValueVector addVector(ValueVector v) {
-    String name = v.getField().getType().getMinorType().name().toLowerCase();
-    MajorType type = v.getField().getType();
+    String name = v.getMinorType().name().toLowerCase();
     Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
-    final ValueVector newVector = internalMap.addOrGet(name, type, (Class<ValueVector>) BasicTypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()));
+    final ValueVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass());
     v.makeTransferPair(newVector).transfer();
     internalMap.putChild(name, newVector);
-    addSubType(v.getField().getType().getMinorType());
+    if (callBack != null) {
+      callBack.doWork();
+    }
     return newVector;
   }
 
@@ -250,8 +249,8 @@ private class TransferImpl implements TransferPair {
 
     UnionVector to;
 
-    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
-      to = new UnionVector(field, allocator, null);
+    public TransferImpl(String name, BufferAllocator allocator) {
+      to = new UnionVector(name, allocator, null);
     }
 
     public TransferImpl(UnionVector to) {
@@ -357,7 +356,7 @@ public class Accessor extends BaseValueVector.BaseAccessor {
     public Object getObject(int index) {
       int type = typeVector.getAccessor().get(index);
       switch (MinorType.values()[type]) {
-      case LATE:
+      case NULL:
         return null;
       <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
       <#assign fields = minor.fields!type.fields />
@@ -421,7 +420,7 @@ public void setSafe(int index, UnionHolder holder) {
         writer = new UnionWriter(UnionVector.this);
       }
       writer.setPosition(index);
-      MinorType type = reader.getType().getMinorType();
+      MinorType type = reader.getMinorType();
       switch (type) {
       <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
       <#assign fields = minor.fields!type.fields />
@@ -460,7 +459,7 @@ public void setSafe(int index, Nullable${name}Holder holder) {
     </#list></#list>
 
     public void setType(int index, MinorType type) {
-      typeVector.getMutator().setSafe(index, type.ordinal());
+      typeVector.getMutator().setSafe(index, (byte) type.ordinal());
     }
 
     @Override
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index c9c29e0dd5f92..1137e2cb0207a 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -37,17 +37,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
   private UnionListWriter listWriter;
   private List<BaseWriter> writers = Lists.newArrayList();
 
-  public UnionWriter(BufferAllocator allocator) {
-    super(null);
-  }
-
   public UnionWriter(UnionVector vector) {
-    super(null);
-    data = vector;
-  }
-
-  public UnionWriter(UnionVector vector, FieldWriter parent) {
-    super(null);
     data = vector;
   }
 
@@ -84,7 +74,7 @@ public void endList() {
 
   private MapWriter getMapWriter() {
     if (mapWriter == null) {
-      mapWriter = new SingleMapWriter(data.getMap(), null, true);
+      mapWriter = new SingleMapWriter(data.getMap());
       mapWriter.setPosition(idx());
       writers.add(mapWriter);
     }
@@ -120,7 +110,7 @@ public ListWriter asList() {
 
   private ${name}Writer get${name}Writer() {
     if (${uncappedName}Writer == null) {
-      ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null);
+      ${uncappedName}Writer = new ${name}WriterImpl(data.get${name}Vector());
       ${uncappedName}Writer.setPosition(idx());
       writers.add(${uncappedName}Writer);
     }
@@ -217,7 +207,7 @@ public void close() throws Exception {
   }
 
   @Override
-  public MaterializedField getField() {
+  public Field getField() {
     return data.getField();
   }
 
diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java
index 2b14194574a58..d744c523265f7 100644
--- a/java/vector/src/main/codegen/templates/ValueHolders.java
+++ b/java/vector/src/main/codegen/templates/ValueHolders.java
@@ -31,10 +31,6 @@
 
 public final class ${className} implements ValueHolder{
   
-  public static final MajorType TYPE = new MajorType(MinorType.${minor.class?upper_case}, DataMode.${mode.name?upper_case});
-
-  public MajorType getType() {return TYPE;}
-  
     <#if mode.name == "Repeated">
     
     /** The first index (inclusive) into the Vector. **/
@@ -49,48 +45,13 @@ public final class ${className} implements ValueHolder{
     <#else>
     public static final int WIDTH = ${type.width};
     
-    <#if mode.name == "Optional">public int isSet;</#if>
+    <#if mode.name == "Optional">public int isSet;
+    <#else>public final int isSet = 1;</#if>
     <#assign fields = minor.fields!type.fields />
     <#list fields as field>
     public ${field.type} ${field.name};
     </#list>
     
-    <#if minor.class.startsWith("Decimal")>
-    public static final int maxPrecision = ${minor.maxPrecisionDigits};
-    <#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")>
-    public static final int nDecimalDigits = ${minor.nDecimalDigits};
-    
-    public static int getInteger(int index, int start, ArrowBuf buffer) {
-      int value = buffer.getInt(start + (index * 4));
-
-      if (index == 0) {
-          /* the first byte contains sign bit, return value without it */
-          <#if minor.class.endsWith("Sparse")>
-          value = (value & 0x7FFFFFFF);
-          <#elseif minor.class.endsWith("Dense")>
-          value = (value & 0x0000007F);
-          </#if>
-      }
-      return value;
-    }
-
-    public static void setInteger(int index, int value, int start, ArrowBuf buffer) {
-        buffer.setInt(start + (index * 4), value);
-    }
-  
-    public static void setSign(boolean sign, int start, ArrowBuf buffer) {
-      // Set MSB to 1 if sign is negative
-      if (sign == true) {
-        int value = getInteger(0, start, buffer);
-        setInteger(0, (value | 0x80000000), start, buffer);
-      }
-    }
-  
-    public static boolean getSign(int start, ArrowBuf buffer) {
-      return ((buffer.getInt(start) & 0x80000000) != 0);
-    }
-    </#if></#if>
-    
     @Deprecated
     public int hashCode(){
       throw new UnsupportedOperationException();
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index 84fb3eb55674f..bcd639ab8c30c 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -56,9 +56,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
   private static final int MIN_BYTE_COUNT = 4096;
 
   public final static String OFFSETS_VECTOR_NAME = "$offsets$";
-  private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
-  final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
-  private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
+  final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(OFFSETS_VECTOR_NAME, allocator);
 
   private final Accessor accessor;
   private final Mutator mutator;
@@ -68,16 +66,42 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
   private int allocationSizeInBytes = INITIAL_BYTE_COUNT;
   private int allocationMonitor = 0;
 
-  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
+  <#if minor.class == "Decimal">
+
+  private final int precision;
+  private final int scale;
+
+  public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) {
+    super(name, allocator);
+    this.oAccessor = offsetVector.getAccessor();
+    this.accessor = new Accessor();
+    this.mutator = new Mutator();
+    this.precision = precision;
+    this.scale = scale;
+  }
+  <#else>
+
+  public ${minor.class}Vector(String name, BufferAllocator allocator) {
+    super(name, allocator);
     this.oAccessor = offsetVector.getAccessor();
     this.accessor = new Accessor();
     this.mutator = new Mutator();
   }
+  </#if>
+
+  @Override
+  public Field getField() {
+    throw new UnsupportedOperationException("internal vector");
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.${minor.class?upper_case};
+  }
 
   @Override
   public FieldReader getReader(){
-    return reader;
+    throw new UnsupportedOperationException("internal vector");
   }
 
   @Override
@@ -125,27 +149,6 @@ public int getVarByteLength(){
     return offsetVector.getAccessor().get(valueCount);
   }
 
-//  @Override
-//  public SerializedField getMetadata() {
-//    return getMetadataBuilder() //
-//             .addChild(offsetVector.getMetadata())
-//             .setValueCount(getAccessor().getValueCount()) //
-//             .setBufferLength(getBufferSize()) //
-//             .build();
-//  }
-//
-//  @Override
-//  public void load(SerializedField metadata, ArrowBuf buffer) {
-//     the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
-//    final SerializedField offsetField = metadata.getChild(0);
-//    offsetVector.load(offsetField, buffer);
-//
-//    final int capacity = buffer.capacity();
-//    final int offsetsLength = offsetField.getBufferLength();
-//    data = buffer.slice(offsetsLength, capacity - offsetsLength);
-//    data.retain();
-//  }
-
   @Override
   public void clear() {
     super.clear();
@@ -175,12 +178,12 @@ public long getOffsetAddr(){
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator){
-    return new TransferImpl(getField(), allocator);
+        return new TransferImpl(name, allocator);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    return new TransferImpl(getField().withPath(ref), allocator);
+    return new TransferImpl(ref, allocator);
   }
 
   @Override
@@ -241,8 +244,12 @@ public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector f
   private class TransferImpl implements TransferPair{
     ${minor.class}Vector to;
 
-    public TransferImpl(MaterializedField field, BufferAllocator allocator){
-      to = new ${minor.class}Vector(field, allocator);
+    public TransferImpl(String name, BufferAllocator allocator){
+      <#if minor.class == "Decimal">
+      to = new ${minor.class}Vector(name, allocator, precision, scale);
+      <#else>
+      to = new ${minor.class}Vector(name, allocator);
+      </#if>
     }
 
     public TransferImpl(${minor.class}Vector to){
@@ -426,10 +433,10 @@ public void get(int index, Nullable${minor.class}Holder holder){
       return text;
     }
     <#break>
-    <#case "Var16Char">
+    <#case "Decimal">
     @Override
     public ${friendlyType} getObject(int index) {
-      return new String(get(index), Charsets.UTF_16);
+      return new BigDecimal(new BigInteger(get(index)), scale);
     }
     <#break>
     <#default>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index b129ea9bcb95f..05b7cf1006723 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -20,7 +20,6 @@
 import io.netty.buffer.ArrowBuf;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.types.MaterializedField;
 
 
 public abstract class BaseDataValueVector extends BaseValueVector {
@@ -29,8 +28,8 @@ public abstract class BaseDataValueVector extends BaseValueVector {
 
   protected ArrowBuf data;
 
-  public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
+  public BaseDataValueVector(String name, BufferAllocator allocator) {
+    super(name, allocator);
     data = allocator.getEmpty();
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 932e6f13caf2b..884cdf0910b8e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -19,8 +19,8 @@
 
 import java.util.Iterator;
 
+import com.google.flatbuffers.FlatBufferBuilder;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.types.MaterializedField;
 import org.apache.arrow.vector.util.TransferPair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,16 +38,16 @@ public abstract class BaseValueVector implements ValueVector {
   public static final int INITIAL_VALUE_ALLOCATION = 4096;
 
   protected final BufferAllocator allocator;
-  protected final MaterializedField field;
+  protected final String name;
 
-  protected BaseValueVector(MaterializedField field, BufferAllocator allocator) {
-    this.field = Preconditions.checkNotNull(field, "field cannot be null");
+  protected BaseValueVector(String name, BufferAllocator allocator) {
     this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
+    this.name = name;
   }
 
   @Override
   public String toString() {
-    return super.toString() + "[field = " + field + ", ...]";
+    return super.toString() + "[name = " + name + ", ...]";
   }
 
   @Override
@@ -60,30 +60,11 @@ public void close() {
     clear();
   }
 
-  @Override
-  public MaterializedField getField() {
-    return field;
-  }
-
-  public MaterializedField getField(String ref){
-    return getField().withPath(ref);
-  }
-
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return getTransferPair(getField().getPath(), allocator);
+    return getTransferPair(name, allocator);
   }
 
-//  public static SerializedField getMetadata(BaseValueVector vector) {
-//    return getMetadataBuilder(vector).build();
-//  }
-//
-//  protected static SerializedField.Builder getMetadataBuilder(BaseValueVector vector) {
-//    return SerializedFieldHelper.getAsBuilder(vector.getField())
-//        .setValueCount(vector.getAccessor().getValueCount())
-//        .setBufferLength(vector.getBufferSize());
-//  }
-
   public abstract static class BaseAccessor implements ValueVector.Accessor {
     protected BaseAccessor() { }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index c5bcb2decc43b..fee6e9cdef73d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -21,11 +21,11 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.complex.impl.BitReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.BitHolder;
 import org.apache.arrow.vector.holders.NullableBitHolder;
-import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
@@ -37,7 +37,6 @@
 public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
 
-  private final FieldReader reader = new BitReaderImpl(BitVector.this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
 
@@ -45,13 +44,23 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
   private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
   private int allocationMonitor = 0;
 
-  public BitVector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
+  public BitVector(String name, BufferAllocator allocator) {
+    super(name, allocator);
+  }
+
+  @Override
+  public Field getField() {
+    throw new UnsupportedOperationException("internal vector");
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.BIT;
   }
 
   @Override
   public FieldReader getReader() {
-    return reader;
+    throw new UnsupportedOperationException("internal vector");
   }
 
   @Override
@@ -180,20 +189,6 @@ public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) {
     return true;
   }
 
-//  @Override
-//  public void load(SerializedField metadata, DrillBuf buffer) {
-//    Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
-//    final int valueCount = metadata.getValueCount();
-//    final int expectedLength = getSizeFromCount(valueCount);
-//    final int actualLength = metadata.getBufferLength();
-//    assert expectedLength == actualLength: "expected and actual buffer sizes do not match";
-//
-//    clear();
-//    data = buffer.slice(0, actualLength);
-//    data.retain();
-//    this.valueCount = valueCount;
-//  }
-
   @Override
   public Mutator getMutator() {
     return new Mutator();
@@ -206,12 +201,12 @@ public Accessor getAccessor() {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new TransferImpl(getField(), allocator);
+    return new TransferImpl(name, allocator);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new TransferImpl(getField().withPath(ref), allocator);
+    return new TransferImpl(ref, allocator);
   }
 
   @Override
@@ -270,8 +265,8 @@ public void splitAndTransferTo(int startIndex, int length, BitVector target) {
   private class TransferImpl implements TransferPair {
     BitVector to;
 
-    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
-      this.to = new BitVector(field, allocator);
+    public TransferImpl(String name, BufferAllocator allocator) {
+      this.to = new BitVector(name, allocator);
     }
 
     public TransferImpl(BitVector to) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
deleted file mode 100644
index b806b180e7014..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ObjectHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.util.TransferPair;
-
-public class ObjectVector extends BaseValueVector {
-  private final Accessor accessor = new Accessor();
-  private final Mutator mutator = new Mutator();
-  private int maxCount = 0;
-  private int count = 0;
-  private int allocationSize = 4096;
-
-  private List<Object[]> objectArrayList = new ArrayList<>();
-
-  public ObjectVector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
-  }
-
-  public void addNewArray() {
-    objectArrayList.add(new Object[allocationSize]);
-    maxCount += allocationSize;
-  }
-
-  @Override
-  public FieldReader getReader() {
-    throw new UnsupportedOperationException("ObjectVector does not support this");
-  }
-
-  public final class Mutator implements ValueVector.Mutator {
-
-    public void set(int index, Object obj) {
-      int listOffset = index / allocationSize;
-      if (listOffset >= objectArrayList.size()) {
-        addNewArray();
-      }
-      objectArrayList.get(listOffset)[index % allocationSize] = obj;
-    }
-
-    public boolean setSafe(int index, long value) {
-      set(index, value);
-      return true;
-    }
-
-    protected void set(int index, ObjectHolder holder) {
-      set(index, holder.obj);
-    }
-
-    public boolean setSafe(int index, ObjectHolder holder){
-      set(index, holder);
-      return true;
-    }
-
-    @Override
-    public void setValueCount(int valueCount) {
-      count = valueCount;
-    }
-
-    @Override
-    public void reset() {
-      count = 0;
-      maxCount = 0;
-      objectArrayList = new ArrayList<>();
-      addNewArray();
-    }
-
-    @Override
-    public void generateTestData(int values) {
-    }
-  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    // NoOp
-  }
-
-  @Override
-  public void allocateNew() throws OutOfMemoryException {
-    addNewArray();
-  }
-
-  public void allocateNew(int valueCount) throws OutOfMemoryException {
-    while (maxCount < valueCount) {
-      addNewArray();
-    }
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    allocateNew();
-    return true;
-  }
-
-  @Override
-  public int getBufferSize() {
-    throw new UnsupportedOperationException("ObjectVector does not support this");
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    throw new UnsupportedOperationException("ObjectVector does not support this");
-  }
-
-  @Override
-  public void close() {
-    clear();
-  }
-
-  @Override
-  public void clear() {
-    objectArrayList.clear();
-    maxCount = 0;
-    count = 0;
-  }
-
-  @Override
-  public MaterializedField getField() {
-    return field;
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    throw new UnsupportedOperationException("ObjectVector does not support this");
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    throw new UnsupportedOperationException("ObjectVector does not support this");
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    throw new UnsupportedOperationException("ObjectVector does not support this");
-  }
-
-  @Override
-  public int getValueCapacity() {
-    return maxCount;
-  }
-
-  @Override
-  public Accessor getAccessor() {
-    return accessor;
-  }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    throw new UnsupportedOperationException("ObjectVector does not support this");
-  }
-
-//  @Override
-//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-//    throw new UnsupportedOperationException("ObjectVector does not support this");
-//  }
-//
-//  @Override
-//  public UserBitShared.SerializedField getMetadata() {
-//    throw new UnsupportedOperationException("ObjectVector does not support this");
-//  }
-
-  @Override
-  public Mutator getMutator() {
-    return mutator;
-  }
-
-  @Override
-  public Iterator<ValueVector> iterator() {
-    throw new UnsupportedOperationException("ObjectVector does not support this");
-  }
-
-  public final class Accessor extends BaseAccessor {
-    @Override
-    public Object getObject(int index) {
-      int listOffset = index / allocationSize;
-      if (listOffset >= objectArrayList.size()) {
-        addNewArray();
-      }
-      return objectArrayList.get(listOffset)[index % allocationSize];
-    }
-
-    @Override
-    public int getValueCount() {
-      return count;
-    }
-
-    public Object get(int index) {
-      return getObject(index);
-    }
-
-    public void get(int index, ObjectHolder holder){
-      holder.obj = getObject(index);
-    }
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
deleted file mode 100644
index 61ce285d61b0c..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.math.BigDecimal;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.holders.BigIntHolder;
-import org.apache.arrow.vector.holders.BitHolder;
-import org.apache.arrow.vector.holders.DateHolder;
-import org.apache.arrow.vector.holders.Decimal18Holder;
-import org.apache.arrow.vector.holders.Decimal28SparseHolder;
-import org.apache.arrow.vector.holders.Decimal38SparseHolder;
-import org.apache.arrow.vector.holders.Decimal9Holder;
-import org.apache.arrow.vector.holders.Float4Holder;
-import org.apache.arrow.vector.holders.Float8Holder;
-import org.apache.arrow.vector.holders.IntHolder;
-import org.apache.arrow.vector.holders.IntervalDayHolder;
-import org.apache.arrow.vector.holders.IntervalYearHolder;
-import org.apache.arrow.vector.holders.NullableBitHolder;
-import org.apache.arrow.vector.holders.TimeHolder;
-import org.apache.arrow.vector.holders.TimeStampHolder;
-import org.apache.arrow.vector.holders.VarCharHolder;
-import org.apache.arrow.vector.util.DecimalUtility;
-
-import com.google.common.base.Charsets;
-
-
-public class ValueHolderHelper {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
-
-  public static IntHolder getIntHolder(int value) {
-    IntHolder holder = new IntHolder();
-    holder.value = value;
-
-    return holder;
-  }
-
-  public static BigIntHolder getBigIntHolder(long value) {
-    BigIntHolder holder = new BigIntHolder();
-    holder.value = value;
-
-    return holder;
-  }
-
-  public static Float4Holder getFloat4Holder(float value) {
-    Float4Holder holder = new Float4Holder();
-    holder.value = value;
-
-    return holder;
-  }
-
-  public static Float8Holder getFloat8Holder(double value) {
-    Float8Holder holder = new Float8Holder();
-    holder.value = value;
-
-    return holder;
-  }
-
-  public static DateHolder getDateHolder(long value) {
-    DateHolder holder = new DateHolder();
-    holder.value = value;
-    return holder;
-  }
-
-  public static TimeHolder getTimeHolder(int value) {
-    TimeHolder holder = new TimeHolder();
-    holder.value = value;
-    return holder;
-  }
-
-  public static TimeStampHolder getTimeStampHolder(long value) {
-    TimeStampHolder holder = new TimeStampHolder();
-    holder.value = value;
-    return holder;
-  }
-
-  public static BitHolder getBitHolder(int value) {
-    BitHolder holder = new BitHolder();
-    holder.value = value;
-
-    return holder;
-  }
-
-  public static NullableBitHolder getNullableBitHolder(boolean isNull, int value) {
-    NullableBitHolder holder = new NullableBitHolder();
-    holder.isSet = isNull? 0 : 1;
-    if (! isNull) {
-      holder.value = value;
-    }
-
-    return holder;
-  }
-
-  public static VarCharHolder getVarCharHolder(ArrowBuf buf, String s){
-    VarCharHolder vch = new VarCharHolder();
-
-    byte[] b = s.getBytes(Charsets.UTF_8);
-    vch.start = 0;
-    vch.end = b.length;
-    vch.buffer = buf.reallocIfNeeded(b.length);
-    vch.buffer.setBytes(0, b);
-    return vch;
-  }
-
-  public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
-    VarCharHolder vch = new VarCharHolder();
-
-    byte[] b = s.getBytes(Charsets.UTF_8);
-    vch.start = 0;
-    vch.end = b.length;
-    vch.buffer = a.buffer(b.length); //
-    vch.buffer.setBytes(0, b);
-    return vch;
-  }
-
-
-  public static IntervalYearHolder getIntervalYearHolder(int intervalYear) {
-    IntervalYearHolder holder = new IntervalYearHolder();
-
-    holder.value = intervalYear;
-    return holder;
-  }
-
-  public static IntervalDayHolder getIntervalDayHolder(int days, int millis) {
-      IntervalDayHolder dch = new IntervalDayHolder();
-
-      dch.days = days;
-      dch.milliseconds = millis;
-      return dch;
-  }
-
-  public static Decimal9Holder getDecimal9Holder(int decimal, int scale, int precision) {
-    Decimal9Holder dch = new Decimal9Holder();
-
-    dch.scale = scale;
-    dch.precision = precision;
-    dch.value = decimal;
-
-    return dch;
-  }
-
-  public static Decimal18Holder getDecimal18Holder(long decimal, int scale, int precision) {
-    Decimal18Holder dch = new Decimal18Holder();
-
-    dch.scale = scale;
-    dch.precision = precision;
-    dch.value = decimal;
-
-    return dch;
-  }
-
-  public static Decimal28SparseHolder getDecimal28Holder(ArrowBuf buf, String decimal) {
-
-    Decimal28SparseHolder dch = new Decimal28SparseHolder();
-
-    BigDecimal bigDecimal = new BigDecimal(decimal);
-
-    dch.scale = bigDecimal.scale();
-    dch.precision = bigDecimal.precision();
-    Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
-    dch.start = 0;
-    dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE);
-    DecimalUtility
-        .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
-
-    return dch;
-  }
-
-  public static Decimal38SparseHolder getDecimal38Holder(ArrowBuf buf, String decimal) {
-
-      Decimal38SparseHolder dch = new Decimal38SparseHolder();
-
-      BigDecimal bigDecimal = new BigDecimal(decimal);
-
-      dch.scale = bigDecimal.scale();
-      dch.precision = bigDecimal.precision();
-      Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
-      dch.start = 0;
-    dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE);
-    DecimalUtility
-        .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
-
-      return dch;
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index a170c59abd7cc..35321c947db0b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -24,8 +24,9 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.TransferPair;
+import org.apache.arrow.vector.types.pojo.Field;
 
 /**
  * An abstraction that is used to store a sequence of values in an individual column.
@@ -33,8 +34,7 @@
  * A {@link ValueVector value vector} stores underlying data in-memory in a columnar fashion that is compact and
  * efficient. The column whose data is stored, is referred by {@link #getField()}.
  *
- * A vector when instantiated, relies on a {@link org.apache.drill.exec.record.DeadBuf dead buffer}. It is important
- * that vector is allocated before attempting to read or write.
+ * It is important that vector is allocated before attempting to read or write.
  *
  * There are a few "rules" around vectors:
  *
@@ -94,7 +94,9 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
   /**
    * Get information about how this field is materialized.
    */
-  MaterializedField getField();
+  Field getField();
+
+  MinorType getMinorType();
 
   /**
    * Returns a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
deleted file mode 100644
index fdad99a333258..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector;
-
-import java.util.Collection;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.MajorType;
-
-public class VectorDescriptor {
-  private static final String DEFAULT_NAME = "NONE";
-
-  private final MaterializedField field;
-
-  public VectorDescriptor(final MajorType type) {
-    this(DEFAULT_NAME, type);
-  }
-
-  public VectorDescriptor(final String name, final MajorType type) {
-    this(MaterializedField.create(name, type));
-  }
-
-  public VectorDescriptor(final MaterializedField field) {
-    this.field = Preconditions.checkNotNull(field, "field cannot be null");
-  }
-
-  public MaterializedField getField() {
-    return field;
-  }
-
-  public MajorType getType() {
-    return field.getType();
-  }
-
-  public String getName() {
-    return field.getLastName();
-  }
-
-  public Collection<MaterializedField> getChildren() {
-    return field.getChildren();
-  }
-
-  public boolean hasName() {
-    return getName() != DEFAULT_NAME;
-  }
-
-  public VectorDescriptor withName(final String name) {
-    return new VectorDescriptor(field.withPath(name));
-  }
-
-  public VectorDescriptor withType(final MajorType type) {
-    return new VectorDescriptor(field.withType(type));
-  }
-
-  public static VectorDescriptor create(final String name, final MajorType type) {
-    return new VectorDescriptor(name, type);
-  }
-
-  public static VectorDescriptor create(final MajorType type) {
-    return new VectorDescriptor(type);
-  }
-
-  public static VectorDescriptor create(final MaterializedField field) {
-    return new VectorDescriptor(field);
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index c94e8d1db090c..705a24b02fe78 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -17,18 +17,20 @@
  */
 package org.apache.arrow.vector;
 
+import com.google.flatbuffers.FlatBufferBuilder;
 import io.netty.buffer.ArrowBuf;
 
 import java.util.Collections;
 import java.util.Iterator;
 
+import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.impl.NullReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.TransferPair;
 
 import com.google.common.collect.Iterators;
@@ -36,7 +38,7 @@
 public class ZeroVector implements ValueVector {
   public final static ZeroVector INSTANCE = new ZeroVector();
 
-  private final MaterializedField field = MaterializedField.create("[DEFAULT]", Types.required(MinorType.LATE));
+  private final String name = "[DEFAULT]";
 
   private final TransferPair defaultPair = new TransferPair() {
     @Override
@@ -91,24 +93,21 @@ public void close() { }
   public void clear() { }
 
   @Override
-  public MaterializedField getField() {
-    return field;
+  public Field getField() {
+    return new Field(name, true, new Null(), null);
   }
 
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.NULL;
+  }
+
+
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
     return defaultPair;
   }
 
-//  @Override
-//  public UserBitShared.SerializedField getMetadata() {
-//    return getField()
-//        .getAsBuilder()
-//        .setBufferLength(getBufferSize())
-//        .setValueCount(getAccessor().getValueCount())
-//        .build();
-//  }
-
   @Override
   public Iterator<ValueVector> iterator() {
     return Collections.emptyIterator();
@@ -176,7 +175,4 @@ public Mutator getMutator() {
   public FieldReader getReader() {
     return NullReader.INSTANCE;
   }
-
-//  @Override
-//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 9fae2382ecb24..ed7797576d679 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -21,12 +21,10 @@
 
 import javax.annotation.Nullable;
 
+import org.apache.arrow.flatbuf.Field;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.CallBack;
 
@@ -43,12 +41,12 @@
 public abstract class AbstractContainerVector implements ValueVector {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
 
-  protected MaterializedField field;
+  protected final String name;
   protected final BufferAllocator allocator;
   protected final CallBack callBack;
 
-  protected AbstractContainerVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    this.field = Preconditions.checkNotNull(field);
+  protected AbstractContainerVector(String name, BufferAllocator allocator, CallBack callBack) {
+    this.name = name;
     this.allocator = allocator;
     this.callBack = callBack;
   }
@@ -64,14 +62,6 @@ public BufferAllocator getAllocator() {
     return allocator;
   }
 
-  /**
-   * Returns the field definition of this instance.
-   */
-  @Override
-  public MaterializedField getField() {
-    return field;
-  }
-
   /**
    * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
    */
@@ -79,19 +69,6 @@ public ValueVector getChild(String name) {
     return getChild(name, ValueVector.class);
   }
 
-  /**
-   * Returns a sequence of field names in the order that they show up in the schema.
-   */
-  protected Collection<String> getChildFieldNames() {
-    return Sets.newLinkedHashSet(Iterables.transform(field.getChildren(), new Function<MaterializedField, String>() {
-      @Nullable
-      @Override
-      public String apply(MaterializedField field) {
-        return Preconditions.checkNotNull(field).getLastName();
-      }
-    }));
-  }
-
   /**
    * Clears out all underlying child vectors.
    */
@@ -109,22 +86,6 @@ protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
     throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Arrow doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
   }
 
-  MajorType getLastPathType() {
-    if((this.getField().getType().getMinorType() == MinorType.LIST  &&
-        this.getField().getType().getMode() == DataMode.REPEATED)) {  // Use Repeated scalar type instead of Required List.
-      VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
-      ValueVector v = vord.vector;
-      if (! (v instanceof  AbstractContainerVector)) {
-        return v.getField().getType();
-      }
-    } else if (this.getField().getType().getMinorType() == MinorType.MAP  &&
-        this.getField().getType().getMode() == DataMode.REPEATED) {  // Use Required Map
-      return new MajorType(MinorType.MAP, DataMode.REQUIRED);
-    }
-
-    return this.getField().getType();
-  }
-
   protected boolean supportsDirectRead() {
     return false;
   }
@@ -133,7 +94,7 @@ protected boolean supportsDirectRead() {
   public abstract int size();
 
   // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
-  public abstract <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz);
+  public abstract <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale);
 
   // return the child vector with the input name
   public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index de6ae829b476d..5964f80079141 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -17,17 +17,17 @@
  */
 package org.apache.arrow.vector.complex;
 
+import com.google.common.collect.ImmutableList;
 import io.netty.buffer.ArrowBuf;
 
-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.arrow.flatbuf.Field;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.MapWithOrdinal;
 
@@ -43,17 +43,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
   // Maintains a map with key as field name and value is the vector itself
   private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
 
-  protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    super(field.clone(), allocator, callBack);
-    MaterializedField clonedField = field.clone();
-    // create the hierarchy of the child vectors based on the materialized field
-    for (MaterializedField child : clonedField.getChildren()) {
-      if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
-        final String fieldName = child.getLastName();
-        final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
-        putVector(fieldName, v);
-      }
-    }
+  protected AbstractMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+    super(name, allocator, callBack);
   }
 
   @Override
@@ -109,8 +100,8 @@ public boolean allocateNewSafe() {
    *   </li>
    * </ul>
    *
-   * @param name name of the field
-   * @param type type of the field
+   * @param name the name of the field
+   * @param minorType the minorType for the vector
    * @param clazz class of expected vector type
    * @param <T> class type of expected vector type
    * @throws java.lang.IllegalStateException raised if there is a hard schema change
@@ -118,7 +109,7 @@ public boolean allocateNewSafe() {
    * @return resultant {@link org.apache.arrow.vector.ValueVector}
    */
   @Override
-  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+  public <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
     final ValueVector existing = getChild(name);
     boolean create = false;
     if (existing == null) {
@@ -130,7 +121,7 @@ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T>
       create = true;
     }
     if (create) {
-      final T vector = (T) BasicTypeHelper.getNewVector(name, allocator, type, callBack);
+      final T vector = (T) minorType.getNewVector(name, allocator, callBack, precisionScale);
       putChild(name, vector);
       if (callBack!=null) {
         callBack.doWork();
@@ -177,7 +168,6 @@ public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
    */
   protected void putChild(String name, ValueVector vector) {
     putVector(name, vector);
-    field.addChild(vector.getField());
   }
 
   /**
@@ -199,8 +189,21 @@ protected void putVector(String name, ValueVector vector) {
   /**
    * Returns a sequence of underlying child vectors.
    */
-  protected Collection<ValueVector> getChildren() {
-    return vectors.values();
+  protected List<ValueVector> getChildren() {
+    int size = vectors.size();
+    List<ValueVector> children = new ArrayList<>();
+    for (int i = 0; i < size; i++) {
+      children.add(vectors.getByOrdinal(i));
+    }
+    return children;
+  }
+
+  protected List<String> getChildFieldNames() {
+    ImmutableList.Builder<String> builder = ImmutableList.builder();
+    for (ValueVector child : getChildren()) {
+      builder.add(child.getField().getName());
+    }
+    return builder.build();
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 6518897fb780d..42262741df93d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -22,22 +22,18 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
 import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
-import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ObjectArrays;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
@@ -45,19 +41,16 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
   public final static String OFFSETS_VECTOR_NAME = "$offsets$";
   public final static String DATA_VECTOR_NAME = "$data$";
 
-  public final static MaterializedField OFFSETS_FIELD =
-    MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
-
   protected final UInt4Vector offsets;
   protected ValueVector vector;
 
-  protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
-    this(field, allocator, DEFAULT_DATA_VECTOR);
+  protected BaseRepeatedValueVector(String name, BufferAllocator allocator) {
+    this(name, allocator, DEFAULT_DATA_VECTOR);
   }
 
-  protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
-    super(field, allocator);
-    this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
+  protected BaseRepeatedValueVector(String name, BufferAllocator allocator, ValueVector vector) {
+    super(name, allocator);
+    this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
     this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
   }
 
@@ -109,13 +102,6 @@ public int getValueCapacity() {
     return Math.min(vector.getValueCapacity(), offsetValueCapacity);
   }
 
-//  @Override
-//  protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
-//    return super.getMetadataBuilder()
-//        .addChild(offsets.getMetadata())
-//        .addChild(vector.getMetadata());
-//  }
-
   @Override
   public int getBufferSize() {
     if (getAccessor().getValueCount() == 0) {
@@ -157,47 +143,24 @@ public ArrowBuf[] getBuffers(boolean clear) {
     return buffers;
   }
 
-//  @Override
-//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-//    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
-//    offsets.load(offsetMetadata, buffer);
-//
-//    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
-//    if (getDataVector() == DEFAULT_DATA_VECTOR) {
-//      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
-//    }
-//
-//    final int offsetLength = offsetMetadata.getBufferLength();
-//    final int vectorLength = vectorMetadata.getBufferLength();
-//    vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
-//  }
-
   /**
    * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
-   *
-   * @see {@link ContainerVectorLike#size}
    */
-  @Override
   public int size() {
     return vector == DEFAULT_DATA_VECTOR ? 0:1;
   }
 
-  @Override
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
     boolean created = false;
-    if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != MinorType.LATE) {
-      final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
-      vector = BasicTypeHelper.getNewVector(field, allocator);
+    if (vector instanceof ZeroVector) {
+      vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, null);
       // returned vector must have the same field
-      assert field.equals(vector.getField());
-      getField().addChild(field);
       created = true;
     }
 
-    final MajorType actual = vector.getField().getType();
-    if (!actual.equals(descriptor.getType())) {
+    if (vector.getField().getType().getTypeType() != minorType.getType().getTypeType()) {
       final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
-          descriptor.getType(), actual);
+          Type.name(minorType.getType().getTypeType()), Type.name(vector.getField().getType().getTypeType()));
       throw new SchemaChangeRuntimeException(msg);
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
deleted file mode 100644
index 655b55a6aa2c6..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-
-/**
- * A mix-in used for introducing container vector-like behaviour.
- */
-public interface ContainerVectorLike {
-
-  /**
-   * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
-   *
-   * @param  descriptor vector descriptor
-   * @return  result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
-   * @throws org.apache.arrow.vector.util.SchemaChangeRuntimeException
-   *    if schema change is not permissible between the given and existing data vector types.
-   */
-  <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
-
-  /**
-   * Returns the number of child vectors in this container vector-like instance.
-   */
-  int size();
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 3e60c76802380..c6c6b090db6b0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -18,6 +18,8 @@
  ******************************************************************************/
 package org.apache.arrow.vector.complex;
 
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
 import io.netty.buffer.ArrowBuf;
 
 import java.util.List;
@@ -28,17 +30,14 @@
 import org.apache.arrow.vector.UInt1Vector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.TransferPair;
@@ -55,11 +54,10 @@ public class ListVector extends BaseRepeatedValueVector {
   private UnionListReader reader;
   private CallBack callBack;
 
-  public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    super(field, allocator);
-    this.bits = new UInt1Vector(MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED)), allocator);
+  public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
+    super(name, allocator);
+    this.bits = new UInt1Vector("$bits$", allocator);
     offsets = getOffsetVector();
-    this.field.addChild(getDataVector().getField());
     this.writer = new UnionListWriter(this);
     this.reader = new UnionListReader(this);
     this.callBack = callBack;
@@ -75,15 +73,6 @@ public void allocateNew() throws OutOfMemoryException {
     bits.allocateNewSafe();
   }
 
-  public void transferTo(ListVector target) {
-    offsets.makeTransferPair(target.offsets).transfer();
-    bits.makeTransferPair(target.bits).transfer();
-    if (target.getDataVector() instanceof ZeroVector) {
-      target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
-    }
-    getDataVector().makeTransferPair(target.getDataVector()).transfer();
-  }
-
   public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
     copyFrom(inIndex, outIndex, from);
   }
@@ -103,7 +92,7 @@ public ValueVector getDataVector() {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new TransferImpl(field.withPath(ref), allocator);
+    return new TransferImpl(ref, allocator);
   }
 
   @Override
@@ -114,20 +103,28 @@ public TransferPair makeTransferPair(ValueVector target) {
   private class TransferImpl implements TransferPair {
 
     ListVector to;
+    TransferPair pairs[] = new TransferPair[3];
 
-    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
-      to = new ListVector(field, allocator, null);
-      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+    public TransferImpl(String name, BufferAllocator allocator) {
+      this(new ListVector(name, allocator, null));
     }
 
     public TransferImpl(ListVector to) {
       this.to = to;
-      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+      to.addOrGetVector(vector.getMinorType());
+      pairs[0] = offsets.makeTransferPair(to.offsets);
+      pairs[1] = bits.makeTransferPair(to.bits);
+      if (to.getDataVector() instanceof ZeroVector) {
+        to.addOrGetVector(vector.getMinorType());
+      }
+      pairs[2] = getDataVector().makeTransferPair(to.getDataVector());
     }
 
     @Override
     public void transfer() {
-      transferTo(to);
+      for (TransferPair pair : pairs) {
+        pair.transfer();
+      }
     }
 
     @Override
@@ -190,17 +187,8 @@ public boolean allocateNewSafe() {
     return success;
   }
 
-//  @Override
-//  protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
-//    return getField().getAsBuilder()
-//            .setValueCount(getAccessor().getValueCount())
-//            .setBufferLength(getBufferSize())
-//            .addChild(offsets.getMetadata())
-//            .addChild(bits.getMetadata())
-//            .addChild(vector.getMetadata());
-//  }
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
-    AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
+    AddOrGetResult<T> result = super.addOrGetVector(minorType);
     reader = new UnionListReader(this);
     return result;
   }
@@ -213,6 +201,17 @@ public int getBufferSize() {
     return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize();
   }
 
+  @Override
+  public Field getField() {
+    return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
+            ImmutableList.of(getDataVector().getField()));
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.LIST;
+  }
+
   @Override
   public void clear() {
     offsets.clear();
@@ -235,28 +234,8 @@ public ArrowBuf[] getBuffers(boolean clear) {
     return buffers;
   }
 
-//  @Override
-//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-//    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
-//    offsets.load(offsetMetadata, buffer);
-//
-//    final int offsetLength = offsetMetadata.getBufferLength();
-//    final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
-//    final int bitLength = bitMetadata.getBufferLength();
-//    bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
-//
-//    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
-//    if (getDataVector() == DEFAULT_DATA_VECTOR) {
-//      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
-//    }
-//
-//    final int vectorLength = vectorMetadata.getBufferLength();
-//    vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
-//  }
-
   public UnionVector promoteToUnion() {
-    MaterializedField newField = MaterializedField.create(getField().getPath(), new MajorType(MinorType.UNION, DataMode.OPTIONAL));
-    UnionVector vector = new UnionVector(newField, allocator, null);
+    UnionVector vector = new UnionVector(name, allocator, null);
     replaceDataVector(vector);
     reader = new UnionListReader(this);
     return vector;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index cc0953a1af8ba..0cb613e2f7acf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -19,8 +19,10 @@
 
 import io.netty.buffer.ArrowBuf;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 import javax.annotation.Nullable;
@@ -28,14 +30,13 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector.MapSingleCopier;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.TransferPair;
@@ -47,19 +48,13 @@
 public class MapVector extends AbstractMapVector {
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
-  public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
-
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
   int valueCount;
 
-  public MapVector(String path, BufferAllocator allocator, CallBack callBack){
-    this(MaterializedField.create(path, TYPE), allocator, callBack);
-  }
-
-  public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
-    super(field, allocator, callBack);
+  public MapVector(String name, BufferAllocator allocator, CallBack callBack){
+    super(name, allocator, callBack);
   }
 
   @Override
@@ -69,7 +64,6 @@ public FieldReader getReader() {
   }
 
   transient private MapTransferPair ephPair;
-  transient private MapSingleCopier ephPair2;
 
   public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
     if(ephPair == null || ephPair.from != from) {
@@ -78,13 +72,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
     ephPair.copyValueSafe(fromIndex, thisIndex);
   }
 
-  public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
-    if(ephPair2 == null || ephPair2.from != from) {
-      ephPair2 = from.makeSingularCopier(this);
-    }
-    ephPair2.copySafe(fromSubIndex, thisIndex);
-  }
-
   @Override
   protected boolean supportsDirectRead() {
     return true;
@@ -139,7 +126,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new MapTransferPair(this, getField().getPath(), allocator);
+    return new MapTransferPair(this, name, allocator);
   }
 
   @Override
@@ -157,8 +144,8 @@ protected static class MapTransferPair implements TransferPair{
     private final MapVector from;
     private final MapVector to;
 
-    public MapTransferPair(MapVector from, String path, BufferAllocator allocator) {
-      this(from, new MapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
+    public MapTransferPair(MapVector from, String name, BufferAllocator allocator) {
+      this(from, new MapVector(name, allocator, from.callBack), false);
     }
 
     public MapTransferPair(MapVector from, MapVector to) {
@@ -170,7 +157,6 @@ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
       this.to = to;
       this.pairs = new TransferPair[from.size()];
       this.to.ephPair = null;
-      this.to.ephPair2 = null;
 
       int i = 0;
       ValueVector vector;
@@ -189,7 +175,7 @@ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
         // (This is similar to what happens in ScanBatch where the children cannot be added till they are
         // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
         // include the hashCode of the children but is based only on MaterializedField$key.
-        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+        final ValueVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass());
         if (allocate && to.size() != preSize) {
           newVector.allocateNew();
         }
@@ -251,46 +237,6 @@ public Accessor getAccessor() {
     return accessor;
   }
 
-//  @Override
-//  public void load(SerializedField metadata, DrillBuf buf) {
-//    final List<SerializedField> fields = metadata.getChildList();
-//    valueCount = metadata.getValueCount();
-//
-//    int bufOffset = 0;
-//    for (final SerializedField child : fields) {
-//      final MaterializedField fieldDef = SerializedFieldHelper.create(child);
-//
-//      ValueVector vector = getChild(fieldDef.getLastName());
-//      if (vector == null) {
-//         if we arrive here, we didn't have a matching vector.
-//        vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
-//        putChild(fieldDef.getLastName(), vector);
-//      }
-//      if (child.getValueCount() == 0) {
-//        vector.clear();
-//      } else {
-//        vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
-//      }
-//      bufOffset += child.getBufferLength();
-//    }
-//
-//    assert bufOffset == buf.capacity();
-//  }
-//
-//  @Override
-//  public SerializedField getMetadata() {
-//    SerializedField.Builder b = getField() //
-//        .getAsBuilder() //
-//        .setBufferLength(getBufferSize()) //
-//        .setValueCount(valueCount);
-//
-//
-//    for(ValueVector v : getChildren()) {
-//      b.addChild(v.getMetadata());
-//    }
-//    return b.build();
-//  }
-
   @Override
   public Mutator getMutator() {
     return mutator;
@@ -303,13 +249,6 @@ public Object getObject(int index) {
       Map<String, Object> vv = new JsonStringHashMap<>();
       for (String child:getChildFieldNames()) {
         ValueVector v = getChild(child);
-        // TODO(DRILL-4001):  Resolve this hack:
-        // The index/value count check in the following if statement is a hack
-        // to work around the current fact that RecordBatchLoader.load and
-        // MapVector.load leave child vectors with a length of zero (as opposed
-        // to matching the lengths of siblings and the parent map vector)
-        // because they don't remove (or set the lengths of) vectors from
-        // previous batches that aren't in the current batch.
         if (v != null && index < v.getAccessor().getValueCount()) {
           Object value = v.getAccessor().getObject(index);
           if (value != null) {
@@ -360,6 +299,20 @@ public void clear() {
     valueCount = 0;
   }
 
+  @Override
+  public Field getField() {
+    List<Field> children = new ArrayList<>();
+    for (ValueVector child : getChildren()) {
+      children.add(child.getField());
+    }
+    return new Field(name, false, Tuple.INSTANCE, children);
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.MAP;
+  }
+
   @Override
   public void close() {
     final Collection<ValueVector> vectors = getChildren();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
deleted file mode 100644
index f337f9c4a60e0..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-import org.apache.arrow.vector.complex.impl.NullReader;
-import org.apache.arrow.vector.complex.impl.RepeatedListReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.holders.RepeatedListHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class RepeatedListVector extends AbstractContainerVector
-    implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
-
-  public final static MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
-  private final RepeatedListReaderImpl reader = new RepeatedListReaderImpl(null, this);
-  final DelegateRepeatedVector delegate;
-
-  protected static class DelegateRepeatedVector extends BaseRepeatedValueVector {
-
-    private final RepeatedListAccessor accessor = new RepeatedListAccessor();
-    private final RepeatedListMutator mutator = new RepeatedListMutator();
-    private final EmptyValuePopulator emptyPopulator;
-    private transient DelegateTransferPair ephPair;
-
-    public class RepeatedListAccessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
-
-      @Override
-      public Object getObject(int index) {
-        final List<Object> list = new JsonStringArrayList<>();
-        final int start = offsets.getAccessor().get(index);
-        final int until = offsets.getAccessor().get(index+1);
-        for (int i = start; i < until; i++) {
-          list.add(vector.getAccessor().getObject(i));
-        }
-        return list;
-      }
-
-      public void get(int index, RepeatedListHolder holder) {
-        assert index <= getValueCapacity();
-        holder.start = getOffsetVector().getAccessor().get(index);
-        holder.end = getOffsetVector().getAccessor().get(index+1);
-      }
-
-      public void get(int index, ComplexHolder holder) {
-        final FieldReader reader = getReader();
-        reader.setPosition(index);
-        holder.reader = reader;
-      }
-
-      public void get(int index, int arrayIndex, ComplexHolder holder) {
-        final RepeatedListHolder listHolder = new RepeatedListHolder();
-        get(index, listHolder);
-        int offset = listHolder.start + arrayIndex;
-        if (offset >= listHolder.end) {
-          holder.reader = NullReader.INSTANCE;
-        } else {
-          FieldReader r = getDataVector().getReader();
-          r.setPosition(offset);
-          holder.reader = r;
-        }
-      }
-    }
-
-    public class RepeatedListMutator extends BaseRepeatedValueVector.BaseRepeatedMutator {
-
-      public int add(int index) {
-        final int curEnd = getOffsetVector().getAccessor().get(index+1);
-        getOffsetVector().getMutator().setSafe(index + 1, curEnd + 1);
-        return curEnd;
-      }
-
-      @Override
-      public void startNewValue(int index) {
-        emptyPopulator.populate(index+1);
-        super.startNewValue(index);
-      }
-
-      @Override
-      public void setValueCount(int valueCount) {
-        emptyPopulator.populate(valueCount);
-        super.setValueCount(valueCount);
-      }
-    }
-
-
-    public class DelegateTransferPair implements TransferPair {
-      private final DelegateRepeatedVector target;
-      private final TransferPair[] children;
-
-      public DelegateTransferPair(DelegateRepeatedVector target) {
-        this.target = Preconditions.checkNotNull(target);
-        if (target.getDataVector() == DEFAULT_DATA_VECTOR) {
-          target.addOrGetVector(VectorDescriptor.create(getDataVector().getField()));
-          target.getDataVector().allocateNew();
-        }
-        this.children = new TransferPair[] {
-            getOffsetVector().makeTransferPair(target.getOffsetVector()),
-            getDataVector().makeTransferPair(target.getDataVector())
-        };
-      }
-
-      @Override
-      public void transfer() {
-        for (TransferPair child:children) {
-          child.transfer();
-        }
-      }
-
-      @Override
-      public ValueVector getTo() {
-        return target;
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-        target.allocateNew();
-        for (int i = 0; i < length; i++) {
-          copyValueSafe(startIndex + i, i);
-        }
-      }
-
-      @Override
-      public void copyValueSafe(int srcIndex, int destIndex) {
-        final RepeatedListHolder holder = new RepeatedListHolder();
-        getAccessor().get(srcIndex, holder);
-        target.emptyPopulator.populate(destIndex+1);
-        final TransferPair vectorTransfer = children[1];
-        int newIndex = target.getOffsetVector().getAccessor().get(destIndex);
-        //todo: make this a bulk copy.
-        for (int i = holder.start; i < holder.end; i++, newIndex++) {
-          vectorTransfer.copyValueSafe(i, newIndex);
-        }
-        target.getOffsetVector().getMutator().setSafe(destIndex + 1, newIndex);
-      }
-    }
-
-    public DelegateRepeatedVector(String path, BufferAllocator allocator) {
-      this(MaterializedField.create(path, TYPE), allocator);
-    }
-
-    public DelegateRepeatedVector(MaterializedField field, BufferAllocator allocator) {
-      super(field, allocator);
-      emptyPopulator = new EmptyValuePopulator(getOffsetVector());
-    }
-
-    @Override
-    public void allocateNew() throws OutOfMemoryException {
-      if (!allocateNewSafe()) {
-        throw new OutOfMemoryException();
-      }
-    }
-
-    @Override
-    public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      return makeTransferPair(new DelegateRepeatedVector(ref, allocator));
-    }
-
-    @Override
-    public TransferPair makeTransferPair(ValueVector target) {
-      return new DelegateTransferPair(DelegateRepeatedVector.class.cast(target));
-    }
-
-    @Override
-    public RepeatedListAccessor getAccessor() {
-      return accessor;
-    }
-
-    @Override
-    public RepeatedListMutator getMutator() {
-      return mutator;
-    }
-
-    @Override
-    public FieldReader getReader() {
-      throw new UnsupportedOperationException();
-    }
-
-    public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) {
-      if(ephPair == null || ephPair.target != from) {
-        ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this));
-      }
-      ephPair.copyValueSafe(fromIndex, thisIndex);
-    }
-
-  }
-
-  protected class RepeatedListTransferPair implements TransferPair {
-    private final TransferPair delegate;
-
-    public RepeatedListTransferPair(TransferPair delegate) {
-      this.delegate = delegate;
-    }
-
-    public void transfer() {
-      delegate.transfer();
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      delegate.splitAndTransfer(startIndex, length);
-    }
-
-    @Override
-    public ValueVector getTo() {
-      final DelegateRepeatedVector delegateVector = DelegateRepeatedVector.class.cast(delegate.getTo());
-      return new RepeatedListVector(getField(), allocator, callBack, delegateVector);
-    }
-
-    @Override
-    public void copyValueSafe(int from, int to) {
-      delegate.copyValueSafe(from, to);
-    }
-  }
-
-  public RepeatedListVector(String path, BufferAllocator allocator, CallBack callBack) {
-    this(MaterializedField.create(path, TYPE), allocator, callBack);
-  }
-
-  public RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    this(field, allocator, callBack, new DelegateRepeatedVector(field, allocator));
-  }
-
-  protected RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack, DelegateRepeatedVector delegate) {
-    super(field, allocator, callBack);
-    this.delegate = Preconditions.checkNotNull(delegate);
-
-    final List<MaterializedField> children = Lists.newArrayList(field.getChildren());
-    final int childSize = children.size();
-    assert childSize < 3;
-    final boolean hasChild = childSize > 0;
-    if (hasChild) {
-      // the last field is data field
-      final MaterializedField child = children.get(childSize-1);
-      addOrGetVector(VectorDescriptor.create(child));
-    }
-  }
-
-
-    @Override
-  public RepeatedListReaderImpl getReader() {
-    return reader;
-  }
-
-  @Override
-  public DelegateRepeatedVector.RepeatedListAccessor getAccessor() {
-    return delegate.getAccessor();
-  }
-
-  @Override
-  public DelegateRepeatedVector.RepeatedListMutator getMutator() {
-    return delegate.getMutator();
-  }
-
-  @Override
-  public UInt4Vector getOffsetVector() {
-    return delegate.getOffsetVector();
-  }
-
-  @Override
-  public ValueVector getDataVector() {
-    return delegate.getDataVector();
-  }
-
-  @Override
-  public void allocateNew() throws OutOfMemoryException {
-    delegate.allocateNew();
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    return delegate.allocateNewSafe();
-  }
-
-  @Override
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
-    final AddOrGetResult<T> result = delegate.addOrGetVector(descriptor);
-    if (result.isCreated() && callBack != null) {
-      callBack.doWork();
-    }
-    return result;
-  }
-
-  @Override
-  public int size() {
-    return delegate.size();
-  }
-
-  @Override
-  public int getBufferSize() {
-    return delegate.getBufferSize();
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    return delegate.getBufferSizeFor(valueCount);
-  }
-
-  @Override
-  public void close() {
-    delegate.close();
-  }
-
-  @Override
-  public void clear() {
-    delegate.clear();
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new RepeatedListTransferPair(delegate.getTransferPair(allocator));
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new RepeatedListTransferPair(delegate.getTransferPair(ref, allocator));
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    final RepeatedListVector target = RepeatedListVector.class.cast(to);
-    return new RepeatedListTransferPair(delegate.makeTransferPair(target.delegate));
-  }
-
-  @Override
-  public int getValueCapacity() {
-    return delegate.getValueCapacity();
-  }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    return delegate.getBuffers(clear);
-  }
-
-
-//  @Override
-//  public void load(SerializedField metadata, DrillBuf buf) {
-//    delegate.load(metadata, buf);
-//  }
-
-//  @Override
-//  public SerializedField getMetadata() {
-//    return delegate.getMetadata();
-//  }
-
-  @Override
-  public Iterator<ValueVector> iterator() {
-    return delegate.iterator();
-  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    delegate.setInitialCapacity(numRecords);
-  }
-
-  /**
-   * @deprecated
-   *   prefer using {@link #addOrGetVector(org.apache.arrow.vector.VectorDescriptor)} instead.
-   */
-  @Override
-  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
-    final AddOrGetResult<T> result = addOrGetVector(VectorDescriptor.create(type));
-    return result.getVector();
-  }
-
-  @Override
-  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
-    if (name != null) {
-      return null;
-    }
-    return typeify(delegate.getDataVector(), clazz);
-  }
-
-  @Override
-  public void allocateNew(int valueCount, int innerValueCount) {
-    clear();
-    getOffsetVector().allocateNew(valueCount + 1);
-    getMutator().reset();
-  }
-
-  @Override
-  public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
-    if (name != null) {
-      return null;
-    }
-    return new VectorWithOrdinal(delegate.getDataVector(), 0);
-  }
-
-  public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) {
-    delegate.copyFromSafe(fromIndex, thisIndex, from.delegate);
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
deleted file mode 100644
index 686414e71cadf..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
+++ /dev/null
@@ -1,584 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.AllocationHelper;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-import org.apache.arrow.vector.complex.impl.NullReader;
-import org.apache.arrow.vector.complex.impl.RepeatedMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-import org.apache.commons.lang3.ArrayUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-public class RepeatedMapVector extends AbstractMapVector
-    implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
-  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
-
-  public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.REPEATED);
-
-  final UInt4Vector offsets;   // offsets to start of each record (considering record indices are 0-indexed)
-  private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
-  private final RepeatedMapAccessor accessor = new RepeatedMapAccessor();
-  private final Mutator mutator = new Mutator();
-  private final EmptyValuePopulator emptyPopulator;
-
-  public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
-    super(field, allocator, callBack);
-    this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator);
-    this.emptyPopulator = new EmptyValuePopulator(offsets);
-  }
-
-  @Override
-  public UInt4Vector getOffsetVector() {
-    return offsets;
-  }
-
-  @Override
-  public ValueVector getDataVector() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    offsets.setInitialCapacity(numRecords + 1);
-    for(final ValueVector v : (Iterable<ValueVector>) this) {
-      v.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
-    }
-  }
-
-  @Override
-  public RepeatedMapReaderImpl getReader() {
-    return reader;
-  }
-
-  @Override
-  public void allocateNew(int groupCount, int innerValueCount) {
-    clear();
-    try {
-      offsets.allocateNew(groupCount + 1);
-      for (ValueVector v : getChildren()) {
-        AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
-      }
-    } catch (OutOfMemoryException e){
-      clear();
-      throw e;
-    }
-    offsets.zeroVector();
-    mutator.reset();
-  }
-
-  public Iterator<String> fieldNameIterator() {
-    return getChildFieldNames().iterator();
-  }
-
-  @Override
-  public List<ValueVector> getPrimitiveVectors() {
-    final List<ValueVector> primitiveVectors = super.getPrimitiveVectors();
-    primitiveVectors.add(offsets);
-    return primitiveVectors;
-  }
-
-  @Override
-  public int getBufferSize() {
-    if (getAccessor().getValueCount() == 0) {
-      return 0;
-    }
-    long bufferSize = offsets.getBufferSize();
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      bufferSize += v.getBufferSize();
-    }
-    return (int) bufferSize;
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
-
-    long bufferSize = 0;
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      bufferSize += v.getBufferSizeFor(valueCount);
-    }
-
-    return (int) bufferSize;
-  }
-
-  @Override
-  public void close() {
-    offsets.close();
-    super.close();
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new RepeatedMapTransferPair(this, getField().getPath(), allocator);
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new RepeatedMapTransferPair(this, (RepeatedMapVector)to);
-  }
-
-  MapSingleCopier makeSingularCopier(MapVector to) {
-    return new MapSingleCopier(this, to);
-  }
-
-  protected static class MapSingleCopier {
-    private final TransferPair[] pairs;
-    public final RepeatedMapVector from;
-
-    public MapSingleCopier(RepeatedMapVector from, MapVector to) {
-      this.from = from;
-      this.pairs = new TransferPair[from.size()];
-
-      int i = 0;
-      ValueVector vector;
-      for (final String child:from.getChildFieldNames()) {
-        int preSize = to.size();
-        vector = from.getChild(child);
-        if (vector == null) {
-          continue;
-        }
-        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
-        if (to.size() != preSize) {
-          newVector.allocateNew();
-        }
-        pairs[i++] = vector.makeTransferPair(newVector);
-      }
-    }
-
-    public void copySafe(int fromSubIndex, int toIndex) {
-      for (TransferPair p : pairs) {
-        p.copyValueSafe(fromSubIndex, toIndex);
-      }
-    }
-  }
-
-  public TransferPair getTransferPairToSingleMap(String reference, BufferAllocator allocator) {
-    return new SingleMapTransferPair(this, reference, allocator);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new RepeatedMapTransferPair(this, ref, allocator);
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    /* boolean to keep track if all the memory allocation were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      if (!offsets.allocateNewSafe()) {
-        return false;
-      }
-      success =  super.allocateNewSafe();
-    } finally {
-      if (!success) {
-        clear();
-      }
-    }
-    offsets.zeroVector();
-    return success;
-  }
-
-  protected static class SingleMapTransferPair implements TransferPair {
-    private final TransferPair[] pairs;
-    private final RepeatedMapVector from;
-    private final MapVector to;
-    private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.REQUIRED);
-
-    public SingleMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
-      this(from, new MapVector(MaterializedField.create(path, MAP_TYPE), allocator, from.callBack), false);
-    }
-
-    public SingleMapTransferPair(RepeatedMapVector from, MapVector to) {
-      this(from, to, true);
-    }
-
-    public SingleMapTransferPair(RepeatedMapVector from, MapVector to, boolean allocate) {
-      this.from = from;
-      this.to = to;
-      this.pairs = new TransferPair[from.size()];
-      int i = 0;
-      ValueVector vector;
-      for (final String child : from.getChildFieldNames()) {
-        int preSize = to.size();
-        vector = from.getChild(child);
-        if (vector == null) {
-          continue;
-        }
-        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
-        if (allocate && to.size() != preSize) {
-          newVector.allocateNew();
-        }
-        pairs[i++] = vector.makeTransferPair(newVector);
-      }
-    }
-
-
-    @Override
-    public void transfer() {
-      for (TransferPair p : pairs) {
-        p.transfer();
-      }
-      to.getMutator().setValueCount(from.getAccessor().getValueCount());
-      from.clear();
-    }
-
-    @Override
-    public ValueVector getTo() {
-      return to;
-    }
-
-    @Override
-    public void copyValueSafe(int from, int to) {
-      for (TransferPair p : pairs) {
-        p.copyValueSafe(from, to);
-      }
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      for (TransferPair p : pairs) {
-        p.splitAndTransfer(startIndex, length);
-      }
-      to.getMutator().setValueCount(length);
-    }
-  }
-
-  private static class RepeatedMapTransferPair implements TransferPair{
-
-    private final TransferPair[] pairs;
-    private final RepeatedMapVector to;
-    private final RepeatedMapVector from;
-
-    public RepeatedMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
-      this(from, new RepeatedMapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
-    }
-
-    public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to) {
-      this(from, to, true);
-    }
-
-    public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to, boolean allocate) {
-      this.from = from;
-      this.to = to;
-      this.pairs = new TransferPair[from.size()];
-      this.to.ephPair = null;
-
-      int i = 0;
-      ValueVector vector;
-      for (final String child : from.getChildFieldNames()) {
-        final int preSize = to.size();
-        vector = from.getChild(child);
-        if (vector == null) {
-          continue;
-        }
-
-        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
-        if (to.size() != preSize) {
-          newVector.allocateNew();
-        }
-
-        pairs[i++] = vector.makeTransferPair(newVector);
-      }
-    }
-
-    @Override
-    public void transfer() {
-      from.offsets.transferTo(to.offsets);
-      for (TransferPair p : pairs) {
-        p.transfer();
-      }
-      from.clear();
-    }
-
-    @Override
-    public ValueVector getTo() {
-      return to;
-    }
-
-    @Override
-    public void copyValueSafe(int srcIndex, int destIndex) {
-      RepeatedMapHolder holder = new RepeatedMapHolder();
-      from.getAccessor().get(srcIndex, holder);
-      to.emptyPopulator.populate(destIndex + 1);
-      int newIndex = to.offsets.getAccessor().get(destIndex);
-      //todo: make these bulk copies
-      for (int i = holder.start; i < holder.end; i++, newIndex++) {
-        for (TransferPair p : pairs) {
-          p.copyValueSafe(i, newIndex);
-        }
-      }
-      to.offsets.getMutator().setSafe(destIndex + 1, newIndex);
-    }
-
-    @Override
-    public void splitAndTransfer(final int groupStart, final int groups) {
-      final UInt4Vector.Accessor a = from.offsets.getAccessor();
-      final UInt4Vector.Mutator m = to.offsets.getMutator();
-
-      final int startPos = a.get(groupStart);
-      final int endPos = a.get(groupStart + groups);
-      final int valuesToCopy = endPos - startPos;
-
-      to.offsets.clear();
-      to.offsets.allocateNew(groups + 1);
-
-      int normalizedPos;
-      for (int i = 0; i < groups + 1; i++) {
-        normalizedPos = a.get(groupStart + i) - startPos;
-        m.set(i, normalizedPos);
-      }
-
-      m.setValueCount(groups + 1);
-      to.emptyPopulator.populate(groups);
-
-      for (final TransferPair p : pairs) {
-        p.splitAndTransfer(startPos, valuesToCopy);
-      }
-    }
-  }
-
-
-  transient private RepeatedMapTransferPair ephPair;
-
-  public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) {
-    if (ephPair == null || ephPair.from != from) {
-      ephPair = (RepeatedMapTransferPair) from.makeTransferPair(this);
-    }
-    ephPair.copyValueSafe(fromIndex, thisIndex);
-  }
-
-  @Override
-  public int getValueCapacity() {
-    return Math.max(offsets.getValueCapacity() - 1, 0);
-  }
-
-  @Override
-  public RepeatedMapAccessor getAccessor() {
-    return accessor;
-  }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    final int expectedBufferSize = getBufferSize();
-    final int actualBufferSize = super.getBufferSize();
-
-    Preconditions.checkArgument(expectedBufferSize == actualBufferSize + offsets.getBufferSize());
-    return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
-  }
-
-
-//  @Override
-//  public void load(SerializedField metadata, DrillBuf buffer) {
-//    final List<SerializedField> children = metadata.getChildList();
-//
-//    final SerializedField offsetField = children.get(0);
-//    offsets.load(offsetField, buffer);
-//    int bufOffset = offsetField.getBufferLength();
-//
-//    for (int i = 1; i < children.size(); i++) {
-//      final SerializedField child = children.get(i);
-//      final MaterializedField fieldDef = SerializedFieldHelper.create(child);
-//      ValueVector vector = getChild(fieldDef.getLastName());
-//      if (vector == null) {
-        // if we arrive here, we didn't have a matching vector.
-//        vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
-//        putChild(fieldDef.getLastName(), vector);
-//      }
-//      final int vectorLength = child.getBufferLength();
-//      vector.load(child, buffer.slice(bufOffset, vectorLength));
-//      bufOffset += vectorLength;
-//    }
-//
-//    assert bufOffset == buffer.capacity();
-//  }
-//
-//
-//  @Override
-//  public SerializedField getMetadata() {
-//    SerializedField.Builder builder = getField() //
-//        .getAsBuilder() //
-//        .setBufferLength(getBufferSize()) //
-        // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
-//        .setValueCount(accessor.getValueCount());
-//    builder.addChild(offsets.getMetadata());
-//    for (final ValueVector child : getChildren()) {
-//      builder.addChild(child.getMetadata());
-//    }
-//    return builder.build();
-//  }
-
-  @Override
-  public Mutator getMutator() {
-    return mutator;
-  }
-
-  public class RepeatedMapAccessor implements RepeatedAccessor {
-    @Override
-    public Object getObject(int index) {
-      final List<Object> list = new JsonStringArrayList<>();
-      final int end = offsets.getAccessor().get(index+1);
-      String fieldName;
-      for (int i =  offsets.getAccessor().get(index); i < end; i++) {
-        final Map<String, Object> vv = Maps.newLinkedHashMap();
-        for (final MaterializedField field : getField().getChildren()) {
-          if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
-            fieldName = field.getLastName();
-            final Object value = getChild(fieldName).getAccessor().getObject(i);
-            if (value != null) {
-              vv.put(fieldName, value);
-            }
-          }
-        }
-        list.add(vv);
-      }
-      return list;
-    }
-
-    @Override
-    public int getValueCount() {
-      return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
-    }
-
-    @Override
-    public int getInnerValueCount() {
-      final int valueCount = getValueCount();
-      if (valueCount == 0) {
-        return 0;
-      }
-      return offsets.getAccessor().get(valueCount);
-    }
-
-    @Override
-    public int getInnerValueCountAt(int index) {
-      return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
-    }
-
-    @Override
-    public boolean isEmpty(int index) {
-      return false;
-    }
-
-    @Override
-    public boolean isNull(int index) {
-      return false;
-    }
-
-    public void get(int index, RepeatedMapHolder holder) {
-      assert index < getValueCapacity() :
-        String.format("Attempted to access index %d when value capacity is %d",
-            index, getValueCapacity());
-      final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
-      holder.start = offsetsAccessor.get(index);
-      holder.end = offsetsAccessor.get(index + 1);
-    }
-
-    public void get(int index, ComplexHolder holder) {
-      final FieldReader reader = getReader();
-      reader.setPosition(index);
-      holder.reader = reader;
-    }
-
-    public void get(int index, int arrayIndex, ComplexHolder holder) {
-      final RepeatedMapHolder h = new RepeatedMapHolder();
-      get(index, h);
-      final int offset = h.start + arrayIndex;
-
-      if (offset >= h.end) {
-        holder.reader = NullReader.INSTANCE;
-      } else {
-        reader.setSinglePosition(index, arrayIndex);
-        holder.reader = reader;
-      }
-    }
-  }
-
-  public class Mutator implements RepeatedMutator {
-    @Override
-    public void startNewValue(int index) {
-      emptyPopulator.populate(index + 1);
-      offsets.getMutator().setSafe(index + 1, offsets.getAccessor().get(index));
-    }
-
-    @Override
-    public void setValueCount(int topLevelValueCount) {
-      emptyPopulator.populate(topLevelValueCount);
-      offsets.getMutator().setValueCount(topLevelValueCount == 0 ? 0 : topLevelValueCount + 1);
-      int childValueCount = offsets.getAccessor().get(topLevelValueCount);
-      for (final ValueVector v : getChildren()) {
-        v.getMutator().setValueCount(childValueCount);
-      }
-    }
-
-    @Override
-    public void reset() {}
-
-    @Override
-    public void generateTestData(int values) {}
-
-    public int add(int index) {
-      final int prevEnd = offsets.getAccessor().get(index + 1);
-      offsets.getMutator().setSafe(index + 1, prevEnd + 1);
-      return prevEnd;
-    }
-  }
-
-  @Override
-  public void clear() {
-    getMutator().reset();
-
-    offsets.clear();
-    for(final ValueVector vector : getChildren()) {
-      vector.clear();
-    }
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index 99c0a0aeb1e2c..54db393e8310d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -28,7 +28,7 @@
  * uses the offset vector to determine the sequence of cells pertaining to an individual value.
  *
  */
-public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
+public interface RepeatedValueVector extends ValueVector {
 
   final static int DEFAULT_REPEAT_PER_RECORD = 5;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
index 264e241e73935..259a954233c06 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
@@ -19,20 +19,20 @@
 
 import java.util.Iterator;
 
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.flatbuf.Union;
+import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.holders.UnionHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 
 
 abstract class AbstractBaseReader implements FieldReader{
 
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseReader.class);
-  private static final MajorType LATE_BIND_TYPE = new MajorType(MinorType.LATE, DataMode.OPTIONAL);
 
   private int index;
 
@@ -58,15 +58,6 @@ public Iterator<String> iterator() {
     throw new IllegalStateException("The current reader doesn't support reading as a map.");
   }
 
-  public MajorType getType(){
-    throw new IllegalStateException("The current reader doesn't support getting type information.");
-  }
-
-  @Override
-  public MaterializedField getField() {
-    return MaterializedField.create("unknown", LATE_BIND_TYPE);
-  }
-
   @Override
   public boolean next() {
     throw new IllegalStateException("The current reader doesn't support getting next information.");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
index 4e1e103a12e7c..e6cf098f16f59 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
@@ -23,25 +23,11 @@
 abstract class AbstractBaseWriter implements FieldWriter {
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseWriter.class);
 
-  final FieldWriter parent;
   private int index;
 
-  public AbstractBaseWriter(FieldWriter parent) {
-    this.parent = parent;
-  }
-
   @Override
   public String toString() {
-    return super.toString() + "[index = " + index + ", parent = " + parent + "]";
-  }
-
-  @Override
-  public FieldWriter getParent() {
-    return parent;
-  }
-
-  public boolean isRoot() {
-    return parent == null;
+    return super.toString() + "[index = " + index + "]";
   }
 
   int idx() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 4e2051fd4efee..4d2adfb32561e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -17,20 +17,20 @@
  */
 package org.apache.arrow.vector.complex.impl;
 
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.StateTool;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
 
 import com.google.common.base.Preconditions;
+import org.apache.arrow.vector.types.pojo.Field;
 
 public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
 
   private SingleMapWriter mapRoot;
-  private SingleListWriter listRoot;
+  private UnionListWriter listRoot;
   private final MapVector container;
 
   Mode mode = Mode.INIT;
@@ -40,7 +40,6 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
   private enum Mode { INIT, MAP, LIST };
 
   public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){
-    super(null);
     this.name = name;
     this.container = container;
     this.unionEnabled = unionEnabled;
@@ -51,7 +50,7 @@ public ComplexWriterImpl(String name, MapVector container){
   }
 
   @Override
-  public MaterializedField getField() {
+  public Field getField() {
     return container.getField();
   }
 
@@ -123,7 +122,7 @@ public MapWriter directMap(){
 
     case INIT:
       MapVector map = (MapVector) container;
-      mapRoot = new SingleMapWriter(map, this, unionEnabled);
+      mapRoot = new SingleMapWriter(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
       break;
@@ -143,8 +142,8 @@ public MapWriter rootAsMap() {
     switch(mode){
 
     case INIT:
-      MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
-      mapRoot = new SingleMapWriter(map, this, unionEnabled);
+      MapVector map = container.addOrGet(name, MinorType.MAP, MapVector.class);
+      mapRoot = new SingleMapWriter(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
       break;
@@ -174,7 +173,12 @@ public ListWriter rootAsList() {
     switch(mode){
 
     case INIT:
-      listRoot = new SingleListWriter(name, container, this);
+      int vectorCount = container.size();
+      ListVector listVector = container.addOrGet(name, MinorType.LIST, ListVector.class);
+      if (container.size() > vectorCount) {
+        listVector.allocateNew();
+      }
+      listRoot = new UnionListWriter(listVector);
       listRoot.setPosition(idx());
       mode = Mode.LIST;
       break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 462ec9dd86a9b..586b1283fe879 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -17,20 +17,14 @@
  */
 package org.apache.arrow.vector.complex.impl;
 
-import java.lang.reflect.Constructor;
-
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
@@ -56,14 +50,12 @@ private enum State {
   private FieldWriter writer;
 
   public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
-    super(null);
     this.parentContainer = parentContainer;
     this.listVector = null;
     init(v);
   }
 
   public PromotableWriter(ValueVector v, ListVector listVector) {
-    super(null);
     this.listVector = listVector;
     this.parentContainer = null;
     init(v);
@@ -84,30 +76,8 @@ private void init(ValueVector v) {
   private void setWriter(ValueVector v) {
     state = State.SINGLE;
     vector = v;
-    type = v.getField().getType().getMinorType();
-    Class<?> writerClass = BasicTypeHelper
-        .getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
-    if (writerClass.equals(SingleListWriter.class)) {
-      writerClass = UnionListWriter.class;
-    }
-    Class<?> vectorClass = BasicTypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField()
-        .getDataMode());
-    try {
-      Constructor<?> constructor = null;
-      for (Constructor<?> c : writerClass.getConstructors()) {
-        if (c.getParameterTypes().length == 3) {
-          constructor = c;
-        }
-      }
-      if (constructor == null) {
-        constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class);
-        writer = (FieldWriter) constructor.newInstance(vector, null);
-      } else {
-        writer = (FieldWriter) constructor.newInstance(vector, null, true);
-      }
-    } catch (ReflectiveOperationException e) {
-      throw new RuntimeException(e);
-    }
+    type = v.getMinorType();
+    writer = type.getNewFieldWriter(vector);
   }
 
   @Override
@@ -129,7 +99,7 @@ protected FieldWriter getWriter(MinorType type) {
       if (type == null) {
         return null;
       }
-      ValueVector v = listVector.addOrGetVector(new VectorDescriptor(new MajorType(type, DataMode.OPTIONAL))).getVector();
+      ValueVector v = listVector.addOrGetVector(type).getVector();
       v.allocateNew();
       setWriter(v);
       writer.setPosition(position);
@@ -150,11 +120,11 @@ protected FieldWriter getWriter() {
   }
 
   private FieldWriter promoteToUnion() {
-    String name = vector.getField().getLastName();
-    TransferPair tp = vector.getTransferPair(vector.getField().getType().getMinorType().name().toLowerCase(), vector.getAllocator());
+    String name = vector.getField().getName();
+    TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator());
     tp.transfer();
     if (parentContainer != null) {
-      unionVector = parentContainer.addOrGet(name, new MajorType(MinorType.UNION, DataMode.OPTIONAL), UnionVector.class);
+      unionVector = parentContainer.addOrGet(name, MinorType.UNION, UnionVector.class);
       unionVector.allocateNew();
     } else if (listVector != null) {
       unionVector = listVector.promoteToUnion();
@@ -163,7 +133,7 @@ private FieldWriter promoteToUnion() {
     writer = new UnionWriter(unionVector);
     writer.setPosition(idx());
     for (int i = 0; i < idx(); i++) {
-      unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
+      unionVector.getMutator().setType(i, vector.getMinorType());
     }
     vector = null;
     state = State.UNION;
@@ -181,7 +151,7 @@ public void clear() {
   }
 
   @Override
-  public MaterializedField getField() {
+  public Field getField() {
     return getWriter().getField();
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
deleted file mode 100644
index dd1a152e2f603..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex.impl;
-
-
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedListVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.holders.RepeatedListHolder;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-
-public class RepeatedListReaderImpl extends AbstractFieldReader{
-  private static final int NO_VALUES = Integer.MAX_VALUE - 1;
-  private static final MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
-  private final String name;
-  private final RepeatedListVector container;
-  private FieldReader reader;
-
-  public RepeatedListReaderImpl(String name, RepeatedListVector container) {
-    super();
-    this.name = name;
-    this.container = container;
-  }
-
-  @Override
-  public MajorType getType() {
-    return TYPE;
-  }
-
-  @Override
-  public void copyAsValue(ListWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    RepeatedListWriter impl = (RepeatedListWriter) writer;
-    impl.container.copyFromSafe(idx(), impl.idx(), container);
-  }
-
-  @Override
-  public void copyAsField(String name, MapWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    RepeatedListWriter impl = (RepeatedListWriter) writer.list(name);
-    impl.container.copyFromSafe(idx(), impl.idx(), container);
-  }
-
-  private int currentOffset;
-  private int maxOffset;
-
-  @Override
-  public void reset() {
-    super.reset();
-    currentOffset = 0;
-    maxOffset = 0;
-    if (reader != null) {
-      reader.reset();
-    }
-    reader = null;
-  }
-
-  @Override
-  public int size() {
-    return maxOffset - currentOffset;
-  }
-
-  @Override
-  public void setPosition(int index) {
-    if (index < 0 || index == NO_VALUES) {
-      currentOffset = NO_VALUES;
-      return;
-    }
-
-    super.setPosition(index);
-    RepeatedListHolder h = new RepeatedListHolder();
-    container.getAccessor().get(index, h);
-    if (h.start == h.end) {
-      currentOffset = NO_VALUES;
-    } else {
-      currentOffset = h.start-1;
-      maxOffset = h.end;
-      if(reader != null) {
-        reader.setPosition(currentOffset);
-      }
-    }
-  }
-
-  @Override
-  public boolean next() {
-    if (currentOffset +1 < maxOffset) {
-      currentOffset++;
-      if (reader != null) {
-        reader.setPosition(currentOffset);
-      }
-      return true;
-    } else {
-      currentOffset = NO_VALUES;
-      return false;
-    }
-  }
-
-  @Override
-  public Object readObject() {
-    return container.getAccessor().getObject(idx());
-  }
-
-  @Override
-  public FieldReader reader() {
-    if (reader == null) {
-      ValueVector child = container.getChild(name);
-      if (child == null) {
-        reader = NullReader.INSTANCE;
-      } else {
-        reader = child.getReader();
-      }
-      reader.setPosition(currentOffset);
-    }
-    return reader;
-  }
-
-  public boolean isSet() {
-    return true;
-  }
-
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
deleted file mode 100644
index 09a831d8329fc..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex.impl;
-
-import java.util.Map;
-
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
-import org.apache.arrow.vector.types.Types.MajorType;
-
-import com.google.common.collect.Maps;
-
-@SuppressWarnings("unused")
-public class RepeatedMapReaderImpl extends AbstractFieldReader{
-  private static final int NO_VALUES = Integer.MAX_VALUE - 1;
-
-  private final RepeatedMapVector vector;
-  private final Map<String, FieldReader> fields = Maps.newHashMap();
-
-  public RepeatedMapReaderImpl(RepeatedMapVector vector) {
-    this.vector = vector;
-  }
-
-  private void setChildrenPosition(int index) {
-    for (FieldReader r : fields.values()) {
-      r.setPosition(index);
-    }
-  }
-
-  @Override
-  public FieldReader reader(String name) {
-    FieldReader reader = fields.get(name);
-    if (reader == null) {
-      ValueVector child = vector.getChild(name);
-      if (child == null) {
-        reader = NullReader.INSTANCE;
-      } else {
-        reader = child.getReader();
-      }
-      fields.put(name, reader);
-      reader.setPosition(currentOffset);
-    }
-    return reader;
-  }
-
-  @Override
-  public FieldReader reader() {
-    if (currentOffset == NO_VALUES) {
-      return NullReader.INSTANCE;
-    }
-
-    setChildrenPosition(currentOffset);
-    return new SingleLikeRepeatedMapReaderImpl(vector, this);
-  }
-
-  private int currentOffset;
-  private int maxOffset;
-
-  @Override
-  public void reset() {
-    super.reset();
-    currentOffset = 0;
-    maxOffset = 0;
-    for (FieldReader reader:fields.values()) {
-      reader.reset();
-    }
-    fields.clear();
-  }
-
-  @Override
-  public int size() {
-    if (isNull()) {
-      return 0;
-    }
-    return maxOffset - (currentOffset < 0 ? 0 : currentOffset);
-  }
-
-  @Override
-  public void setPosition(int index) {
-    if (index < 0 || index == NO_VALUES) {
-      currentOffset = NO_VALUES;
-      return;
-    }
-
-    super.setPosition(index);
-    RepeatedMapHolder h = new RepeatedMapHolder();
-    vector.getAccessor().get(index, h);
-    if (h.start == h.end) {
-      currentOffset = NO_VALUES;
-    } else {
-      currentOffset = h.start-1;
-      maxOffset = h.end;
-      setChildrenPosition(currentOffset);
-    }
-  }
-
-  public void setSinglePosition(int index, int childIndex) {
-    super.setPosition(index);
-    RepeatedMapHolder h = new RepeatedMapHolder();
-    vector.getAccessor().get(index, h);
-    if (h.start == h.end) {
-      currentOffset = NO_VALUES;
-    } else {
-      int singleOffset = h.start + childIndex;
-      assert singleOffset < h.end;
-      currentOffset = singleOffset;
-      maxOffset = singleOffset + 1;
-      setChildrenPosition(singleOffset);
-    }
-  }
-
-  @Override
-  public boolean next() {
-    if (currentOffset +1 < maxOffset) {
-      setChildrenPosition(++currentOffset);
-      return true;
-    } else {
-      currentOffset = NO_VALUES;
-      return false;
-    }
-  }
-
-  public boolean isNull() {
-    return currentOffset == NO_VALUES;
-  }
-
-  @Override
-  public Object readObject() {
-    return vector.getAccessor().getObject(idx());
-  }
-
-  @Override
-  public MajorType getType() {
-    return vector.getField().getType();
-  }
-
-  @Override
-  public java.util.Iterator<String> iterator() {
-    return vector.fieldNameIterator();
-  }
-
-  @Override
-  public boolean isSet() {
-    return true;
-  }
-
-  @Override
-  public void copyAsValue(MapWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    RepeatedMapWriter impl = (RepeatedMapWriter) writer;
-    impl.container.copyFromSafe(idx(), impl.idx(), vector);
-  }
-
-  public void copyAsValueSingle(MapWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    SingleMapWriter impl = (SingleMapWriter) writer;
-    impl.container.copyFromSafe(currentOffset, impl.idx(), vector);
-  }
-
-  @Override
-  public void copyAsField(String name, MapWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    RepeatedMapWriter impl = (RepeatedMapWriter) writer.map(name);
-    impl.container.copyFromSafe(idx(), impl.idx(), vector);
-  }
-
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
deleted file mode 100644
index 086d26e119440..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex.impl;
-
-import java.util.Iterator;
-
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-
-public class SingleLikeRepeatedMapReaderImpl extends AbstractFieldReader{
-
-  private RepeatedMapReaderImpl delegate;
-
-  public SingleLikeRepeatedMapReaderImpl(RepeatedMapVector vector, FieldReader delegate) {
-    this.delegate = (RepeatedMapReaderImpl) delegate;
-  }
-
-  @Override
-  public int size() {
-    throw new UnsupportedOperationException("You can't call size on a single map reader.");
-  }
-
-  @Override
-  public boolean next() {
-    throw new UnsupportedOperationException("You can't call next on a single map reader.");
-  }
-
-  @Override
-  public MajorType getType() {
-    return Types.required(MinorType.MAP);
-  }
-
-
-  @Override
-  public void copyAsValue(MapWriter writer) {
-    delegate.copyAsValueSingle(writer);
-  }
-
-  public void copyAsValueSingle(MapWriter writer){
-    delegate.copyAsValueSingle(writer);
-  }
-
-  @Override
-  public FieldReader reader(String name) {
-    return delegate.reader(name);
-  }
-
-  @Override
-  public void setPosition(int index) {
-    delegate.setPosition(index);
-  }
-
-  @Override
-  public Object readObject() {
-    return delegate.readObject();
-  }
-
-  @Override
-  public Iterator<String> iterator() {
-    return delegate.iterator();
-  }
-
-  @Override
-  public boolean isSet() {
-    return ! delegate.isNull();
-  }
-
-
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
index f16f628603d69..b8f58658eae15 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
@@ -24,14 +24,11 @@
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
 
 @SuppressWarnings("unused")
 public class SingleListReaderImpl extends AbstractFieldReader{
 
-  private static final MajorType TYPE = Types.optional(MinorType.LIST);
   private final String name;
   private final AbstractContainerVector container;
   private FieldReader reader;
@@ -42,12 +39,6 @@ public SingleListReaderImpl(String name, AbstractContainerVector container) {
     this.container = container;
   }
 
-  @Override
-  public MajorType getType() {
-    return TYPE;
-  }
-
-
   @Override
   public void setPosition(int index) {
     super.setPosition(index);
@@ -70,6 +61,11 @@ public FieldReader reader() {
     return reader;
   }
 
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.LIST;
+  }
+
   @Override
   public boolean isSet() {
     return false;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index 84b99801419c4..1c43240901c4f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -27,9 +27,9 @@
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types.MajorType;
 
 import com.google.common.collect.Maps;
+import org.apache.arrow.vector.types.Types.MinorType;
 
 @SuppressWarnings("unused")
 public class SingleMapReaderImpl extends AbstractFieldReader{
@@ -77,13 +77,13 @@ public Object readObject() {
   }
 
   @Override
-  public boolean isSet() {
-    return true;
+  public MinorType getMinorType() {
+    return MinorType.MAP;
   }
 
   @Override
-  public MajorType getType(){
-    return vector.getField().getType();
+  public boolean isSet() {
+    return true;
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
index 9b54d02e571de..39cf00421154b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
@@ -25,8 +25,6 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.holders.UnionHolder;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
 
 public class UnionListReader extends AbstractFieldReader {
@@ -46,12 +44,6 @@ public boolean isSet() {
     return true;
   }
 
-  MajorType type = new MajorType(MinorType.LIST, DataMode.OPTIONAL);
-
-  public MajorType getType() {
-    return type;
-  }
-
   private int currentOffset;
   private int maxOffset;
 
@@ -72,6 +64,11 @@ public Object readObject() {
     return vector.getAccessor().getObject(idx());
   }
 
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.LIST;
+  }
+
   @Override
   public void read(int index, UnionHolder holder) {
     setPosition(idx());
@@ -82,6 +79,12 @@ public void read(int index, UnionHolder holder) {
     holder.isSet = data.getReader().isSet() ? 1 : 0;
   }
 
+  @Override
+  public int size() {
+    int size = maxOffset - currentOffset - 1;
+    return size < 0 ? 0 : size;
+  }
+
   @Override
   public boolean next() {
     if (currentOffset + 1 < maxOffset) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
deleted file mode 100644
index 5a5fe0305d83a..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.holders;
-
-import org.apache.arrow.vector.types.Types;
-
-/*
- * Holder class for the vector ObjectVector. This holder internally stores a
- * reference to an object. The ObjectVector maintains an array of these objects.
- * This holder can be used only as workspace variables in aggregate functions.
- * Using this holder should be avoided and we should stick to native holder types.
- */
-@Deprecated
-public class ObjectHolder implements ValueHolder {
-  public static final Types.MajorType TYPE = Types.required(Types.MinorType.GENERIC_OBJECT);
-
-  public Types.MajorType getType() {
-    return TYPE;
-  }
-
-  public Object obj;
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
index b868a620f985b..b1b695e58a954 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
@@ -18,17 +18,14 @@
 package org.apache.arrow.vector.holders;
 
 import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
 
 public class UnionHolder implements ValueHolder {
-  public static final MajorType TYPE = new MajorType(MinorType.UNION, DataMode.OPTIONAL);
   public FieldReader reader;
   public int isSet;
 
-  public MajorType getType() {
-    return reader.getType();
+  public MinorType getMinorType() {
+    return reader.getMinorType();
   }
 
   public boolean isSet() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java b/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
deleted file mode 100644
index c73098b2a85d7..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.types;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Objects;
-
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
-
-
-public class MaterializedField {
-  private final String name;
-  private final MajorType type;
-  // use an ordered set as existing code relies on order (e,g. parquet writer)
-  private final LinkedHashSet<MaterializedField> children;
-
-  MaterializedField(String name, MajorType type, LinkedHashSet<MaterializedField> children) {
-    this.name = name;
-    this.type = type;
-    this.children = children;
-  }
-
-  public Collection<MaterializedField> getChildren() {
-    return new ArrayList<>(children);
-  }
-
-  public MaterializedField newWithChild(MaterializedField child) {
-    MaterializedField newField = clone();
-    newField.addChild(child);
-    return newField;
-  }
-
-  public void addChild(MaterializedField field){
-    children.add(field);
-  }
-
-  public MaterializedField clone() {
-    return withPathAndType(name, getType());
-  }
-
-  public MaterializedField withType(MajorType type) {
-    return withPathAndType(name, type);
-  }
-
-  public MaterializedField withPath(String name) {
-    return withPathAndType(name, getType());
-  }
-
-  public MaterializedField withPathAndType(String name, final MajorType type) {
-    final LinkedHashSet<MaterializedField> newChildren = new LinkedHashSet<>(children.size());
-    for (final MaterializedField child:children) {
-      newChildren.add(child.clone());
-    }
-    return new MaterializedField(name, type, newChildren);
-  }
-
-//  public String getLastName(){
-//    PathSegment seg = key.path.getRootSegment();
-//    while (seg.getChild() != null) {
-//      seg = seg.getChild();
-//    }
-//    return seg.getNameSegment().getPath();
-//  }
-
-
-  // TODO: rewrite without as direct match rather than conversion then match.
-//  public boolean matches(SerializedField booleanfield){
-//    MaterializedField f = create(field);
-//    return f.equals(this);
-//  }
-
-  public static MaterializedField create(String name, MajorType type){
-    return new MaterializedField(name, type, new LinkedHashSet<MaterializedField>());
-  }
-
-//  public String getName(){
-//    StringBuilder sb = new StringBuilder();
-//    boolean first = true;
-//    for(NamePart np : def.getNameList()){
-//      if(np.getType() == Type.ARRAY){
-//        sb.append("[]");
-//      }else{
-//        if(first){
-//          first = false;
-//        }else{
-//          sb.append(".");
-//        }
-//        sb.append('`');
-//        sb.append(np.getName());
-//        sb.append('`');
-//
-//      }
-//    }
-//    return sb.toString();
-//  }
-
-  public String getPath() {
-    return getName();
-  }
-
-  public String getLastName() {
-    return getName();
-  }
-
-  public String getName() {
-    return name;
-  }
-
-//  public int getWidth() {
-//    return type.getWidth();
-//  }
-
-  public MajorType getType() {
-    return type;
-  }
-
-  public int getScale() {
-      return type.getScale();
-  }
-  public int getPrecision() {
-      return type.getPrecision();
-  }
-  public boolean isNullable() {
-    return type.getMode() == DataMode.OPTIONAL;
-  }
-
-  public DataMode getDataMode() {
-    return type.getMode();
-  }
-
-  public MaterializedField getOtherNullableVersion(){
-    MajorType mt = type;
-    DataMode newDataMode = null;
-    switch (mt.getMode()){
-    case OPTIONAL:
-      newDataMode = DataMode.REQUIRED;
-      break;
-    case REQUIRED:
-      newDataMode = DataMode.OPTIONAL;
-      break;
-    default:
-      throw new UnsupportedOperationException();
-    }
-    return new MaterializedField(name, new MajorType(mt.getMinorType(), newDataMode, mt.getPrecision(), mt.getScale(), mt.getTimezone(), mt.getSubTypes()), children);
-  }
-
-  public Class<?> getValueClass() {
-    return BasicTypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(this.name, this.type, this.children);
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj) {
-      return true;
-    }
-    if (obj == null) {
-      return false;
-    }
-    if (getClass() != obj.getClass()) {
-      return false;
-    }
-    MaterializedField other = (MaterializedField) obj;
-    // DRILL-1872: Compute equals only on key. See also the comment
-    // in MapVector$MapTransferPair
-
-    return this.name.equalsIgnoreCase(other.name) &&
-            Objects.equals(this.type, other.type);
-  }
-
-
-  @Override
-  public String toString() {
-    final int maxLen = 10;
-    String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : "";
-    return name + "(" + type.getMinorType().name() + ":" + type.getMode().name() + ")" + childStr;
-  }
-
-
-  private String toString(Collection<?> collection, int maxLen) {
-    StringBuilder builder = new StringBuilder();
-    builder.append("[");
-    int i = 0;
-    for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
-      if (i > 0){
-        builder.append(", ");
-      }
-      builder.append(iterator.next());
-    }
-    builder.append("]");
-    return builder.toString();
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 88999cb8f5ab8..5ea1456a051f7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -17,150 +17,508 @@
  */
 package org.apache.arrow.vector.types;
 
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.NullableBigIntVector;
+import org.apache.arrow.vector.NullableBitVector;
+import org.apache.arrow.vector.NullableDateVector;
+import org.apache.arrow.vector.NullableDecimalVector;
+import org.apache.arrow.vector.NullableFloat4Vector;
+import org.apache.arrow.vector.NullableFloat8Vector;
+import org.apache.arrow.vector.NullableIntVector;
+import org.apache.arrow.vector.NullableIntervalDayVector;
+import org.apache.arrow.vector.NullableIntervalYearVector;
+import org.apache.arrow.vector.NullableSmallIntVector;
+import org.apache.arrow.vector.NullableTimeStampVector;
+import org.apache.arrow.vector.NullableTimeVector;
+import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.NullableUInt1Vector;
+import org.apache.arrow.vector.NullableUInt2Vector;
+import org.apache.arrow.vector.NullableUInt4Vector;
+import org.apache.arrow.vector.NullableUInt8Vector;
+import org.apache.arrow.vector.NullableVarBinaryVector;
+import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
+import org.apache.arrow.vector.complex.impl.BitWriterImpl;
+import org.apache.arrow.vector.complex.impl.DateWriterImpl;
+import org.apache.arrow.vector.complex.impl.Float4WriterImpl;
+import org.apache.arrow.vector.complex.impl.Float8WriterImpl;
+import org.apache.arrow.vector.complex.impl.IntWriterImpl;
+import org.apache.arrow.vector.complex.impl.IntervalDayWriterImpl;
+import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl;
+import org.apache.arrow.vector.complex.impl.SingleMapWriter;
+import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeWriterImpl;
+import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl;
+import org.apache.arrow.vector.complex.impl.UInt1WriterImpl;
+import org.apache.arrow.vector.complex.impl.UInt2WriterImpl;
+import org.apache.arrow.vector.complex.impl.UInt4WriterImpl;
+import org.apache.arrow.vector.complex.impl.UInt8WriterImpl;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.impl.UnionWriter;
+import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl;
+import org.apache.arrow.vector.complex.impl.VarCharWriterImpl;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
+import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
+import org.apache.arrow.vector.types.pojo.ArrowType.Date;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.IntervalDay;
+import org.apache.arrow.vector.types.pojo.ArrowType.IntervalYear;
+import org.apache.arrow.vector.types.pojo.ArrowType.List;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.ArrowType.Time;
+import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.CallBack;
+
+import java.util.HashMap;
 import java.util.Map;
-import java.util.Objects;
 
 public class Types {
+
+  public static final Field NULL_FIELD = new Field("", true, Null.INSTANCE, null);
+  public static final Field TINYINT_FIELD = new Field("", true, new Int(8, true), null);
+  public static final Field SMALLINT_FIELD = new Field("", true, new Int(16, true), null);
+  public static final Field INT_FIELD = new Field("", true, new Int(32, true), null);
+  public static final Field BIGINT_FIELD = new Field("", true, new Int(64, true), null);
+  public static final Field UINT1_FIELD = new Field("", true, new Int(8, false), null);
+  public static final Field UINT2_FIELD = new Field("", true, new Int(16, false), null);
+  public static final Field UINT4_FIELD = new Field("", true, new Int(32, false), null);
+  public static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
+  public static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
+  public static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
+  public static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null);
+  public static final Field INTERVALDAY_FIELD = new Field("", true, IntervalDay.INSTANCE, null);
+  public static final Field INTERVALYEAR_FIELD = new Field("", true, IntervalYear.INSTANCE, null);
+  public static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(0), null);
+  public static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(1), null);
+  public static final Field LIST_FIELD = new Field("", true, List.INSTANCE, null);
+  public static final Field VARCHAR_FIELD = new Field("", true, Utf8.INSTANCE, null);
+  public static final Field VARBINARY_FIELD = new Field("", true, Binary.INSTANCE, null);
+  public static final Field BIT_FIELD = new Field("", true, Bool.INSTANCE, null);
+
+
   public enum MinorType {
-    LATE,   //  late binding type
-    MAP,   //  an empty map column.  Useful for conceptual setup.  Children listed within here
-
-    TINYINT,   //  single byte signed integer
-    SMALLINT,   //  two byte signed integer
-    INT,   //  four byte signed integer
-    BIGINT,   //  eight byte signed integer
-    DECIMAL9,   //  a decimal supporting precision between 1 and 9
-    DECIMAL18,   //  a decimal supporting precision between 10 and 18
-    DECIMAL28SPARSE,   //  a decimal supporting precision between 19 and 28
-    DECIMAL38SPARSE,   //  a decimal supporting precision between 29 and 38
-    MONEY,   //  signed decimal with two digit precision
-    DATE,   //  days since 4713bc
-    TIME,   //  time in micros before or after 2000/1/1
-    TIMETZ,  //  time in micros before or after 2000/1/1 with timezone
-    TIMESTAMPTZ,   //  unix epoch time in millis
-    TIMESTAMP,   //  TBD
-    INTERVAL,   //  TBD
-    FLOAT4,   //  4 byte ieee 754
-    FLOAT8,   //  8 byte ieee 754
-    BIT,  //  single bit value (boolean)
-    FIXEDCHAR,  //  utf8 fixed length string, padded with spaces
-    FIXED16CHAR,
-    FIXEDBINARY,   //  fixed length binary, padded with 0 bytes
-    VARCHAR,   //  utf8 variable length string
-    VAR16CHAR, // utf16 variable length string
-    VARBINARY,   //  variable length binary
-    UINT1,  //  unsigned 1 byte integer
-    UINT2,  //  unsigned 2 byte integer
-    UINT4,   //  unsigned 4 byte integer
-    UINT8,   //  unsigned 8 byte integer
-    DECIMAL28DENSE, // dense decimal representation, supporting precision between 19 and 28
-    DECIMAL38DENSE, // dense decimal representation, supporting precision between 28 and 38
-    NULL, // a value of unknown type (e.g. a missing reference).
-    INTERVALYEAR, // Interval type specifying YEAR to MONTH
-    INTERVALDAY, // Interval type specifying DAY to SECONDS
-    LIST,
-    GENERIC_OBJECT,
-    UNION
-  }
+    NULL(Null.INSTANCE) {
+      @Override
+      public Field getField() {
+        return NULL_FIELD;
+      }
 
-  public enum DataMode {
-    REQUIRED,
-    OPTIONAL,
-    REPEATED
-  }
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return ZeroVector.INSTANCE;
+      }
 
-  public static class MajorType {
-    private MinorType minorType;
-    private DataMode mode;
-    private int precision;
-    private int scale;
-    private int timezone;
-    private int width;
-    private List<MinorType> subTypes;
-
-    public MajorType(MinorType minorType, DataMode mode) {
-      this(minorType, mode, 0, 0, 0, 0, null);
-    }
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return null;
+      }
+    },
+    MAP(Tuple.INSTANCE) {
+      @Override
+      public Field getField() {
+        throw new UnsupportedOperationException("Cannot get simple field for Map type");
+      }
 
-    public MajorType(MinorType minorType, DataMode mode, int precision, int scale) {
-      this(minorType, mode, precision, scale, 0, 0, null);
-    }
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+         return new MapVector(name, allocator, callBack);
+      }
 
-    public MajorType(MinorType minorType, DataMode mode, int precision, int scale, int timezone, List<MinorType> subTypes) {
-      this(minorType, mode, precision, scale, timezone, 0, subTypes);
-    }
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new SingleMapWriter((MapVector) vector);
+      }
+    },   //  an empty map column.  Useful for conceptual setup.  Children listed within here
 
-    public MajorType(MinorType minorType, DataMode mode, int precision, int scale, int timezone, int width, List<MinorType> subTypes) {
-      this.minorType = minorType;
-      this.mode = mode;
-      this.precision = precision;
-      this.scale = scale;
-      this.timezone = timezone;
-      this.width = width;
-      this.subTypes = subTypes;
-      if (subTypes == null) {
-        this.subTypes = new ArrayList<>();
+    TINYINT(new Int(8, true)) {
+      @Override
+      public Field getField() {
+        return TINYINT_FIELD;
       }
-    }
 
-    public MinorType getMinorType() {
-      return minorType;
-    }
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableTinyIntVector(name, allocator);
+      }
 
-    public DataMode getMode() {
-      return mode;
-    }
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TinyIntWriterImpl((NullableTinyIntVector) vector);
+      }
+    },   //  single byte signed integer
+    SMALLINT(new Int(16, true)) {
+      @Override
+      public Field getField() {
+        return SMALLINT_FIELD;
+      }
 
-    public int getPrecision() {
-      return precision;
-    }
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new SmallIntVector(name, allocator);
+      }
 
-    public int getScale() {
-      return scale;
-    }
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new SmallIntWriterImpl((NullableSmallIntVector) vector);
+      }
+    },   //  two byte signed integer
+    INT(new Int(32, true)) {
+      @Override
+      public Field getField() {
+        return INT_FIELD;
+      }
 
-    public int getTimezone() {
-      return timezone;
-    }
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableIntVector(name, allocator);
+      }
 
-    public List<MinorType> getSubTypes() {
-      return subTypes;
-    }
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new IntWriterImpl((NullableIntVector) vector);
+      }
+    },   //  four byte signed integer
+    BIGINT(new Int(64, true)) {
+      @Override
+      public Field getField() {
+        return BIGINT_FIELD;
+      }
 
-    public int getWidth() {
-      return width;
-    }
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableBigIntVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new BigIntWriterImpl((NullableBigIntVector) vector);
+      }
+    },   //  eight byte signed integer
+    DATE(Date.INSTANCE) {
+      @Override
+      public Field getField() {
+        return DATE_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableDateVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new DateWriterImpl((NullableDateVector) vector);
+      }
+    },   //  days since 4713bc
+    TIME(Time.INSTANCE) {
+      @Override
+      public Field getField() {
+        return TIME_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableTimeVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeWriterImpl((NullableTimeVector) vector);
+      }
+    },   //  time in micros before or after 2000/1/1
+    TIMESTAMP(new Timestamp("")) {
+      @Override
+      public Field getField() {
+        return TIMESTAMP_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableTimeStampVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeStampWriterImpl((NullableTimeStampVector) vector);
+      }
+    },
+    INTERVALDAY(IntervalDay.INSTANCE) {
+      @Override
+      public Field getField() {
+        return INTERVALDAY_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableIntervalDayVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new IntervalDayWriterImpl((NullableIntervalDayVector) vector);
+      }
+    },
+    INTERVALYEAR(IntervalYear.INSTANCE) {
+      @Override
+      public Field getField() {
+        return INTERVALYEAR_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableIntervalDayVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new IntervalYearWriterImpl((NullableIntervalYearVector) vector);
+      }
+    },
+    FLOAT4(new FloatingPoint(0)) {
+      @Override
+      public Field getField() {
+        return FLOAT4_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableFloat4Vector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new Float4WriterImpl((NullableFloat4Vector) vector);
+      }
+    },   //  4 byte ieee 754
+    FLOAT8(new FloatingPoint(1)) {
+      @Override
+      public Field getField() {
+        return FLOAT8_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableFloat8Vector(name, allocator);
+      }
 
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new Float8WriterImpl((NullableFloat8Vector) vector);
+      }
+    },   //  8 byte ieee 754
+    BIT(Bool.INSTANCE) {
+      @Override
+      public Field getField() {
+        return BIT_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableBitVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new BitWriterImpl((NullableBitVector) vector);
+      }
+    },  //  single bit value (boolean)
+    VARCHAR(Utf8.INSTANCE) {
+      @Override
+      public Field getField() {
+        return VARCHAR_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableVarCharVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new VarCharWriterImpl((NullableVarCharVector) vector);
+      }
+    },   //  utf8 variable length string
+    VARBINARY(Binary.INSTANCE) {
+      @Override
+      public Field getField() {
+        return VARBINARY_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableVarBinaryVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new VarBinaryWriterImpl((NullableVarBinaryVector) vector);
+      }
+    },   //  variable length binary
+    DECIMAL(null) {
+      @Override
+      public ArrowType getType() {
+        throw new UnsupportedOperationException("Cannot get simple type for Decimal type");
+      }
+      @Override
+      public Field getField() {
+        throw new UnsupportedOperationException("Cannot get simple field for Decimal type");
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableDecimalVector(name, allocator, precisionScale[0], precisionScale[1]);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new VarBinaryWriterImpl((NullableVarBinaryVector) vector);
+      }
+    },   //  variable length binary
+    UINT1(new Int(8, false)) {
+      @Override
+      public Field getField() {
+        return UINT1_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableUInt1Vector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new UInt1WriterImpl((NullableUInt1Vector) vector);
+      }
+    },  //  unsigned 1 byte integer
+    UINT2(new Int(16, false)) {
+      @Override
+      public Field getField() {
+        return UINT2_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableUInt2Vector(name, allocator);
+      }
 
-    @Override
-    public boolean equals(Object other) {
-      if (other == null) {
-        return false;
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new UInt2WriterImpl((NullableUInt2Vector) vector);
       }
-      if (!(other instanceof MajorType)) {
-        return false;
+    },  //  unsigned 2 byte integer
+    UINT4(new Int(32, false)) {
+      @Override
+      public Field getField() {
+        return UINT8_FIELD;
       }
-      MajorType that = (MajorType) other;
-      return this.minorType == that.minorType &&
-              this.mode == that.mode &&
-              this.precision == that.precision &&
-              this.scale == that.scale &&
-              this.timezone == that.timezone &&
-              this.width == that.width &&
-              Objects.equals(this.subTypes, that.subTypes);
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableUInt4Vector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new UInt4WriterImpl((NullableUInt4Vector) vector);
+      }
+    },   //  unsigned 4 byte integer
+    UINT8(new Int(64, false)) {
+      @Override
+      public Field getField() {
+        return UINT8_FIELD;
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableUInt8Vector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new UInt8WriterImpl((NullableUInt8Vector) vector);
+      }
+    },   //  unsigned 8 byte integer
+    LIST(List.INSTANCE) {
+      @Override
+      public Field getField() {
+        throw new UnsupportedOperationException("Cannot get simple field for List type");
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new ListVector(name, allocator, callBack);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new UnionListWriter((ListVector) vector);
+      }
+    },
+    UNION(Union.INSTANCE) {
+      @Override
+      public Field getField() {
+        throw new UnsupportedOperationException("Cannot get simple field for Union type");
+      }
+
+      @Override
+      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new UnionVector(name, allocator, callBack);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new UnionWriter((UnionVector) vector);
+      }
+    };
+
+    private final ArrowType type;
+
+    MinorType(ArrowType type) {
+      this.type = type;
     }
 
-  }
+    public ArrowType getType() {
+      return type;
+    }
+
+    public abstract Field getField();
 
-  public static MajorType required(MinorType minorType) {
-    return new MajorType(minorType, DataMode.REQUIRED);
+    public abstract ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale);
+
+    public abstract FieldWriter getNewFieldWriter(ValueVector vector);
   }
-  public static MajorType optional(MinorType minorType) {
-    return new MajorType(minorType, DataMode.OPTIONAL);
+
+  private static final Map<ArrowType,MinorType> ARROW_TYPE_MINOR_TYPE_MAP;
+
+  public static MinorType getMinorTypeForArrowType(ArrowType arrowType) {
+    if (arrowType.getTypeType() == Type.Decimal) {
+      return MinorType.DECIMAL;
+    }
+    return ARROW_TYPE_MINOR_TYPE_MAP.get(arrowType);
   }
-  public static MajorType repeated(MinorType minorType) {
-    return new MajorType(minorType, DataMode.REPEATED);
+
+  static {
+    ARROW_TYPE_MINOR_TYPE_MAP = new HashMap<>();
+    for (MinorType minorType : MinorType.values()) {
+      if (minorType != MinorType.DECIMAL) {
+        ARROW_TYPE_MINOR_TYPE_MAP.put(minorType.getType(), minorType);
+      }
+    }
   }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
new file mode 100644
index 0000000000000..49d0503e47036
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.types.pojo;
+
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
+
+public class Field {
+  private final String name;
+  private final boolean nullable;
+  private final ArrowType type;
+  private final List<Field> children;
+
+  public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
+    this.name = name;
+    this.nullable = nullable;
+    this.type = type;
+    if (children == null) {
+      this.children = ImmutableList.of();
+    } else {
+      this.children = children;
+    }
+  }
+
+  public static Field convertField(org.apache.arrow.flatbuf.Field field) {
+    String name = field.name();
+    boolean nullable = field.nullable();
+    ArrowType type = getTypeForField(field);
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    for (int i = 0; i < field.childrenLength(); i++) {
+      childrenBuilder.add(convertField(field.children(i)));
+    }
+    List<Field> children = childrenBuilder.build();
+    return new Field(name, nullable, type, children);
+  }
+
+  public int getField(FlatBufferBuilder builder) {
+    int nameOffset = builder.createString(name);
+    int typeOffset = type.getType(builder);
+    int[] childrenData = new int[children.size()];
+    for (int i = 0; i < children.size(); i++) {
+      childrenData[i] = children.get(i).getField(builder);
+    }
+    int childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, childrenData);
+    org.apache.arrow.flatbuf.Field.startField(builder);
+    org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
+    org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
+    org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeType());
+    org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
+    org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
+    return org.apache.arrow.flatbuf.Field.endField(builder);
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public boolean isNullable() {
+    return nullable;
+  }
+
+  public ArrowType getType() {
+    return type;
+  }
+
+  public List<Field> getChildren() {
+    return children;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof Field)) {
+      return false;
+    }
+    Field that = (Field) obj;
+    return Objects.equals(this.name, that.name) &&
+            Objects.equals(this.nullable, that.nullable) &&
+            Objects.equals(this.type, that.type) &&
+            (Objects.equals(this.children, that.children) ||
+                    (this.children == null && that.children.size() == 0) ||
+                    (this.children.size() == 0 && that.children == null));
+
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
new file mode 100644
index 0000000000000..9e2894170b24b
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.types.pojo;
+
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
+import static org.apache.arrow.vector.types.pojo.Field.convertField;
+
+public class Schema {
+  private List<Field> fields;
+
+  public Schema(List<Field> fields) {
+    this.fields = ImmutableList.copyOf(fields);
+  }
+
+  public int getSchema(FlatBufferBuilder builder) {
+    int[] fieldOffsets = new int[fields.size()];
+    for (int i = 0; i < fields.size(); i++) {
+      fieldOffsets[i] = fields.get(i).getField(builder);
+    }
+    int fieldsOffset = org.apache.arrow.flatbuf.Schema.createFieldsVector(builder, fieldOffsets);
+    org.apache.arrow.flatbuf.Schema.startSchema(builder);
+    org.apache.arrow.flatbuf.Schema.addFields(builder, fieldsOffset);
+    return org.apache.arrow.flatbuf.Schema.endSchema(builder);
+  }
+
+  public List<Field> getFields() {
+    return fields;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hashCode(fields);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof Schema)) {
+      return false;
+    }
+    return Objects.equals(this.fields, ((Schema) obj).fields);
+  }
+
+  public static Schema convertSchema(org.apache.arrow.flatbuf.Schema schema) {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    for (int i = 0; i < schema.fieldsLength(); i++) {
+      childrenBuilder.add(convertField(schema.fields(i)));
+    }
+    List<Field> fields = childrenBuilder.build();
+    return new Schema(fields);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
index b6dd13a06a82d..68b9fb25f2112 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
@@ -180,54 +180,4 @@ private static final int memcmp(final long laddr, int lStart, int lEnd, final by
     return lLen > rLen ? 1 : -1;
   }
 
-  /*
-   * Following are helper functions to interact with sparse decimal represented in a byte array.
-   */
-
-  // Get the integer ignore the sign
-  public static int getInteger(byte[] b, int index) {
-    return getInteger(b, index, true);
-  }
-  // Get the integer, ignore the sign
-  public static int getInteger(byte[] b, int index, boolean ignoreSign) {
-    int startIndex = index * DecimalUtility.INTEGER_SIZE;
-
-    if (index == 0 && ignoreSign == true) {
-      return (b[startIndex + 3] & 0xFF) |
-             (b[startIndex + 2] & 0xFF) << 8 |
-             (b[startIndex + 1] & 0xFF) << 16 |
-             (b[startIndex] & 0x7F) << 24;
-    }
-
-    return ((b[startIndex + 3] & 0xFF) |
-        (b[startIndex + 2] & 0xFF) << 8 |
-        (b[startIndex + 1] & 0xFF) << 16 |
-        (b[startIndex] & 0xFF) << 24);
-
-  }
-
-  // Set integer in the byte array
-  public static void setInteger(byte[] b, int index, int value) {
-    int startIndex = index * DecimalUtility.INTEGER_SIZE;
-    b[startIndex] = (byte) ((value >> 24) & 0xFF);
-    b[startIndex + 1] = (byte) ((value >> 16) & 0xFF);
-    b[startIndex + 2] = (byte) ((value >> 8) & 0xFF);
-    b[startIndex + 3] = (byte) ((value) & 0xFF);
-  }
-
-  // Set the sign in a sparse decimal representation
-  public static void setSign(byte[] b, boolean sign) {
-    int value = getInteger(b, 0);
-    if (sign == true) {
-      setInteger(b, 0, value | 0x80000000);
-    } else {
-      setInteger(b, 0, value & 0x7FFFFFFF);
-    }
-  }
-
-  // Get the sign
-  public static boolean getSign(byte[] b) {
-    return ((getInteger(b, 0, false) & 0x80000000) != 0);
-  }
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
deleted file mode 100644
index 1eb2c13cd4c59..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.util;
-
-import java.math.BigDecimal;
-
-import org.apache.arrow.vector.types.Types;
-
-public class CoreDecimalUtility {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoreDecimalUtility.class);
-
-  public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
-    // Truncate or pad to set the input to the correct scale
-    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
-    return (input.unscaledValue().longValue());
-  }
-
-  public static int getMaxPrecision(Types.MinorType decimalType) {
-    if (decimalType == Types.MinorType.DECIMAL9) {
-      return 9;
-    } else if (decimalType == Types.MinorType.DECIMAL18) {
-      return 18;
-    } else if (decimalType == Types.MinorType.DECIMAL28SPARSE) {
-      return 28;
-    } else if (decimalType == Types.MinorType.DECIMAL38SPARSE) {
-      return 38;
-    }
-    return 0;
-  }
-
-  /*
-   * Function returns the Minor decimal type given the precision
-   */
-  public static Types.MinorType getDecimalDataType(int precision) {
-    if (precision <= 9) {
-      return Types.MinorType.DECIMAL9;
-    } else if (precision <= 18) {
-      return Types.MinorType.DECIMAL18;
-    } else if (precision <= 28) {
-      return Types.MinorType.DECIMAL28SPARSE;
-    } else {
-      return Types.MinorType.DECIMAL38SPARSE;
-    }
-  }
-
-  /*
-   * Given a precision it provides the max precision of that decimal data type;
-   * For eg: given the precision 12, we would use DECIMAL18 to store the data
-   * which has a max precision range of 18 digits
-   */
-  public static int getPrecisionRange(int precision) {
-    return getMaxPrecision(getDecimalDataType(precision));
-  }
-  public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) {
-    // Truncate/ or pad to set the input to the correct scale
-    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
-    return (input.unscaledValue().intValue());
-  }
-
-  /*
-   * Helper function to detect if the given data type is Decimal
-   */
-  public static boolean isDecimalType(Types.MajorType type) {
-    return isDecimalType(type.getMinorType());
-  }
-
-  public static boolean isDecimalType(Types.MinorType minorType) {
-    if (minorType == Types.MinorType.DECIMAL9 || minorType == Types.MinorType.DECIMAL18 ||
-        minorType == Types.MinorType.DECIMAL28SPARSE || minorType == Types.MinorType.DECIMAL38SPARSE) {
-      return true;
-    }
-    return false;
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index a3763cd34f1a1..4eb0d9f2216c1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -26,140 +26,139 @@
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
-import org.apache.arrow.vector.holders.Decimal38SparseHolder;
-
-public class DecimalUtility extends CoreDecimalUtility{
-
-    public final static int MAX_DIGITS = 9;
-    public final static int DIGITS_BASE = 1000000000;
-    public final static int DIGITS_MAX = 999999999;
-    public final static int INTEGER_SIZE = (Integer.SIZE/8);
-
-    public final static String[] decimalToString = {"",
-            "0",
-            "00",
-            "000",
-            "0000",
-            "00000",
-            "000000",
-            "0000000",
-            "00000000",
-            "000000000"};
-
-    public final static long[] scale_long_constants = {
-        1,
-        10,
-        100,
-        1000,
-        10000,
-        100000,
-        1000000,
-        10000000,
-        100000000,
-        1000000000,
-        10000000000l,
-        100000000000l,
-        1000000000000l,
-        10000000000000l,
-        100000000000000l,
-        1000000000000000l,
-        10000000000000000l,
-        100000000000000000l,
-        1000000000000000000l};
-
-    /*
-     * Simple function that returns the static precomputed
-     * power of ten, instead of using Math.pow
-     */
-    public static long getPowerOfTen(int power) {
-      assert power >= 0 && power < scale_long_constants.length;
-      return scale_long_constants[(power)];
-    }
-
-    /*
-     * Math.pow returns a double and while multiplying with large digits
-     * in the decimal data type we encounter noise. So instead of multiplying
-     * with Math.pow we use the static constants to perform the multiplication
-     */
-    public static long adjustScaleMultiply(long input, int factor) {
-      int index = Math.abs(factor);
-      assert index >= 0 && index < scale_long_constants.length;
-      if (factor >= 0) {
-        return input * scale_long_constants[index];
-      } else {
-        return input / scale_long_constants[index];
-      }
-    }
 
-    public static long adjustScaleDivide(long input, int factor) {
-      int index = Math.abs(factor);
-      assert index >= 0 && index < scale_long_constants.length;
-      if (factor >= 0) {
-        return input / scale_long_constants[index];
-      } else {
-        return input * scale_long_constants[index];
-      }
+public class DecimalUtility {
+
+  public final static int MAX_DIGITS = 9;
+  public final static int DIGITS_BASE = 1000000000;
+  public final static int DIGITS_MAX = 999999999;
+  public final static int INTEGER_SIZE = (Integer.SIZE/8);
+
+  public final static String[] decimalToString = {"",
+          "0",
+          "00",
+          "000",
+          "0000",
+          "00000",
+          "000000",
+          "0000000",
+          "00000000",
+          "000000000"};
+
+  public final static long[] scale_long_constants = {
+          1,
+          10,
+          100,
+          1000,
+          10000,
+          100000,
+          1000000,
+          10000000,
+          100000000,
+          1000000000,
+          10000000000l,
+          100000000000l,
+          1000000000000l,
+          10000000000000l,
+          100000000000000l,
+          1000000000000000l,
+          10000000000000000l,
+          100000000000000000l,
+          1000000000000000000l};
+
+  /*
+   * Simple function that returns the static precomputed
+   * power of ten, instead of using Math.pow
+   */
+  public static long getPowerOfTen(int power) {
+    assert power >= 0 && power < scale_long_constants.length;
+    return scale_long_constants[(power)];
+  }
+
+  /*
+   * Math.pow returns a double and while multiplying with large digits
+   * in the decimal data type we encounter noise. So instead of multiplying
+   * with Math.pow we use the static constants to perform the multiplication
+   */
+  public static long adjustScaleMultiply(long input, int factor) {
+    int index = Math.abs(factor);
+    assert index >= 0 && index < scale_long_constants.length;
+    if (factor >= 0) {
+      return input * scale_long_constants[index];
+    } else {
+      return input / scale_long_constants[index];
     }
+  }
 
-    /* Given the number of actual digits this function returns the
-     * number of indexes it will occupy in the array of integers
-     * which are stored in base 1 billion
-     */
-    public static int roundUp(int ndigits) {
-        return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+  public static long adjustScaleDivide(long input, int factor) {
+    int index = Math.abs(factor);
+    assert index >= 0 && index < scale_long_constants.length;
+    if (factor >= 0) {
+      return input / scale_long_constants[index];
+    } else {
+      return input * scale_long_constants[index];
     }
+  }
 
-    /* Returns a string representation of the given integer
-     * If the length of the given integer is less than the
-     * passed length, this function will prepend zeroes to the string
-     */
-    public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
-        String value = ((Integer) number).toString();
-        int length = value.length();
+  /* Given the number of actual digits this function returns the
+   * number of indexes it will occupy in the array of integers
+   * which are stored in base 1 billion
+   */
+  public static int roundUp(int ndigits) {
+    return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+  }
 
-        StringBuilder str = new StringBuilder();
-        str.append(decimalToString[desiredLength - length]);
-        str.append(value);
+  /* Returns a string representation of the given integer
+   * If the length of the given integer is less than the
+   * passed length, this function will prepend zeroes to the string
+   */
+  public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
+    String value = ((Integer) number).toString();
+    int length = value.length();
 
-        return str;
-    }
+    StringBuilder str = new StringBuilder();
+    str.append(decimalToString[desiredLength - length]);
+    str.append(value);
 
-    public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
-        String value = ((Long) number).toString();
-        int length = value.length();
+    return str;
+  }
 
-        StringBuilder str = new StringBuilder();
+  public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
+    String value = ((Long) number).toString();
+    int length = value.length();
 
-        // Desired length can be > MAX_DIGITS
-        int zeroesLength = desiredLength - length;
-        while (zeroesLength > MAX_DIGITS) {
-            str.append(decimalToString[MAX_DIGITS]);
-            zeroesLength -= MAX_DIGITS;
-        }
-        str.append(decimalToString[zeroesLength]);
-        str.append(value);
+    StringBuilder str = new StringBuilder();
 
-        return str;
+    // Desired length can be > MAX_DIGITS
+    int zeroesLength = desiredLength - length;
+    while (zeroesLength > MAX_DIGITS) {
+      str.append(decimalToString[MAX_DIGITS]);
+      zeroesLength -= MAX_DIGITS;
     }
+    str.append(decimalToString[zeroesLength]);
+    str.append(value);
+
+    return str;
+  }
 
   public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
 
-        // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
-        return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false);
-    }
+    // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
+    return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false);
+  }
 
-    public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
+  public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
 
-        // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
-        return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true);
-    }
+    // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
+    return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true);
+  }
 
-    public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) {
-      byte[] value = new byte[length];
-      bytebuf.getBytes(start, value, 0, length);
-      BigInteger unscaledValue = new BigInteger(value);
-      return new BigDecimal(unscaledValue, scale);
-    }
+  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) {
+    byte[] value = new byte[length];
+    bytebuf.getBytes(start, value, 0, length);
+    BigInteger unscaledValue = new BigInteger(value);
+    return new BigDecimal(unscaledValue, scale);
+  }
 
   public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
     byte[] value = new byte[length];
@@ -168,115 +167,123 @@ public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int sta
     return new BigDecimal(unscaledValue, scale);
   }
 
-    /* Create a BigDecimal object using the data in the ArrowBuf.
-     * This function assumes that data is provided in a non-dense format
-     * It works on both sparse and intermediate representations.
-     */
+  public static void writeBigDecimalToArrowBuf(ArrowBuf bytebuf, int startIndex, BigDecimal value) {
+    byte[] bytes = value.unscaledValue().toByteArray();
+    if (bytes.length > 16) {
+      throw new UnsupportedOperationException("Decimal size greater than 16 bytes");
+    }
+    bytebuf.setBytes(startIndex + 16 - bytes.length, bytes, 0, bytes.length);
+  }
+
+  /* Create a BigDecimal object using the data in the ArrowBuf.
+   * This function assumes that data is provided in a non-dense format
+   * It works on both sparse and intermediate representations.
+   */
   public static BigDecimal getBigDecimalFromArrowBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale,
-      boolean truncateScale) {
+                                                     boolean truncateScale) {
 
-        // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
-        int actualDigits;
+    // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
+    int actualDigits;
 
-        // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out
-        BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
+    // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out
+    BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
 
-        BigInteger base = BigInteger.valueOf(DIGITS_BASE);
+    BigInteger base = BigInteger.valueOf(DIGITS_BASE);
 
-        for (int i = 1; i < nDecimalDigits; i++) {
+    for (int i = 1; i < nDecimalDigits; i++) {
 
-            BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
-            decimalDigits = decimalDigits.multiply(base);
-            decimalDigits = decimalDigits.add(temp);
-        }
+      BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
+      decimalDigits = decimalDigits.multiply(base);
+      decimalDigits = decimalDigits.add(temp);
+    }
 
-        // Truncate any additional padding we might have added
-        if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
-            BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
-            decimalDigits = decimalDigits.divide(truncate);
-        }
+    // Truncate any additional padding we might have added
+    if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
+      BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
+      decimalDigits = decimalDigits.divide(truncate);
+    }
 
-        // set the sign
-        if ((data.getInt(startIndex) & 0x80000000) != 0) {
-            decimalDigits = decimalDigits.negate();
-        }
+    // set the sign
+    if ((data.getInt(startIndex) & 0x80000000) != 0) {
+      decimalDigits = decimalDigits.negate();
+    }
 
-        BigDecimal decimal = new BigDecimal(decimalDigits, scale);
+    BigDecimal decimal = new BigDecimal(decimalDigits, scale);
 
-        return decimal;
-    }
+    return decimal;
+  }
 
-    /* This function returns a BigDecimal object from the dense decimal representation.
-     * First step is to convert the dense representation into an intermediate representation
-     * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object
-     */
-    public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
+  /* This function returns a BigDecimal object from the dense decimal representation.
+   * First step is to convert the dense representation into an intermediate representation
+   * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object
+   */
+  public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
 
         /* This method converts the dense representation to
          * an intermediate representation. The intermediate
          * representation has one more integer than the dense
          * representation.
          */
-        byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
-
-        // Start storing from the least significant byte of the first integer
-        int intermediateIndex = 3;
-
-        int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
-        int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
-
-        int maskIndex;
-        int shiftOrder;
-        byte shiftBits;
-
-        // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
-        if (maxPrecision == 38) {
-            maskIndex = 0;
-            shiftOrder = 6;
-            shiftBits = 0x00;
-            intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
-        } else if (maxPrecision == 28) {
-            maskIndex = 1;
-            shiftOrder = 4;
-            shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
-            intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
-        } else {
-            throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
-        }
+    byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
+
+    // Start storing from the least significant byte of the first integer
+    int intermediateIndex = 3;
+
+    int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
+    int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
+
+    int maskIndex;
+    int shiftOrder;
+    byte shiftBits;
+
+    // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
+    if (maxPrecision == 38) {
+      maskIndex = 0;
+      shiftOrder = 6;
+      shiftBits = 0x00;
+      intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
+    } else if (maxPrecision == 28) {
+      maskIndex = 1;
+      shiftOrder = 4;
+      shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
+      intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
+    } else {
+      throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
+    }
 
-        int inputIndex = 1;
-        boolean sign = false;
+    int inputIndex = 1;
+    boolean sign = false;
 
-        if ((data.getByte(startIndex) & 0x80) != 0) {
-            sign = true;
-        }
+    if ((data.getByte(startIndex) & 0x80) != 0) {
+      sign = true;
+    }
 
-        while (inputIndex < width) {
+    while (inputIndex < width) {
 
-            intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
+      intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
 
-            shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
+      shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
 
-            inputIndex++;
-            intermediateIndex++;
+      inputIndex++;
+      intermediateIndex++;
 
-            if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
-                shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
-                maskIndex++;
-                shiftOrder -= 2;
-            }
+      if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
+        shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
+        maskIndex++;
+        shiftOrder -= 2;
+      }
 
-        }
+    }
         /* copy the last byte */
-        intermediateBytes[intermediateIndex] = shiftBits;
+    intermediateBytes[intermediateIndex] = shiftBits;
 
-        if (sign == true) {
-            intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
-        }
+    if (sign == true) {
+      intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
+    }
 
     final ByteBuf intermediate = UnpooledByteBufAllocator.DEFAULT.buffer(intermediateBytes.length);
     try {
-        intermediate.setBytes(0, intermediateBytes);
+      intermediate.setBytes(0, intermediateBytes);
 
       BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
       return ret;
@@ -284,299 +291,296 @@ public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, i
       intermediate.release();
     }
 
-    }
+  }
 
-    /*
-     * Function converts the BigDecimal and stores it in out internal sparse representation
-     */
-  public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
-      int nDecimalDigits) {
+ public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
+                                             int nDecimalDigits) {
 
-        // Initialize the buffer
-        for (int i = 0; i < nDecimalDigits; i++) {
-          data.setInt(startIndex + (i * INTEGER_SIZE), 0);
-        }
+    // Initialize the buffer
+    for (int i = 0; i < nDecimalDigits; i++) {
+      data.setInt(startIndex + (i * INTEGER_SIZE), 0);
+    }
 
-        boolean sign = false;
+    boolean sign = false;
 
-        if (input.signum() == -1) {
-            // negative input
-            sign = true;
-            input = input.abs();
-        }
+    if (input.signum() == -1) {
+      // negative input
+      sign = true;
+      input = input.abs();
+    }
 
-        // Truncate the input as per the scale provided
-        input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+    // Truncate the input as per the scale provided
+    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
 
-        // Separate out the integer part
-        BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
+    // Separate out the integer part
+    BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
 
-        int destIndex = nDecimalDigits - roundUp(scale) - 1;
+    int destIndex = nDecimalDigits - roundUp(scale) - 1;
 
-        // we use base 1 billion integer digits for out integernal representation
-        BigDecimal base = new BigDecimal(DIGITS_BASE);
+    // we use base 1 billion integer digits for out integernal representation
+    BigDecimal base = new BigDecimal(DIGITS_BASE);
 
-        while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
-            // store the modulo as the integer value
-            data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
-            destIndex--;
-            // Divide by base 1 billion
-            integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
-        }
+    while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
+      // store the modulo as the integer value
+      data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
+      destIndex--;
+      // Divide by base 1 billion
+      integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
+    }
 
         /* Sparse representation contains padding of additional zeroes
          * so each digit contains MAX_DIGITS for ease of arithmetic
          */
-        int actualDigits;
-        if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
-            // Pad additional zeroes
-            scale = scale + (MAX_DIGITS - actualDigits);
-            input = input.setScale(scale, BigDecimal.ROUND_DOWN);
-        }
-
-        //separate out the fractional part
-        BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
+    int actualDigits;
+    if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
+      // Pad additional zeroes
+      scale = scale + (MAX_DIGITS - actualDigits);
+      input = input.setScale(scale, BigDecimal.ROUND_DOWN);
+    }
 
-        destIndex = nDecimalDigits - 1;
+    //separate out the fractional part
+    BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
 
-        while (scale > 0) {
-            // Get next set of MAX_DIGITS (9) store it in the ArrowBuf
-            fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
-            BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
+    destIndex = nDecimalDigits - 1;
 
-            data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
-            destIndex--;
+    while (scale > 0) {
+      // Get next set of MAX_DIGITS (9) store it in the ArrowBuf
+      fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
+      BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
 
-            fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
-            scale -= MAX_DIGITS;
-        }
+      data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
+      destIndex--;
 
-        // Set the negative sign
-        if (sign == true) {
-            data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
-        }
+      fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
+      scale -= MAX_DIGITS;
+    }
 
+    // Set the negative sign
+    if (sign == true) {
+      data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
     }
 
+  }
 
-    public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
-        // Truncate or pad to set the input to the correct scale
-        input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
 
-        return (input.unscaledValue().longValue());
-    }
+  public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+    // Truncate or pad to set the input to the correct scale
+    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
 
-    public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
-      return BigDecimal.valueOf(input, scale);
-    }
+    return (input.unscaledValue().longValue());
+  }
 
-    public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
-      return BigDecimal.valueOf(input, scale);
-    }
+  public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
+    return BigDecimal.valueOf(input, scale);
+  }
+
+  public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
+    return BigDecimal.valueOf(input, scale);
+  }
 
 
-    public static int compareDenseBytes(ArrowBuf left, int leftStart, boolean leftSign, ArrowBuf right, int rightStart, boolean rightSign, int width) {
+  public static int compareDenseBytes(ArrowBuf left, int leftStart, boolean leftSign, ArrowBuf right, int rightStart, boolean rightSign, int width) {
 
-      int invert = 1;
+    int invert = 1;
 
       /* If signs are different then simply look at the
        * sign of the two inputs and determine which is greater
        */
-      if (leftSign != rightSign) {
+    if (leftSign != rightSign) {
 
-        return((leftSign == true) ? -1 : 1);
-      } else if(leftSign == true) {
+      return((leftSign == true) ? -1 : 1);
+    } else if(leftSign == true) {
         /* Both inputs are negative, at the end we will
          * have to invert the comparison
          */
-        invert = -1;
-      }
-
-      int cmp = 0;
-
-      for (int i = 0; i < width; i++) {
-        byte leftByte  = left.getByte(leftStart + i);
-        byte rightByte = right.getByte(rightStart + i);
-        // Unsigned byte comparison
-        if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
-          cmp = 1;
-          break;
-        } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
-          cmp = -1;
-          break;
-        }
-      }
-      cmp *= invert; // invert the comparison if both were negative values
-
-      return cmp;
+      invert = -1;
     }
 
-    public static int getIntegerFromSparseBuffer(ArrowBuf buffer, int start, int index) {
-      int value = buffer.getInt(start + (index * 4));
+    int cmp = 0;
 
-      if (index == 0) {
-        /* the first byte contains sign bit, return value without it */
-        value = (value & 0x7FFFFFFF);
+    for (int i = 0; i < width; i++) {
+      byte leftByte  = left.getByte(leftStart + i);
+      byte rightByte = right.getByte(rightStart + i);
+      // Unsigned byte comparison
+      if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
+        cmp = 1;
+        break;
+      } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
+        cmp = -1;
+        break;
       }
-      return value;
     }
+    cmp *= invert; // invert the comparison if both were negative values
 
-    public static void setInteger(ArrowBuf buffer, int start, int index, int value) {
-      buffer.setInt(start + (index * 4), value);
+    return cmp;
+  }
+
+  public static int getIntegerFromSparseBuffer(ArrowBuf buffer, int start, int index) {
+    int value = buffer.getInt(start + (index * 4));
+
+    if (index == 0) {
+        /* the first byte contains sign bit, return value without it */
+      value = (value & 0x7FFFFFFF);
     }
+    return value;
+  }
 
-    public static int compareSparseBytes(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
+  public static void setInteger(ArrowBuf buffer, int start, int index, int value) {
+    buffer.setInt(start + (index * 4), value);
+  }
 
-      int invert = 1;
+  public static int compareSparseBytes(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
 
-      if (absCompare == false) {
-        if (leftSign != rightSign) {
-          return (leftSign == true) ? -1 : 1;
-        }
+    int invert = 1;
 
-        // Both values are negative invert the outcome of the comparison
-        if (leftSign == true) {
-          invert = -1;
-        }
+    if (absCompare == false) {
+      if (leftSign != rightSign) {
+        return (leftSign == true) ? -1 : 1;
       }
 
-      int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
-      return cmp * invert;
+      // Both values are negative invert the outcome of the comparison
+      if (leftSign == true) {
+        invert = -1;
+      }
     }
-    public static int compareSparseBytesInner(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
+
+    int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
+    return cmp * invert;
+  }
+  public static int compareSparseBytesInner(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
       /* compute the number of integer digits in each decimal */
-      int leftInt  = leftPrecision - leftScale;
-      int rightInt = rightPrecision - rightScale;
+    int leftInt  = leftPrecision - leftScale;
+    int rightInt = rightPrecision - rightScale;
 
       /* compute the number of indexes required for storing integer digits */
-      int leftIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftInt);
-      int rightIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightInt);
+    int leftIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftInt);
+    int rightIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightInt);
 
       /* compute number of indexes required for storing scale */
-      int leftScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftScale);
-      int rightScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightScale);
+    int leftScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftScale);
+    int rightScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightScale);
 
       /* compute index of the most significant integer digits */
-      int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
-      int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+    int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+    int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
 
-      int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
-      int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
+    int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
+    int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
 
       /* Discard the zeroes in the integer part */
-      while (leftIndex1 < leftStopIndex) {
-        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
-          break;
-        }
+    while (leftIndex1 < leftStopIndex) {
+      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+        break;
+      }
 
         /* Digit in this location is zero, decrement the actual number
          * of integer digits
          */
-        leftIntRoundedUp--;
-        leftIndex1++;
-      }
+      leftIntRoundedUp--;
+      leftIndex1++;
+    }
 
       /* If we reached the stop index then the number of integers is zero */
-      if (leftIndex1 == leftStopIndex) {
-        leftIntRoundedUp = 0;
-      }
+    if (leftIndex1 == leftStopIndex) {
+      leftIntRoundedUp = 0;
+    }
 
-      while (rightIndex1 < rightStopIndex) {
-        if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
-          break;
-        }
+    while (rightIndex1 < rightStopIndex) {
+      if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+        break;
+      }
 
         /* Digit in this location is zero, decrement the actual number
          * of integer digits
          */
-        rightIntRoundedUp--;
-        rightIndex1++;
-      }
+      rightIntRoundedUp--;
+      rightIndex1++;
+    }
 
-      if (rightIndex1 == rightStopIndex) {
-        rightIntRoundedUp = 0;
-      }
+    if (rightIndex1 == rightStopIndex) {
+      rightIntRoundedUp = 0;
+    }
 
       /* We have the accurate number of non-zero integer digits,
        * if the number of integer digits are different then we can determine
        * which decimal is larger and needn't go down to comparing individual values
        */
-      if (leftIntRoundedUp > rightIntRoundedUp) {
-        return 1;
-      }
-      else if (rightIntRoundedUp > leftIntRoundedUp) {
-        return -1;
-      }
+    if (leftIntRoundedUp > rightIntRoundedUp) {
+      return 1;
+    }
+    else if (rightIntRoundedUp > leftIntRoundedUp) {
+      return -1;
+    }
 
       /* The number of integer digits are the same, set the each index
        * to the first non-zero integer and compare each digit
        */
-      leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
-      rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+    leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+    rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
 
-      while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
-        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
-          return 1;
-        }
-        else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
-          return -1;
-        }
-
-        leftIndex1++;
-        rightIndex1++;
+    while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+        return 1;
+      }
+      else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+        return -1;
       }
 
+      leftIndex1++;
+      rightIndex1++;
+    }
+
       /* The integer part of both the decimal's are equal, now compare
        * each individual fractional part. Set the index to be at the
        * beginning of the fractional part
        */
-      leftIndex1 = leftStopIndex;
-      rightIndex1 = rightStopIndex;
+    leftIndex1 = leftStopIndex;
+    rightIndex1 = rightStopIndex;
 
       /* Stop indexes will be the end of the array */
-      leftStopIndex = nDecimalDigits;
-      rightStopIndex = nDecimalDigits;
+    leftStopIndex = nDecimalDigits;
+    rightStopIndex = nDecimalDigits;
 
       /* compare the two fractional parts of the decimal */
-      while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
-        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
-          return 1;
-        }
-        else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
-          return -1;
-        }
-
-        leftIndex1++;
-        rightIndex1++;
+    while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+        return 1;
+      }
+      else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+        return -1;
       }
 
+      leftIndex1++;
+      rightIndex1++;
+    }
+
       /* Till now the fractional part of the decimals are equal, check
        * if one of the decimal has fractional part that is remaining
        * and is non-zero
        */
-      while (leftIndex1 < leftStopIndex) {
-        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
-          return 1;
-        }
-        leftIndex1++;
+    while (leftIndex1 < leftStopIndex) {
+      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+        return 1;
       }
+      leftIndex1++;
+    }
 
-      while(rightIndex1 < rightStopIndex) {
-        if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
-          return -1;
-        }
-        rightIndex1++;
+    while(rightIndex1 < rightStopIndex) {
+      if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+        return -1;
       }
+      rightIndex1++;
+    }
 
       /* Both decimal values are equal */
-      return 0;
-    }
+    return 0;
+  }
 
-    public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
-      byte[] value = Arrays.copyOfRange(bytes, start, start + length);
-      BigInteger unscaledValue = new BigInteger(value);
-      return new BigDecimal(unscaledValue, scale);
-    }
+  public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
+    byte[] value = Arrays.copyOfRange(bytes, start, start + length);
+    BigInteger unscaledValue = new BigInteger(value);
+    return new BigDecimal(unscaledValue, scale);
+  }
 
   public static void roundDecimal(ArrowBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
     int newScaleRoundedUp  = org.apache.arrow.vector.util.DecimalUtility.roundUp(desiredScale);
@@ -704,34 +708,6 @@ public static int getFirstFractionalDigit(ArrowBuf data, int scale, int start, i
     int index = nDecimalDigits - roundUp(scale);
     return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1));
   }
-
-  public static int compareSparseSamePrecScale(ArrowBuf left, int lStart, byte[] right, int length) {
-    // check the sign first
-    boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
-    boolean rSign = ByteFunctionHelpers.getSign(right);
-    int cmp = 0;
-
-    if (lSign != rSign) {
-      return (lSign == false) ? 1 : -1;
-    }
-
-    // invert the comparison if we are comparing negative numbers
-    int invert = (lSign == true) ? -1 : 1;
-
-    // compare byte by byte
-    int n = 0;
-    int lPos = lStart;
-    int rPos = 0;
-    while (n < length/4) {
-      int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
-      int rightInt = ByteFunctionHelpers.getInteger(right, n);
-      if (leftInt != rightInt) {
-        cmp =  (leftInt - rightInt ) > 0 ? 1 : -1;
-        break;
-      }
-      n++;
-    }
-    return cmp * invert;
-  }
 }
 
+
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
index dea433e99e80f..d7f9d382e4865 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -18,7 +18,9 @@
 package org.apache.arrow.vector.util;
 
 import java.util.AbstractMap;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -241,6 +243,16 @@ public Set<K> keySet() {
     return delegate.keySet();
   }
 
+  public List<K> keyList() {
+    int size = size();
+    Set<K> keys = keySet();
+    List<K> children = new ArrayList<>(size);
+    for (K key : keys) {
+      children.add(getOrdinal(key), key);
+    }
+    return children;
+  }
+
   @Override
   public Set<Entry<K, V>> entrySet() {
     return delegate.entrySet();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
new file mode 100644
index 0000000000000..7ab7db3117b81
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.util.DecimalUtility;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestDecimalVector {
+
+  private static long[] intValues;
+
+  static {
+    intValues = new long[30];
+    for (int i = 0; i < intValues.length; i++) {
+      intValues[i] = 1 << i + 1;
+    }
+  }
+  private int scale = 3;
+
+  @Test
+  public void test() {
+    BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+    NullableDecimalVector decimalVector = new NullableDecimalVector("decimal", allocator, 10, scale);
+    decimalVector.allocateNew();
+    BigDecimal[] values = new BigDecimal[intValues.length];
+    for (int i = 0; i < intValues.length; i++) {
+      BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale);
+      values[i] = decimal;
+      decimalVector.getMutator().setIndexDefined(i);
+      DecimalUtility.writeBigDecimalToArrowBuf(decimalVector.getBuffer(), i * 16, decimal);
+    }
+
+    decimalVector.getMutator().setValueCount(intValues.length);
+
+    for (int i = 0; i < intValues.length; i++) {
+      BigDecimal value = decimalVector.getAccessor().getObject(i);
+      assertEquals(values[i], value);
+    }
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
index 4dee86c9d595a..9baebc5a2992c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
@@ -20,8 +20,6 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.junit.After;
 import org.junit.Before;
@@ -53,8 +51,7 @@ public void terminate() throws Exception {
 
   @Test(expected = OversizedAllocationException.class)
   public void testFixedVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final UInt4Vector vector = new UInt4Vector(field, allocator);
+    final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator);
     // edge case 1: buffer size = max value capacity
     final int expectedValueCapacity = BaseValueVector.MAX_ALLOCATION_SIZE / 4;
     try {
@@ -78,8 +75,7 @@ public void testFixedVectorReallocation() {
 
   @Test(expected = OversizedAllocationException.class)
   public void testBitVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final BitVector vector = new BitVector(field, allocator);
+    final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator);
     // edge case 1: buffer size ~ max value capacity
     final int expectedValueCapacity = 1 << 29;
     try {
@@ -109,8 +105,7 @@ public void testBitVectorReallocation() {
 
   @Test(expected = OversizedAllocationException.class)
   public void testVariableVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final VarCharVector vector = new VarCharVector(field, allocator);
+    final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator);
     // edge case 1: value count = MAX_VALUE_ALLOCATION
     final int expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE;
     final int expectedOffsetSize = 10;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index e4d28c3f88ca6..1bb50b73a9057 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -22,8 +22,6 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.holders.NullableUInt4Holder;
-import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
 import org.apache.arrow.vector.types.Types;
 import org.junit.After;
 import org.junit.Before;
@@ -46,13 +44,12 @@ public void terminate() throws Exception {
 
   @Test
   public void testUnionVector() throws Exception {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
 
     final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder();
     uInt4Holder.value = 100;
     uInt4Holder.isSet = 1;
 
-    try (UnionVector unionVector = new UnionVector(field, allocator, null)) {
+    try (UnionVector unionVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       unionVector.allocateNew();
 
       // write some data
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index ce091ab1ed06b..21cdc4f4d8d3b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -19,15 +19,7 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.RepeatedListVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.holders.*;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.junit.After;
 import org.junit.Before;
@@ -50,9 +42,9 @@ public void init() {
   }
 
   private final static Charset utf8Charset = Charset.forName("UTF-8");
-  private final static byte[] STR1 = new String("AAAAA1").getBytes(utf8Charset);
-  private final static byte[] STR2 = new String("BBBBBBBBB2").getBytes(utf8Charset);
-  private final static byte[] STR3 = new String("CCCC3").getBytes(utf8Charset);
+  private final static byte[] STR1 = "AAAAA1".getBytes(utf8Charset);
+  private final static byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset);
+  private final static byte[] STR3 = "CCCC3".getBytes(utf8Charset);
 
   @After
   public void terminate() throws Exception {
@@ -61,10 +53,9 @@ public void terminate() throws Exception {
 
   @Test
   public void testFixedType() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
 
     // Create a new value vector for 1024 integers.
-    try (final UInt4Vector vector = new UInt4Vector(field, allocator)) {
+    try (final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator)) {
       final UInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -86,10 +77,9 @@ public void testFixedType() {
 
   @Test
   public void testNullableVarLen2() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
 
     // Create a new value vector for 1024 integers.
-    try (final NullableVarCharVector vector = new NullableVarCharVector(field, allocator)) {
+    try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024 * 10, 1024);
 
@@ -115,45 +105,11 @@ public void testNullableVarLen2() {
     }
   }
 
-  @Test
-  public void testRepeatedIntVector() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
-
-    // Create a new value vector.
-    try (final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator)) {
-
-      // Populate the vector.
-      final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
-      final int nRecords = 7;
-      final int nElements = values.length;
-      vector1.allocateNew(nRecords, nRecords * nElements);
-      final RepeatedIntVector.Mutator mutator = vector1.getMutator();
-      for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
-        mutator.startNewValue(recordIndex);
-        for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
-          mutator.add(recordIndex, recordIndex * values[elementIndex]);
-        }
-      }
-      mutator.setValueCount(nRecords);
-
-      // Verify the contents.
-      final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
-      assertEquals(nRecords, accessor1.getValueCount());
-      for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
-        for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
-          final int value = accessor1.get(recordIndex, elementIndex);
-          assertEquals(recordIndex * values[elementIndex], value);
-        }
-      }
-    }
-  }
-
   @Test
   public void testNullableFixedType() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableUInt4Holder.TYPE);
 
     // Create a new value vector for 1024 integers.
-    try (final NullableUInt4Vector vector = new NullableUInt4Vector(field, allocator)) {
+    try (final NullableUInt4Vector vector = new NullableUInt4Vector(EMPTY_SCHEMA_PATH, allocator)) {
       final NullableUInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -222,10 +178,8 @@ public void testNullableFixedType() {
 
   @Test
   public void testNullableFloat() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
-
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = (NullableFloat4Vector) BasicTypeHelper.getNewVector(field, allocator)) {
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -271,10 +225,8 @@ public void testNullableFloat() {
 
   @Test
   public void testBitVector() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
-
     // Create a new value vector for 1024 integers
-    try (final BitVector vector = new BitVector(field, allocator)) {
+    try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
       final BitVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -311,10 +263,8 @@ public void testBitVector() {
 
   @Test
   public void testReAllocNullableFixedWidthVector() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
-
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = (NullableFloat4Vector) BasicTypeHelper.getNewVector(field, allocator)) {
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -346,10 +296,8 @@ public void testReAllocNullableFixedWidthVector() {
 
   @Test
   public void testReAllocNullableVariableWidthVector() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
-
     // Create a new value vector for 1024 integers
-    try (final NullableVarCharVector vector = (NullableVarCharVector) BasicTypeHelper.getNewVector(field, allocator)) {
+    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew();
 
@@ -376,69 +324,4 @@ public void testReAllocNullableVariableWidthVector() {
     }
   }
 
-  @Test
-  public void testVVInitialCapacity() throws Exception {
-    final MaterializedField[] fields = new MaterializedField[9];
-    final ValueVector[] valueVectors = new ValueVector[9];
-
-    fields[0] = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
-    fields[1] = MaterializedField.create(EMPTY_SCHEMA_PATH, IntHolder.TYPE);
-    fields[2] = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
-    fields[3] = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVar16CharHolder.TYPE);
-    fields[4] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedFloat4Holder.TYPE);
-    fields[5] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedVarBinaryHolder.TYPE);
-
-    fields[6] = MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE);
-    fields[6].addChild(fields[0] /*bit*/);
-    fields[6].addChild(fields[2] /*varchar*/);
-
-    fields[7] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE);
-    fields[7].addChild(fields[1] /*int*/);
-    fields[7].addChild(fields[3] /*optional var16char*/);
-
-    fields[8] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE);
-    fields[8].addChild(fields[1] /*int*/);
-
-    final int initialCapacity = 1024;
-
-    try {
-      for (int i = 0; i < valueVectors.length; i++) {
-        valueVectors[i] = BasicTypeHelper.getNewVector(fields[i], allocator);
-        valueVectors[i].setInitialCapacity(initialCapacity);
-        valueVectors[i].allocateNew();
-      }
-
-      for (int i = 0; i < valueVectors.length; i++) {
-        final ValueVector vv = valueVectors[i];
-        final int vvCapacity = vv.getValueCapacity();
-
-        // this can't be equality because Nullables will be allocated using power of two sized buffers (thus need 1025
-        // spots in one vector > power of two is 2048, available capacity will be 2048 => 2047)
-        assertTrue(String.format("Incorrect value capacity for %s [%d]", vv.getField(), vvCapacity),
-                initialCapacity <= vvCapacity);
-      }
-    } finally {
-      for (ValueVector v : valueVectors) {
-        v.close();
-      }
-    }
-  }
-
-  @Test
-  public void testListVectorShouldNotThrowOversizedAllocationException() throws Exception {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH,
-            Types.optional(MinorType.LIST));
-    ListVector vector = new ListVector(field, allocator, null);
-    ListVector vectorFrom = new ListVector(field, allocator, null);
-    vectorFrom.allocateNew();
-
-    for (int i = 0; i < 10000; i++) {
-      vector.allocateNew();
-      vector.copyFromSafe(0, 0, vectorFrom);
-      vector.clear();
-    }
-
-    vectorFrom.clear();
-    vector.clear();
-  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 4c24444d81d18..24f00f14df001 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -27,7 +27,7 @@
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -49,10 +49,9 @@ public void terminate() throws Exception {
 
   @Test
   public void testPromoteToUnion() throws Exception {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
 
-    try (final AbstractMapVector container = new MapVector(field, allocator, null);
-         final MapVector v = container.addOrGet("test", MapVector.TYPE, MapVector.class);
+    try (final AbstractMapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
+         final MapVector v = container.addOrGet("test", MinorType.MAP, MapVector.class);
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
       container.allocateNew();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
new file mode 100644
index 0000000000000..bc17a2b2835c2
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.writer;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.impl.UnionReader;
+import org.apache.arrow.vector.complex.impl.UnionWriter;
+import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestComplexWriter {
+
+  static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+
+  private static final int COUNT = 100;
+
+  @Test
+  public void simpleNestedTypes() {
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    for (int i = 0; i < COUNT; i++) {
+      intWriter.setPosition(i);
+      intWriter.writeInt(i);
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+    }
+    writer.setValueCount(COUNT);
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < COUNT; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
+      Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+    }
+
+    parent.close();
+  }
+
+  @Test
+  public void listScalarType() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 7; j++) {
+        listWriter.writeInt(j);
+      }
+      listWriter.endList();
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      for (int j = 0; j < i % 7; j++) {
+        listReader.next();
+        Assert.assertEquals(j, listReader.reader().readInteger().intValue());
+      }
+    }
+  }
+
+
+  @Test
+  public void listMapType() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    MapWriter mapWriter = listWriter.map();
+    for (int i = 0; i < COUNT; i++) {
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 7; j++) {
+        mapWriter.start();
+        mapWriter.integer("int").writeInt(j);
+        mapWriter.bigInt("bigInt").writeBigInt(j);
+        mapWriter.end();
+      }
+      listWriter.endList();
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      for (int j = 0; j < i % 7; j++) {
+        listReader.next();
+        Assert.assertEquals("record: " + i, j, listReader.reader().reader("int").readInteger().intValue());
+        Assert.assertEquals(j, listReader.reader().reader("bigInt").readLong().longValue());
+      }
+    }
+  }
+
+  @Test
+  public void listListType() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 7; j++) {
+        ListWriter innerListWriter = listWriter.list();
+        innerListWriter.startList();
+        for (int k = 0; k < i % 13; k++) {
+          innerListWriter.integer().writeInt(k);
+        }
+        innerListWriter.endList();
+      }
+      listWriter.endList();
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      for (int j = 0; j < i % 7; j++) {
+        listReader.next();
+        FieldReader innerListReader = listReader.reader();
+        for (int k = 0; k < i % 13; k++) {
+          innerListReader.next();
+          Assert.assertEquals("record: " + i, k, innerListReader.reader().readInteger().intValue());
+        }
+      }
+    }
+    listVector.clear();
+  }
+
+  @Test
+  public void unionListListType() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 7; j++) {
+        ListWriter innerListWriter = listWriter.list();
+        innerListWriter.startList();
+        for (int k = 0; k < i % 13; k++) {
+          if (k % 2 == 0) {
+            innerListWriter.integer().writeInt(k);
+          } else {
+            innerListWriter.bigInt().writeBigInt(k);
+          }
+        }
+        innerListWriter.endList();
+      }
+      listWriter.endList();
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      for (int j = 0; j < i % 7; j++) {
+        listReader.next();
+        FieldReader innerListReader = listReader.reader();
+        for (int k = 0; k < i % 13; k++) {
+          innerListReader.next();
+          if (k % 2 == 0) {
+            Assert.assertEquals("record: " + i, k, innerListReader.reader().readInteger().intValue());
+          } else {
+            Assert.assertEquals("record: " + i, k, innerListReader.reader().readLong().longValue());
+          }
+        }
+      }
+    }
+    listVector.clear();
+  }
+
+  @Test
+  public void simpleUnion() {
+    UnionVector vector = new UnionVector("union", allocator, null);
+    UnionWriter unionWriter = new UnionWriter(vector);
+    unionWriter.allocate();
+    for (int i = 0; i < COUNT; i++) {
+      unionWriter.setPosition(i);
+      if (i % 2 == 0) {
+        unionWriter.writeInt(i);
+      } else {
+        unionWriter.writeFloat4((float) i);
+      }
+    }
+    vector.getMutator().setValueCount(COUNT);
+    UnionReader unionReader = new UnionReader(vector);
+    for (int i = 0; i < COUNT; i++) {
+      unionReader.setPosition(i);
+      if (i % 2 == 0) {
+        Assert.assertEquals(i, i, unionReader.readInteger());
+      } else {
+        Assert.assertEquals((float) i, unionReader.readFloat(), 1e-12);
+      }
+    }
+    vector.close();
+  }
+
+  @Test
+  public void promotableWriter() {
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    for (int i = 0; i < 100; i++) {
+      BigIntWriter bigIntWriter = rootWriter.bigInt("a");
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+    }
+    Field field = parent.getField().getChildren().get(0).getChildren().get(0);
+    Assert.assertEquals("a", field.getName());
+    Assert.assertEquals(Int.TYPE_TYPE, field.getType().getTypeType());
+    Int intType = (Int) field.getType();
+
+    Assert.assertEquals(64, intType.getBitWidth());
+    Assert.assertTrue(intType.getIsSigned());
+    for (int i = 100; i < 200; i++) {
+      VarCharWriter varCharWriter = rootWriter.varChar("a");
+      varCharWriter.setPosition(i);
+      byte[] bytes = Integer.toString(i).getBytes();
+      ArrowBuf tempBuf = allocator.buffer(bytes.length);
+      tempBuf.setBytes(0, bytes);
+      varCharWriter.writeVarChar(0, bytes.length, tempBuf);
+    }
+    field = parent.getField().getChildren().get(0).getChildren().get(0);
+    Assert.assertEquals("a", field.getName());
+    Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeType());
+    Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeType());
+    Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeType());
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < 100; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(i, rootReader.reader("a").readLong().intValue());
+    }
+    for (int i = 100; i < 200; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(Integer.toString(i), rootReader.reader("a").readText().toString());
+    }
+  }
+}
\ No newline at end of file
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
new file mode 100644
index 0000000000000..06a1149c0d6c1
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.pojo;
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test conversion between Flatbuf and Pojo field representations
+ */
+public class TestConvert {
+
+  @Test
+  public void simple() {
+    Field initialField = new Field("a", true, new Int(32, true), null);
+    run(initialField);
+  }
+
+  @Test
+  public void complex() {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
+    childrenBuilder.add(new Field("child2", true, new FloatingPoint(0), ImmutableList.<Field>of()));
+
+    Field initialField = new Field("a", true, Tuple.INSTANCE, childrenBuilder.build());
+    run(initialField);
+  }
+
+  @Test
+  public void schema() {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
+    childrenBuilder.add(new Field("child2", true, new FloatingPoint(0), ImmutableList.<Field>of()));
+    Schema initialSchema = new Schema(childrenBuilder.build());
+    run(initialSchema);
+
+  }
+
+  private void run(Field initialField) {
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    builder.finish(initialField.getField(builder));
+    org.apache.arrow.flatbuf.Field flatBufField = org.apache.arrow.flatbuf.Field.getRootAsField(builder.dataBuffer());
+    Field finalField = Field.convertField(flatBufField);
+    assertEquals(initialField, finalField);
+  }
+
+  private void run(Schema initialSchema) {
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    builder.finish(initialSchema.getSchema(builder));
+    org.apache.arrow.flatbuf.Schema flatBufSchema = org.apache.arrow.flatbuf.Schema.getRootAsSchema(builder.dataBuffer());
+    Schema finalSchema = Schema.convertSchema(flatBufSchema);
+    assertEquals(initialSchema, finalSchema);
+  }
+}

From fd2e52491bc39ae5aa0ddb7dbc21109172cea1c2 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Thu, 18 Aug 2016 16:31:32 -0700
Subject: [PATCH 0114/1644] Revert version to 0.1-SNAPSHOT

---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 2 +-
 java/vector/pom.xml | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index ea27a3072bc9e..cb11b5ff3c45d 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -16,7 +16,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.1-decimal</version>
+  <version>0.1-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 12ff4c81d86c0..44332f5ed14a8 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -15,7 +15,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1-decimal</version>
+    <version>0.1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>arrow-memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 92ab109f939e1..8eb25af7545f4 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -21,7 +21,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.1-decimal</version>
+  <version>0.1-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index fac788cef14d9..1a2921f6ea521 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -15,7 +15,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1-decimal</version>
+    <version>0.1-SNAPSHOT</version>
   </parent>
   <artifactId>vector</artifactId>
   <name>vectors</name>
@@ -25,7 +25,7 @@
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-format</artifactId>
-      <version>0.1-decimal</version>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>

From 282fcacc86c9232c9dc1b1030e9fc9299bbc3f8d Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Fri, 19 Aug 2016 14:28:05 -0700
Subject: [PATCH 0115/1644] ARROW-265: Pad negative decimal values with1

---
 .../codegen/templates/FixedValueVectors.java  |   8 +-
 .../codegen/templates/HolderReaderImpl.java   |   5 +-
 .../arrow/vector/util/DecimalUtility.java     | 579 +-----------------
 .../arrow/vector/TestDecimalVector.java       |   7 +-
 4 files changed, 27 insertions(+), 572 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index fe2b5c5b5bc92..37946f6b76ea6 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+import org.apache.arrow.vector.util.DecimalUtility;
+
 import java.lang.Override;
 
 <@pp.dropOutputFile />
@@ -444,11 +446,7 @@ public void get(int index, Nullable${minor.class}Holder holder) {
 
     @Override
     public ${friendlyType} getObject(int index) {
-      byte[] bytes = new byte[${type.width}];
-      int start = ${type.width} * index;
-      data.getBytes(start, bytes, 0, ${type.width});
-      ${friendlyType} value = new BigDecimal(new BigInteger(bytes), scale);
-      return value;
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(data, index, scale);
     }
 
     <#else>
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index 1ed9287b00eec..d66577bc1e444 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -156,9 +156,11 @@ private Object readSingleObject() {
 </#if>
 
 <#if type.major == "VarLen">
+      <#if minor.class != "Decimal">
       int length = holder.end - holder.start;
       byte[] value = new byte [length];
       holder.buffer.getBytes(holder.start, value, 0, length);
+      </#if>
 
 <#if minor.class == "VarBinary">
       return value;
@@ -169,8 +171,7 @@ private Object readSingleObject() {
       text.set(value);
       return text;
 <#elseif minor.class == "Decimal" >
-        return new BigDecimal(new BigInteger(value), holder.scale);
-
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(holder.buffer, holder.start, holder.scale);
 </#if>
 
 <#elseif minor.class == "Interval">
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index 4eb0d9f2216c1..e171e87360d86 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -66,6 +66,8 @@ public class DecimalUtility {
           100000000000000000l,
           1000000000000000000l};
 
+  public static final int DECIMAL_BYTE_LENGTH = 16;
+
   /*
    * Simple function that returns the static precomputed
    * power of ten, instead of using Math.pow
@@ -100,14 +102,6 @@ public static long adjustScaleDivide(long input, int factor) {
     }
   }
 
-  /* Given the number of actual digits this function returns the
-   * number of indexes it will occupy in the array of integers
-   * which are stored in base 1 billion
-   */
-  public static int roundUp(int ndigits) {
-    return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
-  }
-
   /* Returns a string representation of the given integer
    * If the length of the given integer is less than the
    * passed length, this function will prepend zeroes to the string
@@ -141,572 +135,33 @@ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
     return str;
   }
 
-  public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
-
-    // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
-    return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false);
-  }
-
-  public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
-
-    // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
-    return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true);
-  }
-
-  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) {
-    byte[] value = new byte[length];
-    bytebuf.getBytes(start, value, 0, length);
+  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale) {
+    byte[] value = new byte[DECIMAL_BYTE_LENGTH];
+    final int startIndex = index * DECIMAL_BYTE_LENGTH;
+    bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH);
     BigInteger unscaledValue = new BigInteger(value);
     return new BigDecimal(unscaledValue, scale);
   }
 
-  public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
-    byte[] value = new byte[length];
+  public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int scale) {
+    byte[] value = new byte[DECIMAL_BYTE_LENGTH];
     bytebuf.get(value);
     BigInteger unscaledValue = new BigInteger(value);
     return new BigDecimal(unscaledValue, scale);
   }
 
-  public static void writeBigDecimalToArrowBuf(ArrowBuf bytebuf, int startIndex, BigDecimal value) {
-    byte[] bytes = value.unscaledValue().toByteArray();
-    if (bytes.length > 16) {
+  public static void writeBigDecimalToArrowBuf(BigDecimal value, ArrowBuf bytebuf, int index) {
+    final byte[] bytes = value.unscaledValue().toByteArray();
+    final int startIndex = index * DECIMAL_BYTE_LENGTH;
+    if (bytes.length > DECIMAL_BYTE_LENGTH) {
       throw new UnsupportedOperationException("Decimal size greater than 16 bytes");
     }
-    bytebuf.setBytes(startIndex + 16 - bytes.length, bytes, 0, bytes.length);
-  }
-
-  /* Create a BigDecimal object using the data in the ArrowBuf.
-   * This function assumes that data is provided in a non-dense format
-   * It works on both sparse and intermediate representations.
-   */
-  public static BigDecimal getBigDecimalFromArrowBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale,
-                                                     boolean truncateScale) {
-
-    // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
-    int actualDigits;
-
-    // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out
-    BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
-
-    BigInteger base = BigInteger.valueOf(DIGITS_BASE);
-
-    for (int i = 1; i < nDecimalDigits; i++) {
-
-      BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
-      decimalDigits = decimalDigits.multiply(base);
-      decimalDigits = decimalDigits.add(temp);
-    }
-
-    // Truncate any additional padding we might have added
-    if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
-      BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
-      decimalDigits = decimalDigits.divide(truncate);
-    }
-
-    // set the sign
-    if ((data.getInt(startIndex) & 0x80000000) != 0) {
-      decimalDigits = decimalDigits.negate();
+    final int padLength = DECIMAL_BYTE_LENGTH - bytes.length;
+    final int padValue = value.signum() == -1 ? 0xFF : 0;
+    for (int i = 0; i < padLength; i++) {
+      bytebuf.setByte(startIndex + i, padValue);
     }
-
-    BigDecimal decimal = new BigDecimal(decimalDigits, scale);
-
-    return decimal;
-  }
-
-  /* This function returns a BigDecimal object from the dense decimal representation.
-   * First step is to convert the dense representation into an intermediate representation
-   * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object
-   */
-  public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
-
-        /* This method converts the dense representation to
-         * an intermediate representation. The intermediate
-         * representation has one more integer than the dense
-         * representation.
-         */
-    byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
-
-    // Start storing from the least significant byte of the first integer
-    int intermediateIndex = 3;
-
-    int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
-    int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
-
-    int maskIndex;
-    int shiftOrder;
-    byte shiftBits;
-
-    // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
-    if (maxPrecision == 38) {
-      maskIndex = 0;
-      shiftOrder = 6;
-      shiftBits = 0x00;
-      intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
-    } else if (maxPrecision == 28) {
-      maskIndex = 1;
-      shiftOrder = 4;
-      shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
-      intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
-    } else {
-      throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
-    }
-
-    int inputIndex = 1;
-    boolean sign = false;
-
-    if ((data.getByte(startIndex) & 0x80) != 0) {
-      sign = true;
-    }
-
-    while (inputIndex < width) {
-
-      intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
-
-      shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
-
-      inputIndex++;
-      intermediateIndex++;
-
-      if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
-        shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
-        maskIndex++;
-        shiftOrder -= 2;
-      }
-
-    }
-        /* copy the last byte */
-    intermediateBytes[intermediateIndex] = shiftBits;
-
-    if (sign == true) {
-      intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
-    }
-
-    final ByteBuf intermediate = UnpooledByteBufAllocator.DEFAULT.buffer(intermediateBytes.length);
-    try {
-      intermediate.setBytes(0, intermediateBytes);
-
-      BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
-      return ret;
-    } finally {
-      intermediate.release();
-    }
-
-  }
-
- public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
-                                             int nDecimalDigits) {
-
-    // Initialize the buffer
-    for (int i = 0; i < nDecimalDigits; i++) {
-      data.setInt(startIndex + (i * INTEGER_SIZE), 0);
-    }
-
-    boolean sign = false;
-
-    if (input.signum() == -1) {
-      // negative input
-      sign = true;
-      input = input.abs();
-    }
-
-    // Truncate the input as per the scale provided
-    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
-    // Separate out the integer part
-    BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
-
-    int destIndex = nDecimalDigits - roundUp(scale) - 1;
-
-    // we use base 1 billion integer digits for out integernal representation
-    BigDecimal base = new BigDecimal(DIGITS_BASE);
-
-    while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
-      // store the modulo as the integer value
-      data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
-      destIndex--;
-      // Divide by base 1 billion
-      integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
-    }
-
-        /* Sparse representation contains padding of additional zeroes
-         * so each digit contains MAX_DIGITS for ease of arithmetic
-         */
-    int actualDigits;
-    if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
-      // Pad additional zeroes
-      scale = scale + (MAX_DIGITS - actualDigits);
-      input = input.setScale(scale, BigDecimal.ROUND_DOWN);
-    }
-
-    //separate out the fractional part
-    BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
-
-    destIndex = nDecimalDigits - 1;
-
-    while (scale > 0) {
-      // Get next set of MAX_DIGITS (9) store it in the ArrowBuf
-      fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
-      BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
-
-      data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
-      destIndex--;
-
-      fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
-      scale -= MAX_DIGITS;
-    }
-
-    // Set the negative sign
-    if (sign == true) {
-      data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
-    }
-
-  }
-
-
-  public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
-    // Truncate or pad to set the input to the correct scale
-    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
-    return (input.unscaledValue().longValue());
-  }
-
-  public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
-    return BigDecimal.valueOf(input, scale);
-  }
-
-  public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
-    return BigDecimal.valueOf(input, scale);
-  }
-
-
-  public static int compareDenseBytes(ArrowBuf left, int leftStart, boolean leftSign, ArrowBuf right, int rightStart, boolean rightSign, int width) {
-
-    int invert = 1;
-
-      /* If signs are different then simply look at the
-       * sign of the two inputs and determine which is greater
-       */
-    if (leftSign != rightSign) {
-
-      return((leftSign == true) ? -1 : 1);
-    } else if(leftSign == true) {
-        /* Both inputs are negative, at the end we will
-         * have to invert the comparison
-         */
-      invert = -1;
-    }
-
-    int cmp = 0;
-
-    for (int i = 0; i < width; i++) {
-      byte leftByte  = left.getByte(leftStart + i);
-      byte rightByte = right.getByte(rightStart + i);
-      // Unsigned byte comparison
-      if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
-        cmp = 1;
-        break;
-      } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
-        cmp = -1;
-        break;
-      }
-    }
-    cmp *= invert; // invert the comparison if both were negative values
-
-    return cmp;
-  }
-
-  public static int getIntegerFromSparseBuffer(ArrowBuf buffer, int start, int index) {
-    int value = buffer.getInt(start + (index * 4));
-
-    if (index == 0) {
-        /* the first byte contains sign bit, return value without it */
-      value = (value & 0x7FFFFFFF);
-    }
-    return value;
-  }
-
-  public static void setInteger(ArrowBuf buffer, int start, int index, int value) {
-    buffer.setInt(start + (index * 4), value);
-  }
-
-  public static int compareSparseBytes(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
-
-    int invert = 1;
-
-    if (absCompare == false) {
-      if (leftSign != rightSign) {
-        return (leftSign == true) ? -1 : 1;
-      }
-
-      // Both values are negative invert the outcome of the comparison
-      if (leftSign == true) {
-        invert = -1;
-      }
-    }
-
-    int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
-    return cmp * invert;
-  }
-  public static int compareSparseBytesInner(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
-      /* compute the number of integer digits in each decimal */
-    int leftInt  = leftPrecision - leftScale;
-    int rightInt = rightPrecision - rightScale;
-
-      /* compute the number of indexes required for storing integer digits */
-    int leftIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftInt);
-    int rightIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightInt);
-
-      /* compute number of indexes required for storing scale */
-    int leftScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftScale);
-    int rightScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightScale);
-
-      /* compute index of the most significant integer digits */
-    int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
-    int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
-    int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
-    int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
-
-      /* Discard the zeroes in the integer part */
-    while (leftIndex1 < leftStopIndex) {
-      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
-        break;
-      }
-
-        /* Digit in this location is zero, decrement the actual number
-         * of integer digits
-         */
-      leftIntRoundedUp--;
-      leftIndex1++;
-    }
-
-      /* If we reached the stop index then the number of integers is zero */
-    if (leftIndex1 == leftStopIndex) {
-      leftIntRoundedUp = 0;
-    }
-
-    while (rightIndex1 < rightStopIndex) {
-      if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
-        break;
-      }
-
-        /* Digit in this location is zero, decrement the actual number
-         * of integer digits
-         */
-      rightIntRoundedUp--;
-      rightIndex1++;
-    }
-
-    if (rightIndex1 == rightStopIndex) {
-      rightIntRoundedUp = 0;
-    }
-
-      /* We have the accurate number of non-zero integer digits,
-       * if the number of integer digits are different then we can determine
-       * which decimal is larger and needn't go down to comparing individual values
-       */
-    if (leftIntRoundedUp > rightIntRoundedUp) {
-      return 1;
-    }
-    else if (rightIntRoundedUp > leftIntRoundedUp) {
-      return -1;
-    }
-
-      /* The number of integer digits are the same, set the each index
-       * to the first non-zero integer and compare each digit
-       */
-    leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
-    rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
-    while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
-      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
-        return 1;
-      }
-      else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
-        return -1;
-      }
-
-      leftIndex1++;
-      rightIndex1++;
-    }
-
-      /* The integer part of both the decimal's are equal, now compare
-       * each individual fractional part. Set the index to be at the
-       * beginning of the fractional part
-       */
-    leftIndex1 = leftStopIndex;
-    rightIndex1 = rightStopIndex;
-
-      /* Stop indexes will be the end of the array */
-    leftStopIndex = nDecimalDigits;
-    rightStopIndex = nDecimalDigits;
-
-      /* compare the two fractional parts of the decimal */
-    while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
-      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
-        return 1;
-      }
-      else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
-        return -1;
-      }
-
-      leftIndex1++;
-      rightIndex1++;
-    }
-
-      /* Till now the fractional part of the decimals are equal, check
-       * if one of the decimal has fractional part that is remaining
-       * and is non-zero
-       */
-    while (leftIndex1 < leftStopIndex) {
-      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
-        return 1;
-      }
-      leftIndex1++;
-    }
-
-    while(rightIndex1 < rightStopIndex) {
-      if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
-        return -1;
-      }
-      rightIndex1++;
-    }
-
-      /* Both decimal values are equal */
-    return 0;
-  }
-
-  public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
-    byte[] value = Arrays.copyOfRange(bytes, start, start + length);
-    BigInteger unscaledValue = new BigInteger(value);
-    return new BigDecimal(unscaledValue, scale);
-  }
-
-  public static void roundDecimal(ArrowBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
-    int newScaleRoundedUp  = org.apache.arrow.vector.util.DecimalUtility.roundUp(desiredScale);
-    int origScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(currentScale);
-
-    if (desiredScale < currentScale) {
-
-      boolean roundUp = false;
-
-      //Extract the first digit to be truncated to check if we need to round up
-      int truncatedScaleIndex = desiredScale + 1;
-      if (truncatedScaleIndex <= currentScale) {
-        int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
-        extractDigitIndex += org.apache.arrow.vector.util.DecimalUtility.roundUp(truncatedScaleIndex);
-        int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
-        int temp = org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS);
-        if (temp != 0) {
-          extractDigit = extractDigit / (int) (Math.pow(10, temp));
-        }
-        if ((extractDigit % 10)  > 4) {
-          roundUp = true;
-        }
-      }
-
-      // Get the source index beyond which we will truncate
-      int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
-      int srcIndex = srcIntIndex + newScaleRoundedUp;
-
-      // Truncate the remaining fractional part, move the integer part
-      int destIndex = nDecimalDigits - 1;
-      if (srcIndex != destIndex) {
-        while (srcIndex >= 0) {
-          setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
-        }
-
-        // Set the remaining portion of the decimal to be zeroes
-        while (destIndex >= 0) {
-          setInteger(result, start, destIndex--, 0);
-        }
-        srcIndex = nDecimalDigits - 1;
-      }
-
-      // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
-      int truncateFactor = org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS);
-      if (truncateFactor != org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS) {
-        truncateFactor = (int) Math.pow(10, truncateFactor);
-        int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
-        fractionalDigits /= truncateFactor;
-        setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
-      }
-
-      // Finally round up the digit if needed
-      if (roundUp == true) {
-        srcIndex = nDecimalDigits - 1;
-        int carry;
-        if (truncateFactor != org.apache.arrow.vector.util.DecimalUtility.MAX_DIGITS) {
-          carry = truncateFactor;
-        } else {
-          carry = 1;
-        }
-
-        while (srcIndex >= 0) {
-          int value = getIntegerFromSparseBuffer(result, start, srcIndex);
-          value += carry;
-
-          if (value >= org.apache.arrow.vector.util.DecimalUtility.DIGITS_BASE) {
-            setInteger(result, start, srcIndex--, value % org.apache.arrow.vector.util.DecimalUtility.DIGITS_BASE);
-            carry = value / org.apache.arrow.vector.util.DecimalUtility.DIGITS_BASE;
-          } else {
-            setInteger(result, start, srcIndex--, value);
-            carry = 0;
-            break;
-          }
-        }
-      }
-    } else if (desiredScale > currentScale) {
-      // Add fractional digits to the decimal
-
-      // Check if we need to shift the decimal digits to the left
-      if (newScaleRoundedUp > origScaleRoundedUp) {
-        int srcIndex  = 0;
-        int destIndex = newScaleRoundedUp - origScaleRoundedUp;
-
-        // Check while extending scale, we are not overwriting integer part
-        while (srcIndex < destIndex) {
-          if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
-            throw new RuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
-          }
-        }
-
-        srcIndex = 0;
-        while (destIndex < nDecimalDigits) {
-          setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
-        }
-
-        // Clear the remaining part
-        while (srcIndex < nDecimalDigits) {
-          setInteger(result, start, srcIndex++, 0);
-        }
-      }
-    }
-  }
-
-  public static int getFirstFractionalDigit(int decimal, int scale) {
-    if (scale == 0) {
-      return 0;
-    }
-    int temp = (int) adjustScaleDivide(decimal, scale - 1);
-    return Math.abs(temp % 10);
-  }
-
-  public static int getFirstFractionalDigit(long decimal, int scale) {
-    if (scale == 0) {
-      return 0;
-    }
-    long temp = adjustScaleDivide(decimal, scale - 1);
-    return (int) (Math.abs(temp % 10));
-  }
-
-  public static int getFirstFractionalDigit(ArrowBuf data, int scale, int start, int nDecimalDigits) {
-    if (scale == 0) {
-      return 0;
-    }
-
-    int index = nDecimalDigits - roundUp(scale);
-    return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1));
+    bytebuf.setBytes(startIndex + padLength, bytes, 0, bytes.length);
   }
 }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index 7ab7db3117b81..cca35e44a215d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -33,9 +33,10 @@ public class TestDecimalVector {
   private static long[] intValues;
 
   static {
-    intValues = new long[30];
-    for (int i = 0; i < intValues.length; i++) {
+    intValues = new long[60];
+    for (int i = 0; i < intValues.length / 2; i++) {
       intValues[i] = 1 << i + 1;
+      intValues[2 * i] = -1 * (1 << i + 1);
     }
   }
   private int scale = 3;
@@ -50,7 +51,7 @@ public void test() {
       BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale);
       values[i] = decimal;
       decimalVector.getMutator().setIndexDefined(i);
-      DecimalUtility.writeBigDecimalToArrowBuf(decimalVector.getBuffer(), i * 16, decimal);
+      DecimalUtility.writeBigDecimalToArrowBuf(decimal, decimalVector.getBuffer(), i);
     }
 
     decimalVector.getMutator().setValueCount(intValues.length);

From c2eb1612df34bee7baddc8851d24826d3c33faa6 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Fri, 19 Aug 2016 17:39:36 -0700
Subject: [PATCH 0116/1644] ARROW-265: Fix few decimal bugs

---
 .../AbstractPromotableFieldWriter.java        | 19 ++++++++++++++++---
 .../codegen/templates/FixedValueVectors.java  |  2 +-
 .../main/codegen/templates/MapWriters.java    |  2 +-
 .../org/apache/arrow/vector/types/Types.java  |  3 ++-
 .../arrow/vector/util/DecimalUtility.java     |  3 +--
 5 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 7e60320cfb8ac..d21dcd0f6461c 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -82,7 +82,18 @@ public void write(${name}Holder holder) {
     getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
   }
 
+  <#else>
+  @Override
+  public void write(DecimalHolder holder) {
+    getWriter(MinorType.DECIMAL).write(holder);
+  }
+
+  public void writeDecimal(int start, ArrowBuf buffer) {
+    getWriter(MinorType.DECIMAL).writeDecimal(start, buffer);
+  }
+
   </#if>
+
   </#list></#list>
 
   public void writeNull() {
@@ -113,8 +124,11 @@ public ListWriter list(String name) {
   <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
   <#assign upperName = minor.class?upper_case />
   <#assign capName = minor.class?cap_first />
-  <#if !minor.class?starts_with("Decimal") >
-
+  <#if minor.class?starts_with("Decimal") >
+  public ${capName}Writer ${lowerName}(String name, int scale, int precision) {
+    return getWriter(MinorType.MAP).${lowerName}(name, scale, precision);
+  }
+  </#if>
   @Override
   public ${capName}Writer ${lowerName}(String name) {
     return getWriter(MinorType.MAP).${lowerName}(name);
@@ -125,7 +139,6 @@ public ListWriter list(String name) {
     return getWriter(MinorType.LIST).${lowerName}();
   }
 
-  </#if>
   </#list></#list>
 
   public void copyReader(FieldReader reader) {
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 37946f6b76ea6..7958222f5c1bb 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -446,7 +446,7 @@ public void get(int index, Nullable${minor.class}Holder holder) {
 
     @Override
     public ${friendlyType} getObject(int index) {
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(data, index, scale);
+      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(data, ${type.width} * index, scale);
     }
 
     <#else>
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index af2922826ec4d..8a8983a1497cc 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -198,7 +198,7 @@ public void end() {
     if(writer == null) {
       ValueVector vector;
       ValueVector currentVector = container.getChild(name);
-      ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class);
+      ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class<#if minor.class == "Decimal"> , new int[] {precision, scale}</#if>);
       writer = new PromotableWriter(v, container);
       vector = v;
       if (currentVector == null || currentVector != vector) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 5ea1456a051f7..c34882a8fb12a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -47,6 +47,7 @@
 import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.BitWriterImpl;
 import org.apache.arrow.vector.complex.impl.DateWriterImpl;
+import org.apache.arrow.vector.complex.impl.DecimalWriterImpl;
 import org.apache.arrow.vector.complex.impl.Float4WriterImpl;
 import org.apache.arrow.vector.complex.impl.Float8WriterImpl;
 import org.apache.arrow.vector.complex.impl.IntWriterImpl;
@@ -386,7 +387,7 @@ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new VarBinaryWriterImpl((NullableVarBinaryVector) vector);
+        return new DecimalWriterImpl((NullableDecimalVector) vector);
       }
     },   //  variable length binary
     UINT1(new Int(8, false)) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index e171e87360d86..4c439b2cc1066 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -135,9 +135,8 @@ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
     return str;
   }
 
-  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale) {
+  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int startIndex, int scale) {
     byte[] value = new byte[DECIMAL_BYTE_LENGTH];
-    final int startIndex = index * DECIMAL_BYTE_LENGTH;
     bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH);
     BigInteger unscaledValue = new BigInteger(value);
     return new BigDecimal(unscaledValue, scale);

From 812201a2db1ebabd0f65ebd2774ec8f0880bb8cb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Fri, 19 Aug 2016 18:05:16 -0700
Subject: [PATCH 0117/1644] ARROW-266: [C++] Fix broken build due to
 Flatbuffers namespace change

Author: Wes McKinney <wesm@apache.org>

Closes #122 from wesm/ARROW-266 and squashes the following commits:

6193323 [Wes McKinney] Fix broken build due to Flatbuffers namespace change
---
 cpp/src/arrow/ipc/adapter.cc           | 2 +-
 cpp/src/arrow/ipc/metadata-internal.cc | 2 +-
 cpp/src/arrow/ipc/metadata-internal.h  | 2 +-
 cpp/src/arrow/ipc/metadata.cc          | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 3259980058b8e..40d372bbd3520 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -41,7 +41,7 @@
 
 namespace arrow {
 
-namespace flatbuf = apache::arrow::flatbuf;
+namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 8cd416ff5853f..16ba20f7e90ee 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -37,7 +37,7 @@ typedef flatbuffers::Offset<void> Offset;
 
 namespace arrow {
 
-namespace flatbuf = apache::arrow::flatbuf;
+namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index 871b5bc4bf606..5faa8c947b55d 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -28,7 +28,7 @@
 
 namespace arrow {
 
-namespace flatbuf = apache::arrow::flatbuf;
+namespace flatbuf = org::apache::arrow::flatbuf;
 
 class Buffer;
 struct Field;
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 4fc8ec50eb716..e510755110e04 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -33,7 +33,7 @@
 
 namespace arrow {
 
-namespace flatbuf = apache::arrow::flatbuf;
+namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 

From 78619686f44da5a28319032551b07ddfadc26468 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Sat, 20 Aug 2016 10:39:50 -0700
Subject: [PATCH 0118/1644] ARROW-252: Add implementation guidelines to the
 documentation

Author: Julien Le Dem <julien@dremio.com>

Closes #120 from julienledem/arrow_252_impl_guidelines and squashes the following commits:

caf6994 [Julien Le Dem] ARROW-252: review feedback
6b68ce1 [Julien Le Dem] ARROW-252: Add implementation guidelines to the documentation
---
 format/Guidelines.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 format/Guidelines.md

diff --git a/format/Guidelines.md b/format/Guidelines.md
new file mode 100644
index 0000000000000..14f1057850439
--- /dev/null
+++ b/format/Guidelines.md
@@ -0,0 +1,17 @@
+# Implementation guidelines
+
+An execution engine (or framework, or UDF executor, or storage engine, etc) can implements only a subset of the Arrow spec and/or extend it given the following constraints:
+
+## Implementing a subset the spec
+### If only producing (and not consuming) arrow vectors.
+Any subset of the vector spec and the corresponding metadata can be implemented.
+
+### If consuming and producing vectors
+There is a minimal subset of vectors to be supported.
+Production of a subset of vectors and their corresponding metadata is always fine.
+Consumption of vectors should at least convert the unsupported input vectors to the supported subset (for example Timestamp.millis to timestamp.micros or int32 to int64)
+
+## Extensibility
+An execution engine implementor can also extend their memory representation with their own vectors internally as long as they are never exposed. Before sending data to another system expecting Arrow data these custom vectors should be converted to a type that exist in the Arrow spec.
+An example of this is operating on compressed data.
+These custom vectors are not exchanged externaly and there is no support for custom metadata.

From 8960a2ed4c0d400be32003beb183f150e019c4ec Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Sat, 20 Aug 2016 13:02:45 -0700
Subject: [PATCH 0119/1644] ARROW-255: Finalize Dictionary representation

Author: Julien Le Dem <julien@dremio.com>

Closes #119 from julienledem/arrow_255_dictionary and squashes the following commits:

316745d [Julien Le Dem] ARROW-255: fix typo and linter errors
e28a3c8 [Julien Le Dem] ARROW-255: review feedback
8c27943 [Julien Le Dem] ARROW-255: Finalize Dictionary representation
---
 cpp/src/arrow/ipc/metadata-internal.cc |  3 ++-
 cpp/src/arrow/type.h                   | 11 +++++---
 format/Layout.md                       | 37 ++++++++++++++++++++++++++
 format/Message.fbs                     |  6 ++++-
 4 files changed, 52 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 16ba20f7e90ee..50db730d20832 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -220,7 +220,8 @@ static Status FieldToFlatbuffer(
   auto fb_children = fbb.CreateVector(children);
 
   *offset = flatbuf::CreateField(
-      fbb, fb_name, field->nullable, type_enum, type_data, fb_children);
+      fbb, fb_name, field->nullable, type_enum, type_data, field->dictionary,
+      fb_children);
 
   return Status::OK();
 }
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 4cb37fd1dead8..02677d5e18b90 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -144,8 +144,13 @@ struct ARROW_EXPORT Field {
   // Fields can be nullable
   bool nullable;
 
-  Field(const std::string& name, const TypePtr& type, bool nullable = true)
-      : name(name), type(type), nullable(nullable) {}
+  // optional dictionary id if the field is dictionary encoded
+  // 0 means it's not dictionary encoded
+  int64_t dictionary;
+
+  Field(const std::string& name, const TypePtr& type, bool nullable = true,
+        int64_t dictionary = 0)
+      : name(name), type(type), nullable(nullable), dictionary(dictionary) {}
 
   bool operator==(const Field& other) const { return this->Equals(other); }
 
@@ -154,7 +159,7 @@ struct ARROW_EXPORT Field {
   bool Equals(const Field& other) const {
     return (this == &other) ||
            (this->name == other.name && this->nullable == other.nullable &&
-               this->type->Equals(other.type.get()));
+            this->dictionary == dictionary && this->type->Equals(other.type.get()));
   }
 
   bool Equals(const std::shared_ptr<Field>& other) const { return Equals(*other.get()); }
diff --git a/format/Layout.md b/format/Layout.md
index 5eaefeebf210a..a953930e172e7 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -583,6 +583,43 @@ even if the null bitmap of the parent union array indicates the slot is
 null.  Additionally, a child array may have a non-null slot even if
 the the types array indicates that a slot contains a different type at the index.
 
+## Dictionary encoding
+
+When a field is dictionary encoded, the values are represented by an array of Int32 representing the index of the value in the dictionary.
+The Dictionary is received as a DictionaryBacth whose id is referenced by a dictionary attribute defined in the metadata (Message.fbs) in the Field table.
+The dictionary has the same layout as the type of the field would dictate. Each entry in the dictionary can be accessed by its index in the DictionaryBatch.
+When a Schema references a Dictionary id, it must send a DictionaryBatch for this id before any RecordBatch.
+
+As an example, you could have the following data:
+```
+type: List<String>
+
+[
+ ['a', 'b'],
+ ['a', 'b'],
+ ['a', 'b'],
+ ['c', 'd', 'e'],
+ ['c', 'd', 'e'],
+ ['c', 'd', 'e'],
+ ['c', 'd', 'e'],
+ ['a', 'b']
+]
+```
+In dictionary-encoded form, this could appear as:
+```
+data List<String> (dictionary-encoded, dictionary id i)
+indices: [0, 0, 0, 1, 1, 1, 0]
+
+dictionary i
+
+type: List<String>
+
+[
+ ['a', 'b'],
+ ['c', 'd', 'e'],
+]
+```
+
 ## References
 
 Apache Drill Documentation - [Value Vectors][6] 
diff --git a/format/Message.fbs b/format/Message.fbs
index 2928207db8cc0..a78009b6e5f94 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -104,6 +104,10 @@ table Field {
   name: string;
   nullable: bool;
   type: Type;
+  // present only if the field is dictionary encoded
+  // will point to a dictionary provided by a DictionaryBatch message
+  dictionary: long;
+  // children apply only to Nested data types like Struct, List and Union
   children: [Field];
 }
 
@@ -185,8 +189,8 @@ table RecordBatch {
 /// For sending dictionary encoding information. Any Field can be
 /// dictionary-encoded, but in this case none of its children may be
 /// dictionary-encoded.
+/// There is one dictionary batch per dictionary
 ///
-/// TODO(wesm): To be documented in more detail
 
 table DictionaryBatch {
   id: long;

From ec51d566708f5d6ea0a94a6d53152dc8cc98d6aa Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Mon, 22 Aug 2016 13:10:06 -0700
Subject: [PATCH 0120/1644] ARROW-269: Include typeVector buffers
 UnionVector.getBuffers()

---
 .../main/codegen/templates/UnionVector.java   | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index e2f19f4b33ba5..1fef490d4ec3c 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -16,7 +16,9 @@
  * limitations under the License.
  */
 
+import com.google.common.collect.ImmutableList;
 import com.google.flatbuffers.FlatBufferBuilder;
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.flatbuf.Field;
 import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.flatbuf.Union;
@@ -35,6 +37,7 @@
 package org.apache.arrow.vector.complex;
 
 <#include "/@includes/vv_imports.ftl" />
+import com.google.common.collect.ImmutableList;
 import java.util.ArrayList;
 import java.util.Iterator;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
@@ -219,6 +222,7 @@ public TransferPair makeTransferPair(ValueVector target) {
   }
 
   public void transferTo(org.apache.arrow.vector.complex.UnionVector target) {
+    typeVector.makeTransferPair(target.typeVector).transfer();
     internalMap.makeTransferPair(target.internalMap).transfer();
     target.valueCount = valueCount;
   }
@@ -307,20 +311,9 @@ public FieldWriter getWriter() {
     return mutator.writer;
   }
 
-//  @Override
-//  public UserBitShared.SerializedField getMetadata() {
-//    SerializedField.Builder b = getField() //
-//            .getAsBuilder() //
-//            .setBufferLength(getBufferSize()) //
-//            .setValueCount(valueCount);
-//
-//    b.addChild(internalMap.getMetadata());
-//    return b.build();
-//  }
-
   @Override
   public int getBufferSize() {
-    return internalMap.getBufferSize();
+    return typeVector.getBufferSize() + internalMap.getBufferSize();
   }
 
   @Override
@@ -339,7 +332,11 @@ public int getBufferSizeFor(final int valueCount) {
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    return internalMap.getBuffers(clear);
+    ImmutableList.Builder<ArrowBuf> builder = ImmutableList.builder();
+    builder.add(typeVector.getBuffers(clear));
+    builder.add(internalMap.getBuffers(clear));
+    List<ArrowBuf> list = builder.build();
+    return list.toArray(new ArrowBuf[list.size()]);
   }
 
   @Override
@@ -411,6 +408,7 @@ public class Mutator extends BaseValueVector.BaseMutator {
     @Override
     public void setValueCount(int valueCount) {
       UnionVector.this.valueCount = valueCount;
+      typeVector.getMutator().setValueCount(valueCount);
       internalMap.getMutator().setValueCount(valueCount);
     }
 

From 803afeb502dcdd802fada2ed0d66c145546b8a78 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 26 Aug 2016 08:20:13 -0700
Subject: [PATCH 0121/1644] ARROW-264: File format

This is work in progress

Author: Julien Le Dem <julien@dremio.com>

Closes #123 from julienledem/arrow_264_file_format and squashes the following commits:

252de6d [Julien Le Dem] remove outdated comment
04d797f [Julien Le Dem] maps are not nullable yet
e8359b3 [Julien Le Dem] align on 8 byte boundaries; more tests
8b8b823 [Julien Le Dem] refactoring
31e95e6 [Julien Le Dem] fix list vector
b824938 [Julien Le Dem] fix types; add licenses; more tests; more complex
2fd3bc1 [Julien Le Dem] cleanup
50fe680 [Julien Le Dem] nested support
b0bf6bc [Julien Le Dem] cleanup
4247b1a [Julien Le Dem] fix whitespace
d6a1788 [Julien Le Dem] refactoring
81863c5 [Julien Le Dem] fixed loader
aa1b766 [Julien Le Dem] better test
2067e01 [Julien Le Dem] update format
aacf61e [Julien Le Dem] fix pom
b907aa5 [Julien Le Dem] simplify
e43f26b [Julien Le Dem] add layout spec
0cc9718 [Julien Le Dem] add vector type
ac6902a [Julien Le Dem] ARROW-264: File format
807db51 [Julien Le Dem] move information to schema
f2f0596 [Julien Le Dem] Update FieldNode structure to be more explicit and reflect schema
---
 cpp/src/arrow/ipc/metadata-internal.cc        |   1 +
 format/File.fbs                               |  28 ++
 format/Message.fbs                            |  21 +-
 java/format/pom.xml                           |   1 +
 .../main/java/io/netty/buffer/ArrowBuf.java   |  71 ++--
 .../src/main/codegen/data/ArrowTypes.tdd      |   4 +-
 .../src/main/codegen/templates/ArrowType.java |  29 +-
 .../templates/NullableValueVectors.java       |  49 ++-
 .../main/codegen/templates/UnionVector.java   |  40 ++-
 .../arrow/vector/BaseDataValueVector.java     |  38 +-
 .../org/apache/arrow/vector/BufferBacked.java |  31 ++
 .../org/apache/arrow/vector/FieldVector.java  |  65 ++++
 .../org/apache/arrow/vector/ValueVector.java  |   6 +-
 .../org/apache/arrow/vector/VectorLoader.java |  99 ++++++
 .../apache/arrow/vector/VectorUnloader.java   |  78 +++++
 .../org/apache/arrow/vector/ZeroVector.java   |  39 ++-
 .../complex/AbstractContainerVector.java      |  21 +-
 .../vector/complex/AbstractMapVector.java     |  42 ++-
 .../complex/BaseRepeatedValueVector.java      |  21 +-
 .../arrow/vector/complex/ListVector.java      |  58 ++-
 .../arrow/vector/complex/MapVector.java       |  59 +++-
 .../complex/impl/ComplexWriterImpl.java       |   2 +-
 .../vector/complex/impl/PromotableWriter.java |   3 +-
 .../apache/arrow/vector/file/ArrowBlock.java  |  82 +++++
 .../apache/arrow/vector/file/ArrowFooter.java | 144 ++++++++
 .../apache/arrow/vector/file/ArrowReader.java | 151 ++++++++
 .../apache/arrow/vector/file/ArrowWriter.java | 179 ++++++++++
 .../file/InvalidArrowFileException.java       |  27 ++
 .../arrow/vector/schema/ArrowBuffer.java      |  81 +++++
 .../arrow/vector/schema/ArrowFieldNode.java   |  53 +++
 .../arrow/vector/schema/ArrowRecordBatch.java | 127 +++++++
 .../arrow/vector/schema/ArrowVectorType.java  |  47 +++
 .../arrow/vector/schema/FBSerializable.java   |  24 ++
 .../arrow/vector/schema/FBSerializables.java  |  37 ++
 .../arrow/vector/schema/TypeLayout.java       | 208 +++++++++++
 .../arrow/vector/schema/VectorLayout.java     |  93 +++++
 .../org/apache/arrow/vector/types/Types.java  |  70 ++--
 .../apache/arrow/vector/types/pojo/Field.java |  42 ++-
 .../arrow/vector/types/pojo/Schema.java       |  13 +-
 .../arrow/vector/TestVectorUnloadLoad.java    |  89 +++++
 .../ByteArrayReadableSeekableByteChannel.java |  80 +++++
 .../arrow/vector/file/TestArrowFile.java      | 331 ++++++++++++++++++
 .../arrow/vector/file/TestArrowFooter.java    |  56 +++
 .../vector/file/TestArrowReaderWriter.java    | 106 ++++++
 .../apache/arrow/vector/pojo/TestConvert.java |  38 +-
 45 files changed, 2722 insertions(+), 162 deletions(-)
 create mode 100644 format/File.fbs
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/file/ByteArrayReadableSeekableByteChannel.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java

diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 50db730d20832..c921e4d8e0114 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -219,6 +219,7 @@ static Status FieldToFlatbuffer(
   RETURN_NOT_OK(TypeToFlatbuffer(fbb, field->type, &children, &type_enum, &type_data));
   auto fb_children = fbb.CreateVector(children);
 
+  // TODO: produce the list of VectorTypes
   *offset = flatbuf::CreateField(
       fbb, fb_name, field->nullable, type_enum, type_data, field->dictionary,
       fb_children);
diff --git a/format/File.fbs b/format/File.fbs
new file mode 100644
index 0000000000000..f7ad1e1594a91
--- /dev/null
+++ b/format/File.fbs
@@ -0,0 +1,28 @@
+include "Message.fbs";
+
+namespace org.apache.arrow.flatbuf;
+
+/// ----------------------------------------------------------------------
+/// Arrow File metadata
+///
+
+table Footer {
+
+  schema: org.apache.arrow.flatbuf.Schema;
+
+  dictionaries: [ Block ];
+
+  recordBatches: [ Block ];
+}
+
+struct Block {
+
+  offset: long;
+
+  metaDataLength: int;
+
+  bodyLength: long;
+
+}
+
+root_type Footer;
diff --git a/format/Message.fbs b/format/Message.fbs
index a78009b6e5f94..b02f3fa38694e 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -17,7 +17,7 @@ table Tuple {
 table List {
 }
 
-enum UnionMode:int { Sparse, Dense }
+enum UnionMode:short { Sparse, Dense }
 
 table Union {
   mode: UnionMode;
@@ -28,7 +28,7 @@ table Int {
   is_signed: bool;
 }
 
-enum Precision:int {SINGLE, DOUBLE}
+enum Precision:short {SINGLE, DOUBLE}
 
 table FloatingPoint {
   precision: Precision;
@@ -91,6 +91,17 @@ union Type {
   JSONScalar
 }
 
+enum VectorType: short {
+  /// used in List type Dense Union and variable length primitive types (String, Binary)
+  OFFSET,
+  /// fixed length primitive values
+  VALUES,
+  /// Bit vector indicated if each value is null
+  VALIDITY,
+  /// Type vector used in Union type
+  TYPE
+}
+
 /// ----------------------------------------------------------------------
 /// A field represents a named column in a record / row batch or child of a
 /// nested type.
@@ -109,12 +120,16 @@ table Field {
   dictionary: long;
   // children apply only to Nested data types like Struct, List and Union
   children: [Field];
+  /// the buffers produced for this type (as derived from the Type)
+  /// does not include children
+  /// each recordbatch will return instances of those Buffers.
+  buffers: [ VectorType ];
 }
 
 /// ----------------------------------------------------------------------
 /// Endianness of the platform that produces the RecordBatch
 
-enum Endianness:int { Little, Big }
+enum Endianness:short { Little, Big }
 
 /// ----------------------------------------------------------------------
 /// A Schema describes the columns in a row batch
diff --git a/java/format/pom.xml b/java/format/pom.xml
index cb11b5ff3c45d..dc5897581b5b3 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -106,6 +106,7 @@
               <argument>-o</argument>
               <argument>target/generated-sources/</argument>
               <argument>../../format/Message.fbs</argument>
+              <argument>../../format/File.fbs</argument>
             </arguments>
           </configuration>
         </execution>
diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index bbec26aa85c74..d10f00247e6ee 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -17,8 +17,6 @@
  */
 package io.netty.buffer;
 
-import io.netty.util.internal.PlatformDependent;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -30,16 +28,18 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.arrow.memory.AllocationManager.BufferLedger;
 import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.memory.BaseAllocator.Verbosity;
 import org.apache.arrow.memory.BoundsChecking;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.BufferManager;
-import org.apache.arrow.memory.AllocationManager.BufferLedger;
-import org.apache.arrow.memory.BaseAllocator.Verbosity;
 import org.apache.arrow.memory.util.HistoricalLog;
 
 import com.google.common.base.Preconditions;
 
+import io.netty.util.internal.PlatformDependent;
+
 public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ArrowBuf.class);
 
@@ -307,7 +307,7 @@ public ByteOrder order() {
   }
 
   @Override
-  public ByteBuf order(ByteOrder endianness) {
+  public ArrowBuf order(ByteOrder endianness) {
     return this;
   }
 
@@ -344,7 +344,7 @@ public ByteBuf copy(int index, int length) {
   }
 
   @Override
-  public ByteBuf slice() {
+  public ArrowBuf slice() {
     return slice(readerIndex(), readableBytes());
   }
 
@@ -467,7 +467,7 @@ public boolean equals(Object obj) {
   }
 
   @Override
-  public ByteBuf retain(int increment) {
+  public ArrowBuf retain(int increment) {
     Preconditions.checkArgument(increment > 0, "retain(%d) argument is not positive", increment);
 
     if (isEmpty) {
@@ -484,7 +484,7 @@ public ByteBuf retain(int increment) {
   }
 
   @Override
-  public ByteBuf retain() {
+  public ArrowBuf retain() {
     return retain(1);
   }
 
@@ -535,49 +535,49 @@ public short getShort(int index) {
   }
 
   @Override
-  public ByteBuf setShort(int index, int value) {
+  public ArrowBuf setShort(int index, int value) {
     chk(index, 2);
     PlatformDependent.putShort(addr(index), (short) value);
     return this;
   }
 
   @Override
-  public ByteBuf setInt(int index, int value) {
+  public ArrowBuf setInt(int index, int value) {
     chk(index, 4);
     PlatformDependent.putInt(addr(index), value);
     return this;
   }
 
   @Override
-  public ByteBuf setLong(int index, long value) {
+  public ArrowBuf setLong(int index, long value) {
     chk(index, 8);
     PlatformDependent.putLong(addr(index), value);
     return this;
   }
 
   @Override
-  public ByteBuf setChar(int index, int value) {
+  public ArrowBuf setChar(int index, int value) {
     chk(index, 2);
     PlatformDependent.putShort(addr(index), (short) value);
     return this;
   }
 
   @Override
-  public ByteBuf setFloat(int index, float value) {
+  public ArrowBuf setFloat(int index, float value) {
     chk(index, 4);
     PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value));
     return this;
   }
 
   @Override
-  public ByteBuf setDouble(int index, double value) {
+  public ArrowBuf setDouble(int index, double value) {
     chk(index, 8);
     PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value));
     return this;
   }
 
   @Override
-  public ByteBuf writeShort(int value) {
+  public ArrowBuf writeShort(int value) {
     ensure(2);
     PlatformDependent.putShort(addr(writerIndex), (short) value);
     writerIndex += 2;
@@ -585,7 +585,7 @@ public ByteBuf writeShort(int value) {
   }
 
   @Override
-  public ByteBuf writeInt(int value) {
+  public ArrowBuf writeInt(int value) {
     ensure(4);
     PlatformDependent.putInt(addr(writerIndex), value);
     writerIndex += 4;
@@ -593,7 +593,7 @@ public ByteBuf writeInt(int value) {
   }
 
   @Override
-  public ByteBuf writeLong(long value) {
+  public ArrowBuf writeLong(long value) {
     ensure(8);
     PlatformDependent.putLong(addr(writerIndex), value);
     writerIndex += 8;
@@ -601,7 +601,7 @@ public ByteBuf writeLong(long value) {
   }
 
   @Override
-  public ByteBuf writeChar(int value) {
+  public ArrowBuf writeChar(int value) {
     ensure(2);
     PlatformDependent.putShort(addr(writerIndex), (short) value);
     writerIndex += 2;
@@ -609,7 +609,7 @@ public ByteBuf writeChar(int value) {
   }
 
   @Override
-  public ByteBuf writeFloat(float value) {
+  public ArrowBuf writeFloat(float value) {
     ensure(4);
     PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value));
     writerIndex += 4;
@@ -617,7 +617,7 @@ public ByteBuf writeFloat(float value) {
   }
 
   @Override
-  public ByteBuf writeDouble(double value) {
+  public ArrowBuf writeDouble(double value) {
     ensure(8);
     PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value));
     writerIndex += 8;
@@ -625,19 +625,19 @@ public ByteBuf writeDouble(double value) {
   }
 
   @Override
-  public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+  public ArrowBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
     udle.getBytes(index + offset, dst, dstIndex, length);
     return this;
   }
 
   @Override
-  public ByteBuf getBytes(int index, ByteBuffer dst) {
+  public ArrowBuf getBytes(int index, ByteBuffer dst) {
     udle.getBytes(index + offset, dst);
     return this;
   }
 
   @Override
-  public ByteBuf setByte(int index, int value) {
+  public ArrowBuf setByte(int index, int value) {
     chk(index, 1);
     PlatformDependent.putByte(addr(index), (byte) value);
     return this;
@@ -699,13 +699,13 @@ protected void _setLong(int index, long value) {
   }
 
   @Override
-  public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+  public ArrowBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
     udle.getBytes(index + offset, dst, dstIndex, length);
     return this;
   }
 
   @Override
-  public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
+  public ArrowBuf getBytes(int index, OutputStream out, int length) throws IOException {
     udle.getBytes(index + offset, out, length);
     return this;
   }
@@ -724,12 +724,12 @@ public int getBytes(int index, GatheringByteChannel out, int length) throws IOEx
   }
 
   @Override
-  public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+  public ArrowBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
     udle.setBytes(index + offset, src, srcIndex, length);
     return this;
   }
 
-  public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
+  public ArrowBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
     if (src.isDirect()) {
       checkIndex(index, length);
       PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this.memoryAddress() + index,
@@ -749,13 +749,13 @@ public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
   }
 
   @Override
-  public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+  public ArrowBuf setBytes(int index, byte[] src, int srcIndex, int length) {
     udle.setBytes(index + offset, src, srcIndex, length);
     return this;
   }
 
   @Override
-  public ByteBuf setBytes(int index, ByteBuffer src) {
+  public ArrowBuf setBytes(int index, ByteBuffer src) {
     udle.setBytes(index + offset, src);
     return this;
   }
@@ -860,4 +860,17 @@ public void print(StringBuilder sb, int indent, Verbosity verbosity) {
     }
   }
 
+  @Override
+  public ArrowBuf readerIndex(int readerIndex) {
+    super.readerIndex(readerIndex);
+    return this;
+  }
+
+  @Override
+  public ArrowBuf writerIndex(int writerIndex) {
+    super.writerIndex(writerIndex);
+    return this;
+  }
+
+
 }
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 4ab7f8562f907..2ecad3d31400f 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -30,7 +30,7 @@
     },
     {
       name: "Union",
-      fields: []
+      fields: [{name: "mode", type: short}]
     },
     {
       name: "Int",
@@ -38,7 +38,7 @@
     },
     {
       name: "FloatingPoint",
-      fields: [{name: precision, type: int}]
+      fields: [{name: precision, type: short}]
     },
     {
       name: "Utf8",
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index 6dfaf216ad042..29dee20040a53 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -24,9 +24,8 @@
 
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/types/pojo/ArrowType.java" />
-
-
 <#include "/@includes/license.ftl" />
+
 package org.apache.arrow.vector.types.pojo;
 
 import com.google.flatbuffers.FlatBufferBuilder;
@@ -38,7 +37,13 @@ public abstract class ArrowType {
 
   public abstract byte getTypeType();
   public abstract int getType(FlatBufferBuilder builder);
+  public abstract <T> T accept(ArrowTypeVisitor<T> visitor);
 
+  public static interface ArrowTypeVisitor<T> {
+  <#list arrowTypes.types as type>
+    T visit(${type.name} type);
+  </#list>
+  }
 
   <#list arrowTypes.types as type>
   <#assign name = type.name>
@@ -70,9 +75,14 @@ public byte getTypeType() {
 
     @Override
     public int getType(FlatBufferBuilder builder) {
+      <#list type.fields as field>
+      <#if field.type == "String">
+      int ${field.name} = builder.createString(this.${field.name});
+      </#if>
+      </#list>
       org.apache.arrow.flatbuf.${type.name}.start${type.name}(builder);
       <#list type.fields as field>
-      org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, <#if field.type == "String">builder.createString(${field.name})<#else>${field.name}</#if>);
+      org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, ${field.name});
       </#list>
       return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
     }
@@ -83,6 +93,14 @@ public int getType(FlatBufferBuilder builder) {
     }
     </#list>
 
+    public String toString() {
+      return "${name}{"
+      <#list fields as field>
+      + ", " + ${field.name}
+      </#list>
+      + "}";
+    }
+
     @Override
     public int hashCode() {
       return Objects.hash(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
@@ -102,6 +120,11 @@ public boolean equals(Object obj) {
       </#list>
       </#if>
     }
+
+    @Override
+    public <T> T accept(ArrowTypeVisitor<T> visitor) {
+      return visitor.visit(this);
+    }
   }
   </#list>
 
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index df508979c48b5..6b1aa040a5ba2 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -29,6 +29,9 @@
 
 package org.apache.arrow.vector;
 
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import java.util.Collections;
+
 <#include "/@includes/vv_imports.ftl" />
 
 /**
@@ -39,7 +42,7 @@
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
 @SuppressWarnings("unused")
-public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector{
+public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector, FieldVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
   private final FieldReader reader = new ${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
@@ -54,6 +57,8 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   private final Mutator mutator;
   private final Accessor accessor;
 
+  private final List<BufferBacked> innerVectors;
+
   <#if minor.class == "Decimal">
   private final int precision;
   private final int scale;
@@ -66,6 +71,10 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     mutator = new Mutator();
     accessor = new Accessor();
     field = new Field(name, true, new Decimal(precision, scale), null);
+    innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
+        bits,
+        values
+    ));
   }
   <#else>
   public ${className}(String name, BufferAllocator allocator) {
@@ -88,9 +97,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#elseif minor.class == "Time">
     field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), null);
   <#elseif minor.class == "Float4">
-    field = new Field(name, true, new FloatingPoint(0), null);
+    field = new Field(name, true, new FloatingPoint(org.apache.arrow.flatbuf.Precision.SINGLE), null);
   <#elseif minor.class == "Float8">
-    field = new Field(name, true, new FloatingPoint(1), null);
+    field = new Field(name, true, new FloatingPoint(org.apache.arrow.flatbuf.Precision.DOUBLE), null);
   <#elseif minor.class == "TimeStamp">
     field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(""), null);
   <#elseif minor.class == "IntervalDay">
@@ -104,9 +113,43 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#elseif minor.class == "Bit">
     field = new Field(name, true, new Bool(), null);
   </#if>
+    innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
+        bits,
+        <#if type.major = "VarLen">
+        values.offsetVector,
+        </#if>
+        values
+    ));
   }
   </#if>
 
+  @Override
+  public List<BufferBacked> getFieldInnerVectors() {
+    return innerVectors;
+  }
+
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    if (!children.isEmpty()) {
+      throw new IllegalArgumentException("primitive type vector ${className} can not have children: " + children);
+    }
+  }
+
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    org.apache.arrow.vector.BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+    // TODO: do something with the sizes in fieldNode?
+  }
+
+  public List<ArrowBuf> getFieldBuffers() {
+    return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors());
+  }
+
   @Override
   public Field getField() {
     return field;
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 1fef490d4ec3c..72125fa50fb82 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -42,6 +42,10 @@
 import java.util.Iterator;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+
+import static org.apache.arrow.flatbuf.UnionMode.Sparse;
+
 
 /*
  * This class is generated using freemarker and the ${.template_name} template.
@@ -57,7 +61,7 @@
  * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
  * each time the vector is accessed.
  */
-public class UnionVector implements ValueVector {
+public class UnionVector implements FieldVector {
 
   private String name;
   private BufferAllocator allocator;
@@ -95,6 +99,34 @@ public MinorType getMinorType() {
     return MinorType.UNION;
   }
 
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    getMap().initializeChildrenFromFields(children);
+  }
+
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return getMap().getChildrenFromFields();
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    // TODO
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    // TODO
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public List<BufferBacked> getFieldInnerVectors() {
+    // TODO
+    throw new UnsupportedOperationException();
+  }
+  
   public MapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
@@ -203,7 +235,7 @@ public Field getField() {
     for (ValueVector v : internalMap.getChildren()) {
       childFields.add(v.getField());
     }
-    return new Field(name, true, new ArrowType.Union(), childFields);
+    return new Field(name, true, new ArrowType.Union(Sparse), childFields);
   }
 
   @Override
@@ -237,10 +269,10 @@ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
     copyFrom(inIndex, outIndex, from);
   }
 
-  public ValueVector addVector(ValueVector v) {
+  public FieldVector addVector(FieldVector v) {
     String name = v.getMinorType().name().toLowerCase();
     Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
-    final ValueVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass());
+    final FieldVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass());
     v.makeTransferPair(newVector).transfer();
     internalMap.putChild(name, newVector);
     if (callBack != null) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 05b7cf1006723..c22258d42651b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -17,15 +17,38 @@
  */
 package org.apache.arrow.vector;
 
-import io.netty.buffer.ArrowBuf;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 
+import io.netty.buffer.ArrowBuf;
+
 
-public abstract class BaseDataValueVector extends BaseValueVector {
+public abstract class BaseDataValueVector extends BaseValueVector implements BufferBacked {
 
   protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
 
+  public static void load(List<BufferBacked> vectors, List<ArrowBuf> buffers) {
+    int expectedSize = vectors.size();
+    if (buffers.size() != expectedSize) {
+      throw new IllegalArgumentException("Illegal buffer count, expected " + expectedSize + ", got: " + buffers.size());
+    }
+    for (int i = 0; i < expectedSize; i++) {
+      vectors.get(i).load(buffers.get(i));
+    }
+  }
+
+  public static List<ArrowBuf> unload(List<BufferBacked> vectors) {
+    List<ArrowBuf> result = new ArrayList<>(vectors.size());
+    for (BufferBacked vector : vectors) {
+      result.add(vector.unLoad());
+    }
+    return result;
+  }
+
+  // TODO: Nullable vectors extend BaseDataValueVector but do not use the data field
+  // We should fix the inheritance tree
   protected ArrowBuf data;
 
   public BaseDataValueVector(String name, BufferAllocator allocator) {
@@ -82,6 +105,17 @@ public ArrowBuf getBuffer() {
     return data;
   }
 
+  @Override
+  public void load(ArrowBuf data) {
+    this.data.release();
+    this.data = data.retain(allocator);
+  }
+
+  @Override
+  public ArrowBuf unLoad() {
+    return this.data.readerIndex(0);
+  }
+
   /**
    * This method has a similar effect of allocateNew() without actually clearing and reallocating
    * the value vector. The purpose is to move the value vector to a "mutate" state
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
new file mode 100644
index 0000000000000..d1c262d226556
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Content is backed by a buffer and can be loaded/unloaded
+ */
+public interface BufferBacked {
+
+  void load(ArrowBuf data);
+
+  ArrowBuf unLoad();
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
new file mode 100644
index 0000000000000..b28433cfd0d94
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.util.List;
+
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.Field;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * A vector corresponding to a Field in the schema
+ * It has inner vectors backed by buffers (validity, offsets, data, ...)
+ */
+public interface FieldVector extends ValueVector {
+
+  /**
+   * Initializes the child vectors
+   * to be later loaded with loadBuffers
+   * @param children the schema
+   */
+  void initializeChildrenFromFields(List<Field> children);
+
+  /**
+   * the returned list is the same size as the list passed to initializeChildrenFromFields
+   * @return the children according to schema (empty for primitive types)
+   */
+  List<FieldVector> getChildrenFromFields();
+
+  /**
+   * loads data in the vectors
+   * (ownBuffers must be the same size as getFieldVectors())
+   * @param fieldNode the fieldNode
+   * @param ownBuffers the buffers for this Field (own buffers only, children not included)
+   */
+  void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers);
+
+  /**
+   * (same size as getFieldVectors() since it is their content)
+   * @return the buffers containing the data for this vector (ready for reading)
+   */
+  List<ArrowBuf> getFieldBuffers();
+
+  /**
+   * @return the inner vectors for this field as defined by the TypeLayout
+   */
+  List<BufferBacked> getFieldInnerVectors();
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 35321c947db0b..ba7790e47ef95 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -19,14 +19,14 @@
 
 import java.io.Closeable;
 
-import io.netty.buffer.ArrowBuf;
-
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.TransferPair;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.TransferPair;
+
+import io.netty.buffer.ArrowBuf;
 
 /**
  * An abstraction that is used to store a sequence of values in an individual column.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
new file mode 100644
index 0000000000000..58ac68b82825d
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.schema.VectorLayout;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.collect.Iterators;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Loads buffers into vectors
+ */
+public class VectorLoader {
+  private final List<FieldVector> fieldVectors;
+  private final List<Field> fields;
+
+  /**
+   * will create children in root based on schema
+   * @param schema the expected schema
+   * @param root the root to add vectors to based on schema
+   */
+  public VectorLoader(Schema schema, FieldVector root) {
+    super();
+    this.fields = schema.getFields();
+    root.initializeChildrenFromFields(fields);
+    this.fieldVectors = root.getChildrenFromFields();
+    if (this.fieldVectors.size() != fields.size()) {
+      throw new IllegalArgumentException("The root vector did not create the right number of children. found " + fieldVectors.size() + " expected " + fields.size());
+    }
+  }
+
+  /**
+   * Loads the record batch in the vectors
+   * will not close the record batch
+   * @param recordBatch
+   */
+  public void load(ArrowRecordBatch recordBatch) {
+    Iterator<ArrowBuf> buffers = recordBatch.getBuffers().iterator();
+    Iterator<ArrowFieldNode> nodes = recordBatch.getNodes().iterator();
+    for (int i = 0; i < fields.size(); ++i) {
+      Field field = fields.get(i);
+      FieldVector fieldVector = fieldVectors.get(i);
+      loadBuffers(fieldVector, field, buffers, nodes);
+    }
+    if (nodes.hasNext() || buffers.hasNext()) {
+      throw new IllegalArgumentException("not all nodes and buffers where consumed. nodes: " + Iterators.toString(nodes) + " buffers: " + Iterators.toString(buffers));
+    }
+  }
+
+  private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buffers, Iterator<ArrowFieldNode> nodes) {
+    ArrowFieldNode fieldNode = nodes.next();
+    List<VectorLayout> typeLayout = field.getTypeLayout().getVectors();
+    List<ArrowBuf> ownBuffers = new ArrayList<>(typeLayout.size());
+    for (int j = 0; j < typeLayout.size(); j++) {
+      ownBuffers.add(buffers.next());
+    }
+    try {
+      vector.loadFieldBuffers(fieldNode, ownBuffers);
+    } catch (RuntimeException e) {
+      throw new IllegalArgumentException("Could not load buffers for field " + field);
+    }
+    List<Field> children = field.getChildren();
+    if (children.size() > 0) {
+      List<FieldVector> childrenFromFields = vector.getChildrenFromFields();
+      checkArgument(children.size() == childrenFromFields.size(), "should have as many children as in the schema: found " + childrenFromFields.size() + " expected " + children.size());
+      for (int i = 0; i < childrenFromFields.size(); i++) {
+        Field child = children.get(i);
+        FieldVector fieldVector = childrenFromFields.get(i);
+        loadBuffers(fieldVector, child, buffers, nodes);
+      }
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
new file mode 100644
index 0000000000000..e4d37bf47d114
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.arrow.vector.ValueVector.Accessor;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import io.netty.buffer.ArrowBuf;
+
+public class VectorUnloader {
+
+  private final Schema schema;
+  private final int valueCount;
+  private final List<FieldVector> vectors;
+
+  public VectorUnloader(FieldVector parent) {
+    super();
+    this.schema = new Schema(parent.getField().getChildren());
+    this.valueCount = parent.getAccessor().getValueCount();
+    this.vectors = parent.getChildrenFromFields();
+  }
+
+  public Schema getSchema() {
+    return schema;
+  }
+
+  public ArrowRecordBatch getRecordBatch() {
+    List<ArrowFieldNode> nodes = new ArrayList<>();
+    List<ArrowBuf> buffers = new ArrayList<>();
+    for (FieldVector vector : vectors) {
+      appendNodes(vector, nodes, buffers);
+    }
+    return new ArrowRecordBatch(valueCount, nodes, buffers);
+  }
+
+  private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+    Accessor accessor = vector.getAccessor();
+    int nullCount = 0;
+    // TODO: should not have to do that
+    // we can do that a lot more efficiently (for example with Long.bitCount(i))
+    for (int i = 0; i < accessor.getValueCount(); i++) {
+      if (accessor.isNull(i)) {
+        nullCount ++;
+      }
+    }
+    nodes.add(new ArrowFieldNode(accessor.getValueCount(), nullCount));
+    List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
+    List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
+    if (fieldBuffers.size() != expectedBuffers.size()) {
+      throw new IllegalArgumentException("wrong number of buffers for field " + vector.getField() + ". found: " + fieldBuffers);
+    }
+    buffers.addAll(fieldBuffers);
+    for (FieldVector child : vector.getChildrenFromFields()) {
+      appendNodes(child, nodes, buffers);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 705a24b02fe78..c2482adefecfb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -17,25 +17,23 @@
  */
 package org.apache.arrow.vector;
 
-import com.google.flatbuffers.FlatBufferBuilder;
-import io.netty.buffer.ArrowBuf;
-
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.List;
 
-import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.impl.NullReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.collect.Iterators;
+import io.netty.buffer.ArrowBuf;
 
-public class ZeroVector implements ValueVector {
+public class ZeroVector implements FieldVector {
   public final static ZeroVector INSTANCE = new ZeroVector();
 
   private final String name = "[DEFAULT]";
@@ -175,4 +173,33 @@ public Mutator getMutator() {
   public FieldReader getReader() {
     return NullReader.INSTANCE;
   }
+
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    if (!children.isEmpty()) {
+      throw new IllegalArgumentException("Zero vector has no children");
+    }
+  }
+
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    if (!ownBuffers.isEmpty()) {
+      throw new IllegalArgumentException("Zero vector has no buffers");
+    }
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public List<BufferBacked> getFieldInnerVectors() {
+    return Collections.emptyList();
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index ed7797576d679..2f68886a169b3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -17,22 +17,13 @@
  */
 package org.apache.arrow.vector.complex;
 
-import java.util.Collection;
-
-import javax.annotation.Nullable;
-
-import org.apache.arrow.flatbuf.Field;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.CallBack;
 
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-
 /**
  * Base class for composite vectors.
  *
@@ -65,8 +56,8 @@ public BufferAllocator getAllocator() {
   /**
    * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
    */
-  public ValueVector getChild(String name) {
-    return getChild(name, ValueVector.class);
+  public FieldVector getChild(String name) {
+    return getChild(name, FieldVector.class);
   }
 
   /**
@@ -81,7 +72,7 @@ public void close() {
 
   protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
     if (clazz.isAssignableFrom(v.getClass())) {
-      return (T) v;
+      return clazz.cast(v);
     }
     throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Arrow doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
   }
@@ -94,10 +85,10 @@ protected boolean supportsDirectRead() {
   public abstract int size();
 
   // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
-  public abstract <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale);
+  public abstract <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale);
 
   // return the child vector with the input name
-  public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
+  public abstract <T extends FieldVector> T getChild(String name, Class<T> clazz);
 
   // return the child vector's ordinal in the composite container
   public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index 5964f80079141..23b4997f4f586 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -17,23 +17,24 @@
  */
 package org.apache.arrow.vector.complex;
 
-import com.google.common.collect.ImmutableList;
-import io.netty.buffer.ArrowBuf;
-
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.arrow.flatbuf.Field;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.MapWithOrdinal;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
+import io.netty.buffer.ArrowBuf;
+
 /*
  * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
  */
@@ -41,7 +42,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
 
   // Maintains a map with key as field name and value is the vector itself
-  private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
+  private final MapWithOrdinal<String, FieldVector> vectors =  new MapWithOrdinal<>();
 
   protected AbstractMapVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator, callBack);
@@ -109,19 +110,19 @@ public boolean allocateNewSafe() {
    * @return resultant {@link org.apache.arrow.vector.ValueVector}
    */
   @Override
-  public <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
+  public <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
     final ValueVector existing = getChild(name);
     boolean create = false;
     if (existing == null) {
       create = true;
     } else if (clazz.isAssignableFrom(existing.getClass())) {
-      return (T) existing;
+      return clazz.cast(existing);
     } else if (nullFilled(existing)) {
       existing.clear();
       create = true;
     }
     if (create) {
-      final T vector = (T) minorType.getNewVector(name, allocator, callBack, precisionScale);
+      final T vector = clazz.cast(minorType.getNewVector(name, allocator, callBack, precisionScale));
       putChild(name, vector);
       if (callBack!=null) {
         callBack.doWork();
@@ -153,7 +154,7 @@ public ValueVector getChildByOrdinal(int id) {
    * field name if exists or null.
    */
   @Override
-  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+  public <T extends FieldVector> T getChild(String name, Class<T> clazz) {
     final ValueVector v = vectors.get(name.toLowerCase());
     if (v == null) {
       return null;
@@ -161,12 +162,25 @@ public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
     return typeify(v, clazz);
   }
 
+  protected ValueVector add(String name, MinorType minorType, int... precisionScale) {
+    final ValueVector existing = getChild(name);
+    if (existing != null) {
+      throw new IllegalStateException(String.format("Vector already exists: Existing[%s], Requested[%s] ", existing.getClass().getSimpleName(), minorType));
+    }
+    FieldVector vector = minorType.getNewVector(name, allocator, callBack, precisionScale);
+    putChild(name, vector);
+    if (callBack!=null) {
+      callBack.doWork();
+    }
+    return vector;
+  }
+
   /**
    * Inserts the vector with the given name if it does not exist else replaces it with the new value.
    *
    * Note that this method does not enforce any vector type check nor throws a schema change exception.
    */
-  protected void putChild(String name, ValueVector vector) {
+  protected void putChild(String name, FieldVector vector) {
     putVector(name, vector);
   }
 
@@ -175,7 +189,7 @@ protected void putChild(String name, ValueVector vector) {
    * @param name  field name
    * @param vector  vector to be inserted
    */
-  protected void putVector(String name, ValueVector vector) {
+  protected void putVector(String name, FieldVector vector) {
     final ValueVector old = vectors.put(
         Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
         Preconditions.checkNotNull(vector, "vector cannot be null")
@@ -189,9 +203,9 @@ protected void putVector(String name, ValueVector vector) {
   /**
    * Returns a sequence of underlying child vectors.
    */
-  protected List<ValueVector> getChildren() {
+  protected List<FieldVector> getChildren() {
     int size = vectors.size();
-    List<ValueVector> children = new ArrayList<>();
+    List<FieldVector> children = new ArrayList<>();
     for (int i = 0; i < size; i++) {
       children.add(vectors.getByOrdinal(i));
     }
@@ -216,7 +230,7 @@ public int size() {
 
   @Override
   public Iterator<ValueVector> iterator() {
-    return vectors.values().iterator();
+    return Collections.<ValueVector>unmodifiableCollection(vectors.values()).iterator();
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 42262741df93d..517d20c77a93c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -17,8 +17,6 @@
  */
 package org.apache.arrow.vector.complex;
 
-import io.netty.buffer.ArrowBuf;
-
 import java.util.Collections;
 import java.util.Iterator;
 
@@ -26,29 +24,32 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ObjectArrays;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+
+import io.netty.buffer.ArrowBuf;
 
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
-  public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
+  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
   public final static String OFFSETS_VECTOR_NAME = "$offsets$";
   public final static String DATA_VECTOR_NAME = "$data$";
 
   protected final UInt4Vector offsets;
-  protected ValueVector vector;
+  protected FieldVector vector;
 
   protected BaseRepeatedValueVector(String name, BufferAllocator allocator) {
     this(name, allocator, DEFAULT_DATA_VECTOR);
   }
 
-  protected BaseRepeatedValueVector(String name, BufferAllocator allocator, ValueVector vector) {
+  protected BaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector) {
     super(name, allocator);
     this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
     this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
@@ -83,7 +84,7 @@ public UInt4Vector getOffsetVector() {
   }
 
   @Override
-  public ValueVector getDataVector() {
+  public FieldVector getDataVector() {
     return vector;
   }
 
@@ -121,7 +122,7 @@ public int getBufferSizeFor(int valueCount) {
 
   @Override
   public Iterator<ValueVector> iterator() {
-    return Collections.singleton(getDataVector()).iterator();
+    return Collections.<ValueVector>singleton(getDataVector()).iterator();
   }
 
   @Override
@@ -167,7 +168,7 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorT
     return new AddOrGetResult<>((T)vector, created);
   }
 
-  protected void replaceDataVector(ValueVector v) {
+  protected void replaceDataVector(FieldVector v) {
     vector.clear();
     vector = v;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index c6c6b090db6b0..2984c362514fc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -18,15 +18,18 @@
  ******************************************************************************/
 package org.apache.arrow.vector.complex;
 
-import com.google.common.collect.ImmutableList;
-import com.google.flatbuffers.FlatBufferBuilder;
-import io.netty.buffer.ArrowBuf;
+import static java.util.Collections.singletonList;
 
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BaseDataValueVector;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.UInt1Vector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
@@ -36,18 +39,24 @@
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.TransferPair;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ObjectArrays;
 
-public class ListVector extends BaseRepeatedValueVector {
+import io.netty.buffer.ArrowBuf;
+
+public class ListVector extends BaseRepeatedValueVector implements FieldVector {
 
-  UInt4Vector offsets;
+  final UInt4Vector offsets;
   final UInt1Vector bits;
+  private final List<BufferBacked> innerVectors;
   private Mutator mutator = new Mutator();
   private Accessor accessor = new Accessor();
   private UnionListWriter writer;
@@ -57,12 +66,46 @@ public class ListVector extends BaseRepeatedValueVector {
   public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator);
     this.bits = new UInt1Vector("$bits$", allocator);
-    offsets = getOffsetVector();
+    this.offsets = getOffsetVector();
+    this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
     this.writer = new UnionListWriter(this);
     this.reader = new UnionListReader(this);
     this.callBack = callBack;
   }
 
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    if (children.size() != 1) {
+      throw new IllegalArgumentException("Lists have only one child. Found: " + children);
+    }
+    Field field = children.get(0);
+    MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
+    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(minorType);
+    if (!addOrGetVector.isCreated()) {
+      throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
+    }
+  }
+
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return singletonList(getDataVector());
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    return BaseDataValueVector.unload(getFieldInnerVectors());
+  }
+
+  @Override
+  public List<BufferBacked> getFieldInnerVectors() {
+    return innerVectors;
+  }
+
   public UnionListWriter getWriter() {
     return writer;
   }
@@ -86,7 +129,7 @@ public void copyFrom(int inIndex, int outIndex, ListVector from) {
   }
 
   @Override
-  public ValueVector getDataVector() {
+  public FieldVector getDataVector() {
     return vector;
   }
 
@@ -298,4 +341,5 @@ public void setValueCount(int valueCount) {
       bits.getMutator().setValueCount(valueCount);
     }
   }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 0cb613e2f7acf..e3696588e6006 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -17,10 +17,10 @@
  */
 package org.apache.arrow.vector.complex;
 
-import io.netty.buffer.ArrowBuf;
-
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -28,13 +28,17 @@
 import javax.annotation.Nullable;
 
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.BaseDataValueVector;
 import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
@@ -45,7 +49,9 @@
 import com.google.common.collect.Ordering;
 import com.google.common.primitives.Ints;
 
-public class MapVector extends AbstractMapVector {
+import io.netty.buffer.ArrowBuf;
+
+public class MapVector extends AbstractMapVector implements FieldVector {
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
@@ -53,6 +59,9 @@ public class MapVector extends AbstractMapVector {
   private final Mutator mutator = new Mutator();
   int valueCount;
 
+  // TODO: validity vector
+  private final List<BufferBacked> innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList());
+
   public MapVector(String name, BufferAllocator allocator, CallBack callBack){
     super(name, allocator, callBack);
   }
@@ -120,7 +129,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
     int expectedSize = getBufferSize();
     int actualSize   = super.getBufferSize();
 
-    Preconditions.checkArgument(expectedSize == actualSize);
+    Preconditions.checkArgument(expectedSize == actualSize, expectedSize + " != " + actualSize);
     return super.getBuffers(clear);
   }
 
@@ -159,7 +168,7 @@ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
       this.to.ephPair = null;
 
       int i = 0;
-      ValueVector vector;
+      FieldVector vector;
       for (String child:from.getChildFieldNames()) {
         int preSize = to.size();
         vector = from.getChild(child);
@@ -175,7 +184,7 @@ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
         // (This is similar to what happens in ScanBatch where the children cannot be added till they are
         // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
         // include the hashCode of the children but is based only on MaterializedField$key.
-        final ValueVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass());
+        final FieldVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass());
         if (allocate && to.size() != preSize) {
           newVector.allocateNew();
         }
@@ -315,13 +324,45 @@ public MinorType getMinorType() {
 
   @Override
   public void close() {
-    final Collection<ValueVector> vectors = getChildren();
-    for (final ValueVector v : vectors) {
+    final Collection<FieldVector> vectors = getChildren();
+    for (final FieldVector v : vectors) {
       v.close();
     }
     vectors.clear();
+
     valueCount = 0;
 
     super.close();
  }
+
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    for (Field field : children) {
+      MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
+      FieldVector vector = (FieldVector)this.add(field.getName(), minorType);
+      vector.initializeChildrenFromFields(field.getChildren());
+    }
+  }
+
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return getChildren();
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+    // TODO: something with fieldNode?
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    return BaseDataValueVector.unload(getFieldInnerVectors());
+  }
+
+  @Override
+  public List<BufferBacked> getFieldInnerVectors() {
+    return innerVectors;
+  }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 4d2adfb32561e..89bfefc8f19e3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -22,9 +22,9 @@
 import org.apache.arrow.vector.complex.StateTool;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 
 import com.google.common.base.Preconditions;
-import org.apache.arrow.vector.types.pojo.Field;
 
 public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 586b1283fe879..c282688530b87 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -17,6 +17,7 @@
  */
 package org.apache.arrow.vector.complex.impl;
 
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.AbstractMapVector;
@@ -129,7 +130,7 @@ private FieldWriter promoteToUnion() {
     } else if (listVector != null) {
       unionVector = listVector.promoteToUnion();
     }
-    unionVector.addVector(tp.getTo());
+    unionVector.addVector((FieldVector)tp.getTo());
     writer = new UnionWriter(unionVector);
     writer.setPosition(idx());
     for (int i = 0; i < idx(); i++) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java
new file mode 100644
index 0000000000000..90fb02b059707
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import org.apache.arrow.flatbuf.Block;
+import org.apache.arrow.vector.schema.FBSerializable;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+public class ArrowBlock implements FBSerializable {
+
+  private final long offset;
+  private final int metadataLength;
+  private final long bodyLength;
+
+  public ArrowBlock(long offset, int metadataLength, long bodyLength) {
+    super();
+    this.offset = offset;
+    this.metadataLength = metadataLength;
+    this.bodyLength = bodyLength;
+  }
+
+  public long getOffset() {
+    return offset;
+  }
+
+  public int getMetadataLength() {
+    return metadataLength;
+  }
+
+  public long getBodyLength() {
+    return bodyLength;
+  }
+
+  @Override
+  public int writeTo(FlatBufferBuilder builder) {
+    return Block.createBlock(builder, offset, metadataLength, bodyLength);
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + (int) (bodyLength ^ (bodyLength >>> 32));
+    result = prime * result + metadataLength;
+    result = prime * result + (int) (offset ^ (offset >>> 32));
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ArrowBlock other = (ArrowBlock) obj;
+    if (bodyLength != other.bodyLength)
+      return false;
+    if (metadataLength != other.metadataLength)
+      return false;
+    if (offset != other.offset)
+      return false;
+    return true;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
new file mode 100644
index 0000000000000..01e175b31b8db
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.arrow.flatbuf.Block;
+import org.apache.arrow.flatbuf.Footer;
+import org.apache.arrow.vector.schema.FBSerializable;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+public class ArrowFooter implements FBSerializable {
+
+  private final Schema schema;
+
+  private final List<ArrowBlock> dictionaries;
+
+  private final List<ArrowBlock> recordBatches;
+
+  public ArrowFooter(Schema schema, List<ArrowBlock> dictionaries, List<ArrowBlock> recordBatches) {
+    super();
+    this.schema = schema;
+    this.dictionaries = dictionaries;
+    this.recordBatches = recordBatches;
+  }
+
+  public ArrowFooter(Footer footer) {
+    this(
+        Schema.convertSchema(footer.schema()),
+        dictionaries(footer),
+        recordBatches(footer)
+        );
+  }
+
+  private static List<ArrowBlock> recordBatches(Footer footer) {
+    List<ArrowBlock> recordBatches = new ArrayList<>();
+    Block tempBLock = new Block();
+    int recordBatchesLength = footer.recordBatchesLength();
+    for (int i = 0; i < recordBatchesLength; i++) {
+      Block block = footer.recordBatches(tempBLock, i);
+      recordBatches.add(new ArrowBlock(block.offset(), block.metaDataLength(), block.bodyLength()));
+    }
+    return recordBatches;
+  }
+
+  private static List<ArrowBlock> dictionaries(Footer footer) {
+    List<ArrowBlock> dictionaries = new ArrayList<>();
+    Block tempBLock = new Block();
+    int dictionariesLength = footer.dictionariesLength();
+    for (int i = 0; i < dictionariesLength; i++) {
+      Block block = footer.dictionaries(tempBLock, i);
+      dictionaries.add(new ArrowBlock(block.offset(), block.metaDataLength(), block.bodyLength()));
+    }
+    return dictionaries;
+  }
+
+  public Schema getSchema() {
+    return schema;
+  }
+
+  public List<ArrowBlock> getDictionaries() {
+    return dictionaries;
+  }
+
+  public List<ArrowBlock> getRecordBatches() {
+    return recordBatches;
+  }
+
+  @Override
+  public int writeTo(FlatBufferBuilder builder) {
+    int schemaIndex = schema.getSchema(builder);
+    Footer.startDictionariesVector(builder, dictionaries.size());
+    int dicsOffset = endVector(builder, dictionaries);
+    Footer.startRecordBatchesVector(builder, recordBatches.size());
+    int rbsOffset = endVector(builder, recordBatches);
+    Footer.startFooter(builder);
+    Footer.addSchema(builder, schemaIndex);
+    Footer.addDictionaries(builder, dicsOffset);
+    Footer.addRecordBatches(builder, rbsOffset);
+    return Footer.endFooter(builder);
+  }
+
+  private int endVector(FlatBufferBuilder builder, List<ArrowBlock> blocks) {
+    for (ArrowBlock block : blocks) {
+      block.writeTo(builder);
+    }
+    return builder.endVector();
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((dictionaries == null) ? 0 : dictionaries.hashCode());
+    result = prime * result + ((recordBatches == null) ? 0 : recordBatches.hashCode());
+    result = prime * result + ((schema == null) ? 0 : schema.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ArrowFooter other = (ArrowFooter) obj;
+    if (dictionaries == null) {
+      if (other.dictionaries != null)
+        return false;
+    } else if (!dictionaries.equals(other.dictionaries))
+      return false;
+    if (recordBatches == null) {
+      if (other.recordBatches != null)
+        return false;
+    } else if (!recordBatches.equals(other.recordBatches))
+      return false;
+    if (schema == null) {
+      if (other.schema != null)
+        return false;
+    } else if (!schema.equals(other.schema))
+      return false;
+    return true;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
new file mode 100644
index 0000000000000..bbcd3e9f470e3
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.arrow.flatbuf.Buffer;
+import org.apache.arrow.flatbuf.FieldNode;
+import org.apache.arrow.flatbuf.Footer;
+import org.apache.arrow.flatbuf.RecordBatch;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ArrowBuf;
+
+public class ArrowReader implements AutoCloseable {
+  private static final Logger LOGGER = LoggerFactory.getLogger(ArrowReader.class);
+
+  private static final byte[] MAGIC = "ARROW1".getBytes();
+
+  private final SeekableByteChannel in;
+
+  private final BufferAllocator allocator;
+
+  private ArrowFooter footer;
+
+  public ArrowReader(SeekableByteChannel in, BufferAllocator allocator) {
+    super();
+    this.in = in;
+    this.allocator = allocator;
+  }
+
+  private int readFully(ArrowBuf buffer, int l) throws IOException {
+    int n = readFully(buffer.nioBuffer(buffer.writerIndex(), l));
+    buffer.writerIndex(n);
+    if (n != l) {
+      throw new IllegalStateException(n + " != " + l);
+    }
+    return n;
+  }
+
+  private int readFully(ByteBuffer buffer) throws IOException {
+    int total = 0;
+    int n;
+    do {
+      n = in.read(buffer);
+      total += n;
+    } while (n >= 0 && buffer.remaining() > 0);
+    buffer.flip();
+    return total;
+  }
+
+  private static int bytesToInt(byte[] bytes) {
+    return ((int)(bytes[3] & 255) << 24) +
+           ((int)(bytes[2] & 255) << 16) +
+           ((int)(bytes[1] & 255) <<  8) +
+           ((int)(bytes[0] & 255) <<  0);
+  }
+
+  public ArrowFooter readFooter() throws IOException {
+    if (footer == null) {
+      if (in.size() <= (MAGIC.length * 2 + 4)) {
+        throw new InvalidArrowFileException("file too small: " + in.size());
+      }
+      ByteBuffer buffer = ByteBuffer.allocate(4 + MAGIC.length);
+      long footerLengthOffset = in.size() - buffer.remaining();
+      in.position(footerLengthOffset);
+      readFully(buffer);
+      byte[] array = buffer.array();
+      if (!Arrays.equals(MAGIC, Arrays.copyOfRange(array, 4, array.length))) {
+        throw new InvalidArrowFileException("missing Magic number " + Arrays.toString(buffer.array()));
+      }
+      int footerLength = bytesToInt(array);
+      if (footerLength <= 0 || footerLength + MAGIC.length * 2 + 4 > in.size()) {
+        throw new InvalidArrowFileException("invalid footer length: " + footerLength);
+      }
+      long footerOffset = footerLengthOffset - footerLength;
+      LOGGER.debug(String.format("Footer starts at %d, length: %d", footerOffset, footerLength));
+      ByteBuffer footerBuffer = ByteBuffer.allocate(footerLength);
+      in.position(footerOffset);
+      readFully(footerBuffer);
+      Footer footerFB = Footer.getRootAsFooter(footerBuffer);
+      this.footer = new ArrowFooter(footerFB);
+    }
+    return footer;
+  }
+
+  // TODO: read dictionaries
+
+  public ArrowRecordBatch readRecordBatch(ArrowBlock recordBatchBlock) throws IOException {
+    LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d", recordBatchBlock.getOffset(), recordBatchBlock.getMetadataLength(), recordBatchBlock.getBodyLength()));
+    int l = (int)(recordBatchBlock.getMetadataLength() + recordBatchBlock.getBodyLength());
+    if (l < 0) {
+      throw new InvalidArrowFileException("block invalid: " + recordBatchBlock);
+    }
+    final ArrowBuf buffer = allocator.buffer(l);
+    LOGGER.debug("allocated buffer " + buffer);
+    in.position(recordBatchBlock.getOffset());
+    int n = readFully(buffer, l);
+    if (n != l) {
+      throw new IllegalStateException(n + " != " + l);
+    }
+    RecordBatch recordBatchFB = RecordBatch.getRootAsRecordBatch(buffer.nioBuffer().asReadOnlyBuffer());
+    int nodesLength = recordBatchFB.nodesLength();
+    final ArrowBuf body = buffer.slice(recordBatchBlock.getMetadataLength(), (int)recordBatchBlock.getBodyLength());
+    List<ArrowFieldNode> nodes = new ArrayList<>();
+    for (int i = 0; i < nodesLength; ++i) {
+      FieldNode node = recordBatchFB.nodes(i);
+      nodes.add(new ArrowFieldNode(node.length(), node.nullCount()));
+    }
+    List<ArrowBuf> buffers = new ArrayList<>();
+    for (int i = 0; i < recordBatchFB.buffersLength(); ++i) {
+      Buffer bufferFB = recordBatchFB.buffers(i);
+      LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", bufferFB.offset(), bufferFB.length()));
+      ArrowBuf vectorBuffer = body.slice((int)bufferFB.offset(), (int)bufferFB.length());
+      buffers.add(vectorBuffer);
+    }
+    ArrowRecordBatch arrowRecordBatch = new ArrowRecordBatch(recordBatchFB.length(), nodes, buffers);
+    LOGGER.debug("released buffer " + buffer);
+    buffer.release();
+    return arrowRecordBatch;
+  }
+
+  public void close() throws IOException {
+    in.close();
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
new file mode 100644
index 0000000000000..9881a229c23ea
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.arrow.vector.schema.ArrowBuffer;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.schema.FBSerializable;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import io.netty.buffer.ArrowBuf;
+
+public class ArrowWriter implements AutoCloseable {
+  private static final Logger LOGGER = LoggerFactory.getLogger(ArrowWriter.class);
+
+  private static final byte[] MAGIC = "ARROW1".getBytes();
+
+  private final WritableByteChannel out;
+
+  private final Schema schema;
+
+  private final List<ArrowBlock> recordBatches = new ArrayList<>();
+
+  private long currentPosition = 0;
+
+  private boolean started = false;
+
+  public ArrowWriter(WritableByteChannel out, Schema schema) {
+    this.out = out;
+    this.schema = schema;
+  }
+
+  private void start() throws IOException {
+    writeMagic();
+  }
+
+  private long write(byte[] buffer) throws IOException {
+    return write(ByteBuffer.wrap(buffer));
+  }
+
+  private long writeZeros(int zeroCount) throws IOException {
+    return write(new byte[zeroCount]);
+  }
+
+  private long align() throws IOException {
+    if (currentPosition % 8 != 0) { // align on 8 byte boundaries
+      return writeZeros(8 - (int)(currentPosition % 8));
+    }
+    return 0;
+  }
+
+  private long write(ByteBuffer buffer) throws IOException {
+    long length = buffer.remaining();
+    out.write(buffer);
+    currentPosition += length;
+    return length;
+  }
+
+  private static byte[] intToBytes(int value) {
+    byte[] outBuffer = new byte[4];
+    outBuffer[3] = (byte)(value >>> 24);
+    outBuffer[2] = (byte)(value >>> 16);
+    outBuffer[1] = (byte)(value >>>  8);
+    outBuffer[0] = (byte)(value >>>  0);
+    return outBuffer;
+  }
+
+  private long writeIntLittleEndian(int v) throws IOException {
+    return write(intToBytes(v));
+  }
+
+  // TODO: write dictionaries
+
+  public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
+    checkStarted();
+    align();
+    // write metadata header
+    long offset = currentPosition;
+    write(recordBatch);
+    align();
+    // write body
+    long bodyOffset = currentPosition;
+    List<ArrowBuf> buffers = recordBatch.getBuffers();
+    List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
+    if (buffers.size() != buffersLayout.size()) {
+      throw new IllegalStateException("the layout does not match: " + buffers.size() + " != " + buffersLayout.size());
+    }
+    for (int i = 0; i < buffers.size(); i++) {
+      ArrowBuf buffer = buffers.get(i);
+      ArrowBuffer layout = buffersLayout.get(i);
+      long startPosition = bodyOffset + layout.getOffset();
+      if (startPosition != currentPosition) {
+        writeZeros((int)(startPosition - currentPosition));
+      }
+      write(buffer);
+      if (currentPosition != startPosition + layout.getSize()) {
+        throw new IllegalStateException("wrong buffer size: " + currentPosition + " != " + startPosition + layout.getSize());
+      }
+    }
+    int metadataLength = (int)(bodyOffset - offset);
+    if (metadataLength <= 0) {
+      throw new InvalidArrowFileException("invalid recordBatch");
+    }
+    long bodyLength = currentPosition - bodyOffset;
+    LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d", offset, metadataLength, bodyLength));
+    // add metadata to footer
+    recordBatches.add(new ArrowBlock(offset, metadataLength, bodyLength));
+  }
+
+  private void write(ArrowBuf buffer) throws IOException {
+    write(buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes()));
+  }
+
+  private void checkStarted() throws IOException {
+    if (!started) {
+      started = true;
+      start();
+    }
+  }
+
+  public void close() throws IOException {
+    try {
+      long footerStart = currentPosition;
+      writeFooter();
+      int footerLength = (int)(currentPosition - footerStart);
+      if (footerLength <= 0 ) {
+        throw new InvalidArrowFileException("invalid footer");
+      }
+      writeIntLittleEndian(footerLength);
+      LOGGER.debug(String.format("Footer starts at %d, length: %d", footerStart, footerLength));
+      writeMagic();
+    } finally {
+      out.close();
+    }
+  }
+
+  private void writeMagic() throws IOException {
+    write(MAGIC);
+    LOGGER.debug(String.format("magic written, now at %d", currentPosition));
+  }
+
+  private void writeFooter() throws IOException {
+    // TODO: dictionaries
+    write(new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), recordBatches));
+  }
+
+  private long write(FBSerializable writer) throws IOException {
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    int root = writer.writeTo(builder);
+    builder.finish(root);
+    return write(builder.dataBuffer());
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java b/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java
new file mode 100644
index 0000000000000..3ec75dcb12a2b
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+public class InvalidArrowFileException extends RuntimeException {
+  private static final long serialVersionUID = 1L;
+
+  public InvalidArrowFileException(String message) {
+    super(message);
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
new file mode 100644
index 0000000000000..3aa3e52582b4f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import org.apache.arrow.flatbuf.Buffer;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+public class ArrowBuffer implements FBSerializable {
+
+  private int page;
+  private long offset;
+  private long size;
+
+  public ArrowBuffer(int page, long offset, long size) {
+    super();
+    this.page = page;
+    this.offset = offset;
+    this.size = size;
+  }
+
+  public int getPage() {
+    return page;
+  }
+
+  public long getOffset() {
+    return offset;
+  }
+
+  public long getSize() {
+    return size;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + (int) (offset ^ (offset >>> 32));
+    result = prime * result + page;
+    result = prime * result + (int) (size ^ (size >>> 32));
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ArrowBuffer other = (ArrowBuffer) obj;
+    if (offset != other.offset)
+      return false;
+    if (page != other.page)
+      return false;
+    if (size != other.size)
+      return false;
+    return true;
+  }
+
+  @Override
+  public int writeTo(FlatBufferBuilder builder) {
+    return Buffer.createBuffer(builder, page, offset, size);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
new file mode 100644
index 0000000000000..71dd0abc6bcef
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import org.apache.arrow.flatbuf.FieldNode;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+public class ArrowFieldNode implements FBSerializable {
+
+  private final int length;
+  private final int nullCount;
+
+  public ArrowFieldNode(int length, int nullCount) {
+    super();
+    this.length = length;
+    this.nullCount = nullCount;
+  }
+
+  @Override
+  public int writeTo(FlatBufferBuilder builder) {
+    return FieldNode.createFieldNode(builder, length, nullCount);
+  }
+
+  public int getNullCount() {
+    return nullCount;
+  }
+
+  public int getLength() {
+    return length;
+  }
+
+  @Override
+  public String toString() {
+    return "ArrowFieldNode [length=" + length + ", nullCount=" + nullCount + "]";
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
new file mode 100644
index 0000000000000..9162efd29f864
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.arrow.flatbuf.RecordBatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import io.netty.buffer.ArrowBuf;
+
+public class ArrowRecordBatch implements FBSerializable, AutoCloseable {
+  private static final Logger LOGGER = LoggerFactory.getLogger(ArrowRecordBatch.class);
+
+  /** number of records */
+  private final int length;
+
+  /** Nodes correspond to the pre-ordered flattened logical schema */
+  private final List<ArrowFieldNode> nodes;
+
+  private final List<ArrowBuf> buffers;
+
+  private final List<ArrowBuffer> buffersLayout;
+
+  private boolean closed = false;
+
+  /**
+   * @param length how many rows in this batch
+   * @param nodes field level info
+   * @param buffers will be retained until this recordBatch is closed
+   */
+  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+    super();
+    this.length = length;
+    this.nodes = nodes;
+    this.buffers = buffers;
+    List<ArrowBuffer> arrowBuffers = new ArrayList<>();
+    long offset = 0;
+    for (ArrowBuf arrowBuf : buffers) {
+      arrowBuf.retain();
+      long size = arrowBuf.readableBytes();
+      arrowBuffers.add(new ArrowBuffer(0, offset, size));
+      LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", offset, size));
+      offset += size;
+      if (offset % 8 != 0) { // align on 8 byte boundaries
+        offset += 8 - (offset % 8);
+      }
+    }
+    this.buffersLayout = Collections.unmodifiableList(arrowBuffers);
+  }
+
+  public int getLength() {
+    return length;
+  }
+
+  /**
+   * @return the FieldNodes corresponding to the schema
+   */
+  public List<ArrowFieldNode> getNodes() {
+    return nodes;
+  }
+
+  /**
+   * @return the buffers containing the data
+   */
+  public List<ArrowBuf> getBuffers() {
+    if (closed) {
+      throw new IllegalStateException("already closed");
+    }
+    return buffers;
+  }
+
+  /**
+   * @return the serialized layout if we send the buffers on the wire
+   */
+  public List<ArrowBuffer> getBuffersLayout() {
+    return buffersLayout;
+  }
+
+  @Override
+  public int writeTo(FlatBufferBuilder builder) {
+    RecordBatch.startNodesVector(builder, nodes.size());
+    int nodesOffset = writeAllStructsToVector(builder, nodes);
+    RecordBatch.startBuffersVector(builder, buffers.size());
+    int buffersOffset = writeAllStructsToVector(builder, buffersLayout);
+    RecordBatch.startRecordBatch(builder);
+    RecordBatch.addLength(builder, length);
+    RecordBatch.addNodes(builder, nodesOffset);
+    RecordBatch.addBuffers(builder, buffersOffset);
+    return RecordBatch.endRecordBatch(builder);
+  }
+
+  /**
+   * releases the buffers
+   */
+  public void close() {
+    if (!closed) {
+      closed = true;
+      for (ArrowBuf arrowBuf : buffers) {
+        arrowBuf.release();
+      }
+    }
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
new file mode 100644
index 0000000000000..e3d3e34e0ae24
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import org.apache.arrow.flatbuf.VectorType;
+
+public class ArrowVectorType {
+
+  public static final ArrowVectorType VALUES = new ArrowVectorType(VectorType.VALUES);
+  public static final ArrowVectorType OFFSET = new ArrowVectorType(VectorType.OFFSET);
+  public static final ArrowVectorType VALIDITY = new ArrowVectorType(VectorType.VALIDITY);
+  public static final ArrowVectorType TYPE = new ArrowVectorType(VectorType.TYPE);
+
+  private final short type;
+
+  public ArrowVectorType(short type) {
+    this.type = type;
+  }
+
+  public short getType() {
+    return type;
+  }
+
+  @Override
+  public String toString() {
+    try {
+      return VectorType.name(type);
+    } catch (ArrayIndexOutOfBoundsException e) {
+      return "Unlnown type " + type;
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java
new file mode 100644
index 0000000000000..d23ed91948e5d
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+public interface FBSerializable {
+  int writeTo(FlatBufferBuilder builder);
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java
new file mode 100644
index 0000000000000..31c17ad6df02b
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+public class FBSerializables {
+
+  public static int writeAllStructsToVector(FlatBufferBuilder builder, List<? extends FBSerializable> all) {
+    // struct vectors have to be created in reverse order
+    List<? extends FBSerializable> reversed = new ArrayList<>(all);
+    Collections.reverse(reversed);
+    for (FBSerializable element : reversed) {
+      element.writeTo(builder);
+    }
+    return builder.endVector();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
new file mode 100644
index 0000000000000..1275e0eb5dc45
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import static java.util.Arrays.asList;
+import static org.apache.arrow.flatbuf.Precision.DOUBLE;
+import static org.apache.arrow.flatbuf.Precision.SINGLE;
+import static org.apache.arrow.vector.schema.VectorLayout.booleanVector;
+import static org.apache.arrow.vector.schema.VectorLayout.byteVector;
+import static org.apache.arrow.vector.schema.VectorLayout.dataVector;
+import static org.apache.arrow.vector.schema.VectorLayout.offsetVector;
+import static org.apache.arrow.vector.schema.VectorLayout.typeVector;
+import static org.apache.arrow.vector.schema.VectorLayout.validityVector;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.arrow.flatbuf.UnionMode;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor;
+import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
+import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
+import org.apache.arrow.vector.types.pojo.ArrowType.Date;
+import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.IntervalDay;
+import org.apache.arrow.vector.types.pojo.ArrowType.IntervalYear;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.ArrowType.Time;
+import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+
+/**
+ * The layout of vectors for a given type
+ * It defines its own vectors followed by the vectors for the children
+ * if it is a nested type (Tuple, List, Union)
+ */
+public class TypeLayout {
+
+  public static TypeLayout getTypeLayout(final ArrowType arrowType) {
+    TypeLayout layout = arrowType.accept(new ArrowTypeVisitor<TypeLayout>() {
+
+      @Override public TypeLayout visit(Int type) {
+        return newFixedWidthTypeLayout(dataVector(type.getBitWidth()));
+      }
+
+      @Override public TypeLayout visit(Union type) {
+        List<VectorLayout> vectors;
+        switch (type.getMode()) {
+          case UnionMode.Dense:
+            vectors = asList(
+                // TODO: validate this
+                validityVector(),
+                typeVector(),
+                offsetVector() // offset to find the vector
+                );
+            break;
+          case UnionMode.Sparse:
+            vectors = asList(
+                validityVector(),
+                typeVector()
+                );
+            break;
+          default:
+            throw new UnsupportedOperationException("Unsupported Union Mode: " + type.getMode());
+        }
+        return new TypeLayout(vectors);
+      }
+
+      @Override public TypeLayout visit(Tuple type) {
+        List<VectorLayout> vectors = asList(
+            // TODO: add validity vector in Map
+//            validityVector()
+            );
+        return new TypeLayout(vectors);
+      }
+
+      @Override public TypeLayout visit(Timestamp type) {
+        return newFixedWidthTypeLayout(dataVector(64));
+      }
+
+      @Override public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) {
+        List<VectorLayout> vectors = asList(
+            validityVector(),
+            offsetVector()
+            );
+        return new TypeLayout(vectors);
+      }
+
+      @Override public TypeLayout visit(FloatingPoint type) {
+        int bitWidth;
+        switch (type.getPrecision()) {
+        case SINGLE:
+          bitWidth = 32;
+          break;
+        case DOUBLE:
+          bitWidth = 64;
+          break;
+        default:
+          throw new UnsupportedOperationException("Unsupported Precision: " + type.getPrecision());
+        }
+        return newFixedWidthTypeLayout(dataVector(bitWidth));
+      }
+
+      @Override public TypeLayout visit(Decimal type) {
+        // TODO: check size
+        return newFixedWidthTypeLayout(dataVector(64)); // actually depends on the type fields
+      }
+
+      @Override public TypeLayout visit(Bool type) {
+        return newFixedWidthTypeLayout(booleanVector());
+      }
+
+      @Override public TypeLayout visit(Binary type) {
+        return newVariableWidthTypeLayout();
+      }
+
+      @Override public TypeLayout visit(Utf8 type) {
+        return newVariableWidthTypeLayout();
+      }
+
+      private TypeLayout newVariableWidthTypeLayout() {
+        return newPrimitiveTypeLayout(validityVector(), offsetVector(), byteVector());
+      }
+
+      private TypeLayout newPrimitiveTypeLayout(VectorLayout... vectors) {
+        return new TypeLayout(asList(vectors));
+      }
+
+      public TypeLayout newFixedWidthTypeLayout(VectorLayout dataVector) {
+        return newPrimitiveTypeLayout(validityVector(), dataVector);
+      }
+
+      @Override
+      public TypeLayout visit(Null type) {
+        return new TypeLayout(Collections.<VectorLayout>emptyList());
+      }
+
+      @Override
+      public TypeLayout visit(Date type) {
+        return newFixedWidthTypeLayout(dataVector(64));
+      }
+
+      @Override
+      public TypeLayout visit(Time type) {
+        return newFixedWidthTypeLayout(dataVector(64));
+      }
+
+      @Override
+      public TypeLayout visit(IntervalDay type) { // TODO: check size
+        return newFixedWidthTypeLayout(dataVector(64));
+      }
+
+      @Override
+      public TypeLayout visit(IntervalYear type) { // TODO: check size
+        return newFixedWidthTypeLayout(dataVector(64));
+      }
+    });
+    return layout;
+  }
+
+  private final List<VectorLayout> vectors;
+
+  public TypeLayout(List<VectorLayout> vectors) {
+    super();
+    this.vectors = vectors;
+  }
+
+  public TypeLayout(VectorLayout... vectors) {
+    this(asList(vectors));
+  }
+
+
+  public List<VectorLayout> getVectors() {
+    return vectors;
+  }
+
+  public List<ArrowVectorType> getVectorTypes() {
+    List<ArrowVectorType> types = new ArrayList<>(vectors.size());
+    for (VectorLayout vector : vectors) {
+      types.add(vector.getType());
+    }
+    return types;
+  }
+
+  public String toString() {
+    return "TypeLayout{" + vectors + "}";
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
new file mode 100644
index 0000000000000..421ebcb837677
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET;
+import static org.apache.arrow.vector.schema.ArrowVectorType.TYPE;
+import static org.apache.arrow.vector.schema.ArrowVectorType.VALIDITY;
+import static org.apache.arrow.vector.schema.ArrowVectorType.VALUES;
+
+public class VectorLayout {
+
+  private static final VectorLayout VALIDITY_VECTOR = new VectorLayout(VALIDITY, 1);
+  private static final VectorLayout OFFSET_VECTOR = new VectorLayout(OFFSET, 32);
+  private static final VectorLayout TYPE_VECTOR = new VectorLayout(TYPE, 32);
+  private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(VALUES, 1);
+  private static final VectorLayout VALUES_64 = new VectorLayout(VALUES, 64);
+  private static final VectorLayout VALUES_32 = new VectorLayout(VALUES, 32);
+  private static final VectorLayout VALUES_16 = new VectorLayout(VALUES, 16);
+  private static final VectorLayout VALUES_8 = new VectorLayout(VALUES, 8);
+
+  public static VectorLayout typeVector() {
+    return TYPE_VECTOR;
+  }
+
+  public static VectorLayout offsetVector() {
+    return OFFSET_VECTOR;
+  }
+
+  public static VectorLayout dataVector(int typeBitWidth) {
+    switch (typeBitWidth) {
+    case 8:
+      return VALUES_8;
+    case 16:
+      return VALUES_16;
+    case 32:
+      return VALUES_32;
+    case 64:
+      return VALUES_64;
+    default:
+      throw new IllegalArgumentException("only 8, 16, 32, or 64 bits supported");
+    }
+  }
+
+  public static VectorLayout booleanVector() {
+    return BOOLEAN_VECTOR;
+  }
+
+  public static VectorLayout validityVector() {
+    return VALIDITY_VECTOR;
+  }
+
+  public static VectorLayout byteVector() {
+    return dataVector(8);
+  }
+
+  private final int typeBitWidth;
+
+  private final ArrowVectorType type;
+
+  private VectorLayout(ArrowVectorType type, int typeBitWidth) {
+    super();
+    this.type = type;
+    this.typeBitWidth = typeBitWidth;
+  }
+
+  public int getTypeBitWidth() {
+    return typeBitWidth;
+  }
+
+  public ArrowVectorType getType() {
+    return type;
+  }
+
+  @Override
+  public String toString() {
+    return String.format("{width=%s,type=%s}", typeBitWidth, type);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index c34882a8fb12a..4d0d9ee114ad8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -17,8 +17,14 @@
  */
 package org.apache.arrow.vector.types;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.arrow.flatbuf.Precision;
 import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.NullableBigIntVector;
 import org.apache.arrow.vector.NullableBitVector;
 import org.apache.arrow.vector.NullableDateVector;
@@ -38,7 +44,6 @@
 import org.apache.arrow.vector.NullableUInt8Vector;
 import org.apache.arrow.vector.NullableVarBinaryVector;
 import org.apache.arrow.vector.NullableVarCharVector;
-import org.apache.arrow.vector.SmallIntVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.ListVector;
@@ -85,9 +90,6 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 
-import java.util.HashMap;
-import java.util.Map;
-
 public class Types {
 
   public static final Field NULL_FIELD = new Field("", true, Null.INSTANCE, null);
@@ -104,8 +106,8 @@ public class Types {
   public static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null);
   public static final Field INTERVALDAY_FIELD = new Field("", true, IntervalDay.INSTANCE, null);
   public static final Field INTERVALYEAR_FIELD = new Field("", true, IntervalYear.INSTANCE, null);
-  public static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(0), null);
-  public static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(1), null);
+  public static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(Precision.SINGLE), null);
+  public static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(Precision.DOUBLE), null);
   public static final Field LIST_FIELD = new Field("", true, List.INSTANCE, null);
   public static final Field VARCHAR_FIELD = new Field("", true, Utf8.INSTANCE, null);
   public static final Field VARBINARY_FIELD = new Field("", true, Binary.INSTANCE, null);
@@ -120,7 +122,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return ZeroVector.INSTANCE;
       }
 
@@ -136,7 +138,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
          return new MapVector(name, allocator, callBack);
       }
 
@@ -153,7 +155,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableTinyIntVector(name, allocator);
       }
 
@@ -169,8 +171,8 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new SmallIntVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableSmallIntVector(name, allocator);
       }
 
       @Override
@@ -185,7 +187,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableIntVector(name, allocator);
       }
 
@@ -201,7 +203,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableBigIntVector(name, allocator);
       }
 
@@ -217,7 +219,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableDateVector(name, allocator);
       }
 
@@ -233,7 +235,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableTimeVector(name, allocator);
       }
 
@@ -249,7 +251,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableTimeStampVector(name, allocator);
       }
 
@@ -265,7 +267,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableIntervalDayVector(name, allocator);
       }
 
@@ -281,7 +283,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableIntervalDayVector(name, allocator);
       }
 
@@ -290,14 +292,14 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new IntervalYearWriterImpl((NullableIntervalYearVector) vector);
       }
     },
-    FLOAT4(new FloatingPoint(0)) {
+    FLOAT4(new FloatingPoint(Precision.SINGLE)) {
       @Override
       public Field getField() {
         return FLOAT4_FIELD;
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableFloat4Vector(name, allocator);
       }
 
@@ -306,14 +308,14 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new Float4WriterImpl((NullableFloat4Vector) vector);
       }
     },   //  4 byte ieee 754
-    FLOAT8(new FloatingPoint(1)) {
+    FLOAT8(new FloatingPoint(Precision.DOUBLE)) {
       @Override
       public Field getField() {
         return FLOAT8_FIELD;
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableFloat8Vector(name, allocator);
       }
 
@@ -329,7 +331,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableBitVector(name, allocator);
       }
 
@@ -345,7 +347,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableVarCharVector(name, allocator);
       }
 
@@ -361,7 +363,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableVarBinaryVector(name, allocator);
       }
 
@@ -381,7 +383,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableDecimalVector(name, allocator, precisionScale[0], precisionScale[1]);
       }
 
@@ -397,7 +399,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableUInt1Vector(name, allocator);
       }
 
@@ -413,7 +415,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableUInt2Vector(name, allocator);
       }
 
@@ -429,7 +431,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableUInt4Vector(name, allocator);
       }
 
@@ -445,7 +447,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new NullableUInt8Vector(name, allocator);
       }
 
@@ -461,7 +463,7 @@ public Field getField() {
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new ListVector(name, allocator, callBack);
       }
 
@@ -470,14 +472,14 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UnionListWriter((ListVector) vector);
       }
     },
-    UNION(Union.INSTANCE) {
+    UNION(new Union(UnionMode.Sparse)) {
       @Override
       public Field getField() {
         throw new UnsupportedOperationException("Cannot get simple field for Union type");
       }
 
       @Override
-      public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
         return new UnionVector(name, allocator, callBack);
       }
 
@@ -499,7 +501,7 @@ public ArrowType getType() {
 
     public abstract Field getField();
 
-    public abstract ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale);
+    public abstract FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale);
 
     public abstract FieldWriter getNewFieldWriter(ValueVector vector);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 49d0503e47036..36712b9bea31e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -18,19 +18,24 @@
 package org.apache.arrow.vector.types.pojo;
 
 
-import com.google.common.collect.ImmutableList;
-import com.google.flatbuffers.FlatBufferBuilder;
+import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
-import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
+import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.schema.TypeLayout;
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
 
 public class Field {
   private final String name;
   private final boolean nullable;
   private final ArrowType type;
   private final List<Field> children;
+  private final TypeLayout typeLayout;
 
   public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
     this.name = name;
@@ -41,18 +46,32 @@ public Field(String name, boolean nullable, ArrowType type, List<Field> children
     } else {
       this.children = children;
     }
+    this.typeLayout = TypeLayout.getTypeLayout(type);
   }
 
   public static Field convertField(org.apache.arrow.flatbuf.Field field) {
     String name = field.name();
     boolean nullable = field.nullable();
     ArrowType type = getTypeForField(field);
+    List<ArrowVectorType> buffers = new ArrayList<>();
+    for (int i = 0; i < field.buffersLength(); ++i) {
+      buffers.add(new ArrowVectorType(field.buffers(i)));
+    }
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     for (int i = 0; i < field.childrenLength(); i++) {
       childrenBuilder.add(convertField(field.children(i)));
     }
     List<Field> children = childrenBuilder.build();
-    return new Field(name, nullable, type, children);
+    Field result = new Field(name, nullable, type, children);
+    TypeLayout typeLayout = result.getTypeLayout();
+    if (typeLayout.getVectors().size() != field.buffersLength()) {
+      List<ArrowVectorType> types = new ArrayList<>();
+      for (int i = 0; i < field.buffersLength(); i++) {
+        types.add(new ArrowVectorType(field.buffers(i)));
+      }
+      throw new IllegalArgumentException("Deserialized field does not match expected vectors. expected: " + typeLayout.getVectorTypes() + " got " + types);
+    }
+    return result;
   }
 
   public int getField(FlatBufferBuilder builder) {
@@ -63,12 +82,18 @@ public int getField(FlatBufferBuilder builder) {
       childrenData[i] = children.get(i).getField(builder);
     }
     int childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, childrenData);
+    short[] buffersData = new short[typeLayout.getVectors().size()];
+    for (int i = 0; i < buffersData.length; i++) {
+      buffersData[i] = typeLayout.getVectors().get(i).getType().getType();
+    }
+    int buffersOffset =  org.apache.arrow.flatbuf.Field.createBuffersVector(builder, buffersData );
     org.apache.arrow.flatbuf.Field.startField(builder);
     org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
     org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
     org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeType());
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
     org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
+    org.apache.arrow.flatbuf.Field.addBuffers(builder, buffersOffset);
     return org.apache.arrow.flatbuf.Field.endField(builder);
   }
 
@@ -88,6 +113,10 @@ public List<Field> getChildren() {
     return children;
   }
 
+  public TypeLayout getTypeLayout() {
+    return typeLayout;
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (!(obj instanceof Field)) {
@@ -102,4 +131,9 @@ public boolean equals(Object obj) {
                     (this.children.size() == 0 && that.children == null));
 
   }
+
+  @Override
+  public String toString() {
+    return String.format("Field{name=%s, type=%s, children=%s, layout=%s}", name, type, children, typeLayout);
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
index 9e2894170b24b..231be9bd55ca7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -18,15 +18,13 @@
 package org.apache.arrow.vector.types.pojo;
 
 
-import com.google.common.collect.ImmutableList;
-import com.google.flatbuffers.FlatBufferBuilder;
+import static org.apache.arrow.vector.types.pojo.Field.convertField;
 
-import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.Objects;
 
-import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
-import static org.apache.arrow.vector.types.pojo.Field.convertField;
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
 
 public class Schema {
   private List<Field> fields;
@@ -71,4 +69,9 @@ public static Schema convertSchema(org.apache.arrow.flatbuf.Schema schema) {
     List<Field> fields = childrenBuilder.build();
     return new Schema(fields);
   }
+
+  @Override
+  public String toString() {
+    return "Schema" + fields;
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
new file mode 100644
index 0000000000000..85bb2cfc99f81
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.io.IOException;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestVectorUnloadLoad {
+
+  static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+
+  @Test
+  public void test() throws IOException {
+    int count = 10000;
+    Schema schema;
+
+    try (
+        BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) {
+      ComplexWriter writer = new ComplexWriterImpl("root", parent);
+      MapWriter rootWriter = writer.rootAsMap();
+      IntWriter intWriter = rootWriter.integer("int");
+      BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+      for (int i = 0; i < count; i++) {
+        intWriter.setPosition(i);
+        intWriter.writeInt(i);
+        bigIntWriter.setPosition(i);
+        bigIntWriter.writeBigInt(i);
+      }
+      writer.setValueCount(count);
+
+      VectorUnloader vectorUnloader = new VectorUnloader((MapVector)parent.getChild("root"));
+      schema = vectorUnloader.getSchema();
+
+      try (
+          ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+          BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+          MapVector newParent = new MapVector("parent", finalVectorsAllocator, null)) {
+        MapVector root = newParent.addOrGet("root", MinorType.MAP, MapVector.class);
+        VectorLoader vectorLoader = new VectorLoader(schema, root);
+
+        vectorLoader.load(recordBatch);
+
+        MapReader rootReader = new SingleMapReaderImpl(newParent).reader("root");
+        for (int i = 0; i < count; i++) {
+          rootReader.setPosition(i);
+          Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
+          Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+        }
+      }
+    }
+  }
+
+  @AfterClass
+  public static void afterClass() {
+    allocator.close();
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/ByteArrayReadableSeekableByteChannel.java b/java/vector/src/test/java/org/apache/arrow/vector/file/ByteArrayReadableSeekableByteChannel.java
new file mode 100644
index 0000000000000..7c423d5881aea
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/ByteArrayReadableSeekableByteChannel.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+
+public class ByteArrayReadableSeekableByteChannel implements SeekableByteChannel {
+  private byte[] byteArray;
+  private int position = 0;
+
+  public ByteArrayReadableSeekableByteChannel(byte[] byteArray) {
+    if (byteArray == null) {
+      throw new NullPointerException();
+    }
+    this.byteArray = byteArray;
+  }
+
+  @Override
+  public boolean isOpen() {
+    return byteArray != null;
+  }
+
+  @Override
+  public void close() throws IOException {
+    byteArray = null;
+  }
+
+  @Override
+  public int read(final ByteBuffer dst) throws IOException {
+    int remainingInBuf = byteArray.length - this.position;
+    int length = Math.min(dst.remaining(), remainingInBuf);
+    dst.put(this.byteArray, this.position, length);
+    this.position += length;
+    return length;
+  }
+
+  @Override
+  public long position() throws IOException {
+    return this.position;
+  }
+
+  @Override
+  public SeekableByteChannel position(final long newPosition) throws IOException {
+    this.position = (int)newPosition;
+    return this;
+  }
+
+  @Override
+  public long size() throws IOException {
+    return this.byteArray.length;
+  }
+
+  @Override
+  public int write(final ByteBuffer src) throws IOException {
+    throw new UnsupportedOperationException("Read only");
+  }
+
+  @Override
+  public SeekableByteChannel truncate(final long size) throws IOException {
+    throw new UnsupportedOperationException("Read only");
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
new file mode 100644
index 0000000000000..11de0a2ef00a0
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -0,0 +1,331 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector.Accessor;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.schema.ArrowBuffer;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import io.netty.buffer.ArrowBuf;
+
+public class TestArrowFile {
+  private static final int COUNT = 10;
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Integer.MAX_VALUE);
+  }
+
+  @After
+  public void tearDown() {
+    allocator.close();
+  }
+
+  @Test
+  public void testWrite() throws IOException {
+    File file = new File("target/mytest_write.arrow");
+    int count = COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+      writeData(count, parent);
+      write((MapVector)parent.getChild("root"), file);
+    }
+  }
+
+  @Test
+  public void testWriteComplex() throws IOException {
+    File file = new File("target/mytest_write_complex.arrow");
+    int count = COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+      writeComplexData(count, parent);
+      validateComplexContent(count, parent);
+      write((MapVector)parent.getChild("root"), file);
+    }
+  }
+
+  private void writeComplexData(int count, MapVector parent) {
+    ArrowBuf varchar = allocator.buffer(3);
+    varchar.readerIndex(0);
+    varchar.setByte(0, 'a');
+    varchar.setByte(1, 'b');
+    varchar.setByte(2, 'c');
+    varchar.writerIndex(3);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    ListWriter listWriter = rootWriter.list("list");
+    MapWriter mapWriter = rootWriter.map("map");
+    for (int i = 0; i < count; i++) {
+      intWriter.setPosition(i);
+      intWriter.writeInt(i);
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 3; j++) {
+        listWriter.varChar().writeVarChar(0, 3, varchar);
+      }
+      listWriter.endList();
+      mapWriter.setPosition(i);
+      mapWriter.start();
+      mapWriter.timeStamp("timestamp").writeTimeStamp(i);
+      mapWriter.end();
+    }
+    writer.setValueCount(count);
+    varchar.release();
+  }
+
+
+  private void writeData(int count, MapVector parent) {
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    for (int i = 0; i < count; i++) {
+      intWriter.setPosition(i);
+      intWriter.writeInt(i);
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+    }
+    writer.setValueCount(count);
+  }
+
+  @Test
+  public void testWriteRead() throws IOException {
+    File file = new File("target/mytest.arrow");
+    int count = COUNT;
+
+    // write
+    try (
+        BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+      writeData(count, parent);
+      write((MapVector)parent.getChild("root"), file);
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        FileInputStream fileInputStream = new FileInputStream(file);
+        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)
+        ) {
+      ArrowFooter footer = arrowReader.readFooter();
+      Schema schema = footer.getSchema();
+      System.out.println("reading schema: " + schema);
+
+      // initialize vectors
+
+      MapVector root = parent.addOrGet("root", MinorType.MAP, MapVector.class);
+
+      VectorLoader vectorLoader = new VectorLoader(schema, root);
+
+      List<ArrowBlock> recordBatches = footer.getRecordBatches();
+      for (ArrowBlock rbBlock : recordBatches) {
+        Assert.assertEquals(0, rbBlock.getOffset() % 8);
+        Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
+        try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+          List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
+          for (ArrowBuffer arrowBuffer : buffersLayout) {
+            Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+          }
+          vectorLoader.load(recordBatch);
+        }
+
+        validateContent(count, parent);
+      }
+    }
+  }
+
+  private void validateContent(int count, MapVector parent) {
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < count; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
+      Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+    }
+  }
+
+  @Test
+  public void testWriteReadComplex() throws IOException {
+    File file = new File("target/mytest_complex.arrow");
+    int count = COUNT;
+
+    // write
+    try (
+        BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+      writeComplexData(count, parent);
+      write((MapVector)parent.getChild("root"), file);
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        FileInputStream fileInputStream = new FileInputStream(file);
+        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)
+        ) {
+      ArrowFooter footer = arrowReader.readFooter();
+      Schema schema = footer.getSchema();
+      System.out.println("reading schema: " + schema);
+
+      // initialize vectors
+
+      MapVector root = parent.addOrGet("root", MinorType.MAP, MapVector.class);
+
+      VectorLoader vectorLoader = new VectorLoader(schema, root);
+
+      List<ArrowBlock> recordBatches = footer.getRecordBatches();
+      for (ArrowBlock rbBlock : recordBatches) {
+        try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+          vectorLoader.load(recordBatch);
+        }
+        validateComplexContent(count, parent);
+      }
+    }
+  }
+
+  public void printVectors(List<FieldVector> vectors) {
+    for (FieldVector vector : vectors) {
+      System.out.println(vector.getField().getName());
+      Accessor accessor = vector.getAccessor();
+      int valueCount = accessor.getValueCount();
+      for (int i = 0; i < valueCount; i++) {
+        System.out.println(accessor.getObject(i));
+      }
+    }
+  }
+
+  private void validateComplexContent(int count, MapVector parent) {
+    printVectors(parent.getChildrenFromFields());
+
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < count; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
+      Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+      Assert.assertEquals(i % 3, rootReader.reader("list").size());
+      Assert.assertEquals(i, rootReader.reader("map").reader("timestamp").readDateTime().getMillis() % COUNT);
+    }
+  }
+
+  private void write(MapVector parent, File file) throws FileNotFoundException, IOException {
+    VectorUnloader vectorUnloader = new VectorUnloader(parent);
+    Schema schema = vectorUnloader.getSchema();
+    System.out.println("writing schema: " + schema);
+    try (
+        FileOutputStream fileOutputStream = new FileOutputStream(file);
+        ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
+        ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+            ) {
+      arrowWriter.writeRecordBatch(recordBatch);
+    }
+  }
+
+  @Test
+  public void testWriteReadMultipleRBs() throws IOException {
+    File file = new File("target/mytest_multiple.arrow");
+    int count = COUNT;
+
+    // write
+    try (
+        BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", originalVectorAllocator, null);
+        FileOutputStream fileOutputStream = new FileOutputStream(file);) {
+      writeData(count, parent);
+      VectorUnloader vectorUnloader = new VectorUnloader(parent.getChild("root"));
+      Schema schema = vectorUnloader.getSchema();
+      Assert.assertEquals(2, schema.getFields().size());
+      try (ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);) {
+        try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch()) {
+          arrowWriter.writeRecordBatch(recordBatch);
+        }
+        parent.allocateNew();
+        writeData(count, parent);
+        try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch()) {
+          arrowWriter.writeRecordBatch(recordBatch);
+        }
+      }
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        FileInputStream fileInputStream = new FileInputStream(file);
+        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null);
+        ) {
+      ArrowFooter footer = arrowReader.readFooter();
+      Schema schema = footer.getSchema();
+      System.out.println("reading schema: " + schema);
+      MapVector root = parent.addOrGet("root", MinorType.MAP, MapVector.class);
+      VectorLoader vectorLoader = new VectorLoader(schema, root);
+      List<ArrowBlock> recordBatches = footer.getRecordBatches();
+      Assert.assertEquals(2, recordBatches.size());
+      for (ArrowBlock rbBlock : recordBatches) {
+        Assert.assertEquals(0, rbBlock.getOffset() % 8);
+        Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
+        try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+          List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
+          for (ArrowBuffer arrowBuffer : buffersLayout) {
+            Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+          }
+          vectorLoader.load(recordBatch);
+          validateContent(count, parent);
+        }
+      }
+    }
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
new file mode 100644
index 0000000000000..707dba2af9898
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+
+import org.apache.arrow.flatbuf.Footer;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+public class TestArrowFooter {
+
+  @Test
+  public void test() {
+    Schema schema = new Schema(asList(
+        new Field("a", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())
+        ));
+    ArrowFooter footer = new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), Collections.<ArrowBlock>emptyList());
+    ArrowFooter newFooter = roundTrip(footer);
+    assertEquals(footer, newFooter);
+  }
+
+
+  private ArrowFooter roundTrip(ArrowFooter footer) {
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    int i = footer.writeTo(builder);
+    builder.finish(i);
+    ByteBuffer dataBuffer = builder.dataBuffer();
+    ArrowFooter newFooter = new ArrowFooter(Footer.getRootAsFooter(dataBuffer));
+    return newFooter;
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
new file mode 100644
index 0000000000000..f90329aca11dd
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.file.ArrowBlock;
+import org.apache.arrow.vector.file.ArrowFooter;
+import org.apache.arrow.vector.file.ArrowReader;
+import org.apache.arrow.vector.file.ArrowWriter;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Before;
+import org.junit.Test;
+
+import io.netty.buffer.ArrowBuf;
+
+public class TestArrowReaderWriter {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  ArrowBuf buf(byte[] bytes) {
+    ArrowBuf buffer = allocator.buffer(bytes.length);
+    buffer.writeBytes(bytes);
+    return buffer;
+  }
+
+  byte[] array(ArrowBuf buf) {
+    byte[] bytes = new byte[buf.readableBytes()];
+    buf.readBytes(bytes);
+    return bytes;
+  }
+
+  @Test
+  public void test() throws IOException {
+    Schema schema = new Schema(asList(new Field("testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
+    byte[] validity = new byte[] { (byte)255, 0};
+    // second half is "undefined"
+    byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    try (ArrowWriter writer = new ArrowWriter(Channels.newChannel(out), schema)) {
+      ArrowBuf validityb = buf(validity);
+      ArrowBuf valuesb =  buf(values);
+      writer.writeRecordBatch(new ArrowRecordBatch(16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)));
+    }
+
+    byte[] byteArray = out.toByteArray();
+
+    try (ArrowReader reader = new ArrowReader(new ByteArrayReadableSeekableByteChannel(byteArray), allocator)) {
+      ArrowFooter footer = reader.readFooter();
+      Schema readSchema = footer.getSchema();
+      assertEquals(schema, readSchema);
+      assertTrue(readSchema.getFields().get(0).getTypeLayout().getVectorTypes().toString(), readSchema.getFields().get(0).getTypeLayout().getVectors().size() > 0);
+      // TODO: dictionaries
+      List<ArrowBlock> recordBatches = footer.getRecordBatches();
+      assertEquals(1, recordBatches.size());
+      ArrowRecordBatch recordBatch = reader.readRecordBatch(recordBatches.get(0));
+      List<ArrowFieldNode> nodes = recordBatch.getNodes();
+      assertEquals(1, nodes.size());
+      ArrowFieldNode node = nodes.get(0);
+      assertEquals(16, node.getLength());
+      assertEquals(8, node.getNullCount());
+      List<ArrowBuf> buffers = recordBatch.getBuffers();
+      assertEquals(2, buffers.size());
+      assertArrayEquals(validity, array(buffers.get(0)));
+      assertArrayEquals(values, array(buffers.get(1)));
+
+    }
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 06a1149c0d6c1..61327f1970e83 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -17,19 +17,24 @@
  */
 package org.apache.arrow.vector.pojo;
 
-import com.google.common.collect.ImmutableList;
-import com.google.flatbuffers.FlatBufferBuilder;
+import static org.apache.arrow.flatbuf.Precision.DOUBLE;
+import static org.apache.arrow.flatbuf.Precision.SINGLE;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.List;
+import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Test;
 
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
 
 /**
  * Test conversion between Flatbuf and Pojo field representations
@@ -46,7 +51,7 @@ public void simple() {
   public void complex() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
-    childrenBuilder.add(new Field("child2", true, new FloatingPoint(0), ImmutableList.<Field>of()));
+    childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
 
     Field initialField = new Field("a", true, Tuple.INSTANCE, childrenBuilder.build());
     run(initialField);
@@ -56,10 +61,29 @@ public void complex() {
   public void schema() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
-    childrenBuilder.add(new Field("child2", true, new FloatingPoint(0), ImmutableList.<Field>of()));
+    childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
     Schema initialSchema = new Schema(childrenBuilder.build());
     run(initialSchema);
+  }
 
+  @Test
+  public void nestedSchema() {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
+    childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
+    childrenBuilder.add(new Field("child3", true, new Tuple(), ImmutableList.<Field>of(
+        new Field("child3.1", true, Utf8.INSTANCE, null),
+        new Field("child3.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
+        )));
+    childrenBuilder.add(new Field("child4", true, new List(), ImmutableList.<Field>of(
+        new Field("child4.1", true, Utf8.INSTANCE, null)
+        )));
+    childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse), ImmutableList.<Field>of(
+        new Field("child5.1", true, new Timestamp("UTC"), null),
+        new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
+        )));
+    Schema initialSchema = new Schema(childrenBuilder.build());
+    run(initialSchema);
   }
 
   private void run(Field initialField) {

From 907cc5a1295c4e9227ac50abf5babbe497f1edd1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@apache.org>
Date: Sun, 28 Aug 2016 13:43:01 -0400
Subject: [PATCH 0122/1644] ARROW-262: Start metadata specification document

The purpose of this patch is to:

* Provide exposition and a place to clarify / provide examples illustrating the canonical metadata
* Begin providing definitions of logical types
* Where relevant, the data header metadata generated by a particular logical type (for example: strings produce one fewer buffer compared with List<UInt8> even though the effective memory layout is the same as a the nested type without any nulls in its child array)

This is not a complete specification and will require follow-up JIRAs to address more logical types and fill other gaps.

Author: Wes McKinney <wesm@apache.org>

Closes #121 from wesm/ARROW-262 and squashes the following commits:

bba5e82 [Wes McKinney] int->short
8cc52fd [Wes McKinney] Drafting Metadata specification document
---
 format/Message.fbs |   3 +-
 format/Metadata.md | 258 +++++++++++++++++++++++++++++++++++++++++++++
 format/README.md   |   1 +
 3 files changed, 261 insertions(+), 1 deletion(-)
 create mode 100644 format/Metadata.md

diff --git a/format/Message.fbs b/format/Message.fbs
index b02f3fa38694e..71428b581031f 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -28,12 +28,13 @@ table Int {
   is_signed: bool;
 }
 
-enum Precision:short {SINGLE, DOUBLE}
+enum Precision:short {HALF, SINGLE, DOUBLE}
 
 table FloatingPoint {
   precision: Precision;
 }
 
+/// Unicode with UTF-8 encoding
 table Utf8 {
 }
 
diff --git a/format/Metadata.md b/format/Metadata.md
new file mode 100644
index 0000000000000..e227b8d4afd84
--- /dev/null
+++ b/format/Metadata.md
@@ -0,0 +1,258 @@
+# Metadata: Logical types, schemas, data headers
+
+This is documentation for the Arrow metadata specification, which enables
+systems to communicate the
+
+* Logical array types (which are implemented using the physical memory layouts
+  specified in [Layout.md][1])
+
+* Schemas for table-like collections of Arrow data structures
+
+* "Data headers" indicating the physical locations of memory buffers sufficient
+  to reconstruct a Arrow data structures without copying memory.
+
+## Canonical implementation
+
+We are using [Flatbuffers][2] for low-overhead reading and writing of the Arrow
+metadata. See [Message.fbs][3].
+
+## Schemas
+
+The `Schema` type describes a table-like structure consisting of any number of
+Arrow arrays, each of which can be interpreted as a column in the table. A
+schema by itself does not describe the physical structure of any particular set
+of data.
+
+A schema consists of a sequence of **fields**, which are metadata describing
+the columns. The Flatbuffers IDL for a field is:
+
+```
+table Field {
+  // Name is not required, in i.e. a List
+  name: string;
+  nullable: bool;
+  type: Type;
+  children: [Field];
+}
+```
+
+The `type` is the logical type of the field. Nested types, such as List,
+Struct, and Union, have a sequence of child fields.
+
+## Record data headers
+
+A record batch is a collection of top-level named, equal length Arrow arrays
+(or vectors). If one of the arrays contains nested data, its child arrays are
+not required to be the same length as the top-level arrays.
+
+One can be thought of as a realization of a particular schema. The metadata
+describing a particular record batch is called a "data header". Here is the
+Flatbuffers IDL for a record batch data header
+
+```
+table RecordBatch {
+  length: int;
+  nodes: [FieldNode];
+  buffers: [Buffer];
+}
+```
+
+The `nodes` and `buffers` fields are produced by a depth-first traversal /
+flattening of a schema (possibly containing nested types) for a given in-memory
+data set.
+
+### Buffers
+
+A buffer is metadata describing a contiguous memory region relative to some
+virtual address space. This may include:
+
+* Shared memory, e.g. a memory-mapped file
+* An RPC message received in-memory
+* Data in a file
+
+The key form of the Buffer type is:
+
+```
+struct Buffer {
+  offset: long;
+  length: long;
+}
+```
+
+In the context of a record batch, each field has some number of buffers
+associated with it, which are derived from their physical memory layout.
+
+Each logical type (separate from its children, if it is a nested type) has a
+deterministic number of buffers associated with it. These will be specified in
+the logical types section.
+
+### Field metadata
+
+The `FieldNode` values contain metadata about each level in a nested type
+hierarchy.
+
+```
+struct FieldNode {
+  /// The number of value slots in the Arrow array at this level of a nested
+  /// tree
+  length: int;
+
+  /// The number of observed nulls.
+  null_count: int;
+}
+```
+
+## Flattening of nested data
+
+Nested types are flattened in the record batch in depth-first order. When
+visiting each field in the nested type tree, the metadata is appended to the
+top-level `fields` array and the buffers associated with that field (but not
+its children) are appended to the `buffers` array.
+
+For example, let's consider the schema
+
+```
+col1: Struct<a: Int32, b: List<Int64>, c: Float64>
+col2: Utf8
+```
+
+The flattened version of this is:
+
+```
+FieldNode 0: Struct name='col1'
+FieldNode 1: Int32 name=a'
+FieldNode 2: List name='b'
+FieldNode 3: Int64 name='item'  # arbitrary
+FieldNode 4: Float64 name='c'
+FieldNode 5: Utf8 name='col2'
+```
+
+For the buffers produced, we would have the following (as described in more
+detail for each type below):
+
+```
+buffer 0: field 0 validity bitmap
+
+buffer 1: field 1 validity bitmap
+buffer 2: field 1 values <int32_t*>
+
+buffer 3: field 2 validity bitmap
+buffer 4: field 2 list offsets <int32_t*>
+
+buffer 5: field 3 validity bitmap
+buffer 6: field 3 values <int64_t*>
+
+buffer 7: field 4 validity bitmap
+buffer 8: field 4 values <double*>
+
+buffer 9: field 5 validity bitmap
+buffer 10: field 5 offsets <int32_t*>
+buffer 11: field 5 data <uint8_t*>
+```
+
+## Logical types
+
+A logical type consists of a type name and metadata along with an explicit
+mapping to a physical memory representation. These may fall into some different
+categories:
+
+* Types represented as fixed-width primitive arrays (for example: C-style
+  integers and floating point numbers)
+* Types having equivalent memory layout to a physical nested type (e.g. strings
+  use the list representation, but logically are not nested types)
+
+### Integers
+
+In the first version of Arrow we provide the standard 8-bit through 64-bit size
+standard C integer types, both signed and unsigned:
+
+* Signed types: Int8, Int16, Int32, Int64
+* Unsigned types: UInt8, UInt16, UInt32, UInt64
+
+The IDL looks like:
+
+```
+table Int {
+  bitWidth: int;
+  is_signed: bool;
+}
+```
+
+The integer endianness is currently set globally at the schema level. If a
+schema is set to be little-endian, then all integer types occurring within must
+be little-endian. Integers that are part of other data representations, such as
+list offsets and union types, must have the same endianness as the entire
+record batch.
+
+### Floating point numbers
+
+We provide 3 types of floating point numbers as fixed bit-width primitive array
+
+- Half precision, 16-bit width
+- Single precision, 32-bit width
+- Double precision, 64-bit width
+
+The IDL looks like:
+
+```
+enum Precision:int {HALF, SINGLE, DOUBLE}
+
+table FloatingPoint {
+  precision: Precision;
+}
+```
+
+### Boolean
+
+The Boolean logical type is represented as a 1-bit wide primitive physical
+type. The bits are numbered using least-significant bit (LSB) ordering.
+
+Like other fixed bit-width primitive types, boolean data appears as 2 buffers
+in the data header (one bitmap for the validity vector and one for the values).
+
+### List
+
+The `List` logical type is the logical (and identically-named) counterpart to
+the List physical type.
+
+In data header form, the list field node contains 2 buffers:
+
+* Validity bitmap
+* List offsets
+
+The buffers associated with a list's child field are handled recursively
+according to the child logical type (e.g. `List<Utf8>` vs. `List<Boolean>`).
+
+### Utf8 and Binary
+
+We specify two logical types for variable length bytes:
+
+* `Utf8` data is unicode values with UTF-8 encoding
+* `Binary` is any other variable length bytes
+
+These types both have the same memory layout as the nested type `List<UInt8>`,
+with the constraint that the inner bytes can contain no null values. From a
+logical type perspective they are primitive, not nested types.
+
+In data header form, while `List<UInt8>` would appear as 2 field nodes (`List`
+and `UInt8`) and 4 buffers (2 for each of the nodes, as per above), these types
+have a simplified representation single field node (of `Utf8` or `Binary`
+logical type, which have no children) and 3 buffers:
+
+* Validity bitmap
+* List offsets
+* Byte data
+
+### Decimal
+
+TBD
+
+### Timestamp
+
+TBD
+
+## Dictionary encoding
+
+[1]: https://github.com/apache/arrow/blob/master/format/Layout.md
+[2]: http://github.com/google/flatbuffers
+[3]: https://github.com/apache/arrow/blob/master/format/Message.fbs
diff --git a/format/README.md b/format/README.md
index c84e00772c3d6..3b0e50364d83c 100644
--- a/format/README.md
+++ b/format/README.md
@@ -6,6 +6,7 @@
 
 Currently, the Arrow specification consists of these pieces:
 
+- Metadata specification (see Metadata.md)
 - Physical memory layout specification (see Layout.md)
 - Metadata serialized representation (see Message.fbs)
 

From e081a4c27a5a592251f9f325a05479d4120e30e6 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Sun, 28 Aug 2016 13:45:34 -0400
Subject: [PATCH 0123/1644] ARROW-271: Update Field structure to be more
 explicit

This is a proposal. I have not updated the code depending on this yet.

Author: Julien Le Dem <julien@dremio.com>

Closes #124 from julienledem/record_batch and squashes the following commits:

8e42d74 [Julien Le Dem] ARROW-271: Update Field structure to be more explicit add bit_width to vector layout
---
 format/Message.fbs                            | 26 ++++++---
 .../templates/NullableValueVectors.java       |  6 ++-
 .../arrow/vector/schema/ArrowVectorType.java  |  2 +-
 .../arrow/vector/schema/TypeLayout.java       | 22 +++++++-
 .../arrow/vector/schema/VectorLayout.java     | 54 +++++++++++++++----
 .../apache/arrow/vector/types/pojo/Field.java | 43 ++++++++-------
 .../apache/arrow/vector/pojo/TestConvert.java |  2 +
 7 files changed, 115 insertions(+), 40 deletions(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index 71428b581031f..9c95724897757 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -92,17 +92,31 @@ union Type {
   JSONScalar
 }
 
+/// ----------------------------------------------------------------------
+/// The possible types of a vector
+
 enum VectorType: short {
-  /// used in List type Dense Union and variable length primitive types (String, Binary)
+  /// used in List type, Dense Union and variable length primitive types (String, Binary)
   OFFSET,
-  /// fixed length primitive values
-  VALUES,
-  /// Bit vector indicated if each value is null
+  /// actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
+  DATA,
+  /// Bit vector indicating if each value is null
   VALIDITY,
   /// Type vector used in Union type
   TYPE
 }
 
+/// ----------------------------------------------------------------------
+/// represents the physical layout of a buffer
+/// buffers have fixed width slots of a given type
+
+table VectorLayout {
+  /// the width of a slot in the buffer (typically 1, 8, 16, 32 or 64)
+  bit_width: short;
+  /// the purpose of the vector
+  type: VectorType;
+}
+
 /// ----------------------------------------------------------------------
 /// A field represents a named column in a record / row batch or child of a
 /// nested type.
@@ -121,10 +135,10 @@ table Field {
   dictionary: long;
   // children apply only to Nested data types like Struct, List and Union
   children: [Field];
-  /// the buffers produced for this type (as derived from the Type)
+  /// layout of buffers produced for this type (as derived from the Type)
   /// does not include children
   /// each recordbatch will return instances of those Buffers.
-  buffers: [ VectorType ];
+  layout: [ VectorLayout ];
 }
 
 /// ----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 6b1aa040a5ba2..bb2c00121605c 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -34,6 +34,8 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
+import org.apache.arrow.flatbuf.Precision;
+
 /**
  * Nullable${minor.class} implements a vector of values which could be null.  Elements in the vector
  * are first checked against a fixed length vector of boolean values.  Then the element is retrieved
@@ -97,9 +99,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#elseif minor.class == "Time">
     field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), null);
   <#elseif minor.class == "Float4">
-    field = new Field(name, true, new FloatingPoint(org.apache.arrow.flatbuf.Precision.SINGLE), null);
+    field = new Field(name, true, new FloatingPoint(Precision.SINGLE), null);
   <#elseif minor.class == "Float8">
-    field = new Field(name, true, new FloatingPoint(org.apache.arrow.flatbuf.Precision.DOUBLE), null);
+    field = new Field(name, true, new FloatingPoint(Precision.DOUBLE), null);
   <#elseif minor.class == "TimeStamp">
     field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(""), null);
   <#elseif minor.class == "IntervalDay">
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
index e3d3e34e0ae24..9b7fa45bb9ae3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
@@ -21,7 +21,7 @@
 
 public class ArrowVectorType {
 
-  public static final ArrowVectorType VALUES = new ArrowVectorType(VectorType.VALUES);
+  public static final ArrowVectorType DATA = new ArrowVectorType(VectorType.DATA);
   public static final ArrowVectorType OFFSET = new ArrowVectorType(VectorType.OFFSET);
   public static final ArrowVectorType VALIDITY = new ArrowVectorType(VectorType.VALIDITY);
   public static final ArrowVectorType TYPE = new ArrowVectorType(VectorType.TYPE);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 1275e0eb5dc45..15cd49865bdce 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -49,6 +49,8 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 
+import com.google.common.base.Preconditions;
+
 /**
  * The layout of vectors for a given type
  * It defines its own vectors followed by the vectors for the children
@@ -182,7 +184,7 @@ public TypeLayout visit(IntervalYear type) { // TODO: check size
 
   public TypeLayout(List<VectorLayout> vectors) {
     super();
-    this.vectors = vectors;
+    this.vectors = Preconditions.checkNotNull(vectors);
   }
 
   public TypeLayout(VectorLayout... vectors) {
@@ -205,4 +207,22 @@ public List<ArrowVectorType> getVectorTypes() {
   public String toString() {
     return "TypeLayout{" + vectors + "}";
   }
+
+  @Override
+  public int hashCode() {
+    return vectors.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    TypeLayout other = (TypeLayout) obj;
+    return vectors.equals(other.vectors);
+  }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
index 421ebcb837677..532e9d2328b0f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
@@ -17,21 +17,24 @@
  */
 package org.apache.arrow.vector.schema;
 
+import static org.apache.arrow.vector.schema.ArrowVectorType.DATA;
 import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET;
 import static org.apache.arrow.vector.schema.ArrowVectorType.TYPE;
 import static org.apache.arrow.vector.schema.ArrowVectorType.VALIDITY;
-import static org.apache.arrow.vector.schema.ArrowVectorType.VALUES;
 
-public class VectorLayout {
+import com.google.common.base.Preconditions;
+import com.google.flatbuffers.FlatBufferBuilder;
+
+public class VectorLayout implements FBSerializable {
 
   private static final VectorLayout VALIDITY_VECTOR = new VectorLayout(VALIDITY, 1);
   private static final VectorLayout OFFSET_VECTOR = new VectorLayout(OFFSET, 32);
   private static final VectorLayout TYPE_VECTOR = new VectorLayout(TYPE, 32);
-  private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(VALUES, 1);
-  private static final VectorLayout VALUES_64 = new VectorLayout(VALUES, 64);
-  private static final VectorLayout VALUES_32 = new VectorLayout(VALUES, 32);
-  private static final VectorLayout VALUES_16 = new VectorLayout(VALUES, 16);
-  private static final VectorLayout VALUES_8 = new VectorLayout(VALUES, 8);
+  private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(DATA, 1);
+  private static final VectorLayout VALUES_64 = new VectorLayout(DATA, 64);
+  private static final VectorLayout VALUES_32 = new VectorLayout(DATA, 32);
+  private static final VectorLayout VALUES_16 = new VectorLayout(DATA, 16);
+  private static final VectorLayout VALUES_8 = new VectorLayout(DATA, 8);
 
   public static VectorLayout typeVector() {
     return TYPE_VECTOR;
@@ -68,14 +71,21 @@ public static VectorLayout byteVector() {
     return dataVector(8);
   }
 
-  private final int typeBitWidth;
+  private final short typeBitWidth;
 
   private final ArrowVectorType type;
 
   private VectorLayout(ArrowVectorType type, int typeBitWidth) {
     super();
-    this.type = type;
-    this.typeBitWidth = typeBitWidth;
+    this.type = Preconditions.checkNotNull(type);
+    this.typeBitWidth = (short)typeBitWidth;
+    if (typeBitWidth <= 0) {
+      throw new IllegalArgumentException("bitWidth invalid: " + typeBitWidth);
+    }
+  }
+
+  public VectorLayout(org.apache.arrow.flatbuf.VectorLayout layout) {
+    this(new ArrowVectorType(layout.type()), layout.bitWidth());
   }
 
   public int getTypeBitWidth() {
@@ -90,4 +100,28 @@ public ArrowVectorType getType() {
   public String toString() {
     return String.format("{width=%s,type=%s}", typeBitWidth, type);
   }
+
+  @Override
+  public int hashCode() {
+    return 31 * (31 + type.hashCode()) + typeBitWidth;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    VectorLayout other = (VectorLayout) obj;
+    return type.equals(other.type) && (typeBitWidth == other.typeBitWidth);
+  }
+
+  @Override
+  public int writeTo(FlatBufferBuilder builder) {;
+    return org.apache.arrow.flatbuf.VectorLayout.createVectorLayout(builder, typeBitWidth, type.getType());
+  }
+
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 36712b9bea31e..cfa1ed40aeb8c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -20,12 +20,11 @@
 
 import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
-import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.schema.TypeLayout;
+import org.apache.arrow.vector.schema.VectorLayout;
 
 import com.google.common.collect.ImmutableList;
 import com.google.flatbuffers.FlatBufferBuilder;
@@ -37,7 +36,7 @@ public class Field {
   private final List<Field> children;
   private final TypeLayout typeLayout;
 
-  public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
+  private Field(String name, boolean nullable, ArrowType type, List<Field> children, TypeLayout typeLayout) {
     this.name = name;
     this.nullable = nullable;
     this.type = type;
@@ -46,34 +45,37 @@ public Field(String name, boolean nullable, ArrowType type, List<Field> children
     } else {
       this.children = children;
     }
-    this.typeLayout = TypeLayout.getTypeLayout(type);
+    this.typeLayout = typeLayout;
+  }
+
+  public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
+    this(name, nullable, type, children, TypeLayout.getTypeLayout(type));
   }
 
   public static Field convertField(org.apache.arrow.flatbuf.Field field) {
     String name = field.name();
     boolean nullable = field.nullable();
     ArrowType type = getTypeForField(field);
-    List<ArrowVectorType> buffers = new ArrayList<>();
-    for (int i = 0; i < field.buffersLength(); ++i) {
-      buffers.add(new ArrowVectorType(field.buffers(i)));
+    ImmutableList.Builder<org.apache.arrow.vector.schema.VectorLayout> layout = ImmutableList.builder();
+    for (int i = 0; i < field.layoutLength(); ++i) {
+      layout.add(new org.apache.arrow.vector.schema.VectorLayout(field.layout(i)));
     }
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     for (int i = 0; i < field.childrenLength(); i++) {
       childrenBuilder.add(convertField(field.children(i)));
     }
     List<Field> children = childrenBuilder.build();
-    Field result = new Field(name, nullable, type, children);
-    TypeLayout typeLayout = result.getTypeLayout();
-    if (typeLayout.getVectors().size() != field.buffersLength()) {
-      List<ArrowVectorType> types = new ArrayList<>();
-      for (int i = 0; i < field.buffersLength(); i++) {
-        types.add(new ArrowVectorType(field.buffers(i)));
-      }
-      throw new IllegalArgumentException("Deserialized field does not match expected vectors. expected: " + typeLayout.getVectorTypes() + " got " + types);
-    }
+    Field result = new Field(name, nullable, type, children, new TypeLayout(layout.build()));
     return result;
   }
 
+  public void validate() {
+    TypeLayout expectedLayout = TypeLayout.getTypeLayout(type);
+    if (!expectedLayout.equals(typeLayout)) {
+      throw new IllegalArgumentException("Deserialized field does not match expected vectors. expected: " + expectedLayout + " got " + typeLayout);
+    }
+  }
+
   public int getField(FlatBufferBuilder builder) {
     int nameOffset = builder.createString(name);
     int typeOffset = type.getType(builder);
@@ -82,18 +84,19 @@ public int getField(FlatBufferBuilder builder) {
       childrenData[i] = children.get(i).getField(builder);
     }
     int childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, childrenData);
-    short[] buffersData = new short[typeLayout.getVectors().size()];
+    int[] buffersData = new int[typeLayout.getVectors().size()];
     for (int i = 0; i < buffersData.length; i++) {
-      buffersData[i] = typeLayout.getVectors().get(i).getType().getType();
+      VectorLayout vectorLayout = typeLayout.getVectors().get(i);
+      buffersData[i] = vectorLayout.writeTo(builder);
     }
-    int buffersOffset =  org.apache.arrow.flatbuf.Field.createBuffersVector(builder, buffersData );
+    int layoutOffset =  org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData);
     org.apache.arrow.flatbuf.Field.startField(builder);
     org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
     org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
     org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeType());
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
     org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
-    org.apache.arrow.flatbuf.Field.addBuffers(builder, buffersOffset);
+    org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
     return org.apache.arrow.flatbuf.Field.endField(builder);
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 61327f1970e83..e557cc84f3bae 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -22,6 +22,8 @@
 import static org.junit.Assert.assertEquals;
 
 import org.apache.arrow.flatbuf.UnionMode;
+import static org.junit.Assert.assertEquals;
+
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.List;

From 0a411fd29ed1baac6f1524be82fc15e08f2b28db Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 28 Aug 2016 15:25:35 -0400
Subject: [PATCH 0124/1644] ARROW-242: Support Timestamp Data Type

For the Pandas<->Parquet bridge this is a lossy conversion but must be explicitly activated by the user.

Regarding Parquet 1.0: Yes, the logical type is not supported but should be simply ignored by the reader. Implementation for INT96 timestamps is not in the scope of this PR.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #107 from xhochy/arrow-242 and squashes the following commits:

8db6968 [Uwe L. Korn] Add missing include
34126b1 [Uwe L. Korn] ARROW-242: Support Timestamp Data Type
---
 .../parquet/parquet-reader-writer-test.cc     |  12 +-
 cpp/src/arrow/parquet/parquet-schema-test.cc  |  23 +++-
 cpp/src/arrow/parquet/reader.cc               |   1 +
 cpp/src/arrow/parquet/schema.cc               |  13 ++-
 cpp/src/arrow/parquet/writer.cc               |   1 +
 cpp/src/arrow/types/construct.cc              |   3 +-
 cpp/src/arrow/types/datetime.h                |  12 +-
 cpp/src/arrow/types/primitive.cc              |   1 +
 cpp/src/arrow/types/primitive.h               |  11 ++
 python/pyarrow/array.pyx                      |  40 ++++++-
 python/pyarrow/includes/libarrow.pxd          |   1 +
 python/pyarrow/tests/test_convert_pandas.py   |  24 +++-
 python/pyarrow/tests/test_parquet.py          |   4 +-
 python/src/pyarrow/adapters/pandas.cc         | 107 ++++++++++++++++--
 14 files changed, 232 insertions(+), 21 deletions(-)

diff --git a/cpp/src/arrow/parquet/parquet-reader-writer-test.cc b/cpp/src/arrow/parquet/parquet-reader-writer-test.cc
index bfc27d26d63a1..d7b39dda377d3 100644
--- a/cpp/src/arrow/parquet/parquet-reader-writer-test.cc
+++ b/cpp/src/arrow/parquet/parquet-reader-writer-test.cc
@@ -137,6 +137,15 @@ struct test_traits<Int64Type> {
 
 const int64_t test_traits<Int64Type>::value(-1024);
 
+template <>
+struct test_traits<TimestampType> {
+  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
+  static constexpr LogicalType::type logical_enum = LogicalType::TIMESTAMP_MILLIS;
+  static int64_t const value;
+};
+
+const int64_t test_traits<TimestampType>::value(14695634030000);
+
 template <>
 struct test_traits<FloatType> {
   static constexpr ParquetType::type parquet_enum = ParquetType::FLOAT;
@@ -248,7 +257,8 @@ class TestParquetIO : public ::testing::Test {
 // Parquet version 1.0.
 
 typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
-    Int32Type, UInt64Type, Int64Type, FloatType, DoubleType, StringType> TestTypes;
+    Int32Type, UInt64Type, Int64Type, TimestampType, FloatType, DoubleType,
+    StringType> TestTypes;
 
 TYPED_TEST_CASE(TestParquetIO, TestTypes);
 
diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
index 819cdd3ec4394..a2bcd3e05c307 100644
--- a/cpp/src/arrow/parquet/parquet-schema-test.cc
+++ b/cpp/src/arrow/parquet/parquet-schema-test.cc
@@ -22,6 +22,7 @@
 
 #include "arrow/test-util.h"
 #include "arrow/type.h"
+#include "arrow/types/datetime.h"
 #include "arrow/types/decimal.h"
 #include "arrow/util/status.h"
 
@@ -45,6 +46,9 @@ const auto INT64 = std::make_shared<Int64Type>();
 const auto FLOAT = std::make_shared<FloatType>();
 const auto DOUBLE = std::make_shared<DoubleType>();
 const auto UTF8 = std::make_shared<StringType>();
+const auto TIMESTAMP_MS = std::make_shared<TimestampType>(TimestampType::Unit::MILLI);
+// TODO: This requires parquet-cpp implementing the MICROS enum value
+// const auto TIMESTAMP_US = std::make_shared<TimestampType>(TimestampType::Unit::MICRO);
 const auto BINARY = std::make_shared<ListType>(std::make_shared<Field>("", UINT8));
 const auto DECIMAL_8_4 = std::make_shared<DecimalType>(8, 4);
 
@@ -89,6 +93,14 @@ TEST_F(TestConvertParquetSchema, ParquetFlatPrimitives) {
       PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
   arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
 
+  parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
+      ParquetType::INT64, LogicalType::TIMESTAMP_MILLIS));
+  arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_MS, false));
+
+  // parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
+  //     ParquetType::INT64, LogicalType::TIMESTAMP_MICROS));
+  // arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_US, false));
+
   parquet_fields.push_back(
       PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
   arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
@@ -153,9 +165,6 @@ TEST_F(TestConvertParquetSchema, UnsupportedThings) {
   unsupported_nodes.push_back(PrimitiveNode::Make(
       "int32", Repetition::OPTIONAL, ParquetType::INT32, LogicalType::DATE));
 
-  unsupported_nodes.push_back(PrimitiveNode::Make(
-      "int64", Repetition::OPTIONAL, ParquetType::INT64, LogicalType::TIMESTAMP_MILLIS));
-
   for (const NodePtr& node : unsupported_nodes) {
     ASSERT_RAISES(NotImplemented, ConvertSchema({node}));
   }
@@ -209,6 +218,14 @@ TEST_F(TestConvertArrowSchema, ParquetFlatPrimitives) {
       PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
   arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
 
+  parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
+      ParquetType::INT64, LogicalType::TIMESTAMP_MILLIS));
+  arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_MS, false));
+
+  // parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
+  //     ParquetType::INT64, LogicalType::TIMESTAMP_MICROS));
+  // arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_US, false));
+
   parquet_fields.push_back(
       PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
   arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
index e92967e5363d2..9f6212570dcdc 100644
--- a/cpp/src/arrow/parquet/reader.cc
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -368,6 +368,7 @@ Status FlatColumnReader::Impl::NextBatch(int batch_size, std::shared_ptr<Array>*
     TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
     TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
     TYPED_BATCH_CASE(STRING, StringType, ::parquet::ByteArrayType)
+    TYPED_BATCH_CASE(TIMESTAMP, TimestampType, ::parquet::Int64Type)
     default:
       return Status::NotImplemented(field_->type->ToString());
   }
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index a79342afe2f9d..cd91df32271c1 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -52,6 +52,7 @@ const auto INT64 = std::make_shared<Int64Type>();
 const auto FLOAT = std::make_shared<FloatType>();
 const auto DOUBLE = std::make_shared<DoubleType>();
 const auto UTF8 = std::make_shared<StringType>();
+const auto TIMESTAMP_MS = std::make_shared<TimestampType>(TimestampType::Unit::MILLI);
 const auto BINARY = std::make_shared<ListType>(std::make_shared<Field>("", UINT8));
 
 TypePtr MakeDecimalType(const PrimitiveNode* node) {
@@ -133,6 +134,9 @@ static Status FromInt64(const PrimitiveNode* node, TypePtr* out) {
     case LogicalType::DECIMAL:
       *out = MakeDecimalType(node);
       break;
+    case LogicalType::TIMESTAMP_MILLIS:
+      *out = TIMESTAMP_MS;
+      break;
     default:
       return Status::NotImplemented("Unhandled logical type for int64");
       break;
@@ -289,10 +293,15 @@ Status FieldToNode(const std::shared_ptr<Field>& field,
       type = ParquetType::INT32;
       logical_type = LogicalType::DATE;
       break;
-    case Type::TIMESTAMP:
+    case Type::TIMESTAMP: {
+      auto timestamp_type = static_cast<TimestampType*>(field->type.get());
+      if (timestamp_type->unit != TimestampType::Unit::MILLI) {
+        return Status::NotImplemented(
+            "Other timestamp units than millisecond are not yet support with parquet.");
+      }
       type = ParquetType::INT64;
       logical_type = LogicalType::TIMESTAMP_MILLIS;
-      break;
+    } break;
     case Type::TIMESTAMP_DOUBLE:
       type = ParquetType::INT64;
       // This is specified as seconds since the UNIX epoch
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
index f9514aa2ad2ff..ddee573fa1eb9 100644
--- a/cpp/src/arrow/parquet/writer.cc
+++ b/cpp/src/arrow/parquet/writer.cc
@@ -240,6 +240,7 @@ Status FileWriter::Impl::WriteFlatColumnChunk(
       TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type)
       TYPED_BATCH_CASE(UINT64, UInt64Type, ::parquet::Int64Type)
       TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type)
+      TYPED_BATCH_CASE(TIMESTAMP, TimestampType, ::parquet::Int64Type)
       TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
       TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
     default:
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 5ae9c5ab6d4f9..0b71ea965516c 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -51,6 +51,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(INT32, Int32Builder);
     BUILDER_CASE(UINT64, UInt64Builder);
     BUILDER_CASE(INT64, Int64Builder);
+    BUILDER_CASE(TIMESTAMP, TimestampBuilder);
 
     BUILDER_CASE(BOOL, BooleanBuilder);
 
@@ -105,7 +106,7 @@ Status MakePrimitiveArray(const TypePtr& type, int32_t length,
     MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
     MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
     MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, Int64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
     MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
     MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
     MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP_DOUBLE, DoubleArray);
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
index b782455546c33..241a126d1007f 100644
--- a/cpp/src/arrow/types/datetime.h
+++ b/cpp/src/arrow/types/datetime.h
@@ -18,6 +18,8 @@
 #ifndef ARROW_TYPES_DATETIME_H
 #define ARROW_TYPES_DATETIME_H
 
+#include <string>
+
 #include "arrow/type.h"
 
 namespace arrow {
@@ -34,15 +36,23 @@ struct DateType : public DataType {
   static char const* name() { return "date"; }
 };
 
-struct TimestampType : public DataType {
+struct ARROW_EXPORT TimestampType : public DataType {
   enum class Unit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
 
+  typedef int64_t c_type;
+  static constexpr Type::type type_enum = Type::TIMESTAMP;
+
+  int value_size() const override { return sizeof(int64_t); }
+
   Unit unit;
 
   explicit TimestampType(Unit unit = Unit::MILLI)
       : DataType(Type::TIMESTAMP), unit(unit) {}
 
   TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
+  virtual ~TimestampType() {}
+
+  std::string ToString() const override { return "timestamp"; }
 
   static char const* name() { return "timestamp"; }
 };
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index f4b47f9d2f503..375e94f2bc1c4 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -158,6 +158,7 @@ template class PrimitiveBuilder<Int8Type>;
 template class PrimitiveBuilder<Int16Type>;
 template class PrimitiveBuilder<Int32Type>;
 template class PrimitiveBuilder<Int64Type>;
+template class PrimitiveBuilder<TimestampType>;
 template class PrimitiveBuilder<FloatType>;
 template class PrimitiveBuilder<DoubleType>;
 template class PrimitiveBuilder<BooleanType>;
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index 770de765f1fcc..c643783f681bd 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -26,6 +26,7 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/type.h"
+#include "arrow/types/datetime.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
@@ -100,6 +101,7 @@ NUMERIC_ARRAY_DECL(UInt32Array, UInt32Type);
 NUMERIC_ARRAY_DECL(Int32Array, Int32Type);
 NUMERIC_ARRAY_DECL(UInt64Array, UInt64Type);
 NUMERIC_ARRAY_DECL(Int64Array, Int64Type);
+NUMERIC_ARRAY_DECL(TimestampArray, TimestampType);
 NUMERIC_ARRAY_DECL(FloatArray, FloatType);
 NUMERIC_ARRAY_DECL(DoubleArray, DoubleType);
 
@@ -235,7 +237,15 @@ struct type_traits<Int64Type> {
 
   static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
 };
+
+template <>
+struct type_traits<TimestampType> {
+  typedef TimestampArray ArrayType;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+};
 template <>
+
 struct type_traits<FloatType> {
   typedef FloatArray ArrayType;
 
@@ -260,6 +270,7 @@ typedef NumericBuilder<Int8Type> Int8Builder;
 typedef NumericBuilder<Int16Type> Int16Builder;
 typedef NumericBuilder<Int32Type> Int32Builder;
 typedef NumericBuilder<Int64Type> Int64Builder;
+typedef NumericBuilder<TimestampType> TimestampBuilder;
 
 typedef NumericBuilder<FloatType> FloatBuilder;
 typedef NumericBuilder<DoubleType> DoubleBuilder;
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 619e5ef7e3943..5229b429f58b4 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -19,6 +19,8 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+import numpy as np
+
 from pyarrow.includes.libarrow cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 
@@ -186,6 +188,7 @@ cdef dict _array_classes = {
     Type_DOUBLE: DoubleArray,
     Type_LIST: ListArray,
     Type_STRING: StringArray,
+    Type_TIMESTAMP: Int64Array,
 }
 
 cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
@@ -217,11 +220,28 @@ def from_pylist(object list_obj, DataType type=None):
     return box_arrow_array(sp_array)
 
 
-def from_pandas_series(object series, object mask=None):
+def from_pandas_series(object series, object mask=None, timestamps_to_ms=False):
+    """
+    Convert pandas.Series to an Arrow Array.
+
+    Parameters
+    ----------
+    series: pandas.Series or numpy.ndarray
+
+    mask: pandas.Series or numpy.ndarray
+        array to mask null entries in the series
+
+    timestamps_to_ms: bool
+        Convert datetime columns to ms resolution. This is needed for
+        compability with other functionality like Parquet I/O which
+        only supports milliseconds.
+    """
     cdef:
         shared_ptr[CArray] out
 
     series_values = series_as_ndarray(series)
+    if series_values.dtype.type == np.datetime64 and timestamps_to_ms:
+        series_values = series_values.astype('datetime64[ms]')
 
     if mask is None:
         check_status(pyarrow.PandasToArrow(pyarrow.GetMemoryPool(),
@@ -234,14 +254,28 @@ def from_pandas_series(object series, object mask=None):
     return box_arrow_array(out)
 
 
-def from_pandas_dataframe(object df, name=None):
+def from_pandas_dataframe(object df, name=None, timestamps_to_ms=False):
+    """
+    Convert pandas.DataFrame to an Arrow Table
+
+    Parameters
+    ----------
+    df: pandas.DataFrame
+
+    name: str
+
+    timestamps_to_ms: bool
+        Convert datetime columns to ms resolution. This is needed for
+        compability with other functionality like Parquet I/O which
+        only supports milliseconds.
+    """
     cdef:
         list names = []
         list arrays = []
 
     for name in df.columns:
         col = df[name]
-        arr = from_pandas_series(col)
+        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms)
 
         names.append(name)
         arrays.append(arr)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 91ce069df8f42..854d07d691dbf 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -38,6 +38,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         Type_FLOAT" arrow::Type::FLOAT"
         Type_DOUBLE" arrow::Type::DOUBLE"
 
+        Type_TIMESTAMP" arrow::Type::TIMESTAMP"
         Type_STRING" arrow::Type::STRING"
 
         Type_LIST" arrow::Type::LIST"
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 6dc9c689e249b..55302996f4557 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -33,8 +33,9 @@ def setUp(self):
     def tearDown(self):
         pass
 
-    def _check_pandas_roundtrip(self, df, expected=None):
-        table = A.from_pandas_dataframe(df)
+    def _check_pandas_roundtrip(self, df, expected=None,
+                                timestamps_to_ms=False):
+        table = A.from_pandas_dataframe(df, timestamps_to_ms=timestamps_to_ms)
         result = table.to_pandas()
         if expected is None:
             expected = df
@@ -164,6 +165,25 @@ def test_strings(self):
         expected = pd.DataFrame({'strings': values * repeats})
         self._check_pandas_roundtrip(df, expected)
 
+    def test_timestamps_notimezone(self):
+        df = pd.DataFrame({
+            'datetime64': np.array([
+                '2007-07-13T01:23:34.123',
+                '2006-01-13T12:34:56.432',
+                '2010-08-13T05:46:57.437'],
+                dtype='datetime64[ms]')
+            })
+        self._check_pandas_roundtrip(df, timestamps_to_ms=True)
+
+        df = pd.DataFrame({
+            'datetime64': np.array([
+                '2007-07-13T01:23:34.123456789',
+                '2006-01-13T12:34:56.432539784',
+                '2010-08-13T05:46:57.437699912'],
+                dtype='datetime64[ns]')
+            })
+        self._check_pandas_roundtrip(df, timestamps_to_ms=False)
+
     # def test_category(self):
     #     repeats = 1000
     #     values = [b'foo', None, u'bar', 'qux', np.nan]
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index de9cfbb46e1a2..d89d947b7b6ac 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -57,11 +57,13 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
         'float32': np.arange(size, dtype=np.float32),
         'float64': np.arange(size, dtype=np.float64),
         'bool': np.random.randn(size) > 0,
+        # Pandas only support ns resolution, Arrow at the moment only ms
+        'datetime': np.arange("2016-01-01T00:00:00.001", size, dtype='datetime64[ms]'),
         'str': [str(x) for x in range(size)],
         'str_with_nulls': [None] + [str(x) for x in range(size - 2)] + [None]
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.from_pandas_dataframe(df)
+    arrow_table = A.from_pandas_dataframe(df, timestamps_to_ms=True)
     A.parquet.write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pyarrow.parquet.read_table(filename.strpath)
     df_read = table_read.to_pandas()
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 8dcc2b1c92e11..a4e7fb6f3bb70 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -38,6 +38,7 @@ namespace pyarrow {
 
 using arrow::Array;
 using arrow::Column;
+using arrow::DataType;
 namespace util = arrow::util;
 
 // ----------------------------------------------------------------------
@@ -50,7 +51,7 @@ struct npy_traits {
 template <>
 struct npy_traits<NPY_BOOL> {
   typedef uint8_t value_type;
-  using ArrayType = arrow::BooleanArray;
+  using TypeClass = arrow::BooleanType;
 
   static constexpr bool supports_nulls = false;
   static inline bool isnull(uint8_t v) {
@@ -62,7 +63,7 @@ struct npy_traits<NPY_BOOL> {
   template <>                                       \
   struct npy_traits<NPY_##TYPE> {                   \
     typedef T value_type;                           \
-    using ArrayType = arrow::CapType##Array;        \
+    using TypeClass = arrow::CapType##Type;         \
                                                     \
     static constexpr bool supports_nulls = false;   \
     static inline bool isnull(T v) {                \
@@ -82,7 +83,7 @@ NPY_INT_DECL(UINT64, UInt64, uint64_t);
 template <>
 struct npy_traits<NPY_FLOAT32> {
   typedef float value_type;
-  using ArrayType = arrow::FloatArray;
+  using TypeClass = arrow::FloatType;
 
   static constexpr bool supports_nulls = true;
 
@@ -94,7 +95,7 @@ struct npy_traits<NPY_FLOAT32> {
 template <>
 struct npy_traits<NPY_FLOAT64> {
   typedef double value_type;
-  using ArrayType = arrow::DoubleArray;
+  using TypeClass = arrow::DoubleType;
 
   static constexpr bool supports_nulls = true;
 
@@ -103,6 +104,22 @@ struct npy_traits<NPY_FLOAT64> {
   }
 };
 
+template <>
+struct npy_traits<NPY_DATETIME> {
+  typedef double value_type;
+  using TypeClass = arrow::TimestampType;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(int64_t v) {
+    // NaT = -2**63
+    // = -0x8000000000000000
+    // = -9223372036854775808;
+    // = std::numeric_limits<int64_t>::min()
+    return v == std::numeric_limits<int64_t>::min();
+  }
+};
+
 template <>
 struct npy_traits<NPY_OBJECT> {
   typedef PyObject* value_type;
@@ -206,6 +223,8 @@ class ArrowSerializer {
     return Status::OK();
   }
 
+  Status MakeDataType(std::shared_ptr<DataType>* out);
+
   arrow::MemoryPool* pool_;
 
   PyArrayObject* arr_;
@@ -253,6 +272,39 @@ static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap)
   return null_count;
 }
 
+template <int TYPE>
+inline Status ArrowSerializer<TYPE>::MakeDataType(std::shared_ptr<DataType>* out) {
+  out->reset(new typename npy_traits<TYPE>::TypeClass());
+  return Status::OK();
+}
+
+template <>
+inline Status ArrowSerializer<NPY_DATETIME>::MakeDataType(std::shared_ptr<DataType>* out) {
+  PyArray_Descr* descr = PyArray_DESCR(arr_);
+  auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+  arrow::TimestampType::Unit unit;
+
+  switch (date_dtype->meta.base) {
+      case NPY_FR_s:
+          unit = arrow::TimestampType::Unit::SECOND;
+          break;
+      case NPY_FR_ms:
+          unit = arrow::TimestampType::Unit::MILLI;
+          break;
+      case NPY_FR_us:
+          unit = arrow::TimestampType::Unit::MICRO;
+          break;
+      case NPY_FR_ns:
+          unit = arrow::TimestampType::Unit::NANO;
+          break;
+      default:
+          return Status::ValueError("Unknown NumPy datetime unit");
+  }
+
+  out->reset(new arrow::TimestampType(unit));
+  return Status::OK();
+}
+
 template <int TYPE>
 inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
   typedef npy_traits<TYPE> traits;
@@ -269,9 +321,9 @@ inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
   }
 
   RETURN_NOT_OK(ConvertData());
-  *out = std::make_shared<typename traits::ArrayType>(length_, data_, null_count,
-      null_bitmap_);
-
+  std::shared_ptr<DataType> type;
+  RETURN_NOT_OK(MakeDataType(&type));
+  RETURN_ARROW_NOT_OK(MakePrimitiveArray(type, length_, data_, null_count, null_bitmap_, out));
   return Status::OK();
 }
 
@@ -402,6 +454,7 @@ Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     TO_ARROW_CASE(UINT64);
     TO_ARROW_CASE(FLOAT32);
     TO_ARROW_CASE(FLOAT64);
+    TO_ARROW_CASE(DATETIME);
     TO_ARROW_CASE(OBJECT);
     default:
       std::stringstream ss;
@@ -476,6 +529,17 @@ struct arrow_traits<arrow::Type::DOUBLE> {
   typedef typename npy_traits<NPY_FLOAT64>::value_type T;
 };
 
+template <>
+struct arrow_traits<arrow::Type::TIMESTAMP> {
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr bool supports_nulls = true;
+  static constexpr int64_t na_value = std::numeric_limits<int64_t>::min();
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_integer = true;
+  static constexpr bool is_floating = false;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
+
 template <>
 struct arrow_traits<arrow::Type::STRING> {
   static constexpr int npy_type = NPY_OBJECT;
@@ -494,6 +558,30 @@ static inline PyObject* make_pystring(const uint8_t* data, int32_t length) {
 #endif
 }
 
+inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
+  if (type == NPY_DATETIME) {
+    auto timestamp_type = static_cast<arrow::TimestampType*>(datatype);
+    // We only support ms resolution at the moment
+    PyArray_Descr* descr = PyArray_DESCR(out);
+    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+
+    switch (timestamp_type->unit) {
+      case arrow::TimestampType::Unit::SECOND:
+        date_dtype->meta.base = NPY_FR_s;
+        break;
+      case arrow::TimestampType::Unit::MILLI:
+        date_dtype->meta.base = NPY_FR_ms;
+        break;
+      case arrow::TimestampType::Unit::MICRO:
+        date_dtype->meta.base = NPY_FR_us;
+        break;
+      case arrow::TimestampType::Unit::NANO:
+        date_dtype->meta.base = NPY_FR_ns;
+        break;
+    }
+  }
+}
+
 template <int TYPE>
 class ArrowDeserializer {
  public:
@@ -522,6 +610,8 @@ class ArrowDeserializer {
       return Status::OK();
     }
 
+    set_numpy_metadata(type, col_->type().get(), out_);
+
     return Status::OK();
   }
 
@@ -538,6 +628,8 @@ class ArrowDeserializer {
       return Status::OK();
     }
 
+    set_numpy_metadata(type, col_->type().get(), out_);
+
     if (PyArray_SetBaseObject(out_, py_ref_) == -1) {
       // Error occurred, trust that SetBaseObject set the error state
       return Status::OK();
@@ -713,6 +805,7 @@ Status ArrowToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
     FROM_ARROW_CASE(FLOAT);
     FROM_ARROW_CASE(DOUBLE);
     FROM_ARROW_CASE(STRING);
+    FROM_ARROW_CASE(TIMESTAMP);
     default:
       return Status::NotImplemented("Arrow type reading not implemented");
   }

From e197b2d6e41d0cf6be7c097d6b09c3be29d12cc0 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Mon, 29 Aug 2016 16:08:23 -0700
Subject: [PATCH 0125/1644] ARROW-279: rename vector module to arrow-vector

Author: Julien Le Dem <julien@dremio.com>

Closes #127 from julienledem/rename_vector and squashes the following commits:

cf8a2aa [Julien Le Dem] ARROW-279: rename vector module to arrow-vector
---
 java/memory/pom.xml | 2 +-
 java/vector/pom.xml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 44332f5ed14a8..b91b5981559c3 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -18,7 +18,7 @@
     <version>0.1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
-  <name>arrow-memory</name>
+  <name>Arrow Memory</name>
 
   <dependencies>
 
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 1a2921f6ea521..08f9bc8da4e2c 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -17,8 +17,8 @@
     <artifactId>arrow-java-root</artifactId>
     <version>0.1-SNAPSHOT</version>
   </parent>
-  <artifactId>vector</artifactId>
-  <name>vectors</name>
+  <artifactId>arrow-vector</artifactId>
+  <name>Arrow Vectors</name>
 
   <dependencies>
 

From 2d8ec789365f3c0f82b1f22d76160d5af150dd31 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 6 Sep 2016 11:46:56 -0700
Subject: [PATCH 0126/1644] ARROW-274: Add NullableMapVector to support
 nullable maps

Author: Julien Le Dem <julien@dremio.com>

Closes #128 from julienledem/nullable_map and squashes the following commits:

d98580a [Julien Le Dem] review feedback
ee1dd45 [Julien Le Dem] Fix complex writers/readers
8780f48 [Julien Le Dem] ARROW-274: Add NullableMapVector to support nullable maps
---
 .../main/codegen/templates/MapWriters.java    |  55 ++--
 .../codegen/templates/UnionListWriter.java    |   2 +
 .../main/codegen/templates/UnionVector.java   |   6 +-
 .../main/codegen/templates/UnionWriter.java   |   2 +-
 .../apache/arrow/vector/NullableVector.java   |   2 +-
 .../apache/arrow/vector/VectorUnloader.java   |   4 +-
 .../arrow/vector/complex/MapVector.java       |  53 +---
 .../vector/complex/NullableMapVector.java     | 260 ++++++++++++++++++
 .../complex/impl/AbstractBaseReader.java      |   7 +-
 .../complex/impl/ComplexWriterImpl.java       |  11 +-
 .../complex/impl/NullableMapReaderImpl.java   |  45 +++
 .../complex/impl/SingleMapReaderImpl.java     |   4 +-
 .../arrow/vector/schema/TypeLayout.java       |   3 +-
 .../org/apache/arrow/vector/types/Types.java  |   8 +-
 .../arrow/vector/TestVectorUnloadLoad.java    |   5 +-
 .../complex/impl/TestPromotableWriter.java    |   4 +-
 .../complex/writer/TestComplexWriter.java     |  33 ++-
 .../arrow/vector/file/TestArrowFile.java      |  39 +--
 .../apache/arrow/vector/pojo/TestConvert.java |   2 -
 19 files changed, 408 insertions(+), 137 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java

diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 8a8983a1497cc..7f319a9ca34d8 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -17,14 +17,13 @@
  */
 
 <@pp.dropOutputFile />
-<#list ["Single"] as mode>
+<#list ["Nullable", "Single"] as mode>
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" />
+<#assign index = "idx()">
 <#if mode == "Single">
 <#assign containerClass = "MapVector" />
-<#assign index = "idx()">
 <#else>
-<#assign containerClass = "RepeatedMapVector" />
-<#assign index = "currentChildIndex">
+<#assign containerClass = "NullableMapVector" />
 </#if>
 
 <#include "/@includes/license.ftl" />
@@ -49,9 +48,13 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
 
   protected final ${containerClass} container;
   private final Map<String, FieldWriter> fields = Maps.newHashMap();
-  <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
 
   public ${mode}MapWriter(${containerClass} container) {
+    <#if mode == "Single">
+    if (container instanceof NullableMapVector) {
+      throw new IllegalArgumentException("Invalid container: " + container);
+    }
+    </#if>
     this.container = container;
   }
 
@@ -75,12 +78,12 @@ public MapWriter map(String name) {
       FieldWriter writer = fields.get(name.toLowerCase());
     if(writer == null){
       int vectorCount=container.size();
-      MapVector vector = container.addOrGet(name, MinorType.MAP, MapVector.class);
+      NullableMapVector vector = container.addOrGet(name, MinorType.MAP, NullableMapVector.class);
       writer = new PromotableWriter(vector, container);
       if(vectorCount != container.size()) {
         writer.allocate();
       }
-      writer.setPosition(${index});
+      writer.setPosition(idx());
       fields.put(name.toLowerCase(), writer);
     }
     return writer;
@@ -117,40 +120,12 @@ public ListWriter list(String name) {
       if (container.size() > vectorCount) {
         writer.allocate();
       }
-      writer.setPosition(${index});
+      writer.setPosition(idx());
       fields.put(name.toLowerCase(), writer);
     }
     return writer;
   }
 
-  <#if mode == "Repeated">
-  public void start() {
-      // update the repeated vector to state that there is current+1 objects.
-    final RepeatedMapHolder h = new RepeatedMapHolder();
-    final RepeatedMapVector map = (RepeatedMapVector) container;
-    final RepeatedMapVector.Mutator mutator = map.getMutator();
-
-    // Make sure that the current vector can support the end position of this list.
-    if(container.getValueCapacity() <= idx()) {
-      mutator.setValueCount(idx()+1);
-    }
-
-    map.getAccessor().get(idx(), h);
-    if (h.start >= h.end) {
-      container.getMutator().startNewValue(idx());
-    }
-    currentChildIndex = container.getMutator().add(idx());
-    for(final FieldWriter w : fields.values()) {
-      w.setPosition(currentChildIndex);
-    }
-  }
-
-
-  public void end() {
-    // noop
-  }
-  <#else>
-
   public void setValueCount(int count) {
     container.getMutator().setValueCount(count);
   }
@@ -165,14 +140,16 @@ public void setPosition(int index) {
 
   @Override
   public void start() {
+    <#if mode == "Single">
+    <#else>
+    container.getMutator().setIndexDefined(idx());
+    </#if>
   }
 
   @Override
   public void end() {
   }
 
-  </#if>
-
   <#list vv.types as type><#list type.minor as minor>
   <#assign lowerName = minor.class?uncap_first />
   <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
@@ -204,7 +181,7 @@ public void end() {
       if (currentVector == null || currentVector != vector) {
         vector.allocateNewSafe();
       } 
-      writer.setPosition(${index});
+      writer.setPosition(idx());
       fields.put(name.toLowerCase(), writer);
     }
     return writer;
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index 49d57e716bc8a..d502803d71616 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -160,11 +160,13 @@ public void start() {
     vector.getMutator().setNotNull(idx());
     offsets.getMutator().setSafe(idx() + 1, nextOffset);
     writer.setPosition(nextOffset);
+    writer.start();
   }
 
   @Override
   public void end() {
 //    if (inMap) {
+      writer.end();
       inMap = false;
       final int nextOffset = offsets.getAccessor().get(idx() + 1);
       offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 72125fa50fb82..3014bbba9d52d 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -72,7 +72,7 @@ public class UnionVector implements FieldVector {
   MapVector internalMap;
   UInt1Vector typeVector;
 
-  private MapVector mapVector;
+  private NullableMapVector mapVector;
   private ListVector listVector;
 
   private FieldReader reader;
@@ -127,10 +127,10 @@ public List<BufferBacked> getFieldInnerVectors() {
     throw new UnsupportedOperationException();
   }
   
-  public MapVector getMap() {
+  public NullableMapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
-      mapVector = internalMap.addOrGet("map", MinorType.MAP, MapVector.class);
+      mapVector = internalMap.addOrGet("map", MinorType.MAP, NullableMapVector.class);
       if (internalMap.size() > vectorCount) {
         mapVector.allocateNew();
         if (callBack != null) {
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 1137e2cb0207a..460ec1c0d9586 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -74,7 +74,7 @@ public void endList() {
 
   private MapWriter getMapWriter() {
     if (mapWriter == null) {
-      mapWriter = new SingleMapWriter(data.getMap());
+      mapWriter = new NullableMapWriter(data.getMap());
       mapWriter.setPosition(idx());
       writers.add(mapWriter);
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
index 00c33fc2d6e6c..0212b3c0d7b95 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
@@ -17,7 +17,7 @@
  */
 package org.apache.arrow.vector;
 
-public interface NullableVector extends ValueVector{
+public interface NullableVector extends ValueVector {
 
   ValueVector getValuesVector();
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index e4d37bf47d114..3375a7d5c311b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -68,7 +68,9 @@ private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<Ar
     List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
     List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
     if (fieldBuffers.size() != expectedBuffers.size()) {
-      throw new IllegalArgumentException("wrong number of buffers for field " + vector.getField() + ". found: " + fieldBuffers);
+      throw new IllegalArgumentException(String.format(
+          "wrong number of buffers for field %s in vector %s. found: %s",
+          vector.getField(), vector.getClass().getSimpleName(), fieldBuffers));
     }
     buffers.addAll(fieldBuffers);
     for (FieldVector child : vector.getChildrenFromFields()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index e3696588e6006..1b8483a3d41be 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -18,9 +18,7 @@
 package org.apache.arrow.vector.complex;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -28,15 +26,12 @@
 import javax.annotation.Nullable;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.BaseDataValueVector;
 import org.apache.arrow.vector.BaseValueVector;
-import org.apache.arrow.vector.BufferBacked;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
@@ -49,26 +44,20 @@
 import com.google.common.collect.Ordering;
 import com.google.common.primitives.Ints;
 
-import io.netty.buffer.ArrowBuf;
-
-public class MapVector extends AbstractMapVector implements FieldVector {
+public class MapVector extends AbstractMapVector {
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
-  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
+  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
   int valueCount;
 
-  // TODO: validity vector
-  private final List<BufferBacked> innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList());
-
-  public MapVector(String name, BufferAllocator allocator, CallBack callBack){
+  public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator, callBack);
   }
 
   @Override
   public FieldReader getReader() {
-    //return new SingleMapReaderImpl(MapVector.this);
     return reader;
   }
 
@@ -124,18 +113,9 @@ public int getBufferSizeFor(final int valueCount) {
     return (int) bufferSize;
   }
 
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    int expectedSize = getBufferSize();
-    int actualSize   = super.getBufferSize();
-
-    Preconditions.checkArgument(expectedSize == actualSize, expectedSize + " != " + actualSize);
-    return super.getBuffers(clear);
-  }
-
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new MapTransferPair(this, name, allocator);
+    return new MapTransferPair(this, new MapVector(name, allocator, callBack), false);
   }
 
   @Override
@@ -145,7 +125,7 @@ public TransferPair makeTransferPair(ValueVector to) {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new MapTransferPair(this, ref, allocator);
+    return new MapTransferPair(this, new MapVector(ref, allocator, callBack), false);
   }
 
   protected static class MapTransferPair implements TransferPair{
@@ -153,10 +133,6 @@ protected static class MapTransferPair implements TransferPair{
     private final MapVector from;
     private final MapVector to;
 
-    public MapTransferPair(MapVector from, String name, BufferAllocator allocator) {
-      this(from, new MapVector(name, allocator, from.callBack), false);
-    }
-
     public MapTransferPair(MapVector from, MapVector to) {
       this(from, to, true);
     }
@@ -335,7 +311,6 @@ public void close() {
     super.close();
  }
 
-  @Override
   public void initializeChildrenFromFields(List<Field> children) {
     for (Field field : children) {
       MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
@@ -344,25 +319,9 @@ public void initializeChildrenFromFields(List<Field> children) {
     }
   }
 
-  @Override
+
   public List<FieldVector> getChildrenFromFields() {
     return getChildren();
   }
 
-  @Override
-  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
-    // TODO: something with fieldNode?
-  }
-
-  @Override
-  public List<ArrowBuf> getFieldBuffers() {
-    return BaseDataValueVector.unload(getFieldInnerVectors());
-  }
-
-  @Override
-  public List<BufferBacked> getFieldInnerVectors() {
-    return innerVectors;
-  }
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
new file mode 100644
index 0000000000000..6b257c095d28e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.BaseDataValueVector;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.NullableVectorDefinitionSetter;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+
+public class NullableMapVector extends MapVector implements FieldVector {
+
+  private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
+
+  protected final UInt1Vector bits;
+
+  private final List<BufferBacked> innerVectors;
+
+  private final Accessor accessor;
+  private final Mutator mutator;
+
+  public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+    super(name, checkNotNull(allocator), callBack);
+    this.bits = new UInt1Vector("$bits$", allocator);
+    this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits));
+    this.accessor = new Accessor();
+    this.mutator = new Mutator();
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+    this.valueCount = fieldNode.getLength();
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    return BaseDataValueVector.unload(getFieldInnerVectors());
+  }
+
+  @Override
+  public List<BufferBacked> getFieldInnerVectors() {
+    return innerVectors;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, callBack), false);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new NullableMapTransferPair(this, (NullableMapVector) to, true);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, callBack), false);
+  }
+
+  protected class NullableMapTransferPair extends MapTransferPair {
+
+    private NullableMapVector target;
+
+    protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, boolean allocate) {
+      super(from, to, allocate);
+      this.target = to;
+    }
+
+    @Override
+    public void transfer() {
+      bits.transferTo(target.bits);
+      super.transfer();
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      target.bits.copyFromSafe(fromIndex, toIndex, bits);
+      super.copyValueSafe(fromIndex, toIndex);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      bits.splitAndTransferTo(startIndex, length, target.bits);
+      super.splitAndTransfer(startIndex, length);
+    }
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return Math.min(bits.getValueCapacity(), super.getValueCapacity());
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    return ObjectArrays.concat(bits.getBuffers(clear), super.getBuffers(clear), ArrowBuf.class);
+  }
+
+  @Override
+  public void close() {
+    bits.close();
+    super.close();
+  }
+
+  @Override
+  public void clear() {
+    bits.clear();
+    super.clear();
+  }
+
+
+  @Override
+  public int getBufferSize(){
+    return super.getBufferSize() + bits.getBufferSize();
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+    return super.getBufferSizeFor(valueCount)
+        + bits.getBufferSizeFor(valueCount);
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    bits.setInitialCapacity(numRecords);
+    super.setInitialCapacity(numRecords);
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* Boolean to keep track if all the memory allocations were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      success = super.allocateNewSafe() && bits.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    bits.zeroVector();
+    return success;
+  }
+  public final class Accessor extends MapVector.Accessor  {
+    final UInt1Vector.Accessor bAccessor = bits.getAccessor();
+
+    @Override
+    public Object getObject(int index) {
+      if (isNull(index)) {
+        return null;
+      } else {
+        return super.getObject(index);
+      }
+    }
+
+    @Override
+    public void get(int index, ComplexHolder holder) {
+      holder.isSet = isSet(index);
+      super.get(index, holder);
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return isSet(index) == 0;
+    }
+
+    public int isSet(int index){
+      return bAccessor.get(index);
+    }
+
+  }
+
+  public final class Mutator extends MapVector.Mutator implements NullableVectorDefinitionSetter {
+
+    private Mutator(){
+    }
+
+    @Override
+    public void setIndexDefined(int index){
+      bits.getMutator().setSafe(index, 1);
+    }
+
+    public void setNull(int index){
+      bits.getMutator().setSafe(index, 0);
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      assert valueCount >= 0;
+      super.setValueCount(valueCount);
+      bits.getMutator().setValueCount(valueCount);
+    }
+
+    @Override
+    public void generateTestData(int valueCount){
+      super.generateTestData(valueCount);
+      bits.getMutator().generateTestDataAlt(valueCount);
+    }
+
+    @Override
+    public void reset(){
+      bits.getMutator().setValueCount(0);
+    }
+
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
index 259a954233c06..e7c3c8c7e4b42 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
@@ -19,15 +19,10 @@
 
 import java.util.Iterator;
 
-import com.google.flatbuffers.FlatBufferBuilder;
-import org.apache.arrow.flatbuf.Type;
-import org.apache.arrow.flatbuf.Union;
-import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.holders.UnionHolder;
-import org.apache.arrow.vector.types.pojo.Field;
 
 
 abstract class AbstractBaseReader implements FieldReader{
@@ -44,7 +39,7 @@ public void setPosition(int index){
     this.index = index;
   }
 
-  int idx(){
+  protected int idx(){
     return index;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 89bfefc8f19e3..761b1b43c08aa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -19,6 +19,7 @@
 
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.StateTool;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -29,7 +30,7 @@
 public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
 
-  private SingleMapWriter mapRoot;
+  private NullableMapWriter mapRoot;
   private UnionListWriter listRoot;
   private final MapVector container;
 
@@ -121,8 +122,8 @@ public MapWriter directMap(){
     switch(mode){
 
     case INIT:
-      MapVector map = (MapVector) container;
-      mapRoot = new SingleMapWriter(map);
+      NullableMapVector map = (NullableMapVector) container;
+      mapRoot = new NullableMapWriter(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
       break;
@@ -142,8 +143,8 @@ public MapWriter rootAsMap() {
     switch(mode){
 
     case INIT:
-      MapVector map = container.addOrGet(name, MinorType.MAP, MapVector.class);
-      mapRoot = new SingleMapWriter(map);
+      NullableMapVector map = container.addOrGet(name, MinorType.MAP, NullableMapVector.class);
+      mapRoot = new NullableMapWriter(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
       break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
new file mode 100644
index 0000000000000..18b35c194a184
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
@@ -0,0 +1,45 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex.impl;
+
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+
+public class NullableMapReaderImpl extends SingleMapReaderImpl {
+
+  private NullableMapVector nullableMapVector;
+
+  public NullableMapReaderImpl(MapVector vector) {
+    super((NullableMapVector)vector);
+    this.nullableMapVector = (NullableMapVector)vector;
+  }
+
+  @Override
+  public void copyAsValue(MapWriter writer){
+    NullableMapWriter impl = (NullableMapWriter) writer;
+    impl.container.copyFromSafe(idx(), impl.idx(), nullableMapVector);
+  }
+
+  @Override
+  public void copyAsField(String name, MapWriter writer){
+    NullableMapWriter impl = (NullableMapWriter) writer.map(name);
+    impl.container.copyFromSafe(idx(), impl.idx(), nullableMapVector);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index 1c43240901c4f..ae17b4bbb10dd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -1,5 +1,3 @@
-
-
 /*******************************************************************************
 
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -27,9 +25,9 @@
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.Types.MinorType;
 
 import com.google.common.collect.Maps;
-import org.apache.arrow.vector.types.Types.MinorType;
 
 @SuppressWarnings("unused")
 public class SingleMapReaderImpl extends AbstractFieldReader{
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 15cd49865bdce..9f1efd056cb08 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -90,8 +90,7 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
 
       @Override public TypeLayout visit(Tuple type) {
         List<VectorLayout> vectors = asList(
-            // TODO: add validity vector in Map
-//            validityVector()
+            validityVector()
             );
         return new TypeLayout(vectors);
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 4d0d9ee114ad8..5eef8a008a923 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -47,7 +47,7 @@
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.BitWriterImpl;
@@ -58,7 +58,7 @@
 import org.apache.arrow.vector.complex.impl.IntWriterImpl;
 import org.apache.arrow.vector.complex.impl.IntervalDayWriterImpl;
 import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl;
-import org.apache.arrow.vector.complex.impl.SingleMapWriter;
+import org.apache.arrow.vector.complex.impl.NullableMapWriter;
 import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeWriterImpl;
@@ -139,12 +139,12 @@ public Field getField() {
 
       @Override
       public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-         return new MapVector(name, allocator, callBack);
+         return new NullableMapVector(name, allocator, callBack);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new SingleMapWriter((MapVector) vector);
+        return new NullableMapWriter((NullableMapVector) vector);
       }
     },   //  an empty map column.  Useful for conceptual setup.  Children listed within here
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 85bb2cfc99f81..7dcb8977c0d7f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -22,6 +22,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
@@ -60,14 +61,14 @@ public void test() throws IOException {
       }
       writer.setValueCount(count);
 
-      VectorUnloader vectorUnloader = new VectorUnloader((MapVector)parent.getChild("root"));
+      VectorUnloader vectorUnloader = new VectorUnloader(parent.getChild("root"));
       schema = vectorUnloader.getSchema();
 
       try (
           ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
           BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
           MapVector newParent = new MapVector("parent", finalVectorsAllocator, null)) {
-        MapVector root = newParent.addOrGet("root", MinorType.MAP, MapVector.class);
+        FieldVector root = newParent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
         VectorLoader vectorLoader = new VectorLoader(schema, root);
 
         vectorLoader.load(recordBatch);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 24f00f14df001..689c96fda9202 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -25,8 +25,8 @@
 import org.apache.arrow.vector.DirtyRootAllocator;
 import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
-import org.apache.arrow.vector.holders.UInt4Holder;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.junit.After;
 import org.junit.Before;
@@ -51,7 +51,7 @@ public void terminate() throws Exception {
   public void testPromoteToUnion() throws Exception {
 
     try (final AbstractMapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
-         final MapVector v = container.addOrGet("test", MinorType.MAP, MapVector.class);
+         final NullableMapVector v = container.addOrGet("test", MinorType.MAP, NullableMapVector.class);
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
       container.allocateNew();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index bc17a2b2835c2..fa710dae5eee8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -17,7 +17,6 @@
  */
 package org.apache.arrow.vector.complex.writer;
 
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.ListVector;
@@ -41,6 +40,8 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import io.netty.buffer.ArrowBuf;
+
 public class TestComplexWriter {
 
   static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
@@ -71,6 +72,36 @@ public void simpleNestedTypes() {
     parent.close();
   }
 
+  @Test
+  public void nullableMap() {
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    MapWriter mapWriter = rootWriter.map("map");
+    BigIntWriter nested = mapWriter.bigInt("nested");
+    for (int i = 0; i < COUNT; i++) {
+      if (i % 2 == 0) {
+        mapWriter.setPosition(i);
+        mapWriter.start();
+        nested.writeBigInt(i);
+        mapWriter.end();
+      }
+    }
+    writer.setValueCount(COUNT);
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < COUNT; i++) {
+      rootReader.setPosition(i);
+      if (i % 2 == 0) {
+        Assert.assertNotNull(rootReader.reader("map").readObject());
+        Assert.assertEquals(i, rootReader.reader("map").reader("nested").readLong().longValue());
+      } else {
+        Assert.assertNull(rootReader.reader("map").readObject());
+      }
+    }
+
+    parent.close();
+  }
+
   @Test
   public void listScalarType() {
     ListVector listVector = new ListVector("list", allocator, null);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 11de0a2ef00a0..ad301689cd1e2 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -31,6 +31,7 @@
 import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
@@ -47,10 +48,13 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import io.netty.buffer.ArrowBuf;
 
 public class TestArrowFile {
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestArrowFile.class);
   private static final int COUNT = 10;
   private BufferAllocator allocator;
 
@@ -72,7 +76,7 @@ public void testWrite() throws IOException {
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", vectorAllocator, null)) {
       writeData(count, parent);
-      write((MapVector)parent.getChild("root"), file);
+      write(parent.getChild("root"), file);
     }
   }
 
@@ -82,10 +86,10 @@ public void testWriteComplex() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
       writeComplexData(count, parent);
       validateComplexContent(count, parent);
-      write((MapVector)parent.getChild("root"), file);
+      write(parent.getChild("root"), file);
     }
   }
 
@@ -147,7 +151,7 @@ public void testWriteRead() throws IOException {
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
       writeData(count, parent);
-      write((MapVector)parent.getChild("root"), file);
+      write(parent.getChild("root"), file);
     }
 
     // read
@@ -160,11 +164,11 @@ public void testWriteRead() throws IOException {
         ) {
       ArrowFooter footer = arrowReader.readFooter();
       Schema schema = footer.getSchema();
-      System.out.println("reading schema: " + schema);
+      LOGGER.debug("reading schema: " + schema);
 
       // initialize vectors
 
-      MapVector root = parent.addOrGet("root", MinorType.MAP, MapVector.class);
+      NullableMapVector root = parent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
 
       VectorLoader vectorLoader = new VectorLoader(schema, root);
 
@@ -204,7 +208,7 @@ public void testWriteReadComplex() throws IOException {
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
       writeComplexData(count, parent);
-      write((MapVector)parent.getChild("root"), file);
+      write(parent.getChild("root"), file);
     }
 
     // read
@@ -213,16 +217,15 @@ public void testWriteReadComplex() throws IOException {
         FileInputStream fileInputStream = new FileInputStream(file);
         ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
         BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)
         ) {
       ArrowFooter footer = arrowReader.readFooter();
       Schema schema = footer.getSchema();
-      System.out.println("reading schema: " + schema);
+      LOGGER.debug("reading schema: " + schema);
 
       // initialize vectors
 
-      MapVector root = parent.addOrGet("root", MinorType.MAP, MapVector.class);
-
+      NullableMapVector root = parent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
       VectorLoader vectorLoader = new VectorLoader(schema, root);
 
       List<ArrowBlock> recordBatches = footer.getRecordBatches();
@@ -237,16 +240,16 @@ public void testWriteReadComplex() throws IOException {
 
   public void printVectors(List<FieldVector> vectors) {
     for (FieldVector vector : vectors) {
-      System.out.println(vector.getField().getName());
+      LOGGER.debug(vector.getField().getName());
       Accessor accessor = vector.getAccessor();
       int valueCount = accessor.getValueCount();
       for (int i = 0; i < valueCount; i++) {
-        System.out.println(accessor.getObject(i));
+        LOGGER.debug(String.valueOf(accessor.getObject(i)));
       }
     }
   }
 
-  private void validateComplexContent(int count, MapVector parent) {
+  private void validateComplexContent(int count, NullableMapVector parent) {
     printVectors(parent.getChildrenFromFields());
 
     MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
@@ -259,10 +262,10 @@ private void validateComplexContent(int count, MapVector parent) {
     }
   }
 
-  private void write(MapVector parent, File file) throws FileNotFoundException, IOException {
+  private void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
     VectorUnloader vectorUnloader = new VectorUnloader(parent);
     Schema schema = vectorUnloader.getSchema();
-    System.out.println("writing schema: " + schema);
+    LOGGER.debug("writing schema: " + schema);
     try (
         FileOutputStream fileOutputStream = new FileOutputStream(file);
         ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
@@ -308,8 +311,8 @@ public void testWriteReadMultipleRBs() throws IOException {
         ) {
       ArrowFooter footer = arrowReader.readFooter();
       Schema schema = footer.getSchema();
-      System.out.println("reading schema: " + schema);
-      MapVector root = parent.addOrGet("root", MinorType.MAP, MapVector.class);
+      LOGGER.debug("reading schema: " + schema);
+      NullableMapVector root = parent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
       VectorLoader vectorLoader = new VectorLoader(schema, root);
       List<ArrowBlock> recordBatches = footer.getRecordBatches();
       Assert.assertEquals(2, recordBatches.size());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index e557cc84f3bae..61327f1970e83 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -22,8 +22,6 @@
 import static org.junit.Assert.assertEquals;
 
 import org.apache.arrow.flatbuf.UnionMode;
-import static org.junit.Assert.assertEquals;
-
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.List;

From 637584becb2db88fc510824c22b87e6effb2232f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 6 Sep 2016 23:59:30 -0400
Subject: [PATCH 0127/1644] ARROW-284: Disable arrow_parquet module in Travis
 CI to triage builds

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #132 from wesm/ARROW-284 and squashes the following commits:

e3410cf [Wes McKinney] Install miniconda in $HOME to avoid long prefix issues in conda-build 2.0
9fd94f5 [Wes McKinney] Do not run death test when valgrind is enabled. Gracefully skip pyarrow.parquet when ARROW_PARQUET=off
ccf56f8 [Wes McKinney] Disable arrow_parquet module in Travis CI
---
 ci/travis_before_script_cpp.sh         |  4 +--
 ci/travis_install_conda.sh             |  4 ++-
 ci/travis_script_python.sh             |  6 ++--
 cpp/cmake_modules/FindParquet.cmake    |  1 +
 cpp/src/arrow/util/memory-pool-test.cc |  6 ++++
 python/CMakeLists.txt                  | 41 ++++++++++++++++----------
 python/cmake_modules/FindArrow.cmake   | 26 +++++++++-------
 python/pyarrow/tests/test_io.py        |  1 +
 python/pyarrow/tests/test_parquet.py   | 38 ++++++++++++++++--------
 python/pyarrow/tests/test_table.py     |  7 +----
 python/setup.py                        | 27 ++++++++++-------
 11 files changed, 101 insertions(+), 60 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 08551f3b009a8..2f02ef247af82 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -25,8 +25,8 @@ echo $GTEST_HOME
 
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
--DARROW_PARQUET=ON \
--DARROW_HDFS=on \
+-DARROW_PARQUET=OFF \
+-DARROW_HDFS=ON \
 -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index 3a8f57bf8f1bf..e9225259e6d58 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -9,7 +9,9 @@ else
 fi
 
 wget -O miniconda.sh $MINICONDA_URL
-export MINICONDA=$TRAVIS_BUILD_DIR/miniconda
+
+export MINICONDA=$HOME/miniconda
+
 bash miniconda.sh -b -p $MINICONDA
 export PATH="$MINICONDA/bin:$PATH"
 conda update -y -q conda
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 4a377428ae43a..61c8e444361df 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -5,7 +5,7 @@ set -e
 PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 
 # Re-use conda installation from C++
-export MINICONDA=$TRAVIS_BUILD_DIR/miniconda
+export MINICONDA=$HOME/miniconda
 export PATH="$MINICONDA/bin:$PATH"
 export PARQUET_HOME=$MINICONDA
 
@@ -31,7 +31,9 @@ python_version_tests() {
   # Expensive dependencies install from Continuum package repo
   conda install -y pip numpy pandas cython
 
-  conda install -y parquet-cpp arrow-cpp -c apache/channel/dev
+  # conda install -y parquet-cpp
+
+  conda install -y arrow-cpp -c apache/channel/dev
 
   # Other stuff pip install
   pip install -r requirements.txt
diff --git a/cpp/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
index e3350d6e13da6..36f4828a999d4 100644
--- a/cpp/cmake_modules/FindParquet.cmake
+++ b/cpp/cmake_modules/FindParquet.cmake
@@ -72,6 +72,7 @@ else ()
 endif ()
 
 mark_as_advanced(
+  PARQUET_FOUND
   PARQUET_INCLUDE_DIR
   PARQUET_LIBS
   PARQUET_LIBRARIES
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
index deb7ffd03ba75..e767e9555244d 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -46,6 +46,10 @@ TEST(DefaultMemoryPool, OOM) {
   ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
 }
 
+// Death tests and valgrind are known to not play well 100% of the time. See
+// googletest documentation
+#ifndef ARROW_VALGRIND
+
 TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
   MemoryPool* pool = default_memory_pool();
 
@@ -60,4 +64,6 @@ TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
   pool->Free(data, 100);
 }
 
+#endif // ARROW_VALGRIND
+
 }  // namespace arrow
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index fdbfce99656ca..522895808de5e 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -340,8 +340,10 @@ if (PYARROW_BUILD_TESTS)
 endif()
 
 ## Parquet
-find_package(Parquet REQUIRED)
-include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
+find_package(Parquet)
+if(PARQUET_FOUND)
+  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
+endif()
 
 ## Arrow
 find_package(Arrow REQUIRED)
@@ -350,8 +352,6 @@ ADD_THIRDPARTY_LIB(arrow
   SHARED_LIB ${ARROW_SHARED_LIB})
 ADD_THIRDPARTY_LIB(arrow_io
   SHARED_LIB ${ARROW_IO_SHARED_LIB})
-ADD_THIRDPARTY_LIB(arrow_parquet
-  SHARED_LIB ${ARROW_PARQUET_SHARED_LIB})
 
 ############################################################
 # Linker setup
@@ -418,6 +418,16 @@ endif()
 add_subdirectory(src/pyarrow)
 add_subdirectory(src/pyarrow/util)
 
+set(CYTHON_EXTENSIONS
+  array
+  config
+  error
+  io
+  scalar
+  schema
+  table
+)
+
 set(PYARROW_SRCS
   src/pyarrow/common.cc
   src/pyarrow/config.cc
@@ -431,9 +441,19 @@ set(PYARROW_SRCS
 set(LINK_LIBS
   arrow
   arrow_io
-  arrow_parquet
 )
 
+if(PARQUET_FOUND AND ARROW_PARQUET_FOUND)
+  ADD_THIRDPARTY_LIB(arrow_parquet
+    SHARED_LIB ${ARROW_PARQUET_SHARED_LIB})
+  set(LINK_LIBS
+    ${LINK_LIBS}
+    arrow_parquet)
+  set(CYTHON_EXTENSIONS
+    ${CYTHON_EXTENSIONS}
+    parquet)
+endif()
+
 SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
 add_library(pyarrow SHARED
@@ -448,17 +468,6 @@ endif()
 # Setup and build Cython modules
 ############################################################
 
-set(CYTHON_EXTENSIONS
-  array
-  config
-  error
-  io
-  parquet
-  scalar
-  schema
-  table
-)
-
 foreach(module ${CYTHON_EXTENSIONS})
     string(REPLACE "." ";" directories ${module})
     list(GET directories -1 module_name)
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 6bd305615fce2..5d5efc431a48f 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -52,7 +52,7 @@ find_library(ARROW_IO_LIB_PATH NAMES arrow_io
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
-if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH AND ARROW_PARQUET_LIB_PATH)
+if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
   set(ARROW_IO_LIB_NAME libarrow_io)
@@ -64,18 +64,9 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH AND ARROW_PARQUET_LIB_PATH)
 
   set(ARROW_IO_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IO_LIB_NAME}.a)
   set(ARROW_IO_SHARED_LIB ${ARROW_LIBS}/${ARROW_IO_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-
-  set(ARROW_PARQUET_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PARQUET_LIB_NAME}.a)
-  set(ARROW_PARQUET_SHARED_LIB ${ARROW_LIBS}/${ARROW_PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-else ()
-  set(ARROW_FOUND FALSE)
-endif ()
-
-if (ARROW_FOUND)
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
     message(STATUS "Found the Arrow IO library: ${ARROW_IO_LIB_PATH}")
-    message(STATUS "Found the Arrow Parquet library: ${ARROW_PARQUET_LIB_PATH}")
   endif ()
 else ()
   if (NOT Arrow_FIND_QUIETLY)
@@ -88,8 +79,23 @@ else ()
       message(STATUS "${ARROW_ERR_MSG}")
     endif (Arrow_FIND_REQUIRED)
   endif ()
+  set(ARROW_FOUND FALSE)
 endif ()
 
+if(ARROW_PARQUET_LIB_PATH)
+  set(ARROW_PARQUET_FOUND TRUE)
+  set(ARROW_PARQUET_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PARQUET_LIB_NAME}.a)
+  set(ARROW_PARQUET_SHARED_LIB ${ARROW_LIBS}/${ARROW_PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  if (NOT Arrow_FIND_QUIETLY)
+    message(STATUS "Found the Arrow Parquet library: ${ARROW_PARQUET_LIB_PATH}")
+  endif ()
+else()
+  if (NOT Arrow_FIND_QUIETLY)
+    message(STATUS "Could not find Arrow Parquet library")
+  endif()
+  set(ARROW_PARQUET_FOUND FALSE)
+endif()
+
 mark_as_advanced(
   ARROW_INCLUDE_DIR
   ARROW_LIBS
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 328e923b941a4..eb92e8ea93a1a 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -46,6 +46,7 @@ def hdfs_test_client():
 
 HDFS_TMP_PATH = '/tmp/pyarrow-test-{0}'.format(random.randint(0, 1000))
 
+
 @pytest.fixture(scope='session')
 def hdfs(request):
     fixture = hdfs_test_client()
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index d89d947b7b6ac..8a2d8cab57267 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -15,33 +15,45 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import unittest
-import pyarrow as arrow
-import pyarrow.parquet
+import pytest
 
-A = arrow
+import pyarrow as A
 
 import numpy as np
-import os.path
 import pandas as pd
 
 import pandas.util.testing as pdt
 
+try:
+    import pyarrow.parquet as pq
+    HAVE_PARQUET = True
+except ImportError:
+    HAVE_PARQUET = False
 
+# XXX: Make Parquet tests opt-in rather than skip-if-not-build
+parquet = pytest.mark.skipif(not HAVE_PARQUET,
+                             reason='Parquet support not built')
+
+
+@parquet
 def test_single_pylist_column_roundtrip(tmpdir):
     for dtype in [int, float]:
-        filename = tmpdir.join('single_{}_column.parquet'.format(dtype.__name__))
+        filename = tmpdir.join('single_{}_column.parquet'
+                               .format(dtype.__name__))
         data = [A.from_pylist(list(map(dtype, range(5))))]
         table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
         A.parquet.write_table(table, filename.strpath)
-        table_read = pyarrow.parquet.read_table(filename.strpath)
-        for col_written, col_read in zip(table.itercolumns(), table_read.itercolumns()):
+        table_read = pq.read_table(filename.strpath)
+        for col_written, col_read in zip(table.itercolumns(),
+                                         table_read.itercolumns()):
             assert col_written.name == col_read.name
             assert col_read.data.num_chunks == 1
             data_written = col_written.data.chunk(0)
             data_read = col_read.data.chunk(0)
             assert data_written.equals(data_read)
 
+
+@parquet
 def test_pandas_parquet_2_0_rountrip(tmpdir):
     size = 10000
     np.random.seed(0)
@@ -58,17 +70,20 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
         'float64': np.arange(size, dtype=np.float64),
         'bool': np.random.randn(size) > 0,
         # Pandas only support ns resolution, Arrow at the moment only ms
-        'datetime': np.arange("2016-01-01T00:00:00.001", size, dtype='datetime64[ms]'),
+        'datetime': np.arange("2016-01-01T00:00:00.001", size,
+                              dtype='datetime64[ms]'),
         'str': [str(x) for x in range(size)],
         'str_with_nulls': [None] + [str(x) for x in range(size - 2)] + [None]
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = A.from_pandas_dataframe(df, timestamps_to_ms=True)
     A.parquet.write_table(arrow_table, filename.strpath, version="2.0")
-    table_read = pyarrow.parquet.read_table(filename.strpath)
+    table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
     pdt.assert_frame_equal(df, df_read)
 
+
+@parquet
 def test_pandas_parquet_1_0_rountrip(tmpdir):
     size = 10000
     np.random.seed(0)
@@ -88,11 +103,10 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = A.from_pandas_dataframe(df)
     A.parquet.write_table(arrow_table, filename.strpath, version="1.0")
-    table_read = pyarrow.parquet.read_table(filename.strpath)
+    table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
 
     # We pass uint32_t as int64_t if we write Parquet version 1.0
     df['uint32'] = df['uint32'].values.astype(np.int64)
 
     pdt.assert_frame_equal(df, df_read)
-
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 83fcbb8faff5d..abf143199fe15 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -16,11 +16,7 @@
 # under the License.
 
 from pyarrow.compat import unittest
-import pyarrow as arrow
-
-A = arrow
-
-import pandas as pd
+import pyarrow as A
 
 
 class TestRowBatch(unittest.TestCase):
@@ -76,4 +72,3 @@ def test_pandas(self):
         assert set(df.columns) == set(('a', 'b'))
         assert df.shape == (5, 2)
         assert df.ix[0, 'b'] == -10
-
diff --git a/python/setup.py b/python/setup.py
index 59410d75a61e2..a5db2b025e6ef 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -97,6 +97,18 @@ def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = ''
 
+    CYTHON_MODULE_NAMES = [
+        'array',
+        'config',
+        'error',
+        'io',
+        'parquet',
+        'scalar',
+        'schema',
+        'table']
+
+    CYTHON_ALLOWED_FAILURES = ['parquet']
+
     def _run_cmake(self):
         # The directory containing this setup.py
         source = osp.dirname(osp.abspath(__file__))
@@ -172,10 +184,13 @@ def _run_cmake(self):
 
         # Move the built C-extension to the place expected by the Python build
         self._found_names = []
-        for name in self.get_cmake_cython_names():
+        for name in self.CYTHON_MODULE_NAMES:
             built_path = self.get_ext_built(name)
             if not os.path.exists(built_path):
                 print(built_path)
+                if name in self.CYTHON_ALLOWED_FAILURES:
+                    print('Cython module {0} failure permitted'.format(name))
+                    continue
                 raise RuntimeError('libpyarrow C-extension failed to build:',
                                    os.path.abspath(built_path))
 
@@ -213,16 +228,6 @@ def get_ext_built(self, name):
             suffix = sysconfig.get_config_var('SO')
             return name + suffix
 
-    def get_cmake_cython_names(self):
-        return ['array',
-                'config',
-                'error',
-                'io',
-                'parquet',
-                'scalar',
-                'schema',
-                'table']
-
     def get_names(self):
         return self._found_names
 

From 214b861ae8f40f5fba544247d40c8995b93eca83 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 7 Sep 2016 00:20:51 -0400
Subject: [PATCH 0128/1644] ARROW-283: [C++] Account for upstream changes in
 parquet-cpp

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #131 from wesm/ARROW-283 and squashes the following commits:

52dfb28 [Wes McKinney] Update arrow_parquet for API changes in parquet-cpp
---
 cpp/src/arrow/parquet/reader.cc | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
index 9f6212570dcdc..440ec84e2c74e 100644
--- a/cpp/src/arrow/parquet/reader.cc
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -149,11 +149,13 @@ bool FileReader::Impl::CheckForFlatColumn(const ::parquet::ColumnDescriptor* des
 }
 
 Status FileReader::Impl::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out) {
-  if (!CheckForFlatColumn(reader_->descr()->Column(i))) {
+  const ::parquet::SchemaDescriptor* schema = reader_->metadata()->schema_descriptor();
+
+  if (!CheckForFlatColumn(schema->Column(i))) {
     return Status::Invalid("The requested column is not flat");
   }
   std::unique_ptr<FlatColumnReader::Impl> impl(
-      new FlatColumnReader::Impl(pool_, reader_->descr()->Column(i), reader_.get(), i));
+      new FlatColumnReader::Impl(pool_, schema->Column(i), reader_.get(), i));
   *out = std::unique_ptr<FlatColumnReader>(new FlatColumnReader(std::move(impl)));
   return Status::OK();
 }
@@ -161,16 +163,20 @@ Status FileReader::Impl::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>*
 Status FileReader::Impl::ReadFlatColumn(int i, std::shared_ptr<Array>* out) {
   std::unique_ptr<FlatColumnReader> flat_column_reader;
   RETURN_NOT_OK(GetFlatColumn(i, &flat_column_reader));
-  return flat_column_reader->NextBatch(reader_->num_rows(), out);
+  return flat_column_reader->NextBatch(reader_->metadata()->num_rows(), out);
 }
 
 Status FileReader::Impl::ReadFlatTable(std::shared_ptr<Table>* table) {
-  const std::string& name = reader_->descr()->schema()->name();
+  auto descr = reader_->metadata()->schema_descriptor();
+
+  const std::string& name = descr->schema()->name();
   std::shared_ptr<Schema> schema;
-  RETURN_NOT_OK(FromParquetSchema(reader_->descr(), &schema));
+  RETURN_NOT_OK(FromParquetSchema(descr, &schema));
+
+  int num_columns = reader_->metadata()->num_columns();
 
-  std::vector<std::shared_ptr<Column>> columns(reader_->num_columns());
-  for (int i = 0; i < reader_->num_columns(); i++) {
+  std::vector<std::shared_ptr<Column>> columns(num_columns);
+  for (int i = 0; i < num_columns; i++) {
     std::shared_ptr<Array> array;
     RETURN_NOT_OK(ReadFlatColumn(i, &array));
     columns[i] = std::make_shared<Column>(schema->field(i), array);
@@ -375,7 +381,7 @@ Status FlatColumnReader::Impl::NextBatch(int batch_size, std::shared_ptr<Array>*
 }
 
 void FlatColumnReader::Impl::NextRowGroup() {
-  if (next_row_group_ < reader_->num_row_groups()) {
+  if (next_row_group_ < reader_->metadata()->num_row_groups()) {
     column_reader_ = reader_->RowGroup(next_row_group_)->Column(column_index_);
     next_row_group_++;
   } else {

From 270ab4e94dba3ec45cfd2297d4f901d51d4a053b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 7 Sep 2016 14:15:16 -0700
Subject: [PATCH 0129/1644] ARROW-278: [Format] Rename Tuple to Struct_ in
 flatbuffers IDL

"Struct" is a reserved keyword in generated bindings for C++. We had used "Tuple" to sidestep this but we discussed and decided to mangle "Struct" instead in the Flatbuffers.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #130 from wesm/ARROW-278 and squashes the following commits:

841a721 [Wes McKinney] Rename Tuple to Struct_ in flatbuffers IDL
---
 cpp/src/arrow/ipc/metadata-internal.cc                 |  6 +++---
 format/Message.fbs                                     | 10 +++++-----
 java/vector/src/main/codegen/data/ArrowTypes.tdd       |  2 +-
 .../org/apache/arrow/vector/complex/MapVector.java     |  4 ++--
 .../org/apache/arrow/vector/schema/TypeLayout.java     |  6 +++---
 .../main/java/org/apache/arrow/vector/types/Types.java |  4 ++--
 .../java/org/apache/arrow/vector/pojo/TestConvert.java |  6 +++---
 7 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index c921e4d8e0114..1c15218c0ba12 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -115,7 +115,7 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       }
       *out = std::make_shared<ListType>(children[0]);
       return Status::OK();
-    case flatbuf::Type_Tuple:
+    case flatbuf::Type_Struct_:
       *out = std::make_shared<StructType>(children);
       return Status::OK();
     case flatbuf::Type_Union:
@@ -153,7 +153,7 @@ static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type
     RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), &field));
     out_children->push_back(field);
   }
-  *offset = flatbuf::CreateTuple(fbb).Union();
+  *offset = flatbuf::CreateStruct_(fbb).Union();
   return Status::OK();
 }
 
@@ -197,7 +197,7 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *out_type = flatbuf::Type_List;
       return ListToFlatbuffer(fbb, type, children, offset);
     case Type::STRUCT:
-      *out_type = flatbuf::Type_Tuple;
+      *out_type = flatbuf::Type_Struct_;
       return StructToFlatbuffer(fbb, type, children, offset);
     default:
       *out_type = flatbuf::Type_NONE;  // Make clang-tidy happy
diff --git a/format/Message.fbs b/format/Message.fbs
index 9c95724897757..78bdaeb35f5a5 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -8,10 +8,10 @@ namespace org.apache.arrow.flatbuf;
 table Null {
 }
 
-/// A Tuple in the flatbuffer metadata is the same as an Arrow Struct
-/// (according to the physical memory layout). We used Tuple here as Struct is
-/// a reserved word in Flatbuffers
-table Tuple {
+/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct
+/// (according to the physical memory layout). We used Struct_ here as
+/// Struct is a reserved word in Flatbuffers
+table Struct_ {
 }
 
 table List {
@@ -87,7 +87,7 @@ union Type {
   IntervalDay,
   IntervalYear,
   List,
-  Tuple,
+  Struct_,
   Union,
   JSONScalar
 }
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 2ecad3d31400f..5cb43bed2b69a 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -21,7 +21,7 @@
       fields: []
     },
     {
-      name: "Tuple",
+      name: "Struct_",
       fields: []
     },
     {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 1b8483a3d41be..aaecb956434e9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -34,7 +34,7 @@
 import org.apache.arrow.vector.holders.ComplexHolder;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringHashMap;
@@ -290,7 +290,7 @@ public Field getField() {
     for (ValueVector child : getChildren()) {
       children.add(child.getField());
     }
-    return new Field(name, false, Tuple.INSTANCE, children);
+    return new Field(name, false, Struct_.INSTANCE, children);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 9f1efd056cb08..885ac2ac3d7f2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -45,7 +45,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.ArrowType.Time;
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
-import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 
@@ -54,7 +54,7 @@
 /**
  * The layout of vectors for a given type
  * It defines its own vectors followed by the vectors for the children
- * if it is a nested type (Tuple, List, Union)
+ * if it is a nested type (Struct_, List, Union)
  */
 public class TypeLayout {
 
@@ -88,7 +88,7 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
         return new TypeLayout(vectors);
       }
 
-      @Override public TypeLayout visit(Tuple type) {
+      @Override public TypeLayout visit(Struct_ type) {
         List<VectorLayout> vectors = asList(
             validityVector()
             );
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 5eef8a008a923..66ef7562ceda1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -84,7 +84,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.ArrowType.Time;
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
-import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -131,7 +131,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return null;
       }
     },
-    MAP(Tuple.INSTANCE) {
+    MAP(Struct_.INSTANCE) {
       @Override
       public Field getField() {
         throw new UnsupportedOperationException("Cannot get simple field for Map type");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 61327f1970e83..448117d84dc3e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -26,7 +26,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.List;
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
-import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -53,7 +53,7 @@ public void complex() {
     childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
     childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
 
-    Field initialField = new Field("a", true, Tuple.INSTANCE, childrenBuilder.build());
+    Field initialField = new Field("a", true, Struct_.INSTANCE, childrenBuilder.build());
     run(initialField);
   }
 
@@ -71,7 +71,7 @@ public void nestedSchema() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
     childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
-    childrenBuilder.add(new Field("child3", true, new Tuple(), ImmutableList.<Field>of(
+    childrenBuilder.add(new Field("child3", true, new Struct_(), ImmutableList.<Field>of(
         new Field("child3.1", true, Utf8.INSTANCE, null),
         new Field("child3.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));

From 52089d609dff3d8d2abe99c7b94f7af9fe4735bd Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent@dremio.com>
Date: Thu, 8 Sep 2016 11:35:08 -0700
Subject: [PATCH 0130/1644] ARROW-285: Optional flatc download

For platforms which don't have a flatc compiler artifact on maven central,
allow to skip the download and manually provide a flatc compiler

usage: mvn -Dflatc.download.skip -Dflatc.executable=/usr/local/bin/flatc

Author: Laurent Goujon <laurent@dremio.com>

Closes #129 from laurentgo/laurent/optional-flatc-download and squashes the following commits:

229c6d5 [Laurent Goujon] Optional flatc download
---
 java/format/pom.xml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index dc5897581b5b3..4cf68bbe057e9 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -25,6 +25,8 @@
 
   <properties>
     <fbs.version>1.2.0-3f79e055</fbs.version>
+    <flatc.download.skip>false</flatc.download.skip>
+    <flatc.executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</flatc.executable>
     <maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
     <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
     <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
@@ -71,6 +73,7 @@
                 <outputDirectory>${project.build.directory}</outputDirectory>
               </artifactItem>
             </artifactItems>
+            <skip>${flatc.download.skip}</skip>
           </configuration>
         </execution>
       </executions>
@@ -92,6 +95,7 @@
               <argument>+x</argument>
               <argument>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</argument>
             </arguments>
+            <skip>${flatc.download.skip}</skip>
           </configuration>
         </execution>
         <execution>
@@ -100,11 +104,11 @@
           </goals>
           <phase>generate-sources</phase>
           <configuration>
-            <executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</executable>
+            <executable>${flatc.executable}</executable>
             <arguments>
               <argument>-j</argument>
               <argument>-o</argument>
-              <argument>target/generated-sources/</argument>
+              <argument>target/generated-sources/flatc</argument>
               <argument>../../format/Message.fbs</argument>
               <argument>../../format/File.fbs</argument>
             </arguments>

From a5f28617499a63ec44886bed35253f790e3674e1 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 8 Sep 2016 22:58:37 -0400
Subject: [PATCH 0131/1644] ARROW-286: Build thirdparty dependencies in
 parallel

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #133 from xhochy/ARROW-286 and squashes the following commits:

cb5a990 [Uwe L. Korn] ARROW-286: Build thirdparty dependencies in parallel
---
 cpp/thirdparty/build_thirdparty.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
index f1738ff748299..6cc776d09042a 100755
--- a/cpp/thirdparty/build_thirdparty.sh
+++ b/cpp/thirdparty/build_thirdparty.sh
@@ -62,7 +62,7 @@ if [ -n "$F_ALL" -o -n "$F_GTEST" ]; then
     CXXFLAGS=-fPIC cmake . || { echo "cmake $GOOGLETEST_ERROR"; exit  1; }
   fi
 
-  make VERBOSE=1 || { echo "Make $GOOGLETEST_ERROR" ; exit  1; }
+  make -j$PARALLEL VERBOSE=1 || { echo "Make $GOOGLETEST_ERROR" ; exit  1; }
 fi
 
 # build google benchmark
@@ -76,7 +76,7 @@ if [ -n "$F_ALL" -o -n "$F_GBENCHMARK" ]; then
   fi
   cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_CXX_FLAGS="-fPIC $CMAKE_CXX_FLAGS" . || { echo "cmake $GBENCHMARK_ERROR" ; exit 1; }
 
-  make VERBOSE=1 install || { echo "make $GBENCHMARK_ERROR" ; exit 1; }
+  make -j$PARALLEL VERBOSE=1 install || { echo "make $GBENCHMARK_ERROR" ; exit 1; }
 fi
 
 FLATBUFFERS_ERROR="failed for flatbuffers"

From 077c72bc6adf07c5311785596cb03088ae11ae5e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 9 Sep 2016 00:02:35 -0400
Subject: [PATCH 0132/1644] ARROW-256: [Format] Add a version number to the
 IPC/RPC metadata

See "Schema evolution examples" in https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html. In the future, if we need to add some other message types (like `RecordBatchV2`), then this should permit this without too much trouble.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #125 from wesm/ARROW-256 and squashes the following commits:

60ee5c0 [Wes McKinney] Rename current version to V1_SNAPSHOT to reflect changing nature
bab2749 [Wes McKinney] Add a version number / enum to the Message and File metadata
---
 cpp/src/arrow/ipc/metadata-internal.cc | 3 ++-
 cpp/src/arrow/ipc/metadata-internal.h  | 3 +++
 format/File.fbs                        | 1 +
 format/Message.fbs                     | 5 +++++
 4 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 1c15218c0ba12..8cc902c2967da 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -295,7 +295,8 @@ Status WriteDataHeader(int32_t length, int64_t body_length,
 }
 
 Status MessageBuilder::Finish() {
-  auto message = flatbuf::CreateMessage(fbb_, header_type_, header_, body_length_);
+  auto message = flatbuf::CreateMessage(fbb_, kMetadataVersion,
+      header_type_, header_, body_length_);
   fbb_.Finish(message);
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index 5faa8c947b55d..db9a83f6a8dfb 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -37,6 +37,9 @@ class Status;
 
 namespace ipc {
 
+static constexpr flatbuf::MetadataVersion kMetadataVersion =
+  flatbuf::MetadataVersion_V1_SNAPSHOT;
+
 Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out);
 
 class MessageBuilder {
diff --git a/format/File.fbs b/format/File.fbs
index f7ad1e1594a91..a29bbc694bc13 100644
--- a/format/File.fbs
+++ b/format/File.fbs
@@ -7,6 +7,7 @@ namespace org.apache.arrow.flatbuf;
 ///
 
 table Footer {
+  version: org.apache.arrow.flatbuf.MetadataVersion;
 
   schema: org.apache.arrow.flatbuf.Schema;
 
diff --git a/format/Message.fbs b/format/Message.fbs
index 78bdaeb35f5a5..657904a7032a5 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -1,5 +1,9 @@
 namespace org.apache.arrow.flatbuf;
 
+enum MetadataVersion:short {
+  V1_SNAPSHOT
+}
+
 /// ----------------------------------------------------------------------
 /// Logical types and their metadata (if any)
 ///
@@ -237,6 +241,7 @@ union MessageHeader {
 }
 
 table Message {
+  version: org.apache.arrow.flatbuf.MetadataVersion;
   header: MessageHeader;
   bodyLength: long;
 }

From 6b8abb4402ff1f39fc5944a7df6e3b4755691d87 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 12 Sep 2016 23:15:10 -0400
Subject: [PATCH 0133/1644] ARROW-289: Install test-util.h

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #135 from xhochy/arrow-289 and squashes the following commits:

5e4aadf [Uwe L. Korn] ARROW-289: Install test-util.h
---
 cpp/src/arrow/CMakeLists.txt      |  1 +
 cpp/src/arrow/test-util.h         | 12 ++++++------
 cpp/src/arrow/util/CMakeLists.txt |  1 +
 cpp/src/arrow/util/bit-util.h     |  4 +++-
 4 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 2d42edcfbd499..a9b2feca28cb7 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -24,6 +24,7 @@ install(FILES
   schema.h
   table.h
   type.h
+  test-util.h
   DESTINATION include/arrow)
 
 #######################################
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 055dac7444488..e632ffb1d892d 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -40,22 +40,22 @@
 
 #define ASSERT_RAISES(ENUM, expr)                  \
   do {                                             \
-    Status s = (expr);                             \
+    ::arrow::Status s = (expr);                    \
     if (!s.Is##ENUM()) { FAIL() << s.ToString(); } \
   } while (0)
 
 #define ASSERT_OK(expr)                      \
   do {                                       \
-    Status s = (expr);                       \
+    ::arrow::Status s = (expr);              \
     if (!s.ok()) { FAIL() << s.ToString(); } \
   } while (0)
 
 #define ASSERT_OK_NO_THROW(expr) ASSERT_NO_THROW(ASSERT_OK(expr))
 
-#define EXPECT_OK(expr)  \
-  do {                   \
-    Status s = (expr);   \
-    EXPECT_TRUE(s.ok()); \
+#define EXPECT_OK(expr)         \
+  do {                          \
+    ::arrow::Status s = (expr); \
+    EXPECT_TRUE(s.ok());        \
   } while (0)
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 13c0d7514feef..fd23c1aa3b8b2 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -26,6 +26,7 @@ install(FILES
   logging.h
   macros.h
   memory-pool.h
+  random.h
   status.h
   visibility.h
   DESTINATION include/arrow/util)
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index a6c8dd904d8e0..873a1959865f5 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -22,6 +22,8 @@
 #include <memory>
 #include <vector>
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Buffer;
@@ -76,7 +78,7 @@ static inline bool is_multiple_of_64(int64_t n) {
 }
 
 void bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits);
-Status bytes_to_bits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
+ARROW_EXPORT Status bytes_to_bits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
 }  // namespace util
 

From 6f99156c3bb01329e33f74a57d9aaff1ed8304bc Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 13 Sep 2016 11:13:54 -0700
Subject: [PATCH 0134/1644] ARROW-287: Make nullable vectors use a BitVecor
 instead of UInt1Vector for bits

Author: Julien Le Dem <julien@dremio.com>

Closes #134 from julienledem/bits and squashes the following commits:

d4e5084 [Julien Le Dem] add nullable vector test that verifies Bit based buffers
15fde9d [Julien Le Dem] ARROW-287: Make nullable vectors use a BitVecor instead of UInt1Vector for bits
---
 .../templates/NullableValueVectors.java       |  6 +-
 .../org/apache/arrow/vector/BitVector.java    | 16 ++++-
 .../arrow/vector/complex/ListVector.java      |  6 +-
 .../vector/complex/NullableMapVector.java     |  8 +--
 .../apache/arrow/vector/TestValueVector.java  | 67 +++++++++++++++++--
 5 files changed, 87 insertions(+), 16 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index bb2c00121605c..486cfeefc7a3b 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -53,7 +53,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   private final String valuesField = "$values$";
   private final Field field;
 
-  final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
+  final BitVector bits = new BitVector(bitsField, allocator);
   final ${valuesName} values;
 
   private final Mutator mutator;
@@ -446,7 +446,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vec
   }
 
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
-    final UInt1Vector.Accessor bAccessor = bits.getAccessor();
+    final BitVector.Accessor bAccessor = bits.getAccessor();
     final ${valuesName}.Accessor vAccessor = values.getAccessor();
 
     /**
@@ -545,7 +545,7 @@ public void setIndexDefined(int index){
     public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
       setCount++;
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
-      final UInt1Vector.Mutator bitsMutator = bits.getMutator();
+      final BitVector.Mutator bitsMutator = bits.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
         valuesMutator.set(i, emptyByteArray);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index fee6e9cdef73d..c12db5045c2db 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -17,8 +17,6 @@
  */
 package org.apache.arrow.vector;
 
-import io.netty.buffer.ArrowBuf;
-
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.reader.FieldReader;
@@ -29,6 +27,8 @@
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
+import io.netty.buffer.ArrowBuf;
+
 /**
  * Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start
  * of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0'
@@ -435,6 +435,18 @@ public final void generateTestData(int values) {
       setValueCount(values);
     }
 
+    public void generateTestDataAlt(int size) {
+      setValueCount(size);
+      boolean even = true;
+      final int valueCount = getAccessor().getValueCount();
+      for(int i = 0; i < valueCount; i++, even = !even) {
+        if(even){
+          set(i, (byte) 1);
+        }else{
+          set(i, (byte) 0);
+        }
+      }
+    }
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 2984c362514fc..dd99c734f7ff8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -28,9 +28,9 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseDataValueVector;
+import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.BufferBacked;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.UInt1Vector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
@@ -55,7 +55,7 @@
 public class ListVector extends BaseRepeatedValueVector implements FieldVector {
 
   final UInt4Vector offsets;
-  final UInt1Vector bits;
+  final BitVector bits;
   private final List<BufferBacked> innerVectors;
   private Mutator mutator = new Mutator();
   private Accessor accessor = new Accessor();
@@ -65,7 +65,7 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector {
 
   public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator);
-    this.bits = new UInt1Vector("$bits$", allocator);
+    this.bits = new BitVector("$bits$", allocator);
     this.offsets = getOffsetVector();
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
     this.writer = new UnionListWriter(this);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 6b257c095d28e..8e1bbfabdc907 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -25,10 +25,10 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.BaseDataValueVector;
+import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.BufferBacked;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.NullableVectorDefinitionSetter;
-import org.apache.arrow.vector.UInt1Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
@@ -45,7 +45,7 @@ public class NullableMapVector extends MapVector implements FieldVector {
 
   private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
 
-  protected final UInt1Vector bits;
+  protected final BitVector bits;
 
   private final List<BufferBacked> innerVectors;
 
@@ -54,7 +54,7 @@ public class NullableMapVector extends MapVector implements FieldVector {
 
   public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, checkNotNull(allocator), callBack);
-    this.bits = new UInt1Vector("$bits$", allocator);
+    this.bits = new BitVector("$bits$", allocator);
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits));
     this.accessor = new Accessor();
     this.mutator = new Mutator();
@@ -186,7 +186,7 @@ public boolean allocateNewSafe() {
     return success;
   }
   public final class Accessor extends MapVector.Accessor  {
-    final UInt1Vector.Accessor bAccessor = bits.getAccessor();
+    final BitVector.Accessor bAccessor = bits.getAccessor();
 
     @Override
     public Object getObject(int index) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 21cdc4f4d8d3b..124452e96ee42 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -17,17 +17,23 @@
  */
 package org.apache.arrow.vector;
 
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.charset.Charset;
+import java.util.List;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.schema.TypeLayout;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.nio.charset.Charset;
-
-import static org.junit.Assert.*;
+import io.netty.buffer.ArrowBuf;
 
 
 public class TestValueVector {
@@ -223,6 +229,59 @@ public void testNullableFloat() {
     }
   }
 
+  @Test
+  public void testNullableInt() {
+    // Create a new value vector for 1024 integers
+    try (final NullableIntVector vector = (NullableIntVector) MinorType.INT.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      final NullableIntVector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024);
+
+      // Put and set a few values.
+      m.set(0, 1);
+      m.set(1, 2);
+      m.set(100, 3);
+      m.set(1022, 4);
+      m.set(1023, 5);
+
+      m.setValueCount(1024);
+
+      final NullableIntVector.Accessor accessor = vector.getAccessor();
+      assertEquals(1, accessor.get(0));
+      assertEquals(2, accessor.get(1));
+      assertEquals(3, accessor.get(100));
+      assertEquals(4, accessor.get(1022));
+      assertEquals(5, accessor.get(1023));
+
+      // Ensure null values.
+      assertTrue(vector.getAccessor().isNull(3));
+
+      Field field = vector.getField();
+      TypeLayout typeLayout = field.getTypeLayout();
+
+      List<ArrowBuf> buffers = vector.getFieldBuffers();
+
+      assertEquals(2, typeLayout.getVectors().size());
+      assertEquals(2, buffers.size());
+
+      ArrowBuf validityVectorBuf = buffers.get(0);
+      assertEquals(128, validityVectorBuf.readableBytes());
+      assertEquals(3, validityVectorBuf.getByte(0)); // 1st and second bit defined
+      for (int i = 1; i < 12; i++) {
+        assertEquals(0, validityVectorBuf.getByte(i)); // nothing defined until 100
+      }
+      assertEquals(16, validityVectorBuf.getByte(12)); // 100th bit is defined (12 * 8 + 4)
+      for (int i = 13; i < 127; i++) {
+        assertEquals(0, validityVectorBuf.getByte(i)); // nothing defined between 100th and 1022nd
+      }
+      assertEquals(-64, validityVectorBuf.getByte(127)); // 1022nd and 1023rd bit defined
+
+      vector.allocateNew(2048);
+      // vector has been erased
+      assertTrue(vector.getAccessor().isNull(0));
+    }
+  }
+
+
   @Test
   public void testBitVector() {
     // Create a new value vector for 1024 integers

From 3487c2f0cdc2297a80ba3525c192745313b3da48 Mon Sep 17 00:00:00 2001
From: adeneche <adeneche@dremio.com>
Date: Wed, 14 Sep 2016 14:46:27 -0700
Subject: [PATCH 0135/1644] ARROW-292: [Java] Upgrade Netty to 4.0.41

this closes #137
---
 .../main/java/io/netty/buffer/ArrowBuf.java   |  2 +-
 .../netty/buffer/PooledByteBufAllocatorL.java |  2 +-
 .../buffer/UnsafeDirectLittleEndian.java      | 30 +++++++++++++++----
 java/pom.xml                                  |  2 +-
 4 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index d10f00247e6ee..b7a268a00706c 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -452,7 +452,7 @@ public String toString(int index, int length, Charset charset) {
       return "";
     }
 
-    return ByteBufUtil.decodeString(nioBuffer(index, length), charset);
+    return ByteBufUtil.decodeString(this, index, length, charset);
   }
 
   @Override
diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index 0b6e3f7f8392d..f6feb65cccd09 100644
--- a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -145,7 +145,7 @@ public boolean matches(String name, Metric metric) {
     }
 
     private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCapacity) {
-      PoolThreadCache cache = threadCache.get();
+      PoolThreadCache cache = threadCache();
       PoolArena<ByteBuffer> directArena = cache.directArena;
 
       if (directArena != null) {
diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
index a94c6d1988399..dc93602100e9c 100644
--- a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -20,6 +20,9 @@
 
 import io.netty.util.internal.PlatformDependent;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.ByteOrder;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -93,11 +96,6 @@ public ByteBuf slice(int index, int length) {
     return new SlicedByteBuf(this, index, length);
   }
 
-  @Override
-  public ByteOrder order() {
-    return ByteOrder.LITTLE_ENDIAN;
-  }
-
   @Override
   public ByteBuf order(ByteOrder endianness) {
     return this;
@@ -254,6 +252,28 @@ public boolean release(int decrement) {
     return released;
   }
 
+  @Override
+  public int setBytes(int index, InputStream in, int length) throws IOException {
+    wrapped.checkIndex(index, length);
+    byte[] tmp = new byte[length];
+    int readBytes = in.read(tmp);
+    if (readBytes > 0) {
+      PlatformDependent.copyMemory(tmp, 0, addr(index), readBytes);
+    }
+    return readBytes;
+  }
+
+  @Override
+  public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
+    wrapped.checkIndex(index, length);
+    if (length != 0) {
+      byte[] tmp = new byte[length];
+      PlatformDependent.copyMemory(addr(index), tmp, 0, length);
+      out.write(tmp);
+    }
+    return this;
+  }
+
   @Override
   public int hashCode() {
     return System.identityHashCode(this);
diff --git a/java/pom.xml b/java/pom.xml
index 8eb25af7545f4..a8e24ed054cd5 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -395,7 +395,7 @@
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-handler</artifactId>
-      <version>4.0.27.Final</version>
+      <version>4.0.41.Final</version>
     </dependency>
 
     <dependency>

From 17e90e1d88266ea224244647831f49d5bd1dac72 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 16 Sep 2016 16:01:17 -0700
Subject: [PATCH 0136/1644] ARROW-290: Specialize alloc() in ArrowBuf

Author: Julien Le Dem <julien@dremio.com>

Closes #136 from julienledem/alloc and squashes the following commits:

a19d16f [Julien Le Dem] ARROW-290: Specialize alloc() in ArrowBuf
---
 .../src/main/java/io/netty/buffer/ArrowBuf.java       |  9 +++++----
 .../io/netty/buffer/UnsafeDirectLittleEndian.java     |  2 ++
 .../apache/arrow/memory/ArrowByteBufAllocator.java    |  4 ++++
 .../java/org/apache/arrow/memory/BaseAllocator.java   | 11 +++++------
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index b7a268a00706c..a5989c1518def 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -29,6 +29,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.arrow.memory.AllocationManager.BufferLedger;
+import org.apache.arrow.memory.ArrowByteBufAllocator;
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BaseAllocator.Verbosity;
 import org.apache.arrow.memory.BoundsChecking;
@@ -52,7 +53,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
   private final int offset;
   private final BufferLedger ledger;
   private final BufferManager bufManager;
-  private final ByteBufAllocator alloc;
+  private final ArrowByteBufAllocator alloc;
   private final boolean isEmpty;
   private volatile int length;
   private final HistoricalLog historicalLog = BaseAllocator.DEBUG ?
@@ -63,7 +64,7 @@ public ArrowBuf(
       final BufferLedger ledger,
       final UnsafeDirectLittleEndian byteBuf,
       final BufferManager manager,
-      final ByteBufAllocator alloc,
+      final ArrowByteBufAllocator alloc,
       final int offset,
       final int length,
       boolean isEmpty) {
@@ -297,8 +298,8 @@ public synchronized ArrowBuf capacity(int newCapacity) {
   }
 
   @Override
-  public ByteBufAllocator alloc() {
-    return udle.alloc();
+  public ArrowByteBufAllocator alloc() {
+    return alloc;
   }
 
   @Override
diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
index dc93602100e9c..023a6a2892b80 100644
--- a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -26,6 +26,8 @@
 import java.nio.ByteOrder;
 import java.util.concurrent.atomic.AtomicLong;
 
+import io.netty.util.internal.PlatformDependent;
+
 /**
  * The underlying class we use for little-endian access to memory. Is used underneath ArrowBufs to abstract away the
  * Netty classes and underlying Netty memory management.
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
index f3f72fa57c33a..5dc5ac397bd93 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
@@ -39,6 +39,10 @@ public ArrowByteBufAllocator(BufferAllocator allocator) {
     this.allocator = allocator;
   }
 
+  public BufferAllocator unwrap() {
+    return allocator;
+  }
+
   @Override
   public ByteBuf buffer() {
     return buffer(DEFAULT_BUFFER_SIZE);
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index f1503c902d0be..dbb0705045c35 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -17,10 +17,6 @@
  */
 package org.apache.arrow.memory;
 
-import io.netty.buffer.ArrowBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.UnsafeDirectLittleEndian;
-
 import java.util.Arrays;
 import java.util.IdentityHashMap;
 import java.util.Set;
@@ -33,6 +29,9 @@
 
 import com.google.common.base.Preconditions;
 
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.UnsafeDirectLittleEndian;
+
 public abstract class BaseAllocator extends Accountant implements BufferAllocator {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseAllocator.class);
 
@@ -47,7 +46,7 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato
   private final Object DEBUG_LOCK = DEBUG ? new Object() : null;
 
   private final BaseAllocator parentAllocator;
-  private final ByteBufAllocator thisAsByteBufAllocator;
+  private final ArrowByteBufAllocator thisAsByteBufAllocator;
   private final IdentityHashMap<BaseAllocator, Object> childAllocators;
   private final ArrowBuf empty;
 
@@ -247,7 +246,7 @@ private ArrowBuf bufferWithoutReservation(final int size, BufferManager bufferMa
   }
 
   @Override
-  public ByteBufAllocator getAsByteBufAllocator() {
+  public ArrowByteBufAllocator getAsByteBufAllocator() {
     return thisAsByteBufAllocator;
   }
 

From 559b865226ec0f5d78e87957c2ff0f7711bec9a8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 18 Sep 2016 16:01:58 -0400
Subject: [PATCH 0137/1644] ARROW-280: [C++] Refactor IPC / memory map IO to
 use common arrow_io interfaces. Create arrow_ipc leaf library

Several things here

* Clean up IO interface class structure to be able to indicate precise characteristics of an implementation
* Make the IPC reader/writer use more generic interfaces -- writing only needs an output stream, reading only needs a random access reader. This will unblock ARROW-267
* Create a separate arrow_ipc shared library

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #138 from wesm/ARROW-280 and squashes the following commits:

6a59eb6 [Wes McKinney] * Restructure IO interfaces to accommodate more configurations. * Refactor memory mapped IO interfaces to be in line with other arrow::io   classes. * Split arrow_ipc into a leaf library * Refactor pyarrow and arrow_parquet to suit. Move BufferReader to   arrow_io. Pyarrow parquet tests currently segfault
---
 cpp/CMakeLists.txt                            |   6 -
 cpp/src/arrow/io/CMakeLists.txt               |  11 +-
 cpp/src/arrow/io/hdfs.cc                      |  35 ++-
 cpp/src/arrow/io/hdfs.h                       |  29 +-
 cpp/src/arrow/io/interfaces.h                 |  71 ++++-
 .../io/{hdfs-io-test.cc => io-hdfs-test.cc}   |   2 +-
 .../io-memory-test.cc}                        |  50 ++--
 cpp/src/arrow/io/libhdfs_shim.cc              |   3 +-
 cpp/src/arrow/io/memory.cc                    | 262 ++++++++++++++++++
 cpp/src/arrow/io/memory.h                     | 130 +++++++++
 cpp/src/arrow/io/test-common.h                |  63 +++++
 cpp/src/arrow/ipc/CMakeLists.txt              |  58 +++-
 cpp/src/arrow/ipc/adapter.cc                  |  61 ++--
 cpp/src/arrow/ipc/adapter.h                   |  39 +--
 cpp/src/arrow/ipc/ipc-adapter-test.cc         |  33 ++-
 cpp/src/arrow/ipc/memory.cc                   | 182 ------------
 cpp/src/arrow/ipc/memory.h                    | 150 ----------
 cpp/src/arrow/ipc/metadata-internal.cc        |   9 +-
 cpp/src/arrow/ipc/metadata-internal.h         |   2 +-
 cpp/src/arrow/ipc/metadata.h                  |  11 +-
 cpp/src/arrow/ipc/symbols.map                 |  18 ++
 cpp/src/arrow/ipc/test-common.h               |  25 --
 cpp/src/arrow/ipc/util.h                      |  56 ++++
 cpp/src/arrow/parquet/CMakeLists.txt          |   1 +
 cpp/src/arrow/parquet/io.cc                   |   4 +-
 cpp/src/arrow/parquet/io.h                    |   4 +-
 cpp/src/arrow/parquet/parquet-io-test.cc      |  51 +---
 cpp/src/arrow/parquet/parquet-schema-test.cc  |   3 +-
 cpp/src/arrow/parquet/reader.cc               |   8 +-
 cpp/src/arrow/parquet/reader.h                |   2 +-
 cpp/src/arrow/parquet/schema.cc               |   2 +-
 cpp/src/arrow/parquet/writer.cc               |   2 +-
 cpp/src/arrow/type.h                          |   4 +-
 cpp/src/arrow/util/memory-pool-test.cc        |   2 +-
 python/pyarrow/includes/libarrow_io.pxd       |  42 ++-
 python/pyarrow/includes/parquet.pxd           |  18 +-
 python/pyarrow/io.pxd                         |   7 +-
 python/pyarrow/io.pyx                         |  14 +-
 python/pyarrow/parquet.pyx                    |   6 +-
 39 files changed, 873 insertions(+), 603 deletions(-)
 rename cpp/src/arrow/io/{hdfs-io-test.cc => io-hdfs-test.cc} (99%)
 rename cpp/src/arrow/{ipc/ipc-memory-test.cc => io/io-memory-test.cc} (66%)
 create mode 100644 cpp/src/arrow/io/memory.cc
 create mode 100644 cpp/src/arrow/io/memory.h
 create mode 100644 cpp/src/arrow/io/test-common.h
 delete mode 100644 cpp/src/arrow/ipc/memory.cc
 delete mode 100644 cpp/src/arrow/ipc/memory.h
 create mode 100644 cpp/src/arrow/ipc/symbols.map
 create mode 100644 cpp/src/arrow/ipc/util.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index a39a752123155..be95dabf31897 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -626,12 +626,6 @@ set(ARROW_SRCS
   src/arrow/table.cc
   src/arrow/type.cc
 
-  # IPC / Shared memory library; to be turned into an optional component
-  src/arrow/ipc/adapter.cc
-  src/arrow/ipc/memory.cc
-  src/arrow/ipc/metadata.cc
-  src/arrow/ipc/metadata-internal.cc
-
   src/arrow/types/construct.cc
   src/arrow/types/decimal.cc
   src/arrow/types/json.cc
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index b8c0e138afb06..87e227ef80d80 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -20,6 +20,7 @@
 
 set(ARROW_IO_LINK_LIBS
   arrow_shared
+  dl
 )
 
 if (ARROW_BOOST_USE_SHARED)
@@ -37,6 +38,7 @@ set(ARROW_IO_TEST_LINK_LIBS
   ${ARROW_IO_PRIVATE_LINK_LIBS})
 
 set(ARROW_IO_SRCS
+  memory.cc
 )
 
 if(ARROW_HDFS)
@@ -71,8 +73,8 @@ if(ARROW_HDFS)
     ${ARROW_HDFS_SRCS}
     ${ARROW_IO_SRCS})
 
-  ADD_ARROW_TEST(hdfs-io-test)
-  ARROW_TEST_LINK_LIBRARIES(hdfs-io-test
+  ADD_ARROW_TEST(io-hdfs-test)
+  ARROW_TEST_LINK_LIBRARIES(io-hdfs-test
     ${ARROW_IO_TEST_LINK_LIBS})
 endif()
 
@@ -101,10 +103,15 @@ if (APPLE)
     INSTALL_NAME_DIR "@rpath")
 endif()
 
+ADD_ARROW_TEST(io-memory-test)
+ARROW_TEST_LINK_LIBRARIES(io-memory-test
+  ${ARROW_IO_TEST_LINK_LIBS})
+
 # Headers: top level
 install(FILES
   hdfs.h
   interfaces.h
+  memory.h
   DESTINATION include/arrow/io)
 
 install(TARGETS arrow_io
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 800c3edf4f31a..a6b4b2f3846b1 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -142,6 +142,15 @@ Status HdfsReadableFile::ReadAt(
   return impl_->ReadAt(position, nbytes, bytes_read, buffer);
 }
 
+Status HdfsReadableFile::ReadAt(
+    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  return Status::NotImplemented("Not yet implemented");
+}
+
+bool HdfsReadableFile::supports_zero_copy() const {
+  return false;
+}
+
 Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
   return impl_->Read(nbytes, bytes_read, buffer);
 }
@@ -162,9 +171,9 @@ Status HdfsReadableFile::Tell(int64_t* position) {
 // File writing
 
 // Private implementation for writeable-only files
-class HdfsWriteableFile::HdfsWriteableFileImpl : public HdfsAnyFileImpl {
+class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
  public:
-  HdfsWriteableFileImpl() {}
+  HdfsOutputStreamImpl() {}
 
   Status Close() {
     if (is_open_) {
@@ -185,29 +194,29 @@ class HdfsWriteableFile::HdfsWriteableFileImpl : public HdfsAnyFileImpl {
   }
 };
 
-HdfsWriteableFile::HdfsWriteableFile() {
-  impl_.reset(new HdfsWriteableFileImpl());
+HdfsOutputStream::HdfsOutputStream() {
+  impl_.reset(new HdfsOutputStreamImpl());
 }
 
-HdfsWriteableFile::~HdfsWriteableFile() {
+HdfsOutputStream::~HdfsOutputStream() {
   impl_->Close();
 }
 
-Status HdfsWriteableFile::Close() {
+Status HdfsOutputStream::Close() {
   return impl_->Close();
 }
 
-Status HdfsWriteableFile::Write(
+Status HdfsOutputStream::Write(
     const uint8_t* buffer, int64_t nbytes, int64_t* bytes_read) {
   return impl_->Write(buffer, nbytes, bytes_read);
 }
 
-Status HdfsWriteableFile::Write(const uint8_t* buffer, int64_t nbytes) {
+Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes) {
   int64_t bytes_written_dummy = 0;
   return Write(buffer, nbytes, &bytes_written_dummy);
 }
 
-Status HdfsWriteableFile::Tell(int64_t* position) {
+Status HdfsOutputStream::Tell(int64_t* position) {
   return impl_->Tell(position);
 }
 
@@ -347,7 +356,7 @@ class HdfsClient::HdfsClientImpl {
 
   Status OpenWriteable(const std::string& path, bool append, int32_t buffer_size,
       int16_t replication, int64_t default_block_size,
-      std::shared_ptr<HdfsWriteableFile>* file) {
+      std::shared_ptr<HdfsOutputStream>* file) {
     int flags = O_WRONLY;
     if (append) flags |= O_APPEND;
 
@@ -362,7 +371,7 @@ class HdfsClient::HdfsClientImpl {
     }
 
     // std::make_shared does not work with private ctors
-    *file = std::shared_ptr<HdfsWriteableFile>(new HdfsWriteableFile());
+    *file = std::shared_ptr<HdfsOutputStream>(new HdfsOutputStream());
     (*file)->impl_->set_members(path, fs_, handle);
 
     return Status::OK();
@@ -440,13 +449,13 @@ Status HdfsClient::OpenReadable(
 
 Status HdfsClient::OpenWriteable(const std::string& path, bool append,
     int32_t buffer_size, int16_t replication, int64_t default_block_size,
-    std::shared_ptr<HdfsWriteableFile>* file) {
+    std::shared_ptr<HdfsOutputStream>* file) {
   return impl_->OpenWriteable(
       path, append, buffer_size, replication, default_block_size, file);
 }
 
 Status HdfsClient::OpenWriteable(
-    const std::string& path, bool append, std::shared_ptr<HdfsWriteableFile>* file) {
+    const std::string& path, bool append, std::shared_ptr<HdfsOutputStream>* file) {
   return OpenWriteable(path, append, 0, 0, 0, file);
 }
 
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index b6449fcb88a75..39720cc17e422 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -29,13 +29,14 @@
 
 namespace arrow {
 
+class Buffer;
 class Status;
 
 namespace io {
 
 class HdfsClient;
 class HdfsReadableFile;
-class HdfsWriteableFile;
+class HdfsOutputStream;
 
 struct HdfsPathInfo {
   ObjectType::type kind;
@@ -139,14 +140,14 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   // @param default_block_size, 0 for default
   Status OpenWriteable(const std::string& path, bool append, int32_t buffer_size,
       int16_t replication, int64_t default_block_size,
-      std::shared_ptr<HdfsWriteableFile>* file);
+      std::shared_ptr<HdfsOutputStream>* file);
 
   Status OpenWriteable(
-      const std::string& path, bool append, std::shared_ptr<HdfsWriteableFile>* file);
+      const std::string& path, bool append, std::shared_ptr<HdfsOutputStream>* file);
 
  private:
   friend class HdfsReadableFile;
-  friend class HdfsWriteableFile;
+  friend class HdfsOutputStream;
 
   class ARROW_NO_EXPORT HdfsClientImpl;
   std::unique_ptr<HdfsClientImpl> impl_;
@@ -155,7 +156,7 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   DISALLOW_COPY_AND_ASSIGN(HdfsClient);
 };
 
-class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
+class ARROW_EXPORT HdfsReadableFile : public ReadableFileInterface {
  public:
   ~HdfsReadableFile();
 
@@ -166,6 +167,10 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
   Status ReadAt(
       int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
 
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
+  bool supports_zero_copy() const override;
+
   Status Seek(int64_t position) override;
   Status Tell(int64_t* position) override;
 
@@ -183,9 +188,11 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
   DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
 };
 
-class ARROW_EXPORT HdfsWriteableFile : public WriteableFile {
+// Naming this file OutputStream because it does not support seeking (like the
+// WriteableFile interface)
+class ARROW_EXPORT HdfsOutputStream : public OutputStream {
  public:
-  ~HdfsWriteableFile();
+  ~HdfsOutputStream();
 
   Status Close() override;
 
@@ -196,14 +203,14 @@ class ARROW_EXPORT HdfsWriteableFile : public WriteableFile {
   Status Tell(int64_t* position) override;
 
  private:
-  class ARROW_NO_EXPORT HdfsWriteableFileImpl;
-  std::unique_ptr<HdfsWriteableFileImpl> impl_;
+  class ARROW_NO_EXPORT HdfsOutputStreamImpl;
+  std::unique_ptr<HdfsOutputStreamImpl> impl_;
 
   friend class HdfsClient::HdfsClientImpl;
 
-  HdfsWriteableFile();
+  HdfsOutputStream();
 
-  DISALLOW_COPY_AND_ASSIGN(HdfsWriteableFile);
+  DISALLOW_COPY_AND_ASSIGN(HdfsOutputStream);
 };
 
 Status ARROW_EXPORT ConnectLibHdfs();
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index c21285253714e..fa34b43b2c920 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -21,8 +21,11 @@
 #include <cstdint>
 #include <memory>
 
+#include "arrow/util/macros.h"
+
 namespace arrow {
 
+class Buffer;
 class Status;
 
 namespace io {
@@ -40,30 +43,78 @@ class FileSystemClient {
   virtual ~FileSystemClient() {}
 };
 
-class FileBase {
+class FileInterface {
  public:
+  virtual ~FileInterface() {}
   virtual Status Close() = 0;
   virtual Status Tell(int64_t* position) = 0;
+
+  FileMode::type mode() const { return mode_; }
+
+ protected:
+  FileInterface() {}
+  FileMode::type mode_;
+
+  void set_mode(FileMode::type mode) { mode_ = mode; }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(FileInterface);
 };
 
-class ReadableFile : public FileBase {
+class Seekable {
  public:
-  virtual Status ReadAt(
-      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) = 0;
+  virtual Status Seek(int64_t position) = 0;
+};
 
-  virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) = 0;
+class Writeable {
+ public:
+  virtual Status Write(const uint8_t* data, int64_t nbytes) = 0;
+};
 
-  virtual Status GetSize(int64_t* size) = 0;
+class Readable {
+ public:
+  virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0;
+};
+
+class OutputStream : public FileInterface, public Writeable {
+ protected:
+  OutputStream() {}
 };
 
-class RandomAccessFile : public ReadableFile {
+class InputStream : public FileInterface, public Readable {
+ protected:
+  InputStream() {}
+};
+
+class ReadableFileInterface : public InputStream, public Seekable {
  public:
-  virtual Status Seek(int64_t position) = 0;
+  virtual Status ReadAt(
+      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0;
+
+  virtual Status GetSize(int64_t* size) = 0;
+
+  // Does not copy if not necessary
+  virtual Status ReadAt(
+      int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
+
+  virtual bool supports_zero_copy() const = 0;
+
+ protected:
+  ReadableFileInterface() { set_mode(FileMode::READ); }
 };
 
-class WriteableFile : public FileBase {
+class WriteableFileInterface : public OutputStream, public Seekable {
  public:
-  virtual Status Write(const uint8_t* buffer, int64_t nbytes) = 0;
+  virtual Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) = 0;
+
+ protected:
+  WriteableFileInterface() { set_mode(FileMode::READ); }
+};
+
+class ReadWriteFileInterface : public ReadableFileInterface,
+                               public WriteableFileInterface {
+ protected:
+  ReadWriteFileInterface() { ReadableFileInterface::set_mode(FileMode::READWRITE); }
 };
 
 }  // namespace io
diff --git a/cpp/src/arrow/io/hdfs-io-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
similarity index 99%
rename from cpp/src/arrow/io/hdfs-io-test.cc
rename to cpp/src/arrow/io/io-hdfs-test.cc
index e48a28142fa46..7901932dee676 100644
--- a/cpp/src/arrow/io/hdfs-io-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -49,7 +49,7 @@ class TestHdfsClient : public ::testing::Test {
   Status WriteDummyFile(const std::string& path, const uint8_t* buffer, int64_t size,
       bool append = false, int buffer_size = 0, int replication = 0,
       int default_block_size = 0) {
-    std::shared_ptr<HdfsWriteableFile> file;
+    std::shared_ptr<HdfsOutputStream> file;
     RETURN_NOT_OK(client_->OpenWriteable(
         path, append, buffer_size, replication, default_block_size, &file));
 
diff --git a/cpp/src/arrow/ipc/ipc-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
similarity index 66%
rename from cpp/src/arrow/ipc/ipc-memory-test.cc
rename to cpp/src/arrow/io/io-memory-test.cc
index a2dbd35728c49..6de35dab59b4f 100644
--- a/cpp/src/arrow/ipc/ipc-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -24,20 +24,20 @@
 
 #include "gtest/gtest.h"
 
-#include "arrow/ipc/memory.h"
-#include "arrow/ipc/test-common.h"
+#include "arrow/io/memory.h"
+#include "arrow/io/test-common.h"
 
 namespace arrow {
-namespace ipc {
+namespace io {
 
-class TestMemoryMappedSource : public ::testing::Test, public MemoryMapFixture {
+class TestMemoryMappedFile : public ::testing::Test, public MemoryMapFixture {
  public:
   void TearDown() { MemoryMapFixture::TearDown(); }
 };
 
-TEST_F(TestMemoryMappedSource, InvalidUsages) {}
+TEST_F(TestMemoryMappedFile, InvalidUsages) {}
 
-TEST_F(TestMemoryMappedSource, WriteRead) {
+TEST_F(TestMemoryMappedFile, WriteRead) {
   const int64_t buffer_size = 1024;
   std::vector<uint8_t> buffer(buffer_size);
 
@@ -48,14 +48,13 @@ TEST_F(TestMemoryMappedSource, WriteRead) {
   std::string path = "ipc-write-read-test";
   CreateFile(path, reps * buffer_size);
 
-  std::shared_ptr<MemoryMappedSource> result;
-  ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_WRITE, &result));
+  std::shared_ptr<MemoryMappedFile> result;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &result));
 
   int64_t position = 0;
-
   std::shared_ptr<Buffer> out_buffer;
   for (int i = 0; i < reps; ++i) {
-    ASSERT_OK(result->Write(position, buffer.data(), buffer_size));
+    ASSERT_OK(result->Write(buffer.data(), buffer_size));
     ASSERT_OK(result->ReadAt(position, buffer_size, &out_buffer));
 
     ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
@@ -64,7 +63,7 @@ TEST_F(TestMemoryMappedSource, WriteRead) {
   }
 }
 
-TEST_F(TestMemoryMappedSource, ReadOnly) {
+TEST_F(TestMemoryMappedFile, ReadOnly) {
   const int64_t buffer_size = 1024;
   std::vector<uint8_t> buffer(buffer_size);
 
@@ -75,19 +74,18 @@ TEST_F(TestMemoryMappedSource, ReadOnly) {
   std::string path = "ipc-read-only-test";
   CreateFile(path, reps * buffer_size);
 
-  std::shared_ptr<MemoryMappedSource> rwmmap;
-  ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_WRITE, &rwmmap));
+  std::shared_ptr<MemoryMappedFile> rwmmap;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &rwmmap));
 
   int64_t position = 0;
   for (int i = 0; i < reps; ++i) {
-    ASSERT_OK(rwmmap->Write(position, buffer.data(), buffer_size));
-
+    ASSERT_OK(rwmmap->Write(buffer.data(), buffer_size));
     position += buffer_size;
   }
   rwmmap->Close();
 
-  std::shared_ptr<MemoryMappedSource> rommap;
-  ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_ONLY, &rommap));
+  std::shared_ptr<MemoryMappedFile> rommap;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
 
   position = 0;
   std::shared_ptr<Buffer> out_buffer;
@@ -100,7 +98,7 @@ TEST_F(TestMemoryMappedSource, ReadOnly) {
   rommap->Close();
 }
 
-TEST_F(TestMemoryMappedSource, InvalidMode) {
+TEST_F(TestMemoryMappedFile, InvalidMode) {
   const int64_t buffer_size = 1024;
   std::vector<uint8_t> buffer(buffer_size);
 
@@ -109,19 +107,19 @@ TEST_F(TestMemoryMappedSource, InvalidMode) {
   std::string path = "ipc-invalid-mode-test";
   CreateFile(path, buffer_size);
 
-  std::shared_ptr<MemoryMappedSource> rommap;
-  ASSERT_OK(MemoryMappedSource::Open(path, MemorySource::READ_ONLY, &rommap));
+  std::shared_ptr<MemoryMappedFile> rommap;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
 
-  ASSERT_RAISES(IOError, rommap->Write(0, buffer.data(), buffer_size));
+  ASSERT_RAISES(IOError, rommap->Write(buffer.data(), buffer_size));
 }
 
-TEST_F(TestMemoryMappedSource, InvalidFile) {
+TEST_F(TestMemoryMappedFile, InvalidFile) {
   std::string non_existent_path = "invalid-file-name-asfd";
 
-  std::shared_ptr<MemoryMappedSource> result;
-  ASSERT_RAISES(IOError,
-      MemoryMappedSource::Open(non_existent_path, MemorySource::READ_ONLY, &result));
+  std::shared_ptr<MemoryMappedFile> result;
+  ASSERT_RAISES(
+      IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result));
 }
 
-}  // namespace ipc
+}  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
index 003570d4fdee6..0b805abf94c1b 100644
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -51,8 +51,7 @@ extern "C" {
 #include <type_traits>
 #include <vector>
 
-#include <boost/filesystem.hpp>        // NOLINT
-#include <boost/algorithm/string.hpp>  // NOLINT
+#include <boost/filesystem.hpp>  // NOLINT
 
 #include "arrow/util/status.h"
 #include "arrow/util/visibility.h"
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
new file mode 100644
index 0000000000000..1dd6c3a02304a
--- /dev/null
+++ b/cpp/src/arrow/io/memory.cc
@@ -0,0 +1,262 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/io/memory.h"
+
+#include <sys/mman.h>  // For memory-mapping
+
+#include <algorithm>
+#include <cerrno>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <sstream>
+#include <string>
+
+#include "arrow/io/interfaces.h"
+
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace io {
+
+// Implement MemoryMappedFile
+
+class MemoryMappedFile::MemoryMappedFileImpl {
+ public:
+  MemoryMappedFileImpl()
+      : file_(nullptr), is_open_(false), is_writable_(false), data_(nullptr) {}
+
+  ~MemoryMappedFileImpl() {
+    if (is_open_) {
+      munmap(data_, size_);
+      fclose(file_);
+    }
+  }
+
+  Status Open(const std::string& path, FileMode::type mode) {
+    if (is_open_) { return Status::IOError("A file is already open"); }
+
+    int prot_flags = PROT_READ;
+
+    if (mode == FileMode::READWRITE) {
+      file_ = fopen(path.c_str(), "r+b");
+      prot_flags |= PROT_WRITE;
+      is_writable_ = true;
+    } else {
+      file_ = fopen(path.c_str(), "rb");
+    }
+    if (file_ == nullptr) {
+      std::stringstream ss;
+      ss << "Unable to open file, errno: " << errno;
+      return Status::IOError(ss.str());
+    }
+
+    fseek(file_, 0L, SEEK_END);
+    if (ferror(file_)) { return Status::IOError("Unable to seek to end of file"); }
+    size_ = ftell(file_);
+
+    fseek(file_, 0L, SEEK_SET);
+    is_open_ = true;
+    position_ = 0;
+
+    void* result = mmap(nullptr, size_, prot_flags, MAP_SHARED, fileno(file_), 0);
+    if (result == MAP_FAILED) {
+      std::stringstream ss;
+      ss << "Memory mapping file failed, errno: " << errno;
+      return Status::IOError(ss.str());
+    }
+    data_ = reinterpret_cast<uint8_t*>(result);
+
+    return Status::OK();
+  }
+
+  int64_t size() const { return size_; }
+
+  Status Seek(int64_t position) {
+    if (position < 0 || position >= size_) {
+      return Status::Invalid("position is out of bounds");
+    }
+    position_ = position;
+    return Status::OK();
+  }
+
+  int64_t position() { return position_; }
+
+  void advance(int64_t nbytes) { position_ = std::min(size_, position_ + nbytes); }
+
+  uint8_t* data() { return data_; }
+
+  uint8_t* head() { return data_ + position_; }
+
+  bool writable() { return is_writable_; }
+
+  bool opened() { return is_open_; }
+
+ private:
+  FILE* file_;
+  int64_t position_;
+  int64_t size_;
+  bool is_open_;
+  bool is_writable_;
+
+  // The memory map
+  uint8_t* data_;
+};
+
+MemoryMappedFile::MemoryMappedFile(FileMode::type mode) {
+  ReadableFileInterface::set_mode(mode);
+}
+
+Status MemoryMappedFile::Open(const std::string& path, FileMode::type mode,
+    std::shared_ptr<MemoryMappedFile>* out) {
+  std::shared_ptr<MemoryMappedFile> result(new MemoryMappedFile(mode));
+
+  result->impl_.reset(new MemoryMappedFileImpl());
+  RETURN_NOT_OK(result->impl_->Open(path, mode));
+
+  *out = result;
+  return Status::OK();
+}
+
+Status MemoryMappedFile::GetSize(int64_t* size) {
+  *size = impl_->size();
+  return Status::OK();
+}
+
+Status MemoryMappedFile::Tell(int64_t* position) {
+  *position = impl_->position();
+  return Status::OK();
+}
+
+Status MemoryMappedFile::Seek(int64_t position) {
+  return impl_->Seek(position);
+}
+
+Status MemoryMappedFile::Close() {
+  // munmap handled in pimpl dtor
+  return Status::OK();
+}
+
+Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  nbytes = std::min(nbytes, impl_->size() - impl_->position());
+  std::memcpy(out, impl_->head(), nbytes);
+  *bytes_read = nbytes;
+  impl_->advance(nbytes);
+  return Status::OK();
+}
+
+Status MemoryMappedFile::ReadAt(
+    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  RETURN_NOT_OK(impl_->Seek(position));
+  return Read(nbytes, bytes_read, out);
+}
+
+Status MemoryMappedFile::ReadAt(
+    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  nbytes = std::min(nbytes, impl_->size() - position);
+  RETURN_NOT_OK(impl_->Seek(position));
+  *out = std::make_shared<Buffer>(impl_->head(), nbytes);
+  impl_->advance(nbytes);
+  return Status::OK();
+}
+
+bool MemoryMappedFile::supports_zero_copy() const {
+  return true;
+}
+
+Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+  if (!impl_->opened() || !impl_->writable()) {
+    return Status::IOError("Unable to write");
+  }
+
+  RETURN_NOT_OK(impl_->Seek(position));
+  return WriteInternal(data, nbytes);
+}
+
+Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
+  if (!impl_->opened() || !impl_->writable()) {
+    return Status::IOError("Unable to write");
+  }
+  if (nbytes + impl_->position() > impl_->size()) {
+    return Status::Invalid("Cannot write past end of memory map");
+  }
+
+  return WriteInternal(data, nbytes);
+}
+
+Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
+  memcpy(impl_->head(), data, nbytes);
+  impl_->advance(nbytes);
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// In-memory buffer reader
+
+Status BufferReader::Close() {
+  // no-op
+  return Status::OK();
+}
+
+Status BufferReader::Tell(int64_t* position) {
+  *position = position_;
+  return Status::OK();
+}
+
+Status BufferReader::ReadAt(
+    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, bytes_read, buffer);
+}
+
+Status BufferReader::ReadAt(
+    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  int64_t size = std::min(nbytes, buffer_size_ - position_);
+  *out = std::make_shared<Buffer>(buffer_ + position, size);
+  position_ += nbytes;
+  return Status::OK();
+}
+
+bool BufferReader::supports_zero_copy() const {
+  return true;
+}
+
+Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+  memcpy(buffer, buffer_ + position_, nbytes);
+  *bytes_read = std::min(nbytes, buffer_size_ - position_);
+  position_ += *bytes_read;
+  return Status::OK();
+}
+
+Status BufferReader::GetSize(int64_t* size) {
+  *size = buffer_size_;
+  return Status::OK();
+}
+
+Status BufferReader::Seek(int64_t position) {
+  if (position < 0 || position >= buffer_size_) {
+    return Status::IOError("position out of bounds");
+  }
+
+  position_ = position;
+  return Status::OK();
+}
+
+}  // namespace io
+}  // namespace arrow
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
new file mode 100644
index 0000000000000..6fe47c3b5157a
--- /dev/null
+++ b/cpp/src/arrow/io/memory.h
@@ -0,0 +1,130 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Public API for different memory sharing / IO mechanisms
+
+#ifndef ARROW_IO_MEMORY_H
+#define ARROW_IO_MEMORY_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "arrow/io/interfaces.h"
+
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Buffer;
+class MutableBuffer;
+class Status;
+
+namespace io {
+
+// An output stream that writes to a MutableBuffer, such as one obtained from a
+// memory map
+//
+// TODO(wesm): Implement this class
+class ARROW_EXPORT BufferOutputStream : public OutputStream {
+ public:
+  explicit BufferOutputStream(const std::shared_ptr<MutableBuffer>& buffer)
+      : buffer_(buffer) {}
+
+  // Implement the OutputStream interface
+  Status Close() override;
+  Status Tell(int64_t* position) override;
+  Status Write(const uint8_t* data, int64_t length) override;
+
+  // Returns the number of bytes remaining in the buffer
+  int64_t bytes_remaining() const;
+
+ private:
+  std::shared_ptr<MutableBuffer> buffer_;
+  int64_t capacity_;
+  int64_t position_;
+};
+
+// A memory source that uses memory-mapped files for memory interactions
+class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
+ public:
+  static Status Open(const std::string& path, FileMode::type mode,
+      std::shared_ptr<MemoryMappedFile>* out);
+
+  Status Close() override;
+
+  Status Tell(int64_t* position) override;
+
+  Status Seek(int64_t position) override;
+
+  // Required by ReadableFileInterface, copies memory into out
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+
+  Status ReadAt(
+      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+
+  // Read into a buffer, zero copy if possible
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
+  bool supports_zero_copy() const override;
+
+  Status Write(const uint8_t* data, int64_t nbytes) override;
+
+  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+
+  // @return: the size in bytes of the memory source
+  Status GetSize(int64_t* size) override;
+
+ private:
+  explicit MemoryMappedFile(FileMode::type mode);
+
+  Status WriteInternal(const uint8_t* data, int64_t nbytes);
+
+  // Hide the internal details of this class for now
+  class MemoryMappedFileImpl;
+  std::unique_ptr<MemoryMappedFileImpl> impl_;
+};
+
+class ARROW_EXPORT BufferReader : public ReadableFileInterface {
+ public:
+  BufferReader(const uint8_t* buffer, int buffer_size)
+      : buffer_(buffer), buffer_size_(buffer_size), position_(0) {}
+
+  Status Close() override;
+  Status Tell(int64_t* position) override;
+
+  Status ReadAt(
+      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status GetSize(int64_t* size) override;
+  Status Seek(int64_t position) override;
+
+  bool supports_zero_copy() const override;
+
+ private:
+  const uint8_t* buffer_;
+  int buffer_size_;
+  int64_t position_;
+};
+
+}  // namespace io
+}  // namespace arrow
+
+#endif  // ARROW_IO_MEMORY_H
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
new file mode 100644
index 0000000000000..1954d479e3930
--- /dev/null
+++ b/cpp/src/arrow/io/test-common.h
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IO_TEST_COMMON_H
+#define ARROW_IO_TEST_COMMON_H
+
+#include <algorithm>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/io/memory.h"
+#include "arrow/test-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
+
+namespace arrow {
+namespace io {
+
+class MemoryMapFixture {
+ public:
+  void TearDown() {
+    for (auto path : tmp_files_) {
+      std::remove(path.c_str());
+    }
+  }
+
+  void CreateFile(const std::string path, int64_t size) {
+    FILE* file = fopen(path.c_str(), "w");
+    if (file != nullptr) { tmp_files_.push_back(path); }
+    ftruncate(fileno(file), size);
+    fclose(file);
+  }
+
+  Status InitMemoryMap(
+      int64_t size, const std::string& path, std::shared_ptr<MemoryMappedFile>* mmap) {
+    CreateFile(path, size);
+    return MemoryMappedFile::Open(path, FileMode::READWRITE, mmap);
+  }
+
+ private:
+  std::vector<std::string> tmp_files_;
+};
+
+}  // namespace io
+}  // namespace arrow
+
+#endif  // ARROW_IO_TEST_COMMON_H
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 82634169ed925..e5553a6358115 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -19,16 +19,50 @@
 # arrow_ipc
 #######################################
 
-# Headers: top level
-install(FILES
-  adapter.h
-  metadata.h
-  memory.h
-  DESTINATION include/arrow/ipc)
+set(ARROW_IPC_LINK_LIBS
+  arrow_io
+  arrow_shared
+)
+
+set(ARROW_IPC_PRIVATE_LINK_LIBS
+  )
+
+set(ARROW_IPC_TEST_LINK_LIBS
+  arrow_ipc
+  ${ARROW_IPC_PRIVATE_LINK_LIBS})
+
+set(ARROW_IPC_SRCS
+  adapter.cc
+  metadata.cc
+  metadata-internal.cc
+)
+
+# TODO(wesm): SHARED and STATIC targets
+add_library(arrow_ipc SHARED
+  ${ARROW_IPC_SRCS}
+)
+target_link_libraries(arrow_ipc
+  LINK_PUBLIC ${ARROW_IPC_LINK_LIBS}
+  LINK_PRIVATE ${ARROW_IPC_PRIVATE_LINK_LIBS})
+
+if(NOT APPLE)
+  # Localize thirdparty symbols using a linker version script. This hides them
+  # from the client application. The OS X linker does not support the
+  # version-script option.
+  set(ARROW_IPC_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
+endif()
+
+SET_TARGET_PROPERTIES(arrow_ipc PROPERTIES
+  LINKER_LANGUAGE CXX
+  LINK_FLAGS "${ARROW_IPC_LINK_FLAGS}")
 
 ADD_ARROW_TEST(ipc-adapter-test)
-ADD_ARROW_TEST(ipc-memory-test)
+ARROW_TEST_LINK_LIBRARIES(ipc-adapter-test
+  ${ARROW_IPC_TEST_LINK_LIBS})
+
 ADD_ARROW_TEST(ipc-metadata-test)
+ARROW_TEST_LINK_LIBRARIES(ipc-metadata-test
+  ${ARROW_IPC_TEST_LINK_LIBS})
 
 # make clean will delete the generated file
 set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
@@ -49,3 +83,13 @@ add_custom_command(
 
 add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES})
 add_dependencies(arrow_objlib metadata_fbs)
+
+# Headers: top level
+install(FILES
+  adapter.h
+  metadata.h
+  DESTINATION include/arrow/ipc)
+
+install(TARGETS arrow_ipc
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib)
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 40d372bbd3520..0e101c8930395 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -24,9 +24,11 @@
 
 #include "arrow/array.h"
 #include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/memory.h"
 #include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/metadata.h"
+#include "arrow/ipc/util.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
@@ -144,10 +146,15 @@ class RowBatchWriter {
     return Status::OK();
   }
 
-  Status Write(MemorySource* dst, int64_t position, int64_t* data_header_offset) {
+  Status Write(io::OutputStream* dst, int64_t* data_header_offset) {
     // Write out all the buffers contiguously and compute the total size of the
     // memory payload
     int64_t offset = 0;
+
+    // Get the starting position
+    int64_t position;
+    RETURN_NOT_OK(dst->Tell(&position));
+
     for (size_t i = 0; i < buffers_.size(); ++i) {
       const Buffer* buffer = buffers_[i].get();
       int64_t size = 0;
@@ -171,7 +178,7 @@ class RowBatchWriter {
       buffer_meta_.push_back(flatbuf::Buffer(0, position + offset, size));
 
       if (size > 0) {
-        RETURN_NOT_OK(dst->Write(position + offset, buffer->data(), size));
+        RETURN_NOT_OK(dst->Write(buffer->data(), size));
         offset += size;
       }
     }
@@ -180,7 +187,7 @@ class RowBatchWriter {
     // memory, the data header can be converted to a flatbuffer and written out
     //
     // Note: The memory written here is prefixed by the size of the flatbuffer
-    // itself as an int32_t. On reading from a MemorySource, you will have to
+    // itself as an int32_t. On reading from a input, you will have to
     // determine the data header size then request a buffer such that you can
     // construct the flatbuffer data accessor object (see arrow::ipc::Message)
     std::shared_ptr<Buffer> data_header;
@@ -188,8 +195,7 @@ class RowBatchWriter {
         batch_->num_rows(), offset, field_nodes_, buffer_meta_, &data_header));
 
     // Write the data header at the end
-    RETURN_NOT_OK(
-        dst->Write(position + offset, data_header->data(), data_header->size()));
+    RETURN_NOT_OK(dst->Write(data_header->data(), data_header->size()));
 
     *data_header_offset = position + offset;
     return Status::OK();
@@ -199,9 +205,9 @@ class RowBatchWriter {
   Status GetTotalSize(int64_t* size) {
     // emulates the behavior of Write without actually writing
     int64_t data_header_offset;
-    MockMemorySource source(0);
-    RETURN_NOT_OK(Write(&source, 0, &data_header_offset));
-    *size = source.GetExtentBytesWritten();
+    MockOutputStream dst;
+    RETURN_NOT_OK(Write(&dst, &data_header_offset));
+    *size = dst.GetExtentBytesWritten();
     return Status::OK();
   }
 
@@ -214,12 +220,12 @@ class RowBatchWriter {
   int max_recursion_depth_;
 };
 
-Status WriteRowBatch(MemorySource* dst, const RowBatch* batch, int64_t position,
-    int64_t* header_offset, int max_recursion_depth) {
+Status WriteRowBatch(io::OutputStream* dst, const RowBatch* batch, int64_t* header_offset,
+    int max_recursion_depth) {
   DCHECK_GT(max_recursion_depth, 0);
   RowBatchWriter serializer(batch, max_recursion_depth);
   RETURN_NOT_OK(serializer.AssemblePayload());
-  return serializer.Write(dst, position, header_offset);
+  return serializer.Write(dst, header_offset);
 }
 
 Status GetRowBatchSize(const RowBatch* batch, int64_t* size) {
@@ -234,11 +240,11 @@ Status GetRowBatchSize(const RowBatch* batch, int64_t* size) {
 
 static constexpr int64_t INIT_METADATA_SIZE = 4096;
 
-class RowBatchReader::Impl {
+class RowBatchReader::RowBatchReaderImpl {
  public:
-  Impl(MemorySource* source, const std::shared_ptr<RecordBatchMessage>& metadata,
-      int max_recursion_depth)
-      : source_(source), metadata_(metadata), max_recursion_depth_(max_recursion_depth) {
+  RowBatchReaderImpl(io::ReadableFileInterface* file,
+      const std::shared_ptr<RecordBatchMessage>& metadata, int max_recursion_depth)
+      : file_(file), metadata_(metadata), max_recursion_depth_(max_recursion_depth) {
     num_buffers_ = metadata->num_buffers();
     num_flattened_fields_ = metadata->num_fields();
   }
@@ -339,10 +345,11 @@ class RowBatchReader::Impl {
   Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
     BufferMetadata metadata = metadata_->buffer(buffer_index);
     RETURN_NOT_OK(CheckMultipleOf64(metadata.length));
-    return source_->ReadAt(metadata.offset, metadata.length, out);
+    return file_->ReadAt(metadata.offset, metadata.length, out);
   }
 
-  MemorySource* source_;
+ private:
+  io::ReadableFileInterface* file_;
   std::shared_ptr<RecordBatchMessage> metadata_;
 
   int field_index_;
@@ -352,22 +359,22 @@ class RowBatchReader::Impl {
   int num_flattened_fields_;
 };
 
-Status RowBatchReader::Open(
-    MemorySource* source, int64_t position, std::shared_ptr<RowBatchReader>* out) {
-  return Open(source, position, kMaxIpcRecursionDepth, out);
+Status RowBatchReader::Open(io::ReadableFileInterface* file, int64_t position,
+    std::shared_ptr<RowBatchReader>* out) {
+  return Open(file, position, kMaxIpcRecursionDepth, out);
 }
 
-Status RowBatchReader::Open(MemorySource* source, int64_t position,
+Status RowBatchReader::Open(io::ReadableFileInterface* file, int64_t position,
     int max_recursion_depth, std::shared_ptr<RowBatchReader>* out) {
   std::shared_ptr<Buffer> metadata;
-  RETURN_NOT_OK(source->ReadAt(position, INIT_METADATA_SIZE, &metadata));
+  RETURN_NOT_OK(file->ReadAt(position, INIT_METADATA_SIZE, &metadata));
 
   int32_t metadata_size = *reinterpret_cast<const int32_t*>(metadata->data());
 
-  // We may not need to call source->ReadAt again
+  // We may not need to call ReadAt again
   if (metadata_size > static_cast<int>(INIT_METADATA_SIZE - sizeof(int32_t))) {
     // We don't have enough data, read the indicated metadata size.
-    RETURN_NOT_OK(source->ReadAt(position + sizeof(int32_t), metadata_size, &metadata));
+    RETURN_NOT_OK(file->ReadAt(position + sizeof(int32_t), metadata_size, &metadata));
   }
 
   // TODO(wesm): buffer slicing here would be better in case ReadAt returns
@@ -383,14 +390,14 @@ Status RowBatchReader::Open(MemorySource* source, int64_t position,
   std::shared_ptr<RecordBatchMessage> batch_meta = message->GetRecordBatch();
 
   std::shared_ptr<RowBatchReader> result(new RowBatchReader());
-  result->impl_.reset(new Impl(source, batch_meta, max_recursion_depth));
+  result->impl_.reset(new RowBatchReaderImpl(file, batch_meta, max_recursion_depth));
   *out = result;
 
   return Status::OK();
 }
 
 // Here the explicit destructor is required for compilers to be aware of
-// the complete information of RowBatchReader::Impl class
+// the complete information of RowBatchReader::RowBatchReaderImpl class
 RowBatchReader::~RowBatchReader() {}
 
 Status RowBatchReader::GetRowBatch(
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 6231af66aa180..215b46f8f65d4 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -33,9 +33,15 @@ class RowBatch;
 class Schema;
 class Status;
 
+namespace io {
+
+class ReadableFileInterface;
+class OutputStream;
+
+}  // namespace io
+
 namespace ipc {
 
-class MemorySource;
 class RecordBatchMessage;
 
 // ----------------------------------------------------------------------
@@ -43,22 +49,21 @@ class RecordBatchMessage;
 // We have trouble decoding flatbuffers if the size i > 70, so 64 is a nice round number
 // TODO(emkornfield) investigate this more
 constexpr int kMaxIpcRecursionDepth = 64;
-// Write the RowBatch (collection of equal-length Arrow arrays) to the memory
-// source at the indicated position
+
+// Write the RowBatch (collection of equal-length Arrow arrays) to the output
+// stream
 //
-// First, each of the memory buffers are written out end-to-end in starting at
-// the indicated position.
+// First, each of the memory buffers are written out end-to-end
 //
 // Then, this function writes the batch metadata as a flatbuffer (see
 // format/Message.fbs -- the RecordBatch message type) like so:
 //
 // <int32: metadata size> <uint8*: metadata>
 //
-// Finally, the memory offset to the start of the metadata / data header is
-// returned in an out-variable
-ARROW_EXPORT Status WriteRowBatch(MemorySource* dst, const RowBatch* batch,
-    int64_t position, int64_t* header_offset,
-    int max_recursion_depth = kMaxIpcRecursionDepth);
+// Finally, the absolute offset (relative to the start of the output stream) to
+// the start of the metadata / data header is returned in an out-variable
+ARROW_EXPORT Status WriteRowBatch(io::OutputStream* dst, const RowBatch* batch,
+    int64_t* header_offset, int max_recursion_depth = kMaxIpcRecursionDepth);
 
 // int64_t GetRowBatchMetadata(const RowBatch* batch);
 
@@ -68,16 +73,16 @@ ARROW_EXPORT Status WriteRowBatch(MemorySource* dst, const RowBatch* batch,
 ARROW_EXPORT Status GetRowBatchSize(const RowBatch* batch, int64_t* size);
 
 // ----------------------------------------------------------------------
-// "Read" path; does not copy data if the MemorySource does not
+// "Read" path; does not copy data if the input supports zero copy reads
 
 class ARROW_EXPORT RowBatchReader {
  public:
-  static Status Open(
-      MemorySource* source, int64_t position, std::shared_ptr<RowBatchReader>* out);
-
-  static Status Open(MemorySource* source, int64_t position, int max_recursion_depth,
+  static Status Open(io::ReadableFileInterface* file, int64_t position,
       std::shared_ptr<RowBatchReader>* out);
 
+  static Status Open(io::ReadableFileInterface* file, int64_t position,
+      int max_recursion_depth, std::shared_ptr<RowBatchReader>* out);
+
   virtual ~RowBatchReader();
 
   // Reassemble the row batch. A Schema is required to be able to construct the
@@ -86,8 +91,8 @@ class ARROW_EXPORT RowBatchReader {
       const std::shared_ptr<Schema>& schema, std::shared_ptr<RowBatch>* out);
 
  private:
-  class Impl;
-  std::unique_ptr<Impl> impl_;
+  class RowBatchReaderImpl;
+  std::unique_ptr<RowBatchReaderImpl> impl_;
 };
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 6740e0fc5acc2..ca4d0152b9060 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -24,9 +24,11 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/io/memory.h"
+#include "arrow/io/test-common.h"
 #include "arrow/ipc/adapter.h"
-#include "arrow/ipc/memory.h"
 #include "arrow/ipc/test-common.h"
+#include "arrow/ipc/util.h"
 
 #include "arrow/test-util.h"
 #include "arrow/types/list.h"
@@ -49,17 +51,18 @@ const auto LIST_LIST_INT32 = std::make_shared<ListType>(LIST_INT32);
 typedef Status MakeRowBatch(std::shared_ptr<RowBatch>* out);
 
 class TestWriteRowBatch : public ::testing::TestWithParam<MakeRowBatch*>,
-                          public MemoryMapFixture {
+                          public io::MemoryMapFixture {
  public:
   void SetUp() { pool_ = default_memory_pool(); }
-  void TearDown() { MemoryMapFixture::TearDown(); }
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
 
   Status RoundTripHelper(const RowBatch& batch, int memory_map_size,
       std::shared_ptr<RowBatch>* batch_result) {
     std::string path = "test-write-row-batch";
-    MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
+    io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
     int64_t header_location;
-    RETURN_NOT_OK(WriteRowBatch(mmap_.get(), &batch, 0, &header_location));
+
+    RETURN_NOT_OK(WriteRowBatch(mmap_.get(), &batch, &header_location));
 
     std::shared_ptr<RowBatchReader> reader;
     RETURN_NOT_OK(RowBatchReader::Open(mmap_.get(), header_location, &reader));
@@ -69,7 +72,7 @@ class TestWriteRowBatch : public ::testing::TestWithParam<MakeRowBatch*>,
   }
 
  protected:
-  std::shared_ptr<MemoryMappedSource> mmap_;
+  std::shared_ptr<io::MemoryMappedFile> mmap_;
   MemoryPool* pool_;
 };
 
@@ -276,12 +279,12 @@ INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRowBatch,
                             &MakeStringTypesRowBatch, &MakeStruct));
 
 void TestGetRowBatchSize(std::shared_ptr<RowBatch> batch) {
-  MockMemorySource mock_source(1 << 16);
+  ipc::MockOutputStream mock;
   int64_t mock_header_location = -1;
   int64_t size = -1;
-  ASSERT_OK(WriteRowBatch(&mock_source, batch.get(), 0, &mock_header_location));
+  ASSERT_OK(WriteRowBatch(&mock, batch.get(), &mock_header_location));
   ASSERT_OK(GetRowBatchSize(batch.get(), &size));
-  ASSERT_EQ(mock_source.GetExtentBytesWritten(), size);
+  ASSERT_EQ(mock.GetExtentBytesWritten(), size);
 }
 
 TEST_F(TestWriteRowBatch, IntegerGetRowBatchSize) {
@@ -303,10 +306,10 @@ TEST_F(TestWriteRowBatch, IntegerGetRowBatchSize) {
   TestGetRowBatchSize(batch);
 }
 
-class RecursionLimits : public ::testing::Test, public MemoryMapFixture {
+class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
  public:
   void SetUp() { pool_ = default_memory_pool(); }
-  void TearDown() { MemoryMapFixture::TearDown(); }
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
 
   Status WriteToMmap(int recursion_level, bool override_level,
       int64_t* header_out = nullptr, std::shared_ptr<Schema>* schema_out = nullptr) {
@@ -329,19 +332,19 @@ class RecursionLimits : public ::testing::Test, public MemoryMapFixture {
 
     std::string path = "test-write-past-max-recursion";
     const int memory_map_size = 1 << 16;
-    MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
+    io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
     int64_t header_location;
     int64_t* header_out_param = header_out == nullptr ? &header_location : header_out;
     if (override_level) {
       return WriteRowBatch(
-          mmap_.get(), batch.get(), 0, header_out_param, recursion_level + 1);
+          mmap_.get(), batch.get(), header_out_param, recursion_level + 1);
     } else {
-      return WriteRowBatch(mmap_.get(), batch.get(), 0, header_out_param);
+      return WriteRowBatch(mmap_.get(), batch.get(), header_out_param);
     }
   }
 
  protected:
-  std::shared_ptr<MemoryMappedSource> mmap_;
+  std::shared_ptr<io::MemoryMappedFile> mmap_;
   MemoryPool* pool_;
 };
 
diff --git a/cpp/src/arrow/ipc/memory.cc b/cpp/src/arrow/ipc/memory.cc
deleted file mode 100644
index a6c56d64f4aed..0000000000000
--- a/cpp/src/arrow/ipc/memory.cc
+++ /dev/null
@@ -1,182 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/ipc/memory.h"
-
-#include <sys/mman.h>  // For memory-mapping
-
-#include <algorithm>
-#include <cerrno>
-#include <cstdint>
-#include <cstdio>
-#include <cstring>
-#include <sstream>
-#include <string>
-
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
-
-namespace arrow {
-namespace ipc {
-
-MemorySource::MemorySource(AccessMode access_mode) : access_mode_(access_mode) {}
-
-MemorySource::~MemorySource() {}
-
-// Implement MemoryMappedSource
-
-class MemoryMappedSource::Impl {
- public:
-  Impl() : file_(nullptr), is_open_(false), is_writable_(false), data_(nullptr) {}
-
-  ~Impl() {
-    if (is_open_) {
-      munmap(data_, size_);
-      fclose(file_);
-    }
-  }
-
-  Status Open(const std::string& path, MemorySource::AccessMode mode) {
-    if (is_open_) { return Status::IOError("A file is already open"); }
-
-    int prot_flags = PROT_READ;
-
-    if (mode == MemorySource::READ_WRITE) {
-      file_ = fopen(path.c_str(), "r+b");
-      prot_flags |= PROT_WRITE;
-      is_writable_ = true;
-    } else {
-      file_ = fopen(path.c_str(), "rb");
-    }
-    if (file_ == nullptr) {
-      std::stringstream ss;
-      ss << "Unable to open file, errno: " << errno;
-      return Status::IOError(ss.str());
-    }
-
-    fseek(file_, 0L, SEEK_END);
-    if (ferror(file_)) { return Status::IOError("Unable to seek to end of file"); }
-    size_ = ftell(file_);
-
-    fseek(file_, 0L, SEEK_SET);
-    is_open_ = true;
-
-    void* result = mmap(nullptr, size_, prot_flags, MAP_SHARED, fileno(file_), 0);
-    if (result == MAP_FAILED) {
-      std::stringstream ss;
-      ss << "Memory mapping file failed, errno: " << errno;
-      return Status::IOError(ss.str());
-    }
-    data_ = reinterpret_cast<uint8_t*>(result);
-
-    return Status::OK();
-  }
-
-  int64_t size() const { return size_; }
-
-  uint8_t* data() { return data_; }
-
-  bool writable() { return is_writable_; }
-
-  bool opened() { return is_open_; }
-
- private:
-  FILE* file_;
-  int64_t size_;
-  bool is_open_;
-  bool is_writable_;
-
-  // The memory map
-  uint8_t* data_;
-};
-
-MemoryMappedSource::MemoryMappedSource(AccessMode access_mode)
-    : MemorySource(access_mode) {}
-
-Status MemoryMappedSource::Open(const std::string& path, AccessMode access_mode,
-    std::shared_ptr<MemoryMappedSource>* out) {
-  std::shared_ptr<MemoryMappedSource> result(new MemoryMappedSource(access_mode));
-
-  result->impl_.reset(new Impl());
-  RETURN_NOT_OK(result->impl_->Open(path, access_mode));
-
-  *out = result;
-  return Status::OK();
-}
-
-int64_t MemoryMappedSource::Size() const {
-  return impl_->size();
-}
-
-Status MemoryMappedSource::Close() {
-  // munmap handled in ::Impl dtor
-  return Status::OK();
-}
-
-Status MemoryMappedSource::ReadAt(
-    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  if (position < 0 || position >= impl_->size()) {
-    return Status::Invalid("position is out of bounds");
-  }
-
-  nbytes = std::min(nbytes, impl_->size() - position);
-  *out = std::make_shared<Buffer>(impl_->data() + position, nbytes);
-  return Status::OK();
-}
-
-Status MemoryMappedSource::Write(int64_t position, const uint8_t* data, int64_t nbytes) {
-  if (!impl_->opened() || !impl_->writable()) {
-    return Status::IOError("Unable to write");
-  }
-  if (position < 0 || position >= impl_->size()) {
-    return Status::Invalid("position is out of bounds");
-  }
-
-  // TODO(wesm): verify we are not writing past the end of the buffer
-  uint8_t* dst = impl_->data() + position;
-  memcpy(dst, data, nbytes);
-
-  return Status::OK();
-}
-
-MockMemorySource::MockMemorySource(int64_t size)
-    : size_(size), extent_bytes_written_(0) {}
-
-Status MockMemorySource::Close() {
-  return Status::OK();
-}
-
-Status MockMemorySource::ReadAt(
-    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  return Status::OK();
-}
-
-Status MockMemorySource::Write(int64_t position, const uint8_t* data, int64_t nbytes) {
-  extent_bytes_written_ = std::max(extent_bytes_written_, position + nbytes);
-  return Status::OK();
-}
-
-int64_t MockMemorySource::Size() const {
-  return size_;
-}
-
-int64_t MockMemorySource::GetExtentBytesWritten() const {
-  return extent_bytes_written_;
-}
-
-}  // namespace ipc
-}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/memory.h b/cpp/src/arrow/ipc/memory.h
deleted file mode 100644
index 377401d85c00a..0000000000000
--- a/cpp/src/arrow/ipc/memory.h
+++ /dev/null
@@ -1,150 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Public API for different interprocess memory sharing mechanisms
-
-#ifndef ARROW_IPC_MEMORY_H
-#define ARROW_IPC_MEMORY_H
-
-#include <cstdint>
-#include <memory>
-#include <string>
-
-#include "arrow/util/macros.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Buffer;
-class MutableBuffer;
-class Status;
-
-namespace ipc {
-
-// Abstract output stream
-class OutputStream {
- public:
-  virtual ~OutputStream() {}
-  // Close the output stream
-  virtual Status Close() = 0;
-
-  // The current position in the output stream
-  virtual int64_t Tell() const = 0;
-
-  // Write bytes to the stream
-  virtual Status Write(const uint8_t* data, int64_t length) = 0;
-};
-
-// An output stream that writes to a MutableBuffer, such as one obtained from a
-// memory map
-class BufferOutputStream : public OutputStream {
- public:
-  explicit BufferOutputStream(const std::shared_ptr<MutableBuffer>& buffer)
-      : buffer_(buffer) {}
-
-  // Implement the OutputStream interface
-  Status Close() override;
-  int64_t Tell() const override;
-  Status Write(const uint8_t* data, int64_t length) override;
-
-  // Returns the number of bytes remaining in the buffer
-  int64_t bytes_remaining() const;
-
- private:
-  std::shared_ptr<MutableBuffer> buffer_;
-  int64_t capacity_;
-  int64_t position_;
-};
-
-class ARROW_EXPORT MemorySource {
- public:
-  // Indicates the access permissions of the memory source
-  enum AccessMode { READ_ONLY, READ_WRITE };
-
-  virtual ~MemorySource();
-
-  // Retrieve a buffer of memory from the source of the indicates size and at
-  // the indicated location
-  // @returns: arrow::Status indicating success / failure. The buffer is set
-  // into the *out argument
-  virtual Status ReadAt(
-      int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
-
-  virtual Status Close() = 0;
-
-  virtual Status Write(int64_t position, const uint8_t* data, int64_t nbytes) = 0;
-
-  // @return: the size in bytes of the memory source
-  virtual int64_t Size() const = 0;
-
- protected:
-  explicit MemorySource(AccessMode access_mode = AccessMode::READ_WRITE);
-
-  AccessMode access_mode_;
-
- private:
-  DISALLOW_COPY_AND_ASSIGN(MemorySource);
-};
-
-// A memory source that uses memory-mapped files for memory interactions
-class ARROW_EXPORT MemoryMappedSource : public MemorySource {
- public:
-  static Status Open(const std::string& path, AccessMode access_mode,
-      std::shared_ptr<MemoryMappedSource>* out);
-
-  Status Close() override;
-
-  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
-
-  Status Write(int64_t position, const uint8_t* data, int64_t nbytes) override;
-
-  // @return: the size in bytes of the memory source
-  int64_t Size() const override;
-
- private:
-  explicit MemoryMappedSource(AccessMode access_mode);
-  // Hide the internal details of this class for now
-  class Impl;
-  std::unique_ptr<Impl> impl_;
-};
-
-// A MemorySource that tracks the size of allocations from a memory source
-class MockMemorySource : public MemorySource {
- public:
-  explicit MockMemorySource(int64_t size);
-
-  Status Close() override;
-
-  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
-
-  Status Write(int64_t position, const uint8_t* data, int64_t nbytes) override;
-
-  int64_t Size() const override;
-
-  // @return: the smallest number of bytes containing the modified region of the
-  // MockMemorySource
-  int64_t GetExtentBytesWritten() const;
-
- private:
-  int64_t size_;
-  int64_t extent_bytes_written_;
-};
-
-}  // namespace ipc
-}  // namespace arrow
-
-#endif  // ARROW_IPC_MEMORY_H
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 8cc902c2967da..05e9c7ad4d359 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -220,9 +220,8 @@ static Status FieldToFlatbuffer(
   auto fb_children = fbb.CreateVector(children);
 
   // TODO: produce the list of VectorTypes
-  *offset = flatbuf::CreateField(
-      fbb, fb_name, field->nullable, type_enum, type_data, field->dictionary,
-      fb_children);
+  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_data,
+      field->dictionary, fb_children);
 
   return Status::OK();
 }
@@ -295,8 +294,8 @@ Status WriteDataHeader(int32_t length, int64_t body_length,
 }
 
 Status MessageBuilder::Finish() {
-  auto message = flatbuf::CreateMessage(fbb_, kMetadataVersion,
-      header_type_, header_, body_length_);
+  auto message =
+      flatbuf::CreateMessage(fbb_, kMetadataVersion, header_type_, header_, body_length_);
   fbb_.Finish(message);
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index db9a83f6a8dfb..d38df840ba05e 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -38,7 +38,7 @@ class Status;
 namespace ipc {
 
 static constexpr flatbuf::MetadataVersion kMetadataVersion =
-  flatbuf::MetadataVersion_V1_SNAPSHOT;
+    flatbuf::MetadataVersion_V1_SNAPSHOT;
 
 Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out);
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 838a4a676ea35..d5ec53317e6f2 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -23,6 +23,8 @@
 #include <cstdint>
 #include <memory>
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Buffer;
@@ -36,6 +38,7 @@ namespace ipc {
 // Message read/write APIs
 
 // Serialize arrow::Schema as a Flatbuffer
+ARROW_EXPORT
 Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out);
 
 //----------------------------------------------------------------------
@@ -47,7 +50,7 @@ Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out);
 class Message;
 
 // Container for serialized Schema metadata contained in an IPC message
-class SchemaMessage {
+class ARROW_EXPORT SchemaMessage {
  public:
   // Accepts an opaque flatbuffer pointer
   SchemaMessage(const std::shared_ptr<Message>& message, const void* schema);
@@ -82,7 +85,7 @@ struct BufferMetadata {
 };
 
 // Container for serialized record batch metadata contained in an IPC message
-class RecordBatchMessage {
+class ARROW_EXPORT RecordBatchMessage {
  public:
   // Accepts an opaque flatbuffer pointer
   RecordBatchMessage(const std::shared_ptr<Message>& message, const void* batch_meta);
@@ -102,13 +105,13 @@ class RecordBatchMessage {
   std::unique_ptr<Impl> impl_;
 };
 
-class DictionaryBatchMessage {
+class ARROW_EXPORT DictionaryBatchMessage {
  public:
   int64_t id() const;
   std::unique_ptr<RecordBatchMessage> data() const;
 };
 
-class Message : public std::enable_shared_from_this<Message> {
+class ARROW_EXPORT Message : public std::enable_shared_from_this<Message> {
  public:
   enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH };
 
diff --git a/cpp/src/arrow/ipc/symbols.map b/cpp/src/arrow/ipc/symbols.map
new file mode 100644
index 0000000000000..b4ad98cd7f2d0
--- /dev/null
+++ b/cpp/src/arrow/ipc/symbols.map
@@ -0,0 +1,18 @@
+{
+  # Symbols marked as 'local' are not exported by the DSO and thus may not
+  # be used by client applications.
+  local:
+    # devtoolset / static-libstdc++ symbols
+    __cxa_*;
+
+    extern "C++" {
+      # boost
+      boost::*;
+
+      # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
+      # links c++11 symbols into binaries so that the result may be executed on
+      # a system with an older libstdc++ which doesn't include the necessary
+      # c++11 symbols.
+      std::*;
+    };
+};
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index e7dbb84d790a1..f6582fc883bdc 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -34,31 +34,6 @@
 namespace arrow {
 namespace ipc {
 
-class MemoryMapFixture {
- public:
-  void TearDown() {
-    for (auto path : tmp_files_) {
-      std::remove(path.c_str());
-    }
-  }
-
-  void CreateFile(const std::string path, int64_t size) {
-    FILE* file = fopen(path.c_str(), "w");
-    if (file != nullptr) { tmp_files_.push_back(path); }
-    ftruncate(fileno(file), size);
-    fclose(file);
-  }
-
-  Status InitMemoryMap(
-      int64_t size, const std::string& path, std::shared_ptr<MemoryMappedSource>* mmap) {
-    CreateFile(path, size);
-    return MemoryMappedSource::Open(path, MemorySource::READ_WRITE, mmap);
-  }
-
- private:
-  std::vector<std::string> tmp_files_;
-};
-
 Status MakeRandomInt32Array(
     int32_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* array) {
   std::shared_ptr<PoolBuffer> data;
diff --git a/cpp/src/arrow/ipc/util.h b/cpp/src/arrow/ipc/util.h
new file mode 100644
index 0000000000000..3f4001b21a91b
--- /dev/null
+++ b/cpp/src/arrow/ipc/util.h
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IPC_UTIL_H
+#define ARROW_IPC_UTIL_H
+
+#include <cstdint>
+
+#include "arrow/array.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+// A helper class to tracks the size of allocations
+class MockOutputStream : public io::OutputStream {
+ public:
+  MockOutputStream() : extent_bytes_written_(0) {}
+
+  Status Close() override { return Status::OK(); }
+
+  Status Write(const uint8_t* data, int64_t nbytes) override {
+    extent_bytes_written_ += nbytes;
+    return Status::OK();
+  }
+
+  Status Tell(int64_t* position) override {
+    *position = extent_bytes_written_;
+    return Status::OK();
+  }
+
+  int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
+
+ private:
+  int64_t extent_bytes_written_;
+};
+
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_UTIL_H
diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
index f2a90b71a4968..c400e14ea47f7 100644
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ b/cpp/src/arrow/parquet/CMakeLists.txt
@@ -27,6 +27,7 @@ set(PARQUET_SRCS
 
 set(PARQUET_LIBS
   arrow_shared
+  arrow_io
   parquet_shared
 )
 
diff --git a/cpp/src/arrow/parquet/io.cc b/cpp/src/arrow/parquet/io.cc
index b6fdd67d15b6c..a50d753f3054e 100644
--- a/cpp/src/arrow/parquet/io.cc
+++ b/cpp/src/arrow/parquet/io.cc
@@ -27,7 +27,7 @@
 #include "arrow/util/status.h"
 
 // To assist with readability
-using ArrowROFile = arrow::io::RandomAccessFile;
+using ArrowROFile = arrow::io::ReadableFileInterface;
 
 namespace arrow {
 namespace parquet {
@@ -58,7 +58,7 @@ void ParquetAllocator::Free(uint8_t* buffer, int64_t size) {
 ParquetReadSource::ParquetReadSource(ParquetAllocator* allocator)
     : file_(nullptr), allocator_(allocator) {}
 
-Status ParquetReadSource::Open(const std::shared_ptr<io::RandomAccessFile>& file) {
+Status ParquetReadSource::Open(const std::shared_ptr<io::ReadableFileInterface>& file) {
   int64_t file_size;
   RETURN_NOT_OK(file->GetSize(&file_size));
 
diff --git a/cpp/src/arrow/parquet/io.h b/cpp/src/arrow/parquet/io.h
index 1c59695c6c151..1734863acf1ea 100644
--- a/cpp/src/arrow/parquet/io.h
+++ b/cpp/src/arrow/parquet/io.h
@@ -62,7 +62,7 @@ class ARROW_EXPORT ParquetReadSource : public ::parquet::RandomAccessSource {
   explicit ParquetReadSource(ParquetAllocator* allocator);
 
   // We need to ask for the file size on opening the file, and this can fail
-  Status Open(const std::shared_ptr<io::RandomAccessFile>& file);
+  Status Open(const std::shared_ptr<io::ReadableFileInterface>& file);
 
   void Close() override;
   int64_t Tell() const override;
@@ -72,7 +72,7 @@ class ARROW_EXPORT ParquetReadSource : public ::parquet::RandomAccessSource {
 
  private:
   // An Arrow readable file of some kind
-  std::shared_ptr<io::RandomAccessFile> file_;
+  std::shared_ptr<io::ReadableFileInterface> file_;
 
   // The allocator is required for creating managed buffers
   ParquetAllocator* allocator_;
diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
index 6615457c483f5..208b3e867d374 100644
--- a/cpp/src/arrow/parquet/parquet-io-test.cc
+++ b/cpp/src/arrow/parquet/parquet-io-test.cc
@@ -22,6 +22,7 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/io/memory.h"
 #include "arrow/parquet/io.h"
 #include "arrow/test-util.h"
 #include "arrow/util/memory-pool.h"
@@ -96,61 +97,13 @@ TEST(TestParquetAllocator, CustomPool) {
 // ----------------------------------------------------------------------
 // Read source tests
 
-class BufferReader : public io::RandomAccessFile {
- public:
-  BufferReader(const uint8_t* buffer, int buffer_size)
-      : buffer_(buffer), buffer_size_(buffer_size), position_(0) {}
-
-  Status Close() override {
-    // no-op
-    return Status::OK();
-  }
-
-  Status Tell(int64_t* position) override {
-    *position = position_;
-    return Status::OK();
-  }
-
-  Status ReadAt(
-      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override {
-    RETURN_NOT_OK(Seek(position));
-    return Read(nbytes, bytes_read, buffer);
-  }
-
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override {
-    memcpy(buffer, buffer_ + position_, nbytes);
-    *bytes_read = std::min(nbytes, buffer_size_ - position_);
-    position_ += *bytes_read;
-    return Status::OK();
-  }
-
-  Status GetSize(int64_t* size) override {
-    *size = buffer_size_;
-    return Status::OK();
-  }
-
-  Status Seek(int64_t position) override {
-    if (position < 0 || position >= buffer_size_) {
-      return Status::IOError("position out of bounds");
-    }
-
-    position_ = position;
-    return Status::OK();
-  }
-
- private:
-  const uint8_t* buffer_;
-  int buffer_size_;
-  int64_t position_;
-};
-
 TEST(TestParquetReadSource, Basics) {
   std::string data = "this is the data";
   auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str());
 
   ParquetAllocator allocator(default_memory_pool());
 
-  auto file = std::make_shared<BufferReader>(data_buffer, data.size());
+  auto file = std::make_shared<io::BufferReader>(data_buffer, data.size());
   auto source = std::make_shared<ParquetReadSource>(&allocator);
 
   ASSERT_OK(source->Open(file));
diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
index a2bcd3e05c307..63ad8fba46517 100644
--- a/cpp/src/arrow/parquet/parquet-schema-test.cc
+++ b/cpp/src/arrow/parquet/parquet-schema-test.cc
@@ -178,8 +178,7 @@ class TestConvertArrowSchema : public ::testing::Test {
     NodePtr schema_node = GroupNode::Make("schema", Repetition::REPEATED, nodes);
     const GroupNode* expected_schema_node =
         static_cast<const GroupNode*>(schema_node.get());
-    const GroupNode* result_schema_node =
-        static_cast<const GroupNode*>(result_schema_->schema().get());
+    const GroupNode* result_schema_node = result_schema_->group_node();
 
     ASSERT_EQ(expected_schema_node->field_count(), result_schema_node->field_count());
 
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
index 440ec84e2c74e..0c2fc6e8fc718 100644
--- a/cpp/src/arrow/parquet/reader.cc
+++ b/cpp/src/arrow/parquet/reader.cc
@@ -149,7 +149,7 @@ bool FileReader::Impl::CheckForFlatColumn(const ::parquet::ColumnDescriptor* des
 }
 
 Status FileReader::Impl::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out) {
-  const ::parquet::SchemaDescriptor* schema = reader_->metadata()->schema_descriptor();
+  const ::parquet::SchemaDescriptor* schema = reader_->metadata()->schema();
 
   if (!CheckForFlatColumn(schema->Column(i))) {
     return Status::Invalid("The requested column is not flat");
@@ -167,9 +167,9 @@ Status FileReader::Impl::ReadFlatColumn(int i, std::shared_ptr<Array>* out) {
 }
 
 Status FileReader::Impl::ReadFlatTable(std::shared_ptr<Table>* table) {
-  auto descr = reader_->metadata()->schema_descriptor();
+  auto descr = reader_->metadata()->schema();
 
-  const std::string& name = descr->schema()->name();
+  const std::string& name = descr->name();
   std::shared_ptr<Schema> schema;
   RETURN_NOT_OK(FromParquetSchema(descr, &schema));
 
@@ -193,7 +193,7 @@ FileReader::FileReader(
 FileReader::~FileReader() {}
 
 // Static ctor
-Status OpenFile(const std::shared_ptr<io::RandomAccessFile>& file,
+Status OpenFile(const std::shared_ptr<io::ReadableFileInterface>& file,
     ParquetAllocator* allocator, std::unique_ptr<FileReader>* reader) {
   std::unique_ptr<ParquetReadSource> source(new ParquetReadSource(allocator));
   RETURN_NOT_OK(source->Open(file));
diff --git a/cpp/src/arrow/parquet/reader.h b/cpp/src/arrow/parquet/reader.h
index f1492f64521cb..a9c64eca997b5 100644
--- a/cpp/src/arrow/parquet/reader.h
+++ b/cpp/src/arrow/parquet/reader.h
@@ -137,7 +137,7 @@ class ARROW_EXPORT FlatColumnReader {
 // Helper function to create a file reader from an implementation of an Arrow
 // readable file
 ARROW_EXPORT
-Status OpenFile(const std::shared_ptr<io::RandomAccessFile>& file,
+Status OpenFile(const std::shared_ptr<io::ReadableFileInterface>& file,
     ParquetAllocator* allocator, std::unique_ptr<FileReader>* reader);
 
 }  // namespace parquet
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
index cd91df32271c1..ff32e51bacd8b 100644
--- a/cpp/src/arrow/parquet/schema.cc
+++ b/cpp/src/arrow/parquet/schema.cc
@@ -202,7 +202,7 @@ Status FromParquetSchema(
   // TODO(wesm): Consider adding an arrow::Schema name attribute, which comes
   // from the root Parquet node
   const GroupNode* schema_node =
-      static_cast<const GroupNode*>(parquet_schema->schema().get());
+      static_cast<const GroupNode*>(parquet_schema->group_node());
 
   std::vector<std::shared_ptr<Field>> fields(schema_node->field_count());
   for (int i = 0; i < schema_node->field_count(); i++) {
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
index ddee573fa1eb9..2b47f1461c9f4 100644
--- a/cpp/src/arrow/parquet/writer.cc
+++ b/cpp/src/arrow/parquet/writer.cc
@@ -334,7 +334,7 @@ Status WriteFlatTable(const Table* table, MemoryPool* pool,
   std::shared_ptr<::parquet::SchemaDescriptor> parquet_schema;
   RETURN_NOT_OK(
       ToParquetSchema(table->schema().get(), *properties.get(), &parquet_schema));
-  auto schema_node = std::static_pointer_cast<GroupNode>(parquet_schema->schema());
+  auto schema_node = std::static_pointer_cast<GroupNode>(parquet_schema->schema_root());
   std::unique_ptr<ParquetFileWriter> parquet_writer =
       ParquetFileWriter::Open(sink, schema_node, properties);
   FileWriter writer(pool, std::move(parquet_writer));
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 02677d5e18b90..b4c3721a72895 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -149,7 +149,7 @@ struct ARROW_EXPORT Field {
   int64_t dictionary;
 
   Field(const std::string& name, const TypePtr& type, bool nullable = true,
-        int64_t dictionary = 0)
+      int64_t dictionary = 0)
       : name(name), type(type), nullable(nullable), dictionary(dictionary) {}
 
   bool operator==(const Field& other) const { return this->Equals(other); }
@@ -159,7 +159,7 @@ struct ARROW_EXPORT Field {
   bool Equals(const Field& other) const {
     return (this == &other) ||
            (this->name == other.name && this->nullable == other.nullable &&
-            this->dictionary == dictionary && this->type->Equals(other.type.get()));
+               this->dictionary == dictionary && this->type->Equals(other.type.get()));
   }
 
   bool Equals(const std::shared_ptr<Field>& other) const { return Equals(*other.get()); }
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
index e767e9555244d..5d60376f794ff 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/util/memory-pool-test.cc
@@ -64,6 +64,6 @@ TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
   pool->Free(data, 100);
 }
 
-#endif // ARROW_VALGRIND
+#endif  // ARROW_VALGRIND
 
 }  // namespace arrow
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 734ace6c923b4..f338a436814de 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -29,25 +29,41 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
         ObjectType_FILE" arrow::io::ObjectType::FILE"
         ObjectType_DIRECTORY" arrow::io::ObjectType::DIRECTORY"
 
-    cdef cppclass FileBase:
+    cdef cppclass FileInterface:
         CStatus Close()
         CStatus Tell(int64_t* position)
+        FileMode mode()
 
-    cdef cppclass ReadableFile(FileBase):
+    cdef cppclass Readable:
+        CStatus Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out)
+
+    cdef cppclass Seekable:
+        CStatus Seek(int64_t position)
+
+    cdef cppclass Writeable:
+        CStatus Write(const uint8_t* data, int64_t nbytes)
+
+    cdef cppclass OutputStream(FileInterface, Writeable):
+        pass
+
+    cdef cppclass InputStream(FileInterface, Readable):
+        pass
+
+    cdef cppclass ReadableFileInterface(InputStream, Seekable):
         CStatus GetSize(int64_t* size)
-        CStatus Read(int64_t nbytes, int64_t* bytes_read,
-                     uint8_t* buffer)
 
         CStatus ReadAt(int64_t position, int64_t nbytes,
                        int64_t* bytes_read, uint8_t* buffer)
+        CStatus ReadAt(int64_t position, int64_t nbytes,
+                       int64_t* bytes_read, shared_ptr[Buffer]* out)
 
-    cdef cppclass RandomAccessFile(ReadableFile):
-        CStatus Seek(int64_t position)
+    cdef cppclass WriteableFileInterface(OutputStream, Seekable):
+        CStatus WriteAt(int64_t position, const uint8_t* data,
+                        int64_t nbytes)
 
-    cdef cppclass WriteableFile(FileBase):
-        CStatus Write(const uint8_t* buffer, int64_t nbytes)
-        # CStatus Write(const uint8_t* buffer, int64_t nbytes,
-        #               int64_t* bytes_written)
+    cdef cppclass ReadWriteFileInterface(ReadableFileInterface,
+                                         WriteableFileInterface):
+        pass
 
 
 cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
@@ -70,10 +86,10 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
         int64_t block_size
         int16_t permissions
 
-    cdef cppclass HdfsReadableFile(RandomAccessFile):
+    cdef cppclass HdfsReadableFile(ReadableFileInterface):
         pass
 
-    cdef cppclass HdfsWriteableFile(WriteableFile):
+    cdef cppclass HdfsOutputStream(OutputStream):
         pass
 
     cdef cppclass CHdfsClient" arrow::io::HdfsClient":
@@ -103,4 +119,4 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
         CStatus OpenWriteable(const c_string& path, c_bool append,
                               int32_t buffer_size, int16_t replication,
                               int64_t default_block_size,
-                              shared_ptr[HdfsWriteableFile]* handle)
+                              shared_ptr[HdfsOutputStream]* handle)
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index fe24f593e3294..f932a93149354 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -19,7 +19,7 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport CSchema, CStatus, CTable, MemoryPool
-from pyarrow.includes.libarrow_io cimport RandomAccessFile
+from pyarrow.includes.libarrow_io cimport ReadableFileInterface
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
@@ -78,10 +78,10 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 
 cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
-    cdef cppclass OutputStream:
+    cdef cppclass ParquetOutputStream" parquet::OutputStream":
         pass
 
-    cdef cppclass LocalFileOutputStream(OutputStream):
+    cdef cppclass LocalFileOutputStream(ParquetOutputStream):
         LocalFileOutputStream(const c_string& path)
         void Close()
 
@@ -100,11 +100,11 @@ cdef extern from "arrow/parquet/io.h" namespace "arrow::parquet" nogil:
 
     cdef cppclass ParquetReadSource:
         ParquetReadSource(ParquetAllocator* allocator)
-        Open(const shared_ptr[RandomAccessFile]& file)
+        Open(const shared_ptr[ReadableFileInterface]& file)
 
 
 cdef extern from "arrow/parquet/reader.h" namespace "arrow::parquet" nogil:
-    CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
+    CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
                      ParquetAllocator* allocator,
                      unique_ptr[FileReader]* reader)
 
@@ -121,6 +121,8 @@ cdef extern from "arrow/parquet/schema.h" namespace "arrow::parquet" nogil:
 
 
 cdef extern from "arrow/parquet/writer.h" namespace "arrow::parquet" nogil:
-    cdef CStatus WriteFlatTable(const CTable* table, MemoryPool* pool,
-            const shared_ptr[OutputStream]& sink, int64_t chunk_size,
-            const shared_ptr[WriterProperties]& properties)
+    cdef CStatus WriteFlatTable(
+        const CTable* table, MemoryPool* pool,
+        const shared_ptr[ParquetOutputStream]& sink,
+        int64_t chunk_size,
+        const shared_ptr[WriterProperties]& properties)
diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
index b92af72704ae8..f55fc0ab53ac1 100644
--- a/python/pyarrow/io.pxd
+++ b/python/pyarrow/io.pxd
@@ -19,7 +19,8 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.libarrow_io cimport RandomAccessFile, WriteableFile
+from pyarrow.includes.libarrow_io cimport (ReadableFileInterface,
+                                           OutputStream)
 
 
 cdef class NativeFileInterface:
@@ -28,5 +29,5 @@ cdef class NativeFileInterface:
     # extension classes are technically virtual in the C++ sense)m we can
     # expose the arrow::io abstract file interfaces to other components
     # throughout the suite of Arrow C++ libraries
-    cdef read_handle(self, shared_ptr[RandomAccessFile]* file)
-    cdef write_handle(self, shared_ptr[WriteableFile]* file)
+    cdef read_handle(self, shared_ptr[ReadableFileInterface]* file)
+    cdef write_handle(self, shared_ptr[OutputStream]* file)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index b8bf883562060..f2eee260c331b 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -316,16 +316,16 @@ cdef class HdfsClient:
 
 cdef class NativeFileInterface:
 
-    cdef read_handle(self, shared_ptr[RandomAccessFile]* file):
+    cdef read_handle(self, shared_ptr[ReadableFileInterface]* file):
         raise NotImplementedError
 
-    cdef write_handle(self, shared_ptr[WriteableFile]* file):
+    cdef write_handle(self, shared_ptr[OutputStream]* file):
         raise NotImplementedError
 
 cdef class HdfsFile(NativeFileInterface):
     cdef:
         shared_ptr[HdfsReadableFile] rd_file
-        shared_ptr[HdfsWriteableFile] wr_file
+        shared_ptr[HdfsOutputStream] wr_file
         bint is_readonly
         bint is_open
         object parent
@@ -364,13 +364,13 @@ cdef class HdfsFile(NativeFileInterface):
         if self.is_readonly:
             raise IOError("only valid on writeonly files")
 
-    cdef read_handle(self, shared_ptr[RandomAccessFile]* file):
+    cdef read_handle(self, shared_ptr[ReadableFileInterface]* file):
         self._assert_readable()
-        file[0] = <shared_ptr[RandomAccessFile]> self.rd_file
+        file[0] = <shared_ptr[ReadableFileInterface]> self.rd_file
 
-    cdef write_handle(self, shared_ptr[WriteableFile]* file):
+    cdef write_handle(self, shared_ptr[OutputStream]* file):
         self._assert_writeable()
-        file[0] = <shared_ptr[WriteableFile]> self.wr_file
+        file[0] = <shared_ptr[OutputStream]> self.wr_file
 
     def size(self):
         cdef int64_t size
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index ebba1a17ac742..fb36b2967c096 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -21,7 +21,7 @@
 
 from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.parquet cimport *
-from pyarrow.includes.libarrow_io cimport RandomAccessFile, WriteableFile
+from pyarrow.includes.libarrow_io cimport ReadableFileInterface
 cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.compat import tobytes
@@ -55,7 +55,7 @@ cdef class ParquetReader:
                            ParquetFileReader.OpenFile(path)))
 
     cdef open_native_file(self, NativeFileInterface file):
-        cdef shared_ptr[RandomAccessFile] cpp_handle
+        cdef shared_ptr[ReadableFileInterface] cpp_handle
         file.read_handle(&cpp_handle)
 
         check_cstatus(OpenFile(cpp_handle, &self.allocator, &self.reader))
@@ -105,7 +105,7 @@ def write_table(table, filename, chunk_size=None, version=None):
     """
     cdef Table table_ = table
     cdef CTable* ctable_ = table_.table
-    cdef shared_ptr[OutputStream] sink
+    cdef shared_ptr[ParquetOutputStream] sink
     cdef WriterProperties.Builder properties_builder
     cdef int64_t chunk_size_ = 0
     if chunk_size is None:

From 5f1556c011446a9fc524e91042c859365ed7afc1 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Mon, 19 Sep 2016 14:08:32 -0700
Subject: [PATCH 0138/1644] ARROW-297: Fix Arrow pom for release

Author: Julien Le Dem <julien@dremio.com>

Closes #140 from julienledem/fix_pom_for_release and squashes the following commits:

9618eaf [Julien Le Dem] ARROW-297: Fix Arrow pom for release
---
 java/format/pom.xml | 19 ++++++++++---------
 java/pom.xml        |  2 +-
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 4cf68bbe057e9..78300047862f4 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -27,6 +27,7 @@
     <fbs.version>1.2.0-3f79e055</fbs.version>
     <flatc.download.skip>false</flatc.download.skip>
     <flatc.executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</flatc.executable>
+    <flatc.generated.files>${project.build.directory}/generated-sources/flatc</flatc.generated.files>
     <maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
     <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
     <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
@@ -51,7 +52,7 @@
   </extensions>
 
   <plugins>
-    <plugin>
+    <plugin> <!-- download the flatbuffer compiler -->
       <groupId>org.apache.maven.plugins</groupId>
       <artifactId>maven-dependency-plugin</artifactId>
       <version>${maven-dependency-plugin.version}</version>
@@ -83,7 +84,7 @@
       <artifactId>exec-maven-plugin</artifactId>
       <version>1.4.0</version>
       <executions>
-        <execution>
+        <execution> <!-- make the flatbuffer compiler executable -->
           <id>script-chmod</id>
           <goals>
             <goal>exec</goal>
@@ -98,7 +99,7 @@
             <skip>${flatc.download.skip}</skip>
           </configuration>
         </execution>
-        <execution>
+        <execution> <!-- generate sources by executing the flatbuffer compiler -->
           <goals>
             <goal>exec</goal>
           </goals>
@@ -108,7 +109,7 @@
             <arguments>
               <argument>-j</argument>
               <argument>-o</argument>
-              <argument>target/generated-sources/flatc</argument>
+              <argument>${flatc.generated.files}</argument>
               <argument>../../format/Message.fbs</argument>
               <argument>../../format/File.fbs</argument>
             </arguments>
@@ -116,7 +117,7 @@
         </execution>
       </executions>
     </plugin>
-    <plugin>
+    <plugin> <!-- add the license header to the generated files -->
       <groupId>com.mycila</groupId>
       <artifactId>license-maven-plugin</artifactId>
       <version>2.3</version>
@@ -135,26 +136,26 @@
         </execution>
       </executions>
     </plugin>
-    <plugin>
+    <plugin> <!-- add generated sources to classpath -->
       <groupId>org.codehaus.mojo</groupId>
       <artifactId>build-helper-maven-plugin</artifactId>
       <version>1.9.1</version>
       <executions>
         <execution>
-          <id>add-sources-as-resources</id>
+          <id>add-generated-sources-to-classpath</id>
           <phase>generate-sources</phase>
           <goals>
             <goal>add-source</goal>
           </goals>
           <configuration>
             <sources>
-              <source>${project.build.directory}/generated-sources</source>
+              <source>${flatc.generated.files}</source>
             </sources>
           </configuration>
         </execution>
       </executions>
     </plugin>
-    <plugin>
+    <plugin> <!-- no checkstyle on the generated code -->
       <groupId>org.apache.maven.plugins</groupId>
       <artifactId>maven-checkstyle-plugin</artifactId>
       <configuration>
diff --git a/java/pom.xml b/java/pom.xml
index a8e24ed054cd5..fc2c18d0e517d 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -16,7 +16,7 @@
   <parent>
     <groupId>org.apache</groupId>
     <artifactId>apache</artifactId>
-    <version>14</version>
+    <version>18</version>
   </parent>
 
   <groupId>org.apache.arrow</groupId>

From 53583281b2af3e4ecedd3b130cef588680a44c4f Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 21 Sep 2016 13:38:52 -0700
Subject: [PATCH 0139/1644] ARROW-298: create release scripts

Author: Julien Le Dem <julien@dremio.com>

Closes #141 from julienledem/release and squashes the following commits:

1a5114d [Julien Le Dem] ARROW-298: create release scripts
---
 dev/release/00-prepare.sh | 46 ++++++++++++++++++++++
 dev/release/01-perform.sh | 27 +++++++++++++
 dev/release/02-source.sh  | 80 +++++++++++++++++++++++++++++++++++++++
 dev/release/README        | 15 ++++++++
 java/README.md            | 14 +++++++
 5 files changed, 182 insertions(+)
 create mode 100644 dev/release/00-prepare.sh
 create mode 100644 dev/release/01-perform.sh
 create mode 100644 dev/release/02-source.sh
 create mode 100644 dev/release/README
 create mode 100644 java/README.md

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
new file mode 100644
index 0000000000000..3c1fb9a093892
--- /dev/null
+++ b/dev/release/00-prepare.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ -z "$1" ]; then
+  echo "Usage: $0 <version> <nextVersion>"
+  exit
+fi
+
+if [ -z "$2" ]; then
+  echo "Usage: $0 <version> <nextVersion>"
+  exit
+fi
+
+version=$1
+
+tag=apache-arrow-${version}
+
+nextVersion=$2
+
+cd "${SOURCE_DIR}/../../java"
+
+mvn release:clean
+mvn release:prepare -Dtag=${tag} -DreleaseVersion=${version} -DautoVersionSubmodules -DdevelopmentVersion=${nextVersion}-SNAPSHOT
+
+cd -
+
+echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
diff --git a/dev/release/01-perform.sh b/dev/release/01-perform.sh
new file mode 100644
index 0000000000000..d7140f6cba1e7
--- /dev/null
+++ b/dev/release/01-perform.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "${SOURCE_DIR}/../../java"
+
+mvn release:perform
+
+cd -
diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
new file mode 100644
index 0000000000000..f44692d5e9d83
--- /dev/null
+++ b/dev/release/02-source.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "$1" ]; then
+  echo "Usage: $0 <version> <rc-num>"
+  exit
+fi
+
+if [ -z "$2" ]; then
+  echo "Usage: $0 <version> <rc-num>"
+  exit
+fi
+
+version=$1
+rc=$2
+
+if [ -d tmp/ ]; then
+  echo "Cannot run: tmp/ exists"
+  exit
+fi
+
+tag=apache-arrow-$version
+tagrc=${tag}-rc${rc}
+
+echo "Preparing source for $tagrc"
+
+release_hash=`git rev-list $tag 2> /dev/null | head -n 1 `
+
+if [ -z "$release_hash" ]; then
+  echo "Cannot continue: unknown git tag: $tag"
+  exit
+fi
+
+echo "Using commit $release_hash"
+
+tarball=$tag.tar.gz
+
+# be conservative and use the release hash, even though git produces the same
+# archive (identical hashes) using the scm tag
+git archive $release_hash --prefix $tag/ -o $tarball
+
+# sign the archive
+gpg --armor --output ${tarball}.asc --detach-sig $tarball
+gpg --print-md MD5 $tarball > ${tarball}.md5
+shasum $tarball > ${tarball}.sha
+
+# check out the parquet RC folder
+svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp
+
+# add the release candidate for the tag
+mkdir -p tmp/$tagrc
+cp ${tarball}* tmp/$tagrc
+svn add tmp/$tagrc
+svn ci -m 'Apache Arrow $version RC${rc}' tmp/$tagrc
+
+# clean up
+rm -rf tmp
+
+echo "Success! The release candidate is available here:"
+echo "  https://dist.apache.org/repos/dist/dev/arrow/$tagrc"
+echo ""
+echo "Commit SHA1: $release_hash"
+
diff --git a/dev/release/README b/dev/release/README
new file mode 100644
index 0000000000000..4fcc5d9728c26
--- /dev/null
+++ b/dev/release/README
@@ -0,0 +1,15 @@
+requirements:
+- being a committer to be able to push to dist and maven repository
+- a gpg key to sign the artifacts
+
+to release, run the following (replace 0.1.0 with version to release):
+# prepare release v 0.1.0 (run tests, sign artifacts). Next version will be 0.1.1-SNAPSHOT
+dev/release/00-prepare.sh 0.1.0 0.1.1
+# tag and push to maven repo (repo will have to be finalized separately)
+dev/release/01-perform.sh
+# create the source release
+dev/release/02-source.sh 0.1.0 0
+
+useful commands:
+to set the mvn version in the poms
+mvn versions:set -DnewVersion=0.1-SNAPSHOT
diff --git a/java/README.md b/java/README.md
new file mode 100644
index 0000000000000..5e1d30d9fd26e
--- /dev/null
+++ b/java/README.md
@@ -0,0 +1,14 @@
+# Arrow Java
+
+## Setup Build Environment
+
+install:
+ - java 7 or later
+ - maven 3.3 or later
+
+## Building running tests
+
+```
+cd java
+mvn install
+``` 

From 430bd9576ceb14456cd6853f6d75ca19b333efc2 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 21 Sep 2016 18:14:00 -0400
Subject: [PATCH 0140/1644] ARROW-299: Use absolute namespace in macros

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #142 from xhochy/arrow-299 and squashes the following commits:

b7967fa [Uwe L. Korn] ARROW-299: Use absolute namespace in macros
---
 cpp/src/arrow/util/logging.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 54f67593bec5e..d320d6adb7caa 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -35,7 +35,7 @@ namespace arrow {
 #define ARROW_ERROR 2
 #define ARROW_FATAL 3
 
-#define ARROW_LOG_INTERNAL(level) arrow::internal::CerrLog(level)
+#define ARROW_LOG_INTERNAL(level) ::arrow::internal::CerrLog(level)
 #define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level)
 
 #define ARROW_CHECK(condition)                               \
@@ -47,25 +47,25 @@ namespace arrow {
 
 #define DCHECK(condition) \
   while (false)           \
-  arrow::internal::NullLog()
+  ::arrow::internal::NullLog()
 #define DCHECK_EQ(val1, val2) \
   while (false)               \
-  arrow::internal::NullLog()
+  ::arrow::internal::NullLog()
 #define DCHECK_NE(val1, val2) \
   while (false)               \
-  arrow::internal::NullLog()
+  ::arrow::internal::NullLog()
 #define DCHECK_LE(val1, val2) \
   while (false)               \
-  arrow::internal::NullLog()
+  ::arrow::internal::NullLog()
 #define DCHECK_LT(val1, val2) \
   while (false)               \
-  arrow::internal::NullLog()
+  ::arrow::internal::NullLog()
 #define DCHECK_GE(val1, val2) \
   while (false)               \
-  arrow::internal::NullLog()
+  ::arrow::internal::NullLog()
 #define DCHECK_GT(val1, val2) \
   while (false)               \
-  arrow::internal::NullLog()
+  ::arrow::internal::NullLog()
 
 #else
 #define ARROW_DFATAL ARROW_FATAL

From 7e39747eec05379710e1a42ecbaf1d9795bc3cf0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 21 Sep 2016 18:15:58 -0400
Subject: [PATCH 0141/1644] ARROW-267: [C++] Implement file format layout for
 IPC/RPC

Standing up the PR to get some feedback. I still have to implement the read path for record batches and then add a test suite. I'd also like to add some documentation about the structure of the file format and some of the implicit assumptions (e.g. word alignment) -- I put a placeholder `IPC.md` document here for this.

I also conformed the language re: record batches (had been using "row batch" in the C++ code) to make things more sane.

Note we are not yet able to write OS files here, see ARROW-293. Will tackle that in a follow up PR, and then we should be in a position to integration test.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #139 from wesm/ARROW-267 and squashes the following commits:

9bdbbd4 [Wes McKinney] Get test suite passing, add missing metadata adapters for string, binary
4d3cc1d [Wes McKinney] cpplint
2ec1aad [Wes McKinney] Draft failing file roundtrip test
358309b [Wes McKinney] Move record batch test fixtures into test-common.h
b88bce0 [Wes McKinney] Finish draft of FileReader::GetRecordBatch. Add body end offset to ipc adapter
edf36e7 [Wes McKinney] Start drafting FileReader IPC implementation. Change record batch data header to write metadata size int32_t as suffix rather than prefix
95157f2 [Wes McKinney] Make record batch writes aligned on word boundaries
7c50251 [Wes McKinney] Make the interface for WriteRecordBatch more flexible (not require constructing a RecordBatch object)
ab4056f [Wes McKinney] Drafting file reader/writer API. Implement BufferOutputStream and write file footers to an OutputStream
113ac7b [Wes McKinney] Draft file footer metadata write/read path with simple unit test
---
 NOTICE.txt                             |   6 +
 cpp/src/arrow/io/memory.cc             |  37 ++++
 cpp/src/arrow/io/memory.h              |  18 +-
 cpp/src/arrow/ipc/CMakeLists.txt       |  18 +-
 cpp/src/arrow/ipc/adapter.cc           | 126 ++++++-----
 cpp/src/arrow/ipc/adapter.h            |  47 ++--
 cpp/src/arrow/ipc/file.cc              | 210 ++++++++++++++++++
 cpp/src/arrow/ipc/file.h               | 146 +++++++++++++
 cpp/src/arrow/ipc/ipc-adapter-test.cc  | 284 +++++--------------------
 cpp/src/arrow/ipc/ipc-file-test.cc     | 125 +++++++++++
 cpp/src/arrow/ipc/ipc-metadata-test.cc |  77 ++++++-
 cpp/src/arrow/ipc/metadata-internal.cc |  46 ++--
 cpp/src/arrow/ipc/metadata-internal.h  |   9 +
 cpp/src/arrow/ipc/metadata.cc          | 171 ++++++++++++---
 cpp/src/arrow/ipc/metadata.h           |  64 +++++-
 cpp/src/arrow/ipc/test-common.h        | 193 ++++++++++++++++-
 cpp/src/arrow/ipc/util.h               |   8 +
 cpp/src/arrow/parquet/reader.h         |   2 +-
 cpp/src/arrow/parquet/writer.h         |   2 +-
 cpp/src/arrow/table.cc                 |   4 +-
 cpp/src/arrow/table.h                  |  16 +-
 format/IPC.md                          |   3 +
 format/README.md                       |   1 +
 23 files changed, 1231 insertions(+), 382 deletions(-)
 create mode 100644 cpp/src/arrow/ipc/file.cc
 create mode 100644 cpp/src/arrow/ipc/file.h
 create mode 100644 cpp/src/arrow/ipc/ipc-file-test.cc
 create mode 100644 format/IPC.md

diff --git a/NOTICE.txt b/NOTICE.txt
index a85101617cec8..ce6e567dcb518 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -12,3 +12,9 @@ This product includes software from the Numpy project (BSD-new)
  https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
  * Copyright (c) 1995, 1996, 1997 Jim Hugunin, hugunin@mit.edu
  * Copyright (c) 2005 Travis E. Oliphant oliphant@ee.byu.edu Brigham Young University
+
+This product includes software from the Feather project (Apache 2.0)
+https://github.com/wesm/feather
+
+This product includes software from the DyND project (BSD 2-clause)
+https://github.com/libdynd
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 1dd6c3a02304a..c168c91c5f87c 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -206,6 +206,43 @@ Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// OutputStream that writes to resizable buffer
+
+static constexpr int64_t kBufferMinimumSize = 256;
+
+BufferOutputStream::BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer)
+    : buffer_(buffer),
+      capacity_(buffer->size()),
+      position_(0),
+      mutable_data_(buffer->mutable_data()) {}
+
+Status BufferOutputStream::Close() {
+  return Status::OK();
+}
+
+Status BufferOutputStream::Tell(int64_t* position) {
+  *position = position_;
+  return Status::OK();
+}
+
+Status BufferOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+  RETURN_NOT_OK(Reserve(nbytes));
+  std::memcpy(mutable_data_ + position_, data, nbytes);
+  position_ += nbytes;
+  return Status::OK();
+}
+
+Status BufferOutputStream::Reserve(int64_t nbytes) {
+  while (position_ + nbytes > capacity_) {
+    int64_t new_capacity = std::max(kBufferMinimumSize, capacity_ * 2);
+    RETURN_NOT_OK(buffer_->Resize(new_capacity));
+    capacity_ = new_capacity;
+  }
+  mutable_data_ = buffer_->mutable_data();
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // In-memory buffer reader
 
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 6fe47c3b5157a..51601a0a62678 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -32,32 +32,30 @@
 namespace arrow {
 
 class Buffer;
-class MutableBuffer;
+class ResizableBuffer;
 class Status;
 
 namespace io {
 
 // An output stream that writes to a MutableBuffer, such as one obtained from a
 // memory map
-//
-// TODO(wesm): Implement this class
 class ARROW_EXPORT BufferOutputStream : public OutputStream {
  public:
-  explicit BufferOutputStream(const std::shared_ptr<MutableBuffer>& buffer)
-      : buffer_(buffer) {}
+  explicit BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer);
 
   // Implement the OutputStream interface
   Status Close() override;
   Status Tell(int64_t* position) override;
-  Status Write(const uint8_t* data, int64_t length) override;
-
-  // Returns the number of bytes remaining in the buffer
-  int64_t bytes_remaining() const;
+  Status Write(const uint8_t* data, int64_t nbytes) override;
 
  private:
-  std::shared_ptr<MutableBuffer> buffer_;
+  // Ensures there is sufficient space available to write nbytes
+  Status Reserve(int64_t nbytes);
+
+  std::shared_ptr<ResizableBuffer> buffer_;
   int64_t capacity_;
   int64_t position_;
+  uint8_t* mutable_data_;
 };
 
 // A memory source that uses memory-mapped files for memory interactions
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index e5553a6358115..bde8c5bf73888 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -33,6 +33,7 @@ set(ARROW_IPC_TEST_LINK_LIBS
 
 set(ARROW_IPC_SRCS
   adapter.cc
+  file.cc
   metadata.cc
   metadata-internal.cc
 )
@@ -60,6 +61,10 @@ ADD_ARROW_TEST(ipc-adapter-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-adapter-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
+ADD_ARROW_TEST(ipc-file-test)
+ARROW_TEST_LINK_LIBRARIES(ipc-file-test
+  ${ARROW_IPC_TEST_LINK_LIBS})
+
 ADD_ARROW_TEST(ipc-metadata-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-metadata-test
   ${ARROW_IPC_TEST_LINK_LIBS})
@@ -70,14 +75,20 @@ set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
 set(OUTPUT_DIR ${CMAKE_SOURCE_DIR}/src/arrow/ipc)
 set(FBS_OUTPUT_FILES "${OUTPUT_DIR}/Message_generated.h")
 
-set(FBS_SRC ${CMAKE_SOURCE_DIR}/../format/Message.fbs)
-get_filename_component(ABS_FBS_SRC ${FBS_SRC} ABSOLUTE)
+set(FBS_SRC
+  ${CMAKE_SOURCE_DIR}/../format/Message.fbs
+  ${CMAKE_SOURCE_DIR}/../format/File.fbs)
+
+foreach(FIL ${FBS_SRC})
+  get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+  list(APPEND ABS_FBS_SRC ${ABS_FIL})
+endforeach()
 
 add_custom_command(
   OUTPUT ${FBS_OUTPUT_FILES}
   COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC}
   DEPENDS ${ABS_FBS_SRC}
-  COMMENT "Running flatc compiler on ${FBS_SRC}"
+  COMMENT "Running flatc compiler on ${ABS_FBS_SRC}"
   VERBATIM
 )
 
@@ -87,6 +98,7 @@ add_dependencies(arrow_objlib metadata_fbs)
 # Headers: top level
 install(FILES
   adapter.h
+  file.h
   metadata.h
   DESTINATION include/arrow/ipc)
 
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 0e101c8930395..89b7fb987c63d 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -95,7 +95,7 @@ static bool IsListType(const DataType* type) {
 }
 
 // ----------------------------------------------------------------------
-// Row batch write path
+// Record batch write path
 
 Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes,
     std::vector<std::shared_ptr<Buffer>>* buffers, int max_recursion_depth) {
@@ -132,28 +132,32 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
   return Status::OK();
 }
 
-class RowBatchWriter {
+class RecordBatchWriter {
  public:
-  RowBatchWriter(const RowBatch* batch, int max_recursion_depth)
-      : batch_(batch), max_recursion_depth_(max_recursion_depth) {}
+  RecordBatchWriter(const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows,
+      int max_recursion_depth)
+      : columns_(&columns),
+        num_rows_(num_rows),
+        max_recursion_depth_(max_recursion_depth) {}
 
   Status AssemblePayload() {
     // Perform depth-first traversal of the row-batch
-    for (int i = 0; i < batch_->num_columns(); ++i) {
-      const Array* arr = batch_->column(i).get();
+    for (size_t i = 0; i < columns_->size(); ++i) {
+      const Array* arr = (*columns_)[i].get();
       RETURN_NOT_OK(VisitArray(arr, &field_nodes_, &buffers_, max_recursion_depth_));
     }
     return Status::OK();
   }
 
-  Status Write(io::OutputStream* dst, int64_t* data_header_offset) {
-    // Write out all the buffers contiguously and compute the total size of the
-    // memory payload
-    int64_t offset = 0;
-
+  Status Write(
+      io::OutputStream* dst, int64_t* body_end_offset, int64_t* header_end_offset) {
     // Get the starting position
-    int64_t position;
-    RETURN_NOT_OK(dst->Tell(&position));
+    int64_t start_position;
+    RETURN_NOT_OK(dst->Tell(&start_position));
+
+    // Keep track of the current position so we can determine the size of the
+    // message body
+    int64_t position = start_position;
 
     for (size_t i = 0; i < buffers_.size(); ++i) {
       const Buffer* buffer = buffers_[i].get();
@@ -175,14 +179,16 @@ class RowBatchWriter {
       // are using from any OS-level shared memory. The thought is that systems
       // may (in the future) associate integer page id's with physical memory
       // pages (according to whatever is the desired shared memory mechanism)
-      buffer_meta_.push_back(flatbuf::Buffer(0, position + offset, size));
+      buffer_meta_.push_back(flatbuf::Buffer(0, position, size));
 
       if (size > 0) {
         RETURN_NOT_OK(dst->Write(buffer->data(), size));
-        offset += size;
+        position += size;
       }
     }
 
+    *body_end_offset = position;
+
     // Now that we have computed the locations of all of the buffers in shared
     // memory, the data header can be converted to a flatbuffer and written out
     //
@@ -192,27 +198,43 @@ class RowBatchWriter {
     // construct the flatbuffer data accessor object (see arrow::ipc::Message)
     std::shared_ptr<Buffer> data_header;
     RETURN_NOT_OK(WriteDataHeader(
-        batch_->num_rows(), offset, field_nodes_, buffer_meta_, &data_header));
+        num_rows_, position - start_position, field_nodes_, buffer_meta_, &data_header));
 
     // Write the data header at the end
     RETURN_NOT_OK(dst->Write(data_header->data(), data_header->size()));
 
-    *data_header_offset = position + offset;
+    position += data_header->size();
+    *header_end_offset = position;
+
+    return Align(dst, &position);
+  }
+
+  Status Align(io::OutputStream* dst, int64_t* position) {
+    // Write all buffers here on word boundaries
+    // TODO(wesm): Is there benefit to 64-byte padding in IPC?
+    int64_t remainder = PaddedLength(*position) - *position;
+    if (remainder > 0) {
+      RETURN_NOT_OK(dst->Write(kPaddingBytes, remainder));
+      *position += remainder;
+    }
     return Status::OK();
   }
 
   // This must be called after invoking AssemblePayload
   Status GetTotalSize(int64_t* size) {
     // emulates the behavior of Write without actually writing
+    int64_t body_offset;
     int64_t data_header_offset;
     MockOutputStream dst;
-    RETURN_NOT_OK(Write(&dst, &data_header_offset));
+    RETURN_NOT_OK(Write(&dst, &body_offset, &data_header_offset));
     *size = dst.GetExtentBytesWritten();
     return Status::OK();
   }
 
  private:
-  const RowBatch* batch_;
+  // Do not copy this vector. Ownership must be retained elsewhere
+  const std::vector<std::shared_ptr<Array>>* columns_;
+  int32_t num_rows_;
 
   std::vector<flatbuf::FieldNode> field_nodes_;
   std::vector<flatbuf::Buffer> buffer_meta_;
@@ -220,29 +242,29 @@ class RowBatchWriter {
   int max_recursion_depth_;
 };
 
-Status WriteRowBatch(io::OutputStream* dst, const RowBatch* batch, int64_t* header_offset,
-    int max_recursion_depth) {
+Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
+    int32_t num_rows, io::OutputStream* dst, int64_t* body_end_offset,
+    int64_t* header_end_offset, int max_recursion_depth) {
   DCHECK_GT(max_recursion_depth, 0);
-  RowBatchWriter serializer(batch, max_recursion_depth);
+  RecordBatchWriter serializer(columns, num_rows, max_recursion_depth);
   RETURN_NOT_OK(serializer.AssemblePayload());
-  return serializer.Write(dst, header_offset);
+  return serializer.Write(dst, body_end_offset, header_end_offset);
 }
 
-Status GetRowBatchSize(const RowBatch* batch, int64_t* size) {
-  RowBatchWriter serializer(batch, kMaxIpcRecursionDepth);
+Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size) {
+  RecordBatchWriter serializer(
+      batch->columns(), batch->num_rows(), kMaxIpcRecursionDepth);
   RETURN_NOT_OK(serializer.AssemblePayload());
   RETURN_NOT_OK(serializer.GetTotalSize(size));
   return Status::OK();
 }
 
 // ----------------------------------------------------------------------
-// Row batch read path
+// Record batch read path
 
-static constexpr int64_t INIT_METADATA_SIZE = 4096;
-
-class RowBatchReader::RowBatchReaderImpl {
+class RecordBatchReader::RecordBatchReaderImpl {
  public:
-  RowBatchReaderImpl(io::ReadableFileInterface* file,
+  RecordBatchReaderImpl(io::ReadableFileInterface* file,
       const std::shared_ptr<RecordBatchMessage>& metadata, int max_recursion_depth)
       : file_(file), metadata_(metadata), max_recursion_depth_(max_recursion_depth) {
     num_buffers_ = metadata->num_buffers();
@@ -250,7 +272,7 @@ class RowBatchReader::RowBatchReaderImpl {
   }
 
   Status AssembleBatch(
-      const std::shared_ptr<Schema>& schema, std::shared_ptr<RowBatch>* out) {
+      const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out) {
     std::vector<std::shared_ptr<Array>> arrays(schema->num_fields());
 
     // The field_index and buffer_index are incremented in NextArray based on
@@ -263,7 +285,7 @@ class RowBatchReader::RowBatchReaderImpl {
       RETURN_NOT_OK(NextArray(field, max_recursion_depth_, &arrays[i]));
     }
 
-    *out = std::make_shared<RowBatch>(schema, metadata_->length(), arrays);
+    *out = std::make_shared<RecordBatch>(schema, metadata_->length(), arrays);
     return Status::OK();
   }
 
@@ -359,29 +381,31 @@ class RowBatchReader::RowBatchReaderImpl {
   int num_flattened_fields_;
 };
 
-Status RowBatchReader::Open(io::ReadableFileInterface* file, int64_t position,
-    std::shared_ptr<RowBatchReader>* out) {
-  return Open(file, position, kMaxIpcRecursionDepth, out);
+Status RecordBatchReader::Open(io::ReadableFileInterface* file, int64_t offset,
+    std::shared_ptr<RecordBatchReader>* out) {
+  return Open(file, offset, kMaxIpcRecursionDepth, out);
 }
 
-Status RowBatchReader::Open(io::ReadableFileInterface* file, int64_t position,
-    int max_recursion_depth, std::shared_ptr<RowBatchReader>* out) {
-  std::shared_ptr<Buffer> metadata;
-  RETURN_NOT_OK(file->ReadAt(position, INIT_METADATA_SIZE, &metadata));
+Status RecordBatchReader::Open(io::ReadableFileInterface* file, int64_t offset,
+    int max_recursion_depth, std::shared_ptr<RecordBatchReader>* out) {
+  std::shared_ptr<Buffer> buffer;
+  RETURN_NOT_OK(file->ReadAt(offset - sizeof(int32_t), sizeof(int32_t), &buffer));
 
-  int32_t metadata_size = *reinterpret_cast<const int32_t*>(metadata->data());
+  int32_t metadata_size = *reinterpret_cast<const int32_t*>(buffer->data());
 
-  // We may not need to call ReadAt again
-  if (metadata_size > static_cast<int>(INIT_METADATA_SIZE - sizeof(int32_t))) {
-    // We don't have enough data, read the indicated metadata size.
-    RETURN_NOT_OK(file->ReadAt(position + sizeof(int32_t), metadata_size, &metadata));
+  if (metadata_size + static_cast<int>(sizeof(int32_t)) > offset) {
+    return Status::Invalid("metadata size invalid");
   }
 
+  // Read the metadata
+  RETURN_NOT_OK(
+      file->ReadAt(offset - metadata_size - sizeof(int32_t), metadata_size, &buffer));
+
   // TODO(wesm): buffer slicing here would be better in case ReadAt returns
   // allocated memory
 
   std::shared_ptr<Message> message;
-  RETURN_NOT_OK(Message::Open(metadata, &message));
+  RETURN_NOT_OK(Message::Open(buffer, &message));
 
   if (message->type() != Message::RECORD_BATCH) {
     return Status::Invalid("Metadata message is not a record batch");
@@ -389,19 +413,19 @@ Status RowBatchReader::Open(io::ReadableFileInterface* file, int64_t position,
 
   std::shared_ptr<RecordBatchMessage> batch_meta = message->GetRecordBatch();
 
-  std::shared_ptr<RowBatchReader> result(new RowBatchReader());
-  result->impl_.reset(new RowBatchReaderImpl(file, batch_meta, max_recursion_depth));
+  std::shared_ptr<RecordBatchReader> result(new RecordBatchReader());
+  result->impl_.reset(new RecordBatchReaderImpl(file, batch_meta, max_recursion_depth));
   *out = result;
 
   return Status::OK();
 }
 
 // Here the explicit destructor is required for compilers to be aware of
-// the complete information of RowBatchReader::RowBatchReaderImpl class
-RowBatchReader::~RowBatchReader() {}
+// the complete information of RecordBatchReader::RecordBatchReaderImpl class
+RecordBatchReader::~RecordBatchReader() {}
 
-Status RowBatchReader::GetRowBatch(
-    const std::shared_ptr<Schema>& schema, std::shared_ptr<RowBatch>* out) {
+Status RecordBatchReader::GetRecordBatch(
+    const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out) {
   return impl_->AssembleBatch(schema, out);
 }
 
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 215b46f8f65d4..3fde18dde835b 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -23,13 +23,14 @@
 
 #include <cstdint>
 #include <memory>
+#include <vector>
 
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class Array;
-class RowBatch;
+class RecordBatch;
 class Schema;
 class Status;
 
@@ -50,7 +51,7 @@ class RecordBatchMessage;
 // TODO(emkornfield) investigate this more
 constexpr int kMaxIpcRecursionDepth = 64;
 
-// Write the RowBatch (collection of equal-length Arrow arrays) to the output
+// Write the RecordBatch (collection of equal-length Arrow arrays) to the output
 // stream
 //
 // First, each of the memory buffers are written out end-to-end
@@ -60,39 +61,43 @@ constexpr int kMaxIpcRecursionDepth = 64;
 //
 // <int32: metadata size> <uint8*: metadata>
 //
-// Finally, the absolute offset (relative to the start of the output stream) to
-// the start of the metadata / data header is returned in an out-variable
-ARROW_EXPORT Status WriteRowBatch(io::OutputStream* dst, const RowBatch* batch,
-    int64_t* header_offset, int max_recursion_depth = kMaxIpcRecursionDepth);
+// Finally, the absolute offsets (relative to the start of the output stream)
+// to the end of the body and end of the metadata / data header (suffixed by
+// the header size) is returned in out-variables
+ARROW_EXPORT Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
+    int32_t num_rows, io::OutputStream* dst, int64_t* body_end_offset,
+    int64_t* header_end_offset, int max_recursion_depth = kMaxIpcRecursionDepth);
 
-// int64_t GetRowBatchMetadata(const RowBatch* batch);
+// int64_t GetRecordBatchMetadata(const RecordBatch* batch);
 
 // Compute the precise number of bytes needed in a contiguous memory segment to
-// write the row batch. This involves generating the complete serialized
+// write the record batch. This involves generating the complete serialized
 // Flatbuffers metadata.
-ARROW_EXPORT Status GetRowBatchSize(const RowBatch* batch, int64_t* size);
+ARROW_EXPORT Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size);
 
 // ----------------------------------------------------------------------
 // "Read" path; does not copy data if the input supports zero copy reads
 
-class ARROW_EXPORT RowBatchReader {
+class ARROW_EXPORT RecordBatchReader {
  public:
-  static Status Open(io::ReadableFileInterface* file, int64_t position,
-      std::shared_ptr<RowBatchReader>* out);
+  // The offset is the absolute position to the *end* of the record batch data
+  // header
+  static Status Open(io::ReadableFileInterface* file, int64_t offset,
+      std::shared_ptr<RecordBatchReader>* out);
 
-  static Status Open(io::ReadableFileInterface* file, int64_t position,
-      int max_recursion_depth, std::shared_ptr<RowBatchReader>* out);
+  static Status Open(io::ReadableFileInterface* file, int64_t offset,
+      int max_recursion_depth, std::shared_ptr<RecordBatchReader>* out);
 
-  virtual ~RowBatchReader();
+  virtual ~RecordBatchReader();
 
-  // Reassemble the row batch. A Schema is required to be able to construct the
-  // right array containers
-  Status GetRowBatch(
-      const std::shared_ptr<Schema>& schema, std::shared_ptr<RowBatch>* out);
+  // Reassemble the record batch. A Schema is required to be able to construct
+  // the right array containers
+  Status GetRecordBatch(
+      const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out);
 
  private:
-  class RowBatchReaderImpl;
-  std::unique_ptr<RowBatchReaderImpl> impl_;
+  class RecordBatchReaderImpl;
+  std::unique_ptr<RecordBatchReaderImpl> impl_;
 };
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
new file mode 100644
index 0000000000000..2bf10dde266bd
--- /dev/null
+++ b/cpp/src/arrow/ipc/file.cc
@@ -0,0 +1,210 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/file.h"
+
+#include <cstdint>
+#include <cstring>
+#include <sstream>
+#include <vector>
+
+#include "arrow/ipc/adapter.h"
+#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/util.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+static constexpr const char* kArrowMagicBytes = "ARROW1";
+
+// ----------------------------------------------------------------------
+// Writer implementation
+
+FileWriter::FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
+    : sink_(sink), schema_(schema), position_(-1), started_(false) {}
+
+Status FileWriter::UpdatePosition() {
+  return sink_->Tell(&position_);
+}
+
+Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    std::shared_ptr<FileWriter>* out) {
+  *out = std::shared_ptr<FileWriter>(new FileWriter(sink, schema));  // ctor is private
+  RETURN_NOT_OK((*out)->UpdatePosition());
+  return Status::OK();
+}
+
+Status FileWriter::Write(const uint8_t* data, int64_t nbytes) {
+  RETURN_NOT_OK(sink_->Write(data, nbytes));
+  position_ += nbytes;
+  return Status::OK();
+}
+
+Status FileWriter::Align() {
+  int64_t remainder = PaddedLength(position_) - position_;
+  if (remainder > 0) { return Write(kPaddingBytes, remainder); }
+  return Status::OK();
+}
+
+Status FileWriter::WriteAligned(const uint8_t* data, int64_t nbytes) {
+  RETURN_NOT_OK(Write(data, nbytes));
+  return Align();
+}
+
+Status FileWriter::Start() {
+  RETURN_NOT_OK(WriteAligned(
+      reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
+  started_ = true;
+  return Status::OK();
+}
+
+Status FileWriter::CheckStarted() {
+  if (!started_) { return Start(); }
+  return Status::OK();
+}
+
+Status FileWriter::WriteRecordBatch(
+    const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows) {
+  RETURN_NOT_OK(CheckStarted());
+
+  int64_t offset = position_;
+
+  int64_t body_end_offset;
+  int64_t header_end_offset;
+  RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(
+      columns, num_rows, sink_, &body_end_offset, &header_end_offset));
+  RETURN_NOT_OK(UpdatePosition());
+
+  DCHECK(position_ % 8 == 0) << "ipc::WriteRecordBatch did not perform aligned writes";
+
+  // There may be padding ever the end of the metadata, so we cannot rely on
+  // position_
+  int32_t metadata_length = header_end_offset - body_end_offset;
+  int32_t body_length = body_end_offset - offset;
+
+  // Append metadata, to be written in the footer later
+  record_batches_.emplace_back(offset, metadata_length, body_length);
+
+  return Status::OK();
+}
+
+Status FileWriter::Close() {
+  // Write metadata
+  int64_t initial_position = position_;
+  RETURN_NOT_OK(WriteFileFooter(schema_.get(), dictionaries_, record_batches_, sink_));
+  RETURN_NOT_OK(UpdatePosition());
+
+  // Write footer length
+  int32_t footer_length = position_ - initial_position;
+
+  if (footer_length <= 0) { return Status::Invalid("Invalid file footer"); }
+
+  RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(&footer_length), sizeof(int32_t)));
+
+  // Write magic bytes to end file
+  return Write(
+      reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes));
+}
+
+// ----------------------------------------------------------------------
+// Reader implementation
+
+FileReader::FileReader(
+    const std::shared_ptr<io::ReadableFileInterface>& file, int64_t footer_offset)
+    : file_(file), footer_offset_(footer_offset) {}
+
+FileReader::~FileReader() {}
+
+Status FileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+    std::shared_ptr<FileReader>* reader) {
+  int64_t footer_offset;
+  RETURN_NOT_OK(file->GetSize(&footer_offset));
+  return Open(file, footer_offset, reader);
+}
+
+Status FileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+    int64_t footer_offset, std::shared_ptr<FileReader>* reader) {
+  *reader = std::shared_ptr<FileReader>(new FileReader(file, footer_offset));
+  return (*reader)->ReadFooter();
+}
+
+Status FileReader::ReadFooter() {
+  int magic_size = static_cast<int>(strlen(kArrowMagicBytes));
+
+  if (footer_offset_ <= magic_size * 2 + 4) {
+    std::stringstream ss;
+    ss << "File is too small: " << footer_offset_;
+    return Status::Invalid(ss.str());
+  }
+
+  std::shared_ptr<Buffer> buffer;
+  int file_end_size = magic_size + sizeof(int32_t);
+  RETURN_NOT_OK(file_->ReadAt(footer_offset_ - file_end_size, file_end_size, &buffer));
+
+  if (memcmp(buffer->data() + sizeof(int32_t), kArrowMagicBytes, magic_size)) {
+    return Status::Invalid("Not an Arrow file");
+  }
+
+  int32_t footer_length = *reinterpret_cast<const int32_t*>(buffer->data());
+
+  if (footer_length <= 0 || footer_length + magic_size * 2 + 4 > footer_offset_) {
+    return Status::Invalid("File is smaller than indicated metadata size");
+  }
+
+  // Now read the footer
+  RETURN_NOT_OK(file_->ReadAt(
+      footer_offset_ - footer_length - file_end_size, footer_length, &buffer));
+  RETURN_NOT_OK(FileFooter::Open(buffer, &footer_));
+
+  // Get the schema
+  return footer_->GetSchema(&schema_);
+}
+
+const std::shared_ptr<Schema>& FileReader::schema() const {
+  return schema_;
+}
+
+int FileReader::num_dictionaries() const {
+  return footer_->num_dictionaries();
+}
+
+int FileReader::num_record_batches() const {
+  return footer_->num_record_batches();
+}
+
+MetadataVersion::type FileReader::version() const {
+  return footer_->version();
+}
+
+Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
+  DCHECK_GE(i, 0);
+  DCHECK_LT(i, num_record_batches());
+  FileBlock block = footer_->record_batch(i);
+  int64_t metadata_end_offset = block.offset + block.body_length + block.metadata_length;
+
+  std::shared_ptr<RecordBatchReader> reader;
+  RETURN_NOT_OK(RecordBatchReader::Open(file_.get(), metadata_end_offset, &reader));
+
+  return reader->GetRecordBatch(schema_, batch);
+}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/file.h b/cpp/src/arrow/ipc/file.h
new file mode 100644
index 0000000000000..4b79c98281bbc
--- /dev/null
+++ b/cpp/src/arrow/ipc/file.h
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Implement Arrow file layout for IPC/RPC purposes and short-lived storage
+
+#ifndef ARROW_IPC_FILE_H
+#define ARROW_IPC_FILE_H
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "arrow/ipc/metadata.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+class Buffer;
+struct Field;
+class RecordBatch;
+class Schema;
+class Status;
+
+namespace io {
+
+class OutputStream;
+class ReadableFileInterface;
+
+}  // namespace io
+
+namespace ipc {
+
+class ARROW_EXPORT FileWriter {
+ public:
+  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+      std::shared_ptr<FileWriter>* out);
+
+  // TODO(wesm): Write dictionaries
+
+  Status WriteRecordBatch(
+      const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows);
+
+  Status Close();
+
+ private:
+  FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema);
+
+  Status CheckStarted();
+  Status Start();
+
+  Status UpdatePosition();
+
+  // Adds padding bytes if necessary to ensure all memory blocks are written on
+  // 8-byte boundaries.
+  Status Align();
+
+  // Write data and update position
+  Status Write(const uint8_t* data, int64_t nbytes);
+
+  // Write and align
+  Status WriteAligned(const uint8_t* data, int64_t nbytes);
+
+  io::OutputStream* sink_;
+  std::shared_ptr<Schema> schema_;
+  int64_t position_;
+  bool started_;
+
+  std::vector<FileBlock> dictionaries_;
+  std::vector<FileBlock> record_batches_;
+};
+
+class ARROW_EXPORT FileReader {
+ public:
+  ~FileReader();
+
+  // Open a file-like object that is assumed to be self-contained; i.e., the
+  // end of the file interface is the end of the Arrow file. Note that there
+  // can be any amount of data preceding the Arrow-formatted data, because we
+  // need only locate the end of the Arrow file stream to discover the metadata
+  // and then proceed to read the data into memory.
+  static Status Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+      std::shared_ptr<FileReader>* reader);
+
+  // If the file is embedded within some larger file or memory region, you can
+  // pass the absolute memory offset to the end of the file (which contains the
+  // metadata footer). The metadata must have been written with memory offsets
+  // relative to the start of the containing file
+  //
+  // @param file: the data source
+  // @param footer_offset: the position of the end of the Arrow "file"
+  static Status Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+      int64_t footer_offset, std::shared_ptr<FileReader>* reader);
+
+  const std::shared_ptr<Schema>& schema() const;
+
+  // Shared dictionaries for dictionary-encoding cross record batches
+  // TODO(wesm): Implement dictionary reading when we also have dictionary
+  // encoding
+  int num_dictionaries() const;
+
+  int num_record_batches() const;
+
+  MetadataVersion::type version() const;
+
+  // Read a record batch from the file. Does not copy memory if the input
+  // source supports zero-copy.
+  //
+  // TODO(wesm): Make the copy/zero-copy behavior configurable (e.g. provide an
+  // "always copy" option)
+  Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch);
+
+ private:
+  FileReader(
+      const std::shared_ptr<io::ReadableFileInterface>& file, int64_t footer_offset);
+
+  Status ReadFooter();
+
+  std::shared_ptr<io::ReadableFileInterface> file_;
+
+  // The location where the Arrow file layout ends. May be the end of the file
+  // or some other location if embedded in a larger file.
+  int64_t footer_offset_;
+
+  std::unique_ptr<FileFooter> footer_;
+  std::shared_ptr<Schema> schema_;
+};
+
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_FILE_H
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index ca4d0152b9060..f5611d4840c97 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -43,31 +43,27 @@
 namespace arrow {
 namespace ipc {
 
-// TODO(emkornfield) convert to google style kInt32, etc?
-const auto INT32 = std::make_shared<Int32Type>();
-const auto LIST_INT32 = std::make_shared<ListType>(INT32);
-const auto LIST_LIST_INT32 = std::make_shared<ListType>(LIST_INT32);
-
-typedef Status MakeRowBatch(std::shared_ptr<RowBatch>* out);
-
-class TestWriteRowBatch : public ::testing::TestWithParam<MakeRowBatch*>,
-                          public io::MemoryMapFixture {
+class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
+                             public io::MemoryMapFixture {
  public:
   void SetUp() { pool_ = default_memory_pool(); }
   void TearDown() { io::MemoryMapFixture::TearDown(); }
 
-  Status RoundTripHelper(const RowBatch& batch, int memory_map_size,
-      std::shared_ptr<RowBatch>* batch_result) {
+  Status RoundTripHelper(const RecordBatch& batch, int memory_map_size,
+      std::shared_ptr<RecordBatch>* batch_result) {
     std::string path = "test-write-row-batch";
     io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
-    int64_t header_location;
 
-    RETURN_NOT_OK(WriteRowBatch(mmap_.get(), &batch, &header_location));
+    int64_t body_end_offset;
+    int64_t header_end_offset;
 
-    std::shared_ptr<RowBatchReader> reader;
-    RETURN_NOT_OK(RowBatchReader::Open(mmap_.get(), header_location, &reader));
+    RETURN_NOT_OK(WriteRecordBatch(batch.columns(), batch.num_rows(), mmap_.get(),
+        &body_end_offset, &header_end_offset));
 
-    RETURN_NOT_OK(reader->GetRowBatch(batch.schema(), batch_result));
+    std::shared_ptr<RecordBatchReader> reader;
+    RETURN_NOT_OK(RecordBatchReader::Open(mmap_.get(), header_end_offset, &reader));
+
+    RETURN_NOT_OK(reader->GetRecordBatch(batch.schema(), batch_result));
     return Status::OK();
   }
 
@@ -76,10 +72,10 @@ class TestWriteRowBatch : public ::testing::TestWithParam<MakeRowBatch*>,
   MemoryPool* pool_;
 };
 
-TEST_P(TestWriteRowBatch, RoundTrip) {
-  std::shared_ptr<RowBatch> batch;
+TEST_P(TestWriteRecordBatch, RoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
-  std::shared_ptr<RowBatch> batch_result;
+  std::shared_ptr<RecordBatch> batch_result;
   ASSERT_OK(RoundTripHelper(*batch, 1 << 16, &batch_result));
 
   // do checks
@@ -93,217 +89,39 @@ TEST_P(TestWriteRowBatch, RoundTrip) {
   }
 }
 
-Status MakeIntRowBatch(std::shared_ptr<RowBatch>* out) {
-  const int length = 1000;
-
-  // Make the schema
-  auto f0 = std::make_shared<Field>("f0", INT32);
-  auto f1 = std::make_shared<Field>("f1", INT32);
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
-
-  // Example data
-  std::shared_ptr<Array> a0, a1;
-  MemoryPool* pool = default_memory_pool();
-  RETURN_NOT_OK(MakeRandomInt32Array(length, false, pool, &a0));
-  RETURN_NOT_OK(MakeRandomInt32Array(length, true, pool, &a1));
-  out->reset(new RowBatch(schema, length, {a0, a1}));
-  return Status::OK();
-}
+INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRecordBatch,
+    ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch,
+                            &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,
+                            &MakeStringTypesRecordBatch, &MakeStruct));
 
-template <class Builder, class RawType>
-Status MakeRandomBinaryArray(
-    const TypePtr& type, int32_t length, MemoryPool* pool, ArrayPtr* array) {
-  const std::vector<std::string> values = {
-      "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
-  Builder builder(pool, type);
-  const auto values_len = values.size();
-  for (int32_t i = 0; i < length; ++i) {
-    int values_index = i % values_len;
-    if (values_index == 0) {
-      RETURN_NOT_OK(builder.AppendNull());
-    } else {
-      const std::string& value = values[values_index];
-      RETURN_NOT_OK(
-          builder.Append(reinterpret_cast<const RawType*>(value.data()), value.size()));
-    }
-  }
-  *array = builder.Finish();
-  return Status::OK();
-}
-
-Status MakeStringTypesRowBatch(std::shared_ptr<RowBatch>* out) {
-  const int32_t length = 500;
-  auto string_type = std::make_shared<StringType>();
-  auto binary_type = std::make_shared<BinaryType>();
-  auto f0 = std::make_shared<Field>("f0", string_type);
-  auto f1 = std::make_shared<Field>("f1", binary_type);
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
-
-  std::shared_ptr<Array> a0, a1;
-  MemoryPool* pool = default_memory_pool();
-
-  {
-    auto status =
-        MakeRandomBinaryArray<StringBuilder, char>(string_type, length, pool, &a0);
-    RETURN_NOT_OK(status);
-  }
-  {
-    auto status =
-        MakeRandomBinaryArray<BinaryBuilder, uint8_t>(binary_type, length, pool, &a1);
-    RETURN_NOT_OK(status);
-  }
-  out->reset(new RowBatch(schema, length, {a0, a1}));
-  return Status::OK();
-}
-
-Status MakeListRowBatch(std::shared_ptr<RowBatch>* out) {
-  // Make the schema
-  auto f0 = std::make_shared<Field>("f0", LIST_INT32);
-  auto f1 = std::make_shared<Field>("f1", LIST_LIST_INT32);
-  auto f2 = std::make_shared<Field>("f2", INT32);
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
-
-  // Example data
-
-  MemoryPool* pool = default_memory_pool();
-  const int length = 200;
-  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
-  const bool include_nulls = true;
-  RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool, &leaf_values));
-  RETURN_NOT_OK(
-      MakeRandomListArray(leaf_values, length, include_nulls, pool, &list_array));
-  RETURN_NOT_OK(
-      MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array));
-  RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array));
-  out->reset(new RowBatch(schema, length, {list_array, list_list_array, flat_array}));
-  return Status::OK();
-}
-
-Status MakeZeroLengthRowBatch(std::shared_ptr<RowBatch>* out) {
-  // Make the schema
-  auto f0 = std::make_shared<Field>("f0", LIST_INT32);
-  auto f1 = std::make_shared<Field>("f1", LIST_LIST_INT32);
-  auto f2 = std::make_shared<Field>("f2", INT32);
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
-
-  // Example data
-  MemoryPool* pool = default_memory_pool();
-  const int length = 200;
-  const bool include_nulls = true;
-  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
-  RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &leaf_values));
-  RETURN_NOT_OK(MakeRandomListArray(leaf_values, 0, include_nulls, pool, &list_array));
-  RETURN_NOT_OK(
-      MakeRandomListArray(list_array, 0, include_nulls, pool, &list_list_array));
-  RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &flat_array));
-  out->reset(new RowBatch(schema, length, {list_array, list_list_array, flat_array}));
-  return Status::OK();
-}
-
-Status MakeNonNullRowBatch(std::shared_ptr<RowBatch>* out) {
-  // Make the schema
-  auto f0 = std::make_shared<Field>("f0", LIST_INT32);
-  auto f1 = std::make_shared<Field>("f1", LIST_LIST_INT32);
-  auto f2 = std::make_shared<Field>("f2", INT32);
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
-
-  // Example data
-  MemoryPool* pool = default_memory_pool();
-  const int length = 50;
-  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
-
-  RETURN_NOT_OK(MakeRandomInt32Array(1000, true, pool, &leaf_values));
-  bool include_nulls = false;
-  RETURN_NOT_OK(
-      MakeRandomListArray(leaf_values, length, include_nulls, pool, &list_array));
-  RETURN_NOT_OK(
-      MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array));
-  RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array));
-  out->reset(new RowBatch(schema, length, {list_array, list_list_array, flat_array}));
-  return Status::OK();
-}
-
-Status MakeDeeplyNestedList(std::shared_ptr<RowBatch>* out) {
-  const int batch_length = 5;
-  TypePtr type = INT32;
-
-  MemoryPool* pool = default_memory_pool();
-  ArrayPtr array;
-  const bool include_nulls = true;
-  RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool, &array));
-  for (int i = 0; i < 63; ++i) {
-    type = std::static_pointer_cast<DataType>(std::make_shared<ListType>(type));
-    RETURN_NOT_OK(MakeRandomListArray(array, batch_length, include_nulls, pool, &array));
-  }
-
-  auto f0 = std::make_shared<Field>("f0", type);
-  std::shared_ptr<Schema> schema(new Schema({f0}));
-  std::vector<ArrayPtr> arrays = {array};
-  out->reset(new RowBatch(schema, batch_length, arrays));
-  return Status::OK();
-}
-
-Status MakeStruct(std::shared_ptr<RowBatch>* out) {
-  // reuse constructed list columns
-  std::shared_ptr<RowBatch> list_batch;
-  RETURN_NOT_OK(MakeListRowBatch(&list_batch));
-  std::vector<ArrayPtr> columns = {
-      list_batch->column(0), list_batch->column(1), list_batch->column(2)};
-  auto list_schema = list_batch->schema();
-
-  // Define schema
-  std::shared_ptr<DataType> type(new StructType(
-      {list_schema->field(0), list_schema->field(1), list_schema->field(2)}));
-  auto f0 = std::make_shared<Field>("non_null_struct", type);
-  auto f1 = std::make_shared<Field>("null_struct", type);
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
-
-  // construct individual nullable/non-nullable struct arrays
-  ArrayPtr no_nulls(new StructArray(type, list_batch->num_rows(), columns));
-  std::vector<uint8_t> null_bytes(list_batch->num_rows(), 1);
-  null_bytes[0] = 0;
-  std::shared_ptr<Buffer> null_bitmask;
-  RETURN_NOT_OK(util::bytes_to_bits(null_bytes, &null_bitmask));
-  ArrayPtr with_nulls(
-      new StructArray(type, list_batch->num_rows(), columns, 1, null_bitmask));
-
-  // construct batch
-  std::vector<ArrayPtr> arrays = {no_nulls, with_nulls};
-  out->reset(new RowBatch(schema, list_batch->num_rows(), arrays));
-  return Status::OK();
-}
-
-INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRowBatch,
-    ::testing::Values(&MakeIntRowBatch, &MakeListRowBatch, &MakeNonNullRowBatch,
-                            &MakeZeroLengthRowBatch, &MakeDeeplyNestedList,
-                            &MakeStringTypesRowBatch, &MakeStruct));
-
-void TestGetRowBatchSize(std::shared_ptr<RowBatch> batch) {
+void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   ipc::MockOutputStream mock;
-  int64_t mock_header_location = -1;
+  int64_t mock_header_offset = -1;
+  int64_t mock_body_offset = -1;
   int64_t size = -1;
-  ASSERT_OK(WriteRowBatch(&mock, batch.get(), &mock_header_location));
-  ASSERT_OK(GetRowBatchSize(batch.get(), &size));
+  ASSERT_OK(WriteRecordBatch(batch->columns(), batch->num_rows(), &mock,
+      &mock_body_offset, &mock_header_offset));
+  ASSERT_OK(GetRecordBatchSize(batch.get(), &size));
   ASSERT_EQ(mock.GetExtentBytesWritten(), size);
 }
 
-TEST_F(TestWriteRowBatch, IntegerGetRowBatchSize) {
-  std::shared_ptr<RowBatch> batch;
+TEST_F(TestWriteRecordBatch, IntegerGetRecordBatchSize) {
+  std::shared_ptr<RecordBatch> batch;
 
-  ASSERT_OK(MakeIntRowBatch(&batch));
-  TestGetRowBatchSize(batch);
+  ASSERT_OK(MakeIntRecordBatch(&batch));
+  TestGetRecordBatchSize(batch);
 
-  ASSERT_OK(MakeListRowBatch(&batch));
-  TestGetRowBatchSize(batch);
+  ASSERT_OK(MakeListRecordBatch(&batch));
+  TestGetRecordBatchSize(batch);
 
-  ASSERT_OK(MakeZeroLengthRowBatch(&batch));
-  TestGetRowBatchSize(batch);
+  ASSERT_OK(MakeZeroLengthRecordBatch(&batch));
+  TestGetRecordBatchSize(batch);
 
-  ASSERT_OK(MakeNonNullRowBatch(&batch));
-  TestGetRowBatchSize(batch);
+  ASSERT_OK(MakeNonNullRecordBatch(&batch));
+  TestGetRecordBatchSize(batch);
 
   ASSERT_OK(MakeDeeplyNestedList(&batch));
-  TestGetRowBatchSize(batch);
+  TestGetRecordBatchSize(batch);
 }
 
 class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
@@ -314,7 +132,7 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
   Status WriteToMmap(int recursion_level, bool override_level,
       int64_t* header_out = nullptr, std::shared_ptr<Schema>* schema_out = nullptr) {
     const int batch_length = 5;
-    TypePtr type = INT32;
+    TypePtr type = kInt32;
     ArrayPtr array;
     const bool include_nulls = true;
     RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
@@ -328,18 +146,22 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
     std::shared_ptr<Schema> schema(new Schema({f0}));
     if (schema_out != nullptr) { *schema_out = schema; }
     std::vector<ArrayPtr> arrays = {array};
-    auto batch = std::make_shared<RowBatch>(schema, batch_length, arrays);
+    auto batch = std::make_shared<RecordBatch>(schema, batch_length, arrays);
 
     std::string path = "test-write-past-max-recursion";
     const int memory_map_size = 1 << 16;
     io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
-    int64_t header_location;
-    int64_t* header_out_param = header_out == nullptr ? &header_location : header_out;
+
+    int64_t body_offset;
+    int64_t header_offset;
+
+    int64_t* header_out_param = header_out == nullptr ? &header_offset : header_out;
     if (override_level) {
-      return WriteRowBatch(
-          mmap_.get(), batch.get(), header_out_param, recursion_level + 1);
+      return WriteRecordBatch(batch->columns(), batch->num_rows(), mmap_.get(),
+          &body_offset, header_out_param, recursion_level + 1);
     } else {
-      return WriteRowBatch(mmap_.get(), batch.get(), header_out_param);
+      return WriteRecordBatch(batch->columns(), batch->num_rows(), mmap_.get(),
+          &body_offset, header_out_param);
     }
   }
 
@@ -353,14 +175,14 @@ TEST_F(RecursionLimits, WriteLimit) {
 }
 
 TEST_F(RecursionLimits, ReadLimit) {
-  int64_t header_location = -1;
+  int64_t header_offset = -1;
   std::shared_ptr<Schema> schema;
-  ASSERT_OK(WriteToMmap(64, true, &header_location, &schema));
+  ASSERT_OK(WriteToMmap(64, true, &header_offset, &schema));
 
-  std::shared_ptr<RowBatchReader> reader;
-  ASSERT_OK(RowBatchReader::Open(mmap_.get(), header_location, &reader));
-  std::shared_ptr<RowBatch> batch_result;
-  ASSERT_RAISES(Invalid, reader->GetRowBatch(schema, &batch_result));
+  std::shared_ptr<RecordBatchReader> reader;
+  ASSERT_OK(RecordBatchReader::Open(mmap_.get(), header_offset, &reader));
+  std::shared_ptr<RecordBatch> batch_result;
+  ASSERT_RAISES(Invalid, reader->GetRecordBatch(schema, &batch_result));
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
new file mode 100644
index 0000000000000..cd424bf385cae
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/io/memory.h"
+#include "arrow/io/test-common.h"
+#include "arrow/ipc/adapter.h"
+#include "arrow/ipc/file.h"
+#include "arrow/ipc/test-common.h"
+#include "arrow/ipc/util.h"
+
+#include "arrow/test-util.h"
+#include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+    buffer_ = std::make_shared<PoolBuffer>(pool_);
+    sink_.reset(new io::BufferOutputStream(buffer_));
+  }
+  void TearDown() {}
+
+  Status RoundTripHelper(
+      const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
+    // Write the file
+    RETURN_NOT_OK(FileWriter::Open(sink_.get(), batch.schema(), &file_writer_));
+    int num_batches = 3;
+    for (int i = 0; i < num_batches; ++i) {
+      RETURN_NOT_OK(file_writer_->WriteRecordBatch(batch.columns(), batch.num_rows()));
+    }
+    RETURN_NOT_OK(file_writer_->Close());
+
+    // Current offset into stream is the end of the file
+    int64_t footer_offset;
+    RETURN_NOT_OK(sink_->Tell(&footer_offset));
+
+    // Open the file
+    auto reader = std::make_shared<io::BufferReader>(buffer_->data(), buffer_->size());
+    RETURN_NOT_OK(FileReader::Open(reader, footer_offset, &file_reader_));
+
+    EXPECT_EQ(num_batches, file_reader_->num_record_batches());
+
+    out_batches->resize(num_batches);
+    for (int i = 0; i < num_batches; ++i) {
+      RETURN_NOT_OK(file_reader_->GetRecordBatch(i, &(*out_batches)[i]));
+    }
+
+    return Status::OK();
+  }
+
+  void CompareBatch(const RecordBatch* left, const RecordBatch* right) {
+    ASSERT_TRUE(left->schema()->Equals(right->schema()));
+    ASSERT_EQ(left->num_columns(), right->num_columns())
+        << left->schema()->ToString() << " result: " << right->schema()->ToString();
+    EXPECT_EQ(left->num_rows(), right->num_rows());
+    for (int i = 0; i < left->num_columns(); ++i) {
+      EXPECT_TRUE(left->column(i)->Equals(right->column(i)))
+          << "Idx: " << i << " Name: " << left->column_name(i);
+    }
+  }
+
+ protected:
+  MemoryPool* pool_;
+
+  std::unique_ptr<io::BufferOutputStream> sink_;
+  std::shared_ptr<PoolBuffer> buffer_;
+
+  std::shared_ptr<FileWriter> file_writer_;
+  std::shared_ptr<FileReader> file_reader_;
+};
+
+TEST_P(TestFileFormat, RoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
+
+  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+
+  ASSERT_OK(RoundTripHelper(*batch, &out_batches));
+
+  // Compare batches. Same
+  for (size_t i = 0; i < out_batches.size(); ++i) {
+    CompareBatch(batch.get(), out_batches[i].get());
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(RoundTripTests, TestFileFormat,
+    ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch,
+                            &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,
+                            &MakeStringTypesRecordBatch, &MakeStruct));
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
index 51d79cfb4c4bb..1dc3969233237 100644
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -21,6 +21,7 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/io/memory.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/schema.h"
 #include "arrow/test-util.h"
@@ -31,6 +32,8 @@ namespace arrow {
 
 class Buffer;
 
+namespace ipc {
+
 static inline void assert_schema_equal(const Schema* lhs, const Schema* rhs) {
   if (!lhs->Equals(*rhs)) {
     std::stringstream ss;
@@ -46,14 +49,14 @@ class TestSchemaMessage : public ::testing::Test {
 
   void CheckRoundtrip(const Schema* schema) {
     std::shared_ptr<Buffer> buffer;
-    ASSERT_OK(ipc::WriteSchema(schema, &buffer));
+    ASSERT_OK(WriteSchema(schema, &buffer));
 
-    std::shared_ptr<ipc::Message> message;
-    ASSERT_OK(ipc::Message::Open(buffer, &message));
+    std::shared_ptr<Message> message;
+    ASSERT_OK(Message::Open(buffer, &message));
 
-    ASSERT_EQ(ipc::Message::SCHEMA, message->type());
+    ASSERT_EQ(Message::SCHEMA, message->type());
 
-    std::shared_ptr<ipc::SchemaMessage> schema_msg = message->GetSchema();
+    std::shared_ptr<SchemaMessage> schema_msg = message->GetSchema();
     ASSERT_EQ(schema->num_fields(), schema_msg->num_fields());
 
     std::shared_ptr<Schema> schema2;
@@ -94,4 +97,68 @@ TEST_F(TestSchemaMessage, NestedFields) {
   CheckRoundtrip(&schema);
 }
 
+class TestFileFooter : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  void CheckRoundtrip(const Schema* schema, const std::vector<FileBlock>& dictionaries,
+      const std::vector<FileBlock>& record_batches) {
+    auto buffer = std::make_shared<PoolBuffer>();
+    io::BufferOutputStream stream(buffer);
+
+    ASSERT_OK(WriteFileFooter(schema, dictionaries, record_batches, &stream));
+
+    std::unique_ptr<FileFooter> footer;
+    ASSERT_OK(FileFooter::Open(buffer, &footer));
+
+    ASSERT_EQ(MetadataVersion::V1_SNAPSHOT, footer->version());
+
+    // Check schema
+    std::shared_ptr<Schema> schema2;
+    ASSERT_OK(footer->GetSchema(&schema2));
+    assert_schema_equal(schema, schema2.get());
+
+    // Check blocks
+    ASSERT_EQ(dictionaries.size(), footer->num_dictionaries());
+    ASSERT_EQ(record_batches.size(), footer->num_record_batches());
+
+    for (int i = 0; i < footer->num_dictionaries(); ++i) {
+      CheckBlocks(dictionaries[i], footer->dictionary(i));
+    }
+
+    for (int i = 0; i < footer->num_record_batches(); ++i) {
+      CheckBlocks(record_batches[i], footer->record_batch(i));
+    }
+  }
+
+  void CheckBlocks(const FileBlock& left, const FileBlock& right) {
+    ASSERT_EQ(left.offset, right.offset);
+    ASSERT_EQ(left.metadata_length, right.metadata_length);
+    ASSERT_EQ(left.body_length, right.body_length);
+  }
+
+ private:
+  std::shared_ptr<Schema> example_schema_;
+};
+
+TEST_F(TestFileFooter, Basics) {
+  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>());
+  Schema schema({f0, f1});
+
+  std::vector<FileBlock> dictionaries;
+  dictionaries.emplace_back(8, 92, 900);
+  dictionaries.emplace_back(1000, 100, 1900);
+  dictionaries.emplace_back(3000, 100, 2900);
+
+  std::vector<FileBlock> record_batches;
+  record_batches.emplace_back(6000, 100, 900);
+  record_batches.emplace_back(7000, 100, 1900);
+  record_batches.emplace_back(9000, 100, 2900);
+  record_batches.emplace_back(12000, 100, 3900);
+
+  CheckRoundtrip(&schema, dictionaries, record_batches);
+}
+
+}  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 05e9c7ad4d359..7102012c29a84 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -31,10 +31,6 @@
 #include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
 
-typedef flatbuffers::FlatBufferBuilder FBB;
-typedef flatbuffers::Offset<arrow::flatbuf::Field> FieldOffset;
-typedef flatbuffers::Offset<void> Offset;
-
 namespace arrow {
 
 namespace flatbuf = org::apache::arrow::flatbuf;
@@ -52,6 +48,8 @@ const std::shared_ptr<DataType> UINT32 = std::make_shared<UInt32Type>();
 const std::shared_ptr<DataType> UINT64 = std::make_shared<UInt64Type>();
 const std::shared_ptr<DataType> FLOAT = std::make_shared<FloatType>();
 const std::shared_ptr<DataType> DOUBLE = std::make_shared<DoubleType>();
+const std::shared_ptr<DataType> STRING = std::make_shared<StringType>();
+const std::shared_ptr<DataType> BINARY = std::make_shared<BinaryType>();
 
 static Status IntFromFlatbuffer(
     const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
@@ -102,8 +100,11 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       return FloatFromFlatuffer(
           static_cast<const flatbuf::FloatingPoint*>(type_data), out);
     case flatbuf::Type_Binary:
+      *out = BINARY;
+      return Status::OK();
     case flatbuf::Type_Utf8:
-      return Status::NotImplemented("Type is not implemented");
+      *out = STRING;
+      return Status::OK();
     case flatbuf::Type_Bool:
       *out = BOOL;
       return Status::OK();
@@ -193,6 +194,14 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *out_type = flatbuf::Type_FloatingPoint;
       *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
       break;
+    case Type::BINARY:
+      *out_type = flatbuf::Type_Binary;
+      *offset = flatbuf::CreateBinary(fbb).Union();
+      break;
+    case Type::STRING:
+      *out_type = flatbuf::Type_Utf8;
+      *offset = flatbuf::CreateUtf8(fbb).Union();
+      break;
     case Type::LIST:
       *out_type = flatbuf::Type_List;
       return ListToFlatbuffer(fbb, type, children, offset);
@@ -255,19 +264,26 @@ flatbuf::Endianness endianness() {
   return bint.c[0] == 1 ? flatbuf::Endianness_Big : flatbuf::Endianness_Little;
 }
 
-Status MessageBuilder::SetSchema(const Schema* schema) {
-  header_type_ = flatbuf::MessageHeader_Schema;
-
+Status SchemaToFlatbuffer(
+    FBB& fbb, const Schema* schema, flatbuffers::Offset<flatbuf::Schema>* out) {
   std::vector<FieldOffset> field_offsets;
   for (int i = 0; i < schema->num_fields(); ++i) {
     const std::shared_ptr<Field>& field = schema->field(i);
     FieldOffset offset;
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb_, field, &offset));
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, &offset));
     field_offsets.push_back(offset);
   }
 
-  header_ =
-      flatbuf::CreateSchema(fbb_, endianness(), fbb_.CreateVector(field_offsets)).Union();
+  *out = flatbuf::CreateSchema(fbb, endianness(), fbb.CreateVector(field_offsets));
+  return Status::OK();
+}
+
+Status MessageBuilder::SetSchema(const Schema* schema) {
+  flatbuffers::Offset<flatbuf::Schema> fb_schema;
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb_, schema, &fb_schema));
+
+  header_type_ = flatbuf::MessageHeader_Schema;
+  header_ = fb_schema.Union();
   body_length_ = 0;
   return Status::OK();
 }
@@ -301,17 +317,17 @@ Status MessageBuilder::Finish() {
 }
 
 Status MessageBuilder::GetBuffer(std::shared_ptr<Buffer>* out) {
-  // The message buffer is prefixed by the size of the complete flatbuffer as
+  // The message buffer is suffixed by the size of the complete flatbuffer as
   // int32_t
-  // <int32_t: flatbuffer size><uint8_t*: flatbuffer data>
+  // <uint8_t*: flatbuffer data><int32_t: flatbuffer size>
   int32_t size = fbb_.GetSize();
 
   auto result = std::make_shared<PoolBuffer>();
   RETURN_NOT_OK(result->Resize(size + sizeof(int32_t)));
 
   uint8_t* dst = result->mutable_data();
-  memcpy(dst, reinterpret_cast<int32_t*>(&size), sizeof(int32_t));
-  memcpy(dst + sizeof(int32_t), fbb_.GetBufferPointer(), size);
+  memcpy(dst, fbb_.GetBufferPointer(), size);
+  memcpy(dst + size, reinterpret_cast<int32_t*>(&size), sizeof(int32_t));
 
   *out = result;
   return Status::OK();
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index d38df840ba05e..c404cfde22ca3 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -24,7 +24,9 @@
 
 #include "flatbuffers/flatbuffers.h"
 
+#include "arrow/ipc/File_generated.h"
 #include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/metadata.h"
 
 namespace arrow {
 
@@ -37,11 +39,18 @@ class Status;
 
 namespace ipc {
 
+using FBB = flatbuffers::FlatBufferBuilder;
+using FieldOffset = flatbuffers::Offset<arrow::flatbuf::Field>;
+using Offset = flatbuffers::Offset<void>;
+
 static constexpr flatbuf::MetadataVersion kMetadataVersion =
     flatbuf::MetadataVersion_V1_SNAPSHOT;
 
 Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out);
 
+Status SchemaToFlatbuffer(
+    FBB& fbb, const Schema* schema, flatbuffers::Offset<flatbuf::Schema>* out);
+
 class MessageBuilder {
  public:
   Status SetSchema(const Schema* schema);
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index e510755110e04..66df8a6711fa9 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -23,7 +23,8 @@
 
 #include "flatbuffers/flatbuffers.h"
 
-// Generated C++ flatbuffer IDL
+#include "arrow/io/interfaces.h"
+#include "arrow/ipc/File_generated.h"
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/ipc/metadata-internal.h"
 
@@ -47,9 +48,10 @@ Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out) {
 //----------------------------------------------------------------------
 // Message reader
 
-class Message::Impl {
+class Message::MessageImpl {
  public:
-  explicit Impl(const std::shared_ptr<Buffer>& buffer, const flatbuf::Message* message)
+  explicit MessageImpl(
+      const std::shared_ptr<Buffer>& buffer, const flatbuf::Message* message)
       : buffer_(buffer), message_(message) {}
 
   Message::Type type() const {
@@ -76,31 +78,16 @@ class Message::Impl {
   const flatbuf::Message* message_;
 };
 
-class SchemaMessage::Impl {
- public:
-  explicit Impl(const void* schema)
-      : schema_(static_cast<const flatbuf::Schema*>(schema)) {}
-
-  const flatbuf::Field* field(int i) const { return schema_->fields()->Get(i); }
-
-  int num_fields() const { return schema_->fields()->size(); }
-
- private:
-  const flatbuf::Schema* schema_;
-};
-
 Message::Message() {}
 
 Status Message::Open(
     const std::shared_ptr<Buffer>& buffer, std::shared_ptr<Message>* out) {
   std::shared_ptr<Message> result(new Message());
 
-  // The buffer is prefixed by its size as int32_t
-  const uint8_t* fb_head = buffer->data() + sizeof(int32_t);
-  const flatbuf::Message* message = flatbuf::GetMessage(fb_head);
+  const flatbuf::Message* message = flatbuf::GetMessage(buffer->data());
 
   // TODO(wesm): verify message
-  result->impl_.reset(new Impl(buffer, message));
+  result->impl_.reset(new MessageImpl(buffer, message));
   *out = result;
 
   return Status::OK();
@@ -122,10 +109,26 @@ std::shared_ptr<SchemaMessage> Message::GetSchema() {
   return std::make_shared<SchemaMessage>(this->shared_from_this(), impl_->header());
 }
 
+// ----------------------------------------------------------------------
+// SchemaMessage
+
+class SchemaMessage::SchemaMessageImpl {
+ public:
+  explicit SchemaMessageImpl(const void* schema)
+      : schema_(static_cast<const flatbuf::Schema*>(schema)) {}
+
+  const flatbuf::Field* field(int i) const { return schema_->fields()->Get(i); }
+
+  int num_fields() const { return schema_->fields()->size(); }
+
+ private:
+  const flatbuf::Schema* schema_;
+};
+
 SchemaMessage::SchemaMessage(
     const std::shared_ptr<Message>& message, const void* schema) {
   message_ = message;
-  impl_.reset(new Impl(schema));
+  impl_.reset(new SchemaMessageImpl(schema));
 }
 
 int SchemaMessage::num_fields() const {
@@ -146,9 +149,12 @@ Status SchemaMessage::GetSchema(std::shared_ptr<Schema>* out) const {
   return Status::OK();
 }
 
-class RecordBatchMessage::Impl {
+// ----------------------------------------------------------------------
+// RecordBatchMessage
+
+class RecordBatchMessage::RecordBatchMessageImpl {
  public:
-  explicit Impl(const void* batch)
+  explicit RecordBatchMessageImpl(const void* batch)
       : batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
     nodes_ = batch_->nodes();
     buffers_ = batch_->buffers();
@@ -177,7 +183,7 @@ std::shared_ptr<RecordBatchMessage> Message::GetRecordBatch() {
 RecordBatchMessage::RecordBatchMessage(
     const std::shared_ptr<Message>& message, const void* batch) {
   message_ = message;
-  impl_.reset(new Impl(batch));
+  impl_.reset(new RecordBatchMessageImpl(batch));
 }
 
 // TODO(wesm): Copying the flatbuffer data isn't great, but this will do for
@@ -213,5 +219,122 @@ int RecordBatchMessage::num_fields() const {
   return impl_->num_fields();
 }
 
+// ----------------------------------------------------------------------
+// File footer
+
+static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
+FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
+  std::vector<flatbuf::Block> fb_blocks;
+
+  for (const FileBlock& block : blocks) {
+    fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
+  }
+
+  return fbb.CreateVectorOfStructs(fb_blocks);
+}
+
+Status WriteFileFooter(const Schema* schema, const std::vector<FileBlock>& dictionaries,
+    const std::vector<FileBlock>& record_batches, io::OutputStream* out) {
+  FBB fbb;
+
+  flatbuffers::Offset<flatbuf::Schema> fb_schema;
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, &fb_schema));
+
+  auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
+  auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
+
+  auto footer = flatbuf::CreateFooter(
+      fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
+
+  fbb.Finish(footer);
+
+  int32_t size = fbb.GetSize();
+
+  return out->Write(fbb.GetBufferPointer(), size);
+}
+
+static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
+  return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
+}
+
+class FileFooter::FileFooterImpl {
+ public:
+  FileFooterImpl(const std::shared_ptr<Buffer>& buffer, const flatbuf::Footer* footer)
+      : buffer_(buffer), footer_(footer) {}
+
+  int num_dictionaries() const { return footer_->dictionaries()->size(); }
+
+  int num_record_batches() const { return footer_->recordBatches()->size(); }
+
+  MetadataVersion::type version() const {
+    switch (footer_->version()) {
+      case flatbuf::MetadataVersion_V1_SNAPSHOT:
+        return MetadataVersion::V1_SNAPSHOT;
+      // Add cases as other versions become available
+      default:
+        return MetadataVersion::V1_SNAPSHOT;
+    }
+  }
+
+  FileBlock record_batch(int i) const {
+    return FileBlockFromFlatbuffer(footer_->recordBatches()->Get(i));
+  }
+
+  FileBlock dictionary(int i) const {
+    return FileBlockFromFlatbuffer(footer_->dictionaries()->Get(i));
+  }
+
+  Status GetSchema(std::shared_ptr<Schema>* out) const {
+    auto schema_msg = std::make_shared<SchemaMessage>(nullptr, footer_->schema());
+    return schema_msg->GetSchema(out);
+  }
+
+ private:
+  // Retain reference to memory
+  std::shared_ptr<Buffer> buffer_;
+
+  const flatbuf::Footer* footer_;
+};
+
+FileFooter::FileFooter() {}
+
+FileFooter::~FileFooter() {}
+
+Status FileFooter::Open(
+    const std::shared_ptr<Buffer>& buffer, std::unique_ptr<FileFooter>* out) {
+  const flatbuf::Footer* footer = flatbuf::GetFooter(buffer->data());
+
+  *out = std::unique_ptr<FileFooter>(new FileFooter());
+
+  // TODO(wesm): Verify the footer
+  (*out)->impl_.reset(new FileFooterImpl(buffer, footer));
+
+  return Status::OK();
+}
+
+int FileFooter::num_dictionaries() const {
+  return impl_->num_dictionaries();
+}
+
+int FileFooter::num_record_batches() const {
+  return impl_->num_record_batches();
+}
+
+MetadataVersion::type FileFooter::version() const {
+  return impl_->version();
+}
+
+FileBlock FileFooter::record_batch(int i) const {
+  return impl_->record_batch(i);
+}
+
+FileBlock FileFooter::dictionary(int i) const {
+  return impl_->dictionary(i);
+}
+
+Status FileFooter::GetSchema(std::shared_ptr<Schema>* out) const {
+  return impl_->GetSchema(out);
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index d5ec53317e6f2..2f0e853bf97f0 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -22,6 +22,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <vector>
 
 #include "arrow/util/visibility.h"
 
@@ -32,17 +33,24 @@ struct Field;
 class Schema;
 class Status;
 
+namespace io {
+
+class OutputStream;
+
+}  // namespace io
+
 namespace ipc {
 
+struct MetadataVersion {
+  enum type { V1_SNAPSHOT };
+};
+
 //----------------------------------------------------------------------
-// Message read/write APIs
 
 // Serialize arrow::Schema as a Flatbuffer
 ARROW_EXPORT
 Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out);
 
-//----------------------------------------------------------------------
-
 // Read interface classes. We do not fully deserialize the flatbuffers so that
 // individual fields metadata can be retrieved from very large schema without
 //
@@ -68,8 +76,8 @@ class ARROW_EXPORT SchemaMessage {
   // Parent, owns the flatbuffer data
   std::shared_ptr<Message> message_;
 
-  class Impl;
-  std::unique_ptr<Impl> impl_;
+  class SchemaMessageImpl;
+  std::unique_ptr<SchemaMessageImpl> impl_;
 };
 
 // Field metadata
@@ -101,8 +109,8 @@ class ARROW_EXPORT RecordBatchMessage {
   // Parent, owns the flatbuffer data
   std::shared_ptr<Message> message_;
 
-  class Impl;
-  std::unique_ptr<Impl> impl_;
+  class RecordBatchMessageImpl;
+  std::unique_ptr<RecordBatchMessageImpl> impl_;
 };
 
 class ARROW_EXPORT DictionaryBatchMessage {
@@ -133,8 +141,46 @@ class ARROW_EXPORT Message : public std::enable_shared_from_this<Message> {
   Message();
 
   // Hide serialization details from user API
-  class Impl;
-  std::unique_ptr<Impl> impl_;
+  class MessageImpl;
+  std::unique_ptr<MessageImpl> impl_;
+};
+
+// ----------------------------------------------------------------------
+// File footer for file-like representation
+
+struct FileBlock {
+  FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
+      : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
+
+  int64_t offset;
+  int32_t metadata_length;
+  int64_t body_length;
+};
+
+ARROW_EXPORT
+Status WriteFileFooter(const Schema* schema, const std::vector<FileBlock>& dictionaries,
+    const std::vector<FileBlock>& record_batches, io::OutputStream* out);
+
+class ARROW_EXPORT FileFooter {
+ public:
+  ~FileFooter();
+
+  static Status Open(
+      const std::shared_ptr<Buffer>& buffer, std::unique_ptr<FileFooter>* out);
+
+  int num_dictionaries() const;
+  int num_record_batches() const;
+  MetadataVersion::type version() const;
+
+  FileBlock record_batch(int i) const;
+  FileBlock dictionary(int i) const;
+
+  Status GetSchema(std::shared_ptr<Schema>* out) const;
+
+ private:
+  FileFooter();
+  class FileFooterImpl;
+  std::unique_ptr<FileFooterImpl> impl_;
 };
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index f6582fc883bdc..7d02bc302f40e 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -25,21 +25,28 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/memory-pool.h"
 
 namespace arrow {
 namespace ipc {
 
+const auto kInt32 = std::make_shared<Int32Type>();
+const auto kListInt32 = std::make_shared<ListType>(kInt32);
+const auto kListListInt32 = std::make_shared<ListType>(kListInt32);
+
 Status MakeRandomInt32Array(
     int32_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* array) {
   std::shared_ptr<PoolBuffer> data;
   test::MakeRandomInt32PoolBuffer(length, pool, &data);
-  const auto INT32 = std::make_shared<Int32Type>();
-  Int32Builder builder(pool, INT32);
+  const auto kInt32 = std::make_shared<Int32Type>();
+  Int32Builder builder(pool, kInt32);
   if (include_nulls) {
     std::shared_ptr<PoolBuffer> valid_bytes;
     test::MakeRandomBytePoolBuffer(length, pool, &valid_bytes);
@@ -87,6 +94,188 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
   return (*array)->Validate();
 }
 
+typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
+
+Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
+  const int length = 1000;
+
+  // Make the schema
+  auto f0 = std::make_shared<Field>("f0", kInt32);
+  auto f1 = std::make_shared<Field>("f1", kInt32);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  // Example data
+  std::shared_ptr<Array> a0, a1;
+  MemoryPool* pool = default_memory_pool();
+  RETURN_NOT_OK(MakeRandomInt32Array(length, false, pool, &a0));
+  RETURN_NOT_OK(MakeRandomInt32Array(length, true, pool, &a1));
+  out->reset(new RecordBatch(schema, length, {a0, a1}));
+  return Status::OK();
+}
+
+template <class Builder, class RawType>
+Status MakeRandomBinaryArray(
+    const TypePtr& type, int32_t length, MemoryPool* pool, ArrayPtr* array) {
+  const std::vector<std::string> values = {
+      "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
+  Builder builder(pool, type);
+  const auto values_len = values.size();
+  for (int32_t i = 0; i < length; ++i) {
+    int values_index = i % values_len;
+    if (values_index == 0) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else {
+      const std::string& value = values[values_index];
+      RETURN_NOT_OK(
+          builder.Append(reinterpret_cast<const RawType*>(value.data()), value.size()));
+    }
+  }
+  *array = builder.Finish();
+  return Status::OK();
+}
+
+Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
+  const int32_t length = 500;
+  auto string_type = std::make_shared<StringType>();
+  auto binary_type = std::make_shared<BinaryType>();
+  auto f0 = std::make_shared<Field>("f0", string_type);
+  auto f1 = std::make_shared<Field>("f1", binary_type);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  std::shared_ptr<Array> a0, a1;
+  MemoryPool* pool = default_memory_pool();
+
+  {
+    auto status =
+        MakeRandomBinaryArray<StringBuilder, char>(string_type, length, pool, &a0);
+    RETURN_NOT_OK(status);
+  }
+  {
+    auto status =
+        MakeRandomBinaryArray<BinaryBuilder, uint8_t>(binary_type, length, pool, &a1);
+    RETURN_NOT_OK(status);
+  }
+  out->reset(new RecordBatch(schema, length, {a0, a1}));
+  return Status::OK();
+}
+
+Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out) {
+  // Make the schema
+  auto f0 = std::make_shared<Field>("f0", kListInt32);
+  auto f1 = std::make_shared<Field>("f1", kListListInt32);
+  auto f2 = std::make_shared<Field>("f2", kInt32);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+
+  // Example data
+
+  MemoryPool* pool = default_memory_pool();
+  const int length = 200;
+  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
+  const bool include_nulls = true;
+  RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool, &leaf_values));
+  RETURN_NOT_OK(
+      MakeRandomListArray(leaf_values, length, include_nulls, pool, &list_array));
+  RETURN_NOT_OK(
+      MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array));
+  RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array));
+  out->reset(new RecordBatch(schema, length, {list_array, list_list_array, flat_array}));
+  return Status::OK();
+}
+
+Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out) {
+  // Make the schema
+  auto f0 = std::make_shared<Field>("f0", kListInt32);
+  auto f1 = std::make_shared<Field>("f1", kListListInt32);
+  auto f2 = std::make_shared<Field>("f2", kInt32);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+
+  // Example data
+  MemoryPool* pool = default_memory_pool();
+  const int length = 200;
+  const bool include_nulls = true;
+  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
+  RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &leaf_values));
+  RETURN_NOT_OK(MakeRandomListArray(leaf_values, 0, include_nulls, pool, &list_array));
+  RETURN_NOT_OK(
+      MakeRandomListArray(list_array, 0, include_nulls, pool, &list_list_array));
+  RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &flat_array));
+  out->reset(new RecordBatch(schema, length, {list_array, list_list_array, flat_array}));
+  return Status::OK();
+}
+
+Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
+  // Make the schema
+  auto f0 = std::make_shared<Field>("f0", kListInt32);
+  auto f1 = std::make_shared<Field>("f1", kListListInt32);
+  auto f2 = std::make_shared<Field>("f2", kInt32);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+
+  // Example data
+  MemoryPool* pool = default_memory_pool();
+  const int length = 50;
+  std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
+
+  RETURN_NOT_OK(MakeRandomInt32Array(1000, true, pool, &leaf_values));
+  bool include_nulls = false;
+  RETURN_NOT_OK(
+      MakeRandomListArray(leaf_values, length, include_nulls, pool, &list_array));
+  RETURN_NOT_OK(
+      MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array));
+  RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array));
+  out->reset(new RecordBatch(schema, length, {list_array, list_list_array, flat_array}));
+  return Status::OK();
+}
+
+Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
+  const int batch_length = 5;
+  TypePtr type = kInt32;
+
+  MemoryPool* pool = default_memory_pool();
+  ArrayPtr array;
+  const bool include_nulls = true;
+  RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool, &array));
+  for (int i = 0; i < 63; ++i) {
+    type = std::static_pointer_cast<DataType>(std::make_shared<ListType>(type));
+    RETURN_NOT_OK(MakeRandomListArray(array, batch_length, include_nulls, pool, &array));
+  }
+
+  auto f0 = std::make_shared<Field>("f0", type);
+  std::shared_ptr<Schema> schema(new Schema({f0}));
+  std::vector<ArrayPtr> arrays = {array};
+  out->reset(new RecordBatch(schema, batch_length, arrays));
+  return Status::OK();
+}
+
+Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
+  // reuse constructed list columns
+  std::shared_ptr<RecordBatch> list_batch;
+  RETURN_NOT_OK(MakeListRecordBatch(&list_batch));
+  std::vector<ArrayPtr> columns = {
+      list_batch->column(0), list_batch->column(1), list_batch->column(2)};
+  auto list_schema = list_batch->schema();
+
+  // Define schema
+  std::shared_ptr<DataType> type(new StructType(
+      {list_schema->field(0), list_schema->field(1), list_schema->field(2)}));
+  auto f0 = std::make_shared<Field>("non_null_struct", type);
+  auto f1 = std::make_shared<Field>("null_struct", type);
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  // construct individual nullable/non-nullable struct arrays
+  ArrayPtr no_nulls(new StructArray(type, list_batch->num_rows(), columns));
+  std::vector<uint8_t> null_bytes(list_batch->num_rows(), 1);
+  null_bytes[0] = 0;
+  std::shared_ptr<Buffer> null_bitmask;
+  RETURN_NOT_OK(util::bytes_to_bits(null_bytes, &null_bitmask));
+  ArrayPtr with_nulls(
+      new StructArray(type, list_batch->num_rows(), columns, 1, null_bitmask));
+
+  // construct batch
+  std::vector<ArrayPtr> arrays = {no_nulls, with_nulls};
+  out->reset(new RecordBatch(schema, list_batch->num_rows(), arrays));
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/util.h b/cpp/src/arrow/ipc/util.h
index 3f4001b21a91b..94079a3827777 100644
--- a/cpp/src/arrow/ipc/util.h
+++ b/cpp/src/arrow/ipc/util.h
@@ -27,6 +27,14 @@
 namespace arrow {
 namespace ipc {
 
+// Align on 8-byte boundaries
+static constexpr int kArrowAlignment = 8;
+static constexpr uint8_t kPaddingBytes[kArrowAlignment] = {0};
+
+static inline int64_t PaddedLength(int64_t nbytes, int64_t alignment = kArrowAlignment) {
+  return ((nbytes + alignment - 1) / alignment) * alignment;
+}
+
 // A helper class to tracks the size of allocations
 class MockOutputStream : public io::OutputStream {
  public:
diff --git a/cpp/src/arrow/parquet/reader.h b/cpp/src/arrow/parquet/reader.h
index a9c64eca997b5..2689bebea30ef 100644
--- a/cpp/src/arrow/parquet/reader.h
+++ b/cpp/src/arrow/parquet/reader.h
@@ -31,7 +31,7 @@ namespace arrow {
 
 class Array;
 class MemoryPool;
-class RowBatch;
+class RecordBatch;
 class Status;
 class Table;
 
diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h
index 5aa1ba587176a..ecc6a9f8be3de 100644
--- a/cpp/src/arrow/parquet/writer.h
+++ b/cpp/src/arrow/parquet/writer.h
@@ -30,7 +30,7 @@ namespace arrow {
 class Array;
 class MemoryPool;
 class PrimitiveArray;
-class RowBatch;
+class RecordBatch;
 class Status;
 class StringArray;
 class Table;
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index d9573eae74ddd..3a250df81d0fb 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -27,11 +27,11 @@
 
 namespace arrow {
 
-RowBatch::RowBatch(const std::shared_ptr<Schema>& schema, int num_rows,
+RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int num_rows,
     const std::vector<std::shared_ptr<Array>>& columns)
     : schema_(schema), num_rows_(num_rows), columns_(columns) {}
 
-const std::string& RowBatch::column_name(int i) const {
+const std::string& RecordBatch::column_name(int i) const {
   return schema_->field(i)->name;
 }
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 2088fdf0b6415..36b3c8ecaf43f 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -32,15 +32,15 @@ class Column;
 class Schema;
 class Status;
 
-// A row batch is a simpler and more rigid table data structure intended for
+// A record batch is a simpler and more rigid table data structure intended for
 // use primarily in shared memory IPC. It contains a schema (metadata) and a
-// corresponding vector of equal-length Arrow arrays
-class ARROW_EXPORT RowBatch {
+// corresponding sequence of equal-length Arrow arrays
+class ARROW_EXPORT RecordBatch {
  public:
-  // num_rows is a parameter to allow for row batches of a particular size not
+  // num_rows is a parameter to allow for record batches of a particular size not
   // having any materialized columns. Each array should have the same length as
   // num_rows
-  RowBatch(const std::shared_ptr<Schema>& schema, int num_rows,
+  RecordBatch(const std::shared_ptr<Schema>& schema, int32_t num_rows,
       const std::vector<std::shared_ptr<Array>>& columns);
 
   // @returns: the table's schema
@@ -50,17 +50,19 @@ class ARROW_EXPORT RowBatch {
   // Note: Does not boundscheck
   const std::shared_ptr<Array>& column(int i) const { return columns_[i]; }
 
+  const std::vector<std::shared_ptr<Array>>& columns() const { return columns_; }
+
   const std::string& column_name(int i) const;
 
   // @returns: the number of columns in the table
   int num_columns() const { return columns_.size(); }
 
   // @returns: the number of rows (the corresponding length of each column)
-  int64_t num_rows() const { return num_rows_; }
+  int32_t num_rows() const { return num_rows_; }
 
  private:
   std::shared_ptr<Schema> schema_;
-  int num_rows_;
+  int32_t num_rows_;
   std::vector<std::shared_ptr<Array>> columns_;
 };
 
diff --git a/format/IPC.md b/format/IPC.md
new file mode 100644
index 0000000000000..1f39e762ab70d
--- /dev/null
+++ b/format/IPC.md
@@ -0,0 +1,3 @@
+# Interprocess messaging / communication (IPC)
+
+## File format
diff --git a/format/README.md b/format/README.md
index 3b0e50364d83c..78e15207ee95a 100644
--- a/format/README.md
+++ b/format/README.md
@@ -9,6 +9,7 @@ Currently, the Arrow specification consists of these pieces:
 - Metadata specification (see Metadata.md)
 - Physical memory layout specification (see Layout.md)
 - Metadata serialized representation (see Message.fbs)
+- Mechanics of messaging between Arrow systems (IPC, RPC, etc.) (see IPC.md)
 
 The metadata currently uses Google's [flatbuffers library][1] for serializing a
 couple related pieces of information:

From 32fd692f3aced29cc65a786d5ec63f8cd484853c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 25 Sep 2016 19:28:26 -0400
Subject: [PATCH 0142/1644] ARROW-296: [Python / C++] Remove arrow::parquet,
 make pyarrow link against parquet_arrow

This patch depends on PARQUET-728 (to run the full test suite, including pyarrow Parquet tests)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #145 from wesm/ARROW-296 and squashes the following commits:

d67b4f9 [Wes McKinney] Refactor to link against parquet_arrow, fix up cmake files
---
 cpp/CMakeLists.txt                            |  18 -
 cpp/cmake_modules/FindParquet.cmake           |  44 +-
 cpp/doc/Parquet.md                            |  15 +-
 cpp/src/arrow/parquet/CMakeLists.txt          |  67 ---
 cpp/src/arrow/parquet/io.cc                   | 105 ----
 cpp/src/arrow/parquet/io.h                    |  84 ---
 cpp/src/arrow/parquet/parquet-io-test.cc      | 135 -----
 .../parquet/parquet-reader-writer-test.cc     | 499 ------------------
 cpp/src/arrow/parquet/parquet-schema-test.cc  | 261 ---------
 cpp/src/arrow/parquet/reader.cc               | 401 --------------
 cpp/src/arrow/parquet/reader.h                | 146 -----
 cpp/src/arrow/parquet/schema.cc               | 344 ------------
 cpp/src/arrow/parquet/schema.h                |  53 --
 cpp/src/arrow/parquet/test-util.h             | 193 -------
 cpp/src/arrow/parquet/utils.h                 |  52 --
 cpp/src/arrow/parquet/writer.cc               | 365 -------------
 cpp/src/arrow/parquet/writer.h                |  76 ---
 cpp/src/arrow/types/string.cc                 |   2 +-
 python/CMakeLists.txt                         |  14 +-
 python/cmake_modules/FindArrow.cmake          |  22 -
 python/pyarrow/includes/parquet.pxd           |  10 +-
 21 files changed, 55 insertions(+), 2851 deletions(-)
 delete mode 100644 cpp/src/arrow/parquet/CMakeLists.txt
 delete mode 100644 cpp/src/arrow/parquet/io.cc
 delete mode 100644 cpp/src/arrow/parquet/io.h
 delete mode 100644 cpp/src/arrow/parquet/parquet-io-test.cc
 delete mode 100644 cpp/src/arrow/parquet/parquet-reader-writer-test.cc
 delete mode 100644 cpp/src/arrow/parquet/parquet-schema-test.cc
 delete mode 100644 cpp/src/arrow/parquet/reader.cc
 delete mode 100644 cpp/src/arrow/parquet/reader.h
 delete mode 100644 cpp/src/arrow/parquet/schema.cc
 delete mode 100644 cpp/src/arrow/parquet/schema.h
 delete mode 100644 cpp/src/arrow/parquet/test-util.h
 delete mode 100644 cpp/src/arrow/parquet/utils.h
 delete mode 100644 cpp/src/arrow/parquet/writer.cc
 delete mode 100644 cpp/src/arrow/parquet/writer.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index be95dabf31897..f3f4a7dac0100 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -52,10 +52,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the libarrow shared libraries"
     ON)
 
-  option(ARROW_PARQUET
-    "Build the Parquet adapter and link to libparquet"
-    OFF)
-
   option(ARROW_TEST_MEMCHECK
     "Run the test suite using valgrind --tool=memcheck"
     OFF)
@@ -702,20 +698,6 @@ add_subdirectory(src/arrow/io)
 add_subdirectory(src/arrow/util)
 add_subdirectory(src/arrow/types)
 
-#----------------------------------------------------------------------
-# Parquet adapter library
-
-if(ARROW_PARQUET)
-  find_package(Parquet REQUIRED)
-  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(parquet
-    STATIC_LIB ${PARQUET_STATIC_LIB}
-    SHARED_LIB ${PARQUET_SHARED_LIB})
-
-  add_subdirectory(src/arrow/parquet)
-  list(APPEND LINK_LIBS arrow_parquet parquet)
-endif()
-
 #----------------------------------------------------------------------
 # IPC library
 
diff --git a/cpp/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
index 36f4828a999d4..7445e0919acb6 100644
--- a/cpp/cmake_modules/FindParquet.cmake
+++ b/cpp/cmake_modules/FindParquet.cmake
@@ -29,15 +29,20 @@ endif()
 
 # Try the parameterized roots, if they exist
 if ( _parquet_roots )
-    find_path( PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h
-        PATHS ${_parquet_roots} NO_DEFAULT_PATH
-        PATH_SUFFIXES "include" )
-    find_library( PARQUET_LIBRARIES NAMES parquet
-        PATHS ${_parquet_roots} NO_DEFAULT_PATH
-        PATH_SUFFIXES "lib" )
+  find_path( PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h
+    PATHS ${_parquet_roots} NO_DEFAULT_PATH
+    PATH_SUFFIXES "include" )
+  find_library( PARQUET_LIBRARIES NAMES parquet
+    PATHS ${_parquet_roots} NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+
+  find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
+    PATHS ${_parquet_roots} NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib")
 else ()
-    find_path( PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
-    find_library( PARQUET_LIBRARIES NAMES parquet )
+    find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
+    find_library(PARQUET_LIBRARIES NAMES parquet)
+    find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow)
 endif ()
 
 
@@ -51,6 +56,18 @@ else ()
   set(PARQUET_FOUND FALSE)
 endif ()
 
+if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES)
+  set(PARQUET_ARROW_FOUND TRUE)
+  get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH)
+  set(PARQUET_ARROW_LIB_NAME libparquet_arrow)
+  set(PARQUET_ARROW_STATIC_LIB
+    ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.a)
+  set(PARQUET_ARROW_SHARED_LIB
+    ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+else ()
+  set(PARQUET_ARROW_FOUND FALSE)
+endif ()
+
 if (PARQUET_FOUND)
   if (NOT Parquet_FIND_QUIETLY)
     message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}")
@@ -71,6 +88,12 @@ else ()
   endif ()
 endif ()
 
+if (PARQUET_ARROW_FOUND)
+  if (NOT Parquet_FIND_QUIETLY)
+    message(STATUS "Found the Parquet Arrow library: ${PARQUET_ARROW_LIBS}")
+  endif()
+endif()
+
 mark_as_advanced(
   PARQUET_FOUND
   PARQUET_INCLUDE_DIR
@@ -78,4 +101,9 @@ mark_as_advanced(
   PARQUET_LIBRARIES
   PARQUET_STATIC_LIB
   PARQUET_SHARED_LIB
+
+  PARQUET_ARROW_FOUND
+  PARQUET_ARROW_LIBS
+  PARQUET_ARROW_STATIC_LIB
+  PARQUET_ARROW_SHARED_LIB
 )
diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md
index 370ac833388fc..96471d94835f3 100644
--- a/cpp/doc/Parquet.md
+++ b/cpp/doc/Parquet.md
@@ -1,24 +1,19 @@
 ## Building Arrow-Parquet integration
 
-To build the Arrow C++'s Parquet adapter library, you must first build [parquet-cpp][1]:
+To use Arrow C++ with Parquet, you must first build the Arrow C++ libraries and
+install them someplace. Then, you can build [parquet-cpp][1] with the Arrow
+adapter library:
 
 ```bash
 # Set this to your preferred install location
-export PARQUET_HOME=$HOME/local
+export ARROW_HOME=$HOME/local
 
 git clone https://github.com/apache/parquet-cpp.git
 cd parquet-cpp
 source setup_build_env.sh
-cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME
+cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_ARROW=on
 make -j4
 make install
 ```
 
-Make sure that `$PARQUET_HOME` is set to the installation location. Now, build
-Arrow with the Parquet adapter enabled:
-
-```bash
-cmake -DARROW_PARQUET=ON
-```
-
 [1]: https://github.com/apache/parquet-cpp
\ No newline at end of file
diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt
deleted file mode 100644
index c400e14ea47f7..0000000000000
--- a/cpp/src/arrow/parquet/CMakeLists.txt
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# ----------------------------------------------------------------------
-# arrow_parquet : Arrow <-> Parquet adapter
-
-set(PARQUET_SRCS
-  io.cc
-  reader.cc
-  schema.cc
-  writer.cc
-)
-
-set(PARQUET_LIBS
-  arrow_shared
-  arrow_io
-  parquet_shared
-)
-
-add_library(arrow_parquet SHARED
-  ${PARQUET_SRCS}
-)
-target_link_libraries(arrow_parquet ${PARQUET_LIBS})
-SET_TARGET_PROPERTIES(arrow_parquet PROPERTIES LINKER_LANGUAGE CXX)
-
-if (APPLE)
-  set_target_properties(arrow_parquet
-    PROPERTIES
-    BUILD_WITH_INSTALL_RPATH ON
-    INSTALL_NAME_DIR "@rpath")
-endif()
-
-ADD_ARROW_TEST(parquet-schema-test)
-ARROW_TEST_LINK_LIBRARIES(parquet-schema-test arrow_parquet)
-
-ADD_ARROW_TEST(parquet-io-test)
-ARROW_TEST_LINK_LIBRARIES(parquet-io-test arrow_parquet)
-
-ADD_ARROW_TEST(parquet-reader-writer-test)
-ARROW_TEST_LINK_LIBRARIES(parquet-reader-writer-test arrow_parquet)
-
-# Headers: top level
-install(FILES
-  io.h
-  reader.h
-  schema.h
-  utils.h
-  writer.h
-  DESTINATION include/arrow/parquet)
-
-install(TARGETS arrow_parquet
-  LIBRARY DESTINATION lib
-  ARCHIVE DESTINATION lib)
diff --git a/cpp/src/arrow/parquet/io.cc b/cpp/src/arrow/parquet/io.cc
deleted file mode 100644
index a50d753f3054e..0000000000000
--- a/cpp/src/arrow/parquet/io.cc
+++ /dev/null
@@ -1,105 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/parquet/io.h"
-
-#include <cstdint>
-#include <memory>
-
-#include "parquet/api/io.h"
-
-#include "arrow/parquet/utils.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
-
-// To assist with readability
-using ArrowROFile = arrow::io::ReadableFileInterface;
-
-namespace arrow {
-namespace parquet {
-
-// ----------------------------------------------------------------------
-// ParquetAllocator
-
-ParquetAllocator::ParquetAllocator() : pool_(default_memory_pool()) {}
-
-ParquetAllocator::ParquetAllocator(MemoryPool* pool) : pool_(pool) {}
-
-ParquetAllocator::~ParquetAllocator() {}
-
-uint8_t* ParquetAllocator::Malloc(int64_t size) {
-  uint8_t* result;
-  PARQUET_THROW_NOT_OK(pool_->Allocate(size, &result));
-  return result;
-}
-
-void ParquetAllocator::Free(uint8_t* buffer, int64_t size) {
-  // Does not report Status
-  pool_->Free(buffer, size);
-}
-
-// ----------------------------------------------------------------------
-// ParquetReadSource
-
-ParquetReadSource::ParquetReadSource(ParquetAllocator* allocator)
-    : file_(nullptr), allocator_(allocator) {}
-
-Status ParquetReadSource::Open(const std::shared_ptr<io::ReadableFileInterface>& file) {
-  int64_t file_size;
-  RETURN_NOT_OK(file->GetSize(&file_size));
-
-  file_ = file;
-  size_ = file_size;
-  return Status::OK();
-}
-
-void ParquetReadSource::Close() {
-  // TODO(wesm): Make this a no-op for now. This leaves Python wrappers for
-  // these classes in a borked state. Probably better to explicitly close.
-
-  // PARQUET_THROW_NOT_OK(file_->Close());
-}
-
-int64_t ParquetReadSource::Tell() const {
-  int64_t position;
-  PARQUET_THROW_NOT_OK(file_->Tell(&position));
-  return position;
-}
-
-void ParquetReadSource::Seek(int64_t position) {
-  PARQUET_THROW_NOT_OK(file_->Seek(position));
-}
-
-int64_t ParquetReadSource::Read(int64_t nbytes, uint8_t* out) {
-  int64_t bytes_read;
-  PARQUET_THROW_NOT_OK(file_->Read(nbytes, &bytes_read, out));
-  return bytes_read;
-}
-
-std::shared_ptr<::parquet::Buffer> ParquetReadSource::Read(int64_t nbytes) {
-  // TODO(wesm): This code is duplicated from parquet/util/input.cc; suggests
-  // that there should be more code sharing amongst file-like sources
-  auto result = std::make_shared<::parquet::OwnedMutableBuffer>(0, allocator_);
-  result->Resize(nbytes);
-
-  int64_t bytes_read = Read(nbytes, result->mutable_data());
-  if (bytes_read < nbytes) { result->Resize(bytes_read); }
-  return result;
-}
-
-}  // namespace parquet
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/io.h b/cpp/src/arrow/parquet/io.h
deleted file mode 100644
index 1734863acf1ea..0000000000000
--- a/cpp/src/arrow/parquet/io.h
+++ /dev/null
@@ -1,84 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Bridges Arrow's IO interfaces and Parquet-cpp's IO interfaces
-
-#ifndef ARROW_PARQUET_IO_H
-#define ARROW_PARQUET_IO_H
-
-#include <cstdint>
-#include <memory>
-
-#include "parquet/api/io.h"
-
-#include "arrow/io/interfaces.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class MemoryPool;
-
-namespace parquet {
-
-// An implementation of the Parquet MemoryAllocator API that plugs into an
-// existing Arrow memory pool. This way we can direct all allocations to a
-// single place rather than tracking allocations in different locations (for
-// example: without utilizing parquet-cpp's default allocator)
-class ARROW_EXPORT ParquetAllocator : public ::parquet::MemoryAllocator {
- public:
-  // Uses the default memory pool
-  ParquetAllocator();
-
-  explicit ParquetAllocator(MemoryPool* pool);
-  virtual ~ParquetAllocator();
-
-  uint8_t* Malloc(int64_t size) override;
-  void Free(uint8_t* buffer, int64_t size) override;
-
-  void set_pool(MemoryPool* pool) { pool_ = pool; }
-
-  MemoryPool* pool() const { return pool_; }
-
- private:
-  MemoryPool* pool_;
-};
-
-class ARROW_EXPORT ParquetReadSource : public ::parquet::RandomAccessSource {
- public:
-  explicit ParquetReadSource(ParquetAllocator* allocator);
-
-  // We need to ask for the file size on opening the file, and this can fail
-  Status Open(const std::shared_ptr<io::ReadableFileInterface>& file);
-
-  void Close() override;
-  int64_t Tell() const override;
-  void Seek(int64_t pos) override;
-  int64_t Read(int64_t nbytes, uint8_t* out) override;
-  std::shared_ptr<::parquet::Buffer> Read(int64_t nbytes) override;
-
- private:
-  // An Arrow readable file of some kind
-  std::shared_ptr<io::ReadableFileInterface> file_;
-
-  // The allocator is required for creating managed buffers
-  ParquetAllocator* allocator_;
-};
-
-}  // namespace parquet
-}  // namespace arrow
-
-#endif  // ARROW_PARQUET_IO_H
diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc
deleted file mode 100644
index 208b3e867d374..0000000000000
--- a/cpp/src/arrow/parquet/parquet-io-test.cc
+++ /dev/null
@@ -1,135 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdlib>
-#include <memory>
-#include <string>
-
-#include "gtest/gtest.h"
-
-#include "arrow/io/memory.h"
-#include "arrow/parquet/io.h"
-#include "arrow/test-util.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
-
-#include "parquet/api/io.h"
-
-namespace arrow {
-namespace parquet {
-
-// Allocator tests
-
-TEST(TestParquetAllocator, DefaultCtor) {
-  ParquetAllocator allocator;
-
-  const int buffer_size = 10;
-
-  uint8_t* buffer = nullptr;
-  ASSERT_NO_THROW(buffer = allocator.Malloc(buffer_size););
-
-  // valgrind will complain if we write into nullptr
-  memset(buffer, 0, buffer_size);
-
-  allocator.Free(buffer, buffer_size);
-}
-
-// Pass through to the default memory pool
-class TrackingPool : public MemoryPool {
- public:
-  TrackingPool() : pool_(default_memory_pool()), bytes_allocated_(0) {}
-
-  Status Allocate(int64_t size, uint8_t** out) override {
-    RETURN_NOT_OK(pool_->Allocate(size, out));
-    bytes_allocated_ += size;
-    return Status::OK();
-  }
-
-  void Free(uint8_t* buffer, int64_t size) override {
-    pool_->Free(buffer, size);
-    bytes_allocated_ -= size;
-  }
-
-  int64_t bytes_allocated() const override { return bytes_allocated_; }
-
- private:
-  MemoryPool* pool_;
-  int64_t bytes_allocated_;
-};
-
-TEST(TestParquetAllocator, CustomPool) {
-  TrackingPool pool;
-
-  ParquetAllocator allocator(&pool);
-
-  ASSERT_EQ(&pool, allocator.pool());
-
-  const int buffer_size = 10;
-
-  uint8_t* buffer = nullptr;
-  ASSERT_NO_THROW(buffer = allocator.Malloc(buffer_size););
-
-  ASSERT_EQ(buffer_size, pool.bytes_allocated());
-
-  // valgrind will complain if we write into nullptr
-  memset(buffer, 0, buffer_size);
-
-  allocator.Free(buffer, buffer_size);
-
-  ASSERT_EQ(0, pool.bytes_allocated());
-}
-
-// ----------------------------------------------------------------------
-// Read source tests
-
-TEST(TestParquetReadSource, Basics) {
-  std::string data = "this is the data";
-  auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str());
-
-  ParquetAllocator allocator(default_memory_pool());
-
-  auto file = std::make_shared<io::BufferReader>(data_buffer, data.size());
-  auto source = std::make_shared<ParquetReadSource>(&allocator);
-
-  ASSERT_OK(source->Open(file));
-
-  ASSERT_EQ(0, source->Tell());
-  ASSERT_NO_THROW(source->Seek(5));
-  ASSERT_EQ(5, source->Tell());
-  ASSERT_NO_THROW(source->Seek(0));
-
-  // Seek out of bounds
-  ASSERT_THROW(source->Seek(100), ::parquet::ParquetException);
-
-  uint8_t buffer[50];
-
-  ASSERT_NO_THROW(source->Read(4, buffer));
-  ASSERT_EQ(0, std::memcmp(buffer, "this", 4));
-  ASSERT_EQ(4, source->Tell());
-
-  std::shared_ptr<::parquet::Buffer> pq_buffer;
-
-  ASSERT_NO_THROW(pq_buffer = source->Read(7));
-
-  auto expected_buffer = std::make_shared<::parquet::Buffer>(data_buffer + 4, 7);
-
-  ASSERT_TRUE(expected_buffer->Equals(*pq_buffer.get()));
-}
-
-}  // namespace parquet
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/parquet-reader-writer-test.cc b/cpp/src/arrow/parquet/parquet-reader-writer-test.cc
deleted file mode 100644
index d7b39dda377d3..0000000000000
--- a/cpp/src/arrow/parquet/parquet-reader-writer-test.cc
+++ /dev/null
@@ -1,499 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-#include "arrow/test-util.h"
-#include "arrow/parquet/test-util.h"
-#include "arrow/parquet/reader.h"
-#include "arrow/parquet/writer.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
-
-#include "parquet/api/reader.h"
-#include "parquet/api/writer.h"
-
-using ParquetBuffer = parquet::Buffer;
-using parquet::BufferReader;
-using parquet::default_writer_properties;
-using parquet::InMemoryOutputStream;
-using parquet::LogicalType;
-using parquet::ParquetFileReader;
-using parquet::ParquetFileWriter;
-using parquet::RandomAccessSource;
-using parquet::Repetition;
-using parquet::SchemaDescriptor;
-using parquet::ParquetVersion;
-using ParquetType = parquet::Type;
-using parquet::schema::GroupNode;
-using parquet::schema::NodePtr;
-using parquet::schema::PrimitiveNode;
-
-namespace arrow {
-
-namespace parquet {
-
-const int SMALL_SIZE = 100;
-const int LARGE_SIZE = 10000;
-
-template <typename TestType>
-struct test_traits {};
-
-template <>
-struct test_traits<BooleanType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::BOOLEAN;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static uint8_t const value;
-};
-
-const uint8_t test_traits<BooleanType>::value(1);
-
-template <>
-struct test_traits<UInt8Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::UINT_8;
-  static uint8_t const value;
-};
-
-const uint8_t test_traits<UInt8Type>::value(64);
-
-template <>
-struct test_traits<Int8Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::INT_8;
-  static int8_t const value;
-};
-
-const int8_t test_traits<Int8Type>::value(-64);
-
-template <>
-struct test_traits<UInt16Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::UINT_16;
-  static uint16_t const value;
-};
-
-const uint16_t test_traits<UInt16Type>::value(1024);
-
-template <>
-struct test_traits<Int16Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::INT_16;
-  static int16_t const value;
-};
-
-const int16_t test_traits<Int16Type>::value(-1024);
-
-template <>
-struct test_traits<UInt32Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::UINT_32;
-  static uint32_t const value;
-};
-
-const uint32_t test_traits<UInt32Type>::value(1024);
-
-template <>
-struct test_traits<Int32Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static int32_t const value;
-};
-
-const int32_t test_traits<Int32Type>::value(-1024);
-
-template <>
-struct test_traits<UInt64Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
-  static constexpr LogicalType::type logical_enum = LogicalType::UINT_64;
-  static uint64_t const value;
-};
-
-const uint64_t test_traits<UInt64Type>::value(1024);
-
-template <>
-struct test_traits<Int64Type> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static int64_t const value;
-};
-
-const int64_t test_traits<Int64Type>::value(-1024);
-
-template <>
-struct test_traits<TimestampType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
-  static constexpr LogicalType::type logical_enum = LogicalType::TIMESTAMP_MILLIS;
-  static int64_t const value;
-};
-
-const int64_t test_traits<TimestampType>::value(14695634030000);
-
-template <>
-struct test_traits<FloatType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::FLOAT;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static float const value;
-};
-
-const float test_traits<FloatType>::value(2.1f);
-
-template <>
-struct test_traits<DoubleType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::DOUBLE;
-  static constexpr LogicalType::type logical_enum = LogicalType::NONE;
-  static double const value;
-};
-
-const double test_traits<DoubleType>::value(4.2);
-
-template <>
-struct test_traits<StringType> {
-  static constexpr ParquetType::type parquet_enum = ParquetType::BYTE_ARRAY;
-  static constexpr LogicalType::type logical_enum = LogicalType::UTF8;
-  static std::string const value;
-};
-
-const std::string test_traits<StringType>::value("Test");
-
-template <typename T>
-using ParquetDataType = ::parquet::DataType<test_traits<T>::parquet_enum>;
-
-template <typename T>
-using ParquetWriter = ::parquet::TypedColumnWriter<ParquetDataType<T>>;
-
-template <typename TestType>
-class TestParquetIO : public ::testing::Test {
- public:
-  virtual void SetUp() {}
-
-  std::shared_ptr<GroupNode> MakeSchema(Repetition::type repetition) {
-    auto pnode = PrimitiveNode::Make("column1", repetition,
-        test_traits<TestType>::parquet_enum, test_traits<TestType>::logical_enum);
-    NodePtr node_ =
-        GroupNode::Make("schema", Repetition::REQUIRED, std::vector<NodePtr>({pnode}));
-    return std::static_pointer_cast<GroupNode>(node_);
-  }
-
-  std::unique_ptr<ParquetFileWriter> MakeWriter(
-      const std::shared_ptr<GroupNode>& schema) {
-    sink_ = std::make_shared<InMemoryOutputStream>();
-    return ParquetFileWriter::Open(sink_, schema);
-  }
-
-  std::unique_ptr<ParquetFileReader> ReaderFromSink() {
-    std::shared_ptr<ParquetBuffer> buffer = sink_->GetBuffer();
-    std::unique_ptr<RandomAccessSource> source(new BufferReader(buffer));
-    return ParquetFileReader::Open(std::move(source));
-  }
-
-  void ReadSingleColumnFile(
-      std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Array>* out) {
-    arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
-    std::unique_ptr<arrow::parquet::FlatColumnReader> column_reader;
-    ASSERT_OK_NO_THROW(reader.GetFlatColumn(0, &column_reader));
-    ASSERT_NE(nullptr, column_reader.get());
-
-    ASSERT_OK(column_reader->NextBatch(SMALL_SIZE, out));
-    ASSERT_NE(nullptr, out->get());
-  }
-
-  void ReadAndCheckSingleColumnFile(Array* values) {
-    std::shared_ptr<Array> out;
-    ReadSingleColumnFile(ReaderFromSink(), &out);
-    ASSERT_TRUE(values->Equals(out));
-  }
-
-  void ReadTableFromFile(
-      std::unique_ptr<ParquetFileReader> file_reader, std::shared_ptr<Table>* out) {
-    arrow::parquet::FileReader reader(default_memory_pool(), std::move(file_reader));
-    ASSERT_OK_NO_THROW(reader.ReadFlatTable(out));
-    ASSERT_NE(nullptr, out->get());
-  }
-
-  void ReadAndCheckSingleColumnTable(const std::shared_ptr<Array>& values) {
-    std::shared_ptr<Table> out;
-    ReadTableFromFile(ReaderFromSink(), &out);
-    ASSERT_EQ(1, out->num_columns());
-    ASSERT_EQ(values->length(), out->num_rows());
-
-    std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-    ASSERT_EQ(1, chunked_array->num_chunks());
-    ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
-  }
-
-  template <typename ArrayType>
-  void WriteFlatColumn(const std::shared_ptr<GroupNode>& schema,
-      const std::shared_ptr<ArrayType>& values) {
-    FileWriter writer(default_memory_pool(), MakeWriter(schema));
-    ASSERT_OK_NO_THROW(writer.NewRowGroup(values->length()));
-    ASSERT_OK_NO_THROW(writer.WriteFlatColumnChunk(values.get()));
-    ASSERT_OK_NO_THROW(writer.Close());
-  }
-
-  std::shared_ptr<InMemoryOutputStream> sink_;
-};
-
-// We habe separate tests for UInt32Type as this is currently the only type
-// where a roundtrip does not yield the identical Array structure.
-// There we write an UInt32 Array but receive an Int64 Array as result for
-// Parquet version 1.0.
-
-typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
-    Int32Type, UInt64Type, Int64Type, TimestampType, FloatType, DoubleType,
-    StringType> TestTypes;
-
-TYPED_TEST_CASE(TestParquetIO, TestTypes);
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredWrite) {
-  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
-
-  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
-  this->WriteFlatColumn(schema, values);
-
-  this->ReadAndCheckSingleColumnFile(values.get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableRequiredWrite) {
-  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, false);
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_OK_NO_THROW(WriteFlatTable(table.get(), default_memory_pool(), this->sink_,
-      values->length(), default_writer_properties()));
-
-  std::shared_ptr<Table> out;
-  this->ReadTableFromFile(this->ReaderFromSink(), &out);
-  ASSERT_EQ(1, out->num_columns());
-  ASSERT_EQ(100, out->num_rows());
-
-  std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-  ASSERT_EQ(1, chunked_array->num_chunks());
-  ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnOptionalReadWrite) {
-  // This also tests max_definition_level = 1
-  auto values = NullableArray<TypeParam>(SMALL_SIZE, 10);
-
-  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::OPTIONAL);
-  this->WriteFlatColumn(schema, values);
-
-  this->ReadAndCheckSingleColumnFile(values.get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {
-  // This also tests max_definition_level = 1
-  std::shared_ptr<Array> values = NullableArray<TypeParam>(SMALL_SIZE, 10);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_OK_NO_THROW(WriteFlatTable(table.get(), default_memory_pool(), this->sink_,
-      values->length(), default_writer_properties()));
-
-  this->ReadAndCheckSingleColumnTable(values);
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
-  auto values = NonNullArray<TypeParam>(SMALL_SIZE);
-  int64_t chunk_size = values->length() / 4;
-
-  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
-  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
-  for (int i = 0; i < 4; i++) {
-    ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
-    ASSERT_OK_NO_THROW(
-        writer.WriteFlatColumnChunk(values.get(), i * chunk_size, chunk_size));
-  }
-  ASSERT_OK_NO_THROW(writer.Close());
-
-  this->ReadAndCheckSingleColumnFile(values.get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableRequiredChunkedWrite) {
-  auto values = NonNullArray<TypeParam>(LARGE_SIZE);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, false);
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_OK_NO_THROW(WriteFlatTable(
-      table.get(), default_memory_pool(), this->sink_, 512, default_writer_properties()));
-
-  this->ReadAndCheckSingleColumnTable(values);
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnOptionalChunkedWrite) {
-  int64_t chunk_size = SMALL_SIZE / 4;
-  auto values = NullableArray<TypeParam>(SMALL_SIZE, 10);
-
-  std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::OPTIONAL);
-  FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
-  for (int i = 0; i < 4; i++) {
-    ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
-    ASSERT_OK_NO_THROW(
-        writer.WriteFlatColumnChunk(values.get(), i * chunk_size, chunk_size));
-  }
-  ASSERT_OK_NO_THROW(writer.Close());
-
-  this->ReadAndCheckSingleColumnFile(values.get());
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableOptionalChunkedWrite) {
-  // This also tests max_definition_level = 1
-  auto values = NullableArray<TypeParam>(LARGE_SIZE, 100);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  ASSERT_OK_NO_THROW(WriteFlatTable(
-      table.get(), default_memory_pool(), this->sink_, 512, default_writer_properties()));
-
-  this->ReadAndCheckSingleColumnTable(values);
-}
-
-using TestUInt32ParquetIO = TestParquetIO<UInt32Type>;
-
-TEST_F(TestUInt32ParquetIO, Parquet_2_0_Compability) {
-  // This also tests max_definition_level = 1
-  std::shared_ptr<PrimitiveArray> values = NullableArray<UInt32Type>(LARGE_SIZE, 100);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-
-  // Parquet 2.0 roundtrip should yield an uint32_t column again
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  std::shared_ptr<::parquet::WriterProperties> properties =
-      ::parquet::WriterProperties::Builder()
-          .version(ParquetVersion::PARQUET_2_0)
-          ->build();
-  ASSERT_OK_NO_THROW(
-      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512, properties));
-  this->ReadAndCheckSingleColumnTable(values);
-}
-
-TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compability) {
-  // This also tests max_definition_level = 1
-  std::shared_ptr<PrimitiveArray> values = NullableArray<UInt32Type>(LARGE_SIZE, 100);
-  std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-
-  // Parquet 1.0 returns an int64_t column as there is no way to tell a Parquet 1.0
-  // reader that a column is unsigned.
-  this->sink_ = std::make_shared<InMemoryOutputStream>();
-  std::shared_ptr<::parquet::WriterProperties> properties =
-      ::parquet::WriterProperties::Builder()
-          .version(ParquetVersion::PARQUET_1_0)
-          ->build();
-  ASSERT_OK_NO_THROW(
-      WriteFlatTable(table.get(), default_memory_pool(), this->sink_, 512, properties));
-
-  std::shared_ptr<Array> expected_values;
-  std::shared_ptr<PoolBuffer> int64_data =
-      std::make_shared<PoolBuffer>(default_memory_pool());
-  {
-    ASSERT_OK(int64_data->Resize(sizeof(int64_t) * values->length()));
-    int64_t* int64_data_ptr = reinterpret_cast<int64_t*>(int64_data->mutable_data());
-    const uint32_t* uint32_data_ptr =
-        reinterpret_cast<const uint32_t*>(values->data()->data());
-    // std::copy might be faster but this is explicit on the casts)
-    for (int64_t i = 0; i < values->length(); i++) {
-      int64_data_ptr[i] = static_cast<int64_t>(uint32_data_ptr[i]);
-    }
-  }
-  ASSERT_OK(MakePrimitiveArray(std::make_shared<Int64Type>(), values->length(),
-      int64_data, values->null_count(), values->null_bitmap(), &expected_values));
-  this->ReadAndCheckSingleColumnTable(expected_values);
-}
-
-template <typename T>
-using ParquetCDataType = typename ParquetDataType<T>::c_type;
-
-template <typename TestType>
-class TestPrimitiveParquetIO : public TestParquetIO<TestType> {
- public:
-  typedef typename TestType::c_type T;
-
-  void MakeTestFile(std::vector<T>& values, int num_chunks,
-      std::unique_ptr<ParquetFileReader>* file_reader) {
-    std::shared_ptr<GroupNode> schema = this->MakeSchema(Repetition::REQUIRED);
-    std::unique_ptr<ParquetFileWriter> file_writer = this->MakeWriter(schema);
-    size_t chunk_size = values.size() / num_chunks;
-    // Convert to Parquet's expected physical type
-    std::vector<uint8_t> values_buffer(
-        sizeof(ParquetCDataType<TestType>) * values.size());
-    auto values_parquet =
-        reinterpret_cast<ParquetCDataType<TestType>*>(values_buffer.data());
-    std::copy(values.cbegin(), values.cend(), values_parquet);
-    for (int i = 0; i < num_chunks; i++) {
-      auto row_group_writer = file_writer->AppendRowGroup(chunk_size);
-      auto column_writer =
-          static_cast<ParquetWriter<TestType>*>(row_group_writer->NextColumn());
-      ParquetCDataType<TestType>* data = values_parquet + i * chunk_size;
-      column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
-      column_writer->Close();
-      row_group_writer->Close();
-    }
-    file_writer->Close();
-    *file_reader = this->ReaderFromSink();
-  }
-
-  void CheckSingleColumnRequiredTableRead(int num_chunks) {
-    std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
-    std::unique_ptr<ParquetFileReader> file_reader;
-    ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader));
-
-    std::shared_ptr<Table> out;
-    this->ReadTableFromFile(std::move(file_reader), &out);
-    ASSERT_EQ(1, out->num_columns());
-    ASSERT_EQ(SMALL_SIZE, out->num_rows());
-
-    std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
-    ASSERT_EQ(1, chunked_array->num_chunks());
-    ExpectArray<TestType>(values.data(), chunked_array->chunk(0).get());
-  }
-
-  void CheckSingleColumnRequiredRead(int num_chunks) {
-    std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
-    std::unique_ptr<ParquetFileReader> file_reader;
-    ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader));
-
-    std::shared_ptr<Array> out;
-    this->ReadSingleColumnFile(std::move(file_reader), &out);
-
-    ExpectArray<TestType>(values.data(), out.get());
-  }
-};
-
-typedef ::testing::Types<BooleanType, UInt8Type, Int8Type, UInt16Type, Int16Type,
-    UInt32Type, Int32Type, UInt64Type, Int64Type, FloatType,
-    DoubleType> PrimitiveTestTypes;
-
-TYPED_TEST_CASE(TestPrimitiveParquetIO, PrimitiveTestTypes);
-
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredRead) {
-  this->CheckSingleColumnRequiredRead(1);
-}
-
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredTableRead) {
-  this->CheckSingleColumnRequiredTableRead(1);
-}
-
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedRead) {
-  this->CheckSingleColumnRequiredRead(4);
-}
-
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
-  this->CheckSingleColumnRequiredTableRead(4);
-}
-
-}  // namespace parquet
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/parquet-schema-test.cc b/cpp/src/arrow/parquet/parquet-schema-test.cc
deleted file mode 100644
index 63ad8fba46517..0000000000000
--- a/cpp/src/arrow/parquet/parquet-schema-test.cc
+++ /dev/null
@@ -1,261 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <memory>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/types/datetime.h"
-#include "arrow/types/decimal.h"
-#include "arrow/util/status.h"
-
-#include "arrow/parquet/schema.h"
-
-using ParquetType = parquet::Type;
-using parquet::LogicalType;
-using parquet::Repetition;
-using parquet::schema::NodePtr;
-using parquet::schema::GroupNode;
-using parquet::schema::PrimitiveNode;
-
-namespace arrow {
-
-namespace parquet {
-
-const auto BOOL = std::make_shared<BooleanType>();
-const auto UINT8 = std::make_shared<UInt8Type>();
-const auto INT32 = std::make_shared<Int32Type>();
-const auto INT64 = std::make_shared<Int64Type>();
-const auto FLOAT = std::make_shared<FloatType>();
-const auto DOUBLE = std::make_shared<DoubleType>();
-const auto UTF8 = std::make_shared<StringType>();
-const auto TIMESTAMP_MS = std::make_shared<TimestampType>(TimestampType::Unit::MILLI);
-// TODO: This requires parquet-cpp implementing the MICROS enum value
-// const auto TIMESTAMP_US = std::make_shared<TimestampType>(TimestampType::Unit::MICRO);
-const auto BINARY = std::make_shared<ListType>(std::make_shared<Field>("", UINT8));
-const auto DECIMAL_8_4 = std::make_shared<DecimalType>(8, 4);
-
-class TestConvertParquetSchema : public ::testing::Test {
- public:
-  virtual void SetUp() {}
-
-  void CheckFlatSchema(const std::shared_ptr<Schema>& expected_schema) {
-    ASSERT_EQ(expected_schema->num_fields(), result_schema_->num_fields());
-    for (int i = 0; i < expected_schema->num_fields(); ++i) {
-      auto lhs = result_schema_->field(i);
-      auto rhs = expected_schema->field(i);
-      EXPECT_TRUE(lhs->Equals(rhs)) << i << " " << lhs->ToString()
-                                    << " != " << rhs->ToString();
-    }
-  }
-
-  Status ConvertSchema(const std::vector<NodePtr>& nodes) {
-    NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, nodes);
-    descr_.Init(schema);
-    return FromParquetSchema(&descr_, &result_schema_);
-  }
-
- protected:
-  ::parquet::SchemaDescriptor descr_;
-  std::shared_ptr<Schema> result_schema_;
-};
-
-TEST_F(TestConvertParquetSchema, ParquetFlatPrimitives) {
-  std::vector<NodePtr> parquet_fields;
-  std::vector<std::shared_ptr<Field>> arrow_fields;
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("boolean", Repetition::REQUIRED, ParquetType::BOOLEAN));
-  arrow_fields.push_back(std::make_shared<Field>("boolean", BOOL, false));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
-  arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
-  arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
-
-  parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
-      ParquetType::INT64, LogicalType::TIMESTAMP_MILLIS));
-  arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_MS, false));
-
-  // parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
-  //     ParquetType::INT64, LogicalType::TIMESTAMP_MICROS));
-  // arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_US, false));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
-  arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("double", Repetition::OPTIONAL, ParquetType::DOUBLE));
-  arrow_fields.push_back(std::make_shared<Field>("double", DOUBLE));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("binary", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY));
-  arrow_fields.push_back(std::make_shared<Field>("binary", BINARY));
-
-  parquet_fields.push_back(PrimitiveNode::Make(
-      "string", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::UTF8));
-  arrow_fields.push_back(std::make_shared<Field>("string", UTF8));
-
-  parquet_fields.push_back(PrimitiveNode::Make("flba-binary", Repetition::OPTIONAL,
-      ParquetType::FIXED_LEN_BYTE_ARRAY, LogicalType::NONE, 12));
-  arrow_fields.push_back(std::make_shared<Field>("flba-binary", BINARY));
-
-  auto arrow_schema = std::make_shared<Schema>(arrow_fields);
-  ASSERT_OK(ConvertSchema(parquet_fields));
-
-  CheckFlatSchema(arrow_schema);
-}
-
-TEST_F(TestConvertParquetSchema, ParquetFlatDecimals) {
-  std::vector<NodePtr> parquet_fields;
-  std::vector<std::shared_ptr<Field>> arrow_fields;
-
-  parquet_fields.push_back(PrimitiveNode::Make("flba-decimal", Repetition::OPTIONAL,
-      ParquetType::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 4, 8, 4));
-  arrow_fields.push_back(std::make_shared<Field>("flba-decimal", DECIMAL_8_4));
-
-  parquet_fields.push_back(PrimitiveNode::Make("binary-decimal", Repetition::OPTIONAL,
-      ParquetType::BYTE_ARRAY, LogicalType::DECIMAL, -1, 8, 4));
-  arrow_fields.push_back(std::make_shared<Field>("binary-decimal", DECIMAL_8_4));
-
-  parquet_fields.push_back(PrimitiveNode::Make("int32-decimal", Repetition::OPTIONAL,
-      ParquetType::INT32, LogicalType::DECIMAL, -1, 8, 4));
-  arrow_fields.push_back(std::make_shared<Field>("int32-decimal", DECIMAL_8_4));
-
-  parquet_fields.push_back(PrimitiveNode::Make("int64-decimal", Repetition::OPTIONAL,
-      ParquetType::INT64, LogicalType::DECIMAL, -1, 8, 4));
-  arrow_fields.push_back(std::make_shared<Field>("int64-decimal", DECIMAL_8_4));
-
-  auto arrow_schema = std::make_shared<Schema>(arrow_fields);
-  ASSERT_OK(ConvertSchema(parquet_fields));
-
-  CheckFlatSchema(arrow_schema);
-}
-
-TEST_F(TestConvertParquetSchema, UnsupportedThings) {
-  std::vector<NodePtr> unsupported_nodes;
-
-  unsupported_nodes.push_back(
-      PrimitiveNode::Make("int96", Repetition::REQUIRED, ParquetType::INT96));
-
-  unsupported_nodes.push_back(
-      GroupNode::Make("repeated-group", Repetition::REPEATED, {}));
-
-  unsupported_nodes.push_back(PrimitiveNode::Make(
-      "int32", Repetition::OPTIONAL, ParquetType::INT32, LogicalType::DATE));
-
-  for (const NodePtr& node : unsupported_nodes) {
-    ASSERT_RAISES(NotImplemented, ConvertSchema({node}));
-  }
-}
-
-class TestConvertArrowSchema : public ::testing::Test {
- public:
-  virtual void SetUp() {}
-
-  void CheckFlatSchema(const std::vector<NodePtr>& nodes) {
-    NodePtr schema_node = GroupNode::Make("schema", Repetition::REPEATED, nodes);
-    const GroupNode* expected_schema_node =
-        static_cast<const GroupNode*>(schema_node.get());
-    const GroupNode* result_schema_node = result_schema_->group_node();
-
-    ASSERT_EQ(expected_schema_node->field_count(), result_schema_node->field_count());
-
-    for (int i = 0; i < expected_schema_node->field_count(); i++) {
-      auto lhs = result_schema_node->field(i);
-      auto rhs = expected_schema_node->field(i);
-      EXPECT_TRUE(lhs->Equals(rhs.get()));
-    }
-  }
-
-  Status ConvertSchema(const std::vector<std::shared_ptr<Field>>& fields) {
-    arrow_schema_ = std::make_shared<Schema>(fields);
-    std::shared_ptr<::parquet::WriterProperties> properties =
-        ::parquet::default_writer_properties();
-    return ToParquetSchema(arrow_schema_.get(), *properties.get(), &result_schema_);
-  }
-
- protected:
-  std::shared_ptr<Schema> arrow_schema_;
-  std::shared_ptr<::parquet::SchemaDescriptor> result_schema_;
-};
-
-TEST_F(TestConvertArrowSchema, ParquetFlatPrimitives) {
-  std::vector<NodePtr> parquet_fields;
-  std::vector<std::shared_ptr<Field>> arrow_fields;
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("boolean", Repetition::REQUIRED, ParquetType::BOOLEAN));
-  arrow_fields.push_back(std::make_shared<Field>("boolean", BOOL, false));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
-  arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
-  arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
-
-  parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
-      ParquetType::INT64, LogicalType::TIMESTAMP_MILLIS));
-  arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_MS, false));
-
-  // parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
-  //     ParquetType::INT64, LogicalType::TIMESTAMP_MICROS));
-  // arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_US, false));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
-  arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
-
-  parquet_fields.push_back(
-      PrimitiveNode::Make("double", Repetition::OPTIONAL, ParquetType::DOUBLE));
-  arrow_fields.push_back(std::make_shared<Field>("double", DOUBLE));
-
-  // TODO: String types need to be clarified a bit more in the Arrow spec
-  parquet_fields.push_back(PrimitiveNode::Make(
-      "string", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::UTF8));
-  arrow_fields.push_back(std::make_shared<Field>("string", UTF8));
-
-  ASSERT_OK(ConvertSchema(arrow_fields));
-
-  CheckFlatSchema(parquet_fields);
-}
-
-TEST_F(TestConvertArrowSchema, ParquetFlatDecimals) {
-  std::vector<NodePtr> parquet_fields;
-  std::vector<std::shared_ptr<Field>> arrow_fields;
-
-  // TODO: Test Decimal Arrow -> Parquet conversion
-
-  ASSERT_OK(ConvertSchema(arrow_fields));
-
-  CheckFlatSchema(parquet_fields);
-}
-
-TEST(TestNodeConversion, DateAndTime) {}
-
-}  // namespace parquet
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc
deleted file mode 100644
index 0c2fc6e8fc718..0000000000000
--- a/cpp/src/arrow/parquet/reader.cc
+++ /dev/null
@@ -1,401 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/parquet/reader.h"
-
-#include <algorithm>
-#include <queue>
-#include <string>
-#include <vector>
-
-#include "arrow/column.h"
-#include "arrow/parquet/io.h"
-#include "arrow/parquet/schema.h"
-#include "arrow/parquet/utils.h"
-#include "arrow/schema.h"
-#include "arrow/table.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/util/status.h"
-
-using parquet::ColumnReader;
-using parquet::Repetition;
-using parquet::TypedColumnReader;
-
-// Help reduce verbosity
-using ParquetRAS = parquet::RandomAccessSource;
-using ParquetReader = parquet::ParquetFileReader;
-
-namespace arrow {
-namespace parquet {
-
-template <typename ArrowType>
-struct ArrowTypeTraits {
-  typedef NumericBuilder<ArrowType> builder_type;
-};
-
-template <>
-struct ArrowTypeTraits<BooleanType> {
-  typedef BooleanBuilder builder_type;
-};
-
-template <typename ArrowType>
-using BuilderType = typename ArrowTypeTraits<ArrowType>::builder_type;
-
-class FileReader::Impl {
- public:
-  Impl(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader);
-  virtual ~Impl() {}
-
-  bool CheckForFlatColumn(const ::parquet::ColumnDescriptor* descr);
-  Status GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out);
-  Status ReadFlatColumn(int i, std::shared_ptr<Array>* out);
-  Status ReadFlatTable(std::shared_ptr<Table>* out);
-
- private:
-  MemoryPool* pool_;
-  std::unique_ptr<::parquet::ParquetFileReader> reader_;
-};
-
-class FlatColumnReader::Impl {
- public:
-  Impl(MemoryPool* pool, const ::parquet::ColumnDescriptor* descr,
-      ::parquet::ParquetFileReader* reader, int column_index);
-  virtual ~Impl() {}
-
-  Status NextBatch(int batch_size, std::shared_ptr<Array>* out);
-  template <typename ArrowType, typename ParquetType>
-  Status TypedReadBatch(int batch_size, std::shared_ptr<Array>* out);
-
-  template <typename ArrowType, typename ParquetType>
-  Status ReadNullableFlatBatch(const int16_t* def_levels,
-      typename ParquetType::c_type* values, int64_t values_read, int64_t levels_read,
-      BuilderType<ArrowType>* builder);
-  template <typename ArrowType, typename ParquetType>
-  Status ReadNonNullableBatch(typename ParquetType::c_type* values, int64_t values_read,
-      BuilderType<ArrowType>* builder);
-
- private:
-  void NextRowGroup();
-
-  template <typename InType, typename OutType>
-  struct can_copy_ptr {
-    static constexpr bool value =
-        std::is_same<InType, OutType>::value ||
-        (std::is_integral<InType>{} && std::is_integral<OutType>{} &&
-            (sizeof(InType) == sizeof(OutType)));
-  };
-
-  template <typename InType, typename OutType,
-      typename std::enable_if<can_copy_ptr<InType, OutType>::value>::type* = nullptr>
-  Status ConvertPhysicalType(
-      const InType* in_ptr, int64_t length, const OutType** out_ptr) {
-    *out_ptr = reinterpret_cast<const OutType*>(in_ptr);
-    return Status::OK();
-  }
-
-  template <typename InType, typename OutType,
-      typename std::enable_if<not can_copy_ptr<InType, OutType>::value>::type* = nullptr>
-  Status ConvertPhysicalType(
-      const InType* in_ptr, int64_t length, const OutType** out_ptr) {
-    RETURN_NOT_OK(values_builder_buffer_.Resize(length * sizeof(OutType)));
-    OutType* mutable_out_ptr =
-        reinterpret_cast<OutType*>(values_builder_buffer_.mutable_data());
-    std::copy(in_ptr, in_ptr + length, mutable_out_ptr);
-    *out_ptr = mutable_out_ptr;
-    return Status::OK();
-  }
-
-  MemoryPool* pool_;
-  const ::parquet::ColumnDescriptor* descr_;
-  ::parquet::ParquetFileReader* reader_;
-  int column_index_;
-  int next_row_group_;
-  std::shared_ptr<ColumnReader> column_reader_;
-  std::shared_ptr<Field> field_;
-
-  PoolBuffer values_buffer_;
-  PoolBuffer def_levels_buffer_;
-  PoolBuffer values_builder_buffer_;
-  PoolBuffer valid_bytes_buffer_;
-};
-
-FileReader::Impl::Impl(
-    MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader)
-    : pool_(pool), reader_(std::move(reader)) {}
-
-bool FileReader::Impl::CheckForFlatColumn(const ::parquet::ColumnDescriptor* descr) {
-  if ((descr->max_repetition_level() > 0) || (descr->max_definition_level() > 1)) {
-    return false;
-  } else if ((descr->max_definition_level() == 1) &&
-             (descr->schema_node()->repetition() != Repetition::OPTIONAL)) {
-    return false;
-  }
-  return true;
-}
-
-Status FileReader::Impl::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out) {
-  const ::parquet::SchemaDescriptor* schema = reader_->metadata()->schema();
-
-  if (!CheckForFlatColumn(schema->Column(i))) {
-    return Status::Invalid("The requested column is not flat");
-  }
-  std::unique_ptr<FlatColumnReader::Impl> impl(
-      new FlatColumnReader::Impl(pool_, schema->Column(i), reader_.get(), i));
-  *out = std::unique_ptr<FlatColumnReader>(new FlatColumnReader(std::move(impl)));
-  return Status::OK();
-}
-
-Status FileReader::Impl::ReadFlatColumn(int i, std::shared_ptr<Array>* out) {
-  std::unique_ptr<FlatColumnReader> flat_column_reader;
-  RETURN_NOT_OK(GetFlatColumn(i, &flat_column_reader));
-  return flat_column_reader->NextBatch(reader_->metadata()->num_rows(), out);
-}
-
-Status FileReader::Impl::ReadFlatTable(std::shared_ptr<Table>* table) {
-  auto descr = reader_->metadata()->schema();
-
-  const std::string& name = descr->name();
-  std::shared_ptr<Schema> schema;
-  RETURN_NOT_OK(FromParquetSchema(descr, &schema));
-
-  int num_columns = reader_->metadata()->num_columns();
-
-  std::vector<std::shared_ptr<Column>> columns(num_columns);
-  for (int i = 0; i < num_columns; i++) {
-    std::shared_ptr<Array> array;
-    RETURN_NOT_OK(ReadFlatColumn(i, &array));
-    columns[i] = std::make_shared<Column>(schema->field(i), array);
-  }
-
-  *table = std::make_shared<Table>(name, schema, columns);
-  return Status::OK();
-}
-
-FileReader::FileReader(
-    MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader)
-    : impl_(new FileReader::Impl(pool, std::move(reader))) {}
-
-FileReader::~FileReader() {}
-
-// Static ctor
-Status OpenFile(const std::shared_ptr<io::ReadableFileInterface>& file,
-    ParquetAllocator* allocator, std::unique_ptr<FileReader>* reader) {
-  std::unique_ptr<ParquetReadSource> source(new ParquetReadSource(allocator));
-  RETURN_NOT_OK(source->Open(file));
-
-  // TODO(wesm): reader properties
-  std::unique_ptr<ParquetReader> pq_reader;
-  PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(std::move(source)));
-
-  // Use the same memory pool as the ParquetAllocator
-  reader->reset(new FileReader(allocator->pool(), std::move(pq_reader)));
-  return Status::OK();
-}
-
-Status FileReader::GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out) {
-  return impl_->GetFlatColumn(i, out);
-}
-
-Status FileReader::ReadFlatColumn(int i, std::shared_ptr<Array>* out) {
-  return impl_->ReadFlatColumn(i, out);
-}
-
-Status FileReader::ReadFlatTable(std::shared_ptr<Table>* out) {
-  return impl_->ReadFlatTable(out);
-}
-
-FlatColumnReader::Impl::Impl(MemoryPool* pool, const ::parquet::ColumnDescriptor* descr,
-    ::parquet::ParquetFileReader* reader, int column_index)
-    : pool_(pool),
-      descr_(descr),
-      reader_(reader),
-      column_index_(column_index),
-      next_row_group_(0),
-      values_buffer_(pool),
-      def_levels_buffer_(pool) {
-  NodeToField(descr_->schema_node(), &field_);
-  NextRowGroup();
-}
-
-template <typename ArrowType, typename ParquetType>
-Status FlatColumnReader::Impl::ReadNonNullableBatch(typename ParquetType::c_type* values,
-    int64_t values_read, BuilderType<ArrowType>* builder) {
-  using ArrowCType = typename ArrowType::c_type;
-  using ParquetCType = typename ParquetType::c_type;
-
-  DCHECK(builder);
-  const ArrowCType* values_ptr = nullptr;
-  RETURN_NOT_OK(
-      (ConvertPhysicalType<ParquetCType, ArrowCType>(values, values_read, &values_ptr)));
-  RETURN_NOT_OK(builder->Append(values_ptr, values_read));
-  return Status::OK();
-}
-
-template <typename ArrowType, typename ParquetType>
-Status FlatColumnReader::Impl::ReadNullableFlatBatch(const int16_t* def_levels,
-    typename ParquetType::c_type* values, int64_t values_read, int64_t levels_read,
-    BuilderType<ArrowType>* builder) {
-  using ArrowCType = typename ArrowType::c_type;
-
-  DCHECK(builder);
-  RETURN_NOT_OK(values_builder_buffer_.Resize(levels_read * sizeof(ArrowCType)));
-  RETURN_NOT_OK(valid_bytes_buffer_.Resize(levels_read * sizeof(uint8_t)));
-  auto values_ptr = reinterpret_cast<ArrowCType*>(values_builder_buffer_.mutable_data());
-  uint8_t* valid_bytes = valid_bytes_buffer_.mutable_data();
-  int values_idx = 0;
-  for (int64_t i = 0; i < levels_read; i++) {
-    if (def_levels[i] < descr_->max_definition_level()) {
-      valid_bytes[i] = 0;
-    } else {
-      valid_bytes[i] = 1;
-      values_ptr[i] = values[values_idx++];
-    }
-  }
-  RETURN_NOT_OK(builder->Append(values_ptr, levels_read, valid_bytes));
-  return Status::OK();
-}
-
-template <typename ArrowType, typename ParquetType>
-Status FlatColumnReader::Impl::TypedReadBatch(
-    int batch_size, std::shared_ptr<Array>* out) {
-  using ParquetCType = typename ParquetType::c_type;
-
-  int values_to_read = batch_size;
-  BuilderType<ArrowType> builder(pool_, field_->type);
-  while ((values_to_read > 0) && column_reader_) {
-    values_buffer_.Resize(values_to_read * sizeof(ParquetCType));
-    if (descr_->max_definition_level() > 0) {
-      def_levels_buffer_.Resize(values_to_read * sizeof(int16_t));
-    }
-    auto reader = dynamic_cast<TypedColumnReader<ParquetType>*>(column_reader_.get());
-    int64_t values_read;
-    int64_t levels_read;
-    int16_t* def_levels = reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
-    auto values = reinterpret_cast<ParquetCType*>(values_buffer_.mutable_data());
-    PARQUET_CATCH_NOT_OK(levels_read = reader->ReadBatch(
-                             values_to_read, def_levels, nullptr, values, &values_read));
-    values_to_read -= levels_read;
-    if (descr_->max_definition_level() == 0) {
-      RETURN_NOT_OK(
-          (ReadNonNullableBatch<ArrowType, ParquetType>(values, values_read, &builder)));
-    } else {
-      // As per the defintion and checks for flat columns:
-      // descr_->max_definition_level() == 1
-      RETURN_NOT_OK((ReadNullableFlatBatch<ArrowType, ParquetType>(
-          def_levels, values, values_read, levels_read, &builder)));
-    }
-    if (!column_reader_->HasNext()) { NextRowGroup(); }
-  }
-  *out = builder.Finish();
-  return Status::OK();
-}
-
-template <>
-Status FlatColumnReader::Impl::TypedReadBatch<StringType, ::parquet::ByteArrayType>(
-    int batch_size, std::shared_ptr<Array>* out) {
-  int values_to_read = batch_size;
-  StringBuilder builder(pool_, field_->type);
-  while ((values_to_read > 0) && column_reader_) {
-    values_buffer_.Resize(values_to_read * sizeof(::parquet::ByteArray));
-    if (descr_->max_definition_level() > 0) {
-      def_levels_buffer_.Resize(values_to_read * sizeof(int16_t));
-    }
-    auto reader =
-        dynamic_cast<TypedColumnReader<::parquet::ByteArrayType>*>(column_reader_.get());
-    int64_t values_read;
-    int64_t levels_read;
-    int16_t* def_levels = reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
-    auto values = reinterpret_cast<::parquet::ByteArray*>(values_buffer_.mutable_data());
-    PARQUET_CATCH_NOT_OK(levels_read = reader->ReadBatch(
-                             values_to_read, def_levels, nullptr, values, &values_read));
-    values_to_read -= levels_read;
-    if (descr_->max_definition_level() == 0) {
-      for (int64_t i = 0; i < levels_read; i++) {
-        RETURN_NOT_OK(
-            builder.Append(reinterpret_cast<const char*>(values[i].ptr), values[i].len));
-      }
-    } else {
-      // descr_->max_definition_level() == 1
-      int values_idx = 0;
-      for (int64_t i = 0; i < levels_read; i++) {
-        if (def_levels[i] < descr_->max_definition_level()) {
-          RETURN_NOT_OK(builder.AppendNull());
-        } else {
-          RETURN_NOT_OK(
-              builder.Append(reinterpret_cast<const char*>(values[values_idx].ptr),
-                  values[values_idx].len));
-          values_idx++;
-        }
-      }
-    }
-    if (!column_reader_->HasNext()) { NextRowGroup(); }
-  }
-  *out = builder.Finish();
-  return Status::OK();
-}
-
-#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType)              \
-  case Type::ENUM:                                                  \
-    return TypedReadBatch<ArrowType, ParquetType>(batch_size, out); \
-    break;
-
-Status FlatColumnReader::Impl::NextBatch(int batch_size, std::shared_ptr<Array>* out) {
-  if (!column_reader_) {
-    // Exhausted all row groups.
-    *out = nullptr;
-    return Status::OK();
-  }
-
-  switch (field_->type->type) {
-    TYPED_BATCH_CASE(BOOL, BooleanType, ::parquet::BooleanType)
-    TYPED_BATCH_CASE(UINT8, UInt8Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(INT8, Int8Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(UINT16, UInt16Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(INT16, Int16Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(UINT32, UInt32Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(UINT64, UInt64Type, ::parquet::Int64Type)
-    TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type)
-    TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
-    TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
-    TYPED_BATCH_CASE(STRING, StringType, ::parquet::ByteArrayType)
-    TYPED_BATCH_CASE(TIMESTAMP, TimestampType, ::parquet::Int64Type)
-    default:
-      return Status::NotImplemented(field_->type->ToString());
-  }
-}
-
-void FlatColumnReader::Impl::NextRowGroup() {
-  if (next_row_group_ < reader_->metadata()->num_row_groups()) {
-    column_reader_ = reader_->RowGroup(next_row_group_)->Column(column_index_);
-    next_row_group_++;
-  } else {
-    column_reader_ = nullptr;
-  }
-}
-
-FlatColumnReader::FlatColumnReader(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
-
-FlatColumnReader::~FlatColumnReader() {}
-
-Status FlatColumnReader::NextBatch(int batch_size, std::shared_ptr<Array>* out) {
-  return impl_->NextBatch(batch_size, out);
-}
-
-}  // namespace parquet
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/reader.h b/cpp/src/arrow/parquet/reader.h
deleted file mode 100644
index 2689bebea30ef..0000000000000
--- a/cpp/src/arrow/parquet/reader.h
+++ /dev/null
@@ -1,146 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_PARQUET_READER_H
-#define ARROW_PARQUET_READER_H
-
-#include <memory>
-
-#include "parquet/api/reader.h"
-#include "parquet/api/schema.h"
-
-#include "arrow/io/interfaces.h"
-#include "arrow/parquet/io.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Array;
-class MemoryPool;
-class RecordBatch;
-class Status;
-class Table;
-
-namespace parquet {
-
-class FlatColumnReader;
-
-// Arrow read adapter class for deserializing Parquet files as Arrow row
-// batches.
-//
-// TODO(wesm): nested data does not always make sense with this user
-// interface unless you are only reading a single leaf node from a branch of
-// a table. For example:
-//
-// repeated group data {
-//   optional group record {
-//     optional int32 val1;
-//     optional byte_array val2;
-//     optional bool val3;
-//   }
-//   optional int32 val4;
-// }
-//
-// In the Parquet file, there are 3 leaf nodes:
-//
-// * data.record.val1
-// * data.record.val2
-// * data.record.val3
-// * data.val4
-//
-// When materializing this data in an Arrow array, we would have:
-//
-// data: list<struct<
-//   record: struct<
-//    val1: int32,
-//    val2: string (= list<uint8>),
-//    val3: bool,
-//   >,
-//   val4: int32
-// >>
-//
-// However, in the Parquet format, each leaf node has its own repetition and
-// definition levels describing the structure of the intermediate nodes in
-// this array structure. Thus, we will need to scan the leaf data for a group
-// of leaf nodes part of the same type tree to create a single result Arrow
-// nested array structure.
-//
-// This is additionally complicated "chunky" repeated fields or very large byte
-// arrays
-class ARROW_EXPORT FileReader {
- public:
-  FileReader(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader);
-
-  // Since the distribution of columns amongst a Parquet file's row groups may
-  // be uneven (the number of values in each column chunk can be different), we
-  // provide a column-oriented read interface. The ColumnReader hides the
-  // details of paging through the file's row groups and yielding
-  // fully-materialized arrow::Array instances
-  //
-  // Returns error status if the column of interest is not flat.
-  Status GetFlatColumn(int i, std::unique_ptr<FlatColumnReader>* out);
-  // Read column as a whole into an Array.
-  Status ReadFlatColumn(int i, std::shared_ptr<Array>* out);
-  // Read a table of flat columns into a Table.
-  Status ReadFlatTable(std::shared_ptr<Table>* out);
-
-  virtual ~FileReader();
-
- private:
-  class ARROW_NO_EXPORT Impl;
-  std::unique_ptr<Impl> impl_;
-};
-
-// At this point, the column reader is a stream iterator. It only knows how to
-// read the next batch of values for a particular column from the file until it
-// runs out.
-//
-// We also do not expose any internal Parquet details, such as row groups. This
-// might change in the future.
-class ARROW_EXPORT FlatColumnReader {
- public:
-  virtual ~FlatColumnReader();
-
-  // Scan the next array of the indicated size. The actual size of the
-  // returned array may be less than the passed size depending how much data is
-  // available in the file.
-  //
-  // When all the data in the file has been exhausted, the result is set to
-  // nullptr.
-  //
-  // Returns Status::OK on a successful read, including if you have exhausted
-  // the data available in the file.
-  Status NextBatch(int batch_size, std::shared_ptr<Array>* out);
-
- private:
-  class ARROW_NO_EXPORT Impl;
-  std::unique_ptr<Impl> impl_;
-  explicit FlatColumnReader(std::unique_ptr<Impl> impl);
-
-  friend class FileReader;
-};
-
-// Helper function to create a file reader from an implementation of an Arrow
-// readable file
-ARROW_EXPORT
-Status OpenFile(const std::shared_ptr<io::ReadableFileInterface>& file,
-    ParquetAllocator* allocator, std::unique_ptr<FileReader>* reader);
-
-}  // namespace parquet
-}  // namespace arrow
-
-#endif  // ARROW_PARQUET_READER_H
diff --git a/cpp/src/arrow/parquet/schema.cc b/cpp/src/arrow/parquet/schema.cc
deleted file mode 100644
index ff32e51bacd8b..0000000000000
--- a/cpp/src/arrow/parquet/schema.cc
+++ /dev/null
@@ -1,344 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/parquet/schema.h"
-
-#include <string>
-#include <vector>
-
-#include "parquet/api/schema.h"
-
-#include "arrow/parquet/utils.h"
-#include "arrow/types/decimal.h"
-#include "arrow/types/string.h"
-#include "arrow/util/status.h"
-
-using parquet::Repetition;
-using parquet::schema::Node;
-using parquet::schema::NodePtr;
-using parquet::schema::GroupNode;
-using parquet::schema::PrimitiveNode;
-
-using ParquetType = parquet::Type;
-using parquet::LogicalType;
-
-namespace arrow {
-
-namespace parquet {
-
-const auto BOOL = std::make_shared<BooleanType>();
-const auto UINT8 = std::make_shared<UInt8Type>();
-const auto INT8 = std::make_shared<Int8Type>();
-const auto UINT16 = std::make_shared<UInt16Type>();
-const auto INT16 = std::make_shared<Int16Type>();
-const auto UINT32 = std::make_shared<UInt32Type>();
-const auto INT32 = std::make_shared<Int32Type>();
-const auto UINT64 = std::make_shared<UInt64Type>();
-const auto INT64 = std::make_shared<Int64Type>();
-const auto FLOAT = std::make_shared<FloatType>();
-const auto DOUBLE = std::make_shared<DoubleType>();
-const auto UTF8 = std::make_shared<StringType>();
-const auto TIMESTAMP_MS = std::make_shared<TimestampType>(TimestampType::Unit::MILLI);
-const auto BINARY = std::make_shared<ListType>(std::make_shared<Field>("", UINT8));
-
-TypePtr MakeDecimalType(const PrimitiveNode* node) {
-  int precision = node->decimal_metadata().precision;
-  int scale = node->decimal_metadata().scale;
-  return std::make_shared<DecimalType>(precision, scale);
-}
-
-static Status FromByteArray(const PrimitiveNode* node, TypePtr* out) {
-  switch (node->logical_type()) {
-    case LogicalType::UTF8:
-      *out = UTF8;
-      break;
-    case LogicalType::DECIMAL:
-      *out = MakeDecimalType(node);
-      break;
-    default:
-      // BINARY
-      *out = BINARY;
-      break;
-  }
-  return Status::OK();
-}
-
-static Status FromFLBA(const PrimitiveNode* node, TypePtr* out) {
-  switch (node->logical_type()) {
-    case LogicalType::NONE:
-      *out = BINARY;
-      break;
-    case LogicalType::DECIMAL:
-      *out = MakeDecimalType(node);
-      break;
-    default:
-      return Status::NotImplemented("unhandled type");
-      break;
-  }
-
-  return Status::OK();
-}
-
-static Status FromInt32(const PrimitiveNode* node, TypePtr* out) {
-  switch (node->logical_type()) {
-    case LogicalType::NONE:
-      *out = INT32;
-      break;
-    case LogicalType::UINT_8:
-      *out = UINT8;
-      break;
-    case LogicalType::INT_8:
-      *out = INT8;
-      break;
-    case LogicalType::UINT_16:
-      *out = UINT16;
-      break;
-    case LogicalType::INT_16:
-      *out = INT16;
-      break;
-    case LogicalType::UINT_32:
-      *out = UINT32;
-      break;
-    case LogicalType::DECIMAL:
-      *out = MakeDecimalType(node);
-      break;
-    default:
-      return Status::NotImplemented("Unhandled logical type for int32");
-      break;
-  }
-  return Status::OK();
-}
-
-static Status FromInt64(const PrimitiveNode* node, TypePtr* out) {
-  switch (node->logical_type()) {
-    case LogicalType::NONE:
-      *out = INT64;
-      break;
-    case LogicalType::UINT_64:
-      *out = UINT64;
-      break;
-    case LogicalType::DECIMAL:
-      *out = MakeDecimalType(node);
-      break;
-    case LogicalType::TIMESTAMP_MILLIS:
-      *out = TIMESTAMP_MS;
-      break;
-    default:
-      return Status::NotImplemented("Unhandled logical type for int64");
-      break;
-  }
-  return Status::OK();
-}
-
-// TODO: Logical Type Handling
-Status NodeToField(const NodePtr& node, std::shared_ptr<Field>* out) {
-  std::shared_ptr<DataType> type;
-
-  if (node->is_repeated()) {
-    return Status::NotImplemented("No support yet for repeated node types");
-  }
-
-  if (node->is_group()) {
-    const GroupNode* group = static_cast<const GroupNode*>(node.get());
-    std::vector<std::shared_ptr<Field>> fields(group->field_count());
-    for (int i = 0; i < group->field_count(); i++) {
-      RETURN_NOT_OK(NodeToField(group->field(i), &fields[i]));
-    }
-    type = std::make_shared<StructType>(fields);
-  } else {
-    // Primitive (leaf) node
-    const PrimitiveNode* primitive = static_cast<const PrimitiveNode*>(node.get());
-
-    switch (primitive->physical_type()) {
-      case ParquetType::BOOLEAN:
-        type = BOOL;
-        break;
-      case ParquetType::INT32:
-        RETURN_NOT_OK(FromInt32(primitive, &type));
-        break;
-      case ParquetType::INT64:
-        RETURN_NOT_OK(FromInt64(primitive, &type));
-        break;
-      case ParquetType::INT96:
-        // TODO: Do we have that type in Arrow?
-        // type = TypePtr(new Int96Type());
-        return Status::NotImplemented("int96");
-      case ParquetType::FLOAT:
-        type = FLOAT;
-        break;
-      case ParquetType::DOUBLE:
-        type = DOUBLE;
-        break;
-      case ParquetType::BYTE_ARRAY:
-        // TODO: Do we have that type in Arrow?
-        RETURN_NOT_OK(FromByteArray(primitive, &type));
-        break;
-      case ParquetType::FIXED_LEN_BYTE_ARRAY:
-        RETURN_NOT_OK(FromFLBA(primitive, &type));
-        break;
-    }
-  }
-
-  *out = std::make_shared<Field>(node->name(), type, !node->is_required());
-  return Status::OK();
-}
-
-Status FromParquetSchema(
-    const ::parquet::SchemaDescriptor* parquet_schema, std::shared_ptr<Schema>* out) {
-  // TODO(wesm): Consider adding an arrow::Schema name attribute, which comes
-  // from the root Parquet node
-  const GroupNode* schema_node =
-      static_cast<const GroupNode*>(parquet_schema->group_node());
-
-  std::vector<std::shared_ptr<Field>> fields(schema_node->field_count());
-  for (int i = 0; i < schema_node->field_count(); i++) {
-    RETURN_NOT_OK(NodeToField(schema_node->field(i), &fields[i]));
-  }
-
-  *out = std::make_shared<Schema>(fields);
-  return Status::OK();
-}
-
-Status StructToNode(const std::shared_ptr<StructType>& type, const std::string& name,
-    bool nullable, const ::parquet::WriterProperties& properties, NodePtr* out) {
-  Repetition::type repetition = Repetition::REQUIRED;
-  if (nullable) { repetition = Repetition::OPTIONAL; }
-
-  std::vector<NodePtr> children(type->num_children());
-  for (int i = 0; i < type->num_children(); i++) {
-    RETURN_NOT_OK(FieldToNode(type->child(i), properties, &children[i]));
-  }
-
-  *out = GroupNode::Make(name, repetition, children);
-  return Status::OK();
-}
-
-Status FieldToNode(const std::shared_ptr<Field>& field,
-    const ::parquet::WriterProperties& properties, NodePtr* out) {
-  LogicalType::type logical_type = LogicalType::NONE;
-  ParquetType::type type;
-  Repetition::type repetition = Repetition::REQUIRED;
-  if (field->nullable) { repetition = Repetition::OPTIONAL; }
-  int length = -1;
-
-  switch (field->type->type) {
-    // TODO:
-    // case Type::NA:
-    // break;
-    case Type::BOOL:
-      type = ParquetType::BOOLEAN;
-      break;
-    case Type::UINT8:
-      type = ParquetType::INT32;
-      logical_type = LogicalType::UINT_8;
-      break;
-    case Type::INT8:
-      type = ParquetType::INT32;
-      logical_type = LogicalType::INT_8;
-      break;
-    case Type::UINT16:
-      type = ParquetType::INT32;
-      logical_type = LogicalType::UINT_16;
-      break;
-    case Type::INT16:
-      type = ParquetType::INT32;
-      logical_type = LogicalType::INT_16;
-      break;
-    case Type::UINT32:
-      if (properties.version() == ::parquet::ParquetVersion::PARQUET_1_0) {
-        type = ParquetType::INT64;
-      } else {
-        type = ParquetType::INT32;
-        logical_type = LogicalType::UINT_32;
-      }
-      break;
-    case Type::INT32:
-      type = ParquetType::INT32;
-      break;
-    case Type::UINT64:
-      type = ParquetType::INT64;
-      logical_type = LogicalType::UINT_64;
-      break;
-    case Type::INT64:
-      type = ParquetType::INT64;
-      break;
-    case Type::FLOAT:
-      type = ParquetType::FLOAT;
-      break;
-    case Type::DOUBLE:
-      type = ParquetType::DOUBLE;
-      break;
-    case Type::STRING:
-      type = ParquetType::BYTE_ARRAY;
-      logical_type = LogicalType::UTF8;
-      break;
-    case Type::BINARY:
-      type = ParquetType::BYTE_ARRAY;
-      break;
-    case Type::DATE:
-      type = ParquetType::INT32;
-      logical_type = LogicalType::DATE;
-      break;
-    case Type::TIMESTAMP: {
-      auto timestamp_type = static_cast<TimestampType*>(field->type.get());
-      if (timestamp_type->unit != TimestampType::Unit::MILLI) {
-        return Status::NotImplemented(
-            "Other timestamp units than millisecond are not yet support with parquet.");
-      }
-      type = ParquetType::INT64;
-      logical_type = LogicalType::TIMESTAMP_MILLIS;
-    } break;
-    case Type::TIMESTAMP_DOUBLE:
-      type = ParquetType::INT64;
-      // This is specified as seconds since the UNIX epoch
-      // TODO: Converted type in Parquet?
-      // logical_type = LogicalType::TIMESTAMP_MILLIS;
-      break;
-    case Type::TIME:
-      type = ParquetType::INT64;
-      logical_type = LogicalType::TIME_MILLIS;
-      break;
-    case Type::STRUCT: {
-      auto struct_type = std::static_pointer_cast<StructType>(field->type);
-      return StructToNode(struct_type, field->name, field->nullable, properties, out);
-    } break;
-    default:
-      // TODO: LIST, DENSE_UNION, SPARE_UNION, JSON_SCALAR, DECIMAL, DECIMAL_TEXT, VARCHAR
-      return Status::NotImplemented("unhandled type");
-  }
-  *out = PrimitiveNode::Make(field->name, repetition, type, logical_type, length);
-  return Status::OK();
-}
-
-Status ToParquetSchema(const Schema* arrow_schema,
-    const ::parquet::WriterProperties& properties,
-    std::shared_ptr<::parquet::SchemaDescriptor>* out) {
-  std::vector<NodePtr> nodes(arrow_schema->num_fields());
-  for (int i = 0; i < arrow_schema->num_fields(); i++) {
-    RETURN_NOT_OK(FieldToNode(arrow_schema->field(i), properties, &nodes[i]));
-  }
-
-  NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, nodes);
-  *out = std::make_shared<::parquet::SchemaDescriptor>();
-  PARQUET_CATCH_NOT_OK((*out)->Init(schema));
-
-  return Status::OK();
-}
-
-}  // namespace parquet
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h
deleted file mode 100644
index 88b5977d223a4..0000000000000
--- a/cpp/src/arrow/parquet/schema.h
+++ /dev/null
@@ -1,53 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_PARQUET_SCHEMA_H
-#define ARROW_PARQUET_SCHEMA_H
-
-#include <memory>
-
-#include "parquet/api/schema.h"
-#include "parquet/api/writer.h"
-
-#include "arrow/schema.h"
-#include "arrow/type.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Status;
-
-namespace parquet {
-
-Status ARROW_EXPORT NodeToField(
-    const ::parquet::schema::NodePtr& node, std::shared_ptr<Field>* out);
-
-Status ARROW_EXPORT FromParquetSchema(
-    const ::parquet::SchemaDescriptor* parquet_schema, std::shared_ptr<Schema>* out);
-
-Status ARROW_EXPORT FieldToNode(const std::shared_ptr<Field>& field,
-    const ::parquet::WriterProperties& properties, ::parquet::schema::NodePtr* out);
-
-Status ARROW_EXPORT ToParquetSchema(const Schema* arrow_schema,
-    const ::parquet::WriterProperties& properties,
-    std::shared_ptr<::parquet::SchemaDescriptor>* out);
-
-}  // namespace parquet
-
-}  // namespace arrow
-
-#endif  // ARROW_PARQUET_SCHEMA_H
diff --git a/cpp/src/arrow/parquet/test-util.h b/cpp/src/arrow/parquet/test-util.h
deleted file mode 100644
index 68a7fb94c2aed..0000000000000
--- a/cpp/src/arrow/parquet/test-util.h
+++ /dev/null
@@ -1,193 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <string>
-#include <vector>
-
-#include "arrow/test-util.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-
-namespace arrow {
-
-namespace parquet {
-
-template <typename ArrowType>
-using is_arrow_float = std::is_floating_point<typename ArrowType::c_type>;
-
-template <typename ArrowType>
-using is_arrow_int = std::is_integral<typename ArrowType::c_type>;
-
-template <typename ArrowType>
-using is_arrow_string = std::is_same<ArrowType, StringType>;
-
-template <class ArrowType>
-typename std::enable_if<is_arrow_float<ArrowType>::value,
-    std::shared_ptr<PrimitiveArray>>::type
-NonNullArray(size_t size) {
-  std::vector<typename ArrowType::c_type> values;
-  ::arrow::test::random_real<typename ArrowType::c_type>(size, 0, 0, 1, &values);
-  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
-  builder.Append(values.data(), values.size());
-  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
-}
-
-template <class ArrowType>
-typename std::enable_if<is_arrow_int<ArrowType>::value,
-    std::shared_ptr<PrimitiveArray>>::type
-NonNullArray(size_t size) {
-  std::vector<typename ArrowType::c_type> values;
-  ::arrow::test::randint<typename ArrowType::c_type>(size, 0, 64, &values);
-  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
-  builder.Append(values.data(), values.size());
-  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
-}
-
-template <class ArrowType>
-typename std::enable_if<is_arrow_string<ArrowType>::value,
-    std::shared_ptr<StringArray>>::type
-NonNullArray(size_t size) {
-  StringBuilder builder(default_memory_pool(), std::make_shared<StringType>());
-  for (size_t i = 0; i < size; i++) {
-    builder.Append("test-string");
-  }
-  return std::static_pointer_cast<StringArray>(builder.Finish());
-}
-
-template <>
-std::shared_ptr<PrimitiveArray> NonNullArray<BooleanType>(size_t size) {
-  std::vector<uint8_t> values;
-  ::arrow::test::randint<uint8_t>(size, 0, 1, &values);
-  BooleanBuilder builder(default_memory_pool(), std::make_shared<BooleanType>());
-  builder.Append(values.data(), values.size());
-  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
-}
-
-// This helper function only supports (size/2) nulls.
-template <typename ArrowType>
-typename std::enable_if<is_arrow_float<ArrowType>::value,
-    std::shared_ptr<PrimitiveArray>>::type
-NullableArray(size_t size, size_t num_nulls) {
-  std::vector<typename ArrowType::c_type> values;
-  ::arrow::test::random_real<typename ArrowType::c_type>(size, 0, 0, 1, &values);
-  std::vector<uint8_t> valid_bytes(size, 1);
-
-  for (size_t i = 0; i < num_nulls; i++) {
-    valid_bytes[i * 2] = 0;
-  }
-
-  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
-  builder.Append(values.data(), values.size(), valid_bytes.data());
-  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
-}
-
-// This helper function only supports (size/2) nulls.
-template <typename ArrowType>
-typename std::enable_if<is_arrow_int<ArrowType>::value,
-    std::shared_ptr<PrimitiveArray>>::type
-NullableArray(size_t size, size_t num_nulls) {
-  std::vector<typename ArrowType::c_type> values;
-  ::arrow::test::randint<typename ArrowType::c_type>(size, 0, 64, &values);
-  std::vector<uint8_t> valid_bytes(size, 1);
-
-  for (size_t i = 0; i < num_nulls; i++) {
-    valid_bytes[i * 2] = 0;
-  }
-
-  NumericBuilder<ArrowType> builder(default_memory_pool(), std::make_shared<ArrowType>());
-  builder.Append(values.data(), values.size(), valid_bytes.data());
-  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
-}
-
-// This helper function only supports (size/2) nulls yet.
-template <typename ArrowType>
-typename std::enable_if<is_arrow_string<ArrowType>::value,
-    std::shared_ptr<StringArray>>::type
-NullableArray(size_t size, size_t num_nulls) {
-  std::vector<uint8_t> valid_bytes(size, 1);
-
-  for (size_t i = 0; i < num_nulls; i++) {
-    valid_bytes[i * 2] = 0;
-  }
-
-  StringBuilder builder(default_memory_pool(), std::make_shared<StringType>());
-  for (size_t i = 0; i < size; i++) {
-    builder.Append("test-string");
-  }
-  return std::static_pointer_cast<StringArray>(builder.Finish());
-}
-
-// This helper function only supports (size/2) nulls yet.
-template <>
-std::shared_ptr<PrimitiveArray> NullableArray<BooleanType>(
-    size_t size, size_t num_nulls) {
-  std::vector<uint8_t> values;
-  ::arrow::test::randint<uint8_t>(size, 0, 1, &values);
-  std::vector<uint8_t> valid_bytes(size, 1);
-
-  for (size_t i = 0; i < num_nulls; i++) {
-    valid_bytes[i * 2] = 0;
-  }
-
-  BooleanBuilder builder(default_memory_pool(), std::make_shared<BooleanType>());
-  builder.Append(values.data(), values.size(), valid_bytes.data());
-  return std::static_pointer_cast<PrimitiveArray>(builder.Finish());
-}
-
-std::shared_ptr<Column> MakeColumn(
-    const std::string& name, const std::shared_ptr<Array>& array, bool nullable) {
-  auto field = std::make_shared<Field>(name, array->type(), nullable);
-  return std::make_shared<Column>(field, array);
-}
-
-std::shared_ptr<Table> MakeSimpleTable(
-    const std::shared_ptr<Array>& values, bool nullable) {
-  std::shared_ptr<Column> column = MakeColumn("col", values, nullable);
-  std::vector<std::shared_ptr<Column>> columns({column});
-  std::vector<std::shared_ptr<Field>> fields({column->field()});
-  auto schema = std::make_shared<Schema>(fields);
-  return std::make_shared<Table>("table", schema, columns);
-}
-
-template <typename T>
-void ExpectArray(T* expected, Array* result) {
-  PrimitiveArray* p_array = static_cast<PrimitiveArray*>(result);
-  for (int i = 0; i < result->length(); i++) {
-    EXPECT_EQ(expected[i], reinterpret_cast<const T*>(p_array->data()->data())[i]);
-  }
-}
-
-template <typename ArrowType>
-void ExpectArray(typename ArrowType::c_type* expected, Array* result) {
-  PrimitiveArray* p_array = static_cast<PrimitiveArray*>(result);
-  for (int64_t i = 0; i < result->length(); i++) {
-    EXPECT_EQ(expected[i],
-        reinterpret_cast<const typename ArrowType::c_type*>(p_array->data()->data())[i]);
-  }
-}
-
-template <>
-void ExpectArray<BooleanType>(uint8_t* expected, Array* result) {
-  BooleanBuilder builder(default_memory_pool(), std::make_shared<BooleanType>());
-  builder.Append(expected, result->length());
-  std::shared_ptr<Array> expected_array = builder.Finish();
-  EXPECT_TRUE(result->Equals(expected_array));
-}
-
-}  // namespace parquet
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/utils.h b/cpp/src/arrow/parquet/utils.h
deleted file mode 100644
index bcc46be60e6ec..0000000000000
--- a/cpp/src/arrow/parquet/utils.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_PARQUET_UTILS_H
-#define ARROW_PARQUET_UTILS_H
-
-#include <sstream>
-
-#include "arrow/util/status.h"
-#include "parquet/exception.h"
-
-namespace arrow {
-namespace parquet {
-
-#define PARQUET_CATCH_NOT_OK(s) \
-  try {                         \
-    (s);                        \
-  } catch (const ::parquet::ParquetException& e) { return Status::Invalid(e.what()); }
-
-#define PARQUET_IGNORE_NOT_OK(s) \
-  try {                          \
-    (s);                         \
-  } catch (const ::parquet::ParquetException& e) {}
-
-#define PARQUET_THROW_NOT_OK(s)                    \
-  do {                                             \
-    ::arrow::Status _s = (s);                      \
-    if (!_s.ok()) {                                \
-      std::stringstream ss;                        \
-      ss << "Arrow error: " << _s.ToString();      \
-      throw ::parquet::ParquetException(ss.str()); \
-    }                                              \
-  } while (0);
-
-}  // namespace parquet
-}  // namespace arrow
-
-#endif  // ARROW_PARQUET_UTILS_H
diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc
deleted file mode 100644
index 2b47f1461c9f4..0000000000000
--- a/cpp/src/arrow/parquet/writer.cc
+++ /dev/null
@@ -1,365 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/parquet/writer.h"
-
-#include <algorithm>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/column.h"
-#include "arrow/table.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/parquet/schema.h"
-#include "arrow/parquet/utils.h"
-#include "arrow/util/status.h"
-
-using parquet::ParquetFileWriter;
-using parquet::ParquetVersion;
-using parquet::schema::GroupNode;
-
-namespace arrow {
-namespace parquet {
-
-class FileWriter::Impl {
- public:
-  Impl(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer);
-
-  Status NewRowGroup(int64_t chunk_size);
-  template <typename ParquetType, typename ArrowType>
-  Status TypedWriteBatch(::parquet::ColumnWriter* writer, const PrimitiveArray* data,
-      int64_t offset, int64_t length);
-
-  // TODO(uwe): Same code as in reader.cc the only difference is the name of the temporary
-  // buffer
-  template <typename InType, typename OutType>
-  struct can_copy_ptr {
-    static constexpr bool value =
-        std::is_same<InType, OutType>::value ||
-        (std::is_integral<InType>{} && std::is_integral<OutType>{} &&
-            (sizeof(InType) == sizeof(OutType)));
-  };
-
-  template <typename InType, typename OutType,
-      typename std::enable_if<can_copy_ptr<InType, OutType>::value>::type* = nullptr>
-  Status ConvertPhysicalType(const InType* in_ptr, int64_t, const OutType** out_ptr) {
-    *out_ptr = reinterpret_cast<const OutType*>(in_ptr);
-    return Status::OK();
-  }
-
-  template <typename InType, typename OutType,
-      typename std::enable_if<not can_copy_ptr<InType, OutType>::value>::type* = nullptr>
-  Status ConvertPhysicalType(
-      const InType* in_ptr, int64_t length, const OutType** out_ptr) {
-    RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(OutType)));
-    OutType* mutable_out_ptr = reinterpret_cast<OutType*>(data_buffer_.mutable_data());
-    std::copy(in_ptr, in_ptr + length, mutable_out_ptr);
-    *out_ptr = mutable_out_ptr;
-    return Status::OK();
-  }
-
-  Status WriteFlatColumnChunk(const PrimitiveArray* data, int64_t offset, int64_t length);
-  Status WriteFlatColumnChunk(const StringArray* data, int64_t offset, int64_t length);
-  Status Close();
-
-  virtual ~Impl() {}
-
- private:
-  friend class FileWriter;
-
-  MemoryPool* pool_;
-  // Buffer used for storing the data of an array converted to the physical type
-  // as expected by parquet-cpp.
-  PoolBuffer data_buffer_;
-  PoolBuffer def_levels_buffer_;
-  std::unique_ptr<::parquet::ParquetFileWriter> writer_;
-  ::parquet::RowGroupWriter* row_group_writer_;
-};
-
-FileWriter::Impl::Impl(
-    MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer)
-    : pool_(pool),
-      data_buffer_(pool),
-      writer_(std::move(writer)),
-      row_group_writer_(nullptr) {}
-
-Status FileWriter::Impl::NewRowGroup(int64_t chunk_size) {
-  if (row_group_writer_ != nullptr) { PARQUET_CATCH_NOT_OK(row_group_writer_->Close()); }
-  PARQUET_CATCH_NOT_OK(row_group_writer_ = writer_->AppendRowGroup(chunk_size));
-  return Status::OK();
-}
-
-template <typename ParquetType, typename ArrowType>
-Status FileWriter::Impl::TypedWriteBatch(::parquet::ColumnWriter* column_writer,
-    const PrimitiveArray* data, int64_t offset, int64_t length) {
-  using ArrowCType = typename ArrowType::c_type;
-  using ParquetCType = typename ParquetType::c_type;
-
-  DCHECK((offset + length) <= data->length());
-  auto data_ptr = reinterpret_cast<const ArrowCType*>(data->data()->data()) + offset;
-  auto writer =
-      reinterpret_cast<::parquet::TypedColumnWriter<ParquetType>*>(column_writer);
-  if (writer->descr()->max_definition_level() == 0) {
-    // no nulls, just dump the data
-    const ParquetCType* data_writer_ptr = nullptr;
-    RETURN_NOT_OK((ConvertPhysicalType<ArrowCType, ParquetCType>(
-        data_ptr, length, &data_writer_ptr)));
-    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, nullptr, nullptr, data_writer_ptr));
-  } else if (writer->descr()->max_definition_level() == 1) {
-    RETURN_NOT_OK(def_levels_buffer_.Resize(length * sizeof(int16_t)));
-    int16_t* def_levels_ptr =
-        reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
-    if (data->null_count() == 0) {
-      std::fill(def_levels_ptr, def_levels_ptr + length, 1);
-      const ParquetCType* data_writer_ptr = nullptr;
-      RETURN_NOT_OK((ConvertPhysicalType<ArrowCType, ParquetCType>(
-          data_ptr, length, &data_writer_ptr)));
-      PARQUET_CATCH_NOT_OK(
-          writer->WriteBatch(length, def_levels_ptr, nullptr, data_writer_ptr));
-    } else {
-      RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(ParquetCType)));
-      auto buffer_ptr = reinterpret_cast<ParquetCType*>(data_buffer_.mutable_data());
-      int buffer_idx = 0;
-      for (int i = 0; i < length; i++) {
-        if (data->IsNull(offset + i)) {
-          def_levels_ptr[i] = 0;
-        } else {
-          def_levels_ptr[i] = 1;
-          buffer_ptr[buffer_idx++] = static_cast<ParquetCType>(data_ptr[i]);
-        }
-      }
-      PARQUET_CATCH_NOT_OK(
-          writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
-    }
-  } else {
-    return Status::NotImplemented("no support for max definition level > 1 yet");
-  }
-  PARQUET_CATCH_NOT_OK(writer->Close());
-  return Status::OK();
-}
-
-// This specialization seems quite similar but it significantly differs in two points:
-// * offset is added at the most latest time to the pointer as we have sub-byte access
-// * Arrow data is stored bitwise thus we cannot use std::copy to transform from
-//   ArrowType::c_type to ParquetType::c_type
-template <>
-Status FileWriter::Impl::TypedWriteBatch<::parquet::BooleanType, BooleanType>(
-    ::parquet::ColumnWriter* column_writer, const PrimitiveArray* data, int64_t offset,
-    int64_t length) {
-  DCHECK((offset + length) <= data->length());
-  RETURN_NOT_OK(data_buffer_.Resize(length));
-  auto data_ptr = reinterpret_cast<const uint8_t*>(data->data()->data());
-  auto buffer_ptr = reinterpret_cast<bool*>(data_buffer_.mutable_data());
-  auto writer = reinterpret_cast<::parquet::TypedColumnWriter<::parquet::BooleanType>*>(
-      column_writer);
-  if (writer->descr()->max_definition_level() == 0) {
-    // no nulls, just dump the data
-    for (int64_t i = 0; i < length; i++) {
-      buffer_ptr[i] = util::get_bit(data_ptr, offset + i);
-    }
-    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, nullptr, nullptr, buffer_ptr));
-  } else if (writer->descr()->max_definition_level() == 1) {
-    RETURN_NOT_OK(def_levels_buffer_.Resize(length * sizeof(int16_t)));
-    int16_t* def_levels_ptr =
-        reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
-    if (data->null_count() == 0) {
-      std::fill(def_levels_ptr, def_levels_ptr + length, 1);
-      for (int64_t i = 0; i < length; i++) {
-        buffer_ptr[i] = util::get_bit(data_ptr, offset + i);
-      }
-      // TODO(PARQUET-644): write boolean values as a packed bitmap
-      PARQUET_CATCH_NOT_OK(
-          writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
-    } else {
-      int buffer_idx = 0;
-      for (int i = 0; i < length; i++) {
-        if (data->IsNull(offset + i)) {
-          def_levels_ptr[i] = 0;
-        } else {
-          def_levels_ptr[i] = 1;
-          buffer_ptr[buffer_idx++] = util::get_bit(data_ptr, offset + i);
-        }
-      }
-      PARQUET_CATCH_NOT_OK(
-          writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
-    }
-  } else {
-    return Status::NotImplemented("no support for max definition level > 1 yet");
-  }
-  PARQUET_CATCH_NOT_OK(writer->Close());
-  return Status::OK();
-}
-
-Status FileWriter::Impl::Close() {
-  if (row_group_writer_ != nullptr) { PARQUET_CATCH_NOT_OK(row_group_writer_->Close()); }
-  PARQUET_CATCH_NOT_OK(writer_->Close());
-  return Status::OK();
-}
-
-#define TYPED_BATCH_CASE(ENUM, ArrowType, ParquetType)                            \
-  case Type::ENUM:                                                                \
-    return TypedWriteBatch<ParquetType, ArrowType>(writer, data, offset, length); \
-    break;
-
-Status FileWriter::Impl::WriteFlatColumnChunk(
-    const PrimitiveArray* data, int64_t offset, int64_t length) {
-  ::parquet::ColumnWriter* writer;
-  PARQUET_CATCH_NOT_OK(writer = row_group_writer_->NextColumn());
-  switch (data->type_enum()) {
-    TYPED_BATCH_CASE(BOOL, BooleanType, ::parquet::BooleanType)
-    TYPED_BATCH_CASE(UINT8, UInt8Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(INT8, Int8Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(UINT16, UInt16Type, ::parquet::Int32Type)
-    TYPED_BATCH_CASE(INT16, Int16Type, ::parquet::Int32Type)
-    case Type::UINT32:
-      if (writer_->properties()->version() == ParquetVersion::PARQUET_1_0) {
-        // Parquet 1.0 reader cannot read the UINT_32 logical type. Thus we need
-        // to use the larger Int64Type to store them lossless.
-        return TypedWriteBatch<::parquet::Int64Type, UInt32Type>(
-            writer, data, offset, length);
-      } else {
-        return TypedWriteBatch<::parquet::Int32Type, UInt32Type>(
-            writer, data, offset, length);
-      }
-      TYPED_BATCH_CASE(INT32, Int32Type, ::parquet::Int32Type)
-      TYPED_BATCH_CASE(UINT64, UInt64Type, ::parquet::Int64Type)
-      TYPED_BATCH_CASE(INT64, Int64Type, ::parquet::Int64Type)
-      TYPED_BATCH_CASE(TIMESTAMP, TimestampType, ::parquet::Int64Type)
-      TYPED_BATCH_CASE(FLOAT, FloatType, ::parquet::FloatType)
-      TYPED_BATCH_CASE(DOUBLE, DoubleType, ::parquet::DoubleType)
-    default:
-      return Status::NotImplemented(data->type()->ToString());
-  }
-}
-
-Status FileWriter::Impl::WriteFlatColumnChunk(
-    const StringArray* data, int64_t offset, int64_t length) {
-  ::parquet::ColumnWriter* column_writer;
-  PARQUET_CATCH_NOT_OK(column_writer = row_group_writer_->NextColumn());
-  DCHECK((offset + length) <= data->length());
-  RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(::parquet::ByteArray)));
-  auto buffer_ptr = reinterpret_cast<::parquet::ByteArray*>(data_buffer_.mutable_data());
-  auto values = std::dynamic_pointer_cast<PrimitiveArray>(data->values());
-  auto data_ptr = reinterpret_cast<const uint8_t*>(values->data()->data());
-  DCHECK(values != nullptr);
-  auto writer = reinterpret_cast<::parquet::TypedColumnWriter<::parquet::ByteArrayType>*>(
-      column_writer);
-  if (writer->descr()->max_definition_level() > 0) {
-    RETURN_NOT_OK(def_levels_buffer_.Resize(length * sizeof(int16_t)));
-  }
-  int16_t* def_levels_ptr = reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data());
-  if (writer->descr()->max_definition_level() == 0 || data->null_count() == 0) {
-    // no nulls, just dump the data
-    for (int64_t i = 0; i < length; i++) {
-      buffer_ptr[i] = ::parquet::ByteArray(
-          data->value_length(i + offset), data_ptr + data->value_offset(i));
-    }
-    if (writer->descr()->max_definition_level() > 0) {
-      std::fill(def_levels_ptr, def_levels_ptr + length, 1);
-    }
-    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
-  } else if (writer->descr()->max_definition_level() == 1) {
-    int buffer_idx = 0;
-    for (int64_t i = 0; i < length; i++) {
-      if (data->IsNull(offset + i)) {
-        def_levels_ptr[i] = 0;
-      } else {
-        def_levels_ptr[i] = 1;
-        buffer_ptr[buffer_idx++] = ::parquet::ByteArray(
-            data->value_length(i + offset), data_ptr + data->value_offset(i + offset));
-      }
-    }
-    PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, def_levels_ptr, nullptr, buffer_ptr));
-  } else {
-    return Status::NotImplemented("no support for max definition level > 1 yet");
-  }
-  PARQUET_CATCH_NOT_OK(writer->Close());
-  return Status::OK();
-}
-
-FileWriter::FileWriter(
-    MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer)
-    : impl_(new FileWriter::Impl(pool, std::move(writer))) {}
-
-Status FileWriter::NewRowGroup(int64_t chunk_size) {
-  return impl_->NewRowGroup(chunk_size);
-}
-
-Status FileWriter::WriteFlatColumnChunk(
-    const Array* array, int64_t offset, int64_t length) {
-  int64_t real_length = length;
-  if (length == -1) { real_length = array->length(); }
-  if (array->type_enum() == Type::STRING) {
-    auto string_array = dynamic_cast<const StringArray*>(array);
-    DCHECK(string_array);
-    return impl_->WriteFlatColumnChunk(string_array, offset, real_length);
-  } else {
-    auto primitive_array = dynamic_cast<const PrimitiveArray*>(array);
-    if (!primitive_array) {
-      return Status::NotImplemented("Table must consist of PrimitiveArray instances");
-    }
-    return impl_->WriteFlatColumnChunk(primitive_array, offset, real_length);
-  }
-}
-
-Status FileWriter::Close() {
-  return impl_->Close();
-}
-
-MemoryPool* FileWriter::memory_pool() const {
-  return impl_->pool_;
-}
-
-FileWriter::~FileWriter() {}
-
-Status WriteFlatTable(const Table* table, MemoryPool* pool,
-    const std::shared_ptr<::parquet::OutputStream>& sink, int64_t chunk_size,
-    const std::shared_ptr<::parquet::WriterProperties>& properties) {
-  std::shared_ptr<::parquet::SchemaDescriptor> parquet_schema;
-  RETURN_NOT_OK(
-      ToParquetSchema(table->schema().get(), *properties.get(), &parquet_schema));
-  auto schema_node = std::static_pointer_cast<GroupNode>(parquet_schema->schema_root());
-  std::unique_ptr<ParquetFileWriter> parquet_writer =
-      ParquetFileWriter::Open(sink, schema_node, properties);
-  FileWriter writer(pool, std::move(parquet_writer));
-
-  // TODO(ARROW-232) Support writing chunked arrays.
-  for (int i = 0; i < table->num_columns(); i++) {
-    if (table->column(i)->data()->num_chunks() != 1) {
-      return Status::NotImplemented("No support for writing chunked arrays yet.");
-    }
-  }
-
-  for (int chunk = 0; chunk * chunk_size < table->num_rows(); chunk++) {
-    int64_t offset = chunk * chunk_size;
-    int64_t size = std::min(chunk_size, table->num_rows() - offset);
-    RETURN_NOT_OK_ELSE(writer.NewRowGroup(size), PARQUET_IGNORE_NOT_OK(writer.Close()));
-    for (int i = 0; i < table->num_columns(); i++) {
-      std::shared_ptr<Array> array = table->column(i)->data()->chunk(0);
-      RETURN_NOT_OK_ELSE(writer.WriteFlatColumnChunk(array.get(), offset, size),
-          PARQUET_IGNORE_NOT_OK(writer.Close()));
-    }
-  }
-
-  return writer.Close();
-}
-
-}  // namespace parquet
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h
deleted file mode 100644
index ecc6a9f8be3de..0000000000000
--- a/cpp/src/arrow/parquet/writer.h
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_PARQUET_WRITER_H
-#define ARROW_PARQUET_WRITER_H
-
-#include <memory>
-
-#include "parquet/api/schema.h"
-#include "parquet/api/writer.h"
-
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Array;
-class MemoryPool;
-class PrimitiveArray;
-class RecordBatch;
-class Status;
-class StringArray;
-class Table;
-
-namespace parquet {
-
-/**
- * Iterative API:
- *  Start a new RowGroup/Chunk with NewRowGroup
- *  Write column-by-column the whole column chunk
- */
-class ARROW_EXPORT FileWriter {
- public:
-  FileWriter(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer);
-
-  Status NewRowGroup(int64_t chunk_size);
-  Status WriteFlatColumnChunk(const Array* data, int64_t offset = 0, int64_t length = -1);
-  Status Close();
-
-  virtual ~FileWriter();
-
-  MemoryPool* memory_pool() const;
-
- private:
-  class ARROW_NO_EXPORT Impl;
-  std::unique_ptr<Impl> impl_;
-};
-
-/**
- * Write a flat Table to Parquet.
- *
- * The table shall only consist of nullable, non-repeated columns of primitive type.
- */
-Status ARROW_EXPORT WriteFlatTable(const Table* table, MemoryPool* pool,
-    const std::shared_ptr<::parquet::OutputStream>& sink, int64_t chunk_size,
-    const std::shared_ptr<::parquet::WriterProperties>& properties =
-        ::parquet::default_writer_properties());
-
-}  // namespace parquet
-
-}  // namespace arrow
-
-#endif  // ARROW_PARQUET_WRITER_H
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index 2f0037024c78d..745ed8f7edb99 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -64,7 +64,7 @@ Status StringArray::Validate() const {
 // This used to be a static member variable of BinaryBuilder, but it can cause
 // valgrind to report a (spurious?) memory leak when needed in other shared
 // libraries. The problem came up while adding explicit visibility to libarrow
-// and libarrow_parquet
+// and libparquet_arrow
 static TypePtr kBinaryValueType = TypePtr(new UInt8Type());
 
 BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 522895808de5e..6357e3c1725e3 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -1,5 +1,5 @@
 # Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
+# or more cod ntributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
@@ -294,12 +294,12 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     add_library(${LIB_NAME} STATIC IMPORTED)
     set_target_properties(${LIB_NAME}
       PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
-    message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
+    message(STATUS "Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
   else()
     add_library(${LIB_NAME} SHARED IMPORTED)
     set_target_properties(${LIB_NAME}
       PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
-    message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+    message(STATUS "Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
   endif()
 
   if(ARG_DEPS)
@@ -443,12 +443,12 @@ set(LINK_LIBS
   arrow_io
 )
 
-if(PARQUET_FOUND AND ARROW_PARQUET_FOUND)
-  ADD_THIRDPARTY_LIB(arrow_parquet
-    SHARED_LIB ${ARROW_PARQUET_SHARED_LIB})
+if(PARQUET_FOUND AND PARQUET_ARROW_FOUND)
+  ADD_THIRDPARTY_LIB(parquet_arrow
+    SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
   set(LINK_LIBS
     ${LINK_LIBS}
-    arrow_parquet)
+    parquet_arrow)
   set(CYTHON_EXTENSIONS
     ${CYTHON_EXTENSIONS}
     parquet)
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 5d5efc431a48f..9919746520b4c 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -42,11 +42,6 @@ find_library(ARROW_LIB_PATH NAMES arrow
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
-find_library(ARROW_PARQUET_LIB_PATH NAMES arrow_parquet
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
-
 find_library(ARROW_IO_LIB_PATH NAMES arrow_io
   PATHS
   ${ARROW_SEARCH_LIB_PATH}
@@ -56,7 +51,6 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
   set(ARROW_IO_LIB_NAME libarrow_io)
-  set(ARROW_PARQUET_LIB_NAME libarrow_parquet)
 
   set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
@@ -82,20 +76,6 @@ else ()
   set(ARROW_FOUND FALSE)
 endif ()
 
-if(ARROW_PARQUET_LIB_PATH)
-  set(ARROW_PARQUET_FOUND TRUE)
-  set(ARROW_PARQUET_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PARQUET_LIB_NAME}.a)
-  set(ARROW_PARQUET_SHARED_LIB ${ARROW_LIBS}/${ARROW_PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-  if (NOT Arrow_FIND_QUIETLY)
-    message(STATUS "Found the Arrow Parquet library: ${ARROW_PARQUET_LIB_PATH}")
-  endif ()
-else()
-  if (NOT Arrow_FIND_QUIETLY)
-    message(STATUS "Could not find Arrow Parquet library")
-  endif()
-  set(ARROW_PARQUET_FOUND FALSE)
-endif()
-
 mark_as_advanced(
   ARROW_INCLUDE_DIR
   ARROW_LIBS
@@ -103,6 +83,4 @@ mark_as_advanced(
   ARROW_SHARED_LIB
   ARROW_IO_STATIC_LIB
   ARROW_IO_SHARED_LIB
-  ARROW_PARQUET_STATIC_LIB
-  ARROW_PARQUET_SHARED_LIB
 )
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index f932a93149354..9085b0bb29866 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -44,6 +44,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
   cdef cppclass ColumnDescriptor:
     pass
 
+
 cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
     cdef cppclass ColumnReader:
         pass
@@ -77,6 +78,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         @staticmethod
         unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 
+
 cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
     cdef cppclass ParquetOutputStream" parquet::OutputStream":
         pass
@@ -91,7 +93,7 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
             shared_ptr[WriterProperties] build()
 
 
-cdef extern from "arrow/parquet/io.h" namespace "arrow::parquet" nogil:
+cdef extern from "parquet/arrow/io.h" namespace "parquet::arrow" nogil:
     cdef cppclass ParquetAllocator:
         ParquetAllocator()
         ParquetAllocator(MemoryPool* pool)
@@ -103,7 +105,7 @@ cdef extern from "arrow/parquet/io.h" namespace "arrow::parquet" nogil:
         Open(const shared_ptr[ReadableFileInterface]& file)
 
 
-cdef extern from "arrow/parquet/reader.h" namespace "arrow::parquet" nogil:
+cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
     CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
                      ParquetAllocator* allocator,
                      unique_ptr[FileReader]* reader)
@@ -113,14 +115,14 @@ cdef extern from "arrow/parquet/reader.h" namespace "arrow::parquet" nogil:
         CStatus ReadFlatTable(shared_ptr[CTable]* out);
 
 
-cdef extern from "arrow/parquet/schema.h" namespace "arrow::parquet" nogil:
+cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
     CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema,
                               shared_ptr[CSchema]* out)
     CStatus ToParquetSchema(const CSchema* arrow_schema,
                             shared_ptr[SchemaDescriptor]* out)
 
 
-cdef extern from "arrow/parquet/writer.h" namespace "arrow::parquet" nogil:
+cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
     cdef CStatus WriteFlatTable(
         const CTable* table, MemoryPool* pool,
         const shared_ptr[ParquetOutputStream]& sink,

From 45d88328dd73a331b8099c07dc1332cc585ff8d2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 27 Sep 2016 09:45:05 -0400
Subject: [PATCH 0143/1644] ARROW-293: [C++] Implement Arrow IO interfaces for
 operating system files

I started with the code I put together previously for Feather and conformed it to the `arrow::io` API. There's a bunch of Windows compatibility stuff; I left this until we add CI for Windows and can sort this out.

We should also refactor the memory mapped file interfaces to be based on this common code (see ARROW-294).

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #146 from wesm/ARROW-293 and squashes the following commits:

a2653b7 [Wes McKinney] cpplint
d56ef06 [Wes McKinney] Test the rest of ReadableFile methods
43126ca [Wes McKinney] Drafting OS file IO implementations based on Feather implementation. Work on test suite
---
 cpp/CMakeLists.txt                    |   2 +-
 cpp/src/arrow/io/CMakeLists.txt       |   6 +
 cpp/src/arrow/io/file.cc              | 485 ++++++++++++++++++++++++++
 cpp/src/arrow/io/file.h               |  96 +++++
 cpp/src/arrow/io/io-file-test.cc      | 290 +++++++++++++++
 cpp/src/arrow/io/libhdfs_shim.cc      |   2 +-
 cpp/src/arrow/io/memory.h             |   2 +-
 cpp/src/arrow/io/mman.h               | 189 ++++++++++
 cpp/src/arrow/ipc/adapter.cc          |   4 +-
 cpp/src/arrow/ipc/file.cc             |   2 +-
 cpp/src/arrow/types/primitive-test.cc |   3 +-
 cpp/src/arrow/util/logging.h          |   6 +-
 cpp/src/arrow/util/memory-pool.cc     |   4 +-
 cpp/src/arrow/util/status-test.cc     |   2 +-
 14 files changed, 1080 insertions(+), 13 deletions(-)
 create mode 100644 cpp/src/arrow/io/file.cc
 create mode 100644 cpp/src/arrow/io/file.h
 create mode 100644 cpp/src/arrow/io/io-file-test.cc
 create mode 100644 cpp/src/arrow/io/mman.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f3f4a7dac0100..d65c715319694 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -570,7 +570,7 @@ if (UNIX)
   add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
   --verbose=2
   --linelength=90
-  --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11,-runtime/references
+  --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11,-runtime/references,-build/include_order
   ${FILTERED_LINT_FILES})
 endif (UNIX)
 
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 87e227ef80d80..d2e3491b75f12 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -38,6 +38,7 @@ set(ARROW_IO_TEST_LINK_LIBS
   ${ARROW_IO_PRIVATE_LINK_LIBS})
 
 set(ARROW_IO_SRCS
+  file.cc
   memory.cc
 )
 
@@ -103,12 +104,17 @@ if (APPLE)
     INSTALL_NAME_DIR "@rpath")
 endif()
 
+ADD_ARROW_TEST(io-file-test)
+ARROW_TEST_LINK_LIBRARIES(io-file-test
+  ${ARROW_IO_TEST_LINK_LIBS})
+
 ADD_ARROW_TEST(io-memory-test)
 ARROW_TEST_LINK_LIBRARIES(io-memory-test
   ${ARROW_IO_TEST_LINK_LIBS})
 
 # Headers: top level
 install(FILES
+  file.h
   hdfs.h
   interfaces.h
   memory.h
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
new file mode 100644
index 0000000000000..87bae7f3928ec
--- /dev/null
+++ b/cpp/src/arrow/io/file.cc
@@ -0,0 +1,485 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Ensure 64-bit off_t for platforms where it matters
+#ifdef _FILE_OFFSET_BITS
+#undef _FILE_OFFSET_BITS
+#endif
+
+#define _FILE_OFFSET_BITS 64
+
+#include "arrow/io/file.h"
+
+#if _WIN32 || _WIN64
+#if _WIN64
+#define ENVIRONMENT64
+#else
+#define ENVIRONMENT32
+#endif
+#endif
+
+// sys/mman.h not present in Visual Studio or Cygwin
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include "arrow/io/mman.h"
+#undef Realloc
+#undef Free
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#ifndef _MSC_VER  // POSIX-like platforms
+
+#include <unistd.h>
+
+// Not available on some platforms
+#ifndef errno_t
+#define errno_t int
+#endif
+
+#endif  // _MSC_VER
+
+// defines that
+#if defined(__MINGW32__)
+#define ARROW_WRITE_SHMODE S_IRUSR | S_IWUSR
+#elif defined(_MSC_VER)  // Visual Studio
+
+#else  // gcc / clang on POSIX platforms
+#define ARROW_WRITE_SHMODE S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH
+#endif
+
+// ----------------------------------------------------------------------
+// C++ standard library
+
+#include <algorithm>
+#include <cstring>
+#include <iostream>
+#include <limits>
+#include <sstream>
+#include <vector>
+
+#if defined(_MSC_VER)
+#include <codecvt>
+#include <locale>
+#endif
+
+// ----------------------------------------------------------------------
+// file compatibility stuff
+
+#if defined(__MINGW32__)  // MinGW
+// nothing
+#elif defined(_MSC_VER)  // Visual Studio
+#include <io.h>
+#else  // POSIX / Linux
+// nothing
+#endif
+
+#include <cstdio>
+
+// POSIX systems do not have this
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
+// ----------------------------------------------------------------------
+// Other Arrow includes
+
+#include "arrow/io/interfaces.h"
+
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace io {
+
+// ----------------------------------------------------------------------
+// Cross-platform file compatability layer
+
+static inline Status CheckOpenResult(
+    int ret, int errno_actual, const char* filename, size_t filename_length) {
+  if (ret == -1) {
+    // TODO: errno codes to strings
+    std::stringstream ss;
+    ss << "Failed to open file: ";
+#if defined(_MSC_VER)
+    // using wchar_t
+
+    // this requires c++11
+    std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
+    std::wstring wide_string(
+        reinterpret_cast<const wchar_t*>(filename), filename_length / sizeof(wchar_t));
+    std::string byte_string = converter.to_bytes(wide_string);
+    ss << byte_string;
+#else
+    ss << filename;
+#endif
+    return Status::IOError(ss.str());
+  }
+  return Status::OK();
+}
+
+#define CHECK_LSEEK(retval) \
+  if ((retval) == -1) return Status::IOError("lseek failed");
+
+static inline int64_t lseek64_compat(int fd, int64_t pos, int whence) {
+#if defined(_MSC_VER)
+  return _lseeki64(fd, pos, whence);
+#else
+  return lseek(fd, pos, whence);
+#endif
+}
+
+static inline Status FileOpenReadable(const std::string& filename, int* fd) {
+  int ret;
+  errno_t errno_actual = 0;
+#if defined(_MSC_VER)
+  // https://msdn.microsoft.com/en-us/library/w64k0ytk.aspx
+
+  // See GH #209. Here we are assuming that the filename has been encoded in
+  // utf-16le so that unicode filenames can be supported
+  const int nwchars = static_cast<int>(filename.size()) / sizeof(wchar_t);
+  std::vector<wchar_t> wpath(nwchars + 1);
+  memcpy(wpath.data(), filename.data(), filename.size());
+  memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
+
+  errno_actual = _wsopen_s(fd, wpath.data(), _O_RDONLY | _O_BINARY, _SH_DENYNO, _S_IREAD);
+  ret = *fd;
+#else
+  ret = *fd = open(filename.c_str(), O_RDONLY | O_BINARY);
+  errno_actual = errno;
+#endif
+
+  return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
+}
+
+static inline Status FileOpenWriteable(const std::string& filename, int* fd) {
+  int ret;
+  errno_t errno_actual = 0;
+
+#if defined(_MSC_VER)
+  // https://msdn.microsoft.com/en-us/library/w64k0ytk.aspx
+  // Same story with wchar_t as above
+  const int nwchars = static_cast<int>(filename.size()) / sizeof(wchar_t);
+  std::vector<wchar_t> wpath(nwchars + 1);
+  memcpy(wpath.data(), filename.data(), filename.size());
+  memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
+
+  errno_actual = _wsopen_s(
+      fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY, _SH_DENYNO, _S_IWRITE);
+  ret = *fd;
+
+#else
+  ret = *fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY, ARROW_WRITE_SHMODE);
+#endif
+  return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
+}
+
+static inline Status FileTell(int fd, int64_t* pos) {
+  int64_t current_pos;
+
+#if defined(_MSC_VER)
+  current_pos = _telli64(fd);
+  if (current_pos == -1) { return Status::IOError("_telli64 failed"); }
+#else
+  current_pos = lseek64_compat(fd, 0, SEEK_CUR);
+  CHECK_LSEEK(current_pos);
+#endif
+
+  *pos = current_pos;
+  return Status::OK();
+}
+
+static inline Status FileSeek(int fd, int64_t pos) {
+  int64_t ret = lseek64_compat(fd, pos, SEEK_SET);
+  CHECK_LSEEK(ret);
+  return Status::OK();
+}
+
+static inline Status FileRead(
+    int fd, uint8_t* buffer, int64_t nbytes, int64_t* bytes_read) {
+#if defined(_MSC_VER)
+  if (nbytes > INT32_MAX) { return Status::IOError("Unable to read > 2GB blocks yet"); }
+  *bytes_read = _read(fd, buffer, static_cast<unsigned int>(nbytes));
+#else
+  *bytes_read = read(fd, buffer, nbytes);
+#endif
+
+  if (*bytes_read == -1) {
+    // TODO(wesm): errno to string
+    return Status::IOError("Error reading bytes from file");
+  }
+
+  return Status::OK();
+}
+
+static inline Status FileWrite(int fd, const uint8_t* buffer, int64_t nbytes) {
+  int ret;
+#if defined(_MSC_VER)
+  if (nbytes > INT32_MAX) {
+    return Status::IOError("Unable to write > 2GB blocks to file yet");
+  }
+  ret = _write(fd, buffer, static_cast<unsigned int>(nbytes));
+#else
+  ret = write(fd, buffer, nbytes);
+#endif
+
+  if (ret == -1) {
+    // TODO(wesm): errno to string
+    return Status::IOError("Error writing bytes to file");
+  }
+  return Status::OK();
+}
+
+static inline Status FileGetSize(int fd, int64_t* size) {
+  int64_t ret;
+
+  // Save current position
+  int64_t current_position = lseek64_compat(fd, 0, SEEK_CUR);
+  CHECK_LSEEK(current_position);
+
+  // move to end of the file
+  ret = lseek64_compat(fd, 0, SEEK_END);
+  CHECK_LSEEK(ret);
+
+  // Get file length
+  ret = lseek64_compat(fd, 0, SEEK_CUR);
+  CHECK_LSEEK(ret);
+
+  *size = ret;
+
+  // Restore file position
+  ret = lseek64_compat(fd, current_position, SEEK_SET);
+  CHECK_LSEEK(ret);
+
+  return Status::OK();
+}
+
+static inline Status FileClose(int fd) {
+  int ret;
+
+#if defined(_MSC_VER)
+  ret = _close(fd);
+#else
+  ret = close(fd);
+#endif
+
+  if (ret == -1) { return Status::IOError("error closing file"); }
+  return Status::OK();
+}
+
+class OSFile {
+ public:
+  OSFile() : fd_(-1), is_open_(false), size_(-1) {}
+
+  ~OSFile() {}
+
+  Status OpenWritable(const std::string& path) {
+    RETURN_NOT_OK(FileOpenWriteable(path, &fd_));
+    path_ = path;
+    is_open_ = true;
+    return Status::OK();
+  }
+
+  Status OpenReadable(const std::string& path) {
+    RETURN_NOT_OK(FileOpenReadable(path, &fd_));
+    RETURN_NOT_OK(FileGetSize(fd_, &size_));
+
+    // The position should be 0 after GetSize
+    // RETURN_NOT_OK(Seek(0));
+
+    path_ = path;
+    is_open_ = true;
+    return Status::OK();
+  }
+
+  Status Close() {
+    if (is_open_) {
+      RETURN_NOT_OK(FileClose(fd_));
+      is_open_ = false;
+    }
+    return Status::OK();
+  }
+
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+    return FileRead(fd_, out, nbytes, bytes_read);
+  }
+
+  Status Seek(int64_t pos) {
+    if (pos > size_) { pos = size_; }
+    return FileSeek(fd_, pos);
+  }
+
+  Status Tell(int64_t* pos) const { return FileTell(fd_, pos); }
+
+  Status Write(const uint8_t* data, int64_t length) {
+    if (length < 0) { return Status::IOError("Length must be non-negative"); }
+    return FileWrite(fd_, data, length);
+  }
+
+  int fd() const { return fd_; }
+
+  bool is_open() const { return is_open_; }
+  const std::string& path() const { return path_; }
+
+  int64_t size() const { return size_; }
+
+ private:
+  std::string path_;
+
+  // File descriptor
+  int fd_;
+
+  bool is_open_;
+  int64_t size_;
+};
+
+// ----------------------------------------------------------------------
+// ReadableFile implementation
+
+class ReadableFile::ReadableFileImpl : public OSFile {
+ public:
+  explicit ReadableFileImpl(MemoryPool* pool) : OSFile(), pool_(pool) {}
+
+  Status Open(const std::string& path) { return OpenReadable(path); }
+
+  Status ReadBuffer(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+    auto buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(buffer->Resize(nbytes));
+
+    int64_t bytes_read = 0;
+    RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
+
+    // XXX: heuristic
+    if (bytes_read < nbytes / 2) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
+
+    *out = buffer;
+    return Status::OK();
+  }
+
+ private:
+  MemoryPool* pool_;
+};
+
+ReadableFile::ReadableFile(MemoryPool* pool) {
+  impl_.reset(new ReadableFileImpl(pool));
+}
+
+ReadableFile::~ReadableFile() {
+  impl_->Close();
+}
+
+Status ReadableFile::Open(const std::string& path, std::shared_ptr<ReadableFile>* file) {
+  *file = std::shared_ptr<ReadableFile>(new ReadableFile(default_memory_pool()));
+  return (*file)->impl_->Open(path);
+}
+
+Status ReadableFile::Open(const std::string& path, MemoryPool* memory_pool,
+    std::shared_ptr<ReadableFile>* file) {
+  *file = std::shared_ptr<ReadableFile>(new ReadableFile(memory_pool));
+  return (*file)->impl_->Open(path);
+}
+
+Status ReadableFile::Close() {
+  return impl_->Close();
+}
+
+Status ReadableFile::Tell(int64_t* pos) {
+  return impl_->Tell(pos);
+}
+
+Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  return impl_->Read(nbytes, bytes_read, out);
+}
+
+Status ReadableFile::ReadAt(
+    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  RETURN_NOT_OK(Seek(position));
+  return impl_->Read(nbytes, bytes_read, out);
+}
+
+Status ReadableFile::ReadAt(
+    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  RETURN_NOT_OK(Seek(position));
+  return impl_->ReadBuffer(nbytes, out);
+}
+
+Status ReadableFile::GetSize(int64_t* size) {
+  *size = impl_->size();
+  return Status::OK();
+}
+
+Status ReadableFile::Seek(int64_t pos) {
+  return impl_->Seek(pos);
+}
+
+bool ReadableFile::supports_zero_copy() const {
+  return false;
+}
+
+int ReadableFile::file_descriptor() const {
+  return impl_->fd();
+}
+
+// ----------------------------------------------------------------------
+// FileOutputStream
+
+class FileOutputStream::FileOutputStreamImpl : public OSFile {
+ public:
+  Status Open(const std::string& path) { return OpenWritable(path); }
+};
+
+FileOutputStream::FileOutputStream() {
+  impl_.reset(new FileOutputStreamImpl());
+}
+
+FileOutputStream::~FileOutputStream() {
+  impl_->Close();
+}
+
+Status FileOutputStream::Open(
+    const std::string& path, std::shared_ptr<FileOutputStream>* file) {
+  // private ctor
+  *file = std::shared_ptr<FileOutputStream>(new FileOutputStream());
+  return (*file)->impl_->Open(path);
+}
+
+Status FileOutputStream::Close() {
+  return impl_->Close();
+}
+
+Status FileOutputStream::Tell(int64_t* pos) {
+  return impl_->Tell(pos);
+}
+
+Status FileOutputStream::Write(const uint8_t* data, int64_t length) {
+  return impl_->Write(data, length);
+}
+
+int FileOutputStream::file_descriptor() const {
+  return impl_->fd();
+}
+
+}  // namespace io
+}  // namespace arrow
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
new file mode 100644
index 0000000000000..5e714ea966790
--- /dev/null
+++ b/cpp/src/arrow/io/file.h
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// IO interface implementations for OS files
+
+#ifndef ARROW_IO_FILE_H
+#define ARROW_IO_FILE_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "arrow/io/interfaces.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Buffer;
+class MemoryPool;
+class Status;
+
+namespace io {
+
+class ARROW_EXPORT FileOutputStream : public OutputStream {
+ public:
+  ~FileOutputStream();
+
+  static Status Open(const std::string& path, std::shared_ptr<FileOutputStream>* file);
+
+  // OutputStream interface
+  Status Close() override;
+  Status Tell(int64_t* position) override;
+  Status Write(const uint8_t* data, int64_t nbytes) override;
+
+  int file_descriptor() const;
+
+ private:
+  FileOutputStream();
+
+  class ARROW_NO_EXPORT FileOutputStreamImpl;
+  std::unique_ptr<FileOutputStreamImpl> impl_;
+};
+
+// Operating system file
+class ARROW_EXPORT ReadableFile : public ReadableFileInterface {
+ public:
+  ~ReadableFile();
+
+  // Open file, allocate memory (if needed) from default memory pool
+  static Status Open(const std::string& path, std::shared_ptr<ReadableFile>* file);
+
+  // Open file with one's own memory pool for memory allocations
+  static Status Open(const std::string& path, MemoryPool* memory_pool,
+      std::shared_ptr<ReadableFile>* file);
+
+  Status Close() override;
+  Status Tell(int64_t* position) override;
+
+  Status ReadAt(
+      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status GetSize(int64_t* size) override;
+  Status Seek(int64_t position) override;
+
+  bool supports_zero_copy() const override;
+
+  int file_descriptor() const;
+
+ private:
+  explicit ReadableFile(MemoryPool* pool);
+
+  class ARROW_NO_EXPORT ReadableFileImpl;
+  std::unique_ptr<ReadableFileImpl> impl_;
+};
+
+}  // namespace io
+}  // namespace arrow
+
+#endif  // ARROW_IO_FILE_H
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
new file mode 100644
index 0000000000000..cde769ffb6155
--- /dev/null
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -0,0 +1,290 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <fcntl.h>
+#include <fstream>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include "arrow/io/file.h"
+#include "arrow/io/test-common.h"
+#include "arrow/util/memory-pool.h"
+
+namespace arrow {
+namespace io {
+
+static bool FileExists(const std::string& path) {
+  return std::ifstream(path.c_str()).good();
+}
+
+static bool FileIsClosed(int fd) {
+  if (-1 != fcntl(fd, F_GETFD)) { return false; }
+  return errno == EBADF;
+}
+
+class FileTestFixture : public ::testing::Test {
+ public:
+  void SetUp() {
+    path_ = "arrow-test-io-file-output-stream.txt";
+    EnsureFileDeleted();
+  }
+
+  void TearDown() { EnsureFileDeleted(); }
+
+  void EnsureFileDeleted() {
+    if (FileExists(path_)) { std::remove(path_.c_str()); }
+  }
+
+ protected:
+  std::string path_;
+};
+
+// ----------------------------------------------------------------------
+// File output tests
+
+class TestFileOutputStream : public FileTestFixture {
+ public:
+  void OpenFile() { ASSERT_OK(FileOutputStream::Open(path_, &file_)); }
+
+ protected:
+  std::shared_ptr<FileOutputStream> file_;
+};
+
+TEST_F(TestFileOutputStream, DestructorClosesFile) {
+  int fd;
+  {
+    std::shared_ptr<FileOutputStream> file;
+    ASSERT_OK(FileOutputStream::Open(path_, &file));
+    fd = file->file_descriptor();
+  }
+  ASSERT_TRUE(FileIsClosed(fd));
+}
+
+TEST_F(TestFileOutputStream, Close) {
+  OpenFile();
+
+  const char* data = "testdata";
+  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), strlen(data)));
+
+  int fd = file_->file_descriptor();
+  file_->Close();
+
+  ASSERT_TRUE(FileIsClosed(fd));
+
+  // Idempotent
+  file_->Close();
+
+  std::shared_ptr<ReadableFile> rd_file;
+  ASSERT_OK(ReadableFile::Open(path_, &rd_file));
+
+  int64_t size = 0;
+  ASSERT_OK(rd_file->GetSize(&size));
+  ASSERT_EQ(strlen(data), size);
+}
+
+TEST_F(TestFileOutputStream, InvalidWrites) {
+  OpenFile();
+
+  const char* data = "";
+
+  ASSERT_RAISES(IOError, file_->Write(reinterpret_cast<const uint8_t*>(data), -1));
+}
+
+TEST_F(TestFileOutputStream, Tell) {
+  OpenFile();
+
+  int64_t position;
+
+  ASSERT_OK(file_->Tell(&position));
+  ASSERT_EQ(0, position);
+
+  const char* data = "testdata";
+  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), 8));
+  ASSERT_OK(file_->Tell(&position));
+  ASSERT_EQ(8, position);
+}
+
+// ----------------------------------------------------------------------
+// File input tests
+
+class TestReadableFile : public FileTestFixture {
+ public:
+  void OpenFile() { ASSERT_OK(ReadableFile::Open(path_, &file_)); }
+
+  void MakeTestFile() {
+    std::string data = "testdata";
+    std::ofstream stream;
+    stream.open(path_.c_str());
+    stream << data;
+  }
+
+ protected:
+  std::shared_ptr<ReadableFile> file_;
+};
+
+TEST_F(TestReadableFile, DestructorClosesFile) {
+  MakeTestFile();
+
+  int fd;
+  {
+    std::shared_ptr<ReadableFile> file;
+    ASSERT_OK(ReadableFile::Open(path_, &file));
+    fd = file->file_descriptor();
+  }
+  ASSERT_TRUE(FileIsClosed(fd));
+}
+
+TEST_F(TestReadableFile, Close) {
+  MakeTestFile();
+  OpenFile();
+
+  int fd = file_->file_descriptor();
+  file_->Close();
+
+  ASSERT_TRUE(FileIsClosed(fd));
+
+  // Idempotent
+  file_->Close();
+}
+
+TEST_F(TestReadableFile, SeekTellSize) {
+  MakeTestFile();
+  OpenFile();
+
+  int64_t position;
+  ASSERT_OK(file_->Tell(&position));
+  ASSERT_EQ(0, position);
+
+  ASSERT_OK(file_->Seek(4));
+  ASSERT_OK(file_->Tell(&position));
+  ASSERT_EQ(4, position);
+
+  ASSERT_OK(file_->Seek(100));
+  ASSERT_OK(file_->Tell(&position));
+
+  // now at EOF
+  ASSERT_EQ(8, position);
+
+  int64_t size;
+  ASSERT_OK(file_->GetSize(&size));
+  ASSERT_EQ(8, size);
+
+  // does not support zero copy
+  ASSERT_FALSE(file_->supports_zero_copy());
+}
+
+TEST_F(TestReadableFile, Read) {
+  uint8_t buffer[50];
+
+  MakeTestFile();
+  OpenFile();
+
+  int64_t bytes_read;
+  ASSERT_OK(file_->Read(4, &bytes_read, buffer));
+  ASSERT_EQ(4, bytes_read);
+  ASSERT_EQ(0, std::memcmp(buffer, "test", 4));
+
+  ASSERT_OK(file_->Read(10, &bytes_read, buffer));
+  ASSERT_EQ(4, bytes_read);
+  ASSERT_EQ(0, std::memcmp(buffer, "data", 4));
+}
+
+TEST_F(TestReadableFile, ReadAt) {
+  uint8_t buffer[50];
+  const char* test_data = "testdata";
+
+  MakeTestFile();
+  OpenFile();
+
+  int64_t bytes_read;
+  int64_t position;
+
+  ASSERT_OK(file_->ReadAt(0, 4, &bytes_read, buffer));
+  ASSERT_EQ(4, bytes_read);
+  ASSERT_EQ(0, std::memcmp(buffer, "test", 4));
+
+  // position advanced
+  ASSERT_OK(file_->Tell(&position));
+  ASSERT_EQ(4, position);
+
+  ASSERT_OK(file_->ReadAt(4, 10, &bytes_read, buffer));
+  ASSERT_EQ(4, bytes_read);
+  ASSERT_EQ(0, std::memcmp(buffer, "data", 4));
+
+  // position advanced to EOF
+  ASSERT_OK(file_->Tell(&position));
+  ASSERT_EQ(8, position);
+
+  // Check buffer API
+  std::shared_ptr<Buffer> buffer2;
+
+  ASSERT_OK(file_->ReadAt(0, 4, &buffer2));
+  ASSERT_EQ(4, buffer2->size());
+
+  Buffer expected(reinterpret_cast<const uint8_t*>(test_data), 4);
+  ASSERT_TRUE(buffer2->Equals(expected));
+
+  // position advanced
+  ASSERT_OK(file_->Tell(&position));
+  ASSERT_EQ(4, position);
+}
+
+TEST_F(TestReadableFile, NonExistentFile) {
+  ASSERT_RAISES(IOError, ReadableFile::Open("0xDEADBEEF.txt", &file_));
+}
+
+class MyMemoryPool : public MemoryPool {
+ public:
+  MyMemoryPool() : num_allocations_(0) {}
+
+  Status Allocate(int64_t size, uint8_t** out) override {
+    *out = reinterpret_cast<uint8_t*>(std::malloc(size));
+    ++num_allocations_;
+    return Status::OK();
+  }
+
+  void Free(uint8_t* buffer, int64_t size) override { std::free(buffer); }
+
+  int64_t bytes_allocated() const override { return -1; }
+
+  int64_t num_allocations() const { return num_allocations_; }
+
+ private:
+  int64_t num_allocations_;
+};
+
+TEST_F(TestReadableFile, CustomMemoryPool) {
+  MakeTestFile();
+
+  MyMemoryPool pool;
+  ASSERT_OK(ReadableFile::Open(path_, &pool, &file_));
+
+  std::shared_ptr<Buffer> buffer;
+  ASSERT_OK(file_->ReadAt(0, 4, &buffer));
+  ASSERT_OK(file_->ReadAt(4, 8, &buffer));
+
+  ASSERT_EQ(2, pool.num_allocations());
+}
+
+}  // namespace io
+}  // namespace arrow
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
index 0b805abf94c1b..f256c31b4f4b2 100644
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -33,8 +33,8 @@
 #ifndef _WIN32
 #include <dlfcn.h>
 #else
-#include <winsock2.h>
 #include <windows.h>
+#include <winsock2.h>
 
 // TODO(wesm): address when/if we add windows support
 // #include <util/syserr_reporting.hpp>
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 51601a0a62678..6989d732ca752 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -94,7 +94,7 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   Status WriteInternal(const uint8_t* data, int64_t nbytes);
 
   // Hide the internal details of this class for now
-  class MemoryMappedFileImpl;
+  class ARROW_NO_EXPORT MemoryMappedFileImpl;
   std::unique_ptr<MemoryMappedFileImpl> impl_;
 };
 
diff --git a/cpp/src/arrow/io/mman.h b/cpp/src/arrow/io/mman.h
new file mode 100644
index 0000000000000..00d1f93601df3
--- /dev/null
+++ b/cpp/src/arrow/io/mman.h
@@ -0,0 +1,189 @@
+// Copyright https://code.google.com/p/mman-win32/
+//
+// Licensed under the MIT License;
+// You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/MIT
+
+#ifndef _MMAN_WIN32_H
+#define _MMAN_WIN32_H
+
+// Allow use of features specific to Windows XP or later.
+#ifndef _WIN32_WINNT
+// Change this to the appropriate value to target other versions of Windows.
+#define _WIN32_WINNT 0x0501
+
+#endif
+
+#include <errno.h>
+#include <io.h>
+#include <sys/types.h>
+#include <windows.h>
+
+#define PROT_NONE 0
+#define PROT_READ 1
+#define PROT_WRITE 2
+#define PROT_EXEC 4
+
+#define MAP_FILE 0
+#define MAP_SHARED 1
+#define MAP_PRIVATE 2
+#define MAP_TYPE 0xf
+#define MAP_FIXED 0x10
+#define MAP_ANONYMOUS 0x20
+#define MAP_ANON MAP_ANONYMOUS
+
+#define MAP_FAILED ((void*)-1)
+
+/* Flags for msync. */
+#define MS_ASYNC 1
+#define MS_SYNC 2
+#define MS_INVALIDATE 4
+
+#ifndef FILE_MAP_EXECUTE
+#define FILE_MAP_EXECUTE 0x0020
+#endif
+
+static int __map_mman_error(const DWORD err, const int deferr) {
+  if (err == 0) return 0;
+  // TODO: implement
+  return err;
+}
+
+static DWORD __map_mmap_prot_page(const int prot) {
+  DWORD protect = 0;
+
+  if (prot == PROT_NONE) return protect;
+
+  if ((prot & PROT_EXEC) != 0) {
+    protect = ((prot & PROT_WRITE) != 0) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
+  } else {
+    protect = ((prot & PROT_WRITE) != 0) ? PAGE_READWRITE : PAGE_READONLY;
+  }
+
+  return protect;
+}
+
+static DWORD __map_mmap_prot_file(const int prot) {
+  DWORD desiredAccess = 0;
+
+  if (prot == PROT_NONE) return desiredAccess;
+
+  if ((prot & PROT_READ) != 0) desiredAccess |= FILE_MAP_READ;
+  if ((prot & PROT_WRITE) != 0) desiredAccess |= FILE_MAP_WRITE;
+  if ((prot & PROT_EXEC) != 0) desiredAccess |= FILE_MAP_EXECUTE;
+
+  return desiredAccess;
+}
+
+void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
+  HANDLE fm, h;
+
+  void* map = MAP_FAILED;
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4293)
+#endif
+
+  const DWORD dwFileOffsetLow =
+      (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)off : (DWORD)(off & 0xFFFFFFFFL);
+  const DWORD dwFileOffsetHigh =
+      (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL);
+  const DWORD protect = __map_mmap_prot_page(prot);
+  const DWORD desiredAccess = __map_mmap_prot_file(prot);
+
+  const off_t maxSize = off + (off_t)len;
+
+  const DWORD dwMaxSizeLow =
+      (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
+  const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD))
+                                  ? (DWORD)0
+                                  : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+  errno = 0;
+
+  if (len == 0
+      /* Unsupported flag combinations */
+      || (flags & MAP_FIXED) != 0
+      /* Usupported protection combinations */
+      || prot == PROT_EXEC) {
+    errno = EINVAL;
+    return MAP_FAILED;
+  }
+
+  h = ((flags & MAP_ANONYMOUS) == 0) ? (HANDLE)_get_osfhandle(fildes)
+                                     : INVALID_HANDLE_VALUE;
+
+  if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE) {
+    errno = EBADF;
+    return MAP_FAILED;
+  }
+
+  fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);
+
+  if (fm == NULL) {
+    errno = __map_mman_error(GetLastError(), EPERM);
+    return MAP_FAILED;
+  }
+
+  map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);
+
+  CloseHandle(fm);
+
+  if (map == NULL) {
+    errno = __map_mman_error(GetLastError(), EPERM);
+    return MAP_FAILED;
+  }
+
+  return map;
+}
+
+int munmap(void* addr, size_t len) {
+  if (UnmapViewOfFile(addr)) return 0;
+
+  errno = __map_mman_error(GetLastError(), EPERM);
+
+  return -1;
+}
+
+int mprotect(void* addr, size_t len, int prot) {
+  DWORD newProtect = __map_mmap_prot_page(prot);
+  DWORD oldProtect = 0;
+
+  if (VirtualProtect(addr, len, newProtect, &oldProtect)) return 0;
+
+  errno = __map_mman_error(GetLastError(), EPERM);
+
+  return -1;
+}
+
+int msync(void* addr, size_t len, int flags) {
+  if (FlushViewOfFile(addr, len)) return 0;
+
+  errno = __map_mman_error(GetLastError(), EPERM);
+
+  return -1;
+}
+
+int mlock(const void* addr, size_t len) {
+  if (VirtualLock((LPVOID)addr, len)) return 0;
+
+  errno = __map_mman_error(GetLastError(), EPERM);
+
+  return -1;
+}
+
+int munlock(const void* addr, size_t len) {
+  if (VirtualUnlock((LPVOID)addr, len)) return 0;
+
+  errno = __map_mman_error(GetLastError(), EPERM);
+
+  return -1;
+}
+
+#endif
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 89b7fb987c63d..99974a4a4c7b7 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -23,12 +23,12 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
-#include "arrow/io/interfaces.h"
-#include "arrow/io/memory.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
index 2bf10dde266bd..c68244d50258c 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/file.cc
@@ -22,10 +22,10 @@
 #include <sstream>
 #include <vector>
 
+#include "arrow/io/interfaces.h"
 #include "arrow/ipc/adapter.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
-#include "arrow/io/interfaces.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/status.h"
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 87eb0fe3a8bf7..ffebb9269bdc3 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -238,7 +238,8 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 }
 
 typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
-    PInt32, PInt64, PFloat, PDouble> Primitives;
+    PInt32, PInt64, PFloat, PDouble>
+    Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index d320d6adb7caa..b22f07dd6345f 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -117,10 +117,10 @@ class CerrLog {
 // return so we create a new class to give it a hint.
 class FatalLog : public CerrLog {
  public:
-  FatalLog(int /* severity */)  // NOLINT
-      : CerrLog(ARROW_FATAL) {}
+  explicit FatalLog(int /* severity */)  // NOLINT
+      : CerrLog(ARROW_FATAL){}           // NOLINT
 
-  [[noreturn]] ~FatalLog() {
+            [[noreturn]] ~FatalLog() {
     if (has_logged_) { std::cerr << std::endl; }
     std::exit(1);
   }
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
index fed149bc3598c..9f83afe4cb20f 100644
--- a/cpp/src/arrow/util/memory-pool.cc
+++ b/cpp/src/arrow/util/memory-pool.cc
@@ -17,13 +17,13 @@
 
 #include "arrow/util/memory-pool.h"
 
-#include <stdlib.h>
 #include <cstdlib>
 #include <mutex>
 #include <sstream>
+#include <stdlib.h>
 
-#include "arrow/util/status.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/status-test.cc b/cpp/src/arrow/util/status-test.cc
index 45e0ff361ac22..e0ff20fea1233 100644
--- a/cpp/src/arrow/util/status-test.cc
+++ b/cpp/src/arrow/util/status-test.cc
@@ -17,8 +17,8 @@
 
 #include "gtest/gtest.h"
 
-#include "arrow/util/status.h"
 #include "arrow/test-util.h"
+#include "arrow/util/status.h"
 
 namespace arrow {
 

From 03134b11ffd4f63bda2f3cb448713600df6d8fdb Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 27 Sep 2016 09:45:32 -0700
Subject: [PATCH 0144/1644] ARROW-270: Define more generic Interval logical
 type

Author: Julien Le Dem <julien@dremio.com>

Closes #144 from julienledem/interval and squashes the following commits:

eb76fed [Julien Le Dem] ARROW-270: Define more generic Interval logical type
---
 format/Message.fbs                            | 10 ++++-----
 .../src/main/codegen/data/ArrowTypes.tdd      |  8 ++-----
 .../templates/NullableValueVectors.java       |  4 ++--
 .../arrow/vector/schema/TypeLayout.java       | 21 +++++++++++--------
 .../org/apache/arrow/vector/types/Types.java  | 14 ++++++-------
 5 files changed, 27 insertions(+), 30 deletions(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index 657904a7032a5..07da862c32d5d 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -63,10 +63,9 @@ table Timestamp {
   timezone: string;
 }
 
-table IntervalDay {
-}
-
-table IntervalYear {
+enum IntervalUnit: short { YEAR_MONTH, DAY_TIME}
+table Interval {
+  unit: IntervalUnit;
 }
 
 table JSONScalar {
@@ -88,8 +87,7 @@ union Type {
   Date,
   Time,
   Timestamp,
-  IntervalDay,
-  IntervalYear,
+  Interval,
   List,
   Struct_,
   Union,
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 5cb43bed2b69a..9f81f0e3800ed 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -69,12 +69,8 @@
       fields: [{name: "timezone", type: "String"}]
     },
     {
-      name: "IntervalDay",
-      fields: []
-    },
-    {
-      name: "IntervalYear",
-      fields: []
+      name: "Interval",
+      fields: [{name: "unit", type: short}]
     }
   ]
 }
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 486cfeefc7a3b..8f325afad3920 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -105,9 +105,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#elseif minor.class == "TimeStamp">
     field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(""), null);
   <#elseif minor.class == "IntervalDay">
-    field = new Field(name, true, new IntervalDay(), null);
+    field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME), null);
   <#elseif minor.class == "IntervalYear">
-    field = new Field(name, true, new IntervalYear(), null);
+    field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH), null);
   <#elseif minor.class == "VarChar">
     field = new Field(name, true, new Utf8(), null);
   <#elseif minor.class == "VarBinary">
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 885ac2ac3d7f2..072385a215582 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -31,6 +31,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.arrow.flatbuf.IntervalUnit;
 import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor;
@@ -40,12 +41,11 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
-import org.apache.arrow.vector.types.pojo.ArrowType.IntervalDay;
-import org.apache.arrow.vector.types.pojo.ArrowType.IntervalYear;
+import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
 import org.apache.arrow.vector.types.pojo.ArrowType.Time;
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 
@@ -167,14 +167,17 @@ public TypeLayout visit(Time type) {
       }
 
       @Override
-      public TypeLayout visit(IntervalDay type) { // TODO: check size
-        return newFixedWidthTypeLayout(dataVector(64));
+      public TypeLayout visit(Interval type) { // TODO: check size
+        switch (type.getUnit()) {
+        case IntervalUnit.DAY_TIME:
+          return newFixedWidthTypeLayout(dataVector(64));
+        case IntervalUnit.YEAR_MONTH:
+          return newFixedWidthTypeLayout(dataVector(64));
+        default:
+          throw new UnsupportedOperationException("Unknown unit " + type.getUnit());
+        }
       }
 
-      @Override
-      public TypeLayout visit(IntervalYear type) { // TODO: check size
-        return newFixedWidthTypeLayout(dataVector(64));
-      }
     });
     return layout;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 66ef7562ceda1..181d835368265 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -20,6 +20,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.arrow.flatbuf.IntervalUnit;
 import org.apache.arrow.flatbuf.Precision;
 import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.flatbuf.UnionMode;
@@ -78,13 +79,12 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Date;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
-import org.apache.arrow.vector.types.pojo.ArrowType.IntervalDay;
-import org.apache.arrow.vector.types.pojo.ArrowType.IntervalYear;
+import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
 import org.apache.arrow.vector.types.pojo.ArrowType.List;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
 import org.apache.arrow.vector.types.pojo.ArrowType.Time;
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -104,8 +104,8 @@ public class Types {
   public static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
   public static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
   public static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null);
-  public static final Field INTERVALDAY_FIELD = new Field("", true, IntervalDay.INSTANCE, null);
-  public static final Field INTERVALYEAR_FIELD = new Field("", true, IntervalYear.INSTANCE, null);
+  public static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
+  public static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
   public static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(Precision.SINGLE), null);
   public static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(Precision.DOUBLE), null);
   public static final Field LIST_FIELD = new Field("", true, List.INSTANCE, null);
@@ -260,7 +260,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new TimeStampWriterImpl((NullableTimeStampVector) vector);
       }
     },
-    INTERVALDAY(IntervalDay.INSTANCE) {
+    INTERVALDAY(new Interval(IntervalUnit.DAY_TIME)) {
       @Override
       public Field getField() {
         return INTERVALDAY_FIELD;
@@ -276,7 +276,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new IntervalDayWriterImpl((NullableIntervalDayVector) vector);
       }
     },
-    INTERVALYEAR(IntervalYear.INSTANCE) {
+    INTERVALYEAR(new Interval(IntervalUnit.YEAR_MONTH)) {
       @Override
       public Field getField() {
         return INTERVALYEAR_FIELD;

From bae33d622421e6377ab3e9c81dd054c796ab48a3 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 27 Sep 2016 10:39:09 -0700
Subject: [PATCH 0145/1644] ARROW-304: NullableMapReaderImpl.isSet() always
 returns true

Author: Julien Le Dem <julien@dremio.com>

Closes #147 from julienledem/isSet and squashes the following commits:

c06e048 [Julien Le Dem] review feedback
5a33785 [Julien Le Dem] review feedback
af5d613 [Julien Le Dem] ARROW-304: NullableMapReaderImpl.isSet() always returns true
---
 .../complex/impl/NullableMapReaderImpl.java   |  5 ++
 .../vector/complex/impl/UnionListReader.java  |  2 +-
 .../complex/writer/TestComplexWriter.java     | 57 ++++++++++++++++---
 3 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
index 18b35c194a184..7c389e61ae202 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
@@ -42,4 +42,9 @@ public void copyAsField(String name, MapWriter writer){
     NullableMapWriter impl = (NullableMapWriter) writer.map(name);
     impl.container.copyFromSafe(idx(), impl.idx(), nullableMapVector);
   }
+
+  @Override
+  public boolean isSet(){
+    return !nullableMapVector.getAccessor().isNull(idx());
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
index 39cf00421154b..6c7c230226ea3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
@@ -41,7 +41,7 @@ public UnionListReader(ListVector vector) {
 
   @Override
   public boolean isSet() {
-    return true;
+    return !vector.getAccessor().isNull(idx());
   }
 
   private int currentOffset;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index fa710dae5eee8..c1da104da5780 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -17,6 +17,14 @@
  */
 package org.apache.arrow.vector.complex.writer;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.ListVector;
@@ -77,28 +85,33 @@ public void nullableMap() {
     MapVector parent = new MapVector("parent", allocator, null);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
-    MapWriter mapWriter = rootWriter.map("map");
-    BigIntWriter nested = mapWriter.bigInt("nested");
     for (int i = 0; i < COUNT; i++) {
+      rootWriter.setPosition(i);
+      rootWriter.start();
       if (i % 2 == 0) {
+        MapWriter mapWriter = rootWriter.map("map");
         mapWriter.setPosition(i);
         mapWriter.start();
-        nested.writeBigInt(i);
+        mapWriter.bigInt("nested").writeBigInt(i);
         mapWriter.end();
       }
+      rootWriter.end();
     }
     writer.setValueCount(COUNT);
     MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
     for (int i = 0; i < COUNT; i++) {
       rootReader.setPosition(i);
+      assertTrue("index is set: " + i, rootReader.isSet());
+      FieldReader map = rootReader.reader("map");
       if (i % 2 == 0) {
-        Assert.assertNotNull(rootReader.reader("map").readObject());
-        Assert.assertEquals(i, rootReader.reader("map").reader("nested").readLong().longValue());
+        assertTrue("index is set: " + i, map.isSet());
+        assertNotNull("index is set: " + i, map.readObject());
+        assertEquals(i, map.reader("nested").readLong().longValue());
       } else {
-        Assert.assertNull(rootReader.reader("map").readObject());
+        assertFalse("index is not set: " + i, map.isSet());
+        assertNull("index is not set: " + i, map.readObject());
       }
     }
-
     parent.close();
   }
 
@@ -121,11 +134,39 @@ public void listScalarType() {
       listReader.setPosition(i);
       for (int j = 0; j < i % 7; j++) {
         listReader.next();
-        Assert.assertEquals(j, listReader.reader().readInteger().intValue());
+        assertEquals(j, listReader.reader().readInteger().intValue());
       }
     }
   }
 
+  @Test
+  public void listScalarTypeNullable() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      if (i % 2 == 0) {
+        listWriter.setPosition(i);
+        listWriter.startList();
+        for (int j = 0; j < i % 7; j++) {
+          listWriter.writeInt(j);
+        }
+        listWriter.endList();
+      }
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      if (i % 2 == 0) {
+        assertTrue("index is set: " + i, listReader.isSet());
+        assertEquals("correct length at: " + i, i % 7, ((List<?>)listReader.readObject()).size());
+      } else {
+        assertFalse("index is not set: " + i, listReader.isSet());
+        assertNull("index is not set: " + i, listReader.readObject());
+      }
+    }
+  }
 
   @Test
   public void listMapType() {

From 768c7d0be7dde9942235b5312c1c46ab035af86b Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 27 Sep 2016 11:54:35 -0700
Subject: [PATCH 0146/1644] ARROW-257: Add a typeids Vector to Union type

Author: Julien Le Dem <julien@dremio.com>

Closes #143 from julienledem/union and squashes the following commits:

cd1b711 [Julien Le Dem] ARROW-257: Add a typeids Vector to Union type
---
 format/Message.fbs                            |  5 +++
 .../src/main/codegen/data/ArrowTypes.tdd      |  2 +-
 .../src/main/codegen/templates/ArrowType.java | 38 +++++++++++++++----
 .../main/codegen/templates/UnionVector.java   |  7 +++-
 .../org/apache/arrow/vector/types/Types.java  |  2 +-
 .../apache/arrow/vector/pojo/TestConvert.java |  5 ++-
 6 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index 07da862c32d5d..288f5a1b6b2d0 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -23,8 +23,13 @@ table List {
 
 enum UnionMode:short { Sparse, Dense }
 
+/// A union is a complex type with children in Field
+/// By default ids in the type vector refer to the offsets in the children
+/// optionally typeIds provides an indirection between the child offset and the type id
+/// for each child typeIds[offset] is the id used in the type vector
 table Union {
   mode: UnionMode;
+  typeIds: [ int ]; // optional, describes typeid of each child.
 }
 
 table Int {
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 9f81f0e3800ed..9624fecf6aad1 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -30,7 +30,7 @@
     },
     {
       name: "Union",
-      fields: [{name: "mode", type: short}]
+      fields: [{name: "mode", type: short}, {name: "typeIds", type: "int[]"}]
     },
     {
       name: "Int",
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index 29dee20040a53..30f2c68efe0b3 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -33,12 +33,23 @@
 
 import java.util.Objects;
 
+/**
+ * Arrow types
+ **/
 public abstract class ArrowType {
 
   public abstract byte getTypeType();
   public abstract int getType(FlatBufferBuilder builder);
   public abstract <T> T accept(ArrowTypeVisitor<T> visitor);
 
+  /**
+   * to visit the ArrowTypes
+   * <code>
+   *   type.accept(new ArrowTypeVisitor<Type>() {
+   *   ...
+   *   });
+   * </code>
+   */
   public static interface ArrowTypeVisitor<T> {
   <#list arrowTypes.types as type>
     T visit(${type.name} type);
@@ -55,9 +66,7 @@ public static class ${name} extends ArrowType {
     </#if>
 
     <#list fields as field>
-    <#assign fieldName = field.name>
-    <#assign fieldType = field.type>
-    ${fieldType} ${fieldName};
+    ${field.type} ${field.name};
     </#list>
 
     <#if type.fields?size != 0>
@@ -79,6 +88,9 @@ public int getType(FlatBufferBuilder builder) {
       <#if field.type == "String">
       int ${field.name} = builder.createString(this.${field.name});
       </#if>
+      <#if field.type == "int[]">
+      int ${field.name} = org.apache.arrow.flatbuf.${type.name}.create${field.name?cap_first}Vector(builder, this.${field.name});
+      </#if>
       </#list>
       org.apache.arrow.flatbuf.${type.name}.start${type.name}(builder);
       <#list type.fields as field>
@@ -96,7 +108,7 @@ public int getType(FlatBufferBuilder builder) {
     public String toString() {
       return "${name}{"
       <#list fields as field>
-      + ", " + ${field.name}
+        + <#if field.type == "int[]">java.util.Arrays.toString(${field.name})<#else>${field.name}</#if><#if field_has_next> + ", " </#if>
       </#list>
       + "}";
     }
@@ -115,8 +127,7 @@ public boolean equals(Object obj) {
       return true;
       <#else>
       ${type.name} that = (${type.name}) obj;
-      return
-      <#list type.fields as field>Objects.equals(this.${field.name}, that.${field.name}) <#if field_has_next>&&<#else>;</#if>
+      return <#list type.fields as field>Objects.deepEquals(this.${field.name}, that.${field.name}) <#if field_has_next>&&<#else>;</#if>
       </#list>
       </#if>
     }
@@ -134,9 +145,20 @@ public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.a
     <#assign name = type.name>
     <#assign nameLower = type.name?lower_case>
     <#assign fields = type.fields>
-    case Type.${type.name}:
+    case Type.${type.name}: {
       org.apache.arrow.flatbuf.${type.name} ${nameLower}Type = (org.apache.arrow.flatbuf.${type.name}) field.type(new org.apache.arrow.flatbuf.${type.name}());
-      return new ${type.name}(<#list type.fields as field>${nameLower}Type.${field.name}()<#if field_has_next>, </#if></#list>);
+      <#list type.fields as field>
+      <#if field.type == "int[]">
+      ${field.type} ${field.name} = new int[${nameLower}Type.${field.name}Length()];
+      for (int i = 0; i< ${field.name}.length; ++i) {
+        ${field.name}[i] = ${nameLower}Type.${field.name}(i);
+      }
+      <#else>
+      ${field.type} ${field.name} = ${nameLower}Type.${field.name}();
+      </#if>
+      </#list>
+      return new ${type.name}(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+    }
     </#list>
     default:
       throw new UnsupportedOperationException("Unsupported type: " + field.typeType());
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 3014bbba9d52d..b14314d2b0dbb 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -232,10 +232,13 @@ public void clear() {
   @Override
   public Field getField() {
     List<org.apache.arrow.vector.types.pojo.Field> childFields = new ArrayList<>();
-    for (ValueVector v : internalMap.getChildren()) {
+    List<FieldVector> children = internalMap.getChildren();
+    int[] typeIds = new int[children.size()];
+    for (ValueVector v : children) {
+      typeIds[childFields.size()] = v.getMinorType().ordinal();
       childFields.add(v.getField());
     }
-    return new Field(name, true, new ArrowType.Union(Sparse), childFields);
+    return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 181d835368265..6e63ae232781a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -472,7 +472,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UnionListWriter((ListVector) vector);
       }
     },
-    UNION(new Union(UnionMode.Sparse)) {
+    UNION(new Union(UnionMode.Sparse, null)) {
       @Override
       public Field getField() {
         throw new UnsupportedOperationException("Cannot get simple field for Union type");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 448117d84dc3e..ed740cd0f1b78 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -22,11 +22,12 @@
 import static org.junit.Assert.assertEquals;
 
 import org.apache.arrow.flatbuf.UnionMode;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.List;
-import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
+import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -78,7 +79,7 @@ public void nestedSchema() {
     childrenBuilder.add(new Field("child4", true, new List(), ImmutableList.<Field>of(
         new Field("child4.1", true, Utf8.INSTANCE, null)
         )));
-    childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse), ImmutableList.<Field>of(
+    childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMP.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
         new Field("child5.1", true, new Timestamp("UTC"), null),
         new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));

From bd195e304d82dcf6e2cea266b4d0871bd2b88564 Mon Sep 17 00:00:00 2001
From: adeneche <adeneche@dremio.com>
Date: Wed, 28 Sep 2016 07:26:05 -0700
Subject: [PATCH 0147/1644] ARROW-308: UnionListWriter.setPosition() should not
 call startList()

---
 .../codegen/templates/UnionListWriter.java    |  1 -
 .../complex/writer/TestComplexWriter.java     | 32 ++++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index d502803d71616..04531a72128a0 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -84,7 +84,6 @@ public void close() throws Exception {
   @Override
   public void setPosition(int index) {
     super.setPosition(index);
-    startList();
   }
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index c1da104da5780..398aea915b343 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -52,7 +52,7 @@
 
 public class TestComplexWriter {
 
-  static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+  private static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
 
   private static final int COUNT = 100;
 
@@ -115,6 +115,36 @@ public void nullableMap() {
     parent.close();
   }
 
+  @Test
+  public void listOfLists() {
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+
+    rootWriter.start();
+    rootWriter.bigInt("int").writeBigInt(0);
+    rootWriter.list("list").startList();
+    rootWriter.list("list").bigInt().writeBigInt(0);
+    rootWriter.list("list").endList();
+    rootWriter.end();
+
+    rootWriter.setPosition(1);
+    rootWriter.start();
+    rootWriter.bigInt("int").writeBigInt(1);
+    rootWriter.end();
+
+    writer.setValueCount(2);
+
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+
+    rootReader.setPosition(0);
+    assertTrue("row 0 list is not set", rootReader.reader("list").isSet());
+    assertEquals(Long.valueOf(0), rootReader.reader("list").reader().readLong());
+
+    rootReader.setPosition(1);
+    assertFalse("row 1 list is set", rootReader.reader("list").isSet());
+  }
+
   @Test
   public void listScalarType() {
     ListVector listVector = new ListVector("list", allocator, null);

From bf30235fa3672936013db82ed9dd8949433d802e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 28 Sep 2016 21:44:37 -0400
Subject: [PATCH 0148/1644] ARROW-306: Add option to pass cmake arguments via
 environment variable

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #149 from xhochy/arrow-306 and squashes the following commits:

11a3e66 [Uwe L. Korn] ARROW-306: Add option to pass cmake arguments via environment variable
---
 python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index a5db2b025e6ef..d1be122888e7b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -95,7 +95,7 @@ def run(self):
 
     def initialize_options(self):
         _build_ext.initialize_options(self)
-        self.extra_cmake_args = ''
+        self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
 
     CYTHON_MODULE_NAMES = [
         'array',

From 30f60832a5f4bd3063699061796d2107fb7a9738 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 28 Sep 2016 21:45:46 -0400
Subject: [PATCH 0149/1644] ARROW-305: Add compression and use_dictionary
 options to Parquet

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #148 from xhochy/arrow-305 and squashes the following commits:

93d653b [Uwe L. Korn] ARROW-305: Add compression and use_dictionary options to Parquet interface
---
 python/pyarrow/includes/parquet.pxd  | 12 +++++++
 python/pyarrow/parquet.pyx           | 49 +++++++++++++++++++++++++++-
 python/pyarrow/tests/test_parquet.py | 40 +++++++++++++++++++++++
 3 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index 9085b0bb29866..754eeccecc8e9 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -37,6 +37,13 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
       PARQUET_1_0" parquet::ParquetVersion::PARQUET_1_0"
       PARQUET_2_0" parquet::ParquetVersion::PARQUET_2_0"
 
+  enum Compression" parquet::Compression::type":
+      UNCOMPRESSED" parquet::Compression::UNCOMPRESSED"
+      SNAPPY" parquet::Compression::SNAPPY"
+      GZIP" parquet::Compression::GZIP"
+      LZO" parquet::Compression::LZO"
+      BROTLI" parquet::Compression::BROTLI"
+
   cdef cppclass SchemaDescriptor:
     shared_ptr[Node] schema()
     GroupNode* group()
@@ -90,6 +97,11 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
     cdef cppclass WriterProperties:
         cppclass Builder:
             Builder* version(ParquetVersion version)
+            Builder* compression(Compression codec)
+            Builder* compression(const c_string& path, Compression codec)
+            Builder* disable_dictionary()
+            Builder* enable_dictionary()
+            Builder* enable_dictionary(const c_string& path)
             shared_ptr[WriterProperties] build()
 
 
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index fb36b2967c096..099e148abc16f 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -90,7 +90,8 @@ def read_table(source, columns=None):
     return reader.read_all()
 
 
-def write_table(table, filename, chunk_size=None, version=None):
+def write_table(table, filename, chunk_size=None, version=None,
+                use_dictionary=True, compression=None):
     """
     Write a Table to Parquet format
 
@@ -102,6 +103,11 @@ def write_table(table, filename, chunk_size=None, version=None):
         The maximum number of rows in each Parquet RowGroup
     version : {"1.0", "2.0"}, default "1.0"
         The Parquet format version, defaults to 1.0
+    use_dictionary : bool or list
+        Specify if we should use dictionary encoding in general or only for
+        some columns.
+    compression : str or dict
+        Specify the compression codec, either on a general basis or per-column.
     """
     cdef Table table_ = table
     cdef CTable* ctable_ = table_.table
@@ -121,6 +127,47 @@ def write_table(table, filename, chunk_size=None, version=None):
         else:
             raise ArrowException("Unsupported Parquet format version")
 
+    if isinstance(use_dictionary, bool):
+        if use_dictionary:
+            properties_builder.enable_dictionary()
+        else:
+            properties_builder.disable_dictionary()
+    else:
+        # Deactivate dictionary encoding by default
+        properties_builder.disable_dictionary()
+        for column in use_dictionary:
+            properties_builder.enable_dictionary(column)
+
+    if isinstance(compression, basestring):
+        if compression == "NONE":
+            properties_builder.compression(UNCOMPRESSED)
+        elif compression == "SNAPPY":
+            properties_builder.compression(SNAPPY)
+        elif compression == "GZIP":
+            properties_builder.compression(GZIP)
+        elif compression == "LZO":
+            properties_builder.compression(LZO)
+        elif compression == "BROTLI":
+            properties_builder.compression(BROTLI)
+        else:
+            raise ArrowException("Unsupport compression codec")
+    elif compression is not None:
+        # Deactivate dictionary encoding by default
+        properties_builder.disable_dictionary()
+        for column, codec in compression.iteritems():
+            if codec == "NONE":
+                properties_builder.compression(column, UNCOMPRESSED)
+            elif codec == "SNAPPY":
+                properties_builder.compression(column, SNAPPY)
+            elif codec == "GZIP":
+                properties_builder.compression(column, GZIP)
+            elif codec == "LZO":
+                properties_builder.compression(column, LZO)
+            elif codec == "BROTLI":
+                properties_builder.compression(column, BROTLI)
+            else:
+                raise ArrowException("Unsupport compression codec")
+
     sink.reset(new LocalFileOutputStream(tobytes(filename)))
     with nogil:
         check_cstatus(WriteFlatTable(ctable_, default_memory_pool(), sink,
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 8a2d8cab57267..0f9f2e40813ce 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -110,3 +110,43 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
     df['uint32'] = df['uint32'].values.astype(np.int64)
 
     pdt.assert_frame_equal(df, df_read)
+
+@parquet
+def test_pandas_parquet_configuration_options(tmpdir):
+    size = 10000
+    np.random.seed(0)
+    df = pd.DataFrame({
+        'uint8': np.arange(size, dtype=np.uint8),
+        'uint16': np.arange(size, dtype=np.uint16),
+        'uint32': np.arange(size, dtype=np.uint32),
+        'uint64': np.arange(size, dtype=np.uint64),
+        'int8': np.arange(size, dtype=np.int16),
+        'int16': np.arange(size, dtype=np.int16),
+        'int32': np.arange(size, dtype=np.int32),
+        'int64': np.arange(size, dtype=np.int64),
+        'float32': np.arange(size, dtype=np.float32),
+        'float64': np.arange(size, dtype=np.float64),
+        'bool': np.random.randn(size) > 0
+    })
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = A.from_pandas_dataframe(df)
+
+    for use_dictionary in [True, False]:
+        A.parquet.write_table(
+                arrow_table,
+                filename.strpath,
+                version="2.0",
+                use_dictionary=use_dictionary)
+        table_read = pq.read_table(filename.strpath)
+        df_read = table_read.to_pandas()
+        pdt.assert_frame_equal(df, df_read)
+
+    for compression in ['NONE', 'SNAPPY', 'GZIP']:
+        A.parquet.write_table(
+                arrow_table,
+                filename.strpath,
+                version="2.0",
+                compression=compression)
+        table_read = pq.read_table(filename.strpath)
+        df_read = table_read.to_pandas()
+        pdt.assert_frame_equal(df, df_read)

From 391ab64d05fc9c5ea89fcc9a9938604954047ada Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 30 Sep 2016 08:53:52 -0700
Subject: [PATCH 0150/1644] ARROW-309: Types.getMinorTypeForArrowType() does
 not work for Union type

Author: Julien Le Dem <julien@dremio.com>

Closes #151 from julienledem/fix_union and squashes the following commits:

01bea42 [Julien Le Dem] fix union
---
 .../org/apache/arrow/vector/types/Types.java  | 145 +++++++++++++-----
 1 file changed, 107 insertions(+), 38 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 6e63ae232781a..2ff93d4b98d11 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -17,12 +17,8 @@
  */
 package org.apache.arrow.vector.types;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import org.apache.arrow.flatbuf.IntervalUnit;
 import org.apache.arrow.flatbuf.Precision;
-import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
@@ -74,9 +70,11 @@
 import org.apache.arrow.vector.complex.impl.VarCharWriterImpl;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor;
 import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
 import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
 import org.apache.arrow.vector.types.pojo.ArrowType.Date;
+import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
@@ -92,26 +90,25 @@
 
 public class Types {
 
-  public static final Field NULL_FIELD = new Field("", true, Null.INSTANCE, null);
-  public static final Field TINYINT_FIELD = new Field("", true, new Int(8, true), null);
-  public static final Field SMALLINT_FIELD = new Field("", true, new Int(16, true), null);
-  public static final Field INT_FIELD = new Field("", true, new Int(32, true), null);
-  public static final Field BIGINT_FIELD = new Field("", true, new Int(64, true), null);
-  public static final Field UINT1_FIELD = new Field("", true, new Int(8, false), null);
-  public static final Field UINT2_FIELD = new Field("", true, new Int(16, false), null);
-  public static final Field UINT4_FIELD = new Field("", true, new Int(32, false), null);
-  public static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
-  public static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
-  public static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
-  public static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null);
-  public static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
-  public static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
-  public static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(Precision.SINGLE), null);
-  public static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(Precision.DOUBLE), null);
-  public static final Field LIST_FIELD = new Field("", true, List.INSTANCE, null);
-  public static final Field VARCHAR_FIELD = new Field("", true, Utf8.INSTANCE, null);
-  public static final Field VARBINARY_FIELD = new Field("", true, Binary.INSTANCE, null);
-  public static final Field BIT_FIELD = new Field("", true, Bool.INSTANCE, null);
+  private static final Field NULL_FIELD = new Field("", true, Null.INSTANCE, null);
+  private static final Field TINYINT_FIELD = new Field("", true, new Int(8, true), null);
+  private static final Field SMALLINT_FIELD = new Field("", true, new Int(16, true), null);
+  private static final Field INT_FIELD = new Field("", true, new Int(32, true), null);
+  private static final Field BIGINT_FIELD = new Field("", true, new Int(64, true), null);
+  private static final Field UINT1_FIELD = new Field("", true, new Int(8, false), null);
+  private static final Field UINT2_FIELD = new Field("", true, new Int(16, false), null);
+  private static final Field UINT4_FIELD = new Field("", true, new Int(32, false), null);
+  private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
+  private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
+  private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
+  private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null);
+  private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
+  private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
+  private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(Precision.SINGLE), null);
+  private static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(Precision.DOUBLE), null);
+  private static final Field VARCHAR_FIELD = new Field("", true, Utf8.INSTANCE, null);
+  private static final Field VARBINARY_FIELD = new Field("", true, Binary.INSTANCE, null);
+  private static final Field BIT_FIELD = new Field("", true, Bool.INSTANCE, null);
 
 
   public enum MinorType {
@@ -427,7 +424,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     UINT4(new Int(32, false)) {
       @Override
       public Field getField() {
-        return UINT8_FIELD;
+        return UINT4_FIELD;
       }
 
       @Override
@@ -506,22 +503,94 @@ public ArrowType getType() {
     public abstract FieldWriter getNewFieldWriter(ValueVector vector);
   }
 
-  private static final Map<ArrowType,MinorType> ARROW_TYPE_MINOR_TYPE_MAP;
-
   public static MinorType getMinorTypeForArrowType(ArrowType arrowType) {
-    if (arrowType.getTypeType() == Type.Decimal) {
-      return MinorType.DECIMAL;
-    }
-    return ARROW_TYPE_MINOR_TYPE_MAP.get(arrowType);
-  }
+    return arrowType.accept(new ArrowTypeVisitor<MinorType>() {
+      @Override public MinorType visit(Null type) {
+        return MinorType.NULL;
+      }
 
-  static {
-    ARROW_TYPE_MINOR_TYPE_MAP = new HashMap<>();
-    for (MinorType minorType : MinorType.values()) {
-      if (minorType != MinorType.DECIMAL) {
-        ARROW_TYPE_MINOR_TYPE_MAP.put(minorType.getType(), minorType);
+      @Override public MinorType visit(Struct_ type) {
+        return MinorType.MAP;
       }
-    }
+
+      @Override public MinorType visit(List type) {
+        return MinorType.LIST;
+      }
+
+      @Override public MinorType visit(Union type) {
+        return MinorType.UNION;
+      }
+
+      @Override
+      public MinorType visit(Int type) {
+        switch (type.getBitWidth()) {
+        case 8:
+          return type.getIsSigned() ? MinorType.TINYINT : MinorType.UINT1;
+        case 16:
+          return type.getIsSigned() ? MinorType.SMALLINT : MinorType.UINT2;
+        case 32:
+          return type.getIsSigned() ? MinorType.INT : MinorType.UINT4;
+        case 64:
+          return type.getIsSigned() ? MinorType.BIGINT : MinorType.UINT8;
+        default:
+          throw new IllegalArgumentException("only 8, 16, 32, 64 supported: " + type);
+        }
+      }
+
+      @Override
+      public MinorType visit(FloatingPoint type) {
+        switch (type.getPrecision()) {
+        case Precision.HALF:
+          throw new UnsupportedOperationException("NYI: " + type);
+        case Precision.SINGLE:
+          return MinorType.FLOAT4;
+        case Precision.DOUBLE:
+          return MinorType.FLOAT8;
+        default:
+          throw new IllegalArgumentException("unknown precision: " + type);
+        }
+      }
+
+      @Override public MinorType visit(Utf8 type) {
+        return MinorType.VARCHAR;
+      }
+
+      @Override public MinorType visit(Binary type) {
+        return MinorType.VARBINARY;
+      }
+
+      @Override public MinorType visit(Bool type) {
+        return MinorType.BIT;
+      }
+
+      @Override public MinorType visit(Decimal type) {
+        return MinorType.DECIMAL;
+      }
+
+      @Override public MinorType visit(Date type) {
+        return MinorType.DATE;
+      }
+
+      @Override public MinorType visit(Time type) {
+        return MinorType.TIME;
+      }
+
+      @Override public MinorType visit(Timestamp type) {
+        return MinorType.TIMESTAMP;
+      }
+
+      @Override
+      public MinorType visit(Interval type) {
+        switch (type.getUnit()) {
+        case IntervalUnit.DAY_TIME:
+          return MinorType.INTERVALDAY;
+        case IntervalUnit.YEAR_MONTH:
+          return MinorType.INTERVALYEAR;
+        default:
+          throw new IllegalArgumentException("unknown unit: " + type);
+        }
+      }
+    });
   }
 
 }

From c7b0480f5c8dadb78b9586dc4e40f3964929d8ef Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Mon, 3 Oct 2016 14:54:15 -0700
Subject: [PATCH 0151/1644] ARROW-314: JSONScalar is unnecessary and unused

Author: Julien Le Dem <julien@dremio.com>

Closes #153 from julienledem/jsonscalar and squashes the following commits:

905027c [Julien Le Dem] ARROW-314: JSONScalar is unnecessary and unused
---
 format/Message.fbs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index 288f5a1b6b2d0..e1758bf3638e8 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -73,10 +73,6 @@ table Interval {
   unit: IntervalUnit;
 }
 
-table JSONScalar {
-  dense:bool=true;
-}
-
 /// ----------------------------------------------------------------------
 /// Top-level Type value, enabling extensible type-specific metadata. We can
 /// add new logical types to Type without breaking backwards compatibility
@@ -95,8 +91,7 @@ union Type {
   Interval,
   List,
   Struct_,
-  Union,
-  JSONScalar
+  Union
 }
 
 /// ----------------------------------------------------------------------

From c3930a062b2d71e3d277d4db1785e24e9183276f Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Mon, 3 Oct 2016 15:17:32 -0700
Subject: [PATCH 0152/1644] ARROW-301: Add user field metadata to IPC schemas

Author: Julien Le Dem <julien@dremio.com>

Closes #154 from julienledem/custom and squashes the following commits:

47a02b7 [Julien Le Dem] ARROW-301: Add user field metadata to IPC schemas
---
 format/Message.fbs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/format/Message.fbs b/format/Message.fbs
index e1758bf3638e8..3d877a2f234af 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -119,6 +119,16 @@ table VectorLayout {
   type: VectorType;
 }
 
+
+/// ----------------------------------------------------------------------
+/// user defined key value pairs to add custom metadata to arrow
+/// key namespacing is the responsibility of the user
+
+table KeyValue {
+  key: string;
+  value: [ubyte];
+}
+
 /// ----------------------------------------------------------------------
 /// A field represents a named column in a record / row batch or child of a
 /// nested type.
@@ -141,6 +151,8 @@ table Field {
   /// does not include children
   /// each recordbatch will return instances of those Buffers.
   layout: [ VectorLayout ];
+  // User-defined metadata
+  custom_metadata: [ KeyValue ];
 }
 
 /// ----------------------------------------------------------------------
@@ -159,6 +171,8 @@ table Schema {
   endianness: Endianness=Little;
 
   fields: [Field];
+  // User-defined metadata
+  custom_metadata: [ KeyValue ];
 }
 
 /// ----------------------------------------------------------------------

From c7e6a0716308766766aaaf4faa2effc5445640c6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 3 Oct 2016 23:14:41 -0400
Subject: [PATCH 0153/1644] ARROW-302: [C++/Python] Implement C++ IO interfaces
 for interacting with Python file and bytes objects

This will enable code (such as arrow IPC or Parquet) that only knows about Arrow's IO subsystem to interact with Python objects in various ways. In other words, when we have in C++:

```
std::shared_ptr<io::ReadableFileInterface> handle = ...;
handle->Read(nbytes, &out);
```

then the C++ file handle could be invoking the `read` method of a Python object. Same goes for `arrow::io::OutputStream` and `write` methods. There's data copying in some places overhead because of the rigid memory ownership semantics of the `PyBytes` type, but this can't be avoided here.

Another nice thing is that if we have some data in a Python bytes object that we want to expose to some other C++ component, we can wrap it in the `PyBytesReader` which provides zero-copy read access to the underlying data.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #152 from wesm/ARROW-302 and squashes the following commits:

2de9f97 [Wes McKinney] Fix compiler warning / bug from OS X
316b845 [Wes McKinney] Code review comments
e791893 [Wes McKinney] Python 2.7 fix
0fc4cf1 [Wes McKinney] cpplint
e9b8c60 [Wes McKinney] Test the size() method and fix bug with missing whence
6481e91 [Wes McKinney] Add a zero-copy reader for PyBytes
7e357eb [Wes McKinney] Get basic Python file read/write working
d470133 [Wes McKinney] Share default implementations of ReadAt, add Buffer-based Read API
737a8db [Wes McKinney] Refactoring, more code sharing with native file interfaces
8be433f [Wes McKinney] Draft PyReadableFile implementation, not yet tested
20a3f28 [Wes McKinney] Draft API for Arrow IO wrappers for Python files
---
 cpp/CMakeLists.txt                      |   2 +
 cpp/src/arrow/io/CMakeLists.txt         |   1 +
 cpp/src/arrow/io/file.cc                |  10 +-
 cpp/src/arrow/io/file.h                 |   6 +-
 cpp/src/arrow/io/hdfs.cc                |  46 ++++-
 cpp/src/arrow/io/hdfs.h                 |  13 +-
 cpp/src/arrow/io/interfaces.cc          |  48 ++++++
 cpp/src/arrow/io/interfaces.h           |  26 +--
 cpp/src/arrow/io/memory.cc              |  40 ++---
 cpp/src/arrow/io/memory.h               |  21 ++-
 python/CMakeLists.txt                   |   1 +
 python/pyarrow/__init__.py              |   5 +-
 python/pyarrow/array.pyx                |  31 ----
 python/pyarrow/error.pxd                |   4 +-
 python/pyarrow/error.pyx                |   2 +-
 python/pyarrow/includes/libarrow_io.pxd |  29 ++++
 python/pyarrow/includes/pyarrow.pxd     |  34 +++-
 python/pyarrow/io.pxd                   |  13 +-
 python/pyarrow/io.pyx                   | 136 ++++++++++-----
 python/pyarrow/parquet.pyx              |   8 +-
 python/pyarrow/table.pyx                |  37 +++-
 python/pyarrow/tests/test_hdfs.py       | 128 ++++++++++++++
 python/pyarrow/tests/test_io.py         | 121 ++++++-------
 python/src/pyarrow/adapters/pandas.cc   |   2 +-
 python/src/pyarrow/common.cc            |  15 ++
 python/src/pyarrow/common.h             |  30 +++-
 python/src/pyarrow/io.cc                | 215 ++++++++++++++++++++++++
 python/src/pyarrow/io.h                 |  97 +++++++++++
 28 files changed, 878 insertions(+), 243 deletions(-)
 create mode 100644 cpp/src/arrow/io/interfaces.cc
 create mode 100644 python/pyarrow/tests/test_hdfs.py
 create mode 100644 python/src/pyarrow/io.cc
 create mode 100644 python/src/pyarrow/io.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d65c715319694..f70c8ab4bccef 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -166,6 +166,8 @@ else()
   message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
 endif ()
 
+message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}")
+
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
 
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index d2e3491b75f12..47bb089386371 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -39,6 +39,7 @@ set(ARROW_IO_TEST_LINK_LIBS
 
 set(ARROW_IO_SRCS
   file.cc
+  interfaces.cc
   memory.cc
 )
 
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 87bae7f3928ec..93f0ad91ee86c 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -413,15 +413,7 @@ Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
   return impl_->Read(nbytes, bytes_read, out);
 }
 
-Status ReadableFile::ReadAt(
-    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-  RETURN_NOT_OK(Seek(position));
-  return impl_->Read(nbytes, bytes_read, out);
-}
-
-Status ReadableFile::ReadAt(
-    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  RETURN_NOT_OK(Seek(position));
+Status ReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   return impl_->ReadBuffer(nbytes, out);
 }
 
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 5e714ea966790..10fe16e511210 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -71,11 +71,9 @@ class ARROW_EXPORT ReadableFile : public ReadableFileInterface {
   Status Close() override;
   Status Tell(int64_t* position) override;
 
-  Status ReadAt(
-      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
-  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
-
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
   Status GetSize(int64_t* size) override;
   Status Seek(int64_t position) override;
 
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index a6b4b2f3846b1..b74f84604f18c 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -22,6 +22,8 @@
 #include <string>
 
 #include "arrow/io/hdfs.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
@@ -89,7 +91,7 @@ class HdfsAnyFileImpl {
 // Private implementation for read-only files
 class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
  public:
-  HdfsReadableFileImpl() {}
+  explicit HdfsReadableFileImpl(MemoryPool* pool) : pool_(pool) {}
 
   Status Close() {
     if (is_open_) {
@@ -108,6 +110,19 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+    auto buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(buffer->Resize(nbytes));
+
+    int64_t bytes_read = 0;
+    RETURN_NOT_OK(ReadAt(position, nbytes, &bytes_read, buffer->mutable_data()));
+
+    if (bytes_read < nbytes) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
+
+    *out = buffer;
+    return Status::OK();
+  }
+
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
     tSize ret = hdfsRead(fs_, file_, reinterpret_cast<void*>(buffer), nbytes);
     RETURN_NOT_OK(CheckReadResult(ret));
@@ -115,6 +130,19 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+    auto buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(buffer->Resize(nbytes));
+
+    int64_t bytes_read = 0;
+    RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
+
+    if (bytes_read < nbytes) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
+
+    *out = buffer;
+    return Status::OK();
+  }
+
   Status GetSize(int64_t* size) {
     hdfsFileInfo* entry = hdfsGetPathInfo(fs_, path_.c_str());
     if (entry == nullptr) { return Status::IOError("HDFS: GetPathInfo failed"); }
@@ -123,10 +151,16 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     hdfsFreeFileInfo(entry, 1);
     return Status::OK();
   }
+
+  void set_memory_pool(MemoryPool* pool) { pool_ = pool; }
+
+ private:
+  MemoryPool* pool_;
 };
 
-HdfsReadableFile::HdfsReadableFile() {
-  impl_.reset(new HdfsReadableFileImpl());
+HdfsReadableFile::HdfsReadableFile(MemoryPool* pool) {
+  if (pool == nullptr) { pool = default_memory_pool(); }
+  impl_.reset(new HdfsReadableFileImpl(pool));
 }
 
 HdfsReadableFile::~HdfsReadableFile() {
@@ -144,7 +178,7 @@ Status HdfsReadableFile::ReadAt(
 
 Status HdfsReadableFile::ReadAt(
     int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  return Status::NotImplemented("Not yet implemented");
+  return impl_->ReadAt(position, nbytes, out);
 }
 
 bool HdfsReadableFile::supports_zero_copy() const {
@@ -155,6 +189,10 @@ Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buff
   return impl_->Read(nbytes, bytes_read, buffer);
 }
 
+Status HdfsReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* buffer) {
+  return impl_->Read(nbytes, buffer);
+}
+
 Status HdfsReadableFile::GetSize(int64_t* size) {
   return impl_->GetSize(size);
 }
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 39720cc17e422..4a4e3ec5f5134 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -164,6 +164,12 @@ class ARROW_EXPORT HdfsReadableFile : public ReadableFileInterface {
 
   Status GetSize(int64_t* size) override;
 
+  // NOTE: If you wish to read a particular range of a file in a multithreaded
+  // context, you may prefer to use ReadAt to avoid locking issues
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
   Status ReadAt(
       int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
 
@@ -174,17 +180,16 @@ class ARROW_EXPORT HdfsReadableFile : public ReadableFileInterface {
   Status Seek(int64_t position) override;
   Status Tell(int64_t* position) override;
 
-  // NOTE: If you wish to read a particular range of a file in a multithreaded
-  // context, you may prefer to use ReadAt to avoid locking issues
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  void set_memory_pool(MemoryPool* pool);
 
  private:
+  explicit HdfsReadableFile(MemoryPool* pool = nullptr);
+
   class ARROW_NO_EXPORT HdfsReadableFileImpl;
   std::unique_ptr<HdfsReadableFileImpl> impl_;
 
   friend class HdfsClient::HdfsClientImpl;
 
-  HdfsReadableFile();
   DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
 };
 
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
new file mode 100644
index 0000000000000..44986cee1afc9
--- /dev/null
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/io/interfaces.h"
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/util/buffer.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace io {
+
+FileInterface::~FileInterface() {}
+
+ReadableFileInterface::ReadableFileInterface() {
+  set_mode(FileMode::READ);
+}
+
+Status ReadableFileInterface::ReadAt(
+    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, bytes_read, out);
+}
+
+Status ReadableFileInterface::ReadAt(
+    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, out);
+}
+
+}  // namespace io
+}  // namespace arrow
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index fa34b43b2c920..db0c059c6e286 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -22,10 +22,12 @@
 #include <memory>
 
 #include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class Buffer;
+class MemoryPool;
 class Status;
 
 namespace io {
@@ -43,9 +45,9 @@ class FileSystemClient {
   virtual ~FileSystemClient() {}
 };
 
-class FileInterface {
+class ARROW_EXPORT FileInterface {
  public:
-  virtual ~FileInterface() {}
+  virtual ~FileInterface() = 0;
   virtual Status Close() = 0;
   virtual Status Tell(int64_t* position) = 0;
 
@@ -54,7 +56,6 @@ class FileInterface {
  protected:
   FileInterface() {}
   FileMode::type mode_;
-
   void set_mode(FileMode::type mode) { mode_ = mode; }
 
  private:
@@ -74,6 +75,9 @@ class Writeable {
 class Readable {
  public:
   virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0;
+
+  // Does not copy if not necessary
+  virtual Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
 };
 
 class OutputStream : public FileInterface, public Writeable {
@@ -86,21 +90,21 @@ class InputStream : public FileInterface, public Readable {
   InputStream() {}
 };
 
-class ReadableFileInterface : public InputStream, public Seekable {
+class ARROW_EXPORT ReadableFileInterface : public InputStream, public Seekable {
  public:
-  virtual Status ReadAt(
-      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0;
-
   virtual Status GetSize(int64_t* size) = 0;
 
-  // Does not copy if not necessary
+  virtual bool supports_zero_copy() const = 0;
+
+  // Read at position, provide default implementations using Read(...), but can
+  // be overridden
   virtual Status ReadAt(
-      int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
+      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out);
 
-  virtual bool supports_zero_copy() const = 0;
+  virtual Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out);
 
  protected:
-  ReadableFileInterface() { set_mode(FileMode::READ); }
+  ReadableFileInterface();
 };
 
 class WriteableFileInterface : public OutputStream, public Seekable {
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index c168c91c5f87c..7d6e02e25b43c 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -123,6 +123,8 @@ MemoryMappedFile::MemoryMappedFile(FileMode::type mode) {
   ReadableFileInterface::set_mode(mode);
 }
 
+MemoryMappedFile::~MemoryMappedFile() {}
+
 Status MemoryMappedFile::Open(const std::string& path, FileMode::type mode,
     std::shared_ptr<MemoryMappedFile>* out) {
   std::shared_ptr<MemoryMappedFile> result(new MemoryMappedFile(mode));
@@ -161,16 +163,8 @@ Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out)
   return Status::OK();
 }
 
-Status MemoryMappedFile::ReadAt(
-    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-  RETURN_NOT_OK(impl_->Seek(position));
-  return Read(nbytes, bytes_read, out);
-}
-
-Status MemoryMappedFile::ReadAt(
-    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  nbytes = std::min(nbytes, impl_->size() - position);
-  RETURN_NOT_OK(impl_->Seek(position));
+Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  nbytes = std::min(nbytes, impl_->size() - impl_->position());
   *out = std::make_shared<Buffer>(impl_->head(), nbytes);
   impl_->advance(nbytes);
   return Status::OK();
@@ -246,6 +240,11 @@ Status BufferOutputStream::Reserve(int64_t nbytes) {
 // ----------------------------------------------------------------------
 // In-memory buffer reader
 
+BufferReader::BufferReader(const uint8_t* buffer, int buffer_size)
+    : buffer_(buffer), buffer_size_(buffer_size), position_(0) {}
+
+BufferReader::~BufferReader() {}
+
 Status BufferReader::Close() {
   // no-op
   return Status::OK();
@@ -256,20 +255,6 @@ Status BufferReader::Tell(int64_t* position) {
   return Status::OK();
 }
 
-Status BufferReader::ReadAt(
-    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
-  RETURN_NOT_OK(Seek(position));
-  return Read(nbytes, bytes_read, buffer);
-}
-
-Status BufferReader::ReadAt(
-    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  int64_t size = std::min(nbytes, buffer_size_ - position_);
-  *out = std::make_shared<Buffer>(buffer_ + position, size);
-  position_ += nbytes;
-  return Status::OK();
-}
-
 bool BufferReader::supports_zero_copy() const {
   return true;
 }
@@ -281,6 +266,13 @@ Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer)
   return Status::OK();
 }
 
+Status BufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  int64_t size = std::min(nbytes, buffer_size_ - position_);
+  *out = std::make_shared<Buffer>(buffer_ + position_, size);
+  position_ += nbytes;
+  return Status::OK();
+}
+
 Status BufferReader::GetSize(int64_t* size) {
   *size = buffer_size_;
   return Status::OK();
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 6989d732ca752..df2fe8d6efbfc 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -61,6 +61,8 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
 // A memory source that uses memory-mapped files for memory interactions
 class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
  public:
+  ~MemoryMappedFile();
+
   static Status Open(const std::string& path, FileMode::type mode,
       std::shared_ptr<MemoryMappedFile>* out);
 
@@ -73,11 +75,8 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   // Required by ReadableFileInterface, copies memory into out
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
 
-  Status ReadAt(
-      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
-
-  // Read into a buffer, zero copy if possible
-  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+  // Zero copy read
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   bool supports_zero_copy() const override;
 
@@ -100,17 +99,17 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
 
 class ARROW_EXPORT BufferReader : public ReadableFileInterface {
  public:
-  BufferReader(const uint8_t* buffer, int buffer_size)
-      : buffer_(buffer), buffer_size_(buffer_size), position_(0) {}
+  BufferReader(const uint8_t* buffer, int buffer_size);
+  ~BufferReader();
 
   Status Close() override;
   Status Tell(int64_t* position) override;
 
-  Status ReadAt(
-      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
-  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
-
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+
+  // Zero copy read
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
   Status GetSize(int64_t* size) override;
   Status Seek(int64_t position) override;
 
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6357e3c1725e3..77a771ab21c06 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -432,6 +432,7 @@ set(PYARROW_SRCS
   src/pyarrow/common.cc
   src/pyarrow/config.cc
   src/pyarrow/helpers.cc
+  src/pyarrow/io.cc
   src/pyarrow/status.cc
 
   src/pyarrow/adapters/builtin.cc
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 40a09c2feaef0..7561f6d46df21 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -41,6 +41,5 @@
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
-from pyarrow.array import RowBatch, from_pandas_dataframe
-
-from pyarrow.table import Column, Table
+from pyarrow.array import RowBatch
+from pyarrow.table import Column, Table, from_pandas_dataframe
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 5229b429f58b4..cdbe73ad21f7d 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -35,7 +35,6 @@ from pyarrow.scalar import NA
 from pyarrow.schema cimport Schema
 import pyarrow.schema as schema
 
-from pyarrow.table cimport Table
 
 def total_allocated_bytes():
     cdef MemoryPool* pool = pyarrow.GetMemoryPool()
@@ -254,35 +253,6 @@ def from_pandas_series(object series, object mask=None, timestamps_to_ms=False):
     return box_arrow_array(out)
 
 
-def from_pandas_dataframe(object df, name=None, timestamps_to_ms=False):
-    """
-    Convert pandas.DataFrame to an Arrow Table
-
-    Parameters
-    ----------
-    df: pandas.DataFrame
-
-    name: str
-
-    timestamps_to_ms: bool
-        Convert datetime columns to ms resolution. This is needed for
-        compability with other functionality like Parquet I/O which
-        only supports milliseconds.
-    """
-    cdef:
-        list names = []
-        list arrays = []
-
-    for name in df.columns:
-        col = df[name]
-        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms)
-
-        names.append(name)
-        arrays.append(arr)
-
-    return Table.from_arrays(names, arrays, name=name)
-
-
 cdef object series_as_ndarray(object obj):
     import pandas as pd
 
@@ -324,4 +294,3 @@ cdef class RowBatch:
 
     def __getitem__(self, i):
         return self.arrays[i]
-
diff --git a/python/pyarrow/error.pxd b/python/pyarrow/error.pxd
index 1fb6fad396a8b..891d1ac1c7ea0 100644
--- a/python/pyarrow/error.pxd
+++ b/python/pyarrow/error.pxd
@@ -16,7 +16,7 @@
 # under the License.
 
 from pyarrow.includes.libarrow cimport CStatus
-from pyarrow.includes.pyarrow cimport *
+from pyarrow.includes.pyarrow cimport PyStatus
 
 cdef int check_cstatus(const CStatus& status) nogil except -1
-cdef int check_status(const Status& status) nogil except -1
+cdef int check_status(const PyStatus& status) nogil except -1
diff --git a/python/pyarrow/error.pyx b/python/pyarrow/error.pyx
index 244019321a7fd..a2c53fed8c6a0 100644
--- a/python/pyarrow/error.pyx
+++ b/python/pyarrow/error.pyx
@@ -30,7 +30,7 @@ cdef int check_cstatus(const CStatus& status) nogil except -1:
     with gil:
         raise ArrowException(frombytes(c_message))
 
-cdef int check_status(const Status& status) nogil except -1:
+cdef int check_status(const PyStatus& status) nogil except -1:
     if status.ok():
         return 0
 
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index f338a436814de..56d8d4cf61494 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -18,6 +18,7 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport MemoryPool
 
 cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
     enum FileMode" arrow::io::FileMode::type":
@@ -35,6 +36,7 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
         FileMode mode()
 
     cdef cppclass Readable:
+        CStatus ReadB" Read"(int64_t nbytes, shared_ptr[Buffer]* out)
         CStatus Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out)
 
     cdef cppclass Seekable:
@@ -66,6 +68,24 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
         pass
 
 
+cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
+    cdef cppclass FileOutputStream(OutputStream):
+        @staticmethod
+        CStatus Open(const c_string& path, shared_ptr[FileOutputStream]* file)
+
+        int file_descriptor()
+
+    cdef cppclass ReadableFile(ReadableFileInterface):
+        @staticmethod
+        CStatus Open(const c_string& path, shared_ptr[ReadableFile]* file)
+
+        @staticmethod
+        CStatus Open(const c_string& path, MemoryPool* memory_pool,
+                     shared_ptr[ReadableFile]* file)
+
+        int file_descriptor()
+
+
 cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
     CStatus ConnectLibHdfs()
 
@@ -120,3 +140,12 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
                               int32_t buffer_size, int16_t replication,
                               int64_t default_block_size,
                               shared_ptr[HdfsOutputStream]* handle)
+
+
+cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
+    cdef cppclass BufferReader(ReadableFileInterface):
+        BufferReader(const uint8_t* data, int64_t nbytes)
+
+    cdef cppclass BufferOutputStream(OutputStream):
+        # TODO(wesm)
+        pass
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 92c814706fdd6..4c971665ff6aa 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -18,15 +18,18 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CColumn, CDataType,
+from pyarrow.includes.libarrow cimport (CArray, CColumn, CDataType, CStatus,
                                         Type, MemoryPool)
 
+cimport pyarrow.includes.libarrow_io as arrow_io
+
+
 cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     # We can later add more of the common status factory methods as needed
-    cdef Status Status_OK "Status::OK"()
+    cdef PyStatus PyStatus_OK "Status::OK"()
 
-    cdef cppclass Status:
-        Status()
+    cdef cppclass PyStatus "pyarrow::Status":
+        PyStatus()
 
         c_string ToString()
 
@@ -40,12 +43,25 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
         c_bool IsArrowError()
 
     shared_ptr[CDataType] GetPrimitiveType(Type type)
-    Status ConvertPySequence(object obj, shared_ptr[CArray]* out)
+    PyStatus ConvertPySequence(object obj, shared_ptr[CArray]* out)
 
-    Status PandasToArrow(MemoryPool* pool, object ao, shared_ptr[CArray]* out)
-    Status PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
-                               shared_ptr[CArray]* out)
+    PyStatus PandasToArrow(MemoryPool* pool, object ao,
+                           shared_ptr[CArray]* out)
+    PyStatus PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
+                                 shared_ptr[CArray]* out)
 
-    Status ArrowToPandas(const shared_ptr[CColumn]& arr, object py_ref, PyObject** out)
+    PyStatus ArrowToPandas(const shared_ptr[CColumn]& arr, object py_ref,
+                           PyObject** out)
 
     MemoryPool* GetMemoryPool()
+
+
+cdef extern from "pyarrow/io.h" namespace "pyarrow" nogil:
+    cdef cppclass PyReadableFile(arrow_io.ReadableFileInterface):
+        PyReadableFile(object fo)
+
+    cdef cppclass PyOutputStream(arrow_io.OutputStream):
+        PyOutputStream(object fo)
+
+    cdef cppclass PyBytesReader(arrow_io.BufferReader):
+        PyBytesReader(object fo)
diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
index f55fc0ab53ac1..1dbb3fd76bbfd 100644
--- a/python/pyarrow/io.pxd
+++ b/python/pyarrow/io.pxd
@@ -23,11 +23,16 @@ from pyarrow.includes.libarrow_io cimport (ReadableFileInterface,
                                            OutputStream)
 
 
-cdef class NativeFileInterface:
+cdef class NativeFile:
+    cdef:
+        shared_ptr[ReadableFileInterface] rd_file
+        shared_ptr[OutputStream] wr_file
+        bint is_readonly
+        bint is_open
 
     # By implementing these "virtual" functions (all functions in Cython
-    # extension classes are technically virtual in the C++ sense)m we can
-    # expose the arrow::io abstract file interfaces to other components
-    # throughout the suite of Arrow C++ libraries
+    # extension classes are technically virtual in the C++ sense) we can expose
+    # the arrow::io abstract file interfaces to other components throughout the
+    # suite of Arrow C++ libraries
     cdef read_handle(self, shared_ptr[ReadableFileInterface]* file)
     cdef write_handle(self, shared_ptr[OutputStream]* file)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index f2eee260c331b..e6e2b625e87ca 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -242,6 +242,9 @@ cdef class HdfsClient:
         cdef int16_t c_replication = replication or 0
         cdef int64_t c_default_block_size = default_block_size or 0
 
+        cdef shared_ptr[HdfsOutputStream] wr_handle
+        cdef shared_ptr[HdfsReadableFile] rd_handle
+
         if mode in ('wb', 'ab'):
             if mode == 'ab':
                 append = True
@@ -251,13 +254,17 @@ cdef class HdfsClient:
                     self.client.get()
                     .OpenWriteable(c_path, append, c_buffer_size,
                                    c_replication, c_default_block_size,
-                                   &out.wr_file))
+                                   &wr_handle))
+
+            out.wr_file = <shared_ptr[OutputStream]> wr_handle
 
             out.is_readonly = False
         else:
             with nogil:
                 check_cstatus(self.client.get()
-                              .OpenReadable(c_path, &out.rd_file))
+                              .OpenReadable(c_path, &rd_handle))
+
+            out.rd_file = <shared_ptr[ReadableFileInterface]> rd_handle
             out.is_readonly = True
 
         if c_buffer_size == 0:
@@ -314,25 +321,8 @@ cdef class HdfsClient:
         f = self.open(path, 'rb', buffer_size=buffer_size)
         f.download(stream)
 
-cdef class NativeFileInterface:
-
-    cdef read_handle(self, shared_ptr[ReadableFileInterface]* file):
-        raise NotImplementedError
-
-    cdef write_handle(self, shared_ptr[OutputStream]* file):
-        raise NotImplementedError
-
-cdef class HdfsFile(NativeFileInterface):
-    cdef:
-        shared_ptr[HdfsReadableFile] rd_file
-        shared_ptr[HdfsOutputStream] wr_file
-        bint is_readonly
-        bint is_open
-        object parent
 
-    cdef readonly:
-        int32_t buffer_size
-        object mode
+cdef class NativeFile:
 
     def __cinit__(self):
         self.is_open = False
@@ -356,14 +346,6 @@ cdef class HdfsFile(NativeFileInterface):
                     check_cstatus(self.wr_file.get().Close())
         self.is_open = False
 
-    cdef _assert_readable(self):
-        if not self.is_readonly:
-            raise IOError("only valid on readonly files")
-
-    cdef _assert_writeable(self):
-        if self.is_readonly:
-            raise IOError("only valid on writeonly files")
-
     cdef read_handle(self, shared_ptr[ReadableFileInterface]* file):
         self._assert_readable()
         file[0] = <shared_ptr[ReadableFileInterface]> self.rd_file
@@ -372,6 +354,14 @@ cdef class HdfsFile(NativeFileInterface):
         self._assert_writeable()
         file[0] = <shared_ptr[OutputStream]> self.wr_file
 
+    def _assert_readable(self):
+        if not self.is_readonly:
+            raise IOError("only valid on readonly files")
+
+    def _assert_writeable(self):
+        if self.is_readonly:
+            raise IOError("only valid on writeonly files")
+
     def size(self):
         cdef int64_t size
         self._assert_readable()
@@ -393,6 +383,83 @@ cdef class HdfsFile(NativeFileInterface):
         with nogil:
             check_cstatus(self.rd_file.get().Seek(position))
 
+    def write(self, data):
+        """
+        Write bytes-like (unicode, encoded to UTF-8) to file
+        """
+        self._assert_writeable()
+
+        data = tobytes(data)
+
+        cdef const uint8_t* buf = <const uint8_t*> cp.PyBytes_AS_STRING(data)
+        cdef int64_t bufsize = len(data)
+        with nogil:
+            check_cstatus(self.wr_file.get().Write(buf, bufsize))
+
+    def read(self, int nbytes):
+        cdef:
+            int64_t bytes_read = 0
+            uint8_t* buf
+            shared_ptr[Buffer] out
+
+        self._assert_readable()
+
+        with nogil:
+            check_cstatus(self.rd_file.get()
+                          .ReadB(nbytes, &out))
+
+        result = cp.PyBytes_FromStringAndSize(
+            <const char*>out.get().data(), out.get().size())
+
+        return result
+
+
+# ----------------------------------------------------------------------
+# Python file-like objects
+
+cdef class PythonFileInterface(NativeFile):
+    cdef:
+        object handle
+
+    def __cinit__(self, handle, mode='w'):
+        self.handle = handle
+
+        if mode.startswith('w'):
+            self.wr_file.reset(new pyarrow.PyOutputStream(handle))
+            self.is_readonly = 0
+        elif mode.startswith('r'):
+            self.rd_file.reset(new pyarrow.PyReadableFile(handle))
+            self.is_readonly = 1
+        else:
+            raise ValueError('Invalid file mode: {0}'.format(mode))
+
+        self.is_open = True
+
+
+cdef class BytesReader(NativeFile):
+    cdef:
+        object obj
+
+    def __cinit__(self, obj):
+        if not isinstance(obj, bytes):
+            raise ValueError('Must pass bytes object')
+
+        self.obj = obj
+        self.is_readonly = 1
+        self.is_open = True
+
+        self.rd_file.reset(new pyarrow.PyBytesReader(obj))
+
+# ----------------------------------------------------------------------
+# Specialization for HDFS
+
+
+cdef class HdfsFile(NativeFile):
+    cdef readonly:
+        int32_t buffer_size
+        object mode
+        object parent
+
     def read(self, int nbytes):
         """
         Read indicated number of bytes from the file, up to EOF
@@ -504,16 +571,3 @@ cdef class HdfsFile(NativeFileInterface):
         writer_thread.join()
         if exc_info is not None:
             raise exc_info[0], exc_info[1], exc_info[2]
-
-    def write(self, data):
-        """
-        Write bytes-like (unicode, encoded to UTF-8) to file
-        """
-        self._assert_writeable()
-
-        data = tobytes(data)
-
-        cdef const uint8_t* buf = <const uint8_t*> cp.PyBytes_AS_STRING(data)
-        cdef int64_t bufsize = len(data)
-        with nogil:
-            check_cstatus(self.wr_file.get().Write(buf, bufsize))
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 099e148abc16f..ca0176a7c0403 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -27,10 +27,10 @@ cimport pyarrow.includes.pyarrow as pyarrow
 from pyarrow.compat import tobytes
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_cstatus
-from pyarrow.io import NativeFileInterface
+from pyarrow.io import NativeFile
 from pyarrow.table cimport Table
 
-from pyarrow.io cimport NativeFileInterface
+from pyarrow.io cimport NativeFile
 
 import six
 
@@ -54,7 +54,7 @@ cdef class ParquetReader:
             new FileReader(default_memory_pool(),
                            ParquetFileReader.OpenFile(path)))
 
-    cdef open_native_file(self, NativeFileInterface file):
+    cdef open_native_file(self, NativeFile file):
         cdef shared_ptr[ReadableFileInterface] cpp_handle
         file.read_handle(&cpp_handle)
 
@@ -84,7 +84,7 @@ def read_table(source, columns=None):
 
     if isinstance(source, six.string_types):
         reader.open_local_file(source)
-    elif isinstance(source, NativeFileInterface):
+    elif isinstance(source, NativeFile):
         reader.open_native_file(source)
 
     return reader.read_all()
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index f02d36f520be6..ade82aa676164 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -25,10 +25,12 @@ cimport pyarrow.includes.pyarrow as pyarrow
 import pyarrow.config
 
 from pyarrow.array cimport Array, box_arrow_array
-from pyarrow.compat import frombytes, tobytes
 from pyarrow.error cimport check_status
 from pyarrow.schema cimport box_data_type, box_schema
 
+from pyarrow.compat import frombytes, tobytes
+
+
 cdef class ChunkedArray:
     '''
     Do not call this class's constructor directly.
@@ -161,7 +163,7 @@ cdef class Table:
 
     @staticmethod
     def from_pandas(df, name=None):
-        pass
+        return from_pandas_dataframe(df, name=name)
 
     @staticmethod
     def from_arrays(names, arrays, name=None):
@@ -264,3 +266,34 @@ cdef class Table:
         def __get__(self):
             return (self.num_rows, self.num_columns)
 
+
+
+def from_pandas_dataframe(object df, name=None, timestamps_to_ms=False):
+    """
+    Convert pandas.DataFrame to an Arrow Table
+
+    Parameters
+    ----------
+    df: pandas.DataFrame
+
+    name: str
+
+    timestamps_to_ms: bool
+        Convert datetime columns to ms resolution. This is needed for
+        compability with other functionality like Parquet I/O which
+        only supports milliseconds.
+    """
+    from pyarrow.array import from_pandas_series
+
+    cdef:
+        list names = []
+        list arrays = []
+
+    for name in df.columns:
+        col = df[name]
+        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms)
+
+        names.append(name)
+        arrays.append(arr)
+
+    return Table.from_arrays(names, arrays, name=name)
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
new file mode 100644
index 0000000000000..ed8d41994cdd0
--- /dev/null
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from io import BytesIO
+from os.path import join as pjoin
+import os
+import random
+
+import pytest
+
+import pyarrow.io as io
+
+# ----------------------------------------------------------------------
+# HDFS tests
+
+
+def hdfs_test_client():
+    host = os.environ.get('ARROW_HDFS_TEST_HOST', 'localhost')
+    user = os.environ['ARROW_HDFS_TEST_USER']
+    try:
+        port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 20500))
+    except ValueError:
+        raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
+                         'an integer')
+
+    return io.HdfsClient.connect(host, port, user)
+
+
+libhdfs = pytest.mark.skipif(not io.have_libhdfs(),
+                             reason='No libhdfs available on system')
+
+
+HDFS_TMP_PATH = '/tmp/pyarrow-test-{0}'.format(random.randint(0, 1000))
+
+
+@pytest.fixture(scope='session')
+def hdfs(request):
+    fixture = hdfs_test_client()
+
+    def teardown():
+        fixture.delete(HDFS_TMP_PATH, recursive=True)
+        fixture.close()
+    request.addfinalizer(teardown)
+    return fixture
+
+
+@libhdfs
+def test_hdfs_close():
+    client = hdfs_test_client()
+    assert client.is_open
+    client.close()
+    assert not client.is_open
+
+    with pytest.raises(Exception):
+        client.ls('/')
+
+
+@libhdfs
+def test_hdfs_mkdir(hdfs):
+    path = pjoin(HDFS_TMP_PATH, 'test-dir/test-dir')
+    parent_path = pjoin(HDFS_TMP_PATH, 'test-dir')
+
+    hdfs.mkdir(path)
+    assert hdfs.exists(path)
+
+    hdfs.delete(parent_path, recursive=True)
+    assert not hdfs.exists(path)
+
+
+@libhdfs
+def test_hdfs_ls(hdfs):
+    base_path = pjoin(HDFS_TMP_PATH, 'ls-test')
+    hdfs.mkdir(base_path)
+
+    dir_path = pjoin(base_path, 'a-dir')
+    f1_path = pjoin(base_path, 'a-file-1')
+
+    hdfs.mkdir(dir_path)
+
+    f = hdfs.open(f1_path, 'wb')
+    f.write('a' * 10)
+
+    contents = sorted(hdfs.ls(base_path, False))
+    assert contents == [dir_path, f1_path]
+
+
+@libhdfs
+def test_hdfs_download_upload(hdfs):
+    base_path = pjoin(HDFS_TMP_PATH, 'upload-test')
+
+    data = b'foobarbaz'
+    buf = BytesIO(data)
+    buf.seek(0)
+
+    hdfs.upload(base_path, buf)
+
+    out_buf = BytesIO()
+    hdfs.download(base_path, out_buf)
+    out_buf.seek(0)
+    assert out_buf.getvalue() == data
+
+
+@libhdfs
+def test_hdfs_file_context_manager(hdfs):
+    path = pjoin(HDFS_TMP_PATH, 'ctx-manager')
+
+    data = b'foo'
+    with hdfs.open(path, 'wb') as f:
+        f.write(data)
+
+    with hdfs.open(path, 'rb') as f:
+        assert f.size() == 3
+        result = f.read(10)
+        assert result == data
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index eb92e8ea93a1a..9a41ebe3e8c74 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -16,112 +16,85 @@
 # under the License.
 
 from io import BytesIO
-from os.path import join as pjoin
-import os
-import random
-
 import pytest
 
+from pyarrow.compat import u
 import pyarrow.io as io
 
-#----------------------------------------------------------------------
-# HDFS tests
+# ----------------------------------------------------------------------
+# Python file-like objects
 
 
-def hdfs_test_client():
-    host = os.environ.get('ARROW_HDFS_TEST_HOST', 'localhost')
-    user = os.environ['ARROW_HDFS_TEST_USER']
-    try:
-        port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 20500))
-    except ValueError:
-        raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
-                         'an integer')
+def test_python_file_write():
+    buf = BytesIO()
 
-    return io.HdfsClient.connect(host, port, user)
+    f = io.PythonFileInterface(buf)
 
+    assert f.tell() == 0
 
-libhdfs = pytest.mark.skipif(not io.have_libhdfs(),
-                             reason='No libhdfs available on system')
+    s1 = b'enga\xc3\xb1ado'
+    s2 = b'foobar'
 
+    f.write(s1.decode('utf8'))
+    assert f.tell() == len(s1)
 
-HDFS_TMP_PATH = '/tmp/pyarrow-test-{0}'.format(random.randint(0, 1000))
+    f.write(s2)
 
+    expected = s1 + s2
 
-@pytest.fixture(scope='session')
-def hdfs(request):
-    fixture = hdfs_test_client()
-    def teardown():
-        fixture.delete(HDFS_TMP_PATH, recursive=True)
-        fixture.close()
-    request.addfinalizer(teardown)
-    return fixture
+    result = buf.getvalue()
+    assert result == expected
 
+    f.close()
 
-@libhdfs
-def test_hdfs_close():
-    client = hdfs_test_client()
-    assert client.is_open
-    client.close()
-    assert not client.is_open
 
-    with pytest.raises(Exception):
-        client.ls('/')
+def test_python_file_read():
+    data = b'some sample data'
 
+    buf = BytesIO(data)
+    f = io.PythonFileInterface(buf, mode='r')
 
-@libhdfs
-def test_hdfs_mkdir(hdfs):
-    path = pjoin(HDFS_TMP_PATH, 'test-dir/test-dir')
-    parent_path = pjoin(HDFS_TMP_PATH, 'test-dir')
+    assert f.size() == len(data)
 
-    hdfs.mkdir(path)
-    assert hdfs.exists(path)
+    assert f.tell() == 0
 
-    hdfs.delete(parent_path, recursive=True)
-    assert not hdfs.exists(path)
+    assert f.read(4) == b'some'
+    assert f.tell() == 4
 
+    f.seek(0)
+    assert f.tell() == 0
 
-@libhdfs
-def test_hdfs_ls(hdfs):
-    base_path = pjoin(HDFS_TMP_PATH, 'ls-test')
-    hdfs.mkdir(base_path)
+    f.seek(5)
+    assert f.tell() == 5
 
-    dir_path = pjoin(base_path, 'a-dir')
-    f1_path = pjoin(base_path, 'a-file-1')
+    assert f.read(50) == b'sample data'
 
-    hdfs.mkdir(dir_path)
+    f.close()
 
-    f = hdfs.open(f1_path, 'wb')
-    f.write('a' * 10)
 
-    contents = sorted(hdfs.ls(base_path, False))
-    assert contents == [dir_path, f1_path]
+def test_bytes_reader():
+    # Like a BytesIO, but zero-copy underneath for C++ consumers
+    data = b'some sample data'
+    f = io.BytesReader(data)
 
+    assert f.tell() == 0
 
-@libhdfs
-def test_hdfs_download_upload(hdfs):
-    base_path = pjoin(HDFS_TMP_PATH, 'upload-test')
+    assert f.size() == len(data)
 
-    data = b'foobarbaz'
-    buf = BytesIO(data)
-    buf.seek(0)
+    assert f.read(4) == b'some'
+    assert f.tell() == 4
 
-    hdfs.upload(base_path, buf)
+    f.seek(0)
+    assert f.tell() == 0
 
-    out_buf = BytesIO()
-    hdfs.download(base_path, out_buf)
-    out_buf.seek(0)
-    assert out_buf.getvalue() == data
+    f.seek(5)
+    assert f.tell() == 5
 
+    assert f.read(50) == b'sample data'
 
-@libhdfs
-def test_hdfs_file_context_manager(hdfs):
-    path = pjoin(HDFS_TMP_PATH, 'ctx-manager')
+    f.close()
 
-    data = b'foo'
-    with hdfs.open(path, 'wb') as f:
-        f.write(data)
 
-    with hdfs.open(path, 'rb') as f:
-        assert f.size() == 3
-        result = f.read(10)
-        assert result == data
+def test_bytes_reader_non_bytes():
+    with pytest.raises(ValueError):
+        io.BytesReader(u('some sample data'))
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index a4e7fb6f3bb70..d224074d652cb 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -618,7 +618,7 @@ class ArrowDeserializer {
   Status OutputFromData(int type, void* data) {
     // Zero-Copy. We can pass the data pointer directly to NumPy.
     Py_INCREF(py_ref_);
-    OwnedRef py_ref(py_ref);
+    OwnedRef py_ref(py_ref_);
     npy_intp dims[1] = {col_->length()};
     out_ = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNewFromData(1, dims,
                 type, data));
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index a2748f99b6733..82b14fdf40173 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -68,4 +68,19 @@ arrow::MemoryPool* GetMemoryPool() {
   return &memory_pool;
 }
 
+// ----------------------------------------------------------------------
+// PyBytesBuffer
+
+PyBytesBuffer::PyBytesBuffer(PyObject* obj)
+    : Buffer(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)),
+        PyBytes_GET_SIZE(obj)),
+      obj_(obj) {
+  Py_INCREF(obj_);
+}
+
+PyBytesBuffer::~PyBytesBuffer() {
+  PyGILGuard lock;
+  Py_DECREF(obj_);
+}
+
 } // namespace pyarrow
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index fb0ba3e482296..bc599f84fab50 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -19,9 +19,8 @@
 #define PYARROW_COMMON_H
 
 #include "pyarrow/config.h"
-
 #include "arrow/util/buffer.h"
-
+#include "arrow/util/macros.h"
 #include "pyarrow/visibility.h"
 
 namespace arrow { class MemoryPool; }
@@ -83,6 +82,20 @@ struct PyObjectStringify {
   }
 };
 
+class PyGILGuard {
+ public:
+  PyGILGuard() {
+    state_ = PyGILState_Ensure();
+  }
+
+  ~PyGILGuard() {
+    PyGILState_Release(state_);
+  }
+ private:
+  PyGILState_STATE state_;
+  DISALLOW_COPY_AND_ASSIGN(PyGILGuard);
+};
+
 // TODO(wesm): We can just let errors pass through. To be explored later
 #define RETURN_IF_PYERROR()                         \
   if (PyErr_Occurred()) {                           \
@@ -100,8 +113,8 @@ PYARROW_EXPORT arrow::MemoryPool* GetMemoryPool();
 
 class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
  public:
-  NumPyBuffer(PyArrayObject* arr) :
-      Buffer(nullptr, 0) {
+  NumPyBuffer(PyArrayObject* arr)
+    : Buffer(nullptr, 0) {
     arr_ = arr;
     Py_INCREF(arr);
 
@@ -117,6 +130,15 @@ class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
   PyArrayObject* arr_;
 };
 
+class PYARROW_EXPORT PyBytesBuffer : public arrow::Buffer {
+ public:
+  PyBytesBuffer(PyObject* obj);
+  ~PyBytesBuffer();
+
+ private:
+  PyObject* obj_;
+};
+
 } // namespace pyarrow
 
 #endif // PYARROW_COMMON_H
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
new file mode 100644
index 0000000000000..35054e9025ad4
--- /dev/null
+++ b/python/src/pyarrow/io.cc
@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "pyarrow/io.h"
+
+#include <cstdint>
+#include <cstdlib>
+
+#include <arrow/io/memory.h>
+#include <arrow/util/memory-pool.h>
+#include <arrow/util/status.h>
+
+#include "pyarrow/common.h"
+#include "pyarrow/status.h"
+
+namespace pyarrow {
+
+// ----------------------------------------------------------------------
+// Python file
+
+PythonFile::PythonFile(PyObject* file)
+    : file_(file) {
+  Py_INCREF(file_);
+}
+
+PythonFile::~PythonFile() {
+  Py_DECREF(file_);
+}
+
+static arrow::Status CheckPyError() {
+  if (PyErr_Occurred()) {
+    PyObject *exc_type, *exc_value, *traceback;
+    PyErr_Fetch(&exc_type, &exc_value, &traceback);
+    PyObjectStringify stringified(exc_value);
+    std::string message(stringified.bytes);
+    Py_DECREF(exc_type);
+    Py_DECREF(exc_value);
+    Py_DECREF(traceback);
+    PyErr_Clear();
+    return arrow::Status::IOError(message);
+  }
+  return arrow::Status::OK();
+}
+
+arrow::Status PythonFile::Close() {
+  // whence: 0 for relative to start of file, 2 for end of file
+  PyObject* result = PyObject_CallMethod(file_, "close", "()");
+  Py_XDECREF(result);
+  ARROW_RETURN_NOT_OK(CheckPyError());
+  return arrow::Status::OK();
+}
+
+arrow::Status PythonFile::Seek(int64_t position, int whence) {
+  // whence: 0 for relative to start of file, 2 for end of file
+  PyObject* result = PyObject_CallMethod(file_, "seek", "(ii)", position, whence);
+  Py_XDECREF(result);
+  ARROW_RETURN_NOT_OK(CheckPyError());
+  return arrow::Status::OK();
+}
+
+arrow::Status PythonFile::Read(int64_t nbytes, PyObject** out) {
+  PyObject* result = PyObject_CallMethod(file_, "read", "(i)", nbytes);
+  ARROW_RETURN_NOT_OK(CheckPyError());
+  *out = result;
+  return arrow::Status::OK();
+}
+
+arrow::Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
+  PyObject* py_data = PyBytes_FromStringAndSize(
+      reinterpret_cast<const char*>(data), nbytes);
+  ARROW_RETURN_NOT_OK(CheckPyError());
+
+  PyObject* result = PyObject_CallMethod(file_, "write", "(O)", py_data);
+  Py_DECREF(py_data);
+  Py_XDECREF(result);
+  ARROW_RETURN_NOT_OK(CheckPyError());
+  return arrow::Status::OK();
+}
+
+arrow::Status PythonFile::Tell(int64_t* position) {
+  PyObject* result = PyObject_CallMethod(file_, "tell", "()");
+  ARROW_RETURN_NOT_OK(CheckPyError());
+
+  *position = PyLong_AsLongLong(result);
+  Py_DECREF(result);
+
+  // PyLong_AsLongLong can raise OverflowError
+  ARROW_RETURN_NOT_OK(CheckPyError());
+
+ return arrow::Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Seekable input stream
+
+PyReadableFile::PyReadableFile(PyObject* file) {
+  file_.reset(new PythonFile(file));
+}
+
+PyReadableFile::~PyReadableFile() {}
+
+arrow::Status PyReadableFile::Close() {
+  PyGILGuard lock;
+  return file_->Close();
+}
+
+arrow::Status PyReadableFile::Seek(int64_t position) {
+  PyGILGuard lock;
+  return file_->Seek(position, 0);
+}
+
+arrow::Status PyReadableFile::Tell(int64_t* position) {
+  PyGILGuard lock;
+  return file_->Tell(position);
+}
+
+arrow::Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  PyGILGuard lock;
+  PyObject* bytes_obj;
+  ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
+
+  *bytes_read = PyBytes_GET_SIZE(bytes_obj);
+  std::memcpy(out, PyBytes_AS_STRING(bytes_obj), *bytes_read);
+  Py_DECREF(bytes_obj);
+
+  return arrow::Status::OK();
+}
+
+arrow::Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) {
+  PyGILGuard lock;
+
+  PyObject* bytes_obj;
+  ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
+
+  *out = std::make_shared<PyBytesBuffer>(bytes_obj);
+  Py_DECREF(bytes_obj);
+
+  return arrow::Status::OK();
+}
+
+arrow::Status PyReadableFile::GetSize(int64_t* size) {
+  PyGILGuard lock;
+
+  int64_t current_position;;
+  ARROW_RETURN_NOT_OK(file_->Tell(&current_position));
+
+  ARROW_RETURN_NOT_OK(file_->Seek(0, 2));
+
+  int64_t file_size;
+  ARROW_RETURN_NOT_OK(file_->Tell(&file_size));
+
+  // Restore previous file position
+  ARROW_RETURN_NOT_OK(file_->Seek(current_position, 0));
+
+  *size = file_size;
+  return arrow::Status::OK();
+}
+
+bool PyReadableFile::supports_zero_copy() const {
+  return false;
+}
+
+// ----------------------------------------------------------------------
+// Output stream
+
+PyOutputStream::PyOutputStream(PyObject* file) {
+  file_.reset(new PythonFile(file));
+}
+
+PyOutputStream::~PyOutputStream() {}
+
+arrow::Status PyOutputStream::Close() {
+  PyGILGuard lock;
+  return file_->Close();
+}
+
+arrow::Status PyOutputStream::Tell(int64_t* position) {
+  PyGILGuard lock;
+  return file_->Tell(position);
+}
+
+arrow::Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+  PyGILGuard lock;
+  return file_->Write(data, nbytes);
+}
+
+// ----------------------------------------------------------------------
+// A readable file that is backed by a PyBytes
+
+PyBytesReader::PyBytesReader(PyObject* obj)
+    : arrow::io::BufferReader(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)),
+        PyBytes_GET_SIZE(obj)),
+      obj_(obj) {
+  Py_INCREF(obj_);
+}
+
+PyBytesReader::~PyBytesReader() {
+  Py_DECREF(obj_);
+}
+
+}  // namespace pyarrow
diff --git a/python/src/pyarrow/io.h b/python/src/pyarrow/io.h
new file mode 100644
index 0000000000000..e14aa8cfb27e3
--- /dev/null
+++ b/python/src/pyarrow/io.h
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_IO_H
+#define PYARROW_IO_H
+
+#include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
+
+#include "pyarrow/config.h"
+#include "pyarrow/visibility.h"
+
+namespace arrow { class MemoryPool; }
+
+namespace pyarrow {
+
+// A common interface to a Python file-like object. Must acquire GIL before
+// calling any methods
+class PythonFile {
+ public:
+  PythonFile(PyObject* file);
+  ~PythonFile();
+
+  arrow::Status Close();
+  arrow::Status Seek(int64_t position, int whence);
+  arrow::Status Read(int64_t nbytes, PyObject** out);
+  arrow::Status Tell(int64_t* position);
+  arrow::Status Write(const uint8_t* data, int64_t nbytes);
+
+ private:
+  PyObject* file_;
+};
+
+class PYARROW_EXPORT PyReadableFile : public arrow::io::ReadableFileInterface {
+ public:
+  explicit PyReadableFile(PyObject* file);
+  virtual ~PyReadableFile();
+
+  arrow::Status Close() override;
+
+  arrow::Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+  arrow::Status Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) override;
+
+  arrow::Status GetSize(int64_t* size) override;
+
+  arrow::Status Seek(int64_t position) override;
+
+  arrow::Status Tell(int64_t* position) override;
+
+  bool supports_zero_copy() const override;
+
+ private:
+  std::unique_ptr<PythonFile> file_;
+};
+
+class PYARROW_EXPORT PyOutputStream : public arrow::io::OutputStream {
+ public:
+  explicit PyOutputStream(PyObject* file);
+  virtual ~PyOutputStream();
+
+  arrow::Status Close() override;
+  arrow::Status Tell(int64_t* position) override;
+  arrow::Status Write(const uint8_t* data, int64_t nbytes) override;
+
+ private:
+  std::unique_ptr<PythonFile> file_;
+};
+
+// A zero-copy reader backed by a PyBytes object
+class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader {
+ public:
+  explicit PyBytesReader(PyObject* obj);
+  virtual ~PyBytesReader();
+
+ private:
+  PyObject* obj_;
+};
+
+// TODO(wesm): seekable output files
+
+} // namespace pyarrow
+
+#endif  // PYARROW_IO_H

From c3cfa3d3b3ce017776508f42fe9410bfb99cd94f Mon Sep 17 00:00:00 2001
From: "Christopher C. Aycock" <christopher.aycock@twosigma.com>
Date: Tue, 4 Oct 2016 14:07:59 -0700
Subject: [PATCH 0154/1644] ARROW-313: Build on any version of XCode

Author: Christopher C. Aycock <christopher.aycock@twosigma.com>

Closes #155 from chrisaycock/ARROW-313 and squashes the following commits:

e47cc01 [Christopher C. Aycock] ARROW-313: Build on any version of XCode
---
 cpp/cmake_modules/CompilerInfo.cmake    | 4 ++--
 python/cmake_modules/CompilerInfo.cmake | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index e1c821cca5d45..02f6fd46997a3 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -29,9 +29,9 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
   string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
     COMPILER_VERSION "${COMPILER_VERSION_FULL}")
 
-# clang on Mac OS X, XCode 7. No version replacement is done
+# clang on Mac OS X, XCode 7+. No version replacement is done
 # because Apple no longer advertises the upstream LLVM version.
-elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-70[0-9]\\..*")
+elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-.*")
   set(COMPILER_FAMILY "clang")
 
 # gcc
diff --git a/python/cmake_modules/CompilerInfo.cmake b/python/cmake_modules/CompilerInfo.cmake
index 55f989a1a6c9d..8e85bdea96ea5 100644
--- a/python/cmake_modules/CompilerInfo.cmake
+++ b/python/cmake_modules/CompilerInfo.cmake
@@ -32,9 +32,9 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
   string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
     COMPILER_VERSION "${COMPILER_VERSION_FULL}")
 
-# clang on Mac OS X, XCode 7. No version replacement is done
+# clang on Mac OS X, XCode 7+. No version replacement is done
 # because Apple no longer advertises the upstream LLVM version.
-elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-70[0-9]\\..*")
+elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-.*")
   set(COMPILER_FAMILY "clang")
 
 # gcc

From 7fb4d24a35269db99fa112c0512d4a32c372dd74 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 4 Oct 2016 15:11:56 -0700
Subject: [PATCH 0155/1644] ARROW-315: finalize timestamp

Author: Julien Le Dem <julien@dremio.com>

Closes #156 from julienledem/timestamp and squashes the following commits:

0ee017f [Julien Le Dem] review feedback
86cae98 [Julien Le Dem] ARROW-315: finalize timestamp
---
 format/Message.fbs                            |  5 +-
 .../src/main/codegen/data/ArrowTypes.tdd      |  2 +-
 .../templates/NullableValueVectors.java       |  2 +-
 .../org/apache/arrow/vector/types/Types.java  | 46 +++++++++++--------
 .../apache/arrow/vector/pojo/TestConvert.java |  3 +-
 5 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index 3d877a2f234af..d8fa65006c24d 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -64,8 +64,11 @@ table Date {
 table Time {
 }
 
+enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }
+
+/// time from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
 table Timestamp {
-  timezone: string;
+  unit: TimeUnit;
 }
 
 enum IntervalUnit: short { YEAR_MONTH, DAY_TIME}
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 9624fecf6aad1..11ac99af42414 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -66,7 +66,7 @@
     },
     {
       name: "Timestamp",
-      fields: [{name: "timezone", type: "String"}]
+      fields: [{name: "unit", type: short}]
     },
     {
       name: "Interval",
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 8f325afad3920..bafa31760205a 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -103,7 +103,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#elseif minor.class == "Float8">
     field = new Field(name, true, new FloatingPoint(Precision.DOUBLE), null);
   <#elseif minor.class == "TimeStamp">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(""), null);
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null);
   <#elseif minor.class == "IntervalDay">
     field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME), null);
   <#elseif minor.class == "IntervalYear">
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 2ff93d4b98d11..d9593673156bf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -19,6 +19,7 @@
 
 import org.apache.arrow.flatbuf.IntervalUnit;
 import org.apache.arrow.flatbuf.Precision;
+import org.apache.arrow.flatbuf.TimeUnit;
 import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
@@ -101,7 +102,7 @@ public class Types {
   private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
   private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
   private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
-  private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null);
+  private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null);
   private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
   private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
   private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(Precision.SINGLE), null);
@@ -143,8 +144,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new NullableMapWriter((NullableMapVector) vector);
       }
-    },   //  an empty map column.  Useful for conceptual setup.  Children listed within here
-
+    },
     TINYINT(new Int(8, true)) {
       @Override
       public Field getField() {
@@ -160,7 +160,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new TinyIntWriterImpl((NullableTinyIntVector) vector);
       }
-    },   //  single byte signed integer
+    },
     SMALLINT(new Int(16, true)) {
       @Override
       public Field getField() {
@@ -176,7 +176,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new SmallIntWriterImpl((NullableSmallIntVector) vector);
       }
-    },   //  two byte signed integer
+    },
     INT(new Int(32, true)) {
       @Override
       public Field getField() {
@@ -192,7 +192,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new IntWriterImpl((NullableIntVector) vector);
       }
-    },   //  four byte signed integer
+    },
     BIGINT(new Int(64, true)) {
       @Override
       public Field getField() {
@@ -208,7 +208,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new BigIntWriterImpl((NullableBigIntVector) vector);
       }
-    },   //  eight byte signed integer
+    },
     DATE(Date.INSTANCE) {
       @Override
       public Field getField() {
@@ -224,7 +224,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new DateWriterImpl((NullableDateVector) vector);
       }
-    },   //  days since 4713bc
+    },
     TIME(Time.INSTANCE) {
       @Override
       public Field getField() {
@@ -240,8 +240,9 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new TimeWriterImpl((NullableTimeVector) vector);
       }
-    },   //  time in micros before or after 2000/1/1
-    TIMESTAMP(new Timestamp("")) {
+    },
+    // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
+    TIMESTAMP(new Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND)) {
       @Override
       public Field getField() {
         return TIMESTAMP_FIELD;
@@ -289,6 +290,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new IntervalYearWriterImpl((NullableIntervalYearVector) vector);
       }
     },
+    //  4 byte ieee 754
     FLOAT4(new FloatingPoint(Precision.SINGLE)) {
       @Override
       public Field getField() {
@@ -304,7 +306,8 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new Float4WriterImpl((NullableFloat4Vector) vector);
       }
-    },   //  4 byte ieee 754
+    },
+    //  8 byte ieee 754
     FLOAT8(new FloatingPoint(Precision.DOUBLE)) {
       @Override
       public Field getField() {
@@ -320,7 +323,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new Float8WriterImpl((NullableFloat8Vector) vector);
       }
-    },   //  8 byte ieee 754
+    },
     BIT(Bool.INSTANCE) {
       @Override
       public Field getField() {
@@ -336,7 +339,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new BitWriterImpl((NullableBitVector) vector);
       }
-    },  //  single bit value (boolean)
+    },
     VARCHAR(Utf8.INSTANCE) {
       @Override
       public Field getField() {
@@ -352,7 +355,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new VarCharWriterImpl((NullableVarCharVector) vector);
       }
-    },   //  utf8 variable length string
+    },
     VARBINARY(Binary.INSTANCE) {
       @Override
       public Field getField() {
@@ -368,7 +371,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new VarBinaryWriterImpl((NullableVarBinaryVector) vector);
       }
-    },   //  variable length binary
+    },
     DECIMAL(null) {
       @Override
       public ArrowType getType() {
@@ -388,7 +391,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new DecimalWriterImpl((NullableDecimalVector) vector);
       }
-    },   //  variable length binary
+    },
     UINT1(new Int(8, false)) {
       @Override
       public Field getField() {
@@ -404,7 +407,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UInt1WriterImpl((NullableUInt1Vector) vector);
       }
-    },  //  unsigned 1 byte integer
+    },
     UINT2(new Int(16, false)) {
       @Override
       public Field getField() {
@@ -420,7 +423,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UInt2WriterImpl((NullableUInt2Vector) vector);
       }
-    },  //  unsigned 2 byte integer
+    },
     UINT4(new Int(32, false)) {
       @Override
       public Field getField() {
@@ -436,7 +439,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UInt4WriterImpl((NullableUInt4Vector) vector);
       }
-    },   //  unsigned 4 byte integer
+    },
     UINT8(new Int(64, false)) {
       @Override
       public Field getField() {
@@ -452,7 +455,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UInt8WriterImpl((NullableUInt8Vector) vector);
       }
-    },   //  unsigned 8 byte integer
+    },
     LIST(List.INSTANCE) {
       @Override
       public Field getField() {
@@ -576,6 +579,9 @@ public MinorType visit(FloatingPoint type) {
       }
 
       @Override public MinorType visit(Timestamp type) {
+        if (type.getUnit() != TimeUnit.MILLISECOND) {
+          throw new UnsupportedOperationException("Only milliseconds supported: " + type);
+        }
         return MinorType.TIMESTAMP;
       }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index ed740cd0f1b78..3da8db298b4a3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -21,6 +21,7 @@
 import static org.apache.arrow.flatbuf.Precision.SINGLE;
 import static org.junit.Assert.assertEquals;
 
+import org.apache.arrow.flatbuf.TimeUnit;
 import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
@@ -80,7 +81,7 @@ public void nestedSchema() {
         new Field("child4.1", true, Utf8.INSTANCE, null)
         )));
     childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMP.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
-        new Field("child5.1", true, new Timestamp("UTC"), null),
+        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null),
         new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));
     Schema initialSchema = new Schema(childrenBuilder.build());

From dd1b95b90e73c3b0b69bfd6284e329eea41f689d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 5 Oct 2016 16:15:16 -0700
Subject: [PATCH 0156/1644] ARROW-318: Revise python/README.md given recent
 changes in codebase

Also removes a redundant LICENSE.txt from cpp/

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #157 from wesm/ARROW-318 and squashes the following commits:

9e802f2 [Wes McKinney] Update python/README.md. Remove redundant LICENSE.txt from cpp/
---
 cpp/LICENSE.txt  | 202 -----------------------------------------------
 python/README.md |  36 ++++-----
 2 files changed, 14 insertions(+), 224 deletions(-)
 delete mode 100644 cpp/LICENSE.txt

diff --git a/cpp/LICENSE.txt b/cpp/LICENSE.txt
deleted file mode 100644
index d645695673349..0000000000000
--- a/cpp/LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/python/README.md b/python/README.md
index bafe71b05ec22..3235d18377d56 100644
--- a/python/README.md
+++ b/python/README.md
@@ -19,25 +19,17 @@ These are the various projects that PyArrow depends on.
 1. **g++ and gcc Version >= 4.8**
 2. **cmake > 2.8.6**
 3. **boost**
-4. **Parquet-cpp**
-
-  The preferred way to install parquet-cpp is to use conda.
-  You need to set the ``PARQUET_HOME`` environment variable to where parquet-cpp is installed.
-  ```bash
-  conda install -y --channel apache/channel/dev parquet-cpp
-  ```
-5. **Arrow-cpp and its dependencies***
-
-   The Arrow C++ library must be built with all options enabled and installed with ``ARROW_HOME`` environment variable set to
-   the installation location. Look at (https://github.com/apache/arrow/blob/master/cpp/README.md) for
-   instructions. Alternatively you could just install arrow-cpp
-   from conda.
-  ```bash
-  conda install arrow-cpp -c apache/channel/dev
-  ```
-6. **Python dependencies: numpy, pandas, cython, pytest**
-
-#### Install pyarrow
-  ```bash
-  python setup.py build_ext --inplace
-  ```
+4. **Arrow-cpp and its dependencies***
+
+The Arrow C++ library must be built with all options enabled and installed with
+``ARROW_HOME`` environment variable set to the installation location. Look at
+(https://github.com/apache/arrow/blob/master/cpp/README.md) for instructions.
+
+5. **Python dependencies: numpy, pandas, cython, pytest**
+
+#### Build pyarrow and run the unit tests
+
+```bash
+python setup.py build_ext --inplace
+py.test pyarrow
+```

From 04cf8746f3588d7bfadcc0b9c8dbe71707bdd196 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 5 Oct 2016 16:20:31 -0700
Subject: [PATCH 0157/1644] ARROW-321: fix arrow licenses

Author: Julien Le Dem <julien@dremio.com>

Closes #159 from julienledem/fix_licenses and squashes the following commits:

0c97810 [Julien Le Dem] fix NOTICE
1489289 [Julien Le Dem] more licenses
0eb2aeb [Julien Le Dem] more licenses
9ac1159 [Julien Le Dem] more licenses
eafa0e1 [Julien Le Dem] more licenses
30b0fa1 [Julien Le Dem] more licenses
bcfc75f [Julien Le Dem] add ci
51db31b [Julien Le Dem] ARROW-321: fix arrow licenses
---
 NOTICE.txt                                    | 20 +++++++++++
 README.md                                     | 16 ++++++++-
 ci/travis_before_script_cpp.sh                | 13 ++++++++
 ci/travis_conda_build.sh                      | 12 +++++++
 ci/travis_install_conda.sh                    | 12 +++++++
 ci/travis_script_cpp.sh                       | 12 +++++++
 ci/travis_script_java.sh                      | 12 +++++++
 ci/travis_script_python.sh                    | 12 +++++++
 cpp/README.md                                 | 14 ++++++++
 cpp/cmake_modules/FindGPerf.cmake             | 12 +++++++
 cpp/cmake_modules/san-config.cmake            | 12 +++++++
 cpp/conda.recipe/build.sh                     | 12 +++++++
 cpp/conda.recipe/meta.yaml                    | 12 +++++++
 cpp/doc/HDFS.md                               | 16 ++++++++-
 cpp/doc/Parquet.md                            | 16 ++++++++-
 cpp/setup_build_env.sh                        | 12 +++++++
 cpp/src/arrow/io/symbols.map                  | 12 +++++++
 cpp/src/arrow/ipc/symbols.map                 | 12 +++++++
 cpp/src/arrow/symbols.map                     | 12 +++++++
 cpp/thirdparty/build_thirdparty.sh            | 12 +++++++
 cpp/thirdparty/download_thirdparty.sh         | 12 +++++++
 cpp/thirdparty/set_thirdparty_env.sh          | 12 +++++++
 cpp/thirdparty/versions.sh                    | 12 +++++++
 dev/release/02-source.sh                      | 33 ++++++++++++++++++-
 dev/release/README                            | 12 ++++++-
 format/File.fbs                               | 17 ++++++++++
 format/Guidelines.md                          | 13 ++++++++
 format/IPC.md                                 | 14 ++++++++
 format/Layout.md                              | 14 ++++++++
 format/Message.fbs                            | 17 ++++++++++
 format/Metadata.md                            | 14 ++++++++
 format/README.md                              | 16 ++++++++-
 java/README.md                                | 14 ++++++++
 java/vector/src/main/codegen/config.fmpp      | 22 +++++--------
 .../src/main/codegen/data/ArrowTypes.tdd      | 22 +++++--------
 .../main/codegen/data/ValueVectorTypes.tdd    | 22 +++++--------
 python/README.md                              | 14 ++++++++
 python/asv.conf.json                          | 12 +++++++
 python/conda.recipe/build.sh                  | 13 ++++++++
 python/conda.recipe/meta.yaml                 | 12 +++++++
 python/doc/Benchmarks.md                      | 13 ++++++++
 python/doc/INSTALL.md                         | 16 ++++++++-
 python/pyarrow/config.pyx                     | 12 +++++++
 43 files changed, 575 insertions(+), 46 deletions(-)

diff --git a/NOTICE.txt b/NOTICE.txt
index ce6e567dcb518..679bb59e6a97d 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -18,3 +18,23 @@ https://github.com/wesm/feather
 
 This product includes software from the DyND project (BSD 2-clause)
 https://github.com/libdynd
+
+This product includes software from the LLVM project
+ * distributed under the University of Illinois Open Source
+
+This product includes software from the google-lint project
+ * Copyright (c) 2009 Google Inc. All rights reserved.
+
+This product includes software from the mman-win32 project
+ * Copyright https://code.google.com/p/mman-win32/
+ * Licensed under the MIT License;
+
+This product includes software from the LevelDB project
+ * Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * Moved from Kudu http://github.com/cloudera/kudu
+
+This product includes software from the CMake project
+ * Copyright 2001-2009 Kitware, Inc.
+ * Copyright 2012-2014 Continuum Analytics, Inc.
+ * All rights reserved.
diff --git a/README.md b/README.md
index 84bae78cc7fbe..89114ee39b4a0 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 ## Apache Arrow
 
 <table>
@@ -42,4 +56,4 @@ integrations in other projects, we'd be happy to have you involved:
 
 [1]: mailto:dev-subscribe@arrow.apache.org
 [2]: https://github.com/apache/arrow/tree/master/format
-[3]: https://issues.apache.org/jira/browse/ARROW
\ No newline at end of file
+[3]: https://issues.apache.org/jira/browse/ARROW
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 2f02ef247af82..acd820bbed2d4 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -1,5 +1,18 @@
 #!/usr/bin/env bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+
 set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
diff --git a/ci/travis_conda_build.sh b/ci/travis_conda_build.sh
index a787df79a5574..17a33ae9717bc 100755
--- a/ci/travis_conda_build.sh
+++ b/ci/travis_conda_build.sh
@@ -1,5 +1,17 @@
 #!/usr/bin/env bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index e9225259e6d58..ffa017cbaf5dd 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -1,5 +1,17 @@
 #!/usr/bin/env bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -e
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index a3585507f0a6d..c3bd3b5f207a8 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -1,5 +1,17 @@
 #!/usr/bin/env bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -e
 
 : ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
diff --git a/ci/travis_script_java.sh b/ci/travis_script_java.sh
index 2d11eaeb4c57d..4679f9c6daf87 100755
--- a/ci/travis_script_java.sh
+++ b/ci/travis_script_java.sh
@@ -1,5 +1,17 @@
 #!/usr/bin/env bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -e
 
 JAVA_DIR=${TRAVIS_BUILD_DIR}/java
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 61c8e444361df..a75ff0778bc82 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -1,5 +1,17 @@
 #!/usr/bin/env bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -e
 
 PYTHON_DIR=$TRAVIS_BUILD_DIR/python
diff --git a/cpp/README.md b/cpp/README.md
index 129c5f15b150c..a1c3ef28447f5 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 # Arrow C++
 
 ## Setup Build Environment
diff --git a/cpp/cmake_modules/FindGPerf.cmake b/cpp/cmake_modules/FindGPerf.cmake
index e8310799c3671..e90d4d0039590 100644
--- a/cpp/cmake_modules/FindGPerf.cmake
+++ b/cpp/cmake_modules/FindGPerf.cmake
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 # -*- cmake -*-
 
 # - Find Google perftools
diff --git a/cpp/cmake_modules/san-config.cmake b/cpp/cmake_modules/san-config.cmake
index b847c96657a4a..fe52fef12ea5d 100644
--- a/cpp/cmake_modules/san-config.cmake
+++ b/cpp/cmake_modules/san-config.cmake
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 # Clang does not support using ASAN and TSAN simultaneously.
 if ("${ARROW_USE_ASAN}" AND "${ARROW_USE_TSAN}")
   message(SEND_ERROR "Can only enable one of ASAN or TSAN at a time")
diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh
index 2f2b748266747..6d7454e927265 100644
--- a/cpp/conda.recipe/build.sh
+++ b/cpp/conda.recipe/build.sh
@@ -1,5 +1,17 @@
 #!/bin/bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -e
 set -x
 
diff --git a/cpp/conda.recipe/meta.yaml b/cpp/conda.recipe/meta.yaml
index 75f3a8ba3d98f..31f150c1f0b00 100644
--- a/cpp/conda.recipe/meta.yaml
+++ b/cpp/conda.recipe/meta.yaml
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 package:
   name: arrow-cpp
   version: "0.1"
diff --git a/cpp/doc/HDFS.md b/cpp/doc/HDFS.md
index e0d5dfda21d93..83311db2d2dc2 100644
--- a/cpp/doc/HDFS.md
+++ b/cpp/doc/HDFS.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 ## Using Arrow's HDFS (Apache Hadoop Distributed File System) interface
 
 ### Build requirements
@@ -36,4 +50,4 @@ will set it automatically for you:
 
 ```shell
 export JAVA_HOME=$(/usr/libexec/java_home)
-```
\ No newline at end of file
+```
diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md
index 96471d94835f3..34b83e78d0a5c 100644
--- a/cpp/doc/Parquet.md
+++ b/cpp/doc/Parquet.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 ## Building Arrow-Parquet integration
 
 To use Arrow C++ with Parquet, you must first build the Arrow C++ libraries and
@@ -16,4 +30,4 @@ make -j4
 make install
 ```
 
-[1]: https://github.com/apache/parquet-cpp
\ No newline at end of file
+[1]: https://github.com/apache/parquet-cpp
diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
index fa779fdd5c2a3..546216753b382 100755
--- a/cpp/setup_build_env.sh
+++ b/cpp/setup_build_env.sh
@@ -1,5 +1,17 @@
 #!/bin/bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
 ./thirdparty/download_thirdparty.sh || { echo "download_thirdparty.sh failed" ; return; }
diff --git a/cpp/src/arrow/io/symbols.map b/cpp/src/arrow/io/symbols.map
index b4ad98cd7f2d0..1e87caef9c8c1 100644
--- a/cpp/src/arrow/io/symbols.map
+++ b/cpp/src/arrow/io/symbols.map
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 {
   # Symbols marked as 'local' are not exported by the DSO and thus may not
   # be used by client applications.
diff --git a/cpp/src/arrow/ipc/symbols.map b/cpp/src/arrow/ipc/symbols.map
index b4ad98cd7f2d0..1e87caef9c8c1 100644
--- a/cpp/src/arrow/ipc/symbols.map
+++ b/cpp/src/arrow/ipc/symbols.map
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 {
   # Symbols marked as 'local' are not exported by the DSO and thus may not
   # be used by client applications.
diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map
index 2ca8d7306105f..cc8c9ba3c94bf 100644
--- a/cpp/src/arrow/symbols.map
+++ b/cpp/src/arrow/symbols.map
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 {
   # Symbols marked as 'local' are not exported by the DSO and thus may not
   # be used by client applications.
diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
index 6cc776d09042a..5011e29c01a71 100755
--- a/cpp/thirdparty/build_thirdparty.sh
+++ b/cpp/thirdparty/build_thirdparty.sh
@@ -1,5 +1,17 @@
 #!/bin/bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -x
 set -e
 TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
diff --git a/cpp/thirdparty/download_thirdparty.sh b/cpp/thirdparty/download_thirdparty.sh
index d299afc15222b..b50e7bc06a14c 100755
--- a/cpp/thirdparty/download_thirdparty.sh
+++ b/cpp/thirdparty/download_thirdparty.sh
@@ -1,5 +1,17 @@
 #!/bin/bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -x
 set -e
 
diff --git a/cpp/thirdparty/set_thirdparty_env.sh b/cpp/thirdparty/set_thirdparty_env.sh
index 7e9531cd50864..135972ee9bdce 100755
--- a/cpp/thirdparty/set_thirdparty_env.sh
+++ b/cpp/thirdparty/set_thirdparty_env.sh
@@ -1,5 +1,17 @@
 #!/usr/bash
 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 source $SOURCE_DIR/versions.sh
 
diff --git a/cpp/thirdparty/versions.sh b/cpp/thirdparty/versions.sh
index cb455b4eadd3b..a7b21e19fccd6 100755
--- a/cpp/thirdparty/versions.sh
+++ b/cpp/thirdparty/versions.sh
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 GTEST_VERSION=1.7.0
 GTEST_URL="https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
 GTEST_BASEDIR=googletest-release-$GTEST_VERSION
diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index f44692d5e9d83..1bbe2e92753ce 100644
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -56,12 +56,43 @@ tarball=$tag.tar.gz
 # archive (identical hashes) using the scm tag
 git archive $release_hash --prefix $tag/ -o $tarball
 
+# download apache rat
+curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.12/apache-rat-0.12.jar > apache-rat-0.12.jar
+
+RAT="java -jar apache-rat-0.12.jar -d "
+
+# generate the rat report
+$RAT $tarball \
+  -e ".*" \
+  -e mman.h \
+  -e "*_generated.h" \
+  -e random.h \
+  -e status.cc \
+  -e status.h \
+  -e asan_symbolize.py \
+  -e cpplint.py \
+  -e FindPythonLibsNew.cmake \
+  -e pax_global_header \
+  -e MANIFEST.in \
+  -e __init__.pxd \
+  -e __init__.py \
+  -e requirements.txt \
+  > rat.txt
+UNAPPROVED=`cat rat.txt  | grep "Unknown Licenses" | head -n 1 | cut -d " " -f 1`
+
+if [ "0" -eq "${UNAPPROVED}" ]; then
+  echo "No unnaproved licenses"
+else
+  echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
+  exit
+fi
+
 # sign the archive
 gpg --armor --output ${tarball}.asc --detach-sig $tarball
 gpg --print-md MD5 $tarball > ${tarball}.md5
 shasum $tarball > ${tarball}.sha
 
-# check out the parquet RC folder
+# check out the arrow RC folder
 svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp
 
 # add the release candidate for the tag
diff --git a/dev/release/README b/dev/release/README
index 4fcc5d9728c26..07402030bf699 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -3,6 +3,9 @@ requirements:
 - a gpg key to sign the artifacts
 
 to release, run the following (replace 0.1.0 with version to release):
+
+#create a release branch
+git co -b release-0_1_0
 # prepare release v 0.1.0 (run tests, sign artifacts). Next version will be 0.1.1-SNAPSHOT
 dev/release/00-prepare.sh 0.1.0 0.1.1
 # tag and push to maven repo (repo will have to be finalized separately)
@@ -11,5 +14,12 @@ dev/release/01-perform.sh
 dev/release/02-source.sh 0.1.0 0
 
 useful commands:
-to set the mvn version in the poms
+- to set the mvn version in the poms
 mvn versions:set -DnewVersion=0.1-SNAPSHOT
+- reset your workspace
+git reset --hard
+- setup gpg agent
+eval $(gpg-agent --daemon --allow-preset-passphrase)
+gpg --use-agent -s LICENSE.txt 
+- delete tag localy
+git tag -d apache-arrow-0.1.0
diff --git a/format/File.fbs b/format/File.fbs
index a29bbc694bc13..f28dc204d58d9 100644
--- a/format/File.fbs
+++ b/format/File.fbs
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 include "Message.fbs";
 
 namespace org.apache.arrow.flatbuf;
diff --git a/format/Guidelines.md b/format/Guidelines.md
index 14f1057850439..c75da9f98bebe 100644
--- a/format/Guidelines.md
+++ b/format/Guidelines.md
@@ -1,3 +1,16 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
 # Implementation guidelines
 
 An execution engine (or framework, or UDF executor, or storage engine, etc) can implements only a subset of the Arrow spec and/or extend it given the following constraints:
diff --git a/format/IPC.md b/format/IPC.md
index 1f39e762ab70d..3f78126ef55d3 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 # Interprocess messaging / communication (IPC)
 
 ## File format
diff --git a/format/Layout.md b/format/Layout.md
index a953930e172e7..251af9dd8a128 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 # Arrow: Physical memory layout
 
 ## Definitions / Terminology
diff --git a/format/Message.fbs b/format/Message.fbs
index d8fa65006c24d..2ec9fd1817bd5 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 namespace org.apache.arrow.flatbuf;
 
 enum MetadataVersion:short {
diff --git a/format/Metadata.md b/format/Metadata.md
index e227b8d4afd84..fa5f623ac9797 100644
--- a/format/Metadata.md
+++ b/format/Metadata.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 # Metadata: Logical types, schemas, data headers
 
 This is documentation for the Arrow metadata specification, which enables
diff --git a/format/README.md b/format/README.md
index 78e15207ee95a..048badb12214b 100644
--- a/format/README.md
+++ b/format/README.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 ## Arrow specification documents
 
 > **Work-in-progress specification documents**. These are discussion documents
@@ -21,4 +35,4 @@ couple related pieces of information:
    schema, and enable a system to send and receive Arrow row batches in a form
    that can be precisely disassembled or reconstructed.
 
-[1]: http://github.com/google/flatbuffers
\ No newline at end of file
+[1]: http://github.com/google/flatbuffers
diff --git a/java/README.md b/java/README.md
index 5e1d30d9fd26e..a57e35afbbd20 100644
--- a/java/README.md
+++ b/java/README.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 # Arrow Java
 
 ## Setup Build Environment
diff --git a/java/vector/src/main/codegen/config.fmpp b/java/vector/src/main/codegen/config.fmpp
index 6d92ba830ee2c..92881dc914f2a 100644
--- a/java/vector/src/main/codegen/config.fmpp
+++ b/java/vector/src/main/codegen/config.fmpp
@@ -1,18 +1,14 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
 #
-# http:# www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
 
 data: {
     # TODO:  Rename to ~valueVectorModesAndTypes for clarity.
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 11ac99af42414..c0b942bc3595d 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -1,18 +1,14 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
 #
-# http:# www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
 
 {
   types: [
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index 421dd7ef92e63..f7790bb3d6ddf 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -1,18 +1,14 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
 #
-# http:# www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
 
 {
   modes: [
diff --git a/python/README.md b/python/README.md
index 3235d18377d56..6febcbcbcbfe7 100644
--- a/python/README.md
+++ b/python/README.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 ## Python library for Apache Arrow
 
 This library provides a Pythonic API wrapper for the reference Arrow C++
diff --git a/python/asv.conf.json b/python/asv.conf.json
index 96beba64c2e6e..0c059fd79c1f2 100644
--- a/python/asv.conf.json
+++ b/python/asv.conf.json
@@ -1,3 +1,15 @@
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License. See accompanying LICENSE file.
+
 {
     // The version of the config file format.  Do not change, unless
     // you know what you are doing.
diff --git a/python/conda.recipe/build.sh b/python/conda.recipe/build.sh
index f32710073c7c2..fafe71e7adb75 100644
--- a/python/conda.recipe/build.sh
+++ b/python/conda.recipe/build.sh
@@ -1,4 +1,17 @@
 #!/bin/bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 set -ex
 
 # Build dependency
diff --git a/python/conda.recipe/meta.yaml b/python/conda.recipe/meta.yaml
index 98ae4141e3bd7..b37dfde0a0d6f 100644
--- a/python/conda.recipe/meta.yaml
+++ b/python/conda.recipe/meta.yaml
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 package:
   name: pyarrow
   version: "0.1"
diff --git a/python/doc/Benchmarks.md b/python/doc/Benchmarks.md
index 8edfb6209e4af..1c36801858278 100644
--- a/python/doc/Benchmarks.md
+++ b/python/doc/Benchmarks.md
@@ -1,3 +1,16 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
 ## Benchmark Requirements
 
 The benchmarks are run using [asv][1] which is also their only requirement.
diff --git a/python/doc/INSTALL.md b/python/doc/INSTALL.md
index d30a03046eda7..81eed565d9123 100644
--- a/python/doc/INSTALL.md
+++ b/python/doc/INSTALL.md
@@ -1,3 +1,17 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
 ## Building pyarrow (Apache Arrow Python library)
 
 First, clone the master git repository:
@@ -84,4 +98,4 @@ Out[2]:
 ]
 ```
 
-[1]: https://cmake.org/
\ No newline at end of file
+[1]: https://cmake.org/
diff --git a/python/pyarrow/config.pyx b/python/pyarrow/config.pyx
index 1047a472fe338..778c15a5e655b 100644
--- a/python/pyarrow/config.pyx
+++ b/python/pyarrow/config.pyx
@@ -1,3 +1,15 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
 # cython: profile=False
 # distutils: language = c++
 # cython: embedsignature = True

From f1a4bd176bc2139ba785522200d7630408328911 Mon Sep 17 00:00:00 2001
From: adeneche <adeneche@dremio.com>
Date: Wed, 5 Oct 2016 20:19:42 -0700
Subject: [PATCH 0158/1644] =?UTF-8?q?ARROW-320:=20ComplexCopier.copy(Field?=
 =?UTF-8?q?Reader,=20FieldWriter)=20should=20not=20st=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…art a list if reader is not set

Author: adeneche <adeneche@dremio.com>

Closes #160 from adeneche/ARROW-320 and squashes the following commits:

5c6ebc5 [adeneche] ARROW-320: ComplexCopier.copy(FieldReader, FieldWriter) should not start a list if reader is not set
---
 .../main/codegen/templates/ComplexCopier.java | 14 +--
 .../apache/arrow/vector/TestListVector.java   | 87 +++++++++++++++++++
 2 files changed, 95 insertions(+), 6 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java

diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index a5756a47ad785..0dffe5e30bea0 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -47,23 +47,25 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
       switch (mt) {
 
       case LIST:
-        writer.startList();
-        while (reader.next()) {
-          writeValue(reader.reader(), getListWriterForReader(reader.reader(), writer));
+        if (reader.isSet()) {
+          writer.startList();
+          while (reader.next()) {
+            writeValue(reader.reader(), getListWriterForReader(reader.reader(), writer));
+          }
+          writer.endList();
         }
-        writer.endList();
         break;
       case MAP:
-        writer.start();
         if (reader.isSet()) {
+          writer.start();
           for(String name : reader){
             FieldReader childReader = reader.reader(name);
             if(childReader.isSet()){
               writeValue(childReader, getMapWriterForReader(childReader, writer, name));
             }
           }
+          writer.end();
         }
-        writer.end();
         break;
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
new file mode 100644
index 0000000000000..bb7103365557f
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.ComplexCopier;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestListVector {
+  private final static String EMPTY_SCHEMA_PATH = "";
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testCopyFrom() throws Exception {
+    try (ListVector inVector = new ListVector("input", allocator, null);
+         ListVector outVector = new ListVector("output", allocator, null)) {
+      UnionListWriter writer = inVector.getWriter();
+      writer.allocate();
+
+      // populate input vector with the following records
+      // [1, 2, 3]
+      // null
+      // []
+      writer.setPosition(0); // optional
+      writer.startList();
+      writer.bigInt().writeBigInt(1);
+      writer.bigInt().writeBigInt(2);
+      writer.bigInt().writeBigInt(3);
+      writer.endList();
+
+      writer.setPosition(2);
+      writer.startList();
+      writer.endList();
+
+      writer.setValueCount(3);
+
+      // copy values from input to output
+      outVector.allocateNew();
+      for (int i = 0; i < 3; i++) {
+        outVector.copyFrom(i, i, inVector);
+      }
+      outVector.getMutator().setValueCount(3);
+
+      // assert the output vector is correct
+      FieldReader reader = outVector.getReader();
+      Assert.assertTrue("shouldn't be null", reader.isSet());
+      reader.setPosition(1);
+      Assert.assertFalse("should be null", reader.isSet());
+      reader.setPosition(2);
+      Assert.assertTrue("shouldn't be null", reader.isSet());
+    }
+  }
+}

From 3f85cee51e45165c4be8d251849d2b3765b9b4dd Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 7 Oct 2016 12:12:58 -0700
Subject: [PATCH 0159/1644] ARROW-324: Update arrow metadata diagram

Author: Julien Le Dem <julien@dremio.com>

Closes #161 from julienledem/diagram and squashes the following commits:

f018cf5 [Julien Le Dem] ARROW-324: Update arrow metadata diagram
---
 format/Arrow.graffle | Bin 3646 -> 4142 bytes
 format/Arrow.png     | Bin 86598 -> 112671 bytes
 format/Metadata.md   |  10 ++++++++++
 3 files changed, 10 insertions(+)

diff --git a/format/Arrow.graffle b/format/Arrow.graffle
index 453e85025d8d324310e27daa3692a98e3bceb332..f4eead9223160f6e16f97bb8e9d6ed6a05e290e6 100644
GIT binary patch
literal 4142
zcmV+}5Yg`+iwFP!000030PS6EcbhmC{yg~=Y`^U8nbeVl0Fmi*j}xbpbkZafr|q`m
zIXyDSHd_oX;HH_j|9vH}4aPWjQsX#SZcqHel@Q`S&wY`Qp8e-b-&H=dAas21*C((B
zo+!+-eB1H5zdm_=_2Mac@}K8B&;HRkKfHQ(aik1fCybPf*9WI3hsu+udwcc4z-4=T
zjjM)oaeC6cQg|JEdq-zalqbC?8tm`wU0+|<sF12zeqRWNdl!K}U_o?q$}4-y3)Jk$
ze!@?a)IQZ5{;Ta+(es_3o_%CD&+As?d}gQgh6N{{&AvR_6Tk9<ju){m3!Z~#d*hGe
zBK+KrMI&D2o22GRAsPf!eE8{E7zO;2pYs;h{J!UO1AjQEo%2t>1+>#~nf`2VB$APa
zx><t;Z$8phRR<Z7XM5wSlO_x!zi$6;7)Fy*N5PQojgKEt>!UcOXY<nydX6;_8(sTI
zOh|;|W0Pil{R(y!@O@YL3RT0{RUm-I_wQ2;79|c{8qOAO`mU3nHCN=Mk;;#I6=gd(
z5o<;_E=wiHZB7pID*cb?(ahoGcq5?KX|vLXot<Z!!+Uhn72{9oholcf_TKmVOK8L)
z^*+<^JaD>Bste=JjQcWsE7KB9=WjM6pXu|NQocLvdv(|8dNZwpv#pAyn*(a`Dkky)
zqwaH+t6%(KEXuzKeQwZbra=fm{V5<%fu=xxUnTnhD?bDNnrWS=#1Hc2Uk+V|dCDnM
z8myl$({RFpOK+MMb!Sct6~|M4Rm5Aq*bB#HS2u&1B9!AIFB|J(!?%Wg=0$nHPI?c`
zbl*saL%}Hp0H0RM9Fj6n(jcg5CdLN8p#UQU@vfq(Y7JvkBL;znfeFzC*a+lDz%<Z`
zkd{KNM+dA3YvWFeuGp7ow%og;W`VTpU(bqOvM|=symSW0X(t-$6qFOmQ^&gntl4@p
z6~XC>64?tsbfV176Tinw%=ywZ2r_kyb<NZaVBm>*j|*S=QLN!vt;wDe<eI)WeSfDJ
zY34G*9<{UGmlY^xrFpcJJ##VNt)}k$1?Ds**M{k%sdl%5r~_Nn3!V6nHRwX5p_b)a
zJ{5;Bz{!_z*yi6k2zI`<I=&Z0ZMW3{t&ZpLcdJK(kVQZ_X6|Peaoc*vhU}N`T-Nek
zKj6Z@v;t-$O>1=n#ys(%?G71#=TfZ!4eVC*MbiQQ3n7Z%bRymm!tomc;!PM;@n*&%
z7{?+Sh(&m5@utIAL^rsIsMiTR8Vv*Lwmw<nhsf#nBIb3d#i<VL?8H|Qsz_;B9iWJA
z<4Q(@o9}asP12ulIc8_UHhsG?25P$S95xU@o6iNj&riILKmQgBiGo`^xg|MGib0Vv
zUlTA2CY@j}f8EE7+IJ-bq}~qAVk&|KRCM9<pRzqSVe^bis~V{3nU{uk6%6tB-1U#e
z3D$QNuJif(omRd1{2}g2JqV^6kQy0&^FNM`X3sNXch;rW%~ZJ#b;H;bP1%Dw?%evp
z@)X6ZxvTQ1b9sF(yyly70~s}qYlBW=-x#Tn-ClzNkSx&Dj^nx|>8Z4#4`@Se)`@7F
zTfLDv6@SJ*EoJ%+{XP%LzbFxRu)_Tk&EI@{V&Pk3119JMj~t!4DbXp>xgvC41Ww-(
znXdBjxn=IDdLQ>xCk5C(52Bvm<<WzCKm(EC>+^`!%9I|b_=@CX-?o4hApm4ofdmu~
zVXlIFbe(kAQ<&|rrQK)^bN)3Es%i5)XWiuflL^VK$7ek63VGH)=2Qzj<&qima;h8P
zhwH`){qWe%n;_0mqc}d(R%hpn@oL=N7MG)WGH<iGyiGe}AW!(+4L)=6bv3J~O>K}g
zX6QN5zVhy<e)-|m`PtR+w?_TlhpUrUN4#}Ax0`Np$1hxJF_G;q(B$cmPa5$bG2;4G
z7;%E(0wYdTLmKgIFyi>mMqKj95<H@92aiC!h({ohJhBZuqOLj^Ac#zY0c4aN3}6`y
zR67{Ba{6o|0)aJo`-{*8;2P1TN_15lU1E~iHla)Uz)SeRXe)eR4H}CQ;=q(X@HY6s
zaMeCAXT&vHX(+99wXO8o&~>GimR5QS^<XQkw4s{|th8ZZX{EQoN@G=Kyi4E_&8#|)
z5GcHTvWQ1?ph_Ou1|BgakCee9WIK2SFB&=HZj0oRZQv1I@<<sxVr&PGEX+P?SeHDq
z4LqVr9w~!Iwqz!cv4}?qL6S$dfk)(wz*2YwZ%3XLEsARlBu7KHf=7_#kurEh-3lJj
z%|-r*0i-{&6+8kZkCee9+c7&!H^8Ft8bgyjvJE@}B#)HABk(>vl2NARiKKXVb$K4)
zL{!sE{vz1G5J6!60I#TWbY>e^Lp3F9l)xI;+>W{7x?VHY1rCO$$@NmUf=Y~)u3Km(
znj_Xl(~&SpB~K5>a=ny_PY-|Xar)bc`Qd%vX8THWAo})OgL&aYK3o>JNG!2-Vu$+*
zhrpkIJ#=dx{29`OepEQ2pVFUB!MdcMYST}XyLVQUkbuOOIeGePP59CzzN(EcvFU|G
zSGod%rmb(WK`hr7*@E>&!ie?<8?@SB^XTyWtnsb&PojN0tDl`G-`BR_-~s>|YP!Hr
z`O&Lff~4w8kTA>XQ+FdQ+#ukM5t~xQ@nUNy6nkOq#~(KWd2o%3|6nAMS8e1S_`WM0
z%L%%4vc^R_uw1mG>RH*SLnOLVbZN#K(FG*BDsH<Rv9q7VSBfv7PVlASlJTXQ5?|HE
z*9+ID5$~IL&csHfMl!M4F?BoNn!I%``_}M>ZI^9IZbVZ-jTmWy(m*w?NoAu^j__B1
zN0g1gVB3k_I(F2H(<92i$S%}(RkW+)f46p=@I&ZyJ!bDKZ5~)M>d75aHfZLas;_Am
z-pbE?D|;8-7mn*PyW!Bj@7a%MM_x`SHKG#(A=M-R0n_AaKulo@b>orm|EU%YvI}~-
zbC2i76UQ*jPhlNV2dH5kVU8%>goX;RX{s46NeW;35fz1V6DsF$=#1WMG!CT?@~}Qg
z>Va52$F&3I$<0bvX2rpMtoYhiBS3_3{x)Y;O)Px^{vF*7XqYuK^CF`OSaZaes=Y$a
z$}P^s15VgK<iCe67>gc{d!v~=yyq+%&+IeBqbY!t9oNgb4^^Lh#4#mv*aBWE(rW}8
zI)Mg(I`?-4+90d}#5o#6gt>CWX*En9dn|!g6-z)FOUPK_5ylc7hq-nrBZ*Z-5(wWX
zl7MT<2dWy}Rk*%O7yJGx!AIyXBUP=bnHN_Qt_ktLn2M+LN7uPCjr6jeX*ngN>O<DT
ztZalg?a@BlSCTu`+VO_2OWQ8XjEjkT%-t*+-0Ul^FN9or;}4^K<xk}e6E&U2-<7{w
zJJv9a{QieNi>OT_Dnx!~H*YAv3Cdp&?6sRjt!akA+21gb_@{BISL2CZ7(H@79hocg
z(-FC=pN^plF(lm3a+MWh57JHF07g|Y$ucT@+>Dxc$_2MbmwaW({d;)ih}_pBCs;3V
zu`@YxsB1N?+Fp2dM&*&*=xjj`LGwQFs{F<xb78))X*nt}O}TJY6&9|NJW>Xa=<DK<
z7>m%*);<Kk$e)lTQ5}+y-bOhrv9|4p;`}KB#@eUpz*{*YN$W~SV=FiWuXNx5C;FTW
z6>ud1B@YcCxyVn|m#DLPtWP)M5gO*%4)r`1hWpB?6GbljcHne3JI!xkV*<%|?pyhg
z+_I@}Rm?3<E%GxN@H(K8)ms-5Gq`i?@le|aJS!@8qvv=skpY20ATT+;lGR6a@YsjV
z;Hrkrz#WIpG^FdM4v?;LXd#Wr>1~qR${#oD@gRT&a^&n`qit~S$!rFh4MVplO0CYQ
z86>ORr9n8n8eX5j3r*?oR>|Le#p65LWyi5`SnHg$4SAje8RnW1L?nVLh@c0+PTRL{
zPMOyg%bX+;;{#&}@|#=32H??+ZepSm?vtnl)-<eXSXcRXT{G26M32x|QS=D$J)=jW
z0@Ku0UCz3cJXe17*dBH|EGY9?s9R_S5VfX5!v7Q11Vj_7w1^4ari%D4SSJv{c>NYl
zd+fBLuZmW*`#p218HQ%4rfEuAk+f1it$4o8<O7xuI-476_f1JbAU2K?o3w55bnePL
zR7vFYss;PI-v?ay-r!ndjx<u$0{m47Tyr%45iVf3!kn*Z0<mDB9;dAEI_4e8j8&yF
zR`K)4XWa8zZ_dgcrK;j}y2Z1`A1topQT)S$C$4n3W=vs;sVZUWAkL4h7f=J*Wg|HG
zoeuA}$us(yr0_!{3lNY_$ur;@kR=zRtN0|@OJ@0j9na6(h*dN)hOHE|r4D^u%Wr{W
zXG5roA|;yM<1VuhnbbcgXPHQQbP!(oOUS3KiQ6!AY!;sS)<-reScr&>^cyF1+9?X>
zoF<M5E;zGI&{hoW1pj?5cqqIG7R-pl8S&p48(jHKu~*_mdJ-}uF$KM0L3-v%AeG}g
zap8+{Qi<dl?X!XqS)4|Jv*H&);j=S^>E@>ESS(zw8E+hRy|CJO3YRSLGGFEDp3|oh
zn^B-#j!qo4oWSy5IbOpF)5{#i>m-fWQ`sitprcbc>v6^!<m8i;X^y;ZHeEh$i|t1<
zjqw<be37k422p2DfI#v>VwrdqiKj27P8gTJLj$90f8Z=7KMgkC#E<jPGYhs?wCl0x
zF7hEqPqW9P?406HQ1QyQoenn{qScMeogP<;0MwsC@Dvyd)b}ym2SE8*^xVw3;)*g@
z9UXo-bRFg?r%Y+Ee!kF|AF<^x=RDvwMN=JI{5;ZFnybswl?z@v;P>cbxP+{L@_f*o
zH4pqK^82r7z*~^&LMC@Js;?t<IGTp^=re}X@J;C|Im59ZIDd<$>Ug^yFQ;^~C$e}*
zXcwO7pdS5gAHQ_py@i1N_rQIBa_raf9|wP&G>>}q-uY{w!S>siuGKx{-|h3)@O`g+
z{Kl>G<A+Du)fIaA>HXU?P(OKlRBwLbGB3Wk*N4B|-uzS6iGHMe1f$$m{{{CggQ?tM
zLG`nPcasMPA&eLFHdJI95SsDEx8`)VuHd9XEiTz-Zf014u0DqQaw&YEGH=Y4KPaR6
zX;zJI#JR@=rssGAmyFuyo_n(x?3ey^-F3R&5>5ygZ4Ri#3r~b&)))KUjOjmd<TxW=
zIngXH9a8Tz4X50Pq(q<lr*Sicf0;O4bt`f{FXMB?g@swBp2hB1_O<7H8nQGPyXER%
zhwR9UoZz-@e^c!K&9e7f$Kzq!e2zUF1_AGhxF=Ipn&WMY!$)5R)Z<|LkaTgzyGkwM
zXp|zzI(BKi3&aMZqbw5tOdy($b|*gi)F4RuJoV)G<<L#-<ct8nJ#D(^aJ&Q7>~$kh
zaOE6wO_#QhnZS0=GeKJ3lNlMzOV*vDu5^pJ_?c=op6X>n9`x&~S<nyfeZQag0&`~)
sBJs0@VIdb3jGPpp<EMh!i+;*7(F|*(?*o^^<oVA30iUyrkhkmr031pl3;+NC

literal 3646
zcmV-E4#DvsiwFP!000030PS7>Q`<Ne|9SZ<eEe{8Hw%d*e`ne50HK8*1(wkEwv(A1
z#a0p>j+NR@Ded-uKgmvFCw3BGNdmUmoyLiuq-W`UK6;X#<o(~i_C4hb2|~*JU$<cm
zY%9cfm`nZcuiNjhU+-GmzrEVp|3~BE==zV#V`bpcFj6kxAD*2aDcigC`oUn}k$Syx
z-B2#iPMg;X&!b*HKHpZhdr>sltJiODZ);dc)g0Crf?@qKU;`3FcV|4aU7n!kM(#E*
zCdqwD9sa9Jo#@rpPy09I?$v=4(J$l--;v<dcgffNy7-mfq<%!YBzOh(>*LFD5?(fS
zaKy9xkmNi`go6N!i=XzxDBzF$ifdS7eV=v%HXPJ0_|=;LcRC(1_Uj{&j5IXt8Z^23
z$WT=SWJLDs<E#@Eh7mh(|2GVyNzqX-B=zzAL+sp$B7K(^H|SAkA~t&Vk(iJO$M+^`
zeE$x16!2q5`3_an+)->@`|)E+Uy<O*!(l#clX)~PbSBAZBb6UlD$3SxBG!!VJd#R|
zRZed5EZI$ZH*-6=-w5z+s#dDl`9)S8UZInk7+<9qNfn0V6Jz~*7{n3wzu@p9pk11p
z!niKus?4@znxaX+k(_*5<+()68}|JJk9Pf>QZTPnEZrPnhi5U74+!>NsoePDi?JyG
zBCNTAqnthnKw}qJyFgc<v8P&lKvRAO{4b}S$b<#6>E8}LN_^#vC=Jq|P1B&^z{7V<
z2Yb1Kp+bG-S4DixCVNdia(y?*CBYmOv$C-%HkdQ)6F-`T>!kKjuKGqM912D$fcLab
zPD5d$&l&Ye0Tf0L!Tjjd@38rnEF=mH`Gk01NW=+fD$fyeG8QF%i(6Sd-Ke><cAum;
zA=sU-LI$uI$g1iTW^;LFQ%w>XU?!1}xCo*i>oOmEhd2<V*5`)kWKxe=tw{co(QE~!
z2m-*`QJ^&iyW$K&8fENZTw%K~TVdi}TvWD&Czg;YR|i&2-%(5**7P4UjWd=pRL%#c
zW-h$1A$3OedRkrqL*|+->|vMPW}7Y*C}zodG?{JNV*btNEllbl2&NR6<UhKREh0&E
zM!ML!n@ZBbUN~y~u@Fa87Wq?VIo*?tlU!r!&D93Xeb`x>7TG)$C)!+`WUeHWJd{av
zyX;KD31Ualt;H;&n}%eODzJztS!5L~VpW4hG;NVLVhaf%d87(FVn`lY1&^53;1Osp
z8P({}mON4g9?>O_tb#|3ip-B#`eGhgc3<*HNqEGPJhBQN(W}BEKwsjKSO7{UsREN|
zl1Wy<BwAHgY=E(B#ReeBBvoJ%B$;FtOrlnVNet6o!X$<%M>myV5-6Eu6-<Jv!X!Fc
zz$6HA3C)sBQUxXfl1Wy<B=9jzk};<HQ%TbC1NE^?!l|fcfXpBITg(FGSf&cRq1uu+
zR>2!piDT8xWpS*!A>&vp!6b5}W))1L*;Qc@71n@(mXL`m7i%g(CJzs1o!O?A*{7yK
zZAY<?74KEE;$X%f1vWm1jha0?Ansf~Dh6#)*yu(qFV`64YcyYn>APtawt}bwTi6e2
z{Esu}LZqXX!yJaiEmgCUFX6B)zMEj{d#l5FIBI*X4rq0J%0HbR4nh(EWkSX0WJrGb
z!DSuhv49Kz(h7)+biLIL2=T>5+Zz)8$)#EY9JsCMt8M`PFN7%mFf8$bkQRSfKzxX8
zk>bOSMKmoI(LpT2Q;QD+#v+EvMMS<d@NqN@u-E$RY;6gh+pSKhBBkYYfbzC^aW3|s
z;h?o9HMkCKkQD9`KO}ofn=y}<zV&0y4#l(uFFouKv6HR9GN=90M}WyN3=SYY0h58&
z0VY#|sZ3yc?{nW_132n4m&2$z5cT#~LxAepz=?Y`V>lsm)o?N-oXP~J>*2s7iIuMl
zQ*pV^1Uf)lBXo2Loiag3Y}>CQa^eAzX{nO|k*2R2IhsUHsX86nCt-y9gAMp`5ik22
z^Mx-Lx5hK@;u1h1FaeaIt{PCP1XP&-)gTV-iw<J#z0n}xMT-M(_7rYVt*ua;v$Gdp
zJ_l48cG^v?-oZl>Rb@ieA!A-;9XvihS|f6xL{6zD>7tHRUF4|t8j%Aea!S?bP#*_(
zmBo%YkZNt~tg0=sQzq=Z_85+MO~i2{8<<+ijD?z94UhcQV8gaYp34-&c-?g^9g`}9
z7)sVLRd=#`yiAnzN;Tf_s|Jq@S|}BGPB+pYwQ2N`x=}Blnp6G-mEC`KqhAz5`}f*<
zBpIW-JVsY73aPkP?5k?n1wpG73Tkcg*1q|Bk*y2ER9{oiBW{D@KJ(qxhOB}o3|U3M
ztO2MS$gmB=)>UM2GqB=W3B%k`RPc1&*J~QhbO|GgKaQIP(q6Wn$D&sxjL~_&uCXYJ
zLh#9hYkX!=d5pW+==2d+fSP6?&9FJ<ps51Qw$%)IB#EzB6u;zPCQ^D1Y3A(aS>sST
z59@RuQrE%hQO^yCzqq^p$nlv6)^yXzPI?*DE~1^a9aMxY!Pa)}kdKw`q*>-oAfzWl
zp*i=9)0fEGSl|t8a4XFb|2=$7Nc3|26J3Se3o(<~D4UzR4OTs%mH{v2iXh_vs^WB%
zs&NihkzUhP+qNLE4ckI?yi%`OFWs|8PvzM|>Df!q{@Hu>9ZEbmlz#nF`SnP9j9(8|
zHTf3b=jJTk;m6Ovy+Gd<nPA<%tqcJwR~Uon7dGJB`nXT_lyI~w!XJ7bZhNDx;{^N9
zAd11Z)S7OZ8q`b^iT}7Wj8rrBe$fl}ev$n|-Y>Eq>iud^g&MR}9dOGM^Xt<2l{B|v
z=hwv%magxUO=&;E^F`KUJzq;R3Ow7)QBcs(#hFZHc7DsV@{HsggqPG1v>yYn(!aBB
zF3gvzHbW)0El*u5!>MbMM^?cjYkOy3j74bbYhQR?=1)kHC=W?UZ(}tqv9>qz#pfm=
zU~VEfWdTRTYDq_<8XTfM^?bds=e5n(ql%qoqya5Ek)^%ti7fF$M7zC+_#Ny-ECBf#
z{HD_*ecalxQsZqPYttjI#PbN6iC-?iUWm(ZvvQrU#H)$>GB~(9#F5k6z+Nuu?kOHi
zjw{<ZpmfWwaXi_`guoP0XdGWwyhYLgFMSyfKGiZDc;GUejts*#05Vh#Eu>pA2u*U^
z>eu0Vd{99Gxe~Y8(K~o_5SnS_hqcui^_FK94q8t*ri~q5U0a{O3*|{VrSf;*kubzv
zauRnARorKROnc1;lE>wg5kVWkPMf)RXT<M{b3c+1<A$*WdE?eJ6U2uRXqIYmpG37_
zP1kf?GgSWF(34|&U;OA1nol%(gxVuVkCqB-+fWS|#VL7i^`po3u+t&ID)%rgpcP=L
zH3M4we@nH2rHhkIM1Z)hBL0hJSP;SZ%oAOI>9k@z6|Lxx+Zm~wrf#aXZA)5_w6c0y
z@tI5H#>kCEheQ3bQ&JGb?{%F2s@*DthnHq=q<k&+k_vjvreml*8eI2?xkjp5fWH!f
zYmVi=z(fC@aL(7Z#aWM5^3o1XTf>x<oUtn3w&Qc|d98QO${S5p#YeV_O~-#a+{UB$
zhmB8M8F0;*k}o<e*P-e0{K$F%HNagmf)nd>c)g9MfxXP6@TQRktW4Cc3D<E-h77tb
zchKz>aaiES@p)?n+`O>YOvknow51MxoXeYly4fMr#E=q8?{Sw|h)nXIGiRAd`*;vu
zvwO&=+Qd2xsY}8$=G>4;!a_u3q(9J*wo??&DJC8hTykccpsg6#3I6+B@JM(OB*=-w
zocMnT8C<ibcyZ-KdNO24#uWID1Zl~Wfz%A&i3eXcCzVK^<31@EB8y@am>0hc3isj^
zPB%9_>X7h$YJ8yNc4@Zr432BdhS7YJJMd{AM<i!JGjnv}u0sQdy`z4EhUsID#_Lu(
zUQcD4go6%G%~_9Utig=D7pr;fceB&wV_jSp<&^OWj+mIONET6Z1t5_8A=?5-MPfI{
zRDp5&2PhcbvH`u9{506OiZ}DnlLy;7-1SNH5c!a!r`hA9>>0zKpyC~KX@_?iLhDAR
zrN@~f0F7N>?Lwd+xQC2ARa1TzHJ2+Z&M1S`(e1ZGj}l)wBT9qxXA_;Xh}{2iE&`rY
zG*!XH`<uqn++6ORx!|Qk-bOd!J!A!x=Y!_FdB~!O_21!uYmnMPX6|N`Uq`&`Y8uj`
zYYk4rH>InjgcBCfZ(^4o*PHq?Mwc(h;X^{Vu%N?(=$m`;*8Afl1mwSm-lx+OcA))!
z`1@(|xOdRIcn@^g{`l5&x<~xGd+{EA>a|ZkcnAFc(XoDgjoyC#^zj@VoPIn$Xny80
zufKY?M{gFZe?~gd->DwKD2wvH<i2GvHFsE$eSYz7vJnx&n$7=A71<_)cKmK{dwRC6
zpip5JSL6%tW=MgpzJzwU5<bu>8|In~R+0TQtHup+;S+)BIo`k(!S03c-7N?E6}vs~
zXxG1o6T(HC1MKj`6XBTk#miyF^q)9#oRP0-l;@=*?0>=Gl>3mR=yU%xRzvugiPLr9
zMD)vje6BdL=$5hXkO!uH@6*pik`BfeJpKES9QzRs7FGL)V)rl4-XE#Yhi&sY_GlOc
zye8tBOigKyw=HfTe;r_-gYD0xiZk9-+9QruDT1tHmyUP&VAs$`vq=2ff@pfQJMqz{
z7D3Ye*^~QMLoe+oa{|11+IZRR_?1xk=SHC5$vNbj9&Voyf$bU31nIM$%&5b>BHbzK
z%GLs`#noUVL)3^?Pz#?J>(BatGvx@8515Za!$Kw~ST`xa#!m&g7yXpwp&VzVp97DB
Q<ki;y0j5-&GMvf)041Ut%K!iX

diff --git a/format/Arrow.png b/format/Arrow.png
index 361dc82ed5a453f25914675b95d428f9ba2f9b81..1b09aa2d8be4357c71064da7d972b1d7def9e0c4 100644
GIT binary patch
literal 112671
zcmeFZby!qs8wU)-00R<3Hw@hk(%p>;3aE66NOuk$(jYA@B7&4bNH@};f&z*(g3=w|
zGrGI)?z><7`+e8<?se_%Fem1m=RD7S-@mv|jJ~cKJ}xyb8X6kDhPv`~G&D>LG&GE3
z7zBKC^urGm{14szx|$;TbU#lp_yfm9-Pj!s?FtF%Uv#wWTuQLR`+EjP9!5Ial2*<S
z_$;iQEp7OG9=L$L(a@xQB*DiAHXarTp9hXk?vg$-OuzP!1fNk~^D`lS?c#A?hRH}r
zAED&zW`hvr6X6qJlEp<J5YleewvyMCRsT2~{3XL=@8RJh$<OcY?ak*c%;)T8$1f-$
zA;B*o#4jYo3-;i3_jU5H@ZoiGXa0SXKhL9V<8I}4&&A`Ovl9Y!UJFZSPY)R;Ce#o8
z_0R9?^tfmHpI>rv|Kqm64f3P5@C)(@@c(sgaHuruTS+D72QF?l?(X3HvZB(z_WZ|p
z|M{KY`|H~~dpLtpaJy%v;pAcC1`hVHKwX`z&>u(tU+?kX=hAb#X9Mo~*U^H19R2t2
z{@7odA9dq@8;IYd{OemV%(A%B{C_<%S={Ocp;<IEc{B}W1p^=SjmMq^hTSLCm^HUI
zy4rd=2sW_V-(q>)4fc8%8}C^)8#`NMG{q5%3@y0T-SNZeS`|~kh=uG5-=VC3&7nlo
zVcn&;-G{-77`qSIqPxRVKVN1ESaeuluprQJ)a4<>Nl56QAMJf04`}j`Xn+3w_f1R%
zacwLA-2K<)5H`pFEP&vDhx}_W|2Ja)n=t=aUjJJ!EQtTVgo{yKJmMsIZk(47K6`b9
z<5X&QeP)H@yu3GIdv>(FeLV`wGRlK~I%iw6I&BF#Xy#R2i3j^XCel~cx5Ibv<dR22
z4-pE*b*cKoL-q87^HN0?G*-2(<&8aujy)6lsVM^#6>mb2{`djJ|DTM#nH!@@giCi;
zxBL%hy_u6;7kY*yzgM&ehv8q1zK}%Sk_dH(MQ7ysd!5%3ryF&t@4wuyYPK8|`x20f
zS8n26G0dDOO?>6Jo5CzS_Kx4$ld-z}gwM{ckF7tyasF~rqOI_2?G4DwQz=j(aw4D(
zF>SNnU+5l6S@$LVY&qpKUH_flU@WKo+Y7yflmhZRzms7|BV$2afm8N)ta^bs^yKg-
z14-lM#cA<v@YKL{K_Me@muG4Wfuu8?QG}N}@Az4nf2Yzi8juUuOF1i}p(a}9Jydoq
zI6Y1I0XrJ%+<mYqr=(<&hqm8eH2oM99(nsMmOIaH{TWYj1dPA+Y$yG;Q;g!8$b@#?
zVv1It_nhx`4~<iONZ*8x7@c=z>%~bJHerIon;Exv4$mu9e@y5~ZMwbV^`5rL=q^wQ
z!>WAs!9;jSgd|ukk$%?Z8$ZF2I~nub3nefH@?$vRFN9)mT-js~K3<}D?af>NHS~s#
z==&M>ks;ZLs+JQjQlTeTbk^7#0;A}>GOZFM(v2)*o>#RrdjFi;OB0**xS_t=NhrtS
z(Q>?e+dKr*?_lG2?P)$alx6AQmZ`7J=Lc^dyAMk|&2`fp|Ki+w9(1<dn{*V8L!v8p
zzV{^EB=-7d!%7~whx<Ykau-JnC8@e5Ueh+mw2p7LdkE#uTwqTU($4A^5|knyM3CKF
zoqTp<hi~@Rv(T4?>@9#&CFBGAtLKBE&G#-p8KY<wd2gEfE+x=OeXBqE_K=gz-EC&@
zfXeKfxg|p^_2IK&(T{hIc0c>BJjSu~5dZ90=i0EGecuUOz4npo<>{A8w$(ZRpWQ>^
zZaEKL8{0fGbV=Qyaek{X3XbcvY4X}s&u?dP3IY#-U13D#aEA1TcDqdyPuIp=z^;y>
zJ6TwP$>p^rld90A40$S8+OZCmqz8w@PI2$T&rkQO=+^tRGvCP^OcirF*7mo6yFR-i
zS>|$jwmZ@~>-Bj%<Mz$Fo`y~uYl=5c`o9EU(v6|TfG9y8gYo$y>!o`?>0^=JI4a?H
zJasPpX3E6kNjeBN1jkLgFM)?LmsJcxX5Y4&z#JKOzWf?Ss5r2wtCF<nMnUl7TNK>d
zQYdvu_H@Hzp$}58JfjU4vhnlhJ7$8Si=1EH^`2E3&9cjjxzyyY6gl6USTOP!Q^^_U
zT1eD>mziU%0gAgaQ^-N&`_EU?D{jIj+sl{+U`B0)lB+PZ%Q)fTw&wRYZ`TY)F!#Z#
z?L|BBgas%Aclz05b~~={3k)-REcPX1U=wr+9xtZWX{y5C8Ef%3?<Gk6h|bw(4Fz*4
zA%|f>J@iHEh2Ko3VlJ5boNZl6rCOFhS}!l|cx`?n6C=hZ2BRcr9QQ4SK`1oMGTg)A
zgx|AbZRfTRPFj?F)|WK#XDusv{(ad_TVTK7@0D*{CSnBNb+;2zDfnTNvGr7BDXpC!
zezAL%dy(YpuSt<Tu2)o~QmTG;Oaal3%@E*X$$8NlhuAB^ruRVzyT@@BC8?|m_R=a1
z8a;-~EjT9;mCPi-!%g^Kjw*6sCVe8ibw>d=I>xm)hP>s<O0GLW2AV=;+$e$>hS`I?
zkNrbt^W_B0#5!4JbulasskKP(@vb+WpY@FCFoSW1u+(k}D6aKiK|hT#o}_}0a9DwO
zto>K;6fE}0N$s~4XyL~K_NQPm8O>R2uaBuQbTb5AgF-VaRUbam>hX1?l@~!4ga~qX
z+DnH?+$UB}L?gq%N}!YUXv;UFBo_{{F^u&H-BHxnR+&wMMH5n9d=o#IvZ`u6_*Cl8
zmc#Re^0AaRf$JGJh))HiQ2nr{gaN0UZ^n|SrM!5I9UIq5E!L#0Gv*Y=7=pm}gP#Q*
zs^|Q^3*J)O8i;Q%Q>eRZuPH}WdZN7?W-Cj_BlX(UZ;h3>B!j??e3V-j{L)cDk4mc_
zDgbzXFfJ@Rdm=`F*)~){hhGIx3p%~jIU^z&>-=8TO7CSF{vwATc35QNGG!4(6moKX
z8iPO_!Qv05_54?1L5E2WTG*8&o>qWR_AVJAX80_0SNPw$)bCaO3c6|eNTJXjVKFrv
z1Ag?LmWmaDJ54(S+=ALjCoZLtk~p*>L(P$|nS!ph)cJu2R2UQPVCAOW&{Ys=OVppV
zKRwudM*t~3jFU1Bu%e}kOtEJb(6xEarHKyW_#TO#G*RRyM{ZdE#bN8^`Nio^*Ir(%
z4exk~Qc~ugF-FwrDMR6TLPMou#ZQ^8l;rovjZ1OxU~aUW{_u#Q&^{X*qIc~g3Ov~Z
z%i4WE)jF0BBiKqVCg+7vA56S2efR!$%qhZU_sScnSTHKwPvsY*6Qydl(s8zJ_cEX+
zG8P7Q&3CqTbD>Bbd`Bi&gL0ujhQcmH6g`Mh<ZOwVn3)@)o-)RZx|B0$8c|w)gM?;5
zIgY@=(yB97MVZ`#mt(5!lDBvSbF^GXq<+3tFcay#-0CE>t62rpD`>Sa%IWR4s%q@z
zQLmi7h)r^AafcCW=hgQZgwjZw2Bj*~<|*rkc#s$jH|Rl?kg{z>oF=4S$jO!}HkE=(
zzxF2uLy?Vj#&b%+Yywo}@dN?4hFZKM^V^?o^z01M9OP)u?>a%d@{{t+>J{xZPGZBC
zZ&sHw%U0G=<FOcx54nhp@#a8NocjvvZW~fXBV`D+JJLGLd+Ye&agfwE^T8pI6{WTJ
zuj%=0zWO+&Og^q~^e`+>j*SM|GB3pP@P+YIH~B*KJcQ>C)&l03+3#GOmM@=5T`h;Y
zqGU8HvzgZO1DfD4DXjUgj*yGm*)&n6*L^!yB@b=T57^A@UE)PQm@kin=PwCL$MlbG
zt>g#y(4#AGeE9MDv(v^We6}Xpiz6%YgpR>{68P26j>3^S6?a^39aYv1ONzsdsyWmK
z3tm%)UXR_rbUw%j^D-obNZvoc@~C3GPSm^;Uks1@)Y6HQj#SuAQfaWFpe7=eEc?=e
zoBZ731z(!3(3AW{1lmp@NROxc-+NUByxh8nASK#rIvlrY6upO9c>QcWx&$6jYUR93
z4)Pbzu?16EfgSQZp7Lh&T0vrD4t326lMfR`LLoXq1fGhbW#}EsLU#don&P$J*aDYW
zKd~TmYsduhUL7rGd^JMoYbT{BHkB=D@;=ke!BI?ev9%Akorh|Sj9AU5v7b-D2q^t_
zl=jWDY|?RrAQfZ+em%nnhPUoU#G*^SaFnrm!__bJ(grWK1#O@a22oF!dBdz4hlrM`
zIa5g{V}L>*^0$a*wEKt?1aWr5h<?YJ5-7az;=9=;(?dsN2Tkk<L%GS~Oh!-_91OlN
z3*0y4n+{2wMB6;KZPE`X;yh=0eps>63i3FMx<x2m`{A5F|K#Hdd{7`{pw^`Y^jsi^
zIp}B}`zmZvgKf!6xsfIG4#sJyqZq6dTj^uG4Rn!hZ!Um23AQU5lTO;OR92kOQN)YM
zM!ifakk+OfOE2SRy;`wktXPanaY^zTFoQ}_l`Cu^CX8)4#@s4HhQmHD6rx=glhv+2
z)44j$a))h1v@MEp859Y0gE#BAB0iuke1fMIva7lDzKC5C5;4cI{2Cqjyk5JDgM0`g
zjbquQFw>_QK&szBr=|`kiQG_V!=;F<@1(z_I99?c@`Q4J?$fyz-TgOP>31MD%=*h>
ztn|>fk_^f5pV!b{&se=od}mGvXAr@x*Ip~o$Az6fjUG&txiKGy@nhh9MCoBFrMrbU
zC3f%VEkc^Bjt_ZXBa8=GlMr%o9pN|bjnSFQ-4g#qerZJa8mSXEwa1>#1zVy1n)=;~
z?O?v*&W@$K!q2(p94_H&@8X)wJ1AJDDpan9lu|0p-@#D4+|9pqnf%eUs^iZpUKi=k
zuuZr^U0{!FVv>tBNDde>K#kZdB73C}MX^mY5PWQ)ls=+g{k!n;1JUhitJ@we_TXTa
zW!841b>6bL<$%|{DC&o;ES~^hf`kjT*$nC|)}M>cgt9?C-=LcO@p3}<NC0Xl2id_S
zPRv^7QXPCrP8WZy{N7T_czJ-h>_Uw7Ntj|<8#$NbEA!7}o8{_4w<xlo=P#mtlKN2}
z#+Pg?IbgOCSir0z7E+0=fr)Kh;gLrb=s4(+l3?r1&QD6tAuUB6Dvn^Lv%SbHw77-I
z`-rHE)spn;GaE|HzJ!~)`Rl<&0~(!g!QxeA8;y3)QFU_uZi)*@;L=ohBvSJ_{s8@*
zM{Z8HKpOt@xbkb<=4MJUZWSrkfkovdj6AB3NVaqeLo%D|Xs+P!un~SVxzDSNLF>?V
z0DUbCeNe^a+*Qt%r)OX&8WoX@gmS3^^np2_<M|lwVsNc7Qd#NIF6s0}Y#dti&DTF`
zGhR(&J|BNnc*c3E{0Ic}4@~G^(ncMosLjKSrcROT7meRv_|HdByf(5aAkPoCd^F8o
zZp5Oa1`0J#fHWMMF?oZRaKy_wJZa3+a++~mgB70DJ@+}0y+-B?xuQ=34Y-jq6wZZy
zU?w9j&|jkeM$e$ID~=x>XObJ;2D<;L_QUf4D6L^yqNmm3BFTHoO-J6_T5V!aUO}5L
ze?kL_E`~d~!6SDCD0Zg&w)=lGTp;EUgH*&~zMA$Q4)Hgr!VwVwzkok;bhFtS`ss~x
z@R$n4E7@ZdXnL=v8>Oj%yK692s~e-=g+ec`FFe(^Kq@VE1c2d&!kRl*JE&0&MtN1!
z|8e30RWO{>`Z>u^hdg<Zu}%&)Z|}aZ{sr{@0>SbGU})TsW4HhLH8|0zIk;L8qp;hv
zZbX!FL1#p2v^ArOSB5~@4=R{h_WS^1cY0x4?L>N+Ei;#POcc5T$yx|2@bdi7|3||L
z<O+|#gtqV|cr3Zc6rlvNM_)lj&#T(0@C}>JLx23K@!@0dxrX)f=Wf&=*m4|qK>m#q
zXV3HbrosN$1DvGl^(P16Vetj(`xQjqaQOW3O8(Cel{BAOCB1)~fpY#wO)vc_f&TI%
z0IT*^$IegoJGTM8j4A<e=Mku|>^|Llx;a&|V3To|TpF8zRy`LWUYR?kXU&_hKYzI6
zt}{^Gfy<riyPQSBrVUHx9G}pW3*4yb<$Jr;;XWwX`=jY+-pr)DYFa?LvF)_wLtfK>
zZIvBAaAh~#1l9jBV122e;F4CL(t$(}i39wNO-LUw)Agk<nK^4h=#o7K;Beu=dn4it
zam9(U*;xWNw&5a9GxdCT8wGgPm|VcD4LX>%yM|j0AnR<v?ofWv$vqBel)0Jbd&9?)
zo|^m<;3i}6ECJrH2Ur-D>MCD<T7UL5?_0X}m%xl?vJE7kp6cZTtWC>_rHCI2Cx>5V
zm<35(p6zZc%L|ydq3zgECw@iV^Nk{)a~TBW0ps__UZ3vmzrXcV;2}?KA38Rz04OWZ
zh#u%K@wXf?b@lI4q~gyE$IrBksnReuO!@z8R<a&%cSC22xuGXRW{gt=9iR#w24@0A
zQIa-5FW0$VEv4U1`3RC%Iyc2R>)$E!5D2W=WknA{Y>$sV_a^RhmLiH4;0eUOB$Pce
zFMn1l2{2Z}S+CrM;Lo?a7Xa7@+JoEsG20}T0WdX*20nwVG?Y=uT!M~)6dzz_wGZoA
z22lk7nINI9r$40ljC<@jWsle|-LwQAQ_6UQx_SS@?ZkIriuuNJminwb_EsF_eKVuT
z_p|wsK7-U5B=KVaew5%v&`@28r8JiqO-Ly6YYRxJIFg4m?wzdD*OCEHkJ3`cV+cC3
zs0fONU(mbtG4?6Dph`u1v5mSBS;4&&ExvZxH-O&4M?eJ~LyEvE^a^>R#HT@WPo<4a
zk;qy<?xU$cuj&g7Y|-*C6C+HVv}~la<}(~I6A*ghKYaUe3qstdA}<4JfnBiZ2S7mR
zUf7$^V;Hyj*|wbHd@*H}=z4s#kXU!$96*qV3U`~lx>L2tk26dho*CXWW`*j=9xwWD
zzPjZyBG8F<+qu~@vu0tUH8}W(WdfaI&iN@ooM1|cIij2E$ev~9rmBn@%L}55Ubrw{
zh(`~8iK6$5PGMGnDU{J8a$rAfFNqnCZYb`YoP4ip$hFPB2T$%=7xQP9qUOLMj-rfx
zTL3O#;|5H&57Tlqmiq-|teABlJe<g$ysIEj#$WZ#uZs6Cp`_pu^wX=IAm<DuG@BJ!
z8B!w0b+VCbM@1`hCN4opRByfh@D!|xffjDIj=ZyYvAIlq*v3e2S{sNnBw^W}1#HMm
z+nfjKM4GrqgP_W!y67Xy3cAq&RxLXg<5T&3Ia6Dy+KK*zi3-9-vLmZ1#>gTmfiD5O
zn8DJioW*AH+go_HbSbP|6-wF)zsq1y{{FhWQB4s0OmId&vyMumUxlz>)LRS7X?^Ed
zjOsq19I(lZ5<U(FIImbP!I}x6GT9reO+Q=w*Pdjs!Mek(2OSS{-G^h?mkEU?@ryV^
zHg0p!lk8VpGAM#IH&UX}y>Z)QgG3o^@^LeUAQ4|`QOGI--9}!U4o<O`65O{7_KUXr
z^IA}xjZ2ln{Z-#8s2g$Hq#2%7>5vm-^3<5nEa-SCRl$gX1@`<xd6_>4Qe>eRP;Ev@
zGH>d!G40f*icx{pwTULE7#X%P3XxYut}>oY*`o@yZ&^gbrmr7-PXER#yu=Z17JM$M
z%25>dE%`1n<}%HgC5*$(hV+MKI-I(J={rbte_aeZ4yh(c9;oye@)j0HDfkVAjZV>e
zcPw7yXI)xE(e*rL&vk)FT@i0c!gLvg);`Wd5F9f{+i$FY@%sEgLO+SjlZVtTT|r3r
zH4Pq}WSTIhL?U7>v9Y~4_!6JyD&7+eU3~LSIU$(W#coC_L;xCH!RYH8c?t+MDiQn9
zYie{VRGC3P0okF^j4{Xo3cec<tKNj6dO_cqB}{Alc2di6xi#Bcn4T*<7$x*m{EK+7
zVSRb|HrL30dn%^`Hb+oFlM=(`Co$F^5%p^Q{od3Ix}h`oM=d?(I6_1q>gbsxR+UNW
z+8-ri%ux*C5N&4a8B7L*q53<HHQgW|#js}FAr><^)P9t8S!XhQ;2$b3igbtL;L0W7
zSf`9+h31wVWtdF_FK^~JHQpPKA{Fo60*Ppa7F`#6(Oy0d=2SOybqZ5k^_`a$Rq_HE
zpd&?Udm<t12)8oyQ)Wf&fyl2I&tnaEPv=&{)I%3=;O@%#wsg6D(b~{<>=i@-dB|Eg
zpcp0ZhAT*-I1`(%2I_9}xe<W$L>I<4(r~<bMTQkn<cl{pj1+WAr$;~dGyUvDz`niy
zz;1*4pS3s$y=Y~v7Fu!eA}(WxXx|T0Y2MHbyf|Kg4guzF2+K8yQ5Cl~Ec4v$1VIC>
z4KvEOqaNAu_tNKM@RBSjN5&y;g@Xl*L(A7T`B(=|nm8@hU4QWDzHzgjDtD1TAnR$<
zqNx<M8MTFsnV9|BIv~kPI6oV|^9_3_!xU@Sef=wy)qRT`p5Pjq>e@gJ=j46KN5ok5
z^i|)bBJ~XN3gXy6xg~>sNIe-ya=TrVxNSYC4gwia)(8I$&BU;S!QdiWR)?}&7@)$i
zQ!wm6Q^<orQe*&6YMiTf9f?HJ`XBxn+RW7A;ph!zvU-FT5fK1|GEVD0maHKQt-YqW
zhzWRydCERNVmPVtNZ(&$17ZRPAE?5*m+m<bbKsz}ZNdl|J->U+xc1h#7qa+jYm>U;
z&{83_Lv-*_y72x-vqY!h6Jv}g+}d6PFjDb7z(G4(lVIPb7jS4}i@3s9kC3trJOc2Q
zAv64IG_Irkq*{D=IXOokPvdG4?Mh7#_5Erw_VoA#m8rr`tHe(Pr%rJ72t)$E%?(0w
z$*Iq^fU%JS^x$^-d}ow%{d^>fvW8Su$8%uK*MmzSYTm|muI*=Y4a2}v9+5qD+}WIK
zJ_UtgPF$4~F;X~Xz$NAW-Xw~T`Hkdo7AKSW?!92+3-M#p$J$grxFWzfAFa&BYa<^e
zimuHoB)XKVazK~H*c)1FZr2Vtp<YH9pp^tlY2mb_9{&4k1bU=8$oNQ*vXRBC0yyo~
zkBn{hDO%Vx`%E!5kOJ>}g<v#B)lEbM=<~)MDIBkbn>^M!=zmox_oQ!1XC7F!n#ZuF
zx3R&<ML^tmmnmqqwb+x)to8zMaIi|4Vy+roC$dAUTFqtY(Jf2E@VILAP%e0r6}|6b
zvggOUZvbIWiZ%}^S8dvSonsklo1gp9ESOH1)yVm7z;<{0yV9rDi3X7^Z>F!W8CQXt
zeEpS8Wd<M>1<-G*p8-Hy$=G)LWA^9fTM#O!Pm$4sWzv|EBaCx`o8=M&%*?eJyohQr
zvc(}C>mK!Dx$PkyNUhM!tzHp{Sqx6JNEH{vGv9@HF-16g%(!+g?*OYN_Mkb@-3Y`a
zeCjNe6#0%J_^c-I^dR>dap=**Rg32R51k<=Xe7iXgo5g7d=QSw8Bnd=e(;KHoN7@D
z#s5b29W*yYuYD1M$X?@JSiTGJ<2g|o1Gm}U#xKvoBsRJTOo%J=yw5Jrmo3BhVO*an
z8_w2}t|^w7U2s7Z@eJTBdqS?GNnwgNxT9i&sDlB_vKK#vdYE;-4Du3V%i3$;+f{$l
z(45x3E8^3n7*zA*9{5+M%rP7t+>SwQkmT5#KxY-Y=(#thW|fhqZzAmLqERn%rV^0!
z*3n3;NHFirdVh@bYg8s$pGNR+r2&+7qz^&KYnd!Uhnik0%t%Q|J3}E(6%tvN0qc7J
zB`jKN9w141bFg$dJ`Z3Oh35xf9D^A4?W<mp#?@TB4%Q|tIKH_8+B*m!@3o_mrD$^6
zRxAt5_vO3M@~e|&Z+{Dug%xh3DQGbIT6u_Ze~d1SUzX?F1wxnweZ(e>{e$W?<Pahr
ztNldtcS8RU-~n(o0LOgB37`V10~mqa*;c?$G1G^z>ejF?{p|U1Tna+yP!Sl6Kynq~
zh%bHgHOvT0;W}~}@~v7XPX?CBsNO^906Bt&5bQv@aQk7DvWr3=``P+4@qa9{P;3y*
z_<gTzr$;}~Og(^6(<ZzJ#SIEkrA5m(PoHoRv86r&U+CLHDpUY_`Oc_r0WsJfhRu!O
zzz7yO%Q_CDTb_?<&|ky^Y8MVU(@iiT0KJQbj?NmBe$araRph7|-EEbDp1n1nLWkc@
z{mpD`sJ#d3sJDEbAv$Ii2GysM=MZq55->$i2sD)c8urH;=fD5~%m(R5j*4m~ifB!K
z00JCSm<6=sxbQZ7@yBI%0kEdpVaODtx_F>{Iu)c>sfSY)$HfF&>(8s)3b3wT5r2w&
z*_n>MaYbPF=lkarw}}D6T?vo^XE~Y6b<PZRa{Ayi*R(4~{(vKOt1er{+HuSxVpc^d
z6q!DK!YyNeLQ8uYa50s3_P^7QK0CA#s1`tM1SPfIi-u)IS7iSgkR!Ecf#^|h9*GhF
zttZgc7|n-WisP?Ri#gFAfU&rNd1P*hwh%#^0Cz<!P5UgwPpc%RSpd$yV-Om}*`rVQ
zR|GHk1baCqaQ&5N%+fOn$RJZdqQi^#<qd7J*WQmxa$uT^qTR-*5qP@{C~TtFIHAo-
zJeoihyL#_5NuTd&mTk6=mPfT+2o%tq7puj^`+yEG85t!_0Z}AH#6wFS60hCXpu_y8
z#gm@^U5%`J-kvL%Xe;LAVfZC$l4_e6YkY!51j$wmS8kR!SP}agZOvHw(=R?~{sSnY
z0KYy57?}uS;g(;M=5IxpdVwqs2Xono*Q=X<-PIqpDk4M?Y~o1E-2Mlw`W4&vH9)Nv
z`1<)Plz#Ge(Dk=6?jOY+>L<(eK=Hd?cenD<e;=b>pbs|Tvf2jxqiFh-Ljs9`<etCj
zQ2AfK1I`$M+QcEa07mM+E`A*aj!ztK-DmyRRdA$%O^Pv8X8(N*i+(<!A13XZOaQ0y
zUuTpr1e-`T2ao>icfhJYVF6mS?C19;zsSpf3<oYN*radS^T%5K+es(c!Sndvqe6}5
z{~py}bLRhcRTbPV6r%)_whu9C`;EVILJQ|*CK{o^!=L-AI3fhVs?u}s`_ANK%Nyz3
zAr-LLf*=prv@`Y3pS%qJ+oX*kJ_4LpRnw0~Ab?w)tUcx5GyT1O<gJJS8nk-d^y*Je
zuiyfQ78F51mED?;Vlcfld=>fYO!_R4bs|!nlK#IYbO;wS-~oiS0c~PWvh}>S$e8xm
z5-Ph6_B6QHX~gr_iW;ScSoOpyh`0dbL_D1ox%d5)9n)W1?P<ZzBD%M4-}rN9>gYC;
z`<0?@i+x{$P9LDSnVE*&(LyDfvYnfTzm~6wro2pOddjZspNFvMcSG|)q5k4-)Acr|
z*+#(Oyp7dz_?;pOVqqZG74=ke{52v498i?+01!jF+4o;CiY8obi`Tn${VLI~6P7tc
zeu$R+Jb2~x=K&&2A(-Sw$h%k|%h{Bj0XQ!4-rz>y(R@^VUx(zcKqSwIiWQ3Q>i;}O
zy?`8sG8O@D@UdkqXHPuM#}{{{;wXwVfzLDCgrmvk7d;!INo;?se{kq;yl4Rwsw=ZU
ziH?Dlv=vRneAC#5jJEx%0#2=v#_#Ln217P!=x##w*Zn84AQeN_F*Fd3zoItcG4kq_
ziDOExf5kz2QV?>fpFF92`mbmV4iXCWkd6GK$oX5~rH%kwRPWvU&iSwNsk0ZvAs#0d
z{&m-XhR_gIlw0ST$tS;mpKlyw81)savVUDd!8P!x?9<X;=l%P99^k^q|94COuYXFY
z_>)B6sSgwiL6pz#!$qNB&Ya1g*;?Nf^5D};D!HJ5?uQoW)JgJ9SQT}KAo%PsB9WlH
zkS&Nq*|rL@aRi2{g{zjaY8?S-iOdkYm2^SO!3hw18fjI`pJ)3|5#kUD=4kdkt}FkZ
z^WZ2OC~NoqZ~vZD*{Dfsn>`Tv?@6kOBBnI9v+MsoOKnjBC9AZy;NK@qK@}(0-?H)j
zYo(M?feGwT`i$rQWZIX2*zv#H`aNX--+o&1IXEP2liC)%WzY-Vf(M^p%eDopeKQ<_
z+fS4Z9c?e%FRdpxfV@|0)05wSrSTx6&#3|T^T@}yxHM04Rdy<Q*f*_BH&ZAYS@my2
z;czg`D%t8@CZpvyuYLg7>m?GIk0h7hO(%tTz2kJ@B3ry~%DH2hQsr#(gu0^{>k*T4
z%fIL3I<UMKG=AzF|0{F<H95Idz$Aatm-E}r^Y`rqji^cf;-FgO-zU^W`C&PCmQntt
zKjZ!Hk7aydl9xZ1;feqEs<{r8KCQmKICKlr&4TiY0Wqa17_e3-Z&@#`JE`!!iGSh>
z>PVy?6n|p!=1X0iGawS20+tp?AuT|<s(d-F36k&%7LDW9xE8-oZ6jbOmM5oyBjkRH
zVEpf}0H?g_NI7eJ_6nGNfzUY%*!>Hjt1HXywJD?gSjBpLy}+&NhH|>j;S=q&3f>A;
z*^}EFRk9KdvjS4w+W3=qC>c$UuUIXy)B$)s^Q^5~6iP-8?NL7xa-F0vXyfy%`)?+%
zYuyE2SD@~3c>dsA19i*hTei*5Z`gHJ`NJ<!_EBJ@I9!`x&sSW&bQ%|p@RmEc#>pyD
zc7Yn?)~l~n<5kQ7r_u~0w^4-oUXL?MNdhiDv1#j6J-PFjHS$KcQDeQt+7^8Vl%};W
zt(ODbRWktQCP@&ZJeA(109jfa6G(5h0nzm8aN};PYYHcYU`HBBu)0zUFv<d6rxb9O
z&VVaonP>$9TG=N+QAqb^swT;k5yKK?Afc#&2=I+7hBgI;50*-N(o$OiyZsrkXS(9^
z;ee;}PKnr?&1GR?hJxn|EYA9!P?QHG@~Oes+IM{Ze&3wCqKWzfQN9sSzD&Aw^E85q
z5W>ag0*D#`LzMQ2lH;gsGc9K7U88{91yHbVvhnsQ(4GAE*A4BrZwFWvQY|qAgDUm%
zXBUH|eJS}&A6ZK1*$shGs6&E$z((-px%b>QaO-7p-3i=(|Ndq&N}M}P_TJrm^Ci!@
z<(Oj*SX8b(h<GXeN(;>I&DW1vGuo`<lzS5y2w@^akc6cSvkSA!6N+z(TkUPTfGqkH
zcsNU8Y@JMHm+K1LNcF&pEYw!G-p~fQg7QCs(vsZ^-;(2cB;W=!suqA6<pT=S$T*az
zwxD_osAshWBHt_OikJtIw!;WzSI3I6wd{Z;WC~ww22KKG;#)>Kme|m7r`VTZb|fp}
zz5iujBgYxFsC{St>itb6iDO`Jdx{IBb`sovaRR9g0m**us<%Ru&!_2+zy-4n10`HB
z7qEteVsutL@hG*1-hV^kV|>ab&_+qzaPIwRKAf8gg>;~tldaif{wwRA*Z3Crf=b>3
zJzS8;1ZZ~+%2$a}p5R)CkCa_24%It*NZ$^Y1v#G`@Z5gwdo%_pB2OUUk*q{em?B<n
zjonLE(q`yQVpP)a{mQKPY-gG;<L#|CP)!9S8O7W-N=54$E5<PvV3%7fNlm}&Wvr?O
zl{wq$?5{Acn6@vDpCT0i66a6gBG!14bH~}E=V8oXve*^6CqM#!{D8nH)6%i$#ke&e
zm6IRfYmFcy)l}%cQ#Dp=v|IUr=~<H><+VY_xwQSFvtv2aXZwDwG$(<PcA+_Lqf81z
zVjX3<y;&%$f`!ljP!-AEnGbbHL#tAVPk(FFy?mRDWhPZmRsuu(<Oj;XlqLR!<)tZl
zH>u2eOD(N3t?k7TKTzP$mb@-@tT=gQ=&_oP22pCD65B7A@(FmX9jq7=&q}ko>W_JN
zQRdUP+X~%LW583(9255ynMfiV&`g;U3Hb^$7CADj77<5(&L47<nB!7Q#A8)oXsNdg
zVeAt*>|>dUHUbtFgBXU#OoQK7FzIFYvbyljS5TU|tJSLiv}_MOv}n6&tK*7NDQB~{
zX4Sjb(MiLA5~HBaac+?ySe9U%2Keov7E3FSE=nD)fBM+Vr1bllk&cRa2w@7*-pA<t
zlrgU76~uo2%j^cJ{JMMuCGN{vb}2lb)()ht$|#{ga<0%|=}b#wAnUR3l2So#JTbk0
zmtB8o&HH?zjp|MUpIRg0xf?wOZ#j>Icjmiswq3*RG7)DuHX(~35Hh9RLVTood~V5f
zN9m|<^UOzd722VmXsy9@pvaN5<7A1qRQ~)0lT_>;kdh-(P)61nt$=Vyp||D52$TLn
zlSY85H%VPKkICC*zB8+KN{$}y)XY6B#*Cda*4`a!Fb$xKdFWm5Oqk4j74Fw4lRSbc
z@+bP_kUbqu0)SK?FMu)gbFt_fpmV!6?2T)YP^)GI1PO3=&xH#(D2HpqGw?oUu3LRV
zYRTRT@UKtK3Qi*upthx@Vwr%}Ty#oOe(6Av%8P~25Q&5fGS%Yog&k#ya?D&H*+Ab$
zfhcs~lDK=!H!`l^_8TYC(i=~84cH~#>#c>v^m8$I3yu=PQ^bngRN#+~-+T$u;%LOW
zBjj1=zAW_QP<PF9Av)hFi**2nn}KS>V%&Hru=xdq;(8Pl-_N9-Uax(T1letdJ<;ip
zhP7KW=Rbl=L?|T@CB2ha$qi$|@cO8jpdHlF44%)2rNQC1JB<3nsyK-<?rw9CAfB(@
z8&k;;fMGuzgk`f>aqU@BPUQk$MfeUA^SGt*WT6r^A0c3w`C{56Zrrha6y{L+c5Iq|
z8FXimW8HmAI&%u<UzBf9WE`?9Rc@}fZ5Nmdtc?2Z)6d;JNrFEZEl^PI^AQ{S!bnPl
zc1jAfxm1kSrm}1|3bpj;^&wQiC<KPsfW9*6;QktEsw^!m^aeY$?b>wMlxz40OPMHL
zx#&ivmEk>E*lXj?c5Ev5=zgzhpi<)S>RZoSq*0N#z-%I7GkGq)5|lr!ps?_gI@4EV
z-s@vdj1z)P-cia{*cyaiz2gMrr>;dM8oo}m7MBe*7_NVSRf+=s_sHv=A7*2<0B{G-
zXeour5o9q^ThNwJVA}sINRv=|kd#Ux)A5e?tS36@0YPZwCD0ke^fZ6cecKgl`WVlA
zA+U`gNX;!Mk<TBgV5<kgg)_9nI;LA=@5yoDWp@JBdbBNi=vt;$9o2l6Afd)QFwzbV
zR5$g+k9~5wL!`58=-_>vUv~Xt8NoQeU9ku^bupS6t*bJEYR<#Mp<|STivW*7Hdlqz
zQs~4olay(pKl<S;_MKl3!Im<t<{R;cTlF$!Ubdj+go@O3#D@pw7N6vKbo!3J74gaZ
zCmQC$Ay6FV^S7Po1NX+DHF8XI(W^<8fH^;{F;8O;e*t<)l;`a711%SY#QSRS8Zv=h
zkWGg&7$`Y9ByHJfojgIgdhS3q$_qT0#(P~|f-=c~>>Opl8%wC~|9!g7uC_KelafyZ
zip{`sVTs6SHEPFMcVL7B6X~jX&m54K%(^lwO(EsYl(a=ClEl7%=wmTO!7?g{`HTEs
z4?}BaCBqS(H)3v_&^TxlbQ^DvR}b38QH0V~>=Xu#yp*#hWfJ@BzcTUNt<O>&od_*8
zn3O8>AXSz-zaqm@0TI5F@kCv6%SEFqfq?qzT-J*%*M9b57b{uXBlQSp?5287GNI%x
z!C2NY#*kW~uQfoPyf&=H<qe!PZl-92az!ELW@z(tt=7Jjbb>!RuoY;8kKy5#a&j|k
z&LZoIj6nf{#xHxLZl!oLLNSSyTzXpv4;B!cT_k$S&q5{)N53i8k5-w<^@MXQR3pOg
z@Xw`e!a|Yank<k&#c1+G>A1T(qj6iagFQwWvVgR%HLf%$hRa9cYYA5@T16ry>+J9t
zCPj_k_p*_53$K<IDG%_&+vD(=9?V#1FkrhX>hCxk>2xG1jGrKgLW6~D8&82#=NOpR
zht^5VZnUJ8z36t<JBj~Av6PWPD#&Y%)b1@fm~!~~5z?w%PQPDNRXw(c^}fgvZ-2ZI
zp0nA!yXM(Bx1977DEZ-WNKDldjH7BEs=)#44BX(!t|TI9kX^%<Q?IGBJXVS(u^yVv
zT4xT~A^AFscLK2<s>R)P2m11rB%*MNq>n5cGjTRbp=|{ZwXn>!-;J@g#VaS0QHUTZ
zFBf-Jdnts=?3hU6*3=%xjInpN7Z6aP8(V!3rw!W=Ur3O00>z9xRy!9#tRmH$a4p<O
z=IQs<r^w3UjojGnh3{F<-J=CPCrLs|qo#lbTfZ0z>SrpCetn!KAloaE83p+Q2bims
zF|t*Gl`O^$X7geg+X^_`^r-TBnyMS(DQViT%Bi*il~v}sHsfRI%UX(r%(5r0)sSUh
zPP;rYY8^oR$%~Sh4{rhaJ3Jwo`L2=G%2&$94{<^(6BuGrCw8A98kqj*;=nuyny=u*
zgMF;N#`ab3EX4M92jP4Ti^Ss!)=ApDYs)+|$~Z^R&By?uSi^TwN<#dz#AU>g!2=0W
zWJma`^DUO$A5n#4AeF_^%Rc9<crQ$5XWgBIr^QgX)kj3>8i8)4;Or4S*luV9hiS}$
z(vef%XG}a^RN1^TT0^!2qD^V;WeslE?x)ggBw}IK`jEp7{kr+rC@P}Z$CETV{htaX
z*$U_t=)_(aJ`aWfo6{0WHy3$6lgGS1291oLhq&H%V3#hs2w<F&>-ivLu&kf21W#B2
z*14nvtSm9MdV#y0&*ao6u4OEphJ@#wUs)5g4Uas3j^{4%5xI%g-k50OK$Z3u5R489
zXs1D`HmZ7g5?)5hS|?8r+bT4QRv&6^EhNe%$h!B$XU0AGZhJ6HdjaGzMB1zN#c!rU
zK^_#JzSjz<*KX)9ycA|>$jzfk?P!l;hNTgO3Veqtw14ySQ?+Oyd-z@Zn&1BVqhZFZ
zVkaB}nvktB_Z=9fm9;0hd*-?Y9c#Y$_juNJ>NrG1JDnzsoh6{%iOiVBwfZt~v&{to
ztCD{qCRx7M8=ERFiIS<U(RvCoDyozll;Xm|vI2~z5}D9?!i`dJF>P-{y-Fz??t-d!
zp6>vA9^@)c`eHT7B2>YKf$gEpBPv+kh5P8cq{gTC)zUAA>><>wZ8W!wS@|t%1l@=1
zMK>1l?owX2t|0E=!NQ}2d-pS~+fGm`$tZ@2=g!}Yrs~30hGpPsON2^sWNs%b#p7#0
z6~=5W#Ao681%<JAQZs#O4=tgmjR8_&I_6?GACv$2E`Wg1$KPk0yg3-J$H5ztFGGux
zpF7X8j`~5Cf@6w?Iz}JW^z_M^E>9J!#>c+b_{7sCe2Y*U-nm_1pzW|qW;fGU6!=n2
zo>1>;EVX+d>#t&>7*mM0=B0ROrRYGL2-}Grjk6M@7A&O0pI_v<G(i0Ti;&o0)3=Nb
z)#D=w2#Z#YxBQ^M)n3AJk~XBEF_4}?wWRT@x;u@lBMg%y<d<5ZxNgg0wmsbwfMv%}
z4PjXC7{db2D#e*g20Xc5hh@LwoIpXU`-QFW10Hh?u3X0yi#ma~W@k@=%~`QNhr{;!
z5;QIjK07lg3fj1R4Z)7gTMWIzhY&fPRVGK?4NIc%u2Bxh#SpB!LcSL_iWmQgD=duZ
z5hF!iAKNHa)fVOPEr_TZ8hSaQi@j(%a&_rgzs5CTM8oTbTM>F8>`L}*M>d_@QT1|S
zc#cS_Q1qSToPD*o@s;7`F5E#O+40l%d9uwY596CvsKYAjz<Z2coMOAuC~K2(9tJO;
z8e~5_UwlZl?_jX?BP&~f(G@ydQf(Z#pdEYWJG&IBZHta18vjD8mHdZFFL(wv%#`#D
zvic1Kn${gjKV0(>ONLmdT2Mva!eJg#hjRp|hKJ7COJYMw-d;?vS+nE>_Ctt6v3i`Z
z*(S%Oo;z7q;of*waZ&(Jjx%=k_LM~<>%ATc5%1>0u&(1ELzhapo-8^$PO`PeD`Vwe
z<_>95u5>j2ur5)EaAaz8!F#&+TBzNmb(Xyx_x7mjvkp~GyW{}00S&3Fj04zj3HUeX
z5)z3(2`nz_yx7yVG01o<{A+gHzd3kcxPBsnP+j%eO$aW<^H(-JA4_nK_H?t81zD0b
z+cG%F5<@Ui8Zo8unsfCv&5`<Y_bv1?gI%8ki~lFOr%VMhwbSim$eTVR^iQ5u1AzYD
z*T>6P-Ht^w1CTgx!-W6eqyhl;9Z+;rwn=BL0d@!gr2e2%E9U8p#J6G~RXa3PYry5}
zev)CTc`OI)%NtUFxtIe?4*L-ywMg`CkX7!YO(9e<!J0aV)_rL03#u<loRy;_7`Oml
z0MbX0|5qsztB#nivtgl;I>5UG2gsQ2n9hRq;0r5xyIjfB2E3`F=55GK*i%_E0D8BD
z^wea#7gpPWt@0EYEQf`4iZwvLmh{Jhv%#SnTbTK#FTlU^NuRShUFR@ofdiZTwqp&v
z`Bs(b;Aj4Z#S~lqUEQ5vBIkxB@=@1wKt#0y^pW{)*`1OUKiw?ISGMu;VYwD6Lv6!Y
z{9k{2;-CpWE5UuhN*M;NpTOu`B>HOTDTShBHSmPX0uS=tMvr?srgPjO<0v*w@^$H*
zT`0L(z%~GR&zGN%-NDrKBe-fpi6UM4UIRP=oTfv721o#{Nf;=5sC)Mqa5*Q*6T#A6
z+Hor$Ybyfz&l_@}xIfnse_ig7T0~PzOc&)Y90r|3o&X5WS*PmIgMv_h8_*-$pxxJ!
z(;qMoIj%iaAC0WyOT5mv<~zk^Tsi@h^h_y=06ZE~>wVxata`gMfHI12DJ)e0Pz1d8
zLp*hZp=g=UoVVCT!`piS4*~o=aSk#QUs3y|C=6yLO7HZkP~0Vb^%8`9(PhP2V5IsC
z*t>)a$h%YZS4!6#&QMPIUkylb-jP-Sa*qL>AZ|NI`GXFmz}BF1mo|1^jeX(k{=gfH
zYO$WS&B6CXf*$J!pmm>$D7(2UmvIo)-mKWJWn=a79bZ!%c?i4PA@HFj&wv)*bIS5x
zoz@Ca#SwC+{`WN`n%+9jyYFwAJz>`+oCcg8ssXtjlVS!K_gjE}1|Gciwf#Qui~y!(
zyxITI*zWNw;05)t5qt&^JN-0AqXD29gSOhS4*|e!2|WGf3XvzmS)@)c!5i#rp02_6
zE2g;P(vHH1D=xtA7^MiT9|!rDXY5tJJ*#|A2xFS{UnH|PcyE4sv<HYsGb}lk5{v7^
z985r?Pzk?x{SiCLktUSV89oQPT-!zbpI5#HM95Q)58$D{6FFKfPGozjmoJNQIROK#
zSkReIsA>Xj!;i-A7r+4+Km*zz%w_Swx^ph;2V9#rN2)L1HI6BO1CwM?@aQS%ZwI+Q
zX!4p`ggr<}mN_!Q56-gCP<C2t!G1&w{QUG#O}ouHXKh=R=D{A=*Np;0b07fpIn26r
z;#c%A$C-!SiIw{zjB8xxKzkbv2|SuLJU2YTnZ*HP(z3|P(5?S)i=qjyJ*^E)0p(``
zb}n5+YAnpwO(>f)7~CDu?S?5ujvI}Vm|H+zV`TFL#3jZpvp~-Z%26YAhBDY2>!-zu
zDDp4>7M>TM$w?<wILI9>|0X{L^&?az>42)k`a%G0A3O#a$=~T1bWy0rQ7Slt<*GH#
zz*pPHgiY&$#sCRTlHEl=0ZZp7ZR=5?oSnZrDf&}uyCjr9CM3;5W1x-a;}iy2A83ik
zjZiavKwBsPsM2KscL(q12{vl2DOvKT-Cu0zBG#(rQ#pYNg%*#-(wIS(5N(<TayXM5
z=yOTd{aIFZ9!j_9QT+hel7W1n8DNF8H1ivXZa4~#nJcC(e0PLIZS>G@Ou&^}6Q9VG
zVBH}Ac4hm$oqCNBD!1rPGu7~Syl?K*TSw%^IN@2&a`p_8g;QFBM^AwHRdxB8XPDWf
zXJ{~Y+g?5X{0uzK15kKod>b)OixJDdGM~Qq#oQ7(e{Wp4ekIF}{^~5ISSVpR$pQ<%
zFiby_5+A*HF;y?20G%Oyvc8MRO?hT`fJSZ|);4tYGtZo7A7hlj@Nx=-xo{O!GI*BK
zI@T<(v8OF`2CrMV?KfHU9U#;E*EXiFOL}Q`+nP5HffqyAqzOzfSU8fg^0Ra^4g=mp
zK8n1t0-+`gAc+z@pFR|x<JNx{c<%7-dczd2>M-h|njhA_aIq(yD;xTT&=mSi#}k{m
zW7#%DvNouityvM8I$#MTJG@YiPI^wr$YK2U4zu0~!+OM`;PMk@ijWO##4YXfhg$Of
zP-*{*_3$=e8AU3}I^<i(M41TE*|gBz49Xm@qd*vt1=A|qp69BZb`nVOP>)-wL>m?q
zNP&iYKy5~8Y3~(l#zqEzetmhPQ-Kf8Tm?E^)RPx%>J$=X4k$x$aNGF17WsM}gD@|*
z`{2WS6w&5neIG-!l2MT__&|0LFxY}k;}RUgZhL*{W5UoW+>lpWaoX4FGc6*iwv=y#
zHDh=KJ1?rG(@Zv9a2e32lt%1_bbA}q!f{WnXn&<J6IDdBm0I>7zl=><_o!gP)%z4i
z=ICjyTgqlUGtqe_E~B&Cvp8%641TMOX-QUQYP6EOdvSQc@CS|zG#cMneLh;FRu`lH
zDc}TNv_$Rux}s%{HN*`iK<a*4HZ*~h;Uu9y|3Ebz-g#+MHnOEQ*CX<X!Cxm;0Y)$?
zq;vAsR%oOY&1<7BJDfJef`MB_P*=z`ZCyh`1~v;_geiE+^;qxYh}lA>kjT7sl$#Mt
zw$6S>1)lUyZqD>N_QQJ{vnpM~{8`uPBwwN$tzH_dTeF<oyCXK_kBEQTaPvELZH{3#
zhy9YSP+s_*pHuEO?Uu~zcdu2@7P<CAndeeA4#L=v-K=VFD^15bNzWC5h|NJBqCudQ
z;%rZwNhWz<e>M5+dD2^%nMbq?%lQd{tolKq3>g@iq193`2NDp=z0yNU9_{7inkreU
zt`kDZB1b+QKdk1k5<({t^<)Bbzu0rD88<-Ge*w9a%5|r;q>Q7_hBmv+K82J+y1@mG
z_6e7RAHgI4Z2sFr{^6zk;L^&NeoJb(F7p0I!Nh}hpdHGdw%|ra8d124Gz-6A1Y+Qx
z4kNJk5+|Ykx~KW<pp&(=l`seSxK5js@cbry-GomTGtu22?AHWJi@`0~%Y>xkYtj0@
z%zMiz0t>n1)DlyVsI;Xck$?K|-c=#U>(mH__0o#|s0QykGPRd+F&92wXs($_b%_XO
zp%1uN_|#pdCxF1q4`D7;uw!I4xyabyF<+1|t8dkm#6EfMxTYo@l6?EBPC@y=B=r@{
z+~F;dYnOL`vUalvI*H3&(WZf2(-s$)OaV;}@cXu9J)v;D4tXR(9*Pkc)a(B&$m2W8
zF*s@o@%7r^T#F(V5Tjh!x@TUDK5M?$?=|{#TS3F?es1M1D4%S5Cqb{HJ?kI0MBMa7
zi~wY>_=ykuW;X_Reu8e&%r90fy(QVLv|>n61z8r(N4@6gMd7B+euY;@a3Cc~g*V%p
zjkwJlPGi{f<WEVt16My6Ly#t%QQt~Ra*1Dpmq}bNr}Lg6iowEhWBLjpUMC+;8s?nO
z0{$X(ZbNqxDakbI4JpOrcP^~kgQQiTwBuUUNifa&?ERD*q-C>WDwj78`J`B;+hY^w
z3#_Ovis+szc$fsPZ61&&-s*BVv$ERYcCs3MyU|A=)<s!|4D`Un$xZAFQ4*4f#4ABs
z90h+?=2es{3Jl5`oz&Cu(FdjZfj%8L?A3M1iQD?F9W6XqH;z=z?<1Pk_wIQhOHh&=
z;;<xN`<k_u-T-hwARfzKbf#qLo=k-$(_ei`Ndg*G-aau_lMPg_6T{NOG7+B&%L?rC
z7`TfC;2O#Vkowy9Q7#(S33%~|AbF+42;n>5T(2fzx-VIY)#3Cw%__1APYkPoV^;^u
zE|4$p(q<sz(0D~zh(Lj@q}t|_%zQ3nx$GOG$;H4}g0a}QY`AfJ7xea(3~nmqAREMR
zuTb)sBOW$vkOr49W?t<ZjnN?wTX#G;H0$Y2G(p4}f%mS2KI@Kqmq~iih^As8KwJIE
z;dy&F9b8akEn0$UA3-2Q5cw%uBE{9kJ(fjK&{Q7ELYKQA&7MvnROE{SC&1MShRZ#u
zv1zZ4@Y>op#O?4*k1-=AbB+;qB)wEK%amO+ka4Q-bh_D8^K@=1obsHjDzPR#L{kis
ztzTo}{h=b>{Z#rEljmCgWi^ZOd0Gv2c8(FdjhS`i+LOi2_Q!e*DS@~46g>Kjb-_Cn
zP%;g@jHh}*IqYnx-o)_HV28SS3iQa~>iu&}Q6ILg2$R!GP&QTbr3$Ip^rDyVM-z7-
z6YrLSHisM1aEp{D53WK~y+@UY<`xF*(LAyCc01A0x99F$2r5$1Rxj<Q)9$0l*NWFF
z++U8<LT0i1MB%j6ffVsL0!&`>6`Bb518Z5%Nx$4Xdd`&}-|c7>I6`-PW7wu^^lD$i
z+}8?{r1XT?h*saMwXeQWdJ{QLZeoDy)N{CTJmZu*K-{+y!xfB^OoUjm>r0j~P$$Ql
zmidl55B*jWT-mgHFKxp@<AsUTL7z&g!}+c+TKVfO(YpfsFo(uto`7AbfV_Sg5*c34
zsv<8Bwe8Nehp+U+otbbV5IUr|+2`vE4O5m6ZD6qI{CD{Uwdb*~czXI1?7uTEp)>D<
z$_AcPvyCDkp|zsvR2l{CING&V`78_~Pj^V+Nu<k;05}-3V0zs5a02D<vVS;Zj>fVj
zN-ODx;!>xJAJ_}pIXiNIcuZa1P+0*jk3j2bp8C=rlCZj8GZeDGE;!)>>T3h7V;)pT
zTu29wc{h@G2*xH55jH*z;#s1l`@ykY1p5Y^4Qrb$siTGqgecu_AbY!Oa7r<Hj7c_O
zmCKTbQc1lAkM>1BmW&r$QE0C>$+V;brGN=gRckgT0G~^1@$eM9@*w&ET{D;`=qqSt
z(d$tzA5z#(Dt$Fh>XG-zZ6leT13*4s!v_uO1#GY~qyg)Y2Vj+JkI8SyN2EtR#su#r
zp$(&deOfz~k?38>Iq4LTKlkZ@l$I$gwETdzW2zlg10#6upeb+oCiZh;KXT?Ad7mPC
zskk+lPl8hDW3<xXoihTw$#f0MvI{-&3yaK-7{zu`9ynoLfLPQQ-VpstX}e+RGC-V2
z9vOrxIY2ukEJrp|r150^xv_3<0@e=ZB1DU)2AZ7+N`0wNEP`vQ3Gz&wz0!}$B#&86
zfdk@(4}pIVF$t!vmF`X@XZQWNKC8q4lalgpY2VPx%5iT#ut*~^cram9twB<~kSW4!
zWx$P|Kj-j(>Ybr5G$6*I^qE6Rf%G-nTMYn=G0~YX_;HBFGSaHVacX)!PAjC>epEv%
z9dP6#ve8FystFKk`*k;gkgg!51dG^}`-mt>(7-S~uQ*sD6AC2pD5Y!x3(8@7KOofu
z^TOTXYnFsoROO?!Jn#g>Px*bUM^V6w$;e}5_t<kPX!ZV_P@boI2BG|u>{>?0FhkrI
z$G~-!jE2pfKrI@pr7cjcPJI4-B;*0+a7u|O-ZI-g%;t-7=XeymFv*ANPQgu_?+2wh
z7D)%yMR((%K=WXhnilUX12{Yhllzi(JUk9f?DUFw7XRWYz!h%v@O&Bq#$i4jws0c%
zuOp>6H`k`8DQ2bxvye9a>C@Pp+C9mGFwG-;TJa=itbsd}b>L+{kPxi#jaU7w`f&7?
zDxlnnF?<dEQEssKA&p$63qVYs=0%i)0*Q_>zEBcX`#wn&-OC51xKIhAI+G(gX%+MK
z*x9U@@yJ<Zc3p+#*k1*Mms?RKQ$6@Pl4VVGv7Iv1IiK0TIw6GZSIDygI!6|ONN~>>
zckZ`qos4k{h+-~Yfo=;l!bUfDB<4c<12}`I-Y>Ec;Ni2#NO`Qn`PJem+j}FJL#8y_
zLfQjbtn?!nK<-f<mA`ux;P*I;#8g48YsHEa);J-z;a=6w$)B!qif~R{Z_i{jHb5~Y
z5-5{D`-9zbR){*bzBt~Mj`wCiu4f#CVAGp(7Vpz$mVI&UrAyV08cN^LwMGkXDAiCM
zj&8%nqY<TYus$mG@9)E)(WSajX`KNo|LE4nT(ke!4Pa758G<-CL%<6=pbNpPi&5_X
z2XkK@4(0p4J;q?{GswP-eTlMU7h~UdiYWUM5|NN<1_{|hBKwjpl`W~1NtR@%g;r#%
zRF)Pb-s`D8-_Q4V{NDHf_c$DmI%do~&;8u@wVc;^omX1yUi7}jkq^5M-&Y7tvzT^G
ziA)kbRHjL}qoK&SxE2zk_<|XzGDr8}p)aM9`2^&&EJ40m9l&Y1WKsX(6;m_9g<w)Z
z_`QpYM6-?+R24aL(Dvrd?AgQrnw|g>D&g|5tb5{cliyIzjuQYkoCLNB4|mr~=W>M4
z;Z#E0uN3`PU5?%x=l;5=oGRB{>7bBSB$9>};Y%wnaTC1V-hfZx4U2*-KaQ^yRYt{x
zD>EQfQg6iht9JUE((T(JZyiFRqN|@}-d^G7$&e-L23xSYS7iL+81A0*9Ci25eb4O-
zi||iI@2FdH^zD6q>w(G>&!}wF{EnG4SZOUA<WzdC*YEFl)(jebNN3*8M6;GwmU(N`
zY-fXen&h1yNb_|AT(+mU+N(rocZobFIXzaKVXGR=?9~>4Ekq{gIR|+}W}&CelRt4b
z5tu6%JS~pR7EBk0kD5XXKYz%Mc!sL{u9L|Dz|3o~qu~y}^m*1!0;*%D>%eSU9E#E&
z$bsDZ#w^Z#pPt%8KBmUkVhV5^Zw@_NRhYu1v@NyEd*u!z$e4p(np;_9ik04`@7Ot&
z(C?tZ;(eLswz_(X?0910y(=@R6E_j~frnaLRaD-+%;*uiJ~`!f%}fh=@P?;4&6&}l
zmYLo*f17{{uLMm!cHLi_#?3>h^#;`P6jUugx_{Boa_@b4{127dl6_M44b(*o9@4CF
zX5SWxA?As#K0OV%$S|&Lq3rw`Awh9Tx#xju+7LT6_n;cld>c4C9WR8oQaqe8=Dp5I
zD)7X&fm!9}i)RDmfnPU1h53~5K)_lbZ8B&IT(sw|EtSs!DMBp7Tx()ac975f`sfCD
zBrCf+3&u0x6oiZg0E<0Y0&PN{1fi>snF<bTy|s-2))=b=22s`WZ}G}bWL0867XLM9
zN|Yn8`@w_xoCdV9$c0JJ3roE?IasI(C9%+9>Q0P;*<vEM6RO7mmqc(ThNgMe_X(Vh
zO5j!a5k%LeFG9*Cb*x^>+lF%Mxwvm34j3M(&9~dX+*%oF^g7x7`7#xcF)y;&t}Twf
z0s2u8GclVo@+u};T{x+0Jp<h<7`;%6RnC=B>ZvTSxTcRg7LU+!%xsPIM%<D9F@;(S
z51Yy2fZ7@xbf@l2_E##t_}#(-v0HWyRwpa_H5cpkSGs)zQ#WaJ=|xJpSK;!-Ph4g)
zCnKplbq#mtGwS%=i)a(+s1%F1OqLz#+SNq~zX}j_gKH@|?^F)({G{<%+~M`l(Sinr
zVwIUEDRyeMxi_@p81^jb2goul0tIM$b6y8ew{*(y?iSdYM<Tia@+DHnNk@Xm36l#0
zP-RT}BKlU>&TEW{PD+tuijlj5UT#;uD!2(|1CpWq8MbO$bX#?Arq`EC=^_E-k*pii
z?8gzT#cytDWE*aK@=)ieHTmP-gF><Lc}=jnm2S<j-9t?i7_qw8#r4jzIj&tJ2JZQu
zAiZ;&$>r^Yj!<mu5uLf2sm%A{@9SB0TvdKvnr?WAd)<LcKBO-zsNMfTT8w4i;6%)$
z#XS-ou9xYxF7aFu*R5<+;AJ?MqS9)>KpBrMFY?0K?O5^Th-FP?a$rys=6Fx7i#2Zb
zPKkZy)3$T}hCzuK*1)rW!TpI|KttMyP!BYE>)|4{GR&6DTL`7nH}8W(9=?@wSW!=J
z25;AWh;TgV(^+nsQhHr!@p<pVi_&b8d!*JAD%Q>$8AY137D|e3=rcV`9(d8H7nSKJ
z(cK?mGaF=m3Sz8H`Zc!Mp~t!FxawL-hQr63gzWFXGr2l)kx|+6$LVm%kBO4hJDQnd
zC2Q$M3RCv|`dk~3{h`P4jQCfm59HSxQ<~;)SYhSo{g?{|f*Q1|Y2&4%Ap@*08)V9p
z7SD2;JNf9#h}ckCx1E;#35juGe`t+x_oFIOka&rU&99MQ*|+x4M@{~H`}VJ+DHC<n
z6KAvzByr%5#D5DAc{h}aPUAWhNq(RUt)LTi+@}{5<+E5nF!g%dTE^3AKeEJGFERFr
z(#RQ9-cV4?moC#R$mtAyT&-8nr!?DK;rc%73D8B;_Xt<!wNv|?&K4ccb-$sDvyMM$
zAHW>OX{vTQHaDB+J(Fi_2-iynQo`AQT^FL482Om(%mjliw$G1B$bIA5vQ13I*~R~o
zey_^=mo5IN7!@cuaN5FIiQ<V~`I^B|es;l1A`MomrHjTog3IWH;^kn(w+*oQKd-w{
z#0Eu@#D3|Bm0>~)M`ZyM$A?D=5zn6HT`G!M>4n&LH_3ADK}l#lB~3fx9YQi2p`*K`
zpTLx)T|G^AV(oy^3Cly>{<ryMqk9N;{?B}^<u$WUMJweqDY)Q{_K>B>2kM1|Bo>$T
zO@$v}V>@El@Uz|&{1?NkLL;b+WL-;z4L`MGSBml1O3b}6SlMw@N2egju_ROEuRL3n
zcE2X5Z2naazq4MGP$WT`MxEdG$GBUN+f!1<PwGV-us_z!=5$W?N~~LBfayEC|LV0x
zymz%a(q7`4<vdw{iB%)5$E1+?9M@;MS8yD%c#LY6$M<HXAAF##so9U`EO8XH$FlfN
z_D?kkW`CV$%hMaJdh$8sJDalJw*0z&6G1f4chE_%^3W3weWsB`PpJ?slcc*g#wndE
zI&#wcH+Z_F7T8|WbsF1KpTtO6H-00$DmoNcL2nX}ky-LR9d>;6S#v`DuS|Q^3mozG
z*$4KmxVL9CE_<CYV$=St=yGpYV(o?h?o@=h%?Cp>rFhJd-3chU(r$L!?NG*nSXP&>
zz*O7!lh=C%1@2e=@0HFN<qM=z2fg<ANtaLLsoYBH?a($_j4LIFh@PZ-gngH&FrcVb
zFHH%?dSe!Z5`@mrOX*JL^$Q66M6{98sCUkL4Dt`{6`}DL+ZkJ^)BN?}r>LTw?99`R
z+;S03Q^g%k*71jRk}96_Hl0s2sW(X6k<>au@E({$R|wyF6TMSY7RL;llf7|vMo(BM
zuZT9bQwk+rwjpbb+x&^eHd?0I2js5bM2z2m7}zDe2wym1Up-ToFezG{{d5YA_Hrlt
z0!=;tsbAlohXqmD--TL6&KT8_$7rIA<~f>pSz8hJDvhxq&X;-L%e{u2bv@>Wk`+Sb
zttkaKI=_{ML#g<aIduwLPn!I!=Q5y@6BL?U<R%lOIfK(?!if|S?>um`lA3JRk;#>n
zyDK>Eio$$1mTxyyD3ad8xi&2Aeu=8d$3U|BZ5Eo7ry-;Nlkj%-Sf&YXj5bkh{8~n)
zc7h|WQtXq!%>Jf+r-ERTb371=(;YXm51f1XC~eIoDuYg(U4*8EZx~e88n!KL%zCp_
z8_!Z*i35hHz;RkJ#YDmC(}@&Q9;^V-?1B981|UQ=CB>y=-un1Y%NnvmL}vEt>;{9?
zi^PxgOVd7L9l>Gy((`t)b$-w=rdM2*WEp$cnkuA;mTh7u&-%n>Fd7_31wU{Xw*QzR
z`}jk@OoKVsu7)z9Mf|ZZQ2y(VS6}nXeQ;*Eiz|tOQH;;zy(D$-T%Yf@r<_jIBD#uG
z!7^2$VWx*)*|kwz@aLwkCh8+;@u4vz0V8hS*L(Is8I`i$IU%2u7^++`F$pQB4Hju|
z{lOhA+acGfV%a_z9$ud!O6a`~&7!GNYIIF!LEF)@^<LE0<Vs98GLo`e`64!?oY=!2
ze;49jbq1k9X_Fe|3wReB`4aIa#^4l18^H=01*!eLNjOl&OXZN(GIXCI(C<;4p5nsk
zFAy7tANqR+E#!idz-=Ob)>K}b*x8vYV8Q^Mxo-eH!KXE-imSu5chKLom5IF<t@sVB
zze}^Q*TT;%q8yw1o0p5!{iC~9jplCkFJ*o6oZSRwPHGRGSlqBCv5QpVn@S_$)u?FQ
zD0^FSZ{MO(S0RrwXv4;gTs5W>#Om@dLA}NEhA`UWg^-<lM4<(yNVZ5c6HA-%;H*Wd
zv)7~d?<L2Wf}bdiD-$aRR)*4wKV@BgmZ)MQ90f9yYx$@4w^$d6&r6pzkDnbRa<GYE
zUDC-%A~@GdDCm12u61w1Mk{RZz1!1>nv6GYN&GQt+Gi*In!!yrcj@3;wr09F6Gjn-
z=B2FDRqJrLe?OWAUwthtE|tDOH}~KJq)z~ZA^X4IP%3`bhixR6v`~~NK?Sdc7V{sa
z+2{)SV0OeGea5U;(xOepJSwGp(6YE%CP<s_%AZCJDk-E<qv%v_MX3u~T#O`J5?}M&
zWK{NBxh3i1xw7m_aeVSR4|-`9e@dCf8k%B0CI#)8l)|W-;1Jhk2L-zSRLqGJ*lJ@N
zl_$OqF;K`qKh*3k?a3`tz(P4qXCcWcE~H|U!mG@%GfD1EyG^#Iw^-~qOoYf}@BNF}
zvFl%bLsspe9ePPz2HLg9;O?)i)|gbM+Tj<UX4OER{X*||bH`CUkTf}{%1j*+N!CZt
zQu2@df)%SRZmCPVyk_8NthsrOL*?<?kUekCB7d}bbndZ?BX$*$7WmUh@vr@vhIrYb
zjb8Tu@{$Z<nv#9%asPSuKZc8(%nmh)Wr=#!|9+{K0AvA~%MbJr&x0V)|9+#46>#^v
z@+s*5{Zef#G<((h(=z^}Ir8rxR;z*iM|Af7{h)t7vzh^bs@gIh+4%qdn%}RM{C{&}
z#JjapXV3n-67=XFP|Ak<RQq-3TR_(N_#uJ$2KD8;!0rU`yP~94jeq=~6fao?=|MT*
zDIrE)Ve=pBrfY5M9Jm&ViHSbnGn)M&i-PEVLY|ghxQ56ZgL3u&h|-2F43|EMc$6Uw
zweQN^wx?8LW&>u~9o4_9hqc%qD?6Nm5+a%1Y>LMGcYo2080j$)1boji^F;vq0}ZS8
z;AEYT^@_g*1vlu(aPQi&zdtIU>YvBjoJQ)azxHna^$zipkcP6gfi_3DE|;@I`5!M+
zpk^QXqw_5wcY~a9z?zPLM&Z|x?bu2$JiqZ4Y8iq7U(6kzTkF!u%(FXEEo40&HOk{C
z)nNSrhnzaF=}%+PpRXcs<JaJWcHLPJ`AUOODu6Wmy(EsML+k;Nf}22!ukOx=S^>=n
zezg#H4&3G!m4p9WBgaK(IWp`z>^X~EV_Vz6wOnNFOvjo7@+eT^qy(c+JMk~Pxu<!6
znh&&n0lDG%um?B`7OVwOkl%aqdw=zn-L|y2-7cmS$=Zu3^U%Agb_})3*_Cj#c+LMG
z=kX>k5(?vS{wSUY(13B)-yNL!(dWCyEJlGTgW!Xzfh5YM>H{wt4?Uh@cRVY1xZJ^`
zJiEz68#P-|*bg6@nedrc)V;^ni`h361SX@|<-=8QWYss3)cI&9KFR_Gijk@Qqyd_7
z0e>pATiKr`GHqL=iOjp)Avxya%tvaQnuI@%T`m}O1ivB4pKXW8<d}=_lCKL78b}~!
z!@NKz7Rq1@0-32rg8$1FN23cdNit(O-}0|oJM3t=mTGr4)oxH^9hIFGp}5-zW6FCc
zFoh&woT{ALWr<sQZ%G+;zJ)myIdzu6y9T0oc;=;t8wA!fwla(64jd^GY+%F;Ii9|&
zx2>&`ld)qHA|<;^j`gRWxhQiEK?{FBCmT8p^q9(fb|w;dDfdyw$S;HMUJIKaJZ__W
z_3x?oNBWa^OD|q4|8vw!nlU(LkO$)!A#QK|0)QvlfEw5rZ=q?)<%O%|!N0bL?1@;a
zpt4HIKkaP(+~{{b=}0S5gYKiFY3g&tJqS1_d-I7MGc_go?*)BG1#1)?IFeAynxDO<
z>~Qf~&}-b%$hG{+p#J0(KGk60*6Rkk0}$Q`3dj%QG$R(Bta#bmQBcTCY8zZ_bh+L^
z`(63QA0pV!K7<IS7%Rs7TG@#7?JDoi&{U2e;C&GlMO(JhfItUSJmRzmgo%#If|Z3|
z4L5@NiOs;4C@(>7bOO#Yf0W<slq7*MNsz5t75#1bGD5WiLFk`QDcJ&{eME(@E1&{t
zuLoKXh=*SRIiY2Q?c&sXISoS?uJt-6CIzUgI2qE?U=U4=2Gj@yGE?r<m0W}-Jq8hq
z2}dbq*WE=$P+teKL>^EV_|ctExBCq1Q<qIDDOuqI5G;deh^=2}L`0*2FownY7K9pk
z*BQzhy}f^`fl)L1tA8iv21w82?Ny}dBfpmpfgxW~MT0e12F*=`8$YQzuxD&(uZvfz
z@)qYqMhIXOu*0lc(4@!B8wtB{ziX&rGi?iq#{ik<szSg~Im8+^CsWdOwpxMHa|S@b
zL}yd%&2@O(zX0tz;#hX(bxPhi2qJ7lcM!d_lKZ>52@r_cyb-c(Ug8KIu9=}&)x2a7
zyF_*lymk|JvM2iH(YL4ja}_?+8xl0cY<^%)1GNVl-0DkFzo7rjh1rD~q;;NxnL6}i
z$E{domnNG`0@29u&STDNxf12l91PPwf&8pMasei{!HGf(@5S+k#T}L#3pvC-Z0qw9
zpI2?>pye_NDvx;}7ohkd0@TNS*CL^!_C$ybjdD|!`N1DRcMIl2q<R4vI}!n4pi=-?
zAp5Cwe<|Nmh6iJ2Ns>b??lzfQ0dP%3&19o_d|=;=3X(hY621kjZ3nJ7w@TF)ArDd~
zUSjk2M~};Vd39S!s2(^C?6~Udp8%$L4vMAv`VA{=)q&^~XZV^!&9n(jJdIl$ADc=^
zWZ&-Amg`hb>PR|#MTDe)rnOyUB*Z~U4^f}{`22>Ot>RBm`GIPxa76ssPM&&1jsbXM
zhNiC!KsJTq2j_T=kDRW(k=QhVx4QMGJ@eH1?4<^`wOibW;B#+Ula_tpq&6s1>W@K7
zgV7$+GzGLO-F_-J(IU$Wm8M#!aWd$H_wW<|095U>qrn?EF}$&_0W?kf#1s_34FXko
zcGVhMkSHj5AcTU>EMIK>4xT>mN7%JZSxCwUI){O?S8a)Q8q(WCdWPyY#xb}Dl8(zg
zwGV7kkh=*Cp-&i_A@5Qj2ucxzX1v?+>vS2m1@842?I&<1V?ZJ+t2!FXY+iEk;j>rC
zfM4Fm%KN!nE9SFs;AZqH+A#s=Q5(9|OaxwcNqkSZP{g=Leo$kQjNZ+si8=dzXWQ1=
z>tg%pftFRnc9CB+5h}SZ;6-oB9Te$c&aYF9O*j(0a~HV)Qy-%QY*2tE`PIF+VUs^W
zQ>Czoa83Szej~z~CP>pE<h`J~6*p~m^uh5pDfy0l1wux-Vp4~1L!0V3+6Q2LKq<{`
zG{vH=6yi1i>baIiwnvvx5+Q1jRNszwLjj~Vi30S=v(B;{bT&fnZ*7XTPd$U;PWOeU
z4n}<<$fpj{Z^EM}umLO1m2;=agbsmo?dCl<m<=g-eSPIy7k`J2<U8UdstYB`q`i7U
zI0U_C;#ZKhyS1am^Y;VHMKvA>VrT(q{E3e%95b+Pl2#RZtj$lN@=nL$Y>b8p%tw^P
zZMdnvVHw`}1kpRBQ9%5Z19Y8@WBn(AmgojH;R$uEYg}#<Sa}#>1h-G!?e>`B3B3mo
zNF4LLb&Zc@d>#$T@{&pOynFO^x)}eiPk>PrHBMtHBg23hQQpb<h59!y#y4QwDVtA%
zQ*t|e<)WHdrIv&{$aa;`aJeEjzs}j6v+WMHI@amnCy&E_L(1FOlF3yhIDshxF#Ta1
zY8JLT4KIBIQ?nNgBkX|K%68=1R@jM2s9)PFSZuXG&ra>tex=zPmFmxS!oB@fjzgX&
zJ=Go>IB2oVdZ(f3R7r;VWl)o4l~v(~is*g6nT)UTPX4857aG9mb*(&EgvM#NfSN0y
zWWuL<`66B9>-n<1v3HJo?nE9w+>I%U+5x(U_wX^mv>Y9vWr=cetA$LmnakQ{U2GUe
z9s_Rz%6tQ2t%DO7Oh_Tbf6xuuGE#25Q#&}fs;+s5qmj*l`;~vy45Sh%h@*hmN`$u+
zY`s&GUZ6Yp8<6!i(M^J)$9DQXCDzuO1KVo@PYlWp3@bB|y;C+*dNWFK!uDo9vclrL
zOr|CYQl<$Ri5bE^SRV#M5pN5rF#OYBr-DDWeEw|lB|PEY(0Y;1AksE+<6GU3_80Y&
zdq46R5HSXXeVC9v{v&0E*~XF=1{!_(Dnt_%zefsK-KSlS_EX-C$5Fx0ZoJyb?~5I0
z8Eu(kz&~&^Zc1W3;%ILln_U^@7rTm0Bw@w0$YWL3Zeat;V&z?1J1@(3rwi$v`24EI
zzRD=fCUwUR41Dr)&8tyKB;@UpBubu=-`qg{xtAXb)WecF<&qDfiqS>8_Y&?Ha3J4_
zUlLsc`k_^>Jb8CSyV8YggscuQz*Z7xAF-6bVUHulFtfeW4$KiDpy4N2WC<Kw45;8a
zG4~hiw0rkvF3<qQCmfn4?wT6)iXd+i$?!z0@iqrMmFc>4WgWfp8s;N;K)AiWI&U53
z_kl1%Z(^LWP>dST3B~g7joxFPBfEK8Y%p8d9v{L-&XTkqEZWQW6g5KEmm_!nDFt+7
zuzO}z)Wy@0J4z(VV4+Eu_3Hi`MaqHuWUgEO6yG7^Kp4FijyWt@BREP=hWET;Ck)Cx
zIF8h+VMnw>YpN-&j?3+1;<v+)esYrcu6ticq5edxb;S&iBa<*db?qLs`}E>ws7S6>
z)G>%Sek@MJ(dg!@9H}5+O)M@V_cdvVj^x=BG`<Aw1BppXvwfE&ujpAMJSzz`B5Edw
zM(8UG;IYI>AO}dcYXqKtf??#8tp?^0$*7i%3XfH#vUAMJCpl4zNmL)W`2mh=da=x$
z)|8;}uRl+#HI~vE`VJA3gY6j*2VS}}AUu`N8hNHwcWZ7Z65Y|5=W@1Xn6!Wbqk988
zLo}Zzld$n|lv4e3zWupm{Rfwo18sVza?SEwC#Du$P9Y0M!nS6k-mZ(?=t`H}xAGdS
zYGgB1EkSbh`uU4YysFyRN6!h`pUZuP(Pa81O<lF-;~$?*@fuV4Jb>=`4^OZ9xwHc(
z_lo(Y`-_czK-RtPt`G?d*$3*J8i2-_5qTR3y|?e>bO2WndLU_oKEOnt4W!r|NHgzz
z(U4GzolMXGCB9d>>s6E&O;8?$nN6aHt6|$|OJmtSWR>*7U5H6gHL|K4eWTS9)|2f%
z(dg4~QMbH@it0UfdQu{QK?IkL+R5%(g2eKTb@oIqisBp)x>UGuML<*+thT%I7}~<G
zLv{q_+OIcLT>I2FP{xMXdiL=}KFx@rJVl>?r?rODZez4ncR_8*Q+uz^<7`g318op$
zfj+|b+pgzCa6d4KvJ}bU)N+KVdy0RX;~c9{(VoL>+cy5srcZBNF8RF&QJ8uH;Z;gF
zW<$P{j%uB^v**+kSp$MR84gDqK(^<(c1u_Z)A)(jjf}6Kf#RP=_yX}1a4WXg-m!}C
z?yGYeQPDg$gIym-KzA}Von4Z~Bp3}JdT-uI&(sfObG1$tD$!VAuDl3z#F5t|>fuHX
zk5%FWN$3x#=EE0v?jMAQ6Ek$6iT(E}#+tYf7O49tr@Hx3Xh&)~(?|j(3n^D@r*lb*
z*y**ofB+F(G6kd#O^M1WWfSXrZcKNb_cA7;B2MxY9rq4BI?;ILS`-+aBkM>r6^qE@
zvu7E(M(cpC9VtHEB$G&VWgEY9JP?B4<@*o5`TqF)8Va>KkxP@XP)WDaI5446J<8IE
zK`~*W%6#r_pI4BBKVskPKLYRD2B(V`Ff8oZOErP-{3Z7cV+e}{X>3TDQ7W%?6)V4*
zdmz@2nCC>@G6!L#uzIX|(6c%o2AZ$vNLRnd*)<1l2DI|wcK4|4)5T0Dpy&;fG#`Bg
zse*4GpU-=xr^z1Fct7!OjwMj!xBZDm*Y!3`6sU!Iez=PrySOR8ylP+)3%R|j#lhNq
zh~(z;4cb0lZi2%K(cWC4rEP?6;ci(*S)o`C8(&-Rzz`{l=>tLV@LPA=4*@x1^u^7<
z-p>gSWA|Q-#YPrVHokof?Ui<_CtOfloAKIBnqIv5{LsT(dj|B4x30ar5i2+)LOk?R
z<M;<}crLC2l~;;aX8YJxRps_B9KdI!5sP7=Y|u{SSFDQ#1$0~5-PCw-(IhkTcoo_B
zkcK8@FdYP$bafynXr32;45ky&8&S9;K@eGf8RWIYfKrZGYJ3k;|F&8t9LX=vqvBh4
z;PR=rV?kwMs5~ok99akg8ZR=*W=yT5c||30_@GsF@N8g)%`KO#9RM^CP6TI3SegtO
z&FPJbG)jq_c>rzIA6?N?KErU5V8#hZ1;jZeKoS@OI@v+g48SvTE!8_a>?EXfn*SS>
zFk4|7{WhP4I6h=A86k@7+<#q+R+xo`>tHzk1=V9H6kPpCbBXllLK?*CqgH(v73f{~
z%!T>AR&9D%hLhMOmspQGT$Mrt<x4=9!M&yi+sQ+Qgz+B4+)h1(>R}Rog9VY!h_H!v
z7{8KlKeY0j8BxE;Fb!Q>nsW7gQ9>YGpi&t}L^_oXL~@k_bC~K+p*{@HVnT`^NU+~S
z@sjJDn^xT9kG^SxmhRqJPjjt5&h`?H8}u`!SzoVi>Z4xUT`{{5^H^st+dR>f=?A-7
zdLjD{b7^`*B1Xg*WgifO8OFzRqqosx-P@8GDYONTGsVrnFHY!xEwLAwltD|igk0BE
zmiOw0;GK8#n~Y5f>1uqo{GVIjB}vieI9!^9U#NA%MT7!rQ?-D2(8_xHb`Bg)*OP%_
zxmbfF6ctu)CpLq+FkD^|ox)h)!BQ^PoPZ}g;E6A9`j3b>P>;9af>8sMTF0v^Grd{0
zgW(tcIK(n`!mlt;nmc%fFqwnPM12`CXji>_h8O!NjljGVUi151q!ZYT+c0Nu0*<wj
zGRayuvsU3tzeOfkUhoi|2}H0?I~}S?`+b8FSQS6KAFD!IXCT(Zg~^z(JDw*>p-1uY
zW#wUi?kQBC<j*^E{yxw-rhSbrC;qt-CVE`G`u^*#F!4^rH;kzZs}>H199+LkjolTJ
zpQALolwRtGhrhPQ&fkFoP7)dAv71J>;FY<hJ_aHsSEBoeCpAxN9m7gvQ7SV9A|f{!
z38olpwv04N)-6lD^S>|mk6S~oJA(osU>uhV6Ae~jk1%~=1$P<QDv5|iuP9@wjH{5?
z>%V^}mnh#qz3jWV^VXT|Ehe5<+|+_P*v9X;Oj7;W{j!fvm`o9ZW^4uaecPW!`}emR
zHbqG1zG;@kX9pZ%4zR7uI%l@PQK}p4v&CeZ6E$LIGVL_~Y^{HPuR#NKYPl{1GOttg
zKGO!@*W7i<QUiVIEg?w~tw30U7@!zZz})IC_aJ8J-)r*w*nXC%4}U)joUJ+t8iP6Q
zHPEVW1upeT2$1`ZW^}TXxpCmeRU5r~S73}{(90G%+J}KA|9)+V0+p)q&gSu9ysX>(
zO=zWQRt5{Mh{RnMHUSdoB)!R3x5J=6=-y;3W9`0c8qr@D4Dwp$-W$6=h^5KR;@6JJ
zRf2w@UAATMZdqp>3bao}Z+v|Z0xwxLmV@Hkt%vCTYy*)=8Z<cjk1$>kB9FOa9yh=N
z^Q#|=fp~D&CHv{y8KiK`t~O~lQ;esn`&@q>>^`eYyAECuF|^4l-g{CdXVgFF1g;gv
zP+WDWVLQOtOwG)@XAQs}7+dMec1OQ($8YzBqvB#C3Lm;3vmP|jc-uxr=t2e}0<l{2
zq!cS=3X3m>uOVZ_IH`i@UbGzPD^T)z>`>>VuxB<&`OJU{j;ceEnOmMPoL6$Zrd7<}
zO+)zaQePLIq&KG4Hy5UNU9DNpC5+-D7ZXff@rhE1cM5vdx(sw7ACAR-WV(eXyFs|Y
z@y~*2AOHkHgm~Tcwn^@jwy3a78J7uKRzZu@y<nm&Kzd2ZZ_u@bNQRj!XO{-y47Gt?
z)M1KW<-ThNIAV6?X+<eURX}eZ8;3Q*c48ZIfzdP3KR*8ggfd6kKB38{FXwf=N1Xa`
zIOx!7gVSU`{uH9hx3`YE*~7Nmggw97pQi*cgg-2aM#SQHnIU7?<F*s;N6#`AD$D#m
zB}6CeA_RI6lpwutPSNMXB}efFmqXgV?-tN)d*eJ{j<M56iN?ZZHHay3J-@pD=vbvC
z5ZbIQ-R^YTV@`uwQIXi(U}0b8NywF;=`Cbe6U4<{g;-h^{x#5iXm}z7Pz6L-cVV*U
z4Yzi^vkZ7jEKk!TRhoHm1k*6Ba2_f@<{6cKsKhc~g?0$2MOBvh&#t-L0+<~ohd^E9
z^&a}{?nx^Cob>?^q?rv|R)#D{9U{T&8-R2HD3e|w#RvmGSUSwC3)C*)@+%(#{+DVS
z3@wde)hGULd#osu6RQS1SpX=J)vNfzM}~XbAO+M4hx8G1Y0<kZRK9vN5@@3^POd%!
zmv^M1eA*VBGHn-L+}!z5bvfxI8Y&Qz#=&5Hd@HM&oIcq@bwM})e7HkfWGDkjFiE}~
zgZ@JgGSWR7^ersWj`Y1EPO&Oxm+m$9-8|YcYWrlI>TJe9I};9sC&hEn>hhi>t4H^H
ztME-|uTXcZ<-EnQDpsF8XjiM4c#uIl`;cAI$`z(&RtsA+fy%||Ld4ev+y8M>Fu!05
z7{XYua~D1{?9i%RHAyeqdY1y8Ba;8AImTd82ZP;9$&t;!XT>l33A8$m8>#)SJYF_-
zhakAKR@SjLJMRD-UJ!TD-9CKXxv9HVY}~%p$1}*2^%vt@CMZr%*vC{WdOxZMFJu3(
zY8SZ5$`pxMbPk%ksU#gyi&8WANjd=&I*wp0PbVoD5tS*<E8)`uaSE~T$>|D*Sfv#E
zOW%O6{GIB$Vuo2tD!O0pauqNt6}ru5Ky_@BSy(`K^R@ke3C7M8Wk{*_c-Y^CsqRV6
zSKK%j`xLb1<SrI|P)x^--#xKgd)vwbqTNqgduE}b%RZ!~lYz8Gvn6*V^|qLGkycB>
zH=Rnl$|AG6{fpA}L>i*Uw+~Mv@7BNQxj<zZKO|*!6g@3^-Khb+od{`0+%WB8-dcFI
z{@-0<qDjSSxU)IyVj88)MtB{gDArzcAenR(n^7<%E9NThs70^OaVPR}+RM2Pqxj`J
zCz9NqR(@iND6VhaRGMrgFB7O32b>>wq8~x_d&bM&%K_y?HL2<TGGyTfZ@r(iyCz9h
z7t+XyW?@B<?Jw0=Nxm;#fyCHi;e^R%5i18>q5NvGNb=q*v44F_{8^M$2prE}5Q%n-
zx;)XCQq7iX->hZJCj`U=5N@z%2#}wDai4xH<QJ@S*$T}T^7PDZ1?=-{E`Flh!hFk&
zzr^>PUqa4N(Fc(3I>}KtbtKiAYCn2#Fi)xH8G93I04@50cN@K4U|p!840C8wu&tY4
z-;c#BeLc!u`|b5VS^&NQ!vYx*R41tUnX%$$59`<G)N=hXzT_u(iWLH<>x&oO>GO)b
zXhH9ZeOs<icdB-EAx5m_kMozoW*DG)PRCdv6KYk~>8VWoiQA{08nRwwnB+j?WG{?p
zKydpG#|*|e#|=c+{3jT?^~HwC+~GQBr4G9K1j$YmvwwOlrNZQ^P{$K)C9-pq?=N7F
zyR-y(%v#6m8Hwe3Qp{8CQt{ws{BarSAOeVNx=|C#`7~PJ$D{a)<!6$Uv!i&iNz3Qi
z$cvOqs5DgF8wF$Um)jtV!K>hj3<!(i`*qynMOE^SozJRmsefR~STmn2fw7qEN9MD>
z(6eJ*%iqWn$179yMaD8_@LaG#SIC8Hu&l9$L&5NZ!W8u1=zb%HS2OI8X+HW^aGQ%S
ze|VnF^T->B?)*6)XG^~eGduU!M`jqIz1JU5uATn1ad=Tu;*z;Bj(xF6@T*o-_{ohe
z@x~=t`LFLSXIM$k=Rdz-l|RgAB1GBH+2ip7G2W0BTAohxpWZ#m8U;dhxF-4V`R|JW
zU6>q66pyu)%EiY?a|Pe~wvUb9z21HIe%-$TPlq1rt*YI3PYte|H)XHr+uIkGkr7be
zQDs`q39C|6*BZ5=--&&1cfH#=dJj*Y!41=?sb^=7buTa^6_H}y=LZW9edVwm-1F4v
z&#_Lx*yxu_R6EqzG5MJE4R@k~N6Kgf&mMkn@LYlga$vE+d(SM86(_LJXPlwpqAfaI
z1P#^F*y$zOKr>`1aDnK~i3iu2b?!PbW^OFc24_S^&_`W4CL}R)<ax#KKRtxaNLz-W
zsONO1Wl>~a*4!UQJt=~2<*ug}>K_|0jHTtG5zu|vawM+!IS+mfNxwI#&#`D35G!8@
z-f?DoT1?WWY*{#n{&n%eqC{<h*e?K9h=At|&XQYSk!KA0{Qb+@bkDu(`)hnU*fMVD
zcD#wU)FNY-UUnxx**?4a9a760Nr@Qf;~UTGgR?gOHZwk~;2w+{`DI;Wgn4B|fQs?f
zHe)RW=<Jj1vn{N*XYhdWh*GgFmk2C1$D(+76uk*wh9K*EXkv7*O96oimIt0bF(4d9
zi{<FYWkAKXBEQzq6*t~;gfC~k4Qb<U&9zCE)J@T6pOf#}{{fB18lYzNN&=pO+v=nz
zL!?M&9H*9>LT_qwb>B6$-1C*V!)UwH#+U0ixTtR72)S~Hj~3hjpfB@gQOq%?;oF90
zy?7pUzUon=>VrrC=<O0v3n5MRU*Ivm_b2pEByQpevY!EU<WmmdptH9d?@((^vdxbT
z$Jpv$BO4I9F^B+-m#p7g$iI*x>Pr4dYp9nfM3D0kv>dE3!52J(qy(@V=xJ?YehD+y
z;M4s2a;|~vh5xXOTU=C-Br49s=qu=hOH~edqIP;y?Yj7y2n(M|9mEU@mn_l$#Pogi
zla{@#{xNGy?*b|I<e~}JtqWmEkt@m}y{Q@8BDhd=-8M3IT(cj7gY+<7m&Z^dcYf0r
zRpU47b}#xHf*RXbS^(?`RfTk}-G};bc){4aPmYyKAf|*+tF!4PbV8>)VL}U~ip=6=
z?hf|)%sPA9Hgw9X1dZb~dm$U_+9#H}>+cpS5~Z+8os=wM*DRqRE%H$0Z#gy#Op{tn
zP>C779^-B7Zd*-2hg!Kqdw<>Lr$DHXX|q>SF`&N}Oay2HN&+6-gyR(eQcZ*5Et{~2
z(D-gTfMBT*Vl*?UADanLu~C=6*eu7ulY+f<8=`6eA1_0j6N3BX9HWq^>~NhkY5q2U
zT$wdHA$ACGEcvTnzseSL^u4B->d)ffQgG-r$bdZYkPLIOcXJGg?It=?G{PWg4KmYw
zxbAr(1hPQ*X227Ks`kETLIzI2=XA!-UY;m>AZ($qj~XE6`AXbzPe>|ZDqm9|CY$h2
zyl;wzMDZj*ZjqcT6<^8mM#hvu-4dl*cJdiqM%tv1S~#qO1?lZ=$l_mH;oPWQRA@SQ
z>>_{OxU|ZVgSHZQ0PPm)eZI=q87iIO^9w&YrlHB;sl6avsip-KGZf5R2``1V0&)>s
z?gRRNHwIw3L>ic}xJJXs**Ru`<Z#j70t&2?Om<kWj=FYOK^9Oi>Ud6M(hj{TnDJgd
zJ_N%xxd_Gt8EdYQ|2llka!`Mo<JcOo^Clic@-xxMfETBH*?_M8*j|(ZAyI@JfZ9g0
zTZxpts}1`nV30!lzk!C$fMR3t#g8EE2MFX@owI#Qkn@cp!xiIf(lp+H<Aw$o!1shq
z=4o&Mw=hjpj-Zi&=487zUv_@iF{^T?22cJ>g$s!Zl||xLo5+tnBuQaCa8M7l1of^m
zgo=bvuR~dAf??s=wR{eYN0q=C0Vt0zwGnow*i0Y2fBAQ+Ur~q}(vdsRY;sa3_@k~c
z{i_J$SJLU>n9psyGtDrb#FD`0*S&u@12r9n_>DGr^{|PXK_EH@Xn~kzxY@%AIHh)B
zP!(lMngsB*`_=0S;%o?>po{;G#t=F?<9rS+Pj{(;$hBt|!;0C=<BL08%d3U{t^|%s
zk_>Um1^zpb1rI)6lL9{`^h05^DI<+A8vnE%8<~Kz!kAy$CFk5I#yH~fFriQ_v<ysu
z+PigYB&!Dn2QFqr=fL10OnUiw%hs}%KBH^}gjS$w>}R9j5X%&Tk;%lmjDaNJr;W<X
z0U~Zlma&YV{s!p}A4_$f1+z?*fkgRjB*P&R(eXcPU}fQPP#1j*7iZlOv--25Zslu=
zSd199!1zv`$=9pehJ;Z}eZkWzRlN+Tm7HE%^6)F@AW@f!5`K6<_SDKn#aFF}q1tAZ
zm*87GFxlDwVQ?7(KClFGvWisfNRri+Sdz}VL-_QFIbeBVP&yQe0UQ*O3n_ESqM?3%
z5o!$iJf)h%kNc~v1lTUyMfC`rSlUZsW?VLwDrfG`l1x8*+iytF_28qL3%d?#L|l0k
zf4QEa&S1+R>Z_2h=Xqi#28>DkA;FWqVlL4QRJxodCyr=Y+>>hePD6vME<-u%8^n_Y
z`$S?L3ysc`excZxFVtlPIcHRn&Se4^2_EVP5JJx95a?CY#EV0ewx74hgkzwe$Vh;A
zV0ZQ@D7(@4jhG^_6wG!Rfi&Ud*M$dD5J!X@Uk5{KdPP@_=|fZxEjdyg21tp&miPpM
z7piQ9IBoU;J+z(WjI{&$O|Ydp9t)5p-~4?@h=sw-hN{1X5v<t;CV&>C^-UbM^AIMu
z#nrnzNFoLU3PJHFp#q|Dfo2$=VHMkhU&JP6h|%aK+kS|3z*D@iPf_R3)X^`H2ybN9
zMzF7-3R$V+Kz+1rxf`|!7^cCE;tr5@Q9pYdjA_S(VbYMC1g(}GtANHHN4o_PWNU_S
zQMJheO38nN0qwL?jC@x+<a)M|273P+PI_^;1{4OF!98jr2#&8L9b+USYKyiwU>}aB
zDk3$w=n|S8+PwBrG|BRc-fKq>d2=|y{x4O8ni{LlraX<^*(@Kgt<Cl92|t(C5MlLZ
zvx_oaQ*ff9o<s6Q#I#wtgB-9J+scNl-Xln>v1%GV*v%j6M72M_Q<(*6INesz_F)o<
zl`7jFq+%p?p5bRsQ6?*;-BN1uC8KFtbL26aFs>C$mGrHEr;jIILNvdi<>37L^G!&R
zrxDTBn!+}z7)2jMa>Ug6rcWQ|^oZCho3i;Hz~#hV2t1GZ6ABwlwqwREr#(V}g3u}Y
z3i2sodVrq--{WK-<|Nsg<$bMI6U}{WTW+*&pdy|oJGlOxZ3l9xlrRhke+U$8X-P(y
z(Wq2nAehc}nQp#xLwEXB;DE#5xN=LEREKGOZTGawTIfyE`6;bmz!f?2`p!jO&h5xf
zn^vIqVZBY_^&ZD7)vY`~?@FILcm2RkGX}Ke-ivUBTE?jJwcyb9f@Dfjmev(?-)q8^
zPPS|J!ZAq^J<4wTaZYWK%L&YUu}K{*f=QjRg^u@-^dkNODRh_Gsf|6SGz6H{#K;kV
zIV7F3<k0W%v+<gC=R2zh5DL_Hp29pjK%%S))J7@|naYDcm?76)_8WV`IGM&<Tn7YX
zHEg!4VJ2|0>#juJUTac?XX~9j?9^7)RKv^Gt&L*4)ZgED2_@a={01`pABM>s!iQU$
zR)JiwL!S&vFBf0YRlG!|%A0r)<9r7$U-+^FV`fFnMOvs`P5kwR_+TLQ9}q?oE0$PJ
z^#g6KAkw$^;^z5`=Au?*HpRzCNl!6^SrTo^PMB~SO?{$=T&JB0;BmVsfs`oxP~yyP
zv034ZfSMdHDoc^<072yyKsK@_Gw95!c2Y^JN8&q3+DGEi#eK)6kU&LHgm}3W)S5ch
z9HUGY9nK#BM+HSYGvkZ^((<X&80P7pTN2L}<zqg*zN^YJ*Ox7=-?mTWsPoA!D7i)~
z3{OF2PxLugQ8Nr@Q})l1bJ=&@RbAxxTVUzT(C>C`oPiq1a=Z8uwY~LuUsANJ6PA$b
z#=zx^_ih5|X+%4qDX;r1l*_*N?1h3az!qdbf7(;P(vpAm8Y0|O@GJ2T`Dk8&qHKb|
z?p+~&(W?c_!-L0G?c}HNR&~&Gn`pcr0JG+D`5I%vsCc>`U~okq#?!N>xHTLp`JFSz
zOAwhl0Au9Wip=;@4Wm10gFrTsYjyUW18&?mX5B>B{V)V$B9n*FYZez+a!@;qiVRoI
z76Y^-V&G??jk_6+pvk8CK1bOJqFjt$;9K!Qv~sAVR$i5$M~wx}9(3lPgNsP(dfqf|
zz=TGv&e@&;SY|rN@x~#6zHMZbm*$D<&`OcJQEo}_Rt=@Pta@~COi50mQ#mQ5zU2z@
zGWt&Vd(TL42~CunbHn$W{J#Lm^!?VHORjk$f7*vqKwXg0!^tiTb(<5NF72mJL40P*
z$*uV&6$lZso{30sa~U6*6FNd@4tx%mc$|s+{wP3Gwuz8%ZUKAtq2n6DgKdjy5;#bM
z7($aw)LWSao(4j$_9=}G^h<3htVhlS{5ncTmIEnm-g^PmgVy3o2`O~12<1@qA*7@o
zbN^K5oPsJ0xdC>c{5Plxl`#jN-1K$M1IfO(EkikSd<wVJ#MGFNn6L(04df}E)93s?
zm+|^E3zyu1PT|5!yWQSASxu8hKqlv2`z^SrdIO60^CD#$2e!!d7iNoC8%*xQ(JSUn
zFI0WLJ_{P2`ijuwa(M*@=$N>P6N#$+!vuTl&k7c7!^$CRpd~KK_W~2fa0LCP0|`IX
zpWOXPAM*3nZf{pJsg=+u7cVvGQ)Ql5H~l|2r$H=)U`y|=@885p9ccE46&Z+0=GtvV
zJ{TRb2rfTZ#<_pDYiow%FF@EnHGhPP_G5`mgahTpqVV7fDVcoYdPgN?aVjh^iK^%h
zJn?QlvW0tFzxd#Z6tq#gVCpZO(Dr42`3san=AgXoBmCkBLD%J-tW(tZ#KUQ>DRuK|
zc~?Q~4&MW11j@mVjeTmn_y4xue|*57-<b$b?Hy~UbQDK$BBhtew%0uHY<Fuv4Ub7z
zne>@**sgB^crkBXefWbFG(rUJw2yz_n4%pYn)k8FJ24%p&U)%`{FXMLy)aUUK7hy{
z4+hTzO?pnURthtDwoLlwuwkA27>@7y%v>)dTlIntze|ySV|UmD1az-(NhGGXv_j>L
z1nCnhXLE6s{d&&RE#Mh<x16D@xjdLNeYEQk=vj)XkXOG!#)P@s`Gj3|@=qO<O@T3y
zK;$QrF5yZE?x*U_DoIK4tFp_Hh3EbOo&QsyA(|oe20PDyFMH8aG&al`o+UP=dTLlG
zCcH4fSZrC?NOA1km%@+1D9Z27*=QBa2qhRu0^}zN^eHU;cgWt@aa4DyIXH!PnQlqj
zU4FMl>gaIosIcJZp7~Momyz=Cy-n&9HPAu?jTC*f@A=aZ6V9OCHu{)@PFNyK;un-H
zZI+G;=O#p&yYAD<_9gk46zRR<WZ;xFVeM2C+e13@S$_|sH$NeRJYpv6RF5(>k?$6F
z5=W*&2R^IQ&7A3*3|dEIj@flHi-7t8oay;aSxVu938K$`BILjSFp&n@f}NSgr+dbe
z*YAE7p_+nW4IPgQ;?Y1=q0qQPWmU|~V#_-bi-^Xd879BGRa)>o+Xnue(xog7@+Nxg
zHmfb}Nwsa9MRKb;ZYx7&L0aB>Ql?Rf&8;3F7zYVAT0e6=nL5|oiuwsm01QZ$XHZi1
zFnI3`X|esETMy0V&dzA=ax%GkTq5PtHCtg1`hu!-y^c30Z&(*NMkj0EGfv|dK0$c7
z=KAPW(z<al+TIG!ut{?r$TKrHx*Iqq+3eWjRVz!@4@jO=zeC?BUlvW#o<;=PE=-FW
zvhRPl#AfoIPOYt6^4L+4;zuWR*utJm%CI>+t%4Z+7&Znt$nmeH)yFEP%?8CQnbU`1
zxG@;eUHN|TUnN}j0ZR-D?N)jBnrD8L=Pa8MBQU=?QEq72MblXSi)cXH3gv#PFh^Oa
zkY{#c7wF&7uyH2r*CpQSbQ4qT0F$QZ+n*Tg4uuy0xyhE(+{b7&{;QZKlF^EZ42Er|
zQCipkW)$=fOzLl*_;Sd?m7Ocu_cN59$<ztSB9j7W|0s|sAg#1Q(G3}>&cmK)GhAWH
z{-vMzw@!-WqQph|$@V11fW_IwGK@i&NS$L>@{R@*A19XWFWXz`znYM7u*r7*Qa=zk
zbMJ>yCQ^?aXUW*(0j!A%b8aAmK)wx$&;^aYL|pko_Fp9UccPn5KdEExFT><Ju?MI(
zTt!tzh|NZ{u=lMX6?O0g{3?OZm>yCUc$jbfLTW$2j8xYMn`2^zTGied971i@i=Jq9
zUY#G3q~&Bzd}+xod-<AELs`uF3U_~I#!q<PZ0VD<E3wbvgie0blv-!Mj7now4t&;`
zNE&CLXV%O5i~IuqOO6+_fvjd4k7$IWUIq!=JUrIzPB2yRz{+KUoyh3U*SDZuk|lgT
z{;Az<rdzL_f)~CHR@2h|h^(V=D(~5gRC!HUJBlOlbXOr)0WXu-I0-D1l6km8uh?ba
zfVP1qWq}BVEMQIpfOi_@;|ZIU|555CcHrHM`P(-@0#F8UCd}c0iN@L9EJ<dXmb*X;
zMToYU5tnB_1FHoJ=oHy~b=(n1tBEFkMPXIq%brY4IK&k<PeY(^Br-_#OP=TwjA{r~
z>(W5D&OResz<JY<oV7<5dcs2ku2exoez#(Xha!<Dct~rtAHuIuC~!gI1T`qUzyo;5
z?$Ly;A5hRzJ^H2}SIze*iO6yTxNwu7lTo>?8ckwrOttz)f$KqW`C&eqP>j-C^4?S7
zW=HRN)mr2s&I_lLqax)=-O_N7Ml0W298@?a%o?4Oc`IxyI8v{4|7+NA%IL)`$VZ}}
zI4A1~V4il=$_z+2DrzwP1P^czhAJ<fYq~oqp>DJiHrKBZ3&f}VtB}0+H0A_7o2i!A
zpRL}d@1DdaF7<A(cl6S{J_j@ZotU8l@jDlae?V(U#!UnR@eH(cSOXBEfr~J%5;Sl%
za(Q|jKbb<*AsME_h&IJ<{ygAa$@fVhRV?EuQV;><T>Cu*ymxbmw9m~X9{jhjL;g+R
zrV66!pB57O38mEBo6CSifShO(RDjEPC-qdufiCjunv3caD#$klV4gWL?;>Q9X!ROG
z?#0URPcNQ+-3Edxr?g$?#*)|R^j|S>d?wUYPtF;|Wk3}a7%DdJGQ}05?xt+EWu5#8
z^%JY({wVHI`Vr=!bCvG5dP3~vl_PmF4l>sIiz8ql!s(*MSg^+IdQxgk|0s=8u4?7^
zKew44dx-C;-+XCmx*kMF(W=!J7m`RL<?+1*O3%O=fK!6I%6bE1&iwGCi_taWN_+c>
z;pe5=?$)OPSKIYgsfUJQMMj_8>f9avELV~J$BnNwy6waPsClw(t<qSJvAtzJNna<g
zpcb<xf0)CTrzGUTw}(ZJ55^!TSo-=YYiXN6T4(?6<$tDuU$m6*PynAwki1;1VZ^VV
zZ)wz<6OUv~ShyF*fMgf#b;oyz?z~VSpUoQV-(kD*!*g`BKUM4M`zH*~bK+GZpb#<G
zJ-C9j|5)SQZW?6gLEyS~$BQPGpBi;@xyoC|<F7V+4*PzL;eo+4$6SO|f^GNe)AZ27
z^(4c+7R8$-Y&4gI_)ZJ5$th)I6+QmXo&->R619`<4hqOKjglf9<IxOr8PS4MYBM8)
zMY9vVb=3`w?zhihr0Pb`83t`bId2*3_r0HfvZqLW=(v{R*~pBhy~fwvUrTX^uCL54
zpz9M#EV`+FZuU+AMwQ8tl=328&cK)$L$|8ONTHUysQp}V0TK~o@$@t?<;e^}VlPie
zm5{)lEZ`R)@Ry7-23xLp<i(!4q7WS~$8C`Q^c41Gqm;0*a8D?Y%g4w2JcTU8<uo<+
z<0$kC=Wm=fkL^l7{@uvm+2ZxuKJiKBhC{Z##cH9kv<uNN?(qDvXA|Q&w#UkcuDi8s
ze-3kQ^!5l1zW+o2<g+Q|a{>ZU{vr3g2;t%?71uuH>B(P!MDf(=)ivQ5fd7^k^nV7Y
ze6H_txfiXh8%)3c5cd@zR!tL)pMR42KS#yYkkr+-TEjZgFU&w=75a!hZAUJhA^ZD`
z9m>07a9g+IPA`GRTuDRzkA7fSMm6rZ+@n&oz`(UDsDNge11C$8rve)0^aC~Z^pK$R
zWs}}jLF>42(Iu-zS`p2%GYy><WD5o@;>xQ>-E0Sv5I`J4YfSXl!<G6s*xWyrUcyFR
zKy4q>_JquA4}|}bc6l2B?TsWXDe-%<5cfH~EIO`v-(@ZXU|#YSi_QZt%KhEj1+Fws
zKrT02@%{bSxyI!%FP@_xqWl!n1$YBbK5p50Uuv07N#8#G5UE&PWlDM*04J#CPd!g;
z9AdnPK`$K-`VjhA1jWQ~kY$~7jzc13j-%ag$IgA;@2#d_$e38YxNGVs<_l{wZ(Bn}
zm3#5@!3%w#W;f+?b@OQA%4}4thC}70r+<v+$W)v~1&tlMece)->Vmbh#zF23lC~{X
zL$w{Ij7S(C6G-1x&=Ej+Ole|nL1|6e_Il+<i7EN%Y%GXwokv87gr$Ft`!()wpto*5
za||;XOlirw=BGSZbUP}#`J#e!?))eMVXtB3m5W%x@&v|v*vkMLt)FEls-+OP?w-Y`
z^L7v2+2<>0uO7(KP(>V}hzfBXivqfnLggffI!ny`&)m;ngf7V@^L9j3gR^~<MzX$p
zXjz}sb?xS^hn~erJWcBy0~v33aK5P+osxOB3J*c|SIqZC=a;6L%U;g?T(V9XrM43*
zdS6KC1<uF3hK9I;>j|o0{(%r#%agb}n%1)>{X18eE_O(R<KE_)de^%u#wuDDec^bM
znD_6BjQM;A9NP-A#OG7r)7_=@Ub5}iJZJM?@u8;`=^lShUEPPMM9<=cd+bbd4N9|A
zth?iYmx&L}s8SRxUMiNnn*CZ#Qfmre+Vja&_UKV@To*}Fq-eqQuAh~x(X%oOs^hfI
zd=rDheWER>InE_9izPilpFHW+E)k0Fm*~ZYp2!bPCm19d2QTZVNJoA(w5ObH1CXcN
zUc74D&QtazJx#ZGj&K0iomr1>kUvHgIZljWw!-?pv5{Ma-Jj+~7-_rtwP=S^@ttIQ
zI33${G%kr{GN{;l(<vLhDchj_r7bz}#^6JePgNX0b3ZYref3x1aiKZAkIsI2&&yov
z1y<M}4aHfTyu^u=_EG9%)mU6(^{J$f6lkE&R74Pd17P5?5E$$TW;{vwhVI8_*Ct9I
zOtUefrycX|2^si+D=S8wG>G=xn?hR(IJhrEq|Y%<ORD>21s!Lyb|_Xwa&&$r)zN)@
zSURC3d9UpDW9BbGvdh@{-Si(=z04-h$b>Uj()HfPeIpMw1ug_1f3_1b_>T)S3#P<y
z8jU;6#sCAJJeN+lA27kS7e3psKIC3_xW?i{9v<B2Oz)b$3h`UFE=6UP^@peR1bXjF
zMVfcJjptj3rUoaqeD;3q7w7j>^5qLzXlh{<W@9+sOuZ?^Hh~{gw0n+I9f|l4;{f8!
z)(q5rYO9(=!LGh1OPfYtwfVK8L?B|g)WmaAIP(4K_`^4veeQy*1&3LrR;M|v-MZhr
zaqTNoe^7*->7QfwU<?gD{=KFV*qTULbAtJ4IGUZv)BpV5UM`JAnLB>6bxVCR4%N7J
zeyIORY?j{3(W_r;Xkm{22mok;?lucUeroQnc==}L*d6jgVV}2$igko$Rojnv?_2|m
zQCO8hws&HmJVeg0x*%U5`iAO~nlZfun^}1O=x~YGb>GvBmI1EwH~%~5oX3{1EZO45
zrRyL0Q5Q%`L3)lG?5!clSM?XEs}$Mc)SPShFJaKzUY)ZJ$~6WO8D)>IOMLqX^iUg}
zrjtV|nh|u70r(qF<(W8jp0RbeG2f-`A2$(<JM96|+6EST6nv8yiNpKZtw&l;aGC@V
z1u=Eh5Do(4s2aKlR_r-mGy%KDcSey=MSa~zOg89g?GA=mATF!lWtJEPjD2Xgl)_PN
zyzTOXhnrMqT81k*2a6mz|MJ`ZW{!z`0OvyA^AiI-y!-p}y1chtY?#=K(+y82KI^Ra
ziAuNSNkDmH>TF8U3I~_<z3{=;MKa~gPoxZ6oCC^mNVLc>t+ARXPyLL<H<YMVp~gjb
zb%}VUH2QcM`6Bd|Vm8pra_=xjBSPI$FMrBzS)OxJ#rdPcP*xsh^Iz-qFOn{!#?<&{
zuu&#24l&^}rP&4^1tzAEf-i&s<hp>ZzfHJ)qdl?~iroM?{pTm~D0=?Nt}>Sx{gFj2
zT~gN(NGtcexph=X1^wU{!&?94Q^#Az8uXRUAGzN3tAAG;Jy{?3d~u?w+<58sa8%+D
z`Z|LI&y6PS@Vu+oYIAPAGwiLTqT`jS1|peg8F~kL;SE+PHbJ`9aRps3XXB6q{5<js
z*pYOrD;J1nI+VlwR+n7RB1dhjah&K4gR?Z{8h@@Ygq56qt?`xUEiqFkwAd-}ex^`=
zM8TfpJ(eveSK_l3`oR9q5BC#e>K^rcY~3*E`dIk1?RW6%K@VLK9!Kx>NNhnEm{E{k
zut#l`{S)ATBZB#)&K+a|210*hHVHQCNgDQwq*{BU66l8UuD1I+kZo%p`0vuJorf4@
zgV0)tN}yhQ<F#tdK?2j~xthKm19gShyKc>}lf7}{Hv=!xS0|n6+5Yu*8{rxBw*DWc
z&O98-#(U#4V=(p^`xawgvhVvY`z|4A>}1KFHAB`?M1|~9C>4=Z%8)GCB`IZ3Duqat
z_?>yT@9(<&;eD@|@jTBw&pG$G@6R!{%EQZ%A?YE^GO4W9sdOTPCd?em&HuKGr1m{y
z*E?`i$V!C>2RO<LnDJfsMg+}|pK!IoL;VtIkF1|LI2E1J2&i3AM<ozH4i>+z(5Db|
z!di6p+FZzj-jW>svE?W$zBq*z#r457$LKV6H}AyjJ65(f^Er7B&5s`ldF<d`U{WrA
zcSfIVZl4v_!v=m2W|?{6QWstY9S(l<TMX@z1xf3*#x9FBlSoS7a|@)8vXvvM{vgWw
zeL@RGnZ1t!O~na;M>R<cP?{dDvo}7dk)&|bj1HMtHD+M*`mA0g#u5U1u!ONT`<h}&
zjmbj@oit8~JS(X26PSBxs`r#C0L?jE(1gmFJ7@a4$#)PcMfM^LbkoE`FD2;x7Py`)
zUExHV+L^i2`)$#(KZxPYRere>a76t&gs*1vK-YuuT@y_8gg@a6#AtLHg-o+;uK50~
z6s$3!z*K2{j$L}s?)n)#I)>2&CcL@%H?3L81C|%1Tko{qSlxE0A&&r`{`!1y9R2Fn
zkshkzyD{`l`ML!v!EYUJI%aJ_SL1yvBPgx%?y<iI89_YZNV2tKYvgbmTrmaWcAbX>
z%?d9uw|=?vnEX><6eY)K0?o8`0OBkILgjXv#8>mLrRdO_1Q^njcb|jSJm;fl$s)+=
zc*l*)gu+jeJWPQLl{uLNwkmMco{cyzXsPlwLio}!Xn9w*PLH3dA$RwmrV6l~8x_#d
z4g)ysG4xu{MS8uz;L-s`Xa}X(gj=6q-yJRsUcSYd(sj0l!HK?pW~CD1Hqj{XcO0VY
zUOEJgkL{E$z<NCR1(BM<n|fEITt?_q$T6WyEZCGAQh(Vb2pG$&N;Jjyg!xfv{aE}N
z&PP9>YIV_LE+VQNX5RT0AMg!K<$k)k9I*_}-H@DBF&=~ZhHQbh85L0y_4G8CwTs7@
zN+d(0_;l~}!Rs716Xj5l^J0uP|M7R{k#z`h&~e}ebMQo6nQYFx!f&4DL{E`drOO8Y
z+T=R>a$Gt9mYoq8Mk=}sMG`c~t^7^enlWpHsr1{f_BZw+Ja!}f84QU$-SjqNT6iIR
z(g5#<o3MSp4n@g5lBZHAUTon{IwkV8uQT(cV$o+yZJx_;^P6f<N&+3894dq9h6jjv
zORTG86E)1d8iad}p@dYgT@rbxVOW3cF}sfmcv<D#Ltk_sPJLXn*OkSR>bx)xx8I<Q
z4q6zv9Llzkc~}j@Yj6Pfa7{HY9_9P}=pS`s#Zl44k5A3W2k-sN(-X`q*~XGj35K=q
ze^n@do%c3lqKRe#L-xnI2f{g(&9r7%!)+NZSFo19!(`#8Q1y=vI@QO3zXGOJ<V6r5
z5(;=0efl|G=TIWThRWD=Z79(JGX;oYp+~o&HwIlaL~2CNo^U@qJQ;N>(KX~<>&4|T
z)|Qa>CnkpBJwQRkbBnC6@M}hAJHU11^D@sO1O~Oh13q7U4?Hxm>El36gIf&XC&2{(
ztfZ6V;2r(={lFui%_1}8N}f0HUZ=4BR4bg>+ApzjRV_?#VpM87hgaQz9KhO36kR*s
z9XwV|W9YRV3aS0A&#%tu(ch5B!#*TR2<dGs4cdupew}(<0;&H3aIM%d#tQy%2ST(_
z5<!ac41Koq{r?(NK4Cp7H{M?<ADF+-rdA2ui$srBtbCWZ=azN-%94TLKc;Fkaa_yk
z;Wlf-3uRB1CL-3ypFz#7(UbBkC@p3WVLrOD#qxVZ={J~1)Y85vWYm9jnJbQ)?qfK|
zmy9aFQX`Ges2hi(JNvak)Zpb-MymuZu@>)_2#yJ4z|c;lzM}49X&Ky4%sXl&<)SBH
zPNQydl}oUzP{D7Mk-*U-urmdM!Ly-u^$!WF<8>@?v?pw%-J;+4Flle*YHI?^_B5Sc
zsBON4gJ?5ii}eQfknk^WRz=E)D_W|d_{rkndIQ)6mP8cNDA9O;Hkt-Ms}y=Oi<a(C
zlpyLNp7=r3gZSoCmeI$scVfd;!mhO2yIeg@6XPY|5hCS0b`LWkJA|Qt5fw-&V*34g
z(cot}&+i@KL?4lp){Q|RKt*0a$mr7qUwaQu5W1HXlEEJ#h?^tPw4Z~T2N1<PHi8Ih
zh%JFD=QTn$uknpVsI$0RYlC_hB|@4?hKYCnP=^h-E5!>niA8X$gEX}zxsNi$>(XN4
zX%mqzaOsN5)~D_P0%LGn6M_o7QB+qbw{F5b6`VQ$ok_vjC=^sSy<no4x2f6Gx%Uk2
zvl3-&4sn%ldl<;^z$GWqUq;UB-vynCEa(dqLxwLVQ3TSe!|)ZIu!RKIw}#`jZYjF$
zw<}4NFQarp2OMg;F>r!cGs$)S;Tmj1ak-tU9uG4Alm(P>`b^#L5s#^q7%P9EwhU-F
zmDh%BDsC?pm)0_X=eQ<llA}VXIuAkH<|XcNxARNKmBatlKv?8m7LsJXHs9T0LDBCp
z*cP39uUHHE#6yleVJq|`$q#28iSUyZ^(Ho@m&T4GrGA%Q=B(U#(9Ga`=Vg;TL&Ig>
z`DZ7YDA?C*3qc=GrXl;>4EUHiM&4vkQg7u3C$+586qwegK$+>`Nw5)dk&{Fut(V+H
z?ptTRHodOcc)rnd?PMMI|B8<_USz!*4eESa-ACb5XGq(mytgEVJk#_-Fl)j4Q}*Ac
zX)v@lcny=WsXJje?rm+g<fY1_(Xxd<qQMe#x!ZmQ^OUE2iiM|MOIg{cWO`Ifce|c*
z>hp6PEXoyNg6mEpz~pjoFo@wB)np{D7fWL{NcfU3g<T*!B(2vAapt)r^F_C8Jtp`3
z1`0j;CI6DS|E^(bXmZ(a!WOo!pzoVfo+4VK^}R9IZ6_M%`8UF@3uW#|=OR3W^tzD-
z%(>>dycsZxK2W)rI6ymK3cuSU*wR@)d~SAk=1Kf1!2@|hxlKmAcz=Gry_H?f=bzs+
ziP6Q#csG$(9{CNI3z&*82Q4^=N}=Z5m-%%m^MtVDxj%HLu{zpQw@fc)UN{p;-({|6
zMfl7i<7WDy+AKdvaUb=Kt=CY7VgPGflz*=A@GU2COu_(rf=rj{+|%FRyuR(GT&<_J
zev)P5b>JT}C3*Vw0Mk2)m;QOq#Iwz_PonIKb>!_+<VW%Qc(J}jn%V9gk9}|w7&D?2
zaQDo`nusT_CH*<4mk77OZ!_&l@U)$5?9na9B1HL+hxS1H!&{!qaY23wzDIShq&vBM
z8&xgoy*`ZHE1lx5XEHT2RjmJ&#i%!pjzs&j8*JkU=6Y8R{V25Xxm`UuBt{Q9n1C5r
zkv@p`7vVmqLy}O47>sSuJ9KC}_YW=&U^A!91JY#=k-BgG9SHyHdcKP#+{OI1TEVml
zHqd==G(~YtU#iX8oI&{7@z_wrW+Kj6vdj#XDr8_!LRk^BAolRxbuL$X6s>4diZi+*
zE;OAPlPIm?YD8`1=kbmKLM%oJM#ZrmAK!utx-yI0<RyO2k2RH|U>2!*j`#J*D;VK?
zMWI2-y`Yh7S}V8q#*mOABpmyYySh{_lPUDA<`E_VOlAVuYcA75iGLk+RR))A%0fb2
zl6vhNb||hw?Hj-D1mvJ!3@KeW8J1c1Fb>CQ5K?*;g)Gqh+3MED`X2AJQb^A#VUONV
zgW9KN`R%x}Zx<K|Dp+G6t|XB4bQUHNUBMopYuA0TJ#zh$o>n5!LF4CS3&+mn_nh|8
zA6aMH<8N}smLiV2RoCs2!Tj}`zuY{<b#rcb&vu_Y^4LVajjoZ~^(j@O;N39e*(5ZS
z(Sv*YFk*Or!>V}y)uDSvg@z>x5ZWKk5r4|&PrcTncO2U%|LVcNMLiK?RD#gI)+ly4
z1%JMxQDv||b_P5xW0N^DKLsH@8?%1L*vz+J_55Fdk}!%FtG5ob6wFe3Zq;<&4CinP
zSU^&QF=!Wj8|v(t#pAcG1PEho_-B=>GQWQi<oB5k(=E}l3rjkutUru{K^JA?6^&Eh
zR@D36i^?@N%Zor7=GGKCU+bbML^7*s;#-scI(}MAGcj3wY^Yxd6UqppzL5@s_3-s%
zP}~BDH5SJBU9>4aEqDozD>(tS_Opzjou?QGZdh_~DrNBK%RdDv`JW8&s8M`F;A<Ha
zt%5xiC-SQ~-ML{(RCCO|W__vKPnb%r1uux+uKVv6QC*0TFN9bk9xP)Wl#htxRAf`u
z`T^T?Dvi63T1D_L=UboLr%{v7eqnw|95<N7qHqdq2xd#7M#3fgaI4SiyJ8dL96rp<
zNHSR>=Rsyv{Fsf(A*O1!Wyt#$%aBsq`6ihkftsWW#HRfsbi#~>FA@?Z@i5-KhEGrb
z@9w@Pjto~{2%NQ>doQX|d9RnX?DM6gA;lLK&2-=)ls9ZHZ46n1x5+R8DwGbHi8$TU
z+8sREFxNrNA6XUv-;g83Im`8KPZQL&K{V)fDH`|=9dKg0=~7yVKf7g2xD4?QjQ2vY
z_6q+UfYy9O!0lK9p0!!?`9_svA+yxcmVtA57E8_K8@Tjl<ZVp#WfCak8I%P%*ItuX
z8A-i?n)mwsQ^)?_?4QPvm}<2cH3ao%=YZNg%jFCB`OIVL(3sw7lV+A2L^oK2f9e;6
zAHv9F6e*YQV3NJw56>C9JuoW={e<G{pu>#;NbJ_W0W;48ym3D<96ovv6ljG(xP`x}
za}bN~1B3G?Z?K#FlX<J_aR8sXD=VbzyXqtyZcKKYv9H}}hYXDGP|2sDK7qV7x}Q)c
zYQx?GxG}YZw++Ty-!7FnV$@{aU`Ybz9W<ygWE?Ss3bgNd%wjLDi&Mcn^HT&Q<Deub
zFHx8CUY=>!m85a78K$73m;1e8*&#tpiUkYyA{Mno-aGfHi#)96S=mn<*a2V87`RI#
zVjIHFtka9PwG010DFZ+dpy>1%+L0V?S{@+J@OuWo95MTQMoNJtQ1*-Nf~^*c;huSR
z)!z^(=MYQEqr2BEnnHZyJKvQ>o)Y|Ni~8a;@ydvOF%QnB^sz_HFiG3g>j#Hql}<xJ
zt}J#HQMJSl@)(?_@dD|i6Rgf95uag-+Y%N|TgaT-`xAul8}e@%#<AU_hyz{F4>=FC
zZ_l3__X<Bb&IPV;&o{#&Qr}(jSr&*_EE~YB4nsB3WI?BB*5x$_q6b0^{V8|`R?f)y
zB~IV4;!(s3B-vs0sM^y<!gw+JXZ8(XRt{O5uVkR{oF#inmGL>$$O)Mdf4Q?}CcK{S
z2wH#AY#6og|38%nLpG*E2lta{Ls%_9fHcJQ*vTl{+(PWpL_B)ZaohrpVxE1;+b}J1
zd2haPYPgec-TBDz&<;v?o!hu5C^eN1(UAl!=E37z95>PIPn{Y3Odv3P>odfXe5@PI
zYcj}<>od4ddgc*6$JQK;JKE&S;FEt7k(Q!P-hF5fC*5pnk{wdaS?Qg88g!y1<h6)_
zl3c7C88Bk<up-g}aPUc;3GyPKL9#j2jNX!MXqj0{tWRk&o5MetGg32f8%1(C<S--4
z(_o85FYk99^3n)V@Q+q!$LoFo*J*K6cL+)m4KIWzG6Mu4@HOLCM25H&+4D1wTw)zX
z+uD$^C7Z1RF&b$Pp1nb7Qdqv@i+e(n-uT6;CE4B^zuEoJDJZdu9PM_l!U1efT2nhq
z<%{^h1NcON0_w|3HHQ5rfY(Hwe)r?7DhkSKrY|2;DCzEvb%zp5Py@LNtM!UU>YY8L
zHgw^$0e{tj+sGd)=e<<OTvs0UYB6uWX))?#b+-8f4!fX8z~@Qbh-crSl_&Q-LKhQ3
z`wAiG1K5?h&NNG8ab{G~lgZQV5aG%`f=VF|SpBgBB5I<9@3SZAh=OP0u53ozZc=V$
z+oaa@T@<3;Lu~~$pt@0!&t2H}-TJ@4r=8a`CAoHSNSUMRDw5kZOSp$YCHM0`mQ(0R
z@oebH?O3pu+ZV7%vON<S7g@Ojn<e~clzpt6G*F3D^Z6vm)X1xN4d!r=V0gVE95`Ov
z5z-c>O89*s7-bO`z4zw&aIxy$;6urVZcHW07Y{Ck=y{{rr@=3P@5l~Q(vu*i>mE${
zZ4gNG(Cub4E!I30QlnMi9v9G^%~h&=sWE+4o172i3v2{sWoX<XD-Lx_D_+GJsZALx
zf=o$%=9Q!E8^zN6K!Zc;KxUi&>Ee7|)k(d7Q*LChgCmqZlwS6H;r3y-WoWGQ1ZWI4
zvih4qlKl{DwP{v~Ct2`6MugLP8sw-y5$TlCV0Q>g+y}L7x1>;6`UGm?N$ZCxUskoY
z&o^(CDBrLYM(w{)n#e`s9w(v>R<#gpwKhKGmNbipWQEKo%&8{d2b#ww_mKX2wcIF6
z$z^{-waxN^x{s_#OzGD%mq%R&;HClawt><SdQrV=0d5b&eNN|$J}higOSP-k=?g2R
zFixpyO8Q3(0zGKH3~}e0PS~Y|(=Y6b@wnC)SE0M!<A=ue<i+4|K5#M_jnPHs-$wAn
zH}(2OOsi8Jo#`)-6=UQ-x-2?@57_~W$rresf)u4D8a|5Q5X%185#>qpUHsmMP`2&w
zBIEq0(QTO3lUrNsQg}z?4Qv;Ooa{WM=kT!0Tg^Xsu*WbQqSpD2RVJ7JL$RaGA5w}6
zY!(O^UH-z=f{lE()SPrEF`JQ#p~zCYluC2C7W&Gb^Qd7$gxz08U-FYrZx=^UrL?hM
z(B@8Rig=doZ9bd+7;D7^o>|f0^K2rhpq!?=PXQ+}@oe&Aer`^(6+Mc!zIzk{(-VvK
z`^{oE@MkGp$Fk*;C%Fl)_QxG;MxUBq6r9{=87ucs09(u{SCPVP%AiV?AH*x!#=~UG
zW62oDH>&z(n!Z3@hE}+PPt=oec@K9%8OgRFP$r0|rIiG&w1bWZ4p-5e1~3rJa?^{W
zw8k0b)*=geW<B)9T}2IKDB^A~s~^38A}xg!fSDB%!*Ox!Zls+~)#0L~rtZSm<3F?2
z^R4W)hpFNhu0^5fwL>sv)^myImzDo30MWvFv%;)sEn~FStryK;hrxfP*Pb{N-zjF!
z`~Ww5ZAIl`$@lGZejD5brsEZQ$c?+E$<KAw>C}XSDnBLL?(*hYjl3aS)bBoA;*MwQ
z4~n?*yuqa?g0?(C<&{KX8<i5tib&Yow26gKMhR^pH{QsdLh-Cw0YuWGd4F3mtS+hv
z3kh9Cxu+iC)IHs%o_4JZ*glzz{?#8?*{PRU?dVNoi8}}!x>n#7YKwL0NWRq`c!4Vd
zRi*m^ffz~Qu=IXWS?++w%<s)lhI^)+EcRLEX)^>~!Q2d6#w}NS6}ejNYOLowtX5*`
zsc2I!A2cm%x%nzz=gY^P@vh+)e{Oz_s9gJH^l<HlX|<7%-ilwbx8&#&U+K>Gf$Y8$
zk=<kBA^1=}gSCwt1(<CX!hz+b925WbT;6txiSBOIG9_v$tK0lKbzvhI!hu=cz?K&-
z8^1L_FMPET6GpAI3#%)8D;a&teN@|kmhfICcHjzyTIf@m>7}6o$y2C83ix^~U+EZ{
z4<qU$Z(<ih!2TkhMO66OSf+Q1*pV{)L7>};(k2@}Oa2dWK5hCx$gSmSErFKa_<5CB
zQ`5vMt;i<f6K>3eofn3fDf(T-?hokqauQ|BF)H0t<2n|P^(FhT>L@P3*Z1StlR24Z
zPo?apIU(kE*XMIr)tqXeeo{J(Zb_DI$UoUXgeu;Oo|$&xPg7vrZ9_2{-F!~`p>w&A
zZF^CFEg*MM;AS!m0p9p_nN$Qhn#O1op3i2-5sK3|RnxU@$NaC~jwwZq($buV_=VA`
zkWD0dtlRlagA49}Y3busaL~3c9x}}q$JHKttTOefn%*CD;!Ekf@tmFv6)92airc@l
zkL)v7BwyDth2+bpOkW;B3d-f_#Gl9R=H<QnPp-k?*P!8Mp*H7Y5F}1Ku!uz0HD&zx
zYTDzDpFQCfE{n+a<2n|iTcoj$kufdN-TipGG$ArXfCiJb)8jnpFY<{Sd*I@j4s9h%
z@9Rt!G>HDI(nd@%N3jNZRH??M_aB>N?s<hvM)A)UW^((TIh#=Oex%UGq`bCp21S$G
zi`D@+7yy+{eC19XM#LXWZT$t7rfYIn(lgbep7T6S_wFFEzdYVerYwH-mT&|$GLh&N
z#c!nJw73xEmRL3aW9Wo?8J_q_Gx_?plf0j4)aKaodS^JJ?)gRZi4>$h@cXf30C17e
zIjf7N|NYT_e*>JWjco|k<!fs=3YF5aAjiBe%N)Dv?cwp+=6&l`q9^;KUt6YXIGFKy
zpA$Zad?`?TEsLK^)nTKk7s^~qhQ8E9>rb}-ioAF?bgvy~4${Ga8Rz`+{;Uz68CnAz
zAwQ^&%3Ta<2l6zAUjhZNscAqcRL3!$YOlM(zR?B=l{EH@`+p$gO;d+`<g(lZKlK=}
zXYeF7)x!8$FgA_@@~^fj9IVplZ2!pyfyT^|AAJ(}!Vke)sv}KPs`oAx_za|)X})M^
z(~Pp1x8$TLmnPv|3fJB6!5sS=-vWkWy+ahn2eX%YNXwWi^{Vc(Y~+J%?7r}*R4<25
z8y?IH(GvStWUI!i4)~8$AZZx)`J^u=0WIRf+vi333Ir1nNCij3Ah$Wm4;1I;E!O_*
z6X1?a+CT00v9MeI9QY8_4+=T@UJyEzooV~=v(vwa2Y~{;`8^DEU{9?Cx1EqJWIjR6
z<Zc?NWw@E_X#i&HLc;3`I5o*MQZDzZp%xd%^hOraQX)&|3$|C2_KWp;hLAejvlL@W
zPPd!}U0|O5Z{KA?1MpkM4H6u}c*oPSXV2diud6o=oi&~li8zY^TeG0siIr$nwMv)8
zXV_jfP^PW9H4S_WJx{lqwd8#WtKHtTx~jyb5>%+p3upxf;5!0D*DdMK53-pM1|tt)
z<nZ?7+vWiHVqMO^0?rbs$@^gujH%^oFai*!PR1WR+9Gvu3aohwKab05XvJ?yee!@6
z&)h3h5`^Utp5GkUyHF7<capXHF+l~vsagLXz`Dt}t<bWH%K^3`{uMOjU&iYUQN~ZN
z0hrd9{TcH5F$fz#zxL_;$p{T$!^*E75GsJn>`9=05#zrs0I*}?qG9aaQq~D3OEATT
z0?`_9X*As6F25cQqYuoXIcN;KzYI4sP3!~L`+2C&G<qp-vF)yNV@jhf))EppJGWMK
z1LK}m&-B0w`_$cc*MPtj0P4yRPQCi*5}ReXpI1n&!}B!&JXf~JF+e|}8WIUJQ2>X$
z1!iG!cJ@%<{|4llO31PavT~B|UQKPkuY-P3AbZy`U&nI(yRAk_Yujxa4GE_oQZ3;4
z*CCJ)6K7K&`E>OCDrtC7|EQEM%JtjS113u#e2{S;r@upX&7*^yxoisZ^>Ke39UO_Q
zuaamO^OYE4--H#G$=z{4n37U%3@%pmyMTNW@?K+N7_M88=yX3*INc;0xfRNlwc6e5
z8$kf&nZJVP`x1R+bNiXsHR}^x^uJxsH#q?u<&w)eJ_xGt0d_EcL2)1A{G(I>T>Z!2
z!ZpAZkaIwD0=59AcTe%ZOjJx%18B`f0r$iohbJa$6-xpGJXr|SGhkcuuQ>4u1kErc
z@CH(?lSf_Diw!dOTm}@;e=-VBHaO@JY)Rj5tvX`SCCa6&(ypwBQ;uD4iu*|Q9>}>R
zotF?IjOA!W_BiA>#tT#e!l%riGel7x*L&%`Y1BN&q72-sar8U4N5*dfN1}H3@P7s&
zh)(K2{{bQytBjl47)4zYKZ@3DmO?m$$z^w01cjtot7LLs%`;D0cSd{Shy|A}VmHFy
zf-QYa>WoXjOP`Bb^p}bAh7-@gv()%w64wx}lCgNI15LUFJx$^x*VfQVb))z}^Wr}z
zXvZrQ2v#B1jat5=-&XQ{vpt8X5>)4(>?{>8)!S)TJe13*RpjL6NMwW#`0THV=uXee
z_ctLbXL{bj^|SoBZ5TLNSp@MeBcGue8}-G3&ZdG}Vzr1{*(svSsBgT;tDK?!JRbB_
zP>2kXJk)W{k0R^T*{YJROel$tRDA7Xm&?S<Rcbe<!3OzJ_4x0gv|1D}t~YE0*aKax
zthxzVUV~$v+{0zs#c~<qZfZ)EENf13HLyd^g;o3ARcziVsHmXZzoD(lI)!vtIuM0;
zD@glJf>hNcS95_^Jw=~d10S)RK;G$Dc&@3teW3o&!NV++C_#S8UK-_89Xpz+$I31R
z*x5HE9Ym{9H5zw@p%Aj;oM+yki%xc0J4B;*TfeZ0q2+i<g2clmMNBOpR<36-Iwe8k
zv|%yuU90?;eKJ=@q|t_$_>xWgS|Z`)B=0>%@+*eCx%Zr^+e<3qtcdC!<%CkH`Q|&R
z_Q5yU)fQ?kR{v<=yA^i3S$Av_bB$K~3Z~lR<Ou>V-cHsWy<=?G=~;YOo@Z9-Q~+UX
zYm`IIWnye856TXYZNdww?49Wz7oy?BPFJ9J$M8v_`j-TzQC)XF^@ycNSEQOOy<K-S
zF}`Dr`!&tD>d=)A-%=3^_7$NuYWedn`%Qg0Rmp~%3f`(@e4G07w`Etzhf>po-suo?
zt!Fl28!*osKp#<*EF%>BZwQKSLx-MUoTF|ZU5maooo7h=BOIM5$8BZpb$?w39z$*M
ztHL|Fm=cYsvx&#H;!E$xyoUkAs0$1M1o)F_oWIZQ4m&B9QP%G!z71`K=yY4`hQ^mc
zfgd-<?mSXAIz!^94IZeJ&JdsEvy7iQ^Q44k{-Z7=m74jM(!L@NCItqqt|WjR#Gl1>
zbN&FKO;WADTT>RrpmtOv>$dw7?!%P*LA!LlVrXqY3qN(j{t%Ufv{$BnobfjUun2~=
z>PdLTVYCC?)G;1JSwtDD5jOOOO4oaY2~8v~JgLbNdGMh#&88RZiJ?!E7WRjaH&Y>Z
zY9fD~_)pl&wL}Fmfg_+OILo``NYff~?qmI!*hfvOpCC$mT=?)=8f*zu6ym1O$IG0P
z;6baG_wm~`V81@|=LdO)YJ2WpZS)}-xv%mJp&GRg0gUp>KUcL3l%=*~$%`c?(L=8R
zcQmIc*iYz`$GRgGsik&;2?i?D^7lyGGjFE9cSIwKQT0DTo1$}CLRZhdetkl`qyYV8
z=8foeA3+?Umvs0YWjzmgpnMkTwFCKZnQw{V$*nnV7GlwE3iv9lfU}T{*t@4tv4lxR
z<#`c2A3Xs;!}^cV&YB=M^_ABD6|Y8@5L28w`sc{g&YuDh*<i|T)w*;6FLN?b(3<vl
zbaeaIJw?n~c}@EH8S=2v|7U#ItXN8^$2d%fcHfAz)RC9`_3)JYcK?}Hl3Rt$hL`MH
zcViJdib>y}@-m_B%v%cmQC*}{V5?n9lc7*O{biM4;=NyjZb3T`f-OaOJ}t3o=Q4DB
zDkjoVTx1+@xrC~v%Jjr2xpA&JAufG=>4KSPZ?d}CFD}vRn;Jj@wc(IhrQe_nrlFj8
z4GHXnVw|u3I1Mi{=GEQfGp)M94&kgUTQ5`S(GE&4JDP~5)Fpp@GkV8^0?VoKD*sj5
zfAyt=Ad%)HJQ0JduCpGy)$Jibyg&fkbk`!S#mUcIrQjnB{g`;su}(lQcOP6E%wPG7
z6HwFl_Ilz={C8t&7S37MCj(2=(}ifPL1(9kI5z+YWX5x0O~URC1mq|BY-Jq_efCt)
z%RP9Cnkk)Uo0QLYBXg+id9Vij4`Q2slybW`L@_7|GV*_;3OdfxpIeBT98)yWh%oPG
zEfj69a6T-j+%Q6_PB!x0dO}W?b8{}xB<$3?AdigF^PkR^3T-9y4R#5bAq+G=vc~@q
zZA;4Bqq(2SE%@po6IU8XpTx-P+@r}H83Mk8E};8Z)LiRGirYE(82Opv-nOgB8Ejs$
z#Xvk@<0h|>%1{UCnf2z+dVYoqaF9B^kYm<3IVG8@kq_p#b8Sxl#j%omakJD{YdAjM
z(Oc}cA4bv;j4e#1__8R<4*Qf~O(*@&UyXrn%b8+tMAI2|pBrB5Q)$g|?Q!A#?y|^3
z<1$oK&a(1Vo|kk8x>;ae$ue-GH}<ntk}o$rTm-+pO24nxbe)Gw@7v1^(XO8Q#O~za
zgqLGt^g5z6P|$)Sw1-$8Rj<7%+c0WX&e>h;;61*8Wh1H^iF61&Vv5Lg4Dxh%_Na6c
z*;3@-+PR0bPFHCYulVxeR*6Cf-Et}+wfiaYcGQ)}Qn#(#DI$@)_o!}&Tyl^7z{bli
zUc7=>Hpwj1)7`{YB}K11M8BjGy1k(t=}r+YufY>>yp>~3tPu61!#7w)U{U}lQd&@~
zP;>w*3TEsr>$Kc=MXwPBEP}G((wP|DzKpXl-L@eXNVzbZw`~(~Cq|RpPF#A8$FmnW
zhWi~8%<9Tyf}z`UGUm3;fOy<H@~8Vn755L`8&ZUOH)BdqqK3X`Ee#A8L!i`#csRFM
zmrtPUnLANLNgh+0@7Z5ExB-P~QPk57R-UC2Pmh7PF_+cjzB#3)=*amb$*any{VPFK
z)cchW!}LgQoko^$vGA6TvZ(b7=<_DrF&6Qd=T0ebkQyDOE4<g_llMoHr&wW&u~j$M
zb!^{JAuwf-yPtB5y`sR(TDR{~iSNJxk+d+!^-o+i^ts%J&kajNWI9lovF}nGRpZIn
zMran-@kYZlH@cc;x)Pqbd&rT^Yi5XcXn#jvn*c`pgAHM<+N>*Hp|0c4@J&%hOe+i%
zsS~udDgT~@l^@NSYBxK$>|m2=RJFO|*!{udZ4_F^*}2={Od2IEN`VpEHd3Qc&jdqk
zfnDK@t%)Nhc`xnG6tawnNiIFmLkTk`gY|1oE!oR!-ttYUqR%aI+0g3!$U)2Oax@su
zs!KiIZOwlF$X9TV;{pG<aqDdUsMB_#&Lkr_sYX8eSg=DdIWfU!TC~>a#0T2_C};lJ
ziR8-U6-+0SDzEnETSC07uX^?|%Pc>1*(I~|glRz4ql&jN5dF}5Q7CVrcGVj<%awfq
zXJ90D>ppkvc~a8*h9{yei@;l4<3!Hu&wooE>3*gAb&Q1#d`<Lw>&m9_Cp}g=mh>@R
z<)0<9(axjmv|bsXB|B)A7cR~6Bdl~!pyfiug7aC?b`*SZ_<W?DM#{*gws@y#^z@Tc
zxPp?8d|pEb)-T#C6Z@ZAyef&bL*W2Al^oJSU1RQ^1BNW-#C)-Ern9maL06v7)q^JG
zXtTe5!UwgIV(J>B6E$EufbiAmnR^zC)iiKQ3Y)i2;LK1m$DhG4)O4Eb#{nkvCh|hY
zW6t&nd>wU>XXYCBTOZNWNPl0rm3|Y{qxECvyBMQpMDjgl#=AO}YfwCX3_cK9KeI%a
zclYvpt^NVAll>D7Xj`nU%4&=KU)kzRwqzO>(cE+w+7##YpZo>QFSm2QO+T61zjR`r
z2xGLcP!{FC{vrG%h9?>3U@u?8En?`gM8@sFC}FZgu1-)6JXMgZzwz1r7^?(IR9L?{
z?nt}52mmt*=TH5dm3fdB{V?X+(+7DnTjamvk9ICvAG0>MNWbnMBw%WIm>gBoQ^J6~
ztOfi@P$Oh^6S#%mpwpT7pN1`TgV$5>&5_zlKLEdy6d#zU{69|u{9qL>2253%PKEwW
z(fbKLCbN6-mrl@J7czzH#9sgbc#rFc@PKc%3;<Gn8#&<GR(v(4xMm4hTC+G5Ai;`(
zjSizgZNZbMtRmUtPl4qP><8uKr4l)Z%Rte3_xX~|_=lDe84lSGZBfRa<P+TZ?ZGME
zFAK+KBRD_AVzGfR9)z*e9Q4TKS1^f<1PX%SEENi7XX+4mR(I-co~McxJ|jZ3;8y)`
zHSo@x$EWUoyYQTupYl%=+=48-T-FwELU`q>p5Wfhi6K4#!hVyd$SSnDjh=%g>KnHo
zOuIG^Iv}n^wq^kuI6iXG1=#OsGWxxli98^NITo2o9k6Pyoe3*Fpe6hQK%~xJ4i>f?
z(|=%QjjX%Cue%5B;bU8^tV`_VPh?uzo%15dyzt8Q@q^6hrQ!17+oh@}-Gw!dl&2H#
zy?Flpi=h{_80p57VY7Yiaw-qquAWNxP{Y=$sSN(pG0;syYWM-LFMz3?qxIc%vgd4M
zhCem8lK<ME>Wonw{)a3IsEweX0%vR1WCLD2r~S?Ij%=`+XhN~{<byR}Gi4Zkt0kOK
zTtQ}kU2a6g9Us8I)Q}>+!r(0Y<B+rzoW?A}Svb1N`94CWCL4v8WYQBD?DxUECksO?
zbT53?V5tBg(DtGA{;@nt!8BQDdR-qrc0kI2F(&jsPheQIrU3gFuNnr{do$~q*n%B)
z+AMSg&|l%ag|e_i?v8JWD9BJirtK3&jf(1dSHSNrQ;$ebj=gJljY@fxT~J}+FtE<A
z+Bmer_%il;48SxfE?m*DSO~oPWTJ3t$C(4%c4U1CNo^h=(%uCgF|$vHp}^%6B%UCE
zFab2L=HVvM-;N-gWFM0R)Gd_Oapd8%V+RG3Rw&T36no(RkRy*zdVtR49`>AVSz|}C
zSk42b0|p>+b(|#l@*d1u$ubEzWnpq`s0$HfBL*CH97ecDU?dEr-4zIGcamN-&vinN
z4F&iBuXT|^nu_U%6|&QlyiQ#iya%DTUGIZIEzPl8<F(SY2MRtffHFQ0IAGXhoI%!-
z=*3~Rfc>r%P*cCGNLJ7*!MGapI|i^}NZwg5`b)Oo<gncwLwt_xNG6;!RA2j^ZDh1J
zFfc`wvNBR_c<pvHq#-&n(+qc}hp<=}l=Vw$5jp!s-wmq@Flb>?jV|Gd<U%3hMw|6x
z&llreJqfW_fo~ue!rB(IeHs?4g}{VSXA!Uh>*ceMfjuYx9LAb|Y|?7wbGDUcd-BTQ
zD4gIXUzlKLr3>@;ZqsPlDX+Qn^_4x^-U{!eiv-}6q~Ss`uR-{pq3pxOJeUuAx{&q`
z!<3P1M~2lg;bx!@yKHRCXu~F?M+T!L!4hsaw%QJ1<SXK*qK~eQ6w9jsdt+!h*jRM0
z6zF>>E|3z<`r*b!){%Zw3f_9DUdb}*X*=w4iE;O}gYTOhX{9BmOQm;Vsmvld_(rx4
zFz{({wo^Qc1J+7Vv8dHm&RA#4V_*!(d<}3Q;JseEWmT^82j-Jj{RD-PJYS;|dJjqQ
zkj?@?ZLi272hv{I0aHrc8a5f++8e9HVcZGjPHaW=$EL3WJ1i!StoK*$SHv4jfIMOL
zG+;-@yyR2QPs{t5?A71JH{Lw|jFh)B_%h#&BmC)0JoR+m*ZAxOM1+r68DI*$Y)2kN
z1^W3ozt92l*k*vn#S8dsxmSn%Q3*_`A=t_$ShC<($NOjlButMM!`g_K+50{!V=lrG
zy$XC!5a*&JUMxBuZ}L5_-qnS8MFf`VLsG<T^=g)h{aR+VW(`#Iw#zNGBR?izI^A^a
zb4v+NUd3)^M|~cD2>uD@zQVYVkB1+006pvI(}dWUL?WHPhUxB*fCh@#<a5H>!);Jp
z9GFf56i3!=LuBR-Xr@3)<*H$~Ro#`BTh~$mosG!aj9JsWoI_u)HM(v$+iqxA>lx#a
z!Z7%gZ;#U7t8uV;ki6UO)!`#=?#>X7)J?OX0h;5+P$z}TvPNh|^WB%Gd=VD(#(LsZ
zLfOx+Q#}b}$|r2C2%2oD1i96jG6{TD6T)gcH~NGAG~WK`b<FbP3jfuvp?VwJM@QRO
z<*06qjiZ}HTE5B$&lYS$VbYtPEPoE!wAaYcFt9RGNs&=`zSgBk?)cJ>Bwa`S2cTo@
zTW0K%HoS(HB;yNje1s^0yx@&-a)JvVj*f|$7vg@DG%=Rov{P}_cURVH%;vf}5O5tq
zd$BC4c5|k7_T&G2UiPwjJlu)A=m?r~$75hbzBJMXR?C9j<AeblHJOc04#ah0m^>-7
z0*o!Y4qm0HtIZqzFW+P}fgQzy6$?Ud1@l1)&%{b)IR@vi8|8s7R!YaznP#?aQ*)6E
z?s>8mEqXyxvbLcFtVf*QpSimm$~BuZs5?NLv%7!d)!N&Ia@;;@@6&*d@UO?h?d)!v
zbunz&tDhk#y(ZQ1Kz>Gd(roTzT3*3NTI=Uu>RJ5UA%gH1UlZVwQk1bTltM^te>p;g
z6N^-ju1%#MJaCJyOKg;z7R5KQbJ0t^V~>pM8Dybhf3BvuQP?S@xw_5N)NX=Y^<m#X
z4l*=(7LHicBL`*Yi~aaE<u{cnRW>ahBcdrAOPcxmr4&ab-hp`Nc7aVv=M@~nj@rV*
zzkr$YOrt9@aE6QH6b;{Ha!4e3+WE>Po6gw1TLE#vvC;mO0#Mhv@xgNs8`!)K)VNs9
z-9CSn_}ii%QvUhp%@1|(Ue#Ye=j)Y!o%O=I?g*aSXzd2E01-QVAnoHJ+Q0I3s+A%d
z&!_Q5%ujy6MJHdWsYX21wC%9Ng4?a8VVm$MkW;w$Pk$8nM&&y-<Mdrxzo=I#LAsEm
zXfn4;>bv^$ecZlFtIFEG|6N+Uii7pSz3HYr+2K1j56o$kJp9i@8DhBLK?S5oxd}V8
zIY}zzWV-axmQ>DTHAbWlXgT)T2a~n!qpY)QncR#;*M&mo&QAeG%hyTkAoDHC{8}&o
z2?}qyb-8!E9!o=6RI}o2-OsSj*?ZwKlTf1O*uHM(`S062HY~o!JO%_C7DU9a##*Wl
z3tC2iIOzO+!OpU%Fl{KSN$XG+AzIYMu_?7XvWSu**tu#W`&&@97vJLxwM>(8!fS=l
zEeY*jfO~HK8hX6xXKw^k`;Gco=Uo8vAMB<j9@7_20IeXiX}OK}ht_XN3^9P>hg+Hi
zvPtepW0pCw52)kAZh7)vhuUCK?U}?O6eVpor+kW?t$k>Oi)mpto0OfJa==(Mnc()-
zYK5B*5YchaUZ+tG7Sho%v0IyKe`w>7G`T>eVeM?s`ZJ^CRp|y>()caHbYA%*Z9l5}
z%wj)WZ4y=YG2m1y4qM((ZwL7%fVi&6)G>5Ls#hJ*&HFx2X}9kROQr1CM5=7aE;tQ#
zo`#vnGW(97k%j-%8l_J^YjzMlt1H!0_Lz0bv^l0g$<4go2h7!^H<w=s&a$DL|2W&p
zSILi>)e7Whbh5iD$VjZ+KZ8rgrPt7H+&|-bMQu>#qOE6j8r0zdl{JCJcpaLCU^0hs
zgU|1!*Yp1H*Oq6?<)tt&dOgms$16~Tz5}5z`c#iG#yq_3?eXmDyU&>3KSTB<iOfph
zCd%F@b(a<0r~aL<bC6|<jWXIqK7r8a;Uro}MZgoLsFkWzrW$}j>8<8|L-o5Ro>>Dm
zuUuh5utAifwlIYPg;b`Q@9#zU!{VE8v|NzBmEMZ>p36wxo8+~-bJzTtFXdJv5|4v_
zV9+Jof8?TvzZZ{Klb+bFLt(f0(sJr(>FcVz*$$2a$$vhR#4>N$vX0T0Qr0vu8`y1w
zk)d%e%X%<2mZb33&`AgVtIF?-o1R7Zpr<n7HqD4Y-YoI1gi=<AHxa>Sm=LyJec?0<
zCG~{fGA8sWMy**=TOVP=QgmWUGmWxb;_<|(^d1@(k?|KL-6;=r5SBATszWMQR@w|W
zI`@zmZMX5E8|U7da+^XZ_8@w)rL*U9=8QW|ttkB7xL?y~+gj-}1+I<L@n2zP<=;Hp
z(#@WC@==g1jT1gwT|Y6@j6lu&fYOr6pR4nVnzQbW>0c^MZTbtdB^Qnb6keIg1T8p0
za4%5h&ujkd2j`!mxLFoGM7mMgkB4)#<tGj3Dw@a{B(0N;{6mavpNZs^dr8g-?hPk9
zFH<dWH(4*kJY-4E(Q{+CfK78FA>HMAjU#7gP5vpwIhUsla#fWqUS{6EJ&xq>zX8}*
zrH-CokLFSDT3a0R6jG}Rwu<FYeFfu=Nnhq1#eUY7tAm_A%SbEkL)Ul(ap8D&n)X?(
z@>O2X3nS80lz=>a_BB(56P=cels6siY?s@=nQ$vtoKYFM|GQp}D`l0M-y?FF2K7$o
zNt-dgt^DB0!DC|v_%?n-!KSF@U!fcz83VgkD|kVZBZ!lqXMr#{=A47nWgo5o_~z%r
zPT8zwB<I^JYF(Mz$_=MU?}?M0O3w;9R2Y4Y2Axw6`!T(2sV1;XPU6WxPw1U`@>5Ky
z#px!?eymrAx?jJCG0H?(|BiXukktF_$(xF2i+n;HEW73U>mvdzP!p8IsdXJs-%hIC
zv6p1hmdm9v?`&sY`UPaW^-rn$n7q^Z4~c|~S_<cGAx_`nt`CB4=V(k)k|`wV_a4h9
zt=S#Mt2bJVh&C*#;_N?_zta4YRAZlGo{~Nypx++KZO}xj=)RTdfb^;0qeRj&`d37N
z_i|gd(8|;?S>=XM%_i)>B0Fi57s+;(oqFawK5dfS%-;^3=nN4gohZpgPHr{b>~KKv
zTU65p5Azx%G}B8thN_Zgi-JPM^@A1&hsI;rL(e`xb4+)3AR8g&Wy)>BYRHW}o5G=a
ztGcG;z)i3g3lkqRtnW7d>@ddM=kAZu`D%@9h@E@Bq<&-Dnv(ysv|rQHq$fcPu2i@D
zE0~`kG*B9IKZnq~^3prIQ^)o897alUhglC&)VyR^HxGWmYCHMHqU6V=Fq^eHfkMn4
zYtRqPFD_FHXNW4v2yr#KQ2$}7d?brlN>)Xgo=Qnh{W?c|3XRuq$A1yrDY)$!S=$Eo
ze+*`(IAqgqv@3_EY!9hW+eT&IWv=iSwxw%`PJ%^>7_;il9~xqMuUA06a=6S}PUtfL
zLz+(OQH|3|ou5okgq50;EzdJ&!7cl|o!A>%$b@x?gI1iZW7|p<xePS0uHz2D2CD$n
z?!h_ine{fns9)&6Ia!Bq_#q)VlI5U}U2|=Yx2A*yd0$<Z6zT{{p<3fG+Ynt=Z=vjQ
zothL<8G|UrrOYz~HgwCAe*-wV8sy&hLh&8)8BDG~*GDzJ+@5LaE-Ip=X?Zb(4V+|N
zI7QS}q~lYd<Ba!>&yvoa%Vs!idDX0gAn9T~B5=f`HmKN+lzH_XgOaadI4BDfnOE5`
zl2pwUg(CSHWkHNV236Iw%vF5zVOB<lDq=2$l^i7Lk~qo6qutW)-n5gcBXxM<PxSt&
z)?Mr#o~_^u^FINXOcc`PnlbVoxU^SsxpgNjzi}al6IP1yg{aQOzGJWdqELX1u7uST
zYFfW}?4}sOX=+@exE~otHgR1+x@KctJ_L(NcF8BV?la@1$ROEZ$=v04=7JpGWe;hv
zxe1pD%^xRD%Sqj!9SS+TB?$lS^~SsvjX=cY>XAuJdVx0U>!o^a)3VQnOn>_sYX9Y4
zi@`um%Q*RXU9}8SJaw-z^4JC|%!U^|ZOqQ`v<cGju-O?)Wpb`brRkKC*gQYWXm$vy
z;E)6JdEk|TsREDc0FMDSJNqw#CFvD<@@r4dEJO$k={fz7N6sCbBS)0~{c6+GTgSA8
z76@M>+ZVQRtuVYd^DtG3`d+?MBFv23VN8$&LgfDNbP=_y_EJ(pdes1aNp;KKRm{Nn
zx&e>0<lE+4b|W)M=_)HToQT;u*5IntS}D6x^J06mbIfPY$~`A+KiMA8+K9Tcs{M0%
zRsCm4#LUN>i1O7Y-Mgzl%Oc8#H(dUHJK1`GP8q$(m<0WGwm9G7!b*2J+qCNK9Voau
z1Lw^O>^V+u4=4^d>#S#ekz9C~KLCCC!;(psn6H&zLAT+KXL&WCkPJeJ;nZ)z*@M&P
zGI?0MFl=th5NBqkzO}WG+C#B#Ug(DF6X|cVf7Z&J%ct+Q{x_{TlLJZ5LZ+5wKKYo*
z9Mx4M?(o-9!r#X<T=cS7^}2cG!-o~XmV*=nHip4)rW3G!lPm{}v&}b=mkJu`&pX{4
z78A_duV1@ww0s@(RY51ctyCL!8PKZyJjHxC9lG27E_de;z0vDlx61x!fF*li&1mq9
zu!!(fc*!n{_(z!E{q&_O<kN`shk^BCnWA2G*2-B<Gw9$vx}egpKp1fPvZ56(;iQf|
zVnrgUd1_16dT~}XNMu+6cwmZq`vU7Nak;nkZ-ict^?AyS$>$5ZxLl2+8$6pUbJVDL
z?J7FrdwK-j-3s3S1-`J!-gx4cTAbfB2Py?-LGBPJ1Y~pr*B<CyF;6<)i|XrTjy?vq
zcBZ1qN8Tsvz-25Ce<1<Qs#u71jB+}8FI{Eqvva+3kyjg@l6M^590KRc2-p--5?imr
za9PBm^RZZRd}Dg(&K2^ua`ETNOqw>Q2bl%Lt5ak^x6{yrbx+GL2h0#diqHBl@+<jc
z3o=t<VnE3-vMR$yb9s9Xcap8vW}r%yM>hKffh0Hn5X#ky3KfxH<l&P1a+wn~CS*$g
z@e+U9><NM7tM;eOT*=|8f4&w&Q1BU!MyP#)fDFIMQ^;-)seddn)cLJ3bbbalD0*E`
zPq$SggM>l5kQ41WKn(QB0$En~d#_M+R<<HQenEv4s74j6Ov6}MK5#-xqaMTw(=;#-
zUnuJ{+E0L|KAQH}_uq7VsH<P00}^nT2oTNUL7f~Q1GNT(Eu8LCj9RXKL!-CFLm*~<
zYRte6?R_9qqY~%pVqi%(nLh83?1XtAekR6O%HvuPaB_K#Xv7)eI>~ohK9Eb^bQEJE
zLlXJf|B-8|$&aZNf07%#<wzJ6x_peYK_EZjgc0Ey^rOHz1xFVCfK!$7e+L1++5Kd*
z^7u5<ncw9PAb65$Tci;s`8)Q)H}cwS2m*P6hsQq)ip^rhq?1<GIj{pYEPO=q1BqY4
zu<MWd3k+O`)vh21E%!`OT*Kpgi&a<Q31f9K$Y{O4e=nEDHW)r17|e()Ng>64b1Li8
z!d0O%J-v~K5q?`$mq4J2f?o~+vv9Ucn6n6OWNl1n=p^JTzTrEs`k%h|E;C1ai?f^3
zEqo%=kk*|{058yTtPb-ooKtK8!l{x`3{_doJk9Id%<H0fF8wwdv4DN>pcpN^IAs99
zGQPNF0q>ZQDsbo;*qYcu;-mEPTUZDx_{gj$F%Go2ZC<kS73C(D5b6dL*xe(Fw%5%s
zyMxu~EbQ%7r!w9=C+kpitGfVs&*N$Q9SH6NX6zxzvRldvkL>?~?*4Nl#jm4bowx|q
zhYa2;iz@?fUWaApNNeb>wdH##<{U%xY=c%N#Q!HEdS#kc<~;2$GN%D9qO9B3`i0kl
zJ@$PlePhf*RwN84%E~9F-+fS2`UFa$fDb;IG3Bt(VCw|CP#UPRe?YOh1^A8~P&DaR
z<0vi2`+!gG{HA-mxeF!@<`ThTL+|bEIrv3wu_aEv(&$4fH3fmi{S$cA>9~>?<W%(j
zXE;d*qNEl(QmcRj8&62KMSR6uKF`HMm^v@VO9s%CBY{vI2sCV`&(2&P_H6FYowetW
z!Yi^@0S>{}>)Ei0Ix{-E2g~zPafY9~R1;3(wq=bvgPpTM<oSy{&nnBKr-B~{m^avz
z=i;LNKTZ*gK{BR-+Wzs8=l<&06}|K2Xp`2qHe-L<7`FCNnwm|Zwso&QIlPcj{@|&F
zkO?Um`El6j>yMBfTb2sOo1gC7xGp{O>*p09yP*xyk`e5|k~9z>j|eu4dHrGVh=wNG
z7AaSr7C8oa`%QB-I;DM-!238Ge#5f{GKppKxB8mAugD58L~$KK<>9*6PAWuSh@Jq9
zK#n{61&>1h7LP?R?+tmQYXcI2=C^e>laq5wu9{+0XL-ke_BW#L_CrZM9BH?p)#v3`
zVBUlgHel&-WeBkoWlv+ie$bGY)w^%&2Ps4ICBd9^lx8}_Ry{wP<l*@Lu9kUyL_}D5
zNSKVCgjkR2bj-B}Z{a+Ad9ql*Q$SvmRRpo9@K3p2AnXr39EJQ|=!NC8_0%U1S{!|9
zCFHs9WTd;hw0Ty#*Q5CCK1wx}xr2E&DCayPBV{?Hf-<PI_%3oRMkX^%t=umN<|UEy
zKeTaP<u|V+!e8uN?s4`9DM^9<acfD`23T=x!niyOoVnCpO23X><TlF3&E7^|M||wZ
z?u3Bx)6OhZn`IB4F9k-bmHayTBZr&pHB1UR61ggz2s@m`c7wl_XX9Q{KQX*WG`jin
z(2htnS2QZ-1O}I>irn)vluBLD_9XuK!mpxNXf}1&T(`>1LV216I2XJ<DkXgD6giD+
z$4N}Or$2c}5Ndv^AQOi2Nt6fp7^I)R0=<quIAphdA%*cSa{6w-mpMo$EzEvSWw9ou
zkQ_&(%L6Kcb^L~SRjy-{$xscddwc35^KG=$Xz#be02T(=t@D83io#o^OXUYpyTl||
z27jAcg|Rln`RdHhJB15W7m?2=+lLj|y_jyYGm)={GbikS_VtKwENqMqGBe4mu{Ky_
zcey<kX)B+d7S!pAjs0E88h~n{xx>BWeWg3{yip9nIPz7!!FKB%mV!(bmLvK!xKMPA
z>n2H!+kBA<B_aL#rEWdn=sP_;+v`a{$MHUIoOzaaX;Am(wcUZoumEwgyGiWS!7A3M
zQHWuwguLTq8;7}A<MmV5=V*BC?9L<5cQ6*n&7*dj?GRTR@$KnAir(2(Qxb8|I^ZeR
zFY9d1g(-LX)^?Ro&VAOF_SLBsKbi}$fzBQ!-a2#{bCz`Y$T6Q8<ywlP<uK||{lQ>O
zyK_Is!EoHL6?exV-SD@T7gx*Sv~67OaOZZT)mWJoFEMp^Fh{9;yK!Ifs3GGnP3{?0
z<ZC9H8V*8{b$zdiih6d6gWGEVC(&3<JJ<h}l1BI*nu^akZ*<*jbzzplxUbG@fQtf%
z;2cXlgn8CYt<vPZx@RxG_|l`%rl4-5N8(92D0r4nFdc-_$|k*|aioI+hEuX;Rl?Zl
zMxgz68Fl?;bN}{an!4f0b4kAQn~?I;{W0!Pazy^{ijdn!bF?ucSb2-C%~r7cDd#N@
zx+7-$%OSIIC7J1J2MjNqiD=}cKL3a7T%+M(x>Qnm)Hpk)hxKzR)A7x_4YHwOv}@WH
z^{yq4@o4}WGv*~&4Dai<(7^stJ19eOwBIidnv?Cen%`O}ERfX?Gigd5aXj*46|fMc
z7LnJ{+3swG#9=_nFE0K!=>8~=sLU`@2Ii&SyG*^SvTUi0>A0Hv{=DMjp0wJIW@?8Q
z-~9@;==@zJ@jORwb~s`7z^{7ki(WgHhQ?tc6_0s^rWeuA`*!#K{5GWDdv>I2SlyF9
zym~;g)55E-<G1pSl}w5@X7p>Um<t?)Q#u&9CPRaJSI7LX3tT?)h<^7p<+O+G!NCT8
zZ|ZlN!jx?a^}Ox^-F{MS9z#pj-~N36D0-A4NhiShMGL=`0Bii@a1)Q;wF?7m^7`yL
zgE*E7N7%K)uZ82&N#OLW5ja7~$j>|j4`>00#U`ErrEuz0M#xb1cUIi75&5rofNz-f
z5YmngMnCxJc+l>9QM(p6WZ}VEnXhsW7!5oafK1x2OE<-*(V6EiR<ic_G0N+B>1fxA
z>!I!`W3aw2BGd9Xk(6q;OrB7$-dmZrV}H@P5~h?H&Y@$0Rjnu{&Lbv#Mt|v3(=gSN
zM{dFNtF7RR_E`46eEB~Y+g}|Ix$nu1LD~0MX>E9O;UW;0h{@^4B(zHz*NbD@m-RP`
zJ(y|*5@BM@yy=TEXS-*HKX$nNUY7CT%ff%o2FYg8S?8s#og2-vOi$*K3QzQ9;a8~k
z<H@HAg)7FQxIe|U3k#XP7IXT-oa|<WKc;&x;)m6%`9vD@-T?J<<?tqp`jufNN7f^#
zNB#2iY_**FB4+nVt4C%Jh|<(XmL7UEE&GPM`U}ggsK3{B|NlBHQ*Q}ldMH=j{iZaz
ze|wmVIAmt}BJ8l$HuHnsqahE|EK7=8OZ>B-53*-@aKn+DtBGQ}eeb(5{yNR1E?vl}
z&LeYwOfa%h>gXbJ_L#gWt>9YfKjp4)9yA<T)wdhTXAYl6^G_Ee(T1=(h7P}k5O|Y|
z_g>~b4RtiIM_|Ph;exf(88IA^uNWf1a(pbX6_aQEw%f2;?Nz0KyvNz{xc<#d|IEcN
zq+|;Dt-2F^hscJq-k&tV%esxZVlUe7ndcY5YxX6aQc@*%7p5{Om<m5P2?!rFYr-Y7
z3dlt)S-MfO$f+4G-3qv&n@q;pR}ER*aI!Y~3VUa^RrNK9i{u()@0H`#X7t|HgLYir
zifq!(xqW(y*s9$O+W=4UA>$n4HYVSJ?8^{-7?TUU4`G4vvm*v=dHtu8E;iT;-pot3
z6ll8_ZiGL@4BK&d_E`(3Ccl#-C_tZBFMh0-jDm$yato*wTAN~(^R!^(y#C!n_Q@qp
zmw;CiM&yl5uiYZ7Xsw(b22sf{A%^Yp#TB1i{BMcYnd&K|IJO1CB4{|5i-=?hnUYie
zWk59{?|M-9KepZjn(Fxf1HSjVxYxXPWL#TDWzV?wUL{Eg$taak$h;S2Z`m?R8i=Tr
zbweSWhRV2BC`C&|Jnvime&_dpp648=Q|DxTzn}4bzuvF01ry)Zft7vi_F#5oub};F
z@mhMp<76RZtQxX4&x_cy5zM6{VfykL7<#Q&?my|QvICvLM1#*EC>A%qw1B)GdY;we
zG1;F$sCwe<`Zi&S0&ym$_tBOTU0C0#))t7#K^L`xPYyk+;1+!Ux7pH1YMMQI*vDIo
z0teK6=RzJY>)soIrlJ{izLLe~*}b;Gil+(Y>A|1gKCN2?XYsG#FW?UmQYLqLM1uh+
zqK1kK7Y`}EgbBZp6*b2O*zUq@U|tIXX{xixiDJKYtc4Gp2Xl&{@Tckd#UT<l_Gc-7
z6bj~31@b58P`^YB6N`f0`alT*y~LmsN1r}i1p0`8&-<S9!4G@hkZq;uiZ)1B!M)Rf
za1=6s0Tp+UNaG0JxKVgE;6Ni<=s4#ac|t>++ziKjsEx>d>g#s=X<iQ;BOHle)DM<<
z@bXy3G#EI4g;HT*OiRg+6SdXMxcU^2TzfxtIGeU2@WF2>V*+`29NfOvhX-Gsg&yGg
zr?zEC34cOk0wdy){ODIXrYKAvRiDdu0AA5Do&P}!XA^)qqhLM6?^51zp@Uf!?g2$`
zL8|*4$GrO$v@ch0Qm-;j8nzB_q0U&a2mL!p!`JbDWGZ)(zw)&2`Ng_~euJ*2mRPHM
zfJd?kyj{@}Np9f)_|J~OJzVto>be8vJG30`x#qNMIXRdW19fdQ?xWu|Tjj@_KY3J9
zd-}kX!n>Uw^GalSP15>mUv(6iRfC4v$JwxHCA+-8^FVvjherR|5HXwMj?E!;_shS*
zILhJe=JsIJc9zJuWBi}olGY92rq;NGnKLbWbn5nBkVxH}Sb`t33tZm-6Px*E3#G@H
z>q^MvuzVfKnxbZZre4pxe$(xHT0|_hk!aAY$5rKYJ$t4BZ|qGhbZwh3P7sgF@(3iv
z54?ui@MQQmONBRY>prc#d$_M;(n;jPW4ImP^oTyt_gjTA;k})?ttpMU;)VUq9ADoY
zE<f!`^XU0B)_`j9AdK3V7}8Cp2EgzheW(DKCTY9mQJ?5!z1xrNk<>>)hKAw0#k*0z
zWt8id$w)MNNf;0nAOq6CSIK|I;3LxcI%Jh(u02|j${SiOixYed?1Woypm(Pxi)#rQ
z?9rP}+PpN@9s2EJ+Ip>FczL*nt*rtFz&{F80{xcM6C^VUaXT5hZlS>TrC%_)o>o+k
zB*N`wf(}qeDm?^=H&ZMy(o5eYG01y`0+4ZXZ&&&JCP&EcAFsh<g1o1u1taD6!q#|N
ze8BNyF^ck`{rtcidFmiZinj4;2EOrph={wS-HDFlwXV8-{q2PzfqNS%C=OiuQ5eBb
zw%lHsEHLv*BtXS(Ng&*TU^`)lIX8UDW+qtKrFBxh_*Q@N1PJYo_^jD#)wsvx%=RXK
z^|V!)v1=lBeBPmm#URXTm*d^<c>ICr?~Q4Wm#F8GSUj29Mv&Nztv|~qk<`O$r#JH5
z6PQHB#JwN$8KE;X$-wWimQssF?GmW%5=|*&)f3bL)rP71aZBk1te--%CNVB5>Y~@I
z*3+5m<oR#;zR`|~PCB#6icV~FpY{q|jUO`&x$mt$`QG06>x_4m!j&kf_QF11`~2kL
ze)Z)wfAc4=zW{auTCLD=|E~OG@<Qvi6MFhg#mxellQ0|ojMY=P<{WyiA#AMtuzjR7
z6_t~oRPO(yBMC@GblC%A#4}`NpU!tiU%evpbi$S4$kw?rS>;;HoD|Y(Zm4mswJ-Ix
z<%gVOXP>AP(vX22>1vy7reZioYGMewv9R|F4yot5$_E*{8cvf(VEIN+@(w5c++R{%
zY8}BPE=hu;`>m$@-cP3=!dioCr{fjED@zR<v{yLda}#f}>lI2wqRF+R6al>@Co4Gj
zowgC)8-J<}9~6K)b#f`Ed!}h5cR2-pc!7;`C<<|%S~R<YBaIs&7^8uqbE%okNGe$Z
zTeU`@M%nOFb8+K?3Agf3&FUp9u}u3bEWCX6MdgSF4?Cp+L?-tdbB(#iUBdg^4Ja4<
zg;HtfqtH#hqRwormF6-T(&#?Z*#4Z&)N^{DWm6^n5Bh<CgYrfAcx!leK`H`UvT4Pp
zBHE3C1yUScrVCm+1tl?4F8cy6JX%Z?gZ^u9Ng6jq@SGk{8L_7}O6@17e-xA`gw%L`
z%W8`EzAuu_iwa55FTMOhZ2$vofVg2~0j*leHCCpy1o_KXS=?BP_%3?>vY%)WCv)gA
zX;m?+{Sc2u3h%d(l?=Ku80~+7l%>ZvbM{raU~S9!AGVfVgk-t$BXLi!gmQ%RO1^dP
z9E+PgH9X=?u!;4G^xz-<AT<?7rkVt|cSHIJ#<3@ku7nA0!j4DyVBA7Bx>hU^%}Qrx
z+E_xUge6pR;nmT`nmNq^No0ihWI{a2V8nv1v9~ejdwxQc?Xq=tl#X=&MN~L;=?^UK
zV?-Q$rF6{JOg4@f5BZ<^x0J;_3q7-#w(?n<^@S)2{Jma$3Hvkvzx+iDHX_enXnCAZ
zT%Q;uxtT<MaC!f>sp!oyZWe}fC|0di;t&0J&D5w{9W}e#{Czj&cJfZ7>PkLMxs;+I
z@iAxv&;(ISGEocs1iVT4cQu`!*_buxDfQVZx#o6PSWBWQt~TentE^X=3I%_i1Vq>0
z@P=lLZpu6^nsh)elH2*xcazwuyS_Ct&H8gI-QKlU<?%d);T~U9o`bWL-&s8gM?|A;
z1WfP8J^-8kV9DkHJqsI$<qw49rA|>ot2GDe@CX;prG0L1?w#cPLbjRvRurpcJN^6m
zaTbS>bGB8BI)auCS6%+BUlsJ6uH>1s72NEiBy!0HS@Zi*6Opd3`Ii%C%m~5iM$L7^
znO*r2*FJsOYC8Ekm_kPOJsEpF%^BmYVcZ|vq;DyZL5ezd<(VEnXOqiZr^YHIPK{^R
zepdqEf;QZ)--6)&YkDr$c%}O+u667i@!$xsvrMWwnN~DKOa#R|(36wd1G1#m%`0yN
zGwWM@nKQq&dfWfKqyGJgBO#1`)?qMJC&3j_?ULuY1R^nF7DVM>KDiI6i>@*)F|<g;
zAwe@`N)}2F6SPsBHp(iQjxhc~Kt*3~TToZg?ZE^vwb1<^zg*Hf8#xuRI~J^EIKfW^
z4qUbD`PdBz=S`T3|Deu`S!|W;a5h;|?|y}^I9DP8Ee9^2@c4G5p-XjuA;mh1;m_JB
zY0B=d3%^lT-pD@WoXU1ii%^!D%4ktOauq5t^tdYx@2rQNoIZ*)b$68ocxS1aSSBNd
zA`27RiJv(9mqw<x<6m}(e~B<NN)LI@XA2ejXY|MqQO(a3CBk!d%-{9d@MnEWC(0lv
z=68KzuuCq@yzWT(DYMFwg|>TaCF;DA+kEA3cR@HSNv;e!Yh3yxiSPk8q~rTxT!tFJ
zhJS>?wIk?b_u}PUB>e(!vc8;qY<f#&_Yz;lTY*t(0GzvMdQO>lgA`l55&l6`(p)vy
z8RzHLyHP!xs}-|-Is1+eaAzRmWK$b2Q=8<>s?3RHD7*$GR5~^~)jk26sv6AhJ0?f&
zqe@t*CWhU#ky1xzw>G|EJvV-~?fj-vn%grITrBLKGVKm6sbP}2qnF5@o{Tp<S9Ie~
z=kwp!e#-!{ga~`vU#RM(wgd*JwE%|l+J79a9E3rZVpUEJ7uXC%_nmwaO$%PG?kTV-
z5z|cIh!_*~Tvis?SjSr-*rI;O#&RT$=*v8^a`lwqf4?~OFX`=Q=xPev>#bkB$aqi$
z6)o^Z>yeL|0T3O~_*i9-a@T((hxi>cvi9nf+y8#*Ei0;X9Y4?1h1bpo;%f8RhsLxx
z%7}(#*3^6uA@zVxxmr-f!krbATL<14+T*tlRk}Y-ted9hlmMu|{5JCYurah1{XUh&
zM&q3@s-^PB?4g>5*gXif+7Gl=5H$@8e^U)4LhU|oH$xTC0+nuf5VCQPwu3}z!f!8x
zQO8pqYRu_wAFOx|_BTO5n7d||w)epA^ubQ=P4JHLi?woxSfH9Szr?Rze9wzV)Wg+6
z?Zi5X$}Yg;l49{&acFG;gEh43h=GzC_?*JDsaA8n_xN^n9aaOy!boE0Q}H#aLT4+}
zu?qUO0OKdNH1BxC53YV_w@gn8s*h~`TnT0tl?cC@9<Mr}LA7xKRx)ua7OZ-xmMnl~
zJvu5RCNfskpPh{N+Bkjwf96d{ws<pFXcBfU)bk3$r01RIgCwZ4%hmwv243!|qL+_y
zUTnbdY2svZ0u)`e)saDOsp&d}`uw~vs*k;XFb*=Flets{9ykZI9l7fgFFiI~$M!=g
z0vPAtjUqf4EG|IaCVY<*+0X;QTTXgl!YunFHCvn-{5E~)Hw>Zr#7HNBDBZLynecWU
zdN{D0`VN}G80>CO1#69^>+saG2;sO%)6luvQ`dE5!=W#vhGL5B`pjwGw0y6(mTED{
z+WYXed>4Np)TWu#Ztu+n(T^bF9s@V52si~#%#fOpj?)$GTZe|^@*zN&0^)PiNDwN3
z;b{ce+>aEY)#KS4HFbV}=UDRIw<9SwlX6YQ!$>P2{$a>uh)SMxBu*Q=c)hPl>Cj{F
z8Y6!)BuA1FLN|OKh{zeExBdf0Z2ccFf|>s5vlVE0{QK2kdZl&EQ{#l^&$a!2#Z`C0
z!)st3g!xkLcddSb6P)Xbl@GC!o`Y8{=b<rk7>25|BeDq|u@eL|Z^7(!Ye8qn#}K5<
z0|2l>R&3*U-ro@Y*9)T`=mM%<g64gxq<T;8@|LOS=~|JXgt<~~x7^8K)ib*nLGu0J
z;HzH#u{Uo|0`q{4H)OYjT7N*afc$m+v&P_CPo*BKJ_mB)W&Q=YL?x-CwQe&fw%f?%
z$Dc9_y;*5;!G%rqgYdqV@9aLGS$tl50KW~xnLe9N<H32@iETg>p5D$@eQ#at&!y+9
z*o2T-?+Y+ek(c>6RJ3Ot_}aoW6`d!$f14$!cPW@IvH5xZ_(v}JM==o-CpwY)L4q7`
zGoz_@S(3^^PF<{HtgRUfyr*bDTWp><bj0}RL?Ts7U5Ci$WeEjozALZW%XFLcc(nyM
z?ZB6Ki1yRBC92HAJ`~v5C(h!`J<uhS2XA7E!#P#cV;a>(_Yr~-ip;qj@}#Re>}Zuq
z6OMG0>pfBgQW=Ik7qs}w<K6ao5R3&bWWQ#A^5e2FA~^2BUP-NtX%6-;&)A~MfF_B_
ztkVC$A3W|qT`VI>kA?O)W>xvW{@#S!?36iXOUD;wb@Wy*mAxnT=n=oUsU8|gSEoCl
z`^LIdfcUuep-?QUS#7e5n|$U+H7YkxFhs`KB23bk>z?|;DvM}pVPyWe0QX@C>3*p6
z-E7s&dt<MnmON0fVS|*~6GxHbrl4VsA!pr;r5q>LN7&geUlO7Fd19C8f4l%*p^uJ!
zOPj}H+x)$whgOpn2mAt!m`GFT;9CX?I5_)xKS#g+gW=sWAfBL)y#9<`7(46GuHYY@
zVb(k9v|~<eD$08uMQ@Sc20d735D&-kNS718d+)x3^``OK|4Z1iy49ReB9F%7Jv8PV
zmU!e(oad=Nke+6oDSU|4G@4oPZ%(t1>P|S2JNzK<=<W&4nvrR=Z<;_PPy5eF;WKqM
zWJWwY(t>7HlUR!wrODcctSWvX+@C%{zo081|BYnluj^dyEO%Y7+R8=y`W&7eE=E&-
zc4WHpAB=78AS$oy>!=5XOnP+w_ny{e2McbRTUm}pHbRZSw)CgF{ODY<#)aD3^XEat
z<KO62f2nh&uB*H-O3J0Ey@Fe*UqujYVK`Nc_~zjWrqcd=-H&WEjZc!z)DIi+2pkWN
z$EZg?%P%_p1A1V_i2$dMSsX7GYXZeZ*vY48jv-c@$%Ql}h#(#j6=9BxA{nAU07%l-
z5fn*4+k(c%$;!{#Ctlxdi)Ae7w|eS+>~6MdnUZigUmW#wIsy%4X^yzTMH-&?*jzl5
zM8)m=c$Db>b4rt5;ECdLh4Rf&U^tOb@?en@mtJ&h_E+zxdk0U57o1GwASWP#mZ+_>
z*EX~*W>=K2eFgyRN~GV(xT&z2G5_m*9I3lekWTb27+fmWTc)7t6OoSoB%FyC6Er-!
zyExGIZMl=DKSynfSCh9DbA6ENA}&lQd2RINTwHpVt7HzynO2WuG|CaRgAXYg!=+^g
zt8kdOA15;#-qYttYTvb}8iF&4SJBh;EKv|?X1z!#%eA}bVfuJx!wrKj<rcW+HF$Mg
zU}r2&+uEezQN|!YtYDbZM}F<;;U7_xy~G;lu{SdrpTCm&UV@6j_E-_8f?|(9hD+f~
zNaqUcd3*o7<T$bSDLmyM!pWdW#b|`sS*<<wjugSsAOrgbt|sOlUZCHrlYP<c<L>RT
zrX995$E`HV#mNBK(EK07dDM<5OuZV+BeTK1Jjcc>ByE;yXge6_{qs!WtGkW&H}8_T
zBfVsSNIE;dNgcp9vWw{n3og~}A8e1V6Ch|JboF7TJthMu9@W3_Ci@WAZd*OM%npYT
z5!Vwtzy%NyD3v;Wt)5Hll|VoW%v6T4T@&HQW2HvCPQmFghEP1VIoV_P&hA!n0lPa&
zn`3N)LostOof=827{0Sn|C4T?DrXFvAXCt$mYmY`D9vv4h}s^qt@!U`!2@p}>%As=
zCEScP!FB6OqYwXil<|Y#`BcwpQDp^z3+3V@JdGwsQ1boKsDWXE1TH?O^iPrk>ADD6
z(&u2mWWPj5uzKvfhvx<#UuF?7<J&QnQqoxIDhexY?M_oMpv(UHG_6g1&CutW@U0w<
zbw#0&IPFGuDz5G>tCvQ^eiPXr2D4F2-+N|lp6#)BJD9d%OVuXuLC=W^4pl8r*lp7)
zPT%C@6gi|L(X=LHP!J`wm32%zcPn+_#cO5j1oWs?!o~ONVv3B3huqf!!sk4r-otsT
z{E7k0xy(b3EV7kKa9kClEKV0!8d!O*9J!p7_)7R;-freqrm$6e$~UwC$BI|B%+po`
zb5wACzd?hfnEOL5bCn8!u}BPy#8TGi1{wExvBBh^w#@LiyjIqs=<%a1RN$R}zGsMN
z=@iz#?C0@BQ{amX2lF$i6a8DCi)XZ8M<)QH^i3&yCX&vN2KHW(H}O$zSpn#fbeH0B
zm(vh}0Sea+tbw7uQTDSh%T#538%wIg#-{;Wxwc2PQu`-JQ2z*>Ji!A^GykEHFDqsh
zk4T7nV&&|vv!B`=^yc%0Hq(O@`D-8W9}<mB3?V=kt6trzusi>O=CS|<bK<eS6uOWb
zeC1-%p%|oS$1&Ql^Fp7nSfO10<$AyM=f+Y-$9(NK!Mu`JKZQ?O#NOUB6}O&N;FOwN
zP1SzE?>7>IFhhfJ+jOs>rfGfuvmyBdIX4c+6VyncwSk<wB4!;8TK@R5B=aid3|<6}
zAJYwq2w0CuD<EFV2vnhEJ6(V;g<8Y2G-({z8K->Tt&0oPwJI3mxlxNZMQ<in-1rbr
zXvVtcKc)AC1FNo(VuPH=Xj>^Jzrd{=y%n_&*@J0T%KqEsbu%Dk1=(;znau2tqtv3H
zjrN57Gn?IPCK4$Ai(m9)3LD+Q7Kl892s~SqC@h<#X32ukpXoEMdZv@Qp}C*O{e?OV
z8|4jC9ggPzX&DmHHf{v>OZ72O(tcqx_DG2@V7b-U(2yTUmuTu!ZzX3yeF_HnNVLM8
z-@DrKZ+cOu;6=V}8cCJGbb6Wy$>M#@t0=y=P6c<?;yDOO@8%2lK1?!6yT#(JEMq9%
zHO@_O9uKH}VR(#`duR5S?cZ4ETtrC)>`~=N4spCmd!ZDsF8d^>rRkq1o`{HK7&&#o
z(c-g&&276|;QU2lp;~GiMQ24Ng+dy(f7i2xI#p3r!HHD!i0J%B_F}6iyEUb;ISCAV
zuXgx<nr@d$KgV1hJ^7D}hIfE5IJ);x%ik&0RvKb5vk||a>nvkJ!1p#(g#rE~7HT~D
z1E}Upv-UV~lq^d+AMOV(<qfqJIeBt@Y*s1t)H;up^J678iue4)&TKu-kOoCPR;){6
zag|@sJe+veRR5ZqQX*Gk@o#Gw{C~bL6$edt+wX#rJOgGo8#@fy?4)dQLY{wg7W4u0
zkZc6XJ3x5jg^X%xMEVLl+L?qlp_0AVVT`LiH>F*`vBf{-oN5e+zyC3lMjMM=7|S$u
z8a8GN5rT3oZQdFVl`=$ElfVwI*2jWaS_uWYs}^@7j9`wqTzvW?`FI(`nLX;=#qp)k
zC7HwKeh4nTa6H83a0z^Zx&=X-9Jo%m<sTAc(@BsR(QGQ#$yt=%k0R-OBLy$kzc5Ih
zS4`>S7SIa8#T{E7dox(bfjq~`tG%YiM>Z*g(a!MrMZ7NA3t2pSo%~KQ6-(7=p64xB
z{%&t|^?c&wd?yX(%}&L^f%reT!JF7A|1JUOh@KB<i9UEb^?`;}i!y@~7Tu*8<Vzha
z*r(?3wk;X%mcRA2W97&}%YD6BwoWm!*AD6bxyMO)n7N(bT!#)HW+%N6UVS)qeE5Cg
z^oc<Uq0^VV`+FoO)$Xn`?2<4@wi9wWq)uJ!)E=1@r7_ZjQd0gOBiFxQEtY+(H6jc;
zOMlDg;ZZQ3m7lMluCsN#=h5RFJYksp@{pB!1w2Xb-H-AQKzL#-E8#&r%?mI_bfbx9
zu>(Su;Kez%t?Gb3bEo9UQDAQ-9)Fo`wU~d3+D=9oS%m-Xk(t~3>j$-);4|0H)SU|P
ztp3~T8sTX)x>{ro1svFam`)DzI^Wwd#j57OrQUc@JRlZo81-o9nExYTrplkI(0&H)
zg4RM2--ki)3UGFG?MNiFhOnW3LsSEm1h?DHDW8c~q4Sr3imwB-kYD(<AZ1tUwurj}
z<nhrFsktp@q}OPf`IPW(SKu}>Q?maz>hWuzQG5+5+k)@Ykd@b^fXX+7v8Y@-W0?ch
z>5fkCV&XtuUHg58z#9*#g9#XxwcnA?;#c#-r#gRz1puJQ2GjL&0~}%ddM*{czs`%)
zJ}~NQ=$Hydg@2)-k`2iK*%F_dQZ?xFxhFbhY~-WOGEe<kB!g<ROwDw)lT;3(gY?;H
z(8H=j8|v?*$1-V}j%pNP`IaH!*TeQ3D_KNkWFC1@byV>(WJ@TjQ;o5Ip=zWYCGFJw
zD2VMtewSyYn_>%H01N+ONl53Bc?(V`t>GKKi!d9>(B&OSTy4-}Q()N(b(Q5XTs`Lu
zO*p_f%n!P)Hz&Wg#ZBFvD+!cU%oaoj+y9R+p338gd9Vu#5cg0>3Q&uD$g?{@3&}zE
z)HB(MY}o>xm)ha^%rP0?ZaLXISEwjOjpLumNyjF`WkkGKdtoEL9Rfr^F_m1sd+cu7
zP|cmsbMYyk@>qgFlVBjt9E;NcB!V?106I05h{Fv%SGE<Y#B_VPS!KL#{Ijw<&^F4Q
zdo!})_^%ol#VwqVJ}&*}N!RYlE3<tHsS}mo=VrSNovDg>@(Nn5q)Px{{H3|s#L3C6
zh_f+#!B>a};Sum+B~c_)??MHerhkPtmXq7>3{e$*PGB$F!AAab3wt?6=~ljX|12px
zebc}RYm$bq33>Sl*5ExH0iB9<>U`tc?nM`w9eRRRbk%6!)0yaLp{b_4jYk&kg=K}o
zfTHX@-dZH$OY5AW>OX3!>bCBs9;**6cT2@_U>Z7J6kxn~%+P!@ZQk+X>luZXgyCz(
zg`;?n4Y+`x@YjJApBh8^^`T>OS}9jVTjTY<s-7K>A1s4(n}?wG$NvI|Oiy}j9#Ty~
z5PO7suTx$Ju6%D<qxXh)tDE%2Hz#v#4K6`Ci=2C4@Aar6Ir(EZena%{rPl_*Y<J9K
z@tpq8MgZ7a)mX~qNLq7UzQEomxt4ItGm+h7m7`+X0N;=C(;hqxZ0T6(@2!u)`>g(8
z<d)4T4O;s%VEV89qu$A2>qsBHg);d<yZF@8&kF9*yrGrG#cKWgXBM9)8^M&t+anu5
z*YsnU(mDHjPs?L+n>}EzHaxO?#D8S=cl(L3v~qzJ7G~J8>(sRCed<Bg;ZaTH-U-?f
zzraFH-qA;PLq2uh**xT-l#u&W_c_LKTG+p=$8<f!VX~l4AclDeO+97g**mBD9H5Tb
zPv1INb<piFe}EBdPuYRDm*aJsxnxtkW@UgHUR6KPeu@2R%BoLnMLE}SdJm;Ben9?e
z6~sD$`%`-uzN=hyR*^R|**vbOCgilj#aPt1+Jl30Q=o*bItz%$q=wnQz*qtU8^xdu
zz&J=~Ajc3})_*;VCo9pG%gseI?j;L8*^Ef!iSfQtpEdY>UHJ!?1U6c2L-Xh6O)he_
z>O_08<yql>)E-QA;g)s4)!k6wFonluF!IU)R(FpH5vRMH)8Q2asAAUAT8`9k+5Lfy
z4H&CB&jW72M%w~#q_{Z{RSWK(Il|dyfIs1XIc-De_+$vH(TF?*`v4$VTYOc?hh65v
zTFT_%Qf;ETNZ40E*e_u0J6YKrky#FAd7588pE>Qr&sn$Og=yd#;=^M`8+=Bk+>fLk
zq=}9JCey5y4~)?bd|+$Zg|IF<;k#YGHe0Zh1iQfUZ@xO=p+{42PLHq=-1N6rs;~VA
zy133wnrsOh%#)`%MicZRcK_m79qjAvZfjJ@VF+ND0~frCX@EfGK9FAPlHpqH;?Zb3
zWwZWsDOO}|JsuH0z<P7^;ko0oj6;nEY>4}2kyuyyCxbLQSE=UNEOcro%jDV`ZKmf=
zPqZbNaS-6ip$V^W@Z?N&5ubuvAFsD|zDP;A$<b0DrWC9lZg}woZT-@GSZ@y|ex~DW
z)|l4fYSV*m3}&PECvSNK%d_I66e!-+*r_R3!$rYOhi2#w#qIN1Ba3;vI}MF^S@#~f
zuj;<vi}K=$iEvSIaFw@khW;z|5v-KMEgy@qT=_%yH9Q}tjkua{>>Yl9mu4p=$#bmc
z+&f2AKnr)cU7Dxbc#m<0e01QOkCJ6`($-PydqpktwuXd#?{WF_KY7v)>0r*5xp=LE
zSG6t{)zG8(B9lTavFR^L2Z(uOWntY2JFH#<ZwuGI>KI27lO!sF|4+t)Z;EuL8dD&O
zS1kGlzQ5RbKbTnuN)umqKuk}8M0nU&s>7k(&Uygr7k}KOWLRXX>DCD$k)K`cI5+e*
z?WzI!99JY3x4geu^&E?iPHia!-f+YNZ<U=pW0s+hyZJ))01~z5)XN(_f0jQhQrkk`
z3}*cvt%ts7?sscPv)p|1_}Uk%c!Ap!@IGT##h(PrP34X;0;puUPhD#I2sms$p4q%I
z>ut9=W*GR(QbP00NKs9ekpsEZ_q1uC?BMI&HyHTyzP0oc9O#Ewn~sLQ&pe=leQ=MP
z8FT-l$JTNSZ?P3d*S9#!2u;wKn-@qD(AGTDn&*>>jx~Q=9=Yz=T%0$Bj^;Whp`)Dc
zjbC>)+sdT8o9GTH3>h_Xh!0o}=o4k@=gt1cFvrJ#sBy#_D<)CuI$-Cj<{`he?sw&=
zn$mcMIfaO*HL#m$M&+YkG4#E;0)=Ys`bDGn(FFQXfXnMdG=GHvZ^gO)yLs)TD-Rd+
zx1;9&|Gcxh^W8jqH8$#S6k|p+`kTKG@yZ`qDLUw?y!5+0rJ}^*<@9S7h6H()>B54Q
zv(kZfWM9)&eJVB~uQK@MRNeSp{t^#v3EZl6qHG_*@=sslhng08(~TL*TIhWD%suNO
zK+cn3P2r@$i}ZQ$eT=bjqx5%bHyMTEsQyW{P>Am)6q#ocM2V#IsI0Zm@8GP8|6CHm
zdnoo04SLS%710e<IX?GIffpI`wgMKodZ)<yI}?e`E0c9N1_P!9i{I(@L&6BC_CC3r
z3LCy<)A}s?2E@l|!ertq^*zwkUZ@&ZN-@C2(4ys?`0lJen4=gV>M&;X>;4TwXyKQu
zYGjL5Z(Qb}__bP>Y~6Hx!?`KJN34;Vw)(XN7EEuDOE_Mn`b{6eAUkeX%xiW=I|JYD
zm&LySC04qS64~rB?HGX~`A+!wy{+52%>%dSI@cdBI3clNnL{V|&b9pDHh+KY-Hq63
zeW1;M5|g+nv80&gx{^Cc!0yeg{#(J2oOFqwDh`zyKVlI;W`Eu~l8U9K=MCrTcc)CW
zK2qsaFL@;9de<t>0Udi@Z_nCVH(plgsru<%2m76kl6P6`(MxgQ=Ql|5T>iysU2ASU
z`b4Aap^!ETW3L6|E0wiHHT<jtShR=S@m6<G{QB~eK}-nqx-#Ja(wyD`r#>G)HV2%W
zr$}@G=L_RAB%!@Ed|jAb2i#92oIu@}{_2d9XAogRN#Wm<%b$oCF!3qHGdS@2Ri?8w
zKA%Jf^XJE_8vFdi^i0m7;%7H%iWE*V9FGof6D{?k7aVr8vGS+HafcRcS+tY0hD^Mb
zXUa+epioy9N;!@GiusFlvZKP4LysgrdIH<KS$*2a7pE3wbO>RMR$5pu(CEhCdali+
zCR0GmU}#K!mu@M0ceO4Y#Q+ar`O~P*r!(N0`|iTduk3pzm0VX$uhA0MI|kMsb?y<J
z8l{Ro=rFu%C%9JM5>QcS3GT9EBg<n;iskyVAU;<D06}8QMOOe3NRxEn5g52fa#Pfo
zfH|)(il15RRzReFh`S<kmar#-kZ@DEyqNS-{8v`9g_dqJO{CzPDk~VP_#H`Ni?g}$
zj~98J8QHw6bSrI9=&;#1nt~^*vzlF2R^Zh;6j(zX5_)NJBd*_7!9q57P&X9)XbU+6
zd`liS^5R63<KBLq5>-X9ltzL6AeNeh;COGcSw>(WD(xOZ0o5?CUr?JcAFu6E1+%tG
zcD--RgMN4$jUmD0wwBFLk2&bY@;k)}W8w}fXO`=5-n<?Ly(}@d@vf&$t|XAAN`&o+
zpXs7#v(8MJE8LC4XtdpICC%gAR>m>>;RlF3D|A(uHm22uOP*-Maft>fxb&W69^y3r
z=#0K+im2S(-M-ut?96sN#=?bd;>_byVm1%U`1p(_Eh?x@KHdtwel+@tXlm>_GRqEy
zdg!8UHT>Bifq+7FiFt8kvdzgxrR;^WOAR$y+v2b|eI$+M8&jMyBMUEbEx;uVX+Wd4
z@{~#2$%t$nmHfN8j^+zj15m}7=*>K6`__j)s7{2n=J{7qUUqarlKqddg&1zc2)CYk
zqr~-T;^LF}^imcL#`h3RjqFY#Uylxs7OSg^+bz9&(q7nSORx}Z6co9;ShhZ2eD_V^
z$lzF}S_+MpO>AG+kJ4C-SRTT~YF(hO%`Lj?r60AFhYuVh;6CnU*EQfDO9X`G2C$7z
zh*Fb+QjrOV#-JlhzjUdK;aL*MRQV5S9zYeOQ=JQuVIm(!W_^X^VxU~yk0!AXm;AhX
zv;F|#*3~ik5Dg6PL|8%(;@)KtPlYNfUST<&blU0#V#K%b;=<6?-&$ptq16#6r-2DC
zHHwv+N#U7#DI?*#Zn?ZLiclg{oCIN2Y_qtfn1}S|F}kE=3J}xuDiKeSMQnZ#&h3}C
zAiH%^V|^a-sH?c0bMUm%1BztF_w>`Q$BD;{(8xT6a!ukyTv#!{9}R$q-4=M+E+k-X
zfPGaqX^8j2;@LWGJS$3j|A)KxmrfkZ0aJ;hga9fSmxTygfyB~1Ppc91oBT)LS&cL~
zzF59Re2yrz?v-Rh7JLHE+?wF8{9lL}ng?Bz?{l8f3$rDEVehmR-aJ@P&l~h||GM#%
z;Gu9m=BB=QtTf&|-htEHbKTKN#$BnFWPGQj2nOL)m#j`k5A`ud-1CeHZDE=!g+M|z
zSvfgpmsZIB874IIIv-(Lh`8t+_mkK!d=(?FiI#Xv@4(zwsLE=aw1lVl<XikyP^T?+
zaPTZQPp6X*Vd@mJ>v&GQ#g0biqE+l0f;*YV_t*<mN5lT}Lt#WVO4^UoO`F*EP3xpa
z$%K3C9|BqCp7#1%yE(A}8VXx9UpHbNvdrb#D@FV|e+pXfiEtO;jJXn8U5Cp;0yW3}
z!(hkDpw;Gv|1(4_xjbwiyC9hQ^V$`<!nD%_)DGP{FPj!NMq)hE<YeuFaXq<R^ig;$
z=knG>MO5^I@2+Jc&i$)``&$X0DKj>Y3$(Gmoca=%kLaNcHkH0`3%9w3z3(?|YZz24
zx&M?7DB?<gq3nGkzxD(^W4rMXyH~Mi?&K4-jMj{2`ny)F^5UgAEFnV3%kUMyWU;4k
zvM6ic<vr~uFwt^m05m)o7oiKk)@ZdV7=1A{crn|gK0Q0nE7kg7X}}h@$0;hogsj>3
zBEU|)9tb3nrfEdYJAfh(_KJ|PDs9Ug^{%p?Aikm`uTDXe;R`MHKCwgZHO?4O<mMhs
z2{5UxsQZ+zs_0no;5#Kq!CtPJKAB}d($8P&+K=~NvCrd^@gDBJ6>51GE`+l;I_4>Q
z4FLN);=@f7^o#-G=&n?Sbn}9*8Lwp$Q`Vu0a{V5Qse+tOw8+!w%LScAx+^`F*bX#j
z_-f}9&3u!Y+WY<$W`7rLJfW7#O3SGX<)Bh~VccXN2t)|u<XzJ`6M(^$ouwS_YE#-g
z;95+2@L)$uyQ0(IppVr2tTHu`dYaX8k~N69-9V^d)~6(hdf<aLy*u$oJCf+5_g_zR
z;I^R;lS0$m^NOQpszvO9C{R2f`;%j&dhdTA&tCL4?v-Y5%gw(r8ieO;<OQ9*h^@*`
zDc+066&-z2e-42vamN8W>{$~I>=>e@rlHhUn2hU=d#?BM#&8}jTCUv+QNNZXk9c6e
z{e04fU8Bu8k*Xr!^W@3x`D`%8wC2A)OnMF!f%jU|za1=F{?i7oQGtO?rk62)&>k_Q
z%5G>A&3kTnd5$diXTF*dRs3i(AtvJLLRZVUUno|k9wfC{u8J%7qvd^~2VYncFPTP<
z^5NAmKZf`CeO10Y^rTXT>z}}YfJa|9#$l*|p8pO?=%jJPtcrlE*6A$nrFb5)s-&A*
z&-HG8+_UcJUnU>@&YF=^f$KZUz;n47vWBD8ZpjCgpIS0Ne8Js4?`Ru(SA4bb$L8<X
zH0cV~mK0V(W|dPY+B85^3Ea>~>+E<YVgIiv{%i1zW2YEv^-F3nCC_q)pcRbxU}GRl
zh-Uoty2satGOnPBe91!+6SJwbDu(WGe;t)gEkS|$=A^l`2X5OM<hfUGVP|g>Svd-D
z!uJnVS^=w>U&VJ<?$4>Hs2jg*=Np7^>>c9uF0T&jJV6)H+1OZ@3KG#%PkdQ;)@g7r
zVzT=q#YDR$2+BMFH!-7*IA7q>nV~8Ac!|fw0VqQVO}iuTRDj|Smh!Q*$}0#H`n-#v
zJO=gWQ;?OP4B|470U8^^5eosh702A~LcVkhASd~sjDP%R;owkFMnUW*#vlFw|Cw=!
z%8N_ir!Na47jl;Mw<1=`jLB9sI}6B3%y*36nK2mNAU5ree8J=Xj<)VtQA_`By<zBX
zsnHL!x}^{_v%PbbH)z0((|KC6<KqCn!+wvQ@P(BC2HjV7f39yFNSpsrI8B03z#b`t
zGOk|hFLh^KIjwr%GZfen-rIR`?WN=?3WNtx-f9B|dqw&%Fj=x}92_8N_X`U6|AsyQ
zj)QG$^ss`}fQ|H7NSYexagt$51DdJ>1WUu^w9lcIJkNn?0mZx8J5A_sd1+sJ*TmN>
zY86Z%-!Zk{u6Lf6x(B&bopyoDI=>c0%kO4T>AxJ7Lsr_=*_clazq)j`pKRHtIy87^
zoSwI&5UHYtbzA*7s=C;Up(F7uSjp87XfO_*f+UY|9}|al2r$@BUoKe$LIg1FjLUqe
zUtx%UiCyjEIA01Zy*HdVGIAfCfs)A)r0(r0@BZV|QAhw66afJ*KfQ7{#Q$WcdIeHV
z#eQM;w}yOimyHC;hYu=28GHb|PG~!ozuW=V4^UcsnU|^|{43Wf0Q;=Rmut)fxcLPs
zsuf^n{A4T-w-`gm?EY{EcrT4|-yv7+&7FElA=R_|)v#zgJH4Zy%geunQ1mz>@=e=o
zYo8~2S;pyRic>U-jg!LH$TY0Aiju7w`)&(AAdD04P?6y=l&DuW3U@4f-|@-Bu@A=q
zdxKB`R|RVA8+MfnGF>_1;xFnL&0+Pejo3?5Iz5i8i{_d*KWbUU*mC;C^^B*~5aT%k
zstXs8W#2(w&UMxX_nknH+akrTdpZ9v<}JpCj1>6rAO0)rn{+OJjjru{3zmjAV#oG`
zwuABD5{O+m555WGt=8sS_pQIxvqyu}4^zGcM}OiVIvt<hk+b^^NoGc@q*x`F8pif$
z*Lg#*1i0zaqLB;RvSI_Ek19+C@;PSC@2m9yq^IDl)0g-jqYue}hyJ&^A~3s~J^LHb
z?b&<4t#)9>RrYhiDK_{XA}M?|y2P8f$`XjW%1xo{b82VR>|(CGoQ{AQH`q!kx$~4a
z)^C9V;<3mJabFwX`X%1GEB2k4yqwd~cYrT~CJ4BmJOBUbqPgyBz@`V*C>*ZD6<O+t
zDS!>xFs1&f4XE_PvaY$kw=}!9wLcElZi9iro8O_ISZV-{J{Q+bJAM+pEkQ{c0GeVk
zo3;g~2^SOoAyJBN$xryN*IIn5^X9;^Xfc#<kBBE)<jz`BI0g27%!#>$s{F;BMzu2{
zx3^z9cr9M)w8z1$;2WgR#QW`@jc|w@d*J)!?H(Hg9gC&5Ea#X`<V$Z^Wn8ptatbZ_
z0?q{hOSV09?K~e-g1?K(MgVU}5m!-Tg8kRC{V~At5Cxh0y$QB{W2oa>(W_?Ugx_F%
zCx5$3tqm?)NJVHFOy_;|u%+R3V&2&n0eAAs-s0Capwmx3`tYpt3x?&Fpg~2G9k{{t
zkp52O>sSLYuxrWHURu4p{BBs5d=1IM*3A4!DGl2k$74HcR*hu<<12zFr7f!kqslO*
zNzL)IBbOr7F)m!IA5&@RwK<z<F6OTE9|0vO9*0`|y0tmA!*X5bzqbw$We8~`s6#%B
zknMBo&jvL^Pt#E&=tgQA-Ddh2Sj(zA%CbLu`J1AL8R5bQxCWARTGf1vj|iDZo?{ix
zoh(m3Z(_OEUq9XxX|^p)zwT6!t326}wD_v&p)r!gLz>!Ad$SJ!=2RwBE&~wZ$3d3O
zCJvSkK$+lVSCDs3B)r8@+9)8M-vpbE{soiNyGc=GU@W{}S5o@N`)Uw|C;N8Xj5@=`
zpFnwk<p@R`*a2AAeJ-sq0s_Ju>O>rS9X?Pq3B4y~-G?qjSGSrQn-4LBAwAG0=;ccZ
zNHb;+FzDsJ5~*4K@crb#wF}-GF2`f|<=l<Qgyk{Yf>mz+RTDOv;l!U1H8ayp{$XKx
z9@+ey_B-J|$Q!qU?^O<|zvo)C!E+Lnh&O4)QEH{xZmq)3=gi^>nK=AqDeHiSJbax6
z`_2s*9qHL(ns$c=rIR^~Wtlk+#Z}Z1ezr@>Oj>{Gn7RMkw9i5*@lT&iFrzjF{|L<K
z>L@8%zh7PUMFiq<)@#Ts^%<3TCvChemGcYD*9?-Ku%Dk(jzlFLcMaNR(>$MVz~KAx
zHdo~Ktz@z2;C3O(MEcavSS<OxkO2LI&qb>oY_2egsS%dI`mSI56lk*vvys!Zvb2lz
zy7~~X;|-JkEeTu#LV#X`lOj(N2^67LQaK4FhWtvdnFMbdcmC32q-#i3DJ-`d1c~%9
zLvVfmx~%K0?hDoLoKjC$W-~;MSKWz$ogHzkiU*pdLknf(P`9iQ8$2S0=L6UG>F>QP
z!x4T4M~L0Yz3%uH28q8utnAy0VegNPmj=t?tE!Qw4))t;{heW&7@?5AbpBlZBI#o2
z^#<mAm=dKqk+HOD4P9=+ZCCZB!}eW!8SKw|v+q6Wfa3EFF!y}YRrUz$A!Eg+`3drL
zMlMi!vPFzz;_sM>*xYH!zW(|q-wkB9s^cxyvJ6}Zw!qWDairHa(t%o4xF6a2aa271
zG!gwI8gD}{kLE+xWUs~!_@E-nA5ao)yj<UUBZnB6QRVEV60Q}gyK}Q7t&P%%8e;+!
zBGEK6K^9@0pmNw?H#OPa@RUKRt3R{f;v${&p&QAJ($fl8FtLcA;^~o2k}!A<Ww93X
zlvNpE^4Kw+4(#3^7sXq^q59&b!9e^stL<PXt5oQqADc7vkK|2*_HU+`uKXEL7C8!D
z6y}Tiyw4GxJ!SKE#M{R{v~a;Q;Z`f`Tn1eCYL}GGjW@f(9^T!)^YzwXP{FkeL5$yt
z5eh3%OzHiisBCQ?sPB4qUB_L(N_I)zJtIouIqD|Xpi19XDIc@2<G^p=JKt}hMY8BD
z)aMIk>Nb#d63LZPXoS7({49ZRc`)3wtD?i8RqsZeRX@!2hr*vQOWtJ@HCo;G3esV}
zFR(q!@SE1zFb1Xn9sgC`1{O|_<{gDC`uh;EiR7PF2jWP|cp{fN?-LjpC-oro>SXRV
z2S8`8g1mZ+KTxEZF`qTp^V?w4L(H*4Xe*AgL?XdWIaP!;fbc#>dI-So57MGCb6P_A
z_*0rDziimb<E^omJDQ#V*@h%EL`^#luL^Mx)z*%`FeAWX9tY!M$_fpQNCVAyTV7i-
z(Kg*x1leFMIa%h|;BDI(wcmflNTiG^qr8N_L8@h?N-yhsyxh#rzMMp_w&eaB28qcb
zBU~guRS$Q<MRm^#U$0|idwpRcWSx<N$RtThCXW8t>->1k8c?G;TG!D`H~quZyUk-Q
zf6#9c^cuYK;j)~5t5ZEHg(*;{eX_N?1=Bw`A^g-5t8qVuS1a``;>LFBy*%kt^O<H4
zFT8NfHl1s`Te!AWzuZq?V2VWdwcPB%P{V6+5x>6cw!H^Nujbm=2{zNA#2t4HW8S#)
z-AUC7w>5UXcJ;4^dMW+j-aSwCkztYVRZhK-5~Dq<(9OKWJxtS4#@$)O<}!qSLcbBd
zT`RYsA9F_iAR<;kHcJ96bux38k!=<mo0m=EF4P4v{tR~`kHBslBcf8QE!1Kb5o)f2
z${<C`I*RkWGFCB1gJflNU?pYn9;^h4*A46JOMBjQ>P_bBo235h-7>R?+IO@52}a(C
zq*H{hLFudD6|+oVyFLFsEm%>h`xV~{2eg7de$`8Rl_kQSLz_G7MC#9KY*U2F%VkGr
z)@4u2i-J2m-kT>LoG_GX_p=$yQ2SHayaVv_C|fAK=jE7W1g*#%?Q*PhT&F%*NI!Ni
ze?5EH$nOG#dJRUYPJGo9B<*8;ty9sB=@xr&Q`t=+Ua)q2I#WRH9!>9pfU#J&G8kqO
z`*l)ntFo<%4zTRiq53=ztnDeMAF+=MPG+>RZ`YApVmgU#v~q1<G;`gi5rT1+85;v*
z<JtS$U*^m#?02I~(Uw)7+&4UmW1C0a{^uDG1hwz|qcL~0AX7JorX|;f-qExpYf19&
zwwJ%(dLWYCi=}*P$57tKV2X=8HSrGoW1smfg`B=m<2w@7NDCzy7It6!g~-C{uh90L
zsRbvtphOEa_28&FF?0w|jK5N@-#Ng)Qij<aTR7&quDKb`&plI^CA=u)ReqC+g3FGH
zxRZR2#cExy-vbn}alx6O(Cm67Fh3-y+X3wP79qC9Eycs9pI(jvPIo-tQSDINgO<wp
z$XZ=L4kHCURuf5o%Tvq|lPO&0C-}S>YJ*(t+kgsyVcBm)VXwQC<cY!s+`Oz}pmk$T
z@z}o(cTSC3@^5i_Q~TQ`0&!HGvuxB&+%X0tMvr8fo-n9YCR`zhnet`uOHQ9XcDlle
zy*)<BO5V?dQjqT2bUL>vC+p_1hc(WRgT*3zuTPw8K<g1)@_5S9$gKjFNRh(YlFG>S
z_&nD1LC@Xlr~z7=y<{GQ2;!WU-WF$L9Yn@O%hNtF0LzwmkJxRW_yHXwi!rs;0A<q<
z(d(h#^hZkZ^!c=OjIsN4N2aSOM#h!oLhFK2xyf1iigeQXzpgmxb^v~u+<sTq^A8On
zVmb|o9AO@Tjo8&!?=!2F)k8#pZ3D-a4sB3^zAn3(CQ}Bc)NZ9(DAS0U#FB8D@|Eov
z9#7_Wg=SDf)KA|*V6U8-lnR>({i*`7g(lFb9`Q4_M(ECoT#JNL>x_g6;)$qOT5qH)
zH>n1Fkj8+nh70S#JlX5*qeSZ({gsxn%he5}G?NA14k7zGM!)18T$g(O@BS1-wHZGC
zD?EVH3O)`dA)eE&(~FZU3sv6vR3T&=_pEro53_y?eUX>4?QO3U$;k-y=$8Hbl<$*K
z51nq??O9H+pnsC$&CXC+Mk!{B+p)M@hY=_08<bbPx@k*;M4r*t@NaQjrV3u%7)YlP
z*RdfC5G=Wt$R*>jr}s^*TSwGF7(=n>KD;#P^lrH)xAVE~bf_SEQCd7AXZ?P=W|L0G
zc%SJ>>&gzld?@2@5%hn#=2Sc+-3PUfBHtsrM&@BDu4!X&7~0Pef2YxIQ{sv=FdA$0
z7uCF&I#YZIF`o3`)Jujg((?2hns1Y!3NQa%$7S3k_(&gwgdLT7VyJd**D`XQ)&m*`
zjb=z3YqaWYFL_`ubdI)D+u&go*ojyWdvFnmYgm@8TgfNUE_8J~6fq%9+VYgk$?XXT
zGWf;@IeHNnsoCIPPI`;5x(nfENDtNzZ2>816;Hem?g`(?;s=`7Hj`}gT)=Dzr;3xy
z6iLF7M$vPkeX;SG2CA5ck5<=syR(}Gr2lHE(oyKl(JSSifBL^S!sNJfK?Qdp@tyyl
z&QF(5o7<mxto5-(=k&*^wzj6KqxBN+LYl-1D7;=px~$D+*-|7U;+<zM1<Wd%Q8IcK
zgm!l!b_V1-q~^;mQ#T+-C0r%wmyE?_&s5`HC%h7NE(<2JY=PY6KAZeuuvo1$v@Yc|
zMbod_EBTCwp}AjO2LNRk-w^gpe<8jOTg|u9O=}_LdWAYz^mt;HK<VDl*^oj}hSOgj
zjWlthksf1gyG+&Dykg*PACewT9AZiyV2n#$=c^5$k$7?IP}fMasCOyHvVaNPsx?CA
zqFMguRwYdtyx-YS7#8}&IO0KVZM}>;_ITW)T2$gcG`-~Iw5PY!x_-EHM5^)^p!grX
z`~3E#V|2fw@o#Mslir7S;&oWRZQ}yJWd8aljm|^-00^}*$%zTC?bO7hx#{!hYR{=Y
zL6<Y=35Nbw2$h*F!yTvl?wUz^j*Y`;ms_zzB<PKXuM=h{)$dD;48DS>rEr#Rr=UjZ
z(4m0p=7i;`ljqL;D+4;yh~eDN(yBxMD^fH;BdZYQ%(D~E;$G;o;p<VTP}g%SslW6!
z8hYci+>Tfq&>Ybz?`ywuFD&f=DcFpoCV-p0F>LP2t;=7o9=suOjcR@gAO7O_D7636
zI|Ki4t3Uhp@9qUUCk7D6lV`6;NdFflY(Rta%a5p?pHT@Vg4!$zSJT)1mCOI`qsQ#&
zJ!c<J9<4S<2Ph+XTi<_pS0ckB^cT>2DE>c#*CWjzoTvu`N(}Vq2)ElZ(2<LYkXMGq
z3Q}(Tfh27)IBf)LG$;Fea`P^#)u8PM>0{JSfsYw0LMYnbxC#k>XSXKM+b4L{um$G)
zGizDJ3Y~_%2PGd!4qL77$nvnMrEgPR9_#3x*nz#$0MG=R`ENz=l>nOP2#|g@k1!r}
zh8%gU67djWW*=_b6=dLOr_~i<F7u9+0O=aNK<a}hjS~pkjDa1_ulW9h@?F6+Lvl0t
zbe)jyU;p_C(nl_V#Ydmt+vz(WCz^i3o&|0*kOcFX3hzY!c}j4EUjfzOOyZH^e?Tu7
zgm9yP5ZNzh{!-)b7aqx_stbQko;vyWEs3N@qrqMU90t?Ef%gG4yD!k%gS*AZ4a?mt
z8lRz|1>d&fA|OWh=gG(P{=XbH`oV)y)pso2frfvDRZwdk1aub+Cp!U`dY&U!s1vnq
zzW0#_m4hT=w9lnE1gLxi+MCN%9|iDUDa($`8?#iMn5*_#f|7gOuY)<K#_7>lFg$@X
zu?B>!3)zRPbi-G{>&%w(-s4rPY)Ezav$)jFfj?6;e|Px5i)))eb-st;(zCGRfS;M*
z#;8f9K;nILxV>&F7Rx~$bIrs27>8>HpN8O_?RGW-m|(zbGZ+TBWB9NkPeusPKS7H4
z3l6}8qoJ#<th@FjB&kySd|)M=x%>EobfAz)F%Zz;qN)c{6O1DaOmtb}5mj<DXCSKZ
zU0MZ@W;m8<Wm`P^`9in<u{$Z0e6Op#5UOD)e1{R-HvaSI>(L&k)9bgCAOCxQ@nj*)
z`Oo9LD-p{(KuEPlcCFlG*8=U`4b9bOyU%TbA`!T8)%SZq_Xf?%_OI7%0SSujU|r)n
zGMFC)!ngsToCCz*cpn&dJaROB+tvqu*!4~e9KHm3E{y=zKtH>{?oWoFC&7s<Eqm-i
z6<A&^uRgl&m5qU4N}q2ktQ#eZAiYdpOFi2Ty^P)xI5)QekOWM%tn=L;%I_bB=IoEB
z<s2tiJ<f4nOp*ETCYC1wX^I-`1O3^(+du^sp40`2-EwA13l`JpVSXHtPFN$wN-GdX
zO4&c<;=7%u@j<GH)=?%*a*+beh88Qd#`#5xv|Y6$*U4C*)(&yK{M@vnUBzD201lve
ztc?Z3>?Yh!wt$$i2x=`-T^9cKj@QKjOc|Nvv$XiPn++(cDBoH2msI7CF4C6Sd*bo^
z+2~3{u7qu)pBu{tR=g21d(WfytAf*4BG<6CA06YXIk-r~r$0@D-<ulnpQ^Jvc7Wq3
z>Q2p=&fleN(6F!eIop)~CCXQ>jlnXTzXp5w9;y&-PSs@e&oHSQz5jyeYqo9Bu#N$K
z+p`Zkv$-7k1!@y16#~(aqmH;2DQCYt;SQu{lLp46mdAiskw2YR8_Y0LY&2zM*3@0;
z2j=SRSU>9X-ZCx)y^dEgq^gx^?LW&5AnbkfEXvpGSysP#SN6$bTGwaq0Dv<2oJ#Ml
zPU!6{C2KJ{vIHUDMMix&cZn6{Bqnm*@5+B4RD&2iD91^AIkP_}C7w)&^*RRuwgo?)
zUS_!T2^vp%V$II?O0A1u`2+jpZ$<^GHM>4RrCv;6EpZ#H8*~H9CW}aro$;>SV(76(
zJYqk)vtN}wR0>GzI^Hi(B_^uLgC^}EDxCjdi4854Y$!r@;T(=p{eZN**~78;iBu_Y
zf#xIg0`0n{m@|M}ki5Yxpuwq(&gQJLTlvrSs6o%w?{{9QdIhyax(vq8iSOS1?RQh#
zYvX}~PfvUa31+Q5Cffm7!5wv<01w94P<;YYPbu<?bTA+XY@FxuB*>!q3}n-z%^~ue
zfl8P>?0dT9kpw>6%pnRKDu$ZG16l`(AkfS`qvi9Vz5^(?2N4!@wVI3ICliT-IGQDA
zPJ22Ne)x?8K}-vbuupKCw*7c@EMuR80DE^HD5ft!ETsZ(C%w1SP%PAQ`1ney>;1*B
zqyqx2kH_BK{txk^vk;En8B2%X)YF;3kF8B{<Cu@AOTqG{MW|hV1FcWj&)vr)wp!|s
z23!D7zV3^f1CyYgRSfEtdl(MvIDRtqx(Zq}YgQfcxc5VZBusNgy+YEmSzX0<IoD>d
ztI$*K-p4074D(b~j_^<4yRUCad=3Y6aLi+LWh1(xi&j7#PPcno)Ikl9&&SMab9(o+
zjX%0k`9hN(^<q>`Q#B~2kH3)iG8tEVimJD96vSHWx=A$&Jvta_&4p@2kdeYS#I+u)
zcwMCz<PyY+aE2byHPG+0u-kj^dN=f37x7px0<GZf1P)2|+SP_pt83VuJ!(9wDh)~(
zz{~U$ztD&xi8VuE^_;sQ1_PQSbBSZ2u^6pMEfcM}?969q;pT_C0^5)9oX|f^-AvTm
z(EvxuE&6L5;fY@VZc)#5Y2N}>H=G}G-s{M+OKec(dvGI?(pV@8tl~<2^XX%3L~3!y
zy%J+T!V=IjEN_q_uXg)+8Uw*+Ww~)beW`FF_PYGHXA(u`DsxJ62DwU_feo$T*YEwY
zMAUvDV9&hks{q^q8<HhoZoH&a`?n%4VKvH#i*J6nn9+e;Fon=&?sqB@+3GoFxx(+q
zY0cI<H_=ZLJ$Y(oX!2Dim_SOT8FUm>e$m85J3Id&hdrwci5=?*+kA4R^uTV1s#0Bw
zT!4>(^IFt&Ks$#?KluHhv_5CDXU7R-0&KtI=IBFAn~i?Pg4?i=yaBOT()x(}<osNI
zt>>R2=1~s$+>GBp8Oy<J#O__4=+Bov{z$*n3~O_7p?>OFm{6e<Cwfe)4IpXi$V4J%
z%(rCB@ES~S+d;=ckQjdNPn&i}@7qsp02W8~z9@PaHQs)0A2pw9vd4IaSV=%NAB}GW
zmVFdR*<-U8cD;J{d}J);+U)~w?Xlo?lh;eblCnpypJdNZJL}7G5$VX}!T$N``4GuD
zirYYreM{(tz5Nf0V&*|J8_?6Em~{^7(#{9ZEwHFHjug$LGF^i+oc1IqESp2;e5q^n
zO)2+`-Z`Pz%8B!6ryJt?7xuU`H}+KPCH`w#{~zYwJesQheFNq=I0(noATni^D2fJ-
zsR)@VA{u0<D4GxsiZp4GAvuPO843*|C(T7DLusNSky3_)c&}|p-{<$&`^UT1yVkQ>
zo^tlt`?Eivd${iFzHYS45QO96vOLGLgbNypWVym+>`YA0;)~IOoU$5I&h0l^UiDWo
zRYm=}5Da{vo{-+X|76?!XE|thH&#&NEgahZ73~uHFd%F5$(4;kkJ8pqcWr#wo@^ht
z#0=B3uC3em3(dt5k9i6e)z~?Q<U_?$sU&H`3xXMB72Y(CqeeAkX?q*@PC;5EyMV~g
zh1;E~mg;rp%QjbW&8lRx_+(VPKefSX%16#WjQCpaaM3CvlcnFFx`sT<zSY&7{pQvM
zoXj%=5A=$Q+tM~w3;e;51WR>>=<1AoHx;fGOer3Q-0B;%<wiz&YX&36pw|1y&1RcX
zy3DsOOFP%zlih3&%hFHDv{32o3Np3#%;Q9MF^{e`NK`YDTG&^!fb*!o;4{U8#+uF<
zs;n9t(OZAKl`AYg#*U#@X(N0(e)suD_p6L377KS@Kbu$~r!wxrW84W;plEJR8&>zz
zGScYWK)#n!VcMY|H+epNSUL<U(Wdh{2YznZK3LcGtX^I2RzOppY21^gw*oj8@f?bJ
z9cVT?%DcsJeHgD`X14L1T5n&Y*uA&8Hr)0WJe~GbLnibB#V9W7Xr<G(dn5HvR5sl3
z^)g>KuC0c0xexRijlU1)Hh@X+X)lQ^&ce!hjw~m6XV)d|-`9gwem^F%+#@=crT(68
zuI-xh^XHRU`q9!Zf;84E?BCua`a2Y>B{u`1%JFrJH)WVt9rC%i`;Os^_7t^ELdjKC
zYu|yTaQ`Gr=lp;+4$_ani2892y(n5JK?I>@l#Y)JNS6U(Hm}KdF!Gm9>ODIm3Sq4;
zs$S0`O0J6I)eh#}MP<IyIltEV{fTc}ZZr*6A@zG#SNL;A9i&yuU))c+-mAsKl+{z=
z&3gt4&_j2w^U;RscjwT*h*A?$u)(hE(>G5XLHNG@#?pXI8OyBN?okUB=H6^P=&!n7
zYoE6I$ur}7g;n5@9Dt^)T$a^Dj4PSV<;q8^9mG=-3c<##5Pz%AIEd7uU4WXxA{YUt
z0#)K7gJY(JfHRw)MW}?PnIJ`smd+-x6}7eGo*eBPjTr{86{F_o9U?idpMQTwqR^9I
zFi!_%TsK?L0%!2X2utpKROn{RZto9`ilSl1o4YS9kcU3{-yj=lQ6<+%{CkVmo8!<7
z{j3~QuUO}e{x``wZj}2siw(E7*E&U5vHWfqRa-GIcXpj5LnHm%^~e~F=j1J94XBB0
zx(O?kLncMuJRu^JZJQqT`TP%UAZw`G(bh3L1eEWcew-6B*vfg~-JyjGV%rsM<;O8p
z8%jpTRI+$8J~VjM=k{9#B}Zw^g0%y3+k6&W<I0*xs1L4N#^^p-BJXN6r3JuAu0Lkw
z!h9~ZO+NNP8YiKf=_Y!&>G?#)XDFFONc^qcXXA|%I>^tu!)Nz^m82T0I}buw1lXg8
zNThtS+7{*mscqZNy_>v1KyOp$)CG1B7g%!LnuoZ61THZ8Q2Mj=I(11ErmpmLna}R0
zoL2D>f;xBK?3OQHJMPl3-FKnoF@2D?z7MM_Er5p*6luG(cmgFc+cxBwI2mQ93$WhT
zG)_y*zC=XbG+(ls5KTbx#^}`sp?TCFI&pcbK$<7-h1e(@grYEui5>A*l1)_CrHIs6
zW1Y~+lFA>@^auKNWvEBrvz%`ox>T3-3$S}}NQH6DR=5<|@=2HJuqLhc)87|<)eHuA
zs-JI=pO|t`v{WXH!GCHoRBEC1(w806CcTNgXb35ven1C8W#oAJiwsl^+zEZJc0z(D
zNvB*B?dSGTK76ruc^=3VV}lQMjc152tG9d9{#?JkrSy30?8WZKUULDyyAGvuXa<96
z(3IA2Qhf}ywba*`YOv~Za<%5FuZw%kJ}qaxO-jDFOn4$Zz*=$<Yk6<S^ZB^((+Nz3
zh}SWAZ=`Ku$dC9oND<qm8geCbYr8~jy~EH8$JE493~B5G8Dnp+-s1}R-brG^Fysq+
zAsDo?KC2G#A^aRU_l%5bBB+|V1-IAO8iDDo=z18HZGS)ef3&p?CBaqRk{9ndL4wjE
zq0i;Q%D`*4j>Y*d?~J<iIL2fK$#$r2R!h~J<r0Q$S;i~S<cnlm{da^K;@=NwFd_sl
zkc`IQ`*K~%5o{*!?1g6P;wvKGkqo18aFcXvQrYv#;CMU>MuYImFtb)`Opm79$}$`G
zJvcUJ-@60bFmu}SoKNHBSCHp%KdL96EZz=-g8_+NulsgR3V3$knh@|jHhRrz0P;q!
zq{h(dW28v%MX#o!GF%uk3fGejS-3@e>pFxP8!$%mBWM*)uVI6(WM9)57^xy(`g3OO
zvLh?Ik&SK`JE!d6a&-&W>$j7cyXWg`-C+8!R%BDZN2K{)?n~4$x(fp+Kh-2B>%B6S
zSh2tMHJfxO(#+ErP;DyQ>r;o}k5Ai(uBb&PwcF$37jJ_W%JJBbv4^Aez$IegG0_kx
zB<%-J!C5(Ij)q4qiwgE+GKQK0bP4^k#`y!N+2Ov(IW2iHs*?WnmxX^<V>c8)_M`FO
zvZF?274;aJB6gLdROJN<|2IMLXrKa%eiJ=vkQ+ry{!!)KTxHrntAv&9XSh89mC5zj
z+jB}7hBUtD_*2mwr{e7Gg9&9gkeA7jQhIpp!UE|vKg(d1ok8aU0lRMA5f&0@pm(tI
zR!8blt;L^{Mu?eZ<r-_r9v*ig_AvtO1XZVNU$Lr~0bU0o2}D}S(i%NqV)A5G0%rVt
zPskz&`vBsf4;U1g%iZ|*8pV~KdxyV1&<^y0B@J{OcnzTPKGttj!-RHd6B>$HN<s#b
z2@z?Cm0gaK!W-Q5rRsKCJ2UoT`7iX^e3`Q#lxQ|v?)K1o5vemm?7qx3*P1ujVnAWE
zl6g>p+Iq%k<&8`6D@kE2_athS9g91N2I}Z64!6>+emdwJh^vj2jr^Yf@T3I)OfUV}
z*D13#Rb0wmF0b+MRCC$=SoX~tr9rCuVb{I~-YlBCS(_I86sQ-vv2aj}`B~pEWrh);
z0X?(OktnCC?#1fSU55V<PK@11BeByeLr<iXH=TEadgZfE3z9$nLC`ip(0WcA_*`hX
z!g@`5#!oA<Z}ltQd1$ZQ{$1Pnv+pe=!}(c&XOJ_`Jm0~m_5p25?;#Z`C>G}={-)(h
z+ciDywpC8_prSR*hZh@K(@TlI{~VK;&4JF!AO2u|ub!*hc}#}nI>o->lxYz9X=Vge
za>+9!S`Vf?Ws2sB9^L&X=5FI3w9QGjy02{>y7TCL0z-h=yr4R|sy39)jJkH8U?@fH
zyIxFmu5zQ(wNT`W-vdu+{h6u|`U?h&`_JIVllB@I;^Bo3$MOxO+LjUv*dloBBYy$)
zaJ8Y4JW2cR{4`%5D&|lvh-0yQNId-@<@Jm?k4lWLr$TdTLv#6N&-^I*2_Pt+7Jb^^
z^e8pv+&aw}8nsLHoGVM7WRq8fi!bZp3s1T@PI4VLmr+8jzhzn+Vu*C=W*Cc8z(CBk
zdYI%CshU-PFn>ZSGHtW}*Ssh*zMk&BM6tfduM}<rh@7p+*jJ10>)Z#+pO9Sg_Ux`r
z-RAV{)biJBPn>`C9;n^(hU?^Qg9@!SUdWhyuRWY8Dspf>-xldvzUE-YJX9=qzN)?N
zck2<1tv$vT;)4o#@2l5N&(OQ4LQfB`f(-pt1)r^B8b8^~1Ph(qe~Nv5F*kq;@6;x_
zvP>#+EY_r0-`?w#aQqAoVXaxk(&HLLQae!PILb~o7Yo|2|Ga*szEb4I)Q5Bq&fhu_
zcbyJ>V#t5kU0X)WWP7EDQ@KIzB<DBzIQfN7Kh2X0h<RujIn8rIBm+=n&3hel7r0F!
zGGNN>KHbP?(-d}Ui+%JnGZ)QetsLDY`0S}uyp8v3+d$&<Bw9P1F|C^OvhixUVM@{&
zzb#a;!?(ES!=W51VD&lSeA;Ib>DeV`MM3iXd*7bR)9<pc0D-sg$eu-{=U+dglyQ{A
zF30k<M*I0M`xCOfvMv-Gt;p5K!e&0e(1EAgLB>ws!S5JSu<;fC?xjE1hP2P~2tli2
ziqnjBeo*RKD351U7}k*L_$ME@$M%L;ANyK>exI!>f7Z>^^V-~~kT18i`O*~nL!=>R
z<zkNbrJr@lB-8~ZNiS0arF@V4(s{aztFQM!>pqse;O{&_Y(kuIoO<3y0pJg~eBAq!
zkefAG9(MS1a>zwFg@rBQwaZllH!pYG7r%GT?UKDBSCeTLtV`*GlodgPJur2K#g#9I
z4d3m;;R)>8+!PubDz<&N{CC~+EFV{cR=#)H92$#v^ek~RcqVIH{>cO~r+G8gtKUW}
z<`6#X9_<??N4duKZq&++x#oW2-TCT9T=8zl)(VyVQTRu*?=g`Nekxm-a1MsJhkRip
zt<?Qw^s3Swisdr9cAhlV{~%1YVY<|RsuD9`5qoQM#y&qdS?6GF&o8C!#pKmpwzQ5=
zAwyL(AclXr?X4d+igx9Wl}Q<*&m*X9B%d?eGj{#7=pYx*=qY;iN~`?Uk%PzHrh1;d
zZ)hxZ#(TfT+k*jY0s_)h>l@3+Dud|0e|t&xhIq#;NtwZg!!|DiKZ@4cuE3t$V9i@i
z=~ew=@bZ}Xg#yDjX8V1}2LmKdBSDq6)&{;?zS(VkA$p6*COc0LhThv1e*MHhjy|T8
zNXy;^X2t_K+H!(rW`7nN!K8EcA89DEn(ykef0m)L<QnN~92Z1B_RU-Fw<I}^QDoOX
znB>bcJ1b3&7RvI0{X??Em9Q>y1&3DE<`<9RwO^Z7)e{4vu06~rag}{Y(LU)DV%w-)
z@<4swwLfw4zR$nq%{=|!LufGN(CpI^Pi0jF)l^-ZXFSZ0yQFT?Hx}=vQoUX#pQ_I4
zs(qQKo2#_6E!N3ZUxB_ZfWc>b%7(`Oo3h+1>+ppd8%`S@X@L(G0(vtz2g^S4g-Vr0
zy!Gt^>8qgC?Z;#JWJBw_<+GtiP&X&5#bCA*SNakKy`I^U$0QHyM>&z-C<q1#<dHAV
z6joSsUR8Y4aHs)ydw2k)ntmkLRE`Sqed~XHhc*!jzFF`h0+HZV<4^<l6MgFI9}Y{d
z9riVtz1wj6AHi3ANkL^liqz$Fq~}&5UUjNFt7sFY$2=Ufc*}*?S5l0`T9SBm27Gr}
z9Gt<ab#)bo-Y5N|iXAzp#-1(zE{O(e(WRsNN9Up@H#)N6kO$WmZq$U1+IOC*5v0sy
zKDgNWG7?kf#pbX|3+{cSlea;@+imHIC~`IB`E1epeCN4&+{-lb$PO>$^<JZp@PW#@
zmG#-kk<h5Xu%340uI#f|DOUGfkG@Mc5h@VU^BFVrlB}AqDR97dd97Axx?PV{72^#N
z*u_2!HRjde|Di(r$|a^MSwUN(w{_+`=@f?F1N~*8H|X5{hoT;ene5NZoN94K{*aCr
zRPJ`E<EO?B*nu}Dp$DC#a&@mg+<yPn=#^V(QEX!~9v2>UFPMG%q)}cA0foO~{oE_C
zTyG!CYx1g;6TP1#T!c*bYYbFZyX^OD(OxH}rkt@wxl){kgRd@@*@4i(DCa|%aEyT&
zdBwmf+9ScELVDi?wc<asBpY)YO1N4!K6?=SJl(`Sp*7D$beOLt@S|4CwHQ<8;rhOc
zW2@qp8;`~OYCdu!pQ3N#VKCotw(yOr9`d&Z>gOrHP_A37uH9M_wB#Xg+1>5WR*z@K
z<Apmx1KKWL!Mp3z@Qm|$SO<dD*B-L@5T$9w+Ve}C>Hc!=sN^R3Evao6)mT~YN{LFQ
z(i3Mm`JX?bPB&w<8Jn#fX)AM^?csNDL=E&<zL=nr0un<=M2y#7lT!{f>Xo%B@pTkG
z&VwryCAE6OqL)y)*~?PTDUEZ$Cvq?AvylH#4ERVaEHV;V(rl7zYt6Idbw_S+878ts
z^6lbni&^8Z;V_y15x*pg%{sq2uhtNk_+Wx|#^zJ*+-fdtl^=Ds<Z$|IJ-di<Hp-M;
zY?Yu6bsxt|@%_q!nJ|cT;UVJZ{$*TmNv35z-Z>Cx%TI);F(1iW)+BwrP7^X#n@lIw
zsJ!x1;cJX0DcU<mbEnsMg`W7pKmUCp1Q7ghr8%CQDoKr3dWg&ursp^was9`P|JBKz
z_;k!p!bV+Rp3^_^yZ`eit&vEP|Nr!rk`|>4jJw7$x%XTnziUEf{$oVOF=tG%J0@v!
z!xnnrs8)|)GM?gM2XkBURdbsDmF+UNoPU?C0`u<Z7ulxYEAjb>|LE`W;Ed1XTXZ@_
zpZH%Cr%nrWXixw0-%kNXAXYkn@4p`z|6nlK*#BRC-Dq*p@1MG3KhB|gi%KltoJF5d
zv16}AAxja3tSr~bMxXI5k**Gm>iJ;1m*@D-^CW?{inML>{;{!vZ_C$4Z~FdgP!_2C
zSxN11Ze!|^(*jgOvcTCirU0vU7)dFLK~a*9-eO6+x@G>Fvyz-fr$1fryZt`0)Z+bK
z_kHniF%Y6RUbns8*1^G{B5<QvSCh!jcV)kEKYySI3YF71FB3@e5;-a+GIfh;v$B$@
zG#26y$^GOg0dh@I>LrR~Xy|wL_Ugoah-8-U`}P&Joxp*wr7e((?SG4`2co(8PS>CP
zGH>C=@3N4_|6mo8Tt!t2P(6nK*IG||!42un^qCo^=hwam6%m3!YhWzS=>(75cs?Yy
z%s&eyESPiE4L5oKg5<r9UvXeJYKj<Qfzt7FRA`ALqCMv4o<__RFG7Hb3w#me&&}T5
zS+8hA6X0$`F3U9YEmWFMpS%N#WQzoSaP;O32R-76DBSdLPX2FiZrV8aL9*u$W;i75
ztIc;_UNVC89jzm%3Ku+2ILs&(1o6Njt_5-yP{Zy-0|8J%5z#?ZQBhfCyyEWc?h3?T
z5n4EhXUs`JHuC9_K+=B94g~8NlQ-<tNzSI9l{9LvY+h}r1rCjf2S9^HomvZ>Izg5N
zjvItmjW00^ox(nQHC2iy9xFzR<0gpQ0lfw9g3({~?Opk66r#6XJRrA-XMxus8mN>L
z7!BOE?uh({E-u=-=EJ=sBC~4eA40QDQ<jxMJfTyl1nKaj?&_F@w)5-DE8sC(jY4;@
zFOQ2)slX6#FRya`idmLh9v+pbDKIseeAN_L)&~L+&tH6U!K<gg<VNKfOo_`mMm#g|
zQu-ms)0D-!!;9)xF1m^F?zT>X7UK(BrbV*-6{Lt?h5<d|?|qs1&bRt(HgjGU{qW#8
zW=)b7&)0LKu=DsoAqH6Oe3PZl?l|)b@<e+#!RcKxD(oso*Dbj@6Pu>RpHL9-mJHSD
zd{+SlN3Z;M0~>GnN>%RlzDIvtb4n&sP?W;>zA|t?WBW_XqXM6vCEY>~Q-V#)?);zT
zp|s!JoC*pIwTM;JI5&#pPf744R5N>o{_bd|F6ka+<YSc`@t7?_S$4UxqIm+W$R<w|
z_B-Y=KfT^o32yyH9x)vht|2e8V(-_unUen64&Gvp+T5JQ9(DS<nr~^jf(myG*?1z!
z6B`{j9>ThbJi7FAH1Qg#lnrJ3e}MC$$y?$wtb6sXW3y$VT+U03fI+1-&69^99uBDp
z2?K?eK=aWs26+JZQSJOGW;C%YaKk-GEL{SF&tek<CT`WC;Nl!%##?T)E=^TQ;7X36
zwPzCRRviIl9oS1Fa0-nu8V|3BkwKD2hLqtt186iA-?@WVv*)ZHSwQd<u#|~KA2bi$
zMGj7D*Ppmo4|jSh+fY{S#w^PeRyGc^#3j|ed%XwlfL)TkgHLYlg)9@YV<Dl8z0BE3
zX_eKH*ad2<j|dp1+>@Fz6ij~7u40SR5~_D)3DXh6-C?dZcH_6un>I%zmoF|xLEjTd
zO88T$^U2LkG;NHi`~Y+Yr+@CNN>0^nKVCR9yFz+h3Tn%%KTVpCP&W3#8V_`-Bt)=~
zu81%Kg`D(4k_z4m0rXD4!xOLtV<{Hp&L8UfAhk!lV5~Z@yfWrPyZeOcqByfQ%23{J
zdzvH#V{~FxZR+7yMcXhl(igH=TvV;mU3Ijnt$&c%`lE~6`2;8itS<D!tp@zPP6I>l
z%bJvD^m5+omKir^R29<Y^|MthqW4NqnYYO;+~R~+f4l$E-l^4?dWB&|q5YJwHj}!w
zWNOsK&{b<ze<!b*xZ1=e2e26{M=e-b$ojUS<@^SHcXm4o@KA-1zC5&H@?udLEPlMH
zT&s5d_u}}(Y`rL*bvt?fZth8g;=xUoiISUrAN%Hi{)u9R`;;1Ty}97O*Gg5S9g|FO
z6<+<nAD~GZvPTI-bcDBEn^@K93*xra!#u*Q#Qx_Suu-WZF!`zcv9I-Ut+M$s=m^x>
z^?@il4Gf`Bom&N=zl16j&Z}XRz<RZh2lTiU0M7cWTd*rq7lD2;21@w~Sba#M<k{yn
z!+~BtJ%9QRF38Z1F|}*LVu_7%M@bE}G$S(LpOt=WAclcmmY2!_b-G9A7oy}kjIl~9
zHa|E9$PT#pJ`_)DK%PQ(P+0#?&!tnRms*N9q5tdE?rp@82qi;>HxRZJDEbcYU4Urf
zZf^Rv>O-&$UvUdI4~?Jex<xVLH`MXr*6Ed&7KxJhBEUDXg<I$}_=ARChdV_39as){
z#=3g8KS4b$dZ@4xbTk<3j%`0a-2eFCPcqXVL$a%_4)%__s`0h<J<IZuBG#|vF#~r>
z4I%+5Txt~x@|^g}09ykNKysO{EsxQ37dn^Lqo44&;_Rh6FT_f{gzy3EfqyZYCGPCY
zeYW$8eN@kfhp&FE>6%<drUU7iOwT!$equAbNf&}`1blwRycIX#Q_)0P>s*tx{5twQ
zVO^H_wuI`%X?R+r_E0>7{`(dBHn0ehM^3)qX`?r1p^IrN$HK-{c9X{e2jv)g!sU&Y
z6NbxGr1ob7!HaR+n7={We5G-M#uq{o)0G+OS}xIie99(8<YPnC9=!Hu4xY4ymrmvq
z%o5#U=UVU;^|m2TzE{06X3Am=E(+AYUKe|w;Sg3?SBUN_QH@7cmQg*zi>BWJNgBIZ
z^6Lg+>`jU&9SoeTJai>ra}4_YOuWcAWyzxAMKY!v;G=KN?eKV!VQwA|oKem!&hp&$
zmxWSSiylp%14<NbE40wh3P(oLgIp{@Z!YzylP(`yy7-;ESsdeH#Uj(Q3T_q0kIq`y
z9FpJ<dEdob9x;lS7;0!cR<vMR>Y6~oO=_7ve;*V93E?SrwJQ-@t&qE~t7#$jML;*9
z$qt*CX%>GKDQxMS;|qxaMb*yn3QSz4WP8d?75|$JsBS1^`n<pGc#O7~k2p}t1f%i%
z!empE;Az7|wuf<W{d02}v6%jJ(D6~)G)oH6>_fcfF2LHo;fDVg>-&6WT_xM-jWYEL
zY~~WR_ED<W(=Tzp3mH3je->h{MVL|mE)^t#Jy8=;y+8QZfk;w)>00mJDk--7GGHTi
z6qm>AC8N7~hc<>~2zo3u$g|1%HP)QSEDv6=Wj_oqnp*B3pR32>*d;s<Clgpd!kWfw
zQQtDp2+d`ku-z};h<~1ZfYA43SpEttKPh;@WR|yc$JYDFZlmn~<MXadWZB&#_LClh
zDMOat-_RSiSc?cSC;xKtrxXQj0^ww(UvM7(^G~!s*fhiU#}-c;l!>1xMtCnNgSI#2
z|BrcGNYx$6^yR>XsAQ7kt_2iF6QSw-M)f_-qE3=u$=e*pLV73U`rY9BdTSdhg(}Pw
z7HK&7ixdvlc2V+Oo7Cg~b2qrTL6^|k^zX-SlK%HTVcI0|2(vE=?lAn1S!Vu%wF>g7
z68S%mz=21&({t8C=D*jvJxg*`OW@}G|G2JDni_>FLphvjIQO6D88<yW<#?#~bN$$f
zov+1?5M$=&<YdI*c$r&av9464pp`lcF>2t$+e%*3uAwIuAM`P?<|mfxDhXr45DW(5
z6WFK-+I%4)C>419&FraLq&^_L;9#p?7gi>ell1tEd_{i`@`EJ&)HUOebviTLMiEOp
zxp34i`2Dc2^g6;tN?MXcS3`?H+^-+nwu$%t4F5u@#~!;EdUIzLtUjOui5B>44V5o4
zEil4G2ZNRp(k)T9A{d^L#?zZd%Tn?-+iSx!<7;|BhM~ZOb#1uV)l_hD&f--^0JWpq
z`6A;Gw67`wxSxG?TK4Jpin09Dx$}aH8<{Aw32SP>6@`(xtTWe^54CQ)7bSo><cGD-
z`+Vnq^yZd`_1wuvh)97|uk1<jdx}S*X^@3QEgd!{!#|+8bgg*Z_uIp&sAJ|p>Q985
zsz7FZ8-oh7h@|m3rE?ilXg0Q5t(ie%4TCCR^0GZ^*tie#OXOD(u1@^;Q=9tDbWWE)
z`-9eVPo5^4F&AEO_QS{3#KXGGijvR`&W%h)F1M{Z!R}8jv;q>>vmuyz-iApd@hCJX
zTOrH^MD4<L#^;)D$Gh@da-s@0cR4i-TG1hd6=|~2nS2?zBA>8pKAla1+nn(DV)J9*
z?lG4qjzf`VnQnXv`UoE}wPHWm%CKUYK?|0;YBr^xD`4m27hkqTR}_q9Sf`|wfsGfo
zCh3#@h7(L}coE-cMGRSdQ|JqJjXDMf$#}J4;$mNt`sP&!y$uX=-^%3$nbXwna-#0J
zMO^a>)L++W&!0XOVzP)7tWJhMAfEihOib}5rm!>^E_PvH7q9VNdk4YCx!MyPsH9m{
zeoFifEpR>Uq3vLMACrU*a`EHVxzl&#NvhKHZ5B#dN!(Epxi~B|cD`kzMjb6sEbaj!
zaC5G1@2une2oQ4HW|SDy?Umn@2VS2kj$AhK1fQse@ljE)Kl;29wJPCXUSuj0+-JPu
z#pnL@4H`JWYXhZ=Iq|i=7nms7n<Z)-r11!O&8dVVRF(&83{i8}^Gio(D%$$f77G;b
zZ^ai>tAwM+Ms;0b%f_*x^XpsGtWpgful0oU>-fIuu9N(0(dc6$QE9*Q#}eB^2=7c4
zu-tYo<ju%$6(3ADfUE<8GC7NjZ~l~Sdv*Z<PgsV$$1D9`X`l8RO~&WS*G-5wLY&4H
zZ`RSp%}tf{Cs|0WRK({j{;G2<C(U{2Pv5DKwZ}G8lV|0*RI#ZOJpO6?k+J{=t~hz&
zzR16ad*Yu$GjYUM{J;Ff*Ml9|!fh}KH^{{w9DOz88mIjs92ORqHH55zJsKp<d>5(X
zcF8)En=0(rc_zhq;(^HV2(_mS%m0|%p!wKGs|#(9{^vevF~rfj%DKw#|5(pcFoF_y
ztQY+Enf2uG%#Y^HuAWZxCQK)DHDT1rrX`cBJGDp@+?INnWT30YtiM}v^5!J)?L7i2
z4a9TuiL9##J}CBD9%CvnXBfQwFF0?TUwwdXP+~M1x<#MnTz*k8I^^9D4{D|Ibmj_R
z2Pn@}qA5w;k8OaYkuVM@NnY9f@Il6X?xkA-2TkJC+_rbNPGu(8J~ZO-&#q`9o`s~+
z1JVcL!e0ck1eU!`y74ZYg?TFjF044P=t1?Gq-Fhek;$-X_iqTUo%U%I2A+@QgP$j1
z&SOZ(<{QR0^WLt;9YAWlV2~V;F7mv{0PfWH{e1<Z^3LB)Id)wzQP-MtQaHgBKzOqq
z64sD0RzqY7=%b?pM>}`L$OxGC_5n2)SAva!%ytLnkfF%$*HN1SJ~&zz*2tfb(0!NF
z3v2~skM#u9#32P_3q+3`ufh|j|M6=lX+dKf4(E<<ey#N_AeRtGl?1=8S7s^;Vq%vo
zn+^qhh11%I#ueOkrg>t<Rv!OiG>@P!++G@4i0Bc9Wmw~)+U@Me4K0OfzoRwNEt21*
z18F$`F&o77&NqBh5R1crLR#<<j=wNzKR_>7=bV>!!0bl|o<l-nZ@mfN6E0X+Lzt|R
zAJZvAPa>Sl$g1i?gc)4T<6%Z`&Xi&qx~R1)l|+sf6<fadE22y`PCnZ)V&>AX@9#HW
z-<d=r!$CXU4!X&Q!Ce+@gdozPS<+?Y0PGaR3ZXP0zH;BU3!69rWa!Zm1Y})<^-=^1
z@musrjJ(CX<8qc8Tpz_qJ+ZZwuH=3^{dR$3e53?zGp)w9rm1nwH;$oIAD0*l84YoR
zRLD@H2D!+99>&hi?csSBFEMUCSA#^bZ3YC`Ivc#>XsB}ZzxeY4ZREVGYQv&L{e8hA
zM~{<dH`_stAUHuG`L!Kqo8|?dwWR7@;ovo~Y_8hqTYyOnf4#ta;*=yU&?FM?JM#&F
zc}<6fz5RJ}$V2mf^=z)--19|5(Q+-O66B4*DJB+kP}arUOGAuVF3*@Q=y-%VVs=*H
zlwd+V0~1nxpiQ$6Z7|5hnk*Lhxjy;VdT)b~3Fi2`8_yal#18s-)~->nc$e>8%P|>i
zsB!riqW71HP8iBl&13ltmCMT`*f(BF=mU$QcD)}3E1nRyDL$MUCWzhp9^OM_Ut#{g
zUXdS0VkER`9A%N`_ULA!SH)&{mWRF`O0s=xHu7-q{qo#s#{&4SMw!|=L}(qKfn<Uh
zlh4JD?!)I&ZJ?N}+r@gWDU2P-@Cz^Y=tXfNyqkP;p>fo>zM$;q6X?pM--DmrY{`{5
zVd)85bPr*w-dEmi0v)ETVa*eE^|{V5rc7pBBA)}aQbQ>8qKGFd&Y*@a5v`qhxVZ>h
z1vZEu2X8SSW)quq>#0FFhMiKZmU2xW09k6ee$5L}Jw~Nt6Kz=3qm&+NTyMd?a76+A
z^t2Il5pdneR-k6h1xrf3Ov2%4_&~x=*#JH5C7K%7Q~KE*@*FQ6;(#oYCP+0!7OiMh
z^2c=9d(CyU<uYEYm|%~j>>?vojMMhZ=scoM4<ms96KAhn|CM8hR}VNuIZ6wxU1Q~=
zc9mR+`fKB<&U7vJ@~n=Fs}5i{F&~q5edsxvxbNc0MsY0+!HzgX0o=J4mIQTVZ(qHy
z3-=<l1-$I_D*_?SeYrQ(;aIXKugh5CuYf~(+@?hnPJ*%&Zgbh2i(0>q($-OG%ejs|
ztTJyVmyIs{-p{?`xa?Gx^-Lt{cdg~;cR~vc{>K?%W79i$0P)<Ovu*nIO#DQR#{SU%
zk-ghudI2YXGTeyQ*H^8@_8%ugw-g3o6K!!WkO<%Y9vZ?H58+y=J`d*L0!&@@#)6Fn
zMlB_K-s={P{ZNLvl>p;W_i9<+-7~pe)?f=I>4w}spKqc${mm%^tfY~wyzdYTX;j!4
zKsu7`GqR>L?;CYtQ$nXof&1QIjJ$(G8<NB>=!+aHs5~<L`Bq8l=R4t?t-i@`kkCY&
zV{y85OZOYBAV2CUFk;^#A91lx#v4>x`^<^Er*IN@-d&sac>=VSF$m@R;kb^C{;D7(
zk}(GuXIx_W-l`h}$-i?2QfWfg3UG{dJ3?rXcWxq)fV9?b^yUn#>wI}-Ge{dTs6_B`
z58`lEr6eui7pv^>4W*oA`v6A3J2l3U%HR1sPk#)(qbfTj<;%f&JahX%Z-%4w4c}In
zOK6;LhGLi8#`hL0e%u4v5P<bU?;-f=w=r7KA%BC_ZPj#B|F19Wlssj>I(!EP4Iw?`
zKu%XS-A8RbXUW_1YrNv_60-tFM-JHig!gg$d>XKQe`on1m+YKx?2~VA1(E696p<C6
zNk*{)2g=y_8$tW~1A|0aAfOIY4C@ytN;6Ir5A|XI01ndSSdrT6C{~U1BEa?(U`_J)
z7KBX=ORxrfpH(O4Ib(PQDjEc3dYONJr=8vYuEqromf#8^h(3;N0s%v5R+vEh8_;Ut
z>#aTe2fQ0{G|^Xy?>`}`sf3^vRJvm&)=Z@Sm<VwVumX_gG7K|ER|PS@!Hl=`9yXNQ
z^8uf(&L=B;TbP8h+Q4g{+}%l8$~mlXQw4wm(HM78!73H<9SUx`N&b&09TrcICdat7
zuA1Drk^v-_k?&kN<GF}5BZSl0@ie+D#I$#JB$YjG>N>b5j+0k3Jy}$TalJq^+J~<k
z3{FvsYmyFpFanG0qJtQ8g}sITg}ck@@uqONyD}f}bwqb2gOae}2>fH>A_5^}u=&-Y
z@7k+4>?fw(r7S|Dlls<rI$UQa(lf3$g)NrZ*0O%a@6iKf&8!>&rKixHdtKz^h&_AS
zvVL$0&VN}EQlY&JIis_w75^V-<i&Bw#Rc6qu%Pib-&)$%&t3j{h1b;Km*7%6K^qgf
zhs>Z-pXMA;EueNiVez)LyrT}Q-;Puv8wLN>?fDaxjkn9*eNH?%AQLrrY5vR+o0^Ru
z`CKscn#VMpy|429P_`jrAmIQHs44P!cGBOy^Lbf__osLh{i+Y=1)6Q<tg<*HPmE|`
zB{ja`yw84l88PMX+Z*j(8+E=tKl2Pu$)dkJ<p6hkR&!c}oxzsO7-Rrb$2L?zaustU
zz>J8aG{lczuO}?ux3fHcsdm8ef;*%&-fu>cD)&8c_W54DnvE3!UK(G;`t?kgBkx{%
zJJQS7{;BszL2Z!-CoB6bq%?SW8iA~rFofMFm*0mkKKaQ~-4ebqlMW{VvQboC^2PaI
zGZTN-<=r`xYsvA~fJx3Lfa_12+cIFsNOaV3Ui$Uhm~teDDW{Zh^w*WLi8lC*c|_B>
zIO9af)$7-Me^vAuNXCGdCPypk9##xD#AC~h!e<q8q?CtBQyi}`FK!Ml{ng#Q)%YI-
zzlD?V__TIZvsx&u-p;g=RpNoie>W8dosa!eULHHYel07$Nk=KWgiTX`&<IqzTy)t6
zF^tWnH6qo9g}eQafT-A!8OG;Gw$lCh@MOiVkE?P+1#`v_kX|2HZh0bFF`;GA4Q4Pm
z5g$*P=n<Kov9TA8x=kO)QA7qA%aq>A0#0sSR`3q#tS@XJ8SnEswpY1U^+>`_#xAns
zPTEoSHxQ#h|0m+d9E5%dic8c|4s)SW_>C9?0!vuQZ7&OFY%1twN?O3`3newUcElkL
za}^xf&(;n^I}TgE=Jm*A+p#JukGJ?CeXn$_*P($;s-D%eELS#j*U7%`O$O-PaP_F@
zN8i2<tD2hB`G2<tZ5@#><UXvfmZmn6nky{}`br$#pn4DOt&FV{K&anTt$~6f=kR)Z
z7jpu<LkthGtU4j)8Eqb?rWZ)Xxm;~9qnK+G%9tX2g-r}i5;6VjT+=<)1$J4pBbG?k
zuZ<j7^kid^`*CTK<4H9HabqAR48eTd+UFA(Pk9lo`1S3rrIiL3x1bDARQd}MlEO%<
z$$=-+t~4TrR_t(>+dyXL99DAJTAFy^`9nkXa~&KuB+l}fWgz~XXKeMmvmBxx5AUI9
zeCup|A*npN>2ySMC9^H&clH&wghWPe*0Dy-V}-{SL8mMK9Hrjp<YkuMl>MqL@bbTJ
z3dHsW91?IjWH$LqeG_OwnVL}p#69SU)ON7&chw|ldPm4=k|jG!*orym2ZzHc%2pX1
zV?vTqGa`fjZUuTMMXhNH?Ebqwh>h|P_JX<VWCVv)Xy`(6u^?+A&k!kg*>s|+Z^GR3
zWT(R0xxav|xhXSkI|~o?xdnclh*#W(J#I`U@L{H*H#zR~mwEi3WcIcoe2wz@P9qk&
zrAX~g_U1Wty|<hYn7KTj2MOU$o8p320B32ccdh>3G4gLEWYd{twkKdTMkIFJdI>mf
z%j45VJJfyNl_5elbUvImWliuag9LnV@QG;~{*F1O2#5n-GxVQlW&S3hqS3{vhj0qn
z=|EzTxUNTtZ(o6}2>IQsN^nfuvFlIETeu+V!2E(wh%4G_L9BN`=fHfE)0obkfy@&Q
zw`lDZL8Qw8U3u4key%Ctq7b!8;BVC_un*_q_^~gC03NQrRGorerA-K;m!bUsuPdL!
zt1<Ys2)rfTg40m1%(m0z`seXS5EU_#n>V0tN0RYHzc>K6%r)n{Mh1T%czg$K1O!9o
zHW;%vgy|t?mFx(HJ;J5$4Y^HdK+iG&J>b+>v^Y`VXNI!EHg`j>Bd`m&oG$8UhqyTS
z#In4S0p&eCzc3g2GsB>E>_+M168gq-ic`7Pfc9R3ToVr5Iw@Mw40$<lBFkWaqVo{b
z86FYT^r(dc>zPg}Y0NQzUp*Dt4#@06csk5)FYCmd8yCbBdv=r$e1DO7a`(6Q6$Mc=
zOsoe)moWl^loeUvs)rIB!i>C1(M_(+rT7W?PHAx<Qoko7aOSwJJOiMfjm8nkn=xau
zt*fA-(Ki&uhnp8tlSSiP8|CWTpu<%W^n3SKFh2T`P(&|6&7|g!@a}1QNbNZZyk$(w
zPXx(HIt>o4FZwoubBxNp=F;DF=@0IyUb4RiXyYI_A5O<#(894^U7zvZ&DxQOq2gGE
znJzbWIdi%!Sbx>fo7aOcOp`YSK9O5hVHINz@6fu2W+AxptHC)V#Jt?(`Tu-9<uWh~
z=8)eW6f7b`-RE3(DR*QgnM>T*_uU0WJxyx{(q(tMFysa6j~mBG(z&}bOaCP3U|#fw
zbB{5Pm&kK3zE^S=1T(;r|NSxZLcZ+X%HnR)Q1!CEJz!WIt6R9pWZFhaQWaR6nl{nB
zILE=4ASFw*3}L#O(NPN4*2XGyJV&Mp@u}r<B;+dRd9p{#5F!XXGgWFggd_YyX=x+M
zuYNukQLz!fXEQd^q4Z7m@jtIzQh^}<+~1sf3W=98YxS5<UFxBnT0ZvXdfKF?dH^L0
z1)DAb#}e<PFjRUy9x;J?5M2U!o8q)~ZG!AMtNk41lz0*n&_)-o<`ia7v0zH9S}}Lb
zKL;3y9loilHx<|yRfvvKutzd!Pm5+a?IWpp3lpvkvDNx4ODSGz?O}GuI2wGwp*wJp
z*j$14*)?v`;+v24Ai8&~v&{6?Hc0ZWD4V&cb_*-7No_5g$XuPEK~VplP$f8Lm6Y@T
z>p>4)+34c@tP*~W83cH0CH#^4*GaLEZiBVdvfRlF1f5KZM|CASy@AD?a^#T&F$*`-
zbRrx_#d;VO!ye#Vxtj~<EY0LAushtcS)a##e@<3j#n{7%ocMO|pMd)m0${?a9$t`*
zHg^$m1&|PcG=V&$gYjhN9O8+TF`HEME9K8wh3BYFZ^s+-uFrB?vOX$xe&j#9tmhQV
z)2+WLv0jMnV@Lw|>{(aXFiPfyC>`hUMRUZ!fah++bIKcEORW11lVp2fYr+=}d{N^N
z`vA$_WJemoPnrbDdx~fgA-z{T!<&?knQzvb0vAVsPcnV#H&9r~Bl5Ig4?-F@Jf1_E
z@Cna+Q$;2Xa}NdOtQDlFyZ;#K@!c751Sdi5K<yj3e^zw-mrhb8SM^XN`Thx?2wXvu
zDo6Tp_mj>|a~$2JtkoHO*PYsfr`;4a>JqNFj<g%KOf6g5@GPz3tr|S65-6|rvMSk4
zV7$~5Fi(_+na@`K_v$UGc%HWr|KLFrrnu4y`LLGGo^SB+|12?f<vxUT-{huPD1QT6
zd<Du>KeGL5{`<%`kc|5-l_DVfe>OD>-IvI$Pgo=cPBOjz{yWnd6~9|t+T#}*lZinR
zrghB%Q7_UBfJR#Udtm7_QImkx4_tR~(-q;umC6g}Z+<m<?PO(|E-4b#i{X8eJC;=w
zSC<yD<UbPfM*8lL6&uyV5JuiQU*J%HzSQ1`ufdNlqV|;xho^kGaMOM6KpoGGleN^L
z5Bb?s+^8~DrWUE-jI|W;N~}p3t@j#-78Ne=0{RG|-8&4dQp1F)m{OCS(~)*d3JmWk
zb^OUw1&v{Ofe8YxY(_U5PDEQ(yv-#|8uLQhOL`au|GEYsGJGD!&^N$6by;FV5`3O}
zu+;W|W09XL5aZtgKqFHn#?#`c4dF<c%LYxqS}j8?pus$HHL>1dG)a@w^D$Tav}h^7
zy*-2c5;_}7yzCY%i-8jY%8UGU52Wi51K6^fTi@8-7EL(#WdVI1a7G%#X;^GFju|Kp
zSpbuYRdpVMG{%<==UT)6zWWv`_5|Mnss@3cQ21DnD_R<Sha~_+0IdBz^u{bMt`Iec
zM6E7PmtmxR{)X)DaBBn>cLa64>@5gb<w2x6#Kk>JH3+mkq`kaiT0MZvkzALCjNOj@
z2g3CY!<x#kU~eYQ2l+S+B}FqQywow!{S{X+s7yF4FjSVq+lyPTjaa%lW3KJxb-VB#
zSW<^VS{fJOG2wA2PtQ!DNeGQXK!mKAPk3-VgbQo+BFm3h{oF%bXlHzj+9*-A>*fdB
zX6F5ao~;Jr4UI=sZ^;8VmX0pcnl7WaSYR<m>)Rc%eQCZt9=xQYG<u_WeJF{YLXhT=
z6X(1}ne{goh>%yhx}0Es@!>t@0H_~WWE3rYWo-U5AA(B5^;qwHeokC(f;m^CJ60bZ
zPGmECIo!qc0IN0h;Vg6wLt<N<XAWTZ+CRbmrH@s{VNOdsO|N>(6En5GqEczEA*Shd
zc6j|i=a-D#qDOmqQ-Rpo8YIW9Ocf2+l0j^$r61YBnTvPKD};a#(xk1Few)p_W+sb$
zU7^P5c&!jN8dO7~%MeHgq}8wBA&Br`u9(MCa(A9bgV}V&QnHY=tItcJa6wBao+Px1
z{lmRhiKPn}?oVT@@r9j0p=<!CBrMtCwRfp4F}&Cg6`P`38pstm4t7Fvwa?hOB|K8-
z2v)$*T`yY!sT&lbzrVa<5YA_|7Cx}EKD!rv3zv*fRr2iM9afqE4eyQ$u#THjybnrG
za^0OD7d&K16hqU{Z|21}HrcF8lteggxm@_ulXIp#b4nd9?cl=~#4Fh;b!9El1DJX2
zl|fzk3T@0T8>hV;dv$OZ!Nw#mI=CKZE&0w(RPP22Lcbl?J$v);nMt%VDc#6v(&kB8
z(<$$M8#>0aO?>R%P~~58)!#EkT-$Ip-i&{-!vCqOb%4V5m%aDz1*zin!8%2=^)R&r
z9p{5%ZSlFo|J3<fW?(Zt+s{{w_fob1z!aO0gULaPmP05b&M%pF5=xcKBFA}CZBHJ3
z@XtrbSFmL<KChE@Y19AOa#)!)xK+!Qs-pkBS7@!HdUh-}`X84y?gl7IxRhT14G)#c
zmHfLTIz)i6*iq@fegU_IIH=)c>Xm;VdHf}q;U&~~WX~p!t|J{Jo`*+1^<rBFBaT}Z
z2Vja(UQxS!|9lIT!>>|2?>Q5!M_h!NM*g6i!}4(A-UcJV5q}H<UM$RP$fut&wIt|`
z^=uLncI?@-!xkv*`1jtCrartiW+I?+-5eAy$zt~Ua_24ihgn!O6HN^DY+EI+zg+PF
z^NQb;?lILQ^t6a5TM&6jH7H|(U@)!4yamA^G1->>iUS1>9SW+VQ>OG37qZnwbAIA}
z=shA3Nn1S{nT%k64m&%=>;#EC%?jVsfrnhvg&^u8kx!T>Uc;{eqto{9utXgL4#^Ig
z9P%=!v*30$9wAB)0G3eUL%6k5T$=uOqoMQ*=FNyeEeo+}9+B5x&;mIKz7djG`@SpX
z8N$`oT+mLd<D8|s!2@M*98}HOt;qJcr3v{sqoXtJzPiE&L2b*DsC}XGa1cDN$soFD
z%VR-^GEBu!RIn@oKTmNc$GEe43w)WlfB#o_IL{k*;4n13Xsw~pCR+$j*6)=8eU_mN
zTug}}K!(!T*~4>>%~r3hSkUG*y$enel+Lq<C(hPI9)IxNfvEyfej2Bq_6$bJE?QVo
zBa?QQv(LtWpJ-19N$t`(+}mm3mEOO|e=aQBlXLs`-z`Yd(nC_ec0YIVcs(b8jzlLu
z&@!FIm1pQ0EVt&YG1mgA11(Heb2H0QG?2E({Dx*p`XBje6pGG598yW0^Tal;G?qk(
zt9>Wb(em2(yZ7Z&NXxrl=}lv0bV1jAJ;tY-s@P*x7m}z)4EWA(iy9TPe)OC{0**i<
z7~Y6XsqC_#-<m3pbMgShxWgZrLG0%gSv8MZ1`P2)SChr-IUWx8k+B2e@x%*D*>nC4
z@6<V?-b1JsHrQI{PrUIPWN6r>RSTd*Q4R{{ad8EZ!Vxx?l)Cx8WzU=XDzg4M$rJAM
z!#&-*ym5;cNu41-PbQv`_=8O<D8F`lo@L9Y3=Y%R6DBv@==3BHnA%Pn+2$WN#_Bsv
ziO67in^+r^|HkgqWXOE#dJ@)X!n8=THmdm7-eo9Gzt@SM$d<4zEI)2)Og+HF;YCrO
zz*iWNdN%(PSQ0kipPD)0&i}@pV)Mam;0u_ofpS2Z0z;&!Y~5VDNn=2q2r3^D#kMN?
zouovhu>?vS{%FJhF;Grn<^%p@r-|T_nMkDi8FG96`PldkQOk&B{q?GD_dj1a{?%m!
zq>-3VKV?Ct9+1{UY>{`{RzLl(aqIzqnB?PSJ?p=Zd`B5Iqy5R^3#K`k6Ay>mx<@I?
zHkl-AOz&iJ0rG>EgL{h!F0!x4Y;bZf80%nbtPB|~)4s8%%f4yx!Jc8Tr4~yB<7jk;
zcV-LzT{Gq|=|}lSS<P`rpk*xy`UH|Gn-_v|W*w0naGQm=QfV#B%Zp2<pCh4a@b}^A
z>$vY70F_rIw}sefOixmP)4YpgH@-BKFZ&cAhS*2N{&Jik4tNLXI)W&$KRqLVQa8CC
zTNrJadCGz^J#4p{#@fqWQY5H1rdkP5XuZjCWobGGaZN=M?CG<y3W*nT<;<mXxTzaZ
zqknR@2GPUTOTWIpdI3pYz-O_FHYna`0_%bdf)&Qe6GVvMk^Nl{0I<KP2pPlEcL`30
z(s>3-8S4m7;GYE*3Zm?r0?h*dj@mnCvq3^dY<%|i-fl;rMKHnVLFa<aTLG{Xwsp}~
zOl!!`sJ^)J7U1HW62UryE#Wv=*+Ixz7jJ!14`2+qN-mL%+x<-1_wF|uW)SjTkTN(7
z`jU55Sazh(^a8%5M10A;_A+J{tb*3YPzNYC#mqMmT&dV!+>o^7q*<3hocb6_jaHAC
z>9Fs?6C8=`W4o<0%nqR6X3sNCs)3(SeWb%z+7|$(`RlIepAiPimml!q;Yr;c&BZXh
z&*B%aR%dTp3QS8rrz}5LfQ)oV0{sYpnQily{YC)~^Kz|G0R$4CzTXhW6hKK0%1;KT
z4#AmyxrH1eXXA10T^}Omy+=^HXA^cvgRR|8dyS44nKi)Vh^Ydy^|qP@tYI+n*(Kc4
z2=NWxx39h1^Lo6kL2ZTv_w@;qDou@B?S$m)WYHaj-7J$s<jH<qQSEzg!+XMS4!tAX
zS>WiTdvHhqZ(~_cfM<bDMQI<w+K_PZ8MI^0P_X<Qg1`|N;C8q&{09Y5Oa`xjd|<r&
zC<<rXRFvX&<6%%>ze9o!k0WRvC%m@YLH#o?(MrV?^+h;51$3h=YUp-2M%$|KTu&E;
z&rRoSZA8K?Hv?%qk6b_&Prp@M_8)&rgh=XsA64)kB@M~T_H?Gt>YsE)JG0RxqG+97
zI~)X_n?-p;({&D>j$7dTG*jI$QJs6H#~!vl?G5y$;R5Z?l5*AA!Hn;Qa%dSGAo8&6
zvZwOMdlOWAKR)0+&SyC%*R((^H=RR<h+GXsG-_$_Ug0{TMV4U7{8@p%F+1zcLd*7k
z%}P!<&v3HbgfrG?g<{N<0aX`r-54Hsb3B-(N|Q3q6eL=u)82#A@n&7B;gPdyrn*SM
z9KP)0*KN=GZBI_w_x)XhW)FIHJv~Boo-~~%V*_Xka0a?v0y6p#;&#SqY#J<5P+(DA
z-EWgN+bqNL*i0=oUGvRf+cX{T3}s&uI@|b?^Iv)utqM`yy{h9yFqL#Q!P?@Gw|R$W
zuX{>ONL*;09>pPEbwRrqHRl9>#$)z1-u7XQJhgG8S(4RS9<Q!mLSaZE1=Ux_QOh5c
z1+p#HDwW9zbHAT15Er{FR(;O0IeO>1-}9#(K&moxLwHBD)jYyWEN3k`l=6VbW2@f5
z;t2XomHNFLqC7p2LDRuJC`TJwFHiCOJ3I@^B$Lp-*L=MAc4#mt4fbGqTDa9R2(6e!
z-n`96jg-rQcXvNx*a-WpeVkS0a<VQbqH=nbRSEo^wN;+5<)OJW)RiiJx6kdiX9Vwu
z$tm8;82a4&jFu?rq0E{mK8P+*Zxj6m){M78d#(=1)nC`#MGKVmDj^xoJe%#w(>&Aa
z3<V;e%6@Z-w$V)jNArBc<upBx7%teskQMY#edSWuE2{8MT1(&-`-CRI;87{YefDBm
zdZVUoWDqbr(br>udo55rI#e8T$Gf0B<E>Ch7?Ip*exb?m6t~g&+8&)UxWbmF;tNY|
zu0h0qKre{0<g>DNm8@&%<IH}E={$MgJA&85s$}2fcO0~2B^im;$7Nm&P%1rVNv!mw
zi!zctvHs)}+4IR*2%05NVaY8__Cc~P%=ai+x5D&Kb0W=@qx0?UYjcfbRpS3-?KLR_
z8nLK)QABgPWzme{IeHP^Gn*}rsjP%S$=k>H!r#0_@3pj75nIW_gLlFnt>teBnqbRY
zp8I@z;#(=|2yY`^vbF6c{$GOI!<k7r&tZ0Yz6lsjGG&E(Rn5-G9p7W@C)9T(_`#x-
z+PhGHnfY{$e!D+OR;pLwrohDQlL*3m=->uX;&CLKNzEow2)EsH4|ybuJ(u6V&s9<&
zGG=shYx~64M%ltkoqFyI8y4me>h}E3@K=umRg-z)=Diy)OvM7hGhM6ob;JF%r&ehy
zUJ)b8Kq>r(-HAAZRk${5&jN$lMI0WdD&ux+n_cwuZw87@H4Ek3{Q_rl@ZIz=c}D+0
z$k?BKI|ef~M>ZecjX6j_&_HZcxoXnh8HsfHBixUAu5M?MZC=Ko)y{HP(;&%ggnEbc
z^3E4QDInx^L(A!uv(YUjSkK>p8-SF#Z`qpqXlXm4nT2^@(#ivmmKg4p{ynwLLxpgR
zwOSQx;g@|=nfSb>>@XlWuL~|YNFRE^6F9j&>s>0gej>Wvz_Qv8xU0C>1*W%nBSYKj
z&$fmyqjkZ-_SNt|HDClHEPZ-T`8)sy!04E%S+guBqVxvbk-fxi1nxw$5Ghz9sE5At
zY!AdxWsPH8(M&f`RJ@vzKaApul1=xO>w4!1K0cUtE+J!Mk7Cy?2tr+^KAljpwLgKl
z^@624KnEDPZ8Z|=n<9R8YlAU|FDs}mmQURhQ4!GZ6-Woq5Qy|*SY9}MG|3eTLO}DB
z9YJxt7QXHa+)msgoEaf2q&-{n#l`x9-XHwsywW?DLpF71xMwfMNp@!{`~f`%b(A+-
zniErQ)s}<IVwJe2^g^Y@)SGASLkKs)1*E<qfoI{}T%5hP>BSC(WsLk$+@r(+Q$$O%
zK~aQwRobt|0j(oIJ1~um0Vr`mNZ#UNSx0g*XrJ$dJKk4>5Jc#v0%8tQ|EGpa4xEH$
zGD`g#>=@oXtABSxJbGhdxjhu_ps18zWcy&AoaL_)IOqYfKsGZ+b~9x{ZSTGe$BfyQ
zAiRSpd+Yo$^Aqw5td=+2QMF`*ZrQNxU1=ZuAdsjzMsva0`SaoE!p2+&pSy>7y~dD8
z3onFZ+XI3-`pi1&Gw!uP=oHvC4k9|#CiEyH$e5km4EHvYT=OX;m}?_D@^yzpff{>V
zVPO_+ytOy4^c#e>1sQvWWL~?j2?oUmj{o|z-4^Sr0hES~eiF`70{v(-;*II)sU(^*
zXh<8|`hdWjarpoEeR;cl(f8dT!Sz4#>OXuFo^)Un(Q(zB)-?EpK*P>mJFo6pei@1J
zZ5PYe{GQkJK|ttVvpzl`Dc_A9X>IKWj=*?=NZQc(8bd&eOA~TB(Y;y{?b#e=*ACyb
zMmg9`+)n?D49v<;_z$JL#s^1d)dbh~7U$T--8~z!#neoFPji2#%gUgGkXwCU8k>-^
z*UK{0B}ZaqK>d>@*|R>uN^4S!h~}cMEWt4Am$JZiqj@?qNy<t-Awiuz+n&anS1;ZM
zvB-usue;Pe=}+I^e-p8%?~(0;GfU!@bh!xU6>j{1;pUbq%uhF#8VK+n-v|I9QgEQ?
z&5M~)!zpoX`^ziiG{Xr$OV}`G4ct;~2s?mPdhIMr3$umvykXnVH^KHA%6^x<EV5I#
zD|1zjE~e$6aCS-Y8hl*2v%^w5?SLx<Rcn)$ec4(f7+{lU0MgoG**u>;<_%{E(&D+a
zdl0yjXkdiA>5do3K)XpS0%T}`HJ{9}J0oYFgSjLR_^K|h9IhDqQxHZ`a$lHsr!$}~
zZw{w@#^D!epzj7zdhdFwRJ6MX^$x+d6rfs}UAx}-YelPx>4TFJ1+RoUqr21<yRC1^
zNp>8(z4M{$`;q#ytO7R~XrprY{u_ADcGT^Zg1$VL%{>gdW9N@O2;ZL^QMrP-z5t%R
zYUxLB&a+2-%slC0@tt~I<RFmKi;G*k#i97K_w1wYZlFZn8*TDlNE<i!#ESKx1&{PS
zfM`?EvULTTJkUVasB@NLDQP&S>r>01mtKVw`~e`l8vYv(?J@S*S>nZ5Rwep1FZ=5g
zEB?m$8Vu*iVwQ_f-*&zYL^xf4DWvV3g9hzwRjR|-HQ+;zn#@au!xrTy8p*-si}z`#
zFRe*<n7=b`rXHGFu6$ruW$&{I|6IaCKd8rQPV+sfbtBoMrY>j@N5bh$Pj}>|Gbc)A
z6DSNtk@k`xc<K5%jRsdXuDyPFj~KXSDR;&Og3iR6fV84Xs%*0)>JTEz?(;6qyYK-G
zTe}8(THWbfA~W~Ek~u%yJ?1&Owr`y?pXcnbN~-8aMvz6b>)L{KS5ilpAjb)Adm?h4
zTV^=B<B0w|M+>Q|q)&g#4=$+dv>2IzV@78P*OCWOW$<XA9n3AtnfC>ZrLX&}QE=!L
zbSik&aO;KCePNAZ$;#`su$W2h54`2dhAxSA<)2RQ^a*3PPjb)GyU%y|<a+mwY)yV`
zPqvEqS!AVQLhoAMynVW7!JKlw-^r32I`id=We2>Dw8KK3R9oF`HHwHtytQujwpDKl
zhJi`=LXCH*a}NV8{>G*uoZ#7zbFOl!>aakZQ)Bt+mrLBDZ6k||?;#5m(X5RjFR+s|
zFwd@SCJz+#y$BGKOZl8d%F**%xq(CArA1#MZ=>Rn>5G+K{zuLU?a}vJr9r>n#V^}=
zbT`paNgS?9d&%IJ@3fZ^432fA04|^B0g(k?I0ghTt2a>JCr{Jp&Be`^?7~+)?K;>G
z?F3C$m)vU4u<}(39*;^(pJc>A^DUgco?j#KD;w+OWqA?NHzVd=HjvxMHs`XKtp7$)
zf5xT$Vj0Ft-K)j%TN$;hwy<xyAYo*B?~=l?(mOdD(iZduY}6J#(7JoU@$RN|{MDI-
zKD-RyBk!)Pc289om}!gzO@tlNaJqdPidWDd1bE!h-&3F0?C;6W_jQ(?nVzoZa+%J@
zvo2o0V|x)Im*8!7`p~50cf4|~*tEq{lR1lD_UhA9Q>?aD6@*3?l68LJdB-eh3!}@u
zQ)V?eUawJk`wM0C+=sd4Y{k1mZ$7CQ?O)jY;9Xm4^G~(P<KHxn(0E3@O^x~;H9xsH
zZ_4fRZUDv?EqSBDs_6uaV>^*&G$?T8`flN>4s4PtcRzWVw*9Bf_9OOTwIBF$>=!GV
zwUsXK?=GoG%h>;Np}aaCmc}xCT!F^e#X3+YG+=YKpk!{hcJ!i-UW_*II7H$VvOJXV
zmZ8X{l(s`|xbMs6h1;L!Z0sc!f1?2u__|tTxn-1I?u^D9J7vcmma+-`j308*Z$0#*
z%b!tRB=n{}o^|Sey7Y>_#3Y*J1AswM>f-X!sLk+r_}*IRSRlpLX)<q_|D&7@3*#F^
zZ1*{I%~}_KO16G|LmA6?I}W>brXo3K3Vzi%Hq>0+(BTs9ZsM`COmp5HCB}{L(yNXG
z@86;9V5&ds30jmRe(;<>tv)!%&9qNPS1*k0xZZNP&N_*=hxU&8RYdHKqMjnD4xF1+
zBJ>GC+E<<OjYpSv%BVM_oge6|daVpd_!5cy<iIDtI_ncka&v~itnuD#x?_t+V`g7E
zuWDuprg4bANf|t4dVAQy?+X+}Zb(S~t<(0D!ro0(JhiN7U3U|Y_nCbxwk}`jm_3^A
zVu_&ch0D!l#f~3q|L|6nA2abz4{d*)``J_1|Ig5#<IzoCVQ7wi{^Wq;Q(BT!k!6LQ
z*-^eM-ODAv^27o^KymeXRgm4z8h?vKPXVRN=X<}cE<gS8o#1^^v0VhmZ|@pPuN8%s
z%kLq;d0fu-W|nA1eu=v;p6LFvdT`asJ(r3HE8BRR85^W&+h#5jNvnGhO~eM=3U2Oa
zSNyyQY0AX`mlMQWHsxm>&$!TWGfN2;tfKh2XbVGK!0Qvh@C|&gLcGqe>GozE5llGY
zRwMtn=r(A~bQqKBNNXQ&P@*no?KUk~|E*IY=fx%MZugskHM!PNdWTA^QrEY+yl%=a
z$p0~>s+n~+Ti~nc)%-nQG`wlb1Ky;v2M@e2gA5Ew5l3V;9j|oV8v^=`8k`y(IYa{(
z3WpzD&%M*BmOD4QfpbskDp`e;gPir7>*Lt_cbABSWvF$mJOF=Lwev*d|I^-?|HZh!
ze_YcxN~G=3ky0n^3mGX(BhisCo%TJlv`M6sQ_>!#h$N-c%2YFLG*OY_5K2s%LPANQ
zO|&ZN`?|I8Ie)?ThvSDH9+vx_xtHsGU9ao;y58owVG*CYHhv?tiA8bj!$<4+Y8}}=
z#k+zUtvT2I5_XQ}#E7$egW?P37I`{q$rwLERyNp=?E;Gg49<DEM!sr=KtF~ZqP>&~
z+GhhoDg!W-aO4~3TpV^TIWaoXnc=Q{EJW~)NbxpKxvE_W?d9R3-q0b9`^h>*vZ{Ux
z1v?}Wb(WX4hwutoFw7p<ZB%GVJ-(TF@u7LCg}T`)s!z4uye%*Nc|CvIf6k=uqEPQ}
z##&WM=bhdEkeCboZZ`d4QV2>h>!SnrWM7}Cg@<_uLQZ`5!R{oDUVGAhk|RaYDx<_X
z2DlC`Zb-FUs1<AbQj>XOQxSGjelc2#u5?4@+)^FRjlaChA3QzjJE{{fdPwSzf@iul
zM(>LQA<1=b5m8=~e)E!2TI<04@zt-c?RRAIsd>3b_c+px7y1O`Wg4LpF&t~P79A;m
zmJ+e}ecpBR4oryiK(0Ml|69}5eatP)_R$<_Wck7}rE;}|$VQT8IHg0MW+9|3v$iX1
z;Thby2T(m@+`0gkeNmDq5)M6zg$B|9BQ7^x)S`B2ddu-Y6KLI)bG~WryIP(7%il_-
zOYQ5bP4quWstGuG8vP@*E8pU|prTdg+NGRU?cIlbqvRAtc2t)<G|=2Vkk@xiDXl^}
zPs1gqki9KR;JTF^-I%jabD7wf^Aqc)_a>Hgo^yL^*&gJrXK5`PY(0`PO#+!I*@6bc
z*`QTCxfQ1)GuQm%nas`#?h!)qN&hV^KW$9Uv)Uw;Ry3OTjrWLejSYo*p}{pO_=Eku
z7JZ@`a_?`i$?A0p%jyzIa!h-5Whc*kiQ`-8`T1!<o4W6%iIfjm(45g`UA9X5To&1G
znN*&UOI1M0Q~LW*0Y%3Xa(A4vW~n)+TZNwUIPSr3)bREG%93l<DmwGWYwywb6j&~}
zP~}DRI0yx5WUEZbMD);$;!*-HYLVkA)4m)g^kl($)sMQ_+lMD7jJP>SPJSEe2)S_a
z`jIv6z0=F=R3bYoIGg8<p--&y_oMDmO506Z@p|2!Yx#p>C&ZLQioA7xi~vCs8X!!r
zo+ae-RI;XVlR~apr_-cILZ82_ZpBnBHZp<acw|E2B5M7!Je+@KUwYB%FnvBE7Ys#g
z)RXas+D#4$8oSBI<-(I-0Ir+o?pOI*pUHnoK7fwKoKO4@d;Z-j_)dk)OIM7T>B3Da
za@6BsWyt>@{zq666+$;=e8kCa6xi<R0ZI=W5y5y6q!8}x*a2$Q?ZP@hk}(1MIFp~9
z_$ohtudre&%JB;n_p9&?6k4=&O!qi~X|P}jcWysFHqGu&U36+CA)Th^m<ZqslU;*>
zbXfj1LUc6YBZ1He^It25DR$9)dz*9Q(4sQ+L(*ylmcsf2OfT0kp$pDL^lm-s=`n7a
z`UKIE2zd$2><SQaD4h>m9T0DD768&|t^=P+<6{(qp!`&D2<#=YEpBNoLgsZ8%%;&l
zWO%L#!8^kf@};uh^}Bi=paG@Wd8wFz?tp-3Gn%AG;m~JOf+7J(6~Psiu)P$TX(4l8
zRs;r4wno|8eY&APqf7V}Sj{+8J^=w3sZr`2CH8I(L`nX4#-j;1pV79^!uLQjs<Ib1
zpv(dy;n~Koym^NJ?AY)wB}QhWSjn}$)(hZUmvB5xJ(-|TF|}lpLgh(Tp%psas+btJ
z)#!(*h;9MiwgL>N4MP)tno%9E7V0HRH8qo4OolDX3#O=^90JMC=yW)`9ah@(S+^ka
zgP6@lI|eDd*%wF$9AbZG<L)$Y!w?%l5ygj)1*EetW^gN~o#w@qsH>ezekkETlQ6KT
z6CCdcZYbd@=8UsVj3u*r)d=|3VO*sG%Q~oC^#DlVgQjuh+}n&DP=5&UHJLY@`)t8Z
zw7JcLs5?TSL!DDJk%bZ?8Jd<Vu%R0VMMtOOP`lM&>||a0y;RGBRrza9Z833SvIMic
zJd@7L8P1i5s?hdor;xm+>Aa?E`}gZH@P<AH;%?;{E<&)jt+SgXQ2z>5YnyP(ADvXR
zUGShKRm7qN(k1qjbj}4Ihp_OU>QQ*^SDDG6-AvyNPhDhgBsSsFJjTl2fdZ@NE;?jR
zN9jiK)hPvOL5-_9hq(QtsWlQV4IBRcj9H%?rsZ~~_<PknUnjlOKl+<Ndc5b|Tek1Z
zCsab7Yn7N^=EDs^NUKIPSufC`jduV@*8}J(utQYcmlfQznWUzhwv24^6a-ro!?0$N
z+epw-xjB+6vOxma0F+wqF}^PB<IC+HlK+%WNp`{)iGJn4tvU}pKa~ZfJ>5Yaye@6r
zJptJc<?62s9QDd4@K`1ktK0$x3~5W$`hpmV=0d%$kWB?+z==1P#p5yBC^6fn+Y3*=
znhi@c%e|aCu4T0iN%C~--BY)<>&kS)r(S%S6;KGns=VK-6L;mcz(DWI)v)N|VYJ33
z5E4olHB$>aS@Er4`aBn&;B&t&&^fpZhF)3sun>|>1$$AkgdFYU8xjNZqo<VaA@=fP
zcwS5w=Gh5bul-Se0*IG6`ATPGfb^Z9A919v{R6XCbg%zT2_fd40i?dBtatG3@s%1Z
z3&q0EYqz8KZ&Wl#Y&V41$92tHK5KvU)So-GM%|$XO+=sMl7!O}kMMg9oo}=2*6kPU
z)Bd>6q;oQU$r$5i3Df7Ly!bb;Qn?XWiXd5UI`U1wsAN;bxgDb|rTeDe-0zAEe%Rp>
zl>|)=WRs~RAq1n?b_n|QR{QiZf(r?3^4zJvFA!&2EboPKebQI<*@U+|@CMpZknjeN
zmUDhMHTx8CzEdJ^R>5_|B&6A5i?84vdC^6RK^_ckW~Ey+h-i5byYQ8HDPgzLOo%ov
zaTTEZ^?k|$`Q|P5&}+v&^QMV}bUO49i$O{9d4LHBQxKbnhj#XUz_6zCqKLw84&v5c
zhsevRLz-lLj0jkS-~0E|8I!v0gCt59zw{BO9{IoC+?gG(fp}2E97otubc`oVZykW;
z%QWQDjB;xd4PcT&Qr30m!@(LiW3*hrfo6<^Z+Spi7R!=X(pXJj?(c2=BlOEcpEhiJ
zmdU&UGn8_7DWk^TN)>ZLA`69f&0tJiV#lQ`-M@S3VOX7%SY6TT<@~;p9}QCMhN?Vh
zlIhZqEweGlC<9jg0L-AUxQrJN_{=+O&;Xbs?g3hdSCmm|n50p;y0ZLAv38i9jLOkl
zq&KcNo{lI}Bs-Vh2QorT24f8g8m3_lGvdAoqYwdjC=-hj^q`b}Y>c&LxY-+X$t^f(
zn2!$AG_lJ7I*(|M6T6M(1iA~LksN?l4VJ*7RQD%P!;iFv|6)AaVCiUn&mo<wUkfht
z&&ymNSLE3OMP=o$cB%{lar7s`7zTtDD<^91xQ&*^Rj{AM6__npJ=|w)MLBQ4iSbqP
zIO><dIMZlv6@Sq<@rhB~Oax7qO`BNIPAIBj=Yz(>2YaQY)*Z3(QQp*d-%VVQ_JCj^
z&-k_3V`RS1)xk!49l|?p;&UT2doR}0&M4-*+>ao6Y~>p^G?u0Y3?)Jpgm#YW)QsHS
zpT5Or_n|j&+t(%Pc<&KfR4Cn0^JjZk_NL8>jxnle`q}sc@b$k;NUlhSjVTVUF^+6%
zD7tVqcR8Bvwt8DLaCf3sJ=2mCy+4UH8E>X+o-c4o?nXPbAA4>^v4BEG!&cYAP`>4`
zl#0T`*lzeuHFxJ}`Q~5uS7|7|O*dU*8PlgOd)+0v|K%rVJ8{(TL=F=~T_vdblsaL3
z7YnPtBemRcOw70ZxaYINbWdBUgM+`Hl3Gnq_0!1nts`{N)@P;Od$CU-N8Vx^#-GJM
z2atfoK<P5F5h=ioNlLC07O#sDhLe^$jmCEBbhQtGsc)+q9C-Z#e3pB5g>I)?(UYO-
zm-U_5%vyD4q2?1xSy*8K__F4CmG})Z_*Pkc{A#`jT(Bf-0>l6$8ID8bPGQLf1%zno
zN#o{`U1aDK|H(w(emKFw?*q1@t9ugBA>y~lMtgk>lDF^>sWOc?>9M_^?%Vbm44#gs
zO-CGIuaiMp6SbYm67?d8M%dZ~(Y3f>Y1)@Q0j}&6Ec!%cv_6zi(w;!~-_;~+*Wxv^
zI>u?>@Er9q5QIFAZC~26>pex@Jx+bPBu}H|l}P#52_!g_Ajh_TpZH*I{+rl~C+XQo
zhvl~ysU7Kq?z9dFfi7cl!5c}p9bxiYJ3x~m?GYWELpEKGtuO_=T%2y0Uyvn5@xo(T
zQ~d>L(hB8tL_>>{@#sN*Lamh>KFiElMG)k&9G^FTuC8<PVOUE6_6D|8a+!w8Dv8h@
z>hB0a2Sml#>9g|})itgu3XBxahK18Xw{i(-;(SME?8>@1#0r>7lFZ3B=|!5z-bhKJ
zx{rL=eLY}pDKXN={u@S94vsbrREe9iySdSb+p!wrDZR<;C_Me08{EckY*`jQ$1XzL
zWw*g#nYpp|IOWl6RrR!w2!!q|eOSA5C0C49-dVbWXPd@jiZHcn{(EQf<Fe^XxXXBj
z+%%tJf7CO&4KTeHZZo;F-g&&C`0ByjuyfoTW1iyrq>Tk$k#uFniy7(sbbV%@oM<m;
z4|QB_`wCvMxwJQ6!z^|gwo$~o*`3_gb<v?j8n+}Ayg=faQ%5A|RvtH{bKI0+nog~$
zJ{z_9f!@l5j+rqfm0bilw9N?aFig<%+w3X1FQWH_;Ua^>T1oR#!7-BOB^nFKvbzKF
z9MgQ8Y-sFDy8zN0TH;TqSW~~*hW{p(#WrEyb3ZlTMNhwIRfs?`29ZFih;qcd-TMx<
zyIh5YX&(CS!VGr?H8kQqGkoZ_TSSc~Z=*?^G$3{r!${;@ISKhXWJ&cMj{|o#*;AQX
z>prV8gA#Yg>iW2~-?xUT(r2@`8FXB)Hu&J16mPvXz}rmR^oxuf>pJ-uDBT+m4W(Y3
zUMIN}FuBX?wrQFWzL!%lnl%2`cqRWSNbHo1DcC=pG=ZIH617{^kFiukMtiC`d{G^-
zejSo#1+`?#>sxOJdR`3IYQJV}<1}&YFCRRon`*;zt2$aWofcBH6r&h+5|oVP`3(wg
z-aO13LsIMkE_;r=L>dV=2{~kw=(FMVY<uKizq^aN$V8Y&D$Mrfit+4wYhYT%vfi+G
zz;w0bd5U1|lA_zO%jKO9W*n8E8&BS*r-g!Jp(kp^b!L!B%m~4#vFCqiA=9*^wte}=
zE+O;x7#^?}8h?5}Fz@mHzq5PGUPUqQjbU8sro$Pd?>a9PVAycnf?zl7N9qH`zql$m
z|575UG2FP_M$trNuFLzIgEp_qvMn|fW1J|jZdC{N6hCE6WNgxPnINguKb>8y31Bgb
z=CU~ibIA_%G?{w~<eqTT+29{t`SyBqj`=f{$ZYfSgR_-qEKy&Hxs!U2?10vhRAW`G
zmlwIFy50lU#NC{BT)iMA@0>Wt?dwpe%~dbt$pSpjwDMS0FWe0`j9m}tqJ>4m-NEq9
zdrl0SU5qh}TM~iJxLlfL--V52FE>ij7H;HXy$|y1P$z1HUs!jYurlYp<Y}%sti2b#
ze4DeanWxr0xg<~jOWNp#!Xd@j^wK#|7tZgKD)%2GyPvS3+%~;GGk7JLK=KkFhg-n@
znZO(Mi$;FVxa3F79B?S#+xUpL=>xmpUGYxZ?IuN&fP?16$XigTp-G%F6vMr?U>~X&
zBo9RimQ0M+V)+ky4o80Tdr=@ha!$i>@$(LH*?ctzmjbmY)@TER)<wpowG5b>)#V4T
z*k>+a$!0bh947kE{Z9w@)lLw*I=DXxdyWZXV&6C`V^Nu29g04rAzZU)Yq8ah!<Ut@
zzEq>g6mMdS&BjW|BHbl(l>LLKcR6M;?(DW9+h`d#bMG-M<dR7|>08zFesbUa^d7W$
za}7qp)6DnQogRS2s^NqA-e>(xhIGEpSHD~NuKmDM%hAGXoO|T$lZev|9~WD*mARwl
zpN|=^djxXs^q)esQ`%P+1WUX#Wv}<<$u*y2=jQb#hZ*?!fkN*-&ATjCO2h)9v0X3p
zaN98(T@&qB&}tL>y7efU>zP`cN)}!$w|e+w`BTGp!2Wwa-IJa`7lJaO|MFnC2g@kr
zPkyf)ivx><j<andbj4l@{bI-X{B<vf!MIl@$b3ZN6>xcN)TJ<k7*@`PrC@Q@>>lDJ
zCn$)4b<N+TwcE4(3xovi9aj5)DoZzuC-EdX#zKI*Oa6Mqv8jUArD}Cy>&TbR^?s$~
ziH^_#2bYyo$v!~S15;Lu7E3OljEq->?1H>cLw8wA^ur29Ddjss%r7fR+9VI#sWP7e
z-(96E%WZ!zVOdy+ct3di9q=Ji2;p@Iskh%*duQSI)uhhnp3kqypZ&;Y4oei3L|R^;
z?w%^ah%q>zCO+k`mlA7lGVd$DE;Y5B1&>J0tEWB_oQ%O_dN&_K$&LsJm>%d|R4%`H
z?Je2L*GCH1C^b!@K0*IrOm<5jzd=8avy{}S@j*0BONT$WMXY7sx>c#a=1tZnl?R{@
zcs0wVub=(|-%}p^zzBLzYJT;`nfN)gO1k39Bo7(55JK$_sRzFcCQ9H1zWGE6{PW)g
z)e|U4Pb7Qs^f9KB4H80xjApN$_;e*Sop5Hs`93{6?thNPvkC04Q*){E|DIwNX&Ye@
zU@7SPPiY_1rw}Gb6JFC2Z}WejJejcWxo!{pZX^Hs0}C%ec@%u$9pmReB);jYt*GDX
zQ^-9tF>dN)GMcz1q_#)q|GrvYNeI<hYv(u3zz9<phU$cC!c+A(bEXuTI$6ulBkiUy
z2l3H^D~ZvQ(W@<I2<)ko1El@$hRodV$xHU%4f*d4`SD2n_lEozvVMx?Jeyi3ND*hs
Vu29+h>u2HD=8fA7^7WlU{twvW44wc0

literal 86598
zcmeFZbyQVt*FFjxkVaZcT1vXROS(h4OL~(M8>CyMI|W2akVcRaknWUDrBOf-e)sl$
zp7(pccZ~DjIpdsh#_JfuX6?1^b+0?-yyi8pxuP}H6tFSKFc1(Bu%9T(Y9Sz?SRf!E
zwV*-35qDb1XYdQrQ%gY_aeja`82o|mrfBGifPhH=|Br}}nN0#tn0C-HfEuW(2wS;2
zvs+lZTH3JtI=g|h5fDUug~3N>8>j`fuk$k(Phnp%+COIqgU|589JJJbPJud!(Hf{~
zP|LV_*iiGa^Rjc&iepeyQ;T|7+X`#R%Kvpa_)Cn|9tw36=HT%0@nQGjVR!Yg<KPk!
z65`<G=HTXL181;#`nf<YeA!$)>HgKo-|fiScv^WlxIrCUU8v#hT3EVzLB(ik;WzsC
zzkl@;>R|gncXILk>#@KCa=^dg;9}?G_;=gjQc?I(VHsCvHxC<6Ptd+NpXi@6|JSkq
zxzE4O*RXemx`I*gaIkvf0=4k~7eg)Jy%XpD>+1jOGybzJbq@y{@YH{<=KAaEe;)hm
zd{GYgga2b7{x!;fj)Gwp#}MWC_mqia)NF7sBOpj3Jdu^s@kRW;?2|vGb6evssUe9{
zTVSdG{T-1Mik#*<rl0HvK7j`&-z|M+TYkn^D(Dc35Nl{?K;@pz;g-x1$+4mxc?wC?
z%{Ti*WIQk3$&ff4re|bi%p0<P<*Mc&`uyj2`HzGel87k({6T#JQN`z=hCwix^8eS5
zDKrwgW6%G641Nk9qz~<e;y+Gi%9o6XaX}<d|Fab)4Lb9<&e4$nHviwj`R~dB8|=T^
z3Ydcb*Ng&hiWDLW8Piztlh^Zgb^@FP`uf!P_DfAC?+oh-8`Xdr%Yo!R|7PuO+m0f7
zF_SeyM5K{Fh#ZHfGmq%yqeLwU>m-pvU#jqVZCt8%G}3H`v8meW>H3vv*ggTaV=93L
z?4KlVO=U<P^8d23aHCs>$0@(M^B?9H{OJgqzZCjUyDS+(hG>Nxg!4~Z{pZ?z`A(9M
zJf#2In$I7v{Y?3#X>9Gs0ArpkEsx!ZvF}=V_s!vQtHN0hyj!mrI6XG|JJepsb2M+h
zx;&%`IQ}$V=-Tm{|6|#+(xPR*{qoX6>7c8vGz*2Q_MZ#(w%v?}tw-YTO{`zak>#cc
z{c#qAQef!tl9t0DDfFE7WBFxG933~O7c0S3?OxJwpM(%{>E|)?&QK-->!sWuCl&G*
z44DRB<itZwY6-}Fv2%^&@uEa=GF6cRxU0fNHhq5ZVN~KyR3(v&Z@;4Yqn7-CUYB10
zseJl@7PHESQ8knB6oU(rNnUI{EK}#Q+`?8rCjG6l(X=Dzw{ILVe%wP5@6Qi?ilrT5
ziP`kb0)Kqr=n#E+k|1(OeR6X)I{MV;S#9s}T6dA0&q|=r&PdL@|Cdy@S<r`XRjuq#
z&$}==pnDS~Uf*VJ9tyZDz40_1FY#-!l&$#F+f5VdY?dN7$EuwHv!JW+ZIdr-jY|P%
z!;PD-lqb~IzEyYNgb%r447=r8{{Um?bkh64%=yE#%g3v2o>9@)JX<hCB<WlNMm6ib
z31yD<@9)1HHm(G{W>g<Rr|;eelkwA~^<bvKdC_ktC++6$_M$^Qi=WL}9C-b+QHkJq
zzL|U-)Q8ZqI8vT<^4}YY5+mZ*-#dOExZY5l?~ZSPxyMqTS5OLCa46LbcV70IJs2yH
z31BNFg0FxW#!kPbGzbQ-EUel|Mvo)XD^X~h<Y%GP=Y)gBM!WTuyIX;h_1tWS!=KF`
ztwYJLHKxjP_VJZjKLwaxAGUJDHVGR;$L0K52N<%RH7<KE9rOII4*nf0mneFQHfgMI
z*z&E4+@*<T()#-L`lxPq#>jQcdGd|wKwhle8$Yq*P9#?UAJh6-d`^9fxQ0>u7oSk6
zYIse!40QH5yqhO{KAk?+Kn|e|)|QVW`9$pJ-nZd8TV>KM?!h>;h;(1H#@4V%ekI@x
z`-~?$zia;sYuyl7pCe{^dsF2E71n~`wu0)n;KpJ1`hj6!kfpHyIjSn++APO7db07k
z(P1*8@#YczC*)f6E@aji@12@%oYwW~2V7<o$1U5LPTbYE^LBaR+4I68()4K`(#_dS
zk)^h+C+%x{i8pxFSYeG}S%$JI8FgZ=S8`~G_9J)q#V@I2*;{sLw_R&K-NR(9K_GZs
z-R7ZcDi=^YPU|wP$_#+k*5lRrnt;UkM-_x2EKP5tERjV!Equ!85N^XCnD(EKh#U=B
zu`@`=!6++<joqZ!<@17X{U+K@`pAuUYR`Ynno$WunF!8?c<!qY81!ts<C``1JZitN
z&D>FD1+$!!zTN9ZAN%X#Xq4B4hTrvTHe|?9>nt@nk}62h)o0EE^-Z*3(i&)VpquII
zDXn}`0SQfB%<UI(J?Ir%8>APH=~UoROL@@cTI4)$>8En-Qus#|Py7w`G<4G*H153o
zu6VA|A<axq3yAs`Au5lR+%>|?HPI(TZ2U@7DZ*&hmzNufqs+Z`y=YyA=lyhvn<LpH
zgyl)S6WVG*;mPBNVdu=jGI7-1`^Zvr--z@KLX;3Uy}Pjml8`Jc-;_ra2R&eKKiB#S
zk^aD)8bKn(#OYaf+u5>{<3ky%_tfE$+9{2Er_nkN*eD`D`VHFCIrvS6hMU=dvB`gi
zlIbCRUIOe0G9pU?KAG=cNm`RXCXs!Hj3D*tTH!%yUeNjp&i(4#pkEH#+GfcJh|7A~
zu_=wlJL7J!F@%tNRff%Rwzj(?nONltl0-=C6lrEbe%oglX}!iBr#dkK(p%vgTq*l+
z=&1ubw>St&bN%<78<+L!)QaRuV`^`IeT(G1)diaigGQ3icO--=@Enh?_>Fqr2t&iX
zZPxLw$TRjc^q8|XU~N7JisX<z1Eyf9g230NcdT40r*sT!XTYsubU2_aV-!7=*vPg0
z2QjHfDvlMgn_j?nhF!W0ANxqf#n<OYz>-kJ44RGf6(Fd1PO4#7*D#&!-?@&&CWldi
zuiM8?X|r3dP%KOwNhy4m-097O=jZ7X1Zk~Be2>i;E;C4YY_SNCF{4<6Z%#gE-^oCN
z_@bcin$~01Due%J9yrY7@-MetG(%2_Ru|HCgp8|}eK+EvDw4}O>|z^S5XUMlfA)lC
zhJ>TOWM=IqPPq9R5xX#!Mrt!q;#)9sX$~hwUi#{}JkTvrMb}+I{{g>At)!eK1)?T?
zwYih~oZZ-Q@*TDH^&&8I$KCf;6jZV7kI0<4WVi}>f!BsXQ1GdJ2c7xtct&)dh3?t~
z$YiK%oD+_fs#B>&>yP-kXNs+b5wJI{hGLbDL3f_;E1BH8mEU>tMo2e;1QBVTJBZD*
z&>L6f45K2<;np<x_mA<bY%*L;y$pRu47Xo<r7NDJVg)Z=>`tjh6nr5e;~R^je)b3h
z<BWf*Ek6^Xx`y{=OMzbL8H}2JqaF@$0)X@G74V70NdAB@`w{E+Fv{cUyoLqGiZKy=
z9ZRw_?&T_zRt+;=o7<DmRHNO67TpnICp`o{U;P;MW1b&2GZLx#DfVhuh(0QEefyd6
zd7Q&KPT=*=l9y~;XtX#zc=jEGkGN+Cb)HZgRb}~M_4daaYfK!b3DGOh8v~y+kPSE6
z7&N$+$OQvLKlXtacyh-)9>JZ*xEBaRJ%V0-p7f9|k<}@0?=?&y{{j6oDh-NM$T!#m
zUkebB{kmnD?oIL{=da78liqsxCw5eN3ZLzVz1VZHEiXx3*UQ^SKl*sh9^Wr5jvSn^
zIz(kd4*9s|+@IQFTR>!#9C7bK%`rdJ6nEI?k8{tzgz!zn0^xS;B^VHuyI<d`Z?GrF
zX&FeeU>&>yTcmiZ)_(3K%A=5a)%>DQ@d97dMS@)}{Y3jzv^*Rxfe)LcLX0glSZwui
zeR>eV-um%!9g~tGTqIata?X3vxp`@GfXFF));7zz4;c}Ya_}X$juCb&H{r&e-p@Sv
zlyD+C9!(PFm7eor&v|@dlSvF5pW)8VZ}Qx>Rup(VHCGr)#CIp7%{mgfr&f?n&y&qa
ziUvCXUa}4e6{nbngsr)-53)HFoms-`j8^Kj;OuVuCV|9+T)sqA;HBRWPX6J7NJ^y~
zG=Jw}t8gZ&{UWYm%JFU5A$oQSHf%vcZo=p&ba#u;k%Fv6rVN5Vf$}558cHejljVBI
z7L#P&#1!UMXyQm2X+LL4lqkn~G^rP&y=u!B_Et_-;&%Mwxk%GmbPBh<f!MdGPL&_@
zqVUFy5cNmT>jEcL1vGnM!d1LtB_CMkJsiI9G5tU(d#6kdd*<LN;(u;WJkF5Xp<``>
zgBYr-Z<5kjaOTF~Yr`b3xM~JBhWU^%jk~MS;P=rBwNqG(O#VNL{0V*fpMS4K^_e|t
zJHv5x2vMJcEnIu$N~6nL3#y-<=k8p&e!OBJ)k+^rTm`n0>}Sq6_Rp2zySfqKSaakZ
z3o?ZpVBL9f`?VOl_rstAVNLgbV@@nlL&Zqcme1oPm%{1<I`Cw}uK1$;wX9QKs|iO_
zIx}+)>?VXH;M+eLuh6ef7IrdB9kc#3R48~%i(d)Z(jh6dodR`QF_~4_<4yNyK1DO;
z_NzI!|Cm{G4lqXAc8m7wr-=-6sqkZwt3<4Fl2Kw8BMlW1);(#8BdcLu?|OM#S@>Dj
zxL2zVmcd%sEH#ysa;_t3<HaBy0ku#hNA_PUq&pnzODUNuwsDBC4D)(?!EtTS9NxHz
zL8zMFzmjZEjo{)xuQ2kQ5XNu;UHH5>CPM95#@x9G#*GCf?58&Uwp>;@XCI<diAcDu
zxcWo}W@R^tV1(z7{+D#X6Fq-l(u&NzDObvAq4RVfXDs(Imh+bzdmnoUKcmHPURgyb
zXBDF7MCC|)Aj#u2=kcc1A)e89^BF=Ph72$Xf~2JLKJ+k@G4~aM)0FSd^6ogN4zoA;
zoF9?YMYvOCQ1$WU7a~qR^3yDS^=vkt1r8(_#?m3-ll8z@=LbQIeIFe^?|CSG04a0t
zM5<~crz6zpO=h?U4Ip}2BCou6n(2>9Po03<i()W{!alo1pOD!Rp<5O$d8WhecrkBB
zwe5zk_8fT~)0)y5K(NAiy>(e`3C$GWVpdNFl)<~KS8K>}Od$+9=hM<FRZ~?l^piQ#
zlR1o}u%7^8?Iib`2&Z(|Uw1|)x1jTiMd`ay_YDs4Y*COqL#qh9szw^hY9!Yj#&Nq4
znXlY-)CG9!->xc_u&%RN5R#RbK$A5@47qla?-<{vles8^;Dj})=Xn-AmFq=2F2q!q
zP&!3)&pB%WRrHP35-YomL(QNr-!z8(tVTXt>XU|`!of{RgE;O7$5~y7>xVfr2D0XR
zr!~rH>|ysvreNFXymn|G+6Ql_Q*}>nWK*WsYB+)AkOs#5QIF50F72wW+!_u&pAM<|
z%f6}}w6a;HGn0k)GgqB&-E5eGMuG}sTn~vKYj(OS2|AwhNg~yb{S0|~z1-r}y8pIf
z-^3;#UjqG#r7o{5;7Mj81Vg6<#uch05t`a@4)6Ofk}+lhXM)0?DB)j?2hJ`CbWdf>
z5m}@`b2prtDX?3!hk~3gufEmil^OS{RKp<CEWq(TO?-E)D~edn5@VMW$95I1)aUNa
zLC&%&P*M<Db=Lt2D!iTT_Q7CEG7e!FSoXRw{u3kp^M@s%^ejo#;A`cd{9Teccy4s@
z%R~Q9$i#-$FF}CSvNkXA`+QO-&>C8XWjHQI6liFtnDkkWqwN^7@iY<(^**J_d=T)L
z*{60oq1gO$)TO#A@BZ1W903UFcpU|bC3`W?4;C*{ja<qvm)%hR8Rt?1E-{psb3_mL
zu>W=30(2DLyK{Nj5`8F>?^_swnt??)QL2Gu^fEw66U9%)Q$HtCtyJi9Y;!a&d0ZcL
z-07PI60!YSi=@I4-p#j3Gid<v87hSvg4J!foB~xK^yq7Qk+n+Fm8QZ&qCD-La}X|!
zm1{G~cQ5#VtIP{H9Z1@K2TU690sx?mkK~9nI!-%o#0w-m#Z6lMFl+j4zK(JD(TrK}
z@6S}hH=RiLiw_ZO4(Dn&n%p<;;ZXGV%+*@S=)vPN;F)IL+kE+XuxRG9>}9bH;;Sc~
zD}fi`jik457?$<yJ-Uqn_==-tw-B%skv)VCb2S!l+!*t?s_Ak)_F`|6@oQNKw&p3n
z^TIRGB-{co0(dPQQQ4&U&#`(>$SI%nD0zUeUv>4BGCQewz@>wyX?4e~@J0R7-RoXd
zssP+W2QX`0uhqpfL@Zo-y9+aT>|9#*-YlgO;5C8O`bH%KUk-5wgtx7SEl#JK0|mHP
zp>R**wL3=Rnc>oQB5#Si-0IV@7C}x7LgO5tuU7zRTBoRsUxgV3HjGSQvNxIUPnFAZ
zp*hw>nIrB61BL)RNN5H!S_Bqpmv?hKqg$u*)#(?G(H;;yQ}DNsieEpgAD1VzElJT6
zUXP;dizDq0kS70r3ZOOVMDew98uq)9n0Fti^^EYE1visqq%BVib|BLDzkmIp)01Wj
z@raOnU>0!lQh{<nzPkPAQ*TEQ(OVmJ-2QTEsT4H(=MZ_|u71klKq0h`LRe``o^B2|
zwau{cJ%6|z{Bz#!Y*=9BYA0`WQS4%=&ZN~_;_k+?zRd4z+b-9S!4(yYL{hIvJ{}_$
zjr3SBM@WY^o!^;kbicVACzkU0cP{Q5VCnq6SOrk_MqZtCyu)u%i?clhizUnTqmb(b
zSgW?fnSlUITa1Ix!+PSs&$Yj5wSAa*`VH_Q_hZo>)$A^Lj$*r(zmlY;R*^XEr(<ma
zkh#l<|8s(||6WPkqaIH8^&a~Rnb6~lbgKj!hOYBOiCe{k2$}n0yifgy`DVw^*Fi|o
zn+>M}m?Z0?93PxR`;AWNpW-!jX+cN{H^80a@%29;Ki5HdzQ)d^VUO6s98RqE^fsvQ
zm3PlFzy*qo=3<Z_ya48;;E;9@NQ9seqg_pSo3|b`JO=@WFW3{K6OQJ$S38Ns2OuUM
z+GM+bHEUoMk1epo$(jj<3P=2Mm9u>{6!Y-?7xuM;93jBr;5xY{K1`jtx&RRG^5>$f
z1EiDZ>Ue!STQy7=Q%5DAEZE@Xl}b{2hq~wq@*~DjTw={1usPZEsxuDst^lR}1`r03
ze{HZI55;{_f9+MJ7<*4ix`*beQd_>m<ePeJG|wnZ*;v3Jv}?41Ws_l!%!gShcYV-k
z#(NO<Py5EJk?(*s>*EEOIc?a-d+TTAh3a4YC|_vbmuL~XkM8@Wl4BAcQXhOMRIz`(
zQlxO7jCxEwuL{`(pbdM?90DBz^V^0+m#p{66zO%lg|Tmj7!&#r1RfbXmX{RD^)Q!>
zQngo#X^P}6IP7Q3EMsg>n#t5Bw;%!f2AV!P<U2COk?%W%wJhADtk{U7iSs9r`P%?X
zBB8|L@#2n66O|dkA`nhc+Df=25@*Vjs3;p0l>NvLyPL1>c8d}VuT*+?<}G8{n!+i0
z?Z+~mss?CT7<#m4a$Oc1*m(ACsY_a<rcxvibeSjx_Sd2V4Swz91^2b;ylYo#-wDG-
zxK>*XVIuh2CxbM8V;?$0prYMX<}HKfCyV|khWOk5hD4yv&E(VgS5-(pMQr&Mo9bSw
z>I8XvaV8p&<24h_QFtLmJYFFYeWhFkE3=Z`9t*ZeM*pVuId}UWFK_-VtV~EUnc60F
zSQ8UQf>;oFTv+f2x(y{Kk1sN)W@@xu%9CPPfDNx1DypU-%KxA=|61OW3poZ#j^Wg+
z93|h<Got?we-L5}#%Gho)*t0PK8cwBIS8ZF&k!TFidusXZxTfg1!qJz;&|*0JNv^$
z5lXWpGbih!q;3slxu5<x0+WWt)KB2$&t%Q|;<C@)h10SJt&h-JH1smyExG9Lj_Q|m
z3R-(GO(gqcb*#|TyBnKg1YrpM46d1g&97u_HNUm+2WrB5^n;xT7r?bpR{KZ_V4UM9
z>O&?=)zuBTZc}p~MDc}H;clF5OxC|OUm^VhD?|PLfcqY@6%#JCxnQ^N`(%d&hVOuS
zCBzTo+(*1ldDYkW_uRw%rG_iwBH0w2fbOT4B!pf2`9G$|Yy1``d)(OZJKo_igmfUW
z1gvi%SA9d96c8f20D{ae^XbJBi|5_wq(|mTU6dvg3F=F^wO-DUrA?kU)O5_ZeAU|$
z7YB>a>Qn=8gg9f?082C~2#Le#AA><@n*fhF!C@|Hc;MhKZ|xg?HX@R!IvEL}qvkF{
zJfM?S8kApsVM;XrVd0y+C)yKEQACRbd$k;bxPC>IuujwsY7Mr~DgroTIYfhX+Wh<H
zA~RBv^n0Q{!={Y{k=-{Trug5M!gz?lcDp2G(<@`p-kFFPrT?lBNgcNMZ=M()&C3yA
zcI0-(wCM_zO7fw*8Ik)>RkUksm#W!}18-)yL)*tu{5o_Yw&~jDT#5{B%CzUq&t=2!
z#`L*ni%>l@|1E?o6p9XZ^sCNQ$WqHBTIjm_g&G%NNcnTWetx~>?hklotYk%jFladE
z?6W5-yCIasiWEwht1s?zzoC=DjS-uje*8!(LS3iiyL3j{B#Bf8RX%I%j=&>jrq?#i
ztBfq*H*>z^G$z;-`}b=_43rT4m#cXsB{K|k>|ijtC8SbhG&J7h!XM(g_u$YY+dCQu
z?L8K5(!pM7-bl9}EebN)+<Jyx?!Z1YEq#*MQ%R*7S;wHCK@#^ppk9$wX_&|#(VMz`
zLV3W6WzC0z<}txj`w#PVMFc#&nYyQTJ~d>io<MMSzHSGQdh%bwL!Vixy@956CL#6^
z+E0xEp5gZQwRWS#nE_JxW0iw!bSBQ-4dAVx(BWhPtb-U&Sk-nqxWHUtH3ATfR2nXp
zrW6(KbghEThX*=IhJITqq$5@U@w{<X)8UVJP!Nto_5AI@*kJ@r$Drr9gm=Fn5@(?S
z`hD&Mc+5mSQa;Bi0P5yjyqqAW1*DqwQPHTBzOXTJv$e{`ufmS2lBk7g#vbU4%b>eB
zBHwF#{Y&W(v!it)UK$r={A~jzrBEhjUb64%4I76z;xkDf$Sz>I$#mgHFVz!<Bz#5Q
zb$n^0-Dg<5EQQyQKFMHjcntV|ZnQ3dloa)Gka0Yow%-s$suM=S#K}Mi&l8cFuZ-rR
z*IdDR*QGCu?%*UTLA|s;$%BS%u<JvI1?jSj&XwI0@dWs$1p5nPUO<<#XuGf5qWT?5
zMP&n7VCdM3d$kC9q!K}*JccJ=+Z-)?cr5jp7;nT1FES2;qd+2*+_mkcsoWf{*eea9
zs2MJGu^&2(F3bKu_T>=K*fplhe2%_Gpzq&owgFcP)s{Z0tf*>XY2)>e$$J^&c34@b
zGxAOkQ9^Ut-+CFql72%NBRxph=hHv}zLC9e`Non&+sPPHIvhufcRyI8;|1e1R^-c9
zbB#Z6=K%IC59ylRt8pO#6cvK2VGnk2GGozmT$Fq(Jcg*Lw!7PF3p~1Lk8~2F<VGFg
zRh-l@T?1J@WU>xil+}ZJhe-hM@%ot1V=t9=Edchh3*ARrO2Q3{A{Kh72E*WcfqMTd
z020w7xGSW9RQ6Q--Yk%le)f%VmGNzmOBG;Irx`WG+5_BeFW`)-P9y>!nwEFiEu9$v
zE6`=5V)~!X#xS$P%SF!NIjSic^huEK5GT;V$S*;l^PzO>M6~LQLyxt0mnn{Le?syW
zwtzUad6*ozSnr2EdGpA$v;Pe=2~V$}O6uZpc~ARwqifsAz4->$74g9&=Ntgi&y@$Y
z5^MujU)A_uDqm9H^4Cs=^<8d$_Q?}QX+tA`QRE%LX*qa+v#AX7P#@`s6#y9Q&qA0U
zC#`surRCTasnTq?VEtN^B;yr7wcnyjD9_vFFZT+<00g)Ky|rCcH+3hMzqiI#c@GVH
z>;-}t<~sn`Xb|~Oe8>6ZMa63Gy(Qsf<Snj#cv2vartH8h^C<i>W*nO19&hh3!4UOD
z6!3e{MCmhZN9VC!SReHh8MlAYr-V$IAl@m|Y^4Fw4z&@Ya3i^Zi?LELGKeSw8op&!
znL_H1R=q}qb~I9`a}g2AAjk?2y$A!bMQ<-R&atG6VJ?`8GH1RE)MMCEqJd=$o!Pi!
zG^UaUG`X;Z`+iw2)~|gB!_;|RVgAMguu`l*0`lu|{Dnf28->rmAaMu;UB}4G48@Y$
z2u=s}LquaKMZ8(t=ecmVLB<Okm+(^_+3nI*cY>eqxCzrqD9DnG%gBNpSfb!M8U#6+
ze%~QaclSwhADZTgy^W;m<#8%8c7Z@o$Ei_v1p2vHM%6FL<Y)pZ7z+|IihKgafIiH^
z3>)|I6P%_5cSHk$T@36y13()gztG02zk(fhx0Pnb4KWU!@yS_2Q4`uC%zZjJgW&zv
zgY)Mm2<%2=p5<O?KU@Wt%t8|T7pr+T<P=Bs-mV!;<nCkO^#uJ|_kD>ayS%_cN0d07
zK_gmB5M8jQU|s@N>oa#v*)q1ZpS64<<#M%|M2l}fcl*F!P$z#4bcW2u9_P<`(WM2*
zR%8hfhH0k$dmyDH)r;t|efRq)xX!gS$NTASM80l?9<Q)D6LJgBl4C`s?MU{M;OlqN
z`Y1yMv53MP)^1@@FPPL5DaGvN)RWhMhk28Qeos3tRrpU56TT#qvm~uV<XIYJEd5y2
zV6A8!3Pe6rgUy|QtytUDV1}6@?K)AjV3`3D^e+1&x^!^lIg?P}uXEp(NY*W9kgY7#
zEks{nO<Lpr!b&jgUbqGc(uO~PJ1{5i*dT>RJplQ-NPo54dUS<%{FYWCVf!=<mRHt6
ze^V{V0C0Z>)Ht;j99SAQmO(^$tKHdfZ`#jz`7xN9e?Tro6c4!5sYu`Z!}|Z@yD~Mv
z6#Vy!|Blt4HSyn7^8f7qu+RL4@kU=VgfOJf)0CMfutKsO<x+C`xrx3GwGfc6y!Lz-
zTUVO<Ja*%|9<Z%9C}g9#Vy!pI4(R{WFAt(ifY#u&6d=6ZKxE(VWit5~Zj9*RyMI9R
z@bJT54pXP26Oin_=i>?h9-d5hR2&w-zmd#Eey5Pcs92}@_p%>UF$ZWh8~Dp|)y|5c
z7+d#0%VAKb(;wN!-<&n|uLhaMa6>>|oq&vl)4cP1Z!*J0QLm~3+?pwg0`VIj{f<Fv
z>0hBx5_Jf>NlP0@mcl8Vv+dzoK(yh}$<y2KEbGFpQ^+q2gT(xg9#y6A>;D+j+D<r8
z35>QzXRg@u1Twz2amtS4|LTe&@cpC@v0@AVK6_JcwW{^-7qF}#f2!wUWsmeG<8@m0
zu<ReV)PnNNNn`o*Y`gZqo-u?TgSGYh`+Dw6fHu2E8l4xtL9~-4=x&M6wq*CGKgK}K
zvsq~K^l#IdkB>|S)F6QP1Dwxrwon})DWIl`x}kvK)0duSEBZ5*%qW?u9vd!3@VoxG
zRXzd)a9ZTt)@Y=_bNGXtw>9m#ss>CoqvX#8y6E7s-{O>t6#e}qJt$=w=08FNqbC21
zLk7zCI?lR8<^P%Tf3>U_1F%Lhg~NsRzwh<evkd{a$9>(T^Pk6yfe=)2@TIrJe>T<6
zY92@Z-&_1OW0E2NzaOYE-=E*-GqnFKk$%&B00MJ$-|uk$S&{H%s8K4p(HCzwoT<A*
zYsaFY5%LCQczFoG4+jV`S{xqzL4IooIBNv5Wb6BseFw>U9&n7EX#&f%JC8Lq0N+8F
zJPS_$+n)SuV9Q7VwjlB>V=en@ga6s(@E<4ujc90?^8Aw)`m=2<;ex%Pg7^7<SrNEd
z_;VGZ4%q)ve#8Hn0ttzM3(dG92Sfie2Liffg^aSr=lS=>{JV23K!zvPOTmKq_V333
z>KBMy<M96bApdK;^8dd&BO!ftcB574T~9Solo)|3l%RvPWTU!^4OnbnKoRLSSslKD
z^x_}a8?){eKLeTxQc8!8Z5v;W7(@Dwl-{WX*9wQ5cwljp)L;l9fOHZZsg6XDc>crS
zi!O-ez=af=f-fH+0%F->$dw^z`0SgZd-GK#>oiSWrBwOma@Bt>>a9m0rPr1aS@LhU
z`j>sl*8%of>Cut*e{K*a4zNVGyhK^!|Ff+Ue6Xrn{2KrG)_-pT;JW?LN$8lWH!K|f
zwq1YQe5Mqb6;-Oz*8>&}{}6cO3(`YZKrEPQXqy3&J8kDE?~(txsWh~J!%j|KiGQp?
z8$!xsO93Qcv1|=<3r^MTpYD;@J1;gozc>W0b;dZh4tQ}My#vsU=xj0re3!pfG2iTe
znT=4S|C&Kn&(v?r#Ao@3#NCy8eQCDx6E`3$mG_)b75y=#{o)=rtp+*1b%$jO)|uel
zoK<4HH;)P+E^Gl|6A1E+b@PKD;WqbStvhm*kE?@WbClzYv390ph&1NZ?VNz~0u(49
zn}Dv;ZZy|qx6-cv?&J00hSU3g+Snw9E!LhJVz#I6TK3;Q07^Rk@S)q*fpX@6iy}{*
z2>h0^UvPV}!sV4nnBLln2(pL39c#RRIf<JNXYxG<`H#l0{@(^(GmZw@X#cSSypBCY
zSMYq`n9O}IK#T!C44uBmbn%1JY>U^oW{ih^n=e5Y#Rxi^^E<|KSSS&FJLim+>=Z<-
zZRZopA_w)8a24L{4UpKlcpqeYj$}9~5in^)K-R3(VXGhv3;;^L%eNi_IxbtqzR9mU
z06hA3d9pDoxDf~RBG0PA&s&A`X$g?Q(g?yQRLt%K>7jl(w$?zP?8MGGBNOxI9bX3m
zT#x=Fn&=I;NU*3vC{RFVL(e6}3{ZX`PiF}Sb3&fmfLWLD+Shr%7QF&;YvYZsD-?C_
z?qZr<n6A%(bg|Rb(-b+m`GdLKS3Kr^aa3&<4R^@+=Ub4;+7bCUCXHE(DX<dY*?iD%
zl_=JC@$zUJq+RR5Q);&X*1Bp5!@-Y5AK20@N)hEMNimeBpd3CY;>X$&Y}U0S!ryLy
z!hs&`VUTV572pIw2GkDZ1|6s0PWO?!iuxX}CdX?^LFYkY7bIN(&r1D^FDez^W_PBN
zyrcXtac@_|8`Ugg7FHlzMiq^C3e?TzhJ)eTfEt(qnPSSO%J+jTK=-DQJOJcn!#qq+
zC2{FW6@%}tR|aUF?!$9N8cz|`0_(YVmzsG}UV}WaCdS#Mx`ZZ8Ml>L{6DF5IDvLzP
zV5b?7ptZ#rybkyd>|ki$2=$h{yPVcHd+56wk{wpQ*#UHY;gw;c_k}ib^$H+e%%#~~
zq}*Gg*@rhz__$`)<wjbMK8grPgM{N2n^b%fTWe=TI*;v8Iw#{VWwB789^`K`0Xo`l
z*%s`BwzCnCD}Lbc)prBi15<*^OIY@b=L95EwI9C7LsCjn?QXnmQG2X`hM1dk{P1YJ
z2sJ}Hk>OeB^V2#&gt!J@lg|pV`kxLkZe(rOe$g{>mMeli0>ms>SPA+qPcGust}O9A
zH?!m(mH>2|XNz-A)T>K;AhTigBx9Sk7D(B+czET_Eq|TwWmBZDwE20eQBUD-Y2u!J
zyK#BXR)yf;Eqr%#=E>EUVl=ot=fgfQw*tnZ9qXHRvHc~#D<ptBfH}X@80B(sm`}BY
ze)R!+Pw!5K9VPp<;6^am(kESK%JL?L-Wf*cWqqvoUn+LzNSx^#yUX3a7Y_D2L124F
z$^J5<t_NRv$Z*0QB+_lhfxL$NtA~&X(E5fu1P$pKSfIQeIN=$hsFDJ4gBL!5;3{Lh
z(9O0x4LL@q?40cMjSXsrt=SSDiW05H4aayC9^D)hhYQwv>re>~&t5t#10=RlCXMrX
zat@7u0izaPN5$NoaSVh7Aj^|2=YS?pj{NEb*o}%Vd7kV@o@@yRV!YFNpzH4L0iv|e
z$QMF1iy+ya)Y#+ZC+IiC^_g?|y^XTVtltT53Jk?FDo%qS{+yJ*9wgBt&iF4FzcX|^
z+@t8<DU4;OHTwC|sAyNfrg<y1D8YwGm+MaJm8Hvwvkjfb&<lL=pJAixC`IKGHgmw(
zrVm&p9*b@5N4a90HAk$)5=`*miFdPFiYxm~!Fl<*<Z8(lSLgu;!zaGb#CIdH)%O@f
z+=k^Ftch~M%P<CgUXN&i>^}p)!pmI0?F@}q&JFVsUQg6DsIzmL?snYO4vrZ1n$@!E
z%H=gzrr-76l$jf!%B`qsdGMndF^`xv7-VBNrw}48YfA=HNG?Nky<e?=Qqzg1zTw9u
z+!4~9y8^=f%#52VH&=C&)))81(;J<tni-V4KQK!?h04n<hPhQfS<R4Z+oDZ03T)wD
zWs~r@v8<)_xz&t=IoVf<%ZO&liSn)Qk#^+MFi9>W>(P(lR<TCg9P=Yr`g!z1`P%1x
zJ1#uDkPBJIB)n55G|4ILLGuT)M+;%-E`s!po<w(Za<b@_6&w3Gl1YFZi+XL;<SATR
ztF=#doNDwyt*&j@jdH~8Hm>SY=+kM`DX@FNt0AcUsRUk42S>}T0<QN(zH{>9xup7%
z2jh)y4N)m&S;)|%ned+?;o;l4p>aVPJb;Q=>t#sLY!`9!75X>gGmb3w?d>H!en_t<
z)0)+&n9vHuRJ7A|<)awN>WRE>0~L#?hnH+S3A-lHbBm^)$}tr8ei(*JA{L7hgku`%
z&3l->PCQ0!%~&HhxsJU8y7h=Jj2{W#T?2(e*!4{^31Y@)vw$z5J8)rb=Y|vI$I5IO
z>GuyzeM69$S5)_pYLamg4hRjDuEUekxn)r{EbwO?jGOW7&G$eeijrR^yW?4{Ai2k5
zW=KmZ>{2hJ1?BA#eXllk5Q{WS^oHK;IaSx1=#>S|%RO@`x<y2!dm28;X#P$7*oj{Z
z20hudPWz+wG%Qj|RiHBuO?6DH_5fKx(u-C`L4Jz0=Exq<y$A{&UM1g%MmDZ-bi~#x
zpiSRg+Qe|Y4R!e0i1D+_L?b^8(tC$r5{m~U4(x=sz(dK=1y$s&87mE9afQ0oxr5|A
zs_8EgG`<_Epi7HwYr*@G=Qovs@R{n4M|Ef#Ku;kQeD0}F@CIl<<cPzw9#N`knPooU
zYqlTn6bDK_T+jLxVS%2Z<m)6X9H0p|A^lxdYdPiWaxTmJu~J(7!5x9_uGrhqn^-@M
zeXMv*7=fA~jbenWu$N<(oSKfi`Ba&vC=Sh|Sxw{Pu%4mKh3Fa7w&COWs^7g>`r4u*
z=;}%aVmcEl(RgFrOaWp=7E)18VXBcGpuh#Q0DA#PgVPkQW;oOSm<3Wx+_Cmlc$JE~
zNtWvAoHQ&XTj<KI&^_sOxOPLyI1IhPAgr<}dI7@6X8|mvBm`ww6&A~58t3Q<l<lS-
z1CNXg$M0r9U5cRlx~yd3F=EOV&{1><;?XfgG+K8dOIY2M7@6Eo_^6d_65}7IC3*8c
z<V(fmIxVA-v4?e%#w1S}Q!53Iid}@o_Tn}sc(}nOx=aFyK-0*)gsxz-(IYtP>5wca
z{wb!%ErTEwP31T_asOiJXC!ubk5_JFolr1$zh!?2E`q?Fj`eCo-i_bsz?qgmE}HVy
zmm)>A-Zi!zJIB}KpG6uU->6vE71UVr<J>HTnIS%P*JiP#U9)?{5={W~l(59W-GbMH
zSF&4_ew%oGn5*6D_SEMblE(@9ZKB3IZYaGJ5a*?VNI9MFJH60tSKjGx>0i6C12WII
zk5e{Vaf!omWzrLFJsPP@T!0F}P{+|6O0;i<Z%iWSMy>Co)3-K`<KAwcvOqwCi)(W9
z66PtbNQ~`AlH({njMkC$fwaOSfh+W_=)1#YNCtA*mHZvA=s+nsWMX_GPDtl;vs_%v
z;Y2i&tWrm8i^!wkj2yD25~=V@C5eGG<_>!_f486d>oqlc*NyMs&?G=JhaI;+IC2e7
z$cK&|6MZc-dpGpB1<t21eMC^l^fUPET~z8<fR!;?GCm!A0I8$Zzna2LQ0pp2pFEj#
z$k|zYHj-E1>^XM`GL|xNFw{6%)bwtvfiWqS11+vR46`0I)4(6PskU_mNl$DelNe;<
zom{7cFbK_HN48}gop1akyM7A^8~~|4UKj_p+3T@;JS^mR$n1H(UrGoYx271(GovU+
z)yfciQdIa?c6Vb*S)Sq(#GNMktHf;=tGM~e$H7!5XVHzY+8MLG7aYK%65~S4{j8zM
zWRr578p?$yF+KJRCZAuQ9W$X|HVS2oWGW$;rq?*5`3drV(f9^HmRuGhFZX`YrIqcw
z;C*M1HGKGH&k)v8K>OHzw<h_ySCB1^#LY3*gAe0R&!<$6v)lwZs<N&@5lUTol6q>e
zgOq|2OC47YB<Gjb2&+DC1{{4+ARdx_4Zi#B7Gvc`oLkF-jKw&RTxsXYUU{~kXNze~
z^b6B`9oR!GJ~ZhFW~1ni;U6~xHGJ8%%`#1(KBqr2NlLWwlDB9ASIJzhY&mH-qR`|^
zAI(Ql-NZzIc=(f)5Y`oDcfIy{$Vb9>6S_YY?SCoyV9ijhaow2C-Ca(ds^GW>ugM!D
zO-m@SNX9ZEzYsMfX*yr(1vl(i&Wd*?305k;v`N64kBQXsaNfQs-u_xcL^A1gx%eTI
zJEF%beu9fElXu%a`v}$6vKW+i3L3qrvnJa4E+G%S=R&CXa|2J1<gtZW6647Q*yFmq
zCY~_X;&eQ&7cP<_BRVL9DqLn~C{Vb|F3<nwOH&kj=rg{CZH0pOqVyG8hj34?L&vfH
zt9Q4Ndez1hJXMIp*wT5DGtOTIM`Xd^5h5=9?rW40^E<5VCTPCA8QRgxeWEp<#_|cC
z5Brg!tyg@euhzBH?s*>!@kL#--HVgR=O^dm(5fAEN~~9G&ch@Jap|lZtMZ0ipVTGM
zunCaYs_0lT#H#GRYWOZGC4O5f1WHbUI+=i2fHz$rYeggdbcAtE#X#<sk^XQ+_ON7+
z@5LOcrnGce+iqnNW-bzrAk2W=VfQZYBT_evIsj!Nu&9XT_jtl<AusO;oQH!YgfDQ8
zi|J!TWF691)x<^T5~UH0m&X}(m^g=Nce!GmYTyavcXHR%3|b$v2ot|as>#CjlMgA}
zXx9ty(xT28w+J$uyzxtgj;^&NLz*$%ywjaBF_a{C=8QKYe&((9D16T_MK3ZT;kH(s
zYDh&8QeAG6w01x%^wcERia16@?=oVyBW)7VQQjzhIp``&Ye4wJkVIa<S`CPPH$~p{
zt3T{TYd<}WCHKd-#o;>~5j`8ykS-*lare5hDSd0Du!M=C^ay}T%_%zzLFSXVm)>nl
zR*9rM4AM~-hW$g(9W7#aAxaja*W^E9wDmR%0;i}DcN`b8nJE5#7l2x#junp4z`sC=
zKeZJt3mZW}HQ!*TvkGsA_l~oyWQ+H~FA!pe^OEw*Lb_V9j0lpO<Sf*LQG-+!`zYnI
za5T|fVhV-ivhwNW7E{Rwwer(Z9|e5Z?i0tF_Pcz~6GUUfa7q-|uqXba_P#HK>SuTX
z_6P_e%YFy%b_p#ELsxc%JJOB$G=pjC4fEB@INv`}$_rn+?ll{^FRESODjn+oEFtSg
z?B^UEYsur+bcv$e*-wC)L@9OV)&(Ck-|R&V+swMfeO(eo!&dWxTs$w)q8yekWI2{D
zy(L-akiH=dD!MzCe93!=p&zC%Y}P*EbztALjV~oo6)d!lM%`n5WT0W!J-e{iO_6&=
z@PKjD7mL<pU3iCpu(UdCOjZB9Ro8BCFvl<@uWV3^BDD=4g=JSc`OtW96l2F=#A_*a
z$`8%Hq8CYp{p5+ZUFZT&okZ}fSI^2ZA09F_J)$-+7U_n1Znw|`S?>17H;j!th~~(&
zs1o`Gp#&DjuxfV=r|d|rMWd~#79xDG@y+kjk3LL|rq;MjMh1pJNC-E5^_!VUUh*@i
z4iNZ&lCYk_bU9H~fR)A6$8}joUCVIs)D2JfPHzyBvrEuW%DNVFY0H|UkuiK<DXmIz
z?6O}+voKX>FiLfDXd)$YkM5HeQ1jB}W0Lf>I-5tNd~C@p;}#92Oy(A4>ScXJ1+#Sg
z+-MVw{3ogV7i}+##tO=s;>^Az)q^Vwq-cN6=jgz{>z4+37end{*%0}7EbCv){na4D
zd*h#J1VOpMe{`Jx0Gjq&mMFO~DC4=t{*E4>(-k7qbOH2xx7I+b$^WA7Kx>0=)r8-}
z`#{|X$w1i^NKbgrUFu%`PsV9V9h~xe$ry<8mFULWpI^Y^eBMRxihmGBI;|l4;3&~`
zXy%7W;n7b67sT372zk&uSCJf7w|&WKfP#WrwyivF8>IungM|;RfGFs`fuq^l@FL!m
z#%7E#w)f-5mWs~uWbm3S(w9t|!62dU0_rHo8-QXl<n4o~JZCL$e&P8yLy5o>uo9Zh
zIsg}(AQ@5ZhlF_ymx7aW(+`H7WVr$kX4g5DV4g6s02LLR>>T(I>x|L?;ICmDWP~eA
zY%Cg)2x7`dpga|;%;@ubjJyv()m9Do0t?1pKM%eHUW1}a?VUyc@4G-wi~aC1yowZF
zh-cz?Hc8Gx23L;Xbk5SL>uXAeU=V?hu|OZNHh_%3(khVtwSwB;k2j}ZA_%Fo>Tt(H
zmO;n}6yI4cEe<jbTP+xbYG6dyo(&s@P$27>cuj*qk~5Nsh0#Ik^U@pQKDe-(6NETd
z2aPN6lC^U~Noc-$o_NX&>nDJ8zYjqXTih%Numw7UY4%mavV}x9t)HU}_nrt%9NN(!
zQvy~(Ek_t$1r0AA1WX?(zx#}#T~6T7dC|ecrDo3xy-I#SFZdq((kzm924QHWQKKoS
z^M)60m+8QZe}T?@bKrG1#oI=p3NMh0C3aq@r<oh7bWK-xLCysds3{_S<CkxLEZw_1
zpH%lW=@B~v;X%{s;A3r+Ll_cHRD*N_SShBUvbwCJb0d&;m!<-G7K!sviRK7!AxW%X
zIu-%#0W=;$K{viL@Ve^G5TuMw#@p0o^??HrQ!coFe)zxys`(Zv8x+q0HdpP5#*R@7
zPfq~A9yZKv-=8Ia2h+C^wfZFP4nDu%2pCkN{)jJhsc}d6Efgklo*51@C-ZcI++vPj
znvm0v{b{G@N2vxb&0Fw-rs|KsSc<8Df*#Z^b305F=Ky2`^q(#uhAT~DssQUX-uKau
ztbddMv~8st6_&u?CP0ZYVZvhFWMKKt%k;iR+S%aa>KhQk2?`mz_YxH(0k!DeXNlWS
zv!YuukGqBFg)vK7$*jKtQ5@10mG_*5anhz{_dCM=BCu;jEi-lBBg19)Rb+e8M-k*M
ztNhPFKoA5f#c3`#U#mA^gIdYghW>!IQ^^tb0y6VFK%~XOR|Y6{{|&;~8=QxrhT#TH
z3<%1l4uA~UmRaDetwj10D5q9AUjsqJ(QNe!pog9V01|EuG+6~1Vm=E)JNGU@fr1SI
zn7`_A5T(eb!?K)R5*q;r1hSd6#1E>IuhwHZJ_0h#mrR{Aw9l7x`)u(Ejb~+q0s#VU
zlh;m5h7mNBPKCAoeBz|>1|hN?EqE`D2KvYih*q^(T6hFOb>=#{qdL4S5U`s!Kz5HL
zB1V{GzTOiJ=Rg$)#6g7clH9o-ubsrLCQ7`9?wyth)eacqEsU@_?5$C^_cqKIMD(jV
z*%s;&K*x#u1Qa)53gr=GZ{^+rGSxf<XoiA7seAS(P}}b{qWQS8egaQ{;?qEe94JE5
zyaZ|^!FJE>9SY^MZ6?+%3r}`{8;nu7jp6gB<Rd2ep@(nA;Du6Dz=sU9@MfU*hF2hd
z+-BA{CLN&}H)L9#UIp@>zB<}d#w0ujk7f7C#~N6Oxg%G9@W-)pT<IER)@Rw9XR<sO
z3r-wC=DuKsU-LW7d<O(OPOLrWXv@tjKox5*5V|>$5ucFDs6Us*_7wrz<X0!LUA#hW
zkrWfhX%(ZOkU*qu3tdQ{Wt=wGU5<H@jTURX&F@SDL2pw$FY+v`He-7v+i4|<{NSq9
z`#>15q*!Oo$6I|_nV1$Smn&nv2x(m>85aJakh3tvES=%_)1j=xx~n!alS*<N!e@YF
zWb#9VL_cn@5m>~A{q!6B92<!21~y{+jPQ24RnwatHWUVFcTfW$AKT>3$b}bXuXe)7
zBzYpRPfS$C5VBxaxclt&nsqEfe27pANP6>tfS$MzNr%N0uN~U~uE$j4UgO2LY)v!q
zHZ<rNMT|wWIZXv}Iwv-@QVYBjOANom=ADSfTQ<*-1>gcX-0)-cuk&TiJlRvhhAM8Q
z9SjibOv{XoQRFjWpjY(4vCwSu-K)AMbXQ|0OIhR$WQ+MfQK?Ooc6RhTZ_zHlUujX>
zg$R80eSvYM!-YMXV34<{dDz&k&ET@<fQid(BThiSeI<1N04T?s%IL&$#ZP*eryfS7
z!PNm6u6u=fjulhKt`nObx<R14AbQrwRg89Kd{Ov;7$~quW?(|7PI~pBsZZ?YxVw2}
zyBYpw2*a0euyyjW(I<vW54z#XEaxgWg5-?!K<h6l!Vuo@e7n=+c&#-!&2G)W$7e6%
z2H^J*pg?Z6B88OTWig(3RIWZXD?!)cP8Uq^pk!1-6_TOeAnWUA^VN*SP4lzhzJfR4
zzDuNudCI$q^okD+(pL$4T%~pkZ-AN2$M=BV(qJ>r4Uu~QPd%kheO4l!qc7WRdVeYd
zy<~WsLi#;a|HY3*vsnNPL3Syo{Gj2j0?ZNe9ZS7Z+nX8d_VbHh$yYr;n^d>tT4ye4
zA*fUK@uL=*S$<?FW0rZ0`XDOm*(}#dX2=yv*&h%(qupn~RY23dC`uOHv&?jX*ux}q
z<~&D)BFQe7<myZyh<n=#6%U}Hs6O=s4B4ZPKNbjo`$p+CrKy=0PQP6a`5=VmjT;a>
zQ!>vv8TPYCXi6mN4R{Ah2n#{~t#y{mXTLeclnSOPa<EUx<Lpx|%LRw5Ti=mLOlu&<
zXgk%%fH6wq2x&HD>FFpK@71cBMe?$J^_#s%|Grai_5AzM&&eQK22S;4c-fK<4Wy@1
zmsfx0;$UuEzn-WP{}^SbI;BuT;}l?9=7XEp&HF(&8fR>n$a&ge3%IXf@OlI73G2Jm
zLM>7bn=#}n<AIQ_I-+}!{02zM12kfd@-^@x44;;(dDnUxW<R&ju{Clx0}`aRXM#9f
zjuaZf9F%lu_pzeeWQD>s)MPOBTEL7pK<4wFO&)YXFj+gB1$3~ZCXV^fuqB){g@oSL
z_QsqSlKp^^Y;Axn1g~*^P#T8meO}|Q6!^5qD;PuQ^A<kZA#G?H!EuYvERlIlFge~f
zoT`W~J)dnx@0x>-Q^h}&By;I^P)K@}{uRU2cMWA>!WF!g!npW7@06k9lhofvPl?Bl
zbIm3@lv)J6Y#b)keM51)9C)QZ7`RmK6~M+xK~^pWdRDh$iKGqr0N&HVJt3uIUR4+U
zAnZ7Jd+4NiT&m3$aZktq8E4mA^l931;@|UiYMo_KErSCUASqdcs>&d#sg7qa5M<j4
z4t!SUi}T?D-erL~2f#^e!m^c;&{-sGYo8yRaIMKmt+)Cavt;89C@{LeeB?geugVup
z>H_<Ud^J8;Ln_Qlojlg)YW8-RClo=bSm&|1iT~qO_X~gq6;$>8_liD>=^~VM4I+q=
zdW_WU=D)xx)B)vn0(jWm*0P-d>ZIu5>k#{b%@*y%`SC$o2lVrAK)$-6WV?uG<sVS#
zcNRS86K+(5ZtUF!jt_$p`~nOJv5pa1>+4f~CBH%DQch{a>k$$RtaZs*^!kQ;{ct8J
zUOeZM^3vQwtC|j=4V5LHn`c5Kc>J!W3)F*2<;E?LM1-yS_7VtI!XUZ~Nmtfi6uGho
zxf26@`$+A^-ox;8%e-ats-c!$epw&yam%>pt`*DP0(lmHt>TCYDNCBo$Y2Rr+lJKF
z%Hd3TL$gT6f#dfHXC2~fa4WTTWchcJg!Qp_nlCLvy<dJn4H~IQjU{Ay17L%tV{7GG
z{QQ_8AS2op%R0H)6-s>R1?sTIhMp_l=}sN&5a*~}lJLg5Asp;z>#YUluHi@FY0xDo
z_JDfjXgoLP!K5H&=DGmoANoPZQ8+bnWP=lVE2;v^vLo*(9SPiyRKrn(Syu})s+xI*
z7|En(?=k;y-RL#<X}DU^AH2`@T#C~Y<V&$k*xt0ozb|y<%xA)ShzdPiA|-e#DXY9e
zFisrppA<oYn)V}1Xx$;*mhg$zgdZY$6qIv@j%Ux0=23<#C|njs=FWT8yt6}??Dt*f
ztsz$vYlxg$*12qFiD;?KNMT0cE{|6y4Dp<ydt-2H5wjwa%pyY{S`OJmMr%8xOBX8c
zZ9y)*^ixfHqB}Brt3~tinzRbFO!di8alNj}9uD(I|MZb<3a#gr_BaJHefsd!RU=@r
z?B5VzB&T%VL4Eb~9@TMz&j=l3UO#zd_<UVfJr54<KyIv*F$?RWtX?qRQLG@$&HYL2
z%uJ<`0ut?%dIU~T*^j5KVr=@9I(GR%K_t}FgQCkfOj39ZHCv^;LE+0K$`5kC%ZbI<
zY2SNFg*!{_CaG=VsM4u87hcOEW&3^LnFo_>CN!^1`q>vzqB5nBXCGxf#pXwJBM=@l
zVq1RlRDA*&DaqCqOpDOQCMqwRhXOCHnZlekt*ZSr<e9i;F9!`5#PZ1a-qh_?!h;B3
z;o4hH7Po+)5_LOI4M#JU;>B;LZIp`oPj~2&hEXNUq|tz`4EODJ1~sBEbw8gZ45z{x
z4L1qaYAMefbr|0~_{<yz6D`-!*de;d1Ft*+??u_09JbROH@)&l{@kB1g+)JzM0XMj
zia%jH_a9;55@L^GGbg!)H0M6s*_UyC!-|A#E%%@fAXHkd&hiL?5qJ%ZD{{`4A+N_k
z6%gW&xd|;_{UO<&W5T5goD#lQ)3PDTD<(+=%lad8NripfiJ_1+#=SQu08FbChF^p7
z;&8(J2S}N`1nK$D$yh>v9IfQV41f46@z@$~NaQ}GUZ#KpT^^<IB3slt&5D@!TM->i
zP)(VHVsJZDjnJlZHIp~9=mYii`Xw9P*I`;=qW!sAy7mJs3c;l4v7X-`Xej=PNOa;$
z{b>kbX7Rz|B?czS@%nq-quE<(eqw*>Y|C1I)w6>QN^5RYN|>b89=Q2Z!G2e{QKp#~
zmWl!oAs!g;vc1vddVtZqowRZuRCxN$!3CX=X^{gj=b=hEXNbrUh$ZX1i7JsP%a!K0
zj-%J-#y?@x;Z6!CE(E||X=9>H===bv%LnXb&u5jF1qzj+_(RL$NRAN=_pstsI!NLI
zdK6+ud|aE>B7j~qG{}3`c6U3=b=zI|w>_>IHG2)f>zd@XAVNeYBbiGVgk)Hg_?O#@
z7(AB&NngrvI|N&Qg~CqWq3B>Ij1xYN#&01D3?qQ7I6;i!_Mv?NtO=GNyVhz4B)9n1
zX-!8Q`9^k2W?U4XZ8eQ)JhB5pWRx*xjm2ltS)dS!*Z96sz9m{W(j?KNN;Qm{;VIT;
z-8TX&Ad=Ml7H@OzSHJfrYvQ^4ZfX=i3@e^6zFLZ6AJj9x3CCpK_>CtDfD~4??1Kmk
z(R}c36i~DsUmI_)pWf2`-cQ{v%okPHwd{1P&Q3iPCCj;iR2A|&Gi{iLMO6wgEq(2p
zMX!F$>b`yG-TDviVTw5ap46a<aBcud((TR&*aUt2`UV;~Bq;0ju2vomw+21;hQ|$`
z3-ZRntg}DBq80wKtvJg6!``3xL;1FE;J6thV;^gFW6P54S+b2KBv~UQ357^PQB*Vb
zoscC&S(05ONtz_tLkg8b_LQwfvi#1w?Oo63_x%gL-`C6YyoK5B`?}8SJdg7@j<bH?
zh~vmh(Q5|}H}Q7f=*5`MxS!Zs=y79wOX-(Rp{Ihh$@0eXBo+tV2*#t4B%!S>UC^_^
ziJv-gkSXLZcw_><&~IrfcjkP99Jha^dYfwio!2DYqS26t{#b4vX{8hD56yGn)iLdG
z|D2pL(>@x`=Cm`I-s>q*h_Hx2`TZ7Dm4);=yr#C18tpru&OEzOP5Y4XCCe@UIkue=
zZWh`a!2#yk)&!ChEwFnEYhPLXM2L|7gCz;HS68|>i;@Le_-WUf7n-lytv{BKd&DUx
ztsiu-_uwq!f_-*6)%o|b>2kMu5VilV?Pwf6*W=Nf`+*9P@^NgyXjlpepHNzd`jsE(
z5`N7Akb3sE6UAxH(<^XCIx}5<b?KIR6ia29_I1WPYX6ar60M0-U+5VKYMT-K=iGpB
z@~J*Y$sNVfgBo4``c~`glFKeU{g4)jS&jtfQe1<;Lyw}fV>P>Xty`$41U<~kd#aiF
zi2JE<`l4!)q0apGsBp(z4XSLZW65t6A%(&K2Z;?ewC@)I5BPZd!ztdqLkeGdX|r((
z)UF6#AME|rXD7#?qp{DQ5<CHR?pO}7%Tzj%&Z!<CTlRJ4C{d2Fs9+N)CeL08EW2Bi
zDbH^4aM>m?b8P_;n1~BVYG-*j(TJo2Fx7hF^+q4@`r!u`wqV;o)Y)7OrGu<xg_=D4
z9}`iWj$~bC#~`4Y_3#_#M4CB07~|O;ABjlnWpskgwgFYYwn{E28Wjf+d3Ob9>`9c=
zNkpYJdKZEKhW`Fpu;}V_eQ4)Vw%ZHTB@~a(e0T|daMprX%s9Tg3j(jZfr|HgWkW(j
z!FC=L^<jxjU2{>YMOG_O>b_y|Y~tbTeM1MPQaff@KLe|%Cr)6PHQ(kUfC6BvmUmd!
zs6%K~czi?4_=#m79iHL)OoL<Z?yW~P8CZ)B;|kxkwO_0+Wi<NcYjv1rog!s6om*J^
za>J`KS(K>-`x$b3V;yKyHgL9NpREgf_kT1f>T918E2d9Kr!JspGLnlKr_N1(xp-pe
z1-^XrxA5xE?P)_)B(I(4um0eh|M@W(`fZif#|?Pf!HWHx1WFo*1#S4T%Kz#AQr{xs
z!bf@_N!4Hb954YNpkRu89lSU7&yV4+n&B7CJG1_evBX_CXp#cwhW`)Ea}0hl#%=Py
z?=;aKh*fPGUq0Ub_mzq0Q}Bzw;Mw0F#jqTr=Qw}h|8d_S)xj_7!he#s|9ZP32q(%X
z;eKHG|9&`o_=Wu=fHeQUMqq@p9oX1+pS;odZxR0d1H3f+;{Ur1KQ6}qOK-!KZT<tW
zwtbl0)Tb_Ya2HzPlPLs>jAITJ64Ab_3pMJ>f@xFRM%-1EE3W5%@o-?9TpEsi+it;^
ziST2kpVuNh)%_V{Aidr1QhKEhIOAH`twX;4zt<q?92PXh+cg5etO1I65eOR~7rn!;
z+JCnlOh@X01z2`>@(MH@=fLR0-jy5J;aZ5k{b|>2pvhV}Higna1l;b6<PL?FTfaN}
z**8gKaI9p6as~WIS@CW_`~%;V<+`t6@3I#{*^@XPgY*$_r|59*K2|{dj9sHgU{?$r
zCJBI(1+J|u0*iGDAgyO03aoN|0(hGglWk_$?|p0Lj)tc>=$d^R3{RRUFSeW9{-$|;
z^)*28XNQ=M2g_7<$^$!<R`rfs?-=m>Y)(J7T28u~Ar9UcT9lCSQIim6Q#~aH+ZyvA
z=lmBg-=)L~+dol_U3>xPSH4UFXbfA<z14>y)KeZ>V1Kd%pkMgw)^8x;w}rUAl3B6V
zB{qtF@#`P~zf<7Ou7k{E?CcHz8o@L4Xi)s0C_(VSAf79T|4ZY8Ywm;~ll8)z@`Q;l
zw?oC=UZCnZ8>_9Ss&>g3(bFu~-5a1Cw<QLg0Rc^(amF!|PcLYGhY#T9-#_;``rIVK
z_MNGWYNT8WUI6SV`ACZDA;GKg!?SHe#%u&IofQDn+13!klX$*wr;ZQZ5<R>46-bEA
zCj3h~>3%<5t~?c%7D%CJzo*UXYe-JxJwgupK)sU<q?Y@#yJXFp;4{6srW+A-)Jv0g
z@CErP#JBN%xdsXRk0O{5RrRBV@g0^5f07qTGsnr!3=ky?4D3eS4590h@l|65vJlP_
ztH><|ROV)N;@7n<*(&rt)_YSQ>);6wIYKeGSE>qgartUB*B@5@Wzhw&a(z}e0xV=H
zvnuE-jUrYLQN$?pi6+lv3CDvWF8uyq?MT7c9$&$4EZf8lKOXzT@ZTR#hGgLE^tA|%
znudLuN(Ea`w^ius*%eF?0dGFvsjmlquUjPA&~9IV4Fk<lX8<xxk0?JY>MhLlIv$(b
z)4HeQEvI&0eBJl|vLxVVhq32IGy!|s>C$cviMeCoY1>!O0mh(9P<C*f)1A!WMNS>n
z7a#xWyRqqx02gQ)tazAf`+?~S9hXy`>2kXVQ!@=C;eCbjy7~Vj_K6mhJ<=2kAH)^*
zsZ2|Yr<WP;gvg1<oZ3n8wcr1IT4V`Rapba~G?*1VUX+qweS0v+<*-<g<{;T!``QR}
zO&}{%qxn9g2@Tv+pmO?BYQgJ%=eM-#sAOdm?RI)@MK@r7Z#Vf?Xp&vu!tD_RuW297
z?i%A&P>&EH{!&-{xOSv5ngv5-%gWdhiG0q3GrOU?^SbWT<5zp`?0*iYaq;Rxz3*QK
zrMQ_M>H24D&bML)gjf%iVFFwu28<ury8Jx;ek8%U`i@{NQX@YIkV<stk=WeUk>~eS
zL9=>3Xjtv<zZ6d*BeLPUzEi-3y=D07yN5yxb|Y!wB0r+*LiI}h{j3C1lA<E<JlGnY
zd|5LC?apyQvwzjYzusq}9|`E;fWXursBjP^4SH60g){J2C+)k}D~b4F9i{Fc{+sVl
zV!_gbI1aM4lW!W%($o$YnS-A#m&G-a%rkjb+dv`E&RBHskMs+<Q=+)(w+}AnXr4E!
z997M%*&A%pcHr%qxL>^Wzb{1*8k~_{0y(`3tYWvGE~$zuYh0a0Y`uWq&73)vkN%9P
zVFY##{QZ_tR6-;*umVJyc5hN$nu;-^M=BcQxP^-vooD$GU&r6~gs_My()NZV7jnj{
zbq6K$VGZicUb*w<58zMt-y<gq>LPgm7QEDlc)Nt$nhmS!)6N(xU-4mPs&*YmP#PUI
zXQcb1k!IWbdzDw!rk+s)%Ig>K>P6IdH4^r@1P)jkw+Ql4f4_*hSsKOHam1P|kN?H(
z%A(~fitc>{d(%K803X|{bQYoY4-1}-`1_vb+MvIJNFH`n9T`o><`I%n`wVFNApVxH
zq)n5MRfgr|%j)xQ-j}GKYs}3Z+BYPseI;*vju*rIDnNyLNAjPz^W)lFq=9g$hV)vS
zqgZcFZ4NY8Y`BJoftQ!@RrR}|A}^4NcvbE~y0s#HKtA{A-+3G<l=c&#8iGt6A_TIW
z+l$g6=>MrWHAjZ>Gx@6tz=vPD=Enc~4j}g;QI>?5RzVa5p}0iJdhK0pTfTTBsaP<a
zQ$L<+`*)x+l3D<dK`AKkN=k1TMRhDb$W+!}c);J-VGFrtw@Z00;W`Fr=!*b$i5*V^
zIOf+C!}E}C&6deZF6i@|_ykY`h!84zdBxaT_rT*!?6+Ctwb6cV^#~gbx^d9fd;~!3
z00=eYoLgK5HzD*UO+tt)Q|J<m;JpR+a^P-j#_vx-cua#hNP+-m98AcpyhKm-UVpsn
zeJRpJO*!~b%CrE$;-VDcqpIxUwdBepkG3#yHz&M>{_qU9|G**tg3Q-WTbn-oS(-;B
zkOs@w>E1l`ig<)M13uc9MKi;SDM*FvX8JKjM05cz0QMXWgWvlXJ3QB*D=M{H;&tKg
zHDwl$R#cpRb?2mNYN@$!y$CqqZf~#lOf6nR0@=osRu)M!Gzka-$QgA_EI6J|4fuM`
zq2KH{28Axu?z2EA-gSBw8t!y*cv*IaS*f>0=6SVXE&zbxv$UshT=JVfO0fR^<wK_O
z$wv5jHhAe(AM1*iIsK<7689w~Og11E!AnxEY0`)4SiVCl(Ow;pTHFcMP0BaatpM5-
zed2;Iw9z@8?ttJ0^su{z-ykYlXgJ?cgf`ea;3>R?#ygm9AAx=VVj2$A*~cjsN$IrM
zHTgpT`a?KV)Fv75pB2eJ5}6D3obIrt8=+%)?(4_%fS?rW;Vf<~mHoLmUT8Q`rXa(6
zR^y&w-99PU1@|fIuGmc<;oIsH!e)g6@0U=wL_l9W0*(rqs=nKH41vBw3Ui$s1Hp&Y
zbYt^##YH~o1$6>1S%1sNfr`WZB7)w?iQCWYNXUidk)Vo5W1!2*!4bzP=LGn}QT8`q
zW}n9xVGyS-n{mX=^k=|dqnaMb2;Y*Rx$366b!RMb5JI|-bpFxprifx`>QvQ!+LW7Q
zfL0cPJ<(@DZ1oE$0})NXFr*U#W1`0xM?=7<X~SrHncn@7L}L5N*z;ryA?$u*6+nh7
zfboxrD}G142`R#Z?wUG#u%qv|n<%Jn)-77&d33v@M(`~SMGj+<4;$iiE%5@3r!p<-
zBrFT{d<(nInkV3|Ji^>(L3Am`J<m`msxkDZE$R?t`vhgX7~HyXrgMLaXJ{1d7s|cw
zGQ=Un1}5i7$3CoU+rA)4`^hK8bJ0^`g>#HiR2I2eR4;2!>|LyTwYhZhaKBV!G-AKj
zCoqZ_?ROm!a_AiapY1j8?)T8JFPuWxE9F1bQLp+hu{=u)gcKRXr|qdaEGqMx(~8I2
zMALZH&Uh+5p`rR?ENGUiPdt7aaTk{he!O|Ew{)bL)zsw}rXO}UFmb1V-V!s36a#>Z
zF`P67`;IL}j#tY4kiZJo=Wbvq2>4no%TFtR<U*#Dk%3VO8?*hi*-)x1@S^ii-kdID
zrFD^j0`{lhQXCmJ?LL`CAVW3-PhszKiVbM6R3W^8QzVrhaIrU@BWj1--MrC4QKI*p
z*6=L2ru1+<gc<_!k5y>mft&lbf+ITW+kZOS!*U@lF&9OKZy|L^)3i@HTiS}s>6_U;
z>F7fwaKJLWYJxcf;-HTo+^1^6AdCOp^figcPK0JKt)2jvxf28BeCHW@&W?hfjHeF}
zGqn&{P>n}BQumVrL4dS{iQjYYT{gnoUuOY<ixxK8)=zIct<1Pm%g$ViW$srxWuk<R
z<Ej_V^eaTpTv!Hp0w3vUs_iz->Mx4#xWOv^`VPAqL)+%aEu|(;sUWe{YZv<cl;H3x
z;SG!wpA(Okla{0$K3Md+E1p(50F0rRF<E(pU2tU0xMYD}=<m=9->n^RyliF7CF@AI
zVUoR}R?={TQDubUoM_RZiRy;ECuWSKAQxs4piutjVG}Zl&r)BR1r-$*fBUZXw~OWN
zm7y)z_Jv#<k>7g{*?{6QBw9Ebf)(^}Ab`A2Ji&15&VNE-<aUdg6G>S!9%n%^6}!0K
z_WN6yT>#nq3E0$SrA`N5vLM3#&=s#gfe+c(l7tbkAvgp_sH8N3SZYNhIr7fx*ltCA
zvt!iFV}FV+_?(0cn*C0?Aq2PB+)hm+b~D~r-22Jew<7<#1mb3rBtYhi!B^X|(Aa(G
z*3p~oX7e8!KfgE#4%|~9@;N-uYcJ=H6lE`&{ePMD{ERPnY7($F5dKYhGX+X*2nrg+
z0Dt}Sfa+Z3p5z^{Vuzl*y?+4z79C5IGB4F-`uo5YFCa`PDH6&+Jq<c9J|#~Z398{U
zyFrmzRP^ZRk-s6X+(>ln(FI_Z0sLJS`Koida4re|H23+{?4Kk$#Mb(E2%_yjkb7}+
z17dpl^p&I*>}=oyYyH*p9NRvT@4cdBGbA4F0mp%@;=a|0=lqW-87(2}JJ|`iA`(j{
zE+3GdJhOan#}}t-VH}VtS-@*9XI9^}LaOTGrOJBCzg|qd$BYEVq|wua-PBmT+shhv
z@B)Gwbt2rS?t(nzRU+?H?xp?xfW$n)?yr!gAIRhZ0IvK0k97b(3Jq+lb(nTh4ODvT
zhuu8FMZqC9HHRP>MGRyAUcT(Y-XQoo0sOt^z2P~vTVO%FfITj$|28-kA<fOq4d43v
z;-r)io+~O+)PaL1?8!IK+3a9{<#YKGq^9z_I5~cf8~Jrv+O-`4HGoqaG6e%wwrw&;
zEAMJMa3&{4$pwq>9{p<m`%WS&!3RmZH#atO?kh0W+??Iql+oR7ew|)WH<U$GcXy<a
zrJ$&Mzi<l69?3ltmn`~<66K6SBI%h0qjb3BFJEJ6Pr06Sp`zlm`um}$s)&H#{`aiv
zubT7APwpZaF1Lfn+v=~YrBA`d4H9An@3=cLM*1TkzAc)~v$eg}O}<thNqkE)@^UIx
zp?eUV(Hn!lt!Z+6K8(>L{P@BjrGdy@8n8G9IF2TqT$GffKy%x%zo-XH;o$CbrE9z6
z7x8%L9hg~V3ngC1$i8wOZymPuTjQ?9MiPH~5%Ac~9IXeA4}-N?aI!njE{#hZD1W^)
z(LpJ<n_G{U#!w$;P*-e+v4|ssf{t?u@{KFUDF*^}|9le!Ot2o_@C_|UTzx6U7z&;V
z;kmT!TvBAvLWy!{urJyL?611;lYLzGYzqE7Mvu&cVdUGT2#i9=kOT76yuRqjqm|va
z+>vg>2CLoS&cKSKQp{3jtN9mpfk_BwSa5S~#fx7b9A24Bg@$VH3^W>P*~CY54)K><
z2S|Qzs?@_*!B18TJ_f`PgE-E4X!i3kFG!%)*ixkrA+()u%Tr>9Zn?C<V>vd%H2LD|
zcc6vvKfV%gVP;>H1ww9l+pSk95#xPJ_QKi<5Z_c1^Pc6}3*t4<2V{#`m<>;(!3V*k
zFF%Tz7T1s1l8)!^VvGB8SI8$Z5bVA{vtil#-JSR+FArz%G#g2Vb?nhkY)Y=U7DEig
z$V!u&$uGowDPweKm~p6|UjodI$$1bApuKlJVKwVoxsiUNp#4o-8~Nb=EdMw=%xe+H
z26hmxKeINnyGpl%LG;h%Skocmx?#f2r@bc?dQID~@~9opr7y+epC7Y-$Bz0+V{<aB
zyLj<%5kArcZxr+Mz7w$lSri*<;B=Y4SNd-AyY)_^ugKUK;F^^mexSMi^7Knb*2Gx+
z;BrZC6hR0(vex46=6GceK%8$;ZVo4YEg$5`crY#Te6OzQQY#-{^5e4*UHJ4VAlEnA
zV}!KFZ0i~Uf3VgQR%e31xnj-L5Q<vPjX3rEi_{liyKKx9RBZCCh1hDaqnHgN$WSxT
zZk29;s3o*JI^VK?>INW38c-%jv5Kt*@y=doz>Y^AcwBq{6TF)OMRLJBywWF<5Wd?y
zDKIf?^7DKmRY1KW8g+P6O^-9-7F?dSJ%~y1^I|Ky1|qRdI{}HJ8(E^CcQIpJJQA_(
z$8~>irk|^s+<~!Z?jcI3K$(bC^rzoj%a}&2rs|=Sm_?WyuW<ajSHEtWp(Y7d=mqc;
z?#fg)y>Nh~FXZKI4;lM9S;2ibnKzjewa}Yg8RxRt&_G#vftMH$d0A93FMilxv&cWC
z;z@y8`@pjs-7y4NETV(wQS#(e^>L96&ohrCSYr(n@Y_H6mHpnQzwX*R4U+BvJ-1OH
zd<!1!h3k1y#KUG~K#S{9IzVgeqZ2Xx9-cPabXu{bY7jdeA03x$6!mMX3n5R+obo*U
zf1VWiXFv|o1@k6H__Nu+m*mHVVbLbY2kV(x^9YR?puv!pfKpjg0O}!}NLSWm8dro?
zduE@P7_`B$s?`9d*iVc_m#3aGX_fFYFR-D4Y{NEX(zEtQ=6|?(*Ua_LvK=$R8`ygs
zn1CHf5=q=~uokku%xl>JV^JjD+%2NNcjk}%d2<iEgP;4rNz@y7I;D2E+iiDU+*pJ8
z1gnq$&v)*(!vyaefizq}yE=ha)uPI2_m)5q!o9xm{e#FY*Qfk*RUUr7b~_#|v71o4
zMPlbLcPZ8-o#JKx)W*En;l>~^Id2$SafK~j*aDM$a)CtM$|wyWEr=r>4RBQjP;?Z*
z0%dzQi!}b7xQ<2>scF2Qz1*aHrV)B{96<s2K$Y585c5Id7NyrbyuEWkbQR%825aRR
zq{$rKfO=f79Xa}klAz!`2@dkT*LTNkmAL>w=qA(>Us70py-nOnbOe)trr&hpi3z|7
zH{gGB*H~^E!?oCd^?ckRjFZqp=DSDRBnNmk&iVB&J_SI&eH1*1`@udTjBe{>`(=-b
z_RH(vzk(|De86+?3@z#fl_CN<mfKbr0(W@yXo@g;L&)r!4%G&`ejImLGUK^!tyYNj
zt*WqV{pSX?$3WWs0j4Rrg1fBp>Dhk7;}{^Tl-D)xmhkK#$nJu}{XEz=V%em@LJ9W7
zs>jOg&!tOXp&99ox}RmNu9iq;4{cl5tW){`x=(L*W+-%|NQ6jbrqzfr8te;<Ko(#I
z-Df>agCPZK`DQ%REn=d>t%ir9g8Z}rW{WASP`sS7AP0-R&K?C^eiaV8s_N+6E2cn%
z)hX_~hFys#fSE+VlT}!UuAS-HQ{QA9yPkTyfUy`yTj0SD@Y@K|m5?Vih0!I0Rlru2
zuzHk52tZ%SXg6Gcsjcs;+7qy1vgWZxkAB{QMqzR-!#QuzrjCLLcX|z&^if=V*FS(u
z!8b3dS#NbL9FR=K#g-V0NHU0uW0kaTS`_p!rfUQ$35C$fG$byXUR593gPI{Sn`ptX
zD=1a<3wywq&=pm%o1B12jJ)kRlynvHjPF6f-pHOuW@Q<~U%dDIVLXKfT7GYec<)_b
zSC+NRg{Sh+-{`X{-Vbi>-Ecy8c?<ze*4l6IGaa~uF;V-Vs3^ob<ka`d?vB?XfETM+
zb{Fk)RdJ!Cex=N&ll(^cG?@+~BD~NfRm3DTatJ+*%QbS;Iz}@-SAJ>_ncw}<W{`EM
zjW!9_rOY_wA^HG@o=y7j?^V(Bvo5H?u3REOTYXw-`$c+&SS3=MoU?Y&F>0$?Puspa
zLB#J{Zcy+%%sxxs%mjn*@JFyS+c>OsegzaKK_br#2*bA&D0RN(2eI#aR$A6tI6#NE
zWn=DveTQl@C1W%tXOi{$Q`gOnb@gx+`E3P|{aFKP|5S5GIdA=z!Yz8QidxMnF5aI!
z<kglWdi(hEu|tg18c;wHc)Du~Lf_Jemy_PQnyqgo)v-o4CMmPH=?DT?^J0iS`J^u-
z3yytm*3QfNSIYL++&Y00)g#})3<i#FYyvV6@KDFR+NCY2A$6!~VeIHLmG2^5;g{Fv
z10a9)4sxSQB<r)ivv7^?`@K8E2!P)tGCQ``f)lKE$QD6)@&W0Is@3M2XAZ4iai_L+
zImfuN)}DMpWYa*8(*<-2S}-}Bt_QPN7=Qx)Wn-7PPu?@6?ok<8XUMV4x}xS^Ud6M!
zB~JLb!=`L@;8>KH-R<L_p~duSGG$>WYKD2`+-m<>W<yOBnyZI~!*D9g*W~`Y9N6^d
zpeX=zfKHuZ{2uHx(SAw6>$OVqCUjKV&X==Z>eCb@j?$0POs@f&)cqCsedD}X85cNk
z@DcacArOg@y2K7juRTa?NxSx~Li>L5<`pP%_Yg+%Z`2@%gPtIfz;W+fme92-sQ+ET
z_N2GYVv?oyp6vvSWw@j9TBp<2nNsR08Nl<IwHivJ0lS)24|5RN>!qaRz5P3Qx(sb)
zmzmFab|e&CRC$e0)^kvn;{!yFIA1eCclx&rg6Bb&aLvzAvmyVQJn&kfZWC2x^0>f%
z0+MxtKA}O<lJcq4r;!)shxW3n51Yv&B&>z=h5?Sd$j=h95^MdYE#Z(}@f+(u!Ms3=
znk$Oqt%q=?ei|pq^U$}z=(0||p{6RRTjamI5-;Ww{*WlYiw&j4BHKcRxpaNmM3<7W
z_Z!mZ4E!`riry9|GoCp43d<HC=m8TXp{!_s&t2g(jrTSZ_;L7?Y_OH4-7=Dwl5Cn;
z%xKc2VUEza4#yXSezM`T7~lDESG@BI)v>*=`}cejEw2aQ#rDP0Y*Rwzx6hMOqwX+m
zk%a}CVy-i4F0D8W1Me-Kr|%i;982Ti1nNAqD{!(<NhYXT@6Pbbz=Ms)cIRO;D~ajS
zsGIVfHyymfbitQH%Er-ejH1Jtj)Xp)l#oK2ejxjBS^8lyLcyosuh5@G8<<M;WKdVz
zIh&4M38(Pj;lD19ZGeqtIB`~1_}rZqsxzF1I5W?8qwQXj{KMjp6jYi|kam=YVpk$5
zUMfn>pS|0BL~Qm-_tk3F#~d6yYAf%^waI~3aa-8PyvR%f$L}$bmeY*hf)>>{K8&lz
zuiP?}b)@W()W$b+u)~P4<ypR>J#DMrbOomn>2>M(8?a@Fr0j`?0d~{S+Wij1DRZ6&
z^_(%EKMud@j|JvdTp5GbgQ-?Dn2$W_zB{C#%!2W3_9h$08>BX`G~aeX#)bt`(H(oa
zlY;&xZ3orCYkLOo?A$+U%*etTWUkA86T?&o3`GhmOrF(NpEAaEL&7=Cg{+;QZ80p`
zL@Nfgb4l%#C(8qstg#Tbj3E1WLVP1fqP%!a4`<yO8dOy;lL)sb+jl?v3TQK1o}IM;
z_Nr<}@+;fBeL^Eg(G3^Y-<EAY)k2bk8L>Ci%on((xS^fPFr)rmRs4j{t9Qe!JNCJ@
zn!YKKZF?9dm{wtTJ6a`DaA5XlVujyD0;H*%gJ9pmuM#1CV)R=p7OSGi81rRp_4E6x
z7?F7T#??xr$^#jZ3X`}hw2bWLkoG2e5%pj+z~jrxRaK^~YP>9S-<GdVcFMVL;lIk2
z6cR+@H69O;Rn=-qX1h^2T$x6uWvAqK2{qSqA(E+{F$G%F9EMz;{3F{WMqtOG=L(66
z?xNJWoF05GCtca}V?S#Qlo4C=;AmWy4vRso8TICr;<e5`jGp$$wW)G?5q6~6<Fdrh
zVu=HFqfh8T71mqgF679r##}y~g5Oq_-}MB9%ALOT^oVu8=jqDVyE`b3qLv@t(Ylm5
z^i(!*a?M)!)oyRH!#P8utEJVg#xgD{KAVOBxmy&hVOI>8vhUn)nuyzX<1*Qd?M_%L
zc46L8(-38L53exJ!Y$e(lVQk!Jx&&s38+$Jwn$*S4=EuUCf%L;nJJDip+ku}^K1<0
z>Pt>)*<z+wp@?u>KS>f7MBa0hAepJ+S&mlR6<rMNq%v2w!7J8=7S79itows1wY@*)
zBsW5hE<QJ*u?*;Vidz_cbS2tQPI7;dV=Pr7+lUl}GU}fHe2XRJD)X6schP8$9ntRk
zU;^Irm>@tp8KgWiBxm0e=e-69VGAp!p2tuWP|0X(zRA<>&r5?YGQ;&hyj<`m`bRDv
z5JS<JT_sH39wB$XD*GKD-F<5v{UP38B^?Wam`)XH<MVR7x+6}S;w5G-*;#Ysb-)t>
z%Td(1$MWfP-}Y=iWn8sQA9_c7>99}2d<JVbF)`Flw=&&8sVlNGi9Z|5`!?M;RthKD
zj23v?r;;jUjiyB#@-8TR_!Q;lF>>iEfUO#k;2AnBbPx|8Z^qfX(TR_Gpf_K%Eq>-i
zP8kYR2|1Nm+pi~yTlma3@xJ>;)9}PSx!L%o;gp+?ghr^)>B_fqMHORb=og)k%<;QT
z>6V^_rd$fwJ%hgYp#eFK#uYqWxpX9l*R?7%v&`~FH&h9i(0~etEKmc@kopz{FK^{|
z3#HH*YJ|0p5GbDJJY@`mitQ$9RHh{fuse_@9<+%LblnrHgrSt#UxX{hb>Dorf?lSz
z{?2u}A~Fc0Mm4SKL1&t$Mx+n3mVHQ`RBRyn1R24u*UVNqeemCU2A@Ll{>T_NZV4w7
z4sd|$P16Qlb<s>8VS{)iG@hz_(w<>RJy-6`eavL1dtYdEsJctlJ{!H}w7F#8_V0m{
z;s$sQsM>pc!^yj}^3Z7MW|Ve48XY|vLSai!Vi&;^^%FJwgt2{|$E8k*<en&3Y1-FT
zeEz(_xB6v?v-i%P@2Ssg(A1sH!LB4zFuN}3nKc9~E_~#14V4P9Jrt31l#cKDI&QQ5
zr2-RmK~^pc)Zj%5C8zEhlOI%6LN%a&qa5lFleZ62yOG)g8q^3SN6-IGhLH%7cyV5Q
zWTk$Y287+0%+d9E@<zrJzhD>q1&okk_t-%CaVTRA)lnkTh~X;KxI#*CjQnrhnl3*R
z;;=6Kw>BhH(u`xjEGX_`#*wBmh>wlVfSycV^{*7@Cw2nWlnz{N3He^`!tWTQ1c2IO
z3>{&%E3|E9uqrRAV(;GV2g*GMD+N<M@A^zMF-!BG+PBQ4wI=J}`{w-DL-MK}UpAon
zZ%Gg^Rk3~DW)D3o$TV;9iT0m><j)j(NYDTbg7(cefu~jWuV9GN!Om@wrU>YO-n`D{
z4P6XlNS-Z23m&4FqPnxy-la!0!|@XH=neE<M=K+Yoz~0*&-iD_55x0vUpTik+0{WI
zq2KroxAO+1pwoOKcU^%+5b)`7#nX!>vlahy6>v!9jIAz`6A7arc>|GGyEF|d!phW^
z<sA;*$dPTAZKD#9b{=vzYzs0x2N_~xtlsSOC1G7lJ{Sm4bjqmak(&qae1xG5SS@mq
z*1O+c039zIp*(9I&QSB0e#9jQV;#K$VX{Djt<a((jBXWwkC_?mk7nQQ{hUYBh2ul$
z!Zk|8U1#QOFgT!Y09t1cZnDgn%8?42Dxm`>-ZW&0@9aNZ=Y1r9OEdL#ou(<D=#M{m
z3_vIv7LH`pI0-~6peT4xJ)(I3yd*-AWQ?J$39HeOrf+i9R{iJBj(8wR6=a>?L|6&?
zh*av?`1p^hI}T2EWg4tPEp%kGOOkKr1;!XQiKdrVt08Hez(rA%_`R6QH&@Q({J!k@
z-M97StI$Dh0o2)t&<^gh%U*+qC~E=79?a*FArEMWVldmZ#v$)!`9z0PPE3fW*b4{Y
zfHHhG;|%$y>dW-U8G-c@b{8W-271^fP4CFgCR?WS-=Qbn6N_iPOai>Ek5}*nP6PL9
zi)vWZ310yCg*3e#ihks-{{dA53Zz&g?&-%R>5qVJxcUTZNZN@Y#-U{c10v?gX`2EV
zHQk=SuMLVjE)*&vk_m$RaSVnxDPaahqqN+s=6E8ZIQTB@=M5Nqd=g5ObpCwy^3(+k
zKp;*as-9t&q>Pm8ygm9>F*HdhKvVUyq?J4x-KW^Dtf8$>aKV&HdM+GnG1O)DF#Bhp
z3DAs;aX+#+Q@b{J_Vdtg^pe4dkg?TeA52y+CcElh&$s~x@i4`wHK-B;#kRVZ9+E~V
zC(EUKGIJ)8ifWPc?bF9$p5YBBy~DTGssenW`;OuuMFWrHXX0CwaB(G-=k=g(OPrd6
zx~@&CLdmB0V9n~?Mn#41Xq(_P7}Wg_OhjNalU$uJ+BrimWvdGUF{?T3un1hju9mZt
zuoxW9QJDCh(76FsOx)Frp-?i>a!9qF<wZHBkF4d|;wE<Or}q}B1H2`c(OaKbN%1@+
za$<R+W2>B2nem#$7z!1JK1lJ3I{X;Z<=@yRq|{GY0-)itK>uOr<Jn?oU}dz3-MLYp
z-z{<#V1X#tat=xpA}6Mq3f%uPhuoirHgeEzdiDzo<L!gLN!e6V+$Yv3ZK9%kwo-?3
zN;onS19h6Xa12LaXm@#Ki>qKgO|yLEZMl}36gE$;zdkIK!`_?ClW|+mjjd~rG)hL_
zFaG+yg1=q|a*H2$?}`lS%#7`uibMA}*4Mx>5tm8y?d`&B1THSJnb1OWBqW8zNe$WV
zHewUdr;c}W*_6K{%8s+7U>9~q@*aU(fsp{xJ6|2Bip9g*DBrHVW*x?l7Y{Oh|1`4<
z%5CSR1we~g(-4ie*cs=!w37V)q7TFy$yCqSJg1D57v~4uVBR^HS&rZHDtYt~P6qO!
ztHt{;8v@J>wpE8=oD4u4kEan`b+bVO_}0bqYjCS&BZIOiqV+@xV}zn$*t>6l{y6EQ
zF6{NPMxsG7^&^%ITS)AnY~aKjfnf^vu}vpy$OyczlVFUsVkI|;Ydi%#6ChIRK0)ol
zgwxlFd9PYWR`Ij5{y{@YdMqRR=>5Bob%<R0uv)Xj0S~y*XpQXW8AAVuQk)7Z!o~Kk
z!$3448s}XvV(j0#c5rbydg|gkH175qXkd4|=5wt7?D!?&ReBAYORs_V$TwhgM%v?g
zd7PT}I<@6@gO7xnm=Ud>!y2$H<bsi@P%w!M*D)c<YJ%7G-@h`^*0p$gDmlia%s?bp
zO;U&hfH`cP4#jOF9nB5_?K#;~nTR0E^;7v^g;kp)E|FhbcG5gAax3X^%+@;|16~Uw
z54nyLzYdQ*jyZUZP;P(MbC>St5h7j}jh1{uqXQkpVRoF^3Bb{yc#qGgK%rO?FRZWj
zDr<NtC}v7Z{e~+O9nlcFj(pDtaE-=mV4$l_lvHu!>E-Ud`XjXZRxFBWa7~5QAScSP
zXQ9HPpdIROz8dCZ-s(tW{x~yM$~%TQ)0{5njEEdMX*A^reHLO6^}>ojKK(J^KKB(~
z7F(4F(X@v?**8zGt-<{3KD@D6ux_1<Gix>hd~nc~>UO>-L-4L@?gQIsa#aWmO-RWC
zok@rhEvI5`8WvZ2li8af7~6IsRM7GgNf)1{pumi~c_UH!NXJ&z@Zp0;VC)%wk6I?y
z9S#|UTNX;gy5LPpFpf3KQYgPjEz@b9xC+<Xu=-|IB+(!1Sl!mL#Vgai>XS)D!`oz_
zDY0I(34Ob5_4e1m68@GD*vyN$#k<xGC?1ney=%F@I$I*%=N$YTZa411`nj;rZ}ZRH
zd-B>mG_ko$zK;YzcSiU1QtfG@*SZ{%`&;RTpY#G_RADGIL+j!NSd+{gI3=#9Y^(kY
zO>>x=O!axeikaLo*nkL%1liDeudv`R-E}NItfdN(?$3cngsCamrfvv(o5>`){*x94
ztlLxlo?Zb^)3(x4yYJqjmc8GIFIp;ED4rsLNhLfsK=M+C;r0b%t~X;)7VNKZ&?uX3
zY7-aRj`1@%iiAW+rDDjy9d5OQMO5z2Zg#;A19R6Cc8s%{s$a53O;BeN*O)PZ7eZur
zSx?fejKn1bT$t!clQ~e%OC!{%!m|#D?;-7PVA!`Lxdv;&{2+85BF#AO+g&~WVo)$Y
zzq&ZlQjUM}Y0kx!g;6<<_yU_3_t^Gs*OpWOI8aAUkg7Vw>1aIj2xLj)4eLos+gPYS
zTtA@#wyx?tIpAlZGM}Y<lK&j0_Kwl-v(S}>SH+v+{C&9jfwd71A35i9pjEfz!}R=4
z9m~r6J$L&|^9iAmQ)tLAcg1{xuGEo`b)eOzNLtOpvHKz+?}=;I{IEw)E@~rHE=gpG
z%2{p$3pXa@Nkl$V+-4YnTM(yG0?ci=a2>49yh65)Zi$~7Z&LosE)t~?(qx|GQ#yxw
zQ7e2Fg=R72F=xRsc)_xL4;7p@`At$Dv0Qkw>SOy#9}dkvsZfij>5(FoQN%3L@4t3k
z2&X_|5j6?|kBK^a&G)6%-o<`^+%^Zf2hok4qYw+54(}-eI^Oe{O4&w+k~6<LHr^p*
zXiuDNs5iHid_lVUj%urYrRUJC{k#n=YnvN6E|_{cJv<uKaF=Dir-&V|e;!haCc4ps
zUZYK6&}7guqLx~)v0Mi#9GmmLu%L(T9mBxN1vFi=VN0eiYh?XmQtJizfiZ#Zt3rza
zzcF8AK6<jI2f*l2^`p;#_~rGYl<OfDlHu4u{+i}4iF$)Dy)Ct(*);YOkm!~G`WVpa
z0r?r^#*R<DUe_@*_%1>Bm^Y;p&f62Fg8L^W^To`X6g76}#@5th3LvqJVVFghT01o(
zuiJ6JsUzerK#pq{$GsDhTskEG9nz=FYRyoX0t7tvtK+eDwMT2fl!i%{X7zsV^L~4N
zM-UPxjpxgnsD=AJcee_cM?Pjl$qA*hwWY-3cW<+M)G31Mr!}iOR={^bV;fDnSnl?2
zi)a-rVEOVLhwOPuZK{I3)xNN_2LTiBi>wG~J_LHj=O9Z`3FiV!Vn#Z<8n+`dVi-O_
zPUr}J1F=A58jz|6SkmP3x76^*dQh&>jfh<Y4O!{vJ}!<~zbzFCf@$fGpkKif)e`bV
zi_Q6ziL%OVAOpTA37yE>3v@n#>$p(1Nwz-fb9*VU7;3+$(;LVP>}%4nqiL+C-EuL5
zz`_G*8rD~v8`yg%Zy0Cj`1aH9@myj)XcMu?{Jp~X*V2Go1S>{lLFwlKmx4*FAd3^}
zeG3;ka=eQ>EWo@VkJ?yl=()e(RjJ-$)A1$lkUz&P@7TMzCdi~}Zb({}w)Ukx65X5X
z9sO9={S%u_Kr+HNl+)XU4P$&34F=CvyCS6W_dLy38_3e}CI-GSx-5BWy0jR;qq<bN
zqbr3v{LwA*!(frpaQ^!lp~0bQCDKcsDyPST&y`i7G4UvG1A%V^FA$@|f?!T-U^M@i
z_X7!!z!*1E`iR0wld5pBY06zxB2F_b{NH<-IThU!?`ECcFKe6^8_ah)e`B&AxT)^g
z#fTkTm6*Kc`N*K*vzty#-cL^(PzCtxR78rxH+#$uG^fZc?xViBTj{<D%sKm#5v(X4
zC#sL`6CUZ(6JTx?L5-uov_of{GmavVGw?bj9V==oz-(J#pFFE{ykEThw%R5qy1(M^
z+x~FMH6Vy0HJnM;MUyJ03hOfNa}1M@maZlNU2%%-Asr5rFKX}r^r9=Dl`n6VamYG!
zzIY6G6SFNS{rZsQzAK7?&$1*f3+d}AC)w<n;Ipc=Gn%MTnG`%1#cW^UQaZ<lm4TsC
zQEZe`-wKtQ1vtIZ713-g8M?=h_uToNM-gaQ$h0i)_nvXH@Sc-q<nm8Q=Uwueamp@k
zm%)r*dnuz(aN*n~3RC>6veM?S{7`~jM|IABQ*F_WElb95eDBt^e|@U;>VqzqI|BZc
zDmhcm3VRU#YWJhanT7VnSmLpVAf^0ZH*DWJ3i-nxXCm%G`}ix)3%&P3uc>XLFWQUe
zmc78?`8G>^-?k<bz~lZ~bcSif2KK@j@fLK;>s;s~$fX{rA|vUEW1$7*qBQEpX48U$
z5S|<%@coPD<2*b<=T+&bwD;$~{`IloF|<<h@C8k9V`4Lz&&sh;AJ2U0X~jDhj_X28
z(G9-2^G!qr1TqSH7Wghi2Td($8m0B{k-LsSwW0+$u^x?Hn%*<iVs!P6zt|agUkvn2
zl}k0{96Mi6)VJ@Uu(YN^`=j?{*SKJADhzPD2V*%uoWB>BaF}AM_F{k-jIXaIL%x%-
zTV>0)Zuu28KL<@XoY1_ETOB(p$&HZu%36>s-v)UfBr-k5O<X+qCc}G`dt9=VrsvBI
zonT=V(@{O5WlhuVAjo+Jx~0vNq*X7|UH6L;W4MlL+LOBd0ONhiSgDbBb4kCuF91Cx
zx}#N0iVxkZJ$s|wPqe-VM@S$lD@&X%99ahz3av?fIPYP!%0kB_Vd@JjXXXY;@3k8~
zRiEH5*i-vaqV~)OyOcG#cglh^XFk4mmk}&NZKDupgZBF5<pT~gZ|<u#uFu)B0K@t$
z3`nfpNl2i`Kr^Dv-aUDUx&b`du3)XQ!HA;uJTHP)6(0YZ!Y2++Js<X}EtOKaVt$kr
zq<}4xDW`-8iB$ALkt)Q4n0h@oN+;ftOUwq8Ren(qE#@GVHMoP8*`i{g_r~<=Lh*Ei
zz59O)qZNbV;&wsX*`={b*(9!uDMj>2=C1t#ghb;AmxOjlYOX+hx%J8v^AKjxWROGM
zPe-<fv@oC;H|2JbbF@sTPtm%ib`^2rE;8?x^;?2%YI0o=QD6&Njhf-6E-Km;nF+R6
z#{m@hG*iBG!M5`tQ>5TlAu5|%Pp<W51tjeW&3uG(GUdU+a+|tf6%7|1Em~ZnGgFyq
zi26oP65)ER9)&=hAC0>XMEgh1x`odR0InGkuY!zdP*_%6WSxE%Y9pc9BZhDN#D6Es
zcr=#~xt97(_7$ZQSFV-pSD{7iCd8t}$Tv(9zW_q4%&F`A_<h(t#k^rhv7Gz<sc&B-
zA5NKZB&t6*Q<k+}cN?j`_pxTdKudrRWxdqvk+<8PePkcr=-}uL<CJE=u?^IB6ZERT
z3HWgArr-s84t^cl^hc40bWo(h!~o+^$;XXQPB#l}<Eu&{96R<Yeu)ip*#C{C$a5}T
zf`7_K4BUzXSa1r!9o+kc-~Q&l%LIfxHYW&tw1d|#Szy$wR*fXaxIOi00>+1Xg@;?k
zH7<XZLLGi<-uSV>^<>okkLe$ae)j3@MF;`MKFQHwKR9JeIIv3>nXRRKE!uyN0ca@4
z70MTXqQbw@GeQqm>QKER^z&G7b!ruE=bet!1GCk?#OegU7@mvtM9gsbzuN0Z@^2V{
z1QAN{>Klvvx4lQ3IvvX12O;?QVUx^Ve>hG*(K<&g?gm%p+2B1ik?%18b%Z@QK2Ty6
z=CBQ(I?dJK{Qr4siX)dx_TRLqaqIPKpYdy<5wFp1$@sQG+lCF&3giAoO!*aR^W|aD
zHlO^`ae2Ez!ww<u#}zcs=XOFq7jQzq|7Yv$zpBxO1}AV1y13B2jr@$h+<`~kWO2cz
z&&NQi^j)H#;tx{<`3C52V5NXUX+!9;OeHi>OLJ*A&;*uUXu~!r3C||{j`hE|E~OZN
z^PT|}#b$4beG44riKmWjGy==);?taSz)t&(T*5aJdmVws0F3&kuNT@`49R3<b~-W-
z3-US>!olNB|Fz8jdK5g6Ya8)S6U=<i0rfanqs7){V}$Euc!WdRo^zR9a{43!o&|37
z2mZIIA<Hn_hXz_xHVl@6m$dL*iQ?TU9}M<-qPKS6zWo2dG-?$gLDQclqXO<jMuS7z
zV`G9ejgvlmXmAoU<_O2mE&sm4PBbUCVH932`2ql=p3uM?c^DGqZc-+a2K86lM8%8$
zn0P<<J@UqmP}llSb>9cCdikIN=~fjIjEAtUa5%p>_L%Fz{sSus@W_o!h0iws!08rQ
zWS}6x5G*J$-mpz2R*rVO#vdt4ev{~e*^`4l{gYPp=l}7}7#FuhIR{(m9bDR}%&UdL
zte^%Hc<X1)2WBUj>cyr*vzNT`$9Ld5wZE^%FP)wWIji+8Xn%cPkS1xi5i3F>18@nH
zu4M?FAb!xG>AK;P44}BQSuPnFpa_X|3MNq}B$Z#Ac3*9{<UjWrg_i}P3xMZ)2#KvX
z^%TtjiU!Hhwp-@6FSEg|5}~@s4fWh#pcHc}JyL=~+$%>a(YrPSE+oQF3??Ov*6*ek
z0E#0CXDwv@=X&C0fwTq@=s=N~wjv91uUP2SZM%R1D7AF_z&}?|5B~@=UMBmf)xJI#
zA~=u~zXLJdzS#76Knh%P{%p?Ed;jMFp=9Xz?}CE~xRasd?`2V#NJZOP&ms2h)*`t(
zc3%2_T0;~hP$wYJI`loBl6OkZ@}n|%>Ynu7+qp-7WcaT6(|`6G0h0wcYb5{D3LN@T
zu2)w$G?>q7E<Uo3(G(iFR5!#w6P~cm<WQUmc>u_4Y69mS{FEf1Di-~56}4Cfp|gPK
z>|j;FQd+IrL`{RlXw$ylZw8wCM>`6;CMh~y#|7Pv3;UXv|GRsMH?X;SL@_iCHc3T-
zik3|U5Oq-3WjAXU%>*E<#5bW<5B}#NbiX#d!B$BNc3Zp|7+k`pI(iL#X!bM`f6j;Z
zYyER?ai|vsH?n-%5xFQ^n<zdFc18~Ig{ZJ0h0D$#J;VQkM4~8q$vRy$=;I+NE>fVB
zhMgs2!k4|nEdQ<+9O)e<o>z4V>WqrR4;<8!OpwN@xE~Otlp(_GJ?Gj_3c$hJ=ihJT
zN<lMnCexIHx=>Iwidb>-#=A|1Re!_2GS=o}=hx?P%{0B~dgK@k6p{D2cOGYxwjYei
z-g{q@%L)w?*<H)9?@u4wDV>IJfIBC+Lu`iUhN^+lfk<4I_?j2pjW@UDUW~Bu3WEZ0
zCyYFTaCmw9f%t-k_MgX$o*-T~MBJO@A}!ZLATA-Y$Y$j-4pzcQD4;=NsIDR!=2k`2
zmo&e$V^<u6WJFvRp;}%5myK%|YGI{9^(}Ji^leeJgRnPrJvIQK!b~H>_U^w>{``ZI
zw6@z3nB#zTYy!9eU3-^{pVf=*4+CoqRLL;lm%krXxKcM>E&KQ+1;b&?YeBLd6l?l}
zv=)4C2kd<rR6p)7v(qEzU^*WVH<;H{17~l;J->sE70_yg55<BDaJ6m+1{o#@s-rhE
zN$19dP;FTI(3{12_|r6g^KbYxf&nq-!2~UL;>Av%avPU0bsQm+WZeL_c)Klo$q@51
z`k6WceFpfmook=#<ry(-!+1us`xXT?b@%02XASHSt|$kW>BLqk8*d=FbxR|5^uRgu
znH1lFzNt%A8GU5ZzRkjQj{7FEChPV(K8|KbgjfK!zAm%Aee@C;(x*zu?Lh;vBj)q8
z4--owKZ#mw_aPS4CNnaaJd2x{-?sQxw%?iAJArtK(WRqssx;8pDj6F|Fk(r(qjS>a
z-Y0B>s!aJOqj~9a$Ni#0iL;nu>#|j-Lf|6v;DWS3i~NI%tYt+rViR*s1`gGEXiNm3
zOinIVSM);GeycjFq2Bn4;!W6r85KGQS|SJlkXI=TS02#~Dn3xwCV~f%bl-tjRk*}5
zZK)J>i%F1#I6#9EQG5G;v`_HcGmIlOvtp}cu&TyE7yGXxpLivWa8>;*5V}K4gl`@_
zdxE9vhO<R9o5U^WCyZV31INT-)-UmEbD(B@po_T4LjK@=O($$+<7BFZP+~C#$B=GX
zjP9vJVko`@1F7{s<D)Nf?1GQ6ALT9I1SI<AXP{!}hSNvselC{J0REccw)3}lpi#oa
zsU{GuG0JI^TM9-a3E|i*(9{V|Jo43{o&H6)2t}GkpeHz_RDv`ie2gL-@Rg8F3E^S%
zVoHX5_-Tjf%c~=_$jW(GWb`Wpj=aUdx&CDD4uSZ6!Mwdc5!T3TGN7M8<i88exU1lN
zxb2qD+(5C56BaLA2lFiuUjkr(-S!$r2&tyV_l}FAkT?5(!wHU+1R%p#5VxIt&2xID
zZvp~i?2J6Skrc`W1Cc`QU919Z5C@=_ff7Zn>O;wG6{Hay?`?%_fxo^8bC^__!*2ks
zg_q0`orWeVxCTpbY0rIPvF#h=-Vdl6aK_>J*MQj#jD#z#j74QDz(&Ad@U*J_>zUwL
z38fe%uh)Zcq60MhyvRHxy&XF7YTyV_6`=g$VAYH3J~V=kE|{(>I}ZZ$0X*@HBHN+a
zIuEmx1mj9Bt+1HQJz80^`d}+YboSQZycI8ZWT#srOSm1w*&N~qUxSvx>@|2vf9!}D
zRI^Z7)PRC)nsFRxst}$Sc$D;vD=%(dXuSW->T_t?GQ^}!plx=tnTUR!8?wwLkf7%(
z1uznvDC!%btpk{RgOT?$mQ(*6zj!PL2!FN5pG#se^x@v;TCq4>qUKC~vUlv(Q06Ue
z^lE43T646toY3g6<6W8l;2TR1l=lm8$~!>Ul(^-}b=|g0*WbOU{O4&27<JuTx8Ccu
zGC%|2SMp3(MTUM)B$1OA4V}Pz0oWtuGscicAwI?2pGmO10`enf^?}J9qXtaw6TrT<
zRc1~Z_Rw9P;NSavJP-}2^*Y;e3ZX=AG_)6zqmDM+bZykWFlBzRb?x(!1G8JOx??l+
z11BjGVy(|%_%pDl|K_XXpVQ#+&;v41M(x$}4=mw<ZqQ<R*JmF58@~9e3*UqznNy;F
zQNRCLGGqkUTA*PhFNo}8B8wcy{&&v($Zs3*PzlHs?#X8RpW_Yw39L$3Iv*pS|NV@=
zzh4Q4jHNMs+i8n`GnOAwb`0rC)CpVvhJgQ0rAO$HOxfv5|DOLjp^<+^s=?kIi??C;
zXW_#lf*Ti7_i;1_Xd?-na=~k$|2CNa06<LF1Trfopba2qUnQW3PxcSF=A4*C*pHs0
zKx|61y<x(a$r^Ya(0_YmaND4ad7Clo3P=dAeOQCl_#W6%JSyIfvAs{A)9nwCM2=6p
z#D`aRZt%S3XW&*;fz`wh1%mNcqpdBM;X|K#T+;r}Y9T9QY+9@Gfv=MIUSk6if}C@+
z5=favBb*3S69llGhyNnE`Lzgs?p>gyE(6^3E!d(X!ZQdk;dAbg;PQ3@ZVRP3hzh*x
z(@F>y*l|OnlH>dTy}<Y=42~LM#1Em)Y(SXvhlE2-;gB&Cw_*|Ouu}FzrIuErvB*GP
zND&Nup-Rz4jRQ;CKzjrdjJP*Q6AanKP;(Y#d3PXf8K_ZqI5ez4_mFE{Cwrd|&`%Kw
zgh?_P9+#m1402Fzho)(@^MG<@>2zzjv?eadk9S_x$mGUZ>8qFgg{vB#LN`cce}5o=
z!9dIA9!RT!o_u0t4N?Z6rau5LXN2JaXz?oeGsI#UvVA|bY~Nqx4OG+NbMMO_zh{)u
z_eGlD6%O~COfUJ)4azAMKuwwiEH6M&fi5r)5w1oQF!FtA(2llg3_4pX=-If(^KD~x
zHIfESL=oRXmP0R*tLsmb%V<_^uR5zZDMB%Ib=0|yiBp!StEcl?9pNO2-qkI?JIO6y
z<vO|mA6`(ychb1eeb^Zkt@)iW+$u6<6b5(|1J@$#^B#KSs)SeDor<H-(&xM;817f5
zZzb|y)giHG1R%@mcgJpg1sCsQ9sa=d2QI2m&S}2yEs%v9JX!++By{9HBHVE~)ss$N
zB$MkPlt7uIzY0iS7T9?}2DsINsB1kBg60-rp%*QH`kG#%r}o6b<J(x*m*GyV!o{--
zSPOAMb$tPpKM3Mc{)~th7H2cq0Rs~EL&60|zTgO9%fqH5(Fxco1y8(~?_PF{i9b79
zz5)vjk{j|Nk~Q|Sg7#~LZAV_dtUN-g8p#siY`{y{cHo*nVGPdTAmObeiQl4q62Kfj
z#3)f2A&@~4tCMw9a=;&raAZ)4&qFgY-j|RZSh(O!wZ-$lbQ}`;#M4{1U;OS4ZF4LU
z#za?Nb$!192Hu0P2a!f&Y5<sA3@-bpx_n(h03+%TCQ56U<n`=x@?yP#<z!h3z0>}8
zr6g+{4k0lD1Lc|$$MqZ8XSm<%?jlqHp@|QLTBWkl_vTpt5<<=NvV++akFNt!hHZ=w
zlC?sni0D5(QKGMfi24+g{E!m^prD`-Svgr;QXcK_4N^7YDb&#gvq)W9O{C`&@p^y$
zDVOw*8nsT3X%+SAsKK$|_VkIF(ik}0y_pM(+v!Pg><X<<eA^FI7_`i|-LIEXN%+8C
zhT`W?p)uF;+$dW@95ij0=qn!^I{`{IU3<;BMVIQv&AL}RzpaIuUwylkyVyLhrT?a7
z+eMHuy0Hl=I#`vx;7t(C?^mZnI<E@dDvyTT3=aZr`FE5gky*)KlI50GLq%1$`@T5s
z`OZ0}+vBqDoYZ6$og>c!D3QnWmfE6+ZOMh^+A>figFeZ2*Gpp-$SlS(sa&C!HM9rf
z0A#CqT{qnO;NLp!1T-JUGFI`!6%RQJGQ%6_IG0$J=x;z;u=3)`A#_9PTfa5R6UX^A
zgH#?$+#1xY{|Z>)5R}%v0Jn68kz1X<T72<o!&;n~)sXXk{R$x*q;uW}ZuQ!jaK;5F
z7CfKA3h)J^0ipAHR=|n!+E&LXfAXqv_n>Nu0gm{P<pJKX@Di^uxe>Us#u}0J5(%>?
zr(;>4QpL^D3QB7?VYp+0es845n+DZc3pRah+$@`pA3terR+VR`)y!X%KeR59kK3VS
zjyl0lP**oDhqu<dN@e6xxs2v)+vx5NTb68ZMx?){{w6cX;Z7EnC%0-ajsI8B(dVW+
zE}OIByGqi!<lK`_&iY^7VS6iIqRpYkd`}&45{<=j&Q5ZffVGNgtt7IVXenL@1qIx0
zIf8k9i}J~L-+=XQjRy<W-C2I~avw#Vzl}8*A;qg!Z~&|Ix;qQOls<}LuRp6Y)%Kk}
zHg`Z<GhqKHIHx*)Xz)oLg@&SVEqw*^x?&0=R1wEQpF~pNxJHJ1QuAMe(-!@KT`St9
zAk(9z(~l%Z@LWA(cstHB=?VRCWR>~ByT9c_aKwUMtCvr`>T*-=py~!N$tQUE%-?54
zw4gr$xK*3s=Ks>qFt|(bjE8l(eUz_WV0-^a15ltEXY`v_yCh~u_sKLnWFM8>pz~xk
zs@qFJojQ26l0D+%mijw|KJPq;HK(Nc_r{oitb!118S?C_o)mj=>~>^dP~}Os(W5R{
z#*&El1Ay_VB2yJHw~(pG;kLG_lEY9NyC)Y*UVV$jwrVrSM`T#)6Z?tiETL}N$>aUU
zYFJveKO!pNp(hqnakH14D-G7ZV0jgvIi(#j;Ej~GOr-EX8F{HJ0Y829Yp=`E`jW=R
z-3l7V52fp&!RSU#O%bF83@_9CDB~!h#W2xqL7sb)*?0|Sgx=~GpjdkFj`R#zPdv$@
z-JE<v3qNA;)-2q|fGKdgCl1Js4UUuDFWFSNhaEZGY*U2#$yZatjk^YS<~|@&qUHCA
zom~$E;$<(*&WKSyGm{rI3oq57@ny-BxmFaBu6T05n2X1!mFVOt+OEOj%pl3Z1o=&K
zNbziJ6ouAw%Dc4T)RqGbZ!mL$93D!18YGr0)%>&(;&+b>9y?BsjK@4Ypd(ph=Ub&X
z8&n+DpQoFEbBtHq$DctbaQcxjOL-^rGmRWouZ{ZlfjdPH%u4Qr6XxICKgIUsTtcO<
z)P~&3lf&z$2M#@vf3xetbrDzdzBMpkg|t~aAQ+~6+{#aA!hJPm6Q&^T2q~bjD=5l#
zc8PAIU}OY3z?1D6<{z4$3BI`5pzM<xz94e+YspR2JF41XM6m;uDqd!Dv}<p~oy*%t
zIY=#WJeh|Doqcp}L!(Y^md7S|AwJGhw_i6Mdk{ZdnD^Rw#-RkgD2%^|S#&PB@=VaN
z8P&>tLbXpl&}q}QYU}lA4kre7A2+Hjb@jSv>&jr?3#F+kWtM-p$AEHwnE4{^E$FyP
zk1XfxyRfF40F$(`+F-s+lz<y;!^&6;X9>>@iqXfOjLf)&a7U)7>k*OH07;mZ^4_s?
z<Z@zR*LNOs7VN#q!^i3Of{mMP!)sy7$^j?QXe)K@84y$PDjcJ~r{~B19ea>Ka|fGy
z{K>&x0>Nx4&~`d_h_yUvwRf023MS()e1dXZWraDWs&{a{igWvZIe%<=Hud=!!Wa$b
zcHucX&u4473ws|jB}QqIlUuIp=*8`(hyYpg=!+-28D@Zxj@avoBx>Y^5Dd`=->9o&
z$<7TtOVjc<dQ`|PD6Shj+9aBv_VU~gySzY$@?v@5XSM!`5jAj<XJC75ljNlnyJx~p
z9$zLNQ!NkBu(+%2$~Ekx^zGU1<}I3!18x}Iaj~PSR27{?{r@O?3#h8vu3ebD*}$eZ
z9n#&>-7V6nfPr)<NQ$7u29Z!CB&3y4L{TXb6r@81B&3w?Qb3eYgfo{u@B6*q_n-5h
zamILt!+N%BueI*`p7WYlWb4zA-#2P}a<0|N83{9aH(K@Vo#UI;t@*1rY}xGB?mC4v
zuQ~M}`NiLJM{2^U=XhPx_Iik3v7j2-_0+vNoF+kB53PuPR24D{xARXvsHpxBCE)^g
z5*xPo*XJ-@HR4<%P0$VVB#@2DOw;1@xsnz-V%MSRx&JCXPM>$^zs=Q|Q2`n{s%U*S
zE_p2d`*$iQfnMdqHGe|NM?Oj<;<DvZm%_4R+f6;CBX)L5Uuo^>5=ITOx%>y)gA!ey
z`JGF%faT+yn(^-SF@no-XtF7Gf*+{;v5Kn(Ucz*g8^%AOwgTMEvc!^~gL<hp(}tAh
ziS`k)^QF+J>O@cuM}xMsXgQYuAN9h2bRNQbQ<#qwmKPdXOG!wl>KV`1KytABkcsUY
z$79sT=vMCWtCA0Y0t7*$RD8ygXx|WTLh-w1&v)j|XSaVIis1izD6XTtbTm#1XYU<_
zXXeW#lCFf-$~{`I@AfsU_awAhc>=hP_dLyXcET6WHh7(VE=e}4Tg~v`h%$pUI%~dW
z=7-^YmP?|sZv41U=o|Rs_J~I|3s}tMHvmJ`{?fAk-?enT2(z_{WiSZAF9^(Xu+_eE
zEaW$BD_!fM_Gu-Hdr<Z+Ruhf2zuHb2-;l9U4P29s5=5@MpDGih=ruODTjJ{~^6yCz
zjvRU}$NVB`q&39ZAof3kQd+Jb^4LdcF4&`Z@YbjKur&<{>JLGh?PrbH-1gmCF2(Lf
z671}Q``1k0NuEKP?unuJGmk*^`niRft~_9UW4a~wOROq`Q?AH@92(St!rw*#zmd4w
zq5bvI(v8bQ@8RUw__p@Q``;buA-^Pe2#1WRgLLw++tLUiqDss|vwIvC4uU#AXgXd2
zd#CTHmNI?>#?P0LYj7i-`*81`raeHrM&CJJ`5EYWYrE0uK=aw^b4nEX;3PJ5>v|2S
ze4!WwmW(nS;lRf_grd4P8+wqNG7w@&xQr+mze8)8GI0Eb2)dkRX|z&MVFh%t#+y*)
z*n!bYUA3PRlBeBXb00Vmpi{fw2o1wA#EwU!`qfF~T4gYZOB56_5;E2+tcN&+CNTlC
zbsKcdo}2NChb+9<x`l=Q$Yo7kK^r8n*#_*lkX-<P1L!=+XcsYjgcH9jiXAAALQZ2Z
zy*WQuf=%Prub}#&@KhPPt<6u3vmZGzXU2N;C)3)(;m<&vd#zh;c;f(ha}_*Cd|IL@
zJKMm(#9Da+`V1fIL#WPuSK=Qt0F*ZZ>CR9Mf;1ADhjegU>(bw=2Q&+YSg}tMEFTz^
zC(WLRn63<tH;~sQcj05_-J_h>4q(y(4JTM8oEc>VG-L$YZlw@V!kZjBI1fHQS0Vc#
z>V$ST{b%s;-c3qYhdn!CA4>h2@WTUW0t=r49M%5-VBBQ0RfOe%h+LI%cGvw1z@@n1
z0L+{EfM<D!|2WY89>HJJukn7?tXc%S1YD(Kn8gqh>f`A}vpAj;3ia}1;A9580Q3!%
zy7_PC0v*#;F0VLWEF%rOrquRaRLNu9lV)8ATyGA*pdzAS;dnhV>4`9=wWn!2J`J_d
zr2wLGt$GOU@;Mm<zCz}_>sdv?=Q^Fgyb_~C*ZJ`;r0D<B>XBc6fyIuDpN84^dUt68
zE-YD}tiAJS{ddc+D=>}0WJb298)1?XHeYwgLfhJN0VhaQdV~@WI4s)xMbIi6c=7<{
zT@b#p&y=SB1t6X7=1xye!wjOekc1~1MmSvR&!8Z_LB_D5)eBHL?ZXXlPD(1Z2sh;K
z6sTE&W>2~;XA<ts0if!g)018RfkTyY9sp_gT;l^#8nwFs^PiL;vN@9#wdMC<fFExY
z6Q^&s(lpB{fEMloZ;;xg^BSdkHeZlndE?)JvO!nnC1QMWLg|(Wqkj~+O)6O}e{cQi
zGW&?yoA1D@%e%j49dHC+vL?~YzjxA&-{@DIrsKr9k2etvk?#&cLvx&G&qc3sD4B4I
zU}vyA0)=ar45`kgfe%@77w`B(dO>Cjbe{g5m#z=%C+{9KHjV3_doF3TLpypZ^-_)B
zW5Z+~ibe2UqI^zync4Km@<5DLop_4ANg;Ib0G2pIv_^I#NnP2e9DBY<+4oHu$wBcv
zA?bUGK@fHB@dSuepa-tF4Q@@Z;oNc)Htm6=oLKtCJncPB4|I$>UD<<4P+niVd0E9|
ziWQ#&#y$l{?adyXBqj0@Dg)Bh_a0A&Y@fiRZaq1w0bRmpWBQZ!I_6IIyDN`8L8oBv
zJr|=Auhagh4Dd|*iL9)0Hr<LX0Db*C#T^7b<sgdzntMZo%q#I^z7lxm&a=%|z-i~p
zbAB0Ua3!w26G#`c)$YI%CezKUlhzil&7(}CK3Gc<iz3JFpXoSO9W`;Jiz|4Li^f+)
zXqR5LnSe$UhY!G`$}P2(7{$+4w*|sr<39HVA)XBlOc@i5xmaJ<Ea+%Pz)m>L-$U=G
zOw?!=)2DPEgmo9SQ_-G!3X>s~CNY9Mr?jO`rKzgmjzyiH>6+HMuAKN5wCZG}#VfBX
zi~3o3d<IMuxOWup(jT>B`$ct$m};}WB4p2xmYFszVE;M;cZ|gXyMef|QKq!Zfj?(x
zlX1R*Jw08<yiE>O(dBy?94jgW7(a|X1#9FVSXuKG2DhIrz4rRy85<(q#$On)1=Eq0
zi$XKAml^Y@g&%}ybHsz-;#aOUwH{X?#(cY-o#0nhNX{@Tov)vdrTx}kuA5AXj#JI0
z0RJvqicenU8@jDV&9@$;&DM>jN{t&nsr%-{XJ3l;nucMNJ-(6qNvJW=CMv=YbvgpZ
zS@P16LR*)r$jkdHUSzfcPVMwkA;7e%epFlJWBzjhLH~^%H?pH$37c|Xow$Xq(ZXM&
zJ<RIK>ZSML2n1*zrm<harXgmQGti*Vt3CBodfz!O3n3Z;bj_XVEoc-K)6mRJ@0*1R
zoeu9+mhyQ?rAJ!4TTN9Uow6Pom7>}-H|zm^aBKj9lW2%qPRO^n`y6k;>~_u|a|dLs
zmIOP`xM@*htT`q^xiiS9X+{Abjo9a`U^c8V==~E8%fK@#2~!C2*tfyqiU;;cX;?NM
zLaMaonzscTGnB5IR7>hjRD1GQ<kSbtdtGC?kGEK!*X2sVj6AIe>XpyYM3I_3ggD<x
zo+I(K<%#|t=d~yn8P^f3eYepn+AwZ>w!`O>3+$6oU1AA>af^{8JRH!OfekWFFY}72
zgG^-nl*Ue<Ei>yFU{TAgcSfj$lF5+K#tqvc=09^A6EySXtpa>Fubtdn5toUv^0XQn
zbNV%;#`V5wXK8|%@W?(?7Ez5%QSr!E+K1%)Zp~~qCZ6lJHPvV%;%TO~1&BGSiG$L3
zqfnbF=3;`bCaI@;-_!yfp$7Nzg}@5x{qZv#^B^6xg$xg$HCs2d_nz*Vc;;Wdwqj3Z
zS#dedG8)neRquo}$WV^7<B%5wC_k4vO+(Or>5sKhx(_$Hrr^ZE+Q+RLU7~1a<60HO
zyQBugkW8r0F6o({;C;_F1sW+_F>4CSHM23(;wtU$6Khvae7_nXzlVvOzG_kUhZi7-
z3Okb<V`Lzx=_z7IU;1Hg=>}oQ(2vjUKgbFj&799zyh`4+0;E#qCy}B@$7z+K%ch$v
zU3q5=--J_J1*ZR2amic?-WB`Yi2A4>t5WnT)nV}Vq+<H-hoq`K_syNMTlCpeF8c@V
zZa5T?GAAu|CeOkEMB&X>;~Lc1$Jy6ZdW5Iq=o2iD0Z+h#K9}fol*!1~X{QE=(U6}M
zI+(KfFo!dXkP{DM^op?OYhDz+HB0s!CraQfYeVOAlloT3M>1p4LW9J18S+eb)Km+D
z4q^JwR_Qs<)ZoS8JF#*4KY%(N)$SaFKOV$#Uxby6Q`_+D^{PwNNh%_Bc~kS6VnNp(
zbq}}Qn>W5AL$L^8Dw5<j8oTh2%>4A4y?VpKov-4l(0FeNu<pe=7@ix5B)X)J>bs7+
z4+ZtT?tYwDcEcvG@<6ag?MXm&R@~*7EM=bSOT#OkVi?EF=Qb<lF3qB?wVHsC!Ib6F
z%vq>HMi>Q}PVl_e0*Mug>|?~^BGUY!%6g|+ZL;wrUbh=YZn2uXAudaP9~jf0Mz|I0
zv;Q{+Q-nF%bG(CxOY-i?Qk&#+?()EuDZ2#xcY9zi^KoHaR$`nSEu*m5CAlkT8p^fb
z_$*R7ldxA?2`dp8OjySD+;<D<$T=VDC;u)=k)PnK(IYhX8P$QK4UA9gC#P=XuC7T^
zACVV(cKh+SBD&r>!BDdO3DhX-ZYc3!INT~MCi3x528T!Q{1Wl2FK@3h=(Akp`FyjV
zGWVt^s@w1E=GCHaL0!@-OS|i>%f_%X$1*en>GnFMCiD`|H^_it6R#4NWPYRk{y<%a
zw(u<%<oPmOodc*%Rhni6arMc@ME*?qBEUPyppXCT3I?(AM+VPk2MRcT2>;$^Fd(v4
z+M8DOHP~A6Y8`ENMTcN#+E<VCb4?j#Mt|BBW9FM5J%t`^@q6=^1Z!OUh<wqF@Pq^a
zhmN)KQdcx~NayGIH04gcp)u^L?z}$vFgGrKLFM`Fi^@+E+Fn?kl=uUEC!Iw{Nj9RU
zVL*7{Rm3pq7~4u*(%7-h&{a?0C0^vOpm-b2{5xZPP`|;C_l9#;3ye8m1ClRL50gho
zKgpJLc+jJ^b9RwmSMZ^0v(;zg$R`e;_mJJT1r5Y#t)zS~Tg_+zCQGHX!ZO(NS#tNq
zd%d-x?fYCIr`X{rlN-ma6FmmZiypSplap6h#oZEadGN^7^}YcVxau2pjI4)U+&|1R
zrHO7E*<I;GrVs!)jVGaf)hHUgDn@eP0WF*Tj|b#u?K--b;iEo#M_r#I3v+4&s@qp;
zapj+T3@*I7++K(m-I`ysi5B6zMC2gsYh&{|&`Ts!%87Vg{9~DY6WT3T>fu1Bm*fw{
z^=Hf%seH;Qy6+|`c5=jJ4gzC>>j5c@Yil7Rh8TrL4{ZsAwlP@SV`5b&`uCv9*d*+F
z+0Fe;MrFd<E|_2XTHXMrw7R(8NEKQmsE&+RJ;nB<s^{5`2fe@w*gQH?LgE$|*M`n}
zdX0AaMrB^hfogs>6rNFYH_FZ+J{Z$AI*;d{Zh~nifg-hq>1L$i4k8${G>-ZEqO0>a
z9z{kp4bWKy{8|r@H4x7EVQ?Y!U48fkkV8Nx(*yN<Y*g;VHygp6CYZS<wYe2+^_@Q=
zZILgrc@%l|uYCRJ4`B5bYygaG9Q6$6-<}7_0o4D;*1*#oXyGtZGf?$@&uj7lfOz;d
zsap3$Y<J?t8Tj<~SI;2w&m%t&SCMmitj)bitgX*K*S!h(xm*l&PqP!_Q-El|$jBA=
z&nIBC;#CMu;PqN|B>e^KBd&`yfk@2${a*(Fo+L6%|5e%I<`E}O-u`qr4GJE)OH7x6
z_O^%Q;@ZiQBCH~*VPfMk9U$RGaM5Y<KSvzBkEioB`@p{j$_f&h)2H5-?jI|}YU&C(
zqo6B+I9)f<laE^qnOGu(qpJ{WC%AAG7`Zq<*VD&$tD+;3j`B&^{-%Ld5Da+%C)EI(
zTccDVt=|H4;_B~8Z7ZXRWJo2X_CSm+cX1Y~@w>uS3o!kV&Rlz7^$=oY7JB!-c&<Wt
zWT6w%T3+-n;Muagv3P+jCcZ6L$-`XI@`+T_%s%uGY#L!9hUEd&ah2fA^)=v@1)iHO
z`7`)S)GcIPloUc}R9{}i5|8}6FDOqk{}xySt`>{ULl2{nf^9cd^K`=&m_v~!l2+|}
zslh!(Oa-?kw=vcBk9_kV_E-UX|Fo%vqM5|^hM-Hd3gzH&xvijKwCer<V!6Tr(ueNw
zzV^>Vd`8a(epXAf0LC8(XLu8QJ*A>x8XI$EP(8~dc2O*G94rtSE46;zH(pI<wL3}R
z*V{~pmP}rQiDjqzX9w}v83KP2ovqP^aE&p;S??uite1Q93D?El747`q@-Ja+sFn<Q
z9(GhvyWB>4){l$_pBkSKoy?zcI~t!6>ldS&qi98XA;YurK2uz|pa3moO5Ebuq1hA=
zwUUl>PPTpxFL|6mV_~ZGZIZ-rFg<E|O>SnTTn=48;f@Wk8^{<$asv8~slGI{KZW0{
zOQW%O#G9ZJ7ePjwxg*H&WbL*;LmHcMK&<lZ0(H`;pCeplRDi;&%n;9B!56E@`71>d
z($=jqW9kN{6*s^_OB^`8x8|S|Y@W0F01{`DPw-f8H4A;=z0%07afix|*XE^_qO(}f
z`(rY5U?-C*er9NHGHCX8-GvQO;q<T5&|U-=1Iw$JA))9p=_!gjfp5w|Fcssy&jO<u
z{-*}~c3RAB$c?2<CchmABVCo>*oP+@zWu4t9%1@_=1u~9kVu*{I2y4fwhKPdUmW5|
zi`tpTxZmoDbQ9h>@7h|Uf3qGGHmBnDKWYHK0Xj=g9*<K23w<mPet>Oa`X`}8;|K%V
z3Af$RVg}jHq`U!94ibR~^E__E8nEukcAv41s#cIY?Tci$m{9YFKMX0Kzgt&|&)UHd
z_grfRb<PcJ`x@tc!dMqA!0hl&>T5PRKsYN3j&K<4e*F%Zs(@ge0V0O6>c})f8`#Cf
zbmB3CcjDZ5s?8KB!o4(88`n0Z>4@yVtMJXVS{#w)y`<E7W(=4#0Y6|qYq+DwW26x9
zE)-oiyYjEY#@nSc)+UKb^i;fBU`m76XL)B7_}3`Ln06RKU+bbUVfVg;0i`OiIfqQc
z#2k;66*g0;l1{X{W2I0Y#pIl=TVj`zXHq`I6pJPLtXAS$Fc{QAYBGE8BPHF>_qzQ#
zMi!D3_$G3U|2$MCaC_Po3!NSHrqXG5e4ms7Va;mPJFJERpD;almLIaP%R)2MGIMXI
zv+CU}%`I%yzq|lOX;M&u6hSVnQYcMUc~5upt(zOYI@G?M<U*lS^-Vc54<dBzq&?r(
z2)Mr=|M_3t^U{g4r0(gnH-~xCVm|o6I-a)kww0=ocH`>lh}vRd@{MnACXc95x`oLU
z6lgwAmCRMRal`&o&P?B1oyS0aVbk|uXOIlrdg(dp_$*7&m*={p$2;y1a{c=sv#s_&
zr8seR3k-yOzGM(?uFP4|^PVo93c6u=c1!L7U)YT&VZ>Mc2_iR8N>q%IbMSOaTl2K#
z@F!m?In#D`H*yyQe$<%&leqpaDvxWV#GzRFu3ws+7YTcKGuXGtoZqMTOq^Su;HH`M
zskwPY<f_YKy@J+9CRO%sI&TbPy6ztR$#z`ufk9&z@<fwpF_&H~3mjp21m33r;AP3U
znSM`pLgN`ZKMT8CgTKO1jl6Dl;}XqAMv?Zmjkd_N*rmNBt&9-o!-eJ~$8E#{PN3W9
zh`7-zKTSR{n*5B|Q;Z+#+6wq)>(}>kJWNB7x1bNV9{8Oh9IRk+(D-O?Dy96LQ0nef
zl7ad+bP+L(>8He#Zeu4*&B>S*F0oOi3m9kMbI?Du$u_Q%IXGjut$kxJh)%*}pF-mS
z8#1spWJhQ2<|eZDz!pDRVDPf%Y;#sh8n?(2n;%)|<<*3{27>Kq=Z9yM+OH1PPz(~j
zGYFBN?u`-54V5)0k$+0!l6pMSmn=7gULhwssFOUYvhXfbD1K>=SId6-wk%4R;A*O;
zy$`9kdXTKHQd>w8D!*y5O1Er~XUHwEW@sTwu)ki1$8-$Tv%&m2NzNig6btFoX`+I0
zNF&D+A_w4ak~!M7$s=2LdeYN<BxeKTEwB2rm)?H#!~CxFeF|xGwSi=wfj%C7c(J7&
zfZC0EIWI<cM}LF!PD;Mc;{2L87l-MNA92sgQ-d1g-{>_B2;4}v>AnRv2T93-@lPP#
zry_3kCk|NtQZpr&Pma_mQX(EgdXwNEO<}a$=;0ag0p^PzBxWpQtGMO(tpAUGt4JWi
zc)zEEitGykM{LhGEYWUR{RgwEb1T_swr`3)j0qWmfg|*>^P}|TA}K6(4ynyr%57_s
zY+FAa+HcsA+L4-sr6%(vyS+xgK<7|xNP;WKwczG&L2WgO^`R%Ix=HjX@)|M+wi4&5
zNbf05B-fMy3{|}q%e|3pPo1`b1|Ymnl6>Vl(BpWp_5(|v(K=@d75$qBu(f@22808O
zyplwc+fJdkJ`%7*`07v9;m#3MFX?=e9?I@<d}~r^66J~+$}hh{=o@FHL^*kzd-&T>
z>=$C13`wy7^<O4Ze%f~QcxuKF_CC5+5}(=YDD-^OXR!OFZ9hq?V$JQfw}jI&c2rk)
zbbM`Of2(RLYEpaq@e*me>_gM)&VsuiZE3I^sMKm{DWS2|3QvE#&YLt!il2X}_LN@8
z7H)Jn{l2#A#aB=3Wgj%`;p*hvi61w}%gr_4Cw2&}i|n#%+TfQXk`FdM-{LRaaw6o8
z8K&bGEW8q1a`Oq!U-5_0Mu|hHgT9ApF)iZ2soQ_*3>=(q`RwCAn@|UF<#WYIJHKLY
z9wT6B+TYvC-A2a}Z&1a`Yt_W#e~?OFEB?fBW`_Iq*JXLub+o6jJxT2Ed{N5s9s>z{
zCgD%CqHXI%!si0qa<i^+Hq`PdFBv_p`Y!2BMWTy-GbfI@YdgjnIaHcyOI^!fr!{Rd
zIy>-&w~z`PY@L(w7JV;O8uyEeU%zB4*9(hDF5x~e<2HIypNaj7>Voe=NWeG6**=fR
zg;#@DTP1`CXs7gu&JZk$O?5|v2FZ|D+Zdv0RGW985We%2N>#UDY52Ytp`B}nl|<_s
zYTLTK4dtj7rz1~=i}IFY`qq=Gp1CO239vZnSVfN$73yUS@xL=ug;#_2!&z|<@9@PC
zJX=pntyE}EP*9Hue%>r-WoT_PUoe`GfWJ(6d(!@_Y$Zi|Ya^pTH}CEGyRp<Bx<5Wb
z-A>($xfJDVuy#UJ^Jb!mIwOR*mN7&N+L(RA>Alq0KM}7q?NrY6tzA>u-#ew6mNv+%
zwFq~e{3DmI4*q9>Oiq3!U-GjuT|C7RCPNZQpEioL9ohhrnz1m6x()4TJUN(4REG9+
z5wB3|Qaz7TsVGpT7xqY|o%U@dB@T2<sE(YEV0lQ)gZAaY@6*dj(ANY#o3`UfEZbjm
zD|ia~s3rBYwcYhuhmHpB8RD=9C1hq&%r~ZYI2t7P1LzK-7g@PC71L^mg3*5BehzdB
zs5nV8<=KytbZU!tm^n)_pNx@PJG>(abYz(tN!+39<xX83Z^0xQKZ!A130`nK-x$R+
z%AG)<p4N5bI1qn;F-@5_teOxUJL{dHM#v^^brMbpve2#si>2dF@`bJ#p1zx8#%cbc
zC@gH>H_MRIv#40ediewoqV6&LbOCe{T$z}l_YR^*+AcFzd9GCsdY>xlQRj9qCJ-T1
zRX0(8cr}QVkG`P%wnC<UD8Z(d?s@f~Nv$Tt=|U_c_|}ZyM7^hIUHfL0>s?|6ZFw2T
zNOWDfo+`s}*Ik>nlZs5bcU2ti_09;SV~$E}Xhk=#9{*0FIy?U$Di<eNaj-~~nw#Wv
z`FfImaN9HyKc7`h1a~8)=9=-A)b(b)bfs2O?BRKVH+PlVG}x7pG_jbul!ZVZ3veEo
zxJ08-zQ>YG(H1MBedf_C9FMGyfIN-piHF#rC<2Yy)}~;f^aXoPt1%@8*krgBc#lxs
zXUE@<z}M_6nB=MML7%p0kp0*uwm{Z^%4<3ff8#eaIPsUP7fvr8)8aZZ$67=AFs%A@
zaQ_XVem8e)xrU|8iKoS64ZzA#^%|9#x)Ps}toyui`RKLXK9#qhj?@_o5%Z2Qs~Xpy
zslhx?sVhiY@Qp7tT9IN6$sr5&(iCZ3EBY~T;dCOD+RGf%x3LA_<}F*rhjX?kY5Jn8
zQFyk%ic|oXa^(qT=U6u~v(#iH-RN@SRCnYrHC56por-lE@19E|zojMLHU4xc?bC+s
z?NrS8&C-w2368qL$5NHcl`z;9LW_49R_+|%W{zVp6Q}#d*Nh^BUYabdT&Zss8aMm{
zxYb%=$m9+MqAbH9Y@-CD<|gZFGkE4D+P#Z5lKjVsbVe{vi%Accs4BFbC_B>5FQz&H
zcz_>H*2mDs`)0nw#r=pa`L09tJ?vUdiQ(?%?5gcH??%ygrNYWC0m*B>_D>fy=eAK*
z7C}+=Y;9S`@Z|n4u63#_GqaAX+~PmyB@4IGzSi_V#_b8bx?K7|YQgL!?HK}b0$G;e
zMbhv!zt3D1z878chS(ji%^3IcpRit{Ri+xV4yw>}@Bt2Mt6?DHmY(v{y>EJpg#pq}
z8U!pJ4+$d`!v+^Boh*(9CmA?VRVU1K-}}*ae)dgTk5damU_AC|MvZIVgx`^K<86GJ
zXEO#v1Kre7x~cB?YZ**dl?w(m!|q2vtMevl3uoP+JJ9V-)h}&)EqA*6)K~lCBHZW$
z{>F1ACn5ugFzz3Ie@^~=cpw!`=(_7^#d~o|Lo|mtvBw96YIF2dtJ9u$P9?DF(seV`
z)TA@Cu%M3H9l3mERsAUx;F2)Tx<$)#2aUmae*O-6^kLFsJ-5~;shNi8L%IslWXdPv
zf9Prve6;q|jD)C2D#!hMt(S@hWsgz{aTcs+fk(zI0_ApQfkxCI0X?oPx2bv$hCnZz
z%dWIlGP#Vb<bap%UItVCdHjs52yt}=Ez-&NS}ZP@-AKL=KOu-`BK*iaYS45EJ#6Ss
zw^3uJV0nF~gQ1ASNFP@;`q2iNiE|NRV%dGbZNPOZ62sVb6ZO<@wpKEt`?0VFjn4$f
z$QTc=Ce5%CLC$60SOpV0tARZ2;>L&n7!(;wqE0X;*-WnyaQ5$}YPm@kJk*K2_Fy3g
zAv*P&pGL^Imazmwc-JFC*PR}E_b}Wix%Nl`cp80y)l#41WFloElNif$tCb!)gO8^d
z6)9aC`*1!;$|yfvK|~}7-~zreiehsqQ~HBv<1urIT>9oB0{<eCCUKz!jN4(JW;|xz
z9C||`e)tlvqUKS&B9>m$YtWYBZVVgkPWF@oMs!*y=P3j*=~B<nmudly@rNpCs0BEp
zVbihUq<=WdjRP2CtiqTjE-*D?LXlPT1)pvj!SHomO}@ZkU?9fD<DbjTcueCPsp6-+
zAPkQ`jKoFp21tEKH{L}*AwQ!}`p+lhB5?50SX6<|zkdUa!X$d3ei5U>p24qbHD8|{
z7b!AR{3!WGoOI^4j5fNCj#H|Lf}0uuyr`)uNcq!YnFf`KzA@Ja8}3v@9}G5|%*K=~
z(tI3h;4wWw!Wm+JQQ#c`3JdLO-0tP0?NpC!z<4<k(NPuunKdEXODP81+kHW!NOrvy
zSjBEzoFC|;emRzKTH?*m9KS2U->SgfWSCvaEUfsXSqIf00kL1ppl0ItWcF_4PW!7D
zu}mkpa0*GKLJq%lJaHZ)obmlUg(3g`dd{?p^Ql(vU<O`KvfGhhhp${>!tHX6JBq+F
z{QDmPAYV)Je{Vgqis?{jMlM$KQZbG0%%31hoFYa=)<RY##r2YRX`+a3=akn9z&p_N
zBB9CWAD<|x>`{IKuZQWP*plURIzJebBkF`&KNrwrkSdRWZ4Ds)U&deG)meQo4-Df~
z=)K8oxzo&L+WIf=8nuoB<$$V9Lba_LoQ*)U<l38Y^&CWa1SRp$NT&h76fI%P5s;=&
zS`}r&LU|hmRp9ZmT}pclat<Pk?5zv``6^uKOg>4(iOsI$J+h1%Ut%`uBQ>l;UQltW
zv;7k8!o==>1JFy7E2L(7sgH1&fRLEN2Bxr}1N)#I3ghjZnO~4+)EaRdu441>W^9Ce
z#cnXL&?IO<>arWyrc#<O?cU49(`^1{zzsmmCa`$*B3=Rrq1p!X(r+-sRSemc9Df3{
z%0A}}`r{f%_!(=nurZ)=8}tM4VZdgwnp$r-1R2Kx;E<3Xsm|BXNhM&K)5&ka+FzoJ
z(>y5pqY%c#6Ex(ZjwDB?5TKt;%0IJaCer?&7;krlfdEhYr5g3~(+majZCA4os$jGX
z*jKevYj`*0A|{-B3(hZy+m~5+W}<9_PW6)S`~E+ZWqxH7U>7F*=V&?XV9%|t1$VZc
zv6}RAJrEmcxZ7az+*JY}I(@NS*5o@2>jNeK6+k|6nwKeJ0A-Oc2I;MrDYuEHCcqI)
z89M9XF(2;k{9-#!?Q-#N6a4SD?axmp`0<PcM}9n_$e<?oQ>~u4()*pc7XL;06#)dl
zk;j0!ysOs4o6u|55c{6$Wn_*jlnxF`6ewgG?C6h`7yv89{mu-wnLc@Ny)-_9P+A0T
zzady4?qpX40jEG9UF$=sVjRvVtzI|InFjP>6`fDc0aCh4%XQ&&$Px1|JX2EWSBq-$
z+$#ylqO8%!{|+8Ql`(wiuoW&p*Of2xSTvDv`{vsYB7#ZN$!GFp@s+gM2{v(RNBAR}
z(XQk{IzH*sIPM$WzNaq9NQL+WChjk(RWXUsSR?F7hUX$o&_uJMwgQY#Z(SIP!R)_j
zA5vRy$(C?BxBW{ztGw?qV)%ER;MZ0NNAMg?u_eJG0FYmV7_yuX?}-JFd6pygc#M9t
zyTG|`3u$}=K&WDCTd7)n#{HPxQ$Be&t*CQh3M&EYx9FB35%R+S=YL!t7#xjVK%d_E
z&>svOm@X3U@J{mhhnlCyVa3vJV`W^s%7```?FS?TmN*0mIYTxgo^Aom)Yywx|C|#`
z+QlQ83rzRBO1COO=dds7c+5rM4v0tMik?>gw|@vZuUaqwU~jb_@wUI%c=IX{*<h9b
zLW0mBh)u5~R)|nx%!iQ4G*312c<j}w036=02|~s}`haf^Pc>2;gdJFypbyHe@4Y%U
zjNexV$@K`dC8u@?RIrXL`p@?uC!ysDPXu!<%(x%yiZxz#{C+$@{{Ns?gKa<%FVg@l
zcUmffTfae%T6Wniz9rzvHIT-T#`-{N3M3;8w*WE9?--0Qb}xa#GLHz{C7EVsx#~=Z
z_M<prIewMFa0iBN+RPE*C{>#K-w>5Fu*+kH7rK-H6jui#7WllKg%n<pf11E`!H_$0
zG_o?jKTdGNJBBgN+XiX2Vm0@%8&j!Xf%_Z8FI@l9@U+$ck2JjJHX_d92N0qY>%tuh
zV~89yP7L22oO)=cyv*lFSq*YM$P5yD@5JV$e~Ts|IT|jB>HQ66rl|y)mk@;&n~itD
zNYO8EgY?3&R(07029-rur0LkfqebBUGVQ;PhCfFw?DvLfl>CYl4K5=O297kxQSzoX
zFL{qLEBbK##BS2zdW9}fNs(cN_Cb?0Z8r@PhPRnAg6K%FOhsq`J-GN|Q%?jACl3cs
zWq(D(`XZpkKDJT7Tg^Y<t_}NKW9LABI?huRAWo5I7}@u}ZL7)e<lb{+#xlR2{uxqh
z#~p?NNt3}?1nAGfwQw7>9f+U+CJr>S1heNT(hy}@9r%$mh@4OaCsW|?0!XzFc1(@j
z6DF@M8-OHg^t{3@&KYCt_J|c;OjQG}M;fDfgzTkM|6IVt|L+A5HdWxi{Dg4^g91ts
zYyVoky^p_?d58`f*>FXE&4r-}=|zQA$~LDr6%$WNho@@K@U7}tQvRue-eER=x6r-c
zYm%c0OSCvDtIoRpb^Kxvtryn(*UozQ;x5TfXGvFg{P<h(RgcYh?fWnw!Rb2<*@FH+
ziujsgji(z+3gY0yzj=uv%;+CGQ&Tjv02DTe)vFLqkh!L&bugpsg^!QG!0&(-r^rnr
z#(va|0o7784I5W1ejRJ?)%k?rbciR{{W=449^j7E@7PIq%Zx-ZHQR1Nv-M#~(hA8<
zT90176f=JL@>`yv@vkNF$EQ-QOsD1BR|XgU89KrC4C2ydnwm(9v_GG{hy|xwcENT<
zGAIT8VQ96?X%V~I-XN8nv^Uk!>7RQlMwd$-tvj*6VOQaN%(D$468_zWxK1G(p^?!%
z@@oh#=<SdI7r9)83uy45g4hz0`ACoseLa8p^DYZhLc->exdExipCkD2YUr0MFgO6i
z$?G}fE+4@cEltRlA`m7vIXsbf*>@p+1pEBleN9wzD4p^}pe+5diW1%fWipQ(uc~5s
z4S162-S@th|6kGqQ*?#gzlz^T9Rglt&_y=C2Rf{frbr?i#(<^Xi0NsCj6r(RF(!5x
zw<BdpR}e+>7?<Mgu;IQLV!DCEim_iAi@1iKaYf{Bh_M0pi)bqLSN@Bpmz?&!Gc9P3
z9SKII%e7KWB2?tpZrxdJxru^s$_JX?36BA_O&0n@1pvV?_efW+_qe-y7(9Ued&Blk
zz?JwQPveIjatHf)8s9wzJ|heo1>$jj5m4e)yl1sU1oN)`gU%Jq*^&<0hA8%%n*F3)
zT|5Eu(xqTw(9bdxbR&Q?@d)IfJO566O*wZqXugQ=r!+{aD)^<08@&WMl=lRX<PfNn
z%H96U)3gk_gEFC1qz;VGq<I8O(WFBI(mNdNkuYo^!rj+@)Ko-_5+f?3QUnLPXRp61
z>(_y0PLglsCnS%;Ty|;rIcFFT>0E>h%0>O>5-*12p$||)ff`{NwE-c8ZF5n>be2!4
z{weFZ<PanzYwMW>KxCMEjc`K`+c!S|Tg^sAY6^C5ATk}@<sHWU*2L?njO%Vqj*h~$
zfN@<wOOZv46e$~JQX^%tQq^S8Jl}1>NI(#>jOplT?&UJGX0XLm#ja|-0&|(DeJ&ci
zMBx(ee|A2Q_kmyEF6I4c_-BhS5uioO@~tR-1(UK=<}%{tV03#LK_IG~B!3x+!!e3%
zQ~gT%1sgBh#1Spnqq=1*Ep@EyG@;@rE|Od{e8oDeC4>n2&r7d~5ku{XRsC}#?P(i{
zuK`@+>N(VLX4l+8N-~1F*-Th)0Ause_2(!1^X}T-Gs1*7m_l$oOe4@dK<UCPS1dhe
zlJxBn@ji^afbP$yN+LH^nY0dNbROtBo7x}s5hW66u>Kt1=&O3AZribVo|<ZM3<^X7
z4whOl;rN&G_N+i}jjz+zQpL)c*gb@w5O^t9eDwCI07_tl?m<<d^g$$758ay9=}0MM
z_x1H~D(Lv^fFcZ1zi>!~G>B)`Qbp4_UXO78F3@@s<)A`(rgwXn#A6u545Tk1`in>Z
zDztqQ-1dVGbp!lA_>wUJgnW0ZKS8vDC*{8ykqrgZgiI3EhC@vC*;+rSxHD9P{dSY!
zIm3uhLAIQf%E0~l2KpUMgd)vUbZer32!T5tGYX{u&M8)_A4^jD7~@X$HiU<-1cQpD
zq#DA1K;7sGdy)RCMex7Qvj{rc>Nf77KfAId&<IyL#C57{&pPznb_^8B_Imy2EJ{@}
z`Gdj4Ss|to--K1MsbaSML5!Wc{0p>mjwkfa5*DjmHy&PSBLA)Oo+GM6;xB!@NKf;5
zpGUIA8`y0VNtaAMb)?*_d-nG;>E-|1@$xTaEd|n6nf|-2`p=*2YQP(~r2e13_3xkm
zzx<h@2O3npBs7|qtP#ZjG6_25PMXyq{r!h8tVz>YYkj}Huqjp(kA;CJ;$MwK=2;Z9
zLOyBE_vAq!<^pQ~$pgD;_VM2(fij{K4JAW<BccY!wVkidlf;eot);nsav3(}<Tqb{
zT|UqP8#0(n{vd~=`RS)axAh*vr>r}~3Fo_0h_g<x3hQBoEsTawL>Fo6Ixkq#xL~=Z
zT^M;RjFKNwuvi*qp0pMsD~azejNbWmb>j6z=m#aQ;sN!=waT3<hvn;)Lsy3?^M2T#
z*->ryK|unv@T69+m*eTP{j^~`FWV{g?~Mn3ehQohh){BjXpcJo9{o43($Ew#M&It8
zElvBIx>zDm28<90V7dx?1|%2>y02oB?Di+Wz>0bSvSE)uK2r~Dn)AO_{X5vN04x{)
zL?px$9XyqiPoz~z|ILn^y8{W0XCq(O{F5yTPRrVa<cwm>szH*J$o>ek$Zx<LcxqSw
z6P&|;pWpj-%?X5%Jn2ovz>tQ)@D9wfn8Z)(it#qA&j0v(wVmW}wZZvoa(`kl={ubw
zz=#y#WFVv&r-Qv->Ef1s@TIW?sUr}Ml4D($5hrm-Bhh?$2yr`oA_qV$FL*3*{>^@l
ze@_XJOMq|=yHpp&dVdF4N&#U53dDh&Ft8qNv5jBn!uWuoRR4x<92_in>>zclsu)~f
zfMoX-ln)~<l2ITjNiRXc+S3VDhDr;*i(V$sEZ#!pbyv)`0<5+n$M7d?)=IEn2}tY!
zX`h;1xe6Mv*^WfFLtD4pHZ|I4#ije#VaA9{lLSROovC6zP~kzqaoDXeX?(I!$CoyM
zXKA7y$&2p!80|cJWKeB;<oXh^0Q7!Ti`(myhma~Bge-S`cx;=9-CIk3l(n#_m)hU-
z&Bg>W&tV%A86^0aNE;u{LawX!)Y+1hZ{ICt13l~G@`kZ{9n#cNs%4*E`m`5cUi{_L
ztIHQZ^~?IcHKPtE+XoECz4t4$DP~}d0E74%pkWV#iNrJk8)6INkpd2{dypW&2HwY%
zyXUTqr_qt}_ymC8=h*_7Ko{!YMwA$Ex#}+O=(oK`D{iP(B^`6yMS_MQQh;1M#I7eC
z_y{Zs6>lxH31!zxJ_BFNz@7JT7kIh~r@N!?zJ?U7k}W8#yM^<|kq0D02x`;3UHBb#
zSlAf38WU5cuQjue__T0-etJCYZ0t&Ye5goimv_)1iGG}hQ9C<KbC+?PwNQ(<94!5+
zUCf(Zfii{vnh#ghCJ@lL^mp%rwoF%Tgc>a)ChOiypnt)TV+M<HR7|}tqIWmJ#5`ui
z<e$*xxoG^MTgs8g=_Ppdp4im+{=IzyvW0oNgUVw;5x!?-&fluuv@l?2FwK8qR+x3_
z;tBNAQOK^rcUze%-PoD#qQX&yLR};#>jRBe<sKj{=3;7JTDT`h2EMsbQ+iy*;Evws
zXQSE-T`4JsEG+@g3RjdUlWvS79cSown&-cTqo_Ay;qGgQ-1BV3j?`WleC$J(#{Jy~
zV!FzXk>`wPouL2C6??Ia#StTWaQGX5hC+J3!IRcEWX}mQ_igkaKU#i_w5TQQ4JjFT
z5CuU~R&*$@-}sUA#4gw*=HElB$7f*Yd&?v3NRgg4JM}#H#32v{8xL)hFt;1kV|uc;
z8zvDf+cj_xVC8Ez6R}-vJe>ZCq#@Ira}g0SYvH~k-2d<Oi*nZjQo2SsPY;9Comg#a
zIh~VglQ{3n`kHe}K3N@~8!FN#mhrNz&=f492ZyS^>DBi>EW#L#Ps53>(^%!+1j%$M
z#bni1W*VO9TL2xGnqw<gUleyIwjvjtkn5(Yvrjw()>4X~nQKZ;E0f_y|3P<ydih04
zN!`fD(UtBS8qh8)j0X&Wic~AtV?yQ)UMIDse!uw5JsgYhiAUyZhEYCO(<hs5S}@RS
zo1wm{rRq#Z5mG7Q8?_W~mFWTfOotUmJ2$7I@R<JD%*($!CMlE9oef%X^MAr+?`Yxg
zZdk}MU(~dY95LyyD_=AT`dCV7Fd70~h9Qyw*kz8()sE~rbVzwQ&zArTxr`<BsBJ}5
zbU;<@Q%pVc?%BXmm+uDlh8k)_;HTAGL;TI{4NnWnh?!G13LU>)?5_poDQs^HM|kgl
z2oDf<)pV95+B!N0Y$9EKp?h@8TOVFU$NNsv&GXT83*cVM__LA)(pGW_X)Z-9>3ila
zt$Y`b9XHFmq7t&GpC)~_)8y|aP0Meu5vhCZbOL;_jq_;ma7{dAn`jdC@)m$uE^gp{
za=4oFWo#<<3wd~`wo_cYT@B7#@nvPv;kya&MHwO6BFpla01nj5#?9S^3fE8eSyy`=
zkltN3Y-eW?xwl#NIhd#M$hE6IGA5d9gCj*A-5fvBl2Tf0`R<kQb=D5Sd6~ru_bcB`
z%kzN3;}tg>oImAIMM0iBbqZDV?>_zllZB&P-5IgqbT0M25tJtx=kFul2Rge{!P;3|
zM5D3TI@l(RJ2cDi`6L0UQ!TAz<uCYwj<`PJSDp5-m_Q)l%~U%}SyDbu+1zurwM+w^
z<>5_^zuC%M9q4wfpYdWG@+zy7OMc80z;R9xTQMk`4uob=6ZX+r%DkIyA-Bnj?CKo?
z$-le6zE_%=Ogx$7m$7PT-Cr^%)IWLi93{doNwyq<A#dTW{*t^1Vo(G*P*1jIJ5B;S
zkXmW^gqcEE7;`I4b4;ML_xdOcLz<FSnFqM!<Wp-0$m$9eaFCU!P4DQt)C>WaEEBg0
zJaLVgSYGS}pG(o&1^uyVH0t(37%M`_YiJpwvV~?q(HnF9R6y$W(=1ffO7R?p<FH9A
z+1sS4XC(|*I>H^}#q9568nzzgfXR;J3$17?M+N^vqw@LeQ-2p~Iec;2RSys3i|eBV
zH108N#YIcFb-ir)T=&G8;fi|xhi$6#jw7yFw?ZJM4T;jV_yomG<}dNEa!7q7W|;cz
zweQJc!hs}UB_tDxDRfGr&<d(rk?8yw@>Qk^8v-pM4mI&$S0QHr?fuh_ECH-dN75f%
zOxdL{Zbqdvhs&Dz{!H*|qhmzodU6^zmQzYQ;*gA4Mbi_Bes)~Z)!+5M)cL`asP1#I
z_aPu4B?M^>KTyObk~R3QfOLm+t#)!%IQ!zQ?we^vKiN2s5|!)CD}Ih@usg-+EYX7d
zX8(mSJ!tCeRTR_~i(vFoKuC4B#~huVy7BWo=QHge5SYu?Od0TC+T4aa@)%ugqIn7!
zO2iyC$|Tmtipy}HyeZyW^>^9ViY9H?;{GiA&MT;c@va?Z-7n5Vw?CX^v5T=h#wtFI
zyUqCNL_A@!Qq$^BNy=-Atx;RwQx|aEd<oK4p&2B3^P}4DWBhCe60A$R=Jk)dt5?|t
zsj&9TR=GDyuBtruStiE~MwKit*O}~id@?MRG0;igkIxby)UyzfiWHK0GJ3;fF35#7
zP@4Lz4Qb-&GV?JG0q;eKykRpEU@xZ16y-c8%1K9hILcSosHm9vT#pl%^}J8Gv}L#<
z{Lnq6Phx+rS>OrjDPXtL?t-E7Jr0zG{HOl?F893{ohVYnjahEO2IF?BsFy$YDI`Dz
zcgoB@Jm&?q64W_o8POC(mh``=+#~3$$*i3Z4%I_{Mh(&_8u3Lk?N0Xs@O7+_C#N4U
zL)dJSwH5Db5~$c_OFGYH*JysfM?5d%z@2=<CzXLM_|Pm&87-O5dxdG+&1M)+F|AXa
z!JDXhr;jW^IriP6Si0-fN=ODqYTvrWVn|MLA;YP_6T>PL70fuA0!Q+EtALPX;K0A0
z3is6E=v?$;7ZGIr^0l1rw}{8LQ#>tMqH}k2!c^60FNRz-i1D5Jc+YlH$MiJ2XMbRU
zQmd?Yk;jBN>p=AY-XVqBs-#byROV6T+upB_1m&A<jC`F8c=gc6?jEf)mtbfyi=&B^
zAjMJYTU}txFR80aPdjv*6NEZ~e#$S%Omy`jX{lfi_~~W$Zrw?+is?$DR>*Fc{HR*a
zU<hTw<Cn9;1)N$R(|dfl!w90vrPnC3#bP^+Vr)5XR;D~7p&*&kd0RfZt9RqPdRjh-
z)*5#|KsMPL;p&qP8Ydkow4+t~1P*O!#0sBlRS6p5>Ti;fsuVJ7r7n8Y1w>-Y?=*C?
zxQsj#(Q|7P2;3L$NOwK`=VXcy`;U{^&jMeqc67SXELwQ9Mv}at&m$NL?g!mEqLCBt
z*N~{JW0m1h(^j=CXLn~T;;!5empdvLEXXGQS^tda3MOybbV)$d8vWCo&UVV2d9C}J
zQ`UDNQ_uYn5SI${Tc!}~-@dl;oZ3r%LTQU5%^aOidD`Xs?}c+x<F(7-rcWwh>{!*h
zRGOC`<u2pTUSYfU2tyalrFmZ@aG_yDv_FEsE-Z1Jr5}^Hq?;5RzONSGIek)t)2K&U
zFpc|a+BEKvTaw)sd)zlxq)lQhiCbn`{-<u#GZGf6J`WxdzLxwo?{DV===<I;*w&tV
z;CZUW^KVzu$|R|eo9rk=cDZNdk{g*9dpD@WWf`NKh~%!4l}V$;`5zKa<qBSW+SjrN
zlP!uD4rcOCkW|b*hwNv4RUL{qY6vH}hkV9-J8oo>+eGGhOiR-nisAcaLzvx6DIyCg
zg1IyXr{u)Mb+nJ@QetJ#`ghB*xwQT4eXFXY<mqY~(=|F)_@`7ten?tJ>wT*y{<8-g
zdZF4`C5x0^b!@0>d~(JrtkzeuJY_Q6%I2R5)-!u~VLOpsWgC!vr(=Pqr;I-{=4N#3
ze%z%#daagCuwN}=U9yooRfjF&cx(DGIqes=zmgINx~@0IP~@M|m0~`R`xcWWBc@u)
z&3VSq(CzPa5cqp-47|HWqTHt2&|lZ^%KF6L!=MBcPs2AjTL!&cQY8BOg&rJJZ2hqX
zuKT|T)V4EyB|mZYe>YwrIsggCJ9DXt!ODUovouRZCNB?#7AFNVY9!{SbHh`WzR$Rp
z-*7jRy`otaLxt|DX7<vL>utzt8fg`4F2&?mr#3ttI+uPqmP|HZldrM$riJoRx~}sX
zckgbU{`<K2D)CN~clMTT6o}{z$s;7m@Non!<47d;^f@)r@=76g6W@m-&hlD}F;1tN
z#RxH(i;A(-9sQ)v_Vvoj!o8mwK%&;nwzm0!%z)*R<_9!HjRGac!R(QeQ^RbO_DpJB
zeOjVbYRunu(j2~hpm`&66BYXED|k9(h3KloVA4axsR?qcjrDPi(r^o&ZkF+|AKlMX
z^nu8ZQfX$1_GS!U!OnI(+P<x3ZMYD|QWycOf;pEC@uiySXqbdMZv_b4bc#kxSKdL3
zwpW@XF@ik{V7XLjtKFGC`_6m#WLN3GUy8DGa28kJdf$b-r87iX3f}4-{e1dAVA%Jb
zT6Kfq+)f<5D%cU8jgzBsZ<mQ9==K{ba9e4T+`adJOz|B(;oY--q-uNM`GkAO(B(SM
zJf%r}13`r!VcOn6OahQ<XU-~Por^K1e5H-eB#EluI+ywiGl%UVp+n5}Uao(ks?xKN
z6Op9<^zP=G5mDr8t&+o0Ocx@Mi#j0NIQ@p}V0Ihill_}{LIhhx?dS?g?m~(S!_RW(
zb8b`(vK-aaICLKgU#qo!NGhZXk?W;LIzC}q$(PI<#R$)4<&!{L{O((7mkJ9;%QMo0
zT-z<Ci>0DckR<JcX*oo4Zrn3zCXe?&9~}Htad5-@8QE^Cf7)>K9~>Z3Gh$!nQ%o`3
zvR0;zqszPMVw#PXL}K?lYoc4IRN{LHZm~*^;cZDPINf8z<06A(r!}&pn3xUZjy`87
zX61Ny5_Qk-*|*a*$Wg9|lhmI&s)Zu5G;eGuI^52e9>V{au(^<sBj(==J@7+S>_x`?
zFSR9E-2Z%v`v3J)D)-Enjmlt(4`70uO!x)2<GUXOrilQqeSfj7Bs#|LWMgs8lRCy{
z$V11$YTG>yH<_Luz6*WGquTJ}|D4*y|J!emUu8x9K5XkF7AV))qT|g8=`(wOmy8fJ
z_<Q0>PhFmUdws|5JhJsqtrOWCOd&>G!Hc=JsC?Zx+VF7g89%TgPkljQTg2t7w_tXx
zuDJUi`TEAmlHjJN`&5&mujr?EP45bR2^B#;Q3N`Jkolb%*br!u#$eI=3L`q+UjYy?
zWR1-@k0k4WE!nR{jYC@?2p|9fe0#o2ufJ43YGkhq2>dGXiZKqNI+nuEJP8lAvGIjG
z7gcE11<dl;vIR+&0a?iA0l?(We631QqD5vvtl^ZPh&(l|m|kXM)|nH2WE-hlQ!nTE
z|18!1vyIGJS%(C5p|COF8v$Iu3L<gHB~~`I1<?`s^M>V6D5>iT9{zujtU!n&J_nm}
zqT05Nkj#z*dvOowt3rT01dNdfV6T>Ah?1)VVeQP^P7?Bd{8^Wp(pbeIE|$E=%kZ2c
z#d|@ukL1D}sOgGt?~eNUr1lb=W6ry}QUZZ7epf-inJss58|J&J0D2&Q@tWUyzwd?5
z4^57wpP*oYVyYk941U0+!hj2!tK0tR|FH*svu+H-?5(a3GMIBEVwP?m0=j1B+r|K~
zC;6gbU<exyq<o(*qR;E@Fs{L;Bc@x?`agi8Yo<6I1wipv(5QkgT&6yu@H7~=rZNVB
zs(XnZndIKkfgYhC^i9yFV?{l5N>%;KR+@{EFmVY3C}MgH47B^)sh1JQCpuyx=!qlb
zw_r3{@a;^MyA-4jWeotA<S78tFz~T{db-U*qLm-8@-ZQ|RGV5q{$=nu(7gtiAP>Vw
z7)3^@^*va2^jXS}@tYl6hVaN{Aw0Z^ips0-V3c7KE7^`9;ZXGGFRJ{Yo}GO=K|fAW
z(IT%a4s>#WR(?Uw;FxNDP0HGC5ZUYV2YmVT1OA6tc|ZmO*2+69_ki*J8eAuA1M6Mo
z4DyyauB|l@M9F+ml~KJ5HO0T4A0F)stE{zY6CP<%LL-27`jt5XLb)4zA^V#h0O$e<
zN9l@T;avjMmpz!pf4P61Y9?28at-d}ui*7=K6DN<*H=Iq0c(5?%_sQPDnUQ??9I;=
z7|i>{avabb4y4j1C@o|_LN_I+I)ZC07GQXrYr(COi0)VN!x!NnEetj<H2}I<mFe+;
zGk7i;aCwSR;3K6#wggx~x&e|}<!x}A^)mE3KtxAU2f5jQnb^QH0G#kLEE!#Y9=5;{
z2ib4HO6hk{MMn@c(~X@2I6wj-r=J<hV`5MBDslqGdU@gO<J8*A>?DHKMwtU+3*0em
zXt|gfthvr5e^{s%q~>s@%g2rPs-YAX6zzL%h+P+81FLs72lR}JDmqAqnO~$e>fi5w
zqdd{>neIgRb&?}`KjrRpg1er~ruzC^OPJj{1iS2~pSk3aa(Bfz+JI1n4T1WWvAA5*
zi<K}@l3fqBfZMwVCL#tgpzy6i7zW2CQO|ZE5DST0;0xmB{3hAWkxSLA7ktaqn`k@6
zg0yRz>Dz`t9lUL_pN^FYRy}Cpexm|7ZX?fsp<rjFC}gKr#suoQU3+p^Tw*2E!uErt
z#{f8-#)-S7Yj9zk7ley*D!^jJbF4;+ja4O3OvxM6CGv%@dF#d53P^=+`0Kc#wxhb2
ze!*as>G#o*cwXEE!?uD-xECRSl$Eo1*1lN6F@aAB&X;bxb%IwxPmI?YL1LGvwXei2
zl`4{6QtIhrV>1ehP<nK|_@4~}kNym33i=>w3#vU2Zq1ZD`3p6J09f=BYGirdvmUs6
zF3?u7)$|m})R(Kfz2|eIm%wnzJ!W8i3V`XjvL&&`Ghf|PF!kR_)i#Xo>B!z;Nzrg>
zk6$X_7L+0CeS7cjXNQyxOv+wg@9lPVc$V)}r|^Wx8ZJ9Fd#{hAFMkB%i*e=O1>|5I
z7*qYc`te-yz5q0KEX4J9xij~d-Z3;;`qj7q+ejpJ3VF27oG{EU1B_Bf7)?{TO(Vk?
zzn5{fOp>(<M}@KVSrA)oxNEO*eL6;&$$_rg*u{?7Giy^@aQw#-`%LT8)9m%O22LGI
zllwraDt~zbG_uMCF=}<~i*kw{`jq<s@zi-N(q(57Yjj&|w=!zqQkK11nBys(6R|TP
zdwyB+L-rmFd$(cSc3V2hrpAZcpR)BUY4%#NH5wKoJ3D`*=9t2#_<K_lySTL}HPu=A
zCH_9h{+noVQxvV<_leyP2%I_@l1g_QASJzQZW<CnogOiTWfCp-XdPcng{oAbpqcom
zq6SUK?<`>9Yz~V=$3Mju>J{Xu#tM09|7n(xAGi|mzp*n@L5A>cjPT`O%XeA?tcau4
zEFx=(_axT|>VG<EP)K2?@sii=6>oh<+$mLp@_v8ULZU9QvXC-Cp40q3c!4XvUczgA
zV@3TFi@c1&cp=3mT`MmYxDbt~f<rC8W+O-YcE{JrQ(1Xb@xHOjVoA5Nf5MPLdM@#L
z3b)&WNtN#iAg+p4=g{R_OnwqhB#z|8ZX|_<jypIx`$vXrmAu__aotXTyFBE=?av_f
zBmZm|oQ-l3?sl4&;^igPCUjEdw$+2ZKcq&LT4_E%*x(jipJTx`>i(RYD-{ZF2>T85
z5r&^kQMqR$t#^t@l_g1evh{A#J6ymo(bJ}CfO+i5%SmV6tfYK@tu^7z0r1IW;$=k&
z$`McQuee`BYTNG7vM(FSs7hUMU%~;OAAWj<$tzly4US?NZ|Un?DeO_BoU_;{UzYEm
zu>6vgx?=PyPMkB~I`I-@+puaCw*3I7>LQwsKU8ge*LBcYL|YzgjLa<l?&F@vp*XL0
z9(#^Qmk2b*hQ1<1a+<zhc2f^wEfDLxyC-1#6n~PI<znM6w}(gUD+pL$Qur1N12^s$
zz!Q=Ld6&>&mUIo!^o+xM>Ha6MTdb1YoVP6}-VTJgFz>_J?H*>pVko)F;m=tSNN{c@
zShT10TB&d224-Te!}WcVYb$ELjNUQ&x2Gk^DSu}@O^J@n%j24qM{B0gD_xf8%q_Rt
znAfhP`d<@viNr5_Rc`sU=QC98BYpRn2V&%kyxFaEX1HTbD^MIbt7-eS)7Y0{EEtrv
zHZqn9oBF9)9_rb%wgJhvS2Uvd7|BhD-u9g>w{~mB=3&Pb6mxNeRu+~K6v2*L{~&ir
z8Mu2i;!D<a@1KSt0ylN9mf}ZI+i}7S7j#PlK7N1kc0gy#E8<dL=Indh8s7PD->$m5
z;`l6a3H|8eY|V_3M4De+_c~NV*ly<T9VQ!a#OeR9_P#tE>vav6&(LScoGI~{hZG?h
zGDV?`4VnvytW?6P%sxXXp;8etB$^XViu%x?GHb1*l1heB2^lh-`^~u4KIb~uxvq1r
z^UvOY>}zlG8{XghKEwUo&;88g<8u5|;<;PvhhehTRx-Z3nwc)isukUSg)fg^Io*18
zZ9u-CR3NiO?q}wo86(8ptdh*jbP~FfzS~N#>=<3z)~8+P5_^1ilG~XJ5VMRxbvp9T
znWCjq!p>1e`gz-<O&Jk~4?d6hb7rWfgVrE;xh~o6RP)mP63iQZnj|XETAzM&p~AM8
z3qwxTunNUWelg)&PGwuxSEf*Pc#Y&8enalz1BipRsm)QezP>lMrOB>@tK4MH;-wQO
zxPCtUYvRngL&dEGsgEC`Xz>i>sNXt+sd`_~InD?OD%{w@r#7@L&NV6f?Ftuq^bZY*
za~qi}_WZ5y$PvaJS)vd)Lo}JOrcChkz4fl!vJ7M?C0ar2UF#cDFISeu1g0#NP3wHP
zFSRn+cS}>9s^aJ3K>J0Ra^y`!^^O*t=hlHZpwlb1-HW{|b^LG1hp2y&71WQFRf+WE
zSTVcy&rqt(4NdP*F=4U!M}1o!mFu20ND9`j$w-kG*y31u@%{RV=*75~O=CnXby%~4
zR9WP%7`NP7FC=Txpx3(R(R@rNR_HzZoJmue@2%sn5Rgwbo|wE)lnMIjB`h1Y)wNNo
zVkrMru^v#sIhW&dCQnoF+@^-e;1w?H#Mb>}`%O^kuM-KmDUhTP-^d!HT;>`=`}t>~
zDSkf&3+zG&l~r-JciAk9J<P%)xsPVIDl!fP2`_gMU#j5a!Z13lqyCL%Gr}F2xaK`5
zE|s};<CO(mflg6uS{Fv?E4!Ra;T$Ro2`n6KER#rL75H5(OS;&>GkKq4%GJO`N4vwT
ze&2TLztTfEMP-%Jc`N^ze2MWDig!d^n1lVM&crn){)qED%FvWwO_y(tn~+KO%pt74
zEdKxFO@x%wf8J`q^BV`yQNB_RVvi%4*5oH=#~X<mzgdm6kRQ`lb{={Yef#KIkZrn_
zH+-4MO1e0amH4I|S@`dzqN%aP@Hd~k^>XFJN=$r2U;IMud;WFTi91c)v-upBs_;ec
zIQBBRP@Hnt@QY>$Ct6LMtg-!IOfM9|8lGy{vf|&rXr6^%6jnVxtvv0O$c^Gp!8diS
ze7%$WLRaKGs{b|%(jc9=8vH>-#u!w}83E;jlOIvlsTf#TJ5ny%#41;JW););r&T&Y
zS<1QP#FgvhYL6+9Qj^hRVvJSWHJl$t?YTeTidm%{7r@%x8;)T7+=8mu#$Vvh?7tC}
z*v+B&&?9LK=EBNwhA?cRhyBkpy}7;yE3L(szU!pl=$)Do=xhDZZB6NhvF_vs<s%S8
z?Zsd@yy-wjPtrSfUhVyR7W@VT67TixX`fh4hSA9Tbu7Vb<ipa1lvZV1AzixQ2f>%+
zCsi#oaVby}{2<q2nTE<j7I>6D!Kk_mGas{@cE@-f-829b9ZE+qPvnG>eJ1Dw7^KjH
zl&LJpnJ?v0E0IcgQ-CbwyTmg&(H}2;M+XTTSmO;OVQZ!bW6H8PZVA2#jZw&WA{bz2
zEWpsd-6!0sk_VcY1yB(q@<<%L4^YU*h%VYN%IJZxr-18!fPJwR?FvYXAbAtx$v~W5
zQDc*;2c7DjDAuol8XXd!OVE5E=@AI~-KbJ2)46ky0V7$ocyV#f7bwrC!s&v9vMfl0
zFN}z#AjZ)7Y(BYHXpdMs%;TMTU%a-*Qi+Hf1q}rS2R7W^Lt+h-MYkc#etowUX&}(z
zBaO_v8g58ep{6#l1yxaalXeqhdko-~)nnNB2?t<F?RCM*CTdg~$R|Mn6cDF9G(LTn
z)T@~(3Whq+u{>L@Fn$p5^;a%Ko~_)iRK))qUIZTyeH$?wH$(r=dKls}5c){iKZ2n6
zX}~Bq=Z;r-@`EG;k@Vw?zq;Y3%GUq^g3N<*M#g2@xM<4tOF{t<y^bX<;0uw^<N-t#
zFK0mIopm3W*z%5F-@TsD0w_-w{j?}!;_xZOi&J`KEsnIdZN|pAOI4r2+%44QOIRpg
z>;M#JTKAQ+`sh&gK|{3s02(HDGCD{M5b`J1q=R@3_{6_OaQ(6c4VG&uwN|?r#$naT
zw2o+v#PPF=Rksj>Ntt+EedZ1QKAcUrFueBYr;12lP7Mj?GnDMvL@hpPRY){@;+c33
zE{$eoWNtSiE2*j%7y*$$@v1<1Uk9aF^-m@V#Z&pA@iVHZtVNP}`w=|89~m_34#uui
zKhs<xwB^4;hzFHcEwGZrW}wg)|1NkmwVnM#$`*_#bmnxS8NGwb`-YTq$tiQCR1HhP
z+4C{K3+H?vMb(zEi75cU!oFw9IUaBCo)BZbeW}~%3oTO{Jag>fJpf~^=JW9_f@aga
z3cH}hT*rG?Z(gCiMX)%2IO9`r$z$CG>*m6i&~`GT|FgF>zs^P<(@-#zrPA@{5(lMn
zz5NdUggpL%Qf^(1vNli*^-oT12Xm3iUd|i+Zm6K<7{wfoaCJW_`i&0SeN)~@$cLWV
z@139iF6`%8Y%mRKbx^O=ZEgiIZLutdQWz*@r&QLclxA=zbN@k8tdsKM{Gz%AjCe7P
zta8_qU~LXH_3h`$cKrR-EH*HY#@IBNZ+nEtZKd3*@Y;Q<y&T*r&1WHrTt%}`Xg>@C
zxBb~M8T~Ogs!aL1Yr;1Oa0yv{xVwHKxfp#&#8HlNNm3H$gK!7lGntH-Ze$t1j~$JJ
zoJ3aMVikP7OtygF<Q+9Pl+?zon{Zrl^0>jI2h2i-H--cL)3D1Ofse@Y6>>0}Jl<oD
zhy01*1a!EQ;?rGuGjHCyseeR&L43&Rs=w~x)4w$AS(IA@A9rhQ`Ow5i^P|Krv?y}>
zJm)`ri6Hi~u6?(}#A}S*5$D<#=-_9ARo{`iO`UeK3@O;9WznZwr<~i^|BCm+&RF?9
z$<KWJu9I)qY=&#uyjVPCw#MJ<g&>~Wuj+;V)Rr9&lN50+ugn%*T*|nMngTNq%wrW$
zQRhoNNNcYjuY!GjGJ?@r4?17E!GcM@hIjf7E)ue!S@SmZp!dkR6^J}0O5ca2e|ce5
zD|Y9)8$0Yh=<dgOc0~WO5!BkVYW(F^TBcj5yp8G0{Ow=QF(5mLlEcFcpjf*A_{lGT
z-JE;%PA0Z5SOT8lyGaK2)R2*%xAsD-iGZeF&1JV6{dNLw!b4T%F;>5|Y?GtkKIleS
zk*_qdgfw2VNoJI7At|zj9v!9Kl{tV5t<wyPfYM<8JS`caFD5r=Le=)QDX&3-MKKh0
zk6mqanEDX3MI<oEzPg<p8cUid%8ozVYGhY8-}xB(1-hOH1D3>uL@~7!KaPGlmmRfP
z9Pf~ax})hN8b%~Me`t)dK)!<o*#hoe=|zY$Ew<7DKB7jV_I?D2RJkWhi#WW_JpX=$
z3OTkpdOnWb03Xu8^o9>Xugw5l@*Mw;>+P3XQYTNayK++itNTEGkvlG^NQay%85KE0
z!joyyFceu24p8AG9@`PT6g_e&!{uB1K~~}yPgHlJBqJ7aHVg$As+pQnxOQUQjOoGk
zaSSOsu|f<Z(GbSn8S={$(oKu`%_WMJ!eD-m_|7~x#+Um2PIJjS{k&7O9*<zcEUeai
z6(&x?e-E4uS2Bi=9e$g({q!1M;>NRka=tU*eaiZv4^^HL0VQX*lLK$ZxLCjU!$q>O
ze?Ilw3M)FFg75UnfpCz3+ne{5|G}08MjnYscC_9S$aug2UjH}R1t3i*8z=Vzop!sw
z2u(9@Sy?#c2?3WI6A)7zHOAI0cn#kS9D6_V>a11-4A65HkZ^XTuR9EI<I(Pe0Bmns
zs$Y}lk_b1D*0aNim)!Po;uJX_r>R3P-^IFnRW)Xlcy2l6iG(+|Ce_m%DW@GkYxG{N
zd=L~UK17a?*1B}iBHy2k)BAiLp@qwxVH}vR0FGlE;GDNZp9w8R8V5QZyNJ$@nAhED
z>(D^fVQ|22Kkgz-0e>^wU~85Izn@_EcAmb5o!2n01hP<iZ$|ObWUi+ABIZ@Rxi?JM
zg|b!19C~!ol;=g6&3GI;=Q%Y3)qNss^zU2fFR)zm$GeREC``F4G3NGY4L#X~MG9v^
z<D)&CCJ?FZLEp^y0FA9d471DH<qqBeEL$f)Mpw5K=^ad3Ve!>qtoZYyx9{7y^`eQf
z*8Zq!bMd^*gU7!_Ji&=YYq)HG7nQOv7WSPR_i;~2(AVrM{P%yD;UQvX(|&E)_-~Wg
z%#U9zt|~eGA2u-!OV+jO-OkCwF>V{<7gLrV%Tf7#4~ulBx<~T4T8F1uLE0RWCzh`2
zm7HZ^PS@Hrs6I!^d&&1#cc5`QZz6}_%tC3Gvw2&7?f6wEEv_7??UYzf66z)_IGv*G
z7z0*^_S`Fz9)0YOKsrQDlk#Gm(mWOMD8=Iy{vX_gUMToK%Ul|YmoNU+c2;N;cKICD
z9LH42ZAl?6#dr7byfpE|G&?gkVi@S5^X(A(WU{<@^dG4U+wlFa`y)t(teci{Xv$4r
z+%988tiB-~^5hb4W)Vv8!3T!0nV-R>5*frG2#7f*-CKLpXUh5{5J%+|_hR-+Bsjw3
z840rz32TqZhdqQgqkr>JbL`OT*o$OjCWuv~BOMY)D%__qMcOAUan_J*FFR^dKiSAI
zy!F9>r1@cn<bjac*={E2FR+Z6qkr;zF}fWr)kuffp9le=VD^qcqjn#ptOIqvPp~9W
zm3)9J4&9F2NX!7U5TgIZ^MVag5hE5FRh$Lr8~(Npe0jtx4d~?nA9Vl?-TcdU)Md}y
z#&`A3heHnxHZX}@&YKAD(m`_WNT?r3+IK_m42k;WmV^|1sBM;7^!LJbg$MS6zKN>b
z!8f=VM&;cF%^m$4Hc1+vpqfqsR>&Sk6J55G+&)T<Z%i;77gFnku=<a6$soq92i`{{
zkjlNnha{m8jj@1>jj2XuJZ(2-7V+F1Mj2tcavx9B85gX6x(W1Tsz8ryJsdvDcKXNz
zvK1W_HTlDtnA>wtN_Wo|Fib?FZpzsVtp{Tacd~hFmou)jPk4YpGKT||wiAOX3$`sI
z3e>sv%%w9+Qm?JM9?JUtHiyJRBk``msWTDpfK6Zfd9QyyY{dr(#1fqyO5pF32D6?s
z{Rl;_t$sPAfV(u+JHZUgi{|+`qdN|*@b#ZMi0e53E6knMM<On^<&O2JWJmJmJnIpm
z3NENNPe&IM*zn(8<+p<8{U?}nu5%B7C10iu)${uwrRLc&IXFz38QKCoj=;wq%H%s6
za`K5ll3sI`63Qw0q8bLpNLwMVI<NN)MuGW=-9fm6!Wh5y{@1$}En!C+828O1!CK8f
zrpy*CkhOOE=kGnfnT;1HoV2EZF0ml?9<_DQS;2nBEF*LoC;y0ZtxUV<KlY6-sC*9`
zj~x+@Ax$~s=46eC+6i`4U|Et4oBVGQd|kWyfotPy5N|}OT%Tz4ZOfX;_ZScRT*zti
zuzmRJKg^{OF_%x?UYd^LkYgY3fnOZ^8of?x@=T6jw9A}vlehc#pBV4vQs2C3_tuiG
zscj%+v4A9Oy~agz0Pz~pfsg!=!Pv(ky0&5d{NGG|vZTui1SBtYY0-oanD~s0eO#il
z^}97}r|jW)fGSVUa`U#I*%PL9;w9^M<A$%TTR0}JF=4i*JN|#Z`0II&F^>11E(U+%
zA6KI%5QlRt?yux&p5gZzI^M(WJr*S?k_#2gBPmv@vmwZ1q=1F_phfWUU<{z;PWcYh
zJN4JubhH<sC2SyjYH7Nx;3Ll1`%B}dv`EUAm)A&*oSj$l4Q9-uwc=B5!UG#J)*Pav
z7*>8;HsZk5Ni$)5kyFU>ui4%A$L%mYG3jdrs)PO%9C8sd{15tyq<!&PVb}85J-Z;}
zqHFk4aOwe8ZXwXYXjG92n~>6Z3O8pxCY*hKd5sA163quPr)Yl{nkFitDDb8h5hv{V
zW(=-dCc}++bUT*#Tq5NF6w%Q}y%qzuNFNKpg7+jsLLufSOnKe_V%g)aRT$01W7yxc
zdEcu~fTK`)|AdII9-S(w`x^@oe%0IrV&sh&WtC&JfIJG7#wQGiI!oKUrb8X3c-{*?
zB#O8YR0h%y&%fISfVRbuaEf4Gj^D>@gnEp8P?6>DCj*U9Ep+5%4pp!|J%kZh2s*dT
z2dASy9eRpjOQ1LE3+~^D!K<0<E*tY)INC{d1G5fs_i9+xC~-pF_HHS6@<#jvF<Bc#
zKP{E1&lRFECHIKVlK()EG*jm&JtOtyahLuFwLXrv@XXo$fJiQ=8K;GM9=EdW1U*Cj
zJI%U}utl;*NV}m<h!R6FEnvwRG)zV?nIH_+H7iWVc*71N0)29s&_cATf;v!Z9l{;6
z`wjpsRHe`!rLG;VAAo8p%pZtIDhAZYaZ*OR-{@qrNMyPd19MB*p7d=JClpdj$06oy
z3(E&$3p4`l+zLvJUB?sATLKn1W&=EbJpwnEx^D#Y37%)Dza_(DEi|J-UXEbA0V#50
zo>&qZl6SqmyD(oojGuHyF3%1#*71Yo%(HwMmittW>a&IxW!%&R?m_0MVyw3)&viv_
z`nRdvK<N&WVWm4&m8a(tm_^;&DRFN)fVslP)|S?)wBYguPadjN?`^_<9>Vvne+8b3
z=Xr#kU}!J%xK;0iFIlQ!z`sQ1vY#Cw&Rj7WF5pN-UvIqzn85Yf4)J!qz_f~6Rt$;w
z$5F&POm)Iukyb8c+z}Y5qFAAF5N;u^XxB;YtdGVB)*Xg^azu&>F)WyC&FB2N+M}f-
zN9?&Ls%-+QcOX#<WY0E@?E@8QZDkNg%e2MiBa8cRTi!7&ZY)H(-1ZrNOX)StXCjL9
zSp<x6(4#J(D9j<VLK1UA&RQ*FY+!Gej$$~nHV%;t%|j!f@nGcVH5Ax2@|lu#`wP+G
zvpO5CJmo%&_MTPh#_=r6Y$q4;*jCt&E+g5U=Mkx^Foc-O^+o-=!&%s8ia(S*sggBZ
zRmg&5=6n5h*(3X0<sHEs37g@aaB|2ga?WE3R<r@SW}5u1;o`QARr=PAjvTn1Wv(sD
z3o-BJ&NGmg1uC~|hj)lL+e{iodb0?iljh)Me<K8{-5m0hE1Iq6+uP(bl$y`%s@`SI
zq2-(U5oOo{>fFm3&Qa1#QQc+HAL7mx#e+Ethd8KckbjqBn<k-O>zjdHTkZqPY3pFQ
z1Ca|tB&A&lUv%f!O|wzQ>>P7ca|&!q4sDPpc5;xy-C%n>_|1;nq|pjzA~stzW{q=2
z37olrs^hWH`eEG_q;(9SuKvsAf9Z0Kbu#Jtd!CteE-qVXn@3_Q??>q(5M*QZNIu6f
zEg#%y@jDf1DT*GakQ&o{1U)aC+%?$yeeJ(J!)4fim#-hTOs>uNKd80FNBKYei*$>-
zZ*%*`!fVDY*2L|e8zV`1c+8Gug`tFB`(D{U4jbzp;jQKDDsU_g)j4$mg$J~z^DCOh
z@@{YaEMy1}5yOsJ`c>S$WLCH;bRJjO@jE3Y<|vEwGOqSJ-FTS7*&H{WZC3dS!y`a8
z8`cr?3Q*Eto72f5r~qdn`(*78v&r|P`|#CAizu{vrZt|$)lP(bsdWo_L%>uVPYB4g
zmPfu394~|!Do^|w?KT_mme(aG@j8~$omIgjr1z}|^HoWsFW@8ul%r&*tt)AMJkm<e
zJp2P`fOw_*A6<kJvsVBD*#YtwCd!!RgrEpIR|2QS0T2*DOA63z7z7vCzKZc>c!CY4
zLI&2NCSdA^K=DNu7=7qn-SPU?SAhH_LHEZfVJ}c!2Yz)MNF<61TR|ciEZ{q(gV#Ho
zVlUJCnGAu&q=zl=rWm(-3h{OA9W}4oiub4x41-3a<N_g~nu2N0N+=5DBfd*cNIzCO
zG$Fq~yWS8W*3OY%gsgQV^OV}w%9(=v>C)EzBk%<J=$%Y-2-(vMRCyT2+*yFxhI>As
zkXG;=U&4DLTbJ1Tkbf<dUBJ23z2W!s56(B3fGRKHUA_Q11+xtU`UxX<DaW#K?*{-?
z&#gFhY>cl($`HsK{GmN}7}zrsrDd9I=<ZZRl9B@CKZKcubdSQ$kXg1Ftn(M_|Mn(6
zb~yHGWhIQkZp=npyZO&q5#BOuvnUwMOVrWP3wcOH4Wi*sjz<~19dBIty<tby7+Ws|
zk+4Kau<6|$?)^8OgE9w?Qh1zq(6`7oY~VM|>KjWBN6y?J9Upv}I)0j7;F~*p;gPbh
zlh6tEDMS_edOi*AIb_PW^)DXJr46lnihc@5#~J=oYW77+*6zbMf5Vjx3;pKijt;b@
zDONQ<J#H!~-igncZOhEj%cr5!9B*1O$+#NnEai<6isD%s-S)X7qYEw#2;ZL48mJC<
z&C8VZKqT<i$BH^QT)C3zl4@V;0XNj*_gXKupE&zv**J;)f|c;_SrJUuevhf)8?UU-
zS4bUoKOb3v-sG>4$l`L}M*!-=Dqg9-Wl4r0I8xJ<^R5_$I6<eH^&ui7H44!zyo=+j
zOF@TwBz;h_vo{D$Q#DIXD>37A8~AZq-;L|#7VoWDDpF%Y6CX6*L{sSwVd3dV|81$l
z0v_>!nv7lYcyy!3%Hd~UM4GKGxn;BW!DQGGLNQ^8(jTxQND`qV413i02abTRrc`gI
zcv>YTtNr#UD9>HbQg8RYH#HBzi9607oVcELi?6X>^D^z0m9|)=9R}4o@X_u-OyDDW
zgrB(5iU7TypDD!?Vci>P-C{!JxbLmrzPY14L&Ev7J^ddWU)C{(#uE!JDD5086yb#Z
zjXX5AJJt)zC+>{r?dGYzWYk;A7skg!)ubNGw?OgmCA?|uypx7h1X(=vgAi$^uWBn~
zNQ7S(i=wITGU+&zn^dq%D*4IuedbHUZFu90P`{CjFxFC|-OO;ZO$)yJ$?d$@_peFE
z;!jZ8{*p3mcKK$^PU%3A!=>eNTLPjNSe-*hsWp=G3FAt8k|UI)>+PCAXZ?hc*7_U1
zhZsz<gr<e;LS?h*qoZ}&d#<*wAv=6XpdXv6ou?Bfp!QPQQdTj;YWUUi6HYfN4wf!=
zn1`u}yF;E*$jo|}Nt7Nhp(GsmtH;TC$z2Za+1efGKF~u+mSe72g!(n+RRVc(%&V9X
zmbwo0{~%c}^CKk|j+1L$zL}n!?RbtwSk}dz<!>EFd5Y`Yn2M9q6$hA~IVnHn9?XBR
z{^Dxo?9EwmrNBP(f=|M=jb(ekQ*lv~4F<dSIM8nhJ93zAY1iE71RK*B&~2S9{RVlx
z=3xm8mL@vmPfHOB<{8RJhX50yxe&v-KCB6e!MVfChiO2{Pk0)8aFp|t_dcFRXR%si
z!5zv*UEi**%{vGje6tLt-24u@=e8iLh5(c)8ZNhWyCf-O6DcKC-tBI!Xqc}kx{dO9
zr}@6F<f)njZ6@13ulbj>0zNn9QIyq;OASkncHemHeklI{^CeNQD3Oui!^Y+;4pyr6
znPvx5eC9T-S&Rh>H#unPbo@R`jCGSrv(uJR)$Uwx?Dujr<9mS{#onYnp@5H~zm6U*
zawqKPk4a3aX%E|>p<O(hBv!v5XZ$_HqtOOCBd^?2B&6SahB}&-x@N2wF0s)YolUD3
zjvbs@&#t9_HU>(*EuH!iHUq4Lg&y-Dzua5d>r3FOXH&|}nm&XUFz-y7NC?QC;`f&A
zR(YFDDjn`~n{+yF1or5AeH=qe=M?ByjrDNiYIy_(?NCaZTA;D-Nq>R<pZH%@_4+9G
zKjh5az&uyh^+7pCl5`vTLf7Vpt<&aGgaMDMKH5*XVf-c7`l2S|?E;;=CpVKZp9D5}
z=Ufubfx&%IX}>#)CZdk!iy_{^Eh^U{2yi;OqdL79E!t{RwbBsEQ#H`($74YOS|@Z%
zpoVbl)Ii~1h&VrD420AyNETEBk2|5fDjVkx8{#E=Y-t8kojjC#ku6__t_PAT!uV`U
zP`VD^3?io&SnZBXC7@gtKs3kmJo0BGh_Dh{U!_2$zTNXYOgAX!pLW2hVYtQ{Op-uI
zeKazb9r5yaMA9aeh;+?AB~P9Jn4&2z34Ms4c*=j$oJs19mOzP%UO=(W!0)2lRZPRB
z4E2q@$Vz!7bmC152Xb1Cu7}|%XNsyfVPaMrg0YqcDI6Unh_Y6m$T?SafO>}Pc6nY!
zn$alC!U)#Q68Z3Hts2%^XewC3zO<GnnI=6H0$+lVr1KsSYJm7c70j}Su!3!<0r8%b
zPRF15iFk8S0b){mp>UJ6;mn^9==(va-3^7k-yy%_kIZ|CSvn@m<0LN{tkU_&y`i6#
z>nf}b3IfNI=Cfy9B@_*$A*V7LF$qvWA2{Ada2a?~F+QTlOjOsad8x?e9;EBFbJCG$
ziP<<EkL7UMy<~nL7V16{zYdbuqbhBYkP$pOl4QGgD$5O__!5J^C#C2RBFh{GnC)eA
ze?cDd3#OHSSYxF273F!8h#hHaV|o?QD(2>42;B2@L3GLWMluI(NDc7T_2=y#$F3!f
z?^l^0)67cwcyyYWLb)DNMQa%7q5!ANG;+uBAy7M(-f*9yxWWn(qRxYnEt{Fyx|W34
zvm-o_HD+|uN%Diht4D|o`zyW7N({e>Jt~kI`Ih-<?^(!*8DB_)tcB%l>^6V3<O6Wz
zvk_9i6wD%(7B6>8IuDhM8)*yq&$jbsrcQTpWoq=8Ymd4vNp2yDLD?EQ!4-gHI0+OJ
z=cUAO&q;m!hAu6^=Q@*ntztzk<pY(V(um_kcjn?qpfH#XPmpx!i5;cIWN6@l(I$Cs
zo4#qVlpPHP2%)GaF7mVN93xppN8iC)9deAI2rqQZhZF(gHj-GrdKwUJ-(J3|`r?|o
zce$+q1Osnz6E4NLInRV>mdwTWp+_l19Ac+@?h=!JbwTM>qRYgrrr#)++~{)a_FYJZ
z6e(zSnf*nmI>e2Yd6<%@;fTc~8XlM^zjOw(u&Sdo<*sOMc=0M*gFn&2CMe?Q$*jLV
z#uKUw<vYsSt=!5AuAg#~VPWU0mZx8x;c^dhUuV#F24v~v;snb9r>~{;t3G5E$aqUt
z&vXLbzBkWzs<4WB0D&g`Z=FOZ*;5%tN(2-eIuR{_H?jR$LOay@=JH7X`NICicR<<z
zzZcH**Q4Z8odI%l$X9m}CQmfK`gcbhCyZdViE#t~4l8lENd8c!rL9jo&MxY)sAjhk
zbiV7j8w{q^8os{`V+`bNQ1*x;qk4eamCA7K_41qkV4DOy?_M6$f+@45tF}_IzB3^C
z@=$_~4dk}Fr2f}}J0Z9GI@|`rc8r2a^t7r{{Y?{?PEI4C_1yM0C+#-@JOlKdX`sNF
zX?)Tsc}~Gm_$1jAKQ>L6X_`JTguJsZlEC8}0qk<tv#f~ytneGzJzn~B!vwCR!UeeW
zMB3H9><M!{@ta<fmcDzIGUai`lgK=h{RQ1^Or91d{{Gb~C_fBe{taYs#xt1~bJ^{>
za&g)}dP>5q@t&cFT)9CeAGmHlOb}MTnot|}GI^Y`NB1CL`$j9*f8(UnOD>Zp?YD6&
zJBcUqSde5nVd%z99zB7$x30MQ*;Clc*so}W5}x<SMPXvOCN2mZQ`F>{k@KFZ!pzvO
z@K~Tq2p><HW>0@#%fgp{(qqV{g$x~e$G00_t8koN-rBV>7K82=?NKqnh)o=3CqNUv
z-n!R*GZTK}dhyM8j$=d2>kZ)X)#$L7Yhd32Y5ID<Cb;5@3w7`fiHlBkt4w_c%lUS4
zPrtbR#FNlh3jpAK4C%6xIjJbXf?nrdpZgxLphT~y4-)SSJ`YHiKJiV91#+cKiv70y
zjd}#<E#spyOMCmPa(vor-&kRHkLGWS!(oRw4DcHVk{m@w&58ft1Fk@<W6dBx3tHwE
zq+3#+42UXlL$`N@&JBclXy$=5#NnL))pjE~=!f^!g|kpRBDx^-Au!8l5$H6NuNAVW
z3RPKiQ3zve5+AXNY+}rM^<eV}UF6Nm@xS8J&jX7hVwreL#ZUc{$d!=AAM`7~Q^W+p
zrnI-s4|33c7wbnlzOJt^Kuc0RA&upGLwF|(4t^6%_UXQ?f%r6ImYMyz>j723v>zV;
zRLkjq5z|El@IZb_Nv!_+5P<?w=(ax1#}XN(A8s=9p#p%0(mP==VfVp9bg2h>y-XtZ
zSbgI&zN~a2NQUXXQKHQ5h8Zb#=bDGIk40ZuSHflG@l(}$VrQ1ABGMThe*PD6dhs_X
z1aGd+Fm2~DZVFjWpbx;oU$Kn9j32k4WzDqo%wj8Vi!f4WgFqt3Foh~p-7<=SZe`s-
z)4KwZyvB)Xwt$hi>&=}X_;OFR0W$;7jxvpu@G65Fe|dA~+Qy+(8b2t0g7LoK5K3$}
zW_|bmc<Qqu#qUOm0Y8wzfkEirc13+3CQjzNE;YBFvgvXeI9qZ7N)F_3^#{|5iX4#J
z8>5ij6zQ#{r&XY#F1!Nefw~xA9cy4A;oJMcEU7#84tLiTcU@GEqL$@WbQ5xK1el;k
zCgYscbgVYfG3Xt{|5rd4vLk%AYMwd(=h!^OXLvTVq<bKisB>z!moaW|$&t3mISZ%J
z1H;L|>C9M1ki4c*wzlh1+L=GtoT$cHzd|IPdlYC@X#b0YUJl|WSKd^%{-Bm5H&Oju
z%#9_%i`dL{dh|2YjB@UG;iAx<EiK{PbOQ_xGeHDuHrq2-VI@V*B^?Xtf!E;sy5Jvo
zzPd9$ocECmsiD1AG|o>G-0_-M%5ZUSd2ZUP83fhuWNtB(Sonv~oDs~mwxEU_SmNPY
zON4e%A6x^Nfn7s=V=?UcQ#p!QmrJnsK-VYx7WE&}Z~0)+rwq(7I}|A_`|U0t!)D11
zq&USgFh)H35m&>=PidJ>!kr2#2Ah*mR0zH3_OJ5w3~H!)k0YF6&BDybLy*|2CmOnc
zf(rsnhD&6AOjk3l*e(KL;$E4zV#Kk9Uf}bxv)OW{M1qrJsCj!u(a8yW5zj|*knox%
z#L^52HKmRsrZjRY2?uVqzoY)>{NeKD44J5~8N9AOOlP?X1$O@CyVYHzeGuwWs1xH`
znEp89EOR$aW0w;STAK;O#&xcX2c=Gej4JeKIZ(00)yjp<?4nlCP(L~>UtV5+{O%{k
zRoYi87)LyspY0Ydnfa>X55ox~P17ZeD~YP&-`-1KfziNRzP+qG0+F%KTjp53&q^h%
zPQREO<~q7Cv&@7B?Z+dFG&V|W3WZy;+JZ2HewJGGGpHI<IXl2E<F;}%IxM~kINa((
zmYmSDYS^nMFPMYMDYLiI{U48KtA0of^P1|Pu%lJ6?pxlWe|+(!+akm}eV}%ZXd8^~
z{g`TWN#MoR&HATn)K@xJGY5_ku#GhH0qCd+61cpH;UAQ}BrCWW3+G;YobJTs@!0iw
z&@X5?xePygz5Cvgnwbwplyaa~^~%WLf!COu?q2cu%S0{x7iqT*JacpZMN<A@9R8IX
zYTUB%oUKWlL;Qf<(+7r5Mm6+YN2eAK-PeMT)tWmtRZaO%$ihk7mwonegP$&0N;*Z3
zp*16%y)EA^<g>?`smznlqVC%rLw~rUUCe_Aa(cTf&fObp)NZLgvhni7w&3GHDJoNa
z>PY-{W$DHnP+zt9^5lEPzX>vY96D}JW=<r|R;s6fvFl{da#GvTG8D(*rsKPqn}i!W
zw(ZM!vE`a0@4iHevREV$Q$SHFJo_91;lmi+7^ThS=Zq*4N5>a9B~bgVf6sAw#z4hq
z7#8RK{dEnU5h7b0zm}@*@y4m<PIfCQRgJqlK@CLic7<?p{`%PKIt%=`ad7(XXz$n8
z82oM2pxPFVu1x6B^<bxOMKS~v><w0{&1tO*sPezRAH~C`L0Sm&3$6Vm$arHwqm^Cf
z^`{nbI#~flYK-esJ)sJn+5+!NT}4(lXz4#d9E<VvFG)O#k?$BLcDsLn127%Z7#$=z
zt?eNzn&9=#CjQBsSvQ%!KlQyi0&WJ;7z9MT?#9lS-w?Vg)Ih#w_U=1QKs_ZIwx8O3
z;N5+s$ZltRiO8(ls_b$?tfNJg?FRh>(%NVFgHV)xfWzuEy71+u(t_)07$Cez;=_rc
zNS-<HfzMc(VZ{g0b5}9~e0GjsTZR!DSi#*WTk2k0cP8$t#uS|X;yv^P8_|!U^8(2K
zRe>Y4s4k#0NlDa`YFj10O7BIcZm-bV6$pip>#ZoJ<n*ydlgu2B2qX;MwQj9|sFe00
z{E*-jM;hgYZd~B*uQQi&<y|)lP&zaWj6FRCEvqs4n2+0^7B+HiVkf2>s}TM|L{6Il
zD*pj=45cSJ$4K?DOp7@*hzcoy3Hf5VU^7;#U<!_9KwX!jW;7i8JG?Xk7s44feNSSY
zuS4jw;1SaZXC75VGOPc+uvGf-=qiu?Y7av1(R~)w*c|E51H}B=-Om>xuY64NMza`y
zC+dn`%aO+;EFfJ9n=!BvxjBd10yT!oj5%hZ!>p=_TJI0dz3O<J_c{xw+wv0Vn~C5}
zfP~F~bn|Us=~8?eO?+=>pbDJ{8$f#RM~7OCUYUj&$thu&%ahnSVquJ%I0e?%XKQ=V
zc*RkqJa_9&J7l0Z`FVh&zpqAs5Y&F|l%!XgG`NC%i!6k~@}qJ*%cinx<ImL2lTUvK
z9_U$~o%%lZ8xZ;Ip2PcU543F7OD2&I`Z5k2mgGr=#7oK`(6tEbP`1KW#Uh{(kH6OK
z<{ca`L;@VGV3sVGsM87H4$6h{6$i9!bw3HYF3Vz*>>@>E@Hy{{{QB|=6FL{K%~<}P
zFTy*@lPUQ{qyVZU?M=qikXw50r+Y{~wl+F;8OL&+?sCn;LIdyKZ}SOG|HxD%mzk3C
zC@54&+UV_!#H@I#MLoH!?sA_H>#JKPB=^LjW*AA`h${V|-E0|M@A<B?@GN*@Qg3x+
z&o`y(i;i_t1UDM2nCt}7;Hr-cF?v6Qow9*T0=RljdZ6<R4DqDy=`wK&KCos4hiL@Z
z6E|nmuHgP^<&YrO>jpz&K4hHbf`!IAE<J!xFq6nEU?NQib*ZUG7ou|P2aqSbIrkd_
zC4&o9!SlN~><=P3T<~5-mt!TVRfKbA3S_d%bAujhSAuB05}=BoC7*`lCMNzvO12)M
zgLgafo;&xMwWEe`4T-4JaeukuBM$&TNptgW|EOZ$Fol(&uK>36+2ZAUz-aVRV(1UG
zS#8O!YmC)C%X;l^%er$$4T1tzoMk<%S?%JCWZw<_(r*}t_h%bLiNBWCOxC(}Rv>XN
zZFwZCwO{t^^kC++dQ+#oka`1%lE_rwqx1f7T((U8?~wZ(2;j?7k{0l+e=J^>RN#>C
z_yvcAOo+g&M{->MdY-91hxwdfm_*bjIn&HoGv_W4plitvL1-?h&U(>;Uo?r#QlpH>
z>V>pt<*tbrX${AvO>$TaQKyhCVshqPATF~{)70<7!{``($Im9N%eW(>d^Ul70wXWA
ziruA5LOW@{`t#w}4VFE0Qy-<MKbCm9Y)qQg&0yyVK_@%UwA6)b0dEPW&oESLL;`;S
zNJVuB8>PMgbk@JEbvklOky4ASiE_}pBv9=a*Q>5fW0`qc%U$5>)08Gvjp~Ncs--HL
z%+fzXJH?H>M{>~y)ayiJfFRD7Og}a1;{tK(eI1n&IR$|S7h9dyQdXfy3uNAyCJvK8
zD5M&3wX@O3bAVFFwDr0(+sr{AjgOw+KC;bfCHNO>9i-Azt0Y(h`Z``6w;XZ*tL|Ld
z`r-L7e=VxiZfbnGC=}XUm@f%*K1=21ZgMMQ`XaLXBj@PtK}7vT@JDQCC(yvEB+bYY
z#M`I{u<-P?*G1kjZMn(I!OCY&TT)!hVr6SlPDB$n#r`FDUQ@Al5q*#FCb?go!KYqT
zj>3Z3fb4$$HJJ4mW_=WKR&=PZEGhpj!7`Sd7Lpjd@6|d@L<H#!g0)|!U*44HbUwIO
zjfjIv06n=pU~WE)mXPZGAvf7|Q=g6|1%{ZG`-leC1FiRW<;$|KP!DpINe>*sl%oaw
z#zS*8uDwUblVl#kJZhz9!TkXsA2o+>hoeSG`A#XXgioUWKR-W`In2F?CSLL@_C_-j
z2YJ6mo(atSs-{H96K-$*$X_2N=0I6hRva9M;F>zRyGWwU(Q?4RG2Vx!DTi>TFl>6v
z=XsvERe~JYIDq-|-vvE!vr|Xgc^JPaKj!lJCwyW@@$r7qDkjfa4ZYX9l^>r>+J0Jc
z20~?*DCSGj9&2ajtIpfkisDnE6}><dS>EGxj@_-O61sVr2?tUS>pXOwf2Tcs)B{bb
z#W!!Zj3?W6cC21ss@g)mM*l|mv2rEj)_P~DxeF+N@1!YtBBXF$QP_kv%&MYX(v|C7
zGO%&OSiehfU?I_3B3}UI`URmr2cDry?yl(gTJ?Qg%76$-i~MG?a*aT}l=Ry`1HtkI
z=51WtRmo{J#yaS@zj(3(vFtvs(XlZJ&%hZIvMc}1fz>gpi5Sv-Tur@*A~-j<)@1i4
zd8y~)e<~Y$YP_3(u0?jS0&m0X)wCkQYk1jhQ~h=<G8ObWD?BOM<*_QgOgruX3E`B=
zC|XsVUO%tgoAl7o$|0zaRCQ4nAYHog{3|=-XDsX!7?5||r2L?{Xo7|&7iWC&h=R4`
z`khng$aq{?ETF#3F89o-o0Ij7u~^ZLb{oN5^U2Gf{!KBon}$P9hL&Sc(LA6v1@b!!
zCLoas)JsSezhL{~yME(DwlVRCUMs*#^j`6|7h*j6R{_v9yU|htNtD|_D_$UYL6Dzh
z`wpoV3hH1HUqH7>^MQ=}!14&M1b9U&IP@52^r?L9G_2dWffHf$E~%nLIB^3CouDdI
zm$Z_L*+rvo@Q~{$f=1sm>Btx8)I=HM0E+C}zVxARD}`eR1k)Yy9*8aoDBqnm=iM-9
ze+D`0Dq5trM?K%y!lqEJ`d0uFa@`QiVWhY{;6l`SAB6GfMLC)QmLzq3rC=Tbgg!g&
z?B=QbDKz=w^l79{;|7ix;0q{ej36W?odFQ4IlX`IArF-qJRFsE2r9!IhEYc_8GL}`
zn(zdndt(w#sB#FDuW#|;BP`Z!eGjuS-@F{$gX*yKzqY0!XC+z*oAx5|yZ&lA#p()n
zEJ|68P@|GN%mCN+{4*3nAm+UFKIm$!)F^6h<Tv(!J#X*+ght6roJg?jIuVA>M<9e$
zTI;7+43Uin2<39;2+6KIO%JAetSmtR+F~)7fU8D-1z->YM!J$(zfDS_?gRhvErpL>
z3&k@NaYLUy>zUb(tq9r6y867!(_<MM<N!eVAe#MgFwFoo$Y%&Ktv&!Za^7K_u_OW$
zM~Z2oE9Mu3r~3RPW8Za-?r9i8&Oit%nFjXGNXhYdPMUYC5n|Zpu*_c#-b4IXkmSB%
zjy8*M+m&i83C46O>jn09ydu1f152(uKPt0jYkST&=ZFA;jsgVj%Jk&|FPem-%sdmW
za%OZ>=18H|L?pjgIr^NO=7i+<!LO3@<#RB_E3G91j?gw4h*j(ZYlfuhSF>$RsC*&V
zR%x9Tiz84;^SW%0o)180L9Po>9&-S^U7DeZcm;m}>al7VHCE?CRQ=FQaV<w-mR~gW
zt8{yABIAE5>WBg~X*)}RVw(wt=#a)L*!W1$A%Oq=BmzfR_VyCS4ypi(GC>e3s&`2+
z>yWn)G*6%V5S)%$WY5`a<yui6uw5GgEP%A(x{zW^j#bi<(9>=@FQ>K62Z#X1yl+Gy
ziDvtJPaMU%5Oex<b`ng~32R(wf}8=y?=0I#(yV}dc4q6ob;>OZxs#<$Eg=4KzkIpH
zFYqg#S5wtiW|eimFLfWv4=F+4u^-s38Y^VN#EJHJWznaT8>xKg()0wA`XN^$AAJ~;
z%A1pmspb1Eg0_qZilisA=$o!GN|}AXy@o+!_v4F^GCH$&rWv$}t3pXq@oZTeoeTB9
z=-Pi&eU6bXx?_u}mykra&t=F(sLt<%Gk^^_=*vn+wqcaGBV_kzRw)znK~Wc`L%reL
zHTE`WN%-&ps?}Is^#(Bk#S~A<_g-l7&t4+Xk;O|%W!rM#JzK5xI`9V6{Fv>IK;!Nw
zN}ug0><P+j3i$EpAvKZjsO3ylHjBUz@YhJN-Os*EFE?$eAUeba*U+>T|GI}G*1p|1
z=OA~^y=m=2-zj3FZRLHOI1@;sf3k|I{T_qFcz<j6k#n2}u2(V9Ne}C{;h0uMMOn&$
zb5af6<?H$G&N7oPPhO;!)jlfqRjyc-sqnm_$thO7fhUqOoAX_*6&A{^>_p~LyN14h
zBwQ4P<#Oj4+)6Z4Aw$Q{%**Dd1pR6+-^)0MSuUkrhs>lfctQ+HeM-6>8s+c7W1;TU
zsx^}FmdsvVR#;~+Wz#KX;Bzb`!~cTGUM-wx7^t_aquQs{@T@&kIQxO)TYp{S{obnk
z&?P8lRMCo0N5`*;KI2%+@ZyT^xtalyQ7p1l2t?jX>j7cvNjask+buS%jZd28s^HJg
zYnbNd-$pZLzq;fJvv_Zo?%b3TQReIw&8EAEl9IDThZ_gGpOVMi`X9(AG~P0h)!Te$
z*`3tRu|7aLK{)Vr;B@e0I=;HnnpT*y74VZmXZzi%^uTZNOw89liH+LIrtp!TbBwA#
z=-H~3Z<>kx9>WGC_O!fmkq#p~+gGe*R*Qm8s=KrT*t!Rs7g1ibsFaWN#Az2e+&Ip`
zuX{9N7SZ9)J^rP(NwH#0KhN4|8UAn<uu!Wy*u$j(jxA$f!pj}J<HdRYh}@(B)5NeX
z9B=q*_*~bd+p^xQNU4~<z;m1KvRe-BxSWVECDs@;x*bn7JwraRw<|L~P~M>I7yfES
zTXz*-V9UTZ8dEA|$A(};tuBs7LOL7LesYO2s%DRDi`6}NdpKM`RNR>Tpj@>`h*?V&
z4C*s=`gtRYSkywhBhUcT5R?8vRWOlrES8Iq$c$ror`5r$-=W?@TUX?_a2wwilyLVQ
zoFz;cr>9?WioIm090X{inVi8RmuXw}m=*;?5;sCA&*kIUE-KIej~;U$O5ye{63ApF
zGL61acs(Z&ZlUlg@2He9?+^~*8SUWJmg0XQM~oVg3O&I?%s$AW%f#Bk7%L-u45+@(
zPR?7g%1GOEN7_5=l^W$6qotk94s*Hnlg=~&?1<ycP+BGtBf3CFFXAUxW-;{zsd@Ks
zH7Tu7BoO5L+Vb23uu+j(r$jYeER-TWe5@S5KK6|ACq;5-bt4H>tCI>JmdkuDzcEx?
z;ns4rL|^|aUu(iy)AAWM9h||p2S~$9Rt!^Jhx~g`U2eO7R@-<h=G(6u49y+hK5#xa
z?k1_m^RV@FR>jk;#oxmjeA?8|`(|4UH^$<quy700?%Sb`fAz$G&`I@XrQ*=)3TYR|
z7fdH|K5a!q)eg&u^47da3)aojN{hP<T$7A^_l;K*pFErBl|v_i9K^OR{P7hHCdOwt
z>b0M1eqMdoZY;7raid6N{IM}*qgB<aveo&@q~D!=LtLGSiMP_y%w+ZWzbe~_n9O(H
zH&W@E00_rwZvPt6F?H~L(SAX@_NeI0v;g$?Z=Wj1KPr6nbiMt~G3qX9FnR7sQ9D{+
zE3j_CRJ-0SqWM0q*|jMhl{lbuo*HkO0V|@Q_59)AS-^O}(lB>^ZWURpp~hb*>%Xo_
zV@`FU`NX@)<Ebz7XidKC<imT-$HTSmJ2Pch$0J-7)D|ty<+K+eM_EkAh9#1Eqyq$`
zLv%WX)nCR)9i#W*f+`M|@1QRI4)D2;8n5~Pf^fP<N9#am6Z&rRA#Dk!!d=+(vs*p^
zIl2S2*S`8~yncg(eve*Po&j<#oc&ly1HIn6*V~EQ1%ahm&G69?!wJpV0O)?!+D$bG
z#5iZEU^wJHFlK9{1;_{s%ntxo_xuV7i@WSAwsj{?wIbWml1Yt~W4rcm)R*wLKv!|M
zF#=f#B{wi}h7q2Y%$v6^mk|7A$mECppI<~f4c&NJnt{gY@O(HWO-BZk_>fi^JedQJ
z0Z<|k-pKU49o=aJqVfX8F?`iXPSHJ%R||GFF*WvY;9hmQ>sb8L64&L*9;!+SD-+C0
zRm=LzUR!C09Z~hj+8<Gy?xA$lN;Sa4<z#}L*6ab*cVRKjvfqtf4sLmCz+2_-t$pp|
zel?%1FLhh}u6FC&)@>SXP4TXPKso}2XpUnGQ_!J?N3%^cv<9Cs4p`!TFif*I#06j5
zRFDVCDWo#U>w$-ogf>~y7p_<}M5O2mMhBX!5CSMA5syPFPtP6!dPo``Idva^_=9fq
zzB_rQ6jmx9%{YXKnIa`^mpA+PE&TufQK7P6R{cF4?@J?5aH;h6_xB=7D)~SHQ`V!K
z?-sYm;jZv^z=CnlW~(z#HQUqYK`VDlZMX!66m}u(*;(Z#SqxfqN-1i~F0Lj^BD0kW
zj9V03+#X68B4EaYBps|4`g0`t(kEA7>VKsK0F3QvuT!*2j4uVH?_3)|E^B+&xmL;A
z^Sn8`K0)yWKPH{v4J+i$%9i~Cu346Zuy6OPymRxq&dsJvKG1%vWgm%mAunZ@6ynP|
z5WGZ+!m1*o16dOiG@PL9GU_%T?DapxP_#XhqRdjExRVta&3@FcvzK_$Z?&KXAh@X6
z6XjP93L1kFZWVN)l9C@U-RiA_($Nd|`G2CZNz2kRJ`<u|s4>S&n>JkfGsyCTI@047
zt02EKL*3_tQ3PY1vA!l=mtt)%6GFHawZ&+#th?|ePU_nF{Olr|4Pax@4vRNv-XH$}
zWaX9U%u;JMZpRA*#8MC-KilQ@F{HO8KV%g7RK<IM$lu;Q7`?VX?UvTohCo&g{&a&h
z^Pc91x|=Z3?o8EjE_B||yl3V~(4NoiY8?)z_}<#%F#pDt`#8zHgOT@r8aIAMr>OSm
z7MwIlV!Yza$Y0}~*sWYK5YDyB{hJFWfK?3iW~ohX3)Z_L@%%n}&M7cc(k_t<$?FS9
zS!)N~hRXlivrnpAJUjqUc*0k9y?cGAw(pnh9T}<GpXRYIQEUbWRfCJRg5w~s)S2yE
z6%_g{OFYXvqaOK70(ytNF6mkwI;vlFnO>QRDSOT8PnXuoMLyMU;MDElUf9q*ZCTpT
zauT*X;f1tuQ@3zg!-!SWp4-1xxY?xWkScV*(-$Fy{7oMo9roR2(*1NF(^D>uqz7?^
zBzhWT%U;sBH5Zvdp&hN8=NP}s_G0XOLA{|q=g~6P$X8A2aVr)seI<Oo`}-(9O%)dp
zNX^In(agTm&ED^5X1PM!+Y5!E(#7VdhpMD<C}VpI2Zr9qSw?Mq{y~nJvWCM&fs8~T
zlb^0I=PjdE)xw}H*Q@r?;2lEiNYx)<g+8>Z&+9u!L0RyaTRp?1!S%DrKB=<j{CW4B
z1R@I))s42Ji(dKJAkPR~c}mrD9^_@3*tDBePINmzBJA6?Jl_i@jbZVxnEhb_3cTI4
zO-P$nuXWkua5T@A2^<vKzP6;WqzN%vMPBI89aZ15d)z#dixY>rs#0|G!MtE)_1#_?
z85G_<^>u#<=l-3(N9r6ab+t?`F1&cCwS5?g^?-h`<gH+&aM^o&H##jpF4X*JhVnCZ
zI<omM_g-^Qccbo-^8<yf9M^p^gyOBaMtY~1O`2h<+s~Az1Mn%IjvD&2Hv8vAFK%5l
zm(6eM;%nQiK7mr{vd{FJbl=Iy(bdn{h#Z0$Z&F75PyQs6_)|I<Gkm{bNoIJV(gObl
zE$28mganqM;q6j|q2H*G+k~ZJrS{D0OdG!a>uOd44tv@u<1@#TE7<(s>^sBASE79v
zDeDf1%aq7g+Q`)WDWd3bqrbu)EsNue&&7!@$TW}eh)7gsuP~$<9QvhWbqQ5n?eL;K
zYRbl45tj|y-Y$M+a#FW>dt}n4>U%GezOfnh?5rD}^+N8qjl{A1M)^MauC1zQ?YA^0
zy4og2X`{#pSEu{izMb9fyEpKU+0um^g{^At3;(p;I`z!IPdR-)n+xmaB^Q<vi#zod
zRB*;t8#3;i|MSOpg>dvZ&WTU|r&PZ4n~ay)4KRI?{O6C`RvIs>mQVO&>Z;`4>0ZoY
zCZAX?`~U05EAZn}b{EH=cl?+1XL!03yt&{0_2d655AA6DX}HdS!g}9=eyuU^e%aM8
zJvm&>w_ECF@3b6$?ej~TF5TYq&Y^}sb!;7{9{LdL<%LQg>JtC4Y}D1VG}-g6N`IoK
zd)jJJg*eKnhx)Y?#=qIPef^7CM!hJue2(-#f4r2U5h`PNgk#DI<KD&#@zXyJt~CGW
z_cJ~k5|bWfyKVZ-;AxKFr>8Ss@c-+_ax~fhKJxgo{&$t9cC_7pH^$US`R~T~@5cDQ
eV3!#Wm;;tBmOrnYo5h6xST40OySl_N^nU;(ZqPgc

diff --git a/format/Metadata.md b/format/Metadata.md
index fa5f623ac9797..3388a7e6cf00d 100644
--- a/format/Metadata.md
+++ b/format/Metadata.md
@@ -46,7 +46,17 @@ table Field {
   name: string;
   nullable: bool;
   type: Type;
+  // present only if the field is dictionary encoded
+  // will point to a dictionary provided by a DictionaryBatch message
+  dictionary: long;
+  // children apply only to Nested data types like Struct, List and Union
   children: [Field];
+  /// layout of buffers produced for this type (as derived from the Type)
+  /// does not include children
+  /// each recordbatch will return instances of those Buffers.
+  layout: [ VectorLayout ];
+  // User-defined metadata
+  custom_metadata: [ KeyValue ];
 }
 ```
 

From 2d8e82056afdcf125e6e512f96007389ce79c1c7 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 7 Oct 2016 12:13:58 -0700
Subject: [PATCH 0160/1644] ARROW-319: Add canonical Arrow Schema json
 representation

Author: Julien Le Dem <julien@dremio.com>

Closes #158 from julienledem/json and squashes the following commits:

796cc6d [Julien Le Dem] add json documentation
f0b2a39 [Julien Le Dem] add sanity checks
7dd6d45 [Julien Le Dem] fix typo
248d3ec [Julien Le Dem] more tests
f2bc3fb [Julien Le Dem] ARROW-319: Add canonical Arrow Schema json representation
---
 format/Metadata.md                            |  81 +++++++++
 .../src/main/codegen/templates/ArrowType.java | 165 ++++++++++++++++--
 .../arrow/vector/schema/ArrowVectorType.java  |  43 ++++-
 .../arrow/vector/schema/TypeLayout.java       |  11 +-
 .../arrow/vector/schema/VectorLayout.java     |   5 +-
 .../apache/arrow/vector/types/pojo/Field.java |  23 ++-
 .../arrow/vector/types/pojo/Schema.java       |  90 ++++++++--
 .../arrow/vector/types/pojo/TestSchema.java   | 119 +++++++++++++
 8 files changed, 501 insertions(+), 36 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java

diff --git a/format/Metadata.md b/format/Metadata.md
index 3388a7e6cf00d..653a4c73e830e 100644
--- a/format/Metadata.md
+++ b/format/Metadata.md
@@ -63,6 +63,87 @@ table Field {
 The `type` is the logical type of the field. Nested types, such as List,
 Struct, and Union, have a sequence of child fields.
 
+a JSON representation of the schema is also provided:
+Field:
+```
+{
+  "name" : "name_of_the_field",
+  "nullable" : false,
+  "type" : /* Type */,
+  "children" : [ /* Field */ ],
+  "typeLayout" : {
+    "vectors" : [ /* VectorLayout */ ]
+  }
+}
+```
+VectorLayout:
+```
+{
+  "type" : "DATA|OFFSET|VALIDITY|TYPE",
+  "typeBitWidth" : /* int */
+}
+```
+Type:
+```
+{
+  "name" : "null|struct|list|union|int|floatingpoint|utf8|binary|bool|decimal|date|time|timestamp|interval"
+  // fields as defined in the flatbuff depending on the type name
+}
+```
+Union:
+```
+{
+  "name" : "union",
+  "mode" : "Sparse|Dense",
+  "typeIds" : [ /* integer */ ]
+}
+```
+Int:
+```
+{
+  "name" : "int",
+  "bitWidth" : /* integer */,
+  "isSigned" : /* boolean */
+}
+```
+FloatingPoint:
+```
+{
+  "name" : "floatingpoint",
+  "precision" : "HALF|SINGLE|DOUBLE"
+}
+```
+Decimal:
+```
+{
+  "name" : "decimal",
+  "precision" : /* integer */,
+  "scale" : /* integer */
+}
+```
+Timestamp:
+```
+{
+  "name" : "timestamp",
+  "unit" : "SECOND|MILLISECOND|MICROSECOND|NANOSECOND"
+}
+```
+Interval:
+```
+{
+  "name" : "interval",
+  "unit" : "YEAR_MONTH|DAY_TIME"
+}
+```
+Schema:
+```
+{
+  "fields" : [
+    /* Field */
+  ]
+}
+```
+
 ## Record data headers
 
 A record batch is a collection of top-level named, equal length Arrow arrays
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index 30f2c68efe0b3..4069e6061b66e 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -16,12 +16,6 @@
  * limitations under the License.
  */
 
-import org.apache.arrow.flatbuf.Field;
-import org.apache.arrow.flatbuf.Type;
-import org.apache.arrow.vector.types.pojo.ArrowType.Int;
-
-import java.util.Objects;
-
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/types/pojo/ArrowType.java" />
 <#include "/@includes/license.ftl" />
@@ -31,13 +25,150 @@
 import com.google.flatbuffers.FlatBufferBuilder;
 import org.apache.arrow.flatbuf.Type;
 
+import java.io.IOException;
 import java.util.Objects;
 
+import org.apache.arrow.flatbuf.Precision;
+import org.apache.arrow.flatbuf.UnionMode;
+import org.apache.arrow.flatbuf.TimeUnit;
+import org.apache.arrow.flatbuf.IntervalUnit;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
 /**
  * Arrow types
  **/
+@JsonTypeInfo(
+  use = JsonTypeInfo.Id.NAME,
+  include = JsonTypeInfo.As.PROPERTY,
+  property = "name")
+@JsonSubTypes({
+<#list arrowTypes.types as type>
+  @JsonSubTypes.Type(value = ArrowType.${type.name}.class, name = "${type.name?remove_ending("_")?lower_case}"),
+</#list>
+})
 public abstract class ArrowType {
 
+  private static class FloatingPointPrecisionSerializer extends JsonSerializer<Short> {
+    @Override
+    public void serialize(Short precision,
+        JsonGenerator jsonGenerator,
+        SerializerProvider serializerProvider)
+            throws IOException, JsonProcessingException {
+      jsonGenerator.writeObject(Precision.name(precision));
+    }
+  }
+
+  private static class FloatingPointPrecisionDeserializer extends JsonDeserializer<Short> {
+    @Override
+    public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+      String name = p.getText();
+      switch(name) {
+        case "HALF":
+          return Precision.HALF;
+        case "SINGLE":
+          return Precision.SINGLE;
+        case "DOUBLE":
+          return Precision.DOUBLE;
+        default:
+          throw new IllegalArgumentException("unknown precision: " + name);
+      }
+    }
+  }
+
+  private static class UnionModeSerializer extends JsonSerializer<Short> {
+    @Override
+    public void serialize(Short mode,
+        JsonGenerator jsonGenerator,
+        SerializerProvider serializerProvider)
+            throws IOException, JsonProcessingException {
+      jsonGenerator.writeObject(UnionMode.name(mode));
+    }
+  }
+
+  private static class UnionModeDeserializer extends JsonDeserializer<Short> {
+    @Override
+    public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+      String name = p.getText();
+      switch(name) {
+        case "Sparse":
+          return UnionMode.Sparse;
+        case "Dense":
+          return UnionMode.Dense;
+        default:
+          throw new IllegalArgumentException("unknown union mode: " + name);
+      }
+    }
+  }
+
+  private static class TimestampUnitSerializer extends JsonSerializer<Short> {
+    @Override
+    public void serialize(Short unit,
+        JsonGenerator jsonGenerator,
+        SerializerProvider serializerProvider)
+            throws IOException, JsonProcessingException {
+      jsonGenerator.writeObject(TimeUnit.name(unit));
+    }
+  }
+
+  private static class TimestampUnitDeserializer extends JsonDeserializer<Short> {
+    @Override
+    public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+      String name = p.getText();
+      switch(name) {
+        case "SECOND":
+          return TimeUnit.SECOND;
+        case "MILLISECOND":
+          return TimeUnit.MILLISECOND;
+        case "MICROSECOND":
+          return TimeUnit.MICROSECOND;
+        case "NANOSECOND":
+          return TimeUnit.NANOSECOND;
+        default:
+          throw new IllegalArgumentException("unknown time unit: " + name);
+      }
+    }
+  }
+
+  private static class IntervalUnitSerializer extends JsonSerializer<Short> {
+    @Override
+    public void serialize(Short unit,
+        JsonGenerator jsonGenerator,
+        SerializerProvider serializerProvider)
+            throws IOException, JsonProcessingException {
+      jsonGenerator.writeObject(IntervalUnit.name(unit));
+    }
+  }
+
+  private static class IntervalUnitDeserializer extends JsonDeserializer<Short> {
+    @Override
+    public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+      String name = p.getText();
+      switch(name) {
+        case "YEAR_MONTH":
+          return IntervalUnit.YEAR_MONTH;
+        case "DAY_TIME":
+          return IntervalUnit.DAY_TIME;
+        default:
+          throw new IllegalArgumentException("unknown interval unit: " + name);
+      }
+    }
+  }
+
+  @JsonIgnore
   public abstract byte getTypeType();
   public abstract int getType(FlatBufferBuilder builder);
   public abstract <T> T accept(ArrowTypeVisitor<T> visitor);
@@ -70,7 +201,12 @@ public static class ${name} extends ArrowType {
     </#list>
 
     <#if type.fields?size != 0>
-    public ${type.name}(<#list type.fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    @JsonCreator
+    public ${type.name}(
+    <#list type.fields as field>
+      <#if field.type == "short"> @JsonDeserialize(using = ${type.name}${field.name?cap_first}Deserializer.class) </#if>@JsonProperty("${field.name}") ${field.type} ${field.name}<#if field_has_next>, </#if>
+    </#list>
+    ) {
       <#list type.fields as field>
       this.${field.name} = ${field.name};
       </#list>
@@ -86,20 +222,29 @@ public byte getTypeType() {
     public int getType(FlatBufferBuilder builder) {
       <#list type.fields as field>
       <#if field.type == "String">
-      int ${field.name} = builder.createString(this.${field.name});
+      int ${field.name} = this.${field.name} == null ? -1 : builder.createString(this.${field.name});
       </#if>
       <#if field.type == "int[]">
-      int ${field.name} = org.apache.arrow.flatbuf.${type.name}.create${field.name?cap_first}Vector(builder, this.${field.name});
+      int ${field.name} = this.${field.name} == null ? -1 : org.apache.arrow.flatbuf.${type.name}.create${field.name?cap_first}Vector(builder, this.${field.name});
       </#if>
       </#list>
       org.apache.arrow.flatbuf.${type.name}.start${type.name}(builder);
       <#list type.fields as field>
-      org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, ${field.name});
+      <#if field.type == "String" || field.type == "int[]">
+      if (this.${field.name} != null) {
+        org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, ${field.name});
+      }
+      <#else>
+      org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, this.${field.name});
+      </#if>
       </#list>
       return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
     }
 
     <#list fields as field>
+      <#if field.type == "short">
+    @JsonSerialize(using = ${type.name}${field.name?cap_first}Serializer.class)
+      </#if>
     public ${field.type} get${field.name?cap_first}() {
       return ${field.name};
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
index 9b7fa45bb9ae3..8fe8e484496cd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
@@ -17,8 +17,15 @@
  */
 package org.apache.arrow.vector.schema;
 
+import java.util.Map;
+
 import org.apache.arrow.flatbuf.VectorType;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+
 public class ArrowVectorType {
 
   public static final ArrowVectorType DATA = new ArrowVectorType(VectorType.DATA);
@@ -26,22 +33,52 @@ public class ArrowVectorType {
   public static final ArrowVectorType VALIDITY = new ArrowVectorType(VectorType.VALIDITY);
   public static final ArrowVectorType TYPE = new ArrowVectorType(VectorType.TYPE);
 
+  private static final Map<String, ArrowVectorType> typeByName;
+  static {
+    ArrowVectorType[] types = { DATA, OFFSET, VALIDITY, TYPE };
+    Builder<String, ArrowVectorType> builder = ImmutableMap.builder();
+    for (ArrowVectorType type: types) {
+      builder.put(type.getName(), type);
+    }
+    typeByName = builder.build();
+  }
+
+  public static ArrowVectorType fromName(String name) {
+    ArrowVectorType type = typeByName.get(name);
+    if (type == null) {
+      throw new IllegalArgumentException("Unknown type " + name);
+    }
+    return type;
+  }
+
   private final short type;
 
   public ArrowVectorType(short type) {
     this.type = type;
+    // validate that the type is valid
+    getName();
+  }
+
+  @JsonCreator
+  private ArrowVectorType(String name) {
+    this.type = fromName(name).type;
   }
 
   public short getType() {
     return type;
   }
 
-  @Override
-  public String toString() {
+  @JsonValue
+  public String getName() {
     try {
       return VectorType.name(type);
     } catch (ArrayIndexOutOfBoundsException e) {
-      return "Unlnown type " + type;
+      throw new IllegalArgumentException("Unknown type " + type);
     }
   }
+
+  @Override
+  public String toString() {
+    return getName();
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 072385a215582..06ae203bf4422 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -19,6 +19,7 @@
 
 import static java.util.Arrays.asList;
 import static org.apache.arrow.flatbuf.Precision.DOUBLE;
+import static org.apache.arrow.flatbuf.Precision.HALF;
 import static org.apache.arrow.flatbuf.Precision.SINGLE;
 import static org.apache.arrow.vector.schema.VectorLayout.booleanVector;
 import static org.apache.arrow.vector.schema.VectorLayout.byteVector;
@@ -49,6 +50,9 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
 
 /**
@@ -110,6 +114,9 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
       @Override public TypeLayout visit(FloatingPoint type) {
         int bitWidth;
         switch (type.getPrecision()) {
+        case HALF:
+          bitWidth = 16;
+          break;
         case SINGLE:
           bitWidth = 32;
           break;
@@ -184,7 +191,8 @@ public TypeLayout visit(Interval type) { // TODO: check size
 
   private final List<VectorLayout> vectors;
 
-  public TypeLayout(List<VectorLayout> vectors) {
+  @JsonCreator
+  public TypeLayout(@JsonProperty("vectors") List<VectorLayout> vectors) {
     super();
     this.vectors = Preconditions.checkNotNull(vectors);
   }
@@ -198,6 +206,7 @@ public List<VectorLayout> getVectors() {
     return vectors;
   }
 
+  @JsonIgnore
   public List<ArrowVectorType> getVectorTypes() {
     List<ArrowVectorType> types = new ArrayList<>(vectors.size());
     for (VectorLayout vector : vectors) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
index 532e9d2328b0f..931c00a02817b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
@@ -22,6 +22,8 @@
 import static org.apache.arrow.vector.schema.ArrowVectorType.TYPE;
 import static org.apache.arrow.vector.schema.ArrowVectorType.VALIDITY;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
 import com.google.flatbuffers.FlatBufferBuilder;
 
@@ -75,7 +77,8 @@ public static VectorLayout byteVector() {
 
   private final ArrowVectorType type;
 
-  private VectorLayout(ArrowVectorType type, int typeBitWidth) {
+  @JsonCreator
+  private VectorLayout(@JsonProperty("type") ArrowVectorType type, @JsonProperty("typeBitWidth") int typeBitWidth) {
     super();
     this.type = Preconditions.checkNotNull(type);
     this.typeBitWidth = (short)typeBitWidth;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index cfa1ed40aeb8c..49ba524ab0a4f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -18,6 +18,7 @@
 package org.apache.arrow.vector.types.pojo;
 
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
 
 import java.util.List;
@@ -26,6 +27,8 @@
 import org.apache.arrow.vector.schema.TypeLayout;
 import org.apache.arrow.vector.schema.VectorLayout;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableList;
 import com.google.flatbuffers.FlatBufferBuilder;
 
@@ -36,20 +39,26 @@ public class Field {
   private final List<Field> children;
   private final TypeLayout typeLayout;
 
-  private Field(String name, boolean nullable, ArrowType type, List<Field> children, TypeLayout typeLayout) {
+  @JsonCreator
+  private Field(
+      @JsonProperty("name") String name,
+      @JsonProperty("nullable") boolean nullable,
+      @JsonProperty("type") ArrowType type,
+      @JsonProperty("children") List<Field> children,
+      @JsonProperty("typeLayout") TypeLayout typeLayout) {
     this.name = name;
     this.nullable = nullable;
-    this.type = type;
+    this.type = checkNotNull(type);
     if (children == null) {
       this.children = ImmutableList.of();
     } else {
       this.children = children;
     }
-    this.typeLayout = typeLayout;
+    this.typeLayout = checkNotNull(typeLayout);
   }
 
   public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
-    this(name, nullable, type, children, TypeLayout.getTypeLayout(type));
+    this(name, nullable, type, children, TypeLayout.getTypeLayout(checkNotNull(type)));
   }
 
   public static Field convertField(org.apache.arrow.flatbuf.Field field) {
@@ -77,7 +86,7 @@ public void validate() {
   }
 
   public int getField(FlatBufferBuilder builder) {
-    int nameOffset = builder.createString(name);
+    int nameOffset = name == null ? -1 : builder.createString(name);
     int typeOffset = type.getType(builder);
     int[] childrenData = new int[children.size()];
     for (int i = 0; i < children.size(); i++) {
@@ -91,7 +100,9 @@ public int getField(FlatBufferBuilder builder) {
     }
     int layoutOffset =  org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData);
     org.apache.arrow.flatbuf.Field.startField(builder);
-    org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
+    if (name != null) {
+      org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
+    }
     org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
     org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeType());
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
index 231be9bd55ca7..44b877eb730d5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -18,19 +18,91 @@
 package org.apache.arrow.vector.types.pojo;
 
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static org.apache.arrow.vector.types.pojo.Field.convertField;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
 import com.google.common.collect.ImmutableList;
 import com.google.flatbuffers.FlatBufferBuilder;
 
+/**
+ * An Arrow Schema
+ */
 public class Schema {
-  private List<Field> fields;
 
-  public Schema(List<Field> fields) {
-    this.fields = ImmutableList.copyOf(fields);
+  /**
+   * @param the list of the fields
+   * @param name the name of the field to return
+   * @return the corresponding field
+   * @throws IllegalArgumentException if the field was not found
+   */
+  public static Field findField(List<Field> fields, String name) {
+    for (Field field : fields) {
+      if (field.getName().equals(name)) {
+        return field;
+      }
+    }
+    throw new IllegalArgumentException(String.format("field %s not found in %s", name, fields));
+  }
+
+  private static final ObjectMapper mapper = new ObjectMapper();
+  private static final ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
+  private static final ObjectReader reader = mapper.readerFor(Schema.class);
+
+  public static Schema fromJSON(String json) throws IOException {
+    return reader.readValue(checkNotNull(json));
+  }
+
+  public static Schema convertSchema(org.apache.arrow.flatbuf.Schema schema) {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    for (int i = 0; i < schema.fieldsLength(); i++) {
+      childrenBuilder.add(convertField(schema.fields(i)));
+    }
+    List<Field> fields = childrenBuilder.build();
+    return new Schema(fields);
+  }
+
+  private final List<Field> fields;
+
+  @JsonCreator
+  public Schema(@JsonProperty("fields") Iterable<Field> fields) {
+    List<Field> fieldList = new ArrayList<>();
+    for (Field field : fields) {
+      fieldList.add(field);
+    }
+    this.fields = Collections.unmodifiableList(fieldList);
+  }
+
+  public List<Field> getFields() {
+    return fields;
+  }
+
+  /**
+   * @param name the name of the field to return
+   * @return the corresponding field
+   */
+  public Field findField(String name) {
+    return findField(getFields(), name);
+  }
+
+  public String toJson() {
+    try {
+      return writer.writeValueAsString(this);
+    } catch (JsonProcessingException e) {
+      // this should not happen
+      throw new RuntimeException(e);
+    }
   }
 
   public int getSchema(FlatBufferBuilder builder) {
@@ -44,9 +116,6 @@ public int getSchema(FlatBufferBuilder builder) {
     return org.apache.arrow.flatbuf.Schema.endSchema(builder);
   }
 
-  public List<Field> getFields() {
-    return fields;
-  }
 
   @Override
   public int hashCode() {
@@ -61,15 +130,6 @@ public boolean equals(Object obj) {
     return Objects.equals(this.fields, ((Schema) obj).fields);
   }
 
-  public static Schema convertSchema(org.apache.arrow.flatbuf.Schema schema) {
-    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
-    for (int i = 0; i < schema.fieldsLength(); i++) {
-      childrenBuilder.add(convertField(schema.fields(i)));
-    }
-    List<Field> fields = childrenBuilder.build();
-    return new Schema(fields);
-  }
-
   @Override
   public String toString() {
     return "Schema" + fields;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
new file mode 100644
index 0000000000000..0ef8be7ef1b8a
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.types.pojo;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.arrow.flatbuf.IntervalUnit;
+import org.apache.arrow.flatbuf.Precision;
+import org.apache.arrow.flatbuf.TimeUnit;
+import org.apache.arrow.flatbuf.UnionMode;
+import org.junit.Test;
+
+public class TestSchema {
+
+  private static Field field(String name, boolean nullable, ArrowType type, Field... children) {
+    return new Field(name, nullable, type, asList(children));
+  }
+
+  private static Field field(String name, ArrowType type, Field... children) {
+    return field(name, true, type, children);
+  }
+
+  @Test
+  public void testAll() throws IOException {
+    Schema schema = new Schema(asList(
+        field("a", false, new ArrowType.Null()),
+        field("b", new ArrowType.Struct_(), field("ba", new ArrowType.Null())),
+        field("c", new ArrowType.List(), field("ca", new ArrowType.Null())),
+        field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new ArrowType.Null())),
+        field("e", new ArrowType.Int(8, true)),
+        field("f", new ArrowType.FloatingPoint(Precision.SINGLE)),
+        field("g", new ArrowType.Utf8()),
+        field("h", new ArrowType.Binary()),
+        field("i", new ArrowType.Bool()),
+        field("j", new ArrowType.Decimal(5, 5)),
+        field("k", new ArrowType.Date()),
+        field("l", new ArrowType.Time()),
+        field("m", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
+        field("n", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        ));
+    roundTrip(schema);
+  }
+
+  @Test
+  public void testUnion() throws IOException {
+    Schema schema = new Schema(asList(
+        field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new ArrowType.Null()))
+        ));
+    roundTrip(schema);
+    contains(schema, "Sparse");
+  }
+
+  @Test
+  public void testTS() throws IOException {
+    Schema schema = new Schema(asList(
+        field("a", new ArrowType.Timestamp(TimeUnit.SECOND)),
+        field("b", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
+        field("c", new ArrowType.Timestamp(TimeUnit.MICROSECOND)),
+        field("d", new ArrowType.Timestamp(TimeUnit.NANOSECOND))
+        ));
+    roundTrip(schema);
+    contains(schema, "SECOND", "MILLISECOND", "MICROSECOND", "NANOSECOND");
+  }
+
+  @Test
+  public void testInterval() throws IOException {
+    Schema schema = new Schema(asList(
+        field("a", new ArrowType.Interval(IntervalUnit.YEAR_MONTH)),
+        field("b", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        ));
+    roundTrip(schema);
+    contains(schema, "YEAR_MONTH", "DAY_TIME");
+  }
+
+  @Test
+  public void testFP() throws IOException {
+    Schema schema = new Schema(asList(
+        field("a", new ArrowType.FloatingPoint(Precision.HALF)),
+        field("b", new ArrowType.FloatingPoint(Precision.SINGLE)),
+        field("c", new ArrowType.FloatingPoint(Precision.DOUBLE))
+        ));
+    roundTrip(schema);
+    contains(schema, "HALF", "SINGLE", "DOUBLE");
+  }
+
+  private void roundTrip(Schema schema) throws IOException {
+    String json = schema.toJson();
+    Schema actual = Schema.fromJSON(json);
+    assertEquals(schema.toJson(), actual.toJson());
+    assertEquals(schema, actual);
+  }
+
+  private void contains(Schema schema, String... s) throws IOException {
+    String json = schema.toJson();
+    for (String string : s) {
+      assertTrue(json + " contains " + string, json.contains(string));
+    }
+  }
+
+}

From 1196691e221c5b00bbf9bf47eead6f684b61fe62 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Fri, 7 Oct 2016 13:12:35 -0700
Subject: [PATCH 0161/1644] ARROW-326: Initialize nested writers in MapWriter
 based on the underlying MapVector's field

Closes #163
---
 .../main/codegen/templates/MapWriters.java    | 22 +++++++++++++++++++
 .../complex/impl/TestPromotableWriter.java    | 21 +++++++++++++++++-
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 7f319a9ca34d8..9fe20df7a1df0 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -56,6 +56,28 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
     }
     </#if>
     this.container = container;
+    for (Field child : container.getField().getChildren()) {
+      switch (Types.getMinorTypeForArrowType(child.getType())) {
+      case MAP:
+        map(child.getName());
+        break;
+      case LIST:
+        list(child.getName());
+        break;
+      case UNION:
+        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), MinorType.UNION, UnionVector.class));
+        fields.put(child.getName().toLowerCase(), writer);
+        break;
+<#list vv.types as type><#list type.minor as minor>
+<#assign lowerName = minor.class?uncap_first />
+<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+<#assign upperName = minor.class?upper_case />
+      case ${upperName}:
+        ${lowerName}(child.getName());
+        break;
+</#list></#list>
+      }
+    }
   }
 
   @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 689c96fda9202..d439cebeda6ac 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -21,13 +21,16 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.DirtyRootAllocator;
 import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -50,7 +53,7 @@ public void terminate() throws Exception {
   @Test
   public void testPromoteToUnion() throws Exception {
 
-    try (final AbstractMapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
+    try (final MapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
          final NullableMapVector v = container.addOrGet("test", MinorType.MAP, NullableMapVector.class);
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
@@ -92,6 +95,22 @@ public void testPromoteToUnion() throws Exception {
 
       assertFalse("4 shouldn't be null", accessor.isNull(4));
       assertEquals(100, accessor.getObject(4));
+
+      container.clear();
+      container.allocateNew();
+
+      ComplexWriterImpl newWriter = new ComplexWriterImpl(EMPTY_SCHEMA_PATH, container);
+
+      MapWriter newMapWriter = newWriter.rootAsMap();
+
+      newMapWriter.start();
+
+      newMapWriter.setPosition(2);
+      newMapWriter.integer("A").writeInt(10);
+
+      Field childField = container.getField().getChildren().get(0).getChildren().get(0);
+      assertEquals("Child field should be union type: " + childField.getName(), Type.Union, childField.getType().getTypeType());
+
     }
   }
 }

From eb1491a96d1fb92bf9c8bfc1acb7a8768af53a7e Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 7 Oct 2016 17:09:00 -0700
Subject: [PATCH 0162/1644] ARROW-325: make TestArrowFile not dependent on
 timezone

Author: Julien Le Dem <julien@dremio.com>

Closes #162 from julienledem/tz and squashes the following commits:

74b5ee8 [Julien Le Dem] ARROW-325: make TestArrowFile not dependent on timezone
---
 .../org/apache/arrow/vector/file/TestArrowFile.java    | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index ad301689cd1e2..7a5e7b58db98c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -40,10 +40,12 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.holders.NullableTimeStampHolder;
 import org.apache.arrow.vector.schema.ArrowBuffer;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -58,14 +60,18 @@ public class TestArrowFile {
   private static final int COUNT = 10;
   private BufferAllocator allocator;
 
+  private DateTimeZone defaultTimezone = DateTimeZone.getDefault();
+
   @Before
   public void init() {
+    DateTimeZone.setDefault(DateTimeZone.forOffsetHours(2));
     allocator = new RootAllocator(Integer.MAX_VALUE);
   }
 
   @After
   public void tearDown() {
     allocator.close();
+    DateTimeZone.setDefault(defaultTimezone);
   }
 
   @Test
@@ -258,7 +264,9 @@ private void validateComplexContent(int count, NullableMapVector parent) {
       Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
       Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
       Assert.assertEquals(i % 3, rootReader.reader("list").size());
-      Assert.assertEquals(i, rootReader.reader("map").reader("timestamp").readDateTime().getMillis() % COUNT);
+      NullableTimeStampHolder h = new NullableTimeStampHolder();
+      rootReader.reader("map").reader("timestamp").read(h);
+      Assert.assertEquals(i, h.value % COUNT);
     }
   }
 

From e7080ef9f1bd91505996edd4e4b7643cc54f6b5f Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 7 Oct 2016 17:14:58 -0700
Subject: [PATCH 0163/1644] [maven-release-plugin] prepare release
 apache-arrow-0.1.0

---
 java/format/pom.xml | 5 ++---
 java/memory/pom.xml | 5 ++---
 java/pom.xml        | 7 +++----
 java/vector/pom.xml | 5 ++---
 4 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 78300047862f4..c81cfed04d9fa 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -9,14 +9,13 @@
   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
   OF ANY KIND, either express or implied. See the License for the specific 
   language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 <modelVersion>4.0.0</modelVersion>
 
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.1-SNAPSHOT</version>
+  <version>0.1.0</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index b91b5981559c3..8af2313079159 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -9,13 +9,12 @@
   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
   OF ANY KIND, either express or implied. See the License for the specific 
   language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1-SNAPSHOT</version>
+    <version>0.1.0</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index fc2c18d0e517d..8ca8eac76a752 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -9,8 +9,7 @@
   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
   OF ANY KIND, either express or implied. See the License for the specific
   language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
@@ -21,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.1-SNAPSHOT</version>
+  <version>0.1.0</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +41,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>HEAD</tag>
+    <tag>apache-arrow-0.1.0</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 08f9bc8da4e2c..ae48d22a6f4e3 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -9,13 +9,12 @@
   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
   OF ANY KIND, either express or implied. See the License for the specific 
   language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1-SNAPSHOT</version>
+    <version>0.1.0</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 17cd7a6466741d22053d132ea306ad6f05351419 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 7 Oct 2016 17:15:08 -0700
Subject: [PATCH 0164/1644] [maven-release-plugin] prepare for next development
 iteration

---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/vector/pom.xml | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index c81cfed04d9fa..eb045d655e982 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.1.0</version>
+  <version>0.1.1-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 8af2313079159..6ed14480860f2 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1.0</version>
+    <version>0.1.1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 8ca8eac76a752..0147de7035794 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.1.0</version>
+  <version>0.1.1-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -41,7 +41,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.1.0</tag>
+    <tag>HEAD</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index ae48d22a6f4e3..1d06bdece01f8 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1.0</version>
+    <version>0.1.1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From a9747ceac2b6399c6acf027de8074d8661d5eb1d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 10 Oct 2016 11:21:49 -0400
Subject: [PATCH 0165/1644] ARROW-312: Read and write Arrow IPC file format
 from Python

This also adds some IO scaffolding for interacting with `arrow::Buffer` objects from Python and assorted additions to help with testing.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #164 from wesm/ARROW-312 and squashes the following commits:

7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc
be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries
5716601 [Wes McKinney] Fix accidental deletion
77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close
316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object
4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc file writes
a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 in an IPC context, to allow zero-copy writing of NumPy arrays
2c49cd4 [Wes McKinney] Some debugging
ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from Python
---
 cpp/src/arrow/io/io-memory-test.cc       |  25 ++
 cpp/src/arrow/io/memory.cc               |  13 +-
 cpp/src/arrow/ipc/CMakeLists.txt         |   7 +
 cpp/src/arrow/ipc/adapter.cc             |  16 +-
 cpp/src/arrow/ipc/util.h                 |   6 +-
 cpp/src/arrow/table-test.cc              |  27 ++
 cpp/src/arrow/table.cc                   |  16 ++
 cpp/src/arrow/table.h                    |   2 +
 cpp/src/arrow/types/primitive-test.cc    |   3 +-
 cpp/src/arrow/util/bit-util.h            |  13 +
 cpp/src/arrow/util/buffer.cc             |  16 +-
 cpp/src/arrow/util/buffer.h              |   1 -
 cpp/src/arrow/util/logging.h             |   4 +-
 python/CMakeLists.txt                    |   8 +-
 python/cmake_modules/FindArrow.cmake     |  11 +
 python/pyarrow/__init__.py               |   3 +-
 python/pyarrow/array.pyx                 |  44 +--
 python/pyarrow/includes/common.pxd       |   4 -
 python/pyarrow/includes/libarrow.pxd     |  29 +-
 python/pyarrow/includes/libarrow_io.pxd  |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |  52 ++++
 python/pyarrow/includes/pyarrow.pxd      |  13 +-
 python/pyarrow/io.pxd                    |   6 +
 python/pyarrow/io.pyx                    | 340 ++++++++++++++---------
 python/pyarrow/ipc.pyx                   | 155 +++++++++++
 python/pyarrow/table.pxd                 |  17 +-
 python/pyarrow/table.pyx                 | 194 ++++++++++---
 python/pyarrow/tests/test_array.py       |   4 +
 python/pyarrow/tests/test_io.py          |  41 +++
 python/pyarrow/tests/test_ipc.py         | 116 ++++++++
 python/pyarrow/tests/test_table.py       |  82 +++---
 python/setup.py                          |   1 +
 python/src/pyarrow/adapters/builtin.cc   |   2 +-
 python/src/pyarrow/adapters/pandas.cc    |   8 +
 python/src/pyarrow/common.cc             |   2 +-
 python/src/pyarrow/common.h              |  20 +-
 python/src/pyarrow/io.cc                 |   6 +-
 37 files changed, 1012 insertions(+), 309 deletions(-)
 create mode 100644 python/pyarrow/includes/libarrow_ipc.pxd
 create mode 100644 python/pyarrow/ipc.pyx
 create mode 100644 python/pyarrow/tests/test_ipc.py

diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 6de35dab59b4f..a49faf3bd8578 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
       IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result));
 }
 
+class TestBufferOutputStream : public ::testing::Test {
+ public:
+  void SetUp() {
+    buffer_.reset(new PoolBuffer(default_memory_pool()));
+    stream_.reset(new BufferOutputStream(buffer_));
+  }
+
+ protected:
+  std::shared_ptr<PoolBuffer> buffer_;
+  std::unique_ptr<OutputStream> stream_;
+};
+
+TEST_F(TestBufferOutputStream, CloseResizes) {
+  std::string data = "data123456";
+
+  const int64_t nbytes = static_cast<int64_t>(data.size());
+  const int K = 100;
+  for (int i = 0; i < K; ++i) {
+    EXPECT_OK(stream_->Write(reinterpret_cast<const uint8_t*>(data.c_str()), nbytes));
+  }
+
+  ASSERT_OK(stream_->Close());
+  ASSERT_EQ(K * nbytes, buffer_->size());
+}
+
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 7d6e02e25b43c..c7d0ae5d56474 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -212,7 +212,11 @@ BufferOutputStream::BufferOutputStream(const std::shared_ptr<ResizableBuffer>& b
       mutable_data_(buffer->mutable_data()) {}
 
 Status BufferOutputStream::Close() {
-  return Status::OK();
+  if (position_ < capacity_) {
+    return buffer_->Resize(position_);
+  } else {
+    return Status::OK();
+  }
 }
 
 Status BufferOutputStream::Tell(int64_t* position) {
@@ -228,8 +232,11 @@ Status BufferOutputStream::Write(const uint8_t* data, int64_t nbytes) {
 }
 
 Status BufferOutputStream::Reserve(int64_t nbytes) {
-  while (position_ + nbytes > capacity_) {
-    int64_t new_capacity = std::max(kBufferMinimumSize, capacity_ * 2);
+  int64_t new_capacity = capacity_;
+  while (position_ + nbytes > new_capacity) {
+    new_capacity = std::max(kBufferMinimumSize, new_capacity * 2);
+  }
+  if (new_capacity > capacity_) {
     RETURN_NOT_OK(buffer_->Resize(new_capacity));
     capacity_ = new_capacity;
   }
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index bde8c5bf73888..8dcd9ac107189 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -57,6 +57,13 @@ SET_TARGET_PROPERTIES(arrow_ipc PROPERTIES
   LINKER_LANGUAGE CXX
   LINK_FLAGS "${ARROW_IPC_LINK_FLAGS}")
 
+if (APPLE)
+  set_target_properties(arrow_ipc
+    PROPERTIES
+    BUILD_WITH_INSTALL_RPATH ON
+    INSTALL_NAME_DIR "@rpath")
+endif()
+
 ADD_ARROW_TEST(ipc-adapter-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-adapter-test
   ${ARROW_IPC_TEST_LINK_LIBS})
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 99974a4a4c7b7..cd8ab53a31d1f 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -162,15 +162,14 @@ class RecordBatchWriter {
     for (size_t i = 0; i < buffers_.size(); ++i) {
       const Buffer* buffer = buffers_[i].get();
       int64_t size = 0;
+      int64_t padding = 0;
 
       // The buffer might be null if we are handling zero row lengths.
       if (buffer) {
-        // We use capacity here, because size might not reflect the padding
-        // requirements of buffers but capacity always should.
-        size = buffer->capacity();
-        // check that padding is appropriate
-        RETURN_NOT_OK(CheckMultipleOf64(size));
+        size = buffer->size();
+        padding = util::RoundUpToMultipleOf64(size) - size;
       }
+
       // TODO(wesm): We currently have no notion of shared memory page id's,
       // but we've included it in the metadata IDL for when we have it in the
       // future. Use page=0 for now
@@ -179,12 +178,17 @@ class RecordBatchWriter {
       // are using from any OS-level shared memory. The thought is that systems
       // may (in the future) associate integer page id's with physical memory
       // pages (according to whatever is the desired shared memory mechanism)
-      buffer_meta_.push_back(flatbuf::Buffer(0, position, size));
+      buffer_meta_.push_back(flatbuf::Buffer(0, position, size + padding));
 
       if (size > 0) {
         RETURN_NOT_OK(dst->Write(buffer->data(), size));
         position += size;
       }
+
+      if (padding > 0) {
+        RETURN_NOT_OK(dst->Write(kPaddingBytes, padding));
+        position += padding;
+      }
     }
 
     *body_end_offset = position;
diff --git a/cpp/src/arrow/ipc/util.h b/cpp/src/arrow/ipc/util.h
index 94079a3827777..9000d1bb0c6c3 100644
--- a/cpp/src/arrow/ipc/util.h
+++ b/cpp/src/arrow/ipc/util.h
@@ -29,7 +29,11 @@ namespace ipc {
 
 // Align on 8-byte boundaries
 static constexpr int kArrowAlignment = 8;
-static constexpr uint8_t kPaddingBytes[kArrowAlignment] = {0};
+
+// Buffers are padded to 64-byte boundaries (for SIMD)
+static constexpr int kArrowBufferAlignment = 64;
+
+static constexpr uint8_t kPaddingBytes[kArrowBufferAlignment] = {0};
 
 static inline int64_t PaddedLength(int64_t nbytes, int64_t alignment = kArrowAlignment) {
   return ((nbytes + alignment - 1) / alignment) * alignment;
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 385e7d831500a..743fb669700ea 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -123,4 +123,31 @@ TEST_F(TestTable, InvalidColumns) {
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
 }
 
+class TestRecordBatch : public TestBase {};
+
+TEST_F(TestRecordBatch, Equals) {
+  const int length = 10;
+
+  auto f0 = std::make_shared<Field>("f0", INT32);
+  auto f1 = std::make_shared<Field>("f1", UINT8);
+  auto f2 = std::make_shared<Field>("f2", INT16);
+
+  vector<shared_ptr<Field>> fields = {f0, f1, f2};
+  auto schema = std::make_shared<Schema>(fields);
+
+  auto a0 = MakePrimitive<Int32Array>(length);
+  auto a1 = MakePrimitive<UInt8Array>(length);
+  auto a2 = MakePrimitive<Int16Array>(length);
+
+  RecordBatch b1(schema, length, {a0, a1, a2});
+  RecordBatch b2(schema, 5, {a0, a1, a2});
+  RecordBatch b3(schema, length, {a0, a1});
+  RecordBatch b4(schema, length, {a0, a1, a1});
+
+  ASSERT_TRUE(b1.Equals(b1));
+  ASSERT_FALSE(b1.Equals(b2));
+  ASSERT_FALSE(b1.Equals(b3));
+  ASSERT_FALSE(b1.Equals(b4));
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 3a250df81d0fb..af84f27eab557 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -21,6 +21,7 @@
 #include <memory>
 #include <sstream>
 
+#include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/schema.h"
 #include "arrow/util/status.h"
@@ -35,6 +36,21 @@ const std::string& RecordBatch::column_name(int i) const {
   return schema_->field(i)->name;
 }
 
+bool RecordBatch::Equals(const RecordBatch& other) const {
+  if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
+    return false;
+  }
+
+  for (int i = 0; i < num_columns(); ++i) {
+    if (!column(i)->Equals(other.column(i))) { return false; }
+  }
+
+  return true;
+}
+
+// ----------------------------------------------------------------------
+// Table methods
+
 Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
     const std::vector<std::shared_ptr<Column>>& columns)
     : name_(name), schema_(schema), columns_(columns) {
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 36b3c8ecaf43f..1a856c8a436d5 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -43,6 +43,8 @@ class ARROW_EXPORT RecordBatch {
   RecordBatch(const std::shared_ptr<Schema>& schema, int32_t num_rows,
       const std::vector<std::shared_ptr<Array>>& columns);
 
+  bool Equals(const RecordBatch& other) const;
+
   // @returns: the table's schema
   const std::shared_ptr<Schema>& schema() const { return schema_; }
 
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index ffebb9269bdc3..87eb0fe3a8bf7 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -238,8 +238,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 }
 
 typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
-    PInt32, PInt64, PFloat, PDouble>
-    Primitives;
+    PInt32, PInt64, PFloat, PDouble> Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 873a1959865f5..3087ce7784d11 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -19,6 +19,7 @@
 #define ARROW_UTIL_BIT_UTIL_H
 
 #include <cstdint>
+#include <limits>
 #include <memory>
 #include <vector>
 
@@ -77,6 +78,18 @@ static inline bool is_multiple_of_64(int64_t n) {
   return (n & 63) == 0;
 }
 
+inline int64_t RoundUpToMultipleOf64(int64_t num) {
+  // TODO(wesm): is this definitely needed?
+  // DCHECK_GE(num, 0);
+  constexpr int64_t round_to = 64;
+  constexpr int64_t force_carry_addend = round_to - 1;
+  constexpr int64_t truncate_bitmask = ~(round_to - 1);
+  constexpr int64_t max_roundable_num = std::numeric_limits<int64_t>::max() - round_to;
+  if (num <= max_roundable_num) { return (num + force_carry_addend) & truncate_bitmask; }
+  // handle overflow case.  This should result in a malloc error upstream
+  return num;
+}
+
 void bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits);
 ARROW_EXPORT Status bytes_to_bits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
index 703ef8384ac07..6faa048e4e52b 100644
--- a/cpp/src/arrow/util/buffer.cc
+++ b/cpp/src/arrow/util/buffer.cc
@@ -20,25 +20,13 @@
 #include <cstdint>
 #include <limits>
 
+#include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/memory-pool.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
 
-namespace {
-int64_t RoundUpToMultipleOf64(int64_t num) {
-  DCHECK_GE(num, 0);
-  constexpr int64_t round_to = 64;
-  constexpr int64_t force_carry_addend = round_to - 1;
-  constexpr int64_t truncate_bitmask = ~(round_to - 1);
-  constexpr int64_t max_roundable_num = std::numeric_limits<int64_t>::max() - round_to;
-  if (num <= max_roundable_num) { return (num + force_carry_addend) & truncate_bitmask; }
-  // handle overflow case.  This should result in a malloc error upstream
-  return num;
-}
-}  // namespace
-
 Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size) {
   data_ = parent->data() + offset;
   size_ = size;
@@ -64,7 +52,7 @@ PoolBuffer::~PoolBuffer() {
 Status PoolBuffer::Reserve(int64_t new_capacity) {
   if (!mutable_data_ || new_capacity > capacity_) {
     uint8_t* new_data;
-    new_capacity = RoundUpToMultipleOf64(new_capacity);
+    new_capacity = util::RoundUpToMultipleOf64(new_capacity);
     if (mutable_data_) {
       RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
       memcpy(new_data, mutable_data_, size_);
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 1aeebc69b4e14..01e4259c31fd2 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -23,7 +23,6 @@
 #include <cstring>
 #include <memory>
 
-#include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/status.h"
 #include "arrow/util/visibility.h"
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index b22f07dd6345f..06ee8411e283c 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -118,9 +118,9 @@ class CerrLog {
 class FatalLog : public CerrLog {
  public:
   explicit FatalLog(int /* severity */)  // NOLINT
-      : CerrLog(ARROW_FATAL){}           // NOLINT
+      : CerrLog(ARROW_FATAL) {}          // NOLINT
 
-            [[noreturn]] ~FatalLog() {
+  [[noreturn]] ~FatalLog() {
     if (has_logged_) { std::cerr << std::endl; }
     std::exit(1);
   }
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 77a771ab21c06..55f6d0543a108 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -352,6 +352,8 @@ ADD_THIRDPARTY_LIB(arrow
   SHARED_LIB ${ARROW_SHARED_LIB})
 ADD_THIRDPARTY_LIB(arrow_io
   SHARED_LIB ${ARROW_IO_SHARED_LIB})
+ADD_THIRDPARTY_LIB(arrow_ipc
+  SHARED_LIB ${ARROW_IPC_SHARED_LIB})
 
 ############################################################
 # Linker setup
@@ -415,6 +417,8 @@ if (UNIX)
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 endif()
 
+SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
 add_subdirectory(src/pyarrow)
 add_subdirectory(src/pyarrow/util)
 
@@ -423,6 +427,7 @@ set(CYTHON_EXTENSIONS
   config
   error
   io
+  ipc
   scalar
   schema
   table
@@ -442,6 +447,7 @@ set(PYARROW_SRCS
 set(LINK_LIBS
   arrow
   arrow_io
+  arrow_ipc
 )
 
 if(PARQUET_FOUND AND PARQUET_ARROW_FOUND)
@@ -455,8 +461,6 @@ if(PARQUET_FOUND AND PARQUET_ARROW_FOUND)
     parquet)
 endif()
 
-SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-
 add_library(pyarrow SHARED
   ${PYARROW_SRCS})
 target_link_libraries(pyarrow ${LINK_LIBS})
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 9919746520b4c..3c359aac55309 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -47,10 +47,16 @@ find_library(ARROW_IO_LIB_PATH NAMES arrow_io
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
+find_library(ARROW_IPC_LIB_PATH NAMES arrow_ipc
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+
 if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
   set(ARROW_IO_LIB_NAME libarrow_io)
+  set(ARROW_IPC_LIB_NAME libarrow_ipc)
 
   set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
@@ -58,9 +64,14 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
 
   set(ARROW_IO_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IO_LIB_NAME}.a)
   set(ARROW_IO_SHARED_LIB ${ARROW_LIBS}/${ARROW_IO_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+  set(ARROW_IPC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IPC_LIB_NAME}.a)
+  set(ARROW_IPC_SHARED_LIB ${ARROW_LIBS}/${ARROW_IPC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
     message(STATUS "Found the Arrow IO library: ${ARROW_IO_LIB_PATH}")
+    message(STATUS "Found the Arrow IPC library: ${ARROW_IPC_LIB_PATH}")
   endif ()
 else ()
   if (NOT Arrow_FIND_QUIETLY)
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 7561f6d46df21..8b131aaa8f4af 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -41,5 +41,4 @@
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
-from pyarrow.array import RowBatch
-from pyarrow.table import Column, Table, from_pandas_dataframe
+from pyarrow.table import Column, RecordBatch, Table, from_pandas_dataframe
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index cdbe73ad21f7d..84ab4a48c9b65 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -37,7 +37,7 @@ import pyarrow.schema as schema
 
 
 def total_allocated_bytes():
-    cdef MemoryPool* pool = pyarrow.GetMemoryPool()
+    cdef MemoryPool* pool = pyarrow.get_memory_pool()
     return pool.bytes_allocated()
 
 
@@ -243,12 +243,14 @@ def from_pandas_series(object series, object mask=None, timestamps_to_ms=False):
         series_values = series_values.astype('datetime64[ms]')
 
     if mask is None:
-        check_status(pyarrow.PandasToArrow(pyarrow.GetMemoryPool(),
-                                           series_values, &out))
+        with nogil:
+            check_status(pyarrow.PandasToArrow(pyarrow.get_memory_pool(),
+                                               series_values, &out))
     else:
         mask = series_as_ndarray(mask)
-        check_status(pyarrow.PandasMaskedToArrow(
-            pyarrow.GetMemoryPool(), series_values, mask, &out))
+        with nogil:
+            check_status(pyarrow.PandasMaskedToArrow(
+                pyarrow.get_memory_pool(), series_values, mask, &out))
 
     return box_arrow_array(out)
 
@@ -262,35 +264,3 @@ cdef object series_as_ndarray(object obj):
         result = obj
 
     return result
-
-#----------------------------------------------------------------------
-# Table-like data structures
-
-cdef class RowBatch:
-    """
-
-    """
-    cdef readonly:
-        Schema schema
-        int num_rows
-        list arrays
-
-    def __cinit__(self, Schema schema, int num_rows, list arrays):
-        self.schema = schema
-        self.num_rows = num_rows
-        self.arrays = arrays
-
-        if len(self.schema) != len(arrays):
-            raise ValueError('Mismatch number of data arrays and '
-                             'schema fields')
-
-    def __len__(self):
-        return self.num_rows
-
-    property num_columns:
-
-        def __get__(self):
-            return len(self.arrays)
-
-    def __getitem__(self, i):
-        return self.arrays[i]
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 133797bc37b5c..05c0123ee7b7e 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -47,7 +47,3 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsKeyError()
         c_bool IsNotImplemented()
         c_bool IsInvalid()
-
-    cdef cppclass Buffer:
-        uint8_t* data()
-        int64_t size()
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 854d07d691dbf..3ae1789170303 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -54,6 +54,18 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass MemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
 
+    cdef cppclass CBuffer" arrow::Buffer":
+        uint8_t* data()
+        int64_t size()
+
+    cdef cppclass ResizableBuffer(CBuffer):
+        CStatus Resize(int64_t nbytes)
+        CStatus Reserve(int64_t nbytes)
+
+    cdef cppclass PoolBuffer(ResizableBuffer):
+        PoolBuffer()
+        PoolBuffer(MemoryPool*)
+
     cdef MemoryPool* default_memory_pool()
 
     cdef cppclass CListType" arrow::ListType"(CDataType):
@@ -149,6 +161,21 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         const shared_ptr[CDataType]& type()
         const shared_ptr[CChunkedArray]& data()
 
+    cdef cppclass CRecordBatch" arrow::RecordBatch":
+        CRecordBatch(const shared_ptr[CSchema]& schema, int32_t num_rows,
+                     const vector[shared_ptr[CArray]]& columns)
+
+        c_bool Equals(const CRecordBatch& other)
+
+        const shared_ptr[CSchema]& schema()
+        const shared_ptr[CArray]& column(int i)
+        const c_string& column_name(int i)
+
+        const vector[shared_ptr[CArray]]& columns()
+
+        int num_columns()
+        int32_t num_rows()
+
     cdef cppclass CTable" arrow::Table":
         CTable(const c_string& name, const shared_ptr[CSchema]& schema,
                const vector[shared_ptr[CColumn]]& columns)
@@ -186,7 +213,7 @@ cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
         MessageType_DICTIONARY_BATCH" arrow::ipc::Message::DICTIONARY_BATCH"
 
     cdef cppclass Message:
-        CStatus Open(const shared_ptr[Buffer]& buf,
+        CStatus Open(const shared_ptr[CBuffer]& buf,
                      shared_ptr[Message]* out)
         int64_t body_length()
         MessageType type()
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 56d8d4cf61494..8074915508fbe 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -18,7 +18,7 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport MemoryPool
+from pyarrow.includes.libarrow cimport *
 
 cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
     enum FileMode" arrow::io::FileMode::type":
@@ -36,7 +36,7 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
         FileMode mode()
 
     cdef cppclass Readable:
-        CStatus ReadB" Read"(int64_t nbytes, shared_ptr[Buffer]* out)
+        CStatus ReadB" Read"(int64_t nbytes, shared_ptr[CBuffer]* out)
         CStatus Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out)
 
     cdef cppclass Seekable:
@@ -57,7 +57,7 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
         CStatus ReadAt(int64_t position, int64_t nbytes,
                        int64_t* bytes_read, uint8_t* buffer)
         CStatus ReadAt(int64_t position, int64_t nbytes,
-                       int64_t* bytes_read, shared_ptr[Buffer]* out)
+                       int64_t* bytes_read, shared_ptr[CBuffer]* out)
 
     cdef cppclass WriteableFileInterface(OutputStream, Seekable):
         CStatus WriteAt(int64_t position, const uint8_t* data,
@@ -143,9 +143,9 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
 
 
 cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
-    cdef cppclass BufferReader(ReadableFileInterface):
-        BufferReader(const uint8_t* data, int64_t nbytes)
+    cdef cppclass CBufferReader" arrow::io::BufferReader"\
+        (ReadableFileInterface):
+        CBufferReader(const uint8_t* data, int64_t nbytes)
 
     cdef cppclass BufferOutputStream(OutputStream):
-        # TODO(wesm)
-        pass
+        BufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
new file mode 100644
index 0000000000000..eda5b9bae9e31
--- /dev/null
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport (MemoryPool, CArray, CSchema,
+                                        CRecordBatch)
+from pyarrow.includes.libarrow_io cimport (OutputStream, ReadableFileInterface)
+
+cdef extern from "arrow/ipc/file.h" namespace "arrow::ipc" nogil:
+
+    cdef cppclass CFileWriter " arrow::ipc::FileWriter":
+        @staticmethod
+        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
+                     shared_ptr[CFileWriter]* out)
+
+        CStatus WriteRecordBatch(const vector[shared_ptr[CArray]]& columns,
+                                 int32_t num_rows)
+
+        CStatus Close()
+
+    cdef cppclass CFileReader " arrow::ipc::FileReader":
+
+        @staticmethod
+        CStatus Open(const shared_ptr[ReadableFileInterface]& file,
+                     shared_ptr[CFileReader]* out)
+
+        @staticmethod
+        CStatus Open2" Open"(const shared_ptr[ReadableFileInterface]& file,
+                     int64_t footer_offset, shared_ptr[CFileReader]* out)
+
+        const shared_ptr[CSchema]& schema()
+
+        int num_dictionaries()
+        int num_record_batches()
+
+        CStatus GetRecordBatch(int i, shared_ptr[CRecordBatch]* batch)
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 4c971665ff6aa..2fa5a7d63256a 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -18,8 +18,8 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CColumn, CDataType, CStatus,
-                                        Type, MemoryPool)
+from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn,
+                                        CDataType, CStatus, Type, MemoryPool)
 
 cimport pyarrow.includes.libarrow_io as arrow_io
 
@@ -53,7 +53,12 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     PyStatus ArrowToPandas(const shared_ptr[CColumn]& arr, object py_ref,
                            PyObject** out)
 
-    MemoryPool* GetMemoryPool()
+    MemoryPool* get_memory_pool()
+
+
+cdef extern from "pyarrow/common.h" namespace "pyarrow" nogil:
+    cdef cppclass PyBytesBuffer(CBuffer):
+        PyBytesBuffer(object o)
 
 
 cdef extern from "pyarrow/io.h" namespace "pyarrow" nogil:
@@ -63,5 +68,5 @@ cdef extern from "pyarrow/io.h" namespace "pyarrow" nogil:
     cdef cppclass PyOutputStream(arrow_io.OutputStream):
         PyOutputStream(object fo)
 
-    cdef cppclass PyBytesReader(arrow_io.BufferReader):
+    cdef cppclass PyBytesReader(arrow_io.CBufferReader):
         PyBytesReader(object fo)
diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
index 1dbb3fd76bbfd..d6966cdaadd3a 100644
--- a/python/pyarrow/io.pxd
+++ b/python/pyarrow/io.pxd
@@ -22,6 +22,11 @@ from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.libarrow_io cimport (ReadableFileInterface,
                                            OutputStream)
 
+cdef class Buffer:
+    cdef:
+        shared_ptr[CBuffer] buffer
+
+    cdef init(self, const shared_ptr[CBuffer]& buffer)
 
 cdef class NativeFile:
     cdef:
@@ -29,6 +34,7 @@ cdef class NativeFile:
         shared_ptr[OutputStream] wr_file
         bint is_readonly
         bint is_open
+        bint own_file
 
     # By implementing these "virtual" functions (all functions in Cython
     # extension classes are technically virtual in the C++ sense) we can expose
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index e6e2b625e87ca..00a492fc0baf2 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -36,6 +36,217 @@ import re
 import sys
 import threading
 
+
+cdef class NativeFile:
+
+    def __cinit__(self):
+        self.is_open = False
+        self.own_file = False
+
+    def __dealloc__(self):
+        if self.is_open and self.own_file:
+            self.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close()
+
+    def close(self):
+        if self.is_open:
+            with nogil:
+                if self.is_readonly:
+                    check_cstatus(self.rd_file.get().Close())
+                else:
+                    check_cstatus(self.wr_file.get().Close())
+        self.is_open = False
+
+    cdef read_handle(self, shared_ptr[ReadableFileInterface]* file):
+        self._assert_readable()
+        file[0] = <shared_ptr[ReadableFileInterface]> self.rd_file
+
+    cdef write_handle(self, shared_ptr[OutputStream]* file):
+        self._assert_writeable()
+        file[0] = <shared_ptr[OutputStream]> self.wr_file
+
+    def _assert_readable(self):
+        if not self.is_readonly:
+            raise IOError("only valid on readonly files")
+
+        if not self.is_open:
+            raise IOError("file not open")
+
+    def _assert_writeable(self):
+        if self.is_readonly:
+            raise IOError("only valid on writeonly files")
+
+        if not self.is_open:
+            raise IOError("file not open")
+
+    def size(self):
+        cdef int64_t size
+        self._assert_readable()
+        with nogil:
+            check_cstatus(self.rd_file.get().GetSize(&size))
+        return size
+
+    def tell(self):
+        cdef int64_t position
+        with nogil:
+            if self.is_readonly:
+                check_cstatus(self.rd_file.get().Tell(&position))
+            else:
+                check_cstatus(self.wr_file.get().Tell(&position))
+        return position
+
+    def seek(self, int64_t position):
+        self._assert_readable()
+        with nogil:
+            check_cstatus(self.rd_file.get().Seek(position))
+
+    def write(self, data):
+        """
+        Write bytes-like (unicode, encoded to UTF-8) to file
+        """
+        self._assert_writeable()
+
+        data = tobytes(data)
+
+        cdef const uint8_t* buf = <const uint8_t*> cp.PyBytes_AS_STRING(data)
+        cdef int64_t bufsize = len(data)
+        with nogil:
+            check_cstatus(self.wr_file.get().Write(buf, bufsize))
+
+    def read(self, int nbytes):
+        cdef:
+            int64_t bytes_read = 0
+            uint8_t* buf
+            shared_ptr[CBuffer] out
+
+        self._assert_readable()
+
+        with nogil:
+            check_cstatus(self.rd_file.get()
+                          .ReadB(nbytes, &out))
+
+        result = cp.PyBytes_FromStringAndSize(
+            <const char*>out.get().data(), out.get().size())
+
+        return result
+
+
+# ----------------------------------------------------------------------
+# Python file-like objects
+
+cdef class PythonFileInterface(NativeFile):
+    cdef:
+        object handle
+
+    def __cinit__(self, handle, mode='w'):
+        self.handle = handle
+
+        if mode.startswith('w'):
+            self.wr_file.reset(new pyarrow.PyOutputStream(handle))
+            self.is_readonly = 0
+        elif mode.startswith('r'):
+            self.rd_file.reset(new pyarrow.PyReadableFile(handle))
+            self.is_readonly = 1
+        else:
+            raise ValueError('Invalid file mode: {0}'.format(mode))
+
+        self.is_open = True
+
+
+cdef class BytesReader(NativeFile):
+    cdef:
+        object obj
+
+    def __cinit__(self, obj):
+        if not isinstance(obj, bytes):
+            raise ValueError('Must pass bytes object')
+
+        self.obj = obj
+        self.is_readonly = 1
+        self.is_open = True
+
+        self.rd_file.reset(new pyarrow.PyBytesReader(obj))
+
+# ----------------------------------------------------------------------
+# Arrow buffers
+
+
+cdef class Buffer:
+
+    def __cinit__(self):
+        pass
+
+    cdef init(self, const shared_ptr[CBuffer]& buffer):
+        self.buffer = buffer
+
+    def __len__(self):
+        return self.size
+
+    property size:
+
+        def __get__(self):
+            return self.buffer.get().size()
+
+    def __getitem__(self, key):
+        # TODO(wesm): buffer slicing
+        raise NotImplementedError
+
+    def to_pybytes(self):
+        return cp.PyBytes_FromStringAndSize(
+            <const char*>self.buffer.get().data(),
+            self.buffer.get().size())
+
+
+cdef shared_ptr[PoolBuffer] allocate_buffer():
+    cdef shared_ptr[PoolBuffer] result
+    result.reset(new PoolBuffer(pyarrow.get_memory_pool()))
+    return result
+
+
+cdef class InMemoryOutputStream(NativeFile):
+
+    cdef:
+        shared_ptr[PoolBuffer] buffer
+
+    def __cinit__(self):
+        self.buffer = allocate_buffer()
+        self.wr_file.reset(new BufferOutputStream(
+            <shared_ptr[ResizableBuffer]> self.buffer))
+        self.is_readonly = 0
+        self.is_open = True
+
+    def get_result(self):
+        cdef Buffer result = Buffer()
+
+        check_cstatus(self.wr_file.get().Close())
+        result.init(<shared_ptr[CBuffer]> self.buffer)
+
+        self.is_open = False
+        return result
+
+
+def buffer_from_bytes(object obj):
+    """
+    Construct an Arrow buffer from a Python bytes object
+    """
+    if not isinstance(obj, bytes):
+        raise ValueError('Must pass bytes object')
+
+    cdef shared_ptr[CBuffer] buf
+    buf.reset(new pyarrow.PyBytesBuffer(obj))
+
+    cdef Buffer result = Buffer()
+    result.init(buf)
+    return result
+
+# ----------------------------------------------------------------------
+# HDFS IO implementation
+
 _HDFS_PATH_RE = re.compile('hdfs://(.*):(\d+)(.*)')
 
 try:
@@ -274,6 +485,7 @@ cdef class HdfsClient:
         out.buffer_size = c_buffer_size
         out.parent = self
         out.is_open = True
+        out.own_file = True
 
         return out
 
@@ -322,134 +534,6 @@ cdef class HdfsClient:
         f.download(stream)
 
 
-cdef class NativeFile:
-
-    def __cinit__(self):
-        self.is_open = False
-
-    def __dealloc__(self):
-        if self.is_open:
-            self.close()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, tb):
-        self.close()
-
-    def close(self):
-        if self.is_open:
-            with nogil:
-                if self.is_readonly:
-                    check_cstatus(self.rd_file.get().Close())
-                else:
-                    check_cstatus(self.wr_file.get().Close())
-        self.is_open = False
-
-    cdef read_handle(self, shared_ptr[ReadableFileInterface]* file):
-        self._assert_readable()
-        file[0] = <shared_ptr[ReadableFileInterface]> self.rd_file
-
-    cdef write_handle(self, shared_ptr[OutputStream]* file):
-        self._assert_writeable()
-        file[0] = <shared_ptr[OutputStream]> self.wr_file
-
-    def _assert_readable(self):
-        if not self.is_readonly:
-            raise IOError("only valid on readonly files")
-
-    def _assert_writeable(self):
-        if self.is_readonly:
-            raise IOError("only valid on writeonly files")
-
-    def size(self):
-        cdef int64_t size
-        self._assert_readable()
-        with nogil:
-            check_cstatus(self.rd_file.get().GetSize(&size))
-        return size
-
-    def tell(self):
-        cdef int64_t position
-        with nogil:
-            if self.is_readonly:
-                check_cstatus(self.rd_file.get().Tell(&position))
-            else:
-                check_cstatus(self.wr_file.get().Tell(&position))
-        return position
-
-    def seek(self, int64_t position):
-        self._assert_readable()
-        with nogil:
-            check_cstatus(self.rd_file.get().Seek(position))
-
-    def write(self, data):
-        """
-        Write bytes-like (unicode, encoded to UTF-8) to file
-        """
-        self._assert_writeable()
-
-        data = tobytes(data)
-
-        cdef const uint8_t* buf = <const uint8_t*> cp.PyBytes_AS_STRING(data)
-        cdef int64_t bufsize = len(data)
-        with nogil:
-            check_cstatus(self.wr_file.get().Write(buf, bufsize))
-
-    def read(self, int nbytes):
-        cdef:
-            int64_t bytes_read = 0
-            uint8_t* buf
-            shared_ptr[Buffer] out
-
-        self._assert_readable()
-
-        with nogil:
-            check_cstatus(self.rd_file.get()
-                          .ReadB(nbytes, &out))
-
-        result = cp.PyBytes_FromStringAndSize(
-            <const char*>out.get().data(), out.get().size())
-
-        return result
-
-
-# ----------------------------------------------------------------------
-# Python file-like objects
-
-cdef class PythonFileInterface(NativeFile):
-    cdef:
-        object handle
-
-    def __cinit__(self, handle, mode='w'):
-        self.handle = handle
-
-        if mode.startswith('w'):
-            self.wr_file.reset(new pyarrow.PyOutputStream(handle))
-            self.is_readonly = 0
-        elif mode.startswith('r'):
-            self.rd_file.reset(new pyarrow.PyReadableFile(handle))
-            self.is_readonly = 1
-        else:
-            raise ValueError('Invalid file mode: {0}'.format(mode))
-
-        self.is_open = True
-
-
-cdef class BytesReader(NativeFile):
-    cdef:
-        object obj
-
-    def __cinit__(self, obj):
-        if not isinstance(obj, bytes):
-            raise ValueError('Must pass bytes object')
-
-        self.obj = obj
-        self.is_readonly = 1
-        self.is_open = True
-
-        self.rd_file.reset(new pyarrow.PyBytesReader(obj))
-
 # ----------------------------------------------------------------------
 # Specialization for HDFS
 
diff --git a/python/pyarrow/ipc.pyx b/python/pyarrow/ipc.pyx
new file mode 100644
index 0000000000000..f8da3a70da819
--- /dev/null
+++ b/python/pyarrow/ipc.pyx
@@ -0,0 +1,155 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cython wrappers for arrow::ipc
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.libarrow_io cimport *
+from pyarrow.includes.libarrow_ipc cimport *
+cimport pyarrow.includes.pyarrow as pyarrow
+
+from pyarrow.error cimport check_cstatus
+from pyarrow.io cimport NativeFile
+from pyarrow.schema cimport Schema
+from pyarrow.table cimport RecordBatch
+
+from pyarrow.compat import frombytes, tobytes
+import pyarrow.io as io
+
+cimport cpython as cp
+
+
+cdef get_reader(source, shared_ptr[ReadableFileInterface]* reader):
+    cdef NativeFile nf
+
+    if isinstance(source, bytes):
+        source = io.BytesReader(source)
+    elif not isinstance(source, io.NativeFile) and hasattr(source, 'read'):
+        # Optimistically hope this is file-like
+        source = io.PythonFileInterface(source, mode='r')
+
+    if isinstance(source, NativeFile):
+        nf = source
+
+        # TODO: what about read-write sources (e.g. memory maps)
+        if not nf.is_readonly:
+            raise IOError('Native file is not readable')
+
+        nf.read_handle(reader)
+    else:
+        raise TypeError('Unable to read from object of type: {0}'
+                        .format(type(source)))
+
+
+cdef get_writer(source, shared_ptr[OutputStream]* writer):
+    cdef NativeFile nf
+
+    if not isinstance(source, io.NativeFile) and hasattr(source, 'write'):
+        # Optimistically hope this is file-like
+        source = io.PythonFileInterface(source, mode='w')
+
+    if isinstance(source, io.NativeFile):
+        nf = source
+
+        if nf.is_readonly:
+            raise IOError('Native file is not writeable')
+
+        nf.write_handle(writer)
+    else:
+        raise TypeError('Unable to read from object of type: {0}'
+                        .format(type(source)))
+
+
+cdef class ArrowFileWriter:
+    cdef:
+        shared_ptr[CFileWriter] writer
+        shared_ptr[OutputStream] sink
+        bint closed
+
+    def __cinit__(self, sink, Schema schema):
+        self.closed = True
+        get_writer(sink, &self.sink)
+
+        with nogil:
+            check_cstatus(CFileWriter.Open(self.sink.get(), schema.sp_schema,
+                                           &self.writer))
+
+        self.closed = False
+
+    def __dealloc__(self):
+        if not self.closed:
+            self.close()
+
+    def write_record_batch(self, RecordBatch batch):
+        cdef CRecordBatch* bptr = batch.batch
+        with nogil:
+            check_cstatus(self.writer.get()
+                          .WriteRecordBatch(bptr.columns(), bptr.num_rows()))
+
+    def close(self):
+        with nogil:
+            check_cstatus(self.writer.get().Close())
+        self.closed = True
+
+
+cdef class ArrowFileReader:
+    cdef:
+        shared_ptr[CFileReader] reader
+
+    def __cinit__(self, source, footer_offset=None):
+        cdef shared_ptr[ReadableFileInterface] reader
+        get_reader(source, &reader)
+
+        cdef int64_t offset = 0
+        if footer_offset is not None:
+            offset = footer_offset
+
+        with nogil:
+            if offset != 0:
+                check_cstatus(CFileReader.Open2(reader, offset, &self.reader))
+            else:
+                check_cstatus(CFileReader.Open(reader, &self.reader))
+
+    property num_dictionaries:
+
+        def __get__(self):
+            return self.reader.get().num_dictionaries()
+
+    property num_record_batches:
+
+        def __get__(self):
+            return self.reader.get().num_record_batches()
+
+    def get_record_batch(self, int i):
+        cdef:
+            shared_ptr[CRecordBatch] batch
+            RecordBatch result
+
+        if i < 0 or i >= self.num_record_batches:
+            raise ValueError('Batch number {0} out of range'.format(i))
+
+        with nogil:
+            check_cstatus(self.reader.get().GetRecordBatch(i, &batch))
+
+        result = RecordBatch()
+        result.init(batch)
+
+        return result
diff --git a/python/pyarrow/table.pxd b/python/pyarrow/table.pxd
index 0a5c122c95cff..79c9ae3b0a194 100644
--- a/python/pyarrow/table.pxd
+++ b/python/pyarrow/table.pxd
@@ -16,7 +16,10 @@
 # under the License.
 
 from pyarrow.includes.common cimport shared_ptr
-from pyarrow.includes.libarrow cimport CChunkedArray, CColumn, CTable
+from pyarrow.includes.libarrow cimport (CChunkedArray, CColumn, CTable,
+                                        CRecordBatch)
+
+from pyarrow.schema cimport Schema
 
 
 cdef class ChunkedArray:
@@ -41,6 +44,16 @@ cdef class Table:
     cdef:
         shared_ptr[CTable] sp_table
         CTable* table
-    
+
     cdef init(self, const shared_ptr[CTable]& table)
     cdef _check_nullptr(self)
+
+
+cdef class RecordBatch:
+    cdef:
+        shared_ptr[CRecordBatch] sp_batch
+        CRecordBatch* batch
+        Schema _schema
+
+    cdef init(self, const shared_ptr[CRecordBatch]& table)
+    cdef _check_nullptr(self)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index ade82aa676164..a1cadcd1e0f69 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -19,6 +19,8 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+from cython.operator cimport dereference as deref
+
 from pyarrow.includes.libarrow cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 
@@ -45,8 +47,8 @@ cdef class ChunkedArray:
 
     cdef _check_nullptr(self):
         if self.chunked_array == NULL:
-            raise ReferenceError("ChunkedArray object references a NULL pointer."
-                    "Not initialized.")
+            raise ReferenceError("ChunkedArray object references a NULL "
+                                 "pointer. Not initialized.")
 
     def length(self):
         self._check_nullptr()
@@ -144,6 +146,130 @@ cdef class Column:
             return chunked_array
 
 
+cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
+    cdef:
+        Array arr
+        c_string c_name
+        vector[shared_ptr[CField]] fields
+
+    cdef int K = len(arrays)
+
+    fields.resize(K)
+    for i in range(K):
+        arr = arrays[i]
+        c_name = tobytes(names[i])
+        fields[i].reset(new CField(c_name, arr.type.sp_type, True))
+
+    schema.reset(new CSchema(fields))
+
+
+
+cdef _dataframe_to_arrays(df, name, timestamps_to_ms):
+    from pyarrow.array import from_pandas_series
+
+    cdef:
+        list names = []
+        list arrays = []
+
+    for name in df.columns:
+        col = df[name]
+        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms)
+
+        names.append(name)
+        arrays.append(arr)
+
+    return names, arrays
+
+
+cdef class RecordBatch:
+
+    def __cinit__(self):
+        self.batch = NULL
+        self._schema = None
+
+    cdef init(self, const shared_ptr[CRecordBatch]& batch):
+        self.sp_batch = batch
+        self.batch = batch.get()
+
+    cdef _check_nullptr(self):
+        if self.batch == NULL:
+            raise ReferenceError("Object not initialized")
+
+    def __len__(self):
+        self._check_nullptr()
+        return self.batch.num_rows()
+
+    property num_columns:
+
+        def __get__(self):
+            self._check_nullptr()
+            return self.batch.num_columns()
+
+    property num_rows:
+
+        def __get__(self):
+            return len(self)
+
+    property schema:
+
+        def __get__(self):
+            cdef Schema schema
+            self._check_nullptr()
+            if self._schema is None:
+                schema = Schema()
+                schema.init_schema(self.batch.schema())
+                self._schema = schema
+
+            return self._schema
+
+    def __getitem__(self, i):
+        cdef Array arr = Array()
+        arr.init(self.batch.column(i))
+        return arr
+
+    def equals(self, RecordBatch other):
+        self._check_nullptr()
+        other._check_nullptr()
+
+        return self.batch.Equals(deref(other.batch))
+
+    @classmethod
+    def from_pandas(cls, df):
+        """
+        Convert pandas.DataFrame to an Arrow RecordBatch
+        """
+        names, arrays = _dataframe_to_arrays(df, None, False)
+        return cls.from_arrays(names, arrays)
+
+    @staticmethod
+    def from_arrays(names, arrays):
+        cdef:
+            Array arr
+            RecordBatch result
+            c_string c_name
+            shared_ptr[CSchema] schema
+            shared_ptr[CRecordBatch] batch
+            vector[shared_ptr[CArray]] c_arrays
+            int32_t num_rows
+
+        if len(arrays) == 0:
+            raise ValueError('Record batch cannot contain no arrays (for now)')
+
+        num_rows = len(arrays[0])
+        _schema_from_arrays(arrays, names, &schema)
+
+        for i in range(len(arrays)):
+            arr = arrays[i]
+            c_arrays.push_back(arr.sp_array)
+
+        batch.reset(new CRecordBatch(schema, num_rows, c_arrays))
+
+        result = RecordBatch()
+        result.init(batch)
+
+        return result
+
+
 cdef class Table:
     '''
     Do not call this class's constructor directly.
@@ -161,38 +287,50 @@ cdef class Table:
             raise ReferenceError("Table object references a NULL pointer."
                     "Not initialized.")
 
-    @staticmethod
-    def from_pandas(df, name=None):
-        return from_pandas_dataframe(df, name=name)
+    @classmethod
+    def from_pandas(cls, df, name=None, timestamps_to_ms=False):
+        """
+        Convert pandas.DataFrame to an Arrow Table
+
+        Parameters
+        ----------
+        df: pandas.DataFrame
+
+        name: str
+
+        timestamps_to_ms: bool
+            Convert datetime columns to ms resolution. This is needed for
+            compability with other functionality like Parquet I/O which
+            only supports milliseconds.
+        """
+        names, arrays = _dataframe_to_arrays(df, name=name,
+                                             timestamps_to_ms=timestamps_to_ms)
+        return cls.from_arrays(names, arrays, name=name)
 
     @staticmethod
     def from_arrays(names, arrays, name=None):
         cdef:
             Array arr
-            Table result
             c_string c_name
             vector[shared_ptr[CField]] fields
             vector[shared_ptr[CColumn]] columns
+            Table result
             shared_ptr[CSchema] schema
             shared_ptr[CTable] table
 
-        cdef int K = len(arrays)
+        _schema_from_arrays(arrays, names, &schema)
 
-        fields.resize(K)
+        cdef int K = len(arrays)
         columns.resize(K)
         for i in range(K):
             arr = arrays[i]
-            c_name = tobytes(names[i])
-
-            fields[i].reset(new CField(c_name, arr.type.sp_type, True))
-            columns[i].reset(new CColumn(fields[i], arr.sp_array))
+            columns[i].reset(new CColumn(schema.get().field(i), arr.sp_array))
 
         if name is None:
             c_name = ''
         else:
             c_name = tobytes(name)
 
-        schema.reset(new CSchema(fields))
         table.reset(new CTable(c_name, schema, columns))
 
         result = Table()
@@ -268,32 +406,4 @@ cdef class Table:
 
 
 
-def from_pandas_dataframe(object df, name=None, timestamps_to_ms=False):
-    """
-    Convert pandas.DataFrame to an Arrow Table
-
-    Parameters
-    ----------
-    df: pandas.DataFrame
-
-    name: str
-
-    timestamps_to_ms: bool
-        Convert datetime columns to ms resolution. This is needed for
-        compability with other functionality like Parquet I/O which
-        only supports milliseconds.
-    """
-    from pyarrow.array import from_pandas_series
-
-    cdef:
-        list names = []
-        list arrays = []
-
-    for name in df.columns:
-        col = df[name]
-        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms)
-
-        names.append(name)
-        arrays.append(arr)
-
-    return Table.from_arrays(names, arrays, name=name)
+from_pandas_dataframe = Table.from_pandas
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 86147f8df5a11..0a17f691ccd1f 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -19,6 +19,10 @@
 import pyarrow.formatting as fmt
 
 
+def test_total_bytes_allocated():
+    assert pyarrow.total_allocated_bytes() == 0
+
+
 def test_repr_on_pre_init_array():
     arr = pyarrow.array.Array()
     assert len(repr(arr)) > 0
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 9a41ebe3e8c74..211a12bcd92fe 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -98,3 +98,44 @@ def test_bytes_reader():
 def test_bytes_reader_non_bytes():
     with pytest.raises(ValueError):
         io.BytesReader(u('some sample data'))
+
+
+
+# ----------------------------------------------------------------------
+# Buffers
+
+
+def test_buffer_bytes():
+    val = b'some data'
+
+    buf = io.buffer_from_bytes(val)
+    assert isinstance(buf, io.Buffer)
+
+    result = buf.to_pybytes()
+
+    assert result == val
+
+
+def test_memory_output_stream():
+    # 10 bytes
+    val = b'dataabcdef'
+
+    f = io.InMemoryOutputStream()
+
+    K = 1000
+    for i in range(K):
+        f.write(val)
+
+    buf = f.get_result()
+
+    assert len(buf) == len(val) * K
+    assert buf.to_pybytes() == val * K
+
+
+def test_inmemory_write_after_closed():
+    f = io.InMemoryOutputStream()
+    f.write(b'ok')
+    f.get_result()
+
+    with pytest.raises(IOError):
+        f.write(b'not ok')
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
new file mode 100644
index 0000000000000..b9e9e6ed0c423
--- /dev/null
+++ b/python/pyarrow/tests/test_ipc.py
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import io
+
+import numpy as np
+import pandas as pd
+
+import pyarrow as A
+import pyarrow.io as arrow_io
+import pyarrow.ipc as ipc
+
+
+class RoundtripTest(object):
+    # Also tests writing zero-copy NumPy array with additional padding
+
+    def __init__(self):
+        self.sink = self._get_sink()
+
+    def _get_sink(self):
+        return io.BytesIO()
+
+    def _get_source(self):
+        return self.sink.getvalue()
+
+    def run(self):
+        nrows = 5
+        df = pd.DataFrame({
+            'one': np.random.randn(nrows),
+            'two': ['foo', np.nan, 'bar', 'bazbaz', 'qux']})
+
+        batch = A.RecordBatch.from_pandas(df)
+        writer = ipc.ArrowFileWriter(self.sink, batch.schema)
+
+        num_batches = 5
+        frames = []
+        batches = []
+        for i in range(num_batches):
+            unique_df = df.copy()
+            unique_df['one'] = np.random.randn(nrows)
+
+            batch = A.RecordBatch.from_pandas(unique_df)
+            writer.write_record_batch(batch)
+            frames.append(unique_df)
+            batches.append(batch)
+
+        writer.close()
+
+        file_contents = self._get_source()
+        reader = ipc.ArrowFileReader(file_contents)
+
+        assert reader.num_record_batches == num_batches
+
+        for i in range(num_batches):
+            # it works. Must convert back to DataFrame
+            batch = reader.get_record_batch(i)
+            assert batches[i].equals(batch)
+
+
+class InMemoryStreamTest(RoundtripTest):
+
+    def _get_sink(self):
+        return arrow_io.InMemoryOutputStream()
+
+    def _get_source(self):
+        return self.sink.get_result()
+
+
+def test_ipc_file_simple_roundtrip():
+    helper = RoundtripTest()
+    helper.run()
+
+
+# XXX: For benchmarking
+
+def big_batch():
+    df = pd.DataFrame(
+        np.random.randn(2**4, 2**20).T,
+        columns=[str(i) for i in range(2**4)]
+    )
+
+    df = pd.concat([df] * 2 ** 3, ignore_index=True)
+
+    return A.RecordBatch.from_pandas(df)
+
+
+def write_to_memory(batch):
+    sink = io.BytesIO()
+    write_file(batch, sink)
+    return sink.getvalue()
+
+
+def write_file(batch, sink):
+    writer = ipc.ArrowFileWriter(sink, batch.schema)
+    writer.write_record_batch(batch)
+    writer.close()
+
+
+def read_file(source):
+    reader = ipc.ArrowFileReader(source)
+    return [reader.get_record_batch(i)
+            for i in range(reader.num_record_batches)]
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index abf143199fe15..c5130329e02bc 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -15,60 +15,52 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import unittest
 import pyarrow as A
 
 
-class TestRowBatch(unittest.TestCase):
+def test_recordbatch_basics():
+    data = [
+        A.from_pylist(range(5)),
+        A.from_pylist([-10, -5, 0, 5, 10])
+    ]
 
-    def test_basics(self):
-        data = [
-            A.from_pylist(range(5)),
-            A.from_pylist([-10, -5, 0, 5, 10])
-        ]
-        num_rows = 5
+    batch = A.RecordBatch.from_arrays(['c0', 'c1'], data)
 
-        descr = A.schema([A.field('c0', data[0].type),
-                          A.field('c1', data[1].type)])
+    assert len(batch) == 5
+    assert batch.num_rows == 5
+    assert batch.num_columns == len(data)
 
-        batch = A.RowBatch(descr, num_rows, data)
 
-        assert len(batch) == num_rows
-        assert batch.num_rows == num_rows
-        assert batch.num_columns == len(data)
+def test_table_basics():
+    data = [
+        A.from_pylist(range(5)),
+        A.from_pylist([-10, -5, 0, 5, 10])
+    ]
+    table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
+    assert table.name == 'table_name'
+    assert len(table) == 5
+    assert table.num_rows == 5
+    assert table.num_columns == 2
+    assert table.shape == (5, 2)
 
+    for col in table.itercolumns():
+        for chunk in col.data.iterchunks():
+            assert chunk is not None
 
-class TestTable(unittest.TestCase):
 
-    def test_basics(self):
-        data = [
-            A.from_pylist(range(5)),
-            A.from_pylist([-10, -5, 0, 5, 10])
-        ]
-        table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
-        assert table.name == 'table_name'
-        assert len(table) == 5
-        assert table.num_rows == 5
-        assert table.num_columns == 2
-        assert table.shape == (5, 2)
+def test_table_pandas():
+    data = [
+        A.from_pylist(range(5)),
+        A.from_pylist([-10, -5, 0, 5, 10])
+    ]
+    table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
 
-        for col in table.itercolumns():
-            for chunk in col.data.iterchunks():
-                assert chunk is not None
+    # TODO: Use this part once from_pandas is implemented
+    # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}
+    # df = pd.DataFrame(data)
+    # A.Table.from_pandas(df)
 
-    def test_pandas(self):
-        data = [
-            A.from_pylist(range(5)),
-            A.from_pylist([-10, -5, 0, 5, 10])
-        ]
-        table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
-
-        # TODO: Use this part once from_pandas is implemented
-        # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}
-        # df = pd.DataFrame(data)
-        # A.Table.from_pandas(df)
-
-        df = table.to_pandas()
-        assert set(df.columns) == set(('a', 'b'))
-        assert df.shape == (5, 2)
-        assert df.ix[0, 'b'] == -10
+    df = table.to_pandas()
+    assert set(df.columns) == set(('a', 'b'))
+    assert df.shape == (5, 2)
+    assert df.loc[0, 'b'] == -10
diff --git a/python/setup.py b/python/setup.py
index d1be122888e7b..d040ea7e892c5 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -102,6 +102,7 @@ def initialize_options(self):
         'config',
         'error',
         'io',
+        'ipc',
         'parquet',
         'scalar',
         'schema',
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index 78ef1b31f34f1..680f3a539b5fa 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -426,7 +426,7 @@ Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
 
   // Give the sequence converter an array builder
   std::shared_ptr<ArrayBuilder> builder;
-  RETURN_ARROW_NOT_OK(arrow::MakeBuilder(GetMemoryPool(), type, &builder));
+  RETURN_ARROW_NOT_OK(arrow::MakeBuilder(get_memory_pool(), type, &builder));
   converter->Init(builder);
 
   PY_RETURN_NOT_OK(converter->AppendData(obj));
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index d224074d652cb..ae24b7ee5847b 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -602,6 +602,8 @@ class ArrowDeserializer {
   }
 
   Status AllocateOutput(int type) {
+    PyAcquireGIL lock;
+
     npy_intp dims[1] = {col_->length()};
     out_ = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNew(1, dims, type));
 
@@ -616,6 +618,8 @@ class ArrowDeserializer {
   }
 
   Status OutputFromData(int type, void* data) {
+    PyAcquireGIL lock;
+
     // Zero-Copy. We can pass the data pointer directly to NumPy.
     Py_INCREF(py_ref_);
     OwnedRef py_ref(py_ref_);
@@ -706,6 +710,8 @@ class ArrowDeserializer {
   inline typename std::enable_if<
     arrow_traits<T2>::is_boolean, Status>::type
   ConvertValues(const std::shared_ptr<Array>& arr) {
+    PyAcquireGIL lock;
+
     arrow::BooleanArray* bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
 
     if (arr->null_count() > 0) {
@@ -743,6 +749,8 @@ class ArrowDeserializer {
   inline typename std::enable_if<
     T2 == arrow::Type::STRING, Status>::type
   ConvertValues(const std::shared_ptr<Array>& arr) {
+    PyAcquireGIL lock;
+
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
 
     PyObject** out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index 82b14fdf40173..09f3efb5a03bc 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -63,7 +63,7 @@ class PyArrowMemoryPool : public arrow::MemoryPool {
   int64_t bytes_allocated_;
 };
 
-arrow::MemoryPool* GetMemoryPool() {
+arrow::MemoryPool* get_memory_pool() {
   static PyArrowMemoryPool memory_pool;
   return &memory_pool;
 }
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index bc599f84fab50..96eed1654a777 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -109,7 +109,8 @@ class PyGILGuard {
     return Status::UnknownError(message);           \
   }
 
-PYARROW_EXPORT arrow::MemoryPool* GetMemoryPool();
+// Return the common PyArrow memory pool
+PYARROW_EXPORT arrow::MemoryPool* get_memory_pool();
 
 class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
  public:
@@ -120,6 +121,7 @@ class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
 
     data_ = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
     size_ = PyArray_SIZE(arr_);
+    capacity_ = size_ * PyArray_DESCR(arr_)->elsize;
   }
 
   virtual ~NumPyBuffer() {
@@ -139,6 +141,22 @@ class PYARROW_EXPORT PyBytesBuffer : public arrow::Buffer {
   PyObject* obj_;
 };
 
+
+class PyAcquireGIL {
+ public:
+  PyAcquireGIL() {
+    state_ = PyGILState_Ensure();
+  }
+
+  ~PyAcquireGIL() {
+    PyGILState_Release(state_);
+  }
+
+ private:
+  PyGILState_STATE state_;
+  DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL);
+};
+
 } // namespace pyarrow
 
 #endif // PYARROW_COMMON_H
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index 35054e9025ad4..9879b3474bcd0 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -47,9 +47,9 @@ static arrow::Status CheckPyError() {
     PyErr_Fetch(&exc_type, &exc_value, &traceback);
     PyObjectStringify stringified(exc_value);
     std::string message(stringified.bytes);
-    Py_DECREF(exc_type);
-    Py_DECREF(exc_value);
-    Py_DECREF(traceback);
+    Py_XDECREF(exc_type);
+    Py_XDECREF(exc_value);
+    Py_XDECREF(traceback);
     PyErr_Clear();
     return arrow::Status::IOError(message);
   }

From fb799bc8f818574aacf380b2694aec011d2c18dd Mon Sep 17 00:00:00 2001
From: Leif Walsh <leif@twosigma.com>
Date: Mon, 10 Oct 2016 22:49:47 -0400
Subject: [PATCH 0166/1644] ARROW-112: Changed constexprs to kValue naming.

Consistent with Google style.

Author: Leif Walsh <leif@twosigma.com>

Closes #168 from leifwalsh/constant-name-fix-no-enum and squashes the following commits:

37a0b34 [Leif Walsh] ARROW-112: Changed constexprs to kValue naming.
---
 cpp/src/arrow/builder.h               |  2 +-
 cpp/src/arrow/types/json.cc           |  6 +++---
 cpp/src/arrow/types/primitive-test.cc |  8 ++++----
 cpp/src/arrow/types/primitive.cc      |  2 +-
 cpp/src/arrow/util/bit-util.h         | 10 +++++-----
 cpp/src/arrow/util/buffer.h           |  2 --
 6 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 5d9fb992ff0b5..646a6f24e9df8 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -33,7 +33,7 @@ class Array;
 class MemoryPool;
 class PoolBuffer;
 
-static constexpr int32_t MIN_BUILDER_CAPACITY = 1 << 5;
+static constexpr int32_t kMinBuilderCapacity = 1 << 5;
 
 // Base class for all data array builders.
 // This class provides a facilities for incrementally building the null bitmap
diff --git a/cpp/src/arrow/types/json.cc b/cpp/src/arrow/types/json.cc
index a4e0d085620a0..89240fc22bb2c 100644
--- a/cpp/src/arrow/types/json.cc
+++ b/cpp/src/arrow/types/json.cc
@@ -30,8 +30,8 @@ static const TypePtr String(new StringType());
 static const TypePtr Double(new DoubleType());
 static const TypePtr Bool(new BooleanType());
 
-static const std::vector<TypePtr> json_types = {Null, Int32, String, Double, Bool};
-TypePtr JSONScalar::dense_type = TypePtr(new DenseUnionType(json_types));
-TypePtr JSONScalar::sparse_type = TypePtr(new SparseUnionType(json_types));
+static const std::vector<TypePtr> kJsonTypes = {Null, Int32, String, Double, Bool};
+TypePtr JSONScalar::dense_type = TypePtr(new DenseUnionType(kJsonTypes));
+TypePtr JSONScalar::sparse_type = TypePtr(new SparseUnionType(kJsonTypes));
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 87eb0fe3a8bf7..5ac2867932df7 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -460,7 +460,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
 TYPED_TEST(TestPrimitiveBuilder, TestResize) {
   DECL_TYPE();
 
-  int cap = MIN_BUILDER_CAPACITY * 2;
+  int cap = kMinBuilderCapacity * 2;
 
   ASSERT_OK(this->builder_->Reserve(cap));
   ASSERT_EQ(cap, this->builder_->capacity());
@@ -472,13 +472,13 @@ TYPED_TEST(TestPrimitiveBuilder, TestResize) {
 TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
   ASSERT_OK(this->builder_->Reserve(10));
   ASSERT_EQ(0, this->builder_->length());
-  ASSERT_EQ(MIN_BUILDER_CAPACITY, this->builder_->capacity());
+  ASSERT_EQ(kMinBuilderCapacity, this->builder_->capacity());
 
   ASSERT_OK(this->builder_->Reserve(90));
   ASSERT_OK(this->builder_->Advance(100));
-  ASSERT_OK(this->builder_->Reserve(MIN_BUILDER_CAPACITY));
+  ASSERT_OK(this->builder_->Reserve(kMinBuilderCapacity));
 
-  ASSERT_EQ(util::next_power2(MIN_BUILDER_CAPACITY + 100), this->builder_->capacity());
+  ASSERT_EQ(util::next_power2(kMinBuilderCapacity + 100), this->builder_->capacity());
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 375e94f2bc1c4..9ba2ebdcc2d5b 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -86,7 +86,7 @@ Status PrimitiveBuilder<T>::Init(int32_t capacity) {
 template <typename T>
 Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
   // XXX: Set floor size for now
-  if (capacity < MIN_BUILDER_CAPACITY) { capacity = MIN_BUILDER_CAPACITY; }
+  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
 
   if (capacity_ == 0) {
     RETURN_NOT_OK(Init(capacity));
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 3087ce7784d11..c33ef272f05e2 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -44,22 +44,22 @@ static inline int64_t ceil_2bytes(int64_t size) {
   return (size + 15) & ~15;
 }
 
-static constexpr uint8_t BITMASK[] = {1, 2, 4, 8, 16, 32, 64, 128};
+static constexpr uint8_t kBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128};
 
 static inline bool get_bit(const uint8_t* bits, int i) {
-  return static_cast<bool>(bits[i / 8] & BITMASK[i % 8]);
+  return static_cast<bool>(bits[i / 8] & kBitmask[i % 8]);
 }
 
 static inline bool bit_not_set(const uint8_t* bits, int i) {
-  return (bits[i / 8] & BITMASK[i % 8]) == 0;
+  return (bits[i / 8] & kBitmask[i % 8]) == 0;
 }
 
 static inline void clear_bit(uint8_t* bits, int i) {
-  bits[i / 8] &= ~BITMASK[i % 8];
+  bits[i / 8] &= ~kBitmask[i % 8];
 }
 
 static inline void set_bit(uint8_t* bits, int i) {
-  bits[i / 8] |= BITMASK[i % 8];
+  bits[i / 8] |= kBitmask[i % 8];
 }
 
 static inline int64_t next_power2(int64_t n) {
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 01e4259c31fd2..bc0df86221c45 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -141,8 +141,6 @@ class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
   MemoryPool* pool_;
 };
 
-static constexpr int64_t MIN_BUFFER_CAPACITY = 1024;
-
 class BufferBuilder {
  public:
   explicit BufferBuilder(MemoryPool* pool)

From 8c8d341e12efcedecd3c2545aaf349bf5f899bc1 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Mon, 10 Oct 2016 13:42:41 -0700
Subject: [PATCH 0167/1644] ARROW-326: Include scale and precision when
 materializing decimal writer

closes #166
---
 java/vector/src/main/codegen/templates/MapWriters.java   | 5 +++++
 .../arrow/vector/complex/impl/TestPromotableWriter.java  | 9 ++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 9fe20df7a1df0..696bbf655cac9 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -73,7 +73,12 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
 <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
 <#assign upperName = minor.class?upper_case />
       case ${upperName}:
+        <#if lowerName == "decimal" >
+        Decimal decimal = (Decimal)child.getType();
+        decimal(child.getName(), decimal.getScale(), decimal.getPrecision());
+        <#else>
         ${lowerName}(child.getName());
+       </#if>
         break;
 </#list></#list>
       }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index d439cebeda6ac..176ad5195b3a1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -67,6 +67,8 @@ public void testPromoteToUnion() throws Exception {
       writer.setPosition(1);
       writer.bit("A").writeBit(1);
 
+      writer.decimal("dec", 10,10);
+
       writer.setPosition(2);
       writer.integer("A").writeInt(10);
 
@@ -108,9 +110,10 @@ public void testPromoteToUnion() throws Exception {
       newMapWriter.setPosition(2);
       newMapWriter.integer("A").writeInt(10);
 
-      Field childField = container.getField().getChildren().get(0).getChildren().get(0);
-      assertEquals("Child field should be union type: " + childField.getName(), Type.Union, childField.getType().getTypeType());
-
+      Field childField1 = container.getField().getChildren().get(0).getChildren().get(0);
+      Field childField2 = container.getField().getChildren().get(0).getChildren().get(1);
+      assertEquals("Child field should be union type: " + childField1.getName(), Type.Union, childField1.getType().getTypeType());
+      assertEquals("Child field should be decimal type: " + childField2.getName(), Type.Decimal, childField2.getType().getTypeType());
     }
   }
 }

From 994aa5a903917aca0c9dd372341d4dcbc8be3aa5 Mon Sep 17 00:00:00 2001
From: Leif Walsh <leif@twosigma.com>
Date: Tue, 11 Oct 2016 14:00:36 -0400
Subject: [PATCH 0168/1644] ARROW-189: Build 3rd party with ExternalProject.

When third party env vars *_HOME are not present, use cmake's
ExternalProject to fetch and build them.  When those vars are present,
we just use them.

Author: Leif Walsh <leif@twosigma.com>

Closes #167 from leifwalsh/cmake-externalproject and squashes the following commits:

e4fb63a [Leif Walsh] ARROW-189: Remove 3rd party from conda build.
7892bae [Leif Walsh] ARROW-189: Fix darwin build.
8630428 [Leif Walsh] ARROW-189: Addressed CR comments.
8215abc [Leif Walsh] ARROW-189: Build 3rd party with ExternalProject.
---
 ci/travis_before_script_cpp.sh        |   8 --
 ci/travis_script_python.sh            |   5 --
 cpp/CMakeLists.txt                    | 107 ++++++++++++++++++++++----
 cpp/README.md                         |  18 +----
 cpp/conda.recipe/build.sh             |  10 ---
 cpp/doc/Parquet.md                    |   1 -
 cpp/setup_build_env.sh                |  21 -----
 cpp/src/arrow/ipc/CMakeLists.txt      |  10 ++-
 cpp/thirdparty/build_thirdparty.sh    | 104 -------------------------
 cpp/thirdparty/download_thirdparty.sh |  44 -----------
 cpp/thirdparty/set_thirdparty_env.sh  |  24 ------
 cpp/thirdparty/versions.sh            |  23 ------
 12 files changed, 101 insertions(+), 274 deletions(-)
 delete mode 100755 cpp/setup_build_env.sh
 delete mode 100755 cpp/thirdparty/build_thirdparty.sh
 delete mode 100755 cpp/thirdparty/download_thirdparty.sh
 delete mode 100755 cpp/thirdparty/set_thirdparty_env.sh
 delete mode 100755 cpp/thirdparty/versions.sh

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index acd820bbed2d4..2d4224b33336f 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -26,14 +26,6 @@ pushd $CPP_BUILD_DIR
 
 CPP_DIR=$TRAVIS_BUILD_DIR/cpp
 
-# Build an isolated thirdparty
-cp -r $CPP_DIR/thirdparty .
-cp $CPP_DIR/setup_build_env.sh .
-
-source setup_build_env.sh
-
-echo $GTEST_HOME
-
 : ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
 
 CMAKE_COMMON_FLAGS="\
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index a75ff0778bc82..97f0563240c75 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -21,11 +21,6 @@ export MINICONDA=$HOME/miniconda
 export PATH="$MINICONDA/bin:$PATH"
 export PARQUET_HOME=$MINICONDA
 
-# Share environment with C++
-pushd $CPP_BUILD_DIR
-source setup_build_env.sh
-popd
-
 pushd $PYTHON_DIR
 
 python_version_tests() {
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f70c8ab4bccef..d682dc76f8ced 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -21,10 +21,15 @@ project(arrow)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
 include(CMakeParseArguments)
+include(ExternalProject)
 
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 
+set(GTEST_VERSION "1.7.0")
+set(GBENCHMARK_VERSION "1.0.0")
+set(FLATBUFFERS_VERSION "1.3.0")
+
 find_package(ClangTools)
 if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
   # Generate a Clang compile_commands.json "compilation database" file for use
@@ -422,16 +427,6 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
   endif()
 endfunction()
 
-## GTest
-if ("$ENV{GTEST_HOME}" STREQUAL "")
-  set(GTest_HOME ${THIRDPARTY_DIR}/googletest-release-1.7.0)
-endif()
-
-## Google Benchmark
-if ("$ENV{GBENCHMARK_HOME}" STREQUAL "")
-  set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed)
-endif()
-
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
@@ -476,18 +471,78 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 
 if(ARROW_BUILD_TESTS)
   add_custom_target(unittest ctest -L unittest)
-  find_package(GTest REQUIRED)
+
+  if("$ENV{GTEST_HOME}" STREQUAL "")
+    if(APPLE)
+      set(GTEST_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
+    else()
+      set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
+    endif()
+
+    ExternalProject_Add(googletest_ep
+      URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+      CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
+      # googletest doesn't define install rules, so just build in the
+      # source dir and don't try to install.  See its README for
+      # details.
+      BUILD_IN_SOURCE 1
+      INSTALL_COMMAND "")
+
+    set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
+    set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
+    set(GTEST_STATIC_LIB "${GTEST_PREFIX}/libgtest.a")
+    set(GTEST_VENDORED 1)
+  else()
+    find_package(GTest REQUIRED)
+    set(GTEST_VENDORED 0)
+  endif()
+
+  message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}")
+  message(STATUS "GTest static library: ${GTEST_STATIC_LIB}")
   include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(gtest
     STATIC_LIB ${GTEST_STATIC_LIB})
+
+  if(GTEST_VENDORED)
+    add_dependencies(gtest googletest_ep)
+  endif()
 endif()
 
 if(ARROW_BUILD_BENCHMARKS)
   add_custom_target(runbenchmark ctest -L benchmark)
-  find_package(GBenchmark REQUIRED)
+
+  if("$ENV{GBENCHMARK_HOME}" STREQUAL "")
+    if(APPLE)
+      set(GBENCHMARK_CMAKE_CXX_FLAGS "-std=c++11 -stdlib=libc++")
+    else()
+      set(GBENCHMARK_CMAKE_CXX_FLAGS "--std=c++11")
+    endif()
+
+    set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install")
+    ExternalProject_Add(gbenchmark_ep
+      URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
+      CMAKE_ARGS
+        "-DCMAKE_BUILD_TYPE=Release"
+        "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}"
+        "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}")
+
+    set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include")
+    set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/libbenchmark.a")
+    set(GBENCHMARK_VENDORED 1)
+  else()
+    find_package(GBenchmark REQUIRED)
+    set(GBENCHMARK_VENDORED 0)
+  endif()
+
+  message(STATUS "GBenchmark include dir: ${GBENCHMARK_INCLUDE_DIR}")
+  message(STATUS "GBenchmark static library: ${GBENCHMARK_STATIC_LIB}")
   include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(benchmark
     STATIC_LIB ${GBENCHMARK_STATIC_LIB})
+
+  if(GBENCHMARK_VENDORED)
+    add_dependencies(benchmark gbenchmark_ep)
+  endif()
 endif()
 
 ## Google PerfTools
@@ -705,14 +760,34 @@ add_subdirectory(src/arrow/types)
 
 ## Flatbuffers
 if(ARROW_IPC)
-  find_package(Flatbuffers REQUIRED)
+  if("$ENV{FLATBUFFERS_HOME}" STREQUAL "")
+    set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
+    ExternalProject_Add(flatbuffers_ep
+      URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
+      CMAKE_ARGS
+        "-DCMAKE_CXX_FLAGS=-fPIC"
+        "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
+        "-DFLATBUFFERS_BUILD_TESTS=OFF")
+
+    set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
+    set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_PREFIX}/libflatbuffers.a")
+    set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
+    set(FLATBUFFERS_VENDORED 1)
+  else()
+    find_package(Flatbuffers REQUIRED)
+    set(FLATBUFFERS_VENDORED 0)
+  endif()
+
   message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
   message(STATUS "Flatbuffers static library: ${FLATBUFFERS_STATIC_LIB}")
   message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
   include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
-  add_library(flatbuffers STATIC IMPORTED)
-  set_target_properties(flatbuffers PROPERTIES
-    IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB})
+  ADD_THIRDPARTY_LIB(flatbuffers
+    STATIC_LIB ${FLATBUFFERS_STATIC_LIB})
+
+  if(FLATBUFFERS_VENDORED)
+    add_dependencies(flatbuffers flatbuffers_ep)
+  endif()
 
   add_subdirectory(src/arrow/ipc)
 endif()
diff --git a/cpp/README.md b/cpp/README.md
index a1c3ef28447f5..190e6f85b429d 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -22,23 +22,6 @@ out-of-source builds with the latter one being preferred.
 Arrow requires a C++11-enabled compiler. On Linux, gcc 4.8 and higher should be
 sufficient.
 
-To build the thirdparty build dependencies, run:
-
-```
-./thirdparty/download_thirdparty.sh
-./thirdparty/build_thirdparty.sh
-source ./thirdparty/set_thirdparty_env.sh
-```
-
-You can also run from the root of the C++ tree
-
-```
-source setup_build_env.sh
-```
-
-Arrow is configured to use the `thirdparty` directory by default for its build
-dependencies. To set up a custom toolchain see below.
-
 Simple debug build:
 
     mkdir debug
@@ -76,6 +59,7 @@ variables
 * Googletest: `GTEST_HOME` (only required to build the unit tests)
 * Google Benchmark: `GBENCHMARK_HOME` (only required if building benchmarks)
 * Flatbuffers: `FLATBUFFERS_HOME` (only required for the IPC extensions)
+* Hadoop: `HADOOP_HOME` (only required for the HDFS I/O extensions)
 
 ## Continuous Integration
 
diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh
index 6d7454e927265..0536fd99b5ca5 100644
--- a/cpp/conda.recipe/build.sh
+++ b/cpp/conda.recipe/build.sh
@@ -38,19 +38,9 @@ cd ..
 
 rm -rf conda-build
 mkdir conda-build
-
-cp -r thirdparty conda-build/
-
 cd conda-build
 pwd
 
-# Build googletest for running unit tests
-./thirdparty/download_thirdparty.sh
-./thirdparty/build_thirdparty.sh gtest
-
-source thirdparty/versions.sh
-export GTEST_HOME=`pwd`/thirdparty/$GTEST_BASEDIR
-
 # if [ `uname` == Linux ]; then
 #     SHARED_LINKER_FLAGS='-static-libstdc++'
 # elif [ `uname` == Darwin ]; then
diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md
index 34b83e78d0a5c..4985dd3b0bc2d 100644
--- a/cpp/doc/Parquet.md
+++ b/cpp/doc/Parquet.md
@@ -24,7 +24,6 @@ export ARROW_HOME=$HOME/local
 
 git clone https://github.com/apache/parquet-cpp.git
 cd parquet-cpp
-source setup_build_env.sh
 cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_ARROW=on
 make -j4
 make install
diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh
deleted file mode 100755
index 546216753b382..0000000000000
--- a/cpp/setup_build_env.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-
-./thirdparty/download_thirdparty.sh || { echo "download_thirdparty.sh failed" ; return; }
-./thirdparty/build_thirdparty.sh || { echo "build_thirdparty.sh failed" ; return; }
-source ./thirdparty/set_thirdparty_env.sh || { echo "source set_thirdparty_env.sh failed" ; return; }
-
-echo "Build env initialized"
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 8dcd9ac107189..d2db339de7ea2 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -42,6 +42,9 @@ set(ARROW_IPC_SRCS
 add_library(arrow_ipc SHARED
   ${ARROW_IPC_SRCS}
 )
+if(FLATBUFFERS_VENDORED)
+  add_dependencies(arrow_ipc flatbuffers_ep)
+endif()
 target_link_libraries(arrow_ipc
   LINK_PUBLIC ${ARROW_IPC_LINK_LIBS}
   LINK_PRIVATE ${ARROW_IPC_PRIVATE_LINK_LIBS})
@@ -91,10 +94,15 @@ foreach(FIL ${FBS_SRC})
   list(APPEND ABS_FBS_SRC ${ABS_FIL})
 endforeach()
 
+if(FLATBUFFERS_VENDORED)
+  set(FBS_DEPENDS ${ABS_FBS_SRC} flatbuffers_ep)
+else()
+  set(FBS_DEPENDS ${ABS_FBS_SRC})
+endif()
 add_custom_command(
   OUTPUT ${FBS_OUTPUT_FILES}
   COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC}
-  DEPENDS ${ABS_FBS_SRC}
+  DEPENDS ${FBS_DEPENDS}
   COMMENT "Running flatc compiler on ${ABS_FBS_SRC}"
   VERBATIM
 )
diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh
deleted file mode 100755
index 5011e29c01a71..0000000000000
--- a/cpp/thirdparty/build_thirdparty.sh
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/bin/bash
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-set -x
-set -e
-TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-
-source $TP_DIR/versions.sh
-PREFIX=$TP_DIR/installed
-
-################################################################################
-
-if [ "$#" = "0" ]; then
-  F_ALL=1
-else
-  # Allow passing specific libs to build on the command line
-  for arg in "$*"; do
-    case $arg in
-      "gtest")      F_GTEST=1 ;;
-      "gbenchmark")      F_GBENCHMARK=1 ;;
-      "flatbuffers")      F_FLATBUFFERS=1 ;;
-      *)            echo "Unknown module: $arg"; exit 1 ;;
-    esac
-  done
-fi
-
-################################################################################
-
-# Determine how many parallel jobs to use for make based on the number of cores
-if [[ "$OSTYPE" =~ ^linux ]]; then
-  PARALLEL=$(grep -c processor /proc/cpuinfo)
-elif [[ "$OSTYPE" == "darwin"* ]]; then
-  PARALLEL=$(sysctl -n hw.ncpu)
-else
-  echo Unsupported platform $OSTYPE
-  exit 1
-fi
-
-mkdir -p "$PREFIX/include"
-mkdir -p "$PREFIX/lib"
-
-# On some systems, autotools installs libraries to lib64 rather than lib.  Fix
-# this by setting up lib64 as a symlink to lib.  We have to do this step first
-# to handle cases where one third-party library depends on another.
-ln -sf lib "$PREFIX/lib64"
-
-# use the compiled tools
-export PATH=$PREFIX/bin:$PATH
-
-type cmake >/dev/null 2>&1 || { echo >&2 "cmake not installed.  Aborting."; exit 1; }
-type make >/dev/null 2>&1 || { echo >&2 "make not installed.  Aborting."; exit 1; }
-
-STANDARD_DARWIN_FLAGS="-std=c++11 -stdlib=libc++"
-
-# build googletest
-GOOGLETEST_ERROR="failed for googletest!"
-if [ -n "$F_ALL" -o -n "$F_GTEST" ]; then
-  cd $TP_DIR/$GTEST_BASEDIR
-
-  if [[ "$OSTYPE" == "darwin"* ]]; then
-    CXXFLAGS=-fPIC cmake -DCMAKE_CXX_FLAGS="$STANDARD_DARWIN_FLAGS -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes" || { echo "cmake $GOOGLETEST_ERROR" ; exit  1; }
-  else
-    CXXFLAGS=-fPIC cmake . || { echo "cmake $GOOGLETEST_ERROR"; exit  1; }
-  fi
-
-  make -j$PARALLEL VERBOSE=1 || { echo "Make $GOOGLETEST_ERROR" ; exit  1; }
-fi
-
-# build google benchmark
-GBENCHMARK_ERROR="failed for google benchmark"
-if [ -n "$F_ALL" -o -n "$F_GBENCHMARK" ]; then
-  cd $TP_DIR/$GBENCHMARK_BASEDIR
-
-  CMAKE_CXX_FLAGS="--std=c++11"
-  if [[ "$OSTYPE" == "darwin"* ]]; then
-    CMAKE_CXX_FLAGS=$STANDARD_DARWIN_FLAGS
-  fi
-  cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_CXX_FLAGS="-fPIC $CMAKE_CXX_FLAGS" . || { echo "cmake $GBENCHMARK_ERROR" ; exit 1; }
-
-  make -j$PARALLEL VERBOSE=1 install || { echo "make $GBENCHMARK_ERROR" ; exit 1; }
-fi
-
-FLATBUFFERS_ERROR="failed for flatbuffers"
-if [ -n "$F_ALL" -o -n "$F_FLATBUFFERS" ]; then
-  cd $TP_DIR/$FLATBUFFERS_BASEDIR
-
-  CXXFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX -DFLATBUFFERS_BUILD_TESTS=OFF . || { echo "cmake $FLATBUFFERS_ERROR" ; exit 1; }
-  make VERBOSE=1 -j$PARALLEL || { echo "make $FLATBUFFERS_ERROR" ; exit 1; }
-  make install || { echo "install $FLATBUFFERS_ERROR" ; exit 1; }
-fi
-
-echo "---------------------"
-echo "Thirdparty dependencies built and installed into $PREFIX successfully"
diff --git a/cpp/thirdparty/download_thirdparty.sh b/cpp/thirdparty/download_thirdparty.sh
deleted file mode 100755
index b50e7bc06a14c..0000000000000
--- a/cpp/thirdparty/download_thirdparty.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-set -x
-set -e
-
-TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-
-source $TP_DIR/versions.sh
-
-download_extract_and_cleanup() {
-	type curl >/dev/null 2>&1 || { echo >&2 "curl not installed.  Aborting."; exit 1; }
-	filename=$TP_DIR/$(basename "$1")
-	curl -#LC - "$1" -o $filename
-	tar xzf $filename -C $TP_DIR
-	rm $filename
-}
-
-if [ ! -d ${GTEST_BASEDIR} ]; then
-  echo "Fetching gtest"
-  download_extract_and_cleanup $GTEST_URL
-fi
-
-echo ${GBENCHMARK_BASEDIR}
-if [ ! -d ${GBENCHMARK_BASEDIR} ]; then
-  echo "Fetching google benchmark"
-  download_extract_and_cleanup $GBENCHMARK_URL
-fi
-
-if [ ! -d ${FLATBUFFERS_BASEDIR} ]; then
-  echo "Fetching flatbuffers"
-  download_extract_and_cleanup $FLATBUFFERS_URL
-fi
diff --git a/cpp/thirdparty/set_thirdparty_env.sh b/cpp/thirdparty/set_thirdparty_env.sh
deleted file mode 100755
index 135972ee9bdce..0000000000000
--- a/cpp/thirdparty/set_thirdparty_env.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bash
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-source $SOURCE_DIR/versions.sh
-
-if [ -z "$THIRDPARTY_DIR" ]; then
-	THIRDPARTY_DIR=$SOURCE_DIR
-fi
-
-export GTEST_HOME=$THIRDPARTY_DIR/$GTEST_BASEDIR
-export GBENCHMARK_HOME=$THIRDPARTY_DIR/installed
-export FLATBUFFERS_HOME=$THIRDPARTY_DIR/installed
diff --git a/cpp/thirdparty/versions.sh b/cpp/thirdparty/versions.sh
deleted file mode 100755
index a7b21e19fccd6..0000000000000
--- a/cpp/thirdparty/versions.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-GTEST_VERSION=1.7.0
-GTEST_URL="https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-GTEST_BASEDIR=googletest-release-$GTEST_VERSION
-
-GBENCHMARK_VERSION=1.0.0
-GBENCHMARK_URL="https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
-GBENCHMARK_BASEDIR=benchmark-$GBENCHMARK_VERSION
-
-FLATBUFFERS_VERSION=1.3.0
-FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
-FLATBUFFERS_BASEDIR=flatbuffers-$FLATBUFFERS_VERSION

From caa843bdaf395b915a739bf5e1d6c5eabe1f4693 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 11 Oct 2016 17:29:25 -0700
Subject: [PATCH 0169/1644] ARROW-333: Make writers update their internal
 schema even when no data is written

Make PromotableWriter predefine writers when asked

Author: Julien Le Dem <julien@dremio.com>

Closes #170 from julienledem/promotable_writer_preset and squashes the following commits:

972eb9c [Julien Le Dem] ARROW-333: Make writers update their internal schema even when no data is written Make PromotableWriter predefine writers when asked
---
 .../main/codegen/templates/MapWriters.java    | 15 +++++++++
 .../main/codegen/templates/UnionWriter.java   | 24 ++++++++++++++
 .../vector/complex/impl/PromotableWriter.java | 14 ++++----
 .../complex/writer/TestComplexWriter.java     | 32 +++++++++++++++++--
 4 files changed, 76 insertions(+), 9 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 696bbf655cac9..51327b43af0fa 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -112,6 +112,11 @@ public MapWriter map(String name) {
       }
       writer.setPosition(idx());
       fields.put(name.toLowerCase(), writer);
+    } else {
+      if (writer instanceof PromotableWriter) {
+        // ensure writers are initialized
+        ((PromotableWriter)writer).getWriter(MinorType.MAP);
+      }
     }
     return writer;
   }
@@ -149,6 +154,11 @@ public ListWriter list(String name) {
       }
       writer.setPosition(idx());
       fields.put(name.toLowerCase(), writer);
+    } else {
+      if (writer instanceof PromotableWriter) {
+        // ensure writers are initialized
+        ((PromotableWriter)writer).getWriter(MinorType.LIST);
+      }
     }
     return writer;
   }
@@ -210,6 +220,11 @@ public void end() {
       } 
       writer.setPosition(idx());
       fields.put(name.toLowerCase(), writer);
+    } else {
+      if (writer instanceof PromotableWriter) {
+        // ensure writers are initialized
+        ((PromotableWriter)writer).getWriter(MinorType.${upperName});
+      }
     }
     return writer;
   }
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 460ec1c0d9586..efb66f168f5f8 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -25,6 +25,8 @@
 package org.apache.arrow.vector.complex.impl;
 
 <#include "/@includes/vv_imports.ftl" />
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.types.Types.MinorType;
 
 /*
  * This class is generated using freemarker and the ${.template_name} template.
@@ -100,6 +102,28 @@ public ListWriter asList() {
     return getListWriter();
   }
 
+  BaseWriter getWriter(MinorType minorType) {
+    switch (minorType) {
+    case MAP:
+      return getMapWriter();
+    case LIST:
+      return getListWriter();
+    <#list vv.types as type>
+    <#list type.minor as minor>
+      <#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#if !minor.class?starts_with("Decimal")>
+    case ${name?upper_case}:
+      return get${name}Writer();
+      </#if>
+    </#list>
+    </#list>
+    default:
+      throw new UnsupportedOperationException("Unknown type: " + minorType);
+    }
+  }
+
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
   <#assign uncappedName = name?uncap_first/>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index c282688530b87..94ff82c04bd18 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -94,19 +94,19 @@ public void setPosition(int index) {
 
   protected FieldWriter getWriter(MinorType type) {
     if (state == State.UNION) {
-      return writer;
-    }
-    if (state == State.UNTYPED) {
+      ((UnionWriter)writer).getWriter(type);
+    } else if (state == State.UNTYPED) {
       if (type == null) {
+        // ???
         return null;
       }
       ValueVector v = listVector.addOrGetVector(type).getVector();
       v.allocateNew();
       setWriter(v);
       writer.setPosition(position);
-    }
-    if (type != this.type) {
-      return promoteToUnion();
+    } else if (type != this.type) {
+      promoteToUnion();
+      ((UnionWriter)writer).getWriter(type);
     }
     return writer;
   }
@@ -133,7 +133,7 @@ private FieldWriter promoteToUnion() {
     unionVector.addVector((FieldVector)tp.getTo());
     writer = new UnionWriter(unionVector);
     writer.setPosition(idx());
-    for (int i = 0; i < idx(); i++) {
+    for (int i = 0; i <= idx(); i++) {
       unionVector.getMutator().setType(i, vector.getMinorType());
     }
     vector = null;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 398aea915b343..9419f88de5b74 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -45,6 +45,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.Text;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -362,11 +363,38 @@ public void promotableWriter() {
     MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
     for (int i = 0; i < 100; i++) {
       rootReader.setPosition(i);
-      Assert.assertEquals(i, rootReader.reader("a").readLong().intValue());
+      FieldReader reader = rootReader.reader("a");
+      Long value = reader.readLong();
+      Assert.assertNotNull("index: " + i, value);
+      Assert.assertEquals(i, value.intValue());
     }
     for (int i = 100; i < 200; i++) {
       rootReader.setPosition(i);
-      Assert.assertEquals(Integer.toString(i), rootReader.reader("a").readText().toString());
+      FieldReader reader = rootReader.reader("a");
+      Text value = reader.readText();
+      Assert.assertEquals(Integer.toString(i), value.toString());
     }
   }
+
+  /**
+   * Even without writing to the writer, the union schema is created correctly
+   */
+  @Test
+  public void promotableWriterSchema() {
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    BigIntWriter bigIntWriter = rootWriter.bigInt("a");
+    VarCharWriter varCharWriter = rootWriter.varChar("a");
+
+    Field field = parent.getField().getChildren().get(0).getChildren().get(0);
+    Assert.assertEquals("a", field.getName());
+    Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeType());
+
+    Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeType());
+    Int intType = (Int) field.getChildren().get(0).getType();
+    Assert.assertEquals(64, intType.getBitWidth());
+    Assert.assertTrue(intType.getIsSigned());
+    Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeType());
+  }
 }
\ No newline at end of file

From 3919a277884cf504fdca5d730cf128e36db6f700 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 11 Oct 2016 23:08:48 -0400
Subject: [PATCH 0170/1644] ARROW-332: Add RecordBatch.to_pandas method

This makes testing and IPC data wrangling a little easier.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #165 from wesm/ARROW-332 and squashes the following commits:

5f19b97 [Wes McKinney] Add simple arrow::Array->NumPy-for-pandas conversion helper and RecordBatch.to_pandas
---
 python/pyarrow/includes/pyarrow.pxd   |  7 +++--
 python/pyarrow/io.pyx                 | 12 ++++++++
 python/pyarrow/table.pyx              | 25 ++++++++++++++--
 python/pyarrow/tests/test_ipc.py      | 40 ++++++++++++++++++++++++--
 python/pyarrow/tests/test_table.py    | 41 ++++++++++++++++++++-------
 python/src/pyarrow/adapters/pandas.cc | 19 +++++++++++--
 python/src/pyarrow/adapters/pandas.h  |  7 ++++-
 python/src/pyarrow/common.h           |  4 +--
 python/src/pyarrow/io.cc              |  2 +-
 9 files changed, 133 insertions(+), 24 deletions(-)

diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 2fa5a7d63256a..7c47f21854e33 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -50,8 +50,11 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     PyStatus PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
                                  shared_ptr[CArray]* out)
 
-    PyStatus ArrowToPandas(const shared_ptr[CColumn]& arr, object py_ref,
-                           PyObject** out)
+    PyStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
+                                  object py_ref, PyObject** out)
+
+    PyStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
+                                   object py_ref, PyObject** out)
 
     MemoryPool* get_memory_pool()
 
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 00a492fc0baf2..8970e06effdd0 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -230,6 +230,18 @@ cdef class InMemoryOutputStream(NativeFile):
         return result
 
 
+cdef class BufferReader(NativeFile):
+    cdef:
+        Buffer buffer
+
+    def __cinit__(self, Buffer buffer):
+        self.buffer = buffer
+        self.rd_file.reset(new CBufferReader(buffer.buffer.get().data(),
+                                             buffer.buffer.get().size()))
+        self.is_readonly = 1
+        self.is_open = True
+
+
 def buffer_from_bytes(object obj):
     """
     Construct an Arrow buffer from a Python bytes object
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index a1cadcd1e0f69..969571262ca44 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -100,7 +100,7 @@ cdef class Column:
 
         import pandas as pd
 
-        check_status(pyarrow.ArrowToPandas(self.sp_column, self, &arr))
+        check_status(pyarrow.ConvertColumnToPandas(self.sp_column, self, &arr))
         return pd.Series(<object>arr, name=self.name)
 
     cdef _check_nullptr(self):
@@ -233,6 +233,27 @@ cdef class RecordBatch:
 
         return self.batch.Equals(deref(other.batch))
 
+    def to_pandas(self):
+        """
+        Convert the arrow::RecordBatch to a pandas DataFrame
+        """
+        cdef:
+            PyObject* np_arr
+            shared_ptr[CArray] arr
+            Column column
+
+        import pandas as pd
+
+        names = []
+        data = []
+        for i in range(self.batch.num_columns()):
+            arr = self.batch.column(i)
+            check_status(pyarrow.ConvertArrayToPandas(arr, self, &np_arr))
+            names.append(frombytes(self.batch.column_name(i)))
+            data.append(<object> np_arr)
+
+        return pd.DataFrame(dict(zip(names, data)), columns=names)
+
     @classmethod
     def from_pandas(cls, df):
         """
@@ -354,7 +375,7 @@ cdef class Table:
         for i in range(self.table.num_columns()):
             col = self.table.column(i)
             column = self.column(i)
-            check_status(pyarrow.ArrowToPandas(col, column, &arr))
+            check_status(pyarrow.ConvertColumnToPandas(col, column, &arr))
             names.append(frombytes(col.get().name()))
             data.append(<object> arr)
 
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index b9e9e6ed0c423..14cbb30d5d48b 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -18,6 +18,8 @@
 import io
 
 import numpy as np
+
+from pandas.util.testing import assert_frame_equal
 import pandas as pd
 
 import pyarrow as A
@@ -85,17 +87,40 @@ def test_ipc_file_simple_roundtrip():
     helper.run()
 
 
+def test_ipc_zero_copy_numpy():
+    df = pd.DataFrame({'foo': [1.5]})
+
+    batch = A.RecordBatch.from_pandas(df)
+    sink = arrow_io.InMemoryOutputStream()
+    write_file(batch, sink)
+    buffer = sink.get_result()
+    reader = arrow_io.BufferReader(buffer)
+
+    batches = read_file(reader)
+
+    data = batches[0].to_pandas()
+    rdf = pd.DataFrame(data)
+    assert_frame_equal(df, rdf)
+
+
 # XXX: For benchmarking
 
 def big_batch():
+    K = 2**4
+    N = 2**20
     df = pd.DataFrame(
-        np.random.randn(2**4, 2**20).T,
-        columns=[str(i) for i in range(2**4)]
+        np.random.randn(K, N).T,
+        columns=[str(i) for i in range(K)]
     )
 
     df = pd.concat([df] * 2 ** 3, ignore_index=True)
+    return df
+
 
-    return A.RecordBatch.from_pandas(df)
+def write_to_memory2(batch):
+    sink = arrow_io.InMemoryOutputStream()
+    write_file(batch, sink)
+    return sink.get_result()
 
 
 def write_to_memory(batch):
@@ -114,3 +139,12 @@ def read_file(source):
     reader = ipc.ArrowFileReader(source)
     return [reader.get_record_batch(i)
             for i in range(reader.num_record_batches)]
+
+# df = big_batch()
+# batch = A.RecordBatch.from_pandas(df)
+# mem = write_to_memory(batch)
+# batches = read_file(mem)
+# data = batches[0].to_pandas()
+# rdf = pd.DataFrame(data)
+
+# [x.to_pandas() for x in batches]
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index c5130329e02bc..4c9d302106af8 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -15,28 +15,47 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import pyarrow as A
+import numpy as np
+
+from pandas.util.testing import assert_frame_equal
+import pandas as pd
+
+import pyarrow as pa
 
 
 def test_recordbatch_basics():
     data = [
-        A.from_pylist(range(5)),
-        A.from_pylist([-10, -5, 0, 5, 10])
+        pa.from_pylist(range(5)),
+        pa.from_pylist([-10, -5, 0, 5, 10])
     ]
 
-    batch = A.RecordBatch.from_arrays(['c0', 'c1'], data)
+    batch = pa.RecordBatch.from_arrays(['c0', 'c1'], data)
 
     assert len(batch) == 5
     assert batch.num_rows == 5
     assert batch.num_columns == len(data)
 
 
+def test_recordbatch_from_to_pandas():
+    data = pd.DataFrame({
+        'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),
+        'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'),
+        'c2': np.random.randn(5),
+        'c3': ['foo', 'bar', None, 'baz', 'qux'],
+        'c4': [False, True, False, True, False]
+    })
+
+    batch = pa.RecordBatch.from_pandas(data)
+    result = batch.to_pandas()
+    assert_frame_equal(data, result)
+
+
 def test_table_basics():
     data = [
-        A.from_pylist(range(5)),
-        A.from_pylist([-10, -5, 0, 5, 10])
+        pa.from_pylist(range(5)),
+        pa.from_pylist([-10, -5, 0, 5, 10])
     ]
-    table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
+    table = pa.Table.from_arrays(('a', 'b'), data, 'table_name')
     assert table.name == 'table_name'
     assert len(table) == 5
     assert table.num_rows == 5
@@ -50,15 +69,15 @@ def test_table_basics():
 
 def test_table_pandas():
     data = [
-        A.from_pylist(range(5)),
-        A.from_pylist([-10, -5, 0, 5, 10])
+        pa.from_pylist(range(5)),
+        pa.from_pylist([-10, -5, 0, 5, 10])
     ]
-    table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
+    table = pa.Table.from_arrays(('a', 'b'), data, 'table_name')
 
     # TODO: Use this part once from_pandas is implemented
     # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}
     # df = pd.DataFrame(data)
-    # A.Table.from_pandas(df)
+    # pa.Table.from_pandas(df)
 
     df = table.to_pandas()
     assert set(df.columns) == set(('a', 'b'))
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index ae24b7ee5847b..b2fcd37aec944 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -21,6 +21,8 @@
 
 #include "pyarrow/numpy_interop.h"
 
+#include "pyarrow/adapters/pandas.h"
+
 #include <cmath>
 #include <cstdint>
 #include <memory>
@@ -38,6 +40,7 @@ namespace pyarrow {
 
 using arrow::Array;
 using arrow::Column;
+using arrow::Field;
 using arrow::DataType;
 namespace util = arrow::util;
 
@@ -106,7 +109,7 @@ struct npy_traits<NPY_FLOAT64> {
 
 template <>
 struct npy_traits<NPY_DATETIME> {
-  typedef double value_type;
+  typedef int64_t value_type;
   using TypeClass = arrow::TimestampType;
 
   static constexpr bool supports_nulls = true;
@@ -163,6 +166,8 @@ class ArrowSerializer {
   Status ConvertData();
 
   Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
+    PyAcquireGIL lock;
+
     PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     arrow::TypePtr string_type(new arrow::StringType());
     arrow::StringBuilder string_builder(pool_, string_type);
@@ -197,6 +202,8 @@ class ArrowSerializer {
   }
 
   Status ConvertBooleans(std::shared_ptr<Array>* out) {
+    PyAcquireGIL lock;
+
     PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
 
     int nbytes = util::bytes_for_bits(length_);
@@ -798,7 +805,15 @@ class ArrowDeserializer {
     }                                                               \
     break;
 
-Status ArrowToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
+Status ConvertArrayToPandas(const std::shared_ptr<Array>& arr, PyObject* py_ref,
+        PyObject** out) {
+  static std::string dummy_name = "dummy";
+  auto field = std::make_shared<Field>(dummy_name, arr->type());
+  auto col = std::make_shared<Column>(field, arr);
+  return ConvertColumnToPandas(col, py_ref, out);
+}
+
+Status ConvertColumnToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
         PyObject** out) {
   switch(col->type()->type) {
     FROM_ARROW_CASE(BOOL);
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index c3377685bcce9..141d1219e64db 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -31,6 +31,7 @@ namespace arrow {
 
 class Array;
 class Column;
+class MemoryPool;
 
 } // namespace arrow
 
@@ -39,7 +40,11 @@ namespace pyarrow {
 class Status;
 
 PYARROW_EXPORT
-Status ArrowToPandas(const std::shared_ptr<arrow::Column>& col, PyObject* py_ref,
+Status ConvertArrayToPandas(const std::shared_ptr<arrow::Array>& arr, PyObject* py_ref,
+    PyObject** out);
+
+PYARROW_EXPORT
+Status ConvertColumnToPandas(const std::shared_ptr<arrow::Column>& col, PyObject* py_ref,
     PyObject** out);
 
 PYARROW_EXPORT
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 96eed1654a777..50c2577b93c9b 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -120,8 +120,8 @@ class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
     Py_INCREF(arr);
 
     data_ = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
-    size_ = PyArray_SIZE(arr_);
-    capacity_ = size_ * PyArray_DESCR(arr_)->elsize;
+    size_ = PyArray_SIZE(arr_) * PyArray_DESCR(arr_)->elsize;
+    capacity_ = size_;
   }
 
   virtual ~NumPyBuffer() {
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index 9879b3474bcd0..7bf32ffa8d22b 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -85,7 +85,7 @@ arrow::Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
   ARROW_RETURN_NOT_OK(CheckPyError());
 
   PyObject* result = PyObject_CallMethod(file_, "write", "(O)", py_data);
-  Py_DECREF(py_data);
+  Py_XDECREF(py_data);
   Py_XDECREF(result);
   ARROW_RETURN_NOT_OK(CheckPyError());
   return arrow::Status::OK();

From bf749f55a1e24d79b08813a39ce51e9aaf6fb425 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 11 Oct 2016 20:11:48 -0700
Subject: [PATCH 0171/1644] ARROW-275: Add tests for UnionVector in Arrow File

Author: Julien Le Dem <julien@dremio.com>

Closes #169 from julienledem/union_test and squashes the following commits:

120f504 [Julien Le Dem] ARROW-275: Add tests for UnionVector in Arrow File
---
 .../main/codegen/templates/UnionReader.java   |   4 +
 .../main/codegen/templates/UnionVector.java   |  30 ++---
 .../org/apache/arrow/vector/VectorLoader.java |   2 +
 .../arrow/vector/schema/TypeLayout.java       |   3 +-
 .../arrow/vector/file/TestArrowFile.java      | 110 +++++++++++++++++-
 5 files changed, 127 insertions(+), 22 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index 7351ae3776f57..c56e95c89dc81 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -134,6 +134,10 @@ public void copyAsValue(UnionWriter writer) {
 
   </#list>
 
+  public int size() {
+    return getReaderForIndex(idx()).size();
+  }
+
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
           <#assign uncappedName = name?uncap_first/>
   <#assign boxedType = (minor.boxedType!type.boxedType) />
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index b14314d2b0dbb..5ca3f90148449 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -15,17 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-import com.google.common.collect.ImmutableList;
-import com.google.flatbuffers.FlatBufferBuilder;
-import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.flatbuf.Field;
-import org.apache.arrow.flatbuf.Type;
-import org.apache.arrow.flatbuf.Union;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-
-import java.util.ArrayList;
 import java.util.List;
 
 <@pp.dropOutputFile />
@@ -39,7 +28,9 @@
 <#include "/@includes/vv_imports.ftl" />
 import com.google.common.collect.ImmutableList;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
+import org.apache.arrow.vector.BaseDataValueVector;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
@@ -47,6 +38,7 @@
 import static org.apache.arrow.flatbuf.UnionMode.Sparse;
 
 
+
 /*
  * This class is generated using freemarker and the ${.template_name} template.
  */
@@ -81,6 +73,7 @@ public class UnionVector implements FieldVector {
   private ValueVector singleVector;
 
   private final CallBack callBack;
+  private final List<BufferBacked> innerVectors;
 
   public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
     this.name = name;
@@ -88,6 +81,7 @@ public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
     this.internalMap = new MapVector("internal", allocator, callBack);
     this.typeVector = new UInt1Vector("types", allocator);
     this.callBack = callBack;
+    this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
   }
 
   public BufferAllocator getAllocator() {
@@ -101,30 +95,28 @@ public MinorType getMinorType() {
 
   @Override
   public void initializeChildrenFromFields(List<Field> children) {
-    getMap().initializeChildrenFromFields(children);
+    internalMap.initializeChildrenFromFields(children);
   }
 
   @Override
   public List<FieldVector> getChildrenFromFields() {
-    return getMap().getChildrenFromFields();
+    return internalMap.getChildrenFromFields();
   }
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    // TODO
-    throw new UnsupportedOperationException();
+    BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+    this.valueCount = fieldNode.getLength();
   }
 
   @Override
   public List<ArrowBuf> getFieldBuffers() {
-    // TODO
-    throw new UnsupportedOperationException();
+    return BaseDataValueVector.unload(getFieldInnerVectors());
   }
 
   @Override
   public List<BufferBacked> getFieldInnerVectors() {
-    // TODO
-    throw new UnsupportedOperationException();
+     return this.innerVectors;
   }
   
   public NullableMapVector getMap() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 58ac68b82825d..b7040da9d8203 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -74,6 +74,8 @@ public void load(ArrowRecordBatch recordBatch) {
   }
 
   private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buffers, Iterator<ArrowFieldNode> nodes) {
+    checkArgument(nodes.hasNext(),
+        "no more field nodes for for field " + field + " and vector " + vector);
     ArrowFieldNode fieldNode = nodes.next();
     List<VectorLayout> typeLayout = field.getTypeLayout().getVectors();
     List<ArrowBuf> ownBuffers = new ArrayList<>(typeLayout.size());
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 06ae203bf4422..c5f53fe508d9f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -82,8 +82,7 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
             break;
           case UnionMode.Sparse:
             vectors = asList(
-                validityVector(),
-                typeVector()
+                typeVector() // type of the value at the index or 0 if null
                 );
             break;
           default:
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 7a5e7b58db98c..0f28d53295c37 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -266,7 +266,7 @@ private void validateComplexContent(int count, NullableMapVector parent) {
       Assert.assertEquals(i % 3, rootReader.reader("list").size());
       NullableTimeStampHolder h = new NullableTimeStampHolder();
       rootReader.reader("map").reader("timestamp").read(h);
-      Assert.assertEquals(i, h.value % COUNT);
+      Assert.assertEquals(i, h.value);
     }
   }
 
@@ -339,4 +339,112 @@ public void testWriteReadMultipleRBs() throws IOException {
     }
   }
 
+  @Test
+  public void testWriteReadUnion() throws IOException {
+    File file = new File("target/mytest_write_union.arrow");
+    int count = COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
+
+      writeUnionData(count, parent);
+
+      printVectors(parent.getChildrenFromFields());
+
+      validateUnionData(count, parent);
+
+      write(parent.getChild("root"), file);
+    }
+ // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        FileInputStream fileInputStream = new FileInputStream(file);
+        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)
+        ) {
+      ArrowFooter footer = arrowReader.readFooter();
+      Schema schema = footer.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+
+      NullableMapVector root = parent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
+      VectorLoader vectorLoader = new VectorLoader(schema, root);
+
+      List<ArrowBlock> recordBatches = footer.getRecordBatches();
+      for (ArrowBlock rbBlock : recordBatches) {
+        try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+          vectorLoader.load(recordBatch);
+        }
+        validateUnionData(count, parent);
+      }
+    }
+  }
+
+  public void validateUnionData(int count, MapVector parent) {
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < count; i++) {
+      rootReader.setPosition(i);
+      switch (i % 4) {
+      case 0:
+        Assert.assertEquals(i, rootReader.reader("union").readInteger().intValue());
+        break;
+      case 1:
+        Assert.assertEquals(i, rootReader.reader("union").readLong().longValue());
+        break;
+      case 2:
+        Assert.assertEquals(i % 3, rootReader.reader("union").size());
+        break;
+      case 3:
+        NullableTimeStampHolder h = new NullableTimeStampHolder();
+        rootReader.reader("union").reader("timestamp").read(h);
+        Assert.assertEquals(i, h.value);
+        break;
+      }
+    }
+  }
+
+  public void writeUnionData(int count, NullableMapVector parent) {
+    ArrowBuf varchar = allocator.buffer(3);
+    varchar.readerIndex(0);
+    varchar.setByte(0, 'a');
+    varchar.setByte(1, 'b');
+    varchar.setByte(2, 'c');
+    varchar.writerIndex(3);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("union");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("union");
+    ListWriter listWriter = rootWriter.list("union");
+    MapWriter mapWriter = rootWriter.map("union");
+    for (int i = 0; i < count; i++) {
+      switch (i % 4) {
+      case 0:
+        intWriter.setPosition(i);
+        intWriter.writeInt(i);
+        break;
+      case 1:
+        bigIntWriter.setPosition(i);
+        bigIntWriter.writeBigInt(i);
+        break;
+      case 2:
+        listWriter.setPosition(i);
+        listWriter.startList();
+        for (int j = 0; j < i % 3; j++) {
+          listWriter.varChar().writeVarChar(0, 3, varchar);
+        }
+        listWriter.endList();
+        break;
+      case 3:
+        mapWriter.setPosition(i);
+        mapWriter.start();
+        mapWriter.timeStamp("timestamp").writeTimeStamp(i);
+        mapWriter.end();
+        break;
+      }
+    }
+    writer.setValueCount(count);
+    varchar.release();
+  }
 }

From 4ecf327636c1373f601679fac18b7fcf7f382e1b Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 16 Oct 2016 16:21:59 -0400
Subject: [PATCH 0172/1644] ARROW-191: Python: Provide infrastructure for
 manylinux1 wheels

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #173 from xhochy/ARROW-191 and squashes the following commits:

278f8b0 [Uwe L. Korn] ARROW-191: Python: Provide infrastructure for manylinux1 wheels
---
 NOTICE.txt                                    |  3 +
 .../Dockerfile-parquet_arrow-base-x86_64      | 40 ++++++++++
 python/manylinux1/Dockerfile-x86_64           | 47 ++++++++++++
 python/manylinux1/README.md                   | 40 ++++++++++
 python/manylinux1/build_arrow.sh              | 76 +++++++++++++++++++
 5 files changed, 206 insertions(+)
 create mode 100644 python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
 create mode 100644 python/manylinux1/Dockerfile-x86_64
 create mode 100644 python/manylinux1/README.md
 create mode 100755 python/manylinux1/build_arrow.sh

diff --git a/NOTICE.txt b/NOTICE.txt
index 679bb59e6a97d..5c699ca022c1b 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -38,3 +38,6 @@ This product includes software from the CMake project
  * Copyright 2001-2009 Kitware, Inc.
  * Copyright 2012-2014 Continuum Analytics, Inc.
  * All rights reserved.
+
+This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
+ * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
new file mode 100644
index 0000000000000..714fa1a91b39e
--- /dev/null
+++ b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
@@ -0,0 +1,40 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+FROM arrow-base-x86_64
+
+WORKDIR /
+ADD http://zlib.net/zlib-1.2.8.tar.gz /zlib-1.2.8.tar.gz
+RUN tar xf zlib-1.2.8.tar.gz
+WORKDIR zlib-1.2.8
+RUN CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DCMAKE_BUILD_TYPE=Release .
+RUN make -j5 install
+
+WORKDIR /
+ADD https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz /snappy-1.1.3.tar.gz
+RUN tar xf snappy-1.1.3.tar.gz
+WORKDIR /snappy-1.1.3
+RUN ./configure --with-pic --prefix=/usr
+RUN make -j5 install
+
+WORKDIR /
+ADD http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz /thrift-0.9.1.tar.gz
+RUN tar xf thrift-0.9.1.tar.gz
+WORKDIR /thrift-0.9.1
+RUN ./configure LDFLAGS='-L/usr/lib64' CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --without-tests --with-cpp --prefix=/usr --disable-shared --enable-static
+RUN make -j5 install
+
+WORKDIR /
+RUN git clone https://github.com/apache/parquet-cpp.git
+WORKDIR /parquet-cpp
+RUN ARROW_HOME=/usr THRIFT_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
+RUN make -j5 install
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
new file mode 100644
index 0000000000000..e62a60111af4a
--- /dev/null
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -0,0 +1,47 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+FROM quay.io/pypa/manylinux1_x86_64:latest
+
+# Install dependencies
+RUN yum install -y flex openssl-devel
+
+WORKDIR /
+ADD http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz /boost_1_60_0.tar.gz
+RUN tar xf boost_1_60_0.tar.gz
+WORKDIR /boost_1_60_0
+RUN ./bootstrap.sh
+RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install
+
+WORKDIR /
+ADD https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz /cmake-3.5.2.tar.gz
+RUN tar xf cmake-3.5.2.tar.gz
+WORKDIR /cmake-3.5.2
+RUN ./configure --prefix=/usr
+RUN make -j5 install
+
+WORKDIR /
+ADD https://github.com/google/flatbuffers/archive/v1.3.0.tar.gz /flatbuffers-1.3.0.tar.gz
+RUN tar xf flatbuffers-1.3.0.tar.gz
+WORKDIR /flatbuffers-1.3.0
+RUN CXXFLAGS='-fPIC' cmake -DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .
+RUN make -j5 install
+
+WORKDIR /
+RUN git clone https://github.com/matthew-brett/multibuild.git
+WORKDIR /multibuild
+RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
+
+ADD arrow /arrow
+WORKDIR /arrow/cpp
+RUN FLATBUFFERS_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF .
+RUN make -j5 install
diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md
new file mode 100644
index 0000000000000..8cd9f6db004e5
--- /dev/null
+++ b/python/manylinux1/README.md
@@ -0,0 +1,40 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+## Manylinux1 wheels for Apache Arrow
+
+This folder provides base Docker images and an infrastructure to build
+`manylinux1` compatible Python wheels that should be installable on all
+Linux distributions published in last four years.
+
+The process is split up in two parts: There are base Docker images that build
+the native, Python-indenpendent dependencies. For these you can select if you
+want to also build the dependencies used for the Parquet support. Depending on
+these images, there is also a bash script that will build the pyarrow wheels
+for all supported Python versions and place them in the `dist` folder.
+
+### Build instructions
+
+```bash
+# Create a clean copy of the arrow source tree
+git clone ../../ arrow
+# Build the native baseimage
+docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
+# (optionally) build parquet-cpp
+docker build -t parquet_arrow-base-x86_64 -f Dockerfile-parquet_arrow-base-x86_64 .
+# Build the python packages
+docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
+# Now the new packages are located in the dist/ folder
+ls -l dist/
+```
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
new file mode 100755
index 0000000000000..0786b6f490a16
--- /dev/null
+++ b/python/manylinux1/build_arrow.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+# Usage:
+#   docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
+# or with Parquet support
+#   docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
+
+# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
+# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
+
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5}"
+
+# Package index with only manylinux1 builds
+MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
+
+source /multibuild/manylinux_utils.sh
+
+cd /arrow/python
+
+# PyArrow build configuration
+export PYARROW_CMAKE_OPTIONS='-DCMAKE_BUILD_TYPE=Release'
+# Need as otherwise arrow_io is sometimes not linked
+export LDFLAGS="-Wl,--no-as-needed"
+export ARROW_HOME="/usr"
+
+# Ensure the target directory exists
+mkdir -p /io/dist
+# Temporary directory to store the wheels that should be sent through auditwheel
+rm_mkdir unfixed_wheels
+
+PY35_BIN=/opt/python/cp35-cp35m/bin
+$PY35_BIN/pip install 'pyelftools<0.24'
+$PY35_BIN/pip install 'git+https://github.com/xhochy/auditwheel.git@pyarrow-fixes'
+
+# Override repair_wheelhouse function
+function repair_wheelhouse {
+    local in_dir=$1
+    local out_dir=$2
+    for whl in $in_dir/*.whl; do
+        if [[ $whl == *none-any.whl ]]; then
+            cp $whl $out_dir
+        else
+            # Store libraries directly in . not .libs to fix problems with libpyarrow.so linkage.
+            auditwheel -v repair -L . $whl -w $out_dir/
+        fi
+    done
+    chmod -R a+rwX $out_dir
+}
+
+for PYTHON in ${PYTHON_VERSIONS}; do
+    PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
+    PIP="$(cpython_path $PYTHON)/bin/pip"
+    PIPI_IO="$PIP install -f $MANYLINUX_URL"
+    PATH="$PATH:$(cpython_path $PYTHON)"
+
+    $PIPI_IO "numpy==1.9.0"
+    $PIPI_IO "cython==0.24"
+
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
+
+    rm_mkdir fixed_wheels
+    repair_wheelhouse dist /io/dist
+done
+

From 8520061d38c4aa407ac6453aff786833efa5cbaa Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 16 Oct 2016 16:23:04 -0400
Subject: [PATCH 0173/1644] ARROW-336: Run Apache Rat in Travis builds

@julienledem Integrated the rat call in the cpp build. It should fail if licenses are not matching. We could also make a separate `lint` Travis build but for the moment this seemed overkill to me.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #174 from xhochy/ARROW-336 and squashes the following commits:

25f797c [Uwe L. Korn] Make run-rat executable
6b6221f [Uwe L. Korn] ARROW-336: Run Apache Rat in Travis builds
---
 ci/travis_script_cpp.sh  |  4 +++
 dev/release/02-source.sh | 37 ++++------------------------
 dev/release/run-rat.sh   | 53 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 62 insertions(+), 32 deletions(-)
 create mode 100755 dev/release/run-rat.sh

diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index c3bd3b5f207a8..d555cab3e640c 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -16,6 +16,10 @@ set -e
 
 : ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
 
+# Check licenses according to Apache policy
+git archive HEAD -o arrow-src.tar.gz
+./dev/release/run-rat.sh arrow-src.tar.gz
+
 pushd $CPP_BUILD_DIR
 
 make lint
diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index 1bbe2e92753ce..bdaa5cc9340fe 100644
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,6 +18,8 @@
 # under the License.
 #
 
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
 if [ -z "$1" ]; then
   echo "Usage: $0 <version> <rc-num>"
   exit
@@ -56,36 +58,7 @@ tarball=$tag.tar.gz
 # archive (identical hashes) using the scm tag
 git archive $release_hash --prefix $tag/ -o $tarball
 
-# download apache rat
-curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.12/apache-rat-0.12.jar > apache-rat-0.12.jar
-
-RAT="java -jar apache-rat-0.12.jar -d "
-
-# generate the rat report
-$RAT $tarball \
-  -e ".*" \
-  -e mman.h \
-  -e "*_generated.h" \
-  -e random.h \
-  -e status.cc \
-  -e status.h \
-  -e asan_symbolize.py \
-  -e cpplint.py \
-  -e FindPythonLibsNew.cmake \
-  -e pax_global_header \
-  -e MANIFEST.in \
-  -e __init__.pxd \
-  -e __init__.py \
-  -e requirements.txt \
-  > rat.txt
-UNAPPROVED=`cat rat.txt  | grep "Unknown Licenses" | head -n 1 | cut -d " " -f 1`
-
-if [ "0" -eq "${UNAPPROVED}" ]; then
-  echo "No unnaproved licenses"
-else
-  echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
-  exit
-fi
+${SOURCE_DIR}/run-rat.sh $tarball
 
 # sign the archive
 gpg --armor --output ${tarball}.asc --detach-sig $tarball
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
new file mode 100755
index 0000000000000..d8ec6507fc4e5
--- /dev/null
+++ b/dev/release/run-rat.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# download apache rat
+curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.12/apache-rat-0.12.jar > apache-rat-0.12.jar
+
+RAT="java -jar apache-rat-0.12.jar -d "
+
+# generate the rat report
+$RAT $1 \
+  -e ".*" \
+  -e mman.h \
+  -e "*_generated.h" \
+  -e random.h \
+  -e status.cc \
+  -e status.h \
+  -e asan_symbolize.py \
+  -e cpplint.py \
+  -e FindPythonLibsNew.cmake \
+  -e pax_global_header \
+  -e MANIFEST.in \
+  -e __init__.pxd \
+  -e __init__.py \
+  -e requirements.txt \
+  > rat.txt
+cat rat.txt
+UNAPPROVED=`cat rat.txt  | grep "Unknown Licenses" | head -n 1 | cut -d " " -f 1`
+
+if [ "0" -eq "${UNAPPROVED}" ]; then
+  echo "No unnaproved licenses"
+else
+  echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
+  exit 1
+fi
+
+

From 8e8b17f992aa3bb3a642a93b44beb9b87d589fea Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 16 Oct 2016 16:23:54 -0400
Subject: [PATCH 0174/1644] ARROW-97: API documentation via sphinx-apidoc

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #175 from xhochy/ARROW-97 and squashes the following commits:

2ec3e11 [Uwe L. Korn] Add license headers
d838e81 [Uwe L. Korn] ARROW-97: API documentation via sphinx-apidoc
---
 ci/travis_script_python.sh  |   7 +
 python/README.md            |   7 +
 python/doc/.gitignore       |   3 +
 python/doc/Makefile         | 237 +++++++++++++++++++++++
 python/doc/conf.py          | 369 ++++++++++++++++++++++++++++++++++++
 python/doc/index.rst        |  28 +++
 python/doc/requirements.txt |   3 +
 python/pyarrow/parquet.pyx  |   8 +-
 8 files changed, 661 insertions(+), 1 deletion(-)
 create mode 100644 python/doc/.gitignore
 create mode 100644 python/doc/Makefile
 create mode 100644 python/doc/conf.py
 create mode 100644 python/doc/index.rst
 create mode 100644 python/doc/requirements.txt

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 97f0563240c75..55cb2a76f6db1 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -51,6 +51,13 @@ python_version_tests() {
 		 --inplace
 
   python -m pytest -vv -r sxX pyarrow
+
+  # Build documentation once
+  if [[ "$PYTHON_VERSION" == "3.5" ]]
+  then
+      pip install -r doc/requirements.txt
+      python setup.py build_sphinx
+  fi
 }
 
 # run tests for python 2.7 and 3.5
diff --git a/python/README.md b/python/README.md
index 6febcbcbcbfe7..e11f64564558c 100644
--- a/python/README.md
+++ b/python/README.md
@@ -47,3 +47,10 @@ The Arrow C++ library must be built with all options enabled and installed with
 python setup.py build_ext --inplace
 py.test pyarrow
 ```
+
+#### Build the documentation
+
+```bash
+pip install -r doc/requirements.txt
+python setup.py build_sphinx
+```
diff --git a/python/doc/.gitignore b/python/doc/.gitignore
new file mode 100644
index 0000000000000..87d04134d6fc3
--- /dev/null
+++ b/python/doc/.gitignore
@@ -0,0 +1,3 @@
+# auto-generated module documentation
+pyarrow*.rst
+modules.rst
diff --git a/python/doc/Makefile b/python/doc/Makefile
new file mode 100644
index 0000000000000..7257583952481
--- /dev/null
+++ b/python/doc/Makefile
@@ -0,0 +1,237 @@
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  applehelp  to make an Apple Help Book"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  epub3      to make an epub3"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+	@echo "  dummy      to check syntax errors of document sources"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*
+
+.PHONY: html
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: dirhtml
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+.PHONY: singlehtml
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+.PHONY: pickle
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+.PHONY: json
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+.PHONY: htmlhelp
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+.PHONY: qthelp
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pyarrow.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pyarrow.qhc"
+
+.PHONY: applehelp
+applehelp:
+	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+	@echo
+	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+	@echo "N.B. You won't be able to view it unless you put it in" \
+	      "~/Library/Documentation/Help or install it in your application" \
+	      "bundle."
+
+.PHONY: devhelp
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/pyarrow"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pyarrow"
+	@echo "# devhelp"
+
+.PHONY: epub
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+.PHONY: epub3
+epub3:
+	$(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3
+	@echo
+	@echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3."
+
+.PHONY: latex
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+.PHONY: latexpdf
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: latexpdfja
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: text
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+.PHONY: man
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+.PHONY: texinfo
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+.PHONY: info
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+.PHONY: gettext
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+.PHONY: changes
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+.PHONY: linkcheck
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."
+
+.PHONY: xml
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+.PHONY: pseudoxml
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+.PHONY: dummy
+dummy:
+	$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
+	@echo
+	@echo "Build finished. Dummy builder generates no files."
diff --git a/python/doc/conf.py b/python/doc/conf.py
new file mode 100644
index 0000000000000..99ac3512ec9d4
--- /dev/null
+++ b/python/doc/conf.py
@@ -0,0 +1,369 @@
+# -*- coding: utf-8 -*-
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import inspect
+import os
+import sys
+
+from sphinx import apidoc
+
+import sphinx_rtd_theme
+
+
+__location__ = os.path.join(os.getcwd(), os.path.dirname(
+        inspect.getfile(inspect.currentframe())))
+output_dir = os.path.join(__location__)
+module_dir = os.path.join(__location__, "..", "pyarrow")
+cmd_line_template = "sphinx-apidoc -f -e -o {outputdir} {moduledir}"
+cmd_line = cmd_line_template.format(outputdir=output_dir, moduledir=module_dir)
+apidoc.main(cmd_line.split(" "))
+
+sys.path.insert(0, os.path.abspath('..'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.doctest',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.viewcode',
+    'numpydoc'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'pyarrow'
+copyright = u'2016 Apache Software Foundation'
+author = u'Apache Software Foundation'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u''
+# The full version, including alpha/beta/rc tags.
+release = u''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#
+# today = ''
+#
+# Else, today_fmt is used as the format for a strftime call.
+#
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# The name for this set of Sphinx documents.
+# "<project> v<release> documentation" by default.
+#
+# html_title = u'pyarrow v0.1.0'
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#
+# html_logo = None
+
+# The name of an image file (relative to this directory) to use as a favicon of
+# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#
+# html_extra_path = []
+
+# If not None, a 'Last updated on:' timestamp is inserted at every page
+# bottom, using the given strftime format.
+# The empty string is equivalent to '%b %d, %Y'.
+#
+# html_last_updated_fmt = None
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+#
+# html_domain_indices = True
+
+# If false, no index is generated.
+#
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'
+#
+# html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# 'ja' uses this config value.
+# 'zh' user can custom change `jieba` dictionary path.
+#
+# html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#
+# html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'pyarrowdoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+     # The paper size ('letterpaper' or 'a4paper').
+     #
+     # 'papersize': 'letterpaper',
+
+     # The font size ('10pt', '11pt' or '12pt').
+     #
+     # 'pointsize': '10pt',
+
+     # Additional stuff for the LaTeX preamble.
+     #
+     # 'preamble': '',
+
+     # Latex figure (float) alignment
+     #
+     # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'pyarrow.tex', u'pyarrow Documentation',
+     u'Apache Arrow Team', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+#
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#
+# latex_appendices = []
+
+# It false, will not define \strong, \code, 	itleref, \crossref ... but only
+# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
+# packages.
+#
+# latex_keep_old_macro_names = True
+
+# If false, no module index is generated.
+#
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'pyarrow', u'pyarrow Documentation',
+     [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'pyarrow', u'pyarrow Documentation',
+     author, 'pyarrow', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+#
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#
+# texinfo_no_detailmenu = False
diff --git a/python/doc/index.rst b/python/doc/index.rst
new file mode 100644
index 0000000000000..550e544eef9e8
--- /dev/null
+++ b/python/doc/index.rst
@@ -0,0 +1,28 @@
+Apache Arrow (Python)
+=====================
+
+Arrow is a columnar in-memory analytics layer designed to accelerate big data.
+It houses a set of canonical in-memory representations of flat and hierarchical
+data along with multiple language-bindings for structure manipulation. It also
+provides IPC and common algorithm implementations.
+
+This is the documentation of the Python API of Apache Arrow. For more details
+on the format and other language bindings see
+`the main page for Arrow <https://arrow.apache.org/>`_. Here will we only
+detail the usage of the Python API for Arrow and the leaf libraries that add
+additional functionality such as reading Apache Parquet files into Arrow
+structures.
+
+.. toctree::
+   :maxdepth: 4
+   :hidden:
+
+   Module Reference <modules.rst>
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
diff --git a/python/doc/requirements.txt b/python/doc/requirements.txt
new file mode 100644
index 0000000000000..ce0793c31de26
--- /dev/null
+++ b/python/doc/requirements.txt
@@ -0,0 +1,3 @@
+numpydoc
+sphinx
+sphinx_rtd_theme
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index ca0176a7c0403..2abe57b33ed48 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -34,6 +34,10 @@ from pyarrow.io cimport NativeFile
 
 import six
 
+__all__ = [
+    'read_table',
+    'write_table'
+]
 
 cdef class ParquetReader:
     cdef:
@@ -76,9 +80,11 @@ cdef class ParquetReader:
 def read_table(source, columns=None):
     """
     Read a Table from Parquet format
+    
     Returns
     -------
-    table: pyarrow.Table
+    pyarrow.table.Table
+        Content of the file as a table (of columns)
     """
     cdef ParquetReader reader = ParquetReader()
 

From 732a2059d0c4493e451c566160b9d5d01dfe87be Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Oct 2016 13:44:34 -0400
Subject: [PATCH 0175/1644] ARROW-261: Refactor String/Binary code paths to
 reflect unnested (non-list-based) structure

Per discussions on the mailing list. This should in theory match the Java implementation.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #176 from wesm/ARROW-261 and squashes the following commits:

dca39ce [Wes McKinney] Make binary/string constants static to avoid memory-access-related segfaults in third party libraries
1e65b01 [Wes McKinney] Deprecate pyarrow::Status in favor of just arrow::Status. Conform pyarrow use of ArrayBuilder::Finish
9a1f77e [Wes McKinney] Add license header to index.rst
bd70cab [Wes McKinney] Complete refactoring, fix up IPC tests for flattened string/binary buffer/metadata layout
ae64f2e [Wes McKinney] Refactoring to reflect collaprsed list-like structure of Binary and String types. Not yet complete
---
 cpp/CMakeLists.txt                     |   1 -
 cpp/src/arrow/builder.h                |   2 +-
 cpp/src/arrow/ipc/adapter.cc           |  47 ++++----
 cpp/src/arrow/ipc/test-common.h        |  19 ++--
 cpp/src/arrow/type.h                   |  20 +---
 cpp/src/arrow/types/CMakeLists.txt     |   1 -
 cpp/src/arrow/types/binary.h           |  28 -----
 cpp/src/arrow/types/construct.cc       |  31 +-----
 cpp/src/arrow/types/construct.h        |   8 --
 cpp/src/arrow/types/json.cc            |  37 -------
 cpp/src/arrow/types/json.h             |  36 ------
 cpp/src/arrow/types/list-test.cc       |  15 ++-
 cpp/src/arrow/types/list.cc            |  42 ++++++-
 cpp/src/arrow/types/list.h             |  49 ++-------
 cpp/src/arrow/types/primitive-test.cc  |  26 +++--
 cpp/src/arrow/types/primitive.cc       |  31 ++++--
 cpp/src/arrow/types/primitive.h        |  31 +++---
 cpp/src/arrow/types/string-test.cc     |  33 +++---
 cpp/src/arrow/types/string.cc          | 101 ++++++++++++++---
 cpp/src/arrow/types/string.h           |  49 +++++----
 cpp/src/arrow/types/struct-test.cc     |  21 +++-
 cpp/src/arrow/types/struct.cc          |  14 +++
 cpp/src/arrow/types/struct.h           |  17 +--
 cpp/src/arrow/util/status.cc           |   6 +
 cpp/src/arrow/util/status.h            |  17 ++-
 python/CMakeLists.txt                  |   2 -
 python/doc/index.rst                   |  18 ++-
 python/pyarrow/error.pxd               |   4 +-
 python/pyarrow/error.pyx               |  10 +-
 python/pyarrow/includes/pyarrow.pxd    |  35 ++----
 python/pyarrow/io.pyx                  |  56 +++++-----
 python/pyarrow/ipc.pyx                 |  18 +--
 python/pyarrow/parquet.pyx             |  14 +--
 python/src/pyarrow/adapters/builtin.cc |  39 ++++---
 python/src/pyarrow/adapters/builtin.h  |   9 +-
 python/src/pyarrow/adapters/pandas.cc  |  32 +++---
 python/src/pyarrow/adapters/pandas.h   |  15 ++-
 python/src/pyarrow/api.h               |   2 -
 python/src/pyarrow/common.cc           |  12 +-
 python/src/pyarrow/common.h            |   7 --
 python/src/pyarrow/io.cc               |  59 +++++-----
 python/src/pyarrow/status.cc           |  92 ----------------
 python/src/pyarrow/status.h            | 146 -------------------------
 43 files changed, 484 insertions(+), 768 deletions(-)
 delete mode 100644 cpp/src/arrow/types/binary.h
 delete mode 100644 cpp/src/arrow/types/json.cc
 delete mode 100644 cpp/src/arrow/types/json.h
 delete mode 100644 python/src/pyarrow/status.cc
 delete mode 100644 python/src/pyarrow/status.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d682dc76f8ced..6f954830b6334 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -681,7 +681,6 @@ set(ARROW_SRCS
 
   src/arrow/types/construct.cc
   src/arrow/types/decimal.cc
-  src/arrow/types/json.cc
   src/arrow/types/list.cc
   src/arrow/types/primitive.cc
   src/arrow/types/string.cc
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 646a6f24e9df8..cef17e5aabab9 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -93,7 +93,7 @@ class ARROW_EXPORT ArrayBuilder {
 
   // Creates new array object to hold the contents of the builder and transfers
   // ownership of the data.  This resets all variables on the builder.
-  virtual std::shared_ptr<Array> Finish() = 0;
+  virtual Status Finish(std::shared_ptr<Array>* out) = 0;
 
   const std::shared_ptr<DataType>& type() const { return type_; }
 
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index cd8ab53a31d1f..f84cb264f70e1 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -78,22 +78,6 @@ static bool IsPrimitive(const DataType* type) {
   }
 }
 
-static bool IsListType(const DataType* type) {
-  DCHECK(type != nullptr);
-  switch (type->type) {
-    // TODO(emkornfield) grouping like this are used in a few places in the
-    // code consider using pattern like:
-    // http://stackoverflow.com/questions/26784685/c-macro-for-calling-function-based-on-enum-type
-    //
-    case Type::BINARY:
-    case Type::LIST:
-    case Type::STRING:
-      return true;
-    default:
-      return false;
-  }
-}
-
 // ----------------------------------------------------------------------
 // Record batch write path
 
@@ -115,7 +99,11 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
   if (IsPrimitive(arr_type)) {
     const auto prim_arr = static_cast<const PrimitiveArray*>(arr);
     buffers->push_back(prim_arr->data());
-  } else if (IsListType(arr_type)) {
+  } else if (arr->type_enum() == Type::STRING || arr->type_enum() == Type::BINARY) {
+    const auto binary_arr = static_cast<const BinaryArray*>(arr);
+    buffers->push_back(binary_arr->offsets());
+    buffers->push_back(binary_arr->data());
+  } else if (arr->type_enum() == Type::LIST) {
     const auto list_arr = static_cast<const ListArray*>(arr);
     buffers->push_back(list_arr->offset_buffer());
     RETURN_NOT_OK(VisitArray(
@@ -331,9 +319,21 @@ class RecordBatchReader::RecordBatchReaderImpl {
       }
       return MakePrimitiveArray(
           type, field_meta.length, data, field_meta.null_count, null_bitmap, out);
-    }
+    } else if (type->type == Type::STRING || type->type == Type::BINARY) {
+      std::shared_ptr<Buffer> offsets;
+      std::shared_ptr<Buffer> values;
+      RETURN_NOT_OK(GetBuffer(buffer_index_++, &offsets));
+      RETURN_NOT_OK(GetBuffer(buffer_index_++, &values));
 
-    if (IsListType(type.get())) {
+      if (type->type == Type::STRING) {
+        *out = std::make_shared<StringArray>(
+            field_meta.length, offsets, values, field_meta.null_count, null_bitmap);
+      } else {
+        *out = std::make_shared<BinaryArray>(
+            field_meta.length, offsets, values, field_meta.null_count, null_bitmap);
+      }
+      return Status::OK();
+    } else if (type->type == Type::LIST) {
       std::shared_ptr<Buffer> offsets;
       RETURN_NOT_OK(GetBuffer(buffer_index_++, &offsets));
       const int num_children = type->num_children();
@@ -346,11 +346,10 @@ class RecordBatchReader::RecordBatchReaderImpl {
       std::shared_ptr<Array> values_array;
       RETURN_NOT_OK(
           NextArray(type->child(0).get(), max_recursion_depth - 1, &values_array));
-      return MakeListArray(type, field_meta.length, offsets, values_array,
-          field_meta.null_count, null_bitmap, out);
-    }
-
-    if (type->type == Type::STRUCT) {
+      *out = std::make_shared<ListArray>(type, field_meta.length, offsets, values_array,
+          field_meta.null_count, null_bitmap);
+      return Status::OK();
+    } else if (type->type == Type::STRUCT) {
       const int num_children = type->num_children();
       std::vector<ArrayPtr> fields;
       fields.reserve(num_children);
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 7d02bc302f40e..13bbbebde8aa1 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -42,7 +42,7 @@ const auto kListInt32 = std::make_shared<ListType>(kInt32);
 const auto kListListInt32 = std::make_shared<ListType>(kListInt32);
 
 Status MakeRandomInt32Array(
-    int32_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* array) {
+    int32_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
   std::shared_ptr<PoolBuffer> data;
   test::MakeRandomInt32PoolBuffer(length, pool, &data);
   const auto kInt32 = std::make_shared<Int32Type>();
@@ -52,16 +52,14 @@ Status MakeRandomInt32Array(
     test::MakeRandomBytePoolBuffer(length, pool, &valid_bytes);
     RETURN_NOT_OK(builder.Append(
         reinterpret_cast<const int32_t*>(data->data()), length, valid_bytes->data()));
-    *array = builder.Finish();
-    return Status::OK();
+    return builder.Finish(out);
   }
   RETURN_NOT_OK(builder.Append(reinterpret_cast<const int32_t*>(data->data()), length));
-  *array = builder.Finish();
-  return Status::OK();
+  return builder.Finish(out);
 }
 
 Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_lists,
-    bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* array) {
+    bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
   // Create the null list values
   std::vector<uint8_t> valid_lists(num_lists);
   const double null_percent = include_nulls ? 0.1 : 0;
@@ -90,8 +88,8 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
   }
   ListBuilder builder(pool, child_array);
   RETURN_NOT_OK(builder.Append(offsets.data(), num_lists, valid_lists.data()));
-  *array = builder.Finish();
-  return (*array)->Validate();
+  RETURN_NOT_OK(builder.Finish(out));
+  return (*out)->Validate();
 }
 
 typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
@@ -115,7 +113,7 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
 template <class Builder, class RawType>
 Status MakeRandomBinaryArray(
-    const TypePtr& type, int32_t length, MemoryPool* pool, ArrayPtr* array) {
+    const TypePtr& type, int32_t length, MemoryPool* pool, ArrayPtr* out) {
   const std::vector<std::string> values = {
       "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
   Builder builder(pool, type);
@@ -130,8 +128,7 @@ Status MakeRandomBinaryArray(
           builder.Append(reinterpret_cast<const RawType*>(value.data()), value.size()));
     }
   }
-  *array = builder.Finish();
-  return Status::OK();
+  return builder.Finish(out);
 }
 
 Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index b4c3721a72895..ea8516fc34798 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -242,7 +242,7 @@ struct ARROW_EXPORT DoubleType : public PrimitiveType<DoubleType> {
 struct ARROW_EXPORT ListType : public DataType {
   // List can contain any other logical value type
   explicit ListType(const std::shared_ptr<DataType>& value_type)
-      : ListType(value_type, Type::LIST) {}
+      : ListType(std::make_shared<Field>("item", value_type)) {}
 
   explicit ListType(const std::shared_ptr<Field>& value_field) : DataType(Type::LIST) {
     children_ = {value_field};
@@ -255,26 +255,17 @@ struct ARROW_EXPORT ListType : public DataType {
   static char const* name() { return "list"; }
 
   std::string ToString() const override;
-
- protected:
-  // Constructor for classes that are implemented as List Arrays.
-  ListType(const std::shared_ptr<DataType>& value_type, Type::type logical_type)
-      : DataType(logical_type) {
-    // TODO ARROW-187 this can technically fail, make a constructor method ?
-    children_ = {std::make_shared<Field>("item", value_type)};
-  }
 };
 
 // BinaryType type is reprsents lists of 1-byte values.
-struct ARROW_EXPORT BinaryType : public ListType {
+struct ARROW_EXPORT BinaryType : public DataType {
   BinaryType() : BinaryType(Type::BINARY) {}
   static char const* name() { return "binary"; }
   std::string ToString() const override;
 
  protected:
   // Allow subclasses to change the logical type.
-  explicit BinaryType(Type::type logical_type)
-      : ListType(std::shared_ptr<DataType>(new UInt8Type()), logical_type) {}
+  explicit BinaryType(Type::type logical_type) : DataType(logical_type) {}
 };
 
 // UTF encoded strings
@@ -284,9 +275,6 @@ struct ARROW_EXPORT StringType : public BinaryType {
   static char const* name() { return "string"; }
 
   std::string ToString() const override;
-
- protected:
-  explicit StringType(Type::type logical_type) : BinaryType(logical_type) {}
 };
 
 struct ARROW_EXPORT StructType : public DataType {
@@ -300,7 +288,7 @@ struct ARROW_EXPORT StructType : public DataType {
 
 // These will be defined elsewhere
 template <typename T>
-struct type_traits {};
+struct TypeTraits {};
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
index 72a8e77664610..9f7816989827d 100644
--- a/cpp/src/arrow/types/CMakeLists.txt
+++ b/cpp/src/arrow/types/CMakeLists.txt
@@ -25,7 +25,6 @@ install(FILES
   construct.h
   datetime.h
   decimal.h
-  json.h
   list.h
   primitive.h
   string.h
diff --git a/cpp/src/arrow/types/binary.h b/cpp/src/arrow/types/binary.h
deleted file mode 100644
index 201fbb6e79536..0000000000000
--- a/cpp/src/arrow/types/binary.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_BINARY_H
-#define ARROW_TYPES_BINARY_H
-
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-
-namespace arrow {}  // namespace arrow
-
-#endif  // ARROW_TYPES_BINARY_H
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 0b71ea965516c..67245f8ea1fda 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -59,6 +59,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(DOUBLE, DoubleBuilder);
 
     BUILDER_CASE(STRING, StringBuilder);
+    BUILDER_CASE(BINARY, BinaryBuilder);
 
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
@@ -105,10 +106,10 @@ Status MakePrimitiveArray(const TypePtr& type, int32_t length,
     MAKE_PRIMITIVE_ARRAY_CASE(INT32, Int32Array);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
     MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
     MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
     MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
     MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP_DOUBLE, DoubleArray);
     default:
       return Status::NotImplemented(type->ToString());
@@ -120,30 +121,4 @@ Status MakePrimitiveArray(const TypePtr& type, int32_t length,
 #endif
 }
 
-Status MakeListArray(const TypePtr& type, int32_t length,
-    const std::shared_ptr<Buffer>& offsets, const ArrayPtr& values, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
-  switch (type->type) {
-    case Type::BINARY:
-      out->reset(new BinaryArray(type, length, offsets, values, null_count, null_bitmap));
-      break;
-
-    case Type::LIST:
-      out->reset(new ListArray(type, length, offsets, values, null_count, null_bitmap));
-      break;
-
-    case Type::DECIMAL_TEXT:
-    case Type::STRING:
-      out->reset(new StringArray(type, length, offsets, values, null_count, null_bitmap));
-      break;
-    default:
-      return Status::NotImplemented(type->ToString());
-  }
-#ifdef NDEBUG
-  return Status::OK();
-#else
-  return (*out)->Validate();
-#endif
-}
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
index afdadbe079013..e18e946d1a64c 100644
--- a/cpp/src/arrow/types/construct.h
+++ b/cpp/src/arrow/types/construct.h
@@ -42,14 +42,6 @@ Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
     int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
 
-// Create new list arrays for logical types that are backed by ListArrays (e.g. list of
-// primitives and strings)
-// TODO(emkornfield) split up string vs list?
-Status ARROW_EXPORT MakeListArray(const std::shared_ptr<DataType>& type, int32_t length,
-    const std::shared_ptr<Buffer>& offests, const std::shared_ptr<Array>& values,
-    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
-    std::shared_ptr<Array>* out);
-
 }  // namespace arrow
 
 #endif  // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/types/json.cc b/cpp/src/arrow/types/json.cc
deleted file mode 100644
index 89240fc22bb2c..0000000000000
--- a/cpp/src/arrow/types/json.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/json.h"
-
-#include <vector>
-
-#include "arrow/type.h"
-#include "arrow/types/union.h"
-
-namespace arrow {
-
-static const TypePtr Null(new NullType());
-static const TypePtr Int32(new Int32Type());
-static const TypePtr String(new StringType());
-static const TypePtr Double(new DoubleType());
-static const TypePtr Bool(new BooleanType());
-
-static const std::vector<TypePtr> kJsonTypes = {Null, Int32, String, Double, Bool};
-TypePtr JSONScalar::dense_type = TypePtr(new DenseUnionType(kJsonTypes));
-TypePtr JSONScalar::sparse_type = TypePtr(new SparseUnionType(kJsonTypes));
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/types/json.h b/cpp/src/arrow/types/json.h
deleted file mode 100644
index 9de961f79a60a..0000000000000
--- a/cpp/src/arrow/types/json.h
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_JSON_H
-#define ARROW_TYPES_JSON_H
-
-#include "arrow/type.h"
-
-namespace arrow {
-
-struct JSONScalar : public DataType {
-  bool dense;
-
-  static TypePtr dense_type;
-  static TypePtr sparse_type;
-
-  explicit JSONScalar(bool dense = true) : DataType(Type::JSON_SCALAR), dense(dense) {}
-};
-
-}  // namespace arrow
-
-#endif  // ARROW_TYPES_JSON_H
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index 2e41b4a61caf2..12c539495a28b 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -76,7 +76,11 @@ class TestListBuilder : public TestBuilder {
     builder_ = std::dynamic_pointer_cast<ListBuilder>(tmp);
   }
 
-  void Done() { result_ = std::dynamic_pointer_cast<ListArray>(builder_->Finish()); }
+  void Done() {
+    std::shared_ptr<Array> out;
+    EXPECT_OK(builder_->Finish(&out));
+    result_ = std::dynamic_pointer_cast<ListArray>(out);
+  }
 
  protected:
   TypePtr value_type_;
@@ -98,14 +102,17 @@ TEST_F(TestListBuilder, Equality) {
   // setup two equal arrays
   ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
   ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
-  array = builder_->Finish();
+
+  ASSERT_OK(builder_->Finish(&array));
   ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
   ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
-  equal_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Finish(&equal_array));
   // now an unequal one
   ASSERT_OK(builder_->Append(unequal_offsets.data(), unequal_offsets.size()));
   ASSERT_OK(vb->Append(unequal_values.data(), unequal_values.size()));
-  unequal_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Finish(&unequal_array));
 
   // Test array equality
   EXPECT_TRUE(array->Equals(array));
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index 6334054caf84a..ef2ec22cb5336 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -25,7 +25,7 @@ bool ListArray::EqualsExact(const ListArray& other) const {
   if (null_count_ != other.null_count_) { return false; }
 
   bool equal_offsets =
-      offset_buf_->Equals(*other.offset_buf_, (length_ + 1) * sizeof(int32_t));
+      offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
   if (!equal_offsets) { return false; }
   bool equal_null_bitmap = true;
   if (null_count_ > 0) {
@@ -72,10 +72,10 @@ bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_st
 
 Status ListArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
-  if (!offset_buf_) { return Status::Invalid("offset_buf_ was null"); }
-  if (offset_buf_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
+  if (!offset_buffer_) { return Status::Invalid("offset_buffer_ was null"); }
+  if (offset_buffer_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
     std::stringstream ss;
-    ss << "offset buffer size (bytes): " << offset_buf_->size()
+    ss << "offset buffer size (bytes): " << offset_buffer_->size()
        << " isn't large enough for length: " << length_;
     return Status::Invalid(ss.str());
   }
@@ -121,4 +121,38 @@ Status ListArray::Validate() const {
   return Status::OK();
 }
 
+Status ListBuilder::Init(int32_t elements) {
+  DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+  // one more then requested for offsets
+  return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
+}
+
+Status ListBuilder::Resize(int32_t capacity) {
+  DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
+  // one more then requested for offsets
+  RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
+  return ArrayBuilder::Resize(capacity);
+}
+
+Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Array> items = values_;
+  if (!items) { RETURN_NOT_OK(value_builder_->Finish(&items)); }
+
+  RETURN_NOT_OK(offset_builder_.Append<int32_t>(items->length()));
+  std::shared_ptr<Buffer> offsets = offset_builder_.Finish();
+
+  *out = std::make_shared<ListArray>(
+      type_, length_, offsets, items, null_count_, null_bitmap_);
+
+  Reset();
+
+  return Status::OK();
+}
+
+void ListBuilder::Reset() {
+  capacity_ = length_ = null_count_ = 0;
+  null_bitmap_ = nullptr;
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index f3894510d091a..9440ffed4bf8a 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -43,9 +43,9 @@ class ARROW_EXPORT ListArray : public Array {
       const ArrayPtr& values, int32_t null_count = 0,
       std::shared_ptr<Buffer> null_bitmap = nullptr)
       : Array(type, length, null_count, null_bitmap) {
-    offset_buf_ = offsets;
-    offsets_ = offsets == nullptr ? nullptr
-                                  : reinterpret_cast<const int32_t*>(offset_buf_->data());
+    offset_buffer_ = offsets;
+    offsets_ = offsets == nullptr ? nullptr : reinterpret_cast<const int32_t*>(
+                                                  offset_buffer_->data());
     values_ = values;
   }
 
@@ -57,7 +57,7 @@ class ARROW_EXPORT ListArray : public Array {
   // with this array.
   const std::shared_ptr<Array>& values() const { return values_; }
   const std::shared_ptr<Buffer> offset_buffer() const {
-    return std::static_pointer_cast<Buffer>(offset_buf_);
+    return std::static_pointer_cast<Buffer>(offset_buffer_);
   }
 
   const std::shared_ptr<DataType>& value_type() const { return values_->type(); }
@@ -77,7 +77,7 @@ class ARROW_EXPORT ListArray : public Array {
       const ArrayPtr& arr) const override;
 
  protected:
-  std::shared_ptr<Buffer> offset_buf_;
+  std::shared_ptr<Buffer> offset_buffer_;
   const int32_t* offsets_;
   ArrayPtr values_;
 };
@@ -119,19 +119,9 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
 
   virtual ~ListBuilder() {}
 
-  Status Init(int32_t elements) override {
-    DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
-    RETURN_NOT_OK(ArrayBuilder::Init(elements));
-    // one more then requested for offsets
-    return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
-  }
-
-  Status Resize(int32_t capacity) override {
-    DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
-    // one more then requested for offsets
-    RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
-    return ArrayBuilder::Resize(capacity);
-  }
+  Status Init(int32_t elements) override;
+  Status Resize(int32_t capacity) override;
+  Status Finish(std::shared_ptr<Array>* out) override;
 
   // Vector append
   //
@@ -145,27 +135,6 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  // The same as Finalize but allows for overridding the c++ type
-  template <typename Container>
-  std::shared_ptr<Array> Transfer() {
-    std::shared_ptr<Array> items = values_;
-    if (!items) { items = value_builder_->Finish(); }
-
-    offset_builder_.Append<int32_t>(items->length());
-
-    const auto offsets_buffer = offset_builder_.Finish();
-    auto result = std::make_shared<Container>(
-        type_, length_, offsets_buffer, items, null_count_, null_bitmap_);
-
-    // TODO(emkornfield) make a reset method
-    capacity_ = length_ = null_count_ = 0;
-    null_bitmap_ = nullptr;
-
-    return result;
-  }
-
-  std::shared_ptr<Array> Finish() override { return Transfer<ListArray>(); }
-
   // Start a new variable-length list slot
   //
   // This function should be called before beginning to append elements to the
@@ -188,6 +157,8 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   BufferBuilder offset_builder_;
   std::shared_ptr<ArrayBuilder> value_builder_;
   std::shared_ptr<Array> values_;
+
+  void Reset();
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 5ac2867932df7..121bd4794f291 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -123,8 +123,11 @@ class TestPrimitiveBuilder : public TestBuilder {
 
     auto expected =
         std::make_shared<ArrayType>(size, ex_data, ex_null_count, ex_null_bitmap);
-    std::shared_ptr<ArrayType> result =
-        std::dynamic_pointer_cast<ArrayType>(builder->Finish());
+
+    std::shared_ptr<Array> out;
+    ASSERT_OK(builder->Finish(&out));
+
+    std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(out);
 
     // Builder is now reset
     ASSERT_EQ(0, builder->length());
@@ -216,8 +219,10 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 
   auto expected =
       std::make_shared<BooleanArray>(size, ex_data, ex_null_count, ex_null_bitmap);
-  std::shared_ptr<BooleanArray> result =
-      std::dynamic_pointer_cast<BooleanArray>(builder->Finish());
+
+  std::shared_ptr<Array> out;
+  ASSERT_OK(builder->Finish(&out));
+  std::shared_ptr<BooleanArray> result = std::dynamic_pointer_cast<BooleanArray>(out);
 
   // Builder is now reset
   ASSERT_EQ(0, builder->length());
@@ -254,7 +259,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestInit) {
   int n = 1000;
   ASSERT_OK(this->builder_->Reserve(n));
   ASSERT_EQ(util::next_power2(n), this->builder_->capacity());
-  ASSERT_EQ(util::next_power2(type_traits<Type>::bytes_required(n)),
+  ASSERT_EQ(util::next_power2(TypeTraits<Type>::bytes_required(n)),
       this->builder_->data()->size());
 
   // unsure if this should go in all builder classes
@@ -267,7 +272,8 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
     ASSERT_OK(this->builder_->AppendNull());
   }
 
-  auto result = this->builder_->Finish();
+  std::shared_ptr<Array> result;
+  ASSERT_OK(this->builder_->Finish(&result));
 
   for (int i = 0; i < size; ++i) {
     ASSERT_TRUE(result->IsNull(i)) << i;
@@ -298,7 +304,8 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
   }
 
   do {
-    std::shared_ptr<Array> result = this->builder_->Finish();
+    std::shared_ptr<Array> result;
+    ASSERT_OK(this->builder_->Finish(&result));
   } while (false);
 
   ASSERT_EQ(memory_before, this->pool_->bytes_allocated());
@@ -315,8 +322,7 @@ Status MakeArray(const vector<uint8_t>& valid_bytes, const vector<T>& draws, int
       RETURN_NOT_OK(builder->AppendNull());
     }
   }
-  *out = builder->Finish();
-  return Status::OK();
+  return builder->Finish(out);
 }
 
 TYPED_TEST(TestPrimitiveBuilder, Equality) {
@@ -465,7 +471,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestResize) {
   ASSERT_OK(this->builder_->Reserve(cap));
   ASSERT_EQ(cap, this->builder_->capacity());
 
-  ASSERT_EQ(type_traits<Type>::bytes_required(cap), this->builder_->data()->size());
+  ASSERT_EQ(TypeTraits<Type>::bytes_required(cap), this->builder_->data()->size());
   ASSERT_EQ(util::bytes_for_bits(cap), this->builder_->null_bitmap()->size());
 }
 
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 9ba2ebdcc2d5b..3a05ccfdf1861 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -69,12 +69,25 @@ bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
   return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
 }
 
+template class NumericArray<UInt8Type>;
+template class NumericArray<UInt16Type>;
+template class NumericArray<UInt32Type>;
+template class NumericArray<UInt64Type>;
+template class NumericArray<Int8Type>;
+template class NumericArray<Int16Type>;
+template class NumericArray<Int32Type>;
+template class NumericArray<Int64Type>;
+template class NumericArray<TimestampType>;
+template class NumericArray<FloatType>;
+template class NumericArray<DoubleType>;
+template class NumericArray<BooleanType>;
+
 template <typename T>
 Status PrimitiveBuilder<T>::Init(int32_t capacity) {
   RETURN_NOT_OK(ArrayBuilder::Init(capacity));
   data_ = std::make_shared<PoolBuffer>(pool_);
 
-  int64_t nbytes = type_traits<T>::bytes_required(capacity);
+  int64_t nbytes = TypeTraits<T>::bytes_required(capacity);
   RETURN_NOT_OK(data_->Resize(nbytes));
   // TODO(emkornfield) valgrind complains without this
   memset(data_->mutable_data(), 0, nbytes);
@@ -93,10 +106,9 @@ Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
   } else {
     RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
     const int64_t old_bytes = data_->size();
-    const int64_t new_bytes = type_traits<T>::bytes_required(capacity);
+    const int64_t new_bytes = TypeTraits<T>::bytes_required(capacity);
     RETURN_NOT_OK(data_->Resize(new_bytes));
     raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
-
     memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
   }
   return Status::OK();
@@ -108,7 +120,7 @@ Status PrimitiveBuilder<T>::Append(
   RETURN_NOT_OK(Reserve(length));
 
   if (length > 0) {
-    memcpy(raw_data_ + length_, values, type_traits<T>::bytes_required(length));
+    memcpy(raw_data_ + length_, values, TypeTraits<T>::bytes_required(length));
   }
 
   // length_ is update by these
@@ -118,13 +130,18 @@ Status PrimitiveBuilder<T>::Append(
 }
 
 template <typename T>
-std::shared_ptr<Array> PrimitiveBuilder<T>::Finish() {
-  std::shared_ptr<Array> result = std::make_shared<typename type_traits<T>::ArrayType>(
+Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
+  const int64_t bytes_required = TypeTraits<T>::bytes_required(length_);
+  if (bytes_required > 0 && bytes_required < data_->size()) {
+    // Trim buffers
+    RETURN_NOT_OK(data_->Resize(bytes_required));
+  }
+  *out = std::make_shared<typename TypeTraits<T>::ArrayType>(
       type_, length_, data_, null_count_, null_bitmap_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
-  return result;
+  return Status::OK();
 }
 
 template <>
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index c643783f681bd..f21470d96e45b 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -91,7 +91,9 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   value_type Value(int i) const { return raw_data()[i]; }
 };
 
-#define NUMERIC_ARRAY_DECL(NAME, TypeClass) using NAME = NumericArray<TypeClass>;
+#define NUMERIC_ARRAY_DECL(NAME, TypeClass) \
+  using NAME = NumericArray<TypeClass>;     \
+  extern template class ARROW_EXPORT NumericArray<TypeClass>;
 
 NUMERIC_ARRAY_DECL(UInt8Array, UInt8Type);
 NUMERIC_ARRAY_DECL(Int8Array, Int8Type);
@@ -139,8 +141,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   Status Append(
       const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
 
-  std::shared_ptr<Array> Finish() override;
-
+  Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int32_t capacity) override;
 
   // Increase the capacity of the builder to accommodate at least the indicated
@@ -183,77 +184,77 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
 };
 
 template <>
-struct type_traits<UInt8Type> {
+struct TypeTraits<UInt8Type> {
   typedef UInt8Array ArrayType;
 
   static inline int bytes_required(int elements) { return elements; }
 };
 
 template <>
-struct type_traits<Int8Type> {
+struct TypeTraits<Int8Type> {
   typedef Int8Array ArrayType;
 
   static inline int bytes_required(int elements) { return elements; }
 };
 
 template <>
-struct type_traits<UInt16Type> {
+struct TypeTraits<UInt16Type> {
   typedef UInt16Array ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
 };
 
 template <>
-struct type_traits<Int16Type> {
+struct TypeTraits<Int16Type> {
   typedef Int16Array ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int16_t); }
 };
 
 template <>
-struct type_traits<UInt32Type> {
+struct TypeTraits<UInt32Type> {
   typedef UInt32Array ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(uint32_t); }
 };
 
 template <>
-struct type_traits<Int32Type> {
+struct TypeTraits<Int32Type> {
   typedef Int32Array ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int32_t); }
 };
 
 template <>
-struct type_traits<UInt64Type> {
+struct TypeTraits<UInt64Type> {
   typedef UInt64Array ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(uint64_t); }
 };
 
 template <>
-struct type_traits<Int64Type> {
+struct TypeTraits<Int64Type> {
   typedef Int64Array ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
 };
 
 template <>
-struct type_traits<TimestampType> {
+struct TypeTraits<TimestampType> {
   typedef TimestampArray ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
 };
 template <>
 
-struct type_traits<FloatType> {
+struct TypeTraits<FloatType> {
   typedef FloatArray ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(float); }
 };
 
 template <>
-struct type_traits<DoubleType> {
+struct TypeTraits<DoubleType> {
   typedef DoubleArray ArrayType;
 
   static inline int bytes_required(int elements) { return elements * sizeof(double); }
@@ -293,7 +294,7 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
 };
 
 template <>
-struct type_traits<BooleanType> {
+struct TypeTraits<BooleanType> {
   typedef BooleanArray ArrayType;
 
   static inline int bytes_required(int elements) {
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index 6807b00e8ca99..d897e30a3c6a2 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -66,18 +66,13 @@ class TestStringContainer : public ::testing::Test {
 
   void MakeArray() {
     length_ = offsets_.size() - 1;
-    int nchars = chars_.size();
-
     value_buf_ = test::to_buffer(chars_);
-    values_ = ArrayPtr(new UInt8Array(nchars, value_buf_));
-
     offsets_buf_ = test::to_buffer(offsets_);
-
     null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<StringArray>(
-        length_, offsets_buf_, values_, null_count_, null_bitmap_);
+        length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
   }
 
  protected:
@@ -94,7 +89,6 @@ class TestStringContainer : public ::testing::Test {
   int null_count_;
   int length_;
 
-  ArrayPtr values_;
   std::shared_ptr<StringArray> strings_;
 };
 
@@ -122,7 +116,7 @@ TEST_F(TestStringContainer, TestListFunctions) {
 
 TEST_F(TestStringContainer, TestDestructor) {
   auto arr = std::make_shared<StringArray>(
-      length_, offsets_buf_, values_, null_count_, null_bitmap_);
+      length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
 }
 
 TEST_F(TestStringContainer, TestGetString) {
@@ -147,7 +141,10 @@ class TestStringBuilder : public TestBuilder {
   }
 
   void Done() {
-    result_ = std::dynamic_pointer_cast<StringArray>(builder_->Finish());
+    std::shared_ptr<Array> out;
+    EXPECT_OK(builder_->Finish(&out));
+
+    result_ = std::dynamic_pointer_cast<StringArray>(out);
     result_->Validate();
   }
 
@@ -178,7 +175,7 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
 
   ASSERT_EQ(reps * N, result_->length());
   ASSERT_EQ(reps, result_->null_count());
-  ASSERT_EQ(reps * 6, result_->values()->length());
+  ASSERT_EQ(reps * 6, result_->data()->size());
 
   int32_t length;
   int32_t pos = 0;
@@ -218,18 +215,14 @@ class TestBinaryContainer : public ::testing::Test {
 
   void MakeArray() {
     length_ = offsets_.size() - 1;
-    int nchars = chars_.size();
-
     value_buf_ = test::to_buffer(chars_);
-    values_ = ArrayPtr(new UInt8Array(nchars, value_buf_));
-
     offsets_buf_ = test::to_buffer(offsets_);
 
     null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<BinaryArray>(
-        length_, offsets_buf_, values_, null_count_, null_bitmap_);
+        length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
   }
 
  protected:
@@ -246,7 +239,6 @@ class TestBinaryContainer : public ::testing::Test {
   int null_count_;
   int length_;
 
-  ArrayPtr values_;
   std::shared_ptr<BinaryArray> strings_;
 };
 
@@ -274,7 +266,7 @@ TEST_F(TestBinaryContainer, TestListFunctions) {
 
 TEST_F(TestBinaryContainer, TestDestructor) {
   auto arr = std::make_shared<BinaryArray>(
-      length_, offsets_buf_, values_, null_count_, null_bitmap_);
+      length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
 }
 
 TEST_F(TestBinaryContainer, TestGetValue) {
@@ -298,7 +290,10 @@ class TestBinaryBuilder : public TestBuilder {
   }
 
   void Done() {
-    result_ = std::dynamic_pointer_cast<BinaryArray>(builder_->Finish());
+    std::shared_ptr<Array> out;
+    EXPECT_OK(builder_->Finish(&out));
+
+    result_ = std::dynamic_pointer_cast<BinaryArray>(out);
     result_->Validate();
   }
 
@@ -330,7 +325,7 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
   ASSERT_OK(result_->Validate());
   ASSERT_EQ(reps * N, result_->length());
   ASSERT_EQ(reps, result_->null_count());
-  ASSERT_EQ(reps * 6, result_->values()->length());
+  ASSERT_EQ(reps * 6, result_->data()->size());
 
   int32_t length;
   for (int i = 0; i < N * reps; ++i) {
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index 745ed8f7edb99..d692e13773f56 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/types/string.h"
 
+#include <cstring>
 #include <sstream>
 #include <string>
 
@@ -24,37 +25,77 @@
 
 namespace arrow {
 
-const std::shared_ptr<DataType> BINARY(new BinaryType());
-const std::shared_ptr<DataType> STRING(new StringType());
+static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
+static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
 
 BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-    const ArrayPtr& values, int32_t null_count,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap)
-    : BinaryArray(BINARY, length, offsets, values, null_count, null_bitmap) {}
+    : BinaryArray(kBinary, length, offsets, data, null_count, null_bitmap) {}
 
 BinaryArray::BinaryArray(const TypePtr& type, int32_t length,
-    const std::shared_ptr<Buffer>& offsets, const ArrayPtr& values, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : ListArray(type, length, offsets, values, null_count, null_bitmap),
-      bytes_(std::dynamic_pointer_cast<UInt8Array>(values).get()),
-      raw_bytes_(bytes_->raw_data()) {
-  // Check in case the dynamic cast fails.
-  DCHECK(bytes_);
+    const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
+    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
+    : Array(type, length, null_count, null_bitmap),
+      offset_buffer_(offsets),
+      offsets_(reinterpret_cast<const int32_t*>(offset_buffer_->data())),
+      data_buffer_(data),
+      data_(nullptr) {
+  if (data_buffer_ != nullptr) { data_ = data_buffer_->data(); }
 }
 
 Status BinaryArray::Validate() const {
-  if (values()->null_count() > 0) {
-    std::stringstream ss;
-    ss << type()->ToString() << " can have null values in the value array";
-    Status::Invalid(ss.str());
+  // TODO(wesm): what to do here?
+  return Status::OK();
+}
+
+bool BinaryArray::EqualsExact(const BinaryArray& other) const {
+  if (!Array::EqualsExact(other)) { return false; }
+
+  bool equal_offsets =
+      offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
+  if (!equal_offsets) { return false; }
+
+  return data_buffer_->Equals(*other.data_buffer_, data_buffer_->size());
+}
+
+bool BinaryArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  return EqualsExact(*static_cast<const BinaryArray*>(arr.get()));
+}
+
+bool BinaryArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  const auto other = static_cast<const BinaryArray*>(arr.get());
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    const bool is_null = IsNull(i);
+    if (is_null != arr->IsNull(o_i)) { return false; }
+    if (is_null) continue;
+    const int32_t begin_offset = offset(i);
+    const int32_t end_offset = offset(i + 1);
+    const int32_t other_begin_offset = other->offset(o_i);
+    const int32_t other_end_offset = other->offset(o_i + 1);
+    // Underlying can't be equal if the size isn't equal
+    if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
+      return false;
+    }
+
+    if (std::memcmp(data_ + begin_offset, other->data_ + other_begin_offset,
+            end_offset - begin_offset)) {
+      return false;
+    }
   }
-  return ListArray::Validate();
+  return true;
 }
 
 StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-    const ArrayPtr& values, int32_t null_count,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap)
-    : StringArray(STRING, length, offsets, values, null_count, null_bitmap) {}
+    : BinaryArray(kString, length, offsets, data, null_count, null_bitmap) {}
 
 Status StringArray::Validate() const {
   // TODO(emkornfield) Validate proper UTF8 code points?
@@ -72,4 +113,28 @@ BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
   byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
 }
 
+Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Array> result;
+  RETURN_NOT_OK(ListBuilder::Finish(&result));
+
+  const auto list = std::dynamic_pointer_cast<ListArray>(result);
+  auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
+
+  *out = std::make_shared<BinaryArray>(list->length(), list->offset_buffer(),
+      values->data(), list->null_count(), list->null_bitmap());
+  return Status::OK();
+}
+
+Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Array> result;
+  RETURN_NOT_OK(ListBuilder::Finish(&result));
+
+  const auto list = std::dynamic_pointer_cast<ListArray>(result);
+  auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
+
+  *out = std::make_shared<StringArray>(list->length(), list->offset_buffer(),
+      values->data(), list->null_count(), list->null_bitmap());
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index bab0c58f617b2..aaba49c60237d 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -35,15 +35,16 @@ namespace arrow {
 class Buffer;
 class MemoryPool;
 
-class ARROW_EXPORT BinaryArray : public ListArray {
+class ARROW_EXPORT BinaryArray : public Array {
  public:
   BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const ArrayPtr& values, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
   // Constructor that allows sub-classes/builders to propagate there logical type up the
   // class hierarchy.
   BinaryArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const ArrayPtr& values, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   // Return the pointer to the given elements bytes
@@ -53,28 +54,38 @@ class ARROW_EXPORT BinaryArray : public ListArray {
     DCHECK(out_length);
     const int32_t pos = offsets_[i];
     *out_length = offsets_[i + 1] - pos;
-    return raw_bytes_ + pos;
+    return data_ + pos;
   }
 
+  std::shared_ptr<Buffer> data() const { return data_buffer_; }
+  std::shared_ptr<Buffer> offsets() const { return offset_buffer_; }
+
+  int32_t offset(int i) const { return offsets_[i]; }
+
+  // Neither of these functions will perform boundschecking
+  int32_t value_offset(int i) const { return offsets_[i]; }
+  int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
+
+  bool EqualsExact(const BinaryArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const ArrayPtr& arr) const override;
+
   Status Validate() const override;
 
  private:
-  UInt8Array* bytes_;
-  const uint8_t* raw_bytes_;
+  std::shared_ptr<Buffer> offset_buffer_;
+  const int32_t* offsets_;
+
+  std::shared_ptr<Buffer> data_buffer_;
+  const uint8_t* data_;
 };
 
 class ARROW_EXPORT StringArray : public BinaryArray {
  public:
   StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const ArrayPtr& values, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-  // Constructor that allows overriding the logical type, so subclasses can propagate
-  // there
-  // up the class hierarchy.
-  StringArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const ArrayPtr& values, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr)
-      : BinaryArray(type, length, offsets, values, null_count, null_bitmap) {}
 
   // Construct a std::string
   // TODO: std::bad_alloc possibility
@@ -98,9 +109,7 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
     return byte_builder_->Append(value, length);
   }
 
-  std::shared_ptr<Array> Finish() override {
-    return ListBuilder::Transfer<BinaryArray>();
-  }
+  Status Finish(std::shared_ptr<Array>* out) override;
 
  protected:
   UInt8Builder* byte_builder_;
@@ -112,6 +121,8 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
   explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
       : BinaryBuilder(pool, type) {}
 
+  Status Finish(std::shared_ptr<Array>* out) override;
+
   Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
 
   Status Append(const char* value, int32_t length) {
@@ -119,10 +130,6 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
   }
 
   Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
-
-  std::shared_ptr<Array> Finish() override {
-    return ListBuilder::Transfer<StringArray>();
-  }
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index ccf5a52dc831c..8e82c389a9423 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -119,7 +119,11 @@ class TestStructBuilder : public TestBuilder {
     ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
   }
 
-  void Done() { result_ = std::dynamic_pointer_cast<StructArray>(builder_->Finish()); }
+  void Done() {
+    std::shared_ptr<Array> out;
+    ASSERT_OK(builder_->Finish(&out));
+    result_ = std::dynamic_pointer_cast<StructArray>(out);
+  }
 
  protected:
   std::vector<FieldPtr> value_fields_;
@@ -294,7 +298,8 @@ TEST_F(TestStructBuilder, TestEquality) {
   for (int32_t value : int_values) {
     int_vb->UnsafeAppend(value);
   }
-  array = builder_->Finish();
+
+  ASSERT_OK(builder_->Finish(&array));
 
   ASSERT_OK(builder_->Resize(list_lengths.size()));
   ASSERT_OK(char_vb->Resize(list_values.size()));
@@ -308,7 +313,8 @@ TEST_F(TestStructBuilder, TestEquality) {
   for (int32_t value : int_values) {
     int_vb->UnsafeAppend(value);
   }
-  equal_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Finish(&equal_array));
 
   ASSERT_OK(builder_->Resize(list_lengths.size()));
   ASSERT_OK(char_vb->Resize(list_values.size()));
@@ -323,7 +329,8 @@ TEST_F(TestStructBuilder, TestEquality) {
   for (int32_t value : int_values) {
     int_vb->UnsafeAppend(value);
   }
-  unequal_bitmap_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Finish(&unequal_bitmap_array));
 
   ASSERT_OK(builder_->Resize(list_lengths.size()));
   ASSERT_OK(char_vb->Resize(list_values.size()));
@@ -339,7 +346,8 @@ TEST_F(TestStructBuilder, TestEquality) {
   for (int32_t value : int_values) {
     int_vb->UnsafeAppend(value);
   }
-  unequal_offsets_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Finish(&unequal_offsets_array));
 
   ASSERT_OK(builder_->Resize(list_lengths.size()));
   ASSERT_OK(char_vb->Resize(list_values.size()));
@@ -354,7 +362,8 @@ TEST_F(TestStructBuilder, TestEquality) {
   for (int32_t value : unequal_int_values) {
     int_vb->UnsafeAppend(value);
   }
-  unequal_values_array = builder_->Finish();
+
+  ASSERT_OK(builder_->Finish(&unequal_values_array));
 
   // Test array equality
   EXPECT_TRUE(array->Equals(array));
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
index e8176f08268b4..369c29d15ef96 100644
--- a/cpp/src/arrow/types/struct.cc
+++ b/cpp/src/arrow/types/struct.cc
@@ -87,4 +87,18 @@ Status StructArray::Validate() const {
   return Status::OK();
 }
 
+Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
+  for (size_t i = 0; i < field_builders_.size(); ++i) {
+    RETURN_NOT_OK(field_builders_[i]->Finish(&fields[i]));
+  }
+
+  *out = std::make_shared<StructArray>(type_, length_, fields, null_count_, null_bitmap_);
+
+  null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
+
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index 63955eb31bb7d..65b8daf214a69 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -73,6 +73,8 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
     field_builders_ = field_builders;
   }
 
+  Status Finish(std::shared_ptr<Array>* out) override;
+
   // Null bitmap is of equal length to every child field, and any zero byte
   // will be considered as a null for that field, but users must using app-
   // end methods or advance methods of the child builders' independently to
@@ -83,21 +85,6 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  std::shared_ptr<Array> Finish() override {
-    std::vector<ArrayPtr> fields;
-    for (auto it : field_builders_) {
-      fields.push_back(it->Finish());
-    }
-
-    auto result =
-        std::make_shared<StructArray>(type_, length_, fields, null_count_, null_bitmap_);
-
-    null_bitmap_ = nullptr;
-    capacity_ = length_ = null_count_ = 0;
-
-    return result;
-  }
-
   // Append an element to the Struct. All child-builders' Append method must
   // be called independently to maintain data-structure consistency.
   Status Append(bool is_valid = true) {
diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/util/status.cc
index 8dd07d0d064e7..08e9ae3946e51 100644
--- a/cpp/src/arrow/util/status.cc
+++ b/cpp/src/arrow/util/status.cc
@@ -49,12 +49,18 @@ std::string Status::CodeAsString() const {
     case StatusCode::KeyError:
       type = "Key error";
       break;
+    case StatusCode::TypeError:
+      type = "Type error";
+      break;
     case StatusCode::Invalid:
       type = "Invalid";
       break;
     case StatusCode::IOError:
       type = "IOError";
       break;
+    case StatusCode::UnknownError:
+      type = "Unknown error";
+      break;
     case StatusCode::NotImplemented:
       type = "NotImplemented";
       break;
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
index d5585313c728b..05f5b749b60cb 100644
--- a/cpp/src/arrow/util/status.h
+++ b/cpp/src/arrow/util/status.h
@@ -78,9 +78,10 @@ enum class StatusCode : char {
   OK = 0,
   OutOfMemory = 1,
   KeyError = 2,
-  Invalid = 3,
-  IOError = 4,
-
+  TypeError = 3,
+  Invalid = 4,
+  IOError = 5,
+  UnknownError = 9,
   NotImplemented = 10,
 };
 
@@ -106,6 +107,14 @@ class ARROW_EXPORT Status {
     return Status(StatusCode::KeyError, msg, -1);
   }
 
+  static Status TypeError(const std::string& msg) {
+    return Status(StatusCode::TypeError, msg, -1);
+  }
+
+  static Status UnknownError(const std::string& msg) {
+    return Status(StatusCode::UnknownError, msg, -1);
+  }
+
   static Status NotImplemented(const std::string& msg) {
     return Status(StatusCode::NotImplemented, msg, -1);
   }
@@ -125,6 +134,8 @@ class ARROW_EXPORT Status {
   bool IsKeyError() const { return code() == StatusCode::KeyError; }
   bool IsInvalid() const { return code() == StatusCode::Invalid; }
   bool IsIOError() const { return code() == StatusCode::IOError; }
+
+  bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
   bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
 
   // Return a string representation of this status suitable for printing.
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 55f6d0543a108..4357fa05ff864 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -438,8 +438,6 @@ set(PYARROW_SRCS
   src/pyarrow/config.cc
   src/pyarrow/helpers.cc
   src/pyarrow/io.cc
-  src/pyarrow/status.cc
-
   src/pyarrow/adapters/builtin.cc
   src/pyarrow/adapters/pandas.cc
 )
diff --git a/python/doc/index.rst b/python/doc/index.rst
index 550e544eef9e8..88725badc1e24 100644
--- a/python/doc/index.rst
+++ b/python/doc/index.rst
@@ -1,3 +1,20 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
 Apache Arrow (Python)
 =====================
 
@@ -25,4 +42,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-
diff --git a/python/pyarrow/error.pxd b/python/pyarrow/error.pxd
index 891d1ac1c7ea0..4fb46c25fafe4 100644
--- a/python/pyarrow/error.pxd
+++ b/python/pyarrow/error.pxd
@@ -16,7 +16,5 @@
 # under the License.
 
 from pyarrow.includes.libarrow cimport CStatus
-from pyarrow.includes.pyarrow cimport PyStatus
 
-cdef int check_cstatus(const CStatus& status) nogil except -1
-cdef int check_status(const PyStatus& status) nogil except -1
+cdef int check_status(const CStatus& status) nogil except -1
diff --git a/python/pyarrow/error.pyx b/python/pyarrow/error.pyx
index a2c53fed8c6a0..b8a82b3754c1b 100644
--- a/python/pyarrow/error.pyx
+++ b/python/pyarrow/error.pyx
@@ -22,15 +22,7 @@ from pyarrow.compat import frombytes
 class ArrowException(Exception):
     pass
 
-cdef int check_cstatus(const CStatus& status) nogil except -1:
-    if status.ok():
-        return 0
-
-    cdef c_string c_message = status.ToString()
-    with gil:
-        raise ArrowException(frombytes(c_message))
-
-cdef int check_status(const PyStatus& status) nogil except -1:
+cdef int check_status(const CStatus& status) nogil except -1:
     if status.ok():
         return 0
 
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 7c47f21854e33..e1da1914c5743 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -25,36 +25,19 @@ cimport pyarrow.includes.libarrow_io as arrow_io
 
 
 cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
-    # We can later add more of the common status factory methods as needed
-    cdef PyStatus PyStatus_OK "Status::OK"()
-
-    cdef cppclass PyStatus "pyarrow::Status":
-        PyStatus()
-
-        c_string ToString()
-
-        c_bool ok()
-        c_bool IsOutOfMemory()
-        c_bool IsKeyError()
-        c_bool IsTypeError()
-        c_bool IsIOError()
-        c_bool IsValueError()
-        c_bool IsNotImplemented()
-        c_bool IsArrowError()
-
     shared_ptr[CDataType] GetPrimitiveType(Type type)
-    PyStatus ConvertPySequence(object obj, shared_ptr[CArray]* out)
+    CStatus ConvertPySequence(object obj, shared_ptr[CArray]* out)
 
-    PyStatus PandasToArrow(MemoryPool* pool, object ao,
-                           shared_ptr[CArray]* out)
-    PyStatus PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
-                                 shared_ptr[CArray]* out)
+    CStatus PandasToArrow(MemoryPool* pool, object ao,
+                          shared_ptr[CArray]* out)
+    CStatus PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
+                                shared_ptr[CArray]* out)
 
-    PyStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
-                                  object py_ref, PyObject** out)
+    CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
+                                 object py_ref, PyObject** out)
 
-    PyStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
-                                   object py_ref, PyObject** out)
+    CStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
+                                  object py_ref, PyObject** out)
 
     MemoryPool* get_memory_pool()
 
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 8970e06effdd0..16ebfa1138e46 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -28,7 +28,7 @@ cimport pyarrow.includes.pyarrow as pyarrow
 from pyarrow.includes.libarrow_io cimport *
 
 from pyarrow.compat import frombytes, tobytes
-from pyarrow.error cimport check_cstatus
+from pyarrow.error cimport check_status
 
 cimport cpython as cp
 
@@ -57,9 +57,9 @@ cdef class NativeFile:
         if self.is_open:
             with nogil:
                 if self.is_readonly:
-                    check_cstatus(self.rd_file.get().Close())
+                    check_status(self.rd_file.get().Close())
                 else:
-                    check_cstatus(self.wr_file.get().Close())
+                    check_status(self.wr_file.get().Close())
         self.is_open = False
 
     cdef read_handle(self, shared_ptr[ReadableFileInterface]* file):
@@ -88,22 +88,22 @@ cdef class NativeFile:
         cdef int64_t size
         self._assert_readable()
         with nogil:
-            check_cstatus(self.rd_file.get().GetSize(&size))
+            check_status(self.rd_file.get().GetSize(&size))
         return size
 
     def tell(self):
         cdef int64_t position
         with nogil:
             if self.is_readonly:
-                check_cstatus(self.rd_file.get().Tell(&position))
+                check_status(self.rd_file.get().Tell(&position))
             else:
-                check_cstatus(self.wr_file.get().Tell(&position))
+                check_status(self.wr_file.get().Tell(&position))
         return position
 
     def seek(self, int64_t position):
         self._assert_readable()
         with nogil:
-            check_cstatus(self.rd_file.get().Seek(position))
+            check_status(self.rd_file.get().Seek(position))
 
     def write(self, data):
         """
@@ -116,7 +116,7 @@ cdef class NativeFile:
         cdef const uint8_t* buf = <const uint8_t*> cp.PyBytes_AS_STRING(data)
         cdef int64_t bufsize = len(data)
         with nogil:
-            check_cstatus(self.wr_file.get().Write(buf, bufsize))
+            check_status(self.wr_file.get().Write(buf, bufsize))
 
     def read(self, int nbytes):
         cdef:
@@ -127,8 +127,7 @@ cdef class NativeFile:
         self._assert_readable()
 
         with nogil:
-            check_cstatus(self.rd_file.get()
-                          .ReadB(nbytes, &out))
+            check_status(self.rd_file.get().ReadB(nbytes, &out))
 
         result = cp.PyBytes_FromStringAndSize(
             <const char*>out.get().data(), out.get().size())
@@ -223,7 +222,7 @@ cdef class InMemoryOutputStream(NativeFile):
     def get_result(self):
         cdef Buffer result = Buffer()
 
-        check_cstatus(self.wr_file.get().Close())
+        check_status(self.wr_file.get().Close())
         result.init(<shared_ptr[CBuffer]> self.buffer)
 
         self.is_open = False
@@ -270,7 +269,7 @@ except ImportError:
 
 def have_libhdfs():
     try:
-        check_cstatus(ConnectLibHdfs())
+        check_status(ConnectLibHdfs())
         return True
     except:
         return False
@@ -304,7 +303,7 @@ cdef class HdfsClient:
     def close(self):
         self._ensure_client()
         with nogil:
-            check_cstatus(self.client.get().Disconnect())
+            check_status(self.client.get().Disconnect())
         self.is_open = False
 
     cdef _ensure_client(self):
@@ -341,8 +340,7 @@ cdef class HdfsClient:
         conf.user = tobytes(user)
 
         with nogil:
-            check_cstatus(
-                CHdfsClient.Connect(&conf, &out.client))
+            check_status(CHdfsClient.Connect(&conf, &out.client))
         out.is_open = True
 
         return out
@@ -383,8 +381,8 @@ cdef class HdfsClient:
         self._ensure_client()
 
         with nogil:
-            check_cstatus(self.client.get()
-                          .ListDirectory(c_path, &listing))
+            check_status(self.client.get()
+                         .ListDirectory(c_path, &listing))
 
         cdef const HdfsPathInfo* info
         for i in range(<int> listing.size()):
@@ -422,8 +420,8 @@ cdef class HdfsClient:
 
         cdef c_string c_path = tobytes(path)
         with nogil:
-            check_cstatus(self.client.get()
-                          .CreateDirectory(c_path))
+            check_status(self.client.get()
+                         .CreateDirectory(c_path))
 
     def delete(self, path, bint recursive=False):
         """
@@ -439,8 +437,8 @@ cdef class HdfsClient:
 
         cdef c_string c_path = tobytes(path)
         with nogil:
-            check_cstatus(self.client.get()
-                          .Delete(c_path, recursive))
+            check_status(self.client.get()
+                         .Delete(c_path, recursive))
 
     def open(self, path, mode='rb', buffer_size=None, replication=None,
              default_block_size=None):
@@ -473,7 +471,7 @@ cdef class HdfsClient:
                 append = True
 
             with nogil:
-                check_cstatus(
+                check_status(
                     self.client.get()
                     .OpenWriteable(c_path, append, c_buffer_size,
                                    c_replication, c_default_block_size,
@@ -484,8 +482,8 @@ cdef class HdfsClient:
             out.is_readonly = False
         else:
             with nogil:
-                check_cstatus(self.client.get()
-                              .OpenReadable(c_path, &rd_handle))
+                check_status(self.client.get()
+                             .OpenReadable(c_path, &rd_handle))
 
             out.rd_file = <shared_ptr[ReadableFileInterface]> rd_handle
             out.is_readonly = True
@@ -579,9 +577,9 @@ cdef class HdfsFile(NativeFile):
         try:
             with nogil:
                 while total_bytes < nbytes:
-                    check_cstatus(self.rd_file.get()
-                                  .Read(rpc_chunksize, &bytes_read,
-                                        buf + total_bytes))
+                    check_status(self.rd_file.get()
+                                 .Read(rpc_chunksize, &bytes_read,
+                                       buf + total_bytes))
 
                     total_bytes += bytes_read
 
@@ -647,8 +645,8 @@ cdef class HdfsFile(NativeFile):
         try:
             while True:
                 with nogil:
-                    check_cstatus(self.rd_file.get()
-                                  .Read(self.buffer_size, &bytes_read, buf))
+                    check_status(self.rd_file.get()
+                                 .Read(self.buffer_size, &bytes_read, buf))
 
                 total_bytes += bytes_read
 
diff --git a/python/pyarrow/ipc.pyx b/python/pyarrow/ipc.pyx
index f8da3a70da819..46deb5ad0c35d 100644
--- a/python/pyarrow/ipc.pyx
+++ b/python/pyarrow/ipc.pyx
@@ -26,7 +26,7 @@ from pyarrow.includes.libarrow_io cimport *
 from pyarrow.includes.libarrow_ipc cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 
-from pyarrow.error cimport check_cstatus
+from pyarrow.error cimport check_status
 from pyarrow.io cimport NativeFile
 from pyarrow.schema cimport Schema
 from pyarrow.table cimport RecordBatch
@@ -89,8 +89,8 @@ cdef class ArrowFileWriter:
         get_writer(sink, &self.sink)
 
         with nogil:
-            check_cstatus(CFileWriter.Open(self.sink.get(), schema.sp_schema,
-                                           &self.writer))
+            check_status(CFileWriter.Open(self.sink.get(), schema.sp_schema,
+                                          &self.writer))
 
         self.closed = False
 
@@ -101,12 +101,12 @@ cdef class ArrowFileWriter:
     def write_record_batch(self, RecordBatch batch):
         cdef CRecordBatch* bptr = batch.batch
         with nogil:
-            check_cstatus(self.writer.get()
-                          .WriteRecordBatch(bptr.columns(), bptr.num_rows()))
+            check_status(self.writer.get()
+                         .WriteRecordBatch(bptr.columns(), bptr.num_rows()))
 
     def close(self):
         with nogil:
-            check_cstatus(self.writer.get().Close())
+            check_status(self.writer.get().Close())
         self.closed = True
 
 
@@ -124,9 +124,9 @@ cdef class ArrowFileReader:
 
         with nogil:
             if offset != 0:
-                check_cstatus(CFileReader.Open2(reader, offset, &self.reader))
+                check_status(CFileReader.Open2(reader, offset, &self.reader))
             else:
-                check_cstatus(CFileReader.Open(reader, &self.reader))
+                check_status(CFileReader.Open(reader, &self.reader))
 
     property num_dictionaries:
 
@@ -147,7 +147,7 @@ cdef class ArrowFileReader:
             raise ValueError('Batch number {0} out of range'.format(i))
 
         with nogil:
-            check_cstatus(self.reader.get().GetRecordBatch(i, &batch))
+            check_status(self.reader.get().GetRecordBatch(i, &batch))
 
         result = RecordBatch()
         result.init(batch)
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 2abe57b33ed48..019dd2c1de489 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -26,7 +26,7 @@ cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.compat import tobytes
 from pyarrow.error import ArrowException
-from pyarrow.error cimport check_cstatus
+from pyarrow.error cimport check_status
 from pyarrow.io import NativeFile
 from pyarrow.table cimport Table
 
@@ -62,7 +62,7 @@ cdef class ParquetReader:
         cdef shared_ptr[ReadableFileInterface] cpp_handle
         file.read_handle(&cpp_handle)
 
-        check_cstatus(OpenFile(cpp_handle, &self.allocator, &self.reader))
+        check_status(OpenFile(cpp_handle, &self.allocator, &self.reader))
 
     def read_all(self):
         cdef:
@@ -70,8 +70,8 @@ cdef class ParquetReader:
             shared_ptr[CTable] ctable
 
         with nogil:
-            check_cstatus(self.reader.get()
-                          .ReadFlatTable(&ctable))
+            check_status(self.reader.get()
+                         .ReadFlatTable(&ctable))
 
         table.init(ctable)
         return table
@@ -80,7 +80,7 @@ cdef class ParquetReader:
 def read_table(source, columns=None):
     """
     Read a Table from Parquet format
-    
+
     Returns
     -------
     pyarrow.table.Table
@@ -176,5 +176,5 @@ def write_table(table, filename, chunk_size=None, version=None,
 
     sink.reset(new LocalFileOutputStream(tobytes(filename)))
     with nogil:
-        check_cstatus(WriteFlatTable(ctable_, default_memory_pool(), sink,
-            chunk_size_, properties_builder.build()))
+        check_status(WriteFlatTable(ctable_, default_memory_pool(), sink,
+                                    chunk_size_, properties_builder.build()))
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index 680f3a539b5fa..c034fbd977747 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -20,13 +20,14 @@
 
 #include "pyarrow/adapters/builtin.h"
 
-#include <arrow/api.h>
+#include "arrow/api.h"
+#include "arrow/util/status.h"
 
 #include "pyarrow/helpers.h"
-#include "pyarrow/status.h"
 
 using arrow::ArrayBuilder;
 using arrow::DataType;
+using arrow::Status;
 using arrow::Type;
 
 namespace pyarrow {
@@ -129,7 +130,7 @@ class SeqVisitor {
       PyObject* item = item_ref.obj();
 
       if (PyList_Check(item)) {
-        PY_RETURN_NOT_OK(Visit(item, level + 1));
+        RETURN_NOT_OK(Visit(item, level + 1));
       } else if (PyDict_Check(item)) {
         return Status::NotImplemented("No type inference for dicts");
       } else {
@@ -164,9 +165,9 @@ class SeqVisitor {
   Status Validate() const {
     if (scalars_.total_count() > 0) {
       if (num_nesting_levels() > 1) {
-        return Status::ValueError("Mixed nesting levels not supported");
+        return Status::Invalid("Mixed nesting levels not supported");
       } else if (max_observed_level() < max_nesting_level_) {
-        return Status::ValueError("Mixed nesting levels not supported");
+        return Status::Invalid("Mixed nesting levels not supported");
       }
     }
     return Status::OK();
@@ -216,8 +217,8 @@ static Status InferArrowType(PyObject* obj, int64_t* size,
   }
 
   SeqVisitor seq_visitor;
-  PY_RETURN_NOT_OK(seq_visitor.Visit(obj));
-  PY_RETURN_NOT_OK(seq_visitor.Validate());
+  RETURN_NOT_OK(seq_visitor.Visit(obj));
+  RETURN_NOT_OK(seq_visitor.Validate());
 
   *out_type = seq_visitor.GetType();
 
@@ -259,7 +260,7 @@ class BoolConverter : public TypedConverter<arrow::BooleanBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     Py_ssize_t size = PySequence_Size(seq);
-    RETURN_ARROW_NOT_OK(typed_builder_->Reserve(size));
+    RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
       if (item.obj() == Py_None) {
@@ -281,7 +282,7 @@ class Int64Converter : public TypedConverter<arrow::Int64Builder> {
   Status AppendData(PyObject* seq) override {
     int64_t val;
     Py_ssize_t size = PySequence_Size(seq);
-    RETURN_ARROW_NOT_OK(typed_builder_->Reserve(size));
+    RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
       if (item.obj() == Py_None) {
@@ -301,7 +302,7 @@ class DoubleConverter : public TypedConverter<arrow::DoubleBuilder> {
   Status AppendData(PyObject* seq) override {
     double val;
     Py_ssize_t size = PySequence_Size(seq);
-    RETURN_ARROW_NOT_OK(typed_builder_->Reserve(size));
+    RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
       if (item.obj() == Py_None) {
@@ -330,7 +331,7 @@ class StringConverter : public TypedConverter<arrow::StringBuilder> {
       OwnedRef holder(item);
 
       if (item == Py_None) {
-        RETURN_ARROW_NOT_OK(typed_builder_->AppendNull());
+        RETURN_NOT_OK(typed_builder_->AppendNull());
         continue;
       } else if (PyUnicode_Check(item)) {
         tmp.reset(PyUnicode_AsUTF8String(item));
@@ -344,7 +345,7 @@ class StringConverter : public TypedConverter<arrow::StringBuilder> {
       // No error checking
       length = PyBytes_GET_SIZE(bytes_obj);
       bytes = PyBytes_AS_STRING(bytes_obj);
-      RETURN_ARROW_NOT_OK(typed_builder_->Append(bytes, length));
+      RETURN_NOT_OK(typed_builder_->Append(bytes, length));
     }
     return Status::OK();
   }
@@ -359,10 +360,10 @@ class ListConverter : public TypedConverter<arrow::ListBuilder> {
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
       if (item.obj() == Py_None) {
-        RETURN_ARROW_NOT_OK(typed_builder_->AppendNull());
+        RETURN_NOT_OK(typed_builder_->AppendNull());
       } else {
         typed_builder_->Append();
-        PY_RETURN_NOT_OK(value_converter_->AppendData(item.obj()));
+        RETURN_NOT_OK(value_converter_->AppendData(item.obj()));
       }
     }
     return Status::OK();
@@ -408,7 +409,7 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
 Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
   std::shared_ptr<DataType> type;
   int64_t size;
-  PY_RETURN_NOT_OK(InferArrowType(obj, &size, &type));
+  RETURN_NOT_OK(InferArrowType(obj, &size, &type));
 
   // Handle NA / NullType case
   if (type->type == Type::NA) {
@@ -426,14 +427,12 @@ Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
 
   // Give the sequence converter an array builder
   std::shared_ptr<ArrayBuilder> builder;
-  RETURN_ARROW_NOT_OK(arrow::MakeBuilder(get_memory_pool(), type, &builder));
+  RETURN_NOT_OK(arrow::MakeBuilder(get_memory_pool(), type, &builder));
   converter->Init(builder);
 
-  PY_RETURN_NOT_OK(converter->AppendData(obj));
+  RETURN_NOT_OK(converter->AppendData(obj));
 
-  *out = builder->Finish();
-
-  return Status::OK();
+  return builder->Finish(out);
 }
 
 } // namespace pyarrow
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
index 4e997e31dd690..2ddfdaaf44134 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -30,14 +30,15 @@
 #include "pyarrow/common.h"
 #include "pyarrow/visibility.h"
 
-namespace arrow { class Array; }
+namespace arrow {
+class Array;
+class Status;
+}
 
 namespace pyarrow {
 
-class Status;
-
 PYARROW_EXPORT
-Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out);
+arrow::Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out);
 
 } // namespace pyarrow
 
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index b2fcd37aec944..5902b8341696d 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -31,10 +31,10 @@
 
 #include "arrow/api.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/status.h"
 
 #include "pyarrow/common.h"
 #include "pyarrow/config.h"
-#include "pyarrow/status.h"
 
 namespace pyarrow {
 
@@ -42,6 +42,8 @@ using arrow::Array;
 using arrow::Column;
 using arrow::Field;
 using arrow::DataType;
+using arrow::Status;
+
 namespace util = arrow::util;
 
 // ----------------------------------------------------------------------
@@ -149,7 +151,7 @@ class ArrowSerializer {
     int null_bytes = util::bytes_for_bits(length_);
 
     null_bitmap_ = std::make_shared<arrow::PoolBuffer>(pool_);
-    RETURN_ARROW_NOT_OK(null_bitmap_->Resize(null_bytes));
+    RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
 
     null_bitmap_data_ = null_bitmap_->mutable_data();
     memset(null_bitmap_data_, 0, null_bytes);
@@ -171,9 +173,9 @@ class ArrowSerializer {
     PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     arrow::TypePtr string_type(new arrow::StringType());
     arrow::StringBuilder string_builder(pool_, string_type);
-    RETURN_ARROW_NOT_OK(string_builder.Resize(length_));
+    RETURN_NOT_OK(string_builder.Resize(length_));
 
-    arrow::Status s;
+    Status s;
     PyObject* obj;
     for (int64_t i = 0; i < length_; ++i) {
       obj = objects[i];
@@ -187,18 +189,16 @@ class ArrowSerializer {
         s = string_builder.Append(PyBytes_AS_STRING(obj), length);
         Py_DECREF(obj);
         if (!s.ok()) {
-          return Status::ArrowError(s.ToString());
+          return s;
         }
       } else if (PyBytes_Check(obj)) {
         const int32_t length = PyBytes_GET_SIZE(obj);
-        RETURN_ARROW_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
+        RETURN_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
       } else {
         string_builder.AppendNull();
       }
     }
-    *out = std::shared_ptr<arrow::Array>(string_builder.Finish());
-
-    return Status::OK();
+    return string_builder.Finish(out);
   }
 
   Status ConvertBooleans(std::shared_ptr<Array>* out) {
@@ -208,7 +208,7 @@ class ArrowSerializer {
 
     int nbytes = util::bytes_for_bits(length_);
     auto data = std::make_shared<arrow::PoolBuffer>(pool_);
-    RETURN_ARROW_NOT_OK(data->Resize(nbytes));
+    RETURN_NOT_OK(data->Resize(nbytes));
     uint8_t* bitmap = data->mutable_data();
     memset(bitmap, 0, nbytes);
 
@@ -305,7 +305,7 @@ inline Status ArrowSerializer<NPY_DATETIME>::MakeDataType(std::shared_ptr<DataTy
           unit = arrow::TimestampType::Unit::NANO;
           break;
       default:
-          return Status::ValueError("Unknown NumPy datetime unit");
+          return Status::Invalid("Unknown NumPy datetime unit");
   }
 
   out->reset(new arrow::TimestampType(unit));
@@ -330,7 +330,7 @@ inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
   RETURN_NOT_OK(ConvertData());
   std::shared_ptr<DataType> type;
   RETURN_NOT_OK(MakeDataType(&type));
-  RETURN_ARROW_NOT_OK(MakePrimitiveArray(type, length_, data_, null_count, null_bitmap_, out));
+  RETURN_NOT_OK(MakePrimitiveArray(type, length_, data_, null_count, null_bitmap_, out));
   return Status::OK();
 }
 
@@ -389,7 +389,7 @@ template <int TYPE>
 inline Status ArrowSerializer<TYPE>::ConvertData() {
   // TODO(wesm): strided arrays
   if (is_strided()) {
-    return Status::ValueError("no support for strided data yet");
+    return Status::Invalid("no support for strided data yet");
   }
 
   data_ = std::make_shared<NumPyBuffer>(arr_);
@@ -399,12 +399,12 @@ inline Status ArrowSerializer<TYPE>::ConvertData() {
 template <>
 inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
   if (is_strided()) {
-    return Status::ValueError("no support for strided data yet");
+    return Status::Invalid("no support for strided data yet");
   }
 
   int nbytes = util::bytes_for_bits(length_);
   auto buffer = std::make_shared<arrow::PoolBuffer>(pool_);
-  RETURN_ARROW_NOT_OK(buffer->Resize(nbytes));
+  RETURN_NOT_OK(buffer->Resize(nbytes));
 
   const uint8_t* values = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
@@ -446,7 +446,7 @@ Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
   }
 
   if (PyArray_NDIM(arr) != 1) {
-    return Status::ValueError("only handle 1-dimensional arrays");
+    return Status::Invalid("only handle 1-dimensional arrays");
   }
 
   switch(PyArray_DESCR(arr)->type_num) {
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index 141d1219e64db..532495dd792db 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -32,27 +32,26 @@ namespace arrow {
 class Array;
 class Column;
 class MemoryPool;
+class Status;
 
 } // namespace arrow
 
 namespace pyarrow {
 
-class Status;
-
 PYARROW_EXPORT
-Status ConvertArrayToPandas(const std::shared_ptr<arrow::Array>& arr, PyObject* py_ref,
-    PyObject** out);
+arrow::Status ConvertArrayToPandas(const std::shared_ptr<arrow::Array>& arr,
+    PyObject* py_ref, PyObject** out);
 
 PYARROW_EXPORT
-Status ConvertColumnToPandas(const std::shared_ptr<arrow::Column>& col, PyObject* py_ref,
-    PyObject** out);
+arrow::Status ConvertColumnToPandas(const std::shared_ptr<arrow::Column>& col,
+    PyObject* py_ref, PyObject** out);
 
 PYARROW_EXPORT
-Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
+arrow::Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     std::shared_ptr<arrow::Array>* out);
 
 PYARROW_EXPORT
-Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
+arrow::Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
     std::shared_ptr<arrow::Array>* out);
 
 } // namespace pyarrow
diff --git a/python/src/pyarrow/api.h b/python/src/pyarrow/api.h
index 72be6afe02c76..6dbbc45d40ccc 100644
--- a/python/src/pyarrow/api.h
+++ b/python/src/pyarrow/api.h
@@ -18,8 +18,6 @@
 #ifndef PYARROW_API_H
 #define PYARROW_API_H
 
-#include "pyarrow/status.h"
-
 #include "pyarrow/helpers.h"
 
 #include "pyarrow/adapters/builtin.h"
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index 09f3efb5a03bc..fa875f2b9aba1 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -21,10 +21,10 @@
 #include <mutex>
 #include <sstream>
 
-#include <arrow/util/memory-pool.h>
-#include <arrow/util/status.h>
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
 
-#include "pyarrow/status.h"
+using arrow::Status;
 
 namespace pyarrow {
 
@@ -33,18 +33,18 @@ class PyArrowMemoryPool : public arrow::MemoryPool {
   PyArrowMemoryPool() : bytes_allocated_(0) {}
   virtual ~PyArrowMemoryPool() {}
 
-  arrow::Status Allocate(int64_t size, uint8_t** out) override {
+  Status Allocate(int64_t size, uint8_t** out) override {
     std::lock_guard<std::mutex> guard(pool_lock_);
     *out = static_cast<uint8_t*>(std::malloc(size));
     if (*out == nullptr) {
       std::stringstream ss;
       ss << "malloc of size " << size << " failed";
-      return arrow::Status::OutOfMemory(ss.str());
+      return Status::OutOfMemory(ss.str());
     }
 
     bytes_allocated_ += size;
 
-    return arrow::Status::OK();
+    return Status::OK();
   }
 
   int64_t bytes_allocated() const override {
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 50c2577b93c9b..7f3131ef03dd8 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -29,13 +29,6 @@ namespace pyarrow {
 
 #define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
 
-#define RETURN_ARROW_NOT_OK(s) do {             \
-    arrow::Status _s = (s);                     \
-    if (!_s.ok()) {                             \
-      return Status::ArrowError(s.ToString());  \
-    }                                           \
-  } while (0);
-
 class OwnedRef {
  public:
   OwnedRef() : obj_(nullptr) {}
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index 7bf32ffa8d22b..e6dbc12d429b0 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -20,12 +20,13 @@
 #include <cstdint>
 #include <cstdlib>
 
-#include <arrow/io/memory.h>
-#include <arrow/util/memory-pool.h>
-#include <arrow/util/status.h>
+#include "arrow/io/memory.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
 
 #include "pyarrow/common.h"
-#include "pyarrow/status.h"
+
+using arrow::Status;
 
 namespace pyarrow {
 
@@ -41,7 +42,7 @@ PythonFile::~PythonFile() {
   Py_DECREF(file_);
 }
 
-static arrow::Status CheckPyError() {
+static Status CheckPyError() {
   if (PyErr_Occurred()) {
     PyObject *exc_type, *exc_value, *traceback;
     PyErr_Fetch(&exc_type, &exc_value, &traceback);
@@ -51,35 +52,35 @@ static arrow::Status CheckPyError() {
     Py_XDECREF(exc_value);
     Py_XDECREF(traceback);
     PyErr_Clear();
-    return arrow::Status::IOError(message);
+    return Status::IOError(message);
   }
-  return arrow::Status::OK();
+  return Status::OK();
 }
 
-arrow::Status PythonFile::Close() {
+Status PythonFile::Close() {
   // whence: 0 for relative to start of file, 2 for end of file
   PyObject* result = PyObject_CallMethod(file_, "close", "()");
   Py_XDECREF(result);
   ARROW_RETURN_NOT_OK(CheckPyError());
-  return arrow::Status::OK();
+  return Status::OK();
 }
 
-arrow::Status PythonFile::Seek(int64_t position, int whence) {
+Status PythonFile::Seek(int64_t position, int whence) {
   // whence: 0 for relative to start of file, 2 for end of file
   PyObject* result = PyObject_CallMethod(file_, "seek", "(ii)", position, whence);
   Py_XDECREF(result);
   ARROW_RETURN_NOT_OK(CheckPyError());
-  return arrow::Status::OK();
+  return Status::OK();
 }
 
-arrow::Status PythonFile::Read(int64_t nbytes, PyObject** out) {
+Status PythonFile::Read(int64_t nbytes, PyObject** out) {
   PyObject* result = PyObject_CallMethod(file_, "read", "(i)", nbytes);
   ARROW_RETURN_NOT_OK(CheckPyError());
   *out = result;
-  return arrow::Status::OK();
+  return Status::OK();
 }
 
-arrow::Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
+Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
   PyObject* py_data = PyBytes_FromStringAndSize(
       reinterpret_cast<const char*>(data), nbytes);
   ARROW_RETURN_NOT_OK(CheckPyError());
@@ -88,10 +89,10 @@ arrow::Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
   Py_XDECREF(py_data);
   Py_XDECREF(result);
   ARROW_RETURN_NOT_OK(CheckPyError());
-  return arrow::Status::OK();
+  return Status::OK();
 }
 
-arrow::Status PythonFile::Tell(int64_t* position) {
+Status PythonFile::Tell(int64_t* position) {
   PyObject* result = PyObject_CallMethod(file_, "tell", "()");
   ARROW_RETURN_NOT_OK(CheckPyError());
 
@@ -101,7 +102,7 @@ arrow::Status PythonFile::Tell(int64_t* position) {
   // PyLong_AsLongLong can raise OverflowError
   ARROW_RETURN_NOT_OK(CheckPyError());
 
- return arrow::Status::OK();
+ return Status::OK();
 }
 
 // ----------------------------------------------------------------------
@@ -113,22 +114,22 @@ PyReadableFile::PyReadableFile(PyObject* file) {
 
 PyReadableFile::~PyReadableFile() {}
 
-arrow::Status PyReadableFile::Close() {
+Status PyReadableFile::Close() {
   PyGILGuard lock;
   return file_->Close();
 }
 
-arrow::Status PyReadableFile::Seek(int64_t position) {
+Status PyReadableFile::Seek(int64_t position) {
   PyGILGuard lock;
   return file_->Seek(position, 0);
 }
 
-arrow::Status PyReadableFile::Tell(int64_t* position) {
+Status PyReadableFile::Tell(int64_t* position) {
   PyGILGuard lock;
   return file_->Tell(position);
 }
 
-arrow::Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
   PyGILGuard lock;
   PyObject* bytes_obj;
   ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
@@ -137,10 +138,10 @@ arrow::Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t*
   std::memcpy(out, PyBytes_AS_STRING(bytes_obj), *bytes_read);
   Py_DECREF(bytes_obj);
 
-  return arrow::Status::OK();
+  return Status::OK();
 }
 
-arrow::Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) {
+Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) {
   PyGILGuard lock;
 
   PyObject* bytes_obj;
@@ -149,10 +150,10 @@ arrow::Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer
   *out = std::make_shared<PyBytesBuffer>(bytes_obj);
   Py_DECREF(bytes_obj);
 
-  return arrow::Status::OK();
+  return Status::OK();
 }
 
-arrow::Status PyReadableFile::GetSize(int64_t* size) {
+Status PyReadableFile::GetSize(int64_t* size) {
   PyGILGuard lock;
 
   int64_t current_position;;
@@ -167,7 +168,7 @@ arrow::Status PyReadableFile::GetSize(int64_t* size) {
   ARROW_RETURN_NOT_OK(file_->Seek(current_position, 0));
 
   *size = file_size;
-  return arrow::Status::OK();
+  return Status::OK();
 }
 
 bool PyReadableFile::supports_zero_copy() const {
@@ -183,17 +184,17 @@ PyOutputStream::PyOutputStream(PyObject* file) {
 
 PyOutputStream::~PyOutputStream() {}
 
-arrow::Status PyOutputStream::Close() {
+Status PyOutputStream::Close() {
   PyGILGuard lock;
   return file_->Close();
 }
 
-arrow::Status PyOutputStream::Tell(int64_t* position) {
+Status PyOutputStream::Tell(int64_t* position) {
   PyGILGuard lock;
   return file_->Tell(position);
 }
 
-arrow::Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
   PyGILGuard lock;
   return file_->Write(data, nbytes);
 }
diff --git a/python/src/pyarrow/status.cc b/python/src/pyarrow/status.cc
deleted file mode 100644
index 1cd54f6a78560..0000000000000
--- a/python/src/pyarrow/status.cc
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Status encapsulates the result of an operation.  It may indicate success,
-// or it may indicate an error with an associated error message.
-//
-// Multiple threads can invoke const methods on a Status without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Status must use
-// external synchronization.
-
-#include "pyarrow/status.h"
-
-#include <assert.h>
-#include <cstdint>
-#include <cstring>
-
-namespace pyarrow {
-
-Status::Status(StatusCode code, const std::string& msg, int16_t posix_code) {
-  assert(code != StatusCode::OK);
-  const uint32_t size = msg.size();
-  char* result = new char[size + 7];
-  memcpy(result, &size, sizeof(size));
-  result[4] = static_cast<char>(code);
-  memcpy(result + 5, &posix_code, sizeof(posix_code));
-  memcpy(result + 7, msg.c_str(), msg.size());
-  state_ = result;
-}
-
-const char* Status::CopyState(const char* state) {
-  uint32_t size;
-  memcpy(&size, state, sizeof(size));
-  char* result = new char[size + 7];
-  memcpy(result, state, size + 7);
-  return result;
-}
-
-std::string Status::CodeAsString() const {
-  if (state_ == NULL) {
-    return "OK";
-  }
-
-  const char* type;
-  switch (code()) {
-    case StatusCode::OK:
-      type = "OK";
-      break;
-    case StatusCode::OutOfMemory:
-      type = "Out of memory";
-      break;
-    case StatusCode::KeyError:
-      type = "Key error";
-      break;
-    case StatusCode::TypeError:
-      type = "Value error";
-      break;
-    case StatusCode::ValueError:
-      type = "Value error";
-      break;
-    case StatusCode::IOError:
-      type = "IO error";
-      break;
-    case StatusCode::NotImplemented:
-      type = "Not implemented";
-      break;
-    case StatusCode::ArrowError:
-      type = "Arrow C++ error";
-      break;
-    case StatusCode::UnknownError:
-      type = "Unknown error";
-      break;
-  }
-  return std::string(type);
-}
-
-std::string Status::ToString() const {
-  std::string result(CodeAsString());
-  if (state_ == NULL) {
-    return result;
-  }
-
-  result.append(": ");
-
-  uint32_t length;
-  memcpy(&length, state_, sizeof(length));
-  result.append(reinterpret_cast<const char*>(state_ + 7), length);
-  return result;
-}
-
-} // namespace pyarrow
diff --git a/python/src/pyarrow/status.h b/python/src/pyarrow/status.h
deleted file mode 100644
index 67cd66c58eeb3..0000000000000
--- a/python/src/pyarrow/status.h
+++ /dev/null
@@ -1,146 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Status encapsulates the result of an operation.  It may indicate success,
-// or it may indicate an error with an associated error message.
-//
-// Multiple threads can invoke const methods on a Status without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Status must use
-// external synchronization.
-
-#ifndef PYARROW_STATUS_H_
-#define PYARROW_STATUS_H_
-
-#include <cstdint>
-#include <cstring>
-#include <string>
-
-#include "pyarrow/visibility.h"
-
-namespace pyarrow {
-
-#define PY_RETURN_NOT_OK(s) do {                \
-    Status _s = (s);                            \
-    if (!_s.ok()) return _s;                    \
-  } while (0);
-
-enum class StatusCode: char {
-  OK = 0,
-  OutOfMemory = 1,
-  KeyError = 2,
-  TypeError = 3,
-  ValueError = 4,
-  IOError = 5,
-  NotImplemented = 6,
-
-  ArrowError = 7,
-
-  UnknownError = 10
-};
-
-class PYARROW_EXPORT Status {
- public:
-  // Create a success status.
-  Status() : state_(NULL) { }
-  ~Status() { delete[] state_; }
-
-  // Copy the specified status.
-  Status(const Status& s);
-  void operator=(const Status& s);
-
-  // Return a success status.
-  static Status OK() { return Status(); }
-
-  // Return error status of an appropriate type.
-  static Status OutOfMemory(const std::string& msg, int16_t posix_code = -1) {
-    return Status(StatusCode::OutOfMemory, msg, posix_code);
-  }
-
-  static Status KeyError(const std::string& msg) {
-    return Status(StatusCode::KeyError, msg, -1);
-  }
-
-  static Status TypeError(const std::string& msg) {
-    return Status(StatusCode::TypeError, msg, -1);
-  }
-
-  static Status IOError(const std::string& msg) {
-    return Status(StatusCode::IOError, msg, -1);
-  }
-
-  static Status ValueError(const std::string& msg) {
-    return Status(StatusCode::ValueError, msg, -1);
-  }
-
-  static Status NotImplemented(const std::string& msg) {
-    return Status(StatusCode::NotImplemented, msg, -1);
-  }
-
-  static Status UnknownError(const std::string& msg) {
-    return Status(StatusCode::UnknownError, msg, -1);
-  }
-
-  static Status ArrowError(const std::string& msg) {
-    return Status(StatusCode::ArrowError, msg, -1);
-  }
-
-  // Returns true iff the status indicates success.
-  bool ok() const { return (state_ == NULL); }
-
-  bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; }
-  bool IsKeyError() const { return code() == StatusCode::KeyError; }
-  bool IsIOError() const { return code() == StatusCode::IOError; }
-  bool IsTypeError() const { return code() == StatusCode::TypeError; }
-  bool IsValueError() const { return code() == StatusCode::ValueError; }
-
-  bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
-
-  bool IsArrowError() const { return code() == StatusCode::ArrowError; }
-
-  // Return a string representation of this status suitable for printing.
-  // Returns the string "OK" for success.
-  std::string ToString() const;
-
-  // Return a string representation of the status code, without the message
-  // text or posix code information.
-  std::string CodeAsString() const;
-
-  // Get the POSIX code associated with this Status, or -1 if there is none.
-  int16_t posix_code() const;
-
- private:
-  // OK status has a NULL state_.  Otherwise, state_ is a new[] array
-  // of the following form:
-  //    state_[0..3] == length of message
-  //    state_[4]    == code
-  //    state_[5..6] == posix_code
-  //    state_[7..]  == message
-  const char* state_;
-
-  StatusCode code() const {
-    return ((state_ == NULL) ?
-        StatusCode::OK : static_cast<StatusCode>(state_[4]));
-  }
-
-  Status(StatusCode code, const std::string& msg, int16_t posix_code);
-  static const char* CopyState(const char* s);
-};
-
-inline Status::Status(const Status& s) {
-  state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-}
-
-inline void Status::operator=(const Status& s) {
-  // The following condition catches both aliasing (when this == &s),
-  // and the common case where both s and *this are ok.
-  if (state_ != s.state_) {
-    delete[] state_;
-    state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-  }
-}
-
-}  // namespace pyarrow
-
-#endif // PYARROW_STATUS_H_

From 676c32ccea6274c75b2750453c1ddbc5f645c037 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Oct 2016 21:18:30 -0400
Subject: [PATCH 0176/1644] ARROW-317: Add Slice, Copy methods to Buffer

There's also a little bit of naming cleanup in `bit-util.h`, pardon the diff noise.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #177 from wesm/ARROW-317 and squashes the following commits:

0666b22 [Wes McKinney] Fix up pyarrow usage of BitUtil
3ab4e7a [Wes McKinney] Add Slice, Copy methods to Buffer
cb9519d [Wes McKinney] Use more conforming names in bit-util.h
---
 cpp/src/arrow/array.cc                |  3 +-
 cpp/src/arrow/array.h                 |  2 +-
 cpp/src/arrow/builder.cc              | 12 ++++----
 cpp/src/arrow/column-benchmark.cc     |  2 +-
 cpp/src/arrow/ipc/adapter.cc          |  5 ++--
 cpp/src/arrow/ipc/test-common.h       |  3 +-
 cpp/src/arrow/test-util.h             |  6 ++--
 cpp/src/arrow/types/list.cc           |  2 +-
 cpp/src/arrow/types/primitive-test.cc | 16 +++++------
 cpp/src/arrow/types/primitive.cc      | 13 +++++----
 cpp/src/arrow/types/primitive.h       | 12 ++++----
 cpp/src/arrow/util/bit-util-test.cc   | 36 +++++++++++------------
 cpp/src/arrow/util/bit-util.cc        | 10 +++----
 cpp/src/arrow/util/bit-util.h         | 29 +++++++++----------
 cpp/src/arrow/util/buffer-test.cc     | 41 +++++++++++++++++++++++++++
 cpp/src/arrow/util/buffer.cc          | 28 +++++++++++++++++-
 cpp/src/arrow/util/buffer.h           | 23 +++++++++++----
 python/src/pyarrow/adapters/pandas.cc | 20 ++++++-------
 18 files changed, 173 insertions(+), 90 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index d6b081f315532..e432a53781f17 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -19,6 +19,7 @@
 
 #include <cstdint>
 
+#include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
 
@@ -43,7 +44,7 @@ bool Array::EqualsExact(const Array& other) const {
     return false;
   }
   if (null_count_ > 0) {
-    return null_bitmap_->Equals(*other.null_bitmap_, util::bytes_for_bits(length_));
+    return null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
   }
   return true;
 }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index c7ffb23ca18a1..ff37323f60519 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -45,7 +45,7 @@ class ARROW_EXPORT Array {
 
   // Determine if a slot is null. For inner loops. Does *not* boundscheck
   bool IsNull(int i) const {
-    return null_count_ > 0 && util::bit_not_set(null_bitmap_data_, i);
+    return null_count_ > 0 && BitUtil::BitNotSet(null_bitmap_data_, i);
   }
 
   int32_t length() const { return length_; }
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 1fba96169228f..151b257a3d894 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -31,7 +31,7 @@ Status ArrayBuilder::AppendToBitmap(bool is_valid) {
     // TODO(emkornfield) doubling isn't great default allocation practice
     // see https://github.com/facebook/folly/blob/master/folly/docs/FBVector.md
     // fo discussion
-    RETURN_NOT_OK(Resize(util::next_power2(capacity_ + 1)));
+    RETURN_NOT_OK(Resize(BitUtil::NextPower2(capacity_ + 1)));
   }
   UnsafeAppendToBitmap(is_valid);
   return Status::OK();
@@ -45,7 +45,7 @@ Status ArrayBuilder::AppendToBitmap(const uint8_t* valid_bytes, int32_t length)
 }
 
 Status ArrayBuilder::Init(int32_t capacity) {
-  int32_t to_alloc = util::ceil_byte(capacity) / 8;
+  int32_t to_alloc = BitUtil::CeilByte(capacity) / 8;
   null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
   RETURN_NOT_OK(null_bitmap_->Resize(to_alloc));
   // Buffers might allocate more then necessary to satisfy padding requirements
@@ -58,7 +58,7 @@ Status ArrayBuilder::Init(int32_t capacity) {
 
 Status ArrayBuilder::Resize(int32_t new_bits) {
   if (!null_bitmap_) { return Init(new_bits); }
-  int32_t new_bytes = util::ceil_byte(new_bits) / 8;
+  int32_t new_bytes = BitUtil::CeilByte(new_bits) / 8;
   int32_t old_bytes = null_bitmap_->size();
   RETURN_NOT_OK(null_bitmap_->Resize(new_bytes));
   null_bitmap_data_ = null_bitmap_->mutable_data();
@@ -82,7 +82,7 @@ Status ArrayBuilder::Advance(int32_t elements) {
 Status ArrayBuilder::Reserve(int32_t elements) {
   if (length_ + elements > capacity_) {
     // TODO(emkornfield) power of 2 growth is potentially suboptimal
-    int32_t new_capacity = util::next_power2(length_ + elements);
+    int32_t new_capacity = BitUtil::NextPower2(length_ + elements);
     return Resize(new_capacity);
   }
   return Status::OK();
@@ -96,7 +96,7 @@ Status ArrayBuilder::SetNotNull(int32_t length) {
 
 void ArrayBuilder::UnsafeAppendToBitmap(bool is_valid) {
   if (is_valid) {
-    util::set_bit(null_bitmap_data_, length_);
+    BitUtil::SetBit(null_bitmap_data_, length_);
   } else {
     ++null_count_;
   }
@@ -118,7 +118,7 @@ void ArrayBuilder::UnsafeSetNotNull(int32_t length) {
   const int32_t new_length = length + length_;
   // TODO(emkornfield) Optimize for large values of length?
   for (int32_t i = length_; i < new_length; ++i) {
-    util::set_bit(null_bitmap_data_, i);
+    BitUtil::SetBit(null_bitmap_data_, i);
   }
   length_ = new_length;
 }
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index edea0948860de..f429a813c6f20 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -29,7 +29,7 @@ std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
   auto data = std::make_shared<PoolBuffer>(pool);
   auto null_bitmap = std::make_shared<PoolBuffer>(pool);
   data->Resize(length * sizeof(typename ArrayType::value_type));
-  null_bitmap->Resize(util::bytes_for_bits(length));
+  null_bitmap->Resize(BitUtil::BytesForBits(length));
   return std::make_shared<ArrayType>(length, data, 10, null_bitmap);
 }
 }  // anonymous namespace
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index f84cb264f70e1..74786bf85ffb4 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -37,6 +37,7 @@
 #include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
 #include "arrow/types/struct.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/status.h"
@@ -49,7 +50,7 @@ namespace ipc {
 
 namespace {
 Status CheckMultipleOf64(int64_t size) {
-  if (util::is_multiple_of_64(size)) { return Status::OK(); }
+  if (BitUtil::IsMultipleOf64(size)) { return Status::OK(); }
   return Status::Invalid(
       "Attempted to write a buffer that "
       "wasn't a multiple of 64 bytes");
@@ -155,7 +156,7 @@ class RecordBatchWriter {
       // The buffer might be null if we are handling zero row lengths.
       if (buffer) {
         size = buffer->size();
-        padding = util::RoundUpToMultipleOf64(size) - size;
+        padding = BitUtil::RoundUpToMultipleOf64(size) - size;
       }
 
       // TODO(wesm): We currently have no notion of shared memory page id's,
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 13bbbebde8aa1..784e238e977c7 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -31,6 +31,7 @@
 #include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
 #include "arrow/types/struct.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/memory-pool.h"
 
@@ -263,7 +264,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
   std::vector<uint8_t> null_bytes(list_batch->num_rows(), 1);
   null_bytes[0] = 0;
   std::shared_ptr<Buffer> null_bitmask;
-  RETURN_NOT_OK(util::bytes_to_bits(null_bytes, &null_bitmask));
+  RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask));
   ArrayPtr with_nulls(
       new StructArray(type, list_batch->num_rows(), columns, 1, null_bitmask));
 
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index e632ffb1d892d..ac56f5ed0871c 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -69,7 +69,7 @@ class TestBase : public ::testing::Test {
     auto data = std::make_shared<PoolBuffer>(pool_);
     auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
     EXPECT_OK(data->Resize(length * sizeof(typename ArrayType::value_type)));
-    EXPECT_OK(null_bitmap->Resize(util::bytes_for_bits(length)));
+    EXPECT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
     return std::make_shared<ArrayType>(length, data, 10, null_bitmap);
   }
 
@@ -152,7 +152,7 @@ static inline int bitmap_popcount(const uint8_t* data, int length) {
   // versions of popcount but the code complexity is likely not worth it)
   const int loop_tail_index = fast_counts * pop_len;
   for (int i = loop_tail_index; i < length; ++i) {
-    if (util::get_bit(data, i)) { ++count; }
+    if (BitUtil::GetBit(data, i)) { ++count; }
   }
 
   return count;
@@ -170,7 +170,7 @@ std::shared_ptr<Buffer> bytes_to_null_buffer(const std::vector<uint8_t>& bytes)
   std::shared_ptr<Buffer> out;
 
   // TODO(wesm): error checking
-  util::bytes_to_bits(bytes, &out);
+  BitUtil::BytesToBits(bytes, &out);
   return out;
 }
 
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index ef2ec22cb5336..4b1e821472795 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -30,7 +30,7 @@ bool ListArray::EqualsExact(const ListArray& other) const {
   bool equal_null_bitmap = true;
   if (null_count_ > 0) {
     equal_null_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, util::bytes_for_bits(length_));
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
   }
 
   if (!equal_null_bitmap) { return false; }
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index 121bd4794f291..e47f6dc74fb7e 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -236,7 +236,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 
   for (int i = 0; i < result->length(); ++i) {
     if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
-    bool actual = util::get_bit(result->raw_data(), i);
+    bool actual = BitUtil::GetBit(result->raw_data(), i);
     ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
   }
   ASSERT_TRUE(result->EqualsExact(*expected.get()));
@@ -258,8 +258,8 @@ TYPED_TEST(TestPrimitiveBuilder, TestInit) {
 
   int n = 1000;
   ASSERT_OK(this->builder_->Reserve(n));
-  ASSERT_EQ(util::next_power2(n), this->builder_->capacity());
-  ASSERT_EQ(util::next_power2(TypeTraits<Type>::bytes_required(n)),
+  ASSERT_EQ(BitUtil::NextPower2(n), this->builder_->capacity());
+  ASSERT_EQ(BitUtil::NextPower2(TypeTraits<Type>::bytes_required(n)),
       this->builder_->data()->size());
 
   // unsure if this should go in all builder classes
@@ -409,10 +409,10 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   }
 
   ASSERT_EQ(size, this->builder_->length());
-  ASSERT_EQ(util::next_power2(size), this->builder_->capacity());
+  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
 
   ASSERT_EQ(size, this->builder_nn_->length());
-  ASSERT_EQ(util::next_power2(size), this->builder_nn_->capacity());
+  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_nn_->capacity());
 
   this->Check(this->builder_, true);
   this->Check(this->builder_nn_, false);
@@ -444,7 +444,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
   ASSERT_OK(this->builder_nn_->Append(draws.data() + K, size - K));
 
   ASSERT_EQ(size, this->builder_->length());
-  ASSERT_EQ(util::next_power2(size), this->builder_->capacity());
+  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
 
   this->Check(this->builder_, true);
   this->Check(this->builder_nn_, false);
@@ -472,7 +472,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestResize) {
   ASSERT_EQ(cap, this->builder_->capacity());
 
   ASSERT_EQ(TypeTraits<Type>::bytes_required(cap), this->builder_->data()->size());
-  ASSERT_EQ(util::bytes_for_bits(cap), this->builder_->null_bitmap()->size());
+  ASSERT_EQ(BitUtil::BytesForBits(cap), this->builder_->null_bitmap()->size());
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
@@ -484,7 +484,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
   ASSERT_OK(this->builder_->Advance(100));
   ASSERT_OK(this->builder_->Reserve(kMinBuilderCapacity));
 
-  ASSERT_EQ(util::next_power2(kMinBuilderCapacity + 100), this->builder_->capacity());
+  ASSERT_EQ(BitUtil::NextPower2(kMinBuilderCapacity + 100), this->builder_->capacity());
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 3a05ccfdf1861..d2288bafa71da 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -19,6 +19,7 @@
 
 #include <memory>
 
+#include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
 
@@ -41,7 +42,7 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
 
   if (null_count_ > 0) {
     bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, util::ceil_byte(length_) / 8);
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
     if (!equal_bitmap) { return false; }
 
     const uint8_t* this_data = raw_data_;
@@ -156,9 +157,9 @@ Status PrimitiveBuilder<BooleanType>::Append(
     if ((valid_bytes != nullptr) && !valid_bytes[i]) continue;
 
     if (values[i] > 0) {
-      util::set_bit(raw_data_, length_ + i);
+      BitUtil::SetBit(raw_data_, length_ + i);
     } else {
-      util::clear_bit(raw_data_, length_ + i);
+      BitUtil::ClearBit(raw_data_, length_ + i);
     }
   }
 
@@ -196,20 +197,20 @@ bool BooleanArray::EqualsExact(const BooleanArray& other) const {
 
   if (null_count_ > 0) {
     bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, util::bytes_for_bits(length_));
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
     if (!equal_bitmap) { return false; }
 
     const uint8_t* this_data = raw_data_;
     const uint8_t* other_data = other.raw_data_;
 
     for (int i = 0; i < length_; ++i) {
-      if (!IsNull(i) && util::get_bit(this_data, i) != util::get_bit(other_data, i)) {
+      if (!IsNull(i) && BitUtil::GetBit(this_data, i) != BitUtil::GetBit(other_data, i)) {
         return false;
       }
     }
     return true;
   } else {
-    return data_->Equals(*other.data_, util::bytes_for_bits(length_));
+    return data_->Equals(*other.data_, BitUtil::BytesForBits(length_));
   }
 }
 
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index f21470d96e45b..c71df584ffe3f 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -173,7 +173,7 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
 
   // Does not capacity-check; make sure to call Reserve beforehand
   void UnsafeAppend(value_type val) {
-    util::set_bit(null_bitmap_data_, length_);
+    BitUtil::SetBit(null_bitmap_data_, length_);
     raw_data_[length_++] = val;
   }
 
@@ -290,7 +290,7 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
 
   const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
 
-  bool Value(int i) const { return util::get_bit(raw_data(), i); }
+  bool Value(int i) const { return BitUtil::GetBit(raw_data(), i); }
 };
 
 template <>
@@ -298,7 +298,7 @@ struct TypeTraits<BooleanType> {
   typedef BooleanArray ArrayType;
 
   static inline int bytes_required(int elements) {
-    return util::bytes_for_bits(elements);
+    return BitUtil::BytesForBits(elements);
   }
 };
 
@@ -314,11 +314,11 @@ class ARROW_EXPORT BooleanBuilder : public PrimitiveBuilder<BooleanType> {
   // Scalar append
   Status Append(bool val) {
     Reserve(1);
-    util::set_bit(null_bitmap_data_, length_);
+    BitUtil::SetBit(null_bitmap_data_, length_);
     if (val) {
-      util::set_bit(raw_data_, length_);
+      BitUtil::SetBit(raw_data_, length_);
     } else {
-      util::clear_bit(raw_data_, length_);
+      BitUtil::ClearBit(raw_data_, length_);
     }
     ++length_;
     return Status::OK();
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index e1d8a0808b41a..cfdee04f6e2ea 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -22,33 +22,33 @@
 namespace arrow {
 
 TEST(UtilTests, TestIsMultipleOf64) {
-  using util::is_multiple_of_64;
-  EXPECT_TRUE(is_multiple_of_64(64));
-  EXPECT_TRUE(is_multiple_of_64(0));
-  EXPECT_TRUE(is_multiple_of_64(128));
-  EXPECT_TRUE(is_multiple_of_64(192));
-  EXPECT_FALSE(is_multiple_of_64(23));
-  EXPECT_FALSE(is_multiple_of_64(32));
+  using BitUtil::IsMultipleOf64;
+  EXPECT_TRUE(IsMultipleOf64(64));
+  EXPECT_TRUE(IsMultipleOf64(0));
+  EXPECT_TRUE(IsMultipleOf64(128));
+  EXPECT_TRUE(IsMultipleOf64(192));
+  EXPECT_FALSE(IsMultipleOf64(23));
+  EXPECT_FALSE(IsMultipleOf64(32));
 }
 
 TEST(UtilTests, TestNextPower2) {
-  using util::next_power2;
+  using BitUtil::NextPower2;
 
-  ASSERT_EQ(8, next_power2(6));
-  ASSERT_EQ(8, next_power2(8));
+  ASSERT_EQ(8, NextPower2(6));
+  ASSERT_EQ(8, NextPower2(8));
 
-  ASSERT_EQ(1, next_power2(1));
-  ASSERT_EQ(256, next_power2(131));
+  ASSERT_EQ(1, NextPower2(1));
+  ASSERT_EQ(256, NextPower2(131));
 
-  ASSERT_EQ(1024, next_power2(1000));
+  ASSERT_EQ(1024, NextPower2(1000));
 
-  ASSERT_EQ(4096, next_power2(4000));
+  ASSERT_EQ(4096, NextPower2(4000));
 
-  ASSERT_EQ(65536, next_power2(64000));
+  ASSERT_EQ(65536, NextPower2(64000));
 
-  ASSERT_EQ(1LL << 32, next_power2((1LL << 32) - 1));
-  ASSERT_EQ(1LL << 31, next_power2((1LL << 31) - 1));
-  ASSERT_EQ(1LL << 62, next_power2((1LL << 62) - 1));
+  ASSERT_EQ(1LL << 32, NextPower2((1LL << 32) - 1));
+  ASSERT_EQ(1LL << 31, NextPower2((1LL << 31) - 1));
+  ASSERT_EQ(1LL << 62, NextPower2((1LL << 62) - 1));
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 475576e87cadd..7e1cb1867171e 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -24,20 +24,20 @@
 
 namespace arrow {
 
-void util::bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits) {
+void BitUtil::BytesToBits(const std::vector<uint8_t>& bytes, uint8_t* bits) {
   for (size_t i = 0; i < bytes.size(); ++i) {
-    if (bytes[i] > 0) { set_bit(bits, i); }
+    if (bytes[i] > 0) { SetBit(bits, i); }
   }
 }
 
-Status util::bytes_to_bits(
+Status BitUtil::BytesToBits(
     const std::vector<uint8_t>& bytes, std::shared_ptr<Buffer>* out) {
-  int bit_length = util::bytes_for_bits(bytes.size());
+  int bit_length = BitUtil::BytesForBits(bytes.size());
 
   auto buffer = std::make_shared<PoolBuffer>();
   RETURN_NOT_OK(buffer->Resize(bit_length));
   memset(buffer->mutable_data(), 0, bit_length);
-  bytes_to_bits(bytes, buffer->mutable_data());
+  BytesToBits(bytes, buffer->mutable_data());
 
   *out = buffer;
   return Status::OK();
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index c33ef272f05e2..13b7e19593d93 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -30,39 +30,39 @@ namespace arrow {
 class Buffer;
 class Status;
 
-namespace util {
+namespace BitUtil {
 
-static inline int64_t ceil_byte(int64_t size) {
+static inline int64_t CeilByte(int64_t size) {
   return (size + 7) & ~7;
 }
 
-static inline int64_t bytes_for_bits(int64_t size) {
-  return ceil_byte(size) / 8;
+static inline int64_t BytesForBits(int64_t size) {
+  return CeilByte(size) / 8;
 }
 
-static inline int64_t ceil_2bytes(int64_t size) {
+static inline int64_t Ceil2Bytes(int64_t size) {
   return (size + 15) & ~15;
 }
 
 static constexpr uint8_t kBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128};
 
-static inline bool get_bit(const uint8_t* bits, int i) {
+static inline bool GetBit(const uint8_t* bits, int i) {
   return static_cast<bool>(bits[i / 8] & kBitmask[i % 8]);
 }
 
-static inline bool bit_not_set(const uint8_t* bits, int i) {
+static inline bool BitNotSet(const uint8_t* bits, int i) {
   return (bits[i / 8] & kBitmask[i % 8]) == 0;
 }
 
-static inline void clear_bit(uint8_t* bits, int i) {
+static inline void ClearBit(uint8_t* bits, int i) {
   bits[i / 8] &= ~kBitmask[i % 8];
 }
 
-static inline void set_bit(uint8_t* bits, int i) {
+static inline void SetBit(uint8_t* bits, int i) {
   bits[i / 8] |= kBitmask[i % 8];
 }
 
-static inline int64_t next_power2(int64_t n) {
+static inline int64_t NextPower2(int64_t n) {
   n--;
   n |= n >> 1;
   n |= n >> 2;
@@ -74,7 +74,7 @@ static inline int64_t next_power2(int64_t n) {
   return n;
 }
 
-static inline bool is_multiple_of_64(int64_t n) {
+static inline bool IsMultipleOf64(int64_t n) {
   return (n & 63) == 0;
 }
 
@@ -90,11 +90,10 @@ inline int64_t RoundUpToMultipleOf64(int64_t num) {
   return num;
 }
 
-void bytes_to_bits(const std::vector<uint8_t>& bytes, uint8_t* bits);
-ARROW_EXPORT Status bytes_to_bits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
-
-}  // namespace util
+void BytesToBits(const std::vector<uint8_t>& bytes, uint8_t* bits);
+ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
+}  // namespace BitUtil
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_BIT_UTIL_H
diff --git a/cpp/src/arrow/util/buffer-test.cc b/cpp/src/arrow/util/buffer-test.cc
index cc4ec98e4fb29..095b07b7ab309 100644
--- a/cpp/src/arrow/util/buffer-test.cc
+++ b/cpp/src/arrow/util/buffer-test.cc
@@ -31,6 +31,18 @@ namespace arrow {
 
 class TestBuffer : public ::testing::Test {};
 
+TEST_F(TestBuffer, IsMutableFlag) {
+  Buffer buf(nullptr, 0);
+
+  ASSERT_FALSE(buf.is_mutable());
+
+  MutableBuffer mbuf(nullptr, 0);
+  ASSERT_TRUE(mbuf.is_mutable());
+
+  PoolBuffer pbuf;
+  ASSERT_TRUE(pbuf.is_mutable());
+}
+
 TEST_F(TestBuffer, Resize) {
   PoolBuffer buf;
 
@@ -96,4 +108,33 @@ TEST_F(TestBuffer, EqualsWithSameBuffer) {
   pool->Free(rawBuffer, bufferSize);
 }
 
+TEST_F(TestBuffer, Copy) {
+  std::string data_str = "some data to copy";
+
+  auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
+
+  Buffer buf(data, data_str.size());
+
+  std::shared_ptr<Buffer> out;
+
+  ASSERT_OK(buf.Copy(5, 4, &out));
+
+  Buffer expected(data + 5, 4);
+  ASSERT_TRUE(out->Equals(expected));
+}
+
+TEST_F(TestBuffer, SliceBuffer) {
+  std::string data_str = "some data to slice";
+
+  auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
+
+  auto buf = std::make_shared<Buffer>(data, data_str.size());
+
+  std::shared_ptr<Buffer> out = SliceBuffer(buf, 5, 4);
+  Buffer expected(data + 5, 4);
+  ASSERT_TRUE(out->Equals(expected));
+
+  ASSERT_EQ(2, buf.use_count());
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
index 6faa048e4e52b..a230259e5930d 100644
--- a/cpp/src/arrow/util/buffer.cc
+++ b/cpp/src/arrow/util/buffer.cc
@@ -36,6 +36,32 @@ Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t si
 
 Buffer::~Buffer() {}
 
+Status Buffer::Copy(
+    int64_t start, int64_t nbytes, MemoryPool* pool, std::shared_ptr<Buffer>* out) const {
+  // Sanity checks
+  DCHECK_LT(start, size_);
+  DCHECK_LE(nbytes, size_ - start);
+
+  auto new_buffer = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(new_buffer->Resize(nbytes));
+
+  std::memcpy(new_buffer->mutable_data(), data() + start, nbytes);
+
+  *out = new_buffer;
+  return Status::OK();
+}
+
+Status Buffer::Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const {
+  return Copy(start, nbytes, default_memory_pool(), out);
+}
+
+std::shared_ptr<Buffer> SliceBuffer(
+    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
+  DCHECK_LT(offset, buffer->size());
+  DCHECK_LE(length, buffer->size() - offset);
+  return std::make_shared<Buffer>(buffer, offset, length);
+}
+
 std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
   return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
 }
@@ -52,7 +78,7 @@ PoolBuffer::~PoolBuffer() {
 Status PoolBuffer::Reserve(int64_t new_capacity) {
   if (!mutable_data_ || new_capacity > capacity_) {
     uint8_t* new_data;
-    new_capacity = util::RoundUpToMultipleOf64(new_capacity);
+    new_capacity = BitUtil::RoundUpToMultipleOf64(new_capacity);
     if (mutable_data_) {
       RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
       memcpy(new_data, mutable_data_, size_);
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index bc0df86221c45..04ad6c2dffde4 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -43,7 +43,8 @@ class Status;
 // The following invariant is always true: Size < Capacity
 class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
  public:
-  Buffer(const uint8_t* data, int64_t size) : data_(data), size_(size), capacity_(size) {}
+  Buffer(const uint8_t* data, int64_t size)
+      : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
   virtual ~Buffer();
 
   // An offset into data that is owned by another buffer, but we want to be
@@ -57,6 +58,8 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
 
   std::shared_ptr<Buffer> get_shared_ptr() { return shared_from_this(); }
 
+  bool is_mutable() const { return is_mutable_; }
+
   // Return true if both buffers are the same size and contain the same bytes
   // up to the number of compared bytes
   bool Equals(const Buffer& other, int64_t nbytes) const {
@@ -71,18 +74,22 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
                (data_ == other.data_ || !memcmp(data_, other.data_, size_)));
   }
 
+  // Copy section of buffer into a new Buffer
+  Status Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
+      std::shared_ptr<Buffer>* out) const;
+
+  // Default memory pool
+  Status Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const;
+
   int64_t capacity() const { return capacity_; }
   const uint8_t* data() const { return data_; }
 
   int64_t size() const { return size_; }
 
-  // Returns true if this Buffer is referencing memory (possibly) owned by some
-  // other buffer
-  bool is_shared() const { return static_cast<bool>(parent_); }
-
   const std::shared_ptr<Buffer> parent() const { return parent_; }
 
  protected:
+  bool is_mutable_;
   const uint8_t* data_;
   int64_t size_;
   int64_t capacity_;
@@ -94,10 +101,16 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
   DISALLOW_COPY_AND_ASSIGN(Buffer);
 };
 
+// Construct a view on passed buffer at the indicated offset and length. This
+// function cannot fail and does not error checking (except in debug builds)
+std::shared_ptr<Buffer> SliceBuffer(
+    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
+
 // A Buffer whose contents can be mutated. May or may not own its data.
 class ARROW_EXPORT MutableBuffer : public Buffer {
  public:
   MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
+    is_mutable_ = true;
     mutable_data_ = data;
   }
 
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 5902b8341696d..7e70be75da5fc 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -44,7 +44,7 @@ using arrow::Field;
 using arrow::DataType;
 using arrow::Status;
 
-namespace util = arrow::util;
+namespace BitUtil = arrow::BitUtil;
 
 // ----------------------------------------------------------------------
 // Serialization
@@ -148,7 +148,7 @@ class ArrowSerializer {
   }
 
   Status InitNullBitmap() {
-    int null_bytes = util::bytes_for_bits(length_);
+    int null_bytes = BitUtil::BytesForBits(length_);
 
     null_bitmap_ = std::make_shared<arrow::PoolBuffer>(pool_);
     RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
@@ -206,7 +206,7 @@ class ArrowSerializer {
 
     PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
 
-    int nbytes = util::bytes_for_bits(length_);
+    int nbytes = BitUtil::BytesForBits(length_);
     auto data = std::make_shared<arrow::PoolBuffer>(pool_);
     RETURN_NOT_OK(data->Resize(nbytes));
     uint8_t* bitmap = data->mutable_data();
@@ -215,12 +215,12 @@ class ArrowSerializer {
     int64_t null_count = 0;
     for (int64_t i = 0; i < length_; ++i) {
       if (objects[i] == Py_True) {
-        util::set_bit(bitmap, i);
-        util::set_bit(null_bitmap_data_, i);
+        BitUtil::SetBit(bitmap, i);
+        BitUtil::SetBit(null_bitmap_data_, i);
       } else if (objects[i] != Py_False) {
         ++null_count;
       } else {
-        util::set_bit(null_bitmap_data_, i);
+        BitUtil::SetBit(null_bitmap_data_, i);
       }
     }
 
@@ -253,7 +253,7 @@ static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap
     if (mask_values[i]) {
       ++null_count;
     } else {
-      util::set_bit(bitmap, i);
+      BitUtil::SetBit(bitmap, i);
     }
   }
   return null_count;
@@ -272,7 +272,7 @@ static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap)
     if (traits::isnull(values[i])) {
       ++null_count;
     } else {
-      util::set_bit(bitmap, i);
+      BitUtil::SetBit(bitmap, i);
     }
   }
 
@@ -402,7 +402,7 @@ inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
     return Status::Invalid("no support for strided data yet");
   }
 
-  int nbytes = util::bytes_for_bits(length_);
+  int nbytes = BitUtil::BytesForBits(length_);
   auto buffer = std::make_shared<arrow::PoolBuffer>(pool_);
   RETURN_NOT_OK(buffer->Resize(nbytes));
 
@@ -413,7 +413,7 @@ inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
   memset(bitmap, 0, nbytes);
   for (int i = 0; i < length_; ++i) {
     if (values[i] > 0) {
-      util::set_bit(bitmap, i);
+      BitUtil::SetBit(bitmap, i);
     }
   }
 

From e2c0a18316504a0177129cb66b25a9dc54291587 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Oct 2016 22:46:44 -0400
Subject: [PATCH 0177/1644] ARROW-327: [Python] Remove conda builds from Travis
 CI setup

We'll do these builds in conda-forge

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #178 from wesm/ARROW-327 and squashes the following commits:

1303d6e [Wes McKinney] Remove conda builds
---
 .travis.yml              | 18 ----------------
 ci/travis_conda_build.sh | 45 ----------------------------------------
 2 files changed, 63 deletions(-)
 delete mode 100755 ci/travis_conda_build.sh

diff --git a/.travis.yml b/.travis.yml
index 97229b1ceb3bc..a53756c962e88 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -41,24 +41,6 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
-  - compiler: gcc
-    env: ARROW_TEST_GROUP=packaging
-    os: linux
-    before_script:
-    - export CC="gcc-4.9"
-    - export CXX="g++-4.9"
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_conda_build.sh
-  - os: osx
-    env: ARROW_TEST_GROUP=packaging
-    language: objective-c
-    osx_image: xcode6.4
-    compiler: clang
-    addons:
-    before_script:
-    before_install:
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_conda_build.sh
   - language: java
     os: linux
     jdk: oraclejdk7
diff --git a/ci/travis_conda_build.sh b/ci/travis_conda_build.sh
deleted file mode 100755
index 17a33ae9717bc..0000000000000
--- a/ci/travis_conda_build.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-set -ex
-
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-
-# Build libarrow
-
-cd $TRAVIS_BUILD_DIR/cpp
-
-conda build conda.recipe --channel apache/channel/dev
-CONDA_PACKAGE=`conda build --output conda.recipe | grep bz2`
-
-if [ $TRAVIS_BRANCH == "master" ] && [ $TRAVIS_PULL_REQUEST == "false" ]; then
-  anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev;
-fi
-
-# Build pyarrow
-
-cd $TRAVIS_BUILD_DIR/python
-
-build_for_python_version() {
-  PY_VERSION=$1
-  conda build conda.recipe --python $PY_VERSION --channel apache/channel/dev
-  CONDA_PACKAGE=`conda build --python $PY_VERSION --output conda.recipe | grep bz2`
-
-  if [ $TRAVIS_BRANCH == "master" ] && [ $TRAVIS_PULL_REQUEST == "false" ]; then
-	anaconda --token $ANACONDA_TOKEN upload $CONDA_PACKAGE --user apache --channel dev;
-  fi
-}
-
-build_for_python_version 2.7
-build_for_python_version 3.5

From 446ec9bd628244bf675887f5a030d3a94c07645e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Oct 2016 22:49:56 -0400
Subject: [PATCH 0178/1644] ARROW-334: [Python] Remove
 INSTALL_RPATH_USE_LINK_PATH

Will try to verify whether this resolves the issue. See https://travis-ci.org/conda-forge/staged-recipes/builds/166897102

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #171 from wesm/ARROW-334 and squashes the following commits:

ed8fa39 [Wes McKinney] Switch by to xcode 6.4
b8224ce [Wes McKinney] Escape dollar sign in ORIGIN
b76b7ac [Wes McKinney] Fix LD_LIBRARY_PATH
3c8d2dd [Wes McKinney] Clean up Travis CI scripts a bit. Put  in LD_LIBRARY_PATH
30488d7 [Wes McKinney] Don't conda install arrow-cpp during Travis build
afb1dc0 [Wes McKinney] Remove INSTALL_RPATH_USE_LINK_PATH
---
 .travis.yml                    |  1 -
 ci/travis_before_script_cpp.sh |  4 ----
 ci/travis_script_python.sh     | 15 ++++++---------
 python/CMakeLists.txt          |  4 +---
 4 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a53756c962e88..052c22ccc3790 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,7 +32,6 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
   - compiler: clang
-    language: objective-c
     osx_image: xcode6.4
     os: osx
     addons:
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 2d4224b33336f..20307736e672a 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -15,10 +15,6 @@
 
 set -ex
 
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-conda install -y --channel apache/channel/dev parquet-cpp
-export PARQUET_HOME=$MINICONDA
-
 : ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
 
 mkdir $CPP_BUILD_DIR
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 55cb2a76f6db1..179567b595416 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -14,12 +14,16 @@
 
 set -e
 
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
 PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 
 # Re-use conda installation from C++
 export MINICONDA=$HOME/miniconda
 export PATH="$MINICONDA/bin:$PATH"
-export PARQUET_HOME=$MINICONDA
+
+export ARROW_HOME=$ARROW_CPP_INSTALL
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
 
 pushd $PYTHON_DIR
 
@@ -38,17 +42,10 @@ python_version_tests() {
   # Expensive dependencies install from Continuum package repo
   conda install -y pip numpy pandas cython
 
-  # conda install -y parquet-cpp
-
-  conda install -y arrow-cpp -c apache/channel/dev
-
   # Other stuff pip install
   pip install -r requirements.txt
 
-  export ARROW_HOME=$ARROW_CPP_INSTALL
-
-  python setup.py build_ext \
-		 --inplace
+  python setup.py build_ext --inplace
 
   python -m pytest -vv -r sxX pyarrow
 
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 4357fa05ff864..b8be8665af079 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -417,8 +417,6 @@ if (UNIX)
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 endif()
 
-SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-
 add_subdirectory(src/pyarrow)
 add_subdirectory(src/pyarrow/util)
 
@@ -494,7 +492,7 @@ foreach(module ${CYTHON_EXTENSIONS})
     if(APPLE)
         set(module_install_rpath "@loader_path")
     else()
-        set(module_install_rpath "$ORIGIN")
+        set(module_install_rpath "\$ORIGIN")
     endif()
     list(LENGTH directories i)
     while(${i} GREATER 0)

From 2f84493371bd8fae30b8e042984c9d6ba5419c5f Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 21 Oct 2016 16:27:00 -0400
Subject: [PATCH 0179/1644] ARROW-342: Set Python version on release

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #179 from xhochy/ARROW-342 and squashes the following commits:

15d0ce3 [Uwe L. Korn] ARROW-342: Set Python version on release
---
 dev/release/00-prepare.sh  |  9 +++++++--
 python/.gitignore          |  1 +
 python/pyarrow/__init__.py |  1 +
 python/setup.py            | 24 ++++++++++++++++++++----
 4 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
index 3c1fb9a093892..3423a3e6c5bf9 100644
--- a/dev/release/00-prepare.sh
+++ b/dev/release/00-prepare.sh
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -43,4 +43,9 @@ mvn release:prepare -Dtag=${tag} -DreleaseVersion=${version} -DautoVersionSubmod
 
 cd -
 
+cd "${SOURCE_DIR}/../../python"
+sed -i "s/VERSION = '[^']*'/VERSION = '${version}'/g" setup.py
+sed -i "s/ISRELEASED = False/ISRELEASED = True/g" setup.py
+cd -
+
 echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
diff --git a/python/.gitignore b/python/.gitignore
index 7e2e360557ad8..07f28355a252f 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -25,6 +25,7 @@ MANIFEST
 # Generated sources
 *.c
 *.cpp
+pyarrow/version.py
 # Python files
 
 # setup.py working directory
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 8b131aaa8f4af..775ce7ec47578 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -42,3 +42,4 @@
                             DataType, Field, Schema, schema)
 
 from pyarrow.table import Column, RecordBatch, Table, from_pandas_dataframe
+from pyarrow.version import version as __version__
diff --git a/python/setup.py b/python/setup.py
index d040ea7e892c5..990497775148d 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -50,10 +50,25 @@
 if Cython.__version__ < '0.19.1':
     raise Exception('Please upgrade to Cython 0.19.1 or newer')
 
-MAJOR = 0
-MINOR = 1
-MICRO = 0
-VERSION = '%d.%d.%ddev' % (MAJOR, MINOR, MICRO)
+VERSION = '0.1.0'
+ISRELEASED = False
+
+if not ISRELEASED:
+    VERSION += '.dev'
+
+setup_dir = os.path.abspath(os.path.dirname(__file__))
+
+
+def write_version_py(filename=os.path.join(setup_dir, 'pyarrow/version.py')):
+    a = open(filename, 'w')
+    file_content = "\n".join(["",
+                              "# THIS FILE IS GENERATED FROM SETUP.PY",
+                              "version = '%(version)s'",
+                              "isrelease = '%(isrelease)s'"])
+
+    a.write(file_content % {'version': VERSION,
+                            'isrelease': str(ISRELEASED)})
+    a.close()
 
 
 class clean(_clean):
@@ -238,6 +253,7 @@ def get_outputs(self):
         return [self._get_cmake_ext_path(name)
                 for name in self.get_names()]
 
+write_version_py()
 
 DESC = """\
 Python library for Apache Arrow"""

From 3d2e4df219d6b06a3d78821bbca6ba17188908c2 Mon Sep 17 00:00:00 2001
From: adeneche <adeneche@dremio.com>
Date: Wed, 26 Oct 2016 12:09:26 -0700
Subject: [PATCH 0180/1644] =?UTF-8?q?ARROW-337:=20UnionListWriter.list()?=
 =?UTF-8?q?=20is=20doing=20more=20than=20it=20should,=20this=20=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…can cause data corruption

The general idea is to use the "inner" writer's position to update the offset. This involves making sure various writers do indeed update their positions.

UnionListWriter.startList() should explicitly set the inner writer position in case setPosition() was called to move the union list writer's position

Author: adeneche <adeneche@dremio.com>

Closes #183 from adeneche/ARROW-337 and squashes the following commits:

1ae7e00 [adeneche] updated TestComplexWriter to ensure position is set properly by the various writers
7d5aefc [adeneche] ARROW-337: UnionListWriter.list() is doing more than it should, this can cause data corruption
---
 .../AbstractPromotableFieldWriter.java        |   2 +
 .../main/codegen/templates/MapWriters.java    |   1 +
 .../codegen/templates/UnionListWriter.java    |  32 +--
 .../apache/arrow/vector/TestListVector.java   |   4 -
 .../complex/writer/TestComplexWriter.java     | 201 +++++++++++++-----
 5 files changed, 154 insertions(+), 86 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index d21dcd0f6461c..60dd0c7b7adf8 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -58,6 +58,7 @@ public void start() {
   @Override
   public void end() {
     getWriter(MinorType.MAP).end();
+    setPosition(idx() + 1);
   }
 
   @Override
@@ -68,6 +69,7 @@ public void startList() {
   @Override
   public void endList() {
     getWriter(MinorType.LIST).endList();
+    setPosition(idx() + 1);
   }
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 51327b43af0fa..f41b60072c873 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -185,6 +185,7 @@ public void start() {
 
   @Override
   public void end() {
+    setPosition(idx()+1);
   }
 
   <#list vv.types as type><#list type.minor as minor>
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index 04531a72128a0..bb39fe8d29426 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -101,11 +101,7 @@ public void setPosition(int index) {
   public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
 //    assert inMap;
     mapName = name;
-    final int nextOffset = offsets.getAccessor().get(idx() + 1);
-    vector.getMutator().setNotNull(idx());
-    writer.setPosition(nextOffset);
-    ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name);
-    return ${uncappedName}Writer;
+    return writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name);
   }
 
   </#if>
@@ -120,18 +116,11 @@ public MapWriter map() {
 
   @Override
   public ListWriter list() {
-    final int nextOffset = offsets.getAccessor().get(idx() + 1);
-    vector.getMutator().setNotNull(idx());
-    offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
-    writer.setPosition(nextOffset);
     return writer;
   }
 
   @Override
   public ListWriter list(String name) {
-    final int nextOffset = offsets.getAccessor().get(idx() + 1);
-    vector.getMutator().setNotNull(idx());
-    writer.setPosition(nextOffset);
     ListWriter listWriter = writer.list(name);
     return listWriter;
   }
@@ -145,30 +134,26 @@ public MapWriter map(String name) {
   @Override
   public void startList() {
     vector.getMutator().startNewValue(idx());
+    writer.setPosition(offsets.getAccessor().get(idx() + 1));
   }
 
   @Override
   public void endList() {
-
+    offsets.getMutator().set(idx() + 1, writer.idx());
+    setPosition(idx() + 1);
   }
 
   @Override
   public void start() {
 //    assert inMap;
-    final int nextOffset = offsets.getAccessor().get(idx() + 1);
-    vector.getMutator().setNotNull(idx());
-    offsets.getMutator().setSafe(idx() + 1, nextOffset);
-    writer.setPosition(nextOffset);
     writer.start();
   }
 
   @Override
   public void end() {
 //    if (inMap) {
-      writer.end();
-      inMap = false;
-      final int nextOffset = offsets.getAccessor().get(idx() + 1);
-      offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
+    writer.end();
+    inMap = false;
 //    }
   }
 
@@ -181,11 +166,8 @@ public void end() {
   @Override
   public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
 //    assert !inMap;
-    final int nextOffset = offsets.getAccessor().get(idx() + 1);
-    vector.getMutator().setNotNull(idx());
-    writer.setPosition(nextOffset);
     writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
-    offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
+    writer.setPosition(writer.idx()+1);
   }
 
   </#if>
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index bb7103365557f..1f0baaed776a1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -19,18 +19,14 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.impl.ComplexCopier;
-import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 public class TestListVector {
-  private final static String EMPTY_SCHEMA_PATH = "";
 
   private BufferAllocator allocator;
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 9419f88de5b74..6e0e617f299f8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -65,10 +65,10 @@ public void simpleNestedTypes() {
     IntWriter intWriter = rootWriter.integer("int");
     BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
     for (int i = 0; i < COUNT; i++) {
-      intWriter.setPosition(i);
+      rootWriter.start();
       intWriter.writeInt(i);
-      bigIntWriter.setPosition(i);
       bigIntWriter.writeBigInt(i);
+      rootWriter.end();
     }
     writer.setValueCount(COUNT);
     MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
@@ -83,23 +83,52 @@ public void simpleNestedTypes() {
 
   @Test
   public void nullableMap() {
-    MapVector parent = new MapVector("parent", allocator, null);
-    ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
-    for (int i = 0; i < COUNT; i++) {
-      rootWriter.setPosition(i);
-      rootWriter.start();
-      if (i % 2 == 0) {
-        MapWriter mapWriter = rootWriter.map("map");
-        mapWriter.setPosition(i);
-        mapWriter.start();
-        mapWriter.bigInt("nested").writeBigInt(i);
-        mapWriter.end();
+    try (MapVector mapVector = new MapVector("parent", allocator, null)) {
+      ComplexWriter writer = new ComplexWriterImpl("root", mapVector);
+      MapWriter rootWriter = writer.rootAsMap();
+      for (int i = 0; i < COUNT; i++) {
+        rootWriter.start();
+        if (i % 2 == 0) {
+          MapWriter mapWriter = rootWriter.map("map");
+          mapWriter.setPosition(i);
+          mapWriter.start();
+          mapWriter.bigInt("nested").writeBigInt(i);
+          mapWriter.end();
+        }
+        rootWriter.end();
       }
-      rootWriter.end();
+      writer.setValueCount(COUNT);
+      checkNullableMap(mapVector);
     }
-    writer.setValueCount(COUNT);
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+  }
+
+  /**
+   * This test is similar to {@link #nullableMap()} ()} but we get the inner map writer once at the beginning
+   */
+  @Test
+  public void nullableMap2() {
+    try (MapVector mapVector = new MapVector("parent", allocator, null)) {
+      ComplexWriter writer = new ComplexWriterImpl("root", mapVector);
+      MapWriter rootWriter = writer.rootAsMap();
+      MapWriter mapWriter = rootWriter.map("map");
+
+      for (int i = 0; i < COUNT; i++) {
+        rootWriter.start();
+        if (i % 2 == 0) {
+          mapWriter.setPosition(i);
+          mapWriter.start();
+          mapWriter.bigInt("nested").writeBigInt(i);
+          mapWriter.end();
+        }
+        rootWriter.end();
+      }
+      writer.setValueCount(COUNT);
+      checkNullableMap(mapVector);
+    }
+  }
+
+  private void checkNullableMap(MapVector mapVector) {
+    MapReader rootReader = new SingleMapReaderImpl(mapVector).reader("root");
     for (int i = 0; i < COUNT; i++) {
       rootReader.setPosition(i);
       assertTrue("index is set: " + i, rootReader.isSet());
@@ -113,11 +142,10 @@ public void nullableMap() {
         assertNull("index is not set: " + i, map.readObject());
       }
     }
-    parent.close();
   }
 
   @Test
-  public void listOfLists() {
+  public void testList() {
     MapVector parent = new MapVector("parent", allocator, null);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
@@ -129,7 +157,6 @@ public void listOfLists() {
     rootWriter.list("list").endList();
     rootWriter.end();
 
-    rootWriter.setPosition(1);
     rootWriter.start();
     rootWriter.bigInt("int").writeBigInt(1);
     rootWriter.end();
@@ -152,7 +179,6 @@ public void listScalarType() {
     listVector.allocateNew();
     UnionListWriter listWriter = new UnionListWriter(listVector);
     for (int i = 0; i < COUNT; i++) {
-      listWriter.setPosition(i);
       listWriter.startList();
       for (int j = 0; j < i % 7; j++) {
         listWriter.writeInt(j);
@@ -206,7 +232,6 @@ public void listMapType() {
     UnionListWriter listWriter = new UnionListWriter(listVector);
     MapWriter mapWriter = listWriter.map();
     for (int i = 0; i < COUNT; i++) {
-      listWriter.setPosition(i);
       listWriter.startList();
       for (int j = 0; j < i % 7; j++) {
         mapWriter.start();
@@ -230,23 +255,53 @@ public void listMapType() {
 
   @Test
   public void listListType() {
-    ListVector listVector = new ListVector("list", allocator, null);
-    listVector.allocateNew();
-    UnionListWriter listWriter = new UnionListWriter(listVector);
-    for (int i = 0; i < COUNT; i++) {
-      listWriter.setPosition(i);
-      listWriter.startList();
-      for (int j = 0; j < i % 7; j++) {
-        ListWriter innerListWriter = listWriter.list();
-        innerListWriter.startList();
-        for (int k = 0; k < i % 13; k++) {
-          innerListWriter.integer().writeInt(k);
+    try (ListVector listVector = new ListVector("list", allocator, null)) {
+      listVector.allocateNew();
+      UnionListWriter listWriter = new UnionListWriter(listVector);
+      for (int i = 0; i < COUNT; i++) {
+        listWriter.startList();
+        for (int j = 0; j < i % 7; j++) {
+          ListWriter innerListWriter = listWriter.list();
+          innerListWriter.startList();
+          for (int k = 0; k < i % 13; k++) {
+            innerListWriter.integer().writeInt(k);
+          }
+          innerListWriter.endList();
         }
-        innerListWriter.endList();
+        listWriter.endList();
       }
-      listWriter.endList();
+      listWriter.setValueCount(COUNT);
+      checkListOfLists(listVector);
     }
-    listWriter.setValueCount(COUNT);
+  }
+
+  /**
+   * This test is similar to {@link #listListType()} but we get the inner list writer once at the beginning
+   */
+  @Test
+  public void listListType2() {
+    try (ListVector listVector = new ListVector("list", allocator, null)) {
+      listVector.allocateNew();
+      UnionListWriter listWriter = new UnionListWriter(listVector);
+      ListWriter innerListWriter = listWriter.list();
+
+      for (int i = 0; i < COUNT; i++) {
+        listWriter.startList();
+        for (int j = 0; j < i % 7; j++) {
+          innerListWriter.startList();
+          for (int k = 0; k < i % 13; k++) {
+            innerListWriter.integer().writeInt(k);
+          }
+          innerListWriter.endList();
+        }
+        listWriter.endList();
+      }
+      listWriter.setValueCount(COUNT);
+      checkListOfLists(listVector);
+    }
+  }
+
+  private void checkListOfLists(final ListVector listVector) {
     UnionListReader listReader = new UnionListReader(listVector);
     for (int i = 0; i < COUNT; i++) {
       listReader.setPosition(i);
@@ -259,32 +314,65 @@ public void listListType() {
         }
       }
     }
-    listVector.clear();
   }
 
   @Test
   public void unionListListType() {
-    ListVector listVector = new ListVector("list", allocator, null);
-    listVector.allocateNew();
-    UnionListWriter listWriter = new UnionListWriter(listVector);
-    for (int i = 0; i < COUNT; i++) {
-      listWriter.setPosition(i);
-      listWriter.startList();
-      for (int j = 0; j < i % 7; j++) {
-        ListWriter innerListWriter = listWriter.list();
-        innerListWriter.startList();
-        for (int k = 0; k < i % 13; k++) {
-          if (k % 2 == 0) {
-            innerListWriter.integer().writeInt(k);
-          } else {
-            innerListWriter.bigInt().writeBigInt(k);
+    try (ListVector listVector = new ListVector("list", allocator, null)) {
+      listVector.allocateNew();
+      UnionListWriter listWriter = new UnionListWriter(listVector);
+      for (int i = 0; i < COUNT; i++) {
+        listWriter.startList();
+        for (int j = 0; j < i % 7; j++) {
+          ListWriter innerListWriter = listWriter.list();
+          innerListWriter.startList();
+          for (int k = 0; k < i % 13; k++) {
+            if (k % 2 == 0) {
+              innerListWriter.integer().writeInt(k);
+            } else {
+              innerListWriter.bigInt().writeBigInt(k);
+            }
           }
+          innerListWriter.endList();
         }
-        innerListWriter.endList();
+        listWriter.endList();
       }
-      listWriter.endList();
+      listWriter.setValueCount(COUNT);
+      checkUnionList(listVector);
     }
-    listWriter.setValueCount(COUNT);
+  }
+
+  /**
+   * This test is similar to {@link #unionListListType()} but we get the inner list writer once at the beginning
+   */
+  @Test
+  public void unionListListType2() {
+    try (ListVector listVector = new ListVector("list", allocator, null)) {
+      listVector.allocateNew();
+      UnionListWriter listWriter = new UnionListWriter(listVector);
+      ListWriter innerListWriter = listWriter.list();
+
+      for (int i = 0; i < COUNT; i++) {
+        listWriter.startList();
+        for (int j = 0; j < i % 7; j++) {
+          innerListWriter.startList();
+          for (int k = 0; k < i % 13; k++) {
+            if (k % 2 == 0) {
+              innerListWriter.integer().writeInt(k);
+            } else {
+              innerListWriter.bigInt().writeBigInt(k);
+            }
+          }
+          innerListWriter.endList();
+        }
+        listWriter.endList();
+      }
+      listWriter.setValueCount(COUNT);
+      checkUnionList(listVector);
+    }
+  }
+
+  private void checkUnionList(ListVector listVector) {
     UnionListReader listReader = new UnionListReader(listVector);
     for (int i = 0; i < COUNT; i++) {
       listReader.setPosition(i);
@@ -301,7 +389,6 @@ public void unionListListType() {
         }
       }
     }
-    listVector.clear();
   }
 
   @Test
@@ -384,8 +471,8 @@ public void promotableWriterSchema() {
     MapVector parent = new MapVector("parent", allocator, null);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
-    BigIntWriter bigIntWriter = rootWriter.bigInt("a");
-    VarCharWriter varCharWriter = rootWriter.varChar("a");
+    rootWriter.bigInt("a");
+    rootWriter.varChar("a");
 
     Field field = parent.getField().getChildren().get(0).getChildren().get(0);
     Assert.assertEquals("a", field.getName());

From 6178bf7b0f0cf66f52536f5d5fb5ee104e696f3c Mon Sep 17 00:00:00 2001
From: "Christopher C. Aycock" <christopher.aycock@twosigma.com>
Date: Fri, 28 Oct 2016 21:13:02 -0400
Subject: [PATCH 0181/1644] ARROW-350: Added Kerberos to HDFS client

Author: Christopher C. Aycock <christopher.aycock@twosigma.com>

Closes #185 from chrisaycock/ARROW-350 and squashes the following commits:

c2a4e64 [Christopher C. Aycock] Renamed 'kerb' parameter to 'kerb_ticket'
f1d63de [Christopher C. Aycock] ARROW-350: Added Kerberos to HDFS client
8f1052f [Christopher C. Aycock] ARROW-345: Proper locations of libhdfs and libjvm on Mac
---
 cpp/doc/HDFS.md                         | 22 ++++++-
 cpp/src/arrow/io/hdfs.cc                | 16 ++++-
 cpp/src/arrow/io/hdfs.h                 |  9 +--
 cpp/src/arrow/io/libhdfs_shim.cc        | 87 ++++++++++++++++++-------
 python/pyarrow/includes/libarrow_io.pxd |  1 +
 python/pyarrow/io.pyx                   | 29 ++++++---
 6 files changed, 124 insertions(+), 40 deletions(-)

diff --git a/cpp/doc/HDFS.md b/cpp/doc/HDFS.md
index 83311db2d2dc2..6b1bb8c452461 100644
--- a/cpp/doc/HDFS.md
+++ b/cpp/doc/HDFS.md
@@ -43,7 +43,7 @@ LD_LIBRARY_PATH), and relies on some environment variables.
 export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
 ```
 
-#### Setting $JAVA_HOME  automatically on OS X
+### Mac Specifics
 
 The installed location of Java on OS X can vary, however the following snippet
 will set it automatically for you:
@@ -51,3 +51,23 @@ will set it automatically for you:
 ```shell
 export JAVA_HOME=$(/usr/libexec/java_home)
 ```
+
+Homebrew's Hadoop does not have native libs. Apache doesn't build these, so
+users must build Hadoop to get the native libs. See this Stack Overflow
+answer for details:
+
+http://stackoverflow.com/a/40051353/478288
+
+Be sure to include the path to the native libs in `JAVA_LIBRARY_PATH`:
+
+```shell
+export JAVA_LIBRARY_PATH=$HADOOP_HOME/lib/native:$JAVA_LIBRARY_PATH
+```
+
+If you get an error about needing to install Java 6, then add *BundledApp* and
+*JNI* to the `JVMCapabilities` in `$JAVA_HOME/../Info.plist`. See
+
+https://oliverdowling.com.au/2015/10/09/oracles-jre-8-on-mac-os-x-el-capitan/
+
+https://derflounder.wordpress.com/2015/08/08/modifying-oracles-java-sdk-to-run-java-applications-on-os-x/
+
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index b74f84604f18c..6490a7574eea2 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -287,12 +287,25 @@ class HdfsClient::HdfsClientImpl {
   Status Connect(const HdfsConnectionConfig* config) {
     RETURN_NOT_OK(ConnectLibHdfs());
 
-    fs_ = hdfsConnectAsUser(config->host.c_str(), config->port, config->user.c_str());
+    // connect to HDFS with the builder object
+    hdfsBuilder* builder = hdfsNewBuilder();
+    if (!config->host.empty()) {
+      hdfsBuilderSetNameNode(builder, config->host.c_str());
+    }
+    hdfsBuilderSetNameNodePort(builder, config->port);
+    if (!config->user.empty()) {
+      hdfsBuilderSetUserName(builder, config->user.c_str());
+    }
+    if (!config->kerb_ticket.empty()) {
+      hdfsBuilderSetKerbTicketCachePath(builder, config->kerb_ticket.c_str());
+    }
+    fs_ = hdfsBuilderConnect(builder);
 
     if (fs_ == nullptr) { return Status::IOError("HDFS connection failed"); }
     namenode_host_ = config->host;
     port_ = config->port;
     user_ = config->user;
+    kerb_ticket_ = config->kerb_ticket;
 
     return Status::OK();
   }
@@ -425,6 +438,7 @@ class HdfsClient::HdfsClientImpl {
   std::string namenode_host_;
   std::string user_;
   int port_;
+  std::string kerb_ticket_;
 
   hdfsFS fs_;
 };
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 4a4e3ec5f5134..48699c914503e 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -60,19 +60,16 @@ struct HdfsConnectionConfig {
   std::string host;
   int port;
   std::string user;
-
-  // TODO: Kerberos, etc.
+  std::string kerb_ticket;
 };
 
 class ARROW_EXPORT HdfsClient : public FileSystemClient {
  public:
   ~HdfsClient();
 
-  // Connect to an HDFS cluster at indicated host, port, and as user
+  // Connect to an HDFS cluster given a configuration
   //
-  // @param host (in)
-  // @param port (in)
-  // @param user (in): user to identify as
+  // @param config (in): configuration for connecting
   // @param fs (out): the created client
   // @returns Status
   static Status Connect(
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
index f256c31b4f4b2..07eb6250bbe55 100644
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -73,9 +73,17 @@ static HINSTANCE libjvm_handle = NULL;
 
 // NOTE(wesm): cpplint does not like use of short and other imprecise C types
 
-static hdfsFS (*ptr_hdfsConnectAsUser)(
-    const char* host, tPort port, const char* user) = NULL;
-static hdfsFS (*ptr_hdfsConnect)(const char* host, tPort port) = NULL;
+static hdfsBuilder* (*ptr_hdfsNewBuilder)(void) = NULL;
+static void (*ptr_hdfsBuilderSetNameNode)(
+    hdfsBuilder* bld, const char* nn) = NULL;
+static void (*ptr_hdfsBuilderSetNameNodePort)(
+    hdfsBuilder* bld, tPort port) = NULL;
+static void (*ptr_hdfsBuilderSetUserName)(
+    hdfsBuilder* bld, const char* userName) = NULL;
+static void (*ptr_hdfsBuilderSetKerbTicketCachePath)(
+    hdfsBuilder* bld, const char* kerbTicketCachePath) = NULL;
+static hdfsFS (*ptr_hdfsBuilderConnect)(hdfsBuilder* bld) = NULL;
+
 static int (*ptr_hdfsDisconnect)(hdfsFS fs) = NULL;
 
 static hdfsFile (*ptr_hdfsOpenFile)(hdfsFS fs, const char* path, int flags,
@@ -149,18 +157,29 @@ static void* get_symbol(const char* symbol) {
 #endif
 }
 
-hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char* user) {
-  return ptr_hdfsConnectAsUser(host, port, user);
+hdfsBuilder* hdfsNewBuilder(void) {
+  return ptr_hdfsNewBuilder();
 }
 
-// Returns NULL on failure
-hdfsFS hdfsConnect(const char* host, tPort port) {
-  if (ptr_hdfsConnect) {
-    return ptr_hdfsConnect(host, port);
-  } else {
-    // TODO: error reporting when shim setup fails
-    return NULL;
-  }
+void hdfsBuilderSetNameNode(hdfsBuilder* bld, const char* nn) {
+  ptr_hdfsBuilderSetNameNode(bld, nn);
+}
+
+void hdfsBuilderSetNameNodePort(hdfsBuilder* bld, tPort port) {
+  ptr_hdfsBuilderSetNameNodePort(bld, port);
+}
+
+void hdfsBuilderSetUserName(hdfsBuilder* bld, const char* userName) {
+  ptr_hdfsBuilderSetUserName(bld, userName);
+}
+
+void hdfsBuilderSetKerbTicketCachePath(hdfsBuilder* bld,
+    const char* kerbTicketCachePath) {
+  ptr_hdfsBuilderSetKerbTicketCachePath(bld , kerbTicketCachePath);
+}
+
+hdfsFS hdfsBuilderConnect(hdfsBuilder* bld) {
+  return ptr_hdfsBuilderConnect(bld);
 }
 
 int hdfsDisconnect(hdfsFS fs) {
@@ -342,18 +361,36 @@ int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
 }
 
 static std::vector<fs::path> get_potential_libhdfs_paths() {
-  std::vector<fs::path> libhdfs_potential_paths = {
-      // find one in the local directory
-      fs::path("./libhdfs.so"), fs::path("./hdfs.dll"),
-      // find a global libhdfs.so
-      fs::path("libhdfs.so"), fs::path("hdfs.dll"),
+  std::vector<fs::path> libhdfs_potential_paths;
+  std::string file_name;
+
+  // OS-specific file name
+#ifdef __WIN32
+  file_name = "hdfs.dll";
+#elif __APPLE__
+  file_name = "libhdfs.dylib";
+#else
+  file_name = "libhdfs.so";
+#endif
+
+  // Common paths
+  std::vector<fs::path> search_paths = {
+      fs::path(""),
+      fs::path(".")
   };
 
+  // Path from environment variable
   const char* hadoop_home = std::getenv("HADOOP_HOME");
   if (hadoop_home != nullptr) {
-    auto path = fs::path(hadoop_home) / "lib/native/libhdfs.so";
-    libhdfs_potential_paths.push_back(path);
+    auto path = fs::path(hadoop_home) / "lib/native";
+    search_paths.push_back(path);
   }
+
+  // All paths with file name
+  for (auto& path : search_paths) {
+    libhdfs_potential_paths.push_back(path / file_name);
+  }
+
   return libhdfs_potential_paths;
 }
 
@@ -371,7 +408,7 @@ static std::vector<fs::path> get_potential_libjvm_paths() {
   file_name = "jvm.dll";
 #elif __APPLE__
   search_prefixes = {""};
-  search_suffixes = {""};
+  search_suffixes = {"", "/jre/lib/server"};
   file_name = "libjvm.dylib";
 
 // SFrame uses /usr/libexec/java_home to find JAVA_HOME; for now we are
@@ -513,8 +550,12 @@ Status ARROW_EXPORT ConnectLibHdfs() {
     return Status::IOError("Prior attempt to load libhdfs failed");
   }
 
-  GET_SYMBOL_REQUIRED(hdfsConnect);
-  GET_SYMBOL_REQUIRED(hdfsConnectAsUser);
+  GET_SYMBOL_REQUIRED(hdfsNewBuilder);
+  GET_SYMBOL_REQUIRED(hdfsBuilderSetNameNode);
+  GET_SYMBOL_REQUIRED(hdfsBuilderSetNameNodePort);
+  GET_SYMBOL_REQUIRED(hdfsBuilderSetUserName);
+  GET_SYMBOL_REQUIRED(hdfsBuilderSetKerbTicketCachePath);
+  GET_SYMBOL_REQUIRED(hdfsBuilderConnect);
   GET_SYMBOL_REQUIRED(hdfsCreateDirectory);
   GET_SYMBOL_REQUIRED(hdfsDelete);
   GET_SYMBOL_REQUIRED(hdfsDisconnect);
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 8074915508fbe..77034159d2f3a 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -93,6 +93,7 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
         c_string host
         int port
         c_string user
+        c_string kerb_ticket
 
     cdef cppclass HdfsPathInfo:
         ObjectType kind;
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 16ebfa1138e46..0e6b81e984431 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -288,9 +288,6 @@ cdef class HdfsClient:
         shared_ptr[CHdfsClient] client
 
     cdef readonly:
-        object host
-        int port
-        object user
         bint is_open
 
     def __cinit__(self):
@@ -301,6 +298,9 @@ cdef class HdfsClient:
             self.close()
 
     def close(self):
+        """
+        Disconnect from the HDFS cluster
+        """
         self._ensure_client()
         with nogil:
             check_status(self.client.get().Disconnect())
@@ -313,14 +313,21 @@ cdef class HdfsClient:
             raise IOError('HDFS client is closed')
 
     @classmethod
-    def connect(cls, host, port, user):
+    def connect(cls, host="default", port=0, user=None, kerb_ticket=None):
         """
+        Connect to an HDFS cluster. All parameters are optional and should
+        only be set if the defaults need to be overridden.
+
+        Authentication should be automatic if the HDFS cluster uses Kerberos.
+        However, if a username is specified, then the ticket cache will likely
+        be required.
 
         Parameters
         ----------
-        host :
-        port :
-        user :
+        host : NameNode. Set to "default" for fs.defaultFS from core-site.xml.
+        port : NameNode's port. Set to 0 for default or logical (HA) nodes.
+        user : Username when connecting to HDFS; None implies login user.
+        kerb_ticket : Path to Kerberos ticket cache.
 
         Notes
         -----
@@ -335,9 +342,13 @@ cdef class HdfsClient:
             HdfsClient out = HdfsClient()
             HdfsConnectionConfig conf
 
-        conf.host = tobytes(host)
+        if host is not None:
+            conf.host = tobytes(host)
         conf.port = port
-        conf.user = tobytes(user)
+        if user is not None:
+            conf.user = tobytes(user)
+        if kerb_ticket is not None:
+            conf.kerb_ticket = tobytes(kerb_ticket)
 
         with nogil:
             check_status(CHdfsClient.Connect(&conf, &out.client))

From da24c1a0a2aba7ccd42cc3cbcf240eeb22d7ffb6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 29 Oct 2016 10:02:15 +0200
Subject: [PATCH 0182/1644] ARROW-339: Python 3 compatibility in
 merge_arrow_pr.py

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #188 from wesm/ARROW-339 and squashes the following commits:

1f3617f [Wes McKinney] Remove cherry-picking cruft
6b99632 [Wes McKinney] Python 3 compatibility in merge_arrow_pr.py
---
 dev/merge_arrow_pr.py | 193 +++++++++++++++++++-----------------------
 1 file changed, 88 insertions(+), 105 deletions(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index 8f47f93b26dd1..aa899edd62ca4 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -17,22 +17,24 @@
 # limitations under the License.
 #
 
-# Utility for creating well-formed pull request merges and pushing them to Apache.
+# Utility for creating well-formed pull request merges and pushing them to
+# Apache.
 #   usage: ./apache-pr-merge.py    (see config env vars below)
 #
 # This utility assumes you already have a local Arrow git clone and that you
 # have added remotes corresponding to both (i) the Github Apache Arrow mirror
 # and (ii) the apache git repo.
 
-import json
 import os
 import re
 import subprocess
 import sys
-import tempfile
-import urllib2
+import requests
 import getpass
 
+from six.moves import input
+import six
+
 try:
     import jira.client
     JIRA_IMPORTED = True
@@ -42,8 +44,8 @@
 # Location of your Arrow git clone
 ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
 PROJECT_NAME = ARROW_HOME.rsplit("/", 1)[1]
-print "ARROW_HOME = " + ARROW_HOME
-print "PROJECT_NAME = " + PROJECT_NAME
+print("ARROW_HOME = " + ARROW_HOME)
+print("PROJECT_NAME = " + PROJECT_NAME)
 
 # Remote name which points to the Gihub site
 PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github")
@@ -65,46 +67,38 @@
 
 
 def get_json(url):
-    try:
-        from urllib2 import urlopen, Request
-        env_var = 'ARROW_GITHUB_API_TOKEN'
-
-        if env_var in os.environ:
-            token = os.environ[env_var]
-            request = Request(url)
-            request.add_header('Authorization', 'token %s' % token)
-            response = urlopen(request)
-        else:
-            response = urlopen(url)
-        return json.load(response)
-    except urllib2.HTTPError as e:
-        print "Unable to fetch URL, exiting: %s" % url
-        sys.exit(-1)
+    req = requests.get(url)
+    return req.json()
 
 
 def fail(msg):
-    print msg
+    print(msg)
     clean_up()
     sys.exit(-1)
 
 
 def run_cmd(cmd):
+    if isinstance(cmd, six.string_types):
+        cmd = cmd.split(' ')
+
     try:
-        if isinstance(cmd, list):
-            return subprocess.check_output(cmd)
-        else:
-            return subprocess.check_output(cmd.split(" "))
+        output = subprocess.check_output(cmd)
     except subprocess.CalledProcessError as e:
         # this avoids hiding the stdout / stderr of failed processes
-        print 'Command failed: %s' % cmd
-        print 'With output:'
-        print '--------------'
-        print e.output
-        print '--------------'
+        print('Command failed: %s' % cmd)
+        print('With output:')
+        print('--------------')
+        print(e.output)
+        print('--------------')
         raise e
 
+    if isinstance(output, six.binary_type):
+        output = output.decode('utf-8')
+    return output
+
+
 def continue_maybe(prompt):
-    result = raw_input("\n%s (y/n): " % prompt)
+    result = input("\n%s (y/n): " % prompt)
     if result.lower() != "y":
         fail("Okay, exiting")
 
@@ -113,38 +107,44 @@ def continue_maybe(prompt):
 
 
 def clean_up():
-    print "Restoring head pointer to %s" % original_head
+    print("Restoring head pointer to %s" % original_head)
     run_cmd("git checkout %s" % original_head)
 
     branches = run_cmd("git branch").replace(" ", "").split("\n")
 
-    for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches):
-        print "Deleting local branch %s" % branch
+    for branch in [x for x in branches if x.startswith(BRANCH_PREFIX)]:
+        print("Deleting local branch %s" % branch)
         run_cmd("git branch -D %s" % branch)
 
 
 # merge the requested PR and return the merge hash
 def merge_pr(pr_num, target_ref):
     pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num)
-    target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper())
-    run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, pr_branch_name))
-    run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, target_branch_name))
+    target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num,
+                                                target_ref.upper())
+    run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num,
+                                              pr_branch_name))
+    run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref,
+                                    target_branch_name))
     run_cmd("git checkout %s" % target_branch_name)
 
     had_conflicts = False
     try:
         run_cmd(['git', 'merge', pr_branch_name, '--squash'])
     except Exception as e:
-        msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e
+        msg = ("Error merging: %s\nWould you like to "
+               "manually fix-up this merge?" % e)
         continue_maybe(msg)
-        msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
+        msg = ("Okay, please fix any conflicts and 'git add' "
+               "conflicting files... Finished?")
         continue_maybe(msg)
         had_conflicts = True
 
     commit_authors = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,
                              '--pretty=format:%an <%ae>']).split("\n")
     distinct_authors = sorted(set(commit_authors),
-                              key=lambda x: commit_authors.count(x), reverse=True)
+                              key=lambda x: commit_authors.count(x),
+                              reverse=True)
     primary_author = distinct_authors[0]
     commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,
                       '--pretty=format:%h [%an] %s']).split("\n\n")
@@ -152,7 +152,7 @@ def merge_pr(pr_num, target_ref):
     merge_message_flags = []
 
     merge_message_flags += ["-m", title]
-    if body != None:
+    if body is not None:
         merge_message_flags += ["-m", body]
 
     authors = "\n".join(["Author: %s" % a for a in distinct_authors])
@@ -162,14 +162,17 @@ def merge_pr(pr_num, target_ref):
     if had_conflicts:
         committer_name = run_cmd("git config --get user.name").strip()
         committer_email = run_cmd("git config --get user.email").strip()
-        message = "This patch had conflicts when merged, resolved by\nCommitter: %s <%s>" % (
-            committer_name, committer_email)
+        message = ("This patch had conflicts when merged, "
+                   "resolved by\nCommitter: %s <%s>" %
+                   (committer_name, committer_email))
         merge_message_flags += ["-m", message]
 
-    # The string "Closes #%s" string is required for GitHub to correctly close the PR
+    # The string "Closes #%s" string is required for GitHub to correctly close
+    # the PR
     merge_message_flags += [
         "-m",
-        "Closes #%s from %s and squashes the following commits:" % (pr_num, pr_repo_desc)]
+        "Closes #%s from %s and squashes the following commits:"
+        % (pr_num, pr_repo_desc)]
     for c in commits:
         merge_message_flags += ["-m", c]
 
@@ -182,7 +185,8 @@ def merge_pr(pr_num, target_ref):
         target_branch_name, PUSH_REMOTE_NAME))
 
     try:
-        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, target_ref))
+        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name,
+                                       target_ref))
     except Exception as e:
         clean_up()
         fail("Exception while pushing: %s" % e)
@@ -194,65 +198,42 @@ def merge_pr(pr_num, target_ref):
     return merge_hash
 
 
-def cherry_pick(pr_num, merge_hash, default_branch):
-    pick_ref = raw_input("Enter a branch name [%s]: " % default_branch)
-    if pick_ref == "":
-        pick_ref = default_branch
-
-    pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, pick_ref.upper())
-
-    run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))
-    run_cmd("git checkout %s" % pick_branch_name)
-    run_cmd("git cherry-pick -sx %s" % merge_hash)
-
-    continue_maybe("Pick complete (local ref %s). Push to %s?" % (
-        pick_branch_name, PUSH_REMOTE_NAME))
-
-    try:
-        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref))
-    except Exception as e:
-        clean_up()
-        fail("Exception while pushing: %s" % e)
-
-    pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8]
-    clean_up()
-
-    print("Pull request #%s picked into %s!" % (pr_num, pick_ref))
-    print("Pick hash: %s" % pick_hash)
-    return pick_ref
-
-
 def fix_version_from_branch(branch, versions):
-    # Note: Assumes this is a sorted (newest->oldest) list of un-released versions
+    # Note: Assumes this is a sorted (newest->oldest) list of un-released
+    # versions
     if branch == "master":
         return versions[0]
     else:
         branch_ver = branch.replace("branch-", "")
-        return filter(lambda x: x.name.startswith(branch_ver), versions)[-1]
+        return [x for x in versions if x.name.startswith(branch_ver)][-1]
+
 
 def exctract_jira_id(title):
     m = re.search(r'^(ARROW-[0-9]+)\b.*$', title)
     if m and m.groups > 0:
         return m.group(1)
     else:
-        fail("PR title should be prefixed by a jira id \"ARROW-XXX: ...\", found: \"%s\"" % title)
+        fail("PR title should be prefixed by a jira id "
+             "\"ARROW-XXX: ...\", found: \"%s\"" % title)
+
 
 def check_jira(title):
     jira_id = exctract_jira_id(title)
     asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
                                 basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
     try:
-        issue = asf_jira.issue(jira_id)
+        asf_jira.issue(jira_id)
     except Exception as e:
         fail("ASF JIRA could not find %s\n%s" % (jira_id, e))
 
+
 def resolve_jira(title, merge_branches, comment):
     asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
                                 basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
 
     default_jira_id = exctract_jira_id(title)
 
-    jira_id = raw_input("Enter a JIRA id [%s]: " % default_jira_id)
+    jira_id = input("Enter a JIRA id [%s]: " % default_jira_id)
     if jira_id == "":
         jira_id = default_jira_id
 
@@ -271,30 +252,33 @@ def resolve_jira(title, merge_branches, comment):
 
     if cur_status == "Resolved" or cur_status == "Closed":
         fail("JIRA issue %s already has status '%s'" % (jira_id, cur_status))
-    print ("=== JIRA %s ===" % jira_id)
-    print ("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n" % (
-        cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))
+    print("=== JIRA %s ===" % jira_id)
+    print("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%sf\n"
+          % (cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))
 
     resolve = filter(lambda a: a['name'] == "Resolve Issue",
                      asf_jira.transitions(jira_id))[0]
     asf_jira.transition_issue(jira_id, resolve["id"], comment=comment)
 
-    print "Succesfully resolved %s!" % (jira_id)
+    print("Succesfully resolved %s!" % (jira_id))
 
 
 if not JIRA_USERNAME:
-    JIRA_USERNAME =  raw_input("Env JIRA_USERNAME not set, please enter your JIRA username:")
+    JIRA_USERNAME = input("Env JIRA_USERNAME not set, "
+                          "please enter your JIRA username:")
 
 if not JIRA_PASSWORD:
-    JIRA_PASSWORD =  getpass.getpass("Env JIRA_PASSWORD not set, please enter your JIRA password:")
+    JIRA_PASSWORD = getpass.getpass("Env JIRA_PASSWORD not set, please enter "
+                                    "your JIRA password:")
 
 branches = get_json("%s/branches" % GITHUB_API_BASE)
-branch_names = filter(lambda x: x.startswith("branch-"), [x['name'] for x in branches])
+branch_names = [x['name'] for x in branches if x['name'].startswith('branch-')]
+
 # Assumes branch names can be sorted lexicographically
 # Julien: I commented this out as we don't have any "branch-*" branch yet
-#latest_branch = sorted(branch_names, reverse=True)[0]
+# latest_branch = sorted(branch_names, reverse=True)[0]
 
-pr_num = raw_input("Which pull request would you like to merge? (e.g. 34): ")
+pr_num = input("Which pull request would you like to merge? (e.g. 34): ")
 pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num))
 
 url = pr["url"]
@@ -307,42 +291,41 @@ def resolve_jira(title, merge_branches, comment):
 pr_repo_desc = "%s/%s" % (user_login, base_ref)
 
 if pr["merged"] is True:
-    print "Pull request %s has already been merged, assuming you want to backport" % pr_num
+    print("Pull request %s has already been merged, "
+          "assuming you want to backport" % pr_num)
     merge_commit_desc = run_cmd([
         'git', 'log', '--merges', '--first-parent',
         '--grep=pull request #%s' % pr_num, '--oneline']).split("\n")[0]
     if merge_commit_desc == "":
-        fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num)
+        fail("Couldn't find any merge commit for #%s, "
+             "you may need to update HEAD." % pr_num)
 
     merge_hash = merge_commit_desc[:7]
     message = merge_commit_desc[8:]
 
-    print "Found: %s" % message
-    maybe_cherry_pick(pr_num, merge_hash, latest_branch)
+    print("Found: %s" % message)
     sys.exit(0)
 
 if not bool(pr["mergeable"]):
-    msg = "Pull request %s is not mergeable in its current form.\n" % pr_num + \
-        "Continue? (experts only!)"
+    msg = ("Pull request %s is not mergeable in its current form.\n"
+           % pr_num + "Continue? (experts only!)")
     continue_maybe(msg)
 
-print ("\n=== Pull Request #%s ===" % pr_num)
-print ("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" % (
-    title, pr_repo_desc, target_ref, url))
+print("\n=== Pull Request #%s ===" % pr_num)
+print("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s"
+      % (title, pr_repo_desc, target_ref, url))
 continue_maybe("Proceed with merging pull request #%s?" % pr_num)
 
 merged_refs = [target_ref]
 
 merge_hash = merge_pr(pr_num, target_ref)
 
-pick_prompt = "Would you like to pick %s into another branch?" % merge_hash
-while raw_input("\n%s (y/n): " % pick_prompt).lower() == "y":
-    merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)]
-
 if JIRA_IMPORTED:
     continue_maybe("Would you like to update the associated JIRA?")
-    jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % (pr_num, GITHUB_BASE, pr_num)
+    jira_comment = ("Issue resolved by pull request %s\n[%s/%s]"
+                    % (pr_num, GITHUB_BASE, pr_num))
     resolve_jira(title, merged_refs, jira_comment)
 else:
-    print "Could not find jira-python library. Run 'sudo pip install jira-python' to install."
-    print "Exiting without trying to close the associated JIRA."
+    print("Could not find jira-python library. "
+          "Run 'sudo pip install jira-python' to install.")
+    print("Exiting without trying to close the associated JIRA.")

From d946e7917d55cb220becd6469ae93430f2e60764 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 29 Oct 2016 04:36:03 -0400
Subject: [PATCH 0183/1644] ARROW-354: Fix comparison of arrays of empty
 strings

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #189 from xhochy/ARROW-354 and squashes the following commits:

8f75d78 [Uwe L. Korn] ARROW-354: Fix comparison of arrays of empty strings
---
 cpp/src/arrow/types/string-test.cc | 12 ++++++++++++
 cpp/src/arrow/types/string.cc      |  2 ++
 2 files changed, 14 insertions(+)

diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index d897e30a3c6a2..af87a14a8b32e 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -129,6 +129,18 @@ TEST_F(TestStringContainer, TestGetString) {
   }
 }
 
+TEST_F(TestStringContainer, TestEmptyStringComparison) {
+  offsets_ = {0, 0, 0, 0, 0, 0};
+  offsets_buf_ = test::to_buffer(offsets_);
+  length_ = offsets_.size() - 1;
+
+  auto strings_a = std::make_shared<StringArray>(
+      length_, offsets_buf_, nullptr, null_count_, null_bitmap_);
+  auto strings_b = std::make_shared<StringArray>(
+      length_, offsets_buf_, nullptr, null_count_, null_bitmap_);
+  ASSERT_TRUE(strings_a->Equals(strings_b));
+}
+
 // ----------------------------------------------------------------------
 // String builder tests
 
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index d692e13773f56..f6d26df3167c9 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -56,6 +56,8 @@ bool BinaryArray::EqualsExact(const BinaryArray& other) const {
       offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
   if (!equal_offsets) { return false; }
 
+  if (!data_buffer_ && !(other.data_buffer_)) { return true; }
+
   return data_buffer_->Equals(*other.data_buffer_, data_buffer_->size());
 }
 

From 772bc6ea6e5d452ccff1df8d5e83299e434c0d04 Mon Sep 17 00:00:00 2001
From: Peter Hoffmann <ph@peter-hoffmann.com>
Date: Sun, 30 Oct 2016 11:11:28 +0100
Subject: [PATCH 0184/1644] ARROW-349: Add six as a requirement

fixes https://issues.apache.org/jira/browse/ARROW-349

Author: Peter Hoffmann <ph@peter-hoffmann.com>

Closes #184 from hoffmann/patch-1 and squashes the following commits:

1bffc69 [Peter Hoffmann] Add six as a requirement
---
 python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index 990497775148d..cdfdc243e2597 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -271,7 +271,7 @@ def get_outputs(self):
         'clean': clean,
         'build_ext': build_ext
     },
-    install_requires=['cython >= 0.23', 'numpy >= 1.9'],
+    install_requires=['cython >= 0.23', 'numpy >= 1.9', 'six >= 1.0.0'],
     description=DESC,
     license='Apache License, Version 2.0',
     maintainer="Apache Arrow Developers",

From ca088dd19eb4283c71252de39782d811f985649a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 31 Oct 2016 21:16:29 -0400
Subject: [PATCH 0185/1644] ARROW-339: [Dev] Lingering Python 3 fixes

I missed a couple Python 3 things. I'll leave this open until one of us successfully merged another patch with this before we merge it.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #191 from wesm/ARROW-339-2 and squashes the following commits:

78bf094 [Wes McKinney] Lingering Python 3 fixes
---
 dev/merge_arrow_pr.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index aa899edd62ca4..f7e7a37c36e5c 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -210,7 +210,7 @@ def fix_version_from_branch(branch, versions):
 
 def exctract_jira_id(title):
     m = re.search(r'^(ARROW-[0-9]+)\b.*$', title)
-    if m and m.groups > 0:
+    if m:
         return m.group(1)
     else:
         fail("PR title should be prefixed by a jira id "
@@ -256,8 +256,8 @@ def resolve_jira(title, merge_branches, comment):
     print("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%sf\n"
           % (cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))
 
-    resolve = filter(lambda a: a['name'] == "Resolve Issue",
-                     asf_jira.transitions(jira_id))[0]
+    resolve = [x for x in asf_jira.transitions(jira_id)
+               if x['name'] == "Resolve Issue"][0]
     asf_jira.transition_issue(jira_id, resolve["id"], comment=comment)
 
     print("Succesfully resolved %s!" % (jira_id))

From d4148759a266d90dacd1ca2b7b7ff0df7e02578a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 1 Nov 2016 14:21:07 -0400
Subject: [PATCH 0186/1644] ARROW-348: [Python] Add build-type command line
 option to setup.py, build CMake extensions in a build type subdirectory

This also resolves ARROW-230.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #187 from wesm/ARROW-348 and squashes the following commits:

3cdaeaf [Wes McKinney] Cast build_type to lowercase in case env variable is uppercase
74bfa71 [Wes McKinney] Pull default build type from environment variable
d0b3154 [Wes McKinney] Tweak readme
6017948 [Wes McKinney] Add built-type command line option to setup.py, build extensions in release type subdirectory to avoid conflicts with setuptools
---
 python/CMakeLists.txt |  3 +--
 python/README.md      |  9 +++++++++
 python/setup.py       | 34 ++++++++++++++++------------------
 3 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index b8be8665af079..179f02fbc9daa 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -203,8 +203,7 @@ if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
 EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
   ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
 else()
-  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
-  # set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
+  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
 endif()
 
 # where to put generated archives (.a files)
diff --git a/python/README.md b/python/README.md
index e11f64564558c..2a3e1ba9542f5 100644
--- a/python/README.md
+++ b/python/README.md
@@ -48,6 +48,15 @@ python setup.py build_ext --inplace
 py.test pyarrow
 ```
 
+To change the build type, use the `--build-type` option:
+
+```bash
+python setup.py build_ext --build-type=release --inplace
+```
+
+To pass through other build options to CMake, set the environment variable
+`$PYARROW_CMAKE_OPTIONS`.
+
 #### Build the documentation
 
 ```bash
diff --git a/python/setup.py b/python/setup.py
index cdfdc243e2597..b3012e694243a 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -39,14 +39,6 @@
 # Check if we're running 64-bit Python
 is_64_bit = sys.maxsize > 2**32
 
-# Check if this is a debug build of Python.
-# if hasattr(sys, 'gettotalrefcount'):
-#     build_type = 'Debug'
-# else:
-#     build_type = 'Release'
-
-build_type = 'Debug'
-
 if Cython.__version__ < '0.19.1':
     raise Exception('Please upgrade to Cython 0.19.1 or newer')
 
@@ -104,13 +96,14 @@ def run(self):
     # github.com/libdynd/dynd-python
 
     description = "Build the C-extensions for arrow"
-    user_options = ([('extra-cmake-args=', None,
-                      'extra arguments for CMake')] +
-                    _build_ext.user_options)
+    user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
+                     ('build-type=', None, 'build type (debug or release)')]
+                    + _build_ext.user_options)
 
     def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
+        self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
 
     CYTHON_MODULE_NAMES = [
         'array',
@@ -152,9 +145,12 @@ def _run_cmake(self):
         static_lib_option = ''
         build_tests_option = ''
 
+        build_type_option = '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type)
+
         if sys.platform != 'win32':
             cmake_command = ['cmake', self.extra_cmake_args, pyexe_option,
                              build_tests_option,
+                             build_type_option,
                              static_lib_option, source]
 
             self.spawn(cmake_command)
@@ -170,7 +166,8 @@ def _run_cmake(self):
             # Generate the build files
             extra_cmake_args = shlex.split(self.extra_cmake_args)
             cmake_command = (['cmake'] + extra_cmake_args +
-                             [source, pyexe_option,
+                             [source,
+                              pyexe_option,
                               static_lib_option,
                               build_tests_option,
                              '-G', cmake_generator])
@@ -179,7 +176,7 @@ def _run_cmake(self):
 
             self.spawn(cmake_command)
             # Do the build
-            self.spawn(['cmake', '--build', '.', '--config', build_type])
+            self.spawn(['cmake', '--build', '.', '--config', self.build_type])
 
         if self.inplace:
             # a bit hacky
@@ -188,14 +185,15 @@ def _run_cmake(self):
         # Move the built libpyarrow library to the place expected by the Python
         # build
         if sys.platform != 'win32':
-            name, = glob.glob('libpyarrow.*')
+            name, = glob.glob(pjoin(self.build_type, 'libpyarrow.*'))
             try:
                 os.makedirs(pjoin(build_lib, 'pyarrow'))
             except OSError:
                 pass
-            shutil.move(name, pjoin(build_lib, 'pyarrow', name))
+            shutil.move(name,
+                        pjoin(build_lib, 'pyarrow', os.path.split(name)[1]))
         else:
-            shutil.move(pjoin(build_type, 'pyarrow.dll'),
+            shutil.move(pjoin(self.build_type, 'pyarrow.dll'),
                         pjoin(build_lib, 'pyarrow', 'pyarrow.dll'))
 
         # Move the built C-extension to the place expected by the Python build
@@ -239,10 +237,10 @@ def get_ext_built(self, name):
         if sys.platform == 'win32':
             head, tail = os.path.split(name)
             suffix = sysconfig.get_config_var('SO')
-            return pjoin(head, build_type, tail + suffix)
+            return pjoin(head, self.build_type, tail + suffix)
         else:
             suffix = sysconfig.get_config_var('SO')
-            return name + suffix
+            return pjoin(self.build_type, name + suffix)
 
     def get_names(self):
         return self._found_names

From c7db80e729c4b3e984c3ef5630ccbff43f3042b8 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 1 Nov 2016 14:25:01 -0400
Subject: [PATCH 0187/1644] ARROW-355: Add tests for serialising arrays of
 empty strings to Parquet

Depends on https://issues.apache.org/jira/browse/PARQUET-759

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #190 from xhochy/ARROW-355 and squashes the following commits:

e5099ce [Uwe L. Korn] ARROW-355: Add tests for serialising arrays of empty strings to Parquet
---
 python/pyarrow/tests/test_parquet.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 0f9f2e40813ce..922ad3aa9ff73 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -73,7 +73,8 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
         'datetime': np.arange("2016-01-01T00:00:00.001", size,
                               dtype='datetime64[ms]'),
         'str': [str(x) for x in range(size)],
-        'str_with_nulls': [None] + [str(x) for x in range(size - 2)] + [None]
+        'str_with_nulls': [None] + [str(x) for x in range(size - 2)] + [None],
+        'empty_str': [''] * size
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = A.from_pandas_dataframe(df, timestamps_to_ms=True)
@@ -98,7 +99,10 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
         'int64': np.arange(size, dtype=np.int64),
         'float32': np.arange(size, dtype=np.float32),
         'float64': np.arange(size, dtype=np.float64),
-        'bool': np.random.randn(size) > 0
+        'bool': np.random.randn(size) > 0,
+        'str': [str(x) for x in range(size)],
+        'str_with_nulls': [None] + [str(x) for x in range(size - 2)] + [None],
+        'empty_str': [''] * size
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = A.from_pandas_dataframe(df)

From e70d97dbc8dc86161083e94c45d5828f79211f6b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 2 Nov 2016 08:06:29 +0100
Subject: [PATCH 0188/1644] ARROW-358: Add explicit environment variable to
 locate libhdfs in one's environment

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #195 from wesm/ARROW-358 and squashes the following commits:

c00d251 [Wes McKinney] Add explicit environment variable to locate libhdfs in one's environment
---
 cpp/src/arrow/io/libhdfs_shim.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
index 07eb6250bbe55..1fee595d0718b 100644
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -386,6 +386,11 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
     search_paths.push_back(path);
   }
 
+  const char* libhdfs_dir = std::getenv("ARROW_LIBHDFS_DIR");
+  if (libhdfs_dir != nullptr) {
+    search_paths.push_back(fs::path(libhdfs_dir));
+  }
+
   // All paths with file name
   for (auto& path : search_paths) {
     libhdfs_potential_paths.push_back(path / file_name);

From 2a059bd277c58bca80412cbda258a253b801d1a4 Mon Sep 17 00:00:00 2001
From: "Christopher C. Aycock" <christopher.aycock@twosigma.com>
Date: Wed, 2 Nov 2016 12:15:53 -0400
Subject: [PATCH 0189/1644] ARROW-359: Document ARROW_LIBHDFS_DIR

Author: Christopher C. Aycock <christopher.aycock@twosigma.com>

Closes #196 from chrisaycock/ARROW-359 and squashes the following commits:

52ec78e [Christopher C. Aycock] ARROW-359: Document ARROW_LIBHDFS_DIR
---
 cpp/doc/HDFS.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cpp/doc/HDFS.md b/cpp/doc/HDFS.md
index 6b1bb8c452461..180d31e54d573 100644
--- a/cpp/doc/HDFS.md
+++ b/cpp/doc/HDFS.md
@@ -33,16 +33,18 @@ interface to the Java Hadoop client. This library is loaded **at runtime**
 (rather than at link / library load time, since the library may not be in your
 LD_LIBRARY_PATH), and relies on some environment variables.
 
-* `HADOOP_HOME`: the root of your installed Hadoop distribution. Check in the
-  `lib/native` directory to look for `libhdfs.so` if you have any questions
-  about which directory you're after.
-* `JAVA_HOME`: the location of your Java SDK installation
+* `HADOOP_HOME`: the root of your installed Hadoop distribution. Often has
+`lib/native/libhdfs.so`.
+* `JAVA_HOME`: the location of your Java SDK installation.
 * `CLASSPATH`: must contain the Hadoop jars. You can set these using:
 
 ```shell
 export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
 ```
 
+* `ARROW_LIBHDFS_DIR` (optional): explicit location of `libhdfs.so` if it is
+installed somewhere other than `$HADOOP_HOME/lib/native`.
+
 ### Mac Specifics
 
 The installed location of Java on OS X can vary, however the following snippet

From 17c9ae7c4ceb328c897fb6c9025c763a879ebefa Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 2 Nov 2016 12:20:15 -0400
Subject: [PATCH 0190/1644] ARROW-357: Use a single RowGroup for Parquet files
 as default.

This is not the optimal choice, we should rather have an option to optimise for the underlying block size of the filesystem but without the infrastructure for that in ``parquet-cpp``, writing a single RowGroup is the much better choice.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #192 from xhochy/ARROW-357 and squashes the following commits:

9eccefd [Uwe L. Korn] ARROW-357: Use a single RowGroup for Parquet files as default.
---
 python/pyarrow/parquet.pyx | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 019dd2c1de489..a56c1e1456d17 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -106,7 +106,8 @@ def write_table(table, filename, chunk_size=None, version=None,
     table : pyarrow.Table
     filename : string
     chunk_size : int
-        The maximum number of rows in each Parquet RowGroup
+        The maximum number of rows in each Parquet RowGroup. As a default,
+        we will write a single RowGroup per file.
     version : {"1.0", "2.0"}, default "1.0"
         The Parquet format version, defaults to 1.0
     use_dictionary : bool or list
@@ -121,7 +122,7 @@ def write_table(table, filename, chunk_size=None, version=None,
     cdef WriterProperties.Builder properties_builder
     cdef int64_t chunk_size_ = 0
     if chunk_size is None:
-        chunk_size_ = min(ctable_.num_rows(), int(2**16))
+        chunk_size_ = ctable_.num_rows()
     else:
         chunk_size_ = chunk_size
 

From 25e010607542aa7330bd881e145180fe606776c5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 3 Nov 2016 13:22:19 -0400
Subject: [PATCH 0191/1644] ARROW-323: [Python] Opt-in to pyarrow.parquet
 extension rather than attempting and failing silently

Added a couple ways to do this, either via the `--with-parquet` command line option (preferred) or by passing through an option to CMake

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #194 from wesm/ARROW-323 and squashes the following commits:

07c05cc [Wes McKinney] Update readme to illustrate proper use of  with build_ext
3bd9a8d [Wes McKinney] Add --with-parquet option to setup.py
374e254 [Wes McKinney] Add to README about building the parquet extension
cab55cb [Wes McKinney] Opt in to building the pyarrow.parquet extension, do not silently fail
---
 python/CMakeLists.txt |  8 +++++++-
 python/README.md      | 20 +++++++++++++++++++-
 python/setup.py       | 38 ++++++++++++++++++++++++--------------
 3 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 179f02fbc9daa..6ad55f8c9a7b8 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -50,6 +50,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUILD_TESTS
     "Build the PyArrow C++ googletest unit tests"
     OFF)
+  option(PYARROW_BUILD_PARQUET
+    "Build the PyArrow Parquet integration"
+    OFF)
 endif()
 
 find_program(CCACHE_FOUND ccache)
@@ -445,7 +448,10 @@ set(LINK_LIBS
   arrow_ipc
 )
 
-if(PARQUET_FOUND AND PARQUET_ARROW_FOUND)
+if (PYARROW_BUILD_PARQUET)
+  if(NOT (PARQUET_FOUND AND PARQUET_ARROW_FOUND))
+    message(FATAL_ERROR "Unable to locate Parquet libraries")
+  endif()
   ADD_THIRDPARTY_LIB(parquet_arrow
     SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
   set(LINK_LIBS
diff --git a/python/README.md b/python/README.md
index 2a3e1ba9542f5..4fce0d26b2850 100644
--- a/python/README.md
+++ b/python/README.md
@@ -48,7 +48,8 @@ python setup.py build_ext --inplace
 py.test pyarrow
 ```
 
-To change the build type, use the `--build-type` option:
+To change the build type, use the `--build-type` option or set
+`$PYARROW_BUILD_TYPE`:
 
 ```bash
 python setup.py build_ext --build-type=release --inplace
@@ -57,9 +58,26 @@ python setup.py build_ext --build-type=release --inplace
 To pass through other build options to CMake, set the environment variable
 `$PYARROW_CMAKE_OPTIONS`.
 
+#### Build the pyarrow Parquet file extension
+
+To build the integration with [parquet-cpp][1], pass `--with-parquet` to
+the `build_ext` option in setup.py:
+
+```
+python setup.py build_ext --with-parquet install
+```
+
+Alternately, add `-DPYARROW_BUILD_PARQUET=on` to the general CMake options.
+
+```
+export PYARROW_CMAKE_OPTIONS=-DPYARROW_BUILD_PARQUET=on
+```
+
 #### Build the documentation
 
 ```bash
 pip install -r doc/requirements.txt
 python setup.py build_sphinx
 ```
+
+[1]: https://github.com/apache/parquet-cpp
\ No newline at end of file
diff --git a/python/setup.py b/python/setup.py
index b3012e694243a..341cc64aa2cc8 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -97,13 +97,15 @@ def run(self):
 
     description = "Build the C-extensions for arrow"
     user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
-                     ('build-type=', None, 'build type (debug or release)')]
-                    + _build_ext.user_options)
+                     ('build-type=', None, 'build type (debug or release)'),
+                     ('with-parquet', None, 'build the Parquet extension')] +
+                    _build_ext.user_options)
 
     def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
+        self.with_parquet = False
 
     CYTHON_MODULE_NAMES = [
         'array',
@@ -116,8 +118,6 @@ def initialize_options(self):
         'schema',
         'table']
 
-    CYTHON_ALLOWED_FAILURES = ['parquet']
-
     def _run_cmake(self):
         # The directory containing this setup.py
         source = osp.dirname(osp.abspath(__file__))
@@ -141,17 +141,24 @@ def _run_cmake(self):
             if (cachedir != build_temp):
                 return
 
-        pyexe_option = '-DPYTHON_EXECUTABLE=%s' % sys.executable
         static_lib_option = ''
         build_tests_option = ''
 
-        build_type_option = '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type)
+        cmake_options = [
+            '-DPYTHON_EXECUTABLE=%s' % sys.executable,
+            static_lib_option,
+            build_tests_option,
+        ]
+
+        if self.with_parquet:
+            cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
 
         if sys.platform != 'win32':
-            cmake_command = ['cmake', self.extra_cmake_args, pyexe_option,
-                             build_tests_option,
-                             build_type_option,
-                             static_lib_option, source]
+            cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
+                                 .format(self.build_type))
+
+            cmake_command = (['cmake', self.extra_cmake_args] +
+                             cmake_options + [source])
 
             self.spawn(cmake_command)
             args = ['make', 'VERBOSE=1']
@@ -166,10 +173,8 @@ def _run_cmake(self):
             # Generate the build files
             extra_cmake_args = shlex.split(self.extra_cmake_args)
             cmake_command = (['cmake'] + extra_cmake_args +
+                             cmake_options +
                              [source,
-                              pyexe_option,
-                              static_lib_option,
-                              build_tests_option,
                              '-G', cmake_generator])
             if "-G" in self.extra_cmake_args:
                 cmake_command = cmake_command[:-2]
@@ -202,7 +207,7 @@ def _run_cmake(self):
             built_path = self.get_ext_built(name)
             if not os.path.exists(built_path):
                 print(built_path)
-                if name in self.CYTHON_ALLOWED_FAILURES:
+                if self._failure_permitted(name):
                     print('Cython module {0} failure permitted'.format(name))
                     continue
                 raise RuntimeError('libpyarrow C-extension failed to build:',
@@ -219,6 +224,11 @@ def _run_cmake(self):
 
         os.chdir(saved_cwd)
 
+    def _failure_permitted(self, name):
+        if name == 'parquet' and not self.with_parquet:
+            return True
+        return False
+
     def _get_inplace_dir(self):
         pass
 

From e8bc1fe3ba7f94b39f38571a435f93f387e67d37 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Sun, 6 Nov 2016 12:10:06 +0100
Subject: [PATCH 0192/1644] ARROW-368:  Added note for LD_LIBRARY_PATH in
 Python README

Added note to use LD_LIBRARY_PATH env var to add $ARROW_HOME/lib path so PyArrow can locate Arrow-Cpp shared libs.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #199 from BryanCutler/pyarrow-README-note-LD_LIBRARY_PATH-ARROW-368 and squashes the following commits:

15861c4 [Bryan Cutler] Added note for LD_LIBRARY_PATH in Python README
---
 python/README.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/python/README.md b/python/README.md
index 4fce0d26b2850..88ab17e71730f 100644
--- a/python/README.md
+++ b/python/README.md
@@ -33,12 +33,19 @@ These are the various projects that PyArrow depends on.
 1. **g++ and gcc Version >= 4.8**
 2. **cmake > 2.8.6**
 3. **boost**
-4. **Arrow-cpp and its dependencies***
+4. **Arrow-cpp and its dependencies**
 
 The Arrow C++ library must be built with all options enabled and installed with
 ``ARROW_HOME`` environment variable set to the installation location. Look at
 (https://github.com/apache/arrow/blob/master/cpp/README.md) for instructions.
 
+Ensure PyArrow can locate the Arrow-cpp shared libraries by setting the
+LD_LIBRARY_PATH environment variable.
+
+```bash
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_HOME/lib
+```
+
 5. **Python dependencies: numpy, pandas, cython, pytest**
 
 #### Build pyarrow and run the unit tests

From 121e82682344b04bdb26edf16344a9fb2cee240c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 6 Nov 2016 16:08:44 -0500
Subject: [PATCH 0193/1644] ARROW-361: Python: Support reading a
 column-selection from Parquet files

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #197 from xhochy/ARROW-361 and squashes the following commits:

c1fb939 [Uwe L. Korn] Cache column indices
0c32213 [Uwe L. Korn] ARROW-361: Python: Support reading a column-selection from Parquet files
---
 python/pyarrow/includes/parquet.pxd  | 25 ++++++++++---
 python/pyarrow/parquet.pyx           | 53 +++++++++++++++++++++++++++-
 python/pyarrow/tests/test_parquet.py | 16 +++++++++
 3 files changed, 89 insertions(+), 5 deletions(-)

diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index 754eeccecc8e9..57c35ba89445b 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -18,7 +18,7 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport CSchema, CStatus, CTable, MemoryPool
+from pyarrow.includes.libarrow cimport CArray, CSchema, CStatus, CTable, MemoryPool
 from pyarrow.includes.libarrow_io cimport ReadableFileInterface
 
 
@@ -32,6 +32,9 @@ cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
   cdef cppclass PrimitiveNode(Node):
     pass
 
+  cdef cppclass ColumnPath:
+    c_string ToDotString()
+
 cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
   enum ParquetVersion" parquet::ParquetVersion::type":
       PARQUET_1_0" parquet::ParquetVersion::PARQUET_1_0"
@@ -44,13 +47,14 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
       LZO" parquet::Compression::LZO"
       BROTLI" parquet::Compression::BROTLI"
 
+  cdef cppclass ColumnDescriptor:
+    shared_ptr[ColumnPath] path()
+
   cdef cppclass SchemaDescriptor:
+    const ColumnDescriptor* Column(int i)
     shared_ptr[Node] schema()
     GroupNode* group()
 
-  cdef cppclass ColumnDescriptor:
-    pass
-
 
 cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
     cdef cppclass ColumnReader:
@@ -80,10 +84,21 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
     cdef cppclass RowGroupReader:
         pass
 
+    cdef cppclass FileMetaData:
+        uint32_t size()
+        int num_columns()
+        int64_t num_rows()
+        int num_row_groups()
+        int32_t version()
+        const c_string created_by()
+        int num_schema_elements()
+        const SchemaDescriptor* schema()
+
     cdef cppclass ParquetFileReader:
         # TODO: Some default arguments are missing
         @staticmethod
         unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
+        const FileMetaData* metadata();
 
 
 cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
@@ -124,7 +139,9 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 
     cdef cppclass FileReader:
         FileReader(MemoryPool* pool, unique_ptr[ParquetFileReader] reader)
+        CStatus ReadFlatColumn(int i, shared_ptr[CArray]* out);
         CStatus ReadFlatTable(shared_ptr[CTable]* out);
+        const ParquetFileReader* parquet_reader();
 
 
 cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index a56c1e1456d17..2152f89474195 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -24,6 +24,7 @@ from pyarrow.includes.parquet cimport *
 from pyarrow.includes.libarrow_io cimport ReadableFileInterface
 cimport pyarrow.includes.pyarrow as pyarrow
 
+from pyarrow.array cimport Array
 from pyarrow.compat import tobytes
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
@@ -43,6 +44,7 @@ cdef class ParquetReader:
     cdef:
         ParquetAllocator allocator
         unique_ptr[FileReader] reader
+        column_idx_map
 
     def __cinit__(self):
         self.allocator.set_pool(default_memory_pool())
@@ -76,11 +78,55 @@ cdef class ParquetReader:
         table.init(ctable)
         return table
 
+    def column_name_idx(self, column_name):
+        """
+        Find the matching index of a column in the schema.
+
+        Parameter
+        ---------
+        column_name: str
+            Name of the column, separation of nesting levels is done via ".".
+
+        Returns
+        -------
+        column_idx: int
+            Integer index of the position of the column
+        """
+        cdef:
+            const FileMetaData* metadata = self.reader.get().parquet_reader().metadata()
+            int i = 0
+
+        if self.column_idx_map is None:
+            self.column_idx_map = {}
+            for i in range(0, metadata.num_columns()):
+                self.column_idx_map[str(metadata.schema().Column(i).path().get().ToDotString())] = i
+
+        return self.column_idx_map[column_name]
+
+    def read_column(self, int column_index):
+        cdef:
+            Array array = Array()
+            shared_ptr[CArray] carray
+
+        with nogil:
+            check_status(self.reader.get().ReadFlatColumn(column_index, &carray))
+
+        array.init(carray)
+        return array
+
 
 def read_table(source, columns=None):
     """
     Read a Table from Parquet format
 
+    Parameters
+    ----------
+    source: str or pyarrow.io.NativeFile
+        Readable source. For passing Python file objects or byte buffers, see
+        pyarrow.io.PythonFileInterface or pyarrow.io.BytesReader.
+    columns: list
+        If not None, only these columns will be read from the file.
+
     Returns
     -------
     pyarrow.table.Table
@@ -93,7 +139,12 @@ def read_table(source, columns=None):
     elif isinstance(source, NativeFile):
         reader.open_native_file(source)
 
-    return reader.read_all()
+    if columns is None:
+        return reader.read_all()
+    else:
+        column_idxs = [reader.column_name_idx(column) for column in columns]
+        arrays = [reader.read_column(column_idx) for column_idx in column_idxs]
+        return Table.from_arrays(columns, arrays)
 
 
 def write_table(table, filename, chunk_size=None, version=None,
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 922ad3aa9ff73..c1d44ce0d4230 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -115,6 +115,22 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
 
     pdt.assert_frame_equal(df, df_read)
 
+@parquet
+def test_pandas_column_selection(tmpdir):
+    size = 10000
+    np.random.seed(0)
+    df = pd.DataFrame({
+        'uint8': np.arange(size, dtype=np.uint8),
+        'uint16': np.arange(size, dtype=np.uint16)
+    })
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = A.from_pandas_dataframe(df)
+    A.parquet.write_table(arrow_table, filename.strpath)
+    table_read = pq.read_table(filename.strpath, columns=['uint8'])
+    df_read = table_read.to_pandas()
+
+    pdt.assert_frame_equal(df[['uint8']], df_read)
+
 @parquet
 def test_pandas_parquet_configuration_options(tmpdir):
     size = 10000

From 79344b335849c2eb43954b0751018051814019d6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 7 Nov 2016 13:52:32 -0500
Subject: [PATCH 0194/1644] ARROW-362: Fix memory leak in zero-copy arrow to
 NumPy/pandas conversion

close #198

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #200 from wesm/ARROW-362 and squashes the following commits:

99df96b [Wes McKinney] Force gc to avoid non-deterministic failure
d85228f [Wes McKinney] Be more careful about reference counts in zero-copy handoff, add pyarrow.Array.to_pandas method
cc7a6b3 [Uwe L. Korn] ARROW-362: Remove redunant reference count
---
 python/pyarrow/array.pyx                     | 21 ++++++++++++++
 python/pyarrow/includes/common.pxd           |  7 +++++
 python/pyarrow/includes/pyarrow.pxd          |  4 +--
 python/pyarrow/table.pyx                     | 18 ++++++++----
 python/pyarrow/tests/test_array.py           | 29 ++++++++++++++++++++
 python/pyarrow/tests/test_convert_builtin.py |  4 +++
 python/src/pyarrow/adapters/pandas.cc        |  4 +--
 7 files changed, 76 insertions(+), 11 deletions(-)

diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 84ab4a48c9b65..fbe4e3879062c 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -22,6 +22,7 @@
 import numpy as np
 
 from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.common cimport PyObject_to_object
 cimport pyarrow.includes.pyarrow as pyarrow
 
 import pyarrow.config
@@ -35,6 +36,8 @@ from pyarrow.scalar import NA
 from pyarrow.schema cimport Schema
 import pyarrow.schema as schema
 
+cimport cpython
+
 
 def total_allocated_bytes():
     cdef MemoryPool* pool = pyarrow.get_memory_pool()
@@ -111,6 +114,24 @@ cdef class Array:
     def slice(self, start, end):
         pass
 
+    def to_pandas(self):
+        """
+        Convert to an array object suitable for use in pandas
+
+        See also
+        --------
+        Column.to_pandas
+        Table.to_pandas
+        RecordBatch.to_pandas
+        """
+        cdef:
+            PyObject* np_arr
+
+        check_status(pyarrow.ConvertArrayToPandas(
+            self.sp_array, <PyObject*> self, &np_arr))
+
+        return PyObject_to_object(np_arr)
+
 
 cdef class NullArray(Array):
     pass
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 05c0123ee7b7e..f689bdc3fd819 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -47,3 +47,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsKeyError()
         c_bool IsNotImplemented()
         c_bool IsInvalid()
+
+
+cdef inline object PyObject_to_object(PyObject* o):
+    # Cast to "object" increments reference count
+    cdef object result = <object> o
+    cpython.Py_DECREF(result)
+    return result
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index e1da1914c5743..a5444c236bcc8 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -34,10 +34,10 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
                                 shared_ptr[CArray]* out)
 
     CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
-                                 object py_ref, PyObject** out)
+                                 PyObject* py_ref, PyObject** out)
 
     CStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
-                                  object py_ref, PyObject** out)
+                                  PyObject* py_ref, PyObject** out)
 
     MemoryPool* get_memory_pool()
 
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 969571262ca44..c71bc712bffb1 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -22,6 +22,7 @@
 from cython.operator cimport dereference as deref
 
 from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.common cimport PyObject_to_object
 cimport pyarrow.includes.pyarrow as pyarrow
 
 import pyarrow.config
@@ -32,6 +33,7 @@ from pyarrow.schema cimport box_data_type, box_schema
 
 from pyarrow.compat import frombytes, tobytes
 
+cimport cpython
 
 cdef class ChunkedArray:
     '''
@@ -100,8 +102,10 @@ cdef class Column:
 
         import pandas as pd
 
-        check_status(pyarrow.ConvertColumnToPandas(self.sp_column, self, &arr))
-        return pd.Series(<object>arr, name=self.name)
+        check_status(pyarrow.ConvertColumnToPandas(self.sp_column,
+                                                   <PyObject*> self, &arr))
+
+        return pd.Series(PyObject_to_object(arr), name=self.name)
 
     cdef _check_nullptr(self):
         if self.column == NULL:
@@ -248,9 +252,10 @@ cdef class RecordBatch:
         data = []
         for i in range(self.batch.num_columns()):
             arr = self.batch.column(i)
-            check_status(pyarrow.ConvertArrayToPandas(arr, self, &np_arr))
+            check_status(pyarrow.ConvertArrayToPandas(arr, <PyObject*> self,
+                                                      &np_arr))
             names.append(frombytes(self.batch.column_name(i)))
-            data.append(<object> np_arr)
+            data.append(PyObject_to_object(np_arr))
 
         return pd.DataFrame(dict(zip(names, data)), columns=names)
 
@@ -375,9 +380,10 @@ cdef class Table:
         for i in range(self.table.num_columns()):
             col = self.table.column(i)
             column = self.column(i)
-            check_status(pyarrow.ConvertColumnToPandas(col, column, &arr))
+            check_status(pyarrow.ConvertColumnToPandas(
+                col, <PyObject*> column, &arr))
             names.append(frombytes(col.get().name()))
-            data.append(<object> arr)
+            data.append(PyObject_to_object(arr))
 
         return pd.DataFrame(dict(zip(names, data)), columns=names)
 
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 0a17f691ccd1f..ead17dbec4e35 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import sys
+
 import pyarrow
 import pyarrow.formatting as fmt
 
@@ -71,3 +73,30 @@ def test_long_array_format():
   99
 ]"""
     assert result == expected
+
+
+def test_to_pandas_zero_copy():
+    import gc
+
+    arr = pyarrow.from_pylist(range(10))
+
+    for i in range(10):
+        np_arr = arr.to_pandas()
+        assert sys.getrefcount(np_arr) == 2
+        np_arr = None  # noqa
+
+    assert sys.getrefcount(arr) == 2
+
+    for i in range(10):
+        arr = pyarrow.from_pylist(range(10))
+        np_arr = arr.to_pandas()
+        arr = None
+        gc.collect()
+
+        # Ensure base is still valid
+
+        # Because of py.test's assert inspection magic, if you put getrefcount
+        # on the line being examined, it will be 1 higher than you expect
+        base_refcount = sys.getrefcount(np_arr.base)
+        assert base_refcount == 2
+        np_arr.sum()
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 2beb6b39d73ed..8937f8db6941f 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -47,6 +47,10 @@ def test_integer(self):
 
     def test_garbage_collection(self):
         import gc
+
+        # Force the cyclic garbage collector to run
+        gc.collect()
+
         bytes_before = pyarrow.total_allocated_bytes()
         pyarrow.from_pylist([1, None, 3, None])
         gc.collect()
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 7e70be75da5fc..6a3966b748806 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -628,8 +628,6 @@ class ArrowDeserializer {
     PyAcquireGIL lock;
 
     // Zero-Copy. We can pass the data pointer directly to NumPy.
-    Py_INCREF(py_ref_);
-    OwnedRef py_ref(py_ref_);
     npy_intp dims[1] = {col_->length()};
     out_ = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNewFromData(1, dims,
                 type, data));
@@ -646,7 +644,7 @@ class ArrowDeserializer {
       return Status::OK();
     } else {
       // PyArray_SetBaseObject steals our reference to py_ref_
-      py_ref.release();
+      Py_INCREF(py_ref_);
     }
 
     // Arrow data is immutable.

From 6996c17f70dc13659c37dfaa39bc28e7777ca6a6 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 8 Nov 2016 13:29:34 -0500
Subject: [PATCH 0195/1644] ARROW-312: [Java] IPC file round trip tool for
 integration testing

Author: Julien Le Dem <julien@dremio.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #186 from wesm/roundtrip-tool and squashes the following commits:

aee552a [Julien Le Dem] missing file
9d5c078 [Julien Le Dem] fix read-write bug
7f20b36 [Julien Le Dem] simple roundtrip
a04091f [Wes McKinney] Drafting file round trip helper executable
---
 .../main/java/io/netty/buffer/ArrowBuf.java   |   7 +-
 .../arrow/memory/TestBaseAllocator.java       |  24 ++-
 java/pom.xml                                  |   1 +
 java/tools/pom.xml                            |  73 ++++++++
 .../org/apache/arrow/tools/FileRoundtrip.java | 135 +++++++++++++++
 .../apache/arrow/tools/TestFileRoundtrip.java | 159 ++++++++++++++++++
 java/vector/pom.xml                           |  32 ++--
 .../templates/NullableValueVectors.java       |   2 +-
 .../org/apache/arrow/vector/VectorLoader.java |  21 +--
 .../apache/arrow/vector/VectorSchemaRoot.java | 140 +++++++++++++++
 .../apache/arrow/vector/VectorUnloader.java   |  13 +-
 .../arrow/vector/schema/ArrowBuffer.java      |   6 +
 .../arrow/vector/schema/ArrowRecordBatch.java |   8 +
 .../arrow/vector/TestVectorUnloadLoad.java    |  42 +++--
 .../arrow/vector/file/TestArrowFile.java      | 149 ++++++++--------
 15 files changed, 681 insertions(+), 131 deletions(-)
 create mode 100644 java/tools/pom.xml
 create mode 100644 java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
 create mode 100644 java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index a5989c1518def..95d2be5a43a36 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -179,7 +179,10 @@ public ArrowBuf retain(BufferAllocator target) {
       historicalLog.recordEvent("retain(%s)", target.getName());
     }
     final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
-    return otherLedger.newArrowBuf(offset, length, null);
+    ArrowBuf newArrowBuf = otherLedger.newArrowBuf(offset, length, null);
+    newArrowBuf.readerIndex(this.readerIndex);
+    newArrowBuf.writerIndex(this.writerIndex);
+    return newArrowBuf;
   }
 
   /**
@@ -214,6 +217,8 @@ public TransferResult transferOwnership(BufferAllocator target) {
 
     final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
     final ArrowBuf newBuf = otherLedger.newArrowBuf(offset, length, null);
+    newBuf.readerIndex(this.readerIndex);
+    newBuf.writerIndex(this.writerIndex);
     final boolean allocationFit = this.ledger.transferBalance(otherLedger);
     return new TransferResult(allocationFit, newBuf);
   }
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index aa6b70c5c74e2..3c96d57f4e64d 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -22,16 +22,13 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import io.netty.buffer.ArrowBuf;
-import io.netty.buffer.ArrowBuf.TransferResult;
 
-import org.apache.arrow.memory.AllocationReservation;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.memory.RootAllocator;
 import org.junit.Ignore;
 import org.junit.Test;
 
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.ArrowBuf.TransferResult;
+
 public class TestBaseAllocator {
   // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBaseAllocator.class);
 
@@ -134,6 +131,7 @@ public void testAllocator_transferOwnership() throws Exception {
       final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4);
       rootAllocator.verify();
       TransferResult transferOwnership = arrowBuf1.transferOwnership(childAllocator2);
+      assertEquiv(arrowBuf1, transferOwnership.buffer);
       final boolean allocationFit = transferOwnership.allocationFit;
       rootAllocator.verify();
       assertTrue(allocationFit);
@@ -160,6 +158,7 @@ public void testAllocator_shareOwnership() throws Exception {
       rootAllocator.verify();
       assertNotNull(arrowBuf2);
       assertNotEquals(arrowBuf2, arrowBuf1);
+      assertEquiv(arrowBuf1, arrowBuf2);
 
       // release original buffer (thus transferring ownership to allocator 2. (should leave allocator 1 in empty state)
       arrowBuf1.release();
@@ -172,6 +171,7 @@ public void testAllocator_shareOwnership() throws Exception {
       assertNotNull(arrowBuf3);
       assertNotEquals(arrowBuf3, arrowBuf1);
       assertNotEquals(arrowBuf3, arrowBuf2);
+      assertEquiv(arrowBuf1, arrowBuf3);
       rootAllocator.verify();
 
       arrowBuf2.release();
@@ -452,8 +452,10 @@ public void testAllocator_transferSliced() throws Exception {
       rootAllocator.verify();
 
       TransferResult result1 = arrowBuf2s.transferOwnership(childAllocator1);
+      assertEquiv(arrowBuf2s, result1.buffer);
       rootAllocator.verify();
       TransferResult result2 = arrowBuf1s.transferOwnership(childAllocator2);
+      assertEquiv(arrowBuf1s, result2.buffer);
       rootAllocator.verify();
 
       result1.buffer.release();
@@ -482,7 +484,9 @@ public void testAllocator_shareSliced() throws Exception {
       rootAllocator.verify();
 
       final ArrowBuf arrowBuf2s1 = arrowBuf2s.retain(childAllocator1);
+      assertEquiv(arrowBuf2s, arrowBuf2s1);
       final ArrowBuf arrowBuf1s2 = arrowBuf1s.retain(childAllocator2);
+      assertEquiv(arrowBuf1s, arrowBuf1s2);
       rootAllocator.verify();
 
       arrowBuf1s.release(); // releases arrowBuf1
@@ -512,11 +516,13 @@ public void testAllocator_transferShared() throws Exception {
       rootAllocator.verify();
       assertNotNull(arrowBuf2);
       assertNotEquals(arrowBuf2, arrowBuf1);
+      assertEquiv(arrowBuf1, arrowBuf2);
 
       TransferResult result = arrowBuf1.transferOwnership(childAllocator3);
       allocationFit = result.allocationFit;
       final ArrowBuf arrowBuf3 = result.buffer;
       assertTrue(allocationFit);
+      assertEquiv(arrowBuf1, arrowBuf3);
       rootAllocator.verify();
 
       // Since childAllocator3 now has childAllocator1's buffer, 1, can close
@@ -533,6 +539,7 @@ public void testAllocator_transferShared() throws Exception {
       allocationFit = result.allocationFit;
       final ArrowBuf arrowBuf4 = result2.buffer;
       assertTrue(allocationFit);
+      assertEquiv(arrowBuf3, arrowBuf4);
       rootAllocator.verify();
 
       arrowBuf3.release();
@@ -645,4 +652,9 @@ public void multiple() throws Exception {
 
     }
   }
+
+  public void assertEquiv(ArrowBuf origBuf, ArrowBuf newBuf) {
+    assertEquals(origBuf.readerIndex(), newBuf.readerIndex());
+    assertEquals(origBuf.writerIndex(), newBuf.writerIndex());
+  }
 }
diff --git a/java/pom.xml b/java/pom.xml
index 0147de7035794..7221a140d96ec 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -467,5 +467,6 @@
     <module>format</module>
     <module>memory</module>
     <module>vector</module>
+    <module>tools</module>
   </modules>
 </project>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
new file mode 100644
index 0000000000000..84b0b5eb4253c
--- /dev/null
+++ b/java/tools/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.arrow</groupId>
+        <artifactId>arrow-java-root</artifactId>
+        <version>0.1.1-SNAPSHOT</version>
+    </parent>
+    <artifactId>arrow-tools</artifactId>
+    <name>Arrow Tools</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-format</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.4</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>1.2</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+      <plugins>
+        <plugin>
+          <artifactId>maven-assembly-plugin</artifactId>
+          <version>2.6</version>
+          <configuration>
+            <descriptorRefs>
+              <descriptorRef>jar-with-dependencies</descriptorRef>
+            </descriptorRefs>
+          </configuration>
+          <executions>
+            <execution>
+              <id>make-assembly</id>
+              <phase>package</phase>
+              <goals>
+                <goal>single</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </build>
+
+</project>
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
new file mode 100644
index 0000000000000..db7a1c23f9ca6
--- /dev/null
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.arrow.tools;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.file.ArrowBlock;
+import org.apache.arrow.vector.file.ArrowFooter;
+import org.apache.arrow.vector.file.ArrowReader;
+import org.apache.arrow.vector.file.ArrowWriter;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileRoundtrip {
+  private static final Logger LOGGER = LoggerFactory.getLogger(FileRoundtrip.class);
+
+  public static void main(String[] args) {
+    System.exit(new FileRoundtrip(System.out, System.err).run(args));
+  }
+
+  private final Options options;
+  private final PrintStream out;
+  private final PrintStream err;
+
+  FileRoundtrip(PrintStream out, PrintStream err) {
+    this.out = out;
+    this.err = err;
+    this.options = new Options();
+    this.options.addOption("i", "in", true, "input file");
+    this.options.addOption("o", "out", true, "output file");
+
+  }
+
+  private File validateFile(String type, String fileName) {
+    if (fileName == null) {
+      throw new IllegalArgumentException("missing " + type + " file parameter");
+    }
+    File f = new File(fileName);
+    if (!f.exists() || f.isDirectory()) {
+      throw new IllegalArgumentException(type + " file not found: " + f.getAbsolutePath());
+    }
+    return f;
+  }
+
+  int run(String[] args) {
+    try {
+      CommandLineParser parser = new PosixParser();
+      CommandLine cmd = parser.parse(options, args, false);
+
+      String inFileName = cmd.getOptionValue("in");
+      String outFileName = cmd.getOptionValue("out");
+
+      File inFile = validateFile("input", inFileName);
+      File outFile = validateFile("output", outFileName);
+      BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); // TODO: close
+      try(
+          FileInputStream fileInputStream = new FileInputStream(inFile);
+          ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), allocator);) {
+
+        ArrowFooter footer = arrowReader.readFooter();
+        Schema schema = footer.getSchema();
+        LOGGER.debug("Input file size: " + inFile.length());
+        LOGGER.debug("Found schema: " + schema);
+
+        try (
+            FileOutputStream fileOutputStream = new FileOutputStream(outFile);
+            ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
+            ) {
+
+          // initialize vectors
+
+          List<ArrowBlock> recordBatches = footer.getRecordBatches();
+          for (ArrowBlock rbBlock : recordBatches) {
+            try (ArrowRecordBatch inRecordBatch = arrowReader.readRecordBatch(rbBlock);
+                VectorSchemaRoot root = new VectorSchemaRoot(schema, allocator);) {
+
+              VectorLoader vectorLoader = new VectorLoader(root);
+              vectorLoader.load(inRecordBatch);
+
+              VectorUnloader vectorUnloader = new VectorUnloader(root);
+              ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+              arrowWriter.writeRecordBatch(recordBatch);
+            }
+          }
+        }
+        LOGGER.debug("Output file size: " + outFile.length());
+      }
+    } catch (ParseException e) {
+      return fatalError("Invalid parameters", e);
+    } catch (IOException e) {
+      return fatalError("Error accessing files", e);
+    }
+    return 0;
+  }
+
+  private int fatalError(String message, Throwable e) {
+    err.println(message);
+    LOGGER.error(message, e);
+    return 1;
+  }
+
+}
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
new file mode 100644
index 0000000000000..339725e5af1e0
--- /dev/null
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.arrow.tools;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.file.ArrowBlock;
+import org.apache.arrow.vector.file.ArrowFooter;
+import org.apache.arrow.vector.file.ArrowReader;
+import org.apache.arrow.vector.file.ArrowWriter;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class TestFileRoundtrip {
+  private static final int COUNT = 10;
+
+  @Rule
+  public TemporaryFolder testFolder = new TemporaryFolder();
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Integer.MAX_VALUE);
+  }
+
+  @After
+  public void tearDown() {
+    allocator.close();
+  }
+
+  private void writeData(int count, MapVector parent) {
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    for (int i = 0; i < count; i++) {
+      intWriter.setPosition(i);
+      intWriter.writeInt(i);
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+    }
+    writer.setValueCount(count);
+  }
+
+  @Test
+  public void test() throws Exception {
+    File testInFile = testFolder.newFile("testIn.arrow");
+    File testOutFile = testFolder.newFile("testOut.arrow");
+
+    writeInput(testInFile);
+
+    String[] args = { "-i", testInFile.getAbsolutePath(), "-o",  testOutFile.getAbsolutePath()};
+    int result = new FileRoundtrip(System.out, System.err).run(args);
+    assertEquals(0, result);
+
+    validateOutput(testOutFile);
+  }
+
+  private void validateOutput(File testOutFile) throws Exception {
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        FileInputStream fileInputStream = new FileInputStream(testOutFile);
+        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        ) {
+      ArrowFooter footer = arrowReader.readFooter();
+      Schema schema = footer.getSchema();
+
+      // initialize vectors
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, readerAllocator)) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+
+        List<ArrowBlock> recordBatches = footer.getRecordBatches();
+        for (ArrowBlock rbBlock : recordBatches) {
+          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+            vectorLoader.load(recordBatch);
+          }
+          validateContent(COUNT, root);
+        }
+      }
+    }
+  }
+
+  private void validateContent(int count, VectorSchemaRoot root) {
+    Assert.assertEquals(count, root.getRowCount());
+    for (int i = 0; i < count; i++) {
+      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
+    }
+  }
+
+  public void writeInput(File testInFile) throws FileNotFoundException, IOException {
+    int count = COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+      writeData(count, parent);
+      write(parent.getChild("root"), testInFile);
+    }
+  }
+
+  private void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
+    Schema schema = new Schema(parent.getField().getChildren());
+    int valueCount = parent.getAccessor().getValueCount();
+    List<FieldVector> fields = parent.getChildrenFromFields();
+    VectorUnloader vectorUnloader = new VectorUnloader(schema, valueCount, fields);
+    try (
+        FileOutputStream fileOutputStream = new FileOutputStream(file);
+        ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
+        ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+            ) {
+      arrowWriter.writeRecordBatch(recordBatch);
+    }
+  }
+
+}
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 1d06bdece01f8..64b68bf8a1588 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -1,13 +1,13 @@
 <?xml version="1.0"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
-  license agreements. See the NOTICE file distributed with this work for additional 
-  information regarding copyright ownership. The ASF licenses this file to 
-  You under the Apache License, Version 2.0 (the "License"); you may not use 
-  this file except in compliance with the License. You may obtain a copy of 
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
-  by applicable law or agreed to in writing, software distributed under the 
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
-  OF ANY KIND, either express or implied. See the License for the specific 
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
   language governing permissions and limitations under the License. -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
@@ -56,8 +56,6 @@
       <artifactId>commons-lang3</artifactId>
       <version>3.4</version>
     </dependency>
-
-
   </dependencies>
 
     <pluginRepositories>
@@ -72,13 +70,13 @@
                 <enabled>false</enabled>
             </snapshots>
         </pluginRepository>
-    </pluginRepositories>  
-    
+    </pluginRepositories>
+
   <build>
 
     <resources>
       <resource>
-        <!-- Copy freemarker template and fmpp configuration files of Vector's 
+        <!-- Copy freemarker template and fmpp configuration files of Vector's
           to allow clients to leverage definitions. -->
         <directory>${basedir}/src/main/codegen</directory>
         <targetPath>codegen</targetPath>
@@ -129,7 +127,7 @@
     </plugins>
     <pluginManagement>
       <plugins>
-        <!--This plugin's configuration is used to store Eclipse m2e settings 
+        <!--This plugin's configuration is used to store Eclipse m2e settings
           only. It has no influence on the Maven build itself. -->
         <plugin>
           <groupId>org.eclipse.m2e</groupId>
@@ -160,8 +158,8 @@
         </plugin>
       </plugins>
     </pluginManagement>
-    
-  
+
+
   </build>
 
 
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index bafa31760205a..48af7a2bafe4d 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -145,7 +145,7 @@ public List<FieldVector> getChildrenFromFields() {
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
     org.apache.arrow.vector.BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
-    // TODO: do something with the sizes in fieldNode?
+    bits.valueCount = fieldNode.getLength();
   }
 
   public List<ArrowBuf> getFieldBuffers() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index b7040da9d8203..4afd82315d9c3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -27,7 +27,6 @@
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.schema.VectorLayout;
 import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.Schema;
 
 import com.google.common.collect.Iterators;
 
@@ -37,22 +36,16 @@
  * Loads buffers into vectors
  */
 public class VectorLoader {
-  private final List<FieldVector> fieldVectors;
-  private final List<Field> fields;
+  private final VectorSchemaRoot root;
 
   /**
    * will create children in root based on schema
    * @param schema the expected schema
    * @param root the root to add vectors to based on schema
    */
-  public VectorLoader(Schema schema, FieldVector root) {
+  public VectorLoader(VectorSchemaRoot root) {
     super();
-    this.fields = schema.getFields();
-    root.initializeChildrenFromFields(fields);
-    this.fieldVectors = root.getChildrenFromFields();
-    if (this.fieldVectors.size() != fields.size()) {
-      throw new IllegalArgumentException("The root vector did not create the right number of children. found " + fieldVectors.size() + " expected " + fields.size());
-    }
+    this.root = root;
   }
 
   /**
@@ -63,16 +56,19 @@ public VectorLoader(Schema schema, FieldVector root) {
   public void load(ArrowRecordBatch recordBatch) {
     Iterator<ArrowBuf> buffers = recordBatch.getBuffers().iterator();
     Iterator<ArrowFieldNode> nodes = recordBatch.getNodes().iterator();
+    List<Field> fields = root.getSchema().getFields();
     for (int i = 0; i < fields.size(); ++i) {
       Field field = fields.get(i);
-      FieldVector fieldVector = fieldVectors.get(i);
+      FieldVector fieldVector = root.getVector(field.getName());
       loadBuffers(fieldVector, field, buffers, nodes);
     }
+    root.setRowCount(recordBatch.getLength());
     if (nodes.hasNext() || buffers.hasNext()) {
       throw new IllegalArgumentException("not all nodes and buffers where consumed. nodes: " + Iterators.toString(nodes) + " buffers: " + Iterators.toString(buffers));
     }
   }
 
+
   private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buffers, Iterator<ArrowFieldNode> nodes) {
     checkArgument(nodes.hasNext(),
         "no more field nodes for for field " + field + " and vector " + vector);
@@ -85,7 +81,7 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
     try {
       vector.loadFieldBuffers(fieldNode, ownBuffers);
     } catch (RuntimeException e) {
-      throw new IllegalArgumentException("Could not load buffers for field " + field);
+      throw new IllegalArgumentException("Could not load buffers for field " + field, e);
     }
     List<Field> children = field.getChildren();
     if (children.size() > 0) {
@@ -98,4 +94,5 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
       }
     }
   }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
new file mode 100644
index 0000000000000..1cbe18787ef45
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+public class VectorSchemaRoot implements AutoCloseable {
+
+  private final Schema schema;
+  private int rowCount;
+  private final List<FieldVector> fieldVectors;
+  private final Map<String, FieldVector> fieldVectorsMap = new HashMap<>();
+
+  public VectorSchemaRoot(FieldVector parent) {
+    this.schema = new Schema(parent.getField().getChildren());
+    this.rowCount = parent.getAccessor().getValueCount();
+    this.fieldVectors = parent.getChildrenFromFields();
+    for (int i = 0; i < schema.getFields().size(); ++i) {
+      Field field = schema.getFields().get(i);
+      FieldVector vector = fieldVectors.get(i);
+      fieldVectorsMap.put(field.getName(), vector);
+    }
+  }
+
+  public VectorSchemaRoot(Schema schema, BufferAllocator allocator) {
+    super();
+    this.schema = schema;
+    List<FieldVector> fieldVectors = new ArrayList<>();
+    for (Field field : schema.getFields()) {
+      MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
+      FieldVector vector = minorType.getNewVector(field.getName(), allocator, null);
+      vector.initializeChildrenFromFields(field.getChildren());
+      fieldVectors.add(vector);
+      fieldVectorsMap.put(field.getName(), vector);
+    }
+    this.fieldVectors = Collections.unmodifiableList(fieldVectors);
+    if (this.fieldVectors.size() != schema.getFields().size()) {
+      throw new IllegalArgumentException("The root vector did not create the right number of children. found " + fieldVectors.size() + " expected " + schema.getFields().size());
+    }
+  }
+
+  public List<FieldVector> getFieldVectors() {
+    return fieldVectors;
+  }
+
+  public FieldVector getVector(String name) {
+    return fieldVectorsMap.get(name);
+  }
+
+  public Schema getSchema() {
+    return schema;
+  }
+
+  public int getRowCount() {
+    return rowCount;
+  }
+
+  public void setRowCount(int rowCount) {
+    this.rowCount = rowCount;
+  }
+
+  @Override
+  public void close() {
+    RuntimeException ex = null;
+    for (FieldVector fieldVector : fieldVectors) {
+      try {
+        fieldVector.close();
+      } catch (RuntimeException e) {
+        ex = chain(ex, e);
+      }
+    }
+    if (ex!= null) {
+      throw ex;
+    }
+  }
+
+  private RuntimeException chain(RuntimeException root, RuntimeException e) {
+    if (root == null) {
+      root = e;
+    } else {
+      root.addSuppressed(e);
+    }
+    return root;
+  }
+
+  private void printRow(StringBuilder sb, List<Object> row) {
+    boolean first = true;
+    for (Object v : row) {
+      if (first) {
+        first = false;
+      } else {
+        sb.append("\t");
+      }
+      sb.append(v);
+    }
+    sb.append("\n");
+  }
+
+  public String contentToTSVString() {
+    StringBuilder sb = new StringBuilder();
+    List<Object> row = new ArrayList<>(schema.getFields().size());
+    for (Field field : schema.getFields()) {
+      row.add(field.getName());
+    }
+    printRow(sb, row);
+    for (int i = 0; i < rowCount; i++) {
+      row.clear();
+      for (FieldVector v : fieldVectors) {
+        row.add(v.getAccessor().getObject(i));
+      }
+      printRow(sb, row);
+    }
+    return sb.toString();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 3375a7d5c311b..e2462180ffadc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -34,11 +34,15 @@ public class VectorUnloader {
   private final int valueCount;
   private final List<FieldVector> vectors;
 
-  public VectorUnloader(FieldVector parent) {
+  public VectorUnloader(Schema schema, int valueCount, List<FieldVector> vectors) {
     super();
-    this.schema = new Schema(parent.getField().getChildren());
-    this.valueCount = parent.getAccessor().getValueCount();
-    this.vectors = parent.getChildrenFromFields();
+    this.schema = schema;
+    this.valueCount = valueCount;
+    this.vectors = vectors;
+  }
+
+  public VectorUnloader(VectorSchemaRoot root) {
+    this(root.getSchema(), root.getRowCount(), root.getFieldVectors());
   }
 
   public Schema getSchema() {
@@ -77,4 +81,5 @@ private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<Ar
       appendNodes(child, nodes, buffers);
     }
   }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
index 3aa3e52582b4f..4e2e200d67645 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
@@ -78,4 +78,10 @@ public boolean equals(Object obj) {
   public int writeTo(FlatBufferBuilder builder) {
     return Buffer.createBuffer(builder, page, offset, size);
   }
+
+  @Override
+  public String toString() {
+    return "ArrowBuffer [page=" + page + ", offset=" + offset + ", size=" + size + "]";
+  }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index 9162efd29f864..adb99e2f3ffb7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -124,4 +124,12 @@ public void close() {
     }
   }
 
+  @Override
+  public String toString() {
+    return "ArrowRecordBatch [length=" + length + ", nodes=" + nodes + ", #buffers=" + buffers.size() + ", buffersLayout="
+        + buffersLayout + ", closed=" + closed + "]";
+  }
+
+
+
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 7dcb8977c0d7f..78f69eedc1c27 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -18,20 +18,18 @@
 package org.apache.arrow.vector;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
-import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -42,13 +40,15 @@ public class TestVectorUnloadLoad {
   static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
 
   @Test
-  public void test() throws IOException {
+  public void testUnloadLoad() throws IOException {
     int count = 10000;
     Schema schema;
 
     try (
         BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) {
+
+      // write some data
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
       MapWriter rootWriter = writer.rootAsMap();
       IntWriter intWriter = rootWriter.integer("int");
@@ -61,28 +61,40 @@ public void test() throws IOException {
       }
       writer.setValueCount(count);
 
-      VectorUnloader vectorUnloader = new VectorUnloader(parent.getChild("root"));
-      schema = vectorUnloader.getSchema();
-
+      // unload it
+      FieldVector root = parent.getChild("root");
+      schema = new Schema(root.getField().getChildren());
+      VectorUnloader vectorUnloader = newVectorUnloader(root);
       try (
           ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
           BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-          MapVector newParent = new MapVector("parent", finalVectorsAllocator, null)) {
-        FieldVector root = newParent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
-        VectorLoader vectorLoader = new VectorLoader(schema, root);
+          VectorSchemaRoot newRoot = new VectorSchemaRoot(schema, finalVectorsAllocator);
+          ) {
+
+        // load it
+        VectorLoader vectorLoader = new VectorLoader(newRoot);
 
         vectorLoader.load(recordBatch);
 
-        MapReader rootReader = new SingleMapReaderImpl(newParent).reader("root");
+        FieldReader intReader = newRoot.getVector("int").getReader();
+        FieldReader bigIntReader = newRoot.getVector("bigInt").getReader();
         for (int i = 0; i < count; i++) {
-          rootReader.setPosition(i);
-          Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
-          Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+          intReader.setPosition(i);
+          Assert.assertEquals(i, intReader.readInteger().intValue());
+          bigIntReader.setPosition(i);
+          Assert.assertEquals(i, bigIntReader.readLong().longValue());
         }
       }
     }
   }
 
+  public static VectorUnloader newVectorUnloader(FieldVector root) {
+    Schema schema = new Schema(root.getField().getChildren());
+    int valueCount = root.getAccessor().getValueCount();
+    List<FieldVector> fields = root.getChildrenFromFields();
+    return new VectorUnloader(schema, valueCount, fields);
+  }
+
   @AfterClass
   public static void afterClass() {
     allocator.close();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 0f28d53295c37..e97bc14d169b7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -17,6 +17,8 @@
  */
 package org.apache.arrow.vector.file;
 
+import static org.apache.arrow.vector.TestVectorUnloadLoad.newVectorUnloader;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -29,12 +31,12 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
-import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
@@ -43,7 +45,6 @@
 import org.apache.arrow.vector.holders.NullableTimeStampHolder;
 import org.apache.arrow.vector.schema.ArrowBuffer;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.joda.time.DateTimeZone;
 import org.junit.After;
@@ -94,8 +95,9 @@ public void testWriteComplex() throws IOException {
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
       writeComplexData(count, parent);
-      validateComplexContent(count, parent);
-      write(parent.getChild("root"), file);
+      FieldVector root = parent.getChild("root");
+      validateComplexContent(count, new VectorSchemaRoot(root));
+      write(root, file);
     }
   }
 
@@ -174,33 +176,31 @@ public void testWriteRead() throws IOException {
 
       // initialize vectors
 
-      NullableMapVector root = parent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
-
-      VectorLoader vectorLoader = new VectorLoader(schema, root);
-
-      List<ArrowBlock> recordBatches = footer.getRecordBatches();
-      for (ArrowBlock rbBlock : recordBatches) {
-        Assert.assertEquals(0, rbBlock.getOffset() % 8);
-        Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
-        try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-          List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
-          for (ArrowBuffer arrowBuffer : buffersLayout) {
-            Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+
+        List<ArrowBlock> recordBatches = footer.getRecordBatches();
+        for (ArrowBlock rbBlock : recordBatches) {
+          Assert.assertEquals(0, rbBlock.getOffset() % 8);
+          Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
+          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+            List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
+            for (ArrowBuffer arrowBuffer : buffersLayout) {
+              Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+            }
+            vectorLoader.load(recordBatch);
           }
-          vectorLoader.load(recordBatch);
-        }
 
-        validateContent(count, parent);
+          validateContent(count, root);
+        }
       }
     }
   }
 
-  private void validateContent(int count, MapVector parent) {
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+  private void validateContent(int count, VectorSchemaRoot root) {
     for (int i = 0; i < count; i++) {
-      rootReader.setPosition(i);
-      Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
-      Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
     }
   }
 
@@ -231,15 +231,15 @@ public void testWriteReadComplex() throws IOException {
 
       // initialize vectors
 
-      NullableMapVector root = parent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
-      VectorLoader vectorLoader = new VectorLoader(schema, root);
-
-      List<ArrowBlock> recordBatches = footer.getRecordBatches();
-      for (ArrowBlock rbBlock : recordBatches) {
-        try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-          vectorLoader.load(recordBatch);
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+        List<ArrowBlock> recordBatches = footer.getRecordBatches();
+        for (ArrowBlock rbBlock : recordBatches) {
+          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+            vectorLoader.load(recordBatch);
+          }
+          validateComplexContent(count, root);
         }
-        validateComplexContent(count, parent);
       }
     }
   }
@@ -255,23 +255,23 @@ public void printVectors(List<FieldVector> vectors) {
     }
   }
 
-  private void validateComplexContent(int count, NullableMapVector parent) {
-    printVectors(parent.getChildrenFromFields());
-
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+  private void validateComplexContent(int count, VectorSchemaRoot root) {
+    Assert.assertEquals(count, root.getRowCount());
+    printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
-      rootReader.setPosition(i);
-      Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
-      Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
-      Assert.assertEquals(i % 3, rootReader.reader("list").size());
+      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
+      Assert.assertEquals(i % 3, ((List<?>)root.getVector("list").getAccessor().getObject(i)).size());
       NullableTimeStampHolder h = new NullableTimeStampHolder();
-      rootReader.reader("map").reader("timestamp").read(h);
+      FieldReader mapReader = root.getVector("map").getReader();
+      mapReader.setPosition(i);
+      mapReader.reader("timestamp").read(h);
       Assert.assertEquals(i, h.value);
     }
   }
 
   private void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
-    VectorUnloader vectorUnloader = new VectorUnloader(parent);
+    VectorUnloader vectorUnloader = newVectorUnloader(parent);
     Schema schema = vectorUnloader.getSchema();
     LOGGER.debug("writing schema: " + schema);
     try (
@@ -294,7 +294,7 @@ public void testWriteReadMultipleRBs() throws IOException {
         MapVector parent = new MapVector("parent", originalVectorAllocator, null);
         FileOutputStream fileOutputStream = new FileOutputStream(file);) {
       writeData(count, parent);
-      VectorUnloader vectorUnloader = new VectorUnloader(parent.getChild("root"));
+      VectorUnloader vectorUnloader = newVectorUnloader(parent.getChild("root"));
       Schema schema = vectorUnloader.getSchema();
       Assert.assertEquals(2, schema.getFields().size());
       try (ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);) {
@@ -320,20 +320,21 @@ public void testWriteReadMultipleRBs() throws IOException {
       ArrowFooter footer = arrowReader.readFooter();
       Schema schema = footer.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      NullableMapVector root = parent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
-      VectorLoader vectorLoader = new VectorLoader(schema, root);
-      List<ArrowBlock> recordBatches = footer.getRecordBatches();
-      Assert.assertEquals(2, recordBatches.size());
-      for (ArrowBlock rbBlock : recordBatches) {
-        Assert.assertEquals(0, rbBlock.getOffset() % 8);
-        Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
-        try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-          List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
-          for (ArrowBuffer arrowBuffer : buffersLayout) {
-            Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator);) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+        List<ArrowBlock> recordBatches = footer.getRecordBatches();
+        Assert.assertEquals(2, recordBatches.size());
+        for (ArrowBlock rbBlock : recordBatches) {
+          Assert.assertEquals(0, rbBlock.getOffset() % 8);
+          Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
+          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+            List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
+            for (ArrowBuffer arrowBuffer : buffersLayout) {
+              Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+            }
+            vectorLoader.load(recordBatch);
+            validateContent(count, root);
           }
-          vectorLoader.load(recordBatch);
-          validateContent(count, parent);
         }
       }
     }
@@ -351,7 +352,7 @@ public void testWriteReadUnion() throws IOException {
 
       printVectors(parent.getChildrenFromFields());
 
-      validateUnionData(count, parent);
+      validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
 
       write(parent.getChild("root"), file);
     }
@@ -361,44 +362,42 @@ public void testWriteReadUnion() throws IOException {
         FileInputStream fileInputStream = new FileInputStream(file);
         ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
         BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)
         ) {
       ArrowFooter footer = arrowReader.readFooter();
       Schema schema = footer.getSchema();
       LOGGER.debug("reading schema: " + schema);
 
       // initialize vectors
-
-      NullableMapVector root = parent.addOrGet("root", MinorType.MAP, NullableMapVector.class);
-      VectorLoader vectorLoader = new VectorLoader(schema, root);
-
-      List<ArrowBlock> recordBatches = footer.getRecordBatches();
-      for (ArrowBlock rbBlock : recordBatches) {
-        try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-          vectorLoader.load(recordBatch);
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator);) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+        List<ArrowBlock> recordBatches = footer.getRecordBatches();
+        for (ArrowBlock rbBlock : recordBatches) {
+          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+            vectorLoader.load(recordBatch);
+          }
+          validateUnionData(count, root);
         }
-        validateUnionData(count, parent);
       }
     }
   }
 
-  public void validateUnionData(int count, MapVector parent) {
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+  public void validateUnionData(int count, VectorSchemaRoot root) {
+    FieldReader unionReader = root.getVector("union").getReader();
     for (int i = 0; i < count; i++) {
-      rootReader.setPosition(i);
+      unionReader.setPosition(i);
       switch (i % 4) {
       case 0:
-        Assert.assertEquals(i, rootReader.reader("union").readInteger().intValue());
+        Assert.assertEquals(i, unionReader.readInteger().intValue());
         break;
       case 1:
-        Assert.assertEquals(i, rootReader.reader("union").readLong().longValue());
+        Assert.assertEquals(i, unionReader.readLong().longValue());
         break;
       case 2:
-        Assert.assertEquals(i % 3, rootReader.reader("union").size());
+        Assert.assertEquals(i % 3, unionReader.size());
         break;
       case 3:
         NullableTimeStampHolder h = new NullableTimeStampHolder();
-        rootReader.reader("union").reader("timestamp").read(h);
+        unionReader.reader("timestamp").read(h);
         Assert.assertEquals(i, h.value);
         break;
       }

From 4fa7ac4f6ca30c34a73fb84d9d56d54aed96491b Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 9 Nov 2016 08:55:51 -0800
Subject: [PATCH 0196/1644] ARROW-372: json vector serialization format

This format serializes the vectors in JSON.
It is not a generic JSON to arrow converter but rather a human readable version of the vectors to help with tests.

Author: Julien Le Dem <julien@dremio.com>

Closes #201 from julienledem/json_file and squashes the following commits:

2e63bec [Julien Le Dem] add missing license
5588729 [Julien Le Dem] refactor tests, improve format
5ef5356 [Julien Le Dem] improve format to allow empty column name
746430c [Julien Le Dem] ARROW-372: Create JSON arrow file format for integration tests
---
 .../vector/file/json/JsonFileReader.java      | 223 ++++++++++++++++++
 .../vector/file/json/JsonFileWriter.java      | 167 +++++++++++++
 .../arrow/vector/file/BaseFileTest.java       | 220 +++++++++++++++++
 .../arrow/vector/file/TestArrowFile.java      | 200 +---------------
 .../arrow/vector/file/json/TestJSONFile.java  | 120 ++++++++++
 5 files changed, 741 insertions(+), 189 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
new file mode 100644
index 0000000000000..859a3a0e80a50
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -0,0 +1,223 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.file.json;
+
+import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
+import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
+import static com.fasterxml.jackson.core.JsonToken.START_ARRAY;
+import static com.fasterxml.jackson.core.JsonToken.START_OBJECT;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TimeStampVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.UInt2Vector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.UInt8Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ValueVector.Mutator;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.MappingJsonFactory;
+import com.google.common.base.Objects;
+
+public class JsonFileReader {
+  private final File inputFile;
+  private final JsonParser parser;
+  private final BufferAllocator allocator;
+  private Schema schema;
+
+  public JsonFileReader(File inputFile, BufferAllocator allocator) throws JsonParseException, IOException {
+    super();
+    this.inputFile = inputFile;
+    this.allocator = allocator;
+    MappingJsonFactory jsonFactory = new MappingJsonFactory();
+    this.parser = jsonFactory.createParser(inputFile);
+  }
+
+  public Schema start() throws JsonParseException, IOException {
+    readToken(START_OBJECT);
+    {
+      this.schema = readNextField("schema", Schema.class);
+      nextFieldIs("batches");
+      readToken(START_ARRAY);
+      return schema;
+    }
+  }
+
+  public VectorSchemaRoot read() throws IOException {
+    VectorSchemaRoot recordBatch = new VectorSchemaRoot(schema, allocator);
+    readToken(START_OBJECT);
+    {
+      int count = readNextField("count", Integer.class);
+      recordBatch.setRowCount(count);
+      nextFieldIs("columns");
+      readToken(START_ARRAY);
+      {
+        for (Field field : schema.getFields()) {
+          FieldVector vector = recordBatch.getVector(field.getName());
+          readVector(field, vector);
+        }
+      }
+      readToken(END_ARRAY);
+    }
+    readToken(END_OBJECT);
+    return recordBatch;
+  }
+
+  private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException {
+    List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
+    List<BufferBacked> fieldInnerVectors = vector.getFieldInnerVectors();
+    if (vectorTypes.size() != fieldInnerVectors.size()) {
+      throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size());
+    }
+    readToken(START_OBJECT);
+    {
+      String name = readNextField("name", String.class);
+      if (!Objects.equal(field.getName(), name)) {
+        throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name);
+      }
+      int count = readNextField("count", Integer.class);
+      for (int v = 0; v < vectorTypes.size(); v++) {
+        ArrowVectorType vectorType = vectorTypes.get(v);
+        BufferBacked innerVector = fieldInnerVectors.get(v);
+        nextFieldIs(vectorType.getName());
+        readToken(START_ARRAY);
+        ValueVector valueVector = (ValueVector)innerVector;
+        valueVector.allocateNew();
+        Mutator mutator = valueVector.getMutator();
+        mutator.setValueCount(count);
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          setValueFromParser(valueVector, i);
+        }
+        readToken(END_ARRAY);
+      }
+      // if children
+      List<Field> fields = field.getChildren();
+      if (!fields.isEmpty()) {
+        List<FieldVector> vectorChildren = vector.getChildrenFromFields();
+        if (fields.size() != vectorChildren.size()) {
+          throw new IllegalArgumentException("fields and children are not the same size: " + fields.size() + " != " + vectorChildren.size());
+        }
+        nextFieldIs("children");
+        readToken(START_ARRAY);
+        for (int i = 0; i < fields.size(); i++) {
+          Field childField = fields.get(i);
+          FieldVector childVector = vectorChildren.get(i);
+          readVector(childField, childVector);
+        }
+        readToken(END_ARRAY);
+      }
+    }
+    readToken(END_OBJECT);
+  }
+
+  private void setValueFromParser(ValueVector valueVector, int i) throws IOException {
+    switch (valueVector.getMinorType()) {
+    case BIT:
+      ((BitVector)valueVector).getMutator().set(i, parser.readValueAs(Boolean.class) ? 1 : 0);
+      break;
+    case TINYINT:
+      ((TinyIntVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case SMALLINT:
+      ((SmallIntVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case INT:
+      ((IntVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case BIGINT:
+      ((BigIntVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case UINT1:
+      ((UInt1Vector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case UINT2:
+      ((UInt2Vector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case UINT4:
+      ((UInt4Vector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case UINT8:
+      ((UInt8Vector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case FLOAT4:
+      ((Float4Vector)valueVector).getMutator().set(i, parser.readValueAs(Float.class));
+      break;
+    case FLOAT8:
+      ((Float8Vector)valueVector).getMutator().set(i, parser.readValueAs(Double.class));
+      break;
+    case VARCHAR:
+      ((VarCharVector)valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8));
+      break;
+    case TIMESTAMP:
+      ((TimeStampVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    default:
+      throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
+    }
+  }
+
+  public void close() throws IOException {
+    readToken(END_ARRAY);
+    readToken(END_OBJECT);
+    parser.close();
+  }
+
+  private <T> T readNextField(String expectedFieldName, Class<T> c) throws IOException, JsonParseException {
+    nextFieldIs(expectedFieldName);
+    parser.nextToken();
+    return parser.readValueAs(c);
+  }
+
+  private void nextFieldIs(String expectedFieldName) throws IOException, JsonParseException {
+    String name = parser.nextFieldName();
+    if (name == null || !name.equals(expectedFieldName)) {
+      throw new IllegalStateException("Expected " + expectedFieldName + " but got " + name);
+    }
+  }
+
+  private void readToken(JsonToken expected) throws JsonParseException, IOException {
+    JsonToken t = parser.nextToken();
+    if (t != expected) {
+      throw new IllegalStateException("Expected " + expected + " but got " + t);
+    }
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
new file mode 100644
index 0000000000000..47c1a7dabef11
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -0,0 +1,167 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.file.json;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.TimeStampVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ValueVector.Accessor;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
+import com.fasterxml.jackson.databind.MappingJsonFactory;
+
+public class JsonFileWriter {
+
+  public static final class JSONWriteConfig {
+    private final boolean pretty;
+    private JSONWriteConfig(boolean pretty) {
+      this.pretty = pretty;
+    }
+    private JSONWriteConfig() {
+      this.pretty = false;
+    }
+    public JSONWriteConfig pretty(boolean pretty) {
+      return new JSONWriteConfig(pretty);
+    }
+  }
+
+  public static JSONWriteConfig config() {
+    return new JSONWriteConfig();
+  }
+
+  private final JsonGenerator generator;
+  private Schema schema;
+
+  public JsonFileWriter(File outputFile) throws IOException {
+    this(outputFile, config());
+  }
+
+  public JsonFileWriter(File outputFile, JSONWriteConfig config) throws IOException {
+    MappingJsonFactory jsonFactory = new MappingJsonFactory();
+    this.generator = jsonFactory.createGenerator(outputFile, JsonEncoding.UTF8);
+    if (config.pretty) {
+      DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
+      prettyPrinter.indentArraysWith(NopIndenter.instance);
+      this.generator.setPrettyPrinter(prettyPrinter);
+    }
+  }
+
+  public void start(Schema schema) throws IOException {
+    this.schema = schema;
+    generator.writeStartObject();
+    generator.writeObjectField("schema", schema);
+    generator.writeArrayFieldStart("batches");
+  }
+
+  public void write(VectorSchemaRoot recordBatch) throws IOException {
+    if (!recordBatch.getSchema().equals(schema)) {
+      throw new IllegalArgumentException("record batches must have the same schema: " + schema);
+    }
+    generator.writeStartObject();
+    {
+      generator.writeObjectField("count", recordBatch.getRowCount());
+      generator.writeArrayFieldStart("columns");
+      for (Field field : schema.getFields()) {
+        FieldVector vector = recordBatch.getVector(field.getName());
+        writeVector(field, vector);
+      }
+      generator.writeEndArray();
+    }
+    generator.writeEndObject();
+  }
+
+  private void writeVector(Field field, FieldVector vector) throws IOException {
+    List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
+    List<BufferBacked> fieldInnerVectors = vector.getFieldInnerVectors();
+    if (vectorTypes.size() != fieldInnerVectors.size()) {
+      throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size());
+    }
+    generator.writeStartObject();
+    {
+      generator.writeObjectField("name", field.getName());
+      int valueCount = vector.getAccessor().getValueCount();
+      generator.writeObjectField("count", valueCount);
+      for (int v = 0; v < vectorTypes.size(); v++) {
+        ArrowVectorType vectorType = vectorTypes.get(v);
+        BufferBacked innerVector = fieldInnerVectors.get(v);
+        generator.writeArrayFieldStart(vectorType.getName());
+        ValueVector valueVector = (ValueVector)innerVector;
+        for (int i = 0; i < valueCount; i++) {
+          writeValueToGenerator(valueVector, i);
+        }
+        generator.writeEndArray();
+      }
+      List<Field> fields = field.getChildren();
+      List<FieldVector> children = vector.getChildrenFromFields();
+      if (fields.size() != children.size()) {
+        throw new IllegalArgumentException("fields and children are not the same size: " + fields.size() + " != " + children.size());
+      }
+      if (fields.size() > 0) {
+        generator.writeArrayFieldStart("children");
+        for (int i = 0; i < fields.size(); i++) {
+          Field childField = fields.get(i);
+          FieldVector childVector = children.get(i);
+          writeVector(childField, childVector);
+        }
+        generator.writeEndArray();
+      }
+    }
+    generator.writeEndObject();
+  }
+
+  private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException {
+    switch (valueVector.getMinorType()) {
+      case TIMESTAMP:
+        generator.writeNumber(((TimeStampVector)valueVector).getAccessor().get(i));
+        break;
+      case BIT:
+        generator.writeNumber(((BitVector)valueVector).getAccessor().get(i));
+        break;
+      default:
+        // TODO: each type
+        Accessor accessor = valueVector.getAccessor();
+        Object value = accessor.getObject(i);
+        if (value instanceof Number || value instanceof Boolean) {
+          generator.writeObject(value);
+        } else {
+          generator.writeObject(value.toString());
+        }
+        break;
+    }
+  }
+
+  public void close() throws IOException {
+    generator.writeEndArray();
+    generator.writeEndObject();
+    generator.close();
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
new file mode 100644
index 0000000000000..6e577b500a6bd
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector.Accessor;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.holders.NullableTimeStampHolder;
+import org.joda.time.DateTimeZone;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Helps testing the file formats
+ */
+public class BaseFileTest {
+  private static final Logger LOGGER = LoggerFactory.getLogger(BaseFileTest.class);
+  protected static final int COUNT = 10;
+  protected BufferAllocator allocator;
+
+  private DateTimeZone defaultTimezone = DateTimeZone.getDefault();
+
+  @Before
+  public void init() {
+    DateTimeZone.setDefault(DateTimeZone.forOffsetHours(2));
+    allocator = new RootAllocator(Integer.MAX_VALUE);
+  }
+
+  @After
+  public void tearDown() {
+    allocator.close();
+    DateTimeZone.setDefault(defaultTimezone);
+  }
+
+  protected void validateContent(int count, VectorSchemaRoot root) {
+    for (int i = 0; i < count; i++) {
+      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
+    }
+  }
+
+  protected void writeComplexData(int count, MapVector parent) {
+    ArrowBuf varchar = allocator.buffer(3);
+    varchar.readerIndex(0);
+    varchar.setByte(0, 'a');
+    varchar.setByte(1, 'b');
+    varchar.setByte(2, 'c');
+    varchar.writerIndex(3);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    ListWriter listWriter = rootWriter.list("list");
+    MapWriter mapWriter = rootWriter.map("map");
+    for (int i = 0; i < count; i++) {
+      if (i % 5 != 3) {
+        intWriter.setPosition(i);
+        intWriter.writeInt(i);
+      }
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 3; j++) {
+        listWriter.varChar().writeVarChar(0, 3, varchar);
+      }
+      listWriter.endList();
+      mapWriter.setPosition(i);
+      mapWriter.start();
+      mapWriter.timeStamp("timestamp").writeTimeStamp(i);
+      mapWriter.end();
+    }
+    writer.setValueCount(count);
+    varchar.release();
+  }
+
+  public void printVectors(List<FieldVector> vectors) {
+    for (FieldVector vector : vectors) {
+      LOGGER.debug(vector.getField().getName());
+      Accessor accessor = vector.getAccessor();
+      int valueCount = accessor.getValueCount();
+      for (int i = 0; i < valueCount; i++) {
+        LOGGER.debug(String.valueOf(accessor.getObject(i)));
+      }
+    }
+  }
+
+  protected void validateComplexContent(int count, VectorSchemaRoot root) {
+    Assert.assertEquals(count, root.getRowCount());
+    printVectors(root.getFieldVectors());
+    for (int i = 0; i < count; i++) {
+      Object intVal = root.getVector("int").getAccessor().getObject(i);
+      if (i % 5 != 3) {
+        Assert.assertEquals(i, intVal);
+      } else {
+        Assert.assertNull(intVal);
+      }
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
+      Assert.assertEquals(i % 3, ((List<?>)root.getVector("list").getAccessor().getObject(i)).size());
+      NullableTimeStampHolder h = new NullableTimeStampHolder();
+      FieldReader mapReader = root.getVector("map").getReader();
+      mapReader.setPosition(i);
+      mapReader.reader("timestamp").read(h);
+      Assert.assertEquals(i, h.value);
+    }
+  }
+
+  protected void writeData(int count, MapVector parent) {
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    for (int i = 0; i < count; i++) {
+      intWriter.setPosition(i);
+      intWriter.writeInt(i);
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+    }
+    writer.setValueCount(count);
+  }
+
+  public void validateUnionData(int count, VectorSchemaRoot root) {
+    FieldReader unionReader = root.getVector("union").getReader();
+    for (int i = 0; i < count; i++) {
+      unionReader.setPosition(i);
+      switch (i % 4) {
+      case 0:
+        Assert.assertEquals(i, unionReader.readInteger().intValue());
+        break;
+      case 1:
+        Assert.assertEquals(i, unionReader.readLong().longValue());
+        break;
+      case 2:
+        Assert.assertEquals(i % 3, unionReader.size());
+        break;
+      case 3:
+        NullableTimeStampHolder h = new NullableTimeStampHolder();
+        unionReader.reader("timestamp").read(h);
+        Assert.assertEquals(i, h.value);
+        break;
+      }
+    }
+  }
+
+  public void writeUnionData(int count, NullableMapVector parent) {
+    ArrowBuf varchar = allocator.buffer(3);
+    varchar.readerIndex(0);
+    varchar.setByte(0, 'a');
+    varchar.setByte(1, 'b');
+    varchar.setByte(2, 'c');
+    varchar.writerIndex(3);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("union");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("union");
+    ListWriter listWriter = rootWriter.list("union");
+    MapWriter mapWriter = rootWriter.map("union");
+    for (int i = 0; i < count; i++) {
+      switch (i % 4) {
+      case 0:
+        intWriter.setPosition(i);
+        intWriter.writeInt(i);
+        break;
+      case 1:
+        bigIntWriter.setPosition(i);
+        bigIntWriter.writeBigInt(i);
+        break;
+      case 2:
+        listWriter.setPosition(i);
+        listWriter.startList();
+        for (int j = 0; j < i % 3; j++) {
+          listWriter.varChar().writeVarChar(0, 3, varchar);
+        }
+        listWriter.endList();
+        break;
+      case 3:
+        mapWriter.setPosition(i);
+        mapWriter.start();
+        mapWriter.timeStamp("timestamp").writeTimeStamp(i);
+        mapWriter.end();
+        break;
+      }
+    }
+    writer.setValueCount(count);
+    varchar.release();
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index e97bc14d169b7..c9e60ee047bfe 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -27,53 +27,22 @@
 import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
-import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.complex.writer.BigIntWriter;
-import org.apache.arrow.vector.complex.writer.IntWriter;
-import org.apache.arrow.vector.holders.NullableTimeStampHolder;
 import org.apache.arrow.vector.schema.ArrowBuffer;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.Schema;
-import org.joda.time.DateTimeZone;
-import org.junit.After;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import io.netty.buffer.ArrowBuf;
-
-public class TestArrowFile {
+public class TestArrowFile extends BaseFileTest {
   private static final Logger LOGGER = LoggerFactory.getLogger(TestArrowFile.class);
-  private static final int COUNT = 10;
-  private BufferAllocator allocator;
-
-  private DateTimeZone defaultTimezone = DateTimeZone.getDefault();
-
-  @Before
-  public void init() {
-    DateTimeZone.setDefault(DateTimeZone.forOffsetHours(2));
-    allocator = new RootAllocator(Integer.MAX_VALUE);
-  }
-
-  @After
-  public void tearDown() {
-    allocator.close();
-    DateTimeZone.setDefault(defaultTimezone);
-  }
 
   @Test
   public void testWrite() throws IOException {
@@ -101,54 +70,6 @@ public void testWriteComplex() throws IOException {
     }
   }
 
-  private void writeComplexData(int count, MapVector parent) {
-    ArrowBuf varchar = allocator.buffer(3);
-    varchar.readerIndex(0);
-    varchar.setByte(0, 'a');
-    varchar.setByte(1, 'b');
-    varchar.setByte(2, 'c');
-    varchar.writerIndex(3);
-    ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
-    IntWriter intWriter = rootWriter.integer("int");
-    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
-    ListWriter listWriter = rootWriter.list("list");
-    MapWriter mapWriter = rootWriter.map("map");
-    for (int i = 0; i < count; i++) {
-      intWriter.setPosition(i);
-      intWriter.writeInt(i);
-      bigIntWriter.setPosition(i);
-      bigIntWriter.writeBigInt(i);
-      listWriter.setPosition(i);
-      listWriter.startList();
-      for (int j = 0; j < i % 3; j++) {
-        listWriter.varChar().writeVarChar(0, 3, varchar);
-      }
-      listWriter.endList();
-      mapWriter.setPosition(i);
-      mapWriter.start();
-      mapWriter.timeStamp("timestamp").writeTimeStamp(i);
-      mapWriter.end();
-    }
-    writer.setValueCount(count);
-    varchar.release();
-  }
-
-
-  private void writeData(int count, MapVector parent) {
-    ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
-    IntWriter intWriter = rootWriter.integer("int");
-    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
-    for (int i = 0; i < count; i++) {
-      intWriter.setPosition(i);
-      intWriter.writeInt(i);
-      bigIntWriter.setPosition(i);
-      bigIntWriter.writeBigInt(i);
-    }
-    writer.setValueCount(count);
-  }
-
   @Test
   public void testWriteRead() throws IOException {
     File file = new File("target/mytest.arrow");
@@ -197,13 +118,6 @@ public void testWriteRead() throws IOException {
     }
   }
 
-  private void validateContent(int count, VectorSchemaRoot root) {
-    for (int i = 0; i < count; i++) {
-      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
-      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
-    }
-  }
-
   @Test
   public void testWriteReadComplex() throws IOException {
     File file = new File("target/mytest_complex.arrow");
@@ -244,45 +158,6 @@ public void testWriteReadComplex() throws IOException {
     }
   }
 
-  public void printVectors(List<FieldVector> vectors) {
-    for (FieldVector vector : vectors) {
-      LOGGER.debug(vector.getField().getName());
-      Accessor accessor = vector.getAccessor();
-      int valueCount = accessor.getValueCount();
-      for (int i = 0; i < valueCount; i++) {
-        LOGGER.debug(String.valueOf(accessor.getObject(i)));
-      }
-    }
-  }
-
-  private void validateComplexContent(int count, VectorSchemaRoot root) {
-    Assert.assertEquals(count, root.getRowCount());
-    printVectors(root.getFieldVectors());
-    for (int i = 0; i < count; i++) {
-      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
-      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
-      Assert.assertEquals(i % 3, ((List<?>)root.getVector("list").getAccessor().getObject(i)).size());
-      NullableTimeStampHolder h = new NullableTimeStampHolder();
-      FieldReader mapReader = root.getVector("map").getReader();
-      mapReader.setPosition(i);
-      mapReader.reader("timestamp").read(h);
-      Assert.assertEquals(i, h.value);
-    }
-  }
-
-  private void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
-    VectorUnloader vectorUnloader = newVectorUnloader(parent);
-    Schema schema = vectorUnloader.getSchema();
-    LOGGER.debug("writing schema: " + schema);
-    try (
-        FileOutputStream fileOutputStream = new FileOutputStream(file);
-        ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
-        ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
-            ) {
-      arrowWriter.writeRecordBatch(recordBatch);
-    }
-  }
-
   @Test
   public void testWriteReadMultipleRBs() throws IOException {
     File file = new File("target/mytest_multiple.arrow");
@@ -381,69 +256,16 @@ public void testWriteReadUnion() throws IOException {
     }
   }
 
-  public void validateUnionData(int count, VectorSchemaRoot root) {
-    FieldReader unionReader = root.getVector("union").getReader();
-    for (int i = 0; i < count; i++) {
-      unionReader.setPosition(i);
-      switch (i % 4) {
-      case 0:
-        Assert.assertEquals(i, unionReader.readInteger().intValue());
-        break;
-      case 1:
-        Assert.assertEquals(i, unionReader.readLong().longValue());
-        break;
-      case 2:
-        Assert.assertEquals(i % 3, unionReader.size());
-        break;
-      case 3:
-        NullableTimeStampHolder h = new NullableTimeStampHolder();
-        unionReader.reader("timestamp").read(h);
-        Assert.assertEquals(i, h.value);
-        break;
-      }
-    }
-  }
-
-  public void writeUnionData(int count, NullableMapVector parent) {
-    ArrowBuf varchar = allocator.buffer(3);
-    varchar.readerIndex(0);
-    varchar.setByte(0, 'a');
-    varchar.setByte(1, 'b');
-    varchar.setByte(2, 'c');
-    varchar.writerIndex(3);
-    ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
-    IntWriter intWriter = rootWriter.integer("union");
-    BigIntWriter bigIntWriter = rootWriter.bigInt("union");
-    ListWriter listWriter = rootWriter.list("union");
-    MapWriter mapWriter = rootWriter.map("union");
-    for (int i = 0; i < count; i++) {
-      switch (i % 4) {
-      case 0:
-        intWriter.setPosition(i);
-        intWriter.writeInt(i);
-        break;
-      case 1:
-        bigIntWriter.setPosition(i);
-        bigIntWriter.writeBigInt(i);
-        break;
-      case 2:
-        listWriter.setPosition(i);
-        listWriter.startList();
-        for (int j = 0; j < i % 3; j++) {
-          listWriter.varChar().writeVarChar(0, 3, varchar);
-        }
-        listWriter.endList();
-        break;
-      case 3:
-        mapWriter.setPosition(i);
-        mapWriter.start();
-        mapWriter.timeStamp("timestamp").writeTimeStamp(i);
-        mapWriter.end();
-        break;
-      }
+  private void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
+    VectorUnloader vectorUnloader = newVectorUnloader(parent);
+    Schema schema = vectorUnloader.getSchema();
+    LOGGER.debug("writing schema: " + schema);
+    try (
+        FileOutputStream fileOutputStream = new FileOutputStream(file);
+        ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
+        ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+            ) {
+      arrowWriter.writeRecordBatch(recordBatch);
     }
-    writer.setValueCount(count);
-    varchar.release();
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
new file mode 100644
index 0000000000000..7d25003f8b335
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file.json;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.file.BaseFileTest;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestJSONFile extends BaseFileTest {
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestJSONFile.class);
+
+  @Test
+  public void testWriteReadComplexJSON() throws IOException {
+    File file = new File("target/mytest_complex.json");
+    int count = COUNT;
+
+    // write
+    try (
+        BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+      writeComplexData(count, parent);
+      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")));
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateComplexContent(count, root);
+      }
+      reader.close();
+    }
+  }
+
+  @Test
+  public void testWriteComplexJSON() throws IOException {
+    File file = new File("target/mytest_write_complex.json");
+    int count = COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
+      writeComplexData(count, parent);
+      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
+      validateComplexContent(root.getRowCount(), root);
+      writeJSON(file, root);
+    }
+  }
+
+  public void writeJSON(File file, VectorSchemaRoot root) throws IOException {
+    JsonFileWriter writer = new JsonFileWriter(file, JsonFileWriter.config().pretty(true));
+    writer.start(root.getSchema());
+    writer.write(root);
+    writer.close();
+  }
+
+
+  @Test
+  public void testWriteReadUnionJSON() throws IOException {
+    File file = new File("target/mytest_write_union.json");
+    int count = COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
+
+      writeUnionData(count, parent);
+
+      printVectors(parent.getChildrenFromFields());
+
+      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
+      validateUnionData(count, root);
+
+      writeJSON(file, root);
+    }
+ // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateUnionData(count, root);
+      }
+    }
+  }
+
+}

From 7f048a4b8bdc6a20cd8f6eeca928ecbb6db7dd96 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 11 Nov 2016 14:18:09 -0500
Subject: [PATCH 0197/1644] ARROW-356: Add documentation about reading Parquet

Assumes #192.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #193 from xhochy/ARROW-356 and squashes the following commits:

530484f [Uwe L. Korn] Mention new setup instructions
06b2f9c [Uwe L. Korn] Add tables describing dtype support
0467e0e [Uwe L. Korn] Move installation instructions into Sphinx docs
744202a [Uwe L. Korn] Document Pandas<->Arrow conversion
b5b4df5 [Uwe L. Korn] ARROW-356: Add documentation about reading Parquet
---
 python/doc/INSTALL.md    | 101 --------------------------
 python/doc/index.rst     |  16 +++--
 python/doc/install.rst   | 151 +++++++++++++++++++++++++++++++++++++++
 python/doc/pandas.rst    | 114 +++++++++++++++++++++++++++++
 python/doc/parquet.rst   |  66 +++++++++++++++++
 python/pyarrow/table.pyx |  15 ++++
 6 files changed, 355 insertions(+), 108 deletions(-)
 delete mode 100644 python/doc/INSTALL.md
 create mode 100644 python/doc/install.rst
 create mode 100644 python/doc/pandas.rst
 create mode 100644 python/doc/parquet.rst

diff --git a/python/doc/INSTALL.md b/python/doc/INSTALL.md
deleted file mode 100644
index 81eed565d9123..0000000000000
--- a/python/doc/INSTALL.md
+++ /dev/null
@@ -1,101 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-## Building pyarrow (Apache Arrow Python library)
-
-First, clone the master git repository:
-
-```bash
-git clone https://github.com/apache/arrow.git arrow
-```
-
-#### System requirements
-
-Building pyarrow requires:
-
-* A C++11 compiler
-
-  * Linux: gcc >= 4.8 or clang >= 3.5
-  * OS X: XCode 6.4 or higher preferred
-
-* [cmake][1]
-
-#### Python requirements
-
-You will need Python (CPython) 2.7, 3.4, or 3.5 installed. Earlier releases and
-are not being targeted.
-
-> This library targets CPython only due to an emphasis on interoperability with
-> pandas and NumPy, which are only available for CPython.
-
-The build requires NumPy, Cython, and a few other Python dependencies:
-
-```bash
-pip install cython
-cd arrow/python
-pip install -r requirements.txt
-```
-
-#### Installing Arrow C++ library
-
-First, you should choose an installation location for Arrow C++. In the future
-using the default system install location will work, but for now we are being
-explicit:
-
-```bash
-export ARROW_HOME=$HOME/local
-```
-
-Now, we build Arrow:
-
-```bash
-cd arrow/cpp
-
-mkdir dev-build
-cd dev-build
-
-cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME ..
-
-make
-
-# Use sudo here if $ARROW_HOME requires it
-make install
-```
-
-#### Install `pyarrow`
-
-```bash
-cd arrow/python
-
-python setup.py install
-```
-
-> On XCode 6 and prior there are some known OS X `@rpath` issues. If you are
-> unable to import pyarrow, upgrading XCode may be the solution.
-
-
-```python
-In [1]: import pyarrow
-
-In [2]: pyarrow.from_pylist([1,2,3])
-Out[2]:
-<pyarrow.array.Int64Array object at 0x7f899f3e60e8>
-[
-  1,
-  2,
-  3
-]
-```
-
-[1]: https://cmake.org/
diff --git a/python/doc/index.rst b/python/doc/index.rst
index 88725badc1e24..6725ae707d90b 100644
--- a/python/doc/index.rst
+++ b/python/doc/index.rst
@@ -31,14 +31,16 @@ additional functionality such as reading Apache Parquet files into Arrow
 structures.
 
 .. toctree::
-   :maxdepth: 4
-   :hidden:
+   :maxdepth: 2
+   :caption: Getting Started
 
+   Installing pyarrow <install.rst>
+   Pandas <pandas.rst>
    Module Reference <modules.rst>
 
-Indices and tables
-==================
+.. toctree::
+   :maxdepth: 2
+   :caption: Additional Features
+
+   Parquet format <parquet.rst>
 
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
diff --git a/python/doc/install.rst b/python/doc/install.rst
new file mode 100644
index 0000000000000..1bab017301633
--- /dev/null
+++ b/python/doc/install.rst
@@ -0,0 +1,151 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Install PyArrow
+===============
+
+Conda
+-----
+
+To install the latest version of PyArrow from conda-forge using conda:
+
+.. code-block:: bash
+
+    conda install -c conda-forge pyarrow
+
+Pip
+---
+
+Install the latest version from PyPI:
+
+.. code-block:: bash
+
+    pip install pyarrow
+
+.. note::
+    Currently there are only binary artifcats available for Linux and MacOS.
+    Otherwise this will only pull the python sources and assumes an existing
+    installation of the C++ part of Arrow.
+    To retrieve the binary artifacts, you'll need a recent ``pip`` version that
+    supports features like the ``manylinux1`` tag.
+
+Building from source
+--------------------
+
+First, clone the master git repository:
+
+.. code-block:: bash
+
+    git clone https://github.com/apache/arrow.git arrow
+
+System requirements
+~~~~~~~~~~~~~~~~~~~
+
+Building pyarrow requires:
+
+* A C++11 compiler
+
+  * Linux: gcc >= 4.8 or clang >= 3.5
+  * OS X: XCode 6.4 or higher preferred
+
+* `CMake <https://cmake.org/>`_
+
+Python requirements
+~~~~~~~~~~~~~~~~~~~
+
+You will need Python (CPython) 2.7, 3.4, or 3.5 installed. Earlier releases and
+are not being targeted.
+
+.. note::
+    This library targets CPython only due to an emphasis on interoperability with
+    pandas and NumPy, which are only available for CPython.
+
+The build requires NumPy, Cython, and a few other Python dependencies:
+
+.. code-block:: bash
+
+    pip install cython
+    cd arrow/python
+    pip install -r requirements.txt
+
+Installing Arrow C++ library
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+First, you should choose an installation location for Arrow C++. In the future
+using the default system install location will work, but for now we are being
+explicit:
+
+.. code-block:: bash
+    
+    export ARROW_HOME=$HOME/local
+
+Now, we build Arrow:
+
+.. code-block:: bash
+
+    cd arrow/cpp
+    
+    mkdir dev-build
+    cd dev-build
+    
+    cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME ..
+    
+    make
+    
+    # Use sudo here if $ARROW_HOME requires it
+    make install
+
+To get the optional Parquet support, you should also build and install 
+`parquet-cpp <https://github.com/apache/parquet-cpp/blob/master/README.md>`_.
+
+Install `pyarrow`
+~~~~~~~~~~~~~~~~~
+
+
+.. code-block:: bash
+
+    cd arrow/python
+
+    # --with-parquet enable the Apache Parquet support in PyArrow
+    # --build-type=release disables debugging information and turns on
+    #       compiler optimizations for native code
+    python setup.py build_ext --with-parquet --build-type=release install
+    python setup.py install
+
+.. warning::
+    On XCode 6 and prior there are some known OS X `@rpath` issues. If you are
+    unable to import pyarrow, upgrading XCode may be the solution.
+
+.. note::
+    In development installations, you will also need to set a correct
+    ``LD_LIBARY_PATH``. This is most probably done with
+    ``export LD_LIBARY_PATH=$ARROW_HOME/lib:$LD_LIBARY_PATH``.
+
+
+.. code-block:: python
+    
+    In [1]: import pyarrow
+
+    In [2]: pyarrow.from_pylist([1,2,3])
+    Out[2]:
+    <pyarrow.array.Int64Array object at 0x7f899f3e60e8>
+    [
+      1,
+      2,
+      3
+    ]
+
diff --git a/python/doc/pandas.rst b/python/doc/pandas.rst
new file mode 100644
index 0000000000000..7c70074817835
--- /dev/null
+++ b/python/doc/pandas.rst
@@ -0,0 +1,114 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Pandas Interface
+================
+
+To interface with Pandas, PyArrow provides various conversion routines to
+consume Pandas structures and convert back to them.
+
+DataFrames
+----------
+
+The equivalent to a Pandas DataFrame in Arrow is a :class:`pyarrow.table.Table`.
+Both consist of a set of named columns of equal length. While Pandas only
+supports flat columns, the Table also provides nested columns, thus it can
+represent more data than a DataFrame, so a full conversion is not always possible.
+
+Conversion from a Table to a DataFrame is done by calling
+:meth:`pyarrow.table.Table.to_pandas`. The inverse is then achieved by using
+:meth:`pyarrow.from_pandas_dataframe`. This conversion routine provides the
+convience parameter ``timestamps_to_ms``. Although Arrow supports timestamps of
+different resolutions, Pandas only supports nanosecond timestamps and most
+other systems (e.g. Parquet) only work on millisecond timestamps. This parameter
+can be used to already do the time conversion during the Pandas to Arrow
+conversion.
+
+.. code-block:: python
+
+    import pyarrow as pa
+    import pandas as pd
+
+    df = pd.DataFrame({"a": [1, 2, 3]})
+    # Convert from Pandas to Arrow
+    table = pa.from_pandas_dataframe(df)
+    # Convert back to Pandas
+    df_new = table.to_pandas()
+
+
+Series
+------
+
+In Arrow, the most similar structure to a Pandas Series is an Array.
+It is a vector that contains data of the same type as linear memory. You can
+convert a Pandas Series to an Arrow Array using :meth:`pyarrow.array.from_pandas_series`.
+As Arrow Arrays are always nullable, you can supply an optional mask using
+the ``mask`` parameter to mark all null-entries.
+
+Type differences
+----------------
+
+With the current design of Pandas and Arrow, it is not possible to convert all
+column types unmodified. One of the main issues here is that Pandas has no
+support for nullable columns of arbitrary type. Also ``datetime64`` is currently
+fixed to nanosecond resolution. On the other side, Arrow might be still missing
+support for some types.
+
+Pandas -> Arrow Conversion
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++------------------------+--------------------------+
+| Source Type (Pandas)   | Destination Type (Arrow) |
++========================+==========================+
+| ``bool``               | ``BOOL``                 |
++------------------------+--------------------------+
+| ``(u)int{8,16,32,64}`` | ``(U)INT{8,16,32,64}``   |
++------------------------+--------------------------+
+| ``float32``            | ``FLOAT``                |
++------------------------+--------------------------+
+| ``float64``            | ``DOUBLE``               |
++------------------------+--------------------------+
+| ``str`` / ``unicode``  | ``STRING``               |
++------------------------+--------------------------+
+| ``pd.Timestamp``       | ``TIMESTAMP(unit=ns)``   |
++------------------------+--------------------------+
+| ``pd.Categorical``     | *not supported*          |
++------------------------+--------------------------+
+
+Arrow -> Pandas Conversion
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++-------------------------------------+--------------------------------------------------------+
+| Source Type (Arrow)                 | Destination Type (Pandas)                              |
++=====================================+========================================================+
+| ``BOOL``                            | ``bool``                                               |
++-------------------------------------+--------------------------------------------------------+
+| ``BOOL`` *with nulls*               | ``object`` (with values ``True``, ``False``, ``None``) |
++-------------------------------------+--------------------------------------------------------+
+| ``(U)INT{8,16,32,64}``              | ``(u)int{8,16,32,64}``                                 |
++-------------------------------------+--------------------------------------------------------+
+| ``(U)INT{8,16,32,64}`` *with nulls* | ``float64``                                            |
++-------------------------------------+--------------------------------------------------------+
+| ``FLOAT``                           | ``float32``                                            |
++-------------------------------------+--------------------------------------------------------+
+| ``DOUBLE``                          | ``float64``                                            |
++-------------------------------------+--------------------------------------------------------+
+| ``STRING``                          | ``str``                                                |
++-------------------------------------+--------------------------------------------------------+
+| ``TIMESTAMP(unit=*)``               | ``pd.Timestamp`` (``np.datetime64[ns]``)               |
++-------------------------------------+--------------------------------------------------------+
+
diff --git a/python/doc/parquet.rst b/python/doc/parquet.rst
new file mode 100644
index 0000000000000..674ed80f27ce3
--- /dev/null
+++ b/python/doc/parquet.rst
@@ -0,0 +1,66 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Reading/Writing Parquet files
+=============================
+
+If you have built ``pyarrow`` with Parquet support, i.e. ``parquet-cpp`` was
+found during the build, you can read files in the Parquet format to/from Arrow
+memory structures. The Parquet support code is located in the
+:mod:`pyarrow.parquet` module and your package needs to be built with the
+``--with-parquet`` flag for ``build_ext``.
+
+Reading Parquet
+---------------
+
+To read a Parquet file into Arrow memory, you can use the following code
+snippet. It will read the whole Parquet file into memory as an
+:class:`pyarrow.table.Table`.
+
+.. code-block:: python
+
+    import pyarrow
+    import pyarrow.parquet
+
+    A = pyarrow
+
+    table = A.parquet.read_table('<filename>')
+
+Writing Parquet
+---------------
+
+Given an instance of :class:`pyarrow.table.Table`, the most simple way to
+persist it to Parquet is by using the :meth:`pyarrow.parquet.write_table`
+method.
+
+.. code-block:: python
+
+    import pyarrow
+    import pyarrow.parquet
+
+    A = pyarrow
+
+    table = A.Table(..)
+    A.parquet.write_table(table, '<filename>')
+
+By default this will write the Table as a single RowGroup using ``DICTIONARY``
+encoding. To increase the potential of parallelism a query engine can process
+a Parquet file, set the ``chunk_size`` to a fraction of the total number of rows.
+
+If you also want to compress the columns, you can select a compression
+method using the ``compression`` argument. Typically, ``GZIP`` is the choice if
+you want to minimize size and ``SNAPPY`` for performance.
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index c71bc712bffb1..5459f26b80aa4 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -298,6 +298,8 @@ cdef class RecordBatch:
 
 cdef class Table:
     '''
+    A collection of top-level named, equal length Arrow arrays.
+
     Do not call this class's constructor directly.
     '''
 
@@ -335,6 +337,19 @@ cdef class Table:
 
     @staticmethod
     def from_arrays(names, arrays, name=None):
+        """
+        Construct a Table from Arrow Arrays
+
+        Parameters
+        ----------
+
+        names: list of str
+            Names for the table columns
+        arrays: list of pyarrow.array.Array
+            Equal-length arrays that should form the table.
+        name: str
+            (optional) name for the Table
+        """
         cdef:
             Array arr
             c_string c_name

From 48f9780a8677546cb143a09b25b0b57c1946ba07 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 11 Nov 2016 14:20:36 -0500
Subject: [PATCH 0198/1644] ARROW-375: Fix unicode Python 3 issue in columns
 argument of parquet.read_table

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #204 from wesm/ARROW-375 and squashes the following commits:

9e6f2a6 [Wes McKinney] BUG: convert unicode to utf8 bytes for column filtering
---
 python/pyarrow/parquet.pyx | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 2152f89474195..a6e3ac30684b4 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -93,15 +93,18 @@ cdef class ParquetReader:
             Integer index of the position of the column
         """
         cdef:
-            const FileMetaData* metadata = self.reader.get().parquet_reader().metadata()
+            const FileMetaData* metadata = (self.reader.get()
+                                            .parquet_reader().metadata())
             int i = 0
 
         if self.column_idx_map is None:
             self.column_idx_map = {}
             for i in range(0, metadata.num_columns()):
-                self.column_idx_map[str(metadata.schema().Column(i).path().get().ToDotString())] = i
+                col_bytes = tobytes(metadata.schema().Column(i)
+                                    .path().get().ToDotString())
+                self.column_idx_map[col_bytes] = i
 
-        return self.column_idx_map[column_name]
+        return self.column_idx_map[tobytes(column_name)]
 
     def read_column(self, int column_index):
         cdef:
@@ -109,7 +112,8 @@ cdef class ParquetReader:
             shared_ptr[CArray] carray
 
         with nogil:
-            check_status(self.reader.get().ReadFlatColumn(column_index, &carray))
+            check_status(self.reader.get()
+                         .ReadFlatColumn(column_index, &carray))
 
         array.init(carray)
         return array

From 78288b5fca8ff527257e487d45c7e68f7dbd8cd2 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 16 Nov 2016 16:18:50 -0500
Subject: [PATCH 0199/1644] ARROW-371: Handle pandas-nullable types correctly

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #205 from xhochy/ARROW-371 and squashes the following commits:

1f73e8b [Uwe L. Korn] ARROW-371: Handle pandas-nullable types correctly
---
 python/pyarrow/tests/test_convert_pandas.py | 22 +++++++++-
 python/src/pyarrow/adapters/pandas.cc       | 46 ++++++++++-----------
 2 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 55302996f4557..b527ca7e80816 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -165,7 +165,7 @@ def test_strings(self):
         expected = pd.DataFrame({'strings': values * repeats})
         self._check_pandas_roundtrip(df, expected)
 
-    def test_timestamps_notimezone(self):
+    def test_timestamps_notimezone_no_nulls(self):
         df = pd.DataFrame({
             'datetime64': np.array([
                 '2007-07-13T01:23:34.123',
@@ -184,6 +184,26 @@ def test_timestamps_notimezone(self):
             })
         self._check_pandas_roundtrip(df, timestamps_to_ms=False)
 
+    def test_timestamps_notimezone_nulls(self):
+        df = pd.DataFrame({
+            'datetime64': np.array([
+                '2007-07-13T01:23:34.123',
+                None,
+                '2010-08-13T05:46:57.437'],
+                dtype='datetime64[ms]')
+            })
+        df.info()
+        self._check_pandas_roundtrip(df, timestamps_to_ms=True)
+
+        df = pd.DataFrame({
+            'datetime64': np.array([
+                '2007-07-13T01:23:34.123456789',
+                None,
+                '2010-08-13T05:46:57.437699912'],
+                dtype='datetime64[ns]')
+            })
+        self._check_pandas_roundtrip(df, timestamps_to_ms=False)
+
     # def test_category(self):
     #     repeats = 1000
     #     values = [b'foo', None, u'bar', 'qux', np.nan]
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 6a3966b748806..1f5b7009e6a08 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -489,20 +489,20 @@ struct arrow_traits<arrow::Type::BOOL> {
   static constexpr int npy_type = NPY_BOOL;
   static constexpr bool supports_nulls = false;
   static constexpr bool is_boolean = true;
-  static constexpr bool is_integer = false;
-  static constexpr bool is_floating = false;
+  static constexpr bool is_pandas_numeric_not_nullable = false;
+  static constexpr bool is_pandas_numeric_nullable = false;
 };
 
-#define INT_DECL(TYPE)                                      \
-  template <>                                               \
-  struct arrow_traits<arrow::Type::TYPE> {              \
-    static constexpr int npy_type = NPY_##TYPE;             \
-    static constexpr bool supports_nulls = false;           \
-    static constexpr double na_value = NAN;                 \
-    static constexpr bool is_boolean = false;               \
-    static constexpr bool is_integer = true;                \
-    static constexpr bool is_floating = false;              \
-    typedef typename npy_traits<NPY_##TYPE>::value_type T;  \
+#define INT_DECL(TYPE)                                           \
+  template <>                                                    \
+  struct arrow_traits<arrow::Type::TYPE> {                       \
+    static constexpr int npy_type = NPY_##TYPE;                  \
+    static constexpr bool supports_nulls = false;                \
+    static constexpr double na_value = NAN;                      \
+    static constexpr bool is_boolean = false;                    \
+    static constexpr bool is_pandas_numeric_not_nullable = true; \
+    static constexpr bool is_pandas_numeric_nullable = false;    \
+    typedef typename npy_traits<NPY_##TYPE>::value_type T;       \
   };
 
 INT_DECL(INT8);
@@ -520,8 +520,8 @@ struct arrow_traits<arrow::Type::FLOAT> {
   static constexpr bool supports_nulls = true;
   static constexpr float na_value = NAN;
   static constexpr bool is_boolean = false;
-  static constexpr bool is_integer = false;
-  static constexpr bool is_floating = true;
+  static constexpr bool is_pandas_numeric_not_nullable = false;
+  static constexpr bool is_pandas_numeric_nullable = true;
   typedef typename npy_traits<NPY_FLOAT32>::value_type T;
 };
 
@@ -531,8 +531,8 @@ struct arrow_traits<arrow::Type::DOUBLE> {
   static constexpr bool supports_nulls = true;
   static constexpr double na_value = NAN;
   static constexpr bool is_boolean = false;
-  static constexpr bool is_integer = false;
-  static constexpr bool is_floating = true;
+  static constexpr bool is_pandas_numeric_not_nullable = false;
+  static constexpr bool is_pandas_numeric_nullable = true;
   typedef typename npy_traits<NPY_FLOAT64>::value_type T;
 };
 
@@ -542,8 +542,8 @@ struct arrow_traits<arrow::Type::TIMESTAMP> {
   static constexpr bool supports_nulls = true;
   static constexpr int64_t na_value = std::numeric_limits<int64_t>::min();
   static constexpr bool is_boolean = false;
-  static constexpr bool is_integer = true;
-  static constexpr bool is_floating = false;
+  static constexpr bool is_pandas_numeric_not_nullable = false;
+  static constexpr bool is_pandas_numeric_nullable = true;
   typedef typename npy_traits<NPY_DATETIME>::value_type T;
 };
 
@@ -552,8 +552,8 @@ struct arrow_traits<arrow::Type::STRING> {
   static constexpr int npy_type = NPY_OBJECT;
   static constexpr bool supports_nulls = true;
   static constexpr bool is_boolean = false;
-  static constexpr bool is_integer = false;
-  static constexpr bool is_floating = false;
+  static constexpr bool is_pandas_numeric_not_nullable = false;
+  static constexpr bool is_pandas_numeric_nullable = false;
 };
 
 
@@ -655,7 +655,7 @@ class ArrowDeserializer {
 
   template <int T2>
   inline typename std::enable_if<
-    arrow_traits<T2>::is_floating, Status>::type
+    arrow_traits<T2>::is_pandas_numeric_nullable, Status>::type
   ConvertValues(const std::shared_ptr<Array>& arr) {
     typedef typename arrow_traits<T2>::T T;
 
@@ -668,7 +668,7 @@ class ArrowDeserializer {
 
       T* out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
       for (int64_t i = 0; i < arr->length(); ++i) {
-        out_values[i] = arr->IsNull(i) ? NAN : in_values[i];
+        out_values[i] = arr->IsNull(i) ? arrow_traits<T2>::na_value : in_values[i];
       }
     } else {
       // Zero-Copy. We can pass the data pointer directly to NumPy.
@@ -683,7 +683,7 @@ class ArrowDeserializer {
   // Integer specialization
   template <int T2>
   inline typename std::enable_if<
-    arrow_traits<T2>::is_integer, Status>::type
+    arrow_traits<T2>::is_pandas_numeric_not_nullable, Status>::type
   ConvertValues(const std::shared_ptr<Array>& arr) {
     typedef typename arrow_traits<T2>::T T;
 

From 84170962712b976fd6f68f10ba55e219155a57db Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 18 Nov 2016 11:09:28 -0500
Subject: [PATCH 0200/1644] ARROW-367: converter json <=> Arrow file format for
 Integration tests

Author: Julien Le Dem <julien@dremio.com>

Closes #203 from julienledem/integration and squashes the following commits:

b3cd326 [Julien Le Dem] add license
fdbe03f [Julien Le Dem] ARROW-367: converter json <=> Arrow file format for Integration tests
---
 .../org/apache/arrow/tools/Integration.java   | 262 ++++++++++++++++++
 .../arrow/tools/ArrowFileTestFixtures.java    | 122 ++++++++
 .../apache/arrow/tools/TestFileRoundtrip.java | 101 +------
 .../apache/arrow/tools/TestIntegration.java   | 143 ++++++++++
 .../vector/file/json/JsonFileReader.java      |  37 +--
 .../vector/file/json/JsonFileWriter.java      |   3 +-
 6 files changed, 554 insertions(+), 114 deletions(-)
 create mode 100644 java/tools/src/main/java/org/apache/arrow/tools/Integration.java
 create mode 100644 java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
 create mode 100644 java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
new file mode 100644
index 0000000000000..29f0ee29e3ca8
--- /dev/null
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.arrow.tools;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.file.ArrowBlock;
+import org.apache.arrow.vector.file.ArrowFooter;
+import org.apache.arrow.vector.file.ArrowReader;
+import org.apache.arrow.vector.file.ArrowWriter;
+import org.apache.arrow.vector.file.json.JsonFileReader;
+import org.apache.arrow.vector.file.json.JsonFileWriter;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Objects;
+
+public class Integration {
+  private static final Logger LOGGER = LoggerFactory.getLogger(Integration.class);
+
+  public static void main(String[] args) {
+    try {
+      new Integration().run(args);
+    } catch (ParseException e) {
+      fatalError("Invalid parameters", e);
+    } catch (IOException e) {
+      fatalError("Error accessing files", e);
+    } catch (RuntimeException e) {
+      fatalError("Incompatible files", e);
+    }
+  }
+
+  private final Options options;
+
+  enum Command {
+    ARROW_TO_JSON(true, false) {
+      @Override
+      public void execute(File arrowFile, File jsonFile) throws IOException {
+        try(
+            BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+            FileInputStream fileInputStream = new FileInputStream(arrowFile);
+            ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), allocator);) {
+          ArrowFooter footer = arrowReader.readFooter();
+          Schema schema = footer.getSchema();
+          LOGGER.debug("Input file size: " + arrowFile.length());
+          LOGGER.debug("Found schema: " + schema);
+          try (JsonFileWriter writer = new JsonFileWriter(jsonFile);) {
+            writer.start(schema);
+            List<ArrowBlock> recordBatches = footer.getRecordBatches();
+            for (ArrowBlock rbBlock : recordBatches) {
+              try (ArrowRecordBatch inRecordBatch = arrowReader.readRecordBatch(rbBlock);
+                  VectorSchemaRoot root = new VectorSchemaRoot(schema, allocator);) {
+                VectorLoader vectorLoader = new VectorLoader(root);
+                vectorLoader.load(inRecordBatch);
+                writer.write(root);
+              }
+            }
+          }
+          LOGGER.debug("Output file size: " + jsonFile.length());
+        }
+      }
+    },
+    JSON_TO_ARROW(false, true) {
+      @Override
+      public void execute(File arrowFile, File jsonFile) throws IOException {
+        try (
+            BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+            JsonFileReader reader = new JsonFileReader(jsonFile, allocator);
+            ) {
+          Schema schema = reader.start();
+          LOGGER.debug("Input file size: " + jsonFile.length());
+          LOGGER.debug("Found schema: " + schema);
+          try (
+              FileOutputStream fileOutputStream = new FileOutputStream(arrowFile);
+              ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
+              ) {
+
+            // initialize vectors
+            VectorSchemaRoot root;
+            while ((root = reader.read()) != null) {
+              VectorUnloader vectorUnloader = new VectorUnloader(root);
+              try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();) {
+                arrowWriter.writeRecordBatch(recordBatch);
+              }
+              root.close();
+            }
+          }
+          LOGGER.debug("Output file size: " + arrowFile.length());
+        }
+      }
+    },
+    VALIDATE(true, true) {
+      @Override
+      public void execute(File arrowFile, File jsonFile) throws IOException {
+        try (
+            BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+            JsonFileReader jsonReader = new JsonFileReader(jsonFile, allocator);
+            FileInputStream fileInputStream = new FileInputStream(arrowFile);
+            ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), allocator);
+            ) {
+          Schema jsonSchema = jsonReader.start();
+          ArrowFooter footer = arrowReader.readFooter();
+          Schema arrowSchema = footer.getSchema();
+          LOGGER.debug("Arrow Input file size: " + arrowFile.length());
+          LOGGER.debug("ARROW schema: " + arrowSchema);
+          LOGGER.debug("JSON Input file size: " + jsonFile.length());
+          LOGGER.debug("JSON schema: " + jsonSchema);
+          compareSchemas(jsonSchema, arrowSchema);
+
+          List<ArrowBlock> recordBatches = footer.getRecordBatches();
+          Iterator<ArrowBlock> iterator = recordBatches.iterator();
+          VectorSchemaRoot jsonRoot;
+          while ((jsonRoot = jsonReader.read()) != null && iterator.hasNext()) {
+            ArrowBlock rbBlock = iterator.next();
+            try (ArrowRecordBatch inRecordBatch = arrowReader.readRecordBatch(rbBlock);
+                VectorSchemaRoot arrowRoot = new VectorSchemaRoot(arrowSchema, allocator);) {
+              VectorLoader vectorLoader = new VectorLoader(arrowRoot);
+              vectorLoader.load(inRecordBatch);
+              // TODO: compare
+              compare(arrowRoot, jsonRoot);
+            }
+            jsonRoot.close();
+          }
+          boolean hasMoreJSON = jsonRoot != null;
+          boolean hasMoreArrow = iterator.hasNext();
+          if (hasMoreJSON || hasMoreArrow) {
+            throw new IllegalArgumentException("Unexpected RecordBatches. J:" + hasMoreJSON + " A:" + hasMoreArrow);
+          }
+        }
+      }
+    };
+
+    public final boolean arrowExists;
+    public final boolean jsonExists;
+
+    Command(boolean arrowExists, boolean jsonExists) {
+      this.arrowExists = arrowExists;
+      this.jsonExists = jsonExists;
+    }
+
+    abstract public void execute(File arrowFile, File jsonFile) throws IOException;
+
+  }
+
+  Integration() {
+    this.options = new Options();
+    this.options.addOption("a", "arrow", true, "arrow file");
+    this.options.addOption("j", "json", true, "json file");
+    this.options.addOption("c", "command", true, "command to execute: " + Arrays.toString(Command.values()));
+  }
+
+  private File validateFile(String type, String fileName, boolean shouldExist) {
+    if (fileName == null) {
+      throw new IllegalArgumentException("missing " + type + " file parameter");
+    }
+    File f = new File(fileName);
+    if (shouldExist && (!f.exists() || f.isDirectory())) {
+      throw new IllegalArgumentException(type + " file not found: " + f.getAbsolutePath());
+    }
+    if (!shouldExist && f.exists()) {
+      throw new IllegalArgumentException(type + " file already exists: " + f.getAbsolutePath());
+    }
+    return f;
+  }
+
+  void run(String[] args) throws ParseException, IOException {
+    CommandLineParser parser = new PosixParser();
+    CommandLine cmd = parser.parse(options, args, false);
+
+
+    Command command = toCommand(cmd.getOptionValue("command"));
+    File arrowFile = validateFile("arrow", cmd.getOptionValue("arrow"), command.arrowExists);
+    File jsonFile = validateFile("json", cmd.getOptionValue("json"), command.jsonExists);
+    command.execute(arrowFile, jsonFile);
+  }
+
+  private Command toCommand(String commandName) {
+    try {
+      return Command.valueOf(commandName);
+    } catch (IllegalArgumentException e) {
+      throw new IllegalArgumentException("Unknown command: " + commandName + " expected one of " + Arrays.toString(Command.values()));
+    }
+  }
+
+  private static void fatalError(String message, Throwable e) {
+    System.err.println(message);
+    LOGGER.error(message, e);
+    System.exit(1);
+  }
+
+
+  private static void compare(VectorSchemaRoot arrowRoot, VectorSchemaRoot jsonRoot) {
+    compareSchemas(jsonRoot.getSchema(), arrowRoot.getSchema());
+    if (arrowRoot.getRowCount() != jsonRoot.getRowCount()) {
+      throw new IllegalArgumentException("Different row count:\n" + arrowRoot.getRowCount() + "\n" + jsonRoot.getRowCount());
+    }
+    List<FieldVector> arrowVectors = arrowRoot.getFieldVectors();
+    List<FieldVector> jsonVectors = jsonRoot.getFieldVectors();
+    if (arrowVectors.size() != jsonVectors.size()) {
+      throw new IllegalArgumentException("Different column count:\n" + arrowVectors.size() + "\n" + jsonVectors.size());
+    }
+    for (int i = 0; i < arrowVectors.size(); i++) {
+      Field field = arrowRoot.getSchema().getFields().get(i);
+      FieldVector arrowVector = arrowVectors.get(i);
+      FieldVector jsonVector = jsonVectors.get(i);
+      int valueCount = arrowVector.getAccessor().getValueCount();
+      if (valueCount != jsonVector.getAccessor().getValueCount()) {
+        throw new IllegalArgumentException("Different value count for field " + field + " : " + valueCount + " != " + jsonVector.getAccessor().getValueCount());
+      }
+      for (int j = 0; j < valueCount; j++) {
+        Object arrow = arrowVector.getAccessor().getObject(j);
+        Object json = jsonVector.getAccessor().getObject(j);
+        if (!Objects.equal(arrow, json)) {
+          throw new IllegalArgumentException(
+              "Different values in column:\n" + field + " at index " + j + ": " + arrow + " != " + json);
+        }
+      }
+    }
+  }
+
+  private static void compareSchemas(Schema jsonSchema, Schema arrowSchema) {
+    if (!arrowSchema.equals(jsonSchema)) {
+      throw new IllegalArgumentException("Different schemas:\n" + arrowSchema + "\n" + jsonSchema);
+    }
+  }
+}
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
new file mode 100644
index 0000000000000..4cfc52fe08631
--- /dev/null
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.arrow.tools;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.file.ArrowBlock;
+import org.apache.arrow.vector.file.ArrowFooter;
+import org.apache.arrow.vector.file.ArrowReader;
+import org.apache.arrow.vector.file.ArrowWriter;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Assert;
+
+public class ArrowFileTestFixtures {
+  static final int COUNT = 10;
+
+  static void writeData(int count, MapVector parent) {
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    for (int i = 0; i < count; i++) {
+      intWriter.setPosition(i);
+      intWriter.writeInt(i);
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+    }
+    writer.setValueCount(count);
+  }
+
+  static void validateOutput(File testOutFile, BufferAllocator allocator) throws Exception {
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        FileInputStream fileInputStream = new FileInputStream(testOutFile);
+        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        ) {
+      ArrowFooter footer = arrowReader.readFooter();
+      Schema schema = footer.getSchema();
+
+      // initialize vectors
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, readerAllocator)) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+
+        List<ArrowBlock> recordBatches = footer.getRecordBatches();
+        for (ArrowBlock rbBlock : recordBatches) {
+          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+            vectorLoader.load(recordBatch);
+          }
+          validateContent(COUNT, root);
+        }
+      }
+    }
+  }
+
+  static void validateContent(int count, VectorSchemaRoot root) {
+    Assert.assertEquals(count, root.getRowCount());
+    for (int i = 0; i < count; i++) {
+      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
+    }
+  }
+
+  static void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
+    Schema schema = new Schema(parent.getField().getChildren());
+    int valueCount = parent.getAccessor().getValueCount();
+    List<FieldVector> fields = parent.getChildrenFromFields();
+    VectorUnloader vectorUnloader = new VectorUnloader(schema, valueCount, fields);
+    try (
+        FileOutputStream fileOutputStream = new FileOutputStream(file);
+        ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
+        ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+            ) {
+      arrowWriter.writeRecordBatch(recordBatch);
+    }
+  }
+
+
+  static void writeInput(File testInFile, BufferAllocator allocator) throws FileNotFoundException, IOException {
+    int count = ArrowFileTestFixtures.COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+      writeData(count, parent);
+      write(parent.getChild("root"), testInFile);
+    }
+  }
+}
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
index 339725e5af1e0..ee39f5e92c7b0 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
@@ -18,42 +18,21 @@
  */
 package org.apache.arrow.tools;
 
+import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput;
+import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
 import static org.junit.Assert.assertEquals;
 
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.VectorLoader;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
-import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.complex.writer.BigIntWriter;
-import org.apache.arrow.vector.complex.writer.IntWriter;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowFooter;
-import org.apache.arrow.vector.file.ArrowReader;
-import org.apache.arrow.vector.file.ArrowWriter;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 public class TestFileRoundtrip {
-  private static final int COUNT = 10;
 
   @Rule
   public TemporaryFolder testFolder = new TemporaryFolder();
@@ -70,90 +49,18 @@ public void tearDown() {
     allocator.close();
   }
 
-  private void writeData(int count, MapVector parent) {
-    ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
-    IntWriter intWriter = rootWriter.integer("int");
-    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
-    for (int i = 0; i < count; i++) {
-      intWriter.setPosition(i);
-      intWriter.writeInt(i);
-      bigIntWriter.setPosition(i);
-      bigIntWriter.writeBigInt(i);
-    }
-    writer.setValueCount(count);
-  }
-
   @Test
   public void test() throws Exception {
     File testInFile = testFolder.newFile("testIn.arrow");
     File testOutFile = testFolder.newFile("testOut.arrow");
 
-    writeInput(testInFile);
+    writeInput(testInFile, allocator);
 
     String[] args = { "-i", testInFile.getAbsolutePath(), "-o",  testOutFile.getAbsolutePath()};
     int result = new FileRoundtrip(System.out, System.err).run(args);
     assertEquals(0, result);
 
-    validateOutput(testOutFile);
-  }
-
-  private void validateOutput(File testOutFile) throws Exception {
-    // read
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        FileInputStream fileInputStream = new FileInputStream(testOutFile);
-        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        ) {
-      ArrowFooter footer = arrowReader.readFooter();
-      Schema schema = footer.getSchema();
-
-      // initialize vectors
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, readerAllocator)) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-
-        List<ArrowBlock> recordBatches = footer.getRecordBatches();
-        for (ArrowBlock rbBlock : recordBatches) {
-          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-            vectorLoader.load(recordBatch);
-          }
-          validateContent(COUNT, root);
-        }
-      }
-    }
-  }
-
-  private void validateContent(int count, VectorSchemaRoot root) {
-    Assert.assertEquals(count, root.getRowCount());
-    for (int i = 0; i < count; i++) {
-      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
-      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
-    }
-  }
-
-  public void writeInput(File testInFile) throws FileNotFoundException, IOException {
-    int count = COUNT;
-    try (
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
-      writeData(count, parent);
-      write(parent.getChild("root"), testInFile);
-    }
-  }
-
-  private void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
-    Schema schema = new Schema(parent.getField().getChildren());
-    int valueCount = parent.getAccessor().getValueCount();
-    List<FieldVector> fields = parent.getChildrenFromFields();
-    VectorUnloader vectorUnloader = new VectorUnloader(schema, valueCount, fields);
-    try (
-        FileOutputStream fileOutputStream = new FileOutputStream(file);
-        ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
-        ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
-            ) {
-      arrowWriter.writeRecordBatch(recordBatch);
-    }
+    validateOutput(testOutFile, allocator);
   }
 
 }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
new file mode 100644
index 0000000000000..bb69ed1498e26
--- /dev/null
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.arrow.tools;
+
+import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput;
+import static org.apache.arrow.tools.ArrowFileTestFixtures.write;
+import static org.apache.arrow.tools.ArrowFileTestFixtures.writeData;
+import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.tools.Integration.Command;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class TestIntegration {
+
+  @Rule
+  public TemporaryFolder testFolder = new TemporaryFolder();
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Integer.MAX_VALUE);
+  }
+
+  @After
+  public void tearDown() {
+    allocator.close();
+  }
+
+  @Test
+  public void testValid() throws Exception {
+    File testInFile = testFolder.newFile("testIn.arrow");
+    File testJSONFile = testFolder.newFile("testOut.json");
+    testJSONFile.delete();
+    File testOutFile = testFolder.newFile("testOut.arrow");
+    testOutFile.delete();
+
+    // generate an arow file
+    writeInput(testInFile, allocator);
+
+    Integration integration = new Integration();
+
+    // convert it to json
+    String[] args1 = { "-arrow", testInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    integration.run(args1);
+
+    // convert back to arrow
+    String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
+    integration.run(args2);
+
+    // check it is the same
+    validateOutput(testOutFile, allocator);
+
+    // validate arrow against json
+    String[] args3 = { "-arrow", testInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()};
+    integration.run(args3);
+  }
+
+  @Test
+  public void testInvalid() throws Exception {
+    File testValidInFile = testFolder.newFile("testValidIn.arrow");
+    File testInvalidInFile = testFolder.newFile("testInvalidIn.arrow");
+    File testJSONFile = testFolder.newFile("testInvalidOut.json");
+    testJSONFile.delete();
+
+    // generate an arrow file
+    writeInput(testValidInFile, allocator);
+    // generate a different arrow file
+    writeInput2(testInvalidInFile, allocator);
+
+    Integration integration = new Integration();
+
+    // convert the "valid" file to json
+    String[] args1 = { "-arrow", testValidInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    integration.run(args1);
+
+    // compare the "invalid" file to the "valid" json
+    String[] args3 = { "-arrow", testInvalidInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()};
+    // this should fail
+    try {
+      integration.run(args3);
+      fail("should have failed");
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage(), e.getMessage().contains("Different values in column"));
+      Assert.assertTrue(e.getMessage(), e.getMessage().contains("999"));
+    }
+
+  }
+
+  static void writeInput2(File testInFile, BufferAllocator allocator) throws FileNotFoundException, IOException {
+    int count = ArrowFileTestFixtures.COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+      writeData(count, parent);
+      ComplexWriter writer = new ComplexWriterImpl("root", parent);
+      MapWriter rootWriter = writer.rootAsMap();
+      IntWriter intWriter = rootWriter.integer("int");
+      BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+      intWriter.setPosition(5);
+      intWriter.writeInt(999);
+      bigIntWriter.setPosition(4);
+      bigIntWriter.writeBigInt(777L);
+      writer.setValueCount(count);
+      write(parent.getChild("root"), testInFile);
+    }
+  }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 859a3a0e80a50..f07b517250732 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -56,7 +56,7 @@
 import com.fasterxml.jackson.databind.MappingJsonFactory;
 import com.google.common.base.Objects;
 
-public class JsonFileReader {
+public class JsonFileReader implements AutoCloseable {
   private final File inputFile;
   private final JsonParser parser;
   private final BufferAllocator allocator;
@@ -81,23 +81,29 @@ public Schema start() throws JsonParseException, IOException {
   }
 
   public VectorSchemaRoot read() throws IOException {
-    VectorSchemaRoot recordBatch = new VectorSchemaRoot(schema, allocator);
-    readToken(START_OBJECT);
-    {
-      int count = readNextField("count", Integer.class);
-      recordBatch.setRowCount(count);
-      nextFieldIs("columns");
-      readToken(START_ARRAY);
+    JsonToken t = parser.nextToken();
+    if (t == START_OBJECT) {
+      VectorSchemaRoot recordBatch = new VectorSchemaRoot(schema, allocator);
       {
-        for (Field field : schema.getFields()) {
-          FieldVector vector = recordBatch.getVector(field.getName());
-          readVector(field, vector);
+        int count = readNextField("count", Integer.class);
+        recordBatch.setRowCount(count);
+        nextFieldIs("columns");
+        readToken(START_ARRAY);
+        {
+          for (Field field : schema.getFields()) {
+            FieldVector vector = recordBatch.getVector(field.getName());
+            readVector(field, vector);
+          }
         }
+        readToken(END_ARRAY);
       }
-      readToken(END_ARRAY);
+      readToken(END_OBJECT);
+      return recordBatch;
+    } else if (t == END_ARRAY) {
+      return null;
+    } else {
+      throw new IllegalArgumentException("Invalid token: " + t);
     }
-    readToken(END_OBJECT);
-    return recordBatch;
   }
 
   private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException {
@@ -194,9 +200,8 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
     }
   }
 
+  @Override
   public void close() throws IOException {
-    readToken(END_ARRAY);
-    readToken(END_OBJECT);
     parser.close();
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index 47c1a7dabef11..812b3da32f83c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -38,7 +38,7 @@
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
 import com.fasterxml.jackson.databind.MappingJsonFactory;
 
-public class JsonFileWriter {
+public class JsonFileWriter implements AutoCloseable {
 
   public static final class JSONWriteConfig {
     private final boolean pretty;
@@ -158,6 +158,7 @@ private void writeValueToGenerator(ValueVector valueVector, int i) throws IOExce
     }
   }
 
+  @Override
   public void close() throws IOException {
     generator.writeEndArray();
     generator.writeEndObject();

From ed6ec3b76e1ac27fab85cd4bc74fbd61e8dfb27f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 18 Nov 2016 14:58:46 -0500
Subject: [PATCH 0201/1644] ARROW-373: [C++] JSON serialization format for
 testing

C++ version of ARROW-372

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #202 from wesm/ARROW-373 and squashes the following commits:

d13a05f [Wes McKinney] Compiler warning
72c24fe [Wes McKinney] Add a minimal literal JSON example
a2cf47b [Wes McKinney] cpplint
3d9fcc2 [Wes McKinney] Complete round trip json file test with multiple record batches
2753449 [Wes McKinney] Complete draft json roundtrip implementation. tests not complete yet
3d6bbbd [Wes McKinney] Start high level writer scaffold
6bbd669 [Wes McKinney] Tweaks
e2e86b5 [Wes McKinney] Test JSON array roundtrip for numeric types, strings, lists, structs
82f108b [Wes McKinney] Refactoring. Array test scaffold
0891378 [Wes McKinney] Declare loop variables
6566343 [Wes McKinney] Recursively construct children for list/struct
35c2f85 [Wes McKinney] Refactoring. Start drafting string/list reader
f26402a [Wes McKinney] Install type_traits.h. cpplint
4fc7294 [Wes McKinney] Refactoring, type attribute consistency. Array reader compiles
2c93cce [Wes McKinney] WIP JSON array reader code path
932ba7a [Wes McKinney] Add ArrayVisitor methods, add enough metaprogramming to detect presence of c_type type member
15c1094 [Wes McKinney] Add type traits, refactoring, drafting json array writing. not working yet
209ba48 [Wes McKinney] More types refactoring. Strange linker error in pyarrow
379da3c [Wes McKinney] Implement union metadata JSON serialization
5fbea41 [Wes McKinney] Implement some more json types and add convenience factory functions
1c08233 [Wes McKinney] JSON schema roundtrip passing for many types
86c9559 [Wes McKinney] Add convenience factory functions for common types
3b9d14e [Wes McKinney] Add type-specific JSON metadata to schema writer
820b0f2 [Wes McKinney] Drafting JSON schema read/write
68ee7ab [Wes McKinney] Move forward declarations into type_fwd.h
1edf2a9 [Wes McKinney] Prototyping out visitor pattern for json serialization
24c1d5d [Wes McKinney] Some Types refactoring, add TypeVisitor abstract class. Add RapidJSON as external project
---
 cpp/CMakeLists.txt                    |   19 +
 cpp/src/arrow/CMakeLists.txt          |    2 +
 cpp/src/arrow/array.cc                |   15 +
 cpp/src/arrow/array.h                 |   12 +
 cpp/src/arrow/column-test.cc          |    1 +
 cpp/src/arrow/io/hdfs.cc              |    8 +-
 cpp/src/arrow/io/libhdfs_shim.cc      |   26 +-
 cpp/src/arrow/ipc/CMakeLists.txt      |    7 +
 cpp/src/arrow/ipc/adapter.cc          |    2 +-
 cpp/src/arrow/ipc/ipc-json-test.cc    |  353 ++++++++
 cpp/src/arrow/ipc/json-internal.cc    | 1113 +++++++++++++++++++++++++
 cpp/src/arrow/ipc/json-internal.h     |  111 +++
 cpp/src/arrow/ipc/json.cc             |  219 +++++
 cpp/src/arrow/ipc/json.h              |   92 ++
 cpp/src/arrow/ipc/test-common.h       |   14 +-
 cpp/src/arrow/schema-test.cc          |   52 +-
 cpp/src/arrow/schema.cc               |   15 +
 cpp/src/arrow/schema.h                |   12 +-
 cpp/src/arrow/test-util.h             |   51 +-
 cpp/src/arrow/type.cc                 |  122 ++-
 cpp/src/arrow/type.h                  |  338 ++++++--
 cpp/src/arrow/type_fwd.h              |  157 ++++
 cpp/src/arrow/type_traits.h           |  197 +++++
 cpp/src/arrow/types/CMakeLists.txt    |    1 -
 cpp/src/arrow/types/collection.h      |   41 -
 cpp/src/arrow/types/datetime.h        |   37 +-
 cpp/src/arrow/types/decimal.h         |   14 +-
 cpp/src/arrow/types/list-test.cc      |    2 +-
 cpp/src/arrow/types/list.cc           |    4 +
 cpp/src/arrow/types/list.h            |    8 +-
 cpp/src/arrow/types/primitive-test.cc |   36 +-
 cpp/src/arrow/types/primitive.cc      |   97 ++-
 cpp/src/arrow/types/primitive.h       |  190 ++---
 cpp/src/arrow/types/string-test.cc    |   12 +-
 cpp/src/arrow/types/string.cc         |   16 +-
 cpp/src/arrow/types/string.h          |   24 +-
 cpp/src/arrow/types/struct-test.cc    |    2 +-
 cpp/src/arrow/types/struct.cc         |    4 +
 cpp/src/arrow/types/struct.h          |    4 +
 cpp/src/arrow/types/test-common.h     |   16 +
 cpp/src/arrow/types/union.cc          |   23 +-
 cpp/src/arrow/types/union.h           |   21 -
 cpp/src/arrow/util/logging.h          |    4 +-
 format/Metadata.md                    |    5 +
 44 files changed, 3049 insertions(+), 450 deletions(-)
 create mode 100644 cpp/src/arrow/ipc/ipc-json-test.cc
 create mode 100644 cpp/src/arrow/ipc/json-internal.cc
 create mode 100644 cpp/src/arrow/ipc/json-internal.h
 create mode 100644 cpp/src/arrow/ipc/json.cc
 create mode 100644 cpp/src/arrow/ipc/json.h
 create mode 100644 cpp/src/arrow/type_fwd.h
 create mode 100644 cpp/src/arrow/type_traits.h
 delete mode 100644 cpp/src/arrow/types/collection.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6f954830b6334..0bff7528578d1 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -545,6 +545,25 @@ if(ARROW_BUILD_BENCHMARKS)
   endif()
 endif()
 
+# RapidJSON, header only dependency
+if("$ENV{RAPIDJSON_HOME}" STREQUAL "")
+  ExternalProject_Add(rapidjson_ep
+    PREFIX "${CMAKE_BINARY_DIR}"
+    URL "https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz"
+    URL_MD5 "badd12c511e081fec6c89c43a7027bce"
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    BUILD_IN_SOURCE 1
+    INSTALL_COMMAND "")
+
+  ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR)
+  set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include")
+else()
+  set(RAPIDJSON_INCLUDE_DIR "$ENV{RAPIDJSON_HOME}/include")
+endif()
+message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
+include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
+
 ## Google PerfTools
 ##
 ## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index a9b2feca28cb7..81851bc5b3eb1 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -24,6 +24,8 @@ install(FILES
   schema.h
   table.h
   type.h
+  type_fwd.h
+  type_traits.h
   test-util.h
   DESTINATION include/arrow)
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index e432a53781f17..3262425e99b66 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -18,6 +18,7 @@
 #include "arrow/array.h"
 
 #include <cstdint>
+#include <cstring>
 
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
@@ -25,6 +26,16 @@
 
 namespace arrow {
 
+Status GetEmptyBitmap(
+    MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result) {
+  auto buffer = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(buffer->Resize(BitUtil::BytesForBits(length)));
+  memset(buffer->mutable_data(), 0, buffer->size());
+
+  *result = buffer;
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // Base array class
 
@@ -66,4 +77,8 @@ bool NullArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_st
   return true;
 }
 
+Status NullArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index ff37323f60519..ff2b70e213b1b 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -29,6 +29,8 @@
 namespace arrow {
 
 class Buffer;
+class MemoryPool;
+class MutableBuffer;
 class Status;
 
 // Immutable data array with some logical type and some length. Any memory is
@@ -70,6 +72,8 @@ class ARROW_EXPORT Array {
   // returning Status::OK.  This can be an expensive check.
   virtual Status Validate() const;
 
+  virtual Status Accept(ArrayVisitor* visitor) const = 0;
+
  protected:
   std::shared_ptr<DataType> type_;
   int32_t null_count_;
@@ -86,6 +90,8 @@ class ARROW_EXPORT Array {
 // Degenerate null type Array
 class ARROW_EXPORT NullArray : public Array {
  public:
+  using TypeClass = NullType;
+
   NullArray(const std::shared_ptr<DataType>& type, int32_t length)
       : Array(type, length, length, nullptr) {}
 
@@ -94,9 +100,15 @@ class ARROW_EXPORT NullArray : public Array {
   bool Equals(const std::shared_ptr<Array>& arr) const override;
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_index,
       const std::shared_ptr<Array>& arr) const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
 };
 
 typedef std::shared_ptr<Array> ArrayPtr;
+
+Status ARROW_EXPORT GetEmptyBitmap(
+    MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result);
+
 }  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
index 1edf313d49bf6..ac3636d1b6dab 100644
--- a/cpp/src/arrow/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -22,6 +22,7 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/schema.h"
 #include "arrow/test-util.h"
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 6490a7574eea2..13491e780e21b 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -289,13 +289,9 @@ class HdfsClient::HdfsClientImpl {
 
     // connect to HDFS with the builder object
     hdfsBuilder* builder = hdfsNewBuilder();
-    if (!config->host.empty()) {
-      hdfsBuilderSetNameNode(builder, config->host.c_str());
-    }
+    if (!config->host.empty()) { hdfsBuilderSetNameNode(builder, config->host.c_str()); }
     hdfsBuilderSetNameNodePort(builder, config->port);
-    if (!config->user.empty()) {
-      hdfsBuilderSetUserName(builder, config->user.c_str());
-    }
+    if (!config->user.empty()) { hdfsBuilderSetUserName(builder, config->user.c_str()); }
     if (!config->kerb_ticket.empty()) {
       hdfsBuilderSetKerbTicketCachePath(builder, config->kerb_ticket.c_str());
     }
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
index 1fee595d0718b..36b8a4ec980a9 100644
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -74,12 +74,9 @@ static HINSTANCE libjvm_handle = NULL;
 // NOTE(wesm): cpplint does not like use of short and other imprecise C types
 
 static hdfsBuilder* (*ptr_hdfsNewBuilder)(void) = NULL;
-static void (*ptr_hdfsBuilderSetNameNode)(
-    hdfsBuilder* bld, const char* nn) = NULL;
-static void (*ptr_hdfsBuilderSetNameNodePort)(
-    hdfsBuilder* bld, tPort port) = NULL;
-static void (*ptr_hdfsBuilderSetUserName)(
-    hdfsBuilder* bld, const char* userName) = NULL;
+static void (*ptr_hdfsBuilderSetNameNode)(hdfsBuilder* bld, const char* nn) = NULL;
+static void (*ptr_hdfsBuilderSetNameNodePort)(hdfsBuilder* bld, tPort port) = NULL;
+static void (*ptr_hdfsBuilderSetUserName)(hdfsBuilder* bld, const char* userName) = NULL;
 static void (*ptr_hdfsBuilderSetKerbTicketCachePath)(
     hdfsBuilder* bld, const char* kerbTicketCachePath) = NULL;
 static hdfsFS (*ptr_hdfsBuilderConnect)(hdfsBuilder* bld) = NULL;
@@ -173,9 +170,9 @@ void hdfsBuilderSetUserName(hdfsBuilder* bld, const char* userName) {
   ptr_hdfsBuilderSetUserName(bld, userName);
 }
 
-void hdfsBuilderSetKerbTicketCachePath(hdfsBuilder* bld,
-    const char* kerbTicketCachePath) {
-  ptr_hdfsBuilderSetKerbTicketCachePath(bld , kerbTicketCachePath);
+void hdfsBuilderSetKerbTicketCachePath(
+    hdfsBuilder* bld, const char* kerbTicketCachePath) {
+  ptr_hdfsBuilderSetKerbTicketCachePath(bld, kerbTicketCachePath);
 }
 
 hdfsFS hdfsBuilderConnect(hdfsBuilder* bld) {
@@ -364,7 +361,7 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
   std::vector<fs::path> libhdfs_potential_paths;
   std::string file_name;
 
-  // OS-specific file name
+// OS-specific file name
 #ifdef __WIN32
   file_name = "hdfs.dll";
 #elif __APPLE__
@@ -374,10 +371,7 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
 #endif
 
   // Common paths
-  std::vector<fs::path> search_paths = {
-      fs::path(""),
-      fs::path(".")
-  };
+  std::vector<fs::path> search_paths = {fs::path(""), fs::path(".")};
 
   // Path from environment variable
   const char* hadoop_home = std::getenv("HADOOP_HOME");
@@ -387,9 +381,7 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
   }
 
   const char* libhdfs_dir = std::getenv("ARROW_LIBHDFS_DIR");
-  if (libhdfs_dir != nullptr) {
-    search_paths.push_back(fs::path(libhdfs_dir));
-  }
+  if (libhdfs_dir != nullptr) { search_paths.push_back(fs::path(libhdfs_dir)); }
 
   // All paths with file name
   for (auto& path : search_paths) {
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index d2db339de7ea2..6955bcb6c233e 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -34,6 +34,8 @@ set(ARROW_IPC_TEST_LINK_LIBS
 set(ARROW_IPC_SRCS
   adapter.cc
   file.cc
+  json.cc
+  json-internal.cc
   metadata.cc
   metadata-internal.cc
 )
@@ -79,6 +81,10 @@ ADD_ARROW_TEST(ipc-metadata-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-metadata-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
+ADD_ARROW_TEST(ipc-json-test)
+ARROW_TEST_LINK_LIBRARIES(ipc-json-test
+  ${ARROW_IPC_TEST_LINK_LIBS})
+
 # make clean will delete the generated file
 set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
 
@@ -114,6 +120,7 @@ add_dependencies(arrow_objlib metadata_fbs)
 install(FILES
   adapter.h
   file.h
+  json.h
   metadata.h
   DESTINATION include/arrow/ipc)
 
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 74786bf85ffb4..da718c08d5480 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -106,7 +106,7 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
     buffers->push_back(binary_arr->data());
   } else if (arr->type_enum() == Type::LIST) {
     const auto list_arr = static_cast<const ListArray*>(arr);
-    buffers->push_back(list_arr->offset_buffer());
+    buffers->push_back(list_arr->offsets());
     RETURN_NOT_OK(VisitArray(
         list_arr->values().get(), field_nodes, buffers, max_recursion_depth - 1));
   } else if (arr->type_enum() == Type::STRUCT) {
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
new file mode 100644
index 0000000000000..a51371c62005b
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -0,0 +1,353 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/ipc/json-internal.h"
+#include "arrow/ipc/json.h"
+#include "arrow/table.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+void TestSchemaRoundTrip(const Schema& schema) {
+  rj::StringBuffer sb;
+  rj::Writer<rj::StringBuffer> writer(sb);
+
+  ASSERT_OK(WriteJsonSchema(schema, &writer));
+
+  rj::Document d;
+  d.Parse(sb.GetString());
+
+  std::shared_ptr<Schema> out;
+  ASSERT_OK(ReadJsonSchema(d, &out));
+
+  ASSERT_TRUE(schema.Equals(out));
+}
+
+void TestArrayRoundTrip(const Array& array) {
+  static std::string name = "dummy";
+
+  rj::StringBuffer sb;
+  rj::Writer<rj::StringBuffer> writer(sb);
+
+  ASSERT_OK(WriteJsonArray(name, array, &writer));
+
+  std::string array_as_json = sb.GetString();
+
+  rj::Document d;
+  d.Parse(array_as_json);
+
+  if (d.HasParseError()) { FAIL() << "JSON parsing failed"; }
+
+  std::shared_ptr<Array> out;
+  ASSERT_OK(ReadJsonArray(default_memory_pool(), d, array.type(), &out));
+
+  ASSERT_TRUE(array.Equals(out)) << array_as_json;
+}
+
+template <typename T, typename ValueType>
+void CheckPrimitive(const std::shared_ptr<DataType>& type,
+    const std::vector<bool>& is_valid, const std::vector<ValueType>& values) {
+  MemoryPool* pool = default_memory_pool();
+  typename TypeTraits<T>::BuilderType builder(pool, type);
+
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid[i]) {
+      ASSERT_OK(builder.Append(values[i]));
+    } else {
+      ASSERT_OK(builder.AppendNull());
+    }
+  }
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+  TestArrayRoundTrip(*array.get());
+}
+
+template <typename TYPE, typename C_TYPE>
+void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& is_valid,
+    const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
+  std::shared_ptr<Buffer> values_buffer;
+  std::shared_ptr<Buffer> values_bitmap;
+
+  ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
+  ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap));
+
+  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
+
+  int32_t null_count = 0;
+  for (bool val : is_valid) {
+    if (!val) { ++null_count; }
+  }
+
+  *out = std::make_shared<ArrayType>(type, static_cast<int32_t>(values.size()),
+      values_buffer, null_count, values_bitmap);
+}
+
+TEST(TestJsonSchemaWriter, FlatTypes) {
+  std::vector<std::shared_ptr<Field>> fields = {field("f0", int8()),
+      field("f1", int16(), false), field("f2", int32()), field("f3", int64(), false),
+      field("f4", uint8()), field("f5", uint16()), field("f6", uint32()),
+      field("f7", uint64()), field("f8", float32()), field("f9", float64()),
+      field("f10", utf8()), field("f11", binary()), field("f12", list(int32())),
+      field("f13", struct_({field("s1", int32()), field("s2", utf8())})),
+      field("f14", date()), field("f15", timestamp(TimeUnit::NANO)),
+      field("f16", time(TimeUnit::MICRO)),
+      field("f17", union_({field("u1", int8()), field("u2", time(TimeUnit::MILLI))},
+                       {0, 1}, UnionMode::DENSE))};
+
+  Schema schema(fields);
+  TestSchemaRoundTrip(schema);
+}
+
+template <typename T>
+void PrimitiveTypesCheckOne() {
+  using c_type = typename T::c_type;
+
+  std::vector<bool> is_valid = {true, false, true, true, true, false, true, true};
+  std::vector<c_type> values = {0, 1, 2, 3, 4, 5, 6, 7};
+  CheckPrimitive<T, c_type>(std::make_shared<T>(), is_valid, values);
+}
+
+TEST(TestJsonArrayWriter, PrimitiveTypes) {
+  PrimitiveTypesCheckOne<Int8Type>();
+  PrimitiveTypesCheckOne<Int16Type>();
+  PrimitiveTypesCheckOne<Int32Type>();
+  PrimitiveTypesCheckOne<Int64Type>();
+  PrimitiveTypesCheckOne<UInt8Type>();
+  PrimitiveTypesCheckOne<UInt16Type>();
+  PrimitiveTypesCheckOne<UInt32Type>();
+  PrimitiveTypesCheckOne<UInt64Type>();
+  PrimitiveTypesCheckOne<FloatType>();
+  PrimitiveTypesCheckOne<DoubleType>();
+
+  std::vector<bool> is_valid = {true, false, true, true, true, false, true, true};
+  std::vector<std::string> values = {"foo", "bar", "", "baz", "qux", "foo", "a", "1"};
+
+  CheckPrimitive<StringType, std::string>(utf8(), is_valid, values);
+  CheckPrimitive<BinaryType, std::string>(binary(), is_valid, values);
+}
+
+TEST(TestJsonArrayWriter, NestedTypes) {
+  auto value_type = int32();
+
+  std::vector<bool> values_is_valid = {true, false, true, true, false, true, true};
+  std::vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+
+  std::shared_ptr<Array> values_array;
+  MakeArray<Int32Type, int32_t>(int32(), values_is_valid, values, &values_array);
+
+  // List
+  std::vector<bool> list_is_valid = {true, false, true, true, true};
+  std::vector<int32_t> offsets = {0, 0, 0, 1, 4, 7};
+
+  std::shared_ptr<Buffer> list_bitmap;
+  ASSERT_OK(test::GetBitmapFromBoolVector(list_is_valid, &list_bitmap));
+  std::shared_ptr<Buffer> offsets_buffer = test::GetBufferFromVector(offsets);
+
+  ListArray list_array(list(value_type), 5, offsets_buffer, values_array, 1, list_bitmap);
+
+  TestArrayRoundTrip(list_array);
+
+  // Struct
+  std::vector<bool> struct_is_valid = {true, false, true, true, true, false, true};
+  std::shared_ptr<Buffer> struct_bitmap;
+  ASSERT_OK(test::GetBitmapFromBoolVector(struct_is_valid, &struct_bitmap));
+
+  auto struct_type =
+      struct_({field("f1", int32()), field("f2", int32()), field("f3", int32())});
+
+  std::vector<std::shared_ptr<Array>> fields = {values_array, values_array, values_array};
+  StructArray struct_array(
+      struct_type, static_cast<int>(struct_is_valid.size()), fields, 2, struct_bitmap);
+  TestArrayRoundTrip(struct_array);
+}
+
+// Data generation for test case below
+void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
+    std::vector<std::shared_ptr<Array>>* arrays) {
+  std::vector<bool> is_valid;
+  test::random_is_valid(num_rows, 0.25, &is_valid);
+
+  std::vector<int8_t> v1_values;
+  std::vector<int32_t> v2_values;
+
+  test::randint<int8_t>(num_rows, 0, 100, &v1_values);
+  test::randint<int32_t>(num_rows, 0, 100, &v2_values);
+
+  std::shared_ptr<Array> v1;
+  MakeArray<Int8Type, int8_t>(schema->field(0)->type, is_valid, v1_values, &v1);
+
+  std::shared_ptr<Array> v2;
+  MakeArray<Int32Type, int32_t>(schema->field(1)->type, is_valid, v2_values, &v2);
+
+  static const int kBufferSize = 10;
+  static uint8_t buffer[kBufferSize];
+  static uint32_t seed = 0;
+  StringBuilder string_builder(default_memory_pool(), utf8());
+  for (int i = 0; i < num_rows; ++i) {
+    if (!is_valid[i]) {
+      string_builder.AppendNull();
+    } else {
+      test::random_ascii(kBufferSize, seed++, buffer);
+      string_builder.Append(buffer, kBufferSize);
+    }
+  }
+  std::shared_ptr<Array> v3;
+  ASSERT_OK(string_builder.Finish(&v3));
+
+  arrays->emplace_back(v1);
+  arrays->emplace_back(v2);
+  arrays->emplace_back(v3);
+}
+
+TEST(TestJsonFileReadWrite, BasicRoundTrip) {
+  auto v1_type = int8();
+  auto v2_type = int32();
+  auto v3_type = utf8();
+
+  std::shared_ptr<Schema> schema(
+      new Schema({field("f1", v1_type), field("f2", v2_type), field("f3", v3_type)}));
+
+  std::unique_ptr<JsonWriter> writer;
+  ASSERT_OK(JsonWriter::Open(schema, &writer));
+
+  const int nbatches = 3;
+  std::vector<std::shared_ptr<RecordBatch>> batches;
+  for (int i = 0; i < nbatches; ++i) {
+    int32_t num_rows = 5 + i * 5;
+    std::vector<std::shared_ptr<Array>> arrays;
+
+    MakeBatchArrays(schema, num_rows, &arrays);
+    batches.emplace_back(std::make_shared<RecordBatch>(schema, num_rows, arrays));
+    ASSERT_OK(writer->WriteRecordBatch(arrays, num_rows));
+  }
+
+  std::string result;
+  ASSERT_OK(writer->Finish(&result));
+
+  std::unique_ptr<JsonReader> reader;
+
+  auto buffer = std::make_shared<Buffer>(
+      reinterpret_cast<const uint8_t*>(result.c_str()), static_cast<int>(result.size()));
+
+  ASSERT_OK(JsonReader::Open(buffer, &reader));
+  ASSERT_TRUE(reader->schema()->Equals(*schema.get()));
+
+  ASSERT_EQ(nbatches, reader->num_record_batches());
+
+  for (int i = 0; i < nbatches; ++i) {
+    std::shared_ptr<RecordBatch> batch;
+    ASSERT_OK(reader->GetRecordBatch(i, &batch));
+    ASSERT_TRUE(batch->Equals(*batches[i].get()));
+  }
+}
+
+TEST(TestJsonFileReadWrite, MinimalFormatExample) {
+  static const char* example = R"example(
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "foo",
+        "type": {"name": "int", "isSigned": true, "bitWidth": 32},
+        "nullable": true, "children": [],
+        "typeLayout": [
+          {"type": "VALIDITY", "typeBitWidth": 1},
+          {"type": "DATA", "typeBitWidth": 32}
+        ]
+      },
+      {
+        "name": "bar",
+        "type": {"name": "floatingpoint", "precision": "DOUBLE"},
+        "nullable": true, "children": [],
+        "typeLayout": [
+          {"type": "VALIDITY", "typeBitWidth": 1},
+          {"type": "DATA", "typeBitWidth": 64}
+        ]
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 5,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 5,
+          "DATA": [1, 2, 3, 4, 5],
+          "VALIDITY": [1, 0, 1, 1, 1]
+        },
+        {
+          "name": "bar",
+          "count": 5,
+          "DATA": [1.0, 2.0, 3.0, 4.0, 5.0],
+          "VALIDITY": [1, 0, 0, 1, 1]
+        }
+      ]
+    }
+  ]
+}
+)example";
+
+  auto buffer = std::make_shared<Buffer>(
+      reinterpret_cast<const uint8_t*>(example), strlen(example));
+
+  std::unique_ptr<JsonReader> reader;
+  ASSERT_OK(JsonReader::Open(buffer, &reader));
+
+  Schema ex_schema({field("foo", int32()), field("bar", float64())});
+
+  ASSERT_TRUE(reader->schema()->Equals(ex_schema));
+  ASSERT_EQ(1, reader->num_record_batches());
+
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(reader->GetRecordBatch(0, &batch));
+
+  std::vector<bool> foo_valid = {true, false, true, true, true};
+  std::vector<int32_t> foo_values = {1, 2, 3, 4, 5};
+  std::shared_ptr<Array> foo;
+  MakeArray<Int32Type, int32_t>(int32(), foo_valid, foo_values, &foo);
+  ASSERT_TRUE(batch->column(0)->Equals(foo));
+
+  std::vector<bool> bar_valid = {true, false, false, true, true};
+  std::vector<double> bar_values = {1, 2, 3, 4, 5};
+  std::shared_ptr<Array> bar;
+  MakeArray<DoubleType, double>(float64(), bar_valid, bar_values, &bar);
+  ASSERT_TRUE(batch->column(1)->Equals(bar));
+}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
new file mode 100644
index 0000000000000..31fe35b44cef7
--- /dev/null
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -0,0 +1,1113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/json-internal.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/writer.h"
+
+#include "arrow/array.h"
+#include "arrow/schema.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+using RjArray = rj::Value::ConstArray;
+using RjObject = rj::Value::ConstObject;
+
+enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
+
+static std::string GetBufferTypeName(BufferType type) {
+  switch (type) {
+    case BufferType::DATA:
+      return "DATA";
+    case BufferType::OFFSET:
+      return "OFFSET";
+    case BufferType::TYPE:
+      return "TYPE";
+    case BufferType::VALIDITY:
+      return "VALIDITY";
+    default:
+      break;
+  }
+  return "UNKNOWN";
+}
+
+static std::string GetFloatingPrecisionName(FloatingPointMeta::Precision precision) {
+  switch (precision) {
+    case FloatingPointMeta::HALF:
+      return "HALF";
+    case FloatingPointMeta::SINGLE:
+      return "SINGLE";
+    case FloatingPointMeta::DOUBLE:
+      return "DOUBLE";
+    default:
+      break;
+  }
+  return "UNKNOWN";
+}
+
+static std::string GetTimeUnitName(TimeUnit unit) {
+  switch (unit) {
+    case TimeUnit::SECOND:
+      return "SECOND";
+    case TimeUnit::MILLI:
+      return "MILLISECOND";
+    case TimeUnit::MICRO:
+      return "MICROSECOND";
+    case TimeUnit::NANO:
+      return "NANOSECOND";
+    default:
+      break;
+  }
+  return "UNKNOWN";
+}
+
+class BufferLayout {
+ public:
+  BufferLayout(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
+
+  BufferType type() const { return type_; }
+  int bit_width() const { return bit_width_; }
+
+ private:
+  BufferType type_;
+  int bit_width_;
+};
+
+static const BufferLayout kValidityBuffer(BufferType::VALIDITY, 1);
+static const BufferLayout kOffsetBuffer(BufferType::OFFSET, 32);
+static const BufferLayout kTypeBuffer(BufferType::TYPE, 32);
+static const BufferLayout kBooleanBuffer(BufferType::DATA, 1);
+static const BufferLayout kValues64(BufferType::DATA, 64);
+static const BufferLayout kValues32(BufferType::DATA, 32);
+static const BufferLayout kValues16(BufferType::DATA, 16);
+static const BufferLayout kValues8(BufferType::DATA, 8);
+
+class JsonSchemaWriter : public TypeVisitor {
+ public:
+  explicit JsonSchemaWriter(const Schema& schema, RjWriter* writer)
+      : schema_(schema), writer_(writer) {}
+
+  Status Write() {
+    writer_->StartObject();
+    writer_->Key("fields");
+    writer_->StartArray();
+    for (const std::shared_ptr<Field>& field : schema_.fields()) {
+      RETURN_NOT_OK(VisitField(*field.get()));
+    }
+    writer_->EndArray();
+    writer_->EndObject();
+    return Status::OK();
+  }
+
+  Status VisitField(const Field& field) {
+    writer_->StartObject();
+
+    writer_->Key("name");
+    writer_->String(field.name.c_str());
+
+    writer_->Key("nullable");
+    writer_->Bool(field.nullable);
+
+    // Visit the type
+    RETURN_NOT_OK(field.type->Accept(this));
+    writer_->EndObject();
+
+    return Status::OK();
+  }
+
+  void SetNoChildren() {
+    writer_->Key("children");
+    writer_->StartArray();
+    writer_->EndArray();
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
+                              std::is_base_of<BooleanType, T>::value ||
+                              std::is_base_of<NullType, T>::value,
+      void>::type
+  WriteTypeMetadata(const T& type) {}
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<IntegerMeta, T>::value, void>::type
+  WriteTypeMetadata(const T& type) {
+    writer_->Key("bitWidth");
+    writer_->Int(type.bit_width());
+    writer_->Key("isSigned");
+    writer_->Bool(type.is_signed());
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<FloatingPointMeta, T>::value, void>::type
+  WriteTypeMetadata(const T& type) {
+    writer_->Key("precision");
+    writer_->String(GetFloatingPrecisionName(type.precision()));
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<IntervalType, T>::value, void>::type
+  WriteTypeMetadata(const T& type) {
+    writer_->Key("unit");
+    switch (type.unit) {
+      case IntervalType::Unit::YEAR_MONTH:
+        writer_->String("YEAR_MONTH");
+        break;
+      case IntervalType::Unit::DAY_TIME:
+        writer_->String("DAY_TIME");
+        break;
+    }
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<TimeType, T>::value ||
+                              std::is_base_of<TimestampType, T>::value,
+      void>::type
+  WriteTypeMetadata(const T& type) {
+    writer_->Key("unit");
+    writer_->String(GetTimeUnitName(type.unit));
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<DecimalType, T>::value, void>::type
+  WriteTypeMetadata(const T& type) {
+    writer_->Key("precision");
+    writer_->Int(type.precision);
+    writer_->Key("scale");
+    writer_->Int(type.scale);
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<UnionType, T>::value, void>::type
+  WriteTypeMetadata(const T& type) {
+    writer_->Key("mode");
+    switch (type.mode) {
+      case UnionMode::SPARSE:
+        writer_->String("SPARSE");
+        break;
+      case UnionMode::DENSE:
+        writer_->String("DENSE");
+        break;
+    }
+
+    // Write type ids
+    writer_->Key("typeIds");
+    writer_->StartArray();
+    for (size_t i = 0; i < type.type_ids.size(); ++i) {
+      writer_->Uint(type.type_ids[i]);
+    }
+    writer_->EndArray();
+  }
+
+  // TODO(wesm): Other Type metadata
+
+  template <typename T>
+  void WriteName(const std::string& typeclass, const T& type) {
+    writer_->Key("type");
+    writer_->StartObject();
+    writer_->Key("name");
+    writer_->String(typeclass);
+    WriteTypeMetadata(type);
+    writer_->EndObject();
+  }
+
+  template <typename T>
+  Status WritePrimitive(const std::string& typeclass, const T& type,
+      const std::vector<BufferLayout>& buffer_layout) {
+    WriteName(typeclass, type);
+    SetNoChildren();
+    WriteBufferLayout(buffer_layout);
+    return Status::OK();
+  }
+
+  template <typename T>
+  Status WriteVarBytes(const std::string& typeclass, const T& type) {
+    WriteName(typeclass, type);
+    SetNoChildren();
+    WriteBufferLayout({kValidityBuffer, kOffsetBuffer, kValues8});
+    return Status::OK();
+  }
+
+  void WriteBufferLayout(const std::vector<BufferLayout>& buffer_layout) {
+    writer_->Key("typeLayout");
+    writer_->StartArray();
+
+    for (const BufferLayout& buffer : buffer_layout) {
+      writer_->StartObject();
+      writer_->Key("type");
+      writer_->String(GetBufferTypeName(buffer.type()));
+
+      writer_->Key("typeBitWidth");
+      writer_->Int(buffer.bit_width());
+
+      writer_->EndObject();
+    }
+    writer_->EndArray();
+  }
+
+  Status WriteChildren(const std::vector<std::shared_ptr<Field>>& children) {
+    writer_->Key("children");
+    writer_->StartArray();
+    for (const std::shared_ptr<Field>& field : children) {
+      RETURN_NOT_OK(VisitField(*field.get()));
+    }
+    writer_->EndArray();
+    return Status::OK();
+  }
+
+  Status Visit(const NullType& type) override { return WritePrimitive("null", type, {}); }
+
+  Status Visit(const BooleanType& type) override {
+    return WritePrimitive("bool", type, {kValidityBuffer, kBooleanBuffer});
+  }
+
+  Status Visit(const Int8Type& type) override {
+    return WritePrimitive("int", type, {kValidityBuffer, kValues8});
+  }
+
+  Status Visit(const Int16Type& type) override {
+    return WritePrimitive("int", type, {kValidityBuffer, kValues16});
+  }
+
+  Status Visit(const Int32Type& type) override {
+    return WritePrimitive("int", type, {kValidityBuffer, kValues32});
+  }
+
+  Status Visit(const Int64Type& type) override {
+    return WritePrimitive("int", type, {kValidityBuffer, kValues64});
+  }
+
+  Status Visit(const UInt8Type& type) override {
+    return WritePrimitive("int", type, {kValidityBuffer, kValues8});
+  }
+
+  Status Visit(const UInt16Type& type) override {
+    return WritePrimitive("int", type, {kValidityBuffer, kValues16});
+  }
+
+  Status Visit(const UInt32Type& type) override {
+    return WritePrimitive("int", type, {kValidityBuffer, kValues32});
+  }
+
+  Status Visit(const UInt64Type& type) override {
+    return WritePrimitive("int", type, {kValidityBuffer, kValues64});
+  }
+
+  Status Visit(const HalfFloatType& type) override {
+    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues16});
+  }
+
+  Status Visit(const FloatType& type) override {
+    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues32});
+  }
+
+  Status Visit(const DoubleType& type) override {
+    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues64});
+  }
+
+  Status Visit(const StringType& type) override { return WriteVarBytes("utf8", type); }
+
+  Status Visit(const BinaryType& type) override { return WriteVarBytes("binary", type); }
+
+  Status Visit(const DateType& type) override {
+    return WritePrimitive("date", type, {kValidityBuffer, kValues64});
+  }
+
+  Status Visit(const TimeType& type) override {
+    return WritePrimitive("time", type, {kValidityBuffer, kValues64});
+  }
+
+  Status Visit(const TimestampType& type) override {
+    return WritePrimitive("timestamp", type, {kValidityBuffer, kValues64});
+  }
+
+  Status Visit(const IntervalType& type) override {
+    return WritePrimitive("interval", type, {kValidityBuffer, kValues64});
+  }
+
+  Status Visit(const DecimalType& type) override { return Status::NotImplemented("NYI"); }
+
+  Status Visit(const ListType& type) override {
+    WriteName("list", type);
+    RETURN_NOT_OK(WriteChildren(type.children()));
+    WriteBufferLayout({kValidityBuffer, kOffsetBuffer});
+    return Status::OK();
+  }
+
+  Status Visit(const StructType& type) override {
+    WriteName("struct", type);
+    WriteChildren(type.children());
+    WriteBufferLayout({kValidityBuffer, kTypeBuffer});
+    return Status::OK();
+  }
+
+  Status Visit(const UnionType& type) override {
+    WriteName("union", type);
+    WriteChildren(type.children());
+
+    if (type.mode == UnionMode::SPARSE) {
+      WriteBufferLayout({kValidityBuffer, kTypeBuffer});
+    } else {
+      WriteBufferLayout({kValidityBuffer, kTypeBuffer, kOffsetBuffer});
+    }
+    return Status::OK();
+  }
+
+ private:
+  const Schema& schema_;
+  RjWriter* writer_;
+};
+
+class JsonArrayWriter : public ArrayVisitor {
+ public:
+  explicit JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
+      : name_(name), array_(array), writer_(writer) {}
+
+  Status Write() { return VisitArray(name_, array_); }
+
+  Status VisitArray(const std::string& name, const Array& arr) {
+    writer_->StartObject();
+    writer_->Key("name");
+    writer_->String(name);
+
+    writer_->Key("count");
+    writer_->Int(arr.length());
+
+    RETURN_NOT_OK(arr.Accept(this));
+
+    writer_->EndObject();
+    return Status::OK();
+  }
+
+  template <typename T>
+  typename std::enable_if<IsSignedInt<T>::value, void>::type WriteDataValues(
+      const T& arr) {
+    const auto data = arr.raw_data();
+    for (int i = 0; i < arr.length(); ++i) {
+      writer_->Int64(data[i]);
+    }
+  }
+
+  template <typename T>
+  typename std::enable_if<IsUnsignedInt<T>::value, void>::type WriteDataValues(
+      const T& arr) {
+    const auto data = arr.raw_data();
+    for (int i = 0; i < arr.length(); ++i) {
+      writer_->Uint64(data[i]);
+    }
+  }
+
+  template <typename T>
+  typename std::enable_if<IsFloatingPoint<T>::value, void>::type WriteDataValues(
+      const T& arr) {
+    const auto data = arr.raw_data();
+    for (int i = 0; i < arr.length(); ++i) {
+      writer_->Double(data[i]);
+    }
+  }
+
+  // String (Utf8), Binary
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BinaryArray, T>::value, void>::type
+  WriteDataValues(const T& arr) {
+    for (int i = 0; i < arr.length(); ++i) {
+      int32_t length;
+      const char* buf = reinterpret_cast<const char*>(arr.GetValue(i, &length));
+      writer_->String(buf, length);
+    }
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
+  WriteDataValues(const T& arr) {
+    for (int i = 0; i < arr.length(); ++i) {
+      writer_->Bool(arr.Value(i));
+    }
+  }
+
+  template <typename T>
+  void WriteDataField(const T& arr) {
+    writer_->Key("DATA");
+    writer_->StartArray();
+    WriteDataValues(arr);
+    writer_->EndArray();
+  }
+
+  template <typename T>
+  void WriteOffsetsField(const T* offsets, int32_t length) {
+    writer_->Key("OFFSETS");
+    writer_->StartArray();
+    for (int i = 0; i < length; ++i) {
+      writer_->Int64(offsets[i]);
+    }
+    writer_->EndArray();
+  }
+
+  void WriteValidityField(const Array& arr) {
+    writer_->Key("VALIDITY");
+    writer_->StartArray();
+    if (arr.null_count() > 0) {
+      for (int i = 0; i < arr.length(); ++i) {
+        writer_->Int(arr.IsNull(i) ? 0 : 1);
+      }
+    } else {
+      for (int i = 0; i < arr.length(); ++i) {
+        writer_->Int(1);
+      }
+    }
+    writer_->EndArray();
+  }
+
+  void SetNoChildren() {
+    writer_->Key("children");
+    writer_->StartArray();
+    writer_->EndArray();
+  }
+
+  template <typename T>
+  Status WritePrimitive(const T& array) {
+    WriteValidityField(array);
+    WriteDataField(array);
+    SetNoChildren();
+    return Status::OK();
+  }
+
+  template <typename T>
+  Status WriteVarBytes(const T& array) {
+    WriteValidityField(array);
+    WriteOffsetsField(array.raw_offsets(), array.length() + 1);
+    WriteDataField(array);
+    SetNoChildren();
+    return Status::OK();
+  }
+
+  Status WriteChildren(const std::vector<std::shared_ptr<Field>>& fields,
+      const std::vector<std::shared_ptr<Array>>& arrays) {
+    writer_->Key("children");
+    writer_->StartArray();
+    for (size_t i = 0; i < fields.size(); ++i) {
+      RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
+    }
+    writer_->EndArray();
+    return Status::OK();
+  }
+
+  Status Visit(const NullArray& array) override {
+    SetNoChildren();
+    return Status::OK();
+  }
+
+  Status Visit(const BooleanArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int8Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int16Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int32Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int64Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt8Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt16Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt32Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt64Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const HalfFloatArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const FloatArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const DoubleArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const StringArray& array) override { return WriteVarBytes(array); }
+
+  Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); }
+
+  Status Visit(const DateArray& array) override { return Status::NotImplemented("date"); }
+
+  Status Visit(const TimeArray& array) override { return Status::NotImplemented("time"); }
+
+  Status Visit(const TimestampArray& array) override {
+    return Status::NotImplemented("timestamp");
+  }
+
+  Status Visit(const IntervalArray& array) override {
+    return Status::NotImplemented("interval");
+  }
+
+  Status Visit(const DecimalArray& array) override {
+    return Status::NotImplemented("decimal");
+  }
+
+  Status Visit(const ListArray& array) override {
+    WriteValidityField(array);
+    WriteOffsetsField(array.raw_offsets(), array.length() + 1);
+    auto type = static_cast<const ListType*>(array.type().get());
+    return WriteChildren(type->children(), {array.values()});
+  }
+
+  Status Visit(const StructArray& array) override {
+    WriteValidityField(array);
+    auto type = static_cast<const StructType*>(array.type().get());
+    return WriteChildren(type->children(), array.fields());
+  }
+
+  Status Visit(const UnionArray& array) override {
+    return Status::NotImplemented("union");
+  }
+
+ private:
+  const std::string& name_;
+  const Array& array_;
+  RjWriter* writer_;
+};
+
+class JsonSchemaReader {
+ public:
+  explicit JsonSchemaReader(const rj::Value& json_schema) : json_schema_(json_schema) {}
+
+  Status GetSchema(std::shared_ptr<Schema>* schema) {
+    const auto& obj_schema = json_schema_.GetObject();
+
+    const auto& json_fields = obj_schema.FindMember("fields");
+    RETURN_NOT_ARRAY("fields", json_fields, obj_schema);
+
+    std::vector<std::shared_ptr<Field>> fields;
+    RETURN_NOT_OK(GetFieldsFromArray(json_fields->value, &fields));
+
+    *schema = std::make_shared<Schema>(fields);
+    return Status::OK();
+  }
+
+  Status GetFieldsFromArray(
+      const rj::Value& obj, std::vector<std::shared_ptr<Field>>* fields) {
+    const auto& values = obj.GetArray();
+
+    fields->resize(values.Size());
+    for (size_t i = 0; i < fields->size(); ++i) {
+      RETURN_NOT_OK(GetField(values[i], &(*fields)[i]));
+    }
+    return Status::OK();
+  }
+
+  Status GetField(const rj::Value& obj, std::shared_ptr<Field>* field) {
+    if (!obj.IsObject()) { return Status::Invalid("Field was not a JSON object"); }
+    const auto& json_field = obj.GetObject();
+
+    const auto& json_name = json_field.FindMember("name");
+    RETURN_NOT_STRING("name", json_name, json_field);
+
+    const auto& json_nullable = json_field.FindMember("nullable");
+    RETURN_NOT_BOOL("nullable", json_nullable, json_field);
+
+    const auto& json_type = json_field.FindMember("type");
+    RETURN_NOT_OBJECT("type", json_type, json_field);
+
+    const auto& json_children = json_field.FindMember("children");
+    RETURN_NOT_ARRAY("children", json_children, json_field);
+
+    std::vector<std::shared_ptr<Field>> children;
+    RETURN_NOT_OK(GetFieldsFromArray(json_children->value, &children));
+
+    std::shared_ptr<DataType> type;
+    RETURN_NOT_OK(GetType(json_type->value.GetObject(), children, &type));
+
+    *field = std::make_shared<Field>(
+        json_name->value.GetString(), type, json_nullable->value.GetBool());
+    return Status::OK();
+  }
+
+  Status GetInteger(
+      const rj::Value::ConstObject& json_type, std::shared_ptr<DataType>* type) {
+    const auto& json_bit_width = json_type.FindMember("bitWidth");
+    RETURN_NOT_INT("bitWidth", json_bit_width, json_type);
+
+    const auto& json_is_signed = json_type.FindMember("isSigned");
+    RETURN_NOT_BOOL("isSigned", json_is_signed, json_type);
+
+    bool is_signed = json_is_signed->value.GetBool();
+    int bit_width = json_bit_width->value.GetInt();
+
+    switch (bit_width) {
+      case 8:
+        *type = is_signed ? int8() : uint8();
+        break;
+      case 16:
+        *type = is_signed ? int16() : uint16();
+        break;
+      case 32:
+        *type = is_signed ? int32() : uint32();
+        break;
+      case 64:
+        *type = is_signed ? int64() : uint64();
+        break;
+      default:
+        std::stringstream ss;
+        ss << "Invalid bit width: " << bit_width;
+        return Status::Invalid(ss.str());
+    }
+    return Status::OK();
+  }
+
+  Status GetFloatingPoint(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+    const auto& json_precision = json_type.FindMember("precision");
+    RETURN_NOT_STRING("precision", json_precision, json_type);
+
+    std::string precision = json_precision->value.GetString();
+
+    if (precision == "DOUBLE") {
+      *type = float64();
+    } else if (precision == "SINGLE") {
+      *type = float32();
+    } else if (precision == "HALF") {
+      *type = float16();
+    } else {
+      std::stringstream ss;
+      ss << "Invalid precision: " << precision;
+      return Status::Invalid(ss.str());
+    }
+    return Status::OK();
+  }
+
+  template <typename T>
+  Status GetTimeLike(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+    const auto& json_unit = json_type.FindMember("unit");
+    RETURN_NOT_STRING("unit", json_unit, json_type);
+
+    std::string unit_str = json_unit->value.GetString();
+
+    TimeUnit unit;
+
+    if (unit_str == "SECOND") {
+      unit = TimeUnit::SECOND;
+    } else if (unit_str == "MILLISECOND") {
+      unit = TimeUnit::MILLI;
+    } else if (unit_str == "MICROSECOND") {
+      unit = TimeUnit::MICRO;
+    } else if (unit_str == "NANOSECOND") {
+      unit = TimeUnit::NANO;
+    } else {
+      std::stringstream ss;
+      ss << "Invalid time unit: " << unit_str;
+      return Status::Invalid(ss.str());
+    }
+
+    *type = std::make_shared<T>(unit);
+
+    return Status::OK();
+  }
+
+  Status GetUnion(const RjObject& json_type,
+      const std::vector<std::shared_ptr<Field>>& children,
+      std::shared_ptr<DataType>* type) {
+    const auto& json_mode = json_type.FindMember("mode");
+    RETURN_NOT_STRING("mode", json_mode, json_type);
+
+    std::string mode_str = json_mode->value.GetString();
+    UnionMode mode;
+
+    if (mode_str == "SPARSE") {
+      mode = UnionMode::SPARSE;
+    } else if (mode_str == "DENSE") {
+      mode = UnionMode::DENSE;
+    } else {
+      std::stringstream ss;
+      ss << "Invalid union mode: " << mode_str;
+      return Status::Invalid(ss.str());
+    }
+
+    const auto& json_type_ids = json_type.FindMember("typeIds");
+    RETURN_NOT_ARRAY("typeIds", json_type_ids, json_type);
+
+    std::vector<uint8_t> type_ids;
+    const auto& id_array = json_type_ids->value.GetArray();
+    for (const rj::Value& val : id_array) {
+      DCHECK(val.IsUint());
+      type_ids.push_back(val.GetUint());
+    }
+
+    *type = union_(children, type_ids, mode);
+
+    return Status::OK();
+  }
+
+  Status GetType(const RjObject& json_type,
+      const std::vector<std::shared_ptr<Field>>& children,
+      std::shared_ptr<DataType>* type) {
+    const auto& json_type_name = json_type.FindMember("name");
+    RETURN_NOT_STRING("name", json_type_name, json_type);
+
+    std::string type_name = json_type_name->value.GetString();
+
+    if (type_name == "int") {
+      return GetInteger(json_type, type);
+    } else if (type_name == "floatingpoint") {
+      return GetFloatingPoint(json_type, type);
+    } else if (type_name == "bool") {
+      *type = boolean();
+    } else if (type_name == "utf8") {
+      *type = utf8();
+    } else if (type_name == "binary") {
+      *type = binary();
+    } else if (type_name == "null") {
+      *type = null();
+    } else if (type_name == "date") {
+      *type = date();
+    } else if (type_name == "time") {
+      return GetTimeLike<TimeType>(json_type, type);
+    } else if (type_name == "timestamp") {
+      return GetTimeLike<TimestampType>(json_type, type);
+    } else if (type_name == "list") {
+      *type = list(children[0]);
+    } else if (type_name == "struct") {
+      *type = struct_(children);
+    } else {
+      return GetUnion(json_type, children, type);
+    }
+    return Status::OK();
+  }
+
+ private:
+  const rj::Value& json_schema_;
+};
+
+class JsonArrayReader {
+ public:
+  explicit JsonArrayReader(MemoryPool* pool) : pool_(pool) {}
+
+  Status GetValidityBuffer(const std::vector<bool>& is_valid, int32_t* null_count,
+      std::shared_ptr<Buffer>* validity_buffer) {
+    int length = static_cast<int>(is_valid.size());
+
+    std::shared_ptr<MutableBuffer> out_buffer;
+    RETURN_NOT_OK(GetEmptyBitmap(pool_, length, &out_buffer));
+    uint8_t* bitmap = out_buffer->mutable_data();
+
+    *null_count = 0;
+    for (int i = 0; i < length; ++i) {
+      if (!is_valid[i]) {
+        ++(*null_count);
+        continue;
+      }
+      BitUtil::SetBit(bitmap, i);
+    }
+
+    *validity_buffer = out_buffer;
+    return Status::OK();
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
+                              std::is_base_of<BooleanType, T>::value,
+      Status>::type
+  ReadArray(const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+    typename TypeTraits<T>::BuilderType builder(pool_, type);
+
+    const auto& json_data = json_array.FindMember("DATA");
+    RETURN_NOT_ARRAY("DATA", json_data, json_array);
+
+    const auto& json_data_arr = json_data->value.GetArray();
+
+    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length);
+    for (int i = 0; i < length; ++i) {
+      if (!is_valid[i]) {
+        builder.AppendNull();
+        continue;
+      }
+
+      const rj::Value& val = json_data_arr[i];
+      if (IsSignedInt<T>::value) {
+        DCHECK(val.IsInt());
+        builder.Append(val.GetInt64());
+      } else if (IsUnsignedInt<T>::value) {
+        DCHECK(val.IsUint());
+        builder.Append(val.GetUint64());
+      } else if (IsFloatingPoint<T>::value) {
+        DCHECK(val.IsFloat());
+        builder.Append(val.GetFloat());
+      } else if (std::is_base_of<BooleanType, T>::value) {
+        DCHECK(val.IsBool());
+        builder.Append(val.GetBool());
+      } else {
+        // We are in the wrong function
+        return Status::Invalid(type->ToString());
+      }
+    }
+
+    return builder.Finish(array);
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BinaryType, T>::value, Status>::type ReadArray(
+      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+    typename TypeTraits<T>::BuilderType builder(pool_, type);
+
+    const auto& json_data = json_array.FindMember("DATA");
+    RETURN_NOT_ARRAY("DATA", json_data, json_array);
+
+    const auto& json_data_arr = json_data->value.GetArray();
+
+    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length);
+    for (int i = 0; i < length; ++i) {
+      if (!is_valid[i]) {
+        builder.AppendNull();
+        continue;
+      }
+
+      const rj::Value& val = json_data_arr[i];
+      DCHECK(val.IsString());
+      builder.Append(val.GetString());
+    }
+
+    return builder.Finish(array);
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<ListType, T>::value, Status>::type ReadArray(
+      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+    const auto& json_offsets = json_array.FindMember("OFFSETS");
+    RETURN_NOT_ARRAY("OFFSETS", json_offsets, json_array);
+    const auto& json_offsets_arr = json_offsets->value.GetArray();
+
+    int32_t null_count = 0;
+    std::shared_ptr<Buffer> validity_buffer;
+    RETURN_NOT_OK(GetValidityBuffer(is_valid, &null_count, &validity_buffer));
+
+    auto offsets_buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(offsets_buffer->Resize((length + 1) * sizeof(int32_t)));
+    int32_t* offsets = reinterpret_cast<int32_t*>(offsets_buffer->mutable_data());
+
+    for (int i = 0; i < length + 1; ++i) {
+      const rj::Value& val = json_offsets_arr[i];
+      DCHECK(val.IsInt());
+      offsets[i] = val.GetInt();
+    }
+
+    std::vector<std::shared_ptr<Array>> children;
+    RETURN_NOT_OK(GetChildren(json_array, type, &children));
+    DCHECK_EQ(children.size(), 1);
+
+    *array = std::make_shared<ListArray>(
+        type, length, offsets_buffer, children[0], null_count, validity_buffer);
+
+    return Status::OK();
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<StructType, T>::value, Status>::type ReadArray(
+      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+    int32_t null_count = 0;
+    std::shared_ptr<Buffer> validity_buffer;
+    RETURN_NOT_OK(GetValidityBuffer(is_valid, &null_count, &validity_buffer));
+
+    std::vector<std::shared_ptr<Array>> fields;
+    RETURN_NOT_OK(GetChildren(json_array, type, &fields));
+
+    *array =
+        std::make_shared<StructArray>(type, length, fields, null_count, validity_buffer);
+
+    return Status::OK();
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<NullType, T>::value, Status>::type ReadArray(
+      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+    *array = std::make_shared<NullArray>(type, length);
+    return Status::OK();
+  }
+
+  Status GetChildren(const RjObject& json_array, const std::shared_ptr<DataType>& type,
+      std::vector<std::shared_ptr<Array>>* array) {
+    const auto& json_children = json_array.FindMember("children");
+    RETURN_NOT_ARRAY("children", json_children, json_array);
+    const auto& json_children_arr = json_children->value.GetArray();
+
+    if (type->num_children() != static_cast<int>(json_children_arr.Size())) {
+      std::stringstream ss;
+      ss << "Expected " << type->num_children() << " children, but got "
+         << json_children_arr.Size();
+      return Status::Invalid(ss.str());
+    }
+
+    for (int i = 0; i < static_cast<int>(json_children_arr.Size()); ++i) {
+      const rj::Value& json_child = json_children_arr[i];
+      DCHECK(json_child.IsObject());
+
+      std::shared_ptr<Field> child_field = type->child(i);
+
+      auto it = json_child.FindMember("name");
+      RETURN_NOT_STRING("name", it, json_child);
+
+      DCHECK_EQ(it->value.GetString(), child_field->name);
+      std::shared_ptr<Array> child;
+      RETURN_NOT_OK(GetArray(json_children_arr[i], child_field->type, &child));
+      array->emplace_back(child);
+    }
+
+    return Status::OK();
+  }
+
+  Status GetArray(const rj::Value& obj, const std::shared_ptr<DataType>& type,
+      std::shared_ptr<Array>* array) {
+    if (!obj.IsObject()) {
+      return Status::Invalid("Array element was not a JSON object");
+    }
+    const auto& json_array = obj.GetObject();
+
+    const auto& json_length = json_array.FindMember("count");
+    RETURN_NOT_INT("count", json_length, json_array);
+    int32_t length = json_length->value.GetInt();
+
+    const auto& json_valid_iter = json_array.FindMember("VALIDITY");
+    RETURN_NOT_ARRAY("VALIDITY", json_valid_iter, json_array);
+
+    const auto& json_validity = json_valid_iter->value.GetArray();
+
+    DCHECK_EQ(static_cast<int>(json_validity.Size()), length);
+
+    std::vector<bool> is_valid;
+    for (const rj::Value& val : json_validity) {
+      DCHECK(val.IsInt());
+      is_valid.push_back(static_cast<bool>(val.GetInt()));
+    }
+
+#define TYPE_CASE(TYPE) \
+  case TYPE::type_id:   \
+    return ReadArray<TYPE>(json_array, length, is_valid, type, array);
+
+#define NOT_IMPLEMENTED_CASE(TYPE_ENUM)      \
+  case Type::TYPE_ENUM: {                    \
+    std::stringstream ss;                    \
+    ss << type->ToString();                  \
+    return Status::NotImplemented(ss.str()); \
+  }
+
+    switch (type->type) {
+      TYPE_CASE(NullType);
+      TYPE_CASE(BooleanType);
+      TYPE_CASE(UInt8Type);
+      TYPE_CASE(Int8Type);
+      TYPE_CASE(UInt16Type);
+      TYPE_CASE(Int16Type);
+      TYPE_CASE(UInt32Type);
+      TYPE_CASE(Int32Type);
+      TYPE_CASE(UInt64Type);
+      TYPE_CASE(Int64Type);
+      TYPE_CASE(HalfFloatType);
+      TYPE_CASE(FloatType);
+      TYPE_CASE(DoubleType);
+      TYPE_CASE(StringType);
+      TYPE_CASE(BinaryType);
+      NOT_IMPLEMENTED_CASE(DATE);
+      NOT_IMPLEMENTED_CASE(TIMESTAMP);
+      NOT_IMPLEMENTED_CASE(TIME);
+      NOT_IMPLEMENTED_CASE(INTERVAL);
+      TYPE_CASE(ListType);
+      TYPE_CASE(StructType);
+      NOT_IMPLEMENTED_CASE(UNION);
+      default:
+        std::stringstream ss;
+        ss << type->ToString();
+        return Status::NotImplemented(ss.str());
+    }
+
+#undef TYPE_CASE
+#undef NOT_IMPLEMENTED_CASE
+
+    return Status::OK();
+  }
+
+ private:
+  MemoryPool* pool_;
+};
+
+Status WriteJsonSchema(const Schema& schema, RjWriter* json_writer) {
+  JsonSchemaWriter converter(schema, json_writer);
+  return converter.Write();
+}
+
+Status ReadJsonSchema(const rj::Value& json_schema, std::shared_ptr<Schema>* schema) {
+  JsonSchemaReader converter(json_schema);
+  return converter.GetSchema(schema);
+}
+
+Status WriteJsonArray(
+    const std::string& name, const Array& array, RjWriter* json_writer) {
+  JsonArrayWriter converter(name, array, json_writer);
+  return converter.Write();
+}
+
+Status ReadJsonArray(MemoryPool* pool, const rj::Value& json_array,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+  JsonArrayReader converter(pool);
+  return converter.GetArray(json_array, type, array);
+}
+
+Status ReadJsonArray(MemoryPool* pool, const rj::Value& json_array, const Schema& schema,
+    std::shared_ptr<Array>* array) {
+  if (!json_array.IsObject()) { return Status::Invalid("Element was not a JSON object"); }
+
+  const auto& json_obj = json_array.GetObject();
+
+  const auto& json_name = json_obj.FindMember("name");
+  RETURN_NOT_STRING("name", json_name, json_obj);
+
+  std::string name = json_name->value.GetString();
+
+  std::shared_ptr<Field> result = nullptr;
+  for (const std::shared_ptr<Field>& field : schema.fields()) {
+    if (field->name == name) {
+      result = field;
+      break;
+    }
+  }
+
+  if (result == nullptr) {
+    std::stringstream ss;
+    ss << "Field named " << name << " not found in schema";
+    return Status::KeyError(ss.str());
+  }
+
+  return ReadJsonArray(pool, json_array, result->type, array);
+}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/json-internal.h b/cpp/src/arrow/ipc/json-internal.h
new file mode 100644
index 0000000000000..0c167a4ec53a2
--- /dev/null
+++ b/cpp/src/arrow/ipc/json-internal.h
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IPC_JSON_INTERNAL_H
+#define ARROW_IPC_JSON_INTERNAL_H
+
+#define RAPIDJSON_HAS_STDSTRING 1
+#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1
+#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1
+
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include "rapidjson/document.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/writer.h"
+
+#include "arrow/type_fwd.h"
+#include "arrow/util/visibility.h"
+
+namespace rj = rapidjson;
+using RjWriter = rj::Writer<rj::StringBuffer>;
+
+#define RETURN_NOT_FOUND(TOK, NAME, PARENT) \
+  if (NAME == PARENT.MemberEnd()) {         \
+    std::stringstream ss;                   \
+    ss << "field " << TOK << " not found";  \
+    return Status::Invalid(ss.str());       \
+  }
+
+#define RETURN_NOT_STRING(TOK, NAME, PARENT) \
+  RETURN_NOT_FOUND(TOK, NAME, PARENT);       \
+  if (!NAME->value.IsString()) {             \
+    std::stringstream ss;                    \
+    ss << "field was not a string"           \
+       << " line " << __LINE__;              \
+    return Status::Invalid(ss.str());        \
+  }
+
+#define RETURN_NOT_BOOL(TOK, NAME, PARENT) \
+  RETURN_NOT_FOUND(TOK, NAME, PARENT);     \
+  if (!NAME->value.IsBool()) {             \
+    std::stringstream ss;                  \
+    ss << "field was not a boolean"        \
+       << " line " << __LINE__;            \
+    return Status::Invalid(ss.str());      \
+  }
+
+#define RETURN_NOT_INT(TOK, NAME, PARENT) \
+  RETURN_NOT_FOUND(TOK, NAME, PARENT);    \
+  if (!NAME->value.IsInt()) {             \
+    std::stringstream ss;                 \
+    ss << "field was not an int"          \
+       << " line " << __LINE__;           \
+    return Status::Invalid(ss.str());     \
+  }
+
+#define RETURN_NOT_ARRAY(TOK, NAME, PARENT) \
+  RETURN_NOT_FOUND(TOK, NAME, PARENT);      \
+  if (!NAME->value.IsArray()) {             \
+    std::stringstream ss;                   \
+    ss << "field was not an array"          \
+       << " line " << __LINE__;             \
+    return Status::Invalid(ss.str());       \
+  }
+
+#define RETURN_NOT_OBJECT(TOK, NAME, PARENT) \
+  RETURN_NOT_FOUND(TOK, NAME, PARENT);       \
+  if (!NAME->value.IsObject()) {             \
+    std::stringstream ss;                    \
+    ss << "field was not an object"          \
+       << " line " << __LINE__;              \
+    return Status::Invalid(ss.str());        \
+  }
+
+namespace arrow {
+namespace ipc {
+
+// TODO(wesm): Only exporting these because arrow_ipc does not have a static
+// library at the moment. Better to not export
+Status ARROW_EXPORT WriteJsonSchema(const Schema& schema, RjWriter* json_writer);
+Status ARROW_EXPORT WriteJsonArray(
+    const std::string& name, const Array& array, RjWriter* json_writer);
+
+Status ARROW_EXPORT ReadJsonSchema(
+    const rj::Value& json_obj, std::shared_ptr<Schema>* schema);
+Status ARROW_EXPORT ReadJsonArray(MemoryPool* pool, const rj::Value& json_obj,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array);
+
+Status ARROW_EXPORT ReadJsonArray(MemoryPool* pool, const rj::Value& json_obj,
+    const Schema& schema, std::shared_ptr<Array>* array);
+
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_JSON_INTERNAL_H
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
new file mode 100644
index 0000000000000..2281611f8b879
--- /dev/null
+++ b/cpp/src/arrow/ipc/json.cc
@@ -0,0 +1,219 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/json.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/ipc/json-internal.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
+#include "arrow/type.h"
+#include "arrow/util/buffer.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/memory-pool.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+namespace ipc {
+
+// ----------------------------------------------------------------------
+// Writer implementation
+
+class JsonWriter::JsonWriterImpl {
+ public:
+  explicit JsonWriterImpl(const std::shared_ptr<Schema>& schema) : schema_(schema) {
+    writer_.reset(new RjWriter(string_buffer_));
+  }
+
+  Status Start() {
+    writer_->StartObject();
+
+    writer_->Key("schema");
+    RETURN_NOT_OK(WriteJsonSchema(*schema_.get(), writer_.get()));
+
+    // Record batches
+    writer_->Key("batches");
+    writer_->StartArray();
+    return Status::OK();
+  }
+
+  Status Finish(std::string* result) {
+    writer_->EndArray();  // Record batches
+    writer_->EndObject();
+
+    *result = string_buffer_.GetString();
+    return Status::OK();
+  }
+
+  Status WriteRecordBatch(
+      const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows) {
+    DCHECK_EQ(static_cast<int>(columns.size()), schema_->num_fields());
+
+    writer_->StartObject();
+    writer_->Key("count");
+    writer_->Int(num_rows);
+
+    writer_->Key("columns");
+    writer_->StartArray();
+
+    for (int i = 0; i < schema_->num_fields(); ++i) {
+      const std::shared_ptr<Array>& column = columns[i];
+
+      DCHECK_EQ(num_rows, column->length())
+          << "Array length did not match record batch length";
+
+      RETURN_NOT_OK(
+          WriteJsonArray(schema_->field(i)->name, *column.get(), writer_.get()));
+    }
+
+    writer_->EndArray();
+    writer_->EndObject();
+    return Status::OK();
+  }
+
+ private:
+  std::shared_ptr<Schema> schema_;
+
+  rj::StringBuffer string_buffer_;
+  std::unique_ptr<RjWriter> writer_;
+};
+
+JsonWriter::JsonWriter(const std::shared_ptr<Schema>& schema) {
+  impl_.reset(new JsonWriterImpl(schema));
+}
+
+JsonWriter::~JsonWriter() {}
+
+Status JsonWriter::Open(
+    const std::shared_ptr<Schema>& schema, std::unique_ptr<JsonWriter>* writer) {
+  *writer = std::unique_ptr<JsonWriter>(new JsonWriter(schema));
+  return (*writer)->impl_->Start();
+}
+
+Status JsonWriter::Finish(std::string* result) {
+  return impl_->Finish(result);
+}
+
+Status JsonWriter::WriteRecordBatch(
+    const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows) {
+  return impl_->WriteRecordBatch(columns, num_rows);
+}
+
+// ----------------------------------------------------------------------
+// Reader implementation
+
+class JsonReader::JsonReaderImpl {
+ public:
+  JsonReaderImpl(MemoryPool* pool, const std::shared_ptr<Buffer>& data)
+      : pool_(pool), data_(data) {}
+
+  Status ParseAndReadSchema() {
+    doc_.Parse(reinterpret_cast<const rj::Document::Ch*>(data_->data()),
+        static_cast<size_t>(data_->size()));
+    if (doc_.HasParseError()) { return Status::IOError("JSON parsing failed"); }
+
+    auto it = doc_.FindMember("schema");
+    RETURN_NOT_OBJECT("schema", it, doc_);
+    RETURN_NOT_OK(ReadJsonSchema(it->value, &schema_));
+
+    it = doc_.FindMember("batches");
+    RETURN_NOT_ARRAY("batches", it, doc_);
+    record_batches_ = &it->value;
+
+    return Status::OK();
+  }
+
+  Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const {
+    DCHECK_GE(i, 0) << "i out of bounds";
+    DCHECK_LT(i, static_cast<int>(record_batches_->GetArray().Size()))
+        << "i out of bounds";
+
+    const auto& batch_val = record_batches_->GetArray()[i];
+    DCHECK(batch_val.IsObject());
+
+    const auto& batch_obj = batch_val.GetObject();
+
+    auto it = batch_obj.FindMember("count");
+    RETURN_NOT_INT("count", it, batch_obj);
+    int32_t num_rows = static_cast<int32_t>(it->value.GetInt());
+
+    it = batch_obj.FindMember("columns");
+    RETURN_NOT_ARRAY("columns", it, batch_obj);
+    const auto& json_columns = it->value.GetArray();
+
+    std::vector<std::shared_ptr<Array>> columns(json_columns.Size());
+    for (size_t i = 0; i < columns.size(); ++i) {
+      const std::shared_ptr<DataType>& type = schema_->field(i)->type;
+      RETURN_NOT_OK(ReadJsonArray(pool_, json_columns[i], type, &columns[i]));
+    }
+
+    *batch = std::make_shared<RecordBatch>(schema_, num_rows, columns);
+    return Status::OK();
+  }
+
+  std::shared_ptr<Schema> schema() const { return schema_; }
+
+  int num_record_batches() const {
+    return static_cast<int>(record_batches_->GetArray().Size());
+  }
+
+ private:
+  MemoryPool* pool_;
+  std::shared_ptr<Buffer> data_;
+  rj::Document doc_;
+
+  const rj::Value* record_batches_;
+
+  std::shared_ptr<Schema> schema_;
+};
+
+JsonReader::JsonReader(MemoryPool* pool, const std::shared_ptr<Buffer>& data) {
+  impl_.reset(new JsonReaderImpl(pool, data));
+}
+
+JsonReader::~JsonReader() {}
+
+Status JsonReader::Open(
+    const std::shared_ptr<Buffer>& data, std::unique_ptr<JsonReader>* reader) {
+  return Open(default_memory_pool(), data, reader);
+}
+
+Status JsonReader::Open(MemoryPool* pool, const std::shared_ptr<Buffer>& data,
+    std::unique_ptr<JsonReader>* reader) {
+  *reader = std::unique_ptr<JsonReader>(new JsonReader(pool, data));
+  return (*reader)->impl_->ParseAndReadSchema();
+}
+
+std::shared_ptr<Schema> JsonReader::schema() const {
+  return impl_->schema();
+}
+
+int JsonReader::num_record_batches() const {
+  return impl_->num_record_batches();
+}
+
+Status JsonReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const {
+  return impl_->GetRecordBatch(i, batch);
+}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
new file mode 100644
index 0000000000000..7395be43b967d
--- /dev/null
+++ b/cpp/src/arrow/ipc/json.h
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Implement Arrow JSON serialization format
+
+#ifndef ARROW_IPC_JSON_H
+#define ARROW_IPC_JSON_H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/type_fwd.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+namespace io {
+
+class OutputStream;
+class ReadableFileInterface;
+
+}  // namespace io
+
+namespace ipc {
+
+class ARROW_EXPORT JsonWriter {
+ public:
+  ~JsonWriter();
+
+  static Status Open(
+      const std::shared_ptr<Schema>& schema, std::unique_ptr<JsonWriter>* out);
+
+  // TODO(wesm): Write dictionaries
+
+  Status WriteRecordBatch(
+      const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows);
+
+  Status Finish(std::string* result);
+
+ private:
+  explicit JsonWriter(const std::shared_ptr<Schema>& schema);
+
+  // Hide RapidJSON details from public API
+  class JsonWriterImpl;
+  std::unique_ptr<JsonWriterImpl> impl_;
+};
+
+// TODO(wesm): Read from a file stream rather than an in-memory buffer
+class ARROW_EXPORT JsonReader {
+ public:
+  ~JsonReader();
+
+  static Status Open(MemoryPool* pool, const std::shared_ptr<Buffer>& data,
+      std::unique_ptr<JsonReader>* reader);
+
+  // Use the default memory pool
+  static Status Open(
+      const std::shared_ptr<Buffer>& data, std::unique_ptr<JsonReader>* reader);
+
+  std::shared_ptr<Schema> schema() const;
+
+  int num_record_batches() const;
+
+  // Read a record batch from the file
+  Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const;
+
+ private:
+  JsonReader(MemoryPool* pool, const std::shared_ptr<Buffer>& data);
+
+  // Hide RapidJSON details from public API
+  class JsonReaderImpl;
+  std::unique_ptr<JsonReaderImpl> impl_;
+};
+
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_JSON_H
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 784e238e977c7..9abc20d876de4 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -27,6 +27,7 @@
 #include "arrow/array.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
+#include "arrow/type.h"
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"
 #include "arrow/types/string.h"
@@ -39,15 +40,14 @@ namespace arrow {
 namespace ipc {
 
 const auto kInt32 = std::make_shared<Int32Type>();
-const auto kListInt32 = std::make_shared<ListType>(kInt32);
-const auto kListListInt32 = std::make_shared<ListType>(kListInt32);
+const auto kListInt32 = list(kInt32);
+const auto kListListInt32 = list(kListInt32);
 
 Status MakeRandomInt32Array(
     int32_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
   std::shared_ptr<PoolBuffer> data;
   test::MakeRandomInt32PoolBuffer(length, pool, &data);
-  const auto kInt32 = std::make_shared<Int32Type>();
-  Int32Builder builder(pool, kInt32);
+  Int32Builder builder(pool, int32());
   if (include_nulls) {
     std::shared_ptr<PoolBuffer> valid_bytes;
     test::MakeRandomBytePoolBuffer(length, pool, &valid_bytes);
@@ -134,8 +134,8 @@ Status MakeRandomBinaryArray(
 
 Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
   const int32_t length = 500;
-  auto string_type = std::make_shared<StringType>();
-  auto binary_type = std::make_shared<BinaryType>();
+  auto string_type = utf8();
+  auto binary_type = binary();
   auto f0 = std::make_shared<Field>("f0", string_type);
   auto f1 = std::make_shared<Field>("f1", binary_type);
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
@@ -233,7 +233,7 @@ Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
   const bool include_nulls = true;
   RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool, &array));
   for (int i = 0; i < 63; ++i) {
-    type = std::static_pointer_cast<DataType>(std::make_shared<ListType>(type));
+    type = std::static_pointer_cast<DataType>(list(type));
     RETURN_NOT_OK(MakeRandomListArray(array, batch_length, include_nulls, pool, &array));
   }
 
diff --git a/cpp/src/arrow/schema-test.cc b/cpp/src/arrow/schema-test.cc
index 8cc80be120a44..4826199f73de7 100644
--- a/cpp/src/arrow/schema-test.cc
+++ b/cpp/src/arrow/schema-test.cc
@@ -29,23 +29,21 @@ using std::vector;
 
 namespace arrow {
 
-const auto INT32 = std::make_shared<Int32Type>();
-
 TEST(TestField, Basics) {
-  Field f0("f0", INT32);
-  Field f0_nn("f0", INT32, false);
+  Field f0("f0", int32());
+  Field f0_nn("f0", int32(), false);
 
   ASSERT_EQ(f0.name, "f0");
-  ASSERT_EQ(f0.type->ToString(), INT32->ToString());
+  ASSERT_EQ(f0.type->ToString(), int32()->ToString());
 
   ASSERT_TRUE(f0.nullable);
   ASSERT_FALSE(f0_nn.nullable);
 }
 
 TEST(TestField, Equals) {
-  Field f0("f0", INT32);
-  Field f0_nn("f0", INT32, false);
-  Field f0_other("f0", INT32);
+  Field f0("f0", int32());
+  Field f0_nn("f0", int32(), false);
+  Field f0_other("f0", int32());
 
   ASSERT_EQ(f0, f0_other);
   ASSERT_NE(f0, f0_nn);
@@ -57,11 +55,11 @@ class TestSchema : public ::testing::Test {
 };
 
 TEST_F(TestSchema, Basics) {
-  auto f0 = std::make_shared<Field>("f0", INT32);
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(), false);
-  auto f1_optional = std::make_shared<Field>("f1", std::make_shared<UInt8Type>());
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8(), false);
+  auto f1_optional = field("f1", uint8());
 
-  auto f2 = std::make_shared<Field>("f2", std::make_shared<StringType>());
+  auto f2 = field("f2", utf8());
 
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto schema = std::make_shared<Schema>(fields);
@@ -83,11 +81,10 @@ TEST_F(TestSchema, Basics) {
 }
 
 TEST_F(TestSchema, ToString) {
-  auto f0 = std::make_shared<Field>("f0", INT32);
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(), false);
-  auto f2 = std::make_shared<Field>("f2", std::make_shared<StringType>());
-  auto f3 = std::make_shared<Field>(
-      "f3", std::make_shared<ListType>(std::make_shared<Int16Type>()));
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8(), false);
+  auto f2 = field("f2", utf8());
+  auto f3 = field("f3", list(int16()));
 
   vector<shared_ptr<Field>> fields = {f0, f1, f2, f3};
   auto schema = std::make_shared<Schema>(fields);
@@ -101,4 +98,25 @@ f3: list<item: int16>)";
   ASSERT_EQ(expected, result);
 }
 
+TEST_F(TestSchema, GetFieldByName) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8(), false);
+  auto f2 = field("f2", utf8());
+  auto f3 = field("f3", list(int16()));
+
+  vector<shared_ptr<Field>> fields = {f0, f1, f2, f3};
+  auto schema = std::make_shared<Schema>(fields);
+
+  std::shared_ptr<Field> result;
+
+  result = schema->GetFieldByName("f1");
+  ASSERT_TRUE(f1->Equals(result));
+
+  result = schema->GetFieldByName("f3");
+  ASSERT_TRUE(f3->Equals(result));
+
+  result = schema->GetFieldByName("not-found");
+  ASSERT_TRUE(result == nullptr);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/schema.cc b/cpp/src/arrow/schema.cc
index ff3ea1990e551..cd8256e658ec3 100644
--- a/cpp/src/arrow/schema.cc
+++ b/cpp/src/arrow/schema.cc
@@ -42,6 +42,21 @@ bool Schema::Equals(const std::shared_ptr<Schema>& other) const {
   return Equals(*other.get());
 }
 
+std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
+  if (fields_.size() > 0 && name_to_index_.size() == 0) {
+    for (size_t i = 0; i < fields_.size(); ++i) {
+      name_to_index_[fields_[i]->name] = i;
+    }
+  }
+
+  auto it = name_to_index_.find(name);
+  if (it == name_to_index_.end()) {
+    return nullptr;
+  } else {
+    return fields_[it->second];
+  }
+}
+
 std::string Schema::ToString() const {
   std::stringstream buffer;
 
diff --git a/cpp/src/arrow/schema.h b/cpp/src/arrow/schema.h
index 4301968e01578..0e1ab5c368e98 100644
--- a/cpp/src/arrow/schema.h
+++ b/cpp/src/arrow/schema.h
@@ -20,14 +20,14 @@
 
 #include <memory>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
+#include "arrow/type.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
-struct Field;
-
 class ARROW_EXPORT Schema {
  public:
   explicit Schema(const std::vector<std::shared_ptr<Field>>& fields);
@@ -37,7 +37,12 @@ class ARROW_EXPORT Schema {
   bool Equals(const std::shared_ptr<Schema>& other) const;
 
   // Return the ith schema element. Does not boundscheck
-  const std::shared_ptr<Field>& field(int i) const { return fields_[i]; }
+  std::shared_ptr<Field> field(int i) const { return fields_[i]; }
+
+  // Returns nullptr if name not found
+  std::shared_ptr<Field> GetFieldByName(const std::string& name);
+
+  const std::vector<std::shared_ptr<Field>>& fields() const { return fields_; }
 
   // Render a string representation of the schema suitable for debugging
   std::string ToString() const;
@@ -46,6 +51,7 @@ class ARROW_EXPORT Schema {
 
  private:
   std::vector<std::shared_ptr<Field>> fields_;
+  std::unordered_map<std::string, int> name_to_index_;
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index ac56f5ed0871c..ab4b980b3be63 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -27,6 +27,7 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
@@ -102,20 +103,57 @@ void random_real(int n, uint32_t seed, T min_value, T max_value, std::vector<T>*
 }
 
 template <typename T>
-std::shared_ptr<Buffer> to_buffer(const std::vector<T>& values) {
+std::shared_ptr<Buffer> GetBufferFromVector(const std::vector<T>& values) {
   return std::make_shared<Buffer>(
       reinterpret_cast<const uint8_t*>(values.data()), values.size() * sizeof(T));
 }
 
+template <typename T>
+inline Status CopyBufferFromVector(
+    const std::vector<T>& values, std::shared_ptr<Buffer>* result) {
+  int64_t nbytes = static_cast<int>(values.size()) * sizeof(T);
+
+  auto buffer = std::make_shared<PoolBuffer>(default_memory_pool());
+  RETURN_NOT_OK(buffer->Resize(nbytes));
+  memcpy(buffer->mutable_data(), values.data(), nbytes);
+
+  *result = buffer;
+  return Status::OK();
+}
+
+static inline Status GetBitmapFromBoolVector(
+    const std::vector<bool>& is_valid, std::shared_ptr<Buffer>* result) {
+  int length = static_cast<int>(is_valid.size());
+
+  std::shared_ptr<MutableBuffer> buffer;
+  RETURN_NOT_OK(GetEmptyBitmap(default_memory_pool(), length, &buffer));
+
+  uint8_t* bitmap = buffer->mutable_data();
+  for (int i = 0; i < length; ++i) {
+    if (is_valid[i]) { BitUtil::SetBit(bitmap, i); }
+  }
+
+  *result = buffer;
+  return Status::OK();
+}
+
 // Sets approximately pct_null of the first n bytes in null_bytes to zero
 // and the rest to non-zero (true) values.
-void random_null_bytes(int64_t n, double pct_null, uint8_t* null_bytes) {
+static inline void random_null_bytes(int64_t n, double pct_null, uint8_t* null_bytes) {
   Random rng(random_seed());
   for (int i = 0; i < n; ++i) {
     null_bytes[i] = rng.NextDoubleFraction() > pct_null;
   }
 }
 
+static inline void random_is_valid(
+    int64_t n, double pct_null, std::vector<bool>* is_valid) {
+  Random rng(random_seed());
+  for (int i = 0; i < n; ++i) {
+    is_valid->push_back(rng.NextDoubleFraction() > pct_null);
+  }
+}
+
 static inline void random_bytes(int n, uint32_t seed, uint8_t* out) {
   std::mt19937 gen(seed);
   std::uniform_int_distribution<int> d(0, 255);
@@ -125,6 +163,15 @@ static inline void random_bytes(int n, uint32_t seed, uint8_t* out) {
   }
 }
 
+static inline void random_ascii(int n, uint32_t seed, uint8_t* out) {
+  std::mt19937 gen(seed);
+  std::uniform_int_distribution<int> d(65, 122);
+
+  for (int i = 0; i < n; ++i) {
+    out[i] = d(gen) & 0xFF;
+  }
+}
+
 template <typename T>
 void rand_uniform_int(int n, uint32_t seed, T min_value, T max_value, T* out) {
   DCHECK(out);
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 4fd50b7c19365..589bdadb77c64 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -20,6 +20,8 @@
 #include <sstream>
 #include <string>
 
+#include "arrow/util/status.h"
+
 namespace arrow {
 
 std::string Field::ToString() const {
@@ -44,9 +46,24 @@ bool DataType::Equals(const DataType* other) const {
   return equals;
 }
 
+std::string BooleanType::ToString() const {
+  return name();
+}
+
+FloatingPointMeta::Precision HalfFloatType::precision() const {
+  return FloatingPointMeta::HALF;
+}
+
+FloatingPointMeta::Precision FloatType::precision() const {
+  return FloatingPointMeta::SINGLE;
+}
+
+FloatingPointMeta::Precision DoubleType::precision() const {
+  return FloatingPointMeta::DOUBLE;
+}
+
 std::string StringType::ToString() const {
-  std::string result(name());
-  return result;
+  return std::string("string");
 }
 
 std::string ListType::ToString() const {
@@ -56,7 +73,7 @@ std::string ListType::ToString() const {
 }
 
 std::string BinaryType::ToString() const {
-  return std::string(name());
+  return std::string("binary");
 }
 
 std::string StructType::ToString() const {
@@ -71,4 +88,103 @@ std::string StructType::ToString() const {
   return s.str();
 }
 
+std::string UnionType::ToString() const {
+  std::stringstream s;
+
+  if (mode == UnionMode::SPARSE) {
+    s << "union[sparse]<";
+  } else {
+    s << "union[dense]<";
+  }
+
+  for (size_t i = 0; i < children_.size(); ++i) {
+    if (i) { s << ", "; }
+    s << children_[i]->ToString();
+  }
+  s << ">";
+  return s.str();
+}
+
+int NullType::bit_width() const {
+  return 0;
+}
+
+std::string NullType::ToString() const {
+  return name();
+}
+
+// Visitors and template instantiation
+
+#define ACCEPT_VISITOR(TYPE) \
+  Status TYPE::Accept(TypeVisitor* visitor) const { return visitor->Visit(*this); }
+
+ACCEPT_VISITOR(NullType);
+ACCEPT_VISITOR(BooleanType);
+ACCEPT_VISITOR(BinaryType);
+ACCEPT_VISITOR(StringType);
+ACCEPT_VISITOR(ListType);
+ACCEPT_VISITOR(StructType);
+ACCEPT_VISITOR(DecimalType);
+ACCEPT_VISITOR(UnionType);
+ACCEPT_VISITOR(DateType);
+ACCEPT_VISITOR(TimeType);
+ACCEPT_VISITOR(TimestampType);
+ACCEPT_VISITOR(IntervalType);
+
+#define TYPE_FACTORY(NAME, KLASS)                                        \
+  std::shared_ptr<DataType> NAME() {                                     \
+    static std::shared_ptr<DataType> result = std::make_shared<KLASS>(); \
+    return result;                                                       \
+  }
+
+TYPE_FACTORY(null, NullType);
+TYPE_FACTORY(boolean, BooleanType);
+TYPE_FACTORY(int8, Int8Type);
+TYPE_FACTORY(uint8, UInt8Type);
+TYPE_FACTORY(int16, Int16Type);
+TYPE_FACTORY(uint16, UInt16Type);
+TYPE_FACTORY(int32, Int32Type);
+TYPE_FACTORY(uint32, UInt32Type);
+TYPE_FACTORY(int64, Int64Type);
+TYPE_FACTORY(uint64, UInt64Type);
+TYPE_FACTORY(float16, HalfFloatType);
+TYPE_FACTORY(float32, FloatType);
+TYPE_FACTORY(float64, DoubleType);
+TYPE_FACTORY(utf8, StringType);
+TYPE_FACTORY(binary, BinaryType);
+TYPE_FACTORY(date, DateType);
+
+std::shared_ptr<DataType> timestamp(TimeUnit unit) {
+  static std::shared_ptr<DataType> result = std::make_shared<TimestampType>();
+  return result;
+}
+
+std::shared_ptr<DataType> time(TimeUnit unit) {
+  static std::shared_ptr<DataType> result = std::make_shared<TimeType>();
+  return result;
+}
+
+std::shared_ptr<DataType> list(const std::shared_ptr<DataType>& value_type) {
+  return std::make_shared<ListType>(value_type);
+}
+
+std::shared_ptr<DataType> list(const std::shared_ptr<Field>& value_field) {
+  return std::make_shared<ListType>(value_field);
+}
+
+std::shared_ptr<DataType> struct_(const std::vector<std::shared_ptr<Field>>& fields) {
+  return std::make_shared<StructType>(fields);
+}
+
+std::shared_ptr<DataType> ARROW_EXPORT union_(
+    const std::vector<std::shared_ptr<Field>>& child_fields,
+    const std::vector<uint8_t>& type_ids, UnionMode mode) {
+  return std::make_shared<UnionType>(child_fields, type_ids, mode);
+}
+
+std::shared_ptr<Field> field(
+    const std::string& name, const TypePtr& type, bool nullable, int64_t dictionary) {
+  return std::make_shared<Field>(name, type, nullable, dictionary);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index ea8516fc34798..5b4d7bc42bd3d 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -23,7 +23,9 @@
 #include <string>
 #include <vector>
 
+#include "arrow/type_fwd.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/status.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -50,17 +52,20 @@ struct Type {
     UINT64 = 8,
     INT64 = 9,
 
+    // 2-byte floating point value
+    HALF_FLOAT = 10,
+
     // 4-byte floating point value
-    FLOAT = 10,
+    FLOAT = 11,
 
     // 8-byte floating point value
-    DOUBLE = 11,
+    DOUBLE = 12,
 
     // UTF8 variable-length string as List<Char>
     STRING = 13,
 
     // Variable-length bytes (no guarantee of UTF8-ness)
-    BINARY = 15,
+    BINARY = 14,
 
     // By default, int32 days since the UNIX epoch
     DATE = 16,
@@ -69,19 +74,16 @@ struct Type {
     // Default unit millisecond
     TIMESTAMP = 17,
 
-    // Timestamp as double seconds since the UNIX epoch
-    TIMESTAMP_DOUBLE = 18,
-
     // Exact time encoded with int64, default unit millisecond
-    TIME = 19,
+    TIME = 18,
+
+    // YEAR_MONTH or DAY_TIME interval in SQL style
+    INTERVAL = 19,
 
     // Precision- and scale-based decimal type. Storage type depends on the
     // parameters.
     DECIMAL = 20,
 
-    // Decimal value encoded as a text string
-    DECIMAL_TEXT = 21,
-
     // A list of some logical data type
     LIST = 30,
 
@@ -89,19 +91,16 @@ struct Type {
     STRUCT = 31,
 
     // Unions of logical types
-    DENSE_UNION = 32,
-    SPARSE_UNION = 33,
+    UNION = 32,
 
-    // Union<Null, Int32, Double, String, Bool>
-    JSON_SCALAR = 50,
+    // Timestamp as double seconds since the UNIX epoch
+    TIMESTAMP_DOUBLE = 33,
 
-    // User-defined type
-    USER = 60
+    // Decimal value encoded as a text string
+    DECIMAL_TEXT = 34,
   };
 };
 
-struct Field;
-
 struct ARROW_EXPORT DataType {
   Type::type type;
 
@@ -123,15 +122,32 @@ struct ARROW_EXPORT DataType {
 
   const std::shared_ptr<Field>& child(int i) const { return children_[i]; }
 
+  const std::vector<std::shared_ptr<Field>>& children() const { return children_; }
+
   int num_children() const { return children_.size(); }
 
-  virtual int value_size() const { return -1; }
+  virtual Status Accept(TypeVisitor* visitor) const = 0;
 
   virtual std::string ToString() const = 0;
 };
 
 typedef std::shared_ptr<DataType> TypePtr;
 
+struct ARROW_EXPORT FixedWidthMeta {
+  virtual int bit_width() const = 0;
+};
+
+struct ARROW_EXPORT IntegerMeta {
+  virtual bool is_signed() const = 0;
+};
+
+struct ARROW_EXPORT FloatingPointMeta {
+  enum Precision { HALF, SINGLE, DOUBLE };
+  virtual Precision precision() const = 0;
+};
+
+struct NoExtraMeta {};
+
 // A field is a piece of metadata that includes (for now) a name and a data
 // type
 struct ARROW_EXPORT Field {
@@ -139,7 +155,7 @@ struct ARROW_EXPORT Field {
   std::string name;
 
   // The field's data type
-  TypePtr type;
+  std::shared_ptr<DataType> type;
 
   // Fields can be nullable
   bool nullable;
@@ -148,8 +164,8 @@ struct ARROW_EXPORT Field {
   // 0 means it's not dictionary encoded
   int64_t dictionary;
 
-  Field(const std::string& name, const TypePtr& type, bool nullable = true,
-      int64_t dictionary = 0)
+  Field(const std::string& name, const std::shared_ptr<DataType>& type,
+      bool nullable = true, int64_t dictionary = 0)
       : name(name), type(type), nullable(nullable), dictionary(dictionary) {}
 
   bool operator==(const Field& other) const { return this->Equals(other); }
@@ -168,78 +184,112 @@ struct ARROW_EXPORT Field {
 };
 typedef std::shared_ptr<Field> FieldPtr;
 
-template <typename Derived>
-struct ARROW_EXPORT PrimitiveType : public DataType {
-  PrimitiveType() : DataType(Derived::type_enum) {}
+struct PrimitiveCType : public DataType {
+  using DataType::DataType;
+};
+
+template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
+struct ARROW_EXPORT CTypeImpl : public PrimitiveCType, public FixedWidthMeta {
+  using c_type = C_TYPE;
+  static constexpr Type::type type_id = TYPE_ID;
+
+  CTypeImpl() : PrimitiveCType(TYPE_ID) {}
 
+  int bit_width() const override { return sizeof(C_TYPE) * 8; }
+
+  Status Accept(TypeVisitor* visitor) const override {
+    return visitor->Visit(*static_cast<const DERIVED*>(this));
+  }
+
+  std::string ToString() const override { return std::string(DERIVED::name()); }
+};
+
+struct ARROW_EXPORT NullType : public DataType, public FixedWidthMeta {
+  static constexpr Type::type type_id = Type::NA;
+
+  NullType() : DataType(Type::NA) {}
+
+  int bit_width() const override;
+  Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+
+  static std::string name() { return "null"; }
+};
+
+template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
+struct IntegerTypeImpl : public CTypeImpl<DERIVED, TYPE_ID, C_TYPE>, public IntegerMeta {
+  bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
 };
 
-template <typename Derived>
-inline std::string PrimitiveType<Derived>::ToString() const {
-  std::string result(static_cast<const Derived*>(this)->name());
-  return result;
-}
+struct ARROW_EXPORT BooleanType : public DataType, FixedWidthMeta {
+  static constexpr Type::type type_id = Type::BOOL;
 
-#define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME) \
-  typedef C_TYPE c_type;                                   \
-  static constexpr Type::type type_enum = Type::ENUM;      \
-                                                           \
-  TYPENAME() : PrimitiveType<TYPENAME>() {}                \
-                                                           \
-  virtual int value_size() const { return SIZE; }          \
-                                                           \
-  static const char* name() { return NAME; }
+  BooleanType() : DataType(Type::BOOL) {}
 
-struct ARROW_EXPORT NullType : public PrimitiveType<NullType> {
-  PRIMITIVE_DECL(NullType, void, NA, 0, "null");
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override;
+
+  int bit_width() const override { return 1; }
+  static std::string name() { return "bool"; }
 };
 
-struct ARROW_EXPORT BooleanType : public PrimitiveType<BooleanType> {
-  PRIMITIVE_DECL(BooleanType, uint8_t, BOOL, 1, "bool");
+struct ARROW_EXPORT UInt8Type : public IntegerTypeImpl<UInt8Type, Type::UINT8, uint8_t> {
+  static std::string name() { return "uint8"; }
 };
 
-struct ARROW_EXPORT UInt8Type : public PrimitiveType<UInt8Type> {
-  PRIMITIVE_DECL(UInt8Type, uint8_t, UINT8, 1, "uint8");
+struct ARROW_EXPORT Int8Type : public IntegerTypeImpl<Int8Type, Type::INT8, int8_t> {
+  static std::string name() { return "int8"; }
 };
 
-struct ARROW_EXPORT Int8Type : public PrimitiveType<Int8Type> {
-  PRIMITIVE_DECL(Int8Type, int8_t, INT8, 1, "int8");
+struct ARROW_EXPORT UInt16Type
+    : public IntegerTypeImpl<UInt16Type, Type::UINT16, uint16_t> {
+  static std::string name() { return "uint16"; }
 };
 
-struct ARROW_EXPORT UInt16Type : public PrimitiveType<UInt16Type> {
-  PRIMITIVE_DECL(UInt16Type, uint16_t, UINT16, 2, "uint16");
+struct ARROW_EXPORT Int16Type : public IntegerTypeImpl<Int16Type, Type::INT16, int16_t> {
+  static std::string name() { return "int16"; }
 };
 
-struct ARROW_EXPORT Int16Type : public PrimitiveType<Int16Type> {
-  PRIMITIVE_DECL(Int16Type, int16_t, INT16, 2, "int16");
+struct ARROW_EXPORT UInt32Type
+    : public IntegerTypeImpl<UInt32Type, Type::UINT32, uint32_t> {
+  static std::string name() { return "uint32"; }
 };
 
-struct ARROW_EXPORT UInt32Type : public PrimitiveType<UInt32Type> {
-  PRIMITIVE_DECL(UInt32Type, uint32_t, UINT32, 4, "uint32");
+struct ARROW_EXPORT Int32Type : public IntegerTypeImpl<Int32Type, Type::INT32, int32_t> {
+  static std::string name() { return "int32"; }
 };
 
-struct ARROW_EXPORT Int32Type : public PrimitiveType<Int32Type> {
-  PRIMITIVE_DECL(Int32Type, int32_t, INT32, 4, "int32");
+struct ARROW_EXPORT UInt64Type
+    : public IntegerTypeImpl<UInt64Type, Type::UINT64, uint64_t> {
+  static std::string name() { return "uint64"; }
 };
 
-struct ARROW_EXPORT UInt64Type : public PrimitiveType<UInt64Type> {
-  PRIMITIVE_DECL(UInt64Type, uint64_t, UINT64, 8, "uint64");
+struct ARROW_EXPORT Int64Type : public IntegerTypeImpl<Int64Type, Type::INT64, int64_t> {
+  static std::string name() { return "int64"; }
 };
 
-struct ARROW_EXPORT Int64Type : public PrimitiveType<Int64Type> {
-  PRIMITIVE_DECL(Int64Type, int64_t, INT64, 8, "int64");
+struct ARROW_EXPORT HalfFloatType
+    : public CTypeImpl<HalfFloatType, Type::HALF_FLOAT, uint16_t>,
+      public FloatingPointMeta {
+  Precision precision() const override;
+  static std::string name() { return "halffloat"; }
 };
 
-struct ARROW_EXPORT FloatType : public PrimitiveType<FloatType> {
-  PRIMITIVE_DECL(FloatType, float, FLOAT, 4, "float");
+struct ARROW_EXPORT FloatType : public CTypeImpl<FloatType, Type::FLOAT, float>,
+                                public FloatingPointMeta {
+  Precision precision() const override;
+  static std::string name() { return "float"; }
 };
 
-struct ARROW_EXPORT DoubleType : public PrimitiveType<DoubleType> {
-  PRIMITIVE_DECL(DoubleType, double, DOUBLE, 8, "double");
+struct ARROW_EXPORT DoubleType : public CTypeImpl<DoubleType, Type::DOUBLE, double>,
+                                 public FloatingPointMeta {
+  Precision precision() const override;
+  static std::string name() { return "double"; }
 };
 
-struct ARROW_EXPORT ListType : public DataType {
+struct ARROW_EXPORT ListType : public DataType, public NoExtraMeta {
+  static constexpr Type::type type_id = Type::LIST;
+
   // List can contain any other logical value type
   explicit ListType(const std::shared_ptr<DataType>& value_type)
       : ListType(std::make_shared<Field>("item", value_type)) {}
@@ -252,16 +302,21 @@ struct ARROW_EXPORT ListType : public DataType {
 
   const std::shared_ptr<DataType>& value_type() const { return children_[0]->type; }
 
-  static char const* name() { return "list"; }
-
+  Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+
+  static std::string name() { return "list"; }
 };
 
 // BinaryType type is reprsents lists of 1-byte values.
-struct ARROW_EXPORT BinaryType : public DataType {
+struct ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
+  static constexpr Type::type type_id = Type::BINARY;
+
   BinaryType() : BinaryType(Type::BINARY) {}
-  static char const* name() { return "binary"; }
+
+  Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+  static std::string name() { return "binary"; }
 
  protected:
   // Allow subclasses to change the logical type.
@@ -270,25 +325,160 @@ struct ARROW_EXPORT BinaryType : public DataType {
 
 // UTF encoded strings
 struct ARROW_EXPORT StringType : public BinaryType {
-  StringType() : BinaryType(Type::STRING) {}
+  static constexpr Type::type type_id = Type::STRING;
 
-  static char const* name() { return "string"; }
+  StringType() : BinaryType(Type::STRING) {}
 
+  Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+  static std::string name() { return "utf8"; }
 };
 
-struct ARROW_EXPORT StructType : public DataType {
+struct ARROW_EXPORT StructType : public DataType, public NoExtraMeta {
+  static constexpr Type::type type_id = Type::STRUCT;
+
   explicit StructType(const std::vector<std::shared_ptr<Field>>& fields)
       : DataType(Type::STRUCT) {
     children_ = fields;
   }
 
+  Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+  static std::string name() { return "struct"; }
+};
+
+struct ARROW_EXPORT DecimalType : public DataType {
+  static constexpr Type::type type_id = Type::DECIMAL;
+
+  explicit DecimalType(int precision_, int scale_)
+      : DataType(Type::DECIMAL), precision(precision_), scale(scale_) {}
+  int precision;
+  int scale;
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override;
+  static std::string name() { return "decimal"; }
+};
+
+enum class UnionMode : char { SPARSE, DENSE };
+
+struct ARROW_EXPORT UnionType : public DataType {
+  static constexpr Type::type type_id = Type::UNION;
+
+  UnionType(const std::vector<std::shared_ptr<Field>>& child_fields,
+      const std::vector<uint8_t>& type_ids, UnionMode mode = UnionMode::SPARSE)
+      : DataType(Type::UNION), mode(mode), type_ids(type_ids) {
+    children_ = child_fields;
+  }
+
+  std::string ToString() const override;
+  static std::string name() { return "union"; }
+  Status Accept(TypeVisitor* visitor) const override;
+
+  UnionMode mode;
+  std::vector<uint8_t> type_ids;
+};
+
+struct ARROW_EXPORT DateType : public DataType, public NoExtraMeta {
+  static constexpr Type::type type_id = Type::DATE;
+
+  DateType() : DataType(Type::DATE) {}
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override { return name(); }
+  static std::string name() { return "date"; }
+};
+
+enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
+
+struct ARROW_EXPORT TimeType : public DataType {
+  static constexpr Type::type type_id = Type::TIME;
+  using Unit = TimeUnit;
+
+  TimeUnit unit;
+
+  explicit TimeType(TimeUnit unit = TimeUnit::MILLI) : DataType(Type::TIME), unit(unit) {}
+  TimeType(const TimeType& other) : TimeType(other.unit) {}
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override { return name(); }
+  static std::string name() { return "time"; }
+};
+
+struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
+  using Unit = TimeUnit;
+
+  typedef int64_t c_type;
+  static constexpr Type::type type_id = Type::TIMESTAMP;
+
+  int bit_width() const override { return sizeof(int64_t) * 8; }
+
+  TimeUnit unit;
+
+  explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
+      : DataType(Type::TIMESTAMP), unit(unit) {}
+
+  TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override { return name(); }
+  static std::string name() { return "timestamp"; }
+};
+
+struct ARROW_EXPORT IntervalType : public DataType, public FixedWidthMeta {
+  enum class Unit : char { YEAR_MONTH = 0, DAY_TIME = 1 };
+
+  typedef int64_t c_type;
+  static constexpr Type::type type_id = Type::INTERVAL;
+
+  int bit_width() const override { return sizeof(int64_t) * 8; }
+
+  Unit unit;
+
+  explicit IntervalType(Unit unit = Unit::YEAR_MONTH)
+      : DataType(Type::INTERVAL), unit(unit) {}
+
+  IntervalType(const IntervalType& other) : IntervalType(other.unit) {}
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override { return name(); }
+  static std::string name() { return "date"; }
 };
 
-// These will be defined elsewhere
-template <typename T>
-struct TypeTraits {};
+// Factory functions
+
+std::shared_ptr<DataType> ARROW_EXPORT null();
+std::shared_ptr<DataType> ARROW_EXPORT boolean();
+std::shared_ptr<DataType> ARROW_EXPORT int8();
+std::shared_ptr<DataType> ARROW_EXPORT int16();
+std::shared_ptr<DataType> ARROW_EXPORT int32();
+std::shared_ptr<DataType> ARROW_EXPORT int64();
+std::shared_ptr<DataType> ARROW_EXPORT uint8();
+std::shared_ptr<DataType> ARROW_EXPORT uint16();
+std::shared_ptr<DataType> ARROW_EXPORT uint32();
+std::shared_ptr<DataType> ARROW_EXPORT uint64();
+std::shared_ptr<DataType> ARROW_EXPORT float16();
+std::shared_ptr<DataType> ARROW_EXPORT float32();
+std::shared_ptr<DataType> ARROW_EXPORT float64();
+std::shared_ptr<DataType> ARROW_EXPORT utf8();
+std::shared_ptr<DataType> ARROW_EXPORT binary();
+
+std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type);
+std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
+
+std::shared_ptr<DataType> ARROW_EXPORT date();
+std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit unit);
+std::shared_ptr<DataType> ARROW_EXPORT time(TimeUnit unit);
+
+std::shared_ptr<DataType> ARROW_EXPORT struct_(
+    const std::vector<std::shared_ptr<Field>>& fields);
+
+std::shared_ptr<DataType> ARROW_EXPORT union_(
+    const std::vector<std::shared_ptr<Field>>& child_fields,
+    const std::vector<uint8_t>& type_ids, UnionMode mode = UnionMode::SPARSE);
+
+std::shared_ptr<Field> ARROW_EXPORT field(const std::string& name,
+    const std::shared_ptr<DataType>& type, bool nullable = true, int64_t dictionary = 0);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
new file mode 100644
index 0000000000000..6d660f4fdee43
--- /dev/null
+++ b/cpp/src/arrow/type_fwd.h
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPE_FWD_H
+#define ARROW_TYPE_FWD_H
+
+namespace arrow {
+
+class Status;
+
+struct DataType;
+class Array;
+class ArrayBuilder;
+struct Field;
+
+class Buffer;
+class MemoryPool;
+class RecordBatch;
+class Schema;
+
+struct NullType;
+class NullArray;
+
+struct BooleanType;
+class BooleanArray;
+class BooleanBuilder;
+
+struct BinaryType;
+class BinaryArray;
+class BinaryBuilder;
+
+struct StringType;
+class StringArray;
+class StringBuilder;
+
+struct ListType;
+class ListArray;
+class ListBuilder;
+
+struct StructType;
+class StructArray;
+class StructBuilder;
+
+struct DecimalType;
+class DecimalArray;
+
+struct UnionType;
+class UnionArray;
+
+template <typename TypeClass>
+class NumericArray;
+
+template <typename TypeClass>
+class NumericBuilder;
+
+#define _NUMERIC_TYPE_DECL(KLASS)                 \
+  struct KLASS##Type;                             \
+  using KLASS##Array = NumericArray<KLASS##Type>; \
+  using KLASS##Builder = NumericBuilder<KLASS##Type>;
+
+_NUMERIC_TYPE_DECL(Int8);
+_NUMERIC_TYPE_DECL(Int16);
+_NUMERIC_TYPE_DECL(Int32);
+_NUMERIC_TYPE_DECL(Int64);
+_NUMERIC_TYPE_DECL(UInt8);
+_NUMERIC_TYPE_DECL(UInt16);
+_NUMERIC_TYPE_DECL(UInt32);
+_NUMERIC_TYPE_DECL(UInt64);
+_NUMERIC_TYPE_DECL(HalfFloat);
+_NUMERIC_TYPE_DECL(Float);
+_NUMERIC_TYPE_DECL(Double);
+
+#undef _NUMERIC_TYPE_DECL
+
+struct DateType;
+class DateArray;
+
+struct TimeType;
+class TimeArray;
+
+struct TimestampType;
+using TimestampArray = NumericArray<TimestampType>;
+
+struct IntervalType;
+using IntervalArray = NumericArray<IntervalType>;
+
+class TypeVisitor {
+ public:
+  virtual Status Visit(const NullType& type) = 0;
+  virtual Status Visit(const BooleanType& type) = 0;
+  virtual Status Visit(const Int8Type& type) = 0;
+  virtual Status Visit(const Int16Type& type) = 0;
+  virtual Status Visit(const Int32Type& type) = 0;
+  virtual Status Visit(const Int64Type& type) = 0;
+  virtual Status Visit(const UInt8Type& type) = 0;
+  virtual Status Visit(const UInt16Type& type) = 0;
+  virtual Status Visit(const UInt32Type& type) = 0;
+  virtual Status Visit(const UInt64Type& type) = 0;
+  virtual Status Visit(const HalfFloatType& type) = 0;
+  virtual Status Visit(const FloatType& type) = 0;
+  virtual Status Visit(const DoubleType& type) = 0;
+  virtual Status Visit(const StringType& type) = 0;
+  virtual Status Visit(const BinaryType& type) = 0;
+  virtual Status Visit(const DateType& type) = 0;
+  virtual Status Visit(const TimeType& type) = 0;
+  virtual Status Visit(const TimestampType& type) = 0;
+  virtual Status Visit(const IntervalType& type) = 0;
+  virtual Status Visit(const DecimalType& type) = 0;
+  virtual Status Visit(const ListType& type) = 0;
+  virtual Status Visit(const StructType& type) = 0;
+  virtual Status Visit(const UnionType& type) = 0;
+};
+
+class ArrayVisitor {
+ public:
+  virtual Status Visit(const NullArray& array) = 0;
+  virtual Status Visit(const BooleanArray& array) = 0;
+  virtual Status Visit(const Int8Array& array) = 0;
+  virtual Status Visit(const Int16Array& array) = 0;
+  virtual Status Visit(const Int32Array& array) = 0;
+  virtual Status Visit(const Int64Array& array) = 0;
+  virtual Status Visit(const UInt8Array& array) = 0;
+  virtual Status Visit(const UInt16Array& array) = 0;
+  virtual Status Visit(const UInt32Array& array) = 0;
+  virtual Status Visit(const UInt64Array& array) = 0;
+  virtual Status Visit(const HalfFloatArray& array) = 0;
+  virtual Status Visit(const FloatArray& array) = 0;
+  virtual Status Visit(const DoubleArray& array) = 0;
+  virtual Status Visit(const StringArray& array) = 0;
+  virtual Status Visit(const BinaryArray& array) = 0;
+  virtual Status Visit(const DateArray& array) = 0;
+  virtual Status Visit(const TimeArray& array) = 0;
+  virtual Status Visit(const TimestampArray& array) = 0;
+  virtual Status Visit(const IntervalArray& array) = 0;
+  virtual Status Visit(const DecimalArray& array) = 0;
+  virtual Status Visit(const ListArray& array) = 0;
+  virtual Status Visit(const StructArray& array) = 0;
+  virtual Status Visit(const UnionArray& array) = 0;
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_TYPE_FWD_H
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
new file mode 100644
index 0000000000000..bbb807488e3d0
--- /dev/null
+++ b/cpp/src/arrow/type_traits.h
@@ -0,0 +1,197 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TYPE_TRAITS_H
+#define ARROW_TYPE_TRAITS_H
+
+#include <type_traits>
+
+#include "arrow/type_fwd.h"
+#include "arrow/util/bit-util.h"
+
+namespace arrow {
+
+template <typename T>
+struct TypeTraits {};
+
+template <>
+struct TypeTraits<UInt8Type> {
+  using ArrayType = UInt8Array;
+  using BuilderType = UInt8Builder;
+  static inline int bytes_required(int elements) { return elements; }
+};
+
+template <>
+struct TypeTraits<Int8Type> {
+  using ArrayType = Int8Array;
+  using BuilderType = Int8Builder;
+  static inline int bytes_required(int elements) { return elements; }
+};
+
+template <>
+struct TypeTraits<UInt16Type> {
+  using ArrayType = UInt16Array;
+  using BuilderType = UInt16Builder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
+};
+
+template <>
+struct TypeTraits<Int16Type> {
+  using ArrayType = Int16Array;
+  using BuilderType = Int16Builder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(int16_t); }
+};
+
+template <>
+struct TypeTraits<UInt32Type> {
+  using ArrayType = UInt32Array;
+  using BuilderType = UInt32Builder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(uint32_t); }
+};
+
+template <>
+struct TypeTraits<Int32Type> {
+  using ArrayType = Int32Array;
+  using BuilderType = Int32Builder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(int32_t); }
+};
+
+template <>
+struct TypeTraits<UInt64Type> {
+  using ArrayType = UInt64Array;
+  using BuilderType = UInt64Builder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(uint64_t); }
+};
+
+template <>
+struct TypeTraits<Int64Type> {
+  using ArrayType = Int64Array;
+  using BuilderType = Int64Builder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+};
+
+template <>
+struct TypeTraits<TimestampType> {
+  using ArrayType = TimestampArray;
+  // using BuilderType = TimestampBuilder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+};
+
+template <>
+struct TypeTraits<HalfFloatType> {
+  using ArrayType = HalfFloatArray;
+  using BuilderType = HalfFloatBuilder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
+};
+
+template <>
+struct TypeTraits<FloatType> {
+  using ArrayType = FloatArray;
+  using BuilderType = FloatBuilder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(float); }
+};
+
+template <>
+struct TypeTraits<DoubleType> {
+  using ArrayType = DoubleArray;
+  using BuilderType = DoubleBuilder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(double); }
+};
+
+template <>
+struct TypeTraits<BooleanType> {
+  using ArrayType = BooleanArray;
+  using BuilderType = BooleanBuilder;
+
+  static inline int bytes_required(int elements) {
+    return BitUtil::BytesForBits(elements);
+  }
+};
+
+template <>
+struct TypeTraits<StringType> {
+  using ArrayType = StringArray;
+  using BuilderType = StringBuilder;
+};
+
+template <>
+struct TypeTraits<BinaryType> {
+  using ArrayType = BinaryArray;
+  using BuilderType = BinaryBuilder;
+};
+
+// Not all type classes have a c_type
+template <typename T>
+struct as_void {
+  using type = void;
+};
+
+// The partial specialization will match if T has the ATTR_NAME member
+#define GET_ATTR(ATTR_NAME, DEFAULT)                                             \
+  template <typename T, typename Enable = void>                                  \
+  struct GetAttr_##ATTR_NAME {                                                   \
+    using type = DEFAULT;                                                        \
+  };                                                                             \
+                                                                                 \
+  template <typename T>                                                          \
+  struct GetAttr_##ATTR_NAME<T, typename as_void<typename T::ATTR_NAME>::type> { \
+    using type = typename T::ATTR_NAME;                                          \
+  };
+
+GET_ATTR(c_type, void);
+GET_ATTR(TypeClass, void);
+
+#undef GET_ATTR
+
+#define PRIMITIVE_TRAITS(T)                                                           \
+  using TypeClass = typename std::conditional<std::is_base_of<DataType, T>::value, T, \
+      typename GetAttr_TypeClass<T>::type>::type;                                     \
+  using c_type = typename GetAttr_c_type<TypeClass>::type;
+
+template <typename T>
+struct IsUnsignedInt {
+  PRIMITIVE_TRAITS(T);
+  static constexpr bool value =
+      std::is_integral<c_type>::value && std::is_unsigned<c_type>::value;
+};
+
+template <typename T>
+struct IsSignedInt {
+  PRIMITIVE_TRAITS(T);
+  static constexpr bool value =
+      std::is_integral<c_type>::value && std::is_signed<c_type>::value;
+};
+
+template <typename T>
+struct IsFloatingPoint {
+  PRIMITIVE_TRAITS(T);
+  static constexpr bool value = std::is_floating_point<c_type>::value;
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_TYPE_TRAITS_H
diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
index 9f7816989827d..6d59acfdf2eec 100644
--- a/cpp/src/arrow/types/CMakeLists.txt
+++ b/cpp/src/arrow/types/CMakeLists.txt
@@ -21,7 +21,6 @@
 
 # Headers: top level
 install(FILES
-  collection.h
   construct.h
   datetime.h
   decimal.h
diff --git a/cpp/src/arrow/types/collection.h b/cpp/src/arrow/types/collection.h
deleted file mode 100644
index 1712030203fa2..0000000000000
--- a/cpp/src/arrow/types/collection.h
+++ /dev/null
@@ -1,41 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_COLLECTION_H
-#define ARROW_TYPES_COLLECTION_H
-
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-
-namespace arrow {
-
-template <Type::type T>
-struct CollectionType : public DataType {
-  std::vector<TypePtr> child_types_;
-
-  CollectionType() : DataType(T) {}
-
-  const TypePtr& child(int i) const { return child_types_[i]; }
-
-  int num_children() const { return child_types_.size(); }
-};
-
-}  // namespace arrow
-
-#endif  // ARROW_TYPES_COLLECTION_H
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
index 241a126d1007f..a8f863923129a 100644
--- a/cpp/src/arrow/types/datetime.h
+++ b/cpp/src/arrow/types/datetime.h
@@ -22,41 +22,6 @@
 
 #include "arrow/type.h"
 
-namespace arrow {
-
-struct DateType : public DataType {
-  enum class Unit : char { DAY = 0, MONTH = 1, YEAR = 2 };
-
-  Unit unit;
-
-  explicit DateType(Unit unit = Unit::DAY) : DataType(Type::DATE), unit(unit) {}
-
-  DateType(const DateType& other) : DateType(other.unit) {}
-
-  static char const* name() { return "date"; }
-};
-
-struct ARROW_EXPORT TimestampType : public DataType {
-  enum class Unit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
-
-  typedef int64_t c_type;
-  static constexpr Type::type type_enum = Type::TIMESTAMP;
-
-  int value_size() const override { return sizeof(int64_t); }
-
-  Unit unit;
-
-  explicit TimestampType(Unit unit = Unit::MILLI)
-      : DataType(Type::TIMESTAMP), unit(unit) {}
-
-  TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
-  virtual ~TimestampType() {}
-
-  std::string ToString() const override { return "timestamp"; }
-
-  static char const* name() { return "timestamp"; }
-};
-
-}  // namespace arrow
+namespace arrow {}  // namespace arrow
 
 #endif  // ARROW_TYPES_DATETIME_H
diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h
index 6c497c597d987..b3ea3a56d8008 100644
--- a/cpp/src/arrow/types/decimal.h
+++ b/cpp/src/arrow/types/decimal.h
@@ -23,18 +23,6 @@
 #include "arrow/type.h"
 #include "arrow/util/visibility.h"
 
-namespace arrow {
-
-struct ARROW_EXPORT DecimalType : public DataType {
-  explicit DecimalType(int precision_, int scale_)
-      : DataType(Type::DECIMAL), precision(precision_), scale(scale_) {}
-  int precision;
-  int scale;
-  static char const* name() { return "decimal"; }
-
-  std::string ToString() const override;
-};
-
-}  // namespace arrow
+namespace arrow {}  // namespace arrow
 
 #endif  // ARROW_TYPES_DECIMAL_H
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
index 12c539495a28b..cb9a8c12d8ab9 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/types/list-test.cc
@@ -141,7 +141,7 @@ TEST_F(TestListBuilder, TestAppendNull) {
   ASSERT_TRUE(result_->IsNull(0));
   ASSERT_TRUE(result_->IsNull(1));
 
-  ASSERT_EQ(0, result_->offsets()[0]);
+  ASSERT_EQ(0, result_->raw_offsets()[0]);
   ASSERT_EQ(0, result_->offset(1));
   ASSERT_EQ(0, result_->offset(2));
 
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
index 4b1e821472795..d86563253bd5a 100644
--- a/cpp/src/arrow/types/list.cc
+++ b/cpp/src/arrow/types/list.cc
@@ -155,4 +155,8 @@ void ListBuilder::Reset() {
   null_bitmap_ = nullptr;
 }
 
+Status ListArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index 9440ffed4bf8a..bd93e8fdcfa1c 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -39,6 +39,8 @@ class MemoryPool;
 
 class ARROW_EXPORT ListArray : public Array {
  public:
+  using TypeClass = ListType;
+
   ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
       const ArrayPtr& values, int32_t null_count = 0,
       std::shared_ptr<Buffer> null_bitmap = nullptr)
@@ -56,13 +58,13 @@ class ARROW_EXPORT ListArray : public Array {
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   const std::shared_ptr<Array>& values() const { return values_; }
-  const std::shared_ptr<Buffer> offset_buffer() const {
+  std::shared_ptr<Buffer> offsets() const {
     return std::static_pointer_cast<Buffer>(offset_buffer_);
   }
 
   const std::shared_ptr<DataType>& value_type() const { return values_->type(); }
 
-  const int32_t* offsets() const { return offsets_; }
+  const int32_t* raw_offsets() const { return offsets_; }
 
   int32_t offset(int i) const { return offsets_[i]; }
 
@@ -76,6 +78,8 @@ class ARROW_EXPORT ListArray : public Array {
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
       const ArrayPtr& arr) const override;
 
+  Status Accept(ArrayVisitor* visitor) const override;
+
  protected:
   std::shared_ptr<Buffer> offset_buffer_;
   const int32_t* offsets_;
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
index e47f6dc74fb7e..bdc8ec00be02c 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/types/primitive-test.cc
@@ -25,6 +25,7 @@
 #include "arrow/builder.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
+#include "arrow/type_traits.h"
 #include "arrow/types/construct.h"
 #include "arrow/types/primitive.h"
 #include "arrow/types/test-common.h"
@@ -41,15 +42,15 @@ namespace arrow {
 
 class Array;
 
-#define PRIMITIVE_TEST(KLASS, ENUM, NAME) \
-  TEST(TypesTest, TestPrimitive_##ENUM) { \
-    KLASS tp;                             \
-                                          \
-    ASSERT_EQ(tp.type, Type::ENUM);       \
-    ASSERT_EQ(tp.name(), string(NAME));   \
-                                          \
-    KLASS tp_copy = tp;                   \
-    ASSERT_EQ(tp_copy.type, Type::ENUM);  \
+#define PRIMITIVE_TEST(KLASS, ENUM, NAME)   \
+  TEST(TypesTest, TestPrimitive_##ENUM) {   \
+    KLASS tp;                               \
+                                            \
+    ASSERT_EQ(tp.type, Type::ENUM);         \
+    ASSERT_EQ(tp.ToString(), string(NAME)); \
+                                            \
+    KLASS tp_copy = tp;                     \
+    ASSERT_EQ(tp_copy.type, Type::ENUM);    \
   }
 
 PRIMITIVE_TEST(Int8Type, INT8, "int8");
@@ -243,7 +244,8 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 }
 
 typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
-    PInt32, PInt64, PFloat, PDouble> Primitives;
+    PInt32, PInt64, PFloat, PDouble>
+    Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
@@ -311,20 +313,6 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
   ASSERT_EQ(memory_before, this->pool_->bytes_allocated());
 }
 
-template <class T, class Builder>
-Status MakeArray(const vector<uint8_t>& valid_bytes, const vector<T>& draws, int size,
-    Builder* builder, ArrayPtr* out) {
-  // Append the first 1000
-  for (int i = 0; i < size; ++i) {
-    if (valid_bytes[i] > 0) {
-      RETURN_NOT_OK(builder->Append(draws[i]));
-    } else {
-      RETURN_NOT_OK(builder->AppendNull());
-    }
-  }
-  return builder->Finish(out);
-}
-
 TYPED_TEST(TestPrimitiveBuilder, Equality) {
   DECL_T();
 
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index d2288bafa71da..14667ee5b6eac 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -19,6 +19,7 @@
 
 #include <memory>
 
+#include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
@@ -48,13 +49,14 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
     const uint8_t* this_data = raw_data_;
     const uint8_t* other_data = other.raw_data_;
 
-    int value_size = type_->value_size();
-    DCHECK_GT(value_size, 0);
+    auto size_meta = dynamic_cast<const FixedWidthMeta*>(type_.get());
+    int value_byte_size = size_meta->bit_width() / 8;
+    DCHECK_GT(value_byte_size, 0);
 
     for (int i = 0; i < length_; ++i) {
-      if (!IsNull(i) && memcmp(this_data, other_data, value_size)) { return false; }
-      this_data += value_size;
-      other_data += value_size;
+      if (!IsNull(i) && memcmp(this_data, other_data, value_byte_size)) { return false; }
+      this_data += value_byte_size;
+      other_data += value_byte_size;
     }
     return true;
   } else {
@@ -70,6 +72,11 @@ bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
   return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
 }
 
+template <typename T>
+Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 template class NumericArray<UInt8Type>;
 template class NumericArray<UInt16Type>;
 template class NumericArray<UInt32Type>;
@@ -79,9 +86,9 @@ template class NumericArray<Int16Type>;
 template class NumericArray<Int32Type>;
 template class NumericArray<Int64Type>;
 template class NumericArray<TimestampType>;
+template class NumericArray<HalfFloatType>;
 template class NumericArray<FloatType>;
 template class NumericArray<DoubleType>;
-template class NumericArray<BooleanType>;
 
 template <typename T>
 Status PrimitiveBuilder<T>::Init(int32_t capacity) {
@@ -145,8 +152,65 @@ Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
-template <>
-Status PrimitiveBuilder<BooleanType>::Append(
+template class PrimitiveBuilder<UInt8Type>;
+template class PrimitiveBuilder<UInt16Type>;
+template class PrimitiveBuilder<UInt32Type>;
+template class PrimitiveBuilder<UInt64Type>;
+template class PrimitiveBuilder<Int8Type>;
+template class PrimitiveBuilder<Int16Type>;
+template class PrimitiveBuilder<Int32Type>;
+template class PrimitiveBuilder<Int64Type>;
+template class PrimitiveBuilder<TimestampType>;
+template class PrimitiveBuilder<HalfFloatType>;
+template class PrimitiveBuilder<FloatType>;
+template class PrimitiveBuilder<DoubleType>;
+
+Status BooleanBuilder::Init(int32_t capacity) {
+  RETURN_NOT_OK(ArrayBuilder::Init(capacity));
+  data_ = std::make_shared<PoolBuffer>(pool_);
+
+  int64_t nbytes = BitUtil::BytesForBits(capacity);
+  RETURN_NOT_OK(data_->Resize(nbytes));
+  // TODO(emkornfield) valgrind complains without this
+  memset(data_->mutable_data(), 0, nbytes);
+
+  raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
+  return Status::OK();
+}
+
+Status BooleanBuilder::Resize(int32_t capacity) {
+  // XXX: Set floor size for now
+  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+
+  if (capacity_ == 0) {
+    RETURN_NOT_OK(Init(capacity));
+  } else {
+    RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
+    const int64_t old_bytes = data_->size();
+    const int64_t new_bytes = BitUtil::BytesForBits(capacity);
+
+    RETURN_NOT_OK(data_->Resize(new_bytes));
+    raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
+    memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
+  }
+  return Status::OK();
+}
+
+Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
+  const int64_t bytes_required = BitUtil::BytesForBits(length_);
+
+  if (bytes_required > 0 && bytes_required < data_->size()) {
+    // Trim buffers
+    RETURN_NOT_OK(data_->Resize(bytes_required));
+  }
+  *out = std::make_shared<BooleanArray>(type_, length_, data_, null_count_, null_bitmap_);
+
+  data_ = null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
+  return Status::OK();
+}
+
+Status BooleanBuilder::Append(
     const uint8_t* values, int32_t length, const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
 
@@ -168,19 +232,6 @@ Status PrimitiveBuilder<BooleanType>::Append(
   return Status::OK();
 }
 
-template class PrimitiveBuilder<UInt8Type>;
-template class PrimitiveBuilder<UInt16Type>;
-template class PrimitiveBuilder<UInt32Type>;
-template class PrimitiveBuilder<UInt64Type>;
-template class PrimitiveBuilder<Int8Type>;
-template class PrimitiveBuilder<Int16Type>;
-template class PrimitiveBuilder<Int32Type>;
-template class PrimitiveBuilder<Int64Type>;
-template class PrimitiveBuilder<TimestampType>;
-template class PrimitiveBuilder<FloatType>;
-template class PrimitiveBuilder<DoubleType>;
-template class PrimitiveBuilder<BooleanType>;
-
 BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
     int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
     : PrimitiveArray(
@@ -235,4 +286,8 @@ bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
   return true;
 }
 
+Status BooleanArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index c71df584ffe3f..a5a3704e2d2d3 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -26,6 +26,7 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/types/datetime.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
@@ -54,9 +55,10 @@ class ARROW_EXPORT PrimitiveArray : public Array {
   const uint8_t* raw_data_;
 };
 
-template <class TypeClass>
+template <class TYPE>
 class ARROW_EXPORT NumericArray : public PrimitiveArray {
  public:
+  using TypeClass = TYPE;
   using value_type = typename TypeClass::c_type;
   NumericArray(int32_t length, const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
@@ -88,29 +90,15 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
     return reinterpret_cast<const value_type*>(raw_data_);
   }
 
+  Status Accept(ArrayVisitor* visitor) const override;
+
   value_type Value(int i) const { return raw_data()[i]; }
 };
 
-#define NUMERIC_ARRAY_DECL(NAME, TypeClass) \
-  using NAME = NumericArray<TypeClass>;     \
-  extern template class ARROW_EXPORT NumericArray<TypeClass>;
-
-NUMERIC_ARRAY_DECL(UInt8Array, UInt8Type);
-NUMERIC_ARRAY_DECL(Int8Array, Int8Type);
-NUMERIC_ARRAY_DECL(UInt16Array, UInt16Type);
-NUMERIC_ARRAY_DECL(Int16Array, Int16Type);
-NUMERIC_ARRAY_DECL(UInt32Array, UInt32Type);
-NUMERIC_ARRAY_DECL(Int32Array, Int32Type);
-NUMERIC_ARRAY_DECL(UInt64Array, UInt64Type);
-NUMERIC_ARRAY_DECL(Int64Array, Int64Type);
-NUMERIC_ARRAY_DECL(TimestampArray, TimestampType);
-NUMERIC_ARRAY_DECL(FloatArray, FloatType);
-NUMERIC_ARRAY_DECL(DoubleArray, DoubleType);
-
 template <typename Type>
 class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
  public:
-  typedef typename Type::c_type value_type;
+  using value_type = typename Type::c_type;
 
   explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type)
       : ArrayBuilder(pool, type), data_(nullptr) {}
@@ -183,101 +171,27 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
   using PrimitiveBuilder<T>::raw_data_;
 };
 
-template <>
-struct TypeTraits<UInt8Type> {
-  typedef UInt8Array ArrayType;
-
-  static inline int bytes_required(int elements) { return elements; }
-};
-
-template <>
-struct TypeTraits<Int8Type> {
-  typedef Int8Array ArrayType;
-
-  static inline int bytes_required(int elements) { return elements; }
-};
-
-template <>
-struct TypeTraits<UInt16Type> {
-  typedef UInt16Array ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
-};
-
-template <>
-struct TypeTraits<Int16Type> {
-  typedef Int16Array ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(int16_t); }
-};
-
-template <>
-struct TypeTraits<UInt32Type> {
-  typedef UInt32Array ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(uint32_t); }
-};
-
-template <>
-struct TypeTraits<Int32Type> {
-  typedef Int32Array ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(int32_t); }
-};
-
-template <>
-struct TypeTraits<UInt64Type> {
-  typedef UInt64Array ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(uint64_t); }
-};
-
-template <>
-struct TypeTraits<Int64Type> {
-  typedef Int64Array ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
-};
-
-template <>
-struct TypeTraits<TimestampType> {
-  typedef TimestampArray ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
-};
-template <>
-
-struct TypeTraits<FloatType> {
-  typedef FloatArray ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(float); }
-};
-
-template <>
-struct TypeTraits<DoubleType> {
-  typedef DoubleArray ArrayType;
-
-  static inline int bytes_required(int elements) { return elements * sizeof(double); }
-};
-
 // Builders
 
-typedef NumericBuilder<UInt8Type> UInt8Builder;
-typedef NumericBuilder<UInt16Type> UInt16Builder;
-typedef NumericBuilder<UInt32Type> UInt32Builder;
-typedef NumericBuilder<UInt64Type> UInt64Builder;
+using UInt8Builder = NumericBuilder<UInt8Type>;
+using UInt16Builder = NumericBuilder<UInt16Type>;
+using UInt32Builder = NumericBuilder<UInt32Type>;
+using UInt64Builder = NumericBuilder<UInt64Type>;
 
-typedef NumericBuilder<Int8Type> Int8Builder;
-typedef NumericBuilder<Int16Type> Int16Builder;
-typedef NumericBuilder<Int32Type> Int32Builder;
-typedef NumericBuilder<Int64Type> Int64Builder;
-typedef NumericBuilder<TimestampType> TimestampBuilder;
+using Int8Builder = NumericBuilder<Int8Type>;
+using Int16Builder = NumericBuilder<Int16Type>;
+using Int32Builder = NumericBuilder<Int32Type>;
+using Int64Builder = NumericBuilder<Int64Type>;
+using TimestampBuilder = NumericBuilder<TimestampType>;
 
-typedef NumericBuilder<FloatType> FloatBuilder;
-typedef NumericBuilder<DoubleType> DoubleBuilder;
+using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
+using FloatBuilder = NumericBuilder<FloatType>;
+using DoubleBuilder = NumericBuilder<DoubleType>;
 
 class ARROW_EXPORT BooleanArray : public PrimitiveArray {
  public:
+  using TypeClass = BooleanType;
+
   BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
   BooleanArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
@@ -288,28 +202,36 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
       const ArrayPtr& arr) const override;
 
+  Status Accept(ArrayVisitor* visitor) const override;
+
   const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
 
   bool Value(int i) const { return BitUtil::GetBit(raw_data(), i); }
 };
 
-template <>
-struct TypeTraits<BooleanType> {
-  typedef BooleanArray ArrayType;
-
-  static inline int bytes_required(int elements) {
-    return BitUtil::BytesForBits(elements);
-  }
-};
-
-class ARROW_EXPORT BooleanBuilder : public PrimitiveBuilder<BooleanType> {
+class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
  public:
   explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type)
-      : PrimitiveBuilder<BooleanType>(pool, type) {}
+      : ArrayBuilder(pool, type), data_(nullptr) {}
 
   virtual ~BooleanBuilder() {}
 
-  using PrimitiveBuilder<BooleanType>::Append;
+  using ArrayBuilder::Advance;
+
+  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+  Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+    RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(valid_bytes, length);
+    return Status::OK();
+  }
+
+  Status AppendNull() {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(false);
+    return Status::OK();
+  }
+
+  std::shared_ptr<Buffer> data() const { return data_; }
 
   // Scalar append
   Status Append(bool val) {
@@ -324,9 +246,39 @@ class ARROW_EXPORT BooleanBuilder : public PrimitiveBuilder<BooleanType> {
     return Status::OK();
   }
 
-  Status Append(uint8_t val) { return Append(static_cast<bool>(val)); }
+  // Vector append
+  //
+  // If passed, valid_bytes is of equal length to values, and any zero byte
+  // will be considered as a null for that slot
+  Status Append(
+      const uint8_t* values, int32_t length, const uint8_t* valid_bytes = nullptr);
+
+  Status Finish(std::shared_ptr<Array>* out) override;
+  Status Init(int32_t capacity) override;
+
+  // Increase the capacity of the builder to accommodate at least the indicated
+  // number of elements
+  Status Resize(int32_t capacity) override;
+
+ protected:
+  std::shared_ptr<PoolBuffer> data_;
+  uint8_t* raw_data_;
 };
 
+// Only instantiate these templates once
+extern template class ARROW_EXPORT NumericArray<Int8Type>;
+extern template class ARROW_EXPORT NumericArray<UInt8Type>;
+extern template class ARROW_EXPORT NumericArray<Int16Type>;
+extern template class ARROW_EXPORT NumericArray<UInt16Type>;
+extern template class ARROW_EXPORT NumericArray<Int32Type>;
+extern template class ARROW_EXPORT NumericArray<UInt32Type>;
+extern template class ARROW_EXPORT NumericArray<Int64Type>;
+extern template class ARROW_EXPORT NumericArray<UInt64Type>;
+extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
+extern template class ARROW_EXPORT NumericArray<FloatType>;
+extern template class ARROW_EXPORT NumericArray<DoubleType>;
+extern template class ARROW_EXPORT NumericArray<TimestampType>;
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPES_PRIMITIVE_H
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
index af87a14a8b32e..3c4b12b7bc772 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/types/string-test.cc
@@ -47,7 +47,7 @@ TEST(TypesTest, BinaryType) {
 TEST(TypesTest, TestStringType) {
   StringType str;
   ASSERT_EQ(str.type, Type::STRING);
-  ASSERT_EQ(str.name(), std::string("string"));
+  ASSERT_EQ(str.ToString(), std::string("string"));
 }
 
 // ----------------------------------------------------------------------
@@ -66,8 +66,8 @@ class TestStringContainer : public ::testing::Test {
 
   void MakeArray() {
     length_ = offsets_.size() - 1;
-    value_buf_ = test::to_buffer(chars_);
-    offsets_buf_ = test::to_buffer(offsets_);
+    value_buf_ = test::GetBufferFromVector(chars_);
+    offsets_buf_ = test::GetBufferFromVector(offsets_);
     null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
     null_count_ = test::null_count(valid_bytes_);
 
@@ -131,7 +131,7 @@ TEST_F(TestStringContainer, TestGetString) {
 
 TEST_F(TestStringContainer, TestEmptyStringComparison) {
   offsets_ = {0, 0, 0, 0, 0, 0};
-  offsets_buf_ = test::to_buffer(offsets_);
+  offsets_buf_ = test::GetBufferFromVector(offsets_);
   length_ = offsets_.size() - 1;
 
   auto strings_a = std::make_shared<StringArray>(
@@ -227,8 +227,8 @@ class TestBinaryContainer : public ::testing::Test {
 
   void MakeArray() {
     length_ = offsets_.size() - 1;
-    value_buf_ = test::to_buffer(chars_);
-    offsets_buf_ = test::to_buffer(offsets_);
+    value_buf_ = test::GetBufferFromVector(chars_);
+    offsets_buf_ = test::GetBufferFromVector(offsets_);
 
     null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
     null_count_ = test::null_count(valid_bytes_);
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
index f6d26df3167c9..db963dfa0de5f 100644
--- a/cpp/src/arrow/types/string.cc
+++ b/cpp/src/arrow/types/string.cc
@@ -94,6 +94,10 @@ bool BinaryArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_
   return true;
 }
 
+Status BinaryArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
     const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap)
@@ -104,6 +108,10 @@ Status StringArray::Validate() const {
   return BinaryArray::Validate();
 }
 
+Status StringArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 // This used to be a static member variable of BinaryBuilder, but it can cause
 // valgrind to report a (spurious?) memory leak when needed in other shared
 // libraries. The problem came up while adding explicit visibility to libarrow
@@ -122,8 +130,8 @@ Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
   const auto list = std::dynamic_pointer_cast<ListArray>(result);
   auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
 
-  *out = std::make_shared<BinaryArray>(list->length(), list->offset_buffer(),
-      values->data(), list->null_count(), list->null_bitmap());
+  *out = std::make_shared<BinaryArray>(list->length(), list->offsets(), values->data(),
+      list->null_count(), list->null_bitmap());
   return Status::OK();
 }
 
@@ -134,8 +142,8 @@ Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
   const auto list = std::dynamic_pointer_cast<ListArray>(result);
   auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
 
-  *out = std::make_shared<StringArray>(list->length(), list->offset_buffer(),
-      values->data(), list->null_count(), list->null_bitmap());
+  *out = std::make_shared<StringArray>(list->length(), list->offsets(), values->data(),
+      list->null_count(), list->null_bitmap());
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
index aaba49c60237d..c8752439f168c 100644
--- a/cpp/src/arrow/types/string.h
+++ b/cpp/src/arrow/types/string.h
@@ -37,6 +37,8 @@ class MemoryPool;
 
 class ARROW_EXPORT BinaryArray : public Array {
  public:
+  using TypeClass = BinaryType;
+
   BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
       const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
@@ -60,6 +62,8 @@ class ARROW_EXPORT BinaryArray : public Array {
   std::shared_ptr<Buffer> data() const { return data_buffer_; }
   std::shared_ptr<Buffer> offsets() const { return offset_buffer_; }
 
+  const int32_t* raw_offsets() const { return offsets_; }
+
   int32_t offset(int i) const { return offsets_[i]; }
 
   // Neither of these functions will perform boundschecking
@@ -73,6 +77,8 @@ class ARROW_EXPORT BinaryArray : public Array {
 
   Status Validate() const override;
 
+  Status Accept(ArrayVisitor* visitor) const override;
+
  private:
   std::shared_ptr<Buffer> offset_buffer_;
   const int32_t* offsets_;
@@ -83,6 +89,8 @@ class ARROW_EXPORT BinaryArray : public Array {
 
 class ARROW_EXPORT StringArray : public BinaryArray {
  public:
+  using TypeClass = StringType;
+
   StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
       const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
@@ -96,6 +104,8 @@ class ARROW_EXPORT StringArray : public BinaryArray {
   }
 
   Status Validate() const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
 };
 
 // BinaryBuilder : public ListBuilder
@@ -109,6 +119,12 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
     return byte_builder_->Append(value, length);
   }
 
+  Status Append(const char* value, int32_t length) {
+    return Append(reinterpret_cast<const uint8_t*>(value), length);
+  }
+
+  Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
+
   Status Finish(std::shared_ptr<Array>* out) override;
 
  protected:
@@ -121,13 +137,9 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
   explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
       : BinaryBuilder(pool, type) {}
 
-  Status Finish(std::shared_ptr<Array>* out) override;
-
-  Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
+  using BinaryBuilder::Append;
 
-  Status Append(const char* value, int32_t length) {
-    return BinaryBuilder::Append(reinterpret_cast<const uint8_t*>(value), length);
-  }
+  Status Finish(std::shared_ptr<Array>* out) override;
 
   Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
 };
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
index 8e82c389a9423..197d7d4ad1f5e 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/types/struct-test.cc
@@ -80,7 +80,7 @@ void ValidateBasicStructArray(const StructArray* result,
   ASSERT_EQ(4, list_char_arr->length());
   ASSERT_EQ(10, list_char_arr->values()->length());
   for (size_t i = 0; i < list_offsets.size(); ++i) {
-    ASSERT_EQ(list_offsets[i], list_char_arr->offsets()[i]);
+    ASSERT_EQ(list_offsets[i], list_char_arr->raw_offsets()[i]);
   }
   for (size_t i = 0; i < list_values.size(); ++i) {
     ASSERT_EQ(list_values[i], char_arr->Value(i));
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
index 369c29d15ef96..0e0db23544bf7 100644
--- a/cpp/src/arrow/types/struct.cc
+++ b/cpp/src/arrow/types/struct.cc
@@ -87,6 +87,10 @@ Status StructArray::Validate() const {
   return Status::OK();
 }
 
+Status StructArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
   std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
   for (size_t i = 0; i < field_builders_.size(); ++i) {
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index 65b8daf214a69..035af05132572 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -31,6 +31,8 @@ namespace arrow {
 
 class ARROW_EXPORT StructArray : public Array {
  public:
+  using TypeClass = StructType;
+
   StructArray(const TypePtr& type, int32_t length, std::vector<ArrayPtr>& field_arrays,
       int32_t null_count = 0, std::shared_ptr<Buffer> null_bitmap = nullptr)
       : Array(type, length, null_count, null_bitmap) {
@@ -55,6 +57,8 @@ class ARROW_EXPORT StructArray : public Array {
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
       const std::shared_ptr<Array>& arr) const override;
 
+  Status Accept(ArrayVisitor* visitor) const override;
+
  protected:
   // The child arrays corresponding to each field of the struct data type.
   std::vector<ArrayPtr> field_arrays_;
diff --git a/cpp/src/arrow/types/test-common.h b/cpp/src/arrow/types/test-common.h
index 1957636b141fd..6e6ab85ad4eb7 100644
--- a/cpp/src/arrow/types/test-common.h
+++ b/cpp/src/arrow/types/test-common.h
@@ -24,6 +24,8 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/array.h"
+#include "arrow/builder.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/util/memory-pool.h"
@@ -49,6 +51,20 @@ class TestBuilder : public ::testing::Test {
   unique_ptr<ArrayBuilder> builder_nn_;
 };
 
+template <class T, class Builder>
+Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
+    int size, Builder* builder, ArrayPtr* out) {
+  // Append the first 1000
+  for (int i = 0; i < size; ++i) {
+    if (valid_bytes[i] > 0) {
+      RETURN_NOT_OK(builder->Append(values[i]));
+    } else {
+      RETURN_NOT_OK(builder->AppendNull());
+    }
+  }
+  return builder->Finish(out);
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPES_TEST_COMMON_H
diff --git a/cpp/src/arrow/types/union.cc b/cpp/src/arrow/types/union.cc
index c891b4a5357ef..cc2934b2e4adb 100644
--- a/cpp/src/arrow/types/union.cc
+++ b/cpp/src/arrow/types/union.cc
@@ -24,25 +24,4 @@
 
 #include "arrow/type.h"
 
-namespace arrow {
-
-static inline std::string format_union(const std::vector<TypePtr>& child_types) {
-  std::stringstream s;
-  s << "union<";
-  for (size_t i = 0; i < child_types.size(); ++i) {
-    if (i) { s << ", "; }
-    s << child_types[i]->ToString();
-  }
-  s << ">";
-  return s.str();
-}
-
-std::string DenseUnionType::ToString() const {
-  return format_union(child_types_);
-}
-
-std::string SparseUnionType::ToString() const {
-  return format_union(child_types_);
-}
-
-}  // namespace arrow
+namespace arrow {}  // namespace arrow
diff --git a/cpp/src/arrow/types/union.h b/cpp/src/arrow/types/union.h
index d2ee9bde04d0d..44f39cc69942b 100644
--- a/cpp/src/arrow/types/union.h
+++ b/cpp/src/arrow/types/union.h
@@ -24,32 +24,11 @@
 
 #include "arrow/array.h"
 #include "arrow/type.h"
-#include "arrow/types/collection.h"
 
 namespace arrow {
 
 class Buffer;
 
-struct DenseUnionType : public CollectionType<Type::DENSE_UNION> {
-  typedef CollectionType<Type::DENSE_UNION> Base;
-
-  explicit DenseUnionType(const std::vector<TypePtr>& child_types) : Base() {
-    child_types_ = child_types;
-  }
-
-  virtual std::string ToString() const;
-};
-
-struct SparseUnionType : public CollectionType<Type::SPARSE_UNION> {
-  typedef CollectionType<Type::SPARSE_UNION> Base;
-
-  explicit SparseUnionType(const std::vector<TypePtr>& child_types) : Base() {
-    child_types_ = child_types;
-  }
-
-  virtual std::string ToString() const;
-};
-
 class UnionArray : public Array {
  protected:
   // The data are types encoded as int16
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 06ee8411e283c..b22f07dd6345f 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -118,9 +118,9 @@ class CerrLog {
 class FatalLog : public CerrLog {
  public:
   explicit FatalLog(int /* severity */)  // NOLINT
-      : CerrLog(ARROW_FATAL) {}          // NOLINT
+      : CerrLog(ARROW_FATAL){}           // NOLINT
 
-  [[noreturn]] ~FatalLog() {
+            [[noreturn]] ~FatalLog() {
     if (has_logged_) { std::cerr << std::endl; }
     std::exit(1);
   }
diff --git a/format/Metadata.md b/format/Metadata.md
index 653a4c73e830e..a4878f347073f 100644
--- a/format/Metadata.md
+++ b/format/Metadata.md
@@ -98,6 +98,11 @@ Union:
   "typeIds" : [ /* integer */ ]
 }
 ```
+
+The `typeIds` field in the Union are the codes used to denote each type, which
+may be different from the index of the child array. This is so that the union
+type ids do not have to be enumerated from 0.
+
 Int:
 ```
 {

From 997f502ce10d6ae3e0b7b1df55e167fda040fc18 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 21 Nov 2016 13:25:55 -0500
Subject: [PATCH 0202/1644] ARROW-382: Extend Python API documentation

 * Fix numpydoc compilation
 * Add simple examples to the API
 * Move away from deprecated Cython-property declaration
 * Add basic descriptions with return types to functions

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #208 from xhochy/ARROW-382 and squashes the following commits:

31e0cb3 [Uwe L. Korn] ARROW-382: Extend Python API documentation
---
 python/doc/conf.py       |   5 +-
 python/pyarrow/array.pyx |  53 ++++++-
 python/pyarrow/compat.py |   2 +
 python/pyarrow/table.pyx | 333 ++++++++++++++++++++++++++++++---------
 4 files changed, 318 insertions(+), 75 deletions(-)

diff --git a/python/doc/conf.py b/python/doc/conf.py
index 99ac3512ec9d4..4c324a8086c39 100644
--- a/python/doc/conf.py
+++ b/python/doc/conf.py
@@ -59,9 +59,12 @@
     'sphinx.ext.doctest',
     'sphinx.ext.mathjax',
     'sphinx.ext.viewcode',
-    'numpydoc'
+    'sphinx.ext.napoleon'
 ]
 
+# numpydoc configuration
+napoleon_use_rtype = False
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index fbe4e3879062c..6c862751fc218 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -54,6 +54,41 @@ cdef class Array:
 
     @staticmethod
     def from_pandas(obj, mask=None):
+        """
+        Create an array from a pandas.Series
+
+        Parameters
+        ----------
+        obj : pandas.Series or numpy.ndarray
+            vector holding the data
+        mask : numpy.ndarray, optional
+            boolean mask if the object is valid or null
+
+        Returns
+        -------
+        pyarrow.Array
+
+        Examples
+        --------
+
+        >>> import pandas as pd
+        >>> import pyarrow as pa
+        >>> pa.Array.from_pandas(pd.Series([1, 2]))
+        <pyarrow.array.Int64Array object at 0x7f674e4c0e10>
+        [
+          1,
+          2
+        ]
+
+
+        >>> import numpy as np
+        >>> pa.Array.from_pandas(pd.Series([1, 2]), np.array([0, 1], dtype=bool))
+        <pyarrow.array.Int64Array object at 0x7f9019e11208>
+        [
+          1,
+          NA
+        ]
+        """
         return from_pandas_series(obj, mask)
 
     property null_count:
@@ -228,6 +263,14 @@ cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
 def from_pylist(object list_obj, DataType type=None):
     """
     Convert Python list to Arrow array
+
+    Parameters
+    ----------
+    list_obj : array_like
+
+    Returns
+    -------
+    pyarrow.array.Array
     """
     cdef:
         shared_ptr[CArray] sp_array
@@ -246,15 +289,19 @@ def from_pandas_series(object series, object mask=None, timestamps_to_ms=False):
 
     Parameters
     ----------
-    series: pandas.Series or numpy.ndarray
+    series : pandas.Series or numpy.ndarray
 
-    mask: pandas.Series or numpy.ndarray
+    mask : pandas.Series or numpy.ndarray, optional
         array to mask null entries in the series
 
-    timestamps_to_ms: bool
+    timestamps_to_ms : bool, optional
         Convert datetime columns to ms resolution. This is needed for
         compability with other functionality like Parquet I/O which
         only supports milliseconds.
+
+    Returns
+    -------
+    pyarrow.array.Array
     """
     cdef:
         shared_ptr[CArray] out
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 08f0f23796797..2dfdb5041d13e 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -90,3 +90,5 @@ def frombytes(o):
 
 
 integer_types = six.integer_types + (np.integer,)
+
+__all__ = []
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 5459f26b80aa4..a6715b141ce41 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -36,9 +36,13 @@ from pyarrow.compat import frombytes, tobytes
 cimport cpython
 
 cdef class ChunkedArray:
-    '''
+    """
+    Array backed via one or more memory chunks.
+
+    Warning
+    -------
     Do not call this class's constructor directly.
-    '''
+    """
 
     def __cinit__(self):
         self.chunked_array = NULL
@@ -59,19 +63,42 @@ cdef class ChunkedArray:
     def __len__(self):
         return self.length()
 
-    property null_count:
+    @property
+    def null_count(self):
+        """
+        Number of null entires
 
-        def __get__(self):
-            self._check_nullptr()
-            return self.chunked_array.null_count()
+        Returns
+        -------
+        int
+        """
+        self._check_nullptr()
+        return self.chunked_array.null_count()
 
-    property num_chunks:
+    @property
+    def num_chunks(self):
+        """
+        Number of underlying chunks
 
-        def __get__(self):
-            self._check_nullptr()
-            return self.chunked_array.num_chunks()
+        Returns
+        -------
+        int
+        """
+        self._check_nullptr()
+        return self.chunked_array.num_chunks()
 
     def chunk(self, i):
+        """
+        Select a chunk by its index
+
+        Parameters
+        ----------
+        i : int
+
+        Returns
+        -------
+        pyarrow.array.Array
+        """
         self._check_nullptr()
         return box_arrow_array(self.chunked_array.chunk(i))
 
@@ -82,9 +109,13 @@ cdef class ChunkedArray:
 
 
 cdef class Column:
-    '''
+    """
+    Named vector of elements of equal type.
+
+    Warning
+    -------
     Do not call this class's constructor directly.
-    '''
+    """
 
     def __cinit__(self):
         self.column = NULL
@@ -95,7 +126,11 @@ cdef class Column:
 
     def to_pandas(self):
         """
-        Convert the arrow::Column to a pandas Series
+        Convert the arrow::Column to a pandas.Series
+
+        Returns
+        -------
+        pandas.Series
         """
         cdef:
             PyObject* arr
@@ -120,34 +155,64 @@ cdef class Column:
         self._check_nullptr()
         return self.column.length()
 
-    property shape:
+    @property
+    def shape(self):
+        """
+        Dimensions of this columns
 
-        def __get__(self):
-            self._check_nullptr()
-            return (self.length(),)
+        Returns
+        -------
+        (int,)
+        """
+        self._check_nullptr()
+        return (self.length(),)
 
-    property null_count:
+    @property
+    def null_count(self):
+        """
+        Number of null entires
 
-        def __get__(self):
-            self._check_nullptr()
-            return self.column.null_count()
+        Returns
+        -------
+        int
+        """
+        self._check_nullptr()
+        return self.column.null_count()
 
-    property name:
+    @property
+    def name(self):
+        """
+        Label of the column
 
-        def __get__(self):
-            return frombytes(self.column.name())
+        Returns
+        -------
+        str
+        """
+        return frombytes(self.column.name())
 
-    property type:
+    @property
+    def type(self):
+        """
+        Type information for this column
 
-        def __get__(self):
-            return box_data_type(self.column.type())
+        Returns
+        -------
+        pyarrow.schema.DataType
+        """
+        return box_data_type(self.column.type())
 
-    property data:
+    @property
+    def data(self):
+        """
+        The underlying data
 
-        def __get__(self):
-            cdef ChunkedArray chunked_array = ChunkedArray()
-            chunked_array.init(self.column.data())
-            return chunked_array
+        Returns
+        -------
+        pyarrow.table.ChunkedArray
+        """
+        cdef ChunkedArray chunked_array = ChunkedArray()
+        chunked_array.init(self.column.data())
+        return chunked_array
 
 
 cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
@@ -186,6 +251,13 @@ cdef _dataframe_to_arrays(df, name, timestamps_to_ms):
 
 
 cdef class RecordBatch:
+    """
+    Batch of rows of columns of equal length
+
+    Warning
+    -------
+    Do not call this class's constructor directly, use one of the ``from_*`` methods instead.
+    """
 
     def __cinit__(self):
         self.batch = NULL
@@ -203,28 +275,48 @@ cdef class RecordBatch:
         self._check_nullptr()
         return self.batch.num_rows()
 
-    property num_columns:
+    @property
+    def num_columns(self):
+        """
+        Number of columns
 
-        def __get__(self):
-            self._check_nullptr()
-            return self.batch.num_columns()
+        Returns
+        -------
+        int
+        """
+        self._check_nullptr()
+        return self.batch.num_columns()
 
-    property num_rows:
+    @property
+    def num_rows(self):
+        """
+        Number of rows
 
-        def __get__(self):
-            return len(self)
+        Due to the definition of a RecordBatch, all columns have the same number of rows.
 
-    property schema:
+        Returns
+        -------
+        int
+        """
+        return len(self)
 
-        def __get__(self):
-            cdef Schema schema
-            self._check_nullptr()
-            if self._schema is None:
-                schema = Schema()
-                schema.init_schema(self.batch.schema())
-                self._schema = schema
+    @property
+    def schema(self):
+        """
+        Schema of the RecordBatch and its columns
 
-            return self._schema
+        Returns
+        -------
+        pyarrow.schema.Schema
+        """
+        cdef Schema schema
+        self._check_nullptr()
+        if self._schema is None:
+            schema = Schema()
+            schema.init_schema(self.batch.schema())
+            self._schema = schema
+
+        return self._schema
 
     def __getitem__(self, i):
         cdef Array arr = Array()
@@ -240,6 +332,10 @@ cdef class RecordBatch:
     def to_pandas(self):
         """
         Convert the arrow::RecordBatch to a pandas DataFrame
+
+        Returns
+        -------
+        pandas.DataFrame
         """
         cdef:
             PyObject* np_arr
@@ -263,12 +359,34 @@ cdef class RecordBatch:
     def from_pandas(cls, df):
         """
         Convert pandas.DataFrame to an Arrow RecordBatch
+
+        Parameters
+        ----------
+        df: pandas.DataFrame
+
+        Returns
+        -------
+        pyarrow.table.RecordBatch
         """
         names, arrays = _dataframe_to_arrays(df, None, False)
         return cls.from_arrays(names, arrays)
 
     @staticmethod
     def from_arrays(names, arrays):
+        """
+        Construct a RecordBatch from multiple pyarrow.Arrays
+
+        Parameters
+        ----------
+        names: list of str
+            Labels for the columns
+        arrays: list of pyarrow.Array
+            column-wise data vectors
+
+        Returns
+        -------
+        pyarrow.table.RecordBatch
+        """
         cdef:
             Array arr
             RecordBatch result
@@ -297,11 +415,13 @@ cdef class RecordBatch:
 
 
 cdef class Table:
-    '''
+    """
     A collection of top-level named, equal length Arrow arrays.
 
-    Do not call this class's constructor directly.
-    '''
+    Warning
+    -------
+    Do not call this class's constructor directly, use one of the ``from_*`` methods instead.
+    """
 
     def __cinit__(self):
         self.table = NULL
@@ -330,6 +450,22 @@ cdef class Table:
             Convert datetime columns to ms resolution. This is needed for
             compability with other functionality like Parquet I/O which
             only supports milliseconds.
+
+        Returns
+        -------
+        pyarrow.table.Table
+
+        Examples
+        --------
+
+        >>> import pandas as pd
+        >>> import pyarrow as pa
+        >>> df = pd.DataFrame({
+            ...     'int': [1, 2],
+            ...     'str': ['a', 'b']
+            ... })
+        >>> pa.table.from_pandas_dataframe(df)
+        <pyarrow.table.Table object at 0x7f05d1fb1b40>
         """
         names, arrays = _dataframe_to_arrays(df, name=name,
                                              timestamps_to_ms=timestamps_to_ms)
@@ -347,8 +483,13 @@ cdef class Table:
             Names for the table columns
         arrays: list of pyarrow.array.Array
             Equal-length arrays that should form the table.
-        name: str
-            (optional) name for the Table
+        name: str, optional
+            name for the Table
+
+        Returns
+        -------
+        pyarrow.table.Table
+
         """
         cdef:
             Array arr
@@ -382,6 +523,10 @@ cdef class Table:
     def to_pandas(self):
         """
         Convert the arrow::Table to a pandas DataFrame
+
+        Returns
+        -------
+        pandas.DataFrame
         """
         cdef:
             PyObject* arr
@@ -402,18 +547,41 @@ cdef class Table:
 
         return pd.DataFrame(dict(zip(names, data)), columns=names)
 
-    property name:
+    @property
+    def name(self):
+        """
+        Label of the table
 
-        def __get__(self):
-            self._check_nullptr()
-            return frombytes(self.table.name())
+        Returns
+        -------
+        str
+        """
+        self._check_nullptr()
+        return frombytes(self.table.name())
 
-    property schema:
+    @property
+    def schema(self):
+        """
+        Schema of the table and its columns
 
-        def __get__(self):
-            raise box_schema(self.table.schema())
+        Returns
+        -------
+        pyarrow.schema.Schema
+        """
+        return box_schema(self.table.schema())
 
     def column(self, index):
+        """
+        Select a column by its numeric index.
+
+        Parameters
+        ----------
+        index: int
+
+        Returns
+        -------
+        pyarrow.table.Column
+        """
         self._check_nullptr()
         cdef Column column = Column()
         column.init(self.table.column(index))
@@ -423,28 +591,51 @@ cdef class Table:
         return self.column(i)
 
     def itercolumns(self):
+        """
+        Iterator over all columns in their numerical order
+        """
         for i in range(self.num_columns):
             yield self.column(i)
 
-    property num_columns:
+    @property
+    def num_columns(self):
+        """
+        Number of columns in this table
 
-        def __get__(self):
-            self._check_nullptr()
-            return self.table.num_columns()
+        Returns
+        -------
+        int
+        """
+        self._check_nullptr()
+        return self.table.num_columns()
 
-    property num_rows:
+    @property
+    def num_rows(self):
+        """
+        Number of rows in this table.
 
-        def __get__(self):
-            self._check_nullptr()
-            return self.table.num_rows()
+        Due to the definition of a table, all columns have the same number of rows.
+
+        Returns
+        -------
+        int
+        """
+        self._check_nullptr()
+        return self.table.num_rows()
 
     def __len__(self):
         return self.num_rows
 
-    property shape:
+    @property
+    def shape(self):
+        """
+        Dimensions of the table: (#rows, #columns)
 
-        def __get__(self):
-            return (self.num_rows, self.num_columns)
+        Returns
+        -------
+        (int, int)
+        """
+        return (self.num_rows, self.num_columns)
 
 
 

From f082b17323354dc2af31f39c15c58b995ba08360 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 21 Nov 2016 17:48:09 -0500
Subject: [PATCH 0203/1644] ARROW-383: [C++] Integration testing CLI tool

Modeled after Java version in ARROW-367

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #209 from wesm/ARROW-383 and squashes the following commits:

7b29b24 [Wes McKinney] Use git master gflags to avoid valgrind error
f563d3a [Wes McKinney] Build integration test as a normal unit test executable, opt-in to integration testing
bdf1f7a [Wes McKinney] Call the RunCommand method instead
dbc6aab [Wes McKinney] Add unit tests for the integration validator
ca1eade [Wes McKinney] Clean up includes
1752249 [Wes McKinney] Draft integration testing CLI tool, untested
b773d0d [Wes McKinney] Add gflags external project and json-integration-test executable stub
---
 cpp/CMakeLists.txt                         |  44 +++
 cpp/cmake_modules/FindGFlags.cmake         |  60 ++++
 cpp/src/arrow/io/file.cc                   |   7 +-
 cpp/src/arrow/ipc/CMakeLists.txt           |  27 ++
 cpp/src/arrow/ipc/adapter.h                |   2 +-
 cpp/src/arrow/ipc/json-integration-test.cc | 381 +++++++++++++++++++++
 6 files changed, 517 insertions(+), 4 deletions(-)
 create mode 100644 cpp/cmake_modules/FindGFlags.cmake
 create mode 100644 cpp/src/arrow/ipc/json-integration-test.cc

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0bff7528578d1..839ea17dee02e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -26,6 +26,7 @@ include(ExternalProject)
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 
+set(GFLAGS_VERSION "2.1.2")
 set(GTEST_VERSION "1.7.0")
 set(GBENCHMARK_VERSION "1.0.0")
 set(FLATBUFFERS_VERSION "1.3.0")
@@ -506,6 +507,49 @@ if(ARROW_BUILD_TESTS)
   if(GTEST_VENDORED)
     add_dependencies(gtest googletest_ep)
   endif()
+
+  # gflags (formerly Googleflags) command line parsing
+  if("$ENV{GFLAGS_HOME}" STREQUAL "")
+    if(APPLE)
+      set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++")
+    else()
+      set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC")
+    endif()
+
+    set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
+    ExternalProject_Add(gflags_ep
+      GIT_REPOSITORY https://github.com/gflags/gflags.git
+      GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+      # URL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz"
+      BUILD_IN_SOURCE 1
+      CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                 -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
+                 -DBUILD_SHARED_LIBS=OFF
+                 -DBUILD_STATIC_LIBS=ON
+                 -DBUILD_PACKAGING=OFF
+                 -DBUILD_TESTING=OFF
+                 -BUILD_CONFIG_TESTS=OFF
+                 -DINSTALL_HEADERS=ON
+                 -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
+
+    set(GFLAGS_HOME "${GFLAGS_PREFIX}")
+    set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
+    set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
+    set(GFLAGS_VENDORED 1)
+  else()
+    set(GFLAGS_VENDORED 0)
+    find_package(GFlags REQUIRED)
+  endif()
+
+  message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}")
+  message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}")
+  include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(gflags
+    STATIC_LIB ${GFLAGS_STATIC_LIB})
+
+  if(GFLAGS_VENDORED)
+    add_dependencies(gflags gflags_ep)
+  endif()
 endif()
 
 if(ARROW_BUILD_BENCHMARKS)
diff --git a/cpp/cmake_modules/FindGFlags.cmake b/cpp/cmake_modules/FindGFlags.cmake
new file mode 100644
index 0000000000000..eaea83530547b
--- /dev/null
+++ b/cpp/cmake_modules/FindGFlags.cmake
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find GFLAGS (gflags.h, libgflags.a, libgflags.so, and libgflags.so.0)
+# This module defines
+#  GFLAGS_INCLUDE_DIR, directory containing headers
+#  GFLAGS_SHARED_LIB, path to libgflags shared library
+#  GFLAGS_STATIC_LIB, path to libgflags static library
+#  GFLAGS_FOUND, whether gflags has been found
+
+if( NOT "$ENV{GFLAGS_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "$ENV{GFLAGS_HOME}" _native_path )
+    list( APPEND _gflags_roots ${_native_path} )
+elseif ( GFlags_HOME )
+    list( APPEND _gflags_roots ${GFlags_HOME} )
+endif()
+
+if ( _gflags_roots )
+  find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "include" )
+  find_library(GFLAGS_SHARED_LIB NAMES gflags
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+  find_library(GFLAGS_SHARED_LIB NAMES libgflags.a
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+else()
+  find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h
+    # make sure we don't accidentally pick up a different version
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(GFLAGS_SHARED_LIB gflags
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(GFLAGS_STATIC_LIB libgflags.a
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GFLAGS REQUIRED_VARS
+  GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR)
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 93f0ad91ee86c..05fa6663e335d 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -186,12 +186,13 @@ static inline Status FileOpenWriteable(const std::string& filename, int* fd) {
   memcpy(wpath.data(), filename.data(), filename.size());
   memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
 
-  errno_actual = _wsopen_s(
-      fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY, _SH_DENYNO, _S_IWRITE);
+  errno_actual = _wsopen_s(fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY | _O_TRUNC,
+      _SH_DENYNO, _S_IWRITE);
   ret = *fd;
 
 #else
-  ret = *fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY, ARROW_WRITE_SHMODE);
+  ret = *fd =
+      open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY | O_TRUNC, ARROW_WRITE_SHMODE);
 #endif
   return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
 }
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 6955bcb6c233e..f9e7cf792ae51 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -85,6 +85,33 @@ ADD_ARROW_TEST(ipc-json-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-json-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
+ADD_ARROW_TEST(json-integration-test)
+
+if (APPLE)
+  target_link_libraries(json-integration-test
+    arrow_static
+    arrow_io
+    arrow_ipc
+    gflags
+    gtest
+    boost_filesystem_static
+    boost_system_static
+    dl)
+  set_target_properties(json-integration-test
+    PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+else()
+  target_link_libraries(json-integration-test
+    arrow_static
+    arrow_io
+    arrow_ipc
+    gflags
+    gtest
+    pthread
+    boost_filesystem_static
+    boost_system_static
+    dl)
+endif()
+
 # make clean will delete the generated file
 set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
 
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 3fde18dde835b..b02de284dfc7d 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -16,7 +16,7 @@
 // under the License.
 
 // Public API for writing and accessing (with zero copy, if possible) Arrow
-// data in shared memory
+// IPC binary formatted data (e.g. in shared memory, or from some other IO source)
 
 #ifndef ARROW_IPC_ADAPTER_H
 #define ARROW_IPC_ADAPTER_H
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
new file mode 100644
index 0000000000000..5eff8998afbc8
--- /dev/null
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -0,0 +1,381 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include "gflags/gflags.h"
+#include "gtest/gtest.h"
+
+#include <boost/filesystem.hpp>  // NOLINT
+
+#include "arrow/io/file.h"
+#include "arrow/ipc/file.h"
+#include "arrow/ipc/json.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
+#include "arrow/test-util.h"
+#include "arrow/util/status.h"
+
+DEFINE_string(arrow, "", "Arrow file name");
+DEFINE_string(json, "", "JSON file name");
+DEFINE_string(mode, "VALIDATE",
+    "Mode of integration testing tool (ARROW_TO_JSON, JSON_TO_ARROW, VALIDATE)");
+DEFINE_bool(integration, false, "Run in integration test mode");
+DEFINE_bool(verbose, true, "Verbose output");
+
+namespace fs = boost::filesystem;
+
+namespace arrow {
+
+bool file_exists(const char* path) {
+  std::ifstream handle(path);
+  return handle.good();
+}
+
+// Convert JSON file to IPC binary format
+static Status ConvertJsonToArrow(
+    const std::string& json_path, const std::string& arrow_path) {
+  std::shared_ptr<io::ReadableFile> in_file;
+  std::shared_ptr<io::FileOutputStream> out_file;
+
+  RETURN_NOT_OK(io::ReadableFile::Open(json_path, &in_file));
+  RETURN_NOT_OK(io::FileOutputStream::Open(arrow_path, &out_file));
+
+  int64_t file_size = 0;
+  RETURN_NOT_OK(in_file->GetSize(&file_size));
+
+  std::shared_ptr<Buffer> json_buffer;
+  RETURN_NOT_OK(in_file->Read(file_size, &json_buffer));
+
+  std::unique_ptr<ipc::JsonReader> reader;
+  RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &reader));
+
+  if (FLAGS_verbose) {
+    std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
+  }
+
+  std::shared_ptr<ipc::FileWriter> writer;
+  RETURN_NOT_OK(ipc::FileWriter::Open(out_file.get(), reader->schema(), &writer));
+
+  for (int i = 0; i < reader->num_record_batches(); ++i) {
+    std::shared_ptr<RecordBatch> batch;
+    RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
+    RETURN_NOT_OK(writer->WriteRecordBatch(batch->columns(), batch->num_rows()));
+  }
+  return writer->Close();
+}
+
+// Convert IPC binary format to JSON
+static Status ConvertArrowToJson(
+    const std::string& arrow_path, const std::string& json_path) {
+  std::shared_ptr<io::ReadableFile> in_file;
+  std::shared_ptr<io::FileOutputStream> out_file;
+
+  RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &in_file));
+  RETURN_NOT_OK(io::FileOutputStream::Open(json_path, &out_file));
+
+  std::shared_ptr<ipc::FileReader> reader;
+  RETURN_NOT_OK(ipc::FileReader::Open(in_file, &reader));
+
+  if (FLAGS_verbose) {
+    std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
+  }
+
+  std::unique_ptr<ipc::JsonWriter> writer;
+  RETURN_NOT_OK(ipc::JsonWriter::Open(reader->schema(), &writer));
+
+  for (int i = 0; i < reader->num_record_batches(); ++i) {
+    std::shared_ptr<RecordBatch> batch;
+    RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
+    RETURN_NOT_OK(writer->WriteRecordBatch(batch->columns(), batch->num_rows()));
+  }
+
+  std::string result;
+  RETURN_NOT_OK(writer->Finish(&result));
+  return out_file->Write(reinterpret_cast<const uint8_t*>(result.c_str()),
+      static_cast<int64_t>(result.size()));
+}
+
+static Status ValidateArrowVsJson(
+    const std::string& arrow_path, const std::string& json_path) {
+  // Construct JSON reader
+  std::shared_ptr<io::ReadableFile> json_file;
+  RETURN_NOT_OK(io::ReadableFile::Open(json_path, &json_file));
+
+  int64_t file_size = 0;
+  RETURN_NOT_OK(json_file->GetSize(&file_size));
+
+  std::shared_ptr<Buffer> json_buffer;
+  RETURN_NOT_OK(json_file->Read(file_size, &json_buffer));
+
+  std::unique_ptr<ipc::JsonReader> json_reader;
+  RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &json_reader));
+
+  // Construct Arrow reader
+  std::shared_ptr<io::ReadableFile> arrow_file;
+  RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &arrow_file));
+
+  std::shared_ptr<ipc::FileReader> arrow_reader;
+  RETURN_NOT_OK(ipc::FileReader::Open(arrow_file, &arrow_reader));
+
+  auto json_schema = json_reader->schema();
+  auto arrow_schema = arrow_reader->schema();
+
+  if (!json_schema->Equals(arrow_schema)) {
+    std::stringstream ss;
+    ss << "JSON schema: \n"
+       << json_schema->ToString() << "\n"
+       << "Arrow schema: \n"
+       << arrow_schema->ToString();
+
+    if (FLAGS_verbose) { std::cout << ss.str() << std::endl; }
+    return Status::Invalid("Schemas did not match");
+  }
+
+  const int json_nbatches = json_reader->num_record_batches();
+  const int arrow_nbatches = arrow_reader->num_record_batches();
+
+  if (json_nbatches != arrow_nbatches) {
+    std::stringstream ss;
+    ss << "Different number of record batches: " << json_nbatches << " (JSON) vs "
+       << arrow_nbatches << " (Arrow)";
+    return Status::Invalid(ss.str());
+  }
+
+  std::shared_ptr<RecordBatch> arrow_batch;
+  std::shared_ptr<RecordBatch> json_batch;
+  for (int i = 0; i < json_nbatches; ++i) {
+    RETURN_NOT_OK(json_reader->GetRecordBatch(i, &json_batch));
+    RETURN_NOT_OK(arrow_reader->GetRecordBatch(i, &arrow_batch));
+
+    if (!json_batch->Equals(*arrow_batch.get())) {
+      std::stringstream ss;
+      ss << "Record batch " << i << " did not match";
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  return Status::OK();
+}
+
+Status RunCommand(const std::string& json_path, const std::string& arrow_path,
+    const std::string& command) {
+  if (json_path == "") { return Status::Invalid("Must specify json file name"); }
+
+  if (arrow_path == "") { return Status::Invalid("Must specify arrow file name"); }
+
+  if (command == "ARROW_TO_JSON") {
+    if (!file_exists(arrow_path.c_str())) {
+      return Status::Invalid("Input file does not exist");
+    }
+
+    return ConvertArrowToJson(arrow_path, json_path);
+  } else if (command == "JSON_TO_ARROW") {
+    if (!file_exists(json_path.c_str())) {
+      return Status::Invalid("Input file does not exist");
+    }
+
+    return ConvertJsonToArrow(json_path, arrow_path);
+  } else if (command == "VALIDATE") {
+    if (!file_exists(json_path.c_str())) {
+      return Status::Invalid("JSON file does not exist");
+    }
+
+    if (!file_exists(arrow_path.c_str())) {
+      return Status::Invalid("Arrow file does not exist");
+    }
+
+    return ValidateArrowVsJson(arrow_path, json_path);
+  } else {
+    std::stringstream ss;
+    ss << "Unknown command: " << command;
+    return Status::Invalid(ss.str());
+  }
+}
+
+static std::string temp_path() {
+  return (fs::temp_directory_path() / fs::unique_path()).native();
+}
+
+class TestJSONIntegration : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  std::string mkstemp() {
+    auto path = temp_path();
+    tmp_paths_.push_back(path);
+    return path;
+  }
+
+  Status WriteJson(const char* data, const std::string& path) {
+    do {
+      std::shared_ptr<io::FileOutputStream> out;
+      RETURN_NOT_OK(io::FileOutputStream::Open(path, &out));
+      RETURN_NOT_OK(out->Write(
+          reinterpret_cast<const uint8_t*>(data), static_cast<int64_t>(strlen(data))));
+    } while (0);
+    return Status::OK();
+  }
+
+  void TearDown() {
+    for (const std::string path : tmp_paths_) {
+      std::remove(path.c_str());
+    }
+  }
+
+ protected:
+  std::vector<std::string> tmp_paths_;
+};
+
+static const char* JSON_EXAMPLE = R"example(
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "foo",
+        "type": {"name": "int", "isSigned": true, "bitWidth": 32},
+        "nullable": true, "children": [],
+        "typeLayout": [
+          {"type": "VALIDITY", "typeBitWidth": 1},
+          {"type": "DATA", "typeBitWidth": 32}
+        ]
+      },
+      {
+        "name": "bar",
+        "type": {"name": "floatingpoint", "precision": "DOUBLE"},
+        "nullable": true, "children": [],
+        "typeLayout": [
+          {"type": "VALIDITY", "typeBitWidth": 1},
+          {"type": "DATA", "typeBitWidth": 64}
+        ]
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 5,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 5,
+          "DATA": [1, 2, 3, 4, 5],
+          "VALIDITY": [1, 0, 1, 1, 1]
+        },
+        {
+          "name": "bar",
+          "count": 5,
+          "DATA": [1.0, 2.0, 3.0, 4.0, 5.0],
+          "VALIDITY": [1, 0, 0, 1, 1]
+        }
+      ]
+    }
+  ]
+}
+)example";
+
+static const char* JSON_EXAMPLE2 = R"example(
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "foo",
+        "type": {"name": "int", "isSigned": true, "bitWidth": 32},
+        "nullable": true, "children": [],
+        "typeLayout": [
+          {"type": "VALIDITY", "typeBitWidth": 1},
+          {"type": "DATA", "typeBitWidth": 32}
+        ]
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 5,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 5,
+          "DATA": [1, 2, 3, 4, 5],
+          "VALIDITY": [1, 0, 1, 1, 1]
+        }
+      ]
+    }
+  ]
+}
+)example";
+
+TEST_F(TestJSONIntegration, ConvertAndValidate) {
+  std::string json_path = this->mkstemp();
+  std::string arrow_path = this->mkstemp();
+
+  ASSERT_OK(WriteJson(JSON_EXAMPLE, json_path));
+
+  ASSERT_OK(RunCommand(json_path, arrow_path, "JSON_TO_ARROW"));
+  ASSERT_OK(RunCommand(json_path, arrow_path, "VALIDATE"));
+
+  // Convert and overwrite
+  ASSERT_OK(RunCommand(json_path, arrow_path, "ARROW_TO_JSON"));
+
+  // Convert back to arrow, and validate
+  ASSERT_OK(RunCommand(json_path, arrow_path, "JSON_TO_ARROW"));
+  ASSERT_OK(RunCommand(json_path, arrow_path, "VALIDATE"));
+}
+
+TEST_F(TestJSONIntegration, ErrorStates) {
+  std::string json_path = this->mkstemp();
+  std::string json_path2 = this->mkstemp();
+  std::string arrow_path = this->mkstemp();
+
+  ASSERT_OK(WriteJson(JSON_EXAMPLE, json_path));
+  ASSERT_OK(WriteJson(JSON_EXAMPLE2, json_path2));
+
+  ASSERT_OK(ConvertJsonToArrow(json_path, arrow_path));
+  ASSERT_RAISES(Invalid, ValidateArrowVsJson(arrow_path, json_path2));
+
+  ASSERT_RAISES(IOError, ValidateArrowVsJson("does_not_exist-1234", json_path2));
+  ASSERT_RAISES(IOError, ValidateArrowVsJson(arrow_path, "does_not_exist-1234"));
+
+  ASSERT_RAISES(Invalid, RunCommand("", arrow_path, "VALIDATE"));
+  ASSERT_RAISES(Invalid, RunCommand(json_path, "", "VALIDATE"));
+}
+
+}  // namespace arrow
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  int ret = 0;
+
+  if (FLAGS_integration) {
+    arrow::Status result = arrow::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode);
+    if (!result.ok()) {
+      std::cout << "Error message: " << result.ToString() << std::endl;
+      ret = 1;
+    }
+  } else {
+    ::testing::InitGoogleTest(&argc, argv);
+    ret = RUN_ALL_TESTS();
+  }
+  gflags::ShutDownCommandLineFlags();
+  return ret;
+}

From 197120cbc7ae419657bb3d22d1c343b49ec3e984 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 26 Nov 2016 14:14:40 -0500
Subject: [PATCH 0204/1644] ARROW-390: Only specify dependencies for
 json-integration-test on ARROW_BUILD_TESTS=ON

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #215 from xhochy/ARROW-390 and squashes the following commits:

36d34c7 [Uwe L. Korn] ARROW-390: Only specify dependencies for json-integration-test on ARROW_BUILD_TESTS=ON
---
 cpp/src/arrow/ipc/CMakeLists.txt | 48 +++++++++++++++++---------------
 1 file changed, 25 insertions(+), 23 deletions(-)

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index f9e7cf792ae51..6f401dba2495f 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -87,29 +87,31 @@ ARROW_TEST_LINK_LIBRARIES(ipc-json-test
 
 ADD_ARROW_TEST(json-integration-test)
 
-if (APPLE)
-  target_link_libraries(json-integration-test
-    arrow_static
-    arrow_io
-    arrow_ipc
-    gflags
-    gtest
-    boost_filesystem_static
-    boost_system_static
-    dl)
-  set_target_properties(json-integration-test
-    PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-else()
-  target_link_libraries(json-integration-test
-    arrow_static
-    arrow_io
-    arrow_ipc
-    gflags
-    gtest
-    pthread
-    boost_filesystem_static
-    boost_system_static
-    dl)
+if (ARROW_BUILD_TESTS)
+  if (APPLE)
+    target_link_libraries(json-integration-test
+      arrow_static
+      arrow_io
+      arrow_ipc
+      gflags
+      gtest
+      boost_filesystem_static
+      boost_system_static
+      dl)
+    set_target_properties(json-integration-test
+      PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  else()
+    target_link_libraries(json-integration-test
+      arrow_static
+      arrow_io
+      arrow_ipc
+      gflags
+      gtest
+      pthread
+      boost_filesystem_static
+      boost_system_static
+      dl)
+  endif()
 endif()
 
 # make clean will delete the generated file

From 86f56a6073c3254487ede3aff1dc9d117d24adaf Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 26 Nov 2016 14:22:47 -0500
Subject: [PATCH 0205/1644] ARROW-202: Integrate with appveyor ci for windows

This only adds yet a successful compilation for windows. Tests don't
run.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #213 from xhochy/ARROW-202 and squashes the following commits:

d5088a6 [Uwe L. Korn] Correctly reference Kudu in LICENSE and NOTICE
72a583b [Uwe L. Korn] Differentiate Boost libraries based on build type
6c75699 [Uwe L. Korn] Add license header
e33b08c [Uwe L. Korn] Pick up shared Boost libraries correctly
5da5f5d [Uwe L. Korn] ARROW-202: Integrate with appveyor ci for windows
---
 LICENSE.txt                          | 12 ++++++
 NOTICE.txt                           | 14 ++++++
 appveyor.yml                         | 39 +++++++++++++++++
 cpp/CMakeLists.txt                   | 64 ++++++++++++++++------------
 cpp/cmake_modules/CompilerInfo.cmake | 42 +++++++++++-------
 cpp/src/arrow/array-test.cc          |  1 +
 cpp/src/arrow/io/CMakeLists.txt      | 14 ++++--
 cpp/src/arrow/io/io-file-test.cc     |  9 +++-
 cpp/src/arrow/io/memory.cc           | 13 +++++-
 cpp/src/arrow/io/mman.h              | 12 +++---
 cpp/src/arrow/io/test-common.h       | 12 ++++++
 cpp/src/arrow/test-util.h            |  8 ++++
 cpp/src/arrow/type.h                 |  2 +-
 cpp/src/arrow/util/CMakeLists.txt    | 25 ++++++-----
 cpp/src/arrow/util/buffer.h          |  4 +-
 cpp/src/arrow/util/memory-pool.cc    | 14 ++++++
 cpp/src/arrow/util/visibility.h      |  1 +
 17 files changed, 217 insertions(+), 69 deletions(-)
 create mode 100644 appveyor.yml

diff --git a/LICENSE.txt b/LICENSE.txt
index d645695673349..c3bec4385507e 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -200,3 +200,15 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Kudu.
+
+ * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
+
+Copyright: 2016 The Apache Software Foundation.
+Home page: https://kudu.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 5c699ca022c1b..02cb4dd6ee001 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -41,3 +41,17 @@ This product includes software from the CMake project
 
 This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
  * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Kudu, which includes the following in
+its NOTICE file:
+
+  Apache Kudu
+  Copyright 2016 The Apache Software Foundation
+
+  This product includes software developed at
+  The Apache Software Foundation (http://www.apache.org/).
+
+  Portions of this software were developed at
+  Cloudera, Inc (http://www.cloudera.com/).
diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644
index 0000000000000..67478487081b7
--- /dev/null
+++ b/appveyor.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Operating system (build VM template)
+os: Visual Studio 2015
+
+environment:
+  matrix:
+    - GENERATOR: Visual Studio 14 2015 Win64
+    # - GENERATOR: Visual Studio 14 2015
+  MSVC_DEFAULT_OPTIONS: ON
+  BOOST_ROOT: C:\Libraries\boost_1_59_0
+  BOOST_LIBRARYDIR: C:\Libraries\boost_1_59_0\lib64-msvc-14.0
+
+build_script:
+ - cd cpp
+ - mkdir build
+ - cd build
+ # A lot of features are still deactivated as they do not build on Windows
+ #  * gbenchmark doesn't build with MSVC
+ - cmake -G "%GENERATOR%" -DARROW_BOOST_USE_SHARED=OFF -DARROW_IPC=OFF -DARROW_HDFS=OFF -DARROW_BUILD_BENCHMARKS=OFF ..
+ - cmake --build . --config Debug
+
+# test_script:
+#  - ctest -VV
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 839ea17dee02e..0edb8ce410b87 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -141,9 +141,11 @@ endif()
 # For CMAKE_BUILD_TYPE=Release
 #   -O3: Enable all compiler optimizations
 #   -g: Enable symbols for profiler tools (TODO: remove for shipping)
-set(CXX_FLAGS_DEBUG "-ggdb -O0")
-set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
-set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
+if (NOT MSVC)
+  set(CXX_FLAGS_DEBUG "-ggdb -O0")
+  set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
+  set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
+endif()
 
 set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate")
 set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use")
@@ -347,6 +349,8 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
       COMPILE_FLAGS " -DARROW_VALGRIND")
     add_test(${TEST_NAME}
       valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
+  elseif(MSVC)
+    add_test(${TEST_NAME} ${TEST_PATH})
   else()
     add_test(${TEST_NAME}
       ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
@@ -431,40 +435,44 @@ endfunction()
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
-# find boost headers and libs
+# Find static boost headers and libs
+# TODO Differentiate here between release and debug builds
 set(Boost_DEBUG TRUE)
 set(Boost_USE_MULTITHREADED ON)
 set(Boost_USE_STATIC_LIBS ON)
 find_package(Boost COMPONENTS system filesystem REQUIRED)
-include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
-set(BOOST_STATIC_LIBS ${Boost_LIBRARIES})
-list(LENGTH BOOST_STATIC_LIBS BOOST_STATIC_LIBS_LEN)
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+  set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+else()
+  set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+  set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+endif()
 
-# Find Boost shared libraries.
+# Find shared Boost libraries.
 set(Boost_USE_STATIC_LIBS OFF)
 find_package(Boost COMPONENTS system filesystem REQUIRED)
-set(BOOST_SHARED_LIBS ${Boost_LIBRARIES})
-list(LENGTH BOOST_SHARED_LIBS BOOST_SHARED_LIBS_LEN)
-list(SORT BOOST_SHARED_LIBS)
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+  set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+else()
+  set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+  set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+endif()
 
 message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
 message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
 
-math(EXPR LAST_IDX "${BOOST_STATIC_LIBS_LEN} - 1")
-foreach(IDX RANGE ${LAST_IDX})
-  list(GET BOOST_STATIC_LIBS ${IDX} BOOST_STATIC_LIB)
-  list(GET BOOST_SHARED_LIBS ${IDX} BOOST_SHARED_LIB)
+ADD_THIRDPARTY_LIB(boost_system
+    STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}"
+    SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
+
+ADD_THIRDPARTY_LIB(boost_filesystem
+    STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
+    SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
+
+SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 
-  # Remove the prefix/suffix from the library name.
-  #
-  # e.g. libboost_system-mt --> boost_system
-  get_filename_component(LIB_NAME ${BOOST_STATIC_LIB} NAME_WE)
-  string(REGEX REPLACE "lib([^-]*)(-mt)?" "\\1" LIB_NAME_NO_PREFIX_SUFFIX ${LIB_NAME})
-  ADD_THIRDPARTY_LIB(${LIB_NAME_NO_PREFIX_SUFFIX}
-    STATIC_LIB "${BOOST_STATIC_LIB}"
-    SHARED_LIB "${BOOST_SHARED_LIB}")
-  list(APPEND ARROW_BOOST_LIBS ${LIB_NAME_NO_PREFIX_SUFFIX})
-endforeach()
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 
 # ----------------------------------------------------------------------
@@ -482,7 +490,7 @@ if(ARROW_BUILD_TESTS)
 
     ExternalProject_Add(googletest_ep
       URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-      CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
+      CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON
       # googletest doesn't define install rules, so just build in the
       # source dir and don't try to install.  See its README for
       # details.
@@ -491,7 +499,7 @@ if(ARROW_BUILD_TESTS)
 
     set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
     set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
-    set(GTEST_STATIC_LIB "${GTEST_PREFIX}/libgtest.a")
+    set(GTEST_STATIC_LIB "${GTEST_PREFIX}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GTEST_VENDORED 1)
   else()
     find_package(GTest REQUIRED)
@@ -571,7 +579,7 @@ if(ARROW_BUILD_BENCHMARKS)
         "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}")
 
     set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include")
-    set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/libbenchmark.a")
+    set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GBENCHMARK_VENDORED 1)
   else()
     find_package(GBenchmark REQUIRED)
diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index 02f6fd46997a3..187698f54507b 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -1,25 +1,32 @@
-# Copyright 2013 Cloudera, Inc.
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 # Sets COMPILER_FAMILY to 'clang' or 'gcc'
 # Sets COMPILER_VERSION to the version
 execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
                 ERROR_VARIABLE COMPILER_VERSION_FULL)
 message(INFO " ${COMPILER_VERSION_FULL}")
+message(INFO " ${CMAKE_CXX_COMPILER_ID}")
+
+if(MSVC)
+  set(COMPILER_FAMILY "msvc")
 
 # clang on Linux and Mac OS X before 10.9
-if("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
   set(COMPILER_FAMILY "clang")
   string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
     COMPILER_VERSION "${COMPILER_VERSION_FULL}")
@@ -29,10 +36,15 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
   string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
     COMPILER_VERSION "${COMPILER_VERSION_FULL}")
 
-# clang on Mac OS X, XCode 7+. No version replacement is done
-# because Apple no longer advertises the upstream LLVM version.
-elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-.*")
+# clang on Mac OS X, XCode 7.
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-7")
+  set(COMPILER_FAMILY "clang")
+  set(COMPILER_VERSION "3.7.0svn")
+
+# clang on Mac OS X, XCode 8.
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-8")
   set(COMPILER_FAMILY "clang")
+  set(COMPILER_VERSION "3.8.0svn")
 
 # gcc
 elseif("${COMPILER_VERSION_FULL}" MATCHES ".*gcc version.*")
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 3b4736327b47c..158124468992a 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -18,6 +18,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <memory>
+#include <numeric>
 #include <vector>
 
 #include "gtest/gtest.h"
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 47bb089386371..a1892a9294a78 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -18,10 +18,16 @@
 # ----------------------------------------------------------------------
 # arrow_io : Arrow IO interfaces
 
-set(ARROW_IO_LINK_LIBS
-  arrow_shared
-  dl
-)
+if (MSVC)
+  set(ARROW_IO_LINK_LIBS
+    arrow_shared
+  )
+else()
+  set(ARROW_IO_LINK_LIBS
+    arrow_shared
+    dl
+  )
+endif()
 
 if (ARROW_BOOST_USE_SHARED)
   set(ARROW_IO_PRIVATE_LINK_LIBS
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index cde769ffb6155..54c21d2e62ce9 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -18,7 +18,9 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
-#include <fcntl.h>
+#ifndef _MSC_VER
+# include <fcntl.h>
+#endif
 #include <fstream>
 #include <memory>
 #include <sstream>
@@ -38,7 +40,12 @@ static bool FileExists(const std::string& path) {
 }
 
 static bool FileIsClosed(int fd) {
+#ifdef _MSC_VER
+  // Close file a second time, this should set errno to EBADF
+  close(fd);
+#else
   if (-1 != fcntl(fd, F_GETFD)) { return false; }
+#endif
   return errno == EBADF;
 }
 
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index c7d0ae5d56474..71b0f1e29b220 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -17,7 +17,18 @@
 
 #include "arrow/io/memory.h"
 
-#include <sys/mman.h>  // For memory-mapping
+// sys/mman.h not present in Visual Studio or Cygwin
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include "arrow/io/mman.h"
+#undef Realloc
+#undef Free
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
 
 #include <algorithm>
 #include <cerrno>
diff --git a/cpp/src/arrow/io/mman.h b/cpp/src/arrow/io/mman.h
index 00d1f93601df3..27d9736b683fd 100644
--- a/cpp/src/arrow/io/mman.h
+++ b/cpp/src/arrow/io/mman.h
@@ -76,7 +76,7 @@ static DWORD __map_mmap_prot_file(const int prot) {
   return desiredAccess;
 }
 
-void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
+static void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
   HANDLE fm, h;
 
   void* map = MAP_FAILED;
@@ -143,7 +143,7 @@ void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
   return map;
 }
 
-int munmap(void* addr, size_t len) {
+static int munmap(void* addr, size_t len) {
   if (UnmapViewOfFile(addr)) return 0;
 
   errno = __map_mman_error(GetLastError(), EPERM);
@@ -151,7 +151,7 @@ int munmap(void* addr, size_t len) {
   return -1;
 }
 
-int mprotect(void* addr, size_t len, int prot) {
+static int mprotect(void* addr, size_t len, int prot) {
   DWORD newProtect = __map_mmap_prot_page(prot);
   DWORD oldProtect = 0;
 
@@ -162,7 +162,7 @@ int mprotect(void* addr, size_t len, int prot) {
   return -1;
 }
 
-int msync(void* addr, size_t len, int flags) {
+static int msync(void* addr, size_t len, int flags) {
   if (FlushViewOfFile(addr, len)) return 0;
 
   errno = __map_mman_error(GetLastError(), EPERM);
@@ -170,7 +170,7 @@ int msync(void* addr, size_t len, int flags) {
   return -1;
 }
 
-int mlock(const void* addr, size_t len) {
+static int mlock(const void* addr, size_t len) {
   if (VirtualLock((LPVOID)addr, len)) return 0;
 
   errno = __map_mman_error(GetLastError(), EPERM);
@@ -178,7 +178,7 @@ int mlock(const void* addr, size_t len) {
   return -1;
 }
 
-int munlock(const void* addr, size_t len) {
+static int munlock(const void* addr, size_t len) {
   if (VirtualUnlock((LPVOID)addr, len)) return 0;
 
   errno = __map_mman_error(GetLastError(), EPERM);
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index 1954d479e3930..f8fed883cf583 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -24,6 +24,14 @@
 #include <string>
 #include <vector>
 
+#if defined(__MINGW32__)  // MinGW
+// nothing
+#elif defined(_MSC_VER)  // Visual Studio
+#include <io.h>
+#else  // POSIX / Linux
+// nothing
+#endif
+
 #include "arrow/io/memory.h"
 #include "arrow/test-util.h"
 #include "arrow/util/buffer.h"
@@ -43,7 +51,11 @@ class MemoryMapFixture {
   void CreateFile(const std::string path, int64_t size) {
     FILE* file = fopen(path.c_str(), "w");
     if (file != nullptr) { tmp_files_.push_back(path); }
+#ifdef _MSC_VER
+    _chsize(fileno(file), size);
+#else
     ftruncate(fileno(file), size);
+#endif
     fclose(file);
   }
 
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index ab4b980b3be63..93dd5b69b1bb7 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -59,6 +59,14 @@
     EXPECT_TRUE(s.ok());        \
   } while (0)
 
+// Alias MSVC popcount to GCC name
+#ifdef _MSC_VER
+#  include <intrin.h>
+#  define __builtin_popcount __popcnt
+#  include <nmmintrin.h>
+#  define __builtin_popcountll _mm_popcnt_u64
+#endif
+
 namespace arrow {
 
 class TestBase : public ::testing::Test {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 5b4d7bc42bd3d..876d7ea464b1c 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -184,7 +184,7 @@ struct ARROW_EXPORT Field {
 };
 typedef std::shared_ptr<Field> FieldPtr;
 
-struct PrimitiveCType : public DataType {
+struct ARROW_EXPORT PrimitiveCType : public DataType {
   using DataType::DataType;
 };
 
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index fd23c1aa3b8b2..6e19730219553 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -40,17 +40,20 @@ if (ARROW_BUILD_TESTS)
 	test_main.cc)
 
   if (APPLE)
-	target_link_libraries(arrow_test_main
-      gtest
-      dl)
-	set_target_properties(arrow_test_main
-      PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+	  target_link_libraries(arrow_test_main
+        gtest
+        dl)
+	  set_target_properties(arrow_test_main
+        PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  elseif(MSVC)
+	  target_link_libraries(arrow_test_main
+        gtest)
   else()
-	target_link_libraries(arrow_test_main
-      gtest
-      pthread
-      dl
-	  )
+	  target_link_libraries(arrow_test_main
+        gtest
+        pthread
+        dl
+	    )
   endif()
 endif()
 
@@ -71,4 +74,4 @@ endif()
 ADD_ARROW_TEST(bit-util-test)
 ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(memory-pool-test)
-ADD_ARROW_TEST(status-test)
\ No newline at end of file
+ADD_ARROW_TEST(status-test)
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 04ad6c2dffde4..330e15feae152 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -103,7 +103,7 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
 
 // Construct a view on passed buffer at the indicated offset and length. This
 // function cannot fail and does not error checking (except in debug builds)
-std::shared_ptr<Buffer> SliceBuffer(
+ARROW_EXPORT std::shared_ptr<Buffer> SliceBuffer(
     const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
 
 // A Buffer whose contents can be mutated. May or may not own its data.
@@ -154,7 +154,7 @@ class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
   MemoryPool* pool_;
 };
 
-class BufferBuilder {
+class ARROW_EXPORT BufferBuilder {
  public:
   explicit BufferBuilder(MemoryPool* pool)
       : pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
index 9f83afe4cb20f..9aa706693e9f7 100644
--- a/cpp/src/arrow/util/memory-pool.cc
+++ b/cpp/src/arrow/util/memory-pool.cc
@@ -33,6 +33,15 @@ namespace {
 Status AllocateAligned(int64_t size, uint8_t** out) {
   // TODO(emkornfield) find something compatible with windows
   constexpr size_t kAlignment = 64;
+#ifdef _MSC_VER
+  // Special code path for MSVC
+  *out = reinterpret_cast<uint8_t*>(_aligned_malloc(size, kAlignment));
+  if (!*out) {
+    std::stringstream ss;
+    ss << "malloc of size " << size << " failed";
+    return Status::OutOfMemory(ss.str());
+  }
+#else
   const int result = posix_memalign(reinterpret_cast<void**>(out), kAlignment, size);
   if (result == ENOMEM) {
     std::stringstream ss;
@@ -45,6 +54,7 @@ Status AllocateAligned(int64_t size, uint8_t** out) {
     ss << "invalid alignment parameter: " << kAlignment;
     return Status::Invalid(ss.str());
   }
+#endif
   return Status::OK();
 }
 }  // namespace
@@ -83,7 +93,11 @@ int64_t InternalMemoryPool::bytes_allocated() const {
 void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
   std::lock_guard<std::mutex> guard(pool_lock_);
   DCHECK_GE(bytes_allocated_, size);
+#ifdef _MSC_VER
+  _aligned_free(buffer);
+#else
   std::free(buffer);
+#endif
   bytes_allocated_ -= size;
 }
 
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
index b197c198297c8..9321cc550ec1f 100644
--- a/cpp/src/arrow/util/visibility.h
+++ b/cpp/src/arrow/util/visibility.h
@@ -20,6 +20,7 @@
 
 #if defined(_WIN32) || defined(__CYGWIN__)
 #define ARROW_EXPORT __declspec(dllexport)
+#define ARROW_NO_EXPORT
 #else  // Not Windows
 #ifndef ARROW_EXPORT
 #define ARROW_EXPORT __attribute__((visibility("default")))

From e3c167bd101734f92c3a2be2eb7f56f1fba91e67 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 28 Nov 2016 21:29:19 -0500
Subject: [PATCH 0206/1644] ARROW-363: [Java/C++] integration testing harness,
 initial integration tests

This also includes format reconciliation as discussed in ARROW-384.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #211 from wesm/ARROW-363 and squashes the following commits:

6982c3c [Wes McKinney] Permit end of buffer IPC reads if length is 0
4d46c8b [Wes McKinney] Fix logical error with offsets array in JsonFileWriter. Add broken string test case to simple.json
36ab5d6 [Wes McKinney] Increment MetadataVersion in flatbuffer
844257e [Wes McKinney] cpplint
a2711f2 [Wes McKinney] Address other format incompatibilities, write vectorLayout to Arrow metadata
13608ef [Wes McKinney] Relax 64 byte padding. Do not write RecordBatch embedded in Message for now
6a66fc8 [Wes McKinney] Write record batch size prefix in Java
72ea42c [Wes McKinney] Note that padding is 64-bytes at start of file (for now)
c2ffde4 [Wes McKinney] More notes about the file format
aef4382 [Wes McKinney] cpplint
85128f7 [Wes McKinney] Refactor IPC/File record batch read/write structure to reflect discussion in ARROW-384
dbd6ed6 [Wes McKinney] Do not embed metadata length in WriteDataHeader
c529d63 [Wes McKinney] Fix JSON integration test example to make it further
d806aa6 [Wes McKinney] Exclude JSON files from Apache RAT checks
a7e2d4b [Wes McKinney] Draft testing harness
---
 .gitignore                                    |  26 ++
 cpp/CMakeLists.txt                            |   1 -
 cpp/src/arrow/io/io-file-test.cc              |   2 +-
 cpp/src/arrow/io/memory.cc                    |  25 +-
 cpp/src/arrow/io/memory.h                     |   8 +-
 cpp/src/arrow/ipc/adapter.cc                  | 251 ++++++++++--------
 cpp/src/arrow/ipc/adapter.h                   |  65 ++---
 cpp/src/arrow/ipc/file.cc                     |  31 ++-
 cpp/src/arrow/ipc/ipc-adapter-test.cc         |  85 +++---
 cpp/src/arrow/ipc/ipc-file-test.cc            |   2 +-
 cpp/src/arrow/ipc/ipc-json-test.cc            |  20 +-
 cpp/src/arrow/ipc/ipc-metadata-test.cc        |  12 +-
 cpp/src/arrow/ipc/json-integration-test.cc    |  30 ++-
 cpp/src/arrow/ipc/json-internal.cc            | 110 +++-----
 cpp/src/arrow/ipc/metadata-internal.cc        | 100 ++++---
 cpp/src/arrow/ipc/metadata-internal.h         |   6 +-
 cpp/src/arrow/ipc/metadata.cc                 | 115 ++++----
 cpp/src/arrow/ipc/metadata.h                  |  50 ++--
 cpp/src/arrow/ipc/test-common.h               |  15 +-
 cpp/src/arrow/ipc/util.h                      |   6 +-
 cpp/src/arrow/test-util.h                     |   8 +-
 cpp/src/arrow/type.cc                         |  46 +++-
 cpp/src/arrow/type.h                          |  73 +++--
 cpp/src/arrow/types/primitive.cc              |   2 +-
 cpp/src/arrow/util/bit-util.h                 |   4 +
 dev/release/run-rat.sh                        |   3 +-
 format/IPC.md                                 | 106 ++++++++
 format/Message.fbs                            |   3 +-
 integration/data/simple.json                  |  66 +++++
 integration/integration_test.py               | 177 ++++++++++++
 java/pom.xml                                  |   6 +-
 java/tools/pom.xml                            |   6 +
 .../org/apache/arrow/tools/Integration.java   |   1 +
 .../org/apache/arrow/vector/VectorLoader.java |   4 +-
 .../apache/arrow/vector/file/ArrowReader.java |   6 +-
 .../apache/arrow/vector/file/ArrowWriter.java |  23 +-
 .../vector/file/json/JsonFileReader.java      |   9 +-
 .../vector/file/json/JsonFileWriter.java      |   2 +-
 python/.gitignore                             |  10 -
 39 files changed, 1024 insertions(+), 491 deletions(-)
 create mode 100644 .gitignore
 create mode 100644 integration/data/simple.json
 create mode 100644 integration/integration_test.py

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000..a00cbba065a03
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Compiled source
+*.a
+*.dll
+*.o
+*.py[ocd]
+*.so
+*.dylib
+.build_cache_dir
+MANIFEST
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0edb8ce410b87..1a970081234fa 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -528,7 +528,6 @@ if(ARROW_BUILD_TESTS)
     ExternalProject_Add(gflags_ep
       GIT_REPOSITORY https://github.com/gflags/gflags.git
       GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
-      # URL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz"
       BUILD_IN_SOURCE 1
       CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                  -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 54c21d2e62ce9..fad49cef89908 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -19,7 +19,7 @@
 #include <cstdio>
 #include <cstring>
 #ifndef _MSC_VER
-# include <fcntl.h>
+#include <fcntl.h>
 #endif
 #include <fstream>
 #include <memory>
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 71b0f1e29b220..af495e27e5642 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -258,8 +258,11 @@ Status BufferOutputStream::Reserve(int64_t nbytes) {
 // ----------------------------------------------------------------------
 // In-memory buffer reader
 
-BufferReader::BufferReader(const uint8_t* buffer, int buffer_size)
-    : buffer_(buffer), buffer_size_(buffer_size), position_(0) {}
+BufferReader::BufferReader(const std::shared_ptr<Buffer>& buffer)
+    : buffer_(buffer), data_(buffer->data()), size_(buffer->size()), position_(0) {}
+
+BufferReader::BufferReader(const uint8_t* data, int64_t size)
+    : buffer_(nullptr), data_(data), size_(size), position_(0) {}
 
 BufferReader::~BufferReader() {}
 
@@ -278,26 +281,32 @@ bool BufferReader::supports_zero_copy() const {
 }
 
 Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
-  memcpy(buffer, buffer_ + position_, nbytes);
-  *bytes_read = std::min(nbytes, buffer_size_ - position_);
+  memcpy(buffer, data_ + position_, nbytes);
+  *bytes_read = std::min(nbytes, size_ - position_);
   position_ += *bytes_read;
   return Status::OK();
 }
 
 Status BufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  int64_t size = std::min(nbytes, buffer_size_ - position_);
-  *out = std::make_shared<Buffer>(buffer_ + position_, size);
+  int64_t size = std::min(nbytes, size_ - position_);
+
+  if (buffer_ != nullptr) {
+    *out = SliceBuffer(buffer_, position_, size);
+  } else {
+    *out = std::make_shared<Buffer>(data_ + position_, size);
+  }
+
   position_ += nbytes;
   return Status::OK();
 }
 
 Status BufferReader::GetSize(int64_t* size) {
-  *size = buffer_size_;
+  *size = size_;
   return Status::OK();
 }
 
 Status BufferReader::Seek(int64_t position) {
-  if (position < 0 || position >= buffer_size_) {
+  if (position < 0 || position >= size_) {
     return Status::IOError("position out of bounds");
   }
 
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index df2fe8d6efbfc..b72f93b939148 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -99,7 +99,8 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
 
 class ARROW_EXPORT BufferReader : public ReadableFileInterface {
  public:
-  BufferReader(const uint8_t* buffer, int buffer_size);
+  explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
+  BufferReader(const uint8_t* data, int64_t size);
   ~BufferReader();
 
   Status Close() override;
@@ -116,8 +117,9 @@ class ARROW_EXPORT BufferReader : public ReadableFileInterface {
   bool supports_zero_copy() const override;
 
  private:
-  const uint8_t* buffer_;
-  int buffer_size_;
+  std::shared_ptr<Buffer> buffer_;
+  const uint8_t* data_;
+  int64_t size_;
   int64_t position_;
 };
 
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index da718c08d5480..edf716f662753 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -48,15 +48,6 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
-namespace {
-Status CheckMultipleOf64(int64_t size) {
-  if (BitUtil::IsMultipleOf64(size)) { return Status::OK(); }
-  return Status::Invalid(
-      "Attempted to write a buffer that "
-      "wasn't a multiple of 64 bytes");
-}
-}
-
 static bool IsPrimitive(const DataType* type) {
   DCHECK(type != nullptr);
   switch (type->type) {
@@ -124,30 +115,30 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
 class RecordBatchWriter {
  public:
   RecordBatchWriter(const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows,
-      int max_recursion_depth)
+      int64_t buffer_start_offset, int max_recursion_depth)
       : columns_(&columns),
         num_rows_(num_rows),
+        buffer_start_offset_(buffer_start_offset),
         max_recursion_depth_(max_recursion_depth) {}
 
-  Status AssemblePayload() {
+  Status AssemblePayload(int64_t* body_length) {
+    if (field_nodes_.size() > 0) {
+      field_nodes_.clear();
+      buffer_meta_.clear();
+      buffers_.clear();
+    }
+
     // Perform depth-first traversal of the row-batch
     for (size_t i = 0; i < columns_->size(); ++i) {
       const Array* arr = (*columns_)[i].get();
       RETURN_NOT_OK(VisitArray(arr, &field_nodes_, &buffers_, max_recursion_depth_));
     }
-    return Status::OK();
-  }
 
-  Status Write(
-      io::OutputStream* dst, int64_t* body_end_offset, int64_t* header_end_offset) {
-    // Get the starting position
-    int64_t start_position;
-    RETURN_NOT_OK(dst->Tell(&start_position));
-
-    // Keep track of the current position so we can determine the size of the
-    // message body
-    int64_t position = start_position;
+    // The position for the start of a buffer relative to the passed frame of
+    // reference. May be 0 or some other position in an address space
+    int64_t offset = buffer_start_offset_;
 
+    // Construct the buffer metadata for the record batch header
     for (size_t i = 0; i < buffers_.size(); ++i) {
       const Buffer* buffer = buffers_[i].get();
       int64_t size = 0;
@@ -161,65 +152,103 @@ class RecordBatchWriter {
 
       // TODO(wesm): We currently have no notion of shared memory page id's,
       // but we've included it in the metadata IDL for when we have it in the
-      // future. Use page=0 for now
+      // future. Use page = -1 for now
       //
       // Note that page ids are a bespoke notion for Arrow and not a feature we
       // are using from any OS-level shared memory. The thought is that systems
       // may (in the future) associate integer page id's with physical memory
       // pages (according to whatever is the desired shared memory mechanism)
-      buffer_meta_.push_back(flatbuf::Buffer(0, position, size + padding));
-
-      if (size > 0) {
-        RETURN_NOT_OK(dst->Write(buffer->data(), size));
-        position += size;
-      }
-
-      if (padding > 0) {
-        RETURN_NOT_OK(dst->Write(kPaddingBytes, padding));
-        position += padding;
-      }
+      buffer_meta_.push_back(flatbuf::Buffer(-1, offset, size + padding));
+      offset += size + padding;
     }
 
-    *body_end_offset = position;
+    *body_length = offset - buffer_start_offset_;
+    DCHECK(BitUtil::IsMultipleOf64(*body_length));
+
+    return Status::OK();
+  }
 
+  Status WriteMetadata(
+      int64_t body_length, io::OutputStream* dst, int32_t* metadata_length) {
     // Now that we have computed the locations of all of the buffers in shared
     // memory, the data header can be converted to a flatbuffer and written out
     //
     // Note: The memory written here is prefixed by the size of the flatbuffer
-    // itself as an int32_t. On reading from a input, you will have to
-    // determine the data header size then request a buffer such that you can
-    // construct the flatbuffer data accessor object (see arrow::ipc::Message)
-    std::shared_ptr<Buffer> data_header;
-    RETURN_NOT_OK(WriteDataHeader(
-        num_rows_, position - start_position, field_nodes_, buffer_meta_, &data_header));
+    // itself as an int32_t.
+    std::shared_ptr<Buffer> metadata_fb;
+    RETURN_NOT_OK(WriteRecordBatchMetadata(
+        num_rows_, body_length, field_nodes_, buffer_meta_, &metadata_fb));
+
+    // Need to write 4 bytes (metadata size), the metadata, plus padding to
+    // fall on a 64-byte offset
+    int64_t padded_metadata_length =
+        BitUtil::RoundUpToMultipleOf64(metadata_fb->size() + 4);
+
+    // The returned metadata size includes the length prefix, the flatbuffer,
+    // plus padding
+    *metadata_length = padded_metadata_length;
 
-    // Write the data header at the end
-    RETURN_NOT_OK(dst->Write(data_header->data(), data_header->size()));
+    // Write the flatbuffer size prefix
+    int32_t flatbuffer_size = metadata_fb->size();
+    RETURN_NOT_OK(
+        dst->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
 
-    position += data_header->size();
-    *header_end_offset = position;
+    // Write the flatbuffer
+    RETURN_NOT_OK(dst->Write(metadata_fb->data(), metadata_fb->size()));
 
-    return Align(dst, &position);
+    // Write any padding
+    int64_t padding = padded_metadata_length - metadata_fb->size() - 4;
+    if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
+
+    return Status::OK();
   }
 
-  Status Align(io::OutputStream* dst, int64_t* position) {
-    // Write all buffers here on word boundaries
-    // TODO(wesm): Is there benefit to 64-byte padding in IPC?
-    int64_t remainder = PaddedLength(*position) - *position;
-    if (remainder > 0) {
-      RETURN_NOT_OK(dst->Write(kPaddingBytes, remainder));
-      *position += remainder;
+  Status Write(io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
+    RETURN_NOT_OK(AssemblePayload(body_length));
+
+#ifndef NDEBUG
+    int64_t start_position, current_position;
+    RETURN_NOT_OK(dst->Tell(&start_position));
+#endif
+
+    RETURN_NOT_OK(WriteMetadata(*body_length, dst, metadata_length));
+
+#ifndef NDEBUG
+    RETURN_NOT_OK(dst->Tell(&current_position));
+    DCHECK(BitUtil::IsMultipleOf8(current_position));
+#endif
+
+    // Now write the buffers
+    for (size_t i = 0; i < buffers_.size(); ++i) {
+      const Buffer* buffer = buffers_[i].get();
+      int64_t size = 0;
+      int64_t padding = 0;
+
+      // The buffer might be null if we are handling zero row lengths.
+      if (buffer) {
+        size = buffer->size();
+        padding = BitUtil::RoundUpToMultipleOf64(size) - size;
+      }
+
+      if (size > 0) { RETURN_NOT_OK(dst->Write(buffer->data(), size)); }
+
+      if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
     }
+
+#ifndef NDEBUG
+    RETURN_NOT_OK(dst->Tell(&current_position));
+    DCHECK(BitUtil::IsMultipleOf8(current_position));
+#endif
+
     return Status::OK();
   }
 
-  // This must be called after invoking AssemblePayload
   Status GetTotalSize(int64_t* size) {
     // emulates the behavior of Write without actually writing
-    int64_t body_offset;
-    int64_t data_header_offset;
+    int32_t metadata_length;
+    int64_t body_length;
     MockOutputStream dst;
-    RETURN_NOT_OK(Write(&dst, &body_offset, &data_header_offset));
+    RETURN_NOT_OK(Write(&dst, &metadata_length, &body_length));
     *size = dst.GetExtentBytesWritten();
     return Status::OK();
   }
@@ -228,6 +257,7 @@ class RecordBatchWriter {
   // Do not copy this vector. Ownership must be retained elsewhere
   const std::vector<std::shared_ptr<Array>>* columns_;
   int32_t num_rows_;
+  int64_t buffer_start_offset_;
 
   std::vector<flatbuf::FieldNode> field_nodes_;
   std::vector<flatbuf::Buffer> buffer_meta_;
@@ -236,18 +266,17 @@ class RecordBatchWriter {
 };
 
 Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
-    int32_t num_rows, io::OutputStream* dst, int64_t* body_end_offset,
-    int64_t* header_end_offset, int max_recursion_depth) {
+    int32_t num_rows, int64_t buffer_start_offset, io::OutputStream* dst,
+    int32_t* metadata_length, int64_t* body_length, int max_recursion_depth) {
   DCHECK_GT(max_recursion_depth, 0);
-  RecordBatchWriter serializer(columns, num_rows, max_recursion_depth);
-  RETURN_NOT_OK(serializer.AssemblePayload());
-  return serializer.Write(dst, body_end_offset, header_end_offset);
+  RecordBatchWriter serializer(
+      columns, num_rows, buffer_start_offset, max_recursion_depth);
+  return serializer.Write(dst, metadata_length, body_length);
 }
 
 Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size) {
   RecordBatchWriter serializer(
-      batch->columns(), batch->num_rows(), kMaxIpcRecursionDepth);
-  RETURN_NOT_OK(serializer.AssemblePayload());
+      batch->columns(), batch->num_rows(), 0, kMaxIpcRecursionDepth);
   RETURN_NOT_OK(serializer.GetTotalSize(size));
   return Status::OK();
 }
@@ -255,30 +284,33 @@ Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size) {
 // ----------------------------------------------------------------------
 // Record batch read path
 
-class RecordBatchReader::RecordBatchReaderImpl {
+class RecordBatchReader {
  public:
-  RecordBatchReaderImpl(io::ReadableFileInterface* file,
-      const std::shared_ptr<RecordBatchMessage>& metadata, int max_recursion_depth)
-      : file_(file), metadata_(metadata), max_recursion_depth_(max_recursion_depth) {
+  RecordBatchReader(const std::shared_ptr<RecordBatchMetadata>& metadata,
+      const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+      io::ReadableFileInterface* file)
+      : metadata_(metadata),
+        schema_(schema),
+        max_recursion_depth_(max_recursion_depth),
+        file_(file) {
     num_buffers_ = metadata->num_buffers();
     num_flattened_fields_ = metadata->num_fields();
   }
 
-  Status AssembleBatch(
-      const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out) {
-    std::vector<std::shared_ptr<Array>> arrays(schema->num_fields());
+  Status Read(std::shared_ptr<RecordBatch>* out) {
+    std::vector<std::shared_ptr<Array>> arrays(schema_->num_fields());
 
     // The field_index and buffer_index are incremented in NextArray based on
     // how much of the batch is "consumed" (through nested data reconstruction,
     // for example)
     field_index_ = 0;
     buffer_index_ = 0;
-    for (int i = 0; i < schema->num_fields(); ++i) {
-      const Field* field = schema->field(i).get();
+    for (int i = 0; i < schema_->num_fields(); ++i) {
+      const Field* field = schema_->field(i).get();
       RETURN_NOT_OK(NextArray(field, max_recursion_depth_, &arrays[i]));
     }
 
-    *out = std::make_shared<RecordBatch>(schema, metadata_->length(), arrays);
+    *out = std::make_shared<RecordBatch>(schema_, metadata_->length(), arrays);
     return Status::OK();
   }
 
@@ -370,67 +402,56 @@ class RecordBatchReader::RecordBatchReaderImpl {
 
   Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
     BufferMetadata metadata = metadata_->buffer(buffer_index);
-    RETURN_NOT_OK(CheckMultipleOf64(metadata.length));
-    return file_->ReadAt(metadata.offset, metadata.length, out);
+
+    if (metadata.length == 0) {
+      *out = std::make_shared<Buffer>(nullptr, 0);
+      return Status::OK();
+    } else {
+      return file_->ReadAt(metadata.offset, metadata.length, out);
+    }
   }
 
  private:
+  std::shared_ptr<RecordBatchMetadata> metadata_;
+  std::shared_ptr<Schema> schema_;
+  int max_recursion_depth_;
   io::ReadableFileInterface* file_;
-  std::shared_ptr<RecordBatchMessage> metadata_;
 
   int field_index_;
   int buffer_index_;
-  int max_recursion_depth_;
   int num_buffers_;
   int num_flattened_fields_;
 };
 
-Status RecordBatchReader::Open(io::ReadableFileInterface* file, int64_t offset,
-    std::shared_ptr<RecordBatchReader>* out) {
-  return Open(file, offset, kMaxIpcRecursionDepth, out);
-}
-
-Status RecordBatchReader::Open(io::ReadableFileInterface* file, int64_t offset,
-    int max_recursion_depth, std::shared_ptr<RecordBatchReader>* out) {
+Status ReadRecordBatchMetadata(int64_t offset, int32_t metadata_length,
+    io::ReadableFileInterface* file, std::shared_ptr<RecordBatchMetadata>* metadata) {
   std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(file->ReadAt(offset - sizeof(int32_t), sizeof(int32_t), &buffer));
-
-  int32_t metadata_size = *reinterpret_cast<const int32_t*>(buffer->data());
+  RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
 
-  if (metadata_size + static_cast<int>(sizeof(int32_t)) > offset) {
-    return Status::Invalid("metadata size invalid");
-  }
-
-  // Read the metadata
-  RETURN_NOT_OK(
-      file->ReadAt(offset - metadata_size - sizeof(int32_t), metadata_size, &buffer));
-
-  // TODO(wesm): buffer slicing here would be better in case ReadAt returns
-  // allocated memory
-
-  std::shared_ptr<Message> message;
-  RETURN_NOT_OK(Message::Open(buffer, &message));
+  int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
 
-  if (message->type() != Message::RECORD_BATCH) {
-    return Status::Invalid("Metadata message is not a record batch");
+  if (flatbuffer_size + static_cast<int>(sizeof(int32_t)) > metadata_length) {
+    std::stringstream ss;
+    ss << "flatbuffer size " << metadata_length << " invalid. File offset: " << offset
+       << ", metadata length: " << metadata_length;
+    return Status::Invalid(ss.str());
   }
 
-  std::shared_ptr<RecordBatchMessage> batch_meta = message->GetRecordBatch();
-
-  std::shared_ptr<RecordBatchReader> result(new RecordBatchReader());
-  result->impl_.reset(new RecordBatchReaderImpl(file, batch_meta, max_recursion_depth));
-  *out = result;
-
+  *metadata = std::make_shared<RecordBatchMetadata>(buffer, sizeof(int32_t));
   return Status::OK();
 }
 
-// Here the explicit destructor is required for compilers to be aware of
-// the complete information of RecordBatchReader::RecordBatchReaderImpl class
-RecordBatchReader::~RecordBatchReader() {}
+Status ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
+    const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
+    std::shared_ptr<RecordBatch>* out) {
+  return ReadRecordBatch(metadata, schema, kMaxIpcRecursionDepth, file, out);
+}
 
-Status RecordBatchReader::GetRecordBatch(
-    const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out) {
-  return impl_->AssembleBatch(schema, out);
+Status ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
+    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+    io::ReadableFileInterface* file, std::shared_ptr<RecordBatch>* out) {
+  RecordBatchReader reader(metadata, schema, max_recursion_depth, file);
+  return reader.Read(out);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index b02de284dfc7d..963b9ee368537 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -43,7 +43,7 @@ class OutputStream;
 
 namespace ipc {
 
-class RecordBatchMessage;
+class RecordBatchMetadata;
 
 // ----------------------------------------------------------------------
 // Write path
@@ -51,22 +51,30 @@ class RecordBatchMessage;
 // TODO(emkornfield) investigate this more
 constexpr int kMaxIpcRecursionDepth = 64;
 
-// Write the RecordBatch (collection of equal-length Arrow arrays) to the output
-// stream
+// Write the RecordBatch (collection of equal-length Arrow arrays) to the
+// output stream in a contiguous block. The record batch metadata is written as
+// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
+// prefixed by its size, followed by each of the memory buffers in the batch
+// written end to end (with appropriate alignment and padding):
 //
-// First, each of the memory buffers are written out end-to-end
-//
-// Then, this function writes the batch metadata as a flatbuffer (see
-// format/Message.fbs -- the RecordBatch message type) like so:
-//
-// <int32: metadata size> <uint8*: metadata>
+// <int32: metadata size> <uint8*: metadata> <buffers>
 //
 // Finally, the absolute offsets (relative to the start of the output stream)
 // to the end of the body and end of the metadata / data header (suffixed by
 // the header size) is returned in out-variables
+//
+// @param(in) buffer_start_offset: the start offset to use in the buffer metadata,
+// default should be 0
+//
+// @param(out) metadata_length: the size of the length-prefixed flatbuffer
+// including padding to a 64-byte boundary
+//
+// @param(out) body_length: the size of the contiguous buffer block plus
+// padding bytes
 ARROW_EXPORT Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
-    int32_t num_rows, io::OutputStream* dst, int64_t* body_end_offset,
-    int64_t* header_end_offset, int max_recursion_depth = kMaxIpcRecursionDepth);
+    int32_t num_rows, int64_t buffer_start_offset, io::OutputStream* dst,
+    int32_t* metadata_length, int64_t* body_length,
+    int max_recursion_depth = kMaxIpcRecursionDepth);
 
 // int64_t GetRecordBatchMetadata(const RecordBatch* batch);
 
@@ -78,27 +86,20 @@ ARROW_EXPORT Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size);
 // ----------------------------------------------------------------------
 // "Read" path; does not copy data if the input supports zero copy reads
 
-class ARROW_EXPORT RecordBatchReader {
- public:
-  // The offset is the absolute position to the *end* of the record batch data
-  // header
-  static Status Open(io::ReadableFileInterface* file, int64_t offset,
-      std::shared_ptr<RecordBatchReader>* out);
-
-  static Status Open(io::ReadableFileInterface* file, int64_t offset,
-      int max_recursion_depth, std::shared_ptr<RecordBatchReader>* out);
-
-  virtual ~RecordBatchReader();
-
-  // Reassemble the record batch. A Schema is required to be able to construct
-  // the right array containers
-  Status GetRecordBatch(
-      const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out);
-
- private:
-  class RecordBatchReaderImpl;
-  std::unique_ptr<RecordBatchReaderImpl> impl_;
-};
+// Read the record batch flatbuffer metadata starting at the indicated file offset
+//
+// The flatbuffer is expected to be length-prefixed, so the metadata_length
+// includes at least the length prefix and the flatbuffer
+Status ARROW_EXPORT ReadRecordBatchMetadata(int64_t offset, int32_t metadata_length,
+    io::ReadableFileInterface* file, std::shared_ptr<RecordBatchMetadata>* metadata);
+
+Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
+    const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
+    std::shared_ptr<RecordBatch>* out);
+
+Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
+    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+    io::ReadableFileInterface* file, std::shared_ptr<RecordBatch>* out);
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
index c68244d50258c..06001cc1c77bc 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/file.cc
@@ -23,6 +23,7 @@
 #include <vector>
 
 #include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
 #include "arrow/ipc/adapter.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
@@ -87,19 +88,19 @@ Status FileWriter::WriteRecordBatch(
 
   int64_t offset = position_;
 
-  int64_t body_end_offset;
-  int64_t header_end_offset;
+  // There may be padding ever the end of the metadata, so we cannot rely on
+  // position_
+  int32_t metadata_length;
+  int64_t body_length;
+
+  // Frame of reference in file format is 0, see ARROW-384
+  const int64_t buffer_start_offset = 0;
   RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(
-      columns, num_rows, sink_, &body_end_offset, &header_end_offset));
+      columns, num_rows, buffer_start_offset, sink_, &metadata_length, &body_length));
   RETURN_NOT_OK(UpdatePosition());
 
   DCHECK(position_ % 8 == 0) << "ipc::WriteRecordBatch did not perform aligned writes";
 
-  // There may be padding ever the end of the metadata, so we cannot rely on
-  // position_
-  int32_t metadata_length = header_end_offset - body_end_offset;
-  int32_t body_length = body_end_offset - offset;
-
   // Append metadata, to be written in the footer later
   record_batches_.emplace_back(offset, metadata_length, body_length);
 
@@ -198,12 +199,18 @@ Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
   DCHECK_GE(i, 0);
   DCHECK_LT(i, num_record_batches());
   FileBlock block = footer_->record_batch(i);
-  int64_t metadata_end_offset = block.offset + block.body_length + block.metadata_length;
 
-  std::shared_ptr<RecordBatchReader> reader;
-  RETURN_NOT_OK(RecordBatchReader::Open(file_.get(), metadata_end_offset, &reader));
+  std::shared_ptr<RecordBatchMetadata> metadata;
+  RETURN_NOT_OK(ReadRecordBatchMetadata(
+      block.offset, block.metadata_length, file_.get(), &metadata));
+
+  // TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
+  // ARROW-384).
+  std::shared_ptr<Buffer> buffer_block;
+  RETURN_NOT_OK(file_->Read(block.body_length, &buffer_block));
+  io::BufferReader reader(buffer_block);
 
-  return reader->GetRecordBatch(schema_, batch);
+  return ReadRecordBatch(metadata, schema_, &reader, batch);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index f5611d4840c97..1accfde7c4842 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -54,17 +54,24 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
     std::string path = "test-write-row-batch";
     io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
 
-    int64_t body_end_offset;
-    int64_t header_end_offset;
+    int32_t metadata_length;
+    int64_t body_length;
 
-    RETURN_NOT_OK(WriteRecordBatch(batch.columns(), batch.num_rows(), mmap_.get(),
-        &body_end_offset, &header_end_offset));
+    const int64_t buffer_offset = 0;
 
-    std::shared_ptr<RecordBatchReader> reader;
-    RETURN_NOT_OK(RecordBatchReader::Open(mmap_.get(), header_end_offset, &reader));
+    RETURN_NOT_OK(WriteRecordBatch(batch.columns(), batch.num_rows(), buffer_offset,
+        mmap_.get(), &metadata_length, &body_length));
 
-    RETURN_NOT_OK(reader->GetRecordBatch(batch.schema(), batch_result));
-    return Status::OK();
+    std::shared_ptr<RecordBatchMetadata> metadata;
+    RETURN_NOT_OK(ReadRecordBatchMetadata(0, metadata_length, mmap_.get(), &metadata));
+
+    // The buffer offsets start at 0, so we must construct a
+    // ReadableFileInterface according to that frame of reference
+    std::shared_ptr<Buffer> buffer_payload;
+    RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
+    io::BufferReader buffer_reader(buffer_payload);
+
+    return ReadRecordBatch(metadata, batch.schema(), &buffer_reader, batch_result);
   }
 
  protected:
@@ -96,11 +103,11 @@ INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRecordBatch,
 
 void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   ipc::MockOutputStream mock;
-  int64_t mock_header_offset = -1;
-  int64_t mock_body_offset = -1;
+  int32_t mock_metadata_length = -1;
+  int64_t mock_body_length = -1;
   int64_t size = -1;
-  ASSERT_OK(WriteRecordBatch(batch->columns(), batch->num_rows(), &mock,
-      &mock_body_offset, &mock_header_offset));
+  ASSERT_OK(WriteRecordBatch(batch->columns(), batch->num_rows(), 0, &mock,
+      &mock_metadata_length, &mock_body_length));
   ASSERT_OK(GetRecordBatchSize(batch.get(), &size));
   ASSERT_EQ(mock.GetExtentBytesWritten(), size);
 }
@@ -129,39 +136,36 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
   void SetUp() { pool_ = default_memory_pool(); }
   void TearDown() { io::MemoryMapFixture::TearDown(); }
 
-  Status WriteToMmap(int recursion_level, bool override_level,
-      int64_t* header_out = nullptr, std::shared_ptr<Schema>* schema_out = nullptr) {
+  Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
+      int64_t* body_length, std::shared_ptr<Schema>* schema) {
     const int batch_length = 5;
-    TypePtr type = kInt32;
+    TypePtr type = int32();
     ArrayPtr array;
     const bool include_nulls = true;
     RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
     for (int i = 0; i < recursion_level; ++i) {
-      type = std::static_pointer_cast<DataType>(std::make_shared<ListType>(type));
+      type = list(type);
       RETURN_NOT_OK(
           MakeRandomListArray(array, batch_length, include_nulls, pool_, &array));
     }
 
-    auto f0 = std::make_shared<Field>("f0", type);
-    std::shared_ptr<Schema> schema(new Schema({f0}));
-    if (schema_out != nullptr) { *schema_out = schema; }
+    auto f0 = field("f0", type);
+
+    *schema = std::shared_ptr<Schema>(new Schema({f0}));
+
     std::vector<ArrayPtr> arrays = {array};
-    auto batch = std::make_shared<RecordBatch>(schema, batch_length, arrays);
+    auto batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
 
     std::string path = "test-write-past-max-recursion";
     const int memory_map_size = 1 << 16;
     io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
 
-    int64_t body_offset;
-    int64_t header_offset;
-
-    int64_t* header_out_param = header_out == nullptr ? &header_offset : header_out;
     if (override_level) {
-      return WriteRecordBatch(batch->columns(), batch->num_rows(), mmap_.get(),
-          &body_offset, header_out_param, recursion_level + 1);
+      return WriteRecordBatch(batch->columns(), batch->num_rows(), 0, mmap_.get(),
+          metadata_length, body_length, recursion_level + 1);
     } else {
-      return WriteRecordBatch(batch->columns(), batch->num_rows(), mmap_.get(),
-          &body_offset, header_out_param);
+      return WriteRecordBatch(batch->columns(), batch->num_rows(), 0, mmap_.get(),
+          metadata_length, body_length);
     }
   }
 
@@ -171,18 +175,29 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
 };
 
 TEST_F(RecursionLimits, WriteLimit) {
-  ASSERT_RAISES(Invalid, WriteToMmap((1 << 8) + 1, false));
+  int32_t metadata_length = -1;
+  int64_t body_length = -1;
+  std::shared_ptr<Schema> schema;
+  ASSERT_RAISES(
+      Invalid, WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &schema));
 }
 
 TEST_F(RecursionLimits, ReadLimit) {
-  int64_t header_offset = -1;
+  int32_t metadata_length = -1;
+  int64_t body_length = -1;
   std::shared_ptr<Schema> schema;
-  ASSERT_OK(WriteToMmap(64, true, &header_offset, &schema));
+  ASSERT_OK(WriteToMmap(64, true, &metadata_length, &body_length, &schema));
 
-  std::shared_ptr<RecordBatchReader> reader;
-  ASSERT_OK(RecordBatchReader::Open(mmap_.get(), header_offset, &reader));
-  std::shared_ptr<RecordBatch> batch_result;
-  ASSERT_RAISES(Invalid, reader->GetRecordBatch(schema, &batch_result));
+  std::shared_ptr<RecordBatchMetadata> metadata;
+  ASSERT_OK(ReadRecordBatchMetadata(0, metadata_length, mmap_.get(), &metadata));
+
+  std::shared_ptr<Buffer> payload;
+  ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
+
+  io::BufferReader reader(payload);
+
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_RAISES(Invalid, ReadRecordBatch(metadata, schema, &reader, &batch));
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index cd424bf385cae..a1feac401f24e 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -68,7 +68,7 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
     RETURN_NOT_OK(sink_->Tell(&footer_offset));
 
     // Open the file
-    auto reader = std::make_shared<io::BufferReader>(buffer_->data(), buffer_->size());
+    auto reader = std::make_shared<io::BufferReader>(buffer_);
     RETURN_NOT_OK(FileReader::Open(reader, footer_offset, &file_reader_));
 
     EXPECT_EQ(num_batches, file_reader_->num_record_batches());
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index a51371c62005b..e5c3a081fca53 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -284,19 +284,23 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
         "name": "foo",
         "type": {"name": "int", "isSigned": true, "bitWidth": 32},
         "nullable": true, "children": [],
-        "typeLayout": [
-          {"type": "VALIDITY", "typeBitWidth": 1},
-          {"type": "DATA", "typeBitWidth": 32}
-        ]
+        "typeLayout": {
+          "vectors": [
+            {"type": "VALIDITY", "typeBitWidth": 1},
+            {"type": "DATA", "typeBitWidth": 32}
+          ]
+        }
       },
       {
         "name": "bar",
         "type": {"name": "floatingpoint", "precision": "DOUBLE"},
         "nullable": true, "children": [],
-        "typeLayout": [
-          {"type": "VALIDITY", "typeBitWidth": 1},
-          {"type": "DATA", "typeBitWidth": 64}
-        ]
+        "typeLayout": {
+          "vectors": [
+            {"type": "VALIDITY", "typeBitWidth": 1},
+            {"type": "DATA", "typeBitWidth": 64}
+          ]
+        }
       }
     ]
   },
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
index 1dc3969233237..d29583f8488e0 100644
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -43,7 +43,7 @@ static inline void assert_schema_equal(const Schema* lhs, const Schema* rhs) {
   }
 }
 
-class TestSchemaMessage : public ::testing::Test {
+class TestSchemaMetadata : public ::testing::Test {
  public:
   void SetUp() {}
 
@@ -52,11 +52,11 @@ class TestSchemaMessage : public ::testing::Test {
     ASSERT_OK(WriteSchema(schema, &buffer));
 
     std::shared_ptr<Message> message;
-    ASSERT_OK(Message::Open(buffer, &message));
+    ASSERT_OK(Message::Open(buffer, 0, &message));
 
     ASSERT_EQ(Message::SCHEMA, message->type());
 
-    std::shared_ptr<SchemaMessage> schema_msg = message->GetSchema();
+    auto schema_msg = std::make_shared<SchemaMetadata>(message);
     ASSERT_EQ(schema->num_fields(), schema_msg->num_fields());
 
     std::shared_ptr<Schema> schema2;
@@ -68,7 +68,7 @@ class TestSchemaMessage : public ::testing::Test {
 
 const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
 
-TEST_F(TestSchemaMessage, PrimitiveFields) {
+TEST_F(TestSchemaMetadata, PrimitiveFields) {
   auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
   auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>());
   auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
@@ -85,7 +85,7 @@ TEST_F(TestSchemaMessage, PrimitiveFields) {
   CheckRoundtrip(&schema);
 }
 
-TEST_F(TestSchemaMessage, NestedFields) {
+TEST_F(TestSchemaMetadata, NestedFields) {
   auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
   auto f0 = std::make_shared<Field>("f0", type);
 
@@ -111,7 +111,7 @@ class TestFileFooter : public ::testing::Test {
     std::unique_ptr<FileFooter> footer;
     ASSERT_OK(FileFooter::Open(buffer, &footer));
 
-    ASSERT_EQ(MetadataVersion::V1_SNAPSHOT, footer->version());
+    ASSERT_EQ(MetadataVersion::V2, footer->version());
 
     // Check schema
     std::shared_ptr<Schema> schema2;
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 5eff8998afbc8..7a313f791e6c8 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -255,19 +255,23 @@ static const char* JSON_EXAMPLE = R"example(
         "name": "foo",
         "type": {"name": "int", "isSigned": true, "bitWidth": 32},
         "nullable": true, "children": [],
-        "typeLayout": [
-          {"type": "VALIDITY", "typeBitWidth": 1},
-          {"type": "DATA", "typeBitWidth": 32}
-        ]
+        "typeLayout": {
+          "vectors": [
+            {"type": "VALIDITY", "typeBitWidth": 1},
+            {"type": "DATA", "typeBitWidth": 32}
+          ]
+        }
       },
       {
         "name": "bar",
         "type": {"name": "floatingpoint", "precision": "DOUBLE"},
         "nullable": true, "children": [],
-        "typeLayout": [
-          {"type": "VALIDITY", "typeBitWidth": 1},
-          {"type": "DATA", "typeBitWidth": 64}
-        ]
+        "typeLayout": {
+          "vectors": [
+            {"type": "VALIDITY", "typeBitWidth": 1},
+            {"type": "DATA", "typeBitWidth": 64}
+          ]
+        }
       }
     ]
   },
@@ -301,10 +305,12 @@ static const char* JSON_EXAMPLE2 = R"example(
         "name": "foo",
         "type": {"name": "int", "isSigned": true, "bitWidth": 32},
         "nullable": true, "children": [],
-        "typeLayout": [
-          {"type": "VALIDITY", "typeBitWidth": 1},
-          {"type": "DATA", "typeBitWidth": 32}
-        ]
+        "typeLayout": {
+          "vectors": [
+            {"type": "VALIDITY", "typeBitWidth": 1},
+            {"type": "DATA", "typeBitWidth": 32}
+          ]
+        }
       }
     ]
   },
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 31fe35b44cef7..e56bcb32b9488 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -45,8 +45,6 @@ namespace ipc {
 using RjArray = rj::Value::ConstArray;
 using RjObject = rj::Value::ConstObject;
 
-enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
-
 static std::string GetBufferTypeName(BufferType type) {
   switch (type) {
     case BufferType::DATA:
@@ -93,27 +91,6 @@ static std::string GetTimeUnitName(TimeUnit unit) {
   return "UNKNOWN";
 }
 
-class BufferLayout {
- public:
-  BufferLayout(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
-
-  BufferType type() const { return type_; }
-  int bit_width() const { return bit_width_; }
-
- private:
-  BufferType type_;
-  int bit_width_;
-};
-
-static const BufferLayout kValidityBuffer(BufferType::VALIDITY, 1);
-static const BufferLayout kOffsetBuffer(BufferType::OFFSET, 32);
-static const BufferLayout kTypeBuffer(BufferType::TYPE, 32);
-static const BufferLayout kBooleanBuffer(BufferType::DATA, 1);
-static const BufferLayout kValues64(BufferType::DATA, 64);
-static const BufferLayout kValues32(BufferType::DATA, 32);
-static const BufferLayout kValues16(BufferType::DATA, 16);
-static const BufferLayout kValues8(BufferType::DATA, 8);
-
 class JsonSchemaWriter : public TypeVisitor {
  public:
   explicit JsonSchemaWriter(const Schema& schema, RjWriter* writer)
@@ -154,9 +131,9 @@ class JsonSchemaWriter : public TypeVisitor {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
-                              std::is_base_of<BooleanType, T>::value ||
-                              std::is_base_of<NullType, T>::value,
+  typename std::enable_if<
+      std::is_base_of<NoExtraMeta, T>::value || std::is_base_of<BooleanType, T>::value ||
+          std::is_base_of<DateType, T>::value || std::is_base_of<NullType, T>::value,
       void>::type
   WriteTypeMetadata(const T& type) {}
 
@@ -243,11 +220,10 @@ class JsonSchemaWriter : public TypeVisitor {
   }
 
   template <typename T>
-  Status WritePrimitive(const std::string& typeclass, const T& type,
-      const std::vector<BufferLayout>& buffer_layout) {
+  Status WritePrimitive(const std::string& typeclass, const T& type) {
     WriteName(typeclass, type);
     SetNoChildren();
-    WriteBufferLayout(buffer_layout);
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
@@ -255,15 +231,17 @@ class JsonSchemaWriter : public TypeVisitor {
   Status WriteVarBytes(const std::string& typeclass, const T& type) {
     WriteName(typeclass, type);
     SetNoChildren();
-    WriteBufferLayout({kValidityBuffer, kOffsetBuffer, kValues8});
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
-  void WriteBufferLayout(const std::vector<BufferLayout>& buffer_layout) {
+  void WriteBufferLayout(const std::vector<BufferDescr>& buffer_layout) {
     writer_->Key("typeLayout");
+    writer_->StartObject();
+    writer_->Key("vectors");
     writer_->StartArray();
 
-    for (const BufferLayout& buffer : buffer_layout) {
+    for (const BufferDescr& buffer : buffer_layout) {
       writer_->StartObject();
       writer_->Key("type");
       writer_->String(GetBufferTypeName(buffer.type()));
@@ -274,6 +252,7 @@ class JsonSchemaWriter : public TypeVisitor {
       writer_->EndObject();
     }
     writer_->EndArray();
+    writer_->EndObject();
   }
 
   Status WriteChildren(const std::vector<std::shared_ptr<Field>>& children) {
@@ -286,74 +265,52 @@ class JsonSchemaWriter : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const NullType& type) override { return WritePrimitive("null", type, {}); }
+  Status Visit(const NullType& type) override { return WritePrimitive("null", type); }
 
-  Status Visit(const BooleanType& type) override {
-    return WritePrimitive("bool", type, {kValidityBuffer, kBooleanBuffer});
-  }
+  Status Visit(const BooleanType& type) override { return WritePrimitive("bool", type); }
 
-  Status Visit(const Int8Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues8});
-  }
+  Status Visit(const Int8Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const Int16Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues16});
-  }
+  Status Visit(const Int16Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const Int32Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues32});
-  }
+  Status Visit(const Int32Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const Int64Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues64});
-  }
+  Status Visit(const Int64Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt8Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues8});
-  }
+  Status Visit(const UInt8Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt16Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues16});
-  }
+  Status Visit(const UInt16Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt32Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues32});
-  }
+  Status Visit(const UInt32Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt64Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues64});
-  }
+  Status Visit(const UInt64Type& type) override { return WritePrimitive("int", type); }
 
   Status Visit(const HalfFloatType& type) override {
-    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues16});
+    return WritePrimitive("floatingpoint", type);
   }
 
   Status Visit(const FloatType& type) override {
-    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues32});
+    return WritePrimitive("floatingpoint", type);
   }
 
   Status Visit(const DoubleType& type) override {
-    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues64});
+    return WritePrimitive("floatingpoint", type);
   }
 
   Status Visit(const StringType& type) override { return WriteVarBytes("utf8", type); }
 
   Status Visit(const BinaryType& type) override { return WriteVarBytes("binary", type); }
 
-  Status Visit(const DateType& type) override {
-    return WritePrimitive("date", type, {kValidityBuffer, kValues64});
-  }
+  Status Visit(const DateType& type) override { return WritePrimitive("date", type); }
 
-  Status Visit(const TimeType& type) override {
-    return WritePrimitive("time", type, {kValidityBuffer, kValues64});
-  }
+  Status Visit(const TimeType& type) override { return WritePrimitive("time", type); }
 
   Status Visit(const TimestampType& type) override {
-    return WritePrimitive("timestamp", type, {kValidityBuffer, kValues64});
+    return WritePrimitive("timestamp", type);
   }
 
   Status Visit(const IntervalType& type) override {
-    return WritePrimitive("interval", type, {kValidityBuffer, kValues64});
+    return WritePrimitive("interval", type);
   }
 
   Status Visit(const DecimalType& type) override { return Status::NotImplemented("NYI"); }
@@ -361,26 +318,21 @@ class JsonSchemaWriter : public TypeVisitor {
   Status Visit(const ListType& type) override {
     WriteName("list", type);
     RETURN_NOT_OK(WriteChildren(type.children()));
-    WriteBufferLayout({kValidityBuffer, kOffsetBuffer});
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
   Status Visit(const StructType& type) override {
     WriteName("struct", type);
     WriteChildren(type.children());
-    WriteBufferLayout({kValidityBuffer, kTypeBuffer});
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
   Status Visit(const UnionType& type) override {
     WriteName("union", type);
     WriteChildren(type.children());
-
-    if (type.mode == UnionMode::SPARSE) {
-      WriteBufferLayout({kValidityBuffer, kTypeBuffer});
-    } else {
-      WriteBufferLayout({kValidityBuffer, kTypeBuffer, kOffsetBuffer});
-    }
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 7102012c29a84..b99522825d902 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -37,20 +37,6 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
-const std::shared_ptr<DataType> BOOL = std::make_shared<BooleanType>();
-const std::shared_ptr<DataType> INT8 = std::make_shared<Int8Type>();
-const std::shared_ptr<DataType> INT16 = std::make_shared<Int16Type>();
-const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
-const std::shared_ptr<DataType> INT64 = std::make_shared<Int64Type>();
-const std::shared_ptr<DataType> UINT8 = std::make_shared<UInt8Type>();
-const std::shared_ptr<DataType> UINT16 = std::make_shared<UInt16Type>();
-const std::shared_ptr<DataType> UINT32 = std::make_shared<UInt32Type>();
-const std::shared_ptr<DataType> UINT64 = std::make_shared<UInt64Type>();
-const std::shared_ptr<DataType> FLOAT = std::make_shared<FloatType>();
-const std::shared_ptr<DataType> DOUBLE = std::make_shared<DoubleType>();
-const std::shared_ptr<DataType> STRING = std::make_shared<StringType>();
-const std::shared_ptr<DataType> BINARY = std::make_shared<BinaryType>();
-
 static Status IntFromFlatbuffer(
     const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
   if (int_data->bitWidth() > 64) {
@@ -62,16 +48,16 @@ static Status IntFromFlatbuffer(
 
   switch (int_data->bitWidth()) {
     case 8:
-      *out = int_data->is_signed() ? INT8 : UINT8;
+      *out = int_data->is_signed() ? int8() : uint8();
       break;
     case 16:
-      *out = int_data->is_signed() ? INT16 : UINT16;
+      *out = int_data->is_signed() ? int16() : uint16();
       break;
     case 32:
-      *out = int_data->is_signed() ? INT32 : UINT32;
+      *out = int_data->is_signed() ? int32() : uint32();
       break;
     case 64:
-      *out = int_data->is_signed() ? INT64 : UINT64;
+      *out = int_data->is_signed() ? int64() : uint64();
       break;
     default:
       return Status::NotImplemented("Integers not in cstdint are not implemented");
@@ -81,10 +67,12 @@ static Status IntFromFlatbuffer(
 
 static Status FloatFromFlatuffer(
     const flatbuf::FloatingPoint* float_data, std::shared_ptr<DataType>* out) {
-  if (float_data->precision() == flatbuf::Precision_SINGLE) {
-    *out = FLOAT;
+  if (float_data->precision() == flatbuf::Precision_HALF) {
+    *out = float16();
+  } else if (float_data->precision() == flatbuf::Precision_SINGLE) {
+    *out = float32();
   } else {
-    *out = DOUBLE;
+    *out = float64();
   }
   return Status::OK();
 }
@@ -100,13 +88,13 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       return FloatFromFlatuffer(
           static_cast<const flatbuf::FloatingPoint*>(type_data), out);
     case flatbuf::Type_Binary:
-      *out = BINARY;
+      *out = binary();
       return Status::OK();
     case flatbuf::Type_Utf8:
-      *out = STRING;
+      *out = utf8();
       return Status::OK();
     case flatbuf::Type_Bool:
-      *out = BOOL;
+      *out = boolean();
       return Status::OK();
     case flatbuf::Type_Decimal:
     case flatbuf::Type_Timestamp:
@@ -164,7 +152,32 @@ static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type
   break;
 
 static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* children, flatbuf::Type* out_type, Offset* offset) {
+    std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
+    flatbuf::Type* out_type, Offset* offset) {
+  std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
+  for (const BufferDescr& descr : buffer_layout) {
+    flatbuf::VectorType vector_type;
+    switch (descr.type()) {
+      case BufferType::OFFSET:
+        vector_type = flatbuf::VectorType_OFFSET;
+        break;
+      case BufferType::DATA:
+        vector_type = flatbuf::VectorType_DATA;
+        break;
+      case BufferType::VALIDITY:
+        vector_type = flatbuf::VectorType_VALIDITY;
+        break;
+      case BufferType::TYPE:
+        vector_type = flatbuf::VectorType_TYPE;
+        break;
+      default:
+        vector_type = flatbuf::VectorType_DATA;
+        break;
+    }
+    auto offset = flatbuf::CreateVectorLayout(fbb, descr.bit_width(), vector_type);
+    layout->push_back(offset);
+  }
+
   switch (type->type) {
     case Type::BOOL:
       *out_type = flatbuf::Type_Bool;
@@ -223,14 +236,18 @@ static Status FieldToFlatbuffer(
 
   flatbuf::Type type_enum;
   Offset type_data;
+  Offset type_layout;
   std::vector<FieldOffset> children;
+  std::vector<VectorLayoutOffset> layout;
 
-  RETURN_NOT_OK(TypeToFlatbuffer(fbb, field->type, &children, &type_enum, &type_data));
+  RETURN_NOT_OK(
+      TypeToFlatbuffer(fbb, field->type, &children, &layout, &type_enum, &type_data));
   auto fb_children = fbb.CreateVector(children);
+  auto fb_layout = fbb.CreateVector(layout);
 
   // TODO: produce the list of VectorTypes
   *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_data,
-      field->dictionary, fb_children);
+      field->dictionary, fb_children, fb_layout);
 
   return Status::OK();
 }
@@ -300,13 +317,26 @@ Status MessageBuilder::SetRecordBatch(int32_t length, int64_t body_length,
   return Status::OK();
 }
 
-Status WriteDataHeader(int32_t length, int64_t body_length,
+Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
     const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
-  MessageBuilder message;
-  RETURN_NOT_OK(message.SetRecordBatch(length, body_length, nodes, buffers));
-  RETURN_NOT_OK(message.Finish());
-  return message.GetBuffer(out);
+  flatbuffers::FlatBufferBuilder fbb;
+
+  auto batch = flatbuf::CreateRecordBatch(
+      fbb, length, fbb.CreateVectorOfStructs(nodes), fbb.CreateVectorOfStructs(buffers));
+
+  fbb.Finish(batch);
+
+  int32_t size = fbb.GetSize();
+
+  auto result = std::make_shared<PoolBuffer>();
+  RETURN_NOT_OK(result->Resize(size));
+
+  uint8_t* dst = result->mutable_data();
+  memcpy(dst, fbb.GetBufferPointer(), size);
+
+  *out = result;
+  return Status::OK();
 }
 
 Status MessageBuilder::Finish() {
@@ -317,17 +347,13 @@ Status MessageBuilder::Finish() {
 }
 
 Status MessageBuilder::GetBuffer(std::shared_ptr<Buffer>* out) {
-  // The message buffer is suffixed by the size of the complete flatbuffer as
-  // int32_t
-  // <uint8_t*: flatbuffer data><int32_t: flatbuffer size>
   int32_t size = fbb_.GetSize();
 
   auto result = std::make_shared<PoolBuffer>();
-  RETURN_NOT_OK(result->Resize(size + sizeof(int32_t)));
+  RETURN_NOT_OK(result->Resize(size));
 
   uint8_t* dst = result->mutable_data();
   memcpy(dst, fbb_.GetBufferPointer(), size);
-  memcpy(dst + size, reinterpret_cast<int32_t*>(&size), sizeof(int32_t));
 
   *out = result;
   return Status::OK();
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index c404cfde22ca3..4826ebe22899d 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -41,10 +41,10 @@ namespace ipc {
 
 using FBB = flatbuffers::FlatBufferBuilder;
 using FieldOffset = flatbuffers::Offset<arrow::flatbuf::Field>;
+using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
 
-static constexpr flatbuf::MetadataVersion kMetadataVersion =
-    flatbuf::MetadataVersion_V1_SNAPSHOT;
+static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
 
 Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out);
 
@@ -70,7 +70,7 @@ class MessageBuilder {
   flatbuffers::FlatBufferBuilder fbb_;
 };
 
-Status WriteDataHeader(int32_t length, int64_t body_length,
+Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
     const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 66df8a6711fa9..44d3939c04f1d 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -50,9 +50,15 @@ Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out) {
 
 class Message::MessageImpl {
  public:
-  explicit MessageImpl(
-      const std::shared_ptr<Buffer>& buffer, const flatbuf::Message* message)
-      : buffer_(buffer), message_(message) {}
+  explicit MessageImpl(const std::shared_ptr<Buffer>& buffer, int64_t offset)
+      : buffer_(buffer), offset_(offset), message_(nullptr) {}
+
+  Status Open() {
+    message_ = flatbuf::GetMessage(buffer_->data() + offset_);
+
+    // TODO(wesm): verify the message
+    return Status::OK();
+  }
 
   Message::Type type() const {
     switch (message_->header_type()) {
@@ -72,25 +78,23 @@ class Message::MessageImpl {
   int64_t body_length() const { return message_->bodyLength(); }
 
  private:
-  // Owns the memory this message accesses
+  // Retain reference to memory
   std::shared_ptr<Buffer> buffer_;
+  int64_t offset_;
 
   const flatbuf::Message* message_;
 };
 
-Message::Message() {}
-
-Status Message::Open(
-    const std::shared_ptr<Buffer>& buffer, std::shared_ptr<Message>* out) {
-  std::shared_ptr<Message> result(new Message());
-
-  const flatbuf::Message* message = flatbuf::GetMessage(buffer->data());
+Message::Message(const std::shared_ptr<Buffer>& buffer, int64_t offset) {
+  impl_.reset(new MessageImpl(buffer, offset));
+}
 
-  // TODO(wesm): verify message
-  result->impl_.reset(new MessageImpl(buffer, message));
-  *out = result;
+Status Message::Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
+    std::shared_ptr<Message>* out) {
+  // ctor is private
 
-  return Status::OK();
+  *out = std::shared_ptr<Message>(new Message(buffer, offset));
+  return (*out)->impl_->Open();
 }
 
 Message::Type Message::type() const {
@@ -101,20 +105,12 @@ int64_t Message::body_length() const {
   return impl_->body_length();
 }
 
-std::shared_ptr<Message> Message::get_shared_ptr() {
-  return this->shared_from_this();
-}
-
-std::shared_ptr<SchemaMessage> Message::GetSchema() {
-  return std::make_shared<SchemaMessage>(this->shared_from_this(), impl_->header());
-}
-
 // ----------------------------------------------------------------------
-// SchemaMessage
+// SchemaMetadata
 
-class SchemaMessage::SchemaMessageImpl {
+class SchemaMetadata::SchemaMetadataImpl {
  public:
-  explicit SchemaMessageImpl(const void* schema)
+  explicit SchemaMetadataImpl(const void* schema)
       : schema_(static_cast<const flatbuf::Schema*>(schema)) {}
 
   const flatbuf::Field* field(int i) const { return schema_->fields()->Get(i); }
@@ -125,22 +121,29 @@ class SchemaMessage::SchemaMessageImpl {
   const flatbuf::Schema* schema_;
 };
 
-SchemaMessage::SchemaMessage(
-    const std::shared_ptr<Message>& message, const void* schema) {
+SchemaMetadata::SchemaMetadata(
+    const std::shared_ptr<Message>& message, const void* flatbuf) {
+  message_ = message;
+  impl_.reset(new SchemaMetadataImpl(flatbuf));
+}
+
+SchemaMetadata::SchemaMetadata(const std::shared_ptr<Message>& message) {
   message_ = message;
-  impl_.reset(new SchemaMessageImpl(schema));
+  impl_.reset(new SchemaMetadataImpl(message->impl_->header()));
 }
 
-int SchemaMessage::num_fields() const {
+SchemaMetadata::~SchemaMetadata() {}
+
+int SchemaMetadata::num_fields() const {
   return impl_->num_fields();
 }
 
-Status SchemaMessage::GetField(int i, std::shared_ptr<Field>* out) const {
+Status SchemaMetadata::GetField(int i, std::shared_ptr<Field>* out) const {
   const flatbuf::Field* field = impl_->field(i);
   return FieldFromFlatbuffer(field, out);
 }
 
-Status SchemaMessage::GetSchema(std::shared_ptr<Schema>* out) const {
+Status SchemaMetadata::GetSchema(std::shared_ptr<Schema>* out) const {
   std::vector<std::shared_ptr<Field>> fields(num_fields());
   for (int i = 0; i < this->num_fields(); ++i) {
     RETURN_NOT_OK(GetField(i, &fields[i]));
@@ -150,11 +153,11 @@ Status SchemaMessage::GetSchema(std::shared_ptr<Schema>* out) const {
 }
 
 // ----------------------------------------------------------------------
-// RecordBatchMessage
+// RecordBatchMetadata
 
-class RecordBatchMessage::RecordBatchMessageImpl {
+class RecordBatchMetadata::RecordBatchMetadataImpl {
  public:
-  explicit RecordBatchMessageImpl(const void* batch)
+  explicit RecordBatchMetadataImpl(const void* batch)
       : batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
     nodes_ = batch_->nodes();
     buffers_ = batch_->buffers();
@@ -176,19 +179,29 @@ class RecordBatchMessage::RecordBatchMessageImpl {
   const flatbuffers::Vector<const flatbuf::Buffer*>* buffers_;
 };
 
-std::shared_ptr<RecordBatchMessage> Message::GetRecordBatch() {
-  return std::make_shared<RecordBatchMessage>(this->shared_from_this(), impl_->header());
+RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message) {
+  message_ = message;
+  impl_.reset(new RecordBatchMetadataImpl(message->impl_->header()));
 }
 
-RecordBatchMessage::RecordBatchMessage(
-    const std::shared_ptr<Message>& message, const void* batch) {
-  message_ = message;
-  impl_.reset(new RecordBatchMessageImpl(batch));
+RecordBatchMetadata::RecordBatchMetadata(
+    const std::shared_ptr<Buffer>& buffer, int64_t offset) {
+  message_ = nullptr;
+  buffer_ = buffer;
+
+  const flatbuf::RecordBatch* metadata =
+      flatbuffers::GetRoot<flatbuf::RecordBatch>(buffer->data() + offset);
+
+  // TODO(wesm): validate table
+
+  impl_.reset(new RecordBatchMetadataImpl(metadata));
 }
 
+RecordBatchMetadata::~RecordBatchMetadata() {}
+
 // TODO(wesm): Copying the flatbuffer data isn't great, but this will do for
 // now
-FieldMetadata RecordBatchMessage::field(int i) const {
+FieldMetadata RecordBatchMetadata::field(int i) const {
   const flatbuf::FieldNode* node = impl_->field(i);
 
   FieldMetadata result;
@@ -197,7 +210,7 @@ FieldMetadata RecordBatchMessage::field(int i) const {
   return result;
 }
 
-BufferMetadata RecordBatchMessage::buffer(int i) const {
+BufferMetadata RecordBatchMetadata::buffer(int i) const {
   const flatbuf::Buffer* buffer = impl_->buffer(i);
 
   BufferMetadata result;
@@ -207,15 +220,15 @@ BufferMetadata RecordBatchMessage::buffer(int i) const {
   return result;
 }
 
-int32_t RecordBatchMessage::length() const {
+int32_t RecordBatchMetadata::length() const {
   return impl_->length();
 }
 
-int RecordBatchMessage::num_buffers() const {
+int RecordBatchMetadata::num_buffers() const {
   return impl_->num_buffers();
 }
 
-int RecordBatchMessage::num_fields() const {
+int RecordBatchMetadata::num_fields() const {
   return impl_->num_fields();
 }
 
@@ -268,11 +281,13 @@ class FileFooter::FileFooterImpl {
 
   MetadataVersion::type version() const {
     switch (footer_->version()) {
-      case flatbuf::MetadataVersion_V1_SNAPSHOT:
-        return MetadataVersion::V1_SNAPSHOT;
+      case flatbuf::MetadataVersion_V1:
+        return MetadataVersion::V1;
+      case flatbuf::MetadataVersion_V2:
+        return MetadataVersion::V2;
       // Add cases as other versions become available
       default:
-        return MetadataVersion::V1_SNAPSHOT;
+        return MetadataVersion::V2;
     }
   }
 
@@ -285,7 +300,7 @@ class FileFooter::FileFooterImpl {
   }
 
   Status GetSchema(std::shared_ptr<Schema>* out) const {
-    auto schema_msg = std::make_shared<SchemaMessage>(nullptr, footer_->schema());
+    auto schema_msg = std::make_shared<SchemaMetadata>(nullptr, footer_->schema());
     return schema_msg->GetSchema(out);
   }
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 2f0e853bf97f0..1c4ef64d62fad 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -42,7 +42,7 @@ class OutputStream;
 namespace ipc {
 
 struct MetadataVersion {
-  enum type { V1_SNAPSHOT };
+  enum type { V1, V2 };
 };
 
 //----------------------------------------------------------------------
@@ -58,10 +58,14 @@ Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out);
 class Message;
 
 // Container for serialized Schema metadata contained in an IPC message
-class ARROW_EXPORT SchemaMessage {
+class ARROW_EXPORT SchemaMetadata {
  public:
+  explicit SchemaMetadata(const std::shared_ptr<Message>& message);
+
   // Accepts an opaque flatbuffer pointer
-  SchemaMessage(const std::shared_ptr<Message>& message, const void* schema);
+  SchemaMetadata(const std::shared_ptr<Message>& message, const void* schema);
+
+  ~SchemaMetadata();
 
   int num_fields() const;
 
@@ -76,8 +80,8 @@ class ARROW_EXPORT SchemaMessage {
   // Parent, owns the flatbuffer data
   std::shared_ptr<Message> message_;
 
-  class SchemaMessageImpl;
-  std::unique_ptr<SchemaMessageImpl> impl_;
+  class SchemaMetadataImpl;
+  std::unique_ptr<SchemaMetadataImpl> impl_;
 };
 
 // Field metadata
@@ -93,10 +97,13 @@ struct BufferMetadata {
 };
 
 // Container for serialized record batch metadata contained in an IPC message
-class ARROW_EXPORT RecordBatchMessage {
+class ARROW_EXPORT RecordBatchMetadata {
  public:
-  // Accepts an opaque flatbuffer pointer
-  RecordBatchMessage(const std::shared_ptr<Message>& message, const void* batch_meta);
+  explicit RecordBatchMetadata(const std::shared_ptr<Message>& message);
+
+  RecordBatchMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
+
+  ~RecordBatchMetadata();
 
   FieldMetadata field(int i) const;
   BufferMetadata buffer(int i) const;
@@ -108,37 +115,34 @@ class ARROW_EXPORT RecordBatchMessage {
  private:
   // Parent, owns the flatbuffer data
   std::shared_ptr<Message> message_;
+  std::shared_ptr<Buffer> buffer_;
 
-  class RecordBatchMessageImpl;
-  std::unique_ptr<RecordBatchMessageImpl> impl_;
+  class RecordBatchMetadataImpl;
+  std::unique_ptr<RecordBatchMetadataImpl> impl_;
 };
 
-class ARROW_EXPORT DictionaryBatchMessage {
+class ARROW_EXPORT DictionaryBatchMetadata {
  public:
   int64_t id() const;
-  std::unique_ptr<RecordBatchMessage> data() const;
+  std::unique_ptr<RecordBatchMetadata> data() const;
 };
 
-class ARROW_EXPORT Message : public std::enable_shared_from_this<Message> {
+class ARROW_EXPORT Message {
  public:
   enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH };
 
-  static Status Open(
-      const std::shared_ptr<Buffer>& buffer, std::shared_ptr<Message>* out);
-
-  std::shared_ptr<Message> get_shared_ptr();
+  static Status Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
+      std::shared_ptr<Message>* out);
 
   int64_t body_length() const;
 
   Type type() const;
 
-  // These methods only to be invoked if you have checked the message type
-  std::shared_ptr<SchemaMessage> GetSchema();
-  std::shared_ptr<RecordBatchMessage> GetRecordBatch();
-  std::shared_ptr<DictionaryBatchMessage> GetDictionaryBatch();
-
  private:
-  Message();
+  Message(const std::shared_ptr<Buffer>& buffer, int64_t offset);
+
+  friend class RecordBatchMetadata;
+  friend class SchemaMetadata;
 
   // Hide serialization details from user API
   class MessageImpl;
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 9abc20d876de4..65b378215222d 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -39,8 +39,7 @@
 namespace arrow {
 namespace ipc {
 
-const auto kInt32 = std::make_shared<Int32Type>();
-const auto kListInt32 = list(kInt32);
+const auto kListInt32 = list(int32());
 const auto kListListInt32 = list(kListInt32);
 
 Status MakeRandomInt32Array(
@@ -99,8 +98,8 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
   const int length = 1000;
 
   // Make the schema
-  auto f0 = std::make_shared<Field>("f0", kInt32);
-  auto f1 = std::make_shared<Field>("f1", kInt32);
+  auto f0 = std::make_shared<Field>("f0", int32());
+  auto f1 = std::make_shared<Field>("f1", int32());
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
   // Example data
@@ -161,7 +160,7 @@ Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out) {
   // Make the schema
   auto f0 = std::make_shared<Field>("f0", kListInt32);
   auto f1 = std::make_shared<Field>("f1", kListListInt32);
-  auto f2 = std::make_shared<Field>("f2", kInt32);
+  auto f2 = std::make_shared<Field>("f2", int32());
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
   // Example data
@@ -184,7 +183,7 @@ Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out) {
   // Make the schema
   auto f0 = std::make_shared<Field>("f0", kListInt32);
   auto f1 = std::make_shared<Field>("f1", kListListInt32);
-  auto f2 = std::make_shared<Field>("f2", kInt32);
+  auto f2 = std::make_shared<Field>("f2", int32());
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
   // Example data
@@ -205,7 +204,7 @@ Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
   // Make the schema
   auto f0 = std::make_shared<Field>("f0", kListInt32);
   auto f1 = std::make_shared<Field>("f1", kListListInt32);
-  auto f2 = std::make_shared<Field>("f2", kInt32);
+  auto f2 = std::make_shared<Field>("f2", int32());
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
   // Example data
@@ -226,7 +225,7 @@ Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
 Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
   const int batch_length = 5;
-  TypePtr type = kInt32;
+  TypePtr type = int32();
 
   MemoryPool* pool = default_memory_pool();
   ArrayPtr array;
diff --git a/cpp/src/arrow/ipc/util.h b/cpp/src/arrow/ipc/util.h
index 9000d1bb0c6c3..242d6624f1e7f 100644
--- a/cpp/src/arrow/ipc/util.h
+++ b/cpp/src/arrow/ipc/util.h
@@ -28,12 +28,10 @@ namespace arrow {
 namespace ipc {
 
 // Align on 8-byte boundaries
-static constexpr int kArrowAlignment = 8;
-
 // Buffers are padded to 64-byte boundaries (for SIMD)
-static constexpr int kArrowBufferAlignment = 64;
+static constexpr int kArrowAlignment = 64;
 
-static constexpr uint8_t kPaddingBytes[kArrowBufferAlignment] = {0};
+static constexpr uint8_t kPaddingBytes[kArrowAlignment] = {0};
 
 static inline int64_t PaddedLength(int64_t nbytes, int64_t alignment = kArrowAlignment) {
   return ((nbytes + alignment - 1) / alignment) * alignment;
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 93dd5b69b1bb7..63c2166a5736b 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -61,10 +61,10 @@
 
 // Alias MSVC popcount to GCC name
 #ifdef _MSC_VER
-#  include <intrin.h>
-#  define __builtin_popcount __popcnt
-#  include <nmmintrin.h>
-#  define __builtin_popcountll _mm_popcnt_u64
+#include <intrin.h>
+#define __builtin_popcount __popcnt
+#include <nmmintrin.h>
+#define __builtin_popcountll _mm_popcnt_u64
 #endif
 
 namespace arrow {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 589bdadb77c64..80f295c487f13 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -105,10 +105,6 @@ std::string UnionType::ToString() const {
   return s.str();
 }
 
-int NullType::bit_width() const {
-  return 0;
-}
-
 std::string NullType::ToString() const {
   return name();
 }
@@ -187,4 +183,46 @@ std::shared_ptr<Field> field(
   return std::make_shared<Field>(name, type, nullable, dictionary);
 }
 
+static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1);
+static const BufferDescr kOffsetBuffer(BufferType::OFFSET, 32);
+static const BufferDescr kTypeBuffer(BufferType::TYPE, 32);
+static const BufferDescr kBooleanBuffer(BufferType::DATA, 1);
+static const BufferDescr kValues64(BufferType::DATA, 64);
+static const BufferDescr kValues32(BufferType::DATA, 32);
+static const BufferDescr kValues16(BufferType::DATA, 16);
+static const BufferDescr kValues8(BufferType::DATA, 8);
+
+std::vector<BufferDescr> FixedWidthType::GetBufferLayout() const {
+  return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
+}
+
+std::vector<BufferDescr> NullType::GetBufferLayout() const {
+  return {};
+}
+
+std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
+  return {kValidityBuffer, kOffsetBuffer, kValues8};
+}
+
+std::vector<BufferDescr> ListType::GetBufferLayout() const {
+  return {kValidityBuffer, kOffsetBuffer};
+}
+
+std::vector<BufferDescr> StructType::GetBufferLayout() const {
+  return {kValidityBuffer, kTypeBuffer};
+}
+
+std::vector<BufferDescr> UnionType::GetBufferLayout() const {
+  if (mode == UnionMode::SPARSE) {
+    return {kValidityBuffer, kTypeBuffer};
+  } else {
+    return {kValidityBuffer, kTypeBuffer, kOffsetBuffer};
+  }
+}
+
+std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
+  // TODO(wesm)
+  return {};
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 876d7ea464b1c..30777384dfb9f 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -101,6 +101,20 @@ struct Type {
   };
 };
 
+enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
+
+class BufferDescr {
+ public:
+  BufferDescr(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
+
+  BufferType type() const { return type_; }
+  int bit_width() const { return bit_width_; }
+
+ private:
+  BufferType type_;
+  int bit_width_;
+};
+
 struct ARROW_EXPORT DataType {
   Type::type type;
 
@@ -129,12 +143,18 @@ struct ARROW_EXPORT DataType {
   virtual Status Accept(TypeVisitor* visitor) const = 0;
 
   virtual std::string ToString() const = 0;
+
+  virtual std::vector<BufferDescr> GetBufferLayout() const = 0;
 };
 
 typedef std::shared_ptr<DataType> TypePtr;
 
-struct ARROW_EXPORT FixedWidthMeta {
+struct ARROW_EXPORT FixedWidthType : public DataType {
+  using DataType::DataType;
+
   virtual int bit_width() const = 0;
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 struct ARROW_EXPORT IntegerMeta {
@@ -184,12 +204,12 @@ struct ARROW_EXPORT Field {
 };
 typedef std::shared_ptr<Field> FieldPtr;
 
-struct ARROW_EXPORT PrimitiveCType : public DataType {
-  using DataType::DataType;
+struct ARROW_EXPORT PrimitiveCType : public FixedWidthType {
+  using FixedWidthType::FixedWidthType;
 };
 
 template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
-struct ARROW_EXPORT CTypeImpl : public PrimitiveCType, public FixedWidthMeta {
+struct ARROW_EXPORT CTypeImpl : public PrimitiveCType {
   using c_type = C_TYPE;
   static constexpr Type::type type_id = TYPE_ID;
 
@@ -204,16 +224,17 @@ struct ARROW_EXPORT CTypeImpl : public PrimitiveCType, public FixedWidthMeta {
   std::string ToString() const override { return std::string(DERIVED::name()); }
 };
 
-struct ARROW_EXPORT NullType : public DataType, public FixedWidthMeta {
+struct ARROW_EXPORT NullType : public DataType {
   static constexpr Type::type type_id = Type::NA;
 
   NullType() : DataType(Type::NA) {}
 
-  int bit_width() const override;
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
 
   static std::string name() { return "null"; }
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
@@ -221,10 +242,10 @@ struct IntegerTypeImpl : public CTypeImpl<DERIVED, TYPE_ID, C_TYPE>, public Inte
   bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
 };
 
-struct ARROW_EXPORT BooleanType : public DataType, FixedWidthMeta {
+struct ARROW_EXPORT BooleanType : public FixedWidthType {
   static constexpr Type::type type_id = Type::BOOL;
 
-  BooleanType() : DataType(Type::BOOL) {}
+  BooleanType() : FixedWidthType(Type::BOOL) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -306,6 +327,8 @@ struct ARROW_EXPORT ListType : public DataType, public NoExtraMeta {
   std::string ToString() const override;
 
   static std::string name() { return "list"; }
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 // BinaryType type is reprsents lists of 1-byte values.
@@ -318,6 +341,8 @@ struct ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
   std::string ToString() const override;
   static std::string name() { return "binary"; }
 
+  std::vector<BufferDescr> GetBufferLayout() const override;
+
  protected:
   // Allow subclasses to change the logical type.
   explicit BinaryType(Type::type logical_type) : DataType(logical_type) {}
@@ -345,6 +370,8 @@ struct ARROW_EXPORT StructType : public DataType, public NoExtraMeta {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "struct"; }
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 struct ARROW_EXPORT DecimalType : public DataType {
@@ -358,6 +385,8 @@ struct ARROW_EXPORT DecimalType : public DataType {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "decimal"; }
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 enum class UnionMode : char { SPARSE, DENSE };
@@ -375,14 +404,20 @@ struct ARROW_EXPORT UnionType : public DataType {
   static std::string name() { return "union"; }
   Status Accept(TypeVisitor* visitor) const override;
 
+  std::vector<BufferDescr> GetBufferLayout() const override;
+
   UnionMode mode;
   std::vector<uint8_t> type_ids;
 };
 
-struct ARROW_EXPORT DateType : public DataType, public NoExtraMeta {
+struct ARROW_EXPORT DateType : public FixedWidthType {
   static constexpr Type::type type_id = Type::DATE;
 
-  DateType() : DataType(Type::DATE) {}
+  using c_type = int32_t;
+
+  DateType() : FixedWidthType(Type::DATE) {}
+
+  int bit_width() const override { return sizeof(c_type) * 8; }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override { return name(); }
@@ -391,13 +426,17 @@ struct ARROW_EXPORT DateType : public DataType, public NoExtraMeta {
 
 enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
 
-struct ARROW_EXPORT TimeType : public DataType {
+struct ARROW_EXPORT TimeType : public FixedWidthType {
   static constexpr Type::type type_id = Type::TIME;
   using Unit = TimeUnit;
+  using c_type = int64_t;
 
   TimeUnit unit;
 
-  explicit TimeType(TimeUnit unit = TimeUnit::MILLI) : DataType(Type::TIME), unit(unit) {}
+  int bit_width() const override { return sizeof(c_type) * 8; }
+
+  explicit TimeType(TimeUnit unit = TimeUnit::MILLI)
+      : FixedWidthType(Type::TIME), unit(unit) {}
   TimeType(const TimeType& other) : TimeType(other.unit) {}
 
   Status Accept(TypeVisitor* visitor) const override;
@@ -405,7 +444,7 @@ struct ARROW_EXPORT TimeType : public DataType {
   static std::string name() { return "time"; }
 };
 
-struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
+struct ARROW_EXPORT TimestampType : public FixedWidthType {
   using Unit = TimeUnit;
 
   typedef int64_t c_type;
@@ -416,7 +455,7 @@ struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
   TimeUnit unit;
 
   explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
-      : DataType(Type::TIMESTAMP), unit(unit) {}
+      : FixedWidthType(Type::TIMESTAMP), unit(unit) {}
 
   TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
 
@@ -425,10 +464,10 @@ struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
   static std::string name() { return "timestamp"; }
 };
 
-struct ARROW_EXPORT IntervalType : public DataType, public FixedWidthMeta {
+struct ARROW_EXPORT IntervalType : public FixedWidthType {
   enum class Unit : char { YEAR_MONTH = 0, DAY_TIME = 1 };
 
-  typedef int64_t c_type;
+  using c_type = int64_t;
   static constexpr Type::type type_id = Type::INTERVAL;
 
   int bit_width() const override { return sizeof(int64_t) * 8; }
@@ -436,7 +475,7 @@ struct ARROW_EXPORT IntervalType : public DataType, public FixedWidthMeta {
   Unit unit;
 
   explicit IntervalType(Unit unit = Unit::YEAR_MONTH)
-      : DataType(Type::INTERVAL), unit(unit) {}
+      : FixedWidthType(Type::INTERVAL), unit(unit) {}
 
   IntervalType(const IntervalType& other) : IntervalType(other.unit) {}
 
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index 14667ee5b6eac..f42a3cac021cd 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -49,7 +49,7 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
     const uint8_t* this_data = raw_data_;
     const uint8_t* other_data = other.raw_data_;
 
-    auto size_meta = dynamic_cast<const FixedWidthMeta*>(type_.get());
+    auto size_meta = dynamic_cast<const FixedWidthType*>(type_.get());
     int value_byte_size = size_meta->bit_width() / 8;
     DCHECK_GT(value_byte_size, 0);
 
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 13b7e19593d93..5c8055f9c6171 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -78,6 +78,10 @@ static inline bool IsMultipleOf64(int64_t n) {
   return (n & 63) == 0;
 }
 
+static inline bool IsMultipleOf8(int64_t n) {
+  return (n & 7) == 0;
+}
+
 inline int64_t RoundUpToMultipleOf64(int64_t num) {
   // TODO(wesm): is this definitely needed?
   // DCHECK_GE(num, 0);
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index d8ec6507fc4e5..e26dd589695b1 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -28,6 +28,7 @@ $RAT $1 \
   -e ".*" \
   -e mman.h \
   -e "*_generated.h" \
+  -e "*.json" \
   -e random.h \
   -e status.cc \
   -e status.h \
@@ -49,5 +50,3 @@ else
   echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
   exit 1
 fi
-
-
diff --git a/format/IPC.md b/format/IPC.md
index 3f78126ef55d3..a55dcdff48117 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -15,3 +15,109 @@
 # Interprocess messaging / communication (IPC)
 
 ## File format
+
+We define a self-contained "file format" containing an Arrow schema along with
+one or more record batches defining a dataset. See [format/File.fbs][1] for the
+precise details of the file metadata.
+
+In general, the file looks like:
+
+```
+<magic number "ARROW1">
+<empty padding bytes [to 64 byte boundary]>
+<DICTIONARY 0>
+...
+<DICTIONARY k - 1>
+<RECORD BATCH 0>
+...
+<RECORD BATCH n - 1>
+<METADATA org.apache.arrow.flatbuf.Footer>
+<metadata_size: int32>
+<magic number "ARROW1">
+```
+
+See the File.fbs document for details about the Flatbuffers metadata. The
+record batches have a particular structure, defined next.
+
+### Record batches
+
+The record batch metadata is written as a flatbuffer (see
+[format/Message.fbs][2] -- the RecordBatch message type) prefixed by its size,
+followed by each of the memory buffers in the batch written end to end (with
+appropriate alignment and padding):
+
+```
+<int32: metadata flatbuffer size>
+<metadata: org.apache.arrow.flatbuf.RecordBatch>
+<padding bytes [to 64-byte boundary]>
+<body: buffers end to end>
+```
+
+The `RecordBatch` metadata contains a depth-first (pre-order) flattened set of
+field metadata and physical memory buffers (some comments from [Message.fbs][2]
+have been shortened / removed):
+
+```
+table RecordBatch {
+  length: int;
+  nodes: [FieldNode];
+  buffers: [Buffer];
+}
+
+struct FieldNode {
+  /// The number of value slots in the Arrow array at this level of a nested
+  /// tree
+  length: int;
+
+  /// The number of observed nulls. Fields with null_count == 0 may choose not
+  /// to write their physical validity bitmap out as a materialized buffer,
+  /// instead setting the length of the bitmap buffer to 0.
+  null_count: int;
+}
+
+struct Buffer {
+  /// The shared memory page id where this buffer is located. Currently this is
+  /// not used
+  page: int;
+
+  /// The relative offset into the shared memory page where the bytes for this
+  /// buffer starts
+  offset: long;
+
+  /// The absolute length (in bytes) of the memory buffer. The memory is found
+  /// from offset (inclusive) to offset + length (non-inclusive).
+  length: long;
+}
+```
+
+In the context of a file, the `page` is not used, and the `Buffer` offsets use
+as a frame of reference the start of the segment where they are written in the
+file. So, while in a general IPC setting these offsets may be anyplace in one
+or more shared memory regions, in the file format the offsets start from 0.
+
+The location of a record batch and the size of the metadata block as well as
+the body of buffers is stored in the file footer:
+
+```
+struct Block {
+  offset: long;
+  metaDataLength: int;
+  bodyLength: long;
+}
+```
+
+Some notes about this
+
+* The `Block` offset indicates the starting byte of the record batch.
+* The metadata length includes the flatbuffer size, the record batch metadata
+  flatbuffer, and any padding bytes
+
+
+### Dictionary batches
+
+Dictionary batches have not yet been implemented, while they are provided for
+in the metadata. For the time being, the `DICTIONARY` segments shown above in
+the file do not appear in any of the file implementations.
+
+[1]: https://github.com/apache/arrow/blob/master/format/File.fbs
+[1]: https://github.com/apache/arrow/blob/master/format/Message.fbs
\ No newline at end of file
diff --git a/format/Message.fbs b/format/Message.fbs
index 2ec9fd1817bd5..d07d0666dce87 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -18,7 +18,8 @@
 namespace org.apache.arrow.flatbuf;
 
 enum MetadataVersion:short {
-  V1_SNAPSHOT
+  V1,
+  V2
 }
 
 /// ----------------------------------------------------------------------
diff --git a/integration/data/simple.json b/integration/data/simple.json
new file mode 100644
index 0000000000000..a91b405d4f0f0
--- /dev/null
+++ b/integration/data/simple.json
@@ -0,0 +1,66 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "foo",
+        "type": {"name": "int", "isSigned": true, "bitWidth": 32},
+        "nullable": true, "children": [],
+        "typeLayout": {
+          "vectors": [
+            {"type": "VALIDITY", "typeBitWidth": 1},
+            {"type": "DATA", "typeBitWidth": 32}
+          ]
+        }
+      },
+      {
+        "name": "bar",
+        "type": {"name": "floatingpoint", "precision": "DOUBLE"},
+        "nullable": true, "children": [],
+        "typeLayout": {
+          "vectors": [
+            {"type": "VALIDITY", "typeBitWidth": 1},
+            {"type": "DATA", "typeBitWidth": 64}
+          ]
+        }
+      },
+      {
+        "name": "baz",
+        "type": {"name": "utf8"},
+        "nullable": true, "children": [],
+        "typeLayout": {
+          "vectors": [
+            {"type": "VALIDITY", "typeBitWidth": 1},
+            {"type": "OFFSET", "typeBitWidth": 32},
+            {"type": "DATA", "typeBitWidth": 64}
+          ]
+        }
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 5,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 5,
+          "VALIDITY": [1, 0, 1, 1, 1],
+          "DATA": [1, 2, 3, 4, 5]
+        },
+        {
+          "name": "bar",
+          "count": 5,
+          "VALIDITY": [1, 0, 0, 1, 1],
+          "DATA": [1.0, 2.0, 3.0, 4.0, 5.0]
+        },
+        {
+          "name": "baz",
+          "count": 5,
+          "VALIDITY": [1, 0, 0, 1, 1],
+          "OFFSET": [0, 2, 2, 2, 5, 9],
+          "DATA": ["aa", "", "", "bbb", "cccc"]
+        }
+      ]
+    }
+  ]
+}
diff --git a/integration/integration_test.py b/integration/integration_test.py
new file mode 100644
index 0000000000000..6ea634d779566
--- /dev/null
+++ b/integration/integration_test.py
@@ -0,0 +1,177 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import glob
+import itertools
+import os
+import six
+import subprocess
+import tempfile
+import uuid
+
+
+ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
+
+
+def guid():
+    return uuid.uuid4().hex
+
+
+def run_cmd(cmd):
+    if isinstance(cmd, six.string_types):
+        cmd = cmd.split(' ')
+
+    try:
+        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        # this avoids hiding the stdout / stderr of failed processes
+        print('Command failed: %s' % ' '.join(cmd))
+        print('With output:')
+        print('--------------')
+        print(e.output)
+        print('--------------')
+        raise e
+
+    if isinstance(output, six.binary_type):
+        output = output.decode('utf-8')
+    return output
+
+
+class IntegrationRunner(object):
+
+    def __init__(self, json_files, testers, debug=False):
+        self.json_files = json_files
+        self.testers = testers
+        self.temp_dir = tempfile.mkdtemp()
+        self.debug = debug
+
+    def run(self):
+        for producer, consumer in itertools.product(self.testers,
+                                                    self.testers):
+            if producer is consumer:
+                continue
+
+            print('-- {0} producing, {1} consuming'.format(producer.name,
+                                                           consumer.name))
+
+            for json_path in self.json_files:
+                print('Testing with {0}'.format(json_path))
+
+                arrow_path = os.path.join(self.temp_dir, guid())
+
+                producer.json_to_arrow(json_path, arrow_path)
+                consumer.validate(json_path, arrow_path)
+
+
+class Tester(object):
+
+    def __init__(self, debug=False):
+        self.debug = debug
+
+    def json_to_arrow(self, json_path, arrow_path):
+        raise NotImplementedError
+
+    def validate(self, json_path, arrow_path):
+        raise NotImplementedError
+
+
+class JavaTester(Tester):
+
+    ARROW_TOOLS_JAR = os.path.join(ARROW_HOME,
+                                   'java/tools/target/arrow-tools-0.1.1-'
+                                   'SNAPSHOT-jar-with-dependencies.jar')
+
+    name = 'Java'
+
+    def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
+        cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
+               'org.apache.arrow.tools.Integration']
+
+        if arrow_path is not None:
+            cmd.extend(['-a', arrow_path])
+
+        if json_path is not None:
+            cmd.extend(['-j', json_path])
+
+        cmd.extend(['-c', command])
+
+        if self.debug:
+            print(' '.join(cmd))
+
+        return run_cmd(cmd)
+
+    def validate(self, json_path, arrow_path):
+        return self._run(arrow_path, json_path, 'VALIDATE')
+
+    def json_to_arrow(self, json_path, arrow_path):
+        return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
+
+
+class CPPTester(Tester):
+
+    CPP_INTEGRATION_EXE = os.environ.get(
+        'ARROW_CPP_TESTER',
+        os.path.join(ARROW_HOME,
+                     'cpp/test-build/debug/json-integration-test'))
+
+    name = 'C++'
+
+    def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
+        cmd = [self.CPP_INTEGRATION_EXE, '--integration']
+
+        if arrow_path is not None:
+            cmd.append('--arrow=' + arrow_path)
+
+        if json_path is not None:
+            cmd.append('--json=' + json_path)
+
+        cmd.append('--mode=' + command)
+
+        if self.debug:
+            print(' '.join(cmd))
+
+        return run_cmd(cmd)
+
+    def validate(self, json_path, arrow_path):
+        return self._run(arrow_path, json_path, 'VALIDATE')
+
+    def json_to_arrow(self, json_path, arrow_path):
+        return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
+
+
+def get_json_files():
+    glob_pattern = os.path.join(ARROW_HOME, 'integration', 'data', '*.json')
+    return glob.glob(glob_pattern)
+
+
+def run_all_tests(debug=False):
+    testers = [JavaTester(debug=debug), CPPTester(debug=debug)]
+    json_files = get_json_files()
+
+    runner = IntegrationRunner(json_files, testers, debug=debug)
+    runner.run()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Arrow integration test CLI')
+    parser.add_argument('--debug', dest='debug', action='store_true',
+                        default=False,
+                        help='Run executables in debug mode as relevant')
+
+    args = parser.parse_args()
+    run_all_tests(debug=args.debug)
diff --git a/java/pom.xml b/java/pom.xml
index 7221a140d96ec..a147d66c98318 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -24,7 +24,7 @@
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
-  <description>Apache arrow is an open source, low latency SQL query engine for Hadoop and NoSQL.</description>
+  <description>Apache Arrow is open source, in-memory columnar data structures and low-overhead messaging</description>
   <url>http://arrow.apache.org/</url>
 
   <properties>
@@ -442,8 +442,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <!-- Mockito needs to be on the class path after JUnit (or Hamcrest) as 
-           long as Mockito _contains_ older Hamcrest classes.  See arrow-2130. --> 
+      <!-- Mockito needs to be on the class path after JUnit (or Hamcrest) as
+           long as Mockito _contains_ older Hamcrest classes.  See arrow-2130. -->
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
       <version>1.9.5</version>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 84b0b5eb4253c..ef96328f7668a 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -45,6 +45,12 @@
             <artifactId>commons-cli</artifactId>
             <version>1.2</version>
         </dependency>
+        <dependency>
+          <groupId>ch.qos.logback</groupId>
+          <artifactId>logback-classic</artifactId>
+          <version>1.0.13</version>
+          <scope>run</scope>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index 29f0ee29e3ca8..fa4bedca7a9bd 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -220,6 +220,7 @@ private Command toCommand(String commandName) {
 
   private static void fatalError(String message, Throwable e) {
     System.err.println(message);
+    System.err.println(e.getMessage());
     LOGGER.error(message, e);
     System.exit(1);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 4afd82315d9c3..c5d642ee0cc72 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -81,7 +81,9 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
     try {
       vector.loadFieldBuffers(fieldNode, ownBuffers);
     } catch (RuntimeException e) {
-      throw new IllegalArgumentException("Could not load buffers for field " + field, e);
+      e.printStackTrace();
+      throw new IllegalArgumentException("Could not load buffers for field " +
+              field + " error message" + e.getMessage(), e);
     }
     List<Field> children = field.getChildren();
     if (children.size() > 0) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index bbcd3e9f470e3..cd520da54f2f5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -123,7 +123,11 @@ public ArrowRecordBatch readRecordBatch(ArrowBlock recordBatchBlock) throws IOEx
     if (n != l) {
       throw new IllegalStateException(n + " != " + l);
     }
-    RecordBatch recordBatchFB = RecordBatch.getRootAsRecordBatch(buffer.nioBuffer().asReadOnlyBuffer());
+
+    // Record batch flatbuffer is prefixed by its size as int32le
+    final ArrowBuf metadata = buffer.slice(4, recordBatchBlock.getMetadataLength() - 4);
+    RecordBatch recordBatchFB = RecordBatch.getRootAsRecordBatch(metadata.nioBuffer().asReadOnlyBuffer());
+
     int nodesLength = recordBatchFB.nodesLength();
     final ArrowBuf body = buffer.slice(recordBatchBlock.getMetadataLength(), (int)recordBatchBlock.getBodyLength());
     List<ArrowFieldNode> nodes = new ArrayList<>();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 9881a229c23ea..1cd87ebc33594 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -99,9 +99,10 @@ private long writeIntLittleEndian(int v) throws IOException {
   public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
     checkStarted();
     align();
-    // write metadata header
+
+    // write metadata header with int32 size prefix
     long offset = currentPosition;
-    write(recordBatch);
+    write(recordBatch, true);
     align();
     // write body
     long bodyOffset = currentPosition;
@@ -117,6 +118,7 @@ public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
       if (startPosition != currentPosition) {
         writeZeros((int)(startPosition - currentPosition));
       }
+
       write(buffer);
       if (currentPosition != startPosition + layout.getSize()) {
         throw new IllegalStateException("wrong buffer size: " + currentPosition + " != " + startPosition + layout.getSize());
@@ -133,7 +135,9 @@ public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
   }
 
   private void write(ArrowBuf buffer) throws IOException {
-    write(buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes()));
+    ByteBuffer nioBuffer = buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes());
+    LOGGER.debug("Writing buffer with size: " + nioBuffer.remaining());
+    write(nioBuffer);
   }
 
   private void checkStarted() throws IOException {
@@ -166,14 +170,21 @@ private void writeMagic() throws IOException {
 
   private void writeFooter() throws IOException {
     // TODO: dictionaries
-    write(new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), recordBatches));
+    write(new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), recordBatches), false);
   }
 
-  private long write(FBSerializable writer) throws IOException {
+  private long write(FBSerializable writer, boolean withSizePrefix) throws IOException {
     FlatBufferBuilder builder = new FlatBufferBuilder();
     int root = writer.writeTo(builder);
     builder.finish(root);
-    return write(builder.dataBuffer());
+
+    ByteBuffer buffer = builder.dataBuffer();
+
+    if (withSizePrefix) {
+      writeIntLittleEndian(buffer.remaining());
+    }
+
+    return write(buffer);
   }
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index f07b517250732..f2059820d23d6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -127,8 +127,13 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         ValueVector valueVector = (ValueVector)innerVector;
         valueVector.allocateNew();
         Mutator mutator = valueVector.getMutator();
-        mutator.setValueCount(count);
-        for (int i = 0; i < count; i++) {
+
+        int innerVectorCount = count;
+        if (vectorType.getName() == "OFFSET") {
+          innerVectorCount++;
+        }
+        mutator.setValueCount(innerVectorCount);
+        for (int i = 0; i < innerVectorCount; i++) {
           parser.nextToken();
           setValueFromParser(valueVector, i);
         }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index 812b3da32f83c..6ff357774486d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -114,7 +114,7 @@ private void writeVector(Field field, FieldVector vector) throws IOException {
         BufferBacked innerVector = fieldInnerVectors.get(v);
         generator.writeArrayFieldStart(vectorType.getName());
         ValueVector valueVector = (ValueVector)innerVector;
-        for (int i = 0; i < valueCount; i++) {
+        for (int i = 0; i < valueVector.getAccessor().getValueCount(); i++) {
           writeValueToGenerator(valueVector, i);
         }
         generator.writeEndArray();
diff --git a/python/.gitignore b/python/.gitignore
index 07f28355a252f..c37efc4b56650 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -12,16 +12,6 @@ Testing/
 # Editor temporary/working/backup files
 *flymake*
 
-# Compiled source
-*.a
-*.dll
-*.o
-*.py[ocd]
-*.so
-*.dylib
-.build_cache_dir
-MANIFEST
-
 # Generated sources
 *.c
 *.cpp

From 65b74b350209ee3f930a00a0270e1d7c3d485c93 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 29 Nov 2016 22:23:19 -0500
Subject: [PATCH 0207/1644] ARROW-393: [JAVA] JSON file reader fails to set the
 buffer size on String data vector

Fixed by calling setValueCount after setting the values instead of before.
Since we set the inner vectors of NullableVarCharVector directly we don't have to worry about it's lastSet field and the way null values are handled.

Author: Julien Le Dem <julien@dremio.com>

Closes #218 from julienledem/json_read_varchar and squashes the following commits:

e147906 [Julien Le Dem] ARROW-393: [JAVA] JSON file reader fails to set the buffer size on String data vector
---
 .../org/apache/arrow/tools/Integration.java   |  2 +-
 .../apache/arrow/tools/TestIntegration.java   | 54 ++++++++++++++++++-
 .../vector/file/json/JsonFileReader.java      |  8 ++-
 .../arrow/vector/schema/ArrowVectorType.java  | 15 ++++++
 4 files changed, 72 insertions(+), 7 deletions(-)

diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index fa4bedca7a9bd..85af30da1e8ae 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -80,7 +80,7 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
           Schema schema = footer.getSchema();
           LOGGER.debug("Input file size: " + arrowFile.length());
           LOGGER.debug("Found schema: " + schema);
-          try (JsonFileWriter writer = new JsonFileWriter(jsonFile);) {
+          try (JsonFileWriter writer = new JsonFileWriter(jsonFile, JsonFileWriter.config().pretty(true));) {
             writer.start(schema);
             List<ArrowBlock> recordBatches = footer.getRecordBatches();
             for (ArrowBlock rbBlock : recordBatches) {
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
index bb69ed1498e26..464144b95a1aa 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -24,9 +24,12 @@
 import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
 import static org.junit.Assert.fail;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.StringReader;
+import java.util.Map;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -44,6 +47,11 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
 public class TestIntegration {
 
   @Rule
@@ -69,7 +77,7 @@ public void testValid() throws Exception {
     File testOutFile = testFolder.newFile("testOut.arrow");
     testOutFile.delete();
 
-    // generate an arow file
+    // generate an arrow file
     writeInput(testInFile, allocator);
 
     Integration integration = new Integration();
@@ -90,6 +98,50 @@ public void testValid() throws Exception {
     integration.run(args3);
   }
 
+  @Test
+  public void testJSONRoundTripWithVariableWidth() throws Exception {
+    File testJSONFile = new File("../../integration/data/simple.json");
+    File testOutFile = testFolder.newFile("testOut.arrow");
+    File testRoundTripJSONFile = testFolder.newFile("testOut.json");
+    testOutFile.delete();
+    testRoundTripJSONFile.delete();
+
+    Integration integration = new Integration();
+
+    // convert to arrow
+    String[] args1 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
+    integration.run(args1);
+
+    // convert back to json
+    String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testRoundTripJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    integration.run(args2);
+
+    BufferedReader orig = readNormalized(testJSONFile);
+    BufferedReader rt = readNormalized(testRoundTripJSONFile);
+    String i, o;
+    int j = 0;
+    while ((i = orig.readLine()) != null && (o = rt.readLine()) != null) {
+      Assert.assertEquals("line: " + j, i, o);
+      ++j;
+    }
+  }
+
+  private ObjectMapper om = new ObjectMapper();
+  {
+    DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
+    prettyPrinter.indentArraysWith(NopIndenter.instance);
+    om.setDefaultPrettyPrinter(prettyPrinter);
+    om.enable(SerializationFeature.INDENT_OUTPUT);
+    om.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
+  }
+
+  private BufferedReader readNormalized(File f) throws IOException {
+    Map<?,?> tree = om.readValue(f, Map.class);
+    String normalized = om.writeValueAsString(tree);
+    return new BufferedReader(new StringReader(normalized));
+  }
+
+
   @Test
   public void testInvalid() throws Exception {
     File testValidInFile = testFolder.newFile("testValidIn.arrow");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index f2059820d23d6..26dd3f6dfe5ae 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -22,6 +22,7 @@
 import static com.fasterxml.jackson.core.JsonToken.START_ARRAY;
 import static com.fasterxml.jackson.core.JsonToken.START_OBJECT;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET;
 
 import java.io.File;
 import java.io.IOException;
@@ -128,15 +129,12 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         valueVector.allocateNew();
         Mutator mutator = valueVector.getMutator();
 
-        int innerVectorCount = count;
-        if (vectorType.getName() == "OFFSET") {
-          innerVectorCount++;
-        }
-        mutator.setValueCount(innerVectorCount);
+        int innerVectorCount = vectorType.equals(OFFSET) ? count + 1 : count;
         for (int i = 0; i < innerVectorCount; i++) {
           parser.nextToken();
           setValueFromParser(valueVector, i);
         }
+        mutator.setValueCount(innerVectorCount);
         readToken(END_ARRAY);
       }
       // if children
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
index 8fe8e484496cd..68da7052f2b8b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
@@ -81,4 +81,19 @@ public String getName() {
   public String toString() {
     return getName();
   }
+
+  @Override
+  public int hashCode() {
+    return type;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj instanceof ArrowVectorType) {
+      ArrowVectorType other = (ArrowVectorType) obj;
+      return type == other.type;
+    }
+    return false;
+  }
+
 }

From 859018b3c79bfc0cb2259bdfc3d5930a9a936432 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 30 Nov 2016 13:25:42 -0500
Subject: [PATCH 0208/1644] ARROW-392: [C++/Java] String IPC integration
 testing / fixes. Add array / record batch pretty-printing

Was blocked by ARROW-393

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #217 from wesm/ARROW-392 and squashes the following commits:

1efeaed [Wes McKinney] Remove debug printing from Java
57e4926 [Wes McKinney] cpplint
56a1c41 [Wes McKinney] We are only padding to 8 byte boundaries
e33bed3 [Wes McKinney] clang-format, add all-OK message to integration_test.py
8e8d6d3 [Wes McKinney] Implement simple C++ pretty printer for record batches. Debugging efforts
---
 cpp/CMakeLists.txt                         |   1 +
 cpp/src/arrow/CMakeLists.txt               |   2 +
 cpp/src/arrow/ipc/ipc-json-test.cc         |  20 ---
 cpp/src/arrow/ipc/json-integration-test.cc |   7 +
 cpp/src/arrow/ipc/json-internal.cc         |   2 +-
 cpp/src/arrow/pretty_print-test.cc         |  87 ++++++++++
 cpp/src/arrow/pretty_print.cc              | 192 +++++++++++++++++++++
 cpp/src/arrow/pretty_print.h               |  35 ++++
 cpp/src/arrow/test-util.h                  |  22 +++
 cpp/src/arrow/type_traits.h                |   6 +
 format/IPC.md                              |   4 +-
 integration/data/simple.json               |   2 +-
 integration/integration_test.py            |   2 +-
 13 files changed, 357 insertions(+), 25 deletions(-)
 create mode 100644 cpp/src/arrow/pretty_print-test.cc
 create mode 100644 cpp/src/arrow/pretty_print.cc
 create mode 100644 cpp/src/arrow/pretty_print.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 1a970081234fa..798d75fe55643 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -745,6 +745,7 @@ set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/builder.cc
   src/arrow/column.cc
+  src/arrow/pretty_print.cc
   src/arrow/schema.cc
   src/arrow/table.cc
   src/arrow/type.cc
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 81851bc5b3eb1..6c0dea20ba7b5 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -21,6 +21,7 @@ install(FILES
   array.h
   column.h
   builder.h
+  pretty_print.h
   schema.h
   table.h
   type.h
@@ -37,6 +38,7 @@ set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 ADD_ARROW_TEST(array-test)
 ADD_ARROW_TEST(column-test)
+ADD_ARROW_TEST(pretty_print-test)
 ADD_ARROW_TEST(schema-test)
 ADD_ARROW_TEST(table-test)
 
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index e5c3a081fca53..ba4d9ca982850 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -96,26 +96,6 @@ void CheckPrimitive(const std::shared_ptr<DataType>& type,
   TestArrayRoundTrip(*array.get());
 }
 
-template <typename TYPE, typename C_TYPE>
-void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& is_valid,
-    const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
-  std::shared_ptr<Buffer> values_buffer;
-  std::shared_ptr<Buffer> values_bitmap;
-
-  ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
-  ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap));
-
-  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
-
-  int32_t null_count = 0;
-  for (bool val : is_valid) {
-    if (!val) { ++null_count; }
-  }
-
-  *out = std::make_shared<ArrayType>(type, static_cast<int32_t>(values.size()),
-      values_buffer, null_count, values_bitmap);
-}
-
 TEST(TestJsonSchemaWriter, FlatTypes) {
   std::vector<std::shared_ptr<Field>> fields = {field("f0", int8()),
       field("f1", int16(), false), field("f2", int32()), field("f3", int64(), false),
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 7a313f791e6c8..c4e68472a19d4 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -31,6 +31,7 @@
 #include "arrow/io/file.h"
 #include "arrow/ipc/file.h"
 #include "arrow/ipc/json.h"
+#include "arrow/pretty_print.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
@@ -171,6 +172,12 @@ static Status ValidateArrowVsJson(
     if (!json_batch->Equals(*arrow_batch.get())) {
       std::stringstream ss;
       ss << "Record batch " << i << " did not match";
+
+      ss << "\nJSON: \n ";
+      RETURN_NOT_OK(PrettyPrint(*json_batch.get(), &ss));
+
+      ss << "\nArrow: \n ";
+      RETURN_NOT_OK(PrettyPrint(*arrow_batch.get(), &ss));
       return Status::Invalid(ss.str());
     }
   }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index e56bcb32b9488..50f5b0cb1bd1e 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -343,7 +343,7 @@ class JsonSchemaWriter : public TypeVisitor {
 
 class JsonArrayWriter : public ArrayVisitor {
  public:
-  explicit JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
+  JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
       : name_(name), array_(array), writer_(writer) {}
 
   Status Write() { return VisitArray(name_, array_); }
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
new file mode 100644
index 0000000000000..10af41d16af13
--- /dev/null
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <sstream>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/pretty_print.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+
+namespace arrow {
+
+class TestArrayPrinter : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  void Print(const Array& array) {}
+
+ private:
+  std::ostringstream sink_;
+};
+
+template <typename TYPE, typename C_TYPE>
+void CheckPrimitive(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
+    const char* expected) {
+  std::ostringstream sink;
+
+  MemoryPool* pool = default_memory_pool();
+  typename TypeTraits<TYPE>::BuilderType builder(pool, std::make_shared<TYPE>());
+
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid[i]) {
+      ASSERT_OK(builder.Append(values[i]));
+    } else {
+      ASSERT_OK(builder.AppendNull());
+    }
+  }
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  ASSERT_OK(PrettyPrint(*array.get(), &sink));
+
+  std::string result = sink.str();
+  ASSERT_EQ(std::string(expected, strlen(expected)), result);
+}
+
+TEST_F(TestArrayPrinter, PrimitiveType) {
+  std::vector<bool> is_valid = {true, true, false, true, false};
+
+  std::vector<int32_t> values = {0, 1, 2, 3, 4};
+  static const char* expected = R"expected([0, 1, null, 3, null])expected";
+  CheckPrimitive<Int32Type, int32_t>(is_valid, values, expected);
+
+  std::vector<std::string> values2 = {"foo", "bar", "", "baz", ""};
+  static const char* ex2 = R"expected(["foo", "bar", null, "baz", null])expected";
+  CheckPrimitive<StringType, std::string>(is_valid, values2, ex2);
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
new file mode 100644
index 0000000000000..c0b4b08274ac1
--- /dev/null
+++ b/cpp/src/arrow/pretty_print.cc
@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <ostream>
+#include <string>
+
+#include "arrow/array.h"
+#include "arrow/pretty_print.h"
+#include "arrow/table.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/types/list.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+class ArrayPrinter : public ArrayVisitor {
+ public:
+  ArrayPrinter(const Array& array, std::ostream* sink) : array_(array), sink_(sink) {}
+
+  Status Print() { return VisitArray(array_); }
+
+  Status VisitArray(const Array& array) { return array.Accept(this); }
+
+  template <typename T>
+  typename std::enable_if<IsNumeric<T>::value, void>::type WriteDataValues(
+      const T& array) {
+    const auto data = array.raw_data();
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        (*sink_) << "null";
+      } else {
+        (*sink_) << data[i];
+      }
+    }
+  }
+
+  // String (Utf8), Binary
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BinaryArray, T>::value, void>::type
+  WriteDataValues(const T& array) {
+    int32_t length;
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        (*sink_) << "null";
+      } else {
+        const char* buf = reinterpret_cast<const char*>(array.GetValue(i, &length));
+        (*sink_) << "\"" << std::string(buf, length) << "\"";
+      }
+    }
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
+  WriteDataValues(const T& array) {
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        (*sink_) << "null";
+      } else {
+        (*sink_) << (array.Value(i) ? "true" : "false");
+      }
+    }
+  }
+
+  void OpenArray() { (*sink_) << "["; }
+
+  void CloseArray() { (*sink_) << "]"; }
+
+  template <typename T>
+  Status WritePrimitive(const T& array) {
+    OpenArray();
+    WriteDataValues(array);
+    CloseArray();
+    return Status::OK();
+  }
+
+  template <typename T>
+  Status WriteVarBytes(const T& array) {
+    OpenArray();
+    WriteDataValues(array);
+    CloseArray();
+    return Status::OK();
+  }
+
+  Status Visit(const NullArray& array) override { return Status::OK(); }
+
+  Status Visit(const BooleanArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int8Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int16Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int32Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int64Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt8Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt16Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt32Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt64Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const HalfFloatArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const FloatArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const DoubleArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const StringArray& array) override { return WriteVarBytes(array); }
+
+  Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); }
+
+  Status Visit(const DateArray& array) override { return Status::NotImplemented("date"); }
+
+  Status Visit(const TimeArray& array) override { return Status::NotImplemented("time"); }
+
+  Status Visit(const TimestampArray& array) override {
+    return Status::NotImplemented("timestamp");
+  }
+
+  Status Visit(const IntervalArray& array) override {
+    return Status::NotImplemented("interval");
+  }
+
+  Status Visit(const DecimalArray& array) override {
+    return Status::NotImplemented("decimal");
+  }
+
+  Status Visit(const ListArray& array) override {
+    // auto type = static_cast<const ListType*>(array.type().get());
+    // for (size_t i = 0; i < fields.size(); ++i) {
+    //   RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
+    // }
+    // return WriteChildren(type->children(), {array.values()});
+    return Status::OK();
+  }
+
+  Status Visit(const StructArray& array) override {
+    // auto type = static_cast<const StructType*>(array.type().get());
+    // for (size_t i = 0; i < fields.size(); ++i) {
+    //   RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
+    // }
+    // return WriteChildren(type->children(), array.fields());
+    return Status::OK();
+  }
+
+  Status Visit(const UnionArray& array) override {
+    return Status::NotImplemented("union");
+  }
+
+ private:
+  const Array& array_;
+  std::ostream* sink_;
+};
+
+Status PrettyPrint(const Array& arr, std::ostream* sink) {
+  ArrayPrinter printer(arr, sink);
+  return printer.Print();
+}
+
+Status PrettyPrint(const RecordBatch& batch, std::ostream* sink) {
+  for (int i = 0; i < batch.num_columns(); ++i) {
+    const std::string& name = batch.column_name(i);
+    (*sink) << name << ": ";
+    RETURN_NOT_OK(PrettyPrint(*batch.column(i).get(), sink));
+    (*sink) << "\n";
+  }
+  return Status::OK();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/pretty_print.h b/cpp/src/arrow/pretty_print.h
new file mode 100644
index 0000000000000..dcb236d726949
--- /dev/null
+++ b/cpp/src/arrow/pretty_print.h
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PRETTY_PRINT_H
+#define ARROW_PRETTY_PRINT_H
+
+#include <ostream>
+
+#include "arrow/type_fwd.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Status;
+
+Status ARROW_EXPORT PrettyPrint(const RecordBatch& batch, std::ostream* sink);
+Status ARROW_EXPORT PrettyPrint(const Array& arr, std::ostream* sink);
+
+}  // namespace arrow
+
+#endif  // ARROW_PRETTY_PRINT_H
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 63c2166a5736b..b86a1809cd0e9 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -32,6 +32,7 @@
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
+#include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
@@ -250,6 +251,27 @@ Status MakeRandomBytePoolBuffer(int32_t length, MemoryPool* pool,
 }
 
 }  // namespace test
+
+template <typename TYPE, typename C_TYPE>
+void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& is_valid,
+    const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
+  std::shared_ptr<Buffer> values_buffer;
+  std::shared_ptr<Buffer> values_bitmap;
+
+  ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
+  ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap));
+
+  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
+
+  int32_t null_count = 0;
+  for (bool val : is_valid) {
+    if (!val) { ++null_count; }
+  }
+
+  *out = std::make_shared<ArrayType>(type, static_cast<int32_t>(values.size()),
+      values_buffer, null_count, values_bitmap);
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_TEST_UTIL_H_
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index bbb807488e3d0..c21c5002035f8 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -192,6 +192,12 @@ struct IsFloatingPoint {
   static constexpr bool value = std::is_floating_point<c_type>::value;
 };
 
+template <typename T>
+struct IsNumeric {
+  PRIMITIVE_TRAITS(T);
+  static constexpr bool value = std::is_arithmetic<c_type>::value;
+};
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPE_TRAITS_H
diff --git a/format/IPC.md b/format/IPC.md
index a55dcdff48117..d386e6048cf12 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -24,7 +24,7 @@ In general, the file looks like:
 
 ```
 <magic number "ARROW1">
-<empty padding bytes [to 64 byte boundary]>
+<empty padding bytes [to 8 byte boundary]>
 <DICTIONARY 0>
 ...
 <DICTIONARY k - 1>
@@ -49,7 +49,7 @@ appropriate alignment and padding):
 ```
 <int32: metadata flatbuffer size>
 <metadata: org.apache.arrow.flatbuf.RecordBatch>
-<padding bytes [to 64-byte boundary]>
+<padding bytes [to 8-byte boundary]>
 <body: buffers end to end>
 ```
 
diff --git a/integration/data/simple.json b/integration/data/simple.json
index a91b405d4f0f0..fb903e7ac4b63 100644
--- a/integration/data/simple.json
+++ b/integration/data/simple.json
@@ -31,7 +31,7 @@
           "vectors": [
             {"type": "VALIDITY", "typeBitWidth": 1},
             {"type": "OFFSET", "typeBitWidth": 32},
-            {"type": "DATA", "typeBitWidth": 64}
+            {"type": "DATA", "typeBitWidth": 8}
           ]
         }
       }
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 6ea634d779566..88dc3ad7971ff 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -165,7 +165,7 @@ def run_all_tests(debug=False):
 
     runner = IntegrationRunner(json_files, testers, debug=debug)
     runner.run()
-
+    print('-- All tests passed!')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Arrow integration test CLI')

From 072b7d671356721bc57da8703ed0939749cf4880 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 30 Nov 2016 18:14:31 -0500
Subject: [PATCH 0209/1644] ARROW-395: Arrow file format writes record batches
 in reverse order.

Author: Julien Le Dem <julien@dremio.com>

Closes #220 from julienledem/rb_order and squashes the following commits:

ae5b7f8 [Julien Le Dem] ARROW-395: Arrow file format writes record batches in reverse order.
---
 .../apache/arrow/vector/file/ArrowFooter.java | 17 +++++--------
 .../arrow/vector/file/TestArrowFile.java      | 25 +++++++++++++------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
index 01e175b31b8db..3be19296cb56d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
@@ -17,6 +17,8 @@
  */
 package org.apache.arrow.vector.file;
 
+import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -52,10 +54,10 @@ public ArrowFooter(Footer footer) {
 
   private static List<ArrowBlock> recordBatches(Footer footer) {
     List<ArrowBlock> recordBatches = new ArrayList<>();
-    Block tempBLock = new Block();
+    Block tempBlock = new Block();
     int recordBatchesLength = footer.recordBatchesLength();
     for (int i = 0; i < recordBatchesLength; i++) {
-      Block block = footer.recordBatches(tempBLock, i);
+      Block block = footer.recordBatches(tempBlock, i);
       recordBatches.add(new ArrowBlock(block.offset(), block.metaDataLength(), block.bodyLength()));
     }
     return recordBatches;
@@ -88,9 +90,9 @@ public List<ArrowBlock> getRecordBatches() {
   public int writeTo(FlatBufferBuilder builder) {
     int schemaIndex = schema.getSchema(builder);
     Footer.startDictionariesVector(builder, dictionaries.size());
-    int dicsOffset = endVector(builder, dictionaries);
+    int dicsOffset = writeAllStructsToVector(builder, dictionaries);
     Footer.startRecordBatchesVector(builder, recordBatches.size());
-    int rbsOffset = endVector(builder, recordBatches);
+    int rbsOffset = writeAllStructsToVector(builder, recordBatches);
     Footer.startFooter(builder);
     Footer.addSchema(builder, schemaIndex);
     Footer.addDictionaries(builder, dicsOffset);
@@ -98,13 +100,6 @@ public int writeTo(FlatBufferBuilder builder) {
     return Footer.endFooter(builder);
   }
 
-  private int endVector(FlatBufferBuilder builder, List<ArrowBlock> blocks) {
-    for (ArrowBlock block : blocks) {
-      block.writeTo(builder);
-    }
-    return builder.endVector();
-  }
-
   @Override
   public int hashCode() {
     final int prime = 31;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index c9e60ee047bfe..5fa18b3ca5339 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -161,24 +161,27 @@ public void testWriteReadComplex() throws IOException {
   @Test
   public void testWriteReadMultipleRBs() throws IOException {
     File file = new File("target/mytest_multiple.arrow");
-    int count = COUNT;
+    int[] counts = { 10, 5 };
 
     // write
     try (
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", originalVectorAllocator, null);
         FileOutputStream fileOutputStream = new FileOutputStream(file);) {
-      writeData(count, parent);
-      VectorUnloader vectorUnloader = newVectorUnloader(parent.getChild("root"));
-      Schema schema = vectorUnloader.getSchema();
+      writeData(counts[0], parent);
+      VectorUnloader vectorUnloader0 = newVectorUnloader(parent.getChild("root"));
+      Schema schema = vectorUnloader0.getSchema();
       Assert.assertEquals(2, schema.getFields().size());
       try (ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);) {
-        try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch()) {
+        try (ArrowRecordBatch recordBatch = vectorUnloader0.getRecordBatch()) {
+          Assert.assertEquals("RB #0", counts[0], recordBatch.getLength());
           arrowWriter.writeRecordBatch(recordBatch);
         }
         parent.allocateNew();
-        writeData(count, parent);
-        try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch()) {
+        writeData(counts[1], parent); // if we write the same data we don't catch that the metadata is stored in the wrong order.
+        VectorUnloader vectorUnloader1 = newVectorUnloader(parent.getChild("root"));
+        try (ArrowRecordBatch recordBatch = vectorUnloader1.getRecordBatch()) {
+          Assert.assertEquals("RB #1", counts[1], recordBatch.getLength());
           arrowWriter.writeRecordBatch(recordBatch);
         }
       }
@@ -195,21 +198,27 @@ public void testWriteReadMultipleRBs() throws IOException {
       ArrowFooter footer = arrowReader.readFooter();
       Schema schema = footer.getSchema();
       LOGGER.debug("reading schema: " + schema);
+      int i = 0;
       try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator);) {
         VectorLoader vectorLoader = new VectorLoader(root);
         List<ArrowBlock> recordBatches = footer.getRecordBatches();
         Assert.assertEquals(2, recordBatches.size());
+        long previousOffset = 0;
         for (ArrowBlock rbBlock : recordBatches) {
+          Assert.assertTrue(rbBlock.getOffset() + " > " + previousOffset, rbBlock.getOffset() > previousOffset);
+          previousOffset = rbBlock.getOffset();
           Assert.assertEquals(0, rbBlock.getOffset() % 8);
           Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
           try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
+            Assert.assertEquals("RB #" + i, counts[i], recordBatch.getLength());
             List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
             for (ArrowBuffer arrowBuffer : buffersLayout) {
               Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
             }
             vectorLoader.load(recordBatch);
-            validateContent(count, root);
+            validateContent(counts[i], root);
           }
+          ++i;
         }
       }
     }

From 3b946b822445f21872c7cb42563c8d0c7bc84b80 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 1 Dec 2016 13:26:43 +0100
Subject: [PATCH 0210/1644] ARROW-396: [Python] Add
 pyarrow.schema.Schema.equals

Added pyarrow api for `Schema.equals` to check if 2 schema's are equal and corresponding test case.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #221 from BryanCutler/add-pyarrow-schema_equals-ARROW-396 and squashes the following commits:

910e943 [Bryan Cutler] added test case for pyarrow Schema equals
24cf982 [Bryan Cutler] added pyarrow Schema equals, and related def for CSchema
---
 python/pyarrow/includes/libarrow.pxd |  3 +++
 python/pyarrow/schema.pyx            |  9 +++++++++
 python/pyarrow/tests/test_schema.py  | 17 +++++++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 3ae1789170303..19da4085e1bb9 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -88,6 +88,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CSchema" arrow::Schema":
         CSchema(const vector[shared_ptr[CField]]& fields)
+
+        c_bool Equals(const shared_ptr[CSchema]& other)
+
         const shared_ptr[CField]& field(int i)
         int num_fields()
         c_string ToString()
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 084c304aed2a2..e0badb9764143 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -110,6 +110,15 @@ cdef class Schema:
         self.schema = schema.get()
         self.sp_schema = schema
 
+    def equals(self, other):
+        """
+        Test if this schema is equal to the other
+        """
+        cdef Schema _other
+        _other = other
+
+        return self.sp_schema.get().Equals(_other.sp_schema)
+
     @classmethod
     def from_fields(cls, fields):
         cdef:
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 2894ea8f84451..4aa8112a91769 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -69,3 +69,20 @@ def test_schema(self):
 foo: int32
 bar: string
 baz: list<item: int8>"""
+
+    def test_schema_equals(self):
+        fields = [
+            A.field('foo', A.int32()),
+            A.field('bar', A.string()),
+            A.field('baz', A.list_(A.int8()))
+        ]
+
+        sch1 = A.schema(fields)
+        print(dir(sch1))
+        sch2 = A.schema(fields)
+        assert sch1.equals(sch2)
+
+        del fields[-1]
+        sch3 = A.schema(fields)
+        assert not sch1.equals(sch3)
+

From 33c731dbd69331b0d7ce0dc924791db4ca461009 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 2 Dec 2016 10:32:36 -0500
Subject: [PATCH 0211/1644] =?UTF-8?q?ARROW-398:=20Java=20file=20format=20r?=
 =?UTF-8?q?equires=20bitmaps=20of=20all=201's=20to=20be=20written=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… when there are no nulls

Author: Julien Le Dem <julien@dremio.com>

Closes #222 from julienledem/empty_buf and squashes the following commits:

c29da53 [Julien Le Dem] fix extraneous bits
4e87d88 [Julien Le Dem] ARROW-398: Java file format requires bitmaps of all 1's to be written when there are no nulls
---
 .../templates/NullableValueVectors.java       |  2 +-
 .../main/codegen/templates/UnionVector.java   |  2 +-
 .../arrow/vector/BaseDataValueVector.java     |  7 +-
 .../org/apache/arrow/vector/BitVector.java    | 36 ++++++++++
 .../org/apache/arrow/vector/BufferBacked.java |  4 +-
 .../org/apache/arrow/vector/ValueVector.java  | 17 -----
 .../org/apache/arrow/vector/VectorLoader.java |  1 -
 .../arrow/vector/complex/ListVector.java      |  2 +-
 .../vector/complex/NullableMapVector.java     |  2 +-
 .../arrow/vector/TestVectorUnloadLoad.java    | 69 +++++++++++++++++++
 .../vector/file/TestArrowReaderWriter.java    |  4 --
 11 files changed, 116 insertions(+), 30 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 48af7a2bafe4d..716fedcf866ef 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -144,7 +144,7 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    org.apache.arrow.vector.BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+    org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
     bits.valueCount = fieldNode.getLength();
   }
 
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 5ca3f90148449..9608b3c48ebd0 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -105,7 +105,7 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+    BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
     this.valueCount = fieldNode.getLength();
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index c22258d42651b..4c6d363f21cda 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -21,6 +21,7 @@
 import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
 
 import io.netty.buffer.ArrowBuf;
 
@@ -29,13 +30,13 @@ public abstract class BaseDataValueVector extends BaseValueVector implements Buf
 
   protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
 
-  public static void load(List<BufferBacked> vectors, List<ArrowBuf> buffers) {
+  public static void load(ArrowFieldNode fieldNode, List<BufferBacked> vectors, List<ArrowBuf> buffers) {
     int expectedSize = vectors.size();
     if (buffers.size() != expectedSize) {
       throw new IllegalArgumentException("Illegal buffer count, expected " + expectedSize + ", got: " + buffers.size());
     }
     for (int i = 0; i < expectedSize; i++) {
-      vectors.get(i).load(buffers.get(i));
+      vectors.get(i).load(fieldNode, buffers.get(i));
     }
   }
 
@@ -106,7 +107,7 @@ public ArrowBuf getBuffer() {
   }
 
   @Override
-  public void load(ArrowBuf data) {
+  public void load(ArrowFieldNode fieldNode, ArrowBuf data) {
     this.data.release();
     this.data = data.retain(allocator);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index c12db5045c2db..7ce1236b2ec30 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -22,6 +22,7 @@
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.BitHolder;
 import org.apache.arrow.vector.holders.NullableBitHolder;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.OversizedAllocationException;
@@ -48,6 +49,41 @@ public BitVector(String name, BufferAllocator allocator) {
     super(name, allocator);
   }
 
+  @Override
+  public void load(ArrowFieldNode fieldNode, ArrowBuf data) {
+    // When the vector is all nulls or all defined, the content of the buffer can be omitted
+    if (data.readableBytes() == 0 && fieldNode.getLength() != 0) {
+      data.release();
+      int count = fieldNode.getLength();
+      allocateNew(count);
+      int n = getSizeFromCount(count);
+      if (fieldNode.getNullCount() == 0) {
+        // all defined
+        // create an all 1s buffer
+        // set full bytes
+        int fullBytesCount = count / 8;
+        for (int i = 0; i < fullBytesCount; ++i) {
+          this.data.setByte(i, 0xFF);
+        }
+        int remainder = count % 8;
+        // set remaining bits
+        if (remainder > 0) {
+          byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7));;
+          this.data.setByte(fullBytesCount, bitMask);
+        }
+      } else if (fieldNode.getNullCount() == fieldNode.getLength()) {
+        // all null
+        // create an all 0s buffer
+        zeroVector();
+      } else {
+        throw new IllegalArgumentException("The buffer can be empty only if there's no data or it's all null or all defined");
+      }
+      this.data.writerIndex(n);
+    } else {
+      super.load(fieldNode, data);
+    }
+  }
+
   @Override
   public Field getField() {
     throw new UnsupportedOperationException("internal vector");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
index d1c262d226556..3c8b3210d77ff 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
@@ -17,6 +17,8 @@
  */
 package org.apache.arrow.vector;
 
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+
 import io.netty.buffer.ArrowBuf;
 
 /**
@@ -24,7 +26,7 @@
  */
 public interface BufferBacked {
 
-  void load(ArrowBuf data);
+  void load(ArrowFieldNode fieldNode, ArrowBuf data);
 
   ArrowBuf unLoad();
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index ba7790e47ef95..5b24a41850d75 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -130,13 +130,6 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
    */
   FieldReader getReader();
 
-  /**
-   * Get the metadata for this field. Used in serialization
-   *
-   * @return FieldMetadata for this field.
-   */
-//  SerializedField getMetadata();
-
   /**
    * Returns the number of bytes that is used by this vector instance.
    */
@@ -166,16 +159,6 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
    */
   ArrowBuf[] getBuffers(boolean clear);
 
-  /**
-   * Load the data provided in the buffer. Typically used when deserializing from the wire.
-   *
-   * @param metadata
-   *          Metadata used to decode the incoming buffer.
-   * @param buffer
-   *          The buffer that contains the ValueVector.
-   */
-//  void load(SerializedField metadata, DrillBuf buffer);
-
   /**
    * An abstraction that is used to read from this vector instance.
    */
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index c5d642ee0cc72..757f061dd5a2f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -81,7 +81,6 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
     try {
       vector.loadFieldBuffers(fieldNode, ownBuffers);
     } catch (RuntimeException e) {
-      e.printStackTrace();
       throw new IllegalArgumentException("Could not load buffers for field " +
               field + " error message" + e.getMessage(), e);
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index dd99c734f7ff8..e18f99f95d780 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -93,7 +93,7 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+    BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 8e1bbfabdc907..f0ddf2727e9ea 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -62,7 +62,7 @@ public NullableMapVector(String name, BufferAllocator allocator, CallBack callBa
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    BaseDataValueVector.load(getFieldInnerVectors(), ownBuffers);
+    BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
     this.valueCount = fieldNode.getLength();
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 78f69eedc1c27..9dfe8d840e49d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -17,7 +17,13 @@
  */
 package org.apache.arrow.vector;
 
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -29,12 +35,17 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Test;
 
+import io.netty.buffer.ArrowBuf;
+
 public class TestVectorUnloadLoad {
 
   static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
@@ -88,6 +99,64 @@ public void testUnloadLoad() throws IOException {
     }
   }
 
+  /**
+   * The validity buffer can be empty if:
+   *  - all values are defined
+   *  - all values are null
+   * @throws IOException
+   */
+  @Test
+  public void testLoadEmptyValidityBuffer() throws IOException {
+    Schema schema = new Schema(asList(
+        new Field("intDefined", true, new ArrowType.Int(32, true), Collections.<Field>emptyList()),
+        new Field("intNull", true, new ArrowType.Int(32, true), Collections.<Field>emptyList())
+        ));
+    int count = 10;
+    ArrowBuf validity = allocator.getEmpty();
+    ArrowBuf values = allocator.buffer(count * 4); // integers
+    for (int i = 0; i < count; i++) {
+      values.setInt(i * 4, i);
+    }
+    try (
+        ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(validity, values, validity, values));
+        BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        VectorSchemaRoot newRoot = new VectorSchemaRoot(schema, finalVectorsAllocator);
+        ) {
+
+      // load it
+      VectorLoader vectorLoader = new VectorLoader(newRoot);
+
+      vectorLoader.load(recordBatch);
+
+      NullableIntVector intDefinedVector = (NullableIntVector)newRoot.getVector("intDefined");
+      NullableIntVector intNullVector = (NullableIntVector)newRoot.getVector("intNull");
+      for (int i = 0; i < count; i++) {
+        assertFalse("#" + i, intDefinedVector.getAccessor().isNull(i));
+        assertEquals("#" + i, i, intDefinedVector.getAccessor().get(i));
+        assertTrue("#" + i, intNullVector.getAccessor().isNull(i));
+      }
+      intDefinedVector.getMutator().setSafe(count + 10, 1234);
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 1));
+      // empty slots should still default to unset
+      intDefinedVector.getMutator().setSafe(count + 1, 789);
+      assertFalse(intDefinedVector.getAccessor().isNull(count + 1));
+      assertEquals(789, intDefinedVector.getAccessor().get(count + 1));
+      assertTrue(intDefinedVector.getAccessor().isNull(count));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 2));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 3));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 4));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 5));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 6));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 7));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 8));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 9));
+      assertFalse(intDefinedVector.getAccessor().isNull(count + 10));
+      assertEquals(1234, intDefinedVector.getAccessor().get(count + 10));
+    } finally {
+      values.release();
+    }
+  }
+
   public static VectorUnloader newVectorUnloader(FieldVector root) {
     Schema schema = new Schema(root.getField().getChildren());
     int valueCount = root.getAccessor().getValueCount();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index f90329aca11dd..8ed89fa347b3b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -30,10 +30,6 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowFooter;
-import org.apache.arrow.vector.file.ArrowReader;
-import org.apache.arrow.vector.file.ArrowWriter;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.ArrowType;

From 06be7aed062aca32b683f2ab3a94a201ae54b4f3 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 2 Dec 2016 11:48:24 -0500
Subject: [PATCH 0212/1644] ARROW-389: Python: Write Parquet files to
 pyarrow.io.NativeFile objects

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #214 from xhochy/ARROW-389 and squashes the following commits:

e66c895 [Uwe L. Korn] Switch image to deprecated group
876cd65 [Uwe L. Korn] ARROW-389: Python: Write Parquet files to pyarrow.io.NativeFile objects
---
 .travis.yml                          |  1 +
 python/pyarrow/includes/parquet.pxd  |  7 +++++--
 python/pyarrow/parquet.pyx           | 18 ++++++++++++------
 python/pyarrow/tests/test_parquet.py | 27 +++++++++++++++++++++++++++
 4 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 052c22ccc3790..bfc2f26b4f590 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,6 +24,7 @@ matrix:
   - compiler: gcc
     language: cpp
     os: linux
+    group: deprecated
     before_script:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index 57c35ba89445b..cb791e16f926d 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -19,7 +19,7 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport CArray, CSchema, CStatus, CTable, MemoryPool
-from pyarrow.includes.libarrow_io cimport ReadableFileInterface
+from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
@@ -131,6 +131,9 @@ cdef extern from "parquet/arrow/io.h" namespace "parquet::arrow" nogil:
         ParquetReadSource(ParquetAllocator* allocator)
         Open(const shared_ptr[ReadableFileInterface]& file)
 
+    cdef cppclass ParquetWriteSink:
+        ParquetWriteSink(const shared_ptr[OutputStream]& file)
+
 
 cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
     CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
@@ -154,6 +157,6 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
 cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
     cdef CStatus WriteFlatTable(
         const CTable* table, MemoryPool* pool,
-        const shared_ptr[ParquetOutputStream]& sink,
+        const shared_ptr[ParquetWriteSink]& sink,
         int64_t chunk_size,
         const shared_ptr[WriterProperties]& properties)
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index a6e3ac30684b4..83fddb287a3f1 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -21,7 +21,7 @@
 
 from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.parquet cimport *
-from pyarrow.includes.libarrow_io cimport ReadableFileInterface
+from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream, FileOutputStream
 cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.array cimport Array
@@ -151,7 +151,7 @@ def read_table(source, columns=None):
         return Table.from_arrays(columns, arrays)
 
 
-def write_table(table, filename, chunk_size=None, version=None,
+def write_table(table, sink, chunk_size=None, version=None,
                 use_dictionary=True, compression=None):
     """
     Write a Table to Parquet format
@@ -159,7 +159,7 @@ def write_table(table, filename, chunk_size=None, version=None,
     Parameters
     ----------
     table : pyarrow.Table
-    filename : string
+    sink: string or pyarrow.io.NativeFile
     chunk_size : int
         The maximum number of rows in each Parquet RowGroup. As a default,
         we will write a single RowGroup per file.
@@ -173,7 +173,8 @@ def write_table(table, filename, chunk_size=None, version=None,
     """
     cdef Table table_ = table
     cdef CTable* ctable_ = table_.table
-    cdef shared_ptr[ParquetOutputStream] sink
+    cdef shared_ptr[ParquetWriteSink] sink_
+    cdef shared_ptr[FileOutputStream] filesink_
     cdef WriterProperties.Builder properties_builder
     cdef int64_t chunk_size_ = 0
     if chunk_size is None:
@@ -230,7 +231,12 @@ def write_table(table, filename, chunk_size=None, version=None,
             else:
                 raise ArrowException("Unsupport compression codec")
 
-    sink.reset(new LocalFileOutputStream(tobytes(filename)))
+    if isinstance(sink, six.string_types):
+       check_status(FileOutputStream.Open(tobytes(sink), &filesink_))
+       sink_.reset(new ParquetWriteSink(<shared_ptr[OutputStream]>filesink_))
+    elif isinstance(sink, NativeFile):
+        sink_.reset(new ParquetWriteSink((<NativeFile>sink).wr_file))
+
     with nogil:
-        check_status(WriteFlatTable(ctable_, default_memory_pool(), sink,
+        check_status(WriteFlatTable(ctable_, default_memory_pool(), sink_,
                                     chunk_size_, properties_builder.build()))
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index c1d44ce0d4230..841830f6fba3b 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -18,6 +18,7 @@
 import pytest
 
 import pyarrow as A
+import pyarrow.io as paio
 
 import numpy as np
 import pandas as pd
@@ -131,6 +132,32 @@ def test_pandas_column_selection(tmpdir):
 
     pdt.assert_frame_equal(df[['uint8']], df_read)
 
+@parquet
+def test_pandas_parquet_native_file_roundtrip(tmpdir):
+    size = 10000
+    np.random.seed(0)
+    df = pd.DataFrame({
+        'uint8': np.arange(size, dtype=np.uint8),
+        'uint16': np.arange(size, dtype=np.uint16),
+        'uint32': np.arange(size, dtype=np.uint32),
+        'uint64': np.arange(size, dtype=np.uint64),
+        'int8': np.arange(size, dtype=np.int16),
+        'int16': np.arange(size, dtype=np.int16),
+        'int32': np.arange(size, dtype=np.int32),
+        'int64': np.arange(size, dtype=np.int64),
+        'float32': np.arange(size, dtype=np.float32),
+        'float64': np.arange(size, dtype=np.float64),
+        'bool': np.random.randn(size) > 0
+    })
+    arrow_table = A.from_pandas_dataframe(df)
+    imos = paio.InMemoryOutputStream()
+    pq.write_table(arrow_table, imos, version="2.0")
+    buf = imos.get_result()
+    reader = paio.BufferReader(buf)
+    df_read = pq.read_table(reader).to_pandas()
+    pdt.assert_frame_equal(df, df_read)
+
+
 @parquet
 def test_pandas_parquet_configuration_options(tmpdir):
     size = 10000

From ebe7dc8f5ff32f5fa86625d4c622b4e075e95ae0 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 2 Dec 2016 11:51:22 -0500
Subject: [PATCH 0213/1644] ARROW-335: Improve Type apis and toString() by
 encapsulating flatbuffers better

Author: Julien Le Dem <julien@dremio.com>

Closes #172 from julienledem/tostring and squashes the following commits:

546aa02 [Julien Le Dem] fix rebase issues
262ae9f [Julien Le Dem] review feedback
41d5627 [Julien Le Dem] ARROW-335: Improve Type apis and toString() by encapsulating flatbuffers better
---
 .../src/main/codegen/data/ArrowTypes.tdd      |   8 +-
 .../src/main/codegen/templates/ArrowType.java | 172 ++++--------------
 .../templates/NullableValueVectors.java       |  10 +-
 .../main/codegen/templates/UnionVector.java   |   4 +-
 .../complex/BaseRepeatedValueVector.java      |   5 +-
 .../arrow/vector/complex/MapVector.java       |   4 +-
 .../arrow/vector/schema/TypeLayout.java       |  19 +-
 .../arrow/vector/schema/VectorLayout.java     |   2 +-
 .../vector/types/FloatingPointPrecision.java  |  47 +++++
 .../arrow/vector/types/IntervalUnit.java      |  44 +++++
 .../apache/arrow/vector/types/TimeUnit.java   |  46 +++++
 .../org/apache/arrow/vector/types/Types.java  |  38 ++--
 .../apache/arrow/vector/types/UnionMode.java  |  44 +++++
 .../apache/arrow/vector/types/pojo/Field.java |  14 +-
 .../arrow/vector/types/pojo/Schema.java       |   3 +-
 .../complex/impl/TestPromotableWriter.java    |   7 +-
 .../complex/writer/TestComplexWriter.java     |  15 +-
 .../apache/arrow/vector/pojo/TestConvert.java |  14 +-
 .../arrow/vector/types/pojo/TestSchema.java   |  36 +++-
 19 files changed, 318 insertions(+), 214 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/TimeUnit.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/UnionMode.java

diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index c0b942bc3595d..01465e585dad2 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -26,7 +26,7 @@
     },
     {
       name: "Union",
-      fields: [{name: "mode", type: short}, {name: "typeIds", type: "int[]"}]
+      fields: [{name: "mode", type: short, valueType: UnionMode}, {name: "typeIds", type: "int[]"}]
     },
     {
       name: "Int",
@@ -34,7 +34,7 @@
     },
     {
       name: "FloatingPoint",
-      fields: [{name: precision, type: short}]
+      fields: [{name: precision, type: short, valueType: FloatingPointPrecision}]
     },
     {
       name: "Utf8",
@@ -62,11 +62,11 @@
     },
     {
       name: "Timestamp",
-      fields: [{name: "unit", type: short}]
+      fields: [{name: "unit", type: short, valueType: TimeUnit}]
     },
     {
       name: "Interval",
-      fields: [{name: "unit", type: short}]
+      fields: [{name: "unit", type: short, valueType: IntervalUnit}]
     }
   ]
 }
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index 4069e6061b66e..85ea3898e09c6 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -23,30 +23,18 @@
 package org.apache.arrow.vector.types.pojo;
 
 import com.google.flatbuffers.FlatBufferBuilder;
-import org.apache.arrow.flatbuf.Type;
 
-import java.io.IOException;
 import java.util.Objects;
 
-import org.apache.arrow.flatbuf.Precision;
-import org.apache.arrow.flatbuf.UnionMode;
-import org.apache.arrow.flatbuf.TimeUnit;
-import org.apache.arrow.flatbuf.IntervalUnit;
+import org.apache.arrow.flatbuf.Type;
+
+import org.apache.arrow.vector.types.*;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 
 /**
  * Arrow types
@@ -57,119 +45,31 @@
   property = "name")
 @JsonSubTypes({
 <#list arrowTypes.types as type>
-  @JsonSubTypes.Type(value = ArrowType.${type.name}.class, name = "${type.name?remove_ending("_")?lower_case}"),
+  @JsonSubTypes.Type(value = ArrowType.${type.name?remove_ending("_")}.class, name = "${type.name?remove_ending("_")?lower_case}"),
 </#list>
 })
 public abstract class ArrowType {
 
-  private static class FloatingPointPrecisionSerializer extends JsonSerializer<Short> {
-    @Override
-    public void serialize(Short precision,
-        JsonGenerator jsonGenerator,
-        SerializerProvider serializerProvider)
-            throws IOException, JsonProcessingException {
-      jsonGenerator.writeObject(Precision.name(precision));
-    }
-  }
-
-  private static class FloatingPointPrecisionDeserializer extends JsonDeserializer<Short> {
-    @Override
-    public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
-      String name = p.getText();
-      switch(name) {
-        case "HALF":
-          return Precision.HALF;
-        case "SINGLE":
-          return Precision.SINGLE;
-        case "DOUBLE":
-          return Precision.DOUBLE;
-        default:
-          throw new IllegalArgumentException("unknown precision: " + name);
-      }
-    }
-  }
-
-  private static class UnionModeSerializer extends JsonSerializer<Short> {
-    @Override
-    public void serialize(Short mode,
-        JsonGenerator jsonGenerator,
-        SerializerProvider serializerProvider)
-            throws IOException, JsonProcessingException {
-      jsonGenerator.writeObject(UnionMode.name(mode));
-    }
-  }
-
-  private static class UnionModeDeserializer extends JsonDeserializer<Short> {
-    @Override
-    public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
-      String name = p.getText();
-      switch(name) {
-        case "Sparse":
-          return UnionMode.Sparse;
-        case "Dense":
-          return UnionMode.Dense;
-        default:
-          throw new IllegalArgumentException("unknown union mode: " + name);
-      }
-    }
-  }
-
-  private static class TimestampUnitSerializer extends JsonSerializer<Short> {
-    @Override
-    public void serialize(Short unit,
-        JsonGenerator jsonGenerator,
-        SerializerProvider serializerProvider)
-            throws IOException, JsonProcessingException {
-      jsonGenerator.writeObject(TimeUnit.name(unit));
-    }
-  }
-
-  private static class TimestampUnitDeserializer extends JsonDeserializer<Short> {
-    @Override
-    public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
-      String name = p.getText();
-      switch(name) {
-        case "SECOND":
-          return TimeUnit.SECOND;
-        case "MILLISECOND":
-          return TimeUnit.MILLISECOND;
-        case "MICROSECOND":
-          return TimeUnit.MICROSECOND;
-        case "NANOSECOND":
-          return TimeUnit.NANOSECOND;
-        default:
-          throw new IllegalArgumentException("unknown time unit: " + name);
-      }
-    }
-  }
+  public static enum ArrowTypeID {
+    <#list arrowTypes.types as type>
+    <#assign name = type.name>
+    ${name?remove_ending("_")}(Type.${name}),
+    </#list>
+    NONE(Type.NONE);
+  
+    private final byte flatbufType;
 
-  private static class IntervalUnitSerializer extends JsonSerializer<Short> {
-    @Override
-    public void serialize(Short unit,
-        JsonGenerator jsonGenerator,
-        SerializerProvider serializerProvider)
-            throws IOException, JsonProcessingException {
-      jsonGenerator.writeObject(IntervalUnit.name(unit));
+    public byte getFlatbufID() {
+      return this.flatbufType;
     }
-  }
 
-  private static class IntervalUnitDeserializer extends JsonDeserializer<Short> {
-    @Override
-    public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
-      String name = p.getText();
-      switch(name) {
-        case "YEAR_MONTH":
-          return IntervalUnit.YEAR_MONTH;
-        case "DAY_TIME":
-          return IntervalUnit.DAY_TIME;
-        default:
-          throw new IllegalArgumentException("unknown interval unit: " + name);
-      }
+    private ArrowTypeID(byte flatbufType) {
+      this.flatbufType = flatbufType;
     }
   }
 
   @JsonIgnore
-  public abstract byte getTypeType();
+  public abstract ArrowTypeID getTypeID();
   public abstract int getType(FlatBufferBuilder builder);
   public abstract <T> T accept(ArrowTypeVisitor<T> visitor);
 
@@ -183,28 +83,30 @@ public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOExc
    */
   public static interface ArrowTypeVisitor<T> {
   <#list arrowTypes.types as type>
-    T visit(${type.name} type);
+    T visit(${type.name?remove_ending("_")} type);
   </#list>
   }
 
   <#list arrowTypes.types as type>
-  <#assign name = type.name>
+  <#assign name = type.name?remove_ending("_")>
   <#assign fields = type.fields>
   public static class ${name} extends ArrowType {
-    public static final byte TYPE_TYPE = Type.${name};
+    public static final ArrowTypeID TYPE_TYPE = ArrowTypeID.${name};
     <#if type.fields?size == 0>
     public static final ${name} INSTANCE = new ${name}();
     </#if>
 
     <#list fields as field>
-    ${field.type} ${field.name};
+    <#assign fieldType = field.valueType!field.type>
+    ${fieldType} ${field.name};
     </#list>
 
     <#if type.fields?size != 0>
     @JsonCreator
     public ${type.name}(
     <#list type.fields as field>
-      <#if field.type == "short"> @JsonDeserialize(using = ${type.name}${field.name?cap_first}Deserializer.class) </#if>@JsonProperty("${field.name}") ${field.type} ${field.name}<#if field_has_next>, </#if>
+    <#assign fieldType = field.valueType!field.type>
+      @JsonProperty("${field.name}") ${fieldType} ${field.name}<#if field_has_next>, </#if>
     </#list>
     ) {
       <#list type.fields as field>
@@ -214,7 +116,7 @@ public static class ${name} extends ArrowType {
     </#if>
 
     @Override
-    public byte getTypeType() {
+    public ArrowTypeID getTypeID() {
       return TYPE_TYPE;
     }
 
@@ -235,27 +137,29 @@ public int getType(FlatBufferBuilder builder) {
         org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, ${field.name});
       }
       <#else>
-      org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, this.${field.name});
+      org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, this.${field.name}<#if field.valueType??>.getFlatbufID()</#if>);
       </#if>
       </#list>
       return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
     }
 
     <#list fields as field>
-      <#if field.type == "short">
-    @JsonSerialize(using = ${type.name}${field.name?cap_first}Serializer.class)
-      </#if>
-    public ${field.type} get${field.name?cap_first}() {
+    <#assign fieldType = field.valueType!field.type>
+    public ${fieldType} get${field.name?cap_first}() {
       return ${field.name};
     }
     </#list>
 
     public String toString() {
-      return "${name}{"
+      return "${name}"
+      <#if fields?size != 0>
+        + "("
       <#list fields as field>
-        + <#if field.type == "int[]">java.util.Arrays.toString(${field.name})<#else>${field.name}</#if><#if field_has_next> + ", " </#if>
+        +   <#if field.type == "int[]">java.util.Arrays.toString(${field.name})<#else>${field.name}</#if><#if field_has_next> + ", " </#if>
       </#list>
-      + "}";
+        + ")"
+      </#if>
+      ;
     }
 
     @Override
@@ -265,7 +169,7 @@ public int hashCode() {
 
     @Override
     public boolean equals(Object obj) {
-      if (!(obj instanceof ${type.name})) {
+      if (!(obj instanceof ${name})) {
         return false;
       }
       <#if type.fields?size == 0>
@@ -287,7 +191,7 @@ public <T> T accept(ArrowTypeVisitor<T> visitor) {
   public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.apache.arrow.flatbuf.Field field) {
     switch(field.typeType()) {
     <#list arrowTypes.types as type>
-    <#assign name = type.name>
+    <#assign name = type.name?remove_ending("_")>
     <#assign nameLower = type.name?lower_case>
     <#assign fields = type.fields>
     case Type.${type.name}: {
@@ -302,7 +206,7 @@ public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.a
       ${field.type} ${field.name} = ${nameLower}Type.${field.name}();
       </#if>
       </#list>
-      return new ${type.name}(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+      return new ${name}(<#list type.fields as field><#if field.valueType??>${field.valueType}.fromFlatbufID(${field.name})<#else>${field.name}</#if><#if field_has_next>, </#if></#list>);
     }
     </#list>
     default:
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 716fedcf866ef..2c4274c13ee58 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -99,15 +99,15 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#elseif minor.class == "Time">
     field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), null);
   <#elseif minor.class == "Float4">
-    field = new Field(name, true, new FloatingPoint(Precision.SINGLE), null);
+    field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), null);
   <#elseif minor.class == "Float8">
-    field = new Field(name, true, new FloatingPoint(Precision.DOUBLE), null);
+    field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), null);
   <#elseif minor.class == "TimeStamp">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null);
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), null);
   <#elseif minor.class == "IntervalDay">
-    field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME), null);
+    field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), null);
   <#elseif minor.class == "IntervalYear">
-    field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH), null);
+    field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.YEAR_MONTH), null);
   <#elseif minor.class == "VarChar">
     field = new Field(name, true, new Utf8(), null);
   <#elseif minor.class == "VarBinary">
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 9608b3c48ebd0..ea1fdf6bd60fb 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -15,8 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import java.util.List;
-
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/UnionVector.java" />
 
@@ -35,7 +33,7 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 
-import static org.apache.arrow.flatbuf.UnionMode.Sparse;
+import static org.apache.arrow.vector.types.UnionMode.Sparse;
 
 
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 517d20c77a93c..7424df474ae89 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -20,7 +20,6 @@
 import java.util.Collections;
 import java.util.Iterator;
 
-import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -159,9 +158,9 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorT
       created = true;
     }
 
-    if (vector.getField().getType().getTypeType() != minorType.getType().getTypeType()) {
+    if (vector.getField().getType().getTypeID() != minorType.getType().getTypeID()) {
       final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
-          Type.name(minorType.getType().getTypeType()), Type.name(vector.getField().getType().getTypeType()));
+          minorType.getType().getTypeID(), vector.getField().getType().getTypeID());
       throw new SchemaChangeRuntimeException(msg);
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index aaecb956434e9..c2f216b197e1d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -34,7 +34,7 @@
 import org.apache.arrow.vector.holders.ComplexHolder;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringHashMap;
@@ -290,7 +290,7 @@ public Field getField() {
     for (ValueVector child : getChildren()) {
       children.add(child.getField());
     }
-    return new Field(name, false, Struct_.INSTANCE, children);
+    return new Field(name, false, Struct.INSTANCE, children);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index c5f53fe508d9f..0b586914bdf85 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -18,9 +18,6 @@
 package org.apache.arrow.vector.schema;
 
 import static java.util.Arrays.asList;
-import static org.apache.arrow.flatbuf.Precision.DOUBLE;
-import static org.apache.arrow.flatbuf.Precision.HALF;
-import static org.apache.arrow.flatbuf.Precision.SINGLE;
 import static org.apache.arrow.vector.schema.VectorLayout.booleanVector;
 import static org.apache.arrow.vector.schema.VectorLayout.byteVector;
 import static org.apache.arrow.vector.schema.VectorLayout.dataVector;
@@ -32,8 +29,6 @@
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.arrow.flatbuf.IntervalUnit;
-import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor;
 import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
@@ -44,7 +39,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.ArrowType.Time;
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
@@ -72,7 +67,7 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
       @Override public TypeLayout visit(Union type) {
         List<VectorLayout> vectors;
         switch (type.getMode()) {
-          case UnionMode.Dense:
+          case Dense:
             vectors = asList(
                 // TODO: validate this
                 validityVector(),
@@ -80,7 +75,7 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
                 offsetVector() // offset to find the vector
                 );
             break;
-          case UnionMode.Sparse:
+          case Sparse:
             vectors = asList(
                 typeVector() // type of the value at the index or 0 if null
                 );
@@ -91,7 +86,7 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
         return new TypeLayout(vectors);
       }
 
-      @Override public TypeLayout visit(Struct_ type) {
+      @Override public TypeLayout visit(Struct type) {
         List<VectorLayout> vectors = asList(
             validityVector()
             );
@@ -175,9 +170,9 @@ public TypeLayout visit(Time type) {
       @Override
       public TypeLayout visit(Interval type) { // TODO: check size
         switch (type.getUnit()) {
-        case IntervalUnit.DAY_TIME:
+        case DAY_TIME:
           return newFixedWidthTypeLayout(dataVector(64));
-        case IntervalUnit.YEAR_MONTH:
+        case YEAR_MONTH:
           return newFixedWidthTypeLayout(dataVector(64));
         default:
           throw new UnsupportedOperationException("Unknown unit " + type.getUnit());
@@ -215,7 +210,7 @@ public List<ArrowVectorType> getVectorTypes() {
   }
 
   public String toString() {
-    return "TypeLayout{" + vectors + "}";
+    return vectors.toString();
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
index 931c00a02817b..2073795b2a199 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
@@ -101,7 +101,7 @@ public ArrowVectorType getType() {
 
   @Override
   public String toString() {
-    return String.format("{width=%s,type=%s}", typeBitWidth, type);
+    return String.format("%s(%s)", type, typeBitWidth);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java b/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java
new file mode 100644
index 0000000000000..3206969fb7ead
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java
@@ -0,0 +1,47 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.types;
+
+import org.apache.arrow.flatbuf.Precision;
+
+public enum FloatingPointPrecision {
+  HALF(Precision.HALF),
+  SINGLE(Precision.SINGLE),
+  DOUBLE(Precision.DOUBLE);
+
+  private static final FloatingPointPrecision[] valuesByFlatbufId = new FloatingPointPrecision[FloatingPointPrecision.values().length];
+  static {
+    for (FloatingPointPrecision v : FloatingPointPrecision.values()) {
+      valuesByFlatbufId[v.flatbufID] = v;
+    }
+  }
+
+  private short flatbufID;
+
+  private FloatingPointPrecision(short flatbufID) {
+    this.flatbufID = flatbufID;
+  }
+
+  public short getFlatbufID() {
+    return flatbufID;
+  }
+
+  public static FloatingPointPrecision fromFlatbufID(short id) {
+    return valuesByFlatbufId[id];
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java b/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java
new file mode 100644
index 0000000000000..b3ddf1fe497de
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java
@@ -0,0 +1,44 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.types;
+
+public enum IntervalUnit {
+  YEAR_MONTH(org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH),
+  DAY_TIME(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME);
+
+  private static final IntervalUnit[] valuesByFlatbufId = new IntervalUnit[IntervalUnit.values().length];
+  static {
+    for (IntervalUnit v : IntervalUnit.values()) {
+      valuesByFlatbufId[v.flatbufID] = v;
+    }
+  }
+
+  private short flatbufID;
+
+  private IntervalUnit(short flatbufID) {
+    this.flatbufID = flatbufID;
+  }
+
+  public short getFlatbufID() {
+    return flatbufID;
+  }
+
+  public static IntervalUnit fromFlatbufID(short id) {
+    return valuesByFlatbufId[id];
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/TimeUnit.java b/java/vector/src/main/java/org/apache/arrow/vector/types/TimeUnit.java
new file mode 100644
index 0000000000000..cea9866965854
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/TimeUnit.java
@@ -0,0 +1,46 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.types;
+
+public enum TimeUnit {
+  SECOND(org.apache.arrow.flatbuf.TimeUnit.SECOND),
+  MILLISECOND(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND),
+  MICROSECOND(org.apache.arrow.flatbuf.TimeUnit.MICROSECOND),
+  NANOSECOND(org.apache.arrow.flatbuf.TimeUnit.NANOSECOND);
+
+  private static final TimeUnit[] valuesByFlatbufId = new TimeUnit[TimeUnit.values().length];
+  static {
+    for (TimeUnit v : TimeUnit.values()) {
+      valuesByFlatbufId[v.flatbufID] = v;
+    }
+  }
+
+  private final short flatbufID;
+
+  TimeUnit(short flatbufID) {
+    this.flatbufID = flatbufID;
+  }
+
+  public short getFlatbufID() {
+    return flatbufID;
+  }
+
+  public static TimeUnit fromFlatbufID(short id) {
+    return valuesByFlatbufId[id];
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index d9593673156bf..2a2fb74bee85c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -17,10 +17,10 @@
  */
 package org.apache.arrow.vector.types;
 
-import org.apache.arrow.flatbuf.IntervalUnit;
-import org.apache.arrow.flatbuf.Precision;
-import org.apache.arrow.flatbuf.TimeUnit;
-import org.apache.arrow.flatbuf.UnionMode;
+import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE;
+import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE;
+import static org.apache.arrow.vector.types.UnionMode.Sparse;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.NullableBigIntVector;
@@ -81,7 +81,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
 import org.apache.arrow.vector.types.pojo.ArrowType.List;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.ArrowType.Time;
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
@@ -102,11 +102,11 @@ public class Types {
   private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
   private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
   private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
-  private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null);
+  private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null);
   private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
   private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
-  private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(Precision.SINGLE), null);
-  private static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(Precision.DOUBLE), null);
+  private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);
+  private static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.DOUBLE), null);
   private static final Field VARCHAR_FIELD = new Field("", true, Utf8.INSTANCE, null);
   private static final Field VARBINARY_FIELD = new Field("", true, Binary.INSTANCE, null);
   private static final Field BIT_FIELD = new Field("", true, Bool.INSTANCE, null);
@@ -129,7 +129,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return null;
       }
     },
-    MAP(Struct_.INSTANCE) {
+    MAP(Struct.INSTANCE) {
       @Override
       public Field getField() {
         throw new UnsupportedOperationException("Cannot get simple field for Map type");
@@ -242,7 +242,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
-    TIMESTAMP(new Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND)) {
+    TIMESTAMP(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) {
       @Override
       public Field getField() {
         return TIMESTAMP_FIELD;
@@ -291,7 +291,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     //  4 byte ieee 754
-    FLOAT4(new FloatingPoint(Precision.SINGLE)) {
+    FLOAT4(new FloatingPoint(SINGLE)) {
       @Override
       public Field getField() {
         return FLOAT4_FIELD;
@@ -308,7 +308,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     //  8 byte ieee 754
-    FLOAT8(new FloatingPoint(Precision.DOUBLE)) {
+    FLOAT8(new FloatingPoint(DOUBLE)) {
       @Override
       public Field getField() {
         return FLOAT8_FIELD;
@@ -472,7 +472,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UnionListWriter((ListVector) vector);
       }
     },
-    UNION(new Union(UnionMode.Sparse, null)) {
+    UNION(new Union(Sparse, null)) {
       @Override
       public Field getField() {
         throw new UnsupportedOperationException("Cannot get simple field for Union type");
@@ -512,7 +512,7 @@ public static MinorType getMinorTypeForArrowType(ArrowType arrowType) {
         return MinorType.NULL;
       }
 
-      @Override public MinorType visit(Struct_ type) {
+      @Override public MinorType visit(Struct type) {
         return MinorType.MAP;
       }
 
@@ -543,11 +543,11 @@ public MinorType visit(Int type) {
       @Override
       public MinorType visit(FloatingPoint type) {
         switch (type.getPrecision()) {
-        case Precision.HALF:
+        case HALF:
           throw new UnsupportedOperationException("NYI: " + type);
-        case Precision.SINGLE:
+        case SINGLE:
           return MinorType.FLOAT4;
-        case Precision.DOUBLE:
+        case DOUBLE:
           return MinorType.FLOAT8;
         default:
           throw new IllegalArgumentException("unknown precision: " + type);
@@ -588,9 +588,9 @@ public MinorType visit(FloatingPoint type) {
       @Override
       public MinorType visit(Interval type) {
         switch (type.getUnit()) {
-        case IntervalUnit.DAY_TIME:
+        case DAY_TIME:
           return MinorType.INTERVALDAY;
-        case IntervalUnit.YEAR_MONTH:
+        case YEAR_MONTH:
           return MinorType.INTERVALYEAR;
         default:
           throw new IllegalArgumentException("unknown unit: " + type);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/UnionMode.java b/java/vector/src/main/java/org/apache/arrow/vector/types/UnionMode.java
new file mode 100644
index 0000000000000..8e957bc0b6e34
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/UnionMode.java
@@ -0,0 +1,44 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.types;
+
+public enum UnionMode {
+  Sparse(org.apache.arrow.flatbuf.UnionMode.Sparse),
+  Dense(org.apache.arrow.flatbuf.UnionMode.Dense);
+
+  private static final UnionMode[] valuesByFlatbufId = new UnionMode[UnionMode.values().length];
+  static {
+    for (UnionMode v : UnionMode.values()) {
+      valuesByFlatbufId[v.flatbufID] = v;
+    }
+  }
+
+  private final short flatbufID;
+
+  private UnionMode(short flatbufID) {
+    this.flatbufID = flatbufID;
+  }
+
+  public short getFlatbufID() {
+    return flatbufID;
+  }
+
+  public static UnionMode fromFlatbufID(short id) {
+    return valuesByFlatbufId[id];
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 49ba524ab0a4f..412fc54b538da 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -29,6 +29,7 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.flatbuffers.FlatBufferBuilder;
 
@@ -104,7 +105,7 @@ public int getField(FlatBufferBuilder builder) {
       org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
     }
     org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
-    org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeType());
+    org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeID().getFlatbufID());
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
     org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
     org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
@@ -143,11 +144,18 @@ public boolean equals(Object obj) {
             (Objects.equals(this.children, that.children) ||
                     (this.children == null && that.children.size() == 0) ||
                     (this.children.size() == 0 && that.children == null));
-
   }
 
   @Override
   public String toString() {
-    return String.format("Field{name=%s, type=%s, children=%s, layout=%s}", name, type, children, typeLayout);
+    StringBuilder sb = new StringBuilder();
+    if (name != null) {
+      sb.append(name).append(": ");
+    }
+    sb.append(type);
+    if (!children.isEmpty()) {
+      sb.append("<").append(Joiner.on(", ").join(children)).append(">");
+    }
+    return sb.toString();
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
index 44b877eb730d5..5ca8ade7891ee 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -33,6 +33,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.flatbuffers.FlatBufferBuilder;
 
@@ -132,6 +133,6 @@ public boolean equals(Object obj) {
 
   @Override
   public String toString() {
-    return "Schema" + fields;
+    return "Schema<" + Joiner.on(", ").join(fields) + ">";
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 176ad5195b3a1..58312b3f9ff9c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -21,15 +21,14 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
-import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.DirtyRootAllocator;
-import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
 import org.junit.Before;
@@ -112,8 +111,8 @@ public void testPromoteToUnion() throws Exception {
 
       Field childField1 = container.getField().getChildren().get(0).getChildren().get(0);
       Field childField2 = container.getField().getChildren().get(0).getChildren().get(1);
-      assertEquals("Child field should be union type: " + childField1.getName(), Type.Union, childField1.getType().getTypeType());
-      assertEquals("Child field should be decimal type: " + childField2.getName(), Type.Decimal, childField2.getType().getTypeType());
+      assertEquals("Child field should be union type: " + childField1.getName(), ArrowTypeID.Union, childField1.getType().getTypeID());
+      assertEquals("Child field should be decimal type: " + childField2.getName(), ArrowTypeID.Decimal, childField2.getType().getTypeID());
     }
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 6e0e617f299f8..caa438aff4761 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -41,6 +41,7 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
@@ -429,7 +430,7 @@ public void promotableWriter() {
     }
     Field field = parent.getField().getChildren().get(0).getChildren().get(0);
     Assert.assertEquals("a", field.getName());
-    Assert.assertEquals(Int.TYPE_TYPE, field.getType().getTypeType());
+    Assert.assertEquals(Int.TYPE_TYPE, field.getType().getTypeID());
     Int intType = (Int) field.getType();
 
     Assert.assertEquals(64, intType.getBitWidth());
@@ -444,9 +445,9 @@ public void promotableWriter() {
     }
     field = parent.getField().getChildren().get(0).getChildren().get(0);
     Assert.assertEquals("a", field.getName());
-    Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeType());
-    Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeType());
-    Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeType());
+    Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeID());
+    Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeID());
+    Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeID());
     MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
     for (int i = 0; i < 100; i++) {
       rootReader.setPosition(i);
@@ -476,12 +477,12 @@ public void promotableWriterSchema() {
 
     Field field = parent.getField().getChildren().get(0).getChildren().get(0);
     Assert.assertEquals("a", field.getName());
-    Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeType());
+    Assert.assertEquals(ArrowTypeID.Union, field.getType().getTypeID());
 
-    Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeType());
+    Assert.assertEquals(ArrowTypeID.Int, field.getChildren().get(0).getType().getTypeID());
     Int intType = (Int) field.getChildren().get(0).getType();
     Assert.assertEquals(64, intType.getBitWidth());
     Assert.assertTrue(intType.getIsSigned());
-    Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeType());
+    Assert.assertEquals(ArrowTypeID.Utf8, field.getChildren().get(1).getType().getTypeID());
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 3da8db298b4a3..5a238bcc0d0c3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -17,17 +17,17 @@
  */
 package org.apache.arrow.vector.pojo;
 
-import static org.apache.arrow.flatbuf.Precision.DOUBLE;
-import static org.apache.arrow.flatbuf.Precision.SINGLE;
+import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE;
+import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE;
 import static org.junit.Assert.assertEquals;
 
-import org.apache.arrow.flatbuf.TimeUnit;
-import org.apache.arrow.flatbuf.UnionMode;
+import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.UnionMode;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.List;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
@@ -55,7 +55,7 @@ public void complex() {
     childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
     childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
 
-    Field initialField = new Field("a", true, Struct_.INSTANCE, childrenBuilder.build());
+    Field initialField = new Field("a", true, Struct.INSTANCE, childrenBuilder.build());
     run(initialField);
   }
 
@@ -73,7 +73,7 @@ public void nestedSchema() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
     childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
-    childrenBuilder.add(new Field("child3", true, new Struct_(), ImmutableList.<Field>of(
+    childrenBuilder.add(new Field("child3", true, new Struct(), ImmutableList.<Field>of(
         new Field("child3.1", true, Utf8.INSTANCE, null),
         new Field("child3.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 0ef8be7ef1b8a..d60d17ea76db8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -23,10 +23,10 @@
 
 import java.io.IOException;
 
-import org.apache.arrow.flatbuf.IntervalUnit;
-import org.apache.arrow.flatbuf.Precision;
-import org.apache.arrow.flatbuf.TimeUnit;
-import org.apache.arrow.flatbuf.UnionMode;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.IntervalUnit;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.UnionMode;
 import org.junit.Test;
 
 public class TestSchema {
@@ -39,15 +39,33 @@ private static Field field(String name, ArrowType type, Field... children) {
     return field(name, true, type, children);
   }
 
+  @Test
+  public void testComplex() throws IOException {
+    Schema schema = new Schema(asList(
+        field("a", false, new ArrowType.Int(8, true)),
+        field("b", new ArrowType.Struct(),
+            field("c", new ArrowType.Int(16, true)),
+            field("d", new ArrowType.Utf8())),
+        field("e", new ArrowType.List(), field(null, new ArrowType.Date())),
+        field("f", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
+        field("g", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
+        field("h", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        ));
+    roundTrip(schema);
+    assertEquals(
+        "Schema<a: Int(8, true), b: Struct<c: Int(16, true), d: Utf8>, e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND), h: Interval(DAY_TIME)>",
+        schema.toString());
+  }
+
   @Test
   public void testAll() throws IOException {
     Schema schema = new Schema(asList(
         field("a", false, new ArrowType.Null()),
-        field("b", new ArrowType.Struct_(), field("ba", new ArrowType.Null())),
+        field("b", new ArrowType.Struct(), field("ba", new ArrowType.Null())),
         field("c", new ArrowType.List(), field("ca", new ArrowType.Null())),
         field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new ArrowType.Null())),
         field("e", new ArrowType.Int(8, true)),
-        field("f", new ArrowType.FloatingPoint(Precision.SINGLE)),
+        field("f", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
         field("g", new ArrowType.Utf8()),
         field("h", new ArrowType.Binary()),
         field("i", new ArrowType.Bool()),
@@ -94,9 +112,9 @@ public void testInterval() throws IOException {
   @Test
   public void testFP() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", new ArrowType.FloatingPoint(Precision.HALF)),
-        field("b", new ArrowType.FloatingPoint(Precision.SINGLE)),
-        field("c", new ArrowType.FloatingPoint(Precision.DOUBLE))
+        field("a", new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)),
+        field("b", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
+        field("c", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))
         ));
     roundTrip(schema);
     contains(schema, "HALF", "SINGLE", "DOUBLE");

From b5de9e56db08480050445dd883643448af12b81b Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Fri, 2 Dec 2016 14:34:47 -0500
Subject: [PATCH 0214/1644] ARROW-369: [Python] Convert multiple record batches
 at once to Pandas

Modified Pandas adapter to handle columns with multiple chunks with `ConvertColumnToPandas`.  This modifies the pyarrow public API by adding a class `RecordBatchList` and static method `toPandas` which takes a list of Arrow RecordBatches and outputs a Pandas DataFrame.

Adds unit test in test_table.py to do the conversion for each column with typed specialization.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #216 from BryanCutler/multi-batch-toPandas-ARROW-369 and squashes the following commits:

b6c9986 [Bryan Cutler] fixed formatting
edf056e [Bryan Cutler] simplified with pyarrow.schema.Schema.equals
068bc1b [Bryan Cutler] Merge remote-tracking branch 'upstream/master' into multi-batch-toPandas-ARROW-369
da65345 [Bryan Cutler] fixed test case for schema checking
9edb0ba [Bryan Cutler] used auto keyword where some typecasting was done in ConvertValues
bd2a720 [Bryan Cutler] added testcase for schema not equal, disabled now
c3d7e8f [Bryan Cutler] Changed conversion to make Table from columns first, now conversion is now just a free function
3ee51e6 [Bryan Cutler] cleanup
398b18d [Bryan Cutler] Fixed case for Integer specialization without nulls
7b29a55 [Bryan Cutler] Initial working version of RecordBatch list to_pandas, need more tests and cleanup
---
 python/pyarrow/__init__.py            |   4 +-
 python/pyarrow/includes/libarrow.pxd  |   3 +
 python/pyarrow/table.pyx              |  47 ++++++
 python/pyarrow/tests/test_table.py    |  35 +++++
 python/src/pyarrow/adapters/pandas.cc | 209 ++++++++++++++++----------
 5 files changed, 219 insertions(+), 79 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 775ce7ec47578..d4d0f00c52346 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -41,5 +41,7 @@
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
-from pyarrow.table import Column, RecordBatch, Table, from_pandas_dataframe
+from pyarrow.table import (Column, RecordBatch, dataframe_from_batches, Table,
+                           from_pandas_dataframe)
+
 from pyarrow.version import version as __version__
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 19da4085e1bb9..350ebe30c9b89 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -158,6 +158,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CColumn(const shared_ptr[CField]& field,
                 const shared_ptr[CArray]& data)
 
+        CColumn(const shared_ptr[CField]& field,
+                const vector[shared_ptr[CArray]]& chunks)
+
         int64_t length()
         int64_t null_count()
         const c_string& name()
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index a6715b141ce41..45cf7becceefa 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -28,6 +28,7 @@ cimport pyarrow.includes.pyarrow as pyarrow
 import pyarrow.config
 
 from pyarrow.array cimport Array, box_arrow_array
+from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
 from pyarrow.schema cimport box_data_type, box_schema
 
@@ -414,6 +415,52 @@ cdef class RecordBatch:
         return result
 
 
+def dataframe_from_batches(batches):
+    """
+    Convert a list of Arrow RecordBatches to a pandas.DataFrame
+
+    Parameters
+    ----------
+
+    batches: list of RecordBatch
+        RecordBatch list to be converted, schemas must be equal
+    """
+
+    cdef:
+        vector[shared_ptr[CArray]] c_array_chunks
+        vector[shared_ptr[CColumn]] c_columns
+        shared_ptr[CTable] c_table
+        Array arr
+        Schema schema
+
+    import pandas as pd
+
+    schema = batches[0].schema
+
+    # check schemas are equal
+    if any((not schema.equals(other.schema) for other in batches[1:])):
+        raise ArrowException("Error converting list of RecordBatches to "
+                "DataFrame, not all schemas are equal")
+
+    cdef int K = batches[0].num_columns
+
+    # create chunked columns from the batches
+    c_columns.resize(K)
+    for i in range(K):
+        for batch in batches:
+            arr = batch[i]
+            c_array_chunks.push_back(arr.sp_array)
+        c_columns[i].reset(new CColumn(schema.sp_schema.get().field(i),
+                           c_array_chunks))
+        c_array_chunks.clear()
+
+    # create a Table from columns and convert to DataFrame
+    c_table.reset(new CTable('', schema.sp_schema, c_columns))
+    table = Table()
+    table.init(c_table)
+    return table.to_pandas()
+
+
 cdef class Table:
     """
     A collection of top-level named, equal length Arrow arrays.
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 4c9d302106af8..dc4f37a830e5f 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -19,6 +19,7 @@
 
 from pandas.util.testing import assert_frame_equal
 import pandas as pd
+import pytest
 
 import pyarrow as pa
 
@@ -50,6 +51,40 @@ def test_recordbatch_from_to_pandas():
     assert_frame_equal(data, result)
 
 
+def test_recordbatchlist_to_pandas():
+    data1 = pd.DataFrame({
+        'c1': np.array([1, 1, 2], dtype='uint32'),
+        'c2': np.array([1.0, 2.0, 3.0], dtype='float64'),
+        'c3': [True, None, False],
+        'c4': ['foo', 'bar', None]
+    })
+
+    data2 = pd.DataFrame({
+        'c1': np.array([3, 5], dtype='uint32'),
+        'c2': np.array([4.0, 5.0], dtype='float64'),
+        'c3': [True, True],
+        'c4': ['baz', 'qux']
+    })
+
+    batch1 = pa.RecordBatch.from_pandas(data1)
+    batch2 = pa.RecordBatch.from_pandas(data2)
+
+    result = pa.dataframe_from_batches([batch1, batch2])
+    data = pd.concat([data1, data2], ignore_index=True)
+    assert_frame_equal(data, result)
+
+
+def test_recordbatchlist_schema_equals():
+    data1 = pd.DataFrame({'c1': np.array([1], dtype='uint32')})
+    data2 = pd.DataFrame({'c1': np.array([4.0, 5.0], dtype='float64')})
+
+    batch1 = pa.RecordBatch.from_pandas(data1)
+    batch2 = pa.RecordBatch.from_pandas(data2)
+
+    with pytest.raises(pa.ArrowException):
+        pa.dataframe_from_batches([batch1, batch2])
+
+
 def test_table_basics():
     data = [
         pa.from_pylist(range(5)),
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 1f5b7009e6a08..adb27e83ef120 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -597,14 +597,10 @@ class ArrowDeserializer {
 
   Status Convert(PyObject** out) {
     const std::shared_ptr<arrow::ChunkedArray> data = col_->data();
-    if (data->num_chunks() > 1) {
-      return Status::NotImplemented("Chunked column conversion NYI");
-    }
-
-    auto chunk = data->chunk(0);
 
-    RETURN_NOT_OK(ConvertValues<TYPE>(chunk));
+    RETURN_NOT_OK(ConvertValues<TYPE>(data));
     *out = reinterpret_cast<PyObject*>(out_);
+
     return Status::OK();
   }
 
@@ -653,28 +649,49 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
+  template <int T2>
+  Status ConvertValuesZeroCopy(std::shared_ptr<Array> arr) {
+    typedef typename arrow_traits<T2>::T T;
+
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+
+    // Zero-Copy. We can pass the data pointer directly to NumPy.
+    void* data = const_cast<T*>(in_values);
+    int type = arrow_traits<TYPE>::npy_type;
+    RETURN_NOT_OK(OutputFromData(type, data));
+
+    return Status::OK();
+  }
+
   template <int T2>
   inline typename std::enable_if<
     arrow_traits<T2>::is_pandas_numeric_nullable, Status>::type
-  ConvertValues(const std::shared_ptr<Array>& arr) {
+  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
     typedef typename arrow_traits<T2>::T T;
+    size_t chunk_offset = 0;
 
-    arrow::PrimitiveArray* prim_arr = static_cast<arrow::PrimitiveArray*>(
-        arr.get());
-    const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    if (data->num_chunks() == 1 && data->null_count() == 0) {
+      return ConvertValuesZeroCopy<TYPE>(data->chunk(0));
+    }
+
+    RETURN_NOT_OK(AllocateOutput(arrow_traits<T2>::npy_type));
 
-    if (arr->null_count() > 0) {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<T2>::npy_type));
+    for (int c = 0; c < data->num_chunks(); c++) {
+      const std::shared_ptr<Array> arr = data->chunk(c);
+      auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+      auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+      auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_)) + chunk_offset;
 
-      T* out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        out_values[i] = arr->IsNull(i) ? arrow_traits<T2>::na_value : in_values[i];
+      if (arr->null_count() > 0) {
+        for (int64_t i = 0; i < arr->length(); ++i) {
+          out_values[i] = arr->IsNull(i) ? arrow_traits<T2>::na_value : in_values[i];
+        }
+      } else {
+        memcpy(out_values, in_values, sizeof(T) * arr->length());
       }
-    } else {
-      // Zero-Copy. We can pass the data pointer directly to NumPy.
-      void* data = const_cast<T*>(in_values);
-      int type = arrow_traits<TYPE>::npy_type;
-      RETURN_NOT_OK(OutputFromData(type, data));
+
+      chunk_offset += arr->length();
     }
 
     return Status::OK();
@@ -684,27 +701,43 @@ class ArrowDeserializer {
   template <int T2>
   inline typename std::enable_if<
     arrow_traits<T2>::is_pandas_numeric_not_nullable, Status>::type
-  ConvertValues(const std::shared_ptr<Array>& arr) {
+  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
     typedef typename arrow_traits<T2>::T T;
+    size_t chunk_offset = 0;
 
-    arrow::PrimitiveArray* prim_arr = static_cast<arrow::PrimitiveArray*>(
-        arr.get());
-
-    const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    if (data->num_chunks() == 1 && data->null_count() == 0) {
+      return ConvertValuesZeroCopy<TYPE>(data->chunk(0));
+    }
 
-    if (arr->null_count() > 0) {
+    if (data->null_count() > 0) {
       RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
 
-      // Upcast to double, set NaN as appropriate
-      double* out_values = reinterpret_cast<double*>(PyArray_DATA(out_));
-      for (int i = 0; i < arr->length(); ++i) {
-        out_values[i] = prim_arr->IsNull(i) ? NAN : in_values[i];
+      for (int c = 0; c < data->num_chunks(); c++) {
+        const std::shared_ptr<Array> arr = data->chunk(c);
+        auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+        auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+        // Upcast to double, set NaN as appropriate
+        auto out_values = reinterpret_cast<double*>(PyArray_DATA(out_)) + chunk_offset;
+
+        for (int i = 0; i < arr->length(); ++i) {
+          out_values[i] = prim_arr->IsNull(i) ? NAN : in_values[i];
+        }
+
+        chunk_offset += arr->length();
       }
     } else {
-      // Zero-Copy. We can pass the data pointer directly to NumPy.
-      void* data = const_cast<T*>(in_values);
-      int type = arrow_traits<TYPE>::npy_type;
-      RETURN_NOT_OK(OutputFromData(type, data));
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+
+      for (int c = 0; c < data->num_chunks(); c++) {
+        const std::shared_ptr<Array> arr = data->chunk(c);
+        auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+        auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+        auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_)) + chunk_offset;
+
+        memcpy(out_values, in_values, sizeof(T) * arr->length());
+
+        chunk_offset += arr->length();
+      }
     }
 
     return Status::OK();
@@ -714,35 +747,48 @@ class ArrowDeserializer {
   template <int T2>
   inline typename std::enable_if<
     arrow_traits<T2>::is_boolean, Status>::type
-  ConvertValues(const std::shared_ptr<Array>& arr) {
+  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
+    size_t chunk_offset = 0;
     PyAcquireGIL lock;
 
-    arrow::BooleanArray* bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
-
-    if (arr->null_count() > 0) {
+    if (data->null_count() > 0) {
       RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
 
-      PyObject** out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        if (bool_arr->IsNull(i)) {
-          Py_INCREF(Py_None);
-          out_values[i] = Py_None;
-        } else if (bool_arr->Value(i)) {
-          // True
-          Py_INCREF(Py_True);
-          out_values[i] = Py_True;
-        } else {
-          // False
-          Py_INCREF(Py_False);
-          out_values[i] = Py_False;
+      for (int c = 0; c < data->num_chunks(); c++) {
+        const std::shared_ptr<Array> arr = data->chunk(c);
+        auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
+        auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_)) + chunk_offset;
+
+        for (int64_t i = 0; i < arr->length(); ++i) {
+          if (bool_arr->IsNull(i)) {
+            Py_INCREF(Py_None);
+            out_values[i] = Py_None;
+          } else if (bool_arr->Value(i)) {
+            // True
+            Py_INCREF(Py_True);
+            out_values[i] = Py_True;
+          } else {
+            // False
+            Py_INCREF(Py_False);
+            out_values[i] = Py_False;
+          }
         }
+
+        chunk_offset += bool_arr->length();
       }
     } else {
       RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
 
-      uint8_t* out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(out_));
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        out_values[i] = static_cast<uint8_t>(bool_arr->Value(i));
+      for (int c = 0; c < data->num_chunks(); c++) {
+        const std::shared_ptr<Array> arr = data->chunk(c);
+        auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
+        auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(out_)) + chunk_offset;
+
+        for (int64_t i = 0; i < arr->length(); ++i) {
+          out_values[i] = static_cast<uint8_t>(bool_arr->Value(i));
+        }
+
+        chunk_offset += bool_arr->length();
       }
     }
 
@@ -753,42 +799,49 @@ class ArrowDeserializer {
   template <int T2>
   inline typename std::enable_if<
     T2 == arrow::Type::STRING, Status>::type
-  ConvertValues(const std::shared_ptr<Array>& arr) {
+  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
+    size_t chunk_offset = 0;
     PyAcquireGIL lock;
 
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
 
-    PyObject** out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
-
-    arrow::StringArray* string_arr = static_cast<arrow::StringArray*>(arr.get());
-
-    const uint8_t* data;
-    int32_t length;
-    if (arr->null_count() > 0) {
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        if (string_arr->IsNull(i)) {
-          Py_INCREF(Py_None);
-          out_values[i] = Py_None;
-        } else {
-          data = string_arr->GetValue(i, &length);
-
-          out_values[i] = make_pystring(data, length);
+    for (int c = 0; c < data->num_chunks(); c++) {
+      const std::shared_ptr<Array> arr = data->chunk(c);
+      auto string_arr = static_cast<arrow::StringArray*>(arr.get());
+      auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_)) + chunk_offset;
+
+      const uint8_t* data_ptr;
+      int32_t length;
+      if (data->null_count() > 0) {
+        for (int64_t i = 0; i < arr->length(); ++i) {
+          if (string_arr->IsNull(i)) {
+            Py_INCREF(Py_None);
+            out_values[i] = Py_None;
+          } else {
+            data_ptr = string_arr->GetValue(i, &length);
+
+            out_values[i] = make_pystring(data_ptr, length);
+            if (out_values[i] == nullptr) {
+              return Status::UnknownError("String initialization failed");
+            }
+          }
+        }
+      } else {
+        for (int64_t i = 0; i < arr->length(); ++i) {
+          data_ptr = string_arr->GetValue(i, &length);
+          out_values[i] = make_pystring(data_ptr, length);
           if (out_values[i] == nullptr) {
             return Status::UnknownError("String initialization failed");
           }
         }
       }
-    } else {
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        data = string_arr->GetValue(i, &length);
-        out_values[i] = make_pystring(data, length);
-        if (out_values[i] == nullptr) {
-          return Status::UnknownError("String initialization failed");
-        }
-      }
+
+      chunk_offset += string_arr->length();
     }
+
     return Status::OK();
   }
+
  private:
   std::shared_ptr<Column> col_;
   PyObject* py_ref_;

From 0ac01a5bf6747a5855d20632c9c7874483b9830a Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 4 Dec 2016 10:50:43 -0500
Subject: [PATCH 0215/1644] ARROW-379: Use setuptools_scm for Python versioning

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #224 from xhochy/ARROW-379 and squashes the following commits:

3a68d9f [Uwe L. Korn] Remove deprecated version import
15fe9b2 [Uwe L. Korn] Add license header
aa9bd49 [Uwe L. Korn] ARROW-379: Use setuptools_scm for Python versioning
---
 dev/release/00-prepare.sh  |  5 -----
 python/.git_archival.txt   |  1 +
 python/.gitattributes      |  1 +
 python/pyarrow/__init__.py | 10 ++++++++--
 python/setup.cfg           | 20 ++++++++++++++++++++
 python/setup.py            | 23 ++---------------------
 6 files changed, 32 insertions(+), 28 deletions(-)
 create mode 100644 python/.git_archival.txt
 create mode 100644 python/.gitattributes
 create mode 100644 python/setup.cfg

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
index 3423a3e6c5bf9..00af5e7768161 100644
--- a/dev/release/00-prepare.sh
+++ b/dev/release/00-prepare.sh
@@ -43,9 +43,4 @@ mvn release:prepare -Dtag=${tag} -DreleaseVersion=${version} -DautoVersionSubmod
 
 cd -
 
-cd "${SOURCE_DIR}/../../python"
-sed -i "s/VERSION = '[^']*'/VERSION = '${version}'/g" setup.py
-sed -i "s/ISRELEASED = False/ISRELEASED = True/g" setup.py
-cd -
-
 echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
diff --git a/python/.git_archival.txt b/python/.git_archival.txt
new file mode 100644
index 0000000000000..95cb3eea4e336
--- /dev/null
+++ b/python/.git_archival.txt
@@ -0,0 +1 @@
+ref-names: $Format:%D$
diff --git a/python/.gitattributes b/python/.gitattributes
new file mode 100644
index 0000000000000..00a7b00c94e08
--- /dev/null
+++ b/python/.gitattributes
@@ -0,0 +1 @@
+.git_archival.txt  export-subst
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index d4d0f00c52346..f366317d04c96 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -17,6 +17,14 @@
 
 # flake8: noqa
 
+from pkg_resources import get_distribution, DistributionNotFound
+try:
+    __version__ = get_distribution(__name__).version
+except DistributionNotFound:
+   # package is not installed
+   pass
+
+
 import pyarrow.config
 
 from pyarrow.array import (Array,
@@ -43,5 +51,3 @@
 
 from pyarrow.table import (Column, RecordBatch, dataframe_from_batches, Table,
                            from_pandas_dataframe)
-
-from pyarrow.version import version as __version__
diff --git a/python/setup.cfg b/python/setup.cfg
new file mode 100644
index 0000000000000..caae3e081b6ca
--- /dev/null
+++ b/python/setup.cfg
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[build_sphinx]
+source-dir = doc/
+build-dir  = doc/_build
diff --git a/python/setup.py b/python/setup.py
index 341cc64aa2cc8..0f6bbda6ec3aa 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -42,27 +42,9 @@
 if Cython.__version__ < '0.19.1':
     raise Exception('Please upgrade to Cython 0.19.1 or newer')
 
-VERSION = '0.1.0'
-ISRELEASED = False
-
-if not ISRELEASED:
-    VERSION += '.dev'
-
 setup_dir = os.path.abspath(os.path.dirname(__file__))
 
 
-def write_version_py(filename=os.path.join(setup_dir, 'pyarrow/version.py')):
-    a = open(filename, 'w')
-    file_content = "\n".join(["",
-                              "# THIS FILE IS GENERATED FROM SETUP.PY",
-                              "version = '%(version)s'",
-                              "isrelease = '%(isrelease)s'"])
-
-    a.write(file_content % {'version': VERSION,
-                            'isrelease': str(ISRELEASED)})
-    a.close()
-
-
 class clean(_clean):
 
     def run(self):
@@ -261,15 +243,12 @@ def get_outputs(self):
         return [self._get_cmake_ext_path(name)
                 for name in self.get_names()]
 
-write_version_py()
-
 DESC = """\
 Python library for Apache Arrow"""
 
 setup(
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
-    version=VERSION,
     zip_safe=False,
     package_data={'pyarrow': ['*.pxd', '*.pyx']},
     # Dummy extension to trigger build_ext
@@ -279,6 +258,8 @@ def get_outputs(self):
         'clean': clean,
         'build_ext': build_ext
     },
+    use_scm_version = {"root": "..", "relative_to": __file__},
+    setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'],
     install_requires=['cython >= 0.23', 'numpy >= 1.9', 'six >= 1.0.0'],
     description=DESC,
     license='Apache License, Version 2.0',

From 599d516a7306de4d1f9d7e0ddc888f13026efd49 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Mon, 5 Dec 2016 14:56:56 -0800
Subject: [PATCH 0216/1644] =?UTF-8?q?ARROW-401:=20Floating=20point=20vecto?=
 =?UTF-8?q?rs=20should=20do=20an=20approximate=20comparison=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… in integration tests

Author: Julien Le Dem <julien@dremio.com>

Closes #223 from julienledem/arrow_401 and squashes the following commits:

a9ee84d [Julien Le Dem] review feedback
da64ca0 [Julien Le Dem] ARROW-401: Floating point vectors should do an approximate comparison in integration tests
---
 .../org/apache/arrow/tools/Integration.java   | 51 ++++++++++-
 .../apache/arrow/tools/TestIntegration.java   | 84 ++++++++++++++++++-
 2 files changed, 130 insertions(+), 5 deletions(-)

diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index 85af30da1e8ae..fd835a63a11ac 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -39,6 +39,8 @@
 import org.apache.arrow.vector.file.json.JsonFileReader;
 import org.apache.arrow.vector.file.json.JsonFileWriter;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.commons.cli.CommandLine;
@@ -247,7 +249,7 @@ private static void compare(VectorSchemaRoot arrowRoot, VectorSchemaRoot jsonRoo
       for (int j = 0; j < valueCount; j++) {
         Object arrow = arrowVector.getAccessor().getObject(j);
         Object json = jsonVector.getAccessor().getObject(j);
-        if (!Objects.equal(arrow, json)) {
+        if (!equals(field.getType(), arrow, json)) {
           throw new IllegalArgumentException(
               "Different values in column:\n" + field + " at index " + j + ": " + arrow + " != " + json);
         }
@@ -255,6 +257,53 @@ private static void compare(VectorSchemaRoot arrowRoot, VectorSchemaRoot jsonRoo
     }
   }
 
+  private static boolean equals(ArrowType type, final Object arrow, final Object json) {
+    if (type instanceof ArrowType.FloatingPoint) {
+      FloatingPoint fpType = (FloatingPoint) type;
+      switch (fpType.getPrecision()) {
+      case DOUBLE:
+        return equalEnough((Double)arrow, (Double)json);
+      case SINGLE:
+        return equalEnough((Float)arrow, (Float)json);
+      case HALF:
+      default:
+        throw new UnsupportedOperationException("unsupported precision: " + fpType);
+      }
+    }
+    return Objects.equal(arrow, json);
+  }
+
+  static boolean equalEnough(Float f1, Float f2) {
+    if (f1 == null || f2 == null) {
+      return f1 == null && f2 == null;
+    }
+    if (f1.isNaN()) {
+      return f2.isNaN();
+    }
+    if (f1.isInfinite()) {
+      return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
+    }
+    float average = Math.abs((f1 + f2) / 2);
+    float differenceScaled = Math.abs(f1 - f2) / (average == 0.0f ? 1f : average);
+    return differenceScaled < 1.0E-6f;
+  }
+
+  static boolean equalEnough(Double f1, Double f2) {
+    if (f1 == null || f2 == null) {
+      return f1 == null && f2 == null;
+    }
+    if (f1.isNaN()) {
+      return f2.isNaN();
+    }
+    if (f1.isInfinite()) {
+      return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
+    }
+    double average = Math.abs((f1 + f2) / 2);
+    double differenceScaled = Math.abs(f1 - f2) / (average == 0.0d ? 1d : average);
+    return differenceScaled < 1.0E-12d;
+  }
+
+
   private static void compareSchemas(Schema jsonSchema, Schema arrowSchema) {
     if (!arrowSchema.equals(jsonSchema)) {
       throw new IllegalArgumentException("Different schemas:\n" + arrowSchema + "\n" + jsonSchema);
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
index 464144b95a1aa..ee6196b74e0dc 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -22,6 +22,10 @@
 import static org.apache.arrow.tools.ArrowFileTestFixtures.write;
 import static org.apache.arrow.tools.ArrowFileTestFixtures.writeData;
 import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
+import static org.apache.arrow.tools.Integration.equalEnough;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.BufferedReader;
@@ -39,9 +43,9 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.Float8Writer;
 import org.apache.arrow.vector.complex.writer.IntWriter;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -121,7 +125,7 @@ public void testJSONRoundTripWithVariableWidth() throws Exception {
     String i, o;
     int j = 0;
     while ((i = orig.readLine()) != null && (o = rt.readLine()) != null) {
-      Assert.assertEquals("line: " + j, i, o);
+      assertEquals("line: " + j, i, o);
       ++j;
     }
   }
@@ -142,6 +146,33 @@ private BufferedReader readNormalized(File f) throws IOException {
   }
 
 
+  /**
+   * the test should not be sensitive to small variations in float representation
+   */
+  @Test
+  public void testFloat() throws Exception {
+    File testValidInFile = testFolder.newFile("testValidFloatIn.arrow");
+    File testInvalidInFile = testFolder.newFile("testAlsoValidFloatIn.arrow");
+    File testJSONFile = testFolder.newFile("testValidOut.json");
+    testJSONFile.delete();
+
+    // generate an arrow file
+    writeInputFloat(testValidInFile, allocator, 912.4140000000002, 912.414);
+    // generate a different arrow file
+    writeInputFloat(testInvalidInFile, allocator, 912.414, 912.4140000000002);
+
+    Integration integration = new Integration();
+
+    // convert the "valid" file to json
+    String[] args1 = { "-arrow", testValidInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    integration.run(args1);
+
+    // compare the "invalid" file to the "valid" json
+    String[] args3 = { "-arrow", testInvalidInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()};
+    // this should fail
+    integration.run(args3);
+  }
+
   @Test
   public void testInvalid() throws Exception {
     File testValidInFile = testFolder.newFile("testValidIn.arrow");
@@ -167,12 +198,28 @@ public void testInvalid() throws Exception {
       integration.run(args3);
       fail("should have failed");
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage(), e.getMessage().contains("Different values in column"));
-      Assert.assertTrue(e.getMessage(), e.getMessage().contains("999"));
+      assertTrue(e.getMessage(), e.getMessage().contains("Different values in column"));
+      assertTrue(e.getMessage(), e.getMessage().contains("999"));
     }
 
   }
 
+  static void writeInputFloat(File testInFile, BufferAllocator allocator, double... f) throws FileNotFoundException, IOException {
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+      ComplexWriter writer = new ComplexWriterImpl("root", parent);
+      MapWriter rootWriter = writer.rootAsMap();
+      Float8Writer floatWriter = rootWriter.float8("float");
+      for (int i = 0; i < f.length; i++) {
+        floatWriter.setPosition(i);
+        floatWriter.writeFloat8(f[i]);
+      }
+      writer.setValueCount(f.length);
+      write(parent.getChild("root"), testInFile);
+    }
+  }
+
   static void writeInput2(File testInFile, BufferAllocator allocator) throws FileNotFoundException, IOException {
     int count = ArrowFileTestFixtures.COUNT;
     try (
@@ -192,4 +239,33 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws FileN
     }
   }
 
+  @Test
+  public void testFloatComp() {
+    assertTrue(equalEnough(912.4140000000002F, 912.414F));
+    assertTrue(equalEnough(912.4140000000002D, 912.414D));
+    assertTrue(equalEnough(912.414F, 912.4140000000002F));
+    assertTrue(equalEnough(912.414D, 912.4140000000002D));
+    assertFalse(equalEnough(912.414D, 912.4140001D));
+    assertFalse(equalEnough(null, 912.414D));
+    assertTrue(equalEnough((Float)null, null));
+    assertTrue(equalEnough((Double)null, null));
+    assertFalse(equalEnough(912.414D, null));
+    assertFalse(equalEnough(Double.MAX_VALUE, Double.MIN_VALUE));
+    assertFalse(equalEnough(Double.MIN_VALUE, Double.MAX_VALUE));
+    assertTrue(equalEnough(Double.MAX_VALUE, Double.MAX_VALUE));
+    assertTrue(equalEnough(Double.MIN_VALUE, Double.MIN_VALUE));
+    assertTrue(equalEnough(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY));
+    assertFalse(equalEnough(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY));
+    assertTrue(equalEnough(Double.NaN, Double.NaN));
+    assertFalse(equalEnough(1.0, Double.NaN));
+    assertFalse(equalEnough(Float.MAX_VALUE, Float.MIN_VALUE));
+    assertFalse(equalEnough(Float.MIN_VALUE, Float.MAX_VALUE));
+    assertTrue(equalEnough(Float.MAX_VALUE, Float.MAX_VALUE));
+    assertTrue(equalEnough(Float.MIN_VALUE, Float.MIN_VALUE));
+    assertTrue(equalEnough(Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY));
+    assertFalse(equalEnough(Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY));
+    assertTrue(equalEnough(Float.NaN, Float.NaN));
+    assertFalse(equalEnough(1.0F, Float.NaN));
+  }
+
 }

From 82575ca3c22db18b2ea69f248b471a0317042b38 Mon Sep 17 00:00:00 2001
From: vkorukanti <venki@dremio.com>
Date: Mon, 5 Dec 2016 21:28:14 -0800
Subject: [PATCH 0217/1644] ARROW-403: [Java] Create transfer pairs for
 internal vectors in UnionVector transfer impl

@StevenMPhillips, @julienledem Could you please review the patch?

Author: vkorukanti <venki@dremio.com>

Closes #225 from vkorukanti/union_vector_schema and squashes the following commits:

431874f [vkorukanti] ARROW-403: [Java] Create transfer pairs for internal vectors in UnionVector transfer impl
---
 .../main/codegen/templates/UnionVector.java   | 19 +++----
 .../apache/arrow/vector/TestUnionVector.java  | 54 +++++++++++++++++++
 2 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index ea1fdf6bd60fb..4e68b681d1404 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -246,12 +246,6 @@ public TransferPair makeTransferPair(ValueVector target) {
     return new TransferImpl((UnionVector) target);
   }
 
-  public void transferTo(org.apache.arrow.vector.complex.UnionVector target) {
-    typeVector.makeTransferPair(target.typeVector).transfer();
-    internalMap.makeTransferPair(target.internalMap).transfer();
-    target.valueCount = valueCount;
-  }
-
   public void copyFrom(int inIndex, int outIndex, UnionVector from) {
     from.getReader().setPosition(inIndex);
     getWriter().setPosition(outIndex);
@@ -275,20 +269,27 @@ public FieldVector addVector(FieldVector v) {
   }
 
   private class TransferImpl implements TransferPair {
-
-    UnionVector to;
+    private final TransferPair internalMapVectorTransferPair;
+    private final TransferPair typeVectorTransferPair;
+    private final UnionVector to;
 
     public TransferImpl(String name, BufferAllocator allocator) {
       to = new UnionVector(name, allocator, null);
+      internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap);
+      typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector);
     }
 
     public TransferImpl(UnionVector to) {
       this.to = to;
+      internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap);
+      typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector);
     }
 
     @Override
     public void transfer() {
-      transferTo(to);
+      internalMapVectorTransferPair.transfer();
+      typeVectorTransferPair.transfer();
+      to.valueCount = valueCount;
     }
 
     @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index 1bb50b73a9057..a5b90ee90b8f9 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -21,8 +21,12 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.holders.NullableBitHolder;
+import org.apache.arrow.vector.holders.NullableIntHolder;
 import org.apache.arrow.vector.holders.NullableUInt4Holder;
 import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.TransferPair;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -76,4 +80,54 @@ public void testUnionVector() throws Exception {
     }
   }
 
+  @Test
+  public void testTransfer() throws Exception {
+    try (UnionVector srcVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      srcVector.allocateNew();
+
+      // write some data
+      final UnionVector.Mutator mutator = srcVector.getMutator();
+      mutator.setType(0, MinorType.INT);
+      mutator.setSafe(0, newIntHolder(5));
+      mutator.setType(1, MinorType.BIT);
+      mutator.setSafe(1, newBitHolder(false));
+      mutator.setType(3, MinorType.INT);
+      mutator.setSafe(3, newIntHolder(10));
+      mutator.setType(5, MinorType.BIT);
+      mutator.setSafe(5, newBitHolder(false));
+      mutator.setValueCount(6);
+
+      try(UnionVector destVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+        TransferPair pair = srcVector.makeTransferPair(destVector);
+
+        // Creating the transfer should transfer the type of the field at least.
+        assertEquals(srcVector.getField(), destVector.getField());
+
+        // transfer
+        pair.transfer();
+
+        assertEquals(srcVector.getField(), destVector.getField());
+
+        // now check the values are transferred
+        assertEquals(srcVector.getAccessor().getValueCount(), destVector.getAccessor().getValueCount());
+        for(int i=0; i<srcVector.getAccessor().getValueCount(); i++) {
+          assertEquals("Different values at index " + i, srcVector.getAccessor().get(i), destVector.getAccessor().get(i));
+        }
+      }
+    }
+  }
+
+  private static NullableIntHolder newIntHolder(int value) {
+    final NullableIntHolder holder = new NullableIntHolder();
+    holder.isSet = 1;
+    holder.value = value;
+    return holder;
+  }
+
+  private static NullableBitHolder newBitHolder(boolean value) {
+    final NullableBitHolder holder = new NullableBitHolder();
+    holder.isSet = 1;
+    holder.value = value ? 1 : 0;
+    return holder;
+  }
 }

From 4b72329fe2d731f445e44925783b9489f4e0d0d5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 6 Dec 2016 11:41:08 -0500
Subject: [PATCH 0218/1644] ARROW-406: [C++] Set explicit 64K HDFS buffer size,
 test large reads

We could not support reads in excess of the default buffer size (typically 64K)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #226 from wesm/ARROW-406 and squashes the following commits:

d09b645 [Wes McKinney] cpplint
0028e90 [Wes McKinney] Set explicit 64K HDFS buffer size, test large reads using buffered chunks
---
 cpp/src/arrow/io/hdfs.cc         | 33 +++++++++++++++++++++++++-------
 cpp/src/arrow/io/hdfs.h          |  3 +++
 cpp/src/arrow/io/io-hdfs-test.cc | 33 ++++++++++++++++++++++++++++++++
 3 files changed, 62 insertions(+), 7 deletions(-)

diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 13491e780e21b..8c6d49f92e606 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -17,6 +17,7 @@
 
 #include <hdfs.h>
 
+#include <algorithm>
 #include <cstdint>
 #include <sstream>
 #include <string>
@@ -51,6 +52,8 @@ static Status CheckReadResult(int ret) {
   return Status::OK();
 }
 
+static constexpr int kDefaultHdfsBufferSize = 1 << 16;
+
 // ----------------------------------------------------------------------
 // File reading
 
@@ -124,9 +127,16 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
   }
 
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
-    tSize ret = hdfsRead(fs_, file_, reinterpret_cast<void*>(buffer), nbytes);
-    RETURN_NOT_OK(CheckReadResult(ret));
-    *bytes_read = ret;
+    int64_t total_bytes = 0;
+    while (total_bytes < nbytes) {
+      tSize ret = hdfsRead(fs_, file_, reinterpret_cast<void*>(buffer + total_bytes),
+          std::min<int64_t>(buffer_size_, nbytes - total_bytes));
+      RETURN_NOT_OK(CheckReadResult(ret));
+      total_bytes += ret;
+      if (ret == 0) { break; }
+    }
+
+    *bytes_read = total_bytes;
     return Status::OK();
   }
 
@@ -136,7 +146,6 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
 
     int64_t bytes_read = 0;
     RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
-
     if (bytes_read < nbytes) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
 
     *out = buffer;
@@ -154,8 +163,11 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
 
   void set_memory_pool(MemoryPool* pool) { pool_ = pool; }
 
+  void set_buffer_size(int32_t buffer_size) { buffer_size_ = buffer_size; }
+
  private:
   MemoryPool* pool_;
+  int32_t buffer_size_;
 };
 
 HdfsReadableFile::HdfsReadableFile(MemoryPool* pool) {
@@ -384,8 +396,9 @@ class HdfsClient::HdfsClientImpl {
     return Status::OK();
   }
 
-  Status OpenReadable(const std::string& path, std::shared_ptr<HdfsReadableFile>* file) {
-    hdfsFile handle = hdfsOpenFile(fs_, path.c_str(), O_RDONLY, 0, 0, 0);
+  Status OpenReadable(const std::string& path, int32_t buffer_size,
+      std::shared_ptr<HdfsReadableFile>* file) {
+    hdfsFile handle = hdfsOpenFile(fs_, path.c_str(), O_RDONLY, buffer_size, 0, 0);
 
     if (handle == nullptr) {
       // TODO(wesm): determine cause of failure
@@ -397,6 +410,7 @@ class HdfsClient::HdfsClientImpl {
     // std::make_shared does not work with private ctors
     *file = std::shared_ptr<HdfsReadableFile>(new HdfsReadableFile());
     (*file)->impl_->set_members(path, fs_, handle);
+    (*file)->impl_->set_buffer_size(buffer_size);
 
     return Status::OK();
   }
@@ -490,9 +504,14 @@ Status HdfsClient::ListDirectory(
   return impl_->ListDirectory(path, listing);
 }
 
+Status HdfsClient::OpenReadable(const std::string& path, int32_t buffer_size,
+    std::shared_ptr<HdfsReadableFile>* file) {
+  return impl_->OpenReadable(path, buffer_size, file);
+}
+
 Status HdfsClient::OpenReadable(
     const std::string& path, std::shared_ptr<HdfsReadableFile>* file) {
-  return impl_->OpenReadable(path, file);
+  return OpenReadable(path, kDefaultHdfsBufferSize, file);
 }
 
 Status HdfsClient::OpenWriteable(const std::string& path, bool append,
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 48699c914503e..1c76f15c397ce 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -128,6 +128,9 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   // status if the file is not found.
   //
   // @param path complete file path
+  Status OpenReadable(const std::string& path, int32_t buffer_size,
+      std::shared_ptr<HdfsReadableFile>* file);
+
   Status OpenReadable(const std::string& path, std::shared_ptr<HdfsReadableFile>* file);
 
   // FileMode::WRITE options
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 7901932dee676..8338de6d96a55 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -293,6 +293,39 @@ TEST_F(TestHdfsClient, ReadableMethods) {
   ASSERT_EQ(60, position);
 }
 
+TEST_F(TestHdfsClient, LargeFile) {
+  SKIP_IF_NO_LIBHDFS();
+
+  ASSERT_OK(MakeScratchDir());
+
+  auto path = ScratchPath("test-large-file");
+  const int size = 1000000;
+
+  std::vector<uint8_t> data = RandomData(size);
+  ASSERT_OK(WriteDummyFile(path, data.data(), size));
+
+  std::shared_ptr<HdfsReadableFile> file;
+  ASSERT_OK(client_->OpenReadable(path, &file));
+
+  auto buffer = std::make_shared<PoolBuffer>();
+  ASSERT_OK(buffer->Resize(size));
+  int64_t bytes_read = 0;
+
+  ASSERT_OK(file->Read(size, &bytes_read, buffer->mutable_data()));
+  ASSERT_EQ(0, std::memcmp(buffer->data(), data.data(), size));
+  ASSERT_EQ(size, bytes_read);
+
+  // explicit buffer size
+  std::shared_ptr<HdfsReadableFile> file2;
+  ASSERT_OK(client_->OpenReadable(path, 1 << 18, &file2));
+
+  auto buffer2 = std::make_shared<PoolBuffer>();
+  ASSERT_OK(buffer2->Resize(size));
+  ASSERT_OK(file2->Read(size, &bytes_read, buffer2->mutable_data()));
+  ASSERT_EQ(0, std::memcmp(buffer2->data(), data.data(), size));
+  ASSERT_EQ(size, bytes_read);
+}
+
 TEST_F(TestHdfsClient, RenameFile) {
   SKIP_IF_NO_LIBHDFS();
 

From 72f80d450e0e8e20812fd80571b0c1d18e88114a Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Wed, 7 Dec 2016 15:00:18 +0100
Subject: [PATCH 0219/1644] ARROW-409: [Python] Change record batches
 conversion to Table

From discussion in ARROW-369, it is more consistent and flexible for the pyarrow.Table API to convert a RecordBatch list first a Table, then Table to pandas.DataFrame.  For example:
```
table = pa.Table.from_batches(batches)
df = table.to_pandas()
```
Also updated conversion to print schemas in exception message if not equal.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #229 from BryanCutler/pyarrow-table-from_batches-ARROW-409 and squashes the following commits:

f5751e0 [Bryan Cutler] fixed schema check to print out if not equal
72ea875 [Bryan Cutler] changed batches conversion to Table instead
---
 python/pyarrow/__init__.py         |  3 +-
 python/pyarrow/table.pyx           | 94 +++++++++++++++---------------
 python/pyarrow/tests/test_table.py |  5 +-
 3 files changed, 52 insertions(+), 50 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index f366317d04c96..5af93fb5865de 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -49,5 +49,4 @@
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
-from pyarrow.table import (Column, RecordBatch, dataframe_from_batches, Table,
-                           from_pandas_dataframe)
+from pyarrow.table import Column, RecordBatch, Table, from_pandas_dataframe
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 45cf7becceefa..0a9805cfdf427 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -415,52 +415,6 @@ cdef class RecordBatch:
         return result
 
 
-def dataframe_from_batches(batches):
-    """
-    Convert a list of Arrow RecordBatches to a pandas.DataFrame
-
-    Parameters
-    ----------
-
-    batches: list of RecordBatch
-        RecordBatch list to be converted, schemas must be equal
-    """
-
-    cdef:
-        vector[shared_ptr[CArray]] c_array_chunks
-        vector[shared_ptr[CColumn]] c_columns
-        shared_ptr[CTable] c_table
-        Array arr
-        Schema schema
-
-    import pandas as pd
-
-    schema = batches[0].schema
-
-    # check schemas are equal
-    if any((not schema.equals(other.schema) for other in batches[1:])):
-        raise ArrowException("Error converting list of RecordBatches to "
-                "DataFrame, not all schemas are equal")
-
-    cdef int K = batches[0].num_columns
-
-    # create chunked columns from the batches
-    c_columns.resize(K)
-    for i in range(K):
-        for batch in batches:
-            arr = batch[i]
-            c_array_chunks.push_back(arr.sp_array)
-        c_columns[i].reset(new CColumn(schema.sp_schema.get().field(i),
-                           c_array_chunks))
-        c_array_chunks.clear()
-
-    # create a Table from columns and convert to DataFrame
-    c_table.reset(new CTable('', schema.sp_schema, c_columns))
-    table = Table()
-    table.init(c_table)
-    return table.to_pandas()
-
-
 cdef class Table:
     """
     A collection of top-level named, equal length Arrow arrays.
@@ -567,6 +521,54 @@ cdef class Table:
 
         return result
 
+    @staticmethod
+    def from_batches(batches):
+        """
+        Construct a Table from a list of Arrow RecordBatches
+
+        Parameters
+        ----------
+
+        batches: list of RecordBatch
+            RecordBatch list to be converted, schemas must be equal
+        """
+
+        cdef:
+            vector[shared_ptr[CArray]] c_array_chunks
+            vector[shared_ptr[CColumn]] c_columns
+            shared_ptr[CTable] c_table
+            Array arr
+            Schema schema
+
+        import pandas as pd
+
+        schema = batches[0].schema
+
+        # check schemas are equal
+        for other in batches[1:]:
+            if not schema.equals(other.schema):
+                raise ArrowException("Error converting list of RecordBatches "
+                        "to DataFrame, not all schemas are equal: {%s} != {%s}"
+                        % (str(schema), str(other.schema)))
+
+        cdef int K = batches[0].num_columns
+
+        # create chunked columns from the batches
+        c_columns.resize(K)
+        for i in range(K):
+            for batch in batches:
+                arr = batch[i]
+                c_array_chunks.push_back(arr.sp_array)
+            c_columns[i].reset(new CColumn(schema.sp_schema.get().field(i),
+                               c_array_chunks))
+            c_array_chunks.clear()
+
+        # create a Table from columns and convert to DataFrame
+        c_table.reset(new CTable('', schema.sp_schema, c_columns))
+        table = Table()
+        table.init(c_table)
+        return table
+
     def to_pandas(self):
         """
         Convert the arrow::Table to a pandas DataFrame
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index dc4f37a830e5f..25463145c00ce 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -69,7 +69,8 @@ def test_recordbatchlist_to_pandas():
     batch1 = pa.RecordBatch.from_pandas(data1)
     batch2 = pa.RecordBatch.from_pandas(data2)
 
-    result = pa.dataframe_from_batches([batch1, batch2])
+    table = pa.Table.from_batches([batch1, batch2])
+    result = table.to_pandas()
     data = pd.concat([data1, data2], ignore_index=True)
     assert_frame_equal(data, result)
 
@@ -82,7 +83,7 @@ def test_recordbatchlist_schema_equals():
     batch2 = pa.RecordBatch.from_pandas(data2)
 
     with pytest.raises(pa.ArrowException):
-        pa.dataframe_from_batches([batch1, batch2])
+        pa.Table.from_batches([batch1, batch2])
 
 
 def test_table_basics():

From c8eb49e4136365f8056e09c36746b6dbb02d2814 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 8 Dec 2016 20:58:48 +0100
Subject: [PATCH 0220/1644] ARROW-408: Remove defunct conda recipes

These are better maintained on conda-forge since there's also an associated cross-platform build environment

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #231 from wesm/ARROW-408 and squashes the following commits:

8c58b75 [Wes McKinney] Remove defunct conda recipes
---
 cpp/conda.recipe/build.sh     | 62 -----------------------------------
 cpp/conda.recipe/meta.yaml    | 43 ------------------------
 python/conda.recipe/build.sh  | 45 -------------------------
 python/conda.recipe/meta.yaml | 54 ------------------------------
 4 files changed, 204 deletions(-)
 delete mode 100644 cpp/conda.recipe/build.sh
 delete mode 100644 cpp/conda.recipe/meta.yaml
 delete mode 100644 python/conda.recipe/build.sh
 delete mode 100644 python/conda.recipe/meta.yaml

diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh
deleted file mode 100644
index 0536fd99b5ca5..0000000000000
--- a/cpp/conda.recipe/build.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/bash
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-set -e
-set -x
-
-cd $RECIPE_DIR
-
-# Build dependencies
-export FLATBUFFERS_HOME=$PREFIX
-export PARQUET_HOME=$PREFIX
-
-if [ "$(uname)" == "Darwin" ]; then
-  # C++11 finagling for Mac OSX
-  export CC=clang
-  export CXX=clang++
-  export MACOSX_VERSION_MIN="10.7"
-  CXXFLAGS="${CXXFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}"
-  CXXFLAGS="${CXXFLAGS} -stdlib=libc++ -std=c++11"
-  export LDFLAGS="${LDFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}"
-  export LDFLAGS="${LDFLAGS} -stdlib=libc++ -std=c++11"
-  export LINKFLAGS="${LDFLAGS}"
-  export MACOSX_DEPLOYMENT_TARGET=10.7
-fi
-
-cd ..
-
-rm -rf conda-build
-mkdir conda-build
-cd conda-build
-pwd
-
-# if [ `uname` == Linux ]; then
-#     SHARED_LINKER_FLAGS='-static-libstdc++'
-# elif [ `uname` == Darwin ]; then
-#     SHARED_LINKER_FLAGS=''
-# fi
-
-# -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \
-
-cmake \
-    -DCMAKE_BUILD_TYPE=release \
-    -DCMAKE_INSTALL_PREFIX=$PREFIX \
-    -DARROW_HDFS=on \
-    -DARROW_IPC=on \
-    -DARROW_PARQUET=on \
-    ..
-
-make
-ctest -L unittest
-make install
diff --git a/cpp/conda.recipe/meta.yaml b/cpp/conda.recipe/meta.yaml
deleted file mode 100644
index 31f150c1f0b00..0000000000000
--- a/cpp/conda.recipe/meta.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-package:
-  name: arrow-cpp
-  version: "0.1"
-
-build:
-  number: {{environ.get('TRAVIS_BUILD_NUMBER', 0)}}    # [unix]
-  skip: true  # [win]
-  script_env:
-    - CC [linux]
-    - CXX [linux]
-    - LD_LIBRARY_PATH [linux]
-
-requirements:
-  build:
-    - cmake
-    - flatbuffers
-    - parquet-cpp
-
-  run:
-    - parquet-cpp
-
-test:
-  commands:
-    - test -f $PREFIX/lib/libarrow.so            # [linux]
-    - test -f $PREFIX/lib/libarrow_parquet.so    # [linux]
-    - test -f $PREFIX/include/arrow/api.h
-
-about:
-  home: http://github.com/apache/arrow
-  license: Apache 2.0
-  summary: 'C++ libraries for the reference Apache Arrow implementation'
diff --git a/python/conda.recipe/build.sh b/python/conda.recipe/build.sh
deleted file mode 100644
index fafe71e7adb75..0000000000000
--- a/python/conda.recipe/build.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-set -ex
-
-# Build dependency
-export ARROW_HOME=$PREFIX
-
-cd $RECIPE_DIR
-
-if [ "$(uname)" == "Darwin" ]; then
-  # C++11 finagling for Mac OSX
-  export CC=clang
-  export CXX=clang++
-  export MACOSX_VERSION_MIN="10.7"
-  CXXFLAGS="${CXXFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}"
-  CXXFLAGS="${CXXFLAGS} -stdlib=libc++ -std=c++11"
-  export LDFLAGS="${LDFLAGS} -mmacosx-version-min=${MACOSX_VERSION_MIN}"
-  export LDFLAGS="${LDFLAGS} -stdlib=libc++ -std=c++11"
-  export LINKFLAGS="${LDFLAGS}"
-  export MACOSX_DEPLOYMENT_TARGET=10.7
-fi
-
-# echo Setting the compiler...
-# if [ `uname` == Linux ]; then
-#   EXTRA_CMAKE_ARGS=-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++
-# elif [ `uname` == Darwin ]; then
-#   EXTRA_CMAKE_ARGS=
-# fi
-
-cd ..
-# $PYTHON setup.py build_ext --extra-cmake-args=$EXTRA_CMAKE_ARGS || exit 1
-$PYTHON setup.py build_ext || exit 1
-$PYTHON setup.py install || exit 1
diff --git a/python/conda.recipe/meta.yaml b/python/conda.recipe/meta.yaml
deleted file mode 100644
index b37dfde0a0d6f..0000000000000
--- a/python/conda.recipe/meta.yaml
+++ /dev/null
@@ -1,54 +0,0 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-package:
-  name: pyarrow
-  version: "0.1"
-
-build:
-  number: {{environ.get('TRAVIS_BUILD_NUMBER', 0)}}    # [unix]
-  rpaths:
-    - lib                                                        # [unix]
-    - lib/python{{environ.get('PY_VER')}}/site-packages/pyarrow  # [unix]
-  script_env:
-    - CC [linux]
-    - CXX [linux]
-    - LD_LIBRARY_PATH [linux]
-  skip: true  # [win]
-
-requirements:
-  build:
-    - cmake
-    - python
-    - setuptools
-    - cython
-    - numpy
-    - pandas
-    - arrow-cpp
-    - pytest
-
-  run:
-    - arrow-cpp
-    - parquet-cpp
-    - python
-    - numpy
-    - pandas
-    - six
-
-test:
-  imports:
-    - pyarrow
-
-about:
-  home: http://github.com/apache/arrow
-  license: Apache 2.0
-  summary: 'Python bindings for Arrow C++ and interoperability tool for pandas and NumPy'

From e139b8b7c11b7f36fa57a625a39d9c8779d033f4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 9 Dec 2016 06:49:49 +0100
Subject: [PATCH 0221/1644] ARROW-404: [Python] Fix segfault caused by
 HdfsClient getting closed before an HdfsFile

The one downside of this patch is that HdfsFile handles don't get garbage-collected until the cyclic GC runs -- I tried to fix this but couldn't get it working. So bytes don't always get flushed to HDFS until `close()` is called. The flush issue should be addressed on the C++ side

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #230 from wesm/ARROW-404 and squashes the following commits:

3a8e641 [Wes McKinney] Use weakref in _HdfsFileNanny to avoid cyclic gc
274d0c5 [Wes McKinney] amend comment
1539a2c [Wes McKinney] Ensure that HdfsClient does not get closed before an open file does when the last user-accessible client reference goes out of scope
---
 python/pyarrow/io.pyx             | 86 ++++++++++++++++++++-----------
 python/pyarrow/tests/test_hdfs.py | 23 +++++++++
 2 files changed, 79 insertions(+), 30 deletions(-)

diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 0e6b81e984431..2fa5fb6b87885 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -504,7 +504,7 @@ cdef class HdfsClient:
 
         out.mode = mode
         out.buffer_size = c_buffer_size
-        out.parent = self
+        out.parent = _HdfsFileNanny(self, out)
         out.is_open = True
         out.own_file = True
 
@@ -516,48 +516,69 @@ cdef class HdfsClient:
         """
         write_queue = Queue(50)
 
-        f = self.open(path, 'wb')
+        with self.open(path, 'wb') as f:
+            done = False
+            exc_info = None
+            def bg_write():
+                try:
+                    while not done or write_queue.qsize() > 0:
+                        try:
+                            buf = write_queue.get(timeout=0.01)
+                        except QueueEmpty:
+                            continue
 
-        done = False
-        exc_info = None
-        def bg_write():
-            try:
-                while not done or write_queue.qsize() > 0:
-                    try:
-                        buf = write_queue.get(timeout=0.01)
-                    except QueueEmpty:
-                        continue
+                        f.write(buf)
 
-                    f.write(buf)
+                except Exception as e:
+                    exc_info = sys.exc_info()
 
-            except Exception as e:
-                exc_info = sys.exc_info()
-
-        writer_thread = threading.Thread(target=bg_write)
-        writer_thread.start()
+            writer_thread = threading.Thread(target=bg_write)
+            writer_thread.start()
 
-        try:
-            while True:
-                buf = stream.read(buffer_size)
-                if not buf:
-                    break
+            try:
+                while True:
+                    buf = stream.read(buffer_size)
+                    if not buf:
+                        break
 
-                write_queue.put_nowait(buf)
-        finally:
-            done = True
+                    write_queue.put_nowait(buf)
+            finally:
+                done = True
 
-        writer_thread.join()
-        if exc_info is not None:
-            raise exc_info[0], exc_info[1], exc_info[2]
+            writer_thread.join()
+            if exc_info is not None:
+                raise exc_info[0], exc_info[1], exc_info[2]
 
     def download(self, path, stream, buffer_size=None):
-        f = self.open(path, 'rb', buffer_size=buffer_size)
-        f.download(stream)
+        with self.open(path, 'rb', buffer_size=buffer_size) as f:
+            f.download(stream)
 
 
 # ----------------------------------------------------------------------
 # Specialization for HDFS
 
+# ARROW-404: Helper class to ensure that files are closed before the
+# client. During deallocation of the extension class, the attributes are
+# decref'd which can cause the client to get closed first if the file has the
+# last remaining reference
+cdef class _HdfsFileNanny:
+    cdef:
+        object client
+        object file_handle_ref
+
+    def __cinit__(self, client, file_handle):
+        import weakref
+        self.client = client
+        self.file_handle_ref = weakref.ref(file_handle)
+
+    def __dealloc__(self):
+        fh = self.file_handle_ref()
+        if fh:
+            fh.close()
+        # avoid cyclic GC
+        self.file_handle_ref = None
+        self.client = None
+
 
 cdef class HdfsFile(NativeFile):
     cdef readonly:
@@ -565,6 +586,11 @@ cdef class HdfsFile(NativeFile):
         object mode
         object parent
 
+    cdef object __weakref__
+
+    def __dealloc__(self):
+        self.parent = None
+
     def read(self, int nbytes):
         """
         Read indicated number of bytes from the file, up to EOF
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index ed8d41994cdd0..c23543b7f0d07 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -98,6 +98,29 @@ def test_hdfs_ls(hdfs):
     assert contents == [dir_path, f1_path]
 
 
+def _make_test_file(hdfs, test_name, test_path, test_data):
+    base_path = pjoin(HDFS_TMP_PATH, test_name)
+    hdfs.mkdir(base_path)
+
+    full_path = pjoin(base_path, test_path)
+
+    f = hdfs.open(full_path, 'wb')
+    f.write(test_data)
+
+    return full_path
+
+
+@libhdfs
+def test_hdfs_orphaned_file():
+    hdfs = hdfs_test_client()
+    file_path = _make_test_file(hdfs, 'orphaned_file_test', 'fname',
+                                'foobarbaz')
+
+    f = hdfs.open(file_path)
+    hdfs = None
+    f = None  # noqa
+
+
 @libhdfs
 def test_hdfs_download_upload(hdfs):
     base_path = pjoin(HDFS_TMP_PATH, 'upload-test')

From a5362c2cbed5f32a468d93d23c8365c9c5528b03 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 9 Dec 2016 11:39:49 -0500
Subject: [PATCH 0222/1644] ARROW-346: Use conda environment to build API docs

As we cannot currently build pyarrow on readthedocs, we have to resort
to building the API docs for the latest version of pyarrow on
conda-forge. All other documentation will though be pulled directly from
the source code.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #228 from xhochy/ARROW-346 and squashes the following commits:

6a4bdc1 [Uwe L. Korn] Add license header
b741141 [Uwe L. Korn] ARROW-346: Use conda environment to build API docs
---
 .readthedocs.yml           |  2 ++
 python/doc/conf.py         |  7 ++++++-
 python/doc/environment.yml | 25 +++++++++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 .readthedocs.yml
 create mode 100644 python/doc/environment.yml

diff --git a/.readthedocs.yml b/.readthedocs.yml
new file mode 100644
index 0000000000000..2e1fe3fbc251a
--- /dev/null
+++ b/.readthedocs.yml
@@ -0,0 +1,2 @@
+conda:
+    file: python/doc/environment.yml
diff --git a/python/doc/conf.py b/python/doc/conf.py
index 4c324a8086c39..e817bbdd42bd3 100644
--- a/python/doc/conf.py
+++ b/python/doc/conf.py
@@ -42,7 +42,12 @@
 cmd_line = cmd_line_template.format(outputdir=output_dir, moduledir=module_dir)
 apidoc.main(cmd_line.split(" "))
 
-sys.path.insert(0, os.path.abspath('..'))
+on_rtd = os.environ.get('READTHEDOCS') == 'True'
+
+if not on_rtd:
+    # Hack: On RTD we use the pyarrow package from conda-forge as we cannot
+    # build pyarrow there.
+    sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration ------------------------------------------------
 
diff --git a/python/doc/environment.yml b/python/doc/environment.yml
new file mode 100644
index 0000000000000..8d1fe9bfb5d11
--- /dev/null
+++ b/python/doc/environment.yml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+channels:
+- defaults
+- conda-forge
+dependencies:
+- arrow-cpp
+- parquet-cpp
+- pyarrow
+- numpydoc

From d06c49144a60faa9af115e803694329e82623a5d Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 9 Dec 2016 11:41:21 -0500
Subject: [PATCH 0223/1644] =?UTF-8?q?ARROW-399:=20ListVector.loadFieldBuff?=
 =?UTF-8?q?ers=20ignores=20the=20ArrowFieldNode=20len=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…gth metadata

Author: Julien Le Dem <julien@dremio.com>

Closes #227 from julienledem/arrow_399 and squashes the following commits:

93a77cb [Julien Le Dem] set padding; add test
462a36c [Julien Le Dem] ARROW-399: ListVector.loadFieldBuffers ignores the ArrowFieldNode length metadata
---
 .../codegen/templates/FixedValueVectors.java  |  2 +
 .../templates/NullableValueVectors.java       | 62 +++++--------
 .../main/codegen/templates/UnionVector.java   |  2 +
 .../arrow/vector/BaseDataValueVector.java     | 17 ++++
 .../org/apache/arrow/vector/BitVector.java    |  2 +-
 .../org/apache/arrow/vector/VectorLoader.java |  2 +-
 .../arrow/vector/complex/ListVector.java      |  2 +
 .../arrow/vector/TestVectorUnloadLoad.java    | 92 ++++++++++++++++++-
 8 files changed, 136 insertions(+), 45 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 7958222f5c1bb..be385d146dbac 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -45,6 +45,8 @@
 public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
 
+  public static final int TYPE_WIDTH = ${type.width};
+
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
 
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 2c4274c13ee58..6a9ce65392f59 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -37,7 +37,7 @@
 import org.apache.arrow.flatbuf.Precision;
 
 /**
- * Nullable${minor.class} implements a vector of values which could be null.  Elements in the vector
+ * ${className} implements a vector of values which could be null.  Elements in the vector
  * are first checked against a fixed length vector of boolean values.  Then the element is retrieved
  * from the base class (if not null).
  *
@@ -47,7 +47,7 @@
 public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector, FieldVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
-  private final FieldReader reader = new ${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
+  private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this);
 
   private final String bitsField = "$bits$";
   private final String valuesField = "$values$";
@@ -67,7 +67,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 
   public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
     super(name, allocator);
-    values = new ${minor.class}Vector(valuesField, allocator, precision, scale);
+    values = new ${valuesName}(valuesField, allocator, precision, scale);
     this.precision = precision;
     this.scale = scale;
     mutator = new Mutator();
@@ -81,7 +81,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#else>
   public ${className}(String name, BufferAllocator allocator) {
     super(name, allocator);
-    values = new ${minor.class}Vector(valuesField, allocator);
+    values = new ${valuesName}(valuesField, allocator);
     mutator = new Mutator();
     accessor = new Accessor();
   <#if minor.class == "TinyInt" ||
@@ -144,6 +144,13 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    <#if type.major = "VarLen">
+    // variable width values: truncate offset vector buffer to size (#1)
+    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.offsetVector.getBufferSizeFor(fieldNode.getLength() + 1));
+    <#else>
+    // fixed width values truncate value vector to size (#1)
+    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.getBufferSizeFor(fieldNode.getLength()));
+    </#if>
     org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
     bits.valueCount = fieldNode.getLength();
   }
@@ -229,13 +236,6 @@ public void setInitialCapacity(int numRecords) {
     values.setInitialCapacity(numRecords);
   }
 
-//  @Override
-//  public SerializedField.Builder getMetadataBuilder() {
-//    return super.getMetadataBuilder()
-//      .addChild(bits.getMetadata())
-//      .addChild(values.getMetadata());
-//  }
-
   @Override
   public void allocateNew() {
     if(!allocateNewSafe()){
@@ -329,20 +329,6 @@ public void zeroVector() {
   }
   </#if>
 
-
-//  @Override
-//  public void load(SerializedField metadata, ArrowBuf buffer) {
-//    clear();
-    // the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
-//    final SerializedField bitsField = metadata.getChild(0);
-//    bits.load(bitsField, buffer);
-//
-//    final int capacity = buffer.capacity();
-//    final int bitsLength = bitsField.getBufferLength();
-//    final SerializedField valuesField = metadata.getChild(1);
-//    values.load(valuesField, buffer.slice(bitsLength, capacity - bitsLength));
-//  }
-
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator){
     return new TransferImpl(name, allocator);
@@ -356,10 +342,10 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator){
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((Nullable${minor.class}Vector) to);
+    return new TransferImpl((${className}) to);
   }
 
-  public void transferTo(Nullable${minor.class}Vector target){
+  public void transferTo(${className} target){
     bits.transferTo(target.bits);
     values.transferTo(target.values);
     <#if type.major == "VarLen">
@@ -368,7 +354,7 @@ public void transferTo(Nullable${minor.class}Vector target){
     clear();
   }
 
-  public void splitAndTransferTo(int startIndex, int length, Nullable${minor.class}Vector target) {
+  public void splitAndTransferTo(int startIndex, int length, ${className} target) {
     bits.splitAndTransferTo(startIndex, length, target.bits);
     values.splitAndTransferTo(startIndex, length, target.values);
     <#if type.major == "VarLen">
@@ -377,22 +363,22 @@ public void splitAndTransferTo(int startIndex, int length, Nullable${minor.class
   }
 
   private class TransferImpl implements TransferPair {
-    Nullable${minor.class}Vector to;
+    ${className} to;
 
     public TransferImpl(String name, BufferAllocator allocator){
       <#if minor.class == "Decimal">
-      to = new Nullable${minor.class}Vector(name, allocator, precision, scale);
+      to = new ${className}(name, allocator, precision, scale);
       <#else>
-      to = new Nullable${minor.class}Vector(name, allocator);
+      to = new ${className}(name, allocator);
       </#if>
     }
 
-    public TransferImpl(Nullable${minor.class}Vector to){
+    public TransferImpl(${className} to){
       this.to = to;
     }
 
     @Override
-    public Nullable${minor.class}Vector getTo(){
+    public ${className} getTo(){
       return to;
     }
 
@@ -408,7 +394,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, Nullable${minor.class}Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, ${className}.this);
     }
   }
 
@@ -422,14 +408,14 @@ public Mutator getMutator(){
     return mutator;
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
+  public void copyFrom(int fromIndex, int thisIndex, ${className} from){
     final Accessor fromAccessor = from.getAccessor();
     if (!fromAccessor.isNull(fromIndex)) {
       mutator.set(thisIndex, fromAccessor.get(fromIndex));
     }
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+  public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from){
     <#if type.major == "VarLen">
     mutator.fillEmpties(thisIndex);
     </#if>
@@ -437,7 +423,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from
     bits.getMutator().setSafe(thisIndex, 1);
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
+  public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
     <#if type.major == "VarLen">
     mutator.fillEmpties(thisIndex);
     </#if>
@@ -640,7 +626,7 @@ public void set(int index, ${minor.class}Holder holder){
     }
 
     public boolean isSafe(int outIndex) {
-      return outIndex < Nullable${minor.class}Vector.this.getValueCapacity();
+      return outIndex < ${className}.this.getValueCapacity();
     }
 
     <#assign fields = minor.fields!type.fields />
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 4e68b681d1404..18acdf4a551b4 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -103,6 +103,8 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    // truncate types vector buffer to size (#0)
+    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 0, typeVector.getBufferSizeFor(fieldNode.getLength()));
     BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
     this.valueCount = fieldNode.getLength();
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 4c6d363f21cda..b7df8d13ee607 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -30,6 +30,9 @@ public abstract class BaseDataValueVector extends BaseValueVector implements Buf
 
   protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
 
+  /** maximum extra size at the end of the buffer */
+  private static final int MAX_BUFFER_PADDING = 64;
+
   public static void load(ArrowFieldNode fieldNode, List<BufferBacked> vectors, List<ArrowBuf> buffers) {
     int expectedSize = vectors.size();
     if (buffers.size() != expectedSize) {
@@ -40,6 +43,20 @@ public static void load(ArrowFieldNode fieldNode, List<BufferBacked> vectors, Li
     }
   }
 
+  public static void truncateBufferBasedOnSize(List<ArrowBuf> buffers, int bufferIndex, int byteSize) {
+    if (bufferIndex >= buffers.size()) {
+      throw new IllegalArgumentException("no buffer at index " + bufferIndex + ": " + buffers);
+    }
+    ArrowBuf buffer = buffers.get(bufferIndex);
+    if (buffer.writerIndex() < byteSize) {
+      throw new IllegalArgumentException("can not truncate buffer to a larger size " + byteSize + ": " + buffer.writerIndex());
+    }
+    if (buffer.writerIndex() - byteSize > MAX_BUFFER_PADDING) {
+      throw new IllegalArgumentException("Buffer too large to resize to " + byteSize + ": " + buffer.writerIndex());
+    }
+    buffer.writerIndex(byteSize);
+  }
+
   public static List<ArrowBuf> unload(List<BufferBacked> vectors) {
     List<ArrowBuf> result = new ArrayList<>(vectors.size());
     for (BufferBacked vector : vectors) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 7ce1236b2ec30..48da8e77d6814 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -68,7 +68,7 @@ public void load(ArrowFieldNode fieldNode, ArrowBuf data) {
         int remainder = count % 8;
         // set remaining bits
         if (remainder > 0) {
-          byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7));;
+          byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7));
           this.data.setByte(fullBytesCount, bitMask);
         }
       } else if (fieldNode.getNullCount() == fieldNode.getLength()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 757f061dd5a2f..5c1176cf95d26 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -82,7 +82,7 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
       vector.loadFieldBuffers(fieldNode, ownBuffers);
     } catch (RuntimeException e) {
       throw new IllegalArgumentException("Could not load buffers for field " +
-              field + " error message" + e.getMessage(), e);
+              field + ". error message: " + e.getMessage(), e);
     }
     List<Field> children = field.getChildren();
     if (children.size() > 0) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index e18f99f95d780..461bdbcda1b52 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -93,6 +93,8 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    // variable width values: truncate offset vector buffer to size (#1)
+    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, offsets.getBufferSizeFor(fieldNode.getLength() + 1));
     BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 9dfe8d840e49d..7a70ffd904758 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -23,6 +23,7 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -32,6 +33,7 @@
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
@@ -99,6 +101,79 @@ public void testUnloadLoad() throws IOException {
     }
   }
 
+  @Test
+  public void testUnloadLoadAddPadding() throws IOException {
+    int count = 10000;
+    Schema schema;
+    try (
+        BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) {
+
+      // write some data
+      ComplexWriter writer = new ComplexWriterImpl("root", parent);
+      MapWriter rootWriter = writer.rootAsMap();
+      ListWriter list = rootWriter.list("list");
+      IntWriter intWriter = list.integer();
+      for (int i = 0; i < count; i++) {
+        list.setPosition(i);
+        list.startList();
+        for (int j = 0; j < i % 4 + 1; j++) {
+          intWriter.writeInt(i);
+        }
+        list.endList();
+      }
+      writer.setValueCount(count);
+
+      // unload it
+      FieldVector root = parent.getChild("root");
+      schema = new Schema(root.getField().getChildren());
+      VectorUnloader vectorUnloader = newVectorUnloader(root);
+      try (
+          ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+          BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+          VectorSchemaRoot newRoot = new VectorSchemaRoot(schema, finalVectorsAllocator);
+          ) {
+        List<ArrowBuf> oldBuffers = recordBatch.getBuffers();
+        List<ArrowBuf> newBuffers = new ArrayList<>();
+        for (ArrowBuf oldBuffer : oldBuffers) {
+          int l = oldBuffer.readableBytes();
+          if (l % 64 != 0) {
+            // pad
+            l = l + 64 - l % 64;
+          }
+          ArrowBuf newBuffer = allocator.buffer(l);
+          for (int i = oldBuffer.readerIndex(); i < oldBuffer.writerIndex(); i++) {
+            newBuffer.setByte(i - oldBuffer.readerIndex(), oldBuffer.getByte(i));
+          }
+          newBuffer.readerIndex(0);
+          newBuffer.writerIndex(l);
+          newBuffers.add(newBuffer);
+        }
+
+        try (ArrowRecordBatch newBatch = new ArrowRecordBatch(recordBatch.getLength(), recordBatch.getNodes(), newBuffers);) {
+          // load it
+          VectorLoader vectorLoader = new VectorLoader(newRoot);
+
+          vectorLoader.load(newBatch);
+
+          FieldReader reader = newRoot.getVector("list").getReader();
+          for (int i = 0; i < count; i++) {
+            reader.setPosition(i);
+            List<Integer> expected = new ArrayList<>();
+            for (int j = 0; j < i % 4 + 1; j++) {
+              expected.add(i);
+            }
+            Assert.assertEquals(expected, reader.readObject());
+          }
+        }
+
+        for (ArrowBuf newBuf : newBuffers) {
+          newBuf.release();
+        }
+      }
+    }
+  }
+
   /**
    * The validity buffer can be empty if:
    *  - all values are defined
@@ -113,12 +188,17 @@ public void testLoadEmptyValidityBuffer() throws IOException {
         ));
     int count = 10;
     ArrowBuf validity = allocator.getEmpty();
-    ArrowBuf values = allocator.buffer(count * 4); // integers
-    for (int i = 0; i < count; i++) {
-      values.setInt(i * 4, i);
+    ArrowBuf[] values = new ArrowBuf[2];
+    for (int i = 0; i < values.length; i++) {
+      ArrowBuf arrowBuf = allocator.buffer(count * 4); // integers
+      values[i] = arrowBuf;
+      for (int j = 0; j < count; j++) {
+        arrowBuf.setInt(j * 4, j);
+      }
+      arrowBuf.writerIndex(count * 4);
     }
     try (
-        ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(validity, values, validity, values));
+        ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(validity, values[0], validity, values[1]));
         BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
         VectorSchemaRoot newRoot = new VectorSchemaRoot(schema, finalVectorsAllocator);
         ) {
@@ -153,7 +233,9 @@ public void testLoadEmptyValidityBuffer() throws IOException {
       assertFalse(intDefinedVector.getAccessor().isNull(count + 10));
       assertEquals(1234, intDefinedVector.getAccessor().get(count + 10));
     } finally {
-      values.release();
+      for (ArrowBuf arrowBuf : values) {
+        arrowBuf.release();
+      }
     }
   }
 

From 14ed1be2d89fedc31f4015456cda28216f926dcc Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Fri, 9 Dec 2016 12:04:25 -0500
Subject: [PATCH 0224/1644] ARROW-400: set struct length on load

Adds unit test, closes #233

Author: Julien Le Dem <julien@dremio.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #234 from wesm/ARROW-400 and squashes the following commits:

f516ba1 [Wes McKinney] Add unit test for ARROW-400
741ff71 [Julien Le Dem] ARROW-400: set struct length on json load
---
 integration/data/struct_example.json          | 237 ++++++++++++++++++
 .../arrow/vector/complex/MapVector.java       |   2 +-
 .../vector/file/json/JsonFileReader.java      |   4 +
 .../arrow/vector/file/json/TestJSONFile.java  |  20 ++
 4 files changed, 262 insertions(+), 1 deletion(-)
 create mode 100644 integration/data/struct_example.json

diff --git a/integration/data/struct_example.json b/integration/data/struct_example.json
new file mode 100644
index 0000000000000..3ea062db7ba32
--- /dev/null
+++ b/integration/data/struct_example.json
@@ -0,0 +1,237 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "struct_nullable",
+        "type": {
+          "name": "struct"
+        },
+        "nullable": true,
+        "children": [
+          {
+            "name": "f1",
+            "type": {
+              "name": "int",
+              "isSigned": true,
+              "bitWidth": 32
+            },
+            "nullable": true,
+            "children": [],
+            "typeLayout": {
+              "vectors": [
+                {
+                  "type": "VALIDITY",
+                  "typeBitWidth": 1
+                },
+                {
+                  "type": "DATA",
+                  "typeBitWidth": 32
+                }
+              ]
+            }
+          },
+          {
+            "name": "f2",
+            "type": {
+              "name": "utf8"
+            },
+            "nullable": true,
+            "children": [],
+            "typeLayout": {
+              "vectors": [
+                {
+                  "type": "VALIDITY",
+                  "typeBitWidth": 1
+                },
+                {
+                  "type": "OFFSET",
+                  "typeBitWidth": 32
+                },
+                {
+                  "type": "DATA",
+                  "typeBitWidth": 8
+                }
+              ]
+            }
+          }
+        ],
+        "typeLayout": {
+          "vectors": [
+            {
+              "type": "VALIDITY",
+              "typeBitWidth": 1
+            }
+          ]
+        }
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "struct_nullable",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "children": [
+            {
+              "name": "f1",
+              "count": 7,
+              "VALIDITY": [
+                1,
+                0,
+                1,
+                1,
+                1,
+                0,
+                0
+              ],
+              "DATA": [
+                1402032511,
+                290876774,
+                137773603,
+                410361374,
+                1959836418,
+                1995074679,
+                -163525262
+              ]
+            },
+            {
+              "name": "f2",
+              "count": 7,
+              "VALIDITY": [
+                0,
+                1,
+                1,
+                1,
+                0,
+                1,
+                0
+              ],
+              "OFFSET": [
+                0,
+                0,
+                7,
+                14,
+                21,
+                21,
+                28,
+                28
+              ],
+              "DATA": [
+                "",
+                "MhRNxD4",
+                "3F9HBxK",
+                "aVd88fp",
+                "",
+                "3loZrRf",
+                ""
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "struct_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "children": [
+            {
+              "name": "f1",
+              "count": 10,
+              "VALIDITY": [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                0,
+                0,
+                0
+              ],
+              "DATA": [
+                -2041500147,
+                1715692943,
+                -35444996,
+                1425496657,
+                112765084,
+                1760754983,
+                413888857,
+                2039738337,
+                -1924327700,
+                670528518
+              ]
+            },
+            {
+              "name": "f2",
+              "count": 10,
+              "VALIDITY": [
+                1,
+                0,
+                0,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0
+              ],
+              "OFFSET": [
+                0,
+                7,
+                7,
+                7,
+                14,
+                21,
+                28,
+                35,
+                42,
+                49,
+                49
+              ],
+              "DATA": [
+                "AS5oARE",
+                "",
+                "",
+                "JGdagcX",
+                "78SLiRw",
+                "vbGf7OY",
+                "5uh5fTs",
+                "0ilsf82",
+                "LjS9MbU",
+                ""
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index c2f216b197e1d..31a1bb74b8e98 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -50,7 +50,7 @@ public class MapVector extends AbstractMapVector {
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
-  int valueCount;
+  public int valueCount;
 
   public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator, callBack);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 26dd3f6dfe5ae..152867c1a11d7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -47,6 +47,7 @@
 import org.apache.arrow.vector.ValueVector.Mutator;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -153,6 +154,9 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         }
         readToken(END_ARRAY);
       }
+      if (vector instanceof NullableMapVector) {
+        ((NullableMapVector)vector).valueCount = count;
+      }
     }
     readToken(END_OBJECT);
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 7d25003f8b335..3720a13b0fce5 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -21,11 +21,13 @@
 import java.io.IOException;
 
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.file.BaseFileTest;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -117,4 +119,22 @@ public void testWriteReadUnionJSON() throws IOException {
     }
   }
 
+  @Test
+  public void testSetStructLength() throws IOException {
+    File file = new File("../../integration/data/struct_example.json");
+    try (
+            BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+    ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        FieldVector vector = root.getVector("struct_nullable");
+        Assert.assertEquals(7, vector.getAccessor().getValueCount());
+      }
+    }
+  }
+
 }

From 8995c923043788f98afef4dd80f72de4688a8e0c Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Thu, 8 Dec 2016 21:24:29 -0800
Subject: [PATCH 0225/1644] ARROW-402: Fix reference counting issue with empty
 buffers. Close #232

Change-Id: I87910c03d7ebca5a8edbf53d01f70c38ef339f04
---
 .../src/main/java/org/apache/arrow/vector/BitVector.java       | 1 -
 .../java/org/apache/arrow/vector/TestVectorUnloadLoad.java     | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 48da8e77d6814..26eeafd51d900 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -53,7 +53,6 @@ public BitVector(String name, BufferAllocator allocator) {
   public void load(ArrowFieldNode fieldNode, ArrowBuf data) {
     // When the vector is all nulls or all defined, the content of the buffer can be omitted
     if (data.readableBytes() == 0 && fieldNode.getLength() != 0) {
-      data.release();
       int count = fieldNode.getLength();
       allocateNew(count);
       int n = getSizeFromCount(count);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 7a70ffd904758..79c9d5046acd6 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -187,7 +187,7 @@ public void testLoadEmptyValidityBuffer() throws IOException {
         new Field("intNull", true, new ArrowType.Int(32, true), Collections.<Field>emptyList())
         ));
     int count = 10;
-    ArrowBuf validity = allocator.getEmpty();
+    ArrowBuf validity = allocator.buffer(10).slice(0, 0);
     ArrowBuf[] values = new ArrowBuf[2];
     for (int i = 0; i < values.length; i++) {
       ArrowBuf arrowBuf = allocator.buffer(count * 4); // integers
@@ -236,6 +236,7 @@ public void testLoadEmptyValidityBuffer() throws IOException {
       for (ArrowBuf arrowBuf : values) {
         arrowBuf.release();
       }
+      validity.release();
     }
   }
 

From 45ed7e7a36fb2a69de468c41132b6b3bbd270c92 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 9 Dec 2016 19:50:19 -0500
Subject: [PATCH 0226/1644] ARROW-394: [Integration] Generate tests cases for
 numeric types, strings, lists, structs

Automatically generating testing files from Python.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #219 from wesm/ARROW-394 and squashes the following commits:

7807f48 [Wes McKinney] OS X doesn't have std::fabs
c0c804c [Wes McKinney] abs -> fabs
8cd1902 [Wes McKinney] Fix compiler warning in OS X from incorrect type declaration
d51581a [Wes McKinney] Add missing apache license
527622d [Wes McKinney] ARROW-414: remove check for maximum buffer padding
2a7b0fc [Wes McKinney] Add JSON generation code to fuzz test numeric types, print integers more nicely. Add integration tests to Travis CI build matrix. Add ApproxEquals method for floating point comparisons. Add boolean, string, struct, list to generated json test case
---
 .travis.yml                                   |  10 +
 ci/travis_script_integration.sh               |  49 ++
 cpp/src/arrow/array.cc                        |   4 +
 cpp/src/arrow/array.h                         |   1 +
 cpp/src/arrow/ipc/ipc-metadata-test.cc        |   4 +-
 cpp/src/arrow/ipc/json-integration-test.cc    |  27 +-
 cpp/src/arrow/ipc/json-internal.cc            |   8 +-
 cpp/src/arrow/ipc/metadata-internal.cc        |   2 +-
 cpp/src/arrow/pretty_print-test.cc            |  10 +-
 cpp/src/arrow/pretty_print.cc                 |  90 +++-
 cpp/src/arrow/pretty_print.h                  |   8 +-
 cpp/src/arrow/table.cc                        |  12 +
 cpp/src/arrow/table.h                         |   2 +
 cpp/src/arrow/type.cc                         |   2 +-
 cpp/src/arrow/type_traits.h                   |   6 +
 cpp/src/arrow/types/primitive.cc              |   1 +
 cpp/src/arrow/types/primitive.h               |  91 +++-
 integration/README.md                         |  59 ++
 integration/integration_test.py               | 508 +++++++++++++++++-
 .../arrow/vector/BaseDataValueVector.java     |   6 -
 20 files changed, 844 insertions(+), 56 deletions(-)
 create mode 100755 ci/travis_script_integration.sh
 create mode 100644 integration/README.md

diff --git a/.travis.yml b/.travis.yml
index bfc2f26b4f590..1634eba443615 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,6 +46,16 @@ matrix:
     jdk: oraclejdk7
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+  - language: java
+    os: linux
+    env: ARROW_TEST_GROUP=integration
+    jdk: oraclejdk7
+    before_script:
+    - export CC="gcc-4.9"
+    - export CXX="g++-4.9"
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
 
 before_install:
 - ulimit -c unlimited -S
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
new file mode 100755
index 0000000000000..d93411b907d47
--- /dev/null
+++ b/ci/travis_script_integration.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+set -e
+
+: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+
+JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+
+pushd $JAVA_DIR
+
+mvn package
+
+popd
+
+pushd $TRAVIS_BUILD_DIR/integration
+
+VERSION=0.1.1-SNAPSHOT
+export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
+export ARROW_CPP_TESTER=$CPP_BUILD_DIR/debug/json-integration-test
+
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+export MINICONDA=$HOME/miniconda
+export PATH="$MINICONDA/bin:$PATH"
+
+CONDA_ENV_NAME=arrow-integration-test
+conda create -y -q -n $CONDA_ENV_NAME python=3.5
+source activate $CONDA_ENV_NAME
+
+# faster builds, please
+conda install -y nomkl
+
+# Expensive dependencies install from Continuum package repo
+conda install -y pip numpy six
+
+python integration_test.py --debug
+
+popd
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 3262425e99b66..1f0bb66e91a3e 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -60,6 +60,10 @@ bool Array::EqualsExact(const Array& other) const {
   return true;
 }
 
+bool Array::ApproxEquals(const std::shared_ptr<Array>& arr) const {
+  return Equals(arr);
+}
+
 Status Array::Validate() const {
   return Status::OK();
 }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index ff2b70e213b1b..78aa2b867e1ea 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -62,6 +62,7 @@ class ARROW_EXPORT Array {
 
   bool EqualsExact(const Array& arr) const;
   virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
+  virtual bool ApproxEquals(const std::shared_ptr<Array>& arr) const;
 
   // Compare if the range of slots specified are equal for the given array and
   // this array.  end_idx exclusive.  This methods does not bounds check.
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
index d29583f8488e0..de08e6dab73c6 100644
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -70,7 +70,7 @@ const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
 
 TEST_F(TestSchemaMetadata, PrimitiveFields) {
   auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>());
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>(), false);
   auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
   auto f3 = std::make_shared<Field>("f3", std::make_shared<Int64Type>());
   auto f4 = std::make_shared<Field>("f4", std::make_shared<UInt8Type>());
@@ -78,7 +78,7 @@ TEST_F(TestSchemaMetadata, PrimitiveFields) {
   auto f6 = std::make_shared<Field>("f6", std::make_shared<UInt32Type>());
   auto f7 = std::make_shared<Field>("f7", std::make_shared<UInt64Type>());
   auto f8 = std::make_shared<Field>("f8", std::make_shared<FloatType>());
-  auto f9 = std::make_shared<Field>("f9", std::make_shared<DoubleType>());
+  auto f9 = std::make_shared<Field>("f9", std::make_shared<DoubleType>(), false);
   auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
 
   Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index c4e68472a19d4..291a719d4e58c 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -169,15 +169,15 @@ static Status ValidateArrowVsJson(
     RETURN_NOT_OK(json_reader->GetRecordBatch(i, &json_batch));
     RETURN_NOT_OK(arrow_reader->GetRecordBatch(i, &arrow_batch));
 
-    if (!json_batch->Equals(*arrow_batch.get())) {
+    if (!json_batch->ApproxEquals(*arrow_batch.get())) {
       std::stringstream ss;
       ss << "Record batch " << i << " did not match";
 
-      ss << "\nJSON: \n ";
-      RETURN_NOT_OK(PrettyPrint(*json_batch.get(), &ss));
+      ss << "\nJSON:\n";
+      RETURN_NOT_OK(PrettyPrint(*json_batch.get(), 0, &ss));
 
-      ss << "\nArrow: \n ";
-      RETURN_NOT_OK(PrettyPrint(*arrow_batch.get(), &ss));
+      ss << "\nArrow:\n";
+      RETURN_NOT_OK(PrettyPrint(*arrow_batch.get(), 0, &ss));
       return Status::Invalid(ss.str());
     }
   }
@@ -299,6 +299,23 @@ static const char* JSON_EXAMPLE = R"example(
           "VALIDITY": [1, 0, 0, 1, 1]
         }
       ]
+    },
+    {
+      "count": 4,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 4,
+          "DATA": [1, 2, 3, 4],
+          "VALIDITY": [1, 0, 1, 1]
+        },
+        {
+          "name": "bar",
+          "count": 4,
+          "DATA": [1.0, 2.0, 3.0, 4.0],
+          "VALIDITY": [1, 0, 0, 1]
+        }
+      ]
     }
   ]
 }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 50f5b0cb1bd1e..ff9f59800be38 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -418,7 +418,7 @@ class JsonArrayWriter : public ArrayVisitor {
 
   template <typename T>
   void WriteOffsetsField(const T* offsets, int32_t length) {
-    writer_->Key("OFFSETS");
+    writer_->Key("OFFSET");
     writer_->StartArray();
     for (int i = 0; i < length; ++i) {
       writer_->Int64(offsets[i]);
@@ -810,7 +810,7 @@ class JsonArrayReader {
         builder.Append(val.GetUint64());
       } else if (IsFloatingPoint<T>::value) {
         DCHECK(val.IsFloat());
-        builder.Append(val.GetFloat());
+        builder.Append(val.GetDouble());
       } else if (std::is_base_of<BooleanType, T>::value) {
         DCHECK(val.IsBool());
         builder.Append(val.GetBool());
@@ -853,8 +853,8 @@ class JsonArrayReader {
   typename std::enable_if<std::is_base_of<ListType, T>::value, Status>::type ReadArray(
       const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
       const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-    const auto& json_offsets = json_array.FindMember("OFFSETS");
-    RETURN_NOT_ARRAY("OFFSETS", json_offsets, json_array);
+    const auto& json_offsets = json_array.FindMember("OFFSET");
+    RETURN_NOT_ARRAY("OFFSET", json_offsets, json_array);
     const auto& json_offsets_arr = json_offsets->value.GetArray();
 
     int32_t null_count = 0;
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index b99522825d902..7a2416165b203 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -264,7 +264,7 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>*
   RETURN_NOT_OK(
       TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
 
-  *out = std::make_shared<Field>(field->name()->str(), type);
+  *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index 10af41d16af13..b1e6a11cedd9b 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -48,8 +48,8 @@ class TestArrayPrinter : public ::testing::Test {
 };
 
 template <typename TYPE, typename C_TYPE>
-void CheckPrimitive(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
-    const char* expected) {
+void CheckPrimitive(int indent, const std::vector<bool>& is_valid,
+    const std::vector<C_TYPE>& values, const char* expected) {
   std::ostringstream sink;
 
   MemoryPool* pool = default_memory_pool();
@@ -66,7 +66,7 @@ void CheckPrimitive(const std::vector<bool>& is_valid, const std::vector<C_TYPE>
   std::shared_ptr<Array> array;
   ASSERT_OK(builder.Finish(&array));
 
-  ASSERT_OK(PrettyPrint(*array.get(), &sink));
+  ASSERT_OK(PrettyPrint(*array.get(), indent, &sink));
 
   std::string result = sink.str();
   ASSERT_EQ(std::string(expected, strlen(expected)), result);
@@ -77,11 +77,11 @@ TEST_F(TestArrayPrinter, PrimitiveType) {
 
   std::vector<int32_t> values = {0, 1, 2, 3, 4};
   static const char* expected = R"expected([0, 1, null, 3, null])expected";
-  CheckPrimitive<Int32Type, int32_t>(is_valid, values, expected);
+  CheckPrimitive<Int32Type, int32_t>(0, is_valid, values, expected);
 
   std::vector<std::string> values2 = {"foo", "bar", "", "baz", ""};
   static const char* ex2 = R"expected(["foo", "bar", null, "baz", null])expected";
-  CheckPrimitive<StringType, std::string>(is_valid, values2, ex2);
+  CheckPrimitive<StringType, std::string>(0, is_valid, values2, ex2);
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index c0b4b08274ac1..c63a9e93e6a63 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -16,7 +16,9 @@
 // under the License.
 
 #include <ostream>
+#include <sstream>
 #include <string>
+#include <vector>
 
 #include "arrow/array.h"
 #include "arrow/pretty_print.h"
@@ -32,20 +34,35 @@ namespace arrow {
 
 class ArrayPrinter : public ArrayVisitor {
  public:
-  ArrayPrinter(const Array& array, std::ostream* sink) : array_(array), sink_(sink) {}
+  ArrayPrinter(const Array& array, int indent, std::ostream* sink)
+      : array_(array), indent_(indent), sink_(sink) {}
 
   Status Print() { return VisitArray(array_); }
 
   Status VisitArray(const Array& array) { return array.Accept(this); }
 
   template <typename T>
-  typename std::enable_if<IsNumeric<T>::value, void>::type WriteDataValues(
+  typename std::enable_if<IsInteger<T>::value, void>::type WriteDataValues(
       const T& array) {
     const auto data = array.raw_data();
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) { (*sink_) << ", "; }
       if (array.IsNull(i)) {
         (*sink_) << "null";
+      } else {
+        (*sink_) << static_cast<int64_t>(data[i]);
+      }
+    }
+  }
+
+  template <typename T>
+  typename std::enable_if<IsFloatingPoint<T>::value, void>::type WriteDataValues(
+      const T& array) {
+    const auto data = array.raw_data();
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        Write("null");
       } else {
         (*sink_) << data[i];
       }
@@ -60,7 +77,7 @@ class ArrayPrinter : public ArrayVisitor {
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) { (*sink_) << ", "; }
       if (array.IsNull(i)) {
-        (*sink_) << "null";
+        Write("null");
       } else {
         const char* buf = reinterpret_cast<const char*>(array.GetValue(i, &length));
         (*sink_) << "\"" << std::string(buf, length) << "\"";
@@ -74,9 +91,9 @@ class ArrayPrinter : public ArrayVisitor {
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) { (*sink_) << ", "; }
       if (array.IsNull(i)) {
-        (*sink_) << "null";
+        Write("null");
       } else {
-        (*sink_) << (array.Value(i) ? "true" : "false");
+        Write(array.Value(i) ? "true" : "false");
       }
     }
   }
@@ -148,20 +165,38 @@ class ArrayPrinter : public ArrayVisitor {
   }
 
   Status Visit(const ListArray& array) override {
-    // auto type = static_cast<const ListType*>(array.type().get());
-    // for (size_t i = 0; i < fields.size(); ++i) {
-    //   RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
-    // }
-    // return WriteChildren(type->children(), {array.values()});
+    Newline();
+    Write("-- is_valid: ");
+    BooleanArray is_valid(array.length(), array.null_bitmap());
+    PrettyPrint(is_valid, indent_ + 2, sink_);
+
+    Newline();
+    Write("-- offsets: ");
+    Int32Array offsets(array.length() + 1, array.offsets());
+    PrettyPrint(offsets, indent_ + 2, sink_);
+
+    Newline();
+    Write("-- values: ");
+    PrettyPrint(*array.values().get(), indent_ + 2, sink_);
+
     return Status::OK();
   }
 
   Status Visit(const StructArray& array) override {
-    // auto type = static_cast<const StructType*>(array.type().get());
-    // for (size_t i = 0; i < fields.size(); ++i) {
-    //   RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
-    // }
-    // return WriteChildren(type->children(), array.fields());
+    Newline();
+    Write("-- is_valid: ");
+    BooleanArray is_valid(array.length(), array.null_bitmap());
+    PrettyPrint(is_valid, indent_ + 2, sink_);
+
+    const std::vector<std::shared_ptr<Array>>& fields = array.fields();
+    for (size_t i = 0; i < fields.size(); ++i) {
+      Newline();
+      std::stringstream ss;
+      ss << "-- child " << i << " type: " << fields[i]->type()->ToString() << " values: ";
+      Write(ss.str());
+      PrettyPrint(*fields[i].get(), indent_ + 2, sink_);
+    }
+
     return Status::OK();
   }
 
@@ -169,21 +204,38 @@ class ArrayPrinter : public ArrayVisitor {
     return Status::NotImplemented("union");
   }
 
+  void Write(const char* data) { (*sink_) << data; }
+
+  void Write(const std::string& data) { (*sink_) << data; }
+
+  void Newline() {
+    (*sink_) << "\n";
+    Indent();
+  }
+
+  void Indent() {
+    for (int i = 0; i < indent_; ++i) {
+      (*sink_) << " ";
+    }
+  }
+
  private:
   const Array& array_;
+  int indent_;
+
   std::ostream* sink_;
 };
 
-Status PrettyPrint(const Array& arr, std::ostream* sink) {
-  ArrayPrinter printer(arr, sink);
+Status PrettyPrint(const Array& arr, int indent, std::ostream* sink) {
+  ArrayPrinter printer(arr, indent, sink);
   return printer.Print();
 }
 
-Status PrettyPrint(const RecordBatch& batch, std::ostream* sink) {
+Status PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink) {
   for (int i = 0; i < batch.num_columns(); ++i) {
     const std::string& name = batch.column_name(i);
     (*sink) << name << ": ";
-    RETURN_NOT_OK(PrettyPrint(*batch.column(i).get(), sink));
+    RETURN_NOT_OK(PrettyPrint(*batch.column(i).get(), indent + 2, sink));
     (*sink) << "\n";
   }
   return Status::OK();
diff --git a/cpp/src/arrow/pretty_print.h b/cpp/src/arrow/pretty_print.h
index dcb236d726949..f508aa042945a 100644
--- a/cpp/src/arrow/pretty_print.h
+++ b/cpp/src/arrow/pretty_print.h
@@ -27,8 +27,12 @@ namespace arrow {
 
 class Status;
 
-Status ARROW_EXPORT PrettyPrint(const RecordBatch& batch, std::ostream* sink);
-Status ARROW_EXPORT PrettyPrint(const Array& arr, std::ostream* sink);
+struct PrettyPrintOptions {
+  int indent;
+};
+
+Status ARROW_EXPORT PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink);
+Status ARROW_EXPORT PrettyPrint(const Array& arr, int indent, std::ostream* sink);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index af84f27eab557..eb1258a73038a 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -48,6 +48,18 @@ bool RecordBatch::Equals(const RecordBatch& other) const {
   return true;
 }
 
+bool RecordBatch::ApproxEquals(const RecordBatch& other) const {
+  if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
+    return false;
+  }
+
+  for (int i = 0; i < num_columns(); ++i) {
+    if (!column(i)->ApproxEquals(other.column(i))) { return false; }
+  }
+
+  return true;
+}
+
 // ----------------------------------------------------------------------
 // Table methods
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 1a856c8a436d5..f2c334ff626a4 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -45,6 +45,8 @@ class ARROW_EXPORT RecordBatch {
 
   bool Equals(const RecordBatch& other) const;
 
+  bool ApproxEquals(const RecordBatch& other) const;
+
   // @returns: the table's schema
   const std::shared_ptr<Schema>& schema() const { return schema_; }
 
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 80f295c487f13..dc955ac62d36c 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -209,7 +209,7 @@ std::vector<BufferDescr> ListType::GetBufferLayout() const {
 }
 
 std::vector<BufferDescr> StructType::GetBufferLayout() const {
-  return {kValidityBuffer, kTypeBuffer};
+  return {kValidityBuffer};
 }
 
 std::vector<BufferDescr> UnionType::GetBufferLayout() const {
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index c21c5002035f8..3aaec0bd5935a 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -186,6 +186,12 @@ struct IsSignedInt {
       std::is_integral<c_type>::value && std::is_signed<c_type>::value;
 };
 
+template <typename T>
+struct IsInteger {
+  PRIMITIVE_TRAITS(T);
+  static constexpr bool value = std::is_integral<c_type>::value;
+};
+
 template <typename T>
 struct IsFloatingPoint {
   PRIMITIVE_TRAITS(T);
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
index f42a3cac021cd..75e5a9ff40e16 100644
--- a/cpp/src/arrow/types/primitive.cc
+++ b/cpp/src/arrow/types/primitive.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/types/primitive.h"
 
+#include <cmath>
 #include <memory>
 
 #include "arrow/type_traits.h"
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index a5a3704e2d2d3..c665218b4448c 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -18,8 +18,10 @@
 #ifndef ARROW_TYPES_PRIMITIVE_H
 #define ARROW_TYPES_PRIMITIVE_H
 
+#include <cmath>
 #include <cstdint>
 #include <cstring>
+#include <iostream>
 #include <memory>
 #include <vector>
 
@@ -55,7 +57,7 @@ class ARROW_EXPORT PrimitiveArray : public Array {
   const uint8_t* raw_data_;
 };
 
-template <class TYPE>
+template <typename TYPE>
 class ARROW_EXPORT NumericArray : public PrimitiveArray {
  public:
   using TypeClass = TYPE;
@@ -69,9 +71,11 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
       : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
 
   bool EqualsExact(const NumericArray<TypeClass>& other) const {
-    return PrimitiveArray::EqualsExact(*static_cast<const PrimitiveArray*>(&other));
+    return PrimitiveArray::EqualsExact(static_cast<const PrimitiveArray&>(other));
   }
 
+  bool ApproxEquals(const std::shared_ptr<Array>& arr) const { return Equals(arr); }
+
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
       const ArrayPtr& arr) const override {
     if (this == arr.get()) { return true; }
@@ -95,6 +99,78 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   value_type Value(int i) const { return raw_data()[i]; }
 };
 
+template <>
+inline bool NumericArray<FloatType>::ApproxEquals(
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+
+  const auto& other = *static_cast<NumericArray<FloatType>*>(arr.get());
+
+  if (this == &other) { return true; }
+  if (null_count_ != other.null_count_) { return false; }
+
+  auto this_data = reinterpret_cast<const float*>(raw_data_);
+  auto other_data = reinterpret_cast<const float*>(other.raw_data_);
+
+  static constexpr float EPSILON = 1E-5;
+
+  if (length_ == 0 && other.length_ == 0) { return true; }
+
+  if (null_count_ > 0) {
+    bool equal_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
+    if (!equal_bitmap) { return false; }
+
+    for (int i = 0; i < length_; ++i) {
+      if (IsNull(i)) continue;
+      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+    }
+  } else {
+    for (int i = 0; i < length_; ++i) {
+      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+    }
+  }
+  return true;
+}
+
+template <>
+inline bool NumericArray<DoubleType>::ApproxEquals(
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+
+  const auto& other = *static_cast<NumericArray<DoubleType>*>(arr.get());
+
+  if (this == &other) { return true; }
+  if (null_count_ != other.null_count_) { return false; }
+
+  auto this_data = reinterpret_cast<const double*>(raw_data_);
+  auto other_data = reinterpret_cast<const double*>(other.raw_data_);
+
+  if (length_ == 0 && other.length_ == 0) { return true; }
+
+  static constexpr double EPSILON = 1E-5;
+
+  if (null_count_ > 0) {
+    bool equal_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
+    if (!equal_bitmap) { return false; }
+
+    for (int i = 0; i < length_; ++i) {
+      if (IsNull(i)) continue;
+      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+    }
+  } else {
+    for (int i = 0; i < length_; ++i) {
+      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+    }
+  }
+  return true;
+}
+
 template <typename Type>
 class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
  public:
@@ -265,6 +341,13 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   uint8_t* raw_data_;
 };
 
+// gcc and clang disagree about how to handle template visibility when you have
+// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif
+
 // Only instantiate these templates once
 extern template class ARROW_EXPORT NumericArray<Int8Type>;
 extern template class ARROW_EXPORT NumericArray<UInt8Type>;
@@ -279,6 +362,10 @@ extern template class ARROW_EXPORT NumericArray<FloatType>;
 extern template class ARROW_EXPORT NumericArray<DoubleType>;
 extern template class ARROW_EXPORT NumericArray<TimestampType>;
 
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPES_PRIMITIVE_H
diff --git a/integration/README.md b/integration/README.md
new file mode 100644
index 0000000000000..b1e4e3a82a734
--- /dev/null
+++ b/integration/README.md
@@ -0,0 +1,59 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Arrow integration testing
+
+Our strategy for integration testing between Arrow implementations is as follows:
+
+* Test datasets are specified in a custom human-readable, JSON-based format
+  designed for Arrow
+
+* Each implementation provides a testing executable capable of converting
+  between the JSON and the binary Arrow file representation
+
+* The test executable is also capable of validating the contents of a binary
+  file against a corresponding JSON file
+
+## Running the existing integration tests
+
+First, build the Java and C++ projects. For Java, you must run
+
+```
+mvn package
+```
+
+Now, the integration tests rely on two environment variables which point to the
+Java `arrow-tool` JAR and the C++ `json-integration-test` executable:
+
+```bash
+JAVA_DIR=$ARROW_HOME/java
+CPP_BUILD_DIR=$ARROW_HOME/cpp/test-build
+
+VERSION=0.1.1-SNAPSHOT
+export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
+export ARROW_CPP_TESTER=$CPP_BUILD_DIR/debug/json-integration-test
+```
+
+Here `$ARROW_HOME` is the location of your Arrow git clone. The
+`$CPP_BUILD_DIR` may be different depending on how you built with CMake
+(in-source of out-of-source).
+
+Once this is done, run the integration tests with (optionally adding `--debug`
+for additional output)
+
+```
+python integration_test.py
+
+python integration_test.py --debug  # additional output
+```
\ No newline at end of file
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 88dc3ad7971ff..417354bc83d9e 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -15,23 +15,53 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from collections import OrderedDict
 import argparse
 import glob
 import itertools
+import json
 import os
 import six
+import string
 import subprocess
 import tempfile
 import uuid
 
+import numpy as np
 
 ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
 
+# Control for flakiness
+np.random.seed(12345)
+
 
 def guid():
     return uuid.uuid4().hex
 
 
+# from pandas
+RANDS_CHARS = np.array(list(string.ascii_letters + string.digits),
+                       dtype=(np.str_, 1))
+
+
+def rands(nchars):
+    """
+    Generate one random byte string.
+
+    See `rands_array` if you want to create an array of random strings.
+
+    """
+    return ''.join(np.random.choice(RANDS_CHARS, nchars))
+
+
+def str_from_bytes(x):
+    if six.PY2:
+        return x
+    else:
+        return x.decode('utf-8')
+
+
+# from the merge_arrow_pr.py script
 def run_cmd(cmd):
     if isinstance(cmd, six.string_types):
         cmd = cmd.split(' ')
@@ -43,13 +73,469 @@ def run_cmd(cmd):
         print('Command failed: %s' % ' '.join(cmd))
         print('With output:')
         print('--------------')
-        print(e.output)
+        print(str_from_bytes(e.output))
         print('--------------')
         raise e
 
-    if isinstance(output, six.binary_type):
-        output = output.decode('utf-8')
-    return output
+    return str_from_bytes(output)
+
+# ----------------------------------------------------------------------
+# Data generation
+
+
+class DataType(object):
+
+    def __init__(self, name, nullable=True):
+        self.name = name
+        self.nullable = nullable
+
+    def get_json(self):
+        return OrderedDict([
+            ('name', self.name),
+            ('type', self._get_type()),
+            ('nullable', self.nullable),
+            ('children', self._get_children()),
+            ('typeLayout', self._get_type_layout())
+        ])
+
+    def _make_is_valid(self, size):
+        if self.nullable:
+            return np.random.randint(0, 2, size=size)
+        else:
+            return np.ones(size)
+
+
+class Column(object):
+
+    def __init__(self, name, count):
+        self.name = name
+        self.count = count
+
+    def _get_children(self):
+        return []
+
+    def _get_buffers(self):
+        return []
+
+    def get_json(self):
+        entries = [
+            ('name', self.name),
+            ('count', self.count)
+        ]
+
+        buffers = self._get_buffers()
+        entries.extend(buffers)
+
+        children = self._get_children()
+        if len(children) > 0:
+            entries.append(('children', children))
+
+        return OrderedDict(entries)
+
+
+class PrimitiveType(DataType):
+
+    def _get_children(self):
+        return []
+
+    def _get_type_layout(self):
+        return OrderedDict([
+            ('vectors',
+             [OrderedDict([('type', 'VALIDITY'),
+                           ('typeBitWidth', 1)]),
+              OrderedDict([('type', 'DATA'),
+                           ('typeBitWidth', self.bit_width)])])])
+
+
+class PrimitiveColumn(Column):
+
+    def __init__(self, name, count, is_valid, values):
+        Column.__init__(self, name, count)
+        self.is_valid = is_valid
+        self.values = values
+
+    def _get_buffers(self):
+        return [
+            ('VALIDITY', [int(v) for v in self.is_valid]),
+            ('DATA', list(self.values))
+        ]
+
+
+TEST_INT_MIN = - 2**31 + 1
+TEST_INT_MAX = 2**31 - 1
+
+
+class IntegerType(PrimitiveType):
+
+    def __init__(self, name, is_signed, bit_width, nullable=True):
+        PrimitiveType.__init__(self, name, nullable=nullable)
+        self.is_signed = is_signed
+        self.bit_width = bit_width
+
+    @property
+    def numpy_type(self):
+        return ('int' if self.is_signed else 'uint') + str(self.bit_width)
+
+    def _get_type(self):
+        return OrderedDict([
+            ('name', 'int'),
+            ('isSigned', self.is_signed),
+            ('bitWidth', self.bit_width)
+        ])
+
+    def generate_column(self, size):
+        iinfo = np.iinfo(self.numpy_type)
+        values = [int(x) for x in
+                  np.random.randint(max(iinfo.min, TEST_INT_MIN),
+                                    min(iinfo.max, TEST_INT_MAX),
+                                    size=size)]
+
+        is_valid = self._make_is_valid(size)
+        return PrimitiveColumn(self.name, size, is_valid, values)
+
+
+class FloatingPointType(PrimitiveType):
+
+    def __init__(self, name, bit_width, nullable=True):
+        PrimitiveType.__init__(self, name, nullable=nullable)
+
+        self.bit_width = bit_width
+        self.precision = {
+            16: 'HALF',
+            32: 'SINGLE',
+            64: 'DOUBLE'
+        }[self.bit_width]
+
+    @property
+    def numpy_type(self):
+        return 'float' + str(self.bit_width)
+
+    def _get_type(self):
+        return OrderedDict([
+            ('name', 'floatingpoint'),
+            ('precision', self.precision)
+        ])
+
+    def generate_column(self, size):
+        values = np.random.randn(size) * 1000
+        values = np.round(values, 3)
+
+        is_valid = self._make_is_valid(size)
+        return PrimitiveColumn(self.name, size, is_valid, values)
+
+
+class BooleanType(PrimitiveType):
+
+    bit_width = 1
+
+    def _get_type(self):
+        return OrderedDict([('name', 'bool')])
+
+    @property
+    def numpy_type(self):
+        return 'bool'
+
+    def generate_column(self, size):
+        values = list(map(bool, np.random.randint(0, 2, size=size)))
+        is_valid = self._make_is_valid(size)
+        return PrimitiveColumn(self.name, size, is_valid, values)
+
+
+class StringType(PrimitiveType):
+
+    @property
+    def numpy_type(self):
+        return object
+
+    def _get_type(self):
+        return OrderedDict([('name', 'utf8')])
+
+    def _get_type_layout(self):
+        return OrderedDict([
+            ('vectors',
+             [OrderedDict([('type', 'VALIDITY'),
+                           ('typeBitWidth', 1)]),
+              OrderedDict([('type', 'OFFSET'),
+                           ('typeBitWidth', 32)]),
+              OrderedDict([('type', 'DATA'),
+                           ('typeBitWidth', 8)])])])
+
+    def generate_column(self, size):
+        K = 7
+        is_valid = self._make_is_valid(size)
+        values = []
+
+        for i in range(size):
+            if is_valid[i]:
+                values.append(rands(K))
+            else:
+                values.append("")
+
+        return StringColumn(self.name, size, is_valid, values)
+
+
+class JSONSchema(object):
+
+    def __init__(self, fields):
+        self.fields = fields
+
+    def get_json(self):
+        return OrderedDict([
+            ('fields', [field.get_json() for field in self.fields])
+        ])
+
+
+class StringColumn(PrimitiveColumn):
+
+    def _get_buffers(self):
+        offset = 0
+        offsets = [0]
+
+        data = []
+        for i, v in enumerate(self.values):
+            if self.is_valid[i]:
+                offset += len(v)
+            else:
+                v = ""
+
+            offsets.append(offset)
+            data.append(v)
+
+        return [
+            ('VALIDITY', [int(x) for x in self.is_valid]),
+            ('OFFSET', offsets),
+            ('DATA', data)
+        ]
+
+
+class ListType(DataType):
+
+    def __init__(self, name, value_type, nullable=True):
+        DataType.__init__(self, name, nullable=nullable)
+        self.value_type = value_type
+
+    def _get_type(self):
+        return OrderedDict([
+            ('name', 'list')
+        ])
+
+    def _get_children(self):
+        return [self.value_type.get_json()]
+
+    def _get_type_layout(self):
+        return OrderedDict([
+            ('vectors',
+             [OrderedDict([('type', 'VALIDITY'),
+                           ('typeBitWidth', 1)]),
+              OrderedDict([('type', 'OFFSET'),
+                           ('typeBitWidth', 32)])])])
+
+    def generate_column(self, size):
+        MAX_LIST_SIZE = 4
+
+        is_valid = self._make_is_valid(size)
+        list_sizes = np.random.randint(0, MAX_LIST_SIZE + 1, size=size)
+        offsets = [0]
+
+        offset = 0
+        for i in range(size):
+            if is_valid[i]:
+                offset += int(list_sizes[i])
+            offsets.append(offset)
+
+        # The offset now is the total number of elements in the child array
+        values = self.value_type.generate_column(offset)
+
+        return ListColumn(self.name, size, is_valid, offsets, values)
+
+
+class ListColumn(Column):
+
+    def __init__(self, name, count, is_valid, offsets, values):
+        Column.__init__(self, name, count)
+        self.is_valid = is_valid
+        self.offsets = offsets
+        self.values = values
+
+    def _get_buffers(self):
+        return [
+            ('VALIDITY', [int(v) for v in self.is_valid]),
+            ('OFFSET', list(self.offsets))
+        ]
+
+    def _get_children(self):
+        return [self.values.get_json()]
+
+
+class StructType(DataType):
+
+    def __init__(self, name, field_types, nullable=True):
+        DataType.__init__(self, name, nullable=nullable)
+        self.field_types = field_types
+
+    def _get_type(self):
+        return OrderedDict([
+            ('name', 'struct')
+        ])
+
+    def _get_children(self):
+        return [type_.get_json() for type_ in self.field_types]
+
+    def _get_type_layout(self):
+        return OrderedDict([
+            ('vectors',
+             [OrderedDict([('type', 'VALIDITY'),
+                           ('typeBitWidth', 1)])])])
+
+    def generate_column(self, size):
+        is_valid = self._make_is_valid(size)
+
+        field_values = [type_.generate_column(size)
+                        for type_ in self.field_types]
+
+        return StructColumn(self.name, size, is_valid, field_values)
+
+
+class StructColumn(Column):
+
+    def __init__(self, name, count, is_valid, field_values):
+        Column.__init__(self, name, count)
+        self.is_valid = is_valid
+        self.field_values = field_values
+
+    def _get_buffers(self):
+        return [
+            ('VALIDITY', [int(v) for v in self.is_valid])
+        ]
+
+    def _get_children(self):
+        return [field.get_json() for field in self.field_values]
+
+
+class JSONRecordBatch(object):
+
+    def __init__(self, count, columns):
+        self.count = count
+        self.columns = columns
+
+    def get_json(self):
+        return OrderedDict([
+            ('count', self.count),
+            ('columns', [col.get_json() for col in self.columns])
+        ])
+
+
+class JSONFile(object):
+
+    def __init__(self, schema, batches):
+        self.schema = schema
+        self.batches = batches
+
+    def get_json(self):
+        return OrderedDict([
+            ('schema', self.schema.get_json()),
+            ('batches', [batch.get_json() for batch in self.batches])
+        ])
+
+    def write(self, path):
+        with open(path, 'wb') as f:
+            f.write(json.dumps(self.get_json(), indent=2).encode('utf-8'))
+
+
+def get_field(name, type_, nullable=True):
+    if type_ == 'utf8':
+        return StringType(name, nullable=nullable)
+
+    dtype = np.dtype(type_)
+
+    if dtype.kind in ('i', 'u'):
+        return IntegerType(name, dtype.kind == 'i', dtype.itemsize * 8,
+                           nullable=nullable)
+    elif dtype.kind == 'f':
+        return FloatingPointType(name, dtype.itemsize * 8,
+                                 nullable=nullable)
+    elif dtype.kind == 'b':
+        return BooleanType(name, nullable=nullable)
+    else:
+        raise TypeError(dtype)
+
+
+def generate_primitive_case():
+    types = ['bool', 'int8', 'int16', 'int32', 'int64',
+             'uint8', 'uint16', 'uint32', 'uint64',
+             'float32', 'float64', 'utf8']
+
+    fields = []
+
+    for type_ in types:
+        fields.append(get_field(type_ + "_nullable", type_, True))
+        fields.append(get_field(type_ + "_nonnullable", type_, False))
+
+    schema = JSONSchema(fields)
+
+    batch_sizes = [7, 10]
+    batches = []
+    for size in batch_sizes:
+        columns = []
+        for field in fields:
+            col = field.generate_column(size)
+            columns.append(col)
+
+        batches.append(JSONRecordBatch(size, columns))
+
+    return JSONFile(schema, batches)
+
+
+def generate_nested_case():
+    fields = [
+        ListType('list_nullable', get_field('item', 'int32')),
+        StructType('struct_nullable', [get_field('f1', 'int32'),
+                                       get_field('f2', 'utf8')]),
+
+        # TODO(wesm): this causes segfault
+        # ListType('list_nonnullable', get_field('item', 'int32'), False),
+    ]
+
+    schema = JSONSchema(fields)
+
+    batch_sizes = [7, 10]
+    batches = []
+    for size in batch_sizes:
+        columns = []
+        for field in fields:
+            col = field.generate_column(size)
+            columns.append(col)
+
+        batches.append(JSONRecordBatch(size, columns))
+
+    return JSONFile(schema, batches)
+
+
+def get_generated_json_files():
+    temp_dir = tempfile.mkdtemp()
+
+    def _temp_path():
+        return
+
+    file_objs = []
+
+    K = 10
+    for i in range(K):
+        file_objs.append(generate_primitive_case())
+
+    file_objs.append(generate_nested_case())
+
+    generated_paths = []
+    for file_obj in file_objs:
+        out_path = os.path.join(temp_dir, guid() + '.json')
+        file_obj.write(out_path)
+        generated_paths.append(out_path)
+
+    return generated_paths
+
+
+# ----------------------------------------------------------------------
+# Testing harness
 
 
 class IntegrationRunner(object):
@@ -92,9 +578,11 @@ def validate(self, json_path, arrow_path):
 
 class JavaTester(Tester):
 
-    ARROW_TOOLS_JAR = os.path.join(ARROW_HOME,
-                                   'java/tools/target/arrow-tools-0.1.1-'
-                                   'SNAPSHOT-jar-with-dependencies.jar')
+    ARROW_TOOLS_JAR = os.environ.get(
+        'ARROW_JAVA_INTEGRATION_JAR',
+        os.path.join(ARROW_HOME,
+                     'java/tools/target/arrow-tools-0.1.1-'
+                     'SNAPSHOT-jar-with-dependencies.jar'))
 
     name = 'Java'
 
@@ -154,14 +642,16 @@ def json_to_arrow(self, json_path, arrow_path):
         return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
 
 
-def get_json_files():
+def get_static_json_files():
     glob_pattern = os.path.join(ARROW_HOME, 'integration', 'data', '*.json')
     return glob.glob(glob_pattern)
 
 
 def run_all_tests(debug=False):
     testers = [JavaTester(debug=debug), CPPTester(debug=debug)]
-    json_files = get_json_files()
+    static_json_files = get_static_json_files()
+    generated_json_files = get_generated_json_files()
+    json_files = static_json_files + generated_json_files
 
     runner = IntegrationRunner(json_files, testers, debug=debug)
     runner.run()
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index b7df8d13ee607..7fe1615da5a27 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -30,9 +30,6 @@ public abstract class BaseDataValueVector extends BaseValueVector implements Buf
 
   protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
 
-  /** maximum extra size at the end of the buffer */
-  private static final int MAX_BUFFER_PADDING = 64;
-
   public static void load(ArrowFieldNode fieldNode, List<BufferBacked> vectors, List<ArrowBuf> buffers) {
     int expectedSize = vectors.size();
     if (buffers.size() != expectedSize) {
@@ -51,9 +48,6 @@ public static void truncateBufferBasedOnSize(List<ArrowBuf> buffers, int bufferI
     if (buffer.writerIndex() < byteSize) {
       throw new IllegalArgumentException("can not truncate buffer to a larger size " + byteSize + ": " + buffer.writerIndex());
     }
-    if (buffer.writerIndex() - byteSize > MAX_BUFFER_PADDING) {
-      throw new IllegalArgumentException("Buffer too large to resize to " + byteSize + ": " + buffer.writerIndex());
-    }
     buffer.writerIndex(byteSize);
   }
 

From 73fe55683c36465972e21bef01b377c3b66579f9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 10 Dec 2016 09:05:48 +0100
Subject: [PATCH 0227/1644] ARROW-328: Return shared_ptr<T> by value instead of
 const-ref

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #235 from wesm/ARROW-328 and squashes the following commits:

f71decc [Wes McKinney] Return shared_ptr<T> by value instead of const-ref
---
 cpp/src/arrow/array.h                    |  4 ++--
 cpp/src/arrow/builder.h                  |  4 ++--
 cpp/src/arrow/column.cc                  |  2 +-
 cpp/src/arrow/column.h                   |  8 ++++----
 cpp/src/arrow/ipc/file.cc                |  2 +-
 cpp/src/arrow/ipc/file.h                 |  2 +-
 cpp/src/arrow/ipc/metadata-internal.cc   |  2 +-
 cpp/src/arrow/table.h                    |  8 ++++----
 cpp/src/arrow/type.cc                    |  2 +-
 cpp/src/arrow/type.h                     |  6 +++---
 cpp/src/arrow/types/construct.cc         |  2 +-
 cpp/src/arrow/types/list.h               |  6 +++---
 cpp/src/arrow/types/primitive.h          |  2 +-
 cpp/src/arrow/types/struct.h             |  4 ++--
 cpp/src/arrow/util/buffer.h              |  2 +-
 python/pyarrow/includes/libarrow.pxd     | 22 +++++++++++-----------
 python/pyarrow/includes/libarrow_ipc.pxd |  2 +-
 17 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 78aa2b867e1ea..91fb93e625494 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -53,10 +53,10 @@ class ARROW_EXPORT Array {
   int32_t length() const { return length_; }
   int32_t null_count() const { return null_count_; }
 
-  const std::shared_ptr<DataType>& type() const { return type_; }
+  std::shared_ptr<DataType> type() const { return type_; }
   Type::type type_enum() const { return type_->type; }
 
-  const std::shared_ptr<Buffer>& null_bitmap() const { return null_bitmap_; }
+  std::shared_ptr<Buffer> null_bitmap() const { return null_bitmap_; }
 
   const uint8_t* null_bitmap_data() const { return null_bitmap_data_; }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index cef17e5aabab9..73e49c0a69674 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -89,13 +89,13 @@ class ARROW_EXPORT ArrayBuilder {
   // this function responsibly.
   Status Advance(int32_t elements);
 
-  const std::shared_ptr<PoolBuffer>& null_bitmap() const { return null_bitmap_; }
+  std::shared_ptr<PoolBuffer> null_bitmap() const { return null_bitmap_; }
 
   // Creates new array object to hold the contents of the builder and transfers
   // ownership of the data.  This resets all variables on the builder.
   virtual Status Finish(std::shared_ptr<Array>* out) = 0;
 
-  const std::shared_ptr<DataType>& type() const { return type_; }
+  std::shared_ptr<DataType> type() const { return type_; }
 
  protected:
   MemoryPool* pool_;
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index 52e4c58e1dc3d..eca5f4d30a698 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -51,7 +51,7 @@ Column::Column(
 
 Status Column::ValidateData() {
   for (int i = 0; i < data_->num_chunks(); ++i) {
-    const std::shared_ptr<DataType>& type = data_->chunk(i)->type();
+    std::shared_ptr<DataType> type = data_->chunk(i)->type();
     if (!this->type()->Equals(type)) {
       std::stringstream ss;
       ss << "In chunk " << i << " expected type " << this->type()->ToString()
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
index d5168cb032ba5..1caafec9db95c 100644
--- a/cpp/src/arrow/column.h
+++ b/cpp/src/arrow/column.h
@@ -46,7 +46,7 @@ class ARROW_EXPORT ChunkedArray {
 
   int num_chunks() const { return chunks_.size(); }
 
-  const std::shared_ptr<Array>& chunk(int i) const { return chunks_[i]; }
+  std::shared_ptr<Array> chunk(int i) const { return chunks_[i]; }
 
  protected:
   ArrayVector chunks_;
@@ -68,16 +68,16 @@ class ARROW_EXPORT Column {
 
   int64_t null_count() const { return data_->null_count(); }
 
-  const std::shared_ptr<Field>& field() const { return field_; }
+  std::shared_ptr<Field> field() const { return field_; }
 
   // @returns: the column's name in the passed metadata
   const std::string& name() const { return field_->name; }
 
   // @returns: the column's type according to the metadata
-  const std::shared_ptr<DataType>& type() const { return field_->type; }
+  std::shared_ptr<DataType> type() const { return field_->type; }
 
   // @returns: the column's data as a chunked logical array
-  const std::shared_ptr<ChunkedArray>& data() const { return data_; }
+  std::shared_ptr<ChunkedArray> data() const { return data_; }
   // Verify that the column's array data is consistent with the passed field's
   // metadata
   Status ValidateData();
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
index 06001cc1c77bc..fa50058ea4200 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/file.cc
@@ -179,7 +179,7 @@ Status FileReader::ReadFooter() {
   return footer_->GetSchema(&schema_);
 }
 
-const std::shared_ptr<Schema>& FileReader::schema() const {
+std::shared_ptr<Schema> FileReader::schema() const {
   return schema_;
 }
 
diff --git a/cpp/src/arrow/ipc/file.h b/cpp/src/arrow/ipc/file.h
index 4b79c98281bbc..4f35c37b03235 100644
--- a/cpp/src/arrow/ipc/file.h
+++ b/cpp/src/arrow/ipc/file.h
@@ -106,7 +106,7 @@ class ARROW_EXPORT FileReader {
   static Status Open(const std::shared_ptr<io::ReadableFileInterface>& file,
       int64_t footer_offset, std::shared_ptr<FileReader>* reader);
 
-  const std::shared_ptr<Schema>& schema() const;
+  std::shared_ptr<Schema> schema() const;
 
   // Shared dictionaries for dictionary-encoding cross record batches
   // TODO(wesm): Implement dictionary reading when we also have dictionary
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 7a2416165b203..5a2758912b759 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -285,7 +285,7 @@ Status SchemaToFlatbuffer(
     FBB& fbb, const Schema* schema, flatbuffers::Offset<flatbuf::Schema>* out) {
   std::vector<FieldOffset> field_offsets;
   for (int i = 0; i < schema->num_fields(); ++i) {
-    const std::shared_ptr<Field>& field = schema->field(i);
+    std::shared_ptr<Field> field = schema->field(i);
     FieldOffset offset;
     RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, &offset));
     field_offsets.push_back(offset);
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index f2c334ff626a4..bf5c39f11e411 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -48,11 +48,11 @@ class ARROW_EXPORT RecordBatch {
   bool ApproxEquals(const RecordBatch& other) const;
 
   // @returns: the table's schema
-  const std::shared_ptr<Schema>& schema() const { return schema_; }
+  std::shared_ptr<Schema> schema() const { return schema_; }
 
   // @returns: the i-th column
   // Note: Does not boundscheck
-  const std::shared_ptr<Array>& column(int i) const { return columns_[i]; }
+  std::shared_ptr<Array> column(int i) const { return columns_[i]; }
 
   const std::vector<std::shared_ptr<Array>>& columns() const { return columns_; }
 
@@ -88,11 +88,11 @@ class ARROW_EXPORT Table {
   const std::string& name() const { return name_; }
 
   // @returns: the table's schema
-  const std::shared_ptr<Schema>& schema() const { return schema_; }
+  std::shared_ptr<Schema> schema() const { return schema_; }
 
   // Note: Does not boundscheck
   // @returns: the i-th column
-  const std::shared_ptr<Column>& column(int i) const { return columns_[i]; }
+  std::shared_ptr<Column> column(int i) const { return columns_[i]; }
 
   // @returns: the number of columns in the table
   int num_columns() const { return columns_.size(); }
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index dc955ac62d36c..75f5086f37de0 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -81,7 +81,7 @@ std::string StructType::ToString() const {
   s << "struct<";
   for (int i = 0; i < this->num_children(); ++i) {
     if (i > 0) { s << ", "; }
-    const std::shared_ptr<Field>& field = this->child(i);
+    std::shared_ptr<Field> field = this->child(i);
     s << field->name << ": " << field->type->ToString();
   }
   s << ">";
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 30777384dfb9f..966706cb520b2 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -134,7 +134,7 @@ struct ARROW_EXPORT DataType {
     return Equals(other.get());
   }
 
-  const std::shared_ptr<Field>& child(int i) const { return children_[i]; }
+  std::shared_ptr<Field> child(int i) const { return children_[i]; }
 
   const std::vector<std::shared_ptr<Field>>& children() const { return children_; }
 
@@ -319,9 +319,9 @@ struct ARROW_EXPORT ListType : public DataType, public NoExtraMeta {
     children_ = {value_field};
   }
 
-  const std::shared_ptr<Field>& value_field() const { return children_[0]; }
+  std::shared_ptr<Field> value_field() const { return children_[0]; }
 
-  const std::shared_ptr<DataType>& value_type() const { return children_[0]->type; }
+  std::shared_ptr<DataType> value_type() const { return children_[0]->type; }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
index 67245f8ea1fda..ab9c59fd4639d 100644
--- a/cpp/src/arrow/types/construct.cc
+++ b/cpp/src/arrow/types/construct.cc
@@ -63,7 +63,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
-      const std::shared_ptr<DataType>& value_type =
+      std::shared_ptr<DataType> value_type =
           static_cast<ListType*>(type.get())->value_type();
       RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
       out->reset(new ListBuilder(pool, value_builder));
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
index bd93e8fdcfa1c..ec09a78afa66c 100644
--- a/cpp/src/arrow/types/list.h
+++ b/cpp/src/arrow/types/list.h
@@ -57,12 +57,12 @@ class ARROW_EXPORT ListArray : public Array {
 
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
-  const std::shared_ptr<Array>& values() const { return values_; }
+  std::shared_ptr<Array> values() const { return values_; }
   std::shared_ptr<Buffer> offsets() const {
     return std::static_pointer_cast<Buffer>(offset_buffer_);
   }
 
-  const std::shared_ptr<DataType>& value_type() const { return values_->type(); }
+  std::shared_ptr<DataType> value_type() const { return values_->type(); }
 
   const int32_t* raw_offsets() const { return offsets_; }
 
@@ -152,7 +152,7 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
 
   Status AppendNull() { return Append(false); }
 
-  const std::shared_ptr<ArrayBuilder>& value_builder() const {
+  std::shared_ptr<ArrayBuilder> value_builder() const {
     DCHECK(!values_) << "Using value builder is pointless when values_ is set";
     return value_builder_;
   }
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
index c665218b4448c..ec578e1e0aee7 100644
--- a/cpp/src/arrow/types/primitive.h
+++ b/cpp/src/arrow/types/primitive.h
@@ -45,7 +45,7 @@ class ARROW_EXPORT PrimitiveArray : public Array {
  public:
   virtual ~PrimitiveArray() {}
 
-  const std::shared_ptr<Buffer>& data() const { return data_; }
+  std::shared_ptr<Buffer> data() const { return data_; }
 
   bool EqualsExact(const PrimitiveArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
index 035af05132572..1e2bf2d9a1223 100644
--- a/cpp/src/arrow/types/struct.h
+++ b/cpp/src/arrow/types/struct.h
@@ -46,7 +46,7 @@ class ARROW_EXPORT StructArray : public Array {
 
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
-  const std::shared_ptr<Array>& field(int32_t pos) const {
+  std::shared_ptr<Array> field(int32_t pos) const {
     DCHECK_GT(field_arrays_.size(), 0);
     return field_arrays_[pos];
   }
@@ -99,7 +99,7 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
 
   Status AppendNull() { return Append(false); }
 
-  const std::shared_ptr<ArrayBuilder> field_builder(int pos) const {
+  std::shared_ptr<ArrayBuilder> field_builder(int pos) const {
     DCHECK_GT(field_builders_.size(), 0);
     return field_builders_[pos];
   }
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
index 330e15feae152..5c87395deebb0 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/util/buffer.h
@@ -86,7 +86,7 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
 
   int64_t size() const { return size_; }
 
-  const std::shared_ptr<Buffer> parent() const { return parent_; }
+  std::shared_ptr<Buffer> parent() const { return parent_; }
 
  protected:
   bool is_mutable_;
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 350ebe30c9b89..15781ced4433a 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -91,12 +91,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_bool Equals(const shared_ptr[CSchema]& other)
 
-        const shared_ptr[CField]& field(int i)
+        shared_ptr[CField] field(int i)
         int num_fields()
         c_string ToString()
 
     cdef cppclass CArray" arrow::Array":
-        const shared_ptr[CDataType]& type()
+        shared_ptr[CDataType] type()
 
         int32_t length()
         int32_t null_count()
@@ -142,8 +142,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         const int32_t* offsets()
         int32_t offset(int i)
         int32_t value_length(int i)
-        const shared_ptr[CArray]& values()
-        const shared_ptr[CDataType]& value_type()
+        shared_ptr[CArray] values()
+        shared_ptr[CDataType] value_type()
 
     cdef cppclass CStringArray" arrow::StringArray"(CListArray):
         c_string GetString(int i)
@@ -152,7 +152,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int64_t length()
         int64_t null_count()
         int num_chunks()
-        const shared_ptr[CArray]& chunk(int i)
+        shared_ptr[CArray] chunk(int i)
 
     cdef cppclass CColumn" arrow::Column":
         CColumn(const shared_ptr[CField]& field,
@@ -164,8 +164,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int64_t length()
         int64_t null_count()
         const c_string& name()
-        const shared_ptr[CDataType]& type()
-        const shared_ptr[CChunkedArray]& data()
+        shared_ptr[CDataType] type()
+        shared_ptr[CChunkedArray] data()
 
     cdef cppclass CRecordBatch" arrow::RecordBatch":
         CRecordBatch(const shared_ptr[CSchema]& schema, int32_t num_rows,
@@ -173,8 +173,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_bool Equals(const CRecordBatch& other)
 
-        const shared_ptr[CSchema]& schema()
-        const shared_ptr[CArray]& column(int i)
+        shared_ptr[CSchema] schema()
+        shared_ptr[CArray] column(int i)
         const c_string& column_name(int i)
 
         const vector[shared_ptr[CArray]]& columns()
@@ -191,8 +191,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         const c_string& name()
 
-        const shared_ptr[CSchema]& schema()
-        const shared_ptr[CColumn]& column(int i)
+        shared_ptr[CSchema] schema()
+        shared_ptr[CColumn] column(int i)
 
 
 cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
index eda5b9bae9e31..b3185b1c1671c 100644
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -44,7 +44,7 @@ cdef extern from "arrow/ipc/file.h" namespace "arrow::ipc" nogil:
         CStatus Open2" Open"(const shared_ptr[ReadableFileInterface]& file,
                      int64_t footer_offset, shared_ptr[CFileReader]* out)
 
-        const shared_ptr[CSchema]& schema()
+        shared_ptr[CSchema] schema()
 
         int num_dictionaries()
         int num_record_batches()

From 2c10d7ccec3c07fb061e1988be16aecaf9916af4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 12 Dec 2016 17:17:31 -0500
Subject: [PATCH 0228/1644] ARROW-418: [C++] Array / Builder class code
 reorganization, flattening

I've been wanting to do this for a while -- it feels cleaner to me. I also am going to promote modules from arrow/util to the top level as well. I'm open to other ideas, too.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #236 from wesm/ARROW-418 and squashes the following commits:

6f556ea [Wes McKinney] Add missing math.h include for clang
9dc2e22 [Wes McKinney] Fix remaining old includes
6f7ae77 [Wes McKinney] Fixes, cpplint
66ac3f7 [Wes McKinney] Promote buffer.h/status.h/memory-pool.h to top level directory
8cdf059 [Wes McKinney] Consolidate Array and Builder classes in array.h, builder.h. Remove arrow/types subdirectory
---
 cpp/CMakeLists.txt                            |  15 +-
 cpp/src/arrow/CMakeLists.txt                  |  11 +
 cpp/src/arrow/api.h                           |  13 +-
 .../decimal-test.cc => array-decimal-test.cc} |   2 +-
 .../list-test.cc => array-list-test.cc}       |   6 +-
 ...mitive-test.cc => array-primitive-test.cc} |   8 +-
 .../string-test.cc => array-string-test.cc}   |   4 +-
 .../struct-test.cc => array-struct-test.cc}   |   7 +-
 cpp/src/arrow/array-test.cc                   |   5 +-
 cpp/src/arrow/array.cc                        | 443 +++++++++++++++++-
 cpp/src/arrow/array.h                         | 373 ++++++++++++++-
 cpp/src/arrow/{util => }/buffer-test.cc       |   4 +-
 cpp/src/arrow/{util => }/buffer.cc            |   6 +-
 cpp/src/arrow/{util => }/buffer.h             |   2 +-
 cpp/src/arrow/builder.cc                      | 329 ++++++++++++-
 cpp/src/arrow/builder.h                       | 315 ++++++++++++-
 cpp/src/arrow/column-benchmark.cc             |   4 +-
 cpp/src/arrow/column-test.cc                  |   1 -
 cpp/src/arrow/column.cc                       |   2 +-
 cpp/src/arrow/io/file.cc                      |   6 +-
 cpp/src/arrow/io/hdfs.cc                      |   6 +-
 cpp/src/arrow/io/interfaces.cc                |   4 +-
 cpp/src/arrow/io/io-file-test.cc              |   2 +-
 cpp/src/arrow/io/io-hdfs-test.cc              |   2 +-
 cpp/src/arrow/io/libhdfs_shim.cc              |   2 +-
 cpp/src/arrow/io/memory.cc                    |   5 +-
 cpp/src/arrow/io/test-common.h                |   4 +-
 cpp/src/arrow/ipc/adapter.cc                  |   9 +-
 cpp/src/arrow/ipc/file.cc                     |   4 +-
 cpp/src/arrow/ipc/ipc-adapter-test.cc         |  10 +-
 cpp/src/arrow/ipc/ipc-file-test.cc            |  11 +-
 cpp/src/arrow/ipc/ipc-json-test.cc            |  18 +-
 cpp/src/arrow/ipc/ipc-metadata-test.cc        |   2 +-
 cpp/src/arrow/ipc/json-integration-test.cc    |   2 +-
 cpp/src/arrow/ipc/json-internal.cc            |  10 +-
 cpp/src/arrow/ipc/json.cc                     |   6 +-
 cpp/src/arrow/ipc/metadata-internal.cc        |   4 +-
 cpp/src/arrow/ipc/metadata.cc                 |   4 +-
 cpp/src/arrow/ipc/test-common.h               |   9 +-
 cpp/src/arrow/ipc/util.h                      |   2 +-
 ...emory-pool-test.cc => memory_pool-test.cc} |   4 +-
 .../{util/memory-pool.cc => memory_pool.cc}   |   4 +-
 .../{util/memory-pool.h => memory_pool.h}     |   0
 cpp/src/arrow/pretty_print-test.cc            |   5 +-
 cpp/src/arrow/pretty_print.cc                 |   5 +-
 cpp/src/arrow/{util => }/status-test.cc       |   2 +-
 cpp/src/arrow/{util => }/status.cc            |   2 +-
 cpp/src/arrow/{util => }/status.h             |   0
 cpp/src/arrow/table-test.cc                   |   4 +-
 cpp/src/arrow/table.cc                        |   2 +-
 cpp/src/arrow/test-util.h                     |  43 +-
 cpp/src/arrow/type.cc                         |   8 +-
 cpp/src/arrow/type.h                          |   2 +-
 cpp/src/arrow/types/CMakeLists.txt            |  39 --
 cpp/src/arrow/types/construct.cc              | 124 -----
 cpp/src/arrow/types/construct.h               |  47 --
 cpp/src/arrow/types/datetime.h                |  27 --
 cpp/src/arrow/types/decimal.cc                |  31 --
 cpp/src/arrow/types/decimal.h                 |  28 --
 cpp/src/arrow/types/list.cc                   | 162 -------
 cpp/src/arrow/types/list.h                    | 170 -------
 cpp/src/arrow/types/primitive.cc              | 294 ------------
 cpp/src/arrow/types/primitive.h               | 371 ---------------
 cpp/src/arrow/types/string.cc                 | 150 ------
 cpp/src/arrow/types/string.h                  | 149 ------
 cpp/src/arrow/types/struct.cc                 | 108 -----
 cpp/src/arrow/types/struct.h                  | 116 -----
 cpp/src/arrow/types/test-common.h             |  70 ---
 cpp/src/arrow/types/union.cc                  |  27 --
 cpp/src/arrow/types/union.h                   |  48 --
 cpp/src/arrow/util/CMakeLists.txt             |   6 -
 cpp/src/arrow/util/bit-util.cc                |   4 +-
 python/src/pyarrow/adapters/builtin.cc        |   2 +-
 python/src/pyarrow/adapters/pandas.cc         |   2 +-
 python/src/pyarrow/common.cc                  |   4 +-
 python/src/pyarrow/common.h                   |   5 +-
 python/src/pyarrow/io.cc                      |   4 +-
 77 files changed, 1607 insertions(+), 2134 deletions(-)
 rename cpp/src/arrow/{types/decimal-test.cc => array-decimal-test.cc} (97%)
 rename cpp/src/arrow/{types/list-test.cc => array-list-test.cc} (97%)
 rename cpp/src/arrow/{types/primitive-test.cc => array-primitive-test.cc} (98%)
 rename cpp/src/arrow/{types/string-test.cc => array-string-test.cc} (98%)
 rename cpp/src/arrow/{types/struct-test.cc => array-struct-test.cc} (98%)
 rename cpp/src/arrow/{util => }/buffer-test.cc (98%)
 rename cpp/src/arrow/{util => }/buffer.cc (96%)
 rename cpp/src/arrow/{util => }/buffer.h (99%)
 rename cpp/src/arrow/{util/memory-pool-test.cc => memory_pool-test.cc} (96%)
 rename cpp/src/arrow/{util/memory-pool.cc => memory_pool.cc} (97%)
 rename cpp/src/arrow/{util/memory-pool.h => memory_pool.h} (100%)
 rename cpp/src/arrow/{util => }/status-test.cc (97%)
 rename cpp/src/arrow/{util => }/status.cc (98%)
 rename cpp/src/arrow/{util => }/status.h (100%)
 delete mode 100644 cpp/src/arrow/types/CMakeLists.txt
 delete mode 100644 cpp/src/arrow/types/construct.cc
 delete mode 100644 cpp/src/arrow/types/construct.h
 delete mode 100644 cpp/src/arrow/types/datetime.h
 delete mode 100644 cpp/src/arrow/types/decimal.cc
 delete mode 100644 cpp/src/arrow/types/decimal.h
 delete mode 100644 cpp/src/arrow/types/list.cc
 delete mode 100644 cpp/src/arrow/types/list.h
 delete mode 100644 cpp/src/arrow/types/primitive.cc
 delete mode 100644 cpp/src/arrow/types/primitive.h
 delete mode 100644 cpp/src/arrow/types/string.cc
 delete mode 100644 cpp/src/arrow/types/string.h
 delete mode 100644 cpp/src/arrow/types/struct.cc
 delete mode 100644 cpp/src/arrow/types/struct.h
 delete mode 100644 cpp/src/arrow/types/test-common.h
 delete mode 100644 cpp/src/arrow/types/union.cc
 delete mode 100644 cpp/src/arrow/types/union.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 798d75fe55643..adcca0e0b49e8 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -743,25 +743,17 @@ set(ARROW_PRIVATE_LINK_LIBS
 
 set(ARROW_SRCS
   src/arrow/array.cc
+  src/arrow/buffer.cc
   src/arrow/builder.cc
   src/arrow/column.cc
+  src/arrow/memory_pool.cc
   src/arrow/pretty_print.cc
   src/arrow/schema.cc
+  src/arrow/status.cc
   src/arrow/table.cc
   src/arrow/type.cc
 
-  src/arrow/types/construct.cc
-  src/arrow/types/decimal.cc
-  src/arrow/types/list.cc
-  src/arrow/types/primitive.cc
-  src/arrow/types/string.cc
-  src/arrow/types/struct.cc
-  src/arrow/types/union.cc
-
   src/arrow/util/bit-util.cc
-  src/arrow/util/buffer.cc
-  src/arrow/util/memory-pool.cc
-  src/arrow/util/status.cc
 )
 
 add_library(arrow_objlib OBJECT
@@ -823,7 +815,6 @@ endif()
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
 add_subdirectory(src/arrow/util)
-add_subdirectory(src/arrow/types)
 
 #----------------------------------------------------------------------
 # IPC library
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 6c0dea20ba7b5..7d7bc29f4abd8 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -20,9 +20,12 @@ install(FILES
   api.h
   array.h
   column.h
+  buffer.h
   builder.h
+  memory_pool.h
   pretty_print.h
   schema.h
+  status.h
   table.h
   type.h
   type_fwd.h
@@ -37,9 +40,17 @@ install(FILES
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 ADD_ARROW_TEST(array-test)
+ADD_ARROW_TEST(array-decimal-test)
+ADD_ARROW_TEST(array-list-test)
+ADD_ARROW_TEST(array-primitive-test)
+ADD_ARROW_TEST(array-string-test)
+ADD_ARROW_TEST(array-struct-test)
+ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(column-test)
+ADD_ARROW_TEST(memory_pool-test)
 ADD_ARROW_TEST(pretty_print-test)
 ADD_ARROW_TEST(schema-test)
+ADD_ARROW_TEST(status-test)
 ADD_ARROW_TEST(table-test)
 
 ADD_ARROW_BENCHMARK(column-benchmark)
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 2d317b49cb7b6..51437d863b8b9 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -21,20 +21,13 @@
 #define ARROW_API_H
 
 #include "arrow/array.h"
+#include "arrow/buffer.h"
 #include "arrow/builder.h"
 #include "arrow/column.h"
+#include "arrow/memory_pool.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
 
-#include "arrow/types/construct.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
-
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
-
 #endif  // ARROW_API_H
diff --git a/cpp/src/arrow/types/decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
similarity index 97%
rename from cpp/src/arrow/types/decimal-test.cc
rename to cpp/src/arrow/array-decimal-test.cc
index 7296ff8176113..9e00fd9a7dd49 100644
--- a/cpp/src/arrow/types/decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -17,7 +17,7 @@
 
 #include "gtest/gtest.h"
 
-#include "arrow/types/decimal.h"
+#include "arrow/type.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/array-list-test.cc
similarity index 97%
rename from cpp/src/arrow/types/list-test.cc
rename to cpp/src/arrow/array-list-test.cc
index cb9a8c12d8ab9..8baaf06a7dbcc 100644
--- a/cpp/src/arrow/types/list-test.cc
+++ b/cpp/src/arrow/array-list-test.cc
@@ -25,13 +25,9 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/test-common.h"
-#include "arrow/util/status.h"
 
 using std::shared_ptr;
 using std::string;
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
similarity index 98%
rename from cpp/src/arrow/types/primitive-test.cc
rename to cpp/src/arrow/array-primitive-test.cc
index bdc8ec00be02c..a10e2404f29c6 100644
--- a/cpp/src/arrow/types/primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -22,16 +22,14 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/array.h"
+#include "arrow/buffer.h"
 #include "arrow/builder.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/test-common.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
 
 using std::string;
 using std::shared_ptr;
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/array-string-test.cc
similarity index 98%
rename from cpp/src/arrow/types/string-test.cc
rename to cpp/src/arrow/array-string-test.cc
index 3c4b12b7bc772..b144c632133d6 100644
--- a/cpp/src/arrow/types/string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -24,11 +24,9 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
+#include "arrow/builder.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/test-common.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/array-struct-test.cc
similarity index 98%
rename from cpp/src/arrow/types/struct-test.cc
rename to cpp/src/arrow/array-struct-test.cc
index 197d7d4ad1f5e..58386fe028fd2 100644
--- a/cpp/src/arrow/types/struct-test.cc
+++ b/cpp/src/arrow/array-struct-test.cc
@@ -23,14 +23,9 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/struct.h"
-#include "arrow/types/test-common.h"
-#include "arrow/util/status.h"
 
 using std::shared_ptr;
 using std::string;
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 158124468992a..783104e874bb7 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -24,11 +24,10 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 1f0bb66e91a3e..7ab61f59f551b 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -19,10 +19,13 @@
 
 #include <cstdint>
 #include <cstring>
+#include <sstream>
 
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+#include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 
@@ -85,4 +88,440 @@ Status NullArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+// ----------------------------------------------------------------------
+// Primitive array base
+
+PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : Array(type, length, null_count, null_bitmap) {
+  data_ = data;
+  raw_data_ = data == nullptr ? nullptr : data_->data();
+}
+
+bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
+  if (this == &other) { return true; }
+  if (null_count_ != other.null_count_) { return false; }
+
+  if (null_count_ > 0) {
+    bool equal_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
+    if (!equal_bitmap) { return false; }
+
+    const uint8_t* this_data = raw_data_;
+    const uint8_t* other_data = other.raw_data_;
+
+    auto size_meta = dynamic_cast<const FixedWidthType*>(type_.get());
+    int value_byte_size = size_meta->bit_width() / 8;
+    DCHECK_GT(value_byte_size, 0);
+
+    for (int i = 0; i < length_; ++i) {
+      if (!IsNull(i) && memcmp(this_data, other_data, value_byte_size)) { return false; }
+      this_data += value_byte_size;
+      other_data += value_byte_size;
+    }
+    return true;
+  } else {
+    if (length_ == 0 && other.length_ == 0) { return true; }
+    return data_->Equals(*other.data_, length_);
+  }
+}
+
+bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
+}
+
+template <typename T>
+Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+template class NumericArray<UInt8Type>;
+template class NumericArray<UInt16Type>;
+template class NumericArray<UInt32Type>;
+template class NumericArray<UInt64Type>;
+template class NumericArray<Int8Type>;
+template class NumericArray<Int16Type>;
+template class NumericArray<Int32Type>;
+template class NumericArray<Int64Type>;
+template class NumericArray<TimestampType>;
+template class NumericArray<HalfFloatType>;
+template class NumericArray<FloatType>;
+template class NumericArray<DoubleType>;
+
+// ----------------------------------------------------------------------
+// BooleanArray
+
+BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
+    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
+    : PrimitiveArray(
+          std::make_shared<BooleanType>(), length, data, null_count, null_bitmap) {}
+
+BooleanArray::BooleanArray(const TypePtr& type, int32_t length,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
+
+bool BooleanArray::EqualsExact(const BooleanArray& other) const {
+  if (this == &other) return true;
+  if (null_count_ != other.null_count_) { return false; }
+
+  if (null_count_ > 0) {
+    bool equal_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
+    if (!equal_bitmap) { return false; }
+
+    const uint8_t* this_data = raw_data_;
+    const uint8_t* other_data = other.raw_data_;
+
+    for (int i = 0; i < length_; ++i) {
+      if (!IsNull(i) && BitUtil::GetBit(this_data, i) != BitUtil::GetBit(other_data, i)) {
+        return false;
+      }
+    }
+    return true;
+  } else {
+    return data_->Equals(*other.data_, BitUtil::BytesForBits(length_));
+  }
+}
+
+bool BooleanArray::Equals(const ArrayPtr& arr) const {
+  if (this == arr.get()) return true;
+  if (Type::BOOL != arr->type_enum()) { return false; }
+  return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
+}
+
+bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
+    int32_t other_start_idx, const ArrayPtr& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  const auto other = static_cast<BooleanArray*>(arr.get());
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    const bool is_null = IsNull(i);
+    if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
+      return false;
+    }
+  }
+  return true;
+}
+
+Status BooleanArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+// ----------------------------------------------------------------------
+// ListArray
+
+bool ListArray::EqualsExact(const ListArray& other) const {
+  if (this == &other) { return true; }
+  if (null_count_ != other.null_count_) { return false; }
+
+  bool equal_offsets =
+      offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
+  if (!equal_offsets) { return false; }
+  bool equal_null_bitmap = true;
+  if (null_count_ > 0) {
+    equal_null_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
+  }
+
+  if (!equal_null_bitmap) { return false; }
+
+  return values()->Equals(other.values());
+}
+
+bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  return EqualsExact(*static_cast<const ListArray*>(arr.get()));
+}
+
+bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  const auto other = static_cast<ListArray*>(arr.get());
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    const bool is_null = IsNull(i);
+    if (is_null != arr->IsNull(o_i)) { return false; }
+    if (is_null) continue;
+    const int32_t begin_offset = offset(i);
+    const int32_t end_offset = offset(i + 1);
+    const int32_t other_begin_offset = other->offset(o_i);
+    const int32_t other_end_offset = other->offset(o_i + 1);
+    // Underlying can't be equal if the size isn't equal
+    if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
+      return false;
+    }
+    if (!values_->RangeEquals(
+            begin_offset, end_offset, other_begin_offset, other->values())) {
+      return false;
+    }
+  }
+  return true;
+}
+
+Status ListArray::Validate() const {
+  if (length_ < 0) { return Status::Invalid("Length was negative"); }
+  if (!offset_buffer_) { return Status::Invalid("offset_buffer_ was null"); }
+  if (offset_buffer_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
+    std::stringstream ss;
+    ss << "offset buffer size (bytes): " << offset_buffer_->size()
+       << " isn't large enough for length: " << length_;
+    return Status::Invalid(ss.str());
+  }
+  const int32_t last_offset = offset(length_);
+  if (last_offset > 0) {
+    if (!values_) {
+      return Status::Invalid("last offset was non-zero and values was null");
+    }
+    if (values_->length() != last_offset) {
+      std::stringstream ss;
+      ss << "Final offset invariant not equal to values length: " << last_offset
+         << "!=" << values_->length();
+      return Status::Invalid(ss.str());
+    }
+
+    const Status child_valid = values_->Validate();
+    if (!child_valid.ok()) {
+      std::stringstream ss;
+      ss << "Child array invalid: " << child_valid.ToString();
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  int32_t prev_offset = offset(0);
+  if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
+  for (int32_t i = 1; i <= length_; ++i) {
+    int32_t current_offset = offset(i);
+    if (IsNull(i - 1) && current_offset != prev_offset) {
+      std::stringstream ss;
+      ss << "Offset invariant failure at: " << i << " inconsistent offsets for null slot"
+         << current_offset << "!=" << prev_offset;
+      return Status::Invalid(ss.str());
+    }
+    if (current_offset < prev_offset) {
+      std::stringstream ss;
+      ss << "Offset invariant failure: " << i
+         << " inconsistent offset for non-null slot: " << current_offset << "<"
+         << prev_offset;
+      return Status::Invalid(ss.str());
+    }
+    prev_offset = current_offset;
+  }
+  return Status::OK();
+}
+
+Status ListArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+// ----------------------------------------------------------------------
+// String and binary
+
+static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
+static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
+
+BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : BinaryArray(kBinary, length, offsets, data, null_count, null_bitmap) {}
+
+BinaryArray::BinaryArray(const TypePtr& type, int32_t length,
+    const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
+    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
+    : Array(type, length, null_count, null_bitmap),
+      offset_buffer_(offsets),
+      offsets_(reinterpret_cast<const int32_t*>(offset_buffer_->data())),
+      data_buffer_(data),
+      data_(nullptr) {
+  if (data_buffer_ != nullptr) { data_ = data_buffer_->data(); }
+}
+
+Status BinaryArray::Validate() const {
+  // TODO(wesm): what to do here?
+  return Status::OK();
+}
+
+bool BinaryArray::EqualsExact(const BinaryArray& other) const {
+  if (!Array::EqualsExact(other)) { return false; }
+
+  bool equal_offsets =
+      offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
+  if (!equal_offsets) { return false; }
+
+  if (!data_buffer_ && !(other.data_buffer_)) { return true; }
+
+  return data_buffer_->Equals(*other.data_buffer_, data_buffer_->size());
+}
+
+bool BinaryArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  return EqualsExact(*static_cast<const BinaryArray*>(arr.get()));
+}
+
+bool BinaryArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  const auto other = static_cast<const BinaryArray*>(arr.get());
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    const bool is_null = IsNull(i);
+    if (is_null != arr->IsNull(o_i)) { return false; }
+    if (is_null) continue;
+    const int32_t begin_offset = offset(i);
+    const int32_t end_offset = offset(i + 1);
+    const int32_t other_begin_offset = other->offset(o_i);
+    const int32_t other_end_offset = other->offset(o_i + 1);
+    // Underlying can't be equal if the size isn't equal
+    if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
+      return false;
+    }
+
+    if (std::memcmp(data_ + begin_offset, other->data_ + other_begin_offset,
+            end_offset - begin_offset)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+Status BinaryArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap)
+    : BinaryArray(kString, length, offsets, data, null_count, null_bitmap) {}
+
+Status StringArray::Validate() const {
+  // TODO(emkornfield) Validate proper UTF8 code points?
+  return BinaryArray::Validate();
+}
+
+Status StringArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+// ----------------------------------------------------------------------
+// Struct
+
+std::shared_ptr<Array> StructArray::field(int32_t pos) const {
+  DCHECK_GT(field_arrays_.size(), 0);
+  return field_arrays_[pos];
+}
+
+bool StructArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+  if (null_count_ != arr->null_count()) { return false; }
+  return RangeEquals(0, length_, 0, arr);
+}
+
+bool StructArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (Type::STRUCT != arr->type_enum()) { return false; }
+  const auto other = static_cast<StructArray*>(arr.get());
+
+  bool equal_fields = true;
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    if (IsNull(i) != arr->IsNull(o_i)) { return false; }
+    if (IsNull(i)) continue;
+    for (size_t j = 0; j < field_arrays_.size(); ++j) {
+      // TODO: really we should be comparing stretches of non-null data rather
+      // than looking at one value at a time.
+      equal_fields = field(j)->RangeEquals(i, i + 1, o_i, other->field(j));
+      if (!equal_fields) { return false; }
+    }
+  }
+
+  return true;
+}
+
+Status StructArray::Validate() const {
+  if (length_ < 0) { return Status::Invalid("Length was negative"); }
+
+  if (null_count() > length_) {
+    return Status::Invalid("Null count exceeds the length of this struct");
+  }
+
+  if (field_arrays_.size() > 0) {
+    // Validate fields
+    int32_t array_length = field_arrays_[0]->length();
+    size_t idx = 0;
+    for (auto it : field_arrays_) {
+      if (it->length() != array_length) {
+        std::stringstream ss;
+        ss << "Length is not equal from field " << it->type()->ToString()
+           << " at position {" << idx << "}";
+        return Status::Invalid(ss.str());
+      }
+
+      const Status child_valid = it->Validate();
+      if (!child_valid.ok()) {
+        std::stringstream ss;
+        ss << "Child array invalid: " << child_valid.ToString() << " at position {" << idx
+           << "}";
+        return Status::Invalid(ss.str());
+      }
+      ++idx;
+    }
+
+    if (array_length > 0 && array_length != length_) {
+      return Status::Invalid("Struct's length is not equal to its child arrays");
+    }
+  }
+  return Status::OK();
+}
+
+Status StructArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+// ----------------------------------------------------------------------
+
+#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                          \
+  case Type::ENUM:                                                          \
+    out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
+    break;
+
+Status MakePrimitiveArray(const TypePtr& type, int32_t length,
+    const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
+  switch (type->type) {
+    MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
+    MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(INT8, Int8Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(UINT16, UInt16Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(INT16, Int16Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(UINT32, UInt32Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(INT32, Int32Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
+    MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
+    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP_DOUBLE, DoubleArray);
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+#ifdef NDEBUG
+  return Status::OK();
+#else
+  return (*out)->Validate();
+#endif
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 91fb93e625494..1a4a9237a1f79 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -18,9 +18,13 @@
 #ifndef ARROW_ARRAY_H
 #define ARROW_ARRAY_H
 
+#include <cmath>
 #include <cstdint>
 #include <memory>
+#include <string>
+#include <vector>
 
+#include "arrow/buffer.h"
 #include "arrow/type.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
@@ -28,7 +32,6 @@
 
 namespace arrow {
 
-class Buffer;
 class MemoryPool;
 class MutableBuffer;
 class Status;
@@ -110,6 +113,374 @@ typedef std::shared_ptr<Array> ArrayPtr;
 Status ARROW_EXPORT GetEmptyBitmap(
     MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result);
 
+// Base class for fixed-size logical types
+class ARROW_EXPORT PrimitiveArray : public Array {
+ public:
+  virtual ~PrimitiveArray() {}
+
+  std::shared_ptr<Buffer> data() const { return data_; }
+
+  bool EqualsExact(const PrimitiveArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+
+ protected:
+  PrimitiveArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  std::shared_ptr<Buffer> data_;
+  const uint8_t* raw_data_;
+};
+
+template <typename TYPE>
+class ARROW_EXPORT NumericArray : public PrimitiveArray {
+ public:
+  using TypeClass = TYPE;
+  using value_type = typename TypeClass::c_type;
+  NumericArray(int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+      : PrimitiveArray(
+            std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {}
+  NumericArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+      : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
+
+  bool EqualsExact(const NumericArray<TypeClass>& other) const {
+    return PrimitiveArray::EqualsExact(static_cast<const PrimitiveArray&>(other));
+  }
+
+  bool ApproxEquals(const std::shared_ptr<Array>& arr) const { return Equals(arr); }
+
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const ArrayPtr& arr) const override {
+    if (this == arr.get()) { return true; }
+    if (!arr) { return false; }
+    if (this->type_enum() != arr->type_enum()) { return false; }
+    const auto other = static_cast<NumericArray<TypeClass>*>(arr.get());
+    for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+      const bool is_null = IsNull(i);
+      if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
+        return false;
+      }
+    }
+    return true;
+  }
+  const value_type* raw_data() const {
+    return reinterpret_cast<const value_type*>(raw_data_);
+  }
+
+  Status Accept(ArrayVisitor* visitor) const override;
+
+  value_type Value(int i) const { return raw_data()[i]; }
+};
+
+template <>
+inline bool NumericArray<FloatType>::ApproxEquals(
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+
+  const auto& other = *static_cast<NumericArray<FloatType>*>(arr.get());
+
+  if (this == &other) { return true; }
+  if (null_count_ != other.null_count_) { return false; }
+
+  auto this_data = reinterpret_cast<const float*>(raw_data_);
+  auto other_data = reinterpret_cast<const float*>(other.raw_data_);
+
+  static constexpr float EPSILON = 1E-5;
+
+  if (length_ == 0 && other.length_ == 0) { return true; }
+
+  if (null_count_ > 0) {
+    bool equal_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
+    if (!equal_bitmap) { return false; }
+
+    for (int i = 0; i < length_; ++i) {
+      if (IsNull(i)) continue;
+      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+    }
+  } else {
+    for (int i = 0; i < length_; ++i) {
+      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+    }
+  }
+  return true;
+}
+
+template <>
+inline bool NumericArray<DoubleType>::ApproxEquals(
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (this->type_enum() != arr->type_enum()) { return false; }
+
+  const auto& other = *static_cast<NumericArray<DoubleType>*>(arr.get());
+
+  if (this == &other) { return true; }
+  if (null_count_ != other.null_count_) { return false; }
+
+  auto this_data = reinterpret_cast<const double*>(raw_data_);
+  auto other_data = reinterpret_cast<const double*>(other.raw_data_);
+
+  if (length_ == 0 && other.length_ == 0) { return true; }
+
+  static constexpr double EPSILON = 1E-5;
+
+  if (null_count_ > 0) {
+    bool equal_bitmap =
+        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
+    if (!equal_bitmap) { return false; }
+
+    for (int i = 0; i < length_; ++i) {
+      if (IsNull(i)) continue;
+      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+    }
+  } else {
+    for (int i = 0; i < length_; ++i) {
+      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+    }
+  }
+  return true;
+}
+
+class ARROW_EXPORT BooleanArray : public PrimitiveArray {
+ public:
+  using TypeClass = BooleanType;
+
+  BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  BooleanArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+  bool EqualsExact(const BooleanArray& other) const;
+  bool Equals(const ArrayPtr& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const ArrayPtr& arr) const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
+
+  const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
+
+  bool Value(int i) const { return BitUtil::GetBit(raw_data(), i); }
+};
+
+// ----------------------------------------------------------------------
+// ListArray
+
+class ARROW_EXPORT ListArray : public Array {
+ public:
+  using TypeClass = ListType;
+
+  ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
+      const ArrayPtr& values, int32_t null_count = 0,
+      std::shared_ptr<Buffer> null_bitmap = nullptr)
+      : Array(type, length, null_count, null_bitmap) {
+    offset_buffer_ = offsets;
+    offsets_ = offsets == nullptr ? nullptr : reinterpret_cast<const int32_t*>(
+                                                  offset_buffer_->data());
+    values_ = values;
+  }
+
+  Status Validate() const override;
+
+  virtual ~ListArray() = default;
+
+  // Return a shared pointer in case the requestor desires to share ownership
+  // with this array.
+  std::shared_ptr<Array> values() const { return values_; }
+  std::shared_ptr<Buffer> offsets() const {
+    return std::static_pointer_cast<Buffer>(offset_buffer_);
+  }
+
+  std::shared_ptr<DataType> value_type() const { return values_->type(); }
+
+  const int32_t* raw_offsets() const { return offsets_; }
+
+  int32_t offset(int i) const { return offsets_[i]; }
+
+  // Neither of these functions will perform boundschecking
+  int32_t value_offset(int i) const { return offsets_[i]; }
+  int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
+
+  bool EqualsExact(const ListArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const ArrayPtr& arr) const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
+
+ protected:
+  std::shared_ptr<Buffer> offset_buffer_;
+  const int32_t* offsets_;
+  ArrayPtr values_;
+};
+
+// ----------------------------------------------------------------------
+// Binary and String
+
+class ARROW_EXPORT BinaryArray : public Array {
+ public:
+  using TypeClass = BinaryType;
+
+  BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+  // Constructor that allows sub-classes/builders to propagate there logical type up the
+  // class hierarchy.
+  BinaryArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+  // Return the pointer to the given elements bytes
+  // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
+  // pointer + offset
+  const uint8_t* GetValue(int i, int32_t* out_length) const {
+    const int32_t pos = offsets_[i];
+    *out_length = offsets_[i + 1] - pos;
+    return data_ + pos;
+  }
+
+  std::shared_ptr<Buffer> data() const { return data_buffer_; }
+  std::shared_ptr<Buffer> offsets() const { return offset_buffer_; }
+
+  const int32_t* raw_offsets() const { return offsets_; }
+
+  int32_t offset(int i) const { return offsets_[i]; }
+
+  // Neither of these functions will perform boundschecking
+  int32_t value_offset(int i) const { return offsets_[i]; }
+  int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
+
+  bool EqualsExact(const BinaryArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const ArrayPtr& arr) const override;
+
+  Status Validate() const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
+
+ private:
+  std::shared_ptr<Buffer> offset_buffer_;
+  const int32_t* offsets_;
+
+  std::shared_ptr<Buffer> data_buffer_;
+  const uint8_t* data_;
+};
+
+class ARROW_EXPORT StringArray : public BinaryArray {
+ public:
+  using TypeClass = StringType;
+
+  StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+  // Construct a std::string
+  // TODO: std::bad_alloc possibility
+  std::string GetString(int i) const {
+    int32_t nchars;
+    const uint8_t* str = GetValue(i, &nchars);
+    return std::string(reinterpret_cast<const char*>(str), nchars);
+  }
+
+  Status Validate() const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
+};
+
+// ----------------------------------------------------------------------
+// Struct
+
+class ARROW_EXPORT StructArray : public Array {
+ public:
+  using TypeClass = StructType;
+
+  StructArray(const TypePtr& type, int32_t length, std::vector<ArrayPtr>& field_arrays,
+      int32_t null_count = 0, std::shared_ptr<Buffer> null_bitmap = nullptr)
+      : Array(type, length, null_count, null_bitmap) {
+    type_ = type;
+    field_arrays_ = field_arrays;
+  }
+
+  Status Validate() const override;
+
+  virtual ~StructArray() {}
+
+  // Return a shared pointer in case the requestor desires to share ownership
+  // with this array.
+  std::shared_ptr<Array> field(int32_t pos) const;
+
+  const std::vector<ArrayPtr>& fields() const { return field_arrays_; }
+
+  bool EqualsExact(const StructArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const std::shared_ptr<Array>& arr) const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
+
+ protected:
+  // The child arrays corresponding to each field of the struct data type.
+  std::vector<ArrayPtr> field_arrays_;
+};
+
+// ----------------------------------------------------------------------
+// Union
+
+class UnionArray : public Array {
+ protected:
+  // The data are types encoded as int16
+  Buffer* types_;
+  std::vector<std::shared_ptr<Array>> children_;
+};
+
+class DenseUnionArray : public UnionArray {
+ protected:
+  Buffer* offset_buf_;
+};
+
+class SparseUnionArray : public UnionArray {};
+
+// ----------------------------------------------------------------------
+// extern templates and other details
+
+// gcc and clang disagree about how to handle template visibility when you have
+// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif
+
+// Only instantiate these templates once
+extern template class ARROW_EXPORT NumericArray<Int8Type>;
+extern template class ARROW_EXPORT NumericArray<UInt8Type>;
+extern template class ARROW_EXPORT NumericArray<Int16Type>;
+extern template class ARROW_EXPORT NumericArray<UInt16Type>;
+extern template class ARROW_EXPORT NumericArray<Int32Type>;
+extern template class ARROW_EXPORT NumericArray<UInt32Type>;
+extern template class ARROW_EXPORT NumericArray<Int64Type>;
+extern template class ARROW_EXPORT NumericArray<UInt64Type>;
+extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
+extern template class ARROW_EXPORT NumericArray<FloatType>;
+extern template class ARROW_EXPORT NumericArray<DoubleType>;
+extern template class ARROW_EXPORT NumericArray<TimestampType>;
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+// ----------------------------------------------------------------------
+// Helper functions
+
+// Create new arrays for logical types that are backed by primitive arrays.
+Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
+
 }  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/util/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
similarity index 98%
rename from cpp/src/arrow/util/buffer-test.cc
rename to cpp/src/arrow/buffer-test.cc
index 095b07b7ab309..c1d027bb653fe 100644
--- a/cpp/src/arrow/util/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -21,9 +21,9 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/buffer.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
 
 using std::string;
 
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/buffer.cc
similarity index 96%
rename from cpp/src/arrow/util/buffer.cc
rename to cpp/src/arrow/buffer.cc
index a230259e5930d..6ffa03a0b5663 100644
--- a/cpp/src/arrow/util/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -15,15 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/buffer.h"
+#include "arrow/buffer.h"
 
 #include <cstdint>
 #include <limits>
 
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/buffer.h
similarity index 99%
rename from cpp/src/arrow/util/buffer.h
rename to cpp/src/arrow/buffer.h
index 5c87395deebb0..27437ca0486c3 100644
--- a/cpp/src/arrow/util/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -23,8 +23,8 @@
 #include <cstring>
 #include <memory>
 
+#include "arrow/status.h"
 #include "arrow/util/macros.h"
-#include "arrow/util/status.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 151b257a3d894..493b5e7ccab9e 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -17,11 +17,17 @@
 
 #include "arrow/builder.h"
 
+#include <cstdint>
 #include <cstring>
+#include <limits>
 
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 
@@ -123,4 +129,323 @@ void ArrayBuilder::UnsafeSetNotNull(int32_t length) {
   length_ = new_length;
 }
 
+template <typename T>
+Status PrimitiveBuilder<T>::Init(int32_t capacity) {
+  RETURN_NOT_OK(ArrayBuilder::Init(capacity));
+  data_ = std::make_shared<PoolBuffer>(pool_);
+
+  int64_t nbytes = TypeTraits<T>::bytes_required(capacity);
+  RETURN_NOT_OK(data_->Resize(nbytes));
+  // TODO(emkornfield) valgrind complains without this
+  memset(data_->mutable_data(), 0, nbytes);
+
+  raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
+  return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
+  // XXX: Set floor size for now
+  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+
+  if (capacity_ == 0) {
+    RETURN_NOT_OK(Init(capacity));
+  } else {
+    RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
+    const int64_t old_bytes = data_->size();
+    const int64_t new_bytes = TypeTraits<T>::bytes_required(capacity);
+    RETURN_NOT_OK(data_->Resize(new_bytes));
+    raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
+    memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
+  }
+  return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Append(
+    const value_type* values, int32_t length, const uint8_t* valid_bytes) {
+  RETURN_NOT_OK(Reserve(length));
+
+  if (length > 0) {
+    memcpy(raw_data_ + length_, values, TypeTraits<T>::bytes_required(length));
+  }
+
+  // length_ is update by these
+  ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
+
+  return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
+  const int64_t bytes_required = TypeTraits<T>::bytes_required(length_);
+  if (bytes_required > 0 && bytes_required < data_->size()) {
+    // Trim buffers
+    RETURN_NOT_OK(data_->Resize(bytes_required));
+  }
+  *out = std::make_shared<typename TypeTraits<T>::ArrayType>(
+      type_, length_, data_, null_count_, null_bitmap_);
+
+  data_ = null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
+  return Status::OK();
+}
+
+template class PrimitiveBuilder<UInt8Type>;
+template class PrimitiveBuilder<UInt16Type>;
+template class PrimitiveBuilder<UInt32Type>;
+template class PrimitiveBuilder<UInt64Type>;
+template class PrimitiveBuilder<Int8Type>;
+template class PrimitiveBuilder<Int16Type>;
+template class PrimitiveBuilder<Int32Type>;
+template class PrimitiveBuilder<Int64Type>;
+template class PrimitiveBuilder<TimestampType>;
+template class PrimitiveBuilder<HalfFloatType>;
+template class PrimitiveBuilder<FloatType>;
+template class PrimitiveBuilder<DoubleType>;
+
+Status BooleanBuilder::Init(int32_t capacity) {
+  RETURN_NOT_OK(ArrayBuilder::Init(capacity));
+  data_ = std::make_shared<PoolBuffer>(pool_);
+
+  int64_t nbytes = BitUtil::BytesForBits(capacity);
+  RETURN_NOT_OK(data_->Resize(nbytes));
+  // TODO(emkornfield) valgrind complains without this
+  memset(data_->mutable_data(), 0, nbytes);
+
+  raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
+  return Status::OK();
+}
+
+Status BooleanBuilder::Resize(int32_t capacity) {
+  // XXX: Set floor size for now
+  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+
+  if (capacity_ == 0) {
+    RETURN_NOT_OK(Init(capacity));
+  } else {
+    RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
+    const int64_t old_bytes = data_->size();
+    const int64_t new_bytes = BitUtil::BytesForBits(capacity);
+
+    RETURN_NOT_OK(data_->Resize(new_bytes));
+    raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
+    memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
+  }
+  return Status::OK();
+}
+
+Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
+  const int64_t bytes_required = BitUtil::BytesForBits(length_);
+
+  if (bytes_required > 0 && bytes_required < data_->size()) {
+    // Trim buffers
+    RETURN_NOT_OK(data_->Resize(bytes_required));
+  }
+  *out = std::make_shared<BooleanArray>(type_, length_, data_, null_count_, null_bitmap_);
+
+  data_ = null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
+  return Status::OK();
+}
+
+Status BooleanBuilder::Append(
+    const uint8_t* values, int32_t length, const uint8_t* valid_bytes) {
+  RETURN_NOT_OK(Reserve(length));
+
+  for (int i = 0; i < length; ++i) {
+    // Skip reading from unitialised memory
+    // TODO: This actually is only to keep valgrind happy but may or may not
+    // have a performance impact.
+    if ((valid_bytes != nullptr) && !valid_bytes[i]) continue;
+
+    if (values[i] > 0) {
+      BitUtil::SetBit(raw_data_, length_ + i);
+    } else {
+      BitUtil::ClearBit(raw_data_, length_ + i);
+    }
+  }
+
+  // this updates length_
+  ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// ListBuilder
+
+ListBuilder::ListBuilder(
+    MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder, const TypePtr& type)
+    : ArrayBuilder(
+          pool, type ? type : std::static_pointer_cast<DataType>(
+                                  std::make_shared<ListType>(value_builder->type()))),
+      offset_builder_(pool),
+      value_builder_(value_builder) {}
+
+ListBuilder::ListBuilder(
+    MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type)
+    : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
+                                           std::make_shared<ListType>(values->type()))),
+      offset_builder_(pool),
+      values_(values) {}
+
+Status ListBuilder::Init(int32_t elements) {
+  DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+  // one more then requested for offsets
+  return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
+}
+
+Status ListBuilder::Resize(int32_t capacity) {
+  DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
+  // one more then requested for offsets
+  RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
+  return ArrayBuilder::Resize(capacity);
+}
+
+Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Array> items = values_;
+  if (!items) { RETURN_NOT_OK(value_builder_->Finish(&items)); }
+
+  RETURN_NOT_OK(offset_builder_.Append<int32_t>(items->length()));
+  std::shared_ptr<Buffer> offsets = offset_builder_.Finish();
+
+  *out = std::make_shared<ListArray>(
+      type_, length_, offsets, items, null_count_, null_bitmap_);
+
+  Reset();
+
+  return Status::OK();
+}
+
+void ListBuilder::Reset() {
+  capacity_ = length_ = null_count_ = 0;
+  null_bitmap_ = nullptr;
+}
+
+std::shared_ptr<ArrayBuilder> ListBuilder::value_builder() const {
+  DCHECK(!values_) << "Using value builder is pointless when values_ is set";
+  return value_builder_;
+}
+
+// ----------------------------------------------------------------------
+// String and binary
+
+// This used to be a static member variable of BinaryBuilder, but it can cause
+// valgrind to report a (spurious?) memory leak when needed in other shared
+// libraries. The problem came up while adding explicit visibility to libarrow
+// and libparquet_arrow
+static TypePtr kBinaryValueType = TypePtr(new UInt8Type());
+
+BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
+    : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, kBinaryValueType), type) {
+  byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
+}
+
+Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Array> result;
+  RETURN_NOT_OK(ListBuilder::Finish(&result));
+
+  const auto list = std::dynamic_pointer_cast<ListArray>(result);
+  auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
+
+  *out = std::make_shared<BinaryArray>(list->length(), list->offsets(), values->data(),
+      list->null_count(), list->null_bitmap());
+  return Status::OK();
+}
+
+Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Array> result;
+  RETURN_NOT_OK(ListBuilder::Finish(&result));
+
+  const auto list = std::dynamic_pointer_cast<ListArray>(result);
+  auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
+
+  *out = std::make_shared<StringArray>(list->length(), list->offsets(), values->data(),
+      list->null_count(), list->null_bitmap());
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Struct
+
+Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
+  for (size_t i = 0; i < field_builders_.size(); ++i) {
+    RETURN_NOT_OK(field_builders_[i]->Finish(&fields[i]));
+  }
+
+  *out = std::make_shared<StructArray>(type_, length_, fields, null_count_, null_bitmap_);
+
+  null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
+
+  return Status::OK();
+}
+
+std::shared_ptr<ArrayBuilder> StructBuilder::field_builder(int pos) const {
+  DCHECK_GT(field_builders_.size(), 0);
+  return field_builders_[pos];
+}
+
+// ----------------------------------------------------------------------
+// Helper functions
+
+#define BUILDER_CASE(ENUM, BuilderType)      \
+  case Type::ENUM:                           \
+    out->reset(new BuilderType(pool, type)); \
+    return Status::OK();
+
+// Initially looked at doing this with vtables, but shared pointers makes it
+// difficult
+//
+// TODO(wesm): come up with a less monolithic strategy
+Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+    std::shared_ptr<ArrayBuilder>* out) {
+  switch (type->type) {
+    BUILDER_CASE(UINT8, UInt8Builder);
+    BUILDER_CASE(INT8, Int8Builder);
+    BUILDER_CASE(UINT16, UInt16Builder);
+    BUILDER_CASE(INT16, Int16Builder);
+    BUILDER_CASE(UINT32, UInt32Builder);
+    BUILDER_CASE(INT32, Int32Builder);
+    BUILDER_CASE(UINT64, UInt64Builder);
+    BUILDER_CASE(INT64, Int64Builder);
+    BUILDER_CASE(TIMESTAMP, TimestampBuilder);
+
+    BUILDER_CASE(BOOL, BooleanBuilder);
+
+    BUILDER_CASE(FLOAT, FloatBuilder);
+    BUILDER_CASE(DOUBLE, DoubleBuilder);
+
+    BUILDER_CASE(STRING, StringBuilder);
+    BUILDER_CASE(BINARY, BinaryBuilder);
+
+    case Type::LIST: {
+      std::shared_ptr<ArrayBuilder> value_builder;
+      std::shared_ptr<DataType> value_type =
+          static_cast<ListType*>(type.get())->value_type();
+      RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
+      out->reset(new ListBuilder(pool, value_builder));
+      return Status::OK();
+    }
+
+    case Type::STRUCT: {
+      std::vector<FieldPtr>& fields = type->children_;
+      std::vector<std::shared_ptr<ArrayBuilder>> values_builder;
+
+      for (auto it : fields) {
+        std::shared_ptr<ArrayBuilder> builder;
+        RETURN_NOT_OK(MakeBuilder(pool, it->type, &builder));
+        values_builder.push_back(builder);
+      }
+      out->reset(new StructBuilder(pool, type, values_builder));
+      return Status::OK();
+    }
+
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 73e49c0a69674..7162d31d2464a 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -20,18 +20,20 @@
 
 #include <cstdint>
 #include <memory>
+#include <string>
 #include <vector>
 
+#include "arrow/buffer.h"
+#include "arrow/status.h"
 #include "arrow/type.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
-#include "arrow/util/status.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class Array;
 class MemoryPool;
-class PoolBuffer;
 
 static constexpr int32_t kMinBuilderCapacity = 1 << 5;
 
@@ -130,6 +132,315 @@ class ARROW_EXPORT ArrayBuilder {
   DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
 };
 
+template <typename Type>
+class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
+ public:
+  using value_type = typename Type::c_type;
+
+  explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type)
+      : ArrayBuilder(pool, type), data_(nullptr) {}
+
+  virtual ~PrimitiveBuilder() {}
+
+  using ArrayBuilder::Advance;
+
+  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+  Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+    RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(valid_bytes, length);
+    return Status::OK();
+  }
+
+  Status AppendNull() {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(false);
+    return Status::OK();
+  }
+
+  std::shared_ptr<Buffer> data() const { return data_; }
+
+  // Vector append
+  //
+  // If passed, valid_bytes is of equal length to values, and any zero byte
+  // will be considered as a null for that slot
+  Status Append(
+      const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
+
+  Status Finish(std::shared_ptr<Array>* out) override;
+  Status Init(int32_t capacity) override;
+
+  // Increase the capacity of the builder to accommodate at least the indicated
+  // number of elements
+  Status Resize(int32_t capacity) override;
+
+ protected:
+  std::shared_ptr<PoolBuffer> data_;
+  value_type* raw_data_;
+};
+
+template <typename T>
+class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
+ public:
+  using typename PrimitiveBuilder<T>::value_type;
+  using PrimitiveBuilder<T>::PrimitiveBuilder;
+
+  using PrimitiveBuilder<T>::Append;
+  using PrimitiveBuilder<T>::Init;
+  using PrimitiveBuilder<T>::Resize;
+  using PrimitiveBuilder<T>::Reserve;
+
+  // Scalar append.
+  Status Append(value_type val) {
+    RETURN_NOT_OK(ArrayBuilder::Reserve(1));
+    UnsafeAppend(val);
+    return Status::OK();
+  }
+
+  // Does not capacity-check; make sure to call Reserve beforehand
+  void UnsafeAppend(value_type val) {
+    BitUtil::SetBit(null_bitmap_data_, length_);
+    raw_data_[length_++] = val;
+  }
+
+ protected:
+  using PrimitiveBuilder<T>::length_;
+  using PrimitiveBuilder<T>::null_bitmap_data_;
+  using PrimitiveBuilder<T>::raw_data_;
+};
+
+// Builders
+
+using UInt8Builder = NumericBuilder<UInt8Type>;
+using UInt16Builder = NumericBuilder<UInt16Type>;
+using UInt32Builder = NumericBuilder<UInt32Type>;
+using UInt64Builder = NumericBuilder<UInt64Type>;
+
+using Int8Builder = NumericBuilder<Int8Type>;
+using Int16Builder = NumericBuilder<Int16Type>;
+using Int32Builder = NumericBuilder<Int32Type>;
+using Int64Builder = NumericBuilder<Int64Type>;
+using TimestampBuilder = NumericBuilder<TimestampType>;
+
+using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
+using FloatBuilder = NumericBuilder<FloatType>;
+using DoubleBuilder = NumericBuilder<DoubleType>;
+
+class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
+ public:
+  explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type)
+      : ArrayBuilder(pool, type), data_(nullptr) {}
+
+  virtual ~BooleanBuilder() {}
+
+  using ArrayBuilder::Advance;
+
+  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+  Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+    RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(valid_bytes, length);
+    return Status::OK();
+  }
+
+  Status AppendNull() {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(false);
+    return Status::OK();
+  }
+
+  std::shared_ptr<Buffer> data() const { return data_; }
+
+  // Scalar append
+  Status Append(bool val) {
+    Reserve(1);
+    BitUtil::SetBit(null_bitmap_data_, length_);
+    if (val) {
+      BitUtil::SetBit(raw_data_, length_);
+    } else {
+      BitUtil::ClearBit(raw_data_, length_);
+    }
+    ++length_;
+    return Status::OK();
+  }
+
+  // Vector append
+  //
+  // If passed, valid_bytes is of equal length to values, and any zero byte
+  // will be considered as a null for that slot
+  Status Append(
+      const uint8_t* values, int32_t length, const uint8_t* valid_bytes = nullptr);
+
+  Status Finish(std::shared_ptr<Array>* out) override;
+  Status Init(int32_t capacity) override;
+
+  // Increase the capacity of the builder to accommodate at least the indicated
+  // number of elements
+  Status Resize(int32_t capacity) override;
+
+ protected:
+  std::shared_ptr<PoolBuffer> data_;
+  uint8_t* raw_data_;
+};
+
+// ----------------------------------------------------------------------
+// List builder
+
+// Builder class for variable-length list array value types
+//
+// To use this class, you must append values to the child array builder and use
+// the Append function to delimit each distinct list value (once the values
+// have been appended to the child array) or use the bulk API to append
+// a sequence of offests and null values.
+//
+// A note on types.  Per arrow/type.h all types in the c++ implementation are
+// logical so even though this class always builds list array, this can
+// represent multiple different logical types.  If no logical type is provided
+// at construction time, the class defaults to List<T> where t is taken from the
+// value_builder/values that the object is constructed with.
+class ARROW_EXPORT ListBuilder : public ArrayBuilder {
+ public:
+  // Use this constructor to incrementally build the value array along with offsets and
+  // null bitmap.
+  ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
+      const TypePtr& type = nullptr);
+
+  // Use this constructor to build the list with a pre-existing values array
+  ListBuilder(
+      MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr);
+
+  virtual ~ListBuilder() {}
+
+  Status Init(int32_t elements) override;
+  Status Resize(int32_t capacity) override;
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+  // Vector append
+  //
+  // If passed, valid_bytes is of equal length to values, and any zero byte
+  // will be considered as a null for that slot
+  Status Append(
+      const int32_t* offsets, int32_t length, const uint8_t* valid_bytes = nullptr) {
+    RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(valid_bytes, length);
+    offset_builder_.UnsafeAppend<int32_t>(offsets, length);
+    return Status::OK();
+  }
+
+  // Start a new variable-length list slot
+  //
+  // This function should be called before beginning to append elements to the
+  // value builder
+  Status Append(bool is_valid = true) {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(is_valid);
+    RETURN_NOT_OK(offset_builder_.Append<int32_t>(value_builder_->length()));
+    return Status::OK();
+  }
+
+  Status AppendNull() { return Append(false); }
+
+  std::shared_ptr<ArrayBuilder> value_builder() const;
+
+ protected:
+  BufferBuilder offset_builder_;
+  std::shared_ptr<ArrayBuilder> value_builder_;
+  std::shared_ptr<Array> values_;
+
+  void Reset();
+};
+
+// ----------------------------------------------------------------------
+// Binary and String
+
+// BinaryBuilder : public ListBuilder
+class ARROW_EXPORT BinaryBuilder : public ListBuilder {
+ public:
+  explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type);
+  virtual ~BinaryBuilder() {}
+
+  Status Append(const uint8_t* value, int32_t length) {
+    RETURN_NOT_OK(ListBuilder::Append());
+    return byte_builder_->Append(value, length);
+  }
+
+  Status Append(const char* value, int32_t length) {
+    return Append(reinterpret_cast<const uint8_t*>(value), length);
+  }
+
+  Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
+
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+ protected:
+  UInt8Builder* byte_builder_;
+};
+
+// String builder
+class ARROW_EXPORT StringBuilder : public BinaryBuilder {
+ public:
+  explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
+      : BinaryBuilder(pool, type) {}
+
+  using BinaryBuilder::Append;
+
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+  Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
+};
+
+// ----------------------------------------------------------------------
+// Struct
+
+// ---------------------------------------------------------------------------------
+// StructArray builder
+// Append, Resize and Reserve methods are acting on StructBuilder.
+// Please make sure all these methods of all child-builders' are consistently
+// called to maintain data-structure consistency.
+class ARROW_EXPORT StructBuilder : public ArrayBuilder {
+ public:
+  StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+      const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders)
+      : ArrayBuilder(pool, type) {
+    field_builders_ = field_builders;
+  }
+
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+  // Null bitmap is of equal length to every child field, and any zero byte
+  // will be considered as a null for that field, but users must using app-
+  // end methods or advance methods of the child builders' independently to
+  // insert data.
+  Status Append(int32_t length, const uint8_t* valid_bytes) {
+    RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(valid_bytes, length);
+    return Status::OK();
+  }
+
+  // Append an element to the Struct. All child-builders' Append method must
+  // be called independently to maintain data-structure consistency.
+  Status Append(bool is_valid = true) {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(is_valid);
+    return Status::OK();
+  }
+
+  Status AppendNull() { return Append(false); }
+
+  std::shared_ptr<ArrayBuilder> field_builder(int pos) const;
+
+  const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders() const {
+    return field_builders_;
+  }
+
+ protected:
+  std::vector<std::shared_ptr<ArrayBuilder>> field_builders_;
+};
+
+// ----------------------------------------------------------------------
+// Helper functions
+
+Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+    std::shared_ptr<ArrayBuilder>* out);
+
 }  // namespace arrow
 
 #endif  // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index f429a813c6f20..650ec90fc0728 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -17,9 +17,9 @@
 
 #include "benchmark/benchmark.h"
 
+#include "arrow/array.h"
+#include "arrow/memory_pool.h"
 #include "arrow/test-util.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/memory-pool.h"
 
 namespace arrow {
 namespace {
diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
index ac3636d1b6dab..9005245b20419 100644
--- a/cpp/src/arrow/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -27,7 +27,6 @@
 #include "arrow/schema.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/primitive.h"
 
 using std::shared_ptr;
 using std::vector;
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index eca5f4d30a698..1d136e7d95a55 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -21,8 +21,8 @@
 #include <sstream>
 
 #include "arrow/array.h"
+#include "arrow/status.h"
 #include "arrow/type.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 05fa6663e335d..c50a9bba28e8e 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -107,9 +107,9 @@
 
 #include "arrow/io/interfaces.h"
 
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 8c6d49f92e606..b8e212026b11c 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -22,10 +22,10 @@
 #include <sstream>
 #include <string>
 
+#include "arrow/buffer.h"
 #include "arrow/io/hdfs.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 44986cee1afc9..68c1ac30f8250 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -20,8 +20,8 @@
 #include <cstdint>
 #include <memory>
 
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/buffer.h"
+#include "arrow/status.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index fad49cef89908..f0ea7ec5e4dea 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -30,7 +30,7 @@
 
 #include "arrow/io/file.h"
 #include "arrow/io/test-common.h"
-#include "arrow/util/memory-pool.h"
+#include "arrow/memory_pool.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 8338de6d96a55..e07eaa3d1b487 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -25,8 +25,8 @@
 #include <boost/filesystem.hpp>  // NOLINT
 
 #include "arrow/io/hdfs.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
index 36b8a4ec980a9..3715376ebb95b 100644
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -53,7 +53,7 @@ extern "C" {
 
 #include <boost/filesystem.hpp>  // NOLINT
 
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 #include "arrow/util/visibility.h"
 
 namespace fs = boost::filesystem;
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index af495e27e5642..b5cf4b77a980d 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -38,10 +38,9 @@
 #include <sstream>
 #include <string>
 
+#include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
-
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index f8fed883cf583..146808371d307 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -32,10 +32,10 @@
 // nothing
 #endif
 
+#include "arrow/buffer.h"
 #include "arrow/io/memory.h"
+#include "arrow/memory_pool.h"
 #include "arrow/test-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index edf716f662753..f813c1dbbc3b0 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -23,6 +23,7 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/Message_generated.h"
@@ -30,17 +31,11 @@
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
index fa50058ea4200..d7d2e613f87db 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/file.cc
@@ -22,14 +22,14 @@
 #include <sstream>
 #include <vector>
 
+#include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/adapter.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
-#include "arrow/util/buffer.h"
+#include "arrow/status.h"
 #include "arrow/util/logging.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 1accfde7c4842..f309b8562f76a 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -30,15 +30,11 @@
 #include "arrow/ipc/test-common.h"
 #include "arrow/ipc/util.h"
 
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index a1feac401f24e..0a9f677966389 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -24,6 +24,7 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/array.h"
 #include "arrow/io/memory.h"
 #include "arrow/io/test-common.h"
 #include "arrow/ipc/adapter.h"
@@ -31,15 +32,11 @@
 #include "arrow/ipc/test-common.h"
 #include "arrow/ipc/util.h"
 
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index ba4d9ca982850..f793a2659579c 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -26,17 +26,15 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
+#include "arrow/builder.h"
 #include "arrow/ipc/json-internal.h"
 #include "arrow/ipc/json.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 namespace ipc {
@@ -147,7 +145,7 @@ TEST(TestJsonArrayWriter, NestedTypes) {
   std::vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
 
   std::shared_ptr<Array> values_array;
-  MakeArray<Int32Type, int32_t>(int32(), values_is_valid, values, &values_array);
+  ArrayFromVector<Int32Type, int32_t>(int32(), values_is_valid, values, &values_array);
 
   // List
   std::vector<bool> list_is_valid = {true, false, true, true, true};
@@ -188,10 +186,10 @@ void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
   test::randint<int32_t>(num_rows, 0, 100, &v2_values);
 
   std::shared_ptr<Array> v1;
-  MakeArray<Int8Type, int8_t>(schema->field(0)->type, is_valid, v1_values, &v1);
+  ArrayFromVector<Int8Type, int8_t>(schema->field(0)->type, is_valid, v1_values, &v1);
 
   std::shared_ptr<Array> v2;
-  MakeArray<Int32Type, int32_t>(schema->field(1)->type, is_valid, v2_values, &v2);
+  ArrayFromVector<Int32Type, int32_t>(schema->field(1)->type, is_valid, v2_values, &v2);
 
   static const int kBufferSize = 10;
   static uint8_t buffer[kBufferSize];
@@ -323,13 +321,13 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
   std::vector<bool> foo_valid = {true, false, true, true, true};
   std::vector<int32_t> foo_values = {1, 2, 3, 4, 5};
   std::shared_ptr<Array> foo;
-  MakeArray<Int32Type, int32_t>(int32(), foo_valid, foo_values, &foo);
+  ArrayFromVector<Int32Type, int32_t>(int32(), foo_valid, foo_values, &foo);
   ASSERT_TRUE(batch->column(0)->Equals(foo));
 
   std::vector<bool> bar_valid = {true, false, false, true, true};
   std::vector<double> bar_values = {1, 2, 3, 4, 5};
   std::shared_ptr<Array> bar;
-  MakeArray<DoubleType, double>(float64(), bar_valid, bar_values, &bar);
+  ArrayFromVector<DoubleType, double>(float64(), bar_valid, bar_values, &bar);
   ASSERT_TRUE(batch->column(1)->Equals(bar));
 }
 
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
index de08e6dab73c6..7c5744a241068 100644
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -24,9 +24,9 @@
 #include "arrow/io/memory.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 291a719d4e58c..5e593560f8cfa 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -33,9 +33,9 @@
 #include "arrow/ipc/json.h"
 #include "arrow/pretty_print.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
-#include "arrow/util/status.h"
 
 DEFINE_string(arrow, "", "Arrow file name");
 DEFINE_string(json, "", "JSON file name");
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index ff9f59800be38..db11b7d0372f7 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -28,16 +28,14 @@
 #include "rapidjson/writer.h"
 
 #include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/memory_pool.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index 2281611f8b879..6e3a9939730f4 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -23,14 +23,14 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/buffer.h"
 #include "arrow/ipc/json-internal.h"
+#include "arrow/memory_pool.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
-#include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 5a2758912b759..16069a8f9dcf0 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -25,11 +25,11 @@
 
 #include "flatbuffers/flatbuffers.h"
 
+#include "arrow/buffer.h"
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/type.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 44d3939c04f1d..f0674ff8d5aeb 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -28,9 +28,9 @@
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/ipc/metadata-internal.h"
 
+#include "arrow/buffer.h"
 #include "arrow/schema.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 65b378215222d..8416f0df57364 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -25,16 +25,13 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/memory_pool.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/util.h b/cpp/src/arrow/ipc/util.h
index 242d6624f1e7f..2000c61e7ed57 100644
--- a/cpp/src/arrow/ipc/util.h
+++ b/cpp/src/arrow/ipc/util.h
@@ -22,7 +22,7 @@
 
 #include "arrow/array.h"
 #include "arrow/io/interfaces.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
similarity index 96%
rename from cpp/src/arrow/util/memory-pool-test.cc
rename to cpp/src/arrow/memory_pool-test.cc
index 5d60376f794ff..d6f323d276305 100644
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -20,9 +20,9 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/memory_pool.cc
similarity index 97%
rename from cpp/src/arrow/util/memory-pool.cc
rename to cpp/src/arrow/memory_pool.cc
index 9aa706693e9f7..f55b1ac668c7c 100644
--- a/cpp/src/arrow/util/memory-pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -15,15 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/memory-pool.h"
+#include "arrow/memory_pool.h"
 
 #include <cstdlib>
 #include <mutex>
 #include <sstream>
 #include <stdlib.h>
 
+#include "arrow/status.h"
 #include "arrow/util/logging.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/memory-pool.h b/cpp/src/arrow/memory_pool.h
similarity index 100%
rename from cpp/src/arrow/util/memory-pool.h
rename to cpp/src/arrow/memory_pool.h
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index b1e6a11cedd9b..c22d3aa632b9d 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -26,14 +26,11 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
+#include "arrow/builder.h"
 #include "arrow/pretty_print.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index c63a9e93e6a63..9c439c47eb82c 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -22,13 +22,10 @@
 
 #include "arrow/array.h"
 #include "arrow/pretty_print.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
-#include "arrow/types/list.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/status-test.cc b/cpp/src/arrow/status-test.cc
similarity index 97%
rename from cpp/src/arrow/util/status-test.cc
rename to cpp/src/arrow/status-test.cc
index e0ff20fea1233..969ba970c154f 100644
--- a/cpp/src/arrow/util/status-test.cc
+++ b/cpp/src/arrow/status-test.cc
@@ -17,8 +17,8 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/status.h"
 #include "arrow/test-util.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/status.cc
similarity index 98%
rename from cpp/src/arrow/util/status.cc
rename to cpp/src/arrow/status.cc
index 08e9ae3946e51..e1a242721eccc 100644
--- a/cpp/src/arrow/util/status.cc
+++ b/cpp/src/arrow/status.cc
@@ -10,7 +10,7 @@
 // non-const method, all threads accessing the same Status must use
 // external synchronization.
 
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 
 #include <assert.h>
 
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/status.h
similarity index 100%
rename from cpp/src/arrow/util/status.h
rename to cpp/src/arrow/status.h
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 743fb669700ea..f62336d07f09a 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -21,13 +21,13 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/status.h"
 
 using std::shared_ptr;
 using std::vector;
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index eb1258a73038a..855d4ec04085d 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -24,7 +24,7 @@
 #include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/schema.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index b86a1809cd0e9..aa310b1a49ebe 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -28,17 +28,18 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
 #include "arrow/column.h"
+#include "arrow/memory_pool.h"
 #include "arrow/schema.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
 #include "arrow/util/logging.h"
-#include "arrow/util/memory-pool.h"
 #include "arrow/util/random.h"
-#include "arrow/util/status.h"
 
 #define ASSERT_RAISES(ENUM, expr)                  \
   do {                                             \
@@ -253,8 +254,9 @@ Status MakeRandomBytePoolBuffer(int32_t length, MemoryPool* pool,
 }  // namespace test
 
 template <typename TYPE, typename C_TYPE>
-void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& is_valid,
-    const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
+void ArrayFromVector(const std::shared_ptr<DataType>& type,
+    const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
+    std::shared_ptr<Array>* out) {
   std::shared_ptr<Buffer> values_buffer;
   std::shared_ptr<Buffer> values_bitmap;
 
@@ -272,6 +274,37 @@ void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& i
       values_buffer, null_count, values_bitmap);
 }
 
+class TestBuilder : public ::testing::Test {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+    type_ = TypePtr(new UInt8Type());
+    builder_.reset(new UInt8Builder(pool_, type_));
+    builder_nn_.reset(new UInt8Builder(pool_, type_));
+  }
+
+ protected:
+  MemoryPool* pool_;
+
+  TypePtr type_;
+  std::unique_ptr<ArrayBuilder> builder_;
+  std::unique_ptr<ArrayBuilder> builder_nn_;
+};
+
+template <class T, class Builder>
+Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
+    int size, Builder* builder, ArrayPtr* out) {
+  // Append the first 1000
+  for (int i = 0; i < size; ++i) {
+    if (valid_bytes[i] > 0) {
+      RETURN_NOT_OK(builder->Append(values[i]));
+    } else {
+      RETURN_NOT_OK(builder->AppendNull());
+    }
+  }
+  return builder->Finish(out);
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_TEST_UTIL_H_
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 75f5086f37de0..5b172e41f6809 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -20,7 +20,7 @@
 #include <sstream>
 #include <string>
 
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 
 namespace arrow {
 
@@ -220,6 +220,12 @@ std::vector<BufferDescr> UnionType::GetBufferLayout() const {
   }
 }
 
+std::string DecimalType::ToString() const {
+  std::stringstream s;
+  s << "decimal(" << precision << ", " << scale << ")";
+  return s.str();
+}
+
 std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
   // TODO(wesm)
   return {};
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 966706cb520b2..8637081acd9b7 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -23,9 +23,9 @@
 #include <string>
 #include <vector>
 
+#include "arrow/status.h"
 #include "arrow/type_fwd.h"
 #include "arrow/util/macros.h"
-#include "arrow/util/status.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
deleted file mode 100644
index 6d59acfdf2eec..0000000000000
--- a/cpp/src/arrow/types/CMakeLists.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#######################################
-# arrow_types
-#######################################
-
-# Headers: top level
-install(FILES
-  construct.h
-  datetime.h
-  decimal.h
-  list.h
-  primitive.h
-  string.h
-  struct.h
-  union.h
-  DESTINATION include/arrow/types)
-
-
-ADD_ARROW_TEST(decimal-test)
-ADD_ARROW_TEST(list-test)
-ADD_ARROW_TEST(primitive-test)
-ADD_ARROW_TEST(string-test)
-ADD_ARROW_TEST(struct-test)
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
deleted file mode 100644
index ab9c59fd4639d..0000000000000
--- a/cpp/src/arrow/types/construct.cc
+++ /dev/null
@@ -1,124 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/construct.h"
-
-#include <memory>
-
-#include "arrow/type.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
-
-namespace arrow {
-
-class ArrayBuilder;
-
-#define BUILDER_CASE(ENUM, BuilderType)      \
-  case Type::ENUM:                           \
-    out->reset(new BuilderType(pool, type)); \
-    return Status::OK();
-
-// Initially looked at doing this with vtables, but shared pointers makes it
-// difficult
-//
-// TODO(wesm): come up with a less monolithic strategy
-Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-    std::shared_ptr<ArrayBuilder>* out) {
-  switch (type->type) {
-    BUILDER_CASE(UINT8, UInt8Builder);
-    BUILDER_CASE(INT8, Int8Builder);
-    BUILDER_CASE(UINT16, UInt16Builder);
-    BUILDER_CASE(INT16, Int16Builder);
-    BUILDER_CASE(UINT32, UInt32Builder);
-    BUILDER_CASE(INT32, Int32Builder);
-    BUILDER_CASE(UINT64, UInt64Builder);
-    BUILDER_CASE(INT64, Int64Builder);
-    BUILDER_CASE(TIMESTAMP, TimestampBuilder);
-
-    BUILDER_CASE(BOOL, BooleanBuilder);
-
-    BUILDER_CASE(FLOAT, FloatBuilder);
-    BUILDER_CASE(DOUBLE, DoubleBuilder);
-
-    BUILDER_CASE(STRING, StringBuilder);
-    BUILDER_CASE(BINARY, BinaryBuilder);
-
-    case Type::LIST: {
-      std::shared_ptr<ArrayBuilder> value_builder;
-      std::shared_ptr<DataType> value_type =
-          static_cast<ListType*>(type.get())->value_type();
-      RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
-      out->reset(new ListBuilder(pool, value_builder));
-      return Status::OK();
-    }
-
-    case Type::STRUCT: {
-      std::vector<FieldPtr>& fields = type->children_;
-      std::vector<std::shared_ptr<ArrayBuilder>> values_builder;
-
-      for (auto it : fields) {
-        std::shared_ptr<ArrayBuilder> builder;
-        RETURN_NOT_OK(MakeBuilder(pool, it->type, &builder));
-        values_builder.push_back(builder);
-      }
-      out->reset(new StructBuilder(pool, type, values_builder));
-      return Status::OK();
-    }
-
-    default:
-      return Status::NotImplemented(type->ToString());
-  }
-}
-
-#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                          \
-  case Type::ENUM:                                                          \
-    out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
-    break;
-
-Status MakePrimitiveArray(const TypePtr& type, int32_t length,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
-  switch (type->type) {
-    MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
-    MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(INT8, Int8Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(UINT16, UInt16Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(INT16, Int16Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(UINT32, UInt32Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(INT32, Int32Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
-    MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP_DOUBLE, DoubleArray);
-    default:
-      return Status::NotImplemented(type->ToString());
-  }
-#ifdef NDEBUG
-  return Status::OK();
-#else
-  return (*out)->Validate();
-#endif
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
deleted file mode 100644
index e18e946d1a64c..0000000000000
--- a/cpp/src/arrow/types/construct.h
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_CONSTRUCT_H
-#define ARROW_TYPES_CONSTRUCT_H
-
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Array;
-class ArrayBuilder;
-class Buffer;
-struct DataType;
-struct Field;
-class MemoryPool;
-class Status;
-
-Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-    std::shared_ptr<ArrayBuilder>* out);
-
-// Create new arrays for logical types that are backed by primitive arrays.
-Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
-
-}  // namespace arrow
-
-#endif  // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
deleted file mode 100644
index a8f863923129a..0000000000000
--- a/cpp/src/arrow/types/datetime.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_DATETIME_H
-#define ARROW_TYPES_DATETIME_H
-
-#include <string>
-
-#include "arrow/type.h"
-
-namespace arrow {}  // namespace arrow
-
-#endif  // ARROW_TYPES_DATETIME_H
diff --git a/cpp/src/arrow/types/decimal.cc b/cpp/src/arrow/types/decimal.cc
deleted file mode 100644
index 1d9a5e50e460b..0000000000000
--- a/cpp/src/arrow/types/decimal.cc
+++ /dev/null
@@ -1,31 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/decimal.h"
-
-#include <sstream>
-#include <string>
-
-namespace arrow {
-
-std::string DecimalType::ToString() const {
-  std::stringstream s;
-  s << "decimal(" << precision << ", " << scale << ")";
-  return s.str();
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h
deleted file mode 100644
index b3ea3a56d8008..0000000000000
--- a/cpp/src/arrow/types/decimal.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_DECIMAL_H
-#define ARROW_TYPES_DECIMAL_H
-
-#include <string>
-
-#include "arrow/type.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {}  // namespace arrow
-
-#endif  // ARROW_TYPES_DECIMAL_H
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
deleted file mode 100644
index d86563253bd5a..0000000000000
--- a/cpp/src/arrow/types/list.cc
+++ /dev/null
@@ -1,162 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-#include "arrow/types/list.h"
-
-#include <sstream>
-
-namespace arrow {
-
-bool ListArray::EqualsExact(const ListArray& other) const {
-  if (this == &other) { return true; }
-  if (null_count_ != other.null_count_) { return false; }
-
-  bool equal_offsets =
-      offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
-  if (!equal_offsets) { return false; }
-  bool equal_null_bitmap = true;
-  if (null_count_ > 0) {
-    equal_null_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
-  }
-
-  if (!equal_null_bitmap) { return false; }
-
-  return values()->Equals(other.values());
-}
-
-bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(*static_cast<const ListArray*>(arr.get()));
-}
-
-bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  const auto other = static_cast<ListArray*>(arr.get());
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    const bool is_null = IsNull(i);
-    if (is_null != arr->IsNull(o_i)) { return false; }
-    if (is_null) continue;
-    const int32_t begin_offset = offset(i);
-    const int32_t end_offset = offset(i + 1);
-    const int32_t other_begin_offset = other->offset(o_i);
-    const int32_t other_end_offset = other->offset(o_i + 1);
-    // Underlying can't be equal if the size isn't equal
-    if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
-      return false;
-    }
-    if (!values_->RangeEquals(
-            begin_offset, end_offset, other_begin_offset, other->values())) {
-      return false;
-    }
-  }
-  return true;
-}
-
-Status ListArray::Validate() const {
-  if (length_ < 0) { return Status::Invalid("Length was negative"); }
-  if (!offset_buffer_) { return Status::Invalid("offset_buffer_ was null"); }
-  if (offset_buffer_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
-    std::stringstream ss;
-    ss << "offset buffer size (bytes): " << offset_buffer_->size()
-       << " isn't large enough for length: " << length_;
-    return Status::Invalid(ss.str());
-  }
-  const int32_t last_offset = offset(length_);
-  if (last_offset > 0) {
-    if (!values_) {
-      return Status::Invalid("last offset was non-zero and values was null");
-    }
-    if (values_->length() != last_offset) {
-      std::stringstream ss;
-      ss << "Final offset invariant not equal to values length: " << last_offset
-         << "!=" << values_->length();
-      return Status::Invalid(ss.str());
-    }
-
-    const Status child_valid = values_->Validate();
-    if (!child_valid.ok()) {
-      std::stringstream ss;
-      ss << "Child array invalid: " << child_valid.ToString();
-      return Status::Invalid(ss.str());
-    }
-  }
-
-  int32_t prev_offset = offset(0);
-  if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
-  for (int32_t i = 1; i <= length_; ++i) {
-    int32_t current_offset = offset(i);
-    if (IsNull(i - 1) && current_offset != prev_offset) {
-      std::stringstream ss;
-      ss << "Offset invariant failure at: " << i << " inconsistent offsets for null slot"
-         << current_offset << "!=" << prev_offset;
-      return Status::Invalid(ss.str());
-    }
-    if (current_offset < prev_offset) {
-      std::stringstream ss;
-      ss << "Offset invariant failure: " << i
-         << " inconsistent offset for non-null slot: " << current_offset << "<"
-         << prev_offset;
-      return Status::Invalid(ss.str());
-    }
-    prev_offset = current_offset;
-  }
-  return Status::OK();
-}
-
-Status ListBuilder::Init(int32_t elements) {
-  DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
-  RETURN_NOT_OK(ArrayBuilder::Init(elements));
-  // one more then requested for offsets
-  return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
-}
-
-Status ListBuilder::Resize(int32_t capacity) {
-  DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
-  // one more then requested for offsets
-  RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
-  return ArrayBuilder::Resize(capacity);
-}
-
-Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<Array> items = values_;
-  if (!items) { RETURN_NOT_OK(value_builder_->Finish(&items)); }
-
-  RETURN_NOT_OK(offset_builder_.Append<int32_t>(items->length()));
-  std::shared_ptr<Buffer> offsets = offset_builder_.Finish();
-
-  *out = std::make_shared<ListArray>(
-      type_, length_, offsets, items, null_count_, null_bitmap_);
-
-  Reset();
-
-  return Status::OK();
-}
-
-void ListBuilder::Reset() {
-  capacity_ = length_ = null_count_ = 0;
-  null_bitmap_ = nullptr;
-}
-
-Status ListArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
deleted file mode 100644
index ec09a78afa66c..0000000000000
--- a/cpp/src/arrow/types/list.h
+++ /dev/null
@@ -1,170 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_LIST_H
-#define ARROW_TYPES_LIST_H
-
-#include <cstdint>
-#include <cstring>
-#include <limits>
-#include <memory>
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/type.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/logging.h"
-#include "arrow/util/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class MemoryPool;
-
-class ARROW_EXPORT ListArray : public Array {
- public:
-  using TypeClass = ListType;
-
-  ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
-      const ArrayPtr& values, int32_t null_count = 0,
-      std::shared_ptr<Buffer> null_bitmap = nullptr)
-      : Array(type, length, null_count, null_bitmap) {
-    offset_buffer_ = offsets;
-    offsets_ = offsets == nullptr ? nullptr : reinterpret_cast<const int32_t*>(
-                                                  offset_buffer_->data());
-    values_ = values;
-  }
-
-  Status Validate() const override;
-
-  virtual ~ListArray() = default;
-
-  // Return a shared pointer in case the requestor desires to share ownership
-  // with this array.
-  std::shared_ptr<Array> values() const { return values_; }
-  std::shared_ptr<Buffer> offsets() const {
-    return std::static_pointer_cast<Buffer>(offset_buffer_);
-  }
-
-  std::shared_ptr<DataType> value_type() const { return values_->type(); }
-
-  const int32_t* raw_offsets() const { return offsets_; }
-
-  int32_t offset(int i) const { return offsets_[i]; }
-
-  // Neither of these functions will perform boundschecking
-  int32_t value_offset(int i) const { return offsets_[i]; }
-  int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
-
-  bool EqualsExact(const ListArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const ArrayPtr& arr) const override;
-
-  Status Accept(ArrayVisitor* visitor) const override;
-
- protected:
-  std::shared_ptr<Buffer> offset_buffer_;
-  const int32_t* offsets_;
-  ArrayPtr values_;
-};
-
-// ----------------------------------------------------------------------
-// Array builder
-
-// Builder class for variable-length list array value types
-//
-// To use this class, you must append values to the child array builder and use
-// the Append function to delimit each distinct list value (once the values
-// have been appended to the child array) or use the bulk API to append
-// a sequence of offests and null values.
-//
-// A note on types.  Per arrow/type.h all types in the c++ implementation are
-// logical so even though this class always builds list array, this can
-// represent multiple different logical types.  If no logical type is provided
-// at construction time, the class defaults to List<T> where t is taken from the
-// value_builder/values that the object is constructed with.
-class ARROW_EXPORT ListBuilder : public ArrayBuilder {
- public:
-  // Use this constructor to incrementally build the value array along with offsets and
-  // null bitmap.
-  ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
-      const TypePtr& type = nullptr)
-      : ArrayBuilder(
-            pool, type ? type : std::static_pointer_cast<DataType>(
-                                    std::make_shared<ListType>(value_builder->type()))),
-        offset_builder_(pool),
-        value_builder_(value_builder) {}
-
-  // Use this constructor to build the list with a pre-existing values array
-  ListBuilder(
-      MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr)
-      : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
-                                             std::make_shared<ListType>(values->type()))),
-        offset_builder_(pool),
-        values_(values) {}
-
-  virtual ~ListBuilder() {}
-
-  Status Init(int32_t elements) override;
-  Status Resize(int32_t capacity) override;
-  Status Finish(std::shared_ptr<Array>* out) override;
-
-  // Vector append
-  //
-  // If passed, valid_bytes is of equal length to values, and any zero byte
-  // will be considered as a null for that slot
-  Status Append(
-      const int32_t* offsets, int32_t length, const uint8_t* valid_bytes = nullptr) {
-    RETURN_NOT_OK(Reserve(length));
-    UnsafeAppendToBitmap(valid_bytes, length);
-    offset_builder_.UnsafeAppend<int32_t>(offsets, length);
-    return Status::OK();
-  }
-
-  // Start a new variable-length list slot
-  //
-  // This function should be called before beginning to append elements to the
-  // value builder
-  Status Append(bool is_valid = true) {
-    RETURN_NOT_OK(Reserve(1));
-    UnsafeAppendToBitmap(is_valid);
-    RETURN_NOT_OK(offset_builder_.Append<int32_t>(value_builder_->length()));
-    return Status::OK();
-  }
-
-  Status AppendNull() { return Append(false); }
-
-  std::shared_ptr<ArrayBuilder> value_builder() const {
-    DCHECK(!values_) << "Using value builder is pointless when values_ is set";
-    return value_builder_;
-  }
-
- protected:
-  BufferBuilder offset_builder_;
-  std::shared_ptr<ArrayBuilder> value_builder_;
-  std::shared_ptr<Array> values_;
-
-  void Reset();
-};
-
-}  // namespace arrow
-
-#endif  // ARROW_TYPES_LIST_H
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
deleted file mode 100644
index 75e5a9ff40e16..0000000000000
--- a/cpp/src/arrow/types/primitive.cc
+++ /dev/null
@@ -1,294 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/primitive.h"
-
-#include <cmath>
-#include <memory>
-
-#include "arrow/type_traits.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/logging.h"
-
-namespace arrow {
-
-// ----------------------------------------------------------------------
-// Primitive array base
-
-PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : Array(type, length, null_count, null_bitmap) {
-  data_ = data;
-  raw_data_ = data == nullptr ? nullptr : data_->data();
-}
-
-bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
-  if (this == &other) { return true; }
-  if (null_count_ != other.null_count_) { return false; }
-
-  if (null_count_ > 0) {
-    bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
-    if (!equal_bitmap) { return false; }
-
-    const uint8_t* this_data = raw_data_;
-    const uint8_t* other_data = other.raw_data_;
-
-    auto size_meta = dynamic_cast<const FixedWidthType*>(type_.get());
-    int value_byte_size = size_meta->bit_width() / 8;
-    DCHECK_GT(value_byte_size, 0);
-
-    for (int i = 0; i < length_; ++i) {
-      if (!IsNull(i) && memcmp(this_data, other_data, value_byte_size)) { return false; }
-      this_data += value_byte_size;
-      other_data += value_byte_size;
-    }
-    return true;
-  } else {
-    if (length_ == 0 && other.length_ == 0) { return true; }
-    return data_->Equals(*other.data_, length_);
-  }
-}
-
-bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
-}
-
-template <typename T>
-Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-template class NumericArray<UInt8Type>;
-template class NumericArray<UInt16Type>;
-template class NumericArray<UInt32Type>;
-template class NumericArray<UInt64Type>;
-template class NumericArray<Int8Type>;
-template class NumericArray<Int16Type>;
-template class NumericArray<Int32Type>;
-template class NumericArray<Int64Type>;
-template class NumericArray<TimestampType>;
-template class NumericArray<HalfFloatType>;
-template class NumericArray<FloatType>;
-template class NumericArray<DoubleType>;
-
-template <typename T>
-Status PrimitiveBuilder<T>::Init(int32_t capacity) {
-  RETURN_NOT_OK(ArrayBuilder::Init(capacity));
-  data_ = std::make_shared<PoolBuffer>(pool_);
-
-  int64_t nbytes = TypeTraits<T>::bytes_required(capacity);
-  RETURN_NOT_OK(data_->Resize(nbytes));
-  // TODO(emkornfield) valgrind complains without this
-  memset(data_->mutable_data(), 0, nbytes);
-
-  raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
-  return Status::OK();
-}
-
-template <typename T>
-Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
-  // XXX: Set floor size for now
-  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
-
-  if (capacity_ == 0) {
-    RETURN_NOT_OK(Init(capacity));
-  } else {
-    RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
-    const int64_t old_bytes = data_->size();
-    const int64_t new_bytes = TypeTraits<T>::bytes_required(capacity);
-    RETURN_NOT_OK(data_->Resize(new_bytes));
-    raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
-    memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
-  }
-  return Status::OK();
-}
-
-template <typename T>
-Status PrimitiveBuilder<T>::Append(
-    const value_type* values, int32_t length, const uint8_t* valid_bytes) {
-  RETURN_NOT_OK(Reserve(length));
-
-  if (length > 0) {
-    memcpy(raw_data_ + length_, values, TypeTraits<T>::bytes_required(length));
-  }
-
-  // length_ is update by these
-  ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
-
-  return Status::OK();
-}
-
-template <typename T>
-Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
-  const int64_t bytes_required = TypeTraits<T>::bytes_required(length_);
-  if (bytes_required > 0 && bytes_required < data_->size()) {
-    // Trim buffers
-    RETURN_NOT_OK(data_->Resize(bytes_required));
-  }
-  *out = std::make_shared<typename TypeTraits<T>::ArrayType>(
-      type_, length_, data_, null_count_, null_bitmap_);
-
-  data_ = null_bitmap_ = nullptr;
-  capacity_ = length_ = null_count_ = 0;
-  return Status::OK();
-}
-
-template class PrimitiveBuilder<UInt8Type>;
-template class PrimitiveBuilder<UInt16Type>;
-template class PrimitiveBuilder<UInt32Type>;
-template class PrimitiveBuilder<UInt64Type>;
-template class PrimitiveBuilder<Int8Type>;
-template class PrimitiveBuilder<Int16Type>;
-template class PrimitiveBuilder<Int32Type>;
-template class PrimitiveBuilder<Int64Type>;
-template class PrimitiveBuilder<TimestampType>;
-template class PrimitiveBuilder<HalfFloatType>;
-template class PrimitiveBuilder<FloatType>;
-template class PrimitiveBuilder<DoubleType>;
-
-Status BooleanBuilder::Init(int32_t capacity) {
-  RETURN_NOT_OK(ArrayBuilder::Init(capacity));
-  data_ = std::make_shared<PoolBuffer>(pool_);
-
-  int64_t nbytes = BitUtil::BytesForBits(capacity);
-  RETURN_NOT_OK(data_->Resize(nbytes));
-  // TODO(emkornfield) valgrind complains without this
-  memset(data_->mutable_data(), 0, nbytes);
-
-  raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
-  return Status::OK();
-}
-
-Status BooleanBuilder::Resize(int32_t capacity) {
-  // XXX: Set floor size for now
-  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
-
-  if (capacity_ == 0) {
-    RETURN_NOT_OK(Init(capacity));
-  } else {
-    RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
-    const int64_t old_bytes = data_->size();
-    const int64_t new_bytes = BitUtil::BytesForBits(capacity);
-
-    RETURN_NOT_OK(data_->Resize(new_bytes));
-    raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
-    memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
-  }
-  return Status::OK();
-}
-
-Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
-  const int64_t bytes_required = BitUtil::BytesForBits(length_);
-
-  if (bytes_required > 0 && bytes_required < data_->size()) {
-    // Trim buffers
-    RETURN_NOT_OK(data_->Resize(bytes_required));
-  }
-  *out = std::make_shared<BooleanArray>(type_, length_, data_, null_count_, null_bitmap_);
-
-  data_ = null_bitmap_ = nullptr;
-  capacity_ = length_ = null_count_ = 0;
-  return Status::OK();
-}
-
-Status BooleanBuilder::Append(
-    const uint8_t* values, int32_t length, const uint8_t* valid_bytes) {
-  RETURN_NOT_OK(Reserve(length));
-
-  for (int i = 0; i < length; ++i) {
-    // Skip reading from unitialised memory
-    // TODO: This actually is only to keep valgrind happy but may or may not
-    // have a performance impact.
-    if ((valid_bytes != nullptr) && !valid_bytes[i]) continue;
-
-    if (values[i] > 0) {
-      BitUtil::SetBit(raw_data_, length_ + i);
-    } else {
-      BitUtil::ClearBit(raw_data_, length_ + i);
-    }
-  }
-
-  // this updates length_
-  ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
-  return Status::OK();
-}
-
-BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
-    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
-    : PrimitiveArray(
-          std::make_shared<BooleanType>(), length, data, null_count, null_bitmap) {}
-
-BooleanArray::BooleanArray(const TypePtr& type, int32_t length,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
-
-bool BooleanArray::EqualsExact(const BooleanArray& other) const {
-  if (this == &other) return true;
-  if (null_count_ != other.null_count_) { return false; }
-
-  if (null_count_ > 0) {
-    bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
-    if (!equal_bitmap) { return false; }
-
-    const uint8_t* this_data = raw_data_;
-    const uint8_t* other_data = other.raw_data_;
-
-    for (int i = 0; i < length_; ++i) {
-      if (!IsNull(i) && BitUtil::GetBit(this_data, i) != BitUtil::GetBit(other_data, i)) {
-        return false;
-      }
-    }
-    return true;
-  } else {
-    return data_->Equals(*other.data_, BitUtil::BytesForBits(length_));
-  }
-}
-
-bool BooleanArray::Equals(const ArrayPtr& arr) const {
-  if (this == arr.get()) return true;
-  if (Type::BOOL != arr->type_enum()) { return false; }
-  return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
-}
-
-bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
-    int32_t other_start_idx, const ArrayPtr& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  const auto other = static_cast<BooleanArray*>(arr.get());
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    const bool is_null = IsNull(i);
-    if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
-      return false;
-    }
-  }
-  return true;
-}
-
-Status BooleanArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
deleted file mode 100644
index ec578e1e0aee7..0000000000000
--- a/cpp/src/arrow/types/primitive.h
+++ /dev/null
@@ -1,371 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_PRIMITIVE_H
-#define ARROW_TYPES_PRIMITIVE_H
-
-#include <cmath>
-#include <cstdint>
-#include <cstring>
-#include <iostream>
-#include <memory>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/type.h"
-#include "arrow/type_fwd.h"
-#include "arrow/types/datetime.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class MemoryPool;
-
-// Base class for fixed-size logical types.  See MakePrimitiveArray
-// (types/construct.h) for constructing a specific subclass.
-class ARROW_EXPORT PrimitiveArray : public Array {
- public:
-  virtual ~PrimitiveArray() {}
-
-  std::shared_ptr<Buffer> data() const { return data_; }
-
-  bool EqualsExact(const PrimitiveArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-
- protected:
-  PrimitiveArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-  std::shared_ptr<Buffer> data_;
-  const uint8_t* raw_data_;
-};
-
-template <typename TYPE>
-class ARROW_EXPORT NumericArray : public PrimitiveArray {
- public:
-  using TypeClass = TYPE;
-  using value_type = typename TypeClass::c_type;
-  NumericArray(int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
-      : PrimitiveArray(
-            std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {}
-  NumericArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
-      : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
-
-  bool EqualsExact(const NumericArray<TypeClass>& other) const {
-    return PrimitiveArray::EqualsExact(static_cast<const PrimitiveArray&>(other));
-  }
-
-  bool ApproxEquals(const std::shared_ptr<Array>& arr) const { return Equals(arr); }
-
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const ArrayPtr& arr) const override {
-    if (this == arr.get()) { return true; }
-    if (!arr) { return false; }
-    if (this->type_enum() != arr->type_enum()) { return false; }
-    const auto other = static_cast<NumericArray<TypeClass>*>(arr.get());
-    for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-      const bool is_null = IsNull(i);
-      if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
-        return false;
-      }
-    }
-    return true;
-  }
-  const value_type* raw_data() const {
-    return reinterpret_cast<const value_type*>(raw_data_);
-  }
-
-  Status Accept(ArrayVisitor* visitor) const override;
-
-  value_type Value(int i) const { return raw_data()[i]; }
-};
-
-template <>
-inline bool NumericArray<FloatType>::ApproxEquals(
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-
-  const auto& other = *static_cast<NumericArray<FloatType>*>(arr.get());
-
-  if (this == &other) { return true; }
-  if (null_count_ != other.null_count_) { return false; }
-
-  auto this_data = reinterpret_cast<const float*>(raw_data_);
-  auto other_data = reinterpret_cast<const float*>(other.raw_data_);
-
-  static constexpr float EPSILON = 1E-5;
-
-  if (length_ == 0 && other.length_ == 0) { return true; }
-
-  if (null_count_ > 0) {
-    bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
-    if (!equal_bitmap) { return false; }
-
-    for (int i = 0; i < length_; ++i) {
-      if (IsNull(i)) continue;
-      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
-    }
-  } else {
-    for (int i = 0; i < length_; ++i) {
-      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
-    }
-  }
-  return true;
-}
-
-template <>
-inline bool NumericArray<DoubleType>::ApproxEquals(
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-
-  const auto& other = *static_cast<NumericArray<DoubleType>*>(arr.get());
-
-  if (this == &other) { return true; }
-  if (null_count_ != other.null_count_) { return false; }
-
-  auto this_data = reinterpret_cast<const double*>(raw_data_);
-  auto other_data = reinterpret_cast<const double*>(other.raw_data_);
-
-  if (length_ == 0 && other.length_ == 0) { return true; }
-
-  static constexpr double EPSILON = 1E-5;
-
-  if (null_count_ > 0) {
-    bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
-    if (!equal_bitmap) { return false; }
-
-    for (int i = 0; i < length_; ++i) {
-      if (IsNull(i)) continue;
-      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
-    }
-  } else {
-    for (int i = 0; i < length_; ++i) {
-      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
-    }
-  }
-  return true;
-}
-
-template <typename Type>
-class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
- public:
-  using value_type = typename Type::c_type;
-
-  explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type)
-      : ArrayBuilder(pool, type), data_(nullptr) {}
-
-  virtual ~PrimitiveBuilder() {}
-
-  using ArrayBuilder::Advance;
-
-  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
-  Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
-    RETURN_NOT_OK(Reserve(length));
-    UnsafeAppendToBitmap(valid_bytes, length);
-    return Status::OK();
-  }
-
-  Status AppendNull() {
-    RETURN_NOT_OK(Reserve(1));
-    UnsafeAppendToBitmap(false);
-    return Status::OK();
-  }
-
-  std::shared_ptr<Buffer> data() const { return data_; }
-
-  // Vector append
-  //
-  // If passed, valid_bytes is of equal length to values, and any zero byte
-  // will be considered as a null for that slot
-  Status Append(
-      const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
-
-  Status Finish(std::shared_ptr<Array>* out) override;
-  Status Init(int32_t capacity) override;
-
-  // Increase the capacity of the builder to accommodate at least the indicated
-  // number of elements
-  Status Resize(int32_t capacity) override;
-
- protected:
-  std::shared_ptr<PoolBuffer> data_;
-  value_type* raw_data_;
-};
-
-template <typename T>
-class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
- public:
-  using typename PrimitiveBuilder<T>::value_type;
-  using PrimitiveBuilder<T>::PrimitiveBuilder;
-
-  using PrimitiveBuilder<T>::Append;
-  using PrimitiveBuilder<T>::Init;
-  using PrimitiveBuilder<T>::Resize;
-  using PrimitiveBuilder<T>::Reserve;
-
-  // Scalar append.
-  Status Append(value_type val) {
-    RETURN_NOT_OK(ArrayBuilder::Reserve(1));
-    UnsafeAppend(val);
-    return Status::OK();
-  }
-
-  // Does not capacity-check; make sure to call Reserve beforehand
-  void UnsafeAppend(value_type val) {
-    BitUtil::SetBit(null_bitmap_data_, length_);
-    raw_data_[length_++] = val;
-  }
-
- protected:
-  using PrimitiveBuilder<T>::length_;
-  using PrimitiveBuilder<T>::null_bitmap_data_;
-  using PrimitiveBuilder<T>::raw_data_;
-};
-
-// Builders
-
-using UInt8Builder = NumericBuilder<UInt8Type>;
-using UInt16Builder = NumericBuilder<UInt16Type>;
-using UInt32Builder = NumericBuilder<UInt32Type>;
-using UInt64Builder = NumericBuilder<UInt64Type>;
-
-using Int8Builder = NumericBuilder<Int8Type>;
-using Int16Builder = NumericBuilder<Int16Type>;
-using Int32Builder = NumericBuilder<Int32Type>;
-using Int64Builder = NumericBuilder<Int64Type>;
-using TimestampBuilder = NumericBuilder<TimestampType>;
-
-using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
-using FloatBuilder = NumericBuilder<FloatType>;
-using DoubleBuilder = NumericBuilder<DoubleType>;
-
-class ARROW_EXPORT BooleanArray : public PrimitiveArray {
- public:
-  using TypeClass = BooleanType;
-
-  BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-  BooleanArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
-  bool EqualsExact(const BooleanArray& other) const;
-  bool Equals(const ArrayPtr& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const ArrayPtr& arr) const override;
-
-  Status Accept(ArrayVisitor* visitor) const override;
-
-  const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
-
-  bool Value(int i) const { return BitUtil::GetBit(raw_data(), i); }
-};
-
-class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
- public:
-  explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type)
-      : ArrayBuilder(pool, type), data_(nullptr) {}
-
-  virtual ~BooleanBuilder() {}
-
-  using ArrayBuilder::Advance;
-
-  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
-  Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
-    RETURN_NOT_OK(Reserve(length));
-    UnsafeAppendToBitmap(valid_bytes, length);
-    return Status::OK();
-  }
-
-  Status AppendNull() {
-    RETURN_NOT_OK(Reserve(1));
-    UnsafeAppendToBitmap(false);
-    return Status::OK();
-  }
-
-  std::shared_ptr<Buffer> data() const { return data_; }
-
-  // Scalar append
-  Status Append(bool val) {
-    Reserve(1);
-    BitUtil::SetBit(null_bitmap_data_, length_);
-    if (val) {
-      BitUtil::SetBit(raw_data_, length_);
-    } else {
-      BitUtil::ClearBit(raw_data_, length_);
-    }
-    ++length_;
-    return Status::OK();
-  }
-
-  // Vector append
-  //
-  // If passed, valid_bytes is of equal length to values, and any zero byte
-  // will be considered as a null for that slot
-  Status Append(
-      const uint8_t* values, int32_t length, const uint8_t* valid_bytes = nullptr);
-
-  Status Finish(std::shared_ptr<Array>* out) override;
-  Status Init(int32_t capacity) override;
-
-  // Increase the capacity of the builder to accommodate at least the indicated
-  // number of elements
-  Status Resize(int32_t capacity) override;
-
- protected:
-  std::shared_ptr<PoolBuffer> data_;
-  uint8_t* raw_data_;
-};
-
-// gcc and clang disagree about how to handle template visibility when you have
-// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-#endif
-
-// Only instantiate these templates once
-extern template class ARROW_EXPORT NumericArray<Int8Type>;
-extern template class ARROW_EXPORT NumericArray<UInt8Type>;
-extern template class ARROW_EXPORT NumericArray<Int16Type>;
-extern template class ARROW_EXPORT NumericArray<UInt16Type>;
-extern template class ARROW_EXPORT NumericArray<Int32Type>;
-extern template class ARROW_EXPORT NumericArray<UInt32Type>;
-extern template class ARROW_EXPORT NumericArray<Int64Type>;
-extern template class ARROW_EXPORT NumericArray<UInt64Type>;
-extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
-extern template class ARROW_EXPORT NumericArray<FloatType>;
-extern template class ARROW_EXPORT NumericArray<DoubleType>;
-extern template class ARROW_EXPORT NumericArray<TimestampType>;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
-
-}  // namespace arrow
-
-#endif  // ARROW_TYPES_PRIMITIVE_H
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
deleted file mode 100644
index db963dfa0de5f..0000000000000
--- a/cpp/src/arrow/types/string.cc
+++ /dev/null
@@ -1,150 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/string.h"
-
-#include <cstring>
-#include <sstream>
-#include <string>
-
-#include "arrow/type.h"
-
-namespace arrow {
-
-static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
-static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
-
-BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : BinaryArray(kBinary, length, offsets, data, null_count, null_bitmap) {}
-
-BinaryArray::BinaryArray(const TypePtr& type, int32_t length,
-    const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
-    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
-    : Array(type, length, null_count, null_bitmap),
-      offset_buffer_(offsets),
-      offsets_(reinterpret_cast<const int32_t*>(offset_buffer_->data())),
-      data_buffer_(data),
-      data_(nullptr) {
-  if (data_buffer_ != nullptr) { data_ = data_buffer_->data(); }
-}
-
-Status BinaryArray::Validate() const {
-  // TODO(wesm): what to do here?
-  return Status::OK();
-}
-
-bool BinaryArray::EqualsExact(const BinaryArray& other) const {
-  if (!Array::EqualsExact(other)) { return false; }
-
-  bool equal_offsets =
-      offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
-  if (!equal_offsets) { return false; }
-
-  if (!data_buffer_ && !(other.data_buffer_)) { return true; }
-
-  return data_buffer_->Equals(*other.data_buffer_, data_buffer_->size());
-}
-
-bool BinaryArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(*static_cast<const BinaryArray*>(arr.get()));
-}
-
-bool BinaryArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  const auto other = static_cast<const BinaryArray*>(arr.get());
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    const bool is_null = IsNull(i);
-    if (is_null != arr->IsNull(o_i)) { return false; }
-    if (is_null) continue;
-    const int32_t begin_offset = offset(i);
-    const int32_t end_offset = offset(i + 1);
-    const int32_t other_begin_offset = other->offset(o_i);
-    const int32_t other_end_offset = other->offset(o_i + 1);
-    // Underlying can't be equal if the size isn't equal
-    if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
-      return false;
-    }
-
-    if (std::memcmp(data_ + begin_offset, other->data_ + other_begin_offset,
-            end_offset - begin_offset)) {
-      return false;
-    }
-  }
-  return true;
-}
-
-Status BinaryArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : BinaryArray(kString, length, offsets, data, null_count, null_bitmap) {}
-
-Status StringArray::Validate() const {
-  // TODO(emkornfield) Validate proper UTF8 code points?
-  return BinaryArray::Validate();
-}
-
-Status StringArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-// This used to be a static member variable of BinaryBuilder, but it can cause
-// valgrind to report a (spurious?) memory leak when needed in other shared
-// libraries. The problem came up while adding explicit visibility to libarrow
-// and libparquet_arrow
-static TypePtr kBinaryValueType = TypePtr(new UInt8Type());
-
-BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
-    : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, kBinaryValueType), type) {
-  byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
-}
-
-Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<Array> result;
-  RETURN_NOT_OK(ListBuilder::Finish(&result));
-
-  const auto list = std::dynamic_pointer_cast<ListArray>(result);
-  auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
-
-  *out = std::make_shared<BinaryArray>(list->length(), list->offsets(), values->data(),
-      list->null_count(), list->null_bitmap());
-  return Status::OK();
-}
-
-Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<Array> result;
-  RETURN_NOT_OK(ListBuilder::Finish(&result));
-
-  const auto list = std::dynamic_pointer_cast<ListArray>(result);
-  auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
-
-  *out = std::make_shared<StringArray>(list->length(), list->offsets(), values->data(),
-      list->null_count(), list->null_bitmap());
-  return Status::OK();
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
deleted file mode 100644
index c8752439f168c..0000000000000
--- a/cpp/src/arrow/types/string.h
+++ /dev/null
@@ -1,149 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_STRING_H
-#define ARROW_TYPES_STRING_H
-
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/type.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Buffer;
-class MemoryPool;
-
-class ARROW_EXPORT BinaryArray : public Array {
- public:
-  using TypeClass = BinaryType;
-
-  BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
-  // Constructor that allows sub-classes/builders to propagate there logical type up the
-  // class hierarchy.
-  BinaryArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
-  // Return the pointer to the given elements bytes
-  // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
-  // pointer + offset
-  const uint8_t* GetValue(int i, int32_t* out_length) const {
-    DCHECK(out_length);
-    const int32_t pos = offsets_[i];
-    *out_length = offsets_[i + 1] - pos;
-    return data_ + pos;
-  }
-
-  std::shared_ptr<Buffer> data() const { return data_buffer_; }
-  std::shared_ptr<Buffer> offsets() const { return offset_buffer_; }
-
-  const int32_t* raw_offsets() const { return offsets_; }
-
-  int32_t offset(int i) const { return offsets_[i]; }
-
-  // Neither of these functions will perform boundschecking
-  int32_t value_offset(int i) const { return offsets_[i]; }
-  int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
-
-  bool EqualsExact(const BinaryArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const ArrayPtr& arr) const override;
-
-  Status Validate() const override;
-
-  Status Accept(ArrayVisitor* visitor) const override;
-
- private:
-  std::shared_ptr<Buffer> offset_buffer_;
-  const int32_t* offsets_;
-
-  std::shared_ptr<Buffer> data_buffer_;
-  const uint8_t* data_;
-};
-
-class ARROW_EXPORT StringArray : public BinaryArray {
- public:
-  using TypeClass = StringType;
-
-  StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
-  // Construct a std::string
-  // TODO: std::bad_alloc possibility
-  std::string GetString(int i) const {
-    int32_t nchars;
-    const uint8_t* str = GetValue(i, &nchars);
-    return std::string(reinterpret_cast<const char*>(str), nchars);
-  }
-
-  Status Validate() const override;
-
-  Status Accept(ArrayVisitor* visitor) const override;
-};
-
-// BinaryBuilder : public ListBuilder
-class ARROW_EXPORT BinaryBuilder : public ListBuilder {
- public:
-  explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type);
-  virtual ~BinaryBuilder() {}
-
-  Status Append(const uint8_t* value, int32_t length) {
-    RETURN_NOT_OK(ListBuilder::Append());
-    return byte_builder_->Append(value, length);
-  }
-
-  Status Append(const char* value, int32_t length) {
-    return Append(reinterpret_cast<const uint8_t*>(value), length);
-  }
-
-  Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
-
-  Status Finish(std::shared_ptr<Array>* out) override;
-
- protected:
-  UInt8Builder* byte_builder_;
-};
-
-// String builder
-class ARROW_EXPORT StringBuilder : public BinaryBuilder {
- public:
-  explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
-      : BinaryBuilder(pool, type) {}
-
-  using BinaryBuilder::Append;
-
-  Status Finish(std::shared_ptr<Array>* out) override;
-
-  Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
-};
-
-}  // namespace arrow
-
-#endif  // ARROW_TYPES_STRING_H
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
deleted file mode 100644
index 0e0db23544bf7..0000000000000
--- a/cpp/src/arrow/types/struct.cc
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/struct.h"
-
-#include <sstream>
-
-namespace arrow {
-
-bool StructArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  if (null_count_ != arr->null_count()) { return false; }
-  return RangeEquals(0, length_, 0, arr);
-}
-
-bool StructArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (Type::STRUCT != arr->type_enum()) { return false; }
-  const auto other = static_cast<StructArray*>(arr.get());
-
-  bool equal_fields = true;
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    if (IsNull(i) != arr->IsNull(o_i)) { return false; }
-    if (IsNull(i)) continue;
-    for (size_t j = 0; j < field_arrays_.size(); ++j) {
-      // TODO: really we should be comparing stretches of non-null data rather
-      // than looking at one value at a time.
-      equal_fields = field(j)->RangeEquals(i, i + 1, o_i, other->field(j));
-      if (!equal_fields) { return false; }
-    }
-  }
-
-  return true;
-}
-
-Status StructArray::Validate() const {
-  if (length_ < 0) { return Status::Invalid("Length was negative"); }
-
-  if (null_count() > length_) {
-    return Status::Invalid("Null count exceeds the length of this struct");
-  }
-
-  if (field_arrays_.size() > 0) {
-    // Validate fields
-    int32_t array_length = field_arrays_[0]->length();
-    size_t idx = 0;
-    for (auto it : field_arrays_) {
-      if (it->length() != array_length) {
-        std::stringstream ss;
-        ss << "Length is not equal from field " << it->type()->ToString()
-           << " at position {" << idx << "}";
-        return Status::Invalid(ss.str());
-      }
-
-      const Status child_valid = it->Validate();
-      if (!child_valid.ok()) {
-        std::stringstream ss;
-        ss << "Child array invalid: " << child_valid.ToString() << " at position {" << idx
-           << "}";
-        return Status::Invalid(ss.str());
-      }
-      ++idx;
-    }
-
-    if (array_length > 0 && array_length != length_) {
-      return Status::Invalid("Struct's length is not equal to its child arrays");
-    }
-  }
-  return Status::OK();
-}
-
-Status StructArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
-  for (size_t i = 0; i < field_builders_.size(); ++i) {
-    RETURN_NOT_OK(field_builders_[i]->Finish(&fields[i]));
-  }
-
-  *out = std::make_shared<StructArray>(type_, length_, fields, null_count_, null_bitmap_);
-
-  null_bitmap_ = nullptr;
-  capacity_ = length_ = null_count_ = 0;
-
-  return Status::OK();
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
deleted file mode 100644
index 1e2bf2d9a1223..0000000000000
--- a/cpp/src/arrow/types/struct.h
+++ /dev/null
@@ -1,116 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_STRUCT_H
-#define ARROW_TYPES_STRUCT_H
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class ARROW_EXPORT StructArray : public Array {
- public:
-  using TypeClass = StructType;
-
-  StructArray(const TypePtr& type, int32_t length, std::vector<ArrayPtr>& field_arrays,
-      int32_t null_count = 0, std::shared_ptr<Buffer> null_bitmap = nullptr)
-      : Array(type, length, null_count, null_bitmap) {
-    type_ = type;
-    field_arrays_ = field_arrays;
-  }
-
-  Status Validate() const override;
-
-  virtual ~StructArray() {}
-
-  // Return a shared pointer in case the requestor desires to share ownership
-  // with this array.
-  std::shared_ptr<Array> field(int32_t pos) const {
-    DCHECK_GT(field_arrays_.size(), 0);
-    return field_arrays_[pos];
-  }
-  const std::vector<ArrayPtr>& fields() const { return field_arrays_; }
-
-  bool EqualsExact(const StructArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const override;
-
-  Status Accept(ArrayVisitor* visitor) const override;
-
- protected:
-  // The child arrays corresponding to each field of the struct data type.
-  std::vector<ArrayPtr> field_arrays_;
-};
-
-// ---------------------------------------------------------------------------------
-// StructArray builder
-// Append, Resize and Reserve methods are acting on StructBuilder.
-// Please make sure all these methods of all child-builders' are consistently
-// called to maintain data-structure consistency.
-class ARROW_EXPORT StructBuilder : public ArrayBuilder {
- public:
-  StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-      const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders)
-      : ArrayBuilder(pool, type) {
-    field_builders_ = field_builders;
-  }
-
-  Status Finish(std::shared_ptr<Array>* out) override;
-
-  // Null bitmap is of equal length to every child field, and any zero byte
-  // will be considered as a null for that field, but users must using app-
-  // end methods or advance methods of the child builders' independently to
-  // insert data.
-  Status Append(int32_t length, const uint8_t* valid_bytes) {
-    RETURN_NOT_OK(Reserve(length));
-    UnsafeAppendToBitmap(valid_bytes, length);
-    return Status::OK();
-  }
-
-  // Append an element to the Struct. All child-builders' Append method must
-  // be called independently to maintain data-structure consistency.
-  Status Append(bool is_valid = true) {
-    RETURN_NOT_OK(Reserve(1));
-    UnsafeAppendToBitmap(is_valid);
-    return Status::OK();
-  }
-
-  Status AppendNull() { return Append(false); }
-
-  std::shared_ptr<ArrayBuilder> field_builder(int pos) const {
-    DCHECK_GT(field_builders_.size(), 0);
-    return field_builders_[pos];
-  }
-  const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders() const {
-    return field_builders_;
-  }
-
- protected:
-  std::vector<std::shared_ptr<ArrayBuilder>> field_builders_;
-};
-
-}  // namespace arrow
-
-#endif  // ARROW_TYPES_STRUCT_H
diff --git a/cpp/src/arrow/types/test-common.h b/cpp/src/arrow/types/test-common.h
deleted file mode 100644
index 6e6ab85ad4eb7..0000000000000
--- a/cpp/src/arrow/types/test-common.h
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_TEST_COMMON_H
-#define ARROW_TYPES_TEST_COMMON_H
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/util/memory-pool.h"
-
-namespace arrow {
-
-using std::unique_ptr;
-
-class TestBuilder : public ::testing::Test {
- public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-    type_ = TypePtr(new UInt8Type());
-    builder_.reset(new UInt8Builder(pool_, type_));
-    builder_nn_.reset(new UInt8Builder(pool_, type_));
-  }
-
- protected:
-  MemoryPool* pool_;
-
-  TypePtr type_;
-  unique_ptr<ArrayBuilder> builder_;
-  unique_ptr<ArrayBuilder> builder_nn_;
-};
-
-template <class T, class Builder>
-Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
-    int size, Builder* builder, ArrayPtr* out) {
-  // Append the first 1000
-  for (int i = 0; i < size; ++i) {
-    if (valid_bytes[i] > 0) {
-      RETURN_NOT_OK(builder->Append(values[i]));
-    } else {
-      RETURN_NOT_OK(builder->AppendNull());
-    }
-  }
-  return builder->Finish(out);
-}
-
-}  // namespace arrow
-
-#endif  // ARROW_TYPES_TEST_COMMON_H
diff --git a/cpp/src/arrow/types/union.cc b/cpp/src/arrow/types/union.cc
deleted file mode 100644
index cc2934b2e4adb..0000000000000
--- a/cpp/src/arrow/types/union.cc
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/union.h"
-
-#include <cstdlib>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-
-namespace arrow {}  // namespace arrow
diff --git a/cpp/src/arrow/types/union.h b/cpp/src/arrow/types/union.h
deleted file mode 100644
index 44f39cc69942b..0000000000000
--- a/cpp/src/arrow/types/union.h
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_UNION_H
-#define ARROW_TYPES_UNION_H
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-class Buffer;
-
-class UnionArray : public Array {
- protected:
-  // The data are types encoded as int16
-  Buffer* types_;
-  std::vector<std::shared_ptr<Array>> children_;
-};
-
-class DenseUnionArray : public UnionArray {
- protected:
-  Buffer* offset_buf_;
-};
-
-class SparseUnionArray : public UnionArray {};
-
-}  // namespace arrow
-
-#endif  // ARROW_TYPES_UNION_H
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 6e19730219553..8d9afccf867df 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -22,12 +22,9 @@
 # Headers: top level
 install(FILES
   bit-util.h
-  buffer.h
   logging.h
   macros.h
-  memory-pool.h
   random.h
-  status.h
   visibility.h
   DESTINATION include/arrow/util)
 
@@ -72,6 +69,3 @@ if (ARROW_BUILD_BENCHMARKS)
 endif()
 
 ADD_ARROW_TEST(bit-util-test)
-ADD_ARROW_TEST(buffer-test)
-ADD_ARROW_TEST(memory-pool-test)
-ADD_ARROW_TEST(status-test)
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 7e1cb1867171e..9c82407ecc092 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -18,9 +18,9 @@
 #include <cstring>
 #include <vector>
 
+#include "arrow/buffer.h"
+#include "arrow/status.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
 
 namespace arrow {
 
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index c034fbd977747..ac2f533c408c7 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -21,7 +21,7 @@
 #include "pyarrow/adapters/builtin.h"
 
 #include "arrow/api.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 
 #include "pyarrow/helpers.h"
 
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index adb27e83ef120..64b708695194a 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -31,7 +31,7 @@
 
 #include "arrow/api.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
 
 #include "pyarrow/common.h"
 #include "pyarrow/config.h"
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index fa875f2b9aba1..fb4d3496ac79f 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -21,8 +21,8 @@
 #include <mutex>
 #include <sstream>
 
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 
 using arrow::Status;
 
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 7f3131ef03dd8..7e3382634a781 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -19,10 +19,11 @@
 #define PYARROW_COMMON_H
 
 #include "pyarrow/config.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/macros.h"
 #include "pyarrow/visibility.h"
 
+#include "arrow/buffer.h"
+#include "arrow/util/macros.h"
+
 namespace arrow { class MemoryPool; }
 
 namespace pyarrow {
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index e6dbc12d429b0..12f5ba0bf2b49 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -21,8 +21,8 @@
 #include <cstdlib>
 
 #include "arrow/io/memory.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 
 #include "pyarrow/common.h"
 

From 7e93075cd48c5f6b1b75f9adc43ba53c831046e7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 13 Dec 2016 06:50:25 +0100
Subject: [PATCH 0229/1644] ARROW-405: Use vendored hdfs.h if not found in
 include/ in $HADOOP_HOME

Not all Hadoop distributions have their files arranged in the same way.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #237 from wesm/ARROW-405 and squashes the following commits:

3a266d3 [Wes McKinney] Use vendored hdfs.h if not found in include/ in $HADOOP_HOME
---
 cpp/src/arrow/io/CMakeLists.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index a1892a9294a78..f285180c5142a 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -56,6 +56,10 @@ if(ARROW_HDFS)
 
   if (DEFINED ENV{HADOOP_HOME})
     set(HADOOP_HOME $ENV{HADOOP_HOME})
+    if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
+      message(STATUS "Did not find hdfs.h in expected location, using vendored one")
+      set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+    endif()
   else()
     set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
   endif()

From 935279091f371716adcf18f6437244f040f98da8 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 14 Dec 2016 15:41:44 -0500
Subject: [PATCH 0230/1644] ARROW-422: IPC should depend on rapidjson_ep if
 RapidJSON is vendored

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #239 from xhochy/ARROW-422 and squashes the following commits:

1545012 [Uwe L. Korn] ARROW-422: IPC should depend on rapidjson_ep if RapidJSON is vendored
---
 cpp/CMakeLists.txt               | 2 ++
 cpp/src/arrow/ipc/CMakeLists.txt | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index adcca0e0b49e8..d288ffb5f7a81 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -609,8 +609,10 @@ if("$ENV{RAPIDJSON_HOME}" STREQUAL "")
 
   ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR)
   set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include")
+  set(RAPIDJSON_VENDORED 1)
 else()
   set(RAPIDJSON_INCLUDE_DIR "$ENV{RAPIDJSON_HOME}/include")
+  set(RAPIDJSON_VENDORED 0)
 endif()
 message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
 include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 6f401dba2495f..b1669c5f7c239 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -44,6 +44,9 @@ set(ARROW_IPC_SRCS
 add_library(arrow_ipc SHARED
   ${ARROW_IPC_SRCS}
 )
+if(RAPIDJSON_VERDORED)
+  add_dependencies(arrow_ipc rapidjson_ep)
+endif()
 if(FLATBUFFERS_VENDORED)
   add_dependencies(arrow_ipc flatbuffers_ep)
 endif()

From 063c190a5252d8f77a37ebf80efcc68b70ffacab Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 15 Dec 2016 13:15:08 -0500
Subject: [PATCH 0231/1644] ARROW-423: Define BUILD_BYPRODUCTS for CMake 3.2+

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #240 from xhochy/ARROW-423 and squashes the following commits:

4c99ba2 [Uwe L. Korn] ARROW-423: Define BUILD_BYPRODUCTS for CMake 3.2+
---
 cpp/CMakeLists.txt           | 91 ++++++++++++++++++++++++------------
 cpp/src/arrow/CMakeLists.txt |  2 -
 2 files changed, 61 insertions(+), 32 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d288ffb5f7a81..315995ce7cb97 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -488,19 +488,32 @@ if(ARROW_BUILD_TESTS)
       set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
     endif()
 
-    ExternalProject_Add(googletest_ep
-      URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-      CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON
-      # googletest doesn't define install rules, so just build in the
-      # source dir and don't try to install.  See its README for
-      # details.
-      BUILD_IN_SOURCE 1
-      INSTALL_COMMAND "")
-
     set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
     set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
     set(GTEST_STATIC_LIB "${GTEST_PREFIX}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GTEST_VENDORED 1)
+
+    if (CMAKE_VERSION VERSION_GREATER "3.2")
+      # BUILD_BYPRODUCTS is a 3.2+ feature
+      ExternalProject_Add(googletest_ep
+        URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+        CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON
+        # googletest doesn't define install rules, so just build in the
+        # source dir and don't try to install.  See its README for
+        # details.
+        BUILD_IN_SOURCE 1
+        BUILD_BYPRODUCTS "${GTEST_STATIC_LIB}"
+        INSTALL_COMMAND "")
+    else()
+      ExternalProject_Add(googletest_ep
+        URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+        CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON
+        # googletest doesn't define install rules, so just build in the
+        # source dir and don't try to install.  See its README for
+        # details.
+        BUILD_IN_SOURCE 1
+        INSTALL_COMMAND "")
+    endif()
   else()
     find_package(GTest REQUIRED)
     set(GTEST_VENDORED 0)
@@ -525,24 +538,34 @@ if(ARROW_BUILD_TESTS)
     endif()
 
     set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
-    ExternalProject_Add(gflags_ep
-      GIT_REPOSITORY https://github.com/gflags/gflags.git
-      GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
-      BUILD_IN_SOURCE 1
-      CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-                 -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
-                 -DBUILD_SHARED_LIBS=OFF
-                 -DBUILD_STATIC_LIBS=ON
-                 -DBUILD_PACKAGING=OFF
-                 -DBUILD_TESTING=OFF
-                 -BUILD_CONFIG_TESTS=OFF
-                 -DINSTALL_HEADERS=ON
-                 -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
-
     set(GFLAGS_HOME "${GFLAGS_PREFIX}")
     set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
     set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
     set(GFLAGS_VENDORED 1)
+    set(GFLAGS_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                          -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
+                          -DBUILD_SHARED_LIBS=OFF
+                          -DBUILD_STATIC_LIBS=ON
+                          -DBUILD_PACKAGING=OFF
+                          -DBUILD_TESTING=OFF
+                          -BUILD_CONFIG_TESTS=OFF
+                          -DINSTALL_HEADERS=ON
+                          -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
+    if (CMAKE_VERSION VERSION_GREATER "3.2")
+      # BUILD_BYPRODUCTS is a 3.2+ feature
+      ExternalProject_Add(gflags_ep
+        GIT_REPOSITORY https://github.com/gflags/gflags.git
+        GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+        BUILD_IN_SOURCE 1
+        BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}"
+        CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
+    else()
+      ExternalProject_Add(gflags_ep
+        GIT_REPOSITORY https://github.com/gflags/gflags.git
+        GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+        BUILD_IN_SOURCE 1
+        CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
+    endif()
   else()
     set(GFLAGS_VENDORED 0)
     find_package(GFlags REQUIRED)
@@ -570,16 +593,24 @@ if(ARROW_BUILD_BENCHMARKS)
     endif()
 
     set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install")
-    ExternalProject_Add(gbenchmark_ep
-      URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
-      CMAKE_ARGS
-        "-DCMAKE_BUILD_TYPE=Release"
-        "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}"
-        "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}")
-
     set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include")
     set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GBENCHMARK_VENDORED 1)
+    set(GBENCHMARK_CMAKE_ARGS
+          "-DCMAKE_BUILD_TYPE=Release"
+          "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}"
+          "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}")
+    if (CMAKE_VERSION VERSION_GREATER "3.2")
+      # BUILD_BYPRODUCTS is a 3.2+ feature
+      ExternalProject_Add(gbenchmark_ep
+        URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
+        BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}"
+        CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
+    else()
+      ExternalProject_Add(gbenchmark_ep
+        URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
+        CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
+    endif()
   else()
     find_package(GBenchmark REQUIRED)
     set(GBENCHMARK_VENDORED 0)
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 7d7bc29f4abd8..b8500ab264f80 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -37,8 +37,6 @@ install(FILES
 # Unit tests
 #######################################
 
-set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
-
 ADD_ARROW_TEST(array-test)
 ADD_ARROW_TEST(array-decimal-test)
 ADD_ARROW_TEST(array-list-test)

From cfb544de2efb260bc0737460e056a0d2a5295e6a Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 15 Dec 2016 13:16:01 -0500
Subject: [PATCH 0232/1644] ARROW-425: Add private API to get python Table from
 a C++ object

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #241 from xhochy/pyarrow-private-api and squashes the following commits:

dc9b814 [Uwe L. Korn] ARROW-425: Add private API to get python Table from a C++ object
---
 python/pyarrow/table.pxd |  2 ++
 python/pyarrow/table.pyx |  4 ++++
 python/setup.py          | 11 +++++++++++
 3 files changed, 17 insertions(+)

diff --git a/python/pyarrow/table.pxd b/python/pyarrow/table.pxd
index 79c9ae3b0a194..df3687ddf9761 100644
--- a/python/pyarrow/table.pxd
+++ b/python/pyarrow/table.pxd
@@ -57,3 +57,5 @@ cdef class RecordBatch:
 
     cdef init(self, const shared_ptr[CRecordBatch]& table)
     cdef _check_nullptr(self)
+
+cdef api object table_from_ctable(const shared_ptr[CTable]& ctable)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 0a9805cfdf427..333686f810ea8 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -687,5 +687,9 @@ cdef class Table:
         return (self.num_rows, self.num_columns)
 
 
+cdef api object table_from_ctable(const shared_ptr[CTable]& ctable):
+    cdef Table table = Table()
+    table.init(ctable)
+    return table
 
 from_pandas_dataframe = Table.from_pandas
diff --git a/python/setup.py b/python/setup.py
index 0f6bbda6ec3aa..5acdca34a0882 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -204,6 +204,10 @@ def _run_cmake(self):
             shutil.move(self.get_ext_built(name), ext_path)
             self._found_names.append(name)
 
+            if os.path.exists(self.get_ext_built_api_header(name)):
+                shutil.move(self.get_ext_built_api_header(name),
+                            pjoin(os.path.dirname(ext_path), name + '_api.h'))
+
         os.chdir(saved_cwd)
 
     def _failure_permitted(self, name):
@@ -225,6 +229,13 @@ def _get_cmake_ext_path(self, name):
         filename = name + suffix
         return pjoin(package_dir, filename)
 
+    def get_ext_built_api_header(self, name):
+        if sys.platform == 'win32':
+            head, tail = os.path.split(name)
+            return pjoin(head, tail + "_api.h")
+        else:
+            return pjoin(name + "_api.h")
+
     def get_ext_built(self, name):
         if sys.platform == 'win32':
             head, tail = os.path.split(name)

From a2ead2f646baad78de01fcb1b90f710fa1eae70b Mon Sep 17 00:00:00 2001
From: Mohamed Zenadi <mohamed@zenadi.com>
Date: Sat, 17 Dec 2016 18:05:04 -0500
Subject: [PATCH 0233/1644] ARROW-380: [Java] optimize null count when
 serializing vectors

I added `getNullCount()` to the `Accessor` interface. I don't know if this is the best way to achieve this. Hence, we'll have both ValueCount and NullCount immediately accessible from the accessor.

Author: Mohamed Zenadi <mohamed@zenadi.com>

Closes #207 from zeapo/ARROW-380 and squashes the following commits:

27c0342 [Mohamed Zenadi] implement missing getNullCount implementation for NullableMapVector
9ff3355 [Mohamed Zenadi] implement the base case of getNullCount()
ad3f24a [Mohamed Zenadi] the used size is not the same as the allocated size
e858432 [Mohamed Zenadi] use the valueCount as basis for counting nulls rather than allocated bytes
0530c85 [Mohamed Zenadi] test the null count byte by byte and the odd length case
95667d3 [Mohamed Zenadi] fix the comment
b12a2a5 [Mohamed Zenadi] fix wrong value returned by the method
f264250 [Mohamed Zenadi] use getNullCount() rather than isNull
baca69c [Mohamed Zenadi] Add methods to count the number null values in the vector
---
 .../apache/arrow/vector/BaseValueVector.java  | 12 +++++
 .../org/apache/arrow/vector/BitVector.java    | 22 ++++++++++
 .../org/apache/arrow/vector/ValueVector.java  |  5 +++
 .../apache/arrow/vector/VectorUnloader.java   | 10 +----
 .../org/apache/arrow/vector/ZeroVector.java   |  5 +++
 .../arrow/vector/complex/ListVector.java      |  5 +++
 .../vector/complex/NullableMapVector.java     |  5 +++
 .../apache/arrow/vector/TestValueVector.java  | 44 +++++++++++++++++++
 8 files changed, 99 insertions(+), 9 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 884cdf0910b8e..2a61403c0dcbe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -72,6 +72,18 @@ protected BaseAccessor() { }
     public boolean isNull(int index) {
       return false;
     }
+
+    @Override
+    // override this in case your implementation is faster, see BitVector
+    public int getNullCount() {
+      int nullCount = 0;
+      for (int i = 0; i < getValueCount(); i++) {
+        if (isNull(i)) {
+          nullCount ++;
+        }
+      }
+      return nullCount;
+    }
   }
 
   public abstract static class BaseMutator implements ValueVector.Mutator {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 26eeafd51d900..9beabcbe46bcc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -379,6 +379,28 @@ public final void get(int index, NullableBitHolder holder) {
       holder.isSet = 1;
       holder.value = get(index);
     }
+
+    /**
+     * Get the number nulls, this correspond to the number of bits set to 0 in the vector
+     * @return the number of bits set to 0
+     */
+    @Override
+    public final int getNullCount() {
+      int count = 0;
+      int sizeInBytes = getSizeFromCount(valueCount);
+
+      for (int i = 0; i < sizeInBytes; ++i) {
+        byte byteValue = data.getByte(i);
+        // Java uses two's complement binary representation, hence 11111111_b which is -1 when converted to Int
+        // will have 32bits set to 1. Masking the MSB and then adding it back solves the issue.
+        count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
+      }
+      int nullCount = (sizeInBytes * 8) - count;
+      // if the valueCount is not a multiple of 8, the bits on the right were counted as null bits
+      int remainder = valueCount % 8;
+      nullCount -= remainder == 0 ? 0 : 8 - remainder;
+      return nullCount;
+    }
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 5b24a41850d75..ff7b94c34d80d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -180,6 +180,11 @@ interface Accessor {
      * Returns true if the value at the given index is null, false otherwise.
      */
     boolean isNull(int index);
+
+    /**
+     * Returns the number of null values
+     */
+    int getNullCount();
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index e2462180ffadc..92d8cb045ae31 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -60,15 +60,7 @@ public ArrowRecordBatch getRecordBatch() {
 
   private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
     Accessor accessor = vector.getAccessor();
-    int nullCount = 0;
-    // TODO: should not have to do that
-    // we can do that a lot more efficiently (for example with Long.bitCount(i))
-    for (int i = 0; i < accessor.getValueCount(); i++) {
-      if (accessor.isNull(i)) {
-        nullCount ++;
-      }
-    }
-    nodes.add(new ArrowFieldNode(accessor.getValueCount(), nullCount));
+    nodes.add(new ArrowFieldNode(accessor.getValueCount(), accessor.getNullCount()));
     List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
     List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
     if (fieldBuffers.size() != expectedBuffers.size()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index c2482adefecfb..e163b4fa9398f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -69,6 +69,11 @@ public int getValueCount() {
     public boolean isNull(int index) {
       return true;
     }
+
+    @Override
+    public int getNullCount() {
+      return 0;
+    }
   };
 
   private final Mutator defaultMutator = new Mutator() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 461bdbcda1b52..074b0aa7e58fa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -310,6 +310,11 @@ public Object getObject(int index) {
     public boolean isNull(int index) {
       return bits.getAccessor().get(index) == 0;
     }
+
+    @Override
+    public int getNullCount() {
+      return bits.getAccessor().getNullCount();
+    }
   }
 
   public class Mutator extends BaseRepeatedMutator {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index f0ddf2727e9ea..5fa35307ab683 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -203,6 +203,11 @@ public void get(int index, ComplexHolder holder) {
       super.get(index, holder);
     }
 
+    @Override
+    public int getNullCount() {
+      return bits.getAccessor().getNullCount();
+    }
+
     @Override
     public boolean isNull(int index) {
       return isSet(index) == 0;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 124452e96ee42..b33919b2790fc 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -288,6 +288,7 @@ public void testBitVector() {
     try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
       final BitVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
+      m.setValueCount(1024);
 
       // Put and set a few values
       m.set(0, 1);
@@ -295,12 +296,16 @@ public void testBitVector() {
       m.set(100, 0);
       m.set(1022, 1);
 
+      m.setValueCount(1024);
+
       final BitVector.Accessor accessor = vector.getAccessor();
       assertEquals(1, accessor.get(0));
       assertEquals(0, accessor.get(1));
       assertEquals(0, accessor.get(100));
       assertEquals(1, accessor.get(1022));
 
+      assertEquals(1022, accessor.getNullCount());
+
       // test setting the same value twice
       m.set(0, 1);
       m.set(0, 1);
@@ -315,8 +320,47 @@ public void testBitVector() {
       assertEquals(0, accessor.get(0));
       assertEquals(1, accessor.get(1));
 
+      // should not change
+      assertEquals(1022, accessor.getNullCount());
+
       // Ensure unallocated space returns 0
       assertEquals(0, accessor.get(3));
+
+      // unset the previously set bits
+      m.set(1, 0);
+      m.set(1022, 0);
+      // this should set all the array to 0
+      assertEquals(1024, accessor.getNullCount());
+
+      // set all the array to 1
+      for (int i = 0; i < 1024; ++i) {
+        assertEquals(1024 - i, accessor.getNullCount());
+        m.set(i, 1);
+      }
+
+      assertEquals(0, accessor.getNullCount());
+
+      vector.allocateNew(1015);
+      m.setValueCount(1015);
+
+      // ensure it has been zeroed
+      assertEquals(1015, accessor.getNullCount());
+
+      m.set(0, 1);
+      m.set(1014, 1); // ensure that the last item of the last byte is allocated
+
+      assertEquals(1013, accessor.getNullCount());
+
+      vector.zeroVector();
+      assertEquals(1015, accessor.getNullCount());
+
+      // set all the array to 1
+      for (int i = 0; i < 1015; ++i) {
+        assertEquals(1015 - i, accessor.getNullCount());
+        m.set(i, 1);
+      }
+
+      assertEquals(0, accessor.getNullCount());
     }
   }
 

From c369709c4f8157cb5e6c8121e1e613b104305aed Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 19 Dec 2016 11:47:32 -0500
Subject: [PATCH 0234/1644] ARROW-426: Python: Conversion from pyarrow.Array to
 a Python list

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #242 from xhochy/ARROW-426 and squashes the following commits:

10739ac [Uwe L. Korn] ARROW-426: Python: Conversion from pyarrow.Array to a Python list
---
 python/pyarrow/array.pyx                     |  6 ++++++
 python/pyarrow/scalar.pyx                    |  4 +++-
 python/pyarrow/table.pyx                     | 15 +++++++++++++++
 python/pyarrow/tests/test_column.py          |  1 +
 python/pyarrow/tests/test_convert_builtin.py | 13 +++++++++++--
 5 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 6c862751fc218..d44212f4aed63 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -167,6 +167,12 @@ cdef class Array:
 
         return PyObject_to_object(np_arr)
 
+    def to_pylist(self):
+        """
+        Convert to an list of native Python objects.
+        """
+        return [x.as_py() for x in self]
+
 
 cdef class NullArray(Array):
     pass
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 0d391e5f26b3e..c2d20e460c37c 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -194,7 +194,9 @@ cdef object box_arrow_scalar(DataType type,
                              const shared_ptr[CArray]& sp_array,
                              int index):
     cdef ArrayValue val
-    if sp_array.get().IsNull(index):
+    if type.type.type == Type_NA:
+        return NA
+    elif sp_array.get().IsNull(index):
         return NA
     else:
         val = _scalar_classes[type.type.type]()
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 333686f810ea8..2f7d4309e4518 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -108,6 +108,15 @@ cdef class ChunkedArray:
         for i in range(self.num_chunks):
             yield self.chunk(i)
 
+    def to_pylist(self):
+        """
+        Convert to a list of native Python objects.
+        """
+        result = []
+        for i in range(self.num_chunks):
+            result += self.chunk(i).to_pylist()
+        return result
+
 
 cdef class Column:
     """
@@ -143,6 +152,12 @@ cdef class Column:
 
         return pd.Series(PyObject_to_object(arr), name=self.name)
 
+    def to_pylist(self):
+        """
+        Convert to a list of native Python objects.
+        """
+        return self.data.to_pylist()
+
     cdef _check_nullptr(self):
         if self.column == NULL:
             raise ReferenceError("Column object references a NULL pointer."
diff --git a/python/pyarrow/tests/test_column.py b/python/pyarrow/tests/test_column.py
index b62f58236e073..32202cb5a9ad8 100644
--- a/python/pyarrow/tests/test_column.py
+++ b/python/pyarrow/tests/test_column.py
@@ -35,6 +35,7 @@ def test_basics(self):
         assert column.length() == 5
         assert len(column) == 5
         assert column.shape == (5,)
+        assert column.to_pylist() == [-10, -5, 0, 5, 10]
 
     def test_pandas(self):
         data = [
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 8937f8db6941f..34371b0bdd7c9 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -22,28 +22,34 @@
 class TestConvertList(unittest.TestCase):
 
     def test_boolean(self):
-        arr = pyarrow.from_pylist([True, None, False, None])
+        expected = [True, None, False, None]
+        arr = pyarrow.from_pylist(expected)
         assert len(arr) == 4
         assert arr.null_count == 2
         assert arr.type == pyarrow.bool_()
+        assert arr.to_pylist() == expected
 
     def test_empty_list(self):
         arr = pyarrow.from_pylist([])
         assert len(arr) == 0
         assert arr.null_count == 0
         assert arr.type == pyarrow.null()
+        assert arr.to_pylist() == []
 
     def test_all_none(self):
         arr = pyarrow.from_pylist([None, None])
         assert len(arr) == 2
         assert arr.null_count == 2
         assert arr.type == pyarrow.null()
+        assert arr.to_pylist() == [None, None]
 
     def test_integer(self):
-        arr = pyarrow.from_pylist([1, None, 3, None])
+        expected = [1, None, 3, None]
+        arr = pyarrow.from_pylist(expected)
         assert len(arr) == 4
         assert arr.null_count == 2
         assert arr.type == pyarrow.int64()
+        assert arr.to_pylist() == expected
 
     def test_garbage_collection(self):
         import gc
@@ -62,6 +68,7 @@ def test_double(self):
         assert len(arr) == 6
         assert arr.null_count == 3
         assert arr.type == pyarrow.double()
+        assert arr.to_pylist() == data
 
     def test_string(self):
         data = ['foo', b'bar', None, 'arrow']
@@ -69,6 +76,7 @@ def test_string(self):
         assert len(arr) == 4
         assert arr.null_count == 1
         assert arr.type == pyarrow.string()
+        assert arr.to_pylist() == ['foo', 'bar', None, 'arrow']
 
     def test_mixed_nesting_levels(self):
         pyarrow.from_pylist([1, 2, None])
@@ -90,3 +98,4 @@ def test_list_of_int(self):
         assert len(arr) == 4
         assert arr.null_count == 1
         assert arr.type == pyarrow.list_(pyarrow.int64())
+        assert arr.to_pylist() == data

From 68e39c6868d449f10243707ca1a7513aaa29761f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 19 Dec 2016 21:11:25 +0100
Subject: [PATCH 0235/1644] ARROW-429: Revert ARROW-379 until git-archive
 issues are resolved

These changes are resulting in GitHub producing archive tarballs with non-deterministic contents.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #243 from wesm/ARROW-429 and squashes the following commits:

49f6edb [Wes McKinney] Revert "ARROW-379: Use setuptools_scm for Python versioning"
---
 dev/release/00-prepare.sh  |  5 +++++
 python/.git_archival.txt   |  1 -
 python/.gitattributes      |  1 -
 python/pyarrow/__init__.py | 10 ++--------
 python/setup.cfg           | 20 --------------------
 python/setup.py            | 23 +++++++++++++++++++++--
 6 files changed, 28 insertions(+), 32 deletions(-)
 delete mode 100644 python/.git_archival.txt
 delete mode 100644 python/.gitattributes
 delete mode 100644 python/setup.cfg

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
index 00af5e7768161..3423a3e6c5bf9 100644
--- a/dev/release/00-prepare.sh
+++ b/dev/release/00-prepare.sh
@@ -43,4 +43,9 @@ mvn release:prepare -Dtag=${tag} -DreleaseVersion=${version} -DautoVersionSubmod
 
 cd -
 
+cd "${SOURCE_DIR}/../../python"
+sed -i "s/VERSION = '[^']*'/VERSION = '${version}'/g" setup.py
+sed -i "s/ISRELEASED = False/ISRELEASED = True/g" setup.py
+cd -
+
 echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
diff --git a/python/.git_archival.txt b/python/.git_archival.txt
deleted file mode 100644
index 95cb3eea4e336..0000000000000
--- a/python/.git_archival.txt
+++ /dev/null
@@ -1 +0,0 @@
-ref-names: $Format:%D$
diff --git a/python/.gitattributes b/python/.gitattributes
deleted file mode 100644
index 00a7b00c94e08..0000000000000
--- a/python/.gitattributes
+++ /dev/null
@@ -1 +0,0 @@
-.git_archival.txt  export-subst
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 5af93fb5865de..b9d386195b436 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -17,14 +17,6 @@
 
 # flake8: noqa
 
-from pkg_resources import get_distribution, DistributionNotFound
-try:
-    __version__ = get_distribution(__name__).version
-except DistributionNotFound:
-   # package is not installed
-   pass
-
-
 import pyarrow.config
 
 from pyarrow.array import (Array,
@@ -50,3 +42,5 @@
                             DataType, Field, Schema, schema)
 
 from pyarrow.table import Column, RecordBatch, Table, from_pandas_dataframe
+
+from pyarrow.version import version as __version__
diff --git a/python/setup.cfg b/python/setup.cfg
deleted file mode 100644
index caae3e081b6ca..0000000000000
--- a/python/setup.cfg
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[build_sphinx]
-source-dir = doc/
-build-dir  = doc/_build
diff --git a/python/setup.py b/python/setup.py
index 5acdca34a0882..5f448f7d50784 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -42,9 +42,27 @@
 if Cython.__version__ < '0.19.1':
     raise Exception('Please upgrade to Cython 0.19.1 or newer')
 
+VERSION = '0.1.0'
+ISRELEASED = False
+
+if not ISRELEASED:
+    VERSION += '.dev'
+
 setup_dir = os.path.abspath(os.path.dirname(__file__))
 
 
+def write_version_py(filename=os.path.join(setup_dir, 'pyarrow/version.py')):
+    a = open(filename, 'w')
+    file_content = "\n".join(["",
+                              "# THIS FILE IS GENERATED FROM SETUP.PY",
+                              "version = '%(version)s'",
+                              "isrelease = '%(isrelease)s'"])
+
+    a.write(file_content % {'version': VERSION,
+                            'isrelease': str(ISRELEASED)})
+    a.close()
+
+
 class clean(_clean):
 
     def run(self):
@@ -254,12 +272,15 @@ def get_outputs(self):
         return [self._get_cmake_ext_path(name)
                 for name in self.get_names()]
 
+write_version_py()
+
 DESC = """\
 Python library for Apache Arrow"""
 
 setup(
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
+    version=VERSION,
     zip_safe=False,
     package_data={'pyarrow': ['*.pxd', '*.pyx']},
     # Dummy extension to trigger build_ext
@@ -269,8 +290,6 @@ def get_outputs(self):
         'clean': clean,
         'build_ext': build_ext
     },
-    use_scm_version = {"root": "..", "relative_to": __file__},
-    setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'],
     install_requires=['cython >= 0.23', 'numpy >= 1.9', 'six >= 1.0.0'],
     description=DESC,
     license='Apache License, Version 2.0',

From cfde4607df453e4b97560e64caff744fb3ba3d1f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 19 Dec 2016 18:26:17 -0500
Subject: [PATCH 0236/1644] ARROW-243: [C++] Add option to switch between
 libhdfs and libhdfs3 when creating HdfsClient

Closes #108

Some users will not have a full Java Hadoop distribution and may wish to use the libhdfs3 package from Pivotal (https://github.com/Pivotal-Data-Attic/pivotalrd-libhdfs3), part of Apache HAWQ (incubating).

In C++, you can switch by setting:

```c++
HdfsConnectionConfig conf;
conf.driver = HdfsDriver::LIBHDFS3;
```

In Python, you can run:

```python
con = arrow.io.HdfsClient.connect(..., driver='libhdfs3')
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #244 from wesm/ARROW-243 and squashes the following commits:

7ae197a [Wes McKinney] Refactor HdfsClient code to support both libhdfs and libhdfs3 at runtime. Add driver option to Python interface
---
 cpp/src/arrow/io/CMakeLists.txt         |   2 +-
 cpp/src/arrow/io/hdfs-internal.cc       | 590 ++++++++++++++++++++++++
 cpp/src/arrow/io/hdfs-internal.h        | 203 ++++++++
 cpp/src/arrow/io/hdfs.cc                | 102 ++--
 cpp/src/arrow/io/hdfs.h                 |   6 +-
 cpp/src/arrow/io/io-hdfs-test.cc        | 211 +++++----
 cpp/src/arrow/io/libhdfs_shim.cc        | 582 -----------------------
 python/.gitignore                       |   1 +
 python/pyarrow/includes/libarrow_io.pxd |   8 +-
 python/pyarrow/io.pyx                   |  45 +-
 python/pyarrow/tests/test_hdfs.py       | 161 +++----
 11 files changed, 1109 insertions(+), 802 deletions(-)
 create mode 100644 cpp/src/arrow/io/hdfs-internal.cc
 create mode 100644 cpp/src/arrow/io/hdfs-internal.h
 delete mode 100644 cpp/src/arrow/io/libhdfs_shim.cc

diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index f285180c5142a..e2b6496cc3f87 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -75,7 +75,7 @@ if(ARROW_HDFS)
 
   set(ARROW_HDFS_SRCS
     hdfs.cc
-    libhdfs_shim.cc)
+    hdfs-internal.cc)
 
   set_property(SOURCE ${ARROW_HDFS_SRCS}
     APPEND_STRING PROPERTY
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
new file mode 100644
index 0000000000000..7094785de02a0
--- /dev/null
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -0,0 +1,590 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// This shim interface to libhdfs (for runtime shared library loading) has been
+// adapted from the SFrame project, released under the ASF-compatible 3-clause
+// BSD license
+//
+// Using this required having the $JAVA_HOME and $HADOOP_HOME environment
+// variables set, so that libjvm and libhdfs can be located easily
+
+// Copyright (C) 2015 Dato, Inc.
+// All rights reserved.
+//
+// This software may be modified and distributed under the terms
+// of the BSD license. See the LICENSE file for details.
+
+#ifdef HAS_HADOOP
+
+#ifndef _WIN32
+#include <dlfcn.h>
+#else
+#include <windows.h>
+#include <winsock2.h>
+
+// TODO(wesm): address when/if we add windows support
+// #include <util/syserr_reporting.hpp>
+#endif
+
+extern "C" {
+#include <hdfs.h>
+}
+
+#include <iostream>
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include <boost/filesystem.hpp>  // NOLINT
+
+#include "arrow/io/hdfs-internal.h"
+#include "arrow/status.h"
+#include "arrow/util/visibility.h"
+
+namespace fs = boost::filesystem;
+
+#ifndef _WIN32
+static void* libjvm_handle = NULL;
+#else
+static HINSTANCE libjvm_handle = NULL;
+#endif
+/*
+ * All the shim pointers
+ */
+
+// Helper functions for dlopens
+static std::vector<fs::path> get_potential_libjvm_paths();
+static std::vector<fs::path> get_potential_libhdfs_paths();
+static std::vector<fs::path> get_potential_libhdfs3_paths();
+static arrow::Status try_dlopen(std::vector<fs::path> potential_paths, const char* name,
+#ifndef _WIN32
+    void*& out_handle);
+#else
+    HINSTANCE& out_handle);
+#endif
+
+static std::vector<fs::path> get_potential_libhdfs_paths() {
+  std::vector<fs::path> libhdfs_potential_paths;
+  std::string file_name;
+
+// OS-specific file name
+#ifdef __WIN32
+  file_name = "hdfs.dll";
+#elif __APPLE__
+  file_name = "libhdfs.dylib";
+#else
+  file_name = "libhdfs.so";
+#endif
+
+  // Common paths
+  std::vector<fs::path> search_paths = {fs::path(""), fs::path(".")};
+
+  // Path from environment variable
+  const char* hadoop_home = std::getenv("HADOOP_HOME");
+  if (hadoop_home != nullptr) {
+    auto path = fs::path(hadoop_home) / "lib/native";
+    search_paths.push_back(path);
+  }
+
+  const char* libhdfs_dir = std::getenv("ARROW_LIBHDFS_DIR");
+  if (libhdfs_dir != nullptr) { search_paths.push_back(fs::path(libhdfs_dir)); }
+
+  // All paths with file name
+  for (auto& path : search_paths) {
+    libhdfs_potential_paths.push_back(path / file_name);
+  }
+
+  return libhdfs_potential_paths;
+}
+
+static std::vector<fs::path> get_potential_libhdfs3_paths() {
+  std::vector<fs::path> potential_paths;
+  std::string file_name;
+
+// OS-specific file name
+#ifdef __WIN32
+  file_name = "hdfs3.dll";
+#elif __APPLE__
+  file_name = "libhdfs3.dylib";
+#else
+  file_name = "libhdfs3.so";
+#endif
+
+  // Common paths
+  std::vector<fs::path> search_paths = {fs::path(""), fs::path(".")};
+
+  const char* libhdfs3_dir = std::getenv("ARROW_LIBHDFS3_DIR");
+  if (libhdfs3_dir != nullptr) { search_paths.push_back(fs::path(libhdfs3_dir)); }
+
+  // All paths with file name
+  for (auto& path : search_paths) {
+    potential_paths.push_back(path / file_name);
+  }
+
+  return potential_paths;
+}
+
+static std::vector<fs::path> get_potential_libjvm_paths() {
+  std::vector<fs::path> libjvm_potential_paths;
+
+  std::vector<fs::path> search_prefixes;
+  std::vector<fs::path> search_suffixes;
+  std::string file_name;
+
+// From heuristics
+#ifdef __WIN32
+  search_prefixes = {""};
+  search_suffixes = {"/jre/bin/server", "/bin/server"};
+  file_name = "jvm.dll";
+#elif __APPLE__
+  search_prefixes = {""};
+  search_suffixes = {"", "/jre/lib/server"};
+  file_name = "libjvm.dylib";
+
+// SFrame uses /usr/libexec/java_home to find JAVA_HOME; for now we are
+// expecting users to set an environment variable
+#else
+  search_prefixes = {
+      "/usr/lib/jvm/default-java",                // ubuntu / debian distros
+      "/usr/lib/jvm/java",                        // rhel6
+      "/usr/lib/jvm",                             // centos6
+      "/usr/lib64/jvm",                           // opensuse 13
+      "/usr/local/lib/jvm/default-java",          // alt ubuntu / debian distros
+      "/usr/local/lib/jvm/java",                  // alt rhel6
+      "/usr/local/lib/jvm",                       // alt centos6
+      "/usr/local/lib64/jvm",                     // alt opensuse 13
+      "/usr/local/lib/jvm/java-7-openjdk-amd64",  // alt ubuntu / debian distros
+      "/usr/lib/jvm/java-7-openjdk-amd64",        // alt ubuntu / debian distros
+      "/usr/local/lib/jvm/java-6-openjdk-amd64",  // alt ubuntu / debian distros
+      "/usr/lib/jvm/java-6-openjdk-amd64",        // alt ubuntu / debian distros
+      "/usr/lib/jvm/java-7-oracle",               // alt ubuntu
+      "/usr/lib/jvm/java-8-oracle",               // alt ubuntu
+      "/usr/lib/jvm/java-6-oracle",               // alt ubuntu
+      "/usr/local/lib/jvm/java-7-oracle",         // alt ubuntu
+      "/usr/local/lib/jvm/java-8-oracle",         // alt ubuntu
+      "/usr/local/lib/jvm/java-6-oracle",         // alt ubuntu
+      "/usr/lib/jvm/default",                     // alt centos
+      "/usr/java/latest",                         // alt centos
+  };
+  search_suffixes = {"/jre/lib/amd64/server"};
+  file_name = "libjvm.so";
+#endif
+  // From direct environment variable
+  char* env_value = NULL;
+  if ((env_value = getenv("JAVA_HOME")) != NULL) {
+    search_prefixes.insert(search_prefixes.begin(), env_value);
+  }
+
+  // Generate cross product between search_prefixes, search_suffixes, and file_name
+  for (auto& prefix : search_prefixes) {
+    for (auto& suffix : search_suffixes) {
+      auto path = (fs::path(prefix) / fs::path(suffix) / fs::path(file_name));
+      libjvm_potential_paths.push_back(path);
+    }
+  }
+
+  return libjvm_potential_paths;
+}
+
+#ifndef _WIN32
+static arrow::Status try_dlopen(
+    std::vector<fs::path> potential_paths, const char* name, void*& out_handle) {
+  std::vector<std::string> error_messages;
+
+  for (auto& i : potential_paths) {
+    i.make_preferred();
+    out_handle = dlopen(i.native().c_str(), RTLD_NOW | RTLD_LOCAL);
+
+    if (out_handle != NULL) {
+      // std::cout << "Loaded " << i << std::endl;
+      break;
+    } else {
+      const char* err_msg = dlerror();
+      if (err_msg != NULL) {
+        error_messages.push_back(std::string(err_msg));
+      } else {
+        error_messages.push_back(std::string(" returned NULL"));
+      }
+    }
+  }
+
+  if (out_handle == NULL) {
+    std::stringstream ss;
+    ss << "Unable to load " << name;
+    return arrow::Status::IOError(ss.str());
+  }
+
+  return arrow::Status::OK();
+}
+
+#else
+static arrow::Status try_dlopen(
+    std::vector<fs::path> potential_paths, const char* name, HINSTANCE& out_handle) {
+  std::vector<std::string> error_messages;
+
+  for (auto& i : potential_paths) {
+    i.make_preferred();
+    out_handle = LoadLibrary(i.string().c_str());
+
+    if (out_handle != NULL) {
+      break;
+    } else {
+      // error_messages.push_back(get_last_err_str(GetLastError()));
+    }
+  }
+
+  if (out_handle == NULL) {
+    std::stringstream ss;
+    ss << "Unable to load " << name;
+    return arrow::Status::IOError(ss.str());
+  }
+
+  return arrow::Status::OK();
+}
+#endif  // _WIN32
+
+static inline void* GetLibrarySymbol(void* handle, const char* symbol) {
+  if (handle == NULL) return NULL;
+#ifndef _WIN32
+  return dlsym(handle, symbol);
+#else
+
+  void* ret = reinterpret_cast<void*>(GetProcAddress(handle, symbol));
+  if (ret == NULL) {
+    // logstream(LOG_INFO) << "GetProcAddress error: "
+    //                     << get_last_err_str(GetLastError()) << std::endl;
+  }
+  return ret;
+#endif
+}
+
+#define GET_SYMBOL_REQUIRED(SHIM, SYMBOL_NAME)                         \
+  do {                                                                 \
+    if (!SHIM->SYMBOL_NAME) {                                          \
+      *reinterpret_cast<void**>(&SHIM->SYMBOL_NAME) =                  \
+          GetLibrarySymbol(SHIM->handle, "" #SYMBOL_NAME);             \
+    }                                                                  \
+    if (!SHIM->SYMBOL_NAME)                                            \
+      return Status::IOError("Getting symbol " #SYMBOL_NAME "failed"); \
+  } while (0)
+
+#define GET_SYMBOL(SHIM, SYMBOL_NAME)                    \
+  if (!SHIM->SYMBOL_NAME) {                              \
+    *reinterpret_cast<void**>(&SHIM->SYMBOL_NAME) =      \
+        GetLibrarySymbol(SHIM->handle, "" #SYMBOL_NAME); \
+  }
+
+namespace arrow {
+namespace io {
+
+static LibHdfsShim libhdfs_shim;
+static LibHdfsShim libhdfs3_shim;
+
+hdfsBuilder* LibHdfsShim::NewBuilder(void) {
+  return this->hdfsNewBuilder();
+}
+
+void LibHdfsShim::BuilderSetNameNode(hdfsBuilder* bld, const char* nn) {
+  this->hdfsBuilderSetNameNode(bld, nn);
+}
+
+void LibHdfsShim::BuilderSetNameNodePort(hdfsBuilder* bld, tPort port) {
+  this->hdfsBuilderSetNameNodePort(bld, port);
+}
+
+void LibHdfsShim::BuilderSetUserName(hdfsBuilder* bld, const char* userName) {
+  this->hdfsBuilderSetUserName(bld, userName);
+}
+
+void LibHdfsShim::BuilderSetKerbTicketCachePath(
+    hdfsBuilder* bld, const char* kerbTicketCachePath) {
+  this->hdfsBuilderSetKerbTicketCachePath(bld, kerbTicketCachePath);
+}
+
+hdfsFS LibHdfsShim::BuilderConnect(hdfsBuilder* bld) {
+  return this->hdfsBuilderConnect(bld);
+}
+
+int LibHdfsShim::Disconnect(hdfsFS fs) {
+  return this->hdfsDisconnect(fs);
+}
+
+hdfsFile LibHdfsShim::OpenFile(hdfsFS fs, const char* path, int flags, int bufferSize,
+    short replication, tSize blocksize) {  // NOLINT
+  return this->hdfsOpenFile(fs, path, flags, bufferSize, replication, blocksize);
+}
+
+int LibHdfsShim::CloseFile(hdfsFS fs, hdfsFile file) {
+  return this->hdfsCloseFile(fs, file);
+}
+
+int LibHdfsShim::Exists(hdfsFS fs, const char* path) {
+  return this->hdfsExists(fs, path);
+}
+
+int LibHdfsShim::Seek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
+  return this->hdfsSeek(fs, file, desiredPos);
+}
+
+tOffset LibHdfsShim::Tell(hdfsFS fs, hdfsFile file) {
+  return this->hdfsTell(fs, file);
+}
+
+tSize LibHdfsShim::Read(hdfsFS fs, hdfsFile file, void* buffer, tSize length) {
+  return this->hdfsRead(fs, file, buffer, length);
+}
+
+bool LibHdfsShim::HasPread() {
+  GET_SYMBOL(this, hdfsPread);
+  return this->hdfsPread != nullptr;
+}
+
+tSize LibHdfsShim::Pread(
+    hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length) {
+  GET_SYMBOL(this, hdfsPread);
+  return this->hdfsPread(fs, file, position, buffer, length);
+}
+
+tSize LibHdfsShim::Write(hdfsFS fs, hdfsFile file, const void* buffer, tSize length) {
+  return this->hdfsWrite(fs, file, buffer, length);
+}
+
+int LibHdfsShim::Flush(hdfsFS fs, hdfsFile file) {
+  return this->hdfsFlush(fs, file);
+}
+
+int LibHdfsShim::Available(hdfsFS fs, hdfsFile file) {
+  GET_SYMBOL(this, hdfsAvailable);
+  if (this->hdfsAvailable)
+    return this->hdfsAvailable(fs, file);
+  else
+    return 0;
+}
+
+int LibHdfsShim::Copy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) {
+  GET_SYMBOL(this, hdfsCopy);
+  if (this->hdfsCopy)
+    return this->hdfsCopy(srcFS, src, dstFS, dst);
+  else
+    return 0;
+}
+
+int LibHdfsShim::Move(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) {
+  GET_SYMBOL(this, hdfsMove);
+  if (this->hdfsMove)
+    return this->hdfsMove(srcFS, src, dstFS, dst);
+  else
+    return 0;
+}
+
+int LibHdfsShim::Delete(hdfsFS fs, const char* path, int recursive) {
+  return this->hdfsDelete(fs, path, recursive);
+}
+
+int LibHdfsShim::Rename(hdfsFS fs, const char* oldPath, const char* newPath) {
+  GET_SYMBOL(this, hdfsRename);
+  if (this->hdfsRename)
+    return this->hdfsRename(fs, oldPath, newPath);
+  else
+    return 0;
+}
+
+char* LibHdfsShim::GetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) {
+  GET_SYMBOL(this, hdfsGetWorkingDirectory);
+  if (this->hdfsGetWorkingDirectory) {
+    return this->hdfsGetWorkingDirectory(fs, buffer, bufferSize);
+  } else {
+    return NULL;
+  }
+}
+
+int LibHdfsShim::SetWorkingDirectory(hdfsFS fs, const char* path) {
+  GET_SYMBOL(this, hdfsSetWorkingDirectory);
+  if (this->hdfsSetWorkingDirectory) {
+    return this->hdfsSetWorkingDirectory(fs, path);
+  } else {
+    return 0;
+  }
+}
+
+int LibHdfsShim::CreateDirectory(hdfsFS fs, const char* path) {
+  return this->hdfsCreateDirectory(fs, path);
+}
+
+int LibHdfsShim::SetReplication(hdfsFS fs, const char* path, int16_t replication) {
+  GET_SYMBOL(this, hdfsSetReplication);
+  if (this->hdfsSetReplication) {
+    return this->hdfsSetReplication(fs, path, replication);
+  } else {
+    return 0;
+  }
+}
+
+hdfsFileInfo* LibHdfsShim::ListDirectory(hdfsFS fs, const char* path, int* numEntries) {
+  return this->hdfsListDirectory(fs, path, numEntries);
+}
+
+hdfsFileInfo* LibHdfsShim::GetPathInfo(hdfsFS fs, const char* path) {
+  return this->hdfsGetPathInfo(fs, path);
+}
+
+void LibHdfsShim::FreeFileInfo(hdfsFileInfo* hdfsFileInfo, int numEntries) {
+  this->hdfsFreeFileInfo(hdfsFileInfo, numEntries);
+}
+
+char*** LibHdfsShim::GetHosts(
+    hdfsFS fs, const char* path, tOffset start, tOffset length) {
+  GET_SYMBOL(this, hdfsGetHosts);
+  if (this->hdfsGetHosts) {
+    return this->hdfsGetHosts(fs, path, start, length);
+  } else {
+    return NULL;
+  }
+}
+
+void LibHdfsShim::FreeHosts(char*** blockHosts) {
+  GET_SYMBOL(this, hdfsFreeHosts);
+  if (this->hdfsFreeHosts) { this->hdfsFreeHosts(blockHosts); }
+}
+
+tOffset LibHdfsShim::GetDefaultBlockSize(hdfsFS fs) {
+  GET_SYMBOL(this, hdfsGetDefaultBlockSize);
+  if (this->hdfsGetDefaultBlockSize) {
+    return this->hdfsGetDefaultBlockSize(fs);
+  } else {
+    return 0;
+  }
+}
+
+tOffset LibHdfsShim::GetCapacity(hdfsFS fs) {
+  return this->hdfsGetCapacity(fs);
+}
+
+tOffset LibHdfsShim::GetUsed(hdfsFS fs) {
+  return this->hdfsGetUsed(fs);
+}
+
+int LibHdfsShim::Chown(
+    hdfsFS fs, const char* path, const char* owner, const char* group) {
+  GET_SYMBOL(this, hdfsChown);
+  if (this->hdfsChown) {
+    return this->hdfsChown(fs, path, owner, group);
+  } else {
+    return 0;
+  }
+}
+
+int LibHdfsShim::Chmod(hdfsFS fs, const char* path, short mode) {  // NOLINT
+  GET_SYMBOL(this, hdfsChmod);
+  if (this->hdfsChmod) {
+    return this->hdfsChmod(fs, path, mode);
+  } else {
+    return 0;
+  }
+}
+
+int LibHdfsShim::Utime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
+  GET_SYMBOL(this, hdfsUtime);
+  if (this->hdfsUtime) {
+    return this->hdfsUtime(fs, path, mtime, atime);
+  } else {
+    return 0;
+  }
+}
+
+Status LibHdfsShim::GetRequiredSymbols() {
+  GET_SYMBOL_REQUIRED(this, hdfsNewBuilder);
+  GET_SYMBOL_REQUIRED(this, hdfsBuilderSetNameNode);
+  GET_SYMBOL_REQUIRED(this, hdfsBuilderSetNameNodePort);
+  GET_SYMBOL_REQUIRED(this, hdfsBuilderSetUserName);
+  GET_SYMBOL_REQUIRED(this, hdfsBuilderSetKerbTicketCachePath);
+  GET_SYMBOL_REQUIRED(this, hdfsBuilderConnect);
+  GET_SYMBOL_REQUIRED(this, hdfsCreateDirectory);
+  GET_SYMBOL_REQUIRED(this, hdfsDelete);
+  GET_SYMBOL_REQUIRED(this, hdfsDisconnect);
+  GET_SYMBOL_REQUIRED(this, hdfsExists);
+  GET_SYMBOL_REQUIRED(this, hdfsFreeFileInfo);
+  GET_SYMBOL_REQUIRED(this, hdfsGetCapacity);
+  GET_SYMBOL_REQUIRED(this, hdfsGetUsed);
+  GET_SYMBOL_REQUIRED(this, hdfsGetPathInfo);
+  GET_SYMBOL_REQUIRED(this, hdfsListDirectory);
+
+  // File methods
+  GET_SYMBOL_REQUIRED(this, hdfsCloseFile);
+  GET_SYMBOL_REQUIRED(this, hdfsFlush);
+  GET_SYMBOL_REQUIRED(this, hdfsOpenFile);
+  GET_SYMBOL_REQUIRED(this, hdfsRead);
+  GET_SYMBOL_REQUIRED(this, hdfsSeek);
+  GET_SYMBOL_REQUIRED(this, hdfsTell);
+  GET_SYMBOL_REQUIRED(this, hdfsWrite);
+
+  return Status::OK();
+}
+
+Status ARROW_EXPORT ConnectLibHdfs(LibHdfsShim** driver) {
+  static std::mutex lock;
+  std::lock_guard<std::mutex> guard(lock);
+
+  LibHdfsShim* shim = &libhdfs_shim;
+
+  static bool shim_attempted = false;
+  if (!shim_attempted) {
+    shim_attempted = true;
+
+    shim->Initialize();
+
+    std::vector<fs::path> libjvm_potential_paths = get_potential_libjvm_paths();
+    RETURN_NOT_OK(try_dlopen(libjvm_potential_paths, "libjvm", libjvm_handle));
+
+    std::vector<fs::path> libhdfs_potential_paths = get_potential_libhdfs_paths();
+    RETURN_NOT_OK(try_dlopen(libhdfs_potential_paths, "libhdfs", shim->handle));
+  } else if (shim->handle == nullptr) {
+    return Status::IOError("Prior attempt to load libhdfs failed");
+  }
+
+  *driver = shim;
+  return shim->GetRequiredSymbols();
+}
+
+Status ARROW_EXPORT ConnectLibHdfs3(LibHdfsShim** driver) {
+  static std::mutex lock;
+  std::lock_guard<std::mutex> guard(lock);
+
+  LibHdfsShim* shim = &libhdfs3_shim;
+
+  static bool shim_attempted = false;
+  if (!shim_attempted) {
+    shim_attempted = true;
+
+    shim->Initialize();
+
+    std::vector<fs::path> libhdfs3_potential_paths = get_potential_libhdfs3_paths();
+    RETURN_NOT_OK(try_dlopen(libhdfs3_potential_paths, "libhdfs3", shim->handle));
+  } else if (shim->handle == nullptr) {
+    return Status::IOError("Prior attempt to load libhdfs3 failed");
+  }
+
+  *driver = shim;
+  return shim->GetRequiredSymbols();
+}
+
+}  // namespace io
+}  // namespace arrow
+
+#endif  // HAS_HADOOP
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
new file mode 100644
index 0000000000000..0ff118a8f57e7
--- /dev/null
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -0,0 +1,203 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IO_HDFS_INTERNAL
+#define ARROW_IO_HDFS_INTERNAL
+
+#include <hdfs.h>
+
+namespace arrow {
+
+class Status;
+
+namespace io {
+
+// NOTE(wesm): cpplint does not like use of short and other imprecise C types
+struct LibHdfsShim {
+#ifndef _WIN32
+  void* handle;
+#else
+  HINSTANCE handle;
+#endif
+
+  hdfsBuilder* (*hdfsNewBuilder)(void);
+  void (*hdfsBuilderSetNameNode)(hdfsBuilder* bld, const char* nn);
+  void (*hdfsBuilderSetNameNodePort)(hdfsBuilder* bld, tPort port);
+  void (*hdfsBuilderSetUserName)(hdfsBuilder* bld, const char* userName);
+  void (*hdfsBuilderSetKerbTicketCachePath)(
+      hdfsBuilder* bld, const char* kerbTicketCachePath);
+  hdfsFS (*hdfsBuilderConnect)(hdfsBuilder* bld);
+
+  int (*hdfsDisconnect)(hdfsFS fs);
+
+  hdfsFile (*hdfsOpenFile)(hdfsFS fs, const char* path, int flags, int bufferSize,
+      short replication, tSize blocksize);  // NOLINT
+
+  int (*hdfsCloseFile)(hdfsFS fs, hdfsFile file);
+  int (*hdfsExists)(hdfsFS fs, const char* path);
+  int (*hdfsSeek)(hdfsFS fs, hdfsFile file, tOffset desiredPos);
+  tOffset (*hdfsTell)(hdfsFS fs, hdfsFile file);
+  tSize (*hdfsRead)(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
+  tSize (*hdfsPread)(
+      hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length);
+  tSize (*hdfsWrite)(hdfsFS fs, hdfsFile file, const void* buffer, tSize length);
+  int (*hdfsFlush)(hdfsFS fs, hdfsFile file);
+  int (*hdfsAvailable)(hdfsFS fs, hdfsFile file);
+  int (*hdfsCopy)(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+  int (*hdfsMove)(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+  int (*hdfsDelete)(hdfsFS fs, const char* path, int recursive);
+  int (*hdfsRename)(hdfsFS fs, const char* oldPath, const char* newPath);
+  char* (*hdfsGetWorkingDirectory)(hdfsFS fs, char* buffer, size_t bufferSize);
+  int (*hdfsSetWorkingDirectory)(hdfsFS fs, const char* path);
+  int (*hdfsCreateDirectory)(hdfsFS fs, const char* path);
+  int (*hdfsSetReplication)(hdfsFS fs, const char* path, int16_t replication);
+  hdfsFileInfo* (*hdfsListDirectory)(hdfsFS fs, const char* path, int* numEntries);
+  hdfsFileInfo* (*hdfsGetPathInfo)(hdfsFS fs, const char* path);
+  void (*hdfsFreeFileInfo)(hdfsFileInfo* hdfsFileInfo, int numEntries);
+  char*** (*hdfsGetHosts)(hdfsFS fs, const char* path, tOffset start, tOffset length);
+  void (*hdfsFreeHosts)(char*** blockHosts);
+  tOffset (*hdfsGetDefaultBlockSize)(hdfsFS fs);
+  tOffset (*hdfsGetCapacity)(hdfsFS fs);
+  tOffset (*hdfsGetUsed)(hdfsFS fs);
+  int (*hdfsChown)(hdfsFS fs, const char* path, const char* owner, const char* group);
+  int (*hdfsChmod)(hdfsFS fs, const char* path, short mode);  // NOLINT
+  int (*hdfsUtime)(hdfsFS fs, const char* path, tTime mtime, tTime atime);
+
+  void Initialize() {
+    this->handle = nullptr;
+    this->hdfsNewBuilder = nullptr;
+    this->hdfsBuilderSetNameNode = nullptr;
+    this->hdfsBuilderSetNameNodePort = nullptr;
+    this->hdfsBuilderSetUserName = nullptr;
+    this->hdfsBuilderSetKerbTicketCachePath = nullptr;
+    this->hdfsBuilderConnect = nullptr;
+    this->hdfsDisconnect = nullptr;
+    this->hdfsOpenFile = nullptr;
+    this->hdfsCloseFile = nullptr;
+    this->hdfsExists = nullptr;
+    this->hdfsSeek = nullptr;
+    this->hdfsTell = nullptr;
+    this->hdfsRead = nullptr;
+    this->hdfsPread = nullptr;
+    this->hdfsWrite = nullptr;
+    this->hdfsFlush = nullptr;
+    this->hdfsAvailable = nullptr;
+    this->hdfsCopy = nullptr;
+    this->hdfsMove = nullptr;
+    this->hdfsDelete = nullptr;
+    this->hdfsRename = nullptr;
+    this->hdfsGetWorkingDirectory = nullptr;
+    this->hdfsSetWorkingDirectory = nullptr;
+    this->hdfsCreateDirectory = nullptr;
+    this->hdfsSetReplication = nullptr;
+    this->hdfsListDirectory = nullptr;
+    this->hdfsGetPathInfo = nullptr;
+    this->hdfsFreeFileInfo = nullptr;
+    this->hdfsGetHosts = nullptr;
+    this->hdfsFreeHosts = nullptr;
+    this->hdfsGetDefaultBlockSize = nullptr;
+    this->hdfsGetCapacity = nullptr;
+    this->hdfsGetUsed = nullptr;
+    this->hdfsChown = nullptr;
+    this->hdfsChmod = nullptr;
+    this->hdfsUtime = nullptr;
+  }
+
+  hdfsBuilder* NewBuilder(void);
+
+  void BuilderSetNameNode(hdfsBuilder* bld, const char* nn);
+
+  void BuilderSetNameNodePort(hdfsBuilder* bld, tPort port);
+
+  void BuilderSetUserName(hdfsBuilder* bld, const char* userName);
+
+  void BuilderSetKerbTicketCachePath(hdfsBuilder* bld, const char* kerbTicketCachePath);
+
+  hdfsFS BuilderConnect(hdfsBuilder* bld);
+
+  int Disconnect(hdfsFS fs);
+
+  hdfsFile OpenFile(hdfsFS fs, const char* path, int flags, int bufferSize,
+      short replication, tSize blocksize);  // NOLINT
+
+  int CloseFile(hdfsFS fs, hdfsFile file);
+
+  int Exists(hdfsFS fs, const char* path);
+
+  int Seek(hdfsFS fs, hdfsFile file, tOffset desiredPos);
+
+  tOffset Tell(hdfsFS fs, hdfsFile file);
+
+  tSize Read(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
+
+  bool HasPread();
+
+  tSize Pread(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length);
+
+  tSize Write(hdfsFS fs, hdfsFile file, const void* buffer, tSize length);
+
+  int Flush(hdfsFS fs, hdfsFile file);
+
+  int Available(hdfsFS fs, hdfsFile file);
+
+  int Copy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+
+  int Move(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+
+  int Delete(hdfsFS fs, const char* path, int recursive);
+
+  int Rename(hdfsFS fs, const char* oldPath, const char* newPath);
+
+  char* GetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize);
+
+  int SetWorkingDirectory(hdfsFS fs, const char* path);
+
+  int CreateDirectory(hdfsFS fs, const char* path);
+
+  int SetReplication(hdfsFS fs, const char* path, int16_t replication);
+
+  hdfsFileInfo* ListDirectory(hdfsFS fs, const char* path, int* numEntries);
+
+  hdfsFileInfo* GetPathInfo(hdfsFS fs, const char* path);
+
+  void FreeFileInfo(hdfsFileInfo* hdfsFileInfo, int numEntries);
+
+  char*** GetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length);
+
+  void FreeHosts(char*** blockHosts);
+
+  tOffset GetDefaultBlockSize(hdfsFS fs);
+  tOffset GetCapacity(hdfsFS fs);
+
+  tOffset GetUsed(hdfsFS fs);
+
+  int Chown(hdfsFS fs, const char* path, const char* owner, const char* group);
+
+  int Chmod(hdfsFS fs, const char* path, short mode);  // NOLINT
+
+  int Utime(hdfsFS fs, const char* path, tTime mtime, tTime atime);
+
+  Status GetRequiredSymbols();
+};
+
+Status ConnectLibHdfs(LibHdfsShim** driver);
+Status ConnectLibHdfs3(LibHdfsShim** driver);
+
+}  // namespace io
+}  // namespace arrow
+
+#endif  // ARROW_IO_HDFS_INTERNAL
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index b8e212026b11c..44e503ff11302 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -23,6 +23,7 @@
 #include <string>
 
 #include "arrow/buffer.h"
+#include "arrow/io/hdfs-internal.h"
 #include "arrow/io/hdfs.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
@@ -59,21 +60,23 @@ static constexpr int kDefaultHdfsBufferSize = 1 << 16;
 
 class HdfsAnyFileImpl {
  public:
-  void set_members(const std::string& path, hdfsFS fs, hdfsFile handle) {
+  void set_members(
+      const std::string& path, LibHdfsShim* driver, hdfsFS fs, hdfsFile handle) {
     path_ = path;
+    driver_ = driver;
     fs_ = fs;
     file_ = handle;
     is_open_ = true;
   }
 
   Status Seek(int64_t position) {
-    int ret = hdfsSeek(fs_, file_, position);
+    int ret = driver_->Seek(fs_, file_, position);
     CHECK_FAILURE(ret, "seek");
     return Status::OK();
   }
 
   Status Tell(int64_t* offset) {
-    int64_t ret = hdfsTell(fs_, file_);
+    int64_t ret = driver_->Tell(fs_, file_);
     CHECK_FAILURE(ret, "tell");
     *offset = ret;
     return Status::OK();
@@ -84,6 +87,8 @@ class HdfsAnyFileImpl {
  protected:
   std::string path_;
 
+  LibHdfsShim* driver_;
+
   // These are pointers in libhdfs, so OK to copy
   hdfsFS fs_;
   hdfsFile file_;
@@ -98,7 +103,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
 
   Status Close() {
     if (is_open_) {
-      int ret = hdfsCloseFile(fs_, file_);
+      int ret = driver_->CloseFile(fs_, file_);
       CHECK_FAILURE(ret, "CloseFile");
       is_open_ = false;
     }
@@ -106,8 +111,14 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
   }
 
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
-    tSize ret = hdfsPread(fs_, file_, static_cast<tOffset>(position),
-        reinterpret_cast<void*>(buffer), nbytes);
+    tSize ret;
+    if (driver_->HasPread()) {
+      ret = driver_->Pread(fs_, file_, static_cast<tOffset>(position),
+          reinterpret_cast<void*>(buffer), nbytes);
+    } else {
+      RETURN_NOT_OK(Seek(position));
+      return Read(nbytes, bytes_read, buffer);
+    }
     RETURN_NOT_OK(CheckReadResult(ret));
     *bytes_read = ret;
     return Status::OK();
@@ -129,7 +140,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
     int64_t total_bytes = 0;
     while (total_bytes < nbytes) {
-      tSize ret = hdfsRead(fs_, file_, reinterpret_cast<void*>(buffer + total_bytes),
+      tSize ret = driver_->Read(fs_, file_, reinterpret_cast<void*>(buffer + total_bytes),
           std::min<int64_t>(buffer_size_, nbytes - total_bytes));
       RETURN_NOT_OK(CheckReadResult(ret));
       total_bytes += ret;
@@ -153,11 +164,11 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
   }
 
   Status GetSize(int64_t* size) {
-    hdfsFileInfo* entry = hdfsGetPathInfo(fs_, path_.c_str());
+    hdfsFileInfo* entry = driver_->GetPathInfo(fs_, path_.c_str());
     if (entry == nullptr) { return Status::IOError("HDFS: GetPathInfo failed"); }
 
     *size = entry->mSize;
-    hdfsFreeFileInfo(entry, 1);
+    driver_->FreeFileInfo(entry, 1);
     return Status::OK();
   }
 
@@ -227,9 +238,9 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
 
   Status Close() {
     if (is_open_) {
-      int ret = hdfsFlush(fs_, file_);
+      int ret = driver_->Flush(fs_, file_);
       CHECK_FAILURE(ret, "Flush");
-      ret = hdfsCloseFile(fs_, file_);
+      ret = driver_->CloseFile(fs_, file_);
       CHECK_FAILURE(ret, "CloseFile");
       is_open_ = false;
     }
@@ -237,7 +248,7 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
   }
 
   Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
-    tSize ret = hdfsWrite(fs_, file_, reinterpret_cast<const void*>(buffer), nbytes);
+    tSize ret = driver_->Write(fs_, file_, reinterpret_cast<const void*>(buffer), nbytes);
     CHECK_FAILURE(ret, "Write");
     *bytes_written = ret;
     return Status::OK();
@@ -297,17 +308,25 @@ class HdfsClient::HdfsClientImpl {
   HdfsClientImpl() {}
 
   Status Connect(const HdfsConnectionConfig* config) {
-    RETURN_NOT_OK(ConnectLibHdfs());
+    if (config->driver == HdfsDriver::LIBHDFS3) {
+      RETURN_NOT_OK(ConnectLibHdfs3(&driver_));
+    } else {
+      RETURN_NOT_OK(ConnectLibHdfs(&driver_));
+    }
 
     // connect to HDFS with the builder object
-    hdfsBuilder* builder = hdfsNewBuilder();
-    if (!config->host.empty()) { hdfsBuilderSetNameNode(builder, config->host.c_str()); }
-    hdfsBuilderSetNameNodePort(builder, config->port);
-    if (!config->user.empty()) { hdfsBuilderSetUserName(builder, config->user.c_str()); }
+    hdfsBuilder* builder = driver_->NewBuilder();
+    if (!config->host.empty()) {
+      driver_->BuilderSetNameNode(builder, config->host.c_str());
+    }
+    driver_->BuilderSetNameNodePort(builder, config->port);
+    if (!config->user.empty()) {
+      driver_->BuilderSetUserName(builder, config->user.c_str());
+    }
     if (!config->kerb_ticket.empty()) {
-      hdfsBuilderSetKerbTicketCachePath(builder, config->kerb_ticket.c_str());
+      driver_->BuilderSetKerbTicketCachePath(builder, config->kerb_ticket.c_str());
     }
-    fs_ = hdfsBuilderConnect(builder);
+    fs_ = driver_->BuilderConnect(builder);
 
     if (fs_ == nullptr) { return Status::IOError("HDFS connection failed"); }
     namenode_host_ = config->host;
@@ -319,19 +338,19 @@ class HdfsClient::HdfsClientImpl {
   }
 
   Status CreateDirectory(const std::string& path) {
-    int ret = hdfsCreateDirectory(fs_, path.c_str());
+    int ret = driver_->CreateDirectory(fs_, path.c_str());
     CHECK_FAILURE(ret, "create directory");
     return Status::OK();
   }
 
   Status Delete(const std::string& path, bool recursive) {
-    int ret = hdfsDelete(fs_, path.c_str(), static_cast<int>(recursive));
+    int ret = driver_->Delete(fs_, path.c_str(), static_cast<int>(recursive));
     CHECK_FAILURE(ret, "delete");
     return Status::OK();
   }
 
   Status Disconnect() {
-    int ret = hdfsDisconnect(fs_);
+    int ret = driver_->Disconnect(fs_);
     CHECK_FAILURE(ret, "hdfsFS::Disconnect");
     return Status::OK();
   }
@@ -339,38 +358,38 @@ class HdfsClient::HdfsClientImpl {
   bool Exists(const std::string& path) {
     // hdfsExists does not distinguish between RPC failure and the file not
     // existing
-    int ret = hdfsExists(fs_, path.c_str());
+    int ret = driver_->Exists(fs_, path.c_str());
     return ret == 0;
   }
 
   Status GetCapacity(int64_t* nbytes) {
-    tOffset ret = hdfsGetCapacity(fs_);
+    tOffset ret = driver_->GetCapacity(fs_);
     CHECK_FAILURE(ret, "GetCapacity");
     *nbytes = ret;
     return Status::OK();
   }
 
   Status GetUsed(int64_t* nbytes) {
-    tOffset ret = hdfsGetUsed(fs_);
+    tOffset ret = driver_->GetUsed(fs_);
     CHECK_FAILURE(ret, "GetUsed");
     *nbytes = ret;
     return Status::OK();
   }
 
   Status GetPathInfo(const std::string& path, HdfsPathInfo* info) {
-    hdfsFileInfo* entry = hdfsGetPathInfo(fs_, path.c_str());
+    hdfsFileInfo* entry = driver_->GetPathInfo(fs_, path.c_str());
 
     if (entry == nullptr) { return Status::IOError("HDFS: GetPathInfo failed"); }
 
     SetPathInfo(entry, info);
-    hdfsFreeFileInfo(entry, 1);
+    driver_->FreeFileInfo(entry, 1);
 
     return Status::OK();
   }
 
   Status ListDirectory(const std::string& path, std::vector<HdfsPathInfo>* listing) {
     int num_entries = 0;
-    hdfsFileInfo* entries = hdfsListDirectory(fs_, path.c_str(), &num_entries);
+    hdfsFileInfo* entries = driver_->ListDirectory(fs_, path.c_str(), &num_entries);
 
     if (entries == nullptr) {
       // If the directory is empty, entries is NULL but errno is 0. Non-zero
@@ -391,14 +410,14 @@ class HdfsClient::HdfsClientImpl {
     }
 
     // Free libhdfs file info
-    hdfsFreeFileInfo(entries, num_entries);
+    driver_->FreeFileInfo(entries, num_entries);
 
     return Status::OK();
   }
 
   Status OpenReadable(const std::string& path, int32_t buffer_size,
       std::shared_ptr<HdfsReadableFile>* file) {
-    hdfsFile handle = hdfsOpenFile(fs_, path.c_str(), O_RDONLY, buffer_size, 0, 0);
+    hdfsFile handle = driver_->OpenFile(fs_, path.c_str(), O_RDONLY, buffer_size, 0, 0);
 
     if (handle == nullptr) {
       // TODO(wesm): determine cause of failure
@@ -409,7 +428,7 @@ class HdfsClient::HdfsClientImpl {
 
     // std::make_shared does not work with private ctors
     *file = std::shared_ptr<HdfsReadableFile>(new HdfsReadableFile());
-    (*file)->impl_->set_members(path, fs_, handle);
+    (*file)->impl_->set_members(path, driver_, fs_, handle);
     (*file)->impl_->set_buffer_size(buffer_size);
 
     return Status::OK();
@@ -421,7 +440,7 @@ class HdfsClient::HdfsClientImpl {
     int flags = O_WRONLY;
     if (append) flags |= O_APPEND;
 
-    hdfsFile handle = hdfsOpenFile(
+    hdfsFile handle = driver_->OpenFile(
         fs_, path.c_str(), flags, buffer_size, replication, default_block_size);
 
     if (handle == nullptr) {
@@ -433,18 +452,20 @@ class HdfsClient::HdfsClientImpl {
 
     // std::make_shared does not work with private ctors
     *file = std::shared_ptr<HdfsOutputStream>(new HdfsOutputStream());
-    (*file)->impl_->set_members(path, fs_, handle);
+    (*file)->impl_->set_members(path, driver_, fs_, handle);
 
     return Status::OK();
   }
 
   Status Rename(const std::string& src, const std::string& dst) {
-    int ret = hdfsRename(fs_, src.c_str(), dst.c_str());
+    int ret = driver_->Rename(fs_, src.c_str(), dst.c_str());
     CHECK_FAILURE(ret, "Rename");
     return Status::OK();
   }
 
  private:
+  LibHdfsShim* driver_;
+
   std::string namenode_host_;
   std::string user_;
   int port_;
@@ -530,5 +551,18 @@ Status HdfsClient::Rename(const std::string& src, const std::string& dst) {
   return impl_->Rename(src, dst);
 }
 
+// ----------------------------------------------------------------------
+// Allow public API users to check whether we are set up correctly
+
+Status HaveLibHdfs() {
+  LibHdfsShim* driver;
+  return ConnectLibHdfs(&driver);
+}
+
+Status HaveLibHdfs3() {
+  LibHdfsShim* driver;
+  return ConnectLibHdfs3(&driver);
+}
+
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 1c76f15c397ce..5cc783e475967 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -56,11 +56,14 @@ struct HdfsPathInfo {
   int16_t permissions;
 };
 
+enum class HdfsDriver : char { LIBHDFS, LIBHDFS3 };
+
 struct HdfsConnectionConfig {
   std::string host;
   int port;
   std::string user;
   std::string kerb_ticket;
+  HdfsDriver driver;
 };
 
 class ARROW_EXPORT HdfsClient : public FileSystemClient {
@@ -218,7 +221,8 @@ class ARROW_EXPORT HdfsOutputStream : public OutputStream {
   DISALLOW_COPY_AND_ASSIGN(HdfsOutputStream);
 };
 
-Status ARROW_EXPORT ConnectLibHdfs();
+Status ARROW_EXPORT HaveLibHdfs();
+Status ARROW_EXPORT HaveLibHdfs3();
 
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index e07eaa3d1b487..4ef47b8babe6e 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -24,6 +24,7 @@
 
 #include <boost/filesystem.hpp>  // NOLINT
 
+#include "arrow/io/hdfs-internal.h"
 #include "arrow/io/hdfs.h"
 #include "arrow/status.h"
 #include "arrow/test-util.h"
@@ -37,6 +38,7 @@ std::vector<uint8_t> RandomData(int64_t size) {
   return buffer;
 }
 
+template <typename DRIVER>
 class TestHdfsClient : public ::testing::Test {
  public:
   Status MakeScratchDir() {
@@ -71,15 +73,34 @@ class TestHdfsClient : public ::testing::Test {
     return ss.str();
   }
 
- protected:
   // Set up shared state between unit tests
-  static void SetUpTestCase() {
-    if (!ConnectLibHdfs().ok()) {
-      std::cout << "Loading libhdfs failed, skipping tests gracefully" << std::endl;
-      return;
+  void SetUp() {
+    LibHdfsShim* driver_shim;
+
+    client_ = nullptr;
+    scratch_dir_ =
+        boost::filesystem::unique_path("/tmp/arrow-hdfs/scratch-%%%%").native();
+
+    loaded_driver_ = false;
+
+    Status msg;
+
+    if (DRIVER::type == HdfsDriver::LIBHDFS) {
+      msg = ConnectLibHdfs(&driver_shim);
+      if (!msg.ok()) {
+        std::cout << "Loading libhdfs failed, skipping tests gracefully" << std::endl;
+        return;
+      }
+    } else {
+      msg = ConnectLibHdfs3(&driver_shim);
+      if (!msg.ok()) {
+        std::cout << "Loading libhdfs3 failed, skipping tests gracefully. "
+                  << msg.ToString() << std::endl;
+        return;
+      }
     }
 
-    loaded_libhdfs_ = true;
+    loaded_driver_ = true;
 
     const char* host = std::getenv("ARROW_HDFS_TEST_HOST");
     const char* port = std::getenv("ARROW_HDFS_TEST_PORT");
@@ -94,151 +115,159 @@ class TestHdfsClient : public ::testing::Test {
     ASSERT_OK(HdfsClient::Connect(&conf_, &client_));
   }
 
-  static void TearDownTestCase() {
+  void TearDown() {
     if (client_) {
-      EXPECT_OK(client_->Delete(scratch_dir_, true));
+      if (client_->Exists(scratch_dir_)) {
+        EXPECT_OK(client_->Delete(scratch_dir_, true));
+      }
       EXPECT_OK(client_->Disconnect());
     }
   }
 
-  static bool loaded_libhdfs_;
+  HdfsConnectionConfig conf_;
+  bool loaded_driver_;
 
   // Resources shared amongst unit tests
-  static HdfsConnectionConfig conf_;
-  static std::string scratch_dir_;
-  static std::shared_ptr<HdfsClient> client_;
+  std::string scratch_dir_;
+  std::shared_ptr<HdfsClient> client_;
 };
 
-bool TestHdfsClient::loaded_libhdfs_ = false;
-HdfsConnectionConfig TestHdfsClient::conf_ = HdfsConnectionConfig();
+#define SKIP_IF_NO_DRIVER()                                  \
+  if (!this->loaded_driver_) {                               \
+    std::cout << "Driver not loaded, skipping" << std::endl; \
+    return;                                                  \
+  }
 
-std::string TestHdfsClient::scratch_dir_ =
-    boost::filesystem::unique_path("/tmp/arrow-hdfs/scratch-%%%%").native();
+struct JNIDriver {
+  static HdfsDriver type;
+};
 
-std::shared_ptr<HdfsClient> TestHdfsClient::client_ = nullptr;
+struct PivotalDriver {
+  static HdfsDriver type;
+};
 
-#define SKIP_IF_NO_LIBHDFS()                          \
-  if (!loaded_libhdfs_) {                             \
-    std::cout << "No libhdfs, skipping" << std::endl; \
-    return;                                           \
-  }
+HdfsDriver JNIDriver::type = HdfsDriver::LIBHDFS;
+HdfsDriver PivotalDriver::type = HdfsDriver::LIBHDFS3;
+
+typedef ::testing::Types<JNIDriver, PivotalDriver> DriverTypes;
+TYPED_TEST_CASE(TestHdfsClient, DriverTypes);
 
-TEST_F(TestHdfsClient, ConnectsAgain) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, ConnectsAgain) {
+  SKIP_IF_NO_DRIVER();
 
   std::shared_ptr<HdfsClient> client;
-  ASSERT_OK(HdfsClient::Connect(&conf_, &client));
+  ASSERT_OK(HdfsClient::Connect(&this->conf_, &client));
   ASSERT_OK(client->Disconnect());
 }
 
-TEST_F(TestHdfsClient, CreateDirectory) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, CreateDirectory) {
+  SKIP_IF_NO_DRIVER();
 
-  std::string path = ScratchPath("create-directory");
+  std::string path = this->ScratchPath("create-directory");
 
-  if (client_->Exists(path)) { ASSERT_OK(client_->Delete(path, true)); }
+  if (this->client_->Exists(path)) { ASSERT_OK(this->client_->Delete(path, true)); }
 
-  ASSERT_OK(client_->CreateDirectory(path));
-  ASSERT_TRUE(client_->Exists(path));
-  EXPECT_OK(client_->Delete(path, true));
-  ASSERT_FALSE(client_->Exists(path));
+  ASSERT_OK(this->client_->CreateDirectory(path));
+  ASSERT_TRUE(this->client_->Exists(path));
+  EXPECT_OK(this->client_->Delete(path, true));
+  ASSERT_FALSE(this->client_->Exists(path));
 }
 
-TEST_F(TestHdfsClient, GetCapacityUsed) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, GetCapacityUsed) {
+  SKIP_IF_NO_DRIVER();
 
   // Who knows what is actually in your DFS cluster, but expect it to have
   // positive used bytes and capacity
   int64_t nbytes = 0;
-  ASSERT_OK(client_->GetCapacity(&nbytes));
+  ASSERT_OK(this->client_->GetCapacity(&nbytes));
   ASSERT_LT(0, nbytes);
 
-  ASSERT_OK(client_->GetUsed(&nbytes));
+  ASSERT_OK(this->client_->GetUsed(&nbytes));
   ASSERT_LT(0, nbytes);
 }
 
-TEST_F(TestHdfsClient, GetPathInfo) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, GetPathInfo) {
+  SKIP_IF_NO_DRIVER();
 
   HdfsPathInfo info;
 
-  ASSERT_OK(MakeScratchDir());
+  ASSERT_OK(this->MakeScratchDir());
 
   // Directory info
-  ASSERT_OK(client_->GetPathInfo(scratch_dir_, &info));
+  ASSERT_OK(this->client_->GetPathInfo(this->scratch_dir_, &info));
   ASSERT_EQ(ObjectType::DIRECTORY, info.kind);
-  ASSERT_EQ(HdfsAbsPath(scratch_dir_), info.name);
-  ASSERT_EQ(conf_.user, info.owner);
+  ASSERT_EQ(this->HdfsAbsPath(this->scratch_dir_), info.name);
+  ASSERT_EQ(this->conf_.user, info.owner);
 
   // TODO(wesm): test group, other attrs
 
-  auto path = ScratchPath("test-file");
+  auto path = this->ScratchPath("test-file");
 
   const int size = 100;
 
   std::vector<uint8_t> buffer = RandomData(size);
 
-  ASSERT_OK(WriteDummyFile(path, buffer.data(), size));
-  ASSERT_OK(client_->GetPathInfo(path, &info));
+  ASSERT_OK(this->WriteDummyFile(path, buffer.data(), size));
+  ASSERT_OK(this->client_->GetPathInfo(path, &info));
 
   ASSERT_EQ(ObjectType::FILE, info.kind);
-  ASSERT_EQ(HdfsAbsPath(path), info.name);
-  ASSERT_EQ(conf_.user, info.owner);
+  ASSERT_EQ(this->HdfsAbsPath(path), info.name);
+  ASSERT_EQ(this->conf_.user, info.owner);
   ASSERT_EQ(size, info.size);
 }
 
-TEST_F(TestHdfsClient, AppendToFile) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, AppendToFile) {
+  SKIP_IF_NO_DRIVER();
 
-  ASSERT_OK(MakeScratchDir());
+  ASSERT_OK(this->MakeScratchDir());
 
-  auto path = ScratchPath("test-file");
+  auto path = this->ScratchPath("test-file");
   const int size = 100;
 
   std::vector<uint8_t> buffer = RandomData(size);
-  ASSERT_OK(WriteDummyFile(path, buffer.data(), size));
+  ASSERT_OK(this->WriteDummyFile(path, buffer.data(), size));
 
   // now append
-  ASSERT_OK(WriteDummyFile(path, buffer.data(), size, true));
+  ASSERT_OK(this->WriteDummyFile(path, buffer.data(), size, true));
 
   HdfsPathInfo info;
-  ASSERT_OK(client_->GetPathInfo(path, &info));
+  ASSERT_OK(this->client_->GetPathInfo(path, &info));
   ASSERT_EQ(size * 2, info.size);
 }
 
-TEST_F(TestHdfsClient, ListDirectory) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, ListDirectory) {
+  SKIP_IF_NO_DRIVER();
 
   const int size = 100;
   std::vector<uint8_t> data = RandomData(size);
 
-  auto p1 = ScratchPath("test-file-1");
-  auto p2 = ScratchPath("test-file-2");
-  auto d1 = ScratchPath("test-dir-1");
+  auto p1 = this->ScratchPath("test-file-1");
+  auto p2 = this->ScratchPath("test-file-2");
+  auto d1 = this->ScratchPath("test-dir-1");
 
-  ASSERT_OK(MakeScratchDir());
-  ASSERT_OK(WriteDummyFile(p1, data.data(), size));
-  ASSERT_OK(WriteDummyFile(p2, data.data(), size / 2));
-  ASSERT_OK(client_->CreateDirectory(d1));
+  ASSERT_OK(this->MakeScratchDir());
+  ASSERT_OK(this->WriteDummyFile(p1, data.data(), size));
+  ASSERT_OK(this->WriteDummyFile(p2, data.data(), size / 2));
+  ASSERT_OK(this->client_->CreateDirectory(d1));
 
   std::vector<HdfsPathInfo> listing;
-  ASSERT_OK(client_->ListDirectory(scratch_dir_, &listing));
+  ASSERT_OK(this->client_->ListDirectory(this->scratch_dir_, &listing));
 
   // Do it again, appends!
-  ASSERT_OK(client_->ListDirectory(scratch_dir_, &listing));
+  ASSERT_OK(this->client_->ListDirectory(this->scratch_dir_, &listing));
 
   ASSERT_EQ(6, static_cast<int>(listing.size()));
 
   // Argh, well, shouldn't expect the listing to be in any particular order
   for (size_t i = 0; i < listing.size(); ++i) {
     const HdfsPathInfo& info = listing[i];
-    if (info.name == HdfsAbsPath(p1)) {
+    if (info.name == this->HdfsAbsPath(p1)) {
       ASSERT_EQ(ObjectType::FILE, info.kind);
       ASSERT_EQ(size, info.size);
-    } else if (info.name == HdfsAbsPath(p2)) {
+    } else if (info.name == this->HdfsAbsPath(p2)) {
       ASSERT_EQ(ObjectType::FILE, info.kind);
       ASSERT_EQ(size / 2, info.size);
-    } else if (info.name == HdfsAbsPath(d1)) {
+    } else if (info.name == this->HdfsAbsPath(d1)) {
       ASSERT_EQ(ObjectType::DIRECTORY, info.kind);
     } else {
       FAIL() << "Unexpected path: " << info.name;
@@ -246,19 +275,19 @@ TEST_F(TestHdfsClient, ListDirectory) {
   }
 }
 
-TEST_F(TestHdfsClient, ReadableMethods) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, ReadableMethods) {
+  SKIP_IF_NO_DRIVER();
 
-  ASSERT_OK(MakeScratchDir());
+  ASSERT_OK(this->MakeScratchDir());
 
-  auto path = ScratchPath("test-file");
+  auto path = this->ScratchPath("test-file");
   const int size = 100;
 
   std::vector<uint8_t> data = RandomData(size);
-  ASSERT_OK(WriteDummyFile(path, data.data(), size));
+  ASSERT_OK(this->WriteDummyFile(path, data.data(), size));
 
   std::shared_ptr<HdfsReadableFile> file;
-  ASSERT_OK(client_->OpenReadable(path, &file));
+  ASSERT_OK(this->client_->OpenReadable(path, &file));
 
   // Test GetSize -- move this into its own unit test if ever needed
   int64_t file_size;
@@ -293,19 +322,19 @@ TEST_F(TestHdfsClient, ReadableMethods) {
   ASSERT_EQ(60, position);
 }
 
-TEST_F(TestHdfsClient, LargeFile) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, LargeFile) {
+  SKIP_IF_NO_DRIVER();
 
-  ASSERT_OK(MakeScratchDir());
+  ASSERT_OK(this->MakeScratchDir());
 
-  auto path = ScratchPath("test-large-file");
+  auto path = this->ScratchPath("test-large-file");
   const int size = 1000000;
 
   std::vector<uint8_t> data = RandomData(size);
-  ASSERT_OK(WriteDummyFile(path, data.data(), size));
+  ASSERT_OK(this->WriteDummyFile(path, data.data(), size));
 
   std::shared_ptr<HdfsReadableFile> file;
-  ASSERT_OK(client_->OpenReadable(path, &file));
+  ASSERT_OK(this->client_->OpenReadable(path, &file));
 
   auto buffer = std::make_shared<PoolBuffer>();
   ASSERT_OK(buffer->Resize(size));
@@ -317,7 +346,7 @@ TEST_F(TestHdfsClient, LargeFile) {
 
   // explicit buffer size
   std::shared_ptr<HdfsReadableFile> file2;
-  ASSERT_OK(client_->OpenReadable(path, 1 << 18, &file2));
+  ASSERT_OK(this->client_->OpenReadable(path, 1 << 18, &file2));
 
   auto buffer2 = std::make_shared<PoolBuffer>();
   ASSERT_OK(buffer2->Resize(size));
@@ -326,22 +355,22 @@ TEST_F(TestHdfsClient, LargeFile) {
   ASSERT_EQ(size, bytes_read);
 }
 
-TEST_F(TestHdfsClient, RenameFile) {
-  SKIP_IF_NO_LIBHDFS();
+TYPED_TEST(TestHdfsClient, RenameFile) {
+  SKIP_IF_NO_DRIVER();
 
-  ASSERT_OK(MakeScratchDir());
+  ASSERT_OK(this->MakeScratchDir());
 
-  auto src_path = ScratchPath("src-file");
-  auto dst_path = ScratchPath("dst-file");
+  auto src_path = this->ScratchPath("src-file");
+  auto dst_path = this->ScratchPath("dst-file");
   const int size = 100;
 
   std::vector<uint8_t> data = RandomData(size);
-  ASSERT_OK(WriteDummyFile(src_path, data.data(), size));
+  ASSERT_OK(this->WriteDummyFile(src_path, data.data(), size));
 
-  ASSERT_OK(client_->Rename(src_path, dst_path));
+  ASSERT_OK(this->client_->Rename(src_path, dst_path));
 
-  ASSERT_FALSE(client_->Exists(src_path));
-  ASSERT_TRUE(client_->Exists(dst_path));
+  ASSERT_FALSE(this->client_->Exists(src_path));
+  ASSERT_TRUE(this->client_->Exists(dst_path));
 }
 
 }  // namespace io
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
deleted file mode 100644
index 3715376ebb95b..0000000000000
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ /dev/null
@@ -1,582 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// This shim interface to libhdfs (for runtime shared library loading) has been
-// adapted from the SFrame project, released under the ASF-compatible 3-clause
-// BSD license
-//
-// Using this required having the $JAVA_HOME and $HADOOP_HOME environment
-// variables set, so that libjvm and libhdfs can be located easily
-
-// Copyright (C) 2015 Dato, Inc.
-// All rights reserved.
-//
-// This software may be modified and distributed under the terms
-// of the BSD license. See the LICENSE file for details.
-
-#ifdef HAS_HADOOP
-
-#ifndef _WIN32
-#include <dlfcn.h>
-#else
-#include <windows.h>
-#include <winsock2.h>
-
-// TODO(wesm): address when/if we add windows support
-// #include <util/syserr_reporting.hpp>
-#endif
-
-extern "C" {
-#include <hdfs.h>
-}
-
-#include <iostream>
-#include <mutex>
-#include <sstream>
-#include <string>
-#include <type_traits>
-#include <vector>
-
-#include <boost/filesystem.hpp>  // NOLINT
-
-#include "arrow/status.h"
-#include "arrow/util/visibility.h"
-
-namespace fs = boost::filesystem;
-
-extern "C" {
-
-#ifndef _WIN32
-static void* libhdfs_handle = NULL;
-static void* libjvm_handle = NULL;
-#else
-static HINSTANCE libhdfs_handle = NULL;
-static HINSTANCE libjvm_handle = NULL;
-#endif
-/*
- * All the shim pointers
- */
-
-// NOTE(wesm): cpplint does not like use of short and other imprecise C types
-
-static hdfsBuilder* (*ptr_hdfsNewBuilder)(void) = NULL;
-static void (*ptr_hdfsBuilderSetNameNode)(hdfsBuilder* bld, const char* nn) = NULL;
-static void (*ptr_hdfsBuilderSetNameNodePort)(hdfsBuilder* bld, tPort port) = NULL;
-static void (*ptr_hdfsBuilderSetUserName)(hdfsBuilder* bld, const char* userName) = NULL;
-static void (*ptr_hdfsBuilderSetKerbTicketCachePath)(
-    hdfsBuilder* bld, const char* kerbTicketCachePath) = NULL;
-static hdfsFS (*ptr_hdfsBuilderConnect)(hdfsBuilder* bld) = NULL;
-
-static int (*ptr_hdfsDisconnect)(hdfsFS fs) = NULL;
-
-static hdfsFile (*ptr_hdfsOpenFile)(hdfsFS fs, const char* path, int flags,
-    int bufferSize, short replication, tSize blocksize) = NULL;  // NOLINT
-
-static int (*ptr_hdfsCloseFile)(hdfsFS fs, hdfsFile file) = NULL;
-static int (*ptr_hdfsExists)(hdfsFS fs, const char* path) = NULL;
-static int (*ptr_hdfsSeek)(hdfsFS fs, hdfsFile file, tOffset desiredPos) = NULL;
-static tOffset (*ptr_hdfsTell)(hdfsFS fs, hdfsFile file) = NULL;
-static tSize (*ptr_hdfsRead)(hdfsFS fs, hdfsFile file, void* buffer, tSize length) = NULL;
-static tSize (*ptr_hdfsPread)(
-    hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length) = NULL;
-static tSize (*ptr_hdfsWrite)(
-    hdfsFS fs, hdfsFile file, const void* buffer, tSize length) = NULL;
-static int (*ptr_hdfsFlush)(hdfsFS fs, hdfsFile file) = NULL;
-static int (*ptr_hdfsAvailable)(hdfsFS fs, hdfsFile file) = NULL;
-static int (*ptr_hdfsCopy)(
-    hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) = NULL;
-static int (*ptr_hdfsMove)(
-    hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) = NULL;
-static int (*ptr_hdfsDelete)(hdfsFS fs, const char* path, int recursive) = NULL;
-static int (*ptr_hdfsRename)(hdfsFS fs, const char* oldPath, const char* newPath) = NULL;
-static char* (*ptr_hdfsGetWorkingDirectory)(
-    hdfsFS fs, char* buffer, size_t bufferSize) = NULL;
-static int (*ptr_hdfsSetWorkingDirectory)(hdfsFS fs, const char* path) = NULL;
-static int (*ptr_hdfsCreateDirectory)(hdfsFS fs, const char* path) = NULL;
-static int (*ptr_hdfsSetReplication)(
-    hdfsFS fs, const char* path, int16_t replication) = NULL;
-static hdfsFileInfo* (*ptr_hdfsListDirectory)(
-    hdfsFS fs, const char* path, int* numEntries) = NULL;
-static hdfsFileInfo* (*ptr_hdfsGetPathInfo)(hdfsFS fs, const char* path) = NULL;
-static void (*ptr_hdfsFreeFileInfo)(hdfsFileInfo* hdfsFileInfo, int numEntries) = NULL;
-static char*** (*ptr_hdfsGetHosts)(
-    hdfsFS fs, const char* path, tOffset start, tOffset length) = NULL;
-static void (*ptr_hdfsFreeHosts)(char*** blockHosts) = NULL;
-static tOffset (*ptr_hdfsGetDefaultBlockSize)(hdfsFS fs) = NULL;
-static tOffset (*ptr_hdfsGetCapacity)(hdfsFS fs) = NULL;
-static tOffset (*ptr_hdfsGetUsed)(hdfsFS fs) = NULL;
-static int (*ptr_hdfsChown)(
-    hdfsFS fs, const char* path, const char* owner, const char* group) = NULL;
-static int (*ptr_hdfsChmod)(hdfsFS fs, const char* path, short mode) = NULL;  // NOLINT
-static int (*ptr_hdfsUtime)(hdfsFS fs, const char* path, tTime mtime, tTime atime) = NULL;
-
-// Helper functions for dlopens
-static std::vector<fs::path> get_potential_libjvm_paths();
-static std::vector<fs::path> get_potential_libhdfs_paths();
-static arrow::Status try_dlopen(std::vector<fs::path> potential_paths, const char* name,
-#ifndef _WIN32
-    void*& out_handle);
-#else
-    HINSTANCE& out_handle);
-#endif
-
-#define GET_SYMBOL(SYMBOL_NAME)                                                  \
-  if (!ptr_##SYMBOL_NAME) {                                                      \
-    *reinterpret_cast<void**>(&ptr_##SYMBOL_NAME) = get_symbol("" #SYMBOL_NAME); \
-  }
-
-static void* get_symbol(const char* symbol) {
-  if (libhdfs_handle == NULL) return NULL;
-#ifndef _WIN32
-  return dlsym(libhdfs_handle, symbol);
-#else
-
-  void* ret = reinterpret_cast<void*>(GetProcAddress(libhdfs_handle, symbol));
-  if (ret == NULL) {
-    // logstream(LOG_INFO) << "GetProcAddress error: "
-    //                     << get_last_err_str(GetLastError()) << std::endl;
-  }
-  return ret;
-#endif
-}
-
-hdfsBuilder* hdfsNewBuilder(void) {
-  return ptr_hdfsNewBuilder();
-}
-
-void hdfsBuilderSetNameNode(hdfsBuilder* bld, const char* nn) {
-  ptr_hdfsBuilderSetNameNode(bld, nn);
-}
-
-void hdfsBuilderSetNameNodePort(hdfsBuilder* bld, tPort port) {
-  ptr_hdfsBuilderSetNameNodePort(bld, port);
-}
-
-void hdfsBuilderSetUserName(hdfsBuilder* bld, const char* userName) {
-  ptr_hdfsBuilderSetUserName(bld, userName);
-}
-
-void hdfsBuilderSetKerbTicketCachePath(
-    hdfsBuilder* bld, const char* kerbTicketCachePath) {
-  ptr_hdfsBuilderSetKerbTicketCachePath(bld, kerbTicketCachePath);
-}
-
-hdfsFS hdfsBuilderConnect(hdfsBuilder* bld) {
-  return ptr_hdfsBuilderConnect(bld);
-}
-
-int hdfsDisconnect(hdfsFS fs) {
-  return ptr_hdfsDisconnect(fs);
-}
-
-hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, int bufferSize,
-    short replication, tSize blocksize) {  // NOLINT
-  return ptr_hdfsOpenFile(fs, path, flags, bufferSize, replication, blocksize);
-}
-
-int hdfsCloseFile(hdfsFS fs, hdfsFile file) {
-  return ptr_hdfsCloseFile(fs, file);
-}
-
-int hdfsExists(hdfsFS fs, const char* path) {
-  return ptr_hdfsExists(fs, path);
-}
-
-int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
-  return ptr_hdfsSeek(fs, file, desiredPos);
-}
-
-tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
-  return ptr_hdfsTell(fs, file);
-}
-
-tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length) {
-  return ptr_hdfsRead(fs, file, buffer, length);
-}
-
-tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length) {
-  return ptr_hdfsPread(fs, file, position, buffer, length);
-}
-
-tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer, tSize length) {
-  return ptr_hdfsWrite(fs, file, buffer, length);
-}
-
-int hdfsFlush(hdfsFS fs, hdfsFile file) {
-  return ptr_hdfsFlush(fs, file);
-}
-
-int hdfsAvailable(hdfsFS fs, hdfsFile file) {
-  GET_SYMBOL(hdfsAvailable);
-  if (ptr_hdfsAvailable)
-    return ptr_hdfsAvailable(fs, file);
-  else
-    return 0;
-}
-
-int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) {
-  GET_SYMBOL(hdfsCopy);
-  if (ptr_hdfsCopy)
-    return ptr_hdfsCopy(srcFS, src, dstFS, dst);
-  else
-    return 0;
-}
-
-int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) {
-  GET_SYMBOL(hdfsMove);
-  if (ptr_hdfsMove)
-    return ptr_hdfsMove(srcFS, src, dstFS, dst);
-  else
-    return 0;
-}
-
-int hdfsDelete(hdfsFS fs, const char* path, int recursive) {
-  return ptr_hdfsDelete(fs, path, recursive);
-}
-
-int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
-  GET_SYMBOL(hdfsRename);
-  if (ptr_hdfsRename)
-    return ptr_hdfsRename(fs, oldPath, newPath);
-  else
-    return 0;
-}
-
-char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) {
-  GET_SYMBOL(hdfsGetWorkingDirectory);
-  if (ptr_hdfsGetWorkingDirectory) {
-    return ptr_hdfsGetWorkingDirectory(fs, buffer, bufferSize);
-  } else {
-    return NULL;
-  }
-}
-
-int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
-  GET_SYMBOL(hdfsSetWorkingDirectory);
-  if (ptr_hdfsSetWorkingDirectory) {
-    return ptr_hdfsSetWorkingDirectory(fs, path);
-  } else {
-    return 0;
-  }
-}
-
-int hdfsCreateDirectory(hdfsFS fs, const char* path) {
-  return ptr_hdfsCreateDirectory(fs, path);
-}
-
-int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) {
-  GET_SYMBOL(hdfsSetReplication);
-  if (ptr_hdfsSetReplication) {
-    return ptr_hdfsSetReplication(fs, path, replication);
-  } else {
-    return 0;
-  }
-}
-
-hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char* path, int* numEntries) {
-  return ptr_hdfsListDirectory(fs, path, numEntries);
-}
-
-hdfsFileInfo* hdfsGetPathInfo(hdfsFS fs, const char* path) {
-  return ptr_hdfsGetPathInfo(fs, path);
-}
-
-void hdfsFreeFileInfo(hdfsFileInfo* hdfsFileInfo, int numEntries) {
-  ptr_hdfsFreeFileInfo(hdfsFileInfo, numEntries);
-}
-
-char*** hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) {
-  GET_SYMBOL(hdfsGetHosts);
-  if (ptr_hdfsGetHosts) {
-    return ptr_hdfsGetHosts(fs, path, start, length);
-  } else {
-    return NULL;
-  }
-}
-
-void hdfsFreeHosts(char*** blockHosts) {
-  GET_SYMBOL(hdfsFreeHosts);
-  if (ptr_hdfsFreeHosts) { ptr_hdfsFreeHosts(blockHosts); }
-}
-
-tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
-  GET_SYMBOL(hdfsGetDefaultBlockSize);
-  if (ptr_hdfsGetDefaultBlockSize) {
-    return ptr_hdfsGetDefaultBlockSize(fs);
-  } else {
-    return 0;
-  }
-}
-
-tOffset hdfsGetCapacity(hdfsFS fs) {
-  return ptr_hdfsGetCapacity(fs);
-}
-
-tOffset hdfsGetUsed(hdfsFS fs) {
-  return ptr_hdfsGetUsed(fs);
-}
-
-int hdfsChown(hdfsFS fs, const char* path, const char* owner, const char* group) {
-  GET_SYMBOL(hdfsChown);
-  if (ptr_hdfsChown) {
-    return ptr_hdfsChown(fs, path, owner, group);
-  } else {
-    return 0;
-  }
-}
-
-int hdfsChmod(hdfsFS fs, const char* path, short mode) {  // NOLINT
-  GET_SYMBOL(hdfsChmod);
-  if (ptr_hdfsChmod) {
-    return ptr_hdfsChmod(fs, path, mode);
-  } else {
-    return 0;
-  }
-}
-
-int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
-  GET_SYMBOL(hdfsUtime);
-  if (ptr_hdfsUtime) {
-    return ptr_hdfsUtime(fs, path, mtime, atime);
-  } else {
-    return 0;
-  }
-}
-
-static std::vector<fs::path> get_potential_libhdfs_paths() {
-  std::vector<fs::path> libhdfs_potential_paths;
-  std::string file_name;
-
-// OS-specific file name
-#ifdef __WIN32
-  file_name = "hdfs.dll";
-#elif __APPLE__
-  file_name = "libhdfs.dylib";
-#else
-  file_name = "libhdfs.so";
-#endif
-
-  // Common paths
-  std::vector<fs::path> search_paths = {fs::path(""), fs::path(".")};
-
-  // Path from environment variable
-  const char* hadoop_home = std::getenv("HADOOP_HOME");
-  if (hadoop_home != nullptr) {
-    auto path = fs::path(hadoop_home) / "lib/native";
-    search_paths.push_back(path);
-  }
-
-  const char* libhdfs_dir = std::getenv("ARROW_LIBHDFS_DIR");
-  if (libhdfs_dir != nullptr) { search_paths.push_back(fs::path(libhdfs_dir)); }
-
-  // All paths with file name
-  for (auto& path : search_paths) {
-    libhdfs_potential_paths.push_back(path / file_name);
-  }
-
-  return libhdfs_potential_paths;
-}
-
-static std::vector<fs::path> get_potential_libjvm_paths() {
-  std::vector<fs::path> libjvm_potential_paths;
-
-  std::vector<fs::path> search_prefixes;
-  std::vector<fs::path> search_suffixes;
-  std::string file_name;
-
-// From heuristics
-#ifdef __WIN32
-  search_prefixes = {""};
-  search_suffixes = {"/jre/bin/server", "/bin/server"};
-  file_name = "jvm.dll";
-#elif __APPLE__
-  search_prefixes = {""};
-  search_suffixes = {"", "/jre/lib/server"};
-  file_name = "libjvm.dylib";
-
-// SFrame uses /usr/libexec/java_home to find JAVA_HOME; for now we are
-// expecting users to set an environment variable
-#else
-  search_prefixes = {
-      "/usr/lib/jvm/default-java",                // ubuntu / debian distros
-      "/usr/lib/jvm/java",                        // rhel6
-      "/usr/lib/jvm",                             // centos6
-      "/usr/lib64/jvm",                           // opensuse 13
-      "/usr/local/lib/jvm/default-java",          // alt ubuntu / debian distros
-      "/usr/local/lib/jvm/java",                  // alt rhel6
-      "/usr/local/lib/jvm",                       // alt centos6
-      "/usr/local/lib64/jvm",                     // alt opensuse 13
-      "/usr/local/lib/jvm/java-7-openjdk-amd64",  // alt ubuntu / debian distros
-      "/usr/lib/jvm/java-7-openjdk-amd64",        // alt ubuntu / debian distros
-      "/usr/local/lib/jvm/java-6-openjdk-amd64",  // alt ubuntu / debian distros
-      "/usr/lib/jvm/java-6-openjdk-amd64",        // alt ubuntu / debian distros
-      "/usr/lib/jvm/java-7-oracle",               // alt ubuntu
-      "/usr/lib/jvm/java-8-oracle",               // alt ubuntu
-      "/usr/lib/jvm/java-6-oracle",               // alt ubuntu
-      "/usr/local/lib/jvm/java-7-oracle",         // alt ubuntu
-      "/usr/local/lib/jvm/java-8-oracle",         // alt ubuntu
-      "/usr/local/lib/jvm/java-6-oracle",         // alt ubuntu
-      "/usr/lib/jvm/default",                     // alt centos
-      "/usr/java/latest",                         // alt centos
-  };
-  search_suffixes = {"/jre/lib/amd64/server"};
-  file_name = "libjvm.so";
-#endif
-  // From direct environment variable
-  char* env_value = NULL;
-  if ((env_value = getenv("JAVA_HOME")) != NULL) {
-    // logstream(LOG_INFO) << "Found environment variable " << env_name << ": " <<
-    // env_value << std::endl;
-    search_prefixes.insert(search_prefixes.begin(), env_value);
-  }
-
-  // Generate cross product between search_prefixes, search_suffixes, and file_name
-  for (auto& prefix : search_prefixes) {
-    for (auto& suffix : search_suffixes) {
-      auto path = (fs::path(prefix) / fs::path(suffix) / fs::path(file_name));
-      libjvm_potential_paths.push_back(path);
-    }
-  }
-
-  return libjvm_potential_paths;
-}
-
-#ifndef _WIN32
-static arrow::Status try_dlopen(
-    std::vector<fs::path> potential_paths, const char* name, void*& out_handle) {
-  std::vector<std::string> error_messages;
-
-  for (auto& i : potential_paths) {
-    i.make_preferred();
-    // logstream(LOG_INFO) << "Trying " << i.string().c_str() << std::endl;
-    out_handle = dlopen(i.native().c_str(), RTLD_NOW | RTLD_LOCAL);
-
-    if (out_handle != NULL) {
-      // logstream(LOG_INFO) << "Success!" << std::endl;
-      break;
-    } else {
-      const char* err_msg = dlerror();
-      if (err_msg != NULL) {
-        error_messages.push_back(std::string(err_msg));
-      } else {
-        error_messages.push_back(std::string(" returned NULL"));
-      }
-    }
-  }
-
-  if (out_handle == NULL) {
-    std::stringstream ss;
-    ss << "Unable to load " << name;
-    return arrow::Status::IOError(ss.str());
-  }
-
-  return arrow::Status::OK();
-}
-
-#else
-static arrow::Status try_dlopen(
-    std::vector<fs::path> potential_paths, const char* name, HINSTANCE& out_handle) {
-  std::vector<std::string> error_messages;
-
-  for (auto& i : potential_paths) {
-    i.make_preferred();
-    // logstream(LOG_INFO) << "Trying " << i.string().c_str() << std::endl;
-
-    out_handle = LoadLibrary(i.string().c_str());
-
-    if (out_handle != NULL) {
-      // logstream(LOG_INFO) << "Success!" << std::endl;
-      break;
-    } else {
-      // error_messages.push_back(get_last_err_str(GetLastError()));
-    }
-  }
-
-  if (out_handle == NULL) {
-    std::stringstream ss;
-    ss << "Unable to load " << name;
-    return arrow::Status::IOError(ss.str());
-  }
-
-  return arrow::Status::OK();
-}
-#endif  // _WIN32
-
-}  // extern "C"
-
-#define GET_SYMBOL_REQUIRED(SYMBOL_NAME)                                           \
-  do {                                                                             \
-    if (!ptr_##SYMBOL_NAME) {                                                      \
-      *reinterpret_cast<void**>(&ptr_##SYMBOL_NAME) = get_symbol("" #SYMBOL_NAME); \
-    }                                                                              \
-    if (!ptr_##SYMBOL_NAME)                                                        \
-      return Status::IOError("Getting symbol " #SYMBOL_NAME "failed");             \
-  } while (0)
-
-namespace arrow {
-namespace io {
-
-Status ARROW_EXPORT ConnectLibHdfs() {
-  static std::mutex lock;
-  std::lock_guard<std::mutex> guard(lock);
-
-  static bool shim_attempted = false;
-  if (!shim_attempted) {
-    shim_attempted = true;
-
-    std::vector<fs::path> libjvm_potential_paths = get_potential_libjvm_paths();
-    RETURN_NOT_OK(try_dlopen(libjvm_potential_paths, "libjvm", libjvm_handle));
-
-    std::vector<fs::path> libhdfs_potential_paths = get_potential_libhdfs_paths();
-    RETURN_NOT_OK(try_dlopen(libhdfs_potential_paths, "libhdfs", libhdfs_handle));
-  } else if (libhdfs_handle == nullptr) {
-    return Status::IOError("Prior attempt to load libhdfs failed");
-  }
-
-  GET_SYMBOL_REQUIRED(hdfsNewBuilder);
-  GET_SYMBOL_REQUIRED(hdfsBuilderSetNameNode);
-  GET_SYMBOL_REQUIRED(hdfsBuilderSetNameNodePort);
-  GET_SYMBOL_REQUIRED(hdfsBuilderSetUserName);
-  GET_SYMBOL_REQUIRED(hdfsBuilderSetKerbTicketCachePath);
-  GET_SYMBOL_REQUIRED(hdfsBuilderConnect);
-  GET_SYMBOL_REQUIRED(hdfsCreateDirectory);
-  GET_SYMBOL_REQUIRED(hdfsDelete);
-  GET_SYMBOL_REQUIRED(hdfsDisconnect);
-  GET_SYMBOL_REQUIRED(hdfsExists);
-  GET_SYMBOL_REQUIRED(hdfsFreeFileInfo);
-  GET_SYMBOL_REQUIRED(hdfsGetCapacity);
-  GET_SYMBOL_REQUIRED(hdfsGetUsed);
-  GET_SYMBOL_REQUIRED(hdfsGetPathInfo);
-  GET_SYMBOL_REQUIRED(hdfsListDirectory);
-
-  // File methods
-  GET_SYMBOL_REQUIRED(hdfsCloseFile);
-  GET_SYMBOL_REQUIRED(hdfsFlush);
-  GET_SYMBOL_REQUIRED(hdfsOpenFile);
-  GET_SYMBOL_REQUIRED(hdfsRead);
-  GET_SYMBOL_REQUIRED(hdfsPread);
-  GET_SYMBOL_REQUIRED(hdfsSeek);
-  GET_SYMBOL_REQUIRED(hdfsTell);
-  GET_SYMBOL_REQUIRED(hdfsWrite);
-
-  return Status::OK();
-}
-
-}  // namespace io
-}  // namespace arrow
-
-#endif  // HAS_HADOOP
diff --git a/python/.gitignore b/python/.gitignore
index c37efc4b56650..4ab802006914e 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -16,6 +16,7 @@ Testing/
 *.c
 *.cpp
 pyarrow/version.py
+pyarrow/table_api.h
 # Python files
 
 # setup.py working directory
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 77034159d2f3a..99f88adf81d2b 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -87,13 +87,19 @@ cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
 
 
 cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
-    CStatus ConnectLibHdfs()
+    CStatus HaveLibHdfs()
+    CStatus HaveLibHdfs3()
+
+    enum HdfsDriver" arrow::io::HdfsDriver":
+        HdfsDriver_LIBHDFS" arrow::io::HdfsDriver::LIBHDFS"
+        HdfsDriver_LIBHDFS3" arrow::io::HdfsDriver::LIBHDFS3"
 
     cdef cppclass HdfsConnectionConfig:
         c_string host
         int port
         c_string user
         c_string kerb_ticket
+        HdfsDriver driver
 
     cdef cppclass HdfsPathInfo:
         ObjectType kind;
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 2fa5fb6b87885..6b0e3924d207c 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -35,6 +35,7 @@ cimport cpython as cp
 import re
 import sys
 import threading
+import time
 
 
 cdef class NativeFile:
@@ -269,7 +270,15 @@ except ImportError:
 
 def have_libhdfs():
     try:
-        check_status(ConnectLibHdfs())
+        check_status(HaveLibHdfs())
+        return True
+    except:
+        return False
+
+
+def have_libhdfs3():
+    try:
+        check_status(HaveLibHdfs3())
         return True
     except:
         return False
@@ -313,7 +322,8 @@ cdef class HdfsClient:
             raise IOError('HDFS client is closed')
 
     @classmethod
-    def connect(cls, host="default", port=0, user=None, kerb_ticket=None):
+    def connect(cls, host="default", port=0, user=None, kerb_ticket=None,
+                driver='libhdfs'):
         """
         Connect to an HDFS cluster. All parameters are optional and should
         only be set if the defaults need to be overridden.
@@ -328,6 +338,9 @@ cdef class HdfsClient:
         port : NameNode's port. Set to 0 for default or logical (HA) nodes.
         user : Username when connecting to HDFS; None implies login user.
         kerb_ticket : Path to Kerberos ticket cache.
+        driver : {'libhdfs', 'libhdfs3'}, default 'libhdfs'
+          Connect using libhdfs (JNI-based) or libhdfs3 (3rd-party C++
+          library from Pivotal Labs)
 
         Notes
         -----
@@ -350,6 +363,13 @@ cdef class HdfsClient:
         if kerb_ticket is not None:
             conf.kerb_ticket = tobytes(kerb_ticket)
 
+        if driver == 'libhdfs':
+            check_status(HaveLibHdfs())
+            conf.driver = HdfsDriver_LIBHDFS
+        else:
+            check_status(HaveLibHdfs3())
+            conf.driver = HdfsDriver_LIBHDFS3
+
         with nogil:
             check_status(CHdfsClient.Connect(&conf, &out.client))
         out.is_open = True
@@ -541,6 +561,12 @@ cdef class HdfsClient:
                     if not buf:
                         break
 
+                    if writer_thread.is_alive():
+                        while write_queue.full():
+                            time.sleep(0.01)
+                    else:
+                        break
+
                     write_queue.put_nowait(buf)
             finally:
                 done = True
@@ -609,22 +635,13 @@ cdef class HdfsFile(NativeFile):
 
         cdef int64_t total_bytes = 0
 
-        cdef int rpc_chunksize = min(self.buffer_size, nbytes)
-
         try:
             with nogil:
-                while total_bytes < nbytes:
-                    check_status(self.rd_file.get()
-                                 .Read(rpc_chunksize, &bytes_read,
-                                       buf + total_bytes))
-
-                    total_bytes += bytes_read
+                check_status(self.rd_file.get()
+                             .Read(nbytes, &bytes_read, buf))
 
-                    # EOF
-                    if bytes_read == 0:
-                        break
             result = cp.PyBytes_FromStringAndSize(<const char*>buf,
-                                                  total_bytes)
+                                                  bytes_read)
         finally:
             free(buf)
 
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index c23543b7f0d07..73d5a66cf4765 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -19,6 +19,7 @@
 from os.path import join as pjoin
 import os
 import random
+import unittest
 
 import pytest
 
@@ -28,7 +29,7 @@
 # HDFS tests
 
 
-def hdfs_test_client():
+def hdfs_test_client(driver='libhdfs'):
     host = os.environ.get('ARROW_HDFS_TEST_HOST', 'localhost')
     user = os.environ['ARROW_HDFS_TEST_USER']
     try:
@@ -37,115 +38,119 @@ def hdfs_test_client():
         raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
                          'an integer')
 
-    return io.HdfsClient.connect(host, port, user)
+    return io.HdfsClient.connect(host, port, user, driver=driver)
 
 
-libhdfs = pytest.mark.skipif(not io.have_libhdfs(),
-                             reason='No libhdfs available on system')
+class HdfsTestCases(object):
 
+    def _make_test_file(self, hdfs, test_name, test_path, test_data):
+        base_path = pjoin(self.tmp_path, test_name)
+        hdfs.mkdir(base_path)
 
-HDFS_TMP_PATH = '/tmp/pyarrow-test-{0}'.format(random.randint(0, 1000))
+        full_path = pjoin(base_path, test_path)
 
+        with hdfs.open(full_path, 'wb') as f:
+            f.write(test_data)
 
-@pytest.fixture(scope='session')
-def hdfs(request):
-    fixture = hdfs_test_client()
+        return full_path
 
-    def teardown():
-        fixture.delete(HDFS_TMP_PATH, recursive=True)
-        fixture.close()
-    request.addfinalizer(teardown)
-    return fixture
+    @classmethod
+    def setUpClass(cls):
+        cls.check_driver()
+        cls.hdfs = hdfs_test_client(cls.DRIVER)
+        cls.tmp_path = '/tmp/pyarrow-test-{0}'.format(random.randint(0, 1000))
+        cls.hdfs.mkdir(cls.tmp_path)
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.hdfs.delete(cls.tmp_path, recursive=True)
+        cls.hdfs.close()
 
-@libhdfs
-def test_hdfs_close():
-    client = hdfs_test_client()
-    assert client.is_open
-    client.close()
-    assert not client.is_open
+    def test_hdfs_close(self):
+        client = hdfs_test_client()
+        assert client.is_open
+        client.close()
+        assert not client.is_open
 
-    with pytest.raises(Exception):
-        client.ls('/')
+        with pytest.raises(Exception):
+            client.ls('/')
 
+    def test_hdfs_mkdir(self):
+        path = pjoin(self.tmp_path, 'test-dir/test-dir')
+        parent_path = pjoin(self.tmp_path, 'test-dir')
 
-@libhdfs
-def test_hdfs_mkdir(hdfs):
-    path = pjoin(HDFS_TMP_PATH, 'test-dir/test-dir')
-    parent_path = pjoin(HDFS_TMP_PATH, 'test-dir')
+        self.hdfs.mkdir(path)
+        assert self.hdfs.exists(path)
 
-    hdfs.mkdir(path)
-    assert hdfs.exists(path)
+        self.hdfs.delete(parent_path, recursive=True)
+        assert not self.hdfs.exists(path)
 
-    hdfs.delete(parent_path, recursive=True)
-    assert not hdfs.exists(path)
+    def test_hdfs_ls(self):
+        base_path = pjoin(self.tmp_path, 'ls-test')
+        self.hdfs.mkdir(base_path)
 
+        dir_path = pjoin(base_path, 'a-dir')
+        f1_path = pjoin(base_path, 'a-file-1')
 
-@libhdfs
-def test_hdfs_ls(hdfs):
-    base_path = pjoin(HDFS_TMP_PATH, 'ls-test')
-    hdfs.mkdir(base_path)
+        self.hdfs.mkdir(dir_path)
 
-    dir_path = pjoin(base_path, 'a-dir')
-    f1_path = pjoin(base_path, 'a-file-1')
+        f = self.hdfs.open(f1_path, 'wb')
+        f.write('a' * 10)
 
-    hdfs.mkdir(dir_path)
+        contents = sorted(self.hdfs.ls(base_path, False))
+        assert contents == [dir_path, f1_path]
 
-    f = hdfs.open(f1_path, 'wb')
-    f.write('a' * 10)
+    def test_hdfs_download_upload(self):
+        base_path = pjoin(self.tmp_path, 'upload-test')
 
-    contents = sorted(hdfs.ls(base_path, False))
-    assert contents == [dir_path, f1_path]
+        data = b'foobarbaz'
+        buf = BytesIO(data)
+        buf.seek(0)
 
+        self.hdfs.upload(base_path, buf)
 
-def _make_test_file(hdfs, test_name, test_path, test_data):
-    base_path = pjoin(HDFS_TMP_PATH, test_name)
-    hdfs.mkdir(base_path)
+        out_buf = BytesIO()
+        self.hdfs.download(base_path, out_buf)
+        out_buf.seek(0)
+        assert out_buf.getvalue() == data
 
-    full_path = pjoin(base_path, test_path)
+    def test_hdfs_file_context_manager(self):
+        path = pjoin(self.tmp_path, 'ctx-manager')
 
-    f = hdfs.open(full_path, 'wb')
-    f.write(test_data)
+        data = b'foo'
+        with self.hdfs.open(path, 'wb') as f:
+            f.write(data)
 
-    return full_path
+        with self.hdfs.open(path, 'rb') as f:
+            assert f.size() == 3
+            result = f.read(10)
+            assert result == data
 
 
-@libhdfs
-def test_hdfs_orphaned_file():
-    hdfs = hdfs_test_client()
-    file_path = _make_test_file(hdfs, 'orphaned_file_test', 'fname',
-                                'foobarbaz')
+class TestLibHdfs(HdfsTestCases, unittest.TestCase):
 
-    f = hdfs.open(file_path)
-    hdfs = None
-    f = None  # noqa
+    DRIVER = 'libhdfs'
 
+    @classmethod
+    def check_driver(cls):
+        if not io.have_libhdfs():
+            pytest.skip('No libhdfs available on system')
 
-@libhdfs
-def test_hdfs_download_upload(hdfs):
-    base_path = pjoin(HDFS_TMP_PATH, 'upload-test')
+    def test_hdfs_orphaned_file(self):
+        hdfs = hdfs_test_client()
+        file_path = self._make_test_file(hdfs, 'orphaned_file_test', 'fname',
+                                         'foobarbaz')
 
-    data = b'foobarbaz'
-    buf = BytesIO(data)
-    buf.seek(0)
+        f = hdfs.open(file_path)
+        hdfs = None
+        f = None  # noqa
 
-    hdfs.upload(base_path, buf)
 
-    out_buf = BytesIO()
-    hdfs.download(base_path, out_buf)
-    out_buf.seek(0)
-    assert out_buf.getvalue() == data
+class TestLibHdfs3(HdfsTestCases, unittest.TestCase):
 
+    DRIVER = 'libhdfs3'
 
-@libhdfs
-def test_hdfs_file_context_manager(hdfs):
-    path = pjoin(HDFS_TMP_PATH, 'ctx-manager')
-
-    data = b'foo'
-    with hdfs.open(path, 'wb') as f:
-        f.write(data)
-
-    with hdfs.open(path, 'rb') as f:
-        assert f.size() == 3
-        result = f.read(10)
-        assert result == data
+    @classmethod
+    def check_driver(cls):
+        if not io.have_libhdfs3():
+            pytest.skip('No libhdfs3 available on system')

From d7845fcd8b8a06248e42ca083c6460c43723c154 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 19 Dec 2016 18:44:09 -0500
Subject: [PATCH 0237/1644] ARROW-420: Align DATE type with Java implementation

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #238 from xhochy/ARROW-420 and squashes the following commits:

e497d9f [Uwe L. Korn] Add datetime.date parsing for numpy conversion
5c21453 [Uwe L. Korn] Add support for datetime.datetime
6bf346f [Uwe L. Korn] Add datetime.date conversions
6fca4da [Uwe L. Korn] ARROW-420: Align DATE type with Java implementation
---
 cpp/src/arrow/array.cc                       |   1 +
 cpp/src/arrow/array.h                        |   1 +
 cpp/src/arrow/builder.cc                     |   2 +
 cpp/src/arrow/builder.h                      |   1 +
 cpp/src/arrow/type.cc                        |   4 +
 cpp/src/arrow/type.h                         |   4 +-
 cpp/src/arrow/type_fwd.h                     |   4 +-
 cpp/src/arrow/type_traits.h                  |   8 ++
 python/pyarrow/__init__.py                   |   1 +
 python/pyarrow/array.pyx                     |   7 +-
 python/pyarrow/includes/libarrow.pxd         |  16 +++
 python/pyarrow/scalar.pyx                    |  31 ++++++
 python/pyarrow/schema.pyx                    |   7 ++
 python/pyarrow/tests/test_convert_builtin.py |  28 +++++
 python/pyarrow/tests/test_convert_pandas.py  |  15 +++
 python/src/pyarrow/adapters/builtin.cc       |  69 +++++++++++++
 python/src/pyarrow/adapters/pandas.cc        | 103 ++++++++++++++++---
 python/src/pyarrow/helpers.cc                |   6 ++
 python/src/pyarrow/helpers.h                 |   2 +
 python/src/pyarrow/util/datetime.h           |  40 +++++++
 20 files changed, 330 insertions(+), 20 deletions(-)
 create mode 100644 python/src/pyarrow/util/datetime.h

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 7ab61f59f551b..d13fa1e108196 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -148,6 +148,7 @@ template class NumericArray<Int16Type>;
 template class NumericArray<Int32Type>;
 template class NumericArray<Int64Type>;
 template class NumericArray<TimestampType>;
+template class NumericArray<DateType>;
 template class NumericArray<HalfFloatType>;
 template class NumericArray<FloatType>;
 template class NumericArray<DoubleType>;
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 1a4a9237a1f79..26d53f7d75896 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -468,6 +468,7 @@ extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
 extern template class ARROW_EXPORT NumericArray<FloatType>;
 extern template class ARROW_EXPORT NumericArray<DoubleType>;
 extern template class ARROW_EXPORT NumericArray<TimestampType>;
+extern template class ARROW_EXPORT NumericArray<DateType>;
 
 #if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic pop
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 493b5e7ccab9e..1d94dbaa0e91d 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -199,6 +199,7 @@ template class PrimitiveBuilder<Int8Type>;
 template class PrimitiveBuilder<Int16Type>;
 template class PrimitiveBuilder<Int32Type>;
 template class PrimitiveBuilder<Int64Type>;
+template class PrimitiveBuilder<DateType>;
 template class PrimitiveBuilder<TimestampType>;
 template class PrimitiveBuilder<HalfFloatType>;
 template class PrimitiveBuilder<FloatType>;
@@ -411,6 +412,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(INT32, Int32Builder);
     BUILDER_CASE(UINT64, UInt64Builder);
     BUILDER_CASE(INT64, Int64Builder);
+    BUILDER_CASE(DATE, DateBuilder);
     BUILDER_CASE(TIMESTAMP, TimestampBuilder);
 
     BUILDER_CASE(BOOL, BooleanBuilder);
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 7162d31d2464a..205139849b44e 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -220,6 +220,7 @@ using Int16Builder = NumericBuilder<Int16Type>;
 using Int32Builder = NumericBuilder<Int32Type>;
 using Int64Builder = NumericBuilder<Int64Type>;
 using TimestampBuilder = NumericBuilder<TimestampType>;
+using DateBuilder = NumericBuilder<DateType>;
 
 using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
 using FloatBuilder = NumericBuilder<FloatType>;
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 5b172e41f6809..4748cc3c04a20 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -88,6 +88,10 @@ std::string StructType::ToString() const {
   return s.str();
 }
 
+std::string DateType::ToString() const {
+  return std::string("date");
+}
+
 std::string UnionType::ToString() const {
   std::stringstream s;
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 8637081acd9b7..73005707c9edc 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -413,14 +413,14 @@ struct ARROW_EXPORT UnionType : public DataType {
 struct ARROW_EXPORT DateType : public FixedWidthType {
   static constexpr Type::type type_id = Type::DATE;
 
-  using c_type = int32_t;
+  using c_type = int64_t;
 
   DateType() : FixedWidthType(Type::DATE) {}
 
   int bit_width() const override { return sizeof(c_type) * 8; }
 
   Status Accept(TypeVisitor* visitor) const override;
-  std::string ToString() const override { return name(); }
+  std::string ToString() const override;
   static std::string name() { return "date"; }
 };
 
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 6d660f4fdee43..a9db32df54dc3 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -87,13 +87,15 @@ _NUMERIC_TYPE_DECL(Double);
 #undef _NUMERIC_TYPE_DECL
 
 struct DateType;
-class DateArray;
+using DateArray = NumericArray<DateType>;
+using DateBuilder = NumericBuilder<DateType>;
 
 struct TimeType;
 class TimeArray;
 
 struct TimestampType;
 using TimestampArray = NumericArray<TimestampType>;
+using TimestampBuilder = NumericBuilder<TimestampType>;
 
 struct IntervalType;
 using IntervalArray = NumericArray<IntervalType>;
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 3aaec0bd5935a..5616018d93400 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -90,6 +90,14 @@ struct TypeTraits<Int64Type> {
   static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
 };
 
+template <>
+struct TypeTraits<DateType> {
+  using ArrayType = DateArray;
+  // using BuilderType = DateBuilder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+};
+
 template <>
 struct TypeTraits<TimestampType> {
   using ArrayType = TimestampArray;
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index b9d386195b436..a42e39cf9865c 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -37,6 +37,7 @@
 from pyarrow.schema import (null, bool_,
                             int8, int16, int32, int64,
                             uint8, uint16, uint32, uint64,
+                            timestamp, date,
                             float_, double, string,
                             list_, struct, field,
                             DataType, Field, Schema, schema)
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index d44212f4aed63..84f17056a19f7 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -218,6 +218,10 @@ cdef class UInt64Array(NumericArray):
     pass
 
 
+cdef class DateArray(NumericArray):
+    pass
+
+
 cdef class FloatArray(NumericArray):
     pass
 
@@ -245,6 +249,7 @@ cdef dict _array_classes = {
     Type_INT16: Int16Array,
     Type_INT32: Int32Array,
     Type_INT64: Int64Array,
+    Type_DATE: DateArray,
     Type_FLOAT: FloatArray,
     Type_DOUBLE: DoubleArray,
     Type_LIST: ListArray,
@@ -284,7 +289,7 @@ def from_pylist(object list_obj, DataType type=None):
     if type is None:
         check_status(pyarrow.ConvertPySequence(list_obj, &sp_array))
     else:
-        raise NotImplementedError
+        raise NotImplementedError()
 
     return box_arrow_array(sp_array)
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 15781ced4433a..419dd74846c92 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -39,11 +39,18 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         Type_DOUBLE" arrow::Type::DOUBLE"
 
         Type_TIMESTAMP" arrow::Type::TIMESTAMP"
+        Type_DATE" arrow::Type::DATE"
         Type_STRING" arrow::Type::STRING"
 
         Type_LIST" arrow::Type::LIST"
         Type_STRUCT" arrow::Type::STRUCT"
 
+    enum TimeUnit" arrow::TimeUnit":
+        TimeUnit_SECOND" arrow::TimeUnit::SECOND"
+        TimeUnit_MILLI" arrow::TimeUnit::MILLI"
+        TimeUnit_MICRO" arrow::TimeUnit::MICRO"
+        TimeUnit_NANO" arrow::TimeUnit::NANO"
+
     cdef cppclass CDataType" arrow::DataType":
         Type type
 
@@ -74,6 +81,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
 
+    cdef cppclass CTimestampType" arrow::TimestampType"(CDataType):
+        TimeUnit unit
+
     cdef cppclass CField" arrow::Field":
         c_string name
         shared_ptr[CDataType] type
@@ -132,6 +142,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CInt64Array" arrow::Int64Array"(CArray):
         int64_t Value(int i)
 
+    cdef cppclass CDateArray" arrow::DateArray"(CArray):
+        int64_t Value(int i)
+
+    cdef cppclass CTimestampArray" arrow::TimestampArray"(CArray):
+        int64_t Value(int i)
+
     cdef cppclass CFloatArray" arrow::FloatArray"(CArray):
         float Value(int i)
 
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index c2d20e460c37c..09f60e2675499 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -20,6 +20,9 @@ from pyarrow.schema cimport DataType, box_data_type
 from pyarrow.compat import frombytes
 import pyarrow.schema as schema
 
+import datetime
+
+
 NA = None
 
 cdef class NAType(Scalar):
@@ -120,6 +123,32 @@ cdef class UInt64Value(ArrayValue):
         return ap.Value(self.index)
 
 
+cdef class DateValue(ArrayValue):
+
+    def as_py(self):
+        cdef CDateArray* ap = <CDateArray*> self.sp_array.get()
+        return datetime.date.fromtimestamp(ap.Value(self.index) / 1000)
+
+
+cdef class TimestampValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
+            CTimestampType* dtype = <CTimestampType*>ap.type().get()
+            int64_t val = ap.Value(self.index)
+
+        if dtype.unit == TimeUnit_SECOND:
+            return datetime.datetime.utcfromtimestamp(val)
+        elif dtype.unit == TimeUnit_MILLI:
+            return datetime.datetime.utcfromtimestamp(float(val) / 1000)
+        elif dtype.unit == TimeUnit_MICRO:
+            return datetime.datetime.utcfromtimestamp(float(val) / 1000000)
+        else:
+            # TimeUnit_NANO
+            raise NotImplementedError("Cannot convert nanosecond timestamps to datetime.datetime")
+
+
 cdef class FloatValue(ArrayValue):
 
     def as_py(self):
@@ -184,6 +213,8 @@ cdef dict _scalar_classes = {
     Type_INT16: Int16Value,
     Type_INT32: Int32Value,
     Type_INT64: Int64Value,
+    Type_DATE: DateValue,
+    Type_TIMESTAMP: TimestampValue,
     Type_FLOAT: FloatValue,
     Type_DOUBLE: DoubleValue,
     Type_LIST: ListValue,
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index e0badb9764143..d05ac9ebc015a 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -164,6 +164,7 @@ cdef set PRIMITIVE_TYPES = set([
     Type_UINT16, Type_INT16,
     Type_UINT32, Type_INT32,
     Type_UINT64, Type_INT64,
+    Type_TIMESTAMP, Type_DATE,
     Type_FLOAT, Type_DOUBLE])
 
 def null():
@@ -196,6 +197,12 @@ def uint64():
 def int64():
     return primitive_type(Type_INT64)
 
+def timestamp():
+    return primitive_type(Type_TIMESTAMP)
+
+def date():
+    return primitive_type(Type_DATE)
+
 def float_():
     return primitive_type(Type_FLOAT)
 
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 34371b0bdd7c9..7dc1c1b2a4828 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -18,6 +18,7 @@
 from pyarrow.compat import unittest
 import pyarrow
 
+import datetime
 
 class TestConvertList(unittest.TestCase):
 
@@ -78,6 +79,33 @@ def test_string(self):
         assert arr.type == pyarrow.string()
         assert arr.to_pylist() == ['foo', 'bar', None, 'arrow']
 
+    def test_date(self):
+        data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1), datetime.date(2040, 2, 26)]
+        arr = pyarrow.from_pylist(data)
+        assert len(arr) == 4
+        assert arr.type == pyarrow.date()
+        assert arr.null_count == 1
+        assert arr[0].as_py() == datetime.date(2000, 1, 1)
+        assert arr[1].as_py() is None
+        assert arr[2].as_py() == datetime.date(1970, 1, 1)
+        assert arr[3].as_py() == datetime.date(2040, 2, 26)
+
+    def test_timestamp(self):
+        data = [
+            datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
+            None,
+            datetime.datetime(2006, 1, 13, 12, 34, 56, 432539),
+            datetime.datetime(2010, 8, 13, 5, 46, 57, 437699)
+        ]
+        arr = pyarrow.from_pylist(data)
+        assert len(arr) == 4
+        assert arr.type == pyarrow.timestamp()
+        assert arr.null_count == 1
+        assert arr[0].as_py() == datetime.datetime(2007, 7, 13, 1, 23, 34, 123456)
+        assert arr[1].as_py() is None
+        assert arr[2].as_py() == datetime.datetime(2006, 1, 13, 12, 34, 56, 432539)
+        assert arr[3].as_py() == datetime.datetime(2010, 8, 13, 5, 46, 57, 437699)
+
     def test_mixed_nesting_levels(self):
         pyarrow.from_pylist([1, 2, None])
         pyarrow.from_pylist([[1], [2], None])
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index b527ca7e80816..cf50f3d1c2c7a 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import datetime
 import unittest
 
 import numpy as np
@@ -204,6 +205,20 @@ def test_timestamps_notimezone_nulls(self):
             })
         self._check_pandas_roundtrip(df, timestamps_to_ms=False)
 
+    def test_date(self):
+        df = pd.DataFrame({
+            'date': [
+                datetime.date(2000, 1, 1),
+                None,
+                datetime.date(1970, 1, 1),
+                datetime.date(2040, 2, 26)
+        ]})
+        table = A.from_pandas_dataframe(df)
+        result = table.to_pandas()
+        expected = df.copy()
+        expected['date'] = pd.to_datetime(df['date'])
+        tm.assert_frame_equal(result, expected)
+
     # def test_category(self):
     #     repeats = 1000
     #     values = [b'foo', None, u'bar', 'qux', np.nan]
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index ac2f533c408c7..e0cb7c20be3d5 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include <Python.h>
+#include <datetime.h>
 #include <sstream>
 
 #include "pyarrow/adapters/builtin.h"
@@ -24,6 +25,7 @@
 #include "arrow/status.h"
 
 #include "pyarrow/helpers.h"
+#include "pyarrow/util/datetime.h"
 
 using arrow::ArrayBuilder;
 using arrow::DataType;
@@ -55,6 +57,8 @@ class ScalarVisitor {
       none_count_(0),
       bool_count_(0),
       int_count_(0),
+      date_count_(0),
+      timestamp_count_(0),
       float_count_(0),
       string_count_(0) {}
 
@@ -68,6 +72,10 @@ class ScalarVisitor {
       ++float_count_;
     } else if (IsPyInteger(obj)) {
       ++int_count_;
+    } else if (PyDate_CheckExact(obj)) {
+      ++date_count_;
+    } else if (PyDateTime_CheckExact(obj)) {
+      ++timestamp_count_;
     } else if (IsPyBaseString(obj)) {
       ++string_count_;
     } else {
@@ -82,6 +90,10 @@ class ScalarVisitor {
     } else if (int_count_) {
       // TODO(wesm): tighter type later
       return INT64;
+    } else if (date_count_) {
+      return DATE;
+    } else if (timestamp_count_) {
+      return TIMESTAMP_US;
     } else if (bool_count_) {
       return BOOL;
     } else if (string_count_) {
@@ -100,6 +112,8 @@ class ScalarVisitor {
   int64_t none_count_;
   int64_t bool_count_;
   int64_t int_count_;
+  int64_t date_count_;
+  int64_t timestamp_count_;
   int64_t float_count_;
   int64_t string_count_;
 
@@ -297,6 +311,56 @@ class Int64Converter : public TypedConverter<arrow::Int64Builder> {
   }
 };
 
+class DateConverter : public TypedConverter<arrow::DateBuilder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    Py_ssize_t size = PySequence_Size(seq);
+    RETURN_NOT_OK(typed_builder_->Reserve(size));
+    for (int64_t i = 0; i < size; ++i) {
+      OwnedRef item(PySequence_GetItem(seq, i));
+      if (item.obj() == Py_None) {
+        typed_builder_->AppendNull();
+      } else {
+        PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
+        typed_builder_->Append(PyDate_to_ms(pydate));
+      }
+    }
+    return Status::OK();
+  }
+};
+
+class TimestampConverter : public TypedConverter<arrow::TimestampBuilder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    Py_ssize_t size = PySequence_Size(seq);
+    RETURN_NOT_OK(typed_builder_->Reserve(size));
+    for (int64_t i = 0; i < size; ++i) {
+      OwnedRef item(PySequence_GetItem(seq, i));
+      if (item.obj() == Py_None) {
+        typed_builder_->AppendNull();
+      } else {
+        PyDateTime_DateTime* pydatetime = reinterpret_cast<PyDateTime_DateTime*>(item.obj());
+        struct tm datetime = {0};
+        datetime.tm_year = PyDateTime_GET_YEAR(pydatetime) - 1900;
+        datetime.tm_mon = PyDateTime_GET_MONTH(pydatetime) - 1;
+        datetime.tm_mday = PyDateTime_GET_DAY(pydatetime);
+        datetime.tm_hour = PyDateTime_DATE_GET_HOUR(pydatetime);
+        datetime.tm_min = PyDateTime_DATE_GET_MINUTE(pydatetime);
+        datetime.tm_sec = PyDateTime_DATE_GET_SECOND(pydatetime);
+        int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
+        RETURN_IF_PYERROR();
+        struct tm epoch = {0};
+        epoch.tm_year = 70;
+        epoch.tm_mday = 1;
+        // Microseconds since the epoch
+        int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000 + us;
+        typed_builder_->Append(val);
+      }
+    }
+    return Status::OK();
+  }
+};
+
 class DoubleConverter : public TypedConverter<arrow::DoubleBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
@@ -379,6 +443,10 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
       return std::make_shared<BoolConverter>();
     case Type::INT64:
       return std::make_shared<Int64Converter>();
+    case Type::DATE:
+      return std::make_shared<DateConverter>();
+    case Type::TIMESTAMP:
+      return std::make_shared<TimestampConverter>();
     case Type::DOUBLE:
       return std::make_shared<DoubleConverter>();
     case Type::STRING:
@@ -409,6 +477,7 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
 Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
   std::shared_ptr<DataType> type;
   int64_t size;
+  PyDateTime_IMPORT;
   RETURN_NOT_OK(InferArrowType(obj, &size, &type));
 
   // Handle NA / NullType case
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 64b708695194a..f8dff6d824153 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -35,6 +35,7 @@
 
 #include "pyarrow/common.h"
 #include "pyarrow/config.h"
+#include "pyarrow/util/datetime.h"
 
 namespace pyarrow {
 
@@ -167,6 +168,28 @@ class ArrowSerializer {
  private:
   Status ConvertData();
 
+  Status ConvertDates(std::shared_ptr<Array>* out) {
+    PyAcquireGIL lock;
+
+    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    arrow::TypePtr string_type(new arrow::DateType());
+    arrow::DateBuilder date_builder(pool_, string_type);
+    RETURN_NOT_OK(date_builder.Resize(length_));
+
+    Status s;
+    PyObject* obj;
+    for (int64_t i = 0; i < length_; ++i) {
+      obj = objects[i];
+      if (PyDate_CheckExact(obj)) {
+        PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(obj);
+        date_builder.Append(PyDate_to_ms(pydate));
+      } else {
+        date_builder.AppendNull();
+      }
+    }
+    return date_builder.Finish(out);
+  }
+
   Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
     PyAcquireGIL lock;
 
@@ -369,6 +392,10 @@ inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out)
 
   // TODO: mask not supported here
   const PyObject** objects = reinterpret_cast<const PyObject**>(PyArray_DATA(arr_));
+  {
+    PyAcquireGIL lock;
+    PyDateTime_IMPORT;
+  }
 
   for (int64_t i = 0; i < length_; ++i) {
     if (PyObject_is_null(objects[i])) {
@@ -377,6 +404,8 @@ inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out)
       return ConvertObjectStrings(out);
     } else if (PyBool_Check(objects[i])) {
       return ConvertBooleans(out);
+    } else if (PyDate_CheckExact(objects[i])) {
+      return ConvertDates(out);
     } else {
       return Status::TypeError("unhandled python type");
     }
@@ -547,6 +576,17 @@ struct arrow_traits<arrow::Type::TIMESTAMP> {
   typedef typename npy_traits<NPY_DATETIME>::value_type T;
 };
 
+template <>
+struct arrow_traits<arrow::Type::DATE> {
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr bool supports_nulls = true;
+  static constexpr int64_t na_value = std::numeric_limits<int64_t>::min();
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_pandas_numeric_not_nullable = false;
+  static constexpr bool is_pandas_numeric_nullable = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
+
 template <>
 struct arrow_traits<arrow::Type::STRING> {
   static constexpr int npy_type = NPY_OBJECT;
@@ -567,24 +607,28 @@ static inline PyObject* make_pystring(const uint8_t* data, int32_t length) {
 
 inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
   if (type == NPY_DATETIME) {
-    auto timestamp_type = static_cast<arrow::TimestampType*>(datatype);
-    // We only support ms resolution at the moment
     PyArray_Descr* descr = PyArray_DESCR(out);
     auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+    if (datatype->type == arrow::Type::TIMESTAMP) {
+      auto timestamp_type = static_cast<arrow::TimestampType*>(datatype);
 
-    switch (timestamp_type->unit) {
-      case arrow::TimestampType::Unit::SECOND:
-        date_dtype->meta.base = NPY_FR_s;
-        break;
-      case arrow::TimestampType::Unit::MILLI:
-        date_dtype->meta.base = NPY_FR_ms;
-        break;
-      case arrow::TimestampType::Unit::MICRO:
-        date_dtype->meta.base = NPY_FR_us;
-        break;
-      case arrow::TimestampType::Unit::NANO:
-        date_dtype->meta.base = NPY_FR_ns;
-        break;
+      switch (timestamp_type->unit) {
+        case arrow::TimestampType::Unit::SECOND:
+          date_dtype->meta.base = NPY_FR_s;
+          break;
+        case arrow::TimestampType::Unit::MILLI:
+          date_dtype->meta.base = NPY_FR_ms;
+          break;
+        case arrow::TimestampType::Unit::MICRO:
+          date_dtype->meta.base = NPY_FR_us;
+          break;
+        case arrow::TimestampType::Unit::NANO:
+          date_dtype->meta.base = NPY_FR_ns;
+          break;
+      }
+    } else {
+      // datatype->type == arrow::Type::DATE
+      date_dtype->meta.base = NPY_FR_D;
     }
   }
 }
@@ -666,7 +710,7 @@ class ArrowDeserializer {
 
   template <int T2>
   inline typename std::enable_if<
-    arrow_traits<T2>::is_pandas_numeric_nullable, Status>::type
+    (T2 != arrow::Type::DATE) & arrow_traits<T2>::is_pandas_numeric_nullable, Status>::type
   ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
     typedef typename arrow_traits<T2>::T T;
     size_t chunk_offset = 0;
@@ -697,6 +741,32 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
+  template <int T2>
+  inline typename std::enable_if<
+    T2 == arrow::Type::DATE, Status>::type
+  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
+    typedef typename arrow_traits<T2>::T T;
+    size_t chunk_offset = 0;
+
+    RETURN_NOT_OK(AllocateOutput(arrow_traits<T2>::npy_type));
+
+    for (int c = 0; c < data->num_chunks(); c++) {
+      const std::shared_ptr<Array> arr = data->chunk(c);
+      auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+      auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+      auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_)) + chunk_offset;
+
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
+        out_values[i] = arr->IsNull(i) ? arrow_traits<T2>::na_value : in_values[i] / 86400000;
+      }
+
+      chunk_offset += arr->length();
+    }
+
+    return Status::OK();
+  }
+
   // Integer specialization
   template <int T2>
   inline typename std::enable_if<
@@ -879,6 +949,7 @@ Status ConvertColumnToPandas(const std::shared_ptr<Column>& col, PyObject* py_re
     FROM_ARROW_CASE(FLOAT);
     FROM_ARROW_CASE(DOUBLE);
     FROM_ARROW_CASE(STRING);
+    FROM_ARROW_CASE(DATE);
     FROM_ARROW_CASE(TIMESTAMP);
     default:
       return Status::NotImplemented("Arrow type reading not implemented");
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
index 08003aabf9f22..af9274484935f 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/python/src/pyarrow/helpers.cc
@@ -33,6 +33,8 @@ const std::shared_ptr<Int8Type> INT8 = std::make_shared<Int8Type>();
 const std::shared_ptr<Int16Type> INT16 = std::make_shared<Int16Type>();
 const std::shared_ptr<Int32Type> INT32 = std::make_shared<Int32Type>();
 const std::shared_ptr<Int64Type> INT64 = std::make_shared<Int64Type>();
+const std::shared_ptr<DateType> DATE = std::make_shared<DateType>();
+const std::shared_ptr<TimestampType> TIMESTAMP_US = std::make_shared<TimestampType>(TimeUnit::MICRO);
 const std::shared_ptr<FloatType> FLOAT = std::make_shared<FloatType>();
 const std::shared_ptr<DoubleType> DOUBLE = std::make_shared<DoubleType>();
 const std::shared_ptr<StringType> STRING = std::make_shared<StringType>();
@@ -54,6 +56,10 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
     GET_PRIMITIVE_TYPE(INT32, Int32Type);
     GET_PRIMITIVE_TYPE(UINT64, UInt64Type);
     GET_PRIMITIVE_TYPE(INT64, Int64Type);
+    GET_PRIMITIVE_TYPE(DATE, DateType);
+    case Type::TIMESTAMP:
+      return TIMESTAMP_US;
+      break;
     GET_PRIMITIVE_TYPE(BOOL, BooleanType);
     GET_PRIMITIVE_TYPE(FLOAT, FloatType);
     GET_PRIMITIVE_TYPE(DOUBLE, DoubleType);
diff --git a/python/src/pyarrow/helpers.h b/python/src/pyarrow/helpers.h
index fa9c713b0c22c..e714bba5db4cc 100644
--- a/python/src/pyarrow/helpers.h
+++ b/python/src/pyarrow/helpers.h
@@ -38,6 +38,8 @@ extern const std::shared_ptr<arrow::Int8Type> INT8;
 extern const std::shared_ptr<arrow::Int16Type> INT16;
 extern const std::shared_ptr<arrow::Int32Type> INT32;
 extern const std::shared_ptr<arrow::Int64Type> INT64;
+extern const std::shared_ptr<arrow::DateType> DATE;
+extern const std::shared_ptr<arrow::TimestampType> TIMESTAMP_US;
 extern const std::shared_ptr<arrow::FloatType> FLOAT;
 extern const std::shared_ptr<arrow::DoubleType> DOUBLE;
 extern const std::shared_ptr<arrow::StringType> STRING;
diff --git a/python/src/pyarrow/util/datetime.h b/python/src/pyarrow/util/datetime.h
new file mode 100644
index 0000000000000..b67accc388f59
--- /dev/null
+++ b/python/src/pyarrow/util/datetime.h
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PYARROW_UTIL_DATETIME_H
+#define PYARROW_UTIL_DATETIME_H
+
+#include <Python.h>
+#include <datetime.h>
+
+namespace pyarrow {
+  
+inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) { 
+  struct tm date = {0};
+  date.tm_year = PyDateTime_GET_YEAR(pydate) - 1900;
+  date.tm_mon = PyDateTime_GET_MONTH(pydate) - 1;
+  date.tm_mday = PyDateTime_GET_DAY(pydate);
+  struct tm epoch = {0};
+  epoch.tm_year = 70;
+  epoch.tm_mday = 1;
+  // Milliseconds since the epoch
+  return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
+}
+
+} // namespace pyarrow
+
+#endif // PYARROW_UTIL_DATETIME_H

From fe53fa409b644ee7b971dc4ed8f877199e91686e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 20 Dec 2016 11:40:04 -0500
Subject: [PATCH 0238/1644] ARROW-435: Fix spelling of RAPIDJSON_VENDORED

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #246 from xhochy/ARROW-435 and squashes the following commits:

9fdbfde [Uwe L. Korn] ARROW-435: Fix spelling of RAPIDJSON_VENDORED
---
 cpp/src/arrow/ipc/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index b1669c5f7c239..619ca7c92cb7a 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -44,7 +44,7 @@ set(ARROW_IPC_SRCS
 add_library(arrow_ipc SHARED
   ${ARROW_IPC_SRCS}
 )
-if(RAPIDJSON_VERDORED)
+if(RAPIDJSON_VENDORED)
   add_dependencies(arrow_ipc rapidjson_ep)
 endif()
 if(FLATBUFFERS_VENDORED)

From 6ff5fcf1bfb67d817d6261596d47cf6a6d9c3c6c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 20 Dec 2016 14:08:07 -0500
Subject: [PATCH 0239/1644] ARROW-433: Correctly handle Arrow to Python date
 conversion for timezones west of London

Verified with `TZ='America/New_York' py.test pyarrow`

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #245 from xhochy/ARROW-433 and squashes the following commits:

06745d8 [Uwe L. Korn] Use more pythonic approach
a55be24 [Uwe L. Korn] ARROW-433: Correctly handle Arrow to Python date conversion for timezones west of London
---
 python/pyarrow/scalar.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 09f60e2675499..623e3e483d4ae 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -127,7 +127,7 @@ cdef class DateValue(ArrayValue):
 
     def as_py(self):
         cdef CDateArray* ap = <CDateArray*> self.sp_array.get()
-        return datetime.date.fromtimestamp(ap.Value(self.index) / 1000)
+        return datetime.datetime.utcfromtimestamp(ap.Value(self.index) / 1000).date()
 
 
 cdef class TimestampValue(ArrayValue):

From f6bf112cd22eeb03725dff79a28c205324fa4f45 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 20 Dec 2016 14:09:32 -0500
Subject: [PATCH 0240/1644] ARROW-434: [Python] Correctly handle Python file
 objects in Parquet read/write paths

While we'd enabled Python file objects for IPC file reader/writer, they hadn't been enabled in the Parquet read/write paths. For example:

```python
with open(filename, 'wb') as f:
    A.parquet.write_table(arrow_table, f, version="1.0")

data = io.BytesIO(open(filename, 'rb').read())

table_read = pq.read_table(data)
```

There was a separate bug reported in ARROW-434, but that's a Parquet type mapping issue, will be fixed in PARQUET-812.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #247 from wesm/ARROW-434 and squashes the following commits:

c704088 [Wes McKinney] Correctly handle Python file objects in Parquet read/write paths
---
 python/pyarrow/io.pxd                |  3 ++
 python/pyarrow/io.pyx                | 40 +++++++++++++++++++++++
 python/pyarrow/ipc.pyx               | 43 +-----------------------
 python/pyarrow/parquet.pyx           | 49 +++++++++++++++-------------
 python/pyarrow/tests/test_parquet.py | 36 ++++++++++++++++++--
 5 files changed, 104 insertions(+), 67 deletions(-)

diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
index d6966cdaadd3a..02265d0a68eb1 100644
--- a/python/pyarrow/io.pxd
+++ b/python/pyarrow/io.pxd
@@ -42,3 +42,6 @@ cdef class NativeFile:
     # suite of Arrow C++ libraries
     cdef read_handle(self, shared_ptr[ReadableFileInterface]* file)
     cdef write_handle(self, shared_ptr[OutputStream]* file)
+
+cdef get_reader(object source, shared_ptr[ReadableFileInterface]* reader)
+cdef get_writer(object source, shared_ptr[OutputStream]* writer)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 6b0e3924d207c..8491aa8964fb9 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -256,6 +256,46 @@ def buffer_from_bytes(object obj):
     result.init(buf)
     return result
 
+cdef get_reader(object source, shared_ptr[ReadableFileInterface]* reader):
+    cdef NativeFile nf
+
+    if isinstance(source, bytes):
+        source = BytesReader(source)
+    elif not isinstance(source, NativeFile) and hasattr(source, 'read'):
+        # Optimistically hope this is file-like
+        source = PythonFileInterface(source, mode='r')
+
+    if isinstance(source, NativeFile):
+        nf = source
+
+        # TODO: what about read-write sources (e.g. memory maps)
+        if not nf.is_readonly:
+            raise IOError('Native file is not readable')
+
+        nf.read_handle(reader)
+    else:
+        raise TypeError('Unable to read from object of type: {0}'
+                        .format(type(source)))
+
+
+cdef get_writer(object source, shared_ptr[OutputStream]* writer):
+    cdef NativeFile nf
+
+    if not isinstance(source, NativeFile) and hasattr(source, 'write'):
+        # Optimistically hope this is file-like
+        source = PythonFileInterface(source, mode='w')
+
+    if isinstance(source, NativeFile):
+        nf = source
+
+        if nf.is_readonly:
+            raise IOError('Native file is not writeable')
+
+        nf.write_handle(writer)
+    else:
+        raise TypeError('Unable to read from object of type: {0}'
+                        .format(type(source)))
+
 # ----------------------------------------------------------------------
 # HDFS IO implementation
 
diff --git a/python/pyarrow/ipc.pyx b/python/pyarrow/ipc.pyx
index 46deb5ad0c35d..abc5e1b11ec4c 100644
--- a/python/pyarrow/ipc.pyx
+++ b/python/pyarrow/ipc.pyx
@@ -27,7 +27,7 @@ from pyarrow.includes.libarrow_ipc cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.error cimport check_status
-from pyarrow.io cimport NativeFile
+from pyarrow.io cimport NativeFile, get_reader, get_writer
 from pyarrow.schema cimport Schema
 from pyarrow.table cimport RecordBatch
 
@@ -37,47 +37,6 @@ import pyarrow.io as io
 cimport cpython as cp
 
 
-cdef get_reader(source, shared_ptr[ReadableFileInterface]* reader):
-    cdef NativeFile nf
-
-    if isinstance(source, bytes):
-        source = io.BytesReader(source)
-    elif not isinstance(source, io.NativeFile) and hasattr(source, 'read'):
-        # Optimistically hope this is file-like
-        source = io.PythonFileInterface(source, mode='r')
-
-    if isinstance(source, NativeFile):
-        nf = source
-
-        # TODO: what about read-write sources (e.g. memory maps)
-        if not nf.is_readonly:
-            raise IOError('Native file is not readable')
-
-        nf.read_handle(reader)
-    else:
-        raise TypeError('Unable to read from object of type: {0}'
-                        .format(type(source)))
-
-
-cdef get_writer(source, shared_ptr[OutputStream]* writer):
-    cdef NativeFile nf
-
-    if not isinstance(source, io.NativeFile) and hasattr(source, 'write'):
-        # Optimistically hope this is file-like
-        source = io.PythonFileInterface(source, mode='w')
-
-    if isinstance(source, io.NativeFile):
-        nf = source
-
-        if nf.is_readonly:
-            raise IOError('Native file is not writeable')
-
-        nf.write_handle(writer)
-    else:
-        raise TypeError('Unable to read from object of type: {0}'
-                        .format(type(source)))
-
-
 cdef class ArrowFileWriter:
     cdef:
         shared_ptr[CFileWriter] writer
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 83fddb287a3f1..043ccf12d9181 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -31,7 +31,7 @@ from pyarrow.error cimport check_status
 from pyarrow.io import NativeFile
 from pyarrow.table cimport Table
 
-from pyarrow.io cimport NativeFile
+from pyarrow.io cimport NativeFile, get_reader, get_writer
 
 import six
 
@@ -49,22 +49,27 @@ cdef class ParquetReader:
     def __cinit__(self):
         self.allocator.set_pool(default_memory_pool())
 
-    cdef open_local_file(self, file_path):
-        cdef c_string path = tobytes(file_path)
+    def open(self, source):
+        self._open(source)
 
-        # Must be in one expression to avoid calling std::move which is not
-        # possible in Cython (due to missing rvalue support)
+    cdef _open(self, object source):
+        cdef:
+            shared_ptr[ReadableFileInterface] rd_handle
+            c_string path
 
-        # TODO(wesm): ParquetFileReader::OpenFIle can throw?
-        self.reader = unique_ptr[FileReader](
-            new FileReader(default_memory_pool(),
-                           ParquetFileReader.OpenFile(path)))
+        if isinstance(source, six.string_types):
+            path = tobytes(source)
 
-    cdef open_native_file(self, NativeFile file):
-        cdef shared_ptr[ReadableFileInterface] cpp_handle
-        file.read_handle(&cpp_handle)
+            # Must be in one expression to avoid calling std::move which is not
+            # possible in Cython (due to missing rvalue support)
 
-        check_status(OpenFile(cpp_handle, &self.allocator, &self.reader))
+            # TODO(wesm): ParquetFileReader::OpenFile can throw?
+            self.reader = unique_ptr[FileReader](
+                new FileReader(default_memory_pool(),
+                               ParquetFileReader.OpenFile(path)))
+        else:
+            get_reader(source, &rd_handle)
+            check_status(OpenFile(rd_handle, &self.allocator, &self.reader))
 
     def read_all(self):
         cdef:
@@ -137,11 +142,7 @@ def read_table(source, columns=None):
         Content of the file as a table (of columns)
     """
     cdef ParquetReader reader = ParquetReader()
-
-    if isinstance(source, six.string_types):
-        reader.open_local_file(source)
-    elif isinstance(source, NativeFile):
-        reader.open_native_file(source)
+    reader._open(source)
 
     if columns is None:
         return reader.read_all()
@@ -174,7 +175,10 @@ def write_table(table, sink, chunk_size=None, version=None,
     cdef Table table_ = table
     cdef CTable* ctable_ = table_.table
     cdef shared_ptr[ParquetWriteSink] sink_
+
     cdef shared_ptr[FileOutputStream] filesink_
+    cdef shared_ptr[OutputStream] general_sink
+
     cdef WriterProperties.Builder properties_builder
     cdef int64_t chunk_size_ = 0
     if chunk_size is None:
@@ -232,10 +236,11 @@ def write_table(table, sink, chunk_size=None, version=None,
                 raise ArrowException("Unsupport compression codec")
 
     if isinstance(sink, six.string_types):
-       check_status(FileOutputStream.Open(tobytes(sink), &filesink_))
-       sink_.reset(new ParquetWriteSink(<shared_ptr[OutputStream]>filesink_))
-    elif isinstance(sink, NativeFile):
-        sink_.reset(new ParquetWriteSink((<NativeFile>sink).wr_file))
+        check_status(FileOutputStream.Open(tobytes(sink), &filesink_))
+        sink_.reset(new ParquetWriteSink(<shared_ptr[OutputStream]>filesink_))
+    else:
+        get_writer(sink, &general_sink)
+        sink_.reset(new ParquetWriteSink(general_sink))
 
     with nogil:
         check_status(WriteFlatTable(ctable_, default_memory_pool(), sink_,
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 841830f6fba3b..7c45732d34573 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import io
 import pytest
 
 import pyarrow as A
@@ -132,9 +133,8 @@ def test_pandas_column_selection(tmpdir):
 
     pdt.assert_frame_equal(df[['uint8']], df_read)
 
-@parquet
-def test_pandas_parquet_native_file_roundtrip(tmpdir):
-    size = 10000
+
+def _test_dataframe(size=10000):
     np.random.seed(0)
     df = pd.DataFrame({
         'uint8': np.arange(size, dtype=np.uint8),
@@ -149,6 +149,12 @@ def test_pandas_parquet_native_file_roundtrip(tmpdir):
         'float64': np.arange(size, dtype=np.float64),
         'bool': np.random.randn(size) > 0
     })
+    return df
+
+
+@parquet
+def test_pandas_parquet_native_file_roundtrip(tmpdir):
+    df = _test_dataframe(10000)
     arrow_table = A.from_pandas_dataframe(df)
     imos = paio.InMemoryOutputStream()
     pq.write_table(arrow_table, imos, version="2.0")
@@ -158,6 +164,30 @@ def test_pandas_parquet_native_file_roundtrip(tmpdir):
     pdt.assert_frame_equal(df, df_read)
 
 
+@parquet
+def test_pandas_parquet_pyfile_roundtrip(tmpdir):
+    filename = tmpdir.join('pandas_pyfile_roundtrip.parquet').strpath
+    size = 5
+    df = pd.DataFrame({
+        'int64': np.arange(size, dtype=np.int64),
+        'float32': np.arange(size, dtype=np.float32),
+        'float64': np.arange(size, dtype=np.float64),
+        'bool': np.random.randn(size) > 0,
+        'strings': ['foo', 'bar', None, 'baz', 'qux']
+    })
+
+    arrow_table = A.from_pandas_dataframe(df)
+
+    with open(filename, 'wb') as f:
+        A.parquet.write_table(arrow_table, f, version="1.0")
+
+    data = io.BytesIO(open(filename, 'rb').read())
+
+    table_read = pq.read_table(data)
+    df_read = table_read.to_pandas()
+    pdt.assert_frame_equal(df, df_read)
+
+
 @parquet
 def test_pandas_parquet_configuration_options(tmpdir):
     size = 10000

From 73455b56f705c3c11d3c29447082641dcab4c63a Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 20 Dec 2016 14:10:21 -0500
Subject: [PATCH 0241/1644] ARROW-430: Improved version handling

This reintroduces `setuptools_scm` versioning for git clones and sdists/wheels. git-archives are handled by a separate chunk of code that will infer the version from the `pom.xml`

As in the Maven world, always the next to-be-released version is specified and Python PEP 440 development versions are based on the previous release, it used the most minimal pre-release version specified there.

I would suggest to keep the conda package versioning as it is currently, i.e. manually set it to 0.1.0postX.

Also: I would rather not parse the Maven XML but that is the most simple way currently to ensure that the versioning system is in a state where we can still make releases with the Maven release plugin.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #248 from xhochy/ARROW-430 and squashes the following commits:

39753f8 [Uwe L. Korn] Infer version from java/pom.xml
05c44ea [Uwe L. Korn] Get rid of setuptools_scm_git_archive
14b8136 [Uwe L. Korn] Revert "ARROW-429: Revert ARROW-379 until git-archive issues are resolved"

Change-Id: I4f6d291e63b2518af47c2a81049aa24a38c92821
---
 dev/release/00-prepare.sh                     |  5 ---
 .../cmake_modules/FindParquet.cmake           |  0
 python/pyarrow/__init__.py                    | 10 ++++--
 python/setup.cfg                              | 20 +++++++++++
 python/setup.py                               | 34 +++++++------------
 5 files changed, 41 insertions(+), 28 deletions(-)
 rename {cpp => python}/cmake_modules/FindParquet.cmake (100%)
 create mode 100644 python/setup.cfg

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
index 3423a3e6c5bf9..00af5e7768161 100644
--- a/dev/release/00-prepare.sh
+++ b/dev/release/00-prepare.sh
@@ -43,9 +43,4 @@ mvn release:prepare -Dtag=${tag} -DreleaseVersion=${version} -DautoVersionSubmod
 
 cd -
 
-cd "${SOURCE_DIR}/../../python"
-sed -i "s/VERSION = '[^']*'/VERSION = '${version}'/g" setup.py
-sed -i "s/ISRELEASED = False/ISRELEASED = True/g" setup.py
-cd -
-
 echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
diff --git a/cpp/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
similarity index 100%
rename from cpp/cmake_modules/FindParquet.cmake
rename to python/cmake_modules/FindParquet.cmake
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index a42e39cf9865c..39ba4c72e7d3c 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -17,6 +17,14 @@
 
 # flake8: noqa
 
+from pkg_resources import get_distribution, DistributionNotFound
+try:
+    __version__ = get_distribution(__name__).version
+except DistributionNotFound:
+   # package is not installed
+   pass
+
+
 import pyarrow.config
 
 from pyarrow.array import (Array,
@@ -43,5 +51,3 @@
                             DataType, Field, Schema, schema)
 
 from pyarrow.table import Column, RecordBatch, Table, from_pandas_dataframe
-
-from pyarrow.version import version as __version__
diff --git a/python/setup.cfg b/python/setup.cfg
new file mode 100644
index 0000000000000..caae3e081b6ca
--- /dev/null
+++ b/python/setup.cfg
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[build_sphinx]
+source-dir = doc/
+build-dir  = doc/_build
diff --git a/python/setup.py b/python/setup.py
index 5f448f7d50784..2e595e2dc870e 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -42,27 +42,9 @@
 if Cython.__version__ < '0.19.1':
     raise Exception('Please upgrade to Cython 0.19.1 or newer')
 
-VERSION = '0.1.0'
-ISRELEASED = False
-
-if not ISRELEASED:
-    VERSION += '.dev'
-
 setup_dir = os.path.abspath(os.path.dirname(__file__))
 
 
-def write_version_py(filename=os.path.join(setup_dir, 'pyarrow/version.py')):
-    a = open(filename, 'w')
-    file_content = "\n".join(["",
-                              "# THIS FILE IS GENERATED FROM SETUP.PY",
-                              "version = '%(version)s'",
-                              "isrelease = '%(isrelease)s'"])
-
-    a.write(file_content % {'version': VERSION,
-                            'isrelease': str(ISRELEASED)})
-    a.close()
-
-
 class clean(_clean):
 
     def run(self):
@@ -272,15 +254,23 @@ def get_outputs(self):
         return [self._get_cmake_ext_path(name)
                 for name in self.get_names()]
 
-write_version_py()
-
 DESC = """\
 Python library for Apache Arrow"""
 
+# In the case of a git-archive, we don't have any version information
+# from the SCM to infer a version. The only source is the java/pom.xml.
+#
+# Note that this is only the case for git-archives. sdist tarballs have
+# all relevant information (but not the Java sources).
+if not os.path.exists('../.git') and os.path.exists('../java/pom.xml'):
+    import xml.etree.ElementTree as ET
+    tree = ET.parse('../java/pom.xml')
+    version_tag = list(tree.getroot().findall('{http://maven.apache.org/POM/4.0.0}version'))[0]
+    os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version_tag.text.replace("-SNAPSHOT", "a0")
+
 setup(
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
-    version=VERSION,
     zip_safe=False,
     package_data={'pyarrow': ['*.pxd', '*.pyx']},
     # Dummy extension to trigger build_ext
@@ -290,6 +280,8 @@ def get_outputs(self):
         'clean': clean,
         'build_ext': build_ext
     },
+    use_scm_version = {"root": "..", "relative_to": __file__},
+    setup_requires=['setuptools_scm'],
     install_requires=['cython >= 0.23', 'numpy >= 1.9', 'six >= 1.0.0'],
     description=DESC,
     license='Apache License, Version 2.0',

From 268ffbeffb1cd0617e52d381d500a2d10f61124c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 21 Dec 2016 09:31:56 +0100
Subject: [PATCH 0242/1644] ARROW-374: More precise handling of bytes vs
 unicode in Python API

Python built-in types that are not all unicode become `arrow::BinaryArray` instead of `arrow::StringArray`, since we cannot be sure that the PyBytes objects are UTF-8-encoded strings.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #249 from wesm/ARROW-374 and squashes the following commits:

1371a30 [Wes McKinney] py3 fixes
8ac3a49 [Wes McKinney] Consistently convert PyBytes to BinaryArray with pandas, too
83d1c05 [Wes McKinney] Remove print statement
c8df606 [Wes McKinney] Timestamp and time cannot be static
4a9aaf4 [Wes McKinney] Add Python interface to BinaryArray, convert PyBytes to binary instead of assuming utf8 unicode
---
 cpp/src/arrow/type.cc                        |  6 +-
 python/pyarrow/__init__.py                   |  5 +-
 python/pyarrow/array.pyx                     |  5 ++
 python/pyarrow/includes/libarrow.pxd         |  6 +-
 python/pyarrow/scalar.pyx                    | 16 +++-
 python/pyarrow/schema.pyx                    |  6 ++
 python/pyarrow/tests/test_convert_builtin.py | 31 ++++++--
 python/pyarrow/tests/test_convert_pandas.py  | 18 ++++-
 python/pyarrow/tests/test_scalars.py         | 22 ++++--
 python/src/pyarrow/adapters/builtin.cc       | 80 ++++++++++++++------
 python/src/pyarrow/adapters/pandas.cc        | 65 +++++++++++++++-
 python/src/pyarrow/helpers.cc                | 50 +++++-------
 python/src/pyarrow/helpers.h                 | 16 ----
 13 files changed, 227 insertions(+), 99 deletions(-)

diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 4748cc3c04a20..8ff9eea87051d 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -155,13 +155,11 @@ TYPE_FACTORY(binary, BinaryType);
 TYPE_FACTORY(date, DateType);
 
 std::shared_ptr<DataType> timestamp(TimeUnit unit) {
-  static std::shared_ptr<DataType> result = std::make_shared<TimestampType>();
-  return result;
+  return std::make_shared<TimestampType>(unit);
 }
 
 std::shared_ptr<DataType> time(TimeUnit unit) {
-  static std::shared_ptr<DataType> result = std::make_shared<TimeType>();
-  return result;
+  return std::make_shared<TimeType>(unit);
 }
 
 std::shared_ptr<DataType> list(const std::shared_ptr<DataType>& value_type) {
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 39ba4c72e7d3c..9ede9348c93de 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -40,13 +40,14 @@
                             BooleanValue,
                             Int8Value, Int16Value, Int32Value, Int64Value,
                             UInt8Value, UInt16Value, UInt32Value, UInt64Value,
-                            FloatValue, DoubleValue, ListValue, StringValue)
+                            FloatValue, DoubleValue, ListValue,
+                            BinaryValue, StringValue)
 
 from pyarrow.schema import (null, bool_,
                             int8, int16, int32, int64,
                             uint8, uint16, uint32, uint64,
                             timestamp, date,
-                            float_, double, string,
+                            float_, double, binary, string,
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 84f17056a19f7..c178d5ccd355b 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -238,6 +238,10 @@ cdef class StringArray(Array):
     pass
 
 
+cdef class BinaryArray(Array):
+    pass
+
+
 cdef dict _array_classes = {
     Type_NA: NullArray,
     Type_BOOL: BooleanArray,
@@ -253,6 +257,7 @@ cdef dict _array_classes = {
     Type_FLOAT: FloatArray,
     Type_DOUBLE: DoubleArray,
     Type_LIST: ListArray,
+    Type_BINARY: BinaryArray,
     Type_STRING: StringArray,
     Type_TIMESTAMP: Int64Array,
 }
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 419dd74846c92..40fb60de07ed3 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -40,6 +40,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         Type_TIMESTAMP" arrow::Type::TIMESTAMP"
         Type_DATE" arrow::Type::DATE"
+        Type_BINARY" arrow::Type::BINARY"
         Type_STRING" arrow::Type::STRING"
 
         Type_LIST" arrow::Type::LIST"
@@ -161,7 +162,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CArray] values()
         shared_ptr[CDataType] value_type()
 
-    cdef cppclass CStringArray" arrow::StringArray"(CListArray):
+    cdef cppclass CBinaryArray" arrow::BinaryArray"(CListArray):
+        const uint8_t* GetValue(int i, int32_t* length)
+
+    cdef cppclass CStringArray" arrow::StringArray"(CBinaryArray):
         c_string GetString(int i)
 
     cdef cppclass CChunkedArray" arrow::ChunkedArray":
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 623e3e483d4ae..a0610a14e6bd0 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -22,6 +22,7 @@ import pyarrow.schema as schema
 
 import datetime
 
+cimport cpython as cp
 
 NA = None
 
@@ -170,6 +171,18 @@ cdef class StringValue(ArrayValue):
         return frombytes(ap.GetString(self.index))
 
 
+cdef class BinaryValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            const uint8_t* ptr
+            int32_t length
+            CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+
+        ptr = ap.GetValue(self.index, &length)
+        return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+
+
 cdef class ListValue(ArrayValue):
 
     def __len__(self):
@@ -218,7 +231,8 @@ cdef dict _scalar_classes = {
     Type_FLOAT: FloatValue,
     Type_DOUBLE: DoubleValue,
     Type_LIST: ListValue,
-    Type_STRING: StringValue
+    Type_BINARY: BinaryValue,
+    Type_STRING: StringValue,
 }
 
 cdef object box_arrow_scalar(DataType type,
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index d05ac9ebc015a..7a69b0f12391a 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -215,6 +215,12 @@ def string():
     """
     return primitive_type(Type_STRING)
 
+def binary():
+    """
+    Binary (PyBytes-like) type
+    """
+    return primitive_type(Type_BINARY)
+
 def list_(DataType value_type):
     cdef DataType out = DataType()
     cdef shared_ptr[CDataType] list_type
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 7dc1c1b2a4828..a5f7aa51c29da 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import unittest
+from pyarrow.compat import unittest, u
 import pyarrow
 
 import datetime
@@ -71,16 +71,28 @@ def test_double(self):
         assert arr.type == pyarrow.double()
         assert arr.to_pylist() == data
 
-    def test_string(self):
-        data = ['foo', b'bar', None, 'arrow']
+    def test_unicode(self):
+        data = [u('foo'), u('bar'), None, u('arrow')]
         arr = pyarrow.from_pylist(data)
         assert len(arr) == 4
         assert arr.null_count == 1
         assert arr.type == pyarrow.string()
-        assert arr.to_pylist() == ['foo', 'bar', None, 'arrow']
+        assert arr.to_pylist() == [u('foo'), u('bar'), None, u('arrow')]
+
+    def test_bytes(self):
+        u1 = b'ma\xc3\xb1ana'
+        data = [b'foo',
+                u1.decode('utf-8'),  # unicode gets encoded,
+                None]
+        arr = pyarrow.from_pylist(data)
+        assert len(arr) == 3
+        assert arr.null_count == 1
+        assert arr.type == pyarrow.binary()
+        assert arr.to_pylist() == [b'foo', u1, None]
 
     def test_date(self):
-        data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1), datetime.date(2040, 2, 26)]
+        data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1),
+                datetime.date(2040, 2, 26)]
         arr = pyarrow.from_pylist(data)
         assert len(arr) == 4
         assert arr.type == pyarrow.date()
@@ -101,10 +113,13 @@ def test_timestamp(self):
         assert len(arr) == 4
         assert arr.type == pyarrow.timestamp()
         assert arr.null_count == 1
-        assert arr[0].as_py() == datetime.datetime(2007, 7, 13, 1, 23, 34, 123456)
+        assert arr[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                   23, 34, 123456)
         assert arr[1].as_py() is None
-        assert arr[2].as_py() == datetime.datetime(2006, 1, 13, 12, 34, 56, 432539)
-        assert arr[3].as_py() == datetime.datetime(2010, 8, 13, 5, 46, 57, 437699)
+        assert arr[2].as_py() == datetime.datetime(2006, 1, 13, 12,
+                                                   34, 56, 432539)
+        assert arr[3].as_py() == datetime.datetime(2010, 8, 13, 5,
+                                                   46, 57, 437699)
 
     def test_mixed_nesting_levels(self):
         pyarrow.from_pylist([1, 2, None])
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index cf50f3d1c2c7a..da34f85588130 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -23,6 +23,7 @@
 import pandas as pd
 import pandas.util.testing as tm
 
+from pyarrow.compat import u
 import pyarrow as A
 
 
@@ -157,13 +158,22 @@ def test_boolean_object_nulls(self):
         df = pd.DataFrame({'bools': arr})
         self._check_pandas_roundtrip(df)
 
-    def test_strings(self):
+    def test_unicode(self):
         repeats = 1000
-        values = [b'foo', None, u'bar', 'qux', np.nan]
+        values = [u('foo'), None, u('bar'), u('qux'), np.nan]
         df = pd.DataFrame({'strings': values * repeats})
 
-        values = ['foo', None, u'bar', 'qux', None]
-        expected = pd.DataFrame({'strings': values * repeats})
+        self._check_pandas_roundtrip(df)
+
+    def test_bytes_to_binary(self):
+        values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan]
+        df = pd.DataFrame({'strings': values})
+
+        table = A.from_pandas_dataframe(df)
+        assert table[0].type == A.binary()
+
+        values2 = [b'qux', b'foo', None, b'bar', b'qux', np.nan]
+        expected = pd.DataFrame({'strings': values2})
         self._check_pandas_roundtrip(df, expected)
 
     def test_timestamps_notimezone_no_nulls(self):
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 4fb850a4d47bf..19cfacbcb6b11 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import unittest, u
+from pyarrow.compat import unittest, u, unicode_type
 import pyarrow as A
 
 
@@ -58,20 +58,32 @@ def test_double(self):
         v = arr[2]
         assert v.as_py() == 3.0
 
-    def test_string(self):
-        arr = A.from_pylist(['foo', None, u('bar')])
+    def test_string_unicode(self):
+        arr = A.from_pylist([u('foo'), None, u('bar')])
 
         v = arr[0]
         assert isinstance(v, A.StringValue)
-        assert repr(v) == "'foo'"
         assert v.as_py() == 'foo'
 
         assert arr[1] is A.NA
 
         v = arr[2].as_py()
-        assert v == 'bar'
+        assert v == u('bar')
         assert isinstance(v, str)
 
+    def test_bytes(self):
+        arr = A.from_pylist([b'foo', None, u('bar')])
+
+        v = arr[0]
+        assert isinstance(v, A.BinaryValue)
+        assert v.as_py() == b'foo'
+
+        assert arr[1] is A.NA
+
+        v = arr[2].as_py()
+        assert v == b'bar'
+        assert isinstance(v, bytes)
+
     def test_list(self):
         arr = A.from_pylist([['foo', None], None, ['bar'], []])
 
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index e0cb7c20be3d5..2a13944b35c1c 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -42,14 +42,6 @@ static inline bool IsPyInteger(PyObject* obj) {
 #endif
 }
 
-static inline bool IsPyBaseString(PyObject* obj) {
-#if PYARROW_IS_PY2
-  return PyString_Check(obj) || PyUnicode_Check(obj);
-#else
-  return PyUnicode_Check(obj);
-#endif
-}
-
 class ScalarVisitor {
  public:
   ScalarVisitor() :
@@ -60,7 +52,8 @@ class ScalarVisitor {
       date_count_(0),
       timestamp_count_(0),
       float_count_(0),
-      string_count_(0) {}
+      binary_count_(0),
+      unicode_count_(0) {}
 
   void Visit(PyObject* obj) {
     ++total_count_;
@@ -76,8 +69,10 @@ class ScalarVisitor {
       ++date_count_;
     } else if (PyDateTime_CheckExact(obj)) {
       ++timestamp_count_;
-    } else if (IsPyBaseString(obj)) {
-      ++string_count_;
+    } else if (PyBytes_Check(obj)) {
+      ++binary_count_;
+    } else if (PyUnicode_Check(obj)) {
+      ++unicode_count_;
     } else {
       // TODO(wesm): accumulate error information somewhere
     }
@@ -86,20 +81,22 @@ class ScalarVisitor {
   std::shared_ptr<DataType> GetType() {
     // TODO(wesm): handling mixed-type cases
     if (float_count_) {
-      return DOUBLE;
+      return arrow::float64();
     } else if (int_count_) {
       // TODO(wesm): tighter type later
-      return INT64;
+      return arrow::int64();
     } else if (date_count_) {
-      return DATE;
+      return arrow::date();
     } else if (timestamp_count_) {
-      return TIMESTAMP_US;
+      return arrow::timestamp(arrow::TimeUnit::MICRO);
     } else if (bool_count_) {
-      return BOOL;
-    } else if (string_count_) {
-      return STRING;
+      return arrow::boolean();
+    } else if (binary_count_) {
+      return arrow::binary();
+    } else if (unicode_count_) {
+      return arrow::utf8();
     } else {
-      return NA;
+      return arrow::null();
     }
   }
 
@@ -115,7 +112,8 @@ class ScalarVisitor {
   int64_t date_count_;
   int64_t timestamp_count_;
   int64_t float_count_;
-  int64_t string_count_;
+  int64_t binary_count_;
+  int64_t unicode_count_;
 
   // Place to accumulate errors
   // std::vector<Status> errors_;
@@ -163,7 +161,7 @@ class SeqVisitor {
   std::shared_ptr<DataType> GetType() {
     if (scalars_.total_count() == 0) {
       if (max_nesting_level_ == 0) {
-        return NA;
+        return arrow::null();
       } else {
         return nullptr;
       }
@@ -227,7 +225,7 @@ static Status InferArrowType(PyObject* obj, int64_t* size,
 
   // For 0-length sequences, refuse to guess
   if (*size == 0) {
-    *out_type = NA;
+    *out_type = arrow::null();
   }
 
   SeqVisitor seq_visitor;
@@ -381,7 +379,7 @@ class DoubleConverter : public TypedConverter<arrow::DoubleBuilder> {
   }
 };
 
-class StringConverter : public TypedConverter<arrow::StringBuilder> {
+class BytesConverter : public TypedConverter<arrow::BinaryBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     PyObject* item;
@@ -415,6 +413,38 @@ class StringConverter : public TypedConverter<arrow::StringBuilder> {
   }
 };
 
+class UTF8Converter : public TypedConverter<arrow::StringBuilder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    PyObject* item;
+    PyObject* bytes_obj;
+    OwnedRef tmp;
+    const char* bytes;
+    int32_t length;
+    Py_ssize_t size = PySequence_Size(seq);
+    for (int64_t i = 0; i < size; ++i) {
+      item = PySequence_GetItem(seq, i);
+      OwnedRef holder(item);
+
+      if (item == Py_None) {
+        RETURN_NOT_OK(typed_builder_->AppendNull());
+        continue;
+      } else if (!PyUnicode_Check(item)) {
+        return Status::TypeError("Non-unicode value encountered");
+      }
+      tmp.reset(PyUnicode_AsUTF8String(item));
+      RETURN_IF_PYERROR();
+      bytes_obj = tmp.obj();
+
+      // No error checking
+      length = PyBytes_GET_SIZE(bytes_obj);
+      bytes = PyBytes_AS_STRING(bytes_obj);
+      RETURN_NOT_OK(typed_builder_->Append(bytes, length));
+    }
+    return Status::OK();
+  }
+};
+
 class ListConverter : public TypedConverter<arrow::ListBuilder> {
  public:
   Status Init(const std::shared_ptr<ArrayBuilder>& builder) override;
@@ -449,8 +479,10 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
       return std::make_shared<TimestampConverter>();
     case Type::DOUBLE:
       return std::make_shared<DoubleConverter>();
+    case Type::BINARY:
+      return std::make_shared<BytesConverter>();
     case Type::STRING:
-      return std::make_shared<StringConverter>();
+      return std::make_shared<UTF8Converter>();
     case Type::LIST:
       return std::make_shared<ListConverter>();
     case Type::STRUCT:
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index f8dff6d824153..38f3b6f5248ee 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -193,6 +193,9 @@ class ArrowSerializer {
   Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
     PyAcquireGIL lock;
 
+    // The output type at this point is inconclusive because there may be bytes
+    // and unicode mixed in the object array
+
     PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     arrow::TypePtr string_type(new arrow::StringType());
     arrow::StringBuilder string_builder(pool_, string_type);
@@ -200,6 +203,7 @@ class ArrowSerializer {
 
     Status s;
     PyObject* obj;
+    bool have_bytes = false;
     for (int64_t i = 0; i < length_; ++i) {
       obj = objects[i];
       if (PyUnicode_Check(obj)) {
@@ -215,13 +219,21 @@ class ArrowSerializer {
           return s;
         }
       } else if (PyBytes_Check(obj)) {
+        have_bytes = true;
         const int32_t length = PyBytes_GET_SIZE(obj);
         RETURN_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
       } else {
         string_builder.AppendNull();
       }
     }
-    return string_builder.Finish(out);
+    RETURN_NOT_OK(string_builder.Finish(out));
+
+    if (have_bytes) {
+      const auto& arr = static_cast<const arrow::StringArray&>(*out->get());
+      *out = std::make_shared<arrow::BinaryArray>(arr.length(), arr.offsets(),
+          arr.data(), arr.null_count(), arr.null_bitmap());
+    }
+    return Status::OK();
   }
 
   Status ConvertBooleans(std::shared_ptr<Array>* out) {
@@ -865,7 +877,7 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
-  // UTF8
+  // UTF8 strings
   template <int T2>
   inline typename std::enable_if<
     T2 == arrow::Type::STRING, Status>::type
@@ -912,6 +924,54 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
+  template <int T2>
+  inline typename std::enable_if<
+    T2 == arrow::Type::BINARY, Status>::type
+  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
+    size_t chunk_offset = 0;
+    PyAcquireGIL lock;
+
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+
+    for (int c = 0; c < data->num_chunks(); c++) {
+      const std::shared_ptr<Array> arr = data->chunk(c);
+      auto binary_arr = static_cast<arrow::BinaryArray*>(arr.get());
+      auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_)) + chunk_offset;
+
+      const uint8_t* data_ptr;
+      int32_t length;
+      if (data->null_count() > 0) {
+        for (int64_t i = 0; i < arr->length(); ++i) {
+          if (binary_arr->IsNull(i)) {
+            Py_INCREF(Py_None);
+            out_values[i] = Py_None;
+          } else {
+            data_ptr = binary_arr->GetValue(i, &length);
+
+            out_values[i] = PyBytes_FromStringAndSize(
+                reinterpret_cast<const char*>(data_ptr), length);
+            if (out_values[i] == nullptr) {
+              return Status::UnknownError("String initialization failed");
+            }
+          }
+        }
+      } else {
+        for (int64_t i = 0; i < arr->length(); ++i) {
+          data_ptr = binary_arr->GetValue(i, &length);
+          out_values[i] = PyBytes_FromStringAndSize(
+              reinterpret_cast<const char*>(data_ptr), length);
+          if (out_values[i] == nullptr) {
+            return Status::UnknownError("String initialization failed");
+          }
+        }
+      }
+
+      chunk_offset += binary_arr->length();
+    }
+
+    return Status::OK();
+  }
+
  private:
   std::shared_ptr<Column> col_;
   PyObject* py_ref_;
@@ -948,6 +1008,7 @@ Status ConvertColumnToPandas(const std::shared_ptr<Column>& col, PyObject* py_re
     FROM_ARROW_CASE(UINT64);
     FROM_ARROW_CASE(FLOAT);
     FROM_ARROW_CASE(DOUBLE);
+    FROM_ARROW_CASE(BINARY);
     FROM_ARROW_CASE(STRING);
     FROM_ARROW_CASE(DATE);
     FROM_ARROW_CASE(TIMESTAMP);
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
index af9274484935f..b42199c8e041c 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/python/src/pyarrow/helpers.cc
@@ -23,47 +23,33 @@ using namespace arrow;
 
 namespace pyarrow {
 
-const std::shared_ptr<NullType> NA = std::make_shared<NullType>();
-const std::shared_ptr<BooleanType> BOOL = std::make_shared<BooleanType>();
-const std::shared_ptr<UInt8Type> UINT8 = std::make_shared<UInt8Type>();
-const std::shared_ptr<UInt16Type> UINT16 = std::make_shared<UInt16Type>();
-const std::shared_ptr<UInt32Type> UINT32 = std::make_shared<UInt32Type>();
-const std::shared_ptr<UInt64Type> UINT64 = std::make_shared<UInt64Type>();
-const std::shared_ptr<Int8Type> INT8 = std::make_shared<Int8Type>();
-const std::shared_ptr<Int16Type> INT16 = std::make_shared<Int16Type>();
-const std::shared_ptr<Int32Type> INT32 = std::make_shared<Int32Type>();
-const std::shared_ptr<Int64Type> INT64 = std::make_shared<Int64Type>();
-const std::shared_ptr<DateType> DATE = std::make_shared<DateType>();
-const std::shared_ptr<TimestampType> TIMESTAMP_US = std::make_shared<TimestampType>(TimeUnit::MICRO);
-const std::shared_ptr<FloatType> FLOAT = std::make_shared<FloatType>();
-const std::shared_ptr<DoubleType> DOUBLE = std::make_shared<DoubleType>();
-const std::shared_ptr<StringType> STRING = std::make_shared<StringType>();
 
-#define GET_PRIMITIVE_TYPE(NAME, Class)         \
+#define GET_PRIMITIVE_TYPE(NAME, FACTORY)       \
   case Type::NAME:                              \
-    return NAME;                                \
+    return FACTORY();                           \
     break;
 
 std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
   switch (type) {
     case Type::NA:
-      return NA;
-    GET_PRIMITIVE_TYPE(UINT8, UInt8Type);
-    GET_PRIMITIVE_TYPE(INT8, Int8Type);
-    GET_PRIMITIVE_TYPE(UINT16, UInt16Type);
-    GET_PRIMITIVE_TYPE(INT16, Int16Type);
-    GET_PRIMITIVE_TYPE(UINT32, UInt32Type);
-    GET_PRIMITIVE_TYPE(INT32, Int32Type);
-    GET_PRIMITIVE_TYPE(UINT64, UInt64Type);
-    GET_PRIMITIVE_TYPE(INT64, Int64Type);
-    GET_PRIMITIVE_TYPE(DATE, DateType);
+      return null();
+    GET_PRIMITIVE_TYPE(UINT8, uint8);
+    GET_PRIMITIVE_TYPE(INT8, int8);
+    GET_PRIMITIVE_TYPE(UINT16, uint16);
+    GET_PRIMITIVE_TYPE(INT16, int16);
+    GET_PRIMITIVE_TYPE(UINT32, uint32);
+    GET_PRIMITIVE_TYPE(INT32, int32);
+    GET_PRIMITIVE_TYPE(UINT64, uint64);
+    GET_PRIMITIVE_TYPE(INT64, int64);
+    GET_PRIMITIVE_TYPE(DATE, date);
     case Type::TIMESTAMP:
-      return TIMESTAMP_US;
+      return arrow::timestamp(arrow::TimeUnit::MICRO);
       break;
-    GET_PRIMITIVE_TYPE(BOOL, BooleanType);
-    GET_PRIMITIVE_TYPE(FLOAT, FloatType);
-    GET_PRIMITIVE_TYPE(DOUBLE, DoubleType);
-    GET_PRIMITIVE_TYPE(STRING, StringType);
+    GET_PRIMITIVE_TYPE(BOOL, boolean);
+    GET_PRIMITIVE_TYPE(FLOAT, float32);
+    GET_PRIMITIVE_TYPE(DOUBLE, float64);
+    GET_PRIMITIVE_TYPE(BINARY, binary);
+    GET_PRIMITIVE_TYPE(STRING, utf8);
     default:
       return nullptr;
   }
diff --git a/python/src/pyarrow/helpers.h b/python/src/pyarrow/helpers.h
index e714bba5db4cc..8334d974c0237 100644
--- a/python/src/pyarrow/helpers.h
+++ b/python/src/pyarrow/helpers.h
@@ -28,22 +28,6 @@ namespace pyarrow {
 using arrow::DataType;
 using arrow::Type;
 
-extern const std::shared_ptr<arrow::NullType> NA;
-extern const std::shared_ptr<arrow::BooleanType> BOOL;
-extern const std::shared_ptr<arrow::UInt8Type> UINT8;
-extern const std::shared_ptr<arrow::UInt16Type> UINT16;
-extern const std::shared_ptr<arrow::UInt32Type> UINT32;
-extern const std::shared_ptr<arrow::UInt64Type> UINT64;
-extern const std::shared_ptr<arrow::Int8Type> INT8;
-extern const std::shared_ptr<arrow::Int16Type> INT16;
-extern const std::shared_ptr<arrow::Int32Type> INT32;
-extern const std::shared_ptr<arrow::Int64Type> INT64;
-extern const std::shared_ptr<arrow::DateType> DATE;
-extern const std::shared_ptr<arrow::TimestampType> TIMESTAMP_US;
-extern const std::shared_ptr<arrow::FloatType> FLOAT;
-extern const std::shared_ptr<arrow::DoubleType> DOUBLE;
-extern const std::shared_ptr<arrow::StringType> STRING;
-
 PYARROW_EXPORT
 std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
 

From fd4eb98af9bbf19b7a640b55e2d8ed5ad87b6af1 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 21 Dec 2016 16:50:55 +0100
Subject: [PATCH 0243/1644] ARROW-440: [C++] Support pkg-config

pkg-config is a tool to get build flags.

If Arrow supports pkg-config, users can set build flags easily.

For example, CMake supports pkg-config.

To support pkg-config, we just install .pc file that includes build
flags information.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #250 from kou/ARROW-440-support-pkg-config and squashes the following commits:

f35fc44 [Kouhei Sutou] ARROW-440: [C++] Support pkg-config
---
 cpp/src/arrow/CMakeLists.txt      |  8 ++++++++
 cpp/src/arrow/arrow.pc.in         | 26 ++++++++++++++++++++++++++
 cpp/src/arrow/io/CMakeLists.txt   |  8 ++++++++
 cpp/src/arrow/io/arrow-io.pc.in   | 27 +++++++++++++++++++++++++++
 cpp/src/arrow/ipc/CMakeLists.txt  |  8 ++++++++
 cpp/src/arrow/ipc/arrow-ipc.pc.in | 27 +++++++++++++++++++++++++++
 6 files changed, 104 insertions(+)
 create mode 100644 cpp/src/arrow/arrow.pc.in
 create mode 100644 cpp/src/arrow/io/arrow-io.pc.in
 create mode 100644 cpp/src/arrow/ipc/arrow-ipc.pc.in

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index b8500ab264f80..f8c50513d31a5 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -33,6 +33,14 @@ install(FILES
   test-util.h
   DESTINATION include/arrow)
 
+# pkg-config support
+configure_file(arrow.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/arrow.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow.pc"
+  DESTINATION "lib/pkgconfig/")
+
 #######################################
 # Unit tests
 #######################################
diff --git a/cpp/src/arrow/arrow.pc.in b/cpp/src/arrow/arrow.pc.in
new file mode 100644
index 0000000000000..5ad429b714893
--- /dev/null
+++ b/cpp/src/arrow/arrow.pc.in
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+libdir=${prefix}/lib
+includedir=${prefix}/include
+
+Name: Apache Arrow
+Description: Arrow is a set of technologies that enable big-data systems to process and move data fast.
+Version: @ARROW_VERSION@
+Libs: -L${libdir} -larrow
+Cflags: -I${includedir}
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index e2b6496cc3f87..2062cd43b7b48 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -134,3 +134,11 @@ install(FILES
 install(TARGETS arrow_io
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
+
+# pkg-config support
+configure_file(arrow-io.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/arrow-io.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-io.pc"
+  DESTINATION "lib/pkgconfig/")
diff --git a/cpp/src/arrow/io/arrow-io.pc.in b/cpp/src/arrow/io/arrow-io.pc.in
new file mode 100644
index 0000000000000..4b4abdd62df42
--- /dev/null
+++ b/cpp/src/arrow/io/arrow-io.pc.in
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+libdir=${prefix}/lib
+includedir=${prefix}/include
+
+Name: Apache Arrow I/O
+Description: I/O interface for Arrow.
+Version: @ARROW_VERSION@
+Libs: -L${libdir} -larrow_io
+Cflags: -I${includedir}
+Requires: arrow
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 619ca7c92cb7a..d3e625a08fbfe 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -159,3 +159,11 @@ install(FILES
 install(TARGETS arrow_ipc
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
+
+# pkg-config support
+configure_file(arrow-ipc.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/arrow-ipc.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-ipc.pc"
+  DESTINATION "lib/pkgconfig/")
diff --git a/cpp/src/arrow/ipc/arrow-ipc.pc.in b/cpp/src/arrow/ipc/arrow-ipc.pc.in
new file mode 100644
index 0000000000000..73b44c99f0430
--- /dev/null
+++ b/cpp/src/arrow/ipc/arrow-ipc.pc.in
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+libdir=${prefix}/lib
+includedir=${prefix}/include
+
+Name: Apache Arrow IPC
+Description: IPC extension for Arrow.
+Version: @ARROW_VERSION@
+Libs: -L${libdir} -larrow_ipc
+Cflags: -I${includedir}
+Requires: arrow-io

From 65af9ea16a3c9241a66203b57cbfe2041a5ee52b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 23 Dec 2016 17:29:17 -0500
Subject: [PATCH 0244/1644] ARROW-432: [Python] Construct precise pandas
 BlockManager structure for zero-copy DataFrame initialization

This avoids "memory tripling" (because pd.DataFrame will often immediately consolidate the arrays) and also will allow the Arrow->pandas deserialization to be parallelized for further performance gains.

@xhochy this also has the effect of coercing all timestamps to `datetime[ns]` -- for pandas I believe this is the proper behavior, but wanted to run it by you.

In a local benchmark on roughly 1GB of data I have:

setup code:

```python
import numpy as np
import pandas as pd
import pyarrow as pa

DATA_SIZE = (1 << 30)
NCOLS = 100

data = np.random.randn(NCOLS, DATA_SIZE / NCOLS / 8).T
data[::2] = np.nan

df = pd.DataFrame(data, columns=['c' + str(i) for i in range(NCOLS)])

table = pa.Table.from_pandas(df)
```

before these changes (I added `block_based` argument to toggle this code path off):

```python
In [4]: %timeit df2 = table.to_pandas(block_based=False)
1 loop, best of 3: 252 ms per loop
```

```python
In [5]: %timeit df2 = table.to_pandas()
10 loops, best of 3: 139 ms per loop
```

This takes the effective in-memory bandwidth on numerical data from 3.97 GB/s to 7.19 GB/s.

I also moved the clang-format files to the top level so we can more easily run code formatting on the C++ code under python/.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #251 from wesm/ARROW-432 and squashes the following commits:

f22e1b5 [Wes McKinney] Remove unneeded import
ea83ded [Wes McKinney] Unit tests pass again
ec239b8 [Wes McKinney] Fix DataFrame constructions, code formatting
af960ee [Wes McKinney] Draft blocks -> DataFrame scaffold
110692f [Wes McKinney] First draft of scaffolding for creating precise pandas.DataFrame block structure
4928a0b [Wes McKinney] Refactor post rebase
c89cfaf [Wes McKinney] Rearrange to-pandas deserialization to better permit reads into pre-allocated memory
---
 cpp/src/.clang-format => .clang-format        |    0
 cpp/src/.clang-tidy => .clang-tidy            |    0
 .../.clang-tidy-ignore => .clang-tidy-ignore  |    0
 cpp/CMakeLists.txt                            |    3 +-
 python/pyarrow/includes/pyarrow.pxd           |    5 +-
 python/pyarrow/table.pyx                      |   54 +-
 python/src/pyarrow/adapters/builtin.cc        |   66 +-
 python/src/pyarrow/adapters/builtin.h         |    4 +-
 python/src/pyarrow/adapters/pandas.cc         | 1184 ++++++++++++-----
 python/src/pyarrow/adapters/pandas.h          |   33 +-
 python/src/pyarrow/api.h                      |    2 +-
 python/src/pyarrow/common.cc                  |    4 +-
 python/src/pyarrow/common.h                   |   52 +-
 python/src/pyarrow/config.cc                  |    5 +-
 python/src/pyarrow/config.h                   |    6 +-
 python/src/pyarrow/helpers.cc                 |   37 +-
 python/src/pyarrow/helpers.h                  |    4 +-
 python/src/pyarrow/io.cc                      |   14 +-
 python/src/pyarrow/io.h                       |    6 +-
 python/src/pyarrow/numpy_interop.h            |    4 +-
 python/src/pyarrow/util/datetime.h            |    8 +-
 python/src/pyarrow/util/test_main.cc          |    2 +-
 22 files changed, 986 insertions(+), 507 deletions(-)
 rename cpp/src/.clang-format => .clang-format (100%)
 rename cpp/src/.clang-tidy => .clang-tidy (100%)
 rename cpp/src/.clang-tidy-ignore => .clang-tidy-ignore (100%)

diff --git a/cpp/src/.clang-format b/.clang-format
similarity index 100%
rename from cpp/src/.clang-format
rename to .clang-format
diff --git a/cpp/src/.clang-tidy b/.clang-tidy
similarity index 100%
rename from cpp/src/.clang-tidy
rename to .clang-tidy
diff --git a/cpp/src/.clang-tidy-ignore b/.clang-tidy-ignore
similarity index 100%
rename from cpp/src/.clang-tidy-ignore
rename to .clang-tidy-ignore
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 315995ce7cb97..93e9853df8972 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -741,7 +741,8 @@ endif (UNIX)
 if (${CLANG_FORMAT_FOUND})
   # runs clang format and updates files in place.
   add_custom_target(format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1
-  `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`)
+    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`
+    `find ${CMAKE_CURRENT_SOURCE_DIR}/../python -name \\*.cc -or -name \\*.h`)
 
   # runs clang format and exits with a non-zero exit code if any files need to be reformatted
   add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index a5444c236bcc8..dc6ccd2025932 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -18,7 +18,7 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn,
+from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CTable,
                                         CDataType, CStatus, Type, MemoryPool)
 
 cimport pyarrow.includes.libarrow_io as arrow_io
@@ -39,6 +39,9 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     CStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
                                   PyObject* py_ref, PyObject** out)
 
+    CStatus ConvertTableToPandas(const shared_ptr[CTable]& table,
+                                 int nthreads, PyObject** out)
+
     MemoryPool* get_memory_pool()
 
 
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 2f7d4309e4518..9375557888490 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -430,6 +430,32 @@ cdef class RecordBatch:
         return result
 
 
+cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
+    cdef:
+        PyObject* result_obj
+        CColumn* col
+        int i
+
+    from pandas.core.internals import BlockManager, make_block
+    from pandas import RangeIndex
+
+    check_status(pyarrow.ConvertTableToPandas(table, nthreads, &result_obj))
+
+    result = PyObject_to_object(result_obj)
+
+    blocks = []
+    for block_arr, placement_arr in result:
+        blocks.append(make_block(block_arr, placement=placement_arr))
+
+    names = []
+    for i in range(table.get().num_columns()):
+        col = table.get().column(i).get()
+        names.append(frombytes(col.name()))
+
+    axes = [names, RangeIndex(table.get().num_rows())]
+    return BlockManager(blocks, axes)
+
+
 cdef class Table:
     """
     A collection of top-level named, equal length Arrow arrays.
@@ -584,7 +610,7 @@ cdef class Table:
         table.init(c_table)
         return table
 
-    def to_pandas(self):
+    def to_pandas(self, nthreads=1, block_based=True):
         """
         Convert the arrow::Table to a pandas DataFrame
 
@@ -599,17 +625,21 @@ cdef class Table:
 
         import pandas as pd
 
-        names = []
-        data = []
-        for i in range(self.table.num_columns()):
-            col = self.table.column(i)
-            column = self.column(i)
-            check_status(pyarrow.ConvertColumnToPandas(
-                col, <PyObject*> column, &arr))
-            names.append(frombytes(col.get().name()))
-            data.append(PyObject_to_object(arr))
-
-        return pd.DataFrame(dict(zip(names, data)), columns=names)
+        if block_based:
+            mgr = table_to_blockmanager(self.sp_table, nthreads)
+            return pd.DataFrame(mgr)
+        else:
+            names = []
+            data = []
+            for i in range(self.table.num_columns()):
+                col = self.table.column(i)
+                column = self.column(i)
+                check_status(pyarrow.ConvertColumnToPandas(
+                    col, <PyObject*> column, &arr))
+                names.append(frombytes(col.get().name()))
+                data.append(PyObject_to_object(arr))
+
+            return pd.DataFrame(dict(zip(names, data)), columns=names)
 
     @property
     def name(self):
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index 2a13944b35c1c..fb7475f0c9407 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -44,16 +44,16 @@ static inline bool IsPyInteger(PyObject* obj) {
 
 class ScalarVisitor {
  public:
-  ScalarVisitor() :
-      total_count_(0),
-      none_count_(0),
-      bool_count_(0),
-      int_count_(0),
-      date_count_(0),
-      timestamp_count_(0),
-      float_count_(0),
-      binary_count_(0),
-      unicode_count_(0) {}
+  ScalarVisitor()
+      : total_count_(0),
+        none_count_(0),
+        bool_count_(0),
+        int_count_(0),
+        date_count_(0),
+        timestamp_count_(0),
+        float_count_(0),
+        binary_count_(0),
+        unicode_count_(0) {}
 
   void Visit(PyObject* obj) {
     ++total_count_;
@@ -100,9 +100,7 @@ class ScalarVisitor {
     }
   }
 
-  int64_t total_count() const {
-    return total_count_;
-  }
+  int64_t total_count() const { return total_count_; }
 
  private:
   int64_t total_count_;
@@ -123,17 +121,14 @@ static constexpr int MAX_NESTING_LEVELS = 32;
 
 class SeqVisitor {
  public:
-  SeqVisitor() :
-      max_nesting_level_(0) {
+  SeqVisitor() : max_nesting_level_(0) {
     memset(nesting_histogram_, 0, MAX_NESTING_LEVELS * sizeof(int));
   }
 
-  Status Visit(PyObject* obj, int level=0) {
+  Status Visit(PyObject* obj, int level = 0) {
     Py_ssize_t size = PySequence_Size(obj);
 
-    if (level > max_nesting_level_) {
-      max_nesting_level_ = level;
-    }
+    if (level > max_nesting_level_) { max_nesting_level_ = level; }
 
     for (int64_t i = 0; i < size; ++i) {
       // TODO(wesm): Error checking?
@@ -188,9 +183,7 @@ class SeqVisitor {
   int max_observed_level() const {
     int result = 0;
     for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
-      if (nesting_histogram_[i] > 0) {
-        result = i;
-      }
+      if (nesting_histogram_[i] > 0) { result = i; }
     }
     return result;
   }
@@ -198,9 +191,7 @@ class SeqVisitor {
   int num_nesting_levels() const {
     int result = 0;
     for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
-      if (nesting_histogram_[i] > 0) {
-        ++result;
-      }
+      if (nesting_histogram_[i] > 0) { ++result; }
     }
     return result;
   }
@@ -214,8 +205,8 @@ class SeqVisitor {
 };
 
 // Non-exhaustive type inference
-static Status InferArrowType(PyObject* obj, int64_t* size,
-    std::shared_ptr<DataType>* out_type) {
+static Status InferArrowType(
+    PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
   *size = PySequence_Size(obj);
   if (PyErr_Occurred()) {
     // Not a sequence
@@ -224,9 +215,7 @@ static Status InferArrowType(PyObject* obj, int64_t* size,
   }
 
   // For 0-length sequences, refuse to guess
-  if (*size == 0) {
-    *out_type = arrow::null();
-  }
+  if (*size == 0) { *out_type = arrow::null(); }
 
   SeqVisitor seq_visitor;
   RETURN_NOT_OK(seq_visitor.Visit(obj));
@@ -234,9 +223,7 @@ static Status InferArrowType(PyObject* obj, int64_t* size,
 
   *out_type = seq_visitor.GetType();
 
-  if (*out_type == nullptr) {
-    return Status::TypeError("Unable to determine data type");
-  }
+  if (*out_type == nullptr) { return Status::TypeError("Unable to determine data type"); }
 
   return Status::OK();
 }
@@ -337,7 +324,8 @@ class TimestampConverter : public TypedConverter<arrow::TimestampBuilder> {
       if (item.obj() == Py_None) {
         typed_builder_->AppendNull();
       } else {
-        PyDateTime_DateTime* pydatetime = reinterpret_cast<PyDateTime_DateTime*>(item.obj());
+        PyDateTime_DateTime* pydatetime =
+            reinterpret_cast<PyDateTime_DateTime*>(item.obj());
         struct tm datetime = {0};
         datetime.tm_year = PyDateTime_GET_YEAR(pydatetime) - 1900;
         datetime.tm_mon = PyDateTime_GET_MONTH(pydatetime) - 1;
@@ -462,6 +450,7 @@ class ListConverter : public TypedConverter<arrow::ListBuilder> {
     }
     return Status::OK();
   }
+
  protected:
   std::shared_ptr<SeqConverter> value_converter_;
 };
@@ -496,8 +485,8 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
   builder_ = builder;
   typed_builder_ = static_cast<arrow::ListBuilder*>(builder.get());
 
-  value_converter_ = GetConverter(static_cast<arrow::ListType*>(
-          builder->type().get())->value_type());
+  value_converter_ =
+      GetConverter(static_cast<arrow::ListType*>(builder->type().get())->value_type());
   if (value_converter_ == nullptr) {
     return Status::NotImplemented("value type not implemented");
   }
@@ -521,8 +510,7 @@ Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
   std::shared_ptr<SeqConverter> converter = GetConverter(type);
   if (converter == nullptr) {
     std::stringstream ss;
-    ss << "No type converter implemented for "
-       << type->ToString();
+    ss << "No type converter implemented for " << type->ToString();
     return Status::NotImplemented(ss.str());
   }
 
@@ -536,4 +524,4 @@ Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
   return builder->Finish(out);
 }
 
-} // namespace pyarrow
+}  // namespace pyarrow
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
index 2ddfdaaf44134..1ff36945c88c7 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -40,6 +40,6 @@ namespace pyarrow {
 PYARROW_EXPORT
 arrow::Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out);
 
-} // namespace pyarrow
+}  // namespace pyarrow
 
-#endif // PYARROW_ADAPTERS_BUILTIN_H
+#endif  // PYARROW_ADAPTERS_BUILTIN_H
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 38f3b6f5248ee..899eb5519d562 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -28,10 +28,13 @@
 #include <memory>
 #include <sstream>
 #include <string>
+#include <unordered_map>
 
 #include "arrow/api.h"
-#include "arrow/util/bit-util.h"
 #include "arrow/status.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/macros.h"
 
 #include "pyarrow/common.h"
 #include "pyarrow/config.h"
@@ -40,10 +43,13 @@
 namespace pyarrow {
 
 using arrow::Array;
+using arrow::ChunkedArray;
 using arrow::Column;
 using arrow::Field;
 using arrow::DataType;
 using arrow::Status;
+using arrow::Table;
+using arrow::Type;
 
 namespace BitUtil = arrow::BitUtil;
 
@@ -51,8 +57,7 @@ namespace BitUtil = arrow::BitUtil;
 // Serialization
 
 template <int TYPE>
-struct npy_traits {
-};
+struct npy_traits {};
 
 template <>
 struct npy_traits<NPY_BOOL> {
@@ -60,21 +65,17 @@ struct npy_traits<NPY_BOOL> {
   using TypeClass = arrow::BooleanType;
 
   static constexpr bool supports_nulls = false;
-  static inline bool isnull(uint8_t v) {
-    return false;
-  }
+  static inline bool isnull(uint8_t v) { return false; }
 };
 
-#define NPY_INT_DECL(TYPE, CapType, T)              \
-  template <>                                       \
-  struct npy_traits<NPY_##TYPE> {                   \
-    typedef T value_type;                           \
-    using TypeClass = arrow::CapType##Type;         \
-                                                    \
-    static constexpr bool supports_nulls = false;   \
-    static inline bool isnull(T v) {                \
-      return false;                                 \
-    }                                               \
+#define NPY_INT_DECL(TYPE, CapType, T)               \
+  template <>                                        \
+  struct npy_traits<NPY_##TYPE> {                    \
+    typedef T value_type;                            \
+    using TypeClass = arrow::CapType##Type;          \
+                                                     \
+    static constexpr bool supports_nulls = false;    \
+    static inline bool isnull(T v) { return false; } \
   };
 
 NPY_INT_DECL(INT8, Int8, int8_t);
@@ -93,9 +94,7 @@ struct npy_traits<NPY_FLOAT32> {
 
   static constexpr bool supports_nulls = true;
 
-  static inline bool isnull(float v) {
-    return v != v;
-  }
+  static inline bool isnull(float v) { return v != v; }
 };
 
 template <>
@@ -105,9 +104,7 @@ struct npy_traits<NPY_FLOAT64> {
 
   static constexpr bool supports_nulls = true;
 
-  static inline bool isnull(double v) {
-    return v != v;
-  }
+  static inline bool isnull(double v) { return v != v; }
 };
 
 template <>
@@ -135,18 +132,14 @@ struct npy_traits<NPY_OBJECT> {
 template <int TYPE>
 class ArrowSerializer {
  public:
-  ArrowSerializer(arrow::MemoryPool* pool, PyArrayObject* arr, PyArrayObject* mask) :
-      pool_(pool),
-      arr_(arr),
-      mask_(mask) {
+  ArrowSerializer(arrow::MemoryPool* pool, PyArrayObject* arr, PyArrayObject* mask)
+      : pool_(pool), arr_(arr), mask_(mask) {
     length_ = PyArray_SIZE(arr_);
   }
 
   Status Convert(std::shared_ptr<Array>* out);
 
-  int stride() const {
-    return PyArray_STRIDES(arr_)[0];
-  }
+  int stride() const { return PyArray_STRIDES(arr_)[0]; }
 
   Status InitNullBitmap() {
     int null_bytes = BitUtil::BytesForBits(length_);
@@ -215,9 +208,7 @@ class ArrowSerializer {
         const int32_t length = PyBytes_GET_SIZE(obj);
         s = string_builder.Append(PyBytes_AS_STRING(obj), length);
         Py_DECREF(obj);
-        if (!s.ok()) {
-          return s;
-        }
+        if (!s.ok()) { return s; }
       } else if (PyBytes_Check(obj)) {
         have_bytes = true;
         const int32_t length = PyBytes_GET_SIZE(obj);
@@ -230,8 +221,8 @@ class ArrowSerializer {
 
     if (have_bytes) {
       const auto& arr = static_cast<const arrow::StringArray&>(*out->get());
-      *out = std::make_shared<arrow::BinaryArray>(arr.length(), arr.offsets(),
-          arr.data(), arr.null_count(), arr.null_bitmap());
+      *out = std::make_shared<arrow::BinaryArray>(
+          arr.length(), arr.offsets(), arr.data(), arr.null_count(), arr.null_bitmap());
     }
     return Status::OK();
   }
@@ -259,8 +250,7 @@ class ArrowSerializer {
       }
     }
 
-    *out = std::make_shared<arrow::BooleanArray>(length_, data, null_count,
-        null_bitmap_);
+    *out = std::make_shared<arrow::BooleanArray>(length_, data, null_count, null_bitmap_);
 
     return Status::OK();
   }
@@ -321,26 +311,27 @@ inline Status ArrowSerializer<TYPE>::MakeDataType(std::shared_ptr<DataType>* out
 }
 
 template <>
-inline Status ArrowSerializer<NPY_DATETIME>::MakeDataType(std::shared_ptr<DataType>* out) {
+inline Status ArrowSerializer<NPY_DATETIME>::MakeDataType(
+    std::shared_ptr<DataType>* out) {
   PyArray_Descr* descr = PyArray_DESCR(arr_);
   auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
   arrow::TimestampType::Unit unit;
 
   switch (date_dtype->meta.base) {
-      case NPY_FR_s:
-          unit = arrow::TimestampType::Unit::SECOND;
-          break;
-      case NPY_FR_ms:
-          unit = arrow::TimestampType::Unit::MILLI;
-          break;
-      case NPY_FR_us:
-          unit = arrow::TimestampType::Unit::MICRO;
-          break;
-      case NPY_FR_ns:
-          unit = arrow::TimestampType::Unit::NANO;
-          break;
-      default:
-          return Status::Invalid("Unknown NumPy datetime unit");
+    case NPY_FR_s:
+      unit = arrow::TimestampType::Unit::SECOND;
+      break;
+    case NPY_FR_ms:
+      unit = arrow::TimestampType::Unit::MILLI;
+      break;
+    case NPY_FR_us:
+      unit = arrow::TimestampType::Unit::MICRO;
+      break;
+    case NPY_FR_ns:
+      unit = arrow::TimestampType::Unit::NANO;
+      break;
+    default:
+      return Status::Invalid("Unknown NumPy datetime unit");
   }
 
   out->reset(new arrow::TimestampType(unit));
@@ -351,9 +342,7 @@ template <int TYPE>
 inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
   typedef npy_traits<TYPE> traits;
 
-  if (mask_ != nullptr || traits::supports_nulls) {
-    RETURN_NOT_OK(InitNullBitmap());
-  }
+  if (mask_ != nullptr || traits::supports_nulls) { RETURN_NOT_OK(InitNullBitmap()); }
 
   int64_t null_count = 0;
   if (mask_ != nullptr) {
@@ -429,9 +418,7 @@ inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out)
 template <int TYPE>
 inline Status ArrowSerializer<TYPE>::ConvertData() {
   // TODO(wesm): strided arrays
-  if (is_strided()) {
-    return Status::Invalid("no support for strided data yet");
-  }
+  if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
 
   data_ = std::make_shared<NumPyBuffer>(arr_);
   return Status::OK();
@@ -439,9 +426,7 @@ inline Status ArrowSerializer<TYPE>::ConvertData() {
 
 template <>
 inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
-  if (is_strided()) {
-    return Status::Invalid("no support for strided data yet");
-  }
+  if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
 
   int nbytes = BitUtil::BytesForBits(length_);
   auto buffer = std::make_shared<arrow::PoolBuffer>(pool_);
@@ -453,9 +438,7 @@ inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
 
   memset(bitmap, 0, nbytes);
   for (int i = 0; i < length_; ++i) {
-    if (values[i] > 0) {
-      BitUtil::SetBit(bitmap, i);
-    }
+    if (values[i] > 0) { BitUtil::SetBit(bitmap, i); }
   }
 
   data_ = buffer;
@@ -468,29 +451,24 @@ inline Status ArrowSerializer<NPY_OBJECT>::ConvertData() {
   return Status::TypeError("NYI");
 }
 
+#define TO_ARROW_CASE(TYPE)                                 \
+  case NPY_##TYPE: {                                        \
+    ArrowSerializer<NPY_##TYPE> converter(pool, arr, mask); \
+    RETURN_NOT_OK(converter.Convert(out));                  \
+  } break;
 
-#define TO_ARROW_CASE(TYPE)                                     \
-  case NPY_##TYPE:                                              \
-    {                                                           \
-      ArrowSerializer<NPY_##TYPE> converter(pool, arr, mask);   \
-      RETURN_NOT_OK(converter.Convert(out));                    \
-    }                                                           \
-    break;
-
-Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
-    std::shared_ptr<Array>* out) {
+Status PandasMaskedToArrow(
+    arrow::MemoryPool* pool, PyObject* ao, PyObject* mo, std::shared_ptr<Array>* out) {
   PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(ao);
   PyArrayObject* mask = nullptr;
 
-  if (mo != nullptr) {
-    mask = reinterpret_cast<PyArrayObject*>(mo);
-  }
+  if (mo != nullptr) { mask = reinterpret_cast<PyArrayObject*>(mo); }
 
   if (PyArray_NDIM(arr) != 1) {
     return Status::Invalid("only handle 1-dimensional arrays");
   }
 
-  switch(PyArray_DESCR(arr)->type_num) {
+  switch (PyArray_DESCR(arr)->type_num) {
     TO_ARROW_CASE(BOOL);
     TO_ARROW_CASE(INT8);
     TO_ARROW_CASE(INT16);
@@ -506,15 +484,13 @@ Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     TO_ARROW_CASE(OBJECT);
     default:
       std::stringstream ss;
-      ss << "unsupported type " << PyArray_DESCR(arr)->type_num
-         << std::endl;
+      ss << "unsupported type " << PyArray_DESCR(arr)->type_num << std::endl;
       return Status::NotImplemented(ss.str());
   }
   return Status::OK();
 }
 
-Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
-    std::shared_ptr<Array>* out) {
+Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, std::shared_ptr<Array>* out) {
   return PandasMaskedToArrow(pool, ao, nullptr, out);
 }
 
@@ -522,28 +498,27 @@ Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
 // Deserialization
 
 template <int TYPE>
-struct arrow_traits {
-};
+struct arrow_traits {};
 
 template <>
 struct arrow_traits<arrow::Type::BOOL> {
   static constexpr int npy_type = NPY_BOOL;
   static constexpr bool supports_nulls = false;
   static constexpr bool is_boolean = true;
-  static constexpr bool is_pandas_numeric_not_nullable = false;
-  static constexpr bool is_pandas_numeric_nullable = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
 };
 
-#define INT_DECL(TYPE)                                           \
-  template <>                                                    \
-  struct arrow_traits<arrow::Type::TYPE> {                       \
-    static constexpr int npy_type = NPY_##TYPE;                  \
-    static constexpr bool supports_nulls = false;                \
-    static constexpr double na_value = NAN;                      \
-    static constexpr bool is_boolean = false;                    \
-    static constexpr bool is_pandas_numeric_not_nullable = true; \
-    static constexpr bool is_pandas_numeric_nullable = false;    \
-    typedef typename npy_traits<NPY_##TYPE>::value_type T;       \
+#define INT_DECL(TYPE)                                     \
+  template <>                                              \
+  struct arrow_traits<arrow::Type::TYPE> {                 \
+    static constexpr int npy_type = NPY_##TYPE;            \
+    static constexpr bool supports_nulls = false;          \
+    static constexpr double na_value = NAN;                \
+    static constexpr bool is_boolean = false;              \
+    static constexpr bool is_numeric_not_nullable = true;  \
+    static constexpr bool is_numeric_nullable = false;     \
+    typedef typename npy_traits<NPY_##TYPE>::value_type T; \
   };
 
 INT_DECL(INT8);
@@ -561,8 +536,8 @@ struct arrow_traits<arrow::Type::FLOAT> {
   static constexpr bool supports_nulls = true;
   static constexpr float na_value = NAN;
   static constexpr bool is_boolean = false;
-  static constexpr bool is_pandas_numeric_not_nullable = false;
-  static constexpr bool is_pandas_numeric_nullable = true;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
   typedef typename npy_traits<NPY_FLOAT32>::value_type T;
 };
 
@@ -572,19 +547,21 @@ struct arrow_traits<arrow::Type::DOUBLE> {
   static constexpr bool supports_nulls = true;
   static constexpr double na_value = NAN;
   static constexpr bool is_boolean = false;
-  static constexpr bool is_pandas_numeric_not_nullable = false;
-  static constexpr bool is_pandas_numeric_nullable = true;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
   typedef typename npy_traits<NPY_FLOAT64>::value_type T;
 };
 
+static constexpr int64_t kPandasTimestampNull = std::numeric_limits<int64_t>::min();
+
 template <>
 struct arrow_traits<arrow::Type::TIMESTAMP> {
   static constexpr int npy_type = NPY_DATETIME;
   static constexpr bool supports_nulls = true;
-  static constexpr int64_t na_value = std::numeric_limits<int64_t>::min();
+  static constexpr int64_t na_value = kPandasTimestampNull;
   static constexpr bool is_boolean = false;
-  static constexpr bool is_pandas_numeric_not_nullable = false;
-  static constexpr bool is_pandas_numeric_nullable = true;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
   typedef typename npy_traits<NPY_DATETIME>::value_type T;
 };
 
@@ -592,10 +569,10 @@ template <>
 struct arrow_traits<arrow::Type::DATE> {
   static constexpr int npy_type = NPY_DATETIME;
   static constexpr bool supports_nulls = true;
-  static constexpr int64_t na_value = std::numeric_limits<int64_t>::min();
+  static constexpr int64_t na_value = kPandasTimestampNull;
   static constexpr bool is_boolean = false;
-  static constexpr bool is_pandas_numeric_not_nullable = false;
-  static constexpr bool is_pandas_numeric_nullable = true;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
   typedef typename npy_traits<NPY_DATETIME>::value_type T;
 };
 
@@ -604,18 +581,39 @@ struct arrow_traits<arrow::Type::STRING> {
   static constexpr int npy_type = NPY_OBJECT;
   static constexpr bool supports_nulls = true;
   static constexpr bool is_boolean = false;
-  static constexpr bool is_pandas_numeric_not_nullable = false;
-  static constexpr bool is_pandas_numeric_nullable = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
+
+template <>
+struct arrow_traits<arrow::Type::BINARY> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
 };
 
+template <typename T>
+struct WrapBytes {};
 
-static inline PyObject* make_pystring(const uint8_t* data, int32_t length) {
+template <>
+struct WrapBytes<arrow::StringArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
 #if PY_MAJOR_VERSION >= 3
-  return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+    return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
 #else
-  return PyString_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+    return PyString_FromStringAndSize(reinterpret_cast<const char*>(data), length);
 #endif
-}
+  }
+};
+
+template <>
+struct WrapBytes<arrow::BinaryArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+  }
+};
 
 inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
   if (type == NPY_DATETIME) {
@@ -645,20 +643,169 @@ inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out)
   }
 }
 
-template <int TYPE>
-class ArrowDeserializer {
- public:
-  ArrowDeserializer(const std::shared_ptr<Column>& col, PyObject* py_ref) :
-      col_(col), py_ref_(py_ref) {}
+template <typename T>
+inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    // Upcast to double, set NaN as appropriate
 
-  Status Convert(PyObject** out) {
-    const std::shared_ptr<arrow::ChunkedArray> data = col_->data();
+    for (int i = 0; i < arr->length(); ++i) {
+      *out_values++ = prim_arr->IsNull(i) ? NAN : in_values[i];
+    }
+  }
+}
 
-    RETURN_NOT_OK(ConvertValues<TYPE>(data));
-    *out = reinterpret_cast<PyObject*>(out_);
+template <typename T>
+inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    memcpy(out_values, in_values, sizeof(T) * arr->length());
+    out_values += arr->length();
+  }
+}
 
-    return Status::OK();
+template <typename InType, typename OutType>
+inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+    for (int32_t i = 0; i < arr->length(); ++i) {
+      *out_values = in_values[i];
+    }
+  }
+}
+
+static Status ConvertBooleanWithNulls(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (bool_arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values++ = Py_None;
+      } else if (bool_arr->Value(i)) {
+        // True
+        Py_INCREF(Py_True);
+        *out_values++ = Py_True;
+      } else {
+        // False
+        Py_INCREF(Py_False);
+        *out_values++ = Py_False;
+      }
+    }
+  }
+  return Status::OK();
+}
+
+static void ConvertBooleanNoNulls(const ChunkedArray& data, uint8_t* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = static_cast<uint8_t>(bool_arr->Value(i));
+    }
+  }
+}
+
+template <typename ArrayType>
+inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = static_cast<ArrayType*>(data.chunk(c).get());
+
+    const uint8_t* data_ptr;
+    int32_t length;
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (has_nulls && arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        data_ptr = arr->GetValue(i, &length);
+        *out_values = WrapBytes<ArrayType>::Wrap(data_ptr, length);
+        if (*out_values == nullptr) {
+          return Status::UnknownError("String initialization failed");
+        }
+      }
+      ++out_values;
+    }
+  }
+  return Status::OK();
+}
+
+template <typename T>
+inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+
+    const uint8_t* valid_bits = arr->null_bitmap_data();
+
+    if (arr->null_count() > 0) {
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i];
+      }
+    } else {
+      memcpy(out_values, in_values, sizeof(T) * arr->length());
+      out_values += arr->length();
+    }
   }
+}
+
+template <typename InType, typename OutType>
+inline void ConvertNumericNullableCast(
+    const ChunkedArray& data, OutType na_value, OutType* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = arr->IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
+    }
+  }
+}
+
+template <typename T>
+inline void ConvertDates(const ChunkedArray& data, T na_value, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
+      *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / 86400000;
+    }
+  }
+}
+
+template <typename InType, int SHIFT>
+inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = arr->IsNull(i) ? kPandasTimestampNull
+                                     : (static_cast<int64_t>(in_values[i]) * SHIFT);
+    }
+  }
+}
+
+class ArrowDeserializer {
+ public:
+  ArrowDeserializer(const std::shared_ptr<Column>& col, PyObject* py_ref)
+      : col_(col), data_(*col->data().get()), py_ref_(py_ref) {}
 
   Status AllocateOutput(int type) {
     PyAcquireGIL lock;
@@ -676,20 +823,29 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
-  Status OutputFromData(int type, void* data) {
+  template <int TYPE>
+  Status ConvertValuesZeroCopy(int npy_type, std::shared_ptr<Array> arr) {
+    typedef typename arrow_traits<TYPE>::T T;
+
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+
+    // Zero-Copy. We can pass the data pointer directly to NumPy.
+    void* data = const_cast<T*>(in_values);
+
     PyAcquireGIL lock;
 
     // Zero-Copy. We can pass the data pointer directly to NumPy.
     npy_intp dims[1] = {col_->length()};
-    out_ = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNewFromData(1, dims,
-                type, data));
+    out_ = reinterpret_cast<PyArrayObject*>(
+        PyArray_SimpleNewFromData(1, dims, npy_type, data));
 
     if (out_ == NULL) {
       // Error occurred, trust that SimpleNew set the error state
       return Status::OK();
     }
 
-    set_numpy_metadata(type, col_->type().get(), out_);
+    set_numpy_metadata(npy_type, col_->type().get(), out_);
 
     if (PyArray_SetBaseObject(out_, py_ref_) == -1) {
       // Error occurred, trust that SetBaseObject set the error state
@@ -705,317 +861,621 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
-  template <int T2>
-  Status ConvertValuesZeroCopy(std::shared_ptr<Array> arr) {
-    typedef typename arrow_traits<T2>::T T;
+  // ----------------------------------------------------------------------
+  // Allocate new array and deserialize. Can do a zero copy conversion for some
+  // types
 
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+  Status Convert(PyObject** out) {
+#define CONVERT_CASE(TYPE)                             \
+  case arrow::Type::TYPE: {                            \
+    RETURN_NOT_OK(ConvertValues<arrow::Type::TYPE>()); \
+  } break;
+
+    switch (col_->type()->type) {
+      CONVERT_CASE(BOOL);
+      CONVERT_CASE(INT8);
+      CONVERT_CASE(INT16);
+      CONVERT_CASE(INT32);
+      CONVERT_CASE(INT64);
+      CONVERT_CASE(UINT8);
+      CONVERT_CASE(UINT16);
+      CONVERT_CASE(UINT32);
+      CONVERT_CASE(UINT64);
+      CONVERT_CASE(FLOAT);
+      CONVERT_CASE(DOUBLE);
+      CONVERT_CASE(BINARY);
+      CONVERT_CASE(STRING);
+      CONVERT_CASE(DATE);
+      CONVERT_CASE(TIMESTAMP);
+      default:
+        return Status::NotImplemented("Arrow type reading not implemented");
+    }
 
-    // Zero-Copy. We can pass the data pointer directly to NumPy.
-    void* data = const_cast<T*>(in_values);
-    int type = arrow_traits<TYPE>::npy_type;
-    RETURN_NOT_OK(OutputFromData(type, data));
+#undef CONVERT_CASE
 
+    *out = reinterpret_cast<PyObject*>(out_);
     return Status::OK();
   }
 
-  template <int T2>
+  template <int TYPE>
   inline typename std::enable_if<
-    (T2 != arrow::Type::DATE) & arrow_traits<T2>::is_pandas_numeric_nullable, Status>::type
-  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
-    typedef typename arrow_traits<T2>::T T;
-    size_t chunk_offset = 0;
+      (TYPE != arrow::Type::DATE) & arrow_traits<TYPE>::is_numeric_nullable, Status>::type
+  ConvertValues() {
+    typedef typename arrow_traits<TYPE>::T T;
+    int npy_type = arrow_traits<TYPE>::npy_type;
 
-    if (data->num_chunks() == 1 && data->null_count() == 0) {
-      return ConvertValuesZeroCopy<TYPE>(data->chunk(0));
+    if (data_.num_chunks() == 1 && data_.null_count() == 0) {
+      return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
     }
 
-    RETURN_NOT_OK(AllocateOutput(arrow_traits<T2>::npy_type));
+    RETURN_NOT_OK(AllocateOutput(npy_type));
+    auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
+    ConvertNumericNullable<T>(data_, arrow_traits<TYPE>::na_value, out_values);
 
-    for (int c = 0; c < data->num_chunks(); c++) {
-      const std::shared_ptr<Array> arr = data->chunk(c);
-      auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-      auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-      auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_)) + chunk_offset;
+    return Status::OK();
+  }
 
-      if (arr->null_count() > 0) {
-        for (int64_t i = 0; i < arr->length(); ++i) {
-          out_values[i] = arr->IsNull(i) ? arrow_traits<T2>::na_value : in_values[i];
-        }
-      } else {
-        memcpy(out_values, in_values, sizeof(T) * arr->length());
-      }
+  template <int TYPE>
+  inline typename std::enable_if<TYPE == arrow::Type::DATE, Status>::type
+  ConvertValues() {
+    typedef typename arrow_traits<TYPE>::T T;
+
+    RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+    auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
+    ConvertDates<T>(data_, arrow_traits<TYPE>::na_value, out_values);
+    return Status::OK();
+  }
 
-      chunk_offset += arr->length();
+  // Integer specialization
+  template <int TYPE>
+  inline
+      typename std::enable_if<arrow_traits<TYPE>::is_numeric_not_nullable, Status>::type
+      ConvertValues() {
+    typedef typename arrow_traits<TYPE>::T T;
+    int npy_type = arrow_traits<TYPE>::npy_type;
+
+    if (data_.num_chunks() == 1 && data_.null_count() == 0) {
+      return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
+    }
+
+    if (data_.null_count() > 0) {
+      RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
+      auto out_values = reinterpret_cast<double*>(PyArray_DATA(out_));
+      ConvertIntegerWithNulls<T>(data_, out_values);
+    } else {
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+      auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
+      ConvertIntegerNoNullsSameType<T>(data_, out_values);
     }
 
     return Status::OK();
   }
 
+  // Boolean specialization
+  template <int TYPE>
+  inline typename std::enable_if<arrow_traits<TYPE>::is_boolean, Status>::type
+  ConvertValues() {
+    if (data_.null_count() > 0) {
+      RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+      auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
+      RETURN_NOT_OK(ConvertBooleanWithNulls(data_, out_values));
+    } else {
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+      auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(out_));
+      ConvertBooleanNoNulls(data_, out_values);
+    }
+    return Status::OK();
+  }
+
+  // UTF8 strings
+  template <int TYPE>
+  inline typename std::enable_if<TYPE == arrow::Type::STRING, Status>::type
+  ConvertValues() {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
+    return ConvertBinaryLike<arrow::StringArray>(data_, out_values);
+  }
+
   template <int T2>
-  inline typename std::enable_if<
-    T2 == arrow::Type::DATE, Status>::type
-  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
-    typedef typename arrow_traits<T2>::T T;
-    size_t chunk_offset = 0;
+  inline typename std::enable_if<T2 == arrow::Type::BINARY, Status>::type
+  ConvertValues() {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
+    return ConvertBinaryLike<arrow::BinaryArray>(data_, out_values);
+  }
+
+ private:
+  std::shared_ptr<Column> col_;
+  const arrow::ChunkedArray& data_;
+  PyObject* py_ref_;
+  PyArrayObject* out_;
+};
+
+Status ConvertArrayToPandas(
+    const std::shared_ptr<Array>& arr, PyObject* py_ref, PyObject** out) {
+  static std::string dummy_name = "dummy";
+  auto field = std::make_shared<Field>(dummy_name, arr->type());
+  auto col = std::make_shared<Column>(field, arr);
+  return ConvertColumnToPandas(col, py_ref, out);
+}
 
-    RETURN_NOT_OK(AllocateOutput(arrow_traits<T2>::npy_type));
+Status ConvertColumnToPandas(
+    const std::shared_ptr<Column>& col, PyObject* py_ref, PyObject** out) {
+  ArrowDeserializer converter(col, py_ref);
+  return converter.Convert(out);
+}
 
-    for (int c = 0; c < data->num_chunks(); c++) {
-      const std::shared_ptr<Array> arr = data->chunk(c);
-      auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-      auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-      auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_)) + chunk_offset;
+// ----------------------------------------------------------------------
+// pandas 0.x DataFrame conversion internals
 
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
-        out_values[i] = arr->IsNull(i) ? arrow_traits<T2>::na_value : in_values[i] / 86400000;
-      }
+class PandasBlock {
+ public:
+  enum type {
+    OBJECT,
+    UINT8,
+    INT8,
+    UINT16,
+    INT16,
+    UINT32,
+    INT32,
+    UINT64,
+    INT64,
+    FLOAT,
+    DOUBLE,
+    BOOL,
+    DATETIME,
+    CATEGORICAL
+  };
+
+  PandasBlock(int64_t num_rows, int num_columns)
+      : num_rows_(num_rows), num_columns_(num_columns) {}
+
+  virtual Status Allocate() = 0;
+  virtual Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) = 0;
 
-      chunk_offset += arr->length();
+  PyObject* block_arr() { return block_arr_.obj(); }
+
+  PyObject* placement_arr() { return placement_arr_.obj(); }
+
+ protected:
+  Status AllocateNDArray(int npy_type) {
+    PyAcquireGIL lock;
+
+    npy_intp block_dims[2] = {num_columns_, num_rows_};
+    PyObject* block_arr = PyArray_SimpleNew(2, block_dims, npy_type);
+    if (block_arr == NULL) {
+      // TODO(wesm): propagating Python exception
+      return Status::OK();
     }
 
+    npy_intp placement_dims[1] = {num_columns_};
+    PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
+    if (placement_arr == NULL) {
+      // TODO(wesm): propagating Python exception
+      return Status::OK();
+    }
+
+    block_arr_.reset(block_arr);
+    placement_arr_.reset(placement_arr);
+    current_placement_index_ = 0;
+
+    block_data_ = reinterpret_cast<uint8_t*>(
+        PyArray_DATA(reinterpret_cast<PyArrayObject*>(block_arr)));
+
+    placement_data_ = reinterpret_cast<int64_t*>(
+        PyArray_DATA(reinterpret_cast<PyArrayObject*>(placement_arr)));
+
     return Status::OK();
   }
 
-  // Integer specialization
-  template <int T2>
-  inline typename std::enable_if<
-    arrow_traits<T2>::is_pandas_numeric_not_nullable, Status>::type
-  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
-    typedef typename arrow_traits<T2>::T T;
-    size_t chunk_offset = 0;
+  int64_t num_rows_;
+  int num_columns_;
+  int current_placement_index_;
 
-    if (data->num_chunks() == 1 && data->null_count() == 0) {
-      return ConvertValuesZeroCopy<TYPE>(data->chunk(0));
-    }
+  OwnedRef block_arr_;
+  uint8_t* block_data_;
 
-    if (data->null_count() > 0) {
-      RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
+  // ndarray<int32>
+  OwnedRef placement_arr_;
+  int64_t* placement_data_;
 
-      for (int c = 0; c < data->num_chunks(); c++) {
-        const std::shared_ptr<Array> arr = data->chunk(c);
-        auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-        auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-        // Upcast to double, set NaN as appropriate
-        auto out_values = reinterpret_cast<double*>(PyArray_DATA(out_)) + chunk_offset;
+  DISALLOW_COPY_AND_ASSIGN(PandasBlock);
+};
 
-        for (int i = 0; i < arr->length(); ++i) {
-          out_values[i] = prim_arr->IsNull(i) ? NAN : in_values[i];
-        }
+class ObjectBlock : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
 
-        chunk_offset += arr->length();
-      }
-    } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+  Status Allocate() override { return AllocateNDArray(NPY_OBJECT); }
 
-      for (int c = 0; c < data->num_chunks(); c++) {
-        const std::shared_ptr<Array> arr = data->chunk(c);
-        auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-        auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-        auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_)) + chunk_offset;
+  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+    Type::type type = col->type()->type;
 
-        memcpy(out_values, in_values, sizeof(T) * arr->length());
+    PyObject** out_buffer =
+        reinterpret_cast<PyObject**>(block_data_) + current_placement_index_ * num_rows_;
 
-        chunk_offset += arr->length();
-      }
+    const ChunkedArray& data = *col->data().get();
+
+    if (type == Type::BOOL) {
+      RETURN_NOT_OK(ConvertBooleanWithNulls(data, out_buffer));
+    } else if (type == Type::BINARY) {
+      RETURN_NOT_OK(ConvertBinaryLike<arrow::BinaryArray>(data, out_buffer));
+    } else if (type == Type::STRING) {
+      RETURN_NOT_OK(ConvertBinaryLike<arrow::StringArray>(data, out_buffer));
+    } else {
+      std::stringstream ss;
+      ss << "Unsupported type for object array output: " << col->type()->ToString();
+      return Status::NotImplemented(ss.str());
     }
 
+    placement_data_[current_placement_index_++] = placement;
     return Status::OK();
   }
+};
 
-  // Boolean specialization
-  template <int T2>
-  inline typename std::enable_if<
-    arrow_traits<T2>::is_boolean, Status>::type
-  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
-    size_t chunk_offset = 0;
-    PyAcquireGIL lock;
+template <int ARROW_TYPE, typename C_TYPE>
+class IntBlock : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
 
-    if (data->null_count() > 0) {
-      RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+  Status Allocate() override {
+    return AllocateNDArray(arrow_traits<ARROW_TYPE>::npy_type);
+  }
 
-      for (int c = 0; c < data->num_chunks(); c++) {
-        const std::shared_ptr<Array> arr = data->chunk(c);
-        auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
-        auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_)) + chunk_offset;
-
-        for (int64_t i = 0; i < arr->length(); ++i) {
-          if (bool_arr->IsNull(i)) {
-            Py_INCREF(Py_None);
-            out_values[i] = Py_None;
-          } else if (bool_arr->Value(i)) {
-            // True
-            Py_INCREF(Py_True);
-            out_values[i] = Py_True;
-          } else {
-            // False
-            Py_INCREF(Py_False);
-            out_values[i] = Py_False;
-          }
-        }
+  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+    Type::type type = col->type()->type;
 
-        chunk_offset += bool_arr->length();
-      }
-    } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+    C_TYPE* out_buffer =
+        reinterpret_cast<C_TYPE*>(block_data_) + current_placement_index_ * num_rows_;
 
-      for (int c = 0; c < data->num_chunks(); c++) {
-        const std::shared_ptr<Array> arr = data->chunk(c);
-        auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
-        auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(out_)) + chunk_offset;
+    const ChunkedArray& data = *col->data().get();
 
-        for (int64_t i = 0; i < arr->length(); ++i) {
-          out_values[i] = static_cast<uint8_t>(bool_arr->Value(i));
-        }
+    if (type != ARROW_TYPE) { return Status::NotImplemented(col->type()->ToString()); }
 
-        chunk_offset += bool_arr->length();
-      }
+    ConvertIntegerNoNullsSameType<C_TYPE>(data, out_buffer);
+    placement_data_[current_placement_index_++] = placement;
+    return Status::OK();
+  }
+};
+
+using UInt8Block = IntBlock<Type::UINT8, uint8_t>;
+using Int8Block = IntBlock<Type::INT8, int8_t>;
+using UInt16Block = IntBlock<Type::UINT16, uint16_t>;
+using Int16Block = IntBlock<Type::INT16, int16_t>;
+using UInt32Block = IntBlock<Type::UINT32, uint32_t>;
+using Int32Block = IntBlock<Type::INT32, int32_t>;
+using UInt64Block = IntBlock<Type::UINT64, uint64_t>;
+using Int64Block = IntBlock<Type::INT64, int64_t>;
+
+class Float32Block : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
+
+  Status Allocate() override { return AllocateNDArray(NPY_FLOAT32); }
+
+  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+    Type::type type = col->type()->type;
+
+    if (type != Type::FLOAT) { return Status::NotImplemented(col->type()->ToString()); }
+
+    float* out_buffer =
+        reinterpret_cast<float*>(block_data_) + current_placement_index_ * num_rows_;
+
+    ConvertNumericNullable<float>(*col->data().get(), NAN, out_buffer);
+    placement_data_[current_placement_index_++] = placement;
+    return Status::OK();
+  }
+};
+
+class Float64Block : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
+
+  Status Allocate() override { return AllocateNDArray(NPY_FLOAT64); }
+
+  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+    Type::type type = col->type()->type;
+
+    double* out_buffer =
+        reinterpret_cast<double*>(block_data_) + current_placement_index_ * num_rows_;
+
+    const ChunkedArray& data = *col->data().get();
+
+#define INTEGER_CASE(IN_TYPE)                         \
+  ConvertIntegerWithNulls<IN_TYPE>(data, out_buffer); \
+  break;
+
+    switch (type) {
+      case Type::UINT8:
+        INTEGER_CASE(uint8_t);
+      case Type::INT8:
+        INTEGER_CASE(int8_t);
+      case Type::UINT16:
+        INTEGER_CASE(uint16_t);
+      case Type::INT16:
+        INTEGER_CASE(int16_t);
+      case Type::UINT32:
+        INTEGER_CASE(uint32_t);
+      case Type::INT32:
+        INTEGER_CASE(int32_t);
+      case Type::UINT64:
+        INTEGER_CASE(uint64_t);
+      case Type::INT64:
+        INTEGER_CASE(int64_t);
+      case Type::FLOAT:
+        ConvertNumericNullableCast<float, double>(data, NAN, out_buffer);
+        break;
+      case Type::DOUBLE:
+        ConvertNumericNullable<double>(data, NAN, out_buffer);
+        break;
+      default:
+        return Status::NotImplemented(col->type()->ToString());
     }
 
+#undef INTEGER_CASE
+
+    placement_data_[current_placement_index_++] = placement;
     return Status::OK();
   }
+};
 
-  // UTF8 strings
-  template <int T2>
-  inline typename std::enable_if<
-    T2 == arrow::Type::STRING, Status>::type
-  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
-    size_t chunk_offset = 0;
-    PyAcquireGIL lock;
+class BoolBlock : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
 
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+  Status Allocate() override { return AllocateNDArray(NPY_BOOL); }
 
-    for (int c = 0; c < data->num_chunks(); c++) {
-      const std::shared_ptr<Array> arr = data->chunk(c);
-      auto string_arr = static_cast<arrow::StringArray*>(arr.get());
-      auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_)) + chunk_offset;
-
-      const uint8_t* data_ptr;
-      int32_t length;
-      if (data->null_count() > 0) {
-        for (int64_t i = 0; i < arr->length(); ++i) {
-          if (string_arr->IsNull(i)) {
-            Py_INCREF(Py_None);
-            out_values[i] = Py_None;
-          } else {
-            data_ptr = string_arr->GetValue(i, &length);
-
-            out_values[i] = make_pystring(data_ptr, length);
-            if (out_values[i] == nullptr) {
-              return Status::UnknownError("String initialization failed");
-            }
-          }
-        }
+  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+    Type::type type = col->type()->type;
+
+    if (type != Type::BOOL) { return Status::NotImplemented(col->type()->ToString()); }
+
+    uint8_t* out_buffer =
+        reinterpret_cast<uint8_t*>(block_data_) + current_placement_index_ * num_rows_;
+
+    ConvertBooleanNoNulls(*col->data().get(), out_buffer);
+    placement_data_[current_placement_index_++] = placement;
+    return Status::OK();
+  }
+};
+
+class DatetimeBlock : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
+
+  Status Allocate() override {
+    RETURN_NOT_OK(AllocateNDArray(NPY_DATETIME));
+
+    PyAcquireGIL lock;
+    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(
+        PyArray_DESCR(reinterpret_cast<PyArrayObject*>(block_arr_.obj()))->c_metadata);
+    date_dtype->meta.base = NPY_FR_ns;
+    return Status::OK();
+  }
+
+  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+    Type::type type = col->type()->type;
+
+    int64_t* out_buffer =
+        reinterpret_cast<int64_t*>(block_data_) + current_placement_index_ * num_rows_;
+
+    const ChunkedArray& data = *col.get()->data();
+
+    if (type == Type::DATE) {
+      // DateType is millisecond timestamp stored as int64_t
+      // TODO(wesm): Do we want to make sure to zero out the milliseconds?
+      ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
+    } else if (type == Type::TIMESTAMP) {
+      auto ts_type = static_cast<arrow::TimestampType*>(col->type().get());
+
+      if (ts_type->unit == arrow::TimeUnit::NANO) {
+        ConvertNumericNullable<int64_t>(data, kPandasTimestampNull, out_buffer);
+      } else if (ts_type->unit == arrow::TimeUnit::MICRO) {
+        ConvertDatetimeNanos<int64_t, 1000L>(data, out_buffer);
+      } else if (ts_type->unit == arrow::TimeUnit::MILLI) {
+        ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
+      } else if (ts_type->unit == arrow::TimeUnit::SECOND) {
+        ConvertDatetimeNanos<int64_t, 1000000000L>(data, out_buffer);
       } else {
-        for (int64_t i = 0; i < arr->length(); ++i) {
-          data_ptr = string_arr->GetValue(i, &length);
-          out_values[i] = make_pystring(data_ptr, length);
-          if (out_values[i] == nullptr) {
-            return Status::UnknownError("String initialization failed");
-          }
-        }
+        return Status::NotImplemented("Unsupported time unit");
       }
-
-      chunk_offset += string_arr->length();
+    } else {
+      return Status::NotImplemented(col->type()->ToString());
     }
 
+    placement_data_[current_placement_index_++] = placement;
     return Status::OK();
   }
+};
 
-  template <int T2>
-  inline typename std::enable_if<
-    T2 == arrow::Type::BINARY, Status>::type
-  ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
-    size_t chunk_offset = 0;
-    PyAcquireGIL lock;
+// class CategoricalBlock : public PandasBlock {};
 
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
+    std::shared_ptr<PandasBlock>* block) {
+#define BLOCK_CASE(NAME, TYPE)                              \
+  case PandasBlock::NAME:                                   \
+    *block = std::make_shared<TYPE>(num_rows, num_columns); \
+    break;
 
-    for (int c = 0; c < data->num_chunks(); c++) {
-      const std::shared_ptr<Array> arr = data->chunk(c);
-      auto binary_arr = static_cast<arrow::BinaryArray*>(arr.get());
-      auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_)) + chunk_offset;
-
-      const uint8_t* data_ptr;
-      int32_t length;
-      if (data->null_count() > 0) {
-        for (int64_t i = 0; i < arr->length(); ++i) {
-          if (binary_arr->IsNull(i)) {
-            Py_INCREF(Py_None);
-            out_values[i] = Py_None;
-          } else {
-            data_ptr = binary_arr->GetValue(i, &length);
-
-            out_values[i] = PyBytes_FromStringAndSize(
-                reinterpret_cast<const char*>(data_ptr), length);
-            if (out_values[i] == nullptr) {
-              return Status::UnknownError("String initialization failed");
-            }
-          }
-        }
+  switch (type) {
+    BLOCK_CASE(OBJECT, ObjectBlock);
+    BLOCK_CASE(UINT8, UInt8Block);
+    BLOCK_CASE(INT8, Int8Block);
+    BLOCK_CASE(UINT16, UInt16Block);
+    BLOCK_CASE(INT16, Int16Block);
+    BLOCK_CASE(UINT32, UInt32Block);
+    BLOCK_CASE(INT32, Int32Block);
+    BLOCK_CASE(UINT64, UInt64Block);
+    BLOCK_CASE(INT64, Int64Block);
+    BLOCK_CASE(FLOAT, Float32Block);
+    BLOCK_CASE(DOUBLE, Float64Block);
+    BLOCK_CASE(BOOL, BoolBlock);
+    BLOCK_CASE(DATETIME, DatetimeBlock);
+    case PandasBlock::CATEGORICAL:
+      return Status::NotImplemented("categorical");
+  }
+
+#undef BLOCK_CASE
+
+  return (*block)->Allocate();
+}
+
+// Construct the exact pandas 0.x "BlockManager" memory layout
+//
+// * For each column determine the correct output pandas type
+// * Allocate 2D blocks (ncols x nrows) for each distinct data type in output
+// * Allocate  block placement arrays
+// * Write Arrow columns out into each slice of memory; populate block
+// * placement arrays as we go
+class DataFrameBlockCreator {
+ public:
+  DataFrameBlockCreator(const std::shared_ptr<Table>& table) : table_(table) {}
+
+  Status Convert(int nthreads, PyObject** output) {
+    column_types_.resize(table_->num_columns());
+    type_counts_.clear();
+    blocks_.clear();
+
+    RETURN_NOT_OK(CountColumnTypes());
+    RETURN_NOT_OK(CreateBlocks());
+    RETURN_NOT_OK(WriteTableToBlocks(nthreads));
+
+    return GetResultList(output);
+  }
+
+  Status CountColumnTypes() {
+    for (int i = 0; i < table_->num_columns(); ++i) {
+      std::shared_ptr<Column> col = table_->column(i);
+      PandasBlock::type output_type;
+
+      switch (col->type()->type) {
+        case Type::BOOL:
+          output_type = col->null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL;
+          break;
+        case Type::UINT8:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT8;
+          break;
+        case Type::INT8:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT8;
+          break;
+        case Type::UINT16:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT16;
+          break;
+        case Type::INT16:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT16;
+          break;
+        case Type::UINT32:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT32;
+          break;
+        case Type::INT32:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT32;
+          break;
+        case Type::INT64:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT64;
+          break;
+        case Type::UINT64:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT64;
+          break;
+        case Type::FLOAT:
+          output_type = PandasBlock::FLOAT;
+          break;
+        case Type::DOUBLE:
+          output_type = PandasBlock::DOUBLE;
+          break;
+        case Type::STRING:
+        case Type::BINARY:
+          output_type = PandasBlock::OBJECT;
+          break;
+        case Type::DATE:
+          output_type = PandasBlock::DATETIME;
+          break;
+        case Type::TIMESTAMP:
+          output_type = PandasBlock::DATETIME;
+          break;
+        default:
+          return Status::NotImplemented(col->type()->ToString());
+      }
+
+      auto it = type_counts_.find(output_type);
+      if (it != type_counts_.end()) {
+        // Increment count
+        it->second += 1;
       } else {
-        for (int64_t i = 0; i < arr->length(); ++i) {
-          data_ptr = binary_arr->GetValue(i, &length);
-          out_values[i] = PyBytes_FromStringAndSize(
-              reinterpret_cast<const char*>(data_ptr), length);
-          if (out_values[i] == nullptr) {
-            return Status::UnknownError("String initialization failed");
-          }
-        }
+        // Add key to map
+        type_counts_[output_type] = 1;
       }
 
-      chunk_offset += binary_arr->length();
+      column_types_[i] = output_type;
     }
+    return Status::OK();
+  }
 
+  Status CreateBlocks() {
+    for (const auto& it : type_counts_) {
+      PandasBlock::type type = static_cast<PandasBlock::type>(it.first);
+      std::shared_ptr<PandasBlock> block;
+      RETURN_NOT_OK(MakeBlock(type, table_->num_rows(), it.second, &block));
+      blocks_[type] = block;
+    }
     return Status::OK();
   }
 
- private:
-  std::shared_ptr<Column> col_;
-  PyObject* py_ref_;
-  PyArrayObject* out_;
-};
+  Status WriteTableToBlocks(int nthreads) {
+    if (nthreads > 1) {
+      return Status::NotImplemented("multithreading not yet implemented");
+    }
 
-#define FROM_ARROW_CASE(TYPE)                                       \
-  case arrow::Type::TYPE:                                           \
-    {                                                               \
-      ArrowDeserializer<arrow::Type::TYPE> converter(col, py_ref);  \
-      return converter.Convert(out);                                \
-    }                                                               \
-    break;
+    for (int i = 0; i < table_->num_columns(); ++i) {
+      std::shared_ptr<Column> col = table_->column(i);
+      PandasBlock::type output_type = column_types_[i];
 
-Status ConvertArrayToPandas(const std::shared_ptr<Array>& arr, PyObject* py_ref,
-        PyObject** out) {
-  static std::string dummy_name = "dummy";
-  auto field = std::make_shared<Field>(dummy_name, arr->type());
-  auto col = std::make_shared<Column>(field, arr);
-  return ConvertColumnToPandas(col, py_ref, out);
-}
+      auto it = blocks_.find(output_type);
+      if (it == blocks_.end()) { return Status::KeyError("No block allocated"); }
+      RETURN_NOT_OK(it->second->WriteNext(col, i));
+    }
+    return Status::OK();
+  }
 
-Status ConvertColumnToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
-        PyObject** out) {
-  switch(col->type()->type) {
-    FROM_ARROW_CASE(BOOL);
-    FROM_ARROW_CASE(INT8);
-    FROM_ARROW_CASE(INT16);
-    FROM_ARROW_CASE(INT32);
-    FROM_ARROW_CASE(INT64);
-    FROM_ARROW_CASE(UINT8);
-    FROM_ARROW_CASE(UINT16);
-    FROM_ARROW_CASE(UINT32);
-    FROM_ARROW_CASE(UINT64);
-    FROM_ARROW_CASE(FLOAT);
-    FROM_ARROW_CASE(DOUBLE);
-    FROM_ARROW_CASE(BINARY);
-    FROM_ARROW_CASE(STRING);
-    FROM_ARROW_CASE(DATE);
-    FROM_ARROW_CASE(TIMESTAMP);
-    default:
-      return Status::NotImplemented("Arrow type reading not implemented");
+  Status GetResultList(PyObject** out) {
+    auto num_blocks = static_cast<Py_ssize_t>(blocks_.size());
+    PyObject* result = PyList_New(num_blocks);
+    RETURN_IF_PYERROR();
+
+    int i = 0;
+    for (const auto& it : blocks_) {
+      const std::shared_ptr<PandasBlock> block = it.second;
+
+      PyObject* item = PyTuple_New(2);
+      RETURN_IF_PYERROR();
+
+      PyObject* block_arr = block->block_arr();
+      PyObject* placement_arr = block->placement_arr();
+      Py_INCREF(block_arr);
+      Py_INCREF(placement_arr);
+      PyTuple_SET_ITEM(item, 0, block_arr);
+      PyTuple_SET_ITEM(item, 1, placement_arr);
+
+      if (PyList_SET_ITEM(result, i++, item) < 0) { RETURN_IF_PYERROR(); }
+    }
+    *out = result;
+    return Status::OK();
   }
-  return Status::OK();
+
+ private:
+  std::shared_ptr<Table> table_;
+  std::vector<PandasBlock::type> column_types_;
+
+  // block type -> type count
+  std::unordered_map<int, int> type_counts_;
+
+  // block type -> block
+  std::unordered_map<int, std::shared_ptr<PandasBlock>> blocks_;
+};
+
+Status ConvertTableToPandas(
+    const std::shared_ptr<Table>& table, int nthreads, PyObject** out) {
+  DataFrameBlockCreator helper(table);
+  return helper.Convert(nthreads, out);
 }
 
-} // namespace pyarrow
+}  // namespace pyarrow
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index 532495dd792db..60dadd473ad3f 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -33,27 +33,42 @@ class Array;
 class Column;
 class MemoryPool;
 class Status;
+class Table;
 
-} // namespace arrow
+}  // namespace arrow
 
 namespace pyarrow {
 
 PYARROW_EXPORT
-arrow::Status ConvertArrayToPandas(const std::shared_ptr<arrow::Array>& arr,
-    PyObject* py_ref, PyObject** out);
+arrow::Status ConvertArrayToPandas(
+    const std::shared_ptr<arrow::Array>& arr, PyObject* py_ref, PyObject** out);
 
 PYARROW_EXPORT
-arrow::Status ConvertColumnToPandas(const std::shared_ptr<arrow::Column>& col,
-    PyObject* py_ref, PyObject** out);
+arrow::Status ConvertColumnToPandas(
+    const std::shared_ptr<arrow::Column>& col, PyObject* py_ref, PyObject** out);
+
+struct PandasOptions {
+  bool strings_to_categorical;
+};
+
+// Convert a whole table as efficiently as possible to a pandas.DataFrame.
+//
+// The returned Python object is a list of tuples consisting of the exact 2D
+// BlockManager structure of the pandas.DataFrame used as of pandas 0.19.x.
+//
+// tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2])
+PYARROW_EXPORT
+arrow::Status ConvertTableToPandas(
+    const std::shared_ptr<arrow::Table>& table, int nthreads, PyObject** out);
 
 PYARROW_EXPORT
 arrow::Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     std::shared_ptr<arrow::Array>* out);
 
 PYARROW_EXPORT
-arrow::Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
-    std::shared_ptr<arrow::Array>* out);
+arrow::Status PandasToArrow(
+    arrow::MemoryPool* pool, PyObject* ao, std::shared_ptr<arrow::Array>* out);
 
-} // namespace pyarrow
+}  // namespace pyarrow
 
-#endif // PYARROW_ADAPTERS_PANDAS_H
+#endif  // PYARROW_ADAPTERS_PANDAS_H
diff --git a/python/src/pyarrow/api.h b/python/src/pyarrow/api.h
index 6dbbc45d40ccc..f65cc097f548f 100644
--- a/python/src/pyarrow/api.h
+++ b/python/src/pyarrow/api.h
@@ -23,4 +23,4 @@
 #include "pyarrow/adapters/builtin.h"
 #include "pyarrow/adapters/pandas.h"
 
-#endif // PYARROW_API_H
+#endif  // PYARROW_API_H
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index fb4d3496ac79f..8660ac8f0cedf 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -73,7 +73,7 @@ arrow::MemoryPool* get_memory_pool() {
 
 PyBytesBuffer::PyBytesBuffer(PyObject* obj)
     : Buffer(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)),
-        PyBytes_GET_SIZE(obj)),
+          PyBytes_GET_SIZE(obj)),
       obj_(obj) {
   Py_INCREF(obj_);
 }
@@ -83,4 +83,4 @@ PyBytesBuffer::~PyBytesBuffer() {
   Py_DECREF(obj_);
 }
 
-} // namespace pyarrow
+}  // namespace pyarrow
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 7e3382634a781..639918d309fe7 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -24,7 +24,9 @@
 #include "arrow/buffer.h"
 #include "arrow/util/macros.h"
 
-namespace arrow { class MemoryPool; }
+namespace arrow {
+class MemoryPool;
+}
 
 namespace pyarrow {
 
@@ -34,27 +36,18 @@ class OwnedRef {
  public:
   OwnedRef() : obj_(nullptr) {}
 
-  OwnedRef(PyObject* obj) :
-      obj_(obj) {}
+  OwnedRef(PyObject* obj) : obj_(obj) {}
 
-  ~OwnedRef() {
-    Py_XDECREF(obj_);
-  }
+  ~OwnedRef() { Py_XDECREF(obj_); }
 
   void reset(PyObject* obj) {
-    if (obj_ != nullptr) {
-      Py_XDECREF(obj_);
-    }
+    if (obj_ != nullptr) { Py_XDECREF(obj_); }
     obj_ = obj;
   }
 
-  void release() {
-    obj_ = nullptr;
-  }
+  void release() { obj_ = nullptr; }
 
-  PyObject* obj() const{
-    return obj_;
-  }
+  PyObject* obj() const { return obj_; }
 
  private:
   PyObject* obj_;
@@ -78,13 +71,10 @@ struct PyObjectStringify {
 
 class PyGILGuard {
  public:
-  PyGILGuard() {
-    state_ = PyGILState_Ensure();
-  }
+  PyGILGuard() { state_ = PyGILState_Ensure(); }
+
+  ~PyGILGuard() { PyGILState_Release(state_); }
 
-  ~PyGILGuard() {
-    PyGILState_Release(state_);
-  }
  private:
   PyGILState_STATE state_;
   DISALLOW_COPY_AND_ASSIGN(PyGILGuard);
@@ -108,8 +98,7 @@ PYARROW_EXPORT arrow::MemoryPool* get_memory_pool();
 
 class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
  public:
-  NumPyBuffer(PyArrayObject* arr)
-    : Buffer(nullptr, 0) {
+  NumPyBuffer(PyArrayObject* arr) : Buffer(nullptr, 0) {
     arr_ = arr;
     Py_INCREF(arr);
 
@@ -118,9 +107,7 @@ class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
     capacity_ = size_;
   }
 
-  virtual ~NumPyBuffer() {
-    Py_XDECREF(arr_);
-  }
+  virtual ~NumPyBuffer() { Py_XDECREF(arr_); }
 
  private:
   PyArrayObject* arr_;
@@ -135,22 +122,17 @@ class PYARROW_EXPORT PyBytesBuffer : public arrow::Buffer {
   PyObject* obj_;
 };
 
-
 class PyAcquireGIL {
  public:
-  PyAcquireGIL() {
-    state_ = PyGILState_Ensure();
-  }
+  PyAcquireGIL() { state_ = PyGILState_Ensure(); }
 
-  ~PyAcquireGIL() {
-    PyGILState_Release(state_);
-  }
+  ~PyAcquireGIL() { PyGILState_Release(state_); }
 
  private:
   PyGILState_STATE state_;
   DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL);
 };
 
-} // namespace pyarrow
+}  // namespace pyarrow
 
-#endif // PYARROW_COMMON_H
+#endif  // PYARROW_COMMON_H
diff --git a/python/src/pyarrow/config.cc b/python/src/pyarrow/config.cc
index 730d2db99a530..e1002bf4fd146 100644
--- a/python/src/pyarrow/config.cc
+++ b/python/src/pyarrow/config.cc
@@ -21,8 +21,7 @@
 
 namespace pyarrow {
 
-void pyarrow_init() {
-}
+void pyarrow_init() {}
 
 PyObject* numpy_nan = nullptr;
 
@@ -31,4 +30,4 @@ void pyarrow_set_numpy_nan(PyObject* obj) {
   numpy_nan = obj;
 }
 
-} // namespace pyarrow
+}  // namespace pyarrow
diff --git a/python/src/pyarrow/config.h b/python/src/pyarrow/config.h
index 82936b1a5f317..386ee4b1e2590 100644
--- a/python/src/pyarrow/config.h
+++ b/python/src/pyarrow/config.h
@@ -24,7 +24,7 @@
 #include "pyarrow/visibility.h"
 
 #if PY_MAJOR_VERSION >= 3
-  #define PyString_Check PyUnicode_Check
+#define PyString_Check PyUnicode_Check
 #endif
 
 namespace pyarrow {
@@ -38,6 +38,6 @@ void pyarrow_init();
 PYARROW_EXPORT
 void pyarrow_set_numpy_nan(PyObject* obj);
 
-} // namespace pyarrow
+}  // namespace pyarrow
 
-#endif // PYARROW_CONFIG_H
+#endif  // PYARROW_CONFIG_H
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
index b42199c8e041c..3f650326e09aa 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/python/src/pyarrow/helpers.cc
@@ -23,36 +23,35 @@ using namespace arrow;
 
 namespace pyarrow {
 
-
-#define GET_PRIMITIVE_TYPE(NAME, FACTORY)       \
-  case Type::NAME:                              \
-    return FACTORY();                           \
+#define GET_PRIMITIVE_TYPE(NAME, FACTORY) \
+  case Type::NAME:                        \
+    return FACTORY();                     \
     break;
 
 std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
   switch (type) {
     case Type::NA:
       return null();
-    GET_PRIMITIVE_TYPE(UINT8, uint8);
-    GET_PRIMITIVE_TYPE(INT8, int8);
-    GET_PRIMITIVE_TYPE(UINT16, uint16);
-    GET_PRIMITIVE_TYPE(INT16, int16);
-    GET_PRIMITIVE_TYPE(UINT32, uint32);
-    GET_PRIMITIVE_TYPE(INT32, int32);
-    GET_PRIMITIVE_TYPE(UINT64, uint64);
-    GET_PRIMITIVE_TYPE(INT64, int64);
-    GET_PRIMITIVE_TYPE(DATE, date);
+      GET_PRIMITIVE_TYPE(UINT8, uint8);
+      GET_PRIMITIVE_TYPE(INT8, int8);
+      GET_PRIMITIVE_TYPE(UINT16, uint16);
+      GET_PRIMITIVE_TYPE(INT16, int16);
+      GET_PRIMITIVE_TYPE(UINT32, uint32);
+      GET_PRIMITIVE_TYPE(INT32, int32);
+      GET_PRIMITIVE_TYPE(UINT64, uint64);
+      GET_PRIMITIVE_TYPE(INT64, int64);
+      GET_PRIMITIVE_TYPE(DATE, date);
     case Type::TIMESTAMP:
       return arrow::timestamp(arrow::TimeUnit::MICRO);
       break;
-    GET_PRIMITIVE_TYPE(BOOL, boolean);
-    GET_PRIMITIVE_TYPE(FLOAT, float32);
-    GET_PRIMITIVE_TYPE(DOUBLE, float64);
-    GET_PRIMITIVE_TYPE(BINARY, binary);
-    GET_PRIMITIVE_TYPE(STRING, utf8);
+      GET_PRIMITIVE_TYPE(BOOL, boolean);
+      GET_PRIMITIVE_TYPE(FLOAT, float32);
+      GET_PRIMITIVE_TYPE(DOUBLE, float64);
+      GET_PRIMITIVE_TYPE(BINARY, binary);
+      GET_PRIMITIVE_TYPE(STRING, utf8);
     default:
       return nullptr;
   }
 }
 
-} // namespace pyarrow
+}  // namespace pyarrow
diff --git a/python/src/pyarrow/helpers.h b/python/src/pyarrow/helpers.h
index 8334d974c0237..788c3eedddfd6 100644
--- a/python/src/pyarrow/helpers.h
+++ b/python/src/pyarrow/helpers.h
@@ -31,6 +31,6 @@ using arrow::Type;
 PYARROW_EXPORT
 std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
 
-} // namespace pyarrow
+}  // namespace pyarrow
 
-#endif // PYARROW_HELPERS_H
+#endif  // PYARROW_HELPERS_H
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index 12f5ba0bf2b49..ac1aa635b40ea 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -33,8 +33,7 @@ namespace pyarrow {
 // ----------------------------------------------------------------------
 // Python file
 
-PythonFile::PythonFile(PyObject* file)
-    : file_(file) {
+PythonFile::PythonFile(PyObject* file) : file_(file) {
   Py_INCREF(file_);
 }
 
@@ -81,8 +80,8 @@ Status PythonFile::Read(int64_t nbytes, PyObject** out) {
 }
 
 Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
-  PyObject* py_data = PyBytes_FromStringAndSize(
-      reinterpret_cast<const char*>(data), nbytes);
+  PyObject* py_data =
+      PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), nbytes);
   ARROW_RETURN_NOT_OK(CheckPyError());
 
   PyObject* result = PyObject_CallMethod(file_, "write", "(O)", py_data);
@@ -102,7 +101,7 @@ Status PythonFile::Tell(int64_t* position) {
   // PyLong_AsLongLong can raise OverflowError
   ARROW_RETURN_NOT_OK(CheckPyError());
 
- return Status::OK();
+  return Status::OK();
 }
 
 // ----------------------------------------------------------------------
@@ -156,7 +155,8 @@ Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out)
 Status PyReadableFile::GetSize(int64_t* size) {
   PyGILGuard lock;
 
-  int64_t current_position;;
+  int64_t current_position;
+  ;
   ARROW_RETURN_NOT_OK(file_->Tell(&current_position));
 
   ARROW_RETURN_NOT_OK(file_->Seek(0, 2));
@@ -204,7 +204,7 @@ Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
 
 PyBytesReader::PyBytesReader(PyObject* obj)
     : arrow::io::BufferReader(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)),
-        PyBytes_GET_SIZE(obj)),
+          PyBytes_GET_SIZE(obj)),
       obj_(obj) {
   Py_INCREF(obj_);
 }
diff --git a/python/src/pyarrow/io.h b/python/src/pyarrow/io.h
index e14aa8cfb27e3..fd3e7c0887207 100644
--- a/python/src/pyarrow/io.h
+++ b/python/src/pyarrow/io.h
@@ -24,7 +24,9 @@
 #include "pyarrow/config.h"
 #include "pyarrow/visibility.h"
 
-namespace arrow { class MemoryPool; }
+namespace arrow {
+class MemoryPool;
+}
 
 namespace pyarrow {
 
@@ -92,6 +94,6 @@ class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader {
 
 // TODO(wesm): seekable output files
 
-} // namespace pyarrow
+}  // namespace pyarrow
 
 #endif  // PYARROW_IO_H
diff --git a/python/src/pyarrow/numpy_interop.h b/python/src/pyarrow/numpy_interop.h
index 882d287c7c559..6326527a67420 100644
--- a/python/src/pyarrow/numpy_interop.h
+++ b/python/src/pyarrow/numpy_interop.h
@@ -53,6 +53,6 @@ inline int import_numpy() {
   return 0;
 }
 
-} // namespace pyarrow
+}  // namespace pyarrow
 
-#endif // PYARROW_NUMPY_INTEROP_H
+#endif  // PYARROW_NUMPY_INTEROP_H
diff --git a/python/src/pyarrow/util/datetime.h b/python/src/pyarrow/util/datetime.h
index b67accc388f59..9ffa691052460 100644
--- a/python/src/pyarrow/util/datetime.h
+++ b/python/src/pyarrow/util/datetime.h
@@ -22,8 +22,8 @@
 #include <datetime.h>
 
 namespace pyarrow {
-  
-inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) { 
+
+inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   struct tm date = {0};
   date.tm_year = PyDateTime_GET_YEAR(pydate) - 1900;
   date.tm_mon = PyDateTime_GET_MONTH(pydate) - 1;
@@ -35,6 +35,6 @@ inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
 }
 
-} // namespace pyarrow
+}  // namespace pyarrow
 
-#endif // PYARROW_UTIL_DATETIME_H
+#endif  // PYARROW_UTIL_DATETIME_H
diff --git a/python/src/pyarrow/util/test_main.cc b/python/src/pyarrow/util/test_main.cc
index 00139f36742ed..6fb7c0536eed3 100644
--- a/python/src/pyarrow/util/test_main.cc
+++ b/python/src/pyarrow/util/test_main.cc
@@ -17,7 +17,7 @@
 
 #include <gtest/gtest.h>
 
-int main(int argc, char **argv) {
+int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
 
   int ret = RUN_ALL_TESTS();

From 1079a3206c58dc053ad6d9ca4ead6446a1c9be80 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 27 Dec 2016 18:04:28 -0500
Subject: [PATCH 0245/1644] ARROW-437: [C++} Fix clang compiler warning

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #254 from wesm/ARROW-437 and squashes the following commits:

a18a651 [Wes McKinney] Fix compiler warning in clang
---
 cpp/src/arrow/array.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 26d53f7d75896..5cd56d6df5427 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -147,7 +147,9 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
     return PrimitiveArray::EqualsExact(static_cast<const PrimitiveArray&>(other));
   }
 
-  bool ApproxEquals(const std::shared_ptr<Array>& arr) const { return Equals(arr); }
+  bool ApproxEquals(const std::shared_ptr<Array>& arr) const override {
+    return Equals(arr);
+  }
 
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
       const ArrayPtr& arr) const override {

From ab5f66a2e9a2b6af312ffdfa2f95c65b1d6f5739 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 28 Dec 2016 07:49:06 -0500
Subject: [PATCH 0246/1644] ARROW-428: [Python] Multithreaded conversion from
 Arrow table to pandas.DataFrame

This yields a substantial speedup on my laptop. On a 1GB numeric dataset, with 1 thread (the default prior to this patch):

```
>>> %timeit df2 = table.to_pandas(nthreads=1)
1 loop, best of 3: 498 ms per loop
```

With 4 threads (this is a true quad-core machine)

```
>>> %timeit df2 = table.to_pandas(nthreads=4)
1 loop, best of 3: 151 ms per loop
```

The default number of cores used is the `os.cpu_count` divided by 2 (since hyperthreading doesn't help with this largely memory-bound operation).

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #252 from wesm/ARROW-428 and squashes the following commits:

da929bf [Wes McKinney] Factor out common compiler flag code between Arrow C++ and Python CMake files. Add pyarrow.cpu_count/set_cpu_count functions per feedback
cad89e9 [Wes McKinney] Tweak pyarrow cmake flags
e70f16d [Wes McKinney] Add missing GIL acquisition. Do not spawn too many threads if few columns
bc4dff7 [Wes McKinney] Return errors from threaded conversion. Add doc about number of cpus used
79f5fd9 [Wes McKinney] Implement multithreaded conversion from Arrow table to pandas.DataFrame. Default to multiprocessing.cpu_count for now
---
 cpp/CMakeLists.txt                          |  71 +-----------
 cpp/cmake_modules/SetupCxxFlags.cmake       |  86 ++++++++++++++
 python/CMakeLists.txt                       |  36 +-----
 python/pyarrow/__init__.py                  |   1 +
 python/pyarrow/config.pyx                   |  23 ++++
 python/pyarrow/table.pyx                    |  38 +++---
 python/pyarrow/tests/test_convert_pandas.py |  42 +++++--
 python/src/pyarrow/adapters/pandas.cc       | 121 ++++++++++++++------
 8 files changed, 250 insertions(+), 168 deletions(-)
 create mode 100644 cpp/cmake_modules/SetupCxxFlags.cmake

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 93e9853df8972..4507e6783e4de 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -105,76 +105,7 @@ endif()
 # Compiler flags
 ############################################################
 
-# Check if the target architecture and compiler supports some special
-# instruction sets that would boost performance.
-include(CheckCXXCompilerFlag)
-# x86/amd64 compiler flags
-CHECK_CXX_COMPILER_FLAG("-msse3" CXX_SUPPORTS_SSE3)
-# power compiler flags
-CHECK_CXX_COMPILER_FLAG("-maltivec" CXX_SUPPORTS_ALTIVEC)
-
-# compiler flags that are common across debug/release builds
-#  - Wall: Enable all warnings.
-set(CXX_COMMON_FLAGS "-std=c++11 -Wall")
-
-# Only enable additional instruction sets if they are supported
-if (CXX_SUPPORTS_SSE3 AND ARROW_SSE3)
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse3")
-endif()
-if (CXX_SUPPORTS_ALTIVEC AND ARROW_ALTIVEC)
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -maltivec")
-endif()
-
-if (APPLE)
-  # Depending on the default OSX_DEPLOYMENT_TARGET (< 10.9), libstdc++ may be
-  # the default standard library which does not support C++11. libc++ is the
-  # default from 10.9 onward.
-  set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++")
-endif()
-
-# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
-# For all builds:
-# For CMAKE_BUILD_TYPE=Debug
-#   -ggdb: Enable gdb debugging
-# For CMAKE_BUILD_TYPE=FastDebug
-#   Same as DEBUG, except with some optimizations on.
-# For CMAKE_BUILD_TYPE=Release
-#   -O3: Enable all compiler optimizations
-#   -g: Enable symbols for profiler tools (TODO: remove for shipping)
-if (NOT MSVC)
-  set(CXX_FLAGS_DEBUG "-ggdb -O0")
-  set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
-  set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
-endif()
-
-set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate")
-set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use")
-
-# if no build build type is specified, default to debug builds
-if (NOT CMAKE_BUILD_TYPE)
-  set(CMAKE_BUILD_TYPE Debug)
-endif(NOT CMAKE_BUILD_TYPE)
-
-string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
-
-
-# Set compile flags based on the build type.
-message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_DEBUG}")
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_FASTDEBUG}")
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_RELEASE}")
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_GEN}")
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_BUILD}")
-else()
-  message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
-endif ()
-
-message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}")
+include(SetupCxxFlags)
 
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
new file mode 100644
index 0000000000000..ee672bd5f6a96
--- /dev/null
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Check if the target architecture and compiler supports some special
+# instruction sets that would boost performance.
+include(CheckCXXCompilerFlag)
+# x86/amd64 compiler flags
+CHECK_CXX_COMPILER_FLAG("-msse3" CXX_SUPPORTS_SSE3)
+# power compiler flags
+CHECK_CXX_COMPILER_FLAG("-maltivec" CXX_SUPPORTS_ALTIVEC)
+
+# compiler flags that are common across debug/release builds
+#  - Wall: Enable all warnings.
+set(CXX_COMMON_FLAGS "-std=c++11 -Wall")
+
+# Only enable additional instruction sets if they are supported
+if (CXX_SUPPORTS_SSE3 AND ARROW_SSE3)
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse3")
+endif()
+if (CXX_SUPPORTS_ALTIVEC AND ARROW_ALTIVEC)
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -maltivec")
+endif()
+
+if (APPLE)
+  # Depending on the default OSX_DEPLOYMENT_TARGET (< 10.9), libstdc++ may be
+  # the default standard library which does not support C++11. libc++ is the
+  # default from 10.9 onward.
+  set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++")
+endif()
+
+# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
+# For all builds:
+# For CMAKE_BUILD_TYPE=Debug
+#   -ggdb: Enable gdb debugging
+# For CMAKE_BUILD_TYPE=FastDebug
+#   Same as DEBUG, except with some optimizations on.
+# For CMAKE_BUILD_TYPE=Release
+#   -O3: Enable all compiler optimizations
+#   -g: Enable symbols for profiler tools (TODO: remove for shipping)
+if (NOT MSVC)
+  set(CXX_FLAGS_DEBUG "-ggdb -O0")
+  set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
+  set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
+endif()
+
+set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate")
+set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use")
+
+# if no build build type is specified, default to debug builds
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Debug)
+endif(NOT CMAKE_BUILD_TYPE)
+
+string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
+
+# Set compile flags based on the build type.
+message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_DEBUG}")
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_FASTDEBUG}")
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_RELEASE}")
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_GEN}")
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_BUILD}")
+else()
+  message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
+endif ()
+
+message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}")
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6ad55f8c9a7b8..6c2477235faaa 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -65,41 +65,7 @@ endif(CCACHE_FOUND)
 # Compiler flags
 ############################################################
 
-# compiler flags that are common across debug/release builds
-set(CXX_COMMON_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall")
-
-# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
-# For all builds:
-# For CMAKE_BUILD_TYPE=Debug
-#   -ggdb: Enable gdb debugging
-# For CMAKE_BUILD_TYPE=FastDebug
-#   Same as DEBUG, except with some optimizations on.
-# For CMAKE_BUILD_TYPE=Release
-#   -O3: Enable all compiler optimizations
-#   -g: Enable symbols for profiler tools (TODO: remove for shipping)
-#   -DNDEBUG: Turn off dchecks/asserts/debug only code.
-set(CXX_FLAGS_DEBUG "-ggdb -O0")
-set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
-set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
-
-# if no build build type is specified, default to debug builds
-if (NOT CMAKE_BUILD_TYPE)
-  set(CMAKE_BUILD_TYPE Debug)
-endif(NOT CMAKE_BUILD_TYPE)
-
-string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
-
-# Set compile flags based on the build type.
-message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_DEBUG})
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
-  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_FASTDEBUG})
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
-  set(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE})
-else()
-  message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
-endif ()
+include(SetupCxxFlags)
 
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 9ede9348c93de..6f81ef470a86c 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -26,6 +26,7 @@
 
 
 import pyarrow.config
+from pyarrow.config import cpu_count, set_cpu_count
 
 from pyarrow.array import (Array,
                            from_pandas_series, from_pylist,
diff --git a/python/pyarrow/config.pyx b/python/pyarrow/config.pyx
index 778c15a5e655b..aa30f097248cd 100644
--- a/python/pyarrow/config.pyx
+++ b/python/pyarrow/config.pyx
@@ -29,3 +29,26 @@ pyarrow_init()
 
 import numpy as np
 pyarrow_set_numpy_nan(np.nan)
+
+import multiprocessing
+import os
+cdef int CPU_COUNT = int(
+    os.environ.get('OMP_NUM_THREADS',
+                   max(multiprocessing.cpu_count() // 2, 1)))
+
+def cpu_count():
+    """
+    Returns
+    -------
+    count : Number of CPUs to use by default in parallel operations. Default is
+      max(1, multiprocessing.cpu_count() / 2), but can be overridden by the
+      OMP_NUM_THREADS environment variable. For the default, we divide the CPU
+      count by 2 because most modern computers have hyperthreading turned on,
+      so doubling the CPU count beyond the number of physical cores does not
+      help.
+    """
+    return CPU_COUNT
+
+def set_cpu_count(count):
+    global CPU_COUNT
+    CPU_COUNT = max(int(count), 1)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 9375557888490..20137e3d4f8d9 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -439,7 +439,9 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
     from pandas.core.internals import BlockManager, make_block
     from pandas import RangeIndex
 
-    check_status(pyarrow.ConvertTableToPandas(table, nthreads, &result_obj))
+    with nogil:
+        check_status(pyarrow.ConvertTableToPandas(table, nthreads,
+                                                  &result_obj))
 
     result = PyObject_to_object(result_obj)
 
@@ -610,36 +612,28 @@ cdef class Table:
         table.init(c_table)
         return table
 
-    def to_pandas(self, nthreads=1, block_based=True):
+    def to_pandas(self, nthreads=None):
         """
         Convert the arrow::Table to a pandas DataFrame
 
+        Parameters
+        ----------
+        nthreads : int, default max(1, multiprocessing.cpu_count() / 2)
+            For the default, we divide the CPU count by 2 because most modern
+            computers have hyperthreading turned on, so doubling the CPU count
+            beyond the number of physical cores does not help
+
         Returns
         -------
         pandas.DataFrame
         """
-        cdef:
-            PyObject* arr
-            shared_ptr[CColumn] col
-            Column column
-
         import pandas as pd
 
-        if block_based:
-            mgr = table_to_blockmanager(self.sp_table, nthreads)
-            return pd.DataFrame(mgr)
-        else:
-            names = []
-            data = []
-            for i in range(self.table.num_columns()):
-                col = self.table.column(i)
-                column = self.column(i)
-                check_status(pyarrow.ConvertColumnToPandas(
-                    col, <PyObject*> column, &arr))
-                names.append(frombytes(col.get().name()))
-                data.append(PyObject_to_object(arr))
-
-            return pd.DataFrame(dict(zip(names, data)), columns=names)
+        if nthreads is None:
+            nthreads = pyarrow.config.cpu_count()
+
+        mgr = table_to_blockmanager(self.sp_table, nthreads)
+        return pd.DataFrame(mgr)
 
     @property
     def name(self):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index da34f85588130..863aa3073fe12 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -27,6 +27,29 @@
 import pyarrow as A
 
 
+def _alltypes_example(size=100):
+    return pd.DataFrame({
+        'uint8': np.arange(size, dtype=np.uint8),
+        'uint16': np.arange(size, dtype=np.uint16),
+        'uint32': np.arange(size, dtype=np.uint32),
+        'uint64': np.arange(size, dtype=np.uint64),
+        'int8': np.arange(size, dtype=np.int16),
+        'int16': np.arange(size, dtype=np.int16),
+        'int32': np.arange(size, dtype=np.int32),
+        'int64': np.arange(size, dtype=np.int64),
+        'float32': np.arange(size, dtype=np.float32),
+        'float64': np.arange(size, dtype=np.float64),
+        'bool': np.random.randn(size) > 0,
+        # TODO(wesm): Pandas only support ns resolution, Arrow supports s, ms,
+        # us, ns
+        'datetime': np.arange("2016-01-01T00:00:00.001", size,
+                              dtype='datetime64[ms]'),
+        'str': [str(x) for x in range(size)],
+        'str_with_nulls': [None] + [str(x) for x in range(size - 2)] + [None],
+        'empty_str': [''] * size
+    })
+
+
 class TestPandasConversion(unittest.TestCase):
 
     def setUp(self):
@@ -35,10 +58,10 @@ def setUp(self):
     def tearDown(self):
         pass
 
-    def _check_pandas_roundtrip(self, df, expected=None,
+    def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
                                 timestamps_to_ms=False):
         table = A.from_pandas_dataframe(df, timestamps_to_ms=timestamps_to_ms)
-        result = table.to_pandas()
+        result = table.to_pandas(nthreads=nthreads)
         if expected is None:
             expected = df
         tm.assert_frame_equal(result, expected)
@@ -217,18 +240,21 @@ def test_timestamps_notimezone_nulls(self):
 
     def test_date(self):
         df = pd.DataFrame({
-            'date': [
-                datetime.date(2000, 1, 1),
-                None,
-                datetime.date(1970, 1, 1),
-                datetime.date(2040, 2, 26)
-        ]})
+            'date': [datetime.date(2000, 1, 1),
+                     None,
+                     datetime.date(1970, 1, 1),
+                     datetime.date(2040, 2, 26)]})
         table = A.from_pandas_dataframe(df)
         result = table.to_pandas()
         expected = df.copy()
         expected['date'] = pd.to_datetime(df['date'])
         tm.assert_frame_equal(result, expected)
 
+    def test_threaded_conversion(self):
+        df = _alltypes_example()
+        self._check_pandas_roundtrip(df, nthreads=2,
+                                     timestamps_to_ms=False)
+
     # def test_category(self):
     #     repeats = 1000
     #     values = [b'foo', None, u'bar', 'qux', np.nan]
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 899eb5519d562..5e5826b8236a6 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -19,15 +19,18 @@
 
 #include <Python.h>
 
-#include "pyarrow/numpy_interop.h"
-
 #include "pyarrow/adapters/pandas.h"
+#include "pyarrow/numpy_interop.h"
 
+#include <algorithm>
+#include <atomic>
 #include <cmath>
 #include <cstdint>
 #include <memory>
+#include <mutex>
 #include <sstream>
 #include <string>
+#include <thread>
 #include <unordered_map>
 
 #include "arrow/api.h"
@@ -1031,7 +1034,8 @@ class PandasBlock {
       : num_rows_(num_rows), num_columns_(num_columns) {}
 
   virtual Status Allocate() = 0;
-  virtual Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) = 0;
+  virtual Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) = 0;
 
   PyObject* block_arr() { return block_arr_.obj(); }
 
@@ -1057,7 +1061,6 @@ class PandasBlock {
 
     block_arr_.reset(block_arr);
     placement_arr_.reset(placement_arr);
-    current_placement_index_ = 0;
 
     block_data_ = reinterpret_cast<uint8_t*>(
         PyArray_DATA(reinterpret_cast<PyArrayObject*>(block_arr)));
@@ -1070,7 +1073,6 @@ class PandasBlock {
 
   int64_t num_rows_;
   int num_columns_;
-  int current_placement_index_;
 
   OwnedRef block_arr_;
   uint8_t* block_data_;
@@ -1088,11 +1090,12 @@ class ObjectBlock : public PandasBlock {
 
   Status Allocate() override { return AllocateNDArray(NPY_OBJECT); }
 
-  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
     Type::type type = col->type()->type;
 
     PyObject** out_buffer =
-        reinterpret_cast<PyObject**>(block_data_) + current_placement_index_ * num_rows_;
+        reinterpret_cast<PyObject**>(block_data_) + rel_placement * num_rows_;
 
     const ChunkedArray& data = *col->data().get();
 
@@ -1108,7 +1111,7 @@ class ObjectBlock : public PandasBlock {
       return Status::NotImplemented(ss.str());
     }
 
-    placement_data_[current_placement_index_++] = placement;
+    placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
 };
@@ -1122,18 +1125,19 @@ class IntBlock : public PandasBlock {
     return AllocateNDArray(arrow_traits<ARROW_TYPE>::npy_type);
   }
 
-  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
     Type::type type = col->type()->type;
 
     C_TYPE* out_buffer =
-        reinterpret_cast<C_TYPE*>(block_data_) + current_placement_index_ * num_rows_;
+        reinterpret_cast<C_TYPE*>(block_data_) + rel_placement * num_rows_;
 
     const ChunkedArray& data = *col->data().get();
 
     if (type != ARROW_TYPE) { return Status::NotImplemented(col->type()->ToString()); }
 
     ConvertIntegerNoNullsSameType<C_TYPE>(data, out_buffer);
-    placement_data_[current_placement_index_++] = placement;
+    placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
 };
@@ -1153,16 +1157,16 @@ class Float32Block : public PandasBlock {
 
   Status Allocate() override { return AllocateNDArray(NPY_FLOAT32); }
 
-  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
     Type::type type = col->type()->type;
 
     if (type != Type::FLOAT) { return Status::NotImplemented(col->type()->ToString()); }
 
-    float* out_buffer =
-        reinterpret_cast<float*>(block_data_) + current_placement_index_ * num_rows_;
+    float* out_buffer = reinterpret_cast<float*>(block_data_) + rel_placement * num_rows_;
 
     ConvertNumericNullable<float>(*col->data().get(), NAN, out_buffer);
-    placement_data_[current_placement_index_++] = placement;
+    placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
 };
@@ -1173,11 +1177,12 @@ class Float64Block : public PandasBlock {
 
   Status Allocate() override { return AllocateNDArray(NPY_FLOAT64); }
 
-  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
     Type::type type = col->type()->type;
 
     double* out_buffer =
-        reinterpret_cast<double*>(block_data_) + current_placement_index_ * num_rows_;
+        reinterpret_cast<double*>(block_data_) + rel_placement * num_rows_;
 
     const ChunkedArray& data = *col->data().get();
 
@@ -1214,7 +1219,7 @@ class Float64Block : public PandasBlock {
 
 #undef INTEGER_CASE
 
-    placement_data_[current_placement_index_++] = placement;
+    placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
 };
@@ -1225,16 +1230,17 @@ class BoolBlock : public PandasBlock {
 
   Status Allocate() override { return AllocateNDArray(NPY_BOOL); }
 
-  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
     Type::type type = col->type()->type;
 
     if (type != Type::BOOL) { return Status::NotImplemented(col->type()->ToString()); }
 
     uint8_t* out_buffer =
-        reinterpret_cast<uint8_t*>(block_data_) + current_placement_index_ * num_rows_;
+        reinterpret_cast<uint8_t*>(block_data_) + rel_placement * num_rows_;
 
     ConvertBooleanNoNulls(*col->data().get(), out_buffer);
-    placement_data_[current_placement_index_++] = placement;
+    placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
 };
@@ -1253,11 +1259,12 @@ class DatetimeBlock : public PandasBlock {
     return Status::OK();
   }
 
-  Status WriteNext(const std::shared_ptr<Column>& col, int64_t placement) override {
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
     Type::type type = col->type()->type;
 
     int64_t* out_buffer =
-        reinterpret_cast<int64_t*>(block_data_) + current_placement_index_ * num_rows_;
+        reinterpret_cast<int64_t*>(block_data_) + rel_placement * num_rows_;
 
     const ChunkedArray& data = *col.get()->data();
 
@@ -1283,7 +1290,7 @@ class DatetimeBlock : public PandasBlock {
       return Status::NotImplemented(col->type()->ToString());
     }
 
-    placement_data_[current_placement_index_++] = placement;
+    placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
 };
@@ -1333,6 +1340,7 @@ class DataFrameBlockCreator {
 
   Status Convert(int nthreads, PyObject** output) {
     column_types_.resize(table_->num_columns());
+    column_block_placement_.resize(table_->num_columns());
     type_counts_.clear();
     blocks_.clear();
 
@@ -1397,7 +1405,9 @@ class DataFrameBlockCreator {
       }
 
       auto it = type_counts_.find(output_type);
+      int block_placement = 0;
       if (it != type_counts_.end()) {
+        block_placement = it->second;
         // Increment count
         it->second += 1;
       } else {
@@ -1406,6 +1416,7 @@ class DataFrameBlockCreator {
       }
 
       column_types_[i] = output_type;
+      column_block_placement_[i] = block_placement;
     }
     return Status::OK();
   }
@@ -1421,22 +1432,61 @@ class DataFrameBlockCreator {
   }
 
   Status WriteTableToBlocks(int nthreads) {
-    if (nthreads > 1) {
-      return Status::NotImplemented("multithreading not yet implemented");
-    }
+    auto WriteColumn = [this](int i) {
+      std::shared_ptr<Column> col = this->table_->column(i);
+      PandasBlock::type output_type = this->column_types_[i];
 
-    for (int i = 0; i < table_->num_columns(); ++i) {
-      std::shared_ptr<Column> col = table_->column(i);
-      PandasBlock::type output_type = column_types_[i];
+      int rel_placement = this->column_block_placement_[i];
+
+      auto it = this->blocks_.find(output_type);
+      if (it == this->blocks_.end()) { return Status::KeyError("No block allocated"); }
+      return it->second->Write(col, i, rel_placement);
+    };
 
-      auto it = blocks_.find(output_type);
-      if (it == blocks_.end()) { return Status::KeyError("No block allocated"); }
-      RETURN_NOT_OK(it->second->WriteNext(col, i));
+    nthreads = std::min<int>(nthreads, table_->num_columns());
+
+    if (nthreads == 1) {
+      for (int i = 0; i < table_->num_columns(); ++i) {
+        RETURN_NOT_OK(WriteColumn(i));
+      }
+    } else {
+      std::vector<std::thread> thread_pool;
+      thread_pool.reserve(nthreads);
+      std::atomic<int> task_counter(0);
+
+      std::mutex error_mtx;
+      bool error_occurred = false;
+      Status error;
+
+      for (int thread_id = 0; thread_id < nthreads; ++thread_id) {
+        thread_pool.emplace_back(
+            [this, &error, &error_occurred, &error_mtx, &task_counter, &WriteColumn]() {
+              int column_num;
+              while (!error_occurred) {
+                column_num = task_counter.fetch_add(1);
+                if (column_num >= this->table_->num_columns()) { break; }
+                Status s = WriteColumn(column_num);
+                if (!s.ok()) {
+                  std::lock_guard<std::mutex> lock(error_mtx);
+                  error_occurred = true;
+                  error = s;
+                  break;
+                }
+              }
+            });
+      }
+      for (auto&& thread : thread_pool) {
+        thread.join();
+      }
+
+      if (error_occurred) { return error; }
     }
     return Status::OK();
   }
 
   Status GetResultList(PyObject** out) {
+    PyAcquireGIL lock;
+
     auto num_blocks = static_cast<Py_ssize_t>(blocks_.size());
     PyObject* result = PyList_New(num_blocks);
     RETURN_IF_PYERROR();
@@ -1463,8 +1513,13 @@ class DataFrameBlockCreator {
 
  private:
   std::shared_ptr<Table> table_;
+
+  // column num -> block type id
   std::vector<PandasBlock::type> column_types_;
 
+  // column num -> relative placement within internal block
+  std::vector<int> column_block_placement_;
+
   // block type -> type count
   std::unordered_map<int, int> type_counts_;
 

From cfbdb680063b15b5068d99175fe2f042d16abf52 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 28 Dec 2016 14:52:43 +0100
Subject: [PATCH 0247/1644] ARROW-322: [C++] Remove ARROW_HDFS option, always
 build the module

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #253 from wesm/ARROW-322 and squashes the following commits:

e793fd1 [Wes McKinney] Use string() instead of native() for file paths because windows uses utf16 native encoding
d0cc376 [Wes McKinney] Add NOMINMAX windows workaround
5e53ddb [Wes McKinney] Visibility fix
ea8fb9d [Wes McKinney] Various Win32 compilation fixes
82c4d2d [Wes McKinney] Remove ARROW_HDFS option, always build the module
---
 ci/travis_before_script_cpp.sh             |  2 -
 cpp/CMakeLists.txt                         |  4 --
 cpp/src/arrow/io/CMakeLists.txt            | 56 ++++++++--------------
 cpp/src/arrow/io/hdfs-internal.cc          | 26 ++--------
 cpp/src/arrow/io/hdfs-internal.h           | 22 ++++++++-
 cpp/src/arrow/io/io-hdfs-test.cc           |  2 +-
 cpp/src/arrow/ipc/json-integration-test.cc |  2 +-
 7 files changed, 47 insertions(+), 67 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 20307736e672a..73bdaeb81fe78 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -26,8 +26,6 @@ CPP_DIR=$TRAVIS_BUILD_DIR/cpp
 
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
--DARROW_PARQUET=OFF \
--DARROW_HDFS=ON \
 -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 4507e6783e4de..47b767119c95b 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -74,10 +74,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow IPC extensions"
     ON)
 
-  option(ARROW_HDFS
-    "Build the Arrow IO extensions for the Hadoop file system"
-    OFF)
-
   option(ARROW_BOOST_USE_SHARED
     "Rely on boost shared libraries where relevant"
     ON)
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 2062cd43b7b48..1e65a1a46abb4 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -45,50 +45,30 @@ set(ARROW_IO_TEST_LINK_LIBS
 
 set(ARROW_IO_SRCS
   file.cc
+  hdfs.cc
+  hdfs-internal.cc
   interfaces.cc
   memory.cc
 )
 
-if(ARROW_HDFS)
-  if(NOT THIRDPARTY_DIR)
-    message(FATAL_ERROR "THIRDPARTY_DIR not set")
-  endif()
-
-  if (DEFINED ENV{HADOOP_HOME})
-    set(HADOOP_HOME $ENV{HADOOP_HOME})
-    if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
-      message(STATUS "Did not find hdfs.h in expected location, using vendored one")
-      set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
-    endif()
-  else()
+# HDFS thirdparty setup
+if (DEFINED ENV{HADOOP_HOME})
+  set(HADOOP_HOME $ENV{HADOOP_HOME})
+  if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
+    message(STATUS "Did not find hdfs.h in expected location, using vendored one")
     set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
   endif()
+else()
+  set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+endif()
 
-  set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
-  if (NOT EXISTS ${HDFS_H_PATH})
-    message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
-  endif()
-  message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
-  message(STATUS "Building libhdfs shim component")
-
-  include_directories(SYSTEM "${HADOOP_HOME}/include")
-
-  set(ARROW_HDFS_SRCS
-    hdfs.cc
-    hdfs-internal.cc)
-
-  set_property(SOURCE ${ARROW_HDFS_SRCS}
-    APPEND_STRING PROPERTY
-    COMPILE_FLAGS "-DHAS_HADOOP")
-
-  set(ARROW_IO_SRCS
-    ${ARROW_HDFS_SRCS}
-    ${ARROW_IO_SRCS})
-
-  ADD_ARROW_TEST(io-hdfs-test)
-  ARROW_TEST_LINK_LIBRARIES(io-hdfs-test
-    ${ARROW_IO_TEST_LINK_LIBS})
+set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
+if (NOT EXISTS ${HDFS_H_PATH})
+  message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
 endif()
+message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
+
+include_directories(SYSTEM "${HADOOP_HOME}/include")
 
 add_library(arrow_io SHARED
   ${ARROW_IO_SRCS}
@@ -119,6 +99,10 @@ ADD_ARROW_TEST(io-file-test)
 ARROW_TEST_LINK_LIBRARIES(io-file-test
   ${ARROW_IO_TEST_LINK_LIBS})
 
+ADD_ARROW_TEST(io-hdfs-test)
+ARROW_TEST_LINK_LIBRARIES(io-hdfs-test
+  ${ARROW_IO_TEST_LINK_LIBS})
+
 ADD_ARROW_TEST(io-memory-test)
 ARROW_TEST_LINK_LIBRARIES(io-memory-test
   ${ARROW_IO_TEST_LINK_LIBS})
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index 7094785de02a0..e4b2cd55978cb 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -28,21 +28,7 @@
 // This software may be modified and distributed under the terms
 // of the BSD license. See the LICENSE file for details.
 
-#ifdef HAS_HADOOP
-
-#ifndef _WIN32
-#include <dlfcn.h>
-#else
-#include <windows.h>
-#include <winsock2.h>
-
-// TODO(wesm): address when/if we add windows support
-// #include <util/syserr_reporting.hpp>
-#endif
-
-extern "C" {
-#include <hdfs.h>
-}
+#include "arrow/io/hdfs-internal.h"
 
 #include <iostream>
 #include <mutex>
@@ -53,7 +39,6 @@ extern "C" {
 
 #include <boost/filesystem.hpp>  // NOLINT
 
-#include "arrow/io/hdfs-internal.h"
 #include "arrow/status.h"
 #include "arrow/util/visibility.h"
 
@@ -265,7 +250,8 @@ static inline void* GetLibrarySymbol(void* handle, const char* symbol) {
   return dlsym(handle, symbol);
 #else
 
-  void* ret = reinterpret_cast<void*>(GetProcAddress(handle, symbol));
+  void* ret = reinterpret_cast<void*>(
+      GetProcAddress(reinterpret_cast<HINSTANCE>(handle), symbol));
   if (ret == NULL) {
     // logstream(LOG_INFO) << "GetProcAddress error: "
     //                     << get_last_err_str(GetLastError()) << std::endl;
@@ -537,7 +523,7 @@ Status LibHdfsShim::GetRequiredSymbols() {
   return Status::OK();
 }
 
-Status ARROW_EXPORT ConnectLibHdfs(LibHdfsShim** driver) {
+Status ConnectLibHdfs(LibHdfsShim** driver) {
   static std::mutex lock;
   std::lock_guard<std::mutex> guard(lock);
 
@@ -562,7 +548,7 @@ Status ARROW_EXPORT ConnectLibHdfs(LibHdfsShim** driver) {
   return shim->GetRequiredSymbols();
 }
 
-Status ARROW_EXPORT ConnectLibHdfs3(LibHdfsShim** driver) {
+Status ConnectLibHdfs3(LibHdfsShim** driver) {
   static std::mutex lock;
   std::lock_guard<std::mutex> guard(lock);
 
@@ -586,5 +572,3 @@ Status ARROW_EXPORT ConnectLibHdfs3(LibHdfsShim** driver) {
 
 }  // namespace io
 }  // namespace arrow
-
-#endif  // HAS_HADOOP
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
index 0ff118a8f57e7..8f9a06758cbaa 100644
--- a/cpp/src/arrow/io/hdfs-internal.h
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -18,8 +18,25 @@
 #ifndef ARROW_IO_HDFS_INTERNAL
 #define ARROW_IO_HDFS_INTERNAL
 
+#ifndef _WIN32
+#include <dlfcn.h>
+#else
+
+// Windows defines min and max macros that mess up std::min/maxa
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <winsock2.h>
+#include <windows.h>
+
+// TODO(wesm): address when/if we add windows support
+// #include <util/syserr_reporting.hpp>
+#endif
+
 #include <hdfs.h>
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Status;
@@ -194,8 +211,9 @@ struct LibHdfsShim {
   Status GetRequiredSymbols();
 };
 
-Status ConnectLibHdfs(LibHdfsShim** driver);
-Status ConnectLibHdfs3(LibHdfsShim** driver);
+// TODO(wesm): Remove these exports when we are linking statically
+Status ARROW_EXPORT ConnectLibHdfs(LibHdfsShim** driver);
+Status ARROW_EXPORT ConnectLibHdfs3(LibHdfsShim** driver);
 
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 4ef47b8babe6e..72e0ba8f2987b 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -79,7 +79,7 @@ class TestHdfsClient : public ::testing::Test {
 
     client_ = nullptr;
     scratch_dir_ =
-        boost::filesystem::unique_path("/tmp/arrow-hdfs/scratch-%%%%").native();
+        boost::filesystem::unique_path("/tmp/arrow-hdfs/scratch-%%%%").string();
 
     loaded_driver_ = false;
 
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 5e593560f8cfa..757e6c00ab243 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -221,7 +221,7 @@ Status RunCommand(const std::string& json_path, const std::string& arrow_path,
 }
 
 static std::string temp_path() {
-  return (fs::temp_directory_path() / fs::unique_path()).native();
+  return (fs::temp_directory_path() / fs::unique_path()).string();
 }
 
 class TestJSONIntegration : public ::testing::Test {

From 8aab00ee16d9dfe7ed578c8dbe59761eaa68670f Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 28 Dec 2016 10:54:27 -0500
Subject: [PATCH 0248/1644] ARROW-330: CMake functions to simplify shared /
 static library configuration

This also fixes ARROW-303

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #255 from xhochy/ARROW-330 and squashes the following commits:

a495d16 [Uwe L. Korn] Fix linking order
17131c9 [Uwe L. Korn] ARROW-330: CMake functions to simplify shared / static library configuration
---
 cpp/CMakeLists.txt                 | 54 ++---------------
 cpp/cmake_modules/BuildUtils.cmake | 77 ++++++++++++++++++++++++
 cpp/src/arrow/io/CMakeLists.txt    | 97 +++++++++++++++---------------
 cpp/src/arrow/io/memory.h          |  2 +-
 cpp/src/arrow/ipc/CMakeLists.txt   | 67 +++++++++------------
 5 files changed, 164 insertions(+), 133 deletions(-)
 create mode 100644 cpp/cmake_modules/BuildUtils.cmake

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 47b767119c95b..bf30543dc4d65 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -717,61 +717,19 @@ set(ARROW_SRCS
   src/arrow/util/bit-util.cc
 )
 
-add_library(arrow_objlib OBJECT
-  ${ARROW_SRCS}
-)
-
-# Necessary to make static linking into other shared libraries work properly
-set_property(TARGET arrow_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
-
 if(NOT APPLE)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
-  set(SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map")
-endif()
-
-if (ARROW_BUILD_SHARED)
-  add_library(arrow_shared SHARED $<TARGET_OBJECTS:arrow_objlib>)
-  if(APPLE)
-    set_target_properties(arrow_shared PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-  endif()
-  set_target_properties(arrow_shared
-    PROPERTIES
-    LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
-    LINK_FLAGS "${SHARED_LINK_FLAGS}"
-    OUTPUT_NAME "arrow")
-  target_link_libraries(arrow_shared
-    LINK_PUBLIC ${ARROW_LINK_LIBS}
-    LINK_PRIVATE ${ARROW_PRIVATE_LINK_LIBS})
-
-  install(TARGETS arrow_shared
-    LIBRARY DESTINATION lib
-    ARCHIVE DESTINATION lib)
+  set(ARROW_SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map")
 endif()
 
-if (ARROW_BUILD_STATIC)
-  add_library(arrow_static STATIC $<TARGET_OBJECTS:arrow_objlib>)
-  set_target_properties(arrow_static
-    PROPERTIES
-    LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
-    OUTPUT_NAME "arrow")
+include(BuildUtils)
 
-  target_link_libraries(arrow_static
-    LINK_PUBLIC ${ARROW_LINK_LIBS}
-    LINK_PRIVATE ${ARROW_PRIVATE_LINK_LIBS})
-
-  install(TARGETS arrow_static
-    LIBRARY DESTINATION lib
-    ARCHIVE DESTINATION lib)
-endif()
-
-if (APPLE)
-  set_target_properties(arrow_shared
-    PROPERTIES
-    BUILD_WITH_INSTALL_RPATH ON
-    INSTALL_NAME_DIR "@rpath")
-endif()
+ADD_ARROW_LIB(arrow
+    SOURCES ${ARROW_SRCS}
+    SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
+)
 
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
new file mode 100644
index 0000000000000..b620de515c126
--- /dev/null
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function(ADD_ARROW_LIB LIB_NAME)
+  set(options)
+  set(one_value_args SHARED_LINK_FLAGS)
+  set(multi_value_args SOURCES STATIC_LINK_LIBS STATIC_PRIVATE_LINK_LIBS SHARED_LINK_LIBS SHARED_PRIVATE_LINK_LIBS)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  add_library(${LIB_NAME}_objlib OBJECT
+      ${ARG_SOURCES}
+  )
+
+  # Necessary to make static linking into other shared libraries work properly
+  set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
+
+  if (ARROW_BUILD_SHARED)
+    add_library(${LIB_NAME}_shared SHARED $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
+    if(APPLE)
+      set_target_properties(${LIB_NAME}_shared PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+    endif()
+    set_target_properties(${LIB_NAME}_shared
+      PROPERTIES
+      LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+      LINK_FLAGS "${ARG_SHARED_LINK_FLAGS}"
+      OUTPUT_NAME ${LIB_NAME})
+  target_link_libraries(${LIB_NAME}_shared
+      LINK_PUBLIC ${ARG_SHARED_LINK_LIBS}
+      LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS})
+  
+  install(TARGETS ${LIB_NAME}_shared
+      LIBRARY DESTINATION lib
+      ARCHIVE DESTINATION lib)
+  endif()
+  
+  if (ARROW_BUILD_STATIC)
+      add_library(${LIB_NAME}_static STATIC $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
+    set_target_properties(${LIB_NAME}_static
+      PROPERTIES
+      LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+      OUTPUT_NAME ${LIB_NAME})
+  
+  target_link_libraries(${LIB_NAME}_static
+      LINK_PUBLIC ${ARG_STATIC_LINK_LIBS}
+      LINK_PRIVATE ${ARG_STATIC_PRIVATE_LINK_LIBS})
+  
+  install(TARGETS ${LIB_NAME}_static
+      LIBRARY DESTINATION lib
+      ARCHIVE DESTINATION lib)
+  endif()
+  
+  if (APPLE)
+      set_target_properties(${LIB_NAME}_shared
+      PROPERTIES
+      BUILD_WITH_INSTALL_RPATH ON
+      INSTALL_NAME_DIR "@rpath")
+  endif()
+
+endfunction()
+
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 1e65a1a46abb4..b8882e46b4893 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -18,30 +18,65 @@
 # ----------------------------------------------------------------------
 # arrow_io : Arrow IO interfaces
 
+# HDFS thirdparty setup
+if (DEFINED ENV{HADOOP_HOME})
+  set(HADOOP_HOME $ENV{HADOOP_HOME})
+  if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
+    message(STATUS "Did not find hdfs.h in expected location, using vendored one")
+    set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+  endif()
+else()
+  set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+endif()
+
+set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
+if (NOT EXISTS ${HDFS_H_PATH})
+  message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
+endif()
+message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
+
+include_directories(SYSTEM "${HADOOP_HOME}/include")
+
+# arrow_io library
 if (MSVC)
-  set(ARROW_IO_LINK_LIBS
+  set(ARROW_IO_STATIC_LINK_LIBS
+    arrow_static
+  )
+  set(ARROW_IO_SHARED_LINK_LIBS
     arrow_shared
   )
 else()
-  set(ARROW_IO_LINK_LIBS
+  set(ARROW_IO_STATIC_LINK_LIBS
+    arrow_static
+    dl
+  )
+  set(ARROW_IO_SHARED_LINK_LIBS
     arrow_shared
     dl
   )
 endif()
 
 if (ARROW_BOOST_USE_SHARED)
-  set(ARROW_IO_PRIVATE_LINK_LIBS
+  set(ARROW_IO_SHARED_PRIVATE_LINK_LIBS
     boost_system_shared
     boost_filesystem_shared)
 else()
-  set(ARROW_IO_PRIVATE_LINK_LIBS
+  set(ARROW_IO_SHARED_PRIVATE_LINK_LIBS
     boost_system_static
     boost_filesystem_static)
 endif()
 
-set(ARROW_IO_TEST_LINK_LIBS
-  arrow_io
-  ${ARROW_IO_PRIVATE_LINK_LIBS})
+set(ARROW_IO_STATIC_PRIVATE_LINK_LIBS
+  boost_system_static
+  boost_filesystem_static)
+
+if (ARROW_BUILD_STATIC)
+  set(ARROW_IO_TEST_LINK_LIBS
+    arrow_io_static)
+else()
+  set(ARROW_IO_TEST_LINK_LIBS
+    arrow_io_shared)
+endif()
 
 set(ARROW_IO_SRCS
   file.cc
@@ -51,32 +86,6 @@ set(ARROW_IO_SRCS
   memory.cc
 )
 
-# HDFS thirdparty setup
-if (DEFINED ENV{HADOOP_HOME})
-  set(HADOOP_HOME $ENV{HADOOP_HOME})
-  if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
-    message(STATUS "Did not find hdfs.h in expected location, using vendored one")
-    set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
-  endif()
-else()
-  set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
-endif()
-
-set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
-if (NOT EXISTS ${HDFS_H_PATH})
-  message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
-endif()
-message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
-
-include_directories(SYSTEM "${HADOOP_HOME}/include")
-
-add_library(arrow_io SHARED
-  ${ARROW_IO_SRCS}
-)
-target_link_libraries(arrow_io
-  LINK_PUBLIC ${ARROW_IO_LINK_LIBS}
-  LINK_PRIVATE ${ARROW_IO_PRIVATE_LINK_LIBS})
-
 if(NOT APPLE)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
@@ -84,16 +93,14 @@ if(NOT APPLE)
   set(ARROW_IO_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
 endif()
 
-SET_TARGET_PROPERTIES(arrow_io PROPERTIES
-  LINKER_LANGUAGE CXX
-  LINK_FLAGS "${ARROW_IO_LINK_FLAGS}")
-
-if (APPLE)
-  set_target_properties(arrow_io
-    PROPERTIES
-    BUILD_WITH_INSTALL_RPATH ON
-    INSTALL_NAME_DIR "@rpath")
-endif()
+ADD_ARROW_LIB(arrow_io
+  SOURCES ${ARROW_IO_SRCS}
+  SHARED_LINK_FLAGS ${ARROW_IO_LINK_FLAGS}
+  SHARED_LINK_LIBS ${ARROW_IO_SHARED_LINK_LIBS}
+  SHARED_PRIVATE_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
+  STATIC_LINK_LIBS ${ARROW_IO_STATIC_LINK_LIBS}
+  STATIC_PRIVATE_LINK_LIBS ${ARROW_IO_STATIC_PRIVATE_LINK_LIBS}
+)
 
 ADD_ARROW_TEST(io-file-test)
 ARROW_TEST_LINK_LIBRARIES(io-file-test
@@ -115,10 +122,6 @@ install(FILES
   memory.h
   DESTINATION include/arrow/io)
 
-install(TARGETS arrow_io
-  LIBRARY DESTINATION lib
-  ARCHIVE DESTINATION lib)
-
 # pkg-config support
 configure_file(arrow-io.pc.in
   "${CMAKE_CURRENT_BINARY_DIR}/arrow-io.pc"
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index b72f93b939148..2faf2804bcbd0 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -101,7 +101,7 @@ class ARROW_EXPORT BufferReader : public ReadableFileInterface {
  public:
   explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
   BufferReader(const uint8_t* data, int64_t size);
-  ~BufferReader();
+  virtual ~BufferReader();
 
   Status Close() override;
   Status Tell(int64_t* position) override;
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index d3e625a08fbfe..11ca19179f3dc 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -19,17 +19,25 @@
 # arrow_ipc
 #######################################
 
-set(ARROW_IPC_LINK_LIBS
-  arrow_io
+set(ARROW_IPC_SHARED_LINK_LIBS
+  arrow_io_shared
   arrow_shared
 )
 
-set(ARROW_IPC_PRIVATE_LINK_LIBS
-  )
+set(ARROW_IPC_STATIC_LINK_LIBS
+  arrow_static
+  arrow_io_static
+)
 
-set(ARROW_IPC_TEST_LINK_LIBS
-  arrow_ipc
-  ${ARROW_IPC_PRIVATE_LINK_LIBS})
+if (ARROW_BUILD_STATIC)
+  set(ARROW_IPC_TEST_LINK_LIBS
+    arrow_io_static
+    arrow_ipc_static)
+else()
+  set(ARROW_IPC_TEST_LINK_LIBS
+    arrow_io_shared
+    arrow_ipc_shared)
+endif()
 
 set(ARROW_IPC_SRCS
   adapter.cc
@@ -40,20 +48,6 @@ set(ARROW_IPC_SRCS
   metadata-internal.cc
 )
 
-# TODO(wesm): SHARED and STATIC targets
-add_library(arrow_ipc SHARED
-  ${ARROW_IPC_SRCS}
-)
-if(RAPIDJSON_VENDORED)
-  add_dependencies(arrow_ipc rapidjson_ep)
-endif()
-if(FLATBUFFERS_VENDORED)
-  add_dependencies(arrow_ipc flatbuffers_ep)
-endif()
-target_link_libraries(arrow_ipc
-  LINK_PUBLIC ${ARROW_IPC_LINK_LIBS}
-  LINK_PRIVATE ${ARROW_IPC_PRIVATE_LINK_LIBS})
-
 if(NOT APPLE)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
@@ -61,15 +55,18 @@ if(NOT APPLE)
   set(ARROW_IPC_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
 endif()
 
-SET_TARGET_PROPERTIES(arrow_ipc PROPERTIES
-  LINKER_LANGUAGE CXX
-  LINK_FLAGS "${ARROW_IPC_LINK_FLAGS}")
+ADD_ARROW_LIB(arrow_ipc
+    SOURCES ${ARROW_IPC_SRCS}
+    SHARED_LINK_FLAGS ${ARROW_IPC_LINK_FLAGS}
+    SHARED_LINK_LIBS ${ARROW_IPC_SHARED_LINK_LIBS}
+    STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
+)
 
-if (APPLE)
-  set_target_properties(arrow_ipc
-    PROPERTIES
-    BUILD_WITH_INSTALL_RPATH ON
-    INSTALL_NAME_DIR "@rpath")
+if(RAPIDJSON_VENDORED)
+  add_dependencies(arrow_ipc_objlib rapidjson_ep)
+endif()
+if(FLATBUFFERS_VENDORED)
+  add_dependencies(arrow_ipc_objlib flatbuffers_ep)
 endif()
 
 ADD_ARROW_TEST(ipc-adapter-test)
@@ -93,9 +90,9 @@ ADD_ARROW_TEST(json-integration-test)
 if (ARROW_BUILD_TESTS)
   if (APPLE)
     target_link_libraries(json-integration-test
+      arrow_ipc_static
+      arrow_io_static
       arrow_static
-      arrow_io
-      arrow_ipc
       gflags
       gtest
       boost_filesystem_static
@@ -105,9 +102,9 @@ if (ARROW_BUILD_TESTS)
       PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
   else()
     target_link_libraries(json-integration-test
+      arrow_ipc_static
+      arrow_io_static
       arrow_static
-      arrow_io
-      arrow_ipc
       gflags
       gtest
       pthread
@@ -156,10 +153,6 @@ install(FILES
   metadata.h
   DESTINATION include/arrow/ipc)
 
-install(TARGETS arrow_ipc
-  LIBRARY DESTINATION lib
-  ARCHIVE DESTINATION lib)
-
 # pkg-config support
 configure_file(arrow-ipc.pc.in
   "${CMAKE_CURRENT_BINARY_DIR}/arrow-ipc.pc"

From 3095f2cb7bc19954d0dfba02486b7ec48d8fef0f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 28 Dec 2016 23:05:50 +0100
Subject: [PATCH 0249/1644] ARROW-444: [Python] Native file reads into
 pre-allocated memory. Some IO API cleanup / niceness

This yields slightly better performance and less memory use. Also deleted some duplicated code

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #257 from wesm/ARROW-444 and squashes the following commits:

30e480d [Wes McKinney] Rename PyBytes_Empty to something more mundane
9db0d81 [Wes McKinney] Native file reads into pre-allocated memory. Deprecated HdfsClient.connect API. Promote pyarrow.io classes into pyarrow namespace
---
 python/pyarrow/__init__.py        |   4 ++
 python/pyarrow/io.pyx             | 109 +++++++++++-------------------
 python/pyarrow/tests/test_hdfs.py |   2 +-
 python/pyarrow/tests/test_io.py   |   4 +-
 4 files changed, 49 insertions(+), 70 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 6f81ef470a86c..02b2b06237de3 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -37,6 +37,10 @@
 
 from pyarrow.error import ArrowException
 
+from pyarrow.io import (HdfsClient, HdfsFile, NativeFile, PythonFileInterface,
+                        BytesReader, Buffer, InMemoryOutputStream,
+                        BufferReader)
+
 from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
                             BooleanValue,
                             Int8Value, Int16Value, Int32Value, Int64Value,
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 8491aa8964fb9..cab6ccb90ee6b 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -37,6 +37,10 @@ import sys
 import threading
 import time
 
+# To let us get a PyObject* and avoid Cython auto-ref-counting
+cdef extern from "Python.h":
+    PyObject* PyBytes_FromStringAndSizeNative" PyBytes_FromStringAndSize"(
+        char *v, Py_ssize_t len) except NULL
 
 cdef class NativeFile:
 
@@ -119,21 +123,24 @@ cdef class NativeFile:
         with nogil:
             check_status(self.wr_file.get().Write(buf, bufsize))
 
-    def read(self, int nbytes):
+    def read(self, int64_t nbytes):
         cdef:
             int64_t bytes_read = 0
-            uint8_t* buf
-            shared_ptr[CBuffer] out
+            PyObject* obj
 
         self._assert_readable()
 
+        # Allocate empty write space
+        obj = PyBytes_FromStringAndSizeNative(NULL, nbytes)
+
+        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AS_STRING(<object> obj)
         with nogil:
-            check_status(self.rd_file.get().ReadB(nbytes, &out))
+            check_status(self.rd_file.get().Read(nbytes, &bytes_read, buf))
 
-        result = cp.PyBytes_FromStringAndSize(
-            <const char*>out.get().data(), out.get().size())
+        if bytes_read < nbytes:
+            cp._PyBytes_Resize(&obj, <Py_ssize_t> bytes_read)
 
-        return result
+        return PyObject_to_object(obj)
 
 
 # ----------------------------------------------------------------------
@@ -339,31 +346,8 @@ cdef class HdfsClient:
     cdef readonly:
         bint is_open
 
-    def __cinit__(self):
-        self.is_open = False
-
-    def __dealloc__(self):
-        if self.is_open:
-            self.close()
-
-    def close(self):
-        """
-        Disconnect from the HDFS cluster
-        """
-        self._ensure_client()
-        with nogil:
-            check_status(self.client.get().Disconnect())
-        self.is_open = False
-
-    cdef _ensure_client(self):
-        if self.client.get() == NULL:
-            raise IOError('HDFS client improperly initialized')
-        elif not self.is_open:
-            raise IOError('HDFS client is closed')
-
-    @classmethod
-    def connect(cls, host="default", port=0, user=None, kerb_ticket=None,
-                driver='libhdfs'):
+    def __cinit__(self, host="default", port=0, user=None, kerb_ticket=None,
+                  driver='libhdfs'):
         """
         Connect to an HDFS cluster. All parameters are optional and should
         only be set if the defaults need to be overridden.
@@ -391,9 +375,7 @@ cdef class HdfsClient:
         -------
         client : HDFSClient
         """
-        cdef:
-            HdfsClient out = HdfsClient()
-            HdfsConnectionConfig conf
+        cdef HdfsConnectionConfig conf
 
         if host is not None:
             conf.host = tobytes(host)
@@ -411,10 +393,31 @@ cdef class HdfsClient:
             conf.driver = HdfsDriver_LIBHDFS3
 
         with nogil:
-            check_status(CHdfsClient.Connect(&conf, &out.client))
-        out.is_open = True
+            check_status(CHdfsClient.Connect(&conf, &self.client))
+        self.is_open = True
 
-        return out
+    @classmethod
+    def connect(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def __dealloc__(self):
+        if self.is_open:
+            self.close()
+
+    def close(self):
+        """
+        Disconnect from the HDFS cluster
+        """
+        self._ensure_client()
+        with nogil:
+            check_status(self.client.get().Disconnect())
+        self.is_open = False
+
+    cdef _ensure_client(self):
+        if self.client.get() == NULL:
+            raise IOError('HDFS client improperly initialized')
+        elif not self.is_open:
+            raise IOError('HDFS client is closed')
 
     def exists(self, path):
         """
@@ -657,36 +660,6 @@ cdef class HdfsFile(NativeFile):
     def __dealloc__(self):
         self.parent = None
 
-    def read(self, int nbytes):
-        """
-        Read indicated number of bytes from the file, up to EOF
-        """
-        cdef:
-            int64_t bytes_read = 0
-            uint8_t* buf
-
-        self._assert_readable()
-
-        # This isn't ideal -- PyBytes_FromStringAndSize copies the data from
-        # the passed buffer, so it's hard for us to avoid doubling the memory
-        buf = <uint8_t*> malloc(nbytes)
-        if buf == NULL:
-            raise MemoryError("Failed to allocate {0} bytes".format(nbytes))
-
-        cdef int64_t total_bytes = 0
-
-        try:
-            with nogil:
-                check_status(self.rd_file.get()
-                             .Read(nbytes, &bytes_read, buf))
-
-            result = cp.PyBytes_FromStringAndSize(<const char*>buf,
-                                                  bytes_read)
-        finally:
-            free(buf)
-
-        return result
-
     def download(self, stream_or_path):
         """
         Read file completely to local path (rather than reading completely into
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 73d5a66cf4765..4ff5a9d42b55e 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -38,7 +38,7 @@ def hdfs_test_client(driver='libhdfs'):
         raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
                          'an integer')
 
-    return io.HdfsClient.connect(host, port, user, driver=driver)
+    return io.HdfsClient(host, port, user, driver=driver)
 
 
 class HdfsTestCases(object):
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 211a12bcd92fe..c10ed0394b1a8 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -67,7 +67,9 @@ def test_python_file_read():
     f.seek(5)
     assert f.tell() == 5
 
-    assert f.read(50) == b'sample data'
+    v = f.read(50)
+    assert v == b'sample data'
+    assert len(v) == 11
 
     f.close()
 

From 4733ee876e1fddb8032fce1dc9e486d68904fbea Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 28 Dec 2016 20:21:48 -0500
Subject: [PATCH 0250/1644] ARROW-445: arrow_ipc_objlib depends on Flatbuffer
 generated files

This is needed as before the depedency was done through the arrow
library on which arrow_ipc depends. But as the arrow_objlib target is
not linked to anything else, it can actually be built independently.
This would lead to races where the flatbuffer generated files were not
existing during arrow_ipc compilation.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #258 from xhochy/ARROW-445 and squashes the following commits:

8bdad8e [Uwe L. Korn] ARROW-445: arrow_ipc_objlib depends on Flatbuffer generated files
---
 cpp/src/arrow/ipc/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 11ca19179f3dc..b7ac5f059749f 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -143,7 +143,7 @@ add_custom_command(
 )
 
 add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES})
-add_dependencies(arrow_objlib metadata_fbs)
+add_dependencies(arrow_ipc_objlib metadata_fbs)
 
 # Headers: top level
 install(FILES

From 23fe6ae02a6fa6ff912986c45079e25b3e5e4deb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 29 Dec 2016 10:22:40 +0100
Subject: [PATCH 0251/1644] ARROW-338: Implement visitor pattern for IPC
 loading/unloading

This is a first cut at getting rid of the if-then-else statements and using the visitor pattern. This also has the benefit of forcing us to provide implementations should we add new types to Arrow.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #256 from wesm/ARROW-338 and squashes the following commits:

59bac66 [Wes McKinney] Fix accidental copy
17214c4 [Wes McKinney] Fix comment
6b00da4 [Wes McKinney] Implement visitor pattern for IPC loading/unloading
---
 cpp/src/arrow/array.h        |   1 +
 cpp/src/arrow/ipc/adapter.cc | 477 ++++++++++++++++++++++-------------
 cpp/src/arrow/type_fwd.h     |   3 +-
 3 files changed, 306 insertions(+), 175 deletions(-)

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 5cd56d6df5427..6239ccc576b8d 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -471,6 +471,7 @@ extern template class ARROW_EXPORT NumericArray<FloatType>;
 extern template class ARROW_EXPORT NumericArray<DoubleType>;
 extern template class ARROW_EXPORT NumericArray<TimestampType>;
 extern template class ARROW_EXPORT NumericArray<DateType>;
+extern template class ARROW_EXPORT NumericArray<TimeType>;
 
 #if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic pop
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index f813c1dbbc3b0..ac4054b376adc 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -34,6 +34,7 @@
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
 
@@ -43,80 +44,34 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
-static bool IsPrimitive(const DataType* type) {
-  DCHECK(type != nullptr);
-  switch (type->type) {
-    // NA is null type or "no type", considered primitive for now
-    case Type::NA:
-    case Type::BOOL:
-    case Type::UINT8:
-    case Type::INT8:
-    case Type::UINT16:
-    case Type::INT16:
-    case Type::UINT32:
-    case Type::INT32:
-    case Type::UINT64:
-    case Type::INT64:
-    case Type::FLOAT:
-    case Type::DOUBLE:
-      return true;
-    default:
-      return false;
-  }
-}
-
 // ----------------------------------------------------------------------
 // Record batch write path
 
-Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes,
-    std::vector<std::shared_ptr<Buffer>>* buffers, int max_recursion_depth) {
-  if (max_recursion_depth <= 0) { return Status::Invalid("Max recursion depth reached"); }
-  DCHECK(arr);
-  DCHECK(field_nodes);
-  // push back all common elements
-  field_nodes->push_back(flatbuf::FieldNode(arr->length(), arr->null_count()));
-  if (arr->null_count() > 0) {
-    buffers->push_back(arr->null_bitmap());
-  } else {
-    // Push a dummy zero-length buffer, not to be copied
-    buffers->push_back(std::make_shared<Buffer>(nullptr, 0));
-  }
-
-  const DataType* arr_type = arr->type().get();
-  if (IsPrimitive(arr_type)) {
-    const auto prim_arr = static_cast<const PrimitiveArray*>(arr);
-    buffers->push_back(prim_arr->data());
-  } else if (arr->type_enum() == Type::STRING || arr->type_enum() == Type::BINARY) {
-    const auto binary_arr = static_cast<const BinaryArray*>(arr);
-    buffers->push_back(binary_arr->offsets());
-    buffers->push_back(binary_arr->data());
-  } else if (arr->type_enum() == Type::LIST) {
-    const auto list_arr = static_cast<const ListArray*>(arr);
-    buffers->push_back(list_arr->offsets());
-    RETURN_NOT_OK(VisitArray(
-        list_arr->values().get(), field_nodes, buffers, max_recursion_depth - 1));
-  } else if (arr->type_enum() == Type::STRUCT) {
-    const auto struct_arr = static_cast<const StructArray*>(arr);
-    for (auto& field : struct_arr->fields()) {
-      RETURN_NOT_OK(
-          VisitArray(field.get(), field_nodes, buffers, max_recursion_depth - 1));
-    }
-  } else {
-    return Status::NotImplemented("Unrecognized type");
-  }
-  return Status::OK();
-}
-
-class RecordBatchWriter {
+class RecordBatchWriter : public ArrayVisitor {
  public:
   RecordBatchWriter(const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows,
       int64_t buffer_start_offset, int max_recursion_depth)
-      : columns_(&columns),
+      : columns_(columns),
         num_rows_(num_rows),
-        buffer_start_offset_(buffer_start_offset),
-        max_recursion_depth_(max_recursion_depth) {}
+        max_recursion_depth_(max_recursion_depth),
+        buffer_start_offset_(buffer_start_offset) {}
 
-  Status AssemblePayload(int64_t* body_length) {
+  Status VisitArray(const Array& arr) {
+    if (max_recursion_depth_ <= 0) {
+      return Status::Invalid("Max recursion depth reached");
+    }
+    // push back all common elements
+    field_nodes_.push_back(flatbuf::FieldNode(arr.length(), arr.null_count()));
+    if (arr.null_count() > 0) {
+      buffers_.push_back(arr.null_bitmap());
+    } else {
+      // Push a dummy zero-length buffer, not to be copied
+      buffers_.push_back(std::make_shared<Buffer>(nullptr, 0));
+    }
+    return arr.Accept(this);
+  }
+
+  Status Assemble(int64_t* body_length) {
     if (field_nodes_.size() > 0) {
       field_nodes_.clear();
       buffer_meta_.clear();
@@ -124,9 +79,8 @@ class RecordBatchWriter {
     }
 
     // Perform depth-first traversal of the row-batch
-    for (size_t i = 0; i < columns_->size(); ++i) {
-      const Array* arr = (*columns_)[i].get();
-      RETURN_NOT_OK(VisitArray(arr, &field_nodes_, &buffers_, max_recursion_depth_));
+    for (size_t i = 0; i < columns_.size(); ++i) {
+      RETURN_NOT_OK(VisitArray(*columns_[i].get()));
     }
 
     // The position for the start of a buffer relative to the passed frame of
@@ -199,7 +153,7 @@ class RecordBatchWriter {
   }
 
   Status Write(io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
-    RETURN_NOT_OK(AssemblePayload(body_length));
+    RETURN_NOT_OK(Assemble(body_length));
 
 #ifndef NDEBUG
     int64_t start_position, current_position;
@@ -249,15 +203,92 @@ class RecordBatchWriter {
   }
 
  private:
+  Status Visit(const NullArray& array) override { return Status::NotImplemented("null"); }
+
+  Status VisitPrimitive(const PrimitiveArray& array) {
+    buffers_.push_back(array.data());
+    return Status::OK();
+  }
+
+  Status VisitBinary(const BinaryArray& array) {
+    buffers_.push_back(array.offsets());
+    buffers_.push_back(array.data());
+    return Status::OK();
+  }
+
+  Status Visit(const BooleanArray& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const Int8Array& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const Int16Array& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const Int32Array& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const Int64Array& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const UInt8Array& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const UInt16Array& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const UInt32Array& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const UInt64Array& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const HalfFloatArray& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const FloatArray& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const DoubleArray& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const StringArray& array) override { return VisitBinary(array); }
+
+  Status Visit(const BinaryArray& array) override { return VisitBinary(array); }
+
+  Status Visit(const DateArray& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const TimeArray& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const TimestampArray& array) override { return VisitPrimitive(array); }
+
+  Status Visit(const IntervalArray& array) override {
+    return Status::NotImplemented("interval");
+  }
+
+  Status Visit(const DecimalArray& array) override {
+    return Status::NotImplemented("decimal");
+  }
+
+  Status Visit(const ListArray& array) override {
+    buffers_.push_back(array.offsets());
+    --max_recursion_depth_;
+    RETURN_NOT_OK(VisitArray(*array.values().get()));
+    ++max_recursion_depth_;
+    return Status::OK();
+  }
+
+  Status Visit(const StructArray& array) override {
+    --max_recursion_depth_;
+    for (const auto& field : array.fields()) {
+      RETURN_NOT_OK(VisitArray(*field.get()));
+    }
+    ++max_recursion_depth_;
+    return Status::OK();
+  }
+
+  Status Visit(const UnionArray& array) override {
+    return Status::NotImplemented("union");
+  }
+
   // Do not copy this vector. Ownership must be retained elsewhere
-  const std::vector<std::shared_ptr<Array>>* columns_;
+  const std::vector<std::shared_ptr<Array>>& columns_;
   int32_t num_rows_;
-  int64_t buffer_start_offset_;
 
   std::vector<flatbuf::FieldNode> field_nodes_;
   std::vector<flatbuf::Buffer> buffer_meta_;
   std::vector<std::shared_ptr<Buffer>> buffers_;
-  int max_recursion_depth_;
+
+  int64_t max_recursion_depth_;
+  int64_t buffer_start_offset_;
 };
 
 Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
@@ -279,143 +310,241 @@ Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size) {
 // ----------------------------------------------------------------------
 // Record batch read path
 
-class RecordBatchReader {
- public:
-  RecordBatchReader(const std::shared_ptr<RecordBatchMetadata>& metadata,
-      const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-      io::ReadableFileInterface* file)
-      : metadata_(metadata),
-        schema_(schema),
-        max_recursion_depth_(max_recursion_depth),
-        file_(file) {
-    num_buffers_ = metadata->num_buffers();
-    num_flattened_fields_ = metadata->num_fields();
-  }
+struct RecordBatchContext {
+  const RecordBatchMetadata* metadata;
+  int buffer_index;
+  int field_index;
+  int max_recursion_depth;
+};
 
-  Status Read(std::shared_ptr<RecordBatch>* out) {
-    std::vector<std::shared_ptr<Array>> arrays(schema_->num_fields());
+// Traverse the flattened record batch metadata and reassemble the
+// corresponding array containers
+class ArrayLoader : public TypeVisitor {
+ public:
+  ArrayLoader(
+      const Field& field, RecordBatchContext* context, io::ReadableFileInterface* file)
+      : field_(field), context_(context), file_(file) {}
 
-    // The field_index and buffer_index are incremented in NextArray based on
-    // how much of the batch is "consumed" (through nested data reconstruction,
-    // for example)
-    field_index_ = 0;
-    buffer_index_ = 0;
-    for (int i = 0; i < schema_->num_fields(); ++i) {
-      const Field* field = schema_->field(i).get();
-      RETURN_NOT_OK(NextArray(field, max_recursion_depth_, &arrays[i]));
+  Status Load(std::shared_ptr<Array>* out) {
+    if (context_->max_recursion_depth <= 0) {
+      return Status::Invalid("Max recursion depth reached");
     }
 
-    *out = std::make_shared<RecordBatch>(schema_, metadata_->length(), arrays);
+    // Load the array
+    RETURN_NOT_OK(field_.type->Accept(this));
+
+    *out = std::move(result_);
     return Status::OK();
   }
 
  private:
-  // Traverse the flattened record batch metadata and reassemble the
-  // corresponding array containers
-  Status NextArray(
-      const Field* field, int max_recursion_depth, std::shared_ptr<Array>* out) {
-    const TypePtr& type = field->type;
-    if (max_recursion_depth <= 0) {
-      return Status::Invalid("Max recursion depth reached");
+  const Field& field_;
+  RecordBatchContext* context_;
+  io::ReadableFileInterface* file_;
+
+  // Used in visitor pattern
+  std::shared_ptr<Array> result_;
+
+  Status LoadChild(const Field& field, std::shared_ptr<Array>* out) {
+    ArrayLoader loader(field, context_, file_);
+    --context_->max_recursion_depth;
+    RETURN_NOT_OK(loader.Load(out));
+    ++context_->max_recursion_depth;
+    return Status::OK();
+  }
+
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
+    BufferMetadata metadata = context_->metadata->buffer(buffer_index);
+
+    if (metadata.length == 0) {
+      *out = std::make_shared<Buffer>(nullptr, 0);
+      return Status::OK();
+    } else {
+      return file_->ReadAt(metadata.offset, metadata.length, out);
     }
+  }
 
+  Status LoadCommon(FieldMetadata* field_meta, std::shared_ptr<Buffer>* null_bitmap) {
     // pop off a field
-    if (field_index_ >= num_flattened_fields_) {
+    if (context_->field_index >= context_->metadata->num_fields()) {
       return Status::Invalid("Ran out of field metadata, likely malformed");
     }
 
     // This only contains the length and null count, which we need to figure
     // out what to do with the buffers. For example, if null_count == 0, then
     // we can skip that buffer without reading from shared memory
-    FieldMetadata field_meta = metadata_->field(field_index_++);
+    *field_meta = context_->metadata->field(context_->field_index++);
 
     // extract null_bitmap which is common to all arrays
+    if (field_meta->null_count == 0) {
+      *null_bitmap = nullptr;
+    } else {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index, null_bitmap));
+    }
+    context_->buffer_index++;
+    return Status::OK();
+  }
+
+  Status LoadPrimitive(const DataType& type) {
+    FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap;
-    if (field_meta.null_count == 0) {
-      ++buffer_index_;
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+
+    std::shared_ptr<Buffer> data;
+    if (field_meta.length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &data));
     } else {
-      RETURN_NOT_OK(GetBuffer(buffer_index_++, &null_bitmap));
+      context_->buffer_index++;
+      data.reset(new Buffer(nullptr, 0));
     }
+    return MakePrimitiveArray(field_.type, field_meta.length, data, field_meta.null_count,
+        null_bitmap, &result_);
+  }
 
-    if (IsPrimitive(type.get())) {
-      std::shared_ptr<Buffer> data;
-      if (field_meta.length > 0) {
-        RETURN_NOT_OK(GetBuffer(buffer_index_++, &data));
-      } else {
-        buffer_index_++;
-        data.reset(new Buffer(nullptr, 0));
-      }
-      return MakePrimitiveArray(
-          type, field_meta.length, data, field_meta.null_count, null_bitmap, out);
-    } else if (type->type == Type::STRING || type->type == Type::BINARY) {
-      std::shared_ptr<Buffer> offsets;
-      std::shared_ptr<Buffer> values;
-      RETURN_NOT_OK(GetBuffer(buffer_index_++, &offsets));
-      RETURN_NOT_OK(GetBuffer(buffer_index_++, &values));
-
-      if (type->type == Type::STRING) {
-        *out = std::make_shared<StringArray>(
-            field_meta.length, offsets, values, field_meta.null_count, null_bitmap);
-      } else {
-        *out = std::make_shared<BinaryArray>(
-            field_meta.length, offsets, values, field_meta.null_count, null_bitmap);
-      }
-      return Status::OK();
-    } else if (type->type == Type::LIST) {
-      std::shared_ptr<Buffer> offsets;
-      RETURN_NOT_OK(GetBuffer(buffer_index_++, &offsets));
-      const int num_children = type->num_children();
-      if (num_children != 1) {
-        std::stringstream ss;
-        ss << "Field: " << field->ToString()
-           << " has wrong number of children:" << num_children;
-        return Status::Invalid(ss.str());
-      }
-      std::shared_ptr<Array> values_array;
-      RETURN_NOT_OK(
-          NextArray(type->child(0).get(), max_recursion_depth - 1, &values_array));
-      *out = std::make_shared<ListArray>(type, field_meta.length, offsets, values_array,
-          field_meta.null_count, null_bitmap);
-      return Status::OK();
-    } else if (type->type == Type::STRUCT) {
-      const int num_children = type->num_children();
-      std::vector<ArrayPtr> fields;
-      fields.reserve(num_children);
-      for (int child_idx = 0; child_idx < num_children; ++child_idx) {
-        std::shared_ptr<Array> field_array;
-        RETURN_NOT_OK(NextArray(
-            type->child(child_idx).get(), max_recursion_depth - 1, &field_array));
-        fields.push_back(field_array);
-      }
-      out->reset(new StructArray(
-          type, field_meta.length, fields, field_meta.null_count, null_bitmap));
-      return Status::OK();
+  template <typename CONTAINER>
+  Status LoadBinary() {
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap;
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+
+    std::shared_ptr<Buffer> offsets;
+    std::shared_ptr<Buffer> values;
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
+
+    result_ = std::make_shared<CONTAINER>(
+        field_meta.length, offsets, values, field_meta.null_count, null_bitmap);
+    return Status::OK();
+  }
+
+  Status Visit(const NullType& type) override { return Status::NotImplemented("null"); }
+
+  Status Visit(const BooleanType& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const Int8Type& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const Int16Type& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const Int32Type& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const Int64Type& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const UInt8Type& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const UInt16Type& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const UInt32Type& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const UInt64Type& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const HalfFloatType& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const FloatType& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const DoubleType& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const StringType& type) override { return LoadBinary<StringArray>(); }
+
+  Status Visit(const BinaryType& type) override { return LoadBinary<BinaryArray>(); }
+
+  Status Visit(const DateType& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const TimeType& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const TimestampType& type) override { return LoadPrimitive(type); }
+
+  Status Visit(const IntervalType& type) override {
+    return Status::NotImplemented(type.ToString());
+  }
+
+  Status Visit(const DecimalType& type) override {
+    return Status::NotImplemented(type.ToString());
+  }
+
+  Status Visit(const ListType& type) override {
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap;
+    std::shared_ptr<Buffer> offsets;
+
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
+
+    const int num_children = type.num_children();
+    if (num_children != 1) {
+      std::stringstream ss;
+      ss << "Wrong number of children: " << num_children;
+      return Status::Invalid(ss.str());
     }
+    std::shared_ptr<Array> values_array;
 
-    return Status::NotImplemented("Non-primitive types not complete yet");
+    RETURN_NOT_OK(LoadChild(*type.child(0).get(), &values_array));
+
+    result_ = std::make_shared<ListArray>(field_.type, field_meta.length, offsets,
+        values_array, field_meta.null_count, null_bitmap);
+    return Status::OK();
   }
 
-  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
-    BufferMetadata metadata = metadata_->buffer(buffer_index);
+  Status Visit(const StructType& type) override {
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap;
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
 
-    if (metadata.length == 0) {
-      *out = std::make_shared<Buffer>(nullptr, 0);
-      return Status::OK();
-    } else {
-      return file_->ReadAt(metadata.offset, metadata.length, out);
+    const int num_children = type.num_children();
+    std::vector<ArrayPtr> fields;
+    fields.reserve(num_children);
+
+    for (int child_idx = 0; child_idx < num_children; ++child_idx) {
+      std::shared_ptr<Array> field_array;
+      RETURN_NOT_OK(LoadChild(*type.child(child_idx).get(), &field_array));
+      fields.emplace_back(field_array);
+    }
+
+    result_ = std::make_shared<StructArray>(
+        field_.type, field_meta.length, fields, field_meta.null_count, null_bitmap);
+    return Status::OK();
+  }
+
+  Status Visit(const UnionType& type) override {
+    return Status::NotImplemented(type.ToString());
+  }
+};
+
+class RecordBatchReader {
+ public:
+  RecordBatchReader(const std::shared_ptr<RecordBatchMetadata>& metadata,
+      const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+      io::ReadableFileInterface* file)
+      : metadata_(metadata),
+        schema_(schema),
+        max_recursion_depth_(max_recursion_depth),
+        file_(file) {}
+
+  Status Read(std::shared_ptr<RecordBatch>* out) {
+    std::vector<std::shared_ptr<Array>> arrays(schema_->num_fields());
+
+    // The field_index and buffer_index are incremented in the ArrayLoader
+    // based on how much of the batch is "consumed" (through nested data
+    // reconstruction, for example)
+    context_.metadata = metadata_.get();
+    context_.field_index = 0;
+    context_.buffer_index = 0;
+    context_.max_recursion_depth = max_recursion_depth_;
+
+    for (int i = 0; i < schema_->num_fields(); ++i) {
+      ArrayLoader loader(*schema_->field(i).get(), &context_, file_);
+      RETURN_NOT_OK(loader.Load(&arrays[i]));
     }
+
+    *out = std::make_shared<RecordBatch>(schema_, metadata_->length(), arrays);
+    return Status::OK();
   }
 
  private:
+  RecordBatchContext context_;
   std::shared_ptr<RecordBatchMetadata> metadata_;
   std::shared_ptr<Schema> schema_;
   int max_recursion_depth_;
   io::ReadableFileInterface* file_;
-
-  int field_index_;
-  int buffer_index_;
-  int num_buffers_;
-  int num_flattened_fields_;
 };
 
 Status ReadRecordBatchMetadata(int64_t offset, int32_t metadata_length,
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index a9db32df54dc3..a14c535b9b3f1 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -91,7 +91,8 @@ using DateArray = NumericArray<DateType>;
 using DateBuilder = NumericBuilder<DateType>;
 
 struct TimeType;
-class TimeArray;
+using TimeArray = NumericArray<TimeType>;
+using TimeBuilder = NumericBuilder<TimeType>;
 
 struct TimestampType;
 using TimestampArray = NumericArray<TimestampType>;

From e15c6a0b3c05b5b42c204f34369d127182450ca0 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 29 Dec 2016 07:05:55 -0500
Subject: [PATCH 0252/1644] ARROW-447: Always return unicode objects for UTF-8
 strings

As the u() function was not working with Unicode characters, this uses
the u'' literal again which was re-introduced with Python 3.3. Thus the
tests will fail with Python3 < 3.3

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #260 from xhochy/ARROW-447 and squashes the following commits:

84d3569 [Uwe L. Korn] ARROW-447: Always return unicode objects for UTF-8 strings
---
 python/pyarrow/scalar.pyx                    | 2 +-
 python/pyarrow/tests/test_convert_builtin.py | 5 +++--
 python/pyarrow/tests/test_convert_pandas.py  | 3 ++-
 python/pyarrow/tests/test_scalars.py         | 7 ++++---
 python/src/pyarrow/adapters/pandas.cc        | 4 ----
 5 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index a0610a14e6bd0..30b90408dc082 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -168,7 +168,7 @@ cdef class StringValue(ArrayValue):
 
     def as_py(self):
         cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
-        return frombytes(ap.GetString(self.index))
+        return ap.GetString(self.index).decode('utf-8')
 
 
 cdef class BinaryValue(ArrayValue):
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index a5f7aa51c29da..61167422de93c 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -72,12 +73,12 @@ def test_double(self):
         assert arr.to_pylist() == data
 
     def test_unicode(self):
-        data = [u('foo'), u('bar'), None, u('arrow')]
+        data = [u'foo', u'bar', None, u'mañana']
         arr = pyarrow.from_pylist(data)
         assert len(arr) == 4
         assert arr.null_count == 1
         assert arr.type == pyarrow.string()
-        assert arr.to_pylist() == [u('foo'), u('bar'), None, u('arrow')]
+        assert arr.to_pylist() == data
 
     def test_bytes(self):
         u1 = b'ma\xc3\xb1ana'
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 863aa3073fe12..bb9f0b3eccab1 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -183,7 +184,7 @@ def test_boolean_object_nulls(self):
 
     def test_unicode(self):
         repeats = 1000
-        values = [u('foo'), None, u('bar'), u('qux'), np.nan]
+        values = [u'foo', None, u'bar', u'mañana', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
 
         self._check_pandas_roundtrip(df)
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 19cfacbcb6b11..62e51f8dee846 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -59,7 +60,7 @@ def test_double(self):
         assert v.as_py() == 3.0
 
     def test_string_unicode(self):
-        arr = A.from_pylist([u('foo'), None, u('bar')])
+        arr = A.from_pylist([u'foo', None, u'mañana'])
 
         v = arr[0]
         assert isinstance(v, A.StringValue)
@@ -68,8 +69,8 @@ def test_string_unicode(self):
         assert arr[1] is A.NA
 
         v = arr[2].as_py()
-        assert v == u('bar')
-        assert isinstance(v, str)
+        assert v == u'mañana'
+        assert isinstance(v, unicode_type)
 
     def test_bytes(self):
         arr = A.from_pylist([b'foo', None, u('bar')])
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 5e5826b8236a6..ad18eca66e42b 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -603,11 +603,7 @@ struct WrapBytes {};
 template <>
 struct WrapBytes<arrow::StringArray> {
   static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
-#if PY_MAJOR_VERSION >= 3
     return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
-#else
-    return PyString_FromStringAndSize(reinterpret_cast<const char*>(data), length);
-#endif
   }
 };
 

From e8b6231b29f59b2978b78a33eff73697d537c5dd Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 2 Jan 2017 02:37:37 -0500
Subject: [PATCH 0253/1644] ARROW-450: Fixes for PARQUET-818

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #263 from xhochy/ARROW-450 and squashes the following commits:

287015a [Uwe L. Korn] ARROW-450: Fixes for PARQUET-818
---
 python/pyarrow/includes/parquet.pxd | 19 ++-----------------
 python/pyarrow/parquet.pyx          | 15 ++++++---------
 2 files changed, 8 insertions(+), 26 deletions(-)

diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index cb791e16f926d..b4d127c871e09 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -120,24 +120,9 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
             shared_ptr[WriterProperties] build()
 
 
-cdef extern from "parquet/arrow/io.h" namespace "parquet::arrow" nogil:
-    cdef cppclass ParquetAllocator:
-        ParquetAllocator()
-        ParquetAllocator(MemoryPool* pool)
-        MemoryPool* pool()
-        void set_pool(MemoryPool* pool)
-
-    cdef cppclass ParquetReadSource:
-        ParquetReadSource(ParquetAllocator* allocator)
-        Open(const shared_ptr[ReadableFileInterface]& file)
-
-    cdef cppclass ParquetWriteSink:
-        ParquetWriteSink(const shared_ptr[OutputStream]& file)
-
-
 cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
     CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
-                     ParquetAllocator* allocator,
+                     MemoryPool* allocator,
                      unique_ptr[FileReader]* reader)
 
     cdef cppclass FileReader:
@@ -157,6 +142,6 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
 cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
     cdef CStatus WriteFlatTable(
         const CTable* table, MemoryPool* pool,
-        const shared_ptr[ParquetWriteSink]& sink,
+        const shared_ptr[OutputStream]& sink,
         int64_t chunk_size,
         const shared_ptr[WriterProperties]& properties)
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 043ccf12d9181..7379456feef2b 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -42,12 +42,12 @@ __all__ = [
 
 cdef class ParquetReader:
     cdef:
-        ParquetAllocator allocator
+        MemoryPool* allocator
         unique_ptr[FileReader] reader
         column_idx_map
 
     def __cinit__(self):
-        self.allocator.set_pool(default_memory_pool())
+        self.allocator = default_memory_pool()
 
     def open(self, source):
         self._open(source)
@@ -69,7 +69,7 @@ cdef class ParquetReader:
                                ParquetFileReader.OpenFile(path)))
         else:
             get_reader(source, &rd_handle)
-            check_status(OpenFile(rd_handle, &self.allocator, &self.reader))
+            check_status(OpenFile(rd_handle, self.allocator, &self.reader))
 
     def read_all(self):
         cdef:
@@ -174,10 +174,8 @@ def write_table(table, sink, chunk_size=None, version=None,
     """
     cdef Table table_ = table
     cdef CTable* ctable_ = table_.table
-    cdef shared_ptr[ParquetWriteSink] sink_
-
     cdef shared_ptr[FileOutputStream] filesink_
-    cdef shared_ptr[OutputStream] general_sink
+    cdef shared_ptr[OutputStream] sink_
 
     cdef WriterProperties.Builder properties_builder
     cdef int64_t chunk_size_ = 0
@@ -237,10 +235,9 @@ def write_table(table, sink, chunk_size=None, version=None,
 
     if isinstance(sink, six.string_types):
         check_status(FileOutputStream.Open(tobytes(sink), &filesink_))
-        sink_.reset(new ParquetWriteSink(<shared_ptr[OutputStream]>filesink_))
+        sink_ = <shared_ptr[OutputStream]>filesink_
     else:
-        get_writer(sink, &general_sink)
-        sink_.reset(new ParquetWriteSink(general_sink))
+        get_writer(sink, &sink_)
 
     with nogil:
         check_status(WriteFlatTable(ctable_, default_memory_pool(), sink_,

From 806239fdd102649b7afa1dbe9aa1c09911f2885e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 2 Jan 2017 08:48:20 -0500
Subject: [PATCH 0254/1644] ARROW-449: Python: Conversion from
 pyarrow.{Table,RecordBatch} to a Python dict

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #262 from xhochy/ARROW-449 and squashes the following commits:

5f15533 [Uwe L. Korn] Fix string conversion routines
9d72c85 [Uwe L. Korn] ARROW-449: Python: Conversion from pyarrow.{Table,RecordBatch} to a Python dict
---
 python/pyarrow/table.pyx           | 36 +++++++++++++++++++++++++++++-
 python/pyarrow/tests/test_table.py | 10 ++++++++-
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 20137e3d4f8d9..925543176c531 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -36,6 +36,9 @@ from pyarrow.compat import frombytes, tobytes
 
 cimport cpython
 
+from collections import OrderedDict
+
+
 cdef class ChunkedArray:
     """
     Array backed via one or more memory chunks.
@@ -204,7 +207,7 @@ cdef class Column:
         -------
         str
         """
-        return frombytes(self.column.name())
+        return bytes(self.column.name()).decode('utf8')
 
     @property
     def type(self):
@@ -345,6 +348,22 @@ cdef class RecordBatch:
 
         return self.batch.Equals(deref(other.batch))
 
+    def to_pydict(self):
+        """
+        Converted the arrow::RecordBatch to an OrderedDict
+
+        Returns
+        -------
+        OrderedDict
+        """
+        entries = []
+        for i in range(self.batch.num_columns()):
+            name = bytes(self.batch.column_name(i)).decode('utf8')
+            column = self[i].to_pylist()
+            entries.append((name, column))
+        return OrderedDict(entries)
+
+
     def to_pandas(self):
         """
         Convert the arrow::RecordBatch to a pandas DataFrame
@@ -635,6 +654,21 @@ cdef class Table:
         mgr = table_to_blockmanager(self.sp_table, nthreads)
         return pd.DataFrame(mgr)
 
+    def to_pydict(self):
+        """
+        Converted the arrow::Table to an OrderedDict
+
+        Returns
+        -------
+        OrderedDict
+        """
+        entries = []
+        for i in range(self.table.num_columns()):
+            name = self.column(i).name
+            column = self.column(i).to_pylist()
+            entries.append((name, column))
+        return OrderedDict(entries)
+
     @property
     def name(self):
         """
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 25463145c00ce..9985b3e29ada1 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from collections import OrderedDict
 import numpy as np
-
 from pandas.util.testing import assert_frame_equal
 import pandas as pd
 import pytest
@@ -35,6 +35,10 @@ def test_recordbatch_basics():
     assert len(batch) == 5
     assert batch.num_rows == 5
     assert batch.num_columns == len(data)
+    assert batch.to_pydict() == OrderedDict([
+        ('c0', [0, 1, 2, 3, 4]),
+        ('c1', [-10, -5, 0, 5, 10])
+    ])
 
 
 def test_recordbatch_from_to_pandas():
@@ -97,6 +101,10 @@ def test_table_basics():
     assert table.num_rows == 5
     assert table.num_columns == 2
     assert table.shape == (5, 2)
+    assert table.to_pydict() == OrderedDict([
+        ('a', [0, 1, 2, 3, 4]),
+        ('b', [-10, -5, 0, 5, 10])
+    ])
 
     for col in table.itercolumns():
         for chunk in col.data.iterchunks():

From 9f7d4ae6da04d9339dfa2811d750ccf616568bc8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 3 Jan 2017 08:27:36 +0100
Subject: [PATCH 0255/1644] ARROW-108: [C++] Add Union implementation and
 IPC/JSON serialization tests

Closes #206.

Still need to add test cases for JSON read/write and dense union IPC. Integration tests can happen in a subsequent PR (but the Java library does not support dense unions yet, so sparse only -- i.e. no offsets vector)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #264 from wesm/ARROW-108 and squashes the following commits:

86c4191 [Wes McKinney] Fix valgrind error
cdfc61d [Wes McKinney] Export UnionArray
3edca1e [Wes McKinney] Implement basic JSON roundtrip for unions
30b7188 [Wes McKinney] Add test case for dense union, implement RangeEquals for it
4887fd2 [Wes McKinney] Move Windows stuff into a compatibility header, exclude from clang-format because of include order sensitivity
5ca9c57 [Wes McKinney] Implement IPC/JSON serializationf or unions. Test UnionMode::SPARSE example in IPC
---
 cpp/CMakeLists.txt                       |   4 +-
 cpp/src/arrow/array-list-test.cc         |   2 +-
 cpp/src/arrow/array-primitive-test.cc    |   2 +-
 cpp/src/arrow/array-struct-test.cc       |   5 +-
 cpp/src/arrow/array-test.cc              |   6 +-
 cpp/src/arrow/array.cc                   | 120 ++++++++++++++++++++---
 cpp/src/arrow/array.h                    |  90 +++++++++++------
 cpp/src/arrow/builder.h                  |   2 +-
 cpp/src/arrow/io/hdfs-internal.h         |  12 +--
 cpp/src/arrow/io/windows_compatibility.h |  36 +++++++
 cpp/src/arrow/ipc/adapter.cc             |  56 ++++++++---
 cpp/src/arrow/ipc/ipc-adapter-test.cc    |   6 +-
 cpp/src/arrow/ipc/ipc-json-test.cc       |  18 +++-
 cpp/src/arrow/ipc/json-internal.cc       |  90 +++++++++++++----
 cpp/src/arrow/ipc/test-common.h          |  83 ++++++++++++++--
 cpp/src/arrow/pretty_print.cc            |  44 ++++++---
 cpp/src/arrow/test-util.h                |  14 ++-
 cpp/src/arrow/type.cc                    |   2 +-
 cpp/src/arrow/type.h                     |   8 +-
 19 files changed, 476 insertions(+), 124 deletions(-)
 create mode 100644 cpp/src/arrow/io/windows_compatibility.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index bf30543dc4d65..13f0354a73b8b 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -668,7 +668,9 @@ endif (UNIX)
 if (${CLANG_FORMAT_FOUND})
   # runs clang format and updates files in place.
   add_custom_target(format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1
-    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`
+    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h |
+    sed -e '/_generated/g' |
+    sed -e '/windows_compatibility.h/g'`
     `find ${CMAKE_CURRENT_SOURCE_DIR}/../python -name \\*.cc -or -name \\*.h`)
 
   # runs clang format and exits with a non-zero exit code if any files need to be reformatted
diff --git a/cpp/src/arrow/array-list-test.cc b/cpp/src/arrow/array-list-test.cc
index 8baaf06a7dbcc..8e4d319f5dca8 100644
--- a/cpp/src/arrow/array-list-test.cc
+++ b/cpp/src/arrow/array-list-test.cc
@@ -89,7 +89,7 @@ class TestListBuilder : public TestBuilder {
 TEST_F(TestListBuilder, Equality) {
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
 
-  ArrayPtr array, equal_array, unequal_array;
+  std::shared_ptr<Array> array, equal_array, unequal_array;
   vector<int32_t> equal_offsets = {0, 1, 2, 5};
   vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2};
   vector<int32_t> unequal_offsets = {0, 1, 4};
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
index a10e2404f29c6..443abac459dbf 100644
--- a/cpp/src/arrow/array-primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -318,7 +318,7 @@ TYPED_TEST(TestPrimitiveBuilder, Equality) {
   this->RandomData(size);
   vector<T>& draws = this->draws_;
   vector<uint8_t>& valid_bytes = this->valid_bytes_;
-  ArrayPtr array, equal_array, unequal_array;
+  std::shared_ptr<Array> array, equal_array, unequal_array;
   auto builder = this->builder_.get();
   ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
   ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &equal_array));
diff --git a/cpp/src/arrow/array-struct-test.cc b/cpp/src/arrow/array-struct-test.cc
index 58386fe028fd2..5827c399dda17 100644
--- a/cpp/src/arrow/array-struct-test.cc
+++ b/cpp/src/arrow/array-struct-test.cc
@@ -261,8 +261,9 @@ TEST_F(TestStructBuilder, BulkAppendInvalid) {
 }
 
 TEST_F(TestStructBuilder, TestEquality) {
-  ArrayPtr array, equal_array;
-  ArrayPtr unequal_bitmap_array, unequal_offsets_array, unequal_values_array;
+  std::shared_ptr<Array> array, equal_array;
+  std::shared_ptr<Array> unequal_bitmap_array, unequal_offsets_array,
+      unequal_values_array;
 
   vector<int32_t> int_values = {1, 2, 3, 4};
   vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 783104e874bb7..a1d8fdfa91e85 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -56,7 +56,8 @@ TEST_F(TestArray, TestLength) {
   ASSERT_EQ(arr->length(), 100);
 }
 
-ArrayPtr MakeArrayFromValidBytes(const std::vector<uint8_t>& v, MemoryPool* pool) {
+std::shared_ptr<Array> MakeArrayFromValidBytes(
+    const std::vector<uint8_t>& v, MemoryPool* pool) {
   int32_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
   std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(v);
 
@@ -65,7 +66,8 @@ ArrayPtr MakeArrayFromValidBytes(const std::vector<uint8_t>& v, MemoryPool* pool
     value_builder.Append<int32_t>(0);
   }
 
-  ArrayPtr arr(new Int32Array(v.size(), value_builder.Finish(), null_count, null_buf));
+  std::shared_ptr<Array> arr(
+      new Int32Array(v.size(), value_builder.Finish(), null_count, null_buf));
   return arr;
 }
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index d13fa1e108196..3d309b8b92f48 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -189,14 +189,14 @@ bool BooleanArray::EqualsExact(const BooleanArray& other) const {
   }
 }
 
-bool BooleanArray::Equals(const ArrayPtr& arr) const {
+bool BooleanArray::Equals(const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) return true;
   if (Type::BOOL != arr->type_enum()) { return false; }
   return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
 }
 
 bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
-    int32_t other_start_idx, const ArrayPtr& arr) const {
+    int32_t other_start_idx, const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) { return true; }
   if (!arr) { return false; }
   if (this->type_enum() != arr->type_enum()) { return false; }
@@ -222,7 +222,7 @@ bool ListArray::EqualsExact(const ListArray& other) const {
   if (null_count_ != other.null_count_) { return false; }
 
   bool equal_offsets =
-      offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
+      offsets_buffer_->Equals(*other.offsets_buffer_, (length_ + 1) * sizeof(int32_t));
   if (!equal_offsets) { return false; }
   bool equal_null_bitmap = true;
   if (null_count_ > 0) {
@@ -269,10 +269,10 @@ bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_st
 
 Status ListArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
-  if (!offset_buffer_) { return Status::Invalid("offset_buffer_ was null"); }
-  if (offset_buffer_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
+  if (!offsets_buffer_) { return Status::Invalid("offsets_buffer_ was null"); }
+  if (offsets_buffer_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
     std::stringstream ss;
-    ss << "offset buffer size (bytes): " << offset_buffer_->size()
+    ss << "offset buffer size (bytes): " << offsets_buffer_->size()
        << " isn't large enough for length: " << length_;
     return Status::Invalid(ss.str());
   }
@@ -337,8 +337,8 @@ BinaryArray::BinaryArray(const TypePtr& type, int32_t length,
     const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
     int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
     : Array(type, length, null_count, null_bitmap),
-      offset_buffer_(offsets),
-      offsets_(reinterpret_cast<const int32_t*>(offset_buffer_->data())),
+      offsets_buffer_(offsets),
+      offsets_(reinterpret_cast<const int32_t*>(offsets_buffer_->data())),
       data_buffer_(data),
       data_(nullptr) {
   if (data_buffer_ != nullptr) { data_ = data_buffer_->data(); }
@@ -353,7 +353,7 @@ bool BinaryArray::EqualsExact(const BinaryArray& other) const {
   if (!Array::EqualsExact(other)) { return false; }
 
   bool equal_offsets =
-      offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
+      offsets_buffer_->Equals(*other.offsets_buffer_, (length_ + 1) * sizeof(int32_t));
   if (!equal_offsets) { return false; }
 
   if (!data_buffer_ && !(other.data_buffer_)) { return true; }
@@ -433,7 +433,7 @@ bool StructArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_
   if (this == arr.get()) { return true; }
   if (!arr) { return false; }
   if (Type::STRUCT != arr->type_enum()) { return false; }
-  const auto other = static_cast<StructArray*>(arr.get());
+  const auto& other = static_cast<const StructArray&>(*arr.get());
 
   bool equal_fields = true;
   for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
@@ -442,7 +442,7 @@ bool StructArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_
     for (size_t j = 0; j < field_arrays_.size(); ++j) {
       // TODO: really we should be comparing stretches of non-null data rather
       // than looking at one value at a time.
-      equal_fields = field(j)->RangeEquals(i, i + 1, o_i, other->field(j));
+      equal_fields = field(j)->RangeEquals(i, i + 1, o_i, other.field(j));
       if (!equal_fields) { return false; }
     }
   }
@@ -490,6 +490,102 @@ Status StructArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+// ----------------------------------------------------------------------
+// UnionArray
+
+UnionArray::UnionArray(const TypePtr& type, int32_t length,
+    const std::vector<std::shared_ptr<Array>>& children,
+    const std::shared_ptr<Buffer>& type_ids, const std::shared_ptr<Buffer>& offsets,
+    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
+    : Array(type, length, null_count, null_bitmap),
+      children_(children),
+      type_ids_buffer_(type_ids),
+      offsets_buffer_(offsets) {
+  type_ids_ = reinterpret_cast<const uint8_t*>(type_ids->data());
+  if (offsets) { offsets_ = reinterpret_cast<const int32_t*>(offsets->data()); }
+}
+
+std::shared_ptr<Array> UnionArray::child(int32_t pos) const {
+  DCHECK_GT(children_.size(), 0);
+  return children_[pos];
+}
+
+bool UnionArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (!this->type_->Equals(arr->type())) { return false; }
+  if (null_count_ != arr->null_count()) { return false; }
+  return RangeEquals(0, length_, 0, arr);
+}
+
+bool UnionArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+    const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (!arr) { return false; }
+  if (Type::UNION != arr->type_enum()) { return false; }
+  const auto& other = static_cast<const UnionArray&>(*arr.get());
+
+  const UnionMode union_mode = mode();
+  if (union_mode != other.mode()) { return false; }
+
+  // Define a mapping from the type id to child number
+  const auto& type_codes = static_cast<const UnionType&>(*arr->type().get()).type_ids;
+  uint8_t max_code = 0;
+  for (uint8_t code : type_codes) {
+    if (code > max_code) { max_code = code; }
+  }
+
+  // Store mapping in a vector for constant time lookups
+  std::vector<uint8_t> type_id_to_child_num(max_code + 1);
+  for (uint8_t i = 0; i < static_cast<uint8_t>(type_codes.size()); ++i) {
+    type_id_to_child_num[type_codes[i]] = i;
+  }
+
+  const uint8_t* this_ids = raw_type_ids();
+  const uint8_t* other_ids = other.raw_type_ids();
+
+  uint8_t id, child_num;
+  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+    if (IsNull(i) != other.IsNull(o_i)) { return false; }
+    if (IsNull(i)) continue;
+    if (this_ids[i] != other_ids[o_i]) { return false; }
+
+    id = this_ids[i];
+    child_num = type_id_to_child_num[id];
+
+    // TODO(wesm): really we should be comparing stretches of non-null data
+    // rather than looking at one value at a time.
+    if (union_mode == UnionMode::SPARSE) {
+      if (!child(child_num)->RangeEquals(i, i + 1, o_i, other.child(child_num))) {
+        return false;
+      }
+    } else {
+      const int32_t offset = offsets_[i];
+      const int32_t o_offset = other.offsets_[i];
+      if (!child(child_num)->RangeEquals(
+              offset, offset + 1, o_offset, other.child(child_num))) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+Status UnionArray::Validate() const {
+  if (length_ < 0) { return Status::Invalid("Length was negative"); }
+
+  if (null_count() > length_) {
+    return Status::Invalid("Null count exceeds the length of this struct");
+  }
+
+  DCHECK(false) << "Validate not yet implemented";
+  return Status::OK();
+}
+
+Status UnionArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 // ----------------------------------------------------------------------
 
 #define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                          \
@@ -499,7 +595,7 @@ Status StructArray::Accept(ArrayVisitor* visitor) const {
 
 Status MakePrimitiveArray(const TypePtr& type, int32_t length,
     const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
+    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out) {
   switch (type->type) {
     MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 6239ccc576b8d..cd42a28e251ca 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -108,8 +108,6 @@ class ARROW_EXPORT NullArray : public Array {
   Status Accept(ArrayVisitor* visitor) const override;
 };
 
-typedef std::shared_ptr<Array> ArrayPtr;
-
 Status ARROW_EXPORT GetEmptyBitmap(
     MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result);
 
@@ -152,7 +150,7 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   }
 
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const ArrayPtr& arr) const override {
+      const std::shared_ptr<Array>& arr) const override {
     if (this == arr.get()) { return true; }
     if (!arr) { return false; }
     if (this->type_enum() != arr->type_enum()) { return false; }
@@ -256,9 +254,9 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
       int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   bool EqualsExact(const BooleanArray& other) const;
-  bool Equals(const ArrayPtr& arr) const override;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const ArrayPtr& arr) const override;
+      const std::shared_ptr<Array>& arr) const override;
 
   Status Accept(ArrayVisitor* visitor) const override;
 
@@ -274,13 +272,13 @@ class ARROW_EXPORT ListArray : public Array {
  public:
   using TypeClass = ListType;
 
-  ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
-      const ArrayPtr& values, int32_t null_count = 0,
-      std::shared_ptr<Buffer> null_bitmap = nullptr)
+  ListArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
+      const std::shared_ptr<Array>& values, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr)
       : Array(type, length, null_count, null_bitmap) {
-    offset_buffer_ = offsets;
+    offsets_buffer_ = offsets;
     offsets_ = offsets == nullptr ? nullptr : reinterpret_cast<const int32_t*>(
-                                                  offset_buffer_->data());
+                                                  offsets_buffer_->data());
     values_ = values;
   }
 
@@ -291,9 +289,7 @@ class ARROW_EXPORT ListArray : public Array {
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   std::shared_ptr<Array> values() const { return values_; }
-  std::shared_ptr<Buffer> offsets() const {
-    return std::static_pointer_cast<Buffer>(offset_buffer_);
-  }
+  std::shared_ptr<Buffer> offsets() const { return offsets_buffer_; }
 
   std::shared_ptr<DataType> value_type() const { return values_->type(); }
 
@@ -309,14 +305,14 @@ class ARROW_EXPORT ListArray : public Array {
   bool Equals(const std::shared_ptr<Array>& arr) const override;
 
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const ArrayPtr& arr) const override;
+      const std::shared_ptr<Array>& arr) const override;
 
   Status Accept(ArrayVisitor* visitor) const override;
 
  protected:
-  std::shared_ptr<Buffer> offset_buffer_;
+  std::shared_ptr<Buffer> offsets_buffer_;
   const int32_t* offsets_;
-  ArrayPtr values_;
+  std::shared_ptr<Array> values_;
 };
 
 // ----------------------------------------------------------------------
@@ -346,7 +342,7 @@ class ARROW_EXPORT BinaryArray : public Array {
   }
 
   std::shared_ptr<Buffer> data() const { return data_buffer_; }
-  std::shared_ptr<Buffer> offsets() const { return offset_buffer_; }
+  std::shared_ptr<Buffer> offsets() const { return offsets_buffer_; }
 
   const int32_t* raw_offsets() const { return offsets_; }
 
@@ -359,14 +355,14 @@ class ARROW_EXPORT BinaryArray : public Array {
   bool EqualsExact(const BinaryArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const ArrayPtr& arr) const override;
+      const std::shared_ptr<Array>& arr) const override;
 
   Status Validate() const override;
 
   Status Accept(ArrayVisitor* visitor) const override;
 
  private:
-  std::shared_ptr<Buffer> offset_buffer_;
+  std::shared_ptr<Buffer> offsets_buffer_;
   const int32_t* offsets_;
 
   std::shared_ptr<Buffer> data_buffer_;
@@ -401,8 +397,9 @@ class ARROW_EXPORT StructArray : public Array {
  public:
   using TypeClass = StructType;
 
-  StructArray(const TypePtr& type, int32_t length, std::vector<ArrayPtr>& field_arrays,
-      int32_t null_count = 0, std::shared_ptr<Buffer> null_bitmap = nullptr)
+  StructArray(const TypePtr& type, int32_t length,
+      const std::vector<std::shared_ptr<Array>>& field_arrays, int32_t null_count = 0,
+      std::shared_ptr<Buffer> null_bitmap = nullptr)
       : Array(type, length, null_count, null_bitmap) {
     type_ = type;
     field_arrays_ = field_arrays;
@@ -416,7 +413,7 @@ class ARROW_EXPORT StructArray : public Array {
   // with this array.
   std::shared_ptr<Array> field(int32_t pos) const;
 
-  const std::vector<ArrayPtr>& fields() const { return field_arrays_; }
+  const std::vector<std::shared_ptr<Array>>& fields() const { return field_arrays_; }
 
   bool EqualsExact(const StructArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
@@ -427,25 +424,54 @@ class ARROW_EXPORT StructArray : public Array {
 
  protected:
   // The child arrays corresponding to each field of the struct data type.
-  std::vector<ArrayPtr> field_arrays_;
+  std::vector<std::shared_ptr<Array>> field_arrays_;
 };
 
 // ----------------------------------------------------------------------
 // Union
 
-class UnionArray : public Array {
+class ARROW_EXPORT UnionArray : public Array {
+ public:
+  using TypeClass = UnionType;
+
+  UnionArray(const TypePtr& type, int32_t length,
+      const std::vector<std::shared_ptr<Array>>& children,
+      const std::shared_ptr<Buffer>& type_ids,
+      const std::shared_ptr<Buffer>& offsets = nullptr, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+  Status Validate() const override;
+
+  virtual ~UnionArray() {}
+
+  std::shared_ptr<Buffer> type_ids() const { return type_ids_buffer_; }
+  const uint8_t* raw_type_ids() const { return type_ids_; }
+
+  std::shared_ptr<Buffer> offsets() const { return offsets_buffer_; }
+  const int32_t* raw_offsets() const { return offsets_; }
+
+  UnionMode mode() const { return static_cast<const UnionType&>(*type_.get()).mode; }
+
+  std::shared_ptr<Array> child(int32_t pos) const;
+
+  const std::vector<std::shared_ptr<Array>>& children() const { return children_; }
+
+  bool EqualsExact(const UnionArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const std::shared_ptr<Array>& arr) const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
+
  protected:
-  // The data are types encoded as int16
-  Buffer* types_;
   std::vector<std::shared_ptr<Array>> children_;
-};
 
-class DenseUnionArray : public UnionArray {
- protected:
-  Buffer* offset_buf_;
-};
+  std::shared_ptr<Buffer> type_ids_buffer_;
+  const uint8_t* type_ids_;
 
-class SparseUnionArray : public UnionArray {};
+  std::shared_ptr<Buffer> offsets_buffer_;
+  const int32_t* offsets_;
+};
 
 // ----------------------------------------------------------------------
 // extern templates and other details
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 205139849b44e..1837340cedc81 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -228,7 +228,7 @@ using DoubleBuilder = NumericBuilder<DoubleType>;
 
 class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
  public:
-  explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type)
+  explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type = boolean())
       : ArrayBuilder(pool, type), data_(nullptr) {}
 
   virtual ~BooleanBuilder() {}
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
index 8f9a06758cbaa..01cf1499857d9 100644
--- a/cpp/src/arrow/io/hdfs-internal.h
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -20,21 +20,11 @@
 
 #ifndef _WIN32
 #include <dlfcn.h>
-#else
-
-// Windows defines min and max macros that mess up std::min/maxa
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-#include <winsock2.h>
-#include <windows.h>
-
-// TODO(wesm): address when/if we add windows support
-// #include <util/syserr_reporting.hpp>
 #endif
 
 #include <hdfs.h>
 
+#include "arrow/io/windows_compatibility.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/io/windows_compatibility.h b/cpp/src/arrow/io/windows_compatibility.h
new file mode 100644
index 0000000000000..ac8f6aeeb5cac
--- /dev/null
+++ b/cpp/src/arrow/io/windows_compatibility.h
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IO_WINDOWS_COMPATIBILITY
+#define ARROW_IO_WINDOWS_COMPATIBILITY
+
+#ifdef _WIN32
+
+// Windows defines min and max macros that mess up std::min/max
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+
+#include <winsock2.h>
+#include <windows.h>
+
+// TODO(wesm): address when/if we add windows support
+// #include <util/syserr_reporting.hpp>
+
+#endif  // _WIN32
+
+#endif  // ARROW_IO_WINDOWS_COMPATIBILITY
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index ac4054b376adc..9bfd11fd01b5a 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -276,7 +276,16 @@ class RecordBatchWriter : public ArrayVisitor {
   }
 
   Status Visit(const UnionArray& array) override {
-    return Status::NotImplemented("union");
+    buffers_.push_back(array.type_ids());
+
+    if (array.mode() == UnionMode::DENSE) { buffers_.push_back(array.offsets()); }
+
+    --max_recursion_depth_;
+    for (const auto& field : array.children()) {
+      RETURN_NOT_OK(VisitArray(*field.get()));
+    }
+    ++max_recursion_depth_;
+    return Status::OK();
   }
 
   // Do not copy this vector. Ownership must be retained elsewhere
@@ -464,9 +473,10 @@ class ArrayLoader : public TypeVisitor {
   Status Visit(const ListType& type) override {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap;
-    std::shared_ptr<Buffer> offsets;
 
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+
+    std::shared_ptr<Buffer> offsets;
     RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
 
     const int num_children = type.num_children();
@@ -484,20 +494,25 @@ class ArrayLoader : public TypeVisitor {
     return Status::OK();
   }
 
+  Status LoadChildren(std::vector<std::shared_ptr<Field>> child_fields,
+      std::vector<std::shared_ptr<Array>>* arrays) {
+    arrays->reserve(static_cast<int>(child_fields.size()));
+
+    for (const auto& child_field : child_fields) {
+      std::shared_ptr<Array> field_array;
+      RETURN_NOT_OK(LoadChild(*child_field.get(), &field_array));
+      arrays->emplace_back(field_array);
+    }
+    return Status::OK();
+  }
+
   Status Visit(const StructType& type) override {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap;
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
 
-    const int num_children = type.num_children();
-    std::vector<ArrayPtr> fields;
-    fields.reserve(num_children);
-
-    for (int child_idx = 0; child_idx < num_children; ++child_idx) {
-      std::shared_ptr<Array> field_array;
-      RETURN_NOT_OK(LoadChild(*type.child(child_idx).get(), &field_array));
-      fields.emplace_back(field_array);
-    }
+    std::vector<std::shared_ptr<Array>> fields;
+    RETURN_NOT_OK(LoadChildren(type.children(), &fields));
 
     result_ = std::make_shared<StructArray>(
         field_.type, field_meta.length, fields, field_meta.null_count, null_bitmap);
@@ -505,7 +520,24 @@ class ArrayLoader : public TypeVisitor {
   }
 
   Status Visit(const UnionType& type) override {
-    return Status::NotImplemented(type.ToString());
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap;
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+
+    std::shared_ptr<Buffer> type_ids;
+    std::shared_ptr<Buffer> offsets = nullptr;
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &type_ids));
+
+    if (type.mode == UnionMode::DENSE) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
+    }
+
+    std::vector<std::shared_ptr<Array>> fields;
+    RETURN_NOT_OK(LoadChildren(type.children(), &fields));
+
+    result_ = std::make_shared<UnionArray>(field_.type, field_meta.length, fields,
+        type_ids, offsets, field_meta.null_count, null_bitmap);
+    return Status::OK();
   }
 };
 
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index f309b8562f76a..6ba0a6e16be08 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -95,7 +95,7 @@ TEST_P(TestWriteRecordBatch, RoundTrip) {
 INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRecordBatch,
     ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch,
                             &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,
-                            &MakeStringTypesRecordBatch, &MakeStruct));
+                            &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion));
 
 void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   ipc::MockOutputStream mock;
@@ -136,7 +136,7 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
       int64_t* body_length, std::shared_ptr<Schema>* schema) {
     const int batch_length = 5;
     TypePtr type = int32();
-    ArrayPtr array;
+    std::shared_ptr<Array> array;
     const bool include_nulls = true;
     RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
     for (int i = 0; i < recursion_level; ++i) {
@@ -149,7 +149,7 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
 
     *schema = std::shared_ptr<Schema>(new Schema({f0}));
 
-    std::vector<ArrayPtr> arrays = {array};
+    std::vector<std::shared_ptr<Array>> arrays = {array};
     auto batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
 
     std::string path = "test-write-past-max-recursion";
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index f793a2659579c..07509890da35c 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -29,6 +29,7 @@
 #include "arrow/builder.h"
 #include "arrow/ipc/json-internal.h"
 #include "arrow/ipc/json.h"
+#include "arrow/ipc/test-common.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
@@ -142,11 +143,16 @@ TEST(TestJsonArrayWriter, NestedTypes) {
   auto value_type = int32();
 
   std::vector<bool> values_is_valid = {true, false, true, true, false, true, true};
-  std::vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
 
+  std::vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
   std::shared_ptr<Array> values_array;
   ArrayFromVector<Int32Type, int32_t>(int32(), values_is_valid, values, &values_array);
 
+  std::vector<int16_t> i16_values = {0, 1, 2, 3, 4, 5, 6};
+  std::shared_ptr<Array> i16_values_array;
+  ArrayFromVector<Int16Type, int16_t>(
+      int16(), values_is_valid, i16_values, &i16_values_array);
+
   // List
   std::vector<bool> list_is_valid = {true, false, true, true, true};
   std::vector<int32_t> offsets = {0, 0, 0, 1, 4, 7};
@@ -173,6 +179,16 @@ TEST(TestJsonArrayWriter, NestedTypes) {
   TestArrayRoundTrip(struct_array);
 }
 
+TEST(TestJsonArrayWriter, Unions) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeUnion(&batch));
+
+  for (int i = 0; i < batch->num_columns(); ++i) {
+    std::shared_ptr<Array> col = batch->column(i);
+    TestArrayRoundTrip(*col.get());
+  }
+}
+
 // Data generation for test case below
 void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
     std::vector<std::shared_ptr<Array>>* arrays) {
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index db11b7d0372f7..4f980d3e5d157 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -415,11 +415,11 @@ class JsonArrayWriter : public ArrayVisitor {
   }
 
   template <typename T>
-  void WriteOffsetsField(const T* offsets, int32_t length) {
-    writer_->Key("OFFSET");
+  void WriteIntegerField(const char* name, const T* values, int32_t length) {
+    writer_->Key(name);
     writer_->StartArray();
     for (int i = 0; i < length; ++i) {
-      writer_->Int64(offsets[i]);
+      writer_->Int64(values[i]);
     }
     writer_->EndArray();
   }
@@ -456,7 +456,7 @@ class JsonArrayWriter : public ArrayVisitor {
   template <typename T>
   Status WriteVarBytes(const T& array) {
     WriteValidityField(array);
-    WriteOffsetsField(array.raw_offsets(), array.length() + 1);
+    WriteIntegerField("OFFSET", array.raw_offsets(), array.length() + 1);
     WriteDataField(array);
     SetNoChildren();
     return Status::OK();
@@ -524,7 +524,7 @@ class JsonArrayWriter : public ArrayVisitor {
 
   Status Visit(const ListArray& array) override {
     WriteValidityField(array);
-    WriteOffsetsField(array.raw_offsets(), array.length() + 1);
+    WriteIntegerField("OFFSET", array.raw_offsets(), array.length() + 1);
     auto type = static_cast<const ListType*>(array.type().get());
     return WriteChildren(type->children(), {array.values()});
   }
@@ -536,7 +536,14 @@ class JsonArrayWriter : public ArrayVisitor {
   }
 
   Status Visit(const UnionArray& array) override {
-    return Status::NotImplemented("union");
+    WriteValidityField(array);
+    auto type = static_cast<const UnionType*>(array.type().get());
+
+    WriteIntegerField("TYPE_ID", array.raw_type_ids(), array.length());
+    if (type->mode == UnionMode::DENSE) {
+      WriteIntegerField("OFFSET", array.raw_offsets(), array.length());
+    }
+    return WriteChildren(type->children(), array.children());
   }
 
  private:
@@ -847,27 +854,35 @@ class JsonArrayReader {
     return builder.Finish(array);
   }
 
+  template <typename T>
+  Status GetIntArray(
+      const RjArray& json_array, const int32_t length, std::shared_ptr<Buffer>* out) {
+    auto buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(buffer->Resize(length * sizeof(T)));
+    T* values = reinterpret_cast<T*>(buffer->mutable_data());
+    for (int i = 0; i < length; ++i) {
+      const rj::Value& val = json_array[i];
+      DCHECK(val.IsInt());
+      values[i] = static_cast<T>(val.GetInt());
+    }
+
+    *out = buffer;
+    return Status::OK();
+  }
+
   template <typename T>
   typename std::enable_if<std::is_base_of<ListType, T>::value, Status>::type ReadArray(
       const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
       const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-    const auto& json_offsets = json_array.FindMember("OFFSET");
-    RETURN_NOT_ARRAY("OFFSET", json_offsets, json_array);
-    const auto& json_offsets_arr = json_offsets->value.GetArray();
-
     int32_t null_count = 0;
     std::shared_ptr<Buffer> validity_buffer;
     RETURN_NOT_OK(GetValidityBuffer(is_valid, &null_count, &validity_buffer));
 
-    auto offsets_buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(offsets_buffer->Resize((length + 1) * sizeof(int32_t)));
-    int32_t* offsets = reinterpret_cast<int32_t*>(offsets_buffer->mutable_data());
-
-    for (int i = 0; i < length + 1; ++i) {
-      const rj::Value& val = json_offsets_arr[i];
-      DCHECK(val.IsInt());
-      offsets[i] = val.GetInt();
-    }
+    const auto& json_offsets = json_array.FindMember("OFFSET");
+    RETURN_NOT_ARRAY("OFFSET", json_offsets, json_array);
+    std::shared_ptr<Buffer> offsets_buffer;
+    RETURN_NOT_OK(GetIntArray<int32_t>(
+        json_offsets->value.GetArray(), length + 1, &offsets_buffer));
 
     std::vector<std::shared_ptr<Array>> children;
     RETURN_NOT_OK(GetChildren(json_array, type, &children));
@@ -896,6 +911,41 @@ class JsonArrayReader {
     return Status::OK();
   }
 
+  template <typename T>
+  typename std::enable_if<std::is_base_of<UnionType, T>::value, Status>::type ReadArray(
+      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+    int32_t null_count = 0;
+
+    const auto& union_type = static_cast<const UnionType&>(*type.get());
+
+    std::shared_ptr<Buffer> validity_buffer;
+    std::shared_ptr<Buffer> type_id_buffer;
+    std::shared_ptr<Buffer> offsets_buffer;
+
+    RETURN_NOT_OK(GetValidityBuffer(is_valid, &null_count, &validity_buffer));
+
+    const auto& json_type_ids = json_array.FindMember("TYPE_ID");
+    RETURN_NOT_ARRAY("TYPE_ID", json_type_ids, json_array);
+    RETURN_NOT_OK(
+        GetIntArray<uint8_t>(json_type_ids->value.GetArray(), length, &type_id_buffer));
+
+    if (union_type.mode == UnionMode::DENSE) {
+      const auto& json_offsets = json_array.FindMember("OFFSET");
+      RETURN_NOT_ARRAY("OFFSET", json_offsets, json_array);
+      RETURN_NOT_OK(
+          GetIntArray<int32_t>(json_offsets->value.GetArray(), length, &offsets_buffer));
+    }
+
+    std::vector<std::shared_ptr<Array>> children;
+    RETURN_NOT_OK(GetChildren(json_array, type, &children));
+
+    *array = std::make_shared<UnionArray>(type, length, children, type_id_buffer,
+        offsets_buffer, null_count, validity_buffer);
+
+    return Status::OK();
+  }
+
   template <typename T>
   typename std::enable_if<std::is_base_of<NullType, T>::value, Status>::type ReadArray(
       const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
@@ -992,7 +1042,7 @@ class JsonArrayReader {
       NOT_IMPLEMENTED_CASE(INTERVAL);
       TYPE_CASE(ListType);
       TYPE_CASE(StructType);
-      NOT_IMPLEMENTED_CASE(UNION);
+      TYPE_CASE(UnionType);
       default:
         std::stringstream ss;
         ss << type->ToString();
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 8416f0df57364..3faeebf956966 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -110,7 +110,7 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
 template <class Builder, class RawType>
 Status MakeRandomBinaryArray(
-    const TypePtr& type, int32_t length, MemoryPool* pool, ArrayPtr* out) {
+    const TypePtr& type, int32_t length, MemoryPool* pool, std::shared_ptr<Array>* out) {
   const std::vector<std::string> values = {
       "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
   Builder builder(pool, type);
@@ -225,7 +225,7 @@ Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
   TypePtr type = int32();
 
   MemoryPool* pool = default_memory_pool();
-  ArrayPtr array;
+  std::shared_ptr<Array> array;
   const bool include_nulls = true;
   RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool, &array));
   for (int i = 0; i < 63; ++i) {
@@ -235,7 +235,7 @@ Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
 
   auto f0 = std::make_shared<Field>("f0", type);
   std::shared_ptr<Schema> schema(new Schema({f0}));
-  std::vector<ArrayPtr> arrays = {array};
+  std::vector<std::shared_ptr<Array>> arrays = {array};
   out->reset(new RecordBatch(schema, batch_length, arrays));
   return Status::OK();
 }
@@ -244,7 +244,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
   // reuse constructed list columns
   std::shared_ptr<RecordBatch> list_batch;
   RETURN_NOT_OK(MakeListRecordBatch(&list_batch));
-  std::vector<ArrayPtr> columns = {
+  std::vector<std::shared_ptr<Array>> columns = {
       list_batch->column(0), list_batch->column(1), list_batch->column(2)};
   auto list_schema = list_batch->schema();
 
@@ -256,20 +256,89 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
   // construct individual nullable/non-nullable struct arrays
-  ArrayPtr no_nulls(new StructArray(type, list_batch->num_rows(), columns));
+  std::shared_ptr<Array> no_nulls(new StructArray(type, list_batch->num_rows(), columns));
   std::vector<uint8_t> null_bytes(list_batch->num_rows(), 1);
   null_bytes[0] = 0;
   std::shared_ptr<Buffer> null_bitmask;
   RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask));
-  ArrayPtr with_nulls(
+  std::shared_ptr<Array> with_nulls(
       new StructArray(type, list_batch->num_rows(), columns, 1, null_bitmask));
 
   // construct batch
-  std::vector<ArrayPtr> arrays = {no_nulls, with_nulls};
+  std::vector<std::shared_ptr<Array>> arrays = {no_nulls, with_nulls};
   out->reset(new RecordBatch(schema, list_batch->num_rows(), arrays));
   return Status::OK();
 }
 
+Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
+  // Define schema
+  std::vector<std::shared_ptr<Field>> union_types(
+      {std::make_shared<Field>("u0", int32()), std::make_shared<Field>("u1", uint8())});
+
+  std::vector<uint8_t> type_codes = {5, 10};
+  auto sparse_type =
+      std::make_shared<UnionType>(union_types, type_codes, UnionMode::SPARSE);
+
+  auto dense_type =
+      std::make_shared<UnionType>(union_types, type_codes, UnionMode::DENSE);
+
+  auto f0 = std::make_shared<Field>("sparse_nonnull", sparse_type, false);
+  auto f1 = std::make_shared<Field>("sparse", sparse_type);
+  auto f2 = std::make_shared<Field>("dense", dense_type);
+
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+
+  // Create data
+  std::vector<std::shared_ptr<Array>> sparse_children(2);
+  std::vector<std::shared_ptr<Array>> dense_children(2);
+
+  const int32_t length = 7;
+
+  std::shared_ptr<Buffer> type_ids_buffer;
+  std::vector<uint8_t> type_ids = {5, 10, 5, 5, 10, 10, 5};
+  RETURN_NOT_OK(test::CopyBufferFromVector(type_ids, &type_ids_buffer));
+
+  std::vector<int32_t> u0_values = {0, 1, 2, 3, 4, 5, 6};
+  ArrayFromVector<Int32Type, int32_t>(
+      sparse_type->child(0)->type, u0_values, &sparse_children[0]);
+
+  std::vector<uint8_t> u1_values = {10, 11, 12, 13, 14, 15, 16};
+  ArrayFromVector<UInt8Type, uint8_t>(
+      sparse_type->child(1)->type, u1_values, &sparse_children[1]);
+
+  // dense children
+  u0_values = {0, 2, 3, 7};
+  ArrayFromVector<Int32Type, int32_t>(
+      dense_type->child(0)->type, u0_values, &dense_children[0]);
+
+  u1_values = {11, 14, 15};
+  ArrayFromVector<UInt8Type, uint8_t>(
+      dense_type->child(1)->type, u1_values, &dense_children[1]);
+
+  std::shared_ptr<Buffer> offsets_buffer;
+  std::vector<int32_t> offsets = {0, 0, 1, 2, 1, 2, 3};
+  RETURN_NOT_OK(test::CopyBufferFromVector(offsets, &offsets_buffer));
+
+  std::vector<uint8_t> null_bytes(length, 1);
+  null_bytes[2] = 0;
+  std::shared_ptr<Buffer> null_bitmask;
+  RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask));
+
+  // construct individual nullable/non-nullable struct arrays
+  auto sparse_no_nulls =
+      std::make_shared<UnionArray>(sparse_type, length, sparse_children, type_ids_buffer);
+  auto sparse = std::make_shared<UnionArray>(
+      sparse_type, length, sparse_children, type_ids_buffer, nullptr, 1, null_bitmask);
+
+  auto dense = std::make_shared<UnionArray>(dense_type, length, dense_children,
+      type_ids_buffer, offsets_buffer, 1, null_bitmask);
+
+  // construct batch
+  std::vector<std::shared_ptr<Array>> arrays = {sparse_no_nulls, sparse, dense};
+  out->reset(new RecordBatch(schema, length, arrays));
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 9c439c47eb82c..324f81bfbfd6b 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -161,44 +161,60 @@ class ArrayPrinter : public ArrayVisitor {
     return Status::NotImplemented("decimal");
   }
 
-  Status Visit(const ListArray& array) override {
+  Status WriteValidityBitmap(const Array& array) {
     Newline();
     Write("-- is_valid: ");
     BooleanArray is_valid(array.length(), array.null_bitmap());
-    PrettyPrint(is_valid, indent_ + 2, sink_);
+    return PrettyPrint(is_valid, indent_ + 2, sink_);
+  }
+
+  Status Visit(const ListArray& array) override {
+    RETURN_NOT_OK(WriteValidityBitmap(array));
 
     Newline();
     Write("-- offsets: ");
     Int32Array offsets(array.length() + 1, array.offsets());
-    PrettyPrint(offsets, indent_ + 2, sink_);
+    RETURN_NOT_OK(PrettyPrint(offsets, indent_ + 2, sink_));
 
     Newline();
     Write("-- values: ");
-    PrettyPrint(*array.values().get(), indent_ + 2, sink_);
+    RETURN_NOT_OK(PrettyPrint(*array.values().get(), indent_ + 2, sink_));
 
     return Status::OK();
   }
 
-  Status Visit(const StructArray& array) override {
-    Newline();
-    Write("-- is_valid: ");
-    BooleanArray is_valid(array.length(), array.null_bitmap());
-    PrettyPrint(is_valid, indent_ + 2, sink_);
-
-    const std::vector<std::shared_ptr<Array>>& fields = array.fields();
+  Status PrintChildren(const std::vector<std::shared_ptr<Array>>& fields) {
     for (size_t i = 0; i < fields.size(); ++i) {
       Newline();
       std::stringstream ss;
       ss << "-- child " << i << " type: " << fields[i]->type()->ToString() << " values: ";
       Write(ss.str());
-      PrettyPrint(*fields[i].get(), indent_ + 2, sink_);
+      RETURN_NOT_OK(PrettyPrint(*fields[i].get(), indent_ + 2, sink_));
     }
-
     return Status::OK();
   }
 
+  Status Visit(const StructArray& array) override {
+    RETURN_NOT_OK(WriteValidityBitmap(array));
+    return PrintChildren(array.fields());
+  }
+
   Status Visit(const UnionArray& array) override {
-    return Status::NotImplemented("union");
+    RETURN_NOT_OK(WriteValidityBitmap(array));
+
+    Newline();
+    Write("-- type_ids: ");
+    UInt8Array type_ids(array.length(), array.type_ids());
+    RETURN_NOT_OK(PrettyPrint(type_ids, indent_ + 2, sink_));
+
+    if (array.mode() == UnionMode::DENSE) {
+      Newline();
+      Write("-- offsets: ");
+      Int32Array offsets(array.length(), array.offsets());
+      RETURN_NOT_OK(PrettyPrint(offsets, indent_ + 2, sink_));
+    }
+
+    return PrintChildren(array.children());
   }
 
   void Write(const char* data) { (*sink_) << data; }
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index aa310b1a49ebe..ce9327d9009e2 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -274,6 +274,18 @@ void ArrayFromVector(const std::shared_ptr<DataType>& type,
       values_buffer, null_count, values_bitmap);
 }
 
+template <typename TYPE, typename C_TYPE>
+void ArrayFromVector(const std::shared_ptr<DataType>& type,
+    const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
+  std::shared_ptr<Buffer> values_buffer;
+
+  ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
+
+  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
+  *out = std::make_shared<ArrayType>(
+      type, static_cast<int32_t>(values.size()), values_buffer);
+}
+
 class TestBuilder : public ::testing::Test {
  public:
   void SetUp() {
@@ -293,7 +305,7 @@ class TestBuilder : public ::testing::Test {
 
 template <class T, class Builder>
 Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
-    int size, Builder* builder, ArrayPtr* out) {
+    int size, Builder* builder, std::shared_ptr<Array>* out) {
   // Append the first 1000
   for (int i = 0; i < size; ++i) {
     if (valid_bytes[i] > 0) {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 8ff9eea87051d..89faab6ec6ae2 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -103,7 +103,7 @@ std::string UnionType::ToString() const {
 
   for (size_t i = 0; i < children_.size(); ++i) {
     if (i) { s << ", "; }
-    s << children_[i]->ToString();
+    s << children_[i]->ToString() << "=" << static_cast<int>(type_ids[i]);
   }
   s << ">";
   return s.str();
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 73005707c9edc..530c3235dc9ab 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -394,10 +394,10 @@ enum class UnionMode : char { SPARSE, DENSE };
 struct ARROW_EXPORT UnionType : public DataType {
   static constexpr Type::type type_id = Type::UNION;
 
-  UnionType(const std::vector<std::shared_ptr<Field>>& child_fields,
+  UnionType(const std::vector<std::shared_ptr<Field>>& fields,
       const std::vector<uint8_t>& type_ids, UnionMode mode = UnionMode::SPARSE)
       : DataType(Type::UNION), mode(mode), type_ids(type_ids) {
-    children_ = child_fields;
+    children_ = fields;
   }
 
   std::string ToString() const override;
@@ -407,6 +407,10 @@ struct ARROW_EXPORT UnionType : public DataType {
   std::vector<BufferDescr> GetBufferLayout() const override;
 
   UnionMode mode;
+
+  // The type id used in the data to indicate each data type in the union. For
+  // example, the first type in the union might be denoted by the id 5 (instead
+  // of 0).
   std::vector<uint8_t> type_ids;
 };
 

From d9df556791fc6051b2c8582668df9c256f675116 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 3 Jan 2017 08:28:46 +0100
Subject: [PATCH 0256/1644] ARROW-294: [C++] Do not use platform-dependent
 fopen/fclose functions for MemoryMappedFile

Also adds a test case for ARROW-340.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #265 from wesm/ARROW-294 and squashes the following commits:

42a83a4 [Wes McKinney] Remove duplicated includes
3928ab0 [Wes McKinney] Base MemoryMappedFile implementation on common OSFile interface. Add test case for ARROW-340.
---
 cpp/src/arrow/io/file.cc           | 208 +++++++++++++++++++++++++++--
 cpp/src/arrow/io/file.h            |  49 +++++++
 cpp/src/arrow/io/io-file-test.cc   | 116 +++++++++++++++-
 cpp/src/arrow/io/io-memory-test.cc |  91 -------------
 cpp/src/arrow/io/memory.cc         | 178 ------------------------
 cpp/src/arrow/io/memory.h          |  39 ------
 cpp/src/arrow/io/test-common.h     |   1 +
 7 files changed, 359 insertions(+), 323 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index c50a9bba28e8e..3182f2dd8a3b5 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -60,7 +60,7 @@
 
 #endif  // _MSC_VER
 
-// defines that
+// defines that don't exist in MinGW
 #if defined(__MINGW32__)
 #define ARROW_WRITE_SHMODE S_IRUSR | S_IWUSR
 #elif defined(_MSC_VER)  // Visual Studio
@@ -174,7 +174,8 @@ static inline Status FileOpenReadable(const std::string& filename, int* fd) {
   return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
 }
 
-static inline Status FileOpenWriteable(const std::string& filename, int* fd) {
+static inline Status FileOpenWriteable(
+    const std::string& filename, bool write_only, bool truncate, int* fd) {
   int ret;
   errno_t errno_actual = 0;
 
@@ -186,13 +187,31 @@ static inline Status FileOpenWriteable(const std::string& filename, int* fd) {
   memcpy(wpath.data(), filename.data(), filename.size());
   memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
 
-  errno_actual = _wsopen_s(fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY | _O_TRUNC,
-      _SH_DENYNO, _S_IWRITE);
+  int oflag = _O_CREAT | _O_BINARY;
+
+  if (truncate) { oflag |= _O_TRUNC; }
+
+  if (write_only) {
+    oflag |= _O_WRONLY;
+  } else {
+    oflag |= _O_RDWR;
+  }
+
+  errno_actual = _wsopen_s(fd, wpath.data(), oflag, _SH_DENYNO, _S_IWRITE);
   ret = *fd;
 
 #else
-  ret = *fd =
-      open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY | O_TRUNC, ARROW_WRITE_SHMODE);
+  int oflag = O_CREAT | O_BINARY;
+
+  if (truncate) { oflag |= O_TRUNC; }
+
+  if (write_only) {
+    oflag |= O_WRONLY;
+  } else {
+    oflag |= O_RDWR;
+  }
+
+  ret = *fd = open(filename.c_str(), oflag, ARROW_WRITE_SHMODE);
 #endif
   return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
 }
@@ -296,10 +315,17 @@ class OSFile {
 
   ~OSFile() {}
 
-  Status OpenWritable(const std::string& path) {
-    RETURN_NOT_OK(FileOpenWriteable(path, &fd_));
+  Status OpenWriteable(const std::string& path, bool append, bool write_only) {
+    RETURN_NOT_OK(FileOpenWriteable(path, write_only, !append, &fd_));
     path_ = path;
     is_open_ = true;
+    mode_ = write_only ? FileMode::READ : FileMode::READWRITE;
+
+    if (append) {
+      RETURN_NOT_OK(FileGetSize(fd_, &size_));
+    } else {
+      size_ = 0;
+    }
     return Status::OK();
   }
 
@@ -307,11 +333,9 @@ class OSFile {
     RETURN_NOT_OK(FileOpenReadable(path, &fd_));
     RETURN_NOT_OK(FileGetSize(fd_, &size_));
 
-    // The position should be 0 after GetSize
-    // RETURN_NOT_OK(Seek(0));
-
     path_ = path;
     is_open_ = true;
+    mode_ = FileMode::READ;
     return Status::OK();
   }
 
@@ -346,12 +370,14 @@ class OSFile {
 
   int64_t size() const { return size_; }
 
- private:
+ protected:
   std::string path_;
 
   // File descriptor
   int fd_;
 
+  FileMode::type mode_;
+
   bool is_open_;
   int64_t size_;
 };
@@ -440,7 +466,9 @@ int ReadableFile::file_descriptor() const {
 
 class FileOutputStream::FileOutputStreamImpl : public OSFile {
  public:
-  Status Open(const std::string& path) { return OpenWritable(path); }
+  Status Open(const std::string& path, bool append) {
+    return OpenWriteable(path, append, true);
+  }
 };
 
 FileOutputStream::FileOutputStream() {
@@ -453,9 +481,14 @@ FileOutputStream::~FileOutputStream() {
 
 Status FileOutputStream::Open(
     const std::string& path, std::shared_ptr<FileOutputStream>* file) {
+  return Open(path, false, file);
+}
+
+Status FileOutputStream::Open(
+    const std::string& path, bool append, std::shared_ptr<FileOutputStream>* file) {
   // private ctor
   *file = std::shared_ptr<FileOutputStream>(new FileOutputStream());
-  return (*file)->impl_->Open(path);
+  return (*file)->impl_->Open(path, append);
 }
 
 Status FileOutputStream::Close() {
@@ -474,5 +507,152 @@ int FileOutputStream::file_descriptor() const {
   return impl_->fd();
 }
 
+// ----------------------------------------------------------------------
+// Implement MemoryMappedFile
+
+class MemoryMappedFile::MemoryMappedFileImpl : public OSFile {
+ public:
+  MemoryMappedFileImpl() : OSFile(), data_(nullptr) {}
+
+  ~MemoryMappedFileImpl() {
+    if (is_open_) {
+      munmap(data_, size_);
+      OSFile::Close();
+    }
+  }
+
+  Status Open(const std::string& path, FileMode::type mode) {
+    int prot_flags = PROT_READ;
+
+    if (mode != FileMode::READ) {
+      prot_flags |= PROT_WRITE;
+      const bool append = true;
+      RETURN_NOT_OK(OSFile::OpenWriteable(path, append, mode == FileMode::WRITE));
+    } else {
+      RETURN_NOT_OK(OSFile::OpenReadable(path));
+    }
+
+    void* result = mmap(nullptr, size_, prot_flags, MAP_SHARED, fd(), 0);
+    if (result == MAP_FAILED) {
+      std::stringstream ss;
+      ss << "Memory mapping file failed, errno: " << errno;
+      return Status::IOError(ss.str());
+    }
+    data_ = reinterpret_cast<uint8_t*>(result);
+    position_ = 0;
+
+    return Status::OK();
+  }
+
+  int64_t size() const { return size_; }
+
+  Status Seek(int64_t position) {
+    if (position < 0 || position >= size_) {
+      return Status::Invalid("position is out of bounds");
+    }
+    position_ = position;
+    return Status::OK();
+  }
+
+  int64_t position() { return position_; }
+
+  void advance(int64_t nbytes) { position_ = std::min(size_, position_ + nbytes); }
+
+  uint8_t* data() { return data_; }
+
+  uint8_t* head() { return data_ + position_; }
+
+  bool writable() { return mode_ != FileMode::READ; }
+
+  bool opened() { return is_open_; }
+
+ private:
+  int64_t position_;
+
+  // The memory map
+  uint8_t* data_;
+};
+
+MemoryMappedFile::MemoryMappedFile(FileMode::type mode) {
+  ReadableFileInterface::set_mode(mode);
+}
+
+MemoryMappedFile::~MemoryMappedFile() {}
+
+Status MemoryMappedFile::Open(const std::string& path, FileMode::type mode,
+    std::shared_ptr<MemoryMappedFile>* out) {
+  std::shared_ptr<MemoryMappedFile> result(new MemoryMappedFile(mode));
+
+  result->impl_.reset(new MemoryMappedFileImpl());
+  RETURN_NOT_OK(result->impl_->Open(path, mode));
+
+  *out = result;
+  return Status::OK();
+}
+
+Status MemoryMappedFile::GetSize(int64_t* size) {
+  *size = impl_->size();
+  return Status::OK();
+}
+
+Status MemoryMappedFile::Tell(int64_t* position) {
+  *position = impl_->position();
+  return Status::OK();
+}
+
+Status MemoryMappedFile::Seek(int64_t position) {
+  return impl_->Seek(position);
+}
+
+Status MemoryMappedFile::Close() {
+  // munmap handled in pimpl dtor
+  return Status::OK();
+}
+
+Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  nbytes = std::min(nbytes, impl_->size() - impl_->position());
+  std::memcpy(out, impl_->head(), nbytes);
+  *bytes_read = nbytes;
+  impl_->advance(nbytes);
+  return Status::OK();
+}
+
+Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  nbytes = std::min(nbytes, impl_->size() - impl_->position());
+  *out = std::make_shared<Buffer>(impl_->head(), nbytes);
+  impl_->advance(nbytes);
+  return Status::OK();
+}
+
+bool MemoryMappedFile::supports_zero_copy() const {
+  return true;
+}
+
+Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+  if (!impl_->opened() || !impl_->writable()) {
+    return Status::IOError("Unable to write");
+  }
+
+  RETURN_NOT_OK(impl_->Seek(position));
+  return WriteInternal(data, nbytes);
+}
+
+Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
+  if (!impl_->opened() || !impl_->writable()) {
+    return Status::IOError("Unable to write");
+  }
+  if (nbytes + impl_->position() > impl_->size()) {
+    return Status::Invalid("Cannot write past end of memory map");
+  }
+
+  return WriteInternal(data, nbytes);
+}
+
+Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
+  memcpy(impl_->head(), data, nbytes);
+  impl_->advance(nbytes);
+  return Status::OK();
+}
+
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 10fe16e511210..9ca9c540e7c22 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -40,8 +40,13 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
  public:
   ~FileOutputStream();
 
+  // When opening a new file, any existing file with the indicated path is
+  // truncated to 0 bytes, deleting any existing memory
   static Status Open(const std::string& path, std::shared_ptr<FileOutputStream>* file);
 
+  static Status Open(
+      const std::string& path, bool append, std::shared_ptr<FileOutputStream>* file);
+
   // OutputStream interface
   Status Close() override;
   Status Tell(int64_t* position) override;
@@ -88,6 +93,50 @@ class ARROW_EXPORT ReadableFile : public ReadableFileInterface {
   std::unique_ptr<ReadableFileImpl> impl_;
 };
 
+// A file interface that uses memory-mapped files for memory interactions,
+// supporting zero copy reads. The same class is used for both reading and
+// writing.
+//
+// If opening a file in a writeable mode, it is not truncated first as with
+// FileOutputStream
+class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
+ public:
+  ~MemoryMappedFile();
+
+  static Status Open(const std::string& path, FileMode::type mode,
+      std::shared_ptr<MemoryMappedFile>* out);
+
+  Status Close() override;
+
+  Status Tell(int64_t* position) override;
+
+  Status Seek(int64_t position) override;
+
+  // Required by ReadableFileInterface, copies memory into out
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+
+  // Zero copy read
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
+  bool supports_zero_copy() const override;
+
+  Status Write(const uint8_t* data, int64_t nbytes) override;
+
+  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+
+  // @return: the size in bytes of the memory source
+  Status GetSize(int64_t* size) override;
+
+ private:
+  explicit MemoryMappedFile(FileMode::type mode);
+
+  Status WriteInternal(const uint8_t* data, int64_t nbytes);
+
+  // Hide the internal details of this class for now
+  class ARROW_NO_EXPORT MemoryMappedFileImpl;
+  std::unique_ptr<MemoryMappedFileImpl> impl_;
+};
+
 }  // namespace io
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index f0ea7ec5e4dea..5f5d639fab0d8 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -71,7 +71,9 @@ class FileTestFixture : public ::testing::Test {
 
 class TestFileOutputStream : public FileTestFixture {
  public:
-  void OpenFile() { ASSERT_OK(FileOutputStream::Open(path_, &file_)); }
+  void OpenFile(bool append = false) {
+    ASSERT_OK(FileOutputStream::Open(path_, append, &file_));
+  }
 
  protected:
   std::shared_ptr<FileOutputStream> file_;
@@ -131,6 +133,24 @@ TEST_F(TestFileOutputStream, Tell) {
   ASSERT_EQ(8, position);
 }
 
+TEST_F(TestFileOutputStream, TruncatesNewFile) {
+  ASSERT_OK(FileOutputStream::Open(path_, &file_));
+
+  const char* data = "testdata";
+  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), strlen(data)));
+  ASSERT_OK(file_->Close());
+
+  ASSERT_OK(FileOutputStream::Open(path_, &file_));
+  ASSERT_OK(file_->Close());
+
+  std::shared_ptr<ReadableFile> rd_file;
+  ASSERT_OK(ReadableFile::Open(path_, &rd_file));
+
+  int64_t size;
+  ASSERT_OK(rd_file->GetSize(&size));
+  ASSERT_EQ(0, size);
+}
+
 // ----------------------------------------------------------------------
 // File input tests
 
@@ -293,5 +313,99 @@ TEST_F(TestReadableFile, CustomMemoryPool) {
   ASSERT_EQ(2, pool.num_allocations());
 }
 
+// ----------------------------------------------------------------------
+// Memory map tests
+
+class TestMemoryMappedFile : public ::testing::Test, public MemoryMapFixture {
+ public:
+  void TearDown() { MemoryMapFixture::TearDown(); }
+};
+
+TEST_F(TestMemoryMappedFile, InvalidUsages) {}
+
+TEST_F(TestMemoryMappedFile, WriteRead) {
+  const int64_t buffer_size = 1024;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(1024, 0, buffer.data());
+
+  const int reps = 5;
+
+  std::string path = "ipc-write-read-test";
+  CreateFile(path, reps * buffer_size);
+
+  std::shared_ptr<MemoryMappedFile> result;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &result));
+
+  int64_t position = 0;
+  std::shared_ptr<Buffer> out_buffer;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(result->Write(buffer.data(), buffer_size));
+    ASSERT_OK(result->ReadAt(position, buffer_size, &out_buffer));
+
+    ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
+
+    position += buffer_size;
+  }
+}
+
+TEST_F(TestMemoryMappedFile, ReadOnly) {
+  const int64_t buffer_size = 1024;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(1024, 0, buffer.data());
+
+  const int reps = 5;
+
+  std::string path = "ipc-read-only-test";
+  CreateFile(path, reps * buffer_size);
+
+  std::shared_ptr<MemoryMappedFile> rwmmap;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &rwmmap));
+
+  int64_t position = 0;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(rwmmap->Write(buffer.data(), buffer_size));
+    position += buffer_size;
+  }
+  rwmmap->Close();
+
+  std::shared_ptr<MemoryMappedFile> rommap;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
+
+  position = 0;
+  std::shared_ptr<Buffer> out_buffer;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(rommap->ReadAt(position, buffer_size, &out_buffer));
+
+    ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
+    position += buffer_size;
+  }
+  rommap->Close();
+}
+
+TEST_F(TestMemoryMappedFile, InvalidMode) {
+  const int64_t buffer_size = 1024;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(1024, 0, buffer.data());
+
+  std::string path = "ipc-invalid-mode-test";
+  CreateFile(path, buffer_size);
+
+  std::shared_ptr<MemoryMappedFile> rommap;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
+
+  ASSERT_RAISES(IOError, rommap->Write(buffer.data(), buffer_size));
+}
+
+TEST_F(TestMemoryMappedFile, InvalidFile) {
+  std::string non_existent_path = "invalid-file-name-asfd";
+
+  std::shared_ptr<MemoryMappedFile> result;
+  ASSERT_RAISES(
+      IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result));
+}
+
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index a49faf3bd8578..246310221e9aa 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -30,97 +30,6 @@
 namespace arrow {
 namespace io {
 
-class TestMemoryMappedFile : public ::testing::Test, public MemoryMapFixture {
- public:
-  void TearDown() { MemoryMapFixture::TearDown(); }
-};
-
-TEST_F(TestMemoryMappedFile, InvalidUsages) {}
-
-TEST_F(TestMemoryMappedFile, WriteRead) {
-  const int64_t buffer_size = 1024;
-  std::vector<uint8_t> buffer(buffer_size);
-
-  test::random_bytes(1024, 0, buffer.data());
-
-  const int reps = 5;
-
-  std::string path = "ipc-write-read-test";
-  CreateFile(path, reps * buffer_size);
-
-  std::shared_ptr<MemoryMappedFile> result;
-  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &result));
-
-  int64_t position = 0;
-  std::shared_ptr<Buffer> out_buffer;
-  for (int i = 0; i < reps; ++i) {
-    ASSERT_OK(result->Write(buffer.data(), buffer_size));
-    ASSERT_OK(result->ReadAt(position, buffer_size, &out_buffer));
-
-    ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
-
-    position += buffer_size;
-  }
-}
-
-TEST_F(TestMemoryMappedFile, ReadOnly) {
-  const int64_t buffer_size = 1024;
-  std::vector<uint8_t> buffer(buffer_size);
-
-  test::random_bytes(1024, 0, buffer.data());
-
-  const int reps = 5;
-
-  std::string path = "ipc-read-only-test";
-  CreateFile(path, reps * buffer_size);
-
-  std::shared_ptr<MemoryMappedFile> rwmmap;
-  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &rwmmap));
-
-  int64_t position = 0;
-  for (int i = 0; i < reps; ++i) {
-    ASSERT_OK(rwmmap->Write(buffer.data(), buffer_size));
-    position += buffer_size;
-  }
-  rwmmap->Close();
-
-  std::shared_ptr<MemoryMappedFile> rommap;
-  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
-
-  position = 0;
-  std::shared_ptr<Buffer> out_buffer;
-  for (int i = 0; i < reps; ++i) {
-    ASSERT_OK(rommap->ReadAt(position, buffer_size, &out_buffer));
-
-    ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
-    position += buffer_size;
-  }
-  rommap->Close();
-}
-
-TEST_F(TestMemoryMappedFile, InvalidMode) {
-  const int64_t buffer_size = 1024;
-  std::vector<uint8_t> buffer(buffer_size);
-
-  test::random_bytes(1024, 0, buffer.data());
-
-  std::string path = "ipc-invalid-mode-test";
-  CreateFile(path, buffer_size);
-
-  std::shared_ptr<MemoryMappedFile> rommap;
-  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
-
-  ASSERT_RAISES(IOError, rommap->Write(buffer.data(), buffer_size));
-}
-
-TEST_F(TestMemoryMappedFile, InvalidFile) {
-  std::string non_existent_path = "invalid-file-name-asfd";
-
-  std::shared_ptr<MemoryMappedFile> result;
-  ASSERT_RAISES(
-      IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result));
-}
-
 class TestBufferOutputStream : public ::testing::Test {
  public:
   void SetUp() {
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index b5cf4b77a980d..4595268372aa2 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -17,19 +17,6 @@
 
 #include "arrow/io/memory.h"
 
-// sys/mman.h not present in Visual Studio or Cygwin
-#ifdef _WIN32
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-#include "arrow/io/mman.h"
-#undef Realloc
-#undef Free
-#include <windows.h>
-#else
-#include <sys/mman.h>
-#endif
-
 #include <algorithm>
 #include <cerrno>
 #include <cstdint>
@@ -45,171 +32,6 @@
 namespace arrow {
 namespace io {
 
-// Implement MemoryMappedFile
-
-class MemoryMappedFile::MemoryMappedFileImpl {
- public:
-  MemoryMappedFileImpl()
-      : file_(nullptr), is_open_(false), is_writable_(false), data_(nullptr) {}
-
-  ~MemoryMappedFileImpl() {
-    if (is_open_) {
-      munmap(data_, size_);
-      fclose(file_);
-    }
-  }
-
-  Status Open(const std::string& path, FileMode::type mode) {
-    if (is_open_) { return Status::IOError("A file is already open"); }
-
-    int prot_flags = PROT_READ;
-
-    if (mode == FileMode::READWRITE) {
-      file_ = fopen(path.c_str(), "r+b");
-      prot_flags |= PROT_WRITE;
-      is_writable_ = true;
-    } else {
-      file_ = fopen(path.c_str(), "rb");
-    }
-    if (file_ == nullptr) {
-      std::stringstream ss;
-      ss << "Unable to open file, errno: " << errno;
-      return Status::IOError(ss.str());
-    }
-
-    fseek(file_, 0L, SEEK_END);
-    if (ferror(file_)) { return Status::IOError("Unable to seek to end of file"); }
-    size_ = ftell(file_);
-
-    fseek(file_, 0L, SEEK_SET);
-    is_open_ = true;
-    position_ = 0;
-
-    void* result = mmap(nullptr, size_, prot_flags, MAP_SHARED, fileno(file_), 0);
-    if (result == MAP_FAILED) {
-      std::stringstream ss;
-      ss << "Memory mapping file failed, errno: " << errno;
-      return Status::IOError(ss.str());
-    }
-    data_ = reinterpret_cast<uint8_t*>(result);
-
-    return Status::OK();
-  }
-
-  int64_t size() const { return size_; }
-
-  Status Seek(int64_t position) {
-    if (position < 0 || position >= size_) {
-      return Status::Invalid("position is out of bounds");
-    }
-    position_ = position;
-    return Status::OK();
-  }
-
-  int64_t position() { return position_; }
-
-  void advance(int64_t nbytes) { position_ = std::min(size_, position_ + nbytes); }
-
-  uint8_t* data() { return data_; }
-
-  uint8_t* head() { return data_ + position_; }
-
-  bool writable() { return is_writable_; }
-
-  bool opened() { return is_open_; }
-
- private:
-  FILE* file_;
-  int64_t position_;
-  int64_t size_;
-  bool is_open_;
-  bool is_writable_;
-
-  // The memory map
-  uint8_t* data_;
-};
-
-MemoryMappedFile::MemoryMappedFile(FileMode::type mode) {
-  ReadableFileInterface::set_mode(mode);
-}
-
-MemoryMappedFile::~MemoryMappedFile() {}
-
-Status MemoryMappedFile::Open(const std::string& path, FileMode::type mode,
-    std::shared_ptr<MemoryMappedFile>* out) {
-  std::shared_ptr<MemoryMappedFile> result(new MemoryMappedFile(mode));
-
-  result->impl_.reset(new MemoryMappedFileImpl());
-  RETURN_NOT_OK(result->impl_->Open(path, mode));
-
-  *out = result;
-  return Status::OK();
-}
-
-Status MemoryMappedFile::GetSize(int64_t* size) {
-  *size = impl_->size();
-  return Status::OK();
-}
-
-Status MemoryMappedFile::Tell(int64_t* position) {
-  *position = impl_->position();
-  return Status::OK();
-}
-
-Status MemoryMappedFile::Seek(int64_t position) {
-  return impl_->Seek(position);
-}
-
-Status MemoryMappedFile::Close() {
-  // munmap handled in pimpl dtor
-  return Status::OK();
-}
-
-Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-  nbytes = std::min(nbytes, impl_->size() - impl_->position());
-  std::memcpy(out, impl_->head(), nbytes);
-  *bytes_read = nbytes;
-  impl_->advance(nbytes);
-  return Status::OK();
-}
-
-Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  nbytes = std::min(nbytes, impl_->size() - impl_->position());
-  *out = std::make_shared<Buffer>(impl_->head(), nbytes);
-  impl_->advance(nbytes);
-  return Status::OK();
-}
-
-bool MemoryMappedFile::supports_zero_copy() const {
-  return true;
-}
-
-Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
-  if (!impl_->opened() || !impl_->writable()) {
-    return Status::IOError("Unable to write");
-  }
-
-  RETURN_NOT_OK(impl_->Seek(position));
-  return WriteInternal(data, nbytes);
-}
-
-Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
-  if (!impl_->opened() || !impl_->writable()) {
-    return Status::IOError("Unable to write");
-  }
-  if (nbytes + impl_->position() > impl_->size()) {
-    return Status::Invalid("Cannot write past end of memory map");
-  }
-
-  return WriteInternal(data, nbytes);
-}
-
-Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
-  memcpy(impl_->head(), data, nbytes);
-  impl_->advance(nbytes);
-  return Status::OK();
-}
-
 // ----------------------------------------------------------------------
 // OutputStream that writes to resizable buffer
 
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 2faf2804bcbd0..2f1d8ec317578 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -58,45 +58,6 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   uint8_t* mutable_data_;
 };
 
-// A memory source that uses memory-mapped files for memory interactions
-class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
- public:
-  ~MemoryMappedFile();
-
-  static Status Open(const std::string& path, FileMode::type mode,
-      std::shared_ptr<MemoryMappedFile>* out);
-
-  Status Close() override;
-
-  Status Tell(int64_t* position) override;
-
-  Status Seek(int64_t position) override;
-
-  // Required by ReadableFileInterface, copies memory into out
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
-
-  // Zero copy read
-  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
-
-  bool supports_zero_copy() const override;
-
-  Status Write(const uint8_t* data, int64_t nbytes) override;
-
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
-
-  // @return: the size in bytes of the memory source
-  Status GetSize(int64_t* size) override;
-
- private:
-  explicit MemoryMappedFile(FileMode::type mode);
-
-  Status WriteInternal(const uint8_t* data, int64_t nbytes);
-
-  // Hide the internal details of this class for now
-  class ARROW_NO_EXPORT MemoryMappedFileImpl;
-  std::unique_ptr<MemoryMappedFileImpl> impl_;
-};
-
 class ARROW_EXPORT BufferReader : public ReadableFileInterface {
  public:
   explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index 146808371d307..6e917135db274 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -33,6 +33,7 @@
 #endif
 
 #include "arrow/buffer.h"
+#include "arrow/io/file.h"
 #include "arrow/io/memory.h"
 #include "arrow/memory_pool.h"
 #include "arrow/test-util.h"

From 26140dca893296d84cea3b76c97c62fbc4052e3f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 3 Jan 2017 08:31:37 +0100
Subject: [PATCH 0257/1644] ARROW-387: [C++] Verify zero-copy Buffer slices
 from BufferReader retain reference to parent Buffer

This is stacked on top of the patch for ARROW-294, will rebase.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #266 from wesm/ARROW-387 and squashes the following commits:

061ef8b [Wes McKinney] Verify BufferReader passes on ownership of parent buffer to zero-copy slices
42a83a4 [Wes McKinney] Remove duplicated includes
3928ab0 [Wes McKinney] Base MemoryMappedFile implementation on common OSFile interface. Add test case for ARROW-340.
---
 cpp/src/arrow/io/interfaces.cc     |  5 +++++
 cpp/src/arrow/io/interfaces.h      |  5 ++++-
 cpp/src/arrow/io/io-memory-test.cc | 23 ++++++++++++++++++++++-
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 68c1ac30f8250..23bef2853b206 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -44,5 +44,10 @@ Status ReadableFileInterface::ReadAt(
   return Read(nbytes, out);
 }
 
+Status Writeable::Write(const std::string& data) {
+  return Write(reinterpret_cast<const uint8_t*>(data.c_str()),
+      static_cast<int64_t>(data.size()));
+}
+
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index db0c059c6e286..8fe2849287064 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <string>
 
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
@@ -67,9 +68,11 @@ class Seekable {
   virtual Status Seek(int64_t position) = 0;
 };
 
-class Writeable {
+class ARROW_EXPORT Writeable {
  public:
   virtual Status Write(const uint8_t* data, int64_t nbytes) = 0;
+
+  Status Write(const std::string& data);
 };
 
 class Readable {
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 246310221e9aa..95d788c03c97e 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -48,12 +48,33 @@ TEST_F(TestBufferOutputStream, CloseResizes) {
   const int64_t nbytes = static_cast<int64_t>(data.size());
   const int K = 100;
   for (int i = 0; i < K; ++i) {
-    EXPECT_OK(stream_->Write(reinterpret_cast<const uint8_t*>(data.c_str()), nbytes));
+    EXPECT_OK(stream_->Write(data));
   }
 
   ASSERT_OK(stream_->Close());
   ASSERT_EQ(K * nbytes, buffer_->size());
 }
 
+TEST(TestBufferReader, RetainParentReference) {
+  // ARROW-387
+  std::string data = "data123456";
+
+  std::shared_ptr<Buffer> slice1;
+  std::shared_ptr<Buffer> slice2;
+  {
+    auto buffer = std::make_shared<PoolBuffer>();
+    ASSERT_OK(buffer->Resize(static_cast<int64_t>(data.size())));
+    std::memcpy(buffer->mutable_data(), data.c_str(), data.size());
+    BufferReader reader(buffer);
+    ASSERT_OK(reader.Read(4, &slice1));
+    ASSERT_OK(reader.Read(6, &slice2));
+  }
+
+  ASSERT_TRUE(slice1->parent() != nullptr);
+
+  ASSERT_EQ(0, std::memcmp(slice1->data(), data.c_str(), 4));
+  ASSERT_EQ(0, std::memcmp(slice2->data(), data.c_str() + 4, 6));
+}
+
 }  // namespace io
 }  // namespace arrow

From fdbc57941fd3615c71b3a61b409b63eb6a48a817 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 3 Jan 2017 07:23:17 -0500
Subject: [PATCH 0258/1644] ARROW-417: Add Equals implementation to compare
 ChunkedArrays

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #259 from xhochy/ARROW-417 and squashes the following commits:

ffc076a [Uwe L. Korn] Add interface for non-shared_ptr-based Equals
3686d6c [Uwe L. Korn] ARROW-415: C++: Add Equals implementation to compare Tables
54cbf54 [Uwe L. Korn] ARROW-416: C++: Add Equals implementation to compare Columns
21e73a0 [Uwe L. Korn] Make signed comparison explicit
8563cb2 [Uwe L. Korn] ARROW-417: Add Equals implementation to compare ChunkedArrays
---
 cpp/src/arrow/column-test.cc | 121 +++++++++++++++++++++++++++++++++--
 cpp/src/arrow/column.cc      |  51 +++++++++++++++
 cpp/src/arrow/column.h       |   7 ++
 cpp/src/arrow/table-test.cc  |  44 +++++++++----
 cpp/src/arrow/table.cc       |  17 +++++
 cpp/src/arrow/table.h        |   3 +
 cpp/src/arrow/test-util.h    |   2 +-
 7 files changed, 228 insertions(+), 17 deletions(-)

diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
index 9005245b20419..1e722ed7de0d6 100644
--- a/cpp/src/arrow/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -33,12 +33,92 @@ using std::vector;
 
 namespace arrow {
 
-const auto INT32 = std::make_shared<Int32Type>();
+class TestChunkedArray : public TestBase {
+ protected:
+  virtual void Construct() {
+    one_ = std::make_shared<ChunkedArray>(arrays_one_);
+    another_ = std::make_shared<ChunkedArray>(arrays_another_);
+  }
+
+  ArrayVector arrays_one_;
+  ArrayVector arrays_another_;
+
+  std::shared_ptr<ChunkedArray> one_;
+  std::shared_ptr<ChunkedArray> another_;
+};
+
+TEST_F(TestChunkedArray, BasicEquals) {
+  std::vector<bool> null_bitmap(100, true);
+  std::vector<int32_t> data(100, 1);
+  std::shared_ptr<Array> array;
+  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap, data, &array);
+  arrays_one_.push_back(array);
+  arrays_another_.push_back(array);
+
+  Construct();
+  ASSERT_TRUE(one_->Equals(one_));
+  ASSERT_FALSE(one_->Equals(nullptr));
+  ASSERT_TRUE(one_->Equals(another_));
+  ASSERT_TRUE(one_->Equals(*another_.get()));
+}
+
+TEST_F(TestChunkedArray, EqualsDifferingTypes) {
+  std::vector<bool> null_bitmap(100, true);
+  std::vector<int32_t> data32(100, 1);
+  std::vector<int64_t> data64(100, 1);
+  std::shared_ptr<Array> array;
+  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap, data32, &array);
+  arrays_one_.push_back(array);
+  ArrayFromVector<Int64Type, int64_t>(int64(), null_bitmap, data64, &array);
+  arrays_another_.push_back(array);
+
+  Construct();
+  ASSERT_FALSE(one_->Equals(another_));
+  ASSERT_FALSE(one_->Equals(*another_.get()));
+}
+
+TEST_F(TestChunkedArray, EqualsDifferingLengths) {
+  std::vector<bool> null_bitmap100(100, true);
+  std::vector<bool> null_bitmap101(101, true);
+  std::vector<int32_t> data100(100, 1);
+  std::vector<int32_t> data101(101, 1);
+  std::shared_ptr<Array> array;
+  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap100, data100, &array);
+  arrays_one_.push_back(array);
+  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap101, data101, &array);
+  arrays_another_.push_back(array);
+
+  Construct();
+  ASSERT_FALSE(one_->Equals(another_));
+  ASSERT_FALSE(one_->Equals(*another_.get()));
+
+  std::vector<bool> null_bitmap1(1, true);
+  std::vector<int32_t> data1(1, 1);
+  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap1, data1, &array);
+  arrays_one_.push_back(array);
 
-class TestColumn : public TestBase {
+  Construct();
+  ASSERT_TRUE(one_->Equals(another_));
+  ASSERT_TRUE(one_->Equals(*another_.get()));
+}
+
+class TestColumn : public TestChunkedArray {
  protected:
+  void Construct() override {
+    TestChunkedArray::Construct();
+
+    one_col_ = std::make_shared<Column>(one_field_, one_);
+    another_col_ = std::make_shared<Column>(another_field_, another_);
+  }
+
   std::shared_ptr<ChunkedArray> data_;
   std::unique_ptr<Column> column_;
+
+  std::shared_ptr<Field> one_field_;
+  std::shared_ptr<Field> another_field_;
+
+  std::shared_ptr<Column> one_col_;
+  std::shared_ptr<Column> another_col_;
 };
 
 TEST_F(TestColumn, BasicAPI) {
@@ -47,11 +127,11 @@ TEST_F(TestColumn, BasicAPI) {
   arrays.push_back(MakePrimitive<Int32Array>(100, 10));
   arrays.push_back(MakePrimitive<Int32Array>(100, 20));
 
-  auto field = std::make_shared<Field>("c0", INT32);
+  auto field = std::make_shared<Field>("c0", int32());
   column_.reset(new Column(field, arrays));
 
   ASSERT_EQ("c0", column_->name());
-  ASSERT_TRUE(column_->type()->Equals(INT32));
+  ASSERT_TRUE(column_->type()->Equals(int32()));
   ASSERT_EQ(300, column_->length());
   ASSERT_EQ(30, column_->null_count());
   ASSERT_EQ(3, column_->data()->num_chunks());
@@ -62,7 +142,7 @@ TEST_F(TestColumn, ChunksInhomogeneous) {
   arrays.push_back(MakePrimitive<Int32Array>(100));
   arrays.push_back(MakePrimitive<Int32Array>(100, 10));
 
-  auto field = std::make_shared<Field>("c0", INT32);
+  auto field = std::make_shared<Field>("c0", int32());
   column_.reset(new Column(field, arrays));
 
   ASSERT_OK(column_->ValidateData());
@@ -72,4 +152,35 @@ TEST_F(TestColumn, ChunksInhomogeneous) {
   ASSERT_RAISES(Invalid, column_->ValidateData());
 }
 
+TEST_F(TestColumn, Equals) {
+  std::vector<bool> null_bitmap(100, true);
+  std::vector<int32_t> data(100, 1);
+  std::shared_ptr<Array> array;
+  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap, data, &array);
+  arrays_one_.push_back(array);
+  arrays_another_.push_back(array);
+
+  one_field_ = std::make_shared<Field>("column", int32());
+  another_field_ = std::make_shared<Field>("column", int32());
+
+  Construct();
+  ASSERT_TRUE(one_col_->Equals(one_col_));
+  ASSERT_FALSE(one_col_->Equals(nullptr));
+  ASSERT_TRUE(one_col_->Equals(another_col_));
+  ASSERT_TRUE(one_col_->Equals(*another_col_.get()));
+
+  // Field is different
+  another_field_ = std::make_shared<Field>("two", int32());
+  Construct();
+  ASSERT_FALSE(one_col_->Equals(another_col_));
+  ASSERT_FALSE(one_col_->Equals(*another_col_.get()));
+
+  // ChunkedArray is different
+  another_field_ = std::make_shared<Field>("column", int32());
+  arrays_another_.push_back(array);
+  Construct();
+  ASSERT_FALSE(one_col_->Equals(another_col_));
+  ASSERT_FALSE(one_col_->Equals(*another_col_.get()));
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index 1d136e7d95a55..3e899563e2cbe 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -35,6 +35,45 @@ ChunkedArray::ChunkedArray(const ArrayVector& chunks) : chunks_(chunks) {
   }
 }
 
+bool ChunkedArray::Equals(const ChunkedArray& other) const {
+  if (length_ != other.length()) { return false; }
+  if (null_count_ != other.null_count()) { return false; }
+
+  // Check contents of the underlying arrays. This checks for equality of
+  // the underlying data independently of the chunk size.
+  int this_chunk_idx = 0;
+  int32_t this_start_idx = 0;
+  int other_chunk_idx = 0;
+  int32_t other_start_idx = 0;
+  while (this_chunk_idx < static_cast<int32_t>(chunks_.size())) {
+    const std::shared_ptr<Array> this_array = chunks_[this_chunk_idx];
+    const std::shared_ptr<Array> other_array = other.chunk(other_chunk_idx);
+    int32_t common_length = std::min(
+        this_array->length() - this_start_idx, other_array->length() - other_start_idx);
+    if (!this_array->RangeEquals(this_start_idx, this_start_idx + common_length,
+            other_start_idx, other_array)) {
+      return false;
+    }
+
+    // If we have exhausted the current chunk, proceed to the next one individually.
+    if (this_start_idx + common_length == this_array->length()) {
+      this_chunk_idx++;
+      this_start_idx = 0;
+    }
+    if (other_start_idx + common_length == other_array->length()) {
+      other_chunk_idx++;
+      other_start_idx = 0;
+    }
+  }
+  return true;
+}
+
+bool ChunkedArray::Equals(const std::shared_ptr<ChunkedArray>& other) const {
+  if (this == other.get()) { return true; }
+  if (!other) { return false; }
+  return Equals(*other.get());
+}
+
 Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks)
     : field_(field) {
   data_ = std::make_shared<ChunkedArray>(chunks);
@@ -49,6 +88,18 @@ Column::Column(
     const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data)
     : field_(field), data_(data) {}
 
+bool Column::Equals(const Column& other) const {
+  if (!field_->Equals(other.field())) { return false; }
+  return data_->Equals(other.data());
+}
+
+bool Column::Equals(const std::shared_ptr<Column>& other) const {
+  if (this == other.get()) { return true; }
+  if (!other) { return false; }
+
+  return Equals(*other.get());
+}
+
 Status Column::ValidateData() {
   for (int i = 0; i < data_->num_chunks(); ++i) {
     std::shared_ptr<DataType> type = data_->chunk(i)->type();
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
index 1caafec9db95c..f71647381743c 100644
--- a/cpp/src/arrow/column.h
+++ b/cpp/src/arrow/column.h
@@ -48,6 +48,9 @@ class ARROW_EXPORT ChunkedArray {
 
   std::shared_ptr<Array> chunk(int i) const { return chunks_[i]; }
 
+  bool Equals(const ChunkedArray& other) const;
+  bool Equals(const std::shared_ptr<ChunkedArray>& other) const;
+
  protected:
   ArrayVector chunks_;
   int64_t length_;
@@ -78,6 +81,10 @@ class ARROW_EXPORT Column {
 
   // @returns: the column's data as a chunked logical array
   std::shared_ptr<ChunkedArray> data() const { return data_; }
+
+  bool Equals(const Column& other) const;
+  bool Equals(const std::shared_ptr<Column>& other) const;
+
   // Verify that the column's array data is consistent with the passed field's
   // metadata
   Status ValidateData();
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index f62336d07f09a..734b94125defc 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -34,16 +34,12 @@ using std::vector;
 
 namespace arrow {
 
-const auto INT16 = std::make_shared<Int16Type>();
-const auto UINT8 = std::make_shared<UInt8Type>();
-const auto INT32 = std::make_shared<Int32Type>();
-
 class TestTable : public TestBase {
  public:
   void MakeExample1(int length) {
-    auto f0 = std::make_shared<Field>("f0", INT32);
-    auto f1 = std::make_shared<Field>("f1", UINT8);
-    auto f2 = std::make_shared<Field>("f2", INT16);
+    auto f0 = std::make_shared<Field>("f0", int32());
+    auto f1 = std::make_shared<Field>("f1", uint8());
+    auto f2 = std::make_shared<Field>("f2", int16());
 
     vector<shared_ptr<Field>> fields = {f0, f1, f2};
     schema_ = std::make_shared<Schema>(fields);
@@ -55,7 +51,7 @@ class TestTable : public TestBase {
   }
 
  protected:
-  std::unique_ptr<Table> table_;
+  std::shared_ptr<Table> table_;
   shared_ptr<Schema> schema_;
   vector<std::shared_ptr<Column>> columns_;
 };
@@ -123,14 +119,40 @@ TEST_F(TestTable, InvalidColumns) {
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
 }
 
+TEST_F(TestTable, Equals) {
+  int length = 100;
+  MakeExample1(length);
+
+  std::string name = "data";
+  table_.reset(new Table(name, schema_, columns_));
+
+  ASSERT_TRUE(table_->Equals(table_));
+  ASSERT_FALSE(table_->Equals(nullptr));
+  // Differing name
+  ASSERT_FALSE(table_->Equals(std::make_shared<Table>("other_name", schema_, columns_)));
+  // Differing schema
+  auto f0 = std::make_shared<Field>("f3", int32());
+  auto f1 = std::make_shared<Field>("f4", uint8());
+  auto f2 = std::make_shared<Field>("f5", int16());
+  vector<shared_ptr<Field>> fields = {f0, f1, f2};
+  auto other_schema = std::make_shared<Schema>(fields);
+  ASSERT_FALSE(table_->Equals(std::make_shared<Table>(name, other_schema, columns_)));
+  // Differing columns
+  std::vector<std::shared_ptr<Column>> other_columns = {
+      std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length, 10)),
+      std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length, 10)),
+      std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length, 10))};
+  ASSERT_FALSE(table_->Equals(std::make_shared<Table>(name, schema_, other_columns)));
+}
+
 class TestRecordBatch : public TestBase {};
 
 TEST_F(TestRecordBatch, Equals) {
   const int length = 10;
 
-  auto f0 = std::make_shared<Field>("f0", INT32);
-  auto f1 = std::make_shared<Field>("f1", UINT8);
-  auto f2 = std::make_shared<Field>("f2", INT16);
+  auto f0 = std::make_shared<Field>("f0", int32());
+  auto f1 = std::make_shared<Field>("f1", uint8());
+  auto f2 = std::make_shared<Field>("f2", int16());
 
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto schema = std::make_shared<Schema>(fields);
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 855d4ec04085d..45f672ec8928b 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -77,6 +77,23 @@ Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
     const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
     : name_(name), schema_(schema), columns_(columns), num_rows_(num_rows) {}
 
+bool Table::Equals(const Table& other) const {
+  if (name_ != other.name()) { return false; }
+  if (!schema_->Equals(other.schema())) { return false; }
+  if (static_cast<int64_t>(columns_.size()) != other.num_columns()) { return false; }
+
+  for (size_t i = 0; i < columns_.size(); i++) {
+    if (!columns_[i]->Equals(other.column(i))) { return false; }
+  }
+  return true;
+}
+
+bool Table::Equals(const std::shared_ptr<Table>& other) const {
+  if (this == other.get()) { return true; }
+  if (!other) { return false; }
+  return Equals(*other.get());
+}
+
 Status Table::ValidateColumns() const {
   if (num_columns() != schema_->num_fields()) {
     return Status::Invalid("Number of columns did not match schema");
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index bf5c39f11e411..0f2418d0e7900 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -100,6 +100,9 @@ class ARROW_EXPORT Table {
   // @returns: the number of rows (the corresponding length of each column)
   int64_t num_rows() const { return num_rows_; }
 
+  bool Equals(const Table& other) const;
+  bool Equals(const std::shared_ptr<Table>& other) const;
+
   // After construction, perform any checks to validate the input arguments
   Status ValidateColumns() const;
 
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index ce9327d9009e2..70e933365cfdf 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -81,7 +81,7 @@ class TestBase : public ::testing::Test {
     auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
     EXPECT_OK(data->Resize(length * sizeof(typename ArrayType::value_type)));
     EXPECT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
-    return std::make_shared<ArrayType>(length, data, 10, null_bitmap);
+    return std::make_shared<ArrayType>(length, data, null_count, null_bitmap);
   }
 
  protected:

From 9513ca7741bc036ff369cbbd3b3ee3f4bcc06722 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Thu, 5 Jan 2017 11:00:32 -0500
Subject: [PATCH 0259/1644] ARROW-411: [Java] Move compactor functions in
 Integration to a separate Validator module

Author: Li Jin <ice.xelloss@gmail.com>

Closes #267 from icexelloss/validator and squashes the following commits:

b4e86c5 [Li Jin] ARROW-411: Move compator functions in Integration to a separate Validator moduleO
---
 .../org/apache/arrow/tools/Integration.java   |  94 +------------
 .../apache/arrow/tools/TestIntegration.java   |  32 -----
 .../apache/arrow/vector/util/Validator.java   | 125 ++++++++++++++++++
 .../arrow/vector/util/TestValidator.java      |  57 ++++++++
 4 files changed, 185 insertions(+), 123 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java

diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index fd835a63a11ac..36d4ee5485470 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -28,7 +28,6 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
@@ -39,10 +38,8 @@
 import org.apache.arrow.vector.file.json.JsonFileReader;
 import org.apache.arrow.vector.file.json.JsonFileWriter;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
-import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.Validator;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.Options;
@@ -51,8 +48,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Objects;
-
 public class Integration {
   private static final Logger LOGGER = LoggerFactory.getLogger(Integration.class);
 
@@ -143,7 +138,7 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
           LOGGER.debug("ARROW schema: " + arrowSchema);
           LOGGER.debug("JSON Input file size: " + jsonFile.length());
           LOGGER.debug("JSON schema: " + jsonSchema);
-          compareSchemas(jsonSchema, arrowSchema);
+          Validator.compareSchemas(jsonSchema, arrowSchema);
 
           List<ArrowBlock> recordBatches = footer.getRecordBatches();
           Iterator<ArrowBlock> iterator = recordBatches.iterator();
@@ -154,8 +149,7 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
                 VectorSchemaRoot arrowRoot = new VectorSchemaRoot(arrowSchema, allocator);) {
               VectorLoader vectorLoader = new VectorLoader(arrowRoot);
               vectorLoader.load(inRecordBatch);
-              // TODO: compare
-              compare(arrowRoot, jsonRoot);
+              Validator.compareVectorSchemaRoot(arrowRoot, jsonRoot);
             }
             jsonRoot.close();
           }
@@ -227,86 +221,4 @@ private static void fatalError(String message, Throwable e) {
     System.exit(1);
   }
 
-
-  private static void compare(VectorSchemaRoot arrowRoot, VectorSchemaRoot jsonRoot) {
-    compareSchemas(jsonRoot.getSchema(), arrowRoot.getSchema());
-    if (arrowRoot.getRowCount() != jsonRoot.getRowCount()) {
-      throw new IllegalArgumentException("Different row count:\n" + arrowRoot.getRowCount() + "\n" + jsonRoot.getRowCount());
-    }
-    List<FieldVector> arrowVectors = arrowRoot.getFieldVectors();
-    List<FieldVector> jsonVectors = jsonRoot.getFieldVectors();
-    if (arrowVectors.size() != jsonVectors.size()) {
-      throw new IllegalArgumentException("Different column count:\n" + arrowVectors.size() + "\n" + jsonVectors.size());
-    }
-    for (int i = 0; i < arrowVectors.size(); i++) {
-      Field field = arrowRoot.getSchema().getFields().get(i);
-      FieldVector arrowVector = arrowVectors.get(i);
-      FieldVector jsonVector = jsonVectors.get(i);
-      int valueCount = arrowVector.getAccessor().getValueCount();
-      if (valueCount != jsonVector.getAccessor().getValueCount()) {
-        throw new IllegalArgumentException("Different value count for field " + field + " : " + valueCount + " != " + jsonVector.getAccessor().getValueCount());
-      }
-      for (int j = 0; j < valueCount; j++) {
-        Object arrow = arrowVector.getAccessor().getObject(j);
-        Object json = jsonVector.getAccessor().getObject(j);
-        if (!equals(field.getType(), arrow, json)) {
-          throw new IllegalArgumentException(
-              "Different values in column:\n" + field + " at index " + j + ": " + arrow + " != " + json);
-        }
-      }
-    }
-  }
-
-  private static boolean equals(ArrowType type, final Object arrow, final Object json) {
-    if (type instanceof ArrowType.FloatingPoint) {
-      FloatingPoint fpType = (FloatingPoint) type;
-      switch (fpType.getPrecision()) {
-      case DOUBLE:
-        return equalEnough((Double)arrow, (Double)json);
-      case SINGLE:
-        return equalEnough((Float)arrow, (Float)json);
-      case HALF:
-      default:
-        throw new UnsupportedOperationException("unsupported precision: " + fpType);
-      }
-    }
-    return Objects.equal(arrow, json);
-  }
-
-  static boolean equalEnough(Float f1, Float f2) {
-    if (f1 == null || f2 == null) {
-      return f1 == null && f2 == null;
-    }
-    if (f1.isNaN()) {
-      return f2.isNaN();
-    }
-    if (f1.isInfinite()) {
-      return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
-    }
-    float average = Math.abs((f1 + f2) / 2);
-    float differenceScaled = Math.abs(f1 - f2) / (average == 0.0f ? 1f : average);
-    return differenceScaled < 1.0E-6f;
-  }
-
-  static boolean equalEnough(Double f1, Double f2) {
-    if (f1 == null || f2 == null) {
-      return f1 == null && f2 == null;
-    }
-    if (f1.isNaN()) {
-      return f2.isNaN();
-    }
-    if (f1.isInfinite()) {
-      return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
-    }
-    double average = Math.abs((f1 + f2) / 2);
-    double differenceScaled = Math.abs(f1 - f2) / (average == 0.0d ? 1d : average);
-    return differenceScaled < 1.0E-12d;
-  }
-
-
-  private static void compareSchemas(Schema jsonSchema, Schema arrowSchema) {
-    if (!arrowSchema.equals(jsonSchema)) {
-      throw new IllegalArgumentException("Different schemas:\n" + arrowSchema + "\n" + jsonSchema);
-    }
-  }
 }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
index ee6196b74e0dc..0ae32bebe0b30 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -22,9 +22,7 @@
 import static org.apache.arrow.tools.ArrowFileTestFixtures.write;
 import static org.apache.arrow.tools.ArrowFileTestFixtures.writeData;
 import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
-import static org.apache.arrow.tools.Integration.equalEnough;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -238,34 +236,4 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws FileN
       write(parent.getChild("root"), testInFile);
     }
   }
-
-  @Test
-  public void testFloatComp() {
-    assertTrue(equalEnough(912.4140000000002F, 912.414F));
-    assertTrue(equalEnough(912.4140000000002D, 912.414D));
-    assertTrue(equalEnough(912.414F, 912.4140000000002F));
-    assertTrue(equalEnough(912.414D, 912.4140000000002D));
-    assertFalse(equalEnough(912.414D, 912.4140001D));
-    assertFalse(equalEnough(null, 912.414D));
-    assertTrue(equalEnough((Float)null, null));
-    assertTrue(equalEnough((Double)null, null));
-    assertFalse(equalEnough(912.414D, null));
-    assertFalse(equalEnough(Double.MAX_VALUE, Double.MIN_VALUE));
-    assertFalse(equalEnough(Double.MIN_VALUE, Double.MAX_VALUE));
-    assertTrue(equalEnough(Double.MAX_VALUE, Double.MAX_VALUE));
-    assertTrue(equalEnough(Double.MIN_VALUE, Double.MIN_VALUE));
-    assertTrue(equalEnough(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY));
-    assertFalse(equalEnough(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY));
-    assertTrue(equalEnough(Double.NaN, Double.NaN));
-    assertFalse(equalEnough(1.0, Double.NaN));
-    assertFalse(equalEnough(Float.MAX_VALUE, Float.MIN_VALUE));
-    assertFalse(equalEnough(Float.MIN_VALUE, Float.MAX_VALUE));
-    assertTrue(equalEnough(Float.MAX_VALUE, Float.MAX_VALUE));
-    assertTrue(equalEnough(Float.MIN_VALUE, Float.MIN_VALUE));
-    assertTrue(equalEnough(Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY));
-    assertFalse(equalEnough(Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY));
-    assertTrue(equalEnough(Float.NaN, Float.NaN));
-    assertFalse(equalEnough(1.0F, Float.NaN));
-  }
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
new file mode 100644
index 0000000000000..a97458254151d
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import java.util.List;
+
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.base.Objects;
+
+/**
+ * Utility class for validating arrow data structures
+ */
+public class Validator {
+
+    /**
+     * Validate two arrow schemas are equal.
+     *
+     * @throws IllegalArgumentException if they are different.
+     */
+    public static void compareSchemas(Schema schema1, Schema schema2) {
+        if (!schema2.equals(schema1)) {
+            throw new IllegalArgumentException("Different schemas:\n" + schema2 + "\n" + schema1);
+        }
+    }
+
+    /**
+     * Validate two arrow vectorSchemaRoot are equal.
+     *
+     * @throws IllegalArgumentException if they are different.
+     */
+    public static void compareVectorSchemaRoot(VectorSchemaRoot root1, VectorSchemaRoot root2) {
+        compareSchemas(root2.getSchema(), root1.getSchema());
+        if (root1.getRowCount() != root2.getRowCount()) {
+            throw new IllegalArgumentException("Different row count:\n" + root1.getRowCount() + "\n" + root2.getRowCount());
+        }
+        List<FieldVector> arrowVectors = root1.getFieldVectors();
+        List<FieldVector> jsonVectors = root2.getFieldVectors();
+        if (arrowVectors.size() != jsonVectors.size()) {
+            throw new IllegalArgumentException("Different column count:\n" + arrowVectors.size() + "\n" + jsonVectors.size());
+        }
+        for (int i = 0; i < arrowVectors.size(); i++) {
+            Field field = root1.getSchema().getFields().get(i);
+            FieldVector arrowVector = arrowVectors.get(i);
+            FieldVector jsonVector = jsonVectors.get(i);
+            int valueCount = arrowVector.getAccessor().getValueCount();
+            if (valueCount != jsonVector.getAccessor().getValueCount()) {
+                throw new IllegalArgumentException("Different value count for field " + field + " : " + valueCount + " != " + jsonVector.getAccessor().getValueCount());
+            }
+            for (int j = 0; j < valueCount; j++) {
+                Object arrow = arrowVector.getAccessor().getObject(j);
+                Object json = jsonVector.getAccessor().getObject(j);
+                if (!equals(field.getType(), arrow, json)) {
+                    throw new IllegalArgumentException(
+                            "Different values in column:\n" + field + " at index " + j + ": " + arrow + " != " + json);
+                }
+            }
+        }
+    }
+
+    static boolean equals(ArrowType type, final Object o1, final Object o2) {
+        if (type instanceof ArrowType.FloatingPoint) {
+            ArrowType.FloatingPoint fpType = (ArrowType.FloatingPoint) type;
+            switch (fpType.getPrecision()) {
+                case DOUBLE:
+                    return equalEnough((Double)o1, (Double)o2);
+                case SINGLE:
+                    return equalEnough((Float)o1, (Float)o2);
+                case HALF:
+                default:
+                    throw new UnsupportedOperationException("unsupported precision: " + fpType);
+            }
+        }
+        return Objects.equal(o1, o2);
+    }
+
+    static boolean equalEnough(Float f1, Float f2) {
+        if (f1 == null || f2 == null) {
+            return f1 == null && f2 == null;
+        }
+        if (f1.isNaN()) {
+            return f2.isNaN();
+        }
+        if (f1.isInfinite()) {
+            return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
+        }
+        float average = Math.abs((f1 + f2) / 2);
+        float differenceScaled = Math.abs(f1 - f2) / (average == 0.0f ? 1f : average);
+        return differenceScaled < 1.0E-6f;
+    }
+
+    static boolean equalEnough(Double f1, Double f2) {
+        if (f1 == null || f2 == null) {
+            return f1 == null && f2 == null;
+        }
+        if (f1.isNaN()) {
+            return f2.isNaN();
+        }
+        if (f1.isInfinite()) {
+            return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
+        }
+        double average = Math.abs((f1 + f2) / 2);
+        double differenceScaled = Math.abs(f1 - f2) / (average == 0.0d ? 1d : average);
+        return differenceScaled < 1.0E-12d;
+    }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java
new file mode 100644
index 0000000000000..7cf638e57d849
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.arrow.vector.util;
+
+import static org.apache.arrow.vector.util.Validator.equalEnough;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class TestValidator {
+
+    @Test
+    public void testFloatComp() {
+        assertTrue(equalEnough(912.4140000000002F, 912.414F));
+        assertTrue(equalEnough(912.4140000000002D, 912.414D));
+        assertTrue(equalEnough(912.414F, 912.4140000000002F));
+        assertTrue(equalEnough(912.414D, 912.4140000000002D));
+        assertFalse(equalEnough(912.414D, 912.4140001D));
+        assertFalse(equalEnough(null, 912.414D));
+        assertTrue(equalEnough((Float)null, null));
+        assertTrue(equalEnough((Double)null, null));
+        assertFalse(equalEnough(912.414D, null));
+        assertFalse(equalEnough(Double.MAX_VALUE, Double.MIN_VALUE));
+        assertFalse(equalEnough(Double.MIN_VALUE, Double.MAX_VALUE));
+        assertTrue(equalEnough(Double.MAX_VALUE, Double.MAX_VALUE));
+        assertTrue(equalEnough(Double.MIN_VALUE, Double.MIN_VALUE));
+        assertTrue(equalEnough(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY));
+        assertFalse(equalEnough(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY));
+        assertTrue(equalEnough(Double.NaN, Double.NaN));
+        assertFalse(equalEnough(1.0, Double.NaN));
+        assertFalse(equalEnough(Float.MAX_VALUE, Float.MIN_VALUE));
+        assertFalse(equalEnough(Float.MIN_VALUE, Float.MAX_VALUE));
+        assertTrue(equalEnough(Float.MAX_VALUE, Float.MAX_VALUE));
+        assertTrue(equalEnough(Float.MIN_VALUE, Float.MIN_VALUE));
+        assertTrue(equalEnough(Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY));
+        assertFalse(equalEnough(Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY));
+        assertTrue(equalEnough(Float.NaN, Float.NaN));
+        assertFalse(equalEnough(1.0F, Float.NaN));
+    }
+}

From 320f5875eef4010762a2146a0691148af1a3f182 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Jan 2017 15:18:24 -0500
Subject: [PATCH 0260/1644] ARROW-455: [C++] Add dtor to BufferOutputStream
 that calls Close()

Since `Close()` can technically fail, it's better to call it yourself (and it's idempotent), but this will help avoid a common class of bugs in small-scale use cases.

An alternative here is that we could remove all `Close()` calls from all destructors and possibly add a `DCHECK(!is_open_)` to the base dtor to force the user to close handles. The downside of this is that it makes RAII more difficult, so I'd prefer to leave the close-in-dtor even though it can fail in unusual scenarios.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #269 from wesm/ARROW-455 and squashes the following commits:

821ee22 [Wes McKinney] Add dtor to BufferOutputStream that calls Close()
---
 cpp/src/arrow/io/file.cc           |  1 +
 cpp/src/arrow/io/io-memory-test.cc | 15 +++++++++++++--
 cpp/src/arrow/io/memory.cc         |  5 +++++
 cpp/src/arrow/io/memory.h          |  2 ++
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 3182f2dd8a3b5..0fb13ea22e39f 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -476,6 +476,7 @@ FileOutputStream::FileOutputStream() {
 }
 
 FileOutputStream::~FileOutputStream() {
+  // This can fail; better to explicitly call close
   impl_->Close();
 }
 
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 95d788c03c97e..c0b01653cb128 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -42,17 +42,28 @@ class TestBufferOutputStream : public ::testing::Test {
   std::unique_ptr<OutputStream> stream_;
 };
 
+TEST_F(TestBufferOutputStream, DtorCloses) {
+  std::string data = "data123456";
+
+  const int K = 100;
+  for (int i = 0; i < K; ++i) {
+    EXPECT_OK(stream_->Write(data));
+  }
+
+  stream_ = nullptr;
+  ASSERT_EQ(static_cast<int64_t>(K * data.size()), buffer_->size());
+}
+
 TEST_F(TestBufferOutputStream, CloseResizes) {
   std::string data = "data123456";
 
-  const int64_t nbytes = static_cast<int64_t>(data.size());
   const int K = 100;
   for (int i = 0; i < K; ++i) {
     EXPECT_OK(stream_->Write(data));
   }
 
   ASSERT_OK(stream_->Close());
-  ASSERT_EQ(K * nbytes, buffer_->size());
+  ASSERT_EQ(static_cast<int64_t>(K * data.size()), buffer_->size());
 }
 
 TEST(TestBufferReader, RetainParentReference) {
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 4595268372aa2..0f5a0dc06979c 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -43,6 +43,11 @@ BufferOutputStream::BufferOutputStream(const std::shared_ptr<ResizableBuffer>& b
       position_(0),
       mutable_data_(buffer->mutable_data()) {}
 
+BufferOutputStream::~BufferOutputStream() {
+  // This can fail, better to explicitly call close
+  Close();
+}
+
 Status BufferOutputStream::Close() {
   if (position_ < capacity_) {
     return buffer_->Resize(position_);
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 2f1d8ec317578..8428a12220a69 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -43,6 +43,8 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
  public:
   explicit BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer);
 
+  ~BufferOutputStream();
+
   // Implement the OutputStream interface
   Status Close() override;
   Status Tell(int64_t* position) override;

From 5bf6ae49ec561eaaef823f0eb16ccca2d2ba7cf3 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 6 Jan 2017 15:57:20 +0100
Subject: [PATCH 0261/1644] ARROW-456: Add jemalloc based MemoryPool

Runtimes of the `builder-benchmark`:

```
BM_BuildPrimitiveArrayNoNulls/repeats:3               901 ms        889 ms          1   576.196MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3               833 ms        829 ms          1     617.6MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3               825 ms        821 ms          1   623.855MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_mean          853 ms        846 ms          1   605.884MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_stddev         34 ms         30 ms          0    21.147MB/s
BM_BuildVectorNoNulls/repeats:3                       712 ms        701 ms          1   729.866MB/s
BM_BuildVectorNoNulls/repeats:3                       671 ms        670 ms          1   764.464MB/s
BM_BuildVectorNoNulls/repeats:3                       688 ms        681 ms          1   751.285MB/s
BM_BuildVectorNoNulls/repeats:3_mean                  690 ms        684 ms          1   748.538MB/s
BM_BuildVectorNoNulls/repeats:3_stddev                 17 ms         13 ms          0   14.2578MB/s
```

With an aligned `Reallocate`, the jemalloc version is 50% faster and even outperforms `std::vector`:

```
BM_BuildPrimitiveArrayNoNulls/repeats:3               565 ms        559 ms          1   916.516MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3               540 ms        537 ms          1   952.727MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3               544 ms        543 ms          1   942.948MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_mean          550 ms        546 ms          1   937.397MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_stddev         11 ms          9 ms          0   15.2949MB/s
```

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #270 from xhochy/ARROW-456 and squashes the following commits:

d3ce3bf [Uwe L. Korn] Zero arrays for now
831399d [Uwe L. Korn] cpplint #2
e6e251b [Uwe L. Korn] cpplint
52b3c76 [Uwe L. Korn] Add Reallocate implementation to PyArrowMemoryPool
113e650 [Uwe L. Korn] Add missing file
d331cd9 [Uwe L. Korn] Add tests for Reallocate
c2be086 [Uwe L. Korn] Add JEMALLOC_HOME to the Readme
bd47f51 [Uwe L. Korn] Add missing return value
5142ac3 [Uwe L. Korn] Don't use deprecated GBenchmark interfaces
b6bff98 [Uwe L. Korn] Add missing (win) include
6f08e19 [Uwe L. Korn] Don't build jemalloc on AppVeyor
834c3b2 [Uwe L. Korn] Add jemalloc to Travis builds
10c6839 [Uwe L. Korn] Implement Reallocate function
a17b313 [Uwe L. Korn] ARROW-456: C++: Add jemalloc based MemoryPool
---
 .travis.yml                                   |  1 +
 appveyor.yml                                  |  2 +-
 ci/travis_before_script_cpp.sh                |  5 ++
 cpp/CMakeLists.txt                            | 30 ++++++-
 cpp/README.md                                 |  1 +
 cpp/cmake_modules/Findjemalloc.cmake          | 86 +++++++++++++++++++
 cpp/src/arrow/CMakeLists.txt                  |  1 +
 cpp/src/arrow/buffer.cc                       |  6 +-
 cpp/src/arrow/builder-benchmark.cc            | 64 ++++++++++++++
 cpp/src/arrow/builder.cc                      |  1 +
 cpp/src/arrow/column-benchmark.cc             |  2 +-
 cpp/src/arrow/io/interfaces.cc                |  4 +-
 cpp/src/arrow/io/io-file-test.cc              | 13 +++
 cpp/src/arrow/jemalloc/CMakeLists.txt         | 80 +++++++++++++++++
 cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in   | 27 ++++++
 .../jemalloc/jemalloc-builder-benchmark.cc    | 47 ++++++++++
 .../jemalloc/jemalloc-memory_pool-test.cc     | 51 +++++++++++
 cpp/src/arrow/jemalloc/memory_pool.cc         | 74 ++++++++++++++++
 cpp/src/arrow/jemalloc/memory_pool.h          | 57 ++++++++++++
 cpp/src/arrow/jemalloc/symbols.map            | 30 +++++++
 cpp/src/arrow/memory_pool-test.cc             | 33 +++----
 cpp/src/arrow/memory_pool-test.h              | 79 +++++++++++++++++
 cpp/src/arrow/memory_pool.cc                  | 24 ++++++
 cpp/src/arrow/memory_pool.h                   |  1 +
 python/src/pyarrow/common.cc                  | 14 +++
 25 files changed, 704 insertions(+), 29 deletions(-)
 create mode 100644 cpp/cmake_modules/Findjemalloc.cmake
 create mode 100644 cpp/src/arrow/builder-benchmark.cc
 create mode 100644 cpp/src/arrow/jemalloc/CMakeLists.txt
 create mode 100644 cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
 create mode 100644 cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
 create mode 100644 cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc
 create mode 100644 cpp/src/arrow/jemalloc/memory_pool.cc
 create mode 100644 cpp/src/arrow/jemalloc/memory_pool.h
 create mode 100644 cpp/src/arrow/jemalloc/symbols.map
 create mode 100644 cpp/src/arrow/memory_pool-test.h

diff --git a/.travis.yml b/.travis.yml
index 1634eba443615..e8d91045c2254 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,7 @@ addons:
     - libboost-dev
     - libboost-filesystem-dev
     - libboost-system-dev
+    - libjemalloc-dev
 
 matrix:
   fast_finish: true
diff --git a/appveyor.yml b/appveyor.yml
index 67478487081b7..17362c993d053 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -32,7 +32,7 @@ build_script:
  - cd build
  # A lot of features are still deactivated as they do not build on Windows
  #  * gbenchmark doesn't build with MSVC
- - cmake -G "%GENERATOR%" -DARROW_BOOST_USE_SHARED=OFF -DARROW_IPC=OFF -DARROW_HDFS=OFF -DARROW_BUILD_BENCHMARKS=OFF ..
+ - cmake -G "%GENERATOR%" -DARROW_BOOST_USE_SHARED=OFF -DARROW_IPC=OFF -DARROW_HDFS=OFF -DARROW_BUILD_BENCHMARKS=OFF -DARROW_JEMALLOC=OFF ..
  - cmake --build . --config Debug
 
 # test_script:
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 73bdaeb81fe78..94a889cff1a78 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -17,6 +17,11 @@ set -ex
 
 : ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
 
+if [ $TRAVIS_OS_NAME == "osx" ]; then
+  brew update > /dev/null
+  brew install jemalloc
+fi
+
 mkdir $CPP_BUILD_DIR
 pushd $CPP_BUILD_DIR
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 13f0354a73b8b..419691b4b68b2 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -28,7 +28,7 @@ set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 
 set(GFLAGS_VERSION "2.1.2")
 set(GTEST_VERSION "1.7.0")
-set(GBENCHMARK_VERSION "1.0.0")
+set(GBENCHMARK_VERSION "1.1.0")
 set(FLATBUFFERS_VERSION "1.3.0")
 
 find_package(ClangTools)
@@ -74,6 +74,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow IPC extensions"
     ON)
 
+  option(ARROW_JEMALLOC
+    "Build the Arrow jemalloc-based allocator"
+    ON)
+
   option(ARROW_BOOST_USE_SHARED
     "Rely on boost shared libraries where relevant"
     ON)
@@ -238,6 +242,16 @@ function(ADD_ARROW_BENCHMARK_DEPENDENCIES REL_BENCHMARK_NAME)
   add_dependencies(${BENCHMARK_NAME} ${ARGN})
 endfunction()
 
+# A wrapper for target_link_libraries() that is compatible with NO_BENCHMARKS.
+function(ARROW_BENCHMARK_LINK_LIBRARIES REL_BENCHMARK_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
+
+  target_link_libraries(${BENCHMARK_NAME} ${ARGN})
+endfunction()
+
 
 ############################################################
 # Testing
@@ -526,7 +540,11 @@ if(ARROW_BUILD_BENCHMARKS)
     set(GBENCHMARK_CMAKE_ARGS
           "-DCMAKE_BUILD_TYPE=Release"
           "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}"
+          "-DBENCHMARK_ENABLE_TESTING=OFF"
           "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}")
+    if (APPLE)
+      set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON")
+    endif()
     if (CMAKE_VERSION VERSION_GREATER "3.2")
       # BUILD_BYPRODUCTS is a 3.2+ feature
       ExternalProject_Add(gbenchmark_ep
@@ -575,6 +593,12 @@ endif()
 message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
 include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
 
+if (ARROW_JEMALLOC)
+  find_package(jemalloc REQUIRED)
+  ADD_THIRDPARTY_LIB(jemalloc
+      SHARED_LIB ${JEMALLOC_SHARED_LIB})
+endif()
+
 ## Google PerfTools
 ##
 ## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment
@@ -737,6 +761,10 @@ add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
 add_subdirectory(src/arrow/util)
 
+if(ARROW_JEMALLOC)
+  add_subdirectory(src/arrow/jemalloc)
+endif()
+
 #----------------------------------------------------------------------
 # IPC library
 
diff --git a/cpp/README.md b/cpp/README.md
index 190e6f85b429d..b77ea990d0659 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -60,6 +60,7 @@ variables
 * Google Benchmark: `GBENCHMARK_HOME` (only required if building benchmarks)
 * Flatbuffers: `FLATBUFFERS_HOME` (only required for the IPC extensions)
 * Hadoop: `HADOOP_HOME` (only required for the HDFS I/O extensions)
+* jemalloc: `JEMALLOC_HOME` (only required for the jemalloc-based memory pool)
 
 ## Continuous Integration
 
diff --git a/cpp/cmake_modules/Findjemalloc.cmake b/cpp/cmake_modules/Findjemalloc.cmake
new file mode 100644
index 0000000000000..e7fbb94a69235
--- /dev/null
+++ b/cpp/cmake_modules/Findjemalloc.cmake
@@ -0,0 +1,86 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find jemalloc headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(jemalloc)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  JEMALLOC_HOME -
+#   When set, this path is inspected instead of standard library locations as
+#   the root of the jemalloc installation.  The environment variable
+#   JEMALLOC_HOME overrides this veriable.
+#
+# This module defines
+#  JEMALLOC_INCLUDE_DIR, directory containing headers
+#  JEMALLOC_SHARED_LIB, path to libjemalloc.so/dylib
+#  JEMALLOC_FOUND, whether flatbuffers has been found
+
+if( NOT "$ENV{JEMALLOC_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "$ENV{JEMALLOC_HOME}" _native_path )
+    list( APPEND _jemalloc_roots ${_native_path} )
+elseif ( JEMALLOC_HOME )
+    list( APPEND _jemalloc_roots ${JEMALLOC_HOME} )
+endif()
+
+set(LIBJEMALLOC_NAMES jemalloc libjemalloc.so.1 libjemalloc.so.2 libjemalloc.dylib)
+
+# Try the parameterized roots, if they exist
+if ( _jemalloc_roots )
+    find_path( JEMALLOC_INCLUDE_DIR NAMES jemalloc/jemalloc.h
+        PATHS ${_jemalloc_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "include" )
+    find_library( JEMALLOC_SHARED_LIB NAMES ${LIBJEMALLOC_NAMES}
+        PATHS ${_jemalloc_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib" )
+else ()
+    find_path( JEMALLOC_INCLUDE_DIR NAMES jemalloc/jemalloc.h )
+    message(STATUS ${JEMALLOC_INCLUDE_DIR})
+    find_library( JEMALLOC_SHARED_LIB NAMES ${LIBJEMALLOC_NAMES})
+    message(STATUS ${JEMALLOC_SHARED_LIB})
+endif ()
+
+if (JEMALLOC_INCLUDE_DIR AND JEMALLOC_SHARED_LIB)
+  set(JEMALLOC_FOUND TRUE)
+else ()
+  set(JEMALLOC_FOUND FALSE)
+endif ()
+
+if (JEMALLOC_FOUND)
+    if (NOT jemalloc_FIND_QUIETLY)
+      message(STATUS "Found the jemalloc library: ${JEMALLOC_LIBRARIES}")
+  endif ()
+else ()
+  if (NOT jemalloc_FIND_QUIETLY)
+    set(JEMALLOC_ERR_MSG "Could not find the jemalloc library. Looked in ")
+    if ( _flatbuffers_roots )
+      set(JEMALLOC_ERR_MSG "${JEMALLOC_ERR_MSG} in ${_jemalloc_roots}.")
+    else ()
+      set(JEMALLOC_ERR_MSG "${JEMALLOC_ERR_MSG} system search paths.")
+    endif ()
+    if (jemalloc_FIND_REQUIRED)
+      message(FATAL_ERROR "${JEMALLOC_ERR_MSG}")
+    else (jemalloc_FIND_REQUIRED)
+      message(STATUS "${JEMALLOC_ERR_MSG}")
+    endif (jemalloc_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  JEMALLOC_INCLUDE_DIR
+  JEMALLOC_SHARED_LIB
+)
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index f8c50513d31a5..16668db798b78 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -59,4 +59,5 @@ ADD_ARROW_TEST(schema-test)
 ADD_ARROW_TEST(status-test)
 ADD_ARROW_TEST(table-test)
 
+ADD_ARROW_BENCHMARK(builder-benchmark)
 ADD_ARROW_BENCHMARK(column-benchmark)
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 6ffa03a0b5663..6d55f88af1e32 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -80,13 +80,11 @@ Status PoolBuffer::Reserve(int64_t new_capacity) {
     uint8_t* new_data;
     new_capacity = BitUtil::RoundUpToMultipleOf64(new_capacity);
     if (mutable_data_) {
-      RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
-      memcpy(new_data, mutable_data_, size_);
-      pool_->Free(mutable_data_, capacity_);
+      RETURN_NOT_OK(pool_->Reallocate(capacity_, new_capacity, &mutable_data_));
     } else {
       RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
+      mutable_data_ = new_data;
     }
-    mutable_data_ = new_data;
     data_ = mutable_data_;
     capacity_ = new_capacity;
   }
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
new file mode 100644
index 0000000000000..67799a3485f23
--- /dev/null
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include "arrow/builder.h"
+#include "arrow/memory_pool.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+constexpr int64_t kFinalSize = 256;
+
+static void BM_BuildPrimitiveArrayNoNulls(
+    benchmark::State& state) {  // NOLINT non-const reference
+  // 2 MiB block
+  std::vector<int64_t> data(256 * 1024, 100);
+  while (state.KeepRunning()) {
+    Int64Builder builder(default_memory_pool(), arrow::int64());
+    for (int i = 0; i < kFinalSize; i++) {
+      // Build up an array of 512 MiB in size
+      builder.Append(data.data(), data.size(), nullptr);
+    }
+    std::shared_ptr<Array> out;
+    builder.Finish(&out);
+  }
+  state.SetBytesProcessed(
+      state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
+}
+
+BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
+
+static void BM_BuildVectorNoNulls(
+    benchmark::State& state) {  // NOLINT non-const reference
+  // 2 MiB block
+  std::vector<int64_t> data(256 * 1024, 100);
+  while (state.KeepRunning()) {
+    std::vector<int64_t> builder;
+    for (int i = 0; i < kFinalSize; i++) {
+      // Build up an array of 512 MiB in size
+      builder.insert(builder.end(), data.cbegin(), data.cend());
+    }
+  }
+  state.SetBytesProcessed(
+      state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
+}
+
+BENCHMARK(BM_BuildVectorNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 1d94dbaa0e91d..a308ea53c570c 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -156,6 +156,7 @@ Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
     const int64_t new_bytes = TypeTraits<T>::bytes_required(capacity);
     RETURN_NOT_OK(data_->Resize(new_bytes));
     raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
+    // TODO(emkornfield) valgrind complains without this
     memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
   }
   return Status::OK();
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index 650ec90fc0728..8a1c775d7376d 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -37,7 +37,7 @@ std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
 static void BM_BuildInt32ColumnByChunk(
     benchmark::State& state) {  // NOLINT non-const reference
   ArrayVector arrays;
-  for (int chunk_n = 0; chunk_n < state.range_x(); ++chunk_n) {
+  for (int chunk_n = 0; chunk_n < state.range(0); ++chunk_n) {
     arrays.push_back(MakePrimitive<Int32Array>(100, 10));
   }
   const auto INT32 = std::make_shared<Int32Type>();
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 23bef2853b206..8040f93836cdc 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -45,8 +45,8 @@ Status ReadableFileInterface::ReadAt(
 }
 
 Status Writeable::Write(const std::string& data) {
-  return Write(reinterpret_cast<const uint8_t*>(data.c_str()),
-      static_cast<int64_t>(data.size()));
+  return Write(
+      reinterpret_cast<const uint8_t*>(data.c_str()), static_cast<int64_t>(data.size()));
 }
 
 }  // namespace io
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 5f5d639fab0d8..378b60e782124 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -292,6 +292,19 @@ class MyMemoryPool : public MemoryPool {
 
   void Free(uint8_t* buffer, int64_t size) override { std::free(buffer); }
 
+  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override {
+    *ptr = reinterpret_cast<uint8_t*>(std::realloc(*ptr, new_size));
+
+    if (*ptr == NULL) {
+      std::stringstream ss;
+      ss << "realloc of size " << new_size << " failed";
+      return Status::OutOfMemory(ss.str());
+    }
+
+
+    return Status::OK();
+  }
+
   int64_t bytes_allocated() const override { return -1; }
 
   int64_t num_allocations() const { return num_allocations_; }
diff --git a/cpp/src/arrow/jemalloc/CMakeLists.txt b/cpp/src/arrow/jemalloc/CMakeLists.txt
new file mode 100644
index 0000000000000..c6663eb8227f0
--- /dev/null
+++ b/cpp/src/arrow/jemalloc/CMakeLists.txt
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ----------------------------------------------------------------------
+# arrow_jemalloc : Arrow jemalloc-based allocator
+
+include_directories(SYSTEM "{JEMALLOC_INCLUDE_DIR}")
+
+# arrow_jemalloc library
+set(ARROW_JEMALLOC_STATIC_LINK_LIBS
+  arrow_static
+  jemalloc
+)
+set(ARROW_JEMALLOC_SHARED_LINK_LIBS
+  arrow_shared
+  jemalloc
+)
+
+if (ARROW_BUILD_STATIC)
+  set(ARROW_JEMALLOC_TEST_LINK_LIBS
+    arrow_jemalloc_static)
+else()
+  set(ARROW_jemalloc_TEST_LINK_LIBS
+    arrow_jemalloc_shared)
+endif()
+
+set(ARROW_JEMALLOC_SRCS
+  memory_pool.cc
+)
+
+if(NOT APPLE)
+  # Localize thirdparty symbols using a linker version script. This hides them
+  # from the client application. The OS X linker does not support the
+  # version-script option.
+  set(ARROW_JEMALLOC_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
+endif()
+
+ADD_ARROW_LIB(arrow_jemalloc
+  SOURCES ${ARROW_JEMALLOC_SRCS}
+  SHARED_LINK_FLAGS ${ARROW_JEMALLOC_LINK_FLAGS}
+  SHARED_LINK_LIBS ${ARROW_JEMALLOC_SHARED_LINK_LIBS}
+  SHARED_PRIVATE_LINK_LIBS ${ARROW_JEMALLOC_SHARED_PRIVATE_LINK_LIBS}
+  STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS}
+  STATIC_PRIVATE_LINK_LIBS ${ARROW_JEMALLOC_STATIC_PRIVATE_LINK_LIBS}
+)
+
+ADD_ARROW_TEST(jemalloc-memory_pool-test)
+ARROW_TEST_LINK_LIBRARIES(jemalloc-memory_pool-test
+  ${ARROW_JEMALLOC_TEST_LINK_LIBS})
+
+ADD_ARROW_BENCHMARK(jemalloc-builder-benchmark)
+ARROW_BENCHMARK_LINK_LIBRARIES(jemalloc-builder-benchmark
+  ${ARROW_JEMALLOC_TEST_LINK_LIBS})
+
+# Headers: top level
+install(FILES
+  memory_pool.h
+  DESTINATION include/arrow/jemalloc)
+
+# pkg-config support
+configure_file(arrow-jemalloc.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/arrow-jemalloc.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-jemalloc.pc"
+  DESTINATION "lib/pkgconfig/")
diff --git a/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in b/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
new file mode 100644
index 0000000000000..0b300fec0b2bf
--- /dev/null
+++ b/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+libdir=${prefix}/lib
+includedir=${prefix}/include
+
+Name: Apache Arrow jemalloc-based allocator
+Description: jemalloc allocator for Arrow.
+Version: @ARROW_VERSION@
+Libs: -L${libdir} -larrow_jemalloc
+Cflags: -I${includedir}
+Requires: arrow
diff --git a/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc b/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
new file mode 100644
index 0000000000000..58dbaa33a1a0f
--- /dev/null
+++ b/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include "arrow/builder.h"
+#include "arrow/jemalloc/memory_pool.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+constexpr int64_t kFinalSize = 256;
+
+static void BM_BuildPrimitiveArrayNoNulls(
+    benchmark::State& state) {  // NOLINT non-const reference
+  // 2 MiB block
+  std::vector<int64_t> data(256 * 1024, 100);
+  while (state.KeepRunning()) {
+    Int64Builder builder(jemalloc::MemoryPool::default_pool(), arrow::int64());
+    for (int i = 0; i < kFinalSize; i++) {
+      // Build up an array of 512 MiB in size
+      builder.Append(data.data(), data.size(), nullptr);
+    }
+    std::shared_ptr<Array> out;
+    builder.Finish(&out);
+  }
+  state.SetBytesProcessed(
+      state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
+}
+
+BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc b/cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc
new file mode 100644
index 0000000000000..a8448abc7d296
--- /dev/null
+++ b/cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <limits>
+
+#include "gtest/gtest.h"
+
+#include "arrow/jemalloc/memory_pool.h"
+#include "arrow/memory_pool-test.h"
+
+namespace arrow {
+namespace jemalloc {
+namespace test {
+
+class TestJemallocMemoryPool : public ::arrow::test::TestMemoryPoolBase {
+ public:
+  ::arrow::MemoryPool* memory_pool() override {
+    return ::arrow::jemalloc::MemoryPool::default_pool();
+  }
+};
+
+TEST_F(TestJemallocMemoryPool, MemoryTracking) {
+  this->TestMemoryTracking();
+}
+
+TEST_F(TestJemallocMemoryPool, OOM) {
+  this->TestOOM();
+}
+
+TEST_F(TestJemallocMemoryPool, Reallocate) {
+  this->TestReallocate();
+}
+
+}  // namespace test
+}  // namespace jemalloc
+}  // namespace arrow
diff --git a/cpp/src/arrow/jemalloc/memory_pool.cc b/cpp/src/arrow/jemalloc/memory_pool.cc
new file mode 100644
index 0000000000000..acc09c7cd7587
--- /dev/null
+++ b/cpp/src/arrow/jemalloc/memory_pool.cc
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/jemalloc/memory_pool.h"
+
+#include <sstream>
+
+#include <jemalloc/jemalloc.h>
+
+#include "arrow/status.h"
+
+constexpr size_t kAlignment = 64;
+
+namespace arrow {
+namespace jemalloc {
+
+MemoryPool* MemoryPool::default_pool() {
+  static MemoryPool pool;
+  return &pool;
+}
+
+MemoryPool::MemoryPool() : allocated_size_(0) {}
+
+MemoryPool::~MemoryPool() {}
+
+Status MemoryPool::Allocate(int64_t size, uint8_t** out) {
+  *out = reinterpret_cast<uint8_t*>(mallocx(size, MALLOCX_ALIGN(kAlignment)));
+  if (*out == NULL) {
+    std::stringstream ss;
+    ss << "malloc of size " << size << " failed";
+    return Status::OutOfMemory(ss.str());
+  }
+  allocated_size_ += size;
+  return Status::OK();
+}
+
+Status MemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
+  *ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
+  if (*ptr == NULL) {
+    std::stringstream ss;
+    ss << "realloc of size " << new_size << " failed";
+    return Status::OutOfMemory(ss.str());
+  }
+
+  allocated_size_ += new_size - old_size;
+
+  return Status::OK();
+}
+
+void MemoryPool::Free(uint8_t* buffer, int64_t size) {
+  allocated_size_ -= size;
+  free(buffer);
+}
+
+int64_t MemoryPool::bytes_allocated() const {
+  return allocated_size_.load();
+}
+
+}  // namespace jemalloc
+}  // namespace arrow
diff --git a/cpp/src/arrow/jemalloc/memory_pool.h b/cpp/src/arrow/jemalloc/memory_pool.h
new file mode 100644
index 0000000000000..0d32b4658e3e8
--- /dev/null
+++ b/cpp/src/arrow/jemalloc/memory_pool.h
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Public API for the jemalloc-based allocator
+
+#ifndef ARROW_JEMALLOC_MEMORY_POOL_H
+#define ARROW_JEMALLOC_MEMORY_POOL_H
+
+#include "arrow/memory_pool.h"
+
+#include <atomic>
+
+namespace arrow {
+
+class Status;
+
+namespace jemalloc {
+
+class ARROW_EXPORT MemoryPool : public ::arrow::MemoryPool {
+ public:
+  static MemoryPool* default_pool();
+
+  MemoryPool(MemoryPool const&) = delete;
+  MemoryPool& operator=(MemoryPool const&) = delete;
+
+  virtual ~MemoryPool();
+
+  Status Allocate(int64_t size, uint8_t** out) override;
+  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
+  void Free(uint8_t* buffer, int64_t size) override;
+
+  int64_t bytes_allocated() const override;
+
+ private:
+  MemoryPool();
+
+  std::atomic<int64_t> allocated_size_;
+};
+
+}  // namespace jemalloc
+}  // namespace arrow
+
+#endif  // ARROW_JEMALLOC_MEMORY_POOL_H
diff --git a/cpp/src/arrow/jemalloc/symbols.map b/cpp/src/arrow/jemalloc/symbols.map
new file mode 100644
index 0000000000000..1e87caef9c8c1
--- /dev/null
+++ b/cpp/src/arrow/jemalloc/symbols.map
@@ -0,0 +1,30 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+{
+  # Symbols marked as 'local' are not exported by the DSO and thus may not
+  # be used by client applications.
+  local:
+    # devtoolset / static-libstdc++ symbols
+    __cxa_*;
+
+    extern "C++" {
+      # boost
+      boost::*;
+
+      # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
+      # links c++11 symbols into binaries so that the result may be executed on
+      # a system with an older libstdc++ which doesn't include the necessary
+      # c++11 symbols.
+      std::*;
+    };
+};
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index d6f323d276305..3daf72755cff2 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -15,35 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "arrow/memory_pool-test.h"
+
 #include <cstdint>
 #include <limits>
 
-#include "gtest/gtest.h"
-
-#include "arrow/memory_pool.h"
-#include "arrow/status.h"
-#include "arrow/test-util.h"
-
 namespace arrow {
 
-TEST(DefaultMemoryPool, MemoryTracking) {
-  MemoryPool* pool = default_memory_pool();
+class TestDefaultMemoryPool : public ::arrow::test::TestMemoryPoolBase {
+ public:
+  ::arrow::MemoryPool* memory_pool() override { return ::arrow::default_memory_pool(); }
+};
 
-  uint8_t* data;
-  ASSERT_OK(pool->Allocate(100, &data));
-  EXPECT_EQ(static_cast<uint64_t>(0), reinterpret_cast<uint64_t>(data) % 64);
-  ASSERT_EQ(100, pool->bytes_allocated());
-
-  pool->Free(data, 100);
-  ASSERT_EQ(0, pool->bytes_allocated());
+TEST_F(TestDefaultMemoryPool, MemoryTracking) {
+  this->TestMemoryTracking();
 }
 
-TEST(DefaultMemoryPool, OOM) {
-  MemoryPool* pool = default_memory_pool();
+TEST_F(TestDefaultMemoryPool, OOM) {
+  this->TestOOM();
+}
 
-  uint8_t* data;
-  int64_t to_alloc = std::numeric_limits<int64_t>::max();
-  ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
+TEST_F(TestDefaultMemoryPool, Reallocate) {
+  this->TestReallocate();
 }
 
 // Death tests and valgrind are known to not play well 100% of the time. See
diff --git a/cpp/src/arrow/memory_pool-test.h b/cpp/src/arrow/memory_pool-test.h
new file mode 100644
index 0000000000000..b9f0337dfac8e
--- /dev/null
+++ b/cpp/src/arrow/memory_pool-test.h
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include <limits>
+
+#include "arrow/memory_pool.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+namespace test {
+
+class TestMemoryPoolBase : public ::testing::Test {
+ public:
+  virtual ::arrow::MemoryPool* memory_pool() = 0;
+
+  void TestMemoryTracking() {
+    auto pool = memory_pool();
+
+    uint8_t* data;
+    ASSERT_OK(pool->Allocate(100, &data));
+    EXPECT_EQ(static_cast<uint64_t>(0), reinterpret_cast<uint64_t>(data) % 64);
+    ASSERT_EQ(100, pool->bytes_allocated());
+
+    pool->Free(data, 100);
+    ASSERT_EQ(0, pool->bytes_allocated());
+  }
+
+  void TestOOM() {
+    auto pool = memory_pool();
+
+    uint8_t* data;
+    int64_t to_alloc = std::numeric_limits<int64_t>::max();
+    ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
+  }
+
+  void TestReallocate() {
+    auto pool = memory_pool();
+
+    uint8_t* data;
+    ASSERT_OK(pool->Allocate(10, &data));
+    ASSERT_EQ(10, pool->bytes_allocated());
+    data[0] = 35;
+    data[9] = 12;
+
+    // Expand
+    ASSERT_OK(pool->Reallocate(10, 20, &data));
+    ASSERT_EQ(data[9], 12);
+    ASSERT_EQ(20, pool->bytes_allocated());
+
+    // Shrink
+    ASSERT_OK(pool->Reallocate(20, 5, &data));
+    ASSERT_EQ(data[0], 35);
+    ASSERT_EQ(5, pool->bytes_allocated());
+
+    // Free
+    pool->Free(data, 5);
+    ASSERT_EQ(0, pool->bytes_allocated());
+  }
+};
+
+}  // namespace test
+}  // namespace arrow
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index f55b1ac668c7c..aea5e210f4980 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/memory_pool.h"
 
+#include <algorithm>
 #include <cstdlib>
 #include <mutex>
 #include <sstream>
@@ -67,6 +68,7 @@ class InternalMemoryPool : public MemoryPool {
   virtual ~InternalMemoryPool();
 
   Status Allocate(int64_t size, uint8_t** out) override;
+  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
 
   void Free(uint8_t* buffer, int64_t size) override;
 
@@ -85,6 +87,28 @@ Status InternalMemoryPool::Allocate(int64_t size, uint8_t** out) {
   return Status::OK();
 }
 
+Status InternalMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
+  std::lock_guard<std::mutex> guard(pool_lock_);
+
+  // Note: We cannot use realloc() here as it doesn't guarantee alignment.
+
+  // Allocate new chunk
+  uint8_t* out;
+  RETURN_NOT_OK(AllocateAligned(new_size, &out));
+  // Copy contents and release old memory chunk
+  memcpy(out, *ptr, std::min(new_size, old_size));
+#ifdef _MSC_VER
+  _aligned_free(*ptr);
+#else
+  std::free(*ptr);
+#endif
+  *ptr = out;
+
+  bytes_allocated_ += new_size - old_size;
+
+  return Status::OK();
+}
+
 int64_t InternalMemoryPool::bytes_allocated() const {
   std::lock_guard<std::mutex> guard(pool_lock_);
   return bytes_allocated_;
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 4c1d699addd50..13a3f129c1a9e 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -31,6 +31,7 @@ class ARROW_EXPORT MemoryPool {
   virtual ~MemoryPool();
 
   virtual Status Allocate(int64_t size, uint8_t** out) = 0;
+  virtual Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) = 0;
   virtual void Free(uint8_t* buffer, int64_t size) = 0;
 
   virtual int64_t bytes_allocated() const = 0;
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index 8660ac8f0cedf..0bdd289953dc4 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -47,6 +47,20 @@ class PyArrowMemoryPool : public arrow::MemoryPool {
     return Status::OK();
   }
 
+  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override {
+    *ptr = reinterpret_cast<uint8_t*>(std::realloc(*ptr, new_size));
+
+    if (*ptr == NULL) {
+      std::stringstream ss;
+      ss << "realloc of size " << new_size << " failed";
+      return Status::OutOfMemory(ss.str());
+    }
+
+    bytes_allocated_ += new_size - old_size;
+
+    return Status::OK();
+  }
+
   int64_t bytes_allocated() const override {
     std::lock_guard<std::mutex> guard(pool_lock_);
     return bytes_allocated_;

From 74685f386307171a90a9f97316e25b7f39cdd0a1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 6 Jan 2017 11:11:43 -0500
Subject: [PATCH 0262/1644] ARROW-427: [C++] Implement dictionary array type

I thought some about this and thought that it made sense to store the reference to the dictionary values themselves in the data type object, similar to `CategoricalDtype` in pandas. This will be at least adequate for the Feather file format merge.

In the IPC metadata, there is no explicit dictionary type -- an array can be dictionary encoded or not. On JIRA we've discussed adding a dictionary type flag indicating whether or not the dictionary values/categories are ordered (also called "ordinal") or unordered (also called "nominal"). That hasn't been done yet.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #268 from wesm/ARROW-427 and squashes the following commits:

5ce3701 [Wes McKinney] cpplint
a6c2896 [Wes McKinney] Revert T::Equals(const T& other) to EqualsExact to appease clang
9a4edb5 [Wes McKinney] Implement rudimentary DictionaryArray::Validate
9efe46b [Wes McKinney] Add tests, implementation for DictionaryArray::Equals and RangeEquals
b06eb86 [Wes McKinney] Implement PrettyPrint for DictionaryArray
17c70de [Wes McKinney] Refactor, compose shared_ptr<DataType> in DictionaryType
b52b3a7 [Wes McKinney] Add rudimentary DictionaryType and DictionaryArray implementation for discussion
---
 cpp/src/arrow/CMakeLists.txt           |   1 +
 cpp/src/arrow/array-dictionary-test.cc | 128 +++++++++++++++++++
 cpp/src/arrow/array-string-test.cc     |   4 +-
 cpp/src/arrow/array.cc                 |  94 +++++++++++---
 cpp/src/arrow/array.h                  | 111 ++++++++++++++---
 cpp/src/arrow/ipc/adapter.cc           |  11 ++
 cpp/src/arrow/ipc/json-internal.cc     |  13 ++
 cpp/src/arrow/pretty_print-test.cc     |  53 ++++----
 cpp/src/arrow/pretty_print.cc          |  12 ++
 cpp/src/arrow/test-util.h              |  36 +++---
 cpp/src/arrow/type.cc                  |  69 +++++++++--
 cpp/src/arrow/type.h                   | 163 +++++++++++++++++++------
 cpp/src/arrow/type_fwd.h               |  57 +--------
 format/Message.fbs                     |   2 +-
 python/pyarrow/includes/libarrow.pxd   |   3 +-
 python/pyarrow/includes/parquet.pxd    |   2 +-
 python/pyarrow/parquet.pyx             |   4 +-
 python/pyarrow/schema.pyx              |   4 +-
 18 files changed, 583 insertions(+), 184 deletions(-)
 create mode 100644 cpp/src/arrow/array-dictionary-test.cc

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 16668db798b78..e5e36ed253cfa 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -47,6 +47,7 @@ install(
 
 ADD_ARROW_TEST(array-test)
 ADD_ARROW_TEST(array-decimal-test)
+ADD_ARROW_TEST(array-dictionary-test)
 ADD_ARROW_TEST(array-list-test)
 ADD_ARROW_TEST(array-primitive-test)
 ADD_ARROW_TEST(array-string-test)
diff --git a/cpp/src/arrow/array-dictionary-test.cc b/cpp/src/arrow/array-dictionary-test.cc
new file mode 100644
index 0000000000000..c290153b95053
--- /dev/null
+++ b/cpp/src/arrow/array-dictionary-test.cc
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <numeric>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+TEST(TestDictionary, Basics) {
+  std::vector<int32_t> values = {100, 1000, 10000, 100000};
+  std::shared_ptr<Array> dict;
+  ArrayFromVector<Int32Type, int32_t>(int32(), values, &dict);
+
+  std::shared_ptr<DictionaryType> type1 =
+      std::dynamic_pointer_cast<DictionaryType>(dictionary(int16(), dict));
+  DictionaryType type2(int16(), dict);
+
+  ASSERT_TRUE(int16()->Equals(type1->index_type()));
+  ASSERT_TRUE(type1->dictionary()->Equals(dict));
+
+  ASSERT_TRUE(int16()->Equals(type2.index_type()));
+  ASSERT_TRUE(type2.dictionary()->Equals(dict));
+
+  ASSERT_EQ("dictionary<int32, int16>", type1->ToString());
+}
+
+TEST(TestDictionary, Equals) {
+  std::vector<bool> is_valid = {true, true, false, true, true, true};
+
+  std::shared_ptr<Array> dict;
+  std::vector<std::string> dict_values = {"foo", "bar", "baz"};
+  ArrayFromVector<StringType, std::string>(utf8(), dict_values, &dict);
+  std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
+
+  std::shared_ptr<Array> dict2;
+  std::vector<std::string> dict2_values = {"foo", "bar", "baz", "qux"};
+  ArrayFromVector<StringType, std::string>(utf8(), dict2_values, &dict2);
+  std::shared_ptr<DataType> dict2_type = dictionary(int16(), dict2);
+
+  std::shared_ptr<Array> indices;
+  std::vector<int16_t> indices_values = {1, 2, -1, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices_values, &indices);
+
+  std::shared_ptr<Array> indices2;
+  std::vector<int16_t> indices2_values = {1, 2, 0, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices2_values, &indices2);
+
+  std::shared_ptr<Array> indices3;
+  std::vector<int16_t> indices3_values = {1, 1, 0, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices3_values, &indices3);
+
+  auto arr = std::make_shared<DictionaryArray>(dict_type, indices);
+  auto arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
+  auto arr3 = std::make_shared<DictionaryArray>(dict2_type, indices);
+  auto arr4 = std::make_shared<DictionaryArray>(dict_type, indices3);
+
+  ASSERT_TRUE(arr->Equals(arr));
+
+  // Equal, because the unequal index is masked by null
+  ASSERT_TRUE(arr->Equals(arr2));
+
+  // Unequal dictionaries
+  ASSERT_FALSE(arr->Equals(arr3));
+
+  // Unequal indices
+  ASSERT_FALSE(arr->Equals(arr4));
+
+  // RangeEquals
+  ASSERT_TRUE(arr->RangeEquals(3, 6, 3, arr4));
+  ASSERT_FALSE(arr->RangeEquals(1, 3, 1, arr4));
+}
+
+TEST(TestDictionary, Validate) {
+  std::vector<bool> is_valid = {true, true, false, true, true, true};
+
+  std::shared_ptr<Array> dict;
+  std::vector<std::string> dict_values = {"foo", "bar", "baz"};
+  ArrayFromVector<StringType, std::string>(utf8(), dict_values, &dict);
+  std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
+
+  std::shared_ptr<Array> indices;
+  std::vector<uint8_t> indices_values = {1, 2, 0, 0, 2, 0};
+  ArrayFromVector<UInt8Type, uint8_t>(uint8(), is_valid, indices_values, &indices);
+
+  std::shared_ptr<Array> indices2;
+  std::vector<float> indices2_values = {1., 2., 0., 0., 2., 0.};
+  ArrayFromVector<FloatType, float>(float32(), is_valid, indices2_values, &indices2);
+
+  std::shared_ptr<Array> indices3;
+  std::vector<int64_t> indices3_values = {1, 2, 0, 0, 2, 0};
+  ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, indices3_values, &indices3);
+
+  std::shared_ptr<Array> arr = std::make_shared<DictionaryArray>(dict_type, indices);
+  std::shared_ptr<Array> arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
+  std::shared_ptr<Array> arr3 = std::make_shared<DictionaryArray>(dict_type, indices3);
+
+  // Only checking index type for now
+  ASSERT_OK(arr->Validate());
+  ASSERT_RAISES(Invalid, arr2->Validate());
+  ASSERT_OK(arr3->Validate());
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index b144c632133d6..024bfd508957d 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -36,8 +36,8 @@ TEST(TypesTest, BinaryType) {
   BinaryType t1;
   BinaryType e1;
   StringType t2;
-  EXPECT_TRUE(t1.Equals(&e1));
-  EXPECT_FALSE(t1.Equals(&t2));
+  EXPECT_TRUE(t1.Equals(e1));
+  EXPECT_FALSE(t1.Equals(t2));
   ASSERT_EQ(t1.type, Type::BINARY);
   ASSERT_EQ(t1.ToString(), std::string("binary"));
 }
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 3d309b8b92f48..7509520d12685 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -42,7 +42,7 @@ Status GetEmptyBitmap(
 // ----------------------------------------------------------------------
 // Base array class
 
-Array::Array(const TypePtr& type, int32_t length, int32_t null_count,
+Array::Array(const std::shared_ptr<DataType>& type, int32_t length, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap) {
   type_ = type;
   length_ = length;
@@ -51,6 +51,12 @@ Array::Array(const TypePtr& type, int32_t length, int32_t null_count,
   if (null_bitmap_) { null_bitmap_data_ = null_bitmap_->data(); }
 }
 
+bool Array::BaseEquals(const std::shared_ptr<Array>& other) const {
+  if (this == other.get()) { return true; }
+  if (!other) { return false; }
+  return EqualsExact(*other.get());
+}
+
 bool Array::EqualsExact(const Array& other) const {
   if (this == &other) { return true; }
   if (length_ != other.length_ || null_count_ != other.null_count_ ||
@@ -91,7 +97,7 @@ Status NullArray::Accept(ArrayVisitor* visitor) const {
 // ----------------------------------------------------------------------
 // Primitive array base
 
-PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
+PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap)
     : Array(type, length, null_count, null_bitmap) {
@@ -100,14 +106,9 @@ PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
 }
 
 bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
-  if (this == &other) { return true; }
-  if (null_count_ != other.null_count_) { return false; }
+  if (!Array::EqualsExact(other)) { return false; }
 
   if (null_count_ > 0) {
-    bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
-    if (!equal_bitmap) { return false; }
-
     const uint8_t* this_data = raw_data_;
     const uint8_t* other_data = other.raw_data_;
 
@@ -131,7 +132,7 @@ bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) { return true; }
   if (!arr) { return false; }
   if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
+  return EqualsExact(static_cast<const PrimitiveArray&>(*arr.get()));
 }
 
 template <typename T>
@@ -161,7 +162,7 @@ BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
     : PrimitiveArray(
           std::make_shared<BooleanType>(), length, data, null_count, null_bitmap) {}
 
-BooleanArray::BooleanArray(const TypePtr& type, int32_t length,
+BooleanArray::BooleanArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap)
     : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
@@ -192,7 +193,7 @@ bool BooleanArray::EqualsExact(const BooleanArray& other) const {
 bool BooleanArray::Equals(const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) return true;
   if (Type::BOOL != arr->type_enum()) { return false; }
-  return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
+  return EqualsExact(static_cast<const BooleanArray&>(*arr.get()));
 }
 
 bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
@@ -238,7 +239,7 @@ bool ListArray::EqualsExact(const ListArray& other) const {
 bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) { return true; }
   if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(*static_cast<const ListArray*>(arr.get()));
+  return EqualsExact(static_cast<const ListArray&>(*arr.get()));
 }
 
 bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
@@ -333,7 +334,7 @@ BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
     const std::shared_ptr<Buffer>& null_bitmap)
     : BinaryArray(kBinary, length, offsets, data, null_count, null_bitmap) {}
 
-BinaryArray::BinaryArray(const TypePtr& type, int32_t length,
+BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
     int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
     : Array(type, length, null_count, null_bitmap),
@@ -364,7 +365,7 @@ bool BinaryArray::EqualsExact(const BinaryArray& other) const {
 bool BinaryArray::Equals(const std::shared_ptr<Array>& arr) const {
   if (this == arr.get()) { return true; }
   if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(*static_cast<const BinaryArray*>(arr.get()));
+  return EqualsExact(static_cast<const BinaryArray&>(*arr.get()));
 }
 
 bool BinaryArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
@@ -493,7 +494,7 @@ Status StructArray::Accept(ArrayVisitor* visitor) const {
 // ----------------------------------------------------------------------
 // UnionArray
 
-UnionArray::UnionArray(const TypePtr& type, int32_t length,
+UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::vector<std::shared_ptr<Array>>& children,
     const std::shared_ptr<Buffer>& type_ids, const std::shared_ptr<Buffer>& offsets,
     int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
@@ -586,6 +587,66 @@ Status UnionArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+// ----------------------------------------------------------------------
+// DictionaryArray
+
+Status DictionaryArray::FromBuffer(const std::shared_ptr<DataType>& type, int32_t length,
+    const std::shared_ptr<Buffer>& indices, int32_t null_count,
+    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<DictionaryArray>* out) {
+  DCHECK_EQ(type->type, Type::DICTIONARY);
+  const auto& dict_type = static_cast<const DictionaryType*>(type.get());
+
+  std::shared_ptr<Array> boxed_indices;
+  RETURN_NOT_OK(MakePrimitiveArray(
+      dict_type->index_type(), length, indices, null_count, null_bitmap, &boxed_indices));
+
+  *out = std::make_shared<DictionaryArray>(type, boxed_indices);
+  return Status::OK();
+}
+
+DictionaryArray::DictionaryArray(
+    const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices)
+    : Array(type, indices->length(), indices->null_count(), indices->null_bitmap()),
+      dict_type_(static_cast<const DictionaryType*>(type.get())),
+      indices_(indices) {
+  DCHECK_EQ(type->type, Type::DICTIONARY);
+}
+
+Status DictionaryArray::Validate() const {
+  Type::type index_type_id = indices_->type()->type;
+  if (!is_integer(index_type_id)) {
+    return Status::Invalid("Dictionary indices must be integer type");
+  }
+  return Status::OK();
+}
+
+std::shared_ptr<Array> DictionaryArray::dictionary() const {
+  return dict_type_->dictionary();
+}
+
+bool DictionaryArray::EqualsExact(const DictionaryArray& other) const {
+  if (!dictionary()->Equals(other.dictionary())) { return false; }
+  return indices_->Equals(other.indices());
+}
+
+bool DictionaryArray::Equals(const std::shared_ptr<Array>& arr) const {
+  if (this == arr.get()) { return true; }
+  if (Type::DICTIONARY != arr->type_enum()) { return false; }
+  return EqualsExact(static_cast<const DictionaryArray&>(*arr.get()));
+}
+
+bool DictionaryArray::RangeEquals(int32_t start_idx, int32_t end_idx,
+    int32_t other_start_idx, const std::shared_ptr<Array>& arr) const {
+  if (Type::DICTIONARY != arr->type_enum()) { return false; }
+  const auto& dict_other = static_cast<const DictionaryArray&>(*arr.get());
+  if (!dictionary()->Equals(dict_other.dictionary())) { return false; }
+  return indices_->RangeEquals(start_idx, end_idx, other_start_idx, dict_other.indices());
+}
+
+Status DictionaryArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 // ----------------------------------------------------------------------
 
 #define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                          \
@@ -593,7 +654,7 @@ Status UnionArray::Accept(ArrayVisitor* visitor) const {
     out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
     break;
 
-Status MakePrimitiveArray(const TypePtr& type, int32_t length,
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::shared_ptr<Buffer>& data, int32_t null_count,
     const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out) {
   switch (type->type) {
@@ -610,7 +671,6 @@ Status MakePrimitiveArray(const TypePtr& type, int32_t length,
     MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
     MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
     MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP_DOUBLE, DoubleArray);
     default:
       return Status::NotImplemented(type->ToString());
   }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index cd42a28e251ca..57214c46d1cc6 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -26,6 +26,7 @@
 
 #include "arrow/buffer.h"
 #include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
@@ -36,6 +37,34 @@ class MemoryPool;
 class MutableBuffer;
 class Status;
 
+class ArrayVisitor {
+ public:
+  virtual Status Visit(const NullArray& array) = 0;
+  virtual Status Visit(const BooleanArray& array) = 0;
+  virtual Status Visit(const Int8Array& array) = 0;
+  virtual Status Visit(const Int16Array& array) = 0;
+  virtual Status Visit(const Int32Array& array) = 0;
+  virtual Status Visit(const Int64Array& array) = 0;
+  virtual Status Visit(const UInt8Array& array) = 0;
+  virtual Status Visit(const UInt16Array& array) = 0;
+  virtual Status Visit(const UInt32Array& array) = 0;
+  virtual Status Visit(const UInt64Array& array) = 0;
+  virtual Status Visit(const HalfFloatArray& array) = 0;
+  virtual Status Visit(const FloatArray& array) = 0;
+  virtual Status Visit(const DoubleArray& array) = 0;
+  virtual Status Visit(const StringArray& array) = 0;
+  virtual Status Visit(const BinaryArray& array) = 0;
+  virtual Status Visit(const DateArray& array) = 0;
+  virtual Status Visit(const TimeArray& array) = 0;
+  virtual Status Visit(const TimestampArray& array) = 0;
+  virtual Status Visit(const IntervalArray& array) = 0;
+  virtual Status Visit(const DecimalArray& array) = 0;
+  virtual Status Visit(const ListArray& array) = 0;
+  virtual Status Visit(const StructArray& array) = 0;
+  virtual Status Visit(const UnionArray& array) = 0;
+  virtual Status Visit(const DictionaryArray& type) = 0;
+};
+
 // Immutable data array with some logical type and some length. Any memory is
 // owned by the respective Buffer instance (or its parents).
 //
@@ -63,6 +92,7 @@ class ARROW_EXPORT Array {
 
   const uint8_t* null_bitmap_data() const { return null_bitmap_data_; }
 
+  bool BaseEquals(const std::shared_ptr<Array>& arr) const;
   bool EqualsExact(const Array& arr) const;
   virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
   virtual bool ApproxEquals(const std::shared_ptr<Array>& arr) const;
@@ -122,8 +152,9 @@ class ARROW_EXPORT PrimitiveArray : public Array {
   bool Equals(const std::shared_ptr<Array>& arr) const override;
 
  protected:
-  PrimitiveArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
   std::shared_ptr<Buffer> data_;
   const uint8_t* raw_data_;
 };
@@ -137,8 +168,9 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
       int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
       : PrimitiveArray(
             std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {}
-  NumericArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+  NumericArray(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr)
       : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
 
   bool EqualsExact(const NumericArray<TypeClass>& other) const {
@@ -146,7 +178,7 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   }
 
   bool ApproxEquals(const std::shared_ptr<Array>& arr) const override {
-    return Equals(arr);
+    return PrimitiveArray::Equals(arr);
   }
 
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
@@ -250,8 +282,9 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
 
   BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
       int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-  BooleanArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  BooleanArray(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   bool EqualsExact(const BooleanArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
@@ -272,9 +305,9 @@ class ARROW_EXPORT ListArray : public Array {
  public:
   using TypeClass = ListType;
 
-  ListArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const std::shared_ptr<Array>& values, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+  ListArray(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Array>& values,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
       : Array(type, length, null_count, null_bitmap) {
     offsets_buffer_ = offsets;
     offsets_ = offsets == nullptr ? nullptr : reinterpret_cast<const int32_t*>(
@@ -328,9 +361,9 @@ class ARROW_EXPORT BinaryArray : public Array {
 
   // Constructor that allows sub-classes/builders to propagate there logical type up the
   // class hierarchy.
-  BinaryArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  BinaryArray(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
+      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
   // Return the pointer to the given elements bytes
   // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
@@ -397,7 +430,7 @@ class ARROW_EXPORT StructArray : public Array {
  public:
   using TypeClass = StructType;
 
-  StructArray(const TypePtr& type, int32_t length,
+  StructArray(const std::shared_ptr<DataType>& type, int32_t length,
       const std::vector<std::shared_ptr<Array>>& field_arrays, int32_t null_count = 0,
       std::shared_ptr<Buffer> null_bitmap = nullptr)
       : Array(type, length, null_count, null_bitmap) {
@@ -434,7 +467,7 @@ class ARROW_EXPORT UnionArray : public Array {
  public:
   using TypeClass = UnionType;
 
-  UnionArray(const TypePtr& type, int32_t length,
+  UnionArray(const std::shared_ptr<DataType>& type, int32_t length,
       const std::vector<std::shared_ptr<Array>>& children,
       const std::shared_ptr<Buffer>& type_ids,
       const std::shared_ptr<Buffer>& offsets = nullptr, int32_t null_count = 0,
@@ -473,6 +506,54 @@ class ARROW_EXPORT UnionArray : public Array {
   const int32_t* offsets_;
 };
 
+// ----------------------------------------------------------------------
+// DictionaryArray (categorical and dictionary-encoded in memory)
+
+// A dictionary array contains an array of non-negative integers (the
+// "dictionary indices") along with a data type containing a "dictionary"
+// corresponding to the distinct values represented in the data.
+//
+// For example, the array
+//
+//   ["foo", "bar", "foo", "bar", "foo", "bar"]
+//
+// with dictionary ["bar", "foo"], would have dictionary array representation
+//
+//   indices: [1, 0, 1, 0, 1, 0]
+//   dictionary: ["bar", "foo"]
+//
+// The indices in principle may have any integer type (signed or unsigned),
+// though presently data in IPC exchanges must be signed int32.
+class ARROW_EXPORT DictionaryArray : public Array {
+ public:
+  using TypeClass = DictionaryType;
+
+  DictionaryArray(
+      const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices);
+
+  // Alternate ctor; other attributes (like null count) are inherited from the
+  // passed indices array
+  static Status FromBuffer(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& indices, int32_t null_count,
+      const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<DictionaryArray>* out);
+
+  Status Validate() const override;
+
+  std::shared_ptr<Array> indices() const { return indices_; }
+  std::shared_ptr<Array> dictionary() const;
+
+  bool EqualsExact(const DictionaryArray& other) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const override;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const std::shared_ptr<Array>& arr) const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
+
+ protected:
+  const DictionaryType* dict_type_;
+  std::shared_ptr<Array> indices_;
+};
+
 // ----------------------------------------------------------------------
 // extern templates and other details
 
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 9bfd11fd01b5a..2b5ef11f861af 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -288,6 +288,13 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
+  Status Visit(const DictionaryArray& array) override {
+    // Dictionary written out separately
+    const auto& indices = static_cast<const PrimitiveArray&>(*array.indices().get());
+    buffers_.push_back(indices.data());
+    return Status::OK();
+  }
+
   // Do not copy this vector. Ownership must be retained elsewhere
   const std::vector<std::shared_ptr<Array>>& columns_;
   int32_t num_rows_;
@@ -539,6 +546,10 @@ class ArrayLoader : public TypeVisitor {
         type_ids, offsets, field_meta.null_count, null_bitmap);
     return Status::OK();
   }
+
+  Status Visit(const DictionaryType& type) override {
+    return Status::NotImplemented("dictionary");
+  };
 };
 
 class RecordBatchReader {
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 4f980d3e5d157..43bd8a4a4e814 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -334,6 +334,14 @@ class JsonSchemaWriter : public TypeVisitor {
     return Status::OK();
   }
 
+  Status Visit(const DictionaryType& type) override {
+    // WriteName("dictionary", type);
+    // WriteChildren(type.children());
+    // WriteBufferLayout(type.GetBufferLayout());
+    // return Status::OK();
+    return Status::NotImplemented("dictionary type");
+  }
+
  private:
   const Schema& schema_;
   RjWriter* writer_;
@@ -546,6 +554,10 @@ class JsonArrayWriter : public ArrayVisitor {
     return WriteChildren(type->children(), array.children());
   }
 
+  Status Visit(const DictionaryArray& array) override {
+    return Status::NotImplemented("dictionary");
+  }
+
  private:
   const std::string& name_;
   const Array& array_;
@@ -1043,6 +1055,7 @@ class JsonArrayReader {
       TYPE_CASE(ListType);
       TYPE_CASE(StructType);
       TYPE_CASE(UnionType);
+      NOT_IMPLEMENTED_CASE(DICTIONARY);
       default:
         std::stringstream ss;
         ss << type->ToString();
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index c22d3aa632b9d..4725d5dd808ee 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -34,7 +34,7 @@
 
 namespace arrow {
 
-class TestArrayPrinter : public ::testing::Test {
+class TestPrettyPrint : public ::testing::Test {
  public:
   void SetUp() {}
 
@@ -44,32 +44,22 @@ class TestArrayPrinter : public ::testing::Test {
   std::ostringstream sink_;
 };
 
+void CheckArray(const Array& arr, int indent, const char* expected) {
+  std::ostringstream sink;
+  ASSERT_OK(PrettyPrint(arr, indent, &sink));
+  std::string result = sink.str();
+  ASSERT_EQ(std::string(expected, strlen(expected)), result);
+}
+
 template <typename TYPE, typename C_TYPE>
 void CheckPrimitive(int indent, const std::vector<bool>& is_valid,
     const std::vector<C_TYPE>& values, const char* expected) {
-  std::ostringstream sink;
-
-  MemoryPool* pool = default_memory_pool();
-  typename TypeTraits<TYPE>::BuilderType builder(pool, std::make_shared<TYPE>());
-
-  for (size_t i = 0; i < values.size(); ++i) {
-    if (is_valid[i]) {
-      ASSERT_OK(builder.Append(values[i]));
-    } else {
-      ASSERT_OK(builder.AppendNull());
-    }
-  }
-
   std::shared_ptr<Array> array;
-  ASSERT_OK(builder.Finish(&array));
-
-  ASSERT_OK(PrettyPrint(*array.get(), indent, &sink));
-
-  std::string result = sink.str();
-  ASSERT_EQ(std::string(expected, strlen(expected)), result);
+  ArrayFromVector<TYPE, C_TYPE>(std::make_shared<TYPE>(), is_valid, values, &array);
+  CheckArray(*array.get(), indent, expected);
 }
 
-TEST_F(TestArrayPrinter, PrimitiveType) {
+TEST_F(TestPrettyPrint, PrimitiveType) {
   std::vector<bool> is_valid = {true, true, false, true, false};
 
   std::vector<int32_t> values = {0, 1, 2, 3, 4};
@@ -81,4 +71,25 @@ TEST_F(TestArrayPrinter, PrimitiveType) {
   CheckPrimitive<StringType, std::string>(0, is_valid, values2, ex2);
 }
 
+TEST_F(TestPrettyPrint, DictionaryType) {
+  std::vector<bool> is_valid = {true, true, false, true, true, true};
+
+  std::shared_ptr<Array> dict;
+  std::vector<std::string> dict_values = {"foo", "bar", "baz"};
+  ArrayFromVector<StringType, std::string>(utf8(), dict_values, &dict);
+  std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
+
+  std::shared_ptr<Array> indices;
+  std::vector<int16_t> indices_values = {1, 2, -1, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices_values, &indices);
+  auto arr = std::make_shared<DictionaryArray>(dict_type, indices);
+
+  static const char* expected = R"expected(
+-- is_valid: [true, true, false, true, true, true]
+-- dictionary: ["foo", "bar", "baz"]
+-- indices: [1, 2, null, 0, 2, 0])expected";
+
+  CheckArray(*arr.get(), 0, expected);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 324f81bfbfd6b..e30f4cc58d7ab 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -217,6 +217,18 @@ class ArrayPrinter : public ArrayVisitor {
     return PrintChildren(array.children());
   }
 
+  Status Visit(const DictionaryArray& array) override {
+    RETURN_NOT_OK(WriteValidityBitmap(array));
+
+    Newline();
+    Write("-- dictionary: ");
+    RETURN_NOT_OK(PrettyPrint(*array.dictionary().get(), indent_ + 2, sink_));
+
+    Newline();
+    Write("-- indices: ");
+    return PrettyPrint(*array.indices().get(), indent_ + 2, sink_);
+  }
+
   void Write(const char* data) { (*sink_) << data; }
 
   void Write(const std::string& data) { (*sink_) << data; }
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 70e933365cfdf..e5957490e0733 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -257,33 +257,27 @@ template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::shared_ptr<DataType>& type,
     const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
     std::shared_ptr<Array>* out) {
-  std::shared_ptr<Buffer> values_buffer;
-  std::shared_ptr<Buffer> values_bitmap;
-
-  ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
-  ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap));
-
-  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
-
-  int32_t null_count = 0;
-  for (bool val : is_valid) {
-    if (!val) { ++null_count; }
+  MemoryPool* pool = default_memory_pool();
+  typename TypeTraits<TYPE>::BuilderType builder(pool, std::make_shared<TYPE>());
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid[i]) {
+      ASSERT_OK(builder.Append(values[i]));
+    } else {
+      ASSERT_OK(builder.AppendNull());
+    }
   }
-
-  *out = std::make_shared<ArrayType>(type, static_cast<int32_t>(values.size()),
-      values_buffer, null_count, values_bitmap);
+  ASSERT_OK(builder.Finish(out));
 }
 
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::shared_ptr<DataType>& type,
     const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
-  std::shared_ptr<Buffer> values_buffer;
-
-  ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
-
-  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
-  *out = std::make_shared<ArrayType>(
-      type, static_cast<int32_t>(values.size()), values_buffer);
+  MemoryPool* pool = default_memory_pool();
+  typename TypeTraits<TYPE>::BuilderType builder(pool, std::make_shared<TYPE>());
+  for (size_t i = 0; i < values.size(); ++i) {
+    ASSERT_OK(builder.Append(values[i]));
+  }
+  ASSERT_OK(builder.Finish(out));
 }
 
 class TestBuilder : public ::testing::Test {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 89faab6ec6ae2..954fba7af9df9 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -20,10 +20,22 @@
 #include <sstream>
 #include <string>
 
+#include "arrow/array.h"
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 
+bool Field::Equals(const Field& other) const {
+  return (this == &other) ||
+         (this->name == other.name && this->nullable == other.nullable &&
+             this->dictionary == dictionary && this->type->Equals(*other.type.get()));
+}
+
+bool Field::Equals(const std::shared_ptr<Field>& other) const {
+  return Equals(*other.get());
+}
+
 std::string Field::ToString() const {
   std::stringstream ss;
   ss << this->name << ": " << this->type->ToString();
@@ -33,14 +45,14 @@ std::string Field::ToString() const {
 
 DataType::~DataType() {}
 
-bool DataType::Equals(const DataType* other) const {
-  bool equals = other && ((this == other) ||
-                             ((this->type == other->type) &&
-                                 ((this->num_children() == other->num_children()))));
+bool DataType::Equals(const DataType& other) const {
+  bool equals =
+      ((this == &other) || ((this->type == other.type) &&
+                               ((this->num_children() == other.num_children()))));
   if (equals) {
     for (int i = 0; i < num_children(); ++i) {
       // TODO(emkornfield) limit recursion
-      if (!children_[i]->Equals(other->children_[i])) { return false; }
+      if (!children_[i]->Equals(other.children_[i])) { return false; }
     }
   }
   return equals;
@@ -109,11 +121,47 @@ std::string UnionType::ToString() const {
   return s.str();
 }
 
+// ----------------------------------------------------------------------
+// DictionaryType
+
+DictionaryType::DictionaryType(
+    const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& dictionary)
+    : FixedWidthType(Type::DICTIONARY),
+      index_type_(index_type),
+      dictionary_(dictionary) {}
+
+int DictionaryType::bit_width() const {
+  return static_cast<const FixedWidthType*>(index_type_.get())->bit_width();
+}
+
+std::shared_ptr<Array> DictionaryType::dictionary() const {
+  return dictionary_;
+}
+
+bool DictionaryType::Equals(const DataType& other) const {
+  if (other.type != Type::DICTIONARY) { return false; }
+  const auto& other_dict = static_cast<const DictionaryType&>(other);
+
+  return index_type_->Equals(other_dict.index_type_) &&
+         dictionary_->Equals(other_dict.dictionary_);
+}
+
+std::string DictionaryType::ToString() const {
+  std::stringstream ss;
+  ss << "dictionary<" << dictionary_->type()->ToString() << ", "
+     << index_type_->ToString() << ">";
+  return ss.str();
+}
+
+// ----------------------------------------------------------------------
+// Null type
+
 std::string NullType::ToString() const {
   return name();
 }
 
-// Visitors and template instantiation
+// ----------------------------------------------------------------------
+// Visitors and factory functions
 
 #define ACCEPT_VISITOR(TYPE) \
   Status TYPE::Accept(TypeVisitor* visitor) const { return visitor->Visit(*this); }
@@ -130,6 +178,7 @@ ACCEPT_VISITOR(DateType);
 ACCEPT_VISITOR(TimeType);
 ACCEPT_VISITOR(TimestampType);
 ACCEPT_VISITOR(IntervalType);
+ACCEPT_VISITOR(DictionaryType);
 
 #define TYPE_FACTORY(NAME, KLASS)                                        \
   std::shared_ptr<DataType> NAME() {                                     \
@@ -174,12 +223,16 @@ std::shared_ptr<DataType> struct_(const std::vector<std::shared_ptr<Field>>& fie
   return std::make_shared<StructType>(fields);
 }
 
-std::shared_ptr<DataType> ARROW_EXPORT union_(
-    const std::vector<std::shared_ptr<Field>>& child_fields,
+std::shared_ptr<DataType> union_(const std::vector<std::shared_ptr<Field>>& child_fields,
     const std::vector<uint8_t>& type_ids, UnionMode mode) {
   return std::make_shared<UnionType>(child_fields, type_ids, mode);
 }
 
+std::shared_ptr<DataType> dictionary(const std::shared_ptr<DataType>& index_type,
+    const std::shared_ptr<Array>& dict_values) {
+  return std::make_shared<DictionaryType>(index_type, dict_values);
+}
+
 std::shared_ptr<Field> field(
     const std::string& name, const TypePtr& type, bool nullable, int64_t dictionary) {
   return std::make_shared<Field>(name, type, nullable, dictionary);
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 530c3235dc9ab..c2a762d279364 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -37,67 +37,64 @@ namespace arrow {
 struct Type {
   enum type {
     // A degenerate NULL type represented as 0 bytes/bits
-    NA = 0,
+    NA,
 
     // A boolean value represented as 1 bit
-    BOOL = 1,
+    BOOL,
 
     // Little-endian integer types
-    UINT8 = 2,
-    INT8 = 3,
-    UINT16 = 4,
-    INT16 = 5,
-    UINT32 = 6,
-    INT32 = 7,
-    UINT64 = 8,
-    INT64 = 9,
+    UINT8,
+    INT8,
+    UINT16,
+    INT16,
+    UINT32,
+    INT32,
+    UINT64,
+    INT64,
 
     // 2-byte floating point value
-    HALF_FLOAT = 10,
+    HALF_FLOAT,
 
     // 4-byte floating point value
-    FLOAT = 11,
+    FLOAT,
 
     // 8-byte floating point value
-    DOUBLE = 12,
+    DOUBLE,
 
     // UTF8 variable-length string as List<Char>
-    STRING = 13,
+    STRING,
 
     // Variable-length bytes (no guarantee of UTF8-ness)
-    BINARY = 14,
+    BINARY,
 
     // By default, int32 days since the UNIX epoch
-    DATE = 16,
+    DATE,
 
     // Exact timestamp encoded with int64 since UNIX epoch
     // Default unit millisecond
-    TIMESTAMP = 17,
+    TIMESTAMP,
 
     // Exact time encoded with int64, default unit millisecond
-    TIME = 18,
+    TIME,
 
     // YEAR_MONTH or DAY_TIME interval in SQL style
-    INTERVAL = 19,
+    INTERVAL,
 
     // Precision- and scale-based decimal type. Storage type depends on the
     // parameters.
-    DECIMAL = 20,
+    DECIMAL,
 
     // A list of some logical data type
-    LIST = 30,
+    LIST,
 
     // Struct of logical types
-    STRUCT = 31,
+    STRUCT,
 
     // Unions of logical types
-    UNION = 32,
+    UNION,
 
-    // Timestamp as double seconds since the UNIX epoch
-    TIMESTAMP_DOUBLE = 33,
-
-    // Decimal value encoded as a text string
-    DECIMAL_TEXT = 34,
+    // Dictionary aka Category type
+    DICTIONARY
   };
 };
 
@@ -115,6 +112,34 @@ class BufferDescr {
   int bit_width_;
 };
 
+class TypeVisitor {
+ public:
+  virtual Status Visit(const NullType& type) = 0;
+  virtual Status Visit(const BooleanType& type) = 0;
+  virtual Status Visit(const Int8Type& type) = 0;
+  virtual Status Visit(const Int16Type& type) = 0;
+  virtual Status Visit(const Int32Type& type) = 0;
+  virtual Status Visit(const Int64Type& type) = 0;
+  virtual Status Visit(const UInt8Type& type) = 0;
+  virtual Status Visit(const UInt16Type& type) = 0;
+  virtual Status Visit(const UInt32Type& type) = 0;
+  virtual Status Visit(const UInt64Type& type) = 0;
+  virtual Status Visit(const HalfFloatType& type) = 0;
+  virtual Status Visit(const FloatType& type) = 0;
+  virtual Status Visit(const DoubleType& type) = 0;
+  virtual Status Visit(const StringType& type) = 0;
+  virtual Status Visit(const BinaryType& type) = 0;
+  virtual Status Visit(const DateType& type) = 0;
+  virtual Status Visit(const TimeType& type) = 0;
+  virtual Status Visit(const TimestampType& type) = 0;
+  virtual Status Visit(const IntervalType& type) = 0;
+  virtual Status Visit(const DecimalType& type) = 0;
+  virtual Status Visit(const ListType& type) = 0;
+  virtual Status Visit(const StructType& type) = 0;
+  virtual Status Visit(const UnionType& type) = 0;
+  virtual Status Visit(const DictionaryType& type) = 0;
+};
+
 struct ARROW_EXPORT DataType {
   Type::type type;
 
@@ -128,10 +153,10 @@ struct ARROW_EXPORT DataType {
   //
   // Types that are logically convertable from one to another e.g. List<UInt8>
   // and Binary are NOT equal).
-  virtual bool Equals(const DataType* other) const;
+  virtual bool Equals(const DataType& other) const;
 
   bool Equals(const std::shared_ptr<DataType>& other) const {
-    return Equals(other.get());
+    return Equals(*other.get());
   }
 
   std::shared_ptr<Field> child(int i) const { return children_[i]; }
@@ -189,16 +214,9 @@ struct ARROW_EXPORT Field {
       : name(name), type(type), nullable(nullable), dictionary(dictionary) {}
 
   bool operator==(const Field& other) const { return this->Equals(other); }
-
   bool operator!=(const Field& other) const { return !this->Equals(other); }
-
-  bool Equals(const Field& other) const {
-    return (this == &other) ||
-           (this->name == other.name && this->nullable == other.nullable &&
-               this->dictionary == dictionary && this->type->Equals(other.type.get()));
-  }
-
-  bool Equals(const std::shared_ptr<Field>& other) const { return Equals(*other.get()); }
+  bool Equals(const Field& other) const;
+  bool Equals(const std::shared_ptr<Field>& other) const;
 
   std::string ToString() const;
 };
@@ -414,6 +432,9 @@ struct ARROW_EXPORT UnionType : public DataType {
   std::vector<uint8_t> type_ids;
 };
 
+// ----------------------------------------------------------------------
+// Date and time types
+
 struct ARROW_EXPORT DateType : public FixedWidthType {
   static constexpr Type::type type_id = Type::DATE;
 
@@ -488,6 +509,35 @@ struct ARROW_EXPORT IntervalType : public FixedWidthType {
   static std::string name() { return "date"; }
 };
 
+// ----------------------------------------------------------------------
+// DictionaryType (for categorical or dictionary-encoded data)
+
+class ARROW_EXPORT DictionaryType : public FixedWidthType {
+ public:
+  static constexpr Type::type type_id = Type::DICTIONARY;
+
+  DictionaryType(const std::shared_ptr<DataType>& index_type,
+      const std::shared_ptr<Array>& dictionary);
+
+  int bit_width() const override;
+
+  std::shared_ptr<DataType> index_type() const { return index_type_; }
+
+  std::shared_ptr<Array> dictionary() const;
+
+  bool Equals(const DataType& other) const override;
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override;
+
+ private:
+  // Must be an integer type (not currently checked)
+  std::shared_ptr<DataType> index_type_;
+
+  std::shared_ptr<Array> dictionary_;
+};
+
+// ----------------------------------------------------------------------
 // Factory functions
 
 std::shared_ptr<DataType> ARROW_EXPORT null();
@@ -520,9 +570,44 @@ std::shared_ptr<DataType> ARROW_EXPORT union_(
     const std::vector<std::shared_ptr<Field>>& child_fields,
     const std::vector<uint8_t>& type_ids, UnionMode mode = UnionMode::SPARSE);
 
+std::shared_ptr<DataType> ARROW_EXPORT dictionary(
+    const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& values);
+
 std::shared_ptr<Field> ARROW_EXPORT field(const std::string& name,
     const std::shared_ptr<DataType>& type, bool nullable = true, int64_t dictionary = 0);
 
+// ----------------------------------------------------------------------
+//
+
+static inline bool is_integer(Type::type type_id) {
+  switch (type_id) {
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::UINT64:
+    case Type::INT64:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
+static inline bool is_floating(Type::type type_id) {
+  switch (type_id) {
+    case Type::HALF_FLOAT:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPE_H
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index a14c535b9b3f1..334abef664426 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -32,6 +32,9 @@ class MemoryPool;
 class RecordBatch;
 class Schema;
 
+class DictionaryType;
+class DictionaryArray;
+
 struct NullType;
 class NullArray;
 
@@ -101,60 +104,6 @@ using TimestampBuilder = NumericBuilder<TimestampType>;
 struct IntervalType;
 using IntervalArray = NumericArray<IntervalType>;
 
-class TypeVisitor {
- public:
-  virtual Status Visit(const NullType& type) = 0;
-  virtual Status Visit(const BooleanType& type) = 0;
-  virtual Status Visit(const Int8Type& type) = 0;
-  virtual Status Visit(const Int16Type& type) = 0;
-  virtual Status Visit(const Int32Type& type) = 0;
-  virtual Status Visit(const Int64Type& type) = 0;
-  virtual Status Visit(const UInt8Type& type) = 0;
-  virtual Status Visit(const UInt16Type& type) = 0;
-  virtual Status Visit(const UInt32Type& type) = 0;
-  virtual Status Visit(const UInt64Type& type) = 0;
-  virtual Status Visit(const HalfFloatType& type) = 0;
-  virtual Status Visit(const FloatType& type) = 0;
-  virtual Status Visit(const DoubleType& type) = 0;
-  virtual Status Visit(const StringType& type) = 0;
-  virtual Status Visit(const BinaryType& type) = 0;
-  virtual Status Visit(const DateType& type) = 0;
-  virtual Status Visit(const TimeType& type) = 0;
-  virtual Status Visit(const TimestampType& type) = 0;
-  virtual Status Visit(const IntervalType& type) = 0;
-  virtual Status Visit(const DecimalType& type) = 0;
-  virtual Status Visit(const ListType& type) = 0;
-  virtual Status Visit(const StructType& type) = 0;
-  virtual Status Visit(const UnionType& type) = 0;
-};
-
-class ArrayVisitor {
- public:
-  virtual Status Visit(const NullArray& array) = 0;
-  virtual Status Visit(const BooleanArray& array) = 0;
-  virtual Status Visit(const Int8Array& array) = 0;
-  virtual Status Visit(const Int16Array& array) = 0;
-  virtual Status Visit(const Int32Array& array) = 0;
-  virtual Status Visit(const Int64Array& array) = 0;
-  virtual Status Visit(const UInt8Array& array) = 0;
-  virtual Status Visit(const UInt16Array& array) = 0;
-  virtual Status Visit(const UInt32Array& array) = 0;
-  virtual Status Visit(const UInt64Array& array) = 0;
-  virtual Status Visit(const HalfFloatArray& array) = 0;
-  virtual Status Visit(const FloatArray& array) = 0;
-  virtual Status Visit(const DoubleArray& array) = 0;
-  virtual Status Visit(const StringArray& array) = 0;
-  virtual Status Visit(const BinaryArray& array) = 0;
-  virtual Status Visit(const DateArray& array) = 0;
-  virtual Status Visit(const TimeArray& array) = 0;
-  virtual Status Visit(const TimestampArray& array) = 0;
-  virtual Status Visit(const IntervalArray& array) = 0;
-  virtual Status Visit(const DecimalArray& array) = 0;
-  virtual Status Visit(const ListArray& array) = 0;
-  virtual Status Visit(const StructArray& array) = 0;
-  virtual Status Visit(const UnionArray& array) = 0;
-};
-
 }  // namespace arrow
 
 #endif  // ARROW_TYPE_FWD_H
diff --git a/format/Message.fbs b/format/Message.fbs
index d07d0666dce87..b2c64649f2687 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -256,7 +256,7 @@ table RecordBatch {
 /// For sending dictionary encoding information. Any Field can be
 /// dictionary-encoded, but in this case none of its children may be
 /// dictionary-encoded.
-/// There is one dictionary batch per dictionary
+/// There is one vector / column per dictionary
 ///
 
 table DictionaryBatch {
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 40fb60de07ed3..3cdfe49a4e7a7 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -55,7 +55,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CDataType" arrow::DataType":
         Type type
 
-        c_bool Equals(const CDataType* other)
+        c_bool Equals(const shared_ptr[CDataType]& other)
+        c_bool Equals(const CDataType& other)
 
         c_string ToString()
 
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index b4d127c871e09..d9e121dd8853e 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -98,7 +98,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         # TODO: Some default arguments are missing
         @staticmethod
         unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
-        const FileMetaData* metadata();
+        shared_ptr[FileMetaData] metadata();
 
 
 cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 7379456feef2b..c0921859e440d 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -98,8 +98,8 @@ cdef class ParquetReader:
             Integer index of the position of the column
         """
         cdef:
-            const FileMetaData* metadata = (self.reader.get()
-                                            .parquet_reader().metadata())
+            const FileMetaData* metadata = (self.reader.get().parquet_reader()
+                                            .metadata().get())
             int i = 0
 
         if self.column_idx_map is None:
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 7a69b0f12391a..d91ae7cb3b193 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -45,9 +45,9 @@ cdef class DataType:
 
     def __richcmp__(DataType self, DataType other, int op):
         if op == cpython.Py_EQ:
-            return self.type.Equals(other.type)
+            return self.type.Equals(other.sp_type)
         elif op == cpython.Py_NE:
-            return not self.type.Equals(other.type)
+            return not self.type.Equals(other.sp_type)
         else:
             raise TypeError('Invalid comparison')
 

From 7d1f1cf91b798259de18ebd772b213e12a6dd194 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 7 Jan 2017 16:16:31 -0500
Subject: [PATCH 0263/1644] ARROW-360: C++: Add method to shrink PoolBuffer
 using realloc

Added no explicit unit test for this as I want to keep the freedom to
the allocator in the future to advise the PoolBuffer on an acceptable
minimal size. In some cases it might be worth it to occupy a whole
page.

Resizing to a smaller size is tested though, so we already have a unit test ensuring that this code runs smoothly.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #272 from xhochy/ARROW-360 and squashes the following commits:

f4992ee [Uwe L. Korn] Adjust DCHECK for zero size arrays
040d3b4 [Uwe L. Korn] ARROW-360: C++: Add method to shrink PoolBuffer using realloc
---
 cpp/src/arrow/buffer.cc   | 20 +++++++++++++++++++-
 cpp/src/arrow/test-util.h |  2 +-
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 6d55f88af1e32..2e64ffd75c263 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -92,7 +92,25 @@ Status PoolBuffer::Reserve(int64_t new_capacity) {
 }
 
 Status PoolBuffer::Resize(int64_t new_size) {
-  RETURN_NOT_OK(Reserve(new_size));
+  if (new_size > size_) {
+    RETURN_NOT_OK(Reserve(new_size));
+  } else {
+    // Buffer is not growing, so shrink to the requested size without
+    // excess space.
+    if (capacity_ != new_size) {
+      // Buffer hasn't got yet the requested size.
+      if (new_size == 0) {
+        pool_->Free(mutable_data_, capacity_);
+        capacity_ = 0;
+        mutable_data_ = nullptr;
+        data_ = nullptr;
+      } else {
+        RETURN_NOT_OK(pool_->Reallocate(capacity_, new_size, &mutable_data_));
+        data_ = mutable_data_;
+        capacity_ = new_size;
+      }
+    }
+  }
   size_ = new_size;
   return Status::OK();
 }
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index e5957490e0733..f2da824084775 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -184,7 +184,7 @@ static inline void random_ascii(int n, uint32_t seed, uint8_t* out) {
 
 template <typename T>
 void rand_uniform_int(int n, uint32_t seed, T min_value, T max_value, T* out) {
-  DCHECK(out);
+  DCHECK(out || (n == 0));
   std::mt19937 gen(seed);
   std::uniform_int_distribution<T> d(min_value, max_value);
   for (int i = 0; i < n; ++i) {

From 1094d89d4094ab3209c2df15826d8e7d3758df97 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 7 Jan 2017 16:29:46 -0500
Subject: [PATCH 0264/1644] ARROW-463: C++: Support jemalloc 4.x

This also fixes some minor bugs in the CMakeLists for jemalloc.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #273 from xhochy/ARROW-463 and squashes the following commits:

d12a97c [Uwe L. Korn] ARROW-463: C++: Support jemalloc 4.x
---
 cpp/CMakeLists.txt                    | 4 +++-
 cpp/src/arrow/jemalloc/memory_pool.cc | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 419691b4b68b2..3522e5c5a7640 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -244,7 +244,7 @@ endfunction()
 
 # A wrapper for target_link_libraries() that is compatible with NO_BENCHMARKS.
 function(ARROW_BENCHMARK_LINK_LIBRARIES REL_BENCHMARK_NAME)
-  if(NO_TESTS)
+    if(NO_BENCHMARKS)
     return()
   endif()
   get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
@@ -595,6 +595,8 @@ include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
 
 if (ARROW_JEMALLOC)
   find_package(jemalloc REQUIRED)
+
+  include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(jemalloc
       SHARED_LIB ${JEMALLOC_SHARED_LIB})
 endif()
diff --git a/cpp/src/arrow/jemalloc/memory_pool.cc b/cpp/src/arrow/jemalloc/memory_pool.cc
index acc09c7cd7587..c568316711717 100644
--- a/cpp/src/arrow/jemalloc/memory_pool.cc
+++ b/cpp/src/arrow/jemalloc/memory_pool.cc
@@ -19,6 +19,8 @@
 
 #include <sstream>
 
+// Needed to support jemalloc 3 and 4
+#define JEMALLOC_MANGLE
 #include <jemalloc/jemalloc.h>
 
 #include "arrow/status.h"

From 3195948f64770520c7ed4c8a7888b33402ad6519 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 8 Jan 2017 10:50:30 -0500
Subject: [PATCH 0265/1644] ARROW-438: [C++/Python] Implement zero-data-copy
 record batch and table concatenation.

This also fixes a bug in ChunkedArray::Equals. This is caught by the Python test suite but would benefit from more C++ unit tests.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #274 from wesm/ARROW-438 and squashes the following commits:

1f39568 [Wes McKinney] py3 compatibility
2e76c5e [Wes McKinney] Implement arrow::ConcatenateTables and Python wrapper. Fix bug in ChunkedArray::Equals
f3cb170 [Wes McKinney] Fix Cython compilation, verify pyarrow.Table.from_batches still works
af28755 [Wes McKinney] Implement Table::FromRecordBatches

Change-Id: I948b61d848c178edefad63465a74d9c303ad1f18
---
 cpp/CMakeLists.txt                          |   2 +-
 cpp/src/arrow/column.cc                     |  11 +-
 cpp/src/arrow/column.h                      |   2 +
 cpp/src/arrow/io/io-file-test.cc            |   1 -
 cpp/src/arrow/table-test.cc                 |  88 ++++++++++--
 cpp/src/arrow/table.cc                      |  71 ++++++++++
 cpp/src/arrow/table.h                       |  13 +-
 cpp/src/arrow/test-util.h                   |  43 +++---
 python/CMakeLists.txt                       |   3 +
 python/benchmarks/array.py                  |   7 +-
 python/doc/pandas.rst                       |   5 +-
 python/pyarrow/__init__.py                  |   2 +-
 python/pyarrow/array.pyx                    |  25 ++++
 python/pyarrow/includes/libarrow.pxd        |  16 +++
 python/pyarrow/table.pyx                    | 147 ++++++++++++++------
 python/pyarrow/tests/test_convert_pandas.py |   6 +-
 python/pyarrow/tests/test_parquet.py        |  12 +-
 python/pyarrow/tests/test_table.py          |  27 ++++
 python/setup.py                             |   5 +-
 19 files changed, 395 insertions(+), 91 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 3522e5c5a7640..87b7841ece52e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -76,7 +76,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
 
   option(ARROW_JEMALLOC
     "Build the Arrow jemalloc-based allocator"
-    ON)
+    OFF)
 
   option(ARROW_BOOST_USE_SHARED
     "Rely on boost shared libraries where relevant"
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index 3e899563e2cbe..9cc0f579dc5bd 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -45,7 +45,9 @@ bool ChunkedArray::Equals(const ChunkedArray& other) const {
   int32_t this_start_idx = 0;
   int other_chunk_idx = 0;
   int32_t other_start_idx = 0;
-  while (this_chunk_idx < static_cast<int32_t>(chunks_.size())) {
+
+  int64_t elements_compared = 0;
+  while (elements_compared < length_) {
     const std::shared_ptr<Array> this_array = chunks_[this_chunk_idx];
     const std::shared_ptr<Array> other_array = other.chunk(other_chunk_idx);
     int32_t common_length = std::min(
@@ -55,14 +57,21 @@ bool ChunkedArray::Equals(const ChunkedArray& other) const {
       return false;
     }
 
+    elements_compared += common_length;
+
     // If we have exhausted the current chunk, proceed to the next one individually.
     if (this_start_idx + common_length == this_array->length()) {
       this_chunk_idx++;
       this_start_idx = 0;
+    } else {
+      this_start_idx += common_length;
     }
+
     if (other_start_idx + common_length == other_array->length()) {
       other_chunk_idx++;
       other_start_idx = 0;
+    } else {
+      other_start_idx += common_length;
     }
   }
   return true;
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
index f71647381743c..a28b2665e9c1c 100644
--- a/cpp/src/arrow/column.h
+++ b/cpp/src/arrow/column.h
@@ -48,6 +48,8 @@ class ARROW_EXPORT ChunkedArray {
 
   std::shared_ptr<Array> chunk(int i) const { return chunks_[i]; }
 
+  const ArrayVector& chunks() const { return chunks_; }
+
   bool Equals(const ChunkedArray& other) const;
   bool Equals(const std::shared_ptr<ChunkedArray>& other) const;
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 378b60e782124..821e71d0212f6 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -301,7 +301,6 @@ class MyMemoryPool : public MemoryPool {
       return Status::OutOfMemory(ss.str());
     }
 
-
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 734b94125defc..67c9f6703f496 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -44,16 +44,20 @@ class TestTable : public TestBase {
     vector<shared_ptr<Field>> fields = {f0, f1, f2};
     schema_ = std::make_shared<Schema>(fields);
 
-    columns_ = {
-        std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length)),
-        std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
-        std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length))};
+    arrays_ = {MakePrimitive<Int32Array>(length), MakePrimitive<UInt8Array>(length),
+        MakePrimitive<Int16Array>(length)};
+
+    columns_ = {std::make_shared<Column>(schema_->field(0), arrays_[0]),
+        std::make_shared<Column>(schema_->field(1), arrays_[1]),
+        std::make_shared<Column>(schema_->field(2), arrays_[2])};
   }
 
  protected:
   std::shared_ptr<Table> table_;
   shared_ptr<Schema> schema_;
-  vector<std::shared_ptr<Column>> columns_;
+
+  std::vector<std::shared_ptr<Array>> arrays_;
+  std::vector<std::shared_ptr<Column>> columns_;
 };
 
 TEST_F(TestTable, EmptySchema) {
@@ -65,7 +69,7 @@ TEST_F(TestTable, EmptySchema) {
 }
 
 TEST_F(TestTable, Ctors) {
-  int length = 100;
+  const int length = 100;
   MakeExample1(length);
 
   std::string name = "data";
@@ -83,7 +87,7 @@ TEST_F(TestTable, Ctors) {
 }
 
 TEST_F(TestTable, Metadata) {
-  int length = 100;
+  const int length = 100;
   MakeExample1(length);
 
   std::string name = "data";
@@ -98,7 +102,7 @@ TEST_F(TestTable, Metadata) {
 
 TEST_F(TestTable, InvalidColumns) {
   // Check that columns are all the same length
-  int length = 100;
+  const int length = 100;
   MakeExample1(length);
 
   table_.reset(new Table("data", schema_, columns_, length - 1));
@@ -120,7 +124,7 @@ TEST_F(TestTable, InvalidColumns) {
 }
 
 TEST_F(TestTable, Equals) {
-  int length = 100;
+  const int length = 100;
   MakeExample1(length);
 
   std::string name = "data";
@@ -145,6 +149,72 @@ TEST_F(TestTable, Equals) {
   ASSERT_FALSE(table_->Equals(std::make_shared<Table>(name, schema_, other_columns)));
 }
 
+TEST_F(TestTable, FromRecordBatches) {
+  const int32_t length = 10;
+  MakeExample1(length);
+
+  auto batch1 = std::make_shared<RecordBatch>(schema_, length, arrays_);
+
+  std::shared_ptr<Table> result, expected;
+  ASSERT_OK(Table::FromRecordBatches("foo", {batch1}, &result));
+
+  expected = std::make_shared<Table>("foo", schema_, columns_);
+  ASSERT_TRUE(result->Equals(expected));
+
+  std::vector<std::shared_ptr<Column>> other_columns;
+  for (int i = 0; i < schema_->num_fields(); ++i) {
+    std::vector<std::shared_ptr<Array>> col_arrays = {arrays_[i], arrays_[i]};
+    other_columns.push_back(std::make_shared<Column>(schema_->field(i), col_arrays));
+  }
+
+  ASSERT_OK(Table::FromRecordBatches("foo", {batch1, batch1}, &result));
+  expected = std::make_shared<Table>("foo", schema_, other_columns);
+  ASSERT_TRUE(result->Equals(expected));
+
+  // Error states
+  std::vector<std::shared_ptr<RecordBatch>> empty_batches;
+  ASSERT_RAISES(Invalid, Table::FromRecordBatches("", empty_batches, &result));
+
+  std::vector<std::shared_ptr<Field>> fields = {schema_->field(0), schema_->field(1)};
+  auto other_schema = std::make_shared<Schema>(fields);
+
+  std::vector<std::shared_ptr<Array>> other_arrays = {arrays_[0], arrays_[1]};
+  auto batch2 = std::make_shared<RecordBatch>(other_schema, length, other_arrays);
+  ASSERT_RAISES(Invalid, Table::FromRecordBatches("", {batch1, batch2}, &result));
+}
+
+TEST_F(TestTable, ConcatenateTables) {
+  const int32_t length = 10;
+
+  MakeExample1(length);
+  auto batch1 = std::make_shared<RecordBatch>(schema_, length, arrays_);
+
+  // generate different data
+  MakeExample1(length);
+  auto batch2 = std::make_shared<RecordBatch>(schema_, length, arrays_);
+
+  std::shared_ptr<Table> t1, t2, t3, result, expected;
+  ASSERT_OK(Table::FromRecordBatches("foo", {batch1}, &t1));
+  ASSERT_OK(Table::FromRecordBatches("foo", {batch2}, &t2));
+
+  ASSERT_OK(ConcatenateTables("bar", {t1, t2}, &result));
+  ASSERT_OK(Table::FromRecordBatches("bar", {batch1, batch2}, &expected));
+  ASSERT_TRUE(result->Equals(expected));
+
+  // Error states
+  std::vector<std::shared_ptr<Table>> empty_tables;
+  ASSERT_RAISES(Invalid, ConcatenateTables("", empty_tables, &result));
+
+  std::vector<std::shared_ptr<Field>> fields = {schema_->field(0), schema_->field(1)};
+  auto other_schema = std::make_shared<Schema>(fields);
+
+  std::vector<std::shared_ptr<Array>> other_arrays = {arrays_[0], arrays_[1]};
+  auto batch3 = std::make_shared<RecordBatch>(other_schema, length, other_arrays);
+  ASSERT_OK(Table::FromRecordBatches("", {batch3}, &t3));
+
+  ASSERT_RAISES(Invalid, ConcatenateTables("foo", {t1, t3}, &result));
+}
+
 class TestRecordBatch : public TestBase {};
 
 TEST_F(TestRecordBatch, Equals) {
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 45f672ec8928b..b3563eaae7b57 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -77,6 +77,77 @@ Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
     const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
     : name_(name), schema_(schema), columns_(columns), num_rows_(num_rows) {}
 
+Status Table::FromRecordBatches(const std::string& name,
+    const std::vector<std::shared_ptr<RecordBatch>>& batches,
+    std::shared_ptr<Table>* table) {
+  if (batches.size() == 0) {
+    return Status::Invalid("Must pass at least one record batch");
+  }
+
+  std::shared_ptr<Schema> schema = batches[0]->schema();
+
+  const int nbatches = static_cast<int>(batches.size());
+  const int ncolumns = static_cast<int>(schema->num_fields());
+
+  for (int i = 1; i < nbatches; ++i) {
+    if (!batches[i]->schema()->Equals(schema)) {
+      std::stringstream ss;
+      ss << "Schema at index " << static_cast<int>(i) << " was different: \n"
+         << schema->ToString() << "\nvs\n"
+         << batches[i]->schema()->ToString();
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  std::vector<std::shared_ptr<Column>> columns(ncolumns);
+  std::vector<std::shared_ptr<Array>> column_arrays(nbatches);
+
+  for (int i = 0; i < ncolumns; ++i) {
+    for (int j = 0; j < nbatches; ++j) {
+      column_arrays[j] = batches[j]->column(i);
+    }
+    columns[i] = std::make_shared<Column>(schema->field(i), column_arrays);
+  }
+
+  *table = std::make_shared<Table>(name, schema, columns);
+  return Status::OK();
+}
+
+Status ConcatenateTables(const std::string& output_name,
+    const std::vector<std::shared_ptr<Table>>& tables, std::shared_ptr<Table>* table) {
+  if (tables.size() == 0) { return Status::Invalid("Must pass at least one table"); }
+
+  std::shared_ptr<Schema> schema = tables[0]->schema();
+
+  const int ntables = static_cast<int>(tables.size());
+  const int ncolumns = static_cast<int>(schema->num_fields());
+
+  for (int i = 1; i < ntables; ++i) {
+    if (!tables[i]->schema()->Equals(schema)) {
+      std::stringstream ss;
+      ss << "Schema at index " << static_cast<int>(i) << " was different: \n"
+         << schema->ToString() << "\nvs\n"
+         << tables[i]->schema()->ToString();
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  std::vector<std::shared_ptr<Column>> columns(ncolumns);
+  for (int i = 0; i < ncolumns; ++i) {
+    std::vector<std::shared_ptr<Array>> column_arrays;
+    for (int j = 0; j < ntables; ++j) {
+      const std::vector<std::shared_ptr<Array>>& chunks =
+          tables[j]->column(i)->data()->chunks();
+      for (const auto& chunk : chunks) {
+        column_arrays.push_back(chunk);
+      }
+    }
+    columns[i] = std::make_shared<Column>(schema->field(i), column_arrays);
+  }
+  *table = std::make_shared<Table>(output_name, schema, columns);
+  return Status::OK();
+}
+
 bool Table::Equals(const Table& other) const {
   if (name_ != other.name()) { return false; }
   if (!schema_->Equals(other.schema())) { return false; }
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 0f2418d0e7900..583847cfbe3e5 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -82,7 +82,13 @@ class ARROW_EXPORT Table {
   // same length as num_rows -- you can validate this using
   // Table::ValidateColumns
   Table(const std::string& name, const std::shared_ptr<Schema>& schema,
-      const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows);
+      const std::vector<std::shared_ptr<Column>>& columns, int64_t nubm_rows);
+
+  // Construct table from RecordBatch, but only if all of the batch schemas are
+  // equal. Returns Status::Invalid if there is some problem
+  static Status FromRecordBatches(const std::string& name,
+      const std::vector<std::shared_ptr<RecordBatch>>& batches,
+      std::shared_ptr<Table>* table);
 
   // @returns: the table's name, if any (may be length 0)
   const std::string& name() const { return name_; }
@@ -116,6 +122,11 @@ class ARROW_EXPORT Table {
   int64_t num_rows_;
 };
 
+// Construct table from multiple input tables. Return Status::Invalid if
+// schemas are not equal
+Status ARROW_EXPORT ConcatenateTables(const std::string& output_name,
+    const std::vector<std::shared_ptr<Table>>& tables, std::shared_ptr<Table>* table);
+
 }  // namespace arrow
 
 #endif  // ARROW_TABLE_H
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index f2da824084775..b59809d9e48e6 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -71,23 +71,6 @@
 
 namespace arrow {
 
-class TestBase : public ::testing::Test {
- public:
-  void SetUp() { pool_ = default_memory_pool(); }
-
-  template <typename ArrayType>
-  std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
-    auto data = std::make_shared<PoolBuffer>(pool_);
-    auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
-    EXPECT_OK(data->Resize(length * sizeof(typename ArrayType::value_type)));
-    EXPECT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
-    return std::make_shared<ArrayType>(length, data, null_count, null_bitmap);
-  }
-
- protected:
-  MemoryPool* pool_;
-};
-
 namespace test {
 
 template <typename T>
@@ -253,6 +236,32 @@ Status MakeRandomBytePoolBuffer(int32_t length, MemoryPool* pool,
 
 }  // namespace test
 
+class TestBase : public ::testing::Test {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+    random_seed_ = 0;
+  }
+
+  template <typename ArrayType>
+  std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
+    auto data = std::make_shared<PoolBuffer>(pool_);
+    const int64_t data_nbytes = length * sizeof(typename ArrayType::value_type);
+    EXPECT_OK(data->Resize(data_nbytes));
+
+    // Fill with random data
+    test::random_bytes(data_nbytes, random_seed_++, data->mutable_data());
+
+    auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
+    EXPECT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
+    return std::make_shared<ArrayType>(length, data, null_count, null_bitmap);
+  }
+
+ protected:
+  uint32_t random_seed_;
+  MemoryPool* pool_;
+};
+
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::shared_ptr<DataType>& type,
     const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6c2477235faaa..e42c45d3f5cc9 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -70,6 +70,9 @@ include(SetupCxxFlags)
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
 
+# Suppress Cython warnings
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
+
 # Determine compiler version
 include(CompilerInfo)
 
diff --git a/python/benchmarks/array.py b/python/benchmarks/array.py
index 4268f0073f292..e22c0f7fc9e70 100644
--- a/python/benchmarks/array.py
+++ b/python/benchmarks/array.py
@@ -49,10 +49,10 @@ class PandasConversionsToArrow(PandasConversionsBase):
     params = ((1, 10 ** 5, 10 ** 6, 10 ** 7), ('int64', 'float64', 'float64_nans', 'str'))
 
     def time_from_series(self, n, dtype):
-        A.from_pandas_dataframe(self.data)
+        A.Table.from_pandas(self.data)
 
     def peakmem_from_series(self, n, dtype):
-        A.from_pandas_dataframe(self.data)
+        A.Table.from_pandas(self.data)
 
 
 class PandasConversionsFromArrow(PandasConversionsBase):
@@ -61,7 +61,7 @@ class PandasConversionsFromArrow(PandasConversionsBase):
 
     def setup(self, n, dtype):
         super(PandasConversionsFromArrow, self).setup(n, dtype)
-        self.arrow_data = A.from_pandas_dataframe(self.data)
+        self.arrow_data = A.Table.from_pandas(self.data)
 
     def time_to_series(self, n, dtype):
         self.arrow_data.to_pandas()
@@ -80,4 +80,3 @@ def setUp(self, n):
     def time_as_py(self, n):
         for i in range(n):
             self._array[i].as_py()
-
diff --git a/python/doc/pandas.rst b/python/doc/pandas.rst
index 7c70074817835..c225d1362c7b6 100644
--- a/python/doc/pandas.rst
+++ b/python/doc/pandas.rst
@@ -31,7 +31,7 @@ represent more data than a DataFrame, so a full conversion is not always possibl
 
 Conversion from a Table to a DataFrame is done by calling
 :meth:`pyarrow.table.Table.to_pandas`. The inverse is then achieved by using
-:meth:`pyarrow.from_pandas_dataframe`. This conversion routine provides the
+:meth:`pyarrow.Table.from_pandas`. This conversion routine provides the
 convience parameter ``timestamps_to_ms``. Although Arrow supports timestamps of
 different resolutions, Pandas only supports nanosecond timestamps and most
 other systems (e.g. Parquet) only work on millisecond timestamps. This parameter
@@ -45,7 +45,7 @@ conversion.
 
     df = pd.DataFrame({"a": [1, 2, 3]})
     # Convert from Pandas to Arrow
-    table = pa.from_pandas_dataframe(df)
+    table = pa.Table.from_pandas(df)
     # Convert back to Pandas
     df_new = table.to_pandas()
 
@@ -111,4 +111,3 @@ Arrow -> Pandas Conversion
 +-------------------------------------+--------------------------------------------------------+
 | ``TIMESTAMP(unit=*)``               | ``pd.Timestamp`` (``np.datetime64[ns]``)               |
 +-------------------------------------+--------------------------------------------------------+
-
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 02b2b06237de3..d25cdd47ea974 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -56,4 +56,4 @@
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
-from pyarrow.table import Column, RecordBatch, Table, from_pandas_dataframe
+from pyarrow.table import Column, RecordBatch, Table, concat_tables
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index c178d5ccd355b..266768f7e0ba5 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -91,6 +91,29 @@ cdef class Array:
         """
         return from_pandas_series(obj, mask)
 
+    @staticmethod
+    def from_list(object list_obj, DataType type=None):
+        """
+        Convert Python list to Arrow array
+
+        Parameters
+        ----------
+        list_obj : array_like
+
+        Returns
+        -------
+        pyarrow.array.Array
+        """
+        cdef:
+            shared_ptr[CArray] sp_array
+
+        if type is None:
+            check_status(pyarrow.ConvertPySequence(list_obj, &sp_array))
+        else:
+            raise NotImplementedError()
+
+        return box_arrow_array(sp_array)
+
     property null_count:
 
         def __get__(self):
@@ -348,3 +371,5 @@ cdef object series_as_ndarray(object obj):
         result = obj
 
     return result
+
+from_pylist = Array.from_list
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 3cdfe49a4e7a7..b0f971d516ce5 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -182,6 +182,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CColumn(const shared_ptr[CField]& field,
                 const vector[shared_ptr[CArray]]& chunks)
 
+        c_bool Equals(const CColumn& other)
+        c_bool Equals(const shared_ptr[CColumn]& other)
+
         int64_t length()
         int64_t null_count()
         const c_string& name()
@@ -207,14 +210,27 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CTable(const c_string& name, const shared_ptr[CSchema]& schema,
                const vector[shared_ptr[CColumn]]& columns)
 
+        @staticmethod
+        CStatus FromRecordBatches(
+            const c_string& name,
+            const vector[shared_ptr[CRecordBatch]]& batches,
+            shared_ptr[CTable]* table)
+
         int num_columns()
         int num_rows()
 
+        c_bool Equals(const CTable& other)
+        c_bool Equals(const shared_ptr[CTable]& other)
+
         const c_string& name()
 
         shared_ptr[CSchema] schema()
         shared_ptr[CColumn] column(int i)
 
+    CStatus ConcatenateTables(const c_string& output_name,
+                              const vector[shared_ptr[CTable]]& tables,
+                              shared_ptr[CTable]* result)
+
 
 cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
     cdef cppclass SchemaMessage:
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 925543176c531..3a046516d961b 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -155,6 +155,31 @@ cdef class Column:
 
         return pd.Series(PyObject_to_object(arr), name=self.name)
 
+    def equals(self, Column other):
+        """
+        Check if contents of two columns are equal
+
+        Parameters
+        ----------
+        other : pyarrow.Column
+
+        Returns
+        -------
+        are_equal : boolean
+        """
+        cdef:
+            CColumn* my_col = self.column
+            CColumn* other_col = other.column
+            c_bool result
+
+        self._check_nullptr()
+        other._check_nullptr()
+
+        with nogil:
+            result = my_col.Equals(deref(other_col))
+
+        return result
+
     def to_pylist(self):
         """
         Convert to a list of native Python objects.
@@ -343,10 +368,18 @@ cdef class RecordBatch:
         return arr
 
     def equals(self, RecordBatch other):
+        cdef:
+            CRecordBatch* my_batch = self.batch
+            CRecordBatch* other_batch = other.batch
+            c_bool result
+
         self._check_nullptr()
         other._check_nullptr()
 
-        return self.batch.Equals(deref(other.batch))
+        with nogil:
+            result = my_batch.Equals(deref(other_batch))
+
+        return result
 
     def to_pydict(self):
         """
@@ -424,7 +457,6 @@ cdef class RecordBatch:
         """
         cdef:
             Array arr
-            RecordBatch result
             c_string c_name
             shared_ptr[CSchema] schema
             shared_ptr[CRecordBatch] batch
@@ -442,11 +474,7 @@ cdef class RecordBatch:
             c_arrays.push_back(arr.sp_array)
 
         batch.reset(new CRecordBatch(schema, num_rows, c_arrays))
-
-        result = RecordBatch()
-        result.init(batch)
-
-        return result
+        return batch_from_cbatch(batch)
 
 
 cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
@@ -498,6 +526,31 @@ cdef class Table:
             raise ReferenceError("Table object references a NULL pointer."
                     "Not initialized.")
 
+    def equals(self, Table other):
+        """
+        Check if contents of two tables are equal
+
+        Parameters
+        ----------
+        other : pyarrow.Table
+
+        Returns
+        -------
+        are_equal : boolean
+        """
+        cdef:
+            CTable* my_table = self.table
+            CTable* other_table = other.table
+            c_bool result
+
+        self._check_nullptr()
+        other._check_nullptr()
+
+        with nogil:
+            result = my_table.Equals(deref(other_table))
+
+        return result
+
     @classmethod
     def from_pandas(cls, df, name=None, timestamps_to_ms=False):
         """
@@ -527,7 +580,7 @@ cdef class Table:
             ...     'int': [1, 2],
             ...     'str': ['a', 'b']
             ... })
-        >>> pa.table.from_pandas_dataframe(df)
+        >>> pa.Table.from_pandas(df)
         <pyarrow.table.Table object at 0x7f05d1fb1b40>
         """
         names, arrays = _dataframe_to_arrays(df, name=name,
@@ -559,7 +612,6 @@ cdef class Table:
             c_string c_name
             vector[shared_ptr[CField]] fields
             vector[shared_ptr[CColumn]] columns
-            Table result
             shared_ptr[CSchema] schema
             shared_ptr[CTable] table
 
@@ -577,14 +629,10 @@ cdef class Table:
             c_name = tobytes(name)
 
         table.reset(new CTable(c_name, schema, columns))
-
-        result = Table()
-        result.init(table)
-
-        return result
+        return table_from_ctable(table)
 
     @staticmethod
-    def from_batches(batches):
+    def from_batches(batches, name=None):
         """
         Construct a Table from a list of Arrow RecordBatches
 
@@ -594,39 +642,21 @@ cdef class Table:
         batches: list of RecordBatch
             RecordBatch list to be converted, schemas must be equal
         """
-
         cdef:
-            vector[shared_ptr[CArray]] c_array_chunks
-            vector[shared_ptr[CColumn]] c_columns
+            vector[shared_ptr[CRecordBatch]] c_batches
             shared_ptr[CTable] c_table
-            Array arr
-            Schema schema
-
-        import pandas as pd
+            RecordBatch batch
+            Table table
+            c_string c_name
 
-        schema = batches[0].schema
+        c_name = b'' if name is None else tobytes(name)
 
-        # check schemas are equal
-        for other in batches[1:]:
-            if not schema.equals(other.schema):
-                raise ArrowException("Error converting list of RecordBatches "
-                        "to DataFrame, not all schemas are equal: {%s} != {%s}"
-                        % (str(schema), str(other.schema)))
+        for batch in batches:
+            c_batches.push_back(batch.sp_batch)
 
-        cdef int K = batches[0].num_columns
+        with nogil:
+            check_status(CTable.FromRecordBatches(c_name, c_batches, &c_table))
 
-        # create chunked columns from the batches
-        c_columns.resize(K)
-        for i in range(K):
-            for batch in batches:
-                arr = batch[i]
-                c_array_chunks.push_back(arr.sp_array)
-            c_columns[i].reset(new CColumn(schema.sp_schema.get().field(i),
-                               c_array_chunks))
-            c_array_chunks.clear()
-
-        # create a Table from columns and convert to DataFrame
-        c_table.reset(new CTable('', schema.sp_schema, c_columns))
         table = Table()
         table.init(c_table)
         return table
@@ -760,9 +790,40 @@ cdef class Table:
         return (self.num_rows, self.num_columns)
 
 
+def concat_tables(tables, output_name=None):
+    """
+    Perform zero-copy concatenation of pyarrow.Table objects. Raises exception
+    if all of the Table schemas are not the same
+
+    Parameters
+    ----------
+    tables : iterable of pyarrow.Table objects
+    output_name : string, default None
+      A name for the output table, if any
+    """
+    cdef:
+        vector[shared_ptr[CTable]] c_tables
+        shared_ptr[CTable] c_result
+        Table table
+        c_string c_name
+
+    c_name = b'' if output_name is None else tobytes(output_name)
+
+    for table in tables:
+        c_tables.push_back(table.sp_table)
+
+    with nogil:
+        check_status(ConcatenateTables(c_name, c_tables, &c_result))
+
+    return table_from_ctable(c_result)
+
+
 cdef api object table_from_ctable(const shared_ptr[CTable]& ctable):
     cdef Table table = Table()
     table.init(ctable)
     return table
 
-from_pandas_dataframe = Table.from_pandas
+cdef api object batch_from_cbatch(const shared_ptr[CRecordBatch]& cbatch):
+    cdef RecordBatch batch = RecordBatch()
+    batch.init(cbatch)
+    return batch
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index bb9f0b3eccab1..12e7a08d795a2 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -61,7 +61,7 @@ def tearDown(self):
 
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
                                 timestamps_to_ms=False):
-        table = A.from_pandas_dataframe(df, timestamps_to_ms=timestamps_to_ms)
+        table = A.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms)
         result = table.to_pandas(nthreads=nthreads)
         if expected is None:
             expected = df
@@ -193,7 +193,7 @@ def test_bytes_to_binary(self):
         values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan]
         df = pd.DataFrame({'strings': values})
 
-        table = A.from_pandas_dataframe(df)
+        table = A.Table.from_pandas(df)
         assert table[0].type == A.binary()
 
         values2 = [b'qux', b'foo', None, b'bar', b'qux', np.nan]
@@ -245,7 +245,7 @@ def test_date(self):
                      None,
                      datetime.date(1970, 1, 1),
                      datetime.date(2040, 2, 26)]})
-        table = A.from_pandas_dataframe(df)
+        table = A.Table.from_pandas(df)
         result = table.to_pandas()
         expected = df.copy()
         expected['date'] = pd.to_datetime(df['date'])
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 7c45732d34573..0fb913cc792d8 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -79,7 +79,7 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
         'empty_str': [''] * size
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.from_pandas_dataframe(df, timestamps_to_ms=True)
+    arrow_table = A.Table.from_pandas(df, timestamps_to_ms=True)
     A.parquet.write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
@@ -107,7 +107,7 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
         'empty_str': [''] * size
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.from_pandas_dataframe(df)
+    arrow_table = A.Table.from_pandas(df)
     A.parquet.write_table(arrow_table, filename.strpath, version="1.0")
     table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
@@ -126,7 +126,7 @@ def test_pandas_column_selection(tmpdir):
         'uint16': np.arange(size, dtype=np.uint16)
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.from_pandas_dataframe(df)
+    arrow_table = A.Table.from_pandas(df)
     A.parquet.write_table(arrow_table, filename.strpath)
     table_read = pq.read_table(filename.strpath, columns=['uint8'])
     df_read = table_read.to_pandas()
@@ -155,7 +155,7 @@ def _test_dataframe(size=10000):
 @parquet
 def test_pandas_parquet_native_file_roundtrip(tmpdir):
     df = _test_dataframe(10000)
-    arrow_table = A.from_pandas_dataframe(df)
+    arrow_table = A.Table.from_pandas(df)
     imos = paio.InMemoryOutputStream()
     pq.write_table(arrow_table, imos, version="2.0")
     buf = imos.get_result()
@@ -176,7 +176,7 @@ def test_pandas_parquet_pyfile_roundtrip(tmpdir):
         'strings': ['foo', 'bar', None, 'baz', 'qux']
     })
 
-    arrow_table = A.from_pandas_dataframe(df)
+    arrow_table = A.Table.from_pandas(df)
 
     with open(filename, 'wb') as f:
         A.parquet.write_table(arrow_table, f, version="1.0")
@@ -206,7 +206,7 @@ def test_pandas_parquet_configuration_options(tmpdir):
         'bool': np.random.randn(size) > 0
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.from_pandas_dataframe(df)
+    arrow_table = A.Table.from_pandas(df)
 
     for use_dictionary in [True, False]:
         A.parquet.write_table(
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 9985b3e29ada1..6f00c7391f66d 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -111,6 +111,33 @@ def test_table_basics():
             assert chunk is not None
 
 
+def test_concat_tables():
+    data = [
+        list(range(5)),
+        [-10., -5., 0., 5., 10.]
+    ]
+    data2 = [
+        list(range(5, 10)),
+        [1., 2., 3., 4., 5.]
+    ]
+
+    t1 = pa.Table.from_arrays(('a', 'b'), [pa.from_pylist(x)
+                                           for x in data], 'table_name')
+    t2 = pa.Table.from_arrays(('a', 'b'), [pa.from_pylist(x)
+                                           for x in data2], 'table_name')
+
+    result = pa.concat_tables([t1, t2], output_name='foo')
+    assert result.name == 'foo'
+    assert len(result) == 10
+
+    expected = pa.Table.from_arrays(
+        ('a', 'b'), [pa.from_pylist(x + y)
+                     for x, y in zip(data, data2)],
+        'foo')
+
+    assert result.equals(expected)
+
+
 def test_table_pandas():
     data = [
         pa.from_pylist(range(5)),
diff --git a/python/setup.py b/python/setup.py
index 2e595e2dc870e..3829a7982d670 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -143,7 +143,10 @@ def _run_cmake(self):
                              cmake_options + [source])
 
             self.spawn(cmake_command)
-            args = ['make', 'VERBOSE=1']
+            args = ['make']
+            if os.environ.get('PYARROW_BUILD_VERBOSE', '0') == '1':
+                args.append('VERBOSE=1')
+
             if 'PYARROW_PARALLEL' in os.environ:
                 args.append('-j{0}'.format(os.environ['PYARROW_PARALLEL']))
             self.spawn(args)

From f44b6a3b91a15461804dd7877840a557caa52e4e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 10 Jan 2017 08:44:01 +0100
Subject: [PATCH 0266/1644] ARROW-442: [Python] Inspect Parquet file metadata
 from Python

I also made the Cython parquet extension "private" so that higher level logic (e.g. upcoming handling of multiple files) can be handled in pure Python (which doesn't need to be compiled)

Requires PARQUET-828 for the test suite to pass.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #275 from wesm/ARROW-442 and squashes the following commits:

a4255a2 [Wes McKinney] Add row group metadata accessor, add smoke tests
75a11cf [Wes McKinney] Add more metadata accessor scaffolding, to be tested
e59ca40 [Wes McKinney] Move parquet Cython wrapper to a private import, add parquet.py for high level logic
---
 python/CMakeLists.txt                |   2 +-
 python/pyarrow/_parquet.pxd          | 217 +++++++++++
 python/pyarrow/_parquet.pyx          | 520 +++++++++++++++++++++++++++
 python/pyarrow/includes/parquet.pxd  | 147 --------
 python/pyarrow/parquet.py            | 116 ++++++
 python/pyarrow/parquet.pyx           | 244 -------------
 python/pyarrow/tests/test_parquet.py |  71 +++-
 python/setup.py                      |   4 +-
 8 files changed, 922 insertions(+), 399 deletions(-)
 create mode 100644 python/pyarrow/_parquet.pxd
 create mode 100644 python/pyarrow/_parquet.pyx
 delete mode 100644 python/pyarrow/includes/parquet.pxd
 create mode 100644 python/pyarrow/parquet.py
 delete mode 100644 python/pyarrow/parquet.pyx

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index e42c45d3f5cc9..45115d49d455e 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -428,7 +428,7 @@ if (PYARROW_BUILD_PARQUET)
     parquet_arrow)
   set(CYTHON_EXTENSIONS
     ${CYTHON_EXTENSIONS}
-    parquet)
+    _parquet)
 endif()
 
 add_library(pyarrow SHARED
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
new file mode 100644
index 0000000000000..faca845167d31
--- /dev/null
+++ b/python/pyarrow/_parquet.pxd
@@ -0,0 +1,217 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
+                                        CTable, MemoryPool)
+from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream
+
+
+cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
+  cdef cppclass Node:
+    pass
+
+  cdef cppclass GroupNode(Node):
+    pass
+
+  cdef cppclass PrimitiveNode(Node):
+    pass
+
+  cdef cppclass ColumnPath:
+    c_string ToDotString()
+
+cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
+    enum ParquetType" parquet::Type::type":
+        ParquetType_BOOLEAN" parquet::Type::BOOLEAN"
+        ParquetType_INT32" parquet::Type::INT32"
+        ParquetType_INT64" parquet::Type::INT64"
+        ParquetType_INT96" parquet::Type::INT96"
+        ParquetType_FLOAT" parquet::Type::FLOAT"
+        ParquetType_DOUBLE" parquet::Type::DOUBLE"
+        ParquetType_BYTE_ARRAY" parquet::Type::BYTE_ARRAY"
+        ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY"
+
+    enum ParquetLogicalType" parquet::LogicalType::type":
+        ParquetLogicalType_NONE" parquet::LogicalType::NONE"
+        ParquetLogicalType_UTF8" parquet::LogicalType::UTF8"
+        ParquetLogicalType_MAP" parquet::LogicalType::MAP"
+        ParquetLogicalType_MAP_KEY_VALUE" parquet::LogicalType::MAP_KEY_VALUE"
+        ParquetLogicalType_LIST" parquet::LogicalType::LIST"
+        ParquetLogicalType_ENUM" parquet::LogicalType::ENUM"
+        ParquetLogicalType_DECIMAL" parquet::LogicalType::DECIMAL"
+        ParquetLogicalType_DATE" parquet::LogicalType::DATE"
+        ParquetLogicalType_TIME_MILLIS" parquet::LogicalType::TIME_MILLIS"
+        ParquetLogicalType_TIME_MICROS" parquet::LogicalType::TIME_MICROS"
+        ParquetLogicalType_TIMESTAMP_MILLIS" parquet::LogicalType::TIMESTAMP_MILLIS"
+        ParquetLogicalType_TIMESTAMP_MICROS" parquet::LogicalType::TIMESTAMP_MICROS"
+        ParquetLogicalType_UINT_8" parquet::LogicalType::UINT_8"
+        ParquetLogicalType_UINT_16" parquet::LogicalType::UINT_16"
+        ParquetLogicalType_UINT_32" parquet::LogicalType::UINT_32"
+        ParquetLogicalType_UINT_64" parquet::LogicalType::UINT_64"
+        ParquetLogicalType_INT_8" parquet::LogicalType::INT_8"
+        ParquetLogicalType_INT_16" parquet::LogicalType::INT_16"
+        ParquetLogicalType_INT_32" parquet::LogicalType::INT_32"
+        ParquetLogicalType_INT_64" parquet::LogicalType::INT_64"
+        ParquetLogicalType_JSON" parquet::LogicalType::JSON"
+        ParquetLogicalType_BSON" parquet::LogicalType::BSON"
+        ParquetLogicalType_INTERVAL" parquet::LogicalType::INTERVAL"
+
+    enum ParquetRepetition" parquet::Repetition::type":
+        ParquetRepetition_REQUIRED" parquet::REPETITION::REQUIRED"
+        ParquetRepetition_OPTIONAL" parquet::REPETITION::OPTIONAL"
+        ParquetRepetition_REPEATED" parquet::REPETITION::REPEATED"
+
+    enum ParquetEncoding" parquet::Encoding::type":
+        ParquetEncoding_PLAIN" parquet::Encoding::PLAIN"
+        ParquetEncoding_PLAIN_DICTIONARY" parquet::Encoding::PLAIN_DICTIONARY"
+        ParquetEncoding_RLE" parquet::Encoding::RLE"
+        ParquetEncoding_BIT_PACKED" parquet::Encoding::BIT_PACKED"
+        ParquetEncoding_DELTA_BINARY_PACKED" parquet::Encoding::DELTA_BINARY_PACKED"
+        ParquetEncoding_DELTA_LENGTH_BYTE_ARRAY" parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY"
+        ParquetEncoding_DELTA_BYTE_ARRAY" parquet::Encoding::DELTA_BYTE_ARRAY"
+        ParquetEncoding_RLE_DICTIONARY" parquet::Encoding::RLE_DICTIONARY"
+
+    enum ParquetCompression" parquet::Compression::type":
+        ParquetCompression_UNCOMPRESSED" parquet::Compression::UNCOMPRESSED"
+        ParquetCompression_SNAPPY" parquet::Compression::SNAPPY"
+        ParquetCompression_GZIP" parquet::Compression::GZIP"
+        ParquetCompression_LZO" parquet::Compression::LZO"
+        ParquetCompression_BROTLI" parquet::Compression::BROTLI"
+
+    enum ParquetVersion" parquet::ParquetVersion::type":
+        ParquetVersion_V1" parquet::ParquetVersion::PARQUET_1_0"
+        ParquetVersion_V2" parquet::ParquetVersion::PARQUET_2_0"
+
+    cdef cppclass ColumnDescriptor:
+        shared_ptr[ColumnPath] path()
+
+        int16_t max_definition_level()
+        int16_t max_repetition_level()
+
+        ParquetType physical_type()
+        ParquetLogicalType logical_type()
+        const c_string& name()
+        int type_length()
+        int type_precision()
+        int type_scale()
+
+    cdef cppclass SchemaDescriptor:
+        const ColumnDescriptor* Column(int i)
+        shared_ptr[Node] schema()
+        GroupNode* group()
+        int num_columns()
+
+
+cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
+    cdef cppclass ColumnReader:
+        pass
+
+    cdef cppclass BoolReader(ColumnReader):
+        pass
+
+    cdef cppclass Int32Reader(ColumnReader):
+        pass
+
+    cdef cppclass Int64Reader(ColumnReader):
+        pass
+
+    cdef cppclass Int96Reader(ColumnReader):
+        pass
+
+    cdef cppclass FloatReader(ColumnReader):
+        pass
+
+    cdef cppclass DoubleReader(ColumnReader):
+        pass
+
+    cdef cppclass ByteArrayReader(ColumnReader):
+        pass
+
+    cdef cppclass RowGroupReader:
+        pass
+
+    cdef cppclass CRowGroupMetaData" parquet::RowGroupMetaData":
+        int num_columns()
+        int64_t num_rows()
+        int64_t total_byte_size()
+
+    cdef cppclass CFileMetaData" parquet::FileMetaData":
+        uint32_t size()
+        int num_columns()
+        int64_t num_rows()
+        int num_row_groups()
+        int32_t version()
+        const c_string created_by()
+        int num_schema_elements()
+
+        unique_ptr[CRowGroupMetaData] RowGroup(int i)
+        const SchemaDescriptor* schema()
+
+    cdef cppclass ParquetFileReader:
+        # TODO: Some default arguments are missing
+        @staticmethod
+        unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
+        shared_ptr[CFileMetaData] metadata();
+
+
+cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
+    cdef cppclass ParquetOutputStream" parquet::OutputStream":
+        pass
+
+    cdef cppclass LocalFileOutputStream(ParquetOutputStream):
+        LocalFileOutputStream(const c_string& path)
+        void Close()
+
+    cdef cppclass WriterProperties:
+        cppclass Builder:
+            Builder* version(ParquetVersion version)
+            Builder* compression(ParquetCompression codec)
+            Builder* compression(const c_string& path,
+                                 ParquetCompression codec)
+            Builder* disable_dictionary()
+            Builder* enable_dictionary()
+            Builder* enable_dictionary(const c_string& path)
+            shared_ptr[WriterProperties] build()
+
+
+cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
+    CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
+                     MemoryPool* allocator,
+                     unique_ptr[FileReader]* reader)
+
+    cdef cppclass FileReader:
+        FileReader(MemoryPool* pool, unique_ptr[ParquetFileReader] reader)
+        CStatus ReadFlatColumn(int i, shared_ptr[CArray]* out);
+        CStatus ReadFlatTable(shared_ptr[CTable]* out);
+        const ParquetFileReader* parquet_reader();
+
+
+cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
+    CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema,
+                              shared_ptr[CSchema]* out)
+    CStatus ToParquetSchema(const CSchema* arrow_schema,
+                            shared_ptr[SchemaDescriptor]* out)
+
+
+cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
+    cdef CStatus WriteFlatTable(
+        const CTable* table, MemoryPool* pool,
+        const shared_ptr[OutputStream]& sink,
+        int64_t chunk_size,
+        const shared_ptr[WriterProperties]& properties)
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
new file mode 100644
index 0000000000000..c0dc3eb460929
--- /dev/null
+++ b/python/pyarrow/_parquet.pyx
@@ -0,0 +1,520 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from pyarrow._parquet cimport *
+
+from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.libarrow_io cimport (ReadableFileInterface, OutputStream,
+                                           FileOutputStream)
+cimport pyarrow.includes.pyarrow as pyarrow
+
+from pyarrow.array cimport Array
+from pyarrow.compat import tobytes, frombytes
+from pyarrow.error import ArrowException
+from pyarrow.error cimport check_status
+from pyarrow.io import NativeFile
+from pyarrow.table cimport Table
+
+from pyarrow.io cimport NativeFile, get_reader, get_writer
+
+import six
+
+
+cdef class RowGroupMetaData:
+    cdef:
+        unique_ptr[CRowGroupMetaData] up_metadata
+        CRowGroupMetaData* metadata
+        object parent
+
+    def __cinit__(self):
+        pass
+
+    cdef init_from_file(self, FileMetaData parent, int i):
+        if i < 0 or i >= parent.num_row_groups:
+            raise IndexError('{0} out of bounds'.format(i))
+        self.up_metadata = parent.metadata.RowGroup(i)
+        self.metadata = self.up_metadata.get()
+        self.parent = parent
+
+    def __repr__(self):
+        return """{0}
+  num_columns: {1}
+  num_rows: {2}
+  total_byte_size: {3}""".format(object.__repr__(self),
+                                 self.num_columns,
+                                 self.num_rows,
+                                 self.total_byte_size)
+
+    property num_columns:
+
+        def __get__(self):
+            return self.metadata.num_columns()
+
+    property num_rows:
+
+        def __get__(self):
+            return self.metadata.num_rows()
+
+    property total_byte_size:
+
+        def __get__(self):
+            return self.metadata.total_byte_size()
+
+
+cdef class FileMetaData:
+    cdef:
+        shared_ptr[CFileMetaData] sp_metadata
+        CFileMetaData* metadata
+        object _schema
+
+    def __cinit__(self):
+        pass
+
+    cdef init(self, const shared_ptr[CFileMetaData]& metadata):
+        self.sp_metadata = metadata
+        self.metadata = metadata.get()
+
+    def __repr__(self):
+        return """{0}
+  created_by: {1}
+  num_columns: {2}
+  num_rows: {3}
+  num_row_groups: {4}
+  format_version: {5}
+  serialized_size: {6}""".format(object.__repr__(self),
+                                 self.created_by, self.num_columns,
+                                 self.num_rows, self.num_row_groups,
+                                 self.format_version,
+                                 self.serialized_size)
+
+    @property
+    def schema(self):
+        if self._schema is not None:
+            return self._schema
+
+        cdef Schema schema = Schema()
+        schema.init_from_filemeta(self)
+        self._schema = schema
+        return schema
+
+    property serialized_size:
+
+        def __get__(self):
+            return self.metadata.size()
+
+    property num_columns:
+
+        def __get__(self):
+            return self.metadata.num_columns()
+
+    property num_rows:
+
+        def __get__(self):
+            return self.metadata.num_rows()
+
+    property num_row_groups:
+
+        def __get__(self):
+            return self.metadata.num_row_groups()
+
+    property format_version:
+
+        def __get__(self):
+            cdef int version = self.metadata.version()
+            if version == 2:
+                return '2.0'
+            elif version == 1:
+                return '1.0'
+            else:
+                print('Unrecognized file version, assuming 1.0: {0}'
+                      .format(version))
+                return '1.0'
+
+    property created_by:
+
+        def __get__(self):
+            return frombytes(self.metadata.created_by())
+
+    def row_group(self, int i):
+        """
+
+        """
+        cdef RowGroupMetaData result = RowGroupMetaData()
+        result.init_from_file(self, i)
+        return result
+
+
+cdef class Schema:
+    cdef:
+        object parent  # the FileMetaData owning the SchemaDescriptor
+        const SchemaDescriptor* schema
+
+    def __cinit__(self):
+        self.parent = None
+        self.schema = NULL
+
+    def __repr__(self):
+        cdef const ColumnDescriptor* descr
+        elements = []
+        for i in range(self.schema.num_columns()):
+            col = self.column(i)
+            logical_type = col.logical_type
+            formatted = '{0}: {1}'.format(col.path, col.physical_type)
+            if logical_type != 'NONE':
+                formatted += ' {0}'.format(logical_type)
+            elements.append(formatted)
+
+        return """{0}
+{1}
+ """.format(object.__repr__(self), '\n'.join(elements))
+
+    cdef init_from_filemeta(self, FileMetaData container):
+        self.parent = container
+        self.schema = container.metadata.schema()
+
+    def __len__(self):
+        return self.schema.num_columns()
+
+    def __getitem__(self, i):
+        return self.column(i)
+
+    def column(self, i):
+        if i < 0 or i >= len(self):
+            raise IndexError('{0} out of bounds'.format(i))
+
+        cdef ColumnSchema col = ColumnSchema()
+        col.init_from_schema(self, i)
+        return col
+
+
+cdef class ColumnSchema:
+    cdef:
+        object parent
+        const ColumnDescriptor* descr
+
+    def __cinit__(self):
+        self.descr = NULL
+
+    cdef init_from_schema(self, Schema schema, int i):
+        self.parent = schema
+        self.descr = schema.schema.Column(i)
+
+    def __repr__(self):
+        physical_type = self.physical_type
+        logical_type = self.logical_type
+        if logical_type == 'DECIMAL':
+            logical_type = 'DECIMAL({0}, {1})'.format(self.precision,
+                                                      self.scale)
+        elif physical_type == 'FIXED_LEN_BYTE_ARRAY':
+            logical_type = ('FIXED_LEN_BYTE_ARRAY(length={0})'
+                            .format(self.length))
+
+        return """<ParquetColumnSchema>
+  name: {0}
+  path: {1}
+  max_definition_level: {2}
+  max_repetition_level: {3}
+  physical_type: {4}
+  logical_type: {5}""".format(self.name, self.path, self.max_definition_level,
+                       self.max_repetition_level, physical_type,
+                       logical_type)
+
+    property name:
+
+        def __get__(self):
+            return frombytes(self.descr.name())
+
+    property path:
+
+        def __get__(self):
+            return frombytes(self.descr.path().get().ToDotString())
+
+    property max_definition_level:
+
+        def __get__(self):
+            return self.descr.max_definition_level()
+
+    property max_repetition_level:
+
+        def __get__(self):
+            return self.descr.max_repetition_level()
+
+    property physical_type:
+
+        def __get__(self):
+            return physical_type_name_from_enum(self.descr.physical_type())
+
+    property logical_type:
+
+        def __get__(self):
+            return logical_type_name_from_enum(self.descr.logical_type())
+
+    # FIXED_LEN_BYTE_ARRAY attribute
+    property length:
+
+        def __get__(self):
+            return self.descr.type_length()
+
+    # Decimal attributes
+    property precision:
+
+        def __get__(self):
+            return self.descr.type_precision()
+
+    property scale:
+
+        def __get__(self):
+            return self.descr.type_scale()
+
+
+cdef physical_type_name_from_enum(ParquetType type_):
+    return {
+        ParquetType_BOOLEAN: 'BOOLEAN',
+        ParquetType_INT32: 'INT32',
+        ParquetType_INT64: 'INT64',
+        ParquetType_INT96: 'INT96',
+        ParquetType_FLOAT: 'FLOAT',
+        ParquetType_DOUBLE: 'DOUBLE',
+        ParquetType_BYTE_ARRAY: 'BYTE_ARRAY',
+        ParquetType_FIXED_LEN_BYTE_ARRAY: 'FIXED_LEN_BYTE_ARRAY',
+    }.get(type_, 'UNKNOWN')
+
+
+cdef logical_type_name_from_enum(ParquetLogicalType type_):
+    return {
+        ParquetLogicalType_NONE: 'NONE',
+        ParquetLogicalType_UTF8: 'UTF8',
+        ParquetLogicalType_MAP: 'MAP',
+        ParquetLogicalType_MAP_KEY_VALUE: 'MAP_KEY_VALUE',
+        ParquetLogicalType_LIST: 'LIST',
+        ParquetLogicalType_ENUM: 'ENUM',
+        ParquetLogicalType_DECIMAL: 'DECIMAL',
+        ParquetLogicalType_DATE: 'DATE',
+        ParquetLogicalType_TIME_MILLIS: 'TIME_MILLIS',
+        ParquetLogicalType_TIME_MICROS: 'TIME_MICROS',
+        ParquetLogicalType_TIMESTAMP_MILLIS: 'TIMESTAMP_MILLIS',
+        ParquetLogicalType_TIMESTAMP_MICROS: 'TIMESTAMP_MICROS',
+        ParquetLogicalType_UINT_8: 'UINT_8',
+        ParquetLogicalType_UINT_16: 'UINT_16',
+        ParquetLogicalType_UINT_32: 'UINT_32',
+        ParquetLogicalType_UINT_64: 'UINT_64',
+        ParquetLogicalType_INT_8: 'INT_8',
+        ParquetLogicalType_INT_16: 'INT_16',
+        ParquetLogicalType_INT_32: 'INT_32',
+        ParquetLogicalType_INT_64: 'UINT_64',
+        ParquetLogicalType_JSON: 'JSON',
+        ParquetLogicalType_BSON: 'BSON',
+        ParquetLogicalType_INTERVAL: 'INTERVAL',
+    }.get(type_, 'UNKNOWN')
+
+
+cdef class ParquetReader:
+    cdef:
+        MemoryPool* allocator
+        unique_ptr[FileReader] reader
+        column_idx_map
+        FileMetaData _metadata
+
+    def __cinit__(self):
+        self.allocator = default_memory_pool()
+        self._metadata = None
+
+    def open(self, object source):
+        cdef:
+            shared_ptr[ReadableFileInterface] rd_handle
+            c_string path
+
+        if isinstance(source, six.string_types):
+            path = tobytes(source)
+
+            # Must be in one expression to avoid calling std::move which is not
+            # possible in Cython (due to missing rvalue support)
+
+            # TODO(wesm): ParquetFileReader::OpenFile can throw?
+            self.reader = unique_ptr[FileReader](
+                new FileReader(default_memory_pool(),
+                               ParquetFileReader.OpenFile(path)))
+        else:
+            get_reader(source, &rd_handle)
+            check_status(OpenFile(rd_handle, self.allocator, &self.reader))
+
+    @property
+    def metadata(self):
+        cdef:
+            shared_ptr[CFileMetaData] metadata
+            FileMetaData result
+        if self._metadata is not None:
+            return self._metadata
+
+        metadata = self.reader.get().parquet_reader().metadata()
+
+        self._metadata = result = FileMetaData()
+        result.init(metadata)
+        return result
+
+    def read_all(self):
+        cdef:
+            Table table = Table()
+            shared_ptr[CTable] ctable
+
+        with nogil:
+            check_status(self.reader.get()
+                         .ReadFlatTable(&ctable))
+
+        table.init(ctable)
+        return table
+
+    def column_name_idx(self, column_name):
+        """
+        Find the matching index of a column in the schema.
+
+        Parameter
+        ---------
+        column_name: str
+            Name of the column, separation of nesting levels is done via ".".
+
+        Returns
+        -------
+        column_idx: int
+            Integer index of the position of the column
+        """
+        cdef:
+            FileMetaData container = self.metadata
+            const CFileMetaData* metadata = container.metadata
+            int i = 0
+
+        if self.column_idx_map is None:
+            self.column_idx_map = {}
+            for i in range(0, metadata.num_columns()):
+                col_bytes = tobytes(metadata.schema().Column(i)
+                                    .path().get().ToDotString())
+                self.column_idx_map[col_bytes] = i
+
+        return self.column_idx_map[tobytes(column_name)]
+
+    def read_column(self, int column_index):
+        cdef:
+            Array array = Array()
+            shared_ptr[CArray] carray
+
+        with nogil:
+            check_status(self.reader.get()
+                         .ReadFlatColumn(column_index, &carray))
+
+        array.init(carray)
+        return array
+
+
+cdef check_compression_name(name):
+    if name.upper() not in ['NONE', 'SNAPPY', 'GZIP', 'LZO', 'BROTLI']:
+        raise ArrowException("Unsupported compression: " + name)
+
+
+cdef ParquetCompression compression_from_name(object name):
+    name = name.upper()
+    if name == "SNAPPY":
+        return ParquetCompression_SNAPPY
+    elif name == "GZIP":
+        return ParquetCompression_GZIP
+    elif name == "LZO":
+        return ParquetCompression_LZO
+    elif name == "BROTLI":
+        return ParquetCompression_BROTLI
+    else:
+        return ParquetCompression_UNCOMPRESSED
+
+
+cdef class ParquetWriter:
+    cdef:
+        shared_ptr[WriterProperties] properties
+        shared_ptr[OutputStream] sink
+
+    cdef readonly:
+        object use_dictionary
+        object compression
+        object version
+        int row_group_size
+
+    def __cinit__(self, where, use_dictionary=None, compression=None,
+                  version=None):
+        cdef shared_ptr[FileOutputStream] filestream
+
+        if isinstance(where, six.string_types):
+            check_status(FileOutputStream.Open(tobytes(where), &filestream))
+            self.sink = <shared_ptr[OutputStream]> filestream
+        else:
+            get_writer(where, &self.sink)
+
+        self.use_dictionary = use_dictionary
+        self.compression = compression
+        self.version = version
+        self._setup_properties()
+
+    cdef _setup_properties(self):
+        cdef WriterProperties.Builder properties_builder
+        self._set_version(&properties_builder)
+        self._set_compression_props(&properties_builder)
+        self._set_dictionary_props(&properties_builder)
+        self.properties = properties_builder.build()
+
+    cdef _set_version(self, WriterProperties.Builder* props):
+        if self.version is not None:
+            if self.version == "1.0":
+                props.version(ParquetVersion_V1)
+            elif self.version == "2.0":
+                props.version(ParquetVersion_V2)
+            else:
+                raise ArrowException("Unsupported Parquet format version")
+
+    cdef _set_compression_props(self, WriterProperties.Builder* props):
+        if isinstance(self.compression, basestring):
+            check_compression_name(self.compression)
+            props.compression(compression_from_name(self.compression))
+        elif self.compression is not None:
+            # Deactivate dictionary encoding by default
+            props.disable_dictionary()
+            for column, codec in self.compression.iteritems():
+                check_compression_name(codec)
+                props.compression(column, compression_from_name(codec))
+
+    cdef _set_dictionary_props(self, WriterProperties.Builder* props):
+        if isinstance(self.use_dictionary, bool):
+            if self.use_dictionary:
+                props.enable_dictionary()
+            else:
+                props.disable_dictionary()
+        else:
+            # Deactivate dictionary encoding by default
+            props.disable_dictionary()
+            for column in self.use_dictionary:
+                props.enable_dictionary(column)
+
+    def write_table(self, Table table, row_group_size=None):
+        cdef CTable* ctable = table.table
+
+        if row_group_size is None:
+            row_group_size = ctable.num_rows()
+
+        cdef int c_row_group_size = row_group_size
+        with nogil:
+            check_status(WriteFlatTable(ctable, default_memory_pool(),
+                                        self.sink, c_row_group_size,
+                                        self.properties))
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
deleted file mode 100644
index d9e121dd8853e..0000000000000
--- a/python/pyarrow/includes/parquet.pxd
+++ /dev/null
@@ -1,147 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# distutils: language = c++
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport CArray, CSchema, CStatus, CTable, MemoryPool
-from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream
-
-
-cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
-  cdef cppclass Node:
-    pass
-
-  cdef cppclass GroupNode(Node):
-    pass
-
-  cdef cppclass PrimitiveNode(Node):
-    pass
-
-  cdef cppclass ColumnPath:
-    c_string ToDotString()
-
-cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
-  enum ParquetVersion" parquet::ParquetVersion::type":
-      PARQUET_1_0" parquet::ParquetVersion::PARQUET_1_0"
-      PARQUET_2_0" parquet::ParquetVersion::PARQUET_2_0"
-
-  enum Compression" parquet::Compression::type":
-      UNCOMPRESSED" parquet::Compression::UNCOMPRESSED"
-      SNAPPY" parquet::Compression::SNAPPY"
-      GZIP" parquet::Compression::GZIP"
-      LZO" parquet::Compression::LZO"
-      BROTLI" parquet::Compression::BROTLI"
-
-  cdef cppclass ColumnDescriptor:
-    shared_ptr[ColumnPath] path()
-
-  cdef cppclass SchemaDescriptor:
-    const ColumnDescriptor* Column(int i)
-    shared_ptr[Node] schema()
-    GroupNode* group()
-
-
-cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
-    cdef cppclass ColumnReader:
-        pass
-
-    cdef cppclass BoolReader(ColumnReader):
-        pass
-
-    cdef cppclass Int32Reader(ColumnReader):
-        pass
-
-    cdef cppclass Int64Reader(ColumnReader):
-        pass
-
-    cdef cppclass Int96Reader(ColumnReader):
-        pass
-
-    cdef cppclass FloatReader(ColumnReader):
-        pass
-
-    cdef cppclass DoubleReader(ColumnReader):
-        pass
-
-    cdef cppclass ByteArrayReader(ColumnReader):
-        pass
-
-    cdef cppclass RowGroupReader:
-        pass
-
-    cdef cppclass FileMetaData:
-        uint32_t size()
-        int num_columns()
-        int64_t num_rows()
-        int num_row_groups()
-        int32_t version()
-        const c_string created_by()
-        int num_schema_elements()
-        const SchemaDescriptor* schema()
-
-    cdef cppclass ParquetFileReader:
-        # TODO: Some default arguments are missing
-        @staticmethod
-        unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
-        shared_ptr[FileMetaData] metadata();
-
-
-cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
-    cdef cppclass ParquetOutputStream" parquet::OutputStream":
-        pass
-
-    cdef cppclass LocalFileOutputStream(ParquetOutputStream):
-        LocalFileOutputStream(const c_string& path)
-        void Close()
-
-    cdef cppclass WriterProperties:
-        cppclass Builder:
-            Builder* version(ParquetVersion version)
-            Builder* compression(Compression codec)
-            Builder* compression(const c_string& path, Compression codec)
-            Builder* disable_dictionary()
-            Builder* enable_dictionary()
-            Builder* enable_dictionary(const c_string& path)
-            shared_ptr[WriterProperties] build()
-
-
-cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
-    CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
-                     MemoryPool* allocator,
-                     unique_ptr[FileReader]* reader)
-
-    cdef cppclass FileReader:
-        FileReader(MemoryPool* pool, unique_ptr[ParquetFileReader] reader)
-        CStatus ReadFlatColumn(int i, shared_ptr[CArray]* out);
-        CStatus ReadFlatTable(shared_ptr[CTable]* out);
-        const ParquetFileReader* parquet_reader();
-
-
-cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
-    CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema,
-                              shared_ptr[CSchema]* out)
-    CStatus ToParquetSchema(const CSchema* arrow_schema,
-                            shared_ptr[SchemaDescriptor]* out)
-
-
-cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
-    cdef CStatus WriteFlatTable(
-        const CTable* table, MemoryPool* pool,
-        const shared_ptr[OutputStream]& sink,
-        int64_t chunk_size,
-        const shared_ptr[WriterProperties]& properties)
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
new file mode 100644
index 0000000000000..2dedb72ebfcc1
--- /dev/null
+++ b/python/pyarrow/parquet.py
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pyarrow._parquet as _parquet
+from pyarrow.table import Table
+
+
+class ParquetFile(object):
+    """
+    Open a Parquet binary file for reading
+
+    Parameters
+    ----------
+    source : str or pyarrow.io.NativeFile
+        Readable source. For passing Python file objects or byte buffers,
+        see pyarrow.io.PythonFileInterface or pyarrow.io.BytesReader.
+    metadata : ParquetFileMetadata, default None
+        Use existing metadata object, rather than reading from file.
+    """
+    def __init__(self, source, metadata=None):
+        self.reader = _parquet.ParquetReader()
+        self.reader.open(source)
+
+    @property
+    def metadata(self):
+        return self.reader.metadata
+
+    @property
+    def schema(self):
+        return self.metadata.schema
+
+    def read(self, nrows=None, columns=None):
+        """
+        Read a Table from Parquet format
+
+        Parameters
+        ----------
+        columns: list
+            If not None, only these columns will be read from the file.
+
+        Returns
+        -------
+        pyarrow.table.Table
+            Content of the file as a table (of columns)
+        """
+        if nrows is not None:
+            raise NotImplementedError("nrows argument")
+
+        if columns is None:
+            return self.reader.read_all()
+        else:
+            column_idxs = [self.reader.column_name_idx(column)
+                           for column in columns]
+            arrays = [self.reader.read_column(column_idx)
+                      for column_idx in column_idxs]
+            return Table.from_arrays(columns, arrays)
+
+
+def read_table(source, columns=None):
+    """
+    Read a Table from Parquet format
+
+    Parameters
+    ----------
+    source: str or pyarrow.io.NativeFile
+        Readable source. For passing Python file objects or byte buffers, see
+        pyarrow.io.PythonFileInterface or pyarrow.io.BytesReader.
+    columns: list
+        If not None, only these columns will be read from the file.
+
+    Returns
+    -------
+    pyarrow.table.Table
+        Content of the file as a table (of columns)
+    """
+    return ParquetFile(source).read(columns=columns)
+
+
+def write_table(table, sink, chunk_size=None, version=None,
+                use_dictionary=True, compression=None):
+    """
+    Write a Table to Parquet format
+
+    Parameters
+    ----------
+    table : pyarrow.Table
+    sink: string or pyarrow.io.NativeFile
+    chunk_size : int
+        The maximum number of rows in each Parquet RowGroup. As a default,
+        we will write a single RowGroup per file.
+    version : {"1.0", "2.0"}, default "1.0"
+        The Parquet format version, defaults to 1.0
+    use_dictionary : bool or list
+        Specify if we should use dictionary encoding in general or only for
+        some columns.
+    compression : str or dict
+        Specify the compression codec, either on a general basis or per-column.
+    """
+    writer = _parquet.ParquetWriter(sink, use_dictionary=use_dictionary,
+                                    compression=compression,
+                                    version=version)
+    writer.write_table(table, row_group_size=chunk_size)
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
deleted file mode 100644
index c0921859e440d..0000000000000
--- a/python/pyarrow/parquet.pyx
+++ /dev/null
@@ -1,244 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.parquet cimport *
-from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream, FileOutputStream
-cimport pyarrow.includes.pyarrow as pyarrow
-
-from pyarrow.array cimport Array
-from pyarrow.compat import tobytes
-from pyarrow.error import ArrowException
-from pyarrow.error cimport check_status
-from pyarrow.io import NativeFile
-from pyarrow.table cimport Table
-
-from pyarrow.io cimport NativeFile, get_reader, get_writer
-
-import six
-
-__all__ = [
-    'read_table',
-    'write_table'
-]
-
-cdef class ParquetReader:
-    cdef:
-        MemoryPool* allocator
-        unique_ptr[FileReader] reader
-        column_idx_map
-
-    def __cinit__(self):
-        self.allocator = default_memory_pool()
-
-    def open(self, source):
-        self._open(source)
-
-    cdef _open(self, object source):
-        cdef:
-            shared_ptr[ReadableFileInterface] rd_handle
-            c_string path
-
-        if isinstance(source, six.string_types):
-            path = tobytes(source)
-
-            # Must be in one expression to avoid calling std::move which is not
-            # possible in Cython (due to missing rvalue support)
-
-            # TODO(wesm): ParquetFileReader::OpenFile can throw?
-            self.reader = unique_ptr[FileReader](
-                new FileReader(default_memory_pool(),
-                               ParquetFileReader.OpenFile(path)))
-        else:
-            get_reader(source, &rd_handle)
-            check_status(OpenFile(rd_handle, self.allocator, &self.reader))
-
-    def read_all(self):
-        cdef:
-            Table table = Table()
-            shared_ptr[CTable] ctable
-
-        with nogil:
-            check_status(self.reader.get()
-                         .ReadFlatTable(&ctable))
-
-        table.init(ctable)
-        return table
-
-    def column_name_idx(self, column_name):
-        """
-        Find the matching index of a column in the schema.
-
-        Parameter
-        ---------
-        column_name: str
-            Name of the column, separation of nesting levels is done via ".".
-
-        Returns
-        -------
-        column_idx: int
-            Integer index of the position of the column
-        """
-        cdef:
-            const FileMetaData* metadata = (self.reader.get().parquet_reader()
-                                            .metadata().get())
-            int i = 0
-
-        if self.column_idx_map is None:
-            self.column_idx_map = {}
-            for i in range(0, metadata.num_columns()):
-                col_bytes = tobytes(metadata.schema().Column(i)
-                                    .path().get().ToDotString())
-                self.column_idx_map[col_bytes] = i
-
-        return self.column_idx_map[tobytes(column_name)]
-
-    def read_column(self, int column_index):
-        cdef:
-            Array array = Array()
-            shared_ptr[CArray] carray
-
-        with nogil:
-            check_status(self.reader.get()
-                         .ReadFlatColumn(column_index, &carray))
-
-        array.init(carray)
-        return array
-
-
-def read_table(source, columns=None):
-    """
-    Read a Table from Parquet format
-
-    Parameters
-    ----------
-    source: str or pyarrow.io.NativeFile
-        Readable source. For passing Python file objects or byte buffers, see
-        pyarrow.io.PythonFileInterface or pyarrow.io.BytesReader.
-    columns: list
-        If not None, only these columns will be read from the file.
-
-    Returns
-    -------
-    pyarrow.table.Table
-        Content of the file as a table (of columns)
-    """
-    cdef ParquetReader reader = ParquetReader()
-    reader._open(source)
-
-    if columns is None:
-        return reader.read_all()
-    else:
-        column_idxs = [reader.column_name_idx(column) for column in columns]
-        arrays = [reader.read_column(column_idx) for column_idx in column_idxs]
-        return Table.from_arrays(columns, arrays)
-
-
-def write_table(table, sink, chunk_size=None, version=None,
-                use_dictionary=True, compression=None):
-    """
-    Write a Table to Parquet format
-
-    Parameters
-    ----------
-    table : pyarrow.Table
-    sink: string or pyarrow.io.NativeFile
-    chunk_size : int
-        The maximum number of rows in each Parquet RowGroup. As a default,
-        we will write a single RowGroup per file.
-    version : {"1.0", "2.0"}, default "1.0"
-        The Parquet format version, defaults to 1.0
-    use_dictionary : bool or list
-        Specify if we should use dictionary encoding in general or only for
-        some columns.
-    compression : str or dict
-        Specify the compression codec, either on a general basis or per-column.
-    """
-    cdef Table table_ = table
-    cdef CTable* ctable_ = table_.table
-    cdef shared_ptr[FileOutputStream] filesink_
-    cdef shared_ptr[OutputStream] sink_
-
-    cdef WriterProperties.Builder properties_builder
-    cdef int64_t chunk_size_ = 0
-    if chunk_size is None:
-        chunk_size_ = ctable_.num_rows()
-    else:
-        chunk_size_ = chunk_size
-
-    if version is not None:
-        if version == "1.0":
-            properties_builder.version(PARQUET_1_0)
-        elif version == "2.0":
-            properties_builder.version(PARQUET_2_0)
-        else:
-            raise ArrowException("Unsupported Parquet format version")
-
-    if isinstance(use_dictionary, bool):
-        if use_dictionary:
-            properties_builder.enable_dictionary()
-        else:
-            properties_builder.disable_dictionary()
-    else:
-        # Deactivate dictionary encoding by default
-        properties_builder.disable_dictionary()
-        for column in use_dictionary:
-            properties_builder.enable_dictionary(column)
-
-    if isinstance(compression, basestring):
-        if compression == "NONE":
-            properties_builder.compression(UNCOMPRESSED)
-        elif compression == "SNAPPY":
-            properties_builder.compression(SNAPPY)
-        elif compression == "GZIP":
-            properties_builder.compression(GZIP)
-        elif compression == "LZO":
-            properties_builder.compression(LZO)
-        elif compression == "BROTLI":
-            properties_builder.compression(BROTLI)
-        else:
-            raise ArrowException("Unsupport compression codec")
-    elif compression is not None:
-        # Deactivate dictionary encoding by default
-        properties_builder.disable_dictionary()
-        for column, codec in compression.iteritems():
-            if codec == "NONE":
-                properties_builder.compression(column, UNCOMPRESSED)
-            elif codec == "SNAPPY":
-                properties_builder.compression(column, SNAPPY)
-            elif codec == "GZIP":
-                properties_builder.compression(column, GZIP)
-            elif codec == "LZO":
-                properties_builder.compression(column, LZO)
-            elif codec == "BROTLI":
-                properties_builder.compression(column, BROTLI)
-            else:
-                raise ArrowException("Unsupport compression codec")
-
-    if isinstance(sink, six.string_types):
-        check_status(FileOutputStream.Open(tobytes(sink), &filesink_))
-        sink_ = <shared_ptr[OutputStream]>filesink_
-    else:
-        get_writer(sink, &sink_)
-
-    with nogil:
-        check_status(WriteFlatTable(ctable_, default_memory_pool(), sink_,
-                                    chunk_size_, properties_builder.build()))
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 0fb913cc792d8..ad4bc580e8b1c 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -55,10 +55,8 @@ def test_single_pylist_column_roundtrip(tmpdir):
             assert data_written.equals(data_read)
 
 
-@parquet
-def test_pandas_parquet_2_0_rountrip(tmpdir):
-    size = 10000
-    np.random.seed(0)
+def alltypes_sample(size=10000, seed=0):
+    np.random.seed(seed)
     df = pd.DataFrame({
         'uint8': np.arange(size, dtype=np.uint8),
         'uint16': np.arange(size, dtype=np.uint16),
@@ -71,13 +69,21 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
         'float32': np.arange(size, dtype=np.float32),
         'float64': np.arange(size, dtype=np.float64),
         'bool': np.random.randn(size) > 0,
-        # Pandas only support ns resolution, Arrow at the moment only ms
+        # TODO(wesm): Test other timestamp resolutions now that arrow supports
+        # them
         'datetime': np.arange("2016-01-01T00:00:00.001", size,
                               dtype='datetime64[ms]'),
         'str': [str(x) for x in range(size)],
         'str_with_nulls': [None] + [str(x) for x in range(size - 2)] + [None],
         'empty_str': [''] * size
     })
+    return df
+
+
+@parquet
+def test_pandas_parquet_2_0_rountrip(tmpdir):
+    df = alltypes_sample(size=10000)
+
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = A.Table.from_pandas(df, timestamps_to_ms=True)
     A.parquet.write_table(arrow_table, filename.strpath, version="2.0")
@@ -117,6 +123,7 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
 
     pdt.assert_frame_equal(df, df_read)
 
+
 @parquet
 def test_pandas_column_selection(tmpdir):
     size = 10000
@@ -227,3 +234,57 @@ def test_pandas_parquet_configuration_options(tmpdir):
         table_read = pq.read_table(filename.strpath)
         df_read = table_read.to_pandas()
         pdt.assert_frame_equal(df, df_read)
+
+
+@parquet
+def test_parquet_metadata_api():
+    df = alltypes_sample(size=10000)
+    df = df.reindex(columns=sorted(df.columns))
+
+    a_table = A.Table.from_pandas(df, timestamps_to_ms=True)
+
+    buf = io.BytesIO()
+    pq.write_table(a_table, buf, compression='snappy', version='2.0')
+
+    buf.seek(0)
+    fileh = pq.ParquetFile(buf)
+
+    ncols = len(df.columns)
+
+    # Series of sniff tests
+    meta = fileh.metadata
+    repr(meta)
+    assert meta.num_rows == len(df)
+    assert meta.num_columns == ncols
+    assert meta.num_row_groups == 1
+    assert meta.format_version == '2.0'
+    assert 'parquet-cpp' in meta.created_by
+
+    # Schema
+    schema = fileh.schema
+    assert meta.schema is schema
+    assert len(schema) == ncols
+    repr(schema)
+
+    col = schema[0]
+    repr(col)
+    assert col.name == df.columns[0]
+    assert col.max_definition_level == 1
+    assert col.max_repetition_level == 0
+    assert col.max_repetition_level == 0
+
+    assert col.physical_type == 'BOOLEAN'
+    assert col.logical_type == 'NONE'
+
+    with pytest.raises(IndexError):
+        schema[ncols]
+
+    with pytest.raises(IndexError):
+        schema[-1]
+
+    # Row group
+    rg_meta = meta.row_group(0)
+    repr(rg_meta)
+
+    assert rg_meta.num_rows == len(df)
+    assert rg_meta.num_columns == ncols
diff --git a/python/setup.py b/python/setup.py
index 3829a7982d670..72ff5842a22a5 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -95,7 +95,7 @@ def initialize_options(self):
         'error',
         'io',
         'ipc',
-        'parquet',
+        '_parquet',
         'scalar',
         'schema',
         'table']
@@ -214,7 +214,7 @@ def _run_cmake(self):
         os.chdir(saved_cwd)
 
     def _failure_permitted(self, name):
-        if name == 'parquet' and not self.with_parquet:
+        if name == '_parquet' and not self.with_parquet:
             return True
         return False
 

From 8d917c1f925c76e9009f6d1b9792551293a572fe Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 10 Jan 2017 13:23:26 -0500
Subject: [PATCH 0267/1644] ARROW-466: Add ExternalProject for jemalloc

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #276 from xhochy/ARROW-466 and squashes the following commits:

9379c45 [Uwe L. Korn] Revert "Enable jemalloc on Windows"
6fd8da8 [Uwe L. Korn] Enable jemalloc on Windows
0e1082f [Uwe L. Korn] ARROW-466: Add ExternalProject for jemalloc
---
 cpp/CMakeLists.txt                    | 37 +++++++++++++++++++++++++--
 cpp/cmake_modules/Findjemalloc.cmake  |  5 ++++
 cpp/src/arrow/jemalloc/CMakeLists.txt | 12 ++++++---
 3 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 87b7841ece52e..8a2cfc5d6f180 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -30,6 +30,7 @@ set(GFLAGS_VERSION "2.1.2")
 set(GTEST_VERSION "1.7.0")
 set(GBENCHMARK_VERSION "1.1.0")
 set(FLATBUFFERS_VERSION "1.3.0")
+set(JEMALLOC_VERSION "4.4.0")
 
 find_package(ClangTools)
 if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
@@ -76,7 +77,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
 
   option(ARROW_JEMALLOC
     "Build the Arrow jemalloc-based allocator"
-    OFF)
+    ON)
 
   option(ARROW_BOOST_USE_SHARED
     "Rely on boost shared libraries where relevant"
@@ -594,11 +595,43 @@ message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
 include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
 
 if (ARROW_JEMALLOC)
-  find_package(jemalloc REQUIRED)
+  find_package(jemalloc)
+
+  if(NOT JEMALLOC_FOUND)
+    set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep")
+    set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
+    set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
+    set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(JEMALLOC_VENDORED 1)
+    if (CMAKE_VERSION VERSION_GREATER "3.2")
+      # BUILD_BYPRODUCTS is a 3.2+ feature
+      ExternalProject_Add(jemalloc_ep
+        URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+        CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND ${MAKE}
+        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}")
+    else()
+      ExternalProject_Add(jemalloc_ep
+        URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+        CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND ${MAKE})
+    endif()
+  else()
+    set(JEMALLOC_VENDORED 0)
+  endif()
 
   include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(jemalloc
+      STATIC_LIB ${JEMALLOC_STATIC_LIB}
       SHARED_LIB ${JEMALLOC_SHARED_LIB})
+
+  if (JEMALLOC_VENDORED)
+    add_dependencies(jemalloc_shared jemalloc_ep)
+    add_dependencies(jemalloc_static jemalloc_ep)
+  endif()
 endif()
 
 ## Google PerfTools
diff --git a/cpp/cmake_modules/Findjemalloc.cmake b/cpp/cmake_modules/Findjemalloc.cmake
index e7fbb94a69235..e511d4dde0f71 100644
--- a/cpp/cmake_modules/Findjemalloc.cmake
+++ b/cpp/cmake_modules/Findjemalloc.cmake
@@ -47,11 +47,16 @@ if ( _jemalloc_roots )
     find_library( JEMALLOC_SHARED_LIB NAMES ${LIBJEMALLOC_NAMES}
         PATHS ${_jemalloc_roots} NO_DEFAULT_PATH
         PATH_SUFFIXES "lib" )
+    find_library( JEMALLOC_STATIC_LIB NAMES jemalloc_pic
+        PATHS ${_jemalloc_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib" )
 else ()
     find_path( JEMALLOC_INCLUDE_DIR NAMES jemalloc/jemalloc.h )
     message(STATUS ${JEMALLOC_INCLUDE_DIR})
     find_library( JEMALLOC_SHARED_LIB NAMES ${LIBJEMALLOC_NAMES})
     message(STATUS ${JEMALLOC_SHARED_LIB})
+    find_library( JEMALLOC_STATIC_LIB NAMES jemalloc_pic)
+    message(STATUS ${JEMALLOC_STATIC_LIB})
 endif ()
 
 if (JEMALLOC_INCLUDE_DIR AND JEMALLOC_SHARED_LIB)
diff --git a/cpp/src/arrow/jemalloc/CMakeLists.txt b/cpp/src/arrow/jemalloc/CMakeLists.txt
index c6663eb8227f0..c0f90eba260f6 100644
--- a/cpp/src/arrow/jemalloc/CMakeLists.txt
+++ b/cpp/src/arrow/jemalloc/CMakeLists.txt
@@ -23,18 +23,24 @@ include_directories(SYSTEM "{JEMALLOC_INCLUDE_DIR}")
 # arrow_jemalloc library
 set(ARROW_JEMALLOC_STATIC_LINK_LIBS
   arrow_static
-  jemalloc
+  jemalloc_static
 )
+
+if (NOT APPLE)
+  set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} pthread)
+endif()
+
 set(ARROW_JEMALLOC_SHARED_LINK_LIBS
   arrow_shared
-  jemalloc
+  jemalloc_shared
 )
 
 if (ARROW_BUILD_STATIC)
   set(ARROW_JEMALLOC_TEST_LINK_LIBS
+    ${ARROW_JEMALLOC_STATIC_LINK_LIBS}
     arrow_jemalloc_static)
 else()
-  set(ARROW_jemalloc_TEST_LINK_LIBS
+  set(ARROW_JEMALLOC_TEST_LINK_LIBS
     arrow_jemalloc_shared)
 endif()
 

From 543e50814c15d58387683a43b5abc661c4acc484 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 10 Jan 2017 13:27:34 -0500
Subject: [PATCH 0268/1644] ARROW-469: C++: Add option so that resize doesn't
 decrease the capacity

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #277 from xhochy/ARROW-469 and squashes the following commits:

f59059f [Uwe L. Korn] ARROW-469: C++: Add option so that resize doesn't decrease the capacity
---
 cpp/src/arrow/buffer-test.cc | 11 ++++++++++-
 cpp/src/arrow/buffer.cc      | 11 ++++++-----
 cpp/src/arrow/buffer.h       | 13 ++++++++-----
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index c1d027bb653fe..2ded1e11f96f8 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -53,8 +53,17 @@ TEST_F(TestBuffer, Resize) {
   ASSERT_EQ(200, buf.size());
 
   // Make it smaller, too
-  ASSERT_OK(buf.Resize(50));
+  ASSERT_OK(buf.Resize(50, true));
   ASSERT_EQ(50, buf.size());
+  // We have actually shrunken in size
+  // The spec requires that capacity is a multiple of 64
+  ASSERT_EQ(64, buf.capacity());
+
+  // Resize to a larger capacity again to test shrink_to_fit = false
+  ASSERT_OK(buf.Resize(100));
+  ASSERT_EQ(128, buf.capacity());
+  ASSERT_OK(buf.Resize(50, false));
+  ASSERT_EQ(128, buf.capacity());
 }
 
 TEST_F(TestBuffer, ResizeOOM) {
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 2e64ffd75c263..6cce0efa37784 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -91,13 +91,14 @@ Status PoolBuffer::Reserve(int64_t new_capacity) {
   return Status::OK();
 }
 
-Status PoolBuffer::Resize(int64_t new_size) {
-  if (new_size > size_) {
+Status PoolBuffer::Resize(int64_t new_size, bool shrink_to_fit) {
+  if (!shrink_to_fit || (new_size > size_)) {
     RETURN_NOT_OK(Reserve(new_size));
   } else {
     // Buffer is not growing, so shrink to the requested size without
     // excess space.
-    if (capacity_ != new_size) {
+    int64_t new_capacity = BitUtil::RoundUpToMultipleOf64(new_size);
+    if (capacity_ != new_capacity) {
       // Buffer hasn't got yet the requested size.
       if (new_size == 0) {
         pool_->Free(mutable_data_, capacity_);
@@ -105,9 +106,9 @@ Status PoolBuffer::Resize(int64_t new_size) {
         mutable_data_ = nullptr;
         data_ = nullptr;
       } else {
-        RETURN_NOT_OK(pool_->Reallocate(capacity_, new_size, &mutable_data_));
+        RETURN_NOT_OK(pool_->Reallocate(capacity_, new_capacity, &mutable_data_));
         data_ = mutable_data_;
-        capacity_ = new_size;
+        capacity_ = new_capacity;
       }
     }
   }
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 27437ca0486c3..ac78808eaf205 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -127,10 +127,13 @@ class ARROW_EXPORT MutableBuffer : public Buffer {
 
 class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
  public:
-  // Change buffer reported size to indicated size, allocating memory if
-  // necessary.  This will ensure that the capacity of the buffer is a multiple
-  // of 64 bytes as defined in Layout.md.
-  virtual Status Resize(int64_t new_size) = 0;
+  /// Change buffer reported size to indicated size, allocating memory if
+  /// necessary.  This will ensure that the capacity of the buffer is a multiple
+  /// of 64 bytes as defined in Layout.md.
+  ///
+  /// @param shrink_to_fit On deactivating this option, the capacity of the Buffer won't
+  /// decrease.
+  virtual Status Resize(int64_t new_size, bool shrink_to_fit = true) = 0;
 
   // Ensure that buffer has enough memory allocated to fit the indicated
   // capacity (and meets the 64 byte padding requirement in Layout.md).
@@ -147,7 +150,7 @@ class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
   explicit PoolBuffer(MemoryPool* pool = nullptr);
   virtual ~PoolBuffer();
 
-  Status Resize(int64_t new_size) override;
+  Status Resize(int64_t new_size, bool shrink_to_fit = true) override;
   Status Reserve(int64_t new_capacity) override;
 
  private:

From 7d3e2a3ab90324625b738e464a020758379f457a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 11 Jan 2017 09:33:29 -0500
Subject: [PATCH 0269/1644] ARROW-421: [Python] Retain parent reference in
 PyBytesReader

Pass Buffer to BufferReader so that zero-copy slices retain reference to PyBytesBuffer, which prevents the bytes object from being garbage collected prematurely. Also added some helper tools for inspecting Arrow Buffer objects in Python.

Close #278

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #279 from wesm/ARROW-421 and squashes the following commits:

acf730e [Wes McKinney] Rename method
50c195a [Wes McKinney] Fix accidental typo
ef20185 [Wes McKinney] Pass Buffer to BufferReader so that zero-copy slices retain reference to PyBytesBuffer, which prevents the bytes object from being garbage collected prematurely
---
 cpp/src/arrow/io/memory.h            |  2 ++
 python/pyarrow/_parquet.pxd          |  2 +-
 python/pyarrow/_parquet.pyx          |  8 ++---
 python/pyarrow/includes/libarrow.pxd |  1 +
 python/pyarrow/io.pyx                | 46 ++++++++++++++++++++++++++--
 python/pyarrow/tests/test_io.py      | 14 +++++++++
 python/src/pyarrow/io.cc             | 10 ++----
 python/src/pyarrow/io.h              |  5 ++-
 8 files changed, 69 insertions(+), 19 deletions(-)

diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 8428a12220a69..2d3df4224e9fb 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -79,6 +79,8 @@ class ARROW_EXPORT BufferReader : public ReadableFileInterface {
 
   bool supports_zero_copy() const override;
 
+  std::shared_ptr<Buffer> buffer() const { return buffer_; }
+
  private:
   std::shared_ptr<Buffer> buffer_;
   const uint8_t* data_;
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index faca845167d31..7e49e9e834b77 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -156,7 +156,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         int num_columns()
         int64_t num_rows()
         int num_row_groups()
-        int32_t version()
+        ParquetVersion version()
         const c_string created_by()
         int num_schema_elements()
 
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index c0dc3eb460929..30e3de417a827 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -138,11 +138,11 @@ cdef class FileMetaData:
     property format_version:
 
         def __get__(self):
-            cdef int version = self.metadata.version()
-            if version == 2:
-                return '2.0'
-            elif version == 1:
+            cdef ParquetVersion version = self.metadata.version()
+            if version == ParquetVersion_V1:
                 return '1.0'
+            if version == ParquetVersion_V2:
+                return '2.0'
             else:
                 print('Unrecognized file version, assuming 1.0: {0}'
                       .format(version))
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index b0f971d516ce5..d1970e5a2c8f1 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -66,6 +66,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CBuffer" arrow::Buffer":
         uint8_t* data()
         int64_t size()
+        shared_ptr[CBuffer] parent()
 
     cdef cppclass ResizableBuffer(CBuffer):
         CStatus Resize(int64_t nbytes)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index cab6ccb90ee6b..b62de6cdd462c 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -123,11 +123,17 @@ cdef class NativeFile:
         with nogil:
             check_status(self.wr_file.get().Write(buf, bufsize))
 
-    def read(self, int64_t nbytes):
+    def read(self, nbytes=None):
         cdef:
+            int64_t c_nbytes
             int64_t bytes_read = 0
             PyObject* obj
 
+        if nbytes is None:
+            c_nbytes = self.size() - self.tell()
+        else:
+            c_nbytes = nbytes
+
         self._assert_readable()
 
         # Allocate empty write space
@@ -135,17 +141,35 @@ cdef class NativeFile:
 
         cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AS_STRING(<object> obj)
         with nogil:
-            check_status(self.rd_file.get().Read(nbytes, &bytes_read, buf))
+            check_status(self.rd_file.get().Read(c_nbytes, &bytes_read, buf))
 
-        if bytes_read < nbytes:
+        if bytes_read < c_nbytes:
             cp._PyBytes_Resize(&obj, <Py_ssize_t> bytes_read)
 
         return PyObject_to_object(obj)
 
+    def read_buffer(self, nbytes=None):
+        cdef:
+            int64_t c_nbytes
+            int64_t bytes_read = 0
+            shared_ptr[CBuffer] output
+        self._assert_readable()
+
+        if nbytes is None:
+            c_nbytes = self.size() - self.tell()
+        else:
+            c_nbytes = nbytes
+
+        with nogil:
+            check_status(self.rd_file.get().ReadB(c_nbytes, &output))
+
+        return wrap_buffer(output)
+
 
 # ----------------------------------------------------------------------
 # Python file-like objects
 
+
 cdef class PythonFileInterface(NativeFile):
     cdef:
         object handle
@@ -199,6 +223,16 @@ cdef class Buffer:
         def __get__(self):
             return self.buffer.get().size()
 
+    property parent:
+
+        def __get__(self):
+            cdef shared_ptr[CBuffer] parent_buf = self.buffer.get().parent()
+
+            if parent_buf.get() == NULL:
+                return None
+            else:
+                return wrap_buffer(parent_buf)
+
     def __getitem__(self, key):
         # TODO(wesm): buffer slicing
         raise NotImplementedError
@@ -209,6 +243,12 @@ cdef class Buffer:
             self.buffer.get().size())
 
 
+cdef wrap_buffer(const shared_ptr[CBuffer]& buffer):
+    cdef Buffer result = Buffer()
+    result.buffer = buffer
+    return result
+
+
 cdef shared_ptr[PoolBuffer] allocate_buffer():
     cdef shared_ptr[PoolBuffer] result
     result.reset(new PoolBuffer(pyarrow.get_memory_pool()))
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index c10ed0394b1a8..3e7a43702aa05 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -102,6 +102,20 @@ def test_bytes_reader_non_bytes():
         io.BytesReader(u('some sample data'))
 
 
+def test_bytes_reader_retains_parent_reference():
+    import gc
+
+    # ARROW-421
+    def get_buffer():
+        data = b'some sample data' * 1000
+        reader = io.BytesReader(data)
+        reader.seek(5)
+        return reader.read_buffer(6)
+
+    buf = get_buffer()
+    gc.collect()
+    assert buf.to_pybytes() == b'sample'
+    assert buf.parent is not None
 
 # ----------------------------------------------------------------------
 # Buffers
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index ac1aa635b40ea..01f851d874075 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -203,14 +203,8 @@ Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
 // A readable file that is backed by a PyBytes
 
 PyBytesReader::PyBytesReader(PyObject* obj)
-    : arrow::io::BufferReader(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)),
-          PyBytes_GET_SIZE(obj)),
-      obj_(obj) {
-  Py_INCREF(obj_);
-}
+    : arrow::io::BufferReader(std::make_shared<PyBytesBuffer>(obj)) {}
 
-PyBytesReader::~PyBytesReader() {
-  Py_DECREF(obj_);
-}
+PyBytesReader::~PyBytesReader() {}
 
 }  // namespace pyarrow
diff --git a/python/src/pyarrow/io.h b/python/src/pyarrow/io.h
index fd3e7c0887207..4cb010f2d4e9f 100644
--- a/python/src/pyarrow/io.h
+++ b/python/src/pyarrow/io.h
@@ -22,6 +22,8 @@
 #include "arrow/io/memory.h"
 
 #include "pyarrow/config.h"
+
+#include "pyarrow/common.h"
 #include "pyarrow/visibility.h"
 
 namespace arrow {
@@ -87,9 +89,6 @@ class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader {
  public:
   explicit PyBytesReader(PyObject* obj);
   virtual ~PyBytesReader();
-
- private:
-  PyObject* obj_;
 };
 
 // TODO(wesm): seekable output files

From c5663c6d00dbd297dac573670156e26dc0593357 Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent@dremio.com>
Date: Thu, 12 Jan 2017 12:21:37 -0500
Subject: [PATCH 0270/1644] ARROW-385: Refactors metric system

Arrow has some support for metrics, but the metrics registry is by default
not configured to export values. It also forces user to user yammer/codahale
metrics library instead of the library of their choice.

To allow for integration with other metrics system, replace it with a notification
mechanism to alert user on allocation/deallocation.

Author: Laurent Goujon <laurent@dremio.com>

Closes #212 from laurentgo/laurent/metrics-refactoring and squashes the following commits:

e6c435b [Laurent Goujon] ARROW-385: Refactors metric system
---
 java/memory/pom.xml                           |   7 -
 .../java/io/netty/buffer/LargeBuffer.java     |  31 +---
 .../netty/buffer/PooledByteBufAllocatorL.java | 157 ++++++++----------
 .../buffer/UnsafeDirectLittleEndian.java      |  34 +---
 .../Metrics.java => AllocationListener.java}  |  40 ++---
 .../arrow/memory/AllocationManager.java       |  13 +-
 .../apache/arrow/memory/BaseAllocator.java    |  30 +++-
 .../apache/arrow/memory/RootAllocator.java    |   7 +-
 .../org/apache/arrow/memory/util/Pointer.java |  28 ----
 9 files changed, 138 insertions(+), 209 deletions(-)
 rename java/memory/src/main/java/org/apache/arrow/memory/{util/Metrics.java => AllocationListener.java} (58%)
 delete mode 100644 java/memory/src/main/java/org/apache/arrow/memory/util/Pointer.java

diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 6ed14480860f2..a4eb65228febf 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -20,13 +20,6 @@
   <name>Arrow Memory</name>
 
   <dependencies>
-
-    <dependency>
-      <groupId>com.codahale.metrics</groupId>
-      <artifactId>metrics-core</artifactId>
-      <version>3.0.1</version>
-    </dependency>
-
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
diff --git a/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java b/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
index 5f5e904fb0429..c026e430d77f3 100644
--- a/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
+++ b/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
@@ -17,43 +17,16 @@
  */
 package io.netty.buffer;
 
-import java.util.concurrent.atomic.AtomicLong;
-
 /**
  * A MutableWrappedByteBuf that also maintains a metric of the number of huge buffer bytes and counts.
  */
 public class LargeBuffer extends MutableWrappedByteBuf {
-
-  private final AtomicLong hugeBufferSize;
-  private final AtomicLong hugeBufferCount;
-
-  private final int initCap;
-
-  public LargeBuffer(ByteBuf buffer, AtomicLong hugeBufferSize, AtomicLong hugeBufferCount) {
+  public LargeBuffer(ByteBuf buffer) {
     super(buffer);
-    initCap = buffer.capacity();
-    this.hugeBufferCount = hugeBufferCount;
-    this.hugeBufferSize = hugeBufferSize;
   }
 
   @Override
   public ByteBuf copy(int index, int length) {
-    return new LargeBuffer(buffer.copy(index, length), hugeBufferSize, hugeBufferCount);
+    return new LargeBuffer(buffer.copy(index, length));
   }
-
-  @Override
-  public boolean release() {
-    return release(1);
-  }
-
-  @Override
-  public boolean release(int decrement) {
-    boolean released = unwrap().release(decrement);
-    if (released) {
-      hugeBufferSize.addAndGet(-initCap);
-      hugeBufferCount.decrementAndGet();
-    }
-    return released;
-  }
-
 }
diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index f6feb65cccd09..a843ac5586e79 100644
--- a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -17,7 +17,7 @@
  */
 package io.netty.buffer;
 
-import io.netty.util.internal.StringUtil;
+import static org.apache.arrow.memory.util.AssertionUtil.ASSERT_ENABLED;
 
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
@@ -25,24 +25,16 @@
 
 import org.apache.arrow.memory.OutOfMemoryException;
 
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
+import io.netty.util.internal.StringUtil;
 
 /**
  * The base allocator that we use for all of Arrow's memory management. Returns UnsafeDirectLittleEndian buffers.
  */
 public class PooledByteBufAllocatorL {
-  private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
+  private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("arrow.allocator");
 
   private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
 
-
-  public static final String METRIC_PREFIX = "drill.allocator.";
-
-  private final MetricRegistry registry;
   private final AtomicLong hugeBufferSize = new AtomicLong(0);
   private final AtomicLong hugeBufferCount = new AtomicLong(0);
   private final AtomicLong normalBufferSize = new AtomicLong(0);
@@ -51,8 +43,7 @@ public class PooledByteBufAllocatorL {
   private final InnerAllocator allocator;
   public final UnsafeDirectLittleEndian empty;
 
-  public PooledByteBufAllocatorL(MetricRegistry registry) {
-    this.registry = registry;
+  public PooledByteBufAllocatorL() {
     allocator = new InnerAllocator();
     empty = new UnsafeDirectLittleEndian(new DuplicatedByteBuf(Unpooled.EMPTY_BUFFER));
   }
@@ -70,13 +61,66 @@ public int getChunkSize() {
     return allocator.chunkSize;
   }
 
-  private class InnerAllocator extends PooledByteBufAllocator {
+  public long getHugeBufferSize() {
+    return hugeBufferSize.get();
+  }
 
+  public long getHugeBufferCount() {
+    return hugeBufferCount.get();
+  }
 
+  public long getNormalBufferSize() {
+    return normalBufferSize.get();
+  }
+
+  public long getNormalBufferCount() {
+    return normalBufferSize.get();
+  }
+
+  private static class AccountedUnsafeDirectLittleEndian extends UnsafeDirectLittleEndian {
+    private final long initialCapacity;
+    private final AtomicLong count;
+    private final AtomicLong size;
+
+    private AccountedUnsafeDirectLittleEndian(LargeBuffer buf, AtomicLong count, AtomicLong size) {
+      super(buf);
+      this.initialCapacity = buf.capacity();
+      this.count = count;
+      this.size = size;
+    }
+
+    private AccountedUnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong count, AtomicLong size) {
+      super(buf);
+      this.initialCapacity = buf.capacity();
+      this.count = count;
+      this.size = size;
+    }
+
+    @Override
+    public ByteBuf copy() {
+      throw new UnsupportedOperationException("copy method is not supported");
+    }
+
+    @Override
+    public ByteBuf copy(int index, int length) {
+      throw new UnsupportedOperationException("copy method is not supported");
+    }
+
+    @Override
+    public boolean release(int decrement) {
+      boolean released = super.release(decrement);
+      if (released) {
+        count.decrementAndGet();
+        size.addAndGet(-initialCapacity);
+      }
+      return released;
+    }
+
+  }
+
+  private class InnerAllocator extends PooledByteBufAllocator {
     private final PoolArena<ByteBuffer>[] directArenas;
     private final MemoryStatusThread statusThread;
-    private final Histogram largeBuffersHist;
-    private final Histogram normalBuffersHist;
     private final int chunkSize;
 
     public InnerAllocator() {
@@ -98,50 +142,6 @@ public InnerAllocator() {
       } else {
         statusThread = null;
       }
-      removeOldMetrics();
-
-      registry.register(METRIC_PREFIX + "normal.size", new Gauge<Long>() {
-        @Override
-        public Long getValue() {
-          return normalBufferSize.get();
-        }
-      });
-
-      registry.register(METRIC_PREFIX + "normal.count", new Gauge<Long>() {
-        @Override
-        public Long getValue() {
-          return normalBufferCount.get();
-        }
-      });
-
-      registry.register(METRIC_PREFIX + "huge.size", new Gauge<Long>() {
-        @Override
-        public Long getValue() {
-          return hugeBufferSize.get();
-        }
-      });
-
-      registry.register(METRIC_PREFIX + "huge.count", new Gauge<Long>() {
-        @Override
-        public Long getValue() {
-          return hugeBufferCount.get();
-        }
-      });
-
-      largeBuffersHist = registry.histogram(METRIC_PREFIX + "huge.hist");
-      normalBuffersHist = registry.histogram(METRIC_PREFIX + "normal.hist");
-
-    }
-
-
-    private synchronized void removeOldMetrics() {
-      registry.removeMatching(new MetricFilter() {
-        @Override
-        public boolean matches(String name, Metric metric) {
-          return name.startsWith("drill.allocator.");
-        }
-
-      });
     }
 
     private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCapacity) {
@@ -154,12 +154,11 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa
           // This is beyond chunk size so we'll allocate separately.
           ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity);
 
-          hugeBufferCount.incrementAndGet();
           hugeBufferSize.addAndGet(buf.capacity());
-          largeBuffersHist.update(buf.capacity());
-          // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
-          return new UnsafeDirectLittleEndian(new LargeBuffer(buf, hugeBufferSize, hugeBufferCount));
+          hugeBufferCount.incrementAndGet();
 
+          // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
+          return new AccountedUnsafeDirectLittleEndian(new LargeBuffer(buf), hugeBufferCount, hugeBufferSize);
         } else {
           // within chunk, use arena.
           ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity);
@@ -167,14 +166,14 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa
             fail();
           }
 
-          normalBuffersHist.update(buf.capacity());
-          if (ASSERT_ENABLED) {
-            normalBufferSize.addAndGet(buf.capacity());
-            normalBufferCount.incrementAndGet();
+          if (!ASSERT_ENABLED) {
+            return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf);
           }
 
-          return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount,
-              normalBufferSize);
+          normalBufferSize.addAndGet(buf.capacity());
+          normalBufferCount.incrementAndGet();
+
+          return new AccountedUnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount, normalBufferSize);
         }
 
       } else {
@@ -184,9 +183,10 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa
 
     private UnsupportedOperationException fail() {
       return new UnsupportedOperationException(
-          "Arrow requries that the JVM used supports access sun.misc.Unsafe.  This platform didn't provide that functionality.");
+          "Arrow requires that the JVM used supports access sun.misc.Unsafe.  This platform didn't provide that functionality.");
     }
 
+    @Override
     public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
       if (initialCapacity == 0 && maxCapacity == 0) {
         newDirectBuffer(initialCapacity, maxCapacity);
@@ -215,9 +215,8 @@ private void validate(int initialCapacity, int maxCapacity) {
     private class MemoryStatusThread extends Thread {
 
       public MemoryStatusThread() {
-        super("memory-status-logger");
+        super("allocation.logger");
         this.setDaemon(true);
-        this.setName("allocation.logger");
       }
 
       @Override
@@ -229,12 +228,11 @@ public void run() {
           } catch (InterruptedException e) {
             return;
           }
-
         }
       }
-
     }
 
+    @Override
     public String toString() {
       StringBuilder buf = new StringBuilder();
       buf.append(directArenas.length);
@@ -260,13 +258,4 @@ public String toString() {
 
 
   }
-
-  public static final boolean ASSERT_ENABLED;
-
-  static {
-    boolean isAssertEnabled = false;
-    assert isAssertEnabled = true;
-    ASSERT_ENABLED = isAssertEnabled;
-  }
-
 }
diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
index 023a6a2892b80..5ea176745f25e 100644
--- a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -18,8 +18,6 @@
 
 package io.netty.buffer;
 
-import io.netty.util.internal.PlatformDependent;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -32,7 +30,7 @@
  * The underlying class we use for little-endian access to memory. Is used underneath ArrowBufs to abstract away the
  * Netty classes and underlying Netty memory management.
  */
-public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
+public class UnsafeDirectLittleEndian extends WrappedByteBuf {
   private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
   private static final AtomicLong ID_GENERATOR = new AtomicLong(0);
 
@@ -40,35 +38,25 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
   private final AbstractByteBuf wrapped;
   private final long memoryAddress;
 
-  private final AtomicLong bufferCount;
-  private final AtomicLong bufferSize;
-  private final long initCap;
-
   UnsafeDirectLittleEndian(DuplicatedByteBuf buf) {
-    this(buf, true, null, null);
+    this(buf, true);
   }
 
   UnsafeDirectLittleEndian(LargeBuffer buf) {
-    this(buf, true, null, null);
+    this(buf, true);
   }
 
-  UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong bufferCount, AtomicLong bufferSize) {
-    this(buf, true, bufferCount, bufferSize);
+  UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf) {
+    this(buf, true);
 
   }
 
-  private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake, AtomicLong bufferCount, AtomicLong bufferSize) {
+  private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake) {
     super(buf);
     if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
       throw new IllegalStateException("Arrow only runs on LittleEndian systems.");
     }
 
-    this.bufferCount = bufferCount;
-    this.bufferSize = bufferSize;
-
-    // initCap is used if we're tracking memory release. If we're in non-debug mode, we'll skip this.
-    this.initCap = ASSERT_ENABLED ? buf.capacity() : -1;
-
     this.wrapped = buf;
     this.memoryAddress = buf.memoryAddress();
   }
@@ -244,16 +232,6 @@ public boolean release() {
     return release(1);
   }
 
-  @Override
-  public boolean release(int decrement) {
-    final boolean released = super.release(decrement);
-    if (ASSERT_ENABLED && released && bufferCount != null && bufferSize != null) {
-      bufferCount.decrementAndGet();
-      bufferSize.addAndGet(-initCap);
-    }
-    return released;
-  }
-
   @Override
   public int setBytes(int index, InputStream in, int length) throws IOException {
     wrapped.checkIndex(index, length);
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/Metrics.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
similarity index 58%
rename from java/memory/src/main/java/org/apache/arrow/memory/util/Metrics.java
rename to java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
index 5177a2478b53a..1b127f8181222 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/Metrics.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,26 +15,26 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.arrow.memory.util;
-
-import com.codahale.metrics.MetricRegistry;
-
-public class Metrics {
-
-  private Metrics() {
-
-  }
+package org.apache.arrow.memory;
 
-  private static class RegistryHolder {
-    public static final MetricRegistry REGISTRY;
-
-    static {
-      REGISTRY = new MetricRegistry();
+/**
+ * An allocation listener being notified for allocation/deallocation
+ *
+ * It is expected to be called from multiple threads and as such,
+ * provider should take care of making the implementation thread-safe
+ */
+public interface AllocationListener {
+  public static final AllocationListener NOOP = new AllocationListener() {
+    @Override
+    public void onAllocation(long size) {
     }
+  };
 
-  }
+  /**
+   * Called each time a new buffer is allocated
+   *
+   * @param size the buffer size being allocated
+   */
+  void onAllocation(long size);
 
-  public static MetricRegistry getInstance() {
-    return RegistryHolder.REGISTRY;
-  }
-}
\ No newline at end of file
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index 43ee9c108d902..f15bb8a40fa01 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -18,9 +18,6 @@
 package org.apache.arrow.memory;
 
 import static org.apache.arrow.memory.BaseAllocator.indent;
-import io.netty.buffer.ArrowBuf;
-import io.netty.buffer.PooledByteBufAllocatorL;
-import io.netty.buffer.UnsafeDirectLittleEndian;
 
 import java.util.IdentityHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -31,10 +28,13 @@
 import org.apache.arrow.memory.BaseAllocator.Verbosity;
 import org.apache.arrow.memory.util.AutoCloseableLock;
 import org.apache.arrow.memory.util.HistoricalLog;
-import org.apache.arrow.memory.util.Metrics;
 
 import com.google.common.base.Preconditions;
 
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.PooledByteBufAllocatorL;
+import io.netty.buffer.UnsafeDirectLittleEndian;
+
 /**
  * Manages the relationship between one or more allocators and a particular UDLE. Ensures that one allocator owns the
  * memory that multiple allocators may be referencing. Manages a BufferLedger between each of its associated allocators.
@@ -56,7 +56,10 @@ public class AllocationManager {
 
   private static final AtomicLong MANAGER_ID_GENERATOR = new AtomicLong(0);
   private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0);
-  static final PooledByteBufAllocatorL INNER_ALLOCATOR = new PooledByteBufAllocatorL(Metrics.getInstance());
+  private static final PooledByteBufAllocatorL INNER_ALLOCATOR = new PooledByteBufAllocatorL();
+
+  static final UnsafeDirectLittleEndian EMPTY = INNER_ALLOCATOR.empty;
+  static final long CHUNK_SIZE = INNER_ALLOCATOR.getChunkSize();
 
   private final RootAllocator root;
   private final long allocatorManagerId = MANAGER_ID_GENERATOR.incrementAndGet();
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index dbb0705045c35..9edafbce082cb 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -21,7 +21,6 @@
 import java.util.IdentityHashMap;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.arrow.memory.AllocationManager.BufferLedger;
 import org.apache.arrow.memory.util.AssertionUtil;
@@ -37,14 +36,12 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato
 
   public static final String DEBUG_ALLOCATOR = "arrow.memory.debug.allocator";
 
-  private static final AtomicLong ID_GENERATOR = new AtomicLong(0);
-  private static final int CHUNK_SIZE = AllocationManager.INNER_ALLOCATOR.getChunkSize();
-
   public static final int DEBUG_LOG_LENGTH = 6;
   public static final boolean DEBUG = AssertionUtil.isAssertionsEnabled()
       || Boolean.parseBoolean(System.getProperty(DEBUG_ALLOCATOR, "false"));
   private final Object DEBUG_LOCK = DEBUG ? new Object() : null;
 
+  private final AllocationListener listener;
   private final BaseAllocator parentAllocator;
   private final ArrowByteBufAllocator thisAsByteBufAllocator;
   private final IdentityHashMap<BaseAllocator, Object> childAllocators;
@@ -61,13 +58,32 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato
   private final IdentityHashMap<Reservation, Object> reservations;
   private final HistoricalLog historicalLog;
 
+  protected BaseAllocator(
+      final AllocationListener listener,
+      final String name,
+      final long initReservation,
+      final long maxAllocation) throws OutOfMemoryException {
+    this(listener, null, name, initReservation, maxAllocation);
+  }
+
   protected BaseAllocator(
       final BaseAllocator parentAllocator,
       final String name,
       final long initReservation,
       final long maxAllocation) throws OutOfMemoryException {
+    this(parentAllocator.listener, parentAllocator, name, initReservation, maxAllocation);
+  }
+
+  private BaseAllocator(
+      final AllocationListener listener,
+      final BaseAllocator parentAllocator,
+      final String name,
+      final long initReservation,
+      final long maxAllocation) throws OutOfMemoryException {
     super(parentAllocator, initReservation, maxAllocation);
 
+    this.listener = listener;
+
     if (parentAllocator != null) {
       this.root = parentAllocator.root;
       empty = parentAllocator.empty;
@@ -192,7 +208,7 @@ public ArrowBuf buffer(final int initialRequestSize) {
   private ArrowBuf createEmpty(){
     assertOpen();
 
-    return new ArrowBuf(new AtomicInteger(), null, AllocationManager.INNER_ALLOCATOR.empty, null, null, 0, 0, true);
+    return new ArrowBuf(new AtomicInteger(), null, AllocationManager.EMPTY, null, null, 0, 0, true);
   }
 
   @Override
@@ -206,7 +222,7 @@ public ArrowBuf buffer(final int initialRequestSize, BufferManager manager) {
     }
 
     // round to next largest power of two if we're within a chunk since that is how our allocator operates
-    final int actualRequestSize = initialRequestSize < CHUNK_SIZE ?
+    final int actualRequestSize = initialRequestSize < AllocationManager.CHUNK_SIZE ?
         nextPowerOfTwo(initialRequestSize)
         : initialRequestSize;
     AllocationOutcome outcome = this.allocateBytes(actualRequestSize);
@@ -218,6 +234,7 @@ public ArrowBuf buffer(final int initialRequestSize, BufferManager manager) {
     try {
       ArrowBuf buffer = bufferWithoutReservation(actualRequestSize, manager);
       success = true;
+      listener.onAllocation(actualRequestSize);
       return buffer;
     } finally {
       if (!success) {
@@ -405,6 +422,7 @@ private ArrowBuf allocate(int nBytes) {
       try {
         final ArrowBuf arrowBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);
 
+        listener.onAllocation(nBytes);
         if (DEBUG) {
           historicalLog.recordEvent("allocate() => %s", String.format("ArrowBuf[%d]", arrowBuf.getId()));
         }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
index 571fc37577209..57a2c0cdae8d8 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
@@ -24,9 +24,12 @@
  * tree of descendant child allocators.
  */
 public class RootAllocator extends BaseAllocator {
-
   public RootAllocator(final long limit) {
-    super(null, "ROOT", 0, limit);
+    this(AllocationListener.NOOP, limit);
+  }
+
+  public RootAllocator(final AllocationListener listener, final long limit) {
+    super(listener, "ROOT", 0, limit);
   }
 
   /**
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/Pointer.java b/java/memory/src/main/java/org/apache/arrow/memory/util/Pointer.java
deleted file mode 100644
index 58ab13b0a16ab..0000000000000
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/Pointer.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.memory.util;
-
-public class Pointer<T> {
-  public T value;
-
-  public Pointer(){}
-
-  public Pointer(T value){
-    this.value = value;
-  }
-}

From 5ffbda1b408951cb5cf49008920f1054544148d3 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 13 Jan 2017 08:46:48 -0500
Subject: [PATCH 0271/1644] ARROW-479: Python: Test for expected schema in
 Pandas conversion

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #281 from xhochy/ARROW-479 and squashes the following commits:

acd9abd [Uwe L. Korn] Use arrow::timestamp()
43dba37 [Uwe L. Korn] Fix tests
7a3f5b8 [Uwe L. Korn] ARROW-479: Python: Test for expected schema in Pandas conversion
---
 python/pyarrow/includes/libarrow.pxd         |  2 +
 python/pyarrow/includes/pyarrow.pxd          |  4 +-
 python/pyarrow/schema.pyx                    | 38 +++++++++-
 python/pyarrow/tests/test_convert_builtin.py |  2 +-
 python/pyarrow/tests/test_convert_pandas.py  | 77 ++++++++++++++------
 python/pyarrow/tests/test_parquet.py         |  2 +-
 python/src/pyarrow/helpers.cc                |  3 -
 7 files changed, 97 insertions(+), 31 deletions(-)

diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index d1970e5a2c8f1..8cfaaf72bf16f 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -60,6 +60,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_string ToString()
 
+    shared_ptr[CDataType] timestamp(TimeUnit unit)
+
     cdef cppclass MemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
 
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index dc6ccd2025932..901e6c9457dfa 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -19,13 +19,15 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CTable,
-                                        CDataType, CStatus, Type, MemoryPool)
+                                        CDataType, CStatus, Type, MemoryPool,
+                                        TimeUnit)
 
 cimport pyarrow.includes.libarrow_io as arrow_io
 
 
 cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     shared_ptr[CDataType] GetPrimitiveType(Type type)
+    shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
     CStatus ConvertPySequence(object obj, shared_ptr[CArray]* out)
 
     CStatus PandasToArrow(MemoryPool* pool, object ao,
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index d91ae7cb3b193..f6a1a10c8dd5c 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -23,8 +23,20 @@
 # cython: embedsignature = True
 
 from pyarrow.compat import frombytes, tobytes
-from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
+                                        Type_NA, Type_BOOL,
+                                        Type_UINT8, Type_INT8,
+                                        Type_UINT16, Type_INT16,
+                                        Type_UINT32, Type_INT32,
+                                        Type_UINT64, Type_INT64,
+                                        Type_TIMESTAMP, Type_DATE,
+                                        Type_FLOAT, Type_DOUBLE,
+                                        Type_STRING, Type_BINARY, 
+                                        TimeUnit_SECOND, TimeUnit_MILLI,
+                                        TimeUnit_MICRO, TimeUnit_NANO,
+                                        Type, TimeUnit)
 cimport pyarrow.includes.pyarrow as pyarrow
+cimport pyarrow.includes.libarrow as libarrow
 
 cimport cpython
 
@@ -197,8 +209,28 @@ def uint64():
 def int64():
     return primitive_type(Type_INT64)
 
-def timestamp():
-    return primitive_type(Type_TIMESTAMP)
+cdef dict _timestamp_type_cache = {}
+
+def timestamp(unit_str):
+    cdef TimeUnit unit
+    if unit_str == "s":
+        unit = TimeUnit_SECOND
+    elif unit_str == 'ms':
+        unit = TimeUnit_MILLI
+    elif unit_str == 'us':
+        unit = TimeUnit_MICRO
+    elif unit_str == 'ns':
+        unit = TimeUnit_NANO
+    else:
+        raise TypeError('Invalid TimeUnit string')
+
+    if unit in _timestamp_type_cache:
+        return _timestamp_type_cache[unit]
+
+    cdef DataType out = DataType()
+    out.init(libarrow.timestamp(unit))
+    _timestamp_type_cache[unit] = out
+    return out
 
 def date():
     return primitive_type(Type_DATE)
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 61167422de93c..72e438910159f 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -112,7 +112,7 @@ def test_timestamp(self):
         ]
         arr = pyarrow.from_pylist(data)
         assert len(arr) == 4
-        assert arr.type == pyarrow.timestamp()
+        assert arr.type == pyarrow.timestamp('us')
         assert arr.null_count == 1
         assert arr[0].as_py() == datetime.datetime(2007, 7, 13, 1,
                                                    23, 34, 123456)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 12e7a08d795a2..261eaa85657ed 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -60,65 +60,79 @@ def tearDown(self):
         pass
 
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
-                                timestamps_to_ms=False):
+                                timestamps_to_ms=False, expected_schema=None):
         table = A.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms)
         result = table.to_pandas(nthreads=nthreads)
+        if expected_schema:
+            assert table.schema.equals(expected_schema)
         if expected is None:
             expected = df
         tm.assert_frame_equal(result, expected)
 
     def test_float_no_nulls(self):
         data = {}
-        numpy_dtypes = ['f4', 'f8']
+        fields = []
+        dtypes = [('f4', A.float_()), ('f8', A.double())]
         num_values = 100
 
-        for dtype in numpy_dtypes:
+        for numpy_dtype, arrow_dtype in dtypes:
             values = np.random.randn(num_values)
-            data[dtype] = values.astype(dtype)
+            data[numpy_dtype] = values.astype(numpy_dtype)
+            fields.append(A.Field.from_py(numpy_dtype, arrow_dtype))
 
         df = pd.DataFrame(data)
-        self._check_pandas_roundtrip(df)
+        schema = A.Schema.from_fields(fields)
+        self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_float_nulls(self):
         num_values = 100
 
         null_mask = np.random.randint(0, 10, size=num_values) < 3
-        dtypes = ['f4', 'f8']
+        dtypes = [('f4', A.float_()), ('f8', A.double())]
+        names = ['f4', 'f8']
         expected_cols = []
 
         arrays = []
-        for name in dtypes:
+        fields = []
+        for name, arrow_dtype in dtypes:
             values = np.random.randn(num_values).astype(name)
 
             arr = A.from_pandas_series(values, null_mask)
             arrays.append(arr)
-
+            fields.append(A.Field.from_py(name, arrow_dtype))
             values[null_mask] = np.nan
 
             expected_cols.append(values)
 
-        ex_frame = pd.DataFrame(dict(zip(dtypes, expected_cols)),
-                                columns=dtypes)
+        ex_frame = pd.DataFrame(dict(zip(names, expected_cols)),
+                                columns=names)
 
-        table = A.Table.from_arrays(dtypes, arrays)
+        table = A.Table.from_arrays(names, arrays)
+        assert table.schema.equals(A.Schema.from_fields(fields))
         result = table.to_pandas()
         tm.assert_frame_equal(result, ex_frame)
 
     def test_integer_no_nulls(self):
         data = {}
+        fields = []
 
-        numpy_dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8']
+        numpy_dtypes = [('i1', A.int8()), ('i2', A.int16()),
+                        ('i4', A.int32()), ('i8', A.int64()),
+                        ('u1', A.uint8()), ('u2', A.uint16()),
+                        ('u4', A.uint32()), ('u8', A.uint64())]
         num_values = 100
 
-        for dtype in numpy_dtypes:
+        for dtype, arrow_dtype in numpy_dtypes:
             info = np.iinfo(dtype)
             values = np.random.randint(info.min,
                                        min(info.max, np.iinfo('i8').max),
                                        size=num_values)
             data[dtype] = values.astype(dtype)
+            fields.append(A.Field.from_py(dtype, arrow_dtype))
 
         df = pd.DataFrame(data)
-        self._check_pandas_roundtrip(df)
+        schema = A.Schema.from_fields(fields)
+        self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_integer_with_nulls(self):
         # pandas requires upcast to float dtype
@@ -155,7 +169,9 @@ def test_boolean_no_nulls(self):
         np.random.seed(0)
 
         df = pd.DataFrame({'bools': np.random.randn(num_values) > 0})
-        self._check_pandas_roundtrip(df)
+        field = A.Field.from_py('bools', A.bool_())
+        schema = A.Schema.from_fields([field])
+        self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_boolean_nulls(self):
         # pandas requires upcast to object dtype
@@ -170,9 +186,12 @@ def test_boolean_nulls(self):
         expected = values.astype(object)
         expected[mask] = None
 
+        field = A.Field.from_py('bools', A.bool_())
+        schema = A.Schema.from_fields([field])
         ex_frame = pd.DataFrame({'bools': expected})
 
         table = A.Table.from_arrays(['bools'], [arr])
+        assert table.schema.equals(schema)
         result = table.to_pandas()
 
         tm.assert_frame_equal(result, ex_frame)
@@ -180,14 +199,18 @@ def test_boolean_nulls(self):
     def test_boolean_object_nulls(self):
         arr = np.array([False, None, True] * 100, dtype=object)
         df = pd.DataFrame({'bools': arr})
-        self._check_pandas_roundtrip(df)
+        field = A.Field.from_py('bools', A.bool_())
+        schema = A.Schema.from_fields([field])
+        self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_unicode(self):
         repeats = 1000
         values = [u'foo', None, u'bar', u'mañana', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
+        field = A.Field.from_py('strings', A.string())
+        schema = A.Schema.from_fields([field])
 
-        self._check_pandas_roundtrip(df)
+        self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_bytes_to_binary(self):
         values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan]
@@ -208,7 +231,9 @@ def test_timestamps_notimezone_no_nulls(self):
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
             })
-        self._check_pandas_roundtrip(df, timestamps_to_ms=True)
+        field = A.Field.from_py('datetime64', A.timestamp('ms'))
+        schema = A.Schema.from_fields([field])
+        self._check_pandas_roundtrip(df, timestamps_to_ms=True, expected_schema=schema)
 
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -217,7 +242,9 @@ def test_timestamps_notimezone_no_nulls(self):
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
             })
-        self._check_pandas_roundtrip(df, timestamps_to_ms=False)
+        field = A.Field.from_py('datetime64', A.timestamp('ns'))
+        schema = A.Schema.from_fields([field])
+        self._check_pandas_roundtrip(df, timestamps_to_ms=False, expected_schema=schema)
 
     def test_timestamps_notimezone_nulls(self):
         df = pd.DataFrame({
@@ -227,8 +254,9 @@ def test_timestamps_notimezone_nulls(self):
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
             })
-        df.info()
-        self._check_pandas_roundtrip(df, timestamps_to_ms=True)
+        field = A.Field.from_py('datetime64', A.timestamp('ms'))
+        schema = A.Schema.from_fields([field])
+        self._check_pandas_roundtrip(df, timestamps_to_ms=True, expected_schema=schema)
 
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -237,7 +265,9 @@ def test_timestamps_notimezone_nulls(self):
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
             })
-        self._check_pandas_roundtrip(df, timestamps_to_ms=False)
+        field = A.Field.from_py('datetime64', A.timestamp('ns'))
+        schema = A.Schema.from_fields([field])
+        self._check_pandas_roundtrip(df, timestamps_to_ms=False, expected_schema=schema)
 
     def test_date(self):
         df = pd.DataFrame({
@@ -246,6 +276,9 @@ def test_date(self):
                      datetime.date(1970, 1, 1),
                      datetime.date(2040, 2, 26)]})
         table = A.Table.from_pandas(df)
+        field = A.Field.from_py('date', A.date())
+        schema = A.Schema.from_fields([field])
+        assert table.schema.equals(schema)
         result = table.to_pandas()
         expected = df.copy()
         expected['date'] = pd.to_datetime(df['date'])
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index ad4bc580e8b1c..e1571557d9aa7 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -244,7 +244,7 @@ def test_parquet_metadata_api():
     a_table = A.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
-    pq.write_table(a_table, buf, compression='snappy', version='2.0')
+    pq.write_table(a_table, buf, compression='SNAPPY', version='2.0')
 
     buf.seek(0)
     fileh = pq.ParquetFile(buf)
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
index 3f650326e09aa..78fad165ac8e6 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/python/src/pyarrow/helpers.cc
@@ -41,9 +41,6 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
       GET_PRIMITIVE_TYPE(UINT64, uint64);
       GET_PRIMITIVE_TYPE(INT64, int64);
       GET_PRIMITIVE_TYPE(DATE, date);
-    case Type::TIMESTAMP:
-      return arrow::timestamp(arrow::TimeUnit::MICRO);
-      break;
       GET_PRIMITIVE_TYPE(BOOL, boolean);
       GET_PRIMITIVE_TYPE(FLOAT, float32);
       GET_PRIMITIVE_TYPE(DOUBLE, float64);

From ad0e57d23257462b9933745949d54ca729da537e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 13 Jan 2017 08:50:14 -0500
Subject: [PATCH 0272/1644] ARROW-472: [Python] Expose more C++ IO interfaces.
 Add equals methods to Parquet schemas. Pass Parquet metadata separately in
 reader

Also includes ARROW-471, ARROW-441.

Needed to compare file schemas easily.

Requires PARQUET-830

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #280 from wesm/ARROW-472 and squashes the following commits:

1c5e27c [Wes McKinney] Name static const variables constexpr instead
25c5c90 [Wes McKinney] Add some tests for io.OSFile
1d22428 [Wes McKinney] Add some memory map Python unit tests
5268b6c [Wes McKinney] Add untested wrapper for operating system files
fd52153 [Wes McKinney] Add unit test for passing metadata down
2316e64 [Wes McKinney] Expose MemoryMappedFile in pyarrow.io, expand parquet::arrow::OpenFile to take metadata, props parameters
a2ce247 [Wes McKinney] Add equals methods to Parquet Schema and ColumnSchema objects
---
 cpp/src/arrow/io/file.cc                |  45 +++++---
 cpp/src/arrow/io/file.h                 |   2 +
 cpp/src/arrow/io/io-file-test.cc        |   4 +-
 python/pyarrow/_parquet.pxd             |  18 +++-
 python/pyarrow/_parquet.pyx             |  37 ++++---
 python/pyarrow/compat.py                |  26 +++++
 python/pyarrow/includes/libarrow_io.pxd |  10 ++
 python/pyarrow/io.pxd                   |   3 +-
 python/pyarrow/io.pyx                   | 125 ++++++++++++++++++++---
 python/pyarrow/parquet.py               |   2 +-
 python/pyarrow/table.pyx                |   2 +-
 python/pyarrow/tests/test_io.py         | 130 +++++++++++++++++++++++-
 python/pyarrow/tests/test_parquet.py    |  52 ++++++++--
 13 files changed, 396 insertions(+), 60 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 0fb13ea22e39f..1de6efa4d811f 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -188,6 +188,8 @@ static inline Status FileOpenWriteable(
   memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
 
   int oflag = _O_CREAT | _O_BINARY;
+  int sh_flag = _S_IWRITE;
+  if (!write_only) { sh_flag |= _S_IREAD; }
 
   if (truncate) { oflag |= _O_TRUNC; }
 
@@ -197,7 +199,7 @@ static inline Status FileOpenWriteable(
     oflag |= _O_RDWR;
   }
 
-  errno_actual = _wsopen_s(fd, wpath.data(), oflag, _SH_DENYNO, _S_IWRITE);
+  errno_actual = _wsopen_s(fd, wpath.data(), oflag, _SH_DENYNO, sh_flag);
   ret = *fd;
 
 #else
@@ -319,7 +321,7 @@ class OSFile {
     RETURN_NOT_OK(FileOpenWriteable(path, write_only, !append, &fd_));
     path_ = path;
     is_open_ = true;
-    mode_ = write_only ? FileMode::READ : FileMode::READWRITE;
+    mode_ = write_only ? FileMode::WRITE : FileMode::READWRITE;
 
     if (append) {
       RETURN_NOT_OK(FileGetSize(fd_, &size_));
@@ -352,7 +354,7 @@ class OSFile {
   }
 
   Status Seek(int64_t pos) {
-    if (pos > size_) { pos = size_; }
+    if (pos < 0) { return Status::Invalid("Invalid position"); }
     return FileSeek(fd_, pos);
   }
 
@@ -523,17 +525,24 @@ class MemoryMappedFile::MemoryMappedFileImpl : public OSFile {
   }
 
   Status Open(const std::string& path, FileMode::type mode) {
-    int prot_flags = PROT_READ;
+    int prot_flags;
+    int map_mode;
 
     if (mode != FileMode::READ) {
-      prot_flags |= PROT_WRITE;
-      const bool append = true;
-      RETURN_NOT_OK(OSFile::OpenWriteable(path, append, mode == FileMode::WRITE));
+      // Memory mapping has permission failures if PROT_READ not set
+      prot_flags = PROT_READ | PROT_WRITE;
+      map_mode = MAP_SHARED;
+      constexpr bool append = true;
+      constexpr bool write_only = false;
+      RETURN_NOT_OK(OSFile::OpenWriteable(path, append, write_only));
+      mode_ = mode;
     } else {
+      prot_flags = PROT_READ;
+      map_mode = MAP_PRIVATE;  // Changes are not to be committed back to the file
       RETURN_NOT_OK(OSFile::OpenReadable(path));
     }
 
-    void* result = mmap(nullptr, size_, prot_flags, MAP_SHARED, fd(), 0);
+    void* result = mmap(nullptr, size_, prot_flags, map_mode, fd(), 0);
     if (result == MAP_FAILED) {
       std::stringstream ss;
       ss << "Memory mapping file failed, errno: " << errno;
@@ -548,16 +557,14 @@ class MemoryMappedFile::MemoryMappedFileImpl : public OSFile {
   int64_t size() const { return size_; }
 
   Status Seek(int64_t position) {
-    if (position < 0 || position >= size_) {
-      return Status::Invalid("position is out of bounds");
-    }
+    if (position < 0) { return Status::Invalid("position is out of bounds"); }
     position_ = position;
     return Status::OK();
   }
 
   int64_t position() { return position_; }
 
-  void advance(int64_t nbytes) { position_ = std::min(size_, position_ + nbytes); }
+  void advance(int64_t nbytes) { position_ = position_ + nbytes; }
 
   uint8_t* data() { return data_; }
 
@@ -611,16 +618,18 @@ Status MemoryMappedFile::Close() {
 }
 
 Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-  nbytes = std::min(nbytes, impl_->size() - impl_->position());
-  std::memcpy(out, impl_->head(), nbytes);
+  nbytes = std::max<int64_t>(0, std::min(nbytes, impl_->size() - impl_->position()));
+  if (nbytes > 0) { std::memcpy(out, impl_->head(), nbytes); }
   *bytes_read = nbytes;
   impl_->advance(nbytes);
   return Status::OK();
 }
 
 Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  nbytes = std::min(nbytes, impl_->size() - impl_->position());
-  *out = std::make_shared<Buffer>(impl_->head(), nbytes);
+  nbytes = std::max<int64_t>(0, std::min(nbytes, impl_->size() - impl_->position()));
+
+  const uint8_t* data = nbytes > 0 ? impl_->head() : nullptr;
+  *out = std::make_shared<Buffer>(data, nbytes);
   impl_->advance(nbytes);
   return Status::OK();
 }
@@ -655,5 +664,9 @@ Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
   return Status::OK();
 }
 
+int MemoryMappedFile::file_descriptor() const {
+  return impl_->fd();
+}
+
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 9ca9c540e7c22..2387232b2157a 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -127,6 +127,8 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   // @return: the size in bytes of the memory source
   Status GetSize(int64_t* size) override;
 
+  int file_descriptor() const;
+
  private:
   explicit MemoryMappedFile(FileMode::type mode);
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 821e71d0212f6..20cd04748f019 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -209,8 +209,8 @@ TEST_F(TestReadableFile, SeekTellSize) {
   ASSERT_OK(file_->Seek(100));
   ASSERT_OK(file_->Tell(&position));
 
-  // now at EOF
-  ASSERT_EQ(8, position);
+  // Can seek past end of file
+  ASSERT_EQ(100, position);
 
   int64_t size;
   ASSERT_OK(file_->GetSize(&size));
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 7e49e9e834b77..cf1da1c3a9e52 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -99,8 +99,9 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
         ParquetVersion_V2" parquet::ParquetVersion::PARQUET_2_0"
 
     cdef cppclass ColumnDescriptor:
-        shared_ptr[ColumnPath] path()
+        c_bool Equals(const ColumnDescriptor& other)
 
+        shared_ptr[ColumnPath] path()
         int16_t max_definition_level()
         int16_t max_repetition_level()
 
@@ -115,6 +116,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
         const ColumnDescriptor* Column(int i)
         shared_ptr[Node] schema()
         GroupNode* group()
+        c_bool Equals(const SchemaDescriptor& other)
         int num_columns()
 
 
@@ -163,8 +165,18 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         unique_ptr[CRowGroupMetaData] RowGroup(int i)
         const SchemaDescriptor* schema()
 
+    cdef cppclass ReaderProperties:
+        pass
+
+    ReaderProperties default_reader_properties()
+
     cdef cppclass ParquetFileReader:
-        # TODO: Some default arguments are missing
+        @staticmethod
+        unique_ptr[ParquetFileReader] Open(
+            const shared_ptr[ReadableFileInterface]& file,
+            const ReaderProperties& props,
+            const shared_ptr[CFileMetaData]& metadata)
+
         @staticmethod
         unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
         shared_ptr[CFileMetaData] metadata();
@@ -193,6 +205,8 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
 cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
     CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
                      MemoryPool* allocator,
+                     const ReaderProperties& properties,
+                     const shared_ptr[CFileMetaData]& metadata,
                      unique_ptr[FileReader]* reader)
 
     cdef cppclass FileReader:
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 30e3de417a827..867fc4cfecbd6 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -19,8 +19,9 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from pyarrow._parquet cimport *
+from cython.operator cimport dereference as deref
 
+from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.libarrow_io cimport (ReadableFileInterface, OutputStream,
                                            FileOutputStream)
@@ -196,6 +197,12 @@ cdef class Schema:
     def __getitem__(self, i):
         return self.column(i)
 
+    def equals(self, Schema other):
+        """
+        Returns True if the Parquet schemas are equal
+        """
+        return self.schema.Equals(deref(other.schema))
+
     def column(self, i):
         if i < 0 or i >= len(self):
             raise IndexError('{0} out of bounds'.format(i))
@@ -217,6 +224,12 @@ cdef class ColumnSchema:
         self.parent = schema
         self.descr = schema.schema.Column(i)
 
+    def equals(self, ColumnSchema other):
+        """
+        Returns True if the column schemas are equal
+        """
+        return self.descr.Equals(deref(other.descr))
+
     def __repr__(self):
         physical_type = self.physical_type
         logical_type = self.logical_type
@@ -337,24 +350,20 @@ cdef class ParquetReader:
         self.allocator = default_memory_pool()
         self._metadata = None
 
-    def open(self, object source):
+    def open(self, object source, FileMetaData metadata=None):
         cdef:
             shared_ptr[ReadableFileInterface] rd_handle
+            shared_ptr[CFileMetaData] c_metadata
+            ReaderProperties properties = default_reader_properties()
             c_string path
 
-        if isinstance(source, six.string_types):
-            path = tobytes(source)
-
-            # Must be in one expression to avoid calling std::move which is not
-            # possible in Cython (due to missing rvalue support)
+        if metadata is not None:
+            c_metadata = metadata.sp_metadata
 
-            # TODO(wesm): ParquetFileReader::OpenFile can throw?
-            self.reader = unique_ptr[FileReader](
-                new FileReader(default_memory_pool(),
-                               ParquetFileReader.OpenFile(path)))
-        else:
-            get_reader(source, &rd_handle)
-            check_status(OpenFile(rd_handle, self.allocator, &self.reader))
+        get_reader(source, &rd_handle)
+        with nogil:
+            check_status(OpenFile(rd_handle, self.allocator, properties,
+                                  c_metadata, &self.reader))
 
     @property
     def metadata(self):
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 2dfdb5041d13e..9148be7d9f8ad 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -54,6 +54,10 @@ def dict_values(x):
     range = xrange
     long = long
 
+    def guid():
+        from uuid import uuid4
+        return uuid4().get_hex()
+
     def u(s):
         return unicode(s, "unicode_escape")
 
@@ -76,6 +80,10 @@ def dict_values(x):
     from decimal import Decimal
     range = range
 
+    def guid():
+        from uuid import uuid4
+        return uuid4().hex
+
     def u(s):
         return s
 
@@ -89,6 +97,24 @@ def frombytes(o):
         return o.decode('utf8')
 
 
+def encode_file_path(path):
+    import os
+    # Windows requires utf-16le encoding for unicode file names
+    if isinstance(path, unicode_type):
+        if os.name == 'nt':
+            # try:
+            #     encoded_path = path.encode('ascii')
+            # except:
+            encoded_path = path.encode('utf-16le')
+        else:
+            # POSIX systems can handle utf-8
+            encoded_path = path.encode('utf-8')
+    else:
+        encoded_path = path
+
+    return encoded_path
+
+
 integer_types = six.integer_types + (np.integer,)
 
 __all__ = []
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 99f88adf81d2b..6b141a3e76f09 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -69,6 +69,8 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
 
 
 cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
+
+
     cdef cppclass FileOutputStream(OutputStream):
         @staticmethod
         CStatus Open(const c_string& path, shared_ptr[FileOutputStream]* file)
@@ -85,6 +87,14 @@ cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
 
         int file_descriptor()
 
+    cdef cppclass CMemoryMappedFile" arrow::io::MemoryMappedFile"\
+        (ReadWriteFileInterface):
+        @staticmethod
+        CStatus Open(const c_string& path, FileMode mode,
+                     shared_ptr[CMemoryMappedFile]* file)
+
+        int file_descriptor()
+
 
 cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
     CStatus HaveLibHdfs()
diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
index 02265d0a68eb1..fffc7c596db76 100644
--- a/python/pyarrow/io.pxd
+++ b/python/pyarrow/io.pxd
@@ -32,7 +32,8 @@ cdef class NativeFile:
     cdef:
         shared_ptr[ReadableFileInterface] rd_file
         shared_ptr[OutputStream] wr_file
-        bint is_readonly
+        bint is_readable
+        bint is_writeable
         bint is_open
         bint own_file
 
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index b62de6cdd462c..2d8e4e8f34242 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -27,12 +27,13 @@ from pyarrow.includes.libarrow cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 from pyarrow.includes.libarrow_io cimport *
 
-from pyarrow.compat import frombytes, tobytes
+from pyarrow.compat import frombytes, tobytes, encode_file_path
 from pyarrow.error cimport check_status
 
 cimport cpython as cp
 
 import re
+import six
 import sys
 import threading
 import time
@@ -42,6 +43,7 @@ cdef extern from "Python.h":
     PyObject* PyBytes_FromStringAndSizeNative" PyBytes_FromStringAndSize"(
         char *v, Py_ssize_t len) except NULL
 
+
 cdef class NativeFile:
 
     def __cinit__(self):
@@ -61,7 +63,7 @@ cdef class NativeFile:
     def close(self):
         if self.is_open:
             with nogil:
-                if self.is_readonly:
+                if self.is_readable:
                     check_status(self.rd_file.get().Close())
                 else:
                     check_status(self.wr_file.get().Close())
@@ -76,15 +78,15 @@ cdef class NativeFile:
         file[0] = <shared_ptr[OutputStream]> self.wr_file
 
     def _assert_readable(self):
-        if not self.is_readonly:
+        if not self.is_readable:
             raise IOError("only valid on readonly files")
 
         if not self.is_open:
             raise IOError("file not open")
 
     def _assert_writeable(self):
-        if self.is_readonly:
-            raise IOError("only valid on writeonly files")
+        if not self.is_writeable:
+            raise IOError("only valid on writeable files")
 
         if not self.is_open:
             raise IOError("file not open")
@@ -99,7 +101,7 @@ cdef class NativeFile:
     def tell(self):
         cdef int64_t position
         with nogil:
-            if self.is_readonly:
+            if self.is_readable:
                 check_status(self.rd_file.get().Tell(&position))
             else:
                 check_status(self.wr_file.get().Tell(&position))
@@ -137,7 +139,7 @@ cdef class NativeFile:
         self._assert_readable()
 
         # Allocate empty write space
-        obj = PyBytes_FromStringAndSizeNative(NULL, nbytes)
+        obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes)
 
         cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AS_STRING(<object> obj)
         with nogil:
@@ -179,16 +181,100 @@ cdef class PythonFileInterface(NativeFile):
 
         if mode.startswith('w'):
             self.wr_file.reset(new pyarrow.PyOutputStream(handle))
-            self.is_readonly = 0
+            self.is_readable = 0
+            self.is_writeable = 1
         elif mode.startswith('r'):
             self.rd_file.reset(new pyarrow.PyReadableFile(handle))
-            self.is_readonly = 1
+            self.is_readable = 1
+            self.is_writeable = 0
+        else:
+            raise ValueError('Invalid file mode: {0}'.format(mode))
+
+        self.is_open = True
+
+
+cdef class MemoryMappedFile(NativeFile):
+    """
+    Supports 'r', 'r+w', 'w' modes
+    """
+    cdef:
+        object path
+
+    def __cinit__(self, path, mode='r'):
+        self.path = path
+
+        cdef:
+            FileMode c_mode
+            shared_ptr[CMemoryMappedFile] handle
+            c_string c_path = encode_file_path(path)
+
+        self.is_readable = self.is_writeable = 0
+
+        if mode in ('r', 'rb'):
+            c_mode = FileMode_READ
+            self.is_readable = 1
+        elif mode in ('w', 'wb'):
+            c_mode = FileMode_WRITE
+            self.is_writeable = 1
+        elif mode == 'r+w':
+            c_mode = FileMode_READWRITE
+            self.is_readable = 1
+            self.is_writeable = 1
         else:
             raise ValueError('Invalid file mode: {0}'.format(mode))
 
+        check_status(CMemoryMappedFile.Open(c_path, c_mode, &handle))
+
+        self.wr_file = <shared_ptr[OutputStream]> handle
+        self.rd_file = <shared_ptr[ReadableFileInterface]> handle
         self.is_open = True
 
 
+cdef class OSFile(NativeFile):
+    """
+    Supports 'r', 'w' modes
+    """
+    cdef:
+        object path
+
+    def __cinit__(self, path, mode='r'):
+        self.path = path
+
+        cdef:
+            FileMode c_mode
+            shared_ptr[Readable] handle
+            c_string c_path = encode_file_path(path)
+
+        self.is_readable = self.is_writeable = 0
+
+        if mode in ('r', 'rb'):
+            self._open_readable(c_path)
+        elif mode in ('w', 'wb'):
+            self._open_writeable(c_path)
+        else:
+            raise ValueError('Invalid file mode: {0}'.format(mode))
+
+        self.is_open = True
+
+    cdef _open_readable(self, c_string path):
+        cdef shared_ptr[ReadableFile] handle
+
+        with nogil:
+            check_status(ReadableFile.Open(path, pyarrow.get_memory_pool(),
+                                           &handle))
+
+        self.is_readable = 1
+        self.rd_file = <shared_ptr[ReadableFileInterface]> handle
+
+    cdef _open_writeable(self, c_string path):
+        cdef shared_ptr[FileOutputStream] handle
+
+        with nogil:
+            check_status(FileOutputStream.Open(path, &handle))
+        self.is_writeable = 1
+        self.wr_file = <shared_ptr[OutputStream]> handle
+
+
 cdef class BytesReader(NativeFile):
     cdef:
         object obj
@@ -198,7 +284,8 @@ cdef class BytesReader(NativeFile):
             raise ValueError('Must pass bytes object')
 
         self.obj = obj
-        self.is_readonly = 1
+        self.is_readable = 1
+        self.is_writeable = 0
         self.is_open = True
 
         self.rd_file.reset(new pyarrow.PyBytesReader(obj))
@@ -264,7 +351,8 @@ cdef class InMemoryOutputStream(NativeFile):
         self.buffer = allocate_buffer()
         self.wr_file.reset(new BufferOutputStream(
             <shared_ptr[ResizableBuffer]> self.buffer))
-        self.is_readonly = 0
+        self.is_readable = 0
+        self.is_writeable = 1
         self.is_open = True
 
     def get_result(self):
@@ -285,7 +373,8 @@ cdef class BufferReader(NativeFile):
         self.buffer = buffer
         self.rd_file.reset(new CBufferReader(buffer.buffer.get().data(),
                                              buffer.buffer.get().size()))
-        self.is_readonly = 1
+        self.is_readable = 1
+        self.is_writeable = 0
         self.is_open = True
 
 
@@ -311,12 +400,14 @@ cdef get_reader(object source, shared_ptr[ReadableFileInterface]* reader):
     elif not isinstance(source, NativeFile) and hasattr(source, 'read'):
         # Optimistically hope this is file-like
         source = PythonFileInterface(source, mode='r')
+    elif isinstance(source, six.string_types):
+        source = MemoryMappedFile(source, mode='r')
 
     if isinstance(source, NativeFile):
         nf = source
 
         # TODO: what about read-write sources (e.g. memory maps)
-        if not nf.is_readonly:
+        if not nf.is_readable:
             raise IOError('Native file is not readable')
 
         nf.read_handle(reader)
@@ -335,7 +426,7 @@ cdef get_writer(object source, shared_ptr[OutputStream]* writer):
     if isinstance(source, NativeFile):
         nf = source
 
-        if nf.is_readonly:
+        if nf.is_readable:
             raise IOError('Native file is not writeable')
 
         nf.write_handle(writer)
@@ -593,14 +684,16 @@ cdef class HdfsClient:
 
             out.wr_file = <shared_ptr[OutputStream]> wr_handle
 
-            out.is_readonly = False
+            out.is_readable = False
+            out.is_writeable = 1
         else:
             with nogil:
                 check_status(self.client.get()
                              .OpenReadable(c_path, &rd_handle))
 
             out.rd_file = <shared_ptr[ReadableFileInterface]> rd_handle
-            out.is_readonly = True
+            out.is_readable = True
+            out.is_writeable = 0
 
         if c_buffer_size == 0:
             c_buffer_size = 2 ** 16
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 2dedb72ebfcc1..708ae65585ae2 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -33,7 +33,7 @@ class ParquetFile(object):
     """
     def __init__(self, source, metadata=None):
         self.reader = _parquet.ParquetReader()
-        self.reader.open(source)
+        self.reader.open(source, metadata=metadata)
 
     @property
     def metadata(self):
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 3a046516d961b..dce125a7b3236 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -22,7 +22,7 @@
 from cython.operator cimport dereference as deref
 
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.common cimport PyObject_to_object
+from pyarrow.includes.common cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 
 import pyarrow.config
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 3e7a43702aa05..224f20dbfbb03 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -16,10 +16,14 @@
 # under the License.
 
 from io import BytesIO
+import os
 import pytest
 
-from pyarrow.compat import u
+import numpy as np
+
+from pyarrow.compat import u, guid
 import pyarrow.io as io
+import pyarrow as pa
 
 # ----------------------------------------------------------------------
 # Python file-like objects
@@ -155,3 +159,127 @@ def test_inmemory_write_after_closed():
 
     with pytest.raises(IOError):
         f.write(b'not ok')
+
+
+# ----------------------------------------------------------------------
+# OS files and memory maps
+
+@pytest.fixture(scope='session')
+def sample_disk_data(request):
+
+    SIZE = 4096
+    arr = np.random.randint(0, 256, size=SIZE).astype('u1')
+    data = arr.tobytes()[:SIZE]
+
+    path = guid()
+    with open(path, 'wb') as f:
+        f.write(data)
+
+    def teardown():
+        _try_delete(path)
+    request.addfinalizer(teardown)
+    return path, data
+
+
+def _check_native_file_reader(KLASS, sample_data):
+    path, data = sample_data
+
+    f = KLASS(path, mode='r')
+
+    assert f.read(10) == data[:10]
+    assert f.read(0) == b''
+    assert f.tell() == 10
+
+    assert f.read() == data[10:]
+
+    assert f.size() == len(data)
+
+    f.seek(0)
+    assert f.tell() == 0
+
+    # Seeking past end of file not supported in memory maps
+    f.seek(len(data) + 1)
+    assert f.tell() == len(data) + 1
+    assert f.read(5) == b''
+
+
+def test_memory_map_reader(sample_disk_data):
+    _check_native_file_reader(io.MemoryMappedFile, sample_disk_data)
+
+
+def test_os_file_reader(sample_disk_data):
+    _check_native_file_reader(io.OSFile, sample_disk_data)
+
+
+def _try_delete(path):
+    try:
+        os.remove(path)
+    except os.error:
+        pass
+
+
+def test_memory_map_writer():
+    SIZE = 4096
+    arr = np.random.randint(0, 256, size=SIZE).astype('u1')
+    data = arr.tobytes()[:SIZE]
+
+    path = guid()
+    try:
+        with open(path, 'wb') as f:
+            f.write(data)
+
+        f = io.MemoryMappedFile(path, mode='r+w')
+
+        f.seek(10)
+        f.write('peekaboo')
+        assert f.tell() == 18
+
+        f.seek(10)
+        assert f.read(8) == b'peekaboo'
+
+        f2 = io.MemoryMappedFile(path, mode='r+w')
+
+        f2.seek(10)
+        f2.write(b'booapeak')
+        f2.seek(10)
+
+        f.seek(10)
+        assert f.read(8) == b'booapeak'
+
+        # Does not truncate file
+        f3 = io.MemoryMappedFile(path, mode='w')
+        f3.write('foo')
+
+        with io.MemoryMappedFile(path) as f4:
+            assert f4.size() == SIZE
+
+        with pytest.raises(IOError):
+            f3.read(5)
+
+        f.seek(0)
+        assert f.read(3) == b'foo'
+    finally:
+        _try_delete(path)
+
+
+def test_os_file_writer():
+    SIZE = 4096
+    arr = np.random.randint(0, 256, size=SIZE).astype('u1')
+    data = arr.tobytes()[:SIZE]
+
+    path = guid()
+    try:
+        with open(path, 'wb') as f:
+            f.write(data)
+
+        # Truncates file
+        f2 = io.OSFile(path, mode='w')
+        f2.write('foo')
+
+        with io.OSFile(path) as f3:
+            assert f3.size() == 3
+
+        with pytest.raises(IOError):
+            f2.read(5)
+    finally:
+        _try_delete(path)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index e1571557d9aa7..9cf860ac28a10 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -236,19 +236,22 @@ def test_pandas_parquet_configuration_options(tmpdir):
         pdt.assert_frame_equal(df, df_read)
 
 
-@parquet
-def test_parquet_metadata_api():
-    df = alltypes_sample(size=10000)
-    df = df.reindex(columns=sorted(df.columns))
-
+def make_sample_file(df):
     a_table = A.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
     pq.write_table(a_table, buf, compression='SNAPPY', version='2.0')
 
     buf.seek(0)
-    fileh = pq.ParquetFile(buf)
+    return pq.ParquetFile(buf)
 
+
+@parquet
+def test_parquet_metadata_api():
+    df = alltypes_sample(size=10000)
+    df = df.reindex(columns=sorted(df.columns))
+
+    fileh = make_sample_file(df)
     ncols = len(df.columns)
 
     # Series of sniff tests
@@ -288,3 +291,40 @@ def test_parquet_metadata_api():
 
     assert rg_meta.num_rows == len(df)
     assert rg_meta.num_columns == ncols
+
+
+@parquet
+def test_compare_schemas():
+    df = alltypes_sample(size=10000)
+
+    fileh = make_sample_file(df)
+    fileh2 = make_sample_file(df)
+    fileh3 = make_sample_file(df[df.columns[::2]])
+
+    assert fileh.schema.equals(fileh.schema)
+    assert fileh.schema.equals(fileh2.schema)
+
+    assert not fileh.schema.equals(fileh3.schema)
+
+    assert fileh.schema[0].equals(fileh.schema[0])
+    assert not fileh.schema[0].equals(fileh.schema[1])
+
+
+@parquet
+def test_pass_separate_metadata():
+    # ARROW-471
+    df = alltypes_sample(size=10000)
+
+    a_table = A.Table.from_pandas(df, timestamps_to_ms=True)
+
+    buf = io.BytesIO()
+    pq.write_table(a_table, buf, compression='snappy', version='2.0')
+
+    buf.seek(0)
+    metadata = pq.ParquetFile(buf).metadata
+
+    buf.seek(0)
+
+    fileh = pq.ParquetFile(buf, metadata=metadata)
+
+    pdt.assert_frame_equal(df, fileh.read().to_pandas())

From cb83b8d30d6bc7d654736c590763145d7c7252ce Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 13 Jan 2017 13:58:45 -0500
Subject: [PATCH 0273/1644] ARROW-96: Add C++ API documentation

This adds a basic `Doxyfile` that can be used to generate the HTML API documentation as well as a small, initial "Getting Started".

The documentation is not yet deployed anywhere. We can either also use `readthedocs.org` for this (via the `breathe` package) or wait for the restructuring of the website as discussed on the ML and then add this to the "update website" scripts. I'd personally prefer the latter.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #271 from xhochy/ARROW-96 and squashes the following commits:

45c98cb [Uwe L. Korn] Add license headers
e7c9849 [Uwe L. Korn] ARROW-96: Add C++ API documentation
---
 cpp/README.md         |    9 +
 cpp/apidoc/.gitignore |    1 +
 cpp/apidoc/Doxyfile   | 2492 +++++++++++++++++++++++++++++++++++++++++
 cpp/apidoc/index.md   |   85 ++
 cpp/src/arrow/array.h |   35 +-
 5 files changed, 2610 insertions(+), 12 deletions(-)
 create mode 100644 cpp/apidoc/.gitignore
 create mode 100644 cpp/apidoc/Doxyfile
 create mode 100644 cpp/apidoc/index.md

diff --git a/cpp/README.md b/cpp/README.md
index b77ea990d0659..542a854990250 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -62,6 +62,15 @@ variables
 * Hadoop: `HADOOP_HOME` (only required for the HDFS I/O extensions)
 * jemalloc: `JEMALLOC_HOME` (only required for the jemalloc-based memory pool)
 
+### API documentation
+
+To generate the (html) API documentation, run the following command in the apidoc
+directoy:
+
+    doxygen Doxyfile
+
+This requires [Doxygen](http://www.doxygen.org) to be installed.
+
 ## Continuous Integration
 
 Pull requests are run through travis-ci for continuous integration.  You can avoid
diff --git a/cpp/apidoc/.gitignore b/cpp/apidoc/.gitignore
new file mode 100644
index 0000000000000..5ccff1a6bea26
--- /dev/null
+++ b/cpp/apidoc/.gitignore
@@ -0,0 +1 @@
+html/
diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile
new file mode 100644
index 0000000000000..7dc55fef834fc
--- /dev/null
+++ b/cpp/apidoc/Doxyfile
@@ -0,0 +1,2492 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Doxyfile 1.8.13
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all text
+# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
+# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
+# for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING      = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Apache Arrow (C++)"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         =
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "A columnar in-memory analytics layer designed to accelerate big data."
+
+# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
+# in the documentation. The maximum height of the logo should not exceed 55
+# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
+# the logo to the output directory.
+
+PROJECT_LOGO           =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       =
+
+# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS         = NO
+
+# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
+# characters to appear in the names of generated files. If set to NO, non-ASCII
+# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
+# U+3044.
+# The default value is: NO.
+
+ALLOW_UNICODE_NAMES    = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE        = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF           = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF       = "The $name class" \
+                         "The $name widget" \
+                         "The $name file" \
+                         is \
+                         provides \
+                         specifies \
+                         contains \
+                         represents \
+                         a \
+                         an \
+                         the
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC    = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB  = NO
+
+# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES        = YES
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH        =
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH    =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES            = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF      = NO
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF           = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS           = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new
+# page for each member. If set to NO, the documentation of a member will be part
+# of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES  = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE               = 4
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines.
+
+ALIASES                =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C  = NO
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA   = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN   = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL   = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
+# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
+# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
+# Fortran. In the later case the parser tries to guess whether the code is fixed
+# or free formatted code, this is the default for Fortran type files), VHDL. For
+# instance to make doxygen treat .inc files as Fortran files (default is PHP),
+# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+#
+# Note: For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING      =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT       = YES
+
+# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up
+# to that level are automatically included in the table of contents, even if
+# they do not have an id attribute.
+# Note: This feature currently applies only to Markdown headings.
+# Minimum value: 0, maximum value: 99, default value: 0.
+# This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
+
+TOC_INCLUDE_HEADINGS   = 0
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT       = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT    = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT        = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT            = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT   = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC   = NO
+
+# If one adds a struct or class to a group and this option is enabled, then also
+# any nested class or struct is added to the same group. By default this option
+# is disabled and one has to add nested compounds explicitly via \ingroup.
+# The default value is: NO.
+
+GROUP_NESTED_COMPOUNDS = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING            = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS  = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT   = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE      = 0
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL            = YES
+
+# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE        = NO
+
+# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE        = NO
+
+# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC         = NO
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO,
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES  = YES
+
+# This flag is only useful for Objective-C code. If set to YES, local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO, only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS  = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES   = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES     = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# (class|struct|union) declarations. If set to NO, these declarations will be
+# included in the documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS  = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO, these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS      = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS          = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES, upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# and Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES       = NO
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES, the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES       = NO
+
+# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will
+# append additional text to a page's title, such as Class Reference. If set to
+# YES the compound reference will be hidden.
+# The default value is: NO.
+
+HIDE_COMPOUND_REFERENCE= NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES     = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC  = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES   = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO            = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS       = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS        = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES       = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME     = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING  = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo
+# list. This list is created by putting \todo commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST      = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
+# list. This list is created by putting \test commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST      = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST       = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS       =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES  = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES, the
+# list will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES        = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES             = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES        = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER    =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE            =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. See also \cite for info how to create references.
+
+CITE_BIB_FILES         =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET                  = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS               = YES
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED   = YES
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR      = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO, doxygen will only warn about wrong or incomplete
+# parameter documentation, but not about the absence of documentation.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC       = NO
+
+# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
+# a warning is encountered.
+# The default value is: NO.
+
+WARN_AS_ERROR          = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT            = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE           =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# Note we include "." here to add the markdown text.
+INPUT                  = ../src .
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08,
+# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf.
+
+FILE_PATTERNS          = *.c \
+                         *.cc \
+                         *.cxx \
+                         *.cpp \
+                         *.c++ \
+                         *.java \
+                         *.ii \
+                         *.ixx \
+                         *.ipp \
+                         *.i++ \
+                         *.inl \
+                         *.idl \
+                         *.ddl \
+                         *.odl \
+                         *.h \
+                         *.hh \
+                         *.hxx \
+                         *.hpp \
+                         *.h++ \
+                         *.cs \
+                         *.d \
+                         *.php \
+                         *.php4 \
+                         *.php5 \
+                         *.phtml \
+                         *.inc \
+                         *.m \
+                         *.markdown \
+                         *.md \
+                         *.mm \
+                         *.dox \
+                         *.py \
+                         *.pyw \
+                         *.f90 \
+                         *.f95 \
+                         *.f03 \
+                         *.f08 \
+                         *.f \
+                         *.for \
+                         *.tcl \
+                         *.vhd \
+                         *.vhdl \
+                         *.ucf \
+                         *.qsf
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       = *-test.cc \
+                         *-benchmark.cc
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS        =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH           =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS       = *
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE      = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH             =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+INPUT_FILTER           =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+FILTER_PATTERNS        =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES    = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER         = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES         = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS    = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# function all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = NO
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION    = NO
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS        = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see http://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS              = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS       = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX     = YES
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX    = 5
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX          =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML          = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT            = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION    = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER            =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER            =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET        =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# cascading style sheets that are included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefore more robust against future updates.
+# Doxygen will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list). For an example see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET  =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES       =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the style sheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE    = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT    = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA  = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to YES can help to show when doxygen was last run and thus if the
+# documentation is up to date.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP         = NO
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS  = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: http://developer.apple.com/tools/xcode/), introduced with
+# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
+# for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET        = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME        = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID       = org.doxygen.Project
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID    = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME  = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP      = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE               =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler (hhc.exe). If non-empty,
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION           =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated
+# (YES) or that it should be included in the master .chm file (NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI           = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING     =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated
+# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
+# enables the Previous and Next buttons.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC             = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND             = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP           = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE               =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE          = org.doxygen.Project
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER     = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME   =
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS  =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS  =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION           =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP   = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID         = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX          = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW      = NO
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH         = 250
+
+# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW    = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE       = 10
+
+# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT    = YES
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# http://www.mathjax.org) which uses client side Javascript for the rendering
+# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX            = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT         = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from http://www.mathjax.org before deployment.
+# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS     =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE       =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE           = YES
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using Javascript. There
+# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
+# setting. When disabled, doxygen will generate a PHP script for searching and
+# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
+# and searching needs to be provided by external tools. See the section
+# "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH    = NO
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH        = NO
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL       =
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE        = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID     =
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS  =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT           = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when enabling USE_PDFLATEX this option is only used for generating
+# bitmaps for formulas in the HTML output, but not in the Makefile that is
+# written to the output directory.
+# The default file is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME         = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME     = makeindex
+
+# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX          = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE             = a4
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. The package can be specified just
+# by its name or with the correct syntax as to be used with the LaTeX
+# \usepackage command. To get the times font for instance you can specify :
+# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times}
+# To use the option intlimits with the amsmath package you can specify:
+# EXTRA_PACKAGES=[intlimits]{amsmath}
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES         =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
+# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
+# string, for the replacement values of the other commands the user is referred
+# to HTML_HEADER.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER           =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer. See
+# LATEX_HEADER for more information on how to generate a default footer and what
+# special commands can be used inside the footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER           =
+
+# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# LaTeX style sheets that are included after the standard style sheets created
+# by doxygen. Using this option one can overrule certain style aspects. Doxygen
+# will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_STYLESHEET =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES      =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS         = YES
+
+# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES, to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX           = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE        = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES     = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE      = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE        = plain
+
+# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_TIMESTAMP        = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF           = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT             = rtf
+
+# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF            = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS         = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's config
+# file, i.e. a series of assignments. You only have to provide replacements,
+# missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE    =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's config file. A template extensions file can be generated
+# using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE    =
+
+# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
+# with syntax highlighting in the RTF output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_SOURCE_CODE        = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN           = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT             = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION          = .3
+
+# The MAN_SUBDIR tag determines the name of the directory created within
+# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
+# MAN_EXTENSION with the initial . removed.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_SUBDIR             =
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS              = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML           = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT             = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING     = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK       = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT         = docbook
+
+# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
+# program listings (including syntax highlighting and cross-referencing
+# information) to the DOCBOOK output. Note that enabling this will significantly
+# increase the size of the DOCBOOK output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_PROGRAMLISTING = NO
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
+# AutoGen Definitions (see http://autogen.sf.net) file that captures the
+# structure of the code including all documentation. Note that this feature is
+# still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF   = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD       = NO
+
+# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX          = NO
+
+# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO, the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY         = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = NO
+
+# If the SEARCH_INCLUDES tag is set to YES, the include files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES        = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           =
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS  =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED             =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have
+# an all uppercase name, and do not end with a semicolon. Such function macros
+# are typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS   = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have a unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES               =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE       =
+
+# If the ALLEXTERNALS tag is set to YES, all external class will be listed in
+# the class index. If set to NO, only the inherited external classes will be
+# listed.
+# The default value is: NO.
+
+ALLEXTERNALS           = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS        = YES
+
+# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES         = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script
+# interpreter (i.e. the result of 'which perl').
+# The default file (with absolute path) is: /usr/bin/perl.
+
+PERL_PATH              = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS         = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see:
+# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
+# default search path.
+
+MSCGEN_PATH            =
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH               =
+
+# If set to YES the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS   = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: NO.
+
+HAVE_DOT               = NO
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS        = 0
+
+# When you want a differently looking font in the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME           = Helvetica
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE           = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH           =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH            = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH    = YES
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS           = YES
+
+# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK               = NO
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS   = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS     = NO
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH          = YES
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH      = YES
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command. Disabling a call graph can be
+# accomplished by means of the command \hidecallgraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH             = NO
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command. Disabling a caller graph can be
+# accomplished by means of the command \hidecallergraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH           = NO
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY    = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH        = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot. For an explanation of the image formats see the section
+# output formats in the documentation of the dot tool (Graphviz (see:
+# http://www.graphviz.org/)).
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo,
+# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
+# png:gdiplus:gdiplus.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT       = png
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG        = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH               =
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS           =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS           =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS           =
+
+# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
+# path where java can find the plantuml.jar file. If left blank, it is assumed
+# PlantUML is not used or called during a preprocessing step. Doxygen will
+# generate a warning when it encounters a \startuml command in this case and
+# will not generate output for the diagram.
+
+PLANTUML_JAR_PATH      =
+
+# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a
+# configuration file for plantuml.
+
+PLANTUML_CFG_FILE      =
+
+# When using plantuml, the specified paths are searched for files specified by
+# the !include statement in a plantuml block.
+
+PLANTUML_INCLUDE_PATH  =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES    = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH    = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT        = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS      = NO
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND        = YES
+
+# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP            = YES
diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
new file mode 100644
index 0000000000000..080f848bb184f
--- /dev/null
+++ b/cpp/apidoc/index.md
@@ -0,0 +1,85 @@
+Apache Arrow C++ API documentation      {#index}
+==================================
+
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+Apache Arrow is a columnar in-memory analytics layer designed to accelerate
+big data. It houses a set of canonical in-memory representations of flat and
+hierarchical data along with multiple language-bindings for structure
+manipulation. It also provides IPC and common algorithm implementations.
+
+This is the documentation of the C++ API of Apache Arrow. For more details
+on the format and other language bindings see
+the [main page for Arrow](https://arrow.apache.org/). Here will we only detail
+the usage of the C++ API for Arrow and the leaf libraries that add additional
+functionality such as using [jemalloc](http://jemalloc.net/) as an allocator
+for Arrow structures.
+
+Getting Started
+---------------
+
+The most basic structure in Arrow is an `arrow::Array`. It holds a sequence
+of values with known length all having the same type. It consists of the data
+itself and an additional bitmap that indicates if the corresponding entry of
+array is a null-value. Note that for array with zero null entries, we can omit
+this bitmap.
+
+As Arrow objects are immutable, there are classes provided that should help you
+build these objects. To build an array of `int64_t` elements, we can use the
+`Int64Builder`. In the following example, we build an array of the range 1 to 8
+where the element that should hold the number 4 is nulled.
+
+    Int64Builder builder(arrow::default_memory_pool(), arrow::int64());
+    builder.Append(1);
+    builder.Append(2);
+    builder.Append(3);
+    builder.AppendNull();
+    builder.Append(5);
+    builder.Append(6);
+    builder.Append(7);
+    builder.Append(8);
+
+    std::shared_ptr<Array> array;
+    builder.Finish(&array);
+
+The resulting Array (which can be casted to `arrow::Int64Array` if you want
+to access its values) then consists of two `arrow::Buffer`. The first one is
+the null bitmap holding a single byte with the bits `0|0|0|0|1|0|0|0`.
+As we use [least-significant bit (LSB) numbering](https://en.wikipedia.org/wiki/Bit_numbering)
+this indicates that the fourth entry in the array is null. The second
+buffer is simply an `int64_t` array containing all the above values.
+As the fourth entry is null, the value at that position in the buffer is
+undefined.
+
+    // Cast the Array to its actual type to access its data
+    std::shared_ptr<Int64Array> int64_array = std::shared_pointer_cast<Int64Array>(array);
+
+    // Get the pointer to the null bitmap.
+    const uint8_t* null_bitmap = int64_array->null_bitmap_data();
+
+    // Get the pointer to the actual data
+    const int64_t* data = int64_array->raw_data();
+
+In the above example, we have yet skipped explaining two things in the code.
+On constructing the builder, we have passed `arrow::int64()` to it. This is
+the type information with which the resulting array will be annotated. In
+this simple form, it is solely a `std::shared_ptr<arrow::Int64Type>`
+instantiation.
+
+Furthermore, we have passed `arrow::default_memory_pool()` to the constructor.
+This `arrow::MemoryPool` is used for the allocations of heap memory. Besides
+tracking the amount of memory allocated, the allocator also ensures that the
+allocated memory regions are 64-byte aligned (as required by the Arrow
+specification).
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 57214c46d1cc6..45f8ab9f56791 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -65,11 +65,12 @@ class ArrayVisitor {
   virtual Status Visit(const DictionaryArray& type) = 0;
 };
 
-// Immutable data array with some logical type and some length. Any memory is
-// owned by the respective Buffer instance (or its parents).
-//
-// The base class is only required to have a null bitmap buffer if the null
-// count is greater than 0
+/// Immutable data array with some logical type and some length.
+///
+/// Any memory is owned by the respective Buffer instance (or its parents).
+///
+/// The base class is only required to have a null bitmap buffer if the null
+/// count is greater than 0
 class ARROW_EXPORT Array {
  public:
   Array(const std::shared_ptr<DataType>& type, int32_t length, int32_t null_count = 0,
@@ -77,19 +78,28 @@ class ARROW_EXPORT Array {
 
   virtual ~Array() = default;
 
-  // Determine if a slot is null. For inner loops. Does *not* boundscheck
+  /// Determine if a slot is null. For inner loops. Does *not* boundscheck
   bool IsNull(int i) const {
     return null_count_ > 0 && BitUtil::BitNotSet(null_bitmap_data_, i);
   }
 
+  /// Size in the number of elements this array contains.
   int32_t length() const { return length_; }
+
+  /// The number of null entries in the array.
   int32_t null_count() const { return null_count_; }
 
   std::shared_ptr<DataType> type() const { return type_; }
   Type::type type_enum() const { return type_->type; }
 
+  /// Buffer for the null bitmap.
+  ///
+  /// Note that for `null_count == 0`, this can be a `nullptr`.
   std::shared_ptr<Buffer> null_bitmap() const { return null_bitmap_; }
 
+  /// Raw pointer to the null bitmap.
+  ///
+  /// Note that for `null_count == 0`, this can be a `nullptr`.
   const uint8_t* null_bitmap_data() const { return null_bitmap_data_; }
 
   bool BaseEquals(const std::shared_ptr<Array>& arr) const;
@@ -97,13 +107,14 @@ class ARROW_EXPORT Array {
   virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
   virtual bool ApproxEquals(const std::shared_ptr<Array>& arr) const;
 
-  // Compare if the range of slots specified are equal for the given array and
-  // this array.  end_idx exclusive.  This methods does not bounds check.
+  /// Compare if the range of slots specified are equal for the given array and
+  /// this array.  end_idx exclusive.  This methods does not bounds check.
   virtual bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
       const std::shared_ptr<Array>& arr) const = 0;
 
-  // Determines if the array is internally consistent.  Defaults to always
-  // returning Status::OK.  This can be an expensive check.
+  /// Determines if the array is internally consistent.
+  ///
+  /// Defaults to always returning Status::OK. This can be an expensive check.
   virtual Status Validate() const;
 
   virtual Status Accept(ArrayVisitor* visitor) const = 0;
@@ -121,7 +132,7 @@ class ARROW_EXPORT Array {
   DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
-// Degenerate null type Array
+/// Degenerate null type Array
 class ARROW_EXPORT NullArray : public Array {
  public:
   using TypeClass = NullType;
@@ -141,7 +152,7 @@ class ARROW_EXPORT NullArray : public Array {
 Status ARROW_EXPORT GetEmptyBitmap(
     MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result);
 
-// Base class for fixed-size logical types
+/// Base class for fixed-size logical types
 class ARROW_EXPORT PrimitiveArray : public Array {
  public:
   virtual ~PrimitiveArray() {}

From 281cb969772a5878cd64a487cf2be3944f801c3d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 13 Jan 2017 13:59:40 -0500
Subject: [PATCH 0274/1644] ARROW-481: [Python] Fix 2.7 regression in Parquet
 path to open file code path

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #283 from wesm/ARROW-481 and squashes the following commits:

8ed919e [Wes McKinney] Fix Python 2.7 regression in Parquet path to open file code path
---
 python/pyarrow/io.pyx            |  6 ++----
 python/pyarrow/tests/test_ipc.py | 12 ++++++------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 2d8e4e8f34242..1939fe83ab3c6 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -395,13 +395,11 @@ def buffer_from_bytes(object obj):
 cdef get_reader(object source, shared_ptr[ReadableFileInterface]* reader):
     cdef NativeFile nf
 
-    if isinstance(source, bytes):
-        source = BytesReader(source)
+    if isinstance(source, six.string_types):
+        source = MemoryMappedFile(source, mode='r')
     elif not isinstance(source, NativeFile) and hasattr(source, 'read'):
         # Optimistically hope this is file-like
         source = PythonFileInterface(source, mode='r')
-    elif isinstance(source, six.string_types):
-        source = MemoryMappedFile(source, mode='r')
 
     if isinstance(source, NativeFile):
         nf = source
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 14cbb30d5d48b..368a9d4e51905 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -23,7 +23,7 @@
 import pandas as pd
 
 import pyarrow as A
-import pyarrow.io as arrow_io
+import pyarrow.io as aio
 import pyarrow.ipc as ipc
 
 
@@ -63,7 +63,7 @@ def run(self):
         writer.close()
 
         file_contents = self._get_source()
-        reader = ipc.ArrowFileReader(file_contents)
+        reader = ipc.ArrowFileReader(aio.BytesReader(file_contents))
 
         assert reader.num_record_batches == num_batches
 
@@ -76,7 +76,7 @@ def run(self):
 class InMemoryStreamTest(RoundtripTest):
 
     def _get_sink(self):
-        return arrow_io.InMemoryOutputStream()
+        return aio.InMemoryOutputStream()
 
     def _get_source(self):
         return self.sink.get_result()
@@ -91,10 +91,10 @@ def test_ipc_zero_copy_numpy():
     df = pd.DataFrame({'foo': [1.5]})
 
     batch = A.RecordBatch.from_pandas(df)
-    sink = arrow_io.InMemoryOutputStream()
+    sink = aio.InMemoryOutputStream()
     write_file(batch, sink)
     buffer = sink.get_result()
-    reader = arrow_io.BufferReader(buffer)
+    reader = aio.BufferReader(buffer)
 
     batches = read_file(reader)
 
@@ -118,7 +118,7 @@ def big_batch():
 
 
 def write_to_memory2(batch):
-    sink = arrow_io.InMemoryOutputStream()
+    sink = aio.InMemoryOutputStream()
     write_file(batch, sink)
     return sink.get_result()
 

From 876ae8509d84b8c4dc269250b2fae47cef897054 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 13 Jan 2017 14:00:12 -0500
Subject: [PATCH 0275/1644] ARROW-239: Test reading remainder of file in HDFS
 with read() with no args

This was already fixed, but adding a unit test to verify.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #284 from wesm/ARROW-239 and squashes the following commits:

7b3aae7 [Wes McKinney] Test reading remainder of file in HDFS with read() with no args
---
 python/pyarrow/schema.pyx         |  8 +++++---
 python/pyarrow/tests/test_hdfs.py | 12 ++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index f6a1a10c8dd5c..a6aa9d597241d 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -22,6 +22,8 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+from cython.operator cimport dereference as deref
+
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
                                         Type_NA, Type_BOOL,
@@ -31,7 +33,7 @@ from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
                                         Type_UINT64, Type_INT64,
                                         Type_TIMESTAMP, Type_DATE,
                                         Type_FLOAT, Type_DOUBLE,
-                                        Type_STRING, Type_BINARY, 
+                                        Type_STRING, Type_BINARY,
                                         TimeUnit_SECOND, TimeUnit_MILLI,
                                         TimeUnit_MICRO, TimeUnit_NANO,
                                         Type, TimeUnit)
@@ -57,9 +59,9 @@ cdef class DataType:
 
     def __richcmp__(DataType self, DataType other, int op):
         if op == cpython.Py_EQ:
-            return self.type.Equals(other.sp_type)
+            return self.type.Equals(deref(other.type))
         elif op == cpython.Py_NE:
-            return not self.type.Equals(other.sp_type)
+            return not self.type.Equals(deref(other.type))
         else:
             raise TypeError('Invalid comparison')
 
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 4ff5a9d42b55e..2056f7ab589da 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -126,6 +126,18 @@ def test_hdfs_file_context_manager(self):
             result = f.read(10)
             assert result == data
 
+    def test_hdfs_read_whole_file(self):
+        path = pjoin(self.tmp_path, 'read-whole-file')
+
+        data = b'foo' * 1000
+        with self.hdfs.open(path, 'wb') as f:
+            f.write(data)
+
+        with self.hdfs.open(path, 'rb') as f:
+            result = f.read()
+
+        assert result == data
+
 
 class TestLibHdfs(HdfsTestCases, unittest.TestCase):
 

From 47115aa3e33d85bbc697a7950da33cf18f4c71be Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 13 Jan 2017 15:45:19 -0500
Subject: [PATCH 0276/1644] ARROW-478: Consolidate BytesReader and BufferReader
 to accept PyBytes or Buffer

API simplification. I think `pyarrow::PyBytesReader` can even be removed.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #285 from wesm/ARROW-478 and squashes the following commits:

cbfc8ad [Wes McKinney] Consolidate BytesReader and BufferReader to accept either PyBytes or Buffer
---
 python/pyarrow/__init__.py              |  3 +-
 python/pyarrow/includes/libarrow_io.pxd |  1 +
 python/pyarrow/io.pyx                   | 57 +++++++++++--------------
 python/pyarrow/parquet.py               |  4 +-
 python/pyarrow/tests/test_io.py         |  7 ++-
 python/pyarrow/tests/test_ipc.py        |  2 +-
 6 files changed, 34 insertions(+), 40 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index d25cdd47ea974..a8c3e8e3158f8 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -38,8 +38,7 @@
 from pyarrow.error import ArrowException
 
 from pyarrow.io import (HdfsClient, HdfsFile, NativeFile, PythonFileInterface,
-                        BytesReader, Buffer, InMemoryOutputStream,
-                        BufferReader)
+                        Buffer, InMemoryOutputStream, BufferReader)
 
 from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
                             BooleanValue,
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 6b141a3e76f09..417af7d67d1ab 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -162,6 +162,7 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
 cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
     cdef cppclass CBufferReader" arrow::io::BufferReader"\
         (ReadableFileInterface):
+        CBufferReader(const shared_ptr[CBuffer]& buffer)
         CBufferReader(const uint8_t* data, int64_t nbytes)
 
     cdef cppclass BufferOutputStream(OutputStream):
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 1939fe83ab3c6..0f626f178abde 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -275,21 +275,6 @@ cdef class OSFile(NativeFile):
         self.wr_file = <shared_ptr[OutputStream]> handle
 
 
-cdef class BytesReader(NativeFile):
-    cdef:
-        object obj
-
-    def __cinit__(self, obj):
-        if not isinstance(obj, bytes):
-            raise ValueError('Must pass bytes object')
-
-        self.obj = obj
-        self.is_readable = 1
-        self.is_writeable = 0
-        self.is_open = True
-
-        self.rd_file.reset(new pyarrow.PyBytesReader(obj))
-
 # ----------------------------------------------------------------------
 # Arrow buffers
 
@@ -330,12 +315,6 @@ cdef class Buffer:
             self.buffer.get().size())
 
 
-cdef wrap_buffer(const shared_ptr[CBuffer]& buffer):
-    cdef Buffer result = Buffer()
-    result.buffer = buffer
-    return result
-
-
 cdef shared_ptr[PoolBuffer] allocate_buffer():
     cdef shared_ptr[PoolBuffer] result
     result.reset(new PoolBuffer(pyarrow.get_memory_pool()))
@@ -356,23 +335,35 @@ cdef class InMemoryOutputStream(NativeFile):
         self.is_open = True
 
     def get_result(self):
-        cdef Buffer result = Buffer()
-
         check_status(self.wr_file.get().Close())
-        result.init(<shared_ptr[CBuffer]> self.buffer)
-
         self.is_open = False
-        return result
+        return wrap_buffer(<shared_ptr[CBuffer]> self.buffer)
 
 
 cdef class BufferReader(NativeFile):
+    """
+    Zero-copy reader from objects convertible to Arrow buffer
+
+    Parameters
+    ----------
+    obj : Python bytes or pyarrow.io.Buffer
+    """
     cdef:
         Buffer buffer
 
-    def __cinit__(self, Buffer buffer):
-        self.buffer = buffer
-        self.rd_file.reset(new CBufferReader(buffer.buffer.get().data(),
-                                             buffer.buffer.get().size()))
+    def __cinit__(self, object obj):
+        cdef shared_ptr[CBuffer] buf
+
+        if isinstance(obj, Buffer):
+            self.buffer = obj
+        elif isinstance(obj, bytes):
+            buf.reset(new pyarrow.PyBytesBuffer(obj))
+            self.buffer = wrap_buffer(buf)
+        else:
+            raise ValueError('Unable to convert value to buffer: {0}'
+                             .format(type(obj)))
+
+        self.rd_file.reset(new CBufferReader(self.buffer.buffer))
         self.is_readable = 1
         self.is_writeable = 0
         self.is_open = True
@@ -382,16 +373,20 @@ def buffer_from_bytes(object obj):
     """
     Construct an Arrow buffer from a Python bytes object
     """
+    cdef shared_ptr[CBuffer] buf
     if not isinstance(obj, bytes):
         raise ValueError('Must pass bytes object')
 
-    cdef shared_ptr[CBuffer] buf
     buf.reset(new pyarrow.PyBytesBuffer(obj))
+    return wrap_buffer(buf)
+
 
+cdef Buffer wrap_buffer(const shared_ptr[CBuffer]& buf):
     cdef Buffer result = Buffer()
     result.init(buf)
     return result
 
+
 cdef get_reader(object source, shared_ptr[ReadableFileInterface]* reader):
     cdef NativeFile nf
 
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 708ae65585ae2..2a1ac9d2db7ed 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -27,7 +27,7 @@ class ParquetFile(object):
     ----------
     source : str or pyarrow.io.NativeFile
         Readable source. For passing Python file objects or byte buffers,
-        see pyarrow.io.PythonFileInterface or pyarrow.io.BytesReader.
+        see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
     metadata : ParquetFileMetadata, default None
         Use existing metadata object, rather than reading from file.
     """
@@ -78,7 +78,7 @@ def read_table(source, columns=None):
     ----------
     source: str or pyarrow.io.NativeFile
         Readable source. For passing Python file objects or byte buffers, see
-        pyarrow.io.PythonFileInterface or pyarrow.io.BytesReader.
+        pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
     columns: list
         If not None, only these columns will be read from the file.
 
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 224f20dbfbb03..f28d44a746c45 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -23,7 +23,6 @@
 
 from pyarrow.compat import u, guid
 import pyarrow.io as io
-import pyarrow as pa
 
 # ----------------------------------------------------------------------
 # Python file-like objects
@@ -81,7 +80,7 @@ def test_python_file_read():
 def test_bytes_reader():
     # Like a BytesIO, but zero-copy underneath for C++ consumers
     data = b'some sample data'
-    f = io.BytesReader(data)
+    f = io.BufferReader(data)
 
     assert f.tell() == 0
 
@@ -103,7 +102,7 @@ def test_bytes_reader():
 
 def test_bytes_reader_non_bytes():
     with pytest.raises(ValueError):
-        io.BytesReader(u('some sample data'))
+        io.BufferReader(u('some sample data'))
 
 
 def test_bytes_reader_retains_parent_reference():
@@ -112,7 +111,7 @@ def test_bytes_reader_retains_parent_reference():
     # ARROW-421
     def get_buffer():
         data = b'some sample data' * 1000
-        reader = io.BytesReader(data)
+        reader = io.BufferReader(data)
         reader.seek(5)
         return reader.read_buffer(6)
 
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 368a9d4e51905..bbd6c6a56705c 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -63,7 +63,7 @@ def run(self):
         writer.close()
 
         file_contents = self._get_source()
-        reader = ipc.ArrowFileReader(aio.BytesReader(file_contents))
+        reader = ipc.ArrowFileReader(aio.BufferReader(file_contents))
 
         assert reader.num_record_batches == num_batches
 

From a098fd04fe229a9ce6f1f531c54c0c9a8eed56c4 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 15 Jan 2017 12:19:37 -0500
Subject: [PATCH 0277/1644] ARROW-486: [C++] Use virtual inheritance for
 diamond inheritance

arrow::io::ReadWriteFileInterface inheritances arrow::io::FileInterface
as diamond style via:

  * ReadableFileInterface -> InputStream -> FileInterface
  * WriteableFileInterface -> OutputStream -> FileInterface

If we have diamond inheritance, we can't cast subclasses of
arrow::io::ReadWriteFileInterface such as arrow::io::MemoryMappedFile to
arrow::io::FileInterface.

C++:

    #include <arrow/io/file.h>

    int
    main(void)
    {
      std::shared_ptr<arrow::io::MemoryMappedFile> memory_mapped_file;

      std::shared_ptr<arrow::io::FileInterface> file = memory_mapped_file;

      return 0;
    }

Build result:

    a.cc: In function 'int main()':
    a.cc:8:52: error: conversion from 'std::shared_ptr<arrow::io::MemoryMappedFile>' to non-scalar type 'std::shared_ptr<arrow::io::FileInterface>' requested
       std::shared_ptr<arrow::io::FileInterface> file = memory_mapped_file;
                                                        ^~~~~~~~~~~~~~~~~~

We can resolve it by using virtual inheritance.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #282 from kou/use-virtual-inheritance-for-diamond-inheritance and squashes the following commits:

e9d3a0b [Kouhei Sutou] ARROW-486: [C++] Use virtual inheritance for diamond inheritance
---
 cpp/src/arrow/io/interfaces.h    | 4 ++--
 cpp/src/arrow/io/io-file-test.cc | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 8fe2849287064..fdb3788188915 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -83,12 +83,12 @@ class Readable {
   virtual Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
 };
 
-class OutputStream : public FileInterface, public Writeable {
+class OutputStream : virtual public FileInterface, public Writeable {
  protected:
   OutputStream() {}
 };
 
-class InputStream : public FileInterface, public Readable {
+class InputStream : virtual public FileInterface, public Readable {
  protected:
   InputStream() {}
 };
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 20cd04748f019..f18f7b649eb9b 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -419,5 +419,10 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
       IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result));
 }
 
+TEST_F(TestMemoryMappedFile, CastableToFileInterface) {
+  std::shared_ptr<MemoryMappedFile> memory_mapped_file;
+  std::shared_ptr<FileInterface> file = memory_mapped_file;
+}
+
 }  // namespace io
 }  // namespace arrow

From 850774efe43370a876cae1a94de3b253777ba29d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 16 Jan 2017 13:46:36 -0500
Subject: [PATCH 0278/1644] ARROW-484: Revise README to include more detail
 about software components

Also closes #14.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #286 from wesm/ARROW-484 and squashes the following commits:

8c6acf6 [Wes McKinney] Tweak description of data containers
ec4b95e [Wes McKinney] Generalize note about binary formats
3d31644 [Wes McKinney] Typos
57b8bf5 [Wes McKinney] Revise README to include more detail about software components
---
 README.md | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 89114ee39b4a0..1eb3f86f98656 100644
--- a/README.md
+++ b/README.md
@@ -32,17 +32,39 @@ Arrow is a set of technologies that enable big-data systems to process and move
 Initial implementations include:
 
  - [The Arrow Format](https://github.com/apache/arrow/tree/master/format)
- - [Arrow Structures and APIs in C++](https://github.com/apache/arrow/tree/master/cpp)
- - [Arrow Structures and APIs in Java](https://github.com/apache/arrow/tree/master/java)
+ - [Java implementation](https://github.com/apache/arrow/tree/master/java)
+ - [C++ implementation](https://github.com/apache/arrow/tree/master/cpp)
+ - [Python interface to C++ libraries](https://github.com/apache/arrow/tree/master/python)
 
-Arrow is an [Apache Software Foundation](www.apache.org) project. More info can be found at [arrow.apache.org](http://arrow.apache.org).
+Arrow is an [Apache Software Foundation](www.apache.org) project. Learn more at
+[arrow.apache.org](http://arrow.apache.org).
+
+#### What's in the Arrow libraries?
+
+The reference Arrow implementations contain a number of distinct software
+components:
+
+- Columnar vector and table-like containers (similar to data frames) supporting
+  flat or nested types
+- Fast, language agnostic metadata messaging layer (using Google's Flatbuffers
+  library)
+- Reference-counted off-heap buffer memory management, for zero-copy memory
+  sharing and handling memory-mapped files
+- Low-overhead IO interfaces to files on disk, HDFS (C++ only)
+- Self-describing binary wire formats (streaming and batch/file-like) for
+  remote procedure calls (RPC) and
+  interprocess communication (IPC)
+- Integration tests for verifying binary compatibility between the
+  implementations (e.g. sending data from Java to C++)
+- Conversions to and from other in-memory data structures (e.g. Python's pandas
+  library)
 
 #### Getting involved
 
-Right now the primary audience for Apache Arrow are the designers and
-developers of data systems; most people will use Apache Arrow indirectly
-through systems that use it for internal data handling and interoperating with
-other Arrow-enabled systems.
+Right now the primary audience for Apache Arrow are the developers of data
+systems; most people will use Apache Arrow indirectly through systems that use
+it for internal data handling and interoperating with other Arrow-enabled
+systems.
 
 Even if you do not plan to contribute to Apache Arrow itself or Arrow
 integrations in other projects, we'd be happy to have you involved:

From b1472305c7262b07f2e9bf3d5db4eeff09ec5dc4 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 17 Jan 2017 19:09:24 -0500
Subject: [PATCH 0279/1644] ARROW-490: Python: Update manylinux1 build scripts

Through the usage of the ExternalProject command, a lot has become much
simpler.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #290 from xhochy/ARROW-490 and squashes the following commits:

64b024b [Uwe L. Korn] ARROW-490: Python: Update manylinux1 build scripts
---
 cpp/CMakeLists.txt                            |  4 ++--
 cpp/cmake_modules/BuildUtils.cmake            | 12 ++++++++++
 .../Dockerfile-parquet_arrow-base-x86_64      | 23 +------------------
 python/manylinux1/Dockerfile-x86_64           | 19 ++++-----------
 python/manylinux1/build_arrow.sh              |  8 ++++---
 5 files changed, 25 insertions(+), 41 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 8a2cfc5d6f180..885ab19256065 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -598,7 +598,7 @@ if (ARROW_JEMALLOC)
   find_package(jemalloc)
 
   if(NOT JEMALLOC_FOUND)
-    set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep")
+    set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
     set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
     set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
     set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
@@ -611,7 +611,7 @@ if (ARROW_JEMALLOC)
         CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
         BUILD_IN_SOURCE 1
         BUILD_COMMAND ${MAKE}
-        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}")
+        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}")
     else()
       ExternalProject_Add(jemalloc_ep
         URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index b620de515c126..9de9de516f996 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -27,6 +27,18 @@ function(ADD_ARROW_LIB LIB_NAME)
   add_library(${LIB_NAME}_objlib OBJECT
       ${ARG_SOURCES}
   )
+  if (ARG_STATIC_LINK_LIBS)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_LINK_LIBS})
+  endif()
+  if (ARG_STATIC_PRIVATE_LINK_LIBS)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_PRIVATE_LINK_LIBS})
+  endif()
+  if (ARG_SHARED_LINK_LIBS)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_LINK_LIBS})
+  endif()
+  if(ARG_SHARED_PRIVATE_LINK_LIBS)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_PRIVATE_LINK_LIBS})
+  endif()
 
   # Necessary to make static linking into other shared libraries work properly
   set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
index 714fa1a91b39e..94f5bc0f3b66e 100644
--- a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
+++ b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
@@ -12,29 +12,8 @@
 
 FROM arrow-base-x86_64
 
-WORKDIR /
-ADD http://zlib.net/zlib-1.2.8.tar.gz /zlib-1.2.8.tar.gz
-RUN tar xf zlib-1.2.8.tar.gz
-WORKDIR zlib-1.2.8
-RUN CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DCMAKE_BUILD_TYPE=Release .
-RUN make -j5 install
-
-WORKDIR /
-ADD https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz /snappy-1.1.3.tar.gz
-RUN tar xf snappy-1.1.3.tar.gz
-WORKDIR /snappy-1.1.3
-RUN ./configure --with-pic --prefix=/usr
-RUN make -j5 install
-
-WORKDIR /
-ADD http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz /thrift-0.9.1.tar.gz
-RUN tar xf thrift-0.9.1.tar.gz
-WORKDIR /thrift-0.9.1
-RUN ./configure LDFLAGS='-L/usr/lib64' CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --without-tests --with-cpp --prefix=/usr --disable-shared --enable-static
-RUN make -j5 install
-
 WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/usr THRIFT_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
+RUN ARROW_HOME=/usr /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
 RUN make -j5 install
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index e62a60111af4a..29e00b0ccbe49 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -16,32 +16,23 @@ FROM quay.io/pypa/manylinux1_x86_64:latest
 RUN yum install -y flex openssl-devel
 
 WORKDIR /
-ADD http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz /boost_1_60_0.tar.gz
+RUN wget http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
 RUN tar xf boost_1_60_0.tar.gz
 WORKDIR /boost_1_60_0
 RUN ./bootstrap.sh
 RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install
 
 WORKDIR /
-ADD https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz /cmake-3.5.2.tar.gz
-RUN tar xf cmake-3.5.2.tar.gz
-WORKDIR /cmake-3.5.2
-RUN ./configure --prefix=/usr
-RUN make -j5 install
-
-WORKDIR /
-ADD https://github.com/google/flatbuffers/archive/v1.3.0.tar.gz /flatbuffers-1.3.0.tar.gz
-RUN tar xf flatbuffers-1.3.0.tar.gz
-WORKDIR /flatbuffers-1.3.0
-RUN CXXFLAGS='-fPIC' cmake -DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .
-RUN make -j5 install
+# Install cmake manylinux1 package
+RUN /opt/python/cp35-cp35m/bin/pip install cmake
 
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git
 WORKDIR /multibuild
 RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
+
 ADD arrow /arrow
 WORKDIR /arrow/cpp
-RUN FLATBUFFERS_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF .
+RUN /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON .
 RUN make -j5 install
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 0786b6f490a16..7e2ad58617793 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -30,10 +30,12 @@ source /multibuild/manylinux_utils.sh
 cd /arrow/python
 
 # PyArrow build configuration
-export PYARROW_CMAKE_OPTIONS='-DCMAKE_BUILD_TYPE=Release'
+export PYARROW_BUILD_TYPE='release'
+export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_PARQUET=ON'
 # Need as otherwise arrow_io is sometimes not linked
 export LDFLAGS="-Wl,--no-as-needed"
 export ARROW_HOME="/usr"
+export PARQUET_HOME="/usr"
 
 # Ensure the target directory exists
 mkdir -p /io/dist
@@ -53,7 +55,7 @@ function repair_wheelhouse {
             cp $whl $out_dir
         else
             # Store libraries directly in . not .libs to fix problems with libpyarrow.so linkage.
-            auditwheel -v repair -L . $whl -w $out_dir/
+            $PY35_BIN/auditwheel -v repair -L . $whl -w $out_dir/
         fi
     done
     chmod -R a+rwX $out_dir
@@ -67,10 +69,10 @@ for PYTHON in ${PYTHON_VERSIONS}; do
 
     $PIPI_IO "numpy==1.9.0"
     $PIPI_IO "cython==0.24"
+    $PIPI_IO "cmake"
 
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
-    rm_mkdir fixed_wheels
     repair_wheelhouse dist /io/dist
 done
 

From 353772f844e227038ea8a3c5328a70e5fe553773 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 18 Jan 2017 18:28:02 -0500
Subject: [PATCH 0280/1644] ARROW-468: Python: Conversion of nested data in
 pd.DataFrames

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #289 from xhochy/ARROW-468 and squashes the following commits:

6fab6b2 [Uwe L. Korn] clang-format
c30da77 [Uwe L. Korn] Conversion for Lists of String and Timestamp
3ac373e [Uwe L. Korn] Fix string conversion
23fdc97 [Uwe L. Korn] Conversion of nested arrays to Pandas
a8197f7 [Uwe L. Korn] ARROW-468: Python: Conversion of nested data in pd.DataFrames
---
 python/pyarrow/array.pyx                    |  14 +-
 python/pyarrow/includes/libarrow.pxd        |   1 +
 python/pyarrow/includes/pyarrow.pxd         |   9 +-
 python/pyarrow/schema.pxd                   |   1 +
 python/pyarrow/schema.pyx                   |  19 +
 python/pyarrow/table.pyx                    |  26 +-
 python/pyarrow/tests/test_convert_pandas.py |  53 ++-
 python/src/pyarrow/adapters/pandas.cc       | 448 ++++++++++++++++----
 python/src/pyarrow/adapters/pandas.h        |   7 +-
 9 files changed, 477 insertions(+), 101 deletions(-)

diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 266768f7e0ba5..4299ba67f0944 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -33,7 +33,7 @@ from pyarrow.error cimport check_status
 cimport pyarrow.scalar as scalar
 from pyarrow.scalar import NA
 
-from pyarrow.schema cimport Schema
+from pyarrow.schema cimport Field, Schema
 import pyarrow.schema as schema
 
 cimport cpython
@@ -322,7 +322,7 @@ def from_pylist(object list_obj, DataType type=None):
     return box_arrow_array(sp_array)
 
 
-def from_pandas_series(object series, object mask=None, timestamps_to_ms=False):
+def from_pandas_series(object series, object mask=None, timestamps_to_ms=False, Field field=None):
     """
     Convert pandas.Series to an Arrow Array.
 
@@ -338,26 +338,32 @@ def from_pandas_series(object series, object mask=None, timestamps_to_ms=False):
         compability with other functionality like Parquet I/O which
         only supports milliseconds.
 
+    field: pyarrow.Field
+        Schema indicator to what type this column should render in Arrow
+
     Returns
     -------
     pyarrow.array.Array
     """
     cdef:
         shared_ptr[CArray] out
+        shared_ptr[CField] c_field
 
     series_values = series_as_ndarray(series)
     if series_values.dtype.type == np.datetime64 and timestamps_to_ms:
         series_values = series_values.astype('datetime64[ms]')
+    if field is not None:
+        c_field = field.sp_field
 
     if mask is None:
         with nogil:
             check_status(pyarrow.PandasToArrow(pyarrow.get_memory_pool(),
-                                               series_values, &out))
+                                               series_values, c_field, &out))
     else:
         mask = series_as_ndarray(mask)
         with nogil:
             check_status(pyarrow.PandasMaskedToArrow(
-                pyarrow.get_memory_pool(), series_values, mask, &out))
+                pyarrow.get_memory_pool(), series_values, mask, c_field, &out))
 
     return box_arrow_array(out)
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 8cfaaf72bf16f..8b0e3b6fb772a 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -107,6 +107,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool Equals(const shared_ptr[CSchema]& other)
 
         shared_ptr[CField] field(int i)
+        shared_ptr[CField] GetFieldByName(c_string& name)
         int num_fields()
         c_string ToString()
 
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 901e6c9457dfa..b7b8d7c63895a 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -18,9 +18,9 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CTable,
-                                        CDataType, CStatus, Type, MemoryPool,
-                                        TimeUnit)
+from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CField,
+                                        CTable, CDataType, CStatus, Type,
+                                        MemoryPool, TimeUnit)
 
 cimport pyarrow.includes.libarrow_io as arrow_io
 
@@ -30,9 +30,10 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
     CStatus ConvertPySequence(object obj, shared_ptr[CArray]* out)
 
-    CStatus PandasToArrow(MemoryPool* pool, object ao,
+    CStatus PandasToArrow(MemoryPool* pool, object ao, shared_ptr[CField] field,
                           shared_ptr[CArray]* out)
     CStatus PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
+                                shared_ptr[CField] field,
                                 shared_ptr[CArray]* out)
 
     CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
index f2cb776eb2e9f..42588d49b694b 100644
--- a/python/pyarrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -44,4 +44,5 @@ cdef class Schema:
     cdef init_schema(self, const shared_ptr[CSchema]& schema)
 
 cdef DataType box_data_type(const shared_ptr[CDataType]& type)
+cdef Field box_field(const shared_ptr[CField]& field)
 cdef Schema box_schema(const shared_ptr[CSchema]& schema)
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index a6aa9d597241d..85b16170b948e 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -133,6 +133,20 @@ cdef class Schema:
 
         return self.sp_schema.get().Equals(_other.sp_schema)
 
+    def field_by_name(self, name):
+        """
+        Access a field by its name rather than the column index.
+
+        Parameters
+        ----------
+        name: str
+
+        Returns
+        -------
+        field: pyarrow.Field
+        """
+        return box_field(self.schema.GetFieldByName(tobytes(name)))
+
     @classmethod
     def from_fields(cls, fields):
         cdef:
@@ -287,6 +301,11 @@ cdef DataType box_data_type(const shared_ptr[CDataType]& type):
     out.init(type)
     return out
 
+cdef Field box_field(const shared_ptr[CField]& field):
+    cdef Field out = Field()
+    out.init(field)
+    return out
+
 cdef Schema box_schema(const shared_ptr[CSchema]& type):
     cdef Schema out = Schema()
     out.init_schema(type)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index dce125a7b3236..b720a4723ec56 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -30,7 +30,7 @@ import pyarrow.config
 from pyarrow.array cimport Array, box_arrow_array
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
-from pyarrow.schema cimport box_data_type, box_schema
+from pyarrow.schema cimport box_data_type, box_schema, Field
 
 from pyarrow.compat import frombytes, tobytes
 
@@ -277,16 +277,20 @@ cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
 
 
 
-cdef _dataframe_to_arrays(df, name, timestamps_to_ms):
+cdef _dataframe_to_arrays(df, name, timestamps_to_ms, Schema schema):
     from pyarrow.array import from_pandas_series
 
     cdef:
         list names = []
         list arrays = []
+        Field field = None
 
     for name in df.columns:
         col = df[name]
-        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms)
+        if schema is not None:
+            field = schema.field_by_name(name)
+        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms,
+                                 field=field)
 
         names.append(name)
         arrays.append(arr)
@@ -424,19 +428,22 @@ cdef class RecordBatch:
         return pd.DataFrame(dict(zip(names, data)), columns=names)
 
     @classmethod
-    def from_pandas(cls, df):
+    def from_pandas(cls, df, schema=None):
         """
         Convert pandas.DataFrame to an Arrow RecordBatch
 
         Parameters
         ----------
         df: pandas.DataFrame
+        schema: pyarrow.Schema (optional)
+            The expected schema of the RecordBatch. This can be used to
+            indicate the type of columns if we cannot infer it automatically.
 
         Returns
         -------
         pyarrow.table.RecordBatch
         """
-        names, arrays = _dataframe_to_arrays(df, None, False)
+        names, arrays = _dataframe_to_arrays(df, None, False, schema)
         return cls.from_arrays(names, arrays)
 
     @staticmethod
@@ -552,7 +559,7 @@ cdef class Table:
         return result
 
     @classmethod
-    def from_pandas(cls, df, name=None, timestamps_to_ms=False):
+    def from_pandas(cls, df, name=None, timestamps_to_ms=False, schema=None):
         """
         Convert pandas.DataFrame to an Arrow Table
 
@@ -567,6 +574,10 @@ cdef class Table:
             compability with other functionality like Parquet I/O which
             only supports milliseconds.
 
+        schema: pyarrow.Schema (optional)
+            The expected schema of the Arrow Table. This can be used to
+            indicate the type of columns if we cannot infer it automatically.
+
         Returns
         -------
         pyarrow.table.Table
@@ -584,7 +595,8 @@ cdef class Table:
         <pyarrow.table.Table object at 0x7f05d1fb1b40>
         """
         names, arrays = _dataframe_to_arrays(df, name=name,
-                                             timestamps_to_ms=timestamps_to_ms)
+                                             timestamps_to_ms=timestamps_to_ms,
+                                             schema=schema)
         return cls.from_arrays(names, arrays, name=name)
 
     @staticmethod
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 261eaa85657ed..3928a1fc1b92b 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -16,6 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from collections import OrderedDict
+
 import datetime
 import unittest
 
@@ -60,8 +62,8 @@ def tearDown(self):
         pass
 
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
-                                timestamps_to_ms=False, expected_schema=None):
-        table = A.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms)
+                                timestamps_to_ms=False, expected_schema=None, schema=None):
+        table = A.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms, schema=schema)
         result = table.to_pandas(nthreads=nthreads)
         if expected_schema:
             assert table.schema.equals(expected_schema)
@@ -284,6 +286,53 @@ def test_date(self):
         expected['date'] = pd.to_datetime(df['date'])
         tm.assert_frame_equal(result, expected)
 
+    def test_column_of_lists(self):
+        dtypes = [('i1', A.int8()), ('i2', A.int16()),
+                  ('i4', A.int32()), ('i8', A.int64()),
+                  ('u1', A.uint8()), ('u2', A.uint16()),
+                  ('u4', A.uint32()), ('u8', A.uint64()),
+                  ('f4', A.float_()), ('f8', A.double())]
+
+        arrays = OrderedDict()
+        fields = []
+        for dtype, arrow_dtype in dtypes:
+            fields.append(A.field(dtype, A.list_(arrow_dtype)))
+            arrays[dtype] = [
+                np.arange(10, dtype=dtype),
+                np.arange(5, dtype=dtype),
+                None,
+                np.arange(1, dtype=dtype)
+            ]
+
+        fields.append(A.field('str', A.list_(A.string())))
+        arrays['str'] = [
+            np.array([u"1", u"ä"], dtype="object"),
+            None,
+            np.array([u"1"], dtype="object"),
+            np.array([u"1", u"2", u"3"], dtype="object")
+        ]
+
+        fields.append(A.field('datetime64', A.list_(A.timestamp('ns'))))
+        arrays['datetime64'] = [
+            np.array(['2007-07-13T01:23:34.123456789',
+                      None,
+                      '2010-08-13T05:46:57.437699912'],
+                      dtype='datetime64[ns]'),
+            None,
+            None,
+            np.array(['2007-07-13T02',
+                      None,
+                      '2010-08-13T05:46:57.437699912'],
+                      dtype='datetime64[ns]'),
+        ]
+
+        df = pd.DataFrame(arrays)
+        schema = A.Schema.from_fields(fields)
+        self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
+        table = A.Table.from_pandas(df, schema=schema)
+        assert table.schema.equals(schema)
+        df_new = table.to_pandas(nthreads=1)
+
     def test_threaded_conversion(self):
         df = _alltypes_example()
         self._check_pandas_roundtrip(df, nthreads=2,
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index ad18eca66e42b..8c2d3506c8a9d 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -36,6 +36,7 @@
 #include "arrow/api.h"
 #include "arrow/status.h"
 #include "arrow/type_fwd.h"
+#include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 
@@ -50,6 +51,8 @@ using arrow::ChunkedArray;
 using arrow::Column;
 using arrow::Field;
 using arrow::DataType;
+using arrow::ListType;
+using arrow::ListBuilder;
 using arrow::Status;
 using arrow::Table;
 using arrow::Type;
@@ -66,6 +69,7 @@ template <>
 struct npy_traits<NPY_BOOL> {
   typedef uint8_t value_type;
   using TypeClass = arrow::BooleanType;
+  using BuilderClass = arrow::BooleanBuilder;
 
   static constexpr bool supports_nulls = false;
   static inline bool isnull(uint8_t v) { return false; }
@@ -76,6 +80,7 @@ struct npy_traits<NPY_BOOL> {
   struct npy_traits<NPY_##TYPE> {                    \
     typedef T value_type;                            \
     using TypeClass = arrow::CapType##Type;          \
+    using BuilderClass = arrow::CapType##Builder;    \
                                                      \
     static constexpr bool supports_nulls = false;    \
     static inline bool isnull(T v) { return false; } \
@@ -94,6 +99,7 @@ template <>
 struct npy_traits<NPY_FLOAT32> {
   typedef float value_type;
   using TypeClass = arrow::FloatType;
+  using BuilderClass = arrow::FloatBuilder;
 
   static constexpr bool supports_nulls = true;
 
@@ -104,6 +110,7 @@ template <>
 struct npy_traits<NPY_FLOAT64> {
   typedef double value_type;
   using TypeClass = arrow::DoubleType;
+  using BuilderClass = arrow::DoubleBuilder;
 
   static constexpr bool supports_nulls = true;
 
@@ -114,6 +121,7 @@ template <>
 struct npy_traits<NPY_DATETIME> {
   typedef int64_t value_type;
   using TypeClass = arrow::TimestampType;
+  using BuilderClass = arrow::TimestampBuilder;
 
   static constexpr bool supports_nulls = true;
 
@@ -132,6 +140,107 @@ struct npy_traits<NPY_OBJECT> {
   static constexpr bool supports_nulls = true;
 };
 
+static inline bool PyObject_is_null(const PyObject* obj) {
+  return obj == Py_None || obj == numpy_nan;
+}
+
+static inline bool PyObject_is_string(const PyObject* obj) {
+#if PY_MAJOR_VERSION >= 3
+  return PyUnicode_Check(obj) || PyBytes_Check(obj);
+#else
+  return PyString_Check(obj) || PyUnicode_Check(obj);
+#endif
+}
+
+static inline bool PyObject_is_bool(const PyObject* obj) {
+#if PY_MAJOR_VERSION >= 3
+  return PyString_Check(obj) || PyBytes_Check(obj);
+#else
+  return PyString_Check(obj) || PyUnicode_Check(obj);
+#endif
+}
+
+template <int TYPE>
+static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap) {
+  typedef npy_traits<TYPE> traits;
+  typedef typename traits::value_type T;
+
+  int64_t null_count = 0;
+  const T* values = reinterpret_cast<const T*>(data);
+
+  // TODO(wesm): striding
+  for (int i = 0; i < length; ++i) {
+    if (traits::isnull(values[i])) {
+      ++null_count;
+    } else {
+      BitUtil::SetBit(bitmap, i);
+    }
+  }
+
+  return null_count;
+}
+
+template <int TYPE>
+static int64_t ValuesToBytemap(const void* data, int64_t length, uint8_t* valid_bytes) {
+  typedef npy_traits<TYPE> traits;
+  typedef typename traits::value_type T;
+
+  int64_t null_count = 0;
+  const T* values = reinterpret_cast<const T*>(data);
+
+  // TODO(wesm): striding
+  for (int i = 0; i < length; ++i) {
+    valid_bytes[i] = not traits::isnull(values[i]);
+    if (traits::isnull(values[i])) null_count++;
+  }
+
+  return null_count;
+}
+
+Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
+  if (PyArray_NDIM(numpy_array) != 1) {
+    return Status::Invalid("only handle 1-dimensional arrays");
+  }
+
+  if (PyArray_DESCR(numpy_array)->type_num != np_type) {
+    return Status::Invalid("can only handle exact conversions");
+  }
+
+  npy_intp* astrides = PyArray_STRIDES(numpy_array);
+  if (astrides[0] != PyArray_DESCR(numpy_array)->elsize) {
+    return Status::Invalid("No support for strided arrays in lists yet");
+  }
+  return Status::OK();
+}
+
+Status AppendObjectStrings(arrow::StringBuilder& string_builder, PyObject** objects,
+    int64_t objects_length, bool* have_bytes) {
+  PyObject* obj;
+
+  for (int64_t i = 0; i < objects_length; ++i) {
+    obj = objects[i];
+    if (PyUnicode_Check(obj)) {
+      obj = PyUnicode_AsUTF8String(obj);
+      if (obj == NULL) {
+        PyErr_Clear();
+        return Status::TypeError("failed converting unicode to UTF8");
+      }
+      const int32_t length = PyBytes_GET_SIZE(obj);
+      Status s = string_builder.Append(PyBytes_AS_STRING(obj), length);
+      Py_DECREF(obj);
+      if (!s.ok()) { return s; }
+    } else if (PyBytes_Check(obj)) {
+      *have_bytes = true;
+      const int32_t length = PyBytes_GET_SIZE(obj);
+      RETURN_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
+    } else {
+      string_builder.AppendNull();
+    }
+  }
+
+  return Status::OK();
+}
+
 template <int TYPE>
 class ArrowSerializer {
  public:
@@ -140,6 +249,8 @@ class ArrowSerializer {
     length_ = PyArray_SIZE(arr_);
   }
 
+  void IndicateType(const std::shared_ptr<Field> field) { field_indicator_ = field; }
+
   Status Convert(std::shared_ptr<Array>* out);
 
   int stride() const { return PyArray_STRIDES(arr_)[0]; }
@@ -198,28 +309,8 @@ class ArrowSerializer {
     RETURN_NOT_OK(string_builder.Resize(length_));
 
     Status s;
-    PyObject* obj;
     bool have_bytes = false;
-    for (int64_t i = 0; i < length_; ++i) {
-      obj = objects[i];
-      if (PyUnicode_Check(obj)) {
-        obj = PyUnicode_AsUTF8String(obj);
-        if (obj == NULL) {
-          PyErr_Clear();
-          return Status::TypeError("failed converting unicode to UTF8");
-        }
-        const int32_t length = PyBytes_GET_SIZE(obj);
-        s = string_builder.Append(PyBytes_AS_STRING(obj), length);
-        Py_DECREF(obj);
-        if (!s.ok()) { return s; }
-      } else if (PyBytes_Check(obj)) {
-        have_bytes = true;
-        const int32_t length = PyBytes_GET_SIZE(obj);
-        RETURN_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
-      } else {
-        string_builder.AppendNull();
-      }
-    }
+    RETURN_NOT_OK(AppendObjectStrings(string_builder, objects, length_, &have_bytes));
     RETURN_NOT_OK(string_builder.Finish(out));
 
     if (have_bytes) {
@@ -258,6 +349,36 @@ class ArrowSerializer {
     return Status::OK();
   }
 
+  template <int ITEM_TYPE, typename ArrowType>
+  Status ConvertTypedLists(
+      const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out);
+
+#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                            \
+  case Type::TYPE: {                                                      \
+    return ConvertTypedLists<NUMPY_TYPE, ::arrow::ArrowType>(field, out); \
+  }
+
+  Status ConvertLists(const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+    switch (field->type->type) {
+      LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
+      LIST_CASE(INT8, NPY_INT8, Int8Type)
+      LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
+      LIST_CASE(INT16, NPY_INT16, Int16Type)
+      LIST_CASE(UINT32, NPY_UINT32, UInt32Type)
+      LIST_CASE(INT32, NPY_INT32, Int32Type)
+      LIST_CASE(UINT64, NPY_UINT64, UInt64Type)
+      LIST_CASE(INT64, NPY_INT64, Int64Type)
+      LIST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
+      LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
+      LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
+      LIST_CASE(STRING, NPY_OBJECT, StringType)
+      default:
+        return Status::TypeError("Unknown list item type");
+    }
+
+    return Status::TypeError("Unknown list type");
+  }
+
   Status MakeDataType(std::shared_ptr<DataType>* out);
 
   arrow::MemoryPool* pool_;
@@ -267,6 +388,7 @@ class ArrowSerializer {
 
   int64_t length_;
 
+  std::shared_ptr<Field> field_indicator_;
   std::shared_ptr<arrow::Buffer> data_;
   std::shared_ptr<arrow::ResizableBuffer> null_bitmap_;
   uint8_t* null_bitmap_data_;
@@ -287,26 +409,6 @@ static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap
   return null_count;
 }
 
-template <int TYPE>
-static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap) {
-  typedef npy_traits<TYPE> traits;
-  typedef typename traits::value_type T;
-
-  int64_t null_count = 0;
-  const T* values = reinterpret_cast<const T*>(data);
-
-  // TODO(wesm): striding
-  for (int i = 0; i < length; ++i) {
-    if (traits::isnull(values[i])) {
-      ++null_count;
-    } else {
-      BitUtil::SetBit(bitmap, i);
-    }
-  }
-
-  return null_count;
-}
-
 template <int TYPE>
 inline Status ArrowSerializer<TYPE>::MakeDataType(std::shared_ptr<DataType>* out) {
   out->reset(new typename npy_traits<TYPE>::TypeClass());
@@ -361,26 +463,6 @@ inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
-static inline bool PyObject_is_null(const PyObject* obj) {
-  return obj == Py_None || obj == numpy_nan;
-}
-
-static inline bool PyObject_is_string(const PyObject* obj) {
-#if PY_MAJOR_VERSION >= 3
-  return PyUnicode_Check(obj) || PyBytes_Check(obj);
-#else
-  return PyString_Check(obj) || PyUnicode_Check(obj);
-#endif
-}
-
-static inline bool PyObject_is_bool(const PyObject* obj) {
-#if PY_MAJOR_VERSION >= 3
-  return PyString_Check(obj) || PyBytes_Check(obj);
-#else
-  return PyString_Check(obj) || PyUnicode_Check(obj);
-#endif
-}
-
 template <>
 inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out) {
   // Python object arrays are annoying, since we could have one of:
@@ -401,17 +483,34 @@ inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out)
     PyDateTime_IMPORT;
   }
 
-  for (int64_t i = 0; i < length_; ++i) {
-    if (PyObject_is_null(objects[i])) {
-      continue;
-    } else if (PyObject_is_string(objects[i])) {
-      return ConvertObjectStrings(out);
-    } else if (PyBool_Check(objects[i])) {
-      return ConvertBooleans(out);
-    } else if (PyDate_CheckExact(objects[i])) {
-      return ConvertDates(out);
-    } else {
-      return Status::TypeError("unhandled python type");
+  if (field_indicator_) {
+    switch (field_indicator_->type->type) {
+      case Type::STRING:
+        return ConvertObjectStrings(out);
+      case Type::BOOL:
+        return ConvertBooleans(out);
+      case Type::DATE:
+        return ConvertDates(out);
+      case Type::LIST: {
+        auto list_field = static_cast<ListType*>(field_indicator_->type.get());
+        return ConvertLists(list_field->value_field(), out);
+      }
+      default:
+        return Status::TypeError("No known conversion to Arrow type");
+    }
+  } else {
+    for (int64_t i = 0; i < length_; ++i) {
+      if (PyObject_is_null(objects[i])) {
+        continue;
+      } else if (PyObject_is_string(objects[i])) {
+        return ConvertObjectStrings(out);
+      } else if (PyBool_Check(objects[i])) {
+        return ConvertBooleans(out);
+      } else if (PyDate_CheckExact(objects[i])) {
+        return ConvertDates(out);
+      } else {
+        return Status::TypeError("unhandled python type");
+      }
     }
   }
 
@@ -449,6 +548,81 @@ inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
   return Status::OK();
 }
 
+template <int TYPE>
+template <int ITEM_TYPE, typename ArrowType>
+inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
+    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+  typedef npy_traits<ITEM_TYPE> traits;
+  typedef typename traits::value_type T;
+  typedef typename traits::BuilderClass BuilderT;
+
+  auto value_builder = std::make_shared<BuilderT>(pool_, field->type);
+  ListBuilder list_builder(pool_, value_builder);
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  for (int64_t i = 0; i < length_; ++i) {
+    if (PyObject_is_null(objects[i])) {
+      RETURN_NOT_OK(list_builder.AppendNull());
+    } else if (PyArray_Check(objects[i])) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
+      RETURN_NOT_OK(list_builder.Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
+
+      int32_t size = PyArray_DIM(numpy_array, 0);
+      auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
+      if (traits::supports_nulls) {
+        null_bitmap_->Resize(size, false);
+        // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
+        // currently support this.
+        // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
+        ValuesToBytemap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
+        RETURN_NOT_OK(value_builder->Append(data, size, null_bitmap_->data()));
+      } else {
+        RETURN_NOT_OK(value_builder->Append(data, size));
+      }
+    } else if (PyList_Check(objects[i])) {
+      return Status::TypeError("Python lists are not yet supported");
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
+  }
+  return list_builder.Finish(out);
+}
+
+template <>
+template <>
+inline Status
+ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
+    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+  // TODO: If there are bytes involed, convert to Binary representation
+  bool have_bytes = false;
+
+  auto value_builder = std::make_shared<arrow::StringBuilder>(pool_, field->type);
+  ListBuilder list_builder(pool_, value_builder);
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  for (int64_t i = 0; i < length_; ++i) {
+    if (PyObject_is_null(objects[i])) {
+      RETURN_NOT_OK(list_builder.AppendNull());
+    } else if (PyArray_Check(objects[i])) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
+      RETURN_NOT_OK(list_builder.Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+
+      int32_t size = PyArray_DIM(numpy_array, 0);
+      auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
+      RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
+    } else if (PyList_Check(objects[i])) {
+      return Status::TypeError("Python lists are not yet supported");
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
+  }
+  return list_builder.Finish(out);
+}
+
 template <>
 inline Status ArrowSerializer<NPY_OBJECT>::ConvertData() {
   return Status::TypeError("NYI");
@@ -460,8 +634,8 @@ inline Status ArrowSerializer<NPY_OBJECT>::ConvertData() {
     RETURN_NOT_OK(converter.Convert(out));                  \
   } break;
 
-Status PandasMaskedToArrow(
-    arrow::MemoryPool* pool, PyObject* ao, PyObject* mo, std::shared_ptr<Array>* out) {
+Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
   PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(ao);
   PyArrayObject* mask = nullptr;
 
@@ -484,7 +658,11 @@ Status PandasMaskedToArrow(
     TO_ARROW_CASE(FLOAT32);
     TO_ARROW_CASE(FLOAT64);
     TO_ARROW_CASE(DATETIME);
-    TO_ARROW_CASE(OBJECT);
+    case NPY_OBJECT: {
+      ArrowSerializer<NPY_OBJECT> converter(pool, arr, mask);
+      converter.IndicateType(field);
+      RETURN_NOT_OK(converter.Convert(out));
+    } break;
     default:
       std::stringstream ss;
       ss << "unsupported type " << PyArray_DESCR(arr)->type_num << std::endl;
@@ -493,8 +671,9 @@ Status PandasMaskedToArrow(
   return Status::OK();
 }
 
-Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, std::shared_ptr<Array>* out) {
-  return PandasMaskedToArrow(pool, ao, nullptr, out);
+Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
+    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+  return PandasMaskedToArrow(pool, ao, nullptr, field, out);
 }
 
 // ----------------------------------------------------------------------
@@ -739,6 +918,56 @@ inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values)
   return Status::OK();
 }
 
+template <typename ArrowType>
+inline Status ConvertListsLike(
+    const std::shared_ptr<Column>& col, PyObject** out_values) {
+  typedef arrow_traits<ArrowType::type_id> traits;
+  typedef typename ::arrow::TypeTraits<ArrowType>::ArrayType ArrayType;
+
+  const ChunkedArray& data = *col->data().get();
+  auto list_type = std::static_pointer_cast<ListType>(col->type());
+
+  // Get column of underlying value arrays
+  std::vector<std::shared_ptr<Array>> value_arrays;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
+    value_arrays.emplace_back(arr->values());
+  }
+  auto flat_column = std::make_shared<Column>(list_type->value_field(), value_arrays);
+  // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
+  //    Storing a reference to the whole Array would be to expensive.
+  PyObject* numpy_array;
+  RETURN_NOT_OK(ConvertColumnToPandas(flat_column, nullptr, &numpy_array));
+
+  PyAcquireGIL lock;
+
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
+
+    const uint8_t* data_ptr;
+    int32_t length;
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (has_nulls && arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        PyObject* start = PyLong_FromLong(arr->value_offset(i));
+        PyObject* end = PyLong_FromLong(arr->value_offset(i + 1));
+        PyObject* slice = PySlice_New(start, end, NULL);
+        *out_values = PyObject_GetItem(numpy_array, slice);
+        Py_DECREF(start);
+        Py_DECREF(end);
+        Py_DECREF(slice);
+      }
+      ++out_values;
+    }
+  }
+
+  Py_XDECREF(numpy_array);
+  return Status::OK();
+}
+
 template <typename T>
 inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
@@ -886,8 +1115,11 @@ class ArrowDeserializer {
       CONVERT_CASE(STRING);
       CONVERT_CASE(DATE);
       CONVERT_CASE(TIMESTAMP);
-      default:
-        return Status::NotImplemented("Arrow type reading not implemented");
+      default: {
+        std::stringstream ss;
+        ss << "Arrow type reading not implemented for " << col_->type()->ToString();
+        return Status::NotImplemented(ss.str());
+      }
     }
 
 #undef CONVERT_CASE
@@ -903,7 +1135,7 @@ class ArrowDeserializer {
     typedef typename arrow_traits<TYPE>::T T;
     int npy_type = arrow_traits<TYPE>::npy_type;
 
-    if (data_.num_chunks() == 1 && data_.null_count() == 0) {
+    if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
       return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
     }
 
@@ -933,7 +1165,7 @@ class ArrowDeserializer {
     typedef typename arrow_traits<TYPE>::T T;
     int npy_type = arrow_traits<TYPE>::npy_type;
 
-    if (data_.num_chunks() == 1 && data_.null_count() == 0) {
+    if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
       return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
     }
 
@@ -1028,6 +1260,7 @@ class PandasBlock {
 
   PandasBlock(int64_t num_rows, int num_columns)
       : num_rows_(num_rows), num_columns_(num_columns) {}
+  virtual ~PandasBlock() {}
 
   virtual Status Allocate() = 0;
   virtual Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
@@ -1080,9 +1313,15 @@ class PandasBlock {
   DISALLOW_COPY_AND_ASSIGN(PandasBlock);
 };
 
+#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum)                         \
+  case Type::ArrowEnum:                                                     \
+    RETURN_NOT_OK((ConvertListsLike<::arrow::ArrowType>(col, out_buffer))); \
+    break;
+
 class ObjectBlock : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
+  virtual ~ObjectBlock() {}
 
   Status Allocate() override { return AllocateNDArray(NPY_OBJECT); }
 
@@ -1101,6 +1340,27 @@ class ObjectBlock : public PandasBlock {
       RETURN_NOT_OK(ConvertBinaryLike<arrow::BinaryArray>(data, out_buffer));
     } else if (type == Type::STRING) {
       RETURN_NOT_OK(ConvertBinaryLike<arrow::StringArray>(data, out_buffer));
+    } else if (type == Type::LIST) {
+      auto list_type = std::static_pointer_cast<ListType>(col->type());
+      switch (list_type->value_type()->type) {
+        CONVERTLISTSLIKE_CASE(UInt8Type, UINT8)
+        CONVERTLISTSLIKE_CASE(Int8Type, INT8)
+        CONVERTLISTSLIKE_CASE(UInt16Type, UINT16)
+        CONVERTLISTSLIKE_CASE(Int16Type, INT16)
+        CONVERTLISTSLIKE_CASE(UInt32Type, UINT32)
+        CONVERTLISTSLIKE_CASE(Int32Type, INT32)
+        CONVERTLISTSLIKE_CASE(UInt64Type, UINT64)
+        CONVERTLISTSLIKE_CASE(Int64Type, INT64)
+        CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP)
+        CONVERTLISTSLIKE_CASE(FloatType, FLOAT)
+        CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE)
+        CONVERTLISTSLIKE_CASE(StringType, STRING)
+        default: {
+          std::stringstream ss;
+          ss << "Not implemented type for lists: " << list_type->value_type()->ToString();
+          return Status::NotImplemented(ss.str());
+        }
+      }
     } else {
       std::stringstream ss;
       ss << "Unsupported type for object array output: " << col->type()->ToString();
@@ -1396,6 +1656,32 @@ class DataFrameBlockCreator {
         case Type::TIMESTAMP:
           output_type = PandasBlock::DATETIME;
           break;
+        case Type::LIST: {
+          auto list_type = std::static_pointer_cast<ListType>(col->type());
+          switch (list_type->value_type()->type) {
+            case Type::UINT8:
+            case Type::INT8:
+            case Type::UINT16:
+            case Type::INT16:
+            case Type::UINT32:
+            case Type::INT32:
+            case Type::INT64:
+            case Type::UINT64:
+            case Type::FLOAT:
+            case Type::DOUBLE:
+            case Type::STRING:
+            case Type::TIMESTAMP:
+              // The above types are all supported.
+              break;
+            default: {
+              std::stringstream ss;
+              ss << "Not implemented type for lists: "
+                 << list_type->value_type()->ToString();
+              return Status::NotImplemented(ss.str());
+            }
+          }
+          output_type = PandasBlock::OBJECT;
+        } break;
         default:
           return Status::NotImplemented(col->type()->ToString());
       }
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index 60dadd473ad3f..664365e398384 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -31,6 +31,7 @@ namespace arrow {
 
 class Array;
 class Column;
+class Field;
 class MemoryPool;
 class Status;
 class Table;
@@ -63,11 +64,11 @@ arrow::Status ConvertTableToPandas(
 
 PYARROW_EXPORT
 arrow::Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
-    std::shared_ptr<arrow::Array>* out);
+    const std::shared_ptr<arrow::Field>& field, std::shared_ptr<arrow::Array>* out);
 
 PYARROW_EXPORT
-arrow::Status PandasToArrow(
-    arrow::MemoryPool* pool, PyObject* ao, std::shared_ptr<arrow::Array>* out);
+arrow::Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
+    const std::shared_ptr<arrow::Field>& field, std::shared_ptr<arrow::Array>* out);
 
 }  // namespace pyarrow
 

From 9b1b3979b499dc06b71a31b2696534550503d6e2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 19 Jan 2017 09:27:32 -0500
Subject: [PATCH 0281/1644] ARROW-461: [Python] Add Python interfaces to
 DictionaryArray data, pandas interop

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #291 from wesm/ARROW-461 and squashes the following commits:

b3efe96 [Wes McKinney] Fix cpp unit test, code review comments
285f863 [Wes McKinney] Accept list input in Array.from_pandas
16aa9d6 [Wes McKinney] Add Categorical conversion for single array or column. Required moving code around a little bit in pandas.cc
3d409e8 [Wes McKinney] First round of DataFrame-level dictionary to Categorical conversion
0b20c38 [Wes McKinney] Draft Python wrapper classes for DictionaryType, DictionaryArray. Avoid segfault when conversion to UTF8 fails. Starting on CategoricalBlock implementation
---
 cpp/src/arrow/array-dictionary-test.cc       |    2 +-
 cpp/src/arrow/array.h                        |    2 +
 cpp/src/arrow/builder.cc                     |    9 +-
 cpp/src/arrow/builder.h                      |    7 +-
 cpp/src/arrow/type.cc                        |    4 +-
 python/CMakeLists.txt                        |   10 +
 python/cmake_modules/FindPythonLibsNew.cmake |    3 +-
 python/pyarrow/__init__.py                   |   11 +-
 python/pyarrow/array.pxd                     |   44 +-
 python/pyarrow/array.pyx                     |  230 +-
 python/pyarrow/includes/libarrow.pxd         |   38 +-
 python/pyarrow/includes/pyarrow.pxd          |    6 +-
 python/pyarrow/schema.pxd                    |    8 +-
 python/pyarrow/schema.pyx                    |   28 +
 python/pyarrow/table.pyx                     |   74 +-
 python/pyarrow/tests/test_column.py          |    1 -
 python/pyarrow/tests/test_convert_pandas.py  |   61 +-
 python/setup.py                              |    1 +
 python/src/pyarrow/CMakeLists.txt            |    2 +
 python/src/pyarrow/adapters/pandas-test.cc   |   64 +
 python/src/pyarrow/adapters/pandas.cc        | 2694 +++++++++---------
 python/src/pyarrow/adapters/pandas.h         |    6 +-
 python/src/pyarrow/common.cc                 |    2 +-
 python/src/pyarrow/common.h                  |   43 +-
 python/src/pyarrow/io.cc                     |   18 +-
 python/src/pyarrow/util/CMakeLists.txt       |    2 +-
 python/src/pyarrow/util/test_main.cc         |   10 +
 27 files changed, 1881 insertions(+), 1499 deletions(-)
 create mode 100644 python/src/pyarrow/adapters/pandas-test.cc

diff --git a/cpp/src/arrow/array-dictionary-test.cc b/cpp/src/arrow/array-dictionary-test.cc
index c290153b95053..1a0d49a118f50 100644
--- a/cpp/src/arrow/array-dictionary-test.cc
+++ b/cpp/src/arrow/array-dictionary-test.cc
@@ -46,7 +46,7 @@ TEST(TestDictionary, Basics) {
   ASSERT_TRUE(int16()->Equals(type2.index_type()));
   ASSERT_TRUE(type2.dictionary()->Equals(dict));
 
-  ASSERT_EQ("dictionary<int32, int16>", type1->ToString());
+  ASSERT_EQ("dictionary<values=int32, indices=int16>", type1->ToString());
 }
 
 TEST(TestDictionary, Equals) {
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 45f8ab9f56791..4f4b727f39ae5 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -553,6 +553,8 @@ class ARROW_EXPORT DictionaryArray : public Array {
   std::shared_ptr<Array> indices() const { return indices_; }
   std::shared_ptr<Array> dictionary() const;
 
+  const DictionaryType* dict_type() { return dict_type_; }
+
   bool EqualsExact(const DictionaryArray& other) const;
   bool Equals(const std::shared_ptr<Array>& arr) const override;
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index a308ea53c570c..b0dc41baf4202 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -421,9 +421,12 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(FLOAT, FloatBuilder);
     BUILDER_CASE(DOUBLE, DoubleBuilder);
 
-    BUILDER_CASE(STRING, StringBuilder);
-    BUILDER_CASE(BINARY, BinaryBuilder);
-
+    case Type::STRING:
+      out->reset(new StringBuilder(pool));
+      return Status::OK();
+    case Type::BINARY:
+      out->reset(new BinaryBuilder(pool, type));
+      return Status::OK();
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 1837340cedc81..735bca1b1bcb3 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -24,6 +24,7 @@
 #include <vector>
 
 #include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/bit-util.h"
@@ -33,7 +34,6 @@
 namespace arrow {
 
 class Array;
-class MemoryPool;
 
 static constexpr int32_t kMinBuilderCapacity = 1 << 5;
 
@@ -378,7 +378,10 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
 // String builder
 class ARROW_EXPORT StringBuilder : public BinaryBuilder {
  public:
-  explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
+  explicit StringBuilder(MemoryPool* pool = default_memory_pool())
+      : BinaryBuilder(pool, utf8()) {}
+
+  explicit StringBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
       : BinaryBuilder(pool, type) {}
 
   using BinaryBuilder::Append;
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 954fba7af9df9..ba775845fcaf2 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -148,8 +148,8 @@ bool DictionaryType::Equals(const DataType& other) const {
 
 std::string DictionaryType::ToString() const {
   std::stringstream ss;
-  ss << "dictionary<" << dictionary_->type()->ToString() << ", "
-     << index_type_->ToString() << ">";
+  ss << "dictionary<values=" << dictionary_->type()->ToString()
+     << ", indices=" << index_type_->ToString() << ">";
   return ss.str();
 }
 
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 45115d49d455e..0a2d4e9facba2 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -55,6 +55,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     OFF)
 endif()
 
+if(NOT PYARROW_BUILD_TESTS)
+  set(NO_TESTS 1)
+endif()
+
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
   set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
@@ -339,6 +343,12 @@ set(PYARROW_MIN_TEST_LIBS
   pyarrow
   ${PYARROW_BASE_LIBS})
 
+if(NOT APPLE)
+  ADD_THIRDPARTY_LIB(python
+    SHARED_LIB "${PYTHON_LIBRARIES}")
+  list(APPEND PYARROW_MIN_TEST_LIBS python)
+endif()
+
 set(PYARROW_TEST_LINK_LIBS ${PYARROW_MIN_TEST_LIBS})
 
 ############################################################
diff --git a/python/cmake_modules/FindPythonLibsNew.cmake b/python/cmake_modules/FindPythonLibsNew.cmake
index 5cb65c9f1a484..1000a957a6269 100644
--- a/python/cmake_modules/FindPythonLibsNew.cmake
+++ b/python/cmake_modules/FindPythonLibsNew.cmake
@@ -161,6 +161,7 @@ else()
         set(_PYTHON_LIBS_SEARCH "${PYTHON_PREFIX}/lib" "${PYTHON_LIBRARY_PATH}")
     endif()
     message(STATUS "Searching for Python libs in ${_PYTHON_LIBS_SEARCH}")
+    message(STATUS "Looking for python${PYTHON_LIBRARY_SUFFIX}")
     # Probably this needs to be more involved. It would be nice if the config
     # information the python interpreter itself gave us were more complete.
     find_library(PYTHON_LIBRARY
@@ -237,4 +238,4 @@ FUNCTION(PYTHON_ADD_MODULE _NAME )
     ENDIF()
 
   ENDIF(PYTHON_ENABLE_MODULE_${_NAME})
-ENDFUNCTION(PYTHON_ADD_MODULE)
\ No newline at end of file
+ENDFUNCTION(PYTHON_ADD_MODULE)
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index a8c3e8e3158f8..efffbf2a4588d 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -31,9 +31,14 @@
 from pyarrow.array import (Array,
                            from_pandas_series, from_pylist,
                            total_allocated_bytes,
-                           BooleanArray, NumericArray,
+                           NumericArray, IntegerArray, FloatingPointArray,
+                           BooleanArray,
                            Int8Array, UInt8Array,
-                           ListArray, StringArray)
+                           Int16Array, UInt16Array,
+                           Int32Array, UInt32Array,
+                           Int64Array, UInt64Array,
+                           ListArray, StringArray,
+                           DictionaryArray)
 
 from pyarrow.error import ArrowException
 
@@ -52,7 +57,7 @@
                             uint8, uint16, uint32, uint64,
                             timestamp, date,
                             float_, double, binary, string,
-                            list_, struct, field,
+                            list_, struct, dictionary, field,
                             DataType, Field, Schema, schema)
 
 from pyarrow.table import Column, RecordBatch, Table, concat_tables
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index 8cd15cd450219..af105354ac2f3 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -22,6 +22,8 @@ from pyarrow.scalar import NA
 
 from pyarrow.schema cimport DataType
 
+from cpython cimport PyObject
+
 cdef extern from "Python.h":
     int PySlice_Check(object)
 
@@ -47,35 +49,50 @@ cdef class NumericArray(Array):
     pass
 
 
-cdef class Int8Array(NumericArray):
+cdef class IntegerArray(NumericArray):
+    pass
+
+cdef class FloatingPointArray(NumericArray):
+    pass
+
+
+cdef class Int8Array(IntegerArray):
+    pass
+
+
+cdef class UInt8Array(IntegerArray):
+    pass
+
+
+cdef class Int16Array(IntegerArray):
     pass
 
 
-cdef class UInt8Array(NumericArray):
+cdef class UInt16Array(IntegerArray):
     pass
 
 
-cdef class Int16Array(NumericArray):
+cdef class Int32Array(IntegerArray):
     pass
 
 
-cdef class UInt16Array(NumericArray):
+cdef class UInt32Array(IntegerArray):
     pass
 
 
-cdef class Int32Array(NumericArray):
+cdef class Int64Array(IntegerArray):
     pass
 
 
-cdef class UInt32Array(NumericArray):
+cdef class UInt64Array(IntegerArray):
     pass
 
 
-cdef class Int64Array(NumericArray):
+cdef class FloatArray(FloatingPointArray):
     pass
 
 
-cdef class UInt64Array(NumericArray):
+cdef class DoubleArray(FloatingPointArray):
     pass
 
 
@@ -85,3 +102,14 @@ cdef class ListArray(Array):
 
 cdef class StringArray(Array):
     pass
+
+
+cdef class BinaryArray(Array):
+    pass
+
+
+cdef class DictionaryArray(Array):
+    pass
+
+
+cdef wrap_array_output(PyObject* output)
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 4299ba67f0944..92206f2451ffb 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -33,12 +33,17 @@ from pyarrow.error cimport check_status
 cimport pyarrow.scalar as scalar
 from pyarrow.scalar import NA
 
-from pyarrow.schema cimport Field, Schema
+from pyarrow.schema cimport Field, Schema, DictionaryType
 import pyarrow.schema as schema
 
 cimport cpython
 
 
+cdef _pandas():
+    import pandas as pd
+    return pd
+
+
 def total_allocated_bytes():
     cdef MemoryPool* pool = pyarrow.get_memory_pool()
     return pool.bytes_allocated()
@@ -53,20 +58,22 @@ cdef class Array:
         self.type.init(self.sp_array.get().type())
 
     @staticmethod
-    def from_pandas(obj, mask=None):
+    def from_pandas(obj, mask=None, timestamps_to_ms=False, Field field=None):
         """
-        Create an array from a pandas.Series
+        Convert pandas.Series to an Arrow Array.
 
         Parameters
         ----------
-        obj : pandas.Series or numpy.ndarray
-            vector holding the data
-        mask : numpy.ndarray, optional
+        series : pandas.Series or numpy.ndarray
+
+        mask : pandas.Series or numpy.ndarray, optional
             boolean mask if the object is valid or null
 
-        Returns
-        -------
-        pyarrow.Array
+        timestamps_to_ms : bool, optional
+            Convert datetime columns to ms resolution. This is needed for
+            compability with other functionality like Parquet I/O which
+            only supports milliseconds.
+
 
         Examples
         --------
@@ -80,16 +87,47 @@ cdef class Array:
           2
         ]
 
-
         >>> import numpy as np
-        >>> pa.Array.from_pandas(pd.Series([1, 2]), np.array([0, 1], dtype=bool))
+        >>> pa.Array.from_pandas(pd.Series([1, 2]), np.array([0, 1],
+        ... dtype=bool))
         <pyarrow.array.Int64Array object at 0x7f9019e11208>
         [
           1,
           NA
         ]
+
+        Returns
+        -------
+        pyarrow.array.Array
         """
-        return from_pandas_series(obj, mask)
+        cdef:
+            shared_ptr[CArray] out
+            shared_ptr[CField] c_field
+
+        pd = _pandas()
+
+        if field is not None:
+            c_field = field.sp_field
+
+        if mask is not None:
+            mask = get_series_values(mask)
+
+        series_values = get_series_values(obj)
+
+        if isinstance(series_values, pd.Categorical):
+            return DictionaryArray.from_arrays(series_values.codes,
+                                               series_values.categories.values,
+                                               mask=mask)
+        else:
+            if series_values.dtype.type == np.datetime64 and timestamps_to_ms:
+                series_values = series_values.astype('datetime64[ms]')
+
+            with nogil:
+                check_status(pyarrow.PandasToArrow(
+                    pyarrow.get_memory_pool(), series_values, mask,
+                    c_field, &out))
+
+            return box_arrow_array(out)
 
     @staticmethod
     def from_list(object list_obj, DataType type=None):
@@ -183,12 +221,13 @@ cdef class Array:
         RecordBatch.to_pandas
         """
         cdef:
-            PyObject* np_arr
-
-        check_status(pyarrow.ConvertArrayToPandas(
-            self.sp_array, <PyObject*> self, &np_arr))
+            PyObject* out
 
-        return PyObject_to_object(np_arr)
+        with nogil:
+            check_status(
+                pyarrow.ConvertArrayToPandas(self.sp_array, <PyObject*> self,
+                                             &out))
+        return wrap_array_output(out)
 
     def to_pylist(self):
         """
@@ -197,6 +236,17 @@ cdef class Array:
         return [x.as_py() for x in self]
 
 
+cdef wrap_array_output(PyObject* output):
+    cdef object obj = PyObject_to_object(output)
+
+    if isinstance(obj, dict):
+        return _pandas().Categorical(obj['indices'],
+                                     categories=obj['dictionary'],
+                                     fastpath=True)
+    else:
+        return obj
+
+
 cdef class NullArray(Array):
     pass
 
@@ -209,35 +259,43 @@ cdef class NumericArray(Array):
     pass
 
 
-cdef class Int8Array(NumericArray):
+cdef class IntegerArray(NumericArray):
+    pass
+
+
+cdef class FloatingPointArray(NumericArray):
+    pass
+
+
+cdef class Int8Array(IntegerArray):
     pass
 
 
-cdef class UInt8Array(NumericArray):
+cdef class UInt8Array(IntegerArray):
     pass
 
 
-cdef class Int16Array(NumericArray):
+cdef class Int16Array(IntegerArray):
     pass
 
 
-cdef class UInt16Array(NumericArray):
+cdef class UInt16Array(IntegerArray):
     pass
 
 
-cdef class Int32Array(NumericArray):
+cdef class Int32Array(IntegerArray):
     pass
 
 
-cdef class UInt32Array(NumericArray):
+cdef class UInt32Array(IntegerArray):
     pass
 
 
-cdef class Int64Array(NumericArray):
+cdef class Int64Array(IntegerArray):
     pass
 
 
-cdef class UInt64Array(NumericArray):
+cdef class UInt64Array(IntegerArray):
     pass
 
 
@@ -245,11 +303,11 @@ cdef class DateArray(NumericArray):
     pass
 
 
-cdef class FloatArray(NumericArray):
+cdef class FloatArray(FloatingPointArray):
     pass
 
 
-cdef class DoubleArray(NumericArray):
+cdef class DoubleArray(FloatingPointArray):
     pass
 
 
@@ -265,6 +323,46 @@ cdef class BinaryArray(Array):
     pass
 
 
+cdef class DictionaryArray(Array):
+
+    @staticmethod
+    def from_arrays(indices, dictionary, mask=None):
+        """
+        Construct Arrow DictionaryArray from array of indices (must be
+        non-negative integers) and corresponding array of dictionary values
+
+        Parameters
+        ----------
+        indices : ndarray or pandas.Series, integer type
+        dictionary : ndarray or pandas.Series
+        mask : ndarray or pandas.Series, boolean type
+            True values indicate that indices are actually null
+
+        Returns
+        -------
+        dict_array : DictionaryArray
+        """
+        cdef:
+            Array arrow_indices, arrow_dictionary
+            DictionaryArray result
+            shared_ptr[CDataType] c_type
+            shared_ptr[CArray] c_result
+
+        arrow_indices = Array.from_pandas(indices, mask=mask)
+        arrow_dictionary = Array.from_pandas(dictionary)
+
+        if not isinstance(arrow_indices, IntegerArray):
+            raise ValueError('Indices must be integer type')
+
+        c_type.reset(new CDictionaryType(arrow_indices.type.sp_type,
+                                         arrow_dictionary.sp_array))
+        c_result.reset(new CDictionaryArray(c_type, arrow_indices.sp_array))
+
+        result = DictionaryArray()
+        result.init(c_result)
+        return result
+
+
 cdef dict _array_classes = {
     Type_NA: NullArray,
     Type_BOOL: BooleanArray,
@@ -283,6 +381,7 @@ cdef dict _array_classes = {
     Type_BINARY: BinaryArray,
     Type_STRING: StringArray,
     Type_TIMESTAMP: Int64Array,
+    Type_DICTIONARY: DictionaryArray
 }
 
 cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
@@ -299,83 +398,18 @@ cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
     return arr
 
 
-def from_pylist(object list_obj, DataType type=None):
-    """
-    Convert Python list to Arrow array
-
-    Parameters
-    ----------
-    list_obj : array_like
-
-    Returns
-    -------
-    pyarrow.array.Array
-    """
-    cdef:
-        shared_ptr[CArray] sp_array
-
-    if type is None:
-        check_status(pyarrow.ConvertPySequence(list_obj, &sp_array))
-    else:
-        raise NotImplementedError()
-
-    return box_arrow_array(sp_array)
-
-
-def from_pandas_series(object series, object mask=None, timestamps_to_ms=False, Field field=None):
-    """
-    Convert pandas.Series to an Arrow Array.
-
-    Parameters
-    ----------
-    series : pandas.Series or numpy.ndarray
-
-    mask : pandas.Series or numpy.ndarray, optional
-        array to mask null entries in the series
-
-    timestamps_to_ms : bool, optional
-        Convert datetime columns to ms resolution. This is needed for
-        compability with other functionality like Parquet I/O which
-        only supports milliseconds.
-
-    field: pyarrow.Field
-        Schema indicator to what type this column should render in Arrow
-
-    Returns
-    -------
-    pyarrow.array.Array
-    """
-    cdef:
-        shared_ptr[CArray] out
-        shared_ptr[CField] c_field
-
-    series_values = series_as_ndarray(series)
-    if series_values.dtype.type == np.datetime64 and timestamps_to_ms:
-        series_values = series_values.astype('datetime64[ms]')
-    if field is not None:
-        c_field = field.sp_field
-
-    if mask is None:
-        with nogil:
-            check_status(pyarrow.PandasToArrow(pyarrow.get_memory_pool(),
-                                               series_values, c_field, &out))
-    else:
-        mask = series_as_ndarray(mask)
-        with nogil:
-            check_status(pyarrow.PandasMaskedToArrow(
-                pyarrow.get_memory_pool(), series_values, mask, c_field, &out))
-
-    return box_arrow_array(out)
-
-
-cdef object series_as_ndarray(object obj):
+cdef object get_series_values(object obj):
     import pandas as pd
 
     if isinstance(obj, pd.Series):
         result = obj.values
-    else:
+    elif isinstance(obj, np.ndarray):
         result = obj
+    else:
+        result = pd.Series(obj).values
 
     return result
 
+
 from_pylist = Array.from_list
+from_pandas_series = Array.from_pandas
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 8b0e3b6fb772a..6284ad3c88a7a 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -45,6 +45,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         Type_LIST" arrow::Type::LIST"
         Type_STRUCT" arrow::Type::STRUCT"
+        Type_DICTIONARY" arrow::Type::DICTIONARY"
 
     enum TimeUnit" arrow::TimeUnit":
         TimeUnit_SECOND" arrow::TimeUnit::SECOND"
@@ -60,6 +61,33 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_string ToString()
 
+    cdef cppclass CArray" arrow::Array":
+        shared_ptr[CDataType] type()
+
+        int32_t length()
+        int32_t null_count()
+        Type type_enum()
+
+        c_bool Equals(const shared_ptr[CArray]& arr)
+        c_bool IsNull(int i)
+
+    cdef cppclass CFixedWidthType" arrow::FixedWidthType"(CDataType):
+        int bit_width()
+
+    cdef cppclass CDictionaryArray" arrow::DictionaryArray"(CArray):
+        CDictionaryArray(const shared_ptr[CDataType]& type,
+                         const shared_ptr[CArray]& indices)
+
+        shared_ptr[CArray] indices()
+        shared_ptr[CArray] dictionary()
+
+    cdef cppclass CDictionaryType" arrow::DictionaryType"(CFixedWidthType):
+        CDictionaryType(const shared_ptr[CDataType]& index_type,
+                        const shared_ptr[CArray]& dictionary)
+
+        shared_ptr[CDataType] index_type()
+        shared_ptr[CArray] dictionary()
+
     shared_ptr[CDataType] timestamp(TimeUnit unit)
 
     cdef cppclass MemoryPool" arrow::MemoryPool":
@@ -111,16 +139,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int num_fields()
         c_string ToString()
 
-    cdef cppclass CArray" arrow::Array":
-        shared_ptr[CDataType] type()
-
-        int32_t length()
-        int32_t null_count()
-        Type type_enum()
-
-        c_bool Equals(const shared_ptr[CArray]& arr)
-        c_bool IsNull(int i)
-
     cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray):
         c_bool Value(int i)
 
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index b7b8d7c63895a..04ad4f32272e6 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -30,11 +30,9 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
     CStatus ConvertPySequence(object obj, shared_ptr[CArray]* out)
 
-    CStatus PandasToArrow(MemoryPool* pool, object ao, shared_ptr[CField] field,
+    CStatus PandasToArrow(MemoryPool* pool, object ao, object mo,
+                          shared_ptr[CField] field,
                           shared_ptr[CArray]* out)
-    CStatus PandasMaskedToArrow(MemoryPool* pool, object ao, object mo,
-                                shared_ptr[CField] field,
-                                shared_ptr[CArray]* out)
 
     CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
                                  PyObject* py_ref, PyObject** out)
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
index 42588d49b694b..390954cfc6bd9 100644
--- a/python/pyarrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -16,7 +16,8 @@
 # under the License.
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport CDataType, CField, CSchema
+from pyarrow.includes.libarrow cimport (CDataType, CDictionaryType,
+                                        CField, CSchema)
 
 cdef class DataType:
     cdef:
@@ -25,6 +26,11 @@ cdef class DataType:
 
     cdef init(self, const shared_ptr[CDataType]& type)
 
+
+cdef class DictionaryType(DataType):
+    cdef:
+        const CDictionaryType* dict_type
+
 cdef class Field:
     cdef:
         shared_ptr[CField] sp_field
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 85b16170b948e..2bcfec1bcf3e2 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -25,6 +25,7 @@
 from cython.operator cimport dereference as deref
 
 from pyarrow.compat import frombytes, tobytes
+from pyarrow.array cimport Array
 from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
                                         Type_NA, Type_BOOL,
                                         Type_UINT8, Type_INT8,
@@ -66,6 +67,19 @@ cdef class DataType:
             raise TypeError('Invalid comparison')
 
 
+cdef class DictionaryType(DataType):
+
+    cdef init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.dict_type = <const CDictionaryType*> type.get()
+
+    def __str__(self):
+        return frombytes(self.type.ToString())
+
+    def __repr__(self):
+        return 'DictionaryType({0})'.format(str(self))
+
+
 cdef class Field:
 
     def __cinit__(self):
@@ -269,6 +283,7 @@ def binary():
     """
     return primitive_type(Type_BINARY)
 
+
 def list_(DataType value_type):
     cdef DataType out = DataType()
     cdef shared_ptr[CDataType] list_type
@@ -276,6 +291,19 @@ def list_(DataType value_type):
     out.init(list_type)
     return out
 
+
+def dictionary(DataType index_type, Array dictionary):
+    """
+    Dictionary (categorical, or simply encoded) type
+    """
+    cdef DictionaryType out = DictionaryType()
+    cdef shared_ptr[CDataType] dict_type
+    dict_type.reset(new CDictionaryType(index_type.sp_type,
+                                        dictionary.sp_array))
+    out.init(dict_type)
+    return out
+
+
 def struct(fields):
     """
 
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index b720a4723ec56..0e3b2bd90dc64 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -27,7 +27,7 @@ cimport pyarrow.includes.pyarrow as pyarrow
 
 import pyarrow.config
 
-from pyarrow.array cimport Array, box_arrow_array
+from pyarrow.array cimport Array, box_arrow_array, wrap_array_output
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
 from pyarrow.schema cimport box_data_type, box_schema, Field
@@ -39,6 +39,11 @@ cimport cpython
 from collections import OrderedDict
 
 
+cdef _pandas():
+    import pandas as pd
+    return pd
+
+
 cdef class ChunkedArray:
     """
     Array backed via one or more memory chunks.
@@ -146,14 +151,12 @@ cdef class Column:
         pandas.Series
         """
         cdef:
-            PyObject* arr
-
-        import pandas as pd
+            PyObject* out
 
         check_status(pyarrow.ConvertColumnToPandas(self.sp_column,
-                                                   <PyObject*> self, &arr))
+                                                   <PyObject*> self, &out))
 
-        return pd.Series(PyObject_to_object(arr), name=self.name)
+        return _pandas().Series(wrap_array_output(out), name=self.name)
 
     def equals(self, Column other):
         """
@@ -278,8 +281,6 @@ cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
 
 
 cdef _dataframe_to_arrays(df, name, timestamps_to_ms, Schema schema):
-    from pyarrow.array import from_pandas_series
-
     cdef:
         list names = []
         list arrays = []
@@ -289,9 +290,8 @@ cdef _dataframe_to_arrays(df, name, timestamps_to_ms, Schema schema):
         col = df[name]
         if schema is not None:
             field = schema.field_by_name(name)
-        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms,
-                                 field=field)
-
+        arr = Array.from_pandas(col, timestamps_to_ms=timestamps_to_ms,
+                                field=field)
         names.append(name)
         arrays.append(arr)
 
@@ -304,7 +304,8 @@ cdef class RecordBatch:
 
     Warning
     -------
-    Do not call this class's constructor directly, use one of the ``from_*`` methods instead.
+    Do not call this class's constructor directly, use one of the ``from_*``
+    methods instead.
     """
 
     def __cinit__(self):
@@ -401,7 +402,7 @@ cdef class RecordBatch:
         return OrderedDict(entries)
 
 
-    def to_pandas(self):
+    def to_pandas(self, nthreads=None):
         """
         Convert the arrow::RecordBatch to a pandas DataFrame
 
@@ -409,23 +410,7 @@ cdef class RecordBatch:
         -------
         pandas.DataFrame
         """
-        cdef:
-            PyObject* np_arr
-            shared_ptr[CArray] arr
-            Column column
-
-        import pandas as pd
-
-        names = []
-        data = []
-        for i in range(self.batch.num_columns()):
-            arr = self.batch.column(i)
-            check_status(pyarrow.ConvertArrayToPandas(arr, <PyObject*> self,
-                                                      &np_arr))
-            names.append(frombytes(self.batch.column_name(i)))
-            data.append(PyObject_to_object(np_arr))
-
-        return pd.DataFrame(dict(zip(names, data)), columns=names)
+        return Table.from_batches([self]).to_pandas(nthreads=nthreads)
 
     @classmethod
     def from_pandas(cls, df, schema=None):
@@ -490,8 +475,8 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
         CColumn* col
         int i
 
-    from pandas.core.internals import BlockManager, make_block
-    from pandas import RangeIndex
+    import pandas.core.internals as _int
+    from pandas import RangeIndex, Categorical
 
     with nogil:
         check_status(pyarrow.ConvertTableToPandas(table, nthreads,
@@ -500,8 +485,19 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
     result = PyObject_to_object(result_obj)
 
     blocks = []
-    for block_arr, placement_arr in result:
-        blocks.append(make_block(block_arr, placement=placement_arr))
+    for item in result:
+        block_arr = item['block']
+        placement = item['placement']
+        if 'dictionary' in item:
+            cat = Categorical(block_arr,
+                              categories=item['dictionary'],
+                              ordered=False, fastpath=True)
+            block = _int.make_block(cat, placement=placement,
+                                    klass=_int.CategoricalBlock,
+                                    fastpath=True)
+        else:
+            block = _int.make_block(block_arr, placement=placement)
+        blocks.append(block)
 
     names = []
     for i in range(table.get().num_columns()):
@@ -509,7 +505,7 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
         names.append(frombytes(col.name()))
 
     axes = [names, RangeIndex(table.get().num_rows())]
-    return BlockManager(blocks, axes)
+    return _int.BlockManager(blocks, axes)
 
 
 cdef class Table:
@@ -518,7 +514,8 @@ cdef class Table:
 
     Warning
     -------
-    Do not call this class's constructor directly, use one of the ``from_*`` methods instead.
+    Do not call this class's constructor directly, use one of the ``from_*``
+    methods instead.
     """
 
     def __cinit__(self):
@@ -688,13 +685,11 @@ cdef class Table:
         -------
         pandas.DataFrame
         """
-        import pandas as pd
-
         if nthreads is None:
             nthreads = pyarrow.config.cpu_count()
 
         mgr = table_to_blockmanager(self.sp_table, nthreads)
-        return pd.DataFrame(mgr)
+        return _pandas().DataFrame(mgr)
 
     def to_pydict(self):
         """
@@ -835,6 +830,7 @@ cdef api object table_from_ctable(const shared_ptr[CTable]& ctable):
     table.init(ctable)
     return table
 
+
 cdef api object batch_from_cbatch(const shared_ptr[CRecordBatch]& cbatch):
     cdef RecordBatch batch = RecordBatch()
     batch.init(cbatch)
diff --git a/python/pyarrow/tests/test_column.py b/python/pyarrow/tests/test_column.py
index 32202cb5a9ad8..1a507c81030f8 100644
--- a/python/pyarrow/tests/test_column.py
+++ b/python/pyarrow/tests/test_column.py
@@ -47,4 +47,3 @@ def test_pandas(self):
         assert series.name == 'a'
         assert series.shape == (5,)
         assert series.iloc[0] == -10
-
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 3928a1fc1b92b..a2f50620d8925 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -62,8 +62,10 @@ def tearDown(self):
         pass
 
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
-                                timestamps_to_ms=False, expected_schema=None, schema=None):
-        table = A.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms, schema=schema)
+                                timestamps_to_ms=False, expected_schema=None,
+                                schema=None):
+        table = A.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms,
+                                    schema=schema)
         result = table.to_pandas(nthreads=nthreads)
         if expected_schema:
             assert table.schema.equals(expected_schema)
@@ -71,6 +73,13 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
             expected = df
         tm.assert_frame_equal(result, expected)
 
+    def _check_array_roundtrip(self, values, expected=None,
+                                timestamps_to_ms=False, field=None):
+        arr = A.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
+                                  field=field)
+        result = arr.to_pandas()
+        tm.assert_series_equal(pd.Series(result), pd.Series(values))
+
     def test_float_no_nulls(self):
         data = {}
         fields = []
@@ -235,7 +244,8 @@ def test_timestamps_notimezone_no_nulls(self):
             })
         field = A.Field.from_py('datetime64', A.timestamp('ms'))
         schema = A.Schema.from_fields([field])
-        self._check_pandas_roundtrip(df, timestamps_to_ms=True, expected_schema=schema)
+        self._check_pandas_roundtrip(df, timestamps_to_ms=True,
+                                     expected_schema=schema)
 
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -246,7 +256,8 @@ def test_timestamps_notimezone_no_nulls(self):
             })
         field = A.Field.from_py('datetime64', A.timestamp('ns'))
         schema = A.Schema.from_fields([field])
-        self._check_pandas_roundtrip(df, timestamps_to_ms=False, expected_schema=schema)
+        self._check_pandas_roundtrip(df, timestamps_to_ms=False,
+                                     expected_schema=schema)
 
     def test_timestamps_notimezone_nulls(self):
         df = pd.DataFrame({
@@ -258,7 +269,8 @@ def test_timestamps_notimezone_nulls(self):
             })
         field = A.Field.from_py('datetime64', A.timestamp('ms'))
         schema = A.Schema.from_fields([field])
-        self._check_pandas_roundtrip(df, timestamps_to_ms=True, expected_schema=schema)
+        self._check_pandas_roundtrip(df, timestamps_to_ms=True,
+                                     expected_schema=schema)
 
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -269,7 +281,8 @@ def test_timestamps_notimezone_nulls(self):
             })
         field = A.Field.from_py('datetime64', A.timestamp('ns'))
         schema = A.Schema.from_fields([field])
-        self._check_pandas_roundtrip(df, timestamps_to_ms=False, expected_schema=schema)
+        self._check_pandas_roundtrip(df, timestamps_to_ms=False,
+                                     expected_schema=schema)
 
     def test_date(self):
         df = pd.DataFrame({
@@ -317,13 +330,13 @@ def test_column_of_lists(self):
             np.array(['2007-07-13T01:23:34.123456789',
                       None,
                       '2010-08-13T05:46:57.437699912'],
-                      dtype='datetime64[ns]'),
+                     dtype='datetime64[ns]'),
             None,
             None,
             np.array(['2007-07-13T02',
                       None,
                       '2010-08-13T05:46:57.437699912'],
-                      dtype='datetime64[ns]'),
+                     dtype='datetime64[ns]'),
         ]
 
         df = pd.DataFrame(arrays)
@@ -331,16 +344,34 @@ def test_column_of_lists(self):
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
         table = A.Table.from_pandas(df, schema=schema)
         assert table.schema.equals(schema)
-        df_new = table.to_pandas(nthreads=1)
+
+        # it works!
+        table.to_pandas(nthreads=1)
 
     def test_threaded_conversion(self):
         df = _alltypes_example()
         self._check_pandas_roundtrip(df, nthreads=2,
                                      timestamps_to_ms=False)
 
-    # def test_category(self):
-    #     repeats = 1000
-    #     values = [b'foo', None, u'bar', 'qux', np.nan]
-    #     df = pd.DataFrame({'strings': values * repeats})
-    #     df['strings'] = df['strings'].astype('category')
-    #     self._check_pandas_roundtrip(df)
+    def test_category(self):
+        repeats = 5
+        v1 = ['foo', None, 'bar', 'qux', np.nan]
+        v2 = [4, 5, 6, 7, 8]
+        v3 = [b'foo', None, b'bar', b'qux', np.nan]
+        df = pd.DataFrame({'cat_strings': pd.Categorical(v1 * repeats),
+                           'cat_ints': pd.Categorical(v2 * repeats),
+                           'cat_binary': pd.Categorical(v3 * repeats),
+                           'ints': v2 * repeats,
+                           'ints2': v2 * repeats,
+                           'strings': v1 * repeats,
+                           'strings2': v1 * repeats,
+                           'strings3': v3 * repeats})
+        self._check_pandas_roundtrip(df)
+
+        arrays = [
+            pd.Categorical(v1 * repeats),
+            pd.Categorical(v2 * repeats),
+            pd.Categorical(v3 * repeats)
+        ]
+        for values in arrays:
+            self._check_array_roundtrip(values)
diff --git a/python/setup.py b/python/setup.py
index 72ff5842a22a5..de59a92905895 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -128,6 +128,7 @@ def _run_cmake(self):
 
         cmake_options = [
             '-DPYTHON_EXECUTABLE=%s' % sys.executable,
+            '-DPYARROW_BUILD_TESTS=off',
             static_lib_option,
             build_tests_option,
         ]
diff --git a/python/src/pyarrow/CMakeLists.txt b/python/src/pyarrow/CMakeLists.txt
index e20c3238b5f78..9e69718dfa7c7 100644
--- a/python/src/pyarrow/CMakeLists.txt
+++ b/python/src/pyarrow/CMakeLists.txt
@@ -18,3 +18,5 @@
 #######################################
 # Unit tests
 #######################################
+
+ADD_PYARROW_TEST(adapters/pandas-test)
diff --git a/python/src/pyarrow/adapters/pandas-test.cc b/python/src/pyarrow/adapters/pandas-test.cc
new file mode 100644
index 0000000000000..e286ccc2c8dc4
--- /dev/null
+++ b/python/src/pyarrow/adapters/pandas-test.cc
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "pyarrow/adapters/pandas.h"
+
+using namespace arrow;
+
+namespace pyarrow {
+
+TEST(PandasConversionTest, TestObjectBlockWriteFails) {
+  StringBuilder builder;
+  const char value[] = {'\xf1', '\0'};
+
+  for (int i = 0; i < 1000; ++i) {
+    builder.Append(value, strlen(value));
+  }
+
+  std::shared_ptr<Array> arr;
+  ASSERT_OK(builder.Finish(&arr));
+
+  auto f1 = field("f1", utf8());
+  auto f2 = field("f2", utf8());
+  auto f3 = field("f3", utf8());
+  std::vector<std::shared_ptr<Field>> fields = {f1, f2, f3};
+  std::vector<std::shared_ptr<Column>> cols = {std::make_shared<Column>(f1, arr),
+      std::make_shared<Column>(f2, arr), std::make_shared<Column>(f3, arr)};
+
+  auto schema = std::make_shared<Schema>(fields);
+  auto table = std::make_shared<Table>("", schema, cols);
+
+  PyObject* out;
+  Py_BEGIN_ALLOW_THREADS;
+  ASSERT_RAISES(UnknownError, ConvertTableToPandas(table, 2, &out));
+  Py_END_ALLOW_THREADS;
+}
+
+}  // namespace arrow
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 8c2d3506c8a9d..6623e239880bc 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -49,6 +49,7 @@ namespace pyarrow {
 using arrow::Array;
 using arrow::ChunkedArray;
 using arrow::Column;
+using arrow::DictionaryType;
 using arrow::Field;
 using arrow::DataType;
 using arrow::ListType;
@@ -60,7 +61,7 @@ using arrow::Type;
 namespace BitUtil = arrow::BitUtil;
 
 // ----------------------------------------------------------------------
-// Serialization
+// Utility code
 
 template <int TYPE>
 struct npy_traits {};
@@ -242,1577 +243,1730 @@ Status AppendObjectStrings(arrow::StringBuilder& string_builder, PyObject** obje
 }
 
 template <int TYPE>
-class ArrowSerializer {
- public:
-  ArrowSerializer(arrow::MemoryPool* pool, PyArrayObject* arr, PyArrayObject* mask)
-      : pool_(pool), arr_(arr), mask_(mask) {
-    length_ = PyArray_SIZE(arr_);
-  }
+struct arrow_traits {};
 
-  void IndicateType(const std::shared_ptr<Field> field) { field_indicator_ = field; }
+template <>
+struct arrow_traits<Type::BOOL> {
+  static constexpr int npy_type = NPY_BOOL;
+  static constexpr bool supports_nulls = false;
+  static constexpr bool is_boolean = true;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
 
-  Status Convert(std::shared_ptr<Array>* out);
+#define INT_DECL(TYPE)                                      \
+  template <>                                               \
+  struct arrow_traits<Type::TYPE> {                         \
+    static constexpr int npy_type = NPY_##TYPE;             \
+    static constexpr bool supports_nulls = false;           \
+    static constexpr double na_value = NAN;                 \
+    static constexpr bool is_boolean = false;               \
+    static constexpr bool is_numeric_not_nullable = true;   \
+    static constexpr bool is_numeric_nullable = false;      \
+    typedef typename npy_traits<NPY_##TYPE>::value_type T;  \
+  };
 
-  int stride() const { return PyArray_STRIDES(arr_)[0]; }
+INT_DECL(INT8);
+INT_DECL(INT16);
+INT_DECL(INT32);
+INT_DECL(INT64);
+INT_DECL(UINT8);
+INT_DECL(UINT16);
+INT_DECL(UINT32);
+INT_DECL(UINT64);
 
-  Status InitNullBitmap() {
-    int null_bytes = BitUtil::BytesForBits(length_);
+template <>
+struct arrow_traits<Type::FLOAT> {
+  static constexpr int npy_type = NPY_FLOAT32;
+  static constexpr bool supports_nulls = true;
+  static constexpr float na_value = NAN;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_FLOAT32>::value_type T;
+};
 
-    null_bitmap_ = std::make_shared<arrow::PoolBuffer>(pool_);
-    RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
+template <>
+struct arrow_traits<Type::DOUBLE> {
+  static constexpr int npy_type = NPY_FLOAT64;
+  static constexpr bool supports_nulls = true;
+  static constexpr double na_value = NAN;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_FLOAT64>::value_type T;
+};
 
-    null_bitmap_data_ = null_bitmap_->mutable_data();
-    memset(null_bitmap_data_, 0, null_bytes);
+static constexpr int64_t kPandasTimestampNull = std::numeric_limits<int64_t>::min();
 
-    return Status::OK();
-  }
+template <>
+struct arrow_traits<Type::TIMESTAMP> {
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr bool supports_nulls = true;
+  static constexpr int64_t na_value = kPandasTimestampNull;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
 
-  bool is_strided() const {
-    npy_intp* astrides = PyArray_STRIDES(arr_);
-    return astrides[0] != PyArray_DESCR(arr_)->elsize;
-  }
+template <>
+struct arrow_traits<Type::DATE> {
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr bool supports_nulls = true;
+  static constexpr int64_t na_value = kPandasTimestampNull;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
 
- private:
-  Status ConvertData();
+template <>
+struct arrow_traits<Type::STRING> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
 
-  Status ConvertDates(std::shared_ptr<Array>* out) {
-    PyAcquireGIL lock;
+template <>
+struct arrow_traits<Type::BINARY> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
 
-    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    arrow::TypePtr string_type(new arrow::DateType());
-    arrow::DateBuilder date_builder(pool_, string_type);
-    RETURN_NOT_OK(date_builder.Resize(length_));
+template <typename T>
+struct WrapBytes {};
 
-    Status s;
-    PyObject* obj;
-    for (int64_t i = 0; i < length_; ++i) {
-      obj = objects[i];
-      if (PyDate_CheckExact(obj)) {
-        PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(obj);
-        date_builder.Append(PyDate_to_ms(pydate));
-      } else {
-        date_builder.AppendNull();
-      }
-    }
-    return date_builder.Finish(out);
+template <>
+struct WrapBytes<arrow::StringArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
   }
+};
 
-  Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
-    PyAcquireGIL lock;
+template <>
+struct WrapBytes<arrow::BinaryArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+  }
+};
 
-    // The output type at this point is inconclusive because there may be bytes
-    // and unicode mixed in the object array
+inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
+  if (type == NPY_DATETIME) {
+    PyArray_Descr* descr = PyArray_DESCR(out);
+    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+    if (datatype->type == Type::TIMESTAMP) {
+      auto timestamp_type = static_cast<arrow::TimestampType*>(datatype);
 
-    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    arrow::TypePtr string_type(new arrow::StringType());
-    arrow::StringBuilder string_builder(pool_, string_type);
-    RETURN_NOT_OK(string_builder.Resize(length_));
+      switch (timestamp_type->unit) {
+        case arrow::TimestampType::Unit::SECOND:
+          date_dtype->meta.base = NPY_FR_s;
+          break;
+        case arrow::TimestampType::Unit::MILLI:
+          date_dtype->meta.base = NPY_FR_ms;
+          break;
+        case arrow::TimestampType::Unit::MICRO:
+          date_dtype->meta.base = NPY_FR_us;
+          break;
+        case arrow::TimestampType::Unit::NANO:
+          date_dtype->meta.base = NPY_FR_ns;
+          break;
+      }
+    } else {
+      // datatype->type == Type::DATE
+      date_dtype->meta.base = NPY_FR_D;
+    }
+  }
+}
 
-    Status s;
-    bool have_bytes = false;
-    RETURN_NOT_OK(AppendObjectStrings(string_builder, objects, length_, &have_bytes));
-    RETURN_NOT_OK(string_builder.Finish(out));
+template <typename T>
+inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    // Upcast to double, set NaN as appropriate
 
-    if (have_bytes) {
-      const auto& arr = static_cast<const arrow::StringArray&>(*out->get());
-      *out = std::make_shared<arrow::BinaryArray>(
-          arr.length(), arr.offsets(), arr.data(), arr.null_count(), arr.null_bitmap());
+    for (int i = 0; i < arr->length(); ++i) {
+      *out_values++ = prim_arr->IsNull(i) ? NAN : in_values[i];
     }
-    return Status::OK();
   }
+}
 
-  Status ConvertBooleans(std::shared_ptr<Array>* out) {
-    PyAcquireGIL lock;
+template <typename T>
+inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    memcpy(out_values, in_values, sizeof(T) * arr->length());
+    out_values += arr->length();
+  }
+}
 
-    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+template <typename InType, typename OutType>
+inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+    for (int32_t i = 0; i < arr->length(); ++i) {
+      *out_values = in_values[i];
+    }
+  }
+}
 
-    int nbytes = BitUtil::BytesForBits(length_);
-    auto data = std::make_shared<arrow::PoolBuffer>(pool_);
-    RETURN_NOT_OK(data->Resize(nbytes));
-    uint8_t* bitmap = data->mutable_data();
-    memset(bitmap, 0, nbytes);
+static Status ConvertBooleanWithNulls(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
 
-    int64_t null_count = 0;
-    for (int64_t i = 0; i < length_; ++i) {
-      if (objects[i] == Py_True) {
-        BitUtil::SetBit(bitmap, i);
-        BitUtil::SetBit(null_bitmap_data_, i);
-      } else if (objects[i] != Py_False) {
-        ++null_count;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (bool_arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values++ = Py_None;
+      } else if (bool_arr->Value(i)) {
+        // True
+        Py_INCREF(Py_True);
+        *out_values++ = Py_True;
       } else {
-        BitUtil::SetBit(null_bitmap_data_, i);
+        // False
+        Py_INCREF(Py_False);
+        *out_values++ = Py_False;
       }
     }
-
-    *out = std::make_shared<arrow::BooleanArray>(length_, data, null_count, null_bitmap_);
-
-    return Status::OK();
   }
+  return Status::OK();
+}
 
-  template <int ITEM_TYPE, typename ArrowType>
-  Status ConvertTypedLists(
-      const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out);
-
-#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                            \
-  case Type::TYPE: {                                                      \
-    return ConvertTypedLists<NUMPY_TYPE, ::arrow::ArrowType>(field, out); \
+static void ConvertBooleanNoNulls(const ChunkedArray& data, uint8_t* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = static_cast<uint8_t>(bool_arr->Value(i));
+    }
   }
+}
 
-  Status ConvertLists(const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-    switch (field->type->type) {
-      LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
-      LIST_CASE(INT8, NPY_INT8, Int8Type)
-      LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
-      LIST_CASE(INT16, NPY_INT16, Int16Type)
-      LIST_CASE(UINT32, NPY_UINT32, UInt32Type)
-      LIST_CASE(INT32, NPY_INT32, Int32Type)
-      LIST_CASE(UINT64, NPY_UINT64, UInt64Type)
-      LIST_CASE(INT64, NPY_INT64, Int64Type)
-      LIST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
-      LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
-      LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
-      LIST_CASE(STRING, NPY_OBJECT, StringType)
-      default:
-        return Status::TypeError("Unknown list item type");
-    }
-
-    return Status::TypeError("Unknown list type");
-  }
-
-  Status MakeDataType(std::shared_ptr<DataType>* out);
-
-  arrow::MemoryPool* pool_;
-
-  PyArrayObject* arr_;
-  PyArrayObject* mask_;
-
-  int64_t length_;
-
-  std::shared_ptr<Field> field_indicator_;
-  std::shared_ptr<arrow::Buffer> data_;
-  std::shared_ptr<arrow::ResizableBuffer> null_bitmap_;
-  uint8_t* null_bitmap_data_;
-};
+template <typename ArrayType>
+inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = static_cast<ArrayType*>(data.chunk(c).get());
 
-// Returns null count
-static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
-  int64_t null_count = 0;
-  const uint8_t* mask_values = static_cast<const uint8_t*>(PyArray_DATA(mask));
-  // TODO(wesm): strided null mask
-  for (int i = 0; i < length; ++i) {
-    if (mask_values[i]) {
-      ++null_count;
-    } else {
-      BitUtil::SetBit(bitmap, i);
+    const uint8_t* data_ptr;
+    int32_t length;
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (has_nulls && arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        data_ptr = arr->GetValue(i, &length);
+        *out_values = WrapBytes<ArrayType>::Wrap(data_ptr, length);
+        if (*out_values == nullptr) {
+          PyErr_Clear();
+          std::stringstream ss;
+          ss << "Wrapping "
+             << std::string(reinterpret_cast<const char*>(data_ptr), length) << " failed";
+          return Status::UnknownError(ss.str());
+        }
+      }
+      ++out_values;
     }
   }
-  return null_count;
-}
-
-template <int TYPE>
-inline Status ArrowSerializer<TYPE>::MakeDataType(std::shared_ptr<DataType>* out) {
-  out->reset(new typename npy_traits<TYPE>::TypeClass());
   return Status::OK();
 }
 
-template <>
-inline Status ArrowSerializer<NPY_DATETIME>::MakeDataType(
-    std::shared_ptr<DataType>* out) {
-  PyArray_Descr* descr = PyArray_DESCR(arr_);
-  auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-  arrow::TimestampType::Unit unit;
+template <typename ArrowType>
+inline Status ConvertListsLike(
+    const std::shared_ptr<Column>& col, PyObject** out_values) {
+  const ChunkedArray& data = *col->data().get();
+  auto list_type = std::static_pointer_cast<ListType>(col->type());
 
-  switch (date_dtype->meta.base) {
-    case NPY_FR_s:
-      unit = arrow::TimestampType::Unit::SECOND;
-      break;
-    case NPY_FR_ms:
-      unit = arrow::TimestampType::Unit::MILLI;
-      break;
-    case NPY_FR_us:
-      unit = arrow::TimestampType::Unit::MICRO;
-      break;
-    case NPY_FR_ns:
-      unit = arrow::TimestampType::Unit::NANO;
-      break;
-    default:
-      return Status::Invalid("Unknown NumPy datetime unit");
+  // Get column of underlying value arrays
+  std::vector<std::shared_ptr<Array>> value_arrays;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
+    value_arrays.emplace_back(arr->values());
   }
+  auto flat_column = std::make_shared<Column>(list_type->value_field(), value_arrays);
+  // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
+  //    Storing a reference to the whole Array would be to expensive.
+  PyObject* numpy_array;
+  RETURN_NOT_OK(ConvertColumnToPandas(flat_column, nullptr, &numpy_array));
 
-  out->reset(new arrow::TimestampType(unit));
-  return Status::OK();
-}
-
-template <int TYPE>
-inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
-  typedef npy_traits<TYPE> traits;
+  PyAcquireGIL lock;
 
-  if (mask_ != nullptr || traits::supports_nulls) { RETURN_NOT_OK(InitNullBitmap()); }
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
 
-  int64_t null_count = 0;
-  if (mask_ != nullptr) {
-    null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
-  } else if (traits::supports_nulls) {
-    null_count = ValuesToBitmap<TYPE>(PyArray_DATA(arr_), length_, null_bitmap_data_);
+    const uint8_t* data_ptr;
+    int32_t length;
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (has_nulls && arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        PyObject* start = PyLong_FromLong(arr->value_offset(i));
+        PyObject* end = PyLong_FromLong(arr->value_offset(i + 1));
+        PyObject* slice = PySlice_New(start, end, NULL);
+        *out_values = PyObject_GetItem(numpy_array, slice);
+        Py_DECREF(start);
+        Py_DECREF(end);
+        Py_DECREF(slice);
+      }
+      ++out_values;
+    }
   }
 
-  RETURN_NOT_OK(ConvertData());
-  std::shared_ptr<DataType> type;
-  RETURN_NOT_OK(MakeDataType(&type));
-  RETURN_NOT_OK(MakePrimitiveArray(type, length_, data_, null_count, null_bitmap_, out));
+  Py_XDECREF(numpy_array);
   return Status::OK();
 }
 
-template <>
-inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out) {
-  // Python object arrays are annoying, since we could have one of:
-  //
-  // * Strings
-  // * Booleans with nulls
-  // * Mixed type (not supported at the moment by arrow format)
-  //
-  // Additionally, nulls may be encoded either as np.nan or None. So we have to
-  // do some type inference and conversion
-
-  RETURN_NOT_OK(InitNullBitmap());
+template <typename T>
+inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
 
-  // TODO: mask not supported here
-  const PyObject** objects = reinterpret_cast<const PyObject**>(PyArray_DATA(arr_));
-  {
-    PyAcquireGIL lock;
-    PyDateTime_IMPORT;
-  }
+    const uint8_t* valid_bits = arr->null_bitmap_data();
 
-  if (field_indicator_) {
-    switch (field_indicator_->type->type) {
-      case Type::STRING:
-        return ConvertObjectStrings(out);
-      case Type::BOOL:
-        return ConvertBooleans(out);
-      case Type::DATE:
-        return ConvertDates(out);
-      case Type::LIST: {
-        auto list_field = static_cast<ListType*>(field_indicator_->type.get());
-        return ConvertLists(list_field->value_field(), out);
-      }
-      default:
-        return Status::TypeError("No known conversion to Arrow type");
-    }
-  } else {
-    for (int64_t i = 0; i < length_; ++i) {
-      if (PyObject_is_null(objects[i])) {
-        continue;
-      } else if (PyObject_is_string(objects[i])) {
-        return ConvertObjectStrings(out);
-      } else if (PyBool_Check(objects[i])) {
-        return ConvertBooleans(out);
-      } else if (PyDate_CheckExact(objects[i])) {
-        return ConvertDates(out);
-      } else {
-        return Status::TypeError("unhandled python type");
+    if (arr->null_count() > 0) {
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i];
       }
+    } else {
+      memcpy(out_values, in_values, sizeof(T) * arr->length());
+      out_values += arr->length();
     }
   }
-
-  return Status::TypeError("Unable to infer type of object array, were all null");
 }
 
-template <int TYPE>
-inline Status ArrowSerializer<TYPE>::ConvertData() {
-  // TODO(wesm): strided arrays
-  if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
+template <typename InType, typename OutType>
+inline void ConvertNumericNullableCast(
+    const ChunkedArray& data, OutType na_value, OutType* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
 
-  data_ = std::make_shared<NumPyBuffer>(arr_);
-  return Status::OK();
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = arr->IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
+    }
+  }
 }
 
-template <>
-inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
-  if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
-
-  int nbytes = BitUtil::BytesForBits(length_);
-  auto buffer = std::make_shared<arrow::PoolBuffer>(pool_);
-  RETURN_NOT_OK(buffer->Resize(nbytes));
-
-  const uint8_t* values = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
-
-  uint8_t* bitmap = buffer->mutable_data();
+template <typename T>
+inline void ConvertDates(const ChunkedArray& data, T na_value, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
 
-  memset(bitmap, 0, nbytes);
-  for (int i = 0; i < length_; ++i) {
-    if (values[i] > 0) { BitUtil::SetBit(bitmap, i); }
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
+      *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / 86400000;
+    }
   }
+}
 
-  data_ = buffer;
+template <typename InType, int SHIFT>
+inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
 
-  return Status::OK();
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = arr->IsNull(i) ? kPandasTimestampNull
+                                     : (static_cast<int64_t>(in_values[i]) * SHIFT);
+    }
+  }
 }
 
-template <int TYPE>
-template <int ITEM_TYPE, typename ArrowType>
-inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
-    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-  typedef npy_traits<ITEM_TYPE> traits;
-  typedef typename traits::value_type T;
-  typedef typename traits::BuilderClass BuilderT;
+// ----------------------------------------------------------------------
+// pandas 0.x DataFrame conversion internals
 
-  auto value_builder = std::make_shared<BuilderT>(pool_, field->type);
-  ListBuilder list_builder(pool_, value_builder);
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  for (int64_t i = 0; i < length_; ++i) {
-    if (PyObject_is_null(objects[i])) {
-      RETURN_NOT_OK(list_builder.AppendNull());
-    } else if (PyArray_Check(objects[i])) {
-      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
-      RETURN_NOT_OK(list_builder.Append(true));
+class PandasBlock {
+ public:
+  enum type {
+    OBJECT,
+    UINT8,
+    INT8,
+    UINT16,
+    INT16,
+    UINT32,
+    INT32,
+    UINT64,
+    INT64,
+    FLOAT,
+    DOUBLE,
+    BOOL,
+    DATETIME,
+    CATEGORICAL
+  };
 
-      // TODO(uwe): Support more complex numpy array structures
-      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
+  PandasBlock(int64_t num_rows, int num_columns)
+      : num_rows_(num_rows), num_columns_(num_columns) {}
+  virtual ~PandasBlock() {}
 
-      int32_t size = PyArray_DIM(numpy_array, 0);
-      auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
-      if (traits::supports_nulls) {
-        null_bitmap_->Resize(size, false);
-        // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
-        // currently support this.
-        // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        ValuesToBytemap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        RETURN_NOT_OK(value_builder->Append(data, size, null_bitmap_->data()));
-      } else {
-        RETURN_NOT_OK(value_builder->Append(data, size));
-      }
-    } else if (PyList_Check(objects[i])) {
-      return Status::TypeError("Python lists are not yet supported");
-    } else {
-      return Status::TypeError("Unsupported Python type for list items");
-    }
-  }
-  return list_builder.Finish(out);
-}
+  virtual Status Allocate() = 0;
+  virtual Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) = 0;
 
-template <>
-template <>
-inline Status
-ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
-    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-  // TODO: If there are bytes involed, convert to Binary representation
-  bool have_bytes = false;
+  PyObject* block_arr() const { return block_arr_.obj(); }
 
-  auto value_builder = std::make_shared<arrow::StringBuilder>(pool_, field->type);
-  ListBuilder list_builder(pool_, value_builder);
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  for (int64_t i = 0; i < length_; ++i) {
-    if (PyObject_is_null(objects[i])) {
-      RETURN_NOT_OK(list_builder.AppendNull());
-    } else if (PyArray_Check(objects[i])) {
-      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
-      RETURN_NOT_OK(list_builder.Append(true));
+  virtual Status GetPyResult(PyObject** output) {
+    PyObject* result = PyDict_New();
+    RETURN_IF_PYERROR();
 
-      // TODO(uwe): Support more complex numpy array structures
-      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+    PyDict_SetItemString(result, "block", block_arr_.obj());
+    PyDict_SetItemString(result, "placement", placement_arr_.obj());
 
-      int32_t size = PyArray_DIM(numpy_array, 0);
-      auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
-      RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
-    } else if (PyList_Check(objects[i])) {
-      return Status::TypeError("Python lists are not yet supported");
+    *output = result;
+
+    return Status::OK();
+  }
+
+ protected:
+  Status AllocateNDArray(int npy_type, int ndim = 2) {
+    PyAcquireGIL lock;
+
+    PyObject* block_arr;
+    if (ndim == 2) {
+      npy_intp block_dims[2] = {num_columns_, num_rows_};
+      block_arr = PyArray_SimpleNew(2, block_dims, npy_type);
     } else {
-      return Status::TypeError("Unsupported Python type for list items");
+      npy_intp block_dims[1] = {num_rows_};
+      block_arr = PyArray_SimpleNew(1, block_dims, npy_type);
     }
-  }
-  return list_builder.Finish(out);
-}
 
-template <>
-inline Status ArrowSerializer<NPY_OBJECT>::ConvertData() {
-  return Status::TypeError("NYI");
-}
+    if (block_arr == NULL) {
+      // TODO(wesm): propagating Python exception
+      return Status::OK();
+    }
 
-#define TO_ARROW_CASE(TYPE)                                 \
-  case NPY_##TYPE: {                                        \
-    ArrowSerializer<NPY_##TYPE> converter(pool, arr, mask); \
-    RETURN_NOT_OK(converter.Convert(out));                  \
-  } break;
+    npy_intp placement_dims[1] = {num_columns_};
+    PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
+    if (placement_arr == NULL) {
+      // TODO(wesm): propagating Python exception
+      return Status::OK();
+    }
 
-Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-  PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(ao);
-  PyArrayObject* mask = nullptr;
+    block_arr_.reset(block_arr);
+    placement_arr_.reset(placement_arr);
 
-  if (mo != nullptr) { mask = reinterpret_cast<PyArrayObject*>(mo); }
+    block_data_ = reinterpret_cast<uint8_t*>(
+        PyArray_DATA(reinterpret_cast<PyArrayObject*>(block_arr)));
 
-  if (PyArray_NDIM(arr) != 1) {
-    return Status::Invalid("only handle 1-dimensional arrays");
+    placement_data_ = reinterpret_cast<int64_t*>(
+        PyArray_DATA(reinterpret_cast<PyArrayObject*>(placement_arr)));
+
+    return Status::OK();
   }
 
-  switch (PyArray_DESCR(arr)->type_num) {
-    TO_ARROW_CASE(BOOL);
-    TO_ARROW_CASE(INT8);
-    TO_ARROW_CASE(INT16);
-    TO_ARROW_CASE(INT32);
-    TO_ARROW_CASE(INT64);
-    TO_ARROW_CASE(UINT8);
-    TO_ARROW_CASE(UINT16);
-    TO_ARROW_CASE(UINT32);
-    TO_ARROW_CASE(UINT64);
-    TO_ARROW_CASE(FLOAT32);
-    TO_ARROW_CASE(FLOAT64);
-    TO_ARROW_CASE(DATETIME);
-    case NPY_OBJECT: {
-      ArrowSerializer<NPY_OBJECT> converter(pool, arr, mask);
-      converter.IndicateType(field);
-      RETURN_NOT_OK(converter.Convert(out));
-    } break;
-    default:
+  int64_t num_rows_;
+  int num_columns_;
+
+  OwnedRef block_arr_;
+  uint8_t* block_data_;
+
+  // ndarray<int32>
+  OwnedRef placement_arr_;
+  int64_t* placement_data_;
+
+  DISALLOW_COPY_AND_ASSIGN(PandasBlock);
+};
+
+#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum)                         \
+  case Type::ArrowEnum:                                                     \
+    RETURN_NOT_OK((ConvertListsLike<::arrow::ArrowType>(col, out_buffer))); \
+    break;
+
+class ObjectBlock : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
+  virtual ~ObjectBlock() {}
+
+  Status Allocate() override { return AllocateNDArray(NPY_OBJECT); }
+
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
+    Type::type type = col->type()->type;
+
+    PyObject** out_buffer =
+        reinterpret_cast<PyObject**>(block_data_) + rel_placement * num_rows_;
+
+    const ChunkedArray& data = *col->data().get();
+
+    if (type == Type::BOOL) {
+      RETURN_NOT_OK(ConvertBooleanWithNulls(data, out_buffer));
+    } else if (type == Type::BINARY) {
+      RETURN_NOT_OK(ConvertBinaryLike<arrow::BinaryArray>(data, out_buffer));
+    } else if (type == Type::STRING) {
+      RETURN_NOT_OK(ConvertBinaryLike<arrow::StringArray>(data, out_buffer));
+    } else if (type == Type::LIST) {
+      auto list_type = std::static_pointer_cast<ListType>(col->type());
+      switch (list_type->value_type()->type) {
+        CONVERTLISTSLIKE_CASE(UInt8Type, UINT8)
+        CONVERTLISTSLIKE_CASE(Int8Type, INT8)
+        CONVERTLISTSLIKE_CASE(UInt16Type, UINT16)
+        CONVERTLISTSLIKE_CASE(Int16Type, INT16)
+        CONVERTLISTSLIKE_CASE(UInt32Type, UINT32)
+        CONVERTLISTSLIKE_CASE(Int32Type, INT32)
+        CONVERTLISTSLIKE_CASE(UInt64Type, UINT64)
+        CONVERTLISTSLIKE_CASE(Int64Type, INT64)
+        CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP)
+        CONVERTLISTSLIKE_CASE(FloatType, FLOAT)
+        CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE)
+        CONVERTLISTSLIKE_CASE(StringType, STRING)
+        default: {
+          std::stringstream ss;
+          ss << "Not implemented type for lists: " << list_type->value_type()->ToString();
+          return Status::NotImplemented(ss.str());
+        }
+      }
+    } else {
       std::stringstream ss;
-      ss << "unsupported type " << PyArray_DESCR(arr)->type_num << std::endl;
+      ss << "Unsupported type for object array output: " << col->type()->ToString();
       return Status::NotImplemented(ss.str());
+    }
+
+    placement_data_[rel_placement] = abs_placement;
+    return Status::OK();
   }
-  return Status::OK();
-}
+};
 
-Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
-    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-  return PandasMaskedToArrow(pool, ao, nullptr, field, out);
-}
+template <int ARROW_TYPE, typename C_TYPE>
+class IntBlock : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
 
-// ----------------------------------------------------------------------
-// Deserialization
+  Status Allocate() override {
+    return AllocateNDArray(arrow_traits<ARROW_TYPE>::npy_type);
+  }
 
-template <int TYPE>
-struct arrow_traits {};
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
+    Type::type type = col->type()->type;
 
-template <>
-struct arrow_traits<arrow::Type::BOOL> {
-  static constexpr int npy_type = NPY_BOOL;
-  static constexpr bool supports_nulls = false;
-  static constexpr bool is_boolean = true;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
-};
+    C_TYPE* out_buffer =
+        reinterpret_cast<C_TYPE*>(block_data_) + rel_placement * num_rows_;
 
-#define INT_DECL(TYPE)                                     \
-  template <>                                              \
-  struct arrow_traits<arrow::Type::TYPE> {                 \
-    static constexpr int npy_type = NPY_##TYPE;            \
-    static constexpr bool supports_nulls = false;          \
-    static constexpr double na_value = NAN;                \
-    static constexpr bool is_boolean = false;              \
-    static constexpr bool is_numeric_not_nullable = true;  \
-    static constexpr bool is_numeric_nullable = false;     \
-    typedef typename npy_traits<NPY_##TYPE>::value_type T; \
-  };
+    const ChunkedArray& data = *col->data().get();
 
-INT_DECL(INT8);
-INT_DECL(INT16);
-INT_DECL(INT32);
-INT_DECL(INT64);
-INT_DECL(UINT8);
-INT_DECL(UINT16);
-INT_DECL(UINT32);
-INT_DECL(UINT64);
+    if (type != ARROW_TYPE) { return Status::NotImplemented(col->type()->ToString()); }
 
-template <>
-struct arrow_traits<arrow::Type::FLOAT> {
-  static constexpr int npy_type = NPY_FLOAT32;
-  static constexpr bool supports_nulls = true;
-  static constexpr float na_value = NAN;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
-  typedef typename npy_traits<NPY_FLOAT32>::value_type T;
+    ConvertIntegerNoNullsSameType<C_TYPE>(data, out_buffer);
+    placement_data_[rel_placement] = abs_placement;
+    return Status::OK();
+  }
 };
 
-template <>
-struct arrow_traits<arrow::Type::DOUBLE> {
-  static constexpr int npy_type = NPY_FLOAT64;
-  static constexpr bool supports_nulls = true;
-  static constexpr double na_value = NAN;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
-  typedef typename npy_traits<NPY_FLOAT64>::value_type T;
-};
+using UInt8Block = IntBlock<Type::UINT8, uint8_t>;
+using Int8Block = IntBlock<Type::INT8, int8_t>;
+using UInt16Block = IntBlock<Type::UINT16, uint16_t>;
+using Int16Block = IntBlock<Type::INT16, int16_t>;
+using UInt32Block = IntBlock<Type::UINT32, uint32_t>;
+using Int32Block = IntBlock<Type::INT32, int32_t>;
+using UInt64Block = IntBlock<Type::UINT64, uint64_t>;
+using Int64Block = IntBlock<Type::INT64, int64_t>;
 
-static constexpr int64_t kPandasTimestampNull = std::numeric_limits<int64_t>::min();
+class Float32Block : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
 
-template <>
-struct arrow_traits<arrow::Type::TIMESTAMP> {
-  static constexpr int npy_type = NPY_DATETIME;
-  static constexpr bool supports_nulls = true;
-  static constexpr int64_t na_value = kPandasTimestampNull;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
-  typedef typename npy_traits<NPY_DATETIME>::value_type T;
-};
+  Status Allocate() override { return AllocateNDArray(NPY_FLOAT32); }
 
-template <>
-struct arrow_traits<arrow::Type::DATE> {
-  static constexpr int npy_type = NPY_DATETIME;
-  static constexpr bool supports_nulls = true;
-  static constexpr int64_t na_value = kPandasTimestampNull;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
-  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
+    Type::type type = col->type()->type;
+
+    if (type != Type::FLOAT) { return Status::NotImplemented(col->type()->ToString()); }
+
+    float* out_buffer = reinterpret_cast<float*>(block_data_) + rel_placement * num_rows_;
+
+    ConvertNumericNullable<float>(*col->data().get(), NAN, out_buffer);
+    placement_data_[rel_placement] = abs_placement;
+    return Status::OK();
+  }
 };
 
-template <>
-struct arrow_traits<arrow::Type::STRING> {
-  static constexpr int npy_type = NPY_OBJECT;
-  static constexpr bool supports_nulls = true;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
-};
+class Float64Block : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
+
+  Status Allocate() override { return AllocateNDArray(NPY_FLOAT64); }
+
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
+    Type::type type = col->type()->type;
+
+    double* out_buffer =
+        reinterpret_cast<double*>(block_data_) + rel_placement * num_rows_;
+
+    const ChunkedArray& data = *col->data().get();
+
+#define INTEGER_CASE(IN_TYPE)                         \
+  ConvertIntegerWithNulls<IN_TYPE>(data, out_buffer); \
+  break;
+
+    switch (type) {
+      case Type::UINT8:
+        INTEGER_CASE(uint8_t);
+      case Type::INT8:
+        INTEGER_CASE(int8_t);
+      case Type::UINT16:
+        INTEGER_CASE(uint16_t);
+      case Type::INT16:
+        INTEGER_CASE(int16_t);
+      case Type::UINT32:
+        INTEGER_CASE(uint32_t);
+      case Type::INT32:
+        INTEGER_CASE(int32_t);
+      case Type::UINT64:
+        INTEGER_CASE(uint64_t);
+      case Type::INT64:
+        INTEGER_CASE(int64_t);
+      case Type::FLOAT:
+        ConvertNumericNullableCast<float, double>(data, NAN, out_buffer);
+        break;
+      case Type::DOUBLE:
+        ConvertNumericNullable<double>(data, NAN, out_buffer);
+        break;
+      default:
+        return Status::NotImplemented(col->type()->ToString());
+    }
+
+#undef INTEGER_CASE
 
-template <>
-struct arrow_traits<arrow::Type::BINARY> {
-  static constexpr int npy_type = NPY_OBJECT;
-  static constexpr bool supports_nulls = true;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
+    placement_data_[rel_placement] = abs_placement;
+    return Status::OK();
+  }
 };
 
-template <typename T>
-struct WrapBytes {};
+class BoolBlock : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
 
-template <>
-struct WrapBytes<arrow::StringArray> {
-  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
-    return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
-  }
-};
+  Status Allocate() override { return AllocateNDArray(NPY_BOOL); }
 
-template <>
-struct WrapBytes<arrow::BinaryArray> {
-  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
-    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
-  }
-};
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
+    Type::type type = col->type()->type;
 
-inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
-  if (type == NPY_DATETIME) {
-    PyArray_Descr* descr = PyArray_DESCR(out);
-    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-    if (datatype->type == arrow::Type::TIMESTAMP) {
-      auto timestamp_type = static_cast<arrow::TimestampType*>(datatype);
+    if (type != Type::BOOL) { return Status::NotImplemented(col->type()->ToString()); }
 
-      switch (timestamp_type->unit) {
-        case arrow::TimestampType::Unit::SECOND:
-          date_dtype->meta.base = NPY_FR_s;
-          break;
-        case arrow::TimestampType::Unit::MILLI:
-          date_dtype->meta.base = NPY_FR_ms;
-          break;
-        case arrow::TimestampType::Unit::MICRO:
-          date_dtype->meta.base = NPY_FR_us;
-          break;
-        case arrow::TimestampType::Unit::NANO:
-          date_dtype->meta.base = NPY_FR_ns;
-          break;
-      }
-    } else {
-      // datatype->type == arrow::Type::DATE
-      date_dtype->meta.base = NPY_FR_D;
-    }
+    uint8_t* out_buffer =
+        reinterpret_cast<uint8_t*>(block_data_) + rel_placement * num_rows_;
+
+    ConvertBooleanNoNulls(*col->data().get(), out_buffer);
+    placement_data_[rel_placement] = abs_placement;
+    return Status::OK();
   }
-}
+};
 
-template <typename T>
-inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-    // Upcast to double, set NaN as appropriate
+class DatetimeBlock : public PandasBlock {
+ public:
+  using PandasBlock::PandasBlock;
 
-    for (int i = 0; i < arr->length(); ++i) {
-      *out_values++ = prim_arr->IsNull(i) ? NAN : in_values[i];
-    }
-  }
-}
+  Status Allocate() override {
+    RETURN_NOT_OK(AllocateNDArray(NPY_DATETIME));
 
-template <typename T>
-inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-    memcpy(out_values, in_values, sizeof(T) * arr->length());
-    out_values += arr->length();
+    PyAcquireGIL lock;
+    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(
+        PyArray_DESCR(reinterpret_cast<PyArrayObject*>(block_arr_.obj()))->c_metadata);
+    date_dtype->meta.base = NPY_FR_ns;
+    return Status::OK();
   }
-}
 
-template <typename InType, typename OutType>
-inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
-    for (int32_t i = 0; i < arr->length(); ++i) {
-      *out_values = in_values[i];
-    }
-  }
-}
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
+    Type::type type = col->type()->type;
 
-static Status ConvertBooleanWithNulls(const ChunkedArray& data, PyObject** out_values) {
-  PyAcquireGIL lock;
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
+    int64_t* out_buffer =
+        reinterpret_cast<int64_t*>(block_data_) + rel_placement * num_rows_;
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (bool_arr->IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values++ = Py_None;
-      } else if (bool_arr->Value(i)) {
-        // True
-        Py_INCREF(Py_True);
-        *out_values++ = Py_True;
+    const ChunkedArray& data = *col.get()->data();
+
+    if (type == Type::DATE) {
+      // DateType is millisecond timestamp stored as int64_t
+      // TODO(wesm): Do we want to make sure to zero out the milliseconds?
+      ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
+    } else if (type == Type::TIMESTAMP) {
+      auto ts_type = static_cast<arrow::TimestampType*>(col->type().get());
+
+      if (ts_type->unit == arrow::TimeUnit::NANO) {
+        ConvertNumericNullable<int64_t>(data, kPandasTimestampNull, out_buffer);
+      } else if (ts_type->unit == arrow::TimeUnit::MICRO) {
+        ConvertDatetimeNanos<int64_t, 1000L>(data, out_buffer);
+      } else if (ts_type->unit == arrow::TimeUnit::MILLI) {
+        ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
+      } else if (ts_type->unit == arrow::TimeUnit::SECOND) {
+        ConvertDatetimeNanos<int64_t, 1000000000L>(data, out_buffer);
       } else {
-        // False
-        Py_INCREF(Py_False);
-        *out_values++ = Py_False;
+        return Status::NotImplemented("Unsupported time unit");
       }
+    } else {
+      return Status::NotImplemented(col->type()->ToString());
     }
+
+    placement_data_[rel_placement] = abs_placement;
+    return Status::OK();
   }
-  return Status::OK();
-}
+};
 
-static void ConvertBooleanNoNulls(const ChunkedArray& data, uint8_t* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = static_cast<uint8_t>(bool_arr->Value(i));
+template <int ARROW_INDEX_TYPE>
+class CategoricalBlock : public PandasBlock {
+ public:
+  CategoricalBlock(int64_t num_rows) : PandasBlock(num_rows, 1) {}
+
+  Status Allocate() override {
+    constexpr int npy_type = arrow_traits<ARROW_INDEX_TYPE>::npy_type;
+
+    if (!(npy_type == NPY_INT8 || npy_type == NPY_INT16 || npy_type == NPY_INT32 ||
+            npy_type == NPY_INT64)) {
+      return Status::Invalid("Category indices must be signed integers");
     }
+    return AllocateNDArray(npy_type, 1);
   }
-}
 
-template <typename ArrayType>
-inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values) {
-  PyAcquireGIL lock;
-  for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = static_cast<ArrayType*>(data.chunk(c).get());
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+      int64_t rel_placement) override {
+    using T = typename arrow_traits<ARROW_INDEX_TYPE>::T;
 
-    const uint8_t* data_ptr;
-    int32_t length;
-    const bool has_nulls = data.null_count() > 0;
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (has_nulls && arr->IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values = Py_None;
-      } else {
-        data_ptr = arr->GetValue(i, &length);
-        *out_values = WrapBytes<ArrayType>::Wrap(data_ptr, length);
-        if (*out_values == nullptr) {
-          return Status::UnknownError("String initialization failed");
-        }
+    T* out_values = reinterpret_cast<T*>(block_data_) + rel_placement * num_rows_;
+
+    const ChunkedArray& data = *col->data().get();
+
+    for (int c = 0; c < data.num_chunks(); c++) {
+      const std::shared_ptr<Array> arr = data.chunk(c);
+      const auto& dict_arr = static_cast<const arrow::DictionaryArray&>(*arr);
+      const auto& indices =
+          static_cast<const arrow::PrimitiveArray&>(*dict_arr.indices());
+      auto in_values = reinterpret_cast<const T*>(indices.data()->data());
+
+      // Null is -1 in CategoricalBlock
+      for (int i = 0; i < arr->length(); ++i) {
+        *out_values++ = indices.IsNull(i) ? -1 : in_values[i];
       }
-      ++out_values;
     }
-  }
-  return Status::OK();
-}
 
-template <typename ArrowType>
-inline Status ConvertListsLike(
-    const std::shared_ptr<Column>& col, PyObject** out_values) {
-  typedef arrow_traits<ArrowType::type_id> traits;
-  typedef typename ::arrow::TypeTraits<ArrowType>::ArrayType ArrayType;
+    placement_data_[rel_placement] = abs_placement;
 
-  const ChunkedArray& data = *col->data().get();
-  auto list_type = std::static_pointer_cast<ListType>(col->type());
+    auto dict_type = static_cast<const DictionaryType*>(col->type().get());
 
-  // Get column of underlying value arrays
-  std::vector<std::shared_ptr<Array>> value_arrays;
-  for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
-    value_arrays.emplace_back(arr->values());
+    PyObject* dict;
+    RETURN_NOT_OK(ConvertArrayToPandas(dict_type->dictionary(), nullptr, &dict));
+    dictionary_.reset(dict);
+
+    return Status::OK();
   }
-  auto flat_column = std::make_shared<Column>(list_type->value_field(), value_arrays);
-  // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
-  //    Storing a reference to the whole Array would be to expensive.
-  PyObject* numpy_array;
-  RETURN_NOT_OK(ConvertColumnToPandas(flat_column, nullptr, &numpy_array));
 
-  PyAcquireGIL lock;
+  Status GetPyResult(PyObject** output) override {
+    PyObject* result = PyDict_New();
+    RETURN_IF_PYERROR();
 
-  for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
+    PyDict_SetItemString(result, "block", block_arr_.obj());
+    PyDict_SetItemString(result, "dictionary", dictionary_.obj());
+    PyDict_SetItemString(result, "placement", placement_arr_.obj());
 
-    const uint8_t* data_ptr;
-    int32_t length;
-    const bool has_nulls = data.null_count() > 0;
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (has_nulls && arr->IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values = Py_None;
-      } else {
-        PyObject* start = PyLong_FromLong(arr->value_offset(i));
-        PyObject* end = PyLong_FromLong(arr->value_offset(i + 1));
-        PyObject* slice = PySlice_New(start, end, NULL);
-        *out_values = PyObject_GetItem(numpy_array, slice);
-        Py_DECREF(start);
-        Py_DECREF(end);
-        Py_DECREF(slice);
-      }
-      ++out_values;
-    }
-  }
+    *output = result;
 
-  Py_XDECREF(numpy_array);
-  return Status::OK();
-}
+    return Status::OK();
+  }
 
-template <typename T>
-inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+ protected:
+  OwnedRef dictionary_;
+};
 
-    const uint8_t* valid_bits = arr->null_bitmap_data();
+Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
+    std::shared_ptr<PandasBlock>* block) {
+#define BLOCK_CASE(NAME, TYPE)                              \
+  case PandasBlock::NAME:                                   \
+    *block = std::make_shared<TYPE>(num_rows, num_columns); \
+    break;
 
-    if (arr->null_count() > 0) {
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i];
-      }
-    } else {
-      memcpy(out_values, in_values, sizeof(T) * arr->length());
-      out_values += arr->length();
-    }
+  switch (type) {
+    BLOCK_CASE(OBJECT, ObjectBlock);
+    BLOCK_CASE(UINT8, UInt8Block);
+    BLOCK_CASE(INT8, Int8Block);
+    BLOCK_CASE(UINT16, UInt16Block);
+    BLOCK_CASE(INT16, Int16Block);
+    BLOCK_CASE(UINT32, UInt32Block);
+    BLOCK_CASE(INT32, Int32Block);
+    BLOCK_CASE(UINT64, UInt64Block);
+    BLOCK_CASE(INT64, Int64Block);
+    BLOCK_CASE(FLOAT, Float32Block);
+    BLOCK_CASE(DOUBLE, Float64Block);
+    BLOCK_CASE(BOOL, BoolBlock);
+    BLOCK_CASE(DATETIME, DatetimeBlock);
+    default:
+      return Status::NotImplemented("Unsupported block type");
   }
-}
 
-template <typename InType, typename OutType>
-inline void ConvertNumericNullableCast(
-    const ChunkedArray& data, OutType na_value, OutType* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+#undef BLOCK_CASE
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = arr->IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
-    }
-  }
+  return (*block)->Allocate();
 }
 
-template <typename T>
-inline void ConvertDates(const ChunkedArray& data, T na_value, T* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
-      *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / 86400000;
-    }
+static inline bool ListTypeSupported(const Type::type type_id) {
+  switch (type_id) {
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::INT64:
+    case Type::UINT64:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+    case Type::STRING:
+    case Type::TIMESTAMP:
+      // The above types are all supported.
+      return true;
+    default:
+      break;
   }
+  return false;
 }
 
-template <typename InType, int SHIFT>
-inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
-
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = arr->IsNull(i) ? kPandasTimestampNull
-                                     : (static_cast<int64_t>(in_values[i]) * SHIFT);
+static inline Status MakeCategoricalBlock(const std::shared_ptr<DataType>& type,
+    int64_t num_rows, std::shared_ptr<PandasBlock>* block) {
+  // All categoricals become a block with a single column
+  auto dict_type = static_cast<const DictionaryType*>(type.get());
+  switch (dict_type->index_type()->type) {
+    case Type::INT8:
+      *block = std::make_shared<CategoricalBlock<Type::INT8>>(num_rows);
+      break;
+    case Type::INT16:
+      *block = std::make_shared<CategoricalBlock<Type::INT16>>(num_rows);
+      break;
+    case Type::INT32:
+      *block = std::make_shared<CategoricalBlock<Type::INT32>>(num_rows);
+      break;
+    case Type::INT64:
+      *block = std::make_shared<CategoricalBlock<Type::INT64>>(num_rows);
+      break;
+    default: {
+      std::stringstream ss;
+      ss << "Categorical index type not implemented: "
+         << dict_type->index_type()->ToString();
+      return Status::NotImplemented(ss.str());
     }
   }
+  return (*block)->Allocate();
 }
 
-class ArrowDeserializer {
+// Construct the exact pandas 0.x "BlockManager" memory layout
+//
+// * For each column determine the correct output pandas type
+// * Allocate 2D blocks (ncols x nrows) for each distinct data type in output
+// * Allocate  block placement arrays
+// * Write Arrow columns out into each slice of memory; populate block
+// * placement arrays as we go
+class DataFrameBlockCreator {
  public:
-  ArrowDeserializer(const std::shared_ptr<Column>& col, PyObject* py_ref)
-      : col_(col), data_(*col->data().get()), py_ref_(py_ref) {}
-
-  Status AllocateOutput(int type) {
-    PyAcquireGIL lock;
-
-    npy_intp dims[1] = {col_->length()};
-    out_ = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNew(1, dims, type));
+  DataFrameBlockCreator(const std::shared_ptr<Table>& table) : table_(table) {}
 
-    if (out_ == NULL) {
-      // Error occurred, trust that SimpleNew set the error state
-      return Status::OK();
-    }
+  Status Convert(int nthreads, PyObject** output) {
+    column_types_.resize(table_->num_columns());
+    column_block_placement_.resize(table_->num_columns());
+    type_counts_.clear();
+    blocks_.clear();
 
-    set_numpy_metadata(type, col_->type().get(), out_);
+    RETURN_NOT_OK(CreateBlocks());
+    RETURN_NOT_OK(WriteTableToBlocks(nthreads));
 
-    return Status::OK();
+    return GetResultList(output);
   }
 
-  template <int TYPE>
-  Status ConvertValuesZeroCopy(int npy_type, std::shared_ptr<Array> arr) {
-    typedef typename arrow_traits<TYPE>::T T;
-
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+  Status CreateBlocks() {
+    for (int i = 0; i < table_->num_columns(); ++i) {
+      std::shared_ptr<Column> col = table_->column(i);
+      PandasBlock::type output_type;
 
-    // Zero-Copy. We can pass the data pointer directly to NumPy.
-    void* data = const_cast<T*>(in_values);
+      Type::type column_type = col->type()->type;
+      switch (column_type) {
+        case Type::BOOL:
+          output_type = col->null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL;
+          break;
+        case Type::UINT8:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT8;
+          break;
+        case Type::INT8:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT8;
+          break;
+        case Type::UINT16:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT16;
+          break;
+        case Type::INT16:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT16;
+          break;
+        case Type::UINT32:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT32;
+          break;
+        case Type::INT32:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT32;
+          break;
+        case Type::INT64:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT64;
+          break;
+        case Type::UINT64:
+          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT64;
+          break;
+        case Type::FLOAT:
+          output_type = PandasBlock::FLOAT;
+          break;
+        case Type::DOUBLE:
+          output_type = PandasBlock::DOUBLE;
+          break;
+        case Type::STRING:
+        case Type::BINARY:
+          output_type = PandasBlock::OBJECT;
+          break;
+        case Type::DATE:
+          output_type = PandasBlock::DATETIME;
+          break;
+        case Type::TIMESTAMP:
+          output_type = PandasBlock::DATETIME;
+          break;
+        case Type::LIST: {
+          auto list_type = std::static_pointer_cast<ListType>(col->type());
+          if (!ListTypeSupported(list_type->value_type()->type)) {
+            std::stringstream ss;
+            ss << "Not implemented type for lists: "
+               << list_type->value_type()->ToString();
+            return Status::NotImplemented(ss.str());
+          }
+          output_type = PandasBlock::OBJECT;
+        } break;
+        case Type::DICTIONARY:
+          output_type = PandasBlock::CATEGORICAL;
+          break;
+        default:
+          return Status::NotImplemented(col->type()->ToString());
+      }
 
-    PyAcquireGIL lock;
+      int block_placement = 0;
+      if (column_type == Type::DICTIONARY) {
+        std::shared_ptr<PandasBlock> block;
+        RETURN_NOT_OK(MakeCategoricalBlock(col->type(), table_->num_rows(), &block));
+        categorical_blocks_[i] = block;
+      } else {
+        auto it = type_counts_.find(output_type);
+        if (it != type_counts_.end()) {
+          block_placement = it->second;
+          // Increment count
+          it->second += 1;
+        } else {
+          // Add key to map
+          type_counts_[output_type] = 1;
+        }
+      }
 
-    // Zero-Copy. We can pass the data pointer directly to NumPy.
-    npy_intp dims[1] = {col_->length()};
-    out_ = reinterpret_cast<PyArrayObject*>(
-        PyArray_SimpleNewFromData(1, dims, npy_type, data));
+      column_types_[i] = output_type;
+      column_block_placement_[i] = block_placement;
+    }
 
-    if (out_ == NULL) {
-      // Error occurred, trust that SimpleNew set the error state
-      return Status::OK();
+    // Create normal non-categorical blocks
+    for (const auto& it : type_counts_) {
+      PandasBlock::type type = static_cast<PandasBlock::type>(it.first);
+      std::shared_ptr<PandasBlock> block;
+      RETURN_NOT_OK(MakeBlock(type, table_->num_rows(), it.second, &block));
+      blocks_[type] = block;
     }
+    return Status::OK();
+  }
 
-    set_numpy_metadata(npy_type, col_->type().get(), out_);
+  Status WriteTableToBlocks(int nthreads) {
+    auto WriteColumn = [this](int i) {
+      std::shared_ptr<Column> col = this->table_->column(i);
+      PandasBlock::type output_type = this->column_types_[i];
 
-    if (PyArray_SetBaseObject(out_, py_ref_) == -1) {
-      // Error occurred, trust that SetBaseObject set the error state
-      return Status::OK();
-    } else {
-      // PyArray_SetBaseObject steals our reference to py_ref_
-      Py_INCREF(py_ref_);
-    }
+      int rel_placement = this->column_block_placement_[i];
 
-    // Arrow data is immutable.
-    PyArray_CLEARFLAGS(out_, NPY_ARRAY_WRITEABLE);
+      std::shared_ptr<PandasBlock> block;
+      if (output_type == PandasBlock::CATEGORICAL) {
+        auto it = this->categorical_blocks_.find(i);
+        if (it == this->blocks_.end()) {
+          return Status::KeyError("No categorical block allocated");
+        }
+        block = it->second;
+      } else {
+        auto it = this->blocks_.find(output_type);
+        if (it == this->blocks_.end()) { return Status::KeyError("No block allocated"); }
+        block = it->second;
+      }
+      return block->Write(col, i, rel_placement);
+    };
 
-    return Status::OK();
-  }
+    nthreads = std::min<int>(nthreads, table_->num_columns());
 
-  // ----------------------------------------------------------------------
-  // Allocate new array and deserialize. Can do a zero copy conversion for some
-  // types
+    if (nthreads == 1) {
+      for (int i = 0; i < table_->num_columns(); ++i) {
+        RETURN_NOT_OK(WriteColumn(i));
+      }
+    } else {
+      std::vector<std::thread> thread_pool;
+      thread_pool.reserve(nthreads);
+      std::atomic<int> task_counter(0);
 
-  Status Convert(PyObject** out) {
-#define CONVERT_CASE(TYPE)                             \
-  case arrow::Type::TYPE: {                            \
-    RETURN_NOT_OK(ConvertValues<arrow::Type::TYPE>()); \
-  } break;
+      std::mutex error_mtx;
+      bool error_occurred = false;
+      Status error;
 
-    switch (col_->type()->type) {
-      CONVERT_CASE(BOOL);
-      CONVERT_CASE(INT8);
-      CONVERT_CASE(INT16);
-      CONVERT_CASE(INT32);
-      CONVERT_CASE(INT64);
-      CONVERT_CASE(UINT8);
-      CONVERT_CASE(UINT16);
-      CONVERT_CASE(UINT32);
-      CONVERT_CASE(UINT64);
-      CONVERT_CASE(FLOAT);
-      CONVERT_CASE(DOUBLE);
-      CONVERT_CASE(BINARY);
-      CONVERT_CASE(STRING);
-      CONVERT_CASE(DATE);
-      CONVERT_CASE(TIMESTAMP);
-      default: {
-        std::stringstream ss;
-        ss << "Arrow type reading not implemented for " << col_->type()->ToString();
-        return Status::NotImplemented(ss.str());
+      for (int thread_id = 0; thread_id < nthreads; ++thread_id) {
+        thread_pool.emplace_back(
+            [this, &error, &error_occurred, &error_mtx, &task_counter, &WriteColumn]() {
+              int column_num;
+              while (!error_occurred) {
+                column_num = task_counter.fetch_add(1);
+                if (column_num >= this->table_->num_columns()) { break; }
+                Status s = WriteColumn(column_num);
+                if (!s.ok()) {
+                  std::lock_guard<std::mutex> lock(error_mtx);
+                  error_occurred = true;
+                  error = s;
+                  break;
+                }
+              }
+            });
+      }
+      for (auto&& thread : thread_pool) {
+        thread.join();
       }
-    }
-
-#undef CONVERT_CASE
 
-    *out = reinterpret_cast<PyObject*>(out_);
+      if (error_occurred) { return error; }
+    }
     return Status::OK();
   }
 
-  template <int TYPE>
-  inline typename std::enable_if<
-      (TYPE != arrow::Type::DATE) & arrow_traits<TYPE>::is_numeric_nullable, Status>::type
-  ConvertValues() {
-    typedef typename arrow_traits<TYPE>::T T;
-    int npy_type = arrow_traits<TYPE>::npy_type;
+  Status GetResultList(PyObject** out) {
+    PyAcquireGIL lock;
 
-    if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
-      return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
+    auto num_blocks =
+        static_cast<Py_ssize_t>(blocks_.size() + categorical_blocks_.size());
+    PyObject* result = PyList_New(num_blocks);
+    RETURN_IF_PYERROR();
+
+    int i = 0;
+    for (const auto& it : blocks_) {
+      const std::shared_ptr<PandasBlock> block = it.second;
+      PyObject* item;
+      RETURN_NOT_OK(block->GetPyResult(&item));
+      if (PyList_SET_ITEM(result, i++, item) < 0) { RETURN_IF_PYERROR(); }
     }
 
-    RETURN_NOT_OK(AllocateOutput(npy_type));
-    auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
-    ConvertNumericNullable<T>(data_, arrow_traits<TYPE>::na_value, out_values);
+    for (const auto& it : categorical_blocks_) {
+      const std::shared_ptr<PandasBlock> block = it.second;
+      PyObject* item;
+      RETURN_NOT_OK(block->GetPyResult(&item));
+      if (PyList_SET_ITEM(result, i++, item) < 0) { RETURN_IF_PYERROR(); }
+    }
 
+    *out = result;
     return Status::OK();
   }
 
-  template <int TYPE>
-  inline typename std::enable_if<TYPE == arrow::Type::DATE, Status>::type
-  ConvertValues() {
-    typedef typename arrow_traits<TYPE>::T T;
+ private:
+  std::shared_ptr<Table> table_;
 
-    RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
-    auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
-    ConvertDates<T>(data_, arrow_traits<TYPE>::na_value, out_values);
-    return Status::OK();
-  }
+  // column num -> block type id
+  std::vector<PandasBlock::type> column_types_;
 
-  // Integer specialization
-  template <int TYPE>
-  inline
-      typename std::enable_if<arrow_traits<TYPE>::is_numeric_not_nullable, Status>::type
-      ConvertValues() {
-    typedef typename arrow_traits<TYPE>::T T;
-    int npy_type = arrow_traits<TYPE>::npy_type;
+  // column num -> relative placement within internal block
+  std::vector<int> column_block_placement_;
 
-    if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
-      return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
-    }
+  // block type -> type count
+  std::unordered_map<int, int> type_counts_;
 
-    if (data_.null_count() > 0) {
-      RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
-      auto out_values = reinterpret_cast<double*>(PyArray_DATA(out_));
-      ConvertIntegerWithNulls<T>(data_, out_values);
-    } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
-      auto out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
-      ConvertIntegerNoNullsSameType<T>(data_, out_values);
-    }
+  // block type -> block
+  std::unordered_map<int, std::shared_ptr<PandasBlock>> blocks_;
 
-    return Status::OK();
-  }
+  // column number -> categorical block
+  std::unordered_map<int, std::shared_ptr<PandasBlock>> categorical_blocks_;
+};
 
-  // Boolean specialization
-  template <int TYPE>
-  inline typename std::enable_if<arrow_traits<TYPE>::is_boolean, Status>::type
-  ConvertValues() {
-    if (data_.null_count() > 0) {
-      RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-      auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
-      RETURN_NOT_OK(ConvertBooleanWithNulls(data_, out_values));
-    } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
-      auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(out_));
-      ConvertBooleanNoNulls(data_, out_values);
-    }
-    return Status::OK();
+Status ConvertTableToPandas(
+    const std::shared_ptr<Table>& table, int nthreads, PyObject** out) {
+  DataFrameBlockCreator helper(table);
+  return helper.Convert(nthreads, out);
+}
+
+// ----------------------------------------------------------------------
+// Serialization
+
+template <int TYPE>
+class ArrowSerializer {
+ public:
+  ArrowSerializer(arrow::MemoryPool* pool, PyArrayObject* arr, PyArrayObject* mask)
+      : pool_(pool), arr_(arr), mask_(mask) {
+    length_ = PyArray_SIZE(arr_);
   }
 
-  // UTF8 strings
-  template <int TYPE>
-  inline typename std::enable_if<TYPE == arrow::Type::STRING, Status>::type
-  ConvertValues() {
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
-    return ConvertBinaryLike<arrow::StringArray>(data_, out_values);
+  void IndicateType(const std::shared_ptr<Field> field) { field_indicator_ = field; }
+
+  Status Convert(std::shared_ptr<Array>* out);
+
+  int stride() const { return PyArray_STRIDES(arr_)[0]; }
+
+  Status InitNullBitmap() {
+    int null_bytes = BitUtil::BytesForBits(length_);
+
+    null_bitmap_ = std::make_shared<arrow::PoolBuffer>(pool_);
+    RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
+
+    null_bitmap_data_ = null_bitmap_->mutable_data();
+    memset(null_bitmap_data_, 0, null_bytes);
+
+    return Status::OK();
   }
 
-  template <int T2>
-  inline typename std::enable_if<T2 == arrow::Type::BINARY, Status>::type
-  ConvertValues() {
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(out_));
-    return ConvertBinaryLike<arrow::BinaryArray>(data_, out_values);
+  bool is_strided() const {
+    npy_intp* astrides = PyArray_STRIDES(arr_);
+    return astrides[0] != PyArray_DESCR(arr_)->elsize;
   }
 
  private:
-  std::shared_ptr<Column> col_;
-  const arrow::ChunkedArray& data_;
-  PyObject* py_ref_;
-  PyArrayObject* out_;
-};
+  Status ConvertData();
 
-Status ConvertArrayToPandas(
-    const std::shared_ptr<Array>& arr, PyObject* py_ref, PyObject** out) {
-  static std::string dummy_name = "dummy";
-  auto field = std::make_shared<Field>(dummy_name, arr->type());
-  auto col = std::make_shared<Column>(field, arr);
-  return ConvertColumnToPandas(col, py_ref, out);
-}
+  Status ConvertDates(std::shared_ptr<Array>* out) {
+    PyAcquireGIL lock;
 
-Status ConvertColumnToPandas(
-    const std::shared_ptr<Column>& col, PyObject* py_ref, PyObject** out) {
-  ArrowDeserializer converter(col, py_ref);
-  return converter.Convert(out);
-}
+    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    arrow::TypePtr string_type(new arrow::DateType());
+    arrow::DateBuilder date_builder(pool_, string_type);
+    RETURN_NOT_OK(date_builder.Resize(length_));
 
-// ----------------------------------------------------------------------
-// pandas 0.x DataFrame conversion internals
+    Status s;
+    PyObject* obj;
+    for (int64_t i = 0; i < length_; ++i) {
+      obj = objects[i];
+      if (PyDate_CheckExact(obj)) {
+        PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(obj);
+        date_builder.Append(PyDate_to_ms(pydate));
+      } else {
+        date_builder.AppendNull();
+      }
+    }
+    return date_builder.Finish(out);
+  }
 
-class PandasBlock {
- public:
-  enum type {
-    OBJECT,
-    UINT8,
-    INT8,
-    UINT16,
-    INT16,
-    UINT32,
-    INT32,
-    UINT64,
-    INT64,
-    FLOAT,
-    DOUBLE,
-    BOOL,
-    DATETIME,
-    CATEGORICAL
-  };
+  Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
+    PyAcquireGIL lock;
 
-  PandasBlock(int64_t num_rows, int num_columns)
-      : num_rows_(num_rows), num_columns_(num_columns) {}
-  virtual ~PandasBlock() {}
+    // The output type at this point is inconclusive because there may be bytes
+    // and unicode mixed in the object array
 
-  virtual Status Allocate() = 0;
-  virtual Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) = 0;
+    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    arrow::TypePtr string_type(new arrow::StringType());
+    arrow::StringBuilder string_builder(pool_, string_type);
+    RETURN_NOT_OK(string_builder.Resize(length_));
 
-  PyObject* block_arr() { return block_arr_.obj(); }
+    Status s;
+    bool have_bytes = false;
+    RETURN_NOT_OK(AppendObjectStrings(string_builder, objects, length_, &have_bytes));
+    RETURN_NOT_OK(string_builder.Finish(out));
 
-  PyObject* placement_arr() { return placement_arr_.obj(); }
+    if (have_bytes) {
+      const auto& arr = static_cast<const arrow::StringArray&>(*out->get());
+      *out = std::make_shared<arrow::BinaryArray>(
+          arr.length(), arr.offsets(), arr.data(), arr.null_count(), arr.null_bitmap());
+    }
+    return Status::OK();
+  }
 
- protected:
-  Status AllocateNDArray(int npy_type) {
+  Status ConvertBooleans(std::shared_ptr<Array>* out) {
     PyAcquireGIL lock;
 
-    npy_intp block_dims[2] = {num_columns_, num_rows_};
-    PyObject* block_arr = PyArray_SimpleNew(2, block_dims, npy_type);
-    if (block_arr == NULL) {
-      // TODO(wesm): propagating Python exception
-      return Status::OK();
+    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+
+    int nbytes = BitUtil::BytesForBits(length_);
+    auto data = std::make_shared<arrow::PoolBuffer>(pool_);
+    RETURN_NOT_OK(data->Resize(nbytes));
+    uint8_t* bitmap = data->mutable_data();
+    memset(bitmap, 0, nbytes);
+
+    int64_t null_count = 0;
+    for (int64_t i = 0; i < length_; ++i) {
+      if (objects[i] == Py_True) {
+        BitUtil::SetBit(bitmap, i);
+        BitUtil::SetBit(null_bitmap_data_, i);
+      } else if (objects[i] != Py_False) {
+        ++null_count;
+      } else {
+        BitUtil::SetBit(null_bitmap_data_, i);
+      }
     }
 
-    npy_intp placement_dims[1] = {num_columns_};
-    PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
-    if (placement_arr == NULL) {
-      // TODO(wesm): propagating Python exception
-      return Status::OK();
+    *out = std::make_shared<arrow::BooleanArray>(length_, data, null_count, null_bitmap_);
+
+    return Status::OK();
+  }
+
+  template <int ITEM_TYPE, typename ArrowType>
+  Status ConvertTypedLists(
+      const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out);
+
+#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                            \
+  case Type::TYPE: {                                                      \
+    return ConvertTypedLists<NUMPY_TYPE, ::arrow::ArrowType>(field, out); \
+  }
+
+  Status ConvertLists(const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+    switch (field->type->type) {
+      LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
+      LIST_CASE(INT8, NPY_INT8, Int8Type)
+      LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
+      LIST_CASE(INT16, NPY_INT16, Int16Type)
+      LIST_CASE(UINT32, NPY_UINT32, UInt32Type)
+      LIST_CASE(INT32, NPY_INT32, Int32Type)
+      LIST_CASE(UINT64, NPY_UINT64, UInt64Type)
+      LIST_CASE(INT64, NPY_INT64, Int64Type)
+      LIST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
+      LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
+      LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
+      LIST_CASE(STRING, NPY_OBJECT, StringType)
+      default:
+        return Status::TypeError("Unknown list item type");
     }
 
-    block_arr_.reset(block_arr);
-    placement_arr_.reset(placement_arr);
-
-    block_data_ = reinterpret_cast<uint8_t*>(
-        PyArray_DATA(reinterpret_cast<PyArrayObject*>(block_arr)));
-
-    placement_data_ = reinterpret_cast<int64_t*>(
-        PyArray_DATA(reinterpret_cast<PyArrayObject*>(placement_arr)));
-
-    return Status::OK();
+    return Status::TypeError("Unknown list type");
   }
 
-  int64_t num_rows_;
-  int num_columns_;
-
-  OwnedRef block_arr_;
-  uint8_t* block_data_;
+  Status MakeDataType(std::shared_ptr<DataType>* out);
 
-  // ndarray<int32>
-  OwnedRef placement_arr_;
-  int64_t* placement_data_;
+  arrow::MemoryPool* pool_;
 
-  DISALLOW_COPY_AND_ASSIGN(PandasBlock);
-};
+  PyArrayObject* arr_;
+  PyArrayObject* mask_;
 
-#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum)                         \
-  case Type::ArrowEnum:                                                     \
-    RETURN_NOT_OK((ConvertListsLike<::arrow::ArrowType>(col, out_buffer))); \
-    break;
+  int64_t length_;
 
-class ObjectBlock : public PandasBlock {
- public:
-  using PandasBlock::PandasBlock;
-  virtual ~ObjectBlock() {}
+  std::shared_ptr<Field> field_indicator_;
+  std::shared_ptr<arrow::Buffer> data_;
+  std::shared_ptr<arrow::ResizableBuffer> null_bitmap_;
+  uint8_t* null_bitmap_data_;
+};
 
-  Status Allocate() override { return AllocateNDArray(NPY_OBJECT); }
+// Returns null count
+static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
+  int64_t null_count = 0;
+  const uint8_t* mask_values = static_cast<const uint8_t*>(PyArray_DATA(mask));
+  // TODO(wesm): strided null mask
+  for (int i = 0; i < length; ++i) {
+    if (mask_values[i]) {
+      ++null_count;
+    } else {
+      BitUtil::SetBit(bitmap, i);
+    }
+  }
+  return null_count;
+}
 
-  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+template <int TYPE>
+inline Status ArrowSerializer<TYPE>::MakeDataType(std::shared_ptr<DataType>* out) {
+  out->reset(new typename npy_traits<TYPE>::TypeClass());
+  return Status::OK();
+}
 
-    PyObject** out_buffer =
-        reinterpret_cast<PyObject**>(block_data_) + rel_placement * num_rows_;
+template <>
+inline Status ArrowSerializer<NPY_DATETIME>::MakeDataType(
+    std::shared_ptr<DataType>* out) {
+  PyArray_Descr* descr = PyArray_DESCR(arr_);
+  auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+  arrow::TimestampType::Unit unit;
 
-    const ChunkedArray& data = *col->data().get();
+  switch (date_dtype->meta.base) {
+    case NPY_FR_s:
+      unit = arrow::TimestampType::Unit::SECOND;
+      break;
+    case NPY_FR_ms:
+      unit = arrow::TimestampType::Unit::MILLI;
+      break;
+    case NPY_FR_us:
+      unit = arrow::TimestampType::Unit::MICRO;
+      break;
+    case NPY_FR_ns:
+      unit = arrow::TimestampType::Unit::NANO;
+      break;
+    default:
+      return Status::Invalid("Unknown NumPy datetime unit");
+  }
 
-    if (type == Type::BOOL) {
-      RETURN_NOT_OK(ConvertBooleanWithNulls(data, out_buffer));
-    } else if (type == Type::BINARY) {
-      RETURN_NOT_OK(ConvertBinaryLike<arrow::BinaryArray>(data, out_buffer));
-    } else if (type == Type::STRING) {
-      RETURN_NOT_OK(ConvertBinaryLike<arrow::StringArray>(data, out_buffer));
-    } else if (type == Type::LIST) {
-      auto list_type = std::static_pointer_cast<ListType>(col->type());
-      switch (list_type->value_type()->type) {
-        CONVERTLISTSLIKE_CASE(UInt8Type, UINT8)
-        CONVERTLISTSLIKE_CASE(Int8Type, INT8)
-        CONVERTLISTSLIKE_CASE(UInt16Type, UINT16)
-        CONVERTLISTSLIKE_CASE(Int16Type, INT16)
-        CONVERTLISTSLIKE_CASE(UInt32Type, UINT32)
-        CONVERTLISTSLIKE_CASE(Int32Type, INT32)
-        CONVERTLISTSLIKE_CASE(UInt64Type, UINT64)
-        CONVERTLISTSLIKE_CASE(Int64Type, INT64)
-        CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP)
-        CONVERTLISTSLIKE_CASE(FloatType, FLOAT)
-        CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE)
-        CONVERTLISTSLIKE_CASE(StringType, STRING)
-        default: {
-          std::stringstream ss;
-          ss << "Not implemented type for lists: " << list_type->value_type()->ToString();
-          return Status::NotImplemented(ss.str());
-        }
-      }
-    } else {
-      std::stringstream ss;
-      ss << "Unsupported type for object array output: " << col->type()->ToString();
-      return Status::NotImplemented(ss.str());
-    }
+  out->reset(new arrow::TimestampType(unit));
+  return Status::OK();
+}
 
-    placement_data_[rel_placement] = abs_placement;
-    return Status::OK();
-  }
-};
+template <int TYPE>
+inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
+  typedef npy_traits<TYPE> traits;
 
-template <int ARROW_TYPE, typename C_TYPE>
-class IntBlock : public PandasBlock {
- public:
-  using PandasBlock::PandasBlock;
+  if (mask_ != nullptr || traits::supports_nulls) { RETURN_NOT_OK(InitNullBitmap()); }
 
-  Status Allocate() override {
-    return AllocateNDArray(arrow_traits<ARROW_TYPE>::npy_type);
+  int64_t null_count = 0;
+  if (mask_ != nullptr) {
+    null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
+  } else if (traits::supports_nulls) {
+    null_count = ValuesToBitmap<TYPE>(PyArray_DATA(arr_), length_, null_bitmap_data_);
   }
 
-  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
-    Type::type type = col->type()->type;
-
-    C_TYPE* out_buffer =
-        reinterpret_cast<C_TYPE*>(block_data_) + rel_placement * num_rows_;
+  RETURN_NOT_OK(ConvertData());
+  std::shared_ptr<DataType> type;
+  RETURN_NOT_OK(MakeDataType(&type));
+  RETURN_NOT_OK(MakePrimitiveArray(type, length_, data_, null_count, null_bitmap_, out));
+  return Status::OK();
+}
 
-    const ChunkedArray& data = *col->data().get();
+template <>
+inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out) {
+  // Python object arrays are annoying, since we could have one of:
+  //
+  // * Strings
+  // * Booleans with nulls
+  // * Mixed type (not supported at the moment by arrow format)
+  //
+  // Additionally, nulls may be encoded either as np.nan or None. So we have to
+  // do some type inference and conversion
 
-    if (type != ARROW_TYPE) { return Status::NotImplemented(col->type()->ToString()); }
+  RETURN_NOT_OK(InitNullBitmap());
 
-    ConvertIntegerNoNullsSameType<C_TYPE>(data, out_buffer);
-    placement_data_[rel_placement] = abs_placement;
-    return Status::OK();
+  // TODO: mask not supported here
+  const PyObject** objects = reinterpret_cast<const PyObject**>(PyArray_DATA(arr_));
+  {
+    PyAcquireGIL lock;
+    PyDateTime_IMPORT;
   }
-};
-
-using UInt8Block = IntBlock<Type::UINT8, uint8_t>;
-using Int8Block = IntBlock<Type::INT8, int8_t>;
-using UInt16Block = IntBlock<Type::UINT16, uint16_t>;
-using Int16Block = IntBlock<Type::INT16, int16_t>;
-using UInt32Block = IntBlock<Type::UINT32, uint32_t>;
-using Int32Block = IntBlock<Type::INT32, int32_t>;
-using UInt64Block = IntBlock<Type::UINT64, uint64_t>;
-using Int64Block = IntBlock<Type::INT64, int64_t>;
 
-class Float32Block : public PandasBlock {
- public:
-  using PandasBlock::PandasBlock;
+  if (field_indicator_) {
+    switch (field_indicator_->type->type) {
+      case Type::STRING:
+        return ConvertObjectStrings(out);
+      case Type::BOOL:
+        return ConvertBooleans(out);
+      case Type::DATE:
+        return ConvertDates(out);
+      case Type::LIST: {
+        auto list_field = static_cast<ListType*>(field_indicator_->type.get());
+        return ConvertLists(list_field->value_field(), out);
+      }
+      default:
+        return Status::TypeError("No known conversion to Arrow type");
+    }
+  } else {
+    for (int64_t i = 0; i < length_; ++i) {
+      if (PyObject_is_null(objects[i])) {
+        continue;
+      } else if (PyObject_is_string(objects[i])) {
+        return ConvertObjectStrings(out);
+      } else if (PyBool_Check(objects[i])) {
+        return ConvertBooleans(out);
+      } else if (PyDate_CheckExact(objects[i])) {
+        return ConvertDates(out);
+      } else {
+        return Status::TypeError("unhandled python type");
+      }
+    }
+  }
 
-  Status Allocate() override { return AllocateNDArray(NPY_FLOAT32); }
+  return Status::TypeError("Unable to infer type of object array, were all null");
+}
 
-  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+template <int TYPE>
+inline Status ArrowSerializer<TYPE>::ConvertData() {
+  // TODO(wesm): strided arrays
+  if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
 
-    if (type != Type::FLOAT) { return Status::NotImplemented(col->type()->ToString()); }
+  data_ = std::make_shared<NumPyBuffer>(arr_);
+  return Status::OK();
+}
 
-    float* out_buffer = reinterpret_cast<float*>(block_data_) + rel_placement * num_rows_;
+template <>
+inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
+  if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
 
-    ConvertNumericNullable<float>(*col->data().get(), NAN, out_buffer);
-    placement_data_[rel_placement] = abs_placement;
-    return Status::OK();
-  }
-};
+  int nbytes = BitUtil::BytesForBits(length_);
+  auto buffer = std::make_shared<arrow::PoolBuffer>(pool_);
+  RETURN_NOT_OK(buffer->Resize(nbytes));
 
-class Float64Block : public PandasBlock {
- public:
-  using PandasBlock::PandasBlock;
+  const uint8_t* values = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
-  Status Allocate() override { return AllocateNDArray(NPY_FLOAT64); }
+  uint8_t* bitmap = buffer->mutable_data();
 
-  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+  memset(bitmap, 0, nbytes);
+  for (int i = 0; i < length_; ++i) {
+    if (values[i] > 0) { BitUtil::SetBit(bitmap, i); }
+  }
 
-    double* out_buffer =
-        reinterpret_cast<double*>(block_data_) + rel_placement * num_rows_;
+  data_ = buffer;
 
-    const ChunkedArray& data = *col->data().get();
+  return Status::OK();
+}
 
-#define INTEGER_CASE(IN_TYPE)                         \
-  ConvertIntegerWithNulls<IN_TYPE>(data, out_buffer); \
-  break;
+template <int TYPE>
+template <int ITEM_TYPE, typename ArrowType>
+inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
+    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+  typedef npy_traits<ITEM_TYPE> traits;
+  typedef typename traits::value_type T;
+  typedef typename traits::BuilderClass BuilderT;
 
-    switch (type) {
-      case Type::UINT8:
-        INTEGER_CASE(uint8_t);
-      case Type::INT8:
-        INTEGER_CASE(int8_t);
-      case Type::UINT16:
-        INTEGER_CASE(uint16_t);
-      case Type::INT16:
-        INTEGER_CASE(int16_t);
-      case Type::UINT32:
-        INTEGER_CASE(uint32_t);
-      case Type::INT32:
-        INTEGER_CASE(int32_t);
-      case Type::UINT64:
-        INTEGER_CASE(uint64_t);
-      case Type::INT64:
-        INTEGER_CASE(int64_t);
-      case Type::FLOAT:
-        ConvertNumericNullableCast<float, double>(data, NAN, out_buffer);
-        break;
-      case Type::DOUBLE:
-        ConvertNumericNullable<double>(data, NAN, out_buffer);
-        break;
-      default:
-        return Status::NotImplemented(col->type()->ToString());
-    }
+  auto value_builder = std::make_shared<BuilderT>(pool_, field->type);
+  ListBuilder list_builder(pool_, value_builder);
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  for (int64_t i = 0; i < length_; ++i) {
+    if (PyObject_is_null(objects[i])) {
+      RETURN_NOT_OK(list_builder.AppendNull());
+    } else if (PyArray_Check(objects[i])) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
+      RETURN_NOT_OK(list_builder.Append(true));
 
-#undef INTEGER_CASE
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
 
-    placement_data_[rel_placement] = abs_placement;
-    return Status::OK();
+      int32_t size = PyArray_DIM(numpy_array, 0);
+      auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
+      if (traits::supports_nulls) {
+        null_bitmap_->Resize(size, false);
+        // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
+        // currently support this.
+        // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
+        ValuesToBytemap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
+        RETURN_NOT_OK(value_builder->Append(data, size, null_bitmap_->data()));
+      } else {
+        RETURN_NOT_OK(value_builder->Append(data, size));
+      }
+    } else if (PyList_Check(objects[i])) {
+      return Status::TypeError("Python lists are not yet supported");
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
   }
-};
+  return list_builder.Finish(out);
+}
 
-class BoolBlock : public PandasBlock {
- public:
-  using PandasBlock::PandasBlock;
+template <>
+template <>
+inline Status
+ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
+    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+  // TODO: If there are bytes involed, convert to Binary representation
+  bool have_bytes = false;
 
-  Status Allocate() override { return AllocateNDArray(NPY_BOOL); }
+  auto value_builder = std::make_shared<arrow::StringBuilder>(pool_, field->type);
+  ListBuilder list_builder(pool_, value_builder);
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  for (int64_t i = 0; i < length_; ++i) {
+    if (PyObject_is_null(objects[i])) {
+      RETURN_NOT_OK(list_builder.AppendNull());
+    } else if (PyArray_Check(objects[i])) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
+      RETURN_NOT_OK(list_builder.Append(true));
 
-  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
 
-    if (type != Type::BOOL) { return Status::NotImplemented(col->type()->ToString()); }
+      int32_t size = PyArray_DIM(numpy_array, 0);
+      auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
+      RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
+    } else if (PyList_Check(objects[i])) {
+      return Status::TypeError("Python lists are not yet supported");
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
+  }
+  return list_builder.Finish(out);
+}
 
-    uint8_t* out_buffer =
-        reinterpret_cast<uint8_t*>(block_data_) + rel_placement * num_rows_;
+template <>
+inline Status ArrowSerializer<NPY_OBJECT>::ConvertData() {
+  return Status::TypeError("NYI");
+}
 
-    ConvertBooleanNoNulls(*col->data().get(), out_buffer);
-    placement_data_[rel_placement] = abs_placement;
-    return Status::OK();
-  }
-};
+#define TO_ARROW_CASE(TYPE)                                 \
+  case NPY_##TYPE: {                                        \
+    ArrowSerializer<NPY_##TYPE> converter(pool, arr, mask); \
+    RETURN_NOT_OK(converter.Convert(out));                  \
+  } break;
 
-class DatetimeBlock : public PandasBlock {
- public:
-  using PandasBlock::PandasBlock;
+Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
+  PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(ao);
+  PyArrayObject* mask = nullptr;
 
-  Status Allocate() override {
-    RETURN_NOT_OK(AllocateNDArray(NPY_DATETIME));
+  if (mo != nullptr and mo != Py_None) { mask = reinterpret_cast<PyArrayObject*>(mo); }
 
-    PyAcquireGIL lock;
-    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(
-        PyArray_DESCR(reinterpret_cast<PyArrayObject*>(block_arr_.obj()))->c_metadata);
-    date_dtype->meta.base = NPY_FR_ns;
-    return Status::OK();
+  if (PyArray_NDIM(arr) != 1) {
+    return Status::Invalid("only handle 1-dimensional arrays");
   }
 
-  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+  switch (PyArray_DESCR(arr)->type_num) {
+    TO_ARROW_CASE(BOOL);
+    TO_ARROW_CASE(INT8);
+    TO_ARROW_CASE(INT16);
+    TO_ARROW_CASE(INT32);
+    TO_ARROW_CASE(INT64);
+    TO_ARROW_CASE(UINT8);
+    TO_ARROW_CASE(UINT16);
+    TO_ARROW_CASE(UINT32);
+    TO_ARROW_CASE(UINT64);
+    TO_ARROW_CASE(FLOAT32);
+    TO_ARROW_CASE(FLOAT64);
+    TO_ARROW_CASE(DATETIME);
+    case NPY_OBJECT: {
+      ArrowSerializer<NPY_OBJECT> converter(pool, arr, mask);
+      converter.IndicateType(field);
+      RETURN_NOT_OK(converter.Convert(out));
+    } break;
+    default:
+      std::stringstream ss;
+      ss << "unsupported type " << PyArray_DESCR(arr)->type_num << std::endl;
+      return Status::NotImplemented(ss.str());
+  }
+  return Status::OK();
+}
 
-    int64_t* out_buffer =
-        reinterpret_cast<int64_t*>(block_data_) + rel_placement * num_rows_;
+class ArrowDeserializer {
+ public:
+  ArrowDeserializer(const std::shared_ptr<Column>& col, PyObject* py_ref)
+      : col_(col), data_(*col->data().get()), py_ref_(py_ref) {}
 
-    const ChunkedArray& data = *col.get()->data();
+  Status AllocateOutput(int type) {
+    PyAcquireGIL lock;
 
-    if (type == Type::DATE) {
-      // DateType is millisecond timestamp stored as int64_t
-      // TODO(wesm): Do we want to make sure to zero out the milliseconds?
-      ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
-    } else if (type == Type::TIMESTAMP) {
-      auto ts_type = static_cast<arrow::TimestampType*>(col->type().get());
+    npy_intp dims[1] = {col_->length()};
+    result_ = PyArray_SimpleNew(1, dims, type);
+    arr_ = reinterpret_cast<PyArrayObject*>(result_);
 
-      if (ts_type->unit == arrow::TimeUnit::NANO) {
-        ConvertNumericNullable<int64_t>(data, kPandasTimestampNull, out_buffer);
-      } else if (ts_type->unit == arrow::TimeUnit::MICRO) {
-        ConvertDatetimeNanos<int64_t, 1000L>(data, out_buffer);
-      } else if (ts_type->unit == arrow::TimeUnit::MILLI) {
-        ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
-      } else if (ts_type->unit == arrow::TimeUnit::SECOND) {
-        ConvertDatetimeNanos<int64_t, 1000000000L>(data, out_buffer);
-      } else {
-        return Status::NotImplemented("Unsupported time unit");
-      }
-    } else {
-      return Status::NotImplemented(col->type()->ToString());
+    if (arr_ == NULL) {
+      // Error occurred, trust that SimpleNew set the error state
+      return Status::OK();
     }
 
-    placement_data_[rel_placement] = abs_placement;
+    set_numpy_metadata(type, col_->type().get(), arr_);
+
     return Status::OK();
   }
-};
 
-// class CategoricalBlock : public PandasBlock {};
+  template <int TYPE>
+  Status ConvertValuesZeroCopy(int npy_type, std::shared_ptr<Array> arr) {
+    typedef typename arrow_traits<TYPE>::T T;
 
-Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
-    std::shared_ptr<PandasBlock>* block) {
-#define BLOCK_CASE(NAME, TYPE)                              \
-  case PandasBlock::NAME:                                   \
-    *block = std::make_shared<TYPE>(num_rows, num_columns); \
-    break;
+    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
 
-  switch (type) {
-    BLOCK_CASE(OBJECT, ObjectBlock);
-    BLOCK_CASE(UINT8, UInt8Block);
-    BLOCK_CASE(INT8, Int8Block);
-    BLOCK_CASE(UINT16, UInt16Block);
-    BLOCK_CASE(INT16, Int16Block);
-    BLOCK_CASE(UINT32, UInt32Block);
-    BLOCK_CASE(INT32, Int32Block);
-    BLOCK_CASE(UINT64, UInt64Block);
-    BLOCK_CASE(INT64, Int64Block);
-    BLOCK_CASE(FLOAT, Float32Block);
-    BLOCK_CASE(DOUBLE, Float64Block);
-    BLOCK_CASE(BOOL, BoolBlock);
-    BLOCK_CASE(DATETIME, DatetimeBlock);
-    case PandasBlock::CATEGORICAL:
-      return Status::NotImplemented("categorical");
-  }
+    // Zero-Copy. We can pass the data pointer directly to NumPy.
+    void* data = const_cast<T*>(in_values);
 
-#undef BLOCK_CASE
+    PyAcquireGIL lock;
 
-  return (*block)->Allocate();
-}
+    // Zero-Copy. We can pass the data pointer directly to NumPy.
+    npy_intp dims[1] = {col_->length()};
+    result_ = PyArray_SimpleNewFromData(1, dims, npy_type, data);
+    arr_ = reinterpret_cast<PyArrayObject*>(result_);
 
-// Construct the exact pandas 0.x "BlockManager" memory layout
-//
-// * For each column determine the correct output pandas type
-// * Allocate 2D blocks (ncols x nrows) for each distinct data type in output
-// * Allocate  block placement arrays
-// * Write Arrow columns out into each slice of memory; populate block
-// * placement arrays as we go
-class DataFrameBlockCreator {
- public:
-  DataFrameBlockCreator(const std::shared_ptr<Table>& table) : table_(table) {}
+    if (arr_ == NULL) {
+      // Error occurred, trust that SimpleNew set the error state
+      return Status::OK();
+    }
+
+    set_numpy_metadata(npy_type, col_->type().get(), arr_);
 
-  Status Convert(int nthreads, PyObject** output) {
-    column_types_.resize(table_->num_columns());
-    column_block_placement_.resize(table_->num_columns());
-    type_counts_.clear();
-    blocks_.clear();
+    if (PyArray_SetBaseObject(arr_, py_ref_) == -1) {
+      // Error occurred, trust that SetBaseObject set the error state
+      return Status::OK();
+    } else {
+      // PyArray_SetBaseObject steals our reference to py_ref_
+      Py_INCREF(py_ref_);
+    }
 
-    RETURN_NOT_OK(CountColumnTypes());
-    RETURN_NOT_OK(CreateBlocks());
-    RETURN_NOT_OK(WriteTableToBlocks(nthreads));
+    // Arrow data is immutable.
+    PyArray_CLEARFLAGS(arr_, NPY_ARRAY_WRITEABLE);
 
-    return GetResultList(output);
+    return Status::OK();
   }
 
-  Status CountColumnTypes() {
-    for (int i = 0; i < table_->num_columns(); ++i) {
-      std::shared_ptr<Column> col = table_->column(i);
-      PandasBlock::type output_type;
+  // ----------------------------------------------------------------------
+  // Allocate new array and deserialize. Can do a zero copy conversion for some
+  // types
 
-      switch (col->type()->type) {
-        case Type::BOOL:
-          output_type = col->null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL;
-          break;
-        case Type::UINT8:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT8;
-          break;
-        case Type::INT8:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT8;
-          break;
-        case Type::UINT16:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT16;
-          break;
-        case Type::INT16:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT16;
-          break;
-        case Type::UINT32:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT32;
-          break;
-        case Type::INT32:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT32;
-          break;
-        case Type::INT64:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT64;
-          break;
-        case Type::UINT64:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT64;
-          break;
-        case Type::FLOAT:
-          output_type = PandasBlock::FLOAT;
-          break;
-        case Type::DOUBLE:
-          output_type = PandasBlock::DOUBLE;
-          break;
-        case Type::STRING:
-        case Type::BINARY:
-          output_type = PandasBlock::OBJECT;
-          break;
-        case Type::DATE:
-          output_type = PandasBlock::DATETIME;
-          break;
-        case Type::TIMESTAMP:
-          output_type = PandasBlock::DATETIME;
-          break;
-        case Type::LIST: {
-          auto list_type = std::static_pointer_cast<ListType>(col->type());
-          switch (list_type->value_type()->type) {
-            case Type::UINT8:
-            case Type::INT8:
-            case Type::UINT16:
-            case Type::INT16:
-            case Type::UINT32:
-            case Type::INT32:
-            case Type::INT64:
-            case Type::UINT64:
-            case Type::FLOAT:
-            case Type::DOUBLE:
-            case Type::STRING:
-            case Type::TIMESTAMP:
-              // The above types are all supported.
-              break;
-            default: {
-              std::stringstream ss;
-              ss << "Not implemented type for lists: "
-                 << list_type->value_type()->ToString();
-              return Status::NotImplemented(ss.str());
-            }
-          }
-          output_type = PandasBlock::OBJECT;
-        } break;
-        default:
-          return Status::NotImplemented(col->type()->ToString());
-      }
+  Status Convert(PyObject** out) {
+#define CONVERT_CASE(TYPE)                             \
+  case Type::TYPE: {                            \
+    RETURN_NOT_OK(ConvertValues<Type::TYPE>()); \
+  } break;
 
-      auto it = type_counts_.find(output_type);
-      int block_placement = 0;
-      if (it != type_counts_.end()) {
-        block_placement = it->second;
-        // Increment count
-        it->second += 1;
-      } else {
-        // Add key to map
-        type_counts_[output_type] = 1;
+    switch (col_->type()->type) {
+      CONVERT_CASE(BOOL);
+      CONVERT_CASE(INT8);
+      CONVERT_CASE(INT16);
+      CONVERT_CASE(INT32);
+      CONVERT_CASE(INT64);
+      CONVERT_CASE(UINT8);
+      CONVERT_CASE(UINT16);
+      CONVERT_CASE(UINT32);
+      CONVERT_CASE(UINT64);
+      CONVERT_CASE(FLOAT);
+      CONVERT_CASE(DOUBLE);
+      CONVERT_CASE(BINARY);
+      CONVERT_CASE(STRING);
+      CONVERT_CASE(DATE);
+      CONVERT_CASE(TIMESTAMP);
+      CONVERT_CASE(DICTIONARY);
+      default: {
+        std::stringstream ss;
+        ss << "Arrow type reading not implemented for " << col_->type()->ToString();
+        return Status::NotImplemented(ss.str());
       }
-
-      column_types_[i] = output_type;
-      column_block_placement_[i] = block_placement;
     }
+
+#undef CONVERT_CASE
+
+    *out = result_;
     return Status::OK();
   }
 
-  Status CreateBlocks() {
-    for (const auto& it : type_counts_) {
-      PandasBlock::type type = static_cast<PandasBlock::type>(it.first);
-      std::shared_ptr<PandasBlock> block;
-      RETURN_NOT_OK(MakeBlock(type, table_->num_rows(), it.second, &block));
-      blocks_[type] = block;
+  template <int TYPE>
+  inline typename std::enable_if<
+      (TYPE != Type::DATE) & arrow_traits<TYPE>::is_numeric_nullable, Status>::type
+  ConvertValues() {
+    typedef typename arrow_traits<TYPE>::T T;
+    int npy_type = arrow_traits<TYPE>::npy_type;
+
+    if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
+      return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
     }
+
+    RETURN_NOT_OK(AllocateOutput(npy_type));
+    auto out_values = reinterpret_cast<T*>(PyArray_DATA(arr_));
+    ConvertNumericNullable<T>(data_, arrow_traits<TYPE>::na_value, out_values);
+
     return Status::OK();
   }
 
-  Status WriteTableToBlocks(int nthreads) {
-    auto WriteColumn = [this](int i) {
-      std::shared_ptr<Column> col = this->table_->column(i);
-      PandasBlock::type output_type = this->column_types_[i];
+  template <int TYPE>
+  inline typename std::enable_if<TYPE == Type::DATE, Status>::type
+  ConvertValues() {
+    typedef typename arrow_traits<TYPE>::T T;
 
-      int rel_placement = this->column_block_placement_[i];
+    RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+    auto out_values = reinterpret_cast<T*>(PyArray_DATA(arr_));
+    ConvertDates<T>(data_, arrow_traits<TYPE>::na_value, out_values);
+    return Status::OK();
+  }
 
-      auto it = this->blocks_.find(output_type);
-      if (it == this->blocks_.end()) { return Status::KeyError("No block allocated"); }
-      return it->second->Write(col, i, rel_placement);
-    };
+  // Integer specialization
+  template <int TYPE>
+  inline
+      typename std::enable_if<arrow_traits<TYPE>::is_numeric_not_nullable, Status>::type
+      ConvertValues() {
+    typedef typename arrow_traits<TYPE>::T T;
+    int npy_type = arrow_traits<TYPE>::npy_type;
 
-    nthreads = std::min<int>(nthreads, table_->num_columns());
+    if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
+      return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
+    }
 
-    if (nthreads == 1) {
-      for (int i = 0; i < table_->num_columns(); ++i) {
-        RETURN_NOT_OK(WriteColumn(i));
-      }
+    if (data_.null_count() > 0) {
+      RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
+      auto out_values = reinterpret_cast<double*>(PyArray_DATA(arr_));
+      ConvertIntegerWithNulls<T>(data_, out_values);
     } else {
-      std::vector<std::thread> thread_pool;
-      thread_pool.reserve(nthreads);
-      std::atomic<int> task_counter(0);
-
-      std::mutex error_mtx;
-      bool error_occurred = false;
-      Status error;
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+      auto out_values = reinterpret_cast<T*>(PyArray_DATA(arr_));
+      ConvertIntegerNoNullsSameType<T>(data_, out_values);
+    }
 
-      for (int thread_id = 0; thread_id < nthreads; ++thread_id) {
-        thread_pool.emplace_back(
-            [this, &error, &error_occurred, &error_mtx, &task_counter, &WriteColumn]() {
-              int column_num;
-              while (!error_occurred) {
-                column_num = task_counter.fetch_add(1);
-                if (column_num >= this->table_->num_columns()) { break; }
-                Status s = WriteColumn(column_num);
-                if (!s.ok()) {
-                  std::lock_guard<std::mutex> lock(error_mtx);
-                  error_occurred = true;
-                  error = s;
-                  break;
-                }
-              }
-            });
-      }
-      for (auto&& thread : thread_pool) {
-        thread.join();
-      }
+    return Status::OK();
+  }
 
-      if (error_occurred) { return error; }
+  // Boolean specialization
+  template <int TYPE>
+  inline typename std::enable_if<arrow_traits<TYPE>::is_boolean, Status>::type
+  ConvertValues() {
+    if (data_.null_count() > 0) {
+      RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+      auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+      RETURN_NOT_OK(ConvertBooleanWithNulls(data_, out_values));
+    } else {
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+      auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(arr_));
+      ConvertBooleanNoNulls(data_, out_values);
     }
     return Status::OK();
   }
 
-  Status GetResultList(PyObject** out) {
-    PyAcquireGIL lock;
+  // UTF8 strings
+  template <int TYPE>
+  inline typename std::enable_if<TYPE == Type::STRING, Status>::type
+  ConvertValues() {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    return ConvertBinaryLike<arrow::StringArray>(data_, out_values);
+  }
 
-    auto num_blocks = static_cast<Py_ssize_t>(blocks_.size());
-    PyObject* result = PyList_New(num_blocks);
-    RETURN_IF_PYERROR();
+  template <int T2>
+  inline typename std::enable_if<T2 == Type::BINARY, Status>::type
+  ConvertValues() {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    return ConvertBinaryLike<arrow::BinaryArray>(data_, out_values);
+  }
 
-    int i = 0;
-    for (const auto& it : blocks_) {
-      const std::shared_ptr<PandasBlock> block = it.second;
+  template <int TYPE>
+  inline typename std::enable_if<TYPE == Type::DICTIONARY, Status>::type
+  ConvertValues() {
+    std::shared_ptr<PandasBlock> block;
+    RETURN_NOT_OK(MakeCategoricalBlock(col_->type(), col_->length(), &block));
+    RETURN_NOT_OK(block->Write(col_, 0, 0));
 
-      PyObject* item = PyTuple_New(2);
-      RETURN_IF_PYERROR();
+    auto dict_type = static_cast<const DictionaryType*>(col_->type().get());
 
-      PyObject* block_arr = block->block_arr();
-      PyObject* placement_arr = block->placement_arr();
-      Py_INCREF(block_arr);
-      Py_INCREF(placement_arr);
-      PyTuple_SET_ITEM(item, 0, block_arr);
-      PyTuple_SET_ITEM(item, 1, placement_arr);
+    PyAcquireGIL lock;
+    result_ = PyDict_New();
+    RETURN_IF_PYERROR();
+
+    PyObject* dictionary;
+    RETURN_NOT_OK(ConvertArrayToPandas(dict_type->dictionary(), nullptr, &dictionary));
+
+    PyDict_SetItemString(result_, "indices", block->block_arr());
+    PyDict_SetItemString(result_, "dictionary", dictionary);
 
-      if (PyList_SET_ITEM(result, i++, item) < 0) { RETURN_IF_PYERROR(); }
-    }
-    *out = result;
     return Status::OK();
   }
 
  private:
-  std::shared_ptr<Table> table_;
-
-  // column num -> block type id
-  std::vector<PandasBlock::type> column_types_;
-
-  // column num -> relative placement within internal block
-  std::vector<int> column_block_placement_;
-
-  // block type -> type count
-  std::unordered_map<int, int> type_counts_;
-
-  // block type -> block
-  std::unordered_map<int, std::shared_ptr<PandasBlock>> blocks_;
+  std::shared_ptr<Column> col_;
+  const arrow::ChunkedArray& data_;
+  PyObject* py_ref_;
+  PyArrayObject* arr_;
+  PyObject* result_;
 };
 
-Status ConvertTableToPandas(
-    const std::shared_ptr<Table>& table, int nthreads, PyObject** out) {
-  DataFrameBlockCreator helper(table);
-  return helper.Convert(nthreads, out);
+Status ConvertArrayToPandas(
+    const std::shared_ptr<Array>& arr, PyObject* py_ref, PyObject** out) {
+  static std::string dummy_name = "dummy";
+  auto field = std::make_shared<Field>(dummy_name, arr->type());
+  auto col = std::make_shared<Column>(field, arr);
+  return ConvertColumnToPandas(col, py_ref, out);
+}
+
+Status ConvertColumnToPandas(
+    const std::shared_ptr<Column>& col, PyObject* py_ref, PyObject** out) {
+  ArrowDeserializer converter(col, py_ref);
+  return converter.Convert(out);
 }
 
 }  // namespace pyarrow
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index 664365e398384..b548f9321d75a 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -63,11 +63,7 @@ arrow::Status ConvertTableToPandas(
     const std::shared_ptr<arrow::Table>& table, int nthreads, PyObject** out);
 
 PYARROW_EXPORT
-arrow::Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<arrow::Field>& field, std::shared_ptr<arrow::Array>* out);
-
-PYARROW_EXPORT
-arrow::Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao,
+arrow::Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     const std::shared_ptr<arrow::Field>& field, std::shared_ptr<arrow::Array>* out);
 
 }  // namespace pyarrow
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index 0bdd289953dc4..b8712d7d0a4fc 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -93,7 +93,7 @@ PyBytesBuffer::PyBytesBuffer(PyObject* obj)
 }
 
 PyBytesBuffer::~PyBytesBuffer() {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
   Py_DECREF(obj_);
 }
 
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 639918d309fe7..0733a3b7cf061 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -30,6 +30,17 @@ class MemoryPool;
 
 namespace pyarrow {
 
+class PyAcquireGIL {
+ public:
+  PyAcquireGIL() { state_ = PyGILState_Ensure(); }
+
+  ~PyAcquireGIL() { PyGILState_Release(state_); }
+
+ private:
+  PyGILState_STATE state_;
+  DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL);
+};
+
 #define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
 
 class OwnedRef {
@@ -38,7 +49,10 @@ class OwnedRef {
 
   OwnedRef(PyObject* obj) : obj_(obj) {}
 
-  ~OwnedRef() { Py_XDECREF(obj_); }
+  ~OwnedRef() {
+    PyAcquireGIL lock;
+    Py_XDECREF(obj_);
+  }
 
   void reset(PyObject* obj) {
     if (obj_ != nullptr) { Py_XDECREF(obj_); }
@@ -69,17 +83,6 @@ struct PyObjectStringify {
   }
 };
 
-class PyGILGuard {
- public:
-  PyGILGuard() { state_ = PyGILState_Ensure(); }
-
-  ~PyGILGuard() { PyGILState_Release(state_); }
-
- private:
-  PyGILState_STATE state_;
-  DISALLOW_COPY_AND_ASSIGN(PyGILGuard);
-};
-
 // TODO(wesm): We can just let errors pass through. To be explored later
 #define RETURN_IF_PYERROR()                         \
   if (PyErr_Occurred()) {                           \
@@ -88,8 +91,9 @@ class PyGILGuard {
     PyObjectStringify stringified(exc_value);       \
     std::string message(stringified.bytes);         \
     Py_DECREF(exc_type);                            \
-    Py_DECREF(exc_value);                           \
-    Py_DECREF(traceback);                           \
+    Py_XDECREF(exc_value);                          \
+    Py_XDECREF(traceback);                          \
+    PyErr_Clear();                                  \
     return Status::UnknownError(message);           \
   }
 
@@ -122,17 +126,6 @@ class PYARROW_EXPORT PyBytesBuffer : public arrow::Buffer {
   PyObject* obj_;
 };
 
-class PyAcquireGIL {
- public:
-  PyAcquireGIL() { state_ = PyGILState_Ensure(); }
-
-  ~PyAcquireGIL() { PyGILState_Release(state_); }
-
- private:
-  PyGILState_STATE state_;
-  DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL);
-};
-
 }  // namespace pyarrow
 
 #endif  // PYARROW_COMMON_H
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index 01f851d874075..92352607e62ec 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -114,22 +114,22 @@ PyReadableFile::PyReadableFile(PyObject* file) {
 PyReadableFile::~PyReadableFile() {}
 
 Status PyReadableFile::Close() {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
   return file_->Close();
 }
 
 Status PyReadableFile::Seek(int64_t position) {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
   return file_->Seek(position, 0);
 }
 
 Status PyReadableFile::Tell(int64_t* position) {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
   return file_->Tell(position);
 }
 
 Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
   PyObject* bytes_obj;
   ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
 
@@ -141,7 +141,7 @@ Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
 }
 
 Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
 
   PyObject* bytes_obj;
   ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
@@ -153,7 +153,7 @@ Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out)
 }
 
 Status PyReadableFile::GetSize(int64_t* size) {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
 
   int64_t current_position;
   ;
@@ -185,17 +185,17 @@ PyOutputStream::PyOutputStream(PyObject* file) {
 PyOutputStream::~PyOutputStream() {}
 
 Status PyOutputStream::Close() {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
   return file_->Close();
 }
 
 Status PyOutputStream::Tell(int64_t* position) {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
   return file_->Tell(position);
 }
 
 Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
-  PyGILGuard lock;
+  PyAcquireGIL lock;
   return file_->Write(data, nbytes);
 }
 
diff --git a/python/src/pyarrow/util/CMakeLists.txt b/python/src/pyarrow/util/CMakeLists.txt
index 4afb4d0f912b1..6cd49cb75a4fb 100644
--- a/python/src/pyarrow/util/CMakeLists.txt
+++ b/python/src/pyarrow/util/CMakeLists.txt
@@ -20,7 +20,7 @@
 #######################################
 
 if (PYARROW_BUILD_TESTS)
-  add_library(pyarrow_test_main
+  add_library(pyarrow_test_main STATIC
 	test_main.cc)
 
   if (APPLE)
diff --git a/python/src/pyarrow/util/test_main.cc b/python/src/pyarrow/util/test_main.cc
index 6fb7c0536eed3..02e9a54f65914 100644
--- a/python/src/pyarrow/util/test_main.cc
+++ b/python/src/pyarrow/util/test_main.cc
@@ -15,12 +15,22 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <Python.h>
+
 #include <gtest/gtest.h>
 
+#include "pyarrow/do_import_numpy.h"
+#include "pyarrow/numpy_interop.h"
+
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
 
+  Py_Initialize();
+  pyarrow::import_numpy();
+
   int ret = RUN_ALL_TESTS();
 
+  Py_Finalize();
+
   return ret;
 }

From 6811d3fcfc9da65e24b6d0f2ad5d5d348d879f11 Mon Sep 17 00:00:00 2001
From: Nong Li <nong@cerebrodata.com>
Date: Thu, 19 Jan 2017 14:45:24 -0500
Subject: [PATCH 0282/1644] ARROW-474: [Java] Add initial version of streaming
 serialized format.

This patch proposes a serialized container format for streaming producer and
consumers. The goal is to allow readers and writers to produce/consume arrow
data without requiring intermediate buffering.

This is similar to the File format but reorganizes the pieces. In particular:
  - No magic header. It's likely a reader connects to a 'random' stream to read it.
  - Move footer to header. This includes similar information, including the schema.
  - ArrowRecordBatches follow one by one. Each is prefixed with an i32 length. The
    serialization is identical as the File version.
  - See Stream.fbs for more details.

This patch also implements the Java reader/writer.

Author: Nong Li <nong@cerebrodata.com>

Closes #288 from nongli/streaming and squashes the following commits:

554cc18 [Nong Li] Redo serialization format.
03bee58 [Nong Li] Updates from wes' comments.
7257031 [Nong Li] ARROW-474: [Java] Add initial version of streaming serialized format.
---
 .../apache/arrow/vector/file/ArrowReader.java |  12 +-
 .../apache/arrow/vector/file/ArrowWriter.java | 102 ++-------
 .../apache/arrow/vector/file/ReadChannel.java |  75 ++++++
 .../arrow/vector/file/WriteChannel.java       | 111 +++++++++
 .../arrow/vector/schema/ArrowRecordBatch.java |  23 ++
 .../vector/stream/ArrowStreamReader.java      |  95 ++++++++
 .../vector/stream/ArrowStreamWriter.java      |  71 ++++++
 .../vector/stream/MessageSerializer.java      | 216 ++++++++++++++++++
 .../arrow/vector/types/pojo/Schema.java       |   5 +
 .../arrow/vector/file/TestArrowFile.java      | 149 +++++++++++-
 .../vector/stream/MessageSerializerTest.java  | 115 ++++++++++
 .../arrow/vector/stream/TestArrowStream.java  |  96 ++++++++
 .../vector/stream/TestArrowStreamPipe.java    | 129 +++++++++++
 13 files changed, 1100 insertions(+), 99 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index cd520da54f2f5..58c51605c5600 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -31,6 +31,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.MessageSerializer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,7 +40,7 @@
 public class ArrowReader implements AutoCloseable {
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowReader.class);
 
-  private static final byte[] MAGIC = "ARROW1".getBytes();
+  public static final byte[] MAGIC = "ARROW1".getBytes();
 
   private final SeekableByteChannel in;
 
@@ -73,13 +74,6 @@ private int readFully(ByteBuffer buffer) throws IOException {
     return total;
   }
 
-  private static int bytesToInt(byte[] bytes) {
-    return ((int)(bytes[3] & 255) << 24) +
-           ((int)(bytes[2] & 255) << 16) +
-           ((int)(bytes[1] & 255) <<  8) +
-           ((int)(bytes[0] & 255) <<  0);
-  }
-
   public ArrowFooter readFooter() throws IOException {
     if (footer == null) {
       if (in.size() <= (MAGIC.length * 2 + 4)) {
@@ -93,7 +87,7 @@ public ArrowFooter readFooter() throws IOException {
       if (!Arrays.equals(MAGIC, Arrays.copyOfRange(array, 4, array.length))) {
         throw new InvalidArrowFileException("missing Magic number " + Arrays.toString(buffer.array()));
       }
-      int footerLength = bytesToInt(array);
+      int footerLength = MessageSerializer.bytesToInt(array);
       if (footerLength <= 0 || footerLength + MAGIC.length * 2 + 4 > in.size()) {
         throw new InvalidArrowFileException("invalid footer length: " + footerLength);
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 1cd87ebc33594..3febd11f4c76a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -18,7 +18,6 @@
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -26,32 +25,25 @@
 
 import org.apache.arrow.vector.schema.ArrowBuffer;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.schema.FBSerializable;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.flatbuffers.FlatBufferBuilder;
-
 import io.netty.buffer.ArrowBuf;
 
 public class ArrowWriter implements AutoCloseable {
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowWriter.class);
 
-  private static final byte[] MAGIC = "ARROW1".getBytes();
-
-  private final WritableByteChannel out;
+  private final WriteChannel out;
 
   private final Schema schema;
 
   private final List<ArrowBlock> recordBatches = new ArrayList<>();
 
-  private long currentPosition = 0;
-
   private boolean started = false;
 
   public ArrowWriter(WritableByteChannel out, Schema schema) {
-    this.out = out;
+    this.out = new WriteChannel(out);
     this.schema = schema;
   }
 
@@ -59,53 +51,19 @@ private void start() throws IOException {
     writeMagic();
   }
 
-  private long write(byte[] buffer) throws IOException {
-    return write(ByteBuffer.wrap(buffer));
-  }
-
-  private long writeZeros(int zeroCount) throws IOException {
-    return write(new byte[zeroCount]);
-  }
-
-  private long align() throws IOException {
-    if (currentPosition % 8 != 0) { // align on 8 byte boundaries
-      return writeZeros(8 - (int)(currentPosition % 8));
-    }
-    return 0;
-  }
-
-  private long write(ByteBuffer buffer) throws IOException {
-    long length = buffer.remaining();
-    out.write(buffer);
-    currentPosition += length;
-    return length;
-  }
-
-  private static byte[] intToBytes(int value) {
-    byte[] outBuffer = new byte[4];
-    outBuffer[3] = (byte)(value >>> 24);
-    outBuffer[2] = (byte)(value >>> 16);
-    outBuffer[1] = (byte)(value >>>  8);
-    outBuffer[0] = (byte)(value >>>  0);
-    return outBuffer;
-  }
-
-  private long writeIntLittleEndian(int v) throws IOException {
-    return write(intToBytes(v));
-  }
 
   // TODO: write dictionaries
 
   public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
     checkStarted();
-    align();
+    out.align();
 
     // write metadata header with int32 size prefix
-    long offset = currentPosition;
-    write(recordBatch, true);
-    align();
+    long offset = out.getCurrentPosition();
+    out.write(recordBatch, true);
+    out.align();
     // write body
-    long bodyOffset = currentPosition;
+    long bodyOffset = out.getCurrentPosition();
     List<ArrowBuf> buffers = recordBatch.getBuffers();
     List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
     if (buffers.size() != buffersLayout.size()) {
@@ -115,31 +73,25 @@ public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
       ArrowBuf buffer = buffers.get(i);
       ArrowBuffer layout = buffersLayout.get(i);
       long startPosition = bodyOffset + layout.getOffset();
-      if (startPosition != currentPosition) {
-        writeZeros((int)(startPosition - currentPosition));
+      if (startPosition != out.getCurrentPosition()) {
+        out.writeZeros((int)(startPosition - out.getCurrentPosition()));
       }
 
-      write(buffer);
-      if (currentPosition != startPosition + layout.getSize()) {
-        throw new IllegalStateException("wrong buffer size: " + currentPosition + " != " + startPosition + layout.getSize());
+      out.write(buffer);
+      if (out.getCurrentPosition() != startPosition + layout.getSize()) {
+        throw new IllegalStateException("wrong buffer size: " + out.getCurrentPosition() + " != " + startPosition + layout.getSize());
       }
     }
     int metadataLength = (int)(bodyOffset - offset);
     if (metadataLength <= 0) {
       throw new InvalidArrowFileException("invalid recordBatch");
     }
-    long bodyLength = currentPosition - bodyOffset;
+    long bodyLength = out.getCurrentPosition() - bodyOffset;
     LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d", offset, metadataLength, bodyLength));
     // add metadata to footer
     recordBatches.add(new ArrowBlock(offset, metadataLength, bodyLength));
   }
 
-  private void write(ArrowBuf buffer) throws IOException {
-    ByteBuffer nioBuffer = buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes());
-    LOGGER.debug("Writing buffer with size: " + nioBuffer.remaining());
-    write(nioBuffer);
-  }
-
   private void checkStarted() throws IOException {
     if (!started) {
       started = true;
@@ -147,15 +99,16 @@ private void checkStarted() throws IOException {
     }
   }
 
+  @Override
   public void close() throws IOException {
     try {
-      long footerStart = currentPosition;
+      long footerStart = out.getCurrentPosition();
       writeFooter();
-      int footerLength = (int)(currentPosition - footerStart);
+      int footerLength = (int)(out.getCurrentPosition() - footerStart);
       if (footerLength <= 0 ) {
         throw new InvalidArrowFileException("invalid footer");
       }
-      writeIntLittleEndian(footerLength);
+      out.writeIntLittleEndian(footerLength);
       LOGGER.debug(String.format("Footer starts at %d, length: %d", footerStart, footerLength));
       writeMagic();
     } finally {
@@ -164,27 +117,12 @@ public void close() throws IOException {
   }
 
   private void writeMagic() throws IOException {
-    write(MAGIC);
-    LOGGER.debug(String.format("magic written, now at %d", currentPosition));
+    out.write(ArrowReader.MAGIC);
+    LOGGER.debug(String.format("magic written, now at %d", out.getCurrentPosition()));
   }
 
   private void writeFooter() throws IOException {
     // TODO: dictionaries
-    write(new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), recordBatches), false);
-  }
-
-  private long write(FBSerializable writer, boolean withSizePrefix) throws IOException {
-    FlatBufferBuilder builder = new FlatBufferBuilder();
-    int root = writer.writeTo(builder);
-    builder.finish(root);
-
-    ByteBuffer buffer = builder.dataBuffer();
-
-    if (withSizePrefix) {
-      writeIntLittleEndian(buffer.remaining());
-    }
-
-    return write(buffer);
+    out.write(new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), recordBatches), false);
   }
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
new file mode 100644
index 0000000000000..b062f3826eab3
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ArrowBuf;
+
+public class ReadChannel implements AutoCloseable {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(ReadChannel.class);
+
+  private ReadableByteChannel in;
+  private long bytesRead = 0;
+
+  public ReadChannel(ReadableByteChannel in) {
+    this.in = in;
+  }
+
+  public long bytesRead() { return bytesRead; }
+
+  /**
+   * Reads bytes into buffer until it is full (buffer.remaining() == 0). Returns the
+   * number of bytes read which can be less than full if there are no more.
+   */
+  public int readFully(ByteBuffer buffer) throws IOException {
+    LOGGER.debug("Reading buffer with size: " + buffer.remaining());
+    int totalRead = 0;
+    while (buffer.remaining() != 0) {
+      int read = in.read(buffer);
+      if (read < 0) return totalRead;
+      totalRead += read;
+      if (read == 0) break;
+    }
+    this.bytesRead += totalRead;
+    return totalRead;
+  }
+
+  /**
+   * Reads up to len into buffer. Returns bytes read.
+   */
+  public int readFully(ArrowBuf buffer, int l) throws IOException {
+    int n = readFully(buffer.nioBuffer(buffer.writerIndex(), l));
+    buffer.writerIndex(n);
+    return n;
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (this.in != null) {
+      in.close();
+      in = null;
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
new file mode 100644
index 0000000000000..d99c9a6c99958
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+
+import org.apache.arrow.vector.schema.FBSerializable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Wrapper around a WritableByteChannel that maintains the position as well adding
+ * some common serialization utilities.
+ */
+public class WriteChannel implements AutoCloseable {
+  private static final Logger LOGGER = LoggerFactory.getLogger(WriteChannel.class);
+
+  private long currentPosition = 0;
+
+  private final WritableByteChannel out;
+
+  public WriteChannel(WritableByteChannel out) {
+    this.out = out;
+  }
+
+  @Override
+  public void close() throws IOException {
+    out.close();
+  }
+
+  public long getCurrentPosition() {
+    return currentPosition;
+  }
+
+  public long write(byte[] buffer) throws IOException {
+    return write(ByteBuffer.wrap(buffer));
+  }
+
+  public long writeZeros(int zeroCount) throws IOException {
+    return write(new byte[zeroCount]);
+  }
+
+  public long align() throws IOException {
+    if (currentPosition % 8 != 0) { // align on 8 byte boundaries
+      return writeZeros(8 - (int)(currentPosition % 8));
+    }
+    return 0;
+  }
+
+  public long write(ByteBuffer buffer) throws IOException {
+    long length = buffer.remaining();
+    LOGGER.debug("Writing buffer with size: " + length);
+    out.write(buffer);
+    currentPosition += length;
+    return length;
+  }
+
+  public static byte[] intToBytes(int value) {
+    byte[] outBuffer = new byte[4];
+    outBuffer[3] = (byte)(value >>> 24);
+    outBuffer[2] = (byte)(value >>> 16);
+    outBuffer[1] = (byte)(value >>>  8);
+    outBuffer[0] = (byte)(value >>>  0);
+    return outBuffer;
+  }
+
+  public long writeIntLittleEndian(int v) throws IOException {
+    return write(intToBytes(v));
+  }
+
+  public void write(ArrowBuf buffer) throws IOException {
+    ByteBuffer nioBuffer = buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes());
+    write(nioBuffer);
+  }
+
+  public long write(FBSerializable writer, boolean withSizePrefix) throws IOException {
+    ByteBuffer buffer = serialize(writer);
+    if (withSizePrefix) {
+      writeIntLittleEndian(buffer.remaining());
+    }
+    return write(buffer);
+  }
+
+  public static ByteBuffer serialize(FBSerializable writer) {
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    int root = writer.writeTo(builder);
+    builder.finish(root);
+    return builder.dataBuffer();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index adb99e2f3ffb7..40c2fbfd984f8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -19,6 +19,7 @@
 
 import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -130,6 +131,28 @@ public String toString() {
         + buffersLayout + ", closed=" + closed + "]";
   }
 
+  /**
+   * Computes the size of the serialized body for this recordBatch.
+   */
+  public int computeBodyLength() {
+    int size = 0;
+
+    List<ArrowBuf> buffers = getBuffers();
+    List<ArrowBuffer> buffersLayout = getBuffersLayout();
+    if (buffers.size() != buffersLayout.size()) {
+      throw new IllegalStateException("the layout does not match: " +
+          buffers.size() + " != " + buffersLayout.size());
+    }
 
+    for (int i = 0; i < buffers.size(); i++) {
+      ArrowBuf buffer = buffers.get(i);
+      ArrowBuffer layout = buffersLayout.get(i);
+      size += (layout.getOffset() - size);
+      ByteBuffer nioBuffer =
+          buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes());
+      size += nioBuffer.remaining();
+    }
+    return size;
+  }
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
new file mode 100644
index 0000000000000..f32966c5d5217
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.stream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.file.ReadChannel;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * This classes reads from an input stream and produces ArrowRecordBatches.
+ */
+public class ArrowStreamReader implements AutoCloseable {
+  private ReadChannel in;
+  private final BufferAllocator allocator;
+  private Schema schema;
+
+  /**
+   * Constructs a streaming read, reading bytes from 'in'. Non-blocking.
+   */
+  public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) {
+    super();
+    this.in = new ReadChannel(in);
+    this.allocator = allocator;
+  }
+
+  public ArrowStreamReader(InputStream in, BufferAllocator allocator) {
+    this(Channels.newChannel(in), allocator);
+  }
+
+  /**
+   * Initializes the reader. Must be called before the other APIs. This is blocking.
+   */
+  public void init() throws IOException {
+    Preconditions.checkState(this.schema == null, "Cannot call init() more than once.");
+    this.schema = readSchema();
+  }
+
+  /**
+   * Returns the schema for all records in this stream.
+   */
+  public Schema getSchema () {
+    Preconditions.checkState(this.schema != null, "Must call init() first.");
+    return schema;
+  }
+
+  public long bytesRead() { return in.bytesRead(); }
+
+  /**
+   * Reads and returns the next ArrowRecordBatch. Returns null if this is the end
+   * of stream.
+   */
+  public ArrowRecordBatch nextRecordBatch() throws IOException {
+    Preconditions.checkState(this.in != null, "Cannot call after close()");
+    Preconditions.checkState(this.schema != null, "Must call init() first.");
+    return MessageSerializer.deserializeRecordBatch(in, allocator);
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (this.in != null) {
+      in.close();
+      in = null;
+    }
+  }
+
+  /**
+   * Reads the schema message from the beginning of the stream.
+   */
+  private Schema readSchema() throws IOException {
+    return MessageSerializer.deserializeSchema(in);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
new file mode 100644
index 0000000000000..06acf9f2c140e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.stream;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+
+import org.apache.arrow.vector.file.WriteChannel;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+public class ArrowStreamWriter implements AutoCloseable {
+  private final WriteChannel out;
+  private final Schema schema;
+  private boolean headerSent = false;
+
+  /**
+   * Creates the stream writer. non-blocking.
+   * totalBatches can be set if the writer knows beforehand. Can be -1 if unknown.
+   */
+  public ArrowStreamWriter(WritableByteChannel out, Schema schema, int totalBatches) {
+    this.out = new WriteChannel(out);
+    this.schema = schema;
+  }
+
+  public ArrowStreamWriter(OutputStream out, Schema schema, int totalBatches)
+      throws IOException {
+    this(Channels.newChannel(out), schema, totalBatches);
+  }
+
+  public long bytesWritten() { return out.getCurrentPosition(); }
+
+  public void writeRecordBatch(ArrowRecordBatch batch) throws IOException {
+    // Send the header if we have not yet.
+    checkAndSendHeader();
+    MessageSerializer.serialize(out, batch);
+  }
+
+  @Override
+  public void close() throws IOException {
+    // The header might not have been sent if this is an empty stream. Send it even in
+    // this case so readers see a valid empty stream.
+    checkAndSendHeader();
+    out.close();
+  }
+
+  private void checkAndSendHeader() throws IOException {
+    if (!headerSent) {
+      MessageSerializer.serialize(out, schema);
+      headerSent = true;
+    }
+  }
+}
+
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
new file mode 100644
index 0000000000000..22c46e2817b1e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.stream;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.arrow.flatbuf.Buffer;
+import org.apache.arrow.flatbuf.FieldNode;
+import org.apache.arrow.flatbuf.Message;
+import org.apache.arrow.flatbuf.MessageHeader;
+import org.apache.arrow.flatbuf.MetadataVersion;
+import org.apache.arrow.flatbuf.RecordBatch;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.file.ReadChannel;
+import org.apache.arrow.vector.file.WriteChannel;
+import org.apache.arrow.vector.schema.ArrowBuffer;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import io.netty.buffer.ArrowBuf;
+
+/**
+ * Utility class for serializing Messages. Messages are all serialized a similar way.
+ * 1. 4 byte little endian message header prefix
+ * 2. FB serialized Message: This includes it the body length, which is the serialized
+ *    body and the type of the message.
+ * 3. Serialized message.
+ *
+ * For schema messages, the serialization is simply the FB serialized Schema.
+ *
+ * For RecordBatch messages the serialization is:
+ *   1. 4 byte little endian batch metadata header
+ *   2. FB serialized RowBatch
+ *   3. serialized RowBatch buffers.
+ */
+public class MessageSerializer {
+
+  public static int bytesToInt(byte[] bytes) {
+    return ((bytes[3] & 255) << 24) +
+           ((bytes[2] & 255) << 16) +
+           ((bytes[1] & 255) <<  8) +
+           ((bytes[0] & 255) <<  0);
+  }
+
+  /**
+   * Serialize a schema object.
+   */
+  public static long serialize(WriteChannel out, Schema schema) throws IOException {
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    builder.finish(schema.getSchema(builder));
+    ByteBuffer serializedBody = builder.dataBuffer();
+    ByteBuffer serializedHeader =
+        serializeHeader(MessageHeader.Schema, serializedBody.remaining());
+
+    long size = out.writeIntLittleEndian(serializedHeader.remaining());
+    size += out.write(serializedHeader);
+    size += out.write(serializedBody);
+    return size;
+  }
+
+  /**
+   * Deserializes a schema object. Format is from serialize().
+   */
+  public static Schema deserializeSchema(ReadChannel in) throws IOException {
+    Message header = deserializeHeader(in, MessageHeader.Schema);
+    if (header == null) {
+      throw new IOException("Unexpected end of input. Missing schema.");
+    }
+
+    // Now read the schema.
+    ByteBuffer buffer = ByteBuffer.allocate((int)header.bodyLength());
+    if (in.readFully(buffer) != header.bodyLength()) {
+      throw new IOException("Unexpected end of input trying to read schema.");
+    }
+    buffer.rewind();
+    return Schema.deserialize(buffer);
+  }
+
+  /**
+   * Serializes an ArrowRecordBatch.
+   */
+  public static long serialize(WriteChannel out, ArrowRecordBatch batch)
+      throws IOException {
+    long start = out.getCurrentPosition();
+    int bodyLength = batch.computeBodyLength();
+
+    ByteBuffer metadata = WriteChannel.serialize(batch);
+    ByteBuffer serializedHeader =
+        serializeHeader(MessageHeader.RecordBatch, bodyLength + metadata.remaining() + 4);
+
+    // Write message header.
+    out.writeIntLittleEndian(serializedHeader.remaining());
+    out.write(serializedHeader);
+
+    // Write the metadata, with the 4 byte little endian prefix
+    out.writeIntLittleEndian(metadata.remaining());
+    out.write(metadata);
+
+    // Write batch header.
+    long offset = out.getCurrentPosition();
+    List<ArrowBuf> buffers = batch.getBuffers();
+    List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
+
+    for (int i = 0; i < buffers.size(); i++) {
+      ArrowBuf buffer = buffers.get(i);
+      ArrowBuffer layout = buffersLayout.get(i);
+      long startPosition = offset + layout.getOffset();
+      if (startPosition != out.getCurrentPosition()) {
+        out.writeZeros((int)(startPosition - out.getCurrentPosition()));
+      }
+      out.write(buffer);
+      if (out.getCurrentPosition() != startPosition + layout.getSize()) {
+        throw new IllegalStateException("wrong buffer size: " + out.getCurrentPosition() +
+            " != " + startPosition + layout.getSize());
+      }
+    }
+    return out.getCurrentPosition() - start;
+  }
+
+  /**
+   * Deserializes a RecordBatch
+   */
+  public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in,
+      BufferAllocator alloc) throws IOException {
+    Message header = deserializeHeader(in, MessageHeader.RecordBatch);
+    if (header == null) return null;
+
+    int messageLen = (int)header.bodyLength();
+    // Now read the buffer. This has the metadata followed by the data.
+    ArrowBuf buffer = alloc.buffer(messageLen);
+    if (in.readFully(buffer, messageLen) != messageLen) {
+      throw new IOException("Unexpected end of input trying to read batch.");
+    }
+
+    // Read the metadata. It starts with the 4 byte size of the metadata.
+    int metadataSize = buffer.readInt();
+    RecordBatch recordBatchFB =
+        RecordBatch.getRootAsRecordBatch( buffer.nioBuffer().asReadOnlyBuffer());
+
+    // No read the body
+    final ArrowBuf body = buffer.slice(4 + metadataSize, messageLen - metadataSize - 4);
+    int nodesLength = recordBatchFB.nodesLength();
+    List<ArrowFieldNode> nodes = new ArrayList<>();
+    for (int i = 0; i < nodesLength; ++i) {
+      FieldNode node = recordBatchFB.nodes(i);
+      nodes.add(new ArrowFieldNode(node.length(), node.nullCount()));
+    }
+    List<ArrowBuf> buffers = new ArrayList<>();
+    for (int i = 0; i < recordBatchFB.buffersLength(); ++i) {
+      Buffer bufferFB = recordBatchFB.buffers(i);
+      ArrowBuf vectorBuffer = body.slice((int)bufferFB.offset(), (int)bufferFB.length());
+      buffers.add(vectorBuffer);
+    }
+    ArrowRecordBatch arrowRecordBatch =
+        new ArrowRecordBatch(recordBatchFB.length(), nodes, buffers);
+    buffer.release();
+    return arrowRecordBatch;
+  }
+
+  /**
+   * Serializes a message header.
+   */
+  private static ByteBuffer serializeHeader(byte headerType, int bodyLength) {
+    FlatBufferBuilder headerBuilder = new FlatBufferBuilder();
+    Message.startMessage(headerBuilder);
+    Message.addHeaderType(headerBuilder, headerType);
+    Message.addVersion(headerBuilder, MetadataVersion.V1);
+    Message.addBodyLength(headerBuilder, bodyLength);
+    headerBuilder.finish(Message.endMessage(headerBuilder));
+    return headerBuilder.dataBuffer();
+  }
+
+  private static Message deserializeHeader(ReadChannel in, byte headerType) throws IOException {
+    // Read the header size. There is an i32 little endian prefix.
+    ByteBuffer buffer = ByteBuffer.allocate(4);
+    if (in.readFully(buffer) != 4) {
+      return null;
+    }
+
+    int headerLength = bytesToInt(buffer.array());
+    buffer = ByteBuffer.allocate(headerLength);
+    if (in.readFully(buffer) != headerLength) {
+      throw new IOException(
+          "Unexpected end of stream trying to read header.");
+    }
+    buffer.rewind();
+
+    Message header = Message.getRootAsMessage(buffer);
+    if (header.headerType() != headerType) {
+      throw new IOException("Invalid message: expecting " + headerType +
+          ". Message contained: " + header.headerType());
+    }
+    return header;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
index 5ca8ade7891ee..c33bd6e6e61b0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -22,6 +22,7 @@
 import static org.apache.arrow.vector.types.pojo.Field.convertField;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -65,6 +66,10 @@ public static Schema fromJSON(String json) throws IOException {
     return reader.readValue(checkNotNull(json));
   }
 
+  public static Schema deserialize(ByteBuffer buffer) {
+    return convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(buffer));
+  }
+
   public static Schema convertSchema(org.apache.arrow.flatbuf.Schema schema) {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     for (int i = 0; i < schema.fieldsLength(); i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 5fa18b3ca5339..bf635fb39f5b8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -18,12 +18,16 @@
 package org.apache.arrow.vector.file;
 
 import static org.apache.arrow.vector.TestVectorUnloadLoad.newVectorUnloader;
+import static org.junit.Assert.assertTrue;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -35,6 +39,8 @@
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.schema.ArrowBuffer;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.ArrowStreamReader;
+import org.apache.arrow.vector.stream.ArrowStreamWriter;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Assert;
 import org.junit.Test;
@@ -52,7 +58,7 @@ public void testWrite() throws IOException {
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", vectorAllocator, null)) {
       writeData(count, parent);
-      write(parent.getChild("root"), file);
+      write(parent.getChild("root"), file, new ByteArrayOutputStream());
     }
   }
 
@@ -66,13 +72,14 @@ public void testWriteComplex() throws IOException {
       writeComplexData(count, parent);
       FieldVector root = parent.getChild("root");
       validateComplexContent(count, new VectorSchemaRoot(root));
-      write(root, file);
+      write(root, file, new ByteArrayOutputStream());
     }
   }
 
   @Test
   public void testWriteRead() throws IOException {
     File file = new File("target/mytest.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
     int count = COUNT;
 
     // write
@@ -80,7 +87,7 @@ public void testWriteRead() throws IOException {
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
       writeData(count, parent);
-      write(parent.getChild("root"), file);
+      write(parent.getChild("root"), file, stream);
     }
 
     // read
@@ -116,11 +123,40 @@ public void testWriteRead() throws IOException {
         }
       }
     }
+
+    // Read from stream.
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)
+        ) {
+      arrowReader.init();
+      Schema schema = arrowReader.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+        while (true) {
+          try (ArrowRecordBatch recordBatch = arrowReader.nextRecordBatch()) {
+            if (recordBatch == null) break;
+            List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
+            for (ArrowBuffer arrowBuffer : buffersLayout) {
+              Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+            }
+            vectorLoader.load(recordBatch);
+          }
+        }
+        validateContent(count, root);
+      }
+    }
   }
 
   @Test
   public void testWriteReadComplex() throws IOException {
     File file = new File("target/mytest_complex.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
     int count = COUNT;
 
     // write
@@ -128,7 +164,7 @@ public void testWriteReadComplex() throws IOException {
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
       writeComplexData(count, parent);
-      write(parent.getChild("root"), file);
+      write(parent.getChild("root"), file, stream);
     }
 
     // read
@@ -156,11 +192,36 @@ public void testWriteReadComplex() throws IOException {
         }
       }
     }
+
+    // Read from stream.
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)
+        ) {
+      arrowReader.init();
+      Schema schema = arrowReader.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+        while (true) {
+          try (ArrowRecordBatch recordBatch = arrowReader.nextRecordBatch()) {
+            if (recordBatch == null) break;
+            vectorLoader.load(recordBatch);
+          }
+        }
+        validateComplexContent(count, root);
+      }
+    }
   }
 
   @Test
   public void testWriteReadMultipleRBs() throws IOException {
     File file = new File("target/mytest_multiple.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
     int[] counts = { 10, 5 };
 
     // write
@@ -172,10 +233,12 @@ public void testWriteReadMultipleRBs() throws IOException {
       VectorUnloader vectorUnloader0 = newVectorUnloader(parent.getChild("root"));
       Schema schema = vectorUnloader0.getSchema();
       Assert.assertEquals(2, schema.getFields().size());
-      try (ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);) {
+      try (ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
+          ArrowStreamWriter streamWriter = new ArrowStreamWriter(stream, schema, 2)) {
         try (ArrowRecordBatch recordBatch = vectorUnloader0.getRecordBatch()) {
           Assert.assertEquals("RB #0", counts[0], recordBatch.getLength());
           arrowWriter.writeRecordBatch(recordBatch);
+          streamWriter.writeRecordBatch(recordBatch);
         }
         parent.allocateNew();
         writeData(counts[1], parent); // if we write the same data we don't catch that the metadata is stored in the wrong order.
@@ -183,6 +246,7 @@ public void testWriteReadMultipleRBs() throws IOException {
         try (ArrowRecordBatch recordBatch = vectorUnloader1.getRecordBatch()) {
           Assert.assertEquals("RB #1", counts[1], recordBatch.getLength());
           arrowWriter.writeRecordBatch(recordBatch);
+          streamWriter.writeRecordBatch(recordBatch);
         }
       }
     }
@@ -222,11 +286,42 @@ public void testWriteReadMultipleRBs() throws IOException {
         }
       }
     }
+
+    // read stream
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)
+        ) {
+      arrowReader.init();
+      Schema schema = arrowReader.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      int i = 0;
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator);) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+        for (int n = 0; n < 2; n++) {
+          try (ArrowRecordBatch recordBatch = arrowReader.nextRecordBatch()) {
+            assertTrue(recordBatch != null);
+            Assert.assertEquals("RB #" + i, counts[i], recordBatch.getLength());
+            List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
+            for (ArrowBuffer arrowBuffer : buffersLayout) {
+              Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+            }
+            vectorLoader.load(recordBatch);
+            validateContent(counts[i], root);
+          }
+          ++i;
+        }
+      }
+    }
   }
 
   @Test
   public void testWriteReadUnion() throws IOException {
     File file = new File("target/mytest_write_union.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
@@ -238,9 +333,9 @@ public void testWriteReadUnion() throws IOException {
 
       validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
 
-      write(parent.getChild("root"), file);
+      write(parent.getChild("root"), file, stream);
     }
- // read
+    // read
     try (
         BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
         FileInputStream fileInputStream = new FileInputStream(file);
@@ -263,9 +358,37 @@ public void testWriteReadUnion() throws IOException {
         }
       }
     }
+
+    // Read from stream.
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)
+        ) {
+      arrowReader.init();
+      Schema schema = arrowReader.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
+        VectorLoader vectorLoader = new VectorLoader(root);
+        while (true) {
+          try (ArrowRecordBatch recordBatch = arrowReader.nextRecordBatch()) {
+            if (recordBatch == null) break;
+            vectorLoader.load(recordBatch);
+          }
+        }
+        validateUnionData(count, root);
+      }
+    }
   }
 
-  private void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
+  /**
+   * Writes the contents of parents to file. If outStream is non-null, also writes it
+   * to outStream in the streaming serialized format.
+   */
+  private void write(FieldVector parent, File file, OutputStream outStream) throws FileNotFoundException, IOException {
     VectorUnloader vectorUnloader = newVectorUnloader(parent);
     Schema schema = vectorUnloader.getSchema();
     LOGGER.debug("writing schema: " + schema);
@@ -276,5 +399,15 @@ private void write(FieldVector parent, File file) throws FileNotFoundException,
             ) {
       arrowWriter.writeRecordBatch(recordBatch);
     }
+
+    // Also try serializing to the stream writer.
+    if (outStream != null) {
+      try (
+          ArrowStreamWriter arrowWriter = new ArrowStreamWriter(outStream, schema, -1);
+          ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+          ) {
+        arrowWriter.writeRecordBatch(recordBatch);
+      }
+    }
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
new file mode 100644
index 0000000000000..7b4de80ee03ea
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.stream;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.file.ReadChannel;
+import org.apache.arrow.vector.file.WriteChannel;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import io.netty.buffer.ArrowBuf;
+
+public class MessageSerializerTest {
+
+  public static ArrowBuf buf(BufferAllocator alloc, byte[] bytes) {
+    ArrowBuf buffer = alloc.buffer(bytes.length);
+    buffer.writeBytes(bytes);
+    return buffer;
+  }
+
+  public static byte[] array(ArrowBuf buf) {
+    byte[] bytes = new byte[buf.readableBytes()];
+    buf.readBytes(bytes);
+    return bytes;
+  }
+
+  @Test
+  public void testSchemaMessageSerialization() throws IOException {
+    Schema schema = testSchema();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    long size = MessageSerializer.serialize(
+        new WriteChannel(Channels.newChannel(out)), schema);
+    assertEquals(size, out.toByteArray().length);
+
+    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+    Schema deserialized = MessageSerializer.deserializeSchema(
+        new ReadChannel(Channels.newChannel(in)));
+    assertEquals(schema, deserialized);
+    assertEquals(1, deserialized.getFields().size());
+  }
+
+  @Test
+  public void testSerializeRecordBatch() throws IOException {
+    byte[] validity = new byte[] { (byte)255, 0};
+    // second half is "undefined"
+    byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+
+    BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+    ArrowBuf validityb = buf(alloc, validity);
+    ArrowBuf valuesb =  buf(alloc, values);
+
+    ArrowRecordBatch batch = new ArrowRecordBatch(
+        16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb));
+
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), batch);
+
+    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+    ArrowRecordBatch deserialized = MessageSerializer.deserializeRecordBatch(
+        new ReadChannel(Channels.newChannel(in)), alloc);
+    verifyBatch(deserialized, validity, values);
+  }
+
+  public static Schema testSchema() {
+    return new Schema(asList(new Field(
+        "testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
+  }
+
+  // Verifies batch contents matching test schema.
+  public static void verifyBatch(ArrowRecordBatch batch, byte[] validity, byte[] values) {
+    assertTrue(batch != null);
+    List<ArrowFieldNode> nodes = batch.getNodes();
+    assertEquals(1, nodes.size());
+    ArrowFieldNode node = nodes.get(0);
+    assertEquals(16, node.getLength());
+    assertEquals(8, node.getNullCount());
+    List<ArrowBuf> buffers = batch.getBuffers();
+    assertEquals(2, buffers.size());
+    assertArrayEquals(validity, MessageSerializerTest.array(buffers.get(0)));
+    assertArrayEquals(values, MessageSerializerTest.array(buffers.get(1)));
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
new file mode 100644
index 0000000000000..ba1cdaeeb2262
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.stream;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.file.BaseFileTest;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import io.netty.buffer.ArrowBuf;
+
+public class TestArrowStream extends BaseFileTest {
+  @Test
+  public void testEmptyStream() throws IOException {
+    Schema schema = MessageSerializerTest.testSchema();
+
+    // Write the stream.
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    try (ArrowStreamWriter writer = new ArrowStreamWriter(out, schema, -1)) {
+    }
+
+    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+    try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) {
+      reader.init();
+      assertEquals(schema, reader.getSchema());
+      // Empty should return null. Can be called repeatedly.
+      assertTrue(reader.nextRecordBatch() == null);
+      assertTrue(reader.nextRecordBatch() == null);
+    }
+  }
+
+  @Test
+  public void testReadWrite() throws IOException {
+    Schema schema = MessageSerializerTest.testSchema();
+    byte[] validity = new byte[] { (byte)255, 0};
+    // second half is "undefined"
+    byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+
+    int numBatches = 5;
+    BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    long bytesWritten = 0;
+    try (ArrowStreamWriter writer = new ArrowStreamWriter(out, schema, numBatches)) {
+      ArrowBuf validityb = MessageSerializerTest.buf(alloc, validity);
+      ArrowBuf valuesb =  MessageSerializerTest.buf(alloc, values);
+      for (int i = 0; i < numBatches; i++) {
+        writer.writeRecordBatch(new ArrowRecordBatch(
+            16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)));
+      }
+      bytesWritten = writer.bytesWritten();
+    }
+
+    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+    try (ArrowStreamReader reader = new ArrowStreamReader(in, alloc)) {
+      reader.init();
+      Schema readSchema = reader.getSchema();
+      for (int i = 0; i < numBatches; i++) {
+        assertEquals(schema, readSchema);
+        assertTrue(
+            readSchema.getFields().get(0).getTypeLayout().getVectorTypes().toString(),
+            readSchema.getFields().get(0).getTypeLayout().getVectors().size() > 0);
+        ArrowRecordBatch recordBatch = reader.nextRecordBatch();
+        MessageSerializerTest.verifyBatch(recordBatch, validity, values);
+        assertTrue(recordBatch != null);
+      }
+      assertTrue(reader.nextRecordBatch() == null);
+      assertEquals(bytesWritten, reader.bytesRead());
+    }
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
new file mode 100644
index 0000000000000..e187fa535cada
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.stream;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.channels.Pipe;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import io.netty.buffer.ArrowBuf;
+
+public class TestArrowStreamPipe {
+  Schema schema = MessageSerializerTest.testSchema();
+  // second half is "undefined"
+  byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+
+  private final class WriterThread extends Thread {
+    private final int numBatches;
+    private final ArrowStreamWriter writer;
+
+    public WriterThread(int numBatches, WritableByteChannel sinkChannel)
+        throws IOException {
+      this.numBatches = numBatches;
+      writer = new ArrowStreamWriter(sinkChannel, schema, -1);
+    }
+
+    @Override
+    public void run() {
+      BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+      try {
+        ArrowBuf valuesb =  MessageSerializerTest.buf(alloc, values);
+        for (int i = 0; i < numBatches; i++) {
+          // Send a changing byte id first.
+          byte[] validity = new byte[] { (byte)i, 0};
+          ArrowBuf validityb = MessageSerializerTest.buf(alloc, validity);
+          writer.writeRecordBatch(new ArrowRecordBatch(
+              16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)));
+        }
+        writer.close();
+      } catch (IOException e) {
+        e.printStackTrace();
+        assertTrue(false);
+      }
+    }
+
+    public long bytesWritten() { return writer.bytesWritten(); }
+  }
+
+  private final class ReaderThread extends Thread {
+    private int batchesRead = 0;
+    private final ArrowStreamReader reader;
+    private final BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+
+    public ReaderThread(ReadableByteChannel sourceChannel)
+        throws IOException {
+      reader = new ArrowStreamReader(sourceChannel, alloc);
+    }
+
+    @Override
+    public void run() {
+      try {
+        reader.init();
+        assertEquals(schema, reader.getSchema());
+        assertTrue(
+            reader.getSchema().getFields().get(0).getTypeLayout().getVectorTypes().toString(),
+            reader.getSchema().getFields().get(0).getTypeLayout().getVectors().size() > 0);
+
+        // Read all the batches. Each batch contains an incrementing id and then some
+        // constant data. Verify both.
+        while (true) {
+          ArrowRecordBatch batch = reader.nextRecordBatch();
+          if (batch == null) break;
+          byte[] validity = new byte[] { (byte)batchesRead, 0};
+          MessageSerializerTest.verifyBatch(batch, validity, values);
+          batchesRead++;
+        }
+      } catch (IOException e) {
+        e.printStackTrace();
+        assertTrue(false);
+      }
+    }
+
+    public int getBatchesRead() { return batchesRead; }
+    public long bytesRead() { return reader.bytesRead(); }
+  }
+
+  // Starts up a producer and consumer thread to read/write batches.
+  @Test
+  public void pipeTest() throws IOException, InterruptedException {
+    int NUM_BATCHES = 1000;
+    Pipe pipe = Pipe.open();
+    WriterThread writer = new WriterThread(NUM_BATCHES, pipe.sink());
+    ReaderThread reader = new ReaderThread(pipe.source());
+
+    writer.start();
+    reader.start();
+    reader.join();
+    writer.join();
+
+    assertEquals(NUM_BATCHES, reader.getBatchesRead());
+    assertEquals(writer.bytesWritten(), reader.bytesRead());
+  }
+}

From 512bc160ebaf8d6775ea67994262709e10a72795 Mon Sep 17 00:00:00 2001
From: Jingyuan Wang <jingyuan@live.com>
Date: Fri, 20 Jan 2017 12:43:20 -0500
Subject: [PATCH 0283/1644] ARROW-386: [Java] Respect case of struct / map
 field names

Changes include:
- Remove all toLowerCase() calls on field names in MapWriters.java template file, so that the writers can respect case of the field names.
- Use lower-case keys for internalMap in UnionVector instead of camel-case (e.g. bigInt -> bigint). p.s. I don't know what is the original purpose of using camel case here. It did not conflict because all field names are converted to lower cases in the past.
- Add a simple test case of MapWriter with mixed-case field names.

Author: Jingyuan Wang <jingyuan@live.com>

Closes #261 from alphalfalfa/arrow-386 and squashes the following commits:

cd08145 [Jingyuan Wang] Remove unnecessary handleCase() call
7b28bfc [Jingyuan Wang] Pass caseSensitive Attribute down to nested MapWriters
2fe7bcf [Jingyuan Wang] Separate MapWriters with CaseSensitiveMapWriters
d269e21 [Jingyuan Wang] Configure case sensitivity when constructing ComplexWriterImpl
cba60d1 [Jingyuan Wang] Add option to MapWriters to configure the case sensitivity (defaulted as case-insensitive)
51da2a1 [Jingyuan Wang] Arrow-386: [Java] Respect case of struct / map field names
---
 .../templates/CaseSensitiveMapWriters.java    | 54 +++++++++++++
 .../main/codegen/templates/MapWriters.java    | 35 +++++----
 .../codegen/templates/UnionListWriter.java    |  6 +-
 .../main/codegen/templates/UnionVector.java   |  3 +-
 .../main/codegen/templates/UnionWriter.java   | 12 ++-
 .../vector/complex/AbstractMapVector.java     |  6 +-
 .../complex/impl/ComplexWriterImpl.java       | 17 +++--
 .../impl/NullableMapWriterFactory.java        | 42 ++++++++++
 .../vector/complex/impl/PromotableWriter.java | 31 +++++++-
 .../complex/writer/TestComplexWriter.java     | 76 +++++++++++++++++++
 10 files changed, 253 insertions(+), 29 deletions(-)
 create mode 100644 java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java

diff --git a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
new file mode 100644
index 0000000000000..5357f9b8a9d3a
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<#list ["Nullable", "Single"] as mode>
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}CaseSensitiveMapWriter.java" />
+<#assign index = "idx()">
+<#if mode == "Single">
+<#assign containerClass = "MapVector" />
+<#else>
+<#assign containerClass = "NullableMapVector" />
+</#if>
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+/*
+ * This class is generated using FreeMarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ${mode}CaseSensitiveMapWriter extends ${mode}MapWriter {
+  public ${mode}CaseSensitiveMapWriter(${containerClass} container) {
+    super(container);
+  }
+
+  @Override
+  protected String handleCase(final String input){
+    return input;
+  }
+
+  @Override
+  protected NullableMapWriterFactory getNullableMapWriterFactory() {
+    return NullableMapWriterFactory.getNullableCaseSensitiveMapWriterFactoryInstance();
+  }
+
+}
+</#list>
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index f41b60072c873..4af6eee91b6de 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -48,7 +48,6 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
 
   protected final ${containerClass} container;
   private final Map<String, FieldWriter> fields = Maps.newHashMap();
-
   public ${mode}MapWriter(${containerClass} container) {
     <#if mode == "Single">
     if (container instanceof NullableMapVector) {
@@ -65,8 +64,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
         list(child.getName());
         break;
       case UNION:
-        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), MinorType.UNION, UnionVector.class));
-        fields.put(child.getName().toLowerCase(), writer);
+        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), MinorType.UNION, UnionVector.class), getNullableMapWriterFactory());
+        fields.put(handleCase(child.getName()), writer);
         break;
 <#list vv.types as type><#list type.minor as minor>
 <#assign lowerName = minor.class?uncap_first />
@@ -85,6 +84,14 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
     }
   }
 
+  protected String handleCase(final String input) {
+    return input.toLowerCase();
+  }
+
+  protected NullableMapWriterFactory getNullableMapWriterFactory() {
+    return NullableMapWriterFactory.getNullableMapWriterFactoryInstance();
+  }
+
   @Override
   public int getValueCapacity() {
     return container.getValueCapacity();
@@ -102,16 +109,17 @@ public Field getField() {
 
   @Override
   public MapWriter map(String name) {
-      FieldWriter writer = fields.get(name.toLowerCase());
+    String finalName = handleCase(name);
+    FieldWriter writer = fields.get(finalName);
     if(writer == null){
       int vectorCount=container.size();
       NullableMapVector vector = container.addOrGet(name, MinorType.MAP, NullableMapVector.class);
-      writer = new PromotableWriter(vector, container);
+      writer = new PromotableWriter(vector, container, getNullableMapWriterFactory());
       if(vectorCount != container.size()) {
         writer.allocate();
       }
       writer.setPosition(idx());
-      fields.put(name.toLowerCase(), writer);
+      fields.put(finalName, writer);
     } else {
       if (writer instanceof PromotableWriter) {
         // ensure writers are initialized
@@ -145,15 +153,16 @@ public void clear() {
 
   @Override
   public ListWriter list(String name) {
-    FieldWriter writer = fields.get(name.toLowerCase());
+    String finalName = handleCase(name);
+    FieldWriter writer = fields.get(finalName);
     int vectorCount = container.size();
     if(writer == null) {
-      writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class), container);
+      writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class), container, getNullableMapWriterFactory());
       if (container.size() > vectorCount) {
         writer.allocate();
       }
       writer.setPosition(idx());
-      fields.put(name.toLowerCase(), writer);
+      fields.put(finalName, writer);
     } else {
       if (writer instanceof PromotableWriter) {
         // ensure writers are initialized
@@ -199,7 +208,7 @@ public void end() {
   <#if minor.class?starts_with("Decimal") >
   public ${minor.class}Writer ${lowerName}(String name) {
     // returns existing writer
-    final FieldWriter writer = fields.get(name.toLowerCase());
+    final FieldWriter writer = fields.get(handleCase(name));
     assert writer != null;
     return writer;
   }
@@ -209,18 +218,18 @@ public void end() {
   @Override
   public ${minor.class}Writer ${lowerName}(String name) {
   </#if>
-    FieldWriter writer = fields.get(name.toLowerCase());
+    FieldWriter writer = fields.get(handleCase(name));
     if(writer == null) {
       ValueVector vector;
       ValueVector currentVector = container.getChild(name);
       ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class<#if minor.class == "Decimal"> , new int[] {precision, scale}</#if>);
-      writer = new PromotableWriter(v, container);
+      writer = new PromotableWriter(v, container, getNullableMapWriterFactory());
       vector = v;
       if (currentVector == null || currentVector != vector) {
         vector.allocateNewSafe();
       } 
       writer.setPosition(idx());
-      fields.put(name.toLowerCase(), writer);
+      fields.put(handleCase(name), writer);
     } else {
       if (writer instanceof PromotableWriter) {
         // ensure writers are initialized
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index bb39fe8d29426..d980830923b31 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -43,8 +43,12 @@ public class UnionListWriter extends AbstractFieldWriter {
   private int lastIndex = 0;
 
   public UnionListWriter(ListVector vector) {
+    this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
+  }
+
+  public UnionListWriter(ListVector vector, NullableMapWriterFactory nullableMapWriterFactory) {
     this.vector = vector;
-    this.writer = new PromotableWriter(vector.getDataVector(), vector);
+    this.writer = new PromotableWriter(vector.getDataVector(), vector, nullableMapWriterFactory);
     this.offsets = vector.getOffsetVector();
   }
 
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 18acdf4a551b4..1a6908df2c40d 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -136,6 +136,7 @@ public NullableMapVector getMap() {
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
   <#assign uncappedName = name?uncap_first/>
+  <#assign lowerCaseName = name?lower_case/>
   <#if !minor.class?starts_with("Decimal")>
 
   private Nullable${name}Vector ${uncappedName}Vector;
@@ -143,7 +144,7 @@ public NullableMapVector getMap() {
   public Nullable${name}Vector get${name}Vector() {
     if (${uncappedName}Vector == null) {
       int vectorCount = internalMap.size();
-      ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", MinorType.${name?upper_case}, Nullable${name}Vector.class);
+      ${uncappedName}Vector = internalMap.addOrGet("${lowerCaseName}", MinorType.${name?upper_case}, Nullable${name}Vector.class);
       if (internalMap.size() > vectorCount) {
         ${uncappedName}Vector.allocateNew();
         if (callBack != null) {
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index efb66f168f5f8..880f537c0296f 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+import org.apache.arrow.vector.complex.impl.NullableMapWriterFactory;
+
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionWriter.java" />
 
@@ -38,9 +40,15 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
   private MapWriter mapWriter;
   private UnionListWriter listWriter;
   private List<BaseWriter> writers = Lists.newArrayList();
+  private final NullableMapWriterFactory nullableMapWriterFactory;
 
   public UnionWriter(UnionVector vector) {
+    this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
+  }
+
+  public UnionWriter(UnionVector vector, NullableMapWriterFactory nullableMapWriterFactory) {
     data = vector;
+    this.nullableMapWriterFactory = nullableMapWriterFactory;
   }
 
   @Override
@@ -76,7 +84,7 @@ public void endList() {
 
   private MapWriter getMapWriter() {
     if (mapWriter == null) {
-      mapWriter = new NullableMapWriter(data.getMap());
+      mapWriter = nullableMapWriterFactory.build(data.getMap());
       mapWriter.setPosition(idx());
       writers.add(mapWriter);
     }
@@ -90,7 +98,7 @@ public MapWriter asMap() {
 
   private ListWriter getListWriter() {
     if (listWriter == null) {
-      listWriter = new UnionListWriter(data.getList());
+      listWriter = new UnionListWriter(data.getList(), nullableMapWriterFactory);
       listWriter.setPosition(idx());
       writers.add(listWriter);
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index 23b4997f4f586..f030d166ade8d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -155,7 +155,7 @@ public ValueVector getChildByOrdinal(int id) {
    */
   @Override
   public <T extends FieldVector> T getChild(String name, Class<T> clazz) {
-    final ValueVector v = vectors.get(name.toLowerCase());
+    final ValueVector v = vectors.get(name);
     if (v == null) {
       return null;
     }
@@ -191,7 +191,7 @@ protected void putChild(String name, FieldVector vector) {
    */
   protected void putVector(String name, FieldVector vector) {
     final ValueVector old = vectors.put(
-        Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
+        Preconditions.checkNotNull(name, "field name cannot be null"),
         Preconditions.checkNotNull(vector, "vector cannot be null")
     );
     if (old != null && old != vector) {
@@ -254,7 +254,7 @@ public List<ValueVector> getPrimitiveVectors() {
    */
   @Override
   public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
-    final int ordinal = vectors.getOrdinal(name.toLowerCase());
+    final int ordinal = vectors.getOrdinal(name);
     if (ordinal < 0) {
       return null;
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 761b1b43c08aa..dbdd2050d13ed 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -37,13 +37,20 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
   Mode mode = Mode.INIT;
   private final String name;
   private final boolean unionEnabled;
+  private final NullableMapWriterFactory nullableMapWriterFactory;
 
   private enum Mode { INIT, MAP, LIST };
 
-  public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){
+  public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled, boolean caseSensitive){
     this.name = name;
     this.container = container;
     this.unionEnabled = unionEnabled;
+    nullableMapWriterFactory = caseSensitive? NullableMapWriterFactory.getNullableCaseSensitiveMapWriterFactoryInstance() :
+        NullableMapWriterFactory.getNullableMapWriterFactoryInstance();
+  }
+
+  public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled) {
+    this(name, container, unionEnabled, false);
   }
 
   public ComplexWriterImpl(String name, MapVector container){
@@ -122,8 +129,7 @@ public MapWriter directMap(){
     switch(mode){
 
     case INIT:
-      NullableMapVector map = (NullableMapVector) container;
-      mapRoot = new NullableMapWriter(map);
+      mapRoot = nullableMapWriterFactory.build((NullableMapVector) container);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
       break;
@@ -144,7 +150,7 @@ public MapWriter rootAsMap() {
 
     case INIT:
       NullableMapVector map = container.addOrGet(name, MinorType.MAP, NullableMapVector.class);
-      mapRoot = new NullableMapWriter(map);
+      mapRoot = nullableMapWriterFactory.build(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
       break;
@@ -159,7 +165,6 @@ public MapWriter rootAsMap() {
     return mapRoot;
   }
 
-
   @Override
   public void allocate() {
     if(mapRoot != null) {
@@ -179,7 +184,7 @@ public ListWriter rootAsList() {
       if (container.size() > vectorCount) {
         listVector.allocateNew();
       }
-      listRoot = new UnionListWriter(listVector);
+      listRoot = new UnionListWriter(listVector, nullableMapWriterFactory);
       listRoot.setPosition(idx());
       mode = Mode.LIST;
       break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
new file mode 100644
index 0000000000000..d932cfb3e1287
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.impl;
+
+import org.apache.arrow.vector.complex.NullableMapVector;
+
+public class NullableMapWriterFactory {
+    private final boolean caseSensitive;
+    private static final NullableMapWriterFactory nullableMapWriterFactory = new NullableMapWriterFactory(false);
+    private static final NullableMapWriterFactory nullableCaseSensitiveWriterFactory = new NullableMapWriterFactory(true);
+
+    public NullableMapWriterFactory(boolean caseSensitive) {
+        this.caseSensitive = caseSensitive;
+    }
+
+    public NullableMapWriter build(NullableMapVector container) {
+        return this.caseSensitive? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container);
+    }
+
+    public static NullableMapWriterFactory getNullableMapWriterFactoryInstance() {
+        return nullableMapWriterFactory;
+    }
+
+    public static NullableMapWriterFactory getNullableCaseSensitiveMapWriterFactoryInstance() {
+        return nullableCaseSensitiveWriterFactory;
+    }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 94ff82c04bd18..1f7253bca93c8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -22,6 +22,7 @@
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -38,6 +39,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
 
   private final AbstractMapVector parentContainer;
   private final ListVector listVector;
+  private final NullableMapWriterFactory nullableMapWriterFactory;
   private int position;
 
   private enum State {
@@ -51,14 +53,24 @@ private enum State {
   private FieldWriter writer;
 
   public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
+    this(v, parentContainer, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
+  }
+
+  public PromotableWriter(ValueVector v, AbstractMapVector parentContainer, NullableMapWriterFactory nullableMapWriterFactory) {
     this.parentContainer = parentContainer;
     this.listVector = null;
+    this.nullableMapWriterFactory = nullableMapWriterFactory;
     init(v);
   }
 
   public PromotableWriter(ValueVector v, ListVector listVector) {
+    this(v, listVector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
+  }
+
+  public PromotableWriter(ValueVector v, ListVector listVector, NullableMapWriterFactory nullableMapWriterFactory) {
     this.listVector = listVector;
     this.parentContainer = null;
+    this.nullableMapWriterFactory = nullableMapWriterFactory;
     init(v);
   }
 
@@ -66,7 +78,7 @@ private void init(ValueVector v) {
     if (v instanceof UnionVector) {
       state = State.UNION;
       unionVector = (UnionVector) v;
-      writer = new UnionWriter(unionVector);
+      writer = new UnionWriter(unionVector, nullableMapWriterFactory);
     } else if (v instanceof ZeroVector) {
       state = State.UNTYPED;
     } else {
@@ -78,7 +90,20 @@ private void setWriter(ValueVector v) {
     state = State.SINGLE;
     vector = v;
     type = v.getMinorType();
-    writer = type.getNewFieldWriter(vector);
+    switch (type) {
+      case MAP:
+        writer = nullableMapWriterFactory.build((NullableMapVector) vector);
+        break;
+      case LIST:
+        writer = new UnionListWriter((ListVector) vector, nullableMapWriterFactory);
+        break;
+      case UNION:
+        writer = new UnionWriter((UnionVector) vector, nullableMapWriterFactory);
+        break;
+      default:
+        writer = type.getNewFieldWriter(vector);
+        break;
+    }
   }
 
   @Override
@@ -131,7 +156,7 @@ private FieldWriter promoteToUnion() {
       unionVector = listVector.promoteToUnion();
     }
     unionVector.addVector((FieldVector)tp.getTo());
-    writer = new UnionWriter(unionVector);
+    writer = new UnionWriter(unionVector, nullableMapWriterFactory);
     writer.setPosition(idx());
     for (int i = 0; i <= idx(); i++) {
       unionVector.getMutator().setType(i, vector.getMinorType());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index caa438aff4761..2c0c85328bdfb 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -23,7 +23,9 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -485,4 +487,78 @@ public void promotableWriterSchema() {
     Assert.assertTrue(intType.getIsSigned());
     Assert.assertEquals(ArrowTypeID.Utf8, field.getChildren().get(1).getType().getTypeID());
   }
+
+  private Set<String> getFieldNames(List<Field> fields) {
+    Set<String> fieldNames = new HashSet<>();
+    for (Field field: fields) {
+      fieldNames.add(field.getName());
+      if (!field.getChildren().isEmpty()) {
+        for (String name: getFieldNames(field.getChildren())) {
+          fieldNames.add(field.getName() + "::" + name);
+        }
+      }
+    }
+    return fieldNames;
+  }
+
+  @Test
+  public void mapWriterMixedCaseFieldNames() {
+    // test case-sensitive MapWriter
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("rootCaseSensitive", parent, false, true);
+    MapWriter rootWriterCaseSensitive = writer.rootAsMap();
+    rootWriterCaseSensitive.bigInt("int_field");
+    rootWriterCaseSensitive.bigInt("Int_Field");
+    rootWriterCaseSensitive.float4("float_field");
+    rootWriterCaseSensitive.float4("Float_Field");
+    MapWriter mapFieldWriterCaseSensitive = rootWriterCaseSensitive.map("map_field");
+    mapFieldWriterCaseSensitive.varChar("char_field");
+    mapFieldWriterCaseSensitive.varChar("Char_Field");
+    ListWriter listFieldWriterCaseSensitive = rootWriterCaseSensitive.list("list_field");
+    MapWriter listMapFieldWriterCaseSensitive = listFieldWriterCaseSensitive.map();
+    listMapFieldWriterCaseSensitive.bit("bit_field");
+    listMapFieldWriterCaseSensitive.bit("Bit_Field");
+
+    List<Field> fieldsCaseSensitive = parent.getField().getChildren().get(0).getChildren();
+    Set<String> fieldNamesCaseSensitive = getFieldNames(fieldsCaseSensitive);
+    Assert.assertEquals(11, fieldNamesCaseSensitive.size());
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("int_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("Int_Field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("float_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("Float_Field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field::char_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field::Char_Field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::Bit_Field"));
+
+    // test case-insensitive MapWriter
+    ComplexWriter writerCaseInsensitive = new ComplexWriterImpl("rootCaseInsensitive", parent, false, false);
+    MapWriter rootWriterCaseInsensitive = writerCaseInsensitive.rootAsMap();
+
+    rootWriterCaseInsensitive.bigInt("int_field");
+    rootWriterCaseInsensitive.bigInt("Int_Field");
+    rootWriterCaseInsensitive.float4("float_field");
+    rootWriterCaseInsensitive.float4("Float_Field");
+    MapWriter mapFieldWriterCaseInsensitive = rootWriterCaseInsensitive.map("map_field");
+    mapFieldWriterCaseInsensitive.varChar("char_field");
+    mapFieldWriterCaseInsensitive.varChar("Char_Field");
+    ListWriter listFieldWriterCaseInsensitive = rootWriterCaseInsensitive.list("list_field");
+    MapWriter listMapFieldWriterCaseInsensitive = listFieldWriterCaseInsensitive.map();
+    listMapFieldWriterCaseInsensitive.bit("bit_field");
+    listMapFieldWriterCaseInsensitive.bit("Bit_Field");
+
+    List<Field> fieldsCaseInsensitive = parent.getField().getChildren().get(1).getChildren();
+    Set<String> fieldNamesCaseInsensitive = getFieldNames(fieldsCaseInsensitive);
+    Assert.assertEquals(7, fieldNamesCaseInsensitive.size());
+    Assert.assertTrue(fieldNamesCaseInsensitive.contains("int_field"));
+    Assert.assertTrue(fieldNamesCaseInsensitive.contains("float_field"));
+    Assert.assertTrue(fieldNamesCaseInsensitive.contains("map_field"));
+    Assert.assertTrue(fieldNamesCaseInsensitive.contains("map_field::char_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field"));
+  }
 }
\ No newline at end of file

From 8ca7033fcd3fcf377cb7924eae9be45b8f6ebd5d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 20 Jan 2017 17:56:23 -0500
Subject: [PATCH 0284/1644] ARROW-499: Update file serialization to use the
 streaming serialization format.

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Nong Li <nongli@gmail.com>

Closes #292 from nongli/file and squashes the following commits:

18890a9 [Wes McKinney] Message fixes. Fix Java test suite. Integration tests pass
f187539 [Nong Li] Merge pull request #1 from wesm/file-change-cpp-impl
e3af434 [Wes McKinney] Remove unused variable
664d5be [Wes McKinney] Fixes, stream tests pass again
ba8db91 [Wes McKinney] Redo MessageSerializer with unions. Still has bugs
21854cc [Wes McKinney] Restore Block.bodyLength to long
7c6f7ef [Nong Li] Update to restore Block behavior
27b3909 [Nong Li] [ARROW-499]: [Java] Update file serialization to use the streaming serialization format.
---
 cpp/src/arrow/ipc/adapter.cc                  |  11 +-
 cpp/src/arrow/ipc/metadata-internal.cc        |  21 +--
 format/File.fbs                               |   5 +-
 integration/integration_test.py               |   2 +-
 .../apache/arrow/vector/file/ArrowFooter.java |   5 +-
 .../apache/arrow/vector/file/ArrowReader.java |  64 ++-----
 .../apache/arrow/vector/file/ArrowWriter.java |  43 +----
 .../apache/arrow/vector/file/ReadChannel.java |  11 +-
 .../vector/stream/MessageSerializer.java      | 169 +++++++++++-------
 .../arrow/vector/file/TestArrowFile.java      |   4 -
 .../arrow/vector/file/TestArrowFooter.java    |   8 +
 .../vector/file/TestArrowReaderWriter.java    |  16 ++
 12 files changed, 174 insertions(+), 185 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 2b5ef11f861af..7b4d18c267d43 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -129,13 +129,12 @@ class RecordBatchWriter : public ArrayVisitor {
         num_rows_, body_length, field_nodes_, buffer_meta_, &metadata_fb));
 
     // Need to write 4 bytes (metadata size), the metadata, plus padding to
-    // fall on a 64-byte offset
-    int64_t padded_metadata_length =
-        BitUtil::RoundUpToMultipleOf64(metadata_fb->size() + 4);
+    // fall on an 8-byte offset
+    int64_t padded_metadata_length = BitUtil::CeilByte(metadata_fb->size() + 4);
 
     // The returned metadata size includes the length prefix, the flatbuffer,
     // plus padding
-    *metadata_length = padded_metadata_length;
+    *metadata_length = static_cast<int32_t>(padded_metadata_length);
 
     // Write the flatbuffer size prefix
     int32_t flatbuffer_size = metadata_fb->size();
@@ -604,7 +603,9 @@ Status ReadRecordBatchMetadata(int64_t offset, int32_t metadata_length,
     return Status::Invalid(ss.str());
   }
 
-  *metadata = std::make_shared<RecordBatchMetadata>(buffer, sizeof(int32_t));
+  std::shared_ptr<Message> message;
+  RETURN_NOT_OK(Message::Open(buffer, 4, &message));
+  *metadata = std::make_shared<RecordBatchMetadata>(message);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 16069a8f9dcf0..cc160c42ec9ef 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -320,23 +320,10 @@ Status MessageBuilder::SetRecordBatch(int32_t length, int64_t body_length,
 Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
     const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
-  flatbuffers::FlatBufferBuilder fbb;
-
-  auto batch = flatbuf::CreateRecordBatch(
-      fbb, length, fbb.CreateVectorOfStructs(nodes), fbb.CreateVectorOfStructs(buffers));
-
-  fbb.Finish(batch);
-
-  int32_t size = fbb.GetSize();
-
-  auto result = std::make_shared<PoolBuffer>();
-  RETURN_NOT_OK(result->Resize(size));
-
-  uint8_t* dst = result->mutable_data();
-  memcpy(dst, fbb.GetBufferPointer(), size);
-
-  *out = result;
-  return Status::OK();
+  MessageBuilder builder;
+  RETURN_NOT_OK(builder.SetRecordBatch(length, body_length, nodes, buffers));
+  RETURN_NOT_OK(builder.Finish());
+  return builder.GetBuffer(out);
 }
 
 Status MessageBuilder::Finish() {
diff --git a/format/File.fbs b/format/File.fbs
index f28dc204d58d9..e8d6da4f848ff 100644
--- a/format/File.fbs
+++ b/format/File.fbs
@@ -35,12 +35,15 @@ table Footer {
 
 struct Block {
 
+  /// Index to the start of the RecordBlock (note this is past the Message header)
   offset: long;
 
+  /// Length of the metadata
   metaDataLength: int;
 
+  /// Length of the data (this is aligned so there can be a gap between this and
+  /// the metatdata).
   bodyLength: long;
-
 }
 
 root_type Footer;
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 417354bc83d9e..77510daecc0b4 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -648,7 +648,7 @@ def get_static_json_files():
 
 
 def run_all_tests(debug=False):
-    testers = [JavaTester(debug=debug), CPPTester(debug=debug)]
+    testers = [CPPTester(debug=debug), JavaTester(debug=debug)]
     static_json_files = get_static_json_files()
     generated_json_files = get_generated_json_files()
     json_files = static_json_files + generated_json_files
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
index 3be19296cb56d..38903068570c7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
@@ -65,10 +65,11 @@ private static List<ArrowBlock> recordBatches(Footer footer) {
 
   private static List<ArrowBlock> dictionaries(Footer footer) {
     List<ArrowBlock> dictionaries = new ArrayList<>();
-    Block tempBLock = new Block();
+    Block tempBlock = new Block();
+
     int dictionariesLength = footer.dictionariesLength();
     for (int i = 0; i < dictionariesLength; i++) {
-      Block block = footer.dictionaries(tempBLock, i);
+      Block block = footer.dictionaries(tempBlock, i);
       dictionaries.add(new ArrowBlock(block.offset(), block.metaDataLength(), block.bodyLength()));
     }
     return dictionaries;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index 58c51605c5600..8f4f4978d66cf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -20,23 +20,15 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.SeekableByteChannel;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 
-import org.apache.arrow.flatbuf.Buffer;
-import org.apache.arrow.flatbuf.FieldNode;
 import org.apache.arrow.flatbuf.Footer;
-import org.apache.arrow.flatbuf.RecordBatch;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.stream.MessageSerializer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import io.netty.buffer.ArrowBuf;
-
 public class ArrowReader implements AutoCloseable {
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowReader.class);
 
@@ -54,15 +46,6 @@ public ArrowReader(SeekableByteChannel in, BufferAllocator allocator) {
     this.allocator = allocator;
   }
 
-  private int readFully(ArrowBuf buffer, int l) throws IOException {
-    int n = readFully(buffer.nioBuffer(buffer.writerIndex(), l));
-    buffer.writerIndex(n);
-    if (n != l) {
-      throw new IllegalStateException(n + " != " + l);
-    }
-    return n;
-  }
-
   private int readFully(ByteBuffer buffer) throws IOException {
     int total = 0;
     int n;
@@ -104,46 +87,21 @@ public ArrowFooter readFooter() throws IOException {
 
   // TODO: read dictionaries
 
-  public ArrowRecordBatch readRecordBatch(ArrowBlock recordBatchBlock) throws IOException {
-    LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d", recordBatchBlock.getOffset(), recordBatchBlock.getMetadataLength(), recordBatchBlock.getBodyLength()));
-    int l = (int)(recordBatchBlock.getMetadataLength() + recordBatchBlock.getBodyLength());
-    if (l < 0) {
-      throw new InvalidArrowFileException("block invalid: " + recordBatchBlock);
-    }
-    final ArrowBuf buffer = allocator.buffer(l);
-    LOGGER.debug("allocated buffer " + buffer);
-    in.position(recordBatchBlock.getOffset());
-    int n = readFully(buffer, l);
-    if (n != l) {
-      throw new IllegalStateException(n + " != " + l);
-    }
-
-    // Record batch flatbuffer is prefixed by its size as int32le
-    final ArrowBuf metadata = buffer.slice(4, recordBatchBlock.getMetadataLength() - 4);
-    RecordBatch recordBatchFB = RecordBatch.getRootAsRecordBatch(metadata.nioBuffer().asReadOnlyBuffer());
-
-    int nodesLength = recordBatchFB.nodesLength();
-    final ArrowBuf body = buffer.slice(recordBatchBlock.getMetadataLength(), (int)recordBatchBlock.getBodyLength());
-    List<ArrowFieldNode> nodes = new ArrayList<>();
-    for (int i = 0; i < nodesLength; ++i) {
-      FieldNode node = recordBatchFB.nodes(i);
-      nodes.add(new ArrowFieldNode(node.length(), node.nullCount()));
+  public ArrowRecordBatch readRecordBatch(ArrowBlock block) throws IOException {
+    LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d",
+        block.getOffset(), block.getMetadataLength(),
+        block.getBodyLength()));
+    in.position(block.getOffset());
+    ArrowRecordBatch batch =  MessageSerializer.deserializeRecordBatch(
+        new ReadChannel(in, block.getOffset()), block, allocator);
+    if (batch == null) {
+      throw new IOException("Invalid file. No batch at offset: " + block.getOffset());
     }
-    List<ArrowBuf> buffers = new ArrayList<>();
-    for (int i = 0; i < recordBatchFB.buffersLength(); ++i) {
-      Buffer bufferFB = recordBatchFB.buffers(i);
-      LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", bufferFB.offset(), bufferFB.length()));
-      ArrowBuf vectorBuffer = body.slice((int)bufferFB.offset(), (int)bufferFB.length());
-      buffers.add(vectorBuffer);
-    }
-    ArrowRecordBatch arrowRecordBatch = new ArrowRecordBatch(recordBatchFB.length(), nodes, buffers);
-    LOGGER.debug("released buffer " + buffer);
-    buffer.release();
-    return arrowRecordBatch;
+    return batch;
   }
 
+  @Override
   public void close() throws IOException {
     in.close();
   }
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 3febd11f4c76a..24c667e67d98d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -23,14 +23,12 @@
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.arrow.vector.schema.ArrowBuffer;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.MessageSerializer;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import io.netty.buffer.ArrowBuf;
-
 public class ArrowWriter implements AutoCloseable {
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowWriter.class);
 
@@ -39,7 +37,6 @@ public class ArrowWriter implements AutoCloseable {
   private final Schema schema;
 
   private final List<ArrowBlock> recordBatches = new ArrayList<>();
-
   private boolean started = false;
 
   public ArrowWriter(WritableByteChannel out, Schema schema) {
@@ -49,47 +46,19 @@ public ArrowWriter(WritableByteChannel out, Schema schema) {
 
   private void start() throws IOException {
     writeMagic();
+    MessageSerializer.serialize(out, schema);
   }
 
-
   // TODO: write dictionaries
 
   public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
     checkStarted();
-    out.align();
+    ArrowBlock batchDesc = MessageSerializer.serialize(out, recordBatch);
+    LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d",
+        batchDesc.getOffset(), batchDesc.getMetadataLength(), batchDesc.getBodyLength()));
 
-    // write metadata header with int32 size prefix
-    long offset = out.getCurrentPosition();
-    out.write(recordBatch, true);
-    out.align();
-    // write body
-    long bodyOffset = out.getCurrentPosition();
-    List<ArrowBuf> buffers = recordBatch.getBuffers();
-    List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
-    if (buffers.size() != buffersLayout.size()) {
-      throw new IllegalStateException("the layout does not match: " + buffers.size() + " != " + buffersLayout.size());
-    }
-    for (int i = 0; i < buffers.size(); i++) {
-      ArrowBuf buffer = buffers.get(i);
-      ArrowBuffer layout = buffersLayout.get(i);
-      long startPosition = bodyOffset + layout.getOffset();
-      if (startPosition != out.getCurrentPosition()) {
-        out.writeZeros((int)(startPosition - out.getCurrentPosition()));
-      }
-
-      out.write(buffer);
-      if (out.getCurrentPosition() != startPosition + layout.getSize()) {
-        throw new IllegalStateException("wrong buffer size: " + out.getCurrentPosition() + " != " + startPosition + layout.getSize());
-      }
-    }
-    int metadataLength = (int)(bodyOffset - offset);
-    if (metadataLength <= 0) {
-      throw new InvalidArrowFileException("invalid recordBatch");
-    }
-    long bodyLength = out.getCurrentPosition() - bodyOffset;
-    LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d", offset, metadataLength, bodyLength));
     // add metadata to footer
-    recordBatches.add(new ArrowBlock(offset, metadataLength, bodyLength));
+    recordBatches.add(batchDesc);
   }
 
   private void checkStarted() throws IOException {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
index b062f3826eab3..a9dc1293b8193 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
@@ -32,9 +32,16 @@ public class ReadChannel implements AutoCloseable {
 
   private ReadableByteChannel in;
   private long bytesRead = 0;
+  // The starting byte offset into 'in'.
+  private final long startByteOffset;
 
-  public ReadChannel(ReadableByteChannel in) {
+  public ReadChannel(ReadableByteChannel in, long startByteOffset) {
     this.in = in;
+    this.startByteOffset = startByteOffset;
+  }
+
+  public ReadChannel(ReadableByteChannel in) {
+    this(in, 0);
   }
 
   public long bytesRead() { return bytesRead; }
@@ -65,6 +72,8 @@ public int readFully(ArrowBuf buffer, int l) throws IOException {
     return n;
   }
 
+  public long getCurrentPositiion() { return startByteOffset + bytesRead; }
+
   @Override
   public void close() throws IOException {
     if (this.in != null) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 22c46e2817b1e..6e22dbd164d6e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -29,6 +29,7 @@
 import org.apache.arrow.flatbuf.MetadataVersion;
 import org.apache.arrow.flatbuf.RecordBatch;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.file.ArrowBlock;
 import org.apache.arrow.vector.file.ReadChannel;
 import org.apache.arrow.vector.file.WriteChannel;
 import org.apache.arrow.vector.schema.ArrowBuffer;
@@ -52,7 +53,8 @@
  * For RecordBatch messages the serialization is:
  *   1. 4 byte little endian batch metadata header
  *   2. FB serialized RowBatch
- *   3. serialized RowBatch buffers.
+ *   3. Padding to align to 8 byte boundary.
+ *   4. serialized RowBatch buffers.
  */
 public class MessageSerializer {
 
@@ -68,14 +70,10 @@ public static int bytesToInt(byte[] bytes) {
    */
   public static long serialize(WriteChannel out, Schema schema) throws IOException {
     FlatBufferBuilder builder = new FlatBufferBuilder();
-    builder.finish(schema.getSchema(builder));
-    ByteBuffer serializedBody = builder.dataBuffer();
-    ByteBuffer serializedHeader =
-        serializeHeader(MessageHeader.Schema, serializedBody.remaining());
-
-    long size = out.writeIntLittleEndian(serializedHeader.remaining());
-    size += out.write(serializedHeader);
-    size += out.write(serializedBody);
+    int schemaOffset = schema.getSchema(builder);
+    ByteBuffer serializedMessage = serializeMessage(builder, MessageHeader.Schema, schemaOffset, 0);
+    long size = out.writeIntLittleEndian(serializedMessage.remaining());
+    size += out.write(serializedMessage);
     return size;
   }
 
@@ -83,49 +81,51 @@ public static long serialize(WriteChannel out, Schema schema) throws IOException
    * Deserializes a schema object. Format is from serialize().
    */
   public static Schema deserializeSchema(ReadChannel in) throws IOException {
-    Message header = deserializeHeader(in, MessageHeader.Schema);
-    if (header == null) {
+    Message message = deserializeMessage(in, MessageHeader.Schema);
+    if (message == null) {
       throw new IOException("Unexpected end of input. Missing schema.");
     }
 
-    // Now read the schema.
-    ByteBuffer buffer = ByteBuffer.allocate((int)header.bodyLength());
-    if (in.readFully(buffer) != header.bodyLength()) {
-      throw new IOException("Unexpected end of input trying to read schema.");
-    }
-    buffer.rewind();
-    return Schema.deserialize(buffer);
+    return Schema.convertSchema((org.apache.arrow.flatbuf.Schema)
+        message.header(new org.apache.arrow.flatbuf.Schema()));
   }
 
   /**
-   * Serializes an ArrowRecordBatch.
+   * Serializes an ArrowRecordBatch. Returns the offset and length of the written batch.
    */
-  public static long serialize(WriteChannel out, ArrowRecordBatch batch)
+  public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
       throws IOException {
     long start = out.getCurrentPosition();
     int bodyLength = batch.computeBodyLength();
 
-    ByteBuffer metadata = WriteChannel.serialize(batch);
-    ByteBuffer serializedHeader =
-        serializeHeader(MessageHeader.RecordBatch, bodyLength + metadata.remaining() + 4);
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    int batchOffset = batch.writeTo(builder);
+
+    ByteBuffer serializedMessage = serializeMessage(builder, MessageHeader.RecordBatch,
+        batchOffset, bodyLength);
+
+    int metadataLength = serializedMessage.remaining();
+
+    // Add extra padding bytes so that length prefix + metadata is a multiple
+    // of 8 after alignment
+    if ((start + metadataLength + 4) % 8 != 0) {
+        metadataLength += 8 - (start + metadataLength + 4) % 8;
+    }
 
-    // Write message header.
-    out.writeIntLittleEndian(serializedHeader.remaining());
-    out.write(serializedHeader);
+    out.writeIntLittleEndian(metadataLength);
+    out.write(serializedMessage);
 
-    // Write the metadata, with the 4 byte little endian prefix
-    out.writeIntLittleEndian(metadata.remaining());
-    out.write(metadata);
+    // Align the output to 8 byte boundary.
+    out.align();
 
-    // Write batch header.
-    long offset = out.getCurrentPosition();
+    long bufferStart = out.getCurrentPosition();
     List<ArrowBuf> buffers = batch.getBuffers();
     List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
 
     for (int i = 0; i < buffers.size(); i++) {
       ArrowBuf buffer = buffers.get(i);
       ArrowBuffer layout = buffersLayout.get(i);
-      long startPosition = offset + layout.getOffset();
+      long startPosition = bufferStart + layout.getOffset();
       if (startPosition != out.getCurrentPosition()) {
         out.writeZeros((int)(startPosition - out.getCurrentPosition()));
       }
@@ -135,7 +135,8 @@ public static long serialize(WriteChannel out, ArrowRecordBatch batch)
             " != " + startPosition + layout.getSize());
       }
     }
-    return out.getCurrentPosition() - start;
+    // Metadata size in the Block account for the size prefix
+    return new ArrowBlock(start, metadataLength + 4, out.getCurrentPosition() - bufferStart);
   }
 
   /**
@@ -143,23 +144,62 @@ public static long serialize(WriteChannel out, ArrowRecordBatch batch)
    */
   public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in,
       BufferAllocator alloc) throws IOException {
-    Message header = deserializeHeader(in, MessageHeader.RecordBatch);
-    if (header == null) return null;
+    Message message = deserializeMessage(in, MessageHeader.RecordBatch);
+    if (message == null) return null;
+
+    if (message.bodyLength() > Integer.MAX_VALUE) {
+      throw new IOException("Cannot currently deserialize record batches over 2GB");
+    }
+
+    RecordBatch recordBatchFB = (RecordBatch) message.header(new RecordBatch());
+
+    int bodyLength = (int) message.bodyLength();
+
+    // Now read the record batch body
+    ArrowBuf buffer = alloc.buffer(bodyLength);
+    if (in.readFully(buffer, bodyLength) != bodyLength) {
+      throw new IOException("Unexpected end of input trying to read batch.");
+    }
+    return deserializeRecordBatch(recordBatchFB, buffer);
+  }
+
+  /**
+   * Deserializes a RecordBatch knowing the size of the entire message up front. This
+   * minimizes the number of reads to the underlying stream.
+   */
+  public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock block,
+      BufferAllocator alloc) throws IOException {
+    // Metadata length contains integer prefix plus byte padding
+    long totalLen = block.getMetadataLength() + block.getBodyLength();
 
-    int messageLen = (int)header.bodyLength();
-    // Now read the buffer. This has the metadata followed by the data.
-    ArrowBuf buffer = alloc.buffer(messageLen);
-    if (in.readFully(buffer, messageLen) != messageLen) {
+    if (totalLen > Integer.MAX_VALUE) {
+      throw new IOException("Cannot currently deserialize record batches over 2GB");
+    }
+
+    ArrowBuf buffer = alloc.buffer((int) totalLen);
+    if (in.readFully(buffer, (int) totalLen) != totalLen) {
       throw new IOException("Unexpected end of input trying to read batch.");
     }
 
-    // Read the metadata. It starts with the 4 byte size of the metadata.
-    int metadataSize = buffer.readInt();
-    RecordBatch recordBatchFB =
-        RecordBatch.getRootAsRecordBatch( buffer.nioBuffer().asReadOnlyBuffer());
+    ArrowBuf metadataBuffer = buffer.slice(4, block.getMetadataLength() - 4);
+
+    Message messageFB =
+        Message.getRootAsMessage(metadataBuffer.nioBuffer().asReadOnlyBuffer());
+
+    RecordBatch recordBatchFB = (RecordBatch) messageFB.header(new RecordBatch());
+
+    // Now read the body
+    final ArrowBuf body = buffer.slice(block.getMetadataLength(),
+        (int) totalLen - block.getMetadataLength());
+    ArrowRecordBatch result = deserializeRecordBatch(recordBatchFB, body);
+
+    return result;
+  }
 
-    // No read the body
-    final ArrowBuf body = buffer.slice(4 + metadataSize, messageLen - metadataSize - 4);
+  // Deserializes a record batch given the Flatbuffer metadata and in-memory body
+  private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
+      ArrowBuf body) {
+    // Now read the body
     int nodesLength = recordBatchFB.nodesLength();
     List<ArrowFieldNode> nodes = new ArrayList<>();
     for (int i = 0; i < nodesLength; ++i) {
@@ -174,43 +214,44 @@ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in,
     }
     ArrowRecordBatch arrowRecordBatch =
         new ArrowRecordBatch(recordBatchFB.length(), nodes, buffers);
-    buffer.release();
+    body.release();
     return arrowRecordBatch;
   }
 
   /**
    * Serializes a message header.
    */
-  private static ByteBuffer serializeHeader(byte headerType, int bodyLength) {
-    FlatBufferBuilder headerBuilder = new FlatBufferBuilder();
-    Message.startMessage(headerBuilder);
-    Message.addHeaderType(headerBuilder, headerType);
-    Message.addVersion(headerBuilder, MetadataVersion.V1);
-    Message.addBodyLength(headerBuilder, bodyLength);
-    headerBuilder.finish(Message.endMessage(headerBuilder));
-    return headerBuilder.dataBuffer();
+  private static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte headerType,
+      int headerOffset, int bodyLength) {
+    Message.startMessage(builder);
+    Message.addHeaderType(builder, headerType);
+    Message.addHeader(builder, headerOffset);
+    Message.addVersion(builder, MetadataVersion.V1);
+    Message.addBodyLength(builder, bodyLength);
+    builder.finish(Message.endMessage(builder));
+    return builder.dataBuffer();
   }
 
-  private static Message deserializeHeader(ReadChannel in, byte headerType) throws IOException {
-    // Read the header size. There is an i32 little endian prefix.
+  private static Message deserializeMessage(ReadChannel in, byte headerType) throws IOException {
+    // Read the message size. There is an i32 little endian prefix.
     ByteBuffer buffer = ByteBuffer.allocate(4);
     if (in.readFully(buffer) != 4) {
       return null;
     }
 
-    int headerLength = bytesToInt(buffer.array());
-    buffer = ByteBuffer.allocate(headerLength);
-    if (in.readFully(buffer) != headerLength) {
+    int messageLength = bytesToInt(buffer.array());
+    buffer = ByteBuffer.allocate(messageLength);
+    if (in.readFully(buffer) != messageLength) {
       throw new IOException(
-          "Unexpected end of stream trying to read header.");
+          "Unexpected end of stream trying to read message.");
     }
     buffer.rewind();
 
-    Message header = Message.getRootAsMessage(buffer);
-    if (header.headerType() != headerType) {
+    Message message = Message.getRootAsMessage(buffer);
+    if (message.headerType() != headerType) {
       throw new IOException("Invalid message: expecting " + headerType +
-          ". Message contained: " + header.headerType());
+          ". Message contained: " + message.headerType());
     }
-    return header;
+    return message;
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index bf635fb39f5b8..9b9914480bad0 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -109,8 +109,6 @@ public void testWriteRead() throws IOException {
 
         List<ArrowBlock> recordBatches = footer.getRecordBatches();
         for (ArrowBlock rbBlock : recordBatches) {
-          Assert.assertEquals(0, rbBlock.getOffset() % 8);
-          Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
           try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
             List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
             for (ArrowBuffer arrowBuffer : buffersLayout) {
@@ -271,8 +269,6 @@ public void testWriteReadMultipleRBs() throws IOException {
         for (ArrowBlock rbBlock : recordBatches) {
           Assert.assertTrue(rbBlock.getOffset() + " > " + previousOffset, rbBlock.getOffset() > previousOffset);
           previousOffset = rbBlock.getOffset();
-          Assert.assertEquals(0, rbBlock.getOffset() % 8);
-          Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
           try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
             Assert.assertEquals("RB #" + i, counts[i], recordBatch.getLength());
             List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
index 707dba2af9898..1e514585e502f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
@@ -21,7 +21,9 @@
 import static org.junit.Assert.assertEquals;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 
 import org.apache.arrow.flatbuf.Footer;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -41,6 +43,12 @@ public void test() {
     ArrowFooter footer = new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), Collections.<ArrowBlock>emptyList());
     ArrowFooter newFooter = roundTrip(footer);
     assertEquals(footer, newFooter);
+
+    List<ArrowBlock> ids = new ArrayList<>();
+    ids.add(new ArrowBlock(0, 1, 2));
+    ids.add(new ArrowBlock(4, 5, 6));
+    footer = new ArrowFooter(schema, ids, ids);
+    assertEquals(footer, roundTrip(footer));
   }
 
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index 8ed89fa347b3b..96bcbb1dae71c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -24,10 +24,14 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.arrow.flatbuf.FieldNode;
+import org.apache.arrow.flatbuf.Message;
+import org.apache.arrow.flatbuf.RecordBatch;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
@@ -96,6 +100,18 @@ public void test() throws IOException {
       assertArrayEquals(validity, array(buffers.get(0)));
       assertArrayEquals(values, array(buffers.get(1)));
 
+      // Read just the header. This demonstrates being able to read without need to
+      // deserialize the buffer.
+      ByteBuffer headerBuffer = ByteBuffer.allocate(recordBatches.get(0).getMetadataLength());
+      headerBuffer.put(byteArray, (int)recordBatches.get(0).getOffset(), headerBuffer.capacity());
+      headerBuffer.position(4);
+      Message messageFB = Message.getRootAsMessage(headerBuffer);
+      RecordBatch recordBatchFB = (RecordBatch) messageFB.header(new RecordBatch());
+      assertEquals(2, recordBatchFB.buffersLength());
+      assertEquals(1, recordBatchFB.nodesLength());
+      FieldNode nodeFB = recordBatchFB.nodes(0);
+      assertEquals(16, nodeFB.length());
+      assertEquals(8, nodeFB.nullCount());
     }
   }
 

From 5888e10cffac222e359d1b440b4684d16c061085 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 21 Jan 2017 11:11:06 -0500
Subject: [PATCH 0285/1644] ARROW-495: [C++] Implement streaming binary format,
 refactoring

cc @nongli

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #293 from wesm/ARROW-495 and squashes the following commits:

279583b [Wes McKinney] FileBlock is a struct
c88e61a [Wes McKinney] Fix Python bindings after API changes
645a329 [Wes McKinney] Install stream.h
21378b4 [Wes McKinney] Collapse BaseStreamWriter and StreamWriter
b6c4578 [Wes McKinney] clang-format
12eb2cb [Wes McKinney] Add unit tests for streaming format, fix EOS, metadata length padding issues
3200b17 [Wes McKinney] Implement StreamReader
69fe82e [Wes McKinney] Implement rough draft of StreamWriter, share code with FileWriter
---
 cpp/CMakeLists.txt                         |   1 -
 cpp/src/arrow/io/memory.cc                 |   4 +-
 cpp/src/arrow/ipc/CMakeLists.txt           |   2 +
 cpp/src/arrow/ipc/adapter.cc               |  44 ++---
 cpp/src/arrow/ipc/adapter.h                |  11 +-
 cpp/src/arrow/ipc/file.cc                  | 167 ++++++++++++-----
 cpp/src/arrow/ipc/file.h                   |  54 +++---
 cpp/src/arrow/ipc/ipc-adapter-test.cc      |  16 +-
 cpp/src/arrow/ipc/ipc-file-test.cc         | 188 ++++++++++++++++---
 cpp/src/arrow/ipc/ipc-json-test.cc         |   5 +-
 cpp/src/arrow/ipc/ipc-metadata-test.cc     |  83 +--------
 cpp/src/arrow/ipc/json-integration-test.cc |   4 +-
 cpp/src/arrow/ipc/json.cc                  |  19 +-
 cpp/src/arrow/ipc/json.h                   |   3 +-
 cpp/src/arrow/ipc/metadata-internal.cc     |   8 +-
 cpp/src/arrow/ipc/metadata-internal.h      |   4 +-
 cpp/src/arrow/ipc/metadata.cc              | 121 +-----------
 cpp/src/arrow/ipc/metadata.h               |  32 +---
 cpp/src/arrow/ipc/stream.cc                | 206 +++++++++++++++++++++
 cpp/src/arrow/ipc/stream.h                 | 112 +++++++++++
 cpp/src/arrow/ipc/test-common.h            |   9 +
 python/pyarrow/includes/libarrow_ipc.pxd   |   3 +-
 python/pyarrow/ipc.pyx                     |   5 +-
 python/src/pyarrow/adapters/pandas.cc      |  34 ++--
 24 files changed, 718 insertions(+), 417 deletions(-)
 create mode 100644 cpp/src/arrow/ipc/stream.cc
 create mode 100644 cpp/src/arrow/ipc/stream.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 885ab19256065..9039ffb571b9e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -90,7 +90,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(ARROW_ALTIVEC
     "Build Arrow with Altivec"
     ON)
-
 endif()
 
 if(NOT ARROW_BUILD_TESTS)
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 0f5a0dc06979c..1339a99aa787e 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -116,13 +116,13 @@ Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer)
 Status BufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   int64_t size = std::min(nbytes, size_ - position_);
 
-  if (buffer_ != nullptr) {
+  if (size > 0 && buffer_ != nullptr) {
     *out = SliceBuffer(buffer_, position_, size);
   } else {
     *out = std::make_shared<Buffer>(data_ + position_, size);
   }
 
-  position_ += nbytes;
+  position_ += size;
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index b7ac5f059749f..c047f53d6bf06 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -46,6 +46,7 @@ set(ARROW_IPC_SRCS
   json-internal.cc
   metadata.cc
   metadata-internal.cc
+  stream.cc
 )
 
 if(NOT APPLE)
@@ -151,6 +152,7 @@ install(FILES
   file.h
   json.h
   metadata.h
+  stream.h
   DESTINATION include/arrow/ipc)
 
 # pkg-config support
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 7b4d18c267d43..9da7b3912d4bc 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -49,10 +49,9 @@ namespace ipc {
 
 class RecordBatchWriter : public ArrayVisitor {
  public:
-  RecordBatchWriter(const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows,
-      int64_t buffer_start_offset, int max_recursion_depth)
-      : columns_(columns),
-        num_rows_(num_rows),
+  RecordBatchWriter(
+      const RecordBatch& batch, int64_t buffer_start_offset, int max_recursion_depth)
+      : batch_(batch),
         max_recursion_depth_(max_recursion_depth),
         buffer_start_offset_(buffer_start_offset) {}
 
@@ -79,8 +78,8 @@ class RecordBatchWriter : public ArrayVisitor {
     }
 
     // Perform depth-first traversal of the row-batch
-    for (size_t i = 0; i < columns_.size(); ++i) {
-      RETURN_NOT_OK(VisitArray(*columns_[i].get()));
+    for (int i = 0; i < batch_.num_columns(); ++i) {
+      RETURN_NOT_OK(VisitArray(*batch_.column(i)));
     }
 
     // The position for the start of a buffer relative to the passed frame of
@@ -126,18 +125,23 @@ class RecordBatchWriter : public ArrayVisitor {
     // itself as an int32_t.
     std::shared_ptr<Buffer> metadata_fb;
     RETURN_NOT_OK(WriteRecordBatchMetadata(
-        num_rows_, body_length, field_nodes_, buffer_meta_, &metadata_fb));
+        batch_.num_rows(), body_length, field_nodes_, buffer_meta_, &metadata_fb));
 
     // Need to write 4 bytes (metadata size), the metadata, plus padding to
-    // fall on an 8-byte offset
-    int64_t padded_metadata_length = BitUtil::CeilByte(metadata_fb->size() + 4);
+    // end on an 8-byte offset
+    int64_t start_offset;
+    RETURN_NOT_OK(dst->Tell(&start_offset));
+
+    int64_t padded_metadata_length = metadata_fb->size() + 4;
+    const int remainder = (padded_metadata_length + start_offset) % 8;
+    if (remainder != 0) { padded_metadata_length += 8 - remainder; }
 
     // The returned metadata size includes the length prefix, the flatbuffer,
     // plus padding
     *metadata_length = static_cast<int32_t>(padded_metadata_length);
 
-    // Write the flatbuffer size prefix
-    int32_t flatbuffer_size = metadata_fb->size();
+    // Write the flatbuffer size prefix including padding
+    int32_t flatbuffer_size = padded_metadata_length - 4;
     RETURN_NOT_OK(
         dst->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
 
@@ -294,9 +298,7 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  // Do not copy this vector. Ownership must be retained elsewhere
-  const std::vector<std::shared_ptr<Array>>& columns_;
-  int32_t num_rows_;
+  const RecordBatch& batch_;
 
   std::vector<flatbuf::FieldNode> field_nodes_;
   std::vector<flatbuf::Buffer> buffer_meta_;
@@ -306,18 +308,16 @@ class RecordBatchWriter : public ArrayVisitor {
   int64_t buffer_start_offset_;
 };
 
-Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
-    int32_t num_rows, int64_t buffer_start_offset, io::OutputStream* dst,
-    int32_t* metadata_length, int64_t* body_length, int max_recursion_depth) {
+Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+    int max_recursion_depth) {
   DCHECK_GT(max_recursion_depth, 0);
-  RecordBatchWriter serializer(
-      columns, num_rows, buffer_start_offset, max_recursion_depth);
+  RecordBatchWriter serializer(batch, buffer_start_offset, max_recursion_depth);
   return serializer.Write(dst, metadata_length, body_length);
 }
 
-Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size) {
-  RecordBatchWriter serializer(
-      batch->columns(), batch->num_rows(), 0, kMaxIpcRecursionDepth);
+Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
+  RecordBatchWriter serializer(batch, 0, kMaxIpcRecursionDepth);
   RETURN_NOT_OK(serializer.GetTotalSize(size));
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 963b9ee368537..f9ef7d9fe1202 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -71,17 +71,14 @@ constexpr int kMaxIpcRecursionDepth = 64;
 //
 // @param(out) body_length: the size of the contiguous buffer block plus
 // padding bytes
-ARROW_EXPORT Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
-    int32_t num_rows, int64_t buffer_start_offset, io::OutputStream* dst,
-    int32_t* metadata_length, int64_t* body_length,
-    int max_recursion_depth = kMaxIpcRecursionDepth);
-
-// int64_t GetRecordBatchMetadata(const RecordBatch* batch);
+ARROW_EXPORT Status WriteRecordBatch(const RecordBatch& batch,
+    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
+    int64_t* body_length, int max_recursion_depth = kMaxIpcRecursionDepth);
 
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the record batch. This involves generating the complete serialized
 // Flatbuffers metadata.
-ARROW_EXPORT Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size);
+ARROW_EXPORT Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 
 // ----------------------------------------------------------------------
 // "Read" path; does not copy data if the input supports zero copy reads
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
index d7d2e613f87db..bc086e31519a5 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/file.cc
@@ -26,6 +26,7 @@
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/adapter.h"
+#include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
 #include "arrow/status.h"
@@ -35,82 +36,154 @@ namespace arrow {
 namespace ipc {
 
 static constexpr const char* kArrowMagicBytes = "ARROW1";
-
 // ----------------------------------------------------------------------
-// Writer implementation
+// File footer
+
+static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
+FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
+  std::vector<flatbuf::Block> fb_blocks;
 
-FileWriter::FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
-    : sink_(sink), schema_(schema), position_(-1), started_(false) {}
+  for (const FileBlock& block : blocks) {
+    fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
+  }
 
-Status FileWriter::UpdatePosition() {
-  return sink_->Tell(&position_);
+  return fbb.CreateVectorOfStructs(fb_blocks);
 }
 
-Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-    std::shared_ptr<FileWriter>* out) {
-  *out = std::shared_ptr<FileWriter>(new FileWriter(sink, schema));  // ctor is private
-  RETURN_NOT_OK((*out)->UpdatePosition());
-  return Status::OK();
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+    const std::vector<FileBlock>& record_batches, io::OutputStream* out) {
+  FBB fbb;
+
+  flatbuffers::Offset<flatbuf::Schema> fb_schema;
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, &fb_schema));
+
+  auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
+  auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
+
+  auto footer = flatbuf::CreateFooter(
+      fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
+
+  fbb.Finish(footer);
+
+  int32_t size = fbb.GetSize();
+
+  return out->Write(fbb.GetBufferPointer(), size);
 }
 
-Status FileWriter::Write(const uint8_t* data, int64_t nbytes) {
-  RETURN_NOT_OK(sink_->Write(data, nbytes));
-  position_ += nbytes;
-  return Status::OK();
+static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
+  return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
 }
 
-Status FileWriter::Align() {
-  int64_t remainder = PaddedLength(position_) - position_;
-  if (remainder > 0) { return Write(kPaddingBytes, remainder); }
+class FileFooter::FileFooterImpl {
+ public:
+  FileFooterImpl(const std::shared_ptr<Buffer>& buffer, const flatbuf::Footer* footer)
+      : buffer_(buffer), footer_(footer) {}
+
+  int num_dictionaries() const { return footer_->dictionaries()->size(); }
+
+  int num_record_batches() const { return footer_->recordBatches()->size(); }
+
+  MetadataVersion::type version() const {
+    switch (footer_->version()) {
+      case flatbuf::MetadataVersion_V1:
+        return MetadataVersion::V1;
+      case flatbuf::MetadataVersion_V2:
+        return MetadataVersion::V2;
+      // Add cases as other versions become available
+      default:
+        return MetadataVersion::V2;
+    }
+  }
+
+  FileBlock record_batch(int i) const {
+    return FileBlockFromFlatbuffer(footer_->recordBatches()->Get(i));
+  }
+
+  FileBlock dictionary(int i) const {
+    return FileBlockFromFlatbuffer(footer_->dictionaries()->Get(i));
+  }
+
+  Status GetSchema(std::shared_ptr<Schema>* out) const {
+    auto schema_msg = std::make_shared<SchemaMetadata>(nullptr, footer_->schema());
+    return schema_msg->GetSchema(out);
+  }
+
+ private:
+  // Retain reference to memory
+  std::shared_ptr<Buffer> buffer_;
+
+  const flatbuf::Footer* footer_;
+};
+
+FileFooter::FileFooter() {}
+
+FileFooter::~FileFooter() {}
+
+Status FileFooter::Open(
+    const std::shared_ptr<Buffer>& buffer, std::unique_ptr<FileFooter>* out) {
+  const flatbuf::Footer* footer = flatbuf::GetFooter(buffer->data());
+
+  *out = std::unique_ptr<FileFooter>(new FileFooter());
+
+  // TODO(wesm): Verify the footer
+  (*out)->impl_.reset(new FileFooterImpl(buffer, footer));
+
   return Status::OK();
 }
 
-Status FileWriter::WriteAligned(const uint8_t* data, int64_t nbytes) {
-  RETURN_NOT_OK(Write(data, nbytes));
-  return Align();
+int FileFooter::num_dictionaries() const {
+  return impl_->num_dictionaries();
 }
 
-Status FileWriter::Start() {
-  RETURN_NOT_OK(WriteAligned(
-      reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
-  started_ = true;
-  return Status::OK();
+int FileFooter::num_record_batches() const {
+  return impl_->num_record_batches();
 }
 
-Status FileWriter::CheckStarted() {
-  if (!started_) { return Start(); }
-  return Status::OK();
+MetadataVersion::type FileFooter::version() const {
+  return impl_->version();
 }
 
-Status FileWriter::WriteRecordBatch(
-    const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows) {
-  RETURN_NOT_OK(CheckStarted());
-
-  int64_t offset = position_;
+FileBlock FileFooter::record_batch(int i) const {
+  return impl_->record_batch(i);
+}
 
-  // There may be padding ever the end of the metadata, so we cannot rely on
-  // position_
-  int32_t metadata_length;
-  int64_t body_length;
+FileBlock FileFooter::dictionary(int i) const {
+  return impl_->dictionary(i);
+}
 
-  // Frame of reference in file format is 0, see ARROW-384
-  const int64_t buffer_start_offset = 0;
-  RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(
-      columns, num_rows, buffer_start_offset, sink_, &metadata_length, &body_length));
-  RETURN_NOT_OK(UpdatePosition());
+Status FileFooter::GetSchema(std::shared_ptr<Schema>* out) const {
+  return impl_->GetSchema(out);
+}
 
-  DCHECK(position_ % 8 == 0) << "ipc::WriteRecordBatch did not perform aligned writes";
+// ----------------------------------------------------------------------
+// File writer implementation
 
-  // Append metadata, to be written in the footer later
-  record_batches_.emplace_back(offset, metadata_length, body_length);
+Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    std::shared_ptr<FileWriter>* out) {
+  *out = std::shared_ptr<FileWriter>(new FileWriter(sink, schema));  // ctor is private
+  RETURN_NOT_OK((*out)->UpdatePosition());
+  return Status::OK();
+}
 
+Status FileWriter::Start() {
+  RETURN_NOT_OK(WriteAligned(
+      reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
+  started_ = true;
   return Status::OK();
 }
 
+Status FileWriter::WriteRecordBatch(const RecordBatch& batch) {
+  // Push an empty FileBlock
+  // Append metadata, to be written in the footer later
+  record_batches_.emplace_back(0, 0, 0);
+  return StreamWriter::WriteRecordBatch(
+      batch, &record_batches_[record_batches_.size() - 1]);
+}
+
 Status FileWriter::Close() {
   // Write metadata
   int64_t initial_position = position_;
-  RETURN_NOT_OK(WriteFileFooter(schema_.get(), dictionaries_, record_batches_, sink_));
+  RETURN_NOT_OK(WriteFileFooter(*schema_, dictionaries_, record_batches_, sink_));
   RETURN_NOT_OK(UpdatePosition());
 
   // Write footer length
diff --git a/cpp/src/arrow/ipc/file.h b/cpp/src/arrow/ipc/file.h
index 4f35c37b03235..7696954c188e3 100644
--- a/cpp/src/arrow/ipc/file.h
+++ b/cpp/src/arrow/ipc/file.h
@@ -25,13 +25,12 @@
 #include <vector>
 
 #include "arrow/ipc/metadata.h"
+#include "arrow/ipc/stream.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
-class Array;
 class Buffer;
-struct Field;
 class RecordBatch;
 class Schema;
 class Status;
@@ -45,40 +44,43 @@ class ReadableFileInterface;
 
 namespace ipc {
 
-class ARROW_EXPORT FileWriter {
- public:
-  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-      std::shared_ptr<FileWriter>* out);
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+    const std::vector<FileBlock>& record_batches, io::OutputStream* out);
 
-  // TODO(wesm): Write dictionaries
+class ARROW_EXPORT FileFooter {
+ public:
+  ~FileFooter();
 
-  Status WriteRecordBatch(
-      const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows);
+  static Status Open(
+      const std::shared_ptr<Buffer>& buffer, std::unique_ptr<FileFooter>* out);
 
-  Status Close();
+  int num_dictionaries() const;
+  int num_record_batches() const;
+  MetadataVersion::type version() const;
 
- private:
-  FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema);
+  FileBlock record_batch(int i) const;
+  FileBlock dictionary(int i) const;
 
-  Status CheckStarted();
-  Status Start();
+  Status GetSchema(std::shared_ptr<Schema>* out) const;
 
-  Status UpdatePosition();
+ private:
+  FileFooter();
+  class FileFooterImpl;
+  std::unique_ptr<FileFooterImpl> impl_;
+};
 
-  // Adds padding bytes if necessary to ensure all memory blocks are written on
-  // 8-byte boundaries.
-  Status Align();
+class ARROW_EXPORT FileWriter : public StreamWriter {
+ public:
+  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+      std::shared_ptr<FileWriter>* out);
 
-  // Write data and update position
-  Status Write(const uint8_t* data, int64_t nbytes);
+  Status WriteRecordBatch(const RecordBatch& batch) override;
+  Status Close() override;
 
-  // Write and align
-  Status WriteAligned(const uint8_t* data, int64_t nbytes);
+ private:
+  using StreamWriter::StreamWriter;
 
-  io::OutputStream* sink_;
-  std::shared_ptr<Schema> schema_;
-  int64_t position_;
-  bool started_;
+  Status Start() override;
 
   std::vector<FileBlock> dictionaries_;
   std::vector<FileBlock> record_batches_;
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 6ba0a6e16be08..17868f8f1029e 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -55,8 +55,8 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
 
     const int64_t buffer_offset = 0;
 
-    RETURN_NOT_OK(WriteRecordBatch(batch.columns(), batch.num_rows(), buffer_offset,
-        mmap_.get(), &metadata_length, &body_length));
+    RETURN_NOT_OK(WriteRecordBatch(
+        batch, buffer_offset, mmap_.get(), &metadata_length, &body_length));
 
     std::shared_ptr<RecordBatchMetadata> metadata;
     RETURN_NOT_OK(ReadRecordBatchMetadata(0, metadata_length, mmap_.get(), &metadata));
@@ -102,9 +102,8 @@ void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   int32_t mock_metadata_length = -1;
   int64_t mock_body_length = -1;
   int64_t size = -1;
-  ASSERT_OK(WriteRecordBatch(batch->columns(), batch->num_rows(), 0, &mock,
-      &mock_metadata_length, &mock_body_length));
-  ASSERT_OK(GetRecordBatchSize(batch.get(), &size));
+  ASSERT_OK(WriteRecordBatch(*batch, 0, &mock, &mock_metadata_length, &mock_body_length));
+  ASSERT_OK(GetRecordBatchSize(*batch, &size));
   ASSERT_EQ(mock.GetExtentBytesWritten(), size);
 }
 
@@ -157,11 +156,10 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
     io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
 
     if (override_level) {
-      return WriteRecordBatch(batch->columns(), batch->num_rows(), 0, mmap_.get(),
-          metadata_length, body_length, recursion_level + 1);
+      return WriteRecordBatch(
+          *batch, 0, mmap_.get(), metadata_length, body_length, recursion_level + 1);
     } else {
-      return WriteRecordBatch(batch->columns(), batch->num_rows(), 0, mmap_.get(),
-          metadata_length, body_length);
+      return WriteRecordBatch(*batch, 0, mmap_.get(), metadata_length, body_length);
     }
   }
 
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index 0a9f677966389..15ceb80493632 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -29,6 +29,7 @@
 #include "arrow/io/test-common.h"
 #include "arrow/ipc/adapter.h"
 #include "arrow/ipc/file.h"
+#include "arrow/ipc/stream.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/ipc/util.h"
 
@@ -41,6 +42,19 @@
 namespace arrow {
 namespace ipc {
 
+void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
+  ASSERT_TRUE(left.schema()->Equals(right.schema()));
+  ASSERT_EQ(left.num_columns(), right.num_columns())
+      << left.schema()->ToString() << " result: " << right.schema()->ToString();
+  EXPECT_EQ(left.num_rows(), right.num_rows());
+  for (int i = 0; i < left.num_columns(); ++i) {
+    EXPECT_TRUE(left.column(i)->Equals(right.column(i)))
+        << "Idx: " << i << " Name: " << left.column_name(i);
+  }
+}
+
+using BatchVector = std::vector<std::shared_ptr<RecordBatch>>;
+
 class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
  public:
   void SetUp() {
@@ -50,43 +64,94 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
   }
   void TearDown() {}
 
-  Status RoundTripHelper(
-      const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
+  Status RoundTripHelper(const BatchVector& in_batches, BatchVector* out_batches) {
     // Write the file
-    RETURN_NOT_OK(FileWriter::Open(sink_.get(), batch.schema(), &file_writer_));
-    int num_batches = 3;
-    for (int i = 0; i < num_batches; ++i) {
-      RETURN_NOT_OK(file_writer_->WriteRecordBatch(batch.columns(), batch.num_rows()));
+    std::shared_ptr<FileWriter> writer;
+    RETURN_NOT_OK(FileWriter::Open(sink_.get(), in_batches[0]->schema(), &writer));
+
+    const int num_batches = static_cast<int>(in_batches.size());
+
+    for (const auto& batch : in_batches) {
+      RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
     }
-    RETURN_NOT_OK(file_writer_->Close());
+    RETURN_NOT_OK(writer->Close());
 
     // Current offset into stream is the end of the file
     int64_t footer_offset;
     RETURN_NOT_OK(sink_->Tell(&footer_offset));
 
     // Open the file
-    auto reader = std::make_shared<io::BufferReader>(buffer_);
-    RETURN_NOT_OK(FileReader::Open(reader, footer_offset, &file_reader_));
+    auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+    std::shared_ptr<FileReader> reader;
+    RETURN_NOT_OK(FileReader::Open(buf_reader, footer_offset, &reader));
 
-    EXPECT_EQ(num_batches, file_reader_->num_record_batches());
-
-    out_batches->resize(num_batches);
+    EXPECT_EQ(num_batches, reader->num_record_batches());
     for (int i = 0; i < num_batches; ++i) {
-      RETURN_NOT_OK(file_reader_->GetRecordBatch(i, &(*out_batches)[i]));
+      std::shared_ptr<RecordBatch> chunk;
+      RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
+      out_batches->emplace_back(chunk);
     }
 
     return Status::OK();
   }
 
-  void CompareBatch(const RecordBatch* left, const RecordBatch* right) {
-    ASSERT_TRUE(left->schema()->Equals(right->schema()));
-    ASSERT_EQ(left->num_columns(), right->num_columns())
-        << left->schema()->ToString() << " result: " << right->schema()->ToString();
-    EXPECT_EQ(left->num_rows(), right->num_rows());
-    for (int i = 0; i < left->num_columns(); ++i) {
-      EXPECT_TRUE(left->column(i)->Equals(right->column(i)))
-          << "Idx: " << i << " Name: " << left->column_name(i);
+ protected:
+  MemoryPool* pool_;
+
+  std::unique_ptr<io::BufferOutputStream> sink_;
+  std::shared_ptr<PoolBuffer> buffer_;
+};
+
+TEST_P(TestFileFormat, RoundTrip) {
+  std::shared_ptr<RecordBatch> batch1;
+  std::shared_ptr<RecordBatch> batch2;
+  ASSERT_OK((*GetParam())(&batch1));  // NOLINT clang-tidy gtest issue
+  ASSERT_OK((*GetParam())(&batch2));  // NOLINT clang-tidy gtest issue
+
+  std::vector<std::shared_ptr<RecordBatch>> in_batches = {batch1, batch2};
+  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+
+  ASSERT_OK(RoundTripHelper(in_batches, &out_batches));
+
+  // Compare batches
+  for (size_t i = 0; i < in_batches.size(); ++i) {
+    CompareBatch(*in_batches[i], *out_batches[i]);
+  }
+}
+
+class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+    buffer_ = std::make_shared<PoolBuffer>(pool_);
+    sink_.reset(new io::BufferOutputStream(buffer_));
+  }
+  void TearDown() {}
+
+  Status RoundTripHelper(
+      const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
+    // Write the file
+    std::shared_ptr<StreamWriter> writer;
+    RETURN_NOT_OK(StreamWriter::Open(sink_.get(), batch.schema(), &writer));
+    int num_batches = 5;
+    for (int i = 0; i < num_batches; ++i) {
+      RETURN_NOT_OK(writer->WriteRecordBatch(batch));
+    }
+    RETURN_NOT_OK(writer->Close());
+
+    // Open the file
+    auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+
+    std::shared_ptr<StreamReader> reader;
+    RETURN_NOT_OK(StreamReader::Open(buf_reader, &reader));
+
+    std::shared_ptr<RecordBatch> chunk;
+    while (true) {
+      RETURN_NOT_OK(reader->GetNextRecordBatch(&chunk));
+      if (chunk == nullptr) { break; }
+      out_batches->emplace_back(chunk);
     }
+    return Status::OK();
   }
 
  protected:
@@ -94,12 +159,9 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
 
   std::unique_ptr<io::BufferOutputStream> sink_;
   std::shared_ptr<PoolBuffer> buffer_;
-
-  std::shared_ptr<FileWriter> file_writer_;
-  std::shared_ptr<FileReader> file_reader_;
 };
 
-TEST_P(TestFileFormat, RoundTrip) {
+TEST_P(TestStreamFormat, RoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
 
@@ -109,14 +171,80 @@ TEST_P(TestFileFormat, RoundTrip) {
 
   // Compare batches. Same
   for (size_t i = 0; i < out_batches.size(); ++i) {
-    CompareBatch(batch.get(), out_batches[i].get());
+    CompareBatch(*batch, *out_batches[i]);
   }
 }
 
-INSTANTIATE_TEST_CASE_P(RoundTripTests, TestFileFormat,
-    ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch,
-                            &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,
-                            &MakeStringTypesRecordBatch, &MakeStruct));
+#define BATCH_CASES()                                                                   \
+  ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
+      &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
+      &MakeStruct);
+
+INSTANTIATE_TEST_CASE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
+INSTANTIATE_TEST_CASE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
+
+class TestFileFooter : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  void CheckRoundtrip(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+      const std::vector<FileBlock>& record_batches) {
+    auto buffer = std::make_shared<PoolBuffer>();
+    io::BufferOutputStream stream(buffer);
+
+    ASSERT_OK(WriteFileFooter(schema, dictionaries, record_batches, &stream));
+
+    std::unique_ptr<FileFooter> footer;
+    ASSERT_OK(FileFooter::Open(buffer, &footer));
+
+    ASSERT_EQ(MetadataVersion::V2, footer->version());
+
+    // Check schema
+    std::shared_ptr<Schema> schema2;
+    ASSERT_OK(footer->GetSchema(&schema2));
+    AssertSchemaEqual(schema, *schema2);
+
+    // Check blocks
+    ASSERT_EQ(dictionaries.size(), footer->num_dictionaries());
+    ASSERT_EQ(record_batches.size(), footer->num_record_batches());
+
+    for (int i = 0; i < footer->num_dictionaries(); ++i) {
+      CheckBlocks(dictionaries[i], footer->dictionary(i));
+    }
+
+    for (int i = 0; i < footer->num_record_batches(); ++i) {
+      CheckBlocks(record_batches[i], footer->record_batch(i));
+    }
+  }
+
+  void CheckBlocks(const FileBlock& left, const FileBlock& right) {
+    ASSERT_EQ(left.offset, right.offset);
+    ASSERT_EQ(left.metadata_length, right.metadata_length);
+    ASSERT_EQ(left.body_length, right.body_length);
+  }
+
+ private:
+  std::shared_ptr<Schema> example_schema_;
+};
+
+TEST_F(TestFileFooter, Basics) {
+  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>());
+  Schema schema({f0, f1});
+
+  std::vector<FileBlock> dictionaries;
+  dictionaries.emplace_back(8, 92, 900);
+  dictionaries.emplace_back(1000, 100, 1900);
+  dictionaries.emplace_back(3000, 100, 2900);
+
+  std::vector<FileBlock> record_batches;
+  record_batches.emplace_back(6000, 100, 900);
+  record_batches.emplace_back(7000, 100, 1900);
+  record_batches.emplace_back(9000, 100, 2900);
+  record_batches.emplace_back(12000, 100, 3900);
+
+  CheckRoundtrip(schema, dictionaries, record_batches);
+}
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 07509890da35c..30f968c2bfd8b 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -245,8 +245,9 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
     std::vector<std::shared_ptr<Array>> arrays;
 
     MakeBatchArrays(schema, num_rows, &arrays);
-    batches.emplace_back(std::make_shared<RecordBatch>(schema, num_rows, arrays));
-    ASSERT_OK(writer->WriteRecordBatch(arrays, num_rows));
+    auto batch = std::make_shared<RecordBatch>(schema, num_rows, arrays);
+    batches.push_back(batch);
+    ASSERT_OK(writer->WriteRecordBatch(*batch));
   }
 
   std::string result;
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
index 7c5744a241068..098f996d292a2 100644
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -23,6 +23,7 @@
 
 #include "arrow/io/memory.h"
 #include "arrow/ipc/metadata.h"
+#include "arrow/ipc/test-common.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/test-util.h"
@@ -34,20 +35,11 @@ class Buffer;
 
 namespace ipc {
 
-static inline void assert_schema_equal(const Schema* lhs, const Schema* rhs) {
-  if (!lhs->Equals(*rhs)) {
-    std::stringstream ss;
-    ss << "left schema: " << lhs->ToString() << std::endl
-       << "right schema: " << rhs->ToString() << std::endl;
-    FAIL() << ss.str();
-  }
-}
-
 class TestSchemaMetadata : public ::testing::Test {
  public:
   void SetUp() {}
 
-  void CheckRoundtrip(const Schema* schema) {
+  void CheckRoundtrip(const Schema& schema) {
     std::shared_ptr<Buffer> buffer;
     ASSERT_OK(WriteSchema(schema, &buffer));
 
@@ -57,12 +49,12 @@ class TestSchemaMetadata : public ::testing::Test {
     ASSERT_EQ(Message::SCHEMA, message->type());
 
     auto schema_msg = std::make_shared<SchemaMetadata>(message);
-    ASSERT_EQ(schema->num_fields(), schema_msg->num_fields());
+    ASSERT_EQ(schema.num_fields(), schema_msg->num_fields());
 
     std::shared_ptr<Schema> schema2;
     ASSERT_OK(schema_msg->GetSchema(&schema2));
 
-    assert_schema_equal(schema, schema2.get());
+    AssertSchemaEqual(schema, *schema2);
   }
 };
 
@@ -82,7 +74,7 @@ TEST_F(TestSchemaMetadata, PrimitiveFields) {
   auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
 
   Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
-  CheckRoundtrip(&schema);
+  CheckRoundtrip(schema);
 }
 
 TEST_F(TestSchemaMetadata, NestedFields) {
@@ -94,70 +86,7 @@ TEST_F(TestSchemaMetadata, NestedFields) {
   auto f1 = std::make_shared<Field>("f1", type2);
 
   Schema schema({f0, f1});
-  CheckRoundtrip(&schema);
-}
-
-class TestFileFooter : public ::testing::Test {
- public:
-  void SetUp() {}
-
-  void CheckRoundtrip(const Schema* schema, const std::vector<FileBlock>& dictionaries,
-      const std::vector<FileBlock>& record_batches) {
-    auto buffer = std::make_shared<PoolBuffer>();
-    io::BufferOutputStream stream(buffer);
-
-    ASSERT_OK(WriteFileFooter(schema, dictionaries, record_batches, &stream));
-
-    std::unique_ptr<FileFooter> footer;
-    ASSERT_OK(FileFooter::Open(buffer, &footer));
-
-    ASSERT_EQ(MetadataVersion::V2, footer->version());
-
-    // Check schema
-    std::shared_ptr<Schema> schema2;
-    ASSERT_OK(footer->GetSchema(&schema2));
-    assert_schema_equal(schema, schema2.get());
-
-    // Check blocks
-    ASSERT_EQ(dictionaries.size(), footer->num_dictionaries());
-    ASSERT_EQ(record_batches.size(), footer->num_record_batches());
-
-    for (int i = 0; i < footer->num_dictionaries(); ++i) {
-      CheckBlocks(dictionaries[i], footer->dictionary(i));
-    }
-
-    for (int i = 0; i < footer->num_record_batches(); ++i) {
-      CheckBlocks(record_batches[i], footer->record_batch(i));
-    }
-  }
-
-  void CheckBlocks(const FileBlock& left, const FileBlock& right) {
-    ASSERT_EQ(left.offset, right.offset);
-    ASSERT_EQ(left.metadata_length, right.metadata_length);
-    ASSERT_EQ(left.body_length, right.body_length);
-  }
-
- private:
-  std::shared_ptr<Schema> example_schema_;
-};
-
-TEST_F(TestFileFooter, Basics) {
-  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>());
-  Schema schema({f0, f1});
-
-  std::vector<FileBlock> dictionaries;
-  dictionaries.emplace_back(8, 92, 900);
-  dictionaries.emplace_back(1000, 100, 1900);
-  dictionaries.emplace_back(3000, 100, 2900);
-
-  std::vector<FileBlock> record_batches;
-  record_batches.emplace_back(6000, 100, 900);
-  record_batches.emplace_back(7000, 100, 1900);
-  record_batches.emplace_back(9000, 100, 2900);
-  record_batches.emplace_back(12000, 100, 3900);
-
-  CheckRoundtrip(&schema, dictionaries, record_batches);
+  CheckRoundtrip(schema);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 757e6c00ab243..95bc742054fab 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -81,7 +81,7 @@ static Status ConvertJsonToArrow(
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
     RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
-    RETURN_NOT_OK(writer->WriteRecordBatch(batch->columns(), batch->num_rows()));
+    RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
   }
   return writer->Close();
 }
@@ -108,7 +108,7 @@ static Status ConvertArrowToJson(
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
     RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
-    RETURN_NOT_OK(writer->WriteRecordBatch(batch->columns(), batch->num_rows()));
+    RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
   }
 
   std::string result;
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index 6e3a9939730f4..773fb74a1767a 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -64,25 +64,23 @@ class JsonWriter::JsonWriterImpl {
     return Status::OK();
   }
 
-  Status WriteRecordBatch(
-      const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows) {
-    DCHECK_EQ(static_cast<int>(columns.size()), schema_->num_fields());
+  Status WriteRecordBatch(const RecordBatch& batch) {
+    DCHECK_EQ(batch.num_columns(), schema_->num_fields());
 
     writer_->StartObject();
     writer_->Key("count");
-    writer_->Int(num_rows);
+    writer_->Int(batch.num_rows());
 
     writer_->Key("columns");
     writer_->StartArray();
 
     for (int i = 0; i < schema_->num_fields(); ++i) {
-      const std::shared_ptr<Array>& column = columns[i];
+      const std::shared_ptr<Array>& column = batch.column(i);
 
-      DCHECK_EQ(num_rows, column->length())
+      DCHECK_EQ(batch.num_rows(), column->length())
           << "Array length did not match record batch length";
 
-      RETURN_NOT_OK(
-          WriteJsonArray(schema_->field(i)->name, *column.get(), writer_.get()));
+      RETURN_NOT_OK(WriteJsonArray(schema_->field(i)->name, *column, writer_.get()));
     }
 
     writer_->EndArray();
@@ -113,9 +111,8 @@ Status JsonWriter::Finish(std::string* result) {
   return impl_->Finish(result);
 }
 
-Status JsonWriter::WriteRecordBatch(
-    const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows) {
-  return impl_->WriteRecordBatch(columns, num_rows);
+Status JsonWriter::WriteRecordBatch(const RecordBatch& batch) {
+  return impl_->WriteRecordBatch(batch);
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
index 7395be43b967d..88afdfaa5ff3b 100644
--- a/cpp/src/arrow/ipc/json.h
+++ b/cpp/src/arrow/ipc/json.h
@@ -46,8 +46,7 @@ class ARROW_EXPORT JsonWriter {
 
   // TODO(wesm): Write dictionaries
 
-  Status WriteRecordBatch(
-      const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows);
+  Status WriteRecordBatch(const RecordBatch& batch);
 
   Status Finish(std::string* result);
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index cc160c42ec9ef..cd7722056a3c7 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -282,10 +282,10 @@ flatbuf::Endianness endianness() {
 }
 
 Status SchemaToFlatbuffer(
-    FBB& fbb, const Schema* schema, flatbuffers::Offset<flatbuf::Schema>* out) {
+    FBB& fbb, const Schema& schema, flatbuffers::Offset<flatbuf::Schema>* out) {
   std::vector<FieldOffset> field_offsets;
-  for (int i = 0; i < schema->num_fields(); ++i) {
-    std::shared_ptr<Field> field = schema->field(i);
+  for (int i = 0; i < schema.num_fields(); ++i) {
+    std::shared_ptr<Field> field = schema.field(i);
     FieldOffset offset;
     RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, &offset));
     field_offsets.push_back(offset);
@@ -295,7 +295,7 @@ Status SchemaToFlatbuffer(
   return Status::OK();
 }
 
-Status MessageBuilder::SetSchema(const Schema* schema) {
+Status MessageBuilder::SetSchema(const Schema& schema) {
   flatbuffers::Offset<flatbuf::Schema> fb_schema;
   RETURN_NOT_OK(SchemaToFlatbuffer(fbb_, schema, &fb_schema));
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index 4826ebe22899d..d94a8abc99ab0 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -49,11 +49,11 @@ static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVe
 Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out);
 
 Status SchemaToFlatbuffer(
-    FBB& fbb, const Schema* schema, flatbuffers::Offset<flatbuf::Schema>* out);
+    FBB& fbb, const Schema& schema, flatbuffers::Offset<flatbuf::Schema>* out);
 
 class MessageBuilder {
  public:
-  Status SetSchema(const Schema* schema);
+  Status SetSchema(const Schema& schema);
 
   Status SetRecordBatch(int32_t length, int64_t body_length,
       const std::vector<flatbuf::FieldNode>& nodes,
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index f0674ff8d5aeb..a97965c40d608 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -38,7 +38,7 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
-Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out) {
+Status WriteSchema(const Schema& schema, std::shared_ptr<Buffer>* out) {
   MessageBuilder message;
   RETURN_NOT_OK(message.SetSchema(schema));
   RETURN_NOT_OK(message.Finish());
@@ -232,124 +232,5 @@ int RecordBatchMetadata::num_fields() const {
   return impl_->num_fields();
 }
 
-// ----------------------------------------------------------------------
-// File footer
-
-static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
-FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
-  std::vector<flatbuf::Block> fb_blocks;
-
-  for (const FileBlock& block : blocks) {
-    fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
-  }
-
-  return fbb.CreateVectorOfStructs(fb_blocks);
-}
-
-Status WriteFileFooter(const Schema* schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, io::OutputStream* out) {
-  FBB fbb;
-
-  flatbuffers::Offset<flatbuf::Schema> fb_schema;
-  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, &fb_schema));
-
-  auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
-  auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
-
-  auto footer = flatbuf::CreateFooter(
-      fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
-
-  fbb.Finish(footer);
-
-  int32_t size = fbb.GetSize();
-
-  return out->Write(fbb.GetBufferPointer(), size);
-}
-
-static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
-  return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
-}
-
-class FileFooter::FileFooterImpl {
- public:
-  FileFooterImpl(const std::shared_ptr<Buffer>& buffer, const flatbuf::Footer* footer)
-      : buffer_(buffer), footer_(footer) {}
-
-  int num_dictionaries() const { return footer_->dictionaries()->size(); }
-
-  int num_record_batches() const { return footer_->recordBatches()->size(); }
-
-  MetadataVersion::type version() const {
-    switch (footer_->version()) {
-      case flatbuf::MetadataVersion_V1:
-        return MetadataVersion::V1;
-      case flatbuf::MetadataVersion_V2:
-        return MetadataVersion::V2;
-      // Add cases as other versions become available
-      default:
-        return MetadataVersion::V2;
-    }
-  }
-
-  FileBlock record_batch(int i) const {
-    return FileBlockFromFlatbuffer(footer_->recordBatches()->Get(i));
-  }
-
-  FileBlock dictionary(int i) const {
-    return FileBlockFromFlatbuffer(footer_->dictionaries()->Get(i));
-  }
-
-  Status GetSchema(std::shared_ptr<Schema>* out) const {
-    auto schema_msg = std::make_shared<SchemaMetadata>(nullptr, footer_->schema());
-    return schema_msg->GetSchema(out);
-  }
-
- private:
-  // Retain reference to memory
-  std::shared_ptr<Buffer> buffer_;
-
-  const flatbuf::Footer* footer_;
-};
-
-FileFooter::FileFooter() {}
-
-FileFooter::~FileFooter() {}
-
-Status FileFooter::Open(
-    const std::shared_ptr<Buffer>& buffer, std::unique_ptr<FileFooter>* out) {
-  const flatbuf::Footer* footer = flatbuf::GetFooter(buffer->data());
-
-  *out = std::unique_ptr<FileFooter>(new FileFooter());
-
-  // TODO(wesm): Verify the footer
-  (*out)->impl_.reset(new FileFooterImpl(buffer, footer));
-
-  return Status::OK();
-}
-
-int FileFooter::num_dictionaries() const {
-  return impl_->num_dictionaries();
-}
-
-int FileFooter::num_record_batches() const {
-  return impl_->num_record_batches();
-}
-
-MetadataVersion::type FileFooter::version() const {
-  return impl_->version();
-}
-
-FileBlock FileFooter::record_batch(int i) const {
-  return impl_->record_batch(i);
-}
-
-FileBlock FileFooter::dictionary(int i) const {
-  return impl_->dictionary(i);
-}
-
-Status FileFooter::GetSchema(std::shared_ptr<Schema>* out) const {
-  return impl_->GetSchema(out);
-}
-
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 1c4ef64d62fad..6e15ef353d853 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -49,7 +49,7 @@ struct MetadataVersion {
 
 // Serialize arrow::Schema as a Flatbuffer
 ARROW_EXPORT
-Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out);
+Status WriteSchema(const Schema& schema, std::shared_ptr<Buffer>* out);
 
 // Read interface classes. We do not fully deserialize the flatbuffers so that
 // individual fields metadata can be retrieved from very large schema without
@@ -149,10 +149,8 @@ class ARROW_EXPORT Message {
   std::unique_ptr<MessageImpl> impl_;
 };
 
-// ----------------------------------------------------------------------
-// File footer for file-like representation
-
 struct FileBlock {
+  FileBlock() {}
   FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
       : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
 
@@ -161,32 +159,6 @@ struct FileBlock {
   int64_t body_length;
 };
 
-ARROW_EXPORT
-Status WriteFileFooter(const Schema* schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, io::OutputStream* out);
-
-class ARROW_EXPORT FileFooter {
- public:
-  ~FileFooter();
-
-  static Status Open(
-      const std::shared_ptr<Buffer>& buffer, std::unique_ptr<FileFooter>* out);
-
-  int num_dictionaries() const;
-  int num_record_batches() const;
-  MetadataVersion::type version() const;
-
-  FileBlock record_batch(int i) const;
-  FileBlock dictionary(int i) const;
-
-  Status GetSchema(std::shared_ptr<Schema>* out) const;
-
- private:
-  FileFooter();
-  class FileFooterImpl;
-  std::unique_ptr<FileFooterImpl> impl_;
-};
-
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/stream.cc b/cpp/src/arrow/ipc/stream.cc
new file mode 100644
index 0000000000000..a2ca672fbe0aa
--- /dev/null
+++ b/cpp/src/arrow/ipc/stream.cc
@@ -0,0 +1,206 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/stream.h"
+
+#include <cstdint>
+#include <cstring>
+#include <sstream>
+#include <vector>
+
+#include "arrow/buffer.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
+#include "arrow/ipc/adapter.h"
+#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/util.h"
+#include "arrow/schema.h"
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+namespace ipc {
+
+// ----------------------------------------------------------------------
+// Stream writer implementation
+
+StreamWriter::~StreamWriter() {}
+
+StreamWriter::StreamWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
+    : sink_(sink), schema_(schema), position_(-1), started_(false) {}
+
+Status StreamWriter::UpdatePosition() {
+  return sink_->Tell(&position_);
+}
+
+Status StreamWriter::Write(const uint8_t* data, int64_t nbytes) {
+  RETURN_NOT_OK(sink_->Write(data, nbytes));
+  position_ += nbytes;
+  return Status::OK();
+}
+
+Status StreamWriter::Align() {
+  int64_t remainder = PaddedLength(position_) - position_;
+  if (remainder > 0) { return Write(kPaddingBytes, remainder); }
+  return Status::OK();
+}
+
+Status StreamWriter::WriteAligned(const uint8_t* data, int64_t nbytes) {
+  RETURN_NOT_OK(Write(data, nbytes));
+  return Align();
+}
+
+Status StreamWriter::CheckStarted() {
+  if (!started_) { return Start(); }
+  return Status::OK();
+}
+
+Status StreamWriter::WriteRecordBatch(const RecordBatch& batch, FileBlock* block) {
+  RETURN_NOT_OK(CheckStarted());
+
+  block->offset = position_;
+
+  // Frame of reference in file format is 0, see ARROW-384
+  const int64_t buffer_start_offset = 0;
+  RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(
+      batch, buffer_start_offset, sink_, &block->metadata_length, &block->body_length));
+  RETURN_NOT_OK(UpdatePosition());
+
+  DCHECK(position_ % 8 == 0) << "WriteRecordBatch did not perform aligned writes";
+
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// StreamWriter implementation
+
+Status StreamWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    std::shared_ptr<StreamWriter>* out) {
+  // ctor is private
+  *out = std::shared_ptr<StreamWriter>(new StreamWriter(sink, schema));
+  RETURN_NOT_OK((*out)->UpdatePosition());
+  return Status::OK();
+}
+
+Status StreamWriter::Start() {
+  std::shared_ptr<Buffer> schema_fb;
+  RETURN_NOT_OK(WriteSchema(*schema_, &schema_fb));
+
+  int32_t flatbuffer_size = schema_fb->size();
+  RETURN_NOT_OK(
+      Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
+
+  // Write the flatbuffer
+  RETURN_NOT_OK(Write(schema_fb->data(), flatbuffer_size));
+  started_ = true;
+  return Status::OK();
+}
+
+Status StreamWriter::WriteRecordBatch(const RecordBatch& batch) {
+  // Pass FileBlock, but results not used
+  FileBlock dummy_block;
+  return WriteRecordBatch(batch, &dummy_block);
+}
+
+Status StreamWriter::Close() {
+  // Close the stream
+  RETURN_NOT_OK(CheckStarted());
+  return sink_->Close();
+}
+
+// ----------------------------------------------------------------------
+// StreamReader implementation
+
+StreamReader::StreamReader(const std::shared_ptr<io::InputStream>& stream)
+    : stream_(stream), schema_(nullptr) {}
+
+StreamReader::~StreamReader() {}
+
+Status StreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
+    std::shared_ptr<StreamReader>* reader) {
+  // Private ctor
+  *reader = std::shared_ptr<StreamReader>(new StreamReader(stream));
+  return (*reader)->ReadSchema();
+}
+
+Status StreamReader::ReadSchema() {
+  std::shared_ptr<Message> message;
+  RETURN_NOT_OK(ReadNextMessage(&message));
+
+  if (message->type() != Message::SCHEMA) {
+    return Status::IOError("First message was not schema type");
+  }
+
+  SchemaMetadata schema_meta(message);
+
+  // TODO(wesm): If the schema contains dictionaries, we must read all the
+  // dictionaries from the stream before constructing the final Schema
+  return schema_meta.GetSchema(&schema_);
+}
+
+Status StreamReader::ReadNextMessage(std::shared_ptr<Message>* message) {
+  std::shared_ptr<Buffer> buffer;
+  RETURN_NOT_OK(stream_->Read(sizeof(int32_t), &buffer));
+
+  if (buffer->size() != sizeof(int32_t)) {
+    *message = nullptr;
+    return Status::OK();
+  }
+
+  int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
+
+  RETURN_NOT_OK(stream_->Read(message_length, &buffer));
+  if (buffer->size() != message_length) {
+    return Status::IOError("Unexpected end of stream trying to read message");
+  }
+  return Message::Open(buffer, 0, message);
+}
+
+std::shared_ptr<Schema> StreamReader::schema() const {
+  return schema_;
+}
+
+Status StreamReader::GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+  std::shared_ptr<Message> message;
+  RETURN_NOT_OK(ReadNextMessage(&message));
+
+  if (message == nullptr) {
+    // End of stream
+    *batch = nullptr;
+    return Status::OK();
+  }
+
+  if (message->type() != Message::RECORD_BATCH) {
+    return Status::IOError("Metadata not record batch");
+  }
+
+  auto batch_metadata = std::make_shared<RecordBatchMetadata>(message);
+
+  std::shared_ptr<Buffer> batch_body;
+  RETURN_NOT_OK(stream_->Read(message->body_length(), &batch_body));
+
+  if (batch_body->size() < message->body_length()) {
+    return Status::IOError("Unexpected EOS when reading message body");
+  }
+
+  io::BufferReader reader(batch_body);
+
+  return ReadRecordBatch(batch_metadata, schema_, &reader, batch);
+}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/stream.h b/cpp/src/arrow/ipc/stream.h
new file mode 100644
index 0000000000000..0b0e62f13fc5f
--- /dev/null
+++ b/cpp/src/arrow/ipc/stream.h
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Implement Arrow streaming binary format
+
+#ifndef ARROW_IPC_STREAM_H
+#define ARROW_IPC_STREAM_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+class Buffer;
+struct Field;
+class RecordBatch;
+class Schema;
+class Status;
+
+namespace io {
+
+class InputStream;
+class OutputStream;
+
+}  // namespace io
+
+namespace ipc {
+
+struct FileBlock;
+class Message;
+
+class ARROW_EXPORT StreamWriter {
+ public:
+  virtual ~StreamWriter();
+
+  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+      std::shared_ptr<StreamWriter>* out);
+
+  virtual Status WriteRecordBatch(const RecordBatch& batch);
+  virtual Status Close();
+
+ protected:
+  StreamWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema);
+
+  virtual Status Start();
+
+  Status CheckStarted();
+  Status UpdatePosition();
+
+  Status WriteRecordBatch(const RecordBatch& batch, FileBlock* block);
+
+  // Adds padding bytes if necessary to ensure all memory blocks are written on
+  // 8-byte boundaries.
+  Status Align();
+
+  // Write data and update position
+  Status Write(const uint8_t* data, int64_t nbytes);
+
+  // Write and align
+  Status WriteAligned(const uint8_t* data, int64_t nbytes);
+
+  io::OutputStream* sink_;
+  std::shared_ptr<Schema> schema_;
+  int64_t position_;
+  bool started_;
+};
+
+class ARROW_EXPORT StreamReader {
+ public:
+  ~StreamReader();
+
+  // Open an stream.
+  static Status Open(const std::shared_ptr<io::InputStream>& stream,
+      std::shared_ptr<StreamReader>* reader);
+
+  std::shared_ptr<Schema> schema() const;
+
+  // Returned batch is nullptr when end of stream reached
+  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch);
+
+ private:
+  explicit StreamReader(const std::shared_ptr<io::InputStream>& stream);
+
+  Status ReadSchema();
+
+  Status ReadNextMessage(std::shared_ptr<Message>* message);
+
+  std::shared_ptr<io::InputStream> stream_;
+  std::shared_ptr<Schema> schema_;
+};
+
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_STREAM_H
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 3faeebf956966..ca790ded92191 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -36,6 +36,15 @@
 namespace arrow {
 namespace ipc {
 
+static inline void AssertSchemaEqual(const Schema& lhs, const Schema& rhs) {
+  if (!lhs.Equals(rhs)) {
+    std::stringstream ss;
+    ss << "left schema: " << lhs.ToString() << std::endl
+       << "right schema: " << rhs.ToString() << std::endl;
+    FAIL() << ss.str();
+  }
+}
+
 const auto kListInt32 = list(int32());
 const auto kListListInt32 = list(kListInt32);
 
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
index b3185b1c1671c..82957600d1eb6 100644
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -29,8 +29,7 @@ cdef extern from "arrow/ipc/file.h" namespace "arrow::ipc" nogil:
         CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
                      shared_ptr[CFileWriter]* out)
 
-        CStatus WriteRecordBatch(const vector[shared_ptr[CArray]]& columns,
-                                 int32_t num_rows)
+        CStatus WriteRecordBatch(const CRecordBatch& batch)
 
         CStatus Close()
 
diff --git a/python/pyarrow/ipc.pyx b/python/pyarrow/ipc.pyx
index abc5e1b11ec4c..22069a7290ead 100644
--- a/python/pyarrow/ipc.pyx
+++ b/python/pyarrow/ipc.pyx
@@ -21,6 +21,8 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+from cython.operator cimport dereference as deref
+
 from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.libarrow_io cimport *
 from pyarrow.includes.libarrow_ipc cimport *
@@ -58,10 +60,9 @@ cdef class ArrowFileWriter:
             self.close()
 
     def write_record_batch(self, RecordBatch batch):
-        cdef CRecordBatch* bptr = batch.batch
         with nogil:
             check_status(self.writer.get()
-                         .WriteRecordBatch(bptr.columns(), bptr.num_rows()))
+                         .WriteRecordBatch(deref(batch.batch)))
 
     def close(self):
         with nogil:
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 6623e239880bc..feafa3dfc3875 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -254,16 +254,16 @@ struct arrow_traits<Type::BOOL> {
   static constexpr bool is_numeric_nullable = false;
 };
 
-#define INT_DECL(TYPE)                                      \
-  template <>                                               \
-  struct arrow_traits<Type::TYPE> {                         \
-    static constexpr int npy_type = NPY_##TYPE;             \
-    static constexpr bool supports_nulls = false;           \
-    static constexpr double na_value = NAN;                 \
-    static constexpr bool is_boolean = false;               \
-    static constexpr bool is_numeric_not_nullable = true;   \
-    static constexpr bool is_numeric_nullable = false;      \
-    typedef typename npy_traits<NPY_##TYPE>::value_type T;  \
+#define INT_DECL(TYPE)                                     \
+  template <>                                              \
+  struct arrow_traits<Type::TYPE> {                        \
+    static constexpr int npy_type = NPY_##TYPE;            \
+    static constexpr bool supports_nulls = false;          \
+    static constexpr double na_value = NAN;                \
+    static constexpr bool is_boolean = false;              \
+    static constexpr bool is_numeric_not_nullable = true;  \
+    static constexpr bool is_numeric_nullable = false;     \
+    typedef typename npy_traits<NPY_##TYPE>::value_type T; \
   };
 
 INT_DECL(INT8);
@@ -1803,7 +1803,7 @@ class ArrowDeserializer {
   // types
 
   Status Convert(PyObject** out) {
-#define CONVERT_CASE(TYPE)                             \
+#define CONVERT_CASE(TYPE)                      \
   case Type::TYPE: {                            \
     RETURN_NOT_OK(ConvertValues<Type::TYPE>()); \
   } break;
@@ -1857,8 +1857,7 @@ class ArrowDeserializer {
   }
 
   template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::DATE, Status>::type
-  ConvertValues() {
+  inline typename std::enable_if<TYPE == Type::DATE, Status>::type ConvertValues() {
     typedef typename arrow_traits<TYPE>::T T;
 
     RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
@@ -1910,24 +1909,21 @@ class ArrowDeserializer {
 
   // UTF8 strings
   template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::STRING, Status>::type
-  ConvertValues() {
+  inline typename std::enable_if<TYPE == Type::STRING, Status>::type ConvertValues() {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     return ConvertBinaryLike<arrow::StringArray>(data_, out_values);
   }
 
   template <int T2>
-  inline typename std::enable_if<T2 == Type::BINARY, Status>::type
-  ConvertValues() {
+  inline typename std::enable_if<T2 == Type::BINARY, Status>::type ConvertValues() {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     return ConvertBinaryLike<arrow::BinaryArray>(data_, out_values);
   }
 
   template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::DICTIONARY, Status>::type
-  ConvertValues() {
+  inline typename std::enable_if<TYPE == Type::DICTIONARY, Status>::type ConvertValues() {
     std::shared_ptr<PandasBlock> block;
     RETURN_NOT_OK(MakeCategoricalBlock(col_->type(), col_->length(), &block));
     RETURN_NOT_OK(block->Write(col_, 0, 0));

From 5a161ebc1961b4f784d51322b12fe09e8c8aa08d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 22 Jan 2017 11:13:59 -0500
Subject: [PATCH 0286/1644] ARROW-505: [C++] Fix compiler warning in gcc in
 release mode

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #294 from wesm/fix-release-compile-warning and squashes the following commits:

4189892 [Wes McKinney] Suppress another Cython compiler warning when compiling with clang
9a8ad54 [Wes McKinney] Fix compiler warning in gcc in release mode
---
 cpp/src/arrow/ipc/adapter.cc | 4 ++--
 python/CMakeLists.txt        | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 9da7b3912d4bc..c8e631c564b22 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -197,8 +197,8 @@ class RecordBatchWriter : public ArrayVisitor {
 
   Status GetTotalSize(int64_t* size) {
     // emulates the behavior of Write without actually writing
-    int32_t metadata_length;
-    int64_t body_length;
+    int32_t metadata_length = 0;
+    int64_t body_length = 0;
     MockOutputStream dst;
     RETURN_NOT_OK(Write(&dst, &metadata_length, &body_length));
     *size = dst.GetExtentBytesWritten();
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 0a2d4e9facba2..b3735b1d58653 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -87,6 +87,9 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   #   http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html
   #   http://petereisentraut.blogspot.com/2011/09/ccache-and-clang-part-2.html
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
+
+  # Cython warnings in clang
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality")
 endif()
 
 set(PYARROW_LINK "a")

From 53a478dfb278dcae5ca7f300b70857662553d118 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Jan 2017 06:41:35 -0500
Subject: [PATCH 0287/1644] ARROW-475: [Python] Add support for reading
 multiple Parquet files as a single pyarrow.Table

Also fixes a serious bug in which the data source passed to the ParquetReader gets garbage collected prematurely

Also implements ARROW-470

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #296 from wesm/ARROW-475 and squashes the following commits:

894d2a2 [Wes McKinney] Implement Filesystem abstraction, add Filesystem.read_parquet. Implement rudimentary shim on local filesystem
3927c2c [Wes McKinney] Test read multiple Parquet from HDFS, fix premature garbage collection error
4904b3b [Wes McKinney] Implement read_multiple_files function for multiple Parquet files as a single Arrow table
---
 python/pyarrow/__init__.py                   |   6 +-
 python/pyarrow/_parquet.pyx                  |   3 +
 python/pyarrow/filesystem.py                 | 186 +++++++++++++++++++
 python/pyarrow/includes/libarrow_io.pxd      |   2 +
 python/pyarrow/io.pyx                        |  62 +++----
 python/pyarrow/parquet.py                    |  88 +++++++--
 python/pyarrow/table.pyx                     |  60 ++++--
 python/pyarrow/tests/test_column.py          |  49 -----
 python/pyarrow/tests/test_convert_builtin.py |   3 +-
 python/pyarrow/tests/test_convert_pandas.py  |   8 +-
 python/pyarrow/tests/test_hdfs.py            |  46 ++++-
 python/pyarrow/tests/test_parquet.py         | 155 ++++++++++++----
 python/pyarrow/tests/test_scalars.py         |   2 +-
 python/pyarrow/tests/test_schema.py          |   1 -
 python/pyarrow/tests/test_table.py           |  50 +++--
 python/pyarrow/util.py                       |  25 +++
 16 files changed, 568 insertions(+), 178 deletions(-)
 create mode 100644 python/pyarrow/filesystem.py
 delete mode 100644 python/pyarrow/tests/test_column.py
 create mode 100644 python/pyarrow/util.py

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index efffbf2a4588d..d563c7aa4055d 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -42,7 +42,8 @@
 
 from pyarrow.error import ArrowException
 
-from pyarrow.io import (HdfsClient, HdfsFile, NativeFile, PythonFileInterface,
+from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
+from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
                         Buffer, InMemoryOutputStream, BufferReader)
 
 from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
@@ -61,3 +62,6 @@
                             DataType, Field, Schema, schema)
 
 from pyarrow.table import Column, RecordBatch, Table, concat_tables
+
+
+localfs = LocalFilesystem()
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 867fc4cfecbd6..b11cee3a201fb 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -341,6 +341,7 @@ cdef logical_type_name_from_enum(ParquetLogicalType type_):
 
 cdef class ParquetReader:
     cdef:
+        object source
         MemoryPool* allocator
         unique_ptr[FileReader] reader
         column_idx_map
@@ -360,6 +361,8 @@ cdef class ParquetReader:
         if metadata is not None:
             c_metadata = metadata.sp_metadata
 
+        self.source = source
+
         get_reader(source, &rd_handle)
         with nogil:
             check_status(OpenFile(rd_handle, self.allocator, properties,
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
new file mode 100644
index 0000000000000..82409b7666ab1
--- /dev/null
+++ b/python/pyarrow/filesystem.py
@@ -0,0 +1,186 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from os.path import join as pjoin
+import os
+
+from pyarrow.util import implements
+import pyarrow.io as io
+
+
+class Filesystem(object):
+    """
+    Abstract filesystem interface
+    """
+    def ls(self, path):
+        """
+        Return list of file paths
+        """
+        raise NotImplementedError
+
+    def delete(self, path, recursive=False):
+        """
+        Delete the indicated file or directory
+
+        Parameters
+        ----------
+        path : string
+        recursive : boolean, default False
+            If True, also delete child paths for directories
+        """
+        raise NotImplementedError
+
+    def mkdir(self, path, create_parents=True):
+        raise NotImplementedError
+
+    def exists(self, path):
+        raise NotImplementedError
+
+    def isdir(self, path):
+        """
+        Return True if path is a directory
+        """
+        raise NotImplementedError
+
+    def isfile(self, path):
+        """
+        Return True if path is a file
+        """
+        raise NotImplementedError
+
+    def read_parquet(self, path, columns=None, schema=None):
+        """
+        Read Parquet data from path in file system. Can read from a single file
+        or a directory of files
+
+        Parameters
+        ----------
+        path : str
+            Single file path or directory
+        columns : List[str], optional
+            Subset of columns to read
+        schema : pyarrow.parquet.Schema
+            Known schema to validate files against
+
+        Returns
+        -------
+        table : pyarrow.Table
+        """
+        from pyarrow.parquet import read_multiple_files
+
+        if self.isdir(path):
+            paths_to_read = []
+            for path in self.ls(path):
+                if path == '_metadata' or path == '_common_metadata':
+                    raise ValueError('No support yet for common metadata file')
+                paths_to_read.append(path)
+        else:
+            paths_to_read = [path]
+
+        return read_multiple_files(paths_to_read, columns=columns,
+                                   filesystem=self, schema=schema)
+
+
+class LocalFilesystem(Filesystem):
+
+    @implements(Filesystem.ls)
+    def ls(self, path):
+        return sorted(pjoin(path, x) for x in os.listdir(path))
+
+    @implements(Filesystem.isdir)
+    def isdir(self, path):
+        return os.path.isdir(path)
+
+    @implements(Filesystem.isfile)
+    def isfile(self, path):
+        return os.path.isfile(path)
+
+    @implements(Filesystem.exists)
+    def exists(self, path):
+        return os.path.exists(path)
+
+    def open(self, path, mode='rb'):
+        """
+        Open file for reading or writing
+        """
+        return open(path, mode=mode)
+
+
+class HdfsClient(io._HdfsClient, Filesystem):
+    """
+    Connect to an HDFS cluster. All parameters are optional and should
+    only be set if the defaults need to be overridden.
+
+    Authentication should be automatic if the HDFS cluster uses Kerberos.
+    However, if a username is specified, then the ticket cache will likely
+    be required.
+
+    Parameters
+    ----------
+    host : NameNode. Set to "default" for fs.defaultFS from core-site.xml.
+    port : NameNode's port. Set to 0 for default or logical (HA) nodes.
+    user : Username when connecting to HDFS; None implies login user.
+    kerb_ticket : Path to Kerberos ticket cache.
+    driver : {'libhdfs', 'libhdfs3'}, default 'libhdfs'
+      Connect using libhdfs (JNI-based) or libhdfs3 (3rd-party C++
+      library from Pivotal Labs)
+
+    Notes
+    -----
+    The first time you call this method, it will take longer than usual due
+    to JNI spin-up time.
+
+    Returns
+    -------
+    client : HDFSClient
+    """
+
+    def __init__(self, host="default", port=0, user=None, kerb_ticket=None,
+                 driver='libhdfs'):
+        self._connect(host, port, user, kerb_ticket, driver)
+
+    @implements(Filesystem.isdir)
+    def isdir(self, path):
+        return io._HdfsClient.isdir(self, path)
+
+    @implements(Filesystem.isfile)
+    def isfile(self, path):
+        return io._HdfsClient.isfile(self, path)
+
+    @implements(Filesystem.delete)
+    def delete(self, path, recursive=False):
+        return io._HdfsClient.delete(self, path, recursive)
+
+    @implements(Filesystem.mkdir)
+    def mkdir(self, path, create_parents=True):
+        return io._HdfsClient.mkdir(self, path)
+
+    def ls(self, path, full_info=False):
+        """
+        Retrieve directory contents and metadata, if requested.
+
+        Parameters
+        ----------
+        path : HDFS path
+        full_info : boolean, default False
+            If False, only return list of paths
+
+        Returns
+        -------
+        result : list of dicts (full_info=True) or strings (full_info=False)
+        """
+        return io._HdfsClient.ls(self, path, full_info)
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 417af7d67d1ab..31379386187ee 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -148,6 +148,8 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
         CStatus ListDirectory(const c_string& path,
                               vector[HdfsPathInfo]* listing)
 
+        CStatus GetPathInfo(const c_string& path, HdfsPathInfo* info)
+
         CStatus Rename(const c_string& src, const c_string& dst)
 
         CStatus OpenReadable(const c_string& path,
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 0f626f178abde..26215122b7a23 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -463,42 +463,17 @@ def strip_hdfs_abspath(path):
         return path
 
 
-cdef class HdfsClient:
+cdef class _HdfsClient:
     cdef:
         shared_ptr[CHdfsClient] client
 
     cdef readonly:
         bint is_open
 
-    def __cinit__(self, host="default", port=0, user=None, kerb_ticket=None,
-                  driver='libhdfs'):
-        """
-        Connect to an HDFS cluster. All parameters are optional and should
-        only be set if the defaults need to be overridden.
-
-        Authentication should be automatic if the HDFS cluster uses Kerberos.
-        However, if a username is specified, then the ticket cache will likely
-        be required.
+    def __cinit__(self):
+        pass
 
-        Parameters
-        ----------
-        host : NameNode. Set to "default" for fs.defaultFS from core-site.xml.
-        port : NameNode's port. Set to 0 for default or logical (HA) nodes.
-        user : Username when connecting to HDFS; None implies login user.
-        kerb_ticket : Path to Kerberos ticket cache.
-        driver : {'libhdfs', 'libhdfs3'}, default 'libhdfs'
-          Connect using libhdfs (JNI-based) or libhdfs3 (3rd-party C++
-          library from Pivotal Labs)
-
-        Notes
-        -----
-        The first time you call this method, it will take longer than usual due
-        to JNI spin-up time.
-
-        Returns
-        -------
-        client : HDFSClient
-        """
+    def _connect(self, host, port, user, kerb_ticket, driver):
         cdef HdfsConnectionConfig conf
 
         if host is not None:
@@ -556,20 +531,25 @@ cdef class HdfsClient:
             result = self.client.get().Exists(c_path)
         return result
 
-    def ls(self, path, bint full_info=True):
-        """
-        Retrieve directory contents and metadata, if requested.
+    def isdir(self, path):
+        cdef HdfsPathInfo info
+        self._path_info(path, &info)
+        return info.kind == ObjectType_DIRECTORY
 
-        Parameters
-        ----------
-        path : HDFS path
-        full_info : boolean, default True
-            If False, only return list of paths
+    def isfile(self, path):
+        cdef HdfsPathInfo info
+        self._path_info(path, &info)
+        return info.kind == ObjectType_FILE
 
-        Returns
-        -------
-        result : list of dicts (full_info=True) or strings (full_info=False)
-        """
+    cdef _path_info(self, path, HdfsPathInfo* info):
+        cdef c_string c_path = tobytes(path)
+
+        with nogil:
+            check_status(self.client.get()
+                         .GetPathInfo(c_path, info))
+
+
+    def ls(self, path, bint full_info):
         cdef:
             c_string c_path = tobytes(path)
             vector[HdfsPathInfo] listing
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 2a1ac9d2db7ed..cbe1c6e5d79d9 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -15,8 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import pyarrow._parquet as _parquet
-from pyarrow.table import Table
+from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
+                              RowGroupMetaData, Schema, ParquetWriter)
+import pyarrow._parquet as _parquet  # noqa
+from pyarrow.table import Table, concat_tables
 
 
 class ParquetFile(object):
@@ -32,7 +34,7 @@ class ParquetFile(object):
         Use existing metadata object, rather than reading from file.
     """
     def __init__(self, source, metadata=None):
-        self.reader = _parquet.ParquetReader()
+        self.reader = ParquetReader()
         self.reader.open(source, metadata=metadata)
 
     @property
@@ -67,10 +69,10 @@ def read(self, nrows=None, columns=None):
                            for column in columns]
             arrays = [self.reader.read_column(column_idx)
                       for column_idx in column_idxs]
-            return Table.from_arrays(columns, arrays)
+            return Table.from_arrays(arrays, names=columns)
 
 
-def read_table(source, columns=None):
+def read_table(source, columns=None, metadata=None):
     """
     Read a Table from Parquet format
 
@@ -81,17 +83,79 @@ def read_table(source, columns=None):
         pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
     columns: list
         If not None, only these columns will be read from the file.
+    metadata : FileMetaData
+        If separately computed
 
     Returns
     -------
-    pyarrow.table.Table
+    pyarrow.Table
         Content of the file as a table (of columns)
     """
-    return ParquetFile(source).read(columns=columns)
+    return ParquetFile(source, metadata=metadata).read(columns=columns)
 
 
-def write_table(table, sink, chunk_size=None, version=None,
-                use_dictionary=True, compression=None):
+def read_multiple_files(paths, columns=None, filesystem=None, metadata=None,
+                        schema=None):
+    """
+    Read multiple Parquet files as a single pyarrow.Table
+
+    Parameters
+    ----------
+    paths : List[str]
+        List of file paths
+    columns : List[str]
+        Names of columns to read from the file
+    filesystem : Filesystem, default None
+        If nothing passed, paths assumed to be found in the local on-disk
+        filesystem
+    metadata : pyarrow.parquet.FileMetaData
+        Use metadata obtained elsewhere to validate file schemas
+    schema : pyarrow.parquet.Schema
+        Use schema obtained elsewhere to validate file schemas. Alternative to
+        metadata parameter
+
+    Returns
+    -------
+    pyarrow.Table
+        Content of the file as a table (of columns)
+    """
+    if filesystem is None:
+        def open_file(path, meta=None):
+            return ParquetFile(path, metadata=meta)
+    else:
+        def open_file(path, meta=None):
+            return ParquetFile(filesystem.open(path, mode='rb'), metadata=meta)
+
+    if len(paths) == 0:
+        raise ValueError('Must pass at least one file path')
+
+    if metadata is None and schema is None:
+        schema = open_file(paths[0]).schema
+    elif schema is None:
+        schema = metadata.schema
+
+    # Verify schemas are all equal
+    all_file_metadata = []
+    for path in paths:
+        file_metadata = open_file(path).metadata
+        if not schema.equals(file_metadata.schema):
+            raise ValueError('Schema in {0} was different. {1!s} vs {2!s}'
+                             .format(path, file_metadata.schema, schema))
+        all_file_metadata.append(file_metadata)
+
+    # Read the tables
+    tables = []
+    for path, path_metadata in zip(paths, all_file_metadata):
+        reader = open_file(path, meta=path_metadata)
+        table = reader.read(columns=columns)
+        tables.append(table)
+
+    all_data = concat_tables(tables)
+    return all_data
+
+
+def write_table(table, sink, chunk_size=None, version='1.0',
+                use_dictionary=True, compression='snappy'):
     """
     Write a Table to Parquet format
 
@@ -110,7 +174,7 @@ def write_table(table, sink, chunk_size=None, version=None,
     compression : str or dict
         Specify the compression codec, either on a general basis or per-column.
     """
-    writer = _parquet.ParquetWriter(sink, use_dictionary=use_dictionary,
-                                    compression=compression,
-                                    version=version)
+    writer = ParquetWriter(sink, use_dictionary=use_dictionary,
+                           compression=compression,
+                           version=version)
     writer.write_table(table, row_group_size=chunk_size)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 0e3b2bd90dc64..924233066055e 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -265,16 +265,35 @@ cdef class Column:
 cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
     cdef:
         Array arr
+        Column col
         c_string c_name
         vector[shared_ptr[CField]] fields
+        cdef shared_ptr[CDataType] type_
 
     cdef int K = len(arrays)
 
     fields.resize(K)
-    for i in range(K):
-        arr = arrays[i]
-        c_name = tobytes(names[i])
-        fields[i].reset(new CField(c_name, arr.type.sp_type, True))
+
+    if len(arrays) == 0:
+        raise ValueError('Must pass at least one array')
+
+    if isinstance(arrays[0], Array):
+        if names is None:
+            raise ValueError('Must pass names when constructing '
+                             'from Array objects')
+        for i in range(K):
+            arr = arrays[i]
+            type_ = arr.type.sp_type
+            c_name = tobytes(names[i])
+            fields[i].reset(new CField(c_name, type_, True))
+    elif isinstance(arrays[0], Column):
+        for i in range(K):
+            col = arrays[i]
+            type_ = col.sp_column.get().type()
+            c_name = tobytes(col.name)
+            fields[i].reset(new CField(c_name, type_, True))
+    else:
+        raise TypeError(type(arrays[0]))
 
     schema.reset(new CSchema(fields))
 
@@ -429,19 +448,19 @@ cdef class RecordBatch:
         pyarrow.table.RecordBatch
         """
         names, arrays = _dataframe_to_arrays(df, None, False, schema)
-        return cls.from_arrays(names, arrays)
+        return cls.from_arrays(arrays, names)
 
     @staticmethod
-    def from_arrays(names, arrays):
+    def from_arrays(arrays, names):
         """
         Construct a RecordBatch from multiple pyarrow.Arrays
 
         Parameters
         ----------
-        names: list of str
-            Labels for the columns
         arrays: list of pyarrow.Array
             column-wise data vectors
+        names: list of str
+            Labels for the columns
 
         Returns
         -------
@@ -594,20 +613,20 @@ cdef class Table:
         names, arrays = _dataframe_to_arrays(df, name=name,
                                              timestamps_to_ms=timestamps_to_ms,
                                              schema=schema)
-        return cls.from_arrays(names, arrays, name=name)
+        return cls.from_arrays(arrays, names=names, name=name)
 
     @staticmethod
-    def from_arrays(names, arrays, name=None):
+    def from_arrays(arrays, names=None, name=None):
         """
-        Construct a Table from Arrow Arrays
+        Construct a Table from Arrow arrays or columns
 
         Parameters
         ----------
-
-        names: list of str
-            Names for the table columns
-        arrays: list of pyarrow.array.Array
+        arrays: list of pyarrow.Array or pyarrow.Column
             Equal-length arrays that should form the table.
+        names: list of str, optional
+            Names for the table columns. If Columns passed, will be
+            inferred. If Arrays passed, this argument is required
         name: str, optional
             name for the Table
 
@@ -617,7 +636,6 @@ cdef class Table:
 
         """
         cdef:
-            Array arr
             c_string c_name
             vector[shared_ptr[CField]] fields
             vector[shared_ptr[CColumn]] columns
@@ -628,9 +646,15 @@ cdef class Table:
 
         cdef int K = len(arrays)
         columns.resize(K)
+
         for i in range(K):
-            arr = arrays[i]
-            columns[i].reset(new CColumn(schema.get().field(i), arr.sp_array))
+            if isinstance(arrays[i], Array):
+                columns[i].reset(new CColumn(schema.get().field(i),
+                                             (<Array> arrays[i]).sp_array))
+            elif isinstance(arrays[i], Column):
+                columns[i] = (<Column> arrays[i]).sp_column
+            else:
+                raise ValueError(type(arrays[i]))
 
         if name is None:
             c_name = ''
diff --git a/python/pyarrow/tests/test_column.py b/python/pyarrow/tests/test_column.py
deleted file mode 100644
index 1a507c81030f8..0000000000000
--- a/python/pyarrow/tests/test_column.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyarrow.compat import unittest
-import pyarrow as arrow
-
-A = arrow
-
-import pandas as pd
-
-
-class TestColumn(unittest.TestCase):
-
-    def test_basics(self):
-        data = [
-            A.from_pylist([-10, -5, 0, 5, 10])
-        ]
-        table = A.Table.from_arrays(('a'), data, 'table_name')
-        column = table.column(0)
-        assert column.name == 'a'
-        assert column.length() == 5
-        assert len(column) == 5
-        assert column.shape == (5,)
-        assert column.to_pylist() == [-10, -5, 0, 5, 10]
-
-    def test_pandas(self):
-        data = [
-            A.from_pylist([-10, -5, 0, 5, 10])
-        ]
-        table = A.Table.from_arrays(('a'), data, 'table_name')
-        column = table.column(0)
-        series = column.to_pandas()
-        assert series.name == 'a'
-        assert series.shape == (5,)
-        assert series.iloc[0] == -10
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 72e438910159f..c06d18d19c049 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -16,11 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import unittest, u
+from pyarrow.compat import unittest, u  # noqa
 import pyarrow
 
 import datetime
 
+
 class TestConvertList(unittest.TestCase):
 
     def test_boolean(self):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index a2f50620d8925..30705c4ca2a20 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -74,7 +74,7 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
         tm.assert_frame_equal(result, expected)
 
     def _check_array_roundtrip(self, values, expected=None,
-                                timestamps_to_ms=False, field=None):
+                               timestamps_to_ms=False, field=None):
         arr = A.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
                                   field=field)
         result = arr.to_pandas()
@@ -118,7 +118,7 @@ def test_float_nulls(self):
         ex_frame = pd.DataFrame(dict(zip(names, expected_cols)),
                                 columns=names)
 
-        table = A.Table.from_arrays(names, arrays)
+        table = A.Table.from_arrays(arrays, names)
         assert table.schema.equals(A.Schema.from_fields(fields))
         result = table.to_pandas()
         tm.assert_frame_equal(result, ex_frame)
@@ -169,7 +169,7 @@ def test_integer_with_nulls(self):
         ex_frame = pd.DataFrame(dict(zip(int_dtypes, expected_cols)),
                                 columns=int_dtypes)
 
-        table = A.Table.from_arrays(int_dtypes, arrays)
+        table = A.Table.from_arrays(arrays, int_dtypes)
         result = table.to_pandas()
 
         tm.assert_frame_equal(result, ex_frame)
@@ -201,7 +201,7 @@ def test_boolean_nulls(self):
         schema = A.Schema.from_fields([field])
         ex_frame = pd.DataFrame({'bools': expected})
 
-        table = A.Table.from_arrays(['bools'], [arr])
+        table = A.Table.from_arrays([arr], ['bools'])
         assert table.schema.equals(schema)
         result = table.to_pandas()
 
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 2056f7ab589da..cb24adb73adc9 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -21,9 +21,16 @@
 import random
 import unittest
 
+import numpy as np
+import pandas.util.testing as pdt
 import pytest
 
+from pyarrow.compat import guid
+from pyarrow.filesystem import HdfsClient
 import pyarrow.io as io
+import pyarrow as pa
+
+import pyarrow.tests.test_parquet as test_parquet
 
 # ----------------------------------------------------------------------
 # HDFS tests
@@ -38,7 +45,7 @@ def hdfs_test_client(driver='libhdfs'):
         raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
                          'an integer')
 
-    return io.HdfsClient(host, port, user, driver=driver)
+    return HdfsClient(host, port, user, driver=driver)
 
 
 class HdfsTestCases(object):
@@ -138,6 +145,43 @@ def test_hdfs_read_whole_file(self):
 
         assert result == data
 
+    @test_parquet.parquet
+    def test_hdfs_read_multiple_parquet_files(self):
+        import pyarrow.parquet as pq
+
+        nfiles = 10
+        size = 5
+
+        tmpdir = pjoin(self.tmp_path, 'multi-parquet-' + guid())
+
+        self.hdfs.mkdir(tmpdir)
+
+        test_data = []
+        paths = []
+        for i in range(nfiles):
+            df = test_parquet._test_dataframe(size, seed=i)
+
+            df['index'] = np.arange(i * size, (i + 1) * size)
+
+            # Hack so that we don't have a dtype cast in v1 files
+            df['uint32'] = df['uint32'].astype(np.int64)
+
+            path = pjoin(tmpdir, '{0}.parquet'.format(i))
+
+            table = pa.Table.from_pandas(df)
+            with self.hdfs.open(path, 'wb') as f:
+                pq.write_table(table, f)
+
+            test_data.append(table)
+            paths.append(path)
+
+        result = self.hdfs.read_parquet(tmpdir)
+        expected = pa.concat_tables(test_data)
+
+        pdt.assert_frame_equal(result.to_pandas()
+                               .sort_values(by='index').reset_index(drop=True),
+                               expected.to_pandas())
+
 
 class TestLibHdfs(HdfsTestCases, unittest.TestCase):
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 9cf860ac28a10..a94fe456d3b2b 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -15,10 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from os.path import join as pjoin
 import io
+import os
 import pytest
 
-import pyarrow as A
+from pyarrow.compat import guid
+import pyarrow as pa
 import pyarrow.io as paio
 
 import numpy as np
@@ -42,9 +45,9 @@ def test_single_pylist_column_roundtrip(tmpdir):
     for dtype in [int, float]:
         filename = tmpdir.join('single_{}_column.parquet'
                                .format(dtype.__name__))
-        data = [A.from_pylist(list(map(dtype, range(5))))]
-        table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
-        A.parquet.write_table(table, filename.strpath)
+        data = [pa.from_pylist(list(map(dtype, range(5))))]
+        table = pa.Table.from_arrays(data, names=('a', 'b'), name='table_name')
+        pq.write_table(table, filename.strpath)
         table_read = pq.read_table(filename.strpath)
         for col_written, col_read in zip(table.itercolumns(),
                                          table_read.itercolumns()):
@@ -85,8 +88,8 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
     df = alltypes_sample(size=10000)
 
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.Table.from_pandas(df, timestamps_to_ms=True)
-    A.parquet.write_table(arrow_table, filename.strpath, version="2.0")
+    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    pq.write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
     pdt.assert_frame_equal(df, df_read)
@@ -113,8 +116,8 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
         'empty_str': [''] * size
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.Table.from_pandas(df)
-    A.parquet.write_table(arrow_table, filename.strpath, version="1.0")
+    arrow_table = pa.Table.from_pandas(df)
+    pq.write_table(arrow_table, filename.strpath, version="1.0")
     table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
 
@@ -133,28 +136,39 @@ def test_pandas_column_selection(tmpdir):
         'uint16': np.arange(size, dtype=np.uint16)
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.Table.from_pandas(df)
-    A.parquet.write_table(arrow_table, filename.strpath)
+    arrow_table = pa.Table.from_pandas(df)
+    pq.write_table(arrow_table, filename.strpath)
     table_read = pq.read_table(filename.strpath, columns=['uint8'])
     df_read = table_read.to_pandas()
 
     pdt.assert_frame_equal(df[['uint8']], df_read)
 
 
-def _test_dataframe(size=10000):
-    np.random.seed(0)
+def _random_integers(size, dtype):
+    # We do not generate integers outside the int64 range
+    i64_info = np.iinfo('int64')
+    iinfo = np.iinfo(dtype)
+    return np.random.randint(max(iinfo.min, i64_info.min),
+                             min(iinfo.max, i64_info.max),
+                             size=size).astype(dtype)
+
+
+def _test_dataframe(size=10000, seed=0):
+    np.random.seed(seed)
     df = pd.DataFrame({
-        'uint8': np.arange(size, dtype=np.uint8),
-        'uint16': np.arange(size, dtype=np.uint16),
-        'uint32': np.arange(size, dtype=np.uint32),
-        'uint64': np.arange(size, dtype=np.uint64),
-        'int8': np.arange(size, dtype=np.int16),
-        'int16': np.arange(size, dtype=np.int16),
-        'int32': np.arange(size, dtype=np.int32),
-        'int64': np.arange(size, dtype=np.int64),
-        'float32': np.arange(size, dtype=np.float32),
+        'uint8': _random_integers(size, np.uint8),
+        'uint16': _random_integers(size, np.uint16),
+        'uint32': _random_integers(size, np.uint32),
+        'uint64': _random_integers(size, np.uint64),
+        'int8': _random_integers(size, np.int8),
+        'int16': _random_integers(size, np.int16),
+        'int32': _random_integers(size, np.int32),
+        'int64': _random_integers(size, np.int64),
+        'float32': np.random.randn(size).astype(np.float32),
+        'float64': np.random.randn(size),
         'float64': np.arange(size, dtype=np.float64),
-        'bool': np.random.randn(size) > 0
+        'bool': np.random.randn(size) > 0,
+        'strings': [pdt.rands(10) for i in range(size)]
     })
     return df
 
@@ -162,7 +176,7 @@ def _test_dataframe(size=10000):
 @parquet
 def test_pandas_parquet_native_file_roundtrip(tmpdir):
     df = _test_dataframe(10000)
-    arrow_table = A.Table.from_pandas(df)
+    arrow_table = pa.Table.from_pandas(df)
     imos = paio.InMemoryOutputStream()
     pq.write_table(arrow_table, imos, version="2.0")
     buf = imos.get_result()
@@ -183,10 +197,10 @@ def test_pandas_parquet_pyfile_roundtrip(tmpdir):
         'strings': ['foo', 'bar', None, 'baz', 'qux']
     })
 
-    arrow_table = A.Table.from_pandas(df)
+    arrow_table = pa.Table.from_pandas(df)
 
     with open(filename, 'wb') as f:
-        A.parquet.write_table(arrow_table, f, version="1.0")
+        pq.write_table(arrow_table, f, version="1.0")
 
     data = io.BytesIO(open(filename, 'rb').read())
 
@@ -213,31 +227,27 @@ def test_pandas_parquet_configuration_options(tmpdir):
         'bool': np.random.randn(size) > 0
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = A.Table.from_pandas(df)
+    arrow_table = pa.Table.from_pandas(df)
 
     for use_dictionary in [True, False]:
-        A.parquet.write_table(
-                arrow_table,
-                filename.strpath,
-                version="2.0",
-                use_dictionary=use_dictionary)
+        pq.write_table(arrow_table, filename.strpath,
+                       version="2.0",
+                       use_dictionary=use_dictionary)
         table_read = pq.read_table(filename.strpath)
         df_read = table_read.to_pandas()
         pdt.assert_frame_equal(df, df_read)
 
     for compression in ['NONE', 'SNAPPY', 'GZIP']:
-        A.parquet.write_table(
-                arrow_table,
-                filename.strpath,
-                version="2.0",
-                compression=compression)
+        pq.write_table(arrow_table, filename.strpath,
+                       version="2.0",
+                       compression=compression)
         table_read = pq.read_table(filename.strpath)
         df_read = table_read.to_pandas()
         pdt.assert_frame_equal(df, df_read)
 
 
 def make_sample_file(df):
-    a_table = A.Table.from_pandas(df, timestamps_to_ms=True)
+    a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
     pq.write_table(a_table, buf, compression='SNAPPY', version='2.0')
@@ -315,7 +325,7 @@ def test_pass_separate_metadata():
     # ARROW-471
     df = alltypes_sample(size=10000)
 
-    a_table = A.Table.from_pandas(df, timestamps_to_ms=True)
+    a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
     pq.write_table(a_table, buf, compression='snappy', version='2.0')
@@ -328,3 +338,72 @@ def test_pass_separate_metadata():
     fileh = pq.ParquetFile(buf, metadata=metadata)
 
     pdt.assert_frame_equal(df, fileh.read().to_pandas())
+
+
+@parquet
+def test_read_multiple_files(tmpdir):
+    nfiles = 10
+    size = 5
+
+    dirpath = tmpdir.join(guid()).strpath
+    os.mkdir(dirpath)
+
+    test_data = []
+    paths = []
+    for i in range(nfiles):
+        df = _test_dataframe(size, seed=i)
+
+        # Hack so that we don't have a dtype cast in v1 files
+        df['uint32'] = df['uint32'].astype(np.int64)
+
+        path = pjoin(dirpath, '{0}.parquet'.format(i))
+
+        table = pa.Table.from_pandas(df)
+        pq.write_table(table, path)
+
+        test_data.append(table)
+        paths.append(path)
+
+    result = pq.read_multiple_files(paths)
+    expected = pa.concat_tables(test_data)
+
+    assert result.equals(expected)
+
+    # Read with provided metadata
+    metadata = pq.ParquetFile(paths[0]).metadata
+
+    result2 = pq.read_multiple_files(paths, metadata=metadata)
+    assert result2.equals(expected)
+
+    result3 = pa.localfs.read_parquet(dirpath, schema=metadata.schema)
+    assert result3.equals(expected)
+
+    # Read column subset
+    to_read = [result[0], result[3], result[6]]
+    result = pa.localfs.read_parquet(
+        dirpath, columns=[c.name for c in to_read])
+    expected = pa.Table.from_arrays(to_read)
+    assert result.equals(expected)
+
+    # Test failure modes with non-uniform metadata
+    bad_apple = _test_dataframe(size, seed=i).iloc[:, :4]
+    bad_apple_path = tmpdir.join('{0}.parquet'.format(guid())).strpath
+
+    t = pa.Table.from_pandas(bad_apple)
+    pq.write_table(t, bad_apple_path)
+
+    bad_meta = pq.ParquetFile(bad_apple_path).metadata
+
+    with pytest.raises(ValueError):
+        pq.read_multiple_files(paths + [bad_apple_path])
+
+    with pytest.raises(ValueError):
+        pq.read_multiple_files(paths, metadata=bad_meta)
+
+    mixed_paths = [bad_apple_path, paths[0]]
+
+    with pytest.raises(ValueError):
+        pq.read_multiple_files(mixed_paths, schema=bad_meta.schema)
+
+    with pytest.raises(ValueError):
+        pq.read_multiple_files(mixed_paths)
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 62e51f8dee846..ef600a06296cb 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -32,7 +32,7 @@ def test_bool(self):
         v = arr[0]
         assert isinstance(v, A.BooleanValue)
         assert repr(v) == "True"
-        assert v.as_py() == True
+        assert v.as_py() is True
 
         assert arr[1] is A.NA
 
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 4aa8112a91769..507ebb878d87b 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -85,4 +85,3 @@ def test_schema_equals(self):
         del fields[-1]
         sch3 = A.schema(fields)
         assert not sch1.equals(sch3)
-
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 6f00c7391f66d..d49b33c9f42d6 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -21,16 +21,43 @@
 import pandas as pd
 import pytest
 
+from pyarrow.compat import unittest
 import pyarrow as pa
 
 
+class TestColumn(unittest.TestCase):
+
+    def test_basics(self):
+        data = [
+            pa.from_pylist([-10, -5, 0, 5, 10])
+        ]
+        table = pa.Table.from_arrays(data, names=['a'], name='table_name')
+        column = table.column(0)
+        assert column.name == 'a'
+        assert column.length() == 5
+        assert len(column) == 5
+        assert column.shape == (5,)
+        assert column.to_pylist() == [-10, -5, 0, 5, 10]
+
+    def test_pandas(self):
+        data = [
+            pa.from_pylist([-10, -5, 0, 5, 10])
+        ]
+        table = pa.Table.from_arrays(data, names=['a'], name='table_name')
+        column = table.column(0)
+        series = column.to_pandas()
+        assert series.name == 'a'
+        assert series.shape == (5,)
+        assert series.iloc[0] == -10
+
+
 def test_recordbatch_basics():
     data = [
         pa.from_pylist(range(5)),
         pa.from_pylist([-10, -5, 0, 5, 10])
     ]
 
-    batch = pa.RecordBatch.from_arrays(['c0', 'c1'], data)
+    batch = pa.RecordBatch.from_arrays(data, ['c0', 'c1'])
 
     assert len(batch) == 5
     assert batch.num_rows == 5
@@ -95,7 +122,7 @@ def test_table_basics():
         pa.from_pylist(range(5)),
         pa.from_pylist([-10, -5, 0, 5, 10])
     ]
-    table = pa.Table.from_arrays(('a', 'b'), data, 'table_name')
+    table = pa.Table.from_arrays(data, names=('a', 'b'), name='table_name')
     assert table.name == 'table_name'
     assert len(table) == 5
     assert table.num_rows == 5
@@ -121,19 +148,19 @@ def test_concat_tables():
         [1., 2., 3., 4., 5.]
     ]
 
-    t1 = pa.Table.from_arrays(('a', 'b'), [pa.from_pylist(x)
-                                           for x in data], 'table_name')
-    t2 = pa.Table.from_arrays(('a', 'b'), [pa.from_pylist(x)
-                                           for x in data2], 'table_name')
+    t1 = pa.Table.from_arrays([pa.from_pylist(x) for x in data],
+                              names=('a', 'b'), name='table_name')
+    t2 = pa.Table.from_arrays([pa.from_pylist(x) for x in data2],
+                              names=('a', 'b'), name='table_name')
 
     result = pa.concat_tables([t1, t2], output_name='foo')
     assert result.name == 'foo'
     assert len(result) == 10
 
-    expected = pa.Table.from_arrays(
-        ('a', 'b'), [pa.from_pylist(x + y)
-                     for x, y in zip(data, data2)],
-        'foo')
+    expected = pa.Table.from_arrays([pa.from_pylist(x + y)
+                                     for x, y in zip(data, data2)],
+                                    names=('a', 'b'),
+                                    name='foo')
 
     assert result.equals(expected)
 
@@ -143,7 +170,8 @@ def test_table_pandas():
         pa.from_pylist(range(5)),
         pa.from_pylist([-10, -5, 0, 5, 10])
     ]
-    table = pa.Table.from_arrays(('a', 'b'), data, 'table_name')
+    table = pa.Table.from_arrays(data, names=('a', 'b'),
+                                 name='table_name')
 
     # TODO: Use this part once from_pandas is implemented
     # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}
diff --git a/python/pyarrow/util.py b/python/pyarrow/util.py
new file mode 100644
index 0000000000000..4b6a8356330d5
--- /dev/null
+++ b/python/pyarrow/util.py
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Miscellaneous utility code
+
+
+def implements(f):
+    def decorator(g):
+        g.__doc__ = f.__doc__
+        return g
+    return decorator

From 69cdbd8ce665138ce35bb34d0bbe8087c0e9513e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Jan 2017 06:43:05 -0500
Subject: [PATCH 0288/1644] ARROW-494: [C++] Extend lifetime of memory mapped
 data if any buffers reference it

If you read memory in an IPC scenario and then the `arrow::io::MemoryMappedFile` goes out of scope, before this patch the memory was being unmapped even if there are `arrow::Buffer` object referencing it.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #298 from wesm/ARROW-494 and squashes the following commits:

60222e3 [Wes McKinney] clang-format
2960d17 [Wes McKinney] Add C++ unit test
d7d776a [Wes McKinney] Add Python unit test where memory mapped file is garbage collected
edf1295 [Wes McKinney] Reimplement memory map owner as Buffer subclass so that MemoryMappedFile can be safely destructed without invalidating Buffers referencing the mapped data
---
 cpp/src/arrow/io/file.cc         | 94 ++++++++++++++++++--------------
 cpp/src/arrow/io/file.h          |  7 +--
 cpp/src/arrow/io/io-file-test.cc | 31 +++++++++++
 python/pyarrow/tests/test_io.py  | 20 ++++++-
 4 files changed, 104 insertions(+), 48 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 1de6efa4d811f..3bf8dfa08f2ff 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -372,6 +372,8 @@ class OSFile {
 
   int64_t size() const { return size_; }
 
+  FileMode::type mode() const { return mode_; }
+
  protected:
   std::string path_;
 
@@ -513,14 +515,14 @@ int FileOutputStream::file_descriptor() const {
 // ----------------------------------------------------------------------
 // Implement MemoryMappedFile
 
-class MemoryMappedFile::MemoryMappedFileImpl : public OSFile {
+class MemoryMappedFile::MemoryMap : public MutableBuffer {
  public:
-  MemoryMappedFileImpl() : OSFile(), data_(nullptr) {}
+  MemoryMap() : MutableBuffer(nullptr, 0) {}
 
-  ~MemoryMappedFileImpl() {
-    if (is_open_) {
-      munmap(data_, size_);
-      OSFile::Close();
+  ~MemoryMap() {
+    if (file_->is_open()) {
+      munmap(mutable_data_, size_);
+      file_->Close();
     }
   }
 
@@ -528,27 +530,35 @@ class MemoryMappedFile::MemoryMappedFileImpl : public OSFile {
     int prot_flags;
     int map_mode;
 
+    file_.reset(new OSFile());
+
     if (mode != FileMode::READ) {
       // Memory mapping has permission failures if PROT_READ not set
       prot_flags = PROT_READ | PROT_WRITE;
       map_mode = MAP_SHARED;
       constexpr bool append = true;
       constexpr bool write_only = false;
-      RETURN_NOT_OK(OSFile::OpenWriteable(path, append, write_only));
-      mode_ = mode;
+      RETURN_NOT_OK(file_->OpenWriteable(path, append, write_only));
+
+      is_mutable_ = true;
     } else {
       prot_flags = PROT_READ;
       map_mode = MAP_PRIVATE;  // Changes are not to be committed back to the file
-      RETURN_NOT_OK(OSFile::OpenReadable(path));
+      RETURN_NOT_OK(file_->OpenReadable(path));
+
+      is_mutable_ = false;
     }
 
-    void* result = mmap(nullptr, size_, prot_flags, map_mode, fd(), 0);
+    void* result = mmap(nullptr, file_->size(), prot_flags, map_mode, file_->fd(), 0);
     if (result == MAP_FAILED) {
       std::stringstream ss;
       ss << "Memory mapping file failed, errno: " << errno;
       return Status::IOError(ss.str());
     }
-    data_ = reinterpret_cast<uint8_t*>(result);
+
+    data_ = mutable_data_ = reinterpret_cast<uint8_t*>(result);
+    size_ = file_->size();
+
     position_ = 0;
 
     return Status::OK();
@@ -566,50 +576,45 @@ class MemoryMappedFile::MemoryMappedFileImpl : public OSFile {
 
   void advance(int64_t nbytes) { position_ = position_ + nbytes; }
 
-  uint8_t* data() { return data_; }
+  uint8_t* head() { return mutable_data_ + position_; }
 
-  uint8_t* head() { return data_ + position_; }
+  bool writable() { return file_->mode() != FileMode::READ; }
 
-  bool writable() { return mode_ != FileMode::READ; }
+  bool opened() { return file_->is_open(); }
 
-  bool opened() { return is_open_; }
+  int fd() const { return file_->fd(); }
 
  private:
+  std::unique_ptr<OSFile> file_;
   int64_t position_;
-
-  // The memory map
-  uint8_t* data_;
 };
 
-MemoryMappedFile::MemoryMappedFile(FileMode::type mode) {
-  ReadableFileInterface::set_mode(mode);
-}
-
+MemoryMappedFile::MemoryMappedFile() {}
 MemoryMappedFile::~MemoryMappedFile() {}
 
 Status MemoryMappedFile::Open(const std::string& path, FileMode::type mode,
     std::shared_ptr<MemoryMappedFile>* out) {
-  std::shared_ptr<MemoryMappedFile> result(new MemoryMappedFile(mode));
+  std::shared_ptr<MemoryMappedFile> result(new MemoryMappedFile());
 
-  result->impl_.reset(new MemoryMappedFileImpl());
-  RETURN_NOT_OK(result->impl_->Open(path, mode));
+  result->memory_map_.reset(new MemoryMap());
+  RETURN_NOT_OK(result->memory_map_->Open(path, mode));
 
   *out = result;
   return Status::OK();
 }
 
 Status MemoryMappedFile::GetSize(int64_t* size) {
-  *size = impl_->size();
+  *size = memory_map_->size();
   return Status::OK();
 }
 
 Status MemoryMappedFile::Tell(int64_t* position) {
-  *position = impl_->position();
+  *position = memory_map_->position();
   return Status::OK();
 }
 
 Status MemoryMappedFile::Seek(int64_t position) {
-  return impl_->Seek(position);
+  return memory_map_->Seek(position);
 }
 
 Status MemoryMappedFile::Close() {
@@ -618,19 +623,24 @@ Status MemoryMappedFile::Close() {
 }
 
 Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-  nbytes = std::max<int64_t>(0, std::min(nbytes, impl_->size() - impl_->position()));
-  if (nbytes > 0) { std::memcpy(out, impl_->head(), nbytes); }
+  nbytes = std::max<int64_t>(
+      0, std::min(nbytes, memory_map_->size() - memory_map_->position()));
+  if (nbytes > 0) { std::memcpy(out, memory_map_->head(), nbytes); }
   *bytes_read = nbytes;
-  impl_->advance(nbytes);
+  memory_map_->advance(nbytes);
   return Status::OK();
 }
 
 Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
-  nbytes = std::max<int64_t>(0, std::min(nbytes, impl_->size() - impl_->position()));
+  nbytes = std::max<int64_t>(
+      0, std::min(nbytes, memory_map_->size() - memory_map_->position()));
 
-  const uint8_t* data = nbytes > 0 ? impl_->head() : nullptr;
-  *out = std::make_shared<Buffer>(data, nbytes);
-  impl_->advance(nbytes);
+  if (nbytes > 0) {
+    *out = SliceBuffer(memory_map_, memory_map_->position(), nbytes);
+  } else {
+    *out = std::make_shared<Buffer>(nullptr, 0);
+  }
+  memory_map_->advance(nbytes);
   return Status::OK();
 }
 
@@ -639,19 +649,19 @@ bool MemoryMappedFile::supports_zero_copy() const {
 }
 
 Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
-  if (!impl_->opened() || !impl_->writable()) {
+  if (!memory_map_->opened() || !memory_map_->writable()) {
     return Status::IOError("Unable to write");
   }
 
-  RETURN_NOT_OK(impl_->Seek(position));
+  RETURN_NOT_OK(memory_map_->Seek(position));
   return WriteInternal(data, nbytes);
 }
 
 Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
-  if (!impl_->opened() || !impl_->writable()) {
+  if (!memory_map_->opened() || !memory_map_->writable()) {
     return Status::IOError("Unable to write");
   }
-  if (nbytes + impl_->position() > impl_->size()) {
+  if (nbytes + memory_map_->position() > memory_map_->size()) {
     return Status::Invalid("Cannot write past end of memory map");
   }
 
@@ -659,13 +669,13 @@ Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
 }
 
 Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
-  memcpy(impl_->head(), data, nbytes);
-  impl_->advance(nbytes);
+  memcpy(memory_map_->head(), data, nbytes);
+  memory_map_->advance(nbytes);
   return Status::OK();
 }
 
 int MemoryMappedFile::file_descriptor() const {
-  return impl_->fd();
+  return memory_map_->fd();
 }
 
 }  // namespace io
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 2387232b2157a..930346b0518b3 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -130,13 +130,12 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   int file_descriptor() const;
 
  private:
-  explicit MemoryMappedFile(FileMode::type mode);
+  MemoryMappedFile();
 
   Status WriteInternal(const uint8_t* data, int64_t nbytes);
 
-  // Hide the internal details of this class for now
-  class ARROW_NO_EXPORT MemoryMappedFileImpl;
-  std::unique_ptr<MemoryMappedFileImpl> impl_;
+  class ARROW_NO_EXPORT MemoryMap;
+  std::shared_ptr<MemoryMap> memory_map_;
 };
 
 }  // namespace io
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index f18f7b649eb9b..999b296465544 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -396,6 +396,37 @@ TEST_F(TestMemoryMappedFile, ReadOnly) {
   rommap->Close();
 }
 
+TEST_F(TestMemoryMappedFile, RetainMemoryMapReference) {
+  // ARROW-494
+
+  const int64_t buffer_size = 1024;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(1024, 0, buffer.data());
+
+  std::string path = "ipc-read-only-test";
+  CreateFile(path, buffer_size);
+
+  {
+    std::shared_ptr<MemoryMappedFile> rwmmap;
+    ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &rwmmap));
+    ASSERT_OK(rwmmap->Write(buffer.data(), buffer_size));
+    ASSERT_OK(rwmmap->Close());
+  }
+
+  std::shared_ptr<Buffer> out_buffer;
+
+  {
+    std::shared_ptr<MemoryMappedFile> rommap;
+    ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
+    ASSERT_OK(rommap->Read(buffer_size, &out_buffer));
+    ASSERT_OK(rommap->Close());
+  }
+
+  // valgrind will catch if memory is unmapped
+  ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
+}
+
 TEST_F(TestMemoryMappedFile, InvalidMode) {
   const int64_t buffer_size = 1024;
   std::vector<uint8_t> buffer(buffer_size);
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index f28d44a746c45..dfa84a27e6be9 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -16,6 +16,7 @@
 # under the License.
 
 from io import BytesIO
+import gc
 import os
 import pytest
 
@@ -163,9 +164,8 @@ def test_inmemory_write_after_closed():
 # ----------------------------------------------------------------------
 # OS files and memory maps
 
-@pytest.fixture(scope='session')
+@pytest.fixture
 def sample_disk_data(request):
-
     SIZE = 4096
     arr = np.random.randint(0, 256, size=SIZE).astype('u1')
     data = arr.tobytes()[:SIZE]
@@ -206,6 +206,22 @@ def test_memory_map_reader(sample_disk_data):
     _check_native_file_reader(io.MemoryMappedFile, sample_disk_data)
 
 
+def test_memory_map_retain_buffer_reference(sample_disk_data):
+    path, data = sample_disk_data
+
+    cases = []
+    with io.MemoryMappedFile(path, 'rb') as f:
+        cases.append((f.read_buffer(100), data[:100]))
+        cases.append((f.read_buffer(100), data[100:200]))
+        cases.append((f.read_buffer(100), data[200:300]))
+
+    # Call gc.collect() for good measure
+    gc.collect()
+
+    for buf, expected in cases:
+        assert buf.to_pybytes() == expected
+
+
 def test_os_file_reader(sample_disk_data):
     _check_native_file_reader(io.OSFile, sample_disk_data)
 

From c327b5fd2c35788c90b3fef2bc7b5faf89c07e49 Mon Sep 17 00:00:00 2001
From: Nong Li <nongli@gmail.com>
Date: Mon, 23 Jan 2017 06:43:59 -0500
Subject: [PATCH 0289/1644] ARROW-506: Java: Implement echo server for
 integration testing.

While implementing this, it became clear it made sense for the stream writer to
have an API to indicate EOS without closing the stream. The current message the
reader is expecting is a 4 byte size for the next batch. This patch proposes we
allow 0 as the size to indicate EOS.

Author: Nong Li <nongli@gmail.com>

Closes #295 from nongli/echo_server and squashes the following commits:

c115b02 [Nong Li] Add license header.
a3a50ca [Nong Li] ARROW-506: Java: Implement echo server for integration testing.
---
 .../org/apache/arrow/tools/EchoServer.java    | 130 ++++++++++++++++++
 .../apache/arrow/tools/EchoServerTest.java    | 129 +++++++++++++++++
 .../vector/stream/ArrowStreamWriter.java      |  14 +-
 .../vector/stream/MessageSerializer.java      |   7 +-
 .../arrow/vector/file/TestArrowFile.java      |   4 +-
 .../arrow/vector/stream/TestArrowStream.java  |   4 +-
 .../vector/stream/TestArrowStreamPipe.java    |   2 +-
 7 files changed, 278 insertions(+), 12 deletions(-)
 create mode 100644 java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
 create mode 100644 java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

diff --git a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
new file mode 100644
index 0000000000000..c00620e44b064
--- /dev/null
+++ b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.tools;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.ArrowStreamReader;
+import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+public class EchoServer {
+  private static final Logger LOGGER = LoggerFactory.getLogger(EchoServer.class);
+
+  private boolean closed = false;
+  private final ServerSocket serverSocket;
+
+  public EchoServer(int port) throws IOException {
+    LOGGER.info("Starting echo server.");
+    serverSocket = new ServerSocket(port);
+    LOGGER.info("Running echo server on port: " + port());
+  }
+
+  public int port() { return serverSocket.getLocalPort(); }
+
+  public static class ClientConnection implements AutoCloseable {
+    public final Socket socket;
+    public ClientConnection(Socket socket) {
+      this.socket = socket;
+    }
+
+    public void run() throws IOException {
+      BufferAllocator  allocator = new RootAllocator(Long.MAX_VALUE);
+      List<ArrowRecordBatch> batches = new ArrayList<ArrowRecordBatch>();
+      try (
+        InputStream in = socket.getInputStream();
+        OutputStream out = socket.getOutputStream();
+        ArrowStreamReader reader = new ArrowStreamReader(in, allocator);
+      ) {
+        // Read the entire input stream.
+        reader.init();
+        while (true) {
+          ArrowRecordBatch batch = reader.nextRecordBatch();
+          if (batch == null) break;
+          batches.add(batch);
+        }
+        LOGGER.info(String.format("Received %d batches", batches.size()));
+
+        // Write it back
+        try (ArrowStreamWriter writer = new ArrowStreamWriter(out, reader.getSchema())) {
+          for (ArrowRecordBatch batch: batches) {
+            writer.writeRecordBatch(batch);
+          }
+          writer.end();
+          Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
+        }
+        LOGGER.info("Done writing stream back.");
+      }
+    }
+
+    @Override
+    public void close() throws IOException {
+      socket.close();
+    }
+  }
+
+  public void run() throws IOException {
+    try {
+      while (!closed) {
+        LOGGER.info("Waiting to accept new client connection.");
+        Socket clientSocket = serverSocket.accept();
+        LOGGER.info("Accepted new client connection.");
+        try (ClientConnection client = new ClientConnection(clientSocket)) {
+          try {
+            client.run();
+          } catch (IOException e) {
+            LOGGER.warn("Error handling client connection.", e);
+          }
+        }
+        LOGGER.info("Closed connection with client");
+      }
+    } catch (java.net.SocketException ex) {
+      if (!closed) throw ex;
+    } finally {
+      serverSocket.close();
+      LOGGER.info("Server closed.");
+    }
+  }
+
+  public void close() throws IOException {
+    closed = true;
+    serverSocket.close();
+  }
+
+  public static void main(String[] args) throws Exception {
+    int port;
+    if (args.length > 0) {
+      port = Integer.parseInt(args[0]);
+    } else {
+      port = 8080;
+    }
+    new EchoServer(port).run();
+  }
+}
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
new file mode 100644
index 0000000000000..48d6162f423a3
--- /dev/null
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.tools;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.ArrowStreamReader;
+import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import io.netty.buffer.ArrowBuf;
+
+public class EchoServerTest {
+  public static ArrowBuf buf(BufferAllocator alloc, byte[] bytes) {
+    ArrowBuf buffer = alloc.buffer(bytes.length);
+    buffer.writeBytes(bytes);
+    return buffer;
+  }
+
+  public static byte[] array(ArrowBuf buf) {
+    byte[] bytes = new byte[buf.readableBytes()];
+    buf.readBytes(bytes);
+    return bytes;
+  }
+
+  private void testEchoServer(int serverPort, Schema schema, List<ArrowRecordBatch> batches)
+      throws UnknownHostException, IOException {
+    BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+    try (Socket socket = new Socket("localhost", serverPort);
+        ArrowStreamWriter writer = new ArrowStreamWriter(socket.getOutputStream(), schema);
+        ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), alloc)) {
+      for (ArrowRecordBatch batch: batches) {
+        writer.writeRecordBatch(batch);
+      }
+      writer.end();
+
+      reader.init();
+      assertEquals(schema, reader.getSchema());
+      for (int i = 0; i < batches.size(); i++) {
+        ArrowRecordBatch result = reader.nextRecordBatch();
+        ArrowRecordBatch expected = batches.get(i);
+        assertTrue(result != null);
+        assertEquals(expected.getBuffers().size(), result.getBuffers().size());
+        for (int j = 0; j < expected.getBuffers().size(); j++) {
+          assertTrue(expected.getBuffers().get(j).compareTo(result.getBuffers().get(j)) == 0);
+        }
+      }
+      ArrowRecordBatch result = reader.nextRecordBatch();
+      assertTrue(result == null);
+      assertEquals(reader.bytesRead(), writer.bytesWritten());
+    }
+  }
+
+  @Test
+  public void basicTest() throws InterruptedException, IOException {
+    final EchoServer server = new EchoServer(0);
+    int serverPort = server.port();
+    Thread serverThread = new Thread() {
+      @Override
+      public void run() {
+        try {
+          server.run();
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+    };
+    serverThread.start();
+
+    BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+    byte[] validity = new byte[] { (byte)255, 0};
+    byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+    ArrowBuf validityb = buf(alloc, validity);
+    ArrowBuf valuesb =  buf(alloc, values);
+    ArrowRecordBatch batch = new ArrowRecordBatch(
+        16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb));
+
+    Schema schema = new Schema(asList(new Field(
+        "testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
+
+    // Try an empty stream, just the header.
+    testEchoServer(serverPort, schema, new ArrayList<ArrowRecordBatch>());
+
+    // Try with one batch.
+    List<ArrowRecordBatch> batches = new ArrayList<>();
+    batches.add(batch);
+    testEchoServer(serverPort, schema, batches);
+
+    // Try with a few
+    for (int i = 0; i < 10; i++) {
+      batches.add(batch);
+    }
+    testEchoServer(serverPort, schema, batches);
+
+    server.close();
+    serverThread.join();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
index 06acf9f2c140e..60dc5861c9242 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
@@ -35,14 +35,14 @@ public class ArrowStreamWriter implements AutoCloseable {
    * Creates the stream writer. non-blocking.
    * totalBatches can be set if the writer knows beforehand. Can be -1 if unknown.
    */
-  public ArrowStreamWriter(WritableByteChannel out, Schema schema, int totalBatches) {
+  public ArrowStreamWriter(WritableByteChannel out, Schema schema) {
     this.out = new WriteChannel(out);
     this.schema = schema;
   }
 
-  public ArrowStreamWriter(OutputStream out, Schema schema, int totalBatches)
+  public ArrowStreamWriter(OutputStream out, Schema schema)
       throws IOException {
-    this(Channels.newChannel(out), schema, totalBatches);
+    this(Channels.newChannel(out), schema);
   }
 
   public long bytesWritten() { return out.getCurrentPosition(); }
@@ -53,6 +53,14 @@ public void writeRecordBatch(ArrowRecordBatch batch) throws IOException {
     MessageSerializer.serialize(out, batch);
   }
 
+  /**
+   * End the stream. This is not required and this object can simply be closed.
+   */
+  public void end() throws IOException {
+    checkAndSendHeader();
+    out.writeIntLittleEndian(0);
+  }
+
   @Override
   public void close() throws IOException {
     // The header might not have been sent if this is an empty stream. Send it even in
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 6e22dbd164d6e..7ab740c70782e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -235,11 +235,10 @@ private static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte heade
   private static Message deserializeMessage(ReadChannel in, byte headerType) throws IOException {
     // Read the message size. There is an i32 little endian prefix.
     ByteBuffer buffer = ByteBuffer.allocate(4);
-    if (in.readFully(buffer) != 4) {
-      return null;
-    }
-
+    if (in.readFully(buffer) != 4) return null;
     int messageLength = bytesToInt(buffer.array());
+    if (messageLength == 0) return null;
+
     buffer = ByteBuffer.allocate(messageLength);
     if (in.readFully(buffer) != messageLength) {
       throw new IOException(
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 9b9914480bad0..a83a2833c88bf 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -232,7 +232,7 @@ public void testWriteReadMultipleRBs() throws IOException {
       Schema schema = vectorUnloader0.getSchema();
       Assert.assertEquals(2, schema.getFields().size());
       try (ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
-          ArrowStreamWriter streamWriter = new ArrowStreamWriter(stream, schema, 2)) {
+          ArrowStreamWriter streamWriter = new ArrowStreamWriter(stream, schema)) {
         try (ArrowRecordBatch recordBatch = vectorUnloader0.getRecordBatch()) {
           Assert.assertEquals("RB #0", counts[0], recordBatch.getLength());
           arrowWriter.writeRecordBatch(recordBatch);
@@ -399,7 +399,7 @@ private void write(FieldVector parent, File file, OutputStream outStream) throws
     // Also try serializing to the stream writer.
     if (outStream != null) {
       try (
-          ArrowStreamWriter arrowWriter = new ArrowStreamWriter(outStream, schema, -1);
+          ArrowStreamWriter arrowWriter = new ArrowStreamWriter(outStream, schema);
           ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
           ) {
         arrowWriter.writeRecordBatch(recordBatch);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
index ba1cdaeeb2262..725272a0f072e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
@@ -42,7 +42,7 @@ public void testEmptyStream() throws IOException {
 
     // Write the stream.
     ByteArrayOutputStream out = new ByteArrayOutputStream();
-    try (ArrowStreamWriter writer = new ArrowStreamWriter(out, schema, -1)) {
+    try (ArrowStreamWriter writer = new ArrowStreamWriter(out, schema)) {
     }
 
     ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
@@ -66,7 +66,7 @@ public void testReadWrite() throws IOException {
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     long bytesWritten = 0;
-    try (ArrowStreamWriter writer = new ArrowStreamWriter(out, schema, numBatches)) {
+    try (ArrowStreamWriter writer = new ArrowStreamWriter(out, schema)) {
       ArrowBuf validityb = MessageSerializerTest.buf(alloc, validity);
       ArrowBuf valuesb =  MessageSerializerTest.buf(alloc, values);
       for (int i = 0; i < numBatches; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
index e187fa535cada..a0a7ffa279308 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
@@ -47,7 +47,7 @@ private final class WriterThread extends Thread {
     public WriterThread(int numBatches, WritableByteChannel sinkChannel)
         throws IOException {
       this.numBatches = numBatches;
-      writer = new ArrowStreamWriter(sinkChannel, schema, -1);
+      writer = new ArrowStreamWriter(sinkChannel, schema);
     }
 
     @Override

From 1f81adcc88b138c6ae5f5ffb3250f87239c89dc1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Jan 2017 09:10:18 -0500
Subject: [PATCH 0290/1644] ARROW-503: [Python] Implement Python interface to
 streaming file format

See the new `StreamWriter` and `StreamReader` classes. This patch is stacked on top of the patch for ARROW-475. Will rebase when that is merged.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #299 from wesm/ARROW-503 and squashes the following commits:

e9d918e [Wes McKinney] Close BufferOutputStream after completing file or stream writes
31e519f [Wes McKinney] Add function alias to preserve backwards compatibility
faac28c [Wes McKinney] Fix small bug in BinaryArray::Equals, add rudimentary StreamReader/Writer interface and tests
d9fb3dc [Wes McKinney] Refactoring, consolidate IPC code into io.pyx
---
 cpp/src/arrow/array.cc                   |   2 +-
 cpp/src/arrow/ipc/ipc-file-test.cc       |   2 +
 cpp/src/arrow/ipc/stream.cc              |   6 +-
 cpp/src/arrow/ipc/stream.h               |   3 +
 python/CMakeLists.txt                    |   1 -
 python/pyarrow/__init__.py               |   2 +
 python/pyarrow/includes/libarrow_ipc.pxd |  29 +-
 python/pyarrow/io.pyx                    | 367 ++++++++++++++++-------
 python/pyarrow/ipc.py                    |  83 +++++
 python/pyarrow/ipc.pyx                   | 115 -------
 python/pyarrow/schema.pyx                |   1 +
 python/pyarrow/table.pxd                 |   1 +
 python/pyarrow/tests/test_ipc.py         | 120 ++++----
 python/setup.py                          |   1 -
 14 files changed, 438 insertions(+), 295 deletions(-)
 create mode 100644 python/pyarrow/ipc.py
 delete mode 100644 python/pyarrow/ipc.pyx

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 7509520d12685..aa4a692e85cb9 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -359,7 +359,7 @@ bool BinaryArray::EqualsExact(const BinaryArray& other) const {
 
   if (!data_buffer_ && !(other.data_buffer_)) { return true; }
 
-  return data_buffer_->Equals(*other.data_buffer_, data_buffer_->size());
+  return data_buffer_->Equals(*other.data_buffer_, raw_offsets()[length_]);
 }
 
 bool BinaryArray::Equals(const std::shared_ptr<Array>& arr) const {
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index 15ceb80493632..7cd8054679e44 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -75,6 +75,7 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
       RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
     }
     RETURN_NOT_OK(writer->Close());
+    RETURN_NOT_OK(sink_->Close());
 
     // Current offset into stream is the end of the file
     int64_t footer_offset;
@@ -138,6 +139,7 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
       RETURN_NOT_OK(writer->WriteRecordBatch(batch));
     }
     RETURN_NOT_OK(writer->Close());
+    RETURN_NOT_OK(sink_->Close());
 
     // Open the file
     auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
diff --git a/cpp/src/arrow/ipc/stream.cc b/cpp/src/arrow/ipc/stream.cc
index a2ca672fbe0aa..c9057e860b1e8 100644
--- a/cpp/src/arrow/ipc/stream.cc
+++ b/cpp/src/arrow/ipc/stream.cc
@@ -117,9 +117,9 @@ Status StreamWriter::WriteRecordBatch(const RecordBatch& batch) {
 }
 
 Status StreamWriter::Close() {
-  // Close the stream
-  RETURN_NOT_OK(CheckStarted());
-  return sink_->Close();
+  // Write the schema if not already written
+  // User is responsible for closing the OutputStream
+  return CheckStarted();
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/stream.h b/cpp/src/arrow/ipc/stream.h
index 0b0e62f13fc5f..53f51dc73675f 100644
--- a/cpp/src/arrow/ipc/stream.h
+++ b/cpp/src/arrow/ipc/stream.h
@@ -54,6 +54,9 @@ class ARROW_EXPORT StreamWriter {
       std::shared_ptr<StreamWriter>* out);
 
   virtual Status WriteRecordBatch(const RecordBatch& batch);
+
+  /// Perform any logic necessary to finish the stream. User is responsible for
+  /// closing the actual OutputStream
   virtual Status Close();
 
  protected:
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index b3735b1d58653..d63fff48a011f 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -409,7 +409,6 @@ set(CYTHON_EXTENSIONS
   config
   error
   io
-  ipc
   scalar
   schema
   table
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index d563c7aa4055d..7c521db6280be 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -46,6 +46,8 @@
 from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
                         Buffer, InMemoryOutputStream, BufferReader)
 
+from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
+
 from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
                             BooleanValue,
                             Int8Value, Int16Value, Int32Value, Int64Value,
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
index 82957600d1eb6..bfece14fe6e03 100644
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -20,18 +20,37 @@
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (MemoryPool, CArray, CSchema,
                                         CRecordBatch)
-from pyarrow.includes.libarrow_io cimport (OutputStream, ReadableFileInterface)
+from pyarrow.includes.libarrow_io cimport (InputStream, OutputStream,
+                                           ReadableFileInterface)
 
-cdef extern from "arrow/ipc/file.h" namespace "arrow::ipc" nogil:
 
-    cdef cppclass CFileWriter " arrow::ipc::FileWriter":
+cdef extern from "arrow/ipc/stream.h" namespace "arrow::ipc" nogil:
+
+    cdef cppclass CStreamWriter " arrow::ipc::StreamWriter":
         @staticmethod
         CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
-                     shared_ptr[CFileWriter]* out)
+                     shared_ptr[CStreamWriter]* out)
 
+        CStatus Close()
         CStatus WriteRecordBatch(const CRecordBatch& batch)
 
-        CStatus Close()
+    cdef cppclass CStreamReader " arrow::ipc::StreamReader":
+
+        @staticmethod
+        CStatus Open(const shared_ptr[InputStream]& stream,
+                     shared_ptr[CStreamReader]* out)
+
+        shared_ptr[CSchema] schema()
+
+        CStatus GetNextRecordBatch(shared_ptr[CRecordBatch]* batch)
+
+
+cdef extern from "arrow/ipc/file.h" namespace "arrow::ipc" nogil:
+
+    cdef cppclass CFileWriter " arrow::ipc::FileWriter"(CStreamWriter):
+        @staticmethod
+        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
+                     shared_ptr[CFileWriter]* out)
 
     cdef cppclass CFileReader " arrow::ipc::FileReader":
 
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 26215122b7a23..0755ed8bb4d4f 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -15,20 +15,26 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Cython wrappers for IO interfaces defined in arrow/io
+# Cython wrappers for IO interfaces defined in arrow::io and messaging in
+# arrow::ipc
 
 # cython: profile=False
 # distutils: language = c++
 # cython: embedsignature = True
 
+from cython.operator cimport dereference as deref
+
 from libc.stdlib cimport malloc, free
 
 from pyarrow.includes.libarrow cimport *
-cimport pyarrow.includes.pyarrow as pyarrow
 from pyarrow.includes.libarrow_io cimport *
+from pyarrow.includes.libarrow_ipc cimport *
+cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.compat import frombytes, tobytes, encode_file_path
 from pyarrow.error cimport check_status
+from pyarrow.schema cimport Schema
+from pyarrow.table cimport RecordBatch, batch_from_cbatch
 
 cimport cpython as cp
 
@@ -38,6 +44,11 @@ import sys
 import threading
 import time
 
+
+# 64K
+DEFAULT_BUFFER_SIZE = 2 ** 16
+
+
 # To let us get a PyObject* and avoid Cython auto-ref-counting
 cdef extern from "Python.h":
     PyObject* PyBytes_FromStringAndSizeNative" PyBytes_FromStringAndSize"(
@@ -167,6 +178,129 @@ cdef class NativeFile:
 
         return wrap_buffer(output)
 
+    def download(self, stream_or_path, buffer_size=None):
+        """
+        Read file completely to local path (rather than reading completely into
+        memory). First seeks to the beginning of the file.
+        """
+        cdef:
+            int64_t bytes_read = 0
+            uint8_t* buf
+        self._assert_readable()
+
+        buffer_size = buffer_size or DEFAULT_BUFFER_SIZE
+
+        write_queue = Queue(50)
+
+        if not hasattr(stream_or_path, 'read'):
+            stream = open(stream_or_path, 'wb')
+            cleanup = lambda: stream.close()
+        else:
+            stream = stream_or_path
+            cleanup = lambda: None
+
+        done = False
+        exc_info = None
+        def bg_write():
+            try:
+                while not done or write_queue.qsize() > 0:
+                    try:
+                        buf = write_queue.get(timeout=0.01)
+                    except QueueEmpty:
+                        continue
+                    stream.write(buf)
+            except Exception as e:
+                exc_info = sys.exc_info()
+            finally:
+                cleanup()
+
+        self.seek(0)
+
+        writer_thread = threading.Thread(target=bg_write)
+
+        # This isn't ideal -- PyBytes_FromStringAndSize copies the data from
+        # the passed buffer, so it's hard for us to avoid doubling the memory
+        buf = <uint8_t*> malloc(buffer_size)
+        if buf == NULL:
+            raise MemoryError("Failed to allocate {0} bytes"
+                              .format(buffer_size))
+
+        writer_thread.start()
+
+        cdef int64_t total_bytes = 0
+        cdef int32_t c_buffer_size = buffer_size
+
+        try:
+            while True:
+                with nogil:
+                    check_status(self.rd_file.get()
+                                 .Read(c_buffer_size, &bytes_read, buf))
+
+                total_bytes += bytes_read
+
+                # EOF
+                if bytes_read == 0:
+                    break
+
+                pybuf = cp.PyBytes_FromStringAndSize(<const char*>buf,
+                                                     bytes_read)
+
+                write_queue.put_nowait(pybuf)
+        finally:
+            free(buf)
+            done = True
+
+        writer_thread.join()
+        if exc_info is not None:
+            raise exc_info[0], exc_info[1], exc_info[2]
+
+    def upload(self, stream, buffer_size=None):
+        """
+        Pipe file-like object to file
+        """
+        write_queue = Queue(50)
+        self._assert_writeable()
+
+        buffer_size = buffer_size or DEFAULT_BUFFER_SIZE
+
+        done = False
+        exc_info = None
+        def bg_write():
+            try:
+                while not done or write_queue.qsize() > 0:
+                    try:
+                        buf = write_queue.get(timeout=0.01)
+                    except QueueEmpty:
+                        continue
+
+                    self.write(buf)
+
+            except Exception as e:
+                exc_info = sys.exc_info()
+
+        writer_thread = threading.Thread(target=bg_write)
+        writer_thread.start()
+
+        try:
+            while True:
+                buf = stream.read(buffer_size)
+                if not buf:
+                    break
+
+                if writer_thread.is_alive():
+                    while write_queue.full():
+                        time.sleep(0.01)
+                else:
+                    break
+
+                write_queue.put_nowait(buf)
+        finally:
+            done = True
+
+        writer_thread.join()
+        if exc_info is not None:
+            raise exc_info[0], exc_info[1], exc_info[2]
+
 
 # ----------------------------------------------------------------------
 # Python file-like objects
@@ -679,58 +813,17 @@ cdef class _HdfsClient:
 
         return out
 
-    def upload(self, path, stream, buffer_size=2**16):
+    def download(self, path, stream, buffer_size=None):
+        with self.open(path, 'rb') as f:
+            f.download(stream, buffer_size=buffer_size)
+
+    def upload(self, path, stream, buffer_size=None):
         """
         Upload file-like object to HDFS path
         """
-        write_queue = Queue(50)
-
         with self.open(path, 'wb') as f:
-            done = False
-            exc_info = None
-            def bg_write():
-                try:
-                    while not done or write_queue.qsize() > 0:
-                        try:
-                            buf = write_queue.get(timeout=0.01)
-                        except QueueEmpty:
-                            continue
-
-                        f.write(buf)
-
-                except Exception as e:
-                    exc_info = sys.exc_info()
-
-            writer_thread = threading.Thread(target=bg_write)
-            writer_thread.start()
+            f.upload(stream, buffer_size=buffer_size)
 
-            try:
-                while True:
-                    buf = stream.read(buffer_size)
-                    if not buf:
-                        break
-
-                    if writer_thread.is_alive():
-                        while write_queue.full():
-                            time.sleep(0.01)
-                    else:
-                        break
-
-                    write_queue.put_nowait(buf)
-            finally:
-                done = True
-
-            writer_thread.join()
-            if exc_info is not None:
-                raise exc_info[0], exc_info[1], exc_info[2]
-
-    def download(self, path, stream, buffer_size=None):
-        with self.open(path, 'rb', buffer_size=buffer_size) as f:
-            f.download(stream)
-
-
-# ----------------------------------------------------------------------
-# Specialization for HDFS
 
 # ARROW-404: Helper class to ensure that files are closed before the
 # client. During deallocation of the extension class, the attributes are
@@ -766,75 +859,139 @@ cdef class HdfsFile(NativeFile):
     def __dealloc__(self):
         self.parent = None
 
-    def download(self, stream_or_path):
+# ----------------------------------------------------------------------
+# File and stream readers and writers
+
+cdef class _StreamWriter:
+    cdef:
+        shared_ptr[CStreamWriter] writer
+        shared_ptr[OutputStream] sink
+        bint closed
+
+    def __cinit__(self):
+        self.closed = True
+
+    def __dealloc__(self):
+        if not self.closed:
+            self.close()
+
+    def _open(self, sink, Schema schema):
+        get_writer(sink, &self.sink)
+
+        with nogil:
+            check_status(CStreamWriter.Open(self.sink.get(), schema.sp_schema,
+                                            &self.writer))
+
+        self.closed = False
+
+    def write_batch(self, RecordBatch batch):
+        with nogil:
+            check_status(self.writer.get()
+                         .WriteRecordBatch(deref(batch.batch)))
+
+    def close(self):
+        with nogil:
+            check_status(self.writer.get().Close())
+        self.closed = True
+
+
+cdef class _StreamReader:
+    cdef:
+        shared_ptr[CStreamReader] reader
+
+    cdef readonly:
+        Schema schema
+
+    def __cinit__(self):
+        pass
+
+    def _open(self, source):
+        cdef:
+            shared_ptr[ReadableFileInterface] reader
+            shared_ptr[InputStream] in_stream
+
+        get_reader(source, &reader)
+        in_stream = <shared_ptr[InputStream]> reader
+
+        with nogil:
+            check_status(CStreamReader.Open(in_stream, &self.reader))
+
+        schema = Schema()
+        schema.init_schema(self.reader.get().schema())
+
+    def get_next_batch(self):
         """
-        Read file completely to local path (rather than reading completely into
-        memory). First seeks to the beginning of the file.
+        Read next RecordBatch from the stream. Raises StopIteration at end of
+        stream
         """
-        cdef:
-            int64_t bytes_read = 0
-            uint8_t* buf
-        self._assert_readable()
+        cdef shared_ptr[CRecordBatch] batch
 
-        write_queue = Queue(50)
+        with nogil:
+            check_status(self.reader.get().GetNextRecordBatch(&batch))
 
-        if not hasattr(stream_or_path, 'read'):
-            stream = open(stream_or_path, 'wb')
-            cleanup = lambda: stream.close()
-        else:
-            stream = stream_or_path
-            cleanup = lambda: None
+        if batch.get() == NULL:
+            raise StopIteration
 
-        done = False
-        exc_info = None
-        def bg_write():
-            try:
-                while not done or write_queue.qsize() > 0:
-                    try:
-                        buf = write_queue.get(timeout=0.01)
-                    except QueueEmpty:
-                        continue
-                    stream.write(buf)
-            except Exception as e:
-                exc_info = sys.exc_info()
-            finally:
-                cleanup()
+        return batch_from_cbatch(batch)
 
-        self.seek(0)
 
-        writer_thread = threading.Thread(target=bg_write)
+cdef class _FileWriter(_StreamWriter):
 
-        # This isn't ideal -- PyBytes_FromStringAndSize copies the data from
-        # the passed buffer, so it's hard for us to avoid doubling the memory
-        buf = <uint8_t*> malloc(self.buffer_size)
-        if buf == NULL:
-            raise MemoryError("Failed to allocate {0} bytes"
-                              .format(self.buffer_size))
+    def _open(self, sink, Schema schema):
+        cdef shared_ptr[CFileWriter] writer
+        get_writer(sink, &self.sink)
 
-        writer_thread.start()
+        with nogil:
+            check_status(CFileWriter.Open(self.sink.get(), schema.sp_schema,
+                                          &writer))
 
-        cdef int64_t total_bytes = 0
+        # Cast to base class, because has same interface
+        self.writer = <shared_ptr[CStreamWriter]> writer
+        self.closed = False
 
-        try:
-            while True:
-                with nogil:
-                    check_status(self.rd_file.get()
-                                 .Read(self.buffer_size, &bytes_read, buf))
 
-                total_bytes += bytes_read
+cdef class _FileReader:
+    cdef:
+        shared_ptr[CFileReader] reader
 
-                # EOF
-                if bytes_read == 0:
-                    break
+    def __cinit__(self):
+        pass
 
-                pybuf = cp.PyBytes_FromStringAndSize(<const char*>buf,
-                                                     bytes_read)
+    def _open(self, source, footer_offset=None):
+        cdef shared_ptr[ReadableFileInterface] reader
+        get_reader(source, &reader)
 
-                write_queue.put_nowait(pybuf)
-        finally:
-            free(buf)
-            done = True
+        cdef int64_t offset = 0
+        if footer_offset is not None:
+            offset = footer_offset
 
-        writer_thread.join()
-        if exc_info is not None:
-            raise exc_info[0], exc_info[1], exc_info[2]
+        with nogil:
+            if offset != 0:
+                check_status(CFileReader.Open2(reader, offset, &self.reader))
+            else:
+                check_status(CFileReader.Open(reader, &self.reader))
+
+    property num_dictionaries:
+
+        def __get__(self):
+            return self.reader.get().num_dictionaries()
+
+    property num_record_batches:
+
+        def __get__(self):
+            return self.reader.get().num_record_batches()
+
+    def get_batch(self, int i):
+        cdef shared_ptr[CRecordBatch] batch
+
+        if i < 0 or i >= self.num_record_batches:
+            raise ValueError('Batch number {0} out of range'.format(i))
+
+        with nogil:
+            check_status(self.reader.get().GetRecordBatch(i, &batch))
+
+        return batch_from_cbatch(batch)
+
+    # TODO(wesm): ARROW-503: Function was renamed. Remove after a period of
+    # time has passed
+    get_record_batch = get_batch
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
new file mode 100644
index 0000000000000..5a5616564324c
--- /dev/null
+++ b/python/pyarrow/ipc.py
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Arrow file and stream reader/writer classes, and other messaging tools
+
+import pyarrow.io as io
+
+
+class StreamReader(io._StreamReader):
+    """
+    Reader for the Arrow streaming binary format
+
+    Parameters
+    ----------
+    source : str, pyarrow.NativeFile, or file-like Python object
+        Either a file path, or a readable file object
+    """
+    def __init__(self, source):
+        self._open(source)
+
+    def __iter__(self):
+        while True:
+            yield self.get_next_batch()
+
+
+class StreamWriter(io._StreamWriter):
+    """
+    Writer for the Arrow streaming binary format
+
+    Parameters
+    ----------
+    sink : str, pyarrow.NativeFile, or file-like Python object
+        Either a file path, or a writeable file object
+    schema : pyarrow.Schema
+        The Arrow schema for data to be written to the file
+    """
+    def __init__(self, sink, schema):
+        self._open(sink, schema)
+
+
+class FileReader(io._FileReader):
+    """
+    Class for reading Arrow record batch data from the Arrow binary file format
+
+    Parameters
+    ----------
+    source : str, pyarrow.NativeFile, or file-like Python object
+        Either a file path, or a readable file object
+    footer_offset : int, default None
+        If the file is embedded in some larger file, this is the byte offset to
+        the very end of the file data
+    """
+    def __init__(self, source, footer_offset=None):
+        self._open(source, footer_offset=footer_offset)
+
+
+class FileWriter(io._FileWriter):
+    """
+    Writer to create the Arrow binary file format
+
+    Parameters
+    ----------
+    sink : str, pyarrow.NativeFile, or file-like Python object
+        Either a file path, or a writeable file object
+    schema : pyarrow.Schema
+        The Arrow schema for data to be written to the file
+    """
+    def __init__(self, sink, schema):
+        self._open(sink, schema)
diff --git a/python/pyarrow/ipc.pyx b/python/pyarrow/ipc.pyx
deleted file mode 100644
index 22069a7290ead..0000000000000
--- a/python/pyarrow/ipc.pyx
+++ /dev/null
@@ -1,115 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Cython wrappers for arrow::ipc
-
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from cython.operator cimport dereference as deref
-
-from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.libarrow_io cimport *
-from pyarrow.includes.libarrow_ipc cimport *
-cimport pyarrow.includes.pyarrow as pyarrow
-
-from pyarrow.error cimport check_status
-from pyarrow.io cimport NativeFile, get_reader, get_writer
-from pyarrow.schema cimport Schema
-from pyarrow.table cimport RecordBatch
-
-from pyarrow.compat import frombytes, tobytes
-import pyarrow.io as io
-
-cimport cpython as cp
-
-
-cdef class ArrowFileWriter:
-    cdef:
-        shared_ptr[CFileWriter] writer
-        shared_ptr[OutputStream] sink
-        bint closed
-
-    def __cinit__(self, sink, Schema schema):
-        self.closed = True
-        get_writer(sink, &self.sink)
-
-        with nogil:
-            check_status(CFileWriter.Open(self.sink.get(), schema.sp_schema,
-                                          &self.writer))
-
-        self.closed = False
-
-    def __dealloc__(self):
-        if not self.closed:
-            self.close()
-
-    def write_record_batch(self, RecordBatch batch):
-        with nogil:
-            check_status(self.writer.get()
-                         .WriteRecordBatch(deref(batch.batch)))
-
-    def close(self):
-        with nogil:
-            check_status(self.writer.get().Close())
-        self.closed = True
-
-
-cdef class ArrowFileReader:
-    cdef:
-        shared_ptr[CFileReader] reader
-
-    def __cinit__(self, source, footer_offset=None):
-        cdef shared_ptr[ReadableFileInterface] reader
-        get_reader(source, &reader)
-
-        cdef int64_t offset = 0
-        if footer_offset is not None:
-            offset = footer_offset
-
-        with nogil:
-            if offset != 0:
-                check_status(CFileReader.Open2(reader, offset, &self.reader))
-            else:
-                check_status(CFileReader.Open(reader, &self.reader))
-
-    property num_dictionaries:
-
-        def __get__(self):
-            return self.reader.get().num_dictionaries()
-
-    property num_record_batches:
-
-        def __get__(self):
-            return self.reader.get().num_record_batches()
-
-    def get_record_batch(self, int i):
-        cdef:
-            shared_ptr[CRecordBatch] batch
-            RecordBatch result
-
-        if i < 0 or i >= self.num_record_batches:
-            raise ValueError('Batch number {0} out of range'.format(i))
-
-        with nogil:
-            check_status(self.reader.get().GetRecordBatch(i, &batch))
-
-        result = RecordBatch()
-        result.init(batch)
-
-        return result
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 2bcfec1bcf3e2..52eeeaf717622 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -112,6 +112,7 @@ cdef class Field:
         def __get__(self):
             return frombytes(self.field.name)
 
+
 cdef class Schema:
 
     def __cinit__(self):
diff --git a/python/pyarrow/table.pxd b/python/pyarrow/table.pxd
index df3687ddf9761..389727b4dc1d7 100644
--- a/python/pyarrow/table.pxd
+++ b/python/pyarrow/table.pxd
@@ -59,3 +59,4 @@ cdef class RecordBatch:
     cdef _check_nullptr(self)
 
 cdef api object table_from_ctable(const shared_ptr[CTable]& ctable)
+cdef api object batch_from_cbatch(const shared_ptr[CRecordBatch]& cbatch)
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index bbd6c6a56705c..819d1b71b8546 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -16,21 +16,20 @@
 # under the License.
 
 import io
+import pytest
 
 import numpy as np
 
 from pandas.util.testing import assert_frame_equal
 import pandas as pd
 
-import pyarrow as A
-import pyarrow.io as aio
-import pyarrow.ipc as ipc
+from pyarrow.compat import unittest
+import pyarrow as pa
 
 
-class RoundtripTest(object):
-    # Also tests writing zero-copy NumPy array with additional padding
+class MessagingTest(object):
 
-    def __init__(self):
+    def setUp(self):
         self.sink = self._get_sink()
 
     def _get_sink(self):
@@ -39,14 +38,15 @@ def _get_sink(self):
     def _get_source(self):
         return self.sink.getvalue()
 
-    def run(self):
+    def write_batches(self):
         nrows = 5
         df = pd.DataFrame({
             'one': np.random.randn(nrows),
             'two': ['foo', np.nan, 'bar', 'bazbaz', 'qux']})
 
-        batch = A.RecordBatch.from_pandas(df)
-        writer = ipc.ArrowFileWriter(self.sink, batch.schema)
+        batch = pa.RecordBatch.from_pandas(df)
+
+        writer = self._get_writer(self.sink, batch.schema)
 
         num_batches = 5
         frames = []
@@ -55,46 +55,73 @@ def run(self):
             unique_df = df.copy()
             unique_df['one'] = np.random.randn(nrows)
 
-            batch = A.RecordBatch.from_pandas(unique_df)
-            writer.write_record_batch(batch)
+            batch = pa.RecordBatch.from_pandas(unique_df)
+            writer.write_batch(batch)
             frames.append(unique_df)
             batches.append(batch)
 
         writer.close()
+        return batches
+
+
+class TestFile(MessagingTest, unittest.TestCase):
+    # Also tests writing zero-copy NumPy array with additional padding
+
+    def _get_writer(self, sink, schema):
+        return pa.FileWriter(sink, schema)
 
+    def test_simple_roundtrip(self):
+        batches = self.write_batches()
         file_contents = self._get_source()
-        reader = ipc.ArrowFileReader(aio.BufferReader(file_contents))
 
-        assert reader.num_record_batches == num_batches
+        reader = pa.FileReader(pa.BufferReader(file_contents))
 
-        for i in range(num_batches):
+        assert reader.num_record_batches == len(batches)
+
+        for i, batch in enumerate(batches):
             # it works. Must convert back to DataFrame
-            batch = reader.get_record_batch(i)
+            batch = reader.get_batch(i)
             assert batches[i].equals(batch)
 
 
-class InMemoryStreamTest(RoundtripTest):
+class TestStream(MessagingTest, unittest.TestCase):
+
+    def _get_writer(self, sink, schema):
+        return pa.StreamWriter(sink, schema)
+
+    def test_simple_roundtrip(self):
+        batches = self.write_batches()
+        file_contents = self._get_source()
+        reader = pa.StreamReader(pa.BufferReader(file_contents))
+
+        total = 0
+        for i, next_batch in enumerate(reader):
+            assert next_batch.equals(batches[i])
+            total += 1
+
+        assert total == len(batches)
+
+        with pytest.raises(StopIteration):
+            reader.get_next_batch()
+
+
+class TestInMemoryFile(TestFile):
 
     def _get_sink(self):
-        return aio.InMemoryOutputStream()
+        return pa.InMemoryOutputStream()
 
     def _get_source(self):
         return self.sink.get_result()
 
 
-def test_ipc_file_simple_roundtrip():
-    helper = RoundtripTest()
-    helper.run()
-
-
 def test_ipc_zero_copy_numpy():
     df = pd.DataFrame({'foo': [1.5]})
 
-    batch = A.RecordBatch.from_pandas(df)
-    sink = aio.InMemoryOutputStream()
+    batch = pa.RecordBatch.from_pandas(df)
+    sink = pa.InMemoryOutputStream()
     write_file(batch, sink)
     buffer = sink.get_result()
-    reader = aio.BufferReader(buffer)
+    reader = pa.BufferReader(buffer)
 
     batches = read_file(reader)
 
@@ -103,48 +130,13 @@ def test_ipc_zero_copy_numpy():
     assert_frame_equal(df, rdf)
 
 
-# XXX: For benchmarking
-
-def big_batch():
-    K = 2**4
-    N = 2**20
-    df = pd.DataFrame(
-        np.random.randn(K, N).T,
-        columns=[str(i) for i in range(K)]
-    )
-
-    df = pd.concat([df] * 2 ** 3, ignore_index=True)
-    return df
-
-
-def write_to_memory2(batch):
-    sink = aio.InMemoryOutputStream()
-    write_file(batch, sink)
-    return sink.get_result()
-
-
-def write_to_memory(batch):
-    sink = io.BytesIO()
-    write_file(batch, sink)
-    return sink.getvalue()
-
-
 def write_file(batch, sink):
-    writer = ipc.ArrowFileWriter(sink, batch.schema)
-    writer.write_record_batch(batch)
+    writer = pa.FileWriter(sink, batch.schema)
+    writer.write_batch(batch)
     writer.close()
 
 
 def read_file(source):
-    reader = ipc.ArrowFileReader(source)
-    return [reader.get_record_batch(i)
+    reader = pa.FileReader(source)
+    return [reader.get_batch(i)
             for i in range(reader.num_record_batches)]
-
-# df = big_batch()
-# batch = A.RecordBatch.from_pandas(df)
-# mem = write_to_memory(batch)
-# batches = read_file(mem)
-# data = batches[0].to_pandas()
-# rdf = pd.DataFrame(data)
-
-# [x.to_pandas() for x in batches]
diff --git a/python/setup.py b/python/setup.py
index de59a92905895..9c63e93df3352 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -94,7 +94,6 @@ def initialize_options(self):
         'config',
         'error',
         'io',
-        'ipc',
         '_parquet',
         'scalar',
         'schema',

From 2821030124eb3e884b0e48f09c38b54f00430b13 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Jan 2017 09:11:26 -0500
Subject: [PATCH 0291/1644] ARROW-508: [C++] Add basic threadsafety to normal
 files and memory maps

This patch is stacked on ARROW-494, so will need to be rebased.

* Since the naive `ReadAt` implementation involves a Seek and a Read, this locks until the read is completed.
* Normal file reads block until completion
* File writes block until completion

This covers the threadsafety requirements for parquet-cpp at least. For on-disk files, the following methods are now threadsafe:

* `ArrowInputFile::Read` and `ArrowInputFile::ReadAt`
* `ArrowOutputStream::Write`

parquet-cpp calls `Seek` in a couple places:

https://github.com/apache/parquet-cpp/blob/master/src/parquet/file/reader-internal.cc#L257

Strictly speaking, if two threads are trying to read the same file from the same input source, this could have a race condition in esoteric circumstances. I'm going to report a bug to change these to `ReadAt` which can be more easily made threadsafe

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #300 from wesm/ARROW-508 and squashes the following commits:

e57156c [Wes McKinney] Make base ReadableFileInterface::ReadAt and some file functions threadsafe
---
 cpp/src/arrow/io/file.cc         | 10 ++++-
 cpp/src/arrow/io/file.h          |  9 ++++-
 cpp/src/arrow/io/interfaces.cc   |  3 ++
 cpp/src/arrow/io/interfaces.h    | 12 +++++-
 cpp/src/arrow/io/io-file-test.cc | 69 ++++++++++++++++++++++++++++++++
 5 files changed, 98 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 3bf8dfa08f2ff..ff58e539b9353 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -76,6 +76,7 @@
 #include <cstring>
 #include <iostream>
 #include <limits>
+#include <mutex>
 #include <sstream>
 #include <vector>
 
@@ -350,6 +351,7 @@ class OSFile {
   }
 
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+    std::lock_guard<std::mutex> guard(lock_);
     return FileRead(fd_, out, nbytes, bytes_read);
   }
 
@@ -361,6 +363,7 @@ class OSFile {
   Status Tell(int64_t* pos) const { return FileTell(fd_, pos); }
 
   Status Write(const uint8_t* data, int64_t length) {
+    std::lock_guard<std::mutex> guard(lock_);
     if (length < 0) { return Status::IOError("Length must be non-negative"); }
     return FileWrite(fd_, data, length);
   }
@@ -377,6 +380,8 @@ class OSFile {
  protected:
   std::string path_;
 
+  std::mutex lock_;
+
   // File descriptor
   int fd_;
 
@@ -649,6 +654,8 @@ bool MemoryMappedFile::supports_zero_copy() const {
 }
 
 Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+  std::lock_guard<std::mutex> guard(lock_);
+
   if (!memory_map_->opened() || !memory_map_->writable()) {
     return Status::IOError("Unable to write");
   }
@@ -658,13 +665,14 @@ Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t
 }
 
 Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
+  std::lock_guard<std::mutex> guard(lock_);
+
   if (!memory_map_->opened() || !memory_map_->writable()) {
     return Status::IOError("Unable to write");
   }
   if (nbytes + memory_map_->position() > memory_map_->size()) {
     return Status::Invalid("Cannot write past end of memory map");
   }
-
   return WriteInternal(data, nbytes);
 }
 
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 930346b0518b3..fe55e968e05d7 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -50,6 +50,8 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
   // OutputStream interface
   Status Close() override;
   Status Tell(int64_t* position) override;
+
+  // Write bytes to the stream. Thread-safe
   Status Write(const uint8_t* data, int64_t nbytes) override;
 
   int file_descriptor() const;
@@ -76,6 +78,7 @@ class ARROW_EXPORT ReadableFile : public ReadableFileInterface {
   Status Close() override;
   Status Tell(int64_t* position) override;
 
+  // Read bytes from the file. Thread-safe
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
@@ -112,16 +115,18 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
 
   Status Seek(int64_t position) override;
 
-  // Required by ReadableFileInterface, copies memory into out
+  // Required by ReadableFileInterface, copies memory into out. Not thread-safe
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
 
-  // Zero copy read
+  // Zero copy read. Not thread-safe
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   bool supports_zero_copy() const override;
 
+  /// Write data at the current position in the file. Thread-safe
   Status Write(const uint8_t* data, int64_t nbytes) override;
 
+  /// Write data at a particular position in the file. Thread-safe
   Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
 
   // @return: the size in bytes of the memory source
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 8040f93836cdc..7e78caa04e711 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -19,6 +19,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <mutex>
 
 #include "arrow/buffer.h"
 #include "arrow/status.h"
@@ -34,12 +35,14 @@ ReadableFileInterface::ReadableFileInterface() {
 
 Status ReadableFileInterface::ReadAt(
     int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
 }
 
 Status ReadableFileInterface::ReadAt(
     int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, out);
 }
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index fdb3788188915..78680903d230a 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <mutex>
 #include <string>
 
 #include "arrow/util/macros.h"
@@ -99,14 +100,21 @@ class ARROW_EXPORT ReadableFileInterface : public InputStream, public Seekable {
 
   virtual bool supports_zero_copy() const = 0;
 
-  // Read at position, provide default implementations using Read(...), but can
-  // be overridden
+  /// Read at position, provide default implementations using Read(...), but can
+  /// be overridden
+  ///
+  /// Default implementation is thread-safe
   virtual Status ReadAt(
       int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out);
 
+  /// Default implementation is thread-safe
   virtual Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out);
 
+  std::mutex& lock() { return lock_; }
+
  protected:
+  std::mutex lock_;
+
   ReadableFileInterface();
 };
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 999b296465544..86a3287b84fbc 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <atomic>
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
@@ -25,6 +26,7 @@
 #include <memory>
 #include <sstream>
 #include <string>
+#include <thread>
 
 #include "gtest/gtest.h"
 
@@ -325,6 +327,40 @@ TEST_F(TestReadableFile, CustomMemoryPool) {
   ASSERT_EQ(2, pool.num_allocations());
 }
 
+TEST_F(TestReadableFile, ThreadSafety) {
+  std::string data = "foobar";
+  {
+    std::ofstream stream;
+    stream.open(path_.c_str());
+    stream << data;
+  }
+
+  MyMemoryPool pool;
+  ASSERT_OK(ReadableFile::Open(path_, &pool, &file_));
+
+  std::atomic<int> correct_count(0);
+  const int niter = 10000;
+
+  auto ReadData = [&correct_count, &data, niter, this] () {
+    std::shared_ptr<Buffer> buffer;
+
+    for (int i = 0; i < niter; ++i) {
+      ASSERT_OK(file_->ReadAt(0, 3, &buffer));
+      if (0 == memcmp(data.c_str(), buffer->data(), 3)) {
+        correct_count += 1;
+      }
+    }
+  };
+
+  std::thread thread1(ReadData);
+  std::thread thread2(ReadData);
+
+  thread1.join();
+  thread2.join();
+
+  ASSERT_EQ(niter * 2, correct_count);
+}
+
 // ----------------------------------------------------------------------
 // Memory map tests
 
@@ -455,5 +491,38 @@ TEST_F(TestMemoryMappedFile, CastableToFileInterface) {
   std::shared_ptr<FileInterface> file = memory_mapped_file;
 }
 
+TEST_F(TestMemoryMappedFile, ThreadSafety) {
+  std::string data = "foobar";
+  std::string path = "ipc-multithreading-test";
+  CreateFile(path, static_cast<int>(data.size()));
+
+  std::shared_ptr<MemoryMappedFile> file;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &file));
+  ASSERT_OK(file->Write(reinterpret_cast<const uint8_t*>(data.c_str()),
+          static_cast<int64_t>(data.size())));
+
+  std::atomic<int> correct_count(0);
+  const int niter = 10000;
+
+  auto ReadData = [&correct_count, &data, niter, &file] () {
+    std::shared_ptr<Buffer> buffer;
+
+    for (int i = 0; i < niter; ++i) {
+      ASSERT_OK(file->ReadAt(0, 3, &buffer));
+      if (0 == memcmp(data.c_str(), buffer->data(), 3)) {
+        correct_count += 1;
+      }
+    }
+  };
+
+  std::thread thread1(ReadData);
+  std::thread thread2(ReadData);
+
+  thread1.join();
+  thread2.join();
+
+  ASSERT_EQ(niter * 2, correct_count);
+}
+
 }  // namespace io
 }  // namespace arrow

From 085c8754b0ab2da7fcd245fc88bc4de9a6806a4c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Jan 2017 09:13:39 -0500
Subject: [PATCH 0292/1644] ARROW-81: [Format] Augment dictionary encoding
 metadata to accommodate additional use cases

cc @julienledem @nongli @jacques-n. I am hoping to close the loop on our discussion in https://issues.apache.org/jira/browse/ARROW-81. In my applications, I need the flexibility to transmit:

* Dictionaries encoded in signed integers smaller than int32. For example, with 10 dictionary values, we may send int8 indices
* Indicator that the dictionary is ordered

These features are needed for Python and R support, and in general for statistical computing applications.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #297 from wesm/ARROW-81 and squashes the following commits:

c960bac [Wes McKinney] Augment dictionary encoding metadata to accommodate additional use cases
---
 format/Message.fbs | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index b2c64649f2687..028c56ad51618 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -150,6 +150,26 @@ table KeyValue {
   value: [ubyte];
 }
 
+/// ----------------------------------------------------------------------
+/// Dictionary encoding metadata
+
+table DictionaryEncoding {
+  /// The known dictionary id in the application where this data is used. In
+  /// the file or streaming formats, the dictionary ids are found in the
+  /// DictionaryBatch messages
+  id: long;
+
+  /// The dictionary indices are constrained to be positive integers. If this
+  /// field is null, the indices must be signed int32
+  indexType: Int;
+
+  /// By default, dictionaries are not ordered, or the order does not have
+  /// semantic meaning. In some statistical, applications, dictionary-encoding
+  /// is used to represent ordered categorical data, and we provide a way to
+  /// preserve that metadata here
+  isOrdered: bool;
+}
+
 /// ----------------------------------------------------------------------
 /// A field represents a named column in a record / row batch or child of a
 /// nested type.
@@ -163,9 +183,10 @@ table Field {
   name: string;
   nullable: bool;
   type: Type;
-  // present only if the field is dictionary encoded
-  // will point to a dictionary provided by a DictionaryBatch message
-  dictionary: long;
+
+  // Present only if the field is dictionary encoded
+  dictionary: DictionaryEncoding;
+
   // children apply only to Nested data types like Struct, List and Union
   children: [Field];
   /// layout of buffers produced for this type (as derived from the Type)

From c90ca60c1859b2b70c4f2dd3fb8c41b0f75f02d0 Mon Sep 17 00:00:00 2001
From: ahnj <ahnj@yahoo.com>
Date: Mon, 23 Jan 2017 23:44:22 -0500
Subject: [PATCH 0293/1644] ARROW-378: Python: Respect timezone on conversion
 of Pandas datetime columns

arrow is now pandas datetime timezone aware

Author: ahnj <ahnj@yahoo.com>

Closes #287 from ahnj/timestamp-aware and squashes the following commits:

0221ed0 [ahnj] ARROW-378: Python: Respect timezone on conversion of Pandas datetime columns
---
 python/pyarrow/array.pyx                    |  6 ++++-
 python/pyarrow/tests/test_convert_pandas.py | 29 +++++++++++++++++++--
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 92206f2451ffb..c3a5a045b7dd5 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -71,9 +71,13 @@ cdef class Array:
 
         timestamps_to_ms : bool, optional
             Convert datetime columns to ms resolution. This is needed for
-            compability with other functionality like Parquet I/O which
+            compatibility with other functionality like Parquet I/O which
             only supports milliseconds.
 
+        Notes
+        -----
+        Localized timestamps will currently be returned as UTC (pandas's native representation).
+        Timezone-naive data will be implicitly interpreted as UTC.
 
         Examples
         --------
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 30705c4ca2a20..674a4361d3395 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -63,7 +63,7 @@ def tearDown(self):
 
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
                                 timestamps_to_ms=False, expected_schema=None,
-                                schema=None):
+                                check_dtype=True, schema=None):
         table = A.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms,
                                     schema=schema)
         result = table.to_pandas(nthreads=nthreads)
@@ -71,7 +71,7 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
             assert table.schema.equals(expected_schema)
         if expected is None:
             expected = df
-        tm.assert_frame_equal(result, expected)
+        tm.assert_frame_equal(result, expected, check_dtype=check_dtype)
 
     def _check_array_roundtrip(self, values, expected=None,
                                timestamps_to_ms=False, field=None):
@@ -284,6 +284,31 @@ def test_timestamps_notimezone_nulls(self):
         self._check_pandas_roundtrip(df, timestamps_to_ms=False,
                                      expected_schema=schema)
 
+    def test_timestamps_with_timezone(self):
+        df = pd.DataFrame({
+            'datetime64': np.array([
+                '2007-07-13T01:23:34.123',
+                '2006-01-13T12:34:56.432',
+                '2010-08-13T05:46:57.437'],
+                dtype='datetime64[ms]')
+            })
+        df_est = df['datetime64'].dt.tz_localize('US/Eastern').to_frame()
+        df_utc = df_est['datetime64'].dt.tz_convert('UTC').to_frame()
+        self._check_pandas_roundtrip(df_est, expected=df_utc, timestamps_to_ms=True, check_dtype=False)
+
+        # drop-in a null and ns instead of ms
+        df = pd.DataFrame({
+            'datetime64': np.array([
+                '2007-07-13T01:23:34.123456789',
+                None,
+                '2006-01-13T12:34:56.432539784',
+                '2010-08-13T05:46:57.437699912'],
+                dtype='datetime64[ns]')
+            })
+        df_est = df['datetime64'].dt.tz_localize('US/Eastern').to_frame()
+        df_utc = df_est['datetime64'].dt.tz_convert('UTC').to_frame()
+        self._check_pandas_roundtrip(df_est, expected=df_utc, timestamps_to_ms=False, check_dtype=False)
+
     def test_date(self):
         df = pd.DataFrame({
             'date': [datetime.date(2000, 1, 1),

From 61a54f8a619efc4fd256c446be29905d6484c5e9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 24 Jan 2017 08:30:37 -0500
Subject: [PATCH 0294/1644] ARROW-509: [Python] Add support for multithreaded
 Parquet reads

I'm getting very nice speedups on a Parquet file storing a ~4.5 GB dataset:

```
In [1]: import pyarrow.parquet as pq

In [2]: %time table = pq.read_table('/home/wesm/data/airlines_parquet/4345e5eef217aa1b-c8f16177f35fd983_1150363067_data.0.parq')
CPU times: user 8.21 s, sys: 468 ms, total: 8.68 s
Wall time: 8.68 s

In [3]: %time table = pq.read_table('/home/wesm/data/airlines_parquet/4345e5eef217aa1b-c8f16177f35fd983_1150363067_data.0.parq', nthreads=4)
CPU times: user 8.84 s, sys: 4.28 s, total: 13.1 s
Wall time: 3.91 s

In [4]: %time table = pq.read_table('/home/wesm/data/airlines_parquet/4345e5eef217aa1b-c8f16177f35fd983_1150363067_data.0.parq', nthreads=8)
CPU times: user 13.3 s, sys: 1.15 s, total: 14.4 s
Wall time: 2.86 s
```

This requires a bugfix in parquet-cpp that will come soon in a patch for PARQUET-836

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #301 from wesm/ARROW-509 and squashes the following commits:

9816689 [Wes McKinney] Update docs slightly, flake8 warning
239b086 [Wes McKinney] Add support for nthreads option in parquet::arrow, unit tests
---
 python/pyarrow/_parquet.pxd          |  4 ++++
 python/pyarrow/_parquet.pyx          | 21 ++++++++++++----
 python/pyarrow/parquet.py            | 36 ++++++++++++++++++----------
 python/pyarrow/tests/test_parquet.py | 18 ++++++++++++++
 4 files changed, 62 insertions(+), 17 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index cf1da1c3a9e52..fabee5d5761d7 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -213,8 +213,12 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
         FileReader(MemoryPool* pool, unique_ptr[ParquetFileReader] reader)
         CStatus ReadFlatColumn(int i, shared_ptr[CArray]* out);
         CStatus ReadFlatTable(shared_ptr[CTable]* out);
+        CStatus ReadFlatTable(const vector[int]& column_indices,
+                              shared_ptr[CTable]* out);
         const ParquetFileReader* parquet_reader();
 
+        void set_num_threads(int num_threads)
+
 
 cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
     CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema,
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index b11cee3a201fb..3f847e9808230 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -382,14 +382,27 @@ cdef class ParquetReader:
         result.init(metadata)
         return result
 
-    def read_all(self):
+    def read(self, column_indices=None, nthreads=1):
         cdef:
             Table table = Table()
             shared_ptr[CTable] ctable
+            vector[int] c_column_indices
 
-        with nogil:
-            check_status(self.reader.get()
-                         .ReadFlatTable(&ctable))
+        self.reader.get().set_num_threads(nthreads)
+
+        if column_indices is not None:
+            # Read only desired column indices
+            for index in column_indices:
+                c_column_indices.push_back(index)
+
+            with nogil:
+                check_status(self.reader.get()
+                             .ReadFlatTable(c_column_indices, &ctable))
+        else:
+            # Read all columns
+            with nogil:
+                check_status(self.reader.get()
+                             .ReadFlatTable(&ctable))
 
         table.init(ctable)
         return table
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index cbe1c6e5d79d9..6654b770ba33e 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -18,7 +18,7 @@
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
                               RowGroupMetaData, Schema, ParquetWriter)
 import pyarrow._parquet as _parquet  # noqa
-from pyarrow.table import Table, concat_tables
+from pyarrow.table import concat_tables
 
 
 class ParquetFile(object):
@@ -45,7 +45,7 @@ def metadata(self):
     def schema(self):
         return self.metadata.schema
 
-    def read(self, nrows=None, columns=None):
+    def read(self, nrows=None, columns=None, nthreads=1):
         """
         Read a Table from Parquet format
 
@@ -53,6 +53,9 @@ def read(self, nrows=None, columns=None):
         ----------
         columns: list
             If not None, only these columns will be read from the file.
+        nthreads : int, default 1
+            Number of columns to read in parallel. Requires that the underlying
+            file source is threadsafe
 
         Returns
         -------
@@ -63,16 +66,16 @@ def read(self, nrows=None, columns=None):
             raise NotImplementedError("nrows argument")
 
         if columns is None:
-            return self.reader.read_all()
+            column_indices = None
         else:
-            column_idxs = [self.reader.column_name_idx(column)
-                           for column in columns]
-            arrays = [self.reader.read_column(column_idx)
-                      for column_idx in column_idxs]
-            return Table.from_arrays(arrays, names=columns)
+            column_indices = [self.reader.column_name_idx(column)
+                              for column in columns]
 
+        return self.reader.read(column_indices=column_indices,
+                                nthreads=nthreads)
 
-def read_table(source, columns=None, metadata=None):
+
+def read_table(source, columns=None, nthreads=1, metadata=None):
     """
     Read a Table from Parquet format
 
@@ -83,6 +86,9 @@ def read_table(source, columns=None, metadata=None):
         pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
     columns: list
         If not None, only these columns will be read from the file.
+    nthreads : int, default 1
+        Number of columns to read in parallel. Requires that the underlying
+        file source is threadsafe
     metadata : FileMetaData
         If separately computed
 
@@ -91,11 +97,12 @@ def read_table(source, columns=None, metadata=None):
     pyarrow.Table
         Content of the file as a table (of columns)
     """
-    return ParquetFile(source, metadata=metadata).read(columns=columns)
+    pf = ParquetFile(source, metadata=metadata)
+    return pf.read(columns=columns, nthreads=nthreads)
 
 
-def read_multiple_files(paths, columns=None, filesystem=None, metadata=None,
-                        schema=None):
+def read_multiple_files(paths, columns=None, filesystem=None, nthreads=1,
+                        metadata=None, schema=None):
     """
     Read multiple Parquet files as a single pyarrow.Table
 
@@ -108,6 +115,9 @@ def read_multiple_files(paths, columns=None, filesystem=None, metadata=None,
     filesystem : Filesystem, default None
         If nothing passed, paths assumed to be found in the local on-disk
         filesystem
+    nthreads : int, default 1
+        Number of columns to read in parallel. Requires that the underlying
+        file source is threadsafe
     metadata : pyarrow.parquet.FileMetaData
         Use metadata obtained elsewhere to validate file schemas
     schema : pyarrow.parquet.Schema
@@ -147,7 +157,7 @@ def open_file(path, meta=None):
     tables = []
     for path, path_metadata in zip(paths, all_file_metadata):
         reader = open_file(path, meta=path_metadata)
-        table = reader.read(columns=columns)
+        table = reader.read(columns=columns, nthreads=nthreads)
         tables.append(table)
 
     all_data = concat_tables(tables)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index a94fe456d3b2b..d85f0e513702f 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -320,6 +320,24 @@ def test_compare_schemas():
     assert not fileh.schema[0].equals(fileh.schema[1])
 
 
+@parquet
+def test_multithreaded_read():
+    df = alltypes_sample(size=10000)
+
+    table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+
+    buf = io.BytesIO()
+    pq.write_table(table, buf, compression='SNAPPY', version='2.0')
+
+    buf.seek(0)
+    table1 = pq.read_table(buf, nthreads=4)
+
+    buf.seek(0)
+    table2 = pq.read_table(buf, nthreads=1)
+
+    assert table1.equals(table2)
+
+
 @parquet
 def test_pass_separate_metadata():
     # ARROW-471

From a68af9d168e381d1730ae0cb4dc653bef42562d3 Mon Sep 17 00:00:00 2001
From: Nong Li <nongli@gmail.com>
Date: Wed, 25 Jan 2017 14:49:00 -0500
Subject: [PATCH 0295/1644] ARROW-498 [C++] Add command line utilities that
 convert between stream and file.

These are in the style of unix utilities using stdin/stdout for argument passing.
This makes it easy to chain them together and I think are using for getting started
or testing. As an example, this command line tests a round trip:
 $ build/debug/file-to-stream /tmp/arrow-file | build/debug/stream-to-file > /tmp/copy
 $ diff /tmp/arrow-file /tmp/copy

If we had the same in java, this would make it pretty convenient for integration
testing.

Author: Nong Li <nongli@gmail.com>

Closes #302 from nongli/utils and squashes the following commits:

b970c75 [Nong Li] fix long -> int64_t
a01ef4d [Nong Li] Fix style issues.
da3d98d [Nong Li] ARROW-498 [C++] Add commandline utilities that convert between stream and file.
---
 cpp/CMakeLists.txt                   |  4 ++
 cpp/src/arrow/util/CMakeLists.txt    | 26 ++++++++
 cpp/src/arrow/util/file-to-stream.cc | 60 ++++++++++++++++++
 cpp/src/arrow/util/io-util.h         | 93 ++++++++++++++++++++++++++++
 cpp/src/arrow/util/stream-to-file.cc | 58 +++++++++++++++++
 5 files changed, 241 insertions(+)
 create mode 100644 cpp/src/arrow/util/file-to-stream.cc
 create mode 100644 cpp/src/arrow/util/io-util.h
 create mode 100644 cpp/src/arrow/util/stream-to-file.cc

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 9039ffb571b9e..a0f89f314f683 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -90,6 +90,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(ARROW_ALTIVEC
     "Build Arrow with Altivec"
     ON)
+
+  option(ARROW_BUILD_UTILITIES
+    "Build Arrow commandline utilities"
+    ON)
 endif()
 
 if(NOT ARROW_BUILD_TESTS)
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 8d9afccf867df..0830ee2ed2928 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -68,4 +68,30 @@ if (ARROW_BUILD_BENCHMARKS)
   endif()
 endif()
 
+if (ARROW_BUILD_UTILITIES)
+  if (APPLE)
+    set(UTIL_LINK_LIBS
+      arrow_ipc_static
+      arrow_io_static
+      arrow_static
+      boost_filesystem_static
+      boost_system_static
+      dl)
+  else()
+    set(UTIL_LINK_LIBS
+      arrow_ipc_static
+      arrow_io_static
+      arrow_static
+      pthread
+      boost_filesystem_static
+      boost_system_static
+      dl)
+  endif()
+
+  add_executable(file-to-stream file-to-stream.cc)
+  target_link_libraries(file-to-stream ${UTIL_LINK_LIBS})
+  add_executable(stream-to-file stream-to-file.cc)
+  target_link_libraries(stream-to-file ${UTIL_LINK_LIBS})
+endif()
+
 ADD_ARROW_TEST(bit-util-test)
diff --git a/cpp/src/arrow/util/file-to-stream.cc b/cpp/src/arrow/util/file-to-stream.cc
new file mode 100644
index 0000000000000..42c1d55afd322
--- /dev/null
+++ b/cpp/src/arrow/util/file-to-stream.cc
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+#include "arrow/io/file.h"
+#include "arrow/ipc/file.h"
+#include "arrow/ipc/stream.h"
+#include "arrow/status.h"
+
+#include "arrow/util/io-util.h"
+
+namespace arrow {
+
+// Reads a file on the file system and prints to stdout the stream version of it.
+Status ConvertToStream(const char* path) {
+  std::shared_ptr<io::ReadableFile> in_file;
+  std::shared_ptr<ipc::FileReader> reader;
+
+  RETURN_NOT_OK(io::ReadableFile::Open(path, &in_file));
+  RETURN_NOT_OK(ipc::FileReader::Open(in_file, &reader));
+
+  io::StdoutStream sink;
+  std::shared_ptr<ipc::StreamWriter> writer;
+  RETURN_NOT_OK(ipc::StreamWriter::Open(&sink, reader->schema(), &writer));
+  for (int i = 0; i < reader->num_record_batches(); ++i) {
+    std::shared_ptr<RecordBatch> chunk;
+    RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
+    RETURN_NOT_OK(writer->WriteRecordBatch(*chunk));
+  }
+  return writer->Close();
+}
+
+} // namespace arrow
+
+int main(int argc, char** argv) {
+  if (argc != 2) {
+    std::cerr << "Usage: file-to-stream <input arrow file>" << std::endl;
+    return 1;
+  }
+  arrow::Status status = arrow::ConvertToStream(argv[1]);
+  if (!status.ok()) {
+    std::cerr << "Could not convert to stream: " << status.ToString() << std::endl;
+    return 1;
+  }
+  return 0;
+}
diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
new file mode 100644
index 0000000000000..3e5054d8fa83d
--- /dev/null
+++ b/cpp/src/arrow/util/io-util.h
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_IO_UTIL_H
+#define ARROW_UTIL_IO_UTIL_H
+
+#include <iostream>
+#include "arrow/buffer.h"
+
+namespace arrow {
+namespace io {
+
+// Output stream that just writes to stdout.
+class StdoutStream : public OutputStream {
+ public:
+  StdoutStream() : pos_(0) {
+    set_mode(FileMode::WRITE);
+  }
+  virtual ~StdoutStream() {}
+
+  Status Close() { return Status::OK(); }
+  Status Tell(int64_t* position)  {
+    *position = pos_;
+    return Status::OK();
+  }
+
+  Status Write(const uint8_t* data, int64_t nbytes) {
+    pos_ += nbytes;
+    std::cout.write(reinterpret_cast<const char*>(data), nbytes);
+    return Status::OK();
+  }
+ private:
+  int64_t pos_;
+};
+
+// Input stream that just reads from stdin.
+class StdinStream : public InputStream {
+ public:
+  StdinStream() : pos_(0) {
+    set_mode(FileMode::READ);
+  }
+  virtual ~StdinStream() {}
+
+  Status Close() { return Status::OK(); }
+  Status Tell(int64_t* position)  {
+    *position = pos_;
+    return Status::OK();
+  }
+
+  virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+    std::cin.read(reinterpret_cast<char*>(out), nbytes);
+    if (std::cin) {
+      *bytes_read = nbytes;
+      pos_ += nbytes;
+    } else {
+      *bytes_read = 0;
+    }
+    return Status::OK();
+  }
+
+  virtual Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+    auto buffer = std::make_shared<PoolBuffer>(nullptr);
+    RETURN_NOT_OK(buffer->Resize(nbytes));
+    int64_t bytes_read;
+    RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
+    RETURN_NOT_OK(buffer->Resize(bytes_read, false));
+    *out = buffer;
+    return Status::OK();
+  }
+
+ private:
+  int64_t pos_;
+};
+
+} // namespace io
+} // namespace arrow
+
+#endif // ARROW_UTIL_IO_UTIL_H
+
diff --git a/cpp/src/arrow/util/stream-to-file.cc b/cpp/src/arrow/util/stream-to-file.cc
new file mode 100644
index 0000000000000..7a8ec0bfd952b
--- /dev/null
+++ b/cpp/src/arrow/util/stream-to-file.cc
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+#include "arrow/io/file.h"
+#include "arrow/ipc/file.h"
+#include "arrow/ipc/stream.h"
+#include "arrow/status.h"
+
+#include "arrow/util/io-util.h"
+
+namespace arrow {
+
+// Converts a stream from stdin to a file written to standard out.
+// A typical usage would be:
+// $ <program that produces streaming output> | stream-to-file > file.arrow
+Status ConvertToFile() {
+  std::shared_ptr<io::InputStream> input(new io::StdinStream);
+  std::shared_ptr<ipc::StreamReader> reader;
+  RETURN_NOT_OK(ipc::StreamReader::Open(input, &reader));
+
+  io::StdoutStream sink;
+  std::shared_ptr<ipc::FileWriter> writer;
+  RETURN_NOT_OK(ipc::FileWriter::Open(&sink, reader->schema(), &writer));
+
+  std::shared_ptr<RecordBatch> batch;
+  while (true) {
+    RETURN_NOT_OK(reader->GetNextRecordBatch(&batch));
+    if (batch == nullptr) break;
+    RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
+  }
+  return writer->Close();
+}
+
+} // namespace arrow
+
+int main(int argc, char** argv) {
+  arrow::Status status = arrow::ConvertToFile();
+  if (!status.ok()) {
+    std::cerr << "Could not convert to file: " << status.ToString() << std::endl;
+    return 1;
+  }
+  return 0;
+}

From a90b5f3634bdbd6af01967f288457d07d5f2e2eb Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 26 Jan 2017 13:20:17 -0500
Subject: [PATCH 0296/1644] ARROW-512: C++: Add method to check for primitive
 types

Also includes some documentation updates.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #304 from xhochy/ARROW-512 and squashes the following commits:

cfe9205 [Uwe L. Korn] ARROW-512: C++: Add method to check for primitive types
---
 cpp/apidoc/Doxyfile         |   2 +-
 cpp/apidoc/index.md         |   4 +-
 cpp/src/arrow/buffer.h      |  54 ++++++-------
 cpp/src/arrow/builder.h     | 150 +++++++++++++++++++-----------------
 cpp/src/arrow/memory_pool.h |  21 +++++
 cpp/src/arrow/type.h        |  37 +++++++++
 6 files changed, 167 insertions(+), 101 deletions(-)

diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile
index 7dc55fef834fc..51f5543b2de1b 100644
--- a/cpp/apidoc/Doxyfile
+++ b/cpp/apidoc/Doxyfile
@@ -204,7 +204,7 @@ SHORT_NAMES            = NO
 # description.)
 # The default value is: NO.
 
-JAVADOC_AUTOBRIEF      = NO
+JAVADOC_AUTOBRIEF      = YES
 
 # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
 # line (until the first dot) of a Qt-style comment as the brief description. If
diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
index 080f848bb184f..fdac4969beb9b 100644
--- a/cpp/apidoc/index.md
+++ b/cpp/apidoc/index.md
@@ -38,8 +38,8 @@ this bitmap.
 
 As Arrow objects are immutable, there are classes provided that should help you
 build these objects. To build an array of `int64_t` elements, we can use the
-`Int64Builder`. In the following example, we build an array of the range 1 to 8
-where the element that should hold the number 4 is nulled.
+`arrow::Int64Builder`. In the following example, we build an array of the range
+1 to 8 where the element that should hold the number 4 is nulled.
 
     Int64Builder builder(arrow::default_memory_pool(), arrow::int64());
     builder.Append(1);
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index ac78808eaf205..d43ab0375b725 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -35,33 +35,35 @@ class Status;
 // ----------------------------------------------------------------------
 // Buffer classes
 
-// Immutable API for a chunk of bytes which may or may not be owned by the
-// class instance.  Buffers have two related notions of length: size and
-// capacity.  Size is the number of bytes that might have valid data.
-// Capacity is the number of bytes that where allocated for the buffer in
-// total.
-// The following invariant is always true: Size < Capacity
+/// Immutable API for a chunk of bytes which may or may not be owned by the
+/// class instance.
+///
+/// Buffers have two related notions of length: size and capacity. Size is
+/// the number of bytes that might have valid data. Capacity is the number
+/// of bytes that where allocated for the buffer in total.
+///
+/// The following invariant is always true: Size < Capacity
 class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
  public:
   Buffer(const uint8_t* data, int64_t size)
       : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
   virtual ~Buffer();
 
-  // An offset into data that is owned by another buffer, but we want to be
-  // able to retain a valid pointer to it even after other shared_ptr's to the
-  // parent buffer have been destroyed
-  //
-  // This method makes no assertions about alignment or padding of the buffer but
-  // in general we expected buffers to be aligned and padded to 64 bytes.  In the future
-  // we might add utility methods to help determine if a buffer satisfies this contract.
+  /// An offset into data that is owned by another buffer, but we want to be
+  /// able to retain a valid pointer to it even after other shared_ptr's to the
+  /// parent buffer have been destroyed
+  ///
+  /// This method makes no assertions about alignment or padding of the buffer but
+  /// in general we expected buffers to be aligned and padded to 64 bytes.  In the future
+  /// we might add utility methods to help determine if a buffer satisfies this contract.
   Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
 
   std::shared_ptr<Buffer> get_shared_ptr() { return shared_from_this(); }
 
   bool is_mutable() const { return is_mutable_; }
 
-  // Return true if both buffers are the same size and contain the same bytes
-  // up to the number of compared bytes
+  /// Return true if both buffers are the same size and contain the same bytes
+  /// up to the number of compared bytes
   bool Equals(const Buffer& other, int64_t nbytes) const {
     return this == &other ||
            (size_ >= nbytes && other.size_ >= nbytes &&
@@ -74,11 +76,11 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
                (data_ == other.data_ || !memcmp(data_, other.data_, size_)));
   }
 
-  // Copy section of buffer into a new Buffer
+  /// Copy a section of the buffer into a new Buffer.
   Status Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
       std::shared_ptr<Buffer>* out) const;
 
-  // Default memory pool
+  /// Copy a section of the buffer using the default memory pool into a new Buffer.
   Status Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const;
 
   int64_t capacity() const { return capacity_; }
@@ -101,12 +103,12 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
   DISALLOW_COPY_AND_ASSIGN(Buffer);
 };
 
-// Construct a view on passed buffer at the indicated offset and length. This
-// function cannot fail and does not error checking (except in debug builds)
+/// Construct a view on passed buffer at the indicated offset and length. This
+/// function cannot fail and does not error checking (except in debug builds)
 ARROW_EXPORT std::shared_ptr<Buffer> SliceBuffer(
     const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
 
-// A Buffer whose contents can be mutated. May or may not own its data.
+/// A Buffer whose contents can be mutated. May or may not own its data.
 class ARROW_EXPORT MutableBuffer : public Buffer {
  public:
   MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
@@ -116,7 +118,7 @@ class ARROW_EXPORT MutableBuffer : public Buffer {
 
   uint8_t* mutable_data() { return mutable_data_; }
 
-  // Get a read-only view of this buffer
+  /// Get a read-only view of this buffer
   std::shared_ptr<Buffer> GetImmutableView();
 
  protected:
@@ -135,16 +137,16 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
   /// decrease.
   virtual Status Resize(int64_t new_size, bool shrink_to_fit = true) = 0;
 
-  // Ensure that buffer has enough memory allocated to fit the indicated
-  // capacity (and meets the 64 byte padding requirement in Layout.md).
-  // It does not change buffer's reported size.
+  /// Ensure that buffer has enough memory allocated to fit the indicated
+  /// capacity (and meets the 64 byte padding requirement in Layout.md).
+  /// It does not change buffer's reported size.
   virtual Status Reserve(int64_t new_capacity) = 0;
 
  protected:
   ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {}
 };
 
-// A Buffer whose lifetime is tied to a particular MemoryPool
+/// A Buffer whose lifetime is tied to a particular MemoryPool
 class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
  public:
   explicit PoolBuffer(MemoryPool* pool = nullptr);
@@ -162,7 +164,7 @@ class ARROW_EXPORT BufferBuilder {
   explicit BufferBuilder(MemoryPool* pool)
       : pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
 
-  // Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
+  /// Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
   Status Resize(int32_t elements) {
     if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
     RETURN_NOT_OK(buffer_->Resize(elements));
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 735bca1b1bcb3..747da7ca2d9dd 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -37,10 +37,11 @@ class Array;
 
 static constexpr int32_t kMinBuilderCapacity = 1 << 5;
 
-// Base class for all data array builders.
-// This class provides a facilities for incrementally building the null bitmap
-// (see Append methods) and as a side effect the current number of slots and
-// the null count.
+/// Base class for all data array builders.
+//
+/// This class provides a facilities for incrementally building the null bitmap
+/// (see Append methods) and as a side effect the current number of slots and
+/// the null count.
 class ARROW_EXPORT ArrayBuilder {
  public:
   explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type)
@@ -54,8 +55,8 @@ class ARROW_EXPORT ArrayBuilder {
 
   virtual ~ArrayBuilder() = default;
 
-  // For nested types. Since the objects are owned by this class instance, we
-  // skip shared pointers and just return a raw pointer
+  /// For nested types. Since the objects are owned by this class instance, we
+  /// skip shared pointers and just return a raw pointer
   ArrayBuilder* child(int i) { return children_[i].get(); }
 
   int num_children() const { return children_.size(); }
@@ -64,37 +65,37 @@ class ARROW_EXPORT ArrayBuilder {
   int32_t null_count() const { return null_count_; }
   int32_t capacity() const { return capacity_; }
 
-  // Append to null bitmap
+  /// Append to null bitmap
   Status AppendToBitmap(bool is_valid);
-  // Vector append. Treat each zero byte as a null.   If valid_bytes is null
-  // assume all of length bits are valid.
+  /// Vector append. Treat each zero byte as a null.   If valid_bytes is null
+  /// assume all of length bits are valid.
   Status AppendToBitmap(const uint8_t* valid_bytes, int32_t length);
-  // Set the next length bits to not null (i.e. valid).
+  /// Set the next length bits to not null (i.e. valid).
   Status SetNotNull(int32_t length);
 
-  // Allocates initial capacity requirements for the builder.  In most
-  // cases subclasses should override and call there parent classes
-  // method as well.
+  /// Allocates initial capacity requirements for the builder.  In most
+  /// cases subclasses should override and call there parent classes
+  /// method as well.
   virtual Status Init(int32_t capacity);
 
-  // Resizes the null_bitmap array.  In most
-  // cases subclasses should override and call there parent classes
-  // method as well.
+  /// Resizes the null_bitmap array.  In most
+  /// cases subclasses should override and call there parent classes
+  /// method as well.
   virtual Status Resize(int32_t new_bits);
 
-  // Ensures there is enough space for adding the number of elements by checking
-  // capacity and calling Resize if necessary.
+  /// Ensures there is enough space for adding the number of elements by checking
+  /// capacity and calling Resize if necessary.
   Status Reserve(int32_t elements);
 
-  // For cases where raw data was memcpy'd into the internal buffers, allows us
-  // to advance the length of the builder. It is your responsibility to use
-  // this function responsibly.
+  /// For cases where raw data was memcpy'd into the internal buffers, allows us
+  /// to advance the length of the builder. It is your responsibility to use
+  /// this function responsibly.
   Status Advance(int32_t elements);
 
   std::shared_ptr<PoolBuffer> null_bitmap() const { return null_bitmap_; }
 
-  // Creates new array object to hold the contents of the builder and transfers
-  // ownership of the data.  This resets all variables on the builder.
+  /// Creates new Array object to hold the contents of the builder and transfers
+  /// ownership of the data.  This resets all variables on the builder.
   virtual Status Finish(std::shared_ptr<Array>* out) = 0;
 
   std::shared_ptr<DataType> type() const { return type_; }
@@ -144,7 +145,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
 
   using ArrayBuilder::Advance;
 
-  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+  /// Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
   Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
     RETURN_NOT_OK(Reserve(length));
     UnsafeAppendToBitmap(valid_bytes, length);
@@ -159,18 +160,18 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
 
   std::shared_ptr<Buffer> data() const { return data_; }
 
-  // Vector append
-  //
-  // If passed, valid_bytes is of equal length to values, and any zero byte
-  // will be considered as a null for that slot
+  /// Vector append
+  ///
+  /// If passed, valid_bytes is of equal length to values, and any zero byte
+  /// will be considered as a null for that slot
   Status Append(
       const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
 
   Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int32_t capacity) override;
 
-  // Increase the capacity of the builder to accommodate at least the indicated
-  // number of elements
+  /// Increase the capacity of the builder to accommodate at least the indicated
+  /// number of elements
   Status Resize(int32_t capacity) override;
 
  protected:
@@ -178,6 +179,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   value_type* raw_data_;
 };
 
+/// Base class for all Builders that emit an Array of a scalar numerical type.
 template <typename T>
 class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
  public:
@@ -189,14 +191,18 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
   using PrimitiveBuilder<T>::Resize;
   using PrimitiveBuilder<T>::Reserve;
 
-  // Scalar append.
+  /// Append a single scalar and increase the size if necessary.
   Status Append(value_type val) {
     RETURN_NOT_OK(ArrayBuilder::Reserve(1));
     UnsafeAppend(val);
     return Status::OK();
   }
 
-  // Does not capacity-check; make sure to call Reserve beforehand
+  /// Append a single scalar under the assumption that the underlying Buffer is
+  /// large enough.
+  ///
+  /// This method does not capacity-check; make sure to call Reserve
+  /// beforehand.
   void UnsafeAppend(value_type val) {
     BitUtil::SetBit(null_bitmap_data_, length_);
     raw_data_[length_++] = val;
@@ -235,7 +241,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 
   using ArrayBuilder::Advance;
 
-  // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+  /// Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
   Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
     RETURN_NOT_OK(Reserve(length));
     UnsafeAppendToBitmap(valid_bytes, length);
@@ -250,7 +256,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 
   std::shared_ptr<Buffer> data() const { return data_; }
 
-  // Scalar append
+  /// Scalar append
   Status Append(bool val) {
     Reserve(1);
     BitUtil::SetBit(null_bitmap_data_, length_);
@@ -263,18 +269,18 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  // Vector append
-  //
-  // If passed, valid_bytes is of equal length to values, and any zero byte
-  // will be considered as a null for that slot
+  /// Vector append
+  ///
+  /// If passed, valid_bytes is of equal length to values, and any zero byte
+  /// will be considered as a null for that slot
   Status Append(
       const uint8_t* values, int32_t length, const uint8_t* valid_bytes = nullptr);
 
   Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int32_t capacity) override;
 
-  // Increase the capacity of the builder to accommodate at least the indicated
-  // number of elements
+  /// Increase the capacity of the builder to accommodate at least the indicated
+  /// number of elements
   Status Resize(int32_t capacity) override;
 
  protected:
@@ -285,26 +291,26 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 // ----------------------------------------------------------------------
 // List builder
 
-// Builder class for variable-length list array value types
-//
-// To use this class, you must append values to the child array builder and use
-// the Append function to delimit each distinct list value (once the values
-// have been appended to the child array) or use the bulk API to append
-// a sequence of offests and null values.
-//
-// A note on types.  Per arrow/type.h all types in the c++ implementation are
-// logical so even though this class always builds list array, this can
-// represent multiple different logical types.  If no logical type is provided
-// at construction time, the class defaults to List<T> where t is taken from the
-// value_builder/values that the object is constructed with.
+/// Builder class for variable-length list array value types
+///
+/// To use this class, you must append values to the child array builder and use
+/// the Append function to delimit each distinct list value (once the values
+/// have been appended to the child array) or use the bulk API to append
+/// a sequence of offests and null values.
+///
+/// A note on types.  Per arrow/type.h all types in the c++ implementation are
+/// logical so even though this class always builds list array, this can
+/// represent multiple different logical types.  If no logical type is provided
+/// at construction time, the class defaults to List<T> where t is taken from the
+/// value_builder/values that the object is constructed with.
 class ARROW_EXPORT ListBuilder : public ArrayBuilder {
  public:
-  // Use this constructor to incrementally build the value array along with offsets and
-  // null bitmap.
+  /// Use this constructor to incrementally build the value array along with offsets and
+  /// null bitmap.
   ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
       const TypePtr& type = nullptr);
 
-  // Use this constructor to build the list with a pre-existing values array
+  /// Use this constructor to build the list with a pre-existing values array
   ListBuilder(
       MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr);
 
@@ -314,10 +320,10 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   Status Resize(int32_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
 
-  // Vector append
-  //
-  // If passed, valid_bytes is of equal length to values, and any zero byte
-  // will be considered as a null for that slot
+  /// Vector append
+  ///
+  /// If passed, valid_bytes is of equal length to values, and any zero byte
+  /// will be considered as a null for that slot
   Status Append(
       const int32_t* offsets, int32_t length, const uint8_t* valid_bytes = nullptr) {
     RETURN_NOT_OK(Reserve(length));
@@ -326,10 +332,10 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  // Start a new variable-length list slot
-  //
-  // This function should be called before beginning to append elements to the
-  // value builder
+  /// Start a new variable-length list slot
+  ///
+  /// This function should be called before beginning to append elements to the
+  /// value builder
   Status Append(bool is_valid = true) {
     RETURN_NOT_OK(Reserve(1));
     UnsafeAppendToBitmap(is_valid);
@@ -396,9 +402,9 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
 
 // ---------------------------------------------------------------------------------
 // StructArray builder
-// Append, Resize and Reserve methods are acting on StructBuilder.
-// Please make sure all these methods of all child-builders' are consistently
-// called to maintain data-structure consistency.
+/// Append, Resize and Reserve methods are acting on StructBuilder.
+/// Please make sure all these methods of all child-builders' are consistently
+/// called to maintain data-structure consistency.
 class ARROW_EXPORT StructBuilder : public ArrayBuilder {
  public:
   StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
@@ -409,18 +415,18 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
 
   Status Finish(std::shared_ptr<Array>* out) override;
 
-  // Null bitmap is of equal length to every child field, and any zero byte
-  // will be considered as a null for that field, but users must using app-
-  // end methods or advance methods of the child builders' independently to
-  // insert data.
+  /// Null bitmap is of equal length to every child field, and any zero byte
+  /// will be considered as a null for that field, but users must using app-
+  /// end methods or advance methods of the child builders' independently to
+  /// insert data.
   Status Append(int32_t length, const uint8_t* valid_bytes) {
     RETURN_NOT_OK(Reserve(length));
     UnsafeAppendToBitmap(valid_bytes, length);
     return Status::OK();
   }
 
-  // Append an element to the Struct. All child-builders' Append method must
-  // be called independently to maintain data-structure consistency.
+  /// Append an element to the Struct. All child-builders' Append method must
+  /// be called independently to maintain data-structure consistency.
   Status Append(bool is_valid = true) {
     RETURN_NOT_OK(Reserve(1));
     UnsafeAppendToBitmap(is_valid);
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 13a3f129c1a9e..89477b6ddeab0 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -26,14 +26,35 @@ namespace arrow {
 
 class Status;
 
+/// Base class for memory allocation.
+///
+/// Besides tracking the number of allocated bytes, the allocator also should
+/// take care of the required 64-byte alignment.
 class ARROW_EXPORT MemoryPool {
  public:
   virtual ~MemoryPool();
 
+  /// Allocate a new memory region of at least size bytes.
+  ///
+  /// The allocated region shall be 64-byte aligned.
   virtual Status Allocate(int64_t size, uint8_t** out) = 0;
+
+  /// Resize an already allocated memory section.
+  ///
+  /// As by default most default allocators on a platform don't support aligned
+  /// reallocation, this function can involve a copy of the underlying data.
   virtual Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) = 0;
+
+  /// Free an allocated region.
+  ///
+  /// @param buffer Pointer to the start of the allocated memory region
+  /// @param size Allocated size located at buffer. An allocator implementation
+  ///   may use this for tracking the amount of allocated bytes as well as for
+  ///   faster deallocation if supported by its backend.
   virtual void Free(uint8_t* buffer, int64_t size) = 0;
 
+  /// The number of bytes that were allocated and not yet free'd through
+  /// this allocator.
   virtual int64_t bytes_allocated() const = 0;
 };
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index c2a762d279364..77a70d1d2ddd3 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -608,6 +608,43 @@ static inline bool is_floating(Type::type type_id) {
   return false;
 }
 
+static inline bool is_primitive(Type::type type_id) {
+  switch (type_id) {
+    case Type::NA:
+    case Type::BOOL:
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::UINT64:
+    case Type::INT64:
+    case Type::HALF_FLOAT:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+    case Type::DATE:
+    case Type::TIMESTAMP:
+    case Type::TIME:
+    case Type::INTERVAL:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
+static inline bool is_binary_like(Type::type type_id) {
+  switch (type_id) {
+    case Type::BINARY:
+    case Type::STRING:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPE_H

From aac2e70c1639cb45c5300b18dd94b000ba4b79db Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 26 Jan 2017 18:38:19 -0500
Subject: [PATCH 0297/1644] ARROW-513: [C++] Fixing Appveyor / MSVC build

Visual Studio 2015 (MSVC 19.0) seems to have a compiler bug with inheriting private ctors. It didn't like the private `using StreamWriter::StreamWriter` in the `FileWriter` implementation. This is not consistent with Microsoft's Modern C++ support matrix https://msdn.microsoft.com/en-us/library/hh567368.aspx, so perhaps they now support inheriting *public* constructors.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #305 from wesm/ARROW-513 and squashes the following commits:

9362674 [Wes McKinney] Visual Studio 2015 has limited support for inheriting constructors
93119e5 [Wes McKinney] Export some more classes to appease MSVC
9d4887c [Wes McKinney] Disable MSVC 4251 warning, add some ARROW_EXPORT visibility macros
---
 cpp/src/arrow/io/interfaces.h        | 16 ++++++++--------
 cpp/src/arrow/io/io-file-test.cc     | 16 ++++++----------
 cpp/src/arrow/ipc/file.cc            |  3 +++
 cpp/src/arrow/ipc/file.h             |  2 +-
 cpp/src/arrow/ipc/metadata.h         |  6 +++---
 cpp/src/arrow/util/CMakeLists.txt    | 28 ++++++++++++----------------
 cpp/src/arrow/util/file-to-stream.cc |  4 ++--
 cpp/src/arrow/util/io-util.h         | 22 +++++++++-------------
 cpp/src/arrow/util/stream-to-file.cc |  6 +++---
 cpp/src/arrow/util/visibility.h      |  5 +++++
 10 files changed, 52 insertions(+), 56 deletions(-)

diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 78680903d230a..e9f07f03a1419 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -42,7 +42,7 @@ struct ObjectType {
   enum type { FILE, DIRECTORY };
 };
 
-class FileSystemClient {
+class ARROW_EXPORT FileSystemClient {
  public:
   virtual ~FileSystemClient() {}
 };
@@ -64,7 +64,7 @@ class ARROW_EXPORT FileInterface {
   DISALLOW_COPY_AND_ASSIGN(FileInterface);
 };
 
-class Seekable {
+class ARROW_EXPORT Seekable {
  public:
   virtual Status Seek(int64_t position) = 0;
 };
@@ -76,7 +76,7 @@ class ARROW_EXPORT Writeable {
   Status Write(const std::string& data);
 };
 
-class Readable {
+class ARROW_EXPORT Readable {
  public:
   virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0;
 
@@ -84,12 +84,12 @@ class Readable {
   virtual Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
 };
 
-class OutputStream : virtual public FileInterface, public Writeable {
+class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writeable {
  protected:
   OutputStream() {}
 };
 
-class InputStream : virtual public FileInterface, public Readable {
+class ARROW_EXPORT InputStream : virtual public FileInterface, public Readable {
  protected:
   InputStream() {}
 };
@@ -118,7 +118,7 @@ class ARROW_EXPORT ReadableFileInterface : public InputStream, public Seekable {
   ReadableFileInterface();
 };
 
-class WriteableFileInterface : public OutputStream, public Seekable {
+class ARROW_EXPORT WriteableFileInterface : public OutputStream, public Seekable {
  public:
   virtual Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) = 0;
 
@@ -126,8 +126,8 @@ class WriteableFileInterface : public OutputStream, public Seekable {
   WriteableFileInterface() { set_mode(FileMode::READ); }
 };
 
-class ReadWriteFileInterface : public ReadableFileInterface,
-                               public WriteableFileInterface {
+class ARROW_EXPORT ReadWriteFileInterface : public ReadableFileInterface,
+                                            public WriteableFileInterface {
  protected:
   ReadWriteFileInterface() { ReadableFileInterface::set_mode(FileMode::READWRITE); }
 };
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 86a3287b84fbc..5810c820f6dd7 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -341,14 +341,12 @@ TEST_F(TestReadableFile, ThreadSafety) {
   std::atomic<int> correct_count(0);
   const int niter = 10000;
 
-  auto ReadData = [&correct_count, &data, niter, this] () {
+  auto ReadData = [&correct_count, &data, niter, this]() {
     std::shared_ptr<Buffer> buffer;
 
     for (int i = 0; i < niter; ++i) {
       ASSERT_OK(file_->ReadAt(0, 3, &buffer));
-      if (0 == memcmp(data.c_str(), buffer->data(), 3)) {
-        correct_count += 1;
-      }
+      if (0 == memcmp(data.c_str(), buffer->data(), 3)) { correct_count += 1; }
     }
   };
 
@@ -498,20 +496,18 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) {
 
   std::shared_ptr<MemoryMappedFile> file;
   ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &file));
-  ASSERT_OK(file->Write(reinterpret_cast<const uint8_t*>(data.c_str()),
-          static_cast<int64_t>(data.size())));
+  ASSERT_OK(file->Write(
+      reinterpret_cast<const uint8_t*>(data.c_str()), static_cast<int64_t>(data.size())));
 
   std::atomic<int> correct_count(0);
   const int niter = 10000;
 
-  auto ReadData = [&correct_count, &data, niter, &file] () {
+  auto ReadData = [&correct_count, &data, niter, &file]() {
     std::shared_ptr<Buffer> buffer;
 
     for (int i = 0; i < niter; ++i) {
       ASSERT_OK(file->ReadAt(0, 3, &buffer));
-      if (0 == memcmp(data.c_str(), buffer->data(), 3)) {
-        correct_count += 1;
-      }
+      if (0 == memcmp(data.c_str(), buffer->data(), 3)) { correct_count += 1; }
     }
   };
 
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
index bc086e31519a5..3b1832611024f 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/file.cc
@@ -158,6 +158,9 @@ Status FileFooter::GetSchema(std::shared_ptr<Schema>* out) const {
 // ----------------------------------------------------------------------
 // File writer implementation
 
+FileWriter::FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
+    : StreamWriter(sink, schema) {}
+
 Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
     std::shared_ptr<FileWriter>* out) {
   *out = std::shared_ptr<FileWriter>(new FileWriter(sink, schema));  // ctor is private
diff --git a/cpp/src/arrow/ipc/file.h b/cpp/src/arrow/ipc/file.h
index 7696954c188e3..cf0baab820eef 100644
--- a/cpp/src/arrow/ipc/file.h
+++ b/cpp/src/arrow/ipc/file.h
@@ -78,7 +78,7 @@ class ARROW_EXPORT FileWriter : public StreamWriter {
   Status Close() override;
 
  private:
-  using StreamWriter::StreamWriter;
+  FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema);
 
   Status Start() override;
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 6e15ef353d853..81e3dbdf6c4c0 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -85,12 +85,12 @@ class ARROW_EXPORT SchemaMetadata {
 };
 
 // Field metadata
-struct FieldMetadata {
+struct ARROW_EXPORT FieldMetadata {
   int32_t length;
   int32_t null_count;
 };
 
-struct BufferMetadata {
+struct ARROW_EXPORT BufferMetadata {
   int32_t page;
   int64_t offset;
   int64_t length;
@@ -149,7 +149,7 @@ class ARROW_EXPORT Message {
   std::unique_ptr<MessageImpl> impl_;
 };
 
-struct FileBlock {
+struct ARROW_EXPORT FileBlock {
   FileBlock() {}
   FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
       : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 0830ee2ed2928..19b1e193d4228 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -68,24 +68,20 @@ if (ARROW_BUILD_BENCHMARKS)
   endif()
 endif()
 
-if (ARROW_BUILD_UTILITIES)
-  if (APPLE)
-    set(UTIL_LINK_LIBS
-      arrow_ipc_static
-      arrow_io_static
-      arrow_static
-      boost_filesystem_static
-      boost_system_static
-      dl)
-  else()
+if (ARROW_IPC AND ARROW_BUILD_UTILITIES)
+  set(UTIL_LINK_LIBS
+    arrow_ipc_static
+    arrow_io_static
+    arrow_static
+    boost_filesystem_static
+    boost_system_static
+    dl)
+
+  if (NOT APPLE)
     set(UTIL_LINK_LIBS
-      arrow_ipc_static
-      arrow_io_static
-      arrow_static
-      pthread
+      ${UTIL_LINK_LIBS}
       boost_filesystem_static
-      boost_system_static
-      dl)
+      boost_system_static)
   endif()
 
   add_executable(file-to-stream file-to-stream.cc)
diff --git a/cpp/src/arrow/util/file-to-stream.cc b/cpp/src/arrow/util/file-to-stream.cc
index 42c1d55afd322..7daf26366721d 100644
--- a/cpp/src/arrow/util/file-to-stream.cc
+++ b/cpp/src/arrow/util/file-to-stream.cc
@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <iostream>
 #include "arrow/io/file.h"
 #include "arrow/ipc/file.h"
 #include "arrow/ipc/stream.h"
 #include "arrow/status.h"
+#include <iostream>
 
 #include "arrow/util/io-util.h"
 
@@ -44,7 +44,7 @@ Status ConvertToStream(const char* path) {
   return writer->Close();
 }
 
-} // namespace arrow
+}  // namespace arrow
 
 int main(int argc, char** argv) {
   if (argc != 2) {
diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
index 3e5054d8fa83d..9f2645699004c 100644
--- a/cpp/src/arrow/util/io-util.h
+++ b/cpp/src/arrow/util/io-util.h
@@ -18,8 +18,8 @@
 #ifndef ARROW_UTIL_IO_UTIL_H
 #define ARROW_UTIL_IO_UTIL_H
 
-#include <iostream>
 #include "arrow/buffer.h"
+#include <iostream>
 
 namespace arrow {
 namespace io {
@@ -27,13 +27,11 @@ namespace io {
 // Output stream that just writes to stdout.
 class StdoutStream : public OutputStream {
  public:
-  StdoutStream() : pos_(0) {
-    set_mode(FileMode::WRITE);
-  }
+  StdoutStream() : pos_(0) { set_mode(FileMode::WRITE); }
   virtual ~StdoutStream() {}
 
   Status Close() { return Status::OK(); }
-  Status Tell(int64_t* position)  {
+  Status Tell(int64_t* position) {
     *position = pos_;
     return Status::OK();
   }
@@ -43,6 +41,7 @@ class StdoutStream : public OutputStream {
     std::cout.write(reinterpret_cast<const char*>(data), nbytes);
     return Status::OK();
   }
+
  private:
   int64_t pos_;
 };
@@ -50,13 +49,11 @@ class StdoutStream : public OutputStream {
 // Input stream that just reads from stdin.
 class StdinStream : public InputStream {
  public:
-  StdinStream() : pos_(0) {
-    set_mode(FileMode::READ);
-  }
+  StdinStream() : pos_(0) { set_mode(FileMode::READ); }
   virtual ~StdinStream() {}
 
   Status Close() { return Status::OK(); }
-  Status Tell(int64_t* position)  {
+  Status Tell(int64_t* position) {
     *position = pos_;
     return Status::OK();
   }
@@ -86,8 +83,7 @@ class StdinStream : public InputStream {
   int64_t pos_;
 };
 
-} // namespace io
-} // namespace arrow
-
-#endif // ARROW_UTIL_IO_UTIL_H
+}  // namespace io
+}  // namespace arrow
 
+#endif  // ARROW_UTIL_IO_UTIL_H
diff --git a/cpp/src/arrow/util/stream-to-file.cc b/cpp/src/arrow/util/stream-to-file.cc
index 7a8ec0bfd952b..393b07d8d355f 100644
--- a/cpp/src/arrow/util/stream-to-file.cc
+++ b/cpp/src/arrow/util/stream-to-file.cc
@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <iostream>
+#include "arrow/ipc/stream.h"
 #include "arrow/io/file.h"
 #include "arrow/ipc/file.h"
-#include "arrow/ipc/stream.h"
 #include "arrow/status.h"
+#include <iostream>
 
 #include "arrow/util/io-util.h"
 
@@ -46,7 +46,7 @@ Status ConvertToFile() {
   return writer->Close();
 }
 
-} // namespace arrow
+}  // namespace arrow
 
 int main(int argc, char** argv) {
   arrow::Status status = arrow::ConvertToFile();
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
index 9321cc550ec1f..4819a0061e75f 100644
--- a/cpp/src/arrow/util/visibility.h
+++ b/cpp/src/arrow/util/visibility.h
@@ -19,6 +19,11 @@
 #define ARROW_UTIL_VISIBILITY_H
 
 #if defined(_WIN32) || defined(__CYGWIN__)
+#if defined(_MSC_VER)
+#pragma warning(disable : 4251)
+#else
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif
 #define ARROW_EXPORT __declspec(dllexport)
 #define ARROW_NO_EXPORT
 #else  // Not Windows

From 30bb0d97d584b65ad6ed8ab225c5c4008eafb88c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 26 Jan 2017 22:12:19 -0500
Subject: [PATCH 0298/1644] ARROW-514: [Python] Automatically wrap
 pyarrow.io.Buffer in BufferReader

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #306 from wesm/ARROW-514 and squashes the following commits:

d5e3235 [Wes McKinney] Automatically wrap pyarrow.io.Buffer when passing in to FileReader or StreamReader
---
 python/pyarrow/io.pyx            | 2 ++
 python/pyarrow/tests/test_ipc.py | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 0755ed8bb4d4f..e5f8b7a0abff4 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -526,6 +526,8 @@ cdef get_reader(object source, shared_ptr[ReadableFileInterface]* reader):
 
     if isinstance(source, six.string_types):
         source = MemoryMappedFile(source, mode='r')
+    elif isinstance(source, Buffer):
+        source = BufferReader(source)
     elif not isinstance(source, NativeFile) and hasattr(source, 'read'):
         # Optimistically hope this is file-like
         source = PythonFileInterface(source, mode='r')
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 819d1b71b8546..8ca464f034d14 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -36,7 +36,7 @@ def _get_sink(self):
         return io.BytesIO()
 
     def _get_source(self):
-        return self.sink.getvalue()
+        return pa.BufferReader(self.sink.getvalue())
 
     def write_batches(self):
         nrows = 5
@@ -74,7 +74,7 @@ def test_simple_roundtrip(self):
         batches = self.write_batches()
         file_contents = self._get_source()
 
-        reader = pa.FileReader(pa.BufferReader(file_contents))
+        reader = pa.FileReader(file_contents)
 
         assert reader.num_record_batches == len(batches)
 
@@ -92,7 +92,7 @@ def _get_writer(self, sink, schema):
     def test_simple_roundtrip(self):
         batches = self.write_batches()
         file_contents = self._get_source()
-        reader = pa.StreamReader(pa.BufferReader(file_contents))
+        reader = pa.StreamReader(file_contents)
 
         total = 0
         for i, next_batch in enumerate(reader):

From 4226adfbc6b3dff10b3fe7a6691b30bcc94140bd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 27 Jan 2017 10:46:34 +0100
Subject: [PATCH 0299/1644] ARROW-515: [Python] Add read_all methods to
 FileReader, StreamReader

Stacked on top of ARROW-514

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #307 from wesm/ARROW-515 and squashes the following commits:

6f2185c [Wes McKinney] Add read_all method to StreamReader, FileReader
---
 python/pyarrow/io.pyx            | 44 +++++++++++++++++++++++++++++++-
 python/pyarrow/table.pyx         |  4 +--
 python/pyarrow/tests/test_ipc.py | 19 ++++++++++++++
 3 files changed, 63 insertions(+), 4 deletions(-)

diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index e5f8b7a0abff4..8b5650879f8f1 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -34,7 +34,8 @@ cimport pyarrow.includes.pyarrow as pyarrow
 from pyarrow.compat import frombytes, tobytes, encode_file_path
 from pyarrow.error cimport check_status
 from pyarrow.schema cimport Schema
-from pyarrow.table cimport RecordBatch, batch_from_cbatch
+from pyarrow.table cimport (RecordBatch, batch_from_cbatch,
+                            table_from_ctable)
 
 cimport cpython as cp
 
@@ -936,6 +937,27 @@ cdef class _StreamReader:
 
         return batch_from_cbatch(batch)
 
+    def read_all(self):
+        """
+        Read all record batches as a pyarrow.Table
+        """
+        cdef:
+            vector[shared_ptr[CRecordBatch]] batches
+            shared_ptr[CRecordBatch] batch
+            shared_ptr[CTable] table
+            c_string name = b''
+
+        with nogil:
+            while True:
+                check_status(self.reader.get().GetNextRecordBatch(&batch))
+                if batch.get() == NULL:
+                    break
+                batches.push_back(batch)
+
+            check_status(CTable.FromRecordBatches(name, batches, &table))
+
+        return table_from_ctable(table)
+
 
 cdef class _FileWriter(_StreamWriter):
 
@@ -997,3 +1019,23 @@ cdef class _FileReader:
     # TODO(wesm): ARROW-503: Function was renamed. Remove after a period of
     # time has passed
     get_record_batch = get_batch
+
+    def read_all(self):
+        """
+        Read all record batches as a pyarrow.Table
+        """
+        cdef:
+            vector[shared_ptr[CRecordBatch]] batches
+            shared_ptr[CTable] table
+            c_string name = b''
+            int i, nbatches
+
+        nbatches = self.num_record_batches
+
+        batches.resize(nbatches)
+        with nogil:
+            for i in range(nbatches):
+                check_status(self.reader.get().GetRecordBatch(i, &batches[i]))
+            check_status(CTable.FromRecordBatches(name, batches, &table))
+
+        return table_from_ctable(table)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 924233066055e..17072108f301f 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -690,9 +690,7 @@ cdef class Table:
         with nogil:
             check_status(CTable.FromRecordBatches(c_name, c_batches, &c_table))
 
-        table = Table()
-        table.init(c_table)
-        return table
+        return table_from_ctable(c_table)
 
     def to_pandas(self, nthreads=None):
         """
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 8ca464f034d14..665a63b6d5a38 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -83,6 +83,16 @@ def test_simple_roundtrip(self):
             batch = reader.get_batch(i)
             assert batches[i].equals(batch)
 
+    def test_read_all(self):
+        batches = self.write_batches()
+        file_contents = self._get_source()
+
+        reader = pa.FileReader(file_contents)
+
+        result = reader.read_all()
+        expected = pa.Table.from_batches(batches)
+        assert result.equals(expected)
+
 
 class TestStream(MessagingTest, unittest.TestCase):
 
@@ -104,6 +114,15 @@ def test_simple_roundtrip(self):
         with pytest.raises(StopIteration):
             reader.get_next_batch()
 
+    def test_read_all(self):
+        batches = self.write_batches()
+        file_contents = self._get_source()
+        reader = pa.StreamReader(file_contents)
+
+        result = reader.read_all()
+        expected = pa.Table.from_batches(batches)
+        assert result.equals(expected)
+
 
 class TestInMemoryFile(TestFile):
 

From 7ac320bde52ae47007dadac7398e22a203c6a48d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 29 Jan 2017 21:27:17 -0500
Subject: [PATCH 0300/1644] ARROW-519: [C++] Refactor array comparison code
 into a compare.h / compare.cc in part to resolve Xcode 6.1 linker issue

This should also pave the way for more user-friendly reporting of "why are the arrays not equal" per ARROW-517

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #308 from wesm/ARROW-519 and squashes the following commits:

85b0bf8 [Wes McKinney] Fix invalid memory access when doing RangeEquals on BinaryArray with all empty strings
f5f4593 [Wes McKinney] Remove unused function in pandas.cc. Fix Binary RangeEquals for arrays of length-0 strings
2118ef4 [Wes McKinney] cpplint, compiler warnings
ad54cc6 [Wes McKinney] Remove unneeded ARROW_EXPORT
342a8e6 [Wes McKinney] Refactor array comparison code into a compare.h header and compilation unit. Use visitor pattern. Also may resolve Xcode bug reported in ARROW-519
---
 cpp/CMakeLists.txt                    |   1 +
 cpp/src/arrow/CMakeLists.txt          |   1 +
 cpp/src/arrow/array-primitive-test.cc |   4 +-
 cpp/src/arrow/array-string-test.cc    |  48 ++-
 cpp/src/arrow/array.cc                | 334 ++---------------
 cpp/src/arrow/array.h                 | 145 +-------
 cpp/src/arrow/compare.cc              | 516 ++++++++++++++++++++++++++
 cpp/src/arrow/compare.h               |  46 +++
 cpp/src/arrow/util/macros.h           |   2 +
 python/CMakeLists.txt                 |   3 +
 python/src/pyarrow/adapters/pandas.cc |   8 -
 11 files changed, 641 insertions(+), 467 deletions(-)
 create mode 100644 cpp/src/arrow/compare.cc
 create mode 100644 cpp/src/arrow/compare.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index a0f89f314f683..ff2c1a61b95a6 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -771,6 +771,7 @@ set(ARROW_SRCS
   src/arrow/buffer.cc
   src/arrow/builder.cc
   src/arrow/column.cc
+  src/arrow/compare.cc
   src/arrow/memory_pool.cc
   src/arrow/pretty_print.cc
   src/arrow/schema.cc
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index e5e36ed253cfa..b002bb75ca934 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -20,6 +20,7 @@ install(FILES
   api.h
   array.h
   column.h
+  compare.h
   buffer.h
   builder.h
   memory_pool.h
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
index 443abac459dbf..c839fb9b19234 100644
--- a/cpp/src/arrow/array-primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -135,7 +135,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     ASSERT_EQ(nullptr, builder->data());
 
     ASSERT_EQ(ex_null_count, result->null_count());
-    ASSERT_TRUE(result->EqualsExact(*expected.get()));
+    ASSERT_TRUE(result->Equals(*expected));
   }
 
  protected:
@@ -238,7 +238,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
     bool actual = BitUtil::GetBit(result->raw_data(), i);
     ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
   }
-  ASSERT_TRUE(result->EqualsExact(*expected.get()));
+  ASSERT_TRUE(result->Equals(*expected));
 }
 
 typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index 024bfd508957d..5ea384acb1c57 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -51,7 +51,7 @@ TEST(TypesTest, TestStringType) {
 // ----------------------------------------------------------------------
 // String container
 
-class TestStringContainer : public ::testing::Test {
+class TestStringArray : public ::testing::Test {
  public:
   void SetUp() {
     chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
@@ -90,20 +90,20 @@ class TestStringContainer : public ::testing::Test {
   std::shared_ptr<StringArray> strings_;
 };
 
-TEST_F(TestStringContainer, TestArrayBasics) {
+TEST_F(TestStringArray, TestArrayBasics) {
   ASSERT_EQ(length_, strings_->length());
   ASSERT_EQ(1, strings_->null_count());
   ASSERT_OK(strings_->Validate());
 }
 
-TEST_F(TestStringContainer, TestType) {
+TEST_F(TestStringArray, TestType) {
   TypePtr type = strings_->type();
 
   ASSERT_EQ(Type::STRING, type->type);
   ASSERT_EQ(Type::STRING, strings_->type_enum());
 }
 
-TEST_F(TestStringContainer, TestListFunctions) {
+TEST_F(TestStringArray, TestListFunctions) {
   int pos = 0;
   for (size_t i = 0; i < expected_.size(); ++i) {
     ASSERT_EQ(pos, strings_->value_offset(i));
@@ -112,12 +112,12 @@ TEST_F(TestStringContainer, TestListFunctions) {
   }
 }
 
-TEST_F(TestStringContainer, TestDestructor) {
+TEST_F(TestStringArray, TestDestructor) {
   auto arr = std::make_shared<StringArray>(
       length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
 }
 
-TEST_F(TestStringContainer, TestGetString) {
+TEST_F(TestStringArray, TestGetString) {
   for (size_t i = 0; i < expected_.size(); ++i) {
     if (valid_bytes_[i] == 0) {
       ASSERT_TRUE(strings_->IsNull(i));
@@ -127,7 +127,7 @@ TEST_F(TestStringContainer, TestGetString) {
   }
 }
 
-TEST_F(TestStringContainer, TestEmptyStringComparison) {
+TEST_F(TestStringArray, TestEmptyStringComparison) {
   offsets_ = {0, 0, 0, 0, 0, 0};
   offsets_buf_ = test::GetBufferFromVector(offsets_);
   length_ = offsets_.size() - 1;
@@ -212,7 +212,7 @@ TEST_F(TestStringBuilder, TestZeroLength) {
 // Binary container type
 // TODO(emkornfield) there should be some way to refactor these to avoid code duplicating
 // with String
-class TestBinaryContainer : public ::testing::Test {
+class TestBinaryArray : public ::testing::Test {
  public:
   void SetUp() {
     chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
@@ -252,20 +252,20 @@ class TestBinaryContainer : public ::testing::Test {
   std::shared_ptr<BinaryArray> strings_;
 };
 
-TEST_F(TestBinaryContainer, TestArrayBasics) {
+TEST_F(TestBinaryArray, TestArrayBasics) {
   ASSERT_EQ(length_, strings_->length());
   ASSERT_EQ(1, strings_->null_count());
   ASSERT_OK(strings_->Validate());
 }
 
-TEST_F(TestBinaryContainer, TestType) {
+TEST_F(TestBinaryArray, TestType) {
   TypePtr type = strings_->type();
 
   ASSERT_EQ(Type::BINARY, type->type);
   ASSERT_EQ(Type::BINARY, strings_->type_enum());
 }
 
-TEST_F(TestBinaryContainer, TestListFunctions) {
+TEST_F(TestBinaryArray, TestListFunctions) {
   int pos = 0;
   for (size_t i = 0; i < expected_.size(); ++i) {
     ASSERT_EQ(pos, strings_->value_offset(i));
@@ -274,12 +274,12 @@ TEST_F(TestBinaryContainer, TestListFunctions) {
   }
 }
 
-TEST_F(TestBinaryContainer, TestDestructor) {
+TEST_F(TestBinaryArray, TestDestructor) {
   auto arr = std::make_shared<BinaryArray>(
       length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
 }
 
-TEST_F(TestBinaryContainer, TestGetValue) {
+TEST_F(TestBinaryArray, TestGetValue) {
   for (size_t i = 0; i < expected_.size(); ++i) {
     if (valid_bytes_[i] == 0) {
       ASSERT_TRUE(strings_->IsNull(i));
@@ -291,6 +291,28 @@ TEST_F(TestBinaryContainer, TestGetValue) {
   }
 }
 
+TEST_F(TestBinaryArray, TestEqualsEmptyStrings) {
+  BinaryBuilder builder(default_memory_pool(), arrow::binary());
+
+  std::string empty_string("");
+
+  builder.Append(empty_string);
+  builder.Append(empty_string);
+  builder.Append(empty_string);
+  builder.Append(empty_string);
+  builder.Append(empty_string);
+
+  std::shared_ptr<Array> left_arr;
+  ASSERT_OK(builder.Finish(&left_arr));
+
+  const BinaryArray& left = static_cast<const BinaryArray&>(*left_arr);
+  std::shared_ptr<Array> right = std::make_shared<BinaryArray>(
+      left.length(), left.offsets(), nullptr, left.null_count(), left.null_bitmap());
+
+  ASSERT_TRUE(left.Equals(right));
+  ASSERT_TRUE(left.RangeEquals(0, left.length(), 0, right));
+}
+
 class TestBinaryBuilder : public TestBuilder {
  public:
   void SetUp() {
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index aa4a692e85cb9..6fc7fb60bf364 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -22,6 +22,7 @@
 #include <sstream>
 
 #include "arrow/buffer.h"
+#include "arrow/compare.h"
 #include "arrow/status.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -51,43 +52,42 @@ Array::Array(const std::shared_ptr<DataType>& type, int32_t length, int32_t null
   if (null_bitmap_) { null_bitmap_data_ = null_bitmap_->data(); }
 }
 
-bool Array::BaseEquals(const std::shared_ptr<Array>& other) const {
-  if (this == other.get()) { return true; }
-  if (!other) { return false; }
-  return EqualsExact(*other.get());
+bool Array::Equals(const Array& arr) const {
+  bool are_equal = false;
+  Status error = ArrayEquals(*this, arr, &are_equal);
+  if (!error.ok()) { DCHECK(false) << "Arrays not comparable: " << error.ToString(); }
+  return are_equal;
 }
 
-bool Array::EqualsExact(const Array& other) const {
-  if (this == &other) { return true; }
-  if (length_ != other.length_ || null_count_ != other.null_count_ ||
-      type_enum() != other.type_enum()) {
-    return false;
-  }
-  if (null_count_ > 0) {
-    return null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
-  }
-  return true;
+bool Array::Equals(const std::shared_ptr<Array>& arr) const {
+  if (!arr) { return false; }
+  return Equals(*arr);
 }
 
-bool Array::ApproxEquals(const std::shared_ptr<Array>& arr) const {
-  return Equals(arr);
+bool Array::ApproxEquals(const Array& arr) const {
+  bool are_equal = false;
+  Status error = ArrayApproxEquals(*this, arr, &are_equal);
+  if (!error.ok()) { DCHECK(false) << "Arrays not comparable: " << error.ToString(); }
+  return are_equal;
 }
 
-Status Array::Validate() const {
-  return Status::OK();
-}
-
-bool NullArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (Type::NA != arr->type_enum()) { return false; }
-  return arr->length() == length_;
+bool Array::ApproxEquals(const std::shared_ptr<Array>& arr) const {
+  if (!arr) { return false; }
+  return ApproxEquals(*arr);
 }
 
-bool NullArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_index,
+bool Array::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
     const std::shared_ptr<Array>& arr) const {
   if (!arr) { return false; }
-  if (Type::NA != arr->type_enum()) { return false; }
-  return true;
+  bool are_equal = false;
+  Status error =
+      ArrayRangeEquals(*this, *arr, start_idx, end_idx, other_start_idx, &are_equal);
+  if (!error.ok()) { DCHECK(false) << "Arrays not comparable: " << error.ToString(); }
+  return are_equal;
+}
+
+Status Array::Validate() const {
+  return Status::OK();
 }
 
 Status NullArray::Accept(ArrayVisitor* visitor) const {
@@ -105,36 +105,6 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t le
   raw_data_ = data == nullptr ? nullptr : data_->data();
 }
 
-bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
-  if (!Array::EqualsExact(other)) { return false; }
-
-  if (null_count_ > 0) {
-    const uint8_t* this_data = raw_data_;
-    const uint8_t* other_data = other.raw_data_;
-
-    auto size_meta = dynamic_cast<const FixedWidthType*>(type_.get());
-    int value_byte_size = size_meta->bit_width() / 8;
-    DCHECK_GT(value_byte_size, 0);
-
-    for (int i = 0; i < length_; ++i) {
-      if (!IsNull(i) && memcmp(this_data, other_data, value_byte_size)) { return false; }
-      this_data += value_byte_size;
-      other_data += value_byte_size;
-    }
-    return true;
-  } else {
-    if (length_ == 0 && other.length_ == 0) { return true; }
-    return data_->Equals(*other.data_, length_);
-  }
-}
-
-bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(static_cast<const PrimitiveArray&>(*arr.get()));
-}
-
 template <typename T>
 Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
@@ -150,6 +120,7 @@ template class NumericArray<Int32Type>;
 template class NumericArray<Int64Type>;
 template class NumericArray<TimestampType>;
 template class NumericArray<DateType>;
+template class NumericArray<TimeType>;
 template class NumericArray<HalfFloatType>;
 template class NumericArray<FloatType>;
 template class NumericArray<DoubleType>;
@@ -167,50 +138,6 @@ BooleanArray::BooleanArray(const std::shared_ptr<DataType>& type, int32_t length
     const std::shared_ptr<Buffer>& null_bitmap)
     : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
 
-bool BooleanArray::EqualsExact(const BooleanArray& other) const {
-  if (this == &other) return true;
-  if (null_count_ != other.null_count_) { return false; }
-
-  if (null_count_ > 0) {
-    bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
-    if (!equal_bitmap) { return false; }
-
-    const uint8_t* this_data = raw_data_;
-    const uint8_t* other_data = other.raw_data_;
-
-    for (int i = 0; i < length_; ++i) {
-      if (!IsNull(i) && BitUtil::GetBit(this_data, i) != BitUtil::GetBit(other_data, i)) {
-        return false;
-      }
-    }
-    return true;
-  } else {
-    return data_->Equals(*other.data_, BitUtil::BytesForBits(length_));
-  }
-}
-
-bool BooleanArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) return true;
-  if (Type::BOOL != arr->type_enum()) { return false; }
-  return EqualsExact(static_cast<const BooleanArray&>(*arr.get()));
-}
-
-bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
-    int32_t other_start_idx, const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  const auto other = static_cast<BooleanArray*>(arr.get());
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    const bool is_null = IsNull(i);
-    if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
-      return false;
-    }
-  }
-  return true;
-}
-
 Status BooleanArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
@@ -218,56 +145,6 @@ Status BooleanArray::Accept(ArrayVisitor* visitor) const {
 // ----------------------------------------------------------------------
 // ListArray
 
-bool ListArray::EqualsExact(const ListArray& other) const {
-  if (this == &other) { return true; }
-  if (null_count_ != other.null_count_) { return false; }
-
-  bool equal_offsets =
-      offsets_buffer_->Equals(*other.offsets_buffer_, (length_ + 1) * sizeof(int32_t));
-  if (!equal_offsets) { return false; }
-  bool equal_null_bitmap = true;
-  if (null_count_ > 0) {
-    equal_null_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
-  }
-
-  if (!equal_null_bitmap) { return false; }
-
-  return values()->Equals(other.values());
-}
-
-bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(static_cast<const ListArray&>(*arr.get()));
-}
-
-bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  const auto other = static_cast<ListArray*>(arr.get());
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    const bool is_null = IsNull(i);
-    if (is_null != arr->IsNull(o_i)) { return false; }
-    if (is_null) continue;
-    const int32_t begin_offset = offset(i);
-    const int32_t end_offset = offset(i + 1);
-    const int32_t other_begin_offset = other->offset(o_i);
-    const int32_t other_end_offset = other->offset(o_i + 1);
-    // Underlying can't be equal if the size isn't equal
-    if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
-      return false;
-    }
-    if (!values_->RangeEquals(
-            begin_offset, end_offset, other_begin_offset, other->values())) {
-      return false;
-    }
-  }
-  return true;
-}
-
 Status ListArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
   if (!offsets_buffer_) { return Status::Invalid("offsets_buffer_ was null"); }
@@ -350,51 +227,6 @@ Status BinaryArray::Validate() const {
   return Status::OK();
 }
 
-bool BinaryArray::EqualsExact(const BinaryArray& other) const {
-  if (!Array::EqualsExact(other)) { return false; }
-
-  bool equal_offsets =
-      offsets_buffer_->Equals(*other.offsets_buffer_, (length_ + 1) * sizeof(int32_t));
-  if (!equal_offsets) { return false; }
-
-  if (!data_buffer_ && !(other.data_buffer_)) { return true; }
-
-  return data_buffer_->Equals(*other.data_buffer_, raw_offsets()[length_]);
-}
-
-bool BinaryArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  return EqualsExact(static_cast<const BinaryArray&>(*arr.get()));
-}
-
-bool BinaryArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  const auto other = static_cast<const BinaryArray*>(arr.get());
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    const bool is_null = IsNull(i);
-    if (is_null != arr->IsNull(o_i)) { return false; }
-    if (is_null) continue;
-    const int32_t begin_offset = offset(i);
-    const int32_t end_offset = offset(i + 1);
-    const int32_t other_begin_offset = other->offset(o_i);
-    const int32_t other_end_offset = other->offset(o_i + 1);
-    // Underlying can't be equal if the size isn't equal
-    if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
-      return false;
-    }
-
-    if (std::memcmp(data_ + begin_offset, other->data_ + other_begin_offset,
-            end_offset - begin_offset)) {
-      return false;
-    }
-  }
-  return true;
-}
-
 Status BinaryArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
@@ -421,36 +253,6 @@ std::shared_ptr<Array> StructArray::field(int32_t pos) const {
   return field_arrays_[pos];
 }
 
-bool StructArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-  if (null_count_ != arr->null_count()) { return false; }
-  return RangeEquals(0, length_, 0, arr);
-}
-
-bool StructArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (Type::STRUCT != arr->type_enum()) { return false; }
-  const auto& other = static_cast<const StructArray&>(*arr.get());
-
-  bool equal_fields = true;
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    if (IsNull(i) != arr->IsNull(o_i)) { return false; }
-    if (IsNull(i)) continue;
-    for (size_t j = 0; j < field_arrays_.size(); ++j) {
-      // TODO: really we should be comparing stretches of non-null data rather
-      // than looking at one value at a time.
-      equal_fields = field(j)->RangeEquals(i, i + 1, o_i, other.field(j));
-      if (!equal_fields) { return false; }
-    }
-  }
-
-  return true;
-}
-
 Status StructArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
 
@@ -511,67 +313,6 @@ std::shared_ptr<Array> UnionArray::child(int32_t pos) const {
   return children_[pos];
 }
 
-bool UnionArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (!this->type_->Equals(arr->type())) { return false; }
-  if (null_count_ != arr->null_count()) { return false; }
-  return RangeEquals(0, length_, 0, arr);
-}
-
-bool UnionArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (Type::UNION != arr->type_enum()) { return false; }
-  const auto& other = static_cast<const UnionArray&>(*arr.get());
-
-  const UnionMode union_mode = mode();
-  if (union_mode != other.mode()) { return false; }
-
-  // Define a mapping from the type id to child number
-  const auto& type_codes = static_cast<const UnionType&>(*arr->type().get()).type_ids;
-  uint8_t max_code = 0;
-  for (uint8_t code : type_codes) {
-    if (code > max_code) { max_code = code; }
-  }
-
-  // Store mapping in a vector for constant time lookups
-  std::vector<uint8_t> type_id_to_child_num(max_code + 1);
-  for (uint8_t i = 0; i < static_cast<uint8_t>(type_codes.size()); ++i) {
-    type_id_to_child_num[type_codes[i]] = i;
-  }
-
-  const uint8_t* this_ids = raw_type_ids();
-  const uint8_t* other_ids = other.raw_type_ids();
-
-  uint8_t id, child_num;
-  for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-    if (IsNull(i) != other.IsNull(o_i)) { return false; }
-    if (IsNull(i)) continue;
-    if (this_ids[i] != other_ids[o_i]) { return false; }
-
-    id = this_ids[i];
-    child_num = type_id_to_child_num[id];
-
-    // TODO(wesm): really we should be comparing stretches of non-null data
-    // rather than looking at one value at a time.
-    if (union_mode == UnionMode::SPARSE) {
-      if (!child(child_num)->RangeEquals(i, i + 1, o_i, other.child(child_num))) {
-        return false;
-      }
-    } else {
-      const int32_t offset = offsets_[i];
-      const int32_t o_offset = other.offsets_[i];
-      if (!child(child_num)->RangeEquals(
-              offset, offset + 1, o_offset, other.child(child_num))) {
-        return false;
-      }
-    }
-  }
-  return true;
-}
-
 Status UnionArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
 
@@ -624,25 +365,6 @@ std::shared_ptr<Array> DictionaryArray::dictionary() const {
   return dict_type_->dictionary();
 }
 
-bool DictionaryArray::EqualsExact(const DictionaryArray& other) const {
-  if (!dictionary()->Equals(other.dictionary())) { return false; }
-  return indices_->Equals(other.indices());
-}
-
-bool DictionaryArray::Equals(const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (Type::DICTIONARY != arr->type_enum()) { return false; }
-  return EqualsExact(static_cast<const DictionaryArray&>(*arr.get()));
-}
-
-bool DictionaryArray::RangeEquals(int32_t start_idx, int32_t end_idx,
-    int32_t other_start_idx, const std::shared_ptr<Array>& arr) const {
-  if (Type::DICTIONARY != arr->type_enum()) { return false; }
-  const auto& dict_other = static_cast<const DictionaryArray&>(*arr.get());
-  if (!dictionary()->Equals(dict_other.dictionary())) { return false; }
-  return indices_->RangeEquals(start_idx, end_idx, other_start_idx, dict_other.indices());
-}
-
 Status DictionaryArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 4f4b727f39ae5..3b6e93f9cb34d 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -102,15 +102,16 @@ class ARROW_EXPORT Array {
   /// Note that for `null_count == 0`, this can be a `nullptr`.
   const uint8_t* null_bitmap_data() const { return null_bitmap_data_; }
 
-  bool BaseEquals(const std::shared_ptr<Array>& arr) const;
-  bool EqualsExact(const Array& arr) const;
-  virtual bool Equals(const std::shared_ptr<Array>& arr) const = 0;
-  virtual bool ApproxEquals(const std::shared_ptr<Array>& arr) const;
+  bool Equals(const Array& arr) const;
+  bool Equals(const std::shared_ptr<Array>& arr) const;
+
+  bool ApproxEquals(const std::shared_ptr<Array>& arr) const;
+  bool ApproxEquals(const Array& arr) const;
 
   /// Compare if the range of slots specified are equal for the given array and
   /// this array.  end_idx exclusive.  This methods does not bounds check.
-  virtual bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const = 0;
+  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+      const std::shared_ptr<Array>& arr) const;
 
   /// Determines if the array is internally consistent.
   ///
@@ -142,10 +143,6 @@ class ARROW_EXPORT NullArray : public Array {
 
   explicit NullArray(int32_t length) : NullArray(std::make_shared<NullType>(), length) {}
 
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_index,
-      const std::shared_ptr<Array>& arr) const override;
-
   Status Accept(ArrayVisitor* visitor) const override;
 };
 
@@ -159,9 +156,6 @@ class ARROW_EXPORT PrimitiveArray : public Array {
 
   std::shared_ptr<Buffer> data() const { return data_; }
 
-  bool EqualsExact(const PrimitiveArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-
  protected:
   PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
       const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
@@ -184,28 +178,6 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
       const std::shared_ptr<Buffer>& null_bitmap = nullptr)
       : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
 
-  bool EqualsExact(const NumericArray<TypeClass>& other) const {
-    return PrimitiveArray::EqualsExact(static_cast<const PrimitiveArray&>(other));
-  }
-
-  bool ApproxEquals(const std::shared_ptr<Array>& arr) const override {
-    return PrimitiveArray::Equals(arr);
-  }
-
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const override {
-    if (this == arr.get()) { return true; }
-    if (!arr) { return false; }
-    if (this->type_enum() != arr->type_enum()) { return false; }
-    const auto other = static_cast<NumericArray<TypeClass>*>(arr.get());
-    for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
-      const bool is_null = IsNull(i);
-      if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
-        return false;
-      }
-    }
-    return true;
-  }
   const value_type* raw_data() const {
     return reinterpret_cast<const value_type*>(raw_data_);
   }
@@ -215,78 +187,6 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   value_type Value(int i) const { return raw_data()[i]; }
 };
 
-template <>
-inline bool NumericArray<FloatType>::ApproxEquals(
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-
-  const auto& other = *static_cast<NumericArray<FloatType>*>(arr.get());
-
-  if (this == &other) { return true; }
-  if (null_count_ != other.null_count_) { return false; }
-
-  auto this_data = reinterpret_cast<const float*>(raw_data_);
-  auto other_data = reinterpret_cast<const float*>(other.raw_data_);
-
-  static constexpr float EPSILON = 1E-5;
-
-  if (length_ == 0 && other.length_ == 0) { return true; }
-
-  if (null_count_ > 0) {
-    bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
-    if (!equal_bitmap) { return false; }
-
-    for (int i = 0; i < length_; ++i) {
-      if (IsNull(i)) continue;
-      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
-    }
-  } else {
-    for (int i = 0; i < length_; ++i) {
-      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
-    }
-  }
-  return true;
-}
-
-template <>
-inline bool NumericArray<DoubleType>::ApproxEquals(
-    const std::shared_ptr<Array>& arr) const {
-  if (this == arr.get()) { return true; }
-  if (!arr) { return false; }
-  if (this->type_enum() != arr->type_enum()) { return false; }
-
-  const auto& other = *static_cast<NumericArray<DoubleType>*>(arr.get());
-
-  if (this == &other) { return true; }
-  if (null_count_ != other.null_count_) { return false; }
-
-  auto this_data = reinterpret_cast<const double*>(raw_data_);
-  auto other_data = reinterpret_cast<const double*>(other.raw_data_);
-
-  if (length_ == 0 && other.length_ == 0) { return true; }
-
-  static constexpr double EPSILON = 1E-5;
-
-  if (null_count_ > 0) {
-    bool equal_bitmap =
-        null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
-    if (!equal_bitmap) { return false; }
-
-    for (int i = 0; i < length_; ++i) {
-      if (IsNull(i)) continue;
-      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
-    }
-  } else {
-    for (int i = 0; i < length_; ++i) {
-      if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
-    }
-  }
-  return true;
-}
-
 class ARROW_EXPORT BooleanArray : public PrimitiveArray {
  public:
   using TypeClass = BooleanType;
@@ -297,11 +197,6 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
       const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr);
 
-  bool EqualsExact(const BooleanArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const override;
-
   Status Accept(ArrayVisitor* visitor) const override;
 
   const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
@@ -345,12 +240,6 @@ class ARROW_EXPORT ListArray : public Array {
   int32_t value_offset(int i) const { return offsets_[i]; }
   int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
 
-  bool EqualsExact(const ListArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const override;
-
   Status Accept(ArrayVisitor* visitor) const override;
 
  protected:
@@ -396,11 +285,6 @@ class ARROW_EXPORT BinaryArray : public Array {
   int32_t value_offset(int i) const { return offsets_[i]; }
   int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
 
-  bool EqualsExact(const BinaryArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const override;
-
   Status Validate() const override;
 
   Status Accept(ArrayVisitor* visitor) const override;
@@ -459,11 +343,6 @@ class ARROW_EXPORT StructArray : public Array {
 
   const std::vector<std::shared_ptr<Array>>& fields() const { return field_arrays_; }
 
-  bool EqualsExact(const StructArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const override;
-
   Status Accept(ArrayVisitor* visitor) const override;
 
  protected:
@@ -500,11 +379,6 @@ class ARROW_EXPORT UnionArray : public Array {
 
   const std::vector<std::shared_ptr<Array>>& children() const { return children_; }
 
-  bool EqualsExact(const UnionArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const override;
-
   Status Accept(ArrayVisitor* visitor) const override;
 
  protected:
@@ -555,11 +429,6 @@ class ARROW_EXPORT DictionaryArray : public Array {
 
   const DictionaryType* dict_type() { return dict_type_; }
 
-  bool EqualsExact(const DictionaryArray& other) const;
-  bool Equals(const std::shared_ptr<Array>& arr) const override;
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const override;
-
   Status Accept(ArrayVisitor* visitor) const override;
 
  protected:
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
new file mode 100644
index 0000000000000..d039bba08827c
--- /dev/null
+++ b/cpp/src/arrow/compare.cc
@@ -0,0 +1,516 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Functions for comparing Arrow data structures
+
+#include "arrow/compare.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// Public method implementations
+
+class RangeEqualsVisitor : public ArrayVisitor {
+ public:
+  RangeEqualsVisitor(const Array& right, int32_t left_start_idx, int32_t left_end_idx,
+      int32_t right_start_idx)
+      : right_(right),
+        left_start_idx_(left_start_idx),
+        left_end_idx_(left_end_idx),
+        right_start_idx_(right_start_idx),
+        result_(false) {}
+
+  Status Visit(const NullArray& left) override {
+    UNUSED(left);
+    result_ = true;
+    return Status::OK();
+  }
+
+  template <typename ArrayType>
+  inline Status CompareValues(const ArrayType& left) {
+    const auto& right = static_cast<const ArrayType&>(right_);
+
+    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+         ++i, ++o_i) {
+      const bool is_null = left.IsNull(i);
+      if (is_null != right.IsNull(o_i) ||
+          (!is_null && left.Value(i) != right.Value(o_i))) {
+        result_ = false;
+        return Status::OK();
+      }
+    }
+    result_ = true;
+    return Status::OK();
+  }
+
+  bool CompareBinaryRange(const BinaryArray& left) const {
+    const auto& right = static_cast<const BinaryArray&>(right_);
+
+    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+         ++i, ++o_i) {
+      const bool is_null = left.IsNull(i);
+      if (is_null != right.IsNull(o_i)) { return false; }
+      if (is_null) continue;
+      const int32_t begin_offset = left.offset(i);
+      const int32_t end_offset = left.offset(i + 1);
+      const int32_t right_begin_offset = right.offset(o_i);
+      const int32_t right_end_offset = right.offset(o_i + 1);
+      // Underlying can't be equal if the size isn't equal
+      if (end_offset - begin_offset != right_end_offset - right_begin_offset) {
+        return false;
+      }
+
+      if (end_offset - begin_offset > 0 &&
+          std::memcmp(left.data()->data() + begin_offset,
+              right.data()->data() + right_begin_offset, end_offset - begin_offset)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  Status Visit(const BooleanArray& left) override {
+    return CompareValues<BooleanArray>(left);
+  }
+
+  Status Visit(const Int8Array& left) override { return CompareValues<Int8Array>(left); }
+
+  Status Visit(const Int16Array& left) override {
+    return CompareValues<Int16Array>(left);
+  }
+  Status Visit(const Int32Array& left) override {
+    return CompareValues<Int32Array>(left);
+  }
+  Status Visit(const Int64Array& left) override {
+    return CompareValues<Int64Array>(left);
+  }
+  Status Visit(const UInt8Array& left) override {
+    return CompareValues<UInt8Array>(left);
+  }
+  Status Visit(const UInt16Array& left) override {
+    return CompareValues<UInt16Array>(left);
+  }
+  Status Visit(const UInt32Array& left) override {
+    return CompareValues<UInt32Array>(left);
+  }
+  Status Visit(const UInt64Array& left) override {
+    return CompareValues<UInt64Array>(left);
+  }
+  Status Visit(const FloatArray& left) override {
+    return CompareValues<FloatArray>(left);
+  }
+  Status Visit(const DoubleArray& left) override {
+    return CompareValues<DoubleArray>(left);
+  }
+
+  Status Visit(const HalfFloatArray& left) override {
+    return Status::NotImplemented("Half float type");
+  }
+
+  Status Visit(const StringArray& left) override {
+    result_ = CompareBinaryRange(left);
+    return Status::OK();
+  }
+
+  Status Visit(const BinaryArray& left) override {
+    result_ = CompareBinaryRange(left);
+    return Status::OK();
+  }
+
+  Status Visit(const DateArray& left) override { return CompareValues<DateArray>(left); }
+
+  Status Visit(const TimeArray& left) override { return CompareValues<TimeArray>(left); }
+
+  Status Visit(const TimestampArray& left) override {
+    return CompareValues<TimestampArray>(left);
+  }
+
+  Status Visit(const IntervalArray& left) override {
+    return CompareValues<IntervalArray>(left);
+  }
+
+  Status Visit(const DecimalArray& left) override {
+    return Status::NotImplemented("Decimal type");
+  }
+
+  bool CompareLists(const ListArray& left) {
+    const auto& right = static_cast<const ListArray&>(right_);
+
+    const std::shared_ptr<Array>& left_values = left.values();
+    const std::shared_ptr<Array>& right_values = right.values();
+
+    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+         ++i, ++o_i) {
+      const bool is_null = left.IsNull(i);
+      if (is_null != right.IsNull(o_i)) { return false; }
+      if (is_null) continue;
+      const int32_t begin_offset = left.offset(i);
+      const int32_t end_offset = left.offset(i + 1);
+      const int32_t right_begin_offset = right.offset(o_i);
+      const int32_t right_end_offset = right.offset(o_i + 1);
+      // Underlying can't be equal if the size isn't equal
+      if (end_offset - begin_offset != right_end_offset - right_begin_offset) {
+        return false;
+      }
+      if (!left_values->RangeEquals(
+              begin_offset, end_offset, right_begin_offset, right_values)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  Status Visit(const ListArray& left) override {
+    result_ = CompareLists(left);
+    return Status::OK();
+  }
+
+  bool CompareStructs(const StructArray& left) {
+    const auto& right = static_cast<const StructArray&>(right_);
+    bool equal_fields = true;
+    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+         ++i, ++o_i) {
+      if (left.IsNull(i) != right.IsNull(o_i)) { return false; }
+      if (left.IsNull(i)) continue;
+      for (size_t j = 0; j < left.fields().size(); ++j) {
+        // TODO: really we should be comparing stretches of non-null data rather
+        // than looking at one value at a time.
+        equal_fields = left.field(j)->RangeEquals(i, i + 1, o_i, right.field(j));
+        if (!equal_fields) { return false; }
+      }
+    }
+    return true;
+  }
+
+  Status Visit(const StructArray& left) override {
+    result_ = CompareStructs(left);
+    return Status::OK();
+  }
+
+  bool CompareUnions(const UnionArray& left) const {
+    const auto& right = static_cast<const UnionArray&>(right_);
+
+    const UnionMode union_mode = left.mode();
+    if (union_mode != right.mode()) { return false; }
+
+    const auto& left_type = static_cast<const UnionType&>(*left.type());
+
+    // Define a mapping from the type id to child number
+    uint8_t max_code = 0;
+
+    const std::vector<uint8_t> type_codes = left_type.type_ids;
+    for (size_t i = 0; i < type_codes.size(); ++i) {
+      const uint8_t code = type_codes[i];
+      if (code > max_code) { max_code = code; }
+    }
+
+    // Store mapping in a vector for constant time lookups
+    std::vector<uint8_t> type_id_to_child_num(max_code + 1);
+    for (uint8_t i = 0; i < static_cast<uint8_t>(type_codes.size()); ++i) {
+      type_id_to_child_num[type_codes[i]] = i;
+    }
+
+    const uint8_t* left_ids = left.raw_type_ids();
+    const uint8_t* right_ids = right.raw_type_ids();
+
+    uint8_t id, child_num;
+    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+         ++i, ++o_i) {
+      if (left.IsNull(i) != right.IsNull(o_i)) { return false; }
+      if (left.IsNull(i)) continue;
+      if (left_ids[i] != right_ids[o_i]) { return false; }
+
+      id = left_ids[i];
+      child_num = type_id_to_child_num[id];
+
+      // TODO(wesm): really we should be comparing stretches of non-null data
+      // rather than looking at one value at a time.
+      if (union_mode == UnionMode::SPARSE) {
+        if (!left.child(child_num)->RangeEquals(i, i + 1, o_i, right.child(child_num))) {
+          return false;
+        }
+      } else {
+        const int32_t offset = left.raw_offsets()[i];
+        const int32_t o_offset = right.raw_offsets()[i];
+        if (!left.child(child_num)->RangeEquals(
+                offset, offset + 1, o_offset, right.child(child_num))) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  Status Visit(const UnionArray& left) override {
+    result_ = CompareUnions(left);
+    return Status::OK();
+  }
+
+  Status Visit(const DictionaryArray& left) override {
+    const auto& right = static_cast<const DictionaryArray&>(right_);
+    if (!left.dictionary()->Equals(right.dictionary())) {
+      result_ = false;
+      return Status::OK();
+    }
+    result_ = left.indices()->RangeEquals(
+        left_start_idx_, left_end_idx_, right_start_idx_, right.indices());
+    return Status::OK();
+  }
+
+  bool result() const { return result_; }
+
+ protected:
+  const Array& right_;
+  int32_t left_start_idx_;
+  int32_t left_end_idx_;
+  int32_t right_start_idx_;
+
+  bool result_;
+};
+
+class EqualsVisitor : public RangeEqualsVisitor {
+ public:
+  explicit EqualsVisitor(const Array& right)
+      : RangeEqualsVisitor(right, 0, right.length(), 0) {}
+
+  Status Visit(const NullArray& left) override { return Status::OK(); }
+
+  Status Visit(const BooleanArray& left) override {
+    const auto& right = static_cast<const BooleanArray&>(right_);
+    if (left.null_count() > 0) {
+      const uint8_t* left_data = left.data()->data();
+      const uint8_t* right_data = right.data()->data();
+
+      for (int i = 0; i < left.length(); ++i) {
+        if (!left.IsNull(i) &&
+            BitUtil::GetBit(left_data, i) != BitUtil::GetBit(right_data, i)) {
+          result_ = false;
+          return Status::OK();
+        }
+      }
+      result_ = true;
+    } else {
+      result_ = left.data()->Equals(*right.data(), BitUtil::BytesForBits(left.length()));
+    }
+    return Status::OK();
+  }
+
+  bool IsEqualPrimitive(const PrimitiveArray& left) {
+    const auto& right = static_cast<const PrimitiveArray&>(right_);
+    if (left.null_count() > 0) {
+      const uint8_t* left_data = left.data()->data();
+      const uint8_t* right_data = right.data()->data();
+      const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
+      const int value_byte_size = size_meta.bit_width() / 8;
+      DCHECK_GT(value_byte_size, 0);
+
+      for (int i = 0; i < left.length(); ++i) {
+        if (!left.IsNull(i) && memcmp(left_data, right_data, value_byte_size)) {
+          return false;
+        }
+        left_data += value_byte_size;
+        right_data += value_byte_size;
+      }
+      return true;
+    } else {
+      if (left.length() == 0) { return true; }
+      return left.data()->Equals(*right.data(), left.length());
+    }
+  }
+
+  Status ComparePrimitive(const PrimitiveArray& left) {
+    result_ = IsEqualPrimitive(left);
+    return Status::OK();
+  }
+
+  Status Visit(const Int8Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const Int16Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const Int32Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const Int64Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const UInt8Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const UInt16Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const UInt32Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const UInt64Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const FloatArray& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const DoubleArray& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const DateArray& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const TimeArray& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const TimestampArray& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const IntervalArray& left) override { return ComparePrimitive(left); }
+
+  bool CompareBinary(const BinaryArray& left) {
+    const auto& right = static_cast<const BinaryArray&>(right_);
+    bool equal_offsets =
+        left.offsets()->Equals(*right.offsets(), (left.length() + 1) * sizeof(int32_t));
+    if (!equal_offsets) { return false; }
+    if (!left.data() && !(right.data())) { return true; }
+    return left.data()->Equals(*right.data(), left.raw_offsets()[left.length()]);
+  }
+
+  Status Visit(const StringArray& left) override {
+    result_ = CompareBinary(left);
+    return Status::OK();
+  }
+
+  Status Visit(const BinaryArray& left) override {
+    result_ = CompareBinary(left);
+    return Status::OK();
+  }
+
+  Status Visit(const ListArray& left) override {
+    const auto& right = static_cast<const ListArray&>(right_);
+    if (!left.offsets()->Equals(
+            *right.offsets(), (left.length() + 1) * sizeof(int32_t))) {
+      result_ = false;
+    } else {
+      result_ = left.values()->Equals(right.values());
+    }
+    return Status::OK();
+  }
+
+  Status Visit(const DictionaryArray& left) override {
+    const auto& right = static_cast<const DictionaryArray&>(right_);
+    if (!left.dictionary()->Equals(right.dictionary())) {
+      result_ = false;
+    } else {
+      result_ = left.indices()->Equals(right.indices());
+    }
+    return Status::OK();
+  }
+};
+
+template <typename TYPE>
+inline bool FloatingApproxEquals(
+    const NumericArray<TYPE>& left, const NumericArray<TYPE>& right) {
+  using T = typename TYPE::c_type;
+
+  auto left_data = reinterpret_cast<const T*>(left.data()->data());
+  auto right_data = reinterpret_cast<const T*>(right.data()->data());
+
+  static constexpr T EPSILON = 1E-5;
+
+  if (left.length() == 0 && right.length() == 0) { return true; }
+
+  if (left.null_count() > 0) {
+    for (int32_t i = 0; i < left.length(); ++i) {
+      if (left.IsNull(i)) continue;
+      if (fabs(left_data[i] - right_data[i]) > EPSILON) { return false; }
+    }
+  } else {
+    for (int32_t i = 0; i < left.length(); ++i) {
+      if (fabs(left_data[i] - right_data[i]) > EPSILON) { return false; }
+    }
+  }
+  return true;
+}
+
+class ApproxEqualsVisitor : public EqualsVisitor {
+ public:
+  using EqualsVisitor::EqualsVisitor;
+
+  Status Visit(const FloatArray& left) override {
+    result_ =
+        FloatingApproxEquals<FloatType>(left, static_cast<const FloatArray&>(right_));
+    return Status::OK();
+  }
+
+  Status Visit(const DoubleArray& left) override {
+    result_ =
+        FloatingApproxEquals<DoubleType>(left, static_cast<const DoubleArray&>(right_));
+    return Status::OK();
+  }
+};
+
+static bool BaseDataEquals(const Array& left, const Array& right) {
+  if (left.length() != right.length() || left.null_count() != right.null_count() ||
+      left.type_enum() != right.type_enum()) {
+    return false;
+  }
+  if (left.null_count() > 0) {
+    return left.null_bitmap()->Equals(
+        *right.null_bitmap(), BitUtil::BytesForBits(left.length()));
+  }
+  return true;
+}
+
+Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
+  // The arrays are the same object
+  if (&left == &right) {
+    *are_equal = true;
+  } else if (!BaseDataEquals(left, right)) {
+    *are_equal = false;
+  } else {
+    EqualsVisitor visitor(right);
+    RETURN_NOT_OK(left.Accept(&visitor));
+    *are_equal = visitor.result();
+  }
+  return Status::OK();
+}
+
+Status ArrayRangeEquals(const Array& left, const Array& right, int32_t left_start_idx,
+    int32_t left_end_idx, int32_t right_start_idx, bool* are_equal) {
+  if (&left == &right) {
+    *are_equal = true;
+  } else if (left.type_enum() != right.type_enum()) {
+    *are_equal = false;
+  } else {
+    RangeEqualsVisitor visitor(right, left_start_idx, left_end_idx, right_start_idx);
+    RETURN_NOT_OK(left.Accept(&visitor));
+    *are_equal = visitor.result();
+  }
+  return Status::OK();
+}
+
+Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal) {
+  // The arrays are the same object
+  if (&left == &right) {
+    *are_equal = true;
+  } else if (!BaseDataEquals(left, right)) {
+    *are_equal = false;
+  } else {
+    ApproxEqualsVisitor visitor(right);
+    RETURN_NOT_OK(left.Accept(&visitor));
+    *are_equal = visitor.result();
+  }
+  return Status::OK();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
new file mode 100644
index 0000000000000..2093b65a51a13
--- /dev/null
+++ b/cpp/src/arrow/compare.h
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Functions for comparing Arrow data structures
+
+#ifndef ARROW_COMPARE_H
+#define ARROW_COMPARE_H
+
+#include <cstdint>
+
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+class Status;
+
+/// Returns true if the arrays are exactly equal
+Status ARROW_EXPORT ArrayEquals(const Array& left, const Array& right, bool* are_equal);
+
+/// Returns true if the arrays are approximately equal. For non-floating point
+/// types, this is equivalent to ArrayEquals(left, right)
+Status ARROW_EXPORT ArrayApproxEquals(
+    const Array& left, const Array& right, bool* are_equal);
+
+/// Returns true if indicated equal-length segment of arrays is exactly equal
+Status ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right,
+    int32_t start_idx, int32_t end_idx, int32_t other_start_idx, bool* are_equal);
+
+}  // namespace arrow
+
+#endif  // ARROW_COMPARE_H
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index e2bb355115b42..c4a62a475b92f 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -25,4 +25,6 @@
   TypeName& operator=(const TypeName&) = delete
 #endif
 
+#define UNUSED(x) (void)x
+
 #endif  // ARROW_UTIL_MACROS_H
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index d63fff48a011f..942e74b73aaee 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -74,6 +74,9 @@ include(SetupCxxFlags)
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
 
+# Enable perf and other tools to work properly
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
+
 # Suppress Cython warnings
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
 
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index feafa3dfc3875..920779fe86174 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -153,14 +153,6 @@ static inline bool PyObject_is_string(const PyObject* obj) {
 #endif
 }
 
-static inline bool PyObject_is_bool(const PyObject* obj) {
-#if PY_MAJOR_VERSION >= 3
-  return PyString_Check(obj) || PyBytes_Check(obj);
-#else
-  return PyString_Check(obj) || PyUnicode_Check(obj);
-#endif
-}
-
 template <int TYPE>
 static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap) {
   typedef npy_traits<TYPE> traits;

From be5d73f2cbcbd4c3f4e0a8ba41f69222ecedfc05 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 31 Jan 2017 08:00:49 -0500
Subject: [PATCH 0301/1644] ARROW-410: [C++] Add virtual Writeable::Flush

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #310 from wesm/ARROW-410 and squashes the following commits:

7352f0a [Wes McKinney] Add virtual Writeable::Flush, and move HDFS flush there
---
 cpp/src/arrow/io/hdfs.cc       | 15 ++++++++++++---
 cpp/src/arrow/io/hdfs.h        |  2 ++
 cpp/src/arrow/io/interfaces.cc |  4 ++++
 cpp/src/arrow/io/interfaces.h  |  3 +++
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 44e503ff11302..2845b0d8c448c 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -238,15 +238,20 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
 
   Status Close() {
     if (is_open_) {
-      int ret = driver_->Flush(fs_, file_);
-      CHECK_FAILURE(ret, "Flush");
-      ret = driver_->CloseFile(fs_, file_);
+      RETURN_NOT_OK(Flush());
+      int ret = driver_->CloseFile(fs_, file_);
       CHECK_FAILURE(ret, "CloseFile");
       is_open_ = false;
     }
     return Status::OK();
   }
 
+  Status Flush() {
+    int ret = driver_->Flush(fs_, file_);
+    CHECK_FAILURE(ret, "Flush");
+    return Status::OK();
+  }
+
   Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
     tSize ret = driver_->Write(fs_, file_, reinterpret_cast<const void*>(buffer), nbytes);
     CHECK_FAILURE(ret, "Write");
@@ -277,6 +282,10 @@ Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes) {
   return Write(buffer, nbytes, &bytes_written_dummy);
 }
 
+Status HdfsOutputStream::Flush() {
+  return impl_->Flush();
+}
+
 Status HdfsOutputStream::Tell(int64_t* position) {
   return impl_->Tell(position);
 }
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 5cc783e475967..fbf1d758afb99 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -208,6 +208,8 @@ class ARROW_EXPORT HdfsOutputStream : public OutputStream {
 
   Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written);
 
+  Status Flush() override;
+
   Status Tell(int64_t* position) override;
 
  private:
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 7e78caa04e711..51ed0693e2c57 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -52,5 +52,9 @@ Status Writeable::Write(const std::string& data) {
       reinterpret_cast<const uint8_t*>(data.c_str()), static_cast<int64_t>(data.size()));
 }
 
+Status Writeable::Flush() {
+  return Status::OK();
+}
+
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index e9f07f03a1419..9862a67aed0cd 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -73,6 +73,9 @@ class ARROW_EXPORT Writeable {
  public:
   virtual Status Write(const uint8_t* data, int64_t nbytes) = 0;
 
+  // Default implementation is a no-op
+  virtual Status Flush();
+
   Status Write(const std::string& data);
 };
 

From 0ae4d86e5ef8ee53a8810f4324dce80ec6a9d422 Mon Sep 17 00:00:00 2001
From: Nong Li <nongli@gmail.com>
Date: Thu, 2 Feb 2017 14:36:23 +0100
Subject: [PATCH 0302/1644] ARROW-497: Integration harness for streaming file
 format

These tests pass locally for me. Thanks @nongli for this!

Author: Nong Li <nongli@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #312 from wesm/streaming-integration and squashes the following commits:

8b9ad76 [Wes McKinney] Hook stream<->file tools together and get integration tests working. Quiet test output in TestArrowStreamPipe
c7f0483 [Nong Li] ARROW-XXX: [Java] Add file <=> stream utility tools.
---
 ci/travis_script_integration.sh               |  3 +
 integration/integration_test.py               | 76 ++++++++++++++++---
 .../org/apache/arrow/tools/FileToStream.java  | 68 +++++++++++++++++
 .../org/apache/arrow/tools/StreamToFile.java  | 61 +++++++++++++++
 .../vector/stream/MessageSerializer.java      |  2 +-
 .../vector/stream/TestArrowStreamPipe.java    |  2 +-
 6 files changed, 198 insertions(+), 14 deletions(-)
 create mode 100644 java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
 create mode 100644 java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java

diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index d93411b907d47..c019a4b7ab7ff 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -28,7 +28,10 @@ pushd $TRAVIS_BUILD_DIR/integration
 
 VERSION=0.1.1-SNAPSHOT
 export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
+
 export ARROW_CPP_TESTER=$CPP_BUILD_DIR/debug/json-integration-test
+export ARROW_CPP_STREAM_TO_FILE=$CPP_BUILD_DIR/debug/stream-to-file
+export ARROW_CPP_FILE_TO_STREAM=$CPP_BUILD_DIR/debug/file-to-stream
 
 source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 export MINICONDA=$HOME/miniconda
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 77510daecc0b4..a622bf228a651 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -556,12 +556,25 @@ def run(self):
                                                            consumer.name))
 
             for json_path in self.json_files:
-                print('Testing with {0}'.format(json_path))
+                print('Testing file {0}'.format(json_path))
 
-                arrow_path = os.path.join(self.temp_dir, guid())
+                # Make the random access file
+                print('-- Creating binary inputs')
+                producer_file_path = os.path.join(self.temp_dir, guid())
+                producer.json_to_file(json_path, producer_file_path)
 
-                producer.json_to_arrow(json_path, arrow_path)
-                consumer.validate(json_path, arrow_path)
+                # Validate the file
+                print('-- Validating file')
+                consumer.validate(json_path, producer_file_path)
+
+                print('-- Validating stream')
+                producer_stream_path = os.path.join(self.temp_dir, guid())
+                consumer_file_path = os.path.join(self.temp_dir, guid())
+                producer.file_to_stream(producer_file_path,
+                                        producer_stream_path)
+                consumer.stream_to_file(producer_stream_path,
+                                        consumer_file_path)
+                consumer.validate(json_path, consumer_file_path)
 
 
 class Tester(object):
@@ -569,7 +582,13 @@ class Tester(object):
     def __init__(self, debug=False):
         self.debug = debug
 
-    def json_to_arrow(self, json_path, arrow_path):
+    def json_to_file(self, json_path, arrow_path):
+        raise NotImplementedError
+
+    def stream_to_file(self, stream_path, file_path):
+        raise NotImplementedError
+
+    def file_to_stream(self, file_path, stream_path):
         raise NotImplementedError
 
     def validate(self, json_path, arrow_path):
@@ -601,21 +620,40 @@ def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
         if self.debug:
             print(' '.join(cmd))
 
-        return run_cmd(cmd)
+        run_cmd(cmd)
 
     def validate(self, json_path, arrow_path):
         return self._run(arrow_path, json_path, 'VALIDATE')
 
-    def json_to_arrow(self, json_path, arrow_path):
+    def json_to_file(self, json_path, arrow_path):
         return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
 
+    def stream_to_file(self, stream_path, file_path):
+        cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
+               'org.apache.arrow.tools.StreamToFile',
+               stream_path, file_path]
+        run_cmd(cmd)
+
+    def file_to_stream(self, file_path, stream_path):
+        cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
+               'org.apache.arrow.tools.FileToStream',
+               file_path, stream_path]
+        run_cmd(cmd)
+
 
 class CPPTester(Tester):
 
+    BUILD_PATH = os.path.join(ARROW_HOME, 'cpp/test-build/debug')
     CPP_INTEGRATION_EXE = os.environ.get(
-        'ARROW_CPP_TESTER',
-        os.path.join(ARROW_HOME,
-                     'cpp/test-build/debug/json-integration-test'))
+        'ARROW_CPP_TESTER', os.path.join(BUILD_PATH, 'json-integration-test'))
+
+    STREAM_TO_FILE = os.environ.get(
+        'ARROW_CPP_STREAM_TO_FILE',
+        os.path.join(BUILD_PATH, 'stream-to-file'))
+
+    FILE_TO_STREAM = os.environ.get(
+        'ARROW_CPP_FILE_TO_STREAM',
+        os.path.join(BUILD_PATH, 'file-to-stream'))
 
     name = 'C++'
 
@@ -633,14 +671,28 @@ def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
         if self.debug:
             print(' '.join(cmd))
 
-        return run_cmd(cmd)
+        run_cmd(cmd)
 
     def validate(self, json_path, arrow_path):
         return self._run(arrow_path, json_path, 'VALIDATE')
 
-    def json_to_arrow(self, json_path, arrow_path):
+    def json_to_file(self, json_path, arrow_path):
         return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
 
+    def stream_to_file(self, stream_path, file_path):
+        cmd = ['cat', stream_path, '|', self.STREAM_TO_FILE, '>', file_path]
+        cmd = ' '.join(cmd)
+        if self.debug:
+            print(cmd)
+        os.system(cmd)
+
+    def file_to_stream(self, file_path, stream_path):
+        cmd = [self.FILE_TO_STREAM, file_path, '>', stream_path]
+        cmd = ' '.join(cmd)
+        if self.debug:
+            print(cmd)
+        os.system(cmd)
+
 
 def get_static_json_files():
     glob_pattern = os.path.join(ARROW_HOME, 'integration', 'data', '*.json')
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
new file mode 100644
index 0000000000000..ba6505cb48d08
--- /dev/null
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.tools;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.file.ArrowBlock;
+import org.apache.arrow.vector.file.ArrowFooter;
+import org.apache.arrow.vector.file.ArrowReader;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.ArrowStreamWriter;
+
+/**
+ * Converts an Arrow file to an Arrow stream. The file should be specified as the
+ * first argument and the output is written to standard out.
+ */
+public class FileToStream {
+  public static void convert(FileInputStream in, OutputStream out) throws IOException {
+    BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+    try(
+        ArrowReader reader = new ArrowReader(in.getChannel(), allocator);) {
+      ArrowFooter footer = reader.readFooter();
+      try (
+        ArrowStreamWriter writer = new ArrowStreamWriter(out, footer.getSchema());
+      ) {
+        for (ArrowBlock block: footer.getRecordBatches()) {
+          try (ArrowRecordBatch batch = reader.readRecordBatch(block)) {
+            writer.writeRecordBatch(batch);
+          }
+        }
+      }
+    }
+  }
+
+  public static void main(String[] args) throws IOException {
+    if (args.length != 1 && args.length != 2) {
+      System.err.println("Usage: FileToStream <input file> [output file]");
+      System.exit(1);
+    }
+
+    FileInputStream in = new FileInputStream(new File(args[0]));
+    OutputStream out = args.length == 1 ?
+        System.out : new FileOutputStream(new File(args[1]));
+
+    convert(in, out);
+  }
+}
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
new file mode 100644
index 0000000000000..c8a5c8914afcc
--- /dev/null
+++ b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.tools;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.Channels;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.file.ArrowWriter;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.ArrowStreamReader;
+
+/**
+ * Converts an Arrow stream to an Arrow file.
+ */
+public class StreamToFile {
+  public static void convert(InputStream in, OutputStream out) throws IOException {
+    BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+    try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) {
+      reader.init();
+      try (ArrowWriter writer = new ArrowWriter(Channels.newChannel(out), reader.getSchema());) {
+        while (true) {
+          ArrowRecordBatch batch = reader.nextRecordBatch();
+          if (batch == null) break;
+          writer.writeRecordBatch(batch);
+        }
+      }
+    }
+  }
+
+  public static void main(String[] args) throws IOException {
+    InputStream in = System.in;
+    OutputStream out = System.out;
+    if (args.length == 2) {
+      in = new FileInputStream(new File(args[0]));
+      out = new FileOutputStream(new File(args[1]));
+    }
+    convert(in, out);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 7ab740c70782e..92df2504bcb23 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -226,7 +226,7 @@ private static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte heade
     Message.startMessage(builder);
     Message.addHeaderType(builder, headerType);
     Message.addHeader(builder, headerOffset);
-    Message.addVersion(builder, MetadataVersion.V1);
+    Message.addVersion(builder, MetadataVersion.V2);
     Message.addBodyLength(builder, bodyLength);
     builder.finish(Message.endMessage(builder));
     return builder.dataBuffer();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
index a0a7ffa279308..aa0b77e46a392 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
@@ -113,7 +113,7 @@ public void run() {
   // Starts up a producer and consumer thread to read/write batches.
   @Test
   public void pipeTest() throws IOException, InterruptedException {
-    int NUM_BATCHES = 1000;
+    int NUM_BATCHES = 10;
     Pipe pipe = Pipe.open();
     WriterThread writer = new WriterThread(NUM_BATCHES, pipe.sink());
     ReaderThread reader = new ReaderThread(pipe.source());

From c05292faf74111e826dbbafe9a1e108346eb10dc Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 2 Feb 2017 16:13:40 -0500
Subject: [PATCH 0303/1644] ARROW-523: Python: Account for changes in
 PARQUET-834

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #313 from xhochy/ARROW-523 and squashes the following commits:

ff699ea [Uwe L. Korn] Use relative import
e36dcc8 [Uwe L. Korn] ARROW-523: Python: Account for changes in PARQUET-834
---
 python/pyarrow/_parquet.pxd                 |  8 +--
 python/pyarrow/_parquet.pyx                 |  8 +--
 python/pyarrow/tests/pandas_examples.py     | 78 +++++++++++++++++++++
 python/pyarrow/tests/test_convert_pandas.py | 47 +------------
 python/pyarrow/tests/test_parquet.py        | 11 +++
 5 files changed, 100 insertions(+), 52 deletions(-)
 create mode 100644 python/pyarrow/tests/pandas_examples.py

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index fabee5d5761d7..6b9350ad6782a 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -211,9 +211,9 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 
     cdef cppclass FileReader:
         FileReader(MemoryPool* pool, unique_ptr[ParquetFileReader] reader)
-        CStatus ReadFlatColumn(int i, shared_ptr[CArray]* out);
-        CStatus ReadFlatTable(shared_ptr[CTable]* out);
-        CStatus ReadFlatTable(const vector[int]& column_indices,
+        CStatus ReadColumn(int i, shared_ptr[CArray]* out);
+        CStatus ReadTable(shared_ptr[CTable]* out);
+        CStatus ReadTable(const vector[int]& column_indices,
                               shared_ptr[CTable]* out);
         const ParquetFileReader* parquet_reader();
 
@@ -228,7 +228,7 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
 
 
 cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
-    cdef CStatus WriteFlatTable(
+    cdef CStatus WriteTable(
         const CTable* table, MemoryPool* pool,
         const shared_ptr[OutputStream]& sink,
         int64_t chunk_size,
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 3f847e9808230..fd4670a00c837 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -397,12 +397,12 @@ cdef class ParquetReader:
 
             with nogil:
                 check_status(self.reader.get()
-                             .ReadFlatTable(c_column_indices, &ctable))
+                             .ReadTable(c_column_indices, &ctable))
         else:
             # Read all columns
             with nogil:
                 check_status(self.reader.get()
-                             .ReadFlatTable(&ctable))
+                             .ReadTable(&ctable))
 
         table.init(ctable)
         return table
@@ -442,7 +442,7 @@ cdef class ParquetReader:
 
         with nogil:
             check_status(self.reader.get()
-                         .ReadFlatColumn(column_index, &carray))
+                         .ReadColumn(column_index, &carray))
 
         array.init(carray)
         return array
@@ -540,6 +540,6 @@ cdef class ParquetWriter:
 
         cdef int c_row_group_size = row_group_size
         with nogil:
-            check_status(WriteFlatTable(ctable, default_memory_pool(),
+            check_status(WriteTable(ctable, default_memory_pool(),
                                         self.sink, c_row_group_size,
                                         self.properties))
diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py
new file mode 100644
index 0000000000000..63af42348026c
--- /dev/null
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections import OrderedDict
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+
+
+def dataframe_with_arrays():
+    """
+    Dataframe with numpy arrays columns of every possible primtive type.
+
+    Returns
+    -------
+    df: pandas.DataFrame
+    schema: pyarrow.Schema
+        Arrow schema definition that is in line with the constructed df.
+    """
+    dtypes = [('i1', pa.int8()), ('i2', pa.int16()),
+              ('i4', pa.int32()), ('i8', pa.int64()),
+              ('u1', pa.uint8()), ('u2', pa.uint16()),
+              ('u4', pa.uint32()), ('u8', pa.uint64()),
+              ('f4', pa.float_()), ('f8', pa.double())]
+
+    arrays = OrderedDict()
+    fields = []
+    for dtype, arrow_dtype in dtypes:
+        fields.append(pa.field(dtype, pa.list_(arrow_dtype)))
+        arrays[dtype] = [
+            np.arange(10, dtype=dtype),
+            np.arange(5, dtype=dtype),
+            None,
+            np.arange(1, dtype=dtype)
+        ]
+
+    fields.append(pa.field('str', pa.list_(pa.string())))
+    arrays['str'] = [
+        np.array([u"1", u"ä"], dtype="object"),
+        None,
+        np.array([u"1"], dtype="object"),
+        np.array([u"1", u"2", u"3"], dtype="object")
+    ]
+
+    fields.append(pa.field('datetime64', pa.list_(pa.timestamp('ms'))))
+    arrays['datetime64'] = [
+        np.array(['2007-07-13T01:23:34.123456789',
+                  None,
+                  '2010-08-13T05:46:57.437699912'],
+                 dtype='datetime64[ms]'),
+        None,
+        None,
+        np.array(['2007-07-13T02',
+                  None,
+                  '2010-08-13T05:46:57.437699912'],
+                 dtype='datetime64[ms]'),
+    ]
+
+    df = pd.DataFrame(arrays)
+    schema = pa.Schema.from_fields(fields)
+
+    return df, schema
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 674a4361d3395..ddbb02a770c35 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -29,6 +29,8 @@
 from pyarrow.compat import u
 import pyarrow as A
 
+from .pandas_examples import dataframe_with_arrays
+
 
 def _alltypes_example(size=100):
     return pd.DataFrame({
@@ -325,54 +327,11 @@ def test_date(self):
         tm.assert_frame_equal(result, expected)
 
     def test_column_of_lists(self):
-        dtypes = [('i1', A.int8()), ('i2', A.int16()),
-                  ('i4', A.int32()), ('i8', A.int64()),
-                  ('u1', A.uint8()), ('u2', A.uint16()),
-                  ('u4', A.uint32()), ('u8', A.uint64()),
-                  ('f4', A.float_()), ('f8', A.double())]
-
-        arrays = OrderedDict()
-        fields = []
-        for dtype, arrow_dtype in dtypes:
-            fields.append(A.field(dtype, A.list_(arrow_dtype)))
-            arrays[dtype] = [
-                np.arange(10, dtype=dtype),
-                np.arange(5, dtype=dtype),
-                None,
-                np.arange(1, dtype=dtype)
-            ]
-
-        fields.append(A.field('str', A.list_(A.string())))
-        arrays['str'] = [
-            np.array([u"1", u"ä"], dtype="object"),
-            None,
-            np.array([u"1"], dtype="object"),
-            np.array([u"1", u"2", u"3"], dtype="object")
-        ]
-
-        fields.append(A.field('datetime64', A.list_(A.timestamp('ns'))))
-        arrays['datetime64'] = [
-            np.array(['2007-07-13T01:23:34.123456789',
-                      None,
-                      '2010-08-13T05:46:57.437699912'],
-                     dtype='datetime64[ns]'),
-            None,
-            None,
-            np.array(['2007-07-13T02',
-                      None,
-                      '2010-08-13T05:46:57.437699912'],
-                     dtype='datetime64[ns]'),
-        ]
-
-        df = pd.DataFrame(arrays)
-        schema = A.Schema.from_fields(fields)
+        df, schema = dataframe_with_arrays()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
         table = A.Table.from_pandas(df, schema=schema)
         assert table.schema.equals(schema)
 
-        # it works!
-        table.to_pandas(nthreads=1)
-
     def test_threaded_conversion(self):
         df = _alltypes_example()
         self._check_pandas_roundtrip(df, nthreads=2,
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index d85f0e513702f..80a995fbb6662 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -23,6 +23,7 @@
 from pyarrow.compat import guid
 import pyarrow as pa
 import pyarrow.io as paio
+from .pandas_examples import dataframe_with_arrays
 
 import numpy as np
 import pandas as pd
@@ -319,6 +320,16 @@ def test_compare_schemas():
     assert fileh.schema[0].equals(fileh.schema[0])
     assert not fileh.schema[0].equals(fileh.schema[1])
 
+@parquet
+def test_column_of_lists(tmpdir):
+    df, schema = dataframe_with_arrays()
+
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True, schema=schema)
+    pq.write_table(arrow_table, filename.strpath, version="2.0")
+    table_read = pq.read_table(filename.strpath)
+    df_read = table_read.to_pandas()
+    pdt.assert_frame_equal(df, df_read)
 
 @parquet
 def test_multithreaded_read():

From 720d422fa761e2beab1b412b1b42c041ac2db1a4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 3 Feb 2017 09:08:14 +0100
Subject: [PATCH 0304/1644] ARROW-467: [Python] Run Python parquet-cpp unit
 tests in Travis CI

This means we'll have to tolerate broken builds whenever APIs change (a good incentive to avoid changing them as much as possible)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #311 from wesm/ARROW-467 and squashes the following commits:

a9c285d [Wes McKinney] parquet-cpp build tweaks
661671c [Wes McKinney] Build parquet-cpp from source and run PyArrow Parquet unit tests in Travis CI
---
 ci/travis_script_python.sh | 50 +++++++++++++++++++++++++++++++++++---
 1 file changed, 46 insertions(+), 4 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 179567b595416..c186fd4639fca 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -26,12 +26,52 @@ export ARROW_HOME=$ARROW_CPP_INSTALL
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
 
 pushd $PYTHON_DIR
+export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
+
+build_parquet_cpp() {
+  conda create -y -q -p $PARQUET_HOME thrift-cpp snappy zlib brotli boost
+  source activate $PARQUET_HOME
+
+  export BOOST_ROOT=$PARQUET_HOME
+  export SNAPPY_HOME=$PARQUET_HOME
+  export THRIFT_HOME=$PARQUET_HOME
+  export ZLIB_HOME=$PARQUET_HOME
+  export BROTLI_HOME=$PARQUET_HOME
+
+  PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
+  mkdir -p $PARQUET_DIR
+
+  git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
+
+  pushd $PARQUET_DIR
+  mkdir build-dir
+  cd build-dir
+
+  cmake \
+      -DCMAKE_BUILD_TYPE=debug \
+      -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+      -DPARQUET_ARROW=on \
+      -DPARQUET_BUILD_BENCHMARKS=off \
+      -DPARQUET_BUILD_EXECUTABLES=off \
+      -DPARQUET_ZLIB_VENDORED=off \
+      -DPARQUET_BUILD_TESTS=off \
+      ..
+
+  make -j${CPU_COUNT}
+  make install
+
+  popd
+}
+
+build_parquet_cpp
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PARQUET_HOME/lib
 
 python_version_tests() {
   PYTHON_VERSION=$1
-  CONDA_ENV_NAME="pyarrow-test-${PYTHON_VERSION}"
-  conda create -y -q -n $CONDA_ENV_NAME python=$PYTHON_VERSION
-  source activate $CONDA_ENV_NAME
+  CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
+  conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION
+  source activate $CONDA_ENV_DIR
 
   python --version
   which python
@@ -45,7 +85,9 @@ python_version_tests() {
   # Other stuff pip install
   pip install -r requirements.txt
 
-  python setup.py build_ext --inplace
+  python setup.py build_ext --inplace --with-parquet
+
+  python -c "import pyarrow.parquet"
 
   python -m pytest -vv -r sxX pyarrow
 

From 08f38d97904e8d265dea09cdc67946119998e039 Mon Sep 17 00:00:00 2001
From: Jingyuan Wang <jingyuan@live.com>
Date: Fri, 3 Feb 2017 23:00:25 -0500
Subject: [PATCH 0305/1644] ARROW-477: [Java] Add support for
 second/microsecond/nanosecond timestamps in-memory and in IPC/JSON layer

Changes include:
- add support for TimeStamp data type with second/microsecond/nanosecond time units
- add an additional readLong() method to timestamp readers to support reading raw long values
- add a simple test case for timestamp readers and writers

Author: Jingyuan Wang <jingyuan@live.com>

Closes #303 from alphalfalfa/arrow-477 and squashes the following commits:

0199574 [Jingyuan Wang] rename TimeStamp to TimeStampMilli
068e47f [Jingyuan Wang] use a test value that exhibits micro/nanosecond truncation when converting timestamps to JODA DateTime
bef2330 [Jingyuan Wang] fix a typo
9b4d7b4 [Jingyuan Wang] add support for timestamp data type with second/microsecond/nanosecond time units
---
 .../main/codegen/data/ValueVectorTypes.tdd    |  5 +-
 .../codegen/templates/ComplexReaders.java     | 10 +++
 .../codegen/templates/FixedValueVectors.java  | 31 ++++++-
 .../templates/NullableValueVectors.java       |  8 +-
 .../vector/file/json/JsonFileReader.java      | 18 +++-
 .../vector/file/json/JsonFileWriter.java      | 18 +++-
 .../org/apache/arrow/vector/types/Types.java  | 88 ++++++++++++++++---
 .../complex/writer/TestComplexWriter.java     | 85 ++++++++++++++++++
 .../arrow/vector/file/BaseFileTest.java       | 10 +--
 .../apache/arrow/vector/pojo/TestConvert.java |  2 +-
 10 files changed, 250 insertions(+), 25 deletions(-)

diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index f7790bb3d6ddf..2181cfdc335b4 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -71,7 +71,10 @@
         { class: "UInt8" },
         { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
         { class: "Date", javaType: "long", friendlyType: "DateTime" },
-        { class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
+        { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
+        { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
+        { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
+        { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
       ]
     },
     {
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 74a19a605e21e..d53744539aae8 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -96,6 +96,16 @@ public void read(Nullable${minor.class?cap_first}Holder h){
   public ${friendlyType} read${safeType}(){
     return vector.getAccessor().getObject(idx());
   }
+
+  <#if minor.class == "TimeStampSec" ||
+       minor.class == "TimeStampMilli" ||
+       minor.class == "TimeStampMicro" ||
+       minor.class == "TimeStampNano">
+  @Override
+  public ${minor.boxedType} read${minor.boxedType}(){
+    return vector.getAccessor().get(idx());
+  }
+  </#if>
   
   public void copyValue(FieldWriter w){
     
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index be385d146dbac..d5265f1140ee0 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -490,7 +490,16 @@ public long getTwoAsLong(int index) {
         return date;
     }
 
-    <#elseif minor.class == "TimeStamp">
+    <#elseif minor.class == "TimeStampSec">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
+      org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
+      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      return date;
+    }
+
+    <#elseif minor.class == "TimeStampMilli">
     @Override
     public ${friendlyType} getObject(int index) {
         org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
@@ -498,6 +507,26 @@ public long getTwoAsLong(int index) {
         return date;
     }
 
+    <#elseif minor.class == "TimeStampMicro">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      // value is truncated when converting microseconds to milliseconds in order to use DateTime type
+      long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
+      org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
+      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      return date;
+    }
+
+    <#elseif minor.class == "TimeStampNano">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
+      long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
+      org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
+      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      return date;
+    }
+
     <#elseif minor.class == "IntervalYear">
     @Override
     public ${friendlyType} getObject(int index) {
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 6a9ce65392f59..ce637100cd8bf 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -102,8 +102,14 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), null);
   <#elseif minor.class == "Float8">
     field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), null);
-  <#elseif minor.class == "TimeStamp">
+  <#elseif minor.class == "TimeStampSec">
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND), null);
+  <#elseif minor.class == "TimeStampMilli">
     field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), null);
+  <#elseif minor.class == "TimeStampMicro">
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND), null);
+  <#elseif minor.class == "TimeStampNano">
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND), null);
   <#elseif minor.class == "IntervalDay">
     field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), null);
   <#elseif minor.class == "IntervalYear">
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 152867c1a11d7..71fe88e156a5d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -37,7 +37,10 @@
 import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.SmallIntVector;
-import org.apache.arrow.vector.TimeStampVector;
+import org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampNanoVector;
 import org.apache.arrow.vector.TinyIntVector;
 import org.apache.arrow.vector.UInt1Vector;
 import org.apache.arrow.vector.UInt2Vector;
@@ -199,9 +202,18 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
     case VARCHAR:
       ((VarCharVector)valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8));
       break;
-    case TIMESTAMP:
-      ((TimeStampVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+    case TIMESTAMPSEC:
+      ((TimeStampSecVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
       break;
+    case TIMESTAMPMILLI:
+      ((TimeStampMilliVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case TIMESTAMPMICRO:
+      ((TimeStampMicroVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case TIMESTAMPNANO:
+      ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+    break;
     default:
       throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index 6ff357774486d..ddc80433cb6db 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -24,7 +24,10 @@
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.BufferBacked;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.TimeStampVector;
+import org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampNanoVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VectorSchemaRoot;
@@ -139,8 +142,17 @@ private void writeVector(Field field, FieldVector vector) throws IOException {
 
   private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException {
     switch (valueVector.getMinorType()) {
-      case TIMESTAMP:
-        generator.writeNumber(((TimeStampVector)valueVector).getAccessor().get(i));
+      case TIMESTAMPSEC:
+        generator.writeNumber(((TimeStampSecVector)valueVector).getAccessor().get(i));
+        break;
+      case TIMESTAMPMILLI:
+        generator.writeNumber(((TimeStampMilliVector)valueVector).getAccessor().get(i));
+        break;
+      case TIMESTAMPMICRO:
+        generator.writeNumber(((TimeStampMicroVector)valueVector).getAccessor().get(i));
+        break;
+      case TIMESTAMPNANO:
+        generator.writeNumber(((TimeStampNanoVector)valueVector).getAccessor().get(i));
         break;
       case BIT:
         generator.writeNumber(((BitVector)valueVector).getAccessor().get(i));
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 2a2fb74bee85c..ab539d5dc3b6e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -33,7 +33,10 @@
 import org.apache.arrow.vector.NullableIntervalDayVector;
 import org.apache.arrow.vector.NullableIntervalYearVector;
 import org.apache.arrow.vector.NullableSmallIntVector;
-import org.apache.arrow.vector.NullableTimeStampVector;
+import org.apache.arrow.vector.NullableTimeStampSecVector;
+import org.apache.arrow.vector.NullableTimeStampMilliVector;
+import org.apache.arrow.vector.NullableTimeStampMicroVector;
+import org.apache.arrow.vector.NullableTimeStampNanoVector;
 import org.apache.arrow.vector.NullableTimeVector;
 import org.apache.arrow.vector.NullableTinyIntVector;
 import org.apache.arrow.vector.NullableUInt1Vector;
@@ -58,7 +61,10 @@
 import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl;
 import org.apache.arrow.vector.complex.impl.NullableMapWriter;
 import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl;
-import org.apache.arrow.vector.complex.impl.TimeStampWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeWriterImpl;
 import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.UInt1WriterImpl;
@@ -102,7 +108,10 @@ public class Types {
   private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
   private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
   private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
-  private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null);
+  private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND), null);
+  private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null);
+  private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND), null);
+  private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND), null);
   private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
   private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
   private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);
@@ -241,21 +250,72 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new TimeWriterImpl((NullableTimeVector) vector);
       }
     },
+    // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
+    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND)) {
+      @Override
+      public Field getField() {
+        return TIMESTAMPSEC_FIELD;
+      }
+
+      @Override
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableTimeStampSecVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeStampSecWriterImpl((NullableTimeStampSecVector) vector);
+      }
+    },
     // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
-    TIMESTAMP(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) {
+    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) {
       @Override
       public Field getField() {
-        return TIMESTAMP_FIELD;
+        return TIMESTAMPMILLI_FIELD;
       }
 
       @Override
       public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampVector(name, allocator);
+        return new NullableTimeStampMilliVector(name, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampWriterImpl((NullableTimeStampVector) vector);
+        return new TimeStampMilliWriterImpl((NullableTimeStampMilliVector) vector);
+      }
+    },
+    // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
+    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND)) {
+      @Override
+      public Field getField() {
+        return TIMESTAMPMICRO_FIELD;
+      }
+
+      @Override
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableTimeStampMicroVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeStampMicroWriterImpl((NullableTimeStampMicroVector) vector);
+      }
+    },
+    // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
+    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND)) {
+      @Override
+      public Field getField() {
+        return TIMESTAMPNANO_FIELD;
+      }
+
+      @Override
+      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+        return new NullableTimeStampNanoVector(name, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeStampNanoWriterImpl((NullableTimeStampNanoVector) vector);
       }
     },
     INTERVALDAY(new Interval(IntervalUnit.DAY_TIME)) {
@@ -579,10 +639,18 @@ public MinorType visit(FloatingPoint type) {
       }
 
       @Override public MinorType visit(Timestamp type) {
-        if (type.getUnit() != TimeUnit.MILLISECOND) {
-          throw new UnsupportedOperationException("Only milliseconds supported: " + type);
+        switch (type.getUnit()) {
+          case SECOND:
+            return MinorType.TIMESTAMPSEC;
+          case MILLISECOND:
+            return MinorType.TIMESTAMPMILLI;
+          case MICROSECOND:
+            return MinorType.TIMESTAMPMICRO;
+          case NANOSECOND:
+            return MinorType.TIMESTAMPNANO;
+          default:
+            throw new IllegalArgumentException("unknown unit: " + type);
         }
-        return MinorType.TIMESTAMP;
       }
 
       @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 2c0c85328bdfb..7a2d416241b78 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -43,12 +43,15 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.Text;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -561,4 +564,86 @@ public void mapWriterMixedCaseFieldNames() {
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field"));
   }
+
+  @Test
+  public void timeStampWriters() throws Exception {
+    // test values
+    final long expectedNanos = 981173106123456789L;
+    final long expectedMicros = 981173106123456L;
+    final long expectedMillis = 981173106123L;
+    final long expectedSecs = 981173106L;
+    final DateTime expectedSecDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 0).withZoneRetainFields(DateTimeZone.getDefault());
+    final DateTime expectedMilliDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 123).withZoneRetainFields(DateTimeZone.getDefault());
+    final DateTime expectedMicroDateTime = expectedMilliDateTime;
+    final DateTime expectedNanoDateTime = expectedMilliDateTime;
+
+    // write
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+
+    TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec");
+    timeStampSecWriter.setPosition(0);
+    timeStampSecWriter.writeTimeStampSec(expectedSecs);
+
+    TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli");
+    timeStampWriter.setPosition(1);
+    timeStampWriter.writeTimeStampMilli(expectedMillis);
+
+    TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro");
+    timeStampMicroWriter.setPosition(2);
+    timeStampMicroWriter.writeTimeStampMicro(expectedMicros);
+
+    TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano");
+    timeStampNanoWriter.setPosition(3);
+    timeStampNanoWriter.writeTimeStampNano(expectedNanos);
+
+    // schema
+    Field secField = parent.getField().getChildren().get(0).getChildren().get(0);
+    Assert.assertEquals("sec", secField.getName());
+    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, secField.getType().getTypeID());
+
+    Field milliField = parent.getField().getChildren().get(0).getChildren().get(1);
+    Assert.assertEquals("milli", milliField.getName());
+    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, milliField.getType().getTypeID());
+
+    Field microField = parent.getField().getChildren().get(0).getChildren().get(2);
+    Assert.assertEquals("micro", microField.getName());
+    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, microField.getType().getTypeID());
+
+    Field nanoField = parent.getField().getChildren().get(0).getChildren().get(3);
+    Assert.assertEquals("nano", nanoField.getName());
+    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, nanoField.getType().getTypeID());
+
+    // read
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+
+    FieldReader secReader = rootReader.reader("sec");
+    secReader.setPosition(0);
+    DateTime secDateTime = secReader.readDateTime();
+    Assert.assertEquals(expectedSecDateTime, secDateTime);
+    long secLong = secReader.readLong();
+    Assert.assertEquals(expectedSecs, secLong);
+
+    FieldReader milliReader = rootReader.reader("milli");
+    milliReader.setPosition(1);
+    DateTime milliDateTime = milliReader.readDateTime();
+    Assert.assertEquals(expectedMilliDateTime, milliDateTime);
+    long milliLong = milliReader.readLong();
+    Assert.assertEquals(expectedMillis, milliLong);
+
+    FieldReader microReader = rootReader.reader("micro");
+    microReader.setPosition(2);
+    DateTime microDateTime = microReader.readDateTime();
+    Assert.assertEquals(expectedMicroDateTime, microDateTime);
+    long microLong = microReader.readLong();
+    Assert.assertEquals(expectedMicros, microLong);
+
+    FieldReader nanoReader = rootReader.reader("nano");
+    nanoReader.setPosition(3);
+    DateTime nanoDateTime = nanoReader.readDateTime();
+    Assert.assertEquals(expectedNanoDateTime, nanoDateTime);
+    long nanoLong = nanoReader.readLong();
+    Assert.assertEquals(expectedNanos, nanoLong);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 6e577b500a6bd..774bead3207a7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -33,7 +33,7 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
-import org.apache.arrow.vector.holders.NullableTimeStampHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
 import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.Assert;
@@ -100,7 +100,7 @@ protected void writeComplexData(int count, MapVector parent) {
       listWriter.endList();
       mapWriter.setPosition(i);
       mapWriter.start();
-      mapWriter.timeStamp("timestamp").writeTimeStamp(i);
+      mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
       mapWriter.end();
     }
     writer.setValueCount(count);
@@ -130,7 +130,7 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) {
       }
       Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
       Assert.assertEquals(i % 3, ((List<?>)root.getVector("list").getAccessor().getObject(i)).size());
-      NullableTimeStampHolder h = new NullableTimeStampHolder();
+      NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
       FieldReader mapReader = root.getVector("map").getReader();
       mapReader.setPosition(i);
       mapReader.reader("timestamp").read(h);
@@ -167,7 +167,7 @@ public void validateUnionData(int count, VectorSchemaRoot root) {
         Assert.assertEquals(i % 3, unionReader.size());
         break;
       case 3:
-        NullableTimeStampHolder h = new NullableTimeStampHolder();
+        NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
         unionReader.reader("timestamp").read(h);
         Assert.assertEquals(i, h.value);
         break;
@@ -209,7 +209,7 @@ public void writeUnionData(int count, NullableMapVector parent) {
       case 3:
         mapWriter.setPosition(i);
         mapWriter.start();
-        mapWriter.timeStamp("timestamp").writeTimeStamp(i);
+        mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
         mapWriter.end();
         break;
       }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 5a238bcc0d0c3..65823e2a821a1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -80,7 +80,7 @@ public void nestedSchema() {
     childrenBuilder.add(new Field("child4", true, new List(), ImmutableList.<Field>of(
         new Field("child4.1", true, Utf8.INSTANCE, null)
         )));
-    childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMP.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
+    childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
         new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null),
         new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));

From e881f1155c7c628f79008988fff8ae81d3750984 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 4 Feb 2017 16:20:54 -0500
Subject: [PATCH 0306/1644] ARROW-525: Python: Add more documentation to the
 package

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #317 from xhochy/ARROW-525 and squashes the following commits:

d213e63 [Uwe L. Korn] ARROW-525: Python: Add more documentation to the package
---
 python/setup.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index 9c63e93df3352..a771d23877013 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -257,9 +257,6 @@ def get_outputs(self):
         return [self._get_cmake_ext_path(name)
                 for name in self.get_names()]
 
-DESC = """\
-Python library for Apache Arrow"""
-
 # In the case of a git-archive, we don't have any version information
 # from the SCM to infer a version. The only source is the java/pom.xml.
 #
@@ -271,6 +268,12 @@ def get_outputs(self):
     version_tag = list(tree.getroot().findall('{http://maven.apache.org/POM/4.0.0}version'))[0]
     os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version_tag.text.replace("-SNAPSHOT", "a0")
 
+long_description = """Apache Arrow is a columnar in-memory analytics layer
+designed to accelerate big data. It houses a set of canonical in-memory
+representations of flat and hierarchical data along with multiple
+language-bindings for structure manipulation. It also provides IPC
+and common algorithm implementations."""
+
 setup(
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
@@ -286,9 +289,18 @@ def get_outputs(self):
     use_scm_version = {"root": "..", "relative_to": __file__},
     setup_requires=['setuptools_scm'],
     install_requires=['cython >= 0.23', 'numpy >= 1.9', 'six >= 1.0.0'],
-    description=DESC,
+    description="Python library for Apache Arrow",
+    long_description=long_description,
+    classifiers=[
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6'
+        ],
     license='Apache License, Version 2.0',
     maintainer="Apache Arrow Developers",
     maintainer_email="dev@arrow.apache.org",
-    test_suite="pyarrow.tests"
+    test_suite="pyarrow.tests",
+    url="https://arrow.apache.org/"
 )

From 5b35d6bda94e901d25aaf3d622dbe47214f75488 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 4 Feb 2017 16:23:46 -0500
Subject: [PATCH 0307/1644] ARROW-457: Python: Better control over memory pool

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #315 from xhochy/ARROW-457 and squashes the following commits:

dc5abdb [Uwe L. Korn] Use aligned deallocator
20c8505 [Uwe L. Korn] ARROW-458: Python: Expose jemalloc MemoryPool
2962bd8 [Uwe L. Korn] ARROW-457: Python: Better control over memory pool
---
 ci/travis_script_python.sh                    |  3 +-
 cpp/src/arrow/jemalloc/memory_pool.cc         |  2 +-
 python/CMakeLists.txt                         | 15 +++++
 python/cmake_modules/FindArrow.cmake          | 14 +++++
 python/pyarrow/__init__.py                    |  3 +-
 python/pyarrow/_parquet.pxd                   |  8 +--
 python/pyarrow/_parquet.pyx                   | 13 ++--
 python/pyarrow/array.pyx                      | 32 +++++-----
 python/pyarrow/includes/libarrow.pxd          |  6 +-
 python/pyarrow/includes/libarrow_io.pxd       |  2 +-
 python/pyarrow/includes/libarrow_ipc.pxd      |  3 +-
 python/pyarrow/includes/libarrow_jemalloc.pxd | 27 ++++++++
 python/pyarrow/includes/pyarrow.pxd           |  9 +--
 python/pyarrow/io.pyx                         | 18 +++---
 python/pyarrow/jemalloc.pyx                   | 28 +++++++++
 python/pyarrow/memory.pxd                     | 27 ++++++++
 python/pyarrow/memory.pyx                     | 49 +++++++++++++++
 python/pyarrow/tests/test_jemalloc.py         | 56 +++++++++++++++++
 python/setup.py                               | 11 +++-
 python/src/pyarrow/adapters/builtin.cc        |  6 +-
 python/src/pyarrow/adapters/builtin.h         |  3 +-
 python/src/pyarrow/common.cc                  | 61 ++++---------------
 python/src/pyarrow/common.h                   |  1 +
 23 files changed, 298 insertions(+), 99 deletions(-)
 create mode 100644 python/pyarrow/includes/libarrow_jemalloc.pxd
 create mode 100644 python/pyarrow/jemalloc.pyx
 create mode 100644 python/pyarrow/memory.pxd
 create mode 100644 python/pyarrow/memory.pyx
 create mode 100644 python/pyarrow/tests/test_jemalloc.py

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index c186fd4639fca..11d8d89ca7b6f 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -85,9 +85,10 @@ python_version_tests() {
   # Other stuff pip install
   pip install -r requirements.txt
 
-  python setup.py build_ext --inplace --with-parquet
+  python setup.py build_ext --inplace --with-parquet --with-jemalloc
 
   python -c "import pyarrow.parquet"
+  python -c "import pyarrow.jemalloc"
 
   python -m pytest -vv -r sxX pyarrow
 
diff --git a/cpp/src/arrow/jemalloc/memory_pool.cc b/cpp/src/arrow/jemalloc/memory_pool.cc
index c568316711717..f7a1446a0d27c 100644
--- a/cpp/src/arrow/jemalloc/memory_pool.cc
+++ b/cpp/src/arrow/jemalloc/memory_pool.cc
@@ -65,7 +65,7 @@ Status MemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr)
 
 void MemoryPool::Free(uint8_t* buffer, int64_t size) {
   allocated_size_ -= size;
-  free(buffer);
+  dallocx(buffer, MALLOCX_ALIGN(kAlignment));
 }
 
 int64_t MemoryPool::bytes_allocated() const {
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 942e74b73aaee..898c48ee0e48d 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -53,6 +53,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUILD_PARQUET
     "Build the PyArrow Parquet integration"
     OFF)
+  option(PYARROW_BUILD_JEMALLOC
+    "Build the PyArrow jemalloc integration"
+    OFF)
 endif()
 
 if(NOT PYARROW_BUILD_TESTS)
@@ -412,6 +415,7 @@ set(CYTHON_EXTENSIONS
   config
   error
   io
+  memory
   scalar
   schema
   table
@@ -446,6 +450,17 @@ if (PYARROW_BUILD_PARQUET)
     _parquet)
 endif()
 
+if (PYARROW_BUILD_JEMALLOC)
+  ADD_THIRDPARTY_LIB(arrow_jemalloc
+    SHARED_LIB ${ARROW_JEMALLOC_SHARED_LIB})
+  set(LINK_LIBS
+    ${LINK_LIBS}
+    arrow_jemalloc)
+  set(CYTHON_EXTENSIONS
+    ${CYTHON_EXTENSIONS}
+    jemalloc)
+endif()
+
 add_library(pyarrow SHARED
   ${PYARROW_SRCS})
 target_link_libraries(pyarrow ${LINK_LIBS})
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 3c359aac55309..5d0207d7c7769 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -52,11 +52,17 @@ find_library(ARROW_IPC_LIB_PATH NAMES arrow_ipc
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
+find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+
 if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
   set(ARROW_IO_LIB_NAME libarrow_io)
   set(ARROW_IPC_LIB_NAME libarrow_ipc)
+  set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc)
 
   set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
@@ -68,10 +74,14 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_IPC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IPC_LIB_NAME}.a)
   set(ARROW_IPC_SHARED_LIB ${ARROW_LIBS}/${ARROW_IPC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
+  set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_JEMALLOC_LIB_NAME}.a)
+  set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
     message(STATUS "Found the Arrow IO library: ${ARROW_IO_LIB_PATH}")
     message(STATUS "Found the Arrow IPC library: ${ARROW_IPC_LIB_PATH}")
+    message(STATUS "Found the Arrow jemalloc library: ${ARROW_JEMALLOC_LIB_PATH}")
   endif ()
 else ()
   if (NOT Arrow_FIND_QUIETLY)
@@ -94,4 +104,8 @@ mark_as_advanced(
   ARROW_SHARED_LIB
   ARROW_IO_STATIC_LIB
   ARROW_IO_SHARED_LIB
+  ARROW_IPC_STATIC_LIB
+  ARROW_IPC_SHARED_LIB
+  ARROW_JEMALLOC_STATIC_LIB
+  ARROW_JEMALLOC_SHARED_LIB
 )
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 7c521db6280be..ea4710d4137de 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -30,7 +30,6 @@
 
 from pyarrow.array import (Array,
                            from_pandas_series, from_pylist,
-                           total_allocated_bytes,
                            NumericArray, IntegerArray, FloatingPointArray,
                            BooleanArray,
                            Int8Array, UInt8Array,
@@ -48,6 +47,8 @@
 
 from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
 
+from pyarrow.memory import MemoryPool, total_allocated_bytes
+
 from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
                             BooleanValue,
                             Int8Value, Int16Value, Int32Value, Int64Value,
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 6b9350ad6782a..005be91bdb97f 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -19,7 +19,7 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
-                                        CTable, MemoryPool)
+                                        CTable, CMemoryPool)
 from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream
 
 
@@ -204,13 +204,13 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
 
 cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
     CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
-                     MemoryPool* allocator,
+                     CMemoryPool* allocator,
                      const ReaderProperties& properties,
                      const shared_ptr[CFileMetaData]& metadata,
                      unique_ptr[FileReader]* reader)
 
     cdef cppclass FileReader:
-        FileReader(MemoryPool* pool, unique_ptr[ParquetFileReader] reader)
+        FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
         CStatus ReadColumn(int i, shared_ptr[CArray]* out);
         CStatus ReadTable(shared_ptr[CTable]* out);
         CStatus ReadTable(const vector[int]& column_indices,
@@ -229,7 +229,7 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
 
 cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
     cdef CStatus WriteTable(
-        const CTable* table, MemoryPool* pool,
+        const CTable* table, CMemoryPool* pool,
         const shared_ptr[OutputStream]& sink,
         int64_t chunk_size,
         const shared_ptr[WriterProperties]& properties)
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index fd4670a00c837..796c436ec46f4 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -32,6 +32,7 @@ from pyarrow.compat import tobytes, frombytes
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
 from pyarrow.io import NativeFile
+from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
 from pyarrow.table cimport Table
 
 from pyarrow.io cimport NativeFile, get_reader, get_writer
@@ -342,13 +343,13 @@ cdef logical_type_name_from_enum(ParquetLogicalType type_):
 cdef class ParquetReader:
     cdef:
         object source
-        MemoryPool* allocator
+        CMemoryPool* allocator
         unique_ptr[FileReader] reader
         column_idx_map
         FileMetaData _metadata
 
-    def __cinit__(self):
-        self.allocator = default_memory_pool()
+    def __cinit__(self, MemoryPool memory_pool=None):
+        self.allocator = maybe_unbox_memory_pool(memory_pool)
         self._metadata = None
 
     def open(self, object source, FileMetaData metadata=None):
@@ -471,6 +472,7 @@ cdef class ParquetWriter:
     cdef:
         shared_ptr[WriterProperties] properties
         shared_ptr[OutputStream] sink
+        CMemoryPool* allocator
 
     cdef readonly:
         object use_dictionary
@@ -479,7 +481,7 @@ cdef class ParquetWriter:
         int row_group_size
 
     def __cinit__(self, where, use_dictionary=None, compression=None,
-                  version=None):
+                  version=None, MemoryPool memory_pool=None):
         cdef shared_ptr[FileOutputStream] filestream
 
         if isinstance(where, six.string_types):
@@ -487,6 +489,7 @@ cdef class ParquetWriter:
             self.sink = <shared_ptr[OutputStream]> filestream
         else:
             get_writer(where, &self.sink)
+        self.allocator = maybe_unbox_memory_pool(memory_pool)
 
         self.use_dictionary = use_dictionary
         self.compression = compression
@@ -540,6 +543,6 @@ cdef class ParquetWriter:
 
         cdef int c_row_group_size = row_group_size
         with nogil:
-            check_status(WriteTable(ctable, default_memory_pool(),
+            check_status(WriteTable(ctable, self.allocator,
                                         self.sink, c_row_group_size,
                                         self.properties))
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index c3a5a045b7dd5..9b34f5607b31d 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -29,6 +29,7 @@ import pyarrow.config
 
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.error cimport check_status
+from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
 
 cimport pyarrow.scalar as scalar
 from pyarrow.scalar import NA
@@ -44,11 +45,6 @@ cdef _pandas():
     return pd
 
 
-def total_allocated_bytes():
-    cdef MemoryPool* pool = pyarrow.get_memory_pool()
-    return pool.bytes_allocated()
-
-
 cdef class Array:
 
     cdef init(self, const shared_ptr[CArray]& sp_array):
@@ -58,7 +54,7 @@ cdef class Array:
         self.type.init(self.sp_array.get().type())
 
     @staticmethod
-    def from_pandas(obj, mask=None, timestamps_to_ms=False, Field field=None):
+    def from_pandas(obj, mask=None, timestamps_to_ms=False, Field field=None, MemoryPool memory_pool=None):
         """
         Convert pandas.Series to an Arrow Array.
 
@@ -74,6 +70,9 @@ cdef class Array:
             compatibility with other functionality like Parquet I/O which
             only supports milliseconds.
 
+        memory_pool: MemoryPool, optional
+            Specific memory pool to use to allocate the resulting Arrow array.
+
         Notes
         -----
         Localized timestamps will currently be returned as UTC (pandas's native representation).
@@ -107,6 +106,7 @@ cdef class Array:
         cdef:
             shared_ptr[CArray] out
             shared_ptr[CField] c_field
+            CMemoryPool* pool
 
         pd = _pandas()
 
@@ -121,20 +121,20 @@ cdef class Array:
         if isinstance(series_values, pd.Categorical):
             return DictionaryArray.from_arrays(series_values.codes,
                                                series_values.categories.values,
-                                               mask=mask)
+                                               mask=mask, memory_pool=memory_pool)
         else:
             if series_values.dtype.type == np.datetime64 and timestamps_to_ms:
                 series_values = series_values.astype('datetime64[ms]')
 
+            pool = maybe_unbox_memory_pool(memory_pool)
             with nogil:
                 check_status(pyarrow.PandasToArrow(
-                    pyarrow.get_memory_pool(), series_values, mask,
-                    c_field, &out))
+                    pool, series_values, mask, c_field, &out))
 
             return box_arrow_array(out)
 
     @staticmethod
-    def from_list(object list_obj, DataType type=None):
+    def from_list(object list_obj, DataType type=None, MemoryPool memory_pool=None):
         """
         Convert Python list to Arrow array
 
@@ -147,10 +147,12 @@ cdef class Array:
         pyarrow.array.Array
         """
         cdef:
-            shared_ptr[CArray] sp_array
+           shared_ptr[CArray] sp_array
+           CMemoryPool* pool
 
+        pool = maybe_unbox_memory_pool(memory_pool)
         if type is None:
-            check_status(pyarrow.ConvertPySequence(list_obj, &sp_array))
+            check_status(pyarrow.ConvertPySequence(list_obj, pool, &sp_array))
         else:
             raise NotImplementedError()
 
@@ -330,7 +332,7 @@ cdef class BinaryArray(Array):
 cdef class DictionaryArray(Array):
 
     @staticmethod
-    def from_arrays(indices, dictionary, mask=None):
+    def from_arrays(indices, dictionary, mask=None, MemoryPool memory_pool=None):
         """
         Construct Arrow DictionaryArray from array of indices (must be
         non-negative integers) and corresponding array of dictionary values
@@ -352,8 +354,8 @@ cdef class DictionaryArray(Array):
             shared_ptr[CDataType] c_type
             shared_ptr[CArray] c_result
 
-        arrow_indices = Array.from_pandas(indices, mask=mask)
-        arrow_dictionary = Array.from_pandas(dictionary)
+        arrow_indices = Array.from_pandas(indices, mask=mask, memory_pool=memory_pool)
+        arrow_dictionary = Array.from_pandas(dictionary, memory_pool=memory_pool)
 
         if not isinstance(arrow_indices, IntegerArray):
             raise ValueError('Indices must be integer type')
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 6284ad3c88a7a..38883e811e1cc 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -90,7 +90,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     shared_ptr[CDataType] timestamp(TimeUnit unit)
 
-    cdef cppclass MemoryPool" arrow::MemoryPool":
+    cdef cppclass CMemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
 
     cdef cppclass CBuffer" arrow::Buffer":
@@ -104,9 +104,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass PoolBuffer(ResizableBuffer):
         PoolBuffer()
-        PoolBuffer(MemoryPool*)
+        PoolBuffer(CMemoryPool*)
 
-    cdef MemoryPool* default_memory_pool()
+    cdef CMemoryPool* default_memory_pool()
 
     cdef cppclass CListType" arrow::ListType"(CDataType):
         CListType(const shared_ptr[CDataType]& value_type)
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 31379386187ee..8d0d5248b4db0 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -82,7 +82,7 @@ cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
         CStatus Open(const c_string& path, shared_ptr[ReadableFile]* file)
 
         @staticmethod
-        CStatus Open(const c_string& path, MemoryPool* memory_pool,
+        CStatus Open(const c_string& path, CMemoryPool* memory_pool,
                      shared_ptr[ReadableFile]* file)
 
         int file_descriptor()
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
index bfece14fe6e03..5ab98152add49 100644
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -18,8 +18,7 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (MemoryPool, CArray, CSchema,
-                                        CRecordBatch)
+from pyarrow.includes.libarrow cimport (CArray, CSchema, CRecordBatch)
 from pyarrow.includes.libarrow_io cimport (InputStream, OutputStream,
                                            ReadableFileInterface)
 
diff --git a/python/pyarrow/includes/libarrow_jemalloc.pxd b/python/pyarrow/includes/libarrow_jemalloc.pxd
new file mode 100644
index 0000000000000..0609d1907589a
--- /dev/null
+++ b/python/pyarrow/includes/libarrow_jemalloc.pxd
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport *
+    
+cdef extern from "arrow/jemalloc/memory_pool.h" namespace "arrow::jemalloc" nogil:
+    cdef cppclass CJemallocMemoryPool" arrow::jemalloc::MemoryPool":
+        int64_t bytes_allocated()
+        @staticmethod
+        CMemoryPool* default_pool()
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 04ad4f32272e6..f1d45e0d50f36 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -20,7 +20,7 @@
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CField,
                                         CTable, CDataType, CStatus, Type,
-                                        MemoryPool, TimeUnit)
+                                        CMemoryPool, TimeUnit)
 
 cimport pyarrow.includes.libarrow_io as arrow_io
 
@@ -28,9 +28,9 @@ cimport pyarrow.includes.libarrow_io as arrow_io
 cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     shared_ptr[CDataType] GetPrimitiveType(Type type)
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
-    CStatus ConvertPySequence(object obj, shared_ptr[CArray]* out)
+    CStatus ConvertPySequence(object obj, CMemoryPool* pool, shared_ptr[CArray]* out)
 
-    CStatus PandasToArrow(MemoryPool* pool, object ao, object mo,
+    CStatus PandasToArrow(CMemoryPool* pool, object ao, object mo,
                           shared_ptr[CField] field,
                           shared_ptr[CArray]* out)
 
@@ -43,7 +43,8 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     CStatus ConvertTableToPandas(const shared_ptr[CTable]& table,
                                  int nthreads, PyObject** out)
 
-    MemoryPool* get_memory_pool()
+    void set_default_memory_pool(CMemoryPool* pool)
+    CMemoryPool* get_memory_pool()
 
 
 cdef extern from "pyarrow/common.h" namespace "pyarrow" nogil:
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 8b5650879f8f1..89ce6e785c02b 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -33,6 +33,7 @@ cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.compat import frombytes, tobytes, encode_file_path
 from pyarrow.error cimport check_status
+from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
 from pyarrow.schema cimport Schema
 from pyarrow.table cimport (RecordBatch, batch_from_cbatch,
                             table_from_ctable)
@@ -372,7 +373,7 @@ cdef class OSFile(NativeFile):
     cdef:
         object path
 
-    def __cinit__(self, path, mode='r'):
+    def __cinit__(self, path, mode='r', MemoryPool memory_pool=None):
         self.path = path
 
         cdef:
@@ -383,7 +384,7 @@ cdef class OSFile(NativeFile):
         self.is_readable = self.is_writeable = 0
 
         if mode in ('r', 'rb'):
-            self._open_readable(c_path)
+            self._open_readable(c_path, maybe_unbox_memory_pool(memory_pool))
         elif mode in ('w', 'wb'):
             self._open_writeable(c_path)
         else:
@@ -391,12 +392,11 @@ cdef class OSFile(NativeFile):
 
         self.is_open = True
 
-    cdef _open_readable(self, c_string path):
+    cdef _open_readable(self, c_string path, CMemoryPool* pool):
         cdef shared_ptr[ReadableFile] handle
 
         with nogil:
-            check_status(ReadableFile.Open(path, pyarrow.get_memory_pool(),
-                                           &handle))
+            check_status(ReadableFile.Open(path, pool, &handle))
 
         self.is_readable = 1
         self.rd_file = <shared_ptr[ReadableFileInterface]> handle
@@ -450,9 +450,9 @@ cdef class Buffer:
             self.buffer.get().size())
 
 
-cdef shared_ptr[PoolBuffer] allocate_buffer():
+cdef shared_ptr[PoolBuffer] allocate_buffer(CMemoryPool* pool):
     cdef shared_ptr[PoolBuffer] result
-    result.reset(new PoolBuffer(pyarrow.get_memory_pool()))
+    result.reset(new PoolBuffer(pool))
     return result
 
 
@@ -461,8 +461,8 @@ cdef class InMemoryOutputStream(NativeFile):
     cdef:
         shared_ptr[PoolBuffer] buffer
 
-    def __cinit__(self):
-        self.buffer = allocate_buffer()
+    def __cinit__(self, MemoryPool memory_pool=None):
+        self.buffer = allocate_buffer(maybe_unbox_memory_pool(memory_pool))
         self.wr_file.reset(new BufferOutputStream(
             <shared_ptr[ResizableBuffer]> self.buffer))
         self.is_readable = 0
diff --git a/python/pyarrow/jemalloc.pyx b/python/pyarrow/jemalloc.pyx
new file mode 100644
index 0000000000000..97583f4b0da95
--- /dev/null
+++ b/python/pyarrow/jemalloc.pyx
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from pyarrow.includes.libarrow_jemalloc cimport CJemallocMemoryPool
+from pyarrow.memory cimport MemoryPool
+
+def default_pool():
+    cdef MemoryPool pool = MemoryPool()
+    pool.init(CJemallocMemoryPool.default_pool())
+    return pool
diff --git a/python/pyarrow/memory.pxd b/python/pyarrow/memory.pxd
new file mode 100644
index 0000000000000..3079ccb807b0d
--- /dev/null
+++ b/python/pyarrow/memory.pxd
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyarrow.includes.libarrow cimport CMemoryPool
+
+
+cdef class MemoryPool:
+    cdef:
+        CMemoryPool* pool
+
+    cdef init(self, CMemoryPool* pool)
+
+cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool)
diff --git a/python/pyarrow/memory.pyx b/python/pyarrow/memory.pyx
new file mode 100644
index 0000000000000..18a6de4f15392
--- /dev/null
+++ b/python/pyarrow/memory.pyx
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from pyarrow.includes.libarrow cimport CMemoryPool
+from pyarrow.includes.pyarrow cimport set_default_memory_pool, get_memory_pool
+
+cdef class MemoryPool:
+    cdef init(self, CMemoryPool* pool):
+        self.pool = pool
+
+    def bytes_allocated(self):
+        return self.pool.bytes_allocated()
+
+cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool):
+    if memory_pool is None:
+        return get_memory_pool()
+    else:
+        return memory_pool.pool
+
+def default_pool():
+    cdef: 
+        MemoryPool pool = MemoryPool()
+    pool.init(get_memory_pool())
+    return pool
+
+def set_default_pool(MemoryPool pool):
+    set_default_memory_pool(pool.pool)
+
+def total_allocated_bytes():
+    cdef CMemoryPool* pool = get_memory_pool()
+    return pool.bytes_allocated()
diff --git a/python/pyarrow/tests/test_jemalloc.py b/python/pyarrow/tests/test_jemalloc.py
new file mode 100644
index 0000000000000..8efd514dd0cae
--- /dev/null
+++ b/python/pyarrow/tests/test_jemalloc.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import gc
+import pytest
+
+try:
+    import pyarrow.jemalloc
+    HAVE_JEMALLOC = True
+except ImportError:
+    HAVE_JEMALLOC = False
+
+jemalloc = pytest.mark.skipif(not HAVE_JEMALLOC,
+                              reason='jemalloc support not built')
+
+
+@jemalloc
+def test_different_memory_pool():
+    gc.collect()
+    bytes_before_default = pyarrow.total_allocated_bytes()
+    bytes_before_jemalloc = pyarrow.jemalloc.default_pool().bytes_allocated()
+    array = pyarrow.from_pylist([1, None, 3, None],
+                        memory_pool=pyarrow.jemalloc.default_pool())
+    gc.collect()
+    assert pyarrow.total_allocated_bytes() == bytes_before_default
+    assert pyarrow.jemalloc.default_pool().bytes_allocated() > bytes_before_jemalloc
+
+@jemalloc
+def test_default_memory_pool():
+    gc.collect()
+    bytes_before_default = pyarrow.total_allocated_bytes()
+    bytes_before_jemalloc = pyarrow.jemalloc.default_pool().bytes_allocated()
+
+    old_memory_pool = pyarrow.memory.default_pool()
+    pyarrow.memory.set_default_pool(pyarrow.jemalloc.default_pool())
+    array = pyarrow.from_pylist([1, None, 3, None])
+    pyarrow.memory.set_default_pool(old_memory_pool)
+    gc.collect()
+
+    assert pyarrow.total_allocated_bytes() == bytes_before_default
+    assert pyarrow.jemalloc.default_pool().bytes_allocated() > bytes_before_jemalloc
+
diff --git a/python/setup.py b/python/setup.py
index a771d23877013..5f5e5f3795541 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -80,7 +80,8 @@ def run(self):
     description = "Build the C-extensions for arrow"
     user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
                      ('build-type=', None, 'build type (debug or release)'),
-                     ('with-parquet', None, 'build the Parquet extension')] +
+                     ('with-parquet', None, 'build the Parquet extension'),
+                     ('with-jemalloc', None, 'build the jemalloc extension')] +
                     _build_ext.user_options)
 
     def initialize_options(self):
@@ -88,12 +89,15 @@ def initialize_options(self):
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
         self.with_parquet = False
+        self.with_jemalloc = False
 
     CYTHON_MODULE_NAMES = [
         'array',
         'config',
         'error',
         'io',
+        'jemalloc',
+        'memory',
         '_parquet',
         'scalar',
         'schema',
@@ -135,6 +139,9 @@ def _run_cmake(self):
         if self.with_parquet:
             cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
 
+        if self.with_jemalloc:
+            cmake_options.append('-DPYARROW_BUILD_JEMALLOC=on')
+
         if sys.platform != 'win32':
             cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                                  .format(self.build_type))
@@ -216,6 +223,8 @@ def _run_cmake(self):
     def _failure_permitted(self, name):
         if name == '_parquet' and not self.with_parquet:
             return True
+        if name == 'jemalloc' and not self.with_jemalloc:
+            return True
         return False
 
     def _get_inplace_dir(self):
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index fb7475f0c9407..1abfb4091189e 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -29,6 +29,7 @@
 
 using arrow::ArrayBuilder;
 using arrow::DataType;
+using arrow::MemoryPool;
 using arrow::Status;
 using arrow::Type;
 
@@ -495,7 +496,8 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
   return Status::OK();
 }
 
-Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
+Status ConvertPySequence(
+    PyObject* obj, MemoryPool* pool, std::shared_ptr<arrow::Array>* out) {
   std::shared_ptr<DataType> type;
   int64_t size;
   PyDateTime_IMPORT;
@@ -516,7 +518,7 @@ Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out) {
 
   // Give the sequence converter an array builder
   std::shared_ptr<ArrayBuilder> builder;
-  RETURN_NOT_OK(arrow::MakeBuilder(get_memory_pool(), type, &builder));
+  RETURN_NOT_OK(arrow::MakeBuilder(pool, type, &builder));
   converter->Init(builder);
 
   RETURN_NOT_OK(converter->AppendData(obj));
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
index 1ff36945c88c7..667298e3c5c5f 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -38,7 +38,8 @@ class Status;
 namespace pyarrow {
 
 PYARROW_EXPORT
-arrow::Status ConvertPySequence(PyObject* obj, std::shared_ptr<arrow::Array>* out);
+arrow::Status ConvertPySequence(
+    PyObject* obj, arrow::MemoryPool* pool, std::shared_ptr<arrow::Array>* out);
 
 }  // namespace pyarrow
 
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index b8712d7d0a4fc..d2f5291ea8301 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -28,58 +28,21 @@ using arrow::Status;
 
 namespace pyarrow {
 
-class PyArrowMemoryPool : public arrow::MemoryPool {
- public:
-  PyArrowMemoryPool() : bytes_allocated_(0) {}
-  virtual ~PyArrowMemoryPool() {}
+static std::mutex memory_pool_mutex;
+static arrow::MemoryPool* default_pyarrow_pool = nullptr;
 
-  Status Allocate(int64_t size, uint8_t** out) override {
-    std::lock_guard<std::mutex> guard(pool_lock_);
-    *out = static_cast<uint8_t*>(std::malloc(size));
-    if (*out == nullptr) {
-      std::stringstream ss;
-      ss << "malloc of size " << size << " failed";
-      return Status::OutOfMemory(ss.str());
-    }
-
-    bytes_allocated_ += size;
-
-    return Status::OK();
-  }
-
-  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override {
-    *ptr = reinterpret_cast<uint8_t*>(std::realloc(*ptr, new_size));
-
-    if (*ptr == NULL) {
-      std::stringstream ss;
-      ss << "realloc of size " << new_size << " failed";
-      return Status::OutOfMemory(ss.str());
-    }
-
-    bytes_allocated_ += new_size - old_size;
-
-    return Status::OK();
-  }
-
-  int64_t bytes_allocated() const override {
-    std::lock_guard<std::mutex> guard(pool_lock_);
-    return bytes_allocated_;
-  }
-
-  void Free(uint8_t* buffer, int64_t size) override {
-    std::lock_guard<std::mutex> guard(pool_lock_);
-    std::free(buffer);
-    bytes_allocated_ -= size;
-  }
-
- private:
-  mutable std::mutex pool_lock_;
-  int64_t bytes_allocated_;
-};
+void set_default_memory_pool(arrow::MemoryPool* pool) {
+  std::lock_guard<std::mutex> guard(memory_pool_mutex);
+  default_pyarrow_pool = pool;
+}
 
 arrow::MemoryPool* get_memory_pool() {
-  static PyArrowMemoryPool memory_pool;
-  return &memory_pool;
+  std::lock_guard<std::mutex> guard(memory_pool_mutex);
+  if (default_pyarrow_pool) {
+    return default_pyarrow_pool;
+  } else {
+    return arrow::default_memory_pool();
+  }
 }
 
 // ----------------------------------------------------------------------
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 0733a3b7cf061..ad65ec75eec9e 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -98,6 +98,7 @@ struct PyObjectStringify {
   }
 
 // Return the common PyArrow memory pool
+PYARROW_EXPORT void set_default_memory_pool(arrow::MemoryPool* pool);
 PYARROW_EXPORT arrow::MemoryPool* get_memory_pool();
 
 class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {

From 84f16624bb390aebf16318b62ff2ac8238fc4b7c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 4 Feb 2017 18:33:39 -0500
Subject: [PATCH 0308/1644] ARROW-381: [C++] Simplify primitive array type
 builders to use a default type singleton

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #316 from xhochy/ARROW-381 and squashes the following commits:

7061d9a [Uwe L. Korn] Use TypeTraits
79e07f1 [Uwe L. Korn] ARROW-381: [C++] Simplify primitive array type builders to use a default type singleton
---
 cpp/src/arrow/builder-benchmark.cc |  2 +-
 cpp/src/arrow/builder.h            |  7 +++++++
 cpp/src/arrow/test-util.h          |  4 ++--
 cpp/src/arrow/type.h               | 17 ----------------
 cpp/src/arrow/type_fwd.h           | 22 +++++++++++++++++++++
 cpp/src/arrow/type_traits.h        | 31 ++++++++++++++++++++++++++++++
 6 files changed, 63 insertions(+), 20 deletions(-)

diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index 67799a3485f23..b0c3cd19064de 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -30,7 +30,7 @@ static void BM_BuildPrimitiveArrayNoNulls(
   // 2 MiB block
   std::vector<int64_t> data(256 * 1024, 100);
   while (state.KeepRunning()) {
-    Int64Builder builder(default_memory_pool(), arrow::int64());
+    Int64Builder builder(default_memory_pool());
     for (int i = 0; i < kFinalSize; i++) {
       // Build up an array of 512 MiB in size
       builder.Append(data.data(), data.size(), nullptr);
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 747da7ca2d9dd..672d2d8f23e8f 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -19,6 +19,7 @@
 #define ARROW_BUILDER_H
 
 #include <cstdint>
+#include <functional>
 #include <memory>
 #include <string>
 #include <vector>
@@ -27,6 +28,7 @@
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
+#include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
@@ -186,6 +188,11 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
   using typename PrimitiveBuilder<T>::value_type;
   using PrimitiveBuilder<T>::PrimitiveBuilder;
 
+  template <typename T1 = T>
+  explicit NumericBuilder(
+      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool)
+      : PrimitiveBuilder<T1>(pool, TypeTraits<T1>::type_singleton()) {}
+
   using PrimitiveBuilder<T>::Append;
   using PrimitiveBuilder<T>::Init;
   using PrimitiveBuilder<T>::Resize;
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index b59809d9e48e6..4e525804b47cc 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -294,8 +294,8 @@ class TestBuilder : public ::testing::Test {
   void SetUp() {
     pool_ = default_memory_pool();
     type_ = TypePtr(new UInt8Type());
-    builder_.reset(new UInt8Builder(pool_, type_));
-    builder_nn_.reset(new UInt8Builder(pool_, type_));
+    builder_.reset(new UInt8Builder(pool_));
+    builder_nn_.reset(new UInt8Builder(pool_));
   }
 
  protected:
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 77a70d1d2ddd3..8638a3f4b6e90 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -540,26 +540,9 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 // ----------------------------------------------------------------------
 // Factory functions
 
-std::shared_ptr<DataType> ARROW_EXPORT null();
-std::shared_ptr<DataType> ARROW_EXPORT boolean();
-std::shared_ptr<DataType> ARROW_EXPORT int8();
-std::shared_ptr<DataType> ARROW_EXPORT int16();
-std::shared_ptr<DataType> ARROW_EXPORT int32();
-std::shared_ptr<DataType> ARROW_EXPORT int64();
-std::shared_ptr<DataType> ARROW_EXPORT uint8();
-std::shared_ptr<DataType> ARROW_EXPORT uint16();
-std::shared_ptr<DataType> ARROW_EXPORT uint32();
-std::shared_ptr<DataType> ARROW_EXPORT uint64();
-std::shared_ptr<DataType> ARROW_EXPORT float16();
-std::shared_ptr<DataType> ARROW_EXPORT float32();
-std::shared_ptr<DataType> ARROW_EXPORT float64();
-std::shared_ptr<DataType> ARROW_EXPORT utf8();
-std::shared_ptr<DataType> ARROW_EXPORT binary();
-
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type);
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
 
-std::shared_ptr<DataType> ARROW_EXPORT date();
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit unit);
 std::shared_ptr<DataType> ARROW_EXPORT time(TimeUnit unit);
 
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 334abef664426..fc4ad3d87d8ac 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -18,6 +18,8 @@
 #ifndef ARROW_TYPE_FWD_H
 #define ARROW_TYPE_FWD_H
 
+#include "arrow/util/visibility.h"
+
 namespace arrow {
 
 class Status;
@@ -104,6 +106,26 @@ using TimestampBuilder = NumericBuilder<TimestampType>;
 struct IntervalType;
 using IntervalArray = NumericArray<IntervalType>;
 
+// ----------------------------------------------------------------------
+// (parameter-free) Factory functions
+
+std::shared_ptr<DataType> ARROW_EXPORT null();
+std::shared_ptr<DataType> ARROW_EXPORT boolean();
+std::shared_ptr<DataType> ARROW_EXPORT int8();
+std::shared_ptr<DataType> ARROW_EXPORT int16();
+std::shared_ptr<DataType> ARROW_EXPORT int32();
+std::shared_ptr<DataType> ARROW_EXPORT int64();
+std::shared_ptr<DataType> ARROW_EXPORT uint8();
+std::shared_ptr<DataType> ARROW_EXPORT uint16();
+std::shared_ptr<DataType> ARROW_EXPORT uint32();
+std::shared_ptr<DataType> ARROW_EXPORT uint64();
+std::shared_ptr<DataType> ARROW_EXPORT float16();
+std::shared_ptr<DataType> ARROW_EXPORT float32();
+std::shared_ptr<DataType> ARROW_EXPORT float64();
+std::shared_ptr<DataType> ARROW_EXPORT utf8();
+std::shared_ptr<DataType> ARROW_EXPORT binary();
+std::shared_ptr<DataType> ARROW_EXPORT date();
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPE_FWD_H
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 5616018d93400..5cd5f45466bf7 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -33,6 +33,8 @@ struct TypeTraits<UInt8Type> {
   using ArrayType = UInt8Array;
   using BuilderType = UInt8Builder;
   static inline int bytes_required(int elements) { return elements; }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return uint8(); }
 };
 
 template <>
@@ -40,6 +42,8 @@ struct TypeTraits<Int8Type> {
   using ArrayType = Int8Array;
   using BuilderType = Int8Builder;
   static inline int bytes_required(int elements) { return elements; }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return int8(); }
 };
 
 template <>
@@ -48,6 +52,8 @@ struct TypeTraits<UInt16Type> {
   using BuilderType = UInt16Builder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return uint16(); }
 };
 
 template <>
@@ -56,6 +62,8 @@ struct TypeTraits<Int16Type> {
   using BuilderType = Int16Builder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int16_t); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return int16(); }
 };
 
 template <>
@@ -64,6 +72,8 @@ struct TypeTraits<UInt32Type> {
   using BuilderType = UInt32Builder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(uint32_t); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return uint32(); }
 };
 
 template <>
@@ -72,6 +82,8 @@ struct TypeTraits<Int32Type> {
   using BuilderType = Int32Builder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int32_t); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return int32(); }
 };
 
 template <>
@@ -80,6 +92,8 @@ struct TypeTraits<UInt64Type> {
   using BuilderType = UInt64Builder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(uint64_t); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return uint64(); }
 };
 
 template <>
@@ -88,6 +102,8 @@ struct TypeTraits<Int64Type> {
   using BuilderType = Int64Builder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return int64(); }
 };
 
 template <>
@@ -96,6 +112,8 @@ struct TypeTraits<DateType> {
   // using BuilderType = DateBuilder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return date(); }
 };
 
 template <>
@@ -104,6 +122,7 @@ struct TypeTraits<TimestampType> {
   // using BuilderType = TimestampBuilder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+  constexpr static bool is_parameter_free = false;
 };
 
 template <>
@@ -112,6 +131,8 @@ struct TypeTraits<HalfFloatType> {
   using BuilderType = HalfFloatBuilder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return float16(); }
 };
 
 template <>
@@ -120,6 +141,8 @@ struct TypeTraits<FloatType> {
   using BuilderType = FloatBuilder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(float); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return float32(); }
 };
 
 template <>
@@ -128,6 +151,8 @@ struct TypeTraits<DoubleType> {
   using BuilderType = DoubleBuilder;
 
   static inline int bytes_required(int elements) { return elements * sizeof(double); }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return float64(); }
 };
 
 template <>
@@ -138,18 +163,24 @@ struct TypeTraits<BooleanType> {
   static inline int bytes_required(int elements) {
     return BitUtil::BytesForBits(elements);
   }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return boolean(); }
 };
 
 template <>
 struct TypeTraits<StringType> {
   using ArrayType = StringArray;
   using BuilderType = StringBuilder;
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return utf8(); }
 };
 
 template <>
 struct TypeTraits<BinaryType> {
   using ArrayType = BinaryArray;
   using BuilderType = BinaryBuilder;
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return binary(); }
 };
 
 // Not all type classes have a c_type

From c45c3b3e11e328a6fdd50d7e1577eb3ba6ab9f93 Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent@dremio.com>
Date: Sat, 4 Feb 2017 18:34:40 -0500
Subject: [PATCH 0309/1644] ARROW-527: Remove drill-module.conf file

Remove drill-module.conf file as it is not used by the project.

Author: Laurent Goujon <laurent@dremio.com>

Closes #318 from laurentgo/laurent/ARROW-527 and squashes the following commits:

7cd384d [Laurent Goujon] ARROW-527: Remove drill-module.conf file
---
 .../src/main/resources/drill-module.conf      | 25 -------------------
 1 file changed, 25 deletions(-)
 delete mode 100644 java/memory/src/main/resources/drill-module.conf

diff --git a/java/memory/src/main/resources/drill-module.conf b/java/memory/src/main/resources/drill-module.conf
deleted file mode 100644
index 593ef8e41e76b..0000000000000
--- a/java/memory/src/main/resources/drill-module.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//  This file tells Drill to consider this module when class path scanning.
-//  This file can also include any supplementary configuration information.
-//  This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
-drill: {
-  memory: {
-    debug.error_on_leak: true,
-    top.max: 1000000000000
-  }
-
-}

From 70c05be2130bdbb650a83bc46f7c4f8fc8a231df Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Sun, 5 Feb 2017 14:06:37 +0100
Subject: [PATCH 0310/1644] ARROW-524: provide apis to access nested vectors
 and buffers

Author: Julien Le Dem <julien@dremio.com>

Closes #314 from julienledem/setRangeToOne and squashes the following commits:

0d526bd [Julien Le Dem] ARROW-524: provide apis to access nested vectors and buffers
---
 .../templates/NullableValueVectors.java       | 21 +++--
 .../org/apache/arrow/vector/BitVector.java    | 88 ++++++++++++++++++-
 .../apache/arrow/vector/NullableVector.java   |  2 +
 .../apache/arrow/vector/TestValueVector.java  | 36 ++++++++
 4 files changed, 137 insertions(+), 10 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index ce637100cd8bf..6b25fb36b40c0 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -131,6 +131,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   }
   </#if>
 
+  @Override
+  public BitVector getValidityVector() {
+    return bits;
+  }
+
   @Override
   public List<BufferBacked> getFieldInnerVectors() {
     return innerVectors;
@@ -426,7 +431,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from){
     mutator.fillEmpties(thisIndex);
     </#if>
     values.copyFromSafe(fromIndex, thisIndex, from);
-    bits.getMutator().setSafe(thisIndex, 1);
+    bits.getMutator().setSafeToOne(thisIndex);
   }
 
   public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
@@ -525,7 +530,7 @@ private Mutator(){
 
     @Override
     public void setIndexDefined(int index){
-      bits.getMutator().set(index, 1);
+      bits.getMutator().setToOne(index);
     }
 
     /**
@@ -543,7 +548,7 @@ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.widt
         valuesMutator.set(i, emptyByteArray);
       }
       </#if>
-      bitsMutator.set(index, 1);
+      bitsMutator.setToOne(index);
       valuesMutator.set(index, value);
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
@@ -574,7 +579,7 @@ public void setSafe(int index, byte[] value, int start, int length) {
       <#else>
       fillEmpties(index);
 
-      bits.getMutator().setSafe(index, 1);
+      bits.getMutator().setSafeToOne(index);
       values.getMutator().setSafe(index, value, start, length);
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
@@ -587,7 +592,7 @@ public void setSafe(int index, ByteBuffer value, int start, int length) {
       <#else>
       fillEmpties(index);
 
-      bits.getMutator().setSafe(index, 1);
+      bits.getMutator().setSafeToOne(index);
       values.getMutator().setSafe(index, value, start, length);
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
@@ -626,7 +631,7 @@ public void set(int index, ${minor.class}Holder holder){
         valuesMutator.set(i, emptyByteArray);
       }
       </#if>
-      bits.getMutator().set(index, 1);
+      bits.getMutator().setToOne(index);
       valuesMutator.set(index, holder);
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
@@ -676,7 +681,7 @@ public void setSafe(int index, ${minor.class}Holder value) {
       <#if type.major == "VarLen">
       fillEmpties(index);
       </#if>
-      bits.getMutator().setSafe(index, 1);
+      bits.getMutator().setSafeToOne(index);
       values.getMutator().setSafe(index, value);
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
@@ -687,7 +692,7 @@ public void setSafe(int index, ${minor.javaType!type.javaType} value) {
         <#if type.major == "VarLen">
         fillEmpties(index);
         </#if>
-        bits.getMutator().setSafe(index, 1);
+        bits.getMutator().setSafeToOne(index);
         values.getMutator().setSafe(index, value);
         setCount++;
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 9beabcbe46bcc..d1e9abe5dd111 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -423,8 +423,8 @@ private Mutator() {
      *          value to set (either 1 or 0)
      */
     public final void set(int index, int value) {
-      int byteIndex = index >> 3;
-      int bitIndex = index & 7;
+      int byteIndex = byteIndex(index);
+      int bitIndex = bitIndex(index);
       byte currentByte = data.getByte(byteIndex);
       byte bitMask = (byte) (1L << bitIndex);
       if (value != 0) {
@@ -432,10 +432,87 @@ public final void set(int index, int value) {
       } else {
         currentByte -= (bitMask & currentByte);
       }
+      data.setByte(byteIndex, currentByte);
+    }
 
+    /**
+     * Set the bit at the given index to 1.
+     *
+     * @param index position of the bit to set
+     */
+    public final void setToOne(int index) {
+      int byteIndex = byteIndex(index);
+      int bitIndex = bitIndex(index);
+      byte currentByte = data.getByte(byteIndex);
+      byte bitMask = (byte) (1L << bitIndex);
+      currentByte |= bitMask;
       data.setByte(byteIndex, currentByte);
     }
 
+    /**
+     * set count bits to 1 in data starting at firstBitIndex
+     * @param data the buffer to set
+     * @param firstBitIndex the index of the first bit to set
+     * @param count the number of bits to set
+     */
+    public void setRangeToOne(int firstBitIndex, int count) {
+      int starByteIndex = byteIndex(firstBitIndex);
+      final int lastBitIndex = firstBitIndex + count;
+      final int endByteIndex = byteIndex(lastBitIndex);
+      final int startByteBitIndex = bitIndex(firstBitIndex);
+      final int endBytebitIndex = bitIndex(lastBitIndex);
+      if (count < 8 && starByteIndex == endByteIndex) {
+        // handles the case where we don't have a first and a last byte
+        byte bitMask = 0;
+        for (int i = startByteBitIndex; i < endBytebitIndex; ++i) {
+          bitMask |= (byte) (1L << i);
+        }
+        byte currentByte = data.getByte(starByteIndex);
+        currentByte |= bitMask;
+        data.setByte(starByteIndex, currentByte);
+      } else {
+        // fill in first byte (if it's not full)
+        if (startByteBitIndex != 0) {
+          byte currentByte = data.getByte(starByteIndex);
+          final byte bitMask = (byte) (0xFFL << startByteBitIndex);
+          currentByte |= bitMask;
+          data.setByte(starByteIndex, currentByte);
+          ++ starByteIndex;
+        }
+
+        // fill in one full byte at a time
+        for (int i = starByteIndex; i < endByteIndex; i++) {
+          data.setByte(i, 0xFF);
+        }
+
+        // fill in the last byte (if it's not full)
+        if (endBytebitIndex != 0) {
+          final int byteIndex = byteIndex(lastBitIndex - endBytebitIndex);
+          byte currentByte = data.getByte(byteIndex);
+          final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7));
+          currentByte |= bitMask;
+          data.setByte(byteIndex, currentByte);
+        }
+
+      }
+    }
+
+    /**
+     * @param absoluteBitIndex the index of the bit in the buffer
+     * @return the index of the byte containing that bit
+     */
+    private int byteIndex(int absoluteBitIndex) {
+      return absoluteBitIndex >> 3;
+    }
+
+    /**
+     * @param absoluteBitIndex the index of the bit in the buffer
+     * @return the index of the bit inside the byte
+     */
+    private int bitIndex(int absoluteBitIndex) {
+      return absoluteBitIndex & 7;
+    }
+
     public final void set(int index, BitHolder holder) {
       set(index, holder.value);
     }
@@ -451,6 +528,13 @@ public void setSafe(int index, int value) {
       set(index, value);
     }
 
+    public void setSafeToOne(int index) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      setToOne(index);
+    }
+
     public void setSafe(int index, BitHolder holder) {
       while(index >= getValueCapacity()) {
         reAlloc();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
index 0212b3c0d7b95..b49e9167c2589 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
@@ -19,5 +19,7 @@
 
 public interface NullableVector extends ValueVector {
 
+  BitVector getValidityVector();
+
   ValueVector getValuesVector();
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index b33919b2790fc..774b59e3683e3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -30,6 +30,7 @@
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -364,6 +365,41 @@ public void testBitVector() {
     }
   }
 
+  @Test
+  public void testBitVectorRangeSetAllOnes() {
+    validateRange(1000, 0, 1000);
+    validateRange(1000, 0, 1);
+    validateRange(1000, 1, 2);
+    validateRange(1000, 5, 6);
+    validateRange(1000, 5, 10);
+    validateRange(1000, 5, 150);
+    validateRange(1000, 5, 27);
+    for (int i = 0; i < 8; i++) {
+      for (int j = 0; j < 8; j++) {
+        validateRange(1000, 10 + i, 27 + j);
+        validateRange(1000, i, j);
+      }
+    }
+  }
+
+  private void validateRange(int length, int start, int count) {
+    String desc = "[" + start + ", "  + (start + count) + ") ";
+    try (BitVector bitVector = new BitVector("bits", allocator)) {
+      bitVector.reset();
+      bitVector.allocateNew(length);
+      bitVector.getMutator().setRangeToOne(start, count);
+      for (int i = 0; i < start; i++) {
+        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+      }
+      for (int i = start; i < start + count; i++) {
+        Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i));
+      }
+      for (int i = start + count; i < length; i++) {
+        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+      }
+    }
+  }
+
   @Test
   public void testReAllocNullableFixedWidthVector() {
     // Create a new value vector for 1024 integers

From 5bee596caf6f26b0f10a2c384f025bbaab43e27e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 5 Feb 2017 15:27:39 -0500
Subject: [PATCH 0311/1644] ARROW-529: Python: Add jemalloc and Python 3.6 to
 manylinux1 build

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #319 from xhochy/ARROW-529 and squashes the following commits:

48893a2 [Uwe L. Korn] ARROW-529: Python: Add jemalloc and Python 3.6 to manylinux1 build
---
 python/CMakeLists.txt                                |  2 +-
 .../manylinux1/Dockerfile-parquet_arrow-base-x86_64  |  2 +-
 python/manylinux1/Dockerfile-x86_64                  | 11 ++++++++++-
 python/manylinux1/build_arrow.sh                     | 12 +++++++++---
 4 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 898c48ee0e48d..842a2196dab62 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -352,7 +352,7 @@ set(PYARROW_MIN_TEST_LIBS
   pyarrow
   ${PYARROW_BASE_LIBS})
 
-if(NOT APPLE)
+if(NOT APPLE AND PYARROW_BUILD_TESTS)
   ADD_THIRDPARTY_LIB(python
     SHARED_LIB "${PYTHON_LIBRARIES}")
   list(APPEND PYARROW_MIN_TEST_LIBS python)
diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
index 94f5bc0f3b66e..dcc9321c322b2 100644
--- a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
+++ b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
@@ -15,5 +15,5 @@ FROM arrow-base-x86_64
 WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/usr /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
+RUN ARROW_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
 RUN make -j5 install
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 29e00b0ccbe49..059158856f1f2 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -22,9 +22,18 @@ WORKDIR /boost_1_60_0
 RUN ./bootstrap.sh
 RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install
 
+WORKDIR /
+RUN wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2
+RUN tar xf jemalloc-4.4.0.tar.bz2
+WORKDIR /jemalloc-4.4.0
+RUN ./configure
+RUN make -j5
+RUN make install
+
 WORKDIR /
 # Install cmake manylinux1 package
 RUN /opt/python/cp35-cp35m/bin/pip install cmake
+RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
 
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git
@@ -34,5 +43,5 @@ RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
-RUN /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON .
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON .
 RUN make -j5 install
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 7e2ad58617793..cce5cd2b4d412 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -20,7 +20,7 @@
 # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
 # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
 
-PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5}"
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5 3.6}"
 
 # Package index with only manylinux1 builds
 MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
@@ -29,9 +29,10 @@ source /multibuild/manylinux_utils.sh
 
 cd /arrow/python
 
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
 # PyArrow build configuration
 export PYARROW_BUILD_TYPE='release'
-export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_PARQUET=ON'
+export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_TESTS=ON'
 # Need as otherwise arrow_io is sometimes not linked
 export LDFLAGS="-Wl,--no-as-needed"
 export ARROW_HOME="/usr"
@@ -69,10 +70,15 @@ for PYTHON in ${PYTHON_VERSIONS}; do
 
     $PIPI_IO "numpy==1.9.0"
     $PIPI_IO "cython==0.24"
-    $PIPI_IO "cmake"
 
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-jemalloc
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
+    # Test for optional modules
+    $PIPI_IO -r requirements.txt
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.jemalloc"
+
     repair_wheelhouse dist /io/dist
 done
 

From 74bc4dd480d6153cf1fb5d6fb7cdbb22d1e6e5d9 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 5 Feb 2017 15:29:11 -0500
Subject: [PATCH 0312/1644] ARROW-511: Python: Implement List conversions for
 single arrays

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #320 from xhochy/ARROW-511 and squashes the following commits:

2ff63f9 [Uwe L. Korn] Use _check_pandas_roundtrip
6c8fa6d [Uwe L. Korn] Python: Implement List conversions for single arrays
---
 python/pyarrow/tests/test_convert_pandas.py |  7 ++++-
 python/src/pyarrow/adapters/pandas.cc       | 31 +++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index ddbb02a770c35..f04fbe5b139e4 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -22,6 +22,7 @@
 import unittest
 
 import numpy as np
+import numpy.testing as npt
 
 import pandas as pd
 import pandas.util.testing as tm
@@ -80,7 +81,7 @@ def _check_array_roundtrip(self, values, expected=None,
         arr = A.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
                                   field=field)
         result = arr.to_pandas()
-        tm.assert_series_equal(pd.Series(result), pd.Series(values))
+        tm.assert_series_equal(pd.Series(result), pd.Series(values), check_names=False)
 
     def test_float_no_nulls(self):
         data = {}
@@ -332,6 +333,10 @@ def test_column_of_lists(self):
         table = A.Table.from_pandas(df, schema=schema)
         assert table.schema.equals(schema)
 
+        for column in df.columns:
+            field = schema.field_by_name(column)
+            self._check_array_roundtrip(df[column], field=field)
+
     def test_threaded_conversion(self):
         df = _alltypes_example()
         self._check_pandas_roundtrip(df, nthreads=2,
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 920779fe86174..8d05821c2fd08 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -1817,6 +1817,7 @@ class ArrowDeserializer {
       CONVERT_CASE(DATE);
       CONVERT_CASE(TIMESTAMP);
       CONVERT_CASE(DICTIONARY);
+      CONVERT_CASE(LIST);
       default: {
         std::stringstream ss;
         ss << "Arrow type reading not implemented for " << col_->type()->ToString();
@@ -1914,6 +1915,36 @@ class ArrowDeserializer {
     return ConvertBinaryLike<arrow::BinaryArray>(data_, out_values);
   }
 
+#define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \
+  case Type::ArrowEnum:                                    \
+    return ConvertListsLike<::arrow::ArrowType>(col_, out_values);
+
+  template <int T2>
+  inline typename std::enable_if<T2 == Type::LIST, Status>::type ConvertValues() {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    auto list_type = std::static_pointer_cast<ListType>(col_->type());
+    switch (list_type->value_type()->type) {
+      CONVERTVALUES_LISTSLIKE_CASE(UInt8Type, UINT8)
+      CONVERTVALUES_LISTSLIKE_CASE(Int8Type, INT8)
+      CONVERTVALUES_LISTSLIKE_CASE(UInt16Type, UINT16)
+      CONVERTVALUES_LISTSLIKE_CASE(Int16Type, INT16)
+      CONVERTVALUES_LISTSLIKE_CASE(UInt32Type, UINT32)
+      CONVERTVALUES_LISTSLIKE_CASE(Int32Type, INT32)
+      CONVERTVALUES_LISTSLIKE_CASE(UInt64Type, UINT64)
+      CONVERTVALUES_LISTSLIKE_CASE(Int64Type, INT64)
+      CONVERTVALUES_LISTSLIKE_CASE(TimestampType, TIMESTAMP)
+      CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT)
+      CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE)
+      CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING)
+      default: {
+        std::stringstream ss;
+        ss << "Not implemented type for lists: " << list_type->value_type()->ToString();
+        return Status::NotImplemented(ss.str());
+      }
+    }
+  }
+
   template <int TYPE>
   inline typename std::enable_if<TYPE == Type::DICTIONARY, Status>::type ConvertValues() {
     std::shared_ptr<PandasBlock> block;

From 5439b71586f4b0f9a36544b9e2417ee6ad7b48e8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 6 Feb 2017 11:25:18 -0500
Subject: [PATCH 0313/1644] ARROW-33: [C++] Implement zero-copy array slicing,
 integrate with IPC code paths

This turned into a bit of a refactoring bloodbath. I have sorted through most of the issues that this turned up, so I should have this all completely working within a day or so. There will be some follow up work to do to polish things up

Closes #56.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #322 from wesm/ARROW-33 and squashes the following commits:

61afe42 [Wes McKinney] Some API cleaning in builder.h
86511a3 [Wes McKinney] Python fixes, clang warning fixes
9a00870 [Wes McKinney] Make ApproxEquals for floating point arrays work on slices
2a13929 [Wes McKinney] Implement slicing IPC logic for dense array
4f08628 [Wes McKinney] Add missing include
1a6fcb4 [Wes McKinney] Make some more progress. dense union needs more work
c6d814d [Wes McKinney] Work on adding sliced array support to IPC code path, with pretty printer and comparison fixed for sliced bitmaps, etc. Not all working yet
b6c511e [Wes McKinney] Add RecordBatch::Slice convenience method
8900d58 [Wes McKinney] Add Slice tests for DictionaryArray. Test recomputing the null count
55454d7 [Wes McKinney] Add slice tests for struct, union, string, list
a72653d [Wes McKinney] Rename offsets to value_offsets in list/binary/string/union for better clarity. Test Slice for primitive arrays
0355f71 [Wes McKinney] Implement CopyBitmap function
a228b50 [Wes McKinney] Implement Slice methods on Array classes
e502901 [Wes McKinney] Move null_count and offset as last two parameters of all array ctors. Implement/test bitmap set bit count with offset
bae6922 [Wes McKinney] Temporary work on adding offset parameter to Array classes for slicing
---
 cpp/src/arrow/CMakeLists.txt               |   1 +
 cpp/src/arrow/array-dictionary-test.cc     |  62 +++--
 cpp/src/arrow/array-list-test.cc           |  36 ++-
 cpp/src/arrow/array-primitive-test.cc      |  78 +++++-
 cpp/src/arrow/array-string-test.cc         |  90 +++++--
 cpp/src/arrow/array-struct-test.cc         |  19 +-
 cpp/src/arrow/array-test.cc                |  32 ++-
 cpp/src/arrow/array-union-test.cc          |  67 ++++++
 cpp/src/arrow/array.cc                     | 233 ++++++++++++------
 cpp/src/arrow/array.h                      | 265 +++++++++++++--------
 cpp/src/arrow/buffer.cc                    |  16 ++
 cpp/src/arrow/buffer.h                     |  21 +-
 cpp/src/arrow/builder.cc                   |  64 ++---
 cpp/src/arrow/builder.h                    |  21 +-
 cpp/src/arrow/column-test.cc               |  14 +-
 cpp/src/arrow/compare.cc                   | 122 +++++++---
 cpp/src/arrow/io/file.cc                   |   4 +-
 cpp/src/arrow/io/hdfs.cc                   |   8 +-
 cpp/src/arrow/io/io-hdfs-test.cc           |  10 +-
 cpp/src/arrow/io/io-memory-test.cc         |   4 +-
 cpp/src/arrow/ipc/adapter.cc               | 260 ++++++++++++++++----
 cpp/src/arrow/ipc/adapter.h                |   8 +-
 cpp/src/arrow/ipc/ipc-adapter-test.cc      |  52 +++-
 cpp/src/arrow/ipc/ipc-json-test.cc         |  21 +-
 cpp/src/arrow/ipc/json-integration-test.cc |   6 +-
 cpp/src/arrow/ipc/json-internal.cc         |  37 +--
 cpp/src/arrow/ipc/stream.cc                |  15 +-
 cpp/src/arrow/ipc/stream.h                 |   8 +
 cpp/src/arrow/ipc/test-common.h            |  79 +++---
 cpp/src/arrow/pretty_print-test.cc         |   6 +-
 cpp/src/arrow/pretty_print.cc              |  53 +++--
 cpp/src/arrow/table-test.cc                |  26 ++
 cpp/src/arrow/table.cc                     |  19 +-
 cpp/src/arrow/table.h                      |   4 +
 cpp/src/arrow/test-util.h                  |  43 +---
 cpp/src/arrow/type.cc                      |   6 +-
 cpp/src/arrow/type.h                       |   8 +-
 cpp/src/arrow/type_traits.h                |   9 +
 cpp/src/arrow/util/bit-util-test.cc        |  62 ++++-
 cpp/src/arrow/util/bit-util.cc             |  83 ++++++-
 cpp/src/arrow/util/bit-util.h              |  45 ++++
 cpp/src/arrow/util/logging.h               |   4 +-
 cpp/src/arrow/util/macros.h                |   2 +-
 python/CMakeLists.txt                      |   2 +-
 python/pyarrow/includes/libarrow.pxd       |   4 +-
 python/pyarrow/scalar.pyx                  |   2 +-
 python/src/pyarrow/adapters/builtin.cc     |   2 +-
 python/src/pyarrow/adapters/pandas.cc      |  20 +-
 python/src/pyarrow/io.cc                   |  21 +-
 49 files changed, 1524 insertions(+), 550 deletions(-)
 create mode 100644 cpp/src/arrow/array-union-test.cc

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index b002bb75ca934..824ced1a51eb9 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -53,6 +53,7 @@ ADD_ARROW_TEST(array-list-test)
 ADD_ARROW_TEST(array-primitive-test)
 ADD_ARROW_TEST(array-string-test)
 ADD_ARROW_TEST(array-struct-test)
+ADD_ARROW_TEST(array-union-test)
 ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(column-test)
 ADD_ARROW_TEST(memory_pool-test)
diff --git a/cpp/src/arrow/array-dictionary-test.cc b/cpp/src/arrow/array-dictionary-test.cc
index 1a0d49a118f50..61381b7671180 100644
--- a/cpp/src/arrow/array-dictionary-test.cc
+++ b/cpp/src/arrow/array-dictionary-test.cc
@@ -34,7 +34,7 @@ namespace arrow {
 TEST(TestDictionary, Basics) {
   std::vector<int32_t> values = {100, 1000, 10000, 100000};
   std::shared_ptr<Array> dict;
-  ArrayFromVector<Int32Type, int32_t>(int32(), values, &dict);
+  ArrayFromVector<Int32Type, int32_t>(values, &dict);
 
   std::shared_ptr<DictionaryType> type1 =
       std::dynamic_pointer_cast<DictionaryType>(dictionary(int16(), dict));
@@ -54,45 +54,67 @@ TEST(TestDictionary, Equals) {
 
   std::shared_ptr<Array> dict;
   std::vector<std::string> dict_values = {"foo", "bar", "baz"};
-  ArrayFromVector<StringType, std::string>(utf8(), dict_values, &dict);
+  ArrayFromVector<StringType, std::string>(dict_values, &dict);
   std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
 
   std::shared_ptr<Array> dict2;
   std::vector<std::string> dict2_values = {"foo", "bar", "baz", "qux"};
-  ArrayFromVector<StringType, std::string>(utf8(), dict2_values, &dict2);
+  ArrayFromVector<StringType, std::string>(dict2_values, &dict2);
   std::shared_ptr<DataType> dict2_type = dictionary(int16(), dict2);
 
   std::shared_ptr<Array> indices;
   std::vector<int16_t> indices_values = {1, 2, -1, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices_values, &indices);
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices_values, &indices);
 
   std::shared_ptr<Array> indices2;
   std::vector<int16_t> indices2_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices2_values, &indices2);
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices2_values, &indices2);
 
   std::shared_ptr<Array> indices3;
   std::vector<int16_t> indices3_values = {1, 1, 0, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices3_values, &indices3);
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices3_values, &indices3);
 
-  auto arr = std::make_shared<DictionaryArray>(dict_type, indices);
-  auto arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
-  auto arr3 = std::make_shared<DictionaryArray>(dict2_type, indices);
-  auto arr4 = std::make_shared<DictionaryArray>(dict_type, indices3);
+  auto array = std::make_shared<DictionaryArray>(dict_type, indices);
+  auto array2 = std::make_shared<DictionaryArray>(dict_type, indices2);
+  auto array3 = std::make_shared<DictionaryArray>(dict2_type, indices);
+  auto array4 = std::make_shared<DictionaryArray>(dict_type, indices3);
 
-  ASSERT_TRUE(arr->Equals(arr));
+  ASSERT_TRUE(array->Equals(array));
 
   // Equal, because the unequal index is masked by null
-  ASSERT_TRUE(arr->Equals(arr2));
+  ASSERT_TRUE(array->Equals(array2));
 
   // Unequal dictionaries
-  ASSERT_FALSE(arr->Equals(arr3));
+  ASSERT_FALSE(array->Equals(array3));
 
   // Unequal indices
-  ASSERT_FALSE(arr->Equals(arr4));
+  ASSERT_FALSE(array->Equals(array4));
 
   // RangeEquals
-  ASSERT_TRUE(arr->RangeEquals(3, 6, 3, arr4));
-  ASSERT_FALSE(arr->RangeEquals(1, 3, 1, arr4));
+  ASSERT_TRUE(array->RangeEquals(3, 6, 3, array4));
+  ASSERT_FALSE(array->RangeEquals(1, 3, 1, array4));
+
+  // ARROW-33 Test slices
+  const int size = array->length();
+
+  std::shared_ptr<Array> slice, slice2;
+  slice = array->Array::Slice(2);
+  slice2 = array->Array::Slice(2);
+  ASSERT_EQ(size - 2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(2, array->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Array::Slice(1)->Array::Slice(1);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(1, 3);
+  slice2 = array->Slice(1, 3);
+  ASSERT_EQ(3, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 4, 0, slice));
 }
 
 TEST(TestDictionary, Validate) {
@@ -100,20 +122,20 @@ TEST(TestDictionary, Validate) {
 
   std::shared_ptr<Array> dict;
   std::vector<std::string> dict_values = {"foo", "bar", "baz"};
-  ArrayFromVector<StringType, std::string>(utf8(), dict_values, &dict);
+  ArrayFromVector<StringType, std::string>(dict_values, &dict);
   std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
 
   std::shared_ptr<Array> indices;
   std::vector<uint8_t> indices_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<UInt8Type, uint8_t>(uint8(), is_valid, indices_values, &indices);
+  ArrayFromVector<UInt8Type, uint8_t>(is_valid, indices_values, &indices);
 
   std::shared_ptr<Array> indices2;
   std::vector<float> indices2_values = {1., 2., 0., 0., 2., 0.};
-  ArrayFromVector<FloatType, float>(float32(), is_valid, indices2_values, &indices2);
+  ArrayFromVector<FloatType, float>(is_valid, indices2_values, &indices2);
 
   std::shared_ptr<Array> indices3;
   std::vector<int64_t> indices3_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, indices3_values, &indices3);
+  ArrayFromVector<Int64Type, int64_t>(is_valid, indices3_values, &indices3);
 
   std::shared_ptr<Array> arr = std::make_shared<DictionaryArray>(dict_type, indices);
   std::shared_ptr<Array> arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
diff --git a/cpp/src/arrow/array-list-test.cc b/cpp/src/arrow/array-list-test.cc
index 8e4d319f5dca8..a144fd937d7a0 100644
--- a/cpp/src/arrow/array-list-test.cc
+++ b/cpp/src/arrow/array-list-test.cc
@@ -90,9 +90,9 @@ TEST_F(TestListBuilder, Equality) {
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
 
   std::shared_ptr<Array> array, equal_array, unequal_array;
-  vector<int32_t> equal_offsets = {0, 1, 2, 5};
-  vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2};
-  vector<int32_t> unequal_offsets = {0, 1, 4};
+  vector<int32_t> equal_offsets = {0, 1, 2, 5, 6, 7, 8, 10};
+  vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2, 5, 6};
+  vector<int32_t> unequal_offsets = {0, 1, 4, 7};
   vector<int32_t> unequal_values = {1, 2, 2, 2, 3, 4, 5};
 
   // setup two equal arrays
@@ -122,7 +122,27 @@ TEST_F(TestListBuilder, Equality) {
   EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_array));
   EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
   EXPECT_TRUE(array->RangeEquals(2, 3, 2, unequal_array));
-  EXPECT_TRUE(array->RangeEquals(3, 4, 1, unequal_array));
+
+  // Check with slices, ARROW-33
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(2);
+  slice2 = array->Slice(2);
+  ASSERT_EQ(array->length() - 2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(2, slice->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Slice(1)->Slice(1);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(1, 4);
+  slice2 = array->Slice(1, 4);
+  ASSERT_EQ(4, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 5, 0, slice));
 }
 
 TEST_F(TestListBuilder, TestResize) {}
@@ -137,9 +157,9 @@ TEST_F(TestListBuilder, TestAppendNull) {
   ASSERT_TRUE(result_->IsNull(0));
   ASSERT_TRUE(result_->IsNull(1));
 
-  ASSERT_EQ(0, result_->raw_offsets()[0]);
-  ASSERT_EQ(0, result_->offset(1));
-  ASSERT_EQ(0, result_->offset(2));
+  ASSERT_EQ(0, result_->raw_value_offsets()[0]);
+  ASSERT_EQ(0, result_->value_offset(1));
+  ASSERT_EQ(0, result_->value_offset(2));
 
   Int32Array* values = static_cast<Int32Array*>(result_->values().get());
   ASSERT_EQ(0, values->length());
@@ -154,7 +174,7 @@ void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& valu
   ASSERT_EQ(3, result->length());
   vector<int32_t> ex_offsets = {0, 3, 3, 7};
   for (size_t i = 0; i < ex_offsets.size(); ++i) {
-    ASSERT_EQ(ex_offsets[i], result->offset(i));
+    ASSERT_EQ(ex_offsets[i], result->value_offset(i));
   }
 
   for (int i = 0; i < result->length(); ++i) {
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
index c839fb9b19234..a20fdbf8b9166 100644
--- a/cpp/src/arrow/array-primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -121,7 +121,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     }
 
     auto expected =
-        std::make_shared<ArrayType>(size, ex_data, ex_null_count, ex_null_bitmap);
+        std::make_shared<ArrayType>(size, ex_data, ex_null_bitmap, ex_null_count);
 
     std::shared_ptr<Array> out;
     ASSERT_OK(builder->Finish(&out));
@@ -217,7 +217,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
   }
 
   auto expected =
-      std::make_shared<BooleanArray>(size, ex_data, ex_null_count, ex_null_bitmap);
+      std::make_shared<BooleanArray>(size, ex_data, ex_null_bitmap, ex_null_count);
 
   std::shared_ptr<Array> out;
   ASSERT_OK(builder->Finish(&out));
@@ -235,15 +235,14 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 
   for (int i = 0; i < result->length(); ++i) {
     if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
-    bool actual = BitUtil::GetBit(result->raw_data(), i);
+    bool actual = BitUtil::GetBit(result->data()->data(), i);
     ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
   }
   ASSERT_TRUE(result->Equals(*expected));
 }
 
 typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
-    PInt32, PInt64, PFloat, PDouble>
-    Primitives;
+    PInt32, PInt64, PFloat, PDouble> Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
@@ -347,6 +346,39 @@ TYPED_TEST(TestPrimitiveBuilder, Equality) {
       array->RangeEquals(first_valid_idx + 1, size, first_valid_idx + 1, unequal_array));
 }
 
+TYPED_TEST(TestPrimitiveBuilder, SliceEquality) {
+  DECL_T();
+
+  const int size = 1000;
+  this->RandomData(size);
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
+  auto builder = this->builder_.get();
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
+
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(5);
+  slice2 = array->Slice(5);
+  ASSERT_EQ(size - 5, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, array->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Slice(2)->Slice(3);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(5, 10);
+  slice2 = array->Slice(5, 10);
+  ASSERT_EQ(10, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, 15, 0, slice));
+}
+
 TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   DECL_T();
 
@@ -473,4 +505,40 @@ TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
   ASSERT_EQ(BitUtil::NextPower2(kMinBuilderCapacity + 100), this->builder_->capacity());
 }
 
+template <typename TYPE>
+void CheckSliceApproxEquals() {
+  using T = typename TYPE::c_type;
+
+  const int kSize = 50;
+  std::vector<T> draws1;
+  std::vector<T> draws2;
+
+  const uint32_t kSeed = 0;
+  test::random_real<T>(kSize, kSeed, 0, 100, &draws1);
+  test::random_real<T>(kSize, kSeed + 1, 0, 100, &draws2);
+
+  // Make the draws equal in the sliced segment, but unequal elsewhere (to
+  // catch not using the slice offset)
+  for (int i = 10; i < 30; ++i) {
+    draws2[i] = draws1[i];
+  }
+
+  std::vector<bool> is_valid;
+  test::random_is_valid(kSize, 0.1, &is_valid);
+
+  std::shared_ptr<Array> array1, array2;
+  ArrayFromVector<TYPE, T>(is_valid, draws1, &array1);
+  ArrayFromVector<TYPE, T>(is_valid, draws2, &array2);
+
+  std::shared_ptr<Array> slice1 = array1->Slice(10, 20);
+  std::shared_ptr<Array> slice2 = array2->Slice(10, 20);
+
+  ASSERT_TRUE(slice1->ApproxEquals(slice2));
+}
+
+TEST(TestPrimitiveAdHoc, FloatingSliceApproxEquals) {
+  CheckSliceApproxEquals<FloatType>();
+  CheckSliceApproxEquals<DoubleType>();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index 5ea384acb1c57..8b7eb41d4c3b9 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -27,6 +27,7 @@
 #include "arrow/builder.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
+#include "arrow/type_traits.h"
 
 namespace arrow {
 
@@ -70,7 +71,7 @@ class TestStringArray : public ::testing::Test {
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<StringArray>(
-        length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
+        length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
   }
 
  protected:
@@ -114,7 +115,7 @@ TEST_F(TestStringArray, TestListFunctions) {
 
 TEST_F(TestStringArray, TestDestructor) {
   auto arr = std::make_shared<StringArray>(
-      length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
+      length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
 }
 
 TEST_F(TestStringArray, TestGetString) {
@@ -133,9 +134,9 @@ TEST_F(TestStringArray, TestEmptyStringComparison) {
   length_ = offsets_.size() - 1;
 
   auto strings_a = std::make_shared<StringArray>(
-      length_, offsets_buf_, nullptr, null_count_, null_bitmap_);
+      length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
   auto strings_b = std::make_shared<StringArray>(
-      length_, offsets_buf_, nullptr, null_count_, null_bitmap_);
+      length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
   ASSERT_TRUE(strings_a->Equals(strings_b));
 }
 
@@ -146,8 +147,7 @@ class TestStringBuilder : public TestBuilder {
  public:
   void SetUp() {
     TestBuilder::SetUp();
-    type_ = TypePtr(new StringType());
-    builder_.reset(new StringBuilder(pool_, type_));
+    builder_.reset(new StringBuilder(pool_));
   }
 
   void Done() {
@@ -159,8 +159,6 @@ class TestStringBuilder : public TestBuilder {
   }
 
  protected:
-  TypePtr type_;
-
   std::unique_ptr<StringBuilder> builder_;
   std::shared_ptr<StringArray> result_;
 };
@@ -195,7 +193,7 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
     } else {
       ASSERT_FALSE(result_->IsNull(i));
       result_->GetValue(i, &length);
-      ASSERT_EQ(pos, result_->offset(i));
+      ASSERT_EQ(pos, result_->value_offset(i));
       ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
       ASSERT_EQ(strings[i % N], result_->GetString(i));
 
@@ -232,7 +230,7 @@ class TestBinaryArray : public ::testing::Test {
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<BinaryArray>(
-        length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
+        length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
   }
 
  protected:
@@ -276,7 +274,7 @@ TEST_F(TestBinaryArray, TestListFunctions) {
 
 TEST_F(TestBinaryArray, TestDestructor) {
   auto arr = std::make_shared<BinaryArray>(
-      length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
+      length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
 }
 
 TEST_F(TestBinaryArray, TestGetValue) {
@@ -306,8 +304,8 @@ TEST_F(TestBinaryArray, TestEqualsEmptyStrings) {
   ASSERT_OK(builder.Finish(&left_arr));
 
   const BinaryArray& left = static_cast<const BinaryArray&>(*left_arr);
-  std::shared_ptr<Array> right = std::make_shared<BinaryArray>(
-      left.length(), left.offsets(), nullptr, left.null_count(), left.null_bitmap());
+  std::shared_ptr<Array> right = std::make_shared<BinaryArray>(left.length(),
+      left.value_offsets(), nullptr, left.null_bitmap(), left.null_count());
 
   ASSERT_TRUE(left.Equals(right));
   ASSERT_TRUE(left.RangeEquals(0, left.length(), 0, right));
@@ -317,8 +315,7 @@ class TestBinaryBuilder : public TestBuilder {
  public:
   void SetUp() {
     TestBuilder::SetUp();
-    type_ = TypePtr(new BinaryType());
-    builder_.reset(new BinaryBuilder(pool_, type_));
+    builder_.reset(new BinaryBuilder(pool_));
   }
 
   void Done() {
@@ -330,8 +327,6 @@ class TestBinaryBuilder : public TestBuilder {
   }
 
  protected:
-  TypePtr type_;
-
   std::unique_ptr<BinaryBuilder> builder_;
   std::shared_ptr<BinaryArray> result_;
 };
@@ -348,8 +343,7 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
       if (is_null[i]) {
         builder_->AppendNull();
       } else {
-        builder_->Append(
-            reinterpret_cast<const uint8_t*>(strings[i].data()), strings[i].size());
+        builder_->Append(strings[i]);
       }
     }
   }
@@ -377,4 +371,62 @@ TEST_F(TestBinaryBuilder, TestZeroLength) {
   Done();
 }
 
+// ----------------------------------------------------------------------
+// Slice tests
+
+template <typename TYPE>
+void CheckSliceEquality() {
+  using Traits = TypeTraits<TYPE>;
+  using BuilderType = typename Traits::BuilderType;
+
+  BuilderType builder(default_memory_pool());
+
+  std::vector<std::string> strings = {"foo", "", "bar", "baz", "qux", ""};
+  std::vector<uint8_t> is_null = {0, 1, 0, 1, 0, 0};
+
+  int N = strings.size();
+  int reps = 10;
+
+  for (int j = 0; j < reps; ++j) {
+    for (int i = 0; i < N; ++i) {
+      if (is_null[i]) {
+        builder.AppendNull();
+      } else {
+        builder.Append(strings[i]);
+      }
+    }
+  }
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(5);
+  slice2 = array->Slice(5);
+  ASSERT_EQ(N * reps - 5, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, slice->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Slice(2)->Slice(3);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(5, 20);
+  slice2 = array->Slice(5, 20);
+  ASSERT_EQ(20, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, 25, 0, slice));
+}
+
+TEST_F(TestBinaryArray, TestSliceEquality) {
+  CheckSliceEquality<BinaryType>();
+}
+
+TEST_F(TestStringArray, TestSliceEquality) {
+  CheckSliceEquality<BinaryType>();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array-struct-test.cc b/cpp/src/arrow/array-struct-test.cc
index 5827c399dda17..f4e7409a6232a 100644
--- a/cpp/src/arrow/array-struct-test.cc
+++ b/cpp/src/arrow/array-struct-test.cc
@@ -75,7 +75,7 @@ void ValidateBasicStructArray(const StructArray* result,
   ASSERT_EQ(4, list_char_arr->length());
   ASSERT_EQ(10, list_char_arr->values()->length());
   for (size_t i = 0; i < list_offsets.size(); ++i) {
-    ASSERT_EQ(list_offsets[i], list_char_arr->raw_offsets()[i]);
+    ASSERT_EQ(list_offsets[i], list_char_arr->raw_value_offsets()[i]);
   }
   for (size_t i = 0; i < list_values.size(); ++i) {
     ASSERT_EQ(list_values[i], char_arr->Value(i));
@@ -381,6 +381,23 @@ TEST_F(TestStructBuilder, TestEquality) {
   EXPECT_FALSE(array->RangeEquals(0, 1, 0, unequal_values_array));
   EXPECT_TRUE(array->RangeEquals(1, 3, 1, unequal_values_array));
   EXPECT_FALSE(array->RangeEquals(3, 4, 3, unequal_values_array));
+
+  // ARROW-33 Slice / equality
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(2);
+  slice2 = array->Slice(2);
+  ASSERT_EQ(array->length() - 2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(2, slice->length(), 0, slice));
+
+  slice = array->Slice(1, 2);
+  slice2 = array->Slice(1, 2);
+  ASSERT_EQ(2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 3, 0, slice));
 }
 
 TEST_F(TestStructBuilder, TestZeroLength) {
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index a1d8fdfa91e85..45130d8f64004 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -43,7 +43,7 @@ TEST_F(TestArray, TestNullCount) {
   auto data = std::make_shared<PoolBuffer>(pool_);
   auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
 
-  std::unique_ptr<Int32Array> arr(new Int32Array(100, data, 10, null_bitmap));
+  std::unique_ptr<Int32Array> arr(new Int32Array(100, data, null_bitmap, 10));
   ASSERT_EQ(10, arr->null_count());
 
   std::unique_ptr<Int32Array> arr_no_nulls(new Int32Array(100, data));
@@ -67,7 +67,7 @@ std::shared_ptr<Array> MakeArrayFromValidBytes(
   }
 
   std::shared_ptr<Array> arr(
-      new Int32Array(v.size(), value_builder.Finish(), null_count, null_buf));
+      new Int32Array(v.size(), value_builder.Finish(), null_buf, null_count));
   return arr;
 }
 
@@ -87,6 +87,32 @@ TEST_F(TestArray, TestEquality) {
   EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
 }
 
+TEST_F(TestArray, SliceRecomputeNullCount) {
+  std::vector<uint8_t> valid_bytes = {1, 0, 1, 1, 0, 1, 0, 0};
+
+  auto array = MakeArrayFromValidBytes(valid_bytes, pool_);
+
+  ASSERT_EQ(4, array->null_count());
+
+  auto slice = array->Slice(1, 4);
+  ASSERT_EQ(2, slice->null_count());
+
+  slice = array->Slice(4);
+  ASSERT_EQ(1, slice->null_count());
+
+  slice = array->Slice(0);
+  ASSERT_EQ(4, slice->null_count());
+
+  // No bitmap, compute 0
+  std::shared_ptr<MutableBuffer> data;
+  const int kBufferSize = 64;
+  ASSERT_OK(AllocateBuffer(pool_, kBufferSize, &data));
+  memset(data->mutable_data(), 0, kBufferSize);
+
+  auto arr = std::make_shared<Int32Array>(16, data, nullptr, -1);
+  ASSERT_EQ(0, arr->null_count());
+}
+
 TEST_F(TestArray, TestIsNull) {
   // clang-format off
   std::vector<uint8_t> null_bitmap = {1, 0, 1, 1, 0, 1, 0, 0,
@@ -102,7 +128,7 @@ TEST_F(TestArray, TestIsNull) {
 
   std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(null_bitmap);
   std::unique_ptr<Array> arr;
-  arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_count, null_buf));
+  arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_buf, null_count));
 
   ASSERT_EQ(null_count, arr->null_count());
   ASSERT_EQ(5, null_buf->size());
diff --git a/cpp/src/arrow/array-union-test.cc b/cpp/src/arrow/array-union-test.cc
new file mode 100644
index 0000000000000..eb9bd7da31b13
--- /dev/null
+++ b/cpp/src/arrow/array-union-test.cc
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Tests for UnionArray
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/ipc/test-common.h"
+#include "arrow/status.h"
+#include "arrow/table.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+TEST(TestUnionArrayAdHoc, TestSliceEquals) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(ipc::MakeUnion(&batch));
+
+  const int size = batch->num_rows();
+
+  auto CheckUnion = [&size](std::shared_ptr<Array> array) {
+    std::shared_ptr<Array> slice, slice2;
+    slice = array->Slice(2);
+    slice2 = array->Slice(2);
+    ASSERT_EQ(size - 2, slice->length());
+
+    ASSERT_TRUE(slice->Equals(slice2));
+    ASSERT_TRUE(array->RangeEquals(2, array->length(), 0, slice));
+
+    // Chained slices
+    slice2 = array->Slice(1)->Slice(1);
+    ASSERT_TRUE(slice->Equals(slice2));
+
+    slice = array->Slice(1, 5);
+    slice2 = array->Slice(1, 5);
+    ASSERT_EQ(5, slice->length());
+
+    ASSERT_TRUE(slice->Equals(slice2));
+    ASSERT_TRUE(array->RangeEquals(1, 6, 0, slice));
+  };
+
+  CheckUnion(batch->column(1));
+  CheckUnion(batch->column(2));
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 6fc7fb60bf364..f84023e6c7d31 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/array.h"
 
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <sstream>
@@ -30,28 +31,37 @@
 
 namespace arrow {
 
-Status GetEmptyBitmap(
-    MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result) {
-  auto buffer = std::make_shared<PoolBuffer>(pool);
-  RETURN_NOT_OK(buffer->Resize(BitUtil::BytesForBits(length)));
-  memset(buffer->mutable_data(), 0, buffer->size());
-
-  *result = buffer;
-  return Status::OK();
-}
+// When slicing, we do not know the null count of the sliced range without
+// doing some computation. To avoid doing this eagerly, we set the null count
+// to -1 (any negative number will do). When Array::null_count is called the
+// first time, the null count will be computed. See ARROW-33
+constexpr int32_t kUnknownNullCount = -1;
 
 // ----------------------------------------------------------------------
 // Base array class
 
-Array::Array(const std::shared_ptr<DataType>& type, int32_t length, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap) {
-  type_ = type;
-  length_ = length;
-  null_count_ = null_count;
-  null_bitmap_ = null_bitmap;
+Array::Array(const std::shared_ptr<DataType>& type, int32_t length,
+    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
+    : type_(type),
+      length_(length),
+      offset_(offset),
+      null_count_(null_count),
+      null_bitmap_(null_bitmap),
+      null_bitmap_data_(nullptr) {
   if (null_bitmap_) { null_bitmap_data_ = null_bitmap_->data(); }
 }
 
+int32_t Array::null_count() const {
+  if (null_count_ < 0) {
+    if (null_bitmap_) {
+      null_count_ = CountSetBits(null_bitmap_data_, offset_, length_);
+    } else {
+      null_count_ = 0;
+    }
+  }
+  return null_count_;
+}
+
 bool Array::Equals(const Array& arr) const {
   bool are_equal = false;
   Status error = ArrayEquals(*this, arr, &are_equal);
@@ -86,10 +96,32 @@ bool Array::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_
   return are_equal;
 }
 
+// Last two parameters are in-out parameters
+static inline void ConformSliceParams(
+    int32_t array_offset, int32_t array_length, int32_t* offset, int32_t* length) {
+  DCHECK_LE(*offset, array_length);
+  DCHECK_GE(offset, 0);
+  *length = std::min(array_length - *offset, *length);
+  *offset = array_offset + *offset;
+}
+
+std::shared_ptr<Array> Array::Slice(int32_t offset) const {
+  int32_t slice_length = length_ - offset;
+  return Slice(offset, slice_length);
+}
+
 Status Array::Validate() const {
   return Status::OK();
 }
 
+NullArray::NullArray(int32_t length) : Array(null(), length, nullptr, length) {}
+
+std::shared_ptr<Array> NullArray::Slice(int32_t offset, int32_t length) const {
+  DCHECK_LE(offset, length_);
+  length = std::min(length_ - offset, length);
+  return std::make_shared<NullArray>(length);
+}
+
 Status NullArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
@@ -98,9 +130,9 @@ Status NullArray::Accept(ArrayVisitor* visitor) const {
 // Primitive array base
 
 PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : Array(type, length, null_count, null_bitmap) {
+    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
+    int32_t null_count, int32_t offset)
+    : Array(type, length, null_bitmap, null_count, offset) {
   data_ = data;
   raw_data_ = data == nullptr ? nullptr : data_->data();
 }
@@ -110,6 +142,13 @@ Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+template <typename T>
+std::shared_ptr<Array> NumericArray<T>::Slice(int32_t offset, int32_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<NumericArray<T>>(
+      type_, length, data_, null_bitmap_, kUnknownNullCount, offset);
+}
+
 template class NumericArray<UInt8Type>;
 template class NumericArray<UInt16Type>;
 template class NumericArray<UInt32Type>;
@@ -129,32 +168,33 @@ template class NumericArray<DoubleType>;
 // BooleanArray
 
 BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
-    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
-    : PrimitiveArray(
-          std::make_shared<BooleanType>(), length, data, null_count, null_bitmap) {}
-
-BooleanArray::BooleanArray(const std::shared_ptr<DataType>& type, int32_t length,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
+    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
+    : PrimitiveArray(std::make_shared<BooleanType>(), length, data, null_bitmap,
+          null_count, offset) {}
 
 Status BooleanArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+std::shared_ptr<Array> BooleanArray::Slice(int32_t offset, int32_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<BooleanArray>(
+      length, data_, null_bitmap_, kUnknownNullCount, offset);
+}
+
 // ----------------------------------------------------------------------
 // ListArray
 
 Status ListArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
-  if (!offsets_buffer_) { return Status::Invalid("offsets_buffer_ was null"); }
-  if (offsets_buffer_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
+  if (!value_offsets_) { return Status::Invalid("value_offsets_ was null"); }
+  if (value_offsets_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
     std::stringstream ss;
-    ss << "offset buffer size (bytes): " << offsets_buffer_->size()
+    ss << "offset buffer size (bytes): " << value_offsets_->size()
        << " isn't large enough for length: " << length_;
     return Status::Invalid(ss.str());
   }
-  const int32_t last_offset = offset(length_);
+  const int32_t last_offset = this->value_offset(length_);
   if (last_offset > 0) {
     if (!values_) {
       return Status::Invalid("last offset was non-zero and values was null");
@@ -174,14 +214,15 @@ Status ListArray::Validate() const {
     }
   }
 
-  int32_t prev_offset = offset(0);
+  int32_t prev_offset = this->value_offset(0);
   if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
   for (int32_t i = 1; i <= length_; ++i) {
-    int32_t current_offset = offset(i);
+    int32_t current_offset = this->value_offset(i);
     if (IsNull(i - 1) && current_offset != prev_offset) {
       std::stringstream ss;
-      ss << "Offset invariant failure at: " << i << " inconsistent offsets for null slot"
-         << current_offset << "!=" << prev_offset;
+      ss << "Offset invariant failure at: " << i
+         << " inconsistent value_offsets for null slot" << current_offset
+         << "!=" << prev_offset;
       return Status::Invalid(ss.str());
     }
     if (current_offset < prev_offset) {
@@ -200,26 +241,33 @@ Status ListArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+std::shared_ptr<Array> ListArray::Slice(int32_t offset, int32_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<ListArray>(
+      type_, length, value_offsets_, values_, null_bitmap_, kUnknownNullCount, offset);
+}
+
 // ----------------------------------------------------------------------
 // String and binary
 
 static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
 static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
 
-BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : BinaryArray(kBinary, length, offsets, data, null_count, null_bitmap) {}
+BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& value_offsets,
+    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
+    int32_t null_count, int32_t offset)
+    : BinaryArray(kBinary, length, value_offsets, data, null_bitmap, null_count, offset) {
+}
 
 BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int32_t length,
-    const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
-    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
-    : Array(type, length, null_count, null_bitmap),
-      offsets_buffer_(offsets),
-      offsets_(reinterpret_cast<const int32_t*>(offsets_buffer_->data())),
-      data_buffer_(data),
-      data_(nullptr) {
-  if (data_buffer_ != nullptr) { data_ = data_buffer_->data(); }
+    const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Buffer>& data,
+    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
+    : Array(type, length, null_bitmap, null_count, offset),
+      value_offsets_(value_offsets),
+      raw_value_offsets_(reinterpret_cast<const int32_t*>(value_offsets_->data())),
+      data_(data),
+      raw_data_(nullptr) {
+  if (data_ != nullptr) { raw_data_ = data_->data(); }
 }
 
 Status BinaryArray::Validate() const {
@@ -231,10 +279,17 @@ Status BinaryArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap)
-    : BinaryArray(kString, length, offsets, data, null_count, null_bitmap) {}
+std::shared_ptr<Array> BinaryArray::Slice(int32_t offset, int32_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<BinaryArray>(
+      length, value_offsets_, data_, null_bitmap_, kUnknownNullCount, offset);
+}
+
+StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& value_offsets,
+    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
+    int32_t null_count, int32_t offset)
+    : BinaryArray(kString, length, value_offsets, data, null_bitmap, null_count, offset) {
+}
 
 Status StringArray::Validate() const {
   // TODO(emkornfield) Validate proper UTF8 code points?
@@ -245,12 +300,26 @@ Status StringArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+std::shared_ptr<Array> StringArray::Slice(int32_t offset, int32_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<StringArray>(
+      length, value_offsets_, data_, null_bitmap_, kUnknownNullCount, offset);
+}
+
 // ----------------------------------------------------------------------
 // Struct
 
+StructArray::StructArray(const std::shared_ptr<DataType>& type, int32_t length,
+    const std::vector<std::shared_ptr<Array>>& children,
+    std::shared_ptr<Buffer> null_bitmap, int32_t null_count, int32_t offset)
+    : Array(type, length, null_bitmap, null_count, offset) {
+  type_ = type;
+  children_ = children;
+}
+
 std::shared_ptr<Array> StructArray::field(int32_t pos) const {
-  DCHECK_GT(field_arrays_.size(), 0);
-  return field_arrays_[pos];
+  DCHECK_GT(children_.size(), 0);
+  return children_[pos];
 }
 
 Status StructArray::Validate() const {
@@ -260,11 +329,11 @@ Status StructArray::Validate() const {
     return Status::Invalid("Null count exceeds the length of this struct");
   }
 
-  if (field_arrays_.size() > 0) {
+  if (children_.size() > 0) {
     // Validate fields
-    int32_t array_length = field_arrays_[0]->length();
+    int32_t array_length = children_[0]->length();
     size_t idx = 0;
-    for (auto it : field_arrays_) {
+    for (auto it : children_) {
       if (it->length() != array_length) {
         std::stringstream ss;
         ss << "Length is not equal from field " << it->type()->ToString()
@@ -293,19 +362,27 @@ Status StructArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+std::shared_ptr<Array> StructArray::Slice(int32_t offset, int32_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<StructArray>(
+      type_, length, children_, null_bitmap_, kUnknownNullCount, offset);
+}
+
 // ----------------------------------------------------------------------
 // UnionArray
 
 UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::vector<std::shared_ptr<Array>>& children,
-    const std::shared_ptr<Buffer>& type_ids, const std::shared_ptr<Buffer>& offsets,
-    int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
-    : Array(type, length, null_count, null_bitmap),
+    const std::shared_ptr<Buffer>& type_ids, const std::shared_ptr<Buffer>& value_offsets,
+    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
+    : Array(type, length, null_bitmap, null_count, offset),
       children_(children),
-      type_ids_buffer_(type_ids),
-      offsets_buffer_(offsets) {
-  type_ids_ = reinterpret_cast<const uint8_t*>(type_ids->data());
-  if (offsets) { offsets_ = reinterpret_cast<const int32_t*>(offsets->data()); }
+      type_ids_(type_ids),
+      value_offsets_(value_offsets) {
+  raw_type_ids_ = reinterpret_cast<const uint8_t*>(type_ids->data());
+  if (value_offsets) {
+    raw_value_offsets_ = reinterpret_cast<const int32_t*>(value_offsets->data());
+  }
 }
 
 std::shared_ptr<Array> UnionArray::child(int32_t pos) const {
@@ -328,18 +405,24 @@ Status UnionArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+std::shared_ptr<Array> UnionArray::Slice(int32_t offset, int32_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<UnionArray>(type_, length, children_, type_ids_, value_offsets_,
+      null_bitmap_, kUnknownNullCount, offset);
+}
+
 // ----------------------------------------------------------------------
 // DictionaryArray
 
 Status DictionaryArray::FromBuffer(const std::shared_ptr<DataType>& type, int32_t length,
-    const std::shared_ptr<Buffer>& indices, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<DictionaryArray>* out) {
+    const std::shared_ptr<Buffer>& indices, const std::shared_ptr<Buffer>& null_bitmap,
+    int32_t null_count, int32_t offset, std::shared_ptr<DictionaryArray>* out) {
   DCHECK_EQ(type->type, Type::DICTIONARY);
   const auto& dict_type = static_cast<const DictionaryType*>(type.get());
 
   std::shared_ptr<Array> boxed_indices;
-  RETURN_NOT_OK(MakePrimitiveArray(
-      dict_type->index_type(), length, indices, null_count, null_bitmap, &boxed_indices));
+  RETURN_NOT_OK(MakePrimitiveArray(dict_type->index_type(), length, indices, null_bitmap,
+      null_count, offset, &boxed_indices));
 
   *out = std::make_shared<DictionaryArray>(type, boxed_indices);
   return Status::OK();
@@ -347,7 +430,8 @@ Status DictionaryArray::FromBuffer(const std::shared_ptr<DataType>& type, int32_
 
 DictionaryArray::DictionaryArray(
     const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices)
-    : Array(type, indices->length(), indices->null_count(), indices->null_bitmap()),
+    : Array(type, indices->length(), indices->null_bitmap(), indices->null_count(),
+          indices->offset()),
       dict_type_(static_cast<const DictionaryType*>(type.get())),
       indices_(indices) {
   DCHECK_EQ(type->type, Type::DICTIONARY);
@@ -369,16 +453,21 @@ Status DictionaryArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
+std::shared_ptr<Array> DictionaryArray::Slice(int32_t offset, int32_t length) const {
+  std::shared_ptr<Array> sliced_indices = indices_->Slice(offset, length);
+  return std::make_shared<DictionaryArray>(type_, sliced_indices);
+}
+
 // ----------------------------------------------------------------------
 
-#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                          \
-  case Type::ENUM:                                                          \
-    out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
+#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                                  \
+  case Type::ENUM:                                                                  \
+    out->reset(new ArrayType(type, length, data, null_bitmap, null_count, offset)); \
     break;
 
 Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
-    const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out) {
+    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
+    int32_t null_count, int32_t offset, std::shared_ptr<Array>* out) {
   switch (type->type) {
     MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 3b6e93f9cb34d..f3e8f9a4982f7 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -27,6 +27,7 @@
 #include "arrow/buffer.h"
 #include "arrow/type.h"
 #include "arrow/type_fwd.h"
+#include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
@@ -71,23 +72,36 @@ class ArrayVisitor {
 ///
 /// The base class is only required to have a null bitmap buffer if the null
 /// count is greater than 0
+///
+/// If known, the null count can be provided in the base Array constructor. If
+/// the null count is not known, pass -1 to indicate that the null count is to
+/// be computed on the first call to null_count()
 class ARROW_EXPORT Array {
  public:
-  Array(const std::shared_ptr<DataType>& type, int32_t length, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  Array(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0);
 
   virtual ~Array() = default;
 
   /// Determine if a slot is null. For inner loops. Does *not* boundscheck
   bool IsNull(int i) const {
-    return null_count_ > 0 && BitUtil::BitNotSet(null_bitmap_data_, i);
+    return null_bitmap_data_ != nullptr &&
+           BitUtil::BitNotSet(null_bitmap_data_, i + offset_);
   }
 
   /// Size in the number of elements this array contains.
   int32_t length() const { return length_; }
 
-  /// The number of null entries in the array.
-  int32_t null_count() const { return null_count_; }
+  /// A relative position into another array's data, to enable zero-copy
+  /// slicing. This value defaults to zero
+  int32_t offset() const { return offset_; }
+
+  /// The number of null entries in the array. If the null count was not known
+  /// at time of construction (and set to a negative value), then the null
+  /// count will be computed and cached on the first invocation of this
+  /// function
+  int32_t null_count() const;
 
   std::shared_ptr<DataType> type() const { return type_; }
   Type::type type_enum() const { return type_->type; }
@@ -95,11 +109,13 @@ class ARROW_EXPORT Array {
   /// Buffer for the null bitmap.
   ///
   /// Note that for `null_count == 0`, this can be a `nullptr`.
+  /// This buffer does not account for any slice offset
   std::shared_ptr<Buffer> null_bitmap() const { return null_bitmap_; }
 
   /// Raw pointer to the null bitmap.
   ///
   /// Note that for `null_count == 0`, this can be a `nullptr`.
+  /// This buffer does not account for any slice offset
   const uint8_t* null_bitmap_data() const { return null_bitmap_data_; }
 
   bool Equals(const Array& arr) const;
@@ -120,10 +136,29 @@ class ARROW_EXPORT Array {
 
   virtual Status Accept(ArrayVisitor* visitor) const = 0;
 
+  /// Construct a zero-copy slice of the array with the indicated offset and
+  /// length
+  ///
+  /// \param[in] offset the position of the first element in the constructed slice
+  /// \param[in] length the length of the slice. If there are not enough elements in the
+  /// array,
+  ///     the length will be adjusted accordingly
+  ///
+  /// \return a new object wrapped in std::shared_ptr<Array>
+  virtual std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const = 0;
+
+  /// Slice from offset until end of the array
+  std::shared_ptr<Array> Slice(int32_t offset) const;
+
  protected:
   std::shared_ptr<DataType> type_;
-  int32_t null_count_;
   int32_t length_;
+  int32_t offset_;
+
+  // This member is marked mutable so that it can be modified when null_count()
+  // is called from a const context and the null count has to be computed (if
+  // it is not already known)
+  mutable int32_t null_count_;
 
   std::shared_ptr<Buffer> null_bitmap_;
   const uint8_t* null_bitmap_data_;
@@ -138,28 +173,26 @@ class ARROW_EXPORT NullArray : public Array {
  public:
   using TypeClass = NullType;
 
-  NullArray(const std::shared_ptr<DataType>& type, int32_t length)
-      : Array(type, length, length, nullptr) {}
-
-  explicit NullArray(int32_t length) : NullArray(std::make_shared<NullType>(), length) {}
+  explicit NullArray(int32_t length);
 
   Status Accept(ArrayVisitor* visitor) const override;
-};
 
-Status ARROW_EXPORT GetEmptyBitmap(
-    MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result);
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+};
 
 /// Base class for fixed-size logical types
 class ARROW_EXPORT PrimitiveArray : public Array {
  public:
-  virtual ~PrimitiveArray() {}
+  PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0);
 
+  /// The memory containing this array's data
+  /// This buffer does not account for any slice offset
   std::shared_ptr<Buffer> data() const { return data_; }
 
  protected:
-  PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
   std::shared_ptr<Buffer> data_;
   const uint8_t* raw_data_;
 };
@@ -169,21 +202,28 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
  public:
   using TypeClass = TYPE;
   using value_type = typename TypeClass::c_type;
-  NumericArray(int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
-      : PrimitiveArray(
-            std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {}
-  NumericArray(const std::shared_ptr<DataType>& type, int32_t length,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr)
-      : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
+
+  using PrimitiveArray::PrimitiveArray;
+
+  // Only enable this constructor without a type argument for types without additional
+  // metadata
+  template <typename T1 = TYPE>
+  NumericArray(
+      typename std::enable_if<TypeTraits<T1>::is_parameter_free, int32_t>::type length,
+      const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0)
+      : PrimitiveArray(TypeTraits<T1>::type_singleton(), length, data, null_bitmap,
+            null_count, offset) {}
 
   const value_type* raw_data() const {
-    return reinterpret_cast<const value_type*>(raw_data_);
+    return reinterpret_cast<const value_type*>(raw_data_) + offset_;
   }
 
   Status Accept(ArrayVisitor* visitor) const override;
 
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+
   value_type Value(int i) const { return raw_data()[i]; }
 };
 
@@ -191,17 +231,19 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
  public:
   using TypeClass = BooleanType;
 
+  using PrimitiveArray::PrimitiveArray;
+
   BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-  BooleanArray(const std::shared_ptr<DataType>& type, int32_t length,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0);
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
 
-  bool Value(int i) const { return BitUtil::GetBit(raw_data(), i); }
+  bool Value(int i) const {
+    return BitUtil::GetBit(reinterpret_cast<const uint8_t*>(raw_data_), i + offset_);
+  }
 };
 
 // ----------------------------------------------------------------------
@@ -212,39 +254,45 @@ class ARROW_EXPORT ListArray : public Array {
   using TypeClass = ListType;
 
   ListArray(const std::shared_ptr<DataType>& type, int32_t length,
-      const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Array>& values,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
-      : Array(type, length, null_count, null_bitmap) {
-    offsets_buffer_ = offsets;
-    offsets_ = offsets == nullptr ? nullptr : reinterpret_cast<const int32_t*>(
-                                                  offsets_buffer_->data());
+      const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Array>& values,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0)
+      : Array(type, length, null_bitmap, null_count, offset) {
+    value_offsets_ = value_offsets;
+    raw_value_offsets_ = value_offsets == nullptr
+                             ? nullptr
+                             : reinterpret_cast<const int32_t*>(value_offsets_->data());
     values_ = values;
   }
 
   Status Validate() const override;
 
-  virtual ~ListArray() = default;
-
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   std::shared_ptr<Array> values() const { return values_; }
-  std::shared_ptr<Buffer> offsets() const { return offsets_buffer_; }
 
-  std::shared_ptr<DataType> value_type() const { return values_->type(); }
+  /// Note that this buffer does not account for any slice offset
+  std::shared_ptr<Buffer> value_offsets() const { return value_offsets_; }
 
-  const int32_t* raw_offsets() const { return offsets_; }
+  std::shared_ptr<DataType> value_type() const { return values_->type(); }
 
-  int32_t offset(int i) const { return offsets_[i]; }
+  /// Return pointer to raw value offsets accounting for any slice offset
+  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
 
   // Neither of these functions will perform boundschecking
-  int32_t value_offset(int i) const { return offsets_[i]; }
-  int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
+  int32_t value_offset(int i) const { return raw_value_offsets_[i + offset_]; }
+  int32_t value_length(int i) const {
+    i += offset_;
+    return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
+  }
 
   Status Accept(ArrayVisitor* visitor) const override;
 
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+
  protected:
-  std::shared_ptr<Buffer> offsets_buffer_;
-  const int32_t* offsets_;
+  std::shared_ptr<Buffer> value_offsets_;
+  const int32_t* raw_value_offsets_;
   std::shared_ptr<Array> values_;
 };
 
@@ -255,55 +303,67 @@ class ARROW_EXPORT BinaryArray : public Array {
  public:
   using TypeClass = BinaryType;
 
-  BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
-  // Constructor that allows sub-classes/builders to propagate there logical type up the
-  // class hierarchy.
-  BinaryArray(const std::shared_ptr<DataType>& type, int32_t length,
-      const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
-      int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  BinaryArray(int32_t length, const std::shared_ptr<Buffer>& value_offsets,
+      const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0);
 
   // Return the pointer to the given elements bytes
   // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
   // pointer + offset
   const uint8_t* GetValue(int i, int32_t* out_length) const {
-    const int32_t pos = offsets_[i];
-    *out_length = offsets_[i + 1] - pos;
-    return data_ + pos;
+    // Account for base offset
+    i += offset_;
+
+    const int32_t pos = raw_value_offsets_[i];
+    *out_length = raw_value_offsets_[i + 1] - pos;
+    return raw_data_ + pos;
   }
 
-  std::shared_ptr<Buffer> data() const { return data_buffer_; }
-  std::shared_ptr<Buffer> offsets() const { return offsets_buffer_; }
+  /// Note that this buffer does not account for any slice offset
+  std::shared_ptr<Buffer> data() const { return data_; }
 
-  const int32_t* raw_offsets() const { return offsets_; }
+  /// Note that this buffer does not account for any slice offset
+  std::shared_ptr<Buffer> value_offsets() const { return value_offsets_; }
 
-  int32_t offset(int i) const { return offsets_[i]; }
+  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
 
   // Neither of these functions will perform boundschecking
-  int32_t value_offset(int i) const { return offsets_[i]; }
-  int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
+  int32_t value_offset(int i) const { return raw_value_offsets_[i + offset_]; }
+  int32_t value_length(int i) const {
+    i += offset_;
+    return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
+  }
 
   Status Validate() const override;
 
   Status Accept(ArrayVisitor* visitor) const override;
 
- private:
-  std::shared_ptr<Buffer> offsets_buffer_;
-  const int32_t* offsets_;
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+
+ protected:
+  // Constructor that allows sub-classes/builders to propagate there logical type up the
+  // class hierarchy.
+  BinaryArray(const std::shared_ptr<DataType>& type, int32_t length,
+      const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0);
 
-  std::shared_ptr<Buffer> data_buffer_;
-  const uint8_t* data_;
+  std::shared_ptr<Buffer> value_offsets_;
+  const int32_t* raw_value_offsets_;
+
+  std::shared_ptr<Buffer> data_;
+  const uint8_t* raw_data_;
 };
 
 class ARROW_EXPORT StringArray : public BinaryArray {
  public:
   using TypeClass = StringType;
 
-  StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
-      const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+  StringArray(int32_t length, const std::shared_ptr<Buffer>& value_offsets,
+      const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0);
 
   // Construct a std::string
   // TODO: std::bad_alloc possibility
@@ -316,6 +376,8 @@ class ARROW_EXPORT StringArray : public BinaryArray {
   Status Validate() const override;
 
   Status Accept(ArrayVisitor* visitor) const override;
+
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
 };
 
 // ----------------------------------------------------------------------
@@ -326,28 +388,25 @@ class ARROW_EXPORT StructArray : public Array {
   using TypeClass = StructType;
 
   StructArray(const std::shared_ptr<DataType>& type, int32_t length,
-      const std::vector<std::shared_ptr<Array>>& field_arrays, int32_t null_count = 0,
-      std::shared_ptr<Buffer> null_bitmap = nullptr)
-      : Array(type, length, null_count, null_bitmap) {
-    type_ = type;
-    field_arrays_ = field_arrays;
-  }
+      const std::vector<std::shared_ptr<Array>>& children,
+      std::shared_ptr<Buffer> null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0);
 
   Status Validate() const override;
 
-  virtual ~StructArray() {}
-
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   std::shared_ptr<Array> field(int32_t pos) const;
 
-  const std::vector<std::shared_ptr<Array>>& fields() const { return field_arrays_; }
+  const std::vector<std::shared_ptr<Array>>& fields() const { return children_; }
 
   Status Accept(ArrayVisitor* visitor) const override;
 
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+
  protected:
   // The child arrays corresponding to each field of the struct data type.
-  std::vector<std::shared_ptr<Array>> field_arrays_;
+  std::vector<std::shared_ptr<Array>> children_;
 };
 
 // ----------------------------------------------------------------------
@@ -356,22 +415,25 @@ class ARROW_EXPORT StructArray : public Array {
 class ARROW_EXPORT UnionArray : public Array {
  public:
   using TypeClass = UnionType;
+  using type_id_t = uint8_t;
 
   UnionArray(const std::shared_ptr<DataType>& type, int32_t length,
       const std::vector<std::shared_ptr<Array>>& children,
       const std::shared_ptr<Buffer>& type_ids,
-      const std::shared_ptr<Buffer>& offsets = nullptr, int32_t null_count = 0,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+      const std::shared_ptr<Buffer>& value_offsets = nullptr,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
+      int32_t offset = 0);
 
   Status Validate() const override;
 
-  virtual ~UnionArray() {}
+  /// Note that this buffer does not account for any slice offset
+  std::shared_ptr<Buffer> type_ids() const { return type_ids_; }
 
-  std::shared_ptr<Buffer> type_ids() const { return type_ids_buffer_; }
-  const uint8_t* raw_type_ids() const { return type_ids_; }
+  /// Note that this buffer does not account for any slice offset
+  std::shared_ptr<Buffer> value_offsets() const { return value_offsets_; }
 
-  std::shared_ptr<Buffer> offsets() const { return offsets_buffer_; }
-  const int32_t* raw_offsets() const { return offsets_; }
+  const type_id_t* raw_type_ids() const { return raw_type_ids_ + offset_; }
+  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
 
   UnionMode mode() const { return static_cast<const UnionType&>(*type_.get()).mode; }
 
@@ -381,14 +443,16 @@ class ARROW_EXPORT UnionArray : public Array {
 
   Status Accept(ArrayVisitor* visitor) const override;
 
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+
  protected:
   std::vector<std::shared_ptr<Array>> children_;
 
-  std::shared_ptr<Buffer> type_ids_buffer_;
-  const uint8_t* type_ids_;
+  std::shared_ptr<Buffer> type_ids_;
+  const type_id_t* raw_type_ids_;
 
-  std::shared_ptr<Buffer> offsets_buffer_;
-  const int32_t* offsets_;
+  std::shared_ptr<Buffer> value_offsets_;
+  const int32_t* raw_value_offsets_;
 };
 
 // ----------------------------------------------------------------------
@@ -419,8 +483,8 @@ class ARROW_EXPORT DictionaryArray : public Array {
   // Alternate ctor; other attributes (like null count) are inherited from the
   // passed indices array
   static Status FromBuffer(const std::shared_ptr<DataType>& type, int32_t length,
-      const std::shared_ptr<Buffer>& indices, int32_t null_count,
-      const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<DictionaryArray>* out);
+      const std::shared_ptr<Buffer>& indices, const std::shared_ptr<Buffer>& null_bitmap,
+      int32_t null_count, int32_t offset, std::shared_ptr<DictionaryArray>* out);
 
   Status Validate() const override;
 
@@ -431,6 +495,8 @@ class ARROW_EXPORT DictionaryArray : public Array {
 
   Status Accept(ArrayVisitor* visitor) const override;
 
+  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+
  protected:
   const DictionaryType* dict_type_;
   std::shared_ptr<Array> indices_;
@@ -471,8 +537,9 @@ extern template class ARROW_EXPORT NumericArray<TimeType>;
 
 // Create new arrays for logical types that are backed by primitive arrays.
 Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count,
-    const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
+    int32_t length, const std::shared_ptr<Buffer>& data,
+    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset,
+    std::shared_ptr<Array>* out);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 6cce0efa37784..fb5a010efa225 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -116,4 +116,20 @@ Status PoolBuffer::Resize(int64_t new_size, bool shrink_to_fit) {
   return Status::OK();
 }
 
+Status AllocateBuffer(
+    MemoryPool* pool, int64_t size, std::shared_ptr<MutableBuffer>* out) {
+  auto buffer = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(buffer->Resize(size));
+  *out = buffer;
+  return Status::OK();
+}
+
+Status AllocateResizableBuffer(
+    MemoryPool* pool, int64_t size, std::shared_ptr<ResizableBuffer>* out) {
+  auto buffer = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(buffer->Resize(size));
+  *out = buffer;
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index d43ab0375b725..9c400b1290394 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef ARROW_UTIL_BUFFER_H
-#define ARROW_UTIL_BUFFER_H
+#ifndef ARROW_BUFFER_H
+#define ARROW_BUFFER_H
 
 #include <algorithm>
 #include <cstdint>
@@ -105,7 +105,7 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
 
 /// Construct a view on passed buffer at the indicated offset and length. This
 /// function cannot fail and does not error checking (except in debug builds)
-ARROW_EXPORT std::shared_ptr<Buffer> SliceBuffer(
+std::shared_ptr<Buffer> ARROW_EXPORT SliceBuffer(
     const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
 
 /// A Buffer whose contents can be mutated. May or may not own its data.
@@ -232,6 +232,19 @@ class ARROW_EXPORT BufferBuilder {
   int64_t size_;
 };
 
+/// Allocate a new mutable buffer from a memory pool
+///
+/// \param[in] pool a memory pool
+/// \param[in] size size of buffer to allocate
+/// \param[out] out the allocated buffer with padding
+///
+/// \return Status message
+Status ARROW_EXPORT AllocateBuffer(
+    MemoryPool* pool, int64_t size, std::shared_ptr<MutableBuffer>* out);
+
+Status ARROW_EXPORT AllocateResizableBuffer(
+    MemoryPool* pool, int64_t size, std::shared_ptr<ResizableBuffer>* out);
+
 }  // namespace arrow
 
-#endif  // ARROW_UTIL_BUFFER_H
+#endif  // ARROW_BUFFER_H
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index b0dc41baf4202..dddadeee0dacf 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -185,7 +185,7 @@ Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
   *out = std::make_shared<typename TypeTraits<T>::ArrayType>(
-      type_, length_, data_, null_count_, null_bitmap_);
+      type_, length_, data_, null_bitmap_, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -202,10 +202,19 @@ template class PrimitiveBuilder<Int32Type>;
 template class PrimitiveBuilder<Int64Type>;
 template class PrimitiveBuilder<DateType>;
 template class PrimitiveBuilder<TimestampType>;
+template class PrimitiveBuilder<TimeType>;
 template class PrimitiveBuilder<HalfFloatType>;
 template class PrimitiveBuilder<FloatType>;
 template class PrimitiveBuilder<DoubleType>;
 
+BooleanBuilder::BooleanBuilder(MemoryPool* pool)
+    : ArrayBuilder(pool, boolean()), data_(nullptr), raw_data_(nullptr) {}
+
+BooleanBuilder::BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
+    : BooleanBuilder(pool) {
+  DCHECK_EQ(Type::BOOL, type->type);
+}
+
 Status BooleanBuilder::Init(int32_t capacity) {
   RETURN_NOT_OK(ArrayBuilder::Init(capacity));
   data_ = std::make_shared<PoolBuffer>(pool_);
@@ -244,7 +253,7 @@ Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
-  *out = std::make_shared<BooleanArray>(type_, length_, data_, null_count_, null_bitmap_);
+  *out = std::make_shared<BooleanArray>(type_, length_, data_, null_bitmap_, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -313,7 +322,7 @@ Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<Buffer> offsets = offset_builder_.Finish();
 
   *out = std::make_shared<ListArray>(
-      type_, length_, offsets, items, null_count_, null_bitmap_);
+      type_, length_, offsets, items, null_bitmap_, null_count_);
 
   Reset();
 
@@ -333,14 +342,13 @@ std::shared_ptr<ArrayBuilder> ListBuilder::value_builder() const {
 // ----------------------------------------------------------------------
 // String and binary
 
-// This used to be a static member variable of BinaryBuilder, but it can cause
-// valgrind to report a (spurious?) memory leak when needed in other shared
-// libraries. The problem came up while adding explicit visibility to libarrow
-// and libparquet_arrow
-static TypePtr kBinaryValueType = TypePtr(new UInt8Type());
+BinaryBuilder::BinaryBuilder(MemoryPool* pool)
+    : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, uint8()), binary()) {
+  byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
+}
 
 BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
-    : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, kBinaryValueType), type) {
+    : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, uint8()), type) {
   byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
 }
 
@@ -351,11 +359,13 @@ Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
   const auto list = std::dynamic_pointer_cast<ListArray>(result);
   auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
 
-  *out = std::make_shared<BinaryArray>(list->length(), list->offsets(), values->data(),
-      list->null_count(), list->null_bitmap());
+  *out = std::make_shared<BinaryArray>(list->length(), list->value_offsets(),
+      values->data(), list->null_bitmap(), list->null_count());
   return Status::OK();
 }
 
+StringBuilder::StringBuilder(MemoryPool* pool) : BinaryBuilder(pool, utf8()) {}
+
 Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<Array> result;
   RETURN_NOT_OK(ListBuilder::Finish(&result));
@@ -363,8 +373,8 @@ Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
   const auto list = std::dynamic_pointer_cast<ListArray>(result);
   auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
 
-  *out = std::make_shared<StringArray>(list->length(), list->offsets(), values->data(),
-      list->null_count(), list->null_bitmap());
+  *out = std::make_shared<StringArray>(list->length(), list->value_offsets(),
+      values->data(), list->null_bitmap(), list->null_count());
   return Status::OK();
 }
 
@@ -377,7 +387,7 @@ Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
     RETURN_NOT_OK(field_builders_[i]->Finish(&fields[i]));
   }
 
-  *out = std::make_shared<StructArray>(type_, length_, fields, null_count_, null_bitmap_);
+  *out = std::make_shared<StructArray>(type_, length_, fields, null_bitmap_, null_count_);
 
   null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -393,9 +403,9 @@ std::shared_ptr<ArrayBuilder> StructBuilder::field_builder(int pos) const {
 // ----------------------------------------------------------------------
 // Helper functions
 
-#define BUILDER_CASE(ENUM, BuilderType)      \
-  case Type::ENUM:                           \
-    out->reset(new BuilderType(pool, type)); \
+#define BUILDER_CASE(ENUM, BuilderType) \
+  case Type::ENUM:                      \
+    out->reset(new BuilderType(pool));  \
     return Status::OK();
 
 // Initially looked at doing this with vtables, but shared pointers makes it
@@ -414,19 +424,17 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(UINT64, UInt64Builder);
     BUILDER_CASE(INT64, Int64Builder);
     BUILDER_CASE(DATE, DateBuilder);
-    BUILDER_CASE(TIMESTAMP, TimestampBuilder);
-
-    BUILDER_CASE(BOOL, BooleanBuilder);
-
-    BUILDER_CASE(FLOAT, FloatBuilder);
-    BUILDER_CASE(DOUBLE, DoubleBuilder);
-
-    case Type::STRING:
-      out->reset(new StringBuilder(pool));
+    case Type::TIMESTAMP:
+      out->reset(new TimestampBuilder(pool, type));
       return Status::OK();
-    case Type::BINARY:
-      out->reset(new BinaryBuilder(pool, type));
+    case Type::TIME:
+      out->reset(new TimeBuilder(pool, type));
       return Status::OK();
+      BUILDER_CASE(BOOL, BooleanBuilder);
+      BUILDER_CASE(FLOAT, FloatBuilder);
+      BUILDER_CASE(DOUBLE, DoubleBuilder);
+      BUILDER_CASE(STRING, StringBuilder);
+      BUILDER_CASE(BINARY, BinaryBuilder);
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 672d2d8f23e8f..0b83b9f3c6862 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -141,9 +141,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   using value_type = typename Type::c_type;
 
   explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type)
-      : ArrayBuilder(pool, type), data_(nullptr) {}
-
-  virtual ~PrimitiveBuilder() {}
+      : ArrayBuilder(pool, type), data_(nullptr), raw_data_(nullptr) {}
 
   using ArrayBuilder::Advance;
 
@@ -233,6 +231,7 @@ using Int16Builder = NumericBuilder<Int16Type>;
 using Int32Builder = NumericBuilder<Int32Type>;
 using Int64Builder = NumericBuilder<Int64Type>;
 using TimestampBuilder = NumericBuilder<TimestampType>;
+using TimeBuilder = NumericBuilder<TimeType>;
 using DateBuilder = NumericBuilder<DateType>;
 
 using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
@@ -241,10 +240,8 @@ using DoubleBuilder = NumericBuilder<DoubleType>;
 
 class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
  public:
-  explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type = boolean())
-      : ArrayBuilder(pool, type), data_(nullptr) {}
-
-  virtual ~BooleanBuilder() {}
+  explicit BooleanBuilder(MemoryPool* pool);
+  explicit BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 
   using ArrayBuilder::Advance;
 
@@ -321,8 +318,6 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   ListBuilder(
       MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr);
 
-  virtual ~ListBuilder() {}
-
   Status Init(int32_t elements) override;
   Status Resize(int32_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
@@ -368,8 +363,8 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
 // BinaryBuilder : public ListBuilder
 class ARROW_EXPORT BinaryBuilder : public ListBuilder {
  public:
+  explicit BinaryBuilder(MemoryPool* pool);
   explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type);
-  virtual ~BinaryBuilder() {}
 
   Status Append(const uint8_t* value, int32_t length) {
     RETURN_NOT_OK(ListBuilder::Append());
@@ -391,11 +386,7 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
 // String builder
 class ARROW_EXPORT StringBuilder : public BinaryBuilder {
  public:
-  explicit StringBuilder(MemoryPool* pool = default_memory_pool())
-      : BinaryBuilder(pool, utf8()) {}
-
-  explicit StringBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-      : BinaryBuilder(pool, type) {}
+  explicit StringBuilder(MemoryPool* pool);
 
   using BinaryBuilder::Append;
 
diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
index 1e722ed7de0d6..0bbfc831f5cb9 100644
--- a/cpp/src/arrow/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -51,7 +51,7 @@ TEST_F(TestChunkedArray, BasicEquals) {
   std::vector<bool> null_bitmap(100, true);
   std::vector<int32_t> data(100, 1);
   std::shared_ptr<Array> array;
-  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap, data, &array);
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data, &array);
   arrays_one_.push_back(array);
   arrays_another_.push_back(array);
 
@@ -67,9 +67,9 @@ TEST_F(TestChunkedArray, EqualsDifferingTypes) {
   std::vector<int32_t> data32(100, 1);
   std::vector<int64_t> data64(100, 1);
   std::shared_ptr<Array> array;
-  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap, data32, &array);
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data32, &array);
   arrays_one_.push_back(array);
-  ArrayFromVector<Int64Type, int64_t>(int64(), null_bitmap, data64, &array);
+  ArrayFromVector<Int64Type, int64_t>(null_bitmap, data64, &array);
   arrays_another_.push_back(array);
 
   Construct();
@@ -83,9 +83,9 @@ TEST_F(TestChunkedArray, EqualsDifferingLengths) {
   std::vector<int32_t> data100(100, 1);
   std::vector<int32_t> data101(101, 1);
   std::shared_ptr<Array> array;
-  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap100, data100, &array);
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap100, data100, &array);
   arrays_one_.push_back(array);
-  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap101, data101, &array);
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap101, data101, &array);
   arrays_another_.push_back(array);
 
   Construct();
@@ -94,7 +94,7 @@ TEST_F(TestChunkedArray, EqualsDifferingLengths) {
 
   std::vector<bool> null_bitmap1(1, true);
   std::vector<int32_t> data1(1, 1);
-  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap1, data1, &array);
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap1, data1, &array);
   arrays_one_.push_back(array);
 
   Construct();
@@ -156,7 +156,7 @@ TEST_F(TestColumn, Equals) {
   std::vector<bool> null_bitmap(100, true);
   std::vector<int32_t> data(100, 1);
   std::shared_ptr<Array> array;
-  ArrayFromVector<Int32Type, int32_t>(int32(), null_bitmap, data, &array);
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data, &array);
   arrays_one_.push_back(array);
   arrays_another_.push_back(array);
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index d039bba08827c..27fad7135721c 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -76,10 +76,10 @@ class RangeEqualsVisitor : public ArrayVisitor {
       const bool is_null = left.IsNull(i);
       if (is_null != right.IsNull(o_i)) { return false; }
       if (is_null) continue;
-      const int32_t begin_offset = left.offset(i);
-      const int32_t end_offset = left.offset(i + 1);
-      const int32_t right_begin_offset = right.offset(o_i);
-      const int32_t right_end_offset = right.offset(o_i + 1);
+      const int32_t begin_offset = left.value_offset(i);
+      const int32_t end_offset = left.value_offset(i + 1);
+      const int32_t right_begin_offset = right.value_offset(o_i);
+      const int32_t right_end_offset = right.value_offset(o_i + 1);
       // Underlying can't be equal if the size isn't equal
       if (end_offset - begin_offset != right_end_offset - right_begin_offset) {
         return false;
@@ -169,10 +169,10 @@ class RangeEqualsVisitor : public ArrayVisitor {
       const bool is_null = left.IsNull(i);
       if (is_null != right.IsNull(o_i)) { return false; }
       if (is_null) continue;
-      const int32_t begin_offset = left.offset(i);
-      const int32_t end_offset = left.offset(i + 1);
-      const int32_t right_begin_offset = right.offset(o_i);
-      const int32_t right_end_offset = right.offset(o_i + 1);
+      const int32_t begin_offset = left.value_offset(i);
+      const int32_t end_offset = left.value_offset(i + 1);
+      const int32_t right_begin_offset = right.value_offset(o_i);
+      const int32_t right_end_offset = right.value_offset(o_i + 1);
       // Underlying can't be equal if the size isn't equal
       if (end_offset - begin_offset != right_end_offset - right_begin_offset) {
         return false;
@@ -200,7 +200,11 @@ class RangeEqualsVisitor : public ArrayVisitor {
       for (size_t j = 0; j < left.fields().size(); ++j) {
         // TODO: really we should be comparing stretches of non-null data rather
         // than looking at one value at a time.
-        equal_fields = left.field(j)->RangeEquals(i, i + 1, o_i, right.field(j));
+        const int left_abs_index = i + left.offset();
+        const int right_abs_index = o_i + right.offset();
+
+        equal_fields = left.field(j)->RangeEquals(
+            left_abs_index, left_abs_index + 1, right_abs_index, right.field(j));
         if (!equal_fields) { return false; }
       }
     }
@@ -223,7 +227,7 @@ class RangeEqualsVisitor : public ArrayVisitor {
     // Define a mapping from the type id to child number
     uint8_t max_code = 0;
 
-    const std::vector<uint8_t> type_codes = left_type.type_ids;
+    const std::vector<uint8_t> type_codes = left_type.type_codes;
     for (size_t i = 0; i < type_codes.size(); ++i) {
       const uint8_t code = type_codes[i];
       if (code > max_code) { max_code = code; }
@@ -248,15 +252,19 @@ class RangeEqualsVisitor : public ArrayVisitor {
       id = left_ids[i];
       child_num = type_id_to_child_num[id];
 
+      const int left_abs_index = i + left.offset();
+      const int right_abs_index = o_i + right.offset();
+
       // TODO(wesm): really we should be comparing stretches of non-null data
       // rather than looking at one value at a time.
       if (union_mode == UnionMode::SPARSE) {
-        if (!left.child(child_num)->RangeEquals(i, i + 1, o_i, right.child(child_num))) {
+        if (!left.child(child_num)->RangeEquals(left_abs_index, left_abs_index + 1,
+                right_abs_index, right.child(child_num))) {
           return false;
         }
       } else {
-        const int32_t offset = left.raw_offsets()[i];
-        const int32_t o_offset = right.raw_offsets()[i];
+        const int32_t offset = left.raw_value_offsets()[i];
+        const int32_t o_offset = right.raw_value_offsets()[i];
         if (!left.child(child_num)->RangeEquals(
                 offset, offset + 1, o_offset, right.child(child_num))) {
           return false;
@@ -315,20 +323,29 @@ class EqualsVisitor : public RangeEqualsVisitor {
       }
       result_ = true;
     } else {
-      result_ = left.data()->Equals(*right.data(), BitUtil::BytesForBits(left.length()));
+      result_ = BitmapEquals(left.data()->data(), left.offset(), right.data()->data(),
+          right.offset(), left.length());
     }
     return Status::OK();
   }
 
   bool IsEqualPrimitive(const PrimitiveArray& left) {
     const auto& right = static_cast<const PrimitiveArray&>(right_);
-    if (left.null_count() > 0) {
-      const uint8_t* left_data = left.data()->data();
-      const uint8_t* right_data = right.data()->data();
-      const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
-      const int value_byte_size = size_meta.bit_width() / 8;
-      DCHECK_GT(value_byte_size, 0);
+    const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
+    const int value_byte_size = size_meta.bit_width() / 8;
+    DCHECK_GT(value_byte_size, 0);
+
+    const uint8_t* left_data = nullptr;
+    if (left.length() > 0) {
+      left_data = left.data()->data() + left.offset() * value_byte_size;
+    }
+
+    const uint8_t* right_data = nullptr;
+    if (right.length() > 0) {
+      right_data = right.data()->data() + right.offset() * value_byte_size;
+    }
 
+    if (left.null_count() > 0) {
       for (int i = 0; i < left.length(); ++i) {
         if (!left.IsNull(i) && memcmp(left_data, right_data, value_byte_size)) {
           return false;
@@ -339,7 +356,7 @@ class EqualsVisitor : public RangeEqualsVisitor {
       return true;
     } else {
       if (left.length() == 0) { return true; }
-      return left.data()->Equals(*right.data(), left.length());
+      return memcmp(left_data, right_data, value_byte_size * left.length()) == 0;
     }
   }
 
@@ -376,13 +393,46 @@ class EqualsVisitor : public RangeEqualsVisitor {
 
   Status Visit(const IntervalArray& left) override { return ComparePrimitive(left); }
 
+  template <typename ArrayType>
+  bool ValueOffsetsEqual(const ArrayType& left) {
+    const auto& right = static_cast<const ArrayType&>(right_);
+
+    if (left.offset() == 0 && right.offset() == 0) {
+      return left.value_offsets()->Equals(
+          *right.value_offsets(), (left.length() + 1) * sizeof(int32_t));
+    } else {
+      // One of the arrays is sliced; logic is more complicated because the
+      // value offsets are not both 0-based
+      auto left_offsets =
+          reinterpret_cast<const int32_t*>(left.value_offsets()->data()) + left.offset();
+      auto right_offsets =
+          reinterpret_cast<const int32_t*>(right.value_offsets()->data()) +
+          right.offset();
+
+      for (int32_t i = 0; i < left.length() + 1; ++i) {
+        if (left_offsets[i] - left_offsets[0] != right_offsets[i] - right_offsets[0]) {
+          return false;
+        }
+      }
+      return true;
+    }
+  }
+
   bool CompareBinary(const BinaryArray& left) {
     const auto& right = static_cast<const BinaryArray&>(right_);
-    bool equal_offsets =
-        left.offsets()->Equals(*right.offsets(), (left.length() + 1) * sizeof(int32_t));
+
+    bool equal_offsets = ValueOffsetsEqual<BinaryArray>(left);
     if (!equal_offsets) { return false; }
-    if (!left.data() && !(right.data())) { return true; }
-    return left.data()->Equals(*right.data(), left.raw_offsets()[left.length()]);
+
+    if (left.offset() == 0 && right.offset() == 0) {
+      if (!left.data() && !(right.data())) { return true; }
+      return left.data()->Equals(*right.data(), left.raw_value_offsets()[left.length()]);
+    } else {
+      // Compare the corresponding data range
+      const int64_t total_bytes = left.value_offset(left.length()) - left.value_offset(0);
+      return std::memcmp(left.data()->data() + left.value_offset(0),
+                 right.data()->data() + right.value_offset(0), total_bytes) == 0;
+    }
   }
 
   Status Visit(const StringArray& left) override {
@@ -397,12 +447,20 @@ class EqualsVisitor : public RangeEqualsVisitor {
 
   Status Visit(const ListArray& left) override {
     const auto& right = static_cast<const ListArray&>(right_);
-    if (!left.offsets()->Equals(
-            *right.offsets(), (left.length() + 1) * sizeof(int32_t))) {
+    bool equal_offsets = ValueOffsetsEqual<ListArray>(left);
+    if (!equal_offsets) {
       result_ = false;
-    } else {
+      return Status::OK();
+    }
+
+    if (left.offset() == 0 && right.offset() == 0) {
       result_ = left.values()->Equals(right.values());
+    } else {
+      // One of the arrays is sliced
+      result_ = left.values()->RangeEquals(left.value_offset(0),
+          left.value_offset(left.length()), right.value_offset(0), right.values());
     }
+
     return Status::OK();
   }
 
@@ -422,8 +480,8 @@ inline bool FloatingApproxEquals(
     const NumericArray<TYPE>& left, const NumericArray<TYPE>& right) {
   using T = typename TYPE::c_type;
 
-  auto left_data = reinterpret_cast<const T*>(left.data()->data());
-  auto right_data = reinterpret_cast<const T*>(right.data()->data());
+  const T* left_data = left.raw_data();
+  const T* right_data = right.raw_data();
 
   static constexpr T EPSILON = 1E-5;
 
@@ -465,8 +523,8 @@ static bool BaseDataEquals(const Array& left, const Array& right) {
     return false;
   }
   if (left.null_count() > 0) {
-    return left.null_bitmap()->Equals(
-        *right.null_bitmap(), BitUtil::BytesForBits(left.length()));
+    return BitmapEquals(left.null_bitmap()->data(), left.offset(),
+        right.null_bitmap()->data(), right.offset(), left.length());
   }
   return true;
 }
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index ff58e539b9353..c1f0ea48a9dc9 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -401,8 +401,8 @@ class ReadableFile::ReadableFileImpl : public OSFile {
   Status Open(const std::string& path) { return OpenReadable(path); }
 
   Status ReadBuffer(int64_t nbytes, std::shared_ptr<Buffer>* out) {
-    auto buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(buffer->Resize(nbytes));
+    std::shared_ptr<ResizableBuffer> buffer;
+    RETURN_NOT_OK(AllocateResizableBuffer(pool_, nbytes, &buffer));
 
     int64_t bytes_read = 0;
     RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 2845b0d8c448c..5682f44b94a46 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -125,8 +125,8 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
   }
 
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
-    auto buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(buffer->Resize(nbytes));
+    std::shared_ptr<ResizableBuffer> buffer;
+    RETURN_NOT_OK(AllocateResizableBuffer(pool_, nbytes, &buffer));
 
     int64_t bytes_read = 0;
     RETURN_NOT_OK(ReadAt(position, nbytes, &bytes_read, buffer->mutable_data()));
@@ -152,8 +152,8 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
   }
 
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
-    auto buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(buffer->Resize(nbytes));
+    std::shared_ptr<ResizableBuffer> buffer;
+    RETURN_NOT_OK(AllocateResizableBuffer(pool_, nbytes, &buffer));
 
     int64_t bytes_read = 0;
     RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 72e0ba8f2987b..f0e5a280d3116 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -336,8 +336,9 @@ TYPED_TEST(TestHdfsClient, LargeFile) {
   std::shared_ptr<HdfsReadableFile> file;
   ASSERT_OK(this->client_->OpenReadable(path, &file));
 
-  auto buffer = std::make_shared<PoolBuffer>();
-  ASSERT_OK(buffer->Resize(size));
+  std::shared_ptr<MutableBuffer> buffer;
+  ASSERT_OK(AllocateBuffer(nullptr, size, &buffer));
+
   int64_t bytes_read = 0;
 
   ASSERT_OK(file->Read(size, &bytes_read, buffer->mutable_data()));
@@ -348,8 +349,9 @@ TYPED_TEST(TestHdfsClient, LargeFile) {
   std::shared_ptr<HdfsReadableFile> file2;
   ASSERT_OK(this->client_->OpenReadable(path, 1 << 18, &file2));
 
-  auto buffer2 = std::make_shared<PoolBuffer>();
-  ASSERT_OK(buffer2->Resize(size));
+  std::shared_ptr<MutableBuffer> buffer2;
+  ASSERT_OK(AllocateBuffer(nullptr, size, &buffer2));
+
   ASSERT_OK(file2->Read(size, &bytes_read, buffer2->mutable_data()));
   ASSERT_EQ(0, std::memcmp(buffer2->data(), data.data(), size));
   ASSERT_EQ(size, bytes_read);
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index c0b01653cb128..442cd0c4bbccd 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -73,8 +73,8 @@ TEST(TestBufferReader, RetainParentReference) {
   std::shared_ptr<Buffer> slice1;
   std::shared_ptr<Buffer> slice2;
   {
-    auto buffer = std::make_shared<PoolBuffer>();
-    ASSERT_OK(buffer->Resize(static_cast<int64_t>(data.size())));
+    std::shared_ptr<MutableBuffer> buffer;
+    ASSERT_OK(AllocateBuffer(nullptr, static_cast<int64_t>(data.size()), &buffer));
     std::memcpy(buffer->mutable_data(), data.c_str(), data.size());
     BufferReader reader(buffer);
     ASSERT_OK(reader.Read(4, &slice1));
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index c8e631c564b22..3613ccbadbbab 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/ipc/adapter.h"
 
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <sstream>
@@ -30,6 +31,7 @@
 #include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
+#include "arrow/memory_pool.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
@@ -49,9 +51,10 @@ namespace ipc {
 
 class RecordBatchWriter : public ArrayVisitor {
  public:
-  RecordBatchWriter(
-      const RecordBatch& batch, int64_t buffer_start_offset, int max_recursion_depth)
-      : batch_(batch),
+  RecordBatchWriter(MemoryPool* pool, const RecordBatch& batch,
+      int64_t buffer_start_offset, int max_recursion_depth)
+      : pool_(pool),
+        batch_(batch),
         max_recursion_depth_(max_recursion_depth),
         buffer_start_offset_(buffer_start_offset) {}
 
@@ -62,7 +65,15 @@ class RecordBatchWriter : public ArrayVisitor {
     // push back all common elements
     field_nodes_.push_back(flatbuf::FieldNode(arr.length(), arr.null_count()));
     if (arr.null_count() > 0) {
-      buffers_.push_back(arr.null_bitmap());
+      std::shared_ptr<Buffer> bitmap = arr.null_bitmap();
+
+      if (arr.offset() != 0) {
+        // With a sliced array / non-zero offset, we must copy the bitmap
+        RETURN_NOT_OK(
+            CopyBitmap(pool_, bitmap->data(), arr.offset(), arr.length(), &bitmap));
+      }
+
+      buffers_.push_back(bitmap);
     } else {
       // Push a dummy zero-length buffer, not to be copied
       buffers_.push_back(std::make_shared<Buffer>(nullptr, 0));
@@ -208,50 +219,136 @@ class RecordBatchWriter : public ArrayVisitor {
  private:
   Status Visit(const NullArray& array) override { return Status::NotImplemented("null"); }
 
-  Status VisitPrimitive(const PrimitiveArray& array) {
-    buffers_.push_back(array.data());
+  template <typename ArrayType>
+  Status VisitFixedWidth(const ArrayType& array) {
+    std::shared_ptr<Buffer> data_buffer = array.data();
+
+    if (array.offset() != 0) {
+      // Non-zero offset, slice the buffer
+      const auto& fw_type = static_cast<const FixedWidthType&>(*array.type());
+      const int type_width = fw_type.bit_width() / 8;
+      const int64_t byte_offset = array.offset() * type_width;
+
+      // Send padding if it's available
+      const int64_t buffer_length =
+          std::min(BitUtil::RoundUpToMultipleOf64(array.length() * type_width),
+              data_buffer->size() - byte_offset);
+      data_buffer = SliceBuffer(data_buffer, byte_offset, buffer_length);
+    }
+    buffers_.push_back(data_buffer);
+    return Status::OK();
+  }
+
+  template <typename ArrayType>
+  Status GetZeroBasedValueOffsets(
+      const ArrayType& array, std::shared_ptr<Buffer>* value_offsets) {
+    // Share slicing logic between ListArray and BinaryArray
+
+    auto offsets = array.value_offsets();
+
+    if (array.offset() != 0) {
+      // If we have a non-zero offset, then the value offsets do not start at
+      // zero. We must a) create a new offsets array with shifted offsets and
+      // b) slice the values array accordingly
+
+      std::shared_ptr<MutableBuffer> shifted_offsets;
+      RETURN_NOT_OK(AllocateBuffer(
+          pool_, sizeof(int32_t) * (array.length() + 1), &shifted_offsets));
+
+      int32_t* dest_offsets = reinterpret_cast<int32_t*>(shifted_offsets->mutable_data());
+      const int32_t start_offset = array.value_offset(0);
+
+      for (int i = 0; i < array.length(); ++i) {
+        dest_offsets[i] = array.value_offset(i) - start_offset;
+      }
+      // Final offset
+      dest_offsets[array.length()] = array.value_offset(array.length()) - start_offset;
+      offsets = shifted_offsets;
+    }
+
+    *value_offsets = offsets;
     return Status::OK();
   }
 
   Status VisitBinary(const BinaryArray& array) {
-    buffers_.push_back(array.offsets());
-    buffers_.push_back(array.data());
+    std::shared_ptr<Buffer> value_offsets;
+    RETURN_NOT_OK(GetZeroBasedValueOffsets<BinaryArray>(array, &value_offsets));
+    auto data = array.data();
+
+    if (array.offset() != 0) {
+      // Slice the data buffer to include only the range we need now
+      data = SliceBuffer(data, array.value_offset(0), array.value_offset(array.length()));
+    }
+
+    buffers_.push_back(value_offsets);
+    buffers_.push_back(data);
     return Status::OK();
   }
 
-  Status Visit(const BooleanArray& array) override { return VisitPrimitive(array); }
+  Status Visit(const BooleanArray& array) override {
+    buffers_.push_back(array.data());
+    return Status::OK();
+  }
 
-  Status Visit(const Int8Array& array) override { return VisitPrimitive(array); }
+  Status Visit(const Int8Array& array) override {
+    return VisitFixedWidth<Int8Array>(array);
+  }
 
-  Status Visit(const Int16Array& array) override { return VisitPrimitive(array); }
+  Status Visit(const Int16Array& array) override {
+    return VisitFixedWidth<Int16Array>(array);
+  }
 
-  Status Visit(const Int32Array& array) override { return VisitPrimitive(array); }
+  Status Visit(const Int32Array& array) override {
+    return VisitFixedWidth<Int32Array>(array);
+  }
 
-  Status Visit(const Int64Array& array) override { return VisitPrimitive(array); }
+  Status Visit(const Int64Array& array) override {
+    return VisitFixedWidth<Int64Array>(array);
+  }
 
-  Status Visit(const UInt8Array& array) override { return VisitPrimitive(array); }
+  Status Visit(const UInt8Array& array) override {
+    return VisitFixedWidth<UInt8Array>(array);
+  }
 
-  Status Visit(const UInt16Array& array) override { return VisitPrimitive(array); }
+  Status Visit(const UInt16Array& array) override {
+    return VisitFixedWidth<UInt16Array>(array);
+  }
 
-  Status Visit(const UInt32Array& array) override { return VisitPrimitive(array); }
+  Status Visit(const UInt32Array& array) override {
+    return VisitFixedWidth<UInt32Array>(array);
+  }
 
-  Status Visit(const UInt64Array& array) override { return VisitPrimitive(array); }
+  Status Visit(const UInt64Array& array) override {
+    return VisitFixedWidth<UInt64Array>(array);
+  }
 
-  Status Visit(const HalfFloatArray& array) override { return VisitPrimitive(array); }
+  Status Visit(const HalfFloatArray& array) override {
+    return VisitFixedWidth<HalfFloatArray>(array);
+  }
 
-  Status Visit(const FloatArray& array) override { return VisitPrimitive(array); }
+  Status Visit(const FloatArray& array) override {
+    return VisitFixedWidth<FloatArray>(array);
+  }
 
-  Status Visit(const DoubleArray& array) override { return VisitPrimitive(array); }
+  Status Visit(const DoubleArray& array) override {
+    return VisitFixedWidth<DoubleArray>(array);
+  }
 
   Status Visit(const StringArray& array) override { return VisitBinary(array); }
 
   Status Visit(const BinaryArray& array) override { return VisitBinary(array); }
 
-  Status Visit(const DateArray& array) override { return VisitPrimitive(array); }
+  Status Visit(const DateArray& array) override {
+    return VisitFixedWidth<DateArray>(array);
+  }
 
-  Status Visit(const TimeArray& array) override { return VisitPrimitive(array); }
+  Status Visit(const TimeArray& array) override {
+    return VisitFixedWidth<TimeArray>(array);
+  }
 
-  Status Visit(const TimestampArray& array) override { return VisitPrimitive(array); }
+  Status Visit(const TimestampArray& array) override {
+    return VisitFixedWidth<TimestampArray>(array);
+  }
 
   Status Visit(const IntervalArray& array) override {
     return Status::NotImplemented("interval");
@@ -262,30 +359,109 @@ class RecordBatchWriter : public ArrayVisitor {
   }
 
   Status Visit(const ListArray& array) override {
-    buffers_.push_back(array.offsets());
+    std::shared_ptr<Buffer> value_offsets;
+    RETURN_NOT_OK(GetZeroBasedValueOffsets<ListArray>(array, &value_offsets));
+    buffers_.push_back(value_offsets);
+
     --max_recursion_depth_;
-    RETURN_NOT_OK(VisitArray(*array.values().get()));
+    std::shared_ptr<Array> values = array.values();
+
+    if (array.offset() != 0) {
+      // For non-zero offset, we slice the values array accordingly
+      const int32_t offset = array.value_offset(0);
+      const int32_t length = array.value_offset(array.length()) - offset;
+      values = values->Slice(offset, length);
+    }
+    RETURN_NOT_OK(VisitArray(*values));
     ++max_recursion_depth_;
     return Status::OK();
   }
 
   Status Visit(const StructArray& array) override {
     --max_recursion_depth_;
-    for (const auto& field : array.fields()) {
-      RETURN_NOT_OK(VisitArray(*field.get()));
+    for (std::shared_ptr<Array> field : array.fields()) {
+      if (array.offset() != 0) {
+        // If offset is non-zero, slice the child array
+        field = field->Slice(array.offset(), array.length());
+      }
+      RETURN_NOT_OK(VisitArray(*field));
     }
     ++max_recursion_depth_;
     return Status::OK();
   }
 
   Status Visit(const UnionArray& array) override {
-    buffers_.push_back(array.type_ids());
+    auto type_ids = array.type_ids();
+    if (array.offset() != 0) {
+      type_ids = SliceBuffer(type_ids, array.offset() * sizeof(UnionArray::type_id_t),
+          array.length() * sizeof(UnionArray::type_id_t));
+    }
 
-    if (array.mode() == UnionMode::DENSE) { buffers_.push_back(array.offsets()); }
+    buffers_.push_back(type_ids);
 
     --max_recursion_depth_;
-    for (const auto& field : array.children()) {
-      RETURN_NOT_OK(VisitArray(*field.get()));
+    if (array.mode() == UnionMode::DENSE) {
+      const auto& type = static_cast<const UnionType&>(*array.type());
+      auto value_offsets = array.value_offsets();
+
+      // The Union type codes are not necessary 0-indexed
+      uint8_t max_code = 0;
+      for (uint8_t code : type.type_codes) {
+        if (code > max_code) { max_code = code; }
+      }
+
+      // Allocate an array of child offsets. Set all to -1 to indicate that we
+      // haven't observed a first occurrence of a particular child yet
+      std::vector<int32_t> child_offsets(max_code + 1);
+      std::vector<int32_t> child_lengths(max_code + 1, 0);
+
+      if (array.offset() != 0) {
+        // This is an unpleasant case. Because the offsets are different for
+        // each child array, when we have a sliced array, we need to "rebase"
+        // the value_offsets for each array
+
+        const int32_t* unshifted_offsets = array.raw_value_offsets();
+        const uint8_t* type_ids = array.raw_type_ids();
+
+        // Allocate the shifted offsets
+        std::shared_ptr<MutableBuffer> shifted_offsets_buffer;
+        RETURN_NOT_OK(AllocateBuffer(
+            pool_, array.length() * sizeof(int32_t), &shifted_offsets_buffer));
+        int32_t* shifted_offsets =
+            reinterpret_cast<int32_t*>(shifted_offsets_buffer->mutable_data());
+
+        for (int32_t i = 0; i < array.length(); ++i) {
+          const uint8_t code = type_ids[i];
+          int32_t shift = child_offsets[code];
+          if (shift == -1) { child_offsets[code] = shift = unshifted_offsets[i]; }
+          shifted_offsets[i] = unshifted_offsets[i] - shift;
+
+          // Update the child length to account for observed value
+          ++child_lengths[code];
+        }
+
+        value_offsets = shifted_offsets_buffer;
+      }
+      buffers_.push_back(value_offsets);
+
+      // Visit children and slice accordingly
+      for (int i = 0; i < type.num_children(); ++i) {
+        std::shared_ptr<Array> child = array.child(i);
+        if (array.offset() != 0) {
+          const uint8_t code = type.type_codes[i];
+          child = child->Slice(child_offsets[code], child_lengths[code]);
+        }
+        RETURN_NOT_OK(VisitArray(*child));
+      }
+    } else {
+      for (std::shared_ptr<Array> child : array.children()) {
+        // Sparse union, slicing is simpler
+        if (array.offset() != 0) {
+          // If offset is non-zero, slice the child array
+          child = child->Slice(array.offset(), array.length());
+        }
+        RETURN_NOT_OK(VisitArray(*child));
+      }
     }
     ++max_recursion_depth_;
     return Status::OK();
@@ -298,6 +474,8 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
+  // In some cases, intermediate buffers may need to be allocated (with sliced arrays)
+  MemoryPool* pool_;
   const RecordBatch& batch_;
 
   std::vector<flatbuf::FieldNode> field_nodes_;
@@ -310,14 +488,14 @@ class RecordBatchWriter : public ArrayVisitor {
 
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    int max_recursion_depth) {
+    MemoryPool* pool, int max_recursion_depth) {
   DCHECK_GT(max_recursion_depth, 0);
-  RecordBatchWriter serializer(batch, buffer_start_offset, max_recursion_depth);
+  RecordBatchWriter serializer(pool, batch, buffer_start_offset, max_recursion_depth);
   return serializer.Write(dst, metadata_length, body_length);
 }
 
 Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
-  RecordBatchWriter serializer(batch, 0, kMaxIpcRecursionDepth);
+  RecordBatchWriter serializer(default_memory_pool(), batch, 0, kMaxIpcRecursionDepth);
   RETURN_NOT_OK(serializer.GetTotalSize(size));
   return Status::OK();
 }
@@ -372,7 +550,7 @@ class ArrayLoader : public TypeVisitor {
     BufferMetadata metadata = context_->metadata->buffer(buffer_index);
 
     if (metadata.length == 0) {
-      *out = std::make_shared<Buffer>(nullptr, 0);
+      *out = nullptr;
       return Status::OK();
     } else {
       return file_->ReadAt(metadata.offset, metadata.length, out);
@@ -412,8 +590,8 @@ class ArrayLoader : public TypeVisitor {
       context_->buffer_index++;
       data.reset(new Buffer(nullptr, 0));
     }
-    return MakePrimitiveArray(field_.type, field_meta.length, data, field_meta.null_count,
-        null_bitmap, &result_);
+    return MakePrimitiveArray(field_.type, field_meta.length, data, null_bitmap,
+        field_meta.null_count, 0, &result_);
   }
 
   template <typename CONTAINER>
@@ -428,7 +606,7 @@ class ArrayLoader : public TypeVisitor {
     RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
 
     result_ = std::make_shared<CONTAINER>(
-        field_meta.length, offsets, values, field_meta.null_count, null_bitmap);
+        field_meta.length, offsets, values, null_bitmap, field_meta.null_count);
     return Status::OK();
   }
 
@@ -496,7 +674,7 @@ class ArrayLoader : public TypeVisitor {
     RETURN_NOT_OK(LoadChild(*type.child(0).get(), &values_array));
 
     result_ = std::make_shared<ListArray>(field_.type, field_meta.length, offsets,
-        values_array, field_meta.null_count, null_bitmap);
+        values_array, null_bitmap, field_meta.null_count);
     return Status::OK();
   }
 
@@ -521,7 +699,7 @@ class ArrayLoader : public TypeVisitor {
     RETURN_NOT_OK(LoadChildren(type.children(), &fields));
 
     result_ = std::make_shared<StructArray>(
-        field_.type, field_meta.length, fields, field_meta.null_count, null_bitmap);
+        field_.type, field_meta.length, fields, null_bitmap, field_meta.null_count);
     return Status::OK();
   }
 
@@ -542,7 +720,7 @@ class ArrayLoader : public TypeVisitor {
     RETURN_NOT_OK(LoadChildren(type.children(), &fields));
 
     result_ = std::make_shared<UnionArray>(field_.type, field_meta.length, fields,
-        type_ids, offsets, field_meta.null_count, null_bitmap);
+        type_ids, offsets, null_bitmap, field_meta.null_count);
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index f9ef7d9fe1202..83542d0b066d4 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -30,6 +30,7 @@
 namespace arrow {
 
 class Array;
+class MemoryPool;
 class RecordBatch;
 class Schema;
 class Status;
@@ -71,14 +72,15 @@ constexpr int kMaxIpcRecursionDepth = 64;
 //
 // @param(out) body_length: the size of the contiguous buffer block plus
 // padding bytes
-ARROW_EXPORT Status WriteRecordBatch(const RecordBatch& batch,
+Status ARROW_EXPORT WriteRecordBatch(const RecordBatch& batch,
     int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length, int max_recursion_depth = kMaxIpcRecursionDepth);
+    int64_t* body_length, MemoryPool* pool,
+    int max_recursion_depth = kMaxIpcRecursionDepth);
 
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the record batch. This involves generating the complete serialized
 // Flatbuffers metadata.
-ARROW_EXPORT Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
+Status ARROW_EXPORT GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 
 // ----------------------------------------------------------------------
 // "Read" path; does not copy data if the input supports zero copy reads
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 17868f8f1029e..bae6578f110f2 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -32,6 +32,7 @@
 
 #include "arrow/buffer.h"
 #include "arrow/memory_pool.h"
+#include "arrow/pretty_print.h"
 #include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/util/bit-util.h"
@@ -56,7 +57,7 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
     const int64_t buffer_offset = 0;
 
     RETURN_NOT_OK(WriteRecordBatch(
-        batch, buffer_offset, mmap_.get(), &metadata_length, &body_length));
+        batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
 
     std::shared_ptr<RecordBatchMetadata> metadata;
     RETURN_NOT_OK(ReadRecordBatchMetadata(0, metadata_length, mmap_.get(), &metadata));
@@ -92,17 +93,49 @@ TEST_P(TestWriteRecordBatch, RoundTrip) {
   }
 }
 
-INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRecordBatch,
-    ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch,
-                            &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,
-                            &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion));
+TEST_P(TestWriteRecordBatch, SliceRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
+  std::shared_ptr<RecordBatch> batch_result;
+
+  // Skip the zero-length case
+  if (batch->num_rows() < 2) { return; }
+
+  auto sliced_batch = batch->Slice(2, 10);
+
+  ASSERT_OK(RoundTripHelper(*sliced_batch, 1 << 16, &batch_result));
+
+  EXPECT_EQ(sliced_batch->num_rows(), batch_result->num_rows());
+
+  for (int i = 0; i < sliced_batch->num_columns(); ++i) {
+    const auto& left = *sliced_batch->column(i);
+    const auto& right = *batch_result->column(i);
+    if (!left.Equals(right)) {
+      std::stringstream pp_result;
+      std::stringstream pp_expected;
+
+      ASSERT_OK(PrettyPrint(left, 0, &pp_expected));
+      ASSERT_OK(PrettyPrint(right, 0, &pp_result));
+
+      FAIL() << "Index: " << i << " Expected: " << pp_expected.str()
+             << "\nGot: " << pp_result.str();
+    }
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(
+    RoundTripTests, TestWriteRecordBatch,
+    ::testing::Values(&MakeIntRecordBatch, &MakeStringTypesRecordBatch,
+        &MakeNonNullRecordBatch, &MakeZeroLengthRecordBatch, &MakeListRecordBatch,
+        &MakeDeeplyNestedList, &MakeStruct, &MakeUnion));
 
 void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   ipc::MockOutputStream mock;
   int32_t mock_metadata_length = -1;
   int64_t mock_body_length = -1;
   int64_t size = -1;
-  ASSERT_OK(WriteRecordBatch(*batch, 0, &mock, &mock_metadata_length, &mock_body_length));
+  ASSERT_OK(WriteRecordBatch(
+      *batch, 0, &mock, &mock_metadata_length, &mock_body_length, default_memory_pool()));
   ASSERT_OK(GetRecordBatchSize(*batch, &size));
   ASSERT_EQ(mock.GetExtentBytesWritten(), size);
 }
@@ -156,10 +189,11 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
     io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
 
     if (override_level) {
-      return WriteRecordBatch(
-          *batch, 0, mmap_.get(), metadata_length, body_length, recursion_level + 1);
+      return WriteRecordBatch(*batch, 0, mmap_.get(), metadata_length, body_length, pool_,
+          recursion_level + 1);
     } else {
-      return WriteRecordBatch(*batch, 0, mmap_.get(), metadata_length, body_length);
+      return WriteRecordBatch(
+          *batch, 0, mmap_.get(), metadata_length, body_length, pool_);
     }
   }
 
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 30f968c2bfd8b..3e759cccbbccc 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -80,7 +80,7 @@ template <typename T, typename ValueType>
 void CheckPrimitive(const std::shared_ptr<DataType>& type,
     const std::vector<bool>& is_valid, const std::vector<ValueType>& values) {
   MemoryPool* pool = default_memory_pool();
-  typename TypeTraits<T>::BuilderType builder(pool, type);
+  typename TypeTraits<T>::BuilderType builder(pool);
 
   for (size_t i = 0; i < values.size(); ++i) {
     if (is_valid[i]) {
@@ -146,12 +146,11 @@ TEST(TestJsonArrayWriter, NestedTypes) {
 
   std::vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
   std::shared_ptr<Array> values_array;
-  ArrayFromVector<Int32Type, int32_t>(int32(), values_is_valid, values, &values_array);
+  ArrayFromVector<Int32Type, int32_t>(values_is_valid, values, &values_array);
 
   std::vector<int16_t> i16_values = {0, 1, 2, 3, 4, 5, 6};
   std::shared_ptr<Array> i16_values_array;
-  ArrayFromVector<Int16Type, int16_t>(
-      int16(), values_is_valid, i16_values, &i16_values_array);
+  ArrayFromVector<Int16Type, int16_t>(values_is_valid, i16_values, &i16_values_array);
 
   // List
   std::vector<bool> list_is_valid = {true, false, true, true, true};
@@ -161,7 +160,7 @@ TEST(TestJsonArrayWriter, NestedTypes) {
   ASSERT_OK(test::GetBitmapFromBoolVector(list_is_valid, &list_bitmap));
   std::shared_ptr<Buffer> offsets_buffer = test::GetBufferFromVector(offsets);
 
-  ListArray list_array(list(value_type), 5, offsets_buffer, values_array, 1, list_bitmap);
+  ListArray list_array(list(value_type), 5, offsets_buffer, values_array, list_bitmap, 1);
 
   TestArrayRoundTrip(list_array);
 
@@ -175,7 +174,7 @@ TEST(TestJsonArrayWriter, NestedTypes) {
 
   std::vector<std::shared_ptr<Array>> fields = {values_array, values_array, values_array};
   StructArray struct_array(
-      struct_type, static_cast<int>(struct_is_valid.size()), fields, 2, struct_bitmap);
+      struct_type, static_cast<int>(struct_is_valid.size()), fields, struct_bitmap, 2);
   TestArrayRoundTrip(struct_array);
 }
 
@@ -202,15 +201,15 @@ void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
   test::randint<int32_t>(num_rows, 0, 100, &v2_values);
 
   std::shared_ptr<Array> v1;
-  ArrayFromVector<Int8Type, int8_t>(schema->field(0)->type, is_valid, v1_values, &v1);
+  ArrayFromVector<Int8Type, int8_t>(is_valid, v1_values, &v1);
 
   std::shared_ptr<Array> v2;
-  ArrayFromVector<Int32Type, int32_t>(schema->field(1)->type, is_valid, v2_values, &v2);
+  ArrayFromVector<Int32Type, int32_t>(is_valid, v2_values, &v2);
 
   static const int kBufferSize = 10;
   static uint8_t buffer[kBufferSize];
   static uint32_t seed = 0;
-  StringBuilder string_builder(default_memory_pool(), utf8());
+  StringBuilder string_builder(default_memory_pool());
   for (int i = 0; i < num_rows; ++i) {
     if (!is_valid[i]) {
       string_builder.AppendNull();
@@ -338,13 +337,13 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
   std::vector<bool> foo_valid = {true, false, true, true, true};
   std::vector<int32_t> foo_values = {1, 2, 3, 4, 5};
   std::shared_ptr<Array> foo;
-  ArrayFromVector<Int32Type, int32_t>(int32(), foo_valid, foo_values, &foo);
+  ArrayFromVector<Int32Type, int32_t>(foo_valid, foo_values, &foo);
   ASSERT_TRUE(batch->column(0)->Equals(foo));
 
   std::vector<bool> bar_valid = {true, false, false, true, true};
   std::vector<double> bar_values = {1, 2, 3, 4, 5};
   std::shared_ptr<Array> bar;
-  ArrayFromVector<DoubleType, double>(float64(), bar_valid, bar_values, &bar);
+  ArrayFromVector<DoubleType, double>(bar_valid, bar_values, &bar);
   ASSERT_TRUE(batch->column(1)->Equals(bar));
 }
 
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 95bc742054fab..17ccc4ac1d0da 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -144,10 +144,8 @@ static Status ValidateArrowVsJson(
 
   if (!json_schema->Equals(arrow_schema)) {
     std::stringstream ss;
-    ss << "JSON schema: \n"
-       << json_schema->ToString() << "\n"
-       << "Arrow schema: \n"
-       << arrow_schema->ToString();
+    ss << "JSON schema: \n" << json_schema->ToString() << "\n"
+       << "Arrow schema: \n" << arrow_schema->ToString();
 
     if (FLAGS_verbose) { std::cout << ss.str() << std::endl; }
     return Status::Invalid("Schemas did not match");
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 43bd8a4a4e814..1a95b2ce470b2 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -199,8 +199,8 @@ class JsonSchemaWriter : public TypeVisitor {
     // Write type ids
     writer_->Key("typeIds");
     writer_->StartArray();
-    for (size_t i = 0; i < type.type_ids.size(); ++i) {
-      writer_->Uint(type.type_ids[i]);
+    for (size_t i = 0; i < type.type_codes.size(); ++i) {
+      writer_->Uint(type.type_codes[i]);
     }
     writer_->EndArray();
   }
@@ -464,7 +464,7 @@ class JsonArrayWriter : public ArrayVisitor {
   template <typename T>
   Status WriteVarBytes(const T& array) {
     WriteValidityField(array);
-    WriteIntegerField("OFFSET", array.raw_offsets(), array.length() + 1);
+    WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length() + 1);
     WriteDataField(array);
     SetNoChildren();
     return Status::OK();
@@ -532,7 +532,7 @@ class JsonArrayWriter : public ArrayVisitor {
 
   Status Visit(const ListArray& array) override {
     WriteValidityField(array);
-    WriteIntegerField("OFFSET", array.raw_offsets(), array.length() + 1);
+    WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length() + 1);
     auto type = static_cast<const ListType*>(array.type().get());
     return WriteChildren(type->children(), {array.values()});
   }
@@ -549,7 +549,7 @@ class JsonArrayWriter : public ArrayVisitor {
 
     WriteIntegerField("TYPE_ID", array.raw_type_ids(), array.length());
     if (type->mode == UnionMode::DENSE) {
-      WriteIntegerField("OFFSET", array.raw_offsets(), array.length());
+      WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length());
     }
     return WriteChildren(type->children(), array.children());
   }
@@ -718,17 +718,17 @@ class JsonSchemaReader {
       return Status::Invalid(ss.str());
     }
 
-    const auto& json_type_ids = json_type.FindMember("typeIds");
-    RETURN_NOT_ARRAY("typeIds", json_type_ids, json_type);
+    const auto& json_type_codes = json_type.FindMember("typeIds");
+    RETURN_NOT_ARRAY("typeIds", json_type_codes, json_type);
 
-    std::vector<uint8_t> type_ids;
-    const auto& id_array = json_type_ids->value.GetArray();
+    std::vector<uint8_t> type_codes;
+    const auto& id_array = json_type_codes->value.GetArray();
     for (const rj::Value& val : id_array) {
       DCHECK(val.IsUint());
-      type_ids.push_back(val.GetUint());
+      type_codes.push_back(val.GetUint());
     }
 
-    *type = union_(children, type_ids, mode);
+    *type = union_(children, type_codes, mode);
 
     return Status::OK();
   }
@@ -844,7 +844,7 @@ class JsonArrayReader {
   typename std::enable_if<std::is_base_of<BinaryType, T>::value, Status>::type ReadArray(
       const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
       const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-    typename TypeTraits<T>::BuilderType builder(pool_, type);
+    typename TypeTraits<T>::BuilderType builder(pool_);
 
     const auto& json_data = json_array.FindMember("DATA");
     RETURN_NOT_ARRAY("DATA", json_data, json_array);
@@ -869,8 +869,9 @@ class JsonArrayReader {
   template <typename T>
   Status GetIntArray(
       const RjArray& json_array, const int32_t length, std::shared_ptr<Buffer>* out) {
-    auto buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(buffer->Resize(length * sizeof(T)));
+    std::shared_ptr<MutableBuffer> buffer;
+    RETURN_NOT_OK(AllocateBuffer(pool_, length * sizeof(T), &buffer));
+
     T* values = reinterpret_cast<T*>(buffer->mutable_data());
     for (int i = 0; i < length; ++i) {
       const rj::Value& val = json_array[i];
@@ -901,7 +902,7 @@ class JsonArrayReader {
     DCHECK_EQ(children.size(), 1);
 
     *array = std::make_shared<ListArray>(
-        type, length, offsets_buffer, children[0], null_count, validity_buffer);
+        type, length, offsets_buffer, children[0], validity_buffer, null_count);
 
     return Status::OK();
   }
@@ -918,7 +919,7 @@ class JsonArrayReader {
     RETURN_NOT_OK(GetChildren(json_array, type, &fields));
 
     *array =
-        std::make_shared<StructArray>(type, length, fields, null_count, validity_buffer);
+        std::make_shared<StructArray>(type, length, fields, validity_buffer, null_count);
 
     return Status::OK();
   }
@@ -953,7 +954,7 @@ class JsonArrayReader {
     RETURN_NOT_OK(GetChildren(json_array, type, &children));
 
     *array = std::make_shared<UnionArray>(type, length, children, type_id_buffer,
-        offsets_buffer, null_count, validity_buffer);
+        offsets_buffer, validity_buffer, null_count);
 
     return Status::OK();
   }
@@ -962,7 +963,7 @@ class JsonArrayReader {
   typename std::enable_if<std::is_base_of<NullType, T>::value, Status>::type ReadArray(
       const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
       const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-    *array = std::make_shared<NullArray>(type, length);
+    *array = std::make_shared<NullArray>(length);
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/ipc/stream.cc b/cpp/src/arrow/ipc/stream.cc
index c9057e860b1e8..72eb13465afcc 100644
--- a/cpp/src/arrow/ipc/stream.cc
+++ b/cpp/src/arrow/ipc/stream.cc
@@ -28,6 +28,7 @@
 #include "arrow/ipc/adapter.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
+#include "arrow/memory_pool.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
@@ -41,7 +42,11 @@ namespace ipc {
 StreamWriter::~StreamWriter() {}
 
 StreamWriter::StreamWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
-    : sink_(sink), schema_(schema), position_(-1), started_(false) {}
+    : sink_(sink),
+      schema_(schema),
+      pool_(default_memory_pool()),
+      position_(-1),
+      started_(false) {}
 
 Status StreamWriter::UpdatePosition() {
   return sink_->Tell(&position_);
@@ -76,8 +81,8 @@ Status StreamWriter::WriteRecordBatch(const RecordBatch& batch, FileBlock* block
 
   // Frame of reference in file format is 0, see ARROW-384
   const int64_t buffer_start_offset = 0;
-  RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(
-      batch, buffer_start_offset, sink_, &block->metadata_length, &block->body_length));
+  RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(batch, buffer_start_offset, sink_,
+      &block->metadata_length, &block->body_length, pool_));
   RETURN_NOT_OK(UpdatePosition());
 
   DCHECK(position_ % 8 == 0) << "WriteRecordBatch did not perform aligned writes";
@@ -85,6 +90,10 @@ Status StreamWriter::WriteRecordBatch(const RecordBatch& batch, FileBlock* block
   return Status::OK();
 }
 
+void StreamWriter::set_memory_pool(MemoryPool* pool) {
+  pool_ = pool;
+}
+
 // ----------------------------------------------------------------------
 // StreamWriter implementation
 
diff --git a/cpp/src/arrow/ipc/stream.h b/cpp/src/arrow/ipc/stream.h
index 53f51dc73675f..12414fa2ca0c7 100644
--- a/cpp/src/arrow/ipc/stream.h
+++ b/cpp/src/arrow/ipc/stream.h
@@ -30,6 +30,7 @@ namespace arrow {
 class Array;
 class Buffer;
 struct Field;
+class MemoryPool;
 class RecordBatch;
 class Schema;
 class Status;
@@ -59,6 +60,10 @@ class ARROW_EXPORT StreamWriter {
   /// closing the actual OutputStream
   virtual Status Close();
 
+  // In some cases, writing may require memory allocation. We use the default
+  // memory pool, but provide the option to override
+  void set_memory_pool(MemoryPool* pool);
+
  protected:
   StreamWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema);
 
@@ -81,6 +86,9 @@ class ARROW_EXPORT StreamWriter {
 
   io::OutputStream* sink_;
   std::shared_ptr<Schema> schema_;
+
+  MemoryPool* pool_;
+
   int64_t position_;
   bool started_;
 };
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index ca790ded92191..b4930c4555d44 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <memory>
+#include <numeric>
 #include <string>
 #include <vector>
 
@@ -28,6 +29,7 @@
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
 #include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
@@ -104,8 +106,8 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
   const int length = 1000;
 
   // Make the schema
-  auto f0 = std::make_shared<Field>("f0", int32());
-  auto f1 = std::make_shared<Field>("f1", int32());
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", int32());
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
   // Example data
@@ -119,10 +121,10 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
 template <class Builder, class RawType>
 Status MakeRandomBinaryArray(
-    const TypePtr& type, int32_t length, MemoryPool* pool, std::shared_ptr<Array>* out) {
+    int32_t length, MemoryPool* pool, std::shared_ptr<Array>* out) {
   const std::vector<std::string> values = {
       "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
-  Builder builder(pool, type);
+  Builder builder(pool);
   const auto values_len = values.size();
   for (int32_t i = 0; i < length; ++i) {
     int values_index = i % values_len;
@@ -141,22 +143,22 @@ Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
   const int32_t length = 500;
   auto string_type = utf8();
   auto binary_type = binary();
-  auto f0 = std::make_shared<Field>("f0", string_type);
-  auto f1 = std::make_shared<Field>("f1", binary_type);
+  auto f0 = field("f0", string_type);
+  auto f1 = field("f1", binary_type);
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
   std::shared_ptr<Array> a0, a1;
   MemoryPool* pool = default_memory_pool();
 
+  // Quirk with RETURN_NOT_OK macro and templated functions
   {
-    auto status =
-        MakeRandomBinaryArray<StringBuilder, char>(string_type, length, pool, &a0);
-    RETURN_NOT_OK(status);
+    auto s = MakeRandomBinaryArray<StringBuilder, char>(length, pool, &a0);
+    RETURN_NOT_OK(s);
   }
+
   {
-    auto status =
-        MakeRandomBinaryArray<BinaryBuilder, uint8_t>(binary_type, length, pool, &a1);
-    RETURN_NOT_OK(status);
+    auto s = MakeRandomBinaryArray<BinaryBuilder, uint8_t>(length, pool, &a1);
+    RETURN_NOT_OK(s);
   }
   out->reset(new RecordBatch(schema, length, {a0, a1}));
   return Status::OK();
@@ -164,9 +166,9 @@ Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
 Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out) {
   // Make the schema
-  auto f0 = std::make_shared<Field>("f0", kListInt32);
-  auto f1 = std::make_shared<Field>("f1", kListListInt32);
-  auto f2 = std::make_shared<Field>("f2", int32());
+  auto f0 = field("f0", kListInt32);
+  auto f1 = field("f1", kListListInt32);
+  auto f2 = field("f2", int32());
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
   // Example data
@@ -187,14 +189,13 @@ Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
 Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out) {
   // Make the schema
-  auto f0 = std::make_shared<Field>("f0", kListInt32);
-  auto f1 = std::make_shared<Field>("f1", kListListInt32);
-  auto f2 = std::make_shared<Field>("f2", int32());
+  auto f0 = field("f0", kListInt32);
+  auto f1 = field("f1", kListListInt32);
+  auto f2 = field("f2", int32());
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
   // Example data
   MemoryPool* pool = default_memory_pool();
-  const int length = 200;
   const bool include_nulls = true;
   std::shared_ptr<Array> leaf_values, list_array, list_list_array, flat_array;
   RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &leaf_values));
@@ -202,15 +203,15 @@ Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out) {
   RETURN_NOT_OK(
       MakeRandomListArray(list_array, 0, include_nulls, pool, &list_list_array));
   RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &flat_array));
-  out->reset(new RecordBatch(schema, length, {list_array, list_list_array, flat_array}));
+  out->reset(new RecordBatch(schema, 0, {list_array, list_list_array, flat_array}));
   return Status::OK();
 }
 
 Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
   // Make the schema
-  auto f0 = std::make_shared<Field>("f0", kListInt32);
-  auto f1 = std::make_shared<Field>("f1", kListListInt32);
-  auto f2 = std::make_shared<Field>("f2", int32());
+  auto f0 = field("f0", kListInt32);
+  auto f1 = field("f1", kListListInt32);
+  auto f2 = field("f2", int32());
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
   // Example data
@@ -242,7 +243,7 @@ Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
     RETURN_NOT_OK(MakeRandomListArray(array, batch_length, include_nulls, pool, &array));
   }
 
-  auto f0 = std::make_shared<Field>("f0", type);
+  auto f0 = field("f0", type);
   std::shared_ptr<Schema> schema(new Schema({f0}));
   std::vector<std::shared_ptr<Array>> arrays = {array};
   out->reset(new RecordBatch(schema, batch_length, arrays));
@@ -260,8 +261,8 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
   // Define schema
   std::shared_ptr<DataType> type(new StructType(
       {list_schema->field(0), list_schema->field(1), list_schema->field(2)}));
-  auto f0 = std::make_shared<Field>("non_null_struct", type);
-  auto f1 = std::make_shared<Field>("null_struct", type);
+  auto f0 = field("non_null_struct", type);
+  auto f1 = field("null_struct", type);
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
   // construct individual nullable/non-nullable struct arrays
@@ -271,7 +272,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Buffer> null_bitmask;
   RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask));
   std::shared_ptr<Array> with_nulls(
-      new StructArray(type, list_batch->num_rows(), columns, 1, null_bitmask));
+      new StructArray(type, list_batch->num_rows(), columns, null_bitmask, 1));
 
   // construct batch
   std::vector<std::shared_ptr<Array>> arrays = {no_nulls, with_nulls};
@@ -282,7 +283,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
 Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   // Define schema
   std::vector<std::shared_ptr<Field>> union_types(
-      {std::make_shared<Field>("u0", int32()), std::make_shared<Field>("u1", uint8())});
+      {field("u0", int32()), field("u1", uint8())});
 
   std::vector<uint8_t> type_codes = {5, 10};
   auto sparse_type =
@@ -291,9 +292,9 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   auto dense_type =
       std::make_shared<UnionType>(union_types, type_codes, UnionMode::DENSE);
 
-  auto f0 = std::make_shared<Field>("sparse_nonnull", sparse_type, false);
-  auto f1 = std::make_shared<Field>("sparse", sparse_type);
-  auto f2 = std::make_shared<Field>("dense", dense_type);
+  auto f0 = field("sparse_nonnull", sparse_type, false);
+  auto f1 = field("sparse", sparse_type);
+  auto f2 = field("dense", dense_type);
 
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
@@ -308,21 +309,17 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   RETURN_NOT_OK(test::CopyBufferFromVector(type_ids, &type_ids_buffer));
 
   std::vector<int32_t> u0_values = {0, 1, 2, 3, 4, 5, 6};
-  ArrayFromVector<Int32Type, int32_t>(
-      sparse_type->child(0)->type, u0_values, &sparse_children[0]);
+  ArrayFromVector<Int32Type, int32_t>(u0_values, &sparse_children[0]);
 
   std::vector<uint8_t> u1_values = {10, 11, 12, 13, 14, 15, 16};
-  ArrayFromVector<UInt8Type, uint8_t>(
-      sparse_type->child(1)->type, u1_values, &sparse_children[1]);
+  ArrayFromVector<UInt8Type, uint8_t>(u1_values, &sparse_children[1]);
 
   // dense children
   u0_values = {0, 2, 3, 7};
-  ArrayFromVector<Int32Type, int32_t>(
-      dense_type->child(0)->type, u0_values, &dense_children[0]);
+  ArrayFromVector<Int32Type, int32_t>(u0_values, &dense_children[0]);
 
   u1_values = {11, 14, 15};
-  ArrayFromVector<UInt8Type, uint8_t>(
-      dense_type->child(1)->type, u1_values, &dense_children[1]);
+  ArrayFromVector<UInt8Type, uint8_t>(u1_values, &dense_children[1]);
 
   std::shared_ptr<Buffer> offsets_buffer;
   std::vector<int32_t> offsets = {0, 0, 1, 2, 1, 2, 3};
@@ -337,10 +334,10 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   auto sparse_no_nulls =
       std::make_shared<UnionArray>(sparse_type, length, sparse_children, type_ids_buffer);
   auto sparse = std::make_shared<UnionArray>(
-      sparse_type, length, sparse_children, type_ids_buffer, nullptr, 1, null_bitmask);
+      sparse_type, length, sparse_children, type_ids_buffer, nullptr, null_bitmask, 1);
 
   auto dense = std::make_shared<UnionArray>(dense_type, length, dense_children,
-      type_ids_buffer, offsets_buffer, 1, null_bitmask);
+      type_ids_buffer, offsets_buffer, null_bitmask, 1);
 
   // construct batch
   std::vector<std::shared_ptr<Array>> arrays = {sparse_no_nulls, sparse, dense};
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index 4725d5dd808ee..aca650f0a927b 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -55,7 +55,7 @@ template <typename TYPE, typename C_TYPE>
 void CheckPrimitive(int indent, const std::vector<bool>& is_valid,
     const std::vector<C_TYPE>& values, const char* expected) {
   std::shared_ptr<Array> array;
-  ArrayFromVector<TYPE, C_TYPE>(std::make_shared<TYPE>(), is_valid, values, &array);
+  ArrayFromVector<TYPE, C_TYPE>(is_valid, values, &array);
   CheckArray(*array.get(), indent, expected);
 }
 
@@ -76,12 +76,12 @@ TEST_F(TestPrettyPrint, DictionaryType) {
 
   std::shared_ptr<Array> dict;
   std::vector<std::string> dict_values = {"foo", "bar", "baz"};
-  ArrayFromVector<StringType, std::string>(utf8(), dict_values, &dict);
+  ArrayFromVector<StringType, std::string>(dict_values, &dict);
   std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
 
   std::shared_ptr<Array> indices;
   std::vector<int16_t> indices_values = {1, 2, -1, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices_values, &indices);
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices_values, &indices);
   auto arr = std::make_shared<DictionaryArray>(dict_type, indices);
 
   static const char* expected = R"expected(
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index e30f4cc58d7ab..23c05807c16ee 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -164,39 +164,56 @@ class ArrayPrinter : public ArrayVisitor {
   Status WriteValidityBitmap(const Array& array) {
     Newline();
     Write("-- is_valid: ");
-    BooleanArray is_valid(array.length(), array.null_bitmap());
-    return PrettyPrint(is_valid, indent_ + 2, sink_);
+
+    if (array.null_count() > 0) {
+      BooleanArray is_valid(
+          array.length(), array.null_bitmap(), nullptr, 0, array.offset());
+      return PrettyPrint(is_valid, indent_ + 2, sink_);
+    } else {
+      Write("all not null");
+      return Status::OK();
+    }
   }
 
   Status Visit(const ListArray& array) override {
     RETURN_NOT_OK(WriteValidityBitmap(array));
 
     Newline();
-    Write("-- offsets: ");
-    Int32Array offsets(array.length() + 1, array.offsets());
-    RETURN_NOT_OK(PrettyPrint(offsets, indent_ + 2, sink_));
+    Write("-- value_offsets: ");
+    Int32Array value_offsets(
+        array.length() + 1, array.value_offsets(), nullptr, 0, array.offset());
+    RETURN_NOT_OK(PrettyPrint(value_offsets, indent_ + 2, sink_));
 
     Newline();
     Write("-- values: ");
-    RETURN_NOT_OK(PrettyPrint(*array.values().get(), indent_ + 2, sink_));
+    auto values = array.values();
+    if (array.offset() != 0) {
+      values = values->Slice(array.value_offset(0), array.value_offset(array.length()));
+    }
+    RETURN_NOT_OK(PrettyPrint(*values, indent_ + 2, sink_));
 
     return Status::OK();
   }
 
-  Status PrintChildren(const std::vector<std::shared_ptr<Array>>& fields) {
+  Status PrintChildren(
+      const std::vector<std::shared_ptr<Array>>& fields, int32_t offset, int32_t length) {
     for (size_t i = 0; i < fields.size(); ++i) {
       Newline();
       std::stringstream ss;
       ss << "-- child " << i << " type: " << fields[i]->type()->ToString() << " values: ";
       Write(ss.str());
-      RETURN_NOT_OK(PrettyPrint(*fields[i].get(), indent_ + 2, sink_));
+
+      std::shared_ptr<Array> field = fields[i];
+      if (offset != 0) { field = field->Slice(offset, length); }
+
+      RETURN_NOT_OK(PrettyPrint(*field, indent_ + 2, sink_));
     }
     return Status::OK();
   }
 
   Status Visit(const StructArray& array) override {
     RETURN_NOT_OK(WriteValidityBitmap(array));
-    return PrintChildren(array.fields());
+    return PrintChildren(array.fields(), array.offset(), array.length());
   }
 
   Status Visit(const UnionArray& array) override {
@@ -204,17 +221,19 @@ class ArrayPrinter : public ArrayVisitor {
 
     Newline();
     Write("-- type_ids: ");
-    UInt8Array type_ids(array.length(), array.type_ids());
+    UInt8Array type_ids(array.length(), array.type_ids(), nullptr, 0, array.offset());
     RETURN_NOT_OK(PrettyPrint(type_ids, indent_ + 2, sink_));
 
     if (array.mode() == UnionMode::DENSE) {
       Newline();
-      Write("-- offsets: ");
-      Int32Array offsets(array.length(), array.offsets());
-      RETURN_NOT_OK(PrettyPrint(offsets, indent_ + 2, sink_));
+      Write("-- value_offsets: ");
+      Int32Array value_offsets(
+          array.length(), array.value_offsets(), nullptr, 0, array.offset());
+      RETURN_NOT_OK(PrettyPrint(value_offsets, indent_ + 2, sink_));
     }
 
-    return PrintChildren(array.children());
+    // Print the children without any offset, because the type ids are absolute
+    return PrintChildren(array.children(), 0, array.length() + array.offset());
   }
 
   Status Visit(const DictionaryArray& array) override {
@@ -222,11 +241,11 @@ class ArrayPrinter : public ArrayVisitor {
 
     Newline();
     Write("-- dictionary: ");
-    RETURN_NOT_OK(PrettyPrint(*array.dictionary().get(), indent_ + 2, sink_));
+    RETURN_NOT_OK(PrettyPrint(*array.dictionary(), indent_ + 2, sink_));
 
     Newline();
     Write("-- indices: ");
-    return PrettyPrint(*array.indices().get(), indent_ + 2, sink_);
+    return PrettyPrint(*array.indices(), indent_ + 2, sink_);
   }
 
   void Write(const char* data) { (*sink_) << data; }
@@ -260,7 +279,7 @@ Status PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink) {
   for (int i = 0; i < batch.num_columns(); ++i) {
     const std::string& name = batch.column_name(i);
     (*sink) << name << ": ";
-    RETURN_NOT_OK(PrettyPrint(*batch.column(i).get(), indent + 2, sink));
+    RETURN_NOT_OK(PrettyPrint(*batch.column(i), indent + 2, sink));
     (*sink) << "\n";
   }
   return Status::OK();
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 67c9f6703f496..e7c5d667903e8 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -242,4 +242,30 @@ TEST_F(TestRecordBatch, Equals) {
   ASSERT_FALSE(b1.Equals(b4));
 }
 
+TEST_F(TestRecordBatch, Slice) {
+  const int length = 10;
+
+  auto f0 = std::make_shared<Field>("f0", int32());
+  auto f1 = std::make_shared<Field>("f1", uint8());
+
+  vector<shared_ptr<Field>> fields = {f0, f1};
+  auto schema = std::make_shared<Schema>(fields);
+
+  auto a0 = MakePrimitive<Int32Array>(length);
+  auto a1 = MakePrimitive<UInt8Array>(length);
+
+  RecordBatch batch(schema, length, {a0, a1});
+
+  auto batch_slice = batch.Slice(2);
+  auto batch_slice2 = batch.Slice(1, 5);
+
+  for (int i = 0; i < batch.num_columns(); ++i) {
+    ASSERT_EQ(2, batch_slice->column(i)->offset());
+    ASSERT_EQ(length - 2, batch_slice->column(i)->length());
+
+    ASSERT_EQ(1, batch_slice2->column(i)->offset());
+    ASSERT_EQ(5, batch_slice2->column(i)->length());
+  }
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index b3563eaae7b57..9e31ba5af0ce3 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -60,6 +60,19 @@ bool RecordBatch::ApproxEquals(const RecordBatch& other) const {
   return true;
 }
 
+std::shared_ptr<RecordBatch> RecordBatch::Slice(int32_t offset) {
+  return Slice(offset, this->num_rows() - offset);
+}
+
+std::shared_ptr<RecordBatch> RecordBatch::Slice(int32_t offset, int32_t length) {
+  std::vector<std::shared_ptr<Array>> arrays;
+  arrays.reserve(num_columns());
+  for (const auto& field : columns_) {
+    arrays.emplace_back(field->Slice(offset, length));
+  }
+  return std::make_shared<RecordBatch>(schema_, num_rows_, arrays);
+}
+
 // ----------------------------------------------------------------------
 // Table methods
 
@@ -93,8 +106,7 @@ Status Table::FromRecordBatches(const std::string& name,
     if (!batches[i]->schema()->Equals(schema)) {
       std::stringstream ss;
       ss << "Schema at index " << static_cast<int>(i) << " was different: \n"
-         << schema->ToString() << "\nvs\n"
-         << batches[i]->schema()->ToString();
+         << schema->ToString() << "\nvs\n" << batches[i]->schema()->ToString();
       return Status::Invalid(ss.str());
     }
   }
@@ -126,8 +138,7 @@ Status ConcatenateTables(const std::string& output_name,
     if (!tables[i]->schema()->Equals(schema)) {
       std::stringstream ss;
       ss << "Schema at index " << static_cast<int>(i) << " was different: \n"
-         << schema->ToString() << "\nvs\n"
-         << tables[i]->schema()->ToString();
+         << schema->ToString() << "\nvs\n" << tables[i]->schema()->ToString();
       return Status::Invalid(ss.str());
     }
   }
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 583847cfbe3e5..fa56824a5a1bc 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -64,6 +64,10 @@ class ARROW_EXPORT RecordBatch {
   // @returns: the number of rows (the corresponding length of each column)
   int32_t num_rows() const { return num_rows_; }
 
+  /// Slice each of the arrays in the record batch and construct a new RecordBatch object
+  std::shared_ptr<RecordBatch> Slice(int32_t offset);
+  std::shared_ptr<RecordBatch> Slice(int32_t offset, int32_t length);
+
  private:
   std::shared_ptr<Schema> schema_;
   int32_t num_rows_;
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 4e525804b47cc..ffc78067d1b97 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -61,14 +61,6 @@
     EXPECT_TRUE(s.ok());        \
   } while (0)
 
-// Alias MSVC popcount to GCC name
-#ifdef _MSC_VER
-#include <intrin.h>
-#define __builtin_popcount __popcnt
-#include <nmmintrin.h>
-#define __builtin_popcountll _mm_popcnt_u64
-#endif
-
 namespace arrow {
 
 namespace test {
@@ -175,29 +167,6 @@ void rand_uniform_int(int n, uint32_t seed, T min_value, T max_value, T* out) {
   }
 }
 
-static inline int bitmap_popcount(const uint8_t* data, int length) {
-  // book keeping
-  constexpr int pop_len = sizeof(uint64_t);
-  const uint64_t* i64_data = reinterpret_cast<const uint64_t*>(data);
-  const int fast_counts = length / pop_len;
-  const uint64_t* end = i64_data + fast_counts;
-
-  int count = 0;
-  // popcount as much as possible with the widest possible count
-  for (auto iter = i64_data; iter < end; ++iter) {
-    count += __builtin_popcountll(*iter);
-  }
-
-  // Account for left over bytes (in theory we could fall back to smaller
-  // versions of popcount but the code complexity is likely not worth it)
-  const int loop_tail_index = fast_counts * pop_len;
-  for (int i = loop_tail_index; i < length; ++i) {
-    if (BitUtil::GetBit(data, i)) { ++count; }
-  }
-
-  return count;
-}
-
 static inline int null_count(const std::vector<uint8_t>& valid_bytes) {
   int result = 0;
   for (size_t i = 0; i < valid_bytes.size(); ++i) {
@@ -254,7 +223,7 @@ class TestBase : public ::testing::Test {
 
     auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
     EXPECT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
-    return std::make_shared<ArrayType>(length, data, null_count, null_bitmap);
+    return std::make_shared<ArrayType>(length, data, null_bitmap, null_count);
   }
 
  protected:
@@ -263,11 +232,10 @@ class TestBase : public ::testing::Test {
 };
 
 template <typename TYPE, typename C_TYPE>
-void ArrayFromVector(const std::shared_ptr<DataType>& type,
-    const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
+void ArrayFromVector(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
     std::shared_ptr<Array>* out) {
   MemoryPool* pool = default_memory_pool();
-  typename TypeTraits<TYPE>::BuilderType builder(pool, std::make_shared<TYPE>());
+  typename TypeTraits<TYPE>::BuilderType builder(pool);
   for (size_t i = 0; i < values.size(); ++i) {
     if (is_valid[i]) {
       ASSERT_OK(builder.Append(values[i]));
@@ -279,10 +247,9 @@ void ArrayFromVector(const std::shared_ptr<DataType>& type,
 }
 
 template <typename TYPE, typename C_TYPE>
-void ArrayFromVector(const std::shared_ptr<DataType>& type,
-    const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
+void ArrayFromVector(const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
   MemoryPool* pool = default_memory_pool();
-  typename TypeTraits<TYPE>::BuilderType builder(pool, std::make_shared<TYPE>());
+  typename TypeTraits<TYPE>::BuilderType builder(pool);
   for (size_t i = 0; i < values.size(); ++i) {
     ASSERT_OK(builder.Append(values[i]));
   }
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index ba775845fcaf2..a1c2b79950d59 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -115,7 +115,7 @@ std::string UnionType::ToString() const {
 
   for (size_t i = 0; i < children_.size(); ++i) {
     if (i) { s << ", "; }
-    s << children_[i]->ToString() << "=" << static_cast<int>(type_ids[i]);
+    s << children_[i]->ToString() << "=" << static_cast<int>(type_codes[i]);
   }
   s << ">";
   return s.str();
@@ -224,8 +224,8 @@ std::shared_ptr<DataType> struct_(const std::vector<std::shared_ptr<Field>>& fie
 }
 
 std::shared_ptr<DataType> union_(const std::vector<std::shared_ptr<Field>>& child_fields,
-    const std::vector<uint8_t>& type_ids, UnionMode mode) {
-  return std::make_shared<UnionType>(child_fields, type_ids, mode);
+    const std::vector<uint8_t>& type_codes, UnionMode mode) {
+  return std::make_shared<UnionType>(child_fields, type_codes, mode);
 }
 
 std::shared_ptr<DataType> dictionary(const std::shared_ptr<DataType>& index_type,
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 8638a3f4b6e90..927b8a44fe12f 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -413,8 +413,8 @@ struct ARROW_EXPORT UnionType : public DataType {
   static constexpr Type::type type_id = Type::UNION;
 
   UnionType(const std::vector<std::shared_ptr<Field>>& fields,
-      const std::vector<uint8_t>& type_ids, UnionMode mode = UnionMode::SPARSE)
-      : DataType(Type::UNION), mode(mode), type_ids(type_ids) {
+      const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE)
+      : DataType(Type::UNION), mode(mode), type_codes(type_codes) {
     children_ = fields;
   }
 
@@ -429,7 +429,7 @@ struct ARROW_EXPORT UnionType : public DataType {
   // The type id used in the data to indicate each data type in the union. For
   // example, the first type in the union might be denoted by the id 5 (instead
   // of 0).
-  std::vector<uint8_t> type_ids;
+  std::vector<uint8_t> type_codes;
 };
 
 // ----------------------------------------------------------------------
@@ -551,7 +551,7 @@ std::shared_ptr<DataType> ARROW_EXPORT struct_(
 
 std::shared_ptr<DataType> ARROW_EXPORT union_(
     const std::vector<std::shared_ptr<Field>>& child_fields,
-    const std::vector<uint8_t>& type_ids, UnionMode mode = UnionMode::SPARSE);
+    const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
 
 std::shared_ptr<DataType> ARROW_EXPORT dictionary(
     const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& values);
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 5cd5f45466bf7..c4898b1ac8ce2 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -125,6 +125,15 @@ struct TypeTraits<TimestampType> {
   constexpr static bool is_parameter_free = false;
 };
 
+template <>
+struct TypeTraits<TimeType> {
+  using ArrayType = TimeArray;
+  // using BuilderType = TimestampBuilder;
+
+  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+  constexpr static bool is_parameter_free = false;
+};
+
 template <>
 struct TypeTraits<HalfFloatType> {
   using ArrayType = HalfFloatArray;
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index cfdee04f6e2ea..cb2fd1ab276ad 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -17,11 +17,17 @@
 
 #include "arrow/util/bit-util.h"
 
+#include <cstdint>
+#include <vector>
+
 #include "gtest/gtest.h"
 
+#include "arrow/buffer.h"
+#include "arrow/test-util.h"
+
 namespace arrow {
 
-TEST(UtilTests, TestIsMultipleOf64) {
+TEST(BitUtilTests, TestIsMultipleOf64) {
   using BitUtil::IsMultipleOf64;
   EXPECT_TRUE(IsMultipleOf64(64));
   EXPECT_TRUE(IsMultipleOf64(0));
@@ -31,7 +37,7 @@ TEST(UtilTests, TestIsMultipleOf64) {
   EXPECT_FALSE(IsMultipleOf64(32));
 }
 
-TEST(UtilTests, TestNextPower2) {
+TEST(BitUtilTests, TestNextPower2) {
   using BitUtil::NextPower2;
 
   ASSERT_EQ(8, NextPower2(6));
@@ -51,4 +57,56 @@ TEST(UtilTests, TestNextPower2) {
   ASSERT_EQ(1LL << 62, NextPower2((1LL << 62) - 1));
 }
 
+static inline int64_t SlowCountBits(
+    const uint8_t* data, int64_t bit_offset, int64_t length) {
+  int64_t count = 0;
+  for (int64_t i = bit_offset; i < bit_offset + length; ++i) {
+    if (BitUtil::GetBit(data, i)) { ++count; }
+  }
+  return count;
+}
+
+TEST(BitUtilTests, TestCountSetBits) {
+  const int kBufferSize = 1000;
+  uint8_t buffer[kBufferSize] = {0};
+
+  test::random_bytes(kBufferSize, 0, buffer);
+
+  const int num_bits = kBufferSize * 8;
+
+  std::vector<int64_t> offsets = {
+      0, 12, 16, 32, 37, 63, 64, 128, num_bits - 30, num_bits - 64};
+  for (int64_t offset : offsets) {
+    int64_t result = CountSetBits(buffer, offset, num_bits - offset);
+    int64_t expected = SlowCountBits(buffer, offset, num_bits - offset);
+
+    ASSERT_EQ(expected, result);
+  }
+}
+
+TEST(BitUtilTests, TestCopyBitmap) {
+  const int kBufferSize = 1000;
+
+  std::shared_ptr<MutableBuffer> buffer;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), kBufferSize, &buffer));
+  memset(buffer->mutable_data(), 0, kBufferSize);
+  test::random_bytes(kBufferSize, 0, buffer->mutable_data());
+
+  const int num_bits = kBufferSize * 8;
+
+  const uint8_t* src = buffer->data();
+
+  std::vector<int64_t> offsets = {0, 12, 16, 32, 37, 63, 64, 128};
+  for (int64_t offset : offsets) {
+    const int64_t copy_length = num_bits - offset;
+
+    std::shared_ptr<Buffer> copy;
+    ASSERT_OK(CopyBitmap(default_memory_pool(), src, offset, copy_length, &copy));
+
+    for (int64_t i = 0; i < copy_length; ++i) {
+      ASSERT_EQ(BitUtil::GetBit(src, i + offset), BitUtil::GetBit(copy->data(), i));
+    }
+  }
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 9c82407ecc092..f3fbb41fa54a7 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -15,10 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
+// Alias MSVC popcount to GCC name
+#ifdef _MSC_VER
+#include <intrin.h>
+#define __builtin_popcount __popcnt
+#include <nmmintrin.h>
+#define __builtin_popcountll _mm_popcnt_u64
+#endif
+
+#include <algorithm>
 #include <cstring>
 #include <vector>
 
 #include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/util/bit-util.h"
 
@@ -34,8 +44,9 @@ Status BitUtil::BytesToBits(
     const std::vector<uint8_t>& bytes, std::shared_ptr<Buffer>* out) {
   int bit_length = BitUtil::BytesForBits(bytes.size());
 
-  auto buffer = std::make_shared<PoolBuffer>();
-  RETURN_NOT_OK(buffer->Resize(bit_length));
+  std::shared_ptr<MutableBuffer> buffer;
+  RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), bit_length, &buffer));
+
   memset(buffer->mutable_data(), 0, bit_length);
   BytesToBits(bytes, buffer->mutable_data());
 
@@ -43,4 +54,72 @@ Status BitUtil::BytesToBits(
   return Status::OK();
 }
 
+int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
+  constexpr int64_t pop_len = sizeof(uint64_t) * 8;
+
+  int64_t count = 0;
+
+  // The first bit offset where we can use a 64-bit wide hardware popcount
+  const int64_t fast_count_start = BitUtil::RoundUp(bit_offset, pop_len);
+
+  // The number of bits until fast_count_start
+  const int64_t initial_bits = std::min(length, fast_count_start - bit_offset);
+  for (int64_t i = bit_offset; i < bit_offset + initial_bits; ++i) {
+    if (BitUtil::GetBit(data, i)) { ++count; }
+  }
+
+  const int64_t fast_counts = (length - initial_bits) / pop_len;
+
+  // Advance until the first aligned 8-byte word after the initial bits
+  const uint64_t* u64_data =
+      reinterpret_cast<const uint64_t*>(data) + fast_count_start / pop_len;
+
+  const uint64_t* end = u64_data + fast_counts;
+
+  // popcount as much as possible with the widest possible count
+  for (auto iter = u64_data; iter < end; ++iter) {
+    count += __builtin_popcountll(*iter);
+  }
+
+  // Account for left over bit (in theory we could fall back to smaller
+  // versions of popcount but the code complexity is likely not worth it)
+  const int64_t tail_index = bit_offset + initial_bits + fast_counts * pop_len;
+  for (int64_t i = tail_index; i < bit_offset + length; ++i) {
+    if (BitUtil::GetBit(data, i)) { ++count; }
+  }
+
+  return count;
+}
+
+Status GetEmptyBitmap(
+    MemoryPool* pool, int64_t length, std::shared_ptr<MutableBuffer>* result) {
+  RETURN_NOT_OK(AllocateBuffer(pool, BitUtil::BytesForBits(length), result));
+  memset((*result)->mutable_data(), 0, (*result)->size());
+  return Status::OK();
+}
+
+Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int32_t offset, int32_t length,
+    std::shared_ptr<Buffer>* out) {
+  std::shared_ptr<MutableBuffer> buffer;
+  RETURN_NOT_OK(GetEmptyBitmap(pool, length, &buffer));
+  uint8_t* dest = buffer->mutable_data();
+  for (int64_t i = 0; i < length; ++i) {
+    BitUtil::SetBitTo(dest, i, BitUtil::GetBit(data, i + offset));
+  }
+  *out = buffer;
+  return Status::OK();
+}
+
+bool BitmapEquals(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+    int64_t right_offset, int64_t bit_length) {
+  // TODO(wesm): Make this faster using word-wise comparisons
+  for (int64_t i = 0; i < bit_length; ++i) {
+    if (BitUtil::GetBit(left, left_offset + i) !=
+        BitUtil::GetBit(right, right_offset + i)) {
+      return false;
+    }
+  }
+  return true;
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 5c8055f9c6171..a0fbdd2f92ca1 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -28,6 +28,8 @@
 namespace arrow {
 
 class Buffer;
+class MemoryPool;
+class MutableBuffer;
 class Status;
 
 namespace BitUtil {
@@ -62,6 +64,12 @@ static inline void SetBit(uint8_t* bits, int i) {
   bits[i / 8] |= kBitmask[i % 8];
 }
 
+static inline void SetBitTo(uint8_t* bits, int i, bool bit_is_set) {
+  // See https://graphics.stanford.edu/~seander/bithacks.html
+  // "Conditionally set or clear bits without branching"
+  bits[i / 8] ^= (-bit_is_set ^ bits[i / 8]) & kBitmask[i % 8];
+}
+
 static inline int64_t NextPower2(int64_t n) {
   n--;
   n |= n >> 1;
@@ -82,6 +90,11 @@ static inline bool IsMultipleOf8(int64_t n) {
   return (n & 7) == 0;
 }
 
+/// Returns 'value' rounded up to the nearest multiple of 'factor'
+inline int64_t RoundUp(int64_t value, int64_t factor) {
+  return (value + (factor - 1)) / factor * factor;
+}
+
 inline int64_t RoundUpToMultipleOf64(int64_t num) {
   // TODO(wesm): is this definitely needed?
   // DCHECK_GE(num, 0);
@@ -98,6 +111,38 @@ void BytesToBits(const std::vector<uint8_t>& bytes, uint8_t* bits);
 ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
 }  // namespace BitUtil
+
+// ----------------------------------------------------------------------
+// Bitmap utilities
+
+Status ARROW_EXPORT GetEmptyBitmap(
+    MemoryPool* pool, int64_t length, std::shared_ptr<MutableBuffer>* result);
+
+/// Copy a bit range of an existing bitmap
+///
+/// \param[in] pool memory pool to allocate memory from
+/// \param[in] bitmap source data
+/// \param[in] offset bit offset into the source data
+/// \param[in] length number of bits to copy
+/// \param[out] out the resulting copy
+///
+/// \return Status message
+Status ARROW_EXPORT CopyBitmap(MemoryPool* pool, const uint8_t* bitmap, int32_t offset,
+    int32_t length, std::shared_ptr<Buffer>* out);
+
+/// Compute the number of 1's in the given data array
+///
+/// \param[in] data a packed LSB-ordered bitmap as a byte array
+/// \param[in] bit_offset a bitwise offset into the bitmap
+/// \param[in] length the number of bits to inspect in the bitmap relative to the offset
+///
+/// \return The number of set (1) bits in the range
+int64_t ARROW_EXPORT CountSetBits(
+    const uint8_t* data, int64_t bit_offset, int64_t length);
+
+bool ARROW_EXPORT BitmapEquals(const uint8_t* left, int64_t left_offset,
+    const uint8_t* right, int64_t right_offset, int64_t bit_length);
+
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_BIT_UTIL_H
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index b22f07dd6345f..06ee8411e283c 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -118,9 +118,9 @@ class CerrLog {
 class FatalLog : public CerrLog {
  public:
   explicit FatalLog(int /* severity */)  // NOLINT
-      : CerrLog(ARROW_FATAL){}           // NOLINT
+      : CerrLog(ARROW_FATAL) {}          // NOLINT
 
-            [[noreturn]] ~FatalLog() {
+  [[noreturn]] ~FatalLog() {
     if (has_logged_) { std::cerr << std::endl; }
     std::exit(1);
   }
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index c4a62a475b92f..81a9b0cff5687 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -25,6 +25,6 @@
   TypeName& operator=(const TypeName&) = delete
 #endif
 
-#define UNUSED(x) (void)x
+#define UNUSED(x) (void) x
 
 #endif  // ARROW_UTIL_MACROS_H
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 842a2196dab62..ba26692b32b88 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -95,7 +95,7 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
 
   # Cython warnings in clang
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality -Wno-constant-logical-operand")
 endif()
 
 set(PYARROW_LINK "a")
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 38883e811e1cc..ebfdc410fa004 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -179,8 +179,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         double Value(int i)
 
     cdef cppclass CListArray" arrow::ListArray"(CArray):
-        const int32_t* offsets()
-        int32_t offset(int i)
+        const int32_t* raw_value_offsets()
+        int32_t value_offset(int i)
         int32_t value_length(int i)
         shared_ptr[CArray] values()
         shared_ptr[CDataType] value_type()
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 30b90408dc082..9d2b2b11a80d6 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -202,7 +202,7 @@ cdef class ListValue(ArrayValue):
         self.value_type = box_data_type(self.ap.value_type())
 
     cdef getitem(self, int i):
-        cdef int j = self.ap.offset(self.index) + i
+        cdef int j = self.ap.value_offset(self.index) + i
         return box_arrow_scalar(self.value_type, self.ap.values(), j)
 
     def as_py(self):
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index 1abfb4091189e..5fd8eef23fec5 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -505,7 +505,7 @@ Status ConvertPySequence(
 
   // Handle NA / NullType case
   if (type->type == Type::NA) {
-    out->reset(new arrow::NullArray(type, size));
+    out->reset(new arrow::NullArray(size));
     return Status::OK();
   }
 
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 8d05821c2fd08..345dc9070e5b3 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -1338,8 +1338,7 @@ class ArrowSerializer {
     PyAcquireGIL lock;
 
     PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    arrow::TypePtr string_type(new arrow::DateType());
-    arrow::DateBuilder date_builder(pool_, string_type);
+    arrow::DateBuilder date_builder(pool_);
     RETURN_NOT_OK(date_builder.Resize(length_));
 
     Status s;
@@ -1363,8 +1362,7 @@ class ArrowSerializer {
     // and unicode mixed in the object array
 
     PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    arrow::TypePtr string_type(new arrow::StringType());
-    arrow::StringBuilder string_builder(pool_, string_type);
+    arrow::StringBuilder string_builder(pool_);
     RETURN_NOT_OK(string_builder.Resize(length_));
 
     Status s;
@@ -1374,8 +1372,8 @@ class ArrowSerializer {
 
     if (have_bytes) {
       const auto& arr = static_cast<const arrow::StringArray&>(*out->get());
-      *out = std::make_shared<arrow::BinaryArray>(
-          arr.length(), arr.offsets(), arr.data(), arr.null_count(), arr.null_bitmap());
+      *out = std::make_shared<arrow::BinaryArray>(arr.length(), arr.value_offsets(),
+          arr.data(), arr.null_bitmap(), arr.null_count());
     }
     return Status::OK();
   }
@@ -1403,7 +1401,7 @@ class ArrowSerializer {
       }
     }
 
-    *out = std::make_shared<arrow::BooleanArray>(length_, data, null_count, null_bitmap_);
+    *out = std::make_shared<arrow::BooleanArray>(length_, data, null_bitmap_, null_count);
 
     return Status::OK();
   }
@@ -1515,10 +1513,14 @@ inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
     null_count = ValuesToBitmap<TYPE>(PyArray_DATA(arr_), length_, null_bitmap_data_);
   }
 
+  // For readability
+  constexpr int32_t kOffset = 0;
+
   RETURN_NOT_OK(ConvertData());
   std::shared_ptr<DataType> type;
   RETURN_NOT_OK(MakeDataType(&type));
-  RETURN_NOT_OK(MakePrimitiveArray(type, length_, data_, null_count, null_bitmap_, out));
+  RETURN_NOT_OK(
+      MakePrimitiveArray(type, length_, data_, null_bitmap_, null_count, kOffset, out));
   return Status::OK();
 }
 
@@ -1657,7 +1659,7 @@ ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
   // TODO: If there are bytes involed, convert to Binary representation
   bool have_bytes = false;
 
-  auto value_builder = std::make_shared<arrow::StringBuilder>(pool_, field->type);
+  auto value_builder = std::make_shared<arrow::StringBuilder>(pool_);
   ListBuilder list_builder(pool_, value_builder);
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
   for (int64_t i = 0; i < length_; ++i) {
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index 92352607e62ec..aa4cb7b052c27 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -56,9 +56,20 @@ static Status CheckPyError() {
   return Status::OK();
 }
 
+// This is annoying: because C++11 does not allow implicit conversion of string
+// literals to non-const char*, we need to go through some gymnastics to use
+// PyObject_CallMethod without a lot of pain (its arguments are non-const
+// char*)
+template <typename... ArgTypes>
+static inline PyObject* cpp_PyObject_CallMethod(
+    PyObject* obj, const char* method_name, const char* argspec, ArgTypes... args) {
+  return PyObject_CallMethod(
+      obj, const_cast<char*>(method_name), const_cast<char*>(argspec), args...);
+}
+
 Status PythonFile::Close() {
   // whence: 0 for relative to start of file, 2 for end of file
-  PyObject* result = PyObject_CallMethod(file_, "close", "()");
+  PyObject* result = cpp_PyObject_CallMethod(file_, "close", "()");
   Py_XDECREF(result);
   ARROW_RETURN_NOT_OK(CheckPyError());
   return Status::OK();
@@ -66,14 +77,14 @@ Status PythonFile::Close() {
 
 Status PythonFile::Seek(int64_t position, int whence) {
   // whence: 0 for relative to start of file, 2 for end of file
-  PyObject* result = PyObject_CallMethod(file_, "seek", "(ii)", position, whence);
+  PyObject* result = cpp_PyObject_CallMethod(file_, "seek", "(ii)", position, whence);
   Py_XDECREF(result);
   ARROW_RETURN_NOT_OK(CheckPyError());
   return Status::OK();
 }
 
 Status PythonFile::Read(int64_t nbytes, PyObject** out) {
-  PyObject* result = PyObject_CallMethod(file_, "read", "(i)", nbytes);
+  PyObject* result = cpp_PyObject_CallMethod(file_, "read", "(i)", nbytes);
   ARROW_RETURN_NOT_OK(CheckPyError());
   *out = result;
   return Status::OK();
@@ -84,7 +95,7 @@ Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
       PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), nbytes);
   ARROW_RETURN_NOT_OK(CheckPyError());
 
-  PyObject* result = PyObject_CallMethod(file_, "write", "(O)", py_data);
+  PyObject* result = cpp_PyObject_CallMethod(file_, "write", "(O)", py_data);
   Py_XDECREF(py_data);
   Py_XDECREF(result);
   ARROW_RETURN_NOT_OK(CheckPyError());
@@ -92,7 +103,7 @@ Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
 }
 
 Status PythonFile::Tell(int64_t* position) {
-  PyObject* result = PyObject_CallMethod(file_, "tell", "()");
+  PyObject* result = cpp_PyObject_CallMethod(file_, "tell", "()");
   ARROW_RETURN_NOT_OK(CheckPyError());
 
   *position = PyLong_AsLongLong(result);

From f268e927ada5cb637404769a136506c600582061 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 7 Feb 2017 15:37:33 +0100
Subject: [PATCH 0314/1644] ARROW-540: [C++] Build fixes after ARROW-33,
 PARQUET-866

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Wes McKinney <wesmckinn@gmail.com>

Closes #325 from wesm/ARROW-540 and squashes the following commits:

9070baf [Wes McKinney] Change DCHECK_LT to DCHECK_LE. Not sure why it fixes bug on OS X
eb80701 [Wes McKinney] Fix API change
---
 cpp/src/arrow/buffer.cc           | 2 +-
 cpp/src/arrow/column-benchmark.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index fb5a010efa225..18e9ed2015227 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -57,7 +57,7 @@ Status Buffer::Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out)
 
 std::shared_ptr<Buffer> SliceBuffer(
     const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
-  DCHECK_LT(offset, buffer->size());
+  DCHECK_LE(offset, buffer->size());
   DCHECK_LE(length, buffer->size() - offset);
   return std::make_shared<Buffer>(buffer, offset, length);
 }
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index 8a1c775d7376d..1bab5a8de0ca4 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -30,7 +30,7 @@ std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
   auto null_bitmap = std::make_shared<PoolBuffer>(pool);
   data->Resize(length * sizeof(typename ArrayType::value_type));
   null_bitmap->Resize(BitUtil::BytesForBits(length));
-  return std::make_shared<ArrayType>(length, data, 10, null_bitmap);
+  return std::make_shared<ArrayType>(length, data, null_bitmap, 10);
 }
 }  // anonymous namespace
 

From 4c3481ea5438d52878f390b0f562f6113f111a8f Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 7 Feb 2017 11:13:00 -0500
Subject: [PATCH 0315/1644] ARROW-535: [Python] Add type mapping for
 NPY_LONGLONG

Based on https://github.com/wesm/feather/pull/107

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #323 from xhochy/ARROW-535 and squashes the following commits:

72221fa [Uwe L. Korn] Address review comments
5d3c046 [Uwe L. Korn] ARROW-535: [Python] Add type mapping for NPY_LONGLONG
---
 python/pyarrow/tests/test_convert_pandas.py |  6 +++--
 python/src/pyarrow/adapters/pandas.cc       | 29 +++++++++++++++++++--
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index f04fbe5b139e4..960653dca279e 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -127,13 +127,14 @@ def test_float_nulls(self):
         tm.assert_frame_equal(result, ex_frame)
 
     def test_integer_no_nulls(self):
-        data = {}
+        data = OrderedDict()
         fields = []
 
         numpy_dtypes = [('i1', A.int8()), ('i2', A.int16()),
                         ('i4', A.int32()), ('i8', A.int64()),
                         ('u1', A.uint8()), ('u2', A.uint16()),
-                        ('u4', A.uint32()), ('u8', A.uint64())]
+                        ('u4', A.uint32()), ('u8', A.uint64()),
+                        ('longlong', A.int64()), ('ulonglong', A.uint64())]
         num_values = 100
 
         for dtype, arrow_dtype in numpy_dtypes:
@@ -148,6 +149,7 @@ def test_integer_no_nulls(self):
         schema = A.Schema.from_fields(fields)
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
+
     def test_integer_with_nulls(self):
         # pandas requires upcast to float dtype
 
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 345dc9070e5b3..b4e0d2f9c138e 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -91,11 +91,17 @@ NPY_INT_DECL(INT8, Int8, int8_t);
 NPY_INT_DECL(INT16, Int16, int16_t);
 NPY_INT_DECL(INT32, Int32, int32_t);
 NPY_INT_DECL(INT64, Int64, int64_t);
+
 NPY_INT_DECL(UINT8, UInt8, uint8_t);
 NPY_INT_DECL(UINT16, UInt16, uint16_t);
 NPY_INT_DECL(UINT32, UInt32, uint32_t);
 NPY_INT_DECL(UINT64, UInt64, uint64_t);
 
+#if NPY_INT64 != NPY_LONGLONG
+NPY_INT_DECL(LONGLONG, Int64, int64_t);
+NPY_INT_DECL(ULONGLONG, UInt64, uint64_t);
+#endif
+
 template <>
 struct npy_traits<NPY_FLOAT32> {
   typedef float value_type;
@@ -1706,16 +1712,35 @@ Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     return Status::Invalid("only handle 1-dimensional arrays");
   }
 
-  switch (PyArray_DESCR(arr)->type_num) {
+  int type_num = PyArray_DESCR(arr)->type_num;
+
+#if (NPY_INT64 == NPY_LONGLONG) && (NPY_SIZEOF_LONGLONG == 8)
+  // Both LONGLONG and INT64 can be observed in the wild, which is buggy. We set
+  // U/LONGLONG to U/INT64 so things work properly.
+  if (type_num == NPY_LONGLONG) {
+    type_num = NPY_INT64;
+  }
+  if (type_num == NPY_ULONGLONG) {
+    type_num = NPY_UINT64;
+  }
+#endif
+
+  switch (type_num) {
     TO_ARROW_CASE(BOOL);
     TO_ARROW_CASE(INT8);
     TO_ARROW_CASE(INT16);
     TO_ARROW_CASE(INT32);
     TO_ARROW_CASE(INT64);
+#if (NPY_INT64 != NPY_LONGLONG)
+	TO_ARROW_CASE(LONGLONG);
+#endif
     TO_ARROW_CASE(UINT8);
     TO_ARROW_CASE(UINT16);
     TO_ARROW_CASE(UINT32);
     TO_ARROW_CASE(UINT64);
+#if (NPY_UINT64 != NPY_ULONGLONG)
+	TO_ARROW_CASE(ULONGLONG);
+#endif
     TO_ARROW_CASE(FLOAT32);
     TO_ARROW_CASE(FLOAT64);
     TO_ARROW_CASE(DATETIME);
@@ -1726,7 +1751,7 @@ Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     } break;
     default:
       std::stringstream ss;
-      ss << "unsupported type " << PyArray_DESCR(arr)->type_num << std::endl;
+      ss << "Unsupported numpy type " << PyArray_DESCR(arr)->type_num << std::endl;
       return Status::NotImplemented(ss.str());
   }
   return Status::OK();

From e97fbe6407e8b15c6d3ef745f7a728e01d499a23 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 7 Feb 2017 11:17:28 -0500
Subject: [PATCH 0316/1644] ARROW-531: Python: Document jemalloc, extend Pandas
 section, add Getting Involved

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #321 from xhochy/ARROW-531 and squashes the following commits:

55da9dc [Uwe L. Korn] ARROW-531: Python: Document jemalloc, extend Pandas section, add Getting Involved
---
 python/doc/getting_involved.rst | 37 +++++++++++++++++++++++
 python/doc/index.rst            |  2 ++
 python/doc/install.rst          |  5 ++--
 python/doc/jemalloc.rst         | 52 +++++++++++++++++++++++++++++++++
 python/doc/pandas.rst           |  8 ++++-
 python/doc/parquet.rst          | 47 ++++++++++++++++++++++-------
 6 files changed, 137 insertions(+), 14 deletions(-)
 create mode 100644 python/doc/getting_involved.rst
 create mode 100644 python/doc/jemalloc.rst

diff --git a/python/doc/getting_involved.rst b/python/doc/getting_involved.rst
new file mode 100644
index 0000000000000..90fa3e49aa191
--- /dev/null
+++ b/python/doc/getting_involved.rst
@@ -0,0 +1,37 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Getting Involved
+================
+
+Right now the primary audience for Apache Arrow are the developers of data
+systems; most people will use Apache Arrow indirectly through systems that use
+it for internal data handling and interoperating with other Arrow-enabled
+systems.
+
+Even if you do not plan to contribute to Apache Arrow itself or Arrow
+integrations in other projects, we'd be happy to have you involved:
+
+ * Join the mailing list: send an email to 
+   `dev-subscribe@arrow.apache.org <mailto:dev-subscribe@arrow.apache.org>`_.
+   Share your ideas and use cases for the project or read through the
+   `Archive <http://mail-archives.apache.org/mod_mbox/arrow-dev/>`_.
+ * Follow our activity on `JIRA <https://issues.apache.org/jira/browse/ARROW>`_
+ * Learn the `Format / Specification
+   <https://github.com/apache/arrow/tree/master/format>`_
+ * Chat with us on `Slack <https://apachearrowslackin.herokuapp.com/>`_
+
diff --git a/python/doc/index.rst b/python/doc/index.rst
index 6725ae707d90b..d64354be05520 100644
--- a/python/doc/index.rst
+++ b/python/doc/index.rst
@@ -37,10 +37,12 @@ structures.
    Installing pyarrow <install.rst>
    Pandas <pandas.rst>
    Module Reference <modules.rst>
+   Getting Involved <getting_involved.rst>
 
 .. toctree::
    :maxdepth: 2
    :caption: Additional Features
 
    Parquet format <parquet.rst>
+   jemalloc MemoryPool <jemalloc.rst>
 
diff --git a/python/doc/install.rst b/python/doc/install.rst
index 1bab017301633..4d99fa0caf1de 100644
--- a/python/doc/install.rst
+++ b/python/doc/install.rst
@@ -120,10 +120,11 @@ Install `pyarrow`
 
     cd arrow/python
 
-    # --with-parquet enable the Apache Parquet support in PyArrow
+    # --with-parquet enables the Apache Parquet support in PyArrow
+    # --with-jemalloc enables the jemalloc allocator support in PyArrow
     # --build-type=release disables debugging information and turns on
     #       compiler optimizations for native code
-    python setup.py build_ext --with-parquet --build-type=release install
+    python setup.py build_ext --with-parquet --with--jemalloc --build-type=release install
     python setup.py install
 
 .. warning::
diff --git a/python/doc/jemalloc.rst b/python/doc/jemalloc.rst
new file mode 100644
index 0000000000000..33fe61729c1e9
--- /dev/null
+++ b/python/doc/jemalloc.rst
@@ -0,0 +1,52 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+jemalloc MemoryPool
+===================
+
+Arrow's default :class:`~pyarrow.memory.MemoryPool` uses the system's allocator
+through the POSIX APIs. Although this already provides aligned allocation, the
+POSIX interface doesn't support aligned reallocation. The default reallocation
+strategy is to allocate a new region, copy over the old data and free the
+previous region. Using `jemalloc <http://jemalloc.net/>`_ we can simply extend
+the existing memory allocation to the requested size. While this may still be
+linear in the size of allocated memory, it is magnitudes faster as only the page
+mapping in the kernel is touched, not the actual data.
+
+The :mod:`~pyarrow.jemalloc` allocator is not enabled by default to allow the
+use of the system allocator and/or other allocators like ``tcmalloc``. You can
+either explicitly make it the default allocator or pass it only to single
+operations.
+
+.. code:: python
+
+    import pyarrow as pa
+    import pyarrow.jemalloc
+    import pyarrow.memory
+
+    jemalloc_pool = pyarrow.jemalloc.default_pool()
+
+    # Explicitly use jemalloc for allocating memory for an Arrow Table object
+    array = pa.Array.from_pylist([1, 2, 3], memory_pool=jemalloc_pool)
+
+    # Set the global pool
+    pyarrow.memory.set_default_pool(jemalloc_pool)
+    # This operation has no explicit MemoryPool specified and will thus will
+    # also use jemalloc for its allocations.
+    array = pa.Array.from_pylist([1, 2, 3])
+
+
diff --git a/python/doc/pandas.rst b/python/doc/pandas.rst
index c225d1362c7b6..34445aed517d3 100644
--- a/python/doc/pandas.rst
+++ b/python/doc/pandas.rst
@@ -84,9 +84,11 @@ Pandas -> Arrow Conversion
 +------------------------+--------------------------+
 | ``str`` / ``unicode``  | ``STRING``               |
 +------------------------+--------------------------+
+| ``pd.Categorical``     | ``DICTIONARY``           |
++------------------------+--------------------------+
 | ``pd.Timestamp``       | ``TIMESTAMP(unit=ns)``   |
 +------------------------+--------------------------+
-| ``pd.Categorical``     | *not supported*          |
+| ``datetime.date``      | ``DATE``                 |
 +------------------------+--------------------------+
 
 Arrow -> Pandas Conversion
@@ -109,5 +111,9 @@ Arrow -> Pandas Conversion
 +-------------------------------------+--------------------------------------------------------+
 | ``STRING``                          | ``str``                                                |
 +-------------------------------------+--------------------------------------------------------+
+| ``DICTIONARY``                      | ``pd.Categorical``                                     |
++-------------------------------------+--------------------------------------------------------+
 | ``TIMESTAMP(unit=*)``               | ``pd.Timestamp`` (``np.datetime64[ns]``)               |
 +-------------------------------------+--------------------------------------------------------+
+| ``DATE``                            | ``pd.Timestamp`` (``np.datetime64[ns]``)               |
++-------------------------------------+--------------------------------------------------------+
diff --git a/python/doc/parquet.rst b/python/doc/parquet.rst
index 674ed80f27ce3..8e011e4f19857 100644
--- a/python/doc/parquet.rst
+++ b/python/doc/parquet.rst
@@ -29,16 +29,30 @@ Reading Parquet
 
 To read a Parquet file into Arrow memory, you can use the following code
 snippet. It will read the whole Parquet file into memory as an
-:class:`pyarrow.table.Table`.
+:class:`~pyarrow.table.Table`.
 
 .. code-block:: python
 
-    import pyarrow
-    import pyarrow.parquet
+    import pyarrow.parquet as pq
 
-    A = pyarrow
+    table = pq.read_table('<filename>')
 
-    table = A.parquet.read_table('<filename>')
+As DataFrames stored as Parquet are often stored in multiple files, a
+convenience method :meth:`~pyarrow.parquet.read_multiple_files` is provided.
+
+If you already have the Parquet available in memory or get it via non-file
+source, you can utilize :class:`pyarrow.io.BufferReader` to read it from
+memory. As input to the :class:`~pyarrow.io.BufferReader` you can either supply
+a Python ``bytes`` object or a :class:`pyarrow.io.Buffer`.
+
+.. code:: python
+
+    import pyarrow.io as paio
+    import pyarrow.parquet as pq
+
+    buf = ... # either bytes or paio.Buffer
+    reader = paio.BufferReader(buf)
+    table = pq.read_table(reader)
 
 Writing Parquet
 ---------------
@@ -49,13 +63,11 @@ method.
 
 .. code-block:: python
 
-    import pyarrow
-    import pyarrow.parquet
-
-    A = pyarrow
+    import pyarrow as pa
+    import pyarrow.parquet as pq
 
-    table = A.Table(..)
-    A.parquet.write_table(table, '<filename>')
+    table = pa.Table(..)
+    pq.write_table(table, '<filename>')
 
 By default this will write the Table as a single RowGroup using ``DICTIONARY``
 encoding. To increase the potential of parallelism a query engine can process
@@ -64,3 +76,16 @@ a Parquet file, set the ``chunk_size`` to a fraction of the total number of rows
 If you also want to compress the columns, you can select a compression
 method using the ``compression`` argument. Typically, ``GZIP`` is the choice if
 you want to minimize size and ``SNAPPY`` for performance.
+
+Instead of writing to a file, you can also write to Python ``bytes`` by
+utilizing an :class:`pyarrow.io.InMemoryOutputStream()`:
+
+.. code:: python
+
+    import pyarrow.io as paio
+    import pyarrow.parquet as pq
+
+    table = ...
+    output = paio.InMemoryOutputStream()
+    pq.write_table(table, output)
+    pybytes = output.get_result().to_pybytes()

From c322cbf225b5da5e17ceec0e9e7373852bcba85c Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Tue, 7 Feb 2017 16:44:35 -0500
Subject: [PATCH 0317/1644] ARROW-366 Java Dictionary Vector

I've added a dictionary type, and a partial implementation of a dictionary vector that just wraps an index vector and has a reference to a lookup vector. The spec seems to indicate that any array can be dictionary encoded, but the C++ implementation created a new type, so I went that way.
Feedback would be appreciated - I want to make sure I'm on the right path.

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #309 from elahrvivaz/ARROW-366 and squashes the following commits:

60836ea [Emilio Lahr-Vivaz] removing dictionary ID from encoded vector
0871e13 [Emilio Lahr-Vivaz] ARROW-366 Adding Java dictionary vector
---
 .../vector/complex/DictionaryVector.java      | 229 ++++++++++++++++++
 .../apache/arrow/vector/types/Dictionary.java |  40 +++
 .../apache/arrow/vector/types/pojo/Field.java |  35 ++-
 .../org/apache/arrow/vector/util/Text.java    |  31 ++-
 .../arrow/vector/TestDictionaryVector.java    | 154 ++++++++++++
 5 files changed, 482 insertions(+), 7 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/DictionaryVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/Dictionary.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/DictionaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/DictionaryVector.java
new file mode 100644
index 0000000000000..84760eadf2253
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/DictionaryVector.java
@@ -0,0 +1,229 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.NullableIntVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.Dictionary;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.TransferPair;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class DictionaryVector implements ValueVector {
+
+  private ValueVector indices;
+  private Dictionary dictionary;
+
+  public DictionaryVector(ValueVector indices, Dictionary dictionary) {
+    this.indices = indices;
+    this.dictionary = dictionary;
+  }
+
+  /**
+   * Dictionary encodes a vector. The dictionary will be built using the values from the vector.
+   *
+   * @param vector vector to encode
+   * @return dictionary encoded vector
+   */
+  public static DictionaryVector encode(ValueVector vector) {
+    validateType(vector.getMinorType());
+    Map<Object, Integer> lookUps = new HashMap<>();
+    Map<Integer, Integer> transfers = new HashMap<>();
+
+    ValueVector.Accessor accessor = vector.getAccessor();
+    int count = accessor.getValueCount();
+
+    NullableIntVector indices = new NullableIntVector(vector.getField().getName(), vector.getAllocator());
+    indices.allocateNew(count);
+    NullableIntVector.Mutator mutator = indices.getMutator();
+
+    int nextIndex = 0;
+    for (int i = 0; i < count; i++) {
+      Object value = accessor.getObject(i);
+      if (value != null) { // if it's null leave it null
+        Integer index = lookUps.get(value);
+        if (index == null) {
+          index = nextIndex++;
+          lookUps.put(value, index);
+          transfers.put(i, index);
+        }
+        mutator.set(i, index);
+      }
+    }
+    mutator.setValueCount(count);
+
+    // copy the dictionary values into the dictionary vector
+    TransferPair dictionaryTransfer = vector.getTransferPair(vector.getAllocator());
+    ValueVector dictionaryVector = dictionaryTransfer.getTo();
+    dictionaryVector.allocateNewSafe();
+    for (Map.Entry<Integer, Integer> entry: transfers.entrySet()) {
+      dictionaryTransfer.copyValueSafe(entry.getKey(), entry.getValue());
+    }
+    dictionaryVector.getMutator().setValueCount(transfers.size());
+    Dictionary dictionary = new Dictionary(dictionaryVector, false);
+
+    return new DictionaryVector(indices, dictionary);
+  }
+
+  /**
+   * Dictionary encodes a vector with a provided dictionary. The dictionary must contain all values in the vector.
+   *
+   * @param vector vector to encode
+   * @param dictionary dictionary used for encoding
+   * @return dictionary encoded vector
+   */
+  public static DictionaryVector encode(ValueVector vector, Dictionary dictionary) {
+    validateType(vector.getMinorType());
+    // load dictionary values into a hashmap for lookup
+    ValueVector.Accessor dictionaryAccessor = dictionary.getDictionary().getAccessor();
+    Map<Object, Integer> lookUps = new HashMap<>(dictionaryAccessor.getValueCount());
+    for (int i = 0; i < dictionaryAccessor.getValueCount(); i++) {
+      // for primitive array types we need a wrapper that implements equals and hashcode appropriately
+      lookUps.put(dictionaryAccessor.getObject(i), i);
+    }
+
+    // vector to hold our indices (dictionary encoded values)
+    NullableIntVector indices = new NullableIntVector(vector.getField().getName(), vector.getAllocator());
+    NullableIntVector.Mutator mutator = indices.getMutator();
+
+    ValueVector.Accessor accessor = vector.getAccessor();
+    int count = accessor.getValueCount();
+
+    indices.allocateNew(count);
+
+    for (int i = 0; i < count; i++) {
+      Object value = accessor.getObject(i);
+      if (value != null) { // if it's null leave it null
+        // note: this may fail if value was not included in the dictionary
+        mutator.set(i, lookUps.get(value));
+      }
+    }
+    mutator.setValueCount(count);
+
+    return new DictionaryVector(indices, dictionary);
+  }
+
+  /**
+   * Decodes a dictionary encoded array using the provided dictionary.
+   *
+   * @param indices dictionary encoded values, must be int type
+   * @param dictionary dictionary used to decode the values
+   * @return vector with values restored from dictionary
+   */
+  public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
+    ValueVector.Accessor accessor = indices.getAccessor();
+    int count = accessor.getValueCount();
+    ValueVector dictionaryVector = dictionary.getDictionary();
+    // copy the dictionary values into the decoded vector
+    TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator());
+    transfer.getTo().allocateNewSafe();
+    for (int i = 0; i < count; i++) {
+      Object index = accessor.getObject(i);
+      if (index != null) {
+        transfer.copyValueSafe(((Number) index).intValue(), i);
+      }
+    }
+
+    ValueVector decoded = transfer.getTo();
+    decoded.getMutator().setValueCount(count);
+    return decoded;
+  }
+
+  private static void validateType(MinorType type) {
+    // byte arrays don't work as keys in our dictionary map - we could wrap them with something to
+    // implement equals and hashcode if we want that functionality
+    if (type == MinorType.VARBINARY || type == MinorType.LIST || type == MinorType.MAP || type == MinorType.UNION) {
+      throw new IllegalArgumentException("Dictionary encoding for complex types not implemented");
+    }
+  }
+
+  public ValueVector getIndexVector() { return indices; }
+
+  public ValueVector getDictionaryVector() { return dictionary.getDictionary(); }
+
+  public Dictionary getDictionary() { return dictionary; }
+
+  @Override
+  public MinorType getMinorType() { return indices.getMinorType(); }
+
+  @Override
+  public Field getField() { return indices.getField(); }
+
+  // note: dictionary vector is not closed, as it may be shared
+  @Override
+  public void close() { indices.close(); }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException { indices.allocateNew(); }
+
+  @Override
+  public boolean allocateNewSafe() { return indices.allocateNewSafe(); }
+
+  @Override
+  public BufferAllocator getAllocator() { return indices.getAllocator();  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) { indices.setInitialCapacity(numRecords); }
+
+  @Override
+  public int getValueCapacity() { return indices.getValueCapacity(); }
+
+  @Override
+  public int getBufferSize() { return indices.getBufferSize(); }
+
+  @Override
+  public int getBufferSizeFor(int valueCount) { return indices.getBufferSizeFor(valueCount); }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return indices.iterator();
+  }
+
+  @Override
+  public void clear() { indices.clear(); }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) { return indices.getTransferPair(allocator); }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return indices.getTransferPair(ref, allocator); }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector target) { return indices.makeTransferPair(target); }
+
+  @Override
+  public Accessor getAccessor() { return indices.getAccessor(); }
+
+  @Override
+  public Mutator getMutator() { return indices.getMutator(); }
+
+  @Override
+  public FieldReader getReader() { return indices.getReader(); }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) { return indices.getBuffers(clear); }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Dictionary.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Dictionary.java
new file mode 100644
index 0000000000000..fbe1345f96aa3
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Dictionary.java
@@ -0,0 +1,40 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.types;
+
+import org.apache.arrow.vector.ValueVector;
+
+public class Dictionary {
+
+    private ValueVector dictionary;
+    private boolean ordered;
+
+    public Dictionary(ValueVector dictionary, boolean ordered) {
+        this.dictionary = dictionary;
+        this.ordered = ordered;
+    }
+
+    public ValueVector getDictionary() {
+        return dictionary;
+    }
+
+    public boolean isOrdered() {
+        return ordered;
+    }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 412fc54b538da..2d528e4141907 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -24,6 +24,9 @@
 import java.util.List;
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import org.apache.arrow.flatbuf.DictionaryEncoding;
 import org.apache.arrow.vector.schema.TypeLayout;
 import org.apache.arrow.vector.schema.VectorLayout;
 
@@ -37,6 +40,7 @@ public class Field {
   private final String name;
   private final boolean nullable;
   private final ArrowType type;
+  private final Long dictionary;
   private final List<Field> children;
   private final TypeLayout typeLayout;
 
@@ -45,11 +49,13 @@ private Field(
       @JsonProperty("name") String name,
       @JsonProperty("nullable") boolean nullable,
       @JsonProperty("type") ArrowType type,
+      @JsonProperty("dictionary") Long dictionary,
       @JsonProperty("children") List<Field> children,
       @JsonProperty("typeLayout") TypeLayout typeLayout) {
     this.name = name;
     this.nullable = nullable;
     this.type = checkNotNull(type);
+    this.dictionary = dictionary;
     if (children == null) {
       this.children = ImmutableList.of();
     } else {
@@ -59,13 +65,22 @@ private Field(
   }
 
   public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
-    this(name, nullable, type, children, TypeLayout.getTypeLayout(checkNotNull(type)));
+    this(name, nullable, type, null, children, TypeLayout.getTypeLayout(checkNotNull(type)));
+  }
+
+  public Field(String name, boolean nullable, ArrowType type, Long dictionary, List<Field> children) {
+    this(name, nullable, type, dictionary, children, TypeLayout.getTypeLayout(checkNotNull(type)));
   }
 
   public static Field convertField(org.apache.arrow.flatbuf.Field field) {
     String name = field.name();
     boolean nullable = field.nullable();
     ArrowType type = getTypeForField(field);
+    DictionaryEncoding dictionaryEncoding = field.dictionary();
+    Long dictionary = null;
+    if (dictionaryEncoding != null) {
+      dictionary = dictionaryEncoding.id();
+    }
     ImmutableList.Builder<org.apache.arrow.vector.schema.VectorLayout> layout = ImmutableList.builder();
     for (int i = 0; i < field.layoutLength(); ++i) {
       layout.add(new org.apache.arrow.vector.schema.VectorLayout(field.layout(i)));
@@ -75,8 +90,7 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
       childrenBuilder.add(convertField(field.children(i)));
     }
     List<Field> children = childrenBuilder.build();
-    Field result = new Field(name, nullable, type, children, new TypeLayout(layout.build()));
-    return result;
+    return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()));
   }
 
   public void validate() {
@@ -89,6 +103,11 @@ public void validate() {
   public int getField(FlatBufferBuilder builder) {
     int nameOffset = name == null ? -1 : builder.createString(name);
     int typeOffset = type.getType(builder);
+    int dictionaryOffset = -1;
+    if (dictionary != null) {
+      builder.addLong(dictionary);
+      dictionaryOffset = builder.offset();
+    }
     int[] childrenData = new int[children.size()];
     for (int i = 0; i < children.size(); i++) {
       childrenData[i] = children.get(i).getField(builder);
@@ -107,6 +126,9 @@ public int getField(FlatBufferBuilder builder) {
     org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
     org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeID().getFlatbufID());
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
+    if (dictionary != null) {
+      org.apache.arrow.flatbuf.Field.addDictionary(builder, dictionaryOffset);
+    }
     org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
     org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
     return org.apache.arrow.flatbuf.Field.endField(builder);
@@ -124,6 +146,9 @@ public ArrowType getType() {
     return type;
   }
 
+  @JsonInclude(Include.NON_NULL)
+  public Long getDictionary() { return dictionary; }
+
   public List<Field> getChildren() {
     return children;
   }
@@ -141,6 +166,7 @@ public boolean equals(Object obj) {
     return Objects.equals(this.name, that.name) &&
             Objects.equals(this.nullable, that.nullable) &&
             Objects.equals(this.type, that.type) &&
+           Objects.equals(this.dictionary, that.dictionary) &&
             (Objects.equals(this.children, that.children) ||
                     (this.children == null && that.children.size() == 0) ||
                     (this.children.size() == 0 && that.children == null));
@@ -153,6 +179,9 @@ public String toString() {
       sb.append(name).append(": ");
     }
     sb.append(type);
+    if (dictionary != null) {
+      sb.append("[dictionary: ").append(dictionary).append("]");
+    }
     if (!children.isEmpty()) {
       sb.append("<").append(Joiner.on(", ").join(children)).append(">");
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
index 3919f0606cb20..3db4358ea9155 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
@@ -299,6 +299,11 @@ public void readWithKnownLength(DataInput in, int len) throws IOException {
   /** Returns true iff <code>o</code> is a Text with the same contents. */
   @Override
   public boolean equals(Object o) {
+    if (o == this) {
+      return true;
+    } else if (o == null) {
+      return false;
+    }
     if (!(o instanceof Text)) {
       return false;
     }
@@ -308,15 +313,33 @@ public boolean equals(Object o) {
       return false;
     }
 
-    byte[] thisBytes = Arrays.copyOf(this.getBytes(), getLength());
-    byte[] thatBytes = Arrays.copyOf(that.getBytes(), getLength());
-    return Arrays.equals(thisBytes, thatBytes);
+    // copied from Arrays.equals so we don'thave to copy the byte arrays
+    for (int i = 0; i < length; i++) {
+      if (bytes[i] != that.bytes[i]) {
+        return false;
+      }
+    }
 
+    return true;
   }
 
+  /**
+   * Copied from Arrays.hashCode so we don't have to copy the byte array
+   *
+   * @return
+   */
   @Override
   public int hashCode() {
-    return super.hashCode();
+    if (bytes == null) {
+      return 0;
+    }
+
+    int result = 1;
+    for (int i = 0; i < length; i++) {
+      result = 31 * result + bytes[i];
+    }
+
+    return result;
   }
 
   // / STATIC UTILITIES FROM HERE DOWN
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
new file mode 100644
index 0000000000000..962950abec87a
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.DictionaryVector;
+import org.apache.arrow.vector.types.Dictionary;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class TestDictionaryVector {
+
+  private BufferAllocator allocator;
+
+  byte[] zero = "foo".getBytes(StandardCharsets.UTF_8);
+  byte[] one  = "bar".getBytes(StandardCharsets.UTF_8);
+  byte[] two  = "baz".getBytes(StandardCharsets.UTF_8);
+
+  @Before
+  public void init() {
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testEncodeStringsWithGeneratedDictionary() {
+    // Create a new value vector
+    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("foo", allocator, null)) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
+      vector.allocateNew(512, 5);
+
+      // set some values
+      m.setSafe(0, zero, 0, zero.length);
+      m.setSafe(1, one, 0, one.length);
+      m.setSafe(2, one, 0, one.length);
+      m.setSafe(3, two, 0, two.length);
+      m.setSafe(4, zero, 0, zero.length);
+      m.setValueCount(5);
+
+      DictionaryVector encoded = DictionaryVector.encode(vector);
+
+      try {
+        // verify values in the dictionary
+        ValueVector dictionary = encoded.getDictionaryVector();
+        assertEquals(vector.getClass(), dictionary.getClass());
+
+        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary).getAccessor();
+        assertEquals(3, dictionaryAccessor.getValueCount());
+        assertArrayEquals(zero, dictionaryAccessor.get(0));
+        assertArrayEquals(one, dictionaryAccessor.get(1));
+        assertArrayEquals(two, dictionaryAccessor.get(2));
+
+        // verify indices
+        ValueVector indices = encoded.getIndexVector();
+        assertEquals(NullableIntVector.class, indices.getClass());
+
+        NullableIntVector.Accessor indexAccessor = ((NullableIntVector) indices).getAccessor();
+        assertEquals(5, indexAccessor.getValueCount());
+        assertEquals(0, indexAccessor.get(0));
+        assertEquals(1, indexAccessor.get(1));
+        assertEquals(1, indexAccessor.get(2));
+        assertEquals(2, indexAccessor.get(3));
+        assertEquals(0, indexAccessor.get(4));
+
+        // now run through the decoder and verify we get the original back
+        try (ValueVector decoded = DictionaryVector.decode(indices, encoded.getDictionary())) {
+          assertEquals(vector.getClass(), decoded.getClass());
+          assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount());
+          for (int i = 0; i < 5; i++) {
+            assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i));
+          }
+        }
+      } finally {
+        encoded.getDictionaryVector().close();
+        encoded.getIndexVector().close();
+      }
+    }
+  }
+
+  @Test
+  public void testEncodeStringsWithProvidedDictionary() {
+    // Create a new value vector
+    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("foo", allocator, null);
+         final NullableVarCharVector dictionary = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("dict", allocator, null)) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
+      vector.allocateNew(512, 5);
+
+      // set some values
+      m.setSafe(0, zero, 0, zero.length);
+      m.setSafe(1, one, 0, one.length);
+      m.setSafe(2, one, 0, one.length);
+      m.setSafe(3, two, 0, two.length);
+      m.setSafe(4, zero, 0, zero.length);
+      m.setValueCount(5);
+
+      // set some dictionary values
+      final NullableVarCharVector.Mutator m2 = dictionary.getMutator();
+      dictionary.allocateNew(512, 3);
+      m2.setSafe(0, zero, 0, zero.length);
+      m2.setSafe(1, one, 0, one.length);
+      m2.setSafe(2, two, 0, two.length);
+      m2.setValueCount(3);
+
+      try(final DictionaryVector encoded = DictionaryVector.encode(vector, new Dictionary(dictionary, false))) {
+        // verify indices
+        ValueVector indices = encoded.getIndexVector();
+        assertEquals(NullableIntVector.class, indices.getClass());
+
+        NullableIntVector.Accessor indexAccessor = ((NullableIntVector) indices).getAccessor();
+        assertEquals(5, indexAccessor.getValueCount());
+        assertEquals(0, indexAccessor.get(0));
+        assertEquals(1, indexAccessor.get(1));
+        assertEquals(1, indexAccessor.get(2));
+        assertEquals(2, indexAccessor.get(3));
+        assertEquals(0, indexAccessor.get(4));
+
+        // now run through the decoder and verify we get the original back
+        try (ValueVector decoded = DictionaryVector.decode(indices, encoded.getDictionary())) {
+          assertEquals(vector.getClass(), decoded.getClass());
+          assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount());
+          for (int i = 0; i < 5; i++) {
+            assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i));
+          }
+        }
+      }
+    }
+  }
+}

From 1407abfc90c03e133f198b59fed48469d171c0a9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 8 Feb 2017 09:16:57 +0100
Subject: [PATCH 0318/1644] ARROW-537: [C++] Do not compare String/Binary data
 in null slots when comparing arrays

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #327 from wesm/ARROW-537 and squashes the following commits:

66b1961 [Wes McKinney] Do not compare String/Binary data in null slots when comparing arrays
---
 cpp/src/arrow/array-string-test.cc    | 41 ++++++++++++++++++++
 cpp/src/arrow/array.cc                | 11 ++++--
 cpp/src/arrow/array.h                 |  9 +++--
 cpp/src/arrow/compare.cc              | 55 ++++++++++++++++++---------
 python/src/pyarrow/adapters/pandas.cc | 12 ++----
 5 files changed, 95 insertions(+), 33 deletions(-)

diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index 8b7eb41d4c3b9..c4d9bf40f57f9 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -140,6 +140,47 @@ TEST_F(TestStringArray, TestEmptyStringComparison) {
   ASSERT_TRUE(strings_a->Equals(strings_b));
 }
 
+TEST_F(TestStringArray, CompareNullByteSlots) {
+  StringBuilder builder(default_memory_pool());
+  StringBuilder builder2(default_memory_pool());
+  StringBuilder builder3(default_memory_pool());
+
+  builder.Append("foo");
+  builder2.Append("foo");
+  builder3.Append("foo");
+
+  builder.Append("bar");
+  builder2.AppendNull();
+
+  // same length, but different
+  builder3.Append("xyz");
+
+  builder.Append("baz");
+  builder2.Append("baz");
+  builder3.Append("baz");
+
+  std::shared_ptr<Array> array, array2, array3;
+  ASSERT_OK(builder.Finish(&array));
+  ASSERT_OK(builder2.Finish(&array2));
+  ASSERT_OK(builder3.Finish(&array3));
+
+  const auto& a1 = static_cast<const StringArray&>(*array);
+  const auto& a2 = static_cast<const StringArray&>(*array2);
+  const auto& a3 = static_cast<const StringArray&>(*array3);
+
+  // The validity bitmaps are the same, the data is different, but the unequal
+  // portion is masked out
+  StringArray equal_array(3, a1.value_offsets(), a1.data(), a2.null_bitmap(), 1);
+  StringArray equal_array2(3, a3.value_offsets(), a3.data(), a2.null_bitmap(), 1);
+
+  ASSERT_TRUE(equal_array.Equals(equal_array2));
+  ASSERT_TRUE(a2.RangeEquals(equal_array2, 0, 3, 0));
+
+  ASSERT_TRUE(equal_array.Array::Slice(1)->Equals(equal_array2.Array::Slice(1)));
+  ASSERT_TRUE(
+      equal_array.Array::Slice(1)->RangeEquals(0, 2, 0, equal_array2.Array::Slice(1)));
+}
+
 // ----------------------------------------------------------------------
 // String builder tests
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index f84023e6c7d31..39459a031f4b0 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -87,11 +87,16 @@ bool Array::ApproxEquals(const std::shared_ptr<Array>& arr) const {
 }
 
 bool Array::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-    const std::shared_ptr<Array>& arr) const {
-  if (!arr) { return false; }
+    const std::shared_ptr<Array>& other) const {
+  if (!other) { return false; }
+  return RangeEquals(*other, start_idx, end_idx, other_start_idx);
+}
+
+bool Array::RangeEquals(const Array& other, int32_t start_idx, int32_t end_idx,
+    int32_t other_start_idx) const {
   bool are_equal = false;
   Status error =
-      ArrayRangeEquals(*this, *arr, start_idx, end_idx, other_start_idx, &are_equal);
+      ArrayRangeEquals(*this, other, start_idx, end_idx, other_start_idx, &are_equal);
   if (!error.ok()) { DCHECK(false) << "Arrays not comparable: " << error.ToString(); }
   return are_equal;
 }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index f3e8f9a4982f7..32d156b8cd0f6 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -127,7 +127,10 @@ class ARROW_EXPORT Array {
   /// Compare if the range of slots specified are equal for the given array and
   /// this array.  end_idx exclusive.  This methods does not bounds check.
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
-      const std::shared_ptr<Array>& arr) const;
+      const std::shared_ptr<Array>& other) const;
+
+  bool RangeEquals(const Array& other, int32_t start_idx, int32_t end_idx,
+      int32_t other_start_idx) const;
 
   /// Determines if the array is internally consistent.
   ///
@@ -315,8 +318,8 @@ class ARROW_EXPORT BinaryArray : public Array {
     // Account for base offset
     i += offset_;
 
-    const int32_t pos = raw_value_offsets_[i];
-    *out_length = raw_value_offsets_[i + 1] - pos;
+    const int32_t pos = raw_value_offsets_[i + offset_];
+    *out_length = raw_value_offsets_[i + offset_ + 1] - pos;
     return raw_data_ + pos;
   }
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 27fad7135721c..21fdb6633a9ee 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -335,15 +335,8 @@ class EqualsVisitor : public RangeEqualsVisitor {
     const int value_byte_size = size_meta.bit_width() / 8;
     DCHECK_GT(value_byte_size, 0);
 
-    const uint8_t* left_data = nullptr;
-    if (left.length() > 0) {
-      left_data = left.data()->data() + left.offset() * value_byte_size;
-    }
-
-    const uint8_t* right_data = nullptr;
-    if (right.length() > 0) {
-      right_data = right.data()->data() + right.offset() * value_byte_size;
-    }
+    const uint8_t* left_data = left.data()->data() + left.offset() * value_byte_size;
+    const uint8_t* right_data = right.data()->data() + right.offset() * value_byte_size;
 
     if (left.null_count() > 0) {
       for (int i = 0; i < left.length(); ++i) {
@@ -355,7 +348,6 @@ class EqualsVisitor : public RangeEqualsVisitor {
       }
       return true;
     } else {
-      if (left.length() == 0) { return true; }
       return memcmp(left_data, right_data, value_byte_size * left.length()) == 0;
     }
   }
@@ -424,14 +416,35 @@ class EqualsVisitor : public RangeEqualsVisitor {
     bool equal_offsets = ValueOffsetsEqual<BinaryArray>(left);
     if (!equal_offsets) { return false; }
 
-    if (left.offset() == 0 && right.offset() == 0) {
-      if (!left.data() && !(right.data())) { return true; }
-      return left.data()->Equals(*right.data(), left.raw_value_offsets()[left.length()]);
+    if (!left.data() && !(right.data())) { return true; }
+    if (left.value_offset(left.length()) == 0) { return true; }
+
+    const uint8_t* left_data = left.data()->data();
+    const uint8_t* right_data = right.data()->data();
+
+    if (left.null_count() == 0) {
+      // Fast path for null count 0, single memcmp
+      if (left.offset() == 0 && right.offset() == 0) {
+        return std::memcmp(
+                   left_data, right_data, left.raw_value_offsets()[left.length()]) == 0;
+      } else {
+        const int64_t total_bytes =
+            left.value_offset(left.length()) - left.value_offset(0);
+        return std::memcmp(left_data + left.value_offset(0),
+                   right_data + right.value_offset(0), total_bytes) == 0;
+      }
     } else {
-      // Compare the corresponding data range
-      const int64_t total_bytes = left.value_offset(left.length()) - left.value_offset(0);
-      return std::memcmp(left.data()->data() + left.value_offset(0),
-                 right.data()->data() + right.value_offset(0), total_bytes) == 0;
+      // ARROW-537: Only compare data in non-null slots
+      const int32_t* left_offsets = left.raw_value_offsets();
+      const int32_t* right_offsets = right.raw_value_offsets();
+      for (int32_t i = 0; i < left.length(); ++i) {
+        if (left.IsNull(i)) { continue; }
+        if (std::memcmp(left_data + left_offsets[i], right_data + right_offsets[i],
+                left.value_length(i))) {
+          return false;
+        }
+      }
+      return true;
     }
   }
 
@@ -485,8 +498,6 @@ inline bool FloatingApproxEquals(
 
   static constexpr T EPSILON = 1E-5;
 
-  if (left.length() == 0 && right.length() == 0) { return true; }
-
   if (left.null_count() > 0) {
     for (int32_t i = 0; i < left.length(); ++i) {
       if (left.IsNull(i)) continue;
@@ -535,6 +546,8 @@ Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
     *are_equal = true;
   } else if (!BaseDataEquals(left, right)) {
     *are_equal = false;
+  } else if (left.length() == 0) {
+    *are_equal = true;
   } else {
     EqualsVisitor visitor(right);
     RETURN_NOT_OK(left.Accept(&visitor));
@@ -549,6 +562,8 @@ Status ArrayRangeEquals(const Array& left, const Array& right, int32_t left_star
     *are_equal = true;
   } else if (left.type_enum() != right.type_enum()) {
     *are_equal = false;
+  } else if (left.length() == 0) {
+    *are_equal = true;
   } else {
     RangeEqualsVisitor visitor(right, left_start_idx, left_end_idx, right_start_idx);
     RETURN_NOT_OK(left.Accept(&visitor));
@@ -563,6 +578,8 @@ Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal)
     *are_equal = true;
   } else if (!BaseDataEquals(left, right)) {
     *are_equal = false;
+  } else if (left.length() == 0) {
+    *are_equal = true;
   } else {
     ApproxEqualsVisitor visitor(right);
     RETURN_NOT_OK(left.Accept(&visitor));
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index b4e0d2f9c138e..bdc2cb7d0025f 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -1717,12 +1717,8 @@ Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
 #if (NPY_INT64 == NPY_LONGLONG) && (NPY_SIZEOF_LONGLONG == 8)
   // Both LONGLONG and INT64 can be observed in the wild, which is buggy. We set
   // U/LONGLONG to U/INT64 so things work properly.
-  if (type_num == NPY_LONGLONG) {
-    type_num = NPY_INT64;
-  }
-  if (type_num == NPY_ULONGLONG) {
-    type_num = NPY_UINT64;
-  }
+  if (type_num == NPY_LONGLONG) { type_num = NPY_INT64; }
+  if (type_num == NPY_ULONGLONG) { type_num = NPY_UINT64; }
 #endif
 
   switch (type_num) {
@@ -1732,14 +1728,14 @@ Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
     TO_ARROW_CASE(INT32);
     TO_ARROW_CASE(INT64);
 #if (NPY_INT64 != NPY_LONGLONG)
-	TO_ARROW_CASE(LONGLONG);
+    TO_ARROW_CASE(LONGLONG);
 #endif
     TO_ARROW_CASE(UINT8);
     TO_ARROW_CASE(UINT16);
     TO_ARROW_CASE(UINT32);
     TO_ARROW_CASE(UINT64);
 #if (NPY_UINT64 != NPY_ULONGLONG)
-	TO_ARROW_CASE(ULONGLONG);
+    TO_ARROW_CASE(ULONGLONG);
 #endif
     TO_ARROW_CASE(FLOAT32);
     TO_ARROW_CASE(FLOAT64);

From b99d049c3d1894908b7e52774eb657675dc1f439 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 8 Feb 2017 11:20:48 -0500
Subject: [PATCH 0319/1644] ARROW-351: Time type has no unit

Author: Julien Le Dem <julien@dremio.com>

Closes #328 from julienledem/arrow_351 and squashes the following commits:

2497ee3 [Julien Le Dem] ARROW-351: Time type has no unit
---
 format/Message.fbs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index 028c56ad51618..86dfa87b04807 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -79,11 +79,12 @@ table Decimal {
 table Date {
 }
 
+enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }
+
 table Time {
+  unit: TimeUnit;
 }
 
-enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }
-
 /// time from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
 table Timestamp {
   unit: TimeUnit;

From 4440e4011d88967a53054486f9eb0a0363a1c217 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 8 Feb 2017 15:05:29 -0500
Subject: [PATCH 0320/1644] ARROW-543: C++: Lazily computed null_counts counts
 number of non-null entries

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #329 from xhochy/ARROW-543 and squashes the following commits:

191792b [Uwe L. Korn] ARROW-543: C++: Lazily computed null_counts counts number of non-null entries
---
 cpp/src/arrow/array-test.cc | 8 ++++----
 cpp/src/arrow/array.cc      | 3 ++-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 45130d8f64004..45ab2740b4c16 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -88,20 +88,20 @@ TEST_F(TestArray, TestEquality) {
 }
 
 TEST_F(TestArray, SliceRecomputeNullCount) {
-  std::vector<uint8_t> valid_bytes = {1, 0, 1, 1, 0, 1, 0, 0};
+  std::vector<uint8_t> valid_bytes = {1, 0, 1, 1, 0, 1, 0, 0, 0};
 
   auto array = MakeArrayFromValidBytes(valid_bytes, pool_);
 
-  ASSERT_EQ(4, array->null_count());
+  ASSERT_EQ(5, array->null_count());
 
   auto slice = array->Slice(1, 4);
   ASSERT_EQ(2, slice->null_count());
 
   slice = array->Slice(4);
-  ASSERT_EQ(1, slice->null_count());
+  ASSERT_EQ(4, slice->null_count());
 
   slice = array->Slice(0);
-  ASSERT_EQ(4, slice->null_count());
+  ASSERT_EQ(5, slice->null_count());
 
   // No bitmap, compute 0
   std::shared_ptr<MutableBuffer> data;
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 39459a031f4b0..bf368d91226be 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -48,13 +48,14 @@ Array::Array(const std::shared_ptr<DataType>& type, int32_t length,
       null_count_(null_count),
       null_bitmap_(null_bitmap),
       null_bitmap_data_(nullptr) {
+  if (null_count_ == 0) { null_bitmap_ = nullptr; }
   if (null_bitmap_) { null_bitmap_data_ = null_bitmap_->data(); }
 }
 
 int32_t Array::null_count() const {
   if (null_count_ < 0) {
     if (null_bitmap_) {
-      null_count_ = CountSetBits(null_bitmap_data_, offset_, length_);
+      null_count_ = length_ - CountSetBits(null_bitmap_data_, offset_, length_);
     } else {
       null_count_ = 0;
     }

From 0bdfd5efb2d7360f8ec8f6a65401d4c76a8df597 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 8 Feb 2017 15:07:39 -0500
Subject: [PATCH 0321/1644] ARROW-538: [C++] Set up AddressSanitizer (ASAN)
 builds

Most of the infrastructure was already in place, only needed to fix the gtest build.

We will now build with AddressSanitizer activated on OSX.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #324 from xhochy/ARROW-538 and squashes the following commits:

c2f8dda [Uwe L. Korn] Don't run AddressSanitizer on Travis
f6b65e5 [Uwe L. Korn] Explicitly detected 3.6
8a20d91 [Uwe L. Korn] Log detected COMPILER_VERSION in error message
acf3f69 [Uwe L. Korn] ARROW-538: [C++] Set up AddressSanitizer (ASAN) builds
---
 cpp/CMakeLists.txt                       |  4 +++-
 cpp/build-support/run-test.sh            |  4 ++--
 cpp/build-support/sanitize-blacklist.txt | 22 ++++++++++++++++++++++
 cpp/cmake_modules/CompilerInfo.cmake     |  6 ++++++
 cpp/cmake_modules/san-config.cmake       |  5 +++--
 cpp/src/arrow/buffer-test.cc             |  3 +++
 cpp/src/arrow/memory_pool-test.cc        |  4 +++-
 python/manylinux1/Dockerfile-x86_64      |  2 +-
 8 files changed, 43 insertions(+), 7 deletions(-)
 create mode 100644 cpp/build-support/sanitize-blacklist.txt

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index ff2c1a61b95a6..035cd8f9b90c7 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -428,10 +428,12 @@ if(ARROW_BUILD_TESTS)
 
   if("$ENV{GTEST_HOME}" STREQUAL "")
     if(APPLE)
-      set(GTEST_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
+      set(GTEST_CMAKE_CXX_FLAGS "-fPIC -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
     else()
       set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
     endif()
+    string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
+    set(GTEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}")
 
     set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
     set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh
index f563da53679be..b4da4f3f02ee4 100755
--- a/cpp/build-support/run-test.sh
+++ b/cpp/build-support/run-test.sh
@@ -82,8 +82,8 @@ function setup_sanitizers() {
 
   # Enable leak detection even under LLVM 3.4, where it was disabled by default.
   # This flag only takes effect when running an ASAN build.
-  ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
-  export ASAN_OPTIONS
+  # ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
+  # export ASAN_OPTIONS
 
   # Set up suppressions for LeakSanitizer
   LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt"
diff --git a/cpp/build-support/sanitize-blacklist.txt b/cpp/build-support/sanitize-blacklist.txt
new file mode 100644
index 0000000000000..f6900c643db90
--- /dev/null
+++ b/cpp/build-support/sanitize-blacklist.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Workaround for a problem with gmock where a runtime error is caused by a call on a null pointer,
+# on a mocked object.
+# Seen error:
+# thirdparty/gmock-1.7.0/include/gmock/gmock-spec-builders.h:1529:12: runtime error: member call on null pointer of type 'testing::internal::ActionResultHolder<void>'
+fun:*testing*internal*InvokeWith*
diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index 187698f54507b..fe200be65d502 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -30,6 +30,12 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
   set(COMPILER_FAMILY "clang")
   string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
     COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+
+# LLVM 3.6 on Mac OS X 10.9 and later
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM 3\\.6\\..*")
+  set(COMPILER_FAMILY "clang")
+  set(COMPILER_VERSION "3.6.0svn")
+
 # clang on Mac OS X 10.9 and later
 elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
   set(COMPILER_FAMILY "clang")
diff --git a/cpp/cmake_modules/san-config.cmake b/cpp/cmake_modules/san-config.cmake
index fe52fef12ea5d..1917eabe8b4b2 100644
--- a/cpp/cmake_modules/san-config.cmake
+++ b/cpp/cmake_modules/san-config.cmake
@@ -94,8 +94,9 @@ if ("${ARROW_USE_UBSAN}" OR "${ARROW_USE_ASAN}" OR "${ARROW_USE_TSAN}")
     # Require clang 3.4 or newer; clang 3.3 has issues with TSAN and pthread
     # symbol interception.
     if("${COMPILER_VERSION}" VERSION_LESS "3.4")
-      message(SEND_ERROR "Must use clang 3.4 or newer to run a sanitizer build."
-        " Try using clang from $NATIVE_TOOLCHAIN/")
+        message(SEND_ERROR "Must use clang 3.4 or newer to run a sanitizer build."
+        " Detected unsupported version ${COMPILER_VERSION}."
+        " Try using clang from $NATIVE_TOOLCHAIN/.")
     endif()
     add_definitions("-fsanitize-blacklist=${BUILD_SUPPORT_DIR}/sanitize-blacklist.txt")
   else()
diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index 2ded1e11f96f8..d76e991b54378 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -67,11 +67,14 @@ TEST_F(TestBuffer, Resize) {
 }
 
 TEST_F(TestBuffer, ResizeOOM) {
+  // This test doesn't play nice with AddressSanitizer
+#ifndef ADDRESS_SANITIZER
   // realloc fails, even though there may be no explicit limit
   PoolBuffer buf;
   ASSERT_OK(buf.Resize(100));
   int64_t to_alloc = std::numeric_limits<int64_t>::max();
   ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));
+#endif
 }
 
 TEST_F(TestBuffer, EqualsWithSameContent) {
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 3daf72755cff2..56bb32f0b5b27 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -32,7 +32,9 @@ TEST_F(TestDefaultMemoryPool, MemoryTracking) {
 }
 
 TEST_F(TestDefaultMemoryPool, OOM) {
+#ifndef ADDRESS_SANITIZER
   this->TestOOM();
+#endif
 }
 
 TEST_F(TestDefaultMemoryPool, Reallocate) {
@@ -41,7 +43,7 @@ TEST_F(TestDefaultMemoryPool, Reallocate) {
 
 // Death tests and valgrind are known to not play well 100% of the time. See
 // googletest documentation
-#ifndef ARROW_VALGRIND
+#if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER))
 
 TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
   MemoryPool* pool = default_memory_pool();
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 059158856f1f2..ac47108c84ae7 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -16,7 +16,7 @@ FROM quay.io/pypa/manylinux1_x86_64:latest
 RUN yum install -y flex openssl-devel
 
 WORKDIR /
-RUN wget http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
+RUN wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
 RUN tar xf boost_1_60_0.tar.gz
 WORKDIR /boost_1_60_0
 RUN ./bootstrap.sh

From 31f145dc5296d27cc8010a4cd17ca5b4ae461dff Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 9 Feb 2017 13:47:09 +0100
Subject: [PATCH 0322/1644] ARROW-545: [Python] Ignore non .parq/.parquet files
 when reading directories as Parquet datasets

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #331 from wesm/ARROW-545 and squashes the following commits:

5494167 [Wes McKinney] Docstring typo
92b274c [Wes McKinney] Ignore non .parq/.parquet files when reading directories-as-Parquet-datasets
---
 python/pyarrow/__init__.py           |  2 +-
 python/pyarrow/filesystem.py         | 23 +++++++++++++++++------
 python/pyarrow/parquet.py            | 18 ++++++++++++++++--
 python/pyarrow/tests/test_parquet.py |  4 ++++
 4 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index ea4710d4137de..6724b52e6004e 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -67,4 +67,4 @@
 from pyarrow.table import Column, RecordBatch, Table, concat_tables
 
 
-localfs = LocalFilesystem()
+localfs = LocalFilesystem.get_instance()
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 82409b7666ab1..55bcad044305e 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -62,7 +62,7 @@ def isfile(self, path):
         """
         raise NotImplementedError
 
-    def read_parquet(self, path, columns=None, schema=None):
+    def read_parquet(self, path, columns=None, metadata=None, schema=None):
         """
         Read Parquet data from path in file system. Can read from a single file
         or a directory of files
@@ -73,8 +73,11 @@ def read_parquet(self, path, columns=None, schema=None):
             Single file path or directory
         columns : List[str], optional
             Subset of columns to read
+        metadata : pyarrow.parquet.FileMetaData
+            Known metadata to validate files against
         schema : pyarrow.parquet.Schema
-            Known schema to validate files against
+            Known schema to validate files against. Alternative to metadata
+            argument
 
         Returns
         -------
@@ -85,18 +88,26 @@ def read_parquet(self, path, columns=None, schema=None):
         if self.isdir(path):
             paths_to_read = []
             for path in self.ls(path):
-                if path == '_metadata' or path == '_common_metadata':
-                    raise ValueError('No support yet for common metadata file')
-                paths_to_read.append(path)
+                if path.endswith('parq') or path.endswith('parquet'):
+                    paths_to_read.append(path)
         else:
             paths_to_read = [path]
 
         return read_multiple_files(paths_to_read, columns=columns,
-                                   filesystem=self, schema=schema)
+                                   filesystem=self, schema=schema,
+                                   metadata=metadata)
 
 
 class LocalFilesystem(Filesystem):
 
+    _instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = LocalFilesystem()
+        return cls._instance
+
     @implements(Filesystem.ls)
     def ls(self, path):
         return sorted(pjoin(path, x) for x in os.listdir(path))
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 6654b770ba33e..9766ff6dfa8e8 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -15,12 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import six
+
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
                               RowGroupMetaData, Schema, ParquetWriter)
 import pyarrow._parquet as _parquet  # noqa
 from pyarrow.table import concat_tables
 
 
+EXCLUDED_PARQUET_PATHS = {'_metadata', '_common_metadata', '_SUCCESS'}
+
+
 class ParquetFile(object):
     """
     Open a Parquet binary file for reading
@@ -82,8 +87,9 @@ def read_table(source, columns=None, nthreads=1, metadata=None):
     Parameters
     ----------
     source: str or pyarrow.io.NativeFile
-        Readable source. For passing Python file objects or byte buffers, see
-        pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
+        Location of Parquet dataset. If a string passed, can be a single file
+        name or directory name. For passing Python file objects or byte
+        buffers, see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
     columns: list
         If not None, only these columns will be read from the file.
     nthreads : int, default 1
@@ -97,6 +103,14 @@ def read_table(source, columns=None, nthreads=1, metadata=None):
     pyarrow.Table
         Content of the file as a table (of columns)
     """
+    from pyarrow.filesystem import LocalFilesystem
+
+    if isinstance(source, six.string_types):
+        fs = LocalFilesystem.get_instance()
+        if fs.isdir(source):
+            return fs.read_parquet(source, columns=columns,
+                                   metadata=metadata)
+
     pf = ParquetFile(source, metadata=metadata)
     return pf.read(columns=columns, nthreads=nthreads)
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 80a995fbb6662..969f68b47b497 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -393,6 +393,10 @@ def test_read_multiple_files(tmpdir):
         test_data.append(table)
         paths.append(path)
 
+    # Write a _SUCCESS.crc file
+    with open(pjoin(dirpath, '_SUCCESS.crc'), 'wb') as f:
+        f.write(b'0')
+
     result = pq.read_multiple_files(paths)
     expected = pa.concat_tables(test_data)
 

From dc6cefde46c65ce4753bec3fbafc44a20944f9c9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 9 Feb 2017 10:43:53 -0500
Subject: [PATCH 0323/1644] ARROW-521: [C++] Track peak allocations in default
 memory pool

This should enable us to remove the `parquet::MemoryAllocator` implementation in parquet-cpp

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #330 from wesm/ARROW-521 and squashes the following commits:

10531c4 [Wes McKinney] Move max_memory_ member to DefaultMemoryPool, add default virtual max_memory() to MemoryPool
a0d134d [Wes McKinney] Add max_memory() method to MemoryPool, leave implementation to subclasses
---
 cpp/src/arrow/array-primitive-test.cc      |  3 +-
 cpp/src/arrow/buffer-test.cc               |  2 +-
 cpp/src/arrow/ipc/json-integration-test.cc |  6 ++-
 cpp/src/arrow/memory_pool-test.cc          | 17 +++++++
 cpp/src/arrow/memory_pool.cc               | 54 +++++++++++-----------
 cpp/src/arrow/memory_pool.h                | 31 +++++++++++++
 cpp/src/arrow/table.cc                     |  6 ++-
 cpp/src/arrow/util/logging.h               |  4 +-
 cpp/src/arrow/util/macros.h                |  2 +-
 9 files changed, 89 insertions(+), 36 deletions(-)

diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
index a20fdbf8b9166..f8bbd774d483c 100644
--- a/cpp/src/arrow/array-primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -242,7 +242,8 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 }
 
 typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
-    PInt32, PInt64, PFloat, PDouble> Primitives;
+    PInt32, PInt64, PFloat, PDouble>
+    Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index d76e991b54378..934fcfef14856 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -67,7 +67,7 @@ TEST_F(TestBuffer, Resize) {
 }
 
 TEST_F(TestBuffer, ResizeOOM) {
-  // This test doesn't play nice with AddressSanitizer
+// This test doesn't play nice with AddressSanitizer
 #ifndef ADDRESS_SANITIZER
   // realloc fails, even though there may be no explicit limit
   PoolBuffer buf;
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 17ccc4ac1d0da..95bc742054fab 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -144,8 +144,10 @@ static Status ValidateArrowVsJson(
 
   if (!json_schema->Equals(arrow_schema)) {
     std::stringstream ss;
-    ss << "JSON schema: \n" << json_schema->ToString() << "\n"
-       << "Arrow schema: \n" << arrow_schema->ToString();
+    ss << "JSON schema: \n"
+       << json_schema->ToString() << "\n"
+       << "Arrow schema: \n"
+       << arrow_schema->ToString();
 
     if (FLAGS_verbose) { std::cout << ss.str() << std::endl; }
     return Status::Invalid("Schemas did not match");
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 56bb32f0b5b27..6ab73fb103f50 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -59,6 +59,23 @@ TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
   pool->Free(data, 100);
 }
 
+TEST(DefaultMemoryPoolDeathTest, MaxMemory) {
+  DefaultMemoryPool pool;
+
+  ASSERT_EQ(0, pool.max_memory());
+
+  uint8_t* data;
+  ASSERT_OK(pool.Allocate(100, &data));
+
+  uint8_t* data2;
+  ASSERT_OK(pool.Allocate(100, &data2));
+
+  pool.Free(data, 100);
+  pool.Free(data2, 100);
+
+  ASSERT_EQ(200, pool.max_memory());
+}
+
 #endif  // ARROW_VALGRIND
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index aea5e210f4980..8d85a089a65c9 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -60,36 +60,30 @@ Status AllocateAligned(int64_t size, uint8_t** out) {
 }
 }  // namespace
 
-MemoryPool::~MemoryPool() {}
-
-class InternalMemoryPool : public MemoryPool {
- public:
-  InternalMemoryPool() : bytes_allocated_(0) {}
-  virtual ~InternalMemoryPool();
-
-  Status Allocate(int64_t size, uint8_t** out) override;
-  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
+MemoryPool::MemoryPool() {}
 
-  void Free(uint8_t* buffer, int64_t size) override;
+MemoryPool::~MemoryPool() {}
 
-  int64_t bytes_allocated() const override;
+int64_t MemoryPool::max_memory() const {
+  return -1;
+}
 
- private:
-  mutable std::mutex pool_lock_;
-  int64_t bytes_allocated_;
-};
+DefaultMemoryPool::DefaultMemoryPool() : bytes_allocated_(0) {
+  max_memory_ = 0;
+}
 
-Status InternalMemoryPool::Allocate(int64_t size, uint8_t** out) {
-  std::lock_guard<std::mutex> guard(pool_lock_);
+Status DefaultMemoryPool::Allocate(int64_t size, uint8_t** out) {
   RETURN_NOT_OK(AllocateAligned(size, out));
   bytes_allocated_ += size;
 
+  {
+    std::lock_guard<std::mutex> guard(lock_);
+    if (bytes_allocated_ > max_memory_) { max_memory_ = bytes_allocated_.load(); }
+  }
   return Status::OK();
 }
 
-Status InternalMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
-  std::lock_guard<std::mutex> guard(pool_lock_);
-
+Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
   // Note: We cannot use realloc() here as it doesn't guarantee alignment.
 
   // Allocate new chunk
@@ -105,17 +99,19 @@ Status InternalMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_
   *ptr = out;
 
   bytes_allocated_ += new_size - old_size;
+  {
+    std::lock_guard<std::mutex> guard(lock_);
+    if (bytes_allocated_ > max_memory_) { max_memory_ = bytes_allocated_.load(); }
+  }
 
   return Status::OK();
 }
 
-int64_t InternalMemoryPool::bytes_allocated() const {
-  std::lock_guard<std::mutex> guard(pool_lock_);
-  return bytes_allocated_;
+int64_t DefaultMemoryPool::bytes_allocated() const {
+  return bytes_allocated_.load();
 }
 
-void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
-  std::lock_guard<std::mutex> guard(pool_lock_);
+void DefaultMemoryPool::Free(uint8_t* buffer, int64_t size) {
   DCHECK_GE(bytes_allocated_, size);
 #ifdef _MSC_VER
   _aligned_free(buffer);
@@ -125,10 +121,14 @@ void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
   bytes_allocated_ -= size;
 }
 
-InternalMemoryPool::~InternalMemoryPool() {}
+int64_t DefaultMemoryPool::max_memory() const {
+  return max_memory_.load();
+}
+
+DefaultMemoryPool::~DefaultMemoryPool() {}
 
 MemoryPool* default_memory_pool() {
-  static InternalMemoryPool default_memory_pool_;
+  static DefaultMemoryPool default_memory_pool_;
   return &default_memory_pool_;
 }
 
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 89477b6ddeab0..33d4c3e9aad52 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -18,7 +18,9 @@
 #ifndef ARROW_UTIL_MEMORY_POOL_H
 #define ARROW_UTIL_MEMORY_POOL_H
 
+#include <atomic>
 #include <cstdint>
+#include <mutex>
 
 #include "arrow/util/visibility.h"
 
@@ -56,6 +58,35 @@ class ARROW_EXPORT MemoryPool {
   /// The number of bytes that were allocated and not yet free'd through
   /// this allocator.
   virtual int64_t bytes_allocated() const = 0;
+
+  /// Return peak memory allocation in this memory pool
+  ///
+  /// \return Maximum bytes allocated. If not known (or not implemented),
+  /// returns -1
+  virtual int64_t max_memory() const;
+
+ protected:
+  MemoryPool();
+};
+
+class ARROW_EXPORT DefaultMemoryPool : public MemoryPool {
+ public:
+  DefaultMemoryPool();
+  virtual ~DefaultMemoryPool();
+
+  Status Allocate(int64_t size, uint8_t** out) override;
+  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
+
+  void Free(uint8_t* buffer, int64_t size) override;
+
+  int64_t bytes_allocated() const override;
+
+  int64_t max_memory() const override;
+
+ private:
+  mutable std::mutex lock_;
+  std::atomic<int64_t> bytes_allocated_;
+  std::atomic<int64_t> max_memory_;
 };
 
 ARROW_EXPORT MemoryPool* default_memory_pool();
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 9e31ba5af0ce3..a9e0909b8b741 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -106,7 +106,8 @@ Status Table::FromRecordBatches(const std::string& name,
     if (!batches[i]->schema()->Equals(schema)) {
       std::stringstream ss;
       ss << "Schema at index " << static_cast<int>(i) << " was different: \n"
-         << schema->ToString() << "\nvs\n" << batches[i]->schema()->ToString();
+         << schema->ToString() << "\nvs\n"
+         << batches[i]->schema()->ToString();
       return Status::Invalid(ss.str());
     }
   }
@@ -138,7 +139,8 @@ Status ConcatenateTables(const std::string& output_name,
     if (!tables[i]->schema()->Equals(schema)) {
       std::stringstream ss;
       ss << "Schema at index " << static_cast<int>(i) << " was different: \n"
-         << schema->ToString() << "\nvs\n" << tables[i]->schema()->ToString();
+         << schema->ToString() << "\nvs\n"
+         << tables[i]->schema()->ToString();
       return Status::Invalid(ss.str());
     }
   }
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 06ee8411e283c..b22f07dd6345f 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -118,9 +118,9 @@ class CerrLog {
 class FatalLog : public CerrLog {
  public:
   explicit FatalLog(int /* severity */)  // NOLINT
-      : CerrLog(ARROW_FATAL) {}          // NOLINT
+      : CerrLog(ARROW_FATAL){}           // NOLINT
 
-  [[noreturn]] ~FatalLog() {
+            [[noreturn]] ~FatalLog() {
     if (has_logged_) { std::cerr << std::endl; }
     std::exit(1);
   }
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index 81a9b0cff5687..c4a62a475b92f 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -25,6 +25,6 @@
   TypeName& operator=(const TypeName&) = delete
 #endif
 
-#define UNUSED(x) (void) x
+#define UNUSED(x) (void)x
 
 #endif  // ARROW_UTIL_MACROS_H

From 3add9181f98810bcfeae558bf44093d9ab89bc3f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 9 Feb 2017 10:45:35 -0500
Subject: [PATCH 0324/1644] ARROW-476: Add binary integration test fixture, add
 Java support

@julienledem could you review my Java changes? Thanks

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #326 from wesm/ARROW-476 and squashes the following commits:

a75228d [Wes McKinney] Use PoolBuffer instead of std::vector
e5a96a0 [Wes McKinney] Chain exceptions
b23b852 [Wes McKinney] Use hexadecimal for transporting binary data in JSON
1d4e850 [Wes McKinney] Compare byte[] with Arrays.equals
e5f13d5 [Wes McKinney] Add binary integration test fixture, add to JsonFileReader.java, but fails
---
 cpp/src/arrow/ipc/json-internal.cc            | 55 ++++++++++++++++++-
 integration/integration_test.py               | 55 ++++++++++++++++---
 java/vector/pom.xml                           |  5 ++
 .../vector/file/json/JsonFileReader.java      | 14 +++++
 .../vector/file/json/JsonFileWriter.java      |  6 ++
 .../apache/arrow/vector/util/Validator.java   |  4 ++
 6 files changed, 129 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 1a95b2ce470b2..b9f97dd2bbd15 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -17,7 +17,10 @@
 
 #include "arrow/ipc/json-internal.h"
 
+#include <algorithm>
 #include <cstdint>
+#include <cstdlib>
+#include <iostream>
 #include <memory>
 #include <sstream>
 #include <string>
@@ -40,6 +43,8 @@
 namespace arrow {
 namespace ipc {
 
+static const char* kAsciiTable = "0123456789ABCDEF";
+
 using RjArray = rj::Value::ConstArray;
 using RjObject = rj::Value::ConstObject;
 
@@ -395,14 +400,26 @@ class JsonArrayWriter : public ArrayVisitor {
     }
   }
 
-  // String (Utf8), Binary
+  // Binary, encode to hexadecimal. UTF8 string write as is
   template <typename T>
   typename std::enable_if<std::is_base_of<BinaryArray, T>::value, void>::type
   WriteDataValues(const T& arr) {
     for (int i = 0; i < arr.length(); ++i) {
       int32_t length;
       const char* buf = reinterpret_cast<const char*>(arr.GetValue(i, &length));
-      writer_->String(buf, length);
+
+      if (std::is_base_of<StringArray, T>::value) {
+        writer_->String(buf, length);
+      } else {
+        std::string hex_string;
+        hex_string.reserve(length * 2);
+        for (int32_t j = 0; j < length; ++j) {
+          // Convert to 2 base16 digits
+          hex_string.push_back(kAsciiTable[buf[j] >> 4]);
+          hex_string.push_back(kAsciiTable[buf[j] & 15]);
+        }
+        writer_->String(hex_string);
+      }
     }
   }
 
@@ -773,6 +790,20 @@ class JsonSchemaReader {
   const rj::Value& json_schema_;
 };
 
+static inline Status ParseHexValue(const char* data, uint8_t* out) {
+  char c1 = data[0];
+  char c2 = data[1];
+
+  const char* pos1 = std::lower_bound(kAsciiTable, kAsciiTable + 16, c1);
+  const char* pos2 = std::lower_bound(kAsciiTable, kAsciiTable + 16, c2);
+
+  // Error checking
+  if (*pos1 != c1 || *pos2 != c2) { return Status::Invalid("Encountered non-hex digit"); }
+
+  *out = (pos1 - kAsciiTable) << 4 | (pos2 - kAsciiTable);
+  return Status::OK();
+}
+
 class JsonArrayReader {
  public:
   explicit JsonArrayReader(MemoryPool* pool) : pool_(pool) {}
@@ -852,6 +883,8 @@ class JsonArrayReader {
     const auto& json_data_arr = json_data->value.GetArray();
 
     DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length);
+
+    auto byte_buffer = std::make_shared<PoolBuffer>(pool_);
     for (int i = 0; i < length; ++i) {
       if (!is_valid[i]) {
         builder.AppendNull();
@@ -860,7 +893,23 @@ class JsonArrayReader {
 
       const rj::Value& val = json_data_arr[i];
       DCHECK(val.IsString());
-      builder.Append(val.GetString());
+      if (std::is_base_of<StringType, T>::value) {
+        builder.Append(val.GetString());
+      } else {
+        std::string hex_string = val.GetString();
+
+        DCHECK(hex_string.size() % 2 == 0) << "Expected base16 hex string";
+        int64_t length = static_cast<int>(hex_string.size()) / 2;
+
+        if (byte_buffer->size() < length) { RETURN_NOT_OK(byte_buffer->Resize(length)); }
+
+        const char* hex_data = hex_string.c_str();
+        uint8_t* byte_buffer_data = byte_buffer->mutable_data();
+        for (int64_t j = 0; j < length; ++j) {
+          RETURN_NOT_OK(ParseHexValue(hex_data + j * 2, &byte_buffer_data[j]));
+        }
+        RETURN_NOT_OK(builder.Append(byte_buffer_data, length));
+      }
     }
 
     return builder.Finish(array);
diff --git a/integration/integration_test.py b/integration/integration_test.py
index a622bf228a651..1d8dc29a9f529 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -241,14 +241,18 @@ def generate_column(self, size):
         return PrimitiveColumn(self.name, size, is_valid, values)
 
 
-class StringType(PrimitiveType):
+class BinaryType(PrimitiveType):
 
     @property
     def numpy_type(self):
         return object
 
+    @property
+    def column_class(self):
+        return BinaryColumn
+
     def _get_type(self):
-        return OrderedDict([('name', 'utf8')])
+        return OrderedDict([('name', 'binary')])
 
     def _get_type_layout(self):
         return OrderedDict([
@@ -260,6 +264,32 @@ def _get_type_layout(self):
               OrderedDict([('type', 'DATA'),
                            ('typeBitWidth', 8)])])])
 
+    def generate_column(self, size):
+        K = 7
+        is_valid = self._make_is_valid(size)
+        values = []
+
+        for i in range(size):
+            if is_valid[i]:
+                draw = (np.random.randint(0, 255, size=K)
+                        .astype(np.uint8)
+                        .tostring())
+                values.append(draw)
+            else:
+                values.append("")
+
+        return self.column_class(self.name, size, is_valid, values)
+
+
+class StringType(BinaryType):
+
+    @property
+    def column_class(self):
+        return StringColumn
+
+    def _get_type(self):
+        return OrderedDict([('name', 'utf8')])
+
     def generate_column(self, size):
         K = 7
         is_valid = self._make_is_valid(size)
@@ -271,7 +301,7 @@ def generate_column(self, size):
             else:
                 values.append("")
 
-        return StringColumn(self.name, size, is_valid, values)
+        return self.column_class(self.name, size, is_valid, values)
 
 
 class JSONSchema(object):
@@ -285,7 +315,10 @@ def get_json(self):
         ])
 
 
-class StringColumn(PrimitiveColumn):
+class BinaryColumn(PrimitiveColumn):
+
+    def _encode_value(self, x):
+        return ''.join('{:02x}'.format(c).upper() for c in x)
 
     def _get_buffers(self):
         offset = 0
@@ -299,7 +332,7 @@ def _get_buffers(self):
                 v = ""
 
             offsets.append(offset)
-            data.append(v)
+            data.append(self._encode_value(v))
 
         return [
             ('VALIDITY', [int(x) for x in self.is_valid]),
@@ -308,6 +341,12 @@ def _get_buffers(self):
         ]
 
 
+class StringColumn(BinaryColumn):
+
+    def _encode_value(self, x):
+        return x
+
+
 class ListType(DataType):
 
     def __init__(self, name, value_type, nullable=True):
@@ -443,7 +482,9 @@ def write(self, path):
 
 
 def get_field(name, type_, nullable=True):
-    if type_ == 'utf8':
+    if type_ == 'binary':
+        return BinaryType(name, nullable=nullable)
+    elif type_ == 'utf8':
         return StringType(name, nullable=nullable)
 
     dtype = np.dtype(type_)
@@ -463,7 +504,7 @@ def get_field(name, type_, nullable=True):
 def generate_primitive_case():
     types = ['bool', 'int8', 'int16', 'int32', 'int64',
              'uint8', 'uint16', 'uint32', 'uint64',
-             'float32', 'float64', 'utf8']
+             'float32', 'float64', 'binary', 'utf8']
 
     fields = []
 
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 64b68bf8a1588..8517d4ced80f1 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -56,6 +56,11 @@
       <artifactId>commons-lang3</artifactId>
       <version>3.4</version>
     </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>1.10</version>
+    </dependency>
   </dependencies>
 
     <pluginRepositories>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 71fe88e156a5d..24fdc184523b3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -48,6 +48,7 @@
 import org.apache.arrow.vector.UInt8Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ValueVector.Mutator;
+import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.NullableMapVector;
@@ -60,6 +61,8 @@
 import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.databind.MappingJsonFactory;
 import com.google.common.base.Objects;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
 
 public class JsonFileReader implements AutoCloseable {
   private final File inputFile;
@@ -164,6 +167,14 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
     readToken(END_OBJECT);
   }
 
+  private byte[] decodeHexSafe(String hexString) throws IOException {
+    try {
+      return Hex.decodeHex(hexString.toCharArray());
+    } catch (DecoderException e) {
+      throw new IOException("Unable to decode hex string: " + hexString, e);
+    }
+  }
+
   private void setValueFromParser(ValueVector valueVector, int i) throws IOException {
     switch (valueVector.getMinorType()) {
     case BIT:
@@ -199,6 +210,9 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
     case FLOAT8:
       ((Float8Vector)valueVector).getMutator().set(i, parser.readValueAs(Double.class));
       break;
+    case VARBINARY:
+      ((VarBinaryVector)valueVector).getMutator().setSafe(i, decodeHexSafe(parser.readValueAs(String.class)));
+      break;
     case VARCHAR:
       ((VarCharVector)valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8));
       break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index ddc80433cb6db..99040b67e1cd3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -30,6 +30,7 @@
 import org.apache.arrow.vector.TimeStampNanoVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ValueVector.Accessor;
+import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -40,6 +41,7 @@
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
 import com.fasterxml.jackson.databind.MappingJsonFactory;
+import org.apache.commons.codec.binary.Hex;
 
 public class JsonFileWriter implements AutoCloseable {
 
@@ -157,6 +159,10 @@ private void writeValueToGenerator(ValueVector valueVector, int i) throws IOExce
       case BIT:
         generator.writeNumber(((BitVector)valueVector).getAccessor().get(i));
         break;
+      case VARBINARY:
+        String hexString = Hex.encodeHexString(((VarBinaryVector) valueVector).getAccessor().get(i));
+        generator.writeObject(hexString);
+        break;
       default:
         // TODO: each type
         Accessor accessor = valueVector.getAccessor();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
index a97458254151d..f294e20b029c5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
@@ -17,6 +17,7 @@
  */
 package org.apache.arrow.vector.util;
 
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.arrow.vector.FieldVector;
@@ -89,7 +90,10 @@ static boolean equals(ArrowType type, final Object o1, final Object o2) {
                 default:
                     throw new UnsupportedOperationException("unsupported precision: " + fpType);
             }
+        } else if (type instanceof ArrowType.Binary) {
+            return Arrays.equals((byte[]) o1, (byte[]) o2);
         }
+
         return Objects.equal(o1, o2);
     }
 

From 0ab4252453be025f13df2a825e67fbfbbb608ab9 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 9 Feb 2017 16:19:38 -0500
Subject: [PATCH 0325/1644] ARROW-546: Python: Account for changes in
 PARQUET-867

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #332 from xhochy/ARROW-546 and squashes the following commits:

ca019c5 [Uwe L. Korn] ARROW-546: Python: Account for changes in PARQUET-867
---
 python/pyarrow/_parquet.pxd | 2 +-
 python/pyarrow/_parquet.pyx | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 005be91bdb97f..e106252189f42 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -229,7 +229,7 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
 
 cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
     cdef CStatus WriteTable(
-        const CTable* table, CMemoryPool* pool,
+        const CTable& table, CMemoryPool* pool,
         const shared_ptr[OutputStream]& sink,
         int64_t chunk_size,
         const shared_ptr[WriterProperties]& properties)
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 796c436ec46f4..08c7bb5d8b1bc 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -543,6 +543,6 @@ cdef class ParquetWriter:
 
         cdef int c_row_group_size = row_group_size
         with nogil:
-            check_status(WriteTable(ctable, self.allocator,
-                                        self.sink, c_row_group_size,
-                                        self.properties))
+            check_status(WriteTable(deref(ctable), self.allocator,
+                                    self.sink, c_row_group_size,
+                                    self.properties))

From 42b55d98c151d87e5a7c6442800f3e5b9316499b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 10 Feb 2017 13:15:39 -0500
Subject: [PATCH 0326/1644] ARROW-544: [C++] Test writing zero-length record
 batches, zero-length BinaryArray fixes

I believe this should fix the failure reported in the Spark integration work. We'll need to upgrade the conda test packages to verify. cc @BryanCutler

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #333 from wesm/ARROW-544 and squashes the following commits:

f80d58f [Wes McKinney] Protect zero-length record batches from incomplete buffer metadata
f876dce [Wes McKinney] Test with null value_offsets too
1dc7733 [Wes McKinney] Test writing zero-length record batches, misc zero-length fixes
---
 cpp/src/arrow/array-string-test.cc    |  4 ++
 cpp/src/arrow/array.cc                | 11 +++-
 cpp/src/arrow/ipc/adapter.cc          | 27 +++++---
 cpp/src/arrow/ipc/ipc-adapter-test.cc | 89 +++++++++++++++++++--------
 4 files changed, 94 insertions(+), 37 deletions(-)

diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index c4d9bf40f57f9..d8a35851c1238 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -470,4 +470,8 @@ TEST_F(TestStringArray, TestSliceEquality) {
   CheckSliceEquality<BinaryType>();
 }
 
+TEST_F(TestBinaryArray, LengthZeroCtor) {
+  BinaryArray array(0, nullptr, nullptr);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index bf368d91226be..81678e354a608 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -270,9 +270,12 @@ BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int32_t length,
     const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
     : Array(type, length, null_bitmap, null_count, offset),
       value_offsets_(value_offsets),
-      raw_value_offsets_(reinterpret_cast<const int32_t*>(value_offsets_->data())),
+      raw_value_offsets_(nullptr),
       data_(data),
       raw_data_(nullptr) {
+  if (value_offsets_ != nullptr) {
+    raw_value_offsets_ = reinterpret_cast<const int32_t*>(value_offsets_->data());
+  }
   if (data_ != nullptr) { raw_data_ = data_->data(); }
 }
 
@@ -384,8 +387,10 @@ UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int32_t length,
     : Array(type, length, null_bitmap, null_count, offset),
       children_(children),
       type_ids_(type_ids),
-      value_offsets_(value_offsets) {
-  raw_type_ids_ = reinterpret_cast<const uint8_t*>(type_ids->data());
+      raw_type_ids_(nullptr),
+      value_offsets_(value_offsets),
+      raw_value_offsets_(nullptr) {
+  if (type_ids) { raw_type_ids_ = reinterpret_cast<const uint8_t*>(type_ids->data()); }
   if (value_offsets) {
     raw_value_offsets_ = reinterpret_cast<const int32_t*>(value_offsets->data());
   }
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 3613ccbadbbab..f36ff37db15a3 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -602,8 +602,13 @@ class ArrayLoader : public TypeVisitor {
 
     std::shared_ptr<Buffer> offsets;
     std::shared_ptr<Buffer> values;
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
+    if (field_meta.length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
+    } else {
+      context_->buffer_index += 2;
+      offsets = values = nullptr;
+    }
 
     result_ = std::make_shared<CONTAINER>(
         field_meta.length, offsets, values, null_bitmap, field_meta.null_count);
@@ -661,7 +666,12 @@ class ArrayLoader : public TypeVisitor {
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
 
     std::shared_ptr<Buffer> offsets;
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
+    if (field_meta.length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &offsets));
+    } else {
+      offsets = nullptr;
+    }
+    ++context_->buffer_index;
 
     const int num_children = type.num_children();
     if (num_children != 1) {
@@ -708,13 +718,16 @@ class ArrayLoader : public TypeVisitor {
     std::shared_ptr<Buffer> null_bitmap;
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
 
-    std::shared_ptr<Buffer> type_ids;
+    std::shared_ptr<Buffer> type_ids = nullptr;
     std::shared_ptr<Buffer> offsets = nullptr;
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &type_ids));
 
-    if (type.mode == UnionMode::DENSE) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
+    if (field_meta.length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &type_ids));
+      if (type.mode == UnionMode::DENSE) {
+        RETURN_NOT_OK(GetBuffer(context_->buffer_index + 1, &offsets));
+      }
     }
+    context_->buffer_index += type.mode == UnionMode::DENSE? 2 : 1;
 
     std::vector<std::shared_ptr<Array>> fields;
     RETURN_NOT_OK(LoadChildren(type.children(), &fields));
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index bae6578f110f2..d11b95b167d21 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -71,6 +71,42 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
     return ReadRecordBatch(metadata, batch.schema(), &buffer_reader, batch_result);
   }
 
+  void CheckRoundtrip(const RecordBatch& batch, int64_t buffer_size) {
+    std::shared_ptr<RecordBatch> batch_result;
+
+    ASSERT_OK(RoundTripHelper(batch, 1 << 16, &batch_result));
+    EXPECT_EQ(batch.num_rows(), batch_result->num_rows());
+
+    ASSERT_TRUE(batch.schema()->Equals(batch_result->schema()));
+    ASSERT_EQ(batch.num_columns(), batch_result->num_columns())
+        << batch.schema()->ToString()
+        << " result: " << batch_result->schema()->ToString();
+
+    for (int i = 0; i < batch.num_columns(); ++i) {
+      const auto& left = *batch.column(i);
+      const auto& right = *batch_result->column(i);
+      if (!left.Equals(right)) {
+        std::stringstream pp_result;
+        std::stringstream pp_expected;
+
+        ASSERT_OK(PrettyPrint(left, 0, &pp_expected));
+        ASSERT_OK(PrettyPrint(right, 0, &pp_result));
+
+        FAIL() << "Index: " << i << " Expected: " << pp_expected.str()
+               << "\nGot: " << pp_result.str();
+      }
+    }
+  }
+
+  void CheckRoundtrip(const std::shared_ptr<Array>& array, int64_t buffer_size) {
+    auto f0 = arrow::field("f0", array->type());
+    std::vector<std::shared_ptr<Field>> fields = {f0};
+    auto schema = std::make_shared<Schema>(fields);
+
+    RecordBatch batch(schema, 0, {array});
+    CheckRoundtrip(batch, buffer_size);
+  }
+
  protected:
   std::shared_ptr<io::MemoryMappedFile> mmap_;
   MemoryPool* pool_;
@@ -79,48 +115,47 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
 TEST_P(TestWriteRecordBatch, RoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
-  std::shared_ptr<RecordBatch> batch_result;
-  ASSERT_OK(RoundTripHelper(*batch, 1 << 16, &batch_result));
-
-  // do checks
-  ASSERT_TRUE(batch->schema()->Equals(batch_result->schema()));
-  ASSERT_EQ(batch->num_columns(), batch_result->num_columns())
-      << batch->schema()->ToString() << " result: " << batch_result->schema()->ToString();
-  EXPECT_EQ(batch->num_rows(), batch_result->num_rows());
-  for (int i = 0; i < batch->num_columns(); ++i) {
-    EXPECT_TRUE(batch->column(i)->Equals(batch_result->column(i)))
-        << "Idx: " << i << " Name: " << batch->column_name(i);
-  }
+
+  CheckRoundtrip(*batch, 1 << 20);
 }
 
 TEST_P(TestWriteRecordBatch, SliceRoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
-  std::shared_ptr<RecordBatch> batch_result;
 
   // Skip the zero-length case
   if (batch->num_rows() < 2) { return; }
 
   auto sliced_batch = batch->Slice(2, 10);
+  CheckRoundtrip(*sliced_batch, 1 << 20);
+}
+
+TEST_P(TestWriteRecordBatch, ZeroLengthArrays) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
 
-  ASSERT_OK(RoundTripHelper(*sliced_batch, 1 << 16, &batch_result));
+  std::shared_ptr<RecordBatch> zero_length_batch;
+  if (batch->num_rows() > 2) {
+    zero_length_batch = batch->Slice(2, 0);
+  } else {
+    zero_length_batch = batch->Slice(0, 0);
+  }
 
-  EXPECT_EQ(sliced_batch->num_rows(), batch_result->num_rows());
+  CheckRoundtrip(*zero_length_batch, 1 << 20);
 
-  for (int i = 0; i < sliced_batch->num_columns(); ++i) {
-    const auto& left = *sliced_batch->column(i);
-    const auto& right = *batch_result->column(i);
-    if (!left.Equals(right)) {
-      std::stringstream pp_result;
-      std::stringstream pp_expected;
+  // ARROW-544: check binary array
+  std::shared_ptr<MutableBuffer> value_offsets;
+  ASSERT_OK(AllocateBuffer(pool_, sizeof(int32_t), &value_offsets));
+  *reinterpret_cast<int32_t*>(value_offsets->mutable_data()) = 0;
 
-      ASSERT_OK(PrettyPrint(left, 0, &pp_expected));
-      ASSERT_OK(PrettyPrint(right, 0, &pp_result));
+  std::shared_ptr<Array> bin_array = std::make_shared<BinaryArray>(0, value_offsets,
+      std::make_shared<Buffer>(nullptr, 0), std::make_shared<Buffer>(nullptr, 0));
 
-      FAIL() << "Index: " << i << " Expected: " << pp_expected.str()
-             << "\nGot: " << pp_result.str();
-    }
-  }
+  // null value_offsets
+  std::shared_ptr<Array> bin_array2 = std::make_shared<BinaryArray>(0, nullptr, nullptr);
+
+  CheckRoundtrip(bin_array, 1 << 20);
+  CheckRoundtrip(bin_array2, 1 << 20);
 }
 
 INSTANTIATE_TEST_CASE_P(

From e4845c447cbef12fa7543f372fbc744fa833fee1 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 12 Feb 2017 07:53:40 -0500
Subject: [PATCH 0327/1644] ARROW-551: C++: Construction of Column with nullptr
 Array segfaults

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #335 from xhochy/ARROW-551 and squashes the following commits:

440d4a9 [Uwe L. Korn] ARROW-551: C++: Construction of Column with nullptr Array segfaults
---
 cpp/src/arrow/column-test.cc | 4 ++++
 cpp/src/arrow/column.cc      | 6 +++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
index 0bbfc831f5cb9..24d58c80b9fae 100644
--- a/cpp/src/arrow/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -135,6 +135,10 @@ TEST_F(TestColumn, BasicAPI) {
   ASSERT_EQ(300, column_->length());
   ASSERT_EQ(30, column_->null_count());
   ASSERT_EQ(3, column_->data()->num_chunks());
+
+  // nullptr array should not break
+  column_.reset(new Column(field, std::shared_ptr<Array>(nullptr)));
+  ASSERT_NE(column_.get(), nullptr);
 }
 
 TEST_F(TestColumn, ChunksInhomogeneous) {
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index 9cc0f579dc5bd..1376f6534ece1 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -90,7 +90,11 @@ Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks)
 
 Column::Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data)
     : field_(field) {
-  data_ = std::make_shared<ChunkedArray>(ArrayVector({data}));
+  if (data) {
+    data_ = std::make_shared<ChunkedArray>(ArrayVector({data}));
+  } else {
+    data_ = std::make_shared<ChunkedArray>(ArrayVector({}));
+  }
 }
 
 Column::Column(

From 1f26040f55eb54e00dc5e67ce0c1df64e51a1567 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 13 Feb 2017 09:52:59 +0100
Subject: [PATCH 0328/1644] ARROW-548: [Python] Add nthreads to
 Filesystem.read_parquet and pass through

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #337 from wesm/ARROW-548 and squashes the following commits:

b9aeaeb [Wes McKinney] Add nthreads to Filesystem.read_parquet and pass through
---
 python/pyarrow/filesystem.py         | 9 +++++++--
 python/pyarrow/parquet.py            | 4 ++--
 python/pyarrow/tests/test_parquet.py | 8 +++++++-
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 55bcad044305e..e820806ab4e68 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -62,7 +62,8 @@ def isfile(self, path):
         """
         raise NotImplementedError
 
-    def read_parquet(self, path, columns=None, metadata=None, schema=None):
+    def read_parquet(self, path, columns=None, metadata=None, schema=None,
+                     nthreads=1):
         """
         Read Parquet data from path in file system. Can read from a single file
         or a directory of files
@@ -78,6 +79,9 @@ def read_parquet(self, path, columns=None, metadata=None, schema=None):
         schema : pyarrow.parquet.Schema
             Known schema to validate files against. Alternative to metadata
             argument
+        nthreads : int, default 1
+            Number of columns to read in parallel. If > 1, requires that the
+            underlying file source is threadsafe
 
         Returns
         -------
@@ -95,7 +99,8 @@ def read_parquet(self, path, columns=None, metadata=None, schema=None):
 
         return read_multiple_files(paths_to_read, columns=columns,
                                    filesystem=self, schema=schema,
-                                   metadata=metadata)
+                                   metadata=metadata,
+                                   nthreads=nthreads)
 
 
 class LocalFilesystem(Filesystem):
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 9766ff6dfa8e8..fa96f95698013 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -59,8 +59,8 @@ def read(self, nrows=None, columns=None, nthreads=1):
         columns: list
             If not None, only these columns will be read from the file.
         nthreads : int, default 1
-            Number of columns to read in parallel. Requires that the underlying
-            file source is threadsafe
+            Number of columns to read in parallel. If > 1, requires that the
+            underlying file source is threadsafe
 
         Returns
         -------
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 969f68b47b497..96f2d15e312f2 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -320,17 +320,20 @@ def test_compare_schemas():
     assert fileh.schema[0].equals(fileh.schema[0])
     assert not fileh.schema[0].equals(fileh.schema[1])
 
+
 @parquet
 def test_column_of_lists(tmpdir):
     df, schema = dataframe_with_arrays()
 
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True, schema=schema)
+    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True,
+                                       schema=schema)
     pq.write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
     pdt.assert_frame_equal(df, df_read)
 
+
 @parquet
 def test_multithreaded_read():
     df = alltypes_sample(size=10000)
@@ -418,6 +421,9 @@ def test_read_multiple_files(tmpdir):
     expected = pa.Table.from_arrays(to_read)
     assert result.equals(expected)
 
+    # Read with multiple threads
+    pa.localfs.read_parquet(dirpath, nthreads=2)
+
     # Test failure modes with non-uniform metadata
     bad_apple = _test_dataframe(size, seed=i).iloc[:, :4]
     bad_apple_path = tmpdir.join('{0}.parquet'.format(guid())).strpath

From ad0157547a4f5e6e51fa2f712c2ed9477489a20c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 13 Feb 2017 09:03:46 -0500
Subject: [PATCH 0329/1644] ARROW-553: C++: Faster valid bitmap building

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #338 from xhochy/ARROW-553 and squashes the following commits:

1c1ee3d [Uwe L. Korn] ARROW-553: C++: Faster valid bitmap building
---
 cpp/src/arrow/builder.cc                      | 41 +++++++++++++++++--
 .../jemalloc/jemalloc-builder-benchmark.cc    |  2 +-
 2 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index dddadeee0dacf..f5c13f9e77ef1 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -114,18 +114,51 @@ void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int32_t leng
     UnsafeSetNotNull(length);
     return;
   }
+
+  int byte_offset = length_ / 8;
+  int bit_offset = length_ % 8;
+  uint8_t bitset = null_bitmap_data_[byte_offset];
+
   for (int32_t i = 0; i < length; ++i) {
-    // TODO(emkornfield) Optimize for large values of length?
-    UnsafeAppendToBitmap(valid_bytes[i] > 0);
+    if (valid_bytes[i]) {
+      bitset |= (1 << bit_offset);
+    } else {
+      bitset &= ~(1 << bit_offset);
+      ++null_count_;
+    }
+
+    bit_offset++;
+    if (bit_offset == 8) {
+      bit_offset = 0;
+      null_bitmap_data_[byte_offset] = bitset;
+      byte_offset++;
+      // TODO: Except for the last byte, this shouldn't be needed
+      bitset = null_bitmap_data_[byte_offset];
+    }
   }
+  if (bit_offset != 0) { null_bitmap_data_[byte_offset] = bitset; }
+  length_ += length;
 }
 
 void ArrayBuilder::UnsafeSetNotNull(int32_t length) {
   const int32_t new_length = length + length_;
-  // TODO(emkornfield) Optimize for large values of length?
-  for (int32_t i = length_; i < new_length; ++i) {
+
+  // Fill up the bytes until we have a byte alignment
+  int32_t pad_to_byte = 8 - (length_ % 8);
+  if (pad_to_byte == 8) { pad_to_byte = 0; }
+  for (int32_t i = 0; i < pad_to_byte; ++i) {
+    BitUtil::SetBit(null_bitmap_data_, i);
+  }
+
+  // Fast bitsetting
+  int32_t fast_length = (length - pad_to_byte) / 8;
+  memset(null_bitmap_data_ + ((length_ + pad_to_byte) / 8), 255, fast_length);
+
+  // Trailing bytes
+  for (int32_t i = length_ + pad_to_byte + (fast_length * 8); i < new_length; ++i) {
     BitUtil::SetBit(null_bitmap_data_, i);
   }
+
   length_ = new_length;
 }
 
diff --git a/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc b/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
index 58dbaa33a1a0f..d69c3047587bf 100644
--- a/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
+++ b/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
@@ -42,6 +42,6 @@ static void BM_BuildPrimitiveArrayNoNulls(
       state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
 }
 
-BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
+BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(5)->Unit(benchmark::kMillisecond);
 
 }  // namespace arrow

From 66f650cd359e13f3d5c3d4ef78d89f389d6bcecc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 13 Feb 2017 09:04:37 -0500
Subject: [PATCH 0330/1644] ARROW-547: [Python] Add zero-copy slice methods to
 Array, RecordBatch

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #336 from wesm/ARROW-547 and squashes the following commits:

42037c2 [Wes McKinney] cpplint
2b91b5b [Wes McKinney] Tweak docstring
5f80d80 [Wes McKinney] Add slice methods to pyarrow.Array and RecordBatch. Fix bug in RecordBatch::Slice
20dc23f [Wes McKinney] Draft Array.slice implementation
---
 cpp/src/arrow/ipc/adapter.cc         |  2 +-
 cpp/src/arrow/table-test.cc          |  2 ++
 cpp/src/arrow/table.cc               |  5 +++-
 python/pyarrow/array.pxd             |  2 +-
 python/pyarrow/array.pyx             | 42 ++++++++++++++++++++++------
 python/pyarrow/includes/libarrow.pxd |  6 ++++
 python/pyarrow/scalar.pxd            |  6 ++--
 python/pyarrow/scalar.pyx            |  7 ++---
 python/pyarrow/table.pyx             | 37 ++++++++++++++++++++----
 python/pyarrow/tests/test_array.py   | 36 ++++++++++++++++++++++++
 python/pyarrow/tests/test_table.py   | 32 +++++++++++++++++++++
 11 files changed, 153 insertions(+), 24 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index f36ff37db15a3..a24c007a4056e 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -727,7 +727,7 @@ class ArrayLoader : public TypeVisitor {
         RETURN_NOT_OK(GetBuffer(context_->buffer_index + 1, &offsets));
       }
     }
-    context_->buffer_index += type.mode == UnionMode::DENSE? 2 : 1;
+    context_->buffer_index += type.mode == UnionMode::DENSE ? 2 : 1;
 
     std::vector<std::shared_ptr<Array>> fields;
     RETURN_NOT_OK(LoadChildren(type.children(), &fields));
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index e7c5d667903e8..25f12c4b4300d 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -259,6 +259,8 @@ TEST_F(TestRecordBatch, Slice) {
   auto batch_slice = batch.Slice(2);
   auto batch_slice2 = batch.Slice(1, 5);
 
+  ASSERT_EQ(batch_slice->num_rows(), batch.num_rows() - 2);
+
   for (int i = 0; i < batch.num_columns(); ++i) {
     ASSERT_EQ(2, batch_slice->column(i)->offset());
     ASSERT_EQ(length - 2, batch_slice->column(i)->length());
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index a9e0909b8b741..8ac06b8cb7811 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/table.h"
 
+#include <algorithm>
 #include <cstdlib>
 #include <memory>
 #include <sstream>
@@ -70,7 +71,9 @@ std::shared_ptr<RecordBatch> RecordBatch::Slice(int32_t offset, int32_t length)
   for (const auto& field : columns_) {
     arrays.emplace_back(field->Slice(offset, length));
   }
-  return std::make_shared<RecordBatch>(schema_, num_rows_, arrays);
+
+  int32_t num_rows = std::min(num_rows_ - offset, length);
+  return std::make_shared<RecordBatch>(schema_, num_rows, arrays);
 }
 
 // ----------------------------------------------------------------------
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index af105354ac2f3..9e4d469bcfa5f 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -38,7 +38,7 @@ cdef class Array:
     cdef init(self, const shared_ptr[CArray]& sp_array)
     cdef getitem(self, int i)
 
-cdef object box_arrow_array(const shared_ptr[CArray]& sp_array)
+cdef object box_array(const shared_ptr[CArray]& sp_array)
 
 
 cdef class BooleanArray(Array):
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 9b34f5607b31d..11abf03e35f1d 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -131,7 +131,7 @@ cdef class Array:
                 check_status(pyarrow.PandasToArrow(
                     pool, series_values, mask, c_field, &out))
 
-            return box_arrow_array(out)
+            return box_array(out)
 
     @staticmethod
     def from_list(object list_obj, DataType type=None, MemoryPool memory_pool=None):
@@ -156,7 +156,7 @@ cdef class Array:
         else:
             raise NotImplementedError()
 
-        return box_arrow_array(sp_array)
+        return box_array(sp_array)
 
     property null_count:
 
@@ -201,9 +201,9 @@ cdef class Array:
 
             step = key.step or 1
             if step != 1:
-                raise NotImplementedError
+                raise IndexError('only slices with step 1 supported')
             else:
-                return self.slice(start, stop)
+                return self.slice(start, stop - start)
 
         while key < 0:
             key += len(self)
@@ -211,10 +211,36 @@ cdef class Array:
         return self.getitem(key)
 
     cdef getitem(self, int i):
-        return scalar.box_arrow_scalar(self.type, self.sp_array, i)
+        return scalar.box_scalar(self.type, self.sp_array, i)
 
-    def slice(self, start, end):
-        pass
+    def slice(self, offset=0, length=None):
+        """
+        Compute zero-copy slice of this array
+
+        Parameters
+        ----------
+        offset : int, default 0
+            Offset from start of array to slice
+        length : int, default None
+            Length of slice (default is until end of Array starting from
+            offset)
+
+        Returns
+        -------
+        sliced : RecordBatch
+        """
+        cdef:
+            shared_ptr[CArray] result
+
+        if offset < 0:
+            raise IndexError('Offset must be non-negative')
+
+        if length is None:
+            result = self.ap.Slice(offset)
+        else:
+            result = self.ap.Slice(offset, length)
+
+        return box_array(result)
 
     def to_pandas(self):
         """
@@ -390,7 +416,7 @@ cdef dict _array_classes = {
     Type_DICTIONARY: DictionaryArray
 }
 
-cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
+cdef object box_array(const shared_ptr[CArray]& sp_array):
     if sp_array.get() == NULL:
         raise ValueError('Array was NULL')
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index ebfdc410fa004..702acfbc12e17 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -71,6 +71,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool Equals(const shared_ptr[CArray]& arr)
         c_bool IsNull(int i)
 
+        shared_ptr[CArray] Slice(int32_t offset)
+        shared_ptr[CArray] Slice(int32_t offset, int32_t length)
+
     cdef cppclass CFixedWidthType" arrow::FixedWidthType"(CDataType):
         int bit_width()
 
@@ -228,6 +231,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int num_columns()
         int32_t num_rows()
 
+        shared_ptr[CRecordBatch] Slice(int32_t offset)
+        shared_ptr[CRecordBatch] Slice(int32_t offset, int32_t length)
+
     cdef cppclass CTable" arrow::Table":
         CTable(const c_string& name, const shared_ptr[CSchema]& schema,
                const vector[shared_ptr[CColumn]]& columns)
diff --git a/python/pyarrow/scalar.pxd b/python/pyarrow/scalar.pxd
index b06845718649b..2d55757726183 100644
--- a/python/pyarrow/scalar.pxd
+++ b/python/pyarrow/scalar.pxd
@@ -61,6 +61,6 @@ cdef class ListValue(ArrayValue):
 cdef class StringValue(ArrayValue):
     pass
 
-cdef object box_arrow_scalar(DataType type,
-                             const shared_ptr[CArray]& sp_array,
-                             int index)
+cdef object box_scalar(DataType type,
+                       const shared_ptr[CArray]& sp_array,
+                       int index)
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 9d2b2b11a80d6..57a15ad78344c 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -203,7 +203,7 @@ cdef class ListValue(ArrayValue):
 
     cdef getitem(self, int i):
         cdef int j = self.ap.value_offset(self.index) + i
-        return box_arrow_scalar(self.value_type, self.ap.values(), j)
+        return box_scalar(self.value_type, self.ap.values(), j)
 
     def as_py(self):
         cdef:
@@ -235,9 +235,8 @@ cdef dict _scalar_classes = {
     Type_STRING: StringValue,
 }
 
-cdef object box_arrow_scalar(DataType type,
-                             const shared_ptr[CArray]& sp_array,
-                             int index):
+cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
+                       int index):
     cdef ArrayValue val
     if type.type.type == Type_NA:
         return NA
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 17072108f301f..7d7336246ee79 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -27,7 +27,7 @@ cimport pyarrow.includes.pyarrow as pyarrow
 
 import pyarrow.config
 
-from pyarrow.array cimport Array, box_arrow_array, wrap_array_output
+from pyarrow.array cimport Array, box_array, wrap_array_output
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
 from pyarrow.schema cimport box_data_type, box_schema, Field
@@ -109,8 +109,7 @@ cdef class ChunkedArray:
         pyarrow.array.Array
         """
         self._check_nullptr()
-        return box_arrow_array(self.chunked_array.chunk(i))
-
+        return box_array(self.chunked_array.chunk(i))
 
     def iterchunks(self):
         for i in range(self.num_chunks):
@@ -387,9 +386,35 @@ cdef class RecordBatch:
         return self._schema
 
     def __getitem__(self, i):
-        cdef Array arr = Array()
-        arr.init(self.batch.column(i))
-        return arr
+        return box_array(self.batch.column(i))
+
+    def slice(self, offset=0, length=None):
+        """
+        Compute zero-copy slice of this RecordBatch
+
+        Parameters
+        ----------
+        offset : int, default 0
+            Offset from start of array to slice
+        length : int, default None
+            Length of slice (default is until end of batch starting from
+            offset)
+
+        Returns
+        -------
+        sliced : RecordBatch
+        """
+        cdef shared_ptr[CRecordBatch] result
+
+        if offset < 0:
+            raise IndexError('Offset must be non-negative')
+
+        if length is None:
+            result = self.batch.Slice(offset)
+        else:
+            result = self.batch.Slice(offset, length)
+
+        return batch_from_cbatch(result)
 
     def equals(self, RecordBatch other):
         cdef:
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index ead17dbec4e35..d8b2e2f5d80d6 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -17,6 +17,8 @@
 
 import sys
 
+import pytest
+
 import pyarrow
 import pyarrow.formatting as fmt
 
@@ -100,3 +102,37 @@ def test_to_pandas_zero_copy():
         base_refcount = sys.getrefcount(np_arr.base)
         assert base_refcount == 2
         np_arr.sum()
+
+
+def test_array_slice():
+    arr = pyarrow.from_pylist(range(10))
+
+    sliced = arr.slice(2)
+    expected = pyarrow.from_pylist(range(2, 10))
+    assert sliced.equals(expected)
+
+    sliced2 = arr.slice(2, 4)
+    expected2 = pyarrow.from_pylist(range(2, 6))
+    assert sliced2.equals(expected2)
+
+    # 0 offset
+    assert arr.slice(0).equals(arr)
+
+    # Slice past end of array
+    assert len(arr.slice(len(arr))) == 0
+
+    with pytest.raises(IndexError):
+        arr.slice(-1)
+
+    # Test slice notation
+    assert arr[2:].equals(arr.slice(2))
+
+    assert arr[2:5].equals(arr.slice(2, 3))
+
+    assert arr[-5:].equals(arr.slice(len(arr) - 5))
+
+    with pytest.raises(IndexError):
+        arr[::-1]
+
+    with pytest.raises(IndexError):
+        arr[::2]
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index d49b33c9f42d6..67f1892a9987b 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -68,6 +68,38 @@ def test_recordbatch_basics():
     ])
 
 
+def test_recordbatch_slice():
+    data = [
+        pa.from_pylist(range(5)),
+        pa.from_pylist([-10, -5, 0, 5, 10])
+    ]
+    names = ['c0', 'c1']
+
+    batch = pa.RecordBatch.from_arrays(data, names)
+
+    sliced = batch.slice(2)
+
+    assert sliced.num_rows == 3
+
+    expected = pa.RecordBatch.from_arrays(
+        [x.slice(2) for x in data], names)
+    assert sliced.equals(expected)
+
+    sliced2 = batch.slice(2, 2)
+    expected2 = pa.RecordBatch.from_arrays(
+        [x.slice(2, 2) for x in data], names)
+    assert sliced2.equals(expected2)
+
+    # 0 offset
+    assert batch.slice(0).equals(batch)
+
+    # Slice past end of array
+    assert len(batch.slice(len(batch))) == 0
+
+    with pytest.raises(IndexError):
+        batch.slice(-1)
+
+
 def test_recordbatch_from_to_pandas():
     data = pd.DataFrame({
         'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),

From 69cf69238492e3872e729775bd833aa23e36bdc8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 13 Feb 2017 21:27:24 -0500
Subject: [PATCH 0331/1644] ARROW-556: [Integration] Configure C++ integration
 test executable with a single environment variable. Update README

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #340 from wesm/ARROW-556 and squashes the following commits:

521af12 [Wes McKinney] Configure C++ integration test executable with a single environment variable. Update README.md
---
 ci/travis_script_integration.sh |  4 +---
 integration/README.md           |  4 ++--
 integration/integration_test.py | 16 ++++++----------
 3 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index c019a4b7ab7ff..7bb1dc0a6015c 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -29,9 +29,7 @@ pushd $TRAVIS_BUILD_DIR/integration
 VERSION=0.1.1-SNAPSHOT
 export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
 
-export ARROW_CPP_TESTER=$CPP_BUILD_DIR/debug/json-integration-test
-export ARROW_CPP_STREAM_TO_FILE=$CPP_BUILD_DIR/debug/stream-to-file
-export ARROW_CPP_FILE_TO_STREAM=$CPP_BUILD_DIR/debug/file-to-stream
+export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug
 
 source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 export MINICONDA=$HOME/miniconda
diff --git a/integration/README.md b/integration/README.md
index b1e4e3a82a734..6005b62c41cd5 100644
--- a/integration/README.md
+++ b/integration/README.md
@@ -34,7 +34,7 @@ mvn package
 ```
 
 Now, the integration tests rely on two environment variables which point to the
-Java `arrow-tool` JAR and the C++ `json-integration-test` executable:
+Java `arrow-tool` JAR and the build path for the C++ executables:
 
 ```bash
 JAVA_DIR=$ARROW_HOME/java
@@ -42,7 +42,7 @@ CPP_BUILD_DIR=$ARROW_HOME/cpp/test-build
 
 VERSION=0.1.1-SNAPSHOT
 export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
-export ARROW_CPP_TESTER=$CPP_BUILD_DIR/debug/json-integration-test
+export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug
 ```
 
 Here `$ARROW_HOME` is the location of your Arrow git clone. The
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 1d8dc29a9f529..d5a066be5f246 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -684,17 +684,13 @@ def file_to_stream(self, file_path, stream_path):
 
 class CPPTester(Tester):
 
-    BUILD_PATH = os.path.join(ARROW_HOME, 'cpp/test-build/debug')
-    CPP_INTEGRATION_EXE = os.environ.get(
-        'ARROW_CPP_TESTER', os.path.join(BUILD_PATH, 'json-integration-test'))
+    EXE_PATH = os.environ.get(
+        'ARROW_CPP_EXE_PATH',
+        os.path.join(ARROW_HOME, 'cpp/test-build/debug'))
 
-    STREAM_TO_FILE = os.environ.get(
-        'ARROW_CPP_STREAM_TO_FILE',
-        os.path.join(BUILD_PATH, 'stream-to-file'))
-
-    FILE_TO_STREAM = os.environ.get(
-        'ARROW_CPP_FILE_TO_STREAM',
-        os.path.join(BUILD_PATH, 'file-to-stream'))
+    CPP_INTEGRATION_EXE = os.path.join(EXE_PATH, 'json-integration-test')
+    STREAM_TO_FILE = os.path.join(EXE_PATH, 'stream-to-file')
+    FILE_TO_STREAM = os.path.join(EXE_PATH, 'file-to-stream')
 
     name = 'C++'
 

From d50f1525a999e1de837334adb7a3d7d0da3f0c33 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 14 Feb 2017 08:25:58 -0500
Subject: [PATCH 0332/1644] ARROW-558: Add KEYS files

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #341 from xhochy/ARROW-558 and squashes the following commits:

ea5327b [Uwe L. Korn] ARROW-558: Add KEYS files
---
 KEYS | 180 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 180 insertions(+)
 create mode 100644 KEYS

diff --git a/KEYS b/KEYS
new file mode 100644
index 0000000000000..ad0f5cf6f4340
--- /dev/null
+++ b/KEYS
@@ -0,0 +1,180 @@
+This file contains the PGP keys of various developers.
+
+Users: pgp < KEYS
+  gpg --import KEYS
+Developers: 
+  pgp -kxa <your name> and append it to this file.
+  (pgpk -ll <your name> && pgpk -xa <your name>) >> this file.
+  (gpg --list-sigs <your name>
+    && gpg --armor --export <your name>) >> this file.
+
+pub   2048R/7AE7E47B 2013-04-10 [expires: 2017-04-10]
+uid                  Julien Le Dem <julien@ledem.net>
+sig 3        7AE7E47B 2013-04-10  Julien Le Dem <julien@ledem.net>
+sig          D3924CCD 2014-09-08  Ryan Blue (CODE SIGNING KEY) <blue@apache.org>
+sig          71F0F13B 2014-09-08  Tianshuo Deng <tdeng@twitter.com>
+sub   2048R/03C4E111 2013-04-10 [expires: 2017-04-10]
+sig          7AE7E47B 2013-04-10  Julien Le Dem <julien@ledem.net>
+
+pub   4096R/1679D194 2016-09-19 [expires: 2020-09-19]
+uid                  Julien Le Dem <julien@ledem.net>
+sig 3        1679D194 2016-09-19  Julien Le Dem <julien@ledem.net>
+sub   4096R/61C65CFD 2016-09-19 [expires: 2020-09-19]
+sig          1679D194 2016-09-19  Julien Le Dem <julien@ledem.net>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQENBFFll5kBCACk/tTfHSxUT2W9phkLQzJs6AV4GElqcFo7ZNE1DwAB/gk8uJwR
+Po7WYaO2/91hNu4y1SooDRGnqz0FvZzOA8sW/KujK13MMqmGYb1jJdwPjNq6KOK/
+3EygCxq9DxSS+TILvq3NsFgYGdopdJxRl9zh15Po/3c/jNMPtnGZzP39EsfMhgIS
+YwwiEHPVPB00Q0IGRQMhtJqh1AQ5KrxqK4+uEwwu3Sb52DpBjfgffl8GMGKfH/tk
+VvJ6L+7rPXtNqho5b7i8379//Bn9xwgO2YCtjPoZMVg37M6f6hVWMr3fFmX/OXgU
+UWwLGOTAeuLKWkikFJr5y0rzDaF2qcD9t7wfABEBAAG0IEp1bGllbiBMZSBEZW0g
+PGp1bGllbkBsZWRlbS5uZXQ+iQE9BBMBCgAnBQJRZZeZAhsvBQkHhh+ABQsJCAcD
+BRUKCQgLBRYCAwEAAh4BAheAAAoJEJfX6GR65+R7au4IAIfZVA9eWBZn9NuaWX7L
+Xi+xDtzrfUrsWZxMIP6zkQsIspiX9AThGv3zDn+Tpfw7svV1QfUQX0LHbwMMYqq+
+mRJB/kqYutpLxw7h63zrWR2k2Sdzvole2c3Rfk1vblIdWZk7ArLSivqTk/oGwr7d
+MejvOMmKSzqW0vQF6dNbYerLOiqPr4mKqONWm4nOLZEBzjE3IfbK3gNBSFq+92jV
+iWY6ozqAxydYafNUSZRrcniYskxd9JCSSLZiIZW3X9lToA/74LjpPbmzvQtkH68D
+0EnC1mkPTKCA4r+CLb3a9GJ9Surg2T0OptyPHsXipgViVryXgopD2odA3fh9SY5l
+Ee+JAhwEEAECAAYFAlQN+kQACgkQ/LPL2dOSTM3+OA//dYj9kiZhZNVb6hMfrubn
+OjTmY8Hcax8G+aJWxRrGE8HrCUjEJ4NThK523+fmol1PxNWsguljlsZvJ189YPOh
+weDJzNmKwhLntq/uBgtJyWBN1v9bUzkR9Ud+UdD1tPbNj7sNiIQE1ZqWMxra3sq/
+gcodVgqSADGgjKO9tenQhWvQXxBR55MOqZbxnyazRPEYS0mkN0A0DwtG82tHNRL7
+Z3vs/kG5hoW3kYifCZn5pW3wKtfIY5JH7usYOzA86p7GH4hOfO+dzhDANH+C+u9O
+ZRbCdUE8oEp3fAWY9+3VzlO5ixpFOeHGfbSJp44Jv6wUOxNwRmD/gk+DxVrsS/Yn
+rLFCZgDHgkFHGJ1D7PnxTy4qtwGasYxWYJOUiaAJbOvRa8nbhan2/wsrgnJTbXAH
++7v5tFfCV77Po//V0fojYZNvbkEO8/yRpQL+uKiVRaRD5dMfHRb31OR0A59ssYX9
+63QpBEof/OeELC0VowG+KCc+4CfSMmAGnQMdEhMAUPz+79nJw7ijeF5C82Z5mQof
+v+nf+kdqr80UbG+RoODKtlHFETxJ5STQe6uiPOfvb+EADPA0cZ34u5tD3Z+SMV1k
+Gf7Jxi45jmkn9Z9AkVj6KgdDeSjV7EkRiY0pm43Vvd6WvV5t54cgJcwXrjG+h03f
+65w7F+KBrh7YAcUvrf4JeXKJARwEEAECAAYFAlQN/XwACgkQfNgniXHw8TtU9Af/
+b9CYFtsG9q1ZbnV9SChxjLLUipGsmKTUjCnz7oiZvJJ04e+0np1NQJKJbthGfEDM
+eLt1WiYpTDu66zAuLDA7ACcbv3UUXXsUTEfN76J+9DJHrtK1soHGLkKLW2hZeWKp
+PKya/HRF4Rv3/aAwWtRjEuQr9pLt/wAOedV6mrpyTngOKQn97tzo/yUeDNG7be8A
+xtUStQY/2zJmHkaLeULKOspgUchBQ1S+M4q46dE+tyel47BLyHIECqk/geLOlZmh
+lo6TtVgnBSXC5SqMwh5pz/P5ntQ8FVLedGQI9dwVhxbjoo5DNB/6ntfbwkheiak1
+CFBm0ZVPJjX7F2XFcq7VCrkBDQRRZZeZAQgA4eixR7xHvnTyF12CYLsnFE8x1tI+
+78FCjKm0n1YPCzEYa70bnnZmpW4KCwO0flN4RhhP+g2KRCCov2ZH7bxvhTxe4n/j
+T6I/+61Fpba4I7qExYqX+tylyjUKhynLcWCbvRQnyjOMTaLbMVrftV+ATVmj7fi0
+PdzRW/7QvCSrDsMFtTSaNBdeMbzptpoXAxTgVZOIoHbWOIfovN1uPnFItrmNnKXX
+KGyDPX2s2KCz10G1lrw0l9tqDg+BtqE9/xCtqWoZJMnT8jAJZeJ0V37R1jDBDEHK
+AfPOUKNYf5GWxJeCWYzL77ve8VdItKwPhtjW7zFKuyrqiBHE40fgTLKvNQARAQAB
+iQJEBBgBCgAPBQJRZZeZAhsuBQkHhh+AASkJEJfX6GR65+R7wF0gBBkBCgAGBQJR
+ZZeZAAoJECrRWHEDxOERzmEIAOCrfYGPdLyzBn/xAdymx2FaNTS48ybMIGjcu6Od
+nKzvgBJObLPQf0+WKhkbQf2HEHYinBVpX8K4dNY9RhzIRbQNhCWY5E5/leI/nQ9O
+ZBUMpT8Gw5saj0YtF3By4E9ywxNWiAyX2SAHjPv/lub0PEaUiWWe6s9MaX5fp71C
+TupkdElpxucEpVefUaUOSMQ2ecOniCh/9ltPLYcjwnC1ti+Et8/cAK2N554GNE+x
+fO3qtGXGUleWhpt3fblTcCyO+odAPKxm70jnABLk8m+KpffcdBYSJ5ai5hPkrnyq
+3NBRDPGlLdtDkzn0/xKYnVbLW1d+d2NFwJzEKncQphHoo0T19wf8DSfym7dIsstj
+jwFI8+N/1yCdMD886x8bgmsSsNiD9tro+1083yr+IL5+gUs8Q4ETpsow+IS6sfp2
+fzA0TaLBLEOFYy/XFxnzO+YtVNIDAnrDEgTOMahFUrJ/HVZF9xT+kKwhyHaRNIQL
+CYc4VoSWldqoDVOGI30NjtVo5EGzf3qVWkTm4yplBhJvJanxrMHuJAWRgFX8D48B
+cs/senr8s+O0oXQQYIjz/FkZh/mQFtrgsvnzyUR52SnwEzNMmXjZNkydPZwcY6mu
+cqCIvQIvmBpPdlyaoglwJ8wWb76uIE6VFcN71FF3EfV51/yUeQGJaoExWLY6IH8x
+Xtn3IWkBWJkCDQRX4AxuARAAzzTxE1FGdmJYPZyTys51oDi8+CJ8VXF6wlTkjuOW
+abkGUu0FjnN++D7G9LRDvN7QnVUHU+h6QWPZ0LanmjYh0ABO5SeWCjOX6ajcACkz
+pEzMv2DbOPfJuPJmtuFfiLOQAUVBB1dSSPFMPPaGTco2iE7uLr8edtQBvgpx/PGd
+52lma3qAAZFzonKWyTRonUjV4SU3C96Xhbs+DExTL8H6MX8NzZCz4UZj5u4NsEH+
+oQD0U4tSOe3xgroJpOR6ZPvlhgbWVqlYvkEWt0AaPJsXJwnWe17GgDmxME2cwsuI
+fgv/9shB7VYmLglY6dV/6HYoHh+2qKXndTMjlqXXvUHW0J3uRryoCR+C2gin38/f
+sPFICpt5yJVnR517O/jsviDz4TwjhqFsUUM7Ud8IydriJX02Oj5UitUF7l5MSqkS
+/Z7jwPEErCRWwmfj4ZjjWWV60I9SYgPZhBp0//s2o/gbIBBtIdHI2+xaMt0lWOsA
+Xi7dsY1NLGoSGUlhdSiP032tVHpGiOV3AWwf399Qus4iuwf6N8KEVSTRdaA7b1Um
+b7PepfEHIrOS5oUYjgZJK+JFGey5SOsPvG3Yv9cbEKWqmoEzEDb6y3HI/iRbk/qC
+SWGKvEiqYSo6wlvZFDv1qoApylfBaI8Lf32vawlMCSI37KCWfua1RqbCYMi/4wux
+bfsAEQEAAbQgSnVsaWVuIExlIERlbSA8anVsaWVuQGxlZGVtLm5ldD6JAj8EEwEI
+ACkFAlfgDG4CGwMFCQeGH4AHCwkIBwMCAQYVCAIJCgsEFgIDAQIeAQIXgAAKCRAC
+2r/fFnnRlHqND/oCaPPGn8u5oyVml9J3+lpYWwT69qHwYV5IX+72zqq02uvYEqlY
+CseEwOvkfDClh81KWO4A9kzVwWcu611d/UFsA94EnZuJ46m8DflPeidhJoTRnNpr
+IRzH2lL72QyQFeT9viWHdxu3cKlJkChQuD3zR9yyVH/QVFOlBvdx/ZT0dOFpbgJd
+2n4fy8ExGSXLP6wGf5RQRKEYiZX4VB4Bkz6sK0Con6GPsqqtaUgj9fCxA1ebhGA2
+k1m79mR9gh4oJWeefSuXyf3x8oBoQ46Lury8HiuxLh6cy9SqHZ8uXu3hfQEZ6rhd
+C9yBjK2+8z6GLhjgSasMkJK0OAR8yLgARZJwt9+wV/Ww15Sq76B6IrKJnSR6P4FM
+jA+ItCDtiooGz6rJGFidsH99fU2IydcsSqbTN3h3/2cBXFgxspesHWsEeTvtXSgR
+I82kUyA0g1v9ESY0leiLVzKyL0zmCjaPg0nHoH8tFqFkqaSXSZu75TefnsokkoXN
+ewkDf+yD2J/BMtUHAgFOlvYkywGzS9cbAxxzc607Jvww5LjtPI0wYRIwzOlvZfws
+xoYPrqJe1R7hRy0QS6pnSL7TgdbvwbGtiUAZ9w/Y5FEugV8bgyZMvF7Z9gt3ThMg
+XOSWlZrsDym7jg/yd/h/4aPZuPC73oNvgV4g6OT510fkkMNWbZR8C2uHX7kCDQRX
+4AxuARAAphEmWY5Z3Q+gQ1X9+b55VE17ORMKjXtE2gQnYk9Fxpt31F0kZcoK/25Y
+BItkjcmIaC4LTLjbdwe6IW4zlOjULxaTstTJsfCcrJONlSmEJ0OWaXx9i/tAXt8d
+0IZn2hkQ9aevJuoWqta+wFNhpLdPuPQq6vO6hIl7j0w1tAGFHV+IQ7Q7VFuUVo/h
+gZbtJOjufZWqulz6pMVu4p3TW9OM98CWioO1eidcwKYEsgk/fJ1uKc599SSCz2Cg
++lEho6rHtvojk34TLD9QQHaEcCZToq7WSqwqLi4OCuhcfAVuwydj0RMByE1TOpsg
+RwOh2egBLNplK+0k2jVaQPX38laolOAMNLg+VVRy73T1MpyelY/m+cRr8292X/f4
+GgHNHbmQU+LDzsezC+ryPXdP3FjVo66xNlYHzw1x0hRdnwExkqOYmdTz1YN83Z+6
+p0d2RkTZTpLnE449KiNsgTPttplBGE8QKNqYxoKIk40DlDuya7q/acgTcqe8vW0v
+34E6RRIX8dbCJeTBB2vUDp6bD3ICXGI09EuUAh9yy/FlNv1OdggDfTnF/NztWmmT
+CpNwmdx+GTT2Sv0i6H9RelOl0uGj351+7PSFSFrHV9T3TUaMB0QkkZDxklIvPVZv
+dhx7UGXFJPDjQyJxcN7UW6Pc7m+m2W3/u2MZaL7xPbWRPVkqs48AEQEAAYkCJQQY
+AQgADwUCV+AMbgIbDAUJB4YfgAAKCRAC2r/fFnnRlGwCEACXcfMAz79G2sLs6z1N
+6tMbO0qGGQJ9vAXRKjb7JN/yd+z+zaejs/+cmRM+wHKZtANYtnSzGiWJO4TIP5A+
+DeRE93GJaVr0ly0C+du/uSm6wVg+w1wgy6JE6q/IsMW5qHd0qWi/npq4esfH1Uho
+T7Kl/AxUyT0N23n5oK0GrVUFhFcU9dUx6auhHxEOM5tIgNqV6lAn72lykPYUvV5f
+aAiz2OAlVYxgBb6wxjXTUVlrhaxbgNQ7PPzkjzMVZaE/TZrcyl4Ck3grYDBFZEGi
+jhjsl/HX+/lhJvr5gcFkisG5A2pnrkAe1wnXm4HoKGN2xUWCCipN5oPc3Lw6ge76
+YDX1t5CXqd94cDBlwFDtd4kykI3rJDvTI3P/fevMNqVS3tzW9AwkHkPil1DE+4rI
+/qCib+G6BAgloUGYLuNxSa1ySOd0yckFTrNBB5yk+yWvrLpKGFVdQS7BwUcgdeCJ
+3XU3fyhfXcIn3tMHabZ6laB3Xzi3Gi8iL6SJywSXIqTGw3MmLJlxr1IKWTMNxjjs
+d/XBF7ZpCCisH7s9hyMCAet72YFAxVcB3bwbd3mzcGfTg/Y+sSum82vaSvAJ0QBc
+pp4X8HzEsSsJ88N6ON7IU92r+1mxWhglKZx2NORHIvNFwIrvAzKWhqGdHd5/xq3f
+EwCykGi6RtdCStNFh6h16kCkgA==
+=YkSF
+-----END PGP PUBLIC KEY BLOCK-----
+pub   4096R/8CAAD602 2016-11-06
+uid                  Uwe L. Korn <uwe@apache.org>
+sig 3        8CAAD602 2016-11-06  Uwe L. Korn <uwe@apache.org>
+sub   4096R/7BD1BC86 2016-11-06
+sig          8CAAD602 2016-11-06  Uwe L. Korn <uwe@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBFgfd4wBEACylQqqVH/aK00fgU/v1ZggNwtgJhzH7yswAzQz9eUU5t4Q9kzI
+zdkR1yJvaEDHtZy2D0mCM1CuGVPXzf+0kSFDaRPcm6LNAD15KC7eUzyad1Y4MwNn
+UYE3pZlnvSwUBAigQSN1quw+u1eHc+IJc32iCRcK8DihQgrDivg8yZckoGGZj/6w
+Epfp8SLrI+OmqBgwYYjRqy9uC0aWypKb9waZmc2NIZZu1y3bL6hx54+Dk+4hF01E
+OtT79HQV1e4MyqiuGUKa34QAHb1CGrju+1Z9sDNdI7hBDqfQKjisR2WaJM4kXHjj
+m7Tv3M1LUB4eh1+Yd514d/wpSChkLvMCJ9tYGSpQ8c+qrLAFvgRD7YCYp4ypslcx
+Sg30gU0bcTu8aiIm7qfl9CUjtBYwirUGC/t2SUxnhOpxWuzZdAiUJHi0QFa+LnZa
+ecA5fIoMfqTWAqfQr3noxB6qLLNCgZd7IIH5KXIIhJZHpO3eMCCTJuDXiMS1Z/uo
+D1FvUL8c19nmMjPJSfQo95Uynw6gZKFy0d3xg7NKUvnJBsVI24/PTVabzRrDh/qb
+RCHvQOFjXOSYsPm2sz1BPs+ucV4AoxPZFgsCfUN2t4FRbcb39vr6oYFb+Nd3sIKX
+7wknSwAid6pATvfZuLC9NI8ykjcEDGeLL0sET3kdUeuGYjpj2kuhnrV4cwARAQAB
+tBxVd2UgTC4gS29ybiA8dXdlQGFwYWNoZS5vcmc+iQI4BBMBAgAiBQJYH3eMAhsD
+BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAp2U4ijKrWAos1D/98UBoLbt6L
+c7mnXTww069nkt0vOOHSz/QWJxo5rQsqFSKcSRuBhwLuaVMGTjBqCOLdEmA+XKJ8
+O+OgCZz0QZXuwL3PklX3DFvsYO0wIEIssovEJMu5e3XxDcCf5ZZtfszW5dnbWTjc
+JXP0TlEbjOR5Z0/O/24iysGtoEMiktRTLOz9R5oRXFQLN4jQSykvMfKhanCVFljX
+qEdMszjtvZhLwOiCaWkIOEo3jCrCDhdThI5nTiu/pH3vi7mkFYTNKpiva3XYKH7V
+ITEdn5WO/QNFu/VBRjtOxT+F068vuuNpvAddn5rOtZOyGMCHnEqnlRnqIIZGtJeo
+EJ87N2ytn8CtKpQKhyJIJhQIfW5jS3YW8qj1HeKN2s5wqQKnBYYsJOh+/QC9g3oE
+nllgoSHAKSzys2Y1VoOQbRxYipCqRx7uS2aAqFr6r3hQpzySWeKQuxVZSZD7ar/0
+AFB0Hg4EgUGDl6Lw5icJ7scXTgoQKZWH1UmNc/FwFbG/F1GVbU88R6DlF84D1X/P
+ArtP20eT+B3u5nfO2pCaBVi6GYyMsL2WKHO7AQAgURMgEPk0AQZZpv/OSJFa/TzI
+UQ8xTLgmwZRL/XjjNFYWs+eYecGQsHKLbKNm1BpZMEfbVSFw54PiyJgoOhdMKdyA
+Cmb+aUBkbPXf5S0ScZOoq8e8k1dYseDGOLkCDQRYH3eMARAAx/joL6ScsKMmPGRn
+n79gQ3zbcKxWSfEDMYeeFfSssRgRd2iIrgvjzr9phka2yknzPnQPi7C8GLkUTj5e
+V1dBxIGkGmP28n0DoowMqGb1xqn0WeoxDL0VQycGjkv5SOkxcbCCKS/MHOn6zenh
+patSJsEHkCqk3f4GtPngYN5oMRTXUfUj1s7AooNti1ONSQSvZNbOMKAg8MgAjAHm
+z3A+INLVTa59vqUNr5ptG/n+cB65ggeNhJf3gMaDyUy7oRZtOhrmA4D9CLpy2OBA
+gezgOCZk/mPNP5jW0sbRiL6nYqC9VTp0E+f3hYSdgXNTWGIcxOwK7xe09SRqUQ7u
+WnoKBTjkkYdCaCN4rv8IhJdrufgYdfqMGuldQZ9R/gcN3Iel7JMdon2onk94KZPs
+W58/1DCD2eRuz8CsIgleUHVXJ+mCpkdtAt45ZGyv5pFC/+6s8mS/pBQEdVl7wjEX
+kf2lrtFZCfK1uUiUTDnJJdtXNhwdtvnxJYeRg51jlD9Qg/mPV6m8KFyINtLKedLv
+hChFkAIfFsdC/r1Xt4fMiCv2eZ8Dop2dM6xV/6Ueicti0lywoTpVtugSUWPO1j8a
+N48jUfkZUV0jdELNHAloZaIDeLc7mU0uZJ3JykC4laD+YDwHT8tYUvamtU2uNgh1
+V7I3jrEu8YO4T2fiXe+0EzBwzjEAEQEAAYkCHwQYAQIACQUCWB93jAIbDAAKCRAp
+2U4ijKrWAs3bD/wOE8NLnzKqebz0v+lxQf7fRL+RMaJ8mFda/t7UFtxj6XdePGZy
+HWdqlvBFSDo/K6aEiicmpEIPbMi+V7d1Dg3tGhwtkHzgbpxNVoolR+2cF4jtrkoV
+NC7uAMaDPt0X+wqinGg4E7IFuJoT4WiS+i4lzCUbD8n7lxe6Kj9bDt8tb6gOCgld
+oweGN2k3bc4hIzeRt0jqGu1xm91Zbf8YbI3vyi8WQqmxX3zugY46NWwj8a+4Mhxz
+Ysd7SI1pPs5k7vdHif3MD3Wwx68CCuZSm2KzNsm0iGxrCXSA6dXVflK9rlq6O1Us
+UTxfX60o6S8PdFr4oOPFHYXmvDU5PY575xscWB2VVAyuSCyZWtq8d1BBU9JxcozS
+6PTefVUqgr0XXRwVldAIabSA5q13j+b5+vU6LnAuoeMlFFprRlcJN03XTWKXF/gP
+SpCDscCEMbz7aHpox8wmFckeiT+TgwDLMKO5PKRSMEBErUk+SsOyBnFpuGaPsCem
+Pi6NwQyPCt3eep4Ti0dPo3u/dCUEtdKWMpOhsPIoCvGpgqS7o5PuBC2MDHQCc7q8
+wfxeCKBeSpMuy3pvOnNy8uNYjNqizVlpNBx01I2R1MD8P14Pxteg6APi0jcusXrD
+s8g7c7dzdXM0lxreeXge8JSmxuwcCqVUswac6zbX4li03m/lov2YYxCwuw==
+=ESbR
+-----END PGP PUBLIC KEY BLOCK-----

From fa8d27f314b7c21c611d1c5caaa9b32ae0cb2b06 Mon Sep 17 00:00:00 2001
From: Holden Karau <holden@us.ibm.com>
Date: Wed, 15 Feb 2017 08:55:24 -0500
Subject: [PATCH 0333/1644] ARROW-561:[JAVA][PYTHON] Update java & python
 dependencies to improve downstream packaging experience

The current build for arrow uses a interesting work around for hamcrest conflict between JUNIT and mockito which results in mockito being in the compile scope. This is not suitable for some downstream users.

Python setup file also leaves out test dependency (not overly important but useful for developers) & we can clarify parquet-cpp as an "extra" dependency for people requiring parquet support (already mentioned in the README file but good to have clarity in setup.py as well).

Author: Holden Karau <holden@us.ibm.com>

Closes #342 from holdenk/ARROW-561-improve-deps and squashes the following commits:

5919b32 [Holden Karau] Drop extras_requires
938ed97 [Holden Karau] Mention test requires pytest and add an extra requires for parquet
283d6cd [Holden Karau] Remove mockito from compile scope and fix hamcrest issue with exclusion rule instead
---
 java/pom.xml    | 10 ++++++++--
 python/setup.py |  1 +
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/java/pom.xml b/java/pom.xml
index a147d66c98318..e467cc185be06 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -442,11 +442,17 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <!-- Mockito needs to be on the class path after JUnit (or Hamcrest) as
-           long as Mockito _contains_ older Hamcrest classes.  See arrow-2130. -->
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
       <version>1.9.5</version>
+      <scope>test</scope>
+      <!-- Mockito  contains old Hamcrest classes, causes issues with JUNIT -->
+      <exclusions>
+        <exclusion>
+         <groupId>org.hamcrest</groupId>
+          <artifactId>hamcrest-core</artifactId>
+	</exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>ch.qos.logback</groupId>
diff --git a/python/setup.py b/python/setup.py
index 5f5e5f3795541..54d1cd3af48bc 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -298,6 +298,7 @@ def get_outputs(self):
     use_scm_version = {"root": "..", "relative_to": __file__},
     setup_requires=['setuptools_scm'],
     install_requires=['cython >= 0.23', 'numpy >= 1.9', 'six >= 1.0.0'],
+    test_requires=['pytest'],
     description="Python library for Apache Arrow",
     long_description=long_description,
     classifiers=[

From f6924ad83bc95741f003830892ad4815ca3b70fd Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 15 Feb 2017 15:59:36 +0100
Subject: [PATCH 0334/1644] [maven-release-plugin] prepare release
 apache-arrow-0.2.0

Change-Id: I71a840dd1891d1b738d6a43748642390d7541f42
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index eb045d655e982..055df5b2b0622 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.1.1-SNAPSHOT</version>
+  <version>0.2.0</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index a4eb65228febf..a3085aa506f65 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1.1-SNAPSHOT</version>
+    <version>0.2.0</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index e467cc185be06..ea0d0297ac3b4 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.1.1-SNAPSHOT</version>
+  <version>0.2.0</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -41,7 +41,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>HEAD</tag>
+    <tag>apache-arrow-0.2.0</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index ef96328f7668a..7271778aea9ad 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.1.1-SNAPSHOT</version>
+        <version>0.2.0</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 8517d4ced80f1..8ac42531e7f68 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1.1-SNAPSHOT</version>
+    <version>0.2.0</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From ab15e01c70d12ea163dd9b0109fa9332884e3e7c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 15 Feb 2017 15:59:46 +0100
Subject: [PATCH 0335/1644] [maven-release-plugin] prepare for next development
 iteration

Change-Id: I1a9e3a6d0dc29c1a7933d373a7224a7bbd60e7e9
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 055df5b2b0622..c65a7bc3de197 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.2.0</version>
+  <version>0.2.1-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index a3085aa506f65..f20228b1bee62 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.2.0</version>
+    <version>0.2.1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index ea0d0297ac3b4..fa03783396ffb 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.2.0</version>
+  <version>0.2.1-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -41,7 +41,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.2.0</tag>
+    <tag>HEAD</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 7271778aea9ad..35e5599b3b64c 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.2.0</version>
+        <version>0.2.1-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 8ac42531e7f68..fc3ce66ac1f80 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.2.0</version>
+    <version>0.2.1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From ef6b4655798e9c31631377bd6412c36405001f7f Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 20 Feb 2017 17:23:40 -0500
Subject: [PATCH 0336/1644] ARROW-563: Support non-standard gcc version strings

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #343 from xhochy/ARROW-563 and squashes the following commits:

64d1c93 [Uwe L. Korn] ARROW-563: Support non-standard gcc version strings
---
 cpp/cmake_modules/CompilerInfo.cmake | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index fe200be65d502..079d9d1f3270d 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -21,6 +21,7 @@ execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
                 ERROR_VARIABLE COMPILER_VERSION_FULL)
 message(INFO " ${COMPILER_VERSION_FULL}")
 message(INFO " ${CMAKE_CXX_COMPILER_ID}")
+string(TOLOWER "${COMPILER_VERSION_FULL}" COMPILER_VERSION_FULL_LOWER)
 
 if(MSVC)
   set(COMPILER_FAMILY "msvc")
@@ -53,10 +54,10 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-8")
   set(COMPILER_VERSION "3.8.0svn")
 
 # gcc
-elseif("${COMPILER_VERSION_FULL}" MATCHES ".*gcc version.*")
+elseif("${COMPILER_VERSION_FULL_LOWER}" MATCHES ".*gcc[ -]version.*")
   set(COMPILER_FAMILY "gcc")
-  string(REGEX REPLACE ".*gcc version ([0-9\\.]+).*" "\\1"
-    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+  string(REGEX REPLACE ".*gcc[ -]version ([0-9\\.]+).*" "\\1"
+      COMPILER_VERSION "${COMPILER_VERSION_FULL_LOWER}")
 else()
   message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
 endif()

From 4598c1a36c20de1f4d12dee62c79a67197e8a603 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 21 Feb 2017 14:41:54 +0100
Subject: [PATCH 0337/1644] ARROW-570: Determine Java tools JAR location from
 project metadata

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #346 from xhochy/ARROW-570 and squashes the following commits:

32ece28 [Uwe L. Korn] Add missing )
f1071db [Uwe L. Korn] ARROW-570: Determine Java tools JAR location from project metadata
---
 ci/travis_script_integration.sh |  3 ---
 integration/integration_test.py | 11 +++++++++--
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 7bb1dc0a6015c..8ddd89b1639b0 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -26,9 +26,6 @@ popd
 
 pushd $TRAVIS_BUILD_DIR/integration
 
-VERSION=0.1.1-SNAPSHOT
-export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
-
 export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug
 
 source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
diff --git a/integration/integration_test.py b/integration/integration_test.py
index d5a066be5f246..049436a751f38 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -34,6 +34,12 @@
 # Control for flakiness
 np.random.seed(12345)
 
+def load_version_from_pom():
+    import xml.etree.ElementTree as ET
+    tree = ET.parse(os.path.join(ARROW_HOME, 'java', 'pom.xml'))
+    version_tag = list(tree.getroot().findall('{http://maven.apache.org/POM/4.0.0}version'))[0]
+    return version_tag.text
+
 
 def guid():
     return uuid.uuid4().hex
@@ -638,11 +644,12 @@ def validate(self, json_path, arrow_path):
 
 class JavaTester(Tester):
 
+    _arrow_version = load_version_from_pom()
     ARROW_TOOLS_JAR = os.environ.get(
         'ARROW_JAVA_INTEGRATION_JAR',
         os.path.join(ARROW_HOME,
-                     'java/tools/target/arrow-tools-0.1.1-'
-                     'SNAPSHOT-jar-with-dependencies.jar'))
+                     'java/tools/target/arrow-tools-{}-'
+                     'jar-with-dependencies.jar'.format(_arrow_version)))
 
     name = 'Java'
 

From 5e279f0a73842518caf34c2cda7c941548d55dbf Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 21 Feb 2017 14:44:01 +0100
Subject: [PATCH 0338/1644] ARROW-569: [C++] Set version for *.pc

*.pc.in such as cpp/build/arrow.pc.in refers ARROW_VERSION but it isn't
defined.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #344 from kou/arrow-569-c++-set-version-for-pc and squashes the following commits:

48b366b [Kouhei Sutou] ARROW-569: [C++] Set version for *.pc
---
 cpp/CMakeLists.txt | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 035cd8f9b90c7..0888a8b97faa1 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -18,6 +18,12 @@
 cmake_minimum_required(VERSION 2.7)
 project(arrow)
 
+file(READ "${CMAKE_CURRENT_SOURCE_DIR}/../java/pom.xml" POM_XML)
+string(REGEX MATCHALL
+  "\n  <version>[^<]+</version>" ARROW_VERSION_TAG "${POM_XML}")
+string(REGEX REPLACE
+  "(\n  <version>|</version>)" "" ARROW_VERSION "${ARROW_VERSION_TAG}")
+
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
 include(CMakeParseArguments)

From d28f1c1e0f21bc578b84ab4bed4cf259c333fbc9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 24 Feb 2017 09:16:32 -0500
Subject: [PATCH 0339/1644] ARROW-459: [C++] Dictionary IPC support in file and
 stream formats

Also fixes ARROW-565

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #347 from wesm/ARROW-459 and squashes the following commits:

6a987b7 [Wes McKinney] Fix clang warning with forward declaration
8e0e6fb [Wes McKinney] Fix bug causing valgrind failure
dee044e [Wes McKinney] Review comments
7ac756e [Wes McKinney] Fix Python build
e5cec27 [Wes McKinney] Add some less trivial dictionary-encoded arrays to test case
acfa994 [Wes McKinney] cpplint
ef9dea8 [Wes McKinney] More dictionary support in FileReader. Simple test passes
cb04a41 [Wes McKinney] Refactoring. Remove FileFooter class in favor of private impl in FileReader
1cee0ff [Wes McKinney] More progress toward file/stream roundtrips with dictionaries
ae389fa [Wes McKinney] WIP progress toward stream/file dictionary roundtrip
6858e12 [Wes McKinney] Add union and dictionary to file/stream tests
d537004 [Wes McKinney] Add support for deconstructing and reconstructing DictionaryArray with known schema
---
 cpp/CMakeLists.txt                       |   4 +-
 cpp/src/arrow/array.h                    |   2 +
 cpp/src/arrow/io/CMakeLists.txt          |   9 +-
 cpp/src/arrow/ipc/CMakeLists.txt         |  15 +-
 cpp/src/arrow/ipc/adapter.cc             | 189 +++++++++-----
 cpp/src/arrow/ipc/adapter.h              |  30 +--
 cpp/src/arrow/ipc/file.cc                | 306 ++++++++++++-----------
 cpp/src/arrow/ipc/file.h                 |  51 +---
 cpp/src/arrow/ipc/ipc-adapter-test.cc    |  46 ++--
 cpp/src/arrow/ipc/ipc-file-test.cc       |  78 ++----
 cpp/src/arrow/ipc/ipc-metadata-test.cc   |  17 +-
 cpp/src/arrow/ipc/metadata-internal.cc   | 239 ++++++++++++++----
 cpp/src/arrow/ipc/metadata-internal.h    |  45 ++--
 cpp/src/arrow/ipc/metadata.cc            | 185 ++++++++++++--
 cpp/src/arrow/ipc/metadata.h             |  97 +++++--
 cpp/src/arrow/ipc/stream.cc              | 207 ++++++++++-----
 cpp/src/arrow/ipc/stream.h               |  34 ++-
 cpp/src/arrow/ipc/test-common.h          |  80 ++++++
 cpp/src/arrow/type.cc                    |   6 +-
 cpp/src/arrow/type.h                     |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |   1 -
 python/pyarrow/io.pyx                    |   5 -
 22 files changed, 1093 insertions(+), 567 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0888a8b97faa1..b77f8c79fa024 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -102,7 +102,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     ON)
 endif()
 
-if(NOT ARROW_BUILD_TESTS)
+if(ARROW_BUILD_TESTS)
+  set(ARROW_BUILD_STATIC ON)
+else()
   set(NO_TESTS 1)
 endif()
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 32d156b8cd0f6..9bb06afc9bf6c 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -40,6 +40,8 @@ class Status;
 
 class ArrayVisitor {
  public:
+  virtual ~ArrayVisitor() = default;
+
   virtual Status Visit(const NullArray& array) = 0;
   virtual Status Visit(const BooleanArray& array) = 0;
   virtual Status Visit(const Int8Array& array) = 0;
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index b8882e46b4893..ceb7b7379322a 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -70,13 +70,8 @@ set(ARROW_IO_STATIC_PRIVATE_LINK_LIBS
   boost_system_static
   boost_filesystem_static)
 
-if (ARROW_BUILD_STATIC)
-  set(ARROW_IO_TEST_LINK_LIBS
-    arrow_io_static)
-else()
-  set(ARROW_IO_TEST_LINK_LIBS
-    arrow_io_shared)
-endif()
+set(ARROW_IO_TEST_LINK_LIBS
+  arrow_io_static)
 
 set(ARROW_IO_SRCS
   file.cc
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index c047f53d6bf06..e7a3fdb1dd862 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -24,20 +24,9 @@ set(ARROW_IPC_SHARED_LINK_LIBS
   arrow_shared
 )
 
-set(ARROW_IPC_STATIC_LINK_LIBS
-  arrow_static
+set(ARROW_IPC_TEST_LINK_LIBS
   arrow_io_static
-)
-
-if (ARROW_BUILD_STATIC)
-  set(ARROW_IPC_TEST_LINK_LIBS
-    arrow_io_static
-    arrow_ipc_static)
-else()
-  set(ARROW_IPC_TEST_LINK_LIBS
-    arrow_io_shared
-    arrow_ipc_shared)
-endif()
+  arrow_ipc_static)
 
 set(ARROW_IPC_SRCS
   adapter.cc
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index a24c007a4056e..08ac9832982c1 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -51,12 +51,15 @@ namespace ipc {
 
 class RecordBatchWriter : public ArrayVisitor {
  public:
-  RecordBatchWriter(MemoryPool* pool, const RecordBatch& batch,
-      int64_t buffer_start_offset, int max_recursion_depth)
+  RecordBatchWriter(
+      MemoryPool* pool, int64_t buffer_start_offset, int max_recursion_depth)
       : pool_(pool),
-        batch_(batch),
         max_recursion_depth_(max_recursion_depth),
-        buffer_start_offset_(buffer_start_offset) {}
+        buffer_start_offset_(buffer_start_offset) {
+    DCHECK_GT(max_recursion_depth, 0);
+  }
+
+  virtual ~RecordBatchWriter() = default;
 
   Status VisitArray(const Array& arr) {
     if (max_recursion_depth_ <= 0) {
@@ -81,7 +84,7 @@ class RecordBatchWriter : public ArrayVisitor {
     return arr.Accept(this);
   }
 
-  Status Assemble(int64_t* body_length) {
+  Status Assemble(const RecordBatch& batch, int64_t* body_length) {
     if (field_nodes_.size() > 0) {
       field_nodes_.clear();
       buffer_meta_.clear();
@@ -89,8 +92,8 @@ class RecordBatchWriter : public ArrayVisitor {
     }
 
     // Perform depth-first traversal of the row-batch
-    for (int i = 0; i < batch_.num_columns(); ++i) {
-      RETURN_NOT_OK(VisitArray(*batch_.column(i)));
+    for (int i = 0; i < batch.num_columns(); ++i) {
+      RETURN_NOT_OK(VisitArray(*batch.column(i)));
     }
 
     // The position for the start of a buffer relative to the passed frame of
@@ -127,16 +130,22 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status WriteMetadata(
-      int64_t body_length, io::OutputStream* dst, int32_t* metadata_length) {
+  // Override this for writing dictionary metadata
+  virtual Status WriteMetadataMessage(
+      int32_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) {
+    return WriteRecordBatchMessage(
+        num_rows, body_length, field_nodes_, buffer_meta_, out);
+  }
+
+  Status WriteMetadata(int32_t num_rows, int64_t body_length, io::OutputStream* dst,
+      int32_t* metadata_length) {
     // Now that we have computed the locations of all of the buffers in shared
     // memory, the data header can be converted to a flatbuffer and written out
     //
     // Note: The memory written here is prefixed by the size of the flatbuffer
     // itself as an int32_t.
     std::shared_ptr<Buffer> metadata_fb;
-    RETURN_NOT_OK(WriteRecordBatchMetadata(
-        batch_.num_rows(), body_length, field_nodes_, buffer_meta_, &metadata_fb));
+    RETURN_NOT_OK(WriteMetadataMessage(num_rows, body_length, &metadata_fb));
 
     // Need to write 4 bytes (metadata size), the metadata, plus padding to
     // end on an 8-byte offset
@@ -166,15 +175,16 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status Write(io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
-    RETURN_NOT_OK(Assemble(body_length));
+  Status Write(const RecordBatch& batch, io::OutputStream* dst, int32_t* metadata_length,
+      int64_t* body_length) {
+    RETURN_NOT_OK(Assemble(batch, body_length));
 
 #ifndef NDEBUG
     int64_t start_position, current_position;
     RETURN_NOT_OK(dst->Tell(&start_position));
 #endif
 
-    RETURN_NOT_OK(WriteMetadata(*body_length, dst, metadata_length));
+    RETURN_NOT_OK(WriteMetadata(batch.num_rows(), *body_length, dst, metadata_length));
 
 #ifndef NDEBUG
     RETURN_NOT_OK(dst->Tell(&current_position));
@@ -206,17 +216,17 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status GetTotalSize(int64_t* size) {
+  Status GetTotalSize(const RecordBatch& batch, int64_t* size) {
     // emulates the behavior of Write without actually writing
     int32_t metadata_length = 0;
     int64_t body_length = 0;
     MockOutputStream dst;
-    RETURN_NOT_OK(Write(&dst, &metadata_length, &body_length));
+    RETURN_NOT_OK(Write(batch, &dst, &metadata_length, &body_length));
     *size = dst.GetExtentBytesWritten();
     return Status::OK();
   }
 
- private:
+ protected:
   Status Visit(const NullArray& array) override { return Status::NotImplemented("null"); }
 
   template <typename ArrayType>
@@ -468,15 +478,12 @@ class RecordBatchWriter : public ArrayVisitor {
   }
 
   Status Visit(const DictionaryArray& array) override {
-    // Dictionary written out separately
-    const auto& indices = static_cast<const PrimitiveArray&>(*array.indices().get());
-    buffers_.push_back(indices.data());
-    return Status::OK();
+    // Dictionary written out separately. Slice offset contained in the indices
+    return array.indices()->Accept(this);
   }
 
   // In some cases, intermediate buffers may need to be allocated (with sliced arrays)
   MemoryPool* pool_;
-  const RecordBatch& batch_;
 
   std::vector<flatbuf::FieldNode> field_nodes_;
   std::vector<flatbuf::Buffer> buffer_meta_;
@@ -486,17 +493,51 @@ class RecordBatchWriter : public ArrayVisitor {
   int64_t buffer_start_offset_;
 };
 
+class DictionaryWriter : public RecordBatchWriter {
+ public:
+  using RecordBatchWriter::RecordBatchWriter;
+
+  Status WriteMetadataMessage(
+      int32_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
+    return WriteDictionaryMessage(
+        dictionary_id_, num_rows, body_length, field_nodes_, buffer_meta_, out);
+  }
+
+  Status Write(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
+      io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
+    dictionary_id_ = dictionary_id;
+
+    // Make a dummy record batch. A bit tedious as we have to make a schema
+    std::vector<std::shared_ptr<Field>> fields = {
+        arrow::field("dictionary", dictionary->type())};
+    auto schema = std::make_shared<Schema>(fields);
+    RecordBatch batch(schema, dictionary->length(), {dictionary});
+
+    return RecordBatchWriter::Write(batch, dst, metadata_length, body_length);
+  }
+
+ private:
+  // TODO(wesm): Setting this in Write is a bit unclean, but it works
+  int64_t dictionary_id_;
+};
+
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
     MemoryPool* pool, int max_recursion_depth) {
-  DCHECK_GT(max_recursion_depth, 0);
-  RecordBatchWriter serializer(pool, batch, buffer_start_offset, max_recursion_depth);
-  return serializer.Write(dst, metadata_length, body_length);
+  RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
+  return writer.Write(batch, dst, metadata_length, body_length);
+}
+
+Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
+    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
+    int64_t* body_length, MemoryPool* pool) {
+  DictionaryWriter writer(pool, buffer_start_offset, kMaxIpcRecursionDepth);
+  return writer.Write(dictionary_id, dictionary, dst, metadata_length, body_length);
 }
 
 Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
-  RecordBatchWriter serializer(default_memory_pool(), batch, 0, kMaxIpcRecursionDepth);
-  RETURN_NOT_OK(serializer.GetTotalSize(size));
+  RecordBatchWriter writer(default_memory_pool(), 0, kMaxIpcRecursionDepth);
+  RETURN_NOT_OK(writer.GetTotalSize(batch, size));
   return Status::OK();
 }
 
@@ -580,10 +621,9 @@ class ArrayLoader : public TypeVisitor {
 
   Status LoadPrimitive(const DataType& type) {
     FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap;
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    std::shared_ptr<Buffer> null_bitmap, data;
 
-    std::shared_ptr<Buffer> data;
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
     if (field_meta.length > 0) {
       RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &data));
     } else {
@@ -597,11 +637,9 @@ class ArrayLoader : public TypeVisitor {
   template <typename CONTAINER>
   Status LoadBinary() {
     FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap;
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    std::shared_ptr<Buffer> null_bitmap, offsets, values;
 
-    std::shared_ptr<Buffer> offsets;
-    std::shared_ptr<Buffer> values;
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
     if (field_meta.length > 0) {
       RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
       RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
@@ -661,11 +699,9 @@ class ArrayLoader : public TypeVisitor {
 
   Status Visit(const ListType& type) override {
     FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap;
+    std::shared_ptr<Buffer> null_bitmap, offsets;
 
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-
-    std::shared_ptr<Buffer> offsets;
     if (field_meta.length > 0) {
       RETURN_NOT_OK(GetBuffer(context_->buffer_index, &offsets));
     } else {
@@ -715,12 +751,9 @@ class ArrayLoader : public TypeVisitor {
 
   Status Visit(const UnionType& type) override {
     FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap;
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-
-    std::shared_ptr<Buffer> type_ids = nullptr;
-    std::shared_ptr<Buffer> offsets = nullptr;
+    std::shared_ptr<Buffer> null_bitmap, type_ids, offsets;
 
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
     if (field_meta.length > 0) {
       RETURN_NOT_OK(GetBuffer(context_->buffer_index, &type_ids));
       if (type.mode == UnionMode::DENSE) {
@@ -738,13 +771,23 @@ class ArrayLoader : public TypeVisitor {
   }
 
   Status Visit(const DictionaryType& type) override {
-    return Status::NotImplemented("dictionary");
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap, indices_data;
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &indices_data));
+
+    std::shared_ptr<Array> indices;
+    RETURN_NOT_OK(MakePrimitiveArray(type.index_type(), field_meta.length, indices_data,
+        null_bitmap, field_meta.null_count, 0, &indices));
+
+    result_ = std::make_shared<DictionaryArray>(field_.type, indices);
+    return Status::OK();
   };
 };
 
 class RecordBatchReader {
  public:
-  RecordBatchReader(const std::shared_ptr<RecordBatchMetadata>& metadata,
+  RecordBatchReader(const RecordBatchMetadata& metadata,
       const std::shared_ptr<Schema>& schema, int max_recursion_depth,
       io::ReadableFileInterface* file)
       : metadata_(metadata),
@@ -758,7 +801,7 @@ class RecordBatchReader {
     // The field_index and buffer_index are incremented in the ArrayLoader
     // based on how much of the batch is "consumed" (through nested data
     // reconstruction, for example)
-    context_.metadata = metadata_.get();
+    context_.metadata = &metadata_;
     context_.field_index = 0;
     context_.buffer_index = 0;
     context_.max_recursion_depth = max_recursion_depth_;
@@ -768,50 +811,58 @@ class RecordBatchReader {
       RETURN_NOT_OK(loader.Load(&arrays[i]));
     }
 
-    *out = std::make_shared<RecordBatch>(schema_, metadata_->length(), arrays);
+    *out = std::make_shared<RecordBatch>(schema_, metadata_.length(), arrays);
     return Status::OK();
   }
 
  private:
   RecordBatchContext context_;
-  std::shared_ptr<RecordBatchMetadata> metadata_;
+  const RecordBatchMetadata& metadata_;
   std::shared_ptr<Schema> schema_;
   int max_recursion_depth_;
   io::ReadableFileInterface* file_;
 };
 
-Status ReadRecordBatchMetadata(int64_t offset, int32_t metadata_length,
-    io::ReadableFileInterface* file, std::shared_ptr<RecordBatchMetadata>* metadata) {
-  std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
-
-  int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
-
-  if (flatbuffer_size + static_cast<int>(sizeof(int32_t)) > metadata_length) {
-    std::stringstream ss;
-    ss << "flatbuffer size " << metadata_length << " invalid. File offset: " << offset
-       << ", metadata length: " << metadata_length;
-    return Status::Invalid(ss.str());
-  }
-
-  std::shared_ptr<Message> message;
-  RETURN_NOT_OK(Message::Open(buffer, 4, &message));
-  *metadata = std::make_shared<RecordBatchMetadata>(message);
-  return Status::OK();
-}
-
-Status ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
     const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
     std::shared_ptr<RecordBatch>* out) {
   return ReadRecordBatch(metadata, schema, kMaxIpcRecursionDepth, file, out);
 }
 
-Status ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
     const std::shared_ptr<Schema>& schema, int max_recursion_depth,
     io::ReadableFileInterface* file, std::shared_ptr<RecordBatch>* out) {
   RecordBatchReader reader(metadata, schema, max_recursion_depth, file);
   return reader.Read(out);
 }
 
+Status ReadDictionary(const DictionaryBatchMetadata& metadata,
+    const DictionaryTypeMap& dictionary_types, io::ReadableFileInterface* file,
+    std::shared_ptr<Array>* out) {
+  int64_t id = metadata.id();
+  auto it = dictionary_types.find(id);
+  if (it == dictionary_types.end()) {
+    std::stringstream ss;
+    ss << "Do not have type metadata for dictionary with id: " << id;
+    return Status::KeyError(ss.str());
+  }
+
+  std::vector<std::shared_ptr<Field>> fields = {it->second};
+
+  // We need a schema for the record batch
+  auto dummy_schema = std::make_shared<Schema>(fields);
+
+  // The dictionary is embedded in a record batch with a single column
+  std::shared_ptr<RecordBatch> batch;
+  RETURN_NOT_OK(ReadRecordBatch(metadata.record_batch(), dummy_schema, file, &batch));
+
+  if (batch->num_columns() != 1) {
+    return Status::Invalid("Dictionary record batch must only contain one field");
+  }
+
+  *out = batch->column(0);
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 83542d0b066d4..b7d8fa93d3651 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -25,6 +25,7 @@
 #include <memory>
 #include <vector>
 
+#include "arrow/ipc/metadata.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -44,8 +45,6 @@ class OutputStream;
 
 namespace ipc {
 
-class RecordBatchMetadata;
-
 // ----------------------------------------------------------------------
 // Write path
 // We have trouble decoding flatbuffers if the size i > 70, so 64 is a nice round number
@@ -72,34 +71,35 @@ constexpr int kMaxIpcRecursionDepth = 64;
 //
 // @param(out) body_length: the size of the contiguous buffer block plus
 // padding bytes
-Status ARROW_EXPORT WriteRecordBatch(const RecordBatch& batch,
+Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+    MemoryPool* pool, int max_recursion_depth = kMaxIpcRecursionDepth);
+
+// Write Array as a DictionaryBatch message
+Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
     int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length, MemoryPool* pool,
-    int max_recursion_depth = kMaxIpcRecursionDepth);
+    int64_t* body_length, MemoryPool* pool);
 
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the record batch. This involves generating the complete serialized
 // Flatbuffers metadata.
-Status ARROW_EXPORT GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
+Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 
 // ----------------------------------------------------------------------
 // "Read" path; does not copy data if the input supports zero copy reads
 
-// Read the record batch flatbuffer metadata starting at the indicated file offset
-//
-// The flatbuffer is expected to be length-prefixed, so the metadata_length
-// includes at least the length prefix and the flatbuffer
-Status ARROW_EXPORT ReadRecordBatchMetadata(int64_t offset, int32_t metadata_length,
-    io::ReadableFileInterface* file, std::shared_ptr<RecordBatchMetadata>* metadata);
-
-Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
     const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
     std::shared_ptr<RecordBatch>* out);
 
-Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
     const std::shared_ptr<Schema>& schema, int max_recursion_depth,
     io::ReadableFileInterface* file, std::shared_ptr<RecordBatch>* out);
 
+Status ReadDictionary(const DictionaryBatchMetadata& metadata,
+    const DictionaryTypeMap& dictionary_types, io::ReadableFileInterface* file,
+    std::shared_ptr<Array>* out);
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
index 3b1832611024f..c1d483f1fbba6 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/file.cc
@@ -36,8 +36,6 @@ namespace arrow {
 namespace ipc {
 
 static constexpr const char* kArrowMagicBytes = "ARROW1";
-// ----------------------------------------------------------------------
-// File footer
 
 static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
 FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
@@ -51,11 +49,12 @@ FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
 }
 
 Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, io::OutputStream* out) {
+    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
+    io::OutputStream* out) {
   FBB fbb;
 
   flatbuffers::Offset<flatbuf::Schema> fb_schema;
-  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, &fb_schema));
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
 
   auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
   auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
@@ -74,87 +73,6 @@ static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
   return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
 }
 
-class FileFooter::FileFooterImpl {
- public:
-  FileFooterImpl(const std::shared_ptr<Buffer>& buffer, const flatbuf::Footer* footer)
-      : buffer_(buffer), footer_(footer) {}
-
-  int num_dictionaries() const { return footer_->dictionaries()->size(); }
-
-  int num_record_batches() const { return footer_->recordBatches()->size(); }
-
-  MetadataVersion::type version() const {
-    switch (footer_->version()) {
-      case flatbuf::MetadataVersion_V1:
-        return MetadataVersion::V1;
-      case flatbuf::MetadataVersion_V2:
-        return MetadataVersion::V2;
-      // Add cases as other versions become available
-      default:
-        return MetadataVersion::V2;
-    }
-  }
-
-  FileBlock record_batch(int i) const {
-    return FileBlockFromFlatbuffer(footer_->recordBatches()->Get(i));
-  }
-
-  FileBlock dictionary(int i) const {
-    return FileBlockFromFlatbuffer(footer_->dictionaries()->Get(i));
-  }
-
-  Status GetSchema(std::shared_ptr<Schema>* out) const {
-    auto schema_msg = std::make_shared<SchemaMetadata>(nullptr, footer_->schema());
-    return schema_msg->GetSchema(out);
-  }
-
- private:
-  // Retain reference to memory
-  std::shared_ptr<Buffer> buffer_;
-
-  const flatbuf::Footer* footer_;
-};
-
-FileFooter::FileFooter() {}
-
-FileFooter::~FileFooter() {}
-
-Status FileFooter::Open(
-    const std::shared_ptr<Buffer>& buffer, std::unique_ptr<FileFooter>* out) {
-  const flatbuf::Footer* footer = flatbuf::GetFooter(buffer->data());
-
-  *out = std::unique_ptr<FileFooter>(new FileFooter());
-
-  // TODO(wesm): Verify the footer
-  (*out)->impl_.reset(new FileFooterImpl(buffer, footer));
-
-  return Status::OK();
-}
-
-int FileFooter::num_dictionaries() const {
-  return impl_->num_dictionaries();
-}
-
-int FileFooter::num_record_batches() const {
-  return impl_->num_record_batches();
-}
-
-MetadataVersion::type FileFooter::version() const {
-  return impl_->version();
-}
-
-FileBlock FileFooter::record_batch(int i) const {
-  return impl_->record_batch(i);
-}
-
-FileBlock FileFooter::dictionary(int i) const {
-  return impl_->dictionary(i);
-}
-
-Status FileFooter::GetSchema(std::shared_ptr<Schema>* out) const {
-  return impl_->GetSchema(out);
-}
-
 // ----------------------------------------------------------------------
 // File writer implementation
 
@@ -171,22 +89,17 @@ Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& s
 Status FileWriter::Start() {
   RETURN_NOT_OK(WriteAligned(
       reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
-  started_ = true;
-  return Status::OK();
-}
 
-Status FileWriter::WriteRecordBatch(const RecordBatch& batch) {
-  // Push an empty FileBlock
-  // Append metadata, to be written in the footer later
-  record_batches_.emplace_back(0, 0, 0);
-  return StreamWriter::WriteRecordBatch(
-      batch, &record_batches_[record_batches_.size() - 1]);
+  // We write the schema at the start of the file (and the end). This also
+  // writes all the dictionaries at the beginning of the file
+  return StreamWriter::Start();
 }
 
 Status FileWriter::Close() {
   // Write metadata
   int64_t initial_position = position_;
-  RETURN_NOT_OK(WriteFileFooter(*schema_, dictionaries_, record_batches_, sink_));
+  RETURN_NOT_OK(WriteFileFooter(
+      *schema_, dictionaries_, record_batches_, dictionary_memo_.get(), sink_));
   RETURN_NOT_OK(UpdatePosition());
 
   // Write footer length
@@ -204,89 +117,180 @@ Status FileWriter::Close() {
 // ----------------------------------------------------------------------
 // Reader implementation
 
-FileReader::FileReader(
-    const std::shared_ptr<io::ReadableFileInterface>& file, int64_t footer_offset)
-    : file_(file), footer_offset_(footer_offset) {}
+class FileReader::FileReaderImpl {
+ public:
+  FileReaderImpl() { dictionary_memo_ = std::make_shared<DictionaryMemo>(); }
 
-FileReader::~FileReader() {}
+  Status ReadFooter() {
+    int magic_size = static_cast<int>(strlen(kArrowMagicBytes));
 
-Status FileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
-    std::shared_ptr<FileReader>* reader) {
-  int64_t footer_offset;
-  RETURN_NOT_OK(file->GetSize(&footer_offset));
-  return Open(file, footer_offset, reader);
-}
+    if (footer_offset_ <= magic_size * 2 + 4) {
+      std::stringstream ss;
+      ss << "File is too small: " << footer_offset_;
+      return Status::Invalid(ss.str());
+    }
 
-Status FileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
-    int64_t footer_offset, std::shared_ptr<FileReader>* reader) {
-  *reader = std::shared_ptr<FileReader>(new FileReader(file, footer_offset));
-  return (*reader)->ReadFooter();
-}
+    std::shared_ptr<Buffer> buffer;
+    int file_end_size = magic_size + sizeof(int32_t);
+    RETURN_NOT_OK(file_->ReadAt(footer_offset_ - file_end_size, file_end_size, &buffer));
+
+    if (memcmp(buffer->data() + sizeof(int32_t), kArrowMagicBytes, magic_size)) {
+      return Status::Invalid("Not an Arrow file");
+    }
+
+    int32_t footer_length = *reinterpret_cast<const int32_t*>(buffer->data());
+
+    if (footer_length <= 0 || footer_length + magic_size * 2 + 4 > footer_offset_) {
+      return Status::Invalid("File is smaller than indicated metadata size");
+    }
 
-Status FileReader::ReadFooter() {
-  int magic_size = static_cast<int>(strlen(kArrowMagicBytes));
+    // Now read the footer
+    RETURN_NOT_OK(file_->ReadAt(
+        footer_offset_ - footer_length - file_end_size, footer_length, &footer_buffer_));
 
-  if (footer_offset_ <= magic_size * 2 + 4) {
-    std::stringstream ss;
-    ss << "File is too small: " << footer_offset_;
-    return Status::Invalid(ss.str());
+    // TODO(wesm): Verify the footer
+    footer_ = flatbuf::GetFooter(footer_buffer_->data());
+    schema_metadata_.reset(new SchemaMetadata(nullptr, footer_->schema()));
+
+    return Status::OK();
+  }
+
+  int num_dictionaries() const { return footer_->dictionaries()->size(); }
+
+  int num_record_batches() const { return footer_->recordBatches()->size(); }
+
+  MetadataVersion::type version() const {
+    switch (footer_->version()) {
+      case flatbuf::MetadataVersion_V1:
+        return MetadataVersion::V1;
+      case flatbuf::MetadataVersion_V2:
+        return MetadataVersion::V2;
+      // Add cases as other versions become available
+      default:
+        return MetadataVersion::V2;
+    }
+  }
+
+  FileBlock record_batch(int i) const {
+    return FileBlockFromFlatbuffer(footer_->recordBatches()->Get(i));
+  }
+
+  FileBlock dictionary(int i) const {
+    return FileBlockFromFlatbuffer(footer_->dictionaries()->Get(i));
   }
 
-  std::shared_ptr<Buffer> buffer;
-  int file_end_size = magic_size + sizeof(int32_t);
-  RETURN_NOT_OK(file_->ReadAt(footer_offset_ - file_end_size, file_end_size, &buffer));
+  const SchemaMetadata& schema_metadata() const { return *schema_metadata_; }
+
+  Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
+    DCHECK_GE(i, 0);
+    DCHECK_LT(i, num_record_batches());
+    FileBlock block = record_batch(i);
+
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(
+        ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
+    auto metadata = std::make_shared<RecordBatchMetadata>(message);
 
-  if (memcmp(buffer->data() + sizeof(int32_t), kArrowMagicBytes, magic_size)) {
-    return Status::Invalid("Not an Arrow file");
+    // TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
+    // ARROW-384).
+    std::shared_ptr<Buffer> buffer_block;
+    RETURN_NOT_OK(file_->Read(block.body_length, &buffer_block));
+    io::BufferReader reader(buffer_block);
+
+    return ReadRecordBatch(*metadata, schema_, &reader, batch);
   }
 
-  int32_t footer_length = *reinterpret_cast<const int32_t*>(buffer->data());
+  Status ReadSchema() {
+    RETURN_NOT_OK(schema_metadata_->GetDictionaryTypes(&dictionary_fields_));
+
+    // Read all the dictionaries
+    for (int i = 0; i < num_dictionaries(); ++i) {
+      FileBlock block = dictionary(i);
+      std::shared_ptr<Message> message;
+      RETURN_NOT_OK(
+          ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
+
+      // TODO(wesm): ARROW-577: This code is duplicated, can be fixed with a more
+      // invasive refactor
+      DictionaryBatchMetadata metadata(message);
+
+      // TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
+      // ARROW-384).
+      std::shared_ptr<Buffer> buffer_block;
+      RETURN_NOT_OK(file_->Read(block.body_length, &buffer_block));
+      io::BufferReader reader(buffer_block);
+
+      std::shared_ptr<Array> dictionary;
+      RETURN_NOT_OK(ReadDictionary(metadata, dictionary_fields_, &reader, &dictionary));
+      RETURN_NOT_OK(dictionary_memo_->AddDictionary(metadata.id(), dictionary));
+    }
 
-  if (footer_length <= 0 || footer_length + magic_size * 2 + 4 > footer_offset_) {
-    return Status::Invalid("File is smaller than indicated metadata size");
+    // Get the schema
+    return schema_metadata_->GetSchema(*dictionary_memo_, &schema_);
   }
 
-  // Now read the footer
-  RETURN_NOT_OK(file_->ReadAt(
-      footer_offset_ - footer_length - file_end_size, footer_length, &buffer));
-  RETURN_NOT_OK(FileFooter::Open(buffer, &footer_));
+  Status Open(
+      const std::shared_ptr<io::ReadableFileInterface>& file, int64_t footer_offset) {
+    file_ = file;
+    footer_offset_ = footer_offset;
+    RETURN_NOT_OK(ReadFooter());
+    return ReadSchema();
+  }
+
+  std::shared_ptr<Schema> schema() const { return schema_; }
+
+ private:
+  std::shared_ptr<io::ReadableFileInterface> file_;
 
-  // Get the schema
-  return footer_->GetSchema(&schema_);
+  // The location where the Arrow file layout ends. May be the end of the file
+  // or some other location if embedded in a larger file.
+  int64_t footer_offset_;
+
+  // Footer metadata
+  std::shared_ptr<Buffer> footer_buffer_;
+  const flatbuf::Footer* footer_;
+  std::unique_ptr<SchemaMetadata> schema_metadata_;
+
+  DictionaryTypeMap dictionary_fields_;
+  std::shared_ptr<DictionaryMemo> dictionary_memo_;
+
+  // Reconstructed schema, including any read dictionaries
+  std::shared_ptr<Schema> schema_;
+};
+
+FileReader::FileReader() {
+  impl_.reset(new FileReaderImpl());
 }
 
-std::shared_ptr<Schema> FileReader::schema() const {
-  return schema_;
+FileReader::~FileReader() {}
+
+Status FileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+    std::shared_ptr<FileReader>* reader) {
+  int64_t footer_offset;
+  RETURN_NOT_OK(file->GetSize(&footer_offset));
+  return Open(file, footer_offset, reader);
+}
+
+Status FileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+    int64_t footer_offset, std::shared_ptr<FileReader>* reader) {
+  *reader = std::shared_ptr<FileReader>(new FileReader());
+  return (*reader)->impl_->Open(file, footer_offset);
 }
 
-int FileReader::num_dictionaries() const {
-  return footer_->num_dictionaries();
+std::shared_ptr<Schema> FileReader::schema() const {
+  return impl_->schema();
 }
 
 int FileReader::num_record_batches() const {
-  return footer_->num_record_batches();
+  return impl_->num_record_batches();
 }
 
 MetadataVersion::type FileReader::version() const {
-  return footer_->version();
+  return impl_->version();
 }
 
 Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
-  DCHECK_GE(i, 0);
-  DCHECK_LT(i, num_record_batches());
-  FileBlock block = footer_->record_batch(i);
-
-  std::shared_ptr<RecordBatchMetadata> metadata;
-  RETURN_NOT_OK(ReadRecordBatchMetadata(
-      block.offset, block.metadata_length, file_.get(), &metadata));
-
-  // TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
-  // ARROW-384).
-  std::shared_ptr<Buffer> buffer_block;
-  RETURN_NOT_OK(file_->Read(block.body_length, &buffer_block));
-  io::BufferReader reader(buffer_block);
-
-  return ReadRecordBatch(metadata, schema_, &reader, batch);
+  return impl_->GetRecordBatch(i, batch);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/file.h b/cpp/src/arrow/ipc/file.h
index cf0baab820eef..524766ccb3336 100644
--- a/cpp/src/arrow/ipc/file.h
+++ b/cpp/src/arrow/ipc/file.h
@@ -45,45 +45,21 @@ class ReadableFileInterface;
 namespace ipc {
 
 Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, io::OutputStream* out);
-
-class ARROW_EXPORT FileFooter {
- public:
-  ~FileFooter();
-
-  static Status Open(
-      const std::shared_ptr<Buffer>& buffer, std::unique_ptr<FileFooter>* out);
-
-  int num_dictionaries() const;
-  int num_record_batches() const;
-  MetadataVersion::type version() const;
-
-  FileBlock record_batch(int i) const;
-  FileBlock dictionary(int i) const;
-
-  Status GetSchema(std::shared_ptr<Schema>* out) const;
-
- private:
-  FileFooter();
-  class FileFooterImpl;
-  std::unique_ptr<FileFooterImpl> impl_;
-};
+    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
+    io::OutputStream* out);
 
 class ARROW_EXPORT FileWriter : public StreamWriter {
  public:
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
       std::shared_ptr<FileWriter>* out);
 
-  Status WriteRecordBatch(const RecordBatch& batch) override;
+  using StreamWriter::WriteRecordBatch;
   Status Close() override;
 
  private:
   FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema);
 
   Status Start() override;
-
-  std::vector<FileBlock> dictionaries_;
-  std::vector<FileBlock> record_batches_;
 };
 
 class ARROW_EXPORT FileReader {
@@ -108,13 +84,9 @@ class ARROW_EXPORT FileReader {
   static Status Open(const std::shared_ptr<io::ReadableFileInterface>& file,
       int64_t footer_offset, std::shared_ptr<FileReader>* reader);
 
+  /// The schema includes any dictionaries
   std::shared_ptr<Schema> schema() const;
 
-  // Shared dictionaries for dictionary-encoding cross record batches
-  // TODO(wesm): Implement dictionary reading when we also have dictionary
-  // encoding
-  int num_dictionaries() const;
-
   int num_record_batches() const;
 
   MetadataVersion::type version() const;
@@ -127,19 +99,10 @@ class ARROW_EXPORT FileReader {
   Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch);
 
  private:
-  FileReader(
-      const std::shared_ptr<io::ReadableFileInterface>& file, int64_t footer_offset);
-
-  Status ReadFooter();
-
-  std::shared_ptr<io::ReadableFileInterface> file_;
-
-  // The location where the Arrow file layout ends. May be the end of the file
-  // or some other location if embedded in a larger file.
-  int64_t footer_offset_;
+  FileReader();
 
-  std::unique_ptr<FileFooter> footer_;
-  std::shared_ptr<Schema> schema_;
+  class ARROW_NO_EXPORT FileReaderImpl;
+  std::unique_ptr<FileReaderImpl> impl_;
 };
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index d11b95b167d21..8999363893289 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -27,6 +27,7 @@
 #include "arrow/io/memory.h"
 #include "arrow/io/test-common.h"
 #include "arrow/ipc/adapter.h"
+#include "arrow/ipc/metadata.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/ipc/util.h"
 
@@ -40,12 +41,8 @@
 namespace arrow {
 namespace ipc {
 
-class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
-                             public io::MemoryMapFixture {
+class IpcTestFixture : public io::MemoryMapFixture {
  public:
-  void SetUp() { pool_ = default_memory_pool(); }
-  void TearDown() { io::MemoryMapFixture::TearDown(); }
-
   Status RoundTripHelper(const RecordBatch& batch, int memory_map_size,
       std::shared_ptr<RecordBatch>* batch_result) {
     std::string path = "test-write-row-batch";
@@ -59,8 +56,9 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
     RETURN_NOT_OK(WriteRecordBatch(
         batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
 
-    std::shared_ptr<RecordBatchMetadata> metadata;
-    RETURN_NOT_OK(ReadRecordBatchMetadata(0, metadata_length, mmap_.get(), &metadata));
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+    auto metadata = std::make_shared<RecordBatchMetadata>(message);
 
     // The buffer offsets start at 0, so we must construct a
     // ReadableFileInterface according to that frame of reference
@@ -68,7 +66,7 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
     RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
     io::BufferReader buffer_reader(buffer_payload);
 
-    return ReadRecordBatch(metadata, batch.schema(), &buffer_reader, batch_result);
+    return ReadRecordBatch(*metadata, batch.schema(), &buffer_reader, batch_result);
   }
 
   void CheckRoundtrip(const RecordBatch& batch, int64_t buffer_size) {
@@ -112,14 +110,29 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
   MemoryPool* pool_;
 };
 
-TEST_P(TestWriteRecordBatch, RoundTrip) {
+class TestWriteRecordBatch : public ::testing::Test, public IpcTestFixture {
+ public:
+  void SetUp() { pool_ = default_memory_pool(); }
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
+};
+
+class TestRecordBatchParam : public ::testing::TestWithParam<MakeRecordBatch*>,
+                             public IpcTestFixture {
+ public:
+  void SetUp() { pool_ = default_memory_pool(); }
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
+  using IpcTestFixture::RoundTripHelper;
+  using IpcTestFixture::CheckRoundtrip;
+};
+
+TEST_P(TestRecordBatchParam, RoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
 
   CheckRoundtrip(*batch, 1 << 20);
 }
 
-TEST_P(TestWriteRecordBatch, SliceRoundTrip) {
+TEST_P(TestRecordBatchParam, SliceRoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
 
@@ -130,7 +143,7 @@ TEST_P(TestWriteRecordBatch, SliceRoundTrip) {
   CheckRoundtrip(*sliced_batch, 1 << 20);
 }
 
-TEST_P(TestWriteRecordBatch, ZeroLengthArrays) {
+TEST_P(TestRecordBatchParam, ZeroLengthArrays) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
 
@@ -159,10 +172,10 @@ TEST_P(TestWriteRecordBatch, ZeroLengthArrays) {
 }
 
 INSTANTIATE_TEST_CASE_P(
-    RoundTripTests, TestWriteRecordBatch,
+    RoundTripTests, TestRecordBatchParam,
     ::testing::Values(&MakeIntRecordBatch, &MakeStringTypesRecordBatch,
         &MakeNonNullRecordBatch, &MakeZeroLengthRecordBatch, &MakeListRecordBatch,
-        &MakeDeeplyNestedList, &MakeStruct, &MakeUnion));
+        &MakeDeeplyNestedList, &MakeStruct, &MakeUnion, &MakeDictionary));
 
 void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   ipc::MockOutputStream mock;
@@ -251,8 +264,9 @@ TEST_F(RecursionLimits, ReadLimit) {
   std::shared_ptr<Schema> schema;
   ASSERT_OK(WriteToMmap(64, true, &metadata_length, &body_length, &schema));
 
-  std::shared_ptr<RecordBatchMetadata> metadata;
-  ASSERT_OK(ReadRecordBatchMetadata(0, metadata_length, mmap_.get(), &metadata));
+  std::shared_ptr<Message> message;
+  ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+  auto metadata = std::make_shared<RecordBatchMetadata>(message);
 
   std::shared_ptr<Buffer> payload;
   ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
@@ -260,7 +274,7 @@ TEST_F(RecursionLimits, ReadLimit) {
   io::BufferReader reader(payload);
 
   std::shared_ptr<RecordBatch> batch;
-  ASSERT_RAISES(Invalid, ReadRecordBatch(metadata, schema, &reader, &batch));
+  ASSERT_RAISES(Invalid, ReadRecordBatch(*metadata, schema, &reader, &batch));
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index 7cd8054679e44..4b82aab0e3978 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -180,72 +180,44 @@ TEST_P(TestStreamFormat, RoundTrip) {
 #define BATCH_CASES()                                                                   \
   ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
       &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
-      &MakeStruct);
+      &MakeStruct, &MakeDictionary);
 
 INSTANTIATE_TEST_CASE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
 INSTANTIATE_TEST_CASE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
 
-class TestFileFooter : public ::testing::Test {
- public:
-  void SetUp() {}
-
-  void CheckRoundtrip(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-      const std::vector<FileBlock>& record_batches) {
-    auto buffer = std::make_shared<PoolBuffer>();
-    io::BufferOutputStream stream(buffer);
-
-    ASSERT_OK(WriteFileFooter(schema, dictionaries, record_batches, &stream));
-
-    std::unique_ptr<FileFooter> footer;
-    ASSERT_OK(FileFooter::Open(buffer, &footer));
-
-    ASSERT_EQ(MetadataVersion::V2, footer->version());
+void CheckBatchDictionaries(const RecordBatch& batch) {
+  // Check that dictionaries that should be the same are the same
+  auto schema = batch.schema();
 
-    // Check schema
-    std::shared_ptr<Schema> schema2;
-    ASSERT_OK(footer->GetSchema(&schema2));
-    AssertSchemaEqual(schema, *schema2);
+  const auto& t0 = static_cast<const DictionaryType&>(*schema->field(0)->type);
+  const auto& t1 = static_cast<const DictionaryType&>(*schema->field(1)->type);
 
-    // Check blocks
-    ASSERT_EQ(dictionaries.size(), footer->num_dictionaries());
-    ASSERT_EQ(record_batches.size(), footer->num_record_batches());
+  ASSERT_EQ(t0.dictionary().get(), t1.dictionary().get());
 
-    for (int i = 0; i < footer->num_dictionaries(); ++i) {
-      CheckBlocks(dictionaries[i], footer->dictionary(i));
-    }
-
-    for (int i = 0; i < footer->num_record_batches(); ++i) {
-      CheckBlocks(record_batches[i], footer->record_batch(i));
-    }
-  }
+  // Same dictionary used for list values
+  const auto& t3 = static_cast<const ListType&>(*schema->field(3)->type);
+  const auto& t3_value = static_cast<const DictionaryType&>(*t3.value_type());
+  ASSERT_EQ(t0.dictionary().get(), t3_value.dictionary().get());
+}
 
-  void CheckBlocks(const FileBlock& left, const FileBlock& right) {
-    ASSERT_EQ(left.offset, right.offset);
-    ASSERT_EQ(left.metadata_length, right.metadata_length);
-    ASSERT_EQ(left.body_length, right.body_length);
-  }
+TEST_F(TestStreamFormat, DictionaryRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeDictionary(&batch));
 
- private:
-  std::shared_ptr<Schema> example_schema_;
-};
+  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+  ASSERT_OK(RoundTripHelper(*batch, &out_batches));
 
-TEST_F(TestFileFooter, Basics) {
-  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>());
-  Schema schema({f0, f1});
+  CheckBatchDictionaries(*out_batches[0]);
+}
 
-  std::vector<FileBlock> dictionaries;
-  dictionaries.emplace_back(8, 92, 900);
-  dictionaries.emplace_back(1000, 100, 1900);
-  dictionaries.emplace_back(3000, 100, 2900);
+TEST_F(TestFileFormat, DictionaryRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeDictionary(&batch));
 
-  std::vector<FileBlock> record_batches;
-  record_batches.emplace_back(6000, 100, 900);
-  record_batches.emplace_back(7000, 100, 1900);
-  record_batches.emplace_back(9000, 100, 2900);
-  record_batches.emplace_back(12000, 100, 3900);
+  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+  ASSERT_OK(RoundTripHelper({batch}, &out_batches));
 
-  CheckRoundtrip(schema, dictionaries, record_batches);
+  CheckBatchDictionaries(*out_batches[0]);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
index 098f996d292a2..4fb3204a5b6d2 100644
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -22,6 +22,7 @@
 #include "gtest/gtest.h"
 
 #include "arrow/io/memory.h"
+#include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/schema.h"
@@ -39,9 +40,9 @@ class TestSchemaMetadata : public ::testing::Test {
  public:
   void SetUp() {}
 
-  void CheckRoundtrip(const Schema& schema) {
+  void CheckRoundtrip(const Schema& schema, DictionaryMemo* memo) {
     std::shared_ptr<Buffer> buffer;
-    ASSERT_OK(WriteSchema(schema, &buffer));
+    ASSERT_OK(WriteSchemaMessage(schema, memo, &buffer));
 
     std::shared_ptr<Message> message;
     ASSERT_OK(Message::Open(buffer, 0, &message));
@@ -51,8 +52,10 @@ class TestSchemaMetadata : public ::testing::Test {
     auto schema_msg = std::make_shared<SchemaMetadata>(message);
     ASSERT_EQ(schema.num_fields(), schema_msg->num_fields());
 
+    DictionaryMemo empty_memo;
+
     std::shared_ptr<Schema> schema2;
-    ASSERT_OK(schema_msg->GetSchema(&schema2));
+    ASSERT_OK(schema_msg->GetSchema(empty_memo, &schema2));
 
     AssertSchemaEqual(schema, *schema2);
   }
@@ -74,7 +77,9 @@ TEST_F(TestSchemaMetadata, PrimitiveFields) {
   auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
 
   Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
-  CheckRoundtrip(schema);
+  DictionaryMemo memo;
+
+  CheckRoundtrip(schema, &memo);
 }
 
 TEST_F(TestSchemaMetadata, NestedFields) {
@@ -86,7 +91,9 @@ TEST_F(TestSchemaMetadata, NestedFields) {
   auto f1 = std::make_shared<Field>("f1", type2);
 
   Schema schema({f0, f1});
-  CheckRoundtrip(schema);
+  DictionaryMemo memo;
+
+  CheckRoundtrip(schema, &memo);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index cd7722056a3c7..7c8ddb93c09d1 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -25,6 +25,7 @@
 
 #include "flatbuffers/flatbuffers.h"
 
+#include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/schema.h"
@@ -115,8 +116,8 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 }
 
 // Forward declaration
-static Status FieldToFlatbuffer(
-    FBB& fbb, const std::shared_ptr<Field>& field, FieldOffset* offset);
+static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+    DictionaryMemo* dictionary_memo, FieldOffset* offset);
 
 static Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
   return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
@@ -126,34 +127,73 @@ static Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
   return flatbuf::CreateFloatingPoint(fbb, precision).Union();
 }
 
-static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, Offset* offset) {
+static Status AppendChildFields(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo) {
   FieldOffset field;
-  RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(0), &field));
-  out_children->push_back(field);
+  for (int i = 0; i < type->num_children(); ++i) {
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), dictionary_memo, &field));
+    out_children->push_back(field);
+  }
+  return Status::OK();
+}
+
+static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+    Offset* offset) {
+  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
   *offset = flatbuf::CreateList(fbb).Union();
   return Status::OK();
 }
 
 static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, Offset* offset) {
-  FieldOffset field;
-  for (int i = 0; i < type->num_children(); ++i) {
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), &field));
-    out_children->push_back(field);
-  }
+    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+    Offset* offset) {
+  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
   *offset = flatbuf::CreateStruct_(fbb).Union();
   return Status::OK();
 }
 
+static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+    Offset* offset) {
+  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
+
+  const auto& union_type = static_cast<const UnionType&>(*type);
+
+  flatbuf::UnionMode mode = union_type.mode == UnionMode::SPARSE
+                                ? flatbuf::UnionMode_Sparse
+                                : flatbuf::UnionMode_Dense;
+
+  std::vector<int32_t> type_ids;
+  type_ids.reserve(union_type.type_codes.size());
+  for (uint8_t code : union_type.type_codes) {
+    type_ids.push_back(code);
+  }
+
+  auto fb_type_ids = fbb.CreateVector(type_ids);
+
+  *offset = flatbuf::CreateUnion(fbb, mode, fb_type_ids).Union();
+  return Status::OK();
+}
+
 #define INT_TO_FB_CASE(BIT_WIDTH, IS_SIGNED)            \
   *out_type = flatbuf::Type_Int;                        \
   *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
   break;
 
+// TODO(wesm): Convert this to visitor pattern
 static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
-    flatbuf::Type* out_type, Offset* offset) {
+    flatbuf::Type* out_type, DictionaryMemo* dictionary_memo, Offset* offset) {
+  if (type->type == Type::DICTIONARY) {
+    // In this library, the dictionary "type" is a logical construct. Here we
+    // pass through to the value type, as we've already captured the index
+    // type in the DictionaryEncoding metadata in the parent field
+    const auto& dict_type = static_cast<const DictionaryType&>(*type);
+    return TypeToFlatbuffer(fbb, dict_type.dictionary()->type(), children, layout,
+        out_type, dictionary_memo, offset);
+  }
+
   std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
   for (const BufferDescr& descr : buffer_layout) {
     flatbuf::VectorType vector_type;
@@ -217,10 +257,13 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       break;
     case Type::LIST:
       *out_type = flatbuf::Type_List;
-      return ListToFlatbuffer(fbb, type, children, offset);
+      return ListToFlatbuffer(fbb, type, children, dictionary_memo, offset);
     case Type::STRUCT:
       *out_type = flatbuf::Type_Struct_;
-      return StructToFlatbuffer(fbb, type, children, offset);
+      return StructToFlatbuffer(fbb, type, children, dictionary_memo, offset);
+    case Type::UNION:
+      *out_type = flatbuf::Type_Union;
+      return UnionToFlatBuffer(fbb, type, children, dictionary_memo, offset);
     default:
       *out_type = flatbuf::Type_NONE;  // Make clang-tidy happy
       std::stringstream ss;
@@ -230,35 +273,63 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
   return Status::OK();
 }
 
-static Status FieldToFlatbuffer(
-    FBB& fbb, const std::shared_ptr<Field>& field, FieldOffset* offset) {
+using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
+
+static DictionaryOffset GetDictionaryEncoding(
+    FBB& fbb, const DictionaryType& type, DictionaryMemo* memo) {
+  int64_t dictionary_id = memo->GetId(type.dictionary());
+
+  // We assume that the dictionary index type (as an integer) has already been
+  // validated elsewhere, and can safely assume we are dealing with signed
+  // integers
+  const auto& fw_index_type = static_cast<const FixedWidthType&>(*type.index_type());
+
+  auto index_type_offset = flatbuf::CreateInt(fbb, fw_index_type.bit_width(), true);
+
+  // TODO(wesm): ordered dictionaries
+  return flatbuf::CreateDictionaryEncoding(fbb, dictionary_id, index_type_offset);
+}
+
+static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+    DictionaryMemo* dictionary_memo, FieldOffset* offset) {
   auto fb_name = fbb.CreateString(field->name);
 
   flatbuf::Type type_enum;
-  Offset type_data;
+  Offset type_offset;
   Offset type_layout;
   std::vector<FieldOffset> children;
   std::vector<VectorLayoutOffset> layout;
 
-  RETURN_NOT_OK(
-      TypeToFlatbuffer(fbb, field->type, &children, &layout, &type_enum, &type_data));
+  RETURN_NOT_OK(TypeToFlatbuffer(
+      fbb, field->type, &children, &layout, &type_enum, dictionary_memo, &type_offset));
   auto fb_children = fbb.CreateVector(children);
   auto fb_layout = fbb.CreateVector(layout);
 
+  DictionaryOffset dictionary = 0;
+  if (field->type->type == Type::DICTIONARY) {
+    dictionary = GetDictionaryEncoding(
+        fbb, static_cast<const DictionaryType&>(*field->type), dictionary_memo);
+  }
+
   // TODO: produce the list of VectorTypes
-  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_data,
-      field->dictionary, fb_children, fb_layout);
+  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_offset,
+      dictionary, fb_children, fb_layout);
 
   return Status::OK();
 }
 
-Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out) {
-  std::shared_ptr<DataType> type;
+Status FieldFromFlatbufferDictionary(
+    const flatbuf::Field* field, std::shared_ptr<Field>* out) {
+  // Need an empty memo to pass down for constructing children
+  DictionaryMemo dummy_memo;
+
+  // Any DictionaryEncoding set is ignored here
 
+  std::shared_ptr<DataType> type;
   auto children = field->children();
   std::vector<std::shared_ptr<Field>> child_fields(children->size());
   for (size_t i = 0; i < children->size(); ++i) {
-    RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), &child_fields[i]));
+    RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), dummy_memo, &child_fields[i]));
   }
 
   RETURN_NOT_OK(
@@ -268,6 +339,39 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>*
   return Status::OK();
 }
 
+Status FieldFromFlatbuffer(const flatbuf::Field* field,
+    const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out) {
+  std::shared_ptr<DataType> type;
+
+  const flatbuf::DictionaryEncoding* encoding = field->dictionary();
+
+  if (encoding == nullptr) {
+    // The field is not dictionary encoded. We must potentially visit its
+    // children to fully reconstruct the data type
+    auto children = field->children();
+    std::vector<std::shared_ptr<Field>> child_fields(children->size());
+    for (size_t i = 0; i < children->size(); ++i) {
+      RETURN_NOT_OK(
+          FieldFromFlatbuffer(children->Get(i), dictionary_memo, &child_fields[i]));
+    }
+    RETURN_NOT_OK(
+        TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
+  } else {
+    // The field is dictionary encoded. The type of the dictionary values has
+    // been determined elsewhere, and is stored in the DictionaryMemo. Here we
+    // construct the logical DictionaryType object
+
+    std::shared_ptr<Array> dictionary;
+    RETURN_NOT_OK(dictionary_memo.GetDictionary(encoding->id(), &dictionary));
+
+    std::shared_ptr<DataType> index_type;
+    RETURN_NOT_OK(IntFromFlatbuffer(encoding->indexType(), &index_type));
+    type = std::make_shared<DictionaryType>(index_type, dictionary);
+  }
+  *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
+  return Status::OK();
+}
+
 // Implement MessageBuilder
 
 // will return the endianness of the system we are running on
@@ -281,13 +385,13 @@ flatbuf::Endianness endianness() {
   return bint.c[0] == 1 ? flatbuf::Endianness_Big : flatbuf::Endianness_Little;
 }
 
-Status SchemaToFlatbuffer(
-    FBB& fbb, const Schema& schema, flatbuffers::Offset<flatbuf::Schema>* out) {
+Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema, DictionaryMemo* dictionary_memo,
+    flatbuffers::Offset<flatbuf::Schema>* out) {
   std::vector<FieldOffset> field_offsets;
   for (int i = 0; i < schema.num_fields(); ++i) {
     std::shared_ptr<Field> field = schema.field(i);
     FieldOffset offset;
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, &offset));
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, dictionary_memo, &offset));
     field_offsets.push_back(offset);
   }
 
@@ -295,29 +399,63 @@ Status SchemaToFlatbuffer(
   return Status::OK();
 }
 
-Status MessageBuilder::SetSchema(const Schema& schema) {
-  flatbuffers::Offset<flatbuf::Schema> fb_schema;
-  RETURN_NOT_OK(SchemaToFlatbuffer(fbb_, schema, &fb_schema));
+class MessageBuilder {
+ public:
+  Status SetSchema(const Schema& schema, DictionaryMemo* dictionary_memo) {
+    flatbuffers::Offset<flatbuf::Schema> fb_schema;
+    RETURN_NOT_OK(SchemaToFlatbuffer(fbb_, schema, dictionary_memo, &fb_schema));
 
-  header_type_ = flatbuf::MessageHeader_Schema;
-  header_ = fb_schema.Union();
-  body_length_ = 0;
-  return Status::OK();
-}
+    header_type_ = flatbuf::MessageHeader_Schema;
+    header_ = fb_schema.Union();
+    body_length_ = 0;
+    return Status::OK();
+  }
 
-Status MessageBuilder::SetRecordBatch(int32_t length, int64_t body_length,
-    const std::vector<flatbuf::FieldNode>& nodes,
-    const std::vector<flatbuf::Buffer>& buffers) {
-  header_type_ = flatbuf::MessageHeader_RecordBatch;
-  header_ = flatbuf::CreateRecordBatch(fbb_, length, fbb_.CreateVectorOfStructs(nodes),
-                fbb_.CreateVectorOfStructs(buffers))
-                .Union();
-  body_length_ = body_length;
+  Status SetRecordBatch(int32_t length, int64_t body_length,
+      const std::vector<flatbuf::FieldNode>& nodes,
+      const std::vector<flatbuf::Buffer>& buffers) {
+    header_type_ = flatbuf::MessageHeader_RecordBatch;
+    header_ = flatbuf::CreateRecordBatch(fbb_, length, fbb_.CreateVectorOfStructs(nodes),
+                  fbb_.CreateVectorOfStructs(buffers))
+                  .Union();
+    body_length_ = body_length;
 
-  return Status::OK();
+    return Status::OK();
+  }
+
+  Status SetDictionary(int64_t id, int32_t length, int64_t body_length,
+      const std::vector<flatbuf::FieldNode>& nodes,
+      const std::vector<flatbuf::Buffer>& buffers) {
+    header_type_ = flatbuf::MessageHeader_DictionaryBatch;
+
+    auto record_batch = flatbuf::CreateRecordBatch(fbb_, length,
+        fbb_.CreateVectorOfStructs(nodes), fbb_.CreateVectorOfStructs(buffers));
+
+    header_ = flatbuf::CreateDictionaryBatch(fbb_, id, record_batch).Union();
+    body_length_ = body_length;
+    return Status::OK();
+  }
+
+  Status Finish();
+
+  Status GetBuffer(std::shared_ptr<Buffer>* out);
+
+ private:
+  flatbuf::MessageHeader header_type_;
+  flatbuffers::Offset<void> header_;
+  int64_t body_length_;
+  flatbuffers::FlatBufferBuilder fbb_;
+};
+
+Status WriteSchemaMessage(
+    const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out) {
+  MessageBuilder message;
+  RETURN_NOT_OK(message.SetSchema(schema, dictionary_memo));
+  RETURN_NOT_OK(message.Finish());
+  return message.GetBuffer(out);
 }
 
-Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
+Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
     const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
   MessageBuilder builder;
@@ -326,6 +464,15 @@ Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
   return builder.GetBuffer(out);
 }
 
+Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
+    const std::vector<flatbuf::FieldNode>& nodes,
+    const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
+  MessageBuilder builder;
+  RETURN_NOT_OK(builder.SetDictionary(id, length, body_length, nodes, buffers));
+  RETURN_NOT_OK(builder.Finish());
+  return builder.GetBuffer(out);
+}
+
 Status MessageBuilder::Finish() {
   auto message =
       flatbuf::CreateMessage(fbb_, kMetadataVersion, header_type_, header_, body_length_);
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index d94a8abc99ab0..59afecbcbd27e 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -46,31 +46,34 @@ using Offset = flatbuffers::Offset<void>;
 
 static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
 
-Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out);
+// Construct a field with type for a dictionary-encoded field. None of its
+// children or children's descendents can be dictionary encoded
+Status FieldFromFlatbufferDictionary(
+    const flatbuf::Field* field, std::shared_ptr<Field>* out);
 
-Status SchemaToFlatbuffer(
-    FBB& fbb, const Schema& schema, flatbuffers::Offset<flatbuf::Schema>* out);
+// Construct a field for a non-dictionary-encoded field. Its children may be
+// dictionary encoded
+Status FieldFromFlatbuffer(const flatbuf::Field* field,
+    const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out);
 
-class MessageBuilder {
- public:
-  Status SetSchema(const Schema& schema);
+Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema, DictionaryMemo* dictionary_memo,
+    flatbuffers::Offset<flatbuf::Schema>* out);
 
-  Status SetRecordBatch(int32_t length, int64_t body_length,
-      const std::vector<flatbuf::FieldNode>& nodes,
-      const std::vector<flatbuf::Buffer>& buffers);
-
-  Status Finish();
-
-  Status GetBuffer(std::shared_ptr<Buffer>* out);
-
- private:
-  flatbuf::MessageHeader header_type_;
-  flatbuffers::Offset<void> header_;
-  int64_t body_length_;
-  flatbuffers::FlatBufferBuilder fbb_;
-};
+// Serialize arrow::Schema as a Flatbuffer
+//
+// \param[in] schema a Schema instance
+// \param[inout] dictionary_memo class for tracking dictionaries and assigning
+// dictionary ids
+// \param[out] out the serialized arrow::Buffer
+// \return Status outcome
+Status WriteSchemaMessage(
+    const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out);
+
+Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
+    const std::vector<flatbuf::FieldNode>& nodes,
+    const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
 
-Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
+Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
     const std::vector<flatbuf::FieldNode>& nodes,
     const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index a97965c40d608..2ba44ac618ce3 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -19,6 +19,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <sstream>
 #include <vector>
 
 #include "flatbuffers/flatbuffers.h"
@@ -38,11 +39,60 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
-Status WriteSchema(const Schema& schema, std::shared_ptr<Buffer>* out) {
-  MessageBuilder message;
-  RETURN_NOT_OK(message.SetSchema(schema));
-  RETURN_NOT_OK(message.Finish());
-  return message.GetBuffer(out);
+// ----------------------------------------------------------------------
+// Memoization data structure for handling shared dictionaries
+
+DictionaryMemo::DictionaryMemo() {}
+
+// Returns KeyError if dictionary not found
+Status DictionaryMemo::GetDictionary(
+    int64_t id, std::shared_ptr<Array>* dictionary) const {
+  auto it = id_to_dictionary_.find(id);
+  if (it == id_to_dictionary_.end()) {
+    std::stringstream ss;
+    ss << "Dictionary with id " << id << " not found";
+    return Status::KeyError(ss.str());
+  }
+  *dictionary = it->second;
+  return Status::OK();
+}
+
+int64_t DictionaryMemo::GetId(const std::shared_ptr<Array>& dictionary) {
+  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
+  auto it = dictionary_to_id_.find(address);
+  if (it != dictionary_to_id_.end()) {
+    // Dictionary already observed, return the id
+    return it->second;
+  } else {
+    int64_t new_id = static_cast<int64_t>(dictionary_to_id_.size()) + 1;
+    dictionary_to_id_[address] = new_id;
+    id_to_dictionary_[new_id] = dictionary;
+    return new_id;
+  }
+}
+
+bool DictionaryMemo::HasDictionary(const std::shared_ptr<Array>& dictionary) const {
+  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
+  auto it = dictionary_to_id_.find(address);
+  return it != dictionary_to_id_.end();
+}
+
+bool DictionaryMemo::HasDictionaryId(int64_t id) const {
+  auto it = id_to_dictionary_.find(id);
+  return it != id_to_dictionary_.end();
+}
+
+Status DictionaryMemo::AddDictionary(
+    int64_t id, const std::shared_ptr<Array>& dictionary) {
+  if (HasDictionaryId(id)) {
+    std::stringstream ss;
+    ss << "Dictionary with id " << id << " already exists";
+    return Status::KeyError(ss.str());
+  }
+  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
+  id_to_dictionary_[id] = dictionary;
+  dictionary_to_id_[address] = id;
+  return Status::OK();
 }
 
 //----------------------------------------------------------------------
@@ -113,10 +163,35 @@ class SchemaMetadata::SchemaMetadataImpl {
   explicit SchemaMetadataImpl(const void* schema)
       : schema_(static_cast<const flatbuf::Schema*>(schema)) {}
 
-  const flatbuf::Field* field(int i) const { return schema_->fields()->Get(i); }
+  const flatbuf::Field* get_field(int i) const { return schema_->fields()->Get(i); }
 
   int num_fields() const { return schema_->fields()->size(); }
 
+  Status VisitField(const flatbuf::Field* field, DictionaryTypeMap* id_to_field) const {
+    const flatbuf::DictionaryEncoding* dict_metadata = field->dictionary();
+    if (dict_metadata == nullptr) {
+      // Field is not dictionary encoded. Visit children
+      auto children = field->children();
+      for (flatbuffers::uoffset_t i = 0; i < children->size(); ++i) {
+        RETURN_NOT_OK(VisitField(children->Get(i), id_to_field));
+      }
+    } else {
+      // Field is dictionary encoded. Construct the data type for the
+      // dictionary (no descendents can be dictionary encoded)
+      std::shared_ptr<Field> dictionary_field;
+      RETURN_NOT_OK(FieldFromFlatbufferDictionary(field, &dictionary_field));
+      (*id_to_field)[dict_metadata->id()] = dictionary_field;
+    }
+    return Status::OK();
+  }
+
+  Status GetDictionaryTypes(DictionaryTypeMap* id_to_field) const {
+    for (int i = 0; i < num_fields(); ++i) {
+      RETURN_NOT_OK(VisitField(get_field(i), id_to_field));
+    }
+    return Status::OK();
+  }
+
  private:
   const flatbuf::Schema* schema_;
 };
@@ -138,15 +213,16 @@ int SchemaMetadata::num_fields() const {
   return impl_->num_fields();
 }
 
-Status SchemaMetadata::GetField(int i, std::shared_ptr<Field>* out) const {
-  const flatbuf::Field* field = impl_->field(i);
-  return FieldFromFlatbuffer(field, out);
+Status SchemaMetadata::GetDictionaryTypes(DictionaryTypeMap* id_to_field) const {
+  return impl_->GetDictionaryTypes(id_to_field);
 }
 
-Status SchemaMetadata::GetSchema(std::shared_ptr<Schema>* out) const {
+Status SchemaMetadata::GetSchema(
+    const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out) const {
   std::vector<std::shared_ptr<Field>> fields(num_fields());
   for (int i = 0; i < this->num_fields(); ++i) {
-    RETURN_NOT_OK(GetField(i, &fields[i]));
+    const flatbuf::Field* field = impl_->get_field(i);
+    RETURN_NOT_OK(FieldFromFlatbuffer(field, dictionary_memo, &fields[i]));
   }
   *out = std::make_shared<Schema>(fields);
   return Status::OK();
@@ -173,28 +249,34 @@ class RecordBatchMetadata::RecordBatchMetadataImpl {
 
   int num_fields() const { return batch_->nodes()->size(); }
 
+  void set_message(const std::shared_ptr<Message>& message) { message_ = message; }
+
+  void set_buffer(const std::shared_ptr<Buffer>& buffer) { buffer_ = buffer; }
+
  private:
   const flatbuf::RecordBatch* batch_;
   const flatbuffers::Vector<const flatbuf::FieldNode*>* nodes_;
   const flatbuffers::Vector<const flatbuf::Buffer*>* buffers_;
+
+  // Possible parents, owns the flatbuffer data
+  std::shared_ptr<Message> message_;
+  std::shared_ptr<Buffer> buffer_;
 };
 
 RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message) {
-  message_ = message;
   impl_.reset(new RecordBatchMetadataImpl(message->impl_->header()));
+  impl_->set_message(message);
 }
 
-RecordBatchMetadata::RecordBatchMetadata(
-    const std::shared_ptr<Buffer>& buffer, int64_t offset) {
-  message_ = nullptr;
-  buffer_ = buffer;
-
-  const flatbuf::RecordBatch* metadata =
-      flatbuffers::GetRoot<flatbuf::RecordBatch>(buffer->data() + offset);
-
-  // TODO(wesm): validate table
+RecordBatchMetadata::RecordBatchMetadata(const void* header) {
+  impl_.reset(new RecordBatchMetadataImpl(header));
+}
 
-  impl_.reset(new RecordBatchMetadataImpl(metadata));
+RecordBatchMetadata::RecordBatchMetadata(
+    const std::shared_ptr<Buffer>& buffer, int64_t offset)
+    : RecordBatchMetadata(buffer->data() + offset) {
+  // Preserve ownership
+  impl_->set_buffer(buffer);
 }
 
 RecordBatchMetadata::~RecordBatchMetadata() {}
@@ -232,5 +314,64 @@ int RecordBatchMetadata::num_fields() const {
   return impl_->num_fields();
 }
 
+// ----------------------------------------------------------------------
+// DictionaryBatchMetadata
+
+class DictionaryBatchMetadata::DictionaryBatchMetadataImpl {
+ public:
+  explicit DictionaryBatchMetadataImpl(const void* dictionary)
+      : metadata_(static_cast<const flatbuf::DictionaryBatch*>(dictionary)) {
+    record_batch_.reset(new RecordBatchMetadata(metadata_->data()));
+  }
+
+  int64_t id() const { return metadata_->id(); }
+  const RecordBatchMetadata& record_batch() const { return *record_batch_; }
+
+  void set_message(const std::shared_ptr<Message>& message) { message_ = message; }
+
+ private:
+  const flatbuf::DictionaryBatch* metadata_;
+
+  std::unique_ptr<RecordBatchMetadata> record_batch_;
+
+  // Parent, owns the flatbuffer data
+  std::shared_ptr<Message> message_;
+};
+
+DictionaryBatchMetadata::DictionaryBatchMetadata(
+    const std::shared_ptr<Message>& message) {
+  impl_.reset(new DictionaryBatchMetadataImpl(message->impl_->header()));
+  impl_->set_message(message);
+}
+
+DictionaryBatchMetadata::~DictionaryBatchMetadata() {}
+
+int64_t DictionaryBatchMetadata::id() const {
+  return impl_->id();
+}
+
+const RecordBatchMetadata& DictionaryBatchMetadata::record_batch() const {
+  return impl_->record_batch();
+}
+
+// ----------------------------------------------------------------------
+// Conveniences
+
+Status ReadMessage(int64_t offset, int32_t metadata_length,
+    io::ReadableFileInterface* file, std::shared_ptr<Message>* message) {
+  std::shared_ptr<Buffer> buffer;
+  RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
+
+  int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
+
+  if (flatbuffer_size + static_cast<int>(sizeof(int32_t)) > metadata_length) {
+    std::stringstream ss;
+    ss << "flatbuffer size " << metadata_length << " invalid. File offset: " << offset
+       << ", metadata length: " << metadata_length;
+    return Status::Invalid(ss.str());
+  }
+  return Message::Open(buffer, 4, message);
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 81e3dbdf6c4c0..0091067c3225a 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -22,13 +22,17 @@
 
 #include <cstdint>
 #include <memory>
+#include <unordered_map>
 #include <vector>
 
+#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
+class Array;
 class Buffer;
+struct DataType;
 struct Field;
 class Schema;
 class Status;
@@ -36,6 +40,7 @@ class Status;
 namespace io {
 
 class OutputStream;
+class ReadableFileInterface;
 
 }  // namespace io
 
@@ -47,9 +52,38 @@ struct MetadataVersion {
 
 //----------------------------------------------------------------------
 
-// Serialize arrow::Schema as a Flatbuffer
-ARROW_EXPORT
-Status WriteSchema(const Schema& schema, std::shared_ptr<Buffer>* out);
+using DictionaryMap = std::unordered_map<int64_t, std::shared_ptr<Array>>;
+using DictionaryTypeMap = std::unordered_map<int64_t, std::shared_ptr<Field>>;
+
+// Memoization data structure for handling shared dictionaries
+class DictionaryMemo {
+ public:
+  DictionaryMemo();
+
+  // Returns KeyError if dictionary not found
+  Status GetDictionary(int64_t id, std::shared_ptr<Array>* dictionary) const;
+
+  int64_t GetId(const std::shared_ptr<Array>& dictionary);
+
+  bool HasDictionary(const std::shared_ptr<Array>& dictionary) const;
+  bool HasDictionaryId(int64_t id) const;
+
+  // Add a dictionary to the memo with a particular id. Returns KeyError if
+  // that dictionary already exists
+  Status AddDictionary(int64_t id, const std::shared_ptr<Array>& dictionary);
+
+  const DictionaryMap& id_to_dictionary() const { return id_to_dictionary_; }
+
+ private:
+  // Dictionary memory addresses, to track whether a dictionary has been seen
+  // before
+  std::unordered_map<intptr_t, int64_t> dictionary_to_id_;
+
+  // Map of dictionary id to dictionary array
+  DictionaryMap id_to_dictionary_;
+
+  DISALLOW_COPY_AND_ASSIGN(DictionaryMemo);
+};
 
 // Read interface classes. We do not fully deserialize the flatbuffers so that
 // individual fields metadata can be retrieved from very large schema without
@@ -69,12 +103,15 @@ class ARROW_EXPORT SchemaMetadata {
 
   int num_fields() const;
 
-  // Construct an arrow::Field for the i-th value in the metadata
-  Status GetField(int i, std::shared_ptr<Field>* out) const;
+  // Retrieve a list of all the dictionary ids and types required by the schema for
+  // reconstruction. The presumption is that these will be loaded either from
+  // the stream or file (or they may already be somewhere else in memory)
+  Status GetDictionaryTypes(DictionaryTypeMap* id_to_field) const;
 
   // Construct a complete Schema from the message. May be expensive for very
   // large schemas if you are only interested in a few fields
-  Status GetSchema(std::shared_ptr<Schema>* out) const;
+  Status GetSchema(
+      const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out) const;
 
  private:
   // Parent, owns the flatbuffer data
@@ -82,6 +119,8 @@ class ARROW_EXPORT SchemaMetadata {
 
   class SchemaMetadataImpl;
   std::unique_ptr<SchemaMetadataImpl> impl_;
+
+  DISALLOW_COPY_AND_ASSIGN(SchemaMetadata);
 };
 
 // Field metadata
@@ -99,8 +138,10 @@ struct ARROW_EXPORT BufferMetadata {
 // Container for serialized record batch metadata contained in an IPC message
 class ARROW_EXPORT RecordBatchMetadata {
  public:
+  // Instantiate from opaque pointer. Memory ownership must be preserved
+  // elsewhere (e.g. in a dictionary batch)
+  explicit RecordBatchMetadata(const void* header);
   explicit RecordBatchMetadata(const std::shared_ptr<Message>& message);
-
   RecordBatchMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
 
   ~RecordBatchMetadata();
@@ -113,18 +154,25 @@ class ARROW_EXPORT RecordBatchMetadata {
   int num_fields() const;
 
  private:
-  // Parent, owns the flatbuffer data
-  std::shared_ptr<Message> message_;
-  std::shared_ptr<Buffer> buffer_;
-
   class RecordBatchMetadataImpl;
   std::unique_ptr<RecordBatchMetadataImpl> impl_;
+
+  DISALLOW_COPY_AND_ASSIGN(RecordBatchMetadata);
 };
 
 class ARROW_EXPORT DictionaryBatchMetadata {
  public:
+  explicit DictionaryBatchMetadata(const std::shared_ptr<Message>& message);
+  ~DictionaryBatchMetadata();
+
   int64_t id() const;
-  std::unique_ptr<RecordBatchMetadata> data() const;
+  const RecordBatchMetadata& record_batch() const;
+
+ private:
+  class DictionaryBatchMetadataImpl;
+  std::unique_ptr<DictionaryBatchMetadataImpl> impl_;
+
+  DISALLOW_COPY_AND_ASSIGN(DictionaryBatchMetadata);
 };
 
 class ARROW_EXPORT Message {
@@ -141,24 +189,31 @@ class ARROW_EXPORT Message {
  private:
   Message(const std::shared_ptr<Buffer>& buffer, int64_t offset);
 
+  friend class DictionaryBatchMetadata;
   friend class RecordBatchMetadata;
   friend class SchemaMetadata;
 
   // Hide serialization details from user API
   class MessageImpl;
   std::unique_ptr<MessageImpl> impl_;
-};
 
-struct ARROW_EXPORT FileBlock {
-  FileBlock() {}
-  FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
-      : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
-
-  int64_t offset;
-  int32_t metadata_length;
-  int64_t body_length;
+  DISALLOW_COPY_AND_ASSIGN(Message);
 };
 
+/// Read a length-prefixed message flatbuffer starting at the indicated file
+/// offset
+///
+/// The metadata_length includes at least the length prefix and the flatbuffer
+///
+/// \param[in] offset the position in the file where the message starts. The
+/// first 4 bytes after the offset are the message length
+/// \param[in] metadata_length the total number of bytes to read from file
+/// \param[in] file the seekable file interface to read from
+/// \param[out] message the message read
+/// \return Status success or failure
+Status ReadMessage(int64_t offset, int32_t metadata_length,
+    io::ReadableFileInterface* file, std::shared_ptr<Message>* message);
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/stream.cc b/cpp/src/arrow/ipc/stream.cc
index 72eb13465afcc..7f5c9932330be 100644
--- a/cpp/src/arrow/ipc/stream.cc
+++ b/cpp/src/arrow/ipc/stream.cc
@@ -20,17 +20,20 @@
 #include <cstdint>
 #include <cstring>
 #include <sstream>
+#include <string>
 #include <vector>
 
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/adapter.h"
+#include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
 #include "arrow/memory_pool.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
+#include "arrow/table.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -39,11 +42,10 @@ namespace ipc {
 // ----------------------------------------------------------------------
 // Stream writer implementation
 
-StreamWriter::~StreamWriter() {}
-
 StreamWriter::StreamWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
     : sink_(sink),
       schema_(schema),
+      dictionary_memo_(std::make_shared<DictionaryMemo>()),
       pool_(default_memory_pool()),
       position_(-1),
       started_(false) {}
@@ -107,7 +109,7 @@ Status StreamWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>&
 
 Status StreamWriter::Start() {
   std::shared_ptr<Buffer> schema_fb;
-  RETURN_NOT_OK(WriteSchema(*schema_, &schema_fb));
+  RETURN_NOT_OK(WriteSchemaMessage(*schema_, dictionary_memo_.get(), &schema_fb));
 
   int32_t flatbuffer_size = schema_fb->size();
   RETURN_NOT_OK(
@@ -115,14 +117,41 @@ Status StreamWriter::Start() {
 
   // Write the flatbuffer
   RETURN_NOT_OK(Write(schema_fb->data(), flatbuffer_size));
+
+  // If there are any dictionaries, write them as the next messages
+  RETURN_NOT_OK(WriteDictionaries());
+
   started_ = true;
   return Status::OK();
 }
 
 Status StreamWriter::WriteRecordBatch(const RecordBatch& batch) {
-  // Pass FileBlock, but results not used
-  FileBlock dummy_block;
-  return WriteRecordBatch(batch, &dummy_block);
+  // Push an empty FileBlock. Can be written in the footer later
+  record_batches_.emplace_back(0, 0, 0);
+  return WriteRecordBatch(batch, &record_batches_[record_batches_.size() - 1]);
+}
+
+Status StreamWriter::WriteDictionaries() {
+  const DictionaryMap& id_to_dictionary = dictionary_memo_->id_to_dictionary();
+
+  dictionaries_.resize(id_to_dictionary.size());
+
+  // TODO(wesm): does sorting by id yield any benefit?
+  int dict_index = 0;
+  for (const auto& entry : id_to_dictionary) {
+    FileBlock* block = &dictionaries_[dict_index++];
+
+    block->offset = position_;
+
+    // Frame of reference in file format is 0, see ARROW-384
+    const int64_t buffer_start_offset = 0;
+    RETURN_NOT_OK(WriteDictionary(entry.first, entry.second, buffer_start_offset, sink_,
+        &block->metadata_length, &block->body_length, pool_));
+    RETURN_NOT_OK(UpdatePosition());
+    DCHECK(position_ % 8 == 0) << "WriteDictionary did not perform aligned writes";
+  }
+
+  return Status::OK();
 }
 
 Status StreamWriter::Close() {
@@ -134,81 +163,147 @@ Status StreamWriter::Close() {
 // ----------------------------------------------------------------------
 // StreamReader implementation
 
-StreamReader::StreamReader(const std::shared_ptr<io::InputStream>& stream)
-    : stream_(stream), schema_(nullptr) {}
-
-StreamReader::~StreamReader() {}
-
-Status StreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
-    std::shared_ptr<StreamReader>* reader) {
-  // Private ctor
-  *reader = std::shared_ptr<StreamReader>(new StreamReader(stream));
-  return (*reader)->ReadSchema();
+static inline std::string message_type_name(Message::Type type) {
+  switch (type) {
+    case Message::SCHEMA:
+      return "schema";
+    case Message::RECORD_BATCH:
+      return "record batch";
+    case Message::DICTIONARY_BATCH:
+      return "dictionary";
+    default:
+      break;
+  }
+  return "unknown";
 }
 
-Status StreamReader::ReadSchema() {
-  std::shared_ptr<Message> message;
-  RETURN_NOT_OK(ReadNextMessage(&message));
+class StreamReader::StreamReaderImpl {
+ public:
+  StreamReaderImpl() {}
+  ~StreamReaderImpl() {}
 
-  if (message->type() != Message::SCHEMA) {
-    return Status::IOError("First message was not schema type");
+  Status Open(const std::shared_ptr<io::InputStream>& stream) {
+    stream_ = stream;
+    return ReadSchema();
   }
 
-  SchemaMetadata schema_meta(message);
+  Status ReadNextMessage(Message::Type expected_type, std::shared_ptr<Message>* message) {
+    std::shared_ptr<Buffer> buffer;
+    RETURN_NOT_OK(stream_->Read(sizeof(int32_t), &buffer));
 
-  // TODO(wesm): If the schema contains dictionaries, we must read all the
-  // dictionaries from the stream before constructing the final Schema
-  return schema_meta.GetSchema(&schema_);
-}
+    if (buffer->size() != sizeof(int32_t)) {
+      *message = nullptr;
+      return Status::OK();
+    }
+
+    int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
+
+    RETURN_NOT_OK(stream_->Read(message_length, &buffer));
+    if (buffer->size() != message_length) {
+      return Status::IOError("Unexpected end of stream trying to read message");
+    }
 
-Status StreamReader::ReadNextMessage(std::shared_ptr<Message>* message) {
-  std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(stream_->Read(sizeof(int32_t), &buffer));
+    RETURN_NOT_OK(Message::Open(buffer, 0, message));
 
-  if (buffer->size() != sizeof(int32_t)) {
-    *message = nullptr;
+    if ((*message)->type() != expected_type) {
+      std::stringstream ss;
+      ss << "Message not expected type: " << message_type_name(expected_type)
+         << ", was: " << (*message)->type();
+      return Status::IOError(ss.str());
+    }
     return Status::OK();
   }
 
-  int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
+  Status ReadExact(int64_t size, std::shared_ptr<Buffer>* buffer) {
+    RETURN_NOT_OK(stream_->Read(size, buffer));
 
-  RETURN_NOT_OK(stream_->Read(message_length, &buffer));
-  if (buffer->size() != message_length) {
-    return Status::IOError("Unexpected end of stream trying to read message");
+    if ((*buffer)->size() < size) {
+      return Status::IOError("Unexpected EOS when reading buffer");
+    }
+    return Status::OK();
   }
-  return Message::Open(buffer, 0, message);
-}
 
-std::shared_ptr<Schema> StreamReader::schema() const {
-  return schema_;
-}
+  Status ReadNextDictionary() {
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(ReadNextMessage(Message::DICTIONARY_BATCH, &message));
 
-Status StreamReader::GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
-  std::shared_ptr<Message> message;
-  RETURN_NOT_OK(ReadNextMessage(&message));
+    DictionaryBatchMetadata metadata(message);
 
-  if (message == nullptr) {
-    // End of stream
-    *batch = nullptr;
-    return Status::OK();
+    std::shared_ptr<Buffer> batch_body;
+    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body))
+    io::BufferReader reader(batch_body);
+
+    std::shared_ptr<Array> dictionary;
+    RETURN_NOT_OK(ReadDictionary(metadata, dictionary_types_, &reader, &dictionary));
+    return dictionary_memo_.AddDictionary(metadata.id(), dictionary);
   }
 
-  if (message->type() != Message::RECORD_BATCH) {
-    return Status::IOError("Metadata not record batch");
+  Status ReadSchema() {
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(ReadNextMessage(Message::SCHEMA, &message));
+
+    SchemaMetadata schema_meta(message);
+    RETURN_NOT_OK(schema_meta.GetDictionaryTypes(&dictionary_types_));
+
+    // TODO(wesm): In future, we may want to reconcile the ids in the stream with
+    // those found in the schema
+    int num_dictionaries = static_cast<int>(dictionary_types_.size());
+    for (int i = 0; i < num_dictionaries; ++i) {
+      RETURN_NOT_OK(ReadNextDictionary());
+    }
+
+    return schema_meta.GetSchema(dictionary_memo_, &schema_);
   }
 
-  auto batch_metadata = std::make_shared<RecordBatchMetadata>(message);
+  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(ReadNextMessage(Message::RECORD_BATCH, &message));
+
+    if (message == nullptr) {
+      // End of stream
+      *batch = nullptr;
+      return Status::OK();
+    }
 
-  std::shared_ptr<Buffer> batch_body;
-  RETURN_NOT_OK(stream_->Read(message->body_length(), &batch_body));
+    RecordBatchMetadata batch_metadata(message);
 
-  if (batch_body->size() < message->body_length()) {
-    return Status::IOError("Unexpected EOS when reading message body");
+    std::shared_ptr<Buffer> batch_body;
+    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body));
+    io::BufferReader reader(batch_body);
+    return ReadRecordBatch(batch_metadata, schema_, &reader, batch);
   }
 
-  io::BufferReader reader(batch_body);
+  std::shared_ptr<Schema> schema() const { return schema_; }
+
+ private:
+  // dictionary_id -> type
+  DictionaryTypeMap dictionary_types_;
+
+  DictionaryMemo dictionary_memo_;
+
+  std::shared_ptr<io::InputStream> stream_;
+  std::shared_ptr<Schema> schema_;
+};
+
+StreamReader::StreamReader() {
+  impl_.reset(new StreamReaderImpl());
+}
+
+StreamReader::~StreamReader() {}
+
+Status StreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
+    std::shared_ptr<StreamReader>* reader) {
+  // Private ctor
+  *reader = std::shared_ptr<StreamReader>(new StreamReader());
+  return (*reader)->impl_->Open(stream);
+}
+
+std::shared_ptr<Schema> StreamReader::schema() const {
+  return impl_->schema();
+}
 
-  return ReadRecordBatch(batch_metadata, schema_, &reader, batch);
+Status StreamReader::GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+  return impl_->GetNextRecordBatch(batch);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/stream.h b/cpp/src/arrow/ipc/stream.h
index 12414fa2ca0c7..1c3f65e49af32 100644
--- a/cpp/src/arrow/ipc/stream.h
+++ b/cpp/src/arrow/ipc/stream.h
@@ -22,7 +22,9 @@
 
 #include <cstdint>
 #include <memory>
+#include <vector>
 
+#include "arrow/ipc/metadata.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -44,12 +46,19 @@ class OutputStream;
 
 namespace ipc {
 
-struct FileBlock;
-class Message;
+struct ARROW_EXPORT FileBlock {
+  FileBlock() {}
+  FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
+      : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
+
+  int64_t offset;
+  int32_t metadata_length;
+  int64_t body_length;
+};
 
 class ARROW_EXPORT StreamWriter {
  public:
-  virtual ~StreamWriter();
+  virtual ~StreamWriter() = default;
 
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
       std::shared_ptr<StreamWriter>* out);
@@ -72,6 +81,8 @@ class ARROW_EXPORT StreamWriter {
   Status CheckStarted();
   Status UpdatePosition();
 
+  Status WriteDictionaries();
+
   Status WriteRecordBatch(const RecordBatch& batch, FileBlock* block);
 
   // Adds padding bytes if necessary to ensure all memory blocks are written on
@@ -87,10 +98,17 @@ class ARROW_EXPORT StreamWriter {
   io::OutputStream* sink_;
   std::shared_ptr<Schema> schema_;
 
+  // When writing out the schema, we keep track of all the dictionaries we
+  // encounter, as they must be written out first in the stream
+  std::shared_ptr<DictionaryMemo> dictionary_memo_;
+
   MemoryPool* pool_;
 
   int64_t position_;
   bool started_;
+
+  std::vector<FileBlock> dictionaries_;
+  std::vector<FileBlock> record_batches_;
 };
 
 class ARROW_EXPORT StreamReader {
@@ -107,14 +125,10 @@ class ARROW_EXPORT StreamReader {
   Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch);
 
  private:
-  explicit StreamReader(const std::shared_ptr<io::InputStream>& stream);
-
-  Status ReadSchema();
+  StreamReader();
 
-  Status ReadNextMessage(std::shared_ptr<Message>* message);
-
-  std::shared_ptr<io::InputStream> stream_;
-  std::shared_ptr<Schema> schema_;
+  class ARROW_NO_EXPORT StreamReaderImpl;
+  std::unique_ptr<StreamReaderImpl> impl_;
 };
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index b4930c4555d44..07f786c4d1d77 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -345,6 +345,86 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
+Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
+  const int32_t length = 6;
+
+  std::vector<bool> is_valid = {true, true, false, true, true, true};
+  std::shared_ptr<Array> dict1, dict2;
+
+  std::vector<std::string> dict1_values = {"foo", "bar", "baz"};
+  std::vector<std::string> dict2_values = {"foo", "bar", "baz", "qux"};
+
+  ArrayFromVector<StringType, std::string>(dict1_values, &dict1);
+  ArrayFromVector<StringType, std::string>(dict2_values, &dict2);
+
+  auto f0_type = arrow::dictionary(arrow::int32(), dict1);
+  auto f1_type = arrow::dictionary(arrow::int8(), dict1);
+  auto f2_type = arrow::dictionary(arrow::int32(), dict2);
+
+  std::shared_ptr<Array> indices0, indices1, indices2;
+  std::vector<int32_t> indices0_values = {1, 2, -1, 0, 2, 0};
+  std::vector<int8_t> indices1_values = {0, 0, 2, 2, 1, 1};
+  std::vector<int32_t> indices2_values = {3, 0, 2, 1, 0, 2};
+
+  ArrayFromVector<Int32Type, int32_t>(is_valid, indices0_values, &indices0);
+  ArrayFromVector<Int8Type, int8_t>(is_valid, indices1_values, &indices1);
+  ArrayFromVector<Int32Type, int32_t>(is_valid, indices2_values, &indices2);
+
+  auto a0 = std::make_shared<DictionaryArray>(f0_type, indices0);
+  auto a1 = std::make_shared<DictionaryArray>(f1_type, indices1);
+  auto a2 = std::make_shared<DictionaryArray>(f2_type, indices2);
+
+  // List of dictionary-encoded string
+  auto f3_type = list(f1_type);
+
+  std::vector<int32_t> list_offsets = {0, 0, 2, 2, 5, 6, 9};
+  std::shared_ptr<Array> offsets, indices3;
+  ArrayFromVector<Int32Type, int32_t>(
+      std::vector<bool>(list_offsets.size(), true), list_offsets, &offsets);
+
+  std::vector<int8_t> indices3_values = {0, 1, 2, 0, 1, 2, 0, 1, 2};
+  std::vector<bool> is_valid3(9, true);
+  ArrayFromVector<Int8Type, int8_t>(is_valid3, indices3_values, &indices3);
+
+  std::shared_ptr<Buffer> null_bitmap;
+  RETURN_NOT_OK(test::GetBitmapFromBoolVector(is_valid, &null_bitmap));
+
+  std::shared_ptr<Array> a3 = std::make_shared<ListArray>(f3_type, length,
+      std::static_pointer_cast<PrimitiveArray>(offsets)->data(),
+      std::make_shared<DictionaryArray>(f1_type, indices3), null_bitmap, 1);
+
+  // Dictionary-encoded list of integer
+  auto f4_value_type = list(int8());
+
+  std::shared_ptr<Array> offsets4, values4, indices4;
+
+  std::vector<int32_t> list_offsets4 = {0, 2, 2, 3};
+  ArrayFromVector<Int32Type, int32_t>(
+      std::vector<bool>(4, true), list_offsets4, &offsets4);
+
+  std::vector<int8_t> list_values4 = {0, 1, 2};
+  ArrayFromVector<Int8Type, int8_t>(std::vector<bool>(3, true), list_values4, &values4);
+
+  auto dict3 = std::make_shared<ListArray>(f4_value_type, 3,
+      std::static_pointer_cast<PrimitiveArray>(offsets4)->data(), values4);
+
+  std::vector<int8_t> indices4_values = {0, 1, 2, 0, 1, 2};
+  ArrayFromVector<Int8Type, int8_t>(is_valid, indices4_values, &indices4);
+
+  auto f4_type = dictionary(int8(), dict3);
+  auto a4 = std::make_shared<DictionaryArray>(f4_type, indices4);
+
+  // construct batch
+  std::shared_ptr<Schema> schema(new Schema({field("dict1", f0_type),
+      field("sparse", f1_type), field("dense", f2_type),
+      field("list of encoded string", f3_type), field("encoded list<int8>", f4_type)}));
+
+  std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2, a3, a4};
+
+  out->reset(new RecordBatch(schema, length, arrays));
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index a1c2b79950d59..b97b4657c361c 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -29,7 +29,7 @@ namespace arrow {
 bool Field::Equals(const Field& other) const {
   return (this == &other) ||
          (this->name == other.name && this->nullable == other.nullable &&
-             this->dictionary == dictionary && this->type->Equals(*other.type.get()));
+             this->type->Equals(*other.type.get()));
 }
 
 bool Field::Equals(const std::shared_ptr<Field>& other) const {
@@ -234,8 +234,8 @@ std::shared_ptr<DataType> dictionary(const std::shared_ptr<DataType>& index_type
 }
 
 std::shared_ptr<Field> field(
-    const std::string& name, const TypePtr& type, bool nullable, int64_t dictionary) {
-  return std::make_shared<Field>(name, type, nullable, dictionary);
+    const std::string& name, const TypePtr& type, bool nullable) {
+  return std::make_shared<Field>(name, type, nullable);
 }
 
 static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1);
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 927b8a44fe12f..b15aa277af201 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -114,6 +114,8 @@ class BufferDescr {
 
 class TypeVisitor {
  public:
+  virtual ~TypeVisitor() = default;
+
   virtual Status Visit(const NullType& type) = 0;
   virtual Status Visit(const BooleanType& type) = 0;
   virtual Status Visit(const Int8Type& type) = 0;
@@ -205,13 +207,9 @@ struct ARROW_EXPORT Field {
   // Fields can be nullable
   bool nullable;
 
-  // optional dictionary id if the field is dictionary encoded
-  // 0 means it's not dictionary encoded
-  int64_t dictionary;
-
   Field(const std::string& name, const std::shared_ptr<DataType>& type,
-      bool nullable = true, int64_t dictionary = 0)
-      : name(name), type(type), nullable(nullable), dictionary(dictionary) {}
+      bool nullable = true)
+      : name(name), type(type), nullable(nullable) {}
 
   bool operator==(const Field& other) const { return this->Equals(other); }
   bool operator!=(const Field& other) const { return !this->Equals(other); }
@@ -556,8 +554,8 @@ std::shared_ptr<DataType> ARROW_EXPORT union_(
 std::shared_ptr<DataType> ARROW_EXPORT dictionary(
     const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& values);
 
-std::shared_ptr<Field> ARROW_EXPORT field(const std::string& name,
-    const std::shared_ptr<DataType>& type, bool nullable = true, int64_t dictionary = 0);
+std::shared_ptr<Field> ARROW_EXPORT field(
+    const std::string& name, const std::shared_ptr<DataType>& type, bool nullable = true);
 
 // ----------------------------------------------------------------------
 //
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
index 5ab98152add49..afc7dbd36e5f0 100644
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -63,7 +63,6 @@ cdef extern from "arrow/ipc/file.h" namespace "arrow::ipc" nogil:
 
         shared_ptr[CSchema] schema()
 
-        int num_dictionaries()
         int num_record_batches()
 
         CStatus GetRecordBatch(int i, shared_ptr[CRecordBatch]* batch)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 89ce6e785c02b..4acef212b4dce 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -995,11 +995,6 @@ cdef class _FileReader:
             else:
                 check_status(CFileReader.Open(reader, &self.reader))
 
-    property num_dictionaries:
-
-        def __get__(self):
-            return self.reader.get().num_dictionaries()
-
     property num_record_batches:
 
         def __get__(self):

From 89dc55789b895653ba8184f462c88588928aee15 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 25 Feb 2017 15:29:30 -0500
Subject: [PATCH 0340/1644] ARROW-580: C++: Also provide jemalloc_X targets if
 only a static or shared version is found

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #349 from xhochy/ARROW-580 and squashes the following commits:

6cdeef2 [Uwe L. Korn] ARROW-580: C++: Also provide jemalloc_X targets if only a static or shared version is found
---
 cpp/CMakeLists.txt | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index b77f8c79fa024..06a18925c0d91 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -369,11 +369,19 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     add_library(${LIB_NAME} STATIC IMPORTED)
     set_target_properties(${LIB_NAME}
       PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    SET(AUG_LIB_NAME "${LIB_NAME}_static")
+    add_library(${AUG_LIB_NAME} STATIC IMPORTED)
+    set_target_properties(${AUG_LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
     message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
   elseif(ARG_SHARED_LIB)
     add_library(${LIB_NAME} SHARED IMPORTED)
     set_target_properties(${LIB_NAME}
       PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    SET(AUG_LIB_NAME "${LIB_NAME}_shared")
+    add_library(${AUG_LIB_NAME} SHARED IMPORTED)
+    set_target_properties(${AUG_LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
   else()
     message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")

From 8afe92c6cb966d7a3fa5fe30a24bb10be49afc06 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 25 Feb 2017 15:51:03 -0500
Subject: [PATCH 0341/1644] ARROW-578: [C++] Add -DARROW_CXXFLAGS=... option to
 make CMake more consistent

I've had issues on CMake 2.8.x with `-DCMAKE_CXX_FLAGS=$MY_CXXFLAGS` not passing on the flags to the compiler. But it seems to work properly in our Travis CI setup, so go figure.

Some Google searches seem to confirm this is a known issue, and having a specific "user flags" option is a way around it. We just did the same thing in parquet-cpp.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #348 from wesm/ARROW-578 and squashes the following commits:

1103bed [Wes McKinney] Use ARROW_CXXFLAGS in Travis CI
086d643 [Wes McKinney] Add -DARROW_CXXFLAGS=... option to make CMake behavior more consistent across versions
---
 ci/travis_before_script_cpp.sh | 4 ++--
 cpp/CMakeLists.txt             | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 94a889cff1a78..feacf8f8e361a 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -36,11 +36,11 @@ CMAKE_COMMON_FLAGS="\
 if [ $TRAVIS_OS_NAME == "linux" ]; then
     cmake -DARROW_TEST_MEMCHECK=on \
           $CMAKE_COMMON_FLAGS \
-          -DCMAKE_CXX_FLAGS="-Werror" \
+          -DARROW_CXXFLAGS=-Werror \
           $CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
-          -DCMAKE_CXX_FLAGS="-Werror" \
+          -DARROW_CXXFLAGS=-Werror \
           $CPP_DIR
 fi
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 06a18925c0d91..be3d4b98cf77f 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -57,6 +57,9 @@ endif(CCACHE_FOUND)
 
 # Top level cmake dir
 if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
+  set(ARROW_CXXFLAGS "" CACHE STRING
+    "Compiler flags to append when compiling Arrow")
+
   option(ARROW_BUILD_STATIC
     "Build the libarrow static libraries"
     ON)
@@ -120,7 +123,7 @@ endif()
 include(SetupCxxFlags)
 
 # Add common flags
-set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS "${ARROW_CXXFLAGS} ${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
 
 # Determine compiler version
 include(CompilerInfo)

From ef3b6b34482c36615af5064f474363126e755a18 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 26 Feb 2017 18:25:03 -0500
Subject: [PATCH 0342/1644] ARROW-451: [C++] Implement DataType::Equals as
 TypeVisitor. Add default implementations for TypeVisitor, ArrayVisitor
 methods

This patch also resolves ARROW-568. Added tests for TimeType, TimestampType, which were not having their `unit` metadata compared due to an oversight.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #350 from wesm/ARROW-451 and squashes the following commits:

97e75d8 [Wes McKinney] Export ArrayVisitor, TypeVisitor symbols
a3332be [Wes McKinney] Typo
635e74d [Wes McKinney] Implement DataType::Equals as TypeVisitor, compare child metadata. Add default implementations for TypeVisitor, ArrayVisitor methods
---
 cpp/src/arrow/CMakeLists.txt                  |   2 +-
 cpp/src/arrow/array.cc                        |  36 ++++++
 cpp/src/arrow/array.h                         |  50 ++++----
 cpp/src/arrow/compare.cc                      | 108 +++++++++++++++++-
 cpp/src/arrow/compare.h                       |   5 +
 cpp/src/arrow/ipc/adapter.cc                  |  20 ----
 cpp/src/arrow/ipc/json-internal.cc            |  30 -----
 .../arrow/{schema-test.cc => type-test.cc}    |  34 +++++-
 cpp/src/arrow/type.cc                         |  69 ++++++++---
 cpp/src/arrow/type.h                          |  64 +++++------
 10 files changed, 277 insertions(+), 141 deletions(-)
 rename cpp/src/arrow/{schema-test.cc => type-test.cc} (81%)

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 824ced1a51eb9..d1efa021a496d 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -58,8 +58,8 @@ ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(column-test)
 ADD_ARROW_TEST(memory_pool-test)
 ADD_ARROW_TEST(pretty_print-test)
-ADD_ARROW_TEST(schema-test)
 ADD_ARROW_TEST(status-test)
+ADD_ARROW_TEST(type-test)
 ADD_ARROW_TEST(table-test)
 
 ADD_ARROW_BENCHMARK(builder-benchmark)
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 81678e354a608..eb4c210930fb2 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -503,4 +503,40 @@ Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
 #endif
 }
 
+// ----------------------------------------------------------------------
+// Default implementations of ArrayVisitor methods
+
+#define ARRAY_VISITOR_DEFAULT(ARRAY_CLASS)                   \
+  Status ArrayVisitor::Visit(const ARRAY_CLASS& array) {     \
+    return Status::NotImplemented(array.type()->ToString()); \
+  }
+
+ARRAY_VISITOR_DEFAULT(NullArray);
+ARRAY_VISITOR_DEFAULT(BooleanArray);
+ARRAY_VISITOR_DEFAULT(Int8Array);
+ARRAY_VISITOR_DEFAULT(Int16Array);
+ARRAY_VISITOR_DEFAULT(Int32Array);
+ARRAY_VISITOR_DEFAULT(Int64Array);
+ARRAY_VISITOR_DEFAULT(UInt8Array);
+ARRAY_VISITOR_DEFAULT(UInt16Array);
+ARRAY_VISITOR_DEFAULT(UInt32Array);
+ARRAY_VISITOR_DEFAULT(UInt64Array);
+ARRAY_VISITOR_DEFAULT(HalfFloatArray);
+ARRAY_VISITOR_DEFAULT(FloatArray);
+ARRAY_VISITOR_DEFAULT(DoubleArray);
+ARRAY_VISITOR_DEFAULT(StringArray);
+ARRAY_VISITOR_DEFAULT(BinaryArray);
+ARRAY_VISITOR_DEFAULT(DateArray);
+ARRAY_VISITOR_DEFAULT(TimeArray);
+ARRAY_VISITOR_DEFAULT(TimestampArray);
+ARRAY_VISITOR_DEFAULT(IntervalArray);
+ARRAY_VISITOR_DEFAULT(ListArray);
+ARRAY_VISITOR_DEFAULT(StructArray);
+ARRAY_VISITOR_DEFAULT(UnionArray);
+ARRAY_VISITOR_DEFAULT(DictionaryArray);
+
+Status ArrayVisitor::Visit(const DecimalArray& array) {
+  return Status::NotImplemented("decimal");
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 9bb06afc9bf6c..8bb914e44ad3d 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -38,34 +38,34 @@ class MemoryPool;
 class MutableBuffer;
 class Status;
 
-class ArrayVisitor {
+class ARROW_EXPORT ArrayVisitor {
  public:
   virtual ~ArrayVisitor() = default;
 
-  virtual Status Visit(const NullArray& array) = 0;
-  virtual Status Visit(const BooleanArray& array) = 0;
-  virtual Status Visit(const Int8Array& array) = 0;
-  virtual Status Visit(const Int16Array& array) = 0;
-  virtual Status Visit(const Int32Array& array) = 0;
-  virtual Status Visit(const Int64Array& array) = 0;
-  virtual Status Visit(const UInt8Array& array) = 0;
-  virtual Status Visit(const UInt16Array& array) = 0;
-  virtual Status Visit(const UInt32Array& array) = 0;
-  virtual Status Visit(const UInt64Array& array) = 0;
-  virtual Status Visit(const HalfFloatArray& array) = 0;
-  virtual Status Visit(const FloatArray& array) = 0;
-  virtual Status Visit(const DoubleArray& array) = 0;
-  virtual Status Visit(const StringArray& array) = 0;
-  virtual Status Visit(const BinaryArray& array) = 0;
-  virtual Status Visit(const DateArray& array) = 0;
-  virtual Status Visit(const TimeArray& array) = 0;
-  virtual Status Visit(const TimestampArray& array) = 0;
-  virtual Status Visit(const IntervalArray& array) = 0;
-  virtual Status Visit(const DecimalArray& array) = 0;
-  virtual Status Visit(const ListArray& array) = 0;
-  virtual Status Visit(const StructArray& array) = 0;
-  virtual Status Visit(const UnionArray& array) = 0;
-  virtual Status Visit(const DictionaryArray& type) = 0;
+  virtual Status Visit(const NullArray& array);
+  virtual Status Visit(const BooleanArray& array);
+  virtual Status Visit(const Int8Array& array);
+  virtual Status Visit(const Int16Array& array);
+  virtual Status Visit(const Int32Array& array);
+  virtual Status Visit(const Int64Array& array);
+  virtual Status Visit(const UInt8Array& array);
+  virtual Status Visit(const UInt16Array& array);
+  virtual Status Visit(const UInt32Array& array);
+  virtual Status Visit(const UInt64Array& array);
+  virtual Status Visit(const HalfFloatArray& array);
+  virtual Status Visit(const FloatArray& array);
+  virtual Status Visit(const DoubleArray& array);
+  virtual Status Visit(const StringArray& array);
+  virtual Status Visit(const BinaryArray& array);
+  virtual Status Visit(const DateArray& array);
+  virtual Status Visit(const TimeArray& array);
+  virtual Status Visit(const TimestampArray& array);
+  virtual Status Visit(const IntervalArray& array);
+  virtual Status Visit(const DecimalArray& array);
+  virtual Status Visit(const ListArray& array);
+  virtual Status Visit(const StructArray& array);
+  virtual Status Visit(const UnionArray& array);
+  virtual Status Visit(const DictionaryArray& type);
 };
 
 /// Immutable data array with some logical type and some length.
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 21fdb6633a9ee..ff3c59f638bb0 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -301,9 +301,9 @@ class RangeEqualsVisitor : public ArrayVisitor {
   bool result_;
 };
 
-class EqualsVisitor : public RangeEqualsVisitor {
+class ArrayEqualsVisitor : public RangeEqualsVisitor {
  public:
-  explicit EqualsVisitor(const Array& right)
+  explicit ArrayEqualsVisitor(const Array& right)
       : RangeEqualsVisitor(right, 0, right.length(), 0) {}
 
   Status Visit(const NullArray& left) override { return Status::OK(); }
@@ -511,9 +511,9 @@ inline bool FloatingApproxEquals(
   return true;
 }
 
-class ApproxEqualsVisitor : public EqualsVisitor {
+class ApproxEqualsVisitor : public ArrayEqualsVisitor {
  public:
-  using EqualsVisitor::EqualsVisitor;
+  using ArrayEqualsVisitor::ArrayEqualsVisitor;
 
   Status Visit(const FloatArray& left) override {
     result_ =
@@ -549,7 +549,7 @@ Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
   } else if (left.length() == 0) {
     *are_equal = true;
   } else {
-    EqualsVisitor visitor(right);
+    ArrayEqualsVisitor visitor(right);
     RETURN_NOT_OK(left.Accept(&visitor));
     *are_equal = visitor.result();
   }
@@ -588,4 +588,102 @@ Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal)
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// Implement TypeEquals
+
+class TypeEqualsVisitor : public TypeVisitor {
+ public:
+  explicit TypeEqualsVisitor(const DataType& right) : right_(right), result_(false) {}
+
+  Status VisitChildren(const DataType& left) {
+    if (left.num_children() != right_.num_children()) {
+      result_ = false;
+      return Status::OK();
+    }
+
+    for (int i = 0; i < left.num_children(); ++i) {
+      if (!left.child(i)->Equals(right_.child(i))) {
+        result_ = false;
+        break;
+      }
+    }
+    result_ = true;
+    return Status::OK();
+  }
+
+  Status Visit(const TimeType& left) override {
+    const auto& right = static_cast<const TimeType&>(right_);
+    result_ = left.unit == right.unit;
+    return Status::OK();
+  }
+
+  Status Visit(const TimestampType& left) override {
+    const auto& right = static_cast<const TimestampType&>(right_);
+    result_ = left.unit == right.unit;
+    return Status::OK();
+  }
+
+  Status Visit(const ListType& left) override { return VisitChildren(left); }
+
+  Status Visit(const StructType& left) override { return VisitChildren(left); }
+
+  Status Visit(const UnionType& left) override {
+    const auto& right = static_cast<const UnionType&>(right_);
+
+    if (left.mode != right.mode || left.type_codes.size() != right.type_codes.size()) {
+      result_ = false;
+      return Status::OK();
+    }
+
+    const std::vector<uint8_t> left_codes = left.type_codes;
+    const std::vector<uint8_t> right_codes = right.type_codes;
+
+    for (size_t i = 0; i < left_codes.size(); ++i) {
+      if (left_codes[i] != right_codes[i]) {
+        result_ = false;
+        break;
+      }
+    }
+    result_ = true;
+    return Status::OK();
+  }
+
+  Status Visit(const DictionaryType& left) override {
+    const auto& right = static_cast<const DictionaryType&>(right_);
+    result_ = left.index_type()->Equals(right.index_type()) &&
+              left.dictionary()->Equals(right.dictionary());
+    return Status::OK();
+  }
+
+  bool result() const { return result_; }
+
+ protected:
+  const DataType& right_;
+  bool result_;
+};
+
+Status TypeEquals(const DataType& left, const DataType& right, bool* are_equal) {
+  // The arrays are the same object
+  if (&left == &right) {
+    *are_equal = true;
+  } else if (left.type != right.type) {
+    *are_equal = false;
+  } else {
+    TypeEqualsVisitor visitor(right);
+    Status s = left.Accept(&visitor);
+
+    // We do not implement any type visitors where there is no additional
+    // metadata to compare.
+    if (s.IsNotImplemented()) {
+      // Not implemented means there is no additional metadata to compare
+      *are_equal = true;
+    } else if (!s.ok()) {
+      return s;
+    } else {
+      *are_equal = visitor.result();
+    }
+  }
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index 2093b65a51a13..6a71f9fd573ba 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -27,6 +27,7 @@
 namespace arrow {
 
 class Array;
+struct DataType;
 class Status;
 
 /// Returns true if the arrays are exactly equal
@@ -41,6 +42,10 @@ Status ARROW_EXPORT ArrayApproxEquals(
 Status ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right,
     int32_t start_idx, int32_t end_idx, int32_t other_start_idx, bool* are_equal);
 
+/// Returns true if the type metadata are exactly equal
+Status ARROW_EXPORT TypeEquals(
+    const DataType& left, const DataType& right, bool* are_equal);
+
 }  // namespace arrow
 
 #endif  // ARROW_COMPARE_H
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 08ac9832982c1..2be87a35e7fb3 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -227,8 +227,6 @@ class RecordBatchWriter : public ArrayVisitor {
   }
 
  protected:
-  Status Visit(const NullArray& array) override { return Status::NotImplemented("null"); }
-
   template <typename ArrayType>
   Status VisitFixedWidth(const ArrayType& array) {
     std::shared_ptr<Buffer> data_buffer = array.data();
@@ -360,14 +358,6 @@ class RecordBatchWriter : public ArrayVisitor {
     return VisitFixedWidth<TimestampArray>(array);
   }
 
-  Status Visit(const IntervalArray& array) override {
-    return Status::NotImplemented("interval");
-  }
-
-  Status Visit(const DecimalArray& array) override {
-    return Status::NotImplemented("decimal");
-  }
-
   Status Visit(const ListArray& array) override {
     std::shared_ptr<Buffer> value_offsets;
     RETURN_NOT_OK(GetZeroBasedValueOffsets<ListArray>(array, &value_offsets));
@@ -653,8 +643,6 @@ class ArrayLoader : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const NullType& type) override { return Status::NotImplemented("null"); }
-
   Status Visit(const BooleanType& type) override { return LoadPrimitive(type); }
 
   Status Visit(const Int8Type& type) override { return LoadPrimitive(type); }
@@ -689,14 +677,6 @@ class ArrayLoader : public TypeVisitor {
 
   Status Visit(const TimestampType& type) override { return LoadPrimitive(type); }
 
-  Status Visit(const IntervalType& type) override {
-    return Status::NotImplemented(type.ToString());
-  }
-
-  Status Visit(const DecimalType& type) override {
-    return Status::NotImplemented(type.ToString());
-  }
-
   Status Visit(const ListType& type) override {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap, offsets;
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index b9f97dd2bbd15..6253cd6b43605 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -316,8 +316,6 @@ class JsonSchemaWriter : public TypeVisitor {
     return WritePrimitive("interval", type);
   }
 
-  Status Visit(const DecimalType& type) override { return Status::NotImplemented("NYI"); }
-
   Status Visit(const ListType& type) override {
     WriteName("list", type);
     RETURN_NOT_OK(WriteChildren(type.children()));
@@ -339,14 +337,6 @@ class JsonSchemaWriter : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DictionaryType& type) override {
-    // WriteName("dictionary", type);
-    // WriteChildren(type.children());
-    // WriteBufferLayout(type.GetBufferLayout());
-    // return Status::OK();
-    return Status::NotImplemented("dictionary type");
-  }
-
  private:
   const Schema& schema_;
   RjWriter* writer_;
@@ -531,22 +521,6 @@ class JsonArrayWriter : public ArrayVisitor {
 
   Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); }
 
-  Status Visit(const DateArray& array) override { return Status::NotImplemented("date"); }
-
-  Status Visit(const TimeArray& array) override { return Status::NotImplemented("time"); }
-
-  Status Visit(const TimestampArray& array) override {
-    return Status::NotImplemented("timestamp");
-  }
-
-  Status Visit(const IntervalArray& array) override {
-    return Status::NotImplemented("interval");
-  }
-
-  Status Visit(const DecimalArray& array) override {
-    return Status::NotImplemented("decimal");
-  }
-
   Status Visit(const ListArray& array) override {
     WriteValidityField(array);
     WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length() + 1);
@@ -571,10 +545,6 @@ class JsonArrayWriter : public ArrayVisitor {
     return WriteChildren(type->children(), array.children());
   }
 
-  Status Visit(const DictionaryArray& array) override {
-    return Status::NotImplemented("dictionary");
-  }
-
  private:
   const std::string& name_;
   const Array& array_;
diff --git a/cpp/src/arrow/schema-test.cc b/cpp/src/arrow/type-test.cc
similarity index 81%
rename from cpp/src/arrow/schema-test.cc
rename to cpp/src/arrow/type-test.cc
index 4826199f73de7..fe6c62adb7fba 100644
--- a/cpp/src/arrow/schema-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+// Unit tests for DataType (and subclasses), Field, and Schema
+
 #include <memory>
 #include <string>
 #include <vector>
@@ -45,8 +47,8 @@ TEST(TestField, Equals) {
   Field f0_nn("f0", int32(), false);
   Field f0_other("f0", int32());
 
-  ASSERT_EQ(f0, f0_other);
-  ASSERT_NE(f0, f0_nn);
+  ASSERT_TRUE(f0.Equals(f0_other));
+  ASSERT_FALSE(f0.Equals(f0_nn));
 }
 
 class TestSchema : public ::testing::Test {
@@ -65,9 +67,9 @@ TEST_F(TestSchema, Basics) {
   auto schema = std::make_shared<Schema>(fields);
 
   ASSERT_EQ(3, schema->num_fields());
-  ASSERT_EQ(f0, schema->field(0));
-  ASSERT_EQ(f1, schema->field(1));
-  ASSERT_EQ(f2, schema->field(2));
+  ASSERT_TRUE(f0->Equals(schema->field(0)));
+  ASSERT_TRUE(f1->Equals(schema->field(1)));
+  ASSERT_TRUE(f2->Equals(schema->field(2)));
 
   auto schema2 = std::make_shared<Schema>(fields);
 
@@ -119,4 +121,26 @@ TEST_F(TestSchema, GetFieldByName) {
   ASSERT_TRUE(result == nullptr);
 }
 
+TEST(TestTimeType, Equals) {
+  TimeType t1;
+  TimeType t2;
+  TimeType t3(TimeUnit::NANO);
+  TimeType t4(TimeUnit::NANO);
+
+  ASSERT_TRUE(t1.Equals(t2));
+  ASSERT_FALSE(t1.Equals(t3));
+  ASSERT_TRUE(t3.Equals(t4));
+}
+
+TEST(TestTimestampType, Equals) {
+  TimestampType t1;
+  TimestampType t2;
+  TimestampType t3(TimeUnit::NANO);
+  TimestampType t4(TimeUnit::NANO);
+
+  ASSERT_TRUE(t1.Equals(t2));
+  ASSERT_FALSE(t1.Equals(t3));
+  ASSERT_TRUE(t3.Equals(t4));
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index b97b4657c361c..23fa6812f53d4 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -21,6 +21,7 @@
 #include <string>
 
 #include "arrow/array.h"
+#include "arrow/compare.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
@@ -46,16 +47,14 @@ std::string Field::ToString() const {
 DataType::~DataType() {}
 
 bool DataType::Equals(const DataType& other) const {
-  bool equals =
-      ((this == &other) || ((this->type == other.type) &&
-                               ((this->num_children() == other.num_children()))));
-  if (equals) {
-    for (int i = 0; i < num_children(); ++i) {
-      // TODO(emkornfield) limit recursion
-      if (!children_[i]->Equals(other.children_[i])) { return false; }
-    }
-  }
-  return equals;
+  bool are_equal = false;
+  Status error = TypeEquals(*this, other, &are_equal);
+  if (!error.ok()) { DCHECK(false) << "Types not comparable: " << error.ToString(); }
+  return are_equal;
+}
+
+bool DataType::Equals(const std::shared_ptr<DataType>& other) const {
+  return Equals(*other.get());
 }
 
 std::string BooleanType::ToString() const {
@@ -104,6 +103,15 @@ std::string DateType::ToString() const {
   return std::string("date");
 }
 
+// ----------------------------------------------------------------------
+// Union type
+
+UnionType::UnionType(const std::vector<std::shared_ptr<Field>>& fields,
+    const std::vector<uint8_t>& type_codes, UnionMode mode)
+    : DataType(Type::UNION), mode(mode), type_codes(type_codes) {
+  children_ = fields;
+}
+
 std::string UnionType::ToString() const {
   std::stringstream s;
 
@@ -138,14 +146,6 @@ std::shared_ptr<Array> DictionaryType::dictionary() const {
   return dictionary_;
 }
 
-bool DictionaryType::Equals(const DataType& other) const {
-  if (other.type != Type::DICTIONARY) { return false; }
-  const auto& other_dict = static_cast<const DictionaryType&>(other);
-
-  return index_type_->Equals(other_dict.index_type_) &&
-         dictionary_->Equals(other_dict.dictionary_);
-}
-
 std::string DictionaryType::ToString() const {
   std::stringstream ss;
   ss << "dictionary<values=" << dictionary_->type()->ToString()
@@ -286,4 +286,37 @@ std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
   return {};
 }
 
+// ----------------------------------------------------------------------
+// Default implementations of TypeVisitor methods
+
+#define TYPE_VISITOR_DEFAULT(TYPE_CLASS)              \
+  Status TypeVisitor::Visit(const TYPE_CLASS& type) { \
+    return Status::NotImplemented(type.ToString());   \
+  }
+
+TYPE_VISITOR_DEFAULT(NullType);
+TYPE_VISITOR_DEFAULT(BooleanType);
+TYPE_VISITOR_DEFAULT(Int8Type);
+TYPE_VISITOR_DEFAULT(Int16Type);
+TYPE_VISITOR_DEFAULT(Int32Type);
+TYPE_VISITOR_DEFAULT(Int64Type);
+TYPE_VISITOR_DEFAULT(UInt8Type);
+TYPE_VISITOR_DEFAULT(UInt16Type);
+TYPE_VISITOR_DEFAULT(UInt32Type);
+TYPE_VISITOR_DEFAULT(UInt64Type);
+TYPE_VISITOR_DEFAULT(HalfFloatType);
+TYPE_VISITOR_DEFAULT(FloatType);
+TYPE_VISITOR_DEFAULT(DoubleType);
+TYPE_VISITOR_DEFAULT(StringType);
+TYPE_VISITOR_DEFAULT(BinaryType);
+TYPE_VISITOR_DEFAULT(DateType);
+TYPE_VISITOR_DEFAULT(TimeType);
+TYPE_VISITOR_DEFAULT(TimestampType);
+TYPE_VISITOR_DEFAULT(IntervalType);
+TYPE_VISITOR_DEFAULT(DecimalType);
+TYPE_VISITOR_DEFAULT(ListType);
+TYPE_VISITOR_DEFAULT(StructType);
+TYPE_VISITOR_DEFAULT(UnionType);
+TYPE_VISITOR_DEFAULT(DictionaryType);
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index b15aa277af201..9a97fc30094b9 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -112,34 +112,34 @@ class BufferDescr {
   int bit_width_;
 };
 
-class TypeVisitor {
+class ARROW_EXPORT TypeVisitor {
  public:
   virtual ~TypeVisitor() = default;
 
-  virtual Status Visit(const NullType& type) = 0;
-  virtual Status Visit(const BooleanType& type) = 0;
-  virtual Status Visit(const Int8Type& type) = 0;
-  virtual Status Visit(const Int16Type& type) = 0;
-  virtual Status Visit(const Int32Type& type) = 0;
-  virtual Status Visit(const Int64Type& type) = 0;
-  virtual Status Visit(const UInt8Type& type) = 0;
-  virtual Status Visit(const UInt16Type& type) = 0;
-  virtual Status Visit(const UInt32Type& type) = 0;
-  virtual Status Visit(const UInt64Type& type) = 0;
-  virtual Status Visit(const HalfFloatType& type) = 0;
-  virtual Status Visit(const FloatType& type) = 0;
-  virtual Status Visit(const DoubleType& type) = 0;
-  virtual Status Visit(const StringType& type) = 0;
-  virtual Status Visit(const BinaryType& type) = 0;
-  virtual Status Visit(const DateType& type) = 0;
-  virtual Status Visit(const TimeType& type) = 0;
-  virtual Status Visit(const TimestampType& type) = 0;
-  virtual Status Visit(const IntervalType& type) = 0;
-  virtual Status Visit(const DecimalType& type) = 0;
-  virtual Status Visit(const ListType& type) = 0;
-  virtual Status Visit(const StructType& type) = 0;
-  virtual Status Visit(const UnionType& type) = 0;
-  virtual Status Visit(const DictionaryType& type) = 0;
+  virtual Status Visit(const NullType& type);
+  virtual Status Visit(const BooleanType& type);
+  virtual Status Visit(const Int8Type& type);
+  virtual Status Visit(const Int16Type& type);
+  virtual Status Visit(const Int32Type& type);
+  virtual Status Visit(const Int64Type& type);
+  virtual Status Visit(const UInt8Type& type);
+  virtual Status Visit(const UInt16Type& type);
+  virtual Status Visit(const UInt32Type& type);
+  virtual Status Visit(const UInt64Type& type);
+  virtual Status Visit(const HalfFloatType& type);
+  virtual Status Visit(const FloatType& type);
+  virtual Status Visit(const DoubleType& type);
+  virtual Status Visit(const StringType& type);
+  virtual Status Visit(const BinaryType& type);
+  virtual Status Visit(const DateType& type);
+  virtual Status Visit(const TimeType& type);
+  virtual Status Visit(const TimestampType& type);
+  virtual Status Visit(const IntervalType& type);
+  virtual Status Visit(const DecimalType& type);
+  virtual Status Visit(const ListType& type);
+  virtual Status Visit(const StructType& type);
+  virtual Status Visit(const UnionType& type);
+  virtual Status Visit(const DictionaryType& type);
 };
 
 struct ARROW_EXPORT DataType {
@@ -156,10 +156,7 @@ struct ARROW_EXPORT DataType {
   // Types that are logically convertable from one to another e.g. List<UInt8>
   // and Binary are NOT equal).
   virtual bool Equals(const DataType& other) const;
-
-  bool Equals(const std::shared_ptr<DataType>& other) const {
-    return Equals(*other.get());
-  }
+  bool Equals(const std::shared_ptr<DataType>& other) const;
 
   std::shared_ptr<Field> child(int i) const { return children_[i]; }
 
@@ -211,8 +208,6 @@ struct ARROW_EXPORT Field {
       bool nullable = true)
       : name(name), type(type), nullable(nullable) {}
 
-  bool operator==(const Field& other) const { return this->Equals(other); }
-  bool operator!=(const Field& other) const { return !this->Equals(other); }
   bool Equals(const Field& other) const;
   bool Equals(const std::shared_ptr<Field>& other) const;
 
@@ -411,10 +406,7 @@ struct ARROW_EXPORT UnionType : public DataType {
   static constexpr Type::type type_id = Type::UNION;
 
   UnionType(const std::vector<std::shared_ptr<Field>>& fields,
-      const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE)
-      : DataType(Type::UNION), mode(mode), type_codes(type_codes) {
-    children_ = fields;
-  }
+      const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
 
   std::string ToString() const override;
   static std::string name() { return "union"; }
@@ -523,8 +515,6 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 
   std::shared_ptr<Array> dictionary() const;
 
-  bool Equals(const DataType& other) const override;
-
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
 

From 16c97592bf948c32a8dae9441ace078422d642dd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 26 Feb 2017 19:22:15 -0500
Subject: [PATCH 0343/1644] ARROW-577: [C++] Use private implementation pattern
 in ipc::StreamWriter and ipc::FileWriter

This patch also includes some code reorganization -- I moved the reader and writer classes to their own headers/compilation units. I also moved the stream-to-file and file-to-stream executables to arrow/ipc

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #351 from wesm/ARROW-577 and squashes the following commits:

98c32d2 [Wes McKinney] Only build file/stream utils if ARROW_BUILD_UTILITIES is on
c5fa43f [Wes McKinney] Refactor to give make stream and file writer implementation details private in public ABI
---
 cpp/src/arrow/ipc/CMakeLists.txt              |  32 +-
 cpp/src/arrow/ipc/api.h                       |  27 ++
 cpp/src/arrow/{util => ipc}/file-to-stream.cc |   4 +-
 cpp/src/arrow/ipc/ipc-file-test.cc            |   4 +-
 cpp/src/arrow/ipc/json-integration-test.cc    |   3 +-
 cpp/src/arrow/ipc/metadata.h                  |  12 +
 cpp/src/arrow/ipc/{file.cc => reader.cc}      | 184 +++++++----
 cpp/src/arrow/ipc/{file.h => reader.h}        |  27 +-
 cpp/src/arrow/{util => ipc}/stream-to-file.cc |   4 +-
 cpp/src/arrow/ipc/stream.cc                   | 310 ------------------
 cpp/src/arrow/ipc/writer.cc                   | 287 ++++++++++++++++
 cpp/src/arrow/ipc/{stream.h => writer.h}      |  75 +----
 cpp/src/arrow/util/CMakeLists.txt             |  22 --
 python/pyarrow/includes/libarrow_ipc.pxd      |   5 +-
 14 files changed, 520 insertions(+), 476 deletions(-)
 create mode 100644 cpp/src/arrow/ipc/api.h
 rename cpp/src/arrow/{util => ipc}/file-to-stream.cc (97%)
 rename cpp/src/arrow/ipc/{file.cc => reader.cc} (63%)
 rename cpp/src/arrow/ipc/{file.h => reader.h} (83%)
 rename cpp/src/arrow/{util => ipc}/stream-to-file.cc (96%)
 delete mode 100644 cpp/src/arrow/ipc/stream.cc
 create mode 100644 cpp/src/arrow/ipc/writer.cc
 rename cpp/src/arrow/ipc/{stream.h => writer.h} (52%)

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index e7a3fdb1dd862..08da0a109c963 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -30,12 +30,12 @@ set(ARROW_IPC_TEST_LINK_LIBS
 
 set(ARROW_IPC_SRCS
   adapter.cc
-  file.cc
   json.cc
   json-internal.cc
   metadata.cc
   metadata-internal.cc
-  stream.cc
+  reader.cc
+  writer.cc
 )
 
 if(NOT APPLE)
@@ -138,10 +138,11 @@ add_dependencies(arrow_ipc_objlib metadata_fbs)
 # Headers: top level
 install(FILES
   adapter.h
-  file.h
+  api.h
   json.h
   metadata.h
-  stream.h
+  reader.h
+  writer.h
   DESTINATION include/arrow/ipc)
 
 # pkg-config support
@@ -151,3 +152,26 @@ configure_file(arrow-ipc.pc.in
 install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-ipc.pc"
   DESTINATION "lib/pkgconfig/")
+
+
+set(UTIL_LINK_LIBS
+  arrow_ipc_static
+  arrow_io_static
+  arrow_static
+  boost_filesystem_static
+  boost_system_static
+  dl)
+
+if (NOT APPLE)
+  set(UTIL_LINK_LIBS
+    ${UTIL_LINK_LIBS}
+    boost_filesystem_static
+    boost_system_static)
+endif()
+
+if (ARROW_BUILD_UTILITIES)
+  add_executable(file-to-stream file-to-stream.cc)
+  target_link_libraries(file-to-stream ${UTIL_LINK_LIBS})
+  add_executable(stream-to-file stream-to-file.cc)
+  target_link_libraries(stream-to-file ${UTIL_LINK_LIBS})
+endif()
diff --git a/cpp/src/arrow/ipc/api.h b/cpp/src/arrow/ipc/api.h
new file mode 100644
index 0000000000000..cb854212bbeee
--- /dev/null
+++ b/cpp/src/arrow/ipc/api.h
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IPC_API_H
+#define ARROW_IPC_API_H
+
+#include "arrow/ipc/adapter.h"
+#include "arrow/ipc/json.h"
+#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/reader.h"
+#include "arrow/ipc/writer.h"
+
+#endif  // ARROW_IPC_API_H
diff --git a/cpp/src/arrow/util/file-to-stream.cc b/cpp/src/arrow/ipc/file-to-stream.cc
similarity index 97%
rename from cpp/src/arrow/util/file-to-stream.cc
rename to cpp/src/arrow/ipc/file-to-stream.cc
index 7daf26366721d..8161b191380dc 100644
--- a/cpp/src/arrow/util/file-to-stream.cc
+++ b/cpp/src/arrow/ipc/file-to-stream.cc
@@ -16,8 +16,8 @@
 // under the License.
 
 #include "arrow/io/file.h"
-#include "arrow/ipc/file.h"
-#include "arrow/ipc/stream.h"
+#include "arrow/ipc/reader.h"
+#include "arrow/ipc/writer.h"
 #include "arrow/status.h"
 #include <iostream>
 
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index 4b82aab0e3978..e58f2cfbbe8c9 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -28,10 +28,10 @@
 #include "arrow/io/memory.h"
 #include "arrow/io/test-common.h"
 #include "arrow/ipc/adapter.h"
-#include "arrow/ipc/file.h"
-#include "arrow/ipc/stream.h"
+#include "arrow/ipc/reader.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/ipc/util.h"
+#include "arrow/ipc/writer.h"
 
 #include "arrow/buffer.h"
 #include "arrow/memory_pool.h"
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 95bc742054fab..c16074ee32dc6 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -29,8 +29,9 @@
 #include <boost/filesystem.hpp>  // NOLINT
 
 #include "arrow/io/file.h"
-#include "arrow/ipc/file.h"
 #include "arrow/ipc/json.h"
+#include "arrow/ipc/reader.h"
+#include "arrow/ipc/writer.h"
 #include "arrow/pretty_print.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 0091067c3225a..f12529b5c585e 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -50,6 +50,18 @@ struct MetadataVersion {
   enum type { V1, V2 };
 };
 
+static constexpr const char* kArrowMagicBytes = "ARROW1";
+
+struct ARROW_EXPORT FileBlock {
+  FileBlock() {}
+  FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
+      : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
+
+  int64_t offset;
+  int32_t metadata_length;
+  int64_t body_length;
+};
+
 //----------------------------------------------------------------------
 
 using DictionaryMap = std::unordered_map<int64_t, std::shared_ptr<Array>>;
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/reader.cc
similarity index 63%
rename from cpp/src/arrow/ipc/file.cc
rename to cpp/src/arrow/ipc/reader.cc
index c1d483f1fbba6..1a9af7db3dcdc 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -15,11 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/ipc/file.h"
+#include "arrow/ipc/reader.h"
 
 #include <cstdint>
 #include <cstring>
 #include <sstream>
+#include <string>
 #include <vector>
 
 #include "arrow/buffer.h"
@@ -35,83 +36,154 @@
 namespace arrow {
 namespace ipc {
 
-static constexpr const char* kArrowMagicBytes = "ARROW1";
+// ----------------------------------------------------------------------
+// StreamReader implementation
 
-static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
-FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
-  std::vector<flatbuf::Block> fb_blocks;
+static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
+  return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
+}
 
-  for (const FileBlock& block : blocks) {
-    fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
+static inline std::string message_type_name(Message::Type type) {
+  switch (type) {
+    case Message::SCHEMA:
+      return "schema";
+    case Message::RECORD_BATCH:
+      return "record batch";
+    case Message::DICTIONARY_BATCH:
+      return "dictionary";
+    default:
+      break;
   }
-
-  return fbb.CreateVectorOfStructs(fb_blocks);
+  return "unknown";
 }
 
-Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
-    io::OutputStream* out) {
-  FBB fbb;
+class StreamReader::StreamReaderImpl {
+ public:
+  StreamReaderImpl() {}
+  ~StreamReaderImpl() {}
 
-  flatbuffers::Offset<flatbuf::Schema> fb_schema;
-  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
+  Status Open(const std::shared_ptr<io::InputStream>& stream) {
+    stream_ = stream;
+    return ReadSchema();
+  }
 
-  auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
-  auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
+  Status ReadNextMessage(Message::Type expected_type, std::shared_ptr<Message>* message) {
+    std::shared_ptr<Buffer> buffer;
+    RETURN_NOT_OK(stream_->Read(sizeof(int32_t), &buffer));
 
-  auto footer = flatbuf::CreateFooter(
-      fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
+    if (buffer->size() != sizeof(int32_t)) {
+      *message = nullptr;
+      return Status::OK();
+    }
 
-  fbb.Finish(footer);
+    int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
 
-  int32_t size = fbb.GetSize();
+    RETURN_NOT_OK(stream_->Read(message_length, &buffer));
+    if (buffer->size() != message_length) {
+      return Status::IOError("Unexpected end of stream trying to read message");
+    }
 
-  return out->Write(fbb.GetBufferPointer(), size);
-}
+    RETURN_NOT_OK(Message::Open(buffer, 0, message));
 
-static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
-  return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
-}
+    if ((*message)->type() != expected_type) {
+      std::stringstream ss;
+      ss << "Message not expected type: " << message_type_name(expected_type)
+         << ", was: " << (*message)->type();
+      return Status::IOError(ss.str());
+    }
+    return Status::OK();
+  }
 
-// ----------------------------------------------------------------------
-// File writer implementation
+  Status ReadExact(int64_t size, std::shared_ptr<Buffer>* buffer) {
+    RETURN_NOT_OK(stream_->Read(size, buffer));
 
-FileWriter::FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
-    : StreamWriter(sink, schema) {}
+    if ((*buffer)->size() < size) {
+      return Status::IOError("Unexpected EOS when reading buffer");
+    }
+    return Status::OK();
+  }
 
-Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-    std::shared_ptr<FileWriter>* out) {
-  *out = std::shared_ptr<FileWriter>(new FileWriter(sink, schema));  // ctor is private
-  RETURN_NOT_OK((*out)->UpdatePosition());
-  return Status::OK();
-}
+  Status ReadNextDictionary() {
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(ReadNextMessage(Message::DICTIONARY_BATCH, &message));
 
-Status FileWriter::Start() {
-  RETURN_NOT_OK(WriteAligned(
-      reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
+    DictionaryBatchMetadata metadata(message);
 
-  // We write the schema at the start of the file (and the end). This also
-  // writes all the dictionaries at the beginning of the file
-  return StreamWriter::Start();
-}
+    std::shared_ptr<Buffer> batch_body;
+    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body))
+    io::BufferReader reader(batch_body);
 
-Status FileWriter::Close() {
-  // Write metadata
-  int64_t initial_position = position_;
-  RETURN_NOT_OK(WriteFileFooter(
-      *schema_, dictionaries_, record_batches_, dictionary_memo_.get(), sink_));
-  RETURN_NOT_OK(UpdatePosition());
+    std::shared_ptr<Array> dictionary;
+    RETURN_NOT_OK(ReadDictionary(metadata, dictionary_types_, &reader, &dictionary));
+    return dictionary_memo_.AddDictionary(metadata.id(), dictionary);
+  }
 
-  // Write footer length
-  int32_t footer_length = position_ - initial_position;
+  Status ReadSchema() {
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(ReadNextMessage(Message::SCHEMA, &message));
+
+    SchemaMetadata schema_meta(message);
+    RETURN_NOT_OK(schema_meta.GetDictionaryTypes(&dictionary_types_));
+
+    // TODO(wesm): In future, we may want to reconcile the ids in the stream with
+    // those found in the schema
+    int num_dictionaries = static_cast<int>(dictionary_types_.size());
+    for (int i = 0; i < num_dictionaries; ++i) {
+      RETURN_NOT_OK(ReadNextDictionary());
+    }
+
+    return schema_meta.GetSchema(dictionary_memo_, &schema_);
+  }
+
+  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(ReadNextMessage(Message::RECORD_BATCH, &message));
+
+    if (message == nullptr) {
+      // End of stream
+      *batch = nullptr;
+      return Status::OK();
+    }
+
+    RecordBatchMetadata batch_metadata(message);
 
-  if (footer_length <= 0) { return Status::Invalid("Invalid file footer"); }
+    std::shared_ptr<Buffer> batch_body;
+    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body));
+    io::BufferReader reader(batch_body);
+    return ReadRecordBatch(batch_metadata, schema_, &reader, batch);
+  }
+
+  std::shared_ptr<Schema> schema() const { return schema_; }
 
-  RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(&footer_length), sizeof(int32_t)));
+ private:
+  // dictionary_id -> type
+  DictionaryTypeMap dictionary_types_;
+
+  DictionaryMemo dictionary_memo_;
+
+  std::shared_ptr<io::InputStream> stream_;
+  std::shared_ptr<Schema> schema_;
+};
+
+StreamReader::StreamReader() {
+  impl_.reset(new StreamReaderImpl());
+}
+
+StreamReader::~StreamReader() {}
+
+Status StreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
+    std::shared_ptr<StreamReader>* reader) {
+  // Private ctor
+  *reader = std::shared_ptr<StreamReader>(new StreamReader());
+  return (*reader)->impl_->Open(stream);
+}
+
+std::shared_ptr<Schema> StreamReader::schema() const {
+  return impl_->schema();
+}
 
-  // Write magic bytes to end file
-  return Write(
-      reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes));
+Status StreamReader::GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+  return impl_->GetNextRecordBatch(batch);
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/file.h b/cpp/src/arrow/ipc/reader.h
similarity index 83%
rename from cpp/src/arrow/ipc/file.h
rename to cpp/src/arrow/ipc/reader.h
index 524766ccb3336..6f143e1a1265e 100644
--- a/cpp/src/arrow/ipc/file.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -25,7 +25,6 @@
 #include <vector>
 
 #include "arrow/ipc/metadata.h"
-#include "arrow/ipc/stream.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -37,29 +36,31 @@ class Status;
 
 namespace io {
 
-class OutputStream;
+class InputStream;
 class ReadableFileInterface;
 
 }  // namespace io
 
 namespace ipc {
 
-Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
-    io::OutputStream* out);
-
-class ARROW_EXPORT FileWriter : public StreamWriter {
+class ARROW_EXPORT StreamReader {
  public:
-  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-      std::shared_ptr<FileWriter>* out);
+  ~StreamReader();
+
+  // Open an stream.
+  static Status Open(const std::shared_ptr<io::InputStream>& stream,
+      std::shared_ptr<StreamReader>* reader);
+
+  std::shared_ptr<Schema> schema() const;
 
-  using StreamWriter::WriteRecordBatch;
-  Status Close() override;
+  // Returned batch is nullptr when end of stream reached
+  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch);
 
  private:
-  FileWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema);
+  StreamReader();
 
-  Status Start() override;
+  class ARROW_NO_EXPORT StreamReaderImpl;
+  std::unique_ptr<StreamReaderImpl> impl_;
 };
 
 class ARROW_EXPORT FileReader {
diff --git a/cpp/src/arrow/util/stream-to-file.cc b/cpp/src/arrow/ipc/stream-to-file.cc
similarity index 96%
rename from cpp/src/arrow/util/stream-to-file.cc
rename to cpp/src/arrow/ipc/stream-to-file.cc
index 393b07d8d355f..ec0ac435a9d0d 100644
--- a/cpp/src/arrow/util/stream-to-file.cc
+++ b/cpp/src/arrow/ipc/stream-to-file.cc
@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/ipc/stream.h"
 #include "arrow/io/file.h"
-#include "arrow/ipc/file.h"
+#include "arrow/ipc/reader.h"
+#include "arrow/ipc/writer.h"
 #include "arrow/status.h"
 #include <iostream>
 
diff --git a/cpp/src/arrow/ipc/stream.cc b/cpp/src/arrow/ipc/stream.cc
deleted file mode 100644
index 7f5c9932330be..0000000000000
--- a/cpp/src/arrow/ipc/stream.cc
+++ /dev/null
@@ -1,310 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/ipc/stream.h"
-
-#include <cstdint>
-#include <cstring>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#include "arrow/buffer.h"
-#include "arrow/io/interfaces.h"
-#include "arrow/io/memory.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/metadata-internal.h"
-#include "arrow/ipc/metadata.h"
-#include "arrow/ipc/util.h"
-#include "arrow/memory_pool.h"
-#include "arrow/schema.h"
-#include "arrow/status.h"
-#include "arrow/table.h"
-#include "arrow/util/logging.h"
-
-namespace arrow {
-namespace ipc {
-
-// ----------------------------------------------------------------------
-// Stream writer implementation
-
-StreamWriter::StreamWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
-    : sink_(sink),
-      schema_(schema),
-      dictionary_memo_(std::make_shared<DictionaryMemo>()),
-      pool_(default_memory_pool()),
-      position_(-1),
-      started_(false) {}
-
-Status StreamWriter::UpdatePosition() {
-  return sink_->Tell(&position_);
-}
-
-Status StreamWriter::Write(const uint8_t* data, int64_t nbytes) {
-  RETURN_NOT_OK(sink_->Write(data, nbytes));
-  position_ += nbytes;
-  return Status::OK();
-}
-
-Status StreamWriter::Align() {
-  int64_t remainder = PaddedLength(position_) - position_;
-  if (remainder > 0) { return Write(kPaddingBytes, remainder); }
-  return Status::OK();
-}
-
-Status StreamWriter::WriteAligned(const uint8_t* data, int64_t nbytes) {
-  RETURN_NOT_OK(Write(data, nbytes));
-  return Align();
-}
-
-Status StreamWriter::CheckStarted() {
-  if (!started_) { return Start(); }
-  return Status::OK();
-}
-
-Status StreamWriter::WriteRecordBatch(const RecordBatch& batch, FileBlock* block) {
-  RETURN_NOT_OK(CheckStarted());
-
-  block->offset = position_;
-
-  // Frame of reference in file format is 0, see ARROW-384
-  const int64_t buffer_start_offset = 0;
-  RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(batch, buffer_start_offset, sink_,
-      &block->metadata_length, &block->body_length, pool_));
-  RETURN_NOT_OK(UpdatePosition());
-
-  DCHECK(position_ % 8 == 0) << "WriteRecordBatch did not perform aligned writes";
-
-  return Status::OK();
-}
-
-void StreamWriter::set_memory_pool(MemoryPool* pool) {
-  pool_ = pool;
-}
-
-// ----------------------------------------------------------------------
-// StreamWriter implementation
-
-Status StreamWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-    std::shared_ptr<StreamWriter>* out) {
-  // ctor is private
-  *out = std::shared_ptr<StreamWriter>(new StreamWriter(sink, schema));
-  RETURN_NOT_OK((*out)->UpdatePosition());
-  return Status::OK();
-}
-
-Status StreamWriter::Start() {
-  std::shared_ptr<Buffer> schema_fb;
-  RETURN_NOT_OK(WriteSchemaMessage(*schema_, dictionary_memo_.get(), &schema_fb));
-
-  int32_t flatbuffer_size = schema_fb->size();
-  RETURN_NOT_OK(
-      Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
-
-  // Write the flatbuffer
-  RETURN_NOT_OK(Write(schema_fb->data(), flatbuffer_size));
-
-  // If there are any dictionaries, write them as the next messages
-  RETURN_NOT_OK(WriteDictionaries());
-
-  started_ = true;
-  return Status::OK();
-}
-
-Status StreamWriter::WriteRecordBatch(const RecordBatch& batch) {
-  // Push an empty FileBlock. Can be written in the footer later
-  record_batches_.emplace_back(0, 0, 0);
-  return WriteRecordBatch(batch, &record_batches_[record_batches_.size() - 1]);
-}
-
-Status StreamWriter::WriteDictionaries() {
-  const DictionaryMap& id_to_dictionary = dictionary_memo_->id_to_dictionary();
-
-  dictionaries_.resize(id_to_dictionary.size());
-
-  // TODO(wesm): does sorting by id yield any benefit?
-  int dict_index = 0;
-  for (const auto& entry : id_to_dictionary) {
-    FileBlock* block = &dictionaries_[dict_index++];
-
-    block->offset = position_;
-
-    // Frame of reference in file format is 0, see ARROW-384
-    const int64_t buffer_start_offset = 0;
-    RETURN_NOT_OK(WriteDictionary(entry.first, entry.second, buffer_start_offset, sink_,
-        &block->metadata_length, &block->body_length, pool_));
-    RETURN_NOT_OK(UpdatePosition());
-    DCHECK(position_ % 8 == 0) << "WriteDictionary did not perform aligned writes";
-  }
-
-  return Status::OK();
-}
-
-Status StreamWriter::Close() {
-  // Write the schema if not already written
-  // User is responsible for closing the OutputStream
-  return CheckStarted();
-}
-
-// ----------------------------------------------------------------------
-// StreamReader implementation
-
-static inline std::string message_type_name(Message::Type type) {
-  switch (type) {
-    case Message::SCHEMA:
-      return "schema";
-    case Message::RECORD_BATCH:
-      return "record batch";
-    case Message::DICTIONARY_BATCH:
-      return "dictionary";
-    default:
-      break;
-  }
-  return "unknown";
-}
-
-class StreamReader::StreamReaderImpl {
- public:
-  StreamReaderImpl() {}
-  ~StreamReaderImpl() {}
-
-  Status Open(const std::shared_ptr<io::InputStream>& stream) {
-    stream_ = stream;
-    return ReadSchema();
-  }
-
-  Status ReadNextMessage(Message::Type expected_type, std::shared_ptr<Message>* message) {
-    std::shared_ptr<Buffer> buffer;
-    RETURN_NOT_OK(stream_->Read(sizeof(int32_t), &buffer));
-
-    if (buffer->size() != sizeof(int32_t)) {
-      *message = nullptr;
-      return Status::OK();
-    }
-
-    int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
-
-    RETURN_NOT_OK(stream_->Read(message_length, &buffer));
-    if (buffer->size() != message_length) {
-      return Status::IOError("Unexpected end of stream trying to read message");
-    }
-
-    RETURN_NOT_OK(Message::Open(buffer, 0, message));
-
-    if ((*message)->type() != expected_type) {
-      std::stringstream ss;
-      ss << "Message not expected type: " << message_type_name(expected_type)
-         << ", was: " << (*message)->type();
-      return Status::IOError(ss.str());
-    }
-    return Status::OK();
-  }
-
-  Status ReadExact(int64_t size, std::shared_ptr<Buffer>* buffer) {
-    RETURN_NOT_OK(stream_->Read(size, buffer));
-
-    if ((*buffer)->size() < size) {
-      return Status::IOError("Unexpected EOS when reading buffer");
-    }
-    return Status::OK();
-  }
-
-  Status ReadNextDictionary() {
-    std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::DICTIONARY_BATCH, &message));
-
-    DictionaryBatchMetadata metadata(message);
-
-    std::shared_ptr<Buffer> batch_body;
-    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body))
-    io::BufferReader reader(batch_body);
-
-    std::shared_ptr<Array> dictionary;
-    RETURN_NOT_OK(ReadDictionary(metadata, dictionary_types_, &reader, &dictionary));
-    return dictionary_memo_.AddDictionary(metadata.id(), dictionary);
-  }
-
-  Status ReadSchema() {
-    std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::SCHEMA, &message));
-
-    SchemaMetadata schema_meta(message);
-    RETURN_NOT_OK(schema_meta.GetDictionaryTypes(&dictionary_types_));
-
-    // TODO(wesm): In future, we may want to reconcile the ids in the stream with
-    // those found in the schema
-    int num_dictionaries = static_cast<int>(dictionary_types_.size());
-    for (int i = 0; i < num_dictionaries; ++i) {
-      RETURN_NOT_OK(ReadNextDictionary());
-    }
-
-    return schema_meta.GetSchema(dictionary_memo_, &schema_);
-  }
-
-  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
-    std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::RECORD_BATCH, &message));
-
-    if (message == nullptr) {
-      // End of stream
-      *batch = nullptr;
-      return Status::OK();
-    }
-
-    RecordBatchMetadata batch_metadata(message);
-
-    std::shared_ptr<Buffer> batch_body;
-    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body));
-    io::BufferReader reader(batch_body);
-    return ReadRecordBatch(batch_metadata, schema_, &reader, batch);
-  }
-
-  std::shared_ptr<Schema> schema() const { return schema_; }
-
- private:
-  // dictionary_id -> type
-  DictionaryTypeMap dictionary_types_;
-
-  DictionaryMemo dictionary_memo_;
-
-  std::shared_ptr<io::InputStream> stream_;
-  std::shared_ptr<Schema> schema_;
-};
-
-StreamReader::StreamReader() {
-  impl_.reset(new StreamReaderImpl());
-}
-
-StreamReader::~StreamReader() {}
-
-Status StreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
-    std::shared_ptr<StreamReader>* reader) {
-  // Private ctor
-  *reader = std::shared_ptr<StreamReader>(new StreamReader());
-  return (*reader)->impl_->Open(stream);
-}
-
-std::shared_ptr<Schema> StreamReader::schema() const {
-  return impl_->schema();
-}
-
-Status StreamReader::GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
-  return impl_->GetNextRecordBatch(batch);
-}
-
-}  // namespace ipc
-}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
new file mode 100644
index 0000000000000..975b0d10cae7d
--- /dev/null
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -0,0 +1,287 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/writer.h"
+
+#include <cstdint>
+#include <cstring>
+#include <sstream>
+#include <vector>
+
+#include "arrow/buffer.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
+#include "arrow/ipc/adapter.h"
+#include "arrow/ipc/metadata-internal.h"
+#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/util.h"
+#include "arrow/memory_pool.h"
+#include "arrow/schema.h"
+#include "arrow/status.h"
+#include "arrow/table.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+namespace ipc {
+
+// ----------------------------------------------------------------------
+// Stream writer implementation
+
+class StreamWriter::StreamWriterImpl {
+ public:
+  StreamWriterImpl()
+      : dictionary_memo_(std::make_shared<DictionaryMemo>()),
+        pool_(default_memory_pool()),
+        position_(-1),
+        started_(false) {}
+
+  virtual ~StreamWriterImpl() = default;
+
+  Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema) {
+    sink_ = sink;
+    schema_ = schema;
+    return UpdatePosition();
+  }
+
+  virtual Status Start() {
+    std::shared_ptr<Buffer> schema_fb;
+    RETURN_NOT_OK(WriteSchemaMessage(*schema_, dictionary_memo_.get(), &schema_fb));
+
+    int32_t flatbuffer_size = schema_fb->size();
+    RETURN_NOT_OK(
+        Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
+
+    // Write the flatbuffer
+    RETURN_NOT_OK(Write(schema_fb->data(), flatbuffer_size));
+
+    // If there are any dictionaries, write them as the next messages
+    RETURN_NOT_OK(WriteDictionaries());
+
+    started_ = true;
+    return Status::OK();
+  }
+
+  virtual Status Close() {
+    // Write the schema if not already written
+    // User is responsible for closing the OutputStream
+    return CheckStarted();
+  }
+
+  Status CheckStarted() {
+    if (!started_) { return Start(); }
+    return Status::OK();
+  }
+
+  Status UpdatePosition() { return sink_->Tell(&position_); }
+
+  Status WriteDictionaries() {
+    const DictionaryMap& id_to_dictionary = dictionary_memo_->id_to_dictionary();
+
+    dictionaries_.resize(id_to_dictionary.size());
+
+    // TODO(wesm): does sorting by id yield any benefit?
+    int dict_index = 0;
+    for (const auto& entry : id_to_dictionary) {
+      FileBlock* block = &dictionaries_[dict_index++];
+
+      block->offset = position_;
+
+      // Frame of reference in file format is 0, see ARROW-384
+      const int64_t buffer_start_offset = 0;
+      RETURN_NOT_OK(WriteDictionary(entry.first, entry.second, buffer_start_offset, sink_,
+          &block->metadata_length, &block->body_length, pool_));
+      RETURN_NOT_OK(UpdatePosition());
+      DCHECK(position_ % 8 == 0) << "WriteDictionary did not perform aligned writes";
+    }
+
+    return Status::OK();
+  }
+
+  Status WriteRecordBatch(const RecordBatch& batch, FileBlock* block) {
+    RETURN_NOT_OK(CheckStarted());
+
+    block->offset = position_;
+
+    // Frame of reference in file format is 0, see ARROW-384
+    const int64_t buffer_start_offset = 0;
+    RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(batch, buffer_start_offset, sink_,
+        &block->metadata_length, &block->body_length, pool_));
+    RETURN_NOT_OK(UpdatePosition());
+
+    DCHECK(position_ % 8 == 0) << "WriteRecordBatch did not perform aligned writes";
+
+    return Status::OK();
+  }
+
+  Status WriteRecordBatch(const RecordBatch& batch) {
+    // Push an empty FileBlock. Can be written in the footer later
+    record_batches_.emplace_back(0, 0, 0);
+    return WriteRecordBatch(batch, &record_batches_[record_batches_.size() - 1]);
+  }
+
+  // Adds padding bytes if necessary to ensure all memory blocks are written on
+  // 8-byte boundaries.
+  Status Align() {
+    int64_t remainder = PaddedLength(position_) - position_;
+    if (remainder > 0) { return Write(kPaddingBytes, remainder); }
+    return Status::OK();
+  }
+
+  // Write data and update position
+  Status Write(const uint8_t* data, int64_t nbytes) {
+    RETURN_NOT_OK(sink_->Write(data, nbytes));
+    position_ += nbytes;
+    return Status::OK();
+  }
+
+  // Write and align
+  Status WriteAligned(const uint8_t* data, int64_t nbytes) {
+    RETURN_NOT_OK(Write(data, nbytes));
+    return Align();
+  }
+
+  void set_memory_pool(MemoryPool* pool) { pool_ = pool; }
+
+ protected:
+  io::OutputStream* sink_;
+  std::shared_ptr<Schema> schema_;
+
+  // When writing out the schema, we keep track of all the dictionaries we
+  // encounter, as they must be written out first in the stream
+  std::shared_ptr<DictionaryMemo> dictionary_memo_;
+
+  MemoryPool* pool_;
+
+  int64_t position_;
+  bool started_;
+
+  std::vector<FileBlock> dictionaries_;
+  std::vector<FileBlock> record_batches_;
+};
+
+StreamWriter::StreamWriter() {
+  impl_.reset(new StreamWriterImpl());
+}
+
+Status StreamWriter::WriteRecordBatch(const RecordBatch& batch) {
+  return impl_->WriteRecordBatch(batch);
+}
+
+void StreamWriter::set_memory_pool(MemoryPool* pool) {
+  impl_->set_memory_pool(pool);
+}
+
+Status StreamWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    std::shared_ptr<StreamWriter>* out) {
+  // ctor is private
+  *out = std::shared_ptr<StreamWriter>(new StreamWriter());
+  return (*out)->impl_->Open(sink, schema);
+}
+
+Status StreamWriter::Close() {
+  return impl_->Close();
+}
+
+// ----------------------------------------------------------------------
+// File writer implementation
+
+static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
+FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
+  std::vector<flatbuf::Block> fb_blocks;
+
+  for (const FileBlock& block : blocks) {
+    fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
+  }
+
+  return fbb.CreateVectorOfStructs(fb_blocks);
+}
+
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
+    io::OutputStream* out) {
+  FBB fbb;
+
+  flatbuffers::Offset<flatbuf::Schema> fb_schema;
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
+
+  auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
+  auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
+
+  auto footer = flatbuf::CreateFooter(
+      fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
+
+  fbb.Finish(footer);
+
+  int32_t size = fbb.GetSize();
+
+  return out->Write(fbb.GetBufferPointer(), size);
+}
+
+class FileWriter::FileWriterImpl : public StreamWriter::StreamWriterImpl {
+ public:
+  using BASE = StreamWriter::StreamWriterImpl;
+
+  Status Start() override {
+    RETURN_NOT_OK(WriteAligned(
+        reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
+
+    // We write the schema at the start of the file (and the end). This also
+    // writes all the dictionaries at the beginning of the file
+    return BASE::Start();
+  }
+
+  Status Close() override {
+    // Write metadata
+    int64_t initial_position = position_;
+    RETURN_NOT_OK(WriteFileFooter(
+        *schema_, dictionaries_, record_batches_, dictionary_memo_.get(), sink_));
+    RETURN_NOT_OK(UpdatePosition());
+
+    // Write footer length
+    int32_t footer_length = position_ - initial_position;
+
+    if (footer_length <= 0) { return Status::Invalid("Invalid file footer"); }
+
+    RETURN_NOT_OK(
+        Write(reinterpret_cast<const uint8_t*>(&footer_length), sizeof(int32_t)));
+
+    // Write magic bytes to end file
+    return Write(
+        reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes));
+  }
+};
+
+FileWriter::FileWriter() {
+  impl_.reset(new FileWriterImpl());
+}
+
+Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+    std::shared_ptr<FileWriter>* out) {
+  *out = std::shared_ptr<FileWriter>(new FileWriter());  // ctor is private
+  return (*out)->impl_->Open(sink, schema);
+}
+
+Status FileWriter::WriteRecordBatch(const RecordBatch& batch) {
+  return impl_->WriteRecordBatch(batch);
+}
+
+Status FileWriter::Close() {
+  return impl_->Close();
+}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/stream.h b/cpp/src/arrow/ipc/writer.h
similarity index 52%
rename from cpp/src/arrow/ipc/stream.h
rename to cpp/src/arrow/ipc/writer.h
index 1c3f65e49af32..7aff71e18e486 100644
--- a/cpp/src/arrow/ipc/stream.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -39,23 +39,12 @@ class Status;
 
 namespace io {
 
-class InputStream;
 class OutputStream;
 
 }  // namespace io
 
 namespace ipc {
 
-struct ARROW_EXPORT FileBlock {
-  FileBlock() {}
-  FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
-      : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
-
-  int64_t offset;
-  int32_t metadata_length;
-  int64_t body_length;
-};
-
 class ARROW_EXPORT StreamWriter {
  public:
   virtual ~StreamWriter() = default;
@@ -74,61 +63,27 @@ class ARROW_EXPORT StreamWriter {
   void set_memory_pool(MemoryPool* pool);
 
  protected:
-  StreamWriter(io::OutputStream* sink, const std::shared_ptr<Schema>& schema);
-
-  virtual Status Start();
-
-  Status CheckStarted();
-  Status UpdatePosition();
-
-  Status WriteDictionaries();
-
-  Status WriteRecordBatch(const RecordBatch& batch, FileBlock* block);
-
-  // Adds padding bytes if necessary to ensure all memory blocks are written on
-  // 8-byte boundaries.
-  Status Align();
-
-  // Write data and update position
-  Status Write(const uint8_t* data, int64_t nbytes);
-
-  // Write and align
-  Status WriteAligned(const uint8_t* data, int64_t nbytes);
-
-  io::OutputStream* sink_;
-  std::shared_ptr<Schema> schema_;
-
-  // When writing out the schema, we keep track of all the dictionaries we
-  // encounter, as they must be written out first in the stream
-  std::shared_ptr<DictionaryMemo> dictionary_memo_;
-
-  MemoryPool* pool_;
-
-  int64_t position_;
-  bool started_;
-
-  std::vector<FileBlock> dictionaries_;
-  std::vector<FileBlock> record_batches_;
+  StreamWriter();
+  class ARROW_NO_EXPORT StreamWriterImpl;
+  std::unique_ptr<StreamWriterImpl> impl_;
 };
 
-class ARROW_EXPORT StreamReader {
- public:
-  ~StreamReader();
-
-  // Open an stream.
-  static Status Open(const std::shared_ptr<io::InputStream>& stream,
-      std::shared_ptr<StreamReader>* reader);
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
+    io::OutputStream* out);
 
-  std::shared_ptr<Schema> schema() const;
+class ARROW_EXPORT FileWriter : public StreamWriter {
+ public:
+  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+      std::shared_ptr<FileWriter>* out);
 
-  // Returned batch is nullptr when end of stream reached
-  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch);
+  Status WriteRecordBatch(const RecordBatch& batch) override;
+  Status Close() override;
 
  private:
-  StreamReader();
-
-  class ARROW_NO_EXPORT StreamReaderImpl;
-  std::unique_ptr<StreamReaderImpl> impl_;
+  FileWriter();
+  class ARROW_NO_EXPORT FileWriterImpl;
+  std::unique_ptr<FileWriterImpl> impl_;
 };
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 19b1e193d4228..8d9afccf867df 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -68,26 +68,4 @@ if (ARROW_BUILD_BENCHMARKS)
   endif()
 endif()
 
-if (ARROW_IPC AND ARROW_BUILD_UTILITIES)
-  set(UTIL_LINK_LIBS
-    arrow_ipc_static
-    arrow_io_static
-    arrow_static
-    boost_filesystem_static
-    boost_system_static
-    dl)
-
-  if (NOT APPLE)
-    set(UTIL_LINK_LIBS
-      ${UTIL_LINK_LIBS}
-      boost_filesystem_static
-      boost_system_static)
-  endif()
-
-  add_executable(file-to-stream file-to-stream.cc)
-  target_link_libraries(file-to-stream ${UTIL_LINK_LIBS})
-  add_executable(stream-to-file stream-to-file.cc)
-  target_link_libraries(stream-to-file ${UTIL_LINK_LIBS})
-endif()
-
 ADD_ARROW_TEST(bit-util-test)
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
index afc7dbd36e5f0..10c70a96b0ab2 100644
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -23,7 +23,7 @@ from pyarrow.includes.libarrow_io cimport (InputStream, OutputStream,
                                            ReadableFileInterface)
 
 
-cdef extern from "arrow/ipc/stream.h" namespace "arrow::ipc" nogil:
+cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
 
     cdef cppclass CStreamWriter " arrow::ipc::StreamWriter":
         @staticmethod
@@ -43,9 +43,6 @@ cdef extern from "arrow/ipc/stream.h" namespace "arrow::ipc" nogil:
 
         CStatus GetNextRecordBatch(shared_ptr[CRecordBatch]* batch)
 
-
-cdef extern from "arrow/ipc/file.h" namespace "arrow::ipc" nogil:
-
     cdef cppclass CFileWriter " arrow::ipc::FileWriter"(CStreamWriter):
         @staticmethod
         CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,

From dc103feaf0bb07b95f0c81afe0e342f239319dec Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Feb 2017 08:13:29 +0100
Subject: [PATCH 0344/1644] ARROW-557: [Python] Add option to explicitly opt in
 to HDFS tests, do not implicitly skip

I have

```
$ py.test pyarrow/tests/test_hdfs.py
================================== test session starts ==================================
platform linux2 -- Python 2.7.11, pytest-2.9.0, py-1.4.31, pluggy-0.3.1
rootdir: /home/wesm/code/arrow/python, inifile:
collected 15 items

pyarrow/tests/test_hdfs.py sssssssssssssss
```

But

```
$ py.test pyarrow/tests/test_hdfs.py --hdfs -v
================================== test session starts ==================================
platform linux2 -- Python 2.7.11, pytest-2.9.0, py-1.4.31, pluggy-0.3.1 -- /home/wesm/anaconda3/envs/py27/bin/python
cachedir: .cache
rootdir: /home/wesm/code/arrow/python, inifile:
collected 15 items

pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_close PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_download_upload PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_file_context_manager PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_ls PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_mkdir PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_orphaned_file PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_read_multiple_parquet_files SKIPPED
pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_read_whole_file PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_close PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_download_upload PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_file_context_manager PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_ls PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_mkdir PASSED
pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_read_multiple_parquet_files SKIPPED
pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_read_whole_file PASSED
```

The `py.test pyarrow --only-hdfs` option will run only the HDFS tests.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #353 from wesm/ARROW-557 and squashes the following commits:

52e03db [Wes McKinney] Add conftest.py file, hdfs group to opt in to HDFS tests with --hdfs
---
 LICENSE.txt                       | 12 ------
 NOTICE.txt                        |  4 ++
 python/pyarrow/tests/conftest.py  | 62 +++++++++++++++++++++++++++++++
 python/pyarrow/tests/test_hdfs.py |  5 ++-
 4 files changed, 69 insertions(+), 14 deletions(-)
 create mode 100644 python/pyarrow/tests/conftest.py

diff --git a/LICENSE.txt b/LICENSE.txt
index c3bec4385507e..d645695673349 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -200,15 +200,3 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-
---------------------------------------------------------------------------------
-
-This product includes code from Apache Kudu.
-
- * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
-
-Copyright: 2016 The Apache Software Foundation.
-Home page: https://kudu.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 02cb4dd6ee001..e71835c233de6 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -42,6 +42,10 @@ This product includes software from the CMake project
 This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
  * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
 
+This product includes software from the Ibis project (Apache 2.0)
+ * Copyright (c) 2015 Cloudera, Inc.
+ * https://github.com/cloudera/ibis
+
 --------------------------------------------------------------------------------
 
 This product includes code from Apache Kudu, which includes the following in
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
new file mode 100644
index 0000000000000..d5b4b69d97392
--- /dev/null
+++ b/python/pyarrow/tests/conftest.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pytest import skip
+
+
+groups = ['hdfs']
+
+
+def pytest_configure(config):
+    pass
+
+
+def pytest_addoption(parser):
+    for group in groups:
+        parser.addoption('--{0}'.format(group), action='store_true',
+                         default=False,
+                         help=('Enable the {0} test group'.format(group)))
+
+    for group in groups:
+        parser.addoption('--only-{0}'.format(group), action='store_true',
+                         default=False,
+                         help=('Run only the {0} test group'.format(group)))
+
+
+def pytest_runtest_setup(item):
+    only_set = False
+
+    for group in groups:
+        only_flag = '--only-{0}'.format(group)
+        flag = '--{0}'.format(group)
+
+        if item.config.getoption(only_flag):
+            only_set = True
+        elif getattr(item.obj, group, None):
+            if not item.config.getoption(flag):
+                skip('{0} NOT enabled'.format(flag))
+
+    if only_set:
+        skip_item = True
+        for group in groups:
+            only_flag = '--only-{0}'.format(group)
+            if (getattr(item.obj, group, False) and
+                    item.config.getoption(only_flag)):
+                skip_item = False
+
+        if skip_item:
+            skip('Only running some groups with only flags')
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index cb24adb73adc9..b8f7e25233421 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -48,6 +48,7 @@ def hdfs_test_client(driver='libhdfs'):
     return HdfsClient(host, port, user, driver=driver)
 
 
+@pytest.mark.hdfs
 class HdfsTestCases(object):
 
     def _make_test_file(self, hdfs, test_name, test_path, test_data):
@@ -190,7 +191,7 @@ class TestLibHdfs(HdfsTestCases, unittest.TestCase):
     @classmethod
     def check_driver(cls):
         if not io.have_libhdfs():
-            pytest.skip('No libhdfs available on system')
+            pytest.fail('No libhdfs available on system')
 
     def test_hdfs_orphaned_file(self):
         hdfs = hdfs_test_client()
@@ -209,4 +210,4 @@ class TestLibHdfs3(HdfsTestCases, unittest.TestCase):
     @classmethod
     def check_driver(cls):
         if not io.have_libhdfs3():
-            pytest.skip('No libhdfs3 available on system')
+            pytest.fail('No libhdfs3 available on system')

From 01a67f3ff3f43f504dc92b66e04473a8b29053f1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Feb 2017 08:14:10 +0100
Subject: [PATCH 0345/1644] ARROW-493: [C++] Permit large (length > INT32_MAX)
 arrays in memory

This commit relaxes the INT32_MAX length requirement for in-memory data. It does not change the Arrow memory format, nor does it permit arrays over INT32_MAX elements to be included in a RecordBatch message sent in the streaming or file formats.

The purpose of this change is to enable Arrow containers to do zero-copy addressing of large datasets (generally of fixed-size elements) produced by other systems. Should those systems wish to send messages to Java, they will need to break those large arrays up into smaller pieces. We can create utilities to assist in copy-free segmentation of large in-memory datasets into compatible chunksizes.

If the large data is only being used in C++-land, then there are no problems.

This is a helpful change en route to adding an `arrow::Tensor` type per ARROW-550, and probably some other things.

This also includes ARROW-584, as I wanted to be sure that I caught all the places in the codebase where there were imprecise integer conversions.

cc @pcmoritz @robertnishihara

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #352 from wesm/ARROW-493 and squashes the following commits:

013d8cc [Wes McKinney] Fix some more compiler warnings
13c4067 [Wes McKinney] Do not pass CMAKE_CXX_FLAGS to googletest ep
dc50d80 [Wes McKinney] Fix last imprecise conversions
c8e90bc [Wes McKinney] Fix many imprecise integer conversions
6bacdf3 [Wes McKinney] Permit in-memory arrays with more than INT32_MAX elements in Array and Builder classes. Raise if large arrays used in IPC context
---
 ci/travis_before_script_cpp.sh         |   2 +-
 cpp/CMakeLists.txt                     |   6 +-
 cpp/src/arrow/array-dictionary-test.cc |   2 +-
 cpp/src/arrow/array-primitive-test.cc  |  69 ++++++-------
 cpp/src/arrow/array-string-test.cc     |  24 ++---
 cpp/src/arrow/array-test.cc            |  17 +++-
 cpp/src/arrow/array-union-test.cc      |   2 +-
 cpp/src/arrow/array.cc                 |  84 ++++++++--------
 cpp/src/arrow/array.h                  | 132 ++++++++++++-------------
 cpp/src/arrow/buffer.h                 |  14 +--
 cpp/src/arrow/builder.cc               |  79 +++++++--------
 cpp/src/arrow/builder.h                |  63 ++++++------
 cpp/src/arrow/column-benchmark.cc      |   2 +-
 cpp/src/arrow/column.cc                |   6 +-
 cpp/src/arrow/column.h                 |   2 +-
 cpp/src/arrow/compare.cc               |  48 ++++-----
 cpp/src/arrow/compare.h                |   2 +-
 cpp/src/arrow/io/file.cc               |   8 +-
 cpp/src/arrow/io/hdfs.cc               |  15 +--
 cpp/src/arrow/io/io-hdfs-test.cc       |   2 +-
 cpp/src/arrow/ipc/adapter.cc           |  24 +++--
 cpp/src/arrow/ipc/ipc-json-test.cc     |   2 +-
 cpp/src/arrow/ipc/json-internal.cc     |  61 +++++++-----
 cpp/src/arrow/ipc/json.cc              |   4 +-
 cpp/src/arrow/ipc/metadata-internal.cc |   7 +-
 cpp/src/arrow/ipc/reader.cc            |   2 +-
 cpp/src/arrow/ipc/test-common.h        |  24 ++---
 cpp/src/arrow/ipc/writer.cc            |   4 +-
 cpp/src/arrow/pretty_print.cc          |   2 +-
 cpp/src/arrow/schema.cc                |   2 +-
 cpp/src/arrow/schema.h                 |   2 +-
 cpp/src/arrow/status.cc                |   2 +-
 cpp/src/arrow/table-test.cc            |   4 +-
 cpp/src/arrow/table.cc                 |  10 +-
 cpp/src/arrow/table.h                  |  14 +--
 cpp/src/arrow/test-util.h              |  47 ++++-----
 cpp/src/arrow/type.h                   |  12 +--
 cpp/src/arrow/type_traits.h            |  54 +++++++---
 cpp/src/arrow/util/bit-util.cc         |   4 +-
 cpp/src/arrow/util/bit-util.h          |  25 ++---
 python/pyarrow/array.pxd               |   4 +-
 python/pyarrow/array.pyx               |   2 +-
 python/pyarrow/includes/libarrow.pxd   |  16 +--
 python/pyarrow/scalar.pxd              |   8 +-
 python/pyarrow/scalar.pyx              |  10 +-
 python/pyarrow/table.pyx               |   2 +-
 python/src/pyarrow/adapters/builtin.cc |   4 +-
 python/src/pyarrow/adapters/pandas.cc  |  13 ++-
 48 files changed, 508 insertions(+), 436 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index feacf8f8e361a..f804a38e76484 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -36,7 +36,7 @@ CMAKE_COMMON_FLAGS="\
 if [ $TRAVIS_OS_NAME == "linux" ]; then
     cmake -DARROW_TEST_MEMCHECK=on \
           $CMAKE_COMMON_FLAGS \
-          -DARROW_CXXFLAGS=-Werror \
+          -DARROW_CXXFLAGS="-Wconversion -Werror" \
           $CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index be3d4b98cf77f..f6dab788b26d5 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -123,7 +123,9 @@ endif()
 include(SetupCxxFlags)
 
 # Add common flags
-set(CMAKE_CXX_FLAGS "${ARROW_CXXFLAGS} ${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
+set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS "${ARROW_CXXFLAGS} ${CMAKE_CXX_FLAGS}")
 
 # Determine compiler version
 include(CompilerInfo)
@@ -452,7 +454,7 @@ if(ARROW_BUILD_TESTS)
       set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
     endif()
     string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
-    set(GTEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}")
+    set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}")
 
     set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
     set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
diff --git a/cpp/src/arrow/array-dictionary-test.cc b/cpp/src/arrow/array-dictionary-test.cc
index 61381b7671180..0c4e628111a15 100644
--- a/cpp/src/arrow/array-dictionary-test.cc
+++ b/cpp/src/arrow/array-dictionary-test.cc
@@ -95,7 +95,7 @@ TEST(TestDictionary, Equals) {
   ASSERT_FALSE(array->RangeEquals(1, 3, 1, array4));
 
   // ARROW-33 Test slices
-  const int size = array->length();
+  const int64_t size = array->length();
 
   std::shared_ptr<Array> slice, slice2;
   slice = array->Array::Slice(2);
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
index f8bbd774d483c..7b36275cbabfb 100644
--- a/cpp/src/arrow/array-primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -97,7 +97,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     builder_nn_ = std::dynamic_pointer_cast<BuilderType>(tmp);
   }
 
-  void RandomData(int N, double pct_null = 0.1) {
+  void RandomData(int64_t N, double pct_null = 0.1) {
     Attrs::draw(N, &draws_);
 
     valid_bytes_.resize(N);
@@ -105,13 +105,13 @@ class TestPrimitiveBuilder : public TestBuilder {
   }
 
   void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
-    int size = builder->length();
+    int64_t size = builder->length();
 
     auto ex_data = std::make_shared<Buffer>(
         reinterpret_cast<uint8_t*>(draws_.data()), size * sizeof(T));
 
     std::shared_ptr<Buffer> ex_null_bitmap;
-    int32_t ex_null_count = 0;
+    int64_t ex_null_count = 0;
 
     if (nullable) {
       ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
@@ -157,18 +157,18 @@ class TestPrimitiveBuilder : public TestBuilder {
     return std::shared_ptr<DataType>(new Type()); \
   }
 
-#define PINT_DECL(CapType, c_type, LOWER, UPPER) \
-  struct P##CapType {                            \
-    PTYPE_DECL(CapType, c_type);                 \
-    static void draw(int N, vector<T>* draws) {  \
-      test::randint<T>(N, LOWER, UPPER, draws);  \
-    }                                            \
+#define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
+  struct P##CapType {                               \
+    PTYPE_DECL(CapType, c_type);                    \
+    static void draw(int64_t N, vector<T>* draws) { \
+      test::randint<T>(N, LOWER, UPPER, draws);     \
+    }                                               \
   }
 
 #define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)     \
   struct P##CapType {                                  \
     PTYPE_DECL(CapType, c_type);                       \
-    static void draw(int N, vector<T>* draws) {        \
+    static void draw(int64_t N, vector<T>* draws) {    \
       test::random_real<T>(N, 0, LOWER, UPPER, draws); \
     }                                                  \
   }
@@ -191,7 +191,7 @@ struct PBoolean {
 };
 
 template <>
-void TestPrimitiveBuilder<PBoolean>::RandomData(int N, double pct_null) {
+void TestPrimitiveBuilder<PBoolean>::RandomData(int64_t N, double pct_null) {
   draws_.resize(N);
   valid_bytes_.resize(N);
 
@@ -202,12 +202,12 @@ void TestPrimitiveBuilder<PBoolean>::RandomData(int N, double pct_null) {
 template <>
 void TestPrimitiveBuilder<PBoolean>::Check(
     const std::shared_ptr<BooleanBuilder>& builder, bool nullable) {
-  int size = builder->length();
+  int64_t size = builder->length();
 
   auto ex_data = test::bytes_to_null_buffer(draws_);
 
   std::shared_ptr<Buffer> ex_null_bitmap;
-  int32_t ex_null_count = 0;
+  int64_t ex_null_count = 0;
 
   if (nullable) {
     ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
@@ -233,7 +233,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 
   ASSERT_EQ(expected->length(), result->length());
 
-  for (int i = 0; i < result->length(); ++i) {
+  for (int64_t i = 0; i < result->length(); ++i) {
     if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
     bool actual = BitUtil::GetBit(result->data()->data(), i);
     ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
@@ -256,7 +256,7 @@ TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 TYPED_TEST(TestPrimitiveBuilder, TestInit) {
   DECL_TYPE();
 
-  int n = 1000;
+  int64_t n = 1000;
   ASSERT_OK(this->builder_->Reserve(n));
   ASSERT_EQ(BitUtil::NextPower2(n), this->builder_->capacity());
   ASSERT_EQ(BitUtil::NextPower2(TypeTraits<Type>::bytes_required(n)),
@@ -267,15 +267,15 @@ TYPED_TEST(TestPrimitiveBuilder, TestInit) {
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
-  int size = 1000;
-  for (int i = 0; i < size; ++i) {
+  int64_t size = 1000;
+  for (int64_t i = 0; i < size; ++i) {
     ASSERT_OK(this->builder_->AppendNull());
   }
 
   std::shared_ptr<Array> result;
   ASSERT_OK(this->builder_->Finish(&result));
 
-  for (int i = 0; i < size; ++i) {
+  for (int64_t i = 0; i < size; ++i) {
     ASSERT_TRUE(result->IsNull(i)) << i;
   }
 }
@@ -283,7 +283,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
 TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
   DECL_T();
 
-  int size = 1000;
+  int64_t size = 1000;
 
   vector<T>& draws = this->draws_;
   vector<uint8_t>& valid_bytes = this->valid_bytes_;
@@ -294,7 +294,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
 
   this->builder_->Reserve(size);
 
-  int i;
+  int64_t i;
   for (i = 0; i < size; ++i) {
     if (valid_bytes[i] > 0) {
       this->builder_->Append(draws[i]);
@@ -314,7 +314,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
 TYPED_TEST(TestPrimitiveBuilder, Equality) {
   DECL_T();
 
-  const int size = 1000;
+  const int64_t size = 1000;
   this->RandomData(size);
   vector<T>& draws = this->draws_;
   vector<uint8_t>& valid_bytes = this->valid_bytes_;
@@ -326,10 +326,11 @@ TYPED_TEST(TestPrimitiveBuilder, Equality) {
   // Make the not equal array by negating the first valid element with itself.
   const auto first_valid = std::find_if(
       valid_bytes.begin(), valid_bytes.end(), [](uint8_t valid) { return valid > 0; });
-  const int first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
+  const int64_t first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
   // This should be true with a very high probability, but might introduce flakiness
   ASSERT_LT(first_valid_idx, size - 1);
-  draws[first_valid_idx] = ~*reinterpret_cast<int64_t*>(&draws[first_valid_idx]);
+  draws[first_valid_idx] =
+      static_cast<T>(~*reinterpret_cast<int64_t*>(&draws[first_valid_idx]));
   ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &unequal_array));
 
   // test normal equality
@@ -350,7 +351,7 @@ TYPED_TEST(TestPrimitiveBuilder, Equality) {
 TYPED_TEST(TestPrimitiveBuilder, SliceEquality) {
   DECL_T();
 
-  const int size = 1000;
+  const int64_t size = 1000;
   this->RandomData(size);
   vector<T>& draws = this->draws_;
   vector<uint8_t>& valid_bytes = this->valid_bytes_;
@@ -383,7 +384,7 @@ TYPED_TEST(TestPrimitiveBuilder, SliceEquality) {
 TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   DECL_T();
 
-  const int size = 10000;
+  const int64_t size = 10000;
 
   vector<T>& draws = this->draws_;
   vector<uint8_t>& valid_bytes = this->valid_bytes_;
@@ -393,8 +394,8 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   this->builder_->Reserve(1000);
   this->builder_nn_->Reserve(1000);
 
-  int i;
-  int null_count = 0;
+  int64_t i;
+  int64_t null_count = 0;
   // Append the first 1000
   for (i = 0; i < 1000; ++i) {
     if (valid_bytes[i] > 0) {
@@ -440,14 +441,14 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
 TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
   DECL_T();
 
-  int size = 10000;
+  int64_t size = 10000;
   this->RandomData(size);
 
   vector<T>& draws = this->draws_;
   vector<uint8_t>& valid_bytes = this->valid_bytes_;
 
   // first slug
-  int K = 1000;
+  int64_t K = 1000;
 
   ASSERT_OK(this->builder_->Append(draws.data(), K, valid_bytes.data()));
   ASSERT_OK(this->builder_nn_->Append(draws.data(), K));
@@ -470,7 +471,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
-  int n = 1000;
+  int64_t n = 1000;
   ASSERT_OK(this->builder_->Reserve(n));
 
   ASSERT_OK(this->builder_->Advance(100));
@@ -478,14 +479,14 @@ TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
 
   ASSERT_OK(this->builder_->Advance(900));
 
-  int too_many = this->builder_->capacity() - 1000 + 1;
+  int64_t too_many = this->builder_->capacity() - 1000 + 1;
   ASSERT_RAISES(Invalid, this->builder_->Advance(too_many));
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestResize) {
   DECL_TYPE();
 
-  int cap = kMinBuilderCapacity * 2;
+  int64_t cap = kMinBuilderCapacity * 2;
 
   ASSERT_OK(this->builder_->Reserve(cap));
   ASSERT_EQ(cap, this->builder_->capacity());
@@ -510,7 +511,7 @@ template <typename TYPE>
 void CheckSliceApproxEquals() {
   using T = typename TYPE::c_type;
 
-  const int kSize = 50;
+  const int64_t kSize = 50;
   std::vector<T> draws1;
   std::vector<T> draws2;
 
@@ -520,7 +521,7 @@ void CheckSliceApproxEquals() {
 
   // Make the draws equal in the sliced segment, but unequal elsewhere (to
   // catch not using the slice offset)
-  for (int i = 10; i < 30; ++i) {
+  for (int64_t i = 10; i < 30; ++i) {
     draws2[i] = draws1[i];
   }
 
diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index d8a35851c1238..3fdeb3cefe7d2 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -64,7 +64,7 @@ class TestStringArray : public ::testing::Test {
   }
 
   void MakeArray() {
-    length_ = offsets_.size() - 1;
+    length_ = static_cast<int64_t>(offsets_.size()) - 1;
     value_buf_ = test::GetBufferFromVector(chars_);
     offsets_buf_ = test::GetBufferFromVector(offsets_);
     null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
@@ -85,8 +85,8 @@ class TestStringArray : public ::testing::Test {
   std::shared_ptr<Buffer> offsets_buf_;
   std::shared_ptr<Buffer> null_bitmap_;
 
-  int null_count_;
-  int length_;
+  int64_t null_count_;
+  int64_t length_;
 
   std::shared_ptr<StringArray> strings_;
 };
@@ -109,7 +109,7 @@ TEST_F(TestStringArray, TestListFunctions) {
   for (size_t i = 0; i < expected_.size(); ++i) {
     ASSERT_EQ(pos, strings_->value_offset(i));
     ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
-    pos += expected_[i].size();
+    pos += static_cast<int>(expected_[i].size());
   }
 }
 
@@ -131,7 +131,7 @@ TEST_F(TestStringArray, TestGetString) {
 TEST_F(TestStringArray, TestEmptyStringComparison) {
   offsets_ = {0, 0, 0, 0, 0, 0};
   offsets_buf_ = test::GetBufferFromVector(offsets_);
-  length_ = offsets_.size() - 1;
+  length_ = static_cast<int64_t>(offsets_.size() - 1);
 
   auto strings_a = std::make_shared<StringArray>(
       length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
@@ -208,7 +208,7 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
   std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
   std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
 
-  int N = strings.size();
+  int N = static_cast<int>(strings.size());
   int reps = 1000;
 
   for (int j = 0; j < reps; ++j) {
@@ -263,7 +263,7 @@ class TestBinaryArray : public ::testing::Test {
   }
 
   void MakeArray() {
-    length_ = offsets_.size() - 1;
+    length_ = static_cast<int64_t>(offsets_.size() - 1);
     value_buf_ = test::GetBufferFromVector(chars_);
     offsets_buf_ = test::GetBufferFromVector(offsets_);
 
@@ -285,8 +285,8 @@ class TestBinaryArray : public ::testing::Test {
   std::shared_ptr<Buffer> offsets_buf_;
   std::shared_ptr<Buffer> null_bitmap_;
 
-  int null_count_;
-  int length_;
+  int64_t null_count_;
+  int64_t length_;
 
   std::shared_ptr<BinaryArray> strings_;
 };
@@ -305,7 +305,7 @@ TEST_F(TestBinaryArray, TestType) {
 }
 
 TEST_F(TestBinaryArray, TestListFunctions) {
-  int pos = 0;
+  size_t pos = 0;
   for (size_t i = 0; i < expected_.size(); ++i) {
     ASSERT_EQ(pos, strings_->value_offset(i));
     ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
@@ -376,7 +376,7 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
   std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
   std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
 
-  int N = strings.size();
+  int N = static_cast<int>(strings.size());
   int reps = 1000;
 
   for (int j = 0; j < reps; ++j) {
@@ -425,7 +425,7 @@ void CheckSliceEquality() {
   std::vector<std::string> strings = {"foo", "", "bar", "baz", "qux", ""};
   std::vector<uint8_t> is_null = {0, 1, 0, 1, 0, 0};
 
-  int N = strings.size();
+  int N = static_cast<int>(strings.size());
   int reps = 10;
 
   for (int j = 0; j < reps; ++j) {
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 45ab2740b4c16..854ebb20f53ed 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -58,7 +58,7 @@ TEST_F(TestArray, TestLength) {
 
 std::shared_ptr<Array> MakeArrayFromValidBytes(
     const std::vector<uint8_t>& v, MemoryPool* pool) {
-  int32_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
+  int64_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
   std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(v);
 
   BufferBuilder value_builder(pool);
@@ -121,7 +121,7 @@ TEST_F(TestArray, TestIsNull) {
                                       1, 0, 1, 1, 0, 1, 0, 0,
                                       1, 0, 0, 1};
   // clang-format on
-  int32_t null_count = 0;
+  int64_t null_count = 0;
   for (uint8_t x : null_bitmap) {
     if (x == 0) { ++null_count; }
   }
@@ -140,6 +140,19 @@ TEST_F(TestArray, TestIsNull) {
   }
 }
 
+TEST_F(TestArray, BuildLargeInMemoryArray) {
+  const int64_t length = static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1;
+
+  BooleanBuilder builder(default_memory_pool());
+  ASSERT_OK(builder.Reserve(length));
+  ASSERT_OK(builder.Advance(length));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  ASSERT_EQ(length, result->length());
+}
+
 TEST_F(TestArray, TestCopy) {}
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/array-union-test.cc b/cpp/src/arrow/array-union-test.cc
index eb9bd7da31b13..83c3196cab74b 100644
--- a/cpp/src/arrow/array-union-test.cc
+++ b/cpp/src/arrow/array-union-test.cc
@@ -37,7 +37,7 @@ TEST(TestUnionArrayAdHoc, TestSliceEquals) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK(ipc::MakeUnion(&batch));
 
-  const int size = batch->num_rows();
+  const int64_t size = batch->num_rows();
 
   auto CheckUnion = [&size](std::shared_ptr<Array> array) {
     std::shared_ptr<Array> slice, slice2;
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index eb4c210930fb2..284bb57a02b88 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -35,13 +35,13 @@ namespace arrow {
 // doing some computation. To avoid doing this eagerly, we set the null count
 // to -1 (any negative number will do). When Array::null_count is called the
 // first time, the null count will be computed. See ARROW-33
-constexpr int32_t kUnknownNullCount = -1;
+constexpr int64_t kUnknownNullCount = -1;
 
 // ----------------------------------------------------------------------
 // Base array class
 
-Array::Array(const std::shared_ptr<DataType>& type, int32_t length,
-    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
+Array::Array(const std::shared_ptr<DataType>& type, int64_t length,
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
     : type_(type),
       length_(length),
       offset_(offset),
@@ -52,7 +52,7 @@ Array::Array(const std::shared_ptr<DataType>& type, int32_t length,
   if (null_bitmap_) { null_bitmap_data_ = null_bitmap_->data(); }
 }
 
-int32_t Array::null_count() const {
+int64_t Array::null_count() const {
   if (null_count_ < 0) {
     if (null_bitmap_) {
       null_count_ = length_ - CountSetBits(null_bitmap_data_, offset_, length_);
@@ -87,14 +87,14 @@ bool Array::ApproxEquals(const std::shared_ptr<Array>& arr) const {
   return ApproxEquals(*arr);
 }
 
-bool Array::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+bool Array::RangeEquals(int64_t start_idx, int64_t end_idx, int64_t other_start_idx,
     const std::shared_ptr<Array>& other) const {
   if (!other) { return false; }
   return RangeEquals(*other, start_idx, end_idx, other_start_idx);
 }
 
-bool Array::RangeEquals(const Array& other, int32_t start_idx, int32_t end_idx,
-    int32_t other_start_idx) const {
+bool Array::RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
+    int64_t other_start_idx) const {
   bool are_equal = false;
   Status error =
       ArrayRangeEquals(*this, other, start_idx, end_idx, other_start_idx, &are_equal);
@@ -104,15 +104,15 @@ bool Array::RangeEquals(const Array& other, int32_t start_idx, int32_t end_idx,
 
 // Last two parameters are in-out parameters
 static inline void ConformSliceParams(
-    int32_t array_offset, int32_t array_length, int32_t* offset, int32_t* length) {
+    int64_t array_offset, int64_t array_length, int64_t* offset, int64_t* length) {
   DCHECK_LE(*offset, array_length);
   DCHECK_GE(offset, 0);
   *length = std::min(array_length - *offset, *length);
   *offset = array_offset + *offset;
 }
 
-std::shared_ptr<Array> Array::Slice(int32_t offset) const {
-  int32_t slice_length = length_ - offset;
+std::shared_ptr<Array> Array::Slice(int64_t offset) const {
+  int64_t slice_length = length_ - offset;
   return Slice(offset, slice_length);
 }
 
@@ -120,9 +120,9 @@ Status Array::Validate() const {
   return Status::OK();
 }
 
-NullArray::NullArray(int32_t length) : Array(null(), length, nullptr, length) {}
+NullArray::NullArray(int64_t length) : Array(null(), length, nullptr, length) {}
 
-std::shared_ptr<Array> NullArray::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> NullArray::Slice(int64_t offset, int64_t length) const {
   DCHECK_LE(offset, length_);
   length = std::min(length_ - offset, length);
   return std::make_shared<NullArray>(length);
@@ -135,9 +135,9 @@ Status NullArray::Accept(ArrayVisitor* visitor) const {
 // ----------------------------------------------------------------------
 // Primitive array base
 
-PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
+PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int32_t null_count, int32_t offset)
+    int64_t null_count, int64_t offset)
     : Array(type, length, null_bitmap, null_count, offset) {
   data_ = data;
   raw_data_ = data == nullptr ? nullptr : data_->data();
@@ -149,7 +149,7 @@ Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
 }
 
 template <typename T>
-std::shared_ptr<Array> NumericArray<T>::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> NumericArray<T>::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<NumericArray<T>>(
       type_, length, data_, null_bitmap_, kUnknownNullCount, offset);
@@ -173,8 +173,8 @@ template class NumericArray<DoubleType>;
 // ----------------------------------------------------------------------
 // BooleanArray
 
-BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
+BooleanArray::BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
     : PrimitiveArray(std::make_shared<BooleanType>(), length, data, null_bitmap,
           null_count, offset) {}
 
@@ -182,7 +182,7 @@ Status BooleanArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-std::shared_ptr<Array> BooleanArray::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> BooleanArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<BooleanArray>(
       length, data_, null_bitmap_, kUnknownNullCount, offset);
@@ -222,7 +222,7 @@ Status ListArray::Validate() const {
 
   int32_t prev_offset = this->value_offset(0);
   if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
-  for (int32_t i = 1; i <= length_; ++i) {
+  for (int64_t i = 1; i <= length_; ++i) {
     int32_t current_offset = this->value_offset(i);
     if (IsNull(i - 1) && current_offset != prev_offset) {
       std::stringstream ss;
@@ -247,7 +247,7 @@ Status ListArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-std::shared_ptr<Array> ListArray::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> ListArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<ListArray>(
       type_, length, value_offsets_, values_, null_bitmap_, kUnknownNullCount, offset);
@@ -259,15 +259,15 @@ std::shared_ptr<Array> ListArray::Slice(int32_t offset, int32_t length) const {
 static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
 static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
 
-BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& value_offsets,
+BinaryArray::BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
     const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int32_t null_count, int32_t offset)
+    int64_t null_count, int64_t offset)
     : BinaryArray(kBinary, length, value_offsets, data, null_bitmap, null_count, offset) {
 }
 
-BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int32_t length,
+BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
     : Array(type, length, null_bitmap, null_count, offset),
       value_offsets_(value_offsets),
       raw_value_offsets_(nullptr),
@@ -288,15 +288,15 @@ Status BinaryArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-std::shared_ptr<Array> BinaryArray::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> BinaryArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<BinaryArray>(
       length, value_offsets_, data_, null_bitmap_, kUnknownNullCount, offset);
 }
 
-StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& value_offsets,
+StringArray::StringArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
     const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int32_t null_count, int32_t offset)
+    int64_t null_count, int64_t offset)
     : BinaryArray(kString, length, value_offsets, data, null_bitmap, null_count, offset) {
 }
 
@@ -309,7 +309,7 @@ Status StringArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-std::shared_ptr<Array> StringArray::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<StringArray>(
       length, value_offsets_, data_, null_bitmap_, kUnknownNullCount, offset);
@@ -318,15 +318,15 @@ std::shared_ptr<Array> StringArray::Slice(int32_t offset, int32_t length) const
 // ----------------------------------------------------------------------
 // Struct
 
-StructArray::StructArray(const std::shared_ptr<DataType>& type, int32_t length,
+StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::vector<std::shared_ptr<Array>>& children,
-    std::shared_ptr<Buffer> null_bitmap, int32_t null_count, int32_t offset)
+    std::shared_ptr<Buffer> null_bitmap, int64_t null_count, int64_t offset)
     : Array(type, length, null_bitmap, null_count, offset) {
   type_ = type;
   children_ = children;
 }
 
-std::shared_ptr<Array> StructArray::field(int32_t pos) const {
+std::shared_ptr<Array> StructArray::field(int pos) const {
   DCHECK_GT(children_.size(), 0);
   return children_[pos];
 }
@@ -340,7 +340,7 @@ Status StructArray::Validate() const {
 
   if (children_.size() > 0) {
     // Validate fields
-    int32_t array_length = children_[0]->length();
+    int64_t array_length = children_[0]->length();
     size_t idx = 0;
     for (auto it : children_) {
       if (it->length() != array_length) {
@@ -371,7 +371,7 @@ Status StructArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-std::shared_ptr<Array> StructArray::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> StructArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<StructArray>(
       type_, length, children_, null_bitmap_, kUnknownNullCount, offset);
@@ -380,10 +380,10 @@ std::shared_ptr<Array> StructArray::Slice(int32_t offset, int32_t length) const
 // ----------------------------------------------------------------------
 // UnionArray
 
-UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int32_t length,
+UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::vector<std::shared_ptr<Array>>& children,
     const std::shared_ptr<Buffer>& type_ids, const std::shared_ptr<Buffer>& value_offsets,
-    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset)
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
     : Array(type, length, null_bitmap, null_count, offset),
       children_(children),
       type_ids_(type_ids),
@@ -396,7 +396,7 @@ UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int32_t length,
   }
 }
 
-std::shared_ptr<Array> UnionArray::child(int32_t pos) const {
+std::shared_ptr<Array> UnionArray::child(int pos) const {
   DCHECK_GT(children_.size(), 0);
   return children_[pos];
 }
@@ -416,7 +416,7 @@ Status UnionArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-std::shared_ptr<Array> UnionArray::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> UnionArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<UnionArray>(type_, length, children_, type_ids_, value_offsets_,
       null_bitmap_, kUnknownNullCount, offset);
@@ -425,9 +425,9 @@ std::shared_ptr<Array> UnionArray::Slice(int32_t offset, int32_t length) const {
 // ----------------------------------------------------------------------
 // DictionaryArray
 
-Status DictionaryArray::FromBuffer(const std::shared_ptr<DataType>& type, int32_t length,
+Status DictionaryArray::FromBuffer(const std::shared_ptr<DataType>& type, int64_t length,
     const std::shared_ptr<Buffer>& indices, const std::shared_ptr<Buffer>& null_bitmap,
-    int32_t null_count, int32_t offset, std::shared_ptr<DictionaryArray>* out) {
+    int64_t null_count, int64_t offset, std::shared_ptr<DictionaryArray>* out) {
   DCHECK_EQ(type->type, Type::DICTIONARY);
   const auto& dict_type = static_cast<const DictionaryType*>(type.get());
 
@@ -464,7 +464,7 @@ Status DictionaryArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-std::shared_ptr<Array> DictionaryArray::Slice(int32_t offset, int32_t length) const {
+std::shared_ptr<Array> DictionaryArray::Slice(int64_t offset, int64_t length) const {
   std::shared_ptr<Array> sliced_indices = indices_->Slice(offset, length);
   return std::make_shared<DictionaryArray>(type_, sliced_indices);
 }
@@ -476,9 +476,9 @@ std::shared_ptr<Array> DictionaryArray::Slice(int32_t offset, int32_t length) co
     out->reset(new ArrayType(type, length, data, null_bitmap, null_count, offset)); \
     break;
 
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int32_t null_count, int32_t offset, std::shared_ptr<Array>* out) {
+    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
   switch (type->type) {
     MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 8bb914e44ad3d..f20f212c3a825 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -80,30 +80,30 @@ class ARROW_EXPORT ArrayVisitor {
 /// be computed on the first call to null_count()
 class ARROW_EXPORT Array {
  public:
-  Array(const std::shared_ptr<DataType>& type, int32_t length,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0);
+  Array(const std::shared_ptr<DataType>& type, int64_t length,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
 
   virtual ~Array() = default;
 
   /// Determine if a slot is null. For inner loops. Does *not* boundscheck
-  bool IsNull(int i) const {
+  bool IsNull(int64_t i) const {
     return null_bitmap_data_ != nullptr &&
            BitUtil::BitNotSet(null_bitmap_data_, i + offset_);
   }
 
   /// Size in the number of elements this array contains.
-  int32_t length() const { return length_; }
+  int64_t length() const { return length_; }
 
   /// A relative position into another array's data, to enable zero-copy
   /// slicing. This value defaults to zero
-  int32_t offset() const { return offset_; }
+  int64_t offset() const { return offset_; }
 
   /// The number of null entries in the array. If the null count was not known
   /// at time of construction (and set to a negative value), then the null
   /// count will be computed and cached on the first invocation of this
   /// function
-  int32_t null_count() const;
+  int64_t null_count() const;
 
   std::shared_ptr<DataType> type() const { return type_; }
   Type::type type_enum() const { return type_->type; }
@@ -128,11 +128,11 @@ class ARROW_EXPORT Array {
 
   /// Compare if the range of slots specified are equal for the given array and
   /// this array.  end_idx exclusive.  This methods does not bounds check.
-  bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+  bool RangeEquals(int64_t start_idx, int64_t end_idx, int64_t other_start_idx,
       const std::shared_ptr<Array>& other) const;
 
-  bool RangeEquals(const Array& other, int32_t start_idx, int32_t end_idx,
-      int32_t other_start_idx) const;
+  bool RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
+      int64_t other_start_idx) const;
 
   /// Determines if the array is internally consistent.
   ///
@@ -150,20 +150,20 @@ class ARROW_EXPORT Array {
   ///     the length will be adjusted accordingly
   ///
   /// \return a new object wrapped in std::shared_ptr<Array>
-  virtual std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const = 0;
+  virtual std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const = 0;
 
   /// Slice from offset until end of the array
-  std::shared_ptr<Array> Slice(int32_t offset) const;
+  std::shared_ptr<Array> Slice(int64_t offset) const;
 
  protected:
   std::shared_ptr<DataType> type_;
-  int32_t length_;
-  int32_t offset_;
+  int64_t length_;
+  int64_t offset_;
 
   // This member is marked mutable so that it can be modified when null_count()
   // is called from a const context and the null count has to be computed (if
   // it is not already known)
-  mutable int32_t null_count_;
+  mutable int64_t null_count_;
 
   std::shared_ptr<Buffer> null_bitmap_;
   const uint8_t* null_bitmap_data_;
@@ -178,20 +178,20 @@ class ARROW_EXPORT NullArray : public Array {
  public:
   using TypeClass = NullType;
 
-  explicit NullArray(int32_t length);
+  explicit NullArray(int64_t length);
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
 /// Base class for fixed-size logical types
 class ARROW_EXPORT PrimitiveArray : public Array {
  public:
-  PrimitiveArray(const std::shared_ptr<DataType>& type, int32_t length,
+  PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0);
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
 
   /// The memory containing this array's data
   /// This buffer does not account for any slice offset
@@ -214,10 +214,10 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   // metadata
   template <typename T1 = TYPE>
   NumericArray(
-      typename std::enable_if<TypeTraits<T1>::is_parameter_free, int32_t>::type length,
+      typename std::enable_if<TypeTraits<T1>::is_parameter_free, int64_t>::type length,
       const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0)
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0)
       : PrimitiveArray(TypeTraits<T1>::type_singleton(), length, data, null_bitmap,
             null_count, offset) {}
 
@@ -227,9 +227,9 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
-  value_type Value(int i) const { return raw_data()[i]; }
+  value_type Value(int64_t i) const { return raw_data()[i]; }
 };
 
 class ARROW_EXPORT BooleanArray : public PrimitiveArray {
@@ -238,15 +238,15 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
 
   using PrimitiveArray::PrimitiveArray;
 
-  BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0);
+  BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
-  bool Value(int i) const {
+  bool Value(int64_t i) const {
     return BitUtil::GetBit(reinterpret_cast<const uint8_t*>(raw_data_), i + offset_);
   }
 };
@@ -258,10 +258,10 @@ class ARROW_EXPORT ListArray : public Array {
  public:
   using TypeClass = ListType;
 
-  ListArray(const std::shared_ptr<DataType>& type, int32_t length,
+  ListArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Array>& values,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0)
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0)
       : Array(type, length, null_bitmap, null_count, offset) {
     value_offsets_ = value_offsets;
     raw_value_offsets_ = value_offsets == nullptr
@@ -285,15 +285,15 @@ class ARROW_EXPORT ListArray : public Array {
   const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
 
   // Neither of these functions will perform boundschecking
-  int32_t value_offset(int i) const { return raw_value_offsets_[i + offset_]; }
-  int32_t value_length(int i) const {
+  int32_t value_offset(int64_t i) const { return raw_value_offsets_[i + offset_]; }
+  int32_t value_length(int64_t i) const {
     i += offset_;
     return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
   }
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
   std::shared_ptr<Buffer> value_offsets_;
@@ -308,15 +308,15 @@ class ARROW_EXPORT BinaryArray : public Array {
  public:
   using TypeClass = BinaryType;
 
-  BinaryArray(int32_t length, const std::shared_ptr<Buffer>& value_offsets,
+  BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
       const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0);
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
 
   // Return the pointer to the given elements bytes
   // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
   // pointer + offset
-  const uint8_t* GetValue(int i, int32_t* out_length) const {
+  const uint8_t* GetValue(int64_t i, int32_t* out_length) const {
     // Account for base offset
     i += offset_;
 
@@ -334,8 +334,8 @@ class ARROW_EXPORT BinaryArray : public Array {
   const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
 
   // Neither of these functions will perform boundschecking
-  int32_t value_offset(int i) const { return raw_value_offsets_[i + offset_]; }
-  int32_t value_length(int i) const {
+  int32_t value_offset(int64_t i) const { return raw_value_offsets_[i + offset_]; }
+  int32_t value_length(int64_t i) const {
     i += offset_;
     return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
   }
@@ -344,15 +344,15 @@ class ARROW_EXPORT BinaryArray : public Array {
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
   // Constructor that allows sub-classes/builders to propagate there logical type up the
   // class hierarchy.
-  BinaryArray(const std::shared_ptr<DataType>& type, int32_t length,
+  BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0);
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
 
   std::shared_ptr<Buffer> value_offsets_;
   const int32_t* raw_value_offsets_;
@@ -365,14 +365,14 @@ class ARROW_EXPORT StringArray : public BinaryArray {
  public:
   using TypeClass = StringType;
 
-  StringArray(int32_t length, const std::shared_ptr<Buffer>& value_offsets,
+  StringArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
       const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0);
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
 
   // Construct a std::string
   // TODO: std::bad_alloc possibility
-  std::string GetString(int i) const {
+  std::string GetString(int64_t i) const {
     int32_t nchars;
     const uint8_t* str = GetValue(i, &nchars);
     return std::string(reinterpret_cast<const char*>(str), nchars);
@@ -382,7 +382,7 @@ class ARROW_EXPORT StringArray : public BinaryArray {
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
 // ----------------------------------------------------------------------
@@ -392,22 +392,22 @@ class ARROW_EXPORT StructArray : public Array {
  public:
   using TypeClass = StructType;
 
-  StructArray(const std::shared_ptr<DataType>& type, int32_t length,
+  StructArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::vector<std::shared_ptr<Array>>& children,
-      std::shared_ptr<Buffer> null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0);
+      std::shared_ptr<Buffer> null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
 
   Status Validate() const override;
 
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
-  std::shared_ptr<Array> field(int32_t pos) const;
+  std::shared_ptr<Array> field(int pos) const;
 
   const std::vector<std::shared_ptr<Array>>& fields() const { return children_; }
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
   // The child arrays corresponding to each field of the struct data type.
@@ -422,12 +422,12 @@ class ARROW_EXPORT UnionArray : public Array {
   using TypeClass = UnionType;
   using type_id_t = uint8_t;
 
-  UnionArray(const std::shared_ptr<DataType>& type, int32_t length,
+  UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::vector<std::shared_ptr<Array>>& children,
       const std::shared_ptr<Buffer>& type_ids,
       const std::shared_ptr<Buffer>& value_offsets = nullptr,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int32_t null_count = 0,
-      int32_t offset = 0);
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
 
   Status Validate() const override;
 
@@ -442,13 +442,13 @@ class ARROW_EXPORT UnionArray : public Array {
 
   UnionMode mode() const { return static_cast<const UnionType&>(*type_.get()).mode; }
 
-  std::shared_ptr<Array> child(int32_t pos) const;
+  std::shared_ptr<Array> child(int pos) const;
 
   const std::vector<std::shared_ptr<Array>>& children() const { return children_; }
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
   std::vector<std::shared_ptr<Array>> children_;
@@ -487,9 +487,9 @@ class ARROW_EXPORT DictionaryArray : public Array {
 
   // Alternate ctor; other attributes (like null count) are inherited from the
   // passed indices array
-  static Status FromBuffer(const std::shared_ptr<DataType>& type, int32_t length,
+  static Status FromBuffer(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& indices, const std::shared_ptr<Buffer>& null_bitmap,
-      int32_t null_count, int32_t offset, std::shared_ptr<DictionaryArray>* out);
+      int64_t null_count, int64_t offset, std::shared_ptr<DictionaryArray>* out);
 
   Status Validate() const override;
 
@@ -500,7 +500,7 @@ class ARROW_EXPORT DictionaryArray : public Array {
 
   Status Accept(ArrayVisitor* visitor) const override;
 
-  std::shared_ptr<Array> Slice(int32_t offset, int32_t length) const override;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
   const DictionaryType* dict_type_;
@@ -542,8 +542,8 @@ extern template class ARROW_EXPORT NumericArray<TimeType>;
 
 // Create new arrays for logical types that are backed by primitive arrays.
 Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    int32_t length, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int32_t null_count, int32_t offset,
+    int64_t length, const std::shared_ptr<Buffer>& data,
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset,
     std::shared_ptr<Array>* out);
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 9c400b1290394..be91af3556da4 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -165,7 +165,7 @@ class ARROW_EXPORT BufferBuilder {
       : pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
 
   /// Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
-  Status Resize(int32_t elements) {
+  Status Resize(int64_t elements) {
     if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
     RETURN_NOT_OK(buffer_->Resize(elements));
     capacity_ = buffer_->capacity();
@@ -173,7 +173,7 @@ class ARROW_EXPORT BufferBuilder {
     return Status::OK();
   }
 
-  Status Append(const uint8_t* data, int length) {
+  Status Append(const uint8_t* data, int64_t length) {
     if (capacity_ < length + size_) { RETURN_NOT_OK(Resize(length + size_)); }
     UnsafeAppend(data, length);
     return Status::OK();
@@ -187,7 +187,7 @@ class ARROW_EXPORT BufferBuilder {
   }
 
   template <typename T>
-  Status Append(const T* arithmetic_values, int num_elements) {
+  Status Append(const T* arithmetic_values, int64_t num_elements) {
     static_assert(std::is_arithmetic<T>::value,
         "Convenience buffer append only supports arithmetic types");
     return Append(
@@ -195,7 +195,7 @@ class ARROW_EXPORT BufferBuilder {
   }
 
   // Unsafe methods don't check existing size
-  void UnsafeAppend(const uint8_t* data, int length) {
+  void UnsafeAppend(const uint8_t* data, int64_t length) {
     memcpy(data_ + size_, data, length);
     size_ += length;
   }
@@ -208,7 +208,7 @@ class ARROW_EXPORT BufferBuilder {
   }
 
   template <typename T>
-  void UnsafeAppend(const T* arithmetic_values, int num_elements) {
+  void UnsafeAppend(const T* arithmetic_values, int64_t num_elements) {
     static_assert(std::is_arithmetic<T>::value,
         "Convenience buffer append only supports arithmetic types");
     UnsafeAppend(
@@ -221,8 +221,8 @@ class ARROW_EXPORT BufferBuilder {
     capacity_ = size_ = 0;
     return result;
   }
-  int capacity() { return capacity_; }
-  int length() { return size_; }
+  int64_t capacity() { return capacity_; }
+  int64_t length() { return size_; }
 
  private:
   std::shared_ptr<PoolBuffer> buffer_;
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index f5c13f9e77ef1..63e083e76b660 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -43,33 +43,33 @@ Status ArrayBuilder::AppendToBitmap(bool is_valid) {
   return Status::OK();
 }
 
-Status ArrayBuilder::AppendToBitmap(const uint8_t* valid_bytes, int32_t length) {
+Status ArrayBuilder::AppendToBitmap(const uint8_t* valid_bytes, int64_t length) {
   RETURN_NOT_OK(Reserve(length));
 
   UnsafeAppendToBitmap(valid_bytes, length);
   return Status::OK();
 }
 
-Status ArrayBuilder::Init(int32_t capacity) {
-  int32_t to_alloc = BitUtil::CeilByte(capacity) / 8;
+Status ArrayBuilder::Init(int64_t capacity) {
+  int64_t to_alloc = BitUtil::CeilByte(capacity) / 8;
   null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
   RETURN_NOT_OK(null_bitmap_->Resize(to_alloc));
   // Buffers might allocate more then necessary to satisfy padding requirements
-  const int byte_capacity = null_bitmap_->capacity();
+  const int64_t byte_capacity = null_bitmap_->capacity();
   capacity_ = capacity;
   null_bitmap_data_ = null_bitmap_->mutable_data();
   memset(null_bitmap_data_, 0, byte_capacity);
   return Status::OK();
 }
 
-Status ArrayBuilder::Resize(int32_t new_bits) {
+Status ArrayBuilder::Resize(int64_t new_bits) {
   if (!null_bitmap_) { return Init(new_bits); }
-  int32_t new_bytes = BitUtil::CeilByte(new_bits) / 8;
-  int32_t old_bytes = null_bitmap_->size();
+  int64_t new_bytes = BitUtil::CeilByte(new_bits) / 8;
+  int64_t old_bytes = null_bitmap_->size();
   RETURN_NOT_OK(null_bitmap_->Resize(new_bytes));
   null_bitmap_data_ = null_bitmap_->mutable_data();
   // The buffer might be overpadded to deal with padding according to the spec
-  const int32_t byte_capacity = null_bitmap_->capacity();
+  const int64_t byte_capacity = null_bitmap_->capacity();
   capacity_ = new_bits;
   if (old_bytes < new_bytes) {
     memset(null_bitmap_data_ + old_bytes, 0, byte_capacity - old_bytes);
@@ -77,7 +77,7 @@ Status ArrayBuilder::Resize(int32_t new_bits) {
   return Status::OK();
 }
 
-Status ArrayBuilder::Advance(int32_t elements) {
+Status ArrayBuilder::Advance(int64_t elements) {
   if (length_ + elements > capacity_) {
     return Status::Invalid("Builder must be expanded");
   }
@@ -85,16 +85,16 @@ Status ArrayBuilder::Advance(int32_t elements) {
   return Status::OK();
 }
 
-Status ArrayBuilder::Reserve(int32_t elements) {
+Status ArrayBuilder::Reserve(int64_t elements) {
   if (length_ + elements > capacity_) {
     // TODO(emkornfield) power of 2 growth is potentially suboptimal
-    int32_t new_capacity = BitUtil::NextPower2(length_ + elements);
+    int64_t new_capacity = BitUtil::NextPower2(length_ + elements);
     return Resize(new_capacity);
   }
   return Status::OK();
 }
 
-Status ArrayBuilder::SetNotNull(int32_t length) {
+Status ArrayBuilder::SetNotNull(int64_t length) {
   RETURN_NOT_OK(Reserve(length));
   UnsafeSetNotNull(length);
   return Status::OK();
@@ -109,21 +109,21 @@ void ArrayBuilder::UnsafeAppendToBitmap(bool is_valid) {
   ++length_;
 }
 
-void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int32_t length) {
+void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t length) {
   if (valid_bytes == nullptr) {
     UnsafeSetNotNull(length);
     return;
   }
 
-  int byte_offset = length_ / 8;
-  int bit_offset = length_ % 8;
+  int64_t byte_offset = length_ / 8;
+  int64_t bit_offset = length_ % 8;
   uint8_t bitset = null_bitmap_data_[byte_offset];
 
-  for (int32_t i = 0; i < length; ++i) {
+  for (int64_t i = 0; i < length; ++i) {
     if (valid_bytes[i]) {
-      bitset |= (1 << bit_offset);
+      bitset |= BitUtil::kBitmask[bit_offset];
     } else {
-      bitset &= ~(1 << bit_offset);
+      bitset &= BitUtil::kFlippedBitmask[bit_offset];
       ++null_count_;
     }
 
@@ -140,22 +140,22 @@ void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int32_t leng
   length_ += length;
 }
 
-void ArrayBuilder::UnsafeSetNotNull(int32_t length) {
-  const int32_t new_length = length + length_;
+void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
+  const int64_t new_length = length + length_;
 
   // Fill up the bytes until we have a byte alignment
-  int32_t pad_to_byte = 8 - (length_ % 8);
+  int64_t pad_to_byte = 8 - (length_ % 8);
   if (pad_to_byte == 8) { pad_to_byte = 0; }
-  for (int32_t i = 0; i < pad_to_byte; ++i) {
+  for (int64_t i = 0; i < pad_to_byte; ++i) {
     BitUtil::SetBit(null_bitmap_data_, i);
   }
 
   // Fast bitsetting
-  int32_t fast_length = (length - pad_to_byte) / 8;
+  int64_t fast_length = (length - pad_to_byte) / 8;
   memset(null_bitmap_data_ + ((length_ + pad_to_byte) / 8), 255, fast_length);
 
   // Trailing bytes
-  for (int32_t i = length_ + pad_to_byte + (fast_length * 8); i < new_length; ++i) {
+  for (int64_t i = length_ + pad_to_byte + (fast_length * 8); i < new_length; ++i) {
     BitUtil::SetBit(null_bitmap_data_, i);
   }
 
@@ -163,7 +163,7 @@ void ArrayBuilder::UnsafeSetNotNull(int32_t length) {
 }
 
 template <typename T>
-Status PrimitiveBuilder<T>::Init(int32_t capacity) {
+Status PrimitiveBuilder<T>::Init(int64_t capacity) {
   RETURN_NOT_OK(ArrayBuilder::Init(capacity));
   data_ = std::make_shared<PoolBuffer>(pool_);
 
@@ -177,7 +177,7 @@ Status PrimitiveBuilder<T>::Init(int32_t capacity) {
 }
 
 template <typename T>
-Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
+Status PrimitiveBuilder<T>::Resize(int64_t capacity) {
   // XXX: Set floor size for now
   if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
 
@@ -197,11 +197,12 @@ Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
 
 template <typename T>
 Status PrimitiveBuilder<T>::Append(
-    const value_type* values, int32_t length, const uint8_t* valid_bytes) {
+    const value_type* values, int64_t length, const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
 
   if (length > 0) {
-    memcpy(raw_data_ + length_, values, TypeTraits<T>::bytes_required(length));
+    std::memcpy(raw_data_ + length_, values,
+        static_cast<std::size_t>(TypeTraits<T>::bytes_required(length)));
   }
 
   // length_ is update by these
@@ -248,7 +249,7 @@ BooleanBuilder::BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>
   DCHECK_EQ(Type::BOOL, type->type);
 }
 
-Status BooleanBuilder::Init(int32_t capacity) {
+Status BooleanBuilder::Init(int64_t capacity) {
   RETURN_NOT_OK(ArrayBuilder::Init(capacity));
   data_ = std::make_shared<PoolBuffer>(pool_);
 
@@ -261,7 +262,7 @@ Status BooleanBuilder::Init(int32_t capacity) {
   return Status::OK();
 }
 
-Status BooleanBuilder::Resize(int32_t capacity) {
+Status BooleanBuilder::Resize(int64_t capacity) {
   // XXX: Set floor size for now
   if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
 
@@ -294,10 +295,10 @@ Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
 }
 
 Status BooleanBuilder::Append(
-    const uint8_t* values, int32_t length, const uint8_t* valid_bytes) {
+    const uint8_t* values, int64_t length, const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
 
-  for (int i = 0; i < length; ++i) {
+  for (int64_t i = 0; i < length; ++i) {
     // Skip reading from unitialised memory
     // TODO: This actually is only to keep valgrind happy but may or may not
     // have a performance impact.
@@ -333,17 +334,17 @@ ListBuilder::ListBuilder(
       offset_builder_(pool),
       values_(values) {}
 
-Status ListBuilder::Init(int32_t elements) {
-  DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
+Status ListBuilder::Init(int64_t elements) {
+  DCHECK_LT(elements, std::numeric_limits<int64_t>::max());
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
   // one more then requested for offsets
-  return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
+  return offset_builder_.Resize((elements + 1) * sizeof(int64_t));
 }
 
-Status ListBuilder::Resize(int32_t capacity) {
-  DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
+Status ListBuilder::Resize(int64_t capacity) {
+  DCHECK_LT(capacity, std::numeric_limits<int64_t>::max());
   // one more then requested for offsets
-  RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
+  RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int64_t)));
   return ArrayBuilder::Resize(capacity);
 }
 
@@ -351,7 +352,7 @@ Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<Array> items = values_;
   if (!items) { RETURN_NOT_OK(value_builder_->Finish(&items)); }
 
-  RETURN_NOT_OK(offset_builder_.Append<int32_t>(items->length()));
+  RETURN_NOT_OK(offset_builder_.Append<int64_t>(items->length()));
   std::shared_ptr<Buffer> offsets = offset_builder_.Finish();
 
   *out = std::make_shared<ListArray>(
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 0b83b9f3c6862..e642d3c21a2fd 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -37,7 +37,7 @@ namespace arrow {
 
 class Array;
 
-static constexpr int32_t kMinBuilderCapacity = 1 << 5;
+static constexpr int64_t kMinBuilderCapacity = 1 << 5;
 
 /// Base class for all data array builders.
 //
@@ -61,38 +61,38 @@ class ARROW_EXPORT ArrayBuilder {
   /// skip shared pointers and just return a raw pointer
   ArrayBuilder* child(int i) { return children_[i].get(); }
 
-  int num_children() const { return children_.size(); }
+  int num_children() const { return static_cast<int>(children_.size()); }
 
-  int32_t length() const { return length_; }
-  int32_t null_count() const { return null_count_; }
-  int32_t capacity() const { return capacity_; }
+  int64_t length() const { return length_; }
+  int64_t null_count() const { return null_count_; }
+  int64_t capacity() const { return capacity_; }
 
   /// Append to null bitmap
   Status AppendToBitmap(bool is_valid);
   /// Vector append. Treat each zero byte as a null.   If valid_bytes is null
   /// assume all of length bits are valid.
-  Status AppendToBitmap(const uint8_t* valid_bytes, int32_t length);
+  Status AppendToBitmap(const uint8_t* valid_bytes, int64_t length);
   /// Set the next length bits to not null (i.e. valid).
-  Status SetNotNull(int32_t length);
+  Status SetNotNull(int64_t length);
 
   /// Allocates initial capacity requirements for the builder.  In most
   /// cases subclasses should override and call there parent classes
   /// method as well.
-  virtual Status Init(int32_t capacity);
+  virtual Status Init(int64_t capacity);
 
   /// Resizes the null_bitmap array.  In most
   /// cases subclasses should override and call there parent classes
   /// method as well.
-  virtual Status Resize(int32_t new_bits);
+  virtual Status Resize(int64_t new_bits);
 
   /// Ensures there is enough space for adding the number of elements by checking
   /// capacity and calling Resize if necessary.
-  Status Reserve(int32_t elements);
+  Status Reserve(int64_t elements);
 
   /// For cases where raw data was memcpy'd into the internal buffers, allows us
   /// to advance the length of the builder. It is your responsibility to use
   /// this function responsibly.
-  Status Advance(int32_t elements);
+  Status Advance(int64_t elements);
 
   std::shared_ptr<PoolBuffer> null_bitmap() const { return null_bitmap_; }
 
@@ -109,12 +109,12 @@ class ARROW_EXPORT ArrayBuilder {
 
   // When null_bitmap are first appended to the builder, the null bitmap is allocated
   std::shared_ptr<PoolBuffer> null_bitmap_;
-  int32_t null_count_;
+  int64_t null_count_;
   uint8_t* null_bitmap_data_;
 
   // Array length, so far. Also, the index of the next element to be added
-  int32_t length_;
-  int32_t capacity_;
+  int64_t length_;
+  int64_t capacity_;
 
   // Child value array builders. These are owned by this class
   std::vector<std::unique_ptr<ArrayBuilder>> children_;
@@ -127,9 +127,9 @@ class ARROW_EXPORT ArrayBuilder {
   void UnsafeAppendToBitmap(bool is_valid);
   // Vector append. Treat each zero byte as a nullzero. If valid_bytes is null
   // assume all of length bits are valid.
-  void UnsafeAppendToBitmap(const uint8_t* valid_bytes, int32_t length);
+  void UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t length);
   // Set the next length bits to not null (i.e. valid).
-  void UnsafeSetNotNull(int32_t length);
+  void UnsafeSetNotNull(int64_t length);
 
  private:
   DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
@@ -146,7 +146,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   using ArrayBuilder::Advance;
 
   /// Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
-  Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+  Status AppendNulls(const uint8_t* valid_bytes, int64_t length) {
     RETURN_NOT_OK(Reserve(length));
     UnsafeAppendToBitmap(valid_bytes, length);
     return Status::OK();
@@ -165,14 +165,14 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
   Status Append(
-      const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
+      const value_type* values, int64_t length, const uint8_t* valid_bytes = nullptr);
 
   Status Finish(std::shared_ptr<Array>* out) override;
-  Status Init(int32_t capacity) override;
+  Status Init(int64_t capacity) override;
 
   /// Increase the capacity of the builder to accommodate at least the indicated
   /// number of elements
-  Status Resize(int32_t capacity) override;
+  Status Resize(int64_t capacity) override;
 
  protected:
   std::shared_ptr<PoolBuffer> data_;
@@ -246,7 +246,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   using ArrayBuilder::Advance;
 
   /// Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
-  Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+  Status AppendNulls(const uint8_t* valid_bytes, int64_t length) {
     RETURN_NOT_OK(Reserve(length));
     UnsafeAppendToBitmap(valid_bytes, length);
     return Status::OK();
@@ -278,14 +278,14 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
   Status Append(
-      const uint8_t* values, int32_t length, const uint8_t* valid_bytes = nullptr);
+      const uint8_t* values, int64_t length, const uint8_t* valid_bytes = nullptr);
 
   Status Finish(std::shared_ptr<Array>* out) override;
-  Status Init(int32_t capacity) override;
+  Status Init(int64_t capacity) override;
 
   /// Increase the capacity of the builder to accommodate at least the indicated
   /// number of elements
-  Status Resize(int32_t capacity) override;
+  Status Resize(int64_t capacity) override;
 
  protected:
   std::shared_ptr<PoolBuffer> data_;
@@ -318,8 +318,8 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   ListBuilder(
       MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr);
 
-  Status Init(int32_t elements) override;
-  Status Resize(int32_t capacity) override;
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
 
   /// Vector append
@@ -327,7 +327,7 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
   Status Append(
-      const int32_t* offsets, int32_t length, const uint8_t* valid_bytes = nullptr) {
+      const int32_t* offsets, int64_t length, const uint8_t* valid_bytes = nullptr) {
     RETURN_NOT_OK(Reserve(length));
     UnsafeAppendToBitmap(valid_bytes, length);
     offset_builder_.UnsafeAppend<int32_t>(offsets, length);
@@ -341,7 +341,8 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   Status Append(bool is_valid = true) {
     RETURN_NOT_OK(Reserve(1));
     UnsafeAppendToBitmap(is_valid);
-    RETURN_NOT_OK(offset_builder_.Append<int32_t>(value_builder_->length()));
+    RETURN_NOT_OK(
+        offset_builder_.Append<int32_t>(static_cast<int32_t>(value_builder_->length())));
     return Status::OK();
   }
 
@@ -375,7 +376,9 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
     return Append(reinterpret_cast<const uint8_t*>(value), length);
   }
 
-  Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
+  Status Append(const std::string& value) {
+    return Append(value.c_str(), static_cast<int32_t>(value.size()));
+  }
 
   Status Finish(std::shared_ptr<Array>* out) override;
 
@@ -417,7 +420,7 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
   /// will be considered as a null for that field, but users must using app-
   /// end methods or advance methods of the child builders' independently to
   /// insert data.
-  Status Append(int32_t length, const uint8_t* valid_bytes) {
+  Status Append(int64_t length, const uint8_t* valid_bytes) {
     RETURN_NOT_OK(Reserve(length));
     UnsafeAppendToBitmap(valid_bytes, length);
     return Status::OK();
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index 1bab5a8de0ca4..13076a4788689 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -24,7 +24,7 @@
 namespace arrow {
 namespace {
 template <typename ArrayType>
-std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
+std::shared_ptr<Array> MakePrimitive(int64_t length, int64_t null_count = 0) {
   auto pool = default_memory_pool();
   auto data = std::make_shared<PoolBuffer>(pool);
   auto null_bitmap = std::make_shared<PoolBuffer>(pool);
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index 1376f6534ece1..18228700472c4 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -42,15 +42,15 @@ bool ChunkedArray::Equals(const ChunkedArray& other) const {
   // Check contents of the underlying arrays. This checks for equality of
   // the underlying data independently of the chunk size.
   int this_chunk_idx = 0;
-  int32_t this_start_idx = 0;
+  int64_t this_start_idx = 0;
   int other_chunk_idx = 0;
-  int32_t other_start_idx = 0;
+  int64_t other_start_idx = 0;
 
   int64_t elements_compared = 0;
   while (elements_compared < length_) {
     const std::shared_ptr<Array> this_array = chunks_[this_chunk_idx];
     const std::shared_ptr<Array> other_array = other.chunk(other_chunk_idx);
-    int32_t common_length = std::min(
+    int64_t common_length = std::min(
         this_array->length() - this_start_idx, other_array->length() - other_start_idx);
     if (!this_array->RangeEquals(this_start_idx, this_start_idx + common_length,
             other_start_idx, other_array)) {
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
index a28b2665e9c1c..93a34c7c95fdf 100644
--- a/cpp/src/arrow/column.h
+++ b/cpp/src/arrow/column.h
@@ -44,7 +44,7 @@ class ARROW_EXPORT ChunkedArray {
 
   int64_t null_count() const { return null_count_; }
 
-  int num_chunks() const { return chunks_.size(); }
+  int num_chunks() const { return static_cast<int>(chunks_.size()); }
 
   std::shared_ptr<Array> chunk(int i) const { return chunks_[i]; }
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index ff3c59f638bb0..e94fa74ea6589 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -37,8 +37,8 @@ namespace arrow {
 
 class RangeEqualsVisitor : public ArrayVisitor {
  public:
-  RangeEqualsVisitor(const Array& right, int32_t left_start_idx, int32_t left_end_idx,
-      int32_t right_start_idx)
+  RangeEqualsVisitor(const Array& right, int64_t left_start_idx, int64_t left_end_idx,
+      int64_t right_start_idx)
       : right_(right),
         left_start_idx_(left_start_idx),
         left_end_idx_(left_end_idx),
@@ -55,7 +55,7 @@ class RangeEqualsVisitor : public ArrayVisitor {
   inline Status CompareValues(const ArrayType& left) {
     const auto& right = static_cast<const ArrayType&>(right_);
 
-    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
       const bool is_null = left.IsNull(i);
       if (is_null != right.IsNull(o_i) ||
@@ -71,7 +71,7 @@ class RangeEqualsVisitor : public ArrayVisitor {
   bool CompareBinaryRange(const BinaryArray& left) const {
     const auto& right = static_cast<const BinaryArray&>(right_);
 
-    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
       const bool is_null = left.IsNull(i);
       if (is_null != right.IsNull(o_i)) { return false; }
@@ -164,7 +164,7 @@ class RangeEqualsVisitor : public ArrayVisitor {
     const std::shared_ptr<Array>& left_values = left.values();
     const std::shared_ptr<Array>& right_values = right.values();
 
-    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
       const bool is_null = left.IsNull(i);
       if (is_null != right.IsNull(o_i)) { return false; }
@@ -193,15 +193,15 @@ class RangeEqualsVisitor : public ArrayVisitor {
   bool CompareStructs(const StructArray& left) {
     const auto& right = static_cast<const StructArray&>(right_);
     bool equal_fields = true;
-    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
       if (left.IsNull(i) != right.IsNull(o_i)) { return false; }
       if (left.IsNull(i)) continue;
-      for (size_t j = 0; j < left.fields().size(); ++j) {
+      for (int j = 0; j < static_cast<int>(left.fields().size()); ++j) {
         // TODO: really we should be comparing stretches of non-null data rather
         // than looking at one value at a time.
-        const int left_abs_index = i + left.offset();
-        const int right_abs_index = o_i + right.offset();
+        const int64_t left_abs_index = i + left.offset();
+        const int64_t right_abs_index = o_i + right.offset();
 
         equal_fields = left.field(j)->RangeEquals(
             left_abs_index, left_abs_index + 1, right_abs_index, right.field(j));
@@ -243,7 +243,7 @@ class RangeEqualsVisitor : public ArrayVisitor {
     const uint8_t* right_ids = right.raw_type_ids();
 
     uint8_t id, child_num;
-    for (int32_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
       if (left.IsNull(i) != right.IsNull(o_i)) { return false; }
       if (left.IsNull(i)) continue;
@@ -252,8 +252,8 @@ class RangeEqualsVisitor : public ArrayVisitor {
       id = left_ids[i];
       child_num = type_id_to_child_num[id];
 
-      const int left_abs_index = i + left.offset();
-      const int right_abs_index = o_i + right.offset();
+      const int64_t left_abs_index = i + left.offset();
+      const int64_t right_abs_index = o_i + right.offset();
 
       // TODO(wesm): really we should be comparing stretches of non-null data
       // rather than looking at one value at a time.
@@ -294,9 +294,9 @@ class RangeEqualsVisitor : public ArrayVisitor {
 
  protected:
   const Array& right_;
-  int32_t left_start_idx_;
-  int32_t left_end_idx_;
-  int32_t right_start_idx_;
+  int64_t left_start_idx_;
+  int64_t left_end_idx_;
+  int64_t right_start_idx_;
 
   bool result_;
 };
@@ -314,7 +314,7 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       const uint8_t* left_data = left.data()->data();
       const uint8_t* right_data = right.data()->data();
 
-      for (int i = 0; i < left.length(); ++i) {
+      for (int64_t i = 0; i < left.length(); ++i) {
         if (!left.IsNull(i) &&
             BitUtil::GetBit(left_data, i) != BitUtil::GetBit(right_data, i)) {
           result_ = false;
@@ -339,7 +339,7 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     const uint8_t* right_data = right.data()->data() + right.offset() * value_byte_size;
 
     if (left.null_count() > 0) {
-      for (int i = 0; i < left.length(); ++i) {
+      for (int64_t i = 0; i < left.length(); ++i) {
         if (!left.IsNull(i) && memcmp(left_data, right_data, value_byte_size)) {
           return false;
         }
@@ -401,7 +401,7 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
           reinterpret_cast<const int32_t*>(right.value_offsets()->data()) +
           right.offset();
 
-      for (int32_t i = 0; i < left.length() + 1; ++i) {
+      for (int64_t i = 0; i < left.length() + 1; ++i) {
         if (left_offsets[i] - left_offsets[0] != right_offsets[i] - right_offsets[0]) {
           return false;
         }
@@ -437,7 +437,7 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       // ARROW-537: Only compare data in non-null slots
       const int32_t* left_offsets = left.raw_value_offsets();
       const int32_t* right_offsets = right.raw_value_offsets();
-      for (int32_t i = 0; i < left.length(); ++i) {
+      for (int64_t i = 0; i < left.length(); ++i) {
         if (left.IsNull(i)) { continue; }
         if (std::memcmp(left_data + left_offsets[i], right_data + right_offsets[i],
                 left.value_length(i))) {
@@ -496,15 +496,15 @@ inline bool FloatingApproxEquals(
   const T* left_data = left.raw_data();
   const T* right_data = right.raw_data();
 
-  static constexpr T EPSILON = 1E-5;
+  static constexpr T EPSILON = static_cast<T>(1E-5);
 
   if (left.null_count() > 0) {
-    for (int32_t i = 0; i < left.length(); ++i) {
+    for (int64_t i = 0; i < left.length(); ++i) {
       if (left.IsNull(i)) continue;
       if (fabs(left_data[i] - right_data[i]) > EPSILON) { return false; }
     }
   } else {
-    for (int32_t i = 0; i < left.length(); ++i) {
+    for (int64_t i = 0; i < left.length(); ++i) {
       if (fabs(left_data[i] - right_data[i]) > EPSILON) { return false; }
     }
   }
@@ -556,8 +556,8 @@ Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
   return Status::OK();
 }
 
-Status ArrayRangeEquals(const Array& left, const Array& right, int32_t left_start_idx,
-    int32_t left_end_idx, int32_t right_start_idx, bool* are_equal) {
+Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_start_idx,
+    int64_t left_end_idx, int64_t right_start_idx, bool* are_equal) {
   if (&left == &right) {
     *are_equal = true;
   } else if (left.type_enum() != right.type_enum()) {
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index 6a71f9fd573ba..1ddf0497dd3d9 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -40,7 +40,7 @@ Status ARROW_EXPORT ArrayApproxEquals(
 
 /// Returns true if indicated equal-length segment of arrays is exactly equal
 Status ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right,
-    int32_t start_idx, int32_t end_idx, int32_t other_start_idx, bool* are_equal);
+    int64_t start_idx, int64_t end_idx, int64_t other_start_idx, bool* are_equal);
 
 /// Returns true if the type metadata are exactly equal
 Status ARROW_EXPORT TypeEquals(
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index c1f0ea48a9dc9..230c7fe0fb4a0 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -263,9 +263,9 @@ static inline Status FileWrite(int fd, const uint8_t* buffer, int64_t nbytes) {
   if (nbytes > INT32_MAX) {
     return Status::IOError("Unable to write > 2GB blocks to file yet");
   }
-  ret = _write(fd, buffer, static_cast<unsigned int>(nbytes));
+  ret = static_cast<int>(_write(fd, buffer, static_cast<unsigned int>(nbytes)));
 #else
-  ret = write(fd, buffer, nbytes);
+  ret = static_cast<int>(write(fd, buffer, nbytes));
 #endif
 
   if (ret == -1) {
@@ -303,9 +303,9 @@ static inline Status FileClose(int fd) {
   int ret;
 
 #if defined(_MSC_VER)
-  ret = _close(fd);
+  ret = static_cast<int>(_close(fd));
 #else
-  ret = close(fd);
+  ret = static_cast<int>(close(fd));
 #endif
 
   if (ret == -1) { return Status::IOError("error closing file"); }
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 5682f44b94a46..408b85f8daeb7 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -114,7 +114,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     tSize ret;
     if (driver_->HasPread()) {
       ret = driver_->Pread(fs_, file_, static_cast<tOffset>(position),
-          reinterpret_cast<void*>(buffer), nbytes);
+          reinterpret_cast<void*>(buffer), static_cast<tSize>(nbytes));
     } else {
       RETURN_NOT_OK(Seek(position));
       return Read(nbytes, bytes_read, buffer);
@@ -141,7 +141,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     int64_t total_bytes = 0;
     while (total_bytes < nbytes) {
       tSize ret = driver_->Read(fs_, file_, reinterpret_cast<void*>(buffer + total_bytes),
-          std::min<int64_t>(buffer_size_, nbytes - total_bytes));
+          static_cast<tSize>(std::min<int64_t>(buffer_size_, nbytes - total_bytes)));
       RETURN_NOT_OK(CheckReadResult(ret));
       total_bytes += ret;
       if (ret == 0) { break; }
@@ -253,7 +253,8 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
   }
 
   Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
-    tSize ret = driver_->Write(fs_, file_, reinterpret_cast<const void*>(buffer), nbytes);
+    tSize ret = driver_->Write(
+        fs_, file_, reinterpret_cast<const void*>(buffer), static_cast<tSize>(nbytes));
     CHECK_FAILURE(ret, "Write");
     *bytes_written = ret;
     return Status::OK();
@@ -328,7 +329,7 @@ class HdfsClient::HdfsClientImpl {
     if (!config->host.empty()) {
       driver_->BuilderSetNameNode(builder, config->host.c_str());
     }
-    driver_->BuilderSetNameNodePort(builder, config->port);
+    driver_->BuilderSetNameNodePort(builder, static_cast<tPort>(config->port));
     if (!config->user.empty()) {
       driver_->BuilderSetUserName(builder, config->user.c_str());
     }
@@ -411,7 +412,7 @@ class HdfsClient::HdfsClientImpl {
 
     // Allocate additional space for elements
 
-    int vec_offset = listing->size();
+    int vec_offset = static_cast<int>(listing->size());
     listing->resize(vec_offset + num_entries);
 
     for (int i = 0; i < num_entries; ++i) {
@@ -449,8 +450,8 @@ class HdfsClient::HdfsClientImpl {
     int flags = O_WRONLY;
     if (append) flags |= O_APPEND;
 
-    hdfsFile handle = driver_->OpenFile(
-        fs_, path.c_str(), flags, buffer_size, replication, default_block_size);
+    hdfsFile handle = driver_->OpenFile(fs_, path.c_str(), flags, buffer_size,
+        replication, static_cast<tSize>(default_block_size));
 
     if (handle == nullptr) {
       // TODO(wesm): determine cause of failure
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index f0e5a280d3116..648d4baac9b6f 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -49,7 +49,7 @@ class TestHdfsClient : public ::testing::Test {
   }
 
   Status WriteDummyFile(const std::string& path, const uint8_t* buffer, int64_t size,
-      bool append = false, int buffer_size = 0, int replication = 0,
+      bool append = false, int buffer_size = 0, int16_t replication = 0,
       int default_block_size = 0) {
     std::shared_ptr<HdfsOutputStream> file;
     RETURN_NOT_OK(client_->OpenWriteable(
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 2be87a35e7fb3..f11c88a6e1e4b 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -20,6 +20,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
+#include <limits>
 #include <sstream>
 #include <vector>
 
@@ -65,8 +66,14 @@ class RecordBatchWriter : public ArrayVisitor {
     if (max_recursion_depth_ <= 0) {
       return Status::Invalid("Max recursion depth reached");
     }
+
+    if (arr.length() > std::numeric_limits<int32_t>::max()) {
+      return Status::Invalid("Cannot write arrays larger than 2^31 - 1 in length");
+    }
+
     // push back all common elements
-    field_nodes_.push_back(flatbuf::FieldNode(arr.length(), arr.null_count()));
+    field_nodes_.push_back(flatbuf::FieldNode(
+        static_cast<int32_t>(arr.length()), static_cast<int32_t>(arr.null_count())));
     if (arr.null_count() > 0) {
       std::shared_ptr<Buffer> bitmap = arr.null_bitmap();
 
@@ -152,13 +159,14 @@ class RecordBatchWriter : public ArrayVisitor {
     int64_t start_offset;
     RETURN_NOT_OK(dst->Tell(&start_offset));
 
-    int64_t padded_metadata_length = metadata_fb->size() + 4;
-    const int remainder = (padded_metadata_length + start_offset) % 8;
+    int32_t padded_metadata_length = static_cast<int32_t>(metadata_fb->size()) + 4;
+    const int32_t remainder =
+        (padded_metadata_length + static_cast<int32_t>(start_offset)) % 8;
     if (remainder != 0) { padded_metadata_length += 8 - remainder; }
 
     // The returned metadata size includes the length prefix, the flatbuffer,
     // plus padding
-    *metadata_length = static_cast<int32_t>(padded_metadata_length);
+    *metadata_length = padded_metadata_length;
 
     // Write the flatbuffer size prefix including padding
     int32_t flatbuffer_size = padded_metadata_length - 4;
@@ -169,7 +177,8 @@ class RecordBatchWriter : public ArrayVisitor {
     RETURN_NOT_OK(dst->Write(metadata_fb->data(), metadata_fb->size()));
 
     // Write any padding
-    int64_t padding = padded_metadata_length - metadata_fb->size() - 4;
+    int32_t padding =
+        padded_metadata_length - static_cast<int32_t>(metadata_fb->size()) - 4;
     if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
 
     return Status::OK();
@@ -184,7 +193,8 @@ class RecordBatchWriter : public ArrayVisitor {
     RETURN_NOT_OK(dst->Tell(&start_position));
 #endif
 
-    RETURN_NOT_OK(WriteMetadata(batch.num_rows(), *body_length, dst, metadata_length));
+    RETURN_NOT_OK(WriteMetadata(
+        static_cast<int32_t>(batch.num_rows()), *body_length, dst, metadata_length));
 
 #ifndef NDEBUG
     RETURN_NOT_OK(dst->Tell(&current_position));
@@ -430,7 +440,7 @@ class RecordBatchWriter : public ArrayVisitor {
         int32_t* shifted_offsets =
             reinterpret_cast<int32_t*>(shifted_offsets_buffer->mutable_data());
 
-        for (int32_t i = 0; i < array.length(); ++i) {
+        for (int64_t i = 0; i < array.length(); ++i) {
           const uint8_t code = type_ids[i];
           int32_t shift = child_offsets[code];
           if (shift == -1) { child_offsets[code] = shift = unshifted_offsets[i]; }
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 3e759cccbbccc..4c18a496f4c80 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -240,7 +240,7 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
   const int nbatches = 3;
   std::vector<std::shared_ptr<RecordBatch>> batches;
   for (int i = 0; i < nbatches; ++i) {
-    int32_t num_rows = 5 + i * 5;
+    int num_rows = 5 + i * 5;
     std::vector<std::shared_ptr<Array>> arrays;
 
     MakeBatchArrays(schema, num_rows, &arrays);
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 6253cd6b43605..0458b85f0078a 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -355,7 +355,7 @@ class JsonArrayWriter : public ArrayVisitor {
     writer_->String(name);
 
     writer_->Key("count");
-    writer_->Int(arr.length());
+    writer_->Int(static_cast<int32_t>(arr.length()));
 
     RETURN_NOT_OK(arr.Accept(this));
 
@@ -394,7 +394,7 @@ class JsonArrayWriter : public ArrayVisitor {
   template <typename T>
   typename std::enable_if<std::is_base_of<BinaryArray, T>::value, void>::type
   WriteDataValues(const T& arr) {
-    for (int i = 0; i < arr.length(); ++i) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
       int32_t length;
       const char* buf = reinterpret_cast<const char*>(arr.GetValue(i, &length));
 
@@ -430,7 +430,7 @@ class JsonArrayWriter : public ArrayVisitor {
   }
 
   template <typename T>
-  void WriteIntegerField(const char* name, const T* values, int32_t length) {
+  void WriteIntegerField(const char* name, const T* values, int64_t length) {
     writer_->Key(name);
     writer_->StartArray();
     for (int i = 0; i < length; ++i) {
@@ -573,7 +573,7 @@ class JsonSchemaReader {
     const auto& values = obj.GetArray();
 
     fields->resize(values.Size());
-    for (size_t i = 0; i < fields->size(); ++i) {
+    for (rj::SizeType i = 0; i < fields->size(); ++i) {
       RETURN_NOT_OK(GetField(values[i], &(*fields)[i]));
     }
     return Status::OK();
@@ -712,7 +712,7 @@ class JsonSchemaReader {
     const auto& id_array = json_type_codes->value.GetArray();
     for (const rj::Value& val : id_array) {
       DCHECK(val.IsUint());
-      type_codes.push_back(val.GetUint());
+      type_codes.push_back(static_cast<uint8_t>(val.GetUint()));
     }
 
     *type = union_(children, type_codes, mode);
@@ -770,10 +770,38 @@ static inline Status ParseHexValue(const char* data, uint8_t* out) {
   // Error checking
   if (*pos1 != c1 || *pos2 != c2) { return Status::Invalid("Encountered non-hex digit"); }
 
-  *out = (pos1 - kAsciiTable) << 4 | (pos2 - kAsciiTable);
+  *out = static_cast<uint8_t>((pos1 - kAsciiTable) << 4 | (pos2 - kAsciiTable));
   return Status::OK();
 }
 
+template <typename T>
+inline typename std::enable_if<IsSignedInt<T>::value, typename T::c_type>::type
+UnboxValue(const rj::Value& val) {
+  DCHECK(val.IsInt());
+  return static_cast<typename T::c_type>(val.GetInt64());
+}
+
+template <typename T>
+inline typename std::enable_if<IsUnsignedInt<T>::value, typename T::c_type>::type
+UnboxValue(const rj::Value& val) {
+  DCHECK(val.IsUint());
+  return static_cast<typename T::c_type>(val.GetUint64());
+}
+
+template <typename T>
+inline typename std::enable_if<IsFloatingPoint<T>::value, typename T::c_type>::type
+UnboxValue(const rj::Value& val) {
+  DCHECK(val.IsFloat());
+  return static_cast<typename T::c_type>(val.GetDouble());
+}
+
+template <typename T>
+inline typename std::enable_if<std::is_base_of<BooleanType, T>::value, bool>::type
+UnboxValue(const rj::Value& val) {
+  DCHECK(val.IsBool());
+  return val.GetBool();
+}
+
 class JsonArrayReader {
  public:
   explicit JsonArrayReader(MemoryPool* pool) : pool_(pool) {}
@@ -820,22 +848,7 @@ class JsonArrayReader {
       }
 
       const rj::Value& val = json_data_arr[i];
-      if (IsSignedInt<T>::value) {
-        DCHECK(val.IsInt());
-        builder.Append(val.GetInt64());
-      } else if (IsUnsignedInt<T>::value) {
-        DCHECK(val.IsUint());
-        builder.Append(val.GetUint64());
-      } else if (IsFloatingPoint<T>::value) {
-        DCHECK(val.IsFloat());
-        builder.Append(val.GetDouble());
-      } else if (std::is_base_of<BooleanType, T>::value) {
-        DCHECK(val.IsBool());
-        builder.Append(val.GetBool());
-      } else {
-        // We are in the wrong function
-        return Status::Invalid(type->ToString());
-      }
+      builder.Append(UnboxValue<T>(val));
     }
 
     return builder.Finish(array);
@@ -869,13 +882,13 @@ class JsonArrayReader {
         std::string hex_string = val.GetString();
 
         DCHECK(hex_string.size() % 2 == 0) << "Expected base16 hex string";
-        int64_t length = static_cast<int>(hex_string.size()) / 2;
+        int32_t length = static_cast<int>(hex_string.size()) / 2;
 
         if (byte_buffer->size() < length) { RETURN_NOT_OK(byte_buffer->Resize(length)); }
 
         const char* hex_data = hex_string.c_str();
         uint8_t* byte_buffer_data = byte_buffer->mutable_data();
-        for (int64_t j = 0; j < length; ++j) {
+        for (int32_t j = 0; j < length; ++j) {
           RETURN_NOT_OK(ParseHexValue(hex_data + j * 2, &byte_buffer_data[j]));
         }
         RETURN_NOT_OK(builder.Append(byte_buffer_data, length));
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index 773fb74a1767a..a01be191aa8ad 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -69,7 +69,7 @@ class JsonWriter::JsonWriterImpl {
 
     writer_->StartObject();
     writer_->Key("count");
-    writer_->Int(batch.num_rows());
+    writer_->Int(static_cast<int32_t>(batch.num_rows()));
 
     writer_->Key("columns");
     writer_->StartArray();
@@ -158,7 +158,7 @@ class JsonReader::JsonReaderImpl {
     const auto& json_columns = it->value.GetArray();
 
     std::vector<std::shared_ptr<Array>> columns(json_columns.Size());
-    for (size_t i = 0; i < columns.size(); ++i) {
+    for (int i = 0; i < static_cast<int>(columns.size()); ++i) {
       const std::shared_ptr<DataType>& type = schema_->field(i)->type;
       RETURN_NOT_OK(ReadJsonArray(pool_, json_columns[i], type, &columns[i]));
     }
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 7c8ddb93c09d1..1cc4a235b81bd 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -214,7 +214,8 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
         vector_type = flatbuf::VectorType_DATA;
         break;
     }
-    auto offset = flatbuf::CreateVectorLayout(fbb, descr.bit_width(), vector_type);
+    auto offset = flatbuf::CreateVectorLayout(
+        fbb, static_cast<int16_t>(descr.bit_width()), vector_type);
     layout->push_back(offset);
   }
 
@@ -328,7 +329,7 @@ Status FieldFromFlatbufferDictionary(
   std::shared_ptr<DataType> type;
   auto children = field->children();
   std::vector<std::shared_ptr<Field>> child_fields(children->size());
-  for (size_t i = 0; i < children->size(); ++i) {
+  for (int i = 0; i < static_cast<int>(children->size()); ++i) {
     RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), dummy_memo, &child_fields[i]));
   }
 
@@ -350,7 +351,7 @@ Status FieldFromFlatbuffer(const flatbuf::Field* field,
     // children to fully reconstruct the data type
     auto children = field->children();
     std::vector<std::shared_ptr<Field>> child_fields(children->size());
-    for (size_t i = 0; i < children->size(); ++i) {
+    for (int i = 0; i < static_cast<int>(children->size()); ++i) {
       RETURN_NOT_OK(
           FieldFromFlatbuffer(children->Get(i), dictionary_memo, &child_fields[i]));
     }
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 1a9af7db3dcdc..973416670bdfa 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -203,7 +203,7 @@ class FileReader::FileReaderImpl {
     }
 
     std::shared_ptr<Buffer> buffer;
-    int file_end_size = magic_size + sizeof(int32_t);
+    int file_end_size = static_cast<int>(magic_size + sizeof(int32_t));
     RETURN_NOT_OK(file_->ReadAt(footer_offset_ - file_end_size, file_end_size, &buffer));
 
     if (memcmp(buffer->data() + sizeof(int32_t), kArrowMagicBytes, magic_size)) {
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 07f786c4d1d77..dc823662ee1ef 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -51,7 +51,7 @@ const auto kListInt32 = list(int32());
 const auto kListListInt32 = list(kListInt32);
 
 Status MakeRandomInt32Array(
-    int32_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
+    int64_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
   std::shared_ptr<PoolBuffer> data;
   test::MakeRandomInt32PoolBuffer(length, pool, &data);
   Int32Builder builder(pool, int32());
@@ -79,7 +79,7 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
   std::vector<int32_t> list_sizes(num_lists, 0);
   std::vector<int32_t> offsets(
       num_lists + 1, 0);  // +1 so we can shift for nulls. See partial sum below.
-  const int seed = child_array->length();
+  const uint32_t seed = static_cast<uint32_t>(child_array->length());
   if (num_lists > 0) {
     test::rand_uniform_int(num_lists, seed, 0, max_list_size, list_sizes.data());
     // make sure sizes are consistent with null
@@ -89,7 +89,7 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
     std::partial_sum(list_sizes.begin(), list_sizes.end(), ++offsets.begin());
 
     // Force invariants
-    const int child_length = child_array->length();
+    const int64_t child_length = child_array->length();
     offsets[0] = 0;
     std::replace_if(offsets.begin(), offsets.end(),
         [child_length](int32_t offset) { return offset > child_length; }, child_length);
@@ -121,26 +121,26 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
 template <class Builder, class RawType>
 Status MakeRandomBinaryArray(
-    int32_t length, MemoryPool* pool, std::shared_ptr<Array>* out) {
+    int64_t length, MemoryPool* pool, std::shared_ptr<Array>* out) {
   const std::vector<std::string> values = {
       "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
   Builder builder(pool);
-  const auto values_len = values.size();
-  for (int32_t i = 0; i < length; ++i) {
-    int values_index = i % values_len;
+  const size_t values_len = values.size();
+  for (int64_t i = 0; i < length; ++i) {
+    int64_t values_index = i % values_len;
     if (values_index == 0) {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       const std::string& value = values[values_index];
-      RETURN_NOT_OK(
-          builder.Append(reinterpret_cast<const RawType*>(value.data()), value.size()));
+      RETURN_NOT_OK(builder.Append(reinterpret_cast<const RawType*>(value.data()),
+          static_cast<int32_t>(value.size())));
     }
   }
   return builder.Finish(out);
 }
 
 Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
-  const int32_t length = 500;
+  const int64_t length = 500;
   auto string_type = utf8();
   auto binary_type = binary();
   auto f0 = field("f0", string_type);
@@ -302,7 +302,7 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   std::vector<std::shared_ptr<Array>> sparse_children(2);
   std::vector<std::shared_ptr<Array>> dense_children(2);
 
-  const int32_t length = 7;
+  const int64_t length = 7;
 
   std::shared_ptr<Buffer> type_ids_buffer;
   std::vector<uint8_t> type_ids = {5, 10, 5, 5, 10, 10, 5};
@@ -346,7 +346,7 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
 }
 
 Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
-  const int32_t length = 6;
+  const int64_t length = 6;
 
   std::vector<bool> is_valid = {true, true, false, true, true, true};
   std::shared_ptr<Array> dict1, dict2;
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 975b0d10cae7d..58402b588404c 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -61,7 +61,7 @@ class StreamWriter::StreamWriterImpl {
     std::shared_ptr<Buffer> schema_fb;
     RETURN_NOT_OK(WriteSchemaMessage(*schema_, dictionary_memo_.get(), &schema_fb));
 
-    int32_t flatbuffer_size = schema_fb->size();
+    int32_t flatbuffer_size = static_cast<int32_t>(schema_fb->size());
     RETURN_NOT_OK(
         Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
 
@@ -252,7 +252,7 @@ class FileWriter::FileWriterImpl : public StreamWriter::StreamWriterImpl {
     RETURN_NOT_OK(UpdatePosition());
 
     // Write footer length
-    int32_t footer_length = position_ - initial_position;
+    int32_t footer_length = static_cast<int32_t>(position_ - initial_position);
 
     if (footer_length <= 0) { return Status::Invalid("Invalid file footer"); }
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 23c05807c16ee..7e69e42800e79 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -196,7 +196,7 @@ class ArrayPrinter : public ArrayVisitor {
   }
 
   Status PrintChildren(
-      const std::vector<std::shared_ptr<Array>>& fields, int32_t offset, int32_t length) {
+      const std::vector<std::shared_ptr<Array>>& fields, int64_t offset, int64_t length) {
     for (size_t i = 0; i < fields.size(); ++i) {
       Newline();
       std::stringstream ss;
diff --git a/cpp/src/arrow/schema.cc b/cpp/src/arrow/schema.cc
index cd8256e658ec3..aa38fd3dd9260 100644
--- a/cpp/src/arrow/schema.cc
+++ b/cpp/src/arrow/schema.cc
@@ -45,7 +45,7 @@ bool Schema::Equals(const std::shared_ptr<Schema>& other) const {
 std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
   if (fields_.size() > 0 && name_to_index_.size() == 0) {
     for (size_t i = 0; i < fields_.size(); ++i) {
-      name_to_index_[fields_[i]->name] = i;
+      name_to_index_[fields_[i]->name] = static_cast<int>(i);
     }
   }
 
diff --git a/cpp/src/arrow/schema.h b/cpp/src/arrow/schema.h
index 0e1ab5c368e98..37cdbf7d786a4 100644
--- a/cpp/src/arrow/schema.h
+++ b/cpp/src/arrow/schema.h
@@ -47,7 +47,7 @@ class ARROW_EXPORT Schema {
   // Render a string representation of the schema suitable for debugging
   std::string ToString() const;
 
-  int num_fields() const { return fields_.size(); }
+  int num_fields() const { return static_cast<int>(fields_.size()); }
 
  private:
   std::vector<std::shared_ptr<Field>> fields_;
diff --git a/cpp/src/arrow/status.cc b/cpp/src/arrow/status.cc
index e1a242721eccc..3a39c8409a5f7 100644
--- a/cpp/src/arrow/status.cc
+++ b/cpp/src/arrow/status.cc
@@ -18,7 +18,7 @@ namespace arrow {
 
 Status::Status(StatusCode code, const std::string& msg, int16_t posix_code) {
   assert(code != StatusCode::OK);
-  const uint32_t size = msg.size();
+  const uint32_t size = static_cast<uint32_t>(msg.size());
   char* result = new char[size + 7];
   memcpy(result, &size, sizeof(size));
   result[4] = static_cast<char>(code);
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 25f12c4b4300d..36374731cbb49 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -150,7 +150,7 @@ TEST_F(TestTable, Equals) {
 }
 
 TEST_F(TestTable, FromRecordBatches) {
-  const int32_t length = 10;
+  const int64_t length = 10;
   MakeExample1(length);
 
   auto batch1 = std::make_shared<RecordBatch>(schema_, length, arrays_);
@@ -184,7 +184,7 @@ TEST_F(TestTable, FromRecordBatches) {
 }
 
 TEST_F(TestTable, ConcatenateTables) {
-  const int32_t length = 10;
+  const int64_t length = 10;
 
   MakeExample1(length);
   auto batch1 = std::make_shared<RecordBatch>(schema_, length, arrays_);
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 8ac06b8cb7811..6b957c081e502 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -29,7 +29,7 @@
 
 namespace arrow {
 
-RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int num_rows,
+RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
     const std::vector<std::shared_ptr<Array>>& columns)
     : schema_(schema), num_rows_(num_rows), columns_(columns) {}
 
@@ -61,18 +61,18 @@ bool RecordBatch::ApproxEquals(const RecordBatch& other) const {
   return true;
 }
 
-std::shared_ptr<RecordBatch> RecordBatch::Slice(int32_t offset) {
+std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset) {
   return Slice(offset, this->num_rows() - offset);
 }
 
-std::shared_ptr<RecordBatch> RecordBatch::Slice(int32_t offset, int32_t length) {
+std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length) {
   std::vector<std::shared_ptr<Array>> arrays;
   arrays.reserve(num_columns());
   for (const auto& field : columns_) {
     arrays.emplace_back(field->Slice(offset, length));
   }
 
-  int32_t num_rows = std::min(num_rows_ - offset, length);
+  int64_t num_rows = std::min(num_rows_ - offset, length);
   return std::make_shared<RecordBatch>(schema_, num_rows, arrays);
 }
 
@@ -169,7 +169,7 @@ bool Table::Equals(const Table& other) const {
   if (!schema_->Equals(other.schema())) { return false; }
   if (static_cast<int64_t>(columns_.size()) != other.num_columns()) { return false; }
 
-  for (size_t i = 0; i < columns_.size(); i++) {
+  for (int i = 0; i < static_cast<int>(columns_.size()); i++) {
     if (!columns_[i]->Equals(other.column(i))) { return false; }
   }
   return true;
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index fa56824a5a1bc..68f664b38a365 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -40,7 +40,7 @@ class ARROW_EXPORT RecordBatch {
   // num_rows is a parameter to allow for record batches of a particular size not
   // having any materialized columns. Each array should have the same length as
   // num_rows
-  RecordBatch(const std::shared_ptr<Schema>& schema, int32_t num_rows,
+  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
       const std::vector<std::shared_ptr<Array>>& columns);
 
   bool Equals(const RecordBatch& other) const;
@@ -59,18 +59,18 @@ class ARROW_EXPORT RecordBatch {
   const std::string& column_name(int i) const;
 
   // @returns: the number of columns in the table
-  int num_columns() const { return columns_.size(); }
+  int num_columns() const { return static_cast<int>(columns_.size()); }
 
   // @returns: the number of rows (the corresponding length of each column)
-  int32_t num_rows() const { return num_rows_; }
+  int64_t num_rows() const { return num_rows_; }
 
   /// Slice each of the arrays in the record batch and construct a new RecordBatch object
-  std::shared_ptr<RecordBatch> Slice(int32_t offset);
-  std::shared_ptr<RecordBatch> Slice(int32_t offset, int32_t length);
+  std::shared_ptr<RecordBatch> Slice(int64_t offset);
+  std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length);
 
  private:
   std::shared_ptr<Schema> schema_;
-  int32_t num_rows_;
+  int64_t num_rows_;
   std::vector<std::shared_ptr<Array>> columns_;
 };
 
@@ -105,7 +105,7 @@ class ARROW_EXPORT Table {
   std::shared_ptr<Column> column(int i) const { return columns_[i]; }
 
   // @returns: the number of columns in the table
-  int num_columns() const { return columns_.size(); }
+  int num_columns() const { return static_cast<int>(columns_.size()); }
 
   // @returns: the number of rows (the corresponding length of each column)
   int64_t num_rows() const { return num_rows_; }
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index ffc78067d1b97..5c7d04de6dfbb 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -73,16 +73,17 @@ void randint(int64_t N, T lower, T upper, std::vector<T>* out) {
   T val;
   for (int64_t i = 0; i < N; ++i) {
     draw = rng.Uniform64(span);
-    val = lower + static_cast<T>(draw);
+    val = static_cast<T>(draw + lower);
     out->push_back(val);
   }
 }
 
 template <typename T>
-void random_real(int n, uint32_t seed, T min_value, T max_value, std::vector<T>* out) {
+void random_real(
+    int64_t n, uint32_t seed, T min_value, T max_value, std::vector<T>* out) {
   std::mt19937 gen(seed);
   std::uniform_real_distribution<T> d(min_value, max_value);
-  for (int i = 0; i < n; ++i) {
+  for (int64_t i = 0; i < n; ++i) {
     out->push_back(d(gen));
   }
 }
@@ -108,13 +109,13 @@ inline Status CopyBufferFromVector(
 
 static inline Status GetBitmapFromBoolVector(
     const std::vector<bool>& is_valid, std::shared_ptr<Buffer>* result) {
-  int length = static_cast<int>(is_valid.size());
+  int64_t length = static_cast<int64_t>(is_valid.size());
 
   std::shared_ptr<MutableBuffer> buffer;
   RETURN_NOT_OK(GetEmptyBitmap(default_memory_pool(), length, &buffer));
 
   uint8_t* bitmap = buffer->mutable_data();
-  for (int i = 0; i < length; ++i) {
+  for (int64_t i = 0; i < length; ++i) {
     if (is_valid[i]) { BitUtil::SetBit(bitmap, i); }
   }
 
@@ -126,7 +127,7 @@ static inline Status GetBitmapFromBoolVector(
 // and the rest to non-zero (true) values.
 static inline void random_null_bytes(int64_t n, double pct_null, uint8_t* null_bytes) {
   Random rng(random_seed());
-  for (int i = 0; i < n; ++i) {
+  for (int64_t i = 0; i < n; ++i) {
     null_bytes[i] = rng.NextDoubleFraction() > pct_null;
   }
 }
@@ -134,41 +135,41 @@ static inline void random_null_bytes(int64_t n, double pct_null, uint8_t* null_b
 static inline void random_is_valid(
     int64_t n, double pct_null, std::vector<bool>* is_valid) {
   Random rng(random_seed());
-  for (int i = 0; i < n; ++i) {
+  for (int64_t i = 0; i < n; ++i) {
     is_valid->push_back(rng.NextDoubleFraction() > pct_null);
   }
 }
 
-static inline void random_bytes(int n, uint32_t seed, uint8_t* out) {
+static inline void random_bytes(int64_t n, uint32_t seed, uint8_t* out) {
   std::mt19937 gen(seed);
   std::uniform_int_distribution<int> d(0, 255);
 
-  for (int i = 0; i < n; ++i) {
-    out[i] = d(gen) & 0xFF;
+  for (int64_t i = 0; i < n; ++i) {
+    out[i] = static_cast<uint8_t>(d(gen) & 0xFF);
   }
 }
 
-static inline void random_ascii(int n, uint32_t seed, uint8_t* out) {
+static inline void random_ascii(int64_t n, uint32_t seed, uint8_t* out) {
   std::mt19937 gen(seed);
   std::uniform_int_distribution<int> d(65, 122);
 
-  for (int i = 0; i < n; ++i) {
-    out[i] = d(gen) & 0xFF;
+  for (int64_t i = 0; i < n; ++i) {
+    out[i] = static_cast<uint8_t>(d(gen) & 0xFF);
   }
 }
 
 template <typename T>
-void rand_uniform_int(int n, uint32_t seed, T min_value, T max_value, T* out) {
+void rand_uniform_int(int64_t n, uint32_t seed, T min_value, T max_value, T* out) {
   DCHECK(out || (n == 0));
   std::mt19937 gen(seed);
   std::uniform_int_distribution<T> d(min_value, max_value);
-  for (int i = 0; i < n; ++i) {
-    out[i] = d(gen);
+  for (int64_t i = 0; i < n; ++i) {
+    out[i] = static_cast<T>(d(gen));
   }
 }
 
-static inline int null_count(const std::vector<uint8_t>& valid_bytes) {
-  int result = 0;
+static inline int64_t null_count(const std::vector<uint8_t>& valid_bytes) {
+  int64_t result = 0;
   for (size_t i = 0; i < valid_bytes.size(); ++i) {
     if (valid_bytes[i] == 0) { ++result; }
   }
@@ -183,7 +184,7 @@ std::shared_ptr<Buffer> bytes_to_null_buffer(const std::vector<uint8_t>& bytes)
   return out;
 }
 
-Status MakeRandomInt32PoolBuffer(int32_t length, MemoryPool* pool,
+Status MakeRandomInt32PoolBuffer(int64_t length, MemoryPool* pool,
     std::shared_ptr<PoolBuffer>* pool_buffer, uint32_t seed = 0) {
   DCHECK(pool);
   auto data = std::make_shared<PoolBuffer>(pool);
@@ -194,7 +195,7 @@ Status MakeRandomInt32PoolBuffer(int32_t length, MemoryPool* pool,
   return Status::OK();
 }
 
-Status MakeRandomBytePoolBuffer(int32_t length, MemoryPool* pool,
+Status MakeRandomBytePoolBuffer(int64_t length, MemoryPool* pool,
     std::shared_ptr<PoolBuffer>* pool_buffer, uint32_t seed = 0) {
   auto bytes = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(bytes->Resize(length));
@@ -213,7 +214,7 @@ class TestBase : public ::testing::Test {
   }
 
   template <typename ArrayType>
-  std::shared_ptr<Array> MakePrimitive(int32_t length, int32_t null_count = 0) {
+  std::shared_ptr<Array> MakePrimitive(int64_t length, int64_t null_count = 0) {
     auto data = std::make_shared<PoolBuffer>(pool_);
     const int64_t data_nbytes = length * sizeof(typename ArrayType::value_type);
     EXPECT_OK(data->Resize(data_nbytes));
@@ -275,9 +276,9 @@ class TestBuilder : public ::testing::Test {
 
 template <class T, class Builder>
 Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
-    int size, Builder* builder, std::shared_ptr<Array>* out) {
+    int64_t size, Builder* builder, std::shared_ptr<Array>* out) {
   // Append the first 1000
-  for (int i = 0; i < size; ++i) {
+  for (int64_t i = 0; i < size; ++i) {
     if (valid_bytes[i] > 0) {
       RETURN_NOT_OK(builder->Append(values[i]));
     } else {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 9a97fc30094b9..9b1ab3288eb8c 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -162,7 +162,7 @@ struct ARROW_EXPORT DataType {
 
   const std::vector<std::shared_ptr<Field>>& children() const { return children_; }
 
-  int num_children() const { return children_.size(); }
+  int num_children() const { return static_cast<int>(children_.size()); }
 
   virtual Status Accept(TypeVisitor* visitor) const = 0;
 
@@ -226,7 +226,7 @@ struct ARROW_EXPORT CTypeImpl : public PrimitiveCType {
 
   CTypeImpl() : PrimitiveCType(TYPE_ID) {}
 
-  int bit_width() const override { return sizeof(C_TYPE) * 8; }
+  int bit_width() const override { return static_cast<int>(sizeof(C_TYPE) * 8); }
 
   Status Accept(TypeVisitor* visitor) const override {
     return visitor->Visit(*static_cast<const DERIVED*>(this));
@@ -432,7 +432,7 @@ struct ARROW_EXPORT DateType : public FixedWidthType {
 
   DateType() : FixedWidthType(Type::DATE) {}
 
-  int bit_width() const override { return sizeof(c_type) * 8; }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -448,7 +448,7 @@ struct ARROW_EXPORT TimeType : public FixedWidthType {
 
   TimeUnit unit;
 
-  int bit_width() const override { return sizeof(c_type) * 8; }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
 
   explicit TimeType(TimeUnit unit = TimeUnit::MILLI)
       : FixedWidthType(Type::TIME), unit(unit) {}
@@ -465,7 +465,7 @@ struct ARROW_EXPORT TimestampType : public FixedWidthType {
   typedef int64_t c_type;
   static constexpr Type::type type_id = Type::TIMESTAMP;
 
-  int bit_width() const override { return sizeof(int64_t) * 8; }
+  int bit_width() const override { return static_cast<int>(sizeof(int64_t) * 8); }
 
   TimeUnit unit;
 
@@ -485,7 +485,7 @@ struct ARROW_EXPORT IntervalType : public FixedWidthType {
   using c_type = int64_t;
   static constexpr Type::type type_id = Type::INTERVAL;
 
-  int bit_width() const override { return sizeof(int64_t) * 8; }
+  int bit_width() const override { return static_cast<int>(sizeof(int64_t) * 8); }
 
   Unit unit;
 
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index c4898b1ac8ce2..d6687c11bcf73 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -32,7 +32,7 @@ template <>
 struct TypeTraits<UInt8Type> {
   using ArrayType = UInt8Array;
   using BuilderType = UInt8Builder;
-  static inline int bytes_required(int elements) { return elements; }
+  static inline int64_t bytes_required(int64_t elements) { return elements; }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return uint8(); }
 };
@@ -41,7 +41,7 @@ template <>
 struct TypeTraits<Int8Type> {
   using ArrayType = Int8Array;
   using BuilderType = Int8Builder;
-  static inline int bytes_required(int elements) { return elements; }
+  static inline int64_t bytes_required(int64_t elements) { return elements; }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return int8(); }
 };
@@ -51,7 +51,9 @@ struct TypeTraits<UInt16Type> {
   using ArrayType = UInt16Array;
   using BuilderType = UInt16Builder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(uint16_t);
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return uint16(); }
 };
@@ -61,7 +63,9 @@ struct TypeTraits<Int16Type> {
   using ArrayType = Int16Array;
   using BuilderType = Int16Builder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(int16_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(int16_t);
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return int16(); }
 };
@@ -71,7 +75,9 @@ struct TypeTraits<UInt32Type> {
   using ArrayType = UInt32Array;
   using BuilderType = UInt32Builder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(uint32_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(uint32_t);
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return uint32(); }
 };
@@ -81,7 +87,9 @@ struct TypeTraits<Int32Type> {
   using ArrayType = Int32Array;
   using BuilderType = Int32Builder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(int32_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(int32_t);
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return int32(); }
 };
@@ -91,7 +99,9 @@ struct TypeTraits<UInt64Type> {
   using ArrayType = UInt64Array;
   using BuilderType = UInt64Builder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(uint64_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(uint64_t);
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return uint64(); }
 };
@@ -101,7 +111,9 @@ struct TypeTraits<Int64Type> {
   using ArrayType = Int64Array;
   using BuilderType = Int64Builder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(int64_t);
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return int64(); }
 };
@@ -111,7 +123,9 @@ struct TypeTraits<DateType> {
   using ArrayType = DateArray;
   // using BuilderType = DateBuilder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(int64_t);
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return date(); }
 };
@@ -121,7 +135,9 @@ struct TypeTraits<TimestampType> {
   using ArrayType = TimestampArray;
   // using BuilderType = TimestampBuilder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(int64_t);
+  }
   constexpr static bool is_parameter_free = false;
 };
 
@@ -130,7 +146,9 @@ struct TypeTraits<TimeType> {
   using ArrayType = TimeArray;
   // using BuilderType = TimestampBuilder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(int64_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(int64_t);
+  }
   constexpr static bool is_parameter_free = false;
 };
 
@@ -139,7 +157,9 @@ struct TypeTraits<HalfFloatType> {
   using ArrayType = HalfFloatArray;
   using BuilderType = HalfFloatBuilder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(uint16_t); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(uint16_t);
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return float16(); }
 };
@@ -149,7 +169,9 @@ struct TypeTraits<FloatType> {
   using ArrayType = FloatArray;
   using BuilderType = FloatBuilder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(float); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return static_cast<int64_t>(elements * sizeof(float));
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return float32(); }
 };
@@ -159,7 +181,9 @@ struct TypeTraits<DoubleType> {
   using ArrayType = DoubleArray;
   using BuilderType = DoubleBuilder;
 
-  static inline int bytes_required(int elements) { return elements * sizeof(double); }
+  static inline int64_t bytes_required(int64_t elements) {
+    return static_cast<int64_t>(elements * sizeof(double));
+  }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return float64(); }
 };
@@ -169,7 +193,7 @@ struct TypeTraits<BooleanType> {
   using ArrayType = BooleanArray;
   using BuilderType = BooleanBuilder;
 
-  static inline int bytes_required(int elements) {
+  static inline int64_t bytes_required(int64_t elements) {
     return BitUtil::BytesForBits(elements);
   }
   constexpr static bool is_parameter_free = true;
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index f3fbb41fa54a7..1bbd2384267c9 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -42,7 +42,7 @@ void BitUtil::BytesToBits(const std::vector<uint8_t>& bytes, uint8_t* bits) {
 
 Status BitUtil::BytesToBits(
     const std::vector<uint8_t>& bytes, std::shared_ptr<Buffer>* out) {
-  int bit_length = BitUtil::BytesForBits(bytes.size());
+  int64_t bit_length = BitUtil::BytesForBits(bytes.size());
 
   std::shared_ptr<MutableBuffer> buffer;
   RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), bit_length, &buffer));
@@ -98,7 +98,7 @@ Status GetEmptyBitmap(
   return Status::OK();
 }
 
-Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int32_t offset, int32_t length,
+Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t length,
     std::shared_ptr<Buffer>* out) {
   std::shared_ptr<MutableBuffer> buffer;
   RETURN_NOT_OK(GetEmptyBitmap(pool, length, &buffer));
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index a0fbdd2f92ca1..6e3e8ae9f2160 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -34,6 +34,11 @@ class Status;
 
 namespace BitUtil {
 
+static constexpr uint8_t kBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128};
+
+// the ~i byte version of kBitmaks
+static constexpr uint8_t kFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127};
+
 static inline int64_t CeilByte(int64_t size) {
   return (size + 7) & ~7;
 }
@@ -46,28 +51,26 @@ static inline int64_t Ceil2Bytes(int64_t size) {
   return (size + 15) & ~15;
 }
 
-static constexpr uint8_t kBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128};
-
-static inline bool GetBit(const uint8_t* bits, int i) {
+static inline bool GetBit(const uint8_t* bits, int64_t i) {
   return static_cast<bool>(bits[i / 8] & kBitmask[i % 8]);
 }
 
-static inline bool BitNotSet(const uint8_t* bits, int i) {
+static inline bool BitNotSet(const uint8_t* bits, int64_t i) {
   return (bits[i / 8] & kBitmask[i % 8]) == 0;
 }
 
-static inline void ClearBit(uint8_t* bits, int i) {
-  bits[i / 8] &= ~kBitmask[i % 8];
+static inline void ClearBit(uint8_t* bits, int64_t i) {
+  bits[i / 8] &= kFlippedBitmask[i % 8];
 }
 
-static inline void SetBit(uint8_t* bits, int i) {
+static inline void SetBit(uint8_t* bits, int64_t i) {
   bits[i / 8] |= kBitmask[i % 8];
 }
 
-static inline void SetBitTo(uint8_t* bits, int i, bool bit_is_set) {
+static inline void SetBitTo(uint8_t* bits, int64_t i, bool bit_is_set) {
   // See https://graphics.stanford.edu/~seander/bithacks.html
   // "Conditionally set or clear bits without branching"
-  bits[i / 8] ^= (-bit_is_set ^ bits[i / 8]) & kBitmask[i % 8];
+  bits[i / 8] ^= static_cast<uint8_t>(-bit_is_set ^ bits[i / 8]) & kBitmask[i % 8];
 }
 
 static inline int64_t NextPower2(int64_t n) {
@@ -127,8 +130,8 @@ Status ARROW_EXPORT GetEmptyBitmap(
 /// \param[out] out the resulting copy
 ///
 /// \return Status message
-Status ARROW_EXPORT CopyBitmap(MemoryPool* pool, const uint8_t* bitmap, int32_t offset,
-    int32_t length, std::shared_ptr<Buffer>* out);
+Status ARROW_EXPORT CopyBitmap(MemoryPool* pool, const uint8_t* bitmap, int64_t offset,
+    int64_t length, std::shared_ptr<Buffer>* out);
 
 /// Compute the number of 1's in the given data array
 ///
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index 9e4d469bcfa5f..56bb53d5c97dc 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.includes.common cimport shared_ptr
+from pyarrow.includes.common cimport shared_ptr, int64_t
 from pyarrow.includes.libarrow cimport CArray
 
 from pyarrow.scalar import NA
@@ -36,7 +36,7 @@ cdef class Array:
         DataType type
 
     cdef init(self, const shared_ptr[CArray]& sp_array)
-    cdef getitem(self, int i)
+    cdef getitem(self, int64_t i)
 
 cdef object box_array(const shared_ptr[CArray]& sp_array)
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 11abf03e35f1d..7787e95df5e72 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -210,7 +210,7 @@ cdef class Array:
 
         return self.getitem(key)
 
-    cdef getitem(self, int i):
+    cdef getitem(self, int64_t i):
         return scalar.box_scalar(self.type, self.sp_array, i)
 
     def slice(self, offset=0, length=None):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 702acfbc12e17..253cabbe0a581 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -64,15 +64,15 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CArray" arrow::Array":
         shared_ptr[CDataType] type()
 
-        int32_t length()
-        int32_t null_count()
+        int64_t length()
+        int64_t null_count()
         Type type_enum()
 
         c_bool Equals(const shared_ptr[CArray]& arr)
         c_bool IsNull(int i)
 
-        shared_ptr[CArray] Slice(int32_t offset)
-        shared_ptr[CArray] Slice(int32_t offset, int32_t length)
+        shared_ptr[CArray] Slice(int64_t offset)
+        shared_ptr[CArray] Slice(int64_t offset, int64_t length)
 
     cdef cppclass CFixedWidthType" arrow::FixedWidthType"(CDataType):
         int bit_width()
@@ -217,7 +217,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CChunkedArray] data()
 
     cdef cppclass CRecordBatch" arrow::RecordBatch":
-        CRecordBatch(const shared_ptr[CSchema]& schema, int32_t num_rows,
+        CRecordBatch(const shared_ptr[CSchema]& schema, int64_t num_rows,
                      const vector[shared_ptr[CArray]]& columns)
 
         c_bool Equals(const CRecordBatch& other)
@@ -229,10 +229,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         const vector[shared_ptr[CArray]]& columns()
 
         int num_columns()
-        int32_t num_rows()
+        int64_t num_rows()
 
-        shared_ptr[CRecordBatch] Slice(int32_t offset)
-        shared_ptr[CRecordBatch] Slice(int32_t offset, int32_t length)
+        shared_ptr[CRecordBatch] Slice(int64_t offset)
+        shared_ptr[CRecordBatch] Slice(int64_t offset, int64_t length)
 
     cdef cppclass CTable" arrow::Table":
         CTable(const c_string& name, const shared_ptr[CSchema]& schema,
diff --git a/python/pyarrow/scalar.pxd b/python/pyarrow/scalar.pxd
index 2d55757726183..551aeb9697bf7 100644
--- a/python/pyarrow/scalar.pxd
+++ b/python/pyarrow/scalar.pxd
@@ -32,10 +32,10 @@ cdef class NAType(Scalar):
 cdef class ArrayValue(Scalar):
     cdef:
         shared_ptr[CArray] sp_array
-        int index
+        int64_t index
 
     cdef void init(self, DataType type,
-                   const shared_ptr[CArray]& sp_array, int index)
+                   const shared_ptr[CArray]& sp_array, int64_t index)
 
     cdef void _set_array(self, const shared_ptr[CArray]& sp_array)
 
@@ -55,7 +55,7 @@ cdef class ListValue(ArrayValue):
     cdef:
         CListArray* ap
 
-    cdef getitem(self, int i)
+    cdef getitem(self, int64_t i)
 
 
 cdef class StringValue(ArrayValue):
@@ -63,4 +63,4 @@ cdef class StringValue(ArrayValue):
 
 cdef object box_scalar(DataType type,
                        const shared_ptr[CArray]& sp_array,
-                       int index)
+                       int64_t index)
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 57a15ad78344c..1337b2b2cb198 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -46,7 +46,7 @@ NA = NAType()
 cdef class ArrayValue(Scalar):
 
     cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
-                   int index):
+                   int64_t index):
         self.type = type
         self.index = index
         self._set_array(sp_array)
@@ -201,13 +201,13 @@ cdef class ListValue(ArrayValue):
         self.ap = <CListArray*> sp_array.get()
         self.value_type = box_data_type(self.ap.value_type())
 
-    cdef getitem(self, int i):
-        cdef int j = self.ap.value_offset(self.index) + i
+    cdef getitem(self, int64_t i):
+        cdef int64_t j = self.ap.value_offset(self.index) + i
         return box_scalar(self.value_type, self.ap.values(), j)
 
     def as_py(self):
         cdef:
-            int j
+            int64_t j
             list result = []
 
         for j in range(len(self)):
@@ -236,7 +236,7 @@ cdef dict _scalar_classes = {
 }
 
 cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
-                       int index):
+                       int64_t index):
     cdef ArrayValue val
     if type.type.type == Type_NA:
         return NA
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 7d7336246ee79..93bc6ddcd56f6 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -497,7 +497,7 @@ cdef class RecordBatch:
             shared_ptr[CSchema] schema
             shared_ptr[CRecordBatch] batch
             vector[shared_ptr[CArray]] c_arrays
-            int32_t num_rows
+            int64_t num_rows
 
         if len(arrays) == 0:
             raise ValueError('Record batch cannot contain no arrays (for now)')
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index 5fd8eef23fec5..c125cc078af88 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -375,7 +375,7 @@ class BytesConverter : public TypedConverter<arrow::BinaryBuilder> {
     PyObject* bytes_obj;
     OwnedRef tmp;
     const char* bytes;
-    int32_t length;
+    int64_t length;
     Py_ssize_t size = PySequence_Size(seq);
     for (int64_t i = 0; i < size; ++i) {
       item = PySequence_GetItem(seq, i);
@@ -409,7 +409,7 @@ class UTF8Converter : public TypedConverter<arrow::StringBuilder> {
     PyObject* bytes_obj;
     OwnedRef tmp;
     const char* bytes;
-    int32_t length;
+    int64_t length;
     Py_ssize_t size = PySequence_Size(seq);
     for (int64_t i = 0; i < size; ++i) {
       item = PySequence_GetItem(seq, i);
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index bdc2cb7d0025f..cadb53e0d2ab9 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -224,13 +224,13 @@ Status AppendObjectStrings(arrow::StringBuilder& string_builder, PyObject** obje
         PyErr_Clear();
         return Status::TypeError("failed converting unicode to UTF8");
       }
-      const int32_t length = PyBytes_GET_SIZE(obj);
+      const int64_t length = PyBytes_GET_SIZE(obj);
       Status s = string_builder.Append(PyBytes_AS_STRING(obj), length);
       Py_DECREF(obj);
       if (!s.ok()) { return s; }
     } else if (PyBytes_Check(obj)) {
       *have_bytes = true;
-      const int32_t length = PyBytes_GET_SIZE(obj);
+      const int64_t length = PyBytes_GET_SIZE(obj);
       RETURN_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
     } else {
       string_builder.AppendNull();
@@ -413,7 +413,7 @@ inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_val
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
     auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
-    for (int32_t i = 0; i < arr->length(); ++i) {
+    for (int64_t i = 0; i < arr->length(); ++i) {
       *out_values = in_values[i];
     }
   }
@@ -507,7 +507,6 @@ inline Status ConvertListsLike(
     auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
 
     const uint8_t* data_ptr;
-    int32_t length;
     const bool has_nulls = data.null_count() > 0;
     for (int64_t i = 0; i < arr->length(); ++i) {
       if (has_nulls && arr->IsNull(i)) {
@@ -1520,7 +1519,7 @@ inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
   }
 
   // For readability
-  constexpr int32_t kOffset = 0;
+  constexpr int64_t kOffset = 0;
 
   RETURN_NOT_OK(ConvertData());
   std::shared_ptr<DataType> type;
@@ -1636,7 +1635,7 @@ inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
       // TODO(uwe): Support more complex numpy array structures
       RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
 
-      int32_t size = PyArray_DIM(numpy_array, 0);
+      int64_t size = PyArray_DIM(numpy_array, 0);
       auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
       if (traits::supports_nulls) {
         null_bitmap_->Resize(size, false);
@@ -1678,7 +1677,7 @@ ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
       // TODO(uwe): Support more complex numpy array structures
       RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
 
-      int32_t size = PyArray_DIM(numpy_array, 0);
+      int64_t size = PyArray_DIM(numpy_array, 0);
       auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
       RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
     } else if (PyList_Check(objects[i])) {

From 2c3bd9311b370a45bac3ff90ed2f772991f211e0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Feb 2017 14:09:39 -0500
Subject: [PATCH 0346/1644] ARROW-588: [C++] Fix some 32 bit compiler warnings

I also found that $CMAKE_CXX_FLAGS were not being passed to the gflags external project.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #354 from wesm/32-bit-compiler-warnings and squashes the following commits:

8829a58 [Wes McKinney] Fix cast to wrong type
5a17654 [Wes McKinney] clang format
43687c5 [Wes McKinney] Fix some more compiler warnings
843479c [Wes McKinney] Fixes
9dbd619 [Wes McKinney] 32 bit fixes
---
 cpp/CMakeLists.txt                    |  6 +-----
 cpp/src/arrow/array-primitive-test.cc | 11 +++++------
 cpp/src/arrow/buffer.cc               | 15 ++++++++++++++-
 cpp/src/arrow/buffer.h                | 15 +++------------
 cpp/src/arrow/builder.cc              | 18 +++++++++++-------
 cpp/src/arrow/compare.cc              |  9 ++++++---
 cpp/src/arrow/io/file.cc              | 17 +++++++++--------
 cpp/src/arrow/io/test-common.h        |  4 ++--
 cpp/src/arrow/memory_pool.cc          |  8 +++++---
 cpp/src/arrow/test-util.h             |  4 ++--
 cpp/src/arrow/util/bit-util.cc        |  4 ++--
 11 files changed, 60 insertions(+), 51 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f6dab788b26d5..7d1f9e167d486 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -499,11 +499,7 @@ if(ARROW_BUILD_TESTS)
 
   # gflags (formerly Googleflags) command line parsing
   if("$ENV{GFLAGS_HOME}" STREQUAL "")
-    if(APPLE)
-      set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++")
-    else()
-      set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC")
-    endif()
+    set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS})
 
     set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
     set(GFLAGS_HOME "${GFLAGS_PREFIX}")
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
index 7b36275cbabfb..dfa37a8063767 100644
--- a/cpp/src/arrow/array-primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -100,7 +100,7 @@ class TestPrimitiveBuilder : public TestBuilder {
   void RandomData(int64_t N, double pct_null = 0.1) {
     Attrs::draw(N, &draws_);
 
-    valid_bytes_.resize(N);
+    valid_bytes_.resize(static_cast<size_t>(N));
     test::random_null_bytes(N, pct_null, valid_bytes_.data());
   }
 
@@ -192,8 +192,8 @@ struct PBoolean {
 
 template <>
 void TestPrimitiveBuilder<PBoolean>::RandomData(int64_t N, double pct_null) {
-  draws_.resize(N);
-  valid_bytes_.resize(N);
+  draws_.resize(static_cast<size_t>(N));
+  valid_bytes_.resize(static_cast<size_t>(N));
 
   test::random_null_bytes(N, 0.5, draws_.data());
   test::random_null_bytes(N, pct_null, valid_bytes_.data());
@@ -394,10 +394,9 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   this->builder_->Reserve(1000);
   this->builder_nn_->Reserve(1000);
 
-  int64_t i;
   int64_t null_count = 0;
   // Append the first 1000
-  for (i = 0; i < 1000; ++i) {
+  for (size_t i = 0; i < 1000; ++i) {
     if (valid_bytes[i] > 0) {
       this->builder_->Append(draws[i]);
     } else {
@@ -419,7 +418,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   this->builder_nn_->Reserve(size - 1000);
 
   // Append the next 9000
-  for (i = 1000; i < size; ++i) {
+  for (size_t i = 1000; i < size; ++i) {
     if (valid_bytes[i] > 0) {
       this->builder_->Append(draws[i]);
     } else {
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 18e9ed2015227..a0b78ac0b9f20 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -45,7 +45,7 @@ Status Buffer::Copy(
   auto new_buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(new_buffer->Resize(nbytes));
 
-  std::memcpy(new_buffer->mutable_data(), data() + start, nbytes);
+  std::memcpy(new_buffer->mutable_data(), data() + start, static_cast<size_t>(nbytes));
 
   *out = new_buffer;
   return Status::OK();
@@ -55,6 +55,19 @@ Status Buffer::Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out)
   return Copy(start, nbytes, default_memory_pool(), out);
 }
 
+bool Buffer::Equals(const Buffer& other, int64_t nbytes) const {
+  return this == &other ||
+         (size_ >= nbytes && other.size_ >= nbytes &&
+             (data_ == other.data_ ||
+                 !memcmp(data_, other.data_, static_cast<size_t>(nbytes))));
+}
+
+bool Buffer::Equals(const Buffer& other) const {
+  return this == &other || (size_ == other.size_ && (data_ == other.data_ ||
+                                                        !memcmp(data_, other.data_,
+                                                            static_cast<size_t>(size_))));
+}
+
 std::shared_ptr<Buffer> SliceBuffer(
     const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
   DCHECK_LE(offset, buffer->size());
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index be91af3556da4..0724385a4aff8 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -64,17 +64,8 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
 
   /// Return true if both buffers are the same size and contain the same bytes
   /// up to the number of compared bytes
-  bool Equals(const Buffer& other, int64_t nbytes) const {
-    return this == &other ||
-           (size_ >= nbytes && other.size_ >= nbytes &&
-               (data_ == other.data_ || !memcmp(data_, other.data_, nbytes)));
-  }
-
-  bool Equals(const Buffer& other) const {
-    return this == &other ||
-           (size_ == other.size_ &&
-               (data_ == other.data_ || !memcmp(data_, other.data_, size_)));
-  }
+  bool Equals(const Buffer& other, int64_t nbytes) const;
+  bool Equals(const Buffer& other) const;
 
   /// Copy a section of the buffer into a new Buffer.
   Status Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
@@ -196,7 +187,7 @@ class ARROW_EXPORT BufferBuilder {
 
   // Unsafe methods don't check existing size
   void UnsafeAppend(const uint8_t* data, int64_t length) {
-    memcpy(data_ + size_, data, length);
+    memcpy(data_ + size_, data, static_cast<size_t>(length));
     size_ += length;
   }
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 63e083e76b660..9086598cc5ba7 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -58,7 +58,7 @@ Status ArrayBuilder::Init(int64_t capacity) {
   const int64_t byte_capacity = null_bitmap_->capacity();
   capacity_ = capacity;
   null_bitmap_data_ = null_bitmap_->mutable_data();
-  memset(null_bitmap_data_, 0, byte_capacity);
+  memset(null_bitmap_data_, 0, static_cast<size_t>(byte_capacity));
   return Status::OK();
 }
 
@@ -72,7 +72,8 @@ Status ArrayBuilder::Resize(int64_t new_bits) {
   const int64_t byte_capacity = null_bitmap_->capacity();
   capacity_ = new_bits;
   if (old_bytes < new_bytes) {
-    memset(null_bitmap_data_ + old_bytes, 0, byte_capacity - old_bytes);
+    memset(
+        null_bitmap_data_ + old_bytes, 0, static_cast<size_t>(byte_capacity - old_bytes));
   }
   return Status::OK();
 }
@@ -152,7 +153,8 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
 
   // Fast bitsetting
   int64_t fast_length = (length - pad_to_byte) / 8;
-  memset(null_bitmap_data_ + ((length_ + pad_to_byte) / 8), 255, fast_length);
+  memset(null_bitmap_data_ + ((length_ + pad_to_byte) / 8), 255,
+      static_cast<size_t>(fast_length));
 
   // Trailing bytes
   for (int64_t i = length_ + pad_to_byte + (fast_length * 8); i < new_length; ++i) {
@@ -170,7 +172,7 @@ Status PrimitiveBuilder<T>::Init(int64_t capacity) {
   int64_t nbytes = TypeTraits<T>::bytes_required(capacity);
   RETURN_NOT_OK(data_->Resize(nbytes));
   // TODO(emkornfield) valgrind complains without this
-  memset(data_->mutable_data(), 0, nbytes);
+  memset(data_->mutable_data(), 0, static_cast<size_t>(nbytes));
 
   raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
   return Status::OK();
@@ -190,7 +192,8 @@ Status PrimitiveBuilder<T>::Resize(int64_t capacity) {
     RETURN_NOT_OK(data_->Resize(new_bytes));
     raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
     // TODO(emkornfield) valgrind complains without this
-    memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
+    memset(
+        data_->mutable_data() + old_bytes, 0, static_cast<size_t>(new_bytes - old_bytes));
   }
   return Status::OK();
 }
@@ -256,7 +259,7 @@ Status BooleanBuilder::Init(int64_t capacity) {
   int64_t nbytes = BitUtil::BytesForBits(capacity);
   RETURN_NOT_OK(data_->Resize(nbytes));
   // TODO(emkornfield) valgrind complains without this
-  memset(data_->mutable_data(), 0, nbytes);
+  memset(data_->mutable_data(), 0, static_cast<size_t>(nbytes));
 
   raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
   return Status::OK();
@@ -275,7 +278,8 @@ Status BooleanBuilder::Resize(int64_t capacity) {
 
     RETURN_NOT_OK(data_->Resize(new_bytes));
     raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
-    memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
+    memset(
+        data_->mutable_data() + old_bytes, 0, static_cast<size_t>(new_bytes - old_bytes));
   }
   return Status::OK();
 }
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index e94fa74ea6589..f38f8d67aa796 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -87,7 +87,8 @@ class RangeEqualsVisitor : public ArrayVisitor {
 
       if (end_offset - begin_offset > 0 &&
           std::memcmp(left.data()->data() + begin_offset,
-              right.data()->data() + right_begin_offset, end_offset - begin_offset)) {
+              right.data()->data() + right_begin_offset,
+              static_cast<size_t>(end_offset - begin_offset))) {
         return false;
       }
     }
@@ -348,7 +349,8 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       }
       return true;
     } else {
-      return memcmp(left_data, right_data, value_byte_size * left.length()) == 0;
+      return memcmp(left_data, right_data,
+                 static_cast<size_t>(value_byte_size * left.length())) == 0;
     }
   }
 
@@ -431,7 +433,8 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
         const int64_t total_bytes =
             left.value_offset(left.length()) - left.value_offset(0);
         return std::memcmp(left_data + left.value_offset(0),
-                   right_data + right.value_offset(0), total_bytes) == 0;
+                   right_data + right.value_offset(0),
+                   static_cast<size_t>(total_bytes)) == 0;
       }
     } else {
       // ARROW-537: Only compare data in non-null slots
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 230c7fe0fb4a0..7c14238e8fda4 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -244,9 +244,9 @@ static inline Status FileRead(
     int fd, uint8_t* buffer, int64_t nbytes, int64_t* bytes_read) {
 #if defined(_MSC_VER)
   if (nbytes > INT32_MAX) { return Status::IOError("Unable to read > 2GB blocks yet"); }
-  *bytes_read = _read(fd, buffer, static_cast<unsigned int>(nbytes));
+  *bytes_read = _read(fd, buffer, static_cast<size_t>(nbytes));
 #else
-  *bytes_read = read(fd, buffer, nbytes);
+  *bytes_read = read(fd, buffer, static_cast<size_t>(nbytes));
 #endif
 
   if (*bytes_read == -1) {
@@ -263,9 +263,9 @@ static inline Status FileWrite(int fd, const uint8_t* buffer, int64_t nbytes) {
   if (nbytes > INT32_MAX) {
     return Status::IOError("Unable to write > 2GB blocks to file yet");
   }
-  ret = static_cast<int>(_write(fd, buffer, static_cast<unsigned int>(nbytes)));
+  ret = static_cast<int>(_write(fd, buffer, static_cast<size_t>(nbytes)));
 #else
-  ret = static_cast<int>(write(fd, buffer, nbytes));
+  ret = static_cast<int>(write(fd, buffer, static_cast<size_t>(nbytes)));
 #endif
 
   if (ret == -1) {
@@ -526,7 +526,7 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
 
   ~MemoryMap() {
     if (file_->is_open()) {
-      munmap(mutable_data_, size_);
+      munmap(mutable_data_, static_cast<size_t>(size_));
       file_->Close();
     }
   }
@@ -554,7 +554,8 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
       is_mutable_ = false;
     }
 
-    void* result = mmap(nullptr, file_->size(), prot_flags, map_mode, file_->fd(), 0);
+    void* result = mmap(nullptr, static_cast<size_t>(file_->size()), prot_flags, map_mode,
+        file_->fd(), 0);
     if (result == MAP_FAILED) {
       std::stringstream ss;
       ss << "Memory mapping file failed, errno: " << errno;
@@ -630,7 +631,7 @@ Status MemoryMappedFile::Close() {
 Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
   nbytes = std::max<int64_t>(
       0, std::min(nbytes, memory_map_->size() - memory_map_->position()));
-  if (nbytes > 0) { std::memcpy(out, memory_map_->head(), nbytes); }
+  if (nbytes > 0) { std::memcpy(out, memory_map_->head(), static_cast<size_t>(nbytes)); }
   *bytes_read = nbytes;
   memory_map_->advance(nbytes);
   return Status::OK();
@@ -677,7 +678,7 @@ Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
 }
 
 Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
-  memcpy(memory_map_->head(), data, nbytes);
+  memcpy(memory_map_->head(), data, static_cast<size_t>(nbytes));
   memory_map_->advance(nbytes);
   return Status::OK();
 }
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index 6e917135db274..8355714540e95 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -53,9 +53,9 @@ class MemoryMapFixture {
     FILE* file = fopen(path.c_str(), "w");
     if (file != nullptr) { tmp_files_.push_back(path); }
 #ifdef _MSC_VER
-    _chsize(fileno(file), size);
+    _chsize(fileno(file), static_cast<size_t>(size));
 #else
-    ftruncate(fileno(file), size);
+    ftruncate(fileno(file), static_cast<size_t>(size));
 #endif
     fclose(file);
   }
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 8d85a089a65c9..5a630271a7da7 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -36,14 +36,16 @@ Status AllocateAligned(int64_t size, uint8_t** out) {
   constexpr size_t kAlignment = 64;
 #ifdef _MSC_VER
   // Special code path for MSVC
-  *out = reinterpret_cast<uint8_t*>(_aligned_malloc(size, kAlignment));
+  *out =
+      reinterpret_cast<uint8_t*>(_aligned_malloc(static_cast<size_t>(size), kAlignment));
   if (!*out) {
     std::stringstream ss;
     ss << "malloc of size " << size << " failed";
     return Status::OutOfMemory(ss.str());
   }
 #else
-  const int result = posix_memalign(reinterpret_cast<void**>(out), kAlignment, size);
+  const int result = posix_memalign(
+      reinterpret_cast<void**>(out), kAlignment, static_cast<size_t>(size));
   if (result == ENOMEM) {
     std::stringstream ss;
     ss << "malloc of size " << size << " failed";
@@ -90,7 +92,7 @@ Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t
   uint8_t* out;
   RETURN_NOT_OK(AllocateAligned(new_size, &out));
   // Copy contents and release old memory chunk
-  memcpy(out, *ptr, std::min(new_size, old_size));
+  memcpy(out, *ptr, static_cast<size_t>(std::min(new_size, old_size)));
 #ifdef _MSC_VER
   _aligned_free(*ptr);
 #else
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 5c7d04de6dfbb..11ce50a76a547 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -109,13 +109,13 @@ inline Status CopyBufferFromVector(
 
 static inline Status GetBitmapFromBoolVector(
     const std::vector<bool>& is_valid, std::shared_ptr<Buffer>* result) {
-  int64_t length = static_cast<int64_t>(is_valid.size());
+  size_t length = is_valid.size();
 
   std::shared_ptr<MutableBuffer> buffer;
   RETURN_NOT_OK(GetEmptyBitmap(default_memory_pool(), length, &buffer));
 
   uint8_t* bitmap = buffer->mutable_data();
-  for (int64_t i = 0; i < length; ++i) {
+  for (size_t i = 0; i < static_cast<size_t>(length); ++i) {
     if (is_valid[i]) { BitUtil::SetBit(bitmap, i); }
   }
 
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 1bbd2384267c9..3767ba9e62f4a 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -47,7 +47,7 @@ Status BitUtil::BytesToBits(
   std::shared_ptr<MutableBuffer> buffer;
   RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), bit_length, &buffer));
 
-  memset(buffer->mutable_data(), 0, bit_length);
+  memset(buffer->mutable_data(), 0, static_cast<size_t>(bit_length));
   BytesToBits(bytes, buffer->mutable_data());
 
   *out = buffer;
@@ -94,7 +94,7 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
 Status GetEmptyBitmap(
     MemoryPool* pool, int64_t length, std::shared_ptr<MutableBuffer>* result) {
   RETURN_NOT_OK(AllocateBuffer(pool, BitUtil::BytesForBits(length), result));
-  memset((*result)->mutable_data(), 0, (*result)->size());
+  memset((*result)->mutable_data(), 0, static_cast<size_t>((*result)->size()));
   return Status::OK();
 }
 

From 0637e05d59f20363a9103ffad5712f981314c4df Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 2 Mar 2017 14:41:29 -0500
Subject: [PATCH 0347/1644] ARROW-576: [C++] Complete file/stream
 implementation for union types

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #356 from wesm/ARROW-576 and squashes the following commits:

e239ba1 [Wes McKinney] Fix miniconda links
12fde46 [Wes McKinney] Complete metadata roundtrip for unions
---
 ci/travis_install_conda.sh             |   4 +-
 cpp/src/arrow/ipc/ipc-file-test.cc     |   2 +-
 cpp/src/arrow/ipc/metadata-internal.cc | 101 ++++++++++++++++---------
 3 files changed, 67 insertions(+), 40 deletions(-)

diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index ffa017cbaf5dd..9c13b1bc0f079 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -15,9 +15,9 @@
 set -e
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
-  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
+  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
 else
-  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
+  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
 fi
 
 wget -O miniconda.sh $MINICONDA_URL
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index e58f2cfbbe8c9..0c95c8eca65ca 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -180,7 +180,7 @@ TEST_P(TestStreamFormat, RoundTrip) {
 #define BATCH_CASES()                                                                   \
   ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
       &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
-      &MakeStruct, &MakeDictionary);
+      &MakeStruct, &MakeUnion, &MakeDictionary);
 
 INSTANTIATE_TEST_CASE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
 INSTANTIATE_TEST_CASE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 1cc4a235b81bd..17a3a5fafe626 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -78,43 +78,6 @@ static Status FloatFromFlatuffer(
   return Status::OK();
 }
 
-static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
-    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
-  switch (type) {
-    case flatbuf::Type_NONE:
-      return Status::Invalid("Type metadata cannot be none");
-    case flatbuf::Type_Int:
-      return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
-    case flatbuf::Type_FloatingPoint:
-      return FloatFromFlatuffer(
-          static_cast<const flatbuf::FloatingPoint*>(type_data), out);
-    case flatbuf::Type_Binary:
-      *out = binary();
-      return Status::OK();
-    case flatbuf::Type_Utf8:
-      *out = utf8();
-      return Status::OK();
-    case flatbuf::Type_Bool:
-      *out = boolean();
-      return Status::OK();
-    case flatbuf::Type_Decimal:
-    case flatbuf::Type_Timestamp:
-    case flatbuf::Type_List:
-      if (children.size() != 1) {
-        return Status::Invalid("List must have exactly 1 child field");
-      }
-      *out = std::make_shared<ListType>(children[0]);
-      return Status::OK();
-    case flatbuf::Type_Struct_:
-      *out = std::make_shared<StructType>(children);
-      return Status::OK();
-    case flatbuf::Type_Union:
-      return Status::NotImplemented("Type is not implemented");
-    default:
-      return Status::Invalid("Unrecognized type");
-  }
-}
-
 // Forward declaration
 static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
     DictionaryMemo* dictionary_memo, FieldOffset* offset);
@@ -153,6 +116,32 @@ static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// Union implementation
+
+static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
+    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
+  UnionMode mode = union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE
+                                                                   : UnionMode::DENSE;
+
+  std::vector<uint8_t> type_codes;
+
+  const flatbuffers::Vector<int32_t>* fb_type_ids = union_data->typeIds();
+  if (fb_type_ids == nullptr) {
+    for (uint8_t i = 0; i < children.size(); ++i) {
+      type_codes.push_back(i);
+    }
+  } else {
+    for (int32_t id : (*fb_type_ids)) {
+      // TODO(wesm): can these values exceed 255?
+      type_codes.push_back(static_cast<uint8_t>(id));
+    }
+  }
+
+  *out = union_(children, type_codes, mode);
+  return Status::OK();
+}
+
 static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
     Offset* offset) {
@@ -181,6 +170,44 @@ static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
   *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
   break;
 
+static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
+    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
+  switch (type) {
+    case flatbuf::Type_NONE:
+      return Status::Invalid("Type metadata cannot be none");
+    case flatbuf::Type_Int:
+      return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
+    case flatbuf::Type_FloatingPoint:
+      return FloatFromFlatuffer(
+          static_cast<const flatbuf::FloatingPoint*>(type_data), out);
+    case flatbuf::Type_Binary:
+      *out = binary();
+      return Status::OK();
+    case flatbuf::Type_Utf8:
+      *out = utf8();
+      return Status::OK();
+    case flatbuf::Type_Bool:
+      *out = boolean();
+      return Status::OK();
+    case flatbuf::Type_Decimal:
+    case flatbuf::Type_Timestamp:
+    case flatbuf::Type_List:
+      if (children.size() != 1) {
+        return Status::Invalid("List must have exactly 1 child field");
+      }
+      *out = std::make_shared<ListType>(children[0]);
+      return Status::OK();
+    case flatbuf::Type_Struct_:
+      *out = std::make_shared<StructType>(children);
+      return Status::OK();
+    case flatbuf::Type_Union:
+      return UnionFromFlatbuffer(
+          static_cast<const flatbuf::Union*>(type_data), children, out);
+    default:
+      return Status::Invalid("Unrecognized type");
+  }
+}
+
 // TODO(wesm): Convert this to visitor pattern
 static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,

From 8378c48df53bfdcf0c834aaf3b8b737f74eb212c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 3 Mar 2017 09:51:57 -0500
Subject: [PATCH 0348/1644] ARROW-589: C++: Use system provided shared jemalloc
 if static is unavailable

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #355 from xhochy/ARROW-589 and squashes the following commits:

a9d88bc [Uwe L. Korn] ARROW-589: C++: Use system provided shared jemalloc if static is unavailable
---
 cpp/src/arrow/jemalloc/CMakeLists.txt | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/jemalloc/CMakeLists.txt b/cpp/src/arrow/jemalloc/CMakeLists.txt
index c0f90eba260f6..7caa74a3ebbda 100644
--- a/cpp/src/arrow/jemalloc/CMakeLists.txt
+++ b/cpp/src/arrow/jemalloc/CMakeLists.txt
@@ -20,11 +20,21 @@
 
 include_directories(SYSTEM "{JEMALLOC_INCLUDE_DIR}")
 
-# arrow_jemalloc library
-set(ARROW_JEMALLOC_STATIC_LINK_LIBS
-  arrow_static
-  jemalloc_static
-)
+# In the case that jemalloc is only available as a shared library also use it to
+# link it in the static requirements. In contrast to other libraries we try in
+# most cases to use the system provided version of jemalloc to better align with
+# other potential users of jemalloc.
+if (JEMALLOC_STATIC_LIB)
+  set(ARROW_JEMALLOC_STATIC_LINK_LIBS
+    arrow_static
+    jemalloc_static
+  )
+else()
+  set(ARROW_JEMALLOC_STATIC_LINK_LIBS
+    arrow_static
+    jemalloc_shared
+  )
+endif()
 
 if (NOT APPLE)
   set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} pthread)

From 9deb3251ec89f5afb14b5bc768f2c3a88cad1627 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 5 Mar 2017 08:52:20 -0500
Subject: [PATCH 0349/1644] ARROW-109: [C++] Add nesting stress tests up to 500
 recursion depth

There doesn't appear to be any limit to the nesting depth permitted in the flatbuffers. I think what @emkornfield was running into was the size of the IPC payload exceeding the size of the memory map that was being allocated to accommodate it. I expanded the memory map size and was able to write schemas with 1000 and 5000 levels of nesting. I left a unit test with 500 depth which doesn't take too long to run.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #357 from wesm/ARROW-109 and squashes the following commits:

fa78976 [Wes McKinney] Add nesting stress tests up to 500 recursion depth, expand size of memory map
---
 cpp/src/arrow/ipc/adapter.h           |  6 ++-
 cpp/src/arrow/ipc/ipc-adapter-test.cc | 60 ++++++++++++++++++++++-----
 2 files changed, 53 insertions(+), 13 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index b7d8fa93d3651..933d3a4639fe8 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -47,8 +47,10 @@ namespace ipc {
 
 // ----------------------------------------------------------------------
 // Write path
-// We have trouble decoding flatbuffers if the size i > 70, so 64 is a nice round number
-// TODO(emkornfield) investigate this more
+//
+// ARROW-109: We set this number arbitrarily to help catch user mistakes. For
+// deeply nested schemas, it is expected the user will indicate explicitly the
+// maximum allowed recursion depth
 constexpr int kMaxIpcRecursionDepth = 64;
 
 // Write the RecordBatch (collection of equal-length Arrow arrays) to the
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 8999363893289..6678fd522a86a 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -213,7 +213,8 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
   void TearDown() { io::MemoryMapFixture::TearDown(); }
 
   Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
-      int64_t* body_length, std::shared_ptr<Schema>* schema) {
+      int64_t* body_length, std::shared_ptr<RecordBatch>* batch,
+      std::shared_ptr<Schema>* schema) {
     const int batch_length = 5;
     TypePtr type = int32();
     std::shared_ptr<Array> array;
@@ -230,18 +231,18 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
     *schema = std::shared_ptr<Schema>(new Schema({f0}));
 
     std::vector<std::shared_ptr<Array>> arrays = {array};
-    auto batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
+    *batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
 
     std::string path = "test-write-past-max-recursion";
-    const int memory_map_size = 1 << 16;
+    const int memory_map_size = 1 << 20;
     io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
 
     if (override_level) {
-      return WriteRecordBatch(*batch, 0, mmap_.get(), metadata_length, body_length, pool_,
-          recursion_level + 1);
+      return WriteRecordBatch(**batch, 0, mmap_.get(), metadata_length, body_length,
+          pool_, recursion_level + 1);
     } else {
       return WriteRecordBatch(
-          *batch, 0, mmap_.get(), metadata_length, body_length, pool_);
+          **batch, 0, mmap_.get(), metadata_length, body_length, pool_);
     }
   }
 
@@ -254,15 +255,21 @@ TEST_F(RecursionLimits, WriteLimit) {
   int32_t metadata_length = -1;
   int64_t body_length = -1;
   std::shared_ptr<Schema> schema;
-  ASSERT_RAISES(
-      Invalid, WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &schema));
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_RAISES(Invalid,
+      WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &batch, &schema));
 }
 
 TEST_F(RecursionLimits, ReadLimit) {
   int32_t metadata_length = -1;
   int64_t body_length = -1;
   std::shared_ptr<Schema> schema;
-  ASSERT_OK(WriteToMmap(64, true, &metadata_length, &body_length, &schema));
+
+  const int recursion_depth = 64;
+
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(WriteToMmap(
+      recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
 
   std::shared_ptr<Message> message;
   ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
@@ -273,8 +280,39 @@ TEST_F(RecursionLimits, ReadLimit) {
 
   io::BufferReader reader(payload);
 
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_RAISES(Invalid, ReadRecordBatch(*metadata, schema, &reader, &batch));
+  std::shared_ptr<RecordBatch> result;
+  ASSERT_RAISES(Invalid, ReadRecordBatch(*metadata, schema, &reader, &result));
+}
+
+TEST_F(RecursionLimits, StressLimit) {
+  auto CheckDepth = [this](int recursion_depth, bool* it_works) {
+    int32_t metadata_length = -1;
+    int64_t body_length = -1;
+    std::shared_ptr<Schema> schema;
+    std::shared_ptr<RecordBatch> batch;
+    ASSERT_OK(WriteToMmap(
+        recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
+
+    std::shared_ptr<Message> message;
+    ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+    auto metadata = std::make_shared<RecordBatchMetadata>(message);
+
+    std::shared_ptr<Buffer> payload;
+    ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
+
+    io::BufferReader reader(payload);
+
+    std::shared_ptr<RecordBatch> result;
+    ASSERT_OK(ReadRecordBatch(*metadata, schema, recursion_depth + 1, &reader, &result));
+    *it_works = result->Equals(*batch);
+  };
+
+  bool it_works = false;
+  CheckDepth(100, &it_works);
+  ASSERT_TRUE(it_works);
+
+  CheckDepth(500, &it_works);
+  ASSERT_TRUE(it_works);
 }
 
 }  // namespace ipc

From fb9fbe4981420aaa0a56bfe87254d8b10bd5ba18 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 7 Mar 2017 17:13:57 +0100
Subject: [PATCH 0350/1644] ARROW-604: Python: boxed Field instances are
 missing the reference to their DataType

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #362 from xhochy/ARROW-604 and squashes the following commits:

2e837c8 [Uwe L. Korn] ARROW-604: Python: boxed Field instances are missing the reference to DataType
---
 cpp/src/arrow/type.cc               | 3 +++
 python/pyarrow/schema.pyx           | 5 +++++
 python/pyarrow/tests/test_schema.py | 2 ++
 3 files changed, 10 insertions(+)

diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 23fa6812f53d4..7e5f13af9cf9b 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -54,6 +54,9 @@ bool DataType::Equals(const DataType& other) const {
 }
 
 bool DataType::Equals(const std::shared_ptr<DataType>& other) const {
+  if (!other) {
+    return false;
+  }
   return Equals(*other.get());
 }
 
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 52eeeaf717622..19910aba00427 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -88,6 +88,7 @@ cdef class Field:
     cdef init(self, const shared_ptr[CField]& field):
         self.sp_field = field
         self.field = field.get()
+        self.type = box_data_type(field.get().type)
 
     @classmethod
     def from_py(cls, object name, DataType type, bint nullable=True):
@@ -326,11 +327,15 @@ def schema(fields):
     return Schema.from_fields(fields)
 
 cdef DataType box_data_type(const shared_ptr[CDataType]& type):
+    if type.get() == NULL:
+        return None
     cdef DataType out = DataType()
     out.init(type)
     return out
 
 cdef Field box_field(const shared_ptr[CField]& field):
+    if field.get() == NULL:
+        return None
     cdef Field out = Field()
     out.init(field)
     return out
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 507ebb878d87b..f6dc33c75dfb8 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -64,6 +64,8 @@ def test_schema(self):
         assert len(sch) == 3
         assert sch[0].name == 'foo'
         assert sch[0].type == fields[0].type
+        assert sch.field_by_name('foo').name == 'foo'
+        assert sch.field_by_name('foo').type == fields[0].type
 
         assert repr(sch) == """\
 foo: int32

From b109a246f464eaf641dd7741d348e02069f3a0e9 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 7 Mar 2017 18:04:24 -0500
Subject: [PATCH 0351/1644] ARROW-566: Bundle Arrow libraries in Python package

Depends on https://github.com/apache/parquet-cpp/pull/265

With this change we can also build self-contained OSX wheels, still we have to find a way to build them reproducibly (will take care of that soon).

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #360 from xhochy/ARROW-566 and squashes the following commits:

d6c86de [Uwe L. Korn] Use Apache git again for Parquet
21861de [Uwe L. Korn] Only link to librt if we use GCC
925fce9 [Uwe L. Korn] ARROW-566: Bundle Arrow libraries in Python package
---
 cpp/CMakeLists.txt                            |  8 +++
 cpp/cmake_modules/BuildUtils.cmake            | 14 ++++-
 cpp/src/arrow/jemalloc/CMakeLists.txt         | 27 ++++++++--
 python/CMakeLists.txt                         | 45 ++++++++++++++++
 .../Dockerfile-parquet_arrow-base-x86_64      | 19 -------
 python/manylinux1/Dockerfile-x86_64           | 21 ++++++--
 python/manylinux1/README.md                   |  4 +-
 python/manylinux1/build_arrow.sh              | 52 +++++++++----------
 python/setup.py                               | 50 +++++++++++++-----
 9 files changed, 170 insertions(+), 70 deletions(-)
 delete mode 100644 python/manylinux1/Dockerfile-parquet_arrow-base-x86_64

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 7d1f9e167d486..22c6e9a7acbe5 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -88,6 +88,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow jemalloc-based allocator"
     ON)
 
+  option(ARROW_JEMALLOC_USE_SHARED
+    "Rely on jemalloc shared libraries where relevant"
+    ON)
+
   option(ARROW_BOOST_USE_SHARED
     "Rely on boost shared libraries where relevant"
     ON)
@@ -103,6 +107,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(ARROW_BUILD_UTILITIES
     "Build Arrow commandline utilities"
     ON)
+
+  option(ARROW_RPATH_ORIGIN
+    "Build Arrow libraries with RATH set to \$ORIGIN"
+    OFF)
 endif()
 
 if(ARROW_BUILD_TESTS)
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 9de9de516f996..2da8a05c9c42a 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -53,11 +53,21 @@ function(ADD_ARROW_LIB LIB_NAME)
       LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
       LINK_FLAGS "${ARG_SHARED_LINK_FLAGS}"
       OUTPUT_NAME ${LIB_NAME})
-  target_link_libraries(${LIB_NAME}_shared
+    target_link_libraries(${LIB_NAME}_shared
       LINK_PUBLIC ${ARG_SHARED_LINK_LIBS}
       LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS})
+
+    if (ARROW_RPATH_ORIGIN)
+        if (APPLE)
+            set(_lib_install_rpath "@loader_path")
+        else()
+            set(_lib_install_rpath "\$ORIGIN")
+        endif()
+        set_target_properties(${LIB_NAME}_shared PROPERTIES
+            INSTALL_RPATH ${_lib_install_rpath})
+    endif()
   
-  install(TARGETS ${LIB_NAME}_shared
+    install(TARGETS ${LIB_NAME}_shared
       LIBRARY DESTINATION lib
       ARCHIVE DESTINATION lib)
   endif()
diff --git a/cpp/src/arrow/jemalloc/CMakeLists.txt b/cpp/src/arrow/jemalloc/CMakeLists.txt
index 7caa74a3ebbda..5d5482ab653bf 100644
--- a/cpp/src/arrow/jemalloc/CMakeLists.txt
+++ b/cpp/src/arrow/jemalloc/CMakeLists.txt
@@ -40,10 +40,29 @@ if (NOT APPLE)
   set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} pthread)
 endif()
 
-set(ARROW_JEMALLOC_SHARED_LINK_LIBS
-  arrow_shared
-  jemalloc_shared
-)
+if (ARROW_JEMALLOC_USE_SHARED)
+  set(ARROW_JEMALLOC_SHARED_LINK_LIBS
+    arrow_shared
+    jemalloc_shared
+  )
+else()
+  if (CMAKE_COMPILER_IS_GNUCXX)
+    set(ARROW_JEMALLOC_SHARED_LINK_LIBS
+      arrow_shared
+      jemalloc_static
+      # For glibc <2.17 we need to link to librt.
+      # As we compile with --as-needed by default, the linker will omit this
+      # dependency if not required.
+      rt
+    )
+  else()
+    set(ARROW_JEMALLOC_SHARED_LINK_LIBS
+      arrow_shared
+      jemalloc_static
+    )
+  endif()
+endif()
+
 
 if (ARROW_BUILD_STATIC)
   set(ARROW_JEMALLOC_TEST_LINK_LIBS
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index ba26692b32b88..6e6d609b00007 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -56,6 +56,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUILD_JEMALLOC
     "Build the PyArrow jemalloc integration"
     OFF)
+  option(PYARROW_BUNDLE_ARROW_CPP
+    "Bundle the Arrow C++ libraries" 
+    OFF)
 endif()
 
 if(NOT PYARROW_BUILD_TESTS)
@@ -332,6 +335,25 @@ endif()
 ## Arrow
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
+
+if (PYARROW_BUNDLE_ARROW_CPP)
+  configure_file(${ARROW_SHARED_LIB}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}
+      COPYONLY)
+  SET(ARROW_SHARED_LIB
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+  configure_file(${ARROW_IO_SHARED_LIB}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX}
+      COPYONLY)
+  SET(ARROW_IO_SHARED_LIB
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX})
+  configure_file(${ARROW_IPC_SHARED_LIB}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX}
+      COPYONLY)
+  SET(ARROW_IPC_SHARED_LIB
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX})
+endif()
+
 ADD_THIRDPARTY_LIB(arrow
   SHARED_LIB ${ARROW_SHARED_LIB})
 ADD_THIRDPARTY_LIB(arrow_io
@@ -440,6 +462,18 @@ if (PYARROW_BUILD_PARQUET)
   if(NOT (PARQUET_FOUND AND PARQUET_ARROW_FOUND))
     message(FATAL_ERROR "Unable to locate Parquet libraries")
   endif()
+  if (PYARROW_BUNDLE_ARROW_CPP)
+      configure_file(${PARQUET_SHARED_LIB}
+          ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}
+          COPYONLY)
+      SET(PARQUET_SHARED_LIB
+          ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX})
+      configure_file(${PARQUET_ARROW_SHARED_LIB}
+          ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}
+          COPYONLY)
+      SET(PARQUET_ARROW_SHARED_LIB
+          ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
   ADD_THIRDPARTY_LIB(parquet_arrow
     SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
   set(LINK_LIBS
@@ -451,6 +485,13 @@ if (PYARROW_BUILD_PARQUET)
 endif()
 
 if (PYARROW_BUILD_JEMALLOC)
+  if (PYARROW_BUNDLE_ARROW_CPP)
+    configure_file(${ARROW_JEMALLOC_SHARED_LIB}
+       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}
+       COPYONLY)
+    SET(ARROW_JEMALLOC_SHARED_LIB
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
   ADD_THIRDPARTY_LIB(arrow_jemalloc
     SHARED_LIB ${ARROW_JEMALLOC_SHARED_LIB})
   set(LINK_LIBS
@@ -463,6 +504,10 @@ endif()
 
 add_library(pyarrow SHARED
   ${PYARROW_SRCS})
+if (PYARROW_BUNDLE_ARROW_CPP)
+  set_target_properties(pyarrow PROPERTIES
+      INSTALL_RPATH "\$ORIGIN")
+endif()
 target_link_libraries(pyarrow ${LINK_LIBS})
 
 if(APPLE)
diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
deleted file mode 100644
index dcc9321c322b2..0000000000000
--- a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
+++ /dev/null
@@ -1,19 +0,0 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-FROM arrow-base-x86_64
-
-WORKDIR /
-RUN git clone https://github.com/apache/parquet-cpp.git
-WORKDIR /parquet-cpp
-RUN ARROW_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
-RUN make -j5 install
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index ac47108c84ae7..820b94e306afe 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -13,14 +13,23 @@
 FROM quay.io/pypa/manylinux1_x86_64:latest
 
 # Install dependencies
-RUN yum install -y flex openssl-devel
+RUN yum install -y flex zlib-devel
+
+# Build a newer OpenSSL version to support Thrift 0.10.0, note that we don't trigger the SSL code in Arrow.
+WORKDIR /
+RUN wget --no-check-certificate https://www.openssl.org/source/openssl-1.0.2k.tar.gz -O openssl-1.0.2k.tar.gz
+RUN tar xf openssl-1.0.2k.tar.gz
+WORKDIR openssl-1.0.2k
+RUN ./config -fpic shared --prefix=/usr
+RUN make -j5
+RUN make install
 
 WORKDIR /
 RUN wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
 RUN tar xf boost_1_60_0.tar.gz
 WORKDIR /boost_1_60_0
 RUN ./bootstrap.sh
-RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install
+RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install
 
 WORKDIR /
 RUN wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2
@@ -43,5 +52,11 @@ RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
-RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON .
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF .
+RUN make -j5 install
+
+WORKDIR /
+RUN git clone https://github.com/apache/parquet-cpp.git
+WORKDIR /parquet-cpp
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .
 RUN make -j5 install
diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md
index 8cd9f6db004e5..32af6f31da287 100644
--- a/python/manylinux1/README.md
+++ b/python/manylinux1/README.md
@@ -31,10 +31,8 @@ for all supported Python versions and place them in the `dist` folder.
 git clone ../../ arrow
 # Build the native baseimage
 docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
-# (optionally) build parquet-cpp
-docker build -t parquet_arrow-base-x86_64 -f Dockerfile-parquet_arrow-base-x86_64 .
 # Build the python packages
-docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
+docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
 # Now the new packages are located in the dist/ folder
 ls -l dist/
 ```
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index cce5cd2b4d412..576a983b11c37 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -29,38 +29,19 @@ source /multibuild/manylinux_utils.sh
 
 cd /arrow/python
 
-export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
 # PyArrow build configuration
 export PYARROW_BUILD_TYPE='release'
 export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_TESTS=ON'
+export PYARROW_WITH_PARQUET=1
+export PYARROW_WITH_JEMALLOC=1
+export PYARROW_BUNDLE_ARROW_CPP=1
 # Need as otherwise arrow_io is sometimes not linked
 export LDFLAGS="-Wl,--no-as-needed"
-export ARROW_HOME="/usr"
+export ARROW_HOME="/arrow-dist"
 export PARQUET_HOME="/usr"
 
 # Ensure the target directory exists
 mkdir -p /io/dist
-# Temporary directory to store the wheels that should be sent through auditwheel
-rm_mkdir unfixed_wheels
-
-PY35_BIN=/opt/python/cp35-cp35m/bin
-$PY35_BIN/pip install 'pyelftools<0.24'
-$PY35_BIN/pip install 'git+https://github.com/xhochy/auditwheel.git@pyarrow-fixes'
-
-# Override repair_wheelhouse function
-function repair_wheelhouse {
-    local in_dir=$1
-    local out_dir=$2
-    for whl in $in_dir/*.whl; do
-        if [[ $whl == *none-any.whl ]]; then
-            cp $whl $out_dir
-        else
-            # Store libraries directly in . not .libs to fix problems with libpyarrow.so linkage.
-            $PY35_BIN/auditwheel -v repair -L . $whl -w $out_dir/
-        fi
-    done
-    chmod -R a+rwX $out_dir
-}
 
 for PYTHON in ${PYTHON_VERSIONS}; do
     PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
@@ -68,17 +49,36 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     PIPI_IO="$PIP install -f $MANYLINUX_URL"
     PATH="$PATH:$(cpython_path $PYTHON)"
 
+    echo "=== (${PYTHON}) Installing build dependencies ==="
     $PIPI_IO "numpy==1.9.0"
     $PIPI_IO "cython==0.24"
 
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-jemalloc
+    # Clear output directory
+    rm -rf dist/
+    echo "=== (${PYTHON}) Building wheel ==="
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-jemalloc --bundle-arrow-cpp
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
-    # Test for optional modules
+    echo "=== (${PYTHON}) Test the existence of optional modules ==="
     $PIPI_IO -r requirements.txt
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.jemalloc"
 
-    repair_wheelhouse dist /io/dist
+    echo "=== (${PYTHON}) Tag the wheel with manylinux1 ==="
+    mkdir -p repaired_wheels/
+    auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/
+
+    echo "=== (${PYTHON}) Testing manylinux1 wheel ==="
+    # Fix version to keep build reproducible"
+    $PIPI_IO "virtualenv==15.1.0"
+    rm -rf venv
+    "$(cpython_path $PYTHON)/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download venv
+    source ./venv/bin/activate
+    pip install repaired_wheels/*.whl
+    pip install pytest pandas
+    py.test venv/lib/*/site-packages/pyarrow
+    deactivate
+
+    mv repaired_wheels/*.whl /io/dist
 done
 
diff --git a/python/setup.py b/python/setup.py
index 54d1cd3af48bc..b0f29be4c1b3b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -34,6 +34,7 @@
 from os.path import join as pjoin
 
 from distutils.command.clean import clean as _clean
+from distutils.util import strtobool
 from distutils import sysconfig
 
 # Check if we're running 64-bit Python
@@ -81,15 +82,17 @@ def run(self):
     user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
                      ('build-type=', None, 'build type (debug or release)'),
                      ('with-parquet', None, 'build the Parquet extension'),
-                     ('with-jemalloc', None, 'build the jemalloc extension')] +
+                     ('with-jemalloc', None, 'build the jemalloc extension'),
+                     ('bundle-arrow-cpp', None, 'bundle the Arrow C++ libraries')] +
                     _build_ext.user_options)
 
     def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
-        self.with_parquet = False
-        self.with_jemalloc = False
+        self.with_parquet = strtobool(os.environ.get('PYARROW_WITH_PARQUET', '0'))
+        self.with_jemalloc = strtobool(os.environ.get('PYARROW_WITH_JEMALLOC', '0'))
+        self.bundle_arrow_cpp = strtobool(os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
 
     CYTHON_MODULE_NAMES = [
         'array',
@@ -142,6 +145,9 @@ def _run_cmake(self):
         if self.with_jemalloc:
             cmake_options.append('-DPYARROW_BUILD_JEMALLOC=on')
 
+        if self.bundle_arrow_cpp:
+            cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
+
         if sys.platform != 'win32':
             cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                                  .format(self.build_type))
@@ -181,17 +187,35 @@ def _run_cmake(self):
 
         # Move the built libpyarrow library to the place expected by the Python
         # build
-        if sys.platform != 'win32':
-            name, = glob.glob(pjoin(self.build_type, 'libpyarrow.*'))
-            try:
-                os.makedirs(pjoin(build_lib, 'pyarrow'))
-            except OSError:
-                pass
-            shutil.move(name,
-                        pjoin(build_lib, 'pyarrow', os.path.split(name)[1]))
+        shared_library_prefix = 'lib'
+        if sys.platform == 'darwin':
+            shared_library_suffix = '.dylib'
+        elif sys.platform == 'win32':
+            shared_library_suffix = '.dll'
+            shared_library_prefix = ''
         else:
-            shutil.move(pjoin(self.build_type, 'pyarrow.dll'),
-                        pjoin(build_lib, 'pyarrow', 'pyarrow.dll'))
+            shared_library_suffix = '.so'
+
+        try:
+            os.makedirs(pjoin(build_lib, 'pyarrow'))
+        except OSError:
+            pass
+
+        def move_lib(lib_name):
+            lib_filename = shared_library_prefix + lib_name + shared_library_suffix
+            shutil.move(pjoin(self.build_type, lib_filename),
+                        pjoin(build_lib, 'pyarrow', lib_filename))
+
+        move_lib("pyarrow")
+        if self.bundle_arrow_cpp:
+            move_lib("arrow")
+            move_lib("arrow_io")
+            move_lib("arrow_ipc")
+            if self.with_jemalloc:
+                move_lib("arrow_jemalloc")
+            if self.with_parquet:
+                move_lib("parquet")
+                move_lib("parquet_arrow")
 
         # Move the built C-extension to the place expected by the Python build
         self._found_names = []

From 6b3ae2aecc8cd31425035a021fa04b9ed3385a8d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 9 Mar 2017 14:00:48 -0500
Subject: [PATCH 0352/1644] ARROW-605: [C++] Refactor IPC adapter code into
 generic ArrayLoader class. Add Date32Type

These are various changes introduced to support the Feather merge in ARROW-452 #361

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #365 from wesm/array-loader and squashes the following commits:

bc22872 [Wes McKinney] Revert Array::type_id to type_enum since Parquet uses this API
344e6b1 [Wes McKinney] fix compiler warning
997b7a2 [Wes McKinney] Refactor IPC adapter code into generic ArrayLoader class. Add Date32Type
---
 cpp/CMakeLists.txt                          |   1 +
 cpp/src/arrow/CMakeLists.txt                |   5 +-
 cpp/src/arrow/array.cc                      |  52 +---
 cpp/src/arrow/array.h                       |  17 +-
 cpp/src/arrow/builder.cc                    |   1 +
 cpp/src/arrow/builder.h                     |   1 +
 cpp/src/arrow/column.cc                     |   3 +
 cpp/src/arrow/column.h                      |   3 +
 cpp/src/arrow/compare.cc                    |   8 +-
 cpp/src/arrow/io/memory.cc                  |  19 +-
 cpp/src/arrow/io/memory.h                   |   6 +
 cpp/src/arrow/ipc/adapter.cc                | 252 ++---------------
 cpp/src/arrow/ipc/adapter.h                 |   8 +-
 cpp/src/arrow/ipc/metadata.cc               |   1 +
 cpp/src/arrow/ipc/metadata.h                |   7 +-
 cpp/src/arrow/loader.cc                     | 285 ++++++++++++++++++++
 cpp/src/arrow/loader.h                      |  89 ++++++
 cpp/src/arrow/pretty_print.cc               |   6 +-
 cpp/src/arrow/type.cc                       |  22 +-
 cpp/src/arrow/type.h                        |  38 ++-
 cpp/src/arrow/type_fwd.h                    |   5 +
 cpp/src/arrow/type_traits.h                 |  12 +
 python/pyarrow/array.pyx                    |  31 ++-
 python/pyarrow/table.pyx                    |   3 +-
 python/pyarrow/tests/test_convert_pandas.py |   6 +-
 python/src/pyarrow/adapters/pandas.cc       |  17 +-
 26 files changed, 558 insertions(+), 340 deletions(-)
 create mode 100644 cpp/src/arrow/loader.cc
 create mode 100644 cpp/src/arrow/loader.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 22c6e9a7acbe5..294c439e2b093 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -799,6 +799,7 @@ set(ARROW_SRCS
   src/arrow/builder.cc
   src/arrow/column.cc
   src/arrow/compare.cc
+  src/arrow/loader.cc
   src/arrow/memory_pool.cc
   src/arrow/pretty_print.cc
   src/arrow/schema.cc
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index d1efa021a496d..ddeb81cae7b5b 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -19,10 +19,11 @@
 install(FILES
   api.h
   array.h
-  column.h
-  compare.h
   buffer.h
   builder.h
+  column.h
+  compare.h
+  loader.h
   memory_pool.h
   pretty_print.h
   schema.h
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 284bb57a02b88..49da6bb3197a1 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -165,6 +165,7 @@ template class NumericArray<Int32Type>;
 template class NumericArray<Int64Type>;
 template class NumericArray<TimestampType>;
 template class NumericArray<DateType>;
+template class NumericArray<Date32Type>;
 template class NumericArray<TimeType>;
 template class NumericArray<HalfFloatType>;
 template class NumericArray<FloatType>;
@@ -193,7 +194,7 @@ std::shared_ptr<Array> BooleanArray::Slice(int64_t offset, int64_t length) const
 
 Status ListArray::Validate() const {
   if (length_ < 0) { return Status::Invalid("Length was negative"); }
-  if (!value_offsets_) { return Status::Invalid("value_offsets_ was null"); }
+  if (length_ && !value_offsets_) { return Status::Invalid("value_offsets_ was null"); }
   if (value_offsets_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
     std::stringstream ss;
     ss << "offset buffer size (bytes): " << value_offsets_->size()
@@ -425,20 +426,6 @@ std::shared_ptr<Array> UnionArray::Slice(int64_t offset, int64_t length) const {
 // ----------------------------------------------------------------------
 // DictionaryArray
 
-Status DictionaryArray::FromBuffer(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& indices, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset, std::shared_ptr<DictionaryArray>* out) {
-  DCHECK_EQ(type->type, Type::DICTIONARY);
-  const auto& dict_type = static_cast<const DictionaryType*>(type.get());
-
-  std::shared_ptr<Array> boxed_indices;
-  RETURN_NOT_OK(MakePrimitiveArray(dict_type->index_type(), length, indices, null_bitmap,
-      null_count, offset, &boxed_indices));
-
-  *out = std::make_shared<DictionaryArray>(type, boxed_indices);
-  return Status::OK();
-}
-
 DictionaryArray::DictionaryArray(
     const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices)
     : Array(type, indices->length(), indices->null_bitmap(), indices->null_count(),
@@ -469,40 +456,6 @@ std::shared_ptr<Array> DictionaryArray::Slice(int64_t offset, int64_t length) co
   return std::make_shared<DictionaryArray>(type_, sliced_indices);
 }
 
-// ----------------------------------------------------------------------
-
-#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType)                                  \
-  case Type::ENUM:                                                                  \
-    out->reset(new ArrayType(type, length, data, null_bitmap, null_count, offset)); \
-    break;
-
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
-  switch (type->type) {
-    MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
-    MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(INT8, Int8Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(UINT16, UInt16Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(INT16, Int16Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(UINT32, UInt32Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(INT32, Int32Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
-    MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
-    MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
-    default:
-      return Status::NotImplemented(type->ToString());
-  }
-#ifdef NDEBUG
-  return Status::OK();
-#else
-  return (*out)->Validate();
-#endif
-}
-
 // ----------------------------------------------------------------------
 // Default implementations of ArrayVisitor methods
 
@@ -527,6 +480,7 @@ ARRAY_VISITOR_DEFAULT(DoubleArray);
 ARRAY_VISITOR_DEFAULT(StringArray);
 ARRAY_VISITOR_DEFAULT(BinaryArray);
 ARRAY_VISITOR_DEFAULT(DateArray);
+ARRAY_VISITOR_DEFAULT(Date32Array);
 ARRAY_VISITOR_DEFAULT(TimeArray);
 ARRAY_VISITOR_DEFAULT(TimestampArray);
 ARRAY_VISITOR_DEFAULT(IntervalArray);
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index f20f212c3a825..f111609db4317 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -58,6 +58,7 @@ class ARROW_EXPORT ArrayVisitor {
   virtual Status Visit(const StringArray& array);
   virtual Status Visit(const BinaryArray& array);
   virtual Status Visit(const DateArray& array);
+  virtual Status Visit(const Date32Array& array);
   virtual Status Visit(const TimeArray& array);
   virtual Status Visit(const TimestampArray& array);
   virtual Status Visit(const IntervalArray& array);
@@ -485,12 +486,6 @@ class ARROW_EXPORT DictionaryArray : public Array {
   DictionaryArray(
       const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices);
 
-  // Alternate ctor; other attributes (like null count) are inherited from the
-  // passed indices array
-  static Status FromBuffer(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::shared_ptr<Buffer>& indices, const std::shared_ptr<Buffer>& null_bitmap,
-      int64_t null_count, int64_t offset, std::shared_ptr<DictionaryArray>* out);
-
   Status Validate() const override;
 
   std::shared_ptr<Array> indices() const { return indices_; }
@@ -531,21 +526,13 @@ extern template class ARROW_EXPORT NumericArray<FloatType>;
 extern template class ARROW_EXPORT NumericArray<DoubleType>;
 extern template class ARROW_EXPORT NumericArray<TimestampType>;
 extern template class ARROW_EXPORT NumericArray<DateType>;
+extern template class ARROW_EXPORT NumericArray<Date32Type>;
 extern template class ARROW_EXPORT NumericArray<TimeType>;
 
 #if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic pop
 #endif
 
-// ----------------------------------------------------------------------
-// Helper functions
-
-// Create new arrays for logical types that are backed by primitive arrays.
-Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    int64_t length, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset,
-    std::shared_ptr<Array>* out);
-
 }  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 9086598cc5ba7..4372925fe494b 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -238,6 +238,7 @@ template class PrimitiveBuilder<Int16Type>;
 template class PrimitiveBuilder<Int32Type>;
 template class PrimitiveBuilder<Int64Type>;
 template class PrimitiveBuilder<DateType>;
+template class PrimitiveBuilder<Date32Type>;
 template class PrimitiveBuilder<TimestampType>;
 template class PrimitiveBuilder<TimeType>;
 template class PrimitiveBuilder<HalfFloatType>;
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index e642d3c21a2fd..ebc683ab334e6 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -233,6 +233,7 @@ using Int64Builder = NumericBuilder<Int64Type>;
 using TimestampBuilder = NumericBuilder<TimestampType>;
 using TimeBuilder = NumericBuilder<TimeType>;
 using DateBuilder = NumericBuilder<DateType>;
+using Date32Builder = NumericBuilder<Date32Type>;
 
 using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
 using FloatBuilder = NumericBuilder<FloatType>;
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index 18228700472c4..78501f9393e22 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -97,6 +97,9 @@ Column::Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>
   }
 }
 
+Column::Column(const std::string& name, const std::shared_ptr<Array>& data)
+    : Column(::arrow::field(name, data->type()), data) {}
+
 Column::Column(
     const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data)
     : field_(field), data_(data) {}
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
index 93a34c7c95fdf..bfcfd8ee459c0 100644
--- a/cpp/src/arrow/column.h
+++ b/cpp/src/arrow/column.h
@@ -69,6 +69,9 @@ class ARROW_EXPORT Column {
 
   Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data);
 
+  /// Construct from name and array
+  Column(const std::string& name, const std::shared_ptr<Array>& data);
+
   int64_t length() const { return data_->length(); }
 
   int64_t null_count() const { return data_->null_count(); }
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index f38f8d67aa796..17b883302c658 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -145,6 +145,10 @@ class RangeEqualsVisitor : public ArrayVisitor {
 
   Status Visit(const DateArray& left) override { return CompareValues<DateArray>(left); }
 
+  Status Visit(const Date32Array& left) override {
+    return CompareValues<Date32Array>(left);
+  }
+
   Status Visit(const TimeArray& left) override { return CompareValues<TimeArray>(left); }
 
   Status Visit(const TimestampArray& left) override {
@@ -381,6 +385,8 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
 
   Status Visit(const DateArray& left) override { return ComparePrimitive(left); }
 
+  Status Visit(const Date32Array& left) override { return ComparePrimitive(left); }
+
   Status Visit(const TimeArray& left) override { return ComparePrimitive(left); }
 
   Status Visit(const TimestampArray& left) override { return ComparePrimitive(left); }
@@ -622,7 +628,7 @@ class TypeEqualsVisitor : public TypeVisitor {
 
   Status Visit(const TimestampType& left) override {
     const auto& right = static_cast<const TimestampType&>(right_);
-    result_ = left.unit == right.unit;
+    result_ = left.unit == right.unit && left.timezone == right.timezone;
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 1339a99aa787e..5b5c8649deec4 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -28,6 +28,7 @@
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace io {
@@ -43,9 +44,17 @@ BufferOutputStream::BufferOutputStream(const std::shared_ptr<ResizableBuffer>& b
       position_(0),
       mutable_data_(buffer->mutable_data()) {}
 
+Status BufferOutputStream::Create(int64_t initial_capacity, MemoryPool* pool,
+    std::shared_ptr<BufferOutputStream>* out) {
+  std::shared_ptr<ResizableBuffer> buffer;
+  RETURN_NOT_OK(AllocateResizableBuffer(pool, initial_capacity, &buffer));
+  *out = std::make_shared<BufferOutputStream>(buffer);
+  return Status::OK();
+}
+
 BufferOutputStream::~BufferOutputStream() {
   // This can fail, better to explicitly call close
-  Close();
+  if (buffer_) { Close(); }
 }
 
 Status BufferOutputStream::Close() {
@@ -56,12 +65,20 @@ Status BufferOutputStream::Close() {
   }
 }
 
+Status BufferOutputStream::Finish(std::shared_ptr<Buffer>* result) {
+  RETURN_NOT_OK(Close());
+  *result = buffer_;
+  buffer_ = nullptr;
+  return Status::OK();
+}
+
 Status BufferOutputStream::Tell(int64_t* position) {
   *position = position_;
   return Status::OK();
 }
 
 Status BufferOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+  DCHECK(buffer_);
   RETURN_NOT_OK(Reserve(nbytes));
   std::memcpy(mutable_data_ + position_, data, nbytes);
   position_ += nbytes;
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 2d3df4224e9fb..82807508417d7 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -43,6 +43,9 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
  public:
   explicit BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer);
 
+  static Status Create(int64_t initial_capacity, MemoryPool* pool,
+      std::shared_ptr<BufferOutputStream>* out);
+
   ~BufferOutputStream();
 
   // Implement the OutputStream interface
@@ -50,6 +53,9 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   Status Tell(int64_t* position) override;
   Status Write(const uint8_t* data, int64_t nbytes) override;
 
+  /// Close the stream and return the buffer
+  Status Finish(std::shared_ptr<Buffer>* result);
+
  private:
   // Ensures there is sufficient space available to write nbytes
   Status Reserve(int64_t nbytes);
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index f11c88a6e1e4b..78d58101963dc 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -32,6 +32,7 @@
 #include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
+#include "arrow/loader.h"
 #include "arrow/memory_pool.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
@@ -531,12 +532,12 @@ Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
     int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
     int64_t* body_length, MemoryPool* pool) {
-  DictionaryWriter writer(pool, buffer_start_offset, kMaxIpcRecursionDepth);
+  DictionaryWriter writer(pool, buffer_start_offset, kMaxNestingDepth);
   return writer.Write(dictionary_id, dictionary, dst, metadata_length, body_length);
 }
 
 Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
-  RecordBatchWriter writer(default_memory_pool(), 0, kMaxIpcRecursionDepth);
+  RecordBatchWriter writer(default_memory_pool(), 0, kMaxNestingDepth);
   RETURN_NOT_OK(writer.GetTotalSize(batch, size));
   return Status::OK();
 }
@@ -544,235 +545,33 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
 // ----------------------------------------------------------------------
 // Record batch read path
 
-struct RecordBatchContext {
-  const RecordBatchMetadata* metadata;
-  int buffer_index;
-  int field_index;
-  int max_recursion_depth;
-};
-
-// Traverse the flattened record batch metadata and reassemble the
-// corresponding array containers
-class ArrayLoader : public TypeVisitor {
+class IpcComponentSource : public ArrayComponentSource {
  public:
-  ArrayLoader(
-      const Field& field, RecordBatchContext* context, io::ReadableFileInterface* file)
-      : field_(field), context_(context), file_(file) {}
-
-  Status Load(std::shared_ptr<Array>* out) {
-    if (context_->max_recursion_depth <= 0) {
-      return Status::Invalid("Max recursion depth reached");
-    }
-
-    // Load the array
-    RETURN_NOT_OK(field_.type->Accept(this));
+  IpcComponentSource(const RecordBatchMetadata& metadata, io::ReadableFileInterface* file)
+      : metadata_(metadata), file_(file) {}
 
-    *out = std::move(result_);
-    return Status::OK();
-  }
-
- private:
-  const Field& field_;
-  RecordBatchContext* context_;
-  io::ReadableFileInterface* file_;
-
-  // Used in visitor pattern
-  std::shared_ptr<Array> result_;
-
-  Status LoadChild(const Field& field, std::shared_ptr<Array>* out) {
-    ArrayLoader loader(field, context_, file_);
-    --context_->max_recursion_depth;
-    RETURN_NOT_OK(loader.Load(out));
-    ++context_->max_recursion_depth;
-    return Status::OK();
-  }
-
-  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
-    BufferMetadata metadata = context_->metadata->buffer(buffer_index);
-
-    if (metadata.length == 0) {
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
+    BufferMetadata buffer_meta = metadata_.buffer(buffer_index);
+    if (buffer_meta.length == 0) {
       *out = nullptr;
       return Status::OK();
     } else {
-      return file_->ReadAt(metadata.offset, metadata.length, out);
+      return file_->ReadAt(buffer_meta.offset, buffer_meta.length, out);
     }
   }
 
-  Status LoadCommon(FieldMetadata* field_meta, std::shared_ptr<Buffer>* null_bitmap) {
+  Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
     // pop off a field
-    if (context_->field_index >= context_->metadata->num_fields()) {
+    if (field_index >= metadata_.num_fields()) {
       return Status::Invalid("Ran out of field metadata, likely malformed");
     }
-
-    // This only contains the length and null count, which we need to figure
-    // out what to do with the buffers. For example, if null_count == 0, then
-    // we can skip that buffer without reading from shared memory
-    *field_meta = context_->metadata->field(context_->field_index++);
-
-    // extract null_bitmap which is common to all arrays
-    if (field_meta->null_count == 0) {
-      *null_bitmap = nullptr;
-    } else {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index, null_bitmap));
-    }
-    context_->buffer_index++;
-    return Status::OK();
-  }
-
-  Status LoadPrimitive(const DataType& type) {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, data;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    if (field_meta.length > 0) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &data));
-    } else {
-      context_->buffer_index++;
-      data.reset(new Buffer(nullptr, 0));
-    }
-    return MakePrimitiveArray(field_.type, field_meta.length, data, null_bitmap,
-        field_meta.null_count, 0, &result_);
-  }
-
-  template <typename CONTAINER>
-  Status LoadBinary() {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, offsets, values;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    if (field_meta.length > 0) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
-    } else {
-      context_->buffer_index += 2;
-      offsets = values = nullptr;
-    }
-
-    result_ = std::make_shared<CONTAINER>(
-        field_meta.length, offsets, values, null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status Visit(const BooleanType& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const Int8Type& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const Int16Type& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const Int32Type& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const Int64Type& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const UInt8Type& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const UInt16Type& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const UInt32Type& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const UInt64Type& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const HalfFloatType& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const FloatType& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const DoubleType& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const StringType& type) override { return LoadBinary<StringArray>(); }
-
-  Status Visit(const BinaryType& type) override { return LoadBinary<BinaryArray>(); }
-
-  Status Visit(const DateType& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const TimeType& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const TimestampType& type) override { return LoadPrimitive(type); }
-
-  Status Visit(const ListType& type) override {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, offsets;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    if (field_meta.length > 0) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &offsets));
-    } else {
-      offsets = nullptr;
-    }
-    ++context_->buffer_index;
-
-    const int num_children = type.num_children();
-    if (num_children != 1) {
-      std::stringstream ss;
-      ss << "Wrong number of children: " << num_children;
-      return Status::Invalid(ss.str());
-    }
-    std::shared_ptr<Array> values_array;
-
-    RETURN_NOT_OK(LoadChild(*type.child(0).get(), &values_array));
-
-    result_ = std::make_shared<ListArray>(field_.type, field_meta.length, offsets,
-        values_array, null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status LoadChildren(std::vector<std::shared_ptr<Field>> child_fields,
-      std::vector<std::shared_ptr<Array>>* arrays) {
-    arrays->reserve(static_cast<int>(child_fields.size()));
-
-    for (const auto& child_field : child_fields) {
-      std::shared_ptr<Array> field_array;
-      RETURN_NOT_OK(LoadChild(*child_field.get(), &field_array));
-      arrays->emplace_back(field_array);
-    }
+    *metadata = metadata_.field(field_index);
     return Status::OK();
   }
 
-  Status Visit(const StructType& type) override {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap;
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-
-    std::vector<std::shared_ptr<Array>> fields;
-    RETURN_NOT_OK(LoadChildren(type.children(), &fields));
-
-    result_ = std::make_shared<StructArray>(
-        field_.type, field_meta.length, fields, null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status Visit(const UnionType& type) override {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, type_ids, offsets;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    if (field_meta.length > 0) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &type_ids));
-      if (type.mode == UnionMode::DENSE) {
-        RETURN_NOT_OK(GetBuffer(context_->buffer_index + 1, &offsets));
-      }
-    }
-    context_->buffer_index += type.mode == UnionMode::DENSE ? 2 : 1;
-
-    std::vector<std::shared_ptr<Array>> fields;
-    RETURN_NOT_OK(LoadChildren(type.children(), &fields));
-
-    result_ = std::make_shared<UnionArray>(field_.type, field_meta.length, fields,
-        type_ids, offsets, null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status Visit(const DictionaryType& type) override {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, indices_data;
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &indices_data));
-
-    std::shared_ptr<Array> indices;
-    RETURN_NOT_OK(MakePrimitiveArray(type.index_type(), field_meta.length, indices_data,
-        null_bitmap, field_meta.null_count, 0, &indices));
-
-    result_ = std::make_shared<DictionaryArray>(field_.type, indices);
-    return Status::OK();
-  };
+ private:
+  const RecordBatchMetadata& metadata_;
+  io::ReadableFileInterface* file_;
 };
 
 class RecordBatchReader {
@@ -788,17 +587,15 @@ class RecordBatchReader {
   Status Read(std::shared_ptr<RecordBatch>* out) {
     std::vector<std::shared_ptr<Array>> arrays(schema_->num_fields());
 
-    // The field_index and buffer_index are incremented in the ArrayLoader
-    // based on how much of the batch is "consumed" (through nested data
-    // reconstruction, for example)
-    context_.metadata = &metadata_;
-    context_.field_index = 0;
-    context_.buffer_index = 0;
-    context_.max_recursion_depth = max_recursion_depth_;
+    IpcComponentSource source(metadata_, file_);
+    ArrayLoaderContext context;
+    context.source = &source;
+    context.field_index = 0;
+    context.buffer_index = 0;
+    context.max_recursion_depth = max_recursion_depth_;
 
     for (int i = 0; i < schema_->num_fields(); ++i) {
-      ArrayLoader loader(*schema_->field(i).get(), &context_, file_);
-      RETURN_NOT_OK(loader.Load(&arrays[i]));
+      RETURN_NOT_OK(LoadArray(schema_->field(i)->type, &context, &arrays[i]));
     }
 
     *out = std::make_shared<RecordBatch>(schema_, metadata_.length(), arrays);
@@ -806,7 +603,6 @@ class RecordBatchReader {
   }
 
  private:
-  RecordBatchContext context_;
   const RecordBatchMetadata& metadata_;
   std::shared_ptr<Schema> schema_;
   int max_recursion_depth_;
@@ -816,7 +612,7 @@ class RecordBatchReader {
 Status ReadRecordBatch(const RecordBatchMetadata& metadata,
     const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
     std::shared_ptr<RecordBatch>* out) {
-  return ReadRecordBatch(metadata, schema, kMaxIpcRecursionDepth, file, out);
+  return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
 }
 
 Status ReadRecordBatch(const RecordBatchMetadata& metadata,
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 933d3a4639fe8..21d814db86530 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -26,6 +26,7 @@
 #include <vector>
 
 #include "arrow/ipc/metadata.h"
+#include "arrow/loader.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -47,11 +48,6 @@ namespace ipc {
 
 // ----------------------------------------------------------------------
 // Write path
-//
-// ARROW-109: We set this number arbitrarily to help catch user mistakes. For
-// deeply nested schemas, it is expected the user will indicate explicitly the
-// maximum allowed recursion depth
-constexpr int kMaxIpcRecursionDepth = 64;
 
 // Write the RecordBatch (collection of equal-length Arrow arrays) to the
 // output stream in a contiguous block. The record batch metadata is written as
@@ -75,7 +71,7 @@ constexpr int kMaxIpcRecursionDepth = 64;
 // padding bytes
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth = kMaxIpcRecursionDepth);
+    MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
 
 // Write Array as a DictionaryBatch message
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 2ba44ac618ce3..695e7886e3124 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -289,6 +289,7 @@ FieldMetadata RecordBatchMetadata::field(int i) const {
   FieldMetadata result;
   result.length = node->length();
   result.null_count = node->null_count();
+  result.offset = 0;
   return result;
 }
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index f12529b5c585e..f6a0a3a073faa 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -25,6 +25,7 @@
 #include <unordered_map>
 #include <vector>
 
+#include "arrow/loader.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -135,12 +136,6 @@ class ARROW_EXPORT SchemaMetadata {
   DISALLOW_COPY_AND_ASSIGN(SchemaMetadata);
 };
 
-// Field metadata
-struct ARROW_EXPORT FieldMetadata {
-  int32_t length;
-  int32_t null_count;
-};
-
 struct ARROW_EXPORT BufferMetadata {
   int32_t page;
   int64_t offset;
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
new file mode 100644
index 0000000000000..3cb51ae8fdab7
--- /dev/null
+++ b/cpp/src/arrow/loader.cc
@@ -0,0 +1,285 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/loader.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+struct DataType;
+class Status;
+
+class ArrayLoader : public TypeVisitor {
+ public:
+  ArrayLoader(const std::shared_ptr<DataType>& type, ArrayLoaderContext* context)
+      : type_(type), context_(context) {}
+
+  Status Load(std::shared_ptr<Array>* out) {
+    if (context_->max_recursion_depth <= 0) {
+      return Status::Invalid("Max recursion depth reached");
+    }
+
+    // Load the array
+    RETURN_NOT_OK(type_->Accept(this));
+
+    *out = std::move(result_);
+    return Status::OK();
+  }
+
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
+    return context_->source->GetBuffer(buffer_index, out);
+  }
+
+  Status LoadCommon(FieldMetadata* field_meta, std::shared_ptr<Buffer>* null_bitmap) {
+    // This only contains the length and null count, which we need to figure
+    // out what to do with the buffers. For example, if null_count == 0, then
+    // we can skip that buffer without reading from shared memory
+    RETURN_NOT_OK(
+        context_->source->GetFieldMetadata(context_->field_index++, field_meta));
+
+    // extract null_bitmap which is common to all arrays
+    if (field_meta->null_count == 0) {
+      *null_bitmap = nullptr;
+    } else {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index, null_bitmap));
+    }
+    context_->buffer_index++;
+    return Status::OK();
+  }
+
+  template <typename TYPE>
+  Status LoadPrimitive() {
+    using ArrayType = typename TypeTraits<TYPE>::ArrayType;
+
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap, data;
+
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    if (field_meta.length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &data));
+    } else {
+      context_->buffer_index++;
+      data.reset(new Buffer(nullptr, 0));
+    }
+    result_ = std::make_shared<ArrayType>(type_, field_meta.length, data, null_bitmap,
+        field_meta.null_count, field_meta.offset);
+    return Status::OK();
+  }
+
+  template <typename CONTAINER>
+  Status LoadBinary() {
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap, offsets, values;
+
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    if (field_meta.length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
+    } else {
+      context_->buffer_index += 2;
+      offsets = values = nullptr;
+    }
+
+    result_ = std::make_shared<CONTAINER>(
+        field_meta.length, offsets, values, null_bitmap, field_meta.null_count);
+    return Status::OK();
+  }
+
+  Status LoadChild(const Field& field, std::shared_ptr<Array>* out) {
+    ArrayLoader loader(field.type, context_);
+    --context_->max_recursion_depth;
+    RETURN_NOT_OK(loader.Load(out));
+    ++context_->max_recursion_depth;
+    return Status::OK();
+  }
+
+  Status LoadChildren(std::vector<std::shared_ptr<Field>> child_fields,
+      std::vector<std::shared_ptr<Array>>* arrays) {
+    arrays->reserve(static_cast<int>(child_fields.size()));
+
+    for (const auto& child_field : child_fields) {
+      std::shared_ptr<Array> field_array;
+      RETURN_NOT_OK(LoadChild(*child_field.get(), &field_array));
+      arrays->emplace_back(field_array);
+    }
+    return Status::OK();
+  }
+
+#define VISIT_PRIMITIVE(TYPE) \
+  Status Visit(const TYPE& type) override { return LoadPrimitive<TYPE>(); }
+
+  VISIT_PRIMITIVE(BooleanType);
+  VISIT_PRIMITIVE(Int8Type);
+  VISIT_PRIMITIVE(Int16Type);
+  VISIT_PRIMITIVE(Int32Type);
+  VISIT_PRIMITIVE(Int64Type);
+  VISIT_PRIMITIVE(UInt8Type);
+  VISIT_PRIMITIVE(UInt16Type);
+  VISIT_PRIMITIVE(UInt32Type);
+  VISIT_PRIMITIVE(UInt64Type);
+  VISIT_PRIMITIVE(HalfFloatType);
+  VISIT_PRIMITIVE(FloatType);
+  VISIT_PRIMITIVE(DoubleType);
+  VISIT_PRIMITIVE(DateType);
+  VISIT_PRIMITIVE(Date32Type);
+  VISIT_PRIMITIVE(TimeType);
+  VISIT_PRIMITIVE(TimestampType);
+
+#undef VISIT_PRIMITIVE
+
+  Status Visit(const StringType& type) override { return LoadBinary<StringArray>(); }
+
+  Status Visit(const BinaryType& type) override { return LoadBinary<BinaryArray>(); }
+
+  Status Visit(const ListType& type) override {
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap, offsets;
+
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    if (field_meta.length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &offsets));
+    } else {
+      offsets = nullptr;
+    }
+    ++context_->buffer_index;
+
+    const int num_children = type.num_children();
+    if (num_children != 1) {
+      std::stringstream ss;
+      ss << "Wrong number of children: " << num_children;
+      return Status::Invalid(ss.str());
+    }
+    std::shared_ptr<Array> values_array;
+
+    RETURN_NOT_OK(LoadChild(*type.child(0).get(), &values_array));
+
+    result_ = std::make_shared<ListArray>(type_, field_meta.length, offsets, values_array,
+        null_bitmap, field_meta.null_count);
+    return Status::OK();
+  }
+
+  Status Visit(const StructType& type) override {
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap;
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+
+    std::vector<std::shared_ptr<Array>> fields;
+    RETURN_NOT_OK(LoadChildren(type.children(), &fields));
+
+    result_ = std::make_shared<StructArray>(
+        type_, field_meta.length, fields, null_bitmap, field_meta.null_count);
+    return Status::OK();
+  }
+
+  Status Visit(const UnionType& type) override {
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap, type_ids, offsets;
+
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    if (field_meta.length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &type_ids));
+      if (type.mode == UnionMode::DENSE) {
+        RETURN_NOT_OK(GetBuffer(context_->buffer_index + 1, &offsets));
+      }
+    }
+    context_->buffer_index += type.mode == UnionMode::DENSE ? 2 : 1;
+
+    std::vector<std::shared_ptr<Array>> fields;
+    RETURN_NOT_OK(LoadChildren(type.children(), &fields));
+
+    result_ = std::make_shared<UnionArray>(type_, field_meta.length, fields, type_ids,
+        offsets, null_bitmap, field_meta.null_count);
+    return Status::OK();
+  }
+
+  Status Visit(const DictionaryType& type) override {
+    std::shared_ptr<Array> indices;
+    RETURN_NOT_OK(LoadArray(type.index_type(), context_, &indices));
+    result_ = std::make_shared<DictionaryArray>(type_, indices);
+    return Status::OK();
+  };
+
+  std::shared_ptr<Array> result() const { return result_; }
+
+ private:
+  const std::shared_ptr<DataType> type_;
+  ArrayLoaderContext* context_;
+
+  // Used in visitor pattern
+  std::shared_ptr<Array> result_;
+};
+
+Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
+    ArrayComponentSource* source, std::shared_ptr<Array>* out) {
+  ArrayLoaderContext context;
+  context.source = source;
+  context.field_index = context.buffer_index = 0;
+  context.max_recursion_depth = kMaxNestingDepth;
+  return LoadArray(type, &context, out);
+}
+
+Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
+    ArrayLoaderContext* context, std::shared_ptr<Array>* out) {
+  ArrayLoader loader(type, context);
+  RETURN_NOT_OK(loader.Load(out));
+
+  return Status::OK();
+}
+
+class InMemorySource : public ArrayComponentSource {
+ public:
+  InMemorySource(const std::vector<FieldMetadata>& fields,
+      const std::vector<std::shared_ptr<Buffer>>& buffers)
+      : fields_(fields), buffers_(buffers) {}
+
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
+    DCHECK(buffer_index < static_cast<int>(buffers_.size()));
+    *out = buffers_[buffer_index];
+    return Status::OK();
+  }
+
+  Status GetFieldMetadata(int field_index, FieldMetadata* metadata) {
+    DCHECK(field_index < static_cast<int>(fields_.size()));
+    *metadata = fields_[field_index];
+    return Status::OK();
+  }
+
+ private:
+  const std::vector<FieldMetadata>& fields_;
+  const std::vector<std::shared_ptr<Buffer>>& buffers_;
+};
+
+Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
+    const std::vector<FieldMetadata>& fields,
+    const std::vector<std::shared_ptr<Buffer>>& buffers, std::shared_ptr<Array>* out) {
+  InMemorySource source(fields, buffers);
+  return LoadArray(type, &source, out);
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/loader.h b/cpp/src/arrow/loader.h
new file mode 100644
index 0000000000000..b4949f2556028
--- /dev/null
+++ b/cpp/src/arrow/loader.h
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Function for constructing Array array objects from metadata and raw memory
+// buffers
+
+#ifndef ARROW_LOADER_H
+#define ARROW_LOADER_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/status.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+class Buffer;
+struct DataType;
+
+// ARROW-109: We set this number arbitrarily to help catch user mistakes. For
+// deeply nested schemas, it is expected the user will indicate explicitly the
+// maximum allowed recursion depth
+constexpr int kMaxNestingDepth = 64;
+
+struct ARROW_EXPORT FieldMetadata {
+  int64_t length;
+  int64_t null_count;
+  int64_t offset;
+};
+
+/// Implement this to create new types of Arrow data loaders
+class ARROW_EXPORT ArrayComponentSource {
+ public:
+  virtual ~ArrayComponentSource() = default;
+
+  virtual Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) = 0;
+  virtual Status GetFieldMetadata(int field_index, FieldMetadata* metadata) = 0;
+};
+
+/// Bookkeeping struct for loading array objects from their constituent pieces of raw data
+///
+/// The field_index and buffer_index are incremented in the ArrayLoader
+/// based on how much of the batch is "consumed" (through nested data
+/// reconstruction, for example)
+struct ArrayLoaderContext {
+  ArrayComponentSource* source;
+  int buffer_index;
+  int field_index;
+  int max_recursion_depth;
+};
+
+/// Construct an Array container from type metadata and a collection of memory
+/// buffers
+///
+/// \param[in] field the data type of the array being loaded
+/// \param[in] source an implementation of ArrayComponentSource
+/// \param[out] out the constructed array
+/// \return Status indicating success or failure
+Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
+    ArrayComponentSource* source, std::shared_ptr<Array>* out);
+
+Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& field,
+    ArrayLoaderContext* context, std::shared_ptr<Array>* out);
+
+Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
+    const std::vector<FieldMetadata>& fields,
+    const std::vector<std::shared_ptr<Buffer>>& buffers, std::shared_ptr<Array>* out);
+
+}  // namespace arrow
+
+#endif  // ARROW_LOADER_H
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 7e69e42800e79..2508fa5bd8cde 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -145,9 +145,11 @@ class ArrayPrinter : public ArrayVisitor {
 
   Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); }
 
-  Status Visit(const DateArray& array) override { return Status::NotImplemented("date"); }
+  Status Visit(const DateArray& array) override { return WritePrimitive(array); }
 
-  Status Visit(const TimeArray& array) override { return Status::NotImplemented("time"); }
+  Status Visit(const Date32Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const TimeArray& array) override { return WritePrimitive(array); }
 
   Status Visit(const TimestampArray& array) override {
     return Status::NotImplemented("timestamp");
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 7e5f13af9cf9b..4679a2f5b76b6 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -54,9 +54,7 @@ bool DataType::Equals(const DataType& other) const {
 }
 
 bool DataType::Equals(const std::shared_ptr<DataType>& other) const {
-  if (!other) {
-    return false;
-  }
+  if (!other) { return false; }
   return Equals(*other.get());
 }
 
@@ -106,6 +104,10 @@ std::string DateType::ToString() const {
   return std::string("date");
 }
 
+std::string Date32Type::ToString() const {
+  return std::string("date32");
+}
+
 // ----------------------------------------------------------------------
 // Union type
 
@@ -135,11 +137,12 @@ std::string UnionType::ToString() const {
 // ----------------------------------------------------------------------
 // DictionaryType
 
-DictionaryType::DictionaryType(
-    const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& dictionary)
+DictionaryType::DictionaryType(const std::shared_ptr<DataType>& index_type,
+    const std::shared_ptr<Array>& dictionary, bool ordered)
     : FixedWidthType(Type::DICTIONARY),
       index_type_(index_type),
-      dictionary_(dictionary) {}
+      dictionary_(dictionary),
+      ordered_(ordered) {}
 
 int DictionaryType::bit_width() const {
   return static_cast<const FixedWidthType*>(index_type_.get())->bit_width();
@@ -178,6 +181,7 @@ ACCEPT_VISITOR(StructType);
 ACCEPT_VISITOR(DecimalType);
 ACCEPT_VISITOR(UnionType);
 ACCEPT_VISITOR(DateType);
+ACCEPT_VISITOR(Date32Type);
 ACCEPT_VISITOR(TimeType);
 ACCEPT_VISITOR(TimestampType);
 ACCEPT_VISITOR(IntervalType);
@@ -205,11 +209,16 @@ TYPE_FACTORY(float64, DoubleType);
 TYPE_FACTORY(utf8, StringType);
 TYPE_FACTORY(binary, BinaryType);
 TYPE_FACTORY(date, DateType);
+TYPE_FACTORY(date32, Date32Type);
 
 std::shared_ptr<DataType> timestamp(TimeUnit unit) {
   return std::make_shared<TimestampType>(unit);
 }
 
+std::shared_ptr<DataType> timestamp(const std::string& timezone, TimeUnit unit) {
+  return std::make_shared<TimestampType>(timezone, unit);
+}
+
 std::shared_ptr<DataType> time(TimeUnit unit) {
   return std::make_shared<TimeType>(unit);
 }
@@ -313,6 +322,7 @@ TYPE_VISITOR_DEFAULT(DoubleType);
 TYPE_VISITOR_DEFAULT(StringType);
 TYPE_VISITOR_DEFAULT(BinaryType);
 TYPE_VISITOR_DEFAULT(DateType);
+TYPE_VISITOR_DEFAULT(Date32Type);
 TYPE_VISITOR_DEFAULT(TimeType);
 TYPE_VISITOR_DEFAULT(TimestampType);
 TYPE_VISITOR_DEFAULT(IntervalType);
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 9b1ab3288eb8c..aa0d70e5505e6 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -67,9 +67,12 @@ struct Type {
     // Variable-length bytes (no guarantee of UTF8-ness)
     BINARY,
 
-    // By default, int32 days since the UNIX epoch
+    // int64_t milliseconds since the UNIX epoch
     DATE,
 
+    // int32_t days since the UNIX epoch
+    DATE32,
+
     // Exact timestamp encoded with int64 since UNIX epoch
     // Default unit millisecond
     TIMESTAMP,
@@ -132,6 +135,7 @@ class ARROW_EXPORT TypeVisitor {
   virtual Status Visit(const StringType& type);
   virtual Status Visit(const BinaryType& type);
   virtual Status Visit(const DateType& type);
+  virtual Status Visit(const Date32Type& type);
   virtual Status Visit(const TimeType& type);
   virtual Status Visit(const TimestampType& type);
   virtual Status Visit(const IntervalType& type);
@@ -425,6 +429,7 @@ struct ARROW_EXPORT UnionType : public DataType {
 // ----------------------------------------------------------------------
 // Date and time types
 
+/// Date as int64_t milliseconds since UNIX epoch
 struct ARROW_EXPORT DateType : public FixedWidthType {
   static constexpr Type::type type_id = Type::DATE;
 
@@ -439,6 +444,20 @@ struct ARROW_EXPORT DateType : public FixedWidthType {
   static std::string name() { return "date"; }
 };
 
+/// Date as int32_t days since UNIX epoch
+struct ARROW_EXPORT Date32Type : public FixedWidthType {
+  static constexpr Type::type type_id = Type::DATE32;
+
+  using c_type = int32_t;
+
+  Date32Type() : FixedWidthType(Type::DATE32) {}
+
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override;
+};
+
 enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
 
 struct ARROW_EXPORT TimeType : public FixedWidthType {
@@ -467,16 +486,20 @@ struct ARROW_EXPORT TimestampType : public FixedWidthType {
 
   int bit_width() const override { return static_cast<int>(sizeof(int64_t) * 8); }
 
-  TimeUnit unit;
-
   explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
       : FixedWidthType(Type::TIMESTAMP), unit(unit) {}
 
+  explicit TimestampType(const std::string& timezone, TimeUnit unit = TimeUnit::MILLI)
+      : FixedWidthType(Type::TIMESTAMP), unit(unit), timezone(timezone) {}
+
   TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override { return name(); }
   static std::string name() { return "timestamp"; }
+
+  TimeUnit unit;
+  std::string timezone;
 };
 
 struct ARROW_EXPORT IntervalType : public FixedWidthType {
@@ -507,7 +530,7 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
   static constexpr Type::type type_id = Type::DICTIONARY;
 
   DictionaryType(const std::shared_ptr<DataType>& index_type,
-      const std::shared_ptr<Array>& dictionary);
+      const std::shared_ptr<Array>& dictionary, bool ordered = false);
 
   int bit_width() const override;
 
@@ -518,11 +541,13 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
 
+  bool ordered() const { return ordered_; }
+
  private:
   // Must be an integer type (not currently checked)
   std::shared_ptr<DataType> index_type_;
-
   std::shared_ptr<Array> dictionary_;
+  bool ordered_;
 };
 
 // ----------------------------------------------------------------------
@@ -532,6 +557,8 @@ std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
 
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit unit);
+std::shared_ptr<DataType> ARROW_EXPORT timestamp(
+    const std::string& timezone, TimeUnit unit);
 std::shared_ptr<DataType> ARROW_EXPORT time(TimeUnit unit);
 
 std::shared_ptr<DataType> ARROW_EXPORT struct_(
@@ -595,6 +622,7 @@ static inline bool is_primitive(Type::type type_id) {
     case Type::FLOAT:
     case Type::DOUBLE:
     case Type::DATE:
+    case Type::DATE32:
     case Type::TIMESTAMP:
     case Type::TIME:
     case Type::INTERVAL:
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index fc4ad3d87d8ac..e53afe1a34d36 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -95,6 +95,10 @@ struct DateType;
 using DateArray = NumericArray<DateType>;
 using DateBuilder = NumericBuilder<DateType>;
 
+struct Date32Type;
+using Date32Array = NumericArray<Date32Type>;
+using Date32Builder = NumericBuilder<Date32Type>;
+
 struct TimeType;
 using TimeArray = NumericArray<TimeType>;
 using TimeBuilder = NumericBuilder<TimeType>;
@@ -125,6 +129,7 @@ std::shared_ptr<DataType> ARROW_EXPORT float64();
 std::shared_ptr<DataType> ARROW_EXPORT utf8();
 std::shared_ptr<DataType> ARROW_EXPORT binary();
 std::shared_ptr<DataType> ARROW_EXPORT date();
+std::shared_ptr<DataType> ARROW_EXPORT date32();
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index d6687c11bcf73..2cd14203cdbb1 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -130,6 +130,18 @@ struct TypeTraits<DateType> {
   static inline std::shared_ptr<DataType> type_singleton() { return date(); }
 };
 
+template <>
+struct TypeTraits<Date32Type> {
+  using ArrayType = Date32Array;
+  using BuilderType = Date32Builder;
+
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(int32_t);
+  }
+  constexpr static bool is_parameter_free = true;
+  static inline std::shared_ptr<DataType> type_singleton() { return date32(); }
+};
+
 template <>
 struct TypeTraits<TimestampType> {
   using ArrayType = TimestampArray;
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 7787e95df5e72..6a6b4ba9ad0cb 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -54,7 +54,8 @@ cdef class Array:
         self.type.init(self.sp_array.get().type())
 
     @staticmethod
-    def from_pandas(obj, mask=None, timestamps_to_ms=False, Field field=None, MemoryPool memory_pool=None):
+    def from_pandas(obj, mask=None, timestamps_to_ms=False, Field field=None,
+                    MemoryPool memory_pool=None):
         """
         Convert pandas.Series to an Arrow Array.
 
@@ -75,8 +76,9 @@ cdef class Array:
 
         Notes
         -----
-        Localized timestamps will currently be returned as UTC (pandas's native representation).
-        Timezone-naive data will be implicitly interpreted as UTC.
+        Localized timestamps will currently be returned as UTC (pandas's native
+        representation).  Timezone-naive data will be implicitly interpreted as
+        UTC.
 
         Examples
         --------
@@ -119,9 +121,9 @@ cdef class Array:
         series_values = get_series_values(obj)
 
         if isinstance(series_values, pd.Categorical):
-            return DictionaryArray.from_arrays(series_values.codes,
-                                               series_values.categories.values,
-                                               mask=mask, memory_pool=memory_pool)
+            return DictionaryArray.from_arrays(
+                series_values.codes, series_values.categories.values,
+                mask=mask, memory_pool=memory_pool)
         else:
             if series_values.dtype.type == np.datetime64 and timestamps_to_ms:
                 series_values = series_values.astype('datetime64[ms]')
@@ -134,7 +136,8 @@ cdef class Array:
             return box_array(out)
 
     @staticmethod
-    def from_list(object list_obj, DataType type=None, MemoryPool memory_pool=None):
+    def from_list(object list_obj, DataType type=None,
+                  MemoryPool memory_pool=None):
         """
         Convert Python list to Arrow array
 
@@ -358,7 +361,8 @@ cdef class BinaryArray(Array):
 cdef class DictionaryArray(Array):
 
     @staticmethod
-    def from_arrays(indices, dictionary, mask=None, MemoryPool memory_pool=None):
+    def from_arrays(indices, dictionary, mask=None,
+                    MemoryPool memory_pool=None):
         """
         Construct Arrow DictionaryArray from array of indices (must be
         non-negative integers) and corresponding array of dictionary values
@@ -380,8 +384,15 @@ cdef class DictionaryArray(Array):
             shared_ptr[CDataType] c_type
             shared_ptr[CArray] c_result
 
-        arrow_indices = Array.from_pandas(indices, mask=mask, memory_pool=memory_pool)
-        arrow_dictionary = Array.from_pandas(dictionary, memory_pool=memory_pool)
+        if mask is None:
+            mask = indices == -1
+        else:
+            mask = mask | (indices == -1)
+
+        arrow_indices = Array.from_pandas(indices, mask=mask,
+                                          memory_pool=memory_pool)
+        arrow_dictionary = Array.from_pandas(dictionary,
+                                             memory_pool=memory_pool)
 
         if not isinstance(arrow_indices, IntegerArray):
             raise ValueError('Indices must be integer type')
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 93bc6ddcd56f6..ad5af1b0128ca 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -359,7 +359,8 @@ cdef class RecordBatch:
         """
         Number of rows
 
-        Due to the definition of a RecordBatch, all columns have the same number of rows.
+        Due to the definition of a RecordBatch, all columns have the same
+        number of rows.
 
         Returns
         -------
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 960653dca279e..953fa2c4b9a72 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -81,7 +81,11 @@ def _check_array_roundtrip(self, values, expected=None,
         arr = A.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
                                   field=field)
         result = arr.to_pandas()
-        tm.assert_series_equal(pd.Series(result), pd.Series(values), check_names=False)
+
+        assert arr.null_count == pd.isnull(values).sum()
+
+        tm.assert_series_equal(pd.Series(result), pd.Series(values),
+                               check_names=False)
 
     def test_float_no_nulls(self):
         data = {}
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index cadb53e0d2ab9..c707ada9dd55c 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -34,6 +34,7 @@
 #include <unordered_map>
 
 #include "arrow/api.h"
+#include "arrow/loader.h"
 #include "arrow/status.h"
 #include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
@@ -610,6 +611,7 @@ class PandasBlock {
     DOUBLE,
     BOOL,
     DATETIME,
+    DATETIME_WITH_TZ,
     CATEGORICAL
   };
 
@@ -1157,7 +1159,7 @@ class DataFrameBlockCreator {
       }
 
       int block_placement = 0;
-      if (column_type == Type::DICTIONARY) {
+      if (output_type == PandasBlock::CATEGORICAL) {
         std::shared_ptr<PandasBlock> block;
         RETURN_NOT_OK(MakeCategoricalBlock(col->type(), table_->num_rows(), &block));
         categorical_blocks_[i] = block;
@@ -1518,15 +1520,16 @@ inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
     null_count = ValuesToBitmap<TYPE>(PyArray_DATA(arr_), length_, null_bitmap_data_);
   }
 
-  // For readability
-  constexpr int64_t kOffset = 0;
-
   RETURN_NOT_OK(ConvertData());
   std::shared_ptr<DataType> type;
   RETURN_NOT_OK(MakeDataType(&type));
-  RETURN_NOT_OK(
-      MakePrimitiveArray(type, length_, data_, null_bitmap_, null_count, kOffset, out));
-  return Status::OK();
+
+  std::vector<arrow::FieldMetadata> fields(1);
+  fields[0].length = length_;
+  fields[0].null_count = null_count;
+  fields[0].offset = 0;
+
+  return arrow::LoadArray(type, fields, {null_bitmap_, data_}, out);
 }
 
 template <>

From f7f915d90aee8affb40616bc14877afde9b32898 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Fri, 10 Mar 2017 19:39:35 -0500
Subject: [PATCH 0353/1644] ARROW-615: [Java] Moved
 ByteArrayReadableSeekableByteChannel to src main o.a.a.vector.util

The ByteArrayReadableSeekableByteChannel is useful when reading an ArrowRecordBatch from a byte array with ArrowReader. Currently it is vector.file test package, this change moves the class to src/main/java/o.a.a.vector.util.  Updated test usage.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #370 from BryanCutler/move-ByteArrayReadableSeekableByteChannel-ARROW-615 and squashes the following commits:

46f32a3 [Bryan Cutler] moved ByteArrayReadableSeekableByteChannel to src main o.a.a.vector.util
---
 .../vector/util}/ByteArrayReadableSeekableByteChannel.java      | 2 +-
 .../org/apache/arrow/vector/file/TestArrowReaderWriter.java     | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 rename java/vector/src/{test/java/org/apache/arrow/vector/file => main/java/org/apache/arrow/vector/util}/ByteArrayReadableSeekableByteChannel.java (98%)

diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/ByteArrayReadableSeekableByteChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteArrayReadableSeekableByteChannel.java
similarity index 98%
rename from java/vector/src/test/java/org/apache/arrow/vector/file/ByteArrayReadableSeekableByteChannel.java
rename to java/vector/src/main/java/org/apache/arrow/vector/util/ByteArrayReadableSeekableByteChannel.java
index 7c423d5881aea..69840fefa968b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/ByteArrayReadableSeekableByteChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteArrayReadableSeekableByteChannel.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.util;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index 96bcbb1dae71c..13b04de68fa62 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -39,6 +39,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
 import org.junit.Before;
 import org.junit.Test;
 

From d99958dd3de0ac4fd6a99127d62657249c494448 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 11 Mar 2017 16:58:45 -0500
Subject: [PATCH 0354/1644] ARROW-452: [C++/Python] Incorporate C++ and Python
 codebases for Feather file format

The goal for this patch is to provide an eventual migration path for Feather (https://github.com/wesm/feather) users to use the batch and streaming Arrow file formats internally. Eventually the Feather metadata can be deprecated, but we will need to wait for the R community to build and ship Arrow bindings for R before that can happen. In the meantime, we won't need to maintain multiple Python/C++ codebases for the Python side of things.

The test suite isn't yet passing because support for timestamps with time zones has not been implemented in the conversion to pandas.DataFrame, so I will do that when I can, but this can be reviewed in the meantime.

I would upload a Gerrit code review, but there are some access control settings on gerrit.cloudera.org that need changing.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #361 from wesm/ARROW-452 and squashes the following commits:

b7bfd30 [Wes McKinney] Add missing license header
06cbdca [Wes McKinney] Fix -Wconversion error
244959c [Wes McKinney] Mark datetime+tz tests as xfail
9a95094 [Wes McKinney] Incorporate Feather C++ and Python codebases and do associated refactoring to maximize code reuse with IPC reader/writer classes. Get C++ test suite passing.
---
 cpp/src/arrow/ipc/CMakeLists.txt      |  22 +-
 cpp/src/arrow/ipc/api.h               |   1 +
 cpp/src/arrow/ipc/feather-internal.h  | 232 ++++++++
 cpp/src/arrow/ipc/feather-test.cc     | 437 +++++++++++++++
 cpp/src/arrow/ipc/feather.cc          | 729 ++++++++++++++++++++++++++
 cpp/src/arrow/ipc/feather.fbs         | 147 ++++++
 cpp/src/arrow/ipc/feather.h           | 109 ++++
 python/CMakeLists.txt                 |   1 +
 python/pyarrow/_feather.pyx           | 158 ++++++
 python/pyarrow/feather.py             | 118 +++++
 python/pyarrow/table.pyx              |   5 +
 python/pyarrow/tests/test_feather.py  | 379 +++++++++++++
 python/setup.py                       |   1 +
 python/src/pyarrow/adapters/pandas.cc |  83 ++-
 14 files changed, 2394 insertions(+), 28 deletions(-)
 create mode 100644 cpp/src/arrow/ipc/feather-internal.h
 create mode 100644 cpp/src/arrow/ipc/feather-test.cc
 create mode 100644 cpp/src/arrow/ipc/feather.cc
 create mode 100644 cpp/src/arrow/ipc/feather.fbs
 create mode 100644 cpp/src/arrow/ipc/feather.h
 create mode 100644 python/pyarrow/_feather.pyx
 create mode 100644 python/pyarrow/feather.py
 create mode 100644 python/pyarrow/tests/test_feather.py

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 08da0a109c963..09a959ba69b51 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -25,11 +25,12 @@ set(ARROW_IPC_SHARED_LINK_LIBS
 )
 
 set(ARROW_IPC_TEST_LINK_LIBS
-  arrow_io_static
-  arrow_ipc_static)
+  arrow_ipc_static
+  arrow_io_static)
 
 set(ARROW_IPC_SRCS
   adapter.cc
+  feather.cc
   json.cc
   json-internal.cc
   metadata.cc
@@ -59,6 +60,10 @@ if(FLATBUFFERS_VENDORED)
   add_dependencies(arrow_ipc_objlib flatbuffers_ep)
 endif()
 
+ADD_ARROW_TEST(feather-test)
+ARROW_TEST_LINK_LIBRARIES(feather-test
+  ${ARROW_IPC_TEST_LINK_LIBS})
+
 ADD_ARROW_TEST(ipc-adapter-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-adapter-test
   ${ARROW_IPC_TEST_LINK_LIBS})
@@ -105,14 +110,20 @@ if (ARROW_BUILD_TESTS)
 endif()
 
 # make clean will delete the generated file
-set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
+set_source_files_properties(Message_generated.h PROPERTIES GENERATED TRUE)
+set_source_files_properties(feather_generated.h PROPERTIES GENERATED TRUE)
+set_source_files_properties(File_generated.h PROPERTIES GENERATED TRUE)
 
 set(OUTPUT_DIR ${CMAKE_SOURCE_DIR}/src/arrow/ipc)
-set(FBS_OUTPUT_FILES "${OUTPUT_DIR}/Message_generated.h")
+set(FBS_OUTPUT_FILES
+  "${OUTPUT_DIR}/File_generated.h"
+  "${OUTPUT_DIR}/Message_generated.h"
+  "${OUTPUT_DIR}/feather_generated.h")
 
 set(FBS_SRC
   ${CMAKE_SOURCE_DIR}/../format/Message.fbs
-  ${CMAKE_SOURCE_DIR}/../format/File.fbs)
+  ${CMAKE_SOURCE_DIR}/../format/File.fbs
+  ${CMAKE_CURRENT_SOURCE_DIR}/feather.fbs)
 
 foreach(FIL ${FBS_SRC})
   get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
@@ -139,6 +150,7 @@ add_dependencies(arrow_ipc_objlib metadata_fbs)
 install(FILES
   adapter.h
   api.h
+  feather.h
   json.h
   metadata.h
   reader.h
diff --git a/cpp/src/arrow/ipc/api.h b/cpp/src/arrow/ipc/api.h
index cb854212bbeee..ad7cd84e9f986 100644
--- a/cpp/src/arrow/ipc/api.h
+++ b/cpp/src/arrow/ipc/api.h
@@ -19,6 +19,7 @@
 #define ARROW_IPC_API_H
 
 #include "arrow/ipc/adapter.h"
+#include "arrow/ipc/feather.h"
 #include "arrow/ipc/json.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/reader.h"
diff --git a/cpp/src/arrow/ipc/feather-internal.h b/cpp/src/arrow/ipc/feather-internal.h
new file mode 100644
index 0000000000000..10b0cfd5d5ea2
--- /dev/null
+++ b/cpp/src/arrow/ipc/feather-internal.h
@@ -0,0 +1,232 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Public API for the "Feather" file format, originally created at
+/// http://github.com/wesm/feather
+
+#ifndef ARROW_IPC_FEATHER_INTERNAL_H
+#define ARROW_IPC_FEATHER_INTERNAL_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "flatbuffers/flatbuffers.h"
+
+#include "arrow/buffer.h"
+#include "arrow/ipc/feather.h"
+#include "arrow/ipc/feather_generated.h"
+#include "arrow/type.h"
+
+namespace arrow {
+namespace ipc {
+namespace feather {
+
+typedef std::vector<flatbuffers::Offset<fbs::Column>> ColumnVector;
+typedef flatbuffers::FlatBufferBuilder FBB;
+typedef flatbuffers::Offset<flatbuffers::String> FBString;
+
+struct ColumnType {
+  enum type { PRIMITIVE, CATEGORY, TIMESTAMP, DATE, TIME };
+};
+
+struct ArrayMetadata {
+  ArrayMetadata() {}
+
+  ArrayMetadata(fbs::Type type, int64_t offset, int64_t length, int64_t null_count,
+      int64_t total_bytes)
+      : type(type),
+        offset(offset),
+        length(length),
+        null_count(null_count),
+        total_bytes(total_bytes) {}
+
+  bool Equals(const ArrayMetadata& other) const {
+    return this->type == other.type && this->offset == other.offset &&
+           this->length == other.length && this->null_count == other.null_count &&
+           this->total_bytes == other.total_bytes;
+  }
+
+  fbs::Type type;
+  int64_t offset;
+  int64_t length;
+  int64_t null_count;
+  int64_t total_bytes;
+};
+
+struct CategoryMetadata {
+  ArrayMetadata levels;
+  bool ordered;
+};
+
+struct TimestampMetadata {
+  TimeUnit unit;
+
+  // A timezone name known to the Olson timezone database. For display purposes
+  // because the actual data is all UTC
+  std::string timezone;
+};
+
+struct TimeMetadata {
+  TimeUnit unit;
+};
+
+static constexpr const char* kFeatherMagicBytes = "FEA1";
+static constexpr const int kFeatherDefaultAlignment = 8;
+
+class ColumnBuilder;
+
+class TableBuilder {
+ public:
+  explicit TableBuilder(int64_t num_rows);
+  ~TableBuilder() = default;
+
+  FBB& fbb();
+  Status Finish();
+  std::shared_ptr<Buffer> GetBuffer() const;
+
+  std::unique_ptr<ColumnBuilder> AddColumn(const std::string& name);
+  void SetDescription(const std::string& description);
+  void SetNumRows(int64_t num_rows);
+  void add_column(const flatbuffers::Offset<fbs::Column>& col);
+
+ private:
+  flatbuffers::FlatBufferBuilder fbb_;
+  ColumnVector columns_;
+
+  friend class ColumnBuilder;
+
+  bool finished_;
+  std::string description_;
+  int64_t num_rows_;
+};
+
+class TableMetadata {
+ public:
+  TableMetadata() {}
+  ~TableMetadata() = default;
+
+  Status Open(const std::shared_ptr<Buffer>& buffer) {
+    metadata_buffer_ = buffer;
+    table_ = fbs::GetCTable(buffer->data());
+
+    if (table_->version() < kFeatherVersion) {
+      std::cout << "This Feather file is old"
+                << " and will not be readable beyond the 0.3.0 release" << std::endl;
+    }
+    return Status::OK();
+  }
+
+  bool HasDescription() const { return table_->description() != 0; }
+
+  std::string GetDescription() const {
+    if (!HasDescription()) { return std::string(""); }
+    return table_->description()->str();
+  }
+
+  int version() const { return table_->version(); }
+  int64_t num_rows() const { return table_->num_rows(); }
+  int64_t num_columns() const { return table_->columns()->size(); }
+
+  const fbs::Column* column(int i) { return table_->columns()->Get(i); }
+
+ private:
+  std::shared_ptr<Buffer> metadata_buffer_;
+  const fbs::CTable* table_;
+};
+
+static inline flatbuffers::Offset<fbs::PrimitiveArray> GetPrimitiveArray(
+    FBB& fbb, const ArrayMetadata& array) {
+  return fbs::CreatePrimitiveArray(fbb, array.type, fbs::Encoding_PLAIN, array.offset,
+      array.length, array.null_count, array.total_bytes);
+}
+
+static inline fbs::TimeUnit ToFlatbufferEnum(TimeUnit unit) {
+  return static_cast<fbs::TimeUnit>(static_cast<int>(unit));
+}
+
+static inline TimeUnit FromFlatbufferEnum(fbs::TimeUnit unit) {
+  return static_cast<TimeUnit>(static_cast<int>(unit));
+}
+
+// Convert Feather enums to Flatbuffer enums
+
+const fbs::TypeMetadata COLUMN_TYPE_ENUM_MAPPING[] = {
+    fbs::TypeMetadata_NONE,               // PRIMITIVE
+    fbs::TypeMetadata_CategoryMetadata,   // CATEGORY
+    fbs::TypeMetadata_TimestampMetadata,  // TIMESTAMP
+    fbs::TypeMetadata_DateMetadata,       // DATE
+    fbs::TypeMetadata_TimeMetadata        // TIME
+};
+
+static inline fbs::TypeMetadata ToFlatbufferEnum(ColumnType::type column_type) {
+  return COLUMN_TYPE_ENUM_MAPPING[column_type];
+}
+
+static inline void FromFlatbuffer(const fbs::PrimitiveArray* values, ArrayMetadata* out) {
+  out->type = values->type();
+  out->offset = values->offset();
+  out->length = values->length();
+  out->null_count = values->null_count();
+  out->total_bytes = values->total_bytes();
+}
+
+class ColumnBuilder {
+ public:
+  ColumnBuilder(TableBuilder* parent, const std::string& name);
+  ~ColumnBuilder() = default;
+
+  flatbuffers::Offset<void> CreateColumnMetadata();
+
+  Status Finish();
+  void SetValues(const ArrayMetadata& values);
+  void SetUserMetadata(const std::string& data);
+  void SetCategory(const ArrayMetadata& levels, bool ordered = false);
+  void SetTimestamp(TimeUnit unit);
+  void SetTimestamp(TimeUnit unit, const std::string& timezone);
+  void SetDate();
+  void SetTime(TimeUnit unit);
+  FBB& fbb();
+
+ private:
+  TableBuilder* parent_;
+
+  std::string name_;
+  ArrayMetadata values_;
+  std::string user_metadata_;
+
+  // Column metadata
+
+  // Is this a primitive type, or one of the types having metadata? Default is
+  // primitive
+  ColumnType::type type_;
+
+  // Type-specific metadata union
+  CategoryMetadata meta_category_;
+  TimeMetadata meta_time_;
+
+  TimestampMetadata meta_timestamp_;
+
+  FBB* fbb_;
+};
+
+}  // namespace feather
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_FEATHER_INTERNAL_H
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
new file mode 100644
index 0000000000000..b73246b672260
--- /dev/null
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -0,0 +1,437 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <memory>
+#include <random>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/io/memory.h"
+#include "arrow/ipc/feather-internal.h"
+#include "arrow/ipc/feather.h"
+#include "arrow/ipc/test-common.h"
+#include "arrow/loader.h"
+#include "arrow/pretty_print.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+namespace ipc {
+namespace feather {
+
+template <typename T>
+inline void assert_vector_equal(const std::vector<T>& left, const std::vector<T>& right) {
+  ASSERT_EQ(left.size(), right.size());
+
+  for (size_t i = 0; i < left.size(); ++i) {
+    ASSERT_EQ(left[i], right[i]) << i;
+  }
+}
+
+class TestTableBuilder : public ::testing::Test {
+ public:
+  void SetUp() { tb_.reset(new TableBuilder(1000)); }
+
+  virtual void Finish() {
+    tb_->Finish();
+
+    table_.reset(new TableMetadata());
+    ASSERT_OK(table_->Open(tb_->GetBuffer()));
+  }
+
+ protected:
+  std::unique_ptr<TableBuilder> tb_;
+  std::unique_ptr<TableMetadata> table_;
+};
+
+TEST_F(TestTableBuilder, Version) {
+  Finish();
+  ASSERT_EQ(kFeatherVersion, table_->version());
+}
+
+TEST_F(TestTableBuilder, EmptyTable) {
+  Finish();
+
+  ASSERT_FALSE(table_->HasDescription());
+  ASSERT_EQ("", table_->GetDescription());
+  ASSERT_EQ(1000, table_->num_rows());
+  ASSERT_EQ(0, table_->num_columns());
+}
+
+TEST_F(TestTableBuilder, SetDescription) {
+  std::string desc("this is some good data");
+  tb_->SetDescription(desc);
+  Finish();
+  ASSERT_TRUE(table_->HasDescription());
+  ASSERT_EQ(desc, table_->GetDescription());
+}
+
+void AssertArrayEquals(const ArrayMetadata& left, const ArrayMetadata& right) {
+  EXPECT_EQ(left.type, right.type);
+  EXPECT_EQ(left.offset, right.offset);
+  EXPECT_EQ(left.length, right.length);
+  EXPECT_EQ(left.null_count, right.null_count);
+  EXPECT_EQ(left.total_bytes, right.total_bytes);
+}
+
+TEST_F(TestTableBuilder, AddPrimitiveColumn) {
+  std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("f0");
+
+  ArrayMetadata values1;
+  ArrayMetadata values2;
+  values1.type = fbs::Type_INT32;
+  values1.offset = 10000;
+  values1.length = 1000;
+  values1.null_count = 100;
+  values1.total_bytes = 4000;
+
+  cb->SetValues(values1);
+
+  std::string user_meta = "as you wish";
+  cb->SetUserMetadata(user_meta);
+
+  cb->Finish();
+
+  cb = tb_->AddColumn("f1");
+
+  values2.type = fbs::Type_UTF8;
+  values2.offset = 14000;
+  values2.length = 1000;
+  values2.null_count = 100;
+  values2.total_bytes = 10000;
+
+  cb->SetValues(values2);
+  cb->Finish();
+
+  Finish();
+
+  ASSERT_EQ(2, table_->num_columns());
+
+  auto col = table_->column(0);
+
+  ASSERT_EQ("f0", col->name()->str());
+  ASSERT_EQ(user_meta, col->user_metadata()->str());
+
+  ArrayMetadata values3;
+  FromFlatbuffer(col->values(), &values3);
+  AssertArrayEquals(values3, values1);
+
+  col = table_->column(1);
+  ASSERT_EQ("f1", col->name()->str());
+
+  ArrayMetadata values4;
+  FromFlatbuffer(col->values(), &values4);
+  AssertArrayEquals(values4, values2);
+}
+
+TEST_F(TestTableBuilder, AddCategoryColumn) {
+  ArrayMetadata values1(fbs::Type_UINT8, 10000, 1000, 100, 4000);
+  ArrayMetadata levels(fbs::Type_UTF8, 14000, 10, 0, 300);
+
+  std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("c0");
+  cb->SetValues(values1);
+  cb->SetCategory(levels);
+  cb->Finish();
+
+  cb = tb_->AddColumn("c1");
+  cb->SetValues(values1);
+  cb->SetCategory(levels, true);
+  cb->Finish();
+
+  Finish();
+
+  auto col = table_->column(0);
+  ASSERT_EQ(fbs::TypeMetadata_CategoryMetadata, col->metadata_type());
+
+  ArrayMetadata result;
+  FromFlatbuffer(col->values(), &result);
+  AssertArrayEquals(result, values1);
+
+  auto cat_ptr = static_cast<const fbs::CategoryMetadata*>(col->metadata());
+  ASSERT_FALSE(cat_ptr->ordered());
+
+  FromFlatbuffer(cat_ptr->levels(), &result);
+  AssertArrayEquals(result, levels);
+
+  col = table_->column(1);
+  cat_ptr = static_cast<const fbs::CategoryMetadata*>(col->metadata());
+  ASSERT_TRUE(cat_ptr->ordered());
+  FromFlatbuffer(cat_ptr->levels(), &result);
+  AssertArrayEquals(result, levels);
+}
+
+TEST_F(TestTableBuilder, AddTimestampColumn) {
+  ArrayMetadata values1(fbs::Type_INT64, 10000, 1000, 100, 4000);
+  std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("c0");
+  cb->SetValues(values1);
+  cb->SetTimestamp(TimeUnit::MILLI);
+  cb->Finish();
+
+  cb = tb_->AddColumn("c1");
+
+  std::string tz("America/Los_Angeles");
+
+  cb->SetValues(values1);
+  cb->SetTimestamp(TimeUnit::SECOND, tz);
+  cb->Finish();
+
+  Finish();
+
+  auto col = table_->column(0);
+
+  ASSERT_EQ(fbs::TypeMetadata_TimestampMetadata, col->metadata_type());
+
+  ArrayMetadata result;
+  FromFlatbuffer(col->values(), &result);
+  AssertArrayEquals(result, values1);
+
+  auto ts_ptr = static_cast<const fbs::TimestampMetadata*>(col->metadata());
+  ASSERT_EQ(fbs::TimeUnit_MILLISECOND, ts_ptr->unit());
+
+  col = table_->column(1);
+  ts_ptr = static_cast<const fbs::TimestampMetadata*>(col->metadata());
+  ASSERT_EQ(fbs::TimeUnit_SECOND, ts_ptr->unit());
+  ASSERT_EQ(tz, ts_ptr->timezone()->str());
+}
+
+TEST_F(TestTableBuilder, AddDateColumn) {
+  ArrayMetadata values1(fbs::Type_INT64, 10000, 1000, 100, 4000);
+  std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("d0");
+  cb->SetValues(values1);
+  cb->SetDate();
+  cb->Finish();
+
+  Finish();
+
+  auto col = table_->column(0);
+
+  ASSERT_EQ(fbs::TypeMetadata_DateMetadata, col->metadata_type());
+  ArrayMetadata result;
+  FromFlatbuffer(col->values(), &result);
+  AssertArrayEquals(result, values1);
+}
+
+TEST_F(TestTableBuilder, AddTimeColumn) {
+  ArrayMetadata values1(fbs::Type_INT64, 10000, 1000, 100, 4000);
+  std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("c0");
+  cb->SetValues(values1);
+  cb->SetTime(TimeUnit::SECOND);
+  cb->Finish();
+  Finish();
+
+  auto col = table_->column(0);
+
+  ASSERT_EQ(fbs::TypeMetadata_TimeMetadata, col->metadata_type());
+  ArrayMetadata result;
+  FromFlatbuffer(col->values(), &result);
+  AssertArrayEquals(result, values1);
+
+  auto t_ptr = static_cast<const fbs::TimeMetadata*>(col->metadata());
+  ASSERT_EQ(fbs::TimeUnit_SECOND, t_ptr->unit());
+}
+
+void CheckArrays(const Array& expected, const Array& result) {
+  if (!result.Equals(expected)) {
+    std::stringstream pp_result;
+    std::stringstream pp_expected;
+
+    EXPECT_OK(PrettyPrint(result, 0, &pp_result));
+    EXPECT_OK(PrettyPrint(expected, 0, &pp_expected));
+    FAIL() << "Got: " << pp_result.str() << "\nExpected: " << pp_expected.str();
+  }
+}
+
+class TestTableWriter : public ::testing::Test {
+ public:
+  void SetUp() {
+    ASSERT_OK(io::BufferOutputStream::Create(1024, default_memory_pool(), &stream_));
+    ASSERT_OK(TableWriter::Open(stream_, &writer_));
+  }
+
+  void Finish() {
+    // Write table footer
+    ASSERT_OK(writer_->Finalize());
+
+    ASSERT_OK(stream_->Finish(&output_));
+
+    std::shared_ptr<io::BufferReader> buffer(new io::BufferReader(output_));
+    reader_.reset(new TableReader());
+    ASSERT_OK(reader_->Open(buffer));
+  }
+
+  void CheckBatch(const RecordBatch& batch) {
+    for (int i = 0; i < batch.num_columns(); ++i) {
+      ASSERT_OK(writer_->Append(batch.column_name(i), *batch.column(i)));
+    }
+    Finish();
+
+    std::shared_ptr<Column> col;
+    for (int i = 0; i < batch.num_columns(); ++i) {
+      ASSERT_OK(reader_->GetColumn(i, &col));
+      ASSERT_EQ(batch.column_name(i), col->name());
+      CheckArrays(*batch.column(i), *col->data()->chunk(0));
+    }
+  }
+
+ protected:
+  std::shared_ptr<io::BufferOutputStream> stream_;
+  std::unique_ptr<TableWriter> writer_;
+  std::unique_ptr<TableReader> reader_;
+
+  std::shared_ptr<Buffer> output_;
+};
+
+TEST_F(TestTableWriter, EmptyTable) {
+  Finish();
+
+  ASSERT_FALSE(reader_->HasDescription());
+  ASSERT_EQ("", reader_->GetDescription());
+
+  ASSERT_EQ(0, reader_->num_rows());
+  ASSERT_EQ(0, reader_->num_columns());
+}
+
+TEST_F(TestTableWriter, SetNumRows) {
+  writer_->SetNumRows(1000);
+  Finish();
+  ASSERT_EQ(1000, reader_->num_rows());
+}
+
+TEST_F(TestTableWriter, SetDescription) {
+  std::string desc("contents of the file");
+  writer_->SetDescription(desc);
+  Finish();
+
+  ASSERT_TRUE(reader_->HasDescription());
+  ASSERT_EQ(desc, reader_->GetDescription());
+
+  ASSERT_EQ(0, reader_->num_rows());
+  ASSERT_EQ(0, reader_->num_columns());
+}
+
+TEST_F(TestTableWriter, PrimitiveRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeIntRecordBatch(&batch));
+
+  ASSERT_OK(writer_->Append("f0", *batch->column(0)));
+  ASSERT_OK(writer_->Append("f1", *batch->column(1)));
+  Finish();
+
+  std::shared_ptr<Column> col;
+  ASSERT_OK(reader_->GetColumn(0, &col));
+  ASSERT_TRUE(col->data()->chunk(0)->Equals(batch->column(0)));
+  ASSERT_EQ("f0", col->name());
+
+  ASSERT_OK(reader_->GetColumn(1, &col));
+  ASSERT_TRUE(col->data()->chunk(0)->Equals(batch->column(1)));
+  ASSERT_EQ("f1", col->name());
+}
+
+Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out) {
+  const int64_t length = 6;
+
+  std::vector<bool> is_valid = {true, true, false, true, true, true};
+  std::shared_ptr<Array> dict1, dict2;
+
+  std::vector<std::string> dict1_values = {"foo", "bar", "baz"};
+  std::vector<std::string> dict2_values = {"foo", "bar", "baz", "qux"};
+
+  ArrayFromVector<StringType, std::string>(dict1_values, &dict1);
+  ArrayFromVector<StringType, std::string>(dict2_values, &dict2);
+
+  auto f0_type = arrow::dictionary(arrow::int32(), dict1);
+  auto f1_type = arrow::dictionary(arrow::int8(), dict1);
+  auto f2_type = arrow::dictionary(arrow::int32(), dict2);
+
+  std::shared_ptr<Array> indices0, indices1, indices2;
+  std::vector<int32_t> indices0_values = {1, 2, -1, 0, 2, 0};
+  std::vector<int8_t> indices1_values = {0, 0, 2, 2, 1, 1};
+  std::vector<int32_t> indices2_values = {3, 0, 2, 1, 0, 2};
+
+  ArrayFromVector<Int32Type, int32_t>(is_valid, indices0_values, &indices0);
+  ArrayFromVector<Int8Type, int8_t>(is_valid, indices1_values, &indices1);
+  ArrayFromVector<Int32Type, int32_t>(is_valid, indices2_values, &indices2);
+
+  auto a0 = std::make_shared<DictionaryArray>(f0_type, indices0);
+  auto a1 = std::make_shared<DictionaryArray>(f1_type, indices1);
+  auto a2 = std::make_shared<DictionaryArray>(f2_type, indices2);
+
+  // construct batch
+  std::shared_ptr<Schema> schema(new Schema(
+      {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type)}));
+
+  std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2};
+  out->reset(new RecordBatch(schema, length, arrays));
+  return Status::OK();
+}
+
+TEST_F(TestTableWriter, CategoryRoundtrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeDictionaryFlat(&batch));
+  CheckBatch(*batch);
+}
+
+TEST_F(TestTableWriter, TimeTypes) {
+  std::vector<bool> is_valid = {true, true, true, false, true, true, true};
+  auto f0 = field("f0", date32());
+  auto f1 = field("f1", time(TimeUnit::MILLI));
+  auto f2 = field("f2", timestamp(TimeUnit::NANO));
+  auto f3 = field("f3", timestamp("US/Los_Angeles", TimeUnit::SECOND));
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2, f3}));
+
+  std::vector<int64_t> values_vec = {0, 1, 2, 3, 4, 5, 6};
+  std::shared_ptr<Array> values;
+  ArrayFromVector<Int64Type, int64_t>(is_valid, values_vec, &values);
+
+  std::vector<int32_t> date_values_vec = {0, 1, 2, 3, 4, 5, 6};
+  std::shared_ptr<Array> date_array;
+  ArrayFromVector<Date32Type, int32_t>(is_valid, date_values_vec, &date_array);
+
+  std::vector<FieldMetadata> fields(1);
+  fields[0].length = values->length();
+  fields[0].null_count = values->null_count();
+  fields[0].offset = 0;
+
+  const auto& prim_values = static_cast<const PrimitiveArray&>(*values);
+  std::vector<std::shared_ptr<Buffer>> buffers = {
+      prim_values.null_bitmap(), prim_values.data()};
+
+  std::vector<std::shared_ptr<Array>> arrays;
+  arrays.push_back(date_array);
+
+  for (int i = 1; i < schema->num_fields(); ++i) {
+    std::shared_ptr<Array> arr;
+    LoadArray(schema->field(i)->type, fields, buffers, &arr);
+    arrays.push_back(arr);
+  }
+
+  RecordBatch batch(schema, values->length(), arrays);
+  CheckBatch(batch);
+}
+
+TEST_F(TestTableWriter, VLenPrimitiveRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeStringTypesRecordBatch(&batch));
+  CheckBatch(*batch);
+}
+
+}  // namespace feather
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
new file mode 100644
index 0000000000000..13dfa5830f1bf
--- /dev/null
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -0,0 +1,729 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/feather.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "flatbuffers/flatbuffers.h"
+
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/column.h"
+#include "arrow/io/file.h"
+#include "arrow/ipc/feather-internal.h"
+#include "arrow/ipc/feather_generated.h"
+#include "arrow/loader.h"
+#include "arrow/status.h"
+#include "arrow/util/bit-util.h"
+
+namespace arrow {
+namespace ipc {
+namespace feather {
+
+static const uint8_t kPaddingBytes[kFeatherDefaultAlignment] = {0};
+
+static inline int64_t PaddedLength(int64_t nbytes) {
+  static const int64_t alignment = kFeatherDefaultAlignment;
+  return ((nbytes + alignment - 1) / alignment) * alignment;
+}
+
+// XXX: Hack for Feather 0.3.0 for backwards compatibility with old files
+// Size in-file of written byte buffer
+static int64_t GetOutputLength(int64_t nbytes) {
+  if (kFeatherVersion < 2) {
+    // Feather files < 0.3.0
+    return nbytes;
+  } else {
+    return PaddedLength(nbytes);
+  }
+}
+
+static Status WritePadded(io::OutputStream* stream, const uint8_t* data, int64_t length,
+    int64_t* bytes_written) {
+  RETURN_NOT_OK(stream->Write(data, length));
+
+  int64_t remainder = PaddedLength(length) - length;
+  if (remainder != 0) { RETURN_NOT_OK(stream->Write(kPaddingBytes, remainder)); }
+  *bytes_written = length + remainder;
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// TableBuilder
+
+TableBuilder::TableBuilder(int64_t num_rows) : finished_(false), num_rows_(num_rows) {}
+
+FBB& TableBuilder::fbb() {
+  return fbb_;
+}
+
+Status TableBuilder::Finish() {
+  if (finished_) { return Status::Invalid("can only call this once"); }
+
+  FBString desc = 0;
+  if (!description_.empty()) { desc = fbb_.CreateString(description_); }
+
+  flatbuffers::Offset<flatbuffers::String> metadata = 0;
+
+  auto root = fbs::CreateCTable(
+      fbb_, desc, num_rows_, fbb_.CreateVector(columns_), kFeatherVersion, metadata);
+  fbb_.Finish(root);
+  finished_ = true;
+
+  return Status::OK();
+}
+
+std::shared_ptr<Buffer> TableBuilder::GetBuffer() const {
+  return std::make_shared<Buffer>(
+      fbb_.GetBufferPointer(), static_cast<int64_t>(fbb_.GetSize()));
+}
+
+void TableBuilder::SetDescription(const std::string& description) {
+  description_ = description;
+}
+
+void TableBuilder::SetNumRows(int64_t num_rows) {
+  num_rows_ = num_rows;
+}
+
+void TableBuilder::add_column(const flatbuffers::Offset<fbs::Column>& col) {
+  columns_.push_back(col);
+}
+
+ColumnBuilder::ColumnBuilder(TableBuilder* parent, const std::string& name)
+    : parent_(parent) {
+  fbb_ = &parent->fbb();
+  name_ = name;
+  type_ = ColumnType::PRIMITIVE;
+}
+
+flatbuffers::Offset<void> ColumnBuilder::CreateColumnMetadata() {
+  switch (type_) {
+    case ColumnType::PRIMITIVE:
+      // flatbuffer void
+      return 0;
+    case ColumnType::CATEGORY: {
+      auto cat_meta = fbs::CreateCategoryMetadata(
+          fbb(), GetPrimitiveArray(fbb(), meta_category_.levels), meta_category_.ordered);
+      return cat_meta.Union();
+    }
+    case ColumnType::TIMESTAMP: {
+      // flatbuffer void
+      flatbuffers::Offset<flatbuffers::String> tz = 0;
+      if (!meta_timestamp_.timezone.empty()) {
+        tz = fbb().CreateString(meta_timestamp_.timezone);
+      }
+
+      auto ts_meta =
+          fbs::CreateTimestampMetadata(fbb(), ToFlatbufferEnum(meta_timestamp_.unit), tz);
+      return ts_meta.Union();
+    }
+    case ColumnType::DATE: {
+      auto date_meta = fbs::CreateDateMetadata(fbb());
+      return date_meta.Union();
+    }
+    case ColumnType::TIME: {
+      auto time_meta = fbs::CreateTimeMetadata(fbb(), ToFlatbufferEnum(meta_time_.unit));
+      return time_meta.Union();
+    }
+    default:
+      // null
+      return flatbuffers::Offset<void>();
+  }
+}
+
+Status ColumnBuilder::Finish() {
+  FBB& buf = fbb();
+
+  // values
+  auto values = GetPrimitiveArray(buf, values_);
+  flatbuffers::Offset<void> metadata = CreateColumnMetadata();
+
+  auto column = fbs::CreateColumn(buf, buf.CreateString(name_), values,
+      ToFlatbufferEnum(type_),  // metadata_type
+      metadata, buf.CreateString(user_metadata_));
+
+  // bad coupling, but OK for now
+  parent_->add_column(column);
+  return Status::OK();
+}
+
+void ColumnBuilder::SetValues(const ArrayMetadata& values) {
+  values_ = values;
+}
+
+void ColumnBuilder::SetUserMetadata(const std::string& data) {
+  user_metadata_ = data;
+}
+
+void ColumnBuilder::SetCategory(const ArrayMetadata& levels, bool ordered) {
+  type_ = ColumnType::CATEGORY;
+  meta_category_.levels = levels;
+  meta_category_.ordered = ordered;
+}
+
+void ColumnBuilder::SetTimestamp(TimeUnit unit) {
+  type_ = ColumnType::TIMESTAMP;
+  meta_timestamp_.unit = unit;
+}
+
+void ColumnBuilder::SetTimestamp(TimeUnit unit, const std::string& timezone) {
+  SetTimestamp(unit);
+  meta_timestamp_.timezone = timezone;
+}
+
+void ColumnBuilder::SetDate() {
+  type_ = ColumnType::DATE;
+}
+
+void ColumnBuilder::SetTime(TimeUnit unit) {
+  type_ = ColumnType::TIME;
+  meta_time_.unit = unit;
+}
+
+FBB& ColumnBuilder::fbb() {
+  return *fbb_;
+}
+
+std::unique_ptr<ColumnBuilder> TableBuilder::AddColumn(const std::string& name) {
+  return std::unique_ptr<ColumnBuilder>(new ColumnBuilder(this, name));
+}
+
+// ----------------------------------------------------------------------
+// reader.cc
+
+class TableReader::TableReaderImpl {
+ public:
+  TableReaderImpl() {}
+
+  Status Open(const std::shared_ptr<io::ReadableFileInterface>& source) {
+    source_ = source;
+
+    int magic_size = static_cast<int>(strlen(kFeatherMagicBytes));
+    int footer_size = magic_size + static_cast<int>(sizeof(uint32_t));
+
+    // Pathological issue where the file is smaller than
+    int64_t size = 0;
+    RETURN_NOT_OK(source->GetSize(&size));
+    if (size < magic_size + footer_size) {
+      return Status::Invalid("File is too small to be a well-formed file");
+    }
+
+    std::shared_ptr<Buffer> buffer;
+    RETURN_NOT_OK(source->Read(magic_size, &buffer));
+
+    if (memcmp(buffer->data(), kFeatherMagicBytes, magic_size)) {
+      return Status::Invalid("Not a feather file");
+    }
+
+    // Now get the footer and verify
+    RETURN_NOT_OK(source->ReadAt(size - footer_size, footer_size, &buffer));
+
+    if (memcmp(buffer->data() + sizeof(uint32_t), kFeatherMagicBytes, magic_size)) {
+      return Status::Invalid("Feather file footer incomplete");
+    }
+
+    uint32_t metadata_length = *reinterpret_cast<const uint32_t*>(buffer->data());
+    if (size < magic_size + footer_size + metadata_length) {
+      return Status::Invalid("File is smaller than indicated metadata size");
+    }
+    RETURN_NOT_OK(
+        source->ReadAt(size - footer_size - metadata_length, metadata_length, &buffer));
+
+    metadata_.reset(new TableMetadata());
+    return metadata_->Open(buffer);
+  }
+
+  Status GetDataType(const fbs::PrimitiveArray* values, fbs::TypeMetadata metadata_type,
+      const void* metadata, std::shared_ptr<DataType>* out) {
+#define PRIMITIVE_CASE(CAP_TYPE, FACTORY_FUNC) \
+  case fbs::Type_##CAP_TYPE:                   \
+    *out = FACTORY_FUNC();                     \
+    break;
+
+    switch (metadata_type) {
+      case fbs::TypeMetadata_CategoryMetadata: {
+        auto meta = static_cast<const fbs::CategoryMetadata*>(metadata);
+
+        std::shared_ptr<DataType> index_type;
+        RETURN_NOT_OK(GetDataType(values, fbs::TypeMetadata_NONE, nullptr, &index_type));
+
+        std::shared_ptr<Array> levels;
+        RETURN_NOT_OK(
+            LoadValues(meta->levels(), fbs::TypeMetadata_NONE, nullptr, &levels));
+
+        *out = std::make_shared<DictionaryType>(index_type, levels, meta->ordered());
+        break;
+      }
+      case fbs::TypeMetadata_TimestampMetadata: {
+        auto meta = static_cast<const fbs::TimestampMetadata*>(metadata);
+        TimeUnit unit = FromFlatbufferEnum(meta->unit());
+        std::string tz;
+        // flatbuffer non-null
+        if (meta->timezone() != 0) {
+          tz = meta->timezone()->str();
+        } else {
+          tz = "";
+        }
+        *out = std::make_shared<TimestampType>(tz, unit);
+      } break;
+      case fbs::TypeMetadata_DateMetadata:
+        *out = date32();
+        break;
+      case fbs::TypeMetadata_TimeMetadata: {
+        auto meta = static_cast<const fbs::TimeMetadata*>(metadata);
+        *out = std::make_shared<TimeType>(FromFlatbufferEnum(meta->unit()));
+      } break;
+      default:
+        switch (values->type()) {
+          PRIMITIVE_CASE(BOOL, boolean);
+          PRIMITIVE_CASE(INT8, int8);
+          PRIMITIVE_CASE(INT16, int16);
+          PRIMITIVE_CASE(INT32, int32);
+          PRIMITIVE_CASE(INT64, int64);
+          PRIMITIVE_CASE(UINT8, uint8);
+          PRIMITIVE_CASE(UINT16, uint16);
+          PRIMITIVE_CASE(UINT32, uint32);
+          PRIMITIVE_CASE(UINT64, uint64);
+          PRIMITIVE_CASE(FLOAT, float32);
+          PRIMITIVE_CASE(DOUBLE, float64);
+          PRIMITIVE_CASE(UTF8, utf8);
+          PRIMITIVE_CASE(BINARY, binary);
+          default:
+            return Status::Invalid("Unrecognized type");
+        }
+        break;
+    }
+
+#undef PRIMITIVE_CASE
+
+    return Status::OK();
+  }
+
+  // Retrieve a primitive array from the data source
+  //
+  // @returns: a Buffer instance, the precise type will depend on the kind of
+  // input data source (which may or may not have memory-map like semantics)
+  Status LoadValues(const fbs::PrimitiveArray* meta, fbs::TypeMetadata metadata_type,
+      const void* metadata, std::shared_ptr<Array>* out) {
+    std::shared_ptr<DataType> type;
+    RETURN_NOT_OK(GetDataType(meta, metadata_type, metadata, &type));
+
+    std::vector<FieldMetadata> fields(1);
+    std::vector<std::shared_ptr<Buffer>> buffers;
+
+    // Buffer data from the source (may or may not perform a copy depending on
+    // input source)
+    std::shared_ptr<Buffer> buffer;
+    RETURN_NOT_OK(source_->ReadAt(meta->offset(), meta->total_bytes(), &buffer));
+
+    int64_t offset = 0;
+
+    // If there are nulls, the null bitmask is first
+    if (meta->null_count() > 0) {
+      int64_t null_bitmap_size = GetOutputLength(BitUtil::BytesForBits(meta->length()));
+      buffers.push_back(SliceBuffer(buffer, offset, null_bitmap_size));
+      offset += null_bitmap_size;
+    } else {
+      buffers.push_back(nullptr);
+    }
+
+    if (is_binary_like(type->type)) {
+      int64_t offsets_size = GetOutputLength((meta->length() + 1) * sizeof(int32_t));
+      buffers.push_back(SliceBuffer(buffer, offset, offsets_size));
+      offset += offsets_size;
+    }
+
+    buffers.push_back(SliceBuffer(buffer, offset, buffer->size() - offset));
+
+    fields[0].length = meta->length();
+    fields[0].null_count = meta->null_count();
+    fields[0].offset = 0;
+
+    return LoadArray(type, fields, buffers, out);
+  }
+
+  bool HasDescription() const { return metadata_->HasDescription(); }
+
+  std::string GetDescription() const { return metadata_->GetDescription(); }
+
+  int version() const { return metadata_->version(); }
+  int64_t num_rows() const { return metadata_->num_rows(); }
+  int64_t num_columns() const { return metadata_->num_columns(); }
+
+  std::string GetColumnName(int i) const {
+    const fbs::Column* col_meta = metadata_->column(i);
+    return col_meta->name()->str();
+  }
+
+  Status GetColumn(int i, std::shared_ptr<Column>* out) {
+    const fbs::Column* col_meta = metadata_->column(i);
+
+    // auto user_meta = column->user_metadata();
+    // if (user_meta->size() > 0) { user_metadata_ = user_meta->str(); }
+
+    std::shared_ptr<Array> values;
+    RETURN_NOT_OK(LoadValues(
+        col_meta->values(), col_meta->metadata_type(), col_meta->metadata(), &values));
+    out->reset(new Column(col_meta->name()->str(), values));
+    return Status::OK();
+  }
+
+ private:
+  std::shared_ptr<io::ReadableFileInterface> source_;
+  std::unique_ptr<TableMetadata> metadata_;
+
+  std::shared_ptr<Schema> schema_;
+};
+
+// ----------------------------------------------------------------------
+// TableReader public API
+
+TableReader::TableReader() {
+  impl_.reset(new TableReaderImpl());
+}
+
+TableReader::~TableReader() {}
+
+Status TableReader::Open(const std::shared_ptr<io::ReadableFileInterface>& source) {
+  return impl_->Open(source);
+}
+
+Status TableReader::OpenFile(
+    const std::string& abspath, std::unique_ptr<TableReader>* out) {
+  std::shared_ptr<io::MemoryMappedFile> file;
+  RETURN_NOT_OK(io::MemoryMappedFile::Open(abspath, io::FileMode::READ, &file));
+  out->reset(new TableReader());
+  return (*out)->Open(file);
+}
+
+bool TableReader::HasDescription() const {
+  return impl_->HasDescription();
+}
+
+std::string TableReader::GetDescription() const {
+  return impl_->GetDescription();
+}
+
+int TableReader::version() const {
+  return impl_->version();
+}
+
+int64_t TableReader::num_rows() const {
+  return impl_->num_rows();
+}
+
+int64_t TableReader::num_columns() const {
+  return impl_->num_columns();
+}
+
+std::string TableReader::GetColumnName(int i) const {
+  return impl_->GetColumnName(i);
+}
+
+Status TableReader::GetColumn(int i, std::shared_ptr<Column>* out) {
+  return impl_->GetColumn(i, out);
+}
+
+// ----------------------------------------------------------------------
+// writer.cc
+
+fbs::Type ToFlatbufferType(Type::type type) {
+  switch (type) {
+    case Type::BOOL:
+      return fbs::Type_BOOL;
+    case Type::INT8:
+      return fbs::Type_INT8;
+    case Type::INT16:
+      return fbs::Type_INT16;
+    case Type::INT32:
+      return fbs::Type_INT32;
+    case Type::INT64:
+      return fbs::Type_INT64;
+    case Type::UINT8:
+      return fbs::Type_UINT8;
+    case Type::UINT16:
+      return fbs::Type_UINT16;
+    case Type::UINT32:
+      return fbs::Type_UINT32;
+    case Type::UINT64:
+      return fbs::Type_UINT64;
+    case Type::FLOAT:
+      return fbs::Type_FLOAT;
+    case Type::DOUBLE:
+      return fbs::Type_DOUBLE;
+    case Type::STRING:
+      return fbs::Type_UTF8;
+    case Type::BINARY:
+      return fbs::Type_BINARY;
+    case Type::DATE32:
+      return fbs::Type_DATE;
+    case Type::TIMESTAMP:
+      return fbs::Type_TIMESTAMP;
+    case Type::TIME:
+      return fbs::Type_TIME;
+    case Type::DICTIONARY:
+      return fbs::Type_CATEGORY;
+    default:
+      break;
+  }
+  // prevent compiler warning
+  return fbs::Type_MIN;
+}
+
+class TableWriter::TableWriterImpl : public ArrayVisitor {
+ public:
+  TableWriterImpl() : initialized_stream_(false), metadata_(0) {}
+
+  Status Open(const std::shared_ptr<io::OutputStream>& stream) {
+    stream_ = stream;
+    return Status::OK();
+  }
+
+  void SetDescription(const std::string& desc) { metadata_.SetDescription(desc); }
+
+  void SetNumRows(int64_t num_rows) { metadata_.SetNumRows(num_rows); }
+
+  Status Finalize() {
+    RETURN_NOT_OK(CheckStarted());
+    metadata_.Finish();
+
+    auto buffer = metadata_.GetBuffer();
+
+    // Writer metadata
+    int64_t bytes_written;
+    RETURN_NOT_OK(
+        WritePadded(stream_.get(), buffer->data(), buffer->size(), &bytes_written));
+    uint32_t buffer_size = static_cast<uint32_t>(bytes_written);
+
+    // Footer: metadata length, magic bytes
+    RETURN_NOT_OK(
+        stream_->Write(reinterpret_cast<const uint8_t*>(&buffer_size), sizeof(uint32_t)));
+    RETURN_NOT_OK(stream_->Write(reinterpret_cast<const uint8_t*>(kFeatherMagicBytes),
+        strlen(kFeatherMagicBytes)));
+    return stream_->Close();
+  }
+
+  Status LoadArrayMetadata(const Array& values, ArrayMetadata* meta) {
+    if (!(is_primitive(values.type_enum()) || is_binary_like(values.type_enum()))) {
+      std::stringstream ss;
+      ss << "Array is not primitive type: " << values.type()->ToString();
+      return Status::Invalid(ss.str());
+    }
+
+    meta->type = ToFlatbufferType(values.type_enum());
+
+    RETURN_NOT_OK(stream_->Tell(&meta->offset));
+
+    meta->length = values.length();
+    meta->null_count = values.null_count();
+    meta->total_bytes = 0;
+
+    return Status::OK();
+  }
+
+  Status WriteArray(const Array& values, ArrayMetadata* meta) {
+    RETURN_NOT_OK(CheckStarted());
+    RETURN_NOT_OK(LoadArrayMetadata(values, meta));
+
+    int64_t bytes_written;
+
+    // Write the null bitmask
+    if (values.null_count() > 0) {
+      // We assume there is one bit for each value in values.nulls, aligned on a
+      // byte boundary, and we write this much data into the stream
+      RETURN_NOT_OK(WritePadded(stream_.get(), values.null_bitmap()->data(),
+          values.null_bitmap()->size(), &bytes_written));
+      meta->total_bytes += bytes_written;
+    }
+
+    int64_t values_bytes = 0;
+
+    const uint8_t* values_buffer = nullptr;
+
+    if (is_binary_like(values.type_enum())) {
+      const auto& bin_values = static_cast<const BinaryArray&>(values);
+
+      int64_t offset_bytes = sizeof(int32_t) * (values.length() + 1);
+
+      values_bytes = bin_values.raw_value_offsets()[values.length()];
+
+      // Write the variable-length offsets
+      RETURN_NOT_OK(WritePadded(stream_.get(),
+          reinterpret_cast<const uint8_t*>(bin_values.raw_value_offsets()), offset_bytes,
+          &bytes_written))
+      meta->total_bytes += bytes_written;
+
+      if (bin_values.data()) { values_buffer = bin_values.data()->data(); }
+    } else {
+      const auto& prim_values = static_cast<const PrimitiveArray&>(values);
+      const auto& fw_type = static_cast<const FixedWidthType&>(*values.type());
+
+      if (values.type_enum() == Type::BOOL) {
+        // Booleans are bit-packed
+        values_bytes = BitUtil::BytesForBits(values.length());
+      } else {
+        values_bytes = values.length() * fw_type.bit_width() / 8;
+      }
+
+      if (prim_values.data()) { values_buffer = prim_values.data()->data(); }
+    }
+    RETURN_NOT_OK(
+        WritePadded(stream_.get(), values_buffer, values_bytes, &bytes_written));
+    meta->total_bytes += bytes_written;
+
+    return Status::OK();
+  }
+
+  Status WritePrimitiveValues(const Array& values) {
+    // Prepare metadata payload
+    ArrayMetadata meta;
+    RETURN_NOT_OK(WriteArray(values, &meta));
+    current_column_->SetValues(meta);
+    return Status::OK();
+  }
+
+#define VISIT_PRIMITIVE(TYPE) \
+  Status Visit(const TYPE& values) override { return WritePrimitiveValues(values); }
+
+  VISIT_PRIMITIVE(BooleanArray);
+  VISIT_PRIMITIVE(Int8Array);
+  VISIT_PRIMITIVE(Int16Array);
+  VISIT_PRIMITIVE(Int32Array);
+  VISIT_PRIMITIVE(Int64Array);
+  VISIT_PRIMITIVE(UInt8Array);
+  VISIT_PRIMITIVE(UInt16Array);
+  VISIT_PRIMITIVE(UInt32Array);
+  VISIT_PRIMITIVE(UInt64Array);
+  VISIT_PRIMITIVE(FloatArray);
+  VISIT_PRIMITIVE(DoubleArray);
+  VISIT_PRIMITIVE(BinaryArray);
+  VISIT_PRIMITIVE(StringArray);
+
+#undef VISIT_PRIMITIVE
+
+  Status Visit(const DictionaryArray& values) override {
+    const auto& dict_type = static_cast<const DictionaryType&>(*values.type());
+
+    if (!is_integer(values.indices()->type_enum())) {
+      return Status::Invalid("Category values must be integers");
+    }
+
+    RETURN_NOT_OK(WritePrimitiveValues(*values.indices()));
+
+    ArrayMetadata levels_meta;
+    RETURN_NOT_OK(WriteArray(*dict_type.dictionary(), &levels_meta));
+    current_column_->SetCategory(levels_meta, dict_type.ordered());
+    return Status::OK();
+  }
+
+  Status Visit(const TimestampArray& values) override {
+    RETURN_NOT_OK(WritePrimitiveValues(values));
+    const auto& ts_type = static_cast<const TimestampType&>(*values.type());
+    current_column_->SetTimestamp(ts_type.unit, ts_type.timezone);
+    return Status::OK();
+  }
+
+  Status Visit(const Date32Array& values) override {
+    RETURN_NOT_OK(WritePrimitiveValues(values));
+    current_column_->SetDate();
+    return Status::OK();
+  }
+
+  Status Visit(const TimeArray& values) override {
+    RETURN_NOT_OK(WritePrimitiveValues(values));
+    auto unit = static_cast<const TimeType&>(*values.type()).unit;
+    current_column_->SetTime(unit);
+    return Status::OK();
+  }
+
+  Status Append(const std::string& name, const Array& values) {
+    current_column_ = metadata_.AddColumn(name);
+    RETURN_NOT_OK(values.Accept(this));
+    current_column_->Finish();
+    return Status::OK();
+  }
+
+ private:
+  Status CheckStarted() {
+    if (!initialized_stream_) {
+      int64_t bytes_written_unused;
+      RETURN_NOT_OK(
+          WritePadded(stream_.get(), reinterpret_cast<const uint8_t*>(kFeatherMagicBytes),
+              strlen(kFeatherMagicBytes), &bytes_written_unused));
+      initialized_stream_ = true;
+    }
+    return Status::OK();
+  }
+
+  std::shared_ptr<io::OutputStream> stream_;
+
+  bool initialized_stream_;
+  TableBuilder metadata_;
+
+  std::unique_ptr<ColumnBuilder> current_column_;
+
+  Status AppendPrimitive(const PrimitiveArray& values, ArrayMetadata* out);
+};
+
+TableWriter::TableWriter() {
+  impl_.reset(new TableWriterImpl());
+}
+
+TableWriter::~TableWriter() {}
+
+Status TableWriter::Open(
+    const std::shared_ptr<io::OutputStream>& stream, std::unique_ptr<TableWriter>* out) {
+  out->reset(new TableWriter());
+  return (*out)->impl_->Open(stream);
+}
+
+Status TableWriter::OpenFile(
+    const std::string& abspath, std::unique_ptr<TableWriter>* out) {
+  std::shared_ptr<io::FileOutputStream> file;
+  RETURN_NOT_OK(io::FileOutputStream::Open(abspath, &file));
+  out->reset(new TableWriter());
+  return (*out)->impl_->Open(file);
+}
+
+void TableWriter::SetDescription(const std::string& desc) {
+  impl_->SetDescription(desc);
+}
+
+void TableWriter::SetNumRows(int64_t num_rows) {
+  impl_->SetNumRows(num_rows);
+}
+
+Status TableWriter::Append(const std::string& name, const Array& values) {
+  return impl_->Append(name, values);
+}
+
+Status TableWriter::Finalize() {
+  return impl_->Finalize();
+}
+
+}  // namespace feather
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/feather.fbs b/cpp/src/arrow/ipc/feather.fbs
new file mode 100644
index 0000000000000..a27d39989c620
--- /dev/null
+++ b/cpp/src/arrow/ipc/feather.fbs
@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace arrow.ipc.feather.fbs;
+
+/// Feather is an experimental serialization format implemented using
+/// techniques from Apache Arrow. It was created as a proof-of-concept of an
+/// interoperable file format for storing data frames originating in Python or
+/// R. It enabled the developers to sidestep some of the open design questions
+/// in Arrow from early 2016 and instead create something simple and useful for
+/// the intended use cases.
+
+enum Type : byte {
+  BOOL = 0,
+
+  INT8 = 1,
+  INT16 = 2,
+  INT32 = 3,
+  INT64 = 4,
+
+  UINT8 = 5,
+  UINT16 = 6,
+  UINT32 = 7,
+  UINT64 = 8,
+
+  FLOAT = 9,
+  DOUBLE = 10,
+
+  UTF8 = 11,
+
+  BINARY = 12,
+
+  CATEGORY = 13,
+
+  TIMESTAMP = 14,
+  DATE = 15,
+  TIME = 16
+}
+
+enum Encoding : byte {
+  PLAIN = 0,
+
+  /// Data is stored dictionary-encoded
+  /// dictionary size: <INT32 Dictionary size>
+  /// dictionary data: <TYPE primitive array>
+  /// dictionary index: <INT32 primitive array>
+  ///
+  /// TODO: do we care about storing the index values in a smaller typeclass
+  DICTIONARY = 1
+}
+
+enum TimeUnit : byte {
+  SECOND = 0,
+  MILLISECOND = 1,
+  MICROSECOND = 2,
+  NANOSECOND = 3
+}
+
+table PrimitiveArray {
+  type: Type;
+
+  encoding: Encoding = PLAIN;
+
+  /// Relative memory offset of the start of the array data excluding the size
+  /// of the metadata
+  offset: long;
+
+  /// The number of logical values in the array
+  length: long;
+
+  /// The number of observed nulls
+  null_count: long;
+
+  /// The total size of the actual data in the file
+  total_bytes: long;
+
+  /// TODO: Compression
+}
+
+table CategoryMetadata {
+  /// The category codes are presumed to be integers that are valid indexes into
+  /// the levels array
+
+  levels: PrimitiveArray;
+  ordered: bool = false;
+}
+
+table TimestampMetadata {
+  unit: TimeUnit;
+
+  /// Timestamp data is assumed to be UTC, but the time zone is stored here for
+  /// presentation as localized
+  timezone: string;
+}
+
+table DateMetadata {
+}
+
+table TimeMetadata {
+  unit: TimeUnit;
+}
+
+union TypeMetadata {
+  CategoryMetadata,
+  TimestampMetadata,
+  DateMetadata,
+  TimeMetadata,
+}
+
+table Column {
+  name: string;
+  values: PrimitiveArray;
+  metadata: TypeMetadata;
+
+  /// This should (probably) be JSON
+  user_metadata: string;
+}
+
+table CTable {
+  /// Some text (or a name) metadata about what the file is, optional
+  description: string;
+
+  num_rows: long;
+  columns: [Column];
+
+  /// Version number of the Feather format
+  version: int;
+
+  /// Table metadata (likely JSON), not yet used
+  metadata: string;
+}
+
+root_type CTable;
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
new file mode 100644
index 0000000000000..3d370dfe02bd0
--- /dev/null
+++ b/cpp/src/arrow/ipc/feather.h
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Public API for the "Feather" file format, originally created at
+/// http://github.com/wesm/feather
+
+#ifndef ARROW_IPC_FEATHER_H
+#define ARROW_IPC_FEATHER_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+class Buffer;
+class Column;
+class Status;
+
+namespace io {
+
+class OutputStream;
+class ReadableFileInterface;
+
+}  // namespace io
+
+namespace ipc {
+namespace feather {
+
+static constexpr const int kFeatherVersion = 2;
+
+// ----------------------------------------------------------------------
+// Metadata accessor classes
+
+class ARROW_EXPORT TableReader {
+ public:
+  TableReader();
+  ~TableReader();
+
+  Status Open(const std::shared_ptr<io::ReadableFileInterface>& source);
+
+  static Status OpenFile(const std::string& abspath, std::unique_ptr<TableReader>* out);
+
+  // Optional table description
+  //
+  // This does not return a const std::string& because a string has to be
+  // copied from the flatbuffer to be able to return a non-flatbuffer type
+  std::string GetDescription() const;
+  bool HasDescription() const;
+
+  int version() const;
+
+  int64_t num_rows() const;
+  int64_t num_columns() const;
+
+  std::string GetColumnName(int i) const;
+
+  Status GetColumn(int i, std::shared_ptr<Column>* out);
+
+ private:
+  class ARROW_NO_EXPORT TableReaderImpl;
+  std::unique_ptr<TableReaderImpl> impl_;
+};
+
+class ARROW_EXPORT TableWriter {
+ public:
+  ~TableWriter();
+
+  static Status Open(
+      const std::shared_ptr<io::OutputStream>& stream, std::unique_ptr<TableWriter>* out);
+
+  static Status OpenFile(const std::string& abspath, std::unique_ptr<TableWriter>* out);
+
+  void SetDescription(const std::string& desc);
+  void SetNumRows(int64_t num_rows);
+
+  Status Append(const std::string& name, const Array& values);
+
+  // We are done, write the file metadata and footer
+  Status Finalize();
+
+ private:
+  TableWriter();
+  class ARROW_NO_EXPORT TableWriterImpl;
+  std::unique_ptr<TableWriterImpl> impl_;
+};
+
+}  // namespace feather
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_FEATHER_H
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6e6d609b00007..ef874e3d07959 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -437,6 +437,7 @@ set(CYTHON_EXTENSIONS
   config
   error
   io
+  _feather
   memory
   scalar
   schema
diff --git a/python/pyarrow/_feather.pyx b/python/pyarrow/_feather.pyx
new file mode 100644
index 0000000000000..67f734f6ed77c
--- /dev/null
+++ b/python/pyarrow/_feather.pyx
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from cython.operator cimport dereference as deref
+
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport CArray, CColumn, CSchema, CStatus
+from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream
+
+from libcpp.string cimport string
+from libcpp cimport bool as c_bool
+
+cimport cpython
+
+from pyarrow.compat import frombytes, tobytes, encode_file_path
+
+from pyarrow.array cimport Array
+from pyarrow.error cimport check_status
+from pyarrow.table cimport Column
+
+cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
+
+    cdef cppclass TableWriter:
+        @staticmethod
+        CStatus Open(const shared_ptr[OutputStream]& stream,
+                     unique_ptr[TableWriter]* out)
+
+        @staticmethod
+        CStatus OpenFile(const string& abspath, unique_ptr[TableWriter]* out)
+
+        void SetDescription(const string& desc)
+        void SetNumRows(int64_t num_rows)
+
+        CStatus Append(const string& name, const CArray& values)
+        CStatus Finalize()
+
+    cdef cppclass TableReader:
+        TableReader(const shared_ptr[ReadableFileInterface]& source)
+
+        @staticmethod
+        CStatus OpenFile(const string& abspath, unique_ptr[TableReader]* out)
+
+        string GetDescription()
+        c_bool HasDescription()
+
+        int64_t num_rows()
+        int64_t num_columns()
+
+        shared_ptr[CSchema] schema()
+
+        CStatus GetColumn(int i, shared_ptr[CColumn]* out)
+        c_string GetColumnName(int i)
+
+
+class FeatherError(Exception):
+    pass
+
+
+cdef class FeatherWriter:
+    cdef:
+        unique_ptr[TableWriter] writer
+
+    cdef public:
+        int64_t num_rows
+
+    def __cinit__(self):
+        self.num_rows = -1
+
+    def open(self, object dest):
+        cdef:
+            string c_name = encode_file_path(dest)
+
+        check_status(TableWriter.OpenFile(c_name, &self.writer))
+
+    def close(self):
+        if self.num_rows < 0:
+            self.num_rows = 0
+        self.writer.get().SetNumRows(self.num_rows)
+        check_status(self.writer.get().Finalize())
+
+    def write_array(self, object name, object col, object mask=None):
+        cdef Array arr
+
+        if self.num_rows >= 0:
+            if len(col) != self.num_rows:
+                raise ValueError('prior column had a different number of rows')
+        else:
+            self.num_rows = len(col)
+
+        if isinstance(col, Array):
+            arr = col
+        else:
+            arr = Array.from_pandas(col, mask=mask)
+
+        cdef c_string c_name = tobytes(name)
+
+        with nogil:
+            check_status(
+                self.writer.get().Append(c_name, deref(arr.sp_array)))
+
+
+cdef class FeatherReader:
+    cdef:
+        unique_ptr[TableReader] reader
+
+    def __cinit__(self):
+        pass
+
+    def open(self, source):
+        cdef:
+            string c_name = encode_file_path(source)
+
+        check_status(TableReader.OpenFile(c_name, &self.reader))
+
+    property num_rows:
+
+        def __get__(self):
+            return self.reader.get().num_rows()
+
+    property num_columns:
+
+        def __get__(self):
+            return self.reader.get().num_columns()
+
+    def get_column_name(self, int i):
+        cdef c_string name = self.reader.get().GetColumnName(i)
+        return frombytes(name)
+
+    def get_column(self, int i):
+        if i < 0 or i >= self.num_columns:
+            raise IndexError(i)
+
+        cdef shared_ptr[CColumn] sp_column
+        with nogil:
+            check_status(self.reader.get()
+                         .GetColumn(i, &sp_column))
+
+        cdef Column col = Column()
+        col.init(sp_column)
+        return col
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
new file mode 100644
index 0000000000000..b7dbf96563a41
--- /dev/null
+++ b/python/pyarrow/feather.py
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import six
+from distutils.version import LooseVersion
+import pandas as pd
+
+from pyarrow._feather import FeatherError  # noqa
+from pyarrow.table import Table
+import pyarrow._feather as ext
+
+
+if LooseVersion(pd.__version__) < '0.17.0':
+    raise ImportError("feather requires pandas >= 0.17.0")
+
+if LooseVersion(pd.__version__) < '0.19.0':
+    pdapi = pd.core.common
+else:
+    pdapi = pd.api.types
+
+
+class FeatherReader(ext.FeatherReader):
+
+    def __init__(self, source):
+        self.source = source
+        self.open(source)
+
+    def read(self, columns=None):
+        if columns is not None:
+            column_set = set(columns)
+        else:
+            column_set = None
+
+        columns = []
+        names = []
+        for i in range(self.num_columns):
+            name = self.get_column_name(i)
+            if column_set is None or name in column_set:
+                col = self.get_column(i)
+                columns.append(col)
+                names.append(name)
+
+        table = Table.from_arrays(columns, names=names)
+        return table.to_pandas()
+
+
+def write_feather(df, path):
+    '''
+    Write a pandas.DataFrame to Feather format
+    '''
+    writer = ext.FeatherWriter()
+    writer.open(path)
+
+    if isinstance(df, pd.SparseDataFrame):
+        df = df.to_dense()
+
+    if not df.columns.is_unique:
+        raise ValueError("cannot serialize duplicate column names")
+
+    # TODO(wesm): pipeline conversion to Arrow memory layout
+    for i, name in enumerate(df.columns):
+        col = df.iloc[:, i]
+
+        if pdapi.is_object_dtype(col):
+            inferred_type = pd.lib.infer_dtype(col)
+            msg = ("cannot serialize column {n} "
+                   "named {name} with dtype {dtype}".format(
+                       n=i, name=name, dtype=inferred_type))
+
+            if inferred_type in ['mixed']:
+
+                # allow columns with nulls + an inferable type
+                inferred_type = pd.lib.infer_dtype(col[col.notnull()])
+                if inferred_type in ['mixed']:
+                    raise ValueError(msg)
+
+            elif inferred_type not in ['unicode', 'string']:
+                raise ValueError(msg)
+
+        if not isinstance(name, six.string_types):
+            name = str(name)
+
+        writer.write_array(name, col)
+
+    writer.close()
+
+
+def read_feather(path, columns=None):
+    """
+    Read a pandas.DataFrame from Feather format
+
+    Parameters
+    ----------
+    path : string, path to read from
+    columns : sequence, optional
+        Only read a specific set of columns. If not provided, all columns are
+        read
+
+    Returns
+    -------
+    df : pandas.DataFrame
+    """
+    reader = FeatherReader(path)
+    return reader.read(columns=columns)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index ad5af1b0128ca..5657b973d1306 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -540,6 +540,11 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
             block = _int.make_block(cat, placement=placement,
                                     klass=_int.CategoricalBlock,
                                     fastpath=True)
+        elif 'timezone' in item:
+            from pandas.types.api import DatetimeTZDtype
+            dtype = DatetimeTZDtype('ns', tz=item['timezone'])
+            block = _int.make_block(block_arr, placement=placement,
+                                    dtype=dtype, fastpath=True)
         else:
             block = _int.make_block(block_arr, placement=placement)
         blocks.append(block)
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
new file mode 100644
index 0000000000000..451475b4c6d81
--- /dev/null
+++ b/python/pyarrow/tests/test_feather.py
@@ -0,0 +1,379 @@
+# Copyright 2016 Feather Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+import pytest
+
+from numpy.testing import assert_array_equal
+import numpy as np
+
+from pandas.util.testing import assert_frame_equal
+import pandas as pd
+
+from pyarrow.compat import guid
+from pyarrow.error import ArrowException
+from pyarrow.feather import (read_feather, write_feather,
+                             FeatherReader)
+from pyarrow._feather import FeatherWriter
+
+
+def random_path():
+    return 'feather_{}'.format(guid())
+
+
+class TestFeatherReader(unittest.TestCase):
+
+    def setUp(self):
+        self.test_files = []
+
+    def tearDown(self):
+        for path in self.test_files:
+            try:
+                os.remove(path)
+            except os.error:
+                pass
+
+    def test_file_not_exist(self):
+        with self.assertRaises(ArrowException):
+            FeatherReader('test_invalid_file')
+
+    def _get_null_counts(self, path, columns=None):
+        reader = FeatherReader(path)
+        counts = []
+        for i in range(reader.num_columns):
+            col = reader.get_column(i)
+            if columns is None or col.name in columns:
+                counts.append(col.null_count)
+
+        return counts
+
+    def _check_pandas_roundtrip(self, df, expected=None, path=None,
+                                columns=None, null_counts=None):
+        if path is None:
+            path = random_path()
+
+        self.test_files.append(path)
+        write_feather(df, path)
+        if not os.path.exists(path):
+            raise Exception('file not written')
+
+        result = read_feather(path, columns)
+        if expected is None:
+            expected = df
+
+        assert_frame_equal(result, expected)
+
+        if null_counts is None:
+            null_counts = np.zeros(len(expected.columns))
+
+        np.testing.assert_array_equal(self._get_null_counts(path, columns),
+                                      null_counts)
+
+    def _assert_error_on_write(self, df, exc, path=None):
+        # check that we are raising the exception
+        # on writing
+
+        if path is None:
+            path = random_path()
+
+        self.test_files.append(path)
+
+        def f():
+            write_feather(df, path)
+
+        self.assertRaises(exc, f)
+
+    def test_num_rows_attr(self):
+        df = pd.DataFrame({'foo': [1, 2, 3, 4, 5]})
+        path = random_path()
+        self.test_files.append(path)
+        write_feather(df, path)
+
+        reader = FeatherReader(path)
+        assert reader.num_rows == len(df)
+
+        df = pd.DataFrame({})
+        path = random_path()
+        self.test_files.append(path)
+        write_feather(df, path)
+
+        reader = FeatherReader(path)
+        assert reader.num_rows == 0
+
+    def test_float_no_nulls(self):
+        data = {}
+        numpy_dtypes = ['f4', 'f8']
+        num_values = 100
+
+        for dtype in numpy_dtypes:
+            values = np.random.randn(num_values)
+            data[dtype] = values.astype(dtype)
+
+        df = pd.DataFrame(data)
+        self._check_pandas_roundtrip(df)
+
+    def test_float_nulls(self):
+        num_values = 100
+
+        path = random_path()
+        self.test_files.append(path)
+        writer = FeatherWriter()
+        writer.open(path)
+
+        null_mask = np.random.randint(0, 10, size=num_values) < 3
+        dtypes = ['f4', 'f8']
+        expected_cols = []
+        null_counts = []
+        for name in dtypes:
+            values = np.random.randn(num_values).astype(name)
+            writer.write_array(name, values, null_mask)
+
+            values[null_mask] = np.nan
+
+            expected_cols.append(values)
+            null_counts.append(null_mask.sum())
+
+        writer.close()
+
+        ex_frame = pd.DataFrame(dict(zip(dtypes, expected_cols)),
+                                columns=dtypes)
+
+        result = read_feather(path)
+        assert_frame_equal(result, ex_frame)
+        assert_array_equal(self._get_null_counts(path), null_counts)
+
+    def test_integer_no_nulls(self):
+        data = {}
+
+        numpy_dtypes = ['i1', 'i2', 'i4', 'i8',
+                        'u1', 'u2', 'u4', 'u8']
+        num_values = 100
+
+        for dtype in numpy_dtypes:
+            values = np.random.randint(0, 100, size=num_values)
+            data[dtype] = values.astype(dtype)
+
+        df = pd.DataFrame(data)
+        self._check_pandas_roundtrip(df)
+
+    def test_platform_numpy_integers(self):
+        data = {}
+
+        numpy_dtypes = ['longlong']
+        num_values = 100
+
+        for dtype in numpy_dtypes:
+            values = np.random.randint(0, 100, size=num_values)
+            data[dtype] = values.astype(dtype)
+
+        df = pd.DataFrame(data)
+        self._check_pandas_roundtrip(df)
+
+    def test_integer_with_nulls(self):
+        # pandas requires upcast to float dtype
+        path = random_path()
+        self.test_files.append(path)
+
+        int_dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8']
+        num_values = 100
+
+        writer = FeatherWriter()
+        writer.open(path)
+
+        null_mask = np.random.randint(0, 10, size=num_values) < 3
+        expected_cols = []
+        for name in int_dtypes:
+            values = np.random.randint(0, 100, size=num_values)
+            writer.write_array(name, values, null_mask)
+
+            expected = values.astype('f8')
+            expected[null_mask] = np.nan
+
+            expected_cols.append(expected)
+
+        ex_frame = pd.DataFrame(dict(zip(int_dtypes, expected_cols)),
+                                columns=int_dtypes)
+
+        writer.close()
+
+        result = read_feather(path)
+        assert_frame_equal(result, ex_frame)
+
+    def test_boolean_no_nulls(self):
+        num_values = 100
+
+        np.random.seed(0)
+
+        df = pd.DataFrame({'bools': np.random.randn(num_values) > 0})
+        self._check_pandas_roundtrip(df)
+
+    def test_boolean_nulls(self):
+        # pandas requires upcast to object dtype
+        path = random_path()
+        self.test_files.append(path)
+
+        num_values = 100
+        np.random.seed(0)
+
+        writer = FeatherWriter()
+        writer.open(path)
+
+        mask = np.random.randint(0, 10, size=num_values) < 3
+        values = np.random.randint(0, 10, size=num_values) < 5
+        writer.write_array('bools', values, mask)
+
+        expected = values.astype(object)
+        expected[mask] = None
+
+        writer.close()
+
+        ex_frame = pd.DataFrame({'bools': expected})
+
+        result = read_feather(path)
+        assert_frame_equal(result, ex_frame)
+
+    def test_boolean_object_nulls(self):
+        repeats = 100
+        arr = np.array([False, None, True] * repeats, dtype=object)
+        df = pd.DataFrame({'bools': arr})
+        self._check_pandas_roundtrip(df, null_counts=[1 * repeats])
+
+    def test_strings(self):
+        repeats = 1000
+
+        # we hvae mixed bytes, unicode, strings
+        values = [b'foo', None, u'bar', 'qux', np.nan]
+        df = pd.DataFrame({'strings': values * repeats})
+        self._assert_error_on_write(df, ValueError)
+
+        # embedded nulls are ok
+        values = ['foo', None, 'bar', 'qux', None]
+        df = pd.DataFrame({'strings': values * repeats})
+        expected = pd.DataFrame({'strings': values * repeats})
+        self._check_pandas_roundtrip(df, expected, null_counts=[2 * repeats])
+
+        values = ['foo', None, 'bar', 'qux', np.nan]
+        df = pd.DataFrame({'strings': values * repeats})
+        expected = pd.DataFrame({'strings': values * repeats})
+        self._check_pandas_roundtrip(df, expected, null_counts=[2 * repeats])
+
+    def test_empty_strings(self):
+        df = pd.DataFrame({'strings': [''] * 10})
+        self._check_pandas_roundtrip(df)
+
+    def test_nan_as_null(self):
+        # Create a nan that is not numpy.nan
+        values = np.array(['foo', np.nan, np.nan * 2, 'bar'] * 10)
+        df = pd.DataFrame({'strings': values})
+        self._check_pandas_roundtrip(df)
+
+    def test_category(self):
+        repeats = 1000
+        values = ['foo', None, u'bar', 'qux', np.nan]
+        df = pd.DataFrame({'strings': values * repeats})
+        df['strings'] = df['strings'].astype('category')
+
+        values = ['foo', None, 'bar', 'qux', None]
+        expected = pd.DataFrame({'strings': pd.Categorical(values * repeats)})
+        self._check_pandas_roundtrip(df, expected,
+                                     null_counts=[2 * repeats])
+
+    @pytest.mark.xfail
+    def test_timestamp(self):
+        df = pd.DataFrame({'naive': pd.date_range('2016-03-28', periods=10)})
+        df['with_tz'] = (df.naive.dt.tz_localize('utc')
+                         .dt.tz_convert('America/Los_Angeles'))
+
+        self._check_pandas_roundtrip(df)
+
+    @pytest.mark.xfail
+    def test_timestamp_with_nulls(self):
+        df = pd.DataFrame({'test': [pd.datetime(2016, 1, 1),
+                                    None,
+                                    pd.datetime(2016, 1, 3)]})
+        df['with_tz'] = df.test.dt.tz_localize('utc')
+
+        self._check_pandas_roundtrip(df, null_counts=[1, 1])
+
+    @pytest.mark.xfail
+    def test_out_of_float64_timestamp_with_nulls(self):
+        df = pd.DataFrame(
+            {'test': pd.DatetimeIndex([1451606400000000001,
+                                       None, 14516064000030405])})
+        df['with_tz'] = df.test.dt.tz_localize('utc')
+        self._check_pandas_roundtrip(df, null_counts=[1, 1])
+
+    def test_non_string_columns(self):
+        df = pd.DataFrame({0: [1, 2, 3, 4],
+                           1: [True, False, True, False]})
+
+        expected = df.rename(columns=str)
+        self._check_pandas_roundtrip(df, expected)
+
+    def test_unicode_filename(self):
+        # GH #209
+        name = (b'Besa_Kavaj\xc3\xab.feather').decode('utf-8')
+        df = pd.DataFrame({'foo': [1, 2, 3, 4]})
+        self._check_pandas_roundtrip(df, path=name)
+
+    def test_read_columns(self):
+        data = {'foo': [1, 2, 3, 4],
+                'boo': [5, 6, 7, 8],
+                'woo': [1, 3, 5, 7]}
+        columns = list(data.keys())[1:3]
+        df = pd.DataFrame(data)
+        expected = pd.DataFrame({c: data[c] for c in columns})
+        self._check_pandas_roundtrip(df, expected, columns=columns)
+
+    def test_overwritten_file(self):
+        path = random_path()
+
+        num_values = 100
+        np.random.seed(0)
+
+        values = np.random.randint(0, 10, size=num_values)
+        write_feather(pd.DataFrame({'ints': values}), path)
+
+        df = pd.DataFrame({'ints': values[0: num_values//2]})
+        self._check_pandas_roundtrip(df, path=path)
+
+    def test_sparse_dataframe(self):
+        # GH #221
+        data = {'A': [0, 1, 2],
+                'B': [1, 0, 1]}
+        df = pd.DataFrame(data).to_sparse(fill_value=1)
+        expected = df.to_dense()
+        self._check_pandas_roundtrip(df, expected)
+
+    def test_duplicate_columns(self):
+
+        # https://github.com/wesm/feather/issues/53
+        # not currently able to handle duplicate columns
+        df = pd.DataFrame(np.arange(12).reshape(4, 3),
+                          columns=list('aaa')).copy()
+        self._assert_error_on_write(df, ValueError)
+
+    def test_unsupported(self):
+        # https://github.com/wesm/feather/issues/240
+        # serializing actual python objects
+
+        # period
+        df = pd.DataFrame({'a': pd.period_range('2013', freq='M', periods=3)})
+        self._assert_error_on_write(df, ValueError)
+
+        # non-strings
+        df = pd.DataFrame({'a': ['a', 1, 2.0]})
+        self._assert_error_on_write(df, ValueError)
diff --git a/python/setup.py b/python/setup.py
index b0f29be4c1b3b..a0573fe1fccff 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -101,6 +101,7 @@ def initialize_options(self):
         'io',
         'jemalloc',
         'memory',
+        '_feather',
         '_parquet',
         'scalar',
         'schema',
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index c707ada9dd55c..eb3ab49f58892 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -925,6 +925,32 @@ class DatetimeBlock : public PandasBlock {
   }
 };
 
+class DatetimeTZBlock : public DatetimeBlock {
+ public:
+  DatetimeTZBlock(const std::string& timezone, int64_t num_rows)
+      : DatetimeBlock(num_rows, 1), timezone_(timezone) {}
+
+  Status GetPyResult(PyObject** output) override {
+    PyObject* result = PyDict_New();
+    RETURN_IF_PYERROR();
+
+    PyObject* py_tz = PyUnicode_FromStringAndSize(
+        timezone_.c_str(), static_cast<Py_ssize_t>(timezone_.size()));
+    RETURN_IF_PYERROR();
+
+    PyDict_SetItemString(result, "block", block_arr_.obj());
+    PyDict_SetItemString(result, "timezone", py_tz);
+    PyDict_SetItemString(result, "placement", placement_arr_.obj());
+
+    *output = result;
+
+    return Status::OK();
+  }
+
+ private:
+  std::string timezone_;
+};
+
 template <int ARROW_INDEX_TYPE>
 class CategoricalBlock : public PandasBlock {
  public:
@@ -1068,6 +1094,8 @@ static inline Status MakeCategoricalBlock(const std::shared_ptr<DataType>& type,
   return (*block)->Allocate();
 }
 
+using BlockMap = std::unordered_map<int, std::shared_ptr<PandasBlock>>;
+
 // Construct the exact pandas 0.x "BlockManager" memory layout
 //
 // * For each column determine the correct output pandas type
@@ -1138,9 +1166,14 @@ class DataFrameBlockCreator {
         case Type::DATE:
           output_type = PandasBlock::DATETIME;
           break;
-        case Type::TIMESTAMP:
-          output_type = PandasBlock::DATETIME;
-          break;
+        case Type::TIMESTAMP: {
+          const auto& ts_type = static_cast<const arrow::TimestampType&>(*col->type());
+          if (ts_type.timezone != "") {
+            output_type = PandasBlock::DATETIME_WITH_TZ;
+          } else {
+            output_type = PandasBlock::DATETIME;
+          }
+        } break;
         case Type::LIST: {
           auto list_type = std::static_pointer_cast<ListType>(col->type());
           if (!ListTypeSupported(list_type->value_type()->type)) {
@@ -1159,10 +1192,15 @@ class DataFrameBlockCreator {
       }
 
       int block_placement = 0;
+      std::shared_ptr<PandasBlock> block;
       if (output_type == PandasBlock::CATEGORICAL) {
-        std::shared_ptr<PandasBlock> block;
         RETURN_NOT_OK(MakeCategoricalBlock(col->type(), table_->num_rows(), &block));
         categorical_blocks_[i] = block;
+      } else if (output_type == PandasBlock::DATETIME_WITH_TZ) {
+        const auto& ts_type = static_cast<const arrow::TimestampType&>(*col->type());
+        block = std::make_shared<DatetimeTZBlock>(ts_type.timezone, table_->num_rows());
+        RETURN_NOT_OK(block->Allocate());
+        datetimetz_blocks_[i] = block;
       } else {
         auto it = type_counts_.find(output_type);
         if (it != type_counts_.end()) {
@@ -1252,28 +1290,24 @@ class DataFrameBlockCreator {
     return Status::OK();
   }
 
+  Status AppendBlocks(const BlockMap& blocks, PyObject* list) {
+    for (const auto& it : blocks) {
+      PyObject* item;
+      RETURN_NOT_OK(it.second->GetPyResult(&item));
+      if (PyList_Append(list, item) < 0) { RETURN_IF_PYERROR(); }
+    }
+    return Status::OK();
+  }
+
   Status GetResultList(PyObject** out) {
     PyAcquireGIL lock;
 
-    auto num_blocks =
-        static_cast<Py_ssize_t>(blocks_.size() + categorical_blocks_.size());
-    PyObject* result = PyList_New(num_blocks);
+    PyObject* result = PyList_New(0);
     RETURN_IF_PYERROR();
 
-    int i = 0;
-    for (const auto& it : blocks_) {
-      const std::shared_ptr<PandasBlock> block = it.second;
-      PyObject* item;
-      RETURN_NOT_OK(block->GetPyResult(&item));
-      if (PyList_SET_ITEM(result, i++, item) < 0) { RETURN_IF_PYERROR(); }
-    }
-
-    for (const auto& it : categorical_blocks_) {
-      const std::shared_ptr<PandasBlock> block = it.second;
-      PyObject* item;
-      RETURN_NOT_OK(block->GetPyResult(&item));
-      if (PyList_SET_ITEM(result, i++, item) < 0) { RETURN_IF_PYERROR(); }
-    }
+    RETURN_NOT_OK(AppendBlocks(blocks_, result));
+    RETURN_NOT_OK(AppendBlocks(categorical_blocks_, result));
+    RETURN_NOT_OK(AppendBlocks(datetimetz_blocks_, result));
 
     *out = result;
     return Status::OK();
@@ -1292,10 +1326,13 @@ class DataFrameBlockCreator {
   std::unordered_map<int, int> type_counts_;
 
   // block type -> block
-  std::unordered_map<int, std::shared_ptr<PandasBlock>> blocks_;
+  BlockMap blocks_;
 
   // column number -> categorical block
-  std::unordered_map<int, std::shared_ptr<PandasBlock>> categorical_blocks_;
+  BlockMap categorical_blocks_;
+
+  // column number -> datetimetz block
+  BlockMap datetimetz_blocks_;
 };
 
 Status ConvertTableToPandas(

From fdc25b418273a9a0d9d2512f571236e96cb4e2b4 Mon Sep 17 00:00:00 2001
From: Julien Lafaye <jlafaye@gmail.com>
Date: Sun, 12 Mar 2017 13:28:09 +0100
Subject: [PATCH 0355/1644] ARROW-606: [C++] upgrade flatbuffers version to
 1.6.0

all unittests pass

benchmark (builder, column, jemalloc-builder) results suffer minor
differences (<5%) wrt to flatbuffer 1.3.0

Author: Julien Lafaye <jlafaye@gmail.com>

Closes #373 from jlafaye/master and squashes the following commits:

3d001e5 [Julien Lafaye] ARROW-606: [C++] upgrade flatbuffers version to 1.6.0
---
 cpp/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 294c439e2b093..5ecc34e8a5fc6 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -35,7 +35,7 @@ set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 set(GFLAGS_VERSION "2.1.2")
 set(GTEST_VERSION "1.7.0")
 set(GBENCHMARK_VERSION "1.1.0")
-set(FLATBUFFERS_VERSION "1.3.0")
+set(FLATBUFFERS_VERSION "1.6.0")
 set(JEMALLOC_VERSION "4.4.0")
 
 find_package(ClangTools)

From e5a11dac2ab856001cae4c1cb582cd376fa7f083 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 12 Mar 2017 11:53:04 -0400
Subject: [PATCH 0356/1644] ARROW-534: [C++] Add IPC tests for date/time after
 ARROW-452, fix bugs

Closes #345. I had mostly done this in #361 so this adds tests to `ipc-adapter-test`

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #371 from wesm/ARROW-534 and squashes the following commits:

cab6d4f [Wes McKinney] Add functions to make record batches for date, date32, timestamp, time. Fix bugs
---
 cpp/src/arrow/ipc/adapter.cc          | 75 ++++++---------------
 cpp/src/arrow/ipc/feather-test.cc     | 38 -----------
 cpp/src/arrow/ipc/ipc-adapter-test.cc |  3 +-
 cpp/src/arrow/ipc/test-common.h       | 97 +++++++++++++++++++++++++++
 cpp/src/arrow/test-util.h             | 18 +++++
 cpp/src/arrow/type.h                  |  2 +-
 cpp/src/arrow/type_traits.h           |  6 +-
 7 files changed, 141 insertions(+), 98 deletions(-)

diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 78d58101963dc..a4eff7214aa5f 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -309,66 +309,31 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status Visit(const Int8Array& array) override {
-    return VisitFixedWidth<Int8Array>(array);
-  }
-
-  Status Visit(const Int16Array& array) override {
-    return VisitFixedWidth<Int16Array>(array);
-  }
-
-  Status Visit(const Int32Array& array) override {
-    return VisitFixedWidth<Int32Array>(array);
-  }
-
-  Status Visit(const Int64Array& array) override {
-    return VisitFixedWidth<Int64Array>(array);
-  }
-
-  Status Visit(const UInt8Array& array) override {
-    return VisitFixedWidth<UInt8Array>(array);
-  }
-
-  Status Visit(const UInt16Array& array) override {
-    return VisitFixedWidth<UInt16Array>(array);
-  }
-
-  Status Visit(const UInt32Array& array) override {
-    return VisitFixedWidth<UInt32Array>(array);
-  }
-
-  Status Visit(const UInt64Array& array) override {
-    return VisitFixedWidth<UInt64Array>(array);
-  }
-
-  Status Visit(const HalfFloatArray& array) override {
-    return VisitFixedWidth<HalfFloatArray>(array);
-  }
-
-  Status Visit(const FloatArray& array) override {
-    return VisitFixedWidth<FloatArray>(array);
-  }
-
-  Status Visit(const DoubleArray& array) override {
-    return VisitFixedWidth<DoubleArray>(array);
-  }
+#define VISIT_FIXED_WIDTH(TYPE) \
+  Status Visit(const TYPE& array) override { return VisitFixedWidth<TYPE>(array); }
+
+  VISIT_FIXED_WIDTH(Int8Array);
+  VISIT_FIXED_WIDTH(Int16Array);
+  VISIT_FIXED_WIDTH(Int32Array);
+  VISIT_FIXED_WIDTH(Int64Array);
+  VISIT_FIXED_WIDTH(UInt8Array);
+  VISIT_FIXED_WIDTH(UInt16Array);
+  VISIT_FIXED_WIDTH(UInt32Array);
+  VISIT_FIXED_WIDTH(UInt64Array);
+  VISIT_FIXED_WIDTH(HalfFloatArray);
+  VISIT_FIXED_WIDTH(FloatArray);
+  VISIT_FIXED_WIDTH(DoubleArray);
+  VISIT_FIXED_WIDTH(DateArray);
+  VISIT_FIXED_WIDTH(Date32Array);
+  VISIT_FIXED_WIDTH(TimeArray);
+  VISIT_FIXED_WIDTH(TimestampArray);
+
+#undef VISIT_FIXED_WIDTH
 
   Status Visit(const StringArray& array) override { return VisitBinary(array); }
 
   Status Visit(const BinaryArray& array) override { return VisitBinary(array); }
 
-  Status Visit(const DateArray& array) override {
-    return VisitFixedWidth<DateArray>(array);
-  }
-
-  Status Visit(const TimeArray& array) override {
-    return VisitFixedWidth<TimeArray>(array);
-  }
-
-  Status Visit(const TimestampArray& array) override {
-    return VisitFixedWidth<TimestampArray>(array);
-  }
-
   Status Visit(const ListArray& array) override {
     std::shared_ptr<Buffer> value_offsets;
     RETURN_NOT_OK(GetZeroBasedValueOffsets<ListArray>(array, &value_offsets));
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index b73246b672260..078c3e10aff29 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -344,44 +344,6 @@ TEST_F(TestTableWriter, PrimitiveRoundTrip) {
   ASSERT_EQ("f1", col->name());
 }
 
-Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out) {
-  const int64_t length = 6;
-
-  std::vector<bool> is_valid = {true, true, false, true, true, true};
-  std::shared_ptr<Array> dict1, dict2;
-
-  std::vector<std::string> dict1_values = {"foo", "bar", "baz"};
-  std::vector<std::string> dict2_values = {"foo", "bar", "baz", "qux"};
-
-  ArrayFromVector<StringType, std::string>(dict1_values, &dict1);
-  ArrayFromVector<StringType, std::string>(dict2_values, &dict2);
-
-  auto f0_type = arrow::dictionary(arrow::int32(), dict1);
-  auto f1_type = arrow::dictionary(arrow::int8(), dict1);
-  auto f2_type = arrow::dictionary(arrow::int32(), dict2);
-
-  std::shared_ptr<Array> indices0, indices1, indices2;
-  std::vector<int32_t> indices0_values = {1, 2, -1, 0, 2, 0};
-  std::vector<int8_t> indices1_values = {0, 0, 2, 2, 1, 1};
-  std::vector<int32_t> indices2_values = {3, 0, 2, 1, 0, 2};
-
-  ArrayFromVector<Int32Type, int32_t>(is_valid, indices0_values, &indices0);
-  ArrayFromVector<Int8Type, int8_t>(is_valid, indices1_values, &indices1);
-  ArrayFromVector<Int32Type, int32_t>(is_valid, indices2_values, &indices2);
-
-  auto a0 = std::make_shared<DictionaryArray>(f0_type, indices0);
-  auto a1 = std::make_shared<DictionaryArray>(f1_type, indices1);
-  auto a2 = std::make_shared<DictionaryArray>(f2_type, indices2);
-
-  // construct batch
-  std::shared_ptr<Schema> schema(new Schema(
-      {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type)}));
-
-  std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2};
-  out->reset(new RecordBatch(schema, length, arrays));
-  return Status::OK();
-}
-
 TEST_F(TestTableWriter, CategoryRoundtrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK(MakeDictionaryFlat(&batch));
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 6678fd522a86a..b60b8a9ba68d2 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -175,7 +175,8 @@ INSTANTIATE_TEST_CASE_P(
     RoundTripTests, TestRecordBatchParam,
     ::testing::Values(&MakeIntRecordBatch, &MakeStringTypesRecordBatch,
         &MakeNonNullRecordBatch, &MakeZeroLengthRecordBatch, &MakeListRecordBatch,
-        &MakeDeeplyNestedList, &MakeStruct, &MakeUnion, &MakeDictionary));
+        &MakeDeeplyNestedList, &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDates,
+        &MakeTimestamps, &MakeTimes));
 
 void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   ipc::MockOutputStream mock;
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index dc823662ee1ef..7f33aba812e0f 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -425,6 +425,103 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
+Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out) {
+  const int64_t length = 6;
+
+  std::vector<bool> is_valid = {true, true, false, true, true, true};
+  std::shared_ptr<Array> dict1, dict2;
+
+  std::vector<std::string> dict1_values = {"foo", "bar", "baz"};
+  std::vector<std::string> dict2_values = {"foo", "bar", "baz", "qux"};
+
+  ArrayFromVector<StringType, std::string>(dict1_values, &dict1);
+  ArrayFromVector<StringType, std::string>(dict2_values, &dict2);
+
+  auto f0_type = arrow::dictionary(arrow::int32(), dict1);
+  auto f1_type = arrow::dictionary(arrow::int8(), dict1);
+  auto f2_type = arrow::dictionary(arrow::int32(), dict2);
+
+  std::shared_ptr<Array> indices0, indices1, indices2;
+  std::vector<int32_t> indices0_values = {1, 2, -1, 0, 2, 0};
+  std::vector<int8_t> indices1_values = {0, 0, 2, 2, 1, 1};
+  std::vector<int32_t> indices2_values = {3, 0, 2, 1, 0, 2};
+
+  ArrayFromVector<Int32Type, int32_t>(is_valid, indices0_values, &indices0);
+  ArrayFromVector<Int8Type, int8_t>(is_valid, indices1_values, &indices1);
+  ArrayFromVector<Int32Type, int32_t>(is_valid, indices2_values, &indices2);
+
+  auto a0 = std::make_shared<DictionaryArray>(f0_type, indices0);
+  auto a1 = std::make_shared<DictionaryArray>(f1_type, indices1);
+  auto a2 = std::make_shared<DictionaryArray>(f2_type, indices2);
+
+  // construct batch
+  std::shared_ptr<Schema> schema(new Schema(
+      {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type)}));
+
+  std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2};
+  out->reset(new RecordBatch(schema, length, arrays));
+  return Status::OK();
+}
+
+Status MakeDates(std::shared_ptr<RecordBatch>* out) {
+  std::vector<bool> is_valid = {true, true, true, false, true, true, true};
+  auto f0 = field("f0", date32());
+  auto f1 = field("f1", date());
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  std::vector<int64_t> date_values = {1489269000000, 1489270000000, 1489271000000,
+      1489272000000, 1489272000000, 1489273000000};
+  std::vector<int32_t> date32_values = {0, 1, 2, 3, 4, 5, 6};
+
+  std::shared_ptr<Array> date_array, date32_array;
+  ArrayFromVector<DateType, int64_t>(is_valid, date_values, &date_array);
+  ArrayFromVector<Date32Type, int32_t>(is_valid, date32_values, &date32_array);
+
+  std::vector<std::shared_ptr<Array>> arrays = {date32_array, date_array};
+  *out = std::make_shared<RecordBatch>(schema, date_array->length(), arrays);
+  return Status::OK();
+}
+
+Status MakeTimestamps(std::shared_ptr<RecordBatch>* out) {
+  std::vector<bool> is_valid = {true, true, true, false, true, true, true};
+  auto f0 = field("f0", timestamp(TimeUnit::MILLI));
+  auto f1 = field("f1", timestamp(TimeUnit::NANO));
+  auto f2 = field("f2", timestamp("US/Los_Angeles", TimeUnit::SECOND));
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+
+  std::vector<int64_t> ts_values = {1489269000000, 1489270000000, 1489271000000,
+      1489272000000, 1489272000000, 1489273000000};
+
+  std::shared_ptr<Array> a0, a1, a2;
+  ArrayFromVector<TimestampType, int64_t>(f0->type, is_valid, ts_values, &a0);
+  ArrayFromVector<TimestampType, int64_t>(f1->type, is_valid, ts_values, &a1);
+  ArrayFromVector<TimestampType, int64_t>(f2->type, is_valid, ts_values, &a2);
+
+  ArrayVector arrays = {a0, a1, a2};
+  *out = std::make_shared<RecordBatch>(schema, a0->length(), arrays);
+  return Status::OK();
+}
+
+Status MakeTimes(std::shared_ptr<RecordBatch>* out) {
+  std::vector<bool> is_valid = {true, true, true, false, true, true, true};
+  auto f0 = field("f0", time(TimeUnit::MILLI));
+  auto f1 = field("f1", time(TimeUnit::NANO));
+  auto f2 = field("f2", time(TimeUnit::SECOND));
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+
+  std::vector<int64_t> ts_values = {1489269000000, 1489270000000, 1489271000000,
+      1489272000000, 1489272000000, 1489273000000};
+
+  std::shared_ptr<Array> a0, a1, a2;
+  ArrayFromVector<TimeType, int64_t>(f0->type, is_valid, ts_values, &a0);
+  ArrayFromVector<TimeType, int64_t>(f1->type, is_valid, ts_values, &a1);
+  ArrayFromVector<TimeType, int64_t>(f2->type, is_valid, ts_values, &a2);
+
+  ArrayVector arrays = {a0, a1, a2};
+  *out = std::make_shared<RecordBatch>(schema, a0->length(), arrays);
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 11ce50a76a547..f05a54168b631 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -63,6 +63,8 @@
 
 namespace arrow {
 
+using ArrayVector = std::vector<std::shared_ptr<Array>>;
+
 namespace test {
 
 template <typename T>
@@ -232,6 +234,22 @@ class TestBase : public ::testing::Test {
   MemoryPool* pool_;
 };
 
+template <typename TYPE, typename C_TYPE>
+void ArrayFromVector(const std::shared_ptr<DataType>& type,
+    const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
+    std::shared_ptr<Array>* out) {
+  MemoryPool* pool = default_memory_pool();
+  typename TypeTraits<TYPE>::BuilderType builder(pool, type);
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid[i]) {
+      ASSERT_OK(builder.Append(values[i]));
+    } else {
+      ASSERT_OK(builder.AppendNull());
+    }
+  }
+  ASSERT_OK(builder.Finish(out));
+}
+
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
     std::shared_ptr<Array>* out) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index aa0d70e5505e6..a838082d7e79a 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -452,7 +452,7 @@ struct ARROW_EXPORT Date32Type : public FixedWidthType {
 
   Date32Type() : FixedWidthType(Type::DATE32) {}
 
-  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 2cd14203cdbb1..91461da8c42a6 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -121,7 +121,7 @@ struct TypeTraits<Int64Type> {
 template <>
 struct TypeTraits<DateType> {
   using ArrayType = DateArray;
-  // using BuilderType = DateBuilder;
+  using BuilderType = DateBuilder;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(int64_t);
@@ -145,7 +145,7 @@ struct TypeTraits<Date32Type> {
 template <>
 struct TypeTraits<TimestampType> {
   using ArrayType = TimestampArray;
-  // using BuilderType = TimestampBuilder;
+  using BuilderType = TimestampBuilder;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(int64_t);
@@ -156,7 +156,7 @@ struct TypeTraits<TimestampType> {
 template <>
 struct TypeTraits<TimeType> {
   using ArrayType = TimeArray;
-  // using BuilderType = TimestampBuilder;
+  using BuilderType = TimeBuilder;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(int64_t);

From 344ad1f10f3a4a86692d7a32b17e9939131321a1 Mon Sep 17 00:00:00 2001
From: rvernica <rvernica@gmail.com>
Date: Sun, 12 Mar 2017 13:43:32 -0400
Subject: [PATCH 0357/1644] ARROW-619: Fix typos in setup.py args and
 LD_LIBRARY_PATH

Author: rvernica <rvernica@gmail.com>

Closes #372 from rvernica/patch-1 and squashes the following commits:

b27999a [rvernica] Fix typos in setup.py args and LD_LIBRARY_PATH
---
 python/doc/install.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/doc/install.rst b/python/doc/install.rst
index 4d99fa0caf1de..d93a88f547576 100644
--- a/python/doc/install.rst
+++ b/python/doc/install.rst
@@ -124,7 +124,7 @@ Install `pyarrow`
     # --with-jemalloc enables the jemalloc allocator support in PyArrow
     # --build-type=release disables debugging information and turns on
     #       compiler optimizations for native code
-    python setup.py build_ext --with-parquet --with--jemalloc --build-type=release install
+    python setup.py build_ext --with-parquet --with-jemalloc --build-type=release install
     python setup.py install
 
 .. warning::
@@ -134,7 +134,7 @@ Install `pyarrow`
 .. note::
     In development installations, you will also need to set a correct
     ``LD_LIBARY_PATH``. This is most probably done with
-    ``export LD_LIBARY_PATH=$ARROW_HOME/lib:$LD_LIBARY_PATH``.
+    ``export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH``.
 
 
 .. code-block:: python

From d4ecb5e54eb7bc9392ad2f4e1cf9a0fe42be8cd0 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Sun, 12 Mar 2017 13:49:53 -0400
Subject: [PATCH 0358/1644] ARROW-612: [Java] Added not null to Field.toString
 output

Changed `Field.toString` method to include an additional `not null` description only if the nullable flag is not set.  Changed test to update expected string output.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #368 from BryanCutler/Field-toString-show-nullable-ARROW-612 and squashes the following commits:

9dc633d [Bryan Cutler] added not null to Field.toString output
---
 .../main/java/org/apache/arrow/vector/types/pojo/Field.java    | 3 +++
 .../java/org/apache/arrow/vector/types/pojo/TestSchema.java    | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 2d528e4141907..f9b79ce556338 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -185,6 +185,9 @@ public String toString() {
     if (!children.isEmpty()) {
       sb.append("<").append(Joiner.on(", ").join(children)).append(">");
     }
+    if (!nullable) {
+      sb.append(" not null");
+    }
     return sb.toString();
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index d60d17ea76db8..f04c78ec45d97 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -53,7 +53,7 @@ public void testComplex() throws IOException {
         ));
     roundTrip(schema);
     assertEquals(
-        "Schema<a: Int(8, true), b: Struct<c: Int(16, true), d: Utf8>, e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND), h: Interval(DAY_TIME)>",
+        "Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>, e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND), h: Interval(DAY_TIME)>",
         schema.toString());
   }
 

From 492b3d4395ab735f2de2a797cf13554a6b15936b Mon Sep 17 00:00:00 2001
From: Brian Hulette <hulettbh@gmail.com>
Date: Sun, 12 Mar 2017 15:42:44 -0400
Subject: [PATCH 0359/1644] ARROW-613: WIP TypeScript Implementation

This implementation only supports reading arrow files at the moment - but it works in node and in the browser.

I've implemented basic numeric types as well as Utf8, Date, List, and Struct. I included a couple of example node scripts, one that just dumps an arrow file's schema, and another that converts it to a (very poorly formatted) CSV. I also included an example of reading an arrow file in the browser, currently hosted here: https://theneuralbit.github.io/arrow/ after selecting an arrow file, that page _should_ display the file contents in a table.

So far I've only tested it with this file: https://keybase.pub/hulettbh/example-csv.arrow, generated by my colleague @elahrvivaz

Author: Brian Hulette <hulettbh@gmail.com>
Author: Brian Hulette <brian.hulette@ccri.com>

Closes #358 from TheNeuralBit/javascript and squashes the following commits:

74e8520 [Brian Hulette] added a few more license headers
ce81034 [Brian Hulette] Cleaned up TextDecoder/Utf8Vector
3f0d9f0 [Brian Hulette] Added missing runtime dependency
a5800d9 [Brian Hulette] Added docstrings for Vector functions
3839485 [Brian Hulette] Removed tabs
1f6dcf3 [Brian Hulette] Renamed _arrayToInt
3a92bdd [Brian Hulette] Added ASF Licence headers
8092810 [Brian Hulette] Moved index.html to examples/ directory
0f43270 [Brian Hulette] Replaced table style with an original, basic one
c221f74 [Brian Hulette] Create README.md
71e72df [Brian Hulette] Added support for the browser via webpack
00bb974 [Brian Hulette] Initial typescript implementation
---
 javascript/.gitignore               |   4 +
 javascript/README.md                |  50 +++++
 javascript/bin/arrow2csv.js         |  47 ++++
 javascript/bin/arrow_schema.js      |  25 +++
 javascript/examples/read_file.html  |  79 +++++++
 javascript/lib/Arrow_generated.d.ts |   5 +
 javascript/lib/arrow.ts             | 201 +++++++++++++++++
 javascript/lib/bitarray.ts          |  55 +++++
 javascript/lib/types.ts             | 328 ++++++++++++++++++++++++++++
 javascript/package.json             |  19 ++
 javascript/postinstall.sh           |  18 ++
 javascript/tsconfig.json            |  14 ++
 javascript/webpack.config.js        |  21 ++
 13 files changed, 866 insertions(+)
 create mode 100644 javascript/.gitignore
 create mode 100644 javascript/README.md
 create mode 100755 javascript/bin/arrow2csv.js
 create mode 100755 javascript/bin/arrow_schema.js
 create mode 100644 javascript/examples/read_file.html
 create mode 100644 javascript/lib/Arrow_generated.d.ts
 create mode 100644 javascript/lib/arrow.ts
 create mode 100644 javascript/lib/bitarray.ts
 create mode 100644 javascript/lib/types.ts
 create mode 100644 javascript/package.json
 create mode 100755 javascript/postinstall.sh
 create mode 100644 javascript/tsconfig.json
 create mode 100644 javascript/webpack.config.js

diff --git a/javascript/.gitignore b/javascript/.gitignore
new file mode 100644
index 0000000000000..3b97e3ab95707
--- /dev/null
+++ b/javascript/.gitignore
@@ -0,0 +1,4 @@
+lib/*_generated.js
+dist
+node_modules
+typings
diff --git a/javascript/README.md b/javascript/README.md
new file mode 100644
index 0000000000000..98ef75674ede0
--- /dev/null
+++ b/javascript/README.md
@@ -0,0 +1,50 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+### Installation
+
+From this directory, run:
+
+``` bash
+$ npm install   # pull dependencies
+$ tsc           # build typescript
+$ webpack       # bundle for the browser
+```
+
+### Usage
+The library is designed to be used with node.js or in the browser, this repository contains examples of both.
+
+#### Node
+Import the arrow module:
+
+``` js
+var arrow = require("arrow.js");
+```
+
+See [bin/arrow_schema.js](bin/arrow_schema.js) and [bin/arrow2csv.js](bin/arrow2csv.js) for usage examples.
+
+#### Browser
+Include `dist/arrow-bundle.js` in a `<script />` tag:
+``` html
+<script src="arrow-bundle.js"/>
+```
+See [examples/read_file.html](examples/read_file.html) for a usage example - or try it out now at [theneuralbit.github.io/arrow](http://theneuralbit.github.io/arrow)
+
+### API
+##### `arrow.loadSchema(buffer)`
+Returns a JSON representation of the file's Arrow schema.
+
+##### `arrow.loadVectors(buffer)`
+Returns a dictionary of `Vector` objects, one for each column, indexed by the column's name.
+Vector objects have, at minimum, a `get(i)` method and a `length` attribute.
diff --git a/javascript/bin/arrow2csv.js b/javascript/bin/arrow2csv.js
new file mode 100755
index 0000000000000..48df2f9648319
--- /dev/null
+++ b/javascript/bin/arrow2csv.js
@@ -0,0 +1,47 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+var fs = require('fs')
+var process = require('process');
+var loadVectors = require('../dist/arrow.js').loadVectors;
+var program = require('commander');
+
+function list (val) {
+    return val.split(',');
+}
+
+program
+  .version('0.1.0')
+  .usage('[options] <file>')
+  .option('-s --schema <list>', 'A comma-separated list of column names', list)
+  .parse(process.argv);
+
+if (!program.schema) {
+    program.outputHelp();
+    process.exit(1);
+}
+
+var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
+var vectors = loadVectors(buf);
+
+for (var i = 0; i < vectors[program.schema[0]].length; i += 1|0) {
+    console.log(program.schema.map(function (field) {
+        return '' + vectors[field].get(i);
+    }).join(','));
+}
diff --git a/javascript/bin/arrow_schema.js b/javascript/bin/arrow_schema.js
new file mode 100755
index 0000000000000..704477863bf95
--- /dev/null
+++ b/javascript/bin/arrow_schema.js
@@ -0,0 +1,25 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+var fs = require('fs');
+var process = require('process');
+var loadSchema = require('../dist/arrow.js').loadSchema;
+
+var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
+console.log(JSON.stringify(loadSchema(buf), null, '\t'));
diff --git a/javascript/examples/read_file.html b/javascript/examples/read_file.html
new file mode 100644
index 0000000000000..02b6f0805c770
--- /dev/null
+++ b/javascript/examples/read_file.html
@@ -0,0 +1,79 @@
+<!DOCTYPE html>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<html>
+  <head>
+    <title>arrow.js browser test</title>
+    <meta charset="utf-8">
+    <style>
+table {
+  border-collapse: collapse;
+}
+table, th, td {
+  border: 1px solid black;
+}
+    </style>
+    <script type="text/javascript">
+var reader = new FileReader();
+function addCell (tr, type, name) {
+  var td = document.createElement(type)
+  td.textContent = name;
+  tr.appendChild(td);
+}
+reader.onload = function (evt) {
+  var buf = new Uint8Array(evt.target.result);
+  var schema = arrow.loadSchema(buf);
+  var vectors = arrow.loadVectors(buf);
+  var length = vectors[schema[0].name].length;
+
+  var thead = document.getElementById("thead");
+  var tbody = document.getElementById("tbody");
+  var header_row = document.createElement("tr");
+
+  schema.forEach(function (d) {
+    addCell(header_row, "th", d.name);
+  });
+
+  thead.appendChild(header_row);
+
+  for (var i = 0; i < length; i += 1|0) {
+    var tr = document.createElement("tr");
+    schema.forEach(function (d) { addCell(tr, "td", vectors[d.name].get(i)); });
+    tbody.appendChild(tr);
+  }
+}
+
+function handleFiles(files) {
+  reader.readAsArrayBuffer(files[0]);
+}
+    </script>
+  </head>
+  <body>
+    <input id="arrow-in" type="file" onchange="handleFiles(this.files)" />
+    <table>
+      <thead id="thead">
+      </thead>
+      <tbody id="tbody">
+      </tbody>
+    </table>
+    <script type="text/javascript" src="../dist/arrow-bundle.js"></script>
+  </body>
+</html>
diff --git a/javascript/lib/Arrow_generated.d.ts b/javascript/lib/Arrow_generated.d.ts
new file mode 100644
index 0000000000000..1f5b4547a478c
--- /dev/null
+++ b/javascript/lib/Arrow_generated.d.ts
@@ -0,0 +1,5 @@
+export var org: {
+  apache: {
+    arrow: any
+  }
+}
diff --git a/javascript/lib/arrow.ts b/javascript/lib/arrow.ts
new file mode 100644
index 0000000000000..0762885aef8cc
--- /dev/null
+++ b/javascript/lib/arrow.ts
@@ -0,0 +1,201 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import { org } from './Arrow_generated';
+var arrow = org.apache.arrow;
+import { vectorFromField, Vector } from './types';
+
+export function loadVectors(buf) {
+    var fileLength = buf.length, bb, footerLengthOffset, footerLength,
+        footerOffset, footer, schema, field, type, type_str, i,
+        len, rb_metas, rb_meta, rtrn, recordBatchBlock, recordBatchBlocks = [];
+    var vectors : Vector[] = [];
+
+    bb = new flatbuffers.ByteBuffer(buf);
+
+    footer = _loadFooter(bb);
+
+    schema = footer.schema();
+
+    for (i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
+        field = schema.fields(i);
+        vectors.push(vectorFromField(field));
+    }
+
+    for (i = 0; i < footer.recordBatchesLength(); i += 1|0) {
+        recordBatchBlock = footer.recordBatches(i);
+        recordBatchBlocks.push({
+            offset: recordBatchBlock.offset(),
+            metaDataLength: recordBatchBlock.metaDataLength(),
+            bodyLength: recordBatchBlock.bodyLength(),
+        })
+    }
+
+    loadBuffersIntoVectors(recordBatchBlocks, bb, vectors);
+    var rtrn : any = {};
+    for (var i : any = 0; i < vectors.length; i += 1|0) {
+      rtrn[vectors[i].name] = vectors[i]
+    }
+    return rtrn;
+}
+
+export function loadSchema(buf) {
+    var footer = _loadFooter(new flatbuffers.ByteBuffer(buf));
+    var schema = footer.schema();
+
+    return parseSchema(schema);
+}
+
+function _loadFooter(bb) {
+    var fileLength: number = bb.bytes_.length;
+
+    if (fileLength < MAGIC.length*2 + 4) {
+      console.error("file too small " + fileLength);
+      return;
+    }
+
+    if (!_checkMagic(bb.bytes_, 0)) {
+      console.error("missing magic bytes at beginning of file")
+      return;
+    }
+
+    if (!_checkMagic(bb.bytes_, fileLength - MAGIC.length)) {
+      console.error("missing magic bytes at end of file")
+      return;
+    }
+
+    var footerLengthOffset: number = fileLength - MAGIC.length - 4;
+    bb.setPosition(footerLengthOffset);
+    var footerLength: number = Int64FromByteBuffer(bb, footerLengthOffset)
+
+    if (footerLength <= 0 || footerLength + MAGIC.length*2 + 4 > fileLength)  {
+      console.log("Invalid footer length: " + footerLength)
+    }
+
+    var footerOffset: number = footerLengthOffset - footerLength;
+    bb.setPosition(footerOffset);
+    var footer = arrow.flatbuf.Footer.getRootAsFooter(bb);
+
+    return footer;
+}
+
+function Int64FromByteBuffer(bb, offset) {
+    return ((bb.bytes_[offset + 3] & 255) << 24) |
+           ((bb.bytes_[offset + 2] & 255) << 16) |
+           ((bb.bytes_[offset + 1] & 255) << 8) |
+           ((bb.bytes_[offset] & 255));
+}
+
+
+var MAGIC_STR = "ARROW1";
+var MAGIC = new Uint8Array(MAGIC_STR.length);
+for (var i = 0; i < MAGIC_STR.length; i += 1|0) {
+    MAGIC[i] = MAGIC_STR.charCodeAt(i);
+}
+
+function _checkMagic(buf, index) {
+    for (var i = 0; i < MAGIC.length; i += 1|0) {
+        if (MAGIC[i] != buf[index + i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+var TYPEMAP = {}
+TYPEMAP[arrow.flatbuf.Type.NONE]          = "NONE";
+TYPEMAP[arrow.flatbuf.Type.Null]          = "Null";
+TYPEMAP[arrow.flatbuf.Type.Int]           = "Int";
+TYPEMAP[arrow.flatbuf.Type.FloatingPoint] = "FloatingPoint";
+TYPEMAP[arrow.flatbuf.Type.Binary]        = "Binary";
+TYPEMAP[arrow.flatbuf.Type.Utf8]          = "Utf8";
+TYPEMAP[arrow.flatbuf.Type.Bool]          = "Bool";
+TYPEMAP[arrow.flatbuf.Type.Decimal]       = "Decimal";
+TYPEMAP[arrow.flatbuf.Type.Date]          = "Date";
+TYPEMAP[arrow.flatbuf.Type.Time]          = "Time";
+TYPEMAP[arrow.flatbuf.Type.Timestamp]     = "Timestamp";
+TYPEMAP[arrow.flatbuf.Type.Interval]      = "Interval";
+TYPEMAP[arrow.flatbuf.Type.List]          = "List";
+TYPEMAP[arrow.flatbuf.Type.Struct_]       = "Struct";
+TYPEMAP[arrow.flatbuf.Type.Union]         = "Union";
+
+var VECTORTYPEMAP = {};
+VECTORTYPEMAP[arrow.flatbuf.VectorType.OFFSET]   = 'OFFSET';
+VECTORTYPEMAP[arrow.flatbuf.VectorType.DATA]     = 'DATA';
+VECTORTYPEMAP[arrow.flatbuf.VectorType.VALIDITY] = 'VALIDITY';
+VECTORTYPEMAP[arrow.flatbuf.VectorType.TYPE]     = 'TYPE';
+
+function parseField(field) {
+    var children = [];
+    for (var i = 0; i < field.childrenLength(); i += 1|0) {
+        children.push(parseField(field.children(i)));
+    }
+
+    var layouts = [];
+    for (var i = 0; i < field.layoutLength(); i += 1|0) {
+        layouts.push(VECTORTYPEMAP[field.layout(i).type()]);
+
+    }
+
+    return {
+      name: field.name(),
+      nullable: field.nullable(),
+      type: TYPEMAP[field.typeType()],
+      children: children,
+      layout: layouts
+    };
+}
+
+function parseSchema(schema) {
+    var result = [];
+    var this_result, type;
+    for (var i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
+        result.push(parseField(schema.fields(i)));
+    }
+    return result;
+}
+
+function parseBuffer(buffer) {
+    return {
+        offset: buffer.offset(),
+        length: buffer.length()
+    };
+}
+
+function loadBuffersIntoVectors(recordBatchBlocks, bb, vectors : Vector[]) {
+    var fieldNode, recordBatchBlock, recordBatch, numBuffers, bufReader = {index: 0, node_index: 1}, field_ctr = 0;
+    var buffer = bb.bytes_.buffer;
+    var baseOffset = bb.bytes_.byteOffset;
+    for (var i = recordBatchBlocks.length - 1; i >= 0; i -= 1|0) {
+        recordBatchBlock = recordBatchBlocks[i];
+        bb.setPosition(recordBatchBlock.offset.low);
+        recordBatch = arrow.flatbuf.RecordBatch.getRootAsRecordBatch(bb);
+        bufReader.index = 0;
+        bufReader.node_index = 0;
+        numBuffers = recordBatch.buffersLength();
+
+        //console.log('num buffers: ' + recordBatch.buffersLength());
+        //console.log('num nodes: ' + recordBatch.nodesLength());
+
+        while (bufReader.index < numBuffers) {
+            //console.log('Allocating buffers starting at ' + bufReader.index + '/' + numBuffers + ' to field ' + field_ctr);
+            vectors[field_ctr].loadData(recordBatch, buffer, bufReader, baseOffset + recordBatchBlock.offset.low + recordBatchBlock.metaDataLength)
+            field_ctr += 1;
+        }
+    }
+}
diff --git a/javascript/lib/bitarray.ts b/javascript/lib/bitarray.ts
new file mode 100644
index 0000000000000..82fff32c194fa
--- /dev/null
+++ b/javascript/lib/bitarray.ts
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+export class BitArray {
+    private view: Uint8Array;
+    constructor(buffer: ArrayBuffer, offset: number, length: number) {
+        //if (ArrayBuffer.isView(buffer)) {
+        //    var og_view = buffer;
+        //    buffer = buffer.buffer;
+        //    offset = og_view.offset;
+        //    length = og_view.length/og_view.BYTES_PER_ELEMENT*8;
+        //} else if (buffer instanceof ArrayBuffer) {
+        var offset = offset || 0;
+        var length = length;// || buffer.length*8;
+        //} else if (buffer instanceof Number) {
+        //    length = buffer;
+        //    buffer = new ArrayBuffer(Math.ceil(length/8));
+        //    offset = 0;
+        //}
+
+        this.view = new Uint8Array(buffer, offset, Math.ceil(length/8));
+    }
+
+    get(i) {
+        var index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too.
+        var bit = i % 8;  // i % 8 is just as fast as i & 7
+        return (this.view[index] & (1 << bit)) !== 0;
+    }
+
+    set(i) {
+        var index = (i >> 3) | 0;
+        var bit = i % 8;
+        this.view[index] |= 1 << bit;
+    }
+
+    unset(i) {
+        var index = (i >> 3) | 0;
+        var bit = i % 8;
+        this.view[index] &= ~(1 << bit);
+    }
+}
diff --git a/javascript/lib/types.ts b/javascript/lib/types.ts
new file mode 100644
index 0000000000000..bbc755810056f
--- /dev/null
+++ b/javascript/lib/types.ts
@@ -0,0 +1,328 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { BitArray } from './bitarray';
+import { TextDecoder } from 'text-encoding';
+import { org } from './Arrow_generated';
+var arrow = org.apache.arrow;
+
+interface ArrayView {
+    slice(start: number, end: number) : ArrayView
+    toString() : string
+}
+
+export abstract class Vector {
+    name: string;
+    length: number;
+    null_count: number;
+    constructor(name: string) {
+        this.name = name;
+    }
+    /* Access datum at index i */
+    abstract get(i);
+    /* Return array representing data in the range [start, end) */
+    abstract slice(start: number, end: number);
+
+    /* Use recordBatch fieldNodes and Buffers to construct this Vector */
+    public loadData(recordBatch: any, buffer: any, bufReader: any, baseOffset: any) {
+        var fieldNode = recordBatch.nodes(bufReader.node_index);
+        this.length = fieldNode.length();
+        this.null_count = fieldNode.length();
+        bufReader.node_index += 1|0;
+
+        this.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+    }
+
+    protected abstract loadBuffers(recordBatch: any, buffer: any, bufReader: any, baseOffset: any);
+
+    /* Helper function for loading a VALIDITY buffer (for Nullable types) */
+    static loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset) : BitArray {
+        var buf_meta = recordBatch.buffers(bufReader.index);
+        var offset = baseOffset + buf_meta.offset().low;
+        var length = buf_meta.length().low;
+        bufReader.index += 1|0;
+        return new BitArray(buffer, offset, length*8);
+    }
+
+    /* Helper function for loading an OFFSET buffer */
+    static loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset) : Int32Array {
+        var buf_meta = recordBatch.buffers(bufReader.index);
+        var offset = baseOffset + buf_meta.offset().low;
+        var length = buf_meta.length().low/Int32Array.BYTES_PER_ELEMENT;
+        bufReader.index += 1|0;
+        return new Int32Array(buffer, offset, length);
+    }
+
+}
+
+class SimpleVector<T extends ArrayView> extends Vector {
+    protected dataView: T;
+    private TypedArray: {new(buffer: any, offset: number, length: number) : T, BYTES_PER_ELEMENT: number};
+
+    constructor (TypedArray: {new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number}, name: string) {
+        super(name);
+        this.TypedArray = TypedArray;
+    }
+
+    get(i) {
+        return this.dataView[i];
+    }
+
+    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+        this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+    }
+
+    loadDataBuffer(recordBatch, buffer, bufReader, baseOffset) : T {
+        var buf_meta = recordBatch.buffers(bufReader.index);
+        var offset = baseOffset + buf_meta.offset().low;
+        var length = buf_meta.length().low/this.TypedArray.BYTES_PER_ELEMENT;
+        bufReader.index += 1|0;
+        return new this.TypedArray(buffer, offset, length);
+    }
+
+    getDataView() {
+        return this.dataView;
+    }
+
+    toString() {
+        return this.dataView.toString();
+    }
+
+    slice(start, end) {
+        return this.dataView.slice(start, end);
+    }
+}
+
+class NullableSimpleVector<T extends ArrayView> extends SimpleVector<T> {
+    private validityView: BitArray;
+
+    get(i: number) {
+        if (this.validityView.get(i)) return this.dataView[i];
+        else                          return null
+    }
+
+    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+        this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+    }
+
+}
+
+class Uint8Vector   extends SimpleVector<Uint8Array>   { constructor(name: string) { super(Uint8Array,  name);  }; }
+class Uint16Vector  extends SimpleVector<Uint16Array>  { constructor(name: string) { super(Uint16Array, name);  }; }
+class Uint32Vector  extends SimpleVector<Uint32Array>  { constructor(name: string) { super(Uint32Array, name);  }; }
+class Int8Vector    extends SimpleVector<Uint8Array>   { constructor(name: string) { super(Uint8Array,  name);  }; }
+class Int16Vector   extends SimpleVector<Uint16Array>  { constructor(name: string) { super(Uint16Array, name);  }; }
+class Int32Vector   extends SimpleVector<Uint32Array>  { constructor(name: string) { super(Uint32Array, name);  }; }
+class Float32Vector extends SimpleVector<Float32Array> { constructor(name: string) { super(Float32Array, name); }; }
+class Float64Vector extends SimpleVector<Float64Array> { constructor(name: string) { super(Float64Array, name); }; }
+
+class NullableUint8Vector   extends NullableSimpleVector<Uint8Array>   { constructor(name: string) { super(Uint8Array,  name);  }; }
+class NullableUint16Vector  extends NullableSimpleVector<Uint16Array>  { constructor(name: string) { super(Uint16Array, name);  }; }
+class NullableUint32Vector  extends NullableSimpleVector<Uint32Array>  { constructor(name: string) { super(Uint32Array, name);  }; }
+class NullableInt8Vector    extends NullableSimpleVector<Uint8Array>   { constructor(name: string) { super(Uint8Array,  name);  }; }
+class NullableInt16Vector   extends NullableSimpleVector<Uint16Array>  { constructor(name: string) { super(Uint16Array, name);  }; }
+class NullableInt32Vector   extends NullableSimpleVector<Uint32Array>  { constructor(name: string) { super(Uint32Array, name);  }; }
+class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(name: string) { super(Float32Array, name); }; }
+class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(name: string) { super(Float64Array, name); }; }
+
+class Utf8Vector extends SimpleVector<Uint8Array> {
+    protected offsetView: Int32Array;
+    static decoder: TextDecoder = new TextDecoder('utf8');
+
+    constructor(name: string) {
+        super(Uint8Array, name);
+    }
+
+    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+        this.offsetView = Vector.loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset);
+        this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+    }
+
+    get(i) {
+        return Utf8Vector.decoder.decode
+            (this.dataView.slice(this.offsetView[i], this.offsetView[i + 1]));
+    }
+
+    slice(start: number, end: number) {
+        var rtrn: string[] = [];
+        for (var i: number = start; i < end; i += 1|0) {
+            rtrn.push(this.get(i));
+        }
+        return rtrn;
+    }
+}
+
+class NullableUtf8Vector extends Utf8Vector {
+    private validityView: BitArray;
+
+    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+        this.offsetView = Vector.loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset);
+        this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+    }
+
+    get(i) {
+        if (!this.validityView.get(i)) return null;
+        return super.get(i);
+    }
+}
+
+// Nested Types
+class ListVector extends Uint32Vector {
+    private dataVector: Vector;
+
+    constructor(name, dataVector : Vector) {
+        super(name);
+        this.dataVector = dataVector;
+    }
+
+    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+        super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+        this.dataVector.loadData(recordBatch, buffer, bufReader, baseOffset);
+        this.length -= 1;
+    }
+
+    get(i) {
+        var offset = super.get(i)
+        if (offset === null) {
+            return null;
+        }
+        var next_offset = super.get(i + 1)
+        return this.dataVector.slice(offset, next_offset)
+    }
+
+    toString() {
+        return "length: " + (this.length);
+    }
+
+    slice(start : number, end : number) { return []; };
+}
+
+class NullableListVector extends ListVector {
+    private validityView: BitArray;
+
+    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+        super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+    }
+
+    get(i) {
+        if (!this.validityView.get(i)) return null;
+        return super.get(i);
+    }
+}
+
+class StructVector extends Vector {
+    private validityView: BitArray;
+    private vectors : Vector[];
+    constructor(name: string, vectors: Vector[]) {
+        super(name);
+        this.vectors = vectors;
+    }
+
+    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+        this.vectors.forEach((v: Vector) => v.loadData(recordBatch, buffer, bufReader, baseOffset));
+    }
+
+    get(i : number) {
+        if (!this.validityView.get(i)) return null;
+        return this.vectors.map((v: Vector) => v.get(i));
+    }
+
+    slice(start : number, end : number) {
+        var rtrn = [];
+        for (var i: number = start; i < end; i += 1|0) {
+            rtrn.push(this.get(i));
+        }
+        return rtrn;
+    }
+}
+
+class DateVector extends SimpleVector<Uint32Array> {
+    constructor (name: string) {
+        super(Uint32Array, name);
+    }
+
+    get (i) {
+        return new Date(super.get(2*i+1)*Math.pow(2,32) + super.get(2*i));
+    }
+}
+
+class NullableDateVector extends DateVector {
+    private validityView: BitArray;
+
+    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+        super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+    }
+
+    get (i) {
+        if (!this.validityView.get(i)) return null;
+        return super.get(i);
+    }
+}
+
+var BASIC_TYPES = [arrow.flatbuf.Type.Int, arrow.flatbuf.Type.FloatingPoint, arrow.flatbuf.Type.Utf8, arrow.flatbuf.Type.Date];
+
+export function vectorFromField(field) : Vector {
+    var typeType = field.typeType();
+    if (BASIC_TYPES.indexOf(typeType) >= 0) {
+        var type = field.typeType();
+        if (type === arrow.flatbuf.Type.Int) {
+            type = field.type(new arrow.flatbuf.Int());
+            var VectorConstructor : {new(string) : Vector};
+            if (type.isSigned()) {
+                if (type.bitWidth() == 32)
+                    VectorConstructor = field.nullable() ? NullableInt32Vector : Int32Vector;
+                else if (type.bitWidth() == 16)
+                    VectorConstructor = field.nullable() ? NullableInt16Vector : Int16Vector;
+                else if (type.bitWidth() == 8)
+                    VectorConstructor = field.nullable() ? NullableInt8Vector : Int8Vector;
+            } else {
+                if (type.bitWidth() == 32)
+                    VectorConstructor = field.nullable() ? NullableUint32Vector : Uint32Vector;
+                else if (type.bitWidth() == 16)
+                    VectorConstructor = field.nullable() ? NullableUint16Vector : Uint16Vector;
+                else if (type.bitWidth() == 8)
+                    VectorConstructor = field.nullable() ? NullableUint8Vector : Uint8Vector;
+            }
+        } else if (type === arrow.flatbuf.Type.FloatingPoint) {
+            type = field.type(new arrow.flatbuf.FloatingPoint());
+            if (type.precision() == arrow.flatbuf.Precision.SINGLE)
+                VectorConstructor = field.nullable() ? NullableFloat32Vector : Float32Vector;
+            else if (type.precision() == arrow.flatbuf.Precision.DOUBLE)
+                VectorConstructor = field.nullable() ? NullableFloat64Vector : Float64Vector;
+        } else if (type === arrow.flatbuf.Type.Utf8) {
+            VectorConstructor = field.nullable() ? NullableUtf8Vector : Utf8Vector;
+        } else if (type === arrow.flatbuf.Type.Date) {
+            VectorConstructor = field.nullable() ? NullableDateVector : DateVector;
+        }
+
+        return new VectorConstructor(field.name());
+    } else if (typeType === arrow.flatbuf.Type.List) {
+        var dataVector = vectorFromField(field.children(0));
+        return field.nullable() ? new NullableListVector(field.name(), dataVector) : new ListVector(field.name(), dataVector);
+    } else if (typeType === arrow.flatbuf.Type.Struct_) {
+        var vectors : Vector[] = [];
+        for (var i : number = 0; i < field.childrenLength(); i += 1|0) {
+            vectors.push(vectorFromField(field.children(i)));
+        }
+        return new StructVector(field.name(), vectors);
+    }
+}
diff --git a/javascript/package.json b/javascript/package.json
new file mode 100644
index 0000000000000..b1e583b7d9da6
--- /dev/null
+++ b/javascript/package.json
@@ -0,0 +1,19 @@
+{
+  "name": "arrow",
+  "version": "0.0.0",
+  "description": "",
+  "main": "dist/arrow.js",
+  "scripts": {
+    "postinstall": "./postinstall.sh",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "flatbuffers": "^1.5.0",
+    "text-encoding": "^0.6.4"
+  },
+  "dependencies": {
+    "commander": "^2.9.0"
+  }
+}
diff --git a/javascript/postinstall.sh b/javascript/postinstall.sh
new file mode 100755
index 0000000000000..1e6622fa4f2ee
--- /dev/null
+++ b/javascript/postinstall.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+echo "Compiling flatbuffer schemas..."
+#flatc -o lib --js ../format/Message.fbs ../format/File.fbs
+flatc -o lib --js ../format/*.fbs
+cat lib/*_generated.js > lib/Arrow_generated.js
diff --git a/javascript/tsconfig.json b/javascript/tsconfig.json
new file mode 100644
index 0000000000000..89c31ef85a143
--- /dev/null
+++ b/javascript/tsconfig.json
@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "outDir": "./dist/",
+    "allowJs": true,
+    "target": "es5",
+    "module": "commonjs",
+    "moduleResolution": "node"
+  },
+  "include": [
+    "typings/index.d.ts",
+    "lib/*.js",
+    "lib/*.ts"
+  ]
+}
diff --git a/javascript/webpack.config.js b/javascript/webpack.config.js
new file mode 100644
index 0000000000000..a0ed56370f6b1
--- /dev/null
+++ b/javascript/webpack.config.js
@@ -0,0 +1,21 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License. See accompanying LICENSE file.
+
+module.exports = {
+  entry: './dist/arrow.js',
+  output: {
+    path: __dirname + '/dist',
+    filename: 'arrow-bundle.js',
+    libraryTarget: 'var',
+    library: 'arrow'
+  }
+};

From 2cf36ef2d6aea6a5ddf32c900d33db40d728bcd9 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 12 Mar 2017 23:18:23 -0400
Subject: [PATCH 0360/1644] ARROW-574: Python: Add support for nested Python
 lists in Pandas conversion

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #364 from xhochy/ARROW-574 and squashes the following commits:

3ef02ae [Uwe L. Korn] ARROW-574: Python: Add support for nested Python lists in Pandas conversion
---
 python/pyarrow/tests/pandas_examples.py     | 40 +++++++++++++++++++++
 python/pyarrow/tests/test_convert_pandas.py | 14 ++++++--
 python/pyarrow/tests/test_parquet.py        | 17 +++++++--
 python/src/pyarrow/adapters/builtin.cc      | 27 +++++++-------
 python/src/pyarrow/adapters/builtin.h       |  7 ++++
 python/src/pyarrow/adapters/pandas.cc       | 25 +++++++++++--
 6 files changed, 112 insertions(+), 18 deletions(-)

diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py
index 63af42348026c..c9343fce233d2 100644
--- a/python/pyarrow/tests/pandas_examples.py
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -76,3 +76,43 @@ def dataframe_with_arrays():
     schema = pa.Schema.from_fields(fields)
 
     return df, schema
+
+def dataframe_with_lists():
+    """
+    Dataframe with list columns of every possible primtive type.
+
+    Returns
+    -------
+    df: pandas.DataFrame
+    schema: pyarrow.Schema
+        Arrow schema definition that is in line with the constructed df.
+    """
+    arrays = OrderedDict()
+    fields = []
+
+    fields.append(pa.field('int64', pa.list_(pa.int64())))
+    arrays['int64'] = [
+        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
+        [0, 1, 2, 3, 4],
+        None,
+        [0]
+    ]
+    fields.append(pa.field('double', pa.list_(pa.double())))
+    arrays['double'] = [
+        [0., 1., 2., 3., 4., 5., 6., 7., 8., 9.],
+        [0., 1., 2., 3., 4.],
+        None,
+        [0.]
+    ]
+    fields.append(pa.field('str_list', pa.list_(pa.string())))
+    arrays['str_list'] = [
+        [u"1", u"ä"],
+        None,
+        [u"1"],
+        [u"1", u"2", u"3"]
+    ]
+
+    df = pd.DataFrame(arrays)
+    schema = pa.Schema.from_fields(fields)
+
+    return df, schema
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 953fa2c4b9a72..a79bb2392ea6c 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -30,7 +30,7 @@
 from pyarrow.compat import u
 import pyarrow as A
 
-from .pandas_examples import dataframe_with_arrays
+from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
 
 def _alltypes_example(size=100):
@@ -333,7 +333,7 @@ def test_date(self):
         expected['date'] = pd.to_datetime(df['date'])
         tm.assert_frame_equal(result, expected)
 
-    def test_column_of_lists(self):
+    def test_column_of_arrays(self):
         df, schema = dataframe_with_arrays()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
         table = A.Table.from_pandas(df, schema=schema)
@@ -343,6 +343,16 @@ def test_column_of_lists(self):
             field = schema.field_by_name(column)
             self._check_array_roundtrip(df[column], field=field)
 
+    def test_column_of_lists(self):
+        df, schema = dataframe_with_lists()
+        self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
+        table = A.Table.from_pandas(df, schema=schema)
+        assert table.schema.equals(schema)
+
+        for column in df.columns:
+            field = schema.field_by_name(column)
+            self._check_array_roundtrip(df[column], field=field)
+
     def test_threaded_conversion(self):
         df = _alltypes_example()
         self._check_pandas_roundtrip(df, nthreads=2,
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 96f2d15e312f2..c72ff9e862b76 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -23,7 +23,7 @@
 from pyarrow.compat import guid
 import pyarrow as pa
 import pyarrow.io as paio
-from .pandas_examples import dataframe_with_arrays
+from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
 import numpy as np
 import pandas as pd
@@ -322,7 +322,7 @@ def test_compare_schemas():
 
 
 @parquet
-def test_column_of_lists(tmpdir):
+def test_column_of_arrays(tmpdir):
     df, schema = dataframe_with_arrays()
 
     filename = tmpdir.join('pandas_rountrip.parquet')
@@ -334,6 +334,19 @@ def test_column_of_lists(tmpdir):
     pdt.assert_frame_equal(df, df_read)
 
 
+@parquet
+def test_column_of_lists(tmpdir):
+    df, schema = dataframe_with_lists()
+
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True,
+                                       schema=schema)
+    pq.write_table(arrow_table, filename.strpath, version="2.0")
+    table_read = pq.read_table(filename.strpath)
+    df_read = table_read.to_pandas()
+    pdt.assert_frame_equal(df, df_read)
+
+
 @parquet
 def test_multithreaded_read():
     df = alltypes_sample(size=10000)
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index c125cc078af88..4f7b2cb09e1e6 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -206,8 +206,7 @@ class SeqVisitor {
 };
 
 // Non-exhaustive type inference
-static Status InferArrowType(
-    PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
+Status InferArrowType(PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
   *size = PySequence_Size(obj);
   if (PyErr_Occurred()) {
     // Not a sequence
@@ -496,6 +495,19 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
   return Status::OK();
 }
 
+Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
+    const std::shared_ptr<ArrayBuilder>& builder) {
+  std::shared_ptr<SeqConverter> converter = GetConverter(type);
+  if (converter == nullptr) {
+    std::stringstream ss;
+    ss << "No type converter implemented for " << type->ToString();
+    return Status::NotImplemented(ss.str());
+  }
+  converter->Init(builder);
+
+  return converter->AppendData(obj);
+}
+
 Status ConvertPySequence(
     PyObject* obj, MemoryPool* pool, std::shared_ptr<arrow::Array>* out) {
   std::shared_ptr<DataType> type;
@@ -509,19 +521,10 @@ Status ConvertPySequence(
     return Status::OK();
   }
 
-  std::shared_ptr<SeqConverter> converter = GetConverter(type);
-  if (converter == nullptr) {
-    std::stringstream ss;
-    ss << "No type converter implemented for " << type->ToString();
-    return Status::NotImplemented(ss.str());
-  }
-
   // Give the sequence converter an array builder
   std::shared_ptr<ArrayBuilder> builder;
   RETURN_NOT_OK(arrow::MakeBuilder(pool, type, &builder));
-  converter->Init(builder);
-
-  RETURN_NOT_OK(converter->AppendData(obj));
+  RETURN_NOT_OK(AppendPySequence(obj, type, builder));
 
   return builder->Finish(out);
 }
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
index 667298e3c5c5f..0c863a5631ada 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -37,6 +37,13 @@ class Status;
 
 namespace pyarrow {
 
+PYARROW_EXPORT arrow::Status InferArrowType(
+    PyObject* obj, int64_t* size, std::shared_ptr<arrow::DataType>* out_type);
+
+PYARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj,
+    const std::shared_ptr<arrow::DataType>& type,
+    const std::shared_ptr<arrow::ArrayBuilder>& builder);
+
 PYARROW_EXPORT
 arrow::Status ConvertPySequence(
     PyObject* obj, arrow::MemoryPool* pool, std::shared_ptr<arrow::Array>* out);
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index eb3ab49f58892..40079b49b9638 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -19,6 +19,7 @@
 
 #include <Python.h>
 
+#include "pyarrow/adapters/builtin.h"
 #include "pyarrow/adapters/pandas.h"
 #include "pyarrow/numpy_interop.h"
 
@@ -1661,6 +1662,7 @@ inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
   typedef npy_traits<ITEM_TYPE> traits;
   typedef typename traits::value_type T;
   typedef typename traits::BuilderClass BuilderT;
+  PyAcquireGIL lock;
 
   auto value_builder = std::make_shared<BuilderT>(pool_, field->type);
   ListBuilder list_builder(pool_, value_builder);
@@ -1688,7 +1690,16 @@ inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
         RETURN_NOT_OK(value_builder->Append(data, size));
       }
     } else if (PyList_Check(objects[i])) {
-      return Status::TypeError("Python lists are not yet supported");
+      int64_t size;
+      std::shared_ptr<arrow::DataType> type;
+      RETURN_NOT_OK(list_builder.Append(true));
+      RETURN_NOT_OK(InferArrowType(objects[i], &size, &type));
+      if (type->type != field->type->type) {
+        std::stringstream ss;
+        ss << type->ToString() << " cannot be converted to " << field->type->ToString();
+        return Status::TypeError(ss.str());
+      }
+      RETURN_NOT_OK(AppendPySequence(objects[i], field->type, value_builder));
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }
@@ -1702,6 +1713,7 @@ inline Status
 ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
     const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
   // TODO: If there are bytes involed, convert to Binary representation
+  PyAcquireGIL lock;
   bool have_bytes = false;
 
   auto value_builder = std::make_shared<arrow::StringBuilder>(pool_);
@@ -1721,7 +1733,16 @@ ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
       auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
       RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
     } else if (PyList_Check(objects[i])) {
-      return Status::TypeError("Python lists are not yet supported");
+      int64_t size;
+      std::shared_ptr<arrow::DataType> type;
+      RETURN_NOT_OK(list_builder.Append(true));
+      RETURN_NOT_OK(InferArrowType(objects[i], &size, &type));
+      if (type->type != Type::STRING) {
+        std::stringstream ss;
+        ss << type->ToString() << " cannot be converted to STRING.";
+        return Status::TypeError(ss.str());
+      }
+      RETURN_NOT_OK(AppendPySequence(objects[i], type, value_builder));
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }

From 331be4923ac4b30dafa7e79785b71b89ddeb8f3c Mon Sep 17 00:00:00 2001
From: Miki Tebeka <miki.tebeka@gmail.com>
Date: Mon, 13 Mar 2017 13:16:04 -0400
Subject: [PATCH 0361/1644] ARROW-623: Fix segfault in __repr__ of empty field

Small fix for not segfaulting when printing an empty field.

Author: Miki Tebeka <miki.tebeka@gmail.com>

Closes #374 from tebeka/field-bug and squashes the following commits:

abcd118 [Miki Tebeka] ARROW-623: Fix segfault in __repr__ of empty field
---
 python/pyarrow/schema.pyx           | 3 +++
 python/pyarrow/tests/test_schema.py | 7 +++++++
 2 files changed, 10 insertions(+)

diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 19910aba00427..d636b5a10bb58 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -111,6 +111,9 @@ cdef class Field:
     property name:
 
         def __get__(self):
+            if box_field(self.sp_field) is None:
+                raise ReferenceError(
+                    'Field not initialized (references NULL pointer)')
             return frombytes(self.field.name)
 
 
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index f6dc33c75dfb8..dd68f396a6888 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -87,3 +87,10 @@ def test_schema_equals(self):
         del fields[-1]
         sch3 = A.schema(fields)
         assert not sch1.equals(sch3)
+
+
+class TestField(unittest.TestCase):
+    def test_empty_field(self):
+        f = arrow.Field()
+        with self.assertRaises(ReferenceError):
+            repr(f)

From 00df40ceab48a97fb9f1404ca6a0049e88d0c461 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 13 Mar 2017 16:15:50 -0400
Subject: [PATCH 0362/1644] ARROW-618: [Python/C++] Support timestamp+timezone
 conversion to pandas

This was a massive pain. This patch brings us up to feature parity with the stuff that was in Feather. The diff is larger than I would like mostly from moving around code in `pyarrow/adapters/pandas.cc`. I suggest we split up that file at our earliest opportunity into the "reader" and "writer" portion at least.

The main work here was refactoring so that the data type for non-object arrays is computed up front (so it might be `timestamp('ns', tz='US/Eastern')`, then we use the visitor pattern to produce the right kind of array. This will also permit implicit type casts and conversions to integer from float because the type metadata is an input parameter.

Things are getting to be a bit of a mess here so we should do some refactoring eventually, and probably also add some microbenchmarks since this stuff is performance sensitive.

I also changed the C++ `pyarrow` namespace to `arrow::py` which will make it less painful to move that code tree to `cpp/src/arrow/python` at some point

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #375 from wesm/ARROW-618 and squashes the following commits:

4b18bfa [Wes McKinney] Fix rebase conflict
5bc3724 [Wes McKinney] Fix rebase issues
870986f [Wes McKinney] Refactor ArrowSerializer to not be a template and use visitor pattern using passed-in data type. Fix DatetimeTZDtype pandas logic. Arrow Change pyarrow namespace to arrow::py
---
 cpp/src/arrow/type.cc                       |   25 +
 cpp/src/arrow/type.h                        |    2 +-
 python/pyarrow/__init__.py                  |    3 +
 python/pyarrow/array.pyx                    |   76 +-
 python/pyarrow/compat.py                    |    9 +
 python/pyarrow/config.pyx                   |    4 +-
 python/pyarrow/feather.py                   |    6 +-
 python/pyarrow/includes/libarrow.pxd        |   11 +-
 python/pyarrow/includes/pyarrow.pxd         |   19 +-
 python/pyarrow/schema.pxd                   |   10 +-
 python/pyarrow/schema.pyx                   |  160 +-
 python/pyarrow/table.pyx                    |   14 +-
 python/pyarrow/tests/test_convert_pandas.py |   33 +-
 python/pyarrow/tests/test_feather.py        |    7 +-
 python/pyarrow/tests/test_schema.py         |  142 +-
 python/src/pyarrow/adapters/builtin.cc      |   61 +-
 python/src/pyarrow/adapters/builtin.h       |   19 +-
 python/src/pyarrow/adapters/pandas-test.cc  |    6 +-
 python/src/pyarrow/adapters/pandas.cc       | 1896 +++++++++----------
 python/src/pyarrow/adapters/pandas.h        |   44 +-
 python/src/pyarrow/common.cc                |   16 +-
 python/src/pyarrow/common.h                 |   17 +-
 python/src/pyarrow/config.cc                |    6 +-
 python/src/pyarrow/config.h                 |   15 +-
 python/src/pyarrow/helpers.cc               |    8 +-
 python/src/pyarrow/helpers.h                |   15 +-
 python/src/pyarrow/io.cc                    |   12 +-
 python/src/pyarrow/io.h                     |   43 +-
 python/src/pyarrow/numpy_interop.h          |    6 +-
 python/src/pyarrow/type_traits.h            |  212 +++
 python/src/pyarrow/util/datetime.h          |    6 +-
 python/src/pyarrow/util/test_main.cc        |    2 +-
 python/src/pyarrow/visibility.h             |   32 -
 33 files changed, 1595 insertions(+), 1342 deletions(-)
 create mode 100644 python/src/pyarrow/type_traits.h
 delete mode 100644 python/src/pyarrow/visibility.h

diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 4679a2f5b76b6..0cafdce89e562 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -108,6 +108,31 @@ std::string Date32Type::ToString() const {
   return std::string("date32");
 }
 
+static inline void print_time_unit(TimeUnit unit, std::ostream* stream) {
+  switch (unit) {
+    case TimeUnit::SECOND:
+      (*stream) << "s";
+      break;
+    case TimeUnit::MILLI:
+      (*stream) << "ms";
+      break;
+    case TimeUnit::MICRO:
+      (*stream) << "us";
+      break;
+    case TimeUnit::NANO:
+      (*stream) << "ns";
+      break;
+  }
+}
+
+std::string TimestampType::ToString() const {
+  std::stringstream ss;
+  ss << "timestamp[";
+  print_time_unit(this->unit, &ss);
+  ss << "]";
+  return ss.str();
+}
+
 // ----------------------------------------------------------------------
 // Union type
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index a838082d7e79a..15b99c5ce4f89 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -495,7 +495,7 @@ struct ARROW_EXPORT TimestampType : public FixedWidthType {
   TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
 
   Status Accept(TypeVisitor* visitor) const override;
-  std::string ToString() const override { return name(); }
+  std::string ToString() const override;
   static std::string name() { return "timestamp"; }
 
   TimeUnit unit;
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 6724b52e6004e..a4aac443fae82 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -56,6 +56,8 @@
                             FloatValue, DoubleValue, ListValue,
                             BinaryValue, StringValue)
 
+import pyarrow.schema as _schema
+
 from pyarrow.schema import (null, bool_,
                             int8, int16, int32, int64,
                             uint8, uint16, uint32, uint64,
@@ -64,6 +66,7 @@
                             list_, struct, dictionary, field,
                             DataType, Field, Schema, schema)
 
+
 from pyarrow.table import Column, RecordBatch, Table, concat_tables
 
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 6a6b4ba9ad0cb..11244e7836058 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -34,7 +34,8 @@ from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
 cimport pyarrow.scalar as scalar
 from pyarrow.scalar import NA
 
-from pyarrow.schema cimport Field, Schema, DictionaryType
+from pyarrow.schema cimport (DataType, Field, Schema, DictionaryType,
+                             box_data_type)
 import pyarrow.schema as schema
 
 cimport cpython
@@ -45,16 +46,40 @@ cdef _pandas():
     return pd
 
 
+cdef maybe_coerce_datetime64(values, dtype, DataType type,
+                             timestamps_to_ms=False):
+
+    from pyarrow.compat import DatetimeTZDtype
+
+    if values.dtype.type != np.datetime64:
+        return values, type
+
+    coerce_ms = timestamps_to_ms and values.dtype != 'datetime64[ms]'
+
+    if coerce_ms:
+        values = values.astype('datetime64[ms]')
+
+    if isinstance(dtype, DatetimeTZDtype):
+        tz = dtype.tz
+        unit = 'ms' if coerce_ms else dtype.unit
+        type = schema.timestamp(unit, tz)
+    else:
+        # Trust the NumPy dtype
+        type = schema.type_from_numpy_dtype(values.dtype)
+
+    return values, type
+
+
 cdef class Array:
 
     cdef init(self, const shared_ptr[CArray]& sp_array):
         self.sp_array = sp_array
         self.ap = sp_array.get()
-        self.type = DataType()
-        self.type.init(self.sp_array.get().type())
+        self.type = box_data_type(self.sp_array.get().type())
 
     @staticmethod
-    def from_pandas(obj, mask=None, timestamps_to_ms=False, Field field=None,
+    def from_pandas(obj, mask=None, DataType type=None,
+                    timestamps_to_ms=False,
                     MemoryPool memory_pool=None):
         """
         Convert pandas.Series to an Arrow Array.
@@ -66,6 +91,9 @@ cdef class Array:
         mask : pandas.Series or numpy.ndarray, optional
             boolean mask if the object is valid or null
 
+        type : pyarrow.DataType
+            Explicit type to attempt to coerce to
+
         timestamps_to_ms : bool, optional
             Convert datetime columns to ms resolution. This is needed for
             compatibility with other functionality like Parquet I/O which
@@ -107,33 +135,43 @@ cdef class Array:
         """
         cdef:
             shared_ptr[CArray] out
-            shared_ptr[CField] c_field
+            shared_ptr[CDataType] c_type
             CMemoryPool* pool
 
         pd = _pandas()
 
-        if field is not None:
-            c_field = field.sp_field
-
         if mask is not None:
             mask = get_series_values(mask)
 
-        series_values = get_series_values(obj)
+        values = get_series_values(obj)
+        pool = maybe_unbox_memory_pool(memory_pool)
 
-        if isinstance(series_values, pd.Categorical):
+        if isinstance(values, pd.Categorical):
             return DictionaryArray.from_arrays(
-                series_values.codes, series_values.categories.values,
+                values.codes, values.categories.values,
                 mask=mask, memory_pool=memory_pool)
+        elif values.dtype == object:
+            # Object dtype undergoes a different conversion path as more type
+            # inference may be needed
+            if type is not None:
+                c_type = type.sp_type
+            with nogil:
+                check_status(pyarrow.PandasObjectsToArrow(
+                    pool, values, mask, c_type, &out))
         else:
-            if series_values.dtype.type == np.datetime64 and timestamps_to_ms:
-                series_values = series_values.astype('datetime64[ms]')
+            values, type = maybe_coerce_datetime64(
+                values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
+
+            if type is None:
+                check_status(pyarrow.PandasDtypeToArrow(values.dtype, &c_type))
+            else:
+                c_type = type.sp_type
 
-            pool = maybe_unbox_memory_pool(memory_pool)
             with nogil:
                 check_status(pyarrow.PandasToArrow(
-                    pool, series_values, mask, c_field, &out))
+                    pool, values, mask, c_type, &out))
 
-            return box_array(out)
+        return box_array(out)
 
     @staticmethod
     def from_list(object list_obj, DataType type=None,
@@ -338,6 +376,10 @@ cdef class DateArray(NumericArray):
     pass
 
 
+cdef class TimestampArray(NumericArray):
+    pass
+
+
 cdef class FloatArray(FloatingPointArray):
     pass
 
@@ -423,7 +465,7 @@ cdef dict _array_classes = {
     Type_LIST: ListArray,
     Type_BINARY: BinaryArray,
     Type_STRING: StringArray,
-    Type_TIMESTAMP: Int64Array,
+    Type_TIMESTAMP: TimestampArray,
     Type_DICTIONARY: DictionaryArray
 }
 
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 9148be7d9f8ad..74d7ca2827bc9 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -17,9 +17,11 @@
 
 # flake8: noqa
 
+from distutils.version import LooseVersion
 import itertools
 
 import numpy as np
+import pandas as pd
 
 import sys
 import six
@@ -115,6 +117,13 @@ def encode_file_path(path):
     return encoded_path
 
 
+if LooseVersion(pd.__version__) < '0.19.0':
+    pdapi = pd.core.common
+    from pandas.core.dtypes import DatetimeTZDtype
+else:
+    from pandas.types.dtypes import DatetimeTZDtype
+    pdapi = pd.api.types
+
 integer_types = six.integer_types + (np.integer,)
 
 __all__ = []
diff --git a/python/pyarrow/config.pyx b/python/pyarrow/config.pyx
index aa30f097248cd..5ad7cf53261e3 100644
--- a/python/pyarrow/config.pyx
+++ b/python/pyarrow/config.pyx
@@ -17,10 +17,10 @@
 cdef extern from 'pyarrow/do_import_numpy.h':
     pass
 
-cdef extern from 'pyarrow/numpy_interop.h' namespace 'pyarrow':
+cdef extern from 'pyarrow/numpy_interop.h' namespace 'arrow::py':
     int import_numpy()
 
-cdef extern from 'pyarrow/config.h' namespace 'pyarrow':
+cdef extern from 'pyarrow/config.h' namespace 'arrow::py':
     void pyarrow_init()
     void pyarrow_set_numpy_nan(object o)
 
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index b7dbf96563a41..28424afb093b5 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -19,6 +19,7 @@
 from distutils.version import LooseVersion
 import pandas as pd
 
+from pyarrow.compat import pdapi
 from pyarrow._feather import FeatherError  # noqa
 from pyarrow.table import Table
 import pyarrow._feather as ext
@@ -27,11 +28,6 @@
 if LooseVersion(pd.__version__) < '0.17.0':
     raise ImportError("feather requires pandas >= 0.17.0")
 
-if LooseVersion(pd.__version__) < '0.19.0':
-    pdapi = pd.core.common
-else:
-    pdapi = pd.api.types
-
 
 class FeatherReader(ext.FeatherReader):
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 253cabbe0a581..dee7fd4f8e4e5 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -84,6 +84,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CArray] indices()
         shared_ptr[CArray] dictionary()
 
+    cdef cppclass CTimestampType" arrow::TimestampType"(CFixedWidthType):
+        TimeUnit unit
+        c_string timezone
+
+    cdef cppclass CTimeType" arrow::TimeType"(CFixedWidthType):
+        TimeUnit unit
+
     cdef cppclass CDictionaryType" arrow::DictionaryType"(CFixedWidthType):
         CDictionaryType(const shared_ptr[CDataType]& index_type,
                         const shared_ptr[CArray]& dictionary)
@@ -92,6 +99,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CArray] dictionary()
 
     shared_ptr[CDataType] timestamp(TimeUnit unit)
+    shared_ptr[CDataType] timestamp(const c_string& timezone, TimeUnit unit)
 
     cdef cppclass CMemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
@@ -117,9 +125,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
 
-    cdef cppclass CTimestampType" arrow::TimestampType"(CDataType):
-        TimeUnit unit
-
     cdef cppclass CField" arrow::Field":
         c_string name
         shared_ptr[CDataType] type
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index f1d45e0d50f36..9fbddba3d10c5 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -18,22 +18,29 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CField,
+from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn,
                                         CTable, CDataType, CStatus, Type,
                                         CMemoryPool, TimeUnit)
 
 cimport pyarrow.includes.libarrow_io as arrow_io
 
 
-cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
+cdef extern from "pyarrow/api.h" namespace "arrow::py" nogil:
     shared_ptr[CDataType] GetPrimitiveType(Type type)
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
-    CStatus ConvertPySequence(object obj, CMemoryPool* pool, shared_ptr[CArray]* out)
+    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
+                              shared_ptr[CArray]* out)
+
+    CStatus PandasDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
 
     CStatus PandasToArrow(CMemoryPool* pool, object ao, object mo,
-                          shared_ptr[CField] field,
+                          const shared_ptr[CDataType]& type,
                           shared_ptr[CArray]* out)
 
+    CStatus PandasObjectsToArrow(CMemoryPool* pool, object ao, object mo,
+                                 const shared_ptr[CDataType]& type,
+                                 shared_ptr[CArray]* out)
+
     CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
                                  PyObject* py_ref, PyObject** out)
 
@@ -47,12 +54,12 @@ cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil:
     CMemoryPool* get_memory_pool()
 
 
-cdef extern from "pyarrow/common.h" namespace "pyarrow" nogil:
+cdef extern from "pyarrow/common.h" namespace "arrow::py" nogil:
     cdef cppclass PyBytesBuffer(CBuffer):
         PyBytesBuffer(object o)
 
 
-cdef extern from "pyarrow/io.h" namespace "pyarrow" nogil:
+cdef extern from "pyarrow/io.h" namespace "arrow::py" nogil:
     cdef cppclass PyReadableFile(arrow_io.ReadableFileInterface):
         PyReadableFile(object fo)
 
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
index 390954cfc6bd9..15ee5f19ee5d9 100644
--- a/python/pyarrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -16,7 +16,9 @@
 # under the License.
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CDataType, CDictionaryType,
+from pyarrow.includes.libarrow cimport (CDataType,
+                                        CDictionaryType,
+                                        CTimestampType,
                                         CField, CSchema)
 
 cdef class DataType:
@@ -31,6 +33,12 @@ cdef class DictionaryType(DataType):
     cdef:
         const CDictionaryType* dict_type
 
+
+cdef class TimestampType(DataType):
+    cdef:
+        const CTimestampType* ts_type
+
+
 cdef class Field:
     cdef:
         shared_ptr[CField] sp_field
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index d636b5a10bb58..4bc938df668f8 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -26,23 +26,19 @@ from cython.operator cimport dereference as deref
 
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.array cimport Array
+from pyarrow.error cimport check_status
 from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
-                                        Type_NA, Type_BOOL,
-                                        Type_UINT8, Type_INT8,
-                                        Type_UINT16, Type_INT16,
-                                        Type_UINT32, Type_INT32,
-                                        Type_UINT64, Type_INT64,
-                                        Type_TIMESTAMP, Type_DATE,
-                                        Type_FLOAT, Type_DOUBLE,
-                                        Type_STRING, Type_BINARY,
                                         TimeUnit_SECOND, TimeUnit_MILLI,
                                         TimeUnit_MICRO, TimeUnit_NANO,
                                         Type, TimeUnit)
 cimport pyarrow.includes.pyarrow as pyarrow
-cimport pyarrow.includes.libarrow as libarrow
+cimport pyarrow.includes.libarrow as la
 
 cimport cpython
 
+import six
+
+
 cdef class DataType:
 
     def __cinit__(self):
@@ -73,13 +69,33 @@ cdef class DictionaryType(DataType):
         DataType.init(self, type)
         self.dict_type = <const CDictionaryType*> type.get()
 
-    def __str__(self):
-        return frombytes(self.type.ToString())
-
     def __repr__(self):
         return 'DictionaryType({0})'.format(str(self))
 
 
+cdef class TimestampType(DataType):
+
+    cdef init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.ts_type = <const CTimestampType*> type.get()
+
+    property unit:
+
+        def __get__(self):
+            return timeunit_to_string(self.ts_type.unit)
+
+    property tz:
+
+        def __get__(self):
+            if self.ts_type.timezone.size() > 0:
+                return frombytes(self.ts_type.timezone)
+            else:
+                return None
+
+    def __repr__(self):
+        return 'TimestampType({0})'.format(str(self))
+
+
 cdef class Field:
 
     def __cinit__(self):
@@ -205,49 +221,76 @@ cdef DataType primitive_type(Type type):
 def field(name, type, bint nullable=True):
     return Field.from_py(name, type, nullable)
 
+
 cdef set PRIMITIVE_TYPES = set([
-    Type_NA, Type_BOOL,
-    Type_UINT8, Type_INT8,
-    Type_UINT16, Type_INT16,
-    Type_UINT32, Type_INT32,
-    Type_UINT64, Type_INT64,
-    Type_TIMESTAMP, Type_DATE,
-    Type_FLOAT, Type_DOUBLE])
+    la.Type_NA, la.Type_BOOL,
+    la.Type_UINT8, la.Type_INT8,
+    la.Type_UINT16, la.Type_INT16,
+    la.Type_UINT32, la.Type_INT32,
+    la.Type_UINT64, la.Type_INT64,
+    la.Type_TIMESTAMP, la.Type_DATE,
+    la.Type_FLOAT, la.Type_DOUBLE])
+
 
 def null():
-    return primitive_type(Type_NA)
+    return primitive_type(la.Type_NA)
+
 
 def bool_():
-    return primitive_type(Type_BOOL)
+    return primitive_type(la.Type_BOOL)
+
 
 def uint8():
-    return primitive_type(Type_UINT8)
+    return primitive_type(la.Type_UINT8)
+
 
 def int8():
-    return primitive_type(Type_INT8)
+    return primitive_type(la.Type_INT8)
+
 
 def uint16():
-    return primitive_type(Type_UINT16)
+    return primitive_type(la.Type_UINT16)
+
 
 def int16():
-    return primitive_type(Type_INT16)
+    return primitive_type(la.Type_INT16)
+
 
 def uint32():
-    return primitive_type(Type_UINT32)
+    return primitive_type(la.Type_UINT32)
+
 
 def int32():
-    return primitive_type(Type_INT32)
+    return primitive_type(la.Type_INT32)
+
 
 def uint64():
-    return primitive_type(Type_UINT64)
+    return primitive_type(la.Type_UINT64)
+
 
 def int64():
-    return primitive_type(Type_INT64)
+    return primitive_type(la.Type_INT64)
+
 
 cdef dict _timestamp_type_cache = {}
 
-def timestamp(unit_str):
-    cdef TimeUnit unit
+
+cdef timeunit_to_string(TimeUnit unit):
+    if unit == TimeUnit_SECOND:
+        return 's'
+    elif unit == TimeUnit_MILLI:
+        return 'ms'
+    elif unit == TimeUnit_MICRO:
+        return 'us'
+    elif unit == TimeUnit_NANO:
+        return 'ns'
+
+
+def timestamp(unit_str, tz=None):
+    cdef:
+        TimeUnit unit
+        c_string c_timezone
+
     if unit_str == "s":
         unit = TimeUnit_SECOND
     elif unit_str == 'ms':
@@ -259,34 +302,47 @@ def timestamp(unit_str):
     else:
         raise TypeError('Invalid TimeUnit string')
 
-    if unit in _timestamp_type_cache:
-        return _timestamp_type_cache[unit]
+    cdef TimestampType out = TimestampType()
+
+    if tz is None:
+        out.init(la.timestamp(unit))
+        if unit in _timestamp_type_cache:
+            return _timestamp_type_cache[unit]
+        _timestamp_type_cache[unit] = out
+    else:
+        if not isinstance(tz, six.string_types):
+            tz = tz.zone
+
+        c_timezone = tobytes(tz)
+        out.init(la.timestamp(c_timezone, unit))
 
-    cdef DataType out = DataType()
-    out.init(libarrow.timestamp(unit))
-    _timestamp_type_cache[unit] = out
     return out
 
+
 def date():
-    return primitive_type(Type_DATE)
+    return primitive_type(la.Type_DATE)
+
 
 def float_():
-    return primitive_type(Type_FLOAT)
+    return primitive_type(la.Type_FLOAT)
+
 
 def double():
-    return primitive_type(Type_DOUBLE)
+    return primitive_type(la.Type_DOUBLE)
+
 
 def string():
     """
     UTF8 string
     """
-    return primitive_type(Type_STRING)
+    return primitive_type(la.Type_STRING)
+
 
 def binary():
     """
     Binary (PyBytes-like) type
     """
-    return primitive_type(Type_BINARY)
+    return primitive_type(la.Type_BINARY)
 
 
 def list_(DataType value_type):
@@ -326,13 +382,25 @@ def struct(fields):
     out.init(struct_type)
     return out
 
+
 def schema(fields):
     return Schema.from_fields(fields)
 
+
 cdef DataType box_data_type(const shared_ptr[CDataType]& type):
+    cdef:
+        DataType out
+
     if type.get() == NULL:
         return None
-    cdef DataType out = DataType()
+
+    if type.get().type == la.Type_DICTIONARY:
+        out = DictionaryType()
+    elif type.get().type == la.Type_TIMESTAMP:
+        out = TimestampType()
+    else:
+        out = DataType()
+
     out.init(type)
     return out
 
@@ -347,3 +415,11 @@ cdef Schema box_schema(const shared_ptr[CSchema]& type):
     cdef Schema out = Schema()
     out.init_schema(type)
     return out
+
+
+def type_from_numpy_dtype(object dtype):
+    cdef shared_ptr[CDataType] c_type
+    with nogil:
+        check_status(pyarrow.PandasDtypeToArrow(dtype, &c_type))
+
+    return box_data_type(c_type)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 5657b973d1306..58f5d680393f7 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -30,7 +30,7 @@ import pyarrow.config
 from pyarrow.array cimport Array, box_array, wrap_array_output
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
-from pyarrow.schema cimport box_data_type, box_schema, Field
+from pyarrow.schema cimport box_data_type, box_schema, DataType
 
 from pyarrow.compat import frombytes, tobytes
 
@@ -302,14 +302,15 @@ cdef _dataframe_to_arrays(df, name, timestamps_to_ms, Schema schema):
     cdef:
         list names = []
         list arrays = []
-        Field field = None
+        DataType type = None
 
     for name in df.columns:
         col = df[name]
         if schema is not None:
-            field = schema.field_by_name(name)
-        arr = Array.from_pandas(col, timestamps_to_ms=timestamps_to_ms,
-                                field=field)
+            type = schema.field_by_name(name).type
+
+        arr = Array.from_pandas(col, type=type,
+                                timestamps_to_ms=timestamps_to_ms)
         names.append(name)
         arrays.append(arr)
 
@@ -522,6 +523,7 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
 
     import pandas.core.internals as _int
     from pandas import RangeIndex, Categorical
+    from pyarrow.compat import DatetimeTZDtype
 
     with nogil:
         check_status(pyarrow.ConvertTableToPandas(table, nthreads,
@@ -541,9 +543,9 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
                                     klass=_int.CategoricalBlock,
                                     fastpath=True)
         elif 'timezone' in item:
-            from pandas.types.api import DatetimeTZDtype
             dtype = DatetimeTZDtype('ns', tz=item['timezone'])
             block = _int.make_block(block_arr, placement=placement,
+                                    klass=_int.DatetimeTZBlock,
                                     dtype=dtype, fastpath=True)
         else:
             block = _int.make_block(block_arr, placement=placement)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index a79bb2392ea6c..6b89444b3e824 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -77,9 +77,9 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
         tm.assert_frame_equal(result, expected, check_dtype=check_dtype)
 
     def _check_array_roundtrip(self, values, expected=None,
-                               timestamps_to_ms=False, field=None):
+                               timestamps_to_ms=False, type=None):
         arr = A.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
-                                  field=field)
+                                  type=type)
         result = arr.to_pandas()
 
         assert arr.null_count == pd.isnull(values).sum()
@@ -134,11 +134,13 @@ def test_integer_no_nulls(self):
         data = OrderedDict()
         fields = []
 
-        numpy_dtypes = [('i1', A.int8()), ('i2', A.int16()),
-                        ('i4', A.int32()), ('i8', A.int64()),
-                        ('u1', A.uint8()), ('u2', A.uint16()),
-                        ('u4', A.uint32()), ('u8', A.uint64()),
-                        ('longlong', A.int64()), ('ulonglong', A.uint64())]
+        numpy_dtypes = [
+            ('i1', A.int8()), ('i2', A.int16()),
+            ('i4', A.int32()), ('i8', A.int64()),
+            ('u1', A.uint8()), ('u2', A.uint16()),
+            ('u4', A.uint32()), ('u8', A.uint64()),
+            ('longlong', A.int64()), ('ulonglong', A.uint64())
+        ]
         num_values = 100
 
         for dtype, arrow_dtype in numpy_dtypes:
@@ -153,7 +155,6 @@ def test_integer_no_nulls(self):
         schema = A.Schema.from_fields(fields)
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
-
     def test_integer_with_nulls(self):
         # pandas requires upcast to float dtype
 
@@ -301,9 +302,9 @@ def test_timestamps_with_timezone(self):
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
             })
-        df_est = df['datetime64'].dt.tz_localize('US/Eastern').to_frame()
-        df_utc = df_est['datetime64'].dt.tz_convert('UTC').to_frame()
-        self._check_pandas_roundtrip(df_est, expected=df_utc, timestamps_to_ms=True, check_dtype=False)
+        df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
+                            .to_frame())
+        self._check_pandas_roundtrip(df, timestamps_to_ms=True)
 
         # drop-in a null and ns instead of ms
         df = pd.DataFrame({
@@ -314,9 +315,9 @@ def test_timestamps_with_timezone(self):
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
             })
-        df_est = df['datetime64'].dt.tz_localize('US/Eastern').to_frame()
-        df_utc = df_est['datetime64'].dt.tz_convert('UTC').to_frame()
-        self._check_pandas_roundtrip(df_est, expected=df_utc, timestamps_to_ms=False, check_dtype=False)
+        df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
+                            .to_frame())
+        self._check_pandas_roundtrip(df, timestamps_to_ms=False)
 
     def test_date(self):
         df = pd.DataFrame({
@@ -341,7 +342,7 @@ def test_column_of_arrays(self):
 
         for column in df.columns:
             field = schema.field_by_name(column)
-            self._check_array_roundtrip(df[column], field=field)
+            self._check_array_roundtrip(df[column], type=field.type)
 
     def test_column_of_lists(self):
         df, schema = dataframe_with_lists()
@@ -351,7 +352,7 @@ def test_column_of_lists(self):
 
         for column in df.columns:
             field = schema.field_by_name(column)
-            self._check_array_roundtrip(df[column], field=field)
+            self._check_array_roundtrip(df[column], type=field.type)
 
     def test_threaded_conversion(self):
         df = _alltypes_example()
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 451475b4c6d81..e4b6273ffccf4 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -23,8 +23,8 @@
 from pandas.util.testing import assert_frame_equal
 import pandas as pd
 
+import pyarrow as pa
 from pyarrow.compat import guid
-from pyarrow.error import ArrowException
 from pyarrow.feather import (read_feather, write_feather,
                              FeatherReader)
 from pyarrow._feather import FeatherWriter
@@ -47,7 +47,7 @@ def tearDown(self):
                 pass
 
     def test_file_not_exist(self):
-        with self.assertRaises(ArrowException):
+        with self.assertRaises(pa.ArrowException):
             FeatherReader('test_invalid_file')
 
     def _get_null_counts(self, path, columns=None):
@@ -291,7 +291,6 @@ def test_category(self):
         self._check_pandas_roundtrip(df, expected,
                                      null_counts=[2 * repeats])
 
-    @pytest.mark.xfail
     def test_timestamp(self):
         df = pd.DataFrame({'naive': pd.date_range('2016-03-28', periods=10)})
         df['with_tz'] = (df.naive.dt.tz_localize('utc')
@@ -299,7 +298,6 @@ def test_timestamp(self):
 
         self._check_pandas_roundtrip(df)
 
-    @pytest.mark.xfail
     def test_timestamp_with_nulls(self):
         df = pd.DataFrame({'test': [pd.datetime(2016, 1, 1),
                                     None,
@@ -308,7 +306,6 @@ def test_timestamp_with_nulls(self):
 
         self._check_pandas_roundtrip(df, null_counts=[1, 1])
 
-    @pytest.mark.xfail
     def test_out_of_float64_timestamp_with_nulls(self):
         df = pd.DataFrame(
             {'test': pd.DatetimeIndex([1451606400000000001,
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index dd68f396a6888..5588840cceb1f 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -15,82 +15,108 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import unittest
-import pyarrow as arrow
+import pytest
 
-A = arrow
+import pyarrow as pa
 
+import numpy as np
 
-class TestTypes(unittest.TestCase):
+# XXX: pyarrow.schema.schema masks the module on imports
+sch = pa._schema
 
-    def test_integers(self):
-        dtypes = ['int8', 'int16', 'int32', 'int64',
-                  'uint8', 'uint16', 'uint32', 'uint64']
 
-        for name in dtypes:
-            factory = getattr(arrow, name)
-            t = factory()
-            assert str(t) == name
+def test_type_integers():
+    dtypes = ['int8', 'int16', 'int32', 'int64',
+              'uint8', 'uint16', 'uint32', 'uint64']
 
-    def test_list(self):
-        value_type = arrow.int32()
-        list_type = arrow.list_(value_type)
-        assert str(list_type) == 'list<item: int32>'
+    for name in dtypes:
+        factory = getattr(pa, name)
+        t = factory()
+        assert str(t) == name
 
-    def test_string(self):
-        t = arrow.string()
-        assert str(t) == 'string'
 
-    def test_field(self):
-        t = arrow.string()
-        f = arrow.field('foo', t)
+def test_type_list():
+    value_type = pa.int32()
+    list_type = pa.list_(value_type)
+    assert str(list_type) == 'list<item: int32>'
 
-        assert f.name == 'foo'
-        assert f.nullable
-        assert f.type is t
-        assert repr(f) == "Field('foo', type=string)"
 
-        f = arrow.field('foo', t, False)
-        assert not f.nullable
+def test_type_string():
+    t = pa.string()
+    assert str(t) == 'string'
 
-    def test_schema(self):
-        fields = [
-            A.field('foo', A.int32()),
-            A.field('bar', A.string()),
-            A.field('baz', A.list_(A.int8()))
-        ]
-        sch = A.schema(fields)
 
-        assert len(sch) == 3
-        assert sch[0].name == 'foo'
-        assert sch[0].type == fields[0].type
-        assert sch.field_by_name('foo').name == 'foo'
-        assert sch.field_by_name('foo').type == fields[0].type
+def test_type_timestamp_with_tz():
+    tz = 'America/Los_Angeles'
+    t = pa.timestamp('ns', tz=tz)
+    assert t.unit == 'ns'
+    assert t.tz == tz
 
-        assert repr(sch) == """\
+
+def test_type_from_numpy_dtype_timestamps():
+    cases = [
+        (np.dtype('datetime64[s]'), pa.timestamp('s')),
+        (np.dtype('datetime64[ms]'), pa.timestamp('ms')),
+        (np.dtype('datetime64[us]'), pa.timestamp('us')),
+        (np.dtype('datetime64[ns]'), pa.timestamp('ns'))
+    ]
+
+    for dt, pt in cases:
+        result = sch.type_from_numpy_dtype(dt)
+        assert result == pt
+
+
+def test_field():
+    t = pa.string()
+    f = pa.field('foo', t)
+
+    assert f.name == 'foo'
+    assert f.nullable
+    assert f.type is t
+    assert repr(f) == "Field('foo', type=string)"
+
+    f = pa.field('foo', t, False)
+    assert not f.nullable
+
+
+def test_schema():
+    fields = [
+        pa.field('foo', pa.int32()),
+        pa.field('bar', pa.string()),
+        pa.field('baz', pa.list_(pa.int8()))
+    ]
+    sch = pa.schema(fields)
+
+    assert len(sch) == 3
+    assert sch[0].name == 'foo'
+    assert sch[0].type == fields[0].type
+    assert sch.field_by_name('foo').name == 'foo'
+    assert sch.field_by_name('foo').type == fields[0].type
+
+    assert repr(sch) == """\
 foo: int32
 bar: string
 baz: list<item: int8>"""
 
-    def test_schema_equals(self):
-        fields = [
-            A.field('foo', A.int32()),
-            A.field('bar', A.string()),
-            A.field('baz', A.list_(A.int8()))
-        ]
 
-        sch1 = A.schema(fields)
-        print(dir(sch1))
-        sch2 = A.schema(fields)
-        assert sch1.equals(sch2)
+def test_field_empty():
+    f = pa.Field()
+    with pytest.raises(ReferenceError):
+        repr(f)
+
 
-        del fields[-1]
-        sch3 = A.schema(fields)
-        assert not sch1.equals(sch3)
+def test_schema_equals():
+    fields = [
+        pa.field('foo', pa.int32()),
+        pa.field('bar', pa.string()),
+        pa.field('baz', pa.list_(pa.int8()))
+    ]
 
+    sch1 = pa.schema(fields)
+    print(dir(sch1))
+    sch2 = pa.schema(fields)
+    assert sch1.equals(sch2)
 
-class TestField(unittest.TestCase):
-    def test_empty_field(self):
-        f = arrow.Field()
-        with self.assertRaises(ReferenceError):
-            repr(f)
+    del fields[-1]
+    sch3 = pa.schema(fields)
+    assert not sch1.equals(sch3)
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index 4f7b2cb09e1e6..b197f5845c020 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -27,13 +27,8 @@
 #include "pyarrow/helpers.h"
 #include "pyarrow/util/datetime.h"
 
-using arrow::ArrayBuilder;
-using arrow::DataType;
-using arrow::MemoryPool;
-using arrow::Status;
-using arrow::Type;
-
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 static inline bool IsPyInteger(PyObject* obj) {
 #if PYARROW_IS_PY2
@@ -82,22 +77,22 @@ class ScalarVisitor {
   std::shared_ptr<DataType> GetType() {
     // TODO(wesm): handling mixed-type cases
     if (float_count_) {
-      return arrow::float64();
+      return float64();
     } else if (int_count_) {
       // TODO(wesm): tighter type later
-      return arrow::int64();
+      return int64();
     } else if (date_count_) {
-      return arrow::date();
+      return date();
     } else if (timestamp_count_) {
-      return arrow::timestamp(arrow::TimeUnit::MICRO);
+      return timestamp(TimeUnit::MICRO);
     } else if (bool_count_) {
-      return arrow::boolean();
+      return boolean();
     } else if (binary_count_) {
-      return arrow::binary();
+      return binary();
     } else if (unicode_count_) {
-      return arrow::utf8();
+      return utf8();
     } else {
-      return arrow::null();
+      return null();
     }
   }
 
@@ -157,14 +152,14 @@ class SeqVisitor {
   std::shared_ptr<DataType> GetType() {
     if (scalars_.total_count() == 0) {
       if (max_nesting_level_ == 0) {
-        return arrow::null();
+        return null();
       } else {
         return nullptr;
       }
     } else {
       std::shared_ptr<DataType> result = scalars_.GetType();
       for (int i = 0; i < max_nesting_level_; ++i) {
-        result = std::make_shared<arrow::ListType>(result);
+        result = std::make_shared<ListType>(result);
       }
       return result;
     }
@@ -215,7 +210,7 @@ Status InferArrowType(PyObject* obj, int64_t* size, std::shared_ptr<DataType>* o
   }
 
   // For 0-length sequences, refuse to guess
-  if (*size == 0) { *out_type = arrow::null(); }
+  if (*size == 0) { *out_type = null(); }
 
   SeqVisitor seq_visitor;
   RETURN_NOT_OK(seq_visitor.Visit(obj));
@@ -255,7 +250,7 @@ class TypedConverter : public SeqConverter {
   BuilderType* typed_builder_;
 };
 
-class BoolConverter : public TypedConverter<arrow::BooleanBuilder> {
+class BoolConverter : public TypedConverter<BooleanBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     Py_ssize_t size = PySequence_Size(seq);
@@ -276,7 +271,7 @@ class BoolConverter : public TypedConverter<arrow::BooleanBuilder> {
   }
 };
 
-class Int64Converter : public TypedConverter<arrow::Int64Builder> {
+class Int64Converter : public TypedConverter<Int64Builder> {
  public:
   Status AppendData(PyObject* seq) override {
     int64_t val;
@@ -296,7 +291,7 @@ class Int64Converter : public TypedConverter<arrow::Int64Builder> {
   }
 };
 
-class DateConverter : public TypedConverter<arrow::DateBuilder> {
+class DateConverter : public TypedConverter<DateBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     Py_ssize_t size = PySequence_Size(seq);
@@ -314,7 +309,7 @@ class DateConverter : public TypedConverter<arrow::DateBuilder> {
   }
 };
 
-class TimestampConverter : public TypedConverter<arrow::TimestampBuilder> {
+class TimestampConverter : public TypedConverter<TimestampBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     Py_ssize_t size = PySequence_Size(seq);
@@ -347,7 +342,7 @@ class TimestampConverter : public TypedConverter<arrow::TimestampBuilder> {
   }
 };
 
-class DoubleConverter : public TypedConverter<arrow::DoubleBuilder> {
+class DoubleConverter : public TypedConverter<DoubleBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     double val;
@@ -367,7 +362,7 @@ class DoubleConverter : public TypedConverter<arrow::DoubleBuilder> {
   }
 };
 
-class BytesConverter : public TypedConverter<arrow::BinaryBuilder> {
+class BytesConverter : public TypedConverter<BinaryBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     PyObject* item;
@@ -401,7 +396,7 @@ class BytesConverter : public TypedConverter<arrow::BinaryBuilder> {
   }
 };
 
-class UTF8Converter : public TypedConverter<arrow::StringBuilder> {
+class UTF8Converter : public TypedConverter<StringBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     PyObject* item;
@@ -433,7 +428,7 @@ class UTF8Converter : public TypedConverter<arrow::StringBuilder> {
   }
 };
 
-class ListConverter : public TypedConverter<arrow::ListBuilder> {
+class ListConverter : public TypedConverter<ListBuilder> {
  public:
   Status Init(const std::shared_ptr<ArrayBuilder>& builder) override;
 
@@ -483,10 +478,10 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
 
 Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
   builder_ = builder;
-  typed_builder_ = static_cast<arrow::ListBuilder*>(builder.get());
+  typed_builder_ = static_cast<ListBuilder*>(builder.get());
 
   value_converter_ =
-      GetConverter(static_cast<arrow::ListType*>(builder->type().get())->value_type());
+      GetConverter(static_cast<ListType*>(builder->type().get())->value_type());
   if (value_converter_ == nullptr) {
     return Status::NotImplemented("value type not implemented");
   }
@@ -508,8 +503,7 @@ Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
   return converter->AppendData(obj);
 }
 
-Status ConvertPySequence(
-    PyObject* obj, MemoryPool* pool, std::shared_ptr<arrow::Array>* out) {
+Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out) {
   std::shared_ptr<DataType> type;
   int64_t size;
   PyDateTime_IMPORT;
@@ -517,16 +511,17 @@ Status ConvertPySequence(
 
   // Handle NA / NullType case
   if (type->type == Type::NA) {
-    out->reset(new arrow::NullArray(size));
+    out->reset(new NullArray(size));
     return Status::OK();
   }
 
   // Give the sequence converter an array builder
   std::shared_ptr<ArrayBuilder> builder;
-  RETURN_NOT_OK(arrow::MakeBuilder(pool, type, &builder));
+  RETURN_NOT_OK(MakeBuilder(pool, type, &builder));
   RETURN_NOT_OK(AppendPySequence(obj, type, builder));
 
   return builder->Finish(out);
 }
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
index 0c863a5631ada..2d45e670628b5 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -27,27 +27,28 @@
 
 #include <arrow/type.h>
 
+#include "arrow/util/visibility.h"
+
 #include "pyarrow/common.h"
-#include "pyarrow/visibility.h"
 
 namespace arrow {
+
 class Array;
 class Status;
-}
 
-namespace pyarrow {
+namespace py {
 
-PYARROW_EXPORT arrow::Status InferArrowType(
+ARROW_EXPORT arrow::Status InferArrowType(
     PyObject* obj, int64_t* size, std::shared_ptr<arrow::DataType>* out_type);
 
-PYARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj,
+ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj,
     const std::shared_ptr<arrow::DataType>& type,
     const std::shared_ptr<arrow::ArrayBuilder>& builder);
 
-PYARROW_EXPORT
-arrow::Status ConvertPySequence(
-    PyObject* obj, arrow::MemoryPool* pool, std::shared_ptr<arrow::Array>* out);
+ARROW_EXPORT
+Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out);
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_ADAPTERS_BUILTIN_H
diff --git a/python/src/pyarrow/adapters/pandas-test.cc b/python/src/pyarrow/adapters/pandas-test.cc
index e286ccc2c8dc4..e694e790a38d1 100644
--- a/python/src/pyarrow/adapters/pandas-test.cc
+++ b/python/src/pyarrow/adapters/pandas-test.cc
@@ -30,9 +30,8 @@
 #include "arrow/type.h"
 #include "pyarrow/adapters/pandas.h"
 
-using namespace arrow;
-
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   StringBuilder builder;
@@ -61,4 +60,5 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   Py_END_ALLOW_THREADS;
 }
 
+}  // namespace py
 }  // namespace arrow
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 40079b49b9638..863cf54c9aa1c 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -19,7 +19,6 @@
 
 #include <Python.h>
 
-#include "pyarrow/adapters/builtin.h"
 #include "pyarrow/adapters/pandas.h"
 #include "pyarrow/numpy_interop.h"
 
@@ -34,120 +33,39 @@
 #include <thread>
 #include <unordered_map>
 
-#include "arrow/api.h"
+#include "arrow/array.h"
+#include "arrow/column.h"
 #include "arrow/loader.h"
 #include "arrow/status.h"
+#include "arrow/table.h"
 #include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 
+#include "pyarrow/adapters/builtin.h"
 #include "pyarrow/common.h"
 #include "pyarrow/config.h"
+#include "pyarrow/type_traits.h"
 #include "pyarrow/util/datetime.h"
 
-namespace pyarrow {
-
-using arrow::Array;
-using arrow::ChunkedArray;
-using arrow::Column;
-using arrow::DictionaryType;
-using arrow::Field;
-using arrow::DataType;
-using arrow::ListType;
-using arrow::ListBuilder;
-using arrow::Status;
-using arrow::Table;
-using arrow::Type;
-
-namespace BitUtil = arrow::BitUtil;
+namespace arrow {
+namespace py {
 
 // ----------------------------------------------------------------------
 // Utility code
 
-template <int TYPE>
-struct npy_traits {};
-
-template <>
-struct npy_traits<NPY_BOOL> {
-  typedef uint8_t value_type;
-  using TypeClass = arrow::BooleanType;
-  using BuilderClass = arrow::BooleanBuilder;
-
-  static constexpr bool supports_nulls = false;
-  static inline bool isnull(uint8_t v) { return false; }
-};
-
-#define NPY_INT_DECL(TYPE, CapType, T)               \
-  template <>                                        \
-  struct npy_traits<NPY_##TYPE> {                    \
-    typedef T value_type;                            \
-    using TypeClass = arrow::CapType##Type;          \
-    using BuilderClass = arrow::CapType##Builder;    \
-                                                     \
-    static constexpr bool supports_nulls = false;    \
-    static inline bool isnull(T v) { return false; } \
-  };
-
-NPY_INT_DECL(INT8, Int8, int8_t);
-NPY_INT_DECL(INT16, Int16, int16_t);
-NPY_INT_DECL(INT32, Int32, int32_t);
-NPY_INT_DECL(INT64, Int64, int64_t);
+int cast_npy_type_compat(int type_num) {
+// Both LONGLONG and INT64 can be observed in the wild, which is buggy. We set
+// U/LONGLONG to U/INT64 so things work properly.
 
-NPY_INT_DECL(UINT8, UInt8, uint8_t);
-NPY_INT_DECL(UINT16, UInt16, uint16_t);
-NPY_INT_DECL(UINT32, UInt32, uint32_t);
-NPY_INT_DECL(UINT64, UInt64, uint64_t);
-
-#if NPY_INT64 != NPY_LONGLONG
-NPY_INT_DECL(LONGLONG, Int64, int64_t);
-NPY_INT_DECL(ULONGLONG, UInt64, uint64_t);
+#if (NPY_INT64 == NPY_LONGLONG) && (NPY_SIZEOF_LONGLONG == 8)
+  if (type_num == NPY_LONGLONG) { type_num = NPY_INT64; }
+  if (type_num == NPY_ULONGLONG) { type_num = NPY_UINT64; }
 #endif
 
-template <>
-struct npy_traits<NPY_FLOAT32> {
-  typedef float value_type;
-  using TypeClass = arrow::FloatType;
-  using BuilderClass = arrow::FloatBuilder;
-
-  static constexpr bool supports_nulls = true;
-
-  static inline bool isnull(float v) { return v != v; }
-};
-
-template <>
-struct npy_traits<NPY_FLOAT64> {
-  typedef double value_type;
-  using TypeClass = arrow::DoubleType;
-  using BuilderClass = arrow::DoubleBuilder;
-
-  static constexpr bool supports_nulls = true;
-
-  static inline bool isnull(double v) { return v != v; }
-};
-
-template <>
-struct npy_traits<NPY_DATETIME> {
-  typedef int64_t value_type;
-  using TypeClass = arrow::TimestampType;
-  using BuilderClass = arrow::TimestampBuilder;
-
-  static constexpr bool supports_nulls = true;
-
-  static inline bool isnull(int64_t v) {
-    // NaT = -2**63
-    // = -0x8000000000000000
-    // = -9223372036854775808;
-    // = std::numeric_limits<int64_t>::min()
-    return v == std::numeric_limits<int64_t>::min();
-  }
-};
-
-template <>
-struct npy_traits<NPY_OBJECT> {
-  typedef PyObject* value_type;
-  static constexpr bool supports_nulls = true;
-};
+  return type_num;
+}
 
 static inline bool PyObject_is_null(const PyObject* obj) {
   return obj == Py_None || obj == numpy_nan;
@@ -181,8 +99,24 @@ static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap)
   return null_count;
 }
 
+// Returns null count
+static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
+  int64_t null_count = 0;
+  const uint8_t* mask_values = static_cast<const uint8_t*>(PyArray_DATA(mask));
+  // TODO(wesm): strided null mask
+  for (int i = 0; i < length; ++i) {
+    if (mask_values[i]) {
+      ++null_count;
+    } else {
+      BitUtil::SetBit(bitmap, i);
+    }
+  }
+  return null_count;
+}
+
 template <int TYPE>
-static int64_t ValuesToBytemap(const void* data, int64_t length, uint8_t* valid_bytes) {
+static int64_t ValuesToValidBytes(
+    const void* data, int64_t length, uint8_t* valid_bytes) {
   typedef npy_traits<TYPE> traits;
   typedef typename traits::value_type T;
 
@@ -214,7 +148,7 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
   return Status::OK();
 }
 
-Status AppendObjectStrings(arrow::StringBuilder& string_builder, PyObject** objects,
+Status AppendObjectStrings(StringBuilder& string_builder, PyObject** objects,
     int64_t objects_length, bool* have_bytes) {
   PyObject* obj;
 
@@ -242,360 +176,561 @@ Status AppendObjectStrings(arrow::StringBuilder& string_builder, PyObject** obje
   return Status::OK();
 }
 
-template <int TYPE>
-struct arrow_traits {};
+template <typename T>
+struct WrapBytes {};
 
 template <>
-struct arrow_traits<Type::BOOL> {
-  static constexpr int npy_type = NPY_BOOL;
-  static constexpr bool supports_nulls = false;
-  static constexpr bool is_boolean = true;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
+struct WrapBytes<StringArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+  }
 };
 
-#define INT_DECL(TYPE)                                     \
-  template <>                                              \
-  struct arrow_traits<Type::TYPE> {                        \
-    static constexpr int npy_type = NPY_##TYPE;            \
-    static constexpr bool supports_nulls = false;          \
-    static constexpr double na_value = NAN;                \
-    static constexpr bool is_boolean = false;              \
-    static constexpr bool is_numeric_not_nullable = true;  \
-    static constexpr bool is_numeric_nullable = false;     \
-    typedef typename npy_traits<NPY_##TYPE>::value_type T; \
-  };
-
-INT_DECL(INT8);
-INT_DECL(INT16);
-INT_DECL(INT32);
-INT_DECL(INT64);
-INT_DECL(UINT8);
-INT_DECL(UINT16);
-INT_DECL(UINT32);
-INT_DECL(UINT64);
-
 template <>
-struct arrow_traits<Type::FLOAT> {
-  static constexpr int npy_type = NPY_FLOAT32;
-  static constexpr bool supports_nulls = true;
-  static constexpr float na_value = NAN;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
-  typedef typename npy_traits<NPY_FLOAT32>::value_type T;
+struct WrapBytes<BinaryArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+  }
 };
 
-template <>
-struct arrow_traits<Type::DOUBLE> {
-  static constexpr int npy_type = NPY_FLOAT64;
-  static constexpr bool supports_nulls = true;
-  static constexpr double na_value = NAN;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
-  typedef typename npy_traits<NPY_FLOAT64>::value_type T;
-};
+static inline bool ListTypeSupported(const Type::type type_id) {
+  switch (type_id) {
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::INT64:
+    case Type::UINT64:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+    case Type::STRING:
+    case Type::TIMESTAMP:
+      // The above types are all supported.
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
 
-static constexpr int64_t kPandasTimestampNull = std::numeric_limits<int64_t>::min();
+// ----------------------------------------------------------------------
+// Conversion from NumPy-in-Pandas to Arrow
 
-template <>
-struct arrow_traits<Type::TIMESTAMP> {
-  static constexpr int npy_type = NPY_DATETIME;
-  static constexpr bool supports_nulls = true;
-  static constexpr int64_t na_value = kPandasTimestampNull;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
-  typedef typename npy_traits<NPY_DATETIME>::value_type T;
-};
+class PandasConverter : public TypeVisitor {
+ public:
+  PandasConverter(
+      MemoryPool* pool, PyObject* ao, PyObject* mo, const std::shared_ptr<DataType>& type)
+      : pool_(pool),
+        type_(type),
+        arr_(reinterpret_cast<PyArrayObject*>(ao)),
+        mask_(nullptr) {
+    if (mo != nullptr and mo != Py_None) { mask_ = reinterpret_cast<PyArrayObject*>(mo); }
+    length_ = PyArray_SIZE(arr_);
+  }
 
-template <>
-struct arrow_traits<Type::DATE> {
-  static constexpr int npy_type = NPY_DATETIME;
-  static constexpr bool supports_nulls = true;
-  static constexpr int64_t na_value = kPandasTimestampNull;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
-  typedef typename npy_traits<NPY_DATETIME>::value_type T;
-};
+  bool is_strided() const {
+    npy_intp* astrides = PyArray_STRIDES(arr_);
+    return astrides[0] != PyArray_DESCR(arr_)->elsize;
+  }
 
-template <>
-struct arrow_traits<Type::STRING> {
-  static constexpr int npy_type = NPY_OBJECT;
-  static constexpr bool supports_nulls = true;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
-};
+  Status InitNullBitmap() {
+    int null_bytes = BitUtil::BytesForBits(length_);
 
-template <>
-struct arrow_traits<Type::BINARY> {
-  static constexpr int npy_type = NPY_OBJECT;
-  static constexpr bool supports_nulls = true;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
-};
+    null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
 
-template <typename T>
-struct WrapBytes {};
+    null_bitmap_data_ = null_bitmap_->mutable_data();
+    memset(null_bitmap_data_, 0, null_bytes);
 
-template <>
-struct WrapBytes<arrow::StringArray> {
-  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
-    return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+    return Status::OK();
   }
-};
 
-template <>
-struct WrapBytes<arrow::BinaryArray> {
-  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
-    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
-  }
-};
+  // ----------------------------------------------------------------------
+  // Traditional visitor conversion for non-object arrays
 
-inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
-  if (type == NPY_DATETIME) {
-    PyArray_Descr* descr = PyArray_DESCR(out);
-    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-    if (datatype->type == Type::TIMESTAMP) {
-      auto timestamp_type = static_cast<arrow::TimestampType*>(datatype);
+  template <typename ArrowType>
+  Status ConvertData(std::shared_ptr<Buffer>* data);
 
-      switch (timestamp_type->unit) {
-        case arrow::TimestampType::Unit::SECOND:
-          date_dtype->meta.base = NPY_FR_s;
-          break;
-        case arrow::TimestampType::Unit::MILLI:
-          date_dtype->meta.base = NPY_FR_ms;
-          break;
-        case arrow::TimestampType::Unit::MICRO:
-          date_dtype->meta.base = NPY_FR_us;
-          break;
-        case arrow::TimestampType::Unit::NANO:
-          date_dtype->meta.base = NPY_FR_ns;
-          break;
-      }
-    } else {
-      // datatype->type == Type::DATE
-      date_dtype->meta.base = NPY_FR_D;
+  template <typename ArrowType>
+  Status VisitNative() {
+    using traits = arrow_traits<ArrowType::type_id>;
+
+    if (mask_ != nullptr || traits::supports_nulls) { RETURN_NOT_OK(InitNullBitmap()); }
+
+    std::shared_ptr<Buffer> data;
+    RETURN_NOT_OK(ConvertData<ArrowType>(&data));
+
+    int64_t null_count = 0;
+    if (mask_ != nullptr) {
+      null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
+    } else if (traits::supports_nulls) {
+      // TODO(wesm): this presumes the NumPy C type and arrow C type are the
+      // same
+      null_count = ValuesToBitmap<traits::npy_type>(
+          PyArray_DATA(arr_), length_, null_bitmap_data_);
     }
+
+    std::vector<FieldMetadata> fields(1);
+    fields[0].length = length_;
+    fields[0].null_count = null_count;
+    fields[0].offset = 0;
+
+    return LoadArray(type_, fields, {null_bitmap_, data}, &out_);
   }
-}
 
-template <typename T>
-inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-    // Upcast to double, set NaN as appropriate
+#define VISIT_NATIVE(TYPE) \
+  Status Visit(const TYPE& type) override { return VisitNative<TYPE>(); }
 
-    for (int i = 0; i < arr->length(); ++i) {
-      *out_values++ = prim_arr->IsNull(i) ? NAN : in_values[i];
+  VISIT_NATIVE(BooleanType);
+  VISIT_NATIVE(Int8Type);
+  VISIT_NATIVE(Int16Type);
+  VISIT_NATIVE(Int32Type);
+  VISIT_NATIVE(Int64Type);
+  VISIT_NATIVE(UInt8Type);
+  VISIT_NATIVE(UInt16Type);
+  VISIT_NATIVE(UInt32Type);
+  VISIT_NATIVE(UInt64Type);
+  VISIT_NATIVE(FloatType);
+  VISIT_NATIVE(DoubleType);
+  VISIT_NATIVE(TimestampType);
+
+#undef VISIT_NATIVE
+
+  Status Convert(std::shared_ptr<Array>* out) {
+    if (PyArray_NDIM(arr_) != 1) {
+      return Status::Invalid("only handle 1-dimensional arrays");
     }
+    // TODO(wesm): strided arrays
+    if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
+
+    if (type_ == nullptr) { return Status::Invalid("Must pass data type"); }
+
+    // Visit the type to perform conversion
+    RETURN_NOT_OK(type_->Accept(this));
+
+    *out = out_;
+    return Status::OK();
   }
-}
 
-template <typename T>
-inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-    memcpy(out_values, in_values, sizeof(T) * arr->length());
-    out_values += arr->length();
+  // ----------------------------------------------------------------------
+  // Conversion logic for various object dtype arrays
+
+  template <int ITEM_TYPE, typename ArrowType>
+  Status ConvertTypedLists(
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
+
+  Status ConvertObjectStrings(std::shared_ptr<Array>* out);
+  Status ConvertBooleans(std::shared_ptr<Array>* out);
+  Status ConvertDates(std::shared_ptr<Array>* out);
+  Status ConvertLists(const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
+  Status ConvertObjects(std::shared_ptr<Array>* out);
+
+ protected:
+  MemoryPool* pool_;
+  std::shared_ptr<DataType> type_;
+  PyArrayObject* arr_;
+  PyArrayObject* mask_;
+  int64_t length_;
+
+  // Used in visitor pattern
+  std::shared_ptr<Array> out_;
+
+  std::shared_ptr<ResizableBuffer> null_bitmap_;
+  uint8_t* null_bitmap_data_;
+};
+
+template <typename ArrowType>
+inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
+  using traits = arrow_traits<ArrowType::type_id>;
+
+  // Handle LONGLONG->INT64 and other fun things
+  int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
+
+  if (traits::npy_type != type_num_compat) {
+    return Status::NotImplemented("NumPy type casts not yet implemented");
   }
+
+  *data = std::make_shared<NumPyBuffer>(arr_);
+  return Status::OK();
 }
 
-template <typename InType, typename OutType>
-inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values = in_values[i];
-    }
+template <>
+inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>* data) {
+  int nbytes = BitUtil::BytesForBits(length_);
+  auto buffer = std::make_shared<PoolBuffer>(pool_);
+  RETURN_NOT_OK(buffer->Resize(nbytes));
+
+  const uint8_t* values = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
+
+  uint8_t* bitmap = buffer->mutable_data();
+
+  memset(bitmap, 0, nbytes);
+  for (int i = 0; i < length_; ++i) {
+    if (values[i] > 0) { BitUtil::SetBit(bitmap, i); }
   }
+
+  *data = buffer;
+  return Status::OK();
 }
 
-static Status ConvertBooleanWithNulls(const ChunkedArray& data, PyObject** out_values) {
+Status PandasConverter::ConvertDates(std::shared_ptr<Array>* out) {
   PyAcquireGIL lock;
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (bool_arr->IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values++ = Py_None;
-      } else if (bool_arr->Value(i)) {
-        // True
-        Py_INCREF(Py_True);
-        *out_values++ = Py_True;
-      } else {
-        // False
-        Py_INCREF(Py_False);
-        *out_values++ = Py_False;
-      }
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  DateBuilder date_builder(pool_);
+  RETURN_NOT_OK(date_builder.Resize(length_));
+
+  Status s;
+  PyObject* obj;
+  for (int64_t i = 0; i < length_; ++i) {
+    obj = objects[i];
+    if (PyDate_CheckExact(obj)) {
+      PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(obj);
+      date_builder.Append(PyDate_to_ms(pydate));
+    } else {
+      date_builder.AppendNull();
     }
   }
-  return Status::OK();
+  return date_builder.Finish(out);
 }
 
-static void ConvertBooleanNoNulls(const ChunkedArray& data, uint8_t* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto bool_arr = static_cast<arrow::BooleanArray*>(arr.get());
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = static_cast<uint8_t>(bool_arr->Value(i));
-    }
+Status PandasConverter::ConvertObjectStrings(std::shared_ptr<Array>* out) {
+  PyAcquireGIL lock;
+
+  // The output type at this point is inconclusive because there may be bytes
+  // and unicode mixed in the object array
+
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  StringBuilder string_builder(pool_);
+  RETURN_NOT_OK(string_builder.Resize(length_));
+
+  Status s;
+  bool have_bytes = false;
+  RETURN_NOT_OK(AppendObjectStrings(string_builder, objects, length_, &have_bytes));
+  RETURN_NOT_OK(string_builder.Finish(out));
+
+  if (have_bytes) {
+    const auto& arr = static_cast<const StringArray&>(*out->get());
+    *out = std::make_shared<BinaryArray>(arr.length(), arr.value_offsets(), arr.data(),
+        arr.null_bitmap(), arr.null_count());
   }
+  return Status::OK();
 }
 
-template <typename ArrayType>
-inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values) {
+Status PandasConverter::ConvertBooleans(std::shared_ptr<Array>* out) {
   PyAcquireGIL lock;
-  for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = static_cast<ArrayType*>(data.chunk(c).get());
 
-    const uint8_t* data_ptr;
-    int32_t length;
-    const bool has_nulls = data.null_count() > 0;
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (has_nulls && arr->IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values = Py_None;
-      } else {
-        data_ptr = arr->GetValue(i, &length);
-        *out_values = WrapBytes<ArrayType>::Wrap(data_ptr, length);
-        if (*out_values == nullptr) {
-          PyErr_Clear();
-          std::stringstream ss;
-          ss << "Wrapping "
-             << std::string(reinterpret_cast<const char*>(data_ptr), length) << " failed";
-          return Status::UnknownError(ss.str());
-        }
-      }
-      ++out_values;
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+
+  int nbytes = BitUtil::BytesForBits(length_);
+  auto data = std::make_shared<PoolBuffer>(pool_);
+  RETURN_NOT_OK(data->Resize(nbytes));
+  uint8_t* bitmap = data->mutable_data();
+  memset(bitmap, 0, nbytes);
+
+  int64_t null_count = 0;
+  for (int64_t i = 0; i < length_; ++i) {
+    if (objects[i] == Py_True) {
+      BitUtil::SetBit(bitmap, i);
+      BitUtil::SetBit(null_bitmap_data_, i);
+    } else if (objects[i] != Py_False) {
+      ++null_count;
+    } else {
+      BitUtil::SetBit(null_bitmap_data_, i);
     }
   }
+
+  *out = std::make_shared<BooleanArray>(length_, data, null_bitmap_, null_count);
+
   return Status::OK();
 }
 
-template <typename ArrowType>
-inline Status ConvertListsLike(
-    const std::shared_ptr<Column>& col, PyObject** out_values) {
-  const ChunkedArray& data = *col->data().get();
-  auto list_type = std::static_pointer_cast<ListType>(col->type());
+Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
+  // Python object arrays are annoying, since we could have one of:
+  //
+  // * Strings
+  // * Booleans with nulls
+  // * Mixed type (not supported at the moment by arrow format)
+  //
+  // Additionally, nulls may be encoded either as np.nan or None. So we have to
+  // do some type inference and conversion
 
-  // Get column of underlying value arrays
-  std::vector<std::shared_ptr<Array>> value_arrays;
-  for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
-    value_arrays.emplace_back(arr->values());
-  }
-  auto flat_column = std::make_shared<Column>(list_type->value_field(), value_arrays);
-  // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
-  //    Storing a reference to the whole Array would be to expensive.
-  PyObject* numpy_array;
-  RETURN_NOT_OK(ConvertColumnToPandas(flat_column, nullptr, &numpy_array));
+  RETURN_NOT_OK(InitNullBitmap());
 
-  PyAcquireGIL lock;
+  // TODO: mask not supported here
+  if (mask_ != nullptr) {
+    return Status::NotImplemented("mask not supported in object conversions yet");
+  }
 
-  for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = std::static_pointer_cast<arrow::ListArray>(data.chunk(c));
+  const PyObject** objects;
+  {
+    PyAcquireGIL lock;
+    objects = reinterpret_cast<const PyObject**>(PyArray_DATA(arr_));
+    PyDateTime_IMPORT;
+  }
 
-    const uint8_t* data_ptr;
-    const bool has_nulls = data.null_count() > 0;
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (has_nulls && arr->IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values = Py_None;
+  if (type_) {
+    switch (type_->type) {
+      case Type::STRING:
+        return ConvertObjectStrings(out);
+      case Type::BOOL:
+        return ConvertBooleans(out);
+      case Type::DATE:
+        return ConvertDates(out);
+      case Type::LIST: {
+        const auto& list_field = static_cast<const ListType&>(*type_);
+        return ConvertLists(list_field.value_field()->type, out);
+      }
+      default:
+        return Status::TypeError("No known conversion to Arrow type");
+    }
+  } else {
+    for (int64_t i = 0; i < length_; ++i) {
+      if (PyObject_is_null(objects[i])) {
+        continue;
+      } else if (PyObject_is_string(objects[i])) {
+        return ConvertObjectStrings(out);
+      } else if (PyBool_Check(objects[i])) {
+        return ConvertBooleans(out);
+      } else if (PyDate_CheckExact(objects[i])) {
+        return ConvertDates(out);
       } else {
-        PyObject* start = PyLong_FromLong(arr->value_offset(i));
-        PyObject* end = PyLong_FromLong(arr->value_offset(i + 1));
-        PyObject* slice = PySlice_New(start, end, NULL);
-        *out_values = PyObject_GetItem(numpy_array, slice);
-        Py_DECREF(start);
-        Py_DECREF(end);
-        Py_DECREF(slice);
+        return Status::TypeError("unhandled python type");
       }
-      ++out_values;
     }
   }
 
-  Py_XDECREF(numpy_array);
-  return Status::OK();
+  return Status::TypeError("Unable to infer type of object array, were all null");
 }
 
-template <typename T>
-inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+template <int ITEM_TYPE, typename ArrowType>
+inline Status PandasConverter::ConvertTypedLists(
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+  typedef npy_traits<ITEM_TYPE> traits;
+  typedef typename traits::value_type T;
+  typedef typename traits::BuilderClass BuilderT;
 
-    const uint8_t* valid_bits = arr->null_bitmap_data();
+  PyAcquireGIL lock;
 
-    if (arr->null_count() > 0) {
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i];
+  auto value_builder = std::make_shared<BuilderT>(pool_, type);
+  ListBuilder list_builder(pool_, value_builder);
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  for (int64_t i = 0; i < length_; ++i) {
+    if (PyObject_is_null(objects[i])) {
+      RETURN_NOT_OK(list_builder.AppendNull());
+    } else if (PyArray_Check(objects[i])) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
+      RETURN_NOT_OK(list_builder.Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
+
+      int64_t size = PyArray_DIM(numpy_array, 0);
+      auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
+      if (traits::supports_nulls) {
+        null_bitmap_->Resize(size, false);
+        // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
+        // currently support this.
+        // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
+        ValuesToValidBytes<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
+        RETURN_NOT_OK(value_builder->Append(data, size, null_bitmap_->data()));
+      } else {
+        RETURN_NOT_OK(value_builder->Append(data, size));
+      }
+
+    } else if (PyList_Check(objects[i])) {
+      int64_t size;
+      std::shared_ptr<DataType> inferred_type;
+      RETURN_NOT_OK(list_builder.Append(true));
+      RETURN_NOT_OK(InferArrowType(objects[i], &size, &inferred_type));
+      if (inferred_type->type != type->type) {
+        std::stringstream ss;
+        ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
+        return Status::TypeError(ss.str());
       }
+      RETURN_NOT_OK(AppendPySequence(objects[i], type, value_builder));
     } else {
-      memcpy(out_values, in_values, sizeof(T) * arr->length());
-      out_values += arr->length();
+      return Status::TypeError("Unsupported Python type for list items");
     }
   }
+  return list_builder.Finish(out);
 }
 
-template <typename InType, typename OutType>
-inline void ConvertNumericNullableCast(
-    const ChunkedArray& data, OutType na_value, OutType* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+template <>
+inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+  PyAcquireGIL lock;
+  // TODO: If there are bytes involed, convert to Binary representation
+  bool have_bytes = false;
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = arr->IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
+  auto value_builder = std::make_shared<StringBuilder>(pool_);
+  ListBuilder list_builder(pool_, value_builder);
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  for (int64_t i = 0; i < length_; ++i) {
+    if (PyObject_is_null(objects[i])) {
+      RETURN_NOT_OK(list_builder.AppendNull());
+    } else if (PyArray_Check(objects[i])) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
+      RETURN_NOT_OK(list_builder.Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+
+      int64_t size = PyArray_DIM(numpy_array, 0);
+      auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
+      RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
+    } else if (PyList_Check(objects[i])) {
+      int64_t size;
+      std::shared_ptr<DataType> inferred_type;
+      RETURN_NOT_OK(list_builder.Append(true));
+      RETURN_NOT_OK(InferArrowType(objects[i], &size, &inferred_type));
+      if (inferred_type->type != Type::STRING) {
+        std::stringstream ss;
+        ss << inferred_type->ToString() << " cannot be converted to STRING.";
+        return Status::TypeError(ss.str());
+      }
+      RETURN_NOT_OK(AppendPySequence(objects[i], inferred_type, value_builder));
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
     }
   }
+  return list_builder.Finish(out);
 }
 
-template <typename T>
-inline void ConvertDates(const ChunkedArray& data, T na_value, T* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
-      *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / 86400000;
-    }
+#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                  \
+  case Type::TYPE: {                                            \
+    return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type, out); \
+  }
+
+Status PandasConverter::ConvertLists(
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+  switch (type->type) {
+    LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
+    LIST_CASE(INT8, NPY_INT8, Int8Type)
+    LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
+    LIST_CASE(INT16, NPY_INT16, Int16Type)
+    LIST_CASE(UINT32, NPY_UINT32, UInt32Type)
+    LIST_CASE(INT32, NPY_INT32, Int32Type)
+    LIST_CASE(UINT64, NPY_UINT64, UInt64Type)
+    LIST_CASE(INT64, NPY_INT64, Int64Type)
+    LIST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
+    LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
+    LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
+    LIST_CASE(STRING, NPY_OBJECT, StringType)
+    default:
+      return Status::TypeError("Unknown list item type");
   }
+
+  return Status::TypeError("Unknown list type");
 }
 
-template <typename InType, int SHIFT>
-inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+  PandasConverter converter(pool, ao, mo, type);
+  return converter.Convert(out);
+}
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = arr->IsNull(i) ? kPandasTimestampNull
-                                     : (static_cast<int64_t>(in_values[i]) * SHIFT);
+Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+  PandasConverter converter(pool, ao, mo, type);
+  return converter.ConvertObjects(out);
+}
+
+Status PandasDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
+  PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
+
+  int type_num = cast_npy_type_compat(descr->type_num);
+
+#define TO_ARROW_TYPE_CASE(NPY_NAME, FACTORY) \
+  case NPY_##NPY_NAME:                        \
+    *out = FACTORY();                         \
+    break;
+
+  switch (type_num) {
+    TO_ARROW_TYPE_CASE(BOOL, boolean);
+    TO_ARROW_TYPE_CASE(INT8, int8);
+    TO_ARROW_TYPE_CASE(INT16, int16);
+    TO_ARROW_TYPE_CASE(INT32, int32);
+    TO_ARROW_TYPE_CASE(INT64, int64);
+#if (NPY_INT64 != NPY_LONGLONG)
+    TO_ARROW_TYPE_CASE(LONGLONG, int64);
+#endif
+    TO_ARROW_TYPE_CASE(UINT8, uint8);
+    TO_ARROW_TYPE_CASE(UINT16, uint16);
+    TO_ARROW_TYPE_CASE(UINT32, uint32);
+    TO_ARROW_TYPE_CASE(UINT64, uint64);
+#if (NPY_UINT64 != NPY_ULONGLONG)
+    TO_ARROW_CASE(ULONGLONG);
+#endif
+    TO_ARROW_TYPE_CASE(FLOAT32, float32);
+    TO_ARROW_TYPE_CASE(FLOAT64, float64);
+    case NPY_DATETIME: {
+      auto date_dtype =
+          reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+      TimeUnit unit;
+      switch (date_dtype->meta.base) {
+        case NPY_FR_s:
+          unit = TimeUnit::SECOND;
+          break;
+        case NPY_FR_ms:
+          unit = TimeUnit::MILLI;
+          break;
+        case NPY_FR_us:
+          unit = TimeUnit::MICRO;
+          break;
+        case NPY_FR_ns:
+          unit = TimeUnit::NANO;
+          break;
+        default:
+          return Status::NotImplemented("Unsupported datetime64 time unit");
+      }
+      *out = timestamp(unit);
+    } break;
+    default: {
+      std::stringstream ss;
+      ss << "Unsupported numpy type " << descr->type_num << std::endl;
+      return Status::NotImplemented(ss.str());
     }
   }
+
+#undef TO_ARROW_TYPE_CASE
+
+  return Status::OK();
 }
 
 // ----------------------------------------------------------------------
 // pandas 0.x DataFrame conversion internals
 
+inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
+  if (type == NPY_DATETIME) {
+    PyArray_Descr* descr = PyArray_DESCR(out);
+    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+    if (datatype->type == Type::TIMESTAMP) {
+      auto timestamp_type = static_cast<TimestampType*>(datatype);
+
+      switch (timestamp_type->unit) {
+        case TimestampType::Unit::SECOND:
+          date_dtype->meta.base = NPY_FR_s;
+          break;
+        case TimestampType::Unit::MILLI:
+          date_dtype->meta.base = NPY_FR_ms;
+          break;
+        case TimestampType::Unit::MICRO:
+          date_dtype->meta.base = NPY_FR_us;
+          break;
+        case TimestampType::Unit::NANO:
+          date_dtype->meta.base = NPY_FR_ns;
+          break;
+      }
+    } else {
+      // datatype->type == Type::DATE
+      date_dtype->meta.base = NPY_FR_D;
+    }
+  }
+}
+
 class PandasBlock {
  public:
   enum type {
@@ -688,10 +823,219 @@ class PandasBlock {
   DISALLOW_COPY_AND_ASSIGN(PandasBlock);
 };
 
-#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum)                         \
-  case Type::ArrowEnum:                                                     \
-    RETURN_NOT_OK((ConvertListsLike<::arrow::ArrowType>(col, out_buffer))); \
-    break;
+template <typename T>
+inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    // Upcast to double, set NaN as appropriate
+
+    for (int i = 0; i < arr->length(); ++i) {
+      *out_values++ = prim_arr->IsNull(i) ? NAN : in_values[i];
+    }
+  }
+}
+
+template <typename T>
+inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    memcpy(out_values, in_values, sizeof(T) * arr->length());
+    out_values += arr->length();
+  }
+}
+
+template <typename InType, typename OutType>
+inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values = in_values[i];
+    }
+  }
+}
+
+static Status ConvertBooleanWithNulls(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto bool_arr = static_cast<BooleanArray*>(arr.get());
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (bool_arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values++ = Py_None;
+      } else if (bool_arr->Value(i)) {
+        // True
+        Py_INCREF(Py_True);
+        *out_values++ = Py_True;
+      } else {
+        // False
+        Py_INCREF(Py_False);
+        *out_values++ = Py_False;
+      }
+    }
+  }
+  return Status::OK();
+}
+
+static void ConvertBooleanNoNulls(const ChunkedArray& data, uint8_t* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto bool_arr = static_cast<BooleanArray*>(arr.get());
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = static_cast<uint8_t>(bool_arr->Value(i));
+    }
+  }
+}
+
+template <typename ArrayType>
+inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = static_cast<ArrayType*>(data.chunk(c).get());
+
+    const uint8_t* data_ptr;
+    int32_t length;
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (has_nulls && arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        data_ptr = arr->GetValue(i, &length);
+        *out_values = WrapBytes<ArrayType>::Wrap(data_ptr, length);
+        if (*out_values == nullptr) {
+          PyErr_Clear();
+          std::stringstream ss;
+          ss << "Wrapping "
+             << std::string(reinterpret_cast<const char*>(data_ptr), length) << " failed";
+          return Status::UnknownError(ss.str());
+        }
+      }
+      ++out_values;
+    }
+  }
+  return Status::OK();
+}
+
+template <typename ArrowType>
+inline Status ConvertListsLike(
+    const std::shared_ptr<Column>& col, PyObject** out_values) {
+  const ChunkedArray& data = *col->data().get();
+  auto list_type = std::static_pointer_cast<ListType>(col->type());
+
+  // Get column of underlying value arrays
+  std::vector<std::shared_ptr<Array>> value_arrays;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = std::static_pointer_cast<ListArray>(data.chunk(c));
+    value_arrays.emplace_back(arr->values());
+  }
+  auto flat_column = std::make_shared<Column>(list_type->value_field(), value_arrays);
+  // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
+  //    Storing a reference to the whole Array would be to expensive.
+  PyObject* numpy_array;
+  RETURN_NOT_OK(ConvertColumnToPandas(flat_column, nullptr, &numpy_array));
+
+  PyAcquireGIL lock;
+
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = std::static_pointer_cast<ListArray>(data.chunk(c));
+
+    const uint8_t* data_ptr;
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (has_nulls && arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        PyObject* start = PyLong_FromLong(arr->value_offset(i));
+        PyObject* end = PyLong_FromLong(arr->value_offset(i + 1));
+        PyObject* slice = PySlice_New(start, end, NULL);
+        *out_values = PyObject_GetItem(numpy_array, slice);
+        Py_DECREF(start);
+        Py_DECREF(end);
+        Py_DECREF(slice);
+      }
+      ++out_values;
+    }
+  }
+
+  Py_XDECREF(numpy_array);
+  return Status::OK();
+}
+
+template <typename T>
+inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+
+    const uint8_t* valid_bits = arr->null_bitmap_data();
+
+    if (arr->null_count() > 0) {
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i];
+      }
+    } else {
+      memcpy(out_values, in_values, sizeof(T) * arr->length());
+      out_values += arr->length();
+    }
+  }
+}
+
+template <typename InType, typename OutType>
+inline void ConvertNumericNullableCast(
+    const ChunkedArray& data, OutType na_value, OutType* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = arr->IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
+    }
+  }
+}
+
+template <typename T>
+inline void ConvertDates(const ChunkedArray& data, T na_value, T* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
+      *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / 86400000;
+    }
+  }
+}
+
+template <typename InType, int SHIFT>
+inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const std::shared_ptr<Array> arr = data.chunk(c);
+    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      *out_values++ = arr->IsNull(i) ? kPandasTimestampNull
+                                     : (static_cast<int64_t>(in_values[i]) * SHIFT);
+    }
+  }
+}
+
+#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum)                \
+  case Type::ArrowEnum:                                            \
+    RETURN_NOT_OK((ConvertListsLike<ArrowType>(col, out_buffer))); \
+    break;
 
 class ObjectBlock : public PandasBlock {
  public:
@@ -712,9 +1056,9 @@ class ObjectBlock : public PandasBlock {
     if (type == Type::BOOL) {
       RETURN_NOT_OK(ConvertBooleanWithNulls(data, out_buffer));
     } else if (type == Type::BINARY) {
-      RETURN_NOT_OK(ConvertBinaryLike<arrow::BinaryArray>(data, out_buffer));
+      RETURN_NOT_OK(ConvertBinaryLike<BinaryArray>(data, out_buffer));
     } else if (type == Type::STRING) {
-      RETURN_NOT_OK(ConvertBinaryLike<arrow::StringArray>(data, out_buffer));
+      RETURN_NOT_OK(ConvertBinaryLike<StringArray>(data, out_buffer));
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
       switch (list_type->value_type()->type) {
@@ -880,8 +1224,8 @@ class DatetimeBlock : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
 
-  Status Allocate() override {
-    RETURN_NOT_OK(AllocateNDArray(NPY_DATETIME));
+  Status AllocateDatetime(int ndim) {
+    RETURN_NOT_OK(AllocateNDArray(NPY_DATETIME, ndim));
 
     PyAcquireGIL lock;
     auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(
@@ -890,6 +1234,8 @@ class DatetimeBlock : public PandasBlock {
     return Status::OK();
   }
 
+  Status Allocate() override { return AllocateDatetime(2); }
+
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
       int64_t rel_placement) override {
     Type::type type = col->type()->type;
@@ -904,15 +1250,15 @@ class DatetimeBlock : public PandasBlock {
       // TODO(wesm): Do we want to make sure to zero out the milliseconds?
       ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
     } else if (type == Type::TIMESTAMP) {
-      auto ts_type = static_cast<arrow::TimestampType*>(col->type().get());
+      auto ts_type = static_cast<TimestampType*>(col->type().get());
 
-      if (ts_type->unit == arrow::TimeUnit::NANO) {
+      if (ts_type->unit == TimeUnit::NANO) {
         ConvertNumericNullable<int64_t>(data, kPandasTimestampNull, out_buffer);
-      } else if (ts_type->unit == arrow::TimeUnit::MICRO) {
+      } else if (ts_type->unit == TimeUnit::MICRO) {
         ConvertDatetimeNanos<int64_t, 1000L>(data, out_buffer);
-      } else if (ts_type->unit == arrow::TimeUnit::MILLI) {
+      } else if (ts_type->unit == TimeUnit::MILLI) {
         ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
-      } else if (ts_type->unit == arrow::TimeUnit::SECOND) {
+      } else if (ts_type->unit == TimeUnit::SECOND) {
         ConvertDatetimeNanos<int64_t, 1000000000L>(data, out_buffer);
       } else {
         return Status::NotImplemented("Unsupported time unit");
@@ -931,6 +1277,9 @@ class DatetimeTZBlock : public DatetimeBlock {
   DatetimeTZBlock(const std::string& timezone, int64_t num_rows)
       : DatetimeBlock(num_rows, 1), timezone_(timezone) {}
 
+  // Like Categorical, the internal ndarray is 1-dimensional
+  Status Allocate() override { return AllocateDatetime(1); }
+
   Status GetPyResult(PyObject** output) override {
     PyObject* result = PyDict_New();
     RETURN_IF_PYERROR();
@@ -977,9 +1326,8 @@ class CategoricalBlock : public PandasBlock {
 
     for (int c = 0; c < data.num_chunks(); c++) {
       const std::shared_ptr<Array> arr = data.chunk(c);
-      const auto& dict_arr = static_cast<const arrow::DictionaryArray&>(*arr);
-      const auto& indices =
-          static_cast<const arrow::PrimitiveArray&>(*dict_arr.indices());
+      const auto& dict_arr = static_cast<const DictionaryArray&>(*arr);
+      const auto& indices = static_cast<const PrimitiveArray&>(*dict_arr.indices());
       auto in_values = reinterpret_cast<const T*>(indices.data()->data());
 
       // Null is -1 in CategoricalBlock
@@ -1046,28 +1394,6 @@ Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
   return (*block)->Allocate();
 }
 
-static inline bool ListTypeSupported(const Type::type type_id) {
-  switch (type_id) {
-    case Type::UINT8:
-    case Type::INT8:
-    case Type::UINT16:
-    case Type::INT16:
-    case Type::UINT32:
-    case Type::INT32:
-    case Type::INT64:
-    case Type::UINT64:
-    case Type::FLOAT:
-    case Type::DOUBLE:
-    case Type::STRING:
-    case Type::TIMESTAMP:
-      // The above types are all supported.
-      return true;
-    default:
-      break;
-  }
-  return false;
-}
-
 static inline Status MakeCategoricalBlock(const std::shared_ptr<DataType>& type,
     int64_t num_rows, std::shared_ptr<PandasBlock>* block) {
   // All categoricals become a block with a single column
@@ -1168,7 +1494,7 @@ class DataFrameBlockCreator {
           output_type = PandasBlock::DATETIME;
           break;
         case Type::TIMESTAMP: {
-          const auto& ts_type = static_cast<const arrow::TimestampType&>(*col->type());
+          const auto& ts_type = static_cast<const TimestampType&>(*col->type());
           if (ts_type.timezone != "") {
             output_type = PandasBlock::DATETIME_WITH_TZ;
           } else {
@@ -1182,636 +1508,165 @@ class DataFrameBlockCreator {
             ss << "Not implemented type for lists: "
                << list_type->value_type()->ToString();
             return Status::NotImplemented(ss.str());
-          }
-          output_type = PandasBlock::OBJECT;
-        } break;
-        case Type::DICTIONARY:
-          output_type = PandasBlock::CATEGORICAL;
-          break;
-        default:
-          return Status::NotImplemented(col->type()->ToString());
-      }
-
-      int block_placement = 0;
-      std::shared_ptr<PandasBlock> block;
-      if (output_type == PandasBlock::CATEGORICAL) {
-        RETURN_NOT_OK(MakeCategoricalBlock(col->type(), table_->num_rows(), &block));
-        categorical_blocks_[i] = block;
-      } else if (output_type == PandasBlock::DATETIME_WITH_TZ) {
-        const auto& ts_type = static_cast<const arrow::TimestampType&>(*col->type());
-        block = std::make_shared<DatetimeTZBlock>(ts_type.timezone, table_->num_rows());
-        RETURN_NOT_OK(block->Allocate());
-        datetimetz_blocks_[i] = block;
-      } else {
-        auto it = type_counts_.find(output_type);
-        if (it != type_counts_.end()) {
-          block_placement = it->second;
-          // Increment count
-          it->second += 1;
-        } else {
-          // Add key to map
-          type_counts_[output_type] = 1;
-        }
-      }
-
-      column_types_[i] = output_type;
-      column_block_placement_[i] = block_placement;
-    }
-
-    // Create normal non-categorical blocks
-    for (const auto& it : type_counts_) {
-      PandasBlock::type type = static_cast<PandasBlock::type>(it.first);
-      std::shared_ptr<PandasBlock> block;
-      RETURN_NOT_OK(MakeBlock(type, table_->num_rows(), it.second, &block));
-      blocks_[type] = block;
-    }
-    return Status::OK();
-  }
-
-  Status WriteTableToBlocks(int nthreads) {
-    auto WriteColumn = [this](int i) {
-      std::shared_ptr<Column> col = this->table_->column(i);
-      PandasBlock::type output_type = this->column_types_[i];
-
-      int rel_placement = this->column_block_placement_[i];
-
-      std::shared_ptr<PandasBlock> block;
-      if (output_type == PandasBlock::CATEGORICAL) {
-        auto it = this->categorical_blocks_.find(i);
-        if (it == this->blocks_.end()) {
-          return Status::KeyError("No categorical block allocated");
-        }
-        block = it->second;
-      } else {
-        auto it = this->blocks_.find(output_type);
-        if (it == this->blocks_.end()) { return Status::KeyError("No block allocated"); }
-        block = it->second;
-      }
-      return block->Write(col, i, rel_placement);
-    };
-
-    nthreads = std::min<int>(nthreads, table_->num_columns());
-
-    if (nthreads == 1) {
-      for (int i = 0; i < table_->num_columns(); ++i) {
-        RETURN_NOT_OK(WriteColumn(i));
-      }
-    } else {
-      std::vector<std::thread> thread_pool;
-      thread_pool.reserve(nthreads);
-      std::atomic<int> task_counter(0);
-
-      std::mutex error_mtx;
-      bool error_occurred = false;
-      Status error;
-
-      for (int thread_id = 0; thread_id < nthreads; ++thread_id) {
-        thread_pool.emplace_back(
-            [this, &error, &error_occurred, &error_mtx, &task_counter, &WriteColumn]() {
-              int column_num;
-              while (!error_occurred) {
-                column_num = task_counter.fetch_add(1);
-                if (column_num >= this->table_->num_columns()) { break; }
-                Status s = WriteColumn(column_num);
-                if (!s.ok()) {
-                  std::lock_guard<std::mutex> lock(error_mtx);
-                  error_occurred = true;
-                  error = s;
-                  break;
-                }
-              }
-            });
-      }
-      for (auto&& thread : thread_pool) {
-        thread.join();
-      }
-
-      if (error_occurred) { return error; }
-    }
-    return Status::OK();
-  }
-
-  Status AppendBlocks(const BlockMap& blocks, PyObject* list) {
-    for (const auto& it : blocks) {
-      PyObject* item;
-      RETURN_NOT_OK(it.second->GetPyResult(&item));
-      if (PyList_Append(list, item) < 0) { RETURN_IF_PYERROR(); }
-    }
-    return Status::OK();
-  }
-
-  Status GetResultList(PyObject** out) {
-    PyAcquireGIL lock;
-
-    PyObject* result = PyList_New(0);
-    RETURN_IF_PYERROR();
-
-    RETURN_NOT_OK(AppendBlocks(blocks_, result));
-    RETURN_NOT_OK(AppendBlocks(categorical_blocks_, result));
-    RETURN_NOT_OK(AppendBlocks(datetimetz_blocks_, result));
-
-    *out = result;
-    return Status::OK();
-  }
-
- private:
-  std::shared_ptr<Table> table_;
-
-  // column num -> block type id
-  std::vector<PandasBlock::type> column_types_;
-
-  // column num -> relative placement within internal block
-  std::vector<int> column_block_placement_;
-
-  // block type -> type count
-  std::unordered_map<int, int> type_counts_;
-
-  // block type -> block
-  BlockMap blocks_;
-
-  // column number -> categorical block
-  BlockMap categorical_blocks_;
-
-  // column number -> datetimetz block
-  BlockMap datetimetz_blocks_;
-};
-
-Status ConvertTableToPandas(
-    const std::shared_ptr<Table>& table, int nthreads, PyObject** out) {
-  DataFrameBlockCreator helper(table);
-  return helper.Convert(nthreads, out);
-}
-
-// ----------------------------------------------------------------------
-// Serialization
-
-template <int TYPE>
-class ArrowSerializer {
- public:
-  ArrowSerializer(arrow::MemoryPool* pool, PyArrayObject* arr, PyArrayObject* mask)
-      : pool_(pool), arr_(arr), mask_(mask) {
-    length_ = PyArray_SIZE(arr_);
-  }
-
-  void IndicateType(const std::shared_ptr<Field> field) { field_indicator_ = field; }
-
-  Status Convert(std::shared_ptr<Array>* out);
-
-  int stride() const { return PyArray_STRIDES(arr_)[0]; }
-
-  Status InitNullBitmap() {
-    int null_bytes = BitUtil::BytesForBits(length_);
-
-    null_bitmap_ = std::make_shared<arrow::PoolBuffer>(pool_);
-    RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
-
-    null_bitmap_data_ = null_bitmap_->mutable_data();
-    memset(null_bitmap_data_, 0, null_bytes);
-
-    return Status::OK();
-  }
-
-  bool is_strided() const {
-    npy_intp* astrides = PyArray_STRIDES(arr_);
-    return astrides[0] != PyArray_DESCR(arr_)->elsize;
-  }
-
- private:
-  Status ConvertData();
-
-  Status ConvertDates(std::shared_ptr<Array>* out) {
-    PyAcquireGIL lock;
-
-    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    arrow::DateBuilder date_builder(pool_);
-    RETURN_NOT_OK(date_builder.Resize(length_));
-
-    Status s;
-    PyObject* obj;
-    for (int64_t i = 0; i < length_; ++i) {
-      obj = objects[i];
-      if (PyDate_CheckExact(obj)) {
-        PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(obj);
-        date_builder.Append(PyDate_to_ms(pydate));
-      } else {
-        date_builder.AppendNull();
-      }
-    }
-    return date_builder.Finish(out);
-  }
-
-  Status ConvertObjectStrings(std::shared_ptr<Array>* out) {
-    PyAcquireGIL lock;
-
-    // The output type at this point is inconclusive because there may be bytes
-    // and unicode mixed in the object array
-
-    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    arrow::StringBuilder string_builder(pool_);
-    RETURN_NOT_OK(string_builder.Resize(length_));
-
-    Status s;
-    bool have_bytes = false;
-    RETURN_NOT_OK(AppendObjectStrings(string_builder, objects, length_, &have_bytes));
-    RETURN_NOT_OK(string_builder.Finish(out));
-
-    if (have_bytes) {
-      const auto& arr = static_cast<const arrow::StringArray&>(*out->get());
-      *out = std::make_shared<arrow::BinaryArray>(arr.length(), arr.value_offsets(),
-          arr.data(), arr.null_bitmap(), arr.null_count());
-    }
-    return Status::OK();
-  }
-
-  Status ConvertBooleans(std::shared_ptr<Array>* out) {
-    PyAcquireGIL lock;
-
-    PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-
-    int nbytes = BitUtil::BytesForBits(length_);
-    auto data = std::make_shared<arrow::PoolBuffer>(pool_);
-    RETURN_NOT_OK(data->Resize(nbytes));
-    uint8_t* bitmap = data->mutable_data();
-    memset(bitmap, 0, nbytes);
-
-    int64_t null_count = 0;
-    for (int64_t i = 0; i < length_; ++i) {
-      if (objects[i] == Py_True) {
-        BitUtil::SetBit(bitmap, i);
-        BitUtil::SetBit(null_bitmap_data_, i);
-      } else if (objects[i] != Py_False) {
-        ++null_count;
-      } else {
-        BitUtil::SetBit(null_bitmap_data_, i);
-      }
-    }
-
-    *out = std::make_shared<arrow::BooleanArray>(length_, data, null_bitmap_, null_count);
-
-    return Status::OK();
-  }
-
-  template <int ITEM_TYPE, typename ArrowType>
-  Status ConvertTypedLists(
-      const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out);
-
-#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                            \
-  case Type::TYPE: {                                                      \
-    return ConvertTypedLists<NUMPY_TYPE, ::arrow::ArrowType>(field, out); \
-  }
-
-  Status ConvertLists(const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-    switch (field->type->type) {
-      LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
-      LIST_CASE(INT8, NPY_INT8, Int8Type)
-      LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
-      LIST_CASE(INT16, NPY_INT16, Int16Type)
-      LIST_CASE(UINT32, NPY_UINT32, UInt32Type)
-      LIST_CASE(INT32, NPY_INT32, Int32Type)
-      LIST_CASE(UINT64, NPY_UINT64, UInt64Type)
-      LIST_CASE(INT64, NPY_INT64, Int64Type)
-      LIST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
-      LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
-      LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
-      LIST_CASE(STRING, NPY_OBJECT, StringType)
-      default:
-        return Status::TypeError("Unknown list item type");
-    }
-
-    return Status::TypeError("Unknown list type");
-  }
-
-  Status MakeDataType(std::shared_ptr<DataType>* out);
-
-  arrow::MemoryPool* pool_;
-
-  PyArrayObject* arr_;
-  PyArrayObject* mask_;
-
-  int64_t length_;
-
-  std::shared_ptr<Field> field_indicator_;
-  std::shared_ptr<arrow::Buffer> data_;
-  std::shared_ptr<arrow::ResizableBuffer> null_bitmap_;
-  uint8_t* null_bitmap_data_;
-};
-
-// Returns null count
-static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
-  int64_t null_count = 0;
-  const uint8_t* mask_values = static_cast<const uint8_t*>(PyArray_DATA(mask));
-  // TODO(wesm): strided null mask
-  for (int i = 0; i < length; ++i) {
-    if (mask_values[i]) {
-      ++null_count;
-    } else {
-      BitUtil::SetBit(bitmap, i);
-    }
-  }
-  return null_count;
-}
-
-template <int TYPE>
-inline Status ArrowSerializer<TYPE>::MakeDataType(std::shared_ptr<DataType>* out) {
-  out->reset(new typename npy_traits<TYPE>::TypeClass());
-  return Status::OK();
-}
-
-template <>
-inline Status ArrowSerializer<NPY_DATETIME>::MakeDataType(
-    std::shared_ptr<DataType>* out) {
-  PyArray_Descr* descr = PyArray_DESCR(arr_);
-  auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-  arrow::TimestampType::Unit unit;
-
-  switch (date_dtype->meta.base) {
-    case NPY_FR_s:
-      unit = arrow::TimestampType::Unit::SECOND;
-      break;
-    case NPY_FR_ms:
-      unit = arrow::TimestampType::Unit::MILLI;
-      break;
-    case NPY_FR_us:
-      unit = arrow::TimestampType::Unit::MICRO;
-      break;
-    case NPY_FR_ns:
-      unit = arrow::TimestampType::Unit::NANO;
-      break;
-    default:
-      return Status::Invalid("Unknown NumPy datetime unit");
-  }
-
-  out->reset(new arrow::TimestampType(unit));
-  return Status::OK();
-}
-
-template <int TYPE>
-inline Status ArrowSerializer<TYPE>::Convert(std::shared_ptr<Array>* out) {
-  typedef npy_traits<TYPE> traits;
-
-  if (mask_ != nullptr || traits::supports_nulls) { RETURN_NOT_OK(InitNullBitmap()); }
-
-  int64_t null_count = 0;
-  if (mask_ != nullptr) {
-    null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
-  } else if (traits::supports_nulls) {
-    null_count = ValuesToBitmap<TYPE>(PyArray_DATA(arr_), length_, null_bitmap_data_);
-  }
-
-  RETURN_NOT_OK(ConvertData());
-  std::shared_ptr<DataType> type;
-  RETURN_NOT_OK(MakeDataType(&type));
-
-  std::vector<arrow::FieldMetadata> fields(1);
-  fields[0].length = length_;
-  fields[0].null_count = null_count;
-  fields[0].offset = 0;
-
-  return arrow::LoadArray(type, fields, {null_bitmap_, data_}, out);
-}
-
-template <>
-inline Status ArrowSerializer<NPY_OBJECT>::Convert(std::shared_ptr<Array>* out) {
-  // Python object arrays are annoying, since we could have one of:
-  //
-  // * Strings
-  // * Booleans with nulls
-  // * Mixed type (not supported at the moment by arrow format)
-  //
-  // Additionally, nulls may be encoded either as np.nan or None. So we have to
-  // do some type inference and conversion
-
-  RETURN_NOT_OK(InitNullBitmap());
-
-  // TODO: mask not supported here
-  const PyObject** objects = reinterpret_cast<const PyObject**>(PyArray_DATA(arr_));
-  {
-    PyAcquireGIL lock;
-    PyDateTime_IMPORT;
-  }
-
-  if (field_indicator_) {
-    switch (field_indicator_->type->type) {
-      case Type::STRING:
-        return ConvertObjectStrings(out);
-      case Type::BOOL:
-        return ConvertBooleans(out);
-      case Type::DATE:
-        return ConvertDates(out);
-      case Type::LIST: {
-        auto list_field = static_cast<ListType*>(field_indicator_->type.get());
-        return ConvertLists(list_field->value_field(), out);
+          }
+          output_type = PandasBlock::OBJECT;
+        } break;
+        case Type::DICTIONARY:
+          output_type = PandasBlock::CATEGORICAL;
+          break;
+        default:
+          return Status::NotImplemented(col->type()->ToString());
       }
-      default:
-        return Status::TypeError("No known conversion to Arrow type");
-    }
-  } else {
-    for (int64_t i = 0; i < length_; ++i) {
-      if (PyObject_is_null(objects[i])) {
-        continue;
-      } else if (PyObject_is_string(objects[i])) {
-        return ConvertObjectStrings(out);
-      } else if (PyBool_Check(objects[i])) {
-        return ConvertBooleans(out);
-      } else if (PyDate_CheckExact(objects[i])) {
-        return ConvertDates(out);
+
+      int block_placement = 0;
+      std::shared_ptr<PandasBlock> block;
+      if (output_type == PandasBlock::CATEGORICAL) {
+        RETURN_NOT_OK(MakeCategoricalBlock(col->type(), table_->num_rows(), &block));
+        categorical_blocks_[i] = block;
+      } else if (output_type == PandasBlock::DATETIME_WITH_TZ) {
+        const auto& ts_type = static_cast<const TimestampType&>(*col->type());
+        block = std::make_shared<DatetimeTZBlock>(ts_type.timezone, table_->num_rows());
+        RETURN_NOT_OK(block->Allocate());
+        datetimetz_blocks_[i] = block;
       } else {
-        return Status::TypeError("unhandled python type");
+        auto it = type_counts_.find(output_type);
+        if (it != type_counts_.end()) {
+          block_placement = it->second;
+          // Increment count
+          it->second += 1;
+        } else {
+          // Add key to map
+          type_counts_[output_type] = 1;
+        }
       }
+
+      column_types_[i] = output_type;
+      column_block_placement_[i] = block_placement;
+    }
+
+    // Create normal non-categorical blocks
+    for (const auto& it : type_counts_) {
+      PandasBlock::type type = static_cast<PandasBlock::type>(it.first);
+      std::shared_ptr<PandasBlock> block;
+      RETURN_NOT_OK(MakeBlock(type, table_->num_rows(), it.second, &block));
+      blocks_[type] = block;
     }
+    return Status::OK();
   }
 
-  return Status::TypeError("Unable to infer type of object array, were all null");
-}
+  Status WriteTableToBlocks(int nthreads) {
+    auto WriteColumn = [this](int i) {
+      std::shared_ptr<Column> col = this->table_->column(i);
+      PandasBlock::type output_type = this->column_types_[i];
 
-template <int TYPE>
-inline Status ArrowSerializer<TYPE>::ConvertData() {
-  // TODO(wesm): strided arrays
-  if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
+      int rel_placement = this->column_block_placement_[i];
 
-  data_ = std::make_shared<NumPyBuffer>(arr_);
-  return Status::OK();
-}
+      std::shared_ptr<PandasBlock> block;
+      if (output_type == PandasBlock::CATEGORICAL) {
+        auto it = this->categorical_blocks_.find(i);
+        if (it == this->blocks_.end()) {
+          return Status::KeyError("No categorical block allocated");
+        }
+        block = it->second;
+      } else if (output_type == PandasBlock::DATETIME_WITH_TZ) {
+        auto it = this->datetimetz_blocks_.find(i);
+        if (it == this->datetimetz_blocks_.end()) {
+          return Status::KeyError("No datetimetz block allocated");
+        }
+        block = it->second;
+      } else {
+        auto it = this->blocks_.find(output_type);
+        if (it == this->blocks_.end()) { return Status::KeyError("No block allocated"); }
+        block = it->second;
+      }
+      return block->Write(col, i, rel_placement);
+    };
 
-template <>
-inline Status ArrowSerializer<NPY_BOOL>::ConvertData() {
-  if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
+    nthreads = std::min<int>(nthreads, table_->num_columns());
 
-  int nbytes = BitUtil::BytesForBits(length_);
-  auto buffer = std::make_shared<arrow::PoolBuffer>(pool_);
-  RETURN_NOT_OK(buffer->Resize(nbytes));
+    if (nthreads == 1) {
+      for (int i = 0; i < table_->num_columns(); ++i) {
+        RETURN_NOT_OK(WriteColumn(i));
+      }
+    } else {
+      std::vector<std::thread> thread_pool;
+      thread_pool.reserve(nthreads);
+      std::atomic<int> task_counter(0);
 
-  const uint8_t* values = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
+      std::mutex error_mtx;
+      bool error_occurred = false;
+      Status error;
 
-  uint8_t* bitmap = buffer->mutable_data();
+      for (int thread_id = 0; thread_id < nthreads; ++thread_id) {
+        thread_pool.emplace_back(
+            [this, &error, &error_occurred, &error_mtx, &task_counter, &WriteColumn]() {
+              int column_num;
+              while (!error_occurred) {
+                column_num = task_counter.fetch_add(1);
+                if (column_num >= this->table_->num_columns()) { break; }
+                Status s = WriteColumn(column_num);
+                if (!s.ok()) {
+                  std::lock_guard<std::mutex> lock(error_mtx);
+                  error_occurred = true;
+                  error = s;
+                  break;
+                }
+              }
+            });
+      }
+      for (auto&& thread : thread_pool) {
+        thread.join();
+      }
 
-  memset(bitmap, 0, nbytes);
-  for (int i = 0; i < length_; ++i) {
-    if (values[i] > 0) { BitUtil::SetBit(bitmap, i); }
+      if (error_occurred) { return error; }
+    }
+    return Status::OK();
   }
 
-  data_ = buffer;
-
-  return Status::OK();
-}
-
-template <int TYPE>
-template <int ITEM_TYPE, typename ArrowType>
-inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
-    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-  typedef npy_traits<ITEM_TYPE> traits;
-  typedef typename traits::value_type T;
-  typedef typename traits::BuilderClass BuilderT;
-  PyAcquireGIL lock;
-
-  auto value_builder = std::make_shared<BuilderT>(pool_, field->type);
-  ListBuilder list_builder(pool_, value_builder);
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  for (int64_t i = 0; i < length_; ++i) {
-    if (PyObject_is_null(objects[i])) {
-      RETURN_NOT_OK(list_builder.AppendNull());
-    } else if (PyArray_Check(objects[i])) {
-      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
-      RETURN_NOT_OK(list_builder.Append(true));
-
-      // TODO(uwe): Support more complex numpy array structures
-      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
-
-      int64_t size = PyArray_DIM(numpy_array, 0);
-      auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
-      if (traits::supports_nulls) {
-        null_bitmap_->Resize(size, false);
-        // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
-        // currently support this.
-        // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        ValuesToBytemap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        RETURN_NOT_OK(value_builder->Append(data, size, null_bitmap_->data()));
-      } else {
-        RETURN_NOT_OK(value_builder->Append(data, size));
-      }
-    } else if (PyList_Check(objects[i])) {
-      int64_t size;
-      std::shared_ptr<arrow::DataType> type;
-      RETURN_NOT_OK(list_builder.Append(true));
-      RETURN_NOT_OK(InferArrowType(objects[i], &size, &type));
-      if (type->type != field->type->type) {
-        std::stringstream ss;
-        ss << type->ToString() << " cannot be converted to " << field->type->ToString();
-        return Status::TypeError(ss.str());
-      }
-      RETURN_NOT_OK(AppendPySequence(objects[i], field->type, value_builder));
-    } else {
-      return Status::TypeError("Unsupported Python type for list items");
+  Status AppendBlocks(const BlockMap& blocks, PyObject* list) {
+    for (const auto& it : blocks) {
+      PyObject* item;
+      RETURN_NOT_OK(it.second->GetPyResult(&item));
+      if (PyList_Append(list, item) < 0) { RETURN_IF_PYERROR(); }
     }
+    return Status::OK();
   }
-  return list_builder.Finish(out);
-}
 
-template <>
-template <>
-inline Status
-ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
-    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-  // TODO: If there are bytes involed, convert to Binary representation
-  PyAcquireGIL lock;
-  bool have_bytes = false;
+  Status GetResultList(PyObject** out) {
+    PyAcquireGIL lock;
 
-  auto value_builder = std::make_shared<arrow::StringBuilder>(pool_);
-  ListBuilder list_builder(pool_, value_builder);
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  for (int64_t i = 0; i < length_; ++i) {
-    if (PyObject_is_null(objects[i])) {
-      RETURN_NOT_OK(list_builder.AppendNull());
-    } else if (PyArray_Check(objects[i])) {
-      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
-      RETURN_NOT_OK(list_builder.Append(true));
+    PyObject* result = PyList_New(0);
+    RETURN_IF_PYERROR();
 
-      // TODO(uwe): Support more complex numpy array structures
-      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+    RETURN_NOT_OK(AppendBlocks(blocks_, result));
+    RETURN_NOT_OK(AppendBlocks(categorical_blocks_, result));
+    RETURN_NOT_OK(AppendBlocks(datetimetz_blocks_, result));
 
-      int64_t size = PyArray_DIM(numpy_array, 0);
-      auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
-      RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
-    } else if (PyList_Check(objects[i])) {
-      int64_t size;
-      std::shared_ptr<arrow::DataType> type;
-      RETURN_NOT_OK(list_builder.Append(true));
-      RETURN_NOT_OK(InferArrowType(objects[i], &size, &type));
-      if (type->type != Type::STRING) {
-        std::stringstream ss;
-        ss << type->ToString() << " cannot be converted to STRING.";
-        return Status::TypeError(ss.str());
-      }
-      RETURN_NOT_OK(AppendPySequence(objects[i], type, value_builder));
-    } else {
-      return Status::TypeError("Unsupported Python type for list items");
-    }
+    *out = result;
+    return Status::OK();
   }
-  return list_builder.Finish(out);
-}
 
-template <>
-inline Status ArrowSerializer<NPY_OBJECT>::ConvertData() {
-  return Status::TypeError("NYI");
-}
-
-#define TO_ARROW_CASE(TYPE)                                 \
-  case NPY_##TYPE: {                                        \
-    ArrowSerializer<NPY_##TYPE> converter(pool, arr, mask); \
-    RETURN_NOT_OK(converter.Convert(out));                  \
-  } break;
+ private:
+  std::shared_ptr<Table> table_;
 
-Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
-  PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(ao);
-  PyArrayObject* mask = nullptr;
+  // column num -> block type id
+  std::vector<PandasBlock::type> column_types_;
 
-  if (mo != nullptr and mo != Py_None) { mask = reinterpret_cast<PyArrayObject*>(mo); }
+  // column num -> relative placement within internal block
+  std::vector<int> column_block_placement_;
 
-  if (PyArray_NDIM(arr) != 1) {
-    return Status::Invalid("only handle 1-dimensional arrays");
-  }
+  // block type -> type count
+  std::unordered_map<int, int> type_counts_;
 
-  int type_num = PyArray_DESCR(arr)->type_num;
+  // block type -> block
+  BlockMap blocks_;
 
-#if (NPY_INT64 == NPY_LONGLONG) && (NPY_SIZEOF_LONGLONG == 8)
-  // Both LONGLONG and INT64 can be observed in the wild, which is buggy. We set
-  // U/LONGLONG to U/INT64 so things work properly.
-  if (type_num == NPY_LONGLONG) { type_num = NPY_INT64; }
-  if (type_num == NPY_ULONGLONG) { type_num = NPY_UINT64; }
-#endif
+  // column number -> categorical block
+  BlockMap categorical_blocks_;
 
-  switch (type_num) {
-    TO_ARROW_CASE(BOOL);
-    TO_ARROW_CASE(INT8);
-    TO_ARROW_CASE(INT16);
-    TO_ARROW_CASE(INT32);
-    TO_ARROW_CASE(INT64);
-#if (NPY_INT64 != NPY_LONGLONG)
-    TO_ARROW_CASE(LONGLONG);
-#endif
-    TO_ARROW_CASE(UINT8);
-    TO_ARROW_CASE(UINT16);
-    TO_ARROW_CASE(UINT32);
-    TO_ARROW_CASE(UINT64);
-#if (NPY_UINT64 != NPY_ULONGLONG)
-    TO_ARROW_CASE(ULONGLONG);
-#endif
-    TO_ARROW_CASE(FLOAT32);
-    TO_ARROW_CASE(FLOAT64);
-    TO_ARROW_CASE(DATETIME);
-    case NPY_OBJECT: {
-      ArrowSerializer<NPY_OBJECT> converter(pool, arr, mask);
-      converter.IndicateType(field);
-      RETURN_NOT_OK(converter.Convert(out));
-    } break;
-    default:
-      std::stringstream ss;
-      ss << "Unsupported numpy type " << PyArray_DESCR(arr)->type_num << std::endl;
-      return Status::NotImplemented(ss.str());
-  }
-  return Status::OK();
-}
+  // column number -> datetimetz block
+  BlockMap datetimetz_blocks_;
+};
 
 class ArrowDeserializer {
  public:
@@ -1839,7 +1694,7 @@ class ArrowDeserializer {
   Status ConvertValuesZeroCopy(int npy_type, std::shared_ptr<Array> arr) {
     typedef typename arrow_traits<TYPE>::T T;
 
-    auto prim_arr = static_cast<arrow::PrimitiveArray*>(arr.get());
+    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
     auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
 
     // Zero-Copy. We can pass the data pointer directly to NumPy.
@@ -1988,19 +1843,19 @@ class ArrowDeserializer {
   inline typename std::enable_if<TYPE == Type::STRING, Status>::type ConvertValues() {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertBinaryLike<arrow::StringArray>(data_, out_values);
+    return ConvertBinaryLike<StringArray>(data_, out_values);
   }
 
   template <int T2>
   inline typename std::enable_if<T2 == Type::BINARY, Status>::type ConvertValues() {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertBinaryLike<arrow::BinaryArray>(data_, out_values);
+    return ConvertBinaryLike<BinaryArray>(data_, out_values);
   }
 
 #define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \
   case Type::ArrowEnum:                                    \
-    return ConvertListsLike<::arrow::ArrowType>(col_, out_values);
+    return ConvertListsLike<ArrowType>(col_, out_values);
 
   template <int T2>
   inline typename std::enable_if<T2 == Type::LIST, Status>::type ConvertValues() {
@@ -2051,7 +1906,7 @@ class ArrowDeserializer {
 
  private:
   std::shared_ptr<Column> col_;
-  const arrow::ChunkedArray& data_;
+  const ChunkedArray& data_;
   PyObject* py_ref_;
   PyArrayObject* arr_;
   PyObject* result_;
@@ -2071,4 +1926,11 @@ Status ConvertColumnToPandas(
   return converter.Convert(out);
 }
 
-}  // namespace pyarrow
+Status ConvertTableToPandas(
+    const std::shared_ptr<Table>& table, int nthreads, PyObject** out) {
+  DataFrameBlockCreator helper(table);
+  return helper.Convert(nthreads, out);
+}
+
+}  // namespace py
+}  // namespace arrow
diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h
index b548f9321d75a..6862339d89baf 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/python/src/pyarrow/adapters/pandas.h
@@ -25,28 +25,26 @@
 
 #include <memory>
 
-#include "pyarrow/visibility.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class Array;
 class Column;
-class Field;
+class DataType;
 class MemoryPool;
 class Status;
 class Table;
 
-}  // namespace arrow
-
-namespace pyarrow {
+namespace py {
 
-PYARROW_EXPORT
-arrow::Status ConvertArrayToPandas(
-    const std::shared_ptr<arrow::Array>& arr, PyObject* py_ref, PyObject** out);
+ARROW_EXPORT
+Status ConvertArrayToPandas(
+    const std::shared_ptr<Array>& arr, PyObject* py_ref, PyObject** out);
 
-PYARROW_EXPORT
-arrow::Status ConvertColumnToPandas(
-    const std::shared_ptr<arrow::Column>& col, PyObject* py_ref, PyObject** out);
+ARROW_EXPORT
+Status ConvertColumnToPandas(
+    const std::shared_ptr<Column>& col, PyObject* py_ref, PyObject** out);
 
 struct PandasOptions {
   bool strings_to_categorical;
@@ -58,14 +56,24 @@ struct PandasOptions {
 // BlockManager structure of the pandas.DataFrame used as of pandas 0.19.x.
 //
 // tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2])
-PYARROW_EXPORT
-arrow::Status ConvertTableToPandas(
-    const std::shared_ptr<arrow::Table>& table, int nthreads, PyObject** out);
+ARROW_EXPORT
+Status ConvertTableToPandas(
+    const std::shared_ptr<Table>& table, int nthreads, PyObject** out);
+
+ARROW_EXPORT
+Status PandasDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out);
 
-PYARROW_EXPORT
-arrow::Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<arrow::Field>& field, std::shared_ptr<arrow::Array>* out);
+ARROW_EXPORT
+Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
 
-}  // namespace pyarrow
+/// Convert dtype=object arrays. If target data type is not known, pass a type
+/// with nullptr
+ARROW_EXPORT
+Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
+
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_ADAPTERS_PANDAS_H
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index d2f5291ea8301..c898f634aedbb 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -24,24 +24,23 @@
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 
-using arrow::Status;
-
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 static std::mutex memory_pool_mutex;
-static arrow::MemoryPool* default_pyarrow_pool = nullptr;
+static MemoryPool* default_pyarrow_pool = nullptr;
 
-void set_default_memory_pool(arrow::MemoryPool* pool) {
+void set_default_memory_pool(MemoryPool* pool) {
   std::lock_guard<std::mutex> guard(memory_pool_mutex);
   default_pyarrow_pool = pool;
 }
 
-arrow::MemoryPool* get_memory_pool() {
+MemoryPool* get_memory_pool() {
   std::lock_guard<std::mutex> guard(memory_pool_mutex);
   if (default_pyarrow_pool) {
     return default_pyarrow_pool;
   } else {
-    return arrow::default_memory_pool();
+    return default_memory_pool();
   }
 }
 
@@ -60,4 +59,5 @@ PyBytesBuffer::~PyBytesBuffer() {
   Py_DECREF(obj_);
 }
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index ad65ec75eec9e..0b4c6bebcfe79 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -19,16 +19,16 @@
 #define PYARROW_COMMON_H
 
 #include "pyarrow/config.h"
-#include "pyarrow/visibility.h"
 
 #include "arrow/buffer.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
+
 class MemoryPool;
-}
 
-namespace pyarrow {
+namespace py {
 
 class PyAcquireGIL {
  public:
@@ -98,10 +98,10 @@ struct PyObjectStringify {
   }
 
 // Return the common PyArrow memory pool
-PYARROW_EXPORT void set_default_memory_pool(arrow::MemoryPool* pool);
-PYARROW_EXPORT arrow::MemoryPool* get_memory_pool();
+ARROW_EXPORT void set_default_memory_pool(MemoryPool* pool);
+ARROW_EXPORT MemoryPool* get_memory_pool();
 
-class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
+class ARROW_EXPORT NumPyBuffer : public Buffer {
  public:
   NumPyBuffer(PyArrayObject* arr) : Buffer(nullptr, 0) {
     arr_ = arr;
@@ -118,7 +118,7 @@ class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer {
   PyArrayObject* arr_;
 };
 
-class PYARROW_EXPORT PyBytesBuffer : public arrow::Buffer {
+class ARROW_EXPORT PyBytesBuffer : public Buffer {
  public:
   PyBytesBuffer(PyObject* obj);
   ~PyBytesBuffer();
@@ -127,6 +127,7 @@ class PYARROW_EXPORT PyBytesBuffer : public arrow::Buffer {
   PyObject* obj_;
 };
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_COMMON_H
diff --git a/python/src/pyarrow/config.cc b/python/src/pyarrow/config.cc
index e1002bf4fd146..0be6d962b55ab 100644
--- a/python/src/pyarrow/config.cc
+++ b/python/src/pyarrow/config.cc
@@ -19,7 +19,8 @@
 
 #include "pyarrow/config.h"
 
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 void pyarrow_init() {}
 
@@ -30,4 +31,5 @@ void pyarrow_set_numpy_nan(PyObject* obj) {
   numpy_nan = obj;
 }
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
diff --git a/python/src/pyarrow/config.h b/python/src/pyarrow/config.h
index 386ee4b1e2590..87fc5c2b290f6 100644
--- a/python/src/pyarrow/config.h
+++ b/python/src/pyarrow/config.h
@@ -20,24 +20,27 @@
 
 #include <Python.h>
 
+#include "arrow/util/visibility.h"
+
 #include "pyarrow/numpy_interop.h"
-#include "pyarrow/visibility.h"
 
 #if PY_MAJOR_VERSION >= 3
 #define PyString_Check PyUnicode_Check
 #endif
 
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
-PYARROW_EXPORT
+ARROW_EXPORT
 extern PyObject* numpy_nan;
 
-PYARROW_EXPORT
+ARROW_EXPORT
 void pyarrow_init();
 
-PYARROW_EXPORT
+ARROW_EXPORT
 void pyarrow_set_numpy_nan(PyObject* obj);
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_CONFIG_H
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
index 78fad165ac8e6..edebea6d97c95 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/python/src/pyarrow/helpers.cc
@@ -19,9 +19,8 @@
 
 #include <arrow/api.h>
 
-using namespace arrow;
-
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 #define GET_PRIMITIVE_TYPE(NAME, FACTORY) \
   case Type::NAME:                        \
@@ -51,4 +50,5 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
   }
 }
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
diff --git a/python/src/pyarrow/helpers.h b/python/src/pyarrow/helpers.h
index 788c3eedddfd6..611e814b7d858 100644
--- a/python/src/pyarrow/helpers.h
+++ b/python/src/pyarrow/helpers.h
@@ -18,19 +18,18 @@
 #ifndef PYARROW_HELPERS_H
 #define PYARROW_HELPERS_H
 
-#include <arrow/api.h>
 #include <memory>
 
-#include "pyarrow/visibility.h"
+#include "arrow/type.h"
+#include "arrow/util/visibility.h"
 
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
-using arrow::DataType;
-using arrow::Type;
-
-PYARROW_EXPORT
+ARROW_EXPORT
 std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_HELPERS_H
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index aa4cb7b052c27..0aa61dc811f5c 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -26,9 +26,8 @@
 
 #include "pyarrow/common.h"
 
-using arrow::Status;
-
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 // ----------------------------------------------------------------------
 // Python file
@@ -151,7 +150,7 @@ Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
   return Status::OK();
 }
 
-Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) {
+Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   PyAcquireGIL lock;
 
   PyObject* bytes_obj;
@@ -214,8 +213,9 @@ Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
 // A readable file that is backed by a PyBytes
 
 PyBytesReader::PyBytesReader(PyObject* obj)
-    : arrow::io::BufferReader(std::make_shared<PyBytesBuffer>(obj)) {}
+    : io::BufferReader(std::make_shared<PyBytesBuffer>(obj)) {}
 
 PyBytesReader::~PyBytesReader() {}
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
diff --git a/python/src/pyarrow/io.h b/python/src/pyarrow/io.h
index 4cb010f2d4e9f..a603e81622545 100644
--- a/python/src/pyarrow/io.h
+++ b/python/src/pyarrow/io.h
@@ -20,17 +20,17 @@
 
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
+#include "arrow/util/visibility.h"
 
 #include "pyarrow/config.h"
 
 #include "pyarrow/common.h"
-#include "pyarrow/visibility.h"
 
 namespace arrow {
+
 class MemoryPool;
-}
 
-namespace pyarrow {
+namespace py {
 
 // A common interface to a Python file-like object. Must acquire GIL before
 // calling any methods
@@ -39,31 +39,31 @@ class PythonFile {
   PythonFile(PyObject* file);
   ~PythonFile();
 
-  arrow::Status Close();
-  arrow::Status Seek(int64_t position, int whence);
-  arrow::Status Read(int64_t nbytes, PyObject** out);
-  arrow::Status Tell(int64_t* position);
-  arrow::Status Write(const uint8_t* data, int64_t nbytes);
+  Status Close();
+  Status Seek(int64_t position, int whence);
+  Status Read(int64_t nbytes, PyObject** out);
+  Status Tell(int64_t* position);
+  Status Write(const uint8_t* data, int64_t nbytes);
 
  private:
   PyObject* file_;
 };
 
-class PYARROW_EXPORT PyReadableFile : public arrow::io::ReadableFileInterface {
+class ARROW_EXPORT PyReadableFile : public io::ReadableFileInterface {
  public:
   explicit PyReadableFile(PyObject* file);
   virtual ~PyReadableFile();
 
-  arrow::Status Close() override;
+  Status Close() override;
 
-  arrow::Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
-  arrow::Status Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
-  arrow::Status GetSize(int64_t* size) override;
+  Status GetSize(int64_t* size) override;
 
-  arrow::Status Seek(int64_t position) override;
+  Status Seek(int64_t position) override;
 
-  arrow::Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) override;
 
   bool supports_zero_copy() const override;
 
@@ -71,21 +71,21 @@ class PYARROW_EXPORT PyReadableFile : public arrow::io::ReadableFileInterface {
   std::unique_ptr<PythonFile> file_;
 };
 
-class PYARROW_EXPORT PyOutputStream : public arrow::io::OutputStream {
+class ARROW_EXPORT PyOutputStream : public io::OutputStream {
  public:
   explicit PyOutputStream(PyObject* file);
   virtual ~PyOutputStream();
 
-  arrow::Status Close() override;
-  arrow::Status Tell(int64_t* position) override;
-  arrow::Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Close() override;
+  Status Tell(int64_t* position) override;
+  Status Write(const uint8_t* data, int64_t nbytes) override;
 
  private:
   std::unique_ptr<PythonFile> file_;
 };
 
 // A zero-copy reader backed by a PyBytes object
-class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader {
+class ARROW_EXPORT PyBytesReader : public io::BufferReader {
  public:
   explicit PyBytesReader(PyObject* obj);
   virtual ~PyBytesReader();
@@ -93,6 +93,7 @@ class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader {
 
 // TODO(wesm): seekable output files
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_IO_H
diff --git a/python/src/pyarrow/numpy_interop.h b/python/src/pyarrow/numpy_interop.h
index 6326527a67420..57f3328e87078 100644
--- a/python/src/pyarrow/numpy_interop.h
+++ b/python/src/pyarrow/numpy_interop.h
@@ -42,7 +42,8 @@
 #include <numpy/arrayobject.h>
 #include <numpy/ufuncobject.h>
 
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 inline int import_numpy() {
 #ifdef NUMPY_IMPORT_ARRAY
@@ -53,6 +54,7 @@ inline int import_numpy() {
   return 0;
 }
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_NUMPY_INTEROP_H
diff --git a/python/src/pyarrow/type_traits.h b/python/src/pyarrow/type_traits.h
new file mode 100644
index 0000000000000..f4604d7a9894d
--- /dev/null
+++ b/python/src/pyarrow/type_traits.h
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <Python.h>
+
+#include <cstdint>
+
+#include "pyarrow/numpy_interop.h"
+
+#include "arrow/builder.h"
+#include "arrow/type.h"
+
+namespace arrow {
+namespace py {
+
+template <int TYPE>
+struct npy_traits {};
+
+template <>
+struct npy_traits<NPY_BOOL> {
+  typedef uint8_t value_type;
+  using TypeClass = BooleanType;
+  using BuilderClass = BooleanBuilder;
+
+  static constexpr bool supports_nulls = false;
+  static inline bool isnull(uint8_t v) { return false; }
+};
+
+#define NPY_INT_DECL(TYPE, CapType, T)               \
+  template <>                                        \
+  struct npy_traits<NPY_##TYPE> {                    \
+    typedef T value_type;                            \
+    using TypeClass = CapType##Type;                 \
+    using BuilderClass = CapType##Builder;           \
+                                                     \
+    static constexpr bool supports_nulls = false;    \
+    static inline bool isnull(T v) { return false; } \
+  };
+
+NPY_INT_DECL(INT8, Int8, int8_t);
+NPY_INT_DECL(INT16, Int16, int16_t);
+NPY_INT_DECL(INT32, Int32, int32_t);
+NPY_INT_DECL(INT64, Int64, int64_t);
+
+NPY_INT_DECL(UINT8, UInt8, uint8_t);
+NPY_INT_DECL(UINT16, UInt16, uint16_t);
+NPY_INT_DECL(UINT32, UInt32, uint32_t);
+NPY_INT_DECL(UINT64, UInt64, uint64_t);
+
+#if NPY_INT64 != NPY_LONGLONG
+NPY_INT_DECL(LONGLONG, Int64, int64_t);
+NPY_INT_DECL(ULONGLONG, UInt64, uint64_t);
+#endif
+
+template <>
+struct npy_traits<NPY_FLOAT32> {
+  typedef float value_type;
+  using TypeClass = FloatType;
+  using BuilderClass = FloatBuilder;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(float v) { return v != v; }
+};
+
+template <>
+struct npy_traits<NPY_FLOAT64> {
+  typedef double value_type;
+  using TypeClass = DoubleType;
+  using BuilderClass = DoubleBuilder;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(double v) { return v != v; }
+};
+
+template <>
+struct npy_traits<NPY_DATETIME> {
+  typedef int64_t value_type;
+  using TypeClass = TimestampType;
+  using BuilderClass = TimestampBuilder;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(int64_t v) {
+    // NaT = -2**63
+    // = -0x8000000000000000
+    // = -9223372036854775808;
+    // = std::numeric_limits<int64_t>::min()
+    return v == std::numeric_limits<int64_t>::min();
+  }
+};
+
+template <>
+struct npy_traits<NPY_OBJECT> {
+  typedef PyObject* value_type;
+  static constexpr bool supports_nulls = true;
+};
+
+template <int TYPE>
+struct arrow_traits {};
+
+template <>
+struct arrow_traits<Type::BOOL> {
+  static constexpr int npy_type = NPY_BOOL;
+  static constexpr bool supports_nulls = false;
+  static constexpr bool is_boolean = true;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
+
+#define INT_DECL(TYPE)                                     \
+  template <>                                              \
+  struct arrow_traits<Type::TYPE> {                        \
+    static constexpr int npy_type = NPY_##TYPE;            \
+    static constexpr bool supports_nulls = false;          \
+    static constexpr double na_value = NAN;                \
+    static constexpr bool is_boolean = false;              \
+    static constexpr bool is_numeric_not_nullable = true;  \
+    static constexpr bool is_numeric_nullable = false;     \
+    typedef typename npy_traits<NPY_##TYPE>::value_type T; \
+  };
+
+INT_DECL(INT8);
+INT_DECL(INT16);
+INT_DECL(INT32);
+INT_DECL(INT64);
+INT_DECL(UINT8);
+INT_DECL(UINT16);
+INT_DECL(UINT32);
+INT_DECL(UINT64);
+
+template <>
+struct arrow_traits<Type::FLOAT> {
+  static constexpr int npy_type = NPY_FLOAT32;
+  static constexpr bool supports_nulls = true;
+  static constexpr float na_value = NAN;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_FLOAT32>::value_type T;
+};
+
+template <>
+struct arrow_traits<Type::DOUBLE> {
+  static constexpr int npy_type = NPY_FLOAT64;
+  static constexpr bool supports_nulls = true;
+  static constexpr double na_value = NAN;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_FLOAT64>::value_type T;
+};
+
+static constexpr int64_t kPandasTimestampNull = std::numeric_limits<int64_t>::min();
+
+template <>
+struct arrow_traits<Type::TIMESTAMP> {
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr bool supports_nulls = true;
+  static constexpr int64_t na_value = kPandasTimestampNull;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
+
+template <>
+struct arrow_traits<Type::DATE> {
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr bool supports_nulls = true;
+  static constexpr int64_t na_value = kPandasTimestampNull;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
+
+template <>
+struct arrow_traits<Type::STRING> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
+
+template <>
+struct arrow_traits<Type::BINARY> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
+
+}  // namespace py
+}  // namespace arrow
diff --git a/python/src/pyarrow/util/datetime.h b/python/src/pyarrow/util/datetime.h
index 9ffa691052460..f704a96d91bba 100644
--- a/python/src/pyarrow/util/datetime.h
+++ b/python/src/pyarrow/util/datetime.h
@@ -21,7 +21,8 @@
 #include <Python.h>
 #include <datetime.h>
 
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   struct tm date = {0};
@@ -35,6 +36,7 @@ inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
 }
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_UTIL_DATETIME_H
diff --git a/python/src/pyarrow/util/test_main.cc b/python/src/pyarrow/util/test_main.cc
index 02e9a54f65914..d8d1d030f8f97 100644
--- a/python/src/pyarrow/util/test_main.cc
+++ b/python/src/pyarrow/util/test_main.cc
@@ -26,7 +26,7 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
 
   Py_Initialize();
-  pyarrow::import_numpy();
+  arrow::py::import_numpy();
 
   int ret = RUN_ALL_TESTS();
 
diff --git a/python/src/pyarrow/visibility.h b/python/src/pyarrow/visibility.h
deleted file mode 100644
index 9f0c13b4b2083..0000000000000
--- a/python/src/pyarrow/visibility.h
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PYARROW_VISIBILITY_H
-#define PYARROW_VISIBILITY_H
-
-#if defined(_WIN32) || defined(__CYGWIN__)
-#define PYARROW_EXPORT __declspec(dllexport)
-#else  // Not Windows
-#ifndef PYARROW_EXPORT
-#define PYARROW_EXPORT __attribute__((visibility("default")))
-#endif
-#ifndef PYARROW_NO_EXPORT
-#define PYARROW_NO_EXPORT __attribute__((visibility("hidden")))
-#endif
-#endif  // Non-Windows
-
-#endif  // PYARROW_VISIBILITY_H

From 6aed18f965bea60580e80b086dd72857546abea2 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 13 Mar 2017 16:23:22 -0400
Subject: [PATCH 0363/1644] ARROW-619: [Python] Fixed remaining typo for
 LD_LIBRARY_PATH

Typo with LD_LIBRARY_PATH in install documentation.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #376 from BryanCutler/pyarrow-install-typo-ARROW-619 and squashes the following commits:

e38b588 [Bryan Cutler] fixed typo for LD_LIBRARY_PATH
---
 python/doc/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/doc/install.rst b/python/doc/install.rst
index d93a88f547576..16d19ef123135 100644
--- a/python/doc/install.rst
+++ b/python/doc/install.rst
@@ -133,7 +133,7 @@ Install `pyarrow`
 
 .. note::
     In development installations, you will also need to set a correct
-    ``LD_LIBARY_PATH``. This is most probably done with
+    ``LD_LIBRARY_PATH``. This is most probably done with
     ``export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH``.
 
 

From f442879d3c791d86fb0fdfa098a72329843f5baf Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 14 Mar 2017 09:17:30 +0100
Subject: [PATCH 0364/1644] ARROW-624: [C++] Restore MakePrimitiveArray
 function, use in feather.cc

I verified locally the parquet-cpp test suite passes again

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #378 from wesm/ARROW-624 and squashes the following commits:

023df9b [Wes McKinney] Use passed offset in MakePrimitiveArray
30a553e [Wes McKinney] Restore MakePrimitiveArray function, use in Feather, verify fixes parquet test suite
---
 cpp/src/arrow/api.h          |  3 +++
 cpp/src/arrow/ipc/feather.cc |  8 +-------
 cpp/src/arrow/loader.cc      | 28 +++++++++++++++++++++++-----
 cpp/src/arrow/loader.h       | 10 ++++++++++
 4 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 51437d863b8b9..3bc86662613ed 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -24,7 +24,10 @@
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
 #include "arrow/column.h"
+#include "arrow/compare.h"
+#include "arrow/loader.h"
 #include "arrow/memory_pool.h"
+#include "arrow/pretty_print.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 13dfa5830f1bf..1d165acccbd04 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -331,7 +331,6 @@ class TableReader::TableReaderImpl {
     std::shared_ptr<DataType> type;
     RETURN_NOT_OK(GetDataType(meta, metadata_type, metadata, &type));
 
-    std::vector<FieldMetadata> fields(1);
     std::vector<std::shared_ptr<Buffer>> buffers;
 
     // Buffer data from the source (may or may not perform a copy depending on
@@ -357,12 +356,7 @@ class TableReader::TableReaderImpl {
     }
 
     buffers.push_back(SliceBuffer(buffer, offset, buffer->size() - offset));
-
-    fields[0].length = meta->length();
-    fields[0].null_count = meta->null_count();
-    fields[0].offset = 0;
-
-    return LoadArray(type, fields, buffers, out);
+    return MakePrimitiveArray(type, buffers, meta->length(), meta->null_count(), 0, out);
   }
 
   bool HasDescription() const { return metadata_->HasDescription(); }
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
index 3cb51ae8fdab7..0b3ee1cf0a899 100644
--- a/cpp/src/arrow/loader.cc
+++ b/cpp/src/arrow/loader.cc
@@ -235,8 +235,8 @@ class ArrayLoader : public TypeVisitor {
   std::shared_ptr<Array> result_;
 };
 
-Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
-    ArrayComponentSource* source, std::shared_ptr<Array>* out) {
+Status LoadArray(const std::shared_ptr<DataType>& type, ArrayComponentSource* source,
+    std::shared_ptr<Array>* out) {
   ArrayLoaderContext context;
   context.source = source;
   context.field_index = context.buffer_index = 0;
@@ -244,8 +244,8 @@ Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
   return LoadArray(type, &context, out);
 }
 
-Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
-    ArrayLoaderContext* context, std::shared_ptr<Array>* out) {
+Status LoadArray(const std::shared_ptr<DataType>& type, ArrayLoaderContext* context,
+    std::shared_ptr<Array>* out) {
   ArrayLoader loader(type, context);
   RETURN_NOT_OK(loader.Load(out));
 
@@ -275,11 +275,29 @@ class InMemorySource : public ArrayComponentSource {
   const std::vector<std::shared_ptr<Buffer>>& buffers_;
 };
 
-Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
+Status LoadArray(const std::shared_ptr<DataType>& type,
     const std::vector<FieldMetadata>& fields,
     const std::vector<std::shared_ptr<Buffer>>& buffers, std::shared_ptr<Array>* out) {
   InMemorySource source(fields, buffers);
   return LoadArray(type, &source, out);
 }
 
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
+    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
+    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
+  std::vector<std::shared_ptr<Buffer>> buffers = {null_bitmap, data};
+  return MakePrimitiveArray(type, buffers, length, null_count, offset, out);
+}
+
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
+    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
+  std::vector<FieldMetadata> fields(1);
+  fields[0].length = length;
+  fields[0].null_count = null_count;
+  fields[0].offset = offset;
+
+  return LoadArray(type, fields, buffers, out);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/loader.h b/cpp/src/arrow/loader.h
index b4949f2556028..f116d64f5c0c1 100644
--- a/cpp/src/arrow/loader.h
+++ b/cpp/src/arrow/loader.h
@@ -84,6 +84,16 @@ Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
     const std::vector<FieldMetadata>& fields,
     const std::vector<std::shared_ptr<Buffer>>& buffers, std::shared_ptr<Array>* out);
 
+/// Create new arrays for logical types that are backed by primitive arrays.
+Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    int64_t length, const std::shared_ptr<Buffer>& data,
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset,
+    std::shared_ptr<Array>* out);
+
+Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
+    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out);
+
 }  // namespace arrow
 
 #endif  // ARROW_LOADER_H

From cef46152cc7489c23b67aaed70574dba742d19bb Mon Sep 17 00:00:00 2001
From: Jeff Knupp <jeff@jeffknupp.com>
Date: Tue, 14 Mar 2017 10:58:44 -0400
Subject: [PATCH 0365/1644] ARROW-598: [Python]  Add support for converting
 pyarrow.Buffer to a memoryview with zero copy

WIP, as tests are not all done and I'm assuming we'll need to keep a reference to the underlying buffer so it doesn't get gc'ed.

Author: Jeff Knupp <jeff@jeffknupp.com>
Author: Jeff Knupp <jeff.knupp@enigma.io>

Closes #369 from jeffknupp/master and squashes the following commits:

c300f30 [Jeff Knupp] Initialize members in init; test for lifetime with zero references
13f5dc1 [Jeff Knupp] WIP: python 2 compatability
170d01d [Jeff Knupp] WIP: python 2 compatability
bfbed0f [Jeff Knupp] WIP: add test for buffer protocol reference counting
fd1cb44 [Jeff Knupp] WIP: make buffers read-only; add test for immutability
c24e83a [Jeff Knupp] WIP: make arrow.io.Buffer implement Python's buffer protocol
b2540d4 [Jeff Knupp] ARROW-598: [Python]  Add support for converting pyarrow.Buffer to a memoryview with zero copy
---
 python/pyarrow/io.pxd           |  2 ++
 python/pyarrow/io.pyx           | 16 +++++++++++++-
 python/pyarrow/tests/test_io.py | 39 +++++++++++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
index fffc7c596db76..3d73e1143e15a 100644
--- a/python/pyarrow/io.pxd
+++ b/python/pyarrow/io.pxd
@@ -25,6 +25,8 @@ from pyarrow.includes.libarrow_io cimport (ReadableFileInterface,
 cdef class Buffer:
     cdef:
         shared_ptr[CBuffer] buffer
+        Py_ssize_t shape[1]
+        Py_ssize_t strides[1]
 
     cdef init(self, const shared_ptr[CBuffer]& buffer)
 
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 4acef212b4dce..240ea240c3abe 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -56,7 +56,6 @@ cdef extern from "Python.h":
     PyObject* PyBytes_FromStringAndSizeNative" PyBytes_FromStringAndSize"(
         char *v, Py_ssize_t len) except NULL
 
-
 cdef class NativeFile:
 
     def __cinit__(self):
@@ -421,6 +420,8 @@ cdef class Buffer:
 
     cdef init(self, const shared_ptr[CBuffer]& buffer):
         self.buffer = buffer
+        self.shape[0] = self.size
+        self.strides[0] = <Py_ssize_t>(1)
 
     def __len__(self):
         return self.size
@@ -449,6 +450,19 @@ cdef class Buffer:
             <const char*>self.buffer.get().data(),
             self.buffer.get().size())
 
+    def __getbuffer__(self, cp.Py_buffer* buffer, int flags):
+
+        buffer.buf = <char *>self.buffer.get().data()
+        buffer.format = 'b'
+        buffer.internal = NULL
+        buffer.itemsize = 1
+        buffer.len = self.size
+        buffer.ndim = 1
+        buffer.obj = self
+        buffer.readonly = 1
+        buffer.shape = self.shape
+        buffer.strides = self.strides
+        buffer.suboffsets = NULL
 
 cdef shared_ptr[PoolBuffer] allocate_buffer(CMemoryPool* pool):
     cdef shared_ptr[PoolBuffer] result
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index dfa84a27e6be9..c6caba5ce641a 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -135,6 +135,34 @@ def test_buffer_bytes():
 
     assert result == val
 
+def test_buffer_memoryview():
+    val = b'some data'
+
+    buf = io.buffer_from_bytes(val)
+    assert isinstance(buf, io.Buffer)
+
+    result = memoryview(buf)
+
+    assert result == val
+
+
+def test_buffer_memoryview_is_immutable():
+    val = b'some data'
+
+    buf = io.buffer_from_bytes(val)
+    assert isinstance(buf, io.Buffer)
+
+    result = memoryview(buf)
+
+    with pytest.raises(TypeError) as exc:
+        result[0] = b'h'
+        assert 'cannot modify read-only' in str(exc.value)
+
+    b = bytes(buf)
+    with pytest.raises(TypeError) as exc:
+        b[0] = b'h'
+        assert 'cannot modify read-only' in str(exc.value)
+
 
 def test_memory_output_stream():
     # 10 bytes
@@ -160,6 +188,17 @@ def test_inmemory_write_after_closed():
     with pytest.raises(IOError):
         f.write(b'not ok')
 
+def test_buffer_protocol_ref_counting():
+    import gc
+
+    def make_buffer(bytes_obj):
+        return bytearray(io.buffer_from_bytes(bytes_obj))
+
+    buf = make_buffer(b'foo')
+    gc.collect()
+    assert buf == b'foo'
+
+
 
 # ----------------------------------------------------------------------
 # OS files and memory maps

From a32ae59094be82ad318a73d067f2db680d3ab295 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 14 Mar 2017 11:47:39 -0400
Subject: [PATCH 0366/1644] ARROW-625: [C++] Add TimeUnit to
 TimeType::ToString. Add timezone to TimestampType::ToString if present

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #377 from wesm/ARROW-625 and squashes the following commits:

d76a8d3 [Wes McKinney] Move PrintTimeUnit into operator<< for std::ostream
351f90e [Wes McKinney] Add TimeUnit to TimeType::ToString. Add timezone to TimestampType::ToString if it has one
---
 cpp/src/arrow/type-test.cc | 24 ++++++++++++++++++++++++
 cpp/src/arrow/type.cc      | 23 ++++++-----------------
 cpp/src/arrow/type.h       | 22 ++++++++++++++++++++--
 3 files changed, 50 insertions(+), 19 deletions(-)

diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index fe6c62adb7fba..3adc4d83c3a2d 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -132,6 +132,18 @@ TEST(TestTimeType, Equals) {
   ASSERT_TRUE(t3.Equals(t4));
 }
 
+TEST(TestTimeType, ToString) {
+  auto t1 = time(TimeUnit::MILLI);
+  auto t2 = time(TimeUnit::NANO);
+  auto t3 = time(TimeUnit::SECOND);
+  auto t4 = time(TimeUnit::MICRO);
+
+  ASSERT_EQ("time[ms]", t1->ToString());
+  ASSERT_EQ("time[ns]", t2->ToString());
+  ASSERT_EQ("time[s]", t3->ToString());
+  ASSERT_EQ("time[us]", t4->ToString());
+}
+
 TEST(TestTimestampType, Equals) {
   TimestampType t1;
   TimestampType t2;
@@ -143,4 +155,16 @@ TEST(TestTimestampType, Equals) {
   ASSERT_TRUE(t3.Equals(t4));
 }
 
+TEST(TestTimestampType, ToString) {
+  auto t1 = timestamp(TimeUnit::MILLI);
+  auto t2 = timestamp("US/Eastern", TimeUnit::NANO);
+  auto t3 = timestamp(TimeUnit::SECOND);
+  auto t4 = timestamp(TimeUnit::MICRO);
+
+  ASSERT_EQ("timestamp[ms]", t1->ToString());
+  ASSERT_EQ("timestamp[ns, tz=US/Eastern]", t2->ToString());
+  ASSERT_EQ("timestamp[s]", t3->ToString());
+  ASSERT_EQ("timestamp[us]", t4->ToString());
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 0cafdce89e562..d41b36315a86e 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -108,27 +108,16 @@ std::string Date32Type::ToString() const {
   return std::string("date32");
 }
 
-static inline void print_time_unit(TimeUnit unit, std::ostream* stream) {
-  switch (unit) {
-    case TimeUnit::SECOND:
-      (*stream) << "s";
-      break;
-    case TimeUnit::MILLI:
-      (*stream) << "ms";
-      break;
-    case TimeUnit::MICRO:
-      (*stream) << "us";
-      break;
-    case TimeUnit::NANO:
-      (*stream) << "ns";
-      break;
-  }
+std::string TimeType::ToString() const {
+  std::stringstream ss;
+  ss << "time[" << this->unit << "]";
+  return ss.str();
 }
 
 std::string TimestampType::ToString() const {
   std::stringstream ss;
-  ss << "timestamp[";
-  print_time_unit(this->unit, &ss);
+  ss << "timestamp[" << this->unit;
+  if (this->timezone.size() > 0) { ss << ", tz=" << this->timezone; }
   ss << "]";
   return ss.str();
 }
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 15b99c5ce4f89..9f28875925a4b 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <ostream>
 #include <string>
 #include <vector>
 
@@ -460,6 +461,24 @@ struct ARROW_EXPORT Date32Type : public FixedWidthType {
 
 enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
 
+static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) {
+  switch (unit) {
+    case TimeUnit::SECOND:
+      os << "s";
+      break;
+    case TimeUnit::MILLI:
+      os << "ms";
+      break;
+    case TimeUnit::MICRO:
+      os << "us";
+      break;
+    case TimeUnit::NANO:
+      os << "ns";
+      break;
+  }
+  return os;
+}
+
 struct ARROW_EXPORT TimeType : public FixedWidthType {
   static constexpr Type::type type_id = Type::TIME;
   using Unit = TimeUnit;
@@ -474,8 +493,7 @@ struct ARROW_EXPORT TimeType : public FixedWidthType {
   TimeType(const TimeType& other) : TimeType(other.unit) {}
 
   Status Accept(TypeVisitor* visitor) const override;
-  std::string ToString() const override { return name(); }
-  static std::string name() { return "time"; }
+  std::string ToString() const override;
 };
 
 struct ARROW_EXPORT TimestampType : public FixedWidthType {

From dd8204ce77662110996c6b24e28577c225ef5546 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 14 Mar 2017 12:16:23 -0400
Subject: [PATCH 0367/1644] ARROW-628: [Python] Install nomkl metapackage when
 building parquet-cpp in Travis CI

I was surprised to find conda installing the mkl conda package with what's there now, but this should fix that and make the builds slightly faster

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #380 from wesm/ARROW-628 and squashes the following commits:

3598dcf [Wes McKinney] Install nomkl metapackage when building parquet-cpp in Travis CI
---
 ci/travis_script_python.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 11d8d89ca7b6f..6f4b8e9a090a7 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -29,9 +29,14 @@ pushd $PYTHON_DIR
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 
 build_parquet_cpp() {
-  conda create -y -q -p $PARQUET_HOME thrift-cpp snappy zlib brotli boost
+  conda create -y -q -p $PARQUET_HOME python=3.5
   source activate $PARQUET_HOME
 
+  # In case some package wants to download the MKL
+  conda install -y -q nomkl
+
+  conda install -y -q thrift-cpp snappy zlib brotli boost
+
   export BOOST_ROOT=$PARQUET_HOME
   export SNAPPY_HOME=$PARQUET_HOME
   export THRIFT_HOME=$PARQUET_HOME

From c8d15d467f7a1950cf08bfcc1ead2e7ab828be00 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 15 Mar 2017 11:10:36 -0400
Subject: [PATCH 0368/1644] ARROW-491: [Format / C++] Add FixedWidthBinary type
 to format, C++ implementation

I have a bunch more work to do on the C++ implementation:

- [x] Test builder class
- [x] Test array API (slice, etc.)
- [x] Implement/test ArrayEquals/ArrayRangeEquals
- [x] Implement `PrettyPrint` (may want to encode to hexadecimal, I don't think that BinaryArray prints properly right now for non-ASCII/UTF8 data)
- [x] Add IPC roundtrip tests

In the meantime, @julienledem @nongli or others could you look at the changes to the format Flatbuffers and let me know if that looks right to you? Thanks

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #379 from wesm/ARROW-491 and squashes the following commits:

f948835 [Wes McKinney] Move hex encode/decode to a separate header since including io-util on Windows causes a compilation failure
949fbc8 [Wes McKinney] Hex encode values in binary and fixedwidthbinary. Test PrettyPrint for FW binary
a97c11a [Wes McKinney] Complete IPC implementation for date/time types. Implement IPC for FixedWidthBinary
b679264 [Wes McKinney] Fix bug with fast bitsetting when length is a power of 2
8e76225 [Wes McKinney] Do not needlessly create 0-length buffers
832b363 [Wes McKinney] Implement TypeEquals, ArrayRangeEquals, clang fixes
bf9ecd0 [Wes McKinney] cpplint
ec50654 [Wes McKinney] Add some basic tests for the fixed width binary builder
caa0314 [Wes McKinney] Draft FixedWidthBinaryBuilder. No tests yet
c183639 [Wes McKinney] Consolidate some type tests. Draft FixedWidthBinaryArray class
9143c53 [Wes McKinney] Draft FixedWidthBinaryType
---
 cpp/src/arrow/array-list-test.cc       |  20 ---
 cpp/src/arrow/array-string-test.cc     | 194 +++++++++++++++++++++++--
 cpp/src/arrow/array.cc                 | 139 +++++++++++-------
 cpp/src/arrow/array.h                  |  34 +++++
 cpp/src/arrow/buffer.h                 |  10 ++
 cpp/src/arrow/builder.cc               |  93 ++++++++++--
 cpp/src/arrow/builder.h                |  50 ++++---
 cpp/src/arrow/compare.cc               |  32 ++++
 cpp/src/arrow/ipc/adapter.cc           |  11 ++
 cpp/src/arrow/ipc/ipc-adapter-test.cc  |   4 +-
 cpp/src/arrow/ipc/ipc-file-test.cc     |   8 +-
 cpp/src/arrow/ipc/json-internal.cc     |  26 +---
 cpp/src/arrow/ipc/metadata-internal.cc |  74 +++++++++-
 cpp/src/arrow/ipc/test-common.h        |  65 ++++++++-
 cpp/src/arrow/loader.cc                |  12 ++
 cpp/src/arrow/pretty_print-test.cc     |  26 +++-
 cpp/src/arrow/pretty_print.cc          |  82 +++++++----
 cpp/src/arrow/type-test.cc             |  52 +++++++
 cpp/src/arrow/type.cc                  |  20 +++
 cpp/src/arrow/type.h                   |  30 +++-
 cpp/src/arrow/type_fwd.h               |   4 +
 cpp/src/arrow/type_traits.h            |   7 +
 cpp/src/arrow/util/io-util.h           |   5 +-
 cpp/src/arrow/util/string.h            |  57 ++++++++
 format/Message.fbs                     |   8 +-
 25 files changed, 870 insertions(+), 193 deletions(-)
 create mode 100644 cpp/src/arrow/util/string.h

diff --git a/cpp/src/arrow/array-list-test.cc b/cpp/src/arrow/array-list-test.cc
index a144fd937d7a0..87dfdaaed33a4 100644
--- a/cpp/src/arrow/array-list-test.cc
+++ b/cpp/src/arrow/array-list-test.cc
@@ -36,26 +36,6 @@ using std::vector;
 
 namespace arrow {
 
-TEST(TypesTest, TestListType) {
-  std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
-
-  ListType list_type(vt);
-  ASSERT_EQ(list_type.type, Type::LIST);
-
-  ASSERT_EQ(list_type.name(), string("list"));
-  ASSERT_EQ(list_type.ToString(), string("list<item: uint8>"));
-
-  ASSERT_EQ(list_type.value_type()->type, vt->type);
-  ASSERT_EQ(list_type.value_type()->type, vt->type);
-
-  std::shared_ptr<DataType> st = std::make_shared<StringType>();
-  std::shared_ptr<DataType> lt = std::make_shared<ListType>(st);
-  ASSERT_EQ(lt->ToString(), string("list<item: string>"));
-
-  ListType lt2(lt);
-  ASSERT_EQ(lt2.ToString(), string("list<item: list<item: string>>"));
-}
-
 // ----------------------------------------------------------------------
 // List tests
 
diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index 3fdeb3cefe7d2..cf2ff416032c6 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -33,22 +33,6 @@ namespace arrow {
 
 class Buffer;
 
-TEST(TypesTest, BinaryType) {
-  BinaryType t1;
-  BinaryType e1;
-  StringType t2;
-  EXPECT_TRUE(t1.Equals(e1));
-  EXPECT_FALSE(t1.Equals(t2));
-  ASSERT_EQ(t1.type, Type::BINARY);
-  ASSERT_EQ(t1.ToString(), std::string("binary"));
-}
-
-TEST(TypesTest, TestStringType) {
-  StringType str;
-  ASSERT_EQ(str.type, Type::STRING);
-  ASSERT_EQ(str.ToString(), std::string("string"));
-}
-
 // ----------------------------------------------------------------------
 // String container
 
@@ -474,4 +458,182 @@ TEST_F(TestBinaryArray, LengthZeroCtor) {
   BinaryArray array(0, nullptr, nullptr);
 }
 
+// ----------------------------------------------------------------------
+// FixedWidthBinary tests
+
+class TestFWBinaryArray : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  void InitBuilder(int byte_width) {
+    auto type = fixed_width_binary(byte_width);
+    builder_.reset(new FixedWidthBinaryBuilder(default_memory_pool(), type));
+  }
+
+ protected:
+  std::unique_ptr<FixedWidthBinaryBuilder> builder_;
+};
+
+TEST_F(TestFWBinaryArray, Builder) {
+  const int32_t byte_width = 10;
+  int64_t length = 4096;
+
+  int64_t nbytes = length * byte_width;
+
+  std::vector<uint8_t> data(nbytes);
+  test::random_bytes(nbytes, 0, data.data());
+
+  std::vector<uint8_t> is_valid(length);
+  test::random_null_bytes(length, 0.1, is_valid.data());
+
+  const uint8_t* raw_data = data.data();
+
+  std::shared_ptr<Array> result;
+
+  auto CheckResult = [this, &length, &is_valid, &raw_data, &byte_width](
+      const Array& result) {
+    // Verify output
+    const auto& fw_result = static_cast<const FixedWidthBinaryArray&>(result);
+
+    ASSERT_EQ(length, result.length());
+
+    for (int64_t i = 0; i < result.length(); ++i) {
+      if (is_valid[i]) {
+        ASSERT_EQ(
+            0, memcmp(raw_data + byte_width * i, fw_result.GetValue(i), byte_width));
+      } else {
+        ASSERT_TRUE(fw_result.IsNull(i));
+      }
+    }
+  };
+
+  // Build using iterative API
+  InitBuilder(byte_width);
+  for (int64_t i = 0; i < length; ++i) {
+    if (is_valid[i]) {
+      builder_->Append(raw_data + byte_width * i);
+    } else {
+      builder_->AppendNull();
+    }
+  }
+
+  ASSERT_OK(builder_->Finish(&result));
+  CheckResult(*result);
+
+  // Build using batch API
+  InitBuilder(byte_width);
+
+  const uint8_t* raw_is_valid = is_valid.data();
+
+  ASSERT_OK(builder_->Append(raw_data, 50, raw_is_valid));
+  ASSERT_OK(builder_->Append(raw_data + 50 * byte_width, length - 50, raw_is_valid + 50));
+  ASSERT_OK(builder_->Finish(&result));
+  CheckResult(*result);
+
+  // Build from std::string
+  InitBuilder(byte_width);
+  for (int64_t i = 0; i < length; ++i) {
+    if (is_valid[i]) {
+      builder_->Append(std::string(
+          reinterpret_cast<const char*>(raw_data + byte_width * i), byte_width));
+    } else {
+      builder_->AppendNull();
+    }
+  }
+
+  ASSERT_OK(builder_->Finish(&result));
+  CheckResult(*result);
+}
+
+TEST_F(TestFWBinaryArray, EqualsRangeEquals) {
+  // Check that we don't compare data in null slots
+
+  auto type = fixed_width_binary(4);
+  FixedWidthBinaryBuilder builder1(default_memory_pool(), type);
+  FixedWidthBinaryBuilder builder2(default_memory_pool(), type);
+
+  ASSERT_OK(builder1.Append("foo1"));
+  ASSERT_OK(builder1.AppendNull());
+
+  ASSERT_OK(builder2.Append("foo1"));
+  ASSERT_OK(builder2.Append("foo2"));
+
+  std::shared_ptr<Array> array1, array2;
+  ASSERT_OK(builder1.Finish(&array1));
+  ASSERT_OK(builder2.Finish(&array2));
+
+  const auto& a1 = static_cast<const FixedWidthBinaryArray&>(*array1);
+  const auto& a2 = static_cast<const FixedWidthBinaryArray&>(*array2);
+
+  FixedWidthBinaryArray equal1(type, 2, a1.data(), a1.null_bitmap(), 1);
+  FixedWidthBinaryArray equal2(type, 2, a2.data(), a1.null_bitmap(), 1);
+
+  ASSERT_TRUE(equal1.Equals(equal2));
+  ASSERT_TRUE(equal1.RangeEquals(equal2, 0, 2, 0));
+}
+
+TEST_F(TestFWBinaryArray, ZeroSize) {
+  auto type = fixed_width_binary(0);
+  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
+
+  ASSERT_OK(builder.Append(nullptr));
+  ASSERT_OK(builder.Append(nullptr));
+  ASSERT_OK(builder.Append(nullptr));
+  ASSERT_OK(builder.AppendNull());
+  ASSERT_OK(builder.AppendNull());
+  ASSERT_OK(builder.AppendNull());
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  const auto& fw_array = static_cast<const FixedWidthBinaryArray&>(*array);
+
+  // data is never allocated
+  ASSERT_TRUE(fw_array.data() == nullptr);
+  ASSERT_EQ(0, fw_array.byte_width());
+
+  ASSERT_EQ(6, array->length());
+  ASSERT_EQ(3, array->null_count());
+}
+
+TEST_F(TestFWBinaryArray, Slice) {
+  auto type = fixed_width_binary(4);
+  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
+
+  std::vector<std::string> strings = {"foo1", "foo2", "foo3", "foo4", "foo5"};
+  std::vector<uint8_t> is_null = {0, 1, 0, 0, 0};
+
+  for (int i = 0; i < 5; ++i) {
+    if (is_null[i]) {
+      builder.AppendNull();
+    } else {
+      builder.Append(strings[i]);
+    }
+  }
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(1);
+  slice2 = array->Slice(1);
+  ASSERT_EQ(4, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, slice->length(), 0, slice));
+
+  // Chained slices
+  slice = array->Slice(2);
+  slice2 = array->Slice(1)->Slice(1);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(1, 3);
+  ASSERT_EQ(3, slice->length());
+
+  slice2 = array->Slice(1, 3);
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 3, 0, slice));
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 49da6bb3197a1..36b3fccf79ed0 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -128,10 +128,6 @@ std::shared_ptr<Array> NullArray::Slice(int64_t offset, int64_t length) const {
   return std::make_shared<NullArray>(length);
 }
 
-Status NullArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 // ----------------------------------------------------------------------
 // Primitive array base
 
@@ -143,11 +139,6 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t le
   raw_data_ = data == nullptr ? nullptr : data_->data();
 }
 
-template <typename T>
-Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 template <typename T>
 std::shared_ptr<Array> NumericArray<T>::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
@@ -155,22 +146,6 @@ std::shared_ptr<Array> NumericArray<T>::Slice(int64_t offset, int64_t length) co
       type_, length, data_, null_bitmap_, kUnknownNullCount, offset);
 }
 
-template class NumericArray<UInt8Type>;
-template class NumericArray<UInt16Type>;
-template class NumericArray<UInt32Type>;
-template class NumericArray<UInt64Type>;
-template class NumericArray<Int8Type>;
-template class NumericArray<Int16Type>;
-template class NumericArray<Int32Type>;
-template class NumericArray<Int64Type>;
-template class NumericArray<TimestampType>;
-template class NumericArray<DateType>;
-template class NumericArray<Date32Type>;
-template class NumericArray<TimeType>;
-template class NumericArray<HalfFloatType>;
-template class NumericArray<FloatType>;
-template class NumericArray<DoubleType>;
-
 // ----------------------------------------------------------------------
 // BooleanArray
 
@@ -179,10 +154,6 @@ BooleanArray::BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
     : PrimitiveArray(std::make_shared<BooleanType>(), length, data, null_bitmap,
           null_count, offset) {}
 
-Status BooleanArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 std::shared_ptr<Array> BooleanArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<BooleanArray>(
@@ -244,10 +215,6 @@ Status ListArray::Validate() const {
   return Status::OK();
 }
 
-Status ListArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 std::shared_ptr<Array> ListArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<ListArray>(
@@ -285,10 +252,6 @@ Status BinaryArray::Validate() const {
   return Status::OK();
 }
 
-Status BinaryArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 std::shared_ptr<Array> BinaryArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<BinaryArray>(
@@ -306,16 +269,33 @@ Status StringArray::Validate() const {
   return BinaryArray::Validate();
 }
 
-Status StringArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<StringArray>(
       length, value_offsets_, data_, null_bitmap_, kUnknownNullCount, offset);
 }
 
+// ----------------------------------------------------------------------
+// Fixed width binary
+
+FixedWidthBinaryArray::FixedWidthBinaryArray(const std::shared_ptr<DataType>& type,
+    int64_t length, const std::shared_ptr<Buffer>& data,
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
+    : Array(type, length, null_bitmap, null_count, offset),
+      data_(data),
+      raw_data_(nullptr) {
+  DCHECK(type->type == Type::FIXED_WIDTH_BINARY);
+  byte_width_ = static_cast<const FixedWidthBinaryType&>(*type).byte_width();
+  if (data) { raw_data_ = data->data(); }
+}
+
+std::shared_ptr<Array> FixedWidthBinaryArray::Slice(
+    int64_t offset, int64_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<FixedWidthBinaryArray>(
+      type_, length, data_, null_bitmap_, kUnknownNullCount, offset);
+}
+
 // ----------------------------------------------------------------------
 // Struct
 
@@ -368,10 +348,6 @@ Status StructArray::Validate() const {
   return Status::OK();
 }
 
-Status StructArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 std::shared_ptr<Array> StructArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<StructArray>(
@@ -413,10 +389,6 @@ Status UnionArray::Validate() const {
   return Status::OK();
 }
 
-Status UnionArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 std::shared_ptr<Array> UnionArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<UnionArray>(type_, length, children_, type_ids_, value_offsets_,
@@ -447,17 +419,54 @@ std::shared_ptr<Array> DictionaryArray::dictionary() const {
   return dict_type_->dictionary();
 }
 
-Status DictionaryArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
 std::shared_ptr<Array> DictionaryArray::Slice(int64_t offset, int64_t length) const {
   std::shared_ptr<Array> sliced_indices = indices_->Slice(offset, length);
   return std::make_shared<DictionaryArray>(type_, sliced_indices);
 }
 
 // ----------------------------------------------------------------------
-// Default implementations of ArrayVisitor methods
+// Implement ArrayVisitor methods
+
+Status NullArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+Status BooleanArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+template <typename T>
+Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+Status BinaryArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+Status StringArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+Status FixedWidthBinaryArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+Status ListArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+Status StructArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+Status UnionArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
+Status DictionaryArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
 
 #define ARRAY_VISITOR_DEFAULT(ARRAY_CLASS)                   \
   Status ArrayVisitor::Visit(const ARRAY_CLASS& array) {     \
@@ -477,8 +486,9 @@ ARRAY_VISITOR_DEFAULT(UInt64Array);
 ARRAY_VISITOR_DEFAULT(HalfFloatArray);
 ARRAY_VISITOR_DEFAULT(FloatArray);
 ARRAY_VISITOR_DEFAULT(DoubleArray);
-ARRAY_VISITOR_DEFAULT(StringArray);
 ARRAY_VISITOR_DEFAULT(BinaryArray);
+ARRAY_VISITOR_DEFAULT(StringArray);
+ARRAY_VISITOR_DEFAULT(FixedWidthBinaryArray);
 ARRAY_VISITOR_DEFAULT(DateArray);
 ARRAY_VISITOR_DEFAULT(Date32Array);
 ARRAY_VISITOR_DEFAULT(TimeArray);
@@ -493,4 +503,23 @@ Status ArrayVisitor::Visit(const DecimalArray& array) {
   return Status::NotImplemented("decimal");
 }
 
+// ----------------------------------------------------------------------
+// Instantiate templates
+
+template class NumericArray<UInt8Type>;
+template class NumericArray<UInt16Type>;
+template class NumericArray<UInt32Type>;
+template class NumericArray<UInt64Type>;
+template class NumericArray<Int8Type>;
+template class NumericArray<Int16Type>;
+template class NumericArray<Int32Type>;
+template class NumericArray<Int64Type>;
+template class NumericArray<TimestampType>;
+template class NumericArray<DateType>;
+template class NumericArray<Date32Type>;
+template class NumericArray<TimeType>;
+template class NumericArray<HalfFloatType>;
+template class NumericArray<FloatType>;
+template class NumericArray<DoubleType>;
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index f111609db4317..ecc8ce540b1dd 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -57,6 +57,7 @@ class ARROW_EXPORT ArrayVisitor {
   virtual Status Visit(const DoubleArray& array);
   virtual Status Visit(const StringArray& array);
   virtual Status Visit(const BinaryArray& array);
+  virtual Status Visit(const FixedWidthBinaryArray& array);
   virtual Status Visit(const DateArray& array);
   virtual Status Visit(const Date32Array& array);
   virtual Status Visit(const TimeArray& array);
@@ -386,6 +387,39 @@ class ARROW_EXPORT StringArray : public BinaryArray {
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
+// ----------------------------------------------------------------------
+// Fixed width binary
+
+class ARROW_EXPORT FixedWidthBinaryArray : public Array {
+ public:
+  using TypeClass = FixedWidthBinaryType;
+
+  FixedWidthBinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
+      const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0);
+
+  const uint8_t* GetValue(int64_t i) const {
+    return raw_data_ + (i + offset_) * byte_width_;
+  }
+
+  /// Note that this buffer does not account for any slice offset
+  std::shared_ptr<Buffer> data() const { return data_; }
+
+  int32_t byte_width() const { return byte_width_; }
+
+  const uint8_t* raw_data() const { return raw_data_; }
+
+  Status Accept(ArrayVisitor* visitor) const override;
+
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
+
+ protected:
+  int32_t byte_width_;
+  std::shared_ptr<Buffer> data_;
+  const uint8_t* raw_data_;
+};
+
 // ----------------------------------------------------------------------
 // Struct
 
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 0724385a4aff8..26c8ea60214f6 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -157,6 +157,8 @@ class ARROW_EXPORT BufferBuilder {
 
   /// Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
   Status Resize(int64_t elements) {
+    // Resize(0) is a no-op
+    if (elements == 0) { return Status::OK(); }
     if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
     RETURN_NOT_OK(buffer_->Resize(elements));
     capacity_ = buffer_->capacity();
@@ -170,6 +172,14 @@ class ARROW_EXPORT BufferBuilder {
     return Status::OK();
   }
 
+  // Advance pointer and zero out memory
+  Status Advance(int64_t length) {
+    if (capacity_ < length + size_) { RETURN_NOT_OK(Resize(length + size_)); }
+    memset(data_ + size_, 0, static_cast<size_t>(length));
+    size_ += length;
+    return Status::OK();
+  }
+
   template <typename T>
   Status Append(T arithmetic_value) {
     static_assert(std::is_arithmetic<T>::value,
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 4372925fe494b..b65a4928ec999 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -121,6 +121,14 @@ void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t leng
   uint8_t bitset = null_bitmap_data_[byte_offset];
 
   for (int64_t i = 0; i < length; ++i) {
+    if (bit_offset == 8) {
+      bit_offset = 0;
+      null_bitmap_data_[byte_offset] = bitset;
+      byte_offset++;
+      // TODO: Except for the last byte, this shouldn't be needed
+      bitset = null_bitmap_data_[byte_offset];
+    }
+
     if (valid_bytes[i]) {
       bitset |= BitUtil::kBitmask[bit_offset];
     } else {
@@ -129,13 +137,6 @@ void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t leng
     }
 
     bit_offset++;
-    if (bit_offset == 8) {
-      bit_offset = 0;
-      null_bitmap_data_[byte_offset] = bitset;
-      byte_offset++;
-      // TODO: Except for the last byte, this shouldn't be needed
-      bitset = null_bitmap_data_[byte_offset];
-    }
   }
   if (bit_offset != 0) { null_bitmap_data_[byte_offset] = bitset; }
   length_ += length;
@@ -324,21 +325,37 @@ Status BooleanBuilder::Append(
 // ----------------------------------------------------------------------
 // ListBuilder
 
-ListBuilder::ListBuilder(
-    MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder, const TypePtr& type)
+ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
+    const std::shared_ptr<DataType>& type)
     : ArrayBuilder(
           pool, type ? type : std::static_pointer_cast<DataType>(
                                   std::make_shared<ListType>(value_builder->type()))),
       offset_builder_(pool),
       value_builder_(value_builder) {}
 
-ListBuilder::ListBuilder(
-    MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type)
+ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<Array> values,
+    const std::shared_ptr<DataType>& type)
     : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
                                            std::make_shared<ListType>(values->type()))),
       offset_builder_(pool),
       values_(values) {}
 
+Status ListBuilder::Append(
+    const int32_t* offsets, int64_t length, const uint8_t* valid_bytes) {
+  RETURN_NOT_OK(Reserve(length));
+  UnsafeAppendToBitmap(valid_bytes, length);
+  offset_builder_.UnsafeAppend<int32_t>(offsets, length);
+  return Status::OK();
+}
+
+Status ListBuilder::Append(bool is_valid) {
+  RETURN_NOT_OK(Reserve(1));
+  UnsafeAppendToBitmap(is_valid);
+  RETURN_NOT_OK(
+      offset_builder_.Append<int32_t>(static_cast<int32_t>(value_builder_->length())));
+  return Status::OK();
+}
+
 Status ListBuilder::Init(int64_t elements) {
   DCHECK_LT(elements, std::numeric_limits<int64_t>::max());
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
@@ -386,7 +403,7 @@ BinaryBuilder::BinaryBuilder(MemoryPool* pool)
   byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
 }
 
-BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
+BinaryBuilder::BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
     : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, uint8()), type) {
   byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
 }
@@ -417,6 +434,58 @@ Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// Fixed width binary
+
+FixedWidthBinaryBuilder::FixedWidthBinaryBuilder(
+    MemoryPool* pool, const std::shared_ptr<DataType>& type)
+    : ArrayBuilder(pool, type), byte_builder_(pool) {
+  DCHECK(type->type == Type::FIXED_WIDTH_BINARY);
+  byte_width_ = static_cast<const FixedWidthBinaryType&>(*type).byte_width();
+}
+
+Status FixedWidthBinaryBuilder::Append(const uint8_t* value) {
+  RETURN_NOT_OK(Reserve(1));
+  UnsafeAppendToBitmap(true);
+  return byte_builder_.Append(value, byte_width_);
+}
+
+Status FixedWidthBinaryBuilder::Append(
+    const uint8_t* data, int64_t length, const uint8_t* valid_bytes) {
+  RETURN_NOT_OK(Reserve(length));
+  UnsafeAppendToBitmap(valid_bytes, length);
+  return byte_builder_.Append(data, length * byte_width_);
+}
+
+Status FixedWidthBinaryBuilder::Append(const std::string& value) {
+  return Append(reinterpret_cast<const uint8_t*>(value.c_str()));
+}
+
+Status FixedWidthBinaryBuilder::AppendNull() {
+  RETURN_NOT_OK(Reserve(1));
+  UnsafeAppendToBitmap(false);
+  return byte_builder_.Advance(byte_width_);
+}
+
+Status FixedWidthBinaryBuilder::Init(int64_t elements) {
+  DCHECK_LT(elements, std::numeric_limits<int64_t>::max());
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+  return byte_builder_.Resize(elements * byte_width_);
+}
+
+Status FixedWidthBinaryBuilder::Resize(int64_t capacity) {
+  DCHECK_LT(capacity, std::numeric_limits<int64_t>::max());
+  RETURN_NOT_OK(byte_builder_.Resize(capacity * byte_width_));
+  return ArrayBuilder::Resize(capacity);
+}
+
+Status FixedWidthBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Buffer> data = byte_builder_.Finish();
+  *out = std::make_shared<FixedWidthBinaryArray>(
+      type_, length_, data, null_bitmap_, null_count_);
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // Struct
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index ebc683ab334e6..07b7cfcb3a964 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -46,7 +46,7 @@ static constexpr int64_t kMinBuilderCapacity = 1 << 5;
 /// the null count.
 class ARROW_EXPORT ArrayBuilder {
  public:
-  explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type)
+  explicit ArrayBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
       : pool_(pool),
         type_(type),
         null_bitmap_(nullptr),
@@ -140,7 +140,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
  public:
   using value_type = typename Type::c_type;
 
-  explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type)
+  explicit PrimitiveBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
       : ArrayBuilder(pool, type), data_(nullptr), raw_data_(nullptr) {}
 
   using ArrayBuilder::Advance;
@@ -313,11 +313,11 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   /// Use this constructor to incrementally build the value array along with offsets and
   /// null bitmap.
   ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
-      const TypePtr& type = nullptr);
+      const std::shared_ptr<DataType>& type = nullptr);
 
   /// Use this constructor to build the list with a pre-existing values array
-  ListBuilder(
-      MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr);
+  ListBuilder(MemoryPool* pool, std::shared_ptr<Array> values,
+      const std::shared_ptr<DataType>& type = nullptr);
 
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
@@ -328,24 +328,13 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
   Status Append(
-      const int32_t* offsets, int64_t length, const uint8_t* valid_bytes = nullptr) {
-    RETURN_NOT_OK(Reserve(length));
-    UnsafeAppendToBitmap(valid_bytes, length);
-    offset_builder_.UnsafeAppend<int32_t>(offsets, length);
-    return Status::OK();
-  }
+      const int32_t* offsets, int64_t length, const uint8_t* valid_bytes = nullptr);
 
   /// Start a new variable-length list slot
   ///
   /// This function should be called before beginning to append elements to the
   /// value builder
-  Status Append(bool is_valid = true) {
-    RETURN_NOT_OK(Reserve(1));
-    UnsafeAppendToBitmap(is_valid);
-    RETURN_NOT_OK(
-        offset_builder_.Append<int32_t>(static_cast<int32_t>(value_builder_->length())));
-    return Status::OK();
-  }
+  Status Append(bool is_valid = true);
 
   Status AppendNull() { return Append(false); }
 
@@ -362,11 +351,10 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
 // ----------------------------------------------------------------------
 // Binary and String
 
-// BinaryBuilder : public ListBuilder
 class ARROW_EXPORT BinaryBuilder : public ListBuilder {
  public:
   explicit BinaryBuilder(MemoryPool* pool);
-  explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type);
+  explicit BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 
   Status Append(const uint8_t* value, int32_t length) {
     RETURN_NOT_OK(ListBuilder::Append());
@@ -399,6 +387,28 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
   Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
 };
 
+// ----------------------------------------------------------------------
+// FixedWidthBinaryBuilder
+
+class ARROW_EXPORT FixedWidthBinaryBuilder : public ArrayBuilder {
+ public:
+  FixedWidthBinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+
+  Status Append(const uint8_t* value);
+  Status Append(
+      const uint8_t* data, int64_t length, const uint8_t* valid_bytes = nullptr);
+  Status Append(const std::string& value);
+  Status AppendNull();
+
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+ protected:
+  int32_t byte_width_;
+  BufferBuilder byte_builder_;
+};
+
 // ----------------------------------------------------------------------
 // Struct
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 17b883302c658..86ed8ccecd1ea 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -143,6 +143,32 @@ class RangeEqualsVisitor : public ArrayVisitor {
     return Status::OK();
   }
 
+  Status Visit(const FixedWidthBinaryArray& left) override {
+    const auto& right = static_cast<const FixedWidthBinaryArray&>(right_);
+
+    int32_t width = left.byte_width();
+
+    const uint8_t* left_data = left.raw_data() + left.offset() * width;
+    const uint8_t* right_data = right.raw_data() + right.offset() * width;
+
+    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+         ++i, ++o_i) {
+      const bool is_null = left.IsNull(i);
+      if (is_null != right.IsNull(o_i)) {
+        result_ = false;
+        return Status::OK();
+      }
+      if (is_null) continue;
+
+      if (std::memcmp(left_data + width * i, right_data + width * o_i, width)) {
+        result_ = false;
+        return Status::OK();
+      }
+    }
+    result_ = true;
+    return Status::OK();
+  }
+
   Status Visit(const DateArray& left) override { return CompareValues<DateArray>(left); }
 
   Status Visit(const Date32Array& left) override {
@@ -632,6 +658,12 @@ class TypeEqualsVisitor : public TypeVisitor {
     return Status::OK();
   }
 
+  Status Visit(const FixedWidthBinaryType& left) override {
+    const auto& right = static_cast<const FixedWidthBinaryType&>(right_);
+    result_ = left.byte_width() == right.byte_width();
+    return Status::OK();
+  }
+
   Status Visit(const ListType& left) override { return VisitChildren(left); }
 
   Status Visit(const StructType& left) override { return VisitChildren(left); }
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index a4eff7214aa5f..406ce249eec32 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -304,6 +304,17 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
+  Status Visit(const FixedWidthBinaryArray& array) override {
+    auto data = array.data();
+    int32_t width = array.byte_width();
+
+    if (array.offset() != 0) {
+      data = SliceBuffer(data, array.offset() * width, width * array.length());
+    }
+    buffers_.push_back(data);
+    return Status::OK();
+  }
+
   Status Visit(const BooleanArray& array) override {
     buffers_.push_back(array.data());
     return Status::OK();
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index b60b8a9ba68d2..36a675f5f94f7 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -175,8 +175,8 @@ INSTANTIATE_TEST_CASE_P(
     RoundTripTests, TestRecordBatchParam,
     ::testing::Values(&MakeIntRecordBatch, &MakeStringTypesRecordBatch,
         &MakeNonNullRecordBatch, &MakeZeroLengthRecordBatch, &MakeListRecordBatch,
-        &MakeDeeplyNestedList, &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDates,
-        &MakeTimestamps, &MakeTimes));
+        &MakeDeeplyNestedList, &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate,
+        &MakeTimestamps, &MakeTimes, &MakeFWBinary));
 
 void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   ipc::MockOutputStream mock;
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index 0c95c8eca65ca..b45782220e478 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -43,7 +43,10 @@ namespace arrow {
 namespace ipc {
 
 void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
-  ASSERT_TRUE(left.schema()->Equals(right.schema()));
+  if (!left.schema()->Equals(right.schema())) {
+    FAIL() << "Left schema: " << left.schema()->ToString()
+           << "\nRight schema: " << right.schema()->ToString();
+  }
   ASSERT_EQ(left.num_columns(), right.num_columns())
       << left.schema()->ToString() << " result: " << right.schema()->ToString();
   EXPECT_EQ(left.num_rows(), right.num_rows());
@@ -180,7 +183,8 @@ TEST_P(TestStreamFormat, RoundTrip) {
 #define BATCH_CASES()                                                                   \
   ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
       &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
-      &MakeStruct, &MakeUnion, &MakeDictionary);
+      &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate, &MakeTimestamps, &MakeTimes, \
+      &MakeFWBinary);
 
 INSTANTIATE_TEST_CASE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
 INSTANTIATE_TEST_CASE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 0458b85f0078a..549b26bfe8201 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -39,12 +39,11 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/string.h"
 
 namespace arrow {
 namespace ipc {
 
-static const char* kAsciiTable = "0123456789ABCDEF";
-
 using RjArray = rj::Value::ConstArray;
 using RjObject = rj::Value::ConstObject;
 
@@ -401,14 +400,7 @@ class JsonArrayWriter : public ArrayVisitor {
       if (std::is_base_of<StringArray, T>::value) {
         writer_->String(buf, length);
       } else {
-        std::string hex_string;
-        hex_string.reserve(length * 2);
-        for (int32_t j = 0; j < length; ++j) {
-          // Convert to 2 base16 digits
-          hex_string.push_back(kAsciiTable[buf[j] >> 4]);
-          hex_string.push_back(kAsciiTable[buf[j] & 15]);
-        }
-        writer_->String(hex_string);
+        writer_->String(HexEncode(buf, length));
       }
     }
   }
@@ -760,20 +752,6 @@ class JsonSchemaReader {
   const rj::Value& json_schema_;
 };
 
-static inline Status ParseHexValue(const char* data, uint8_t* out) {
-  char c1 = data[0];
-  char c2 = data[1];
-
-  const char* pos1 = std::lower_bound(kAsciiTable, kAsciiTable + 16, c1);
-  const char* pos2 = std::lower_bound(kAsciiTable, kAsciiTable + 16, c2);
-
-  // Error checking
-  if (*pos1 != c1 || *pos2 != c2) { return Status::Invalid("Encountered non-hex digit"); }
-
-  *out = static_cast<uint8_t>((pos1 - kAsciiTable) << 4 | (pos2 - kAsciiTable));
-  return Status::OK();
-}
-
 template <typename T>
 inline typename std::enable_if<IsSignedInt<T>::value, typename T::c_type>::type
 UnboxValue(const rj::Value& val) {
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 17a3a5fafe626..be0d282f21bbf 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -170,6 +170,39 @@ static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
   *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
   break;
 
+static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) {
+  switch (unit) {
+    case TimeUnit::SECOND:
+      return flatbuf::TimeUnit_SECOND;
+    case TimeUnit::MILLI:
+      return flatbuf::TimeUnit_MILLISECOND;
+    case TimeUnit::MICRO:
+      return flatbuf::TimeUnit_MICROSECOND;
+    case TimeUnit::NANO:
+      return flatbuf::TimeUnit_NANOSECOND;
+    default:
+      break;
+  }
+  return flatbuf::TimeUnit_MIN;
+}
+
+static inline TimeUnit FromFlatbufferUnit(flatbuf::TimeUnit unit) {
+  switch (unit) {
+    case flatbuf::TimeUnit_SECOND:
+      return TimeUnit::SECOND;
+    case flatbuf::TimeUnit_MILLISECOND:
+      return TimeUnit::MILLI;
+    case flatbuf::TimeUnit_MICROSECOND:
+      return TimeUnit::MICRO;
+    case flatbuf::TimeUnit_NANOSECOND:
+      return TimeUnit::NANO;
+    default:
+      break;
+  }
+  // cannot reach
+  return TimeUnit::SECOND;
+}
+
 static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
   switch (type) {
@@ -183,6 +216,11 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     case flatbuf::Type_Binary:
       *out = binary();
       return Status::OK();
+    case flatbuf::Type_FixedWidthBinary: {
+      auto fw_binary = static_cast<const flatbuf::FixedWidthBinary*>(type_data);
+      *out = fixed_width_binary(fw_binary->byteWidth());
+      return Status::OK();
+    }
     case flatbuf::Type_Utf8:
       *out = utf8();
       return Status::OK();
@@ -190,7 +228,22 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       *out = boolean();
       return Status::OK();
     case flatbuf::Type_Decimal:
-    case flatbuf::Type_Timestamp:
+      return Status::NotImplemented("Decimal");
+    case flatbuf::Type_Date:
+      *out = date();
+      return Status::OK();
+    case flatbuf::Type_Time: {
+      auto time_type = static_cast<const flatbuf::Time*>(type_data);
+      *out = time(FromFlatbufferUnit(time_type->unit()));
+      return Status::OK();
+    }
+    case flatbuf::Type_Timestamp: {
+      auto ts_type = static_cast<const flatbuf::Timestamp*>(type_data);
+      *out = timestamp(FromFlatbufferUnit(ts_type->unit()));
+      return Status::OK();
+    }
+    case flatbuf::Type_Interval:
+      return Status::NotImplemented("Interval");
     case flatbuf::Type_List:
       if (children.size() != 1) {
         return Status::Invalid("List must have exactly 1 child field");
@@ -275,6 +328,11 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *out_type = flatbuf::Type_FloatingPoint;
       *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
       break;
+    case Type::FIXED_WIDTH_BINARY: {
+      const auto& fw_type = static_cast<const FixedWidthBinaryType&>(*type);
+      *out_type = flatbuf::Type_FixedWidthBinary;
+      *offset = flatbuf::CreateFixedWidthBinary(fbb, fw_type.byte_width()).Union();
+    } break;
     case Type::BINARY:
       *out_type = flatbuf::Type_Binary;
       *offset = flatbuf::CreateBinary(fbb).Union();
@@ -283,6 +341,20 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *out_type = flatbuf::Type_Utf8;
       *offset = flatbuf::CreateUtf8(fbb).Union();
       break;
+    case Type::DATE:
+      *out_type = flatbuf::Type_Date;
+      *offset = flatbuf::CreateDate(fbb).Union();
+      break;
+    case Type::TIME: {
+      const auto& time_type = static_cast<const TimeType&>(*type);
+      *out_type = flatbuf::Type_Time;
+      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
+    } break;
+    case Type::TIMESTAMP: {
+      const auto& ts_type = static_cast<const TimestampType&>(*type);
+      *out_type = flatbuf::Type_Timestamp;
+      *offset = flatbuf::CreateTimestamp(fbb, ToFlatbufferUnit(ts_type.unit)).Union();
+    } break;
     case Type::LIST:
       *out_type = flatbuf::Type_List;
       return ListToFlatbuffer(fbb, type, children, dictionary_memo, offset);
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 7f33aba812e0f..66a5e09362cf5 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -463,30 +463,42 @@ Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
-Status MakeDates(std::shared_ptr<RecordBatch>* out) {
+Status MakeDate(std::shared_ptr<RecordBatch>* out) {
   std::vector<bool> is_valid = {true, true, true, false, true, true, true};
-  auto f0 = field("f0", date32());
   auto f1 = field("f1", date());
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+  std::shared_ptr<Schema> schema(new Schema({f1}));
 
   std::vector<int64_t> date_values = {1489269000000, 1489270000000, 1489271000000,
       1489272000000, 1489272000000, 1489273000000};
-  std::vector<int32_t> date32_values = {0, 1, 2, 3, 4, 5, 6};
 
-  std::shared_ptr<Array> date_array, date32_array;
+  std::shared_ptr<Array> date_array;
   ArrayFromVector<DateType, int64_t>(is_valid, date_values, &date_array);
-  ArrayFromVector<Date32Type, int32_t>(is_valid, date32_values, &date32_array);
 
-  std::vector<std::shared_ptr<Array>> arrays = {date32_array, date_array};
+  std::vector<std::shared_ptr<Array>> arrays = {date_array};
   *out = std::make_shared<RecordBatch>(schema, date_array->length(), arrays);
   return Status::OK();
 }
 
+Status MakeDate32(std::shared_ptr<RecordBatch>* out) {
+  std::vector<bool> is_valid = {true, true, true, false, true, true, true};
+  auto f0 = field("f0", date32());
+  std::shared_ptr<Schema> schema(new Schema({f0}));
+
+  std::vector<int32_t> date32_values = {0, 1, 2, 3, 4, 5, 6};
+
+  std::shared_ptr<Array> date32_array;
+  ArrayFromVector<Date32Type, int32_t>(is_valid, date32_values, &date32_array);
+
+  std::vector<std::shared_ptr<Array>> arrays = {date32_array};
+  *out = std::make_shared<RecordBatch>(schema, date32_array->length(), arrays);
+  return Status::OK();
+}
+
 Status MakeTimestamps(std::shared_ptr<RecordBatch>* out) {
   std::vector<bool> is_valid = {true, true, true, false, true, true, true};
   auto f0 = field("f0", timestamp(TimeUnit::MILLI));
   auto f1 = field("f1", timestamp(TimeUnit::NANO));
-  auto f2 = field("f2", timestamp("US/Los_Angeles", TimeUnit::SECOND));
+  auto f2 = field("f2", timestamp(TimeUnit::SECOND));
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
   std::vector<int64_t> ts_values = {1489269000000, 1489270000000, 1489271000000,
@@ -522,6 +534,43 @@ Status MakeTimes(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
+template <typename BuilderType, typename T>
+void AppendValues(const std::vector<bool>& is_valid, const std::vector<T>& values,
+    BuilderType* builder) {
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid[i]) {
+      builder->Append(values[i]);
+    } else {
+      builder->AppendNull();
+    }
+  }
+}
+
+Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
+  std::vector<bool> is_valid = {true, true, true, false};
+  auto f0 = field("f0", fixed_width_binary(4));
+  auto f1 = field("f1", fixed_width_binary(0));
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  std::shared_ptr<Array> a1, a2;
+
+  FixedWidthBinaryBuilder b1(default_memory_pool(), f0->type);
+  FixedWidthBinaryBuilder b2(default_memory_pool(), f0->type);
+
+  std::vector<std::string> values1 = {"foo1", "foo2", "foo3", "foo4"};
+  AppendValues(is_valid, values1, &b1);
+
+  std::vector<std::string> values2 = {"", "", "", ""};
+  AppendValues(is_valid, values2, &b2);
+
+  RETURN_NOT_OK(b1.Finish(&a1));
+  RETURN_NOT_OK(b2.Finish(&a2));
+
+  ArrayVector arrays = {a1, a2};
+  *out = std::make_shared<RecordBatch>(schema, a1->length(), arrays);
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
index 0b3ee1cf0a899..fc373715105e1 100644
--- a/cpp/src/arrow/loader.cc
+++ b/cpp/src/arrow/loader.cc
@@ -157,6 +157,18 @@ class ArrayLoader : public TypeVisitor {
 
   Status Visit(const BinaryType& type) override { return LoadBinary<BinaryArray>(); }
 
+  Status Visit(const FixedWidthBinaryType& type) override {
+    FieldMetadata field_meta;
+    std::shared_ptr<Buffer> null_bitmap, data;
+
+    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &data));
+
+    result_ = std::make_shared<FixedWidthBinaryArray>(
+        type_, field_meta.length, data, null_bitmap, field_meta.null_count);
+    return Status::OK();
+  }
+
   Status Visit(const ListType& type) override {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap, offsets;
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index aca650f0a927b..f21383f0cb06f 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -56,7 +56,7 @@ void CheckPrimitive(int indent, const std::vector<bool>& is_valid,
     const std::vector<C_TYPE>& values, const char* expected) {
   std::shared_ptr<Array> array;
   ArrayFromVector<TYPE, C_TYPE>(is_valid, values, &array);
-  CheckArray(*array.get(), indent, expected);
+  CheckArray(*array, indent, expected);
 }
 
 TEST_F(TestPrettyPrint, PrimitiveType) {
@@ -71,6 +71,30 @@ TEST_F(TestPrettyPrint, PrimitiveType) {
   CheckPrimitive<StringType, std::string>(0, is_valid, values2, ex2);
 }
 
+TEST_F(TestPrettyPrint, BinaryType) {
+  std::vector<bool> is_valid = {true, true, false, true, false};
+  std::vector<std::string> values = {"foo", "bar", "", "baz", ""};
+  static const char* ex = R"expected([666F6F, 626172, null, 62617A, null])expected";
+  CheckPrimitive<BinaryType, std::string>(0, is_valid, values, ex);
+}
+
+TEST_F(TestPrettyPrint, FixedWidthBinaryType) {
+  std::vector<bool> is_valid = {true, true, false, true, false};
+  std::vector<std::string> values = {"foo", "bar", "baz"};
+  static const char* ex = R"expected([666F6F, 626172, 62617A])expected";
+
+  std::shared_ptr<Array> array;
+  auto type = fixed_width_binary(3);
+  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
+
+  builder.Append(values[0]);
+  builder.Append(values[1]);
+  builder.Append(values[2]);
+  builder.Finish(&array);
+
+  CheckArray(*array, 0, ex);
+}
+
 TEST_F(TestPrettyPrint, DictionaryType) {
   std::vector<bool> is_valid = {true, true, false, true, true, true};
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 2508fa5bd8cde..87c1a1cf9d9c5 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -26,6 +26,7 @@
 #include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
+#include "arrow/util/string.h"
 
 namespace arrow {
 
@@ -66,9 +67,9 @@ class ArrayPrinter : public ArrayVisitor {
     }
   }
 
-  // String (Utf8), Binary
+  // String (Utf8)
   template <typename T>
-  typename std::enable_if<std::is_base_of<BinaryArray, T>::value, void>::type
+  typename std::enable_if<std::is_same<StringArray, T>::value, void>::type
   WriteDataValues(const T& array) {
     int32_t length;
     for (int i = 0; i < array.length(); ++i) {
@@ -82,6 +83,37 @@ class ArrayPrinter : public ArrayVisitor {
     }
   }
 
+  // Binary
+  template <typename T>
+  typename std::enable_if<std::is_same<BinaryArray, T>::value, void>::type
+  WriteDataValues(const T& array) {
+    int32_t length;
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        Write("null");
+      } else {
+        const char* buf = reinterpret_cast<const char*>(array.GetValue(i, &length));
+        (*sink_) << HexEncode(buf, length);
+      }
+    }
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_same<FixedWidthBinaryArray, T>::value, void>::type
+  WriteDataValues(const T& array) {
+    int32_t width = array.byte_width();
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        Write("null");
+      } else {
+        const char* buf = reinterpret_cast<const char*>(array.GetValue(i));
+        (*sink_) << HexEncode(buf, width);
+      }
+    }
+  }
+
   template <typename T>
   typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
   WriteDataValues(const T& array) {
@@ -100,15 +132,7 @@ class ArrayPrinter : public ArrayVisitor {
   void CloseArray() { (*sink_) << "]"; }
 
   template <typename T>
-  Status WritePrimitive(const T& array) {
-    OpenArray();
-    WriteDataValues(array);
-    CloseArray();
-    return Status::OK();
-  }
-
-  template <typename T>
-  Status WriteVarBytes(const T& array) {
+  Status WriteArray(const T& array) {
     OpenArray();
     WriteDataValues(array);
     CloseArray();
@@ -117,39 +141,41 @@ class ArrayPrinter : public ArrayVisitor {
 
   Status Visit(const NullArray& array) override { return Status::OK(); }
 
-  Status Visit(const BooleanArray& array) override { return WritePrimitive(array); }
+  Status Visit(const BooleanArray& array) override { return WriteArray(array); }
+
+  Status Visit(const Int8Array& array) override { return WriteArray(array); }
 
-  Status Visit(const Int8Array& array) override { return WritePrimitive(array); }
+  Status Visit(const Int16Array& array) override { return WriteArray(array); }
 
-  Status Visit(const Int16Array& array) override { return WritePrimitive(array); }
+  Status Visit(const Int32Array& array) override { return WriteArray(array); }
 
-  Status Visit(const Int32Array& array) override { return WritePrimitive(array); }
+  Status Visit(const Int64Array& array) override { return WriteArray(array); }
 
-  Status Visit(const Int64Array& array) override { return WritePrimitive(array); }
+  Status Visit(const UInt8Array& array) override { return WriteArray(array); }
 
-  Status Visit(const UInt8Array& array) override { return WritePrimitive(array); }
+  Status Visit(const UInt16Array& array) override { return WriteArray(array); }
 
-  Status Visit(const UInt16Array& array) override { return WritePrimitive(array); }
+  Status Visit(const UInt32Array& array) override { return WriteArray(array); }
 
-  Status Visit(const UInt32Array& array) override { return WritePrimitive(array); }
+  Status Visit(const UInt64Array& array) override { return WriteArray(array); }
 
-  Status Visit(const UInt64Array& array) override { return WritePrimitive(array); }
+  Status Visit(const HalfFloatArray& array) override { return WriteArray(array); }
 
-  Status Visit(const HalfFloatArray& array) override { return WritePrimitive(array); }
+  Status Visit(const FloatArray& array) override { return WriteArray(array); }
 
-  Status Visit(const FloatArray& array) override { return WritePrimitive(array); }
+  Status Visit(const DoubleArray& array) override { return WriteArray(array); }
 
-  Status Visit(const DoubleArray& array) override { return WritePrimitive(array); }
+  Status Visit(const StringArray& array) override { return WriteArray(array); }
 
-  Status Visit(const StringArray& array) override { return WriteVarBytes(array); }
+  Status Visit(const BinaryArray& array) override { return WriteArray(array); }
 
-  Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); }
+  Status Visit(const FixedWidthBinaryArray& array) override { return WriteArray(array); }
 
-  Status Visit(const DateArray& array) override { return WritePrimitive(array); }
+  Status Visit(const DateArray& array) override { return WriteArray(array); }
 
-  Status Visit(const Date32Array& array) override { return WritePrimitive(array); }
+  Status Visit(const Date32Array& array) override { return WriteArray(array); }
 
-  Status Visit(const TimeArray& array) override { return WritePrimitive(array); }
+  Status Visit(const TimeArray& array) override { return WriteArray(array); }
 
   Status Visit(const TimestampArray& array) override {
     return Status::NotImplemented("timestamp");
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 3adc4d83c3a2d..ddfff8745b97e 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -121,6 +121,58 @@ TEST_F(TestSchema, GetFieldByName) {
   ASSERT_TRUE(result == nullptr);
 }
 
+TEST(TestBinaryType, ToString) {
+  BinaryType t1;
+  BinaryType e1;
+  StringType t2;
+  EXPECT_TRUE(t1.Equals(e1));
+  EXPECT_FALSE(t1.Equals(t2));
+  ASSERT_EQ(t1.type, Type::BINARY);
+  ASSERT_EQ(t1.ToString(), std::string("binary"));
+}
+
+TEST(TestStringType, ToString) {
+  StringType str;
+  ASSERT_EQ(str.type, Type::STRING);
+  ASSERT_EQ(str.ToString(), std::string("string"));
+}
+
+TEST(TestFixedWidthBinaryType, ToString) {
+  auto t = fixed_width_binary(10);
+  ASSERT_EQ(t->type, Type::FIXED_WIDTH_BINARY);
+  ASSERT_EQ("fixed_width_binary[10]", t->ToString());
+}
+
+TEST(TestFixedWidthBinaryType, Equals) {
+  auto t1 = fixed_width_binary(10);
+  auto t2 = fixed_width_binary(10);
+  auto t3 = fixed_width_binary(3);
+
+  ASSERT_TRUE(t1->Equals(t1));
+  ASSERT_TRUE(t1->Equals(t2));
+  ASSERT_FALSE(t1->Equals(t3));
+}
+
+TEST(TestListType, Basics) {
+  std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
+
+  ListType list_type(vt);
+  ASSERT_EQ(list_type.type, Type::LIST);
+
+  ASSERT_EQ("list", list_type.name());
+  ASSERT_EQ("list<item: uint8>", list_type.ToString());
+
+  ASSERT_EQ(list_type.value_type()->type, vt->type);
+  ASSERT_EQ(list_type.value_type()->type, vt->type);
+
+  std::shared_ptr<DataType> st = std::make_shared<StringType>();
+  std::shared_ptr<DataType> lt = std::make_shared<ListType>(st);
+  ASSERT_EQ("list<item: string>", lt->ToString());
+
+  ListType lt2(lt);
+  ASSERT_EQ("list<item: list<item: string>>", lt2.ToString());
+}
+
 TEST(TestTimeType, Equals) {
   TimeType t1;
   TimeType t2;
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index d41b36315a86e..ee0a89ab8abea 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -88,6 +88,16 @@ std::string BinaryType::ToString() const {
   return std::string("binary");
 }
 
+int FixedWidthBinaryType::bit_width() const {
+  return 8 * byte_width();
+}
+
+std::string FixedWidthBinaryType::ToString() const {
+  std::stringstream ss;
+  ss << "fixed_width_binary[" << byte_width_ << "]";
+  return ss.str();
+}
+
 std::string StructType::ToString() const {
   std::stringstream s;
   s << "struct<";
@@ -189,6 +199,7 @@ std::string NullType::ToString() const {
 ACCEPT_VISITOR(NullType);
 ACCEPT_VISITOR(BooleanType);
 ACCEPT_VISITOR(BinaryType);
+ACCEPT_VISITOR(FixedWidthBinaryType);
 ACCEPT_VISITOR(StringType);
 ACCEPT_VISITOR(ListType);
 ACCEPT_VISITOR(StructType);
@@ -225,6 +236,10 @@ TYPE_FACTORY(binary, BinaryType);
 TYPE_FACTORY(date, DateType);
 TYPE_FACTORY(date32, Date32Type);
 
+std::shared_ptr<DataType> fixed_width_binary(int32_t byte_width) {
+  return std::make_shared<FixedWidthBinaryType>(byte_width);
+}
+
 std::shared_ptr<DataType> timestamp(TimeUnit unit) {
   return std::make_shared<TimestampType>(unit);
 }
@@ -285,6 +300,10 @@ std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
   return {kValidityBuffer, kOffsetBuffer, kValues8};
 }
 
+std::vector<BufferDescr> FixedWidthBinaryType::GetBufferLayout() const {
+  return {kValidityBuffer, BufferDescr(BufferType::DATA, byte_width_ * 8)};
+}
+
 std::vector<BufferDescr> ListType::GetBufferLayout() const {
   return {kValidityBuffer, kOffsetBuffer};
 }
@@ -335,6 +354,7 @@ TYPE_VISITOR_DEFAULT(FloatType);
 TYPE_VISITOR_DEFAULT(DoubleType);
 TYPE_VISITOR_DEFAULT(StringType);
 TYPE_VISITOR_DEFAULT(BinaryType);
+TYPE_VISITOR_DEFAULT(FixedWidthBinaryType);
 TYPE_VISITOR_DEFAULT(DateType);
 TYPE_VISITOR_DEFAULT(Date32Type);
 TYPE_VISITOR_DEFAULT(TimeType);
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 9f28875925a4b..a143d79013fb1 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -68,6 +68,9 @@ struct Type {
     // Variable-length bytes (no guarantee of UTF8-ness)
     BINARY,
 
+    // Fixed-width binary. Each value occupies the same number of bytes
+    FIXED_WIDTH_BINARY,
+
     // int64_t milliseconds since the UNIX epoch
     DATE,
 
@@ -135,6 +138,7 @@ class ARROW_EXPORT TypeVisitor {
   virtual Status Visit(const DoubleType& type);
   virtual Status Visit(const StringType& type);
   virtual Status Visit(const BinaryType& type);
+  virtual Status Visit(const FixedWidthBinaryType& type);
   virtual Status Visit(const DateType& type);
   virtual Status Visit(const Date32Type& type);
   virtual Status Visit(const TimeType& type);
@@ -347,7 +351,7 @@ struct ARROW_EXPORT ListType : public DataType, public NoExtraMeta {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-// BinaryType type is reprsents lists of 1-byte values.
+// BinaryType type is represents lists of 1-byte values.
 struct ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
   static constexpr Type::type type_id = Type::BINARY;
 
@@ -364,7 +368,27 @@ struct ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
   explicit BinaryType(Type::type logical_type) : DataType(logical_type) {}
 };
 
-// UTF encoded strings
+// BinaryType type is represents lists of 1-byte values.
+class ARROW_EXPORT FixedWidthBinaryType : public FixedWidthType {
+ public:
+  static constexpr Type::type type_id = Type::FIXED_WIDTH_BINARY;
+
+  explicit FixedWidthBinaryType(int32_t byte_width)
+      : FixedWidthType(Type::FIXED_WIDTH_BINARY), byte_width_(byte_width) {}
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override;
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
+
+  int32_t byte_width() const { return byte_width_; }
+  int bit_width() const override;
+
+ protected:
+  int32_t byte_width_;
+};
+
+// UTF-8 encoded strings
 struct ARROW_EXPORT StringType : public BinaryType {
   static constexpr Type::type type_id = Type::STRING;
 
@@ -571,6 +595,8 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 // ----------------------------------------------------------------------
 // Factory functions
 
+std::shared_ptr<DataType> ARROW_EXPORT fixed_width_binary(int32_t byte_width);
+
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type);
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
 
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index e53afe1a34d36..7fc36c4bde06b 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -48,6 +48,10 @@ struct BinaryType;
 class BinaryArray;
 class BinaryBuilder;
 
+class FixedWidthBinaryType;
+class FixedWidthBinaryArray;
+class FixedWidthBinaryBuilder;
+
 struct StringType;
 class StringArray;
 class StringBuilder;
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 91461da8c42a6..242e59d10fce4 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -228,6 +228,13 @@ struct TypeTraits<BinaryType> {
   static inline std::shared_ptr<DataType> type_singleton() { return binary(); }
 };
 
+template <>
+struct TypeTraits<FixedWidthBinaryType> {
+  using ArrayType = FixedWidthBinaryArray;
+  using BuilderType = FixedWidthBinaryBuilder;
+  constexpr static bool is_parameter_free = false;
+};
+
 // Not all type classes have a c_type
 template <typename T>
 struct as_void {
diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
index 9f2645699004c..34bee18df5229 100644
--- a/cpp/src/arrow/util/io-util.h
+++ b/cpp/src/arrow/util/io-util.h
@@ -18,9 +18,12 @@
 #ifndef ARROW_UTIL_IO_UTIL_H
 #define ARROW_UTIL_IO_UTIL_H
 
-#include "arrow/buffer.h"
 #include <iostream>
 
+#include "arrow/buffer.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/status.h"
+
 namespace arrow {
 namespace io {
 
diff --git a/cpp/src/arrow/util/string.h b/cpp/src/arrow/util/string.h
new file mode 100644
index 0000000000000..5d9fdc88ced7e
--- /dev/null
+++ b/cpp/src/arrow/util/string.h
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_STRING_UTIL_H
+#define ARROW_UTIL_STRING_UTIL_H
+
+#include <algorithm>
+#include <string>
+
+#include "arrow/status.h"
+
+namespace arrow {
+
+static const char* kAsciiTable = "0123456789ABCDEF";
+
+static inline std::string HexEncode(const char* data, int32_t length) {
+  std::string hex_string;
+  hex_string.reserve(length * 2);
+  for (int32_t j = 0; j < length; ++j) {
+    // Convert to 2 base16 digits
+    hex_string.push_back(kAsciiTable[data[j] >> 4]);
+    hex_string.push_back(kAsciiTable[data[j] & 15]);
+  }
+  return hex_string;
+}
+
+static inline Status ParseHexValue(const char* data, uint8_t* out) {
+  char c1 = data[0];
+  char c2 = data[1];
+
+  const char* pos1 = std::lower_bound(kAsciiTable, kAsciiTable + 16, c1);
+  const char* pos2 = std::lower_bound(kAsciiTable, kAsciiTable + 16, c2);
+
+  // Error checking
+  if (*pos1 != c1 || *pos2 != c2) { return Status::Invalid("Encountered non-hex digit"); }
+
+  *out = static_cast<uint8_t>((pos1 - kAsciiTable) << 4 | (pos2 - kAsciiTable));
+  return Status::OK();
+}
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_STRING_UTIL_H
diff --git a/format/Message.fbs b/format/Message.fbs
index 86dfa87b04807..fb3478de5d2a0 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -68,6 +68,11 @@ table Utf8 {
 table Binary {
 }
 
+table FixedWidthBinary {
+  /// Number of bytes per value
+  byteWidth: int;
+}
+
 table Bool {
 }
 
@@ -113,7 +118,8 @@ union Type {
   Interval,
   List,
   Struct_,
-  Union
+  Union,
+  FixedWidthBinary
 }
 
 /// ----------------------------------------------------------------------

From 3b650014f6c59c6cf6f488572c5cd340bf2da453 Mon Sep 17 00:00:00 2001
From: Johan Mabille <johan.mabille@gmail.com>
Date: Thu, 16 Mar 2017 12:01:13 -0400
Subject: [PATCH 0369/1644] ARROW-520: [C++] STL-compliant allocator

Ready for review

Author: Johan Mabille <johan.mabille@gmail.com>

Closes #381 from JohanMabille/stl_allocator and squashes the following commits:

53c6821 [Johan Mabille] stl allocator
---
 cpp/src/arrow/CMakeLists.txt    |  2 +
 cpp/src/arrow/allocator-test.cc | 72 ++++++++++++++++++++++++
 cpp/src/arrow/allocator.h       | 98 +++++++++++++++++++++++++++++++++
 cpp/src/arrow/memory_pool.h     |  6 +-
 4 files changed, 175 insertions(+), 3 deletions(-)
 create mode 100644 cpp/src/arrow/allocator-test.cc
 create mode 100644 cpp/src/arrow/allocator.h

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index ddeb81cae7b5b..0abd4b9c34b0a 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -17,6 +17,7 @@
 
 # Headers: top level
 install(FILES
+  allocator.h
   api.h
   array.h
   buffer.h
@@ -47,6 +48,7 @@ install(
 # Unit tests
 #######################################
 
+ADD_ARROW_TEST(allocator-test)
 ADD_ARROW_TEST(array-test)
 ADD_ARROW_TEST(array-decimal-test)
 ADD_ARROW_TEST(array-dictionary-test)
diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
new file mode 100644
index 0000000000000..0b242674bf175
--- /dev/null
+++ b/cpp/src/arrow/allocator-test.cc
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+#include "arrow/allocator.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+TEST(stl_allocator, MemoryTracking) {
+  auto pool = default_memory_pool();
+  stl_allocator<uint64_t> alloc;
+  uint64_t* data = alloc.allocate(100);
+
+  ASSERT_EQ(100 * sizeof(uint64_t), pool->bytes_allocated());
+
+  alloc.deallocate(data, 100);
+  ASSERT_EQ(0, pool->bytes_allocated());
+}
+
+#if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER))
+
+TEST(stl_allocator, TestOOM) {
+  stl_allocator<uint64_t> alloc;
+  uint64_t to_alloc = std::numeric_limits<uint64_t>::max();
+  ASSERT_THROW(alloc.allocate(to_alloc), std::bad_alloc);
+}
+
+TEST(stl_allocator, FreeLargeMemory) {
+  stl_allocator<uint8_t> alloc;
+
+  uint8_t* data = alloc.allocate(100);
+
+#ifndef NDEBUG
+  EXPECT_EXIT(alloc.deallocate(data, 120), ::testing::ExitedWithCode(1),
+      ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
+#endif
+
+  alloc.deallocate(data, 100);
+}
+
+TEST(stl_allocator, MaxMemory) {
+  DefaultMemoryPool pool;
+
+  ASSERT_EQ(0, pool.max_memory());
+  stl_allocator<uint8_t> alloc(&pool);
+  uint8_t* data = alloc.allocate(100);
+  uint8_t* data2 = alloc.allocate(100);
+
+  alloc.deallocate(data, 100);
+  alloc.deallocate(data2, 100);
+
+  ASSERT_EQ(200, pool.max_memory());
+}
+
+#endif  // ARROW_VALGRIND
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/allocator.h b/cpp/src/arrow/allocator.h
new file mode 100644
index 0000000000000..c976ba96b8d03
--- /dev/null
+++ b/cpp/src/arrow/allocator.h
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_ALLOCATOR_H
+#define ARROW_ALLOCATOR_H
+
+#include <cstddef>
+#include <memory>
+#include <utility>
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
+
+namespace arrow {
+
+template <class T>
+class stl_allocator {
+ public:
+  using value_type = T;
+  using pointer = T*;
+  using const_pointer = const T*;
+  using reference = T&;
+  using const_reference = const T&;
+  using size_type = std::size_t;
+  using difference_type = std::ptrdiff_t;
+
+  template <class U>
+  struct rebind {
+    using other = stl_allocator<U>;
+  };
+
+  stl_allocator() noexcept : pool_(default_memory_pool()) {}
+  explicit stl_allocator(MemoryPool* pool) noexcept : pool_(pool) {}
+
+  template <class U>
+  stl_allocator(const stl_allocator<U>& rhs) noexcept : pool_(rhs.pool_) {}
+
+  ~stl_allocator() { pool_ = nullptr; }
+
+  pointer address(reference r) const noexcept { return std::addressof(r); }
+
+  const_pointer address(const_reference r) const noexcept { return std::addressof(r); }
+
+  pointer allocate(size_type n, const void* /*hint*/ = nullptr) {
+    uint8_t* data;
+    Status s = pool_->Allocate(n * sizeof(T), &data);
+    if (!s.ok()) throw std::bad_alloc();
+    return reinterpret_cast<pointer>(data);
+  }
+
+  void deallocate(pointer p, size_type n) {
+    pool_->Free(reinterpret_cast<uint8_t*>(p), n * sizeof(T));
+  }
+
+  size_type size_max() const noexcept { return size_type(-1) / sizeof(T); }
+
+  template <class U, class... Args>
+  void construct(U* p, Args&&... args) {
+    new (reinterpret_cast<void*>(p)) U(std::forward<Args>(args)...);
+  }
+
+  template <class U>
+  void destroy(U* p) {
+    p->~U();
+  }
+
+  MemoryPool* pool() const noexcept { return pool_; }
+
+ private:
+  MemoryPool* pool_;
+};
+
+template <class T1, class T2>
+bool operator==(const stl_allocator<T1>& lhs, const stl_allocator<T2>& rhs) noexcept {
+  return lhs.pool() == rhs.pool();
+}
+
+template <class T1, class T2>
+bool operator!=(const stl_allocator<T1>& lhs, const stl_allocator<T2>& rhs) noexcept {
+  return !(lhs == rhs);
+}
+
+}  // namespace arrow
+
+#endif  // ARROW_ALLOCATOR_H
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 33d4c3e9aad52..0edfda635d0e8 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef ARROW_UTIL_MEMORY_POOL_H
-#define ARROW_UTIL_MEMORY_POOL_H
+#ifndef ARROW_MEMORY_POOL_H
+#define ARROW_MEMORY_POOL_H
 
 #include <atomic>
 #include <cstdint>
@@ -93,4 +93,4 @@ ARROW_EXPORT MemoryPool* default_memory_pool();
 
 }  // namespace arrow
 
-#endif  // ARROW_UTIL_MEMORY_POOL_H
+#endif  // ARROW_MEMORY_POOL_H

From 49f666e740208d1e6167537f141f27b6b78b77cb Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Thu, 16 Mar 2017 13:59:53 -0400
Subject: [PATCH 0370/1644] ARROW-542: Adding dictionary encoding to FileWriter

WIP for comments

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #334 from elahrvivaz/ARROW-542 and squashes the following commits:

5339730 [Emilio Lahr-Vivaz] fixing bitvector load of value count, adding struct integration test
00d78d3 [Emilio Lahr-Vivaz] fixing set bit validity value in NullableMapVector load
1679934 [Emilio Lahr-Vivaz] cleaning up license
70639e0 [Emilio Lahr-Vivaz] restoring vector loader test
bde4eee [Wes McKinney] Handle 0-length message indicator for EOS in C++ StreamReader
a24854b [Emilio Lahr-Vivaz] fixing StreamToFile conversion
2ee7cfb [Emilio Lahr-Vivaz] fixing FileToStream conversion
adec200 [Emilio Lahr-Vivaz] making arrow magic static, cleanup
8366288 [Emilio Lahr-Vivaz] making magic array private
127937f [Emilio Lahr-Vivaz] removing qualifier for magic
db9a007 [Emilio Lahr-Vivaz] adding dictionary tests to echo server
95c7b2a [Emilio Lahr-Vivaz] cleanup
45caa02 [Emilio Lahr-Vivaz] reverting basewriter dictionary methods
682db6f [Emilio Lahr-Vivaz] cleanup
a1508b9 [Emilio Lahr-Vivaz] removing dictionary vector method (instead use field.dictionary)
43c28af [Emilio Lahr-Vivaz] adding test for nested dictionary encoded list
92a1e6f [Emilio Lahr-Vivaz] fixing imports
e567564 [Emilio Lahr-Vivaz] adding field size check in vectorschemaroot
568fda5 [Emilio Lahr-Vivaz] imports, formatting
363308e [Emilio Lahr-Vivaz] fixing tests
2f69be1 [Emilio Lahr-Vivaz] not passing around dictionary vectors with dictionary fields, adding dictionary encoding to fields, restoring vector loader/unloader
e5c8e02 [Emilio Lahr-Vivaz] Merging dictionary unloader/loader with arrow writer/reader Creating base class for stream/file writer Creating base class with visitors for arrow messages Indentation fixes Other cleanup
d095f3f [Emilio Lahr-Vivaz] ARROW-542: Adding dictionary encoding to file and stream writing
---
 cpp/src/arrow/ipc/reader.cc                   |   6 +
 integration/integration_test.py               |   4 +
 .../org/apache/arrow/tools/EchoServer.java    |  48 +-
 .../org/apache/arrow/tools/FileRoundtrip.java |  48 +-
 .../org/apache/arrow/tools/FileToStream.java  |  27 +-
 .../org/apache/arrow/tools/Integration.java   |  83 +--
 .../org/apache/arrow/tools/StreamToFile.java  |  19 +-
 .../arrow/tools/ArrowFileTestFixtures.java    |  51 +-
 .../apache/arrow/tools/EchoServerTest.java    | 280 ++++++--
 .../apache/arrow/tools/TestIntegration.java   |  38 +-
 java/tools/tmptestfilesio                     | Bin 0 -> 628 bytes
 .../main/codegen/templates/MapWriters.java    |   8 +-
 .../templates/NullableValueVectors.java       |  40 +-
 .../main/codegen/templates/UnionVector.java   |  10 +-
 .../org/apache/arrow/vector/BitVector.java    |   2 +-
 .../org/apache/arrow/vector/FieldVector.java  |   4 +-
 .../org/apache/arrow/vector/VectorLoader.java |  13 +-
 .../apache/arrow/vector/VectorSchemaRoot.java |  32 +-
 .../apache/arrow/vector/VectorUnloader.java   |  27 +-
 .../complex/AbstractContainerVector.java      |   3 +-
 .../vector/complex/AbstractMapVector.java     |   9 +-
 .../complex/BaseRepeatedValueVector.java      |   5 +-
 .../vector/complex/DictionaryVector.java      | 229 ------
 .../arrow/vector/complex/ListVector.java      |  26 +-
 .../arrow/vector/complex/MapVector.java       |   5 +-
 .../vector/complex/NullableMapVector.java     |   9 +-
 .../complex/impl/ComplexWriterImpl.java       |   6 +-
 .../vector/complex/impl/PromotableWriter.java |   5 +-
 .../arrow/vector/dictionary/Dictionary.java   |  66 ++
 .../vector/dictionary/DictionaryEncoder.java  | 144 ++++
 .../vector/dictionary/DictionaryProvider.java |  47 ++
 .../arrow/vector/file/ArrowFileReader.java    | 142 ++++
 .../arrow/vector/file/ArrowFileWriter.java    |  59 ++
 .../apache/arrow/vector/file/ArrowFooter.java |   1 -
 .../apache/arrow/vector/file/ArrowMagic.java  |  37 +
 .../apache/arrow/vector/file/ArrowReader.java | 222 ++++--
 .../apache/arrow/vector/file/ArrowWriter.java | 173 ++++-
 .../apache/arrow/vector/file/ReadChannel.java |  11 +-
 .../SeekableReadChannel.java}                 |  29 +-
 .../arrow/vector/file/WriteChannel.java       |   7 +-
 .../vector/file/json/JsonFileReader.java      |  26 +-
 .../vector/schema/ArrowDictionaryBatch.java   |  60 ++
 .../arrow/vector/schema/ArrowMessage.java     |  30 +
 .../arrow/vector/schema/ArrowRecordBatch.java |   8 +-
 .../vector/stream/ArrowStreamReader.java      |  88 +--
 .../vector/stream/ArrowStreamWriter.java      |  75 +-
 .../vector/stream/MessageSerializer.java      | 164 ++++-
 .../org/apache/arrow/vector/types/Types.java  | 114 +--
 .../vector/types/pojo/DictionaryEncoding.java |  51 ++
 .../apache/arrow/vector/types/pojo/Field.java |  59 +-
 .../arrow/vector/TestDecimalVector.java       |   2 +-
 .../arrow/vector/TestDictionaryVector.java    |  82 +--
 .../apache/arrow/vector/TestListVector.java   |   4 +-
 .../apache/arrow/vector/TestValueVector.java  |  12 +-
 .../arrow/vector/TestVectorUnloadLoad.java    |  22 +-
 .../complex/impl/TestPromotableWriter.java    |   2 +-
 .../complex/writer/TestComplexWriter.java     |  14 +-
 .../arrow/vector/file/TestArrowFile.java      | 665 +++++++++++-------
 .../vector/file/TestArrowReaderWriter.java    |  28 +-
 .../arrow/vector/file/TestArrowStream.java    | 102 +++
 .../vector/file/TestArrowStreamPipe.java      | 163 +++++
 .../arrow/vector/file/json/TestJSONFile.java  |   4 +-
 .../vector/stream/MessageSerializerTest.java  |   8 +-
 .../arrow/vector/stream/TestArrowStream.java  |  96 ---
 .../vector/stream/TestArrowStreamPipe.java    | 129 ----
 65 files changed, 2497 insertions(+), 1486 deletions(-)
 create mode 100644 java/tools/tmptestfilesio
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/DictionaryVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
 rename java/vector/src/main/java/org/apache/arrow/vector/{types/Dictionary.java => file/SeekableReadChannel.java} (57%)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
 delete mode 100644 java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
 delete mode 100644 java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java

diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 973416670bdfa..4cb5f6cccc4c8 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -78,6 +78,12 @@ class StreamReader::StreamReaderImpl {
 
     int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
 
+    if (message_length == 0) {
+      // Optional 0 EOS control message
+      *message = nullptr;
+      return Status::OK();
+    }
+
     RETURN_NOT_OK(stream_->Read(message_length, &buffer));
     if (buffer->size() != message_length) {
       return Status::IOError("Unexpected end of stream trying to read message");
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 049436a751f38..5cd63c502bd20 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -680,12 +680,16 @@ def stream_to_file(self, stream_path, file_path):
         cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
                'org.apache.arrow.tools.StreamToFile',
                stream_path, file_path]
+        if self.debug:
+            print(' '.join(cmd))
         run_cmd(cmd)
 
     def file_to_stream(self, file_path, stream_path):
         cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
                'org.apache.arrow.tools.FileToStream',
                file_path, stream_path]
+        if self.debug:
+            print(' '.join(cmd))
         run_cmd(cmd)
 
 
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
index c00620e44b064..7c0cadd9d77dd 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
@@ -18,23 +18,19 @@
 package org.apache.arrow.tools;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.ServerSocket;
 import java.net.Socket;
-import java.util.ArrayList;
-import java.util.List;
+
+import com.google.common.base.Preconditions;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.stream.ArrowStreamReader;
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class EchoServer {
   private static final Logger LOGGER = LoggerFactory.getLogger(EchoServer.class);
 
@@ -57,30 +53,28 @@ public ClientConnection(Socket socket) {
 
     public void run() throws IOException {
       BufferAllocator  allocator = new RootAllocator(Long.MAX_VALUE);
-      List<ArrowRecordBatch> batches = new ArrayList<ArrowRecordBatch>();
-      try (
-        InputStream in = socket.getInputStream();
-        OutputStream out = socket.getOutputStream();
-        ArrowStreamReader reader = new ArrowStreamReader(in, allocator);
-      ) {
-        // Read the entire input stream.
-        reader.init();
-        while (true) {
-          ArrowRecordBatch batch = reader.nextRecordBatch();
-          if (batch == null) break;
-          batches.add(batch);
-        }
-        LOGGER.info(String.format("Received %d batches", batches.size()));
-
-        // Write it back
-        try (ArrowStreamWriter writer = new ArrowStreamWriter(out, reader.getSchema())) {
-          for (ArrowRecordBatch batch: batches) {
-            writer.writeRecordBatch(batch);
+      // Read the entire input stream and write it back
+      try (ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
+        VectorSchemaRoot root = reader.getVectorSchemaRoot();
+        // load the first batch before instantiating the writer so that we have any dictionaries
+        reader.loadNextBatch();
+        try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream())) {
+          writer.start();
+          int echoed = 0;
+          while (true) {
+            int rowCount = reader.getVectorSchemaRoot().getRowCount();
+            if (rowCount == 0) {
+              break;
+            } else {
+              writer.writeBatch();
+              echoed += rowCount;
+              reader.loadNextBatch();
+            }
           }
           writer.end();
           Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
+          LOGGER.info(String.format("Echoed %d records", echoed));
         }
-        LOGGER.info("Done writing stream back.");
       }
     }
 
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
index db7a1c23f9ca6..9fa7b761a5772 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
@@ -23,18 +23,12 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
-import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowFooter;
-import org.apache.arrow.vector.file.ArrowReader;
-import org.apache.arrow.vector.file.ArrowWriter;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.file.ArrowFileReader;
+import org.apache.arrow.vector.file.ArrowFileWriter;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -86,35 +80,27 @@ int run(String[] args) {
       File inFile = validateFile("input", inFileName);
       File outFile = validateFile("output", outFileName);
       BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); // TODO: close
-      try(
-          FileInputStream fileInputStream = new FileInputStream(inFile);
-          ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), allocator);) {
+      try (FileInputStream fileInputStream = new FileInputStream(inFile);
+           ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), allocator)) {
 
-        ArrowFooter footer = arrowReader.readFooter();
-        Schema schema = footer.getSchema();
+        VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+        Schema schema = root.getSchema();
         LOGGER.debug("Input file size: " + inFile.length());
         LOGGER.debug("Found schema: " + schema);
 
-        try (
-            FileOutputStream fileOutputStream = new FileOutputStream(outFile);
-            ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
-            ) {
-
-          // initialize vectors
-
-          List<ArrowBlock> recordBatches = footer.getRecordBatches();
-          for (ArrowBlock rbBlock : recordBatches) {
-            try (ArrowRecordBatch inRecordBatch = arrowReader.readRecordBatch(rbBlock);
-                VectorSchemaRoot root = new VectorSchemaRoot(schema, allocator);) {
-
-              VectorLoader vectorLoader = new VectorLoader(root);
-              vectorLoader.load(inRecordBatch);
-
-              VectorUnloader vectorUnloader = new VectorUnloader(root);
-              ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
-              arrowWriter.writeRecordBatch(recordBatch);
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outFile);
+             ArrowFileWriter arrowWriter = new ArrowFileWriter(root, arrowReader, fileOutputStream.getChannel())) {
+          arrowWriter.start();
+          while (true) {
+            arrowReader.loadNextBatch();
+            int loaded = root.getRowCount();
+            if (loaded == 0) {
+              break;
+            } else {
+              arrowWriter.writeBatch();
             }
           }
+          arrowWriter.end();
         }
         LOGGER.debug("Output file size: " + outFile.length());
       }
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
index ba6505cb48d08..d5345535d19dc 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
@@ -25,10 +25,8 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowFooter;
-import org.apache.arrow.vector.file.ArrowReader;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.file.ArrowFileReader;
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
 
 /**
@@ -36,19 +34,20 @@
  * first argument and the output is written to standard out.
  */
 public class FileToStream {
+
   public static void convert(FileInputStream in, OutputStream out) throws IOException {
     BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
-    try(
-        ArrowReader reader = new ArrowReader(in.getChannel(), allocator);) {
-      ArrowFooter footer = reader.readFooter();
-      try (
-        ArrowStreamWriter writer = new ArrowStreamWriter(out, footer.getSchema());
-      ) {
-        for (ArrowBlock block: footer.getRecordBatches()) {
-          try (ArrowRecordBatch batch = reader.readRecordBatch(block)) {
-            writer.writeRecordBatch(batch);
-          }
+    try (ArrowFileReader reader = new ArrowFileReader(in.getChannel(), allocator)) {
+      VectorSchemaRoot root = reader.getVectorSchemaRoot();
+      // load the first batch before instantiating the writer so that we have any dictionaries
+      reader.loadNextBatch();
+      try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, out)) {
+        writer.start();
+        while (root.getRowCount() > 0) {
+          writer.writeBatch();
+          reader.loadNextBatch();
         }
+        writer.end();
       }
     }
   }
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index 36d4ee5485470..5d4849c234383 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -28,16 +28,12 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowFooter;
-import org.apache.arrow.vector.file.ArrowReader;
-import org.apache.arrow.vector.file.ArrowWriter;
+import org.apache.arrow.vector.file.ArrowFileReader;
+import org.apache.arrow.vector.file.ArrowFileWriter;
 import org.apache.arrow.vector.file.json.JsonFileReader;
 import org.apache.arrow.vector.file.json.JsonFileWriter;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.Validator;
 import org.apache.commons.cli.CommandLine;
@@ -69,24 +65,18 @@ enum Command {
     ARROW_TO_JSON(true, false) {
       @Override
       public void execute(File arrowFile, File jsonFile) throws IOException {
-        try(
-            BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+        try(BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
             FileInputStream fileInputStream = new FileInputStream(arrowFile);
-            ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), allocator);) {
-          ArrowFooter footer = arrowReader.readFooter();
-          Schema schema = footer.getSchema();
+            ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), allocator)) {
+          VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+          Schema schema = root.getSchema();
           LOGGER.debug("Input file size: " + arrowFile.length());
           LOGGER.debug("Found schema: " + schema);
-          try (JsonFileWriter writer = new JsonFileWriter(jsonFile, JsonFileWriter.config().pretty(true));) {
+          try (JsonFileWriter writer = new JsonFileWriter(jsonFile, JsonFileWriter.config().pretty(true))) {
             writer.start(schema);
-            List<ArrowBlock> recordBatches = footer.getRecordBatches();
-            for (ArrowBlock rbBlock : recordBatches) {
-              try (ArrowRecordBatch inRecordBatch = arrowReader.readRecordBatch(rbBlock);
-                  VectorSchemaRoot root = new VectorSchemaRoot(schema, allocator);) {
-                VectorLoader vectorLoader = new VectorLoader(root);
-                vectorLoader.load(inRecordBatch);
-                writer.write(root);
-              }
+            for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
+              arrowReader.loadRecordBatch(rbBlock);
+              writer.write(root);
             }
           }
           LOGGER.debug("Output file size: " + jsonFile.length());
@@ -96,27 +86,22 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
     JSON_TO_ARROW(false, true) {
       @Override
       public void execute(File arrowFile, File jsonFile) throws IOException {
-        try (
-            BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
-            JsonFileReader reader = new JsonFileReader(jsonFile, allocator);
-            ) {
+        try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+             JsonFileReader reader = new JsonFileReader(jsonFile, allocator)) {
           Schema schema = reader.start();
           LOGGER.debug("Input file size: " + jsonFile.length());
           LOGGER.debug("Found schema: " + schema);
-          try (
-              FileOutputStream fileOutputStream = new FileOutputStream(arrowFile);
-              ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
-              ) {
-
-            // initialize vectors
-            VectorSchemaRoot root;
-            while ((root = reader.read()) != null) {
-              VectorUnloader vectorUnloader = new VectorUnloader(root);
-              try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();) {
-                arrowWriter.writeRecordBatch(recordBatch);
-              }
-              root.close();
+          try (FileOutputStream fileOutputStream = new FileOutputStream(arrowFile);
+               VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
+               // TODO json dictionaries
+               ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel())) {
+            arrowWriter.start();
+            reader.read(root);
+            while (root.getRowCount() != 0) {
+              arrowWriter.writeBatch();
+              reader.read(root);
             }
+            arrowWriter.end();
           }
           LOGGER.debug("Output file size: " + arrowFile.length());
         }
@@ -125,32 +110,26 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
     VALIDATE(true, true) {
       @Override
       public void execute(File arrowFile, File jsonFile) throws IOException {
-        try (
-            BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
-            JsonFileReader jsonReader = new JsonFileReader(jsonFile, allocator);
-            FileInputStream fileInputStream = new FileInputStream(arrowFile);
-            ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), allocator);
-            ) {
+        try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+             JsonFileReader jsonReader = new JsonFileReader(jsonFile, allocator);
+             FileInputStream fileInputStream = new FileInputStream(arrowFile);
+             ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), allocator)) {
           Schema jsonSchema = jsonReader.start();
-          ArrowFooter footer = arrowReader.readFooter();
-          Schema arrowSchema = footer.getSchema();
+          VectorSchemaRoot arrowRoot = arrowReader.getVectorSchemaRoot();
+          Schema arrowSchema = arrowRoot.getSchema();
           LOGGER.debug("Arrow Input file size: " + arrowFile.length());
           LOGGER.debug("ARROW schema: " + arrowSchema);
           LOGGER.debug("JSON Input file size: " + jsonFile.length());
           LOGGER.debug("JSON schema: " + jsonSchema);
           Validator.compareSchemas(jsonSchema, arrowSchema);
 
-          List<ArrowBlock> recordBatches = footer.getRecordBatches();
+          List<ArrowBlock> recordBatches = arrowReader.getRecordBlocks();
           Iterator<ArrowBlock> iterator = recordBatches.iterator();
           VectorSchemaRoot jsonRoot;
           while ((jsonRoot = jsonReader.read()) != null && iterator.hasNext()) {
             ArrowBlock rbBlock = iterator.next();
-            try (ArrowRecordBatch inRecordBatch = arrowReader.readRecordBatch(rbBlock);
-                VectorSchemaRoot arrowRoot = new VectorSchemaRoot(arrowSchema, allocator);) {
-              VectorLoader vectorLoader = new VectorLoader(arrowRoot);
-              vectorLoader.load(inRecordBatch);
-              Validator.compareVectorSchemaRoot(arrowRoot, jsonRoot);
-            }
+            arrowReader.loadRecordBatch(rbBlock);
+            Validator.compareVectorSchemaRoot(arrowRoot, jsonRoot);
             jsonRoot.close();
           }
           boolean hasMoreJSON = jsonRoot != null;
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
index c8a5c8914afcc..3b79d5b05e116 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
@@ -27,8 +27,8 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.file.ArrowWriter;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.file.ArrowFileWriter;
 import org.apache.arrow.vector.stream.ArrowStreamReader;
 
 /**
@@ -38,13 +38,16 @@ public class StreamToFile {
   public static void convert(InputStream in, OutputStream out) throws IOException {
     BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
     try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) {
-      reader.init();
-      try (ArrowWriter writer = new ArrowWriter(Channels.newChannel(out), reader.getSchema());) {
-        while (true) {
-          ArrowRecordBatch batch = reader.nextRecordBatch();
-          if (batch == null) break;
-          writer.writeRecordBatch(batch);
+      VectorSchemaRoot root = reader.getVectorSchemaRoot();
+      // load the first batch before instantiating the writer so that we have any dictionaries
+      reader.loadNextBatch();
+      try (ArrowFileWriter writer = new ArrowFileWriter(root, reader, Channels.newChannel(out))) {
+        writer.start();
+        while (root.getRowCount() > 0) {
+          writer.writeBatch();
+          reader.loadNextBatch();
         }
+        writer.end();
       }
     }
   }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
index 4cfc52fe08631..f752f7eaa74b9 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -23,13 +23,10 @@
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
@@ -37,10 +34,8 @@
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
 import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowFooter;
-import org.apache.arrow.vector.file.ArrowReader;
-import org.apache.arrow.vector.file.ArrowWriter;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.file.ArrowFileReader;
+import org.apache.arrow.vector.file.ArrowFileWriter;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Assert;
 
@@ -63,26 +58,14 @@ static void writeData(int count, MapVector parent) {
 
   static void validateOutput(File testOutFile, BufferAllocator allocator) throws Exception {
     // read
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        FileInputStream fileInputStream = new FileInputStream(testOutFile);
-        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        ) {
-      ArrowFooter footer = arrowReader.readFooter();
-      Schema schema = footer.getSchema();
-
-      // initialize vectors
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, readerAllocator)) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-
-        List<ArrowBlock> recordBatches = footer.getRecordBatches();
-        for (ArrowBlock rbBlock : recordBatches) {
-          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-            vectorLoader.load(recordBatch);
-          }
-          validateContent(COUNT, root);
-        }
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(testOutFile);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
+        arrowReader.loadRecordBatch(rbBlock);
+        validateContent(COUNT, root);
       }
     }
   }
@@ -96,16 +79,10 @@ static void validateContent(int count, VectorSchemaRoot root) {
   }
 
   static void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
-    Schema schema = new Schema(parent.getField().getChildren());
-    int valueCount = parent.getAccessor().getValueCount();
-    List<FieldVector> fields = parent.getChildrenFromFields();
-    VectorUnloader vectorUnloader = new VectorUnloader(schema, valueCount, fields);
-    try (
-        FileOutputStream fileOutputStream = new FileOutputStream(file);
-        ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
-        ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
-            ) {
-      arrowWriter.writeRecordBatch(recordBatch);
+    VectorSchemaRoot root = new VectorSchemaRoot(parent);
+    try (FileOutputStream fileOutputStream = new FileOutputStream(file);
+         ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel())) {
+      arrowWriter.writeBatch();
     }
   }
 
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 48d6162f423a3..706f8e2ca4d36 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -24,106 +24,268 @@
 import java.io.IOException;
 import java.net.Socket;
 import java.net.UnknownHostException;
-import java.util.ArrayList;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.NullableIntVector;
+import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.stream.ArrowStreamReader;
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.Text;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
-import io.netty.buffer.ArrowBuf;
-
 public class EchoServerTest {
-  public static ArrowBuf buf(BufferAllocator alloc, byte[] bytes) {
-    ArrowBuf buffer = alloc.buffer(bytes.length);
-    buffer.writeBytes(bytes);
-    return buffer;
+
+  private static EchoServer server;
+  private static int serverPort;
+  private static Thread serverThread;
+
+  @BeforeClass
+  public static void startEchoServer() throws IOException {
+    server = new EchoServer(0);
+    serverPort = server.port();
+    serverThread = new Thread() {
+      @Override
+      public void run() {
+        try {
+          server.run();
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+    };
+    serverThread.start();
   }
 
-  public static byte[] array(ArrowBuf buf) {
-    byte[] bytes = new byte[buf.readableBytes()];
-    buf.readBytes(bytes);
-    return bytes;
+  @AfterClass
+  public static void stopEchoServer() throws IOException, InterruptedException {
+    server.close();
+    serverThread.join();
   }
 
-  private void testEchoServer(int serverPort, Schema schema, List<ArrowRecordBatch> batches)
+  private void testEchoServer(int serverPort,
+                              Field field,
+                              NullableTinyIntVector vector,
+                              int batches)
       throws UnknownHostException, IOException {
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+    VectorSchemaRoot root = new VectorSchemaRoot(asList(field), asList((FieldVector) vector), 0);
     try (Socket socket = new Socket("localhost", serverPort);
-        ArrowStreamWriter writer = new ArrowStreamWriter(socket.getOutputStream(), schema);
+        ArrowStreamWriter writer = new ArrowStreamWriter(root, null, socket.getOutputStream());
         ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), alloc)) {
-      for (ArrowRecordBatch batch: batches) {
-        writer.writeRecordBatch(batch);
+      writer.start();
+      for (int i = 0; i < batches; i++) {
+        vector.allocateNew(16);
+        for (int j = 0; j < 8; j++) {
+          vector.getMutator().set(j, j + i);
+          vector.getMutator().set(j + 8, 0, (byte) (j + i));
+        }
+        vector.getMutator().setValueCount(16);
+        root.setRowCount(16);
+        writer.writeBatch();
       }
       writer.end();
 
-      reader.init();
-      assertEquals(schema, reader.getSchema());
-      for (int i = 0; i < batches.size(); i++) {
-        ArrowRecordBatch result = reader.nextRecordBatch();
-        ArrowRecordBatch expected = batches.get(i);
-        assertTrue(result != null);
-        assertEquals(expected.getBuffers().size(), result.getBuffers().size());
-        for (int j = 0; j < expected.getBuffers().size(); j++) {
-          assertTrue(expected.getBuffers().get(j).compareTo(result.getBuffers().get(j)) == 0);
+      assertEquals(new Schema(asList(field)), reader.getVectorSchemaRoot().getSchema());
+
+      NullableTinyIntVector readVector = (NullableTinyIntVector) reader.getVectorSchemaRoot().getFieldVectors().get(0);
+      for (int i = 0; i < batches; i++) {
+        reader.loadNextBatch();
+        assertEquals(16, reader.getVectorSchemaRoot().getRowCount());
+        assertEquals(16, readVector.getAccessor().getValueCount());
+        for (int j = 0; j < 8; j++) {
+          assertEquals(j + i, readVector.getAccessor().get(j));
+          assertTrue(readVector.getAccessor().isNull(j + 8));
         }
       }
-      ArrowRecordBatch result = reader.nextRecordBatch();
-      assertTrue(result == null);
+      reader.loadNextBatch();
+      assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
       assertEquals(reader.bytesRead(), writer.bytesWritten());
     }
   }
 
   @Test
   public void basicTest() throws InterruptedException, IOException {
-    final EchoServer server = new EchoServer(0);
-    int serverPort = server.port();
-    Thread serverThread = new Thread() {
-      @Override
-      public void run() {
-        try {
-          server.run();
-        } catch (IOException e) {
-          e.printStackTrace();
-        }
-      }
-    };
-    serverThread.start();
-
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
-    byte[] validity = new byte[] { (byte)255, 0};
-    byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
-    ArrowBuf validityb = buf(alloc, validity);
-    ArrowBuf valuesb =  buf(alloc, values);
-    ArrowRecordBatch batch = new ArrowRecordBatch(
-        16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb));
 
-    Schema schema = new Schema(asList(new Field(
-        "testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
+    Field field = new Field("testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList());
+    NullableTinyIntVector vector = new NullableTinyIntVector("testField", alloc, null);
+    Schema schema = new Schema(asList(field));
 
     // Try an empty stream, just the header.
-    testEchoServer(serverPort, schema, new ArrayList<ArrowRecordBatch>());
+    testEchoServer(serverPort, field, vector, 0);
 
     // Try with one batch.
-    List<ArrowRecordBatch> batches = new ArrayList<>();
-    batches.add(batch);
-    testEchoServer(serverPort, schema, batches);
+    testEchoServer(serverPort, field, vector, 1);
 
     // Try with a few
-    for (int i = 0; i < 10; i++) {
-      batches.add(batch);
+    testEchoServer(serverPort, field, vector, 10);
+  }
+
+  @Test
+  public void testFlatDictionary() throws IOException {
+    DictionaryEncoding writeEncoding = new DictionaryEncoding(1L, false, null);
+    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+         NullableIntVector writeVector = new NullableIntVector("varchar", allocator, writeEncoding);
+         NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dict", allocator, null)) {
+      writeVector.allocateNewSafe();
+      NullableIntVector.Mutator mutator = writeVector.getMutator();
+      mutator.set(0, 0);
+      mutator.set(1, 1);
+      mutator.set(3, 2);
+      mutator.set(4, 1);
+      mutator.set(5, 2);
+      mutator.setValueCount(6);
+
+      writeDictionaryVector.allocateNewSafe();
+      NullableVarCharVector.Mutator dictionaryMutator = writeDictionaryVector.getMutator();
+      dictionaryMutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+      dictionaryMutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+      dictionaryMutator.set(2, "baz".getBytes(StandardCharsets.UTF_8));
+      dictionaryMutator.setValueCount(3);
+
+      List<Field> fields = ImmutableList.of(writeVector.getField());
+      List<FieldVector> vectors = ImmutableList.of((FieldVector) writeVector);
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 6);
+
+      DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary(writeDictionaryVector, writeEncoding));
+
+      try (Socket socket = new Socket("localhost", serverPort);
+           ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket.getOutputStream());
+           ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
+        writer.start();
+        writer.writeBatch();
+        writer.end();
+
+        reader.loadNextBatch();
+        VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot();
+        Assert.assertEquals(6, readerRoot.getRowCount());
+
+        FieldVector readVector = readerRoot.getFieldVectors().get(0);
+        Assert.assertNotNull(readVector);
+
+        DictionaryEncoding readEncoding = readVector.getField().getDictionary();
+        Assert.assertNotNull(readEncoding);
+        Assert.assertEquals(1L, readEncoding.getId());
+
+        FieldVector.Accessor accessor = readVector.getAccessor();
+        Assert.assertEquals(6, accessor.getValueCount());
+        Assert.assertEquals(0, accessor.getObject(0));
+        Assert.assertEquals(1, accessor.getObject(1));
+        Assert.assertEquals(null, accessor.getObject(2));
+        Assert.assertEquals(2, accessor.getObject(3));
+        Assert.assertEquals(1, accessor.getObject(4));
+        Assert.assertEquals(2, accessor.getObject(5));
+
+        Dictionary dictionary = reader.lookup(1L);
+        Assert.assertNotNull(dictionary);
+        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
+        Assert.assertEquals(3, dictionaryAccessor.getValueCount());
+        Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
+        Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+        Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2));
+      }
     }
-    testEchoServer(serverPort, schema, batches);
+  }
 
-    server.close();
-    serverThread.join();
+  @Test
+  public void testNestedDictionary() throws IOException {
+    DictionaryEncoding writeEncoding = new DictionaryEncoding(2L, false, null);
+    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+         NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dictionary", allocator, null);
+         ListVector writeVector = new ListVector("list", allocator, null, null)) {
+
+      // data being written:
+      // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
+
+      writeDictionaryVector.allocateNew();
+      writeDictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8));
+      writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
+      writeDictionaryVector.getMutator().setValueCount(2);
+
+      writeVector.addOrGetVector(MinorType.INT, writeEncoding);
+      writeVector.allocateNew();
+      UnionListWriter listWriter = new UnionListWriter(writeVector);
+      listWriter.startList();
+      listWriter.writeInt(0);
+      listWriter.writeInt(1);
+      listWriter.endList();
+      listWriter.startList();
+      listWriter.writeInt(0);
+      listWriter.endList();
+      listWriter.startList();
+      listWriter.writeInt(1);
+      listWriter.endList();
+      listWriter.setValueCount(3);
+
+      List<Field> fields = ImmutableList.of(writeVector.getField());
+      List<FieldVector> vectors = ImmutableList.of((FieldVector) writeVector);
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 3);
+
+      DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary(writeDictionaryVector, writeEncoding));
+
+      try (Socket socket = new Socket("localhost", serverPort);
+           ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket.getOutputStream());
+           ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
+        writer.start();
+        writer.writeBatch();
+        writer.end();
+
+        reader.loadNextBatch();
+        VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot();
+        Assert.assertEquals(3, readerRoot.getRowCount());
+
+        ListVector readVector = (ListVector) readerRoot.getFieldVectors().get(0);
+        Assert.assertNotNull(readVector);
+
+        Assert.assertNull(readVector.getField().getDictionary());
+        DictionaryEncoding readEncoding = readVector.getField().getChildren().get(0).getDictionary();
+        Assert.assertNotNull(readEncoding);
+        Assert.assertEquals(2L, readEncoding.getId());
+
+        Field nestedField = readVector.getField().getChildren().get(0);
+
+        DictionaryEncoding encoding = nestedField.getDictionary();
+        Assert.assertNotNull(encoding);
+        Assert.assertEquals(2L, encoding.getId());
+        Assert.assertEquals(new Int(32, true), encoding.getIndexType());
+
+        ListVector.Accessor accessor = readVector.getAccessor();
+        Assert.assertEquals(3, accessor.getValueCount());
+        Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
+        Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
+        Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
+
+        Dictionary readDictionary = reader.lookup(2L);
+        Assert.assertNotNull(readDictionary);
+        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) readDictionary.getVector()).getAccessor();
+        Assert.assertEquals(2, dictionaryAccessor.getValueCount());
+        Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
+        Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+      }
+    }
   }
 }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
index 0ae32bebe0b30..9d4ef5c26505b 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -33,6 +33,11 @@
 import java.io.StringReader;
 import java.util.Map;
 
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.tools.Integration.Command;
@@ -49,11 +54,6 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
-import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
-import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-
 public class TestIntegration {
 
   @Rule
@@ -128,6 +128,34 @@ public void testJSONRoundTripWithVariableWidth() throws Exception {
     }
   }
 
+  @Test
+  public void testJSONRoundTripWithStruct() throws Exception {
+    File testJSONFile = new File("../../integration/data/struct_example.json");
+    File testOutFile = testFolder.newFile("testOutStruct.arrow");
+    File testRoundTripJSONFile = testFolder.newFile("testOutStruct.json");
+    testOutFile.delete();
+    testRoundTripJSONFile.delete();
+
+    Integration integration = new Integration();
+
+    // convert to arrow
+    String[] args1 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
+    integration.run(args1);
+
+    // convert back to json
+    String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testRoundTripJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    integration.run(args2);
+
+    BufferedReader orig = readNormalized(testJSONFile);
+    BufferedReader rt = readNormalized(testRoundTripJSONFile);
+    String i, o;
+    int j = 0;
+    while ((i = orig.readLine()) != null && (o = rt.readLine()) != null) {
+      assertEquals("line: " + j, i, o);
+      ++j;
+    }
+  }
+
   private ObjectMapper om = new ObjectMapper();
   {
     DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
diff --git a/java/tools/tmptestfilesio b/java/tools/tmptestfilesio
new file mode 100644
index 0000000000000000000000000000000000000000..d1b6b6cdb93878637bff514fbacc2b0054dd5f4d
GIT binary patch
literal 628
zcmZ{hJx;?w5QU$UB`gsjgpz<n3h2234JAjQ!~sR=BDtX83S5d~a0G5)-nZV75r}6!
z&(3)M-uhG0!UV@U)l4VK<xHf}SuyhxOFqv{Y-3)U3$1N0v~p&02Wsi}RPBT9&Xf+C
z7r&?Gtw}9?zWTn;?w0+`>{BmY7e6_I-)K^zOlJyU4|WJ7$N6$-zP8=-uI=>U^QdjD
zb6weIOX<cOQFTg_?5yk@=Cb2BgW2dyWi~w?P4vLu80&^FgDwsC`6HPb;Ak-AO>Oi<
zPk<PAMDBLK-?ZJPeS5bvid^{@F6*!08Ut0}^sk6}r_eO6I4ZFJiHM~mhx(d(N30^s
JY0WuCd;|NY6es`y

literal 0
HcmV?d00001

diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 4af6eee91b6de..428ce0427d4b8 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -64,7 +64,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
         list(child.getName());
         break;
       case UNION:
-        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), MinorType.UNION, UnionVector.class), getNullableMapWriterFactory());
+        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), MinorType.UNION, UnionVector.class, null), getNullableMapWriterFactory());
         fields.put(handleCase(child.getName()), writer);
         break;
 <#list vv.types as type><#list type.minor as minor>
@@ -113,7 +113,7 @@ public MapWriter map(String name) {
     FieldWriter writer = fields.get(finalName);
     if(writer == null){
       int vectorCount=container.size();
-      NullableMapVector vector = container.addOrGet(name, MinorType.MAP, NullableMapVector.class);
+      NullableMapVector vector = container.addOrGet(name, MinorType.MAP, NullableMapVector.class, null);
       writer = new PromotableWriter(vector, container, getNullableMapWriterFactory());
       if(vectorCount != container.size()) {
         writer.allocate();
@@ -157,7 +157,7 @@ public ListWriter list(String name) {
     FieldWriter writer = fields.get(finalName);
     int vectorCount = container.size();
     if(writer == null) {
-      writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class), container, getNullableMapWriterFactory());
+      writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class, null), container, getNullableMapWriterFactory());
       if (container.size() > vectorCount) {
         writer.allocate();
       }
@@ -222,7 +222,7 @@ public void end() {
     if(writer == null) {
       ValueVector vector;
       ValueVector currentVector = container.getChild(name);
-      ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class<#if minor.class == "Decimal"> , new int[] {precision, scale}</#if>);
+      ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class, null<#if minor.class == "Decimal"> , new int[] {precision, scale}</#if>);
       writer = new PromotableWriter(v, container, getNullableMapWriterFactory());
       vector = v;
       if (currentVector == null || currentVector != vector) {
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 6b25fb36b40c0..b3e10e3fa87a2 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -65,21 +65,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   private final int precision;
   private final int scale;
 
-  public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
+  public ${className}(String name, BufferAllocator allocator, DictionaryEncoding dictionary, int precision, int scale) {
     super(name, allocator);
     values = new ${valuesName}(valuesField, allocator, precision, scale);
     this.precision = precision;
     this.scale = scale;
     mutator = new Mutator();
     accessor = new Accessor();
-    field = new Field(name, true, new Decimal(precision, scale), null);
+    field = new Field(name, true, new Decimal(precision, scale), dictionary, null);
     innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
         bits,
         values
     ));
   }
   <#else>
-  public ${className}(String name, BufferAllocator allocator) {
+  public ${className}(String name, BufferAllocator allocator, DictionaryEncoding dictionary) {
     super(name, allocator);
     values = new ${valuesName}(valuesField, allocator);
     mutator = new Mutator();
@@ -88,38 +88,38 @@ public final class ${className} extends BaseDataValueVector implements <#if type
         minor.class == "SmallInt" ||
         minor.class == "Int" ||
         minor.class == "BigInt">
-    field = new Field(name, true, new Int(${type.width} * 8, true), null);
+    field = new Field(name, true, new Int(${type.width} * 8, true), dictionary, null);
   <#elseif minor.class == "UInt1" ||
         minor.class == "UInt2" ||
         minor.class == "UInt4" ||
         minor.class == "UInt8">
-    field = new Field(name, true, new Int(${type.width} * 8, false), null);
+    field = new Field(name, true, new Int(${type.width} * 8, false), dictionary, null);
   <#elseif minor.class == "Date">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Date(), null);
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Date(), dictionary, null);
   <#elseif minor.class == "Time">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), null);
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), dictionary, null);
   <#elseif minor.class == "Float4">
-    field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), null);
+    field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), dictionary, null);
   <#elseif minor.class == "Float8">
-    field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), null);
+    field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), dictionary, null);
   <#elseif minor.class == "TimeStampSec">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND), null);
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND), dictionary, null);
   <#elseif minor.class == "TimeStampMilli">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), null);
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), dictionary, null);
   <#elseif minor.class == "TimeStampMicro">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND), null);
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND), dictionary, null);
   <#elseif minor.class == "TimeStampNano">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND), null);
+    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND), dictionary, null);
   <#elseif minor.class == "IntervalDay">
-    field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), null);
+    field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), dictionary, null);
   <#elseif minor.class == "IntervalYear">
-    field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.YEAR_MONTH), null);
+    field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.YEAR_MONTH), dictionary, null);
   <#elseif minor.class == "VarChar">
-    field = new Field(name, true, new Utf8(), null);
+    field = new Field(name, true, new Utf8(), dictionary, null);
   <#elseif minor.class == "VarBinary">
-    field = new Field(name, true, new Binary(), null);
+    field = new Field(name, true, new Binary(), dictionary, null);
   <#elseif minor.class == "Bit">
-    field = new Field(name, true, new Bool(), null);
+    field = new Field(name, true, new Bool(), dictionary, null);
   </#if>
     innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
         bits,
@@ -378,9 +378,9 @@ private class TransferImpl implements TransferPair {
 
     public TransferImpl(String name, BufferAllocator allocator){
       <#if minor.class == "Decimal">
-      to = new ${className}(name, allocator, precision, scale);
+      to = new ${className}(name, allocator, field.getDictionary(), precision, scale);
       <#else>
-      to = new ${className}(name, allocator);
+      to = new ${className}(name, allocator, field.getDictionary());
       </#if>
     }
 
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 1a6908df2c40d..076ed93999623 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -118,11 +118,11 @@ public List<ArrowBuf> getFieldBuffers() {
   public List<BufferBacked> getFieldInnerVectors() {
      return this.innerVectors;
   }
-  
+
   public NullableMapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
-      mapVector = internalMap.addOrGet("map", MinorType.MAP, NullableMapVector.class);
+      mapVector = internalMap.addOrGet("map", MinorType.MAP, NullableMapVector.class, null);
       if (internalMap.size() > vectorCount) {
         mapVector.allocateNew();
         if (callBack != null) {
@@ -144,7 +144,7 @@ public NullableMapVector getMap() {
   public Nullable${name}Vector get${name}Vector() {
     if (${uncappedName}Vector == null) {
       int vectorCount = internalMap.size();
-      ${uncappedName}Vector = internalMap.addOrGet("${lowerCaseName}", MinorType.${name?upper_case}, Nullable${name}Vector.class);
+      ${uncappedName}Vector = internalMap.addOrGet("${lowerCaseName}", MinorType.${name?upper_case}, Nullable${name}Vector.class, null);
       if (internalMap.size() > vectorCount) {
         ${uncappedName}Vector.allocateNew();
         if (callBack != null) {
@@ -162,7 +162,7 @@ public NullableMapVector getMap() {
   public ListVector getList() {
     if (listVector == null) {
       int vectorCount = internalMap.size();
-      listVector = internalMap.addOrGet("list", MinorType.LIST, ListVector.class);
+      listVector = internalMap.addOrGet("list", MinorType.LIST, ListVector.class, null);
       if (internalMap.size() > vectorCount) {
         listVector.allocateNew();
         if (callBack != null) {
@@ -262,7 +262,7 @@ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
   public FieldVector addVector(FieldVector v) {
     String name = v.getMinorType().name().toLowerCase();
     Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
-    final FieldVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass());
+    final FieldVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass(), v.getField().getDictionary());
     v.makeTransferPair(newVector).transfer();
     internalMap.putChild(name, newVector);
     if (callBack != null) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index d1e9abe5dd111..179f2ee879f43 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -81,6 +81,7 @@ public void load(ArrowFieldNode fieldNode, ArrowBuf data) {
     } else {
       super.load(fieldNode, data);
     }
+    this.valueCount = fieldNode.getLength();
   }
 
   @Override
@@ -451,7 +452,6 @@ public final void setToOne(int index) {
 
     /**
      * set count bits to 1 in data starting at firstBitIndex
-     * @param data the buffer to set
      * @param firstBitIndex the index of the first bit to set
      * @param count the number of bits to set
      */
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
index b28433cfd0d94..0fdbc48552aaa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
@@ -19,11 +19,10 @@
 
 import java.util.List;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 
-import io.netty.buffer.ArrowBuf;
-
 /**
  * A vector corresponding to a Field in the schema
  * It has inner vectors backed by buffers (validity, offsets, data, ...)
@@ -61,5 +60,4 @@ public interface FieldVector extends ValueVector {
    * @return the inner vectors for this field as defined by the TypeLayout
    */
   List<BufferBacked> getFieldInnerVectors();
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 5c1176cf95d26..76de250e0e972 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -36,15 +36,14 @@
  * Loads buffers into vectors
  */
 public class VectorLoader {
+
   private final VectorSchemaRoot root;
 
   /**
    * will create children in root based on schema
-   * @param schema the expected schema
    * @param root the root to add vectors to based on schema
    */
   public VectorLoader(VectorSchemaRoot root) {
-    super();
     this.root = root;
   }
 
@@ -57,18 +56,16 @@ public void load(ArrowRecordBatch recordBatch) {
     Iterator<ArrowBuf> buffers = recordBatch.getBuffers().iterator();
     Iterator<ArrowFieldNode> nodes = recordBatch.getNodes().iterator();
     List<Field> fields = root.getSchema().getFields();
-    for (int i = 0; i < fields.size(); ++i) {
-      Field field = fields.get(i);
+    for (Field field: fields) {
       FieldVector fieldVector = root.getVector(field.getName());
       loadBuffers(fieldVector, field, buffers, nodes);
     }
     root.setRowCount(recordBatch.getLength());
     if (nodes.hasNext() || buffers.hasNext()) {
-      throw new IllegalArgumentException("not all nodes and buffers where consumed. nodes: " + Iterators.toString(nodes) + " buffers: " + Iterators.toString(buffers));
+      throw new IllegalArgumentException("not all nodes and buffers were consumed. nodes: " + Iterators.toString(nodes) + " buffers: " + Iterators.toString(buffers));
     }
   }
 
-
   private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buffers, Iterator<ArrowFieldNode> nodes) {
     checkArgument(nodes.hasNext(),
         "no more field nodes for for field " + field + " and vector " + vector);
@@ -82,7 +79,7 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
       vector.loadFieldBuffers(fieldNode, ownBuffers);
     } catch (RuntimeException e) {
       throw new IllegalArgumentException("Could not load buffers for field " +
-              field + ". error message: " + e.getMessage(), e);
+            field + ". error message: " + e.getMessage(), e);
     }
     List<Field> children = field.getChildren();
     if (children.size() > 0) {
@@ -96,4 +93,4 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
     }
   }
 
-}
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
index 1cbe18787ef45..7e626fb14305e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -18,7 +18,6 @@
 package org.apache.arrow.vector;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -29,6 +28,9 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 
+/**
+ * Holder for a set of vectors to be loaded/unloaded
+ */
 public class VectorSchemaRoot implements AutoCloseable {
 
   private final Schema schema;
@@ -37,9 +39,17 @@ public class VectorSchemaRoot implements AutoCloseable {
   private final Map<String, FieldVector> fieldVectorsMap = new HashMap<>();
 
   public VectorSchemaRoot(FieldVector parent) {
-    this.schema = new Schema(parent.getField().getChildren());
-    this.rowCount = parent.getAccessor().getValueCount();
-    this.fieldVectors = parent.getChildrenFromFields();
+    this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getAccessor().getValueCount());
+  }
+
+  public VectorSchemaRoot(List<Field> fields, List<FieldVector> fieldVectors, int rowCount) {
+    if (fields.size() != fieldVectors.size()) {
+      throw new IllegalArgumentException("Fields must match field vectors. Found " +
+          fieldVectors.size() + " vectors and " + fields.size() + " fields");
+    }
+    this.schema = new Schema(fields);
+    this.rowCount = rowCount;
+    this.fieldVectors = fieldVectors;
     for (int i = 0; i < schema.getFields().size(); ++i) {
       Field field = schema.getFields().get(i);
       FieldVector vector = fieldVectors.get(i);
@@ -47,21 +57,19 @@ public VectorSchemaRoot(FieldVector parent) {
     }
   }
 
-  public VectorSchemaRoot(Schema schema, BufferAllocator allocator) {
-    super();
-    this.schema = schema;
+  public static VectorSchemaRoot create(Schema schema, BufferAllocator allocator) {
     List<FieldVector> fieldVectors = new ArrayList<>();
     for (Field field : schema.getFields()) {
       MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
-      FieldVector vector = minorType.getNewVector(field.getName(), allocator, null);
+      FieldVector vector = minorType.getNewVector(field.getName(), allocator, field.getDictionary(), null);
       vector.initializeChildrenFromFields(field.getChildren());
       fieldVectors.add(vector);
-      fieldVectorsMap.put(field.getName(), vector);
     }
-    this.fieldVectors = Collections.unmodifiableList(fieldVectors);
-    if (this.fieldVectors.size() != schema.getFields().size()) {
-      throw new IllegalArgumentException("The root vector did not create the right number of children. found " + fieldVectors.size() + " expected " + schema.getFields().size());
+    if (fieldVectors.size() != schema.getFields().size()) {
+      throw new IllegalArgumentException("The root vector did not create the right number of children. found " +
+        fieldVectors.size() + " expected " + schema.getFields().size());
     }
+    return new VectorSchemaRoot(schema.getFields(), fieldVectors, 0);
   }
 
   public List<FieldVector> getFieldVectors() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 92d8cb045ae31..8e9ff6d462c5c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -20,42 +20,27 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.schema.ArrowVectorType;
-import org.apache.arrow.vector.types.pojo.Schema;
-
-import io.netty.buffer.ArrowBuf;
 
 public class VectorUnloader {
 
-  private final Schema schema;
-  private final int valueCount;
-  private final List<FieldVector> vectors;
-
-  public VectorUnloader(Schema schema, int valueCount, List<FieldVector> vectors) {
-    super();
-    this.schema = schema;
-    this.valueCount = valueCount;
-    this.vectors = vectors;
-  }
+  private final VectorSchemaRoot root;
 
   public VectorUnloader(VectorSchemaRoot root) {
-    this(root.getSchema(), root.getRowCount(), root.getFieldVectors());
-  }
-
-  public Schema getSchema() {
-    return schema;
+    this.root = root;
   }
 
   public ArrowRecordBatch getRecordBatch() {
     List<ArrowFieldNode> nodes = new ArrayList<>();
     List<ArrowBuf> buffers = new ArrayList<>();
-    for (FieldVector vector : vectors) {
+    for (FieldVector vector : root.getFieldVectors()) {
       appendNodes(vector, nodes, buffers);
     }
-    return new ArrowRecordBatch(valueCount, nodes, buffers);
+    return new ArrowRecordBatch(root.getRowCount(), nodes, buffers);
   }
 
   private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
@@ -74,4 +59,4 @@ private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<Ar
     }
   }
 
-}
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 2f68886a169b3..86a5e82119831 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -22,6 +22,7 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.util.CallBack;
 
 /**
@@ -85,7 +86,7 @@ protected boolean supportsDirectRead() {
   public abstract int size();
 
   // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
-  public abstract <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale);
+  public abstract <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, DictionaryEncoding dictionary, int... precisionScale);
 
   // return the child vector with the input name
   public abstract <T extends FieldVector> T getChild(String name, Class<T> clazz);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index f030d166ade8d..baeeb07873714 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -26,6 +26,7 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.MapWithOrdinal;
 
@@ -110,7 +111,7 @@ public boolean allocateNewSafe() {
    * @return resultant {@link org.apache.arrow.vector.ValueVector}
    */
   @Override
-  public <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
+  public <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, DictionaryEncoding dictionary, int... precisionScale) {
     final ValueVector existing = getChild(name);
     boolean create = false;
     if (existing == null) {
@@ -122,7 +123,7 @@ public <T extends FieldVector> T addOrGet(String name, MinorType minorType, Clas
       create = true;
     }
     if (create) {
-      final T vector = clazz.cast(minorType.getNewVector(name, allocator, callBack, precisionScale));
+      final T vector = clazz.cast(minorType.getNewVector(name, allocator, dictionary, callBack, precisionScale));
       putChild(name, vector);
       if (callBack!=null) {
         callBack.doWork();
@@ -162,12 +163,12 @@ public <T extends FieldVector> T getChild(String name, Class<T> clazz) {
     return typeify(v, clazz);
   }
 
-  protected ValueVector add(String name, MinorType minorType, int... precisionScale) {
+  protected ValueVector add(String name, MinorType minorType, DictionaryEncoding dictionary, int... precisionScale) {
     final ValueVector existing = getChild(name);
     if (existing != null) {
       throw new IllegalStateException(String.format("Vector already exists: Existing[%s], Requested[%s] ", existing.getClass().getSimpleName(), minorType));
     }
-    FieldVector vector = minorType.getNewVector(name, allocator, callBack, precisionScale);
+    FieldVector vector = minorType.getNewVector(name, allocator, dictionary, callBack, precisionScale);
     putChild(name, vector);
     if (callBack!=null) {
       callBack.doWork();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 7424df474ae89..eeb8f5830f404 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -28,6 +28,7 @@
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
 import com.google.common.base.Preconditions;
@@ -150,10 +151,10 @@ public int size() {
     return vector == DEFAULT_DATA_VECTOR ? 0:1;
   }
 
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType, DictionaryEncoding dictionary) {
     boolean created = false;
     if (vector instanceof ZeroVector) {
-      vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, null);
+      vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, dictionary, null);
       // returned vector must have the same field
       created = true;
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/DictionaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/DictionaryVector.java
deleted file mode 100644
index 84760eadf2253..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/DictionaryVector.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.NullableIntVector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.types.Dictionary;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.util.TransferPair;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-public class DictionaryVector implements ValueVector {
-
-  private ValueVector indices;
-  private Dictionary dictionary;
-
-  public DictionaryVector(ValueVector indices, Dictionary dictionary) {
-    this.indices = indices;
-    this.dictionary = dictionary;
-  }
-
-  /**
-   * Dictionary encodes a vector. The dictionary will be built using the values from the vector.
-   *
-   * @param vector vector to encode
-   * @return dictionary encoded vector
-   */
-  public static DictionaryVector encode(ValueVector vector) {
-    validateType(vector.getMinorType());
-    Map<Object, Integer> lookUps = new HashMap<>();
-    Map<Integer, Integer> transfers = new HashMap<>();
-
-    ValueVector.Accessor accessor = vector.getAccessor();
-    int count = accessor.getValueCount();
-
-    NullableIntVector indices = new NullableIntVector(vector.getField().getName(), vector.getAllocator());
-    indices.allocateNew(count);
-    NullableIntVector.Mutator mutator = indices.getMutator();
-
-    int nextIndex = 0;
-    for (int i = 0; i < count; i++) {
-      Object value = accessor.getObject(i);
-      if (value != null) { // if it's null leave it null
-        Integer index = lookUps.get(value);
-        if (index == null) {
-          index = nextIndex++;
-          lookUps.put(value, index);
-          transfers.put(i, index);
-        }
-        mutator.set(i, index);
-      }
-    }
-    mutator.setValueCount(count);
-
-    // copy the dictionary values into the dictionary vector
-    TransferPair dictionaryTransfer = vector.getTransferPair(vector.getAllocator());
-    ValueVector dictionaryVector = dictionaryTransfer.getTo();
-    dictionaryVector.allocateNewSafe();
-    for (Map.Entry<Integer, Integer> entry: transfers.entrySet()) {
-      dictionaryTransfer.copyValueSafe(entry.getKey(), entry.getValue());
-    }
-    dictionaryVector.getMutator().setValueCount(transfers.size());
-    Dictionary dictionary = new Dictionary(dictionaryVector, false);
-
-    return new DictionaryVector(indices, dictionary);
-  }
-
-  /**
-   * Dictionary encodes a vector with a provided dictionary. The dictionary must contain all values in the vector.
-   *
-   * @param vector vector to encode
-   * @param dictionary dictionary used for encoding
-   * @return dictionary encoded vector
-   */
-  public static DictionaryVector encode(ValueVector vector, Dictionary dictionary) {
-    validateType(vector.getMinorType());
-    // load dictionary values into a hashmap for lookup
-    ValueVector.Accessor dictionaryAccessor = dictionary.getDictionary().getAccessor();
-    Map<Object, Integer> lookUps = new HashMap<>(dictionaryAccessor.getValueCount());
-    for (int i = 0; i < dictionaryAccessor.getValueCount(); i++) {
-      // for primitive array types we need a wrapper that implements equals and hashcode appropriately
-      lookUps.put(dictionaryAccessor.getObject(i), i);
-    }
-
-    // vector to hold our indices (dictionary encoded values)
-    NullableIntVector indices = new NullableIntVector(vector.getField().getName(), vector.getAllocator());
-    NullableIntVector.Mutator mutator = indices.getMutator();
-
-    ValueVector.Accessor accessor = vector.getAccessor();
-    int count = accessor.getValueCount();
-
-    indices.allocateNew(count);
-
-    for (int i = 0; i < count; i++) {
-      Object value = accessor.getObject(i);
-      if (value != null) { // if it's null leave it null
-        // note: this may fail if value was not included in the dictionary
-        mutator.set(i, lookUps.get(value));
-      }
-    }
-    mutator.setValueCount(count);
-
-    return new DictionaryVector(indices, dictionary);
-  }
-
-  /**
-   * Decodes a dictionary encoded array using the provided dictionary.
-   *
-   * @param indices dictionary encoded values, must be int type
-   * @param dictionary dictionary used to decode the values
-   * @return vector with values restored from dictionary
-   */
-  public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
-    ValueVector.Accessor accessor = indices.getAccessor();
-    int count = accessor.getValueCount();
-    ValueVector dictionaryVector = dictionary.getDictionary();
-    // copy the dictionary values into the decoded vector
-    TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator());
-    transfer.getTo().allocateNewSafe();
-    for (int i = 0; i < count; i++) {
-      Object index = accessor.getObject(i);
-      if (index != null) {
-        transfer.copyValueSafe(((Number) index).intValue(), i);
-      }
-    }
-
-    ValueVector decoded = transfer.getTo();
-    decoded.getMutator().setValueCount(count);
-    return decoded;
-  }
-
-  private static void validateType(MinorType type) {
-    // byte arrays don't work as keys in our dictionary map - we could wrap them with something to
-    // implement equals and hashcode if we want that functionality
-    if (type == MinorType.VARBINARY || type == MinorType.LIST || type == MinorType.MAP || type == MinorType.UNION) {
-      throw new IllegalArgumentException("Dictionary encoding for complex types not implemented");
-    }
-  }
-
-  public ValueVector getIndexVector() { return indices; }
-
-  public ValueVector getDictionaryVector() { return dictionary.getDictionary(); }
-
-  public Dictionary getDictionary() { return dictionary; }
-
-  @Override
-  public MinorType getMinorType() { return indices.getMinorType(); }
-
-  @Override
-  public Field getField() { return indices.getField(); }
-
-  // note: dictionary vector is not closed, as it may be shared
-  @Override
-  public void close() { indices.close(); }
-
-  @Override
-  public void allocateNew() throws OutOfMemoryException { indices.allocateNew(); }
-
-  @Override
-  public boolean allocateNewSafe() { return indices.allocateNewSafe(); }
-
-  @Override
-  public BufferAllocator getAllocator() { return indices.getAllocator();  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) { indices.setInitialCapacity(numRecords); }
-
-  @Override
-  public int getValueCapacity() { return indices.getValueCapacity(); }
-
-  @Override
-  public int getBufferSize() { return indices.getBufferSize(); }
-
-  @Override
-  public int getBufferSizeFor(int valueCount) { return indices.getBufferSizeFor(valueCount); }
-
-  @Override
-  public Iterator<ValueVector> iterator() {
-    return indices.iterator();
-  }
-
-  @Override
-  public void clear() { indices.clear(); }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) { return indices.getTransferPair(allocator); }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return indices.getTransferPair(ref, allocator); }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector target) { return indices.makeTransferPair(target); }
-
-  @Override
-  public Accessor getAccessor() { return indices.getAccessor(); }
-
-  @Override
-  public Mutator getMutator() { return indices.getMutator(); }
-
-  @Override
-  public FieldReader getReader() { return indices.getReader(); }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) { return indices.getBuffers(clear); }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 074b0aa7e58fa..a12440e39e8fe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -24,6 +24,10 @@
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
@@ -42,16 +46,12 @@
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public class ListVector extends BaseRepeatedValueVector implements FieldVector {
 
   final UInt4Vector offsets;
@@ -62,14 +62,16 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector {
   private UnionListWriter writer;
   private UnionListReader reader;
   private CallBack callBack;
+  private final DictionaryEncoding dictionary;
 
-  public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
+  public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
     super(name, allocator);
     this.bits = new BitVector("$bits$", allocator);
     this.offsets = getOffsetVector();
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
     this.writer = new UnionListWriter(this);
     this.reader = new UnionListReader(this);
+    this.dictionary = dictionary;
     this.callBack = callBack;
   }
 
@@ -80,7 +82,7 @@ public void initializeChildrenFromFields(List<Field> children) {
     }
     Field field = children.get(0);
     MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
-    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(minorType);
+    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(minorType, field.getDictionary());
     if (!addOrGetVector.isCreated()) {
       throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
     }
@@ -151,16 +153,16 @@ private class TransferImpl implements TransferPair {
     TransferPair pairs[] = new TransferPair[3];
 
     public TransferImpl(String name, BufferAllocator allocator) {
-      this(new ListVector(name, allocator, null));
+      this(new ListVector(name, allocator, dictionary, null));
     }
 
     public TransferImpl(ListVector to) {
       this.to = to;
-      to.addOrGetVector(vector.getMinorType());
+      to.addOrGetVector(vector.getMinorType(), vector.getField().getDictionary());
       pairs[0] = offsets.makeTransferPair(to.offsets);
       pairs[1] = bits.makeTransferPair(to.bits);
       if (to.getDataVector() instanceof ZeroVector) {
-        to.addOrGetVector(vector.getMinorType());
+        to.addOrGetVector(vector.getMinorType(), vector.getField().getDictionary());
       }
       pairs[2] = getDataVector().makeTransferPair(to.getDataVector());
     }
@@ -232,8 +234,8 @@ public boolean allocateNewSafe() {
     return success;
   }
 
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
-    AddOrGetResult<T> result = super.addOrGetVector(minorType);
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType, DictionaryEncoding dictionary) {
+    AddOrGetResult<T> result = super.addOrGetVector(minorType, dictionary);
     reader = new UnionListReader(this);
     return result;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 31a1bb74b8e98..4d750cad264db 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -160,7 +160,7 @@ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
         // (This is similar to what happens in ScanBatch where the children cannot be added till they are
         // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
         // include the hashCode of the children but is based only on MaterializedField$key.
-        final FieldVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass());
+        final FieldVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass(), vector.getField().getDictionary());
         if (allocate && to.size() != preSize) {
           newVector.allocateNew();
         }
@@ -314,12 +314,11 @@ public void close() {
   public void initializeChildrenFromFields(List<Field> children) {
     for (Field field : children) {
       MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
-      FieldVector vector = (FieldVector)this.add(field.getName(), minorType);
+      FieldVector vector = (FieldVector)this.add(field.getName(), minorType, field.getDictionary());
       vector.initializeChildrenFromFields(field.getChildren());
     }
   }
 
-
   public List<FieldVector> getChildrenFromFields() {
     return getChildren();
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 5fa35307ab683..bb1fdf841a305 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -34,6 +34,7 @@
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.TransferPair;
 
@@ -48,14 +49,16 @@ public class NullableMapVector extends MapVector implements FieldVector {
   protected final BitVector bits;
 
   private final List<BufferBacked> innerVectors;
+  private final DictionaryEncoding dictionary;
 
   private final Accessor accessor;
   private final Mutator mutator;
 
-  public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+  public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
     super(name, checkNotNull(allocator), callBack);
     this.bits = new BitVector("$bits$", allocator);
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits));
+    this.dictionary = dictionary;
     this.accessor = new Accessor();
     this.mutator = new Mutator();
   }
@@ -83,7 +86,7 @@ public FieldReader getReader() {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, callBack), false);
+    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, dictionary, callBack), false);
   }
 
   @Override
@@ -93,7 +96,7 @@ public TransferPair makeTransferPair(ValueVector to) {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, callBack), false);
+    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, callBack), false);
   }
 
   protected class NullableMapTransferPair extends MapTransferPair {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index dbdd2050d13ed..6d0531678488a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -149,7 +149,8 @@ public MapWriter rootAsMap() {
     switch(mode){
 
     case INIT:
-      NullableMapVector map = container.addOrGet(name, MinorType.MAP, NullableMapVector.class);
+      // TODO allow dictionaries in complex types
+      NullableMapVector map = container.addOrGet(name, MinorType.MAP, NullableMapVector.class, null);
       mapRoot = nullableMapWriterFactory.build(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
@@ -180,7 +181,8 @@ public ListWriter rootAsList() {
 
     case INIT:
       int vectorCount = container.size();
-      ListVector listVector = container.addOrGet(name, MinorType.LIST, ListVector.class);
+      // TODO allow dictionaries in complex types
+      ListVector listVector = container.addOrGet(name, MinorType.LIST, ListVector.class, null);
       if (container.size() > vectorCount) {
         listVector.allocateNew();
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 1f7253bca93c8..e33319a2270b1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -125,7 +125,7 @@ protected FieldWriter getWriter(MinorType type) {
         // ???
         return null;
       }
-      ValueVector v = listVector.addOrGetVector(type).getVector();
+      ValueVector v = listVector.addOrGetVector(type, null).getVector();
       v.allocateNew();
       setWriter(v);
       writer.setPosition(position);
@@ -150,7 +150,8 @@ private FieldWriter promoteToUnion() {
     TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator());
     tp.transfer();
     if (parentContainer != null) {
-      unionVector = parentContainer.addOrGet(name, MinorType.UNION, UnionVector.class);
+      // TODO allow dictionaries in complex types
+      unionVector = parentContainer.addOrGet(name, MinorType.UNION, UnionVector.class, null);
       unionVector.allocateNew();
     } else if (listVector != null) {
       unionVector = listVector.promoteToUnion();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java
new file mode 100644
index 0000000000000..0c1cadfdafdbf
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java
@@ -0,0 +1,66 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.dictionary;
+
+import java.util.Objects;
+
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+
+public class Dictionary {
+
+  private final DictionaryEncoding encoding;
+  private final FieldVector dictionary;
+
+  public Dictionary(FieldVector dictionary, DictionaryEncoding encoding) {
+    this.dictionary = dictionary;
+    this.encoding = encoding;
+  }
+
+  public FieldVector getVector() {
+    return dictionary;
+  }
+
+  public DictionaryEncoding getEncoding() {
+    return encoding;
+  }
+
+  public ArrowType getVectorType() {
+    return dictionary.getField().getType();
+  }
+
+  @Override
+  public String toString() {
+    return "Dictionary " + encoding + " " + dictionary;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    Dictionary that = (Dictionary) o;
+    return Objects.equals(encoding, that.encoding) && Objects.equals(dictionary, that.dictionary);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(encoding, dictionary);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
new file mode 100644
index 0000000000000..0666bc4137a9d
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -0,0 +1,144 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.dictionary;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class DictionaryEncoder {
+
+  // TODO recursively examine fields?
+
+  /**
+   * Dictionary encodes a vector with a provided dictionary. The dictionary must contain all values in the vector.
+   *
+   * @param vector vector to encode
+   * @param dictionary dictionary used for encoding
+   * @return dictionary encoded vector
+   */
+  public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
+    validateType(vector.getMinorType());
+    // load dictionary values into a hashmap for lookup
+    ValueVector.Accessor dictionaryAccessor = dictionary.getVector().getAccessor();
+    Map<Object, Integer> lookUps = new HashMap<>(dictionaryAccessor.getValueCount());
+    for (int i = 0; i < dictionaryAccessor.getValueCount(); i++) {
+      // for primitive array types we need a wrapper that implements equals and hashcode appropriately
+      lookUps.put(dictionaryAccessor.getObject(i), i);
+    }
+
+    Field valueField = vector.getField();
+    Field indexField = new Field(valueField.getName(), valueField.isNullable(),
+      dictionary.getEncoding().getIndexType(), dictionary.getEncoding(), null);
+
+    // vector to hold our indices (dictionary encoded values)
+    FieldVector indices = indexField.createVector(vector.getAllocator());
+    ValueVector.Mutator mutator = indices.getMutator();
+
+    // use reflection to pull out the set method
+    // TODO implement a common interface for int vectors
+    Method setter = null;
+    for (Class<?> c: ImmutableList.of(int.class, long.class)) {
+      try {
+        setter = mutator.getClass().getMethod("set", int.class, c);
+        break;
+      } catch(NoSuchMethodException e) {
+        // ignore
+      }
+    }
+    if (setter == null) {
+      throw new IllegalArgumentException("Dictionary encoding does not have a valid int type:" + indices.getClass());
+    }
+
+    ValueVector.Accessor accessor = vector.getAccessor();
+    int count = accessor.getValueCount();
+
+    indices.allocateNew();
+
+    try {
+      for (int i = 0; i < count; i++) {
+        Object value = accessor.getObject(i);
+        if (value != null) { // if it's null leave it null
+          // note: this may fail if value was not included in the dictionary
+          Object encoded = lookUps.get(value);
+          if (encoded == null) {
+            throw new IllegalArgumentException("Dictionary encoding not defined for value:" + value);
+          }
+          setter.invoke(mutator, i, encoded);
+        }
+      }
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException("IllegalAccessException invoking vector mutator set():", e);
+    } catch (InvocationTargetException e) {
+      throw new RuntimeException("InvocationTargetException invoking vector mutator set():", e.getCause());
+    }
+
+    mutator.setValueCount(count);
+
+    return indices;
+  }
+
+  /**
+   * Decodes a dictionary encoded array using the provided dictionary.
+   *
+   * @param indices dictionary encoded values, must be int type
+   * @param dictionary dictionary used to decode the values
+   * @return vector with values restored from dictionary
+   */
+  public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
+    ValueVector.Accessor accessor = indices.getAccessor();
+    int count = accessor.getValueCount();
+    ValueVector dictionaryVector = dictionary.getVector();
+    int dictionaryCount = dictionaryVector.getAccessor().getValueCount();
+    // copy the dictionary values into the decoded vector
+    TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator());
+    transfer.getTo().allocateNewSafe();
+    for (int i = 0; i < count; i++) {
+      Object index = accessor.getObject(i);
+      if (index != null) {
+        int indexAsInt = ((Number) index).intValue();
+        if (indexAsInt > dictionaryCount) {
+          throw new IllegalArgumentException("Provided dictionary does not contain value for index " + indexAsInt);
+        }
+        transfer.copyValueSafe(indexAsInt, i);
+      }
+    }
+    // TODO do we need to worry about the field?
+    ValueVector decoded = transfer.getTo();
+    decoded.getMutator().setValueCount(count);
+    return decoded;
+  }
+
+  private static void validateType(MinorType type) {
+    // byte arrays don't work as keys in our dictionary map - we could wrap them with something to
+    // implement equals and hashcode if we want that functionality
+    if (type == MinorType.VARBINARY || type == MinorType.LIST || type == MinorType.MAP || type == MinorType.UNION) {
+      throw new IllegalArgumentException("Dictionary encoding for complex types not implemented: type " + type);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
new file mode 100644
index 0000000000000..63fde2536da8b
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.dictionary;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public interface DictionaryProvider {
+
+  public Dictionary lookup(long id);
+
+  public static class MapDictionaryProvider implements DictionaryProvider {
+
+    private final Map<Long, Dictionary> map;
+
+    public MapDictionaryProvider(Dictionary... dictionaries) {
+      this.map = new HashMap<>();
+      for (Dictionary dictionary: dictionaries) {
+        put(dictionary);
+      }
+    }
+
+    public void put(Dictionary dictionary) {
+      map.put(dictionary.getEncoding().getId(), dictionary);
+    }
+
+    @Override
+    public Dictionary lookup(long id) {
+      return map.get(id);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
new file mode 100644
index 0000000000000..28440a190ad43
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.arrow.flatbuf.Footer;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
+import org.apache.arrow.vector.schema.ArrowMessage;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ArrowFileReader extends ArrowReader<SeekableReadChannel> {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(ArrowFileReader.class);
+
+  private ArrowFooter footer;
+  private int currentDictionaryBatch = 0;
+  private int currentRecordBatch = 0;
+
+  public ArrowFileReader(SeekableByteChannel in, BufferAllocator allocator) {
+    super(new SeekableReadChannel(in), allocator);
+  }
+
+  public ArrowFileReader(SeekableReadChannel in, BufferAllocator allocator) {
+    super(in, allocator);
+  }
+
+  @Override
+  protected Schema readSchema(SeekableReadChannel in) throws IOException {
+    if (footer == null) {
+      if (in.size() <= (ArrowMagic.MAGIC_LENGTH * 2 + 4)) {
+        throw new InvalidArrowFileException("file too small: " + in.size());
+      }
+      ByteBuffer buffer = ByteBuffer.allocate(4 + ArrowMagic.MAGIC_LENGTH);
+      long footerLengthOffset = in.size() - buffer.remaining();
+      in.setPosition(footerLengthOffset);
+      in.readFully(buffer);
+      buffer.flip();
+      byte[] array = buffer.array();
+      if (!ArrowMagic.validateMagic(Arrays.copyOfRange(array, 4, array.length))) {
+        throw new InvalidArrowFileException("missing Magic number " + Arrays.toString(buffer.array()));
+      }
+      int footerLength = MessageSerializer.bytesToInt(array);
+      if (footerLength <= 0 || footerLength + ArrowMagic.MAGIC_LENGTH * 2 + 4 > in.size()) {
+        throw new InvalidArrowFileException("invalid footer length: " + footerLength);
+      }
+      long footerOffset = footerLengthOffset - footerLength;
+      LOGGER.debug(String.format("Footer starts at %d, length: %d", footerOffset, footerLength));
+      ByteBuffer footerBuffer = ByteBuffer.allocate(footerLength);
+      in.setPosition(footerOffset);
+      in.readFully(footerBuffer);
+      footerBuffer.flip();
+      Footer footerFB = Footer.getRootAsFooter(footerBuffer);
+      this.footer = new ArrowFooter(footerFB);
+    }
+    return footer.getSchema();
+  }
+
+  @Override
+  protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator allocator) throws IOException {
+    if (currentDictionaryBatch < footer.getDictionaries().size()) {
+      ArrowBlock block = footer.getDictionaries().get(currentDictionaryBatch++);
+      return readDictionaryBatch(in, block, allocator);
+    } else if (currentRecordBatch < footer.getRecordBatches().size()) {
+      ArrowBlock block = footer.getRecordBatches().get(currentRecordBatch++);
+      return readRecordBatch(in, block, allocator);
+    } else {
+      return null;
+    }
+  }
+
+  public List<ArrowBlock> getDictionaryBlocks() throws IOException {
+    ensureInitialized();
+    return footer.getDictionaries();
+  }
+
+  public List<ArrowBlock> getRecordBlocks() throws IOException {
+    ensureInitialized();
+    return footer.getRecordBatches();
+  }
+
+  public void loadRecordBatch(ArrowBlock block) throws IOException {
+    ensureInitialized();
+    int blockIndex = footer.getRecordBatches().indexOf(block);
+    if (blockIndex == -1) {
+      throw new IllegalArgumentException("Arrow bock does not exist in record batches: " + block);
+    }
+    currentRecordBatch = blockIndex;
+    loadNextBatch();
+  }
+
+  private ArrowDictionaryBatch readDictionaryBatch(SeekableReadChannel in,
+                                                   ArrowBlock block,
+                                                   BufferAllocator allocator) throws IOException {
+    LOGGER.debug(String.format("DictionaryRecordBatch at %d, metadata: %d, body: %d",
+       block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
+    in.setPosition(block.getOffset());
+    ArrowDictionaryBatch batch = MessageSerializer.deserializeDictionaryBatch(in, block, allocator);
+    if (batch == null) {
+      throw new IOException("Invalid file. No batch at offset: " + block.getOffset());
+    }
+    return batch;
+  }
+
+  private ArrowRecordBatch readRecordBatch(SeekableReadChannel in,
+                                           ArrowBlock block,
+                                           BufferAllocator allocator) throws IOException {
+    LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d",
+        block.getOffset(), block.getMetadataLength(),
+        block.getBodyLength()));
+    in.setPosition(block.getOffset());
+    ArrowRecordBatch batch = MessageSerializer.deserializeRecordBatch(in, block, allocator);
+    if (batch == null) {
+      throw new IOException("Invalid file. No batch at offset: " + block.getOffset());
+    }
+    return batch;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
new file mode 100644
index 0000000000000..23d210a3ee73b
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.IOException;
+import java.nio.channels.WritableByteChannel;
+import java.util.List;
+
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ArrowFileWriter extends ArrowWriter {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(ArrowFileWriter.class);
+
+  public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
+    super(root, provider, out);
+  }
+
+  @Override
+  protected void startInternal(WriteChannel out) throws IOException {
+    ArrowMagic.writeMagic(out);
+  }
+
+  @Override
+  protected void endInternal(WriteChannel out,
+                             Schema schema,
+                             List<ArrowBlock> dictionaries,
+                             List<ArrowBlock> records) throws IOException {
+    long footerStart = out.getCurrentPosition();
+    out.write(new ArrowFooter(schema, dictionaries, records), false);
+    int footerLength = (int)(out.getCurrentPosition() - footerStart);
+    if (footerLength <= 0) {
+      throw new InvalidArrowFileException("invalid footer");
+    }
+    out.writeIntLittleEndian(footerLength);
+    LOGGER.debug(String.format("Footer starts at %d, length: %d", footerStart, footerLength));
+    ArrowMagic.writeMagic(out);
+    LOGGER.debug(String.format("magic written, now at %d", out.getCurrentPosition()));
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
index 38903068570c7..1c0008a9184a0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
@@ -38,7 +38,6 @@ public class ArrowFooter implements FBSerializable {
   private final List<ArrowBlock> recordBatches;
 
   public ArrowFooter(Schema schema, List<ArrowBlock> dictionaries, List<ArrowBlock> recordBatches) {
-    super();
     this.schema = schema;
     this.dictionaries = dictionaries;
     this.recordBatches = recordBatches;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
new file mode 100644
index 0000000000000..99ea96b3856d5
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+public class ArrowMagic {
+
+  private static final byte[] MAGIC = "ARROW1".getBytes(StandardCharsets.UTF_8);
+
+  public static final int MAGIC_LENGTH = MAGIC.length;
+
+  public static void writeMagic(WriteChannel out) throws IOException {
+    out.write(MAGIC);
+  }
+
+  public static boolean validateMagic(byte[] array) {
+    return Arrays.equals(MAGIC, array);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index 8f4f4978d66cf..1646fbe803687 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -18,90 +18,188 @@
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.SeekableByteChannel;
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
 
-import org.apache.arrow.flatbuf.Footer;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
+import org.apache.arrow.vector.schema.ArrowMessage;
+import org.apache.arrow.vector.schema.ArrowMessage.ArrowMessageVisitor;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.stream.MessageSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ArrowReader implements AutoCloseable {
-  private static final Logger LOGGER = LoggerFactory.getLogger(ArrowReader.class);
-
-  public static final byte[] MAGIC = "ARROW1".getBytes();
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
 
-  private final SeekableByteChannel in;
+public abstract class ArrowReader<T extends ReadChannel> implements DictionaryProvider, AutoCloseable {
 
+  private final T in;
   private final BufferAllocator allocator;
 
-  private ArrowFooter footer;
+  private VectorLoader loader;
+  private VectorSchemaRoot root;
+  private Map<Long, Dictionary> dictionaries;
 
-  public ArrowReader(SeekableByteChannel in, BufferAllocator allocator) {
-    super();
+  private boolean initialized = false;
+
+  protected ArrowReader(T in, BufferAllocator allocator) {
     this.in = in;
     this.allocator = allocator;
   }
 
-  private int readFully(ByteBuffer buffer) throws IOException {
-    int total = 0;
-    int n;
-    do {
-      n = in.read(buffer);
-      total += n;
-    } while (n >= 0 && buffer.remaining() > 0);
-    buffer.flip();
-    return total;
+  /**
+   * Returns the vector schema root. This will be loaded with new values on every call to loadNextBatch
+   *
+   * @return the vector schema root
+   * @throws IOException if reading of schema fails
+   */
+  public VectorSchemaRoot getVectorSchemaRoot() throws IOException {
+    ensureInitialized();
+    return root;
   }
 
-  public ArrowFooter readFooter() throws IOException {
-    if (footer == null) {
-      if (in.size() <= (MAGIC.length * 2 + 4)) {
-        throw new InvalidArrowFileException("file too small: " + in.size());
-      }
-      ByteBuffer buffer = ByteBuffer.allocate(4 + MAGIC.length);
-      long footerLengthOffset = in.size() - buffer.remaining();
-      in.position(footerLengthOffset);
-      readFully(buffer);
-      byte[] array = buffer.array();
-      if (!Arrays.equals(MAGIC, Arrays.copyOfRange(array, 4, array.length))) {
-        throw new InvalidArrowFileException("missing Magic number " + Arrays.toString(buffer.array()));
-      }
-      int footerLength = MessageSerializer.bytesToInt(array);
-      if (footerLength <= 0 || footerLength + MAGIC.length * 2 + 4 > in.size()) {
-        throw new InvalidArrowFileException("invalid footer length: " + footerLength);
-      }
-      long footerOffset = footerLengthOffset - footerLength;
-      LOGGER.debug(String.format("Footer starts at %d, length: %d", footerOffset, footerLength));
-      ByteBuffer footerBuffer = ByteBuffer.allocate(footerLength);
-      in.position(footerOffset);
-      readFully(footerBuffer);
-      Footer footerFB = Footer.getRootAsFooter(footerBuffer);
-      this.footer = new ArrowFooter(footerFB);
+  /**
+   * Returns any dictionaries
+   *
+   * @return dictionaries, if any
+   * @throws IOException if reading of schema fails
+   */
+  public Map<Long, Dictionary> getDictionaryVectors() throws IOException {
+    ensureInitialized();
+    return dictionaries;
+  }
+
+  @Override
+  public Dictionary lookup(long id) {
+    if (initialized) {
+      return dictionaries.get(id);
+    } else {
+      return null;
     }
-    return footer;
   }
 
-  // TODO: read dictionaries
-
-  public ArrowRecordBatch readRecordBatch(ArrowBlock block) throws IOException {
-    LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d",
-        block.getOffset(), block.getMetadataLength(),
-        block.getBodyLength()));
-    in.position(block.getOffset());
-    ArrowRecordBatch batch =  MessageSerializer.deserializeRecordBatch(
-        new ReadChannel(in, block.getOffset()), block, allocator);
-    if (batch == null) {
-      throw new IOException("Invalid file. No batch at offset: " + block.getOffset());
+  public void loadNextBatch() throws IOException {
+    ensureInitialized();
+    // read in all dictionary batches, then stop after our first record batch
+    ArrowMessageVisitor<Boolean> visitor = new ArrowMessageVisitor<Boolean>() {
+      @Override
+      public Boolean visit(ArrowDictionaryBatch message) {
+        try { load(message); } finally { message.close(); }
+        return true;
+      }
+      @Override
+      public Boolean visit(ArrowRecordBatch message) {
+        try { loader.load(message); } finally { message.close(); }
+        return false;
+      }
+    };
+    root.setRowCount(0);
+    ArrowMessage message = readMessage(in, allocator);
+    while (message != null && message.accepts(visitor)) {
+      message = readMessage(in, allocator);
     }
-    return batch;
   }
 
+  public long bytesRead() { return in.bytesRead(); }
+
   @Override
   public void close() throws IOException {
+    if (initialized) {
+      root.close();
+      for (Dictionary dictionary: dictionaries.values()) {
+        dictionary.getVector().close();
+      }
+    }
     in.close();
   }
+
+  protected abstract Schema readSchema(T in) throws IOException;
+
+  protected abstract ArrowMessage readMessage(T in, BufferAllocator allocator) throws IOException;
+
+  protected void ensureInitialized() throws IOException {
+    if (!initialized) {
+      initialize();
+      initialized = true;
+    }
+  }
+
+  /**
+   * Reads the schema and initializes the vectors
+   */
+  private void initialize() throws IOException {
+    Schema schema = readSchema(in);
+    List<Field> fields = new ArrayList<>();
+    List<FieldVector> vectors = new ArrayList<>();
+    Map<Long, Dictionary> dictionaries = new HashMap<>();
+
+    for (Field field: schema.getFields()) {
+      Field updated = toMemoryFormat(field, dictionaries);
+      fields.add(updated);
+      vectors.add(updated.createVector(allocator));
+    }
+
+    this.root = new VectorSchemaRoot(fields, vectors, 0);
+    this.loader = new VectorLoader(root);
+    this.dictionaries = Collections.unmodifiableMap(dictionaries);
+  }
+
+  // in the message format, fields have the dictionary type
+  // in the memory format, they have the index type
+  private Field toMemoryFormat(Field field, Map<Long, Dictionary> dictionaries) {
+    DictionaryEncoding encoding = field.getDictionary();
+    List<Field> children = field.getChildren();
+
+    if (encoding == null && children.isEmpty()) {
+      return field;
+    }
+
+    List<Field> updatedChildren = new ArrayList<>(children.size());
+    for (Field child: children) {
+      updatedChildren.add(toMemoryFormat(child, dictionaries));
+    }
+
+    ArrowType type;
+    if (encoding == null) {
+      type = field.getType();
+    } else {
+      // re-type the field for in-memory format
+      type = encoding.getIndexType();
+      if (type == null) {
+        type = new Int(32, true);
+      }
+      // get existing or create dictionary vector
+      if (!dictionaries.containsKey(encoding.getId())) {
+        // create a new dictionary vector for the values
+        Field dictionaryField = new Field(field.getName(), field.isNullable(), field.getType(), null, children);
+        FieldVector dictionaryVector = dictionaryField.createVector(allocator);
+        dictionaries.put(encoding.getId(), new Dictionary(dictionaryVector, encoding));
+      }
+    }
+
+    return new Field(field.getName(), field.isNullable(), type, encoding, updatedChildren);
+  }
+
+  private void load(ArrowDictionaryBatch dictionaryBatch) {
+    long id = dictionaryBatch.getDictionaryId();
+    Dictionary dictionary = dictionaries.get(id);
+    if (dictionary == null) {
+      throw new IllegalArgumentException("Dictionary ID " + id + " not defined in schema");
+    }
+    FieldVector vector = dictionary.getVector();
+    VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), 0);
+    VectorLoader loader = new VectorLoader(root);
+    loader.load(dictionaryBatch.getDictionary());
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 24c667e67d98d..60a6afb565318 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -21,77 +21,172 @@
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
+import com.google.common.collect.ImmutableList;
+
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.stream.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class ArrowWriter implements AutoCloseable {
+public abstract class ArrowWriter implements AutoCloseable {
+
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowWriter.class);
 
+  // schema with fields in message format, not memory format
+  private final Schema schema;
   private final WriteChannel out;
 
-  private final Schema schema;
+  private final VectorUnloader unloader;
+  private final List<ArrowDictionaryBatch> dictionaries;
+
+  private final List<ArrowBlock> dictionaryBlocks = new ArrayList<>();
+  private final List<ArrowBlock> recordBlocks = new ArrayList<>();
 
-  private final List<ArrowBlock> recordBatches = new ArrayList<>();
   private boolean started = false;
+  private boolean ended = false;
 
-  public ArrowWriter(WritableByteChannel out, Schema schema) {
+  /**
+   * Note: fields are not closed when the writer is closed
+   *
+   * @param root
+   * @param provider
+   * @param out
+   */
+  protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
+    this.unloader = new VectorUnloader(root);
     this.out = new WriteChannel(out);
-    this.schema = schema;
+
+    List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
+    Map<Long, ArrowDictionaryBatch> dictionaryBatches = new HashMap<>();
+
+    for (Field field: root.getSchema().getFields()) {
+      fields.add(toMessageFormat(field, provider, dictionaryBatches));
+    }
+
+    this.schema = new Schema(fields);
+    this.dictionaries = Collections.unmodifiableList(new ArrayList<>(dictionaryBatches.values()));
+  }
+
+  // in the message format, fields have the dictionary type
+  // in the memory format, they have the index type
+  private Field toMessageFormat(Field field, DictionaryProvider provider, Map<Long, ArrowDictionaryBatch> batches) {
+    DictionaryEncoding encoding = field.getDictionary();
+    List<Field> children = field.getChildren();
+
+    if (encoding == null && children.isEmpty()) {
+      return field;
+    }
+
+    List<Field> updatedChildren = new ArrayList<>(children.size());
+    for (Field child: children) {
+      updatedChildren.add(toMessageFormat(child, provider, batches));
+    }
+
+    ArrowType type;
+    if (encoding == null) {
+      type = field.getType();
+    } else {
+      long id = encoding.getId();
+      Dictionary dictionary = provider.lookup(id);
+      if (dictionary == null) {
+        throw new IllegalArgumentException("Could not find dictionary with ID " + id);
+      }
+      type = dictionary.getVectorType();
+
+      if (!batches.containsKey(id)) {
+        FieldVector vector = dictionary.getVector();
+        int count = vector.getAccessor().getValueCount();
+        VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(field), ImmutableList.of(vector), count);
+        VectorUnloader unloader = new VectorUnloader(root);
+        ArrowRecordBatch batch = unloader.getRecordBatch();
+        batches.put(id, new ArrowDictionaryBatch(id, batch));
+      }
+    }
+
+    return new Field(field.getName(), field.isNullable(), type, encoding, updatedChildren);
   }
 
-  private void start() throws IOException {
-    writeMagic();
-    MessageSerializer.serialize(out, schema);
+  public void start() throws IOException {
+    ensureStarted();
   }
 
-  // TODO: write dictionaries
+  public void writeBatch() throws IOException {
+    ensureStarted();
+    try (ArrowRecordBatch batch = unloader.getRecordBatch()) {
+      writeRecordBatch(batch);
+    }
+  }
 
-  public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
-    checkStarted();
-    ArrowBlock batchDesc = MessageSerializer.serialize(out, recordBatch);
+  protected void writeRecordBatch(ArrowRecordBatch batch) throws IOException {
+    ArrowBlock block = MessageSerializer.serialize(out, batch);
     LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d",
-        batchDesc.getOffset(), batchDesc.getMetadataLength(), batchDesc.getBodyLength()));
+      block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
+    recordBlocks.add(block);
+  }
 
-    // add metadata to footer
-    recordBatches.add(batchDesc);
+  public void end() throws IOException {
+    ensureStarted();
+    ensureEnded();
   }
 
-  private void checkStarted() throws IOException {
+  public long bytesWritten() { return out.getCurrentPosition(); }
+
+  private void ensureStarted() throws IOException {
     if (!started) {
       started = true;
-      start();
+      startInternal(out);
+      // write the schema - for file formats this is duplicated in the footer, but matches
+      // the streaming format
+      MessageSerializer.serialize(out, schema);
+      // write out any dictionaries
+      for (ArrowDictionaryBatch batch : dictionaries) {
+        try {
+          ArrowBlock block = MessageSerializer.serialize(out, batch);
+          LOGGER.debug(String.format("DictionaryRecordBatch at %d, metadata: %d, body: %d",
+            block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
+          dictionaryBlocks.add(block);
+        } finally {
+          batch.close();
+        }
+      }
     }
   }
 
-  @Override
-  public void close() throws IOException {
-    try {
-      long footerStart = out.getCurrentPosition();
-      writeFooter();
-      int footerLength = (int)(out.getCurrentPosition() - footerStart);
-      if (footerLength <= 0 ) {
-        throw new InvalidArrowFileException("invalid footer");
-      }
-      out.writeIntLittleEndian(footerLength);
-      LOGGER.debug(String.format("Footer starts at %d, length: %d", footerStart, footerLength));
-      writeMagic();
-    } finally {
-      out.close();
+  private void ensureEnded() throws IOException {
+    if (!ended) {
+      ended = true;
+      endInternal(out, schema, dictionaryBlocks, recordBlocks);
     }
   }
 
-  private void writeMagic() throws IOException {
-    out.write(ArrowReader.MAGIC);
-    LOGGER.debug(String.format("magic written, now at %d", out.getCurrentPosition()));
-  }
+  protected abstract void startInternal(WriteChannel out) throws IOException;
+
+  protected abstract void endInternal(WriteChannel out,
+                                      Schema schema,
+                                      List<ArrowBlock> dictionaries,
+                                      List<ArrowBlock> records) throws IOException;
 
-  private void writeFooter() throws IOException {
-    // TODO: dictionaries
-    out.write(new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), recordBatches), false);
+  @Override
+  public void close() {
+    try {
+      end();
+      out.close();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
index a9dc1293b8193..b062f3826eab3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
@@ -32,16 +32,9 @@ public class ReadChannel implements AutoCloseable {
 
   private ReadableByteChannel in;
   private long bytesRead = 0;
-  // The starting byte offset into 'in'.
-  private final long startByteOffset;
-
-  public ReadChannel(ReadableByteChannel in, long startByteOffset) {
-    this.in = in;
-    this.startByteOffset = startByteOffset;
-  }
 
   public ReadChannel(ReadableByteChannel in) {
-    this(in, 0);
+    this.in = in;
   }
 
   public long bytesRead() { return bytesRead; }
@@ -72,8 +65,6 @@ public int readFully(ArrowBuf buffer, int l) throws IOException {
     return n;
   }
 
-  public long getCurrentPositiion() { return startByteOffset + bytesRead; }
-
   @Override
   public void close() throws IOException {
     if (this.in != null) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Dictionary.java b/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java
similarity index 57%
rename from java/vector/src/main/java/org/apache/arrow/vector/types/Dictionary.java
rename to java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java
index fbe1345f96aa3..914c3cb4b33a9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Dictionary.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java
@@ -1,5 +1,4 @@
-/*******************************************************************************
-
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,26 +14,26 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.types;
+ */
+package org.apache.arrow.vector.file;
 
-import org.apache.arrow.vector.ValueVector;
+import java.io.IOException;
+import java.nio.channels.SeekableByteChannel;
 
-public class Dictionary {
+public class SeekableReadChannel extends ReadChannel {
 
-    private ValueVector dictionary;
-    private boolean ordered;
+    private final SeekableByteChannel in;
 
-    public Dictionary(ValueVector dictionary, boolean ordered) {
-        this.dictionary = dictionary;
-        this.ordered = ordered;
+    public SeekableReadChannel(SeekableByteChannel in) {
+        super(in);
+        this.in = in;
     }
 
-    public ValueVector getDictionary() {
-        return dictionary;
+    public void setPosition(long position) throws IOException {
+        in.position(position);
     }
 
-    public boolean isOrdered() {
-        return ordered;
+    public long size() throws IOException {
+        return in.size();
     }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
index d99c9a6c99958..42104d181a2d0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
@@ -21,13 +21,12 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.WritableByteChannel;
 
-import org.apache.arrow.vector.schema.FBSerializable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.google.flatbuffers.FlatBufferBuilder;
 
 import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.vector.schema.FBSerializable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Wrapper around a WritableByteChannel that maintains the position as well adding
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 24fdc184523b3..bdb63b92cb105 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -88,10 +88,34 @@ public Schema start() throws JsonParseException, IOException {
     }
   }
 
+  public void read(VectorSchemaRoot root) throws IOException {
+    JsonToken t = parser.nextToken();
+    if (t == START_OBJECT) {
+      {
+        int count = readNextField("count", Integer.class);
+        root.setRowCount(count);
+        nextFieldIs("columns");
+        readToken(START_ARRAY);
+        {
+          for (Field field : schema.getFields()) {
+            FieldVector vector = root.getVector(field.getName());
+            readVector(field, vector);
+          }
+        }
+        readToken(END_ARRAY);
+      }
+      readToken(END_OBJECT);
+    } else if (t == END_ARRAY) {
+      root.setRowCount(0);
+    } else {
+      throw new IllegalArgumentException("Invalid token: " + t);
+    }
+  }
+
   public VectorSchemaRoot read() throws IOException {
     JsonToken t = parser.nextToken();
     if (t == START_OBJECT) {
-      VectorSchemaRoot recordBatch = new VectorSchemaRoot(schema, allocator);
+      VectorSchemaRoot recordBatch = VectorSchemaRoot.create(schema, allocator);
       {
         int count = readNextField("count", Integer.class);
         recordBatch.setRowCount(count);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java
new file mode 100644
index 0000000000000..901877b7058cd
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.DictionaryBatch;
+
+public class ArrowDictionaryBatch implements ArrowMessage {
+
+    private final long dictionaryId;
+    private final ArrowRecordBatch dictionary;
+
+    public ArrowDictionaryBatch(long dictionaryId, ArrowRecordBatch dictionary) {
+        this.dictionaryId = dictionaryId;
+        this.dictionary = dictionary;
+    }
+
+    public long getDictionaryId() { return dictionaryId; }
+    public ArrowRecordBatch getDictionary() { return dictionary; }
+
+    @Override
+    public int writeTo(FlatBufferBuilder builder) {
+        int dataOffset = dictionary.writeTo(builder);
+        DictionaryBatch.startDictionaryBatch(builder);
+        DictionaryBatch.addId(builder, dictionaryId);
+        DictionaryBatch.addData(builder, dataOffset);
+        return DictionaryBatch.endDictionaryBatch(builder);
+    }
+
+    @Override
+    public int computeBodyLength() { return dictionary.computeBodyLength(); }
+
+    @Override
+    public <T> T accepts(ArrowMessageVisitor<T> visitor) { return visitor.visit(this); }
+
+    @Override
+    public String toString() {
+       return "ArrowDictionaryBatch [dictionaryId=" + dictionaryId + ", dictionary=" + dictionary + "]";
+    }
+
+    @Override
+    public void close() {
+      dictionary.close();
+    }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java
new file mode 100644
index 0000000000000..d307428889b0f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.schema;
+
+public interface ArrowMessage extends FBSerializable, AutoCloseable {
+
+    public int computeBodyLength();
+
+    public <T> T accepts(ArrowMessageVisitor<T> visitor);
+
+    public static interface ArrowMessageVisitor<T> {
+        public T visit(ArrowDictionaryBatch message);
+        public T visit(ArrowRecordBatch message);
+    }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index 40c2fbfd984f8..6ef514e568d2d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -32,7 +32,8 @@
 
 import io.netty.buffer.ArrowBuf;
 
-public class ArrowRecordBatch implements FBSerializable, AutoCloseable {
+public class ArrowRecordBatch implements ArrowMessage {
+
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowRecordBatch.class);
 
   /** number of records */
@@ -113,9 +114,13 @@ public int writeTo(FlatBufferBuilder builder) {
     return RecordBatch.endRecordBatch(builder);
   }
 
+  @Override
+  public <T> T accepts(ArrowMessageVisitor<T> visitor) { return visitor.visit(this); }
+
   /**
    * releases the buffers
    */
+  @Override
   public void close() {
     if (!closed) {
       closed = true;
@@ -134,6 +139,7 @@ public String toString() {
   /**
    * Computes the size of the serialized body for this recordBatch.
    */
+  @Override
   public int computeBodyLength() {
     int size = 0;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
index f32966c5d5217..2deef37cd4e56 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
@@ -17,79 +17,43 @@
  */
 package org.apache.arrow.vector.stream;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.file.ArrowReader;
 import org.apache.arrow.vector.file.ReadChannel;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.schema.ArrowMessage;
 import org.apache.arrow.vector.types.pojo.Schema;
 
-import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 
 /**
  * This classes reads from an input stream and produces ArrowRecordBatches.
  */
-public class ArrowStreamReader implements AutoCloseable {
-  private ReadChannel in;
-  private final BufferAllocator allocator;
-  private Schema schema;
-
-  /**
-   * Constructs a streaming read, reading bytes from 'in'. Non-blocking.
-   */
-  public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) {
-    super();
-    this.in = new ReadChannel(in);
-    this.allocator = allocator;
-  }
-
-  public ArrowStreamReader(InputStream in, BufferAllocator allocator) {
-    this(Channels.newChannel(in), allocator);
-  }
-
-  /**
-   * Initializes the reader. Must be called before the other APIs. This is blocking.
-   */
-  public void init() throws IOException {
-    Preconditions.checkState(this.schema == null, "Cannot call init() more than once.");
-    this.schema = readSchema();
-  }
+public class ArrowStreamReader extends ArrowReader<ReadChannel> {
 
-  /**
-   * Returns the schema for all records in this stream.
-   */
-  public Schema getSchema () {
-    Preconditions.checkState(this.schema != null, "Must call init() first.");
-    return schema;
-  }
-
-  public long bytesRead() { return in.bytesRead(); }
+    /**
+    * Constructs a streaming read, reading bytes from 'in'. Non-blocking.
+    */
+    public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) {
+        super(new ReadChannel(in), allocator);
+    }
 
-  /**
-   * Reads and returns the next ArrowRecordBatch. Returns null if this is the end
-   * of stream.
-   */
-  public ArrowRecordBatch nextRecordBatch() throws IOException {
-    Preconditions.checkState(this.in != null, "Cannot call after close()");
-    Preconditions.checkState(this.schema != null, "Must call init() first.");
-    return MessageSerializer.deserializeRecordBatch(in, allocator);
-  }
+    public ArrowStreamReader(InputStream in, BufferAllocator allocator) {
+        this(Channels.newChannel(in), allocator);
+    }
 
-  @Override
-  public void close() throws IOException {
-    if (this.in != null) {
-      in.close();
-      in = null;
+    /**
+     * Reads the schema message from the beginning of the stream.
+     */
+    @Override
+    protected Schema readSchema(ReadChannel in) throws IOException {
+        return MessageSerializer.deserializeSchema(in);
     }
-  }
 
-  /**
-   * Reads the schema message from the beginning of the stream.
-   */
-  private Schema readSchema() throws IOException {
-    return MessageSerializer.deserializeSchema(in);
-  }
+    @Override
+    protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
+        return MessageSerializer.deserializeMessageBatch(in, allocator);
+    }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
index 60dc5861c9242..ea29cd99804c8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
@@ -17,63 +17,40 @@
  */
 package org.apache.arrow.vector.stream;
 
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.file.ArrowBlock;
+import org.apache.arrow.vector.file.ArrowWriter;
+import org.apache.arrow.vector.file.WriteChannel;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
+import java.util.List;
 
-import org.apache.arrow.vector.file.WriteChannel;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.pojo.Schema;
-
-public class ArrowStreamWriter implements AutoCloseable {
-  private final WriteChannel out;
-  private final Schema schema;
-  private boolean headerSent = false;
+public class ArrowStreamWriter extends ArrowWriter {
 
-  /**
-   * Creates the stream writer. non-blocking.
-   * totalBatches can be set if the writer knows beforehand. Can be -1 if unknown.
-   */
-  public ArrowStreamWriter(WritableByteChannel out, Schema schema) {
-    this.out = new WriteChannel(out);
-    this.schema = schema;
-  }
-
-  public ArrowStreamWriter(OutputStream out, Schema schema)
-      throws IOException {
-    this(Channels.newChannel(out), schema);
-  }
-
-  public long bytesWritten() { return out.getCurrentPosition(); }
-
-  public void writeRecordBatch(ArrowRecordBatch batch) throws IOException {
-    // Send the header if we have not yet.
-    checkAndSendHeader();
-    MessageSerializer.serialize(out, batch);
-  }
+    public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, OutputStream out) {
+       this(root, provider, Channels.newChannel(out));
+    }
 
-  /**
-   * End the stream. This is not required and this object can simply be closed.
-   */
-  public void end() throws IOException {
-    checkAndSendHeader();
-    out.writeIntLittleEndian(0);
-  }
+    public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
+       super(root, provider, out);
+    }
 
-  @Override
-  public void close() throws IOException {
-    // The header might not have been sent if this is an empty stream. Send it even in
-    // this case so readers see a valid empty stream.
-    checkAndSendHeader();
-    out.close();
-  }
+    @Override
+    protected void startInternal(WriteChannel out) throws IOException {}
 
-  private void checkAndSendHeader() throws IOException {
-    if (!headerSent) {
-      MessageSerializer.serialize(out, schema);
-      headerSent = true;
+    @Override
+    protected void endInternal(WriteChannel out,
+                               Schema schema,
+                               List<ArrowBlock> dictionaries,
+                               List<ArrowBlock> records) throws IOException {
+       out.writeIntLittleEndian(0);
     }
-  }
 }
-
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 92df2504bcb23..92a6c0c26ba6e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -22,7 +22,11 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.flatbuf.Buffer;
+import org.apache.arrow.flatbuf.DictionaryBatch;
 import org.apache.arrow.flatbuf.FieldNode;
 import org.apache.arrow.flatbuf.Message;
 import org.apache.arrow.flatbuf.MessageHeader;
@@ -33,14 +37,12 @@
 import org.apache.arrow.vector.file.ReadChannel;
 import org.apache.arrow.vector.file.WriteChannel;
 import org.apache.arrow.vector.schema.ArrowBuffer;
+import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowMessage;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.Schema;
 
-import com.google.flatbuffers.FlatBufferBuilder;
-
-import io.netty.buffer.ArrowBuf;
-
 /**
  * Utility class for serializing Messages. Messages are all serialized a similar way.
  * 1. 4 byte little endian message header prefix
@@ -81,35 +83,39 @@ public static long serialize(WriteChannel out, Schema schema) throws IOException
    * Deserializes a schema object. Format is from serialize().
    */
   public static Schema deserializeSchema(ReadChannel in) throws IOException {
-    Message message = deserializeMessage(in, MessageHeader.Schema);
+    Message message = deserializeMessage(in);
     if (message == null) {
       throw new IOException("Unexpected end of input. Missing schema.");
     }
+    if (message.headerType() != MessageHeader.Schema) {
+      throw new IOException("Expected schema but header was " + message.headerType());
+    }
 
     return Schema.convertSchema((org.apache.arrow.flatbuf.Schema)
         message.header(new org.apache.arrow.flatbuf.Schema()));
   }
 
+
   /**
    * Serializes an ArrowRecordBatch. Returns the offset and length of the written batch.
    */
   public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
-      throws IOException {
+          throws IOException {
+
     long start = out.getCurrentPosition();
     int bodyLength = batch.computeBodyLength();
 
     FlatBufferBuilder builder = new FlatBufferBuilder();
     int batchOffset = batch.writeTo(builder);
 
-    ByteBuffer serializedMessage = serializeMessage(builder, MessageHeader.RecordBatch,
-        batchOffset, bodyLength);
+    ByteBuffer serializedMessage = serializeMessage(builder, MessageHeader.RecordBatch, batchOffset, bodyLength);
 
     int metadataLength = serializedMessage.remaining();
 
-    // Add extra padding bytes so that length prefix + metadata is a multiple
-    // of 8 after alignment
-    if ((start + metadataLength + 4) % 8 != 0) {
-        metadataLength += 8 - (start + metadataLength + 4) % 8;
+    // calculate alignment bytes so that metadata length points to the correct location after alignment
+    int padding = (int)((start + metadataLength + 4) % 8);
+    if (padding != 0) {
+        metadataLength += (8 - padding);
     }
 
     out.writeIntLittleEndian(metadataLength);
@@ -118,6 +124,13 @@ public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
     // Align the output to 8 byte boundary.
     out.align();
 
+    long bufferLength = writeBatchBuffers(out, batch);
+
+    // Metadata size in the Block account for the size prefix
+    return new ArrowBlock(start, metadataLength + 4, bufferLength);
+  }
+
+  private static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) throws IOException {
     long bufferStart = out.getCurrentPosition();
     List<ArrowBuf> buffers = batch.getBuffers();
     List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
@@ -135,22 +148,14 @@ public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
             " != " + startPosition + layout.getSize());
       }
     }
-    // Metadata size in the Block account for the size prefix
-    return new ArrowBlock(start, metadataLength + 4, out.getCurrentPosition() - bufferStart);
+    return out.getCurrentPosition() - bufferStart;
   }
 
   /**
    * Deserializes a RecordBatch
    */
-  public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in,
-      BufferAllocator alloc) throws IOException {
-    Message message = deserializeMessage(in, MessageHeader.RecordBatch);
-    if (message == null) return null;
-
-    if (message.bodyLength() > Integer.MAX_VALUE) {
-      throw new IOException("Cannot currently deserialize record batches over 2GB");
-    }
-
+  private static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message message, BufferAllocator alloc)
+      throws IOException {
     RecordBatch recordBatchFB = (RecordBatch) message.header(new RecordBatch());
 
     int bodyLength = (int) message.bodyLength();
@@ -191,9 +196,7 @@ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock
     // Now read the body
     final ArrowBuf body = buffer.slice(block.getMetadataLength(),
         (int) totalLen - block.getMetadataLength());
-    ArrowRecordBatch result = deserializeRecordBatch(recordBatchFB, body);
-
-    return result;
+    return deserializeRecordBatch(recordBatchFB, body);
   }
 
   // Deserializes a record batch given the Flatbuffer metadata and in-memory body
@@ -218,6 +221,106 @@ private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB
     return arrowRecordBatch;
   }
 
+  /**
+   * Serializes a dictionary ArrowRecordBatch. Returns the offset and length of the written batch.
+   */
+  public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch) throws IOException {
+    long start = out.getCurrentPosition();
+    int bodyLength = batch.computeBodyLength();
+
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    int batchOffset = batch.writeTo(builder);
+
+    ByteBuffer serializedMessage = serializeMessage(builder, MessageHeader.DictionaryBatch, batchOffset, bodyLength);
+
+    int metadataLength = serializedMessage.remaining();
+
+    // Add extra padding bytes so that length prefix + metadata is a multiple
+    // of 8 after alignment
+    if ((start + metadataLength + 4) % 8 != 0) {
+      metadataLength += 8 - (start + metadataLength + 4) % 8;
+    }
+
+    out.writeIntLittleEndian(metadataLength);
+    out.write(serializedMessage);
+
+    // Align the output to 8 byte boundary.
+    out.align();
+
+    // write the embedded record batch
+    long bufferLength = writeBatchBuffers(out, batch.getDictionary());
+
+    // Metadata size in the Block account for the size prefix
+    return new ArrowBlock(start, metadataLength + 4, bufferLength + 8);
+  }
+
+  /**
+   * Deserializes a DictionaryBatch
+   */
+  private static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
+                                                                 Message message,
+                                                                 BufferAllocator alloc) throws IOException {
+    DictionaryBatch dictionaryBatchFB = (DictionaryBatch) message.header(new DictionaryBatch());
+
+    int bodyLength = (int) message.bodyLength();
+
+    // Now read the record batch body
+    ArrowBuf body = alloc.buffer(bodyLength);
+    if (in.readFully(body, bodyLength) != bodyLength) {
+      throw new IOException("Unexpected end of input trying to read batch.");
+    }
+    ArrowRecordBatch recordBatch = deserializeRecordBatch(dictionaryBatchFB.data(), body);
+    return new ArrowDictionaryBatch(dictionaryBatchFB.id(), recordBatch);
+  }
+
+  /**
+   * Deserializes a DictionaryBatch knowing the size of the entire message up front. This
+   * minimizes the number of reads to the underlying stream.
+   */
+  public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
+                                                                ArrowBlock block,
+                                                                BufferAllocator alloc) throws IOException {
+    // Metadata length contains integer prefix plus byte padding
+    long totalLen = block.getMetadataLength() + block.getBodyLength();
+
+    if (totalLen > Integer.MAX_VALUE) {
+      throw new IOException("Cannot currently deserialize record batches over 2GB");
+    }
+
+    ArrowBuf buffer = alloc.buffer((int) totalLen);
+    if (in.readFully(buffer, (int) totalLen) != totalLen) {
+      throw new IOException("Unexpected end of input trying to read batch.");
+    }
+
+    ArrowBuf metadataBuffer = buffer.slice(4, block.getMetadataLength() - 4);
+
+    Message messageFB =
+        Message.getRootAsMessage(metadataBuffer.nioBuffer().asReadOnlyBuffer());
+
+    DictionaryBatch dictionaryBatchFB = (DictionaryBatch) messageFB.header(new DictionaryBatch());
+
+    // Now read the body
+    final ArrowBuf body = buffer.slice(block.getMetadataLength(),
+                                       (int) totalLen - block.getMetadataLength());
+    ArrowRecordBatch recordBatch = deserializeRecordBatch(dictionaryBatchFB.data(), body);
+    return new ArrowDictionaryBatch(dictionaryBatchFB.id(), recordBatch);
+  }
+
+  public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocator alloc) throws IOException {
+    Message message = deserializeMessage(in);
+    if (message == null) {
+      return null;
+    } else if (message.bodyLength() > Integer.MAX_VALUE) {
+      throw new IOException("Cannot currently deserialize record batches over 2GB");
+    }
+
+    switch (message.headerType()) {
+      case MessageHeader.RecordBatch: return deserializeRecordBatch(in, message, alloc);
+      case MessageHeader.DictionaryBatch: return deserializeDictionaryBatch(in, message, alloc);
+      default: throw new IOException("Unexpected message header type " + message.headerType());
+    }
+  }
+
   /**
    * Serializes a message header.
    */
@@ -232,7 +335,7 @@ private static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte heade
     return builder.dataBuffer();
   }
 
-  private static Message deserializeMessage(ReadChannel in, byte headerType) throws IOException {
+  private static Message deserializeMessage(ReadChannel in) throws IOException {
     // Read the message size. There is an i32 little endian prefix.
     ByteBuffer buffer = ByteBuffer.allocate(4);
     if (in.readFully(buffer) != 4) return null;
@@ -246,11 +349,6 @@ private static Message deserializeMessage(ReadChannel in, byte headerType) throw
     }
     buffer.rewind();
 
-    Message message = Message.getRootAsMessage(buffer);
-    if (message.headerType() != headerType) {
-      throw new IOException("Invalid message: expecting " + headerType +
-          ". Message contained: " + message.headerType());
-    }
-    return message;
+    return Message.getRootAsMessage(buffer);
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index ab539d5dc3b6e..8f2d04224c0fd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -33,10 +33,10 @@
 import org.apache.arrow.vector.NullableIntervalDayVector;
 import org.apache.arrow.vector.NullableIntervalYearVector;
 import org.apache.arrow.vector.NullableSmallIntVector;
-import org.apache.arrow.vector.NullableTimeStampSecVector;
-import org.apache.arrow.vector.NullableTimeStampMilliVector;
 import org.apache.arrow.vector.NullableTimeStampMicroVector;
+import org.apache.arrow.vector.NullableTimeStampMilliVector;
 import org.apache.arrow.vector.NullableTimeStampNanoVector;
+import org.apache.arrow.vector.NullableTimeStampSecVector;
 import org.apache.arrow.vector.NullableTimeVector;
 import org.apache.arrow.vector.NullableTinyIntVector;
 import org.apache.arrow.vector.NullableUInt1Vector;
@@ -61,10 +61,10 @@
 import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl;
 import org.apache.arrow.vector.complex.impl.NullableMapWriter;
 import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl;
-import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl;
-import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeWriterImpl;
 import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.UInt1WriterImpl;
@@ -92,6 +92,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 
@@ -129,7 +130,7 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
         return ZeroVector.INSTANCE;
       }
 
@@ -145,8 +146,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-         return new NullableMapVector(name, allocator, callBack);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+         return new NullableMapVector(name, allocator, dictionary, callBack);
       }
 
       @Override
@@ -161,8 +162,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableTinyIntVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableTinyIntVector(name, allocator, dictionary);
       }
 
       @Override
@@ -177,8 +178,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableSmallIntVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableSmallIntVector(name, allocator, dictionary);
       }
 
       @Override
@@ -193,8 +194,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableIntVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableIntVector(name, allocator, dictionary);
       }
 
       @Override
@@ -209,8 +210,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableBigIntVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableBigIntVector(name, allocator, dictionary);
       }
 
       @Override
@@ -225,8 +226,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableDateVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableDateVector(name, allocator, dictionary);
       }
 
       @Override
@@ -241,8 +242,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableTimeVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableTimeVector(name, allocator, dictionary);
       }
 
       @Override
@@ -258,8 +259,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampSecVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableTimeStampSecVector(name, allocator, dictionary);
       }
 
       @Override
@@ -275,8 +276,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampMilliVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableTimeStampMilliVector(name, allocator, dictionary);
       }
 
       @Override
@@ -292,8 +293,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampMicroVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableTimeStampMicroVector(name, allocator, dictionary);
       }
 
       @Override
@@ -309,8 +310,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampNanoVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableTimeStampNanoVector(name, allocator, dictionary);
       }
 
       @Override
@@ -325,8 +326,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableIntervalDayVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableIntervalDayVector(name, allocator, dictionary);
       }
 
       @Override
@@ -341,8 +342,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableIntervalDayVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableIntervalDayVector(name, allocator, dictionary);
       }
 
       @Override
@@ -358,8 +359,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableFloat4Vector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableFloat4Vector(name, allocator, dictionary);
       }
 
       @Override
@@ -375,8 +376,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableFloat8Vector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableFloat8Vector(name, allocator, dictionary);
       }
 
       @Override
@@ -391,8 +392,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableBitVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableBitVector(name, allocator, dictionary);
       }
 
       @Override
@@ -407,8 +408,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableVarCharVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableVarCharVector(name, allocator, dictionary);
       }
 
       @Override
@@ -423,8 +424,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableVarBinaryVector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableVarBinaryVector(name, allocator, dictionary);
       }
 
       @Override
@@ -443,8 +444,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableDecimalVector(name, allocator, precisionScale[0], precisionScale[1]);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableDecimalVector(name, allocator, dictionary, precisionScale[0], precisionScale[1]);
       }
 
       @Override
@@ -459,8 +460,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableUInt1Vector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableUInt1Vector(name, allocator, dictionary);
       }
 
       @Override
@@ -475,8 +476,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableUInt2Vector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableUInt2Vector(name, allocator, dictionary);
       }
 
       @Override
@@ -491,8 +492,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableUInt4Vector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableUInt4Vector(name, allocator, dictionary);
       }
 
       @Override
@@ -507,8 +508,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new NullableUInt8Vector(name, allocator);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new NullableUInt8Vector(name, allocator, dictionary);
       }
 
       @Override
@@ -523,8 +524,8 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
-        return new ListVector(name, allocator, callBack);
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        return new ListVector(name, allocator, dictionary, callBack);
       }
 
       @Override
@@ -539,7 +540,10 @@ public Field getField() {
       }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+        if (dictionary != null) {
+          throw new UnsupportedOperationException("Dictionary encoding not supported for complex types");
+        }
         return new UnionVector(name, allocator, callBack);
       }
 
@@ -561,7 +565,7 @@ public ArrowType getType() {
 
     public abstract Field getField();
 
-    public abstract FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale);
+    public abstract FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale);
 
     public abstract FieldWriter getNewFieldWriter(ValueVector vector);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
new file mode 100644
index 0000000000000..6d35cdef832f9
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
@@ -0,0 +1,51 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.types.pojo;
+
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+
+public class DictionaryEncoding {
+
+  private final long id;
+  private final boolean ordered;
+  private final Int indexType;
+
+  public DictionaryEncoding(long id, boolean ordered, Int indexType) {
+    this.id = id;
+    this.ordered = ordered;
+    this.indexType = indexType == null ? new Int(32, true) : indexType;
+  }
+
+  public long getId() {
+    return id;
+  }
+
+  public boolean isOrdered() {
+    return ordered;
+  }
+
+  public Int getIndexType() {
+    return indexType;
+  }
+
+  @Override
+  public String toString() {
+    return "DictionaryEncoding[id=" + id + ",ordered=" + ordered + ",indexType=" + indexType + "]";
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index f9b79ce556338..bbbd559f10a3d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -24,23 +24,27 @@
 import java.util.List;
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import org.apache.arrow.flatbuf.DictionaryEncoding;
-import org.apache.arrow.vector.schema.TypeLayout;
-import org.apache.arrow.vector.schema.VectorLayout;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.flatbuffers.FlatBufferBuilder;
 
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.schema.TypeLayout;
+import org.apache.arrow.vector.schema.VectorLayout;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+
 public class Field {
   private final String name;
   private final boolean nullable;
   private final ArrowType type;
-  private final Long dictionary;
+  private final DictionaryEncoding dictionary;
   private final List<Field> children;
   private final TypeLayout typeLayout;
 
@@ -49,7 +53,7 @@ private Field(
       @JsonProperty("name") String name,
       @JsonProperty("nullable") boolean nullable,
       @JsonProperty("type") ArrowType type,
-      @JsonProperty("dictionary") Long dictionary,
+      @JsonProperty("dictionary") DictionaryEncoding dictionary,
       @JsonProperty("children") List<Field> children,
       @JsonProperty("typeLayout") TypeLayout typeLayout) {
     this.name = name;
@@ -68,18 +72,30 @@ public Field(String name, boolean nullable, ArrowType type, List<Field> children
     this(name, nullable, type, null, children, TypeLayout.getTypeLayout(checkNotNull(type)));
   }
 
-  public Field(String name, boolean nullable, ArrowType type, Long dictionary, List<Field> children) {
+  public Field(String name, boolean nullable, ArrowType type, DictionaryEncoding dictionary, List<Field> children) {
     this(name, nullable, type, dictionary, children, TypeLayout.getTypeLayout(checkNotNull(type)));
   }
 
+  public FieldVector createVector(BufferAllocator allocator) {
+    MinorType minorType = Types.getMinorTypeForArrowType(type);
+    FieldVector vector = minorType.getNewVector(name, allocator, dictionary, null);
+    vector.initializeChildrenFromFields(children);
+    return vector;
+  }
+
   public static Field convertField(org.apache.arrow.flatbuf.Field field) {
     String name = field.name();
     boolean nullable = field.nullable();
     ArrowType type = getTypeForField(field);
-    DictionaryEncoding dictionaryEncoding = field.dictionary();
-    Long dictionary = null;
-    if (dictionaryEncoding != null) {
-      dictionary = dictionaryEncoding.id();
+    DictionaryEncoding dictionary = null;
+    org.apache.arrow.flatbuf.DictionaryEncoding dictionaryFB = field.dictionary();
+    if (dictionaryFB != null) {
+      Int indexType = null;
+      org.apache.arrow.flatbuf.Int indexTypeFB = dictionaryFB.indexType();
+      if (indexTypeFB != null) {
+        indexType = new Int(indexTypeFB.bitWidth(), indexTypeFB.isSigned());
+      }
+      dictionary = new DictionaryEncoding(dictionaryFB.id(), dictionaryFB.isOrdered(), indexType);
     }
     ImmutableList.Builder<org.apache.arrow.vector.schema.VectorLayout> layout = ImmutableList.builder();
     for (int i = 0; i < field.layoutLength(); ++i) {
@@ -105,8 +121,11 @@ public int getField(FlatBufferBuilder builder) {
     int typeOffset = type.getType(builder);
     int dictionaryOffset = -1;
     if (dictionary != null) {
-      builder.addLong(dictionary);
-      dictionaryOffset = builder.offset();
+      // TODO encode dictionary type - currently type is only signed 32 bit int (default null)
+      org.apache.arrow.flatbuf.DictionaryEncoding.startDictionaryEncoding(builder);
+      org.apache.arrow.flatbuf.DictionaryEncoding.addId(builder, dictionary.getId());
+      org.apache.arrow.flatbuf.DictionaryEncoding.addIsOrdered(builder, dictionary.isOrdered());
+      dictionaryOffset = org.apache.arrow.flatbuf.DictionaryEncoding.endDictionaryEncoding(builder);
     }
     int[] childrenData = new int[children.size()];
     for (int i = 0; i < children.size(); i++) {
@@ -126,11 +145,11 @@ public int getField(FlatBufferBuilder builder) {
     org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
     org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeID().getFlatbufID());
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
+    org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
+    org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
     if (dictionary != null) {
       org.apache.arrow.flatbuf.Field.addDictionary(builder, dictionaryOffset);
     }
-    org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
-    org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
     return org.apache.arrow.flatbuf.Field.endField(builder);
   }
 
@@ -147,7 +166,7 @@ public ArrowType getType() {
   }
 
   @JsonInclude(Include.NON_NULL)
-  public Long getDictionary() { return dictionary; }
+  public DictionaryEncoding getDictionary() { return dictionary; }
 
   public List<Field> getChildren() {
     return children;
@@ -168,8 +187,8 @@ public boolean equals(Object obj) {
             Objects.equals(this.type, that.type) &&
            Objects.equals(this.dictionary, that.dictionary) &&
             (Objects.equals(this.children, that.children) ||
-                    (this.children == null && that.children.size() == 0) ||
-                    (this.children.size() == 0 && that.children == null));
+                    (this.children == null || this.children.size() == 0) &&
+                    (that.children == null || that.children.size() == 0));
   }
 
   @Override
@@ -180,7 +199,7 @@ public String toString() {
     }
     sb.append(type);
     if (dictionary != null) {
-      sb.append("[dictionary: ").append(dictionary).append("]");
+      sb.append("[dictionary: ").append(dictionary.getId()).append("]");
     }
     if (!children.isEmpty()) {
       sb.append("<").append(Joiner.on(", ").join(children)).append(">");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index cca35e44a215d..20f4aa8cf643d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -44,7 +44,7 @@ public class TestDecimalVector {
   @Test
   public void test() {
     BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
-    NullableDecimalVector decimalVector = new NullableDecimalVector("decimal", allocator, 10, scale);
+    NullableDecimalVector decimalVector = new NullableDecimalVector("decimal", allocator, null, 10, scale);
     decimalVector.allocateNew();
     BigDecimal[] values = new BigDecimal[intValues.length];
     for (int i = 0; i < intValues.length; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
index 962950abec87a..e3087ef8c95cc 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -18,16 +18,16 @@
 package org.apache.arrow.vector;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.DictionaryVector;
-import org.apache.arrow.vector.types.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryEncoder;
+import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.nio.charset.StandardCharsets;
 
-import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
 public class TestDictionaryVector {
@@ -49,65 +49,10 @@ public void terminate() throws Exception {
   }
 
   @Test
-  public void testEncodeStringsWithGeneratedDictionary() {
+  public void testEncodeStrings() {
     // Create a new value vector
-    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("foo", allocator, null)) {
-      final NullableVarCharVector.Mutator m = vector.getMutator();
-      vector.allocateNew(512, 5);
-
-      // set some values
-      m.setSafe(0, zero, 0, zero.length);
-      m.setSafe(1, one, 0, one.length);
-      m.setSafe(2, one, 0, one.length);
-      m.setSafe(3, two, 0, two.length);
-      m.setSafe(4, zero, 0, zero.length);
-      m.setValueCount(5);
-
-      DictionaryVector encoded = DictionaryVector.encode(vector);
-
-      try {
-        // verify values in the dictionary
-        ValueVector dictionary = encoded.getDictionaryVector();
-        assertEquals(vector.getClass(), dictionary.getClass());
-
-        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary).getAccessor();
-        assertEquals(3, dictionaryAccessor.getValueCount());
-        assertArrayEquals(zero, dictionaryAccessor.get(0));
-        assertArrayEquals(one, dictionaryAccessor.get(1));
-        assertArrayEquals(two, dictionaryAccessor.get(2));
-
-        // verify indices
-        ValueVector indices = encoded.getIndexVector();
-        assertEquals(NullableIntVector.class, indices.getClass());
-
-        NullableIntVector.Accessor indexAccessor = ((NullableIntVector) indices).getAccessor();
-        assertEquals(5, indexAccessor.getValueCount());
-        assertEquals(0, indexAccessor.get(0));
-        assertEquals(1, indexAccessor.get(1));
-        assertEquals(1, indexAccessor.get(2));
-        assertEquals(2, indexAccessor.get(3));
-        assertEquals(0, indexAccessor.get(4));
-
-        // now run through the decoder and verify we get the original back
-        try (ValueVector decoded = DictionaryVector.decode(indices, encoded.getDictionary())) {
-          assertEquals(vector.getClass(), decoded.getClass());
-          assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount());
-          for (int i = 0; i < 5; i++) {
-            assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i));
-          }
-        }
-      } finally {
-        encoded.getDictionaryVector().close();
-        encoded.getIndexVector().close();
-      }
-    }
-  }
-
-  @Test
-  public void testEncodeStringsWithProvidedDictionary() {
-    // Create a new value vector
-    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("foo", allocator, null);
-         final NullableVarCharVector dictionary = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("dict", allocator, null)) {
+    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("foo", allocator, null, null);
+         final NullableVarCharVector dictionaryVector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("dict", allocator, null, null)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(512, 5);
 
@@ -120,19 +65,20 @@ public void testEncodeStringsWithProvidedDictionary() {
       m.setValueCount(5);
 
       // set some dictionary values
-      final NullableVarCharVector.Mutator m2 = dictionary.getMutator();
-      dictionary.allocateNew(512, 3);
+      final NullableVarCharVector.Mutator m2 = dictionaryVector.getMutator();
+      dictionaryVector.allocateNew(512, 3);
       m2.setSafe(0, zero, 0, zero.length);
       m2.setSafe(1, one, 0, one.length);
       m2.setSafe(2, two, 0, two.length);
       m2.setValueCount(3);
 
-      try(final DictionaryVector encoded = DictionaryVector.encode(vector, new Dictionary(dictionary, false))) {
+      Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null));
+
+      try(final ValueVector encoded = (FieldVector) DictionaryEncoder.encode(vector, dictionary)) {
         // verify indices
-        ValueVector indices = encoded.getIndexVector();
-        assertEquals(NullableIntVector.class, indices.getClass());
+        assertEquals(NullableIntVector.class, encoded.getClass());
 
-        NullableIntVector.Accessor indexAccessor = ((NullableIntVector) indices).getAccessor();
+        NullableIntVector.Accessor indexAccessor = ((NullableIntVector) encoded).getAccessor();
         assertEquals(5, indexAccessor.getValueCount());
         assertEquals(0, indexAccessor.get(0));
         assertEquals(1, indexAccessor.get(1));
@@ -141,7 +87,7 @@ public void testEncodeStringsWithProvidedDictionary() {
         assertEquals(0, indexAccessor.get(4));
 
         // now run through the decoder and verify we get the original back
-        try (ValueVector decoded = DictionaryVector.decode(indices, encoded.getDictionary())) {
+        try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) {
           assertEquals(vector.getClass(), decoded.getClass());
           assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount());
           for (int i = 0; i < 5; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 1f0baaed776a1..18d93b6401e39 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -42,8 +42,8 @@ public void terminate() throws Exception {
 
   @Test
   public void testCopyFrom() throws Exception {
-    try (ListVector inVector = new ListVector("input", allocator, null);
-         ListVector outVector = new ListVector("output", allocator, null)) {
+    try (ListVector inVector = new ListVector("input", allocator, null, null);
+         ListVector outVector = new ListVector("output", allocator, null, null)) {
       UnionListWriter writer = inVector.getWriter();
       writer.allocate();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 774b59e3683e3..6917638d74e4d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -86,7 +86,7 @@ public void testFixedType() {
   public void testNullableVarLen2() {
 
     // Create a new value vector for 1024 integers.
-    try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024 * 10, 1024);
 
@@ -116,7 +116,7 @@ public void testNullableVarLen2() {
   public void testNullableFixedType() {
 
     // Create a new value vector for 1024 integers.
-    try (final NullableUInt4Vector vector = new NullableUInt4Vector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final NullableUInt4Vector vector = new NullableUInt4Vector(EMPTY_SCHEMA_PATH, allocator, null)) {
       final NullableUInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -186,7 +186,7 @@ public void testNullableFixedType() {
   @Test
   public void testNullableFloat() {
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null, null)) {
       final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -233,7 +233,7 @@ public void testNullableFloat() {
   @Test
   public void testNullableInt() {
     // Create a new value vector for 1024 integers
-    try (final NullableIntVector vector = (NullableIntVector) MinorType.INT.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+    try (final NullableIntVector vector = (NullableIntVector) MinorType.INT.getNewVector(EMPTY_SCHEMA_PATH, allocator, null, null)) {
       final NullableIntVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -403,7 +403,7 @@ private void validateRange(int length, int start, int count) {
   @Test
   public void testReAllocNullableFixedWidthVector() {
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null, null)) {
       final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -436,7 +436,7 @@ public void testReAllocNullableFixedWidthVector() {
   @Test
   public void testReAllocNullableVariableWidthVector() {
     // Create a new value vector for 1024 integers
-    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector(EMPTY_SCHEMA_PATH, allocator, null, null)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 79c9d5046acd6..372bcf0da6e9a 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.MapVector;
@@ -46,8 +47,6 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import io.netty.buffer.ArrowBuf;
-
 public class TestVectorUnloadLoad {
 
   static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
@@ -81,8 +80,8 @@ public void testUnloadLoad() throws IOException {
       try (
           ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
           BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-          VectorSchemaRoot newRoot = new VectorSchemaRoot(schema, finalVectorsAllocator);
-          ) {
+          VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator);
+      ) {
 
         // load it
         VectorLoader vectorLoader = new VectorLoader(newRoot);
@@ -131,8 +130,8 @@ public void testUnloadLoadAddPadding() throws IOException {
       try (
           ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
           BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-          VectorSchemaRoot newRoot = new VectorSchemaRoot(schema, finalVectorsAllocator);
-          ) {
+          VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator);
+      ) {
         List<ArrowBuf> oldBuffers = recordBatch.getBuffers();
         List<ArrowBuf> newBuffers = new ArrayList<>();
         for (ArrowBuf oldBuffer : oldBuffers) {
@@ -185,7 +184,7 @@ public void testLoadEmptyValidityBuffer() throws IOException {
     Schema schema = new Schema(asList(
         new Field("intDefined", true, new ArrowType.Int(32, true), Collections.<Field>emptyList()),
         new Field("intNull", true, new ArrowType.Int(32, true), Collections.<Field>emptyList())
-        ));
+                                     ));
     int count = 10;
     ArrowBuf validity = allocator.buffer(10).slice(0, 0);
     ArrowBuf[] values = new ArrowBuf[2];
@@ -200,8 +199,8 @@ public void testLoadEmptyValidityBuffer() throws IOException {
     try (
         ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(validity, values[0], validity, values[1]));
         BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        VectorSchemaRoot newRoot = new VectorSchemaRoot(schema, finalVectorsAllocator);
-        ) {
+        VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator);
+    ) {
 
       // load it
       VectorLoader vectorLoader = new VectorLoader(newRoot);
@@ -244,11 +243,12 @@ public static VectorUnloader newVectorUnloader(FieldVector root) {
     Schema schema = new Schema(root.getField().getChildren());
     int valueCount = root.getAccessor().getValueCount();
     List<FieldVector> fields = root.getChildrenFromFields();
-    return new VectorUnloader(schema, valueCount, fields);
+    VectorSchemaRoot vsr = new VectorSchemaRoot(schema.getFields(), fields, valueCount);
+    return new VectorUnloader(vsr);
   }
 
   @AfterClass
   public static void afterClass() {
     allocator.close();
   }
-}
+}
\ No newline at end of file
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 58312b3f9ff9c..2b49d8ed4b582 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -53,7 +53,7 @@ public void terminate() throws Exception {
   public void testPromoteToUnion() throws Exception {
 
     try (final MapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
-         final NullableMapVector v = container.addOrGet("test", MinorType.MAP, NullableMapVector.class);
+         final NullableMapVector v = container.addOrGet("test", MinorType.MAP, NullableMapVector.class, null);
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
       container.allocateNew();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 7a2d416241b78..a8a2d512c09ec 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -181,7 +181,7 @@ public void testList() {
 
   @Test
   public void listScalarType() {
-    ListVector listVector = new ListVector("list", allocator, null);
+    ListVector listVector = new ListVector("list", allocator, null, null);
     listVector.allocateNew();
     UnionListWriter listWriter = new UnionListWriter(listVector);
     for (int i = 0; i < COUNT; i++) {
@@ -204,7 +204,7 @@ public void listScalarType() {
 
   @Test
   public void listScalarTypeNullable() {
-    ListVector listVector = new ListVector("list", allocator, null);
+    ListVector listVector = new ListVector("list", allocator, null, null);
     listVector.allocateNew();
     UnionListWriter listWriter = new UnionListWriter(listVector);
     for (int i = 0; i < COUNT; i++) {
@@ -233,7 +233,7 @@ public void listScalarTypeNullable() {
 
   @Test
   public void listMapType() {
-    ListVector listVector = new ListVector("list", allocator, null);
+    ListVector listVector = new ListVector("list", allocator, null, null);
     listVector.allocateNew();
     UnionListWriter listWriter = new UnionListWriter(listVector);
     MapWriter mapWriter = listWriter.map();
@@ -261,7 +261,7 @@ public void listMapType() {
 
   @Test
   public void listListType() {
-    try (ListVector listVector = new ListVector("list", allocator, null)) {
+    try (ListVector listVector = new ListVector("list", allocator, null, null)) {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       for (int i = 0; i < COUNT; i++) {
@@ -286,7 +286,7 @@ public void listListType() {
    */
   @Test
   public void listListType2() {
-    try (ListVector listVector = new ListVector("list", allocator, null)) {
+    try (ListVector listVector = new ListVector("list", allocator, null, null)) {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       ListWriter innerListWriter = listWriter.list();
@@ -324,7 +324,7 @@ private void checkListOfLists(final ListVector listVector) {
 
   @Test
   public void unionListListType() {
-    try (ListVector listVector = new ListVector("list", allocator, null)) {
+    try (ListVector listVector = new ListVector("list", allocator, null, null)) {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       for (int i = 0; i < COUNT; i++) {
@@ -353,7 +353,7 @@ public void unionListListType() {
    */
   @Test
   public void unionListListType2() {
-    try (ListVector listVector = new ListVector("list", allocator, null)) {
+    try (ListVector listVector = new ListVector("list", allocator, null, null)) {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       ListWriter innerListWriter = listWriter.list();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index a83a2833c88bf..75e5d2d6e5c98 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -17,31 +17,44 @@
  */
 package org.apache.arrow.vector.file;
 
-import static org.apache.arrow.vector.TestVectorUnloadLoad.newVectorUnloader;
-import static org.junit.Assert.assertTrue;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.NullableVarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
+import org.apache.arrow.vector.dictionary.DictionaryEncoder;
 import org.apache.arrow.vector.schema.ArrowBuffer;
+import org.apache.arrow.vector.schema.ArrowMessage;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.stream.ArrowStreamReader;
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.Text;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -68,7 +81,7 @@ public void testWriteComplex() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
       writeComplexData(count, parent);
       FieldVector root = parent.getChild("root");
       validateComplexContent(count, new VectorSchemaRoot(root));
@@ -83,71 +96,63 @@ public void testWriteRead() throws IOException {
     int count = COUNT;
 
     // write
-    try (
-        BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
       writeData(count, parent);
       write(parent.getChild("root"), file, stream);
     }
 
     // read
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        FileInputStream fileInputStream = new FileInputStream(file);
-        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)
-        ) {
-      ArrowFooter footer = arrowReader.readFooter();
-      Schema schema = footer.getSchema();
-      LOGGER.debug("reading schema: " + schema);
-
-      // initialize vectors
-
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-
-        List<ArrowBlock> recordBatches = footer.getRecordBatches();
-        for (ArrowBlock rbBlock : recordBatches) {
-          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-            List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
-            for (ArrowBuffer arrowBuffer : buffersLayout) {
-              Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator){
+            @Override
+            protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator allocator) throws IOException {
+              ArrowMessage message = super.readMessage(in, allocator);
+              if (message != null) {
+                ArrowRecordBatch batch = (ArrowRecordBatch) message;
+                List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
+                for (ArrowBuffer arrowBuffer : buffersLayout) {
+                  Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+                }
+              }
+              return message;
             }
-            vectorLoader.load(recordBatch);
-          }
-
-          validateContent(count, root);
-        }
+         }) {
+      Schema schema = arrowReader.getVectorSchemaRoot().getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
+        arrowReader.loadRecordBatch(rbBlock);
+        Assert.assertEquals(count, root.getRowCount());
+        validateContent(count, root);
       }
     }
 
     // Read from stream.
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
-        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)
-        ) {
-      arrowReader.init();
-      Schema schema = arrowReader.getSchema();
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator){
+           @Override
+           protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
+             ArrowMessage message = super.readMessage(in, allocator);
+             if (message != null) {
+               ArrowRecordBatch batch = (ArrowRecordBatch) message;
+               List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
+               for (ArrowBuffer arrowBuffer : buffersLayout) {
+                 Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+               }
+             }
+             return message;
+           }
+         }) {
+
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-        while (true) {
-          try (ArrowRecordBatch recordBatch = arrowReader.nextRecordBatch()) {
-            if (recordBatch == null) break;
-            List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
-            for (ArrowBuffer arrowBuffer : buffersLayout) {
-              Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
-            }
-            vectorLoader.load(recordBatch);
-          }
-        }
-        validateContent(count, root);
-      }
+      arrowReader.loadNextBatch();
+      Assert.assertEquals(count, root.getRowCount());
+      validateContent(count, root);
     }
   }
 
@@ -158,61 +163,37 @@ public void testWriteReadComplex() throws IOException {
     int count = COUNT;
 
     // write
-    try (
-        BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
       writeComplexData(count, parent);
       write(parent.getChild("root"), file, stream);
     }
 
     // read
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        FileInputStream fileInputStream = new FileInputStream(file);
-        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)
-        ) {
-      ArrowFooter footer = arrowReader.readFooter();
-      Schema schema = footer.getSchema();
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
 
-      // initialize vectors
-
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-        List<ArrowBlock> recordBatches = footer.getRecordBatches();
-        for (ArrowBlock rbBlock : recordBatches) {
-          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-            vectorLoader.load(recordBatch);
-          }
-          validateComplexContent(count, root);
-        }
+      for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
+        arrowReader.loadRecordBatch(rbBlock);
+        Assert.assertEquals(count, root.getRowCount());
+        validateComplexContent(count, root);
       }
     }
 
     // Read from stream.
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
-        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)
-        ) {
-      arrowReader.init();
-      Schema schema = arrowReader.getSchema();
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-        while (true) {
-          try (ArrowRecordBatch recordBatch = arrowReader.nextRecordBatch()) {
-            if (recordBatch == null) break;
-            vectorLoader.load(recordBatch);
-          }
-        }
-        validateComplexContent(count, root);
-      }
+      arrowReader.loadNextBatch();
+      Assert.assertEquals(count, root.getRowCount());
+      validateComplexContent(count, root);
     }
   }
 
@@ -223,94 +204,70 @@ public void testWriteReadMultipleRBs() throws IOException {
     int[] counts = { 10, 5 };
 
     // write
-    try (
-        BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", originalVectorAllocator, null);
-        FileOutputStream fileOutputStream = new FileOutputStream(file);) {
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         MapVector parent = new MapVector("parent", originalVectorAllocator, null);
+         FileOutputStream fileOutputStream = new FileOutputStream(file)){
       writeData(counts[0], parent);
-      VectorUnloader vectorUnloader0 = newVectorUnloader(parent.getChild("root"));
-      Schema schema = vectorUnloader0.getSchema();
-      Assert.assertEquals(2, schema.getFields().size());
-      try (ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
-          ArrowStreamWriter streamWriter = new ArrowStreamWriter(stream, schema)) {
-        try (ArrowRecordBatch recordBatch = vectorUnloader0.getRecordBatch()) {
-          Assert.assertEquals("RB #0", counts[0], recordBatch.getLength());
-          arrowWriter.writeRecordBatch(recordBatch);
-          streamWriter.writeRecordBatch(recordBatch);
-        }
+      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
+
+      try(ArrowFileWriter fileWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel());
+          ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, null, stream)) {
+        fileWriter.start();
+        streamWriter.start();
+
+        fileWriter.writeBatch();
+        streamWriter.writeBatch();
+
         parent.allocateNew();
         writeData(counts[1], parent); // if we write the same data we don't catch that the metadata is stored in the wrong order.
-        VectorUnloader vectorUnloader1 = newVectorUnloader(parent.getChild("root"));
-        try (ArrowRecordBatch recordBatch = vectorUnloader1.getRecordBatch()) {
-          Assert.assertEquals("RB #1", counts[1], recordBatch.getLength());
-          arrowWriter.writeRecordBatch(recordBatch);
-          streamWriter.writeRecordBatch(recordBatch);
-        }
+        root.setRowCount(counts[1]);
+
+        fileWriter.writeBatch();
+        streamWriter.writeBatch();
+
+        fileWriter.end();
+        streamWriter.end();
       }
     }
 
-    // read
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        FileInputStream fileInputStream = new FileInputStream(file);
-        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null);
-        ) {
-      ArrowFooter footer = arrowReader.readFooter();
-      Schema schema = footer.getSchema();
+    // read file
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
       int i = 0;
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator);) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-        List<ArrowBlock> recordBatches = footer.getRecordBatches();
-        Assert.assertEquals(2, recordBatches.size());
-        long previousOffset = 0;
-        for (ArrowBlock rbBlock : recordBatches) {
-          Assert.assertTrue(rbBlock.getOffset() + " > " + previousOffset, rbBlock.getOffset() > previousOffset);
-          previousOffset = rbBlock.getOffset();
-          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-            Assert.assertEquals("RB #" + i, counts[i], recordBatch.getLength());
-            List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
-            for (ArrowBuffer arrowBuffer : buffersLayout) {
-              Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
-            }
-            vectorLoader.load(recordBatch);
-            validateContent(counts[i], root);
-          }
-          ++i;
-        }
+      List<ArrowBlock> recordBatches = arrowReader.getRecordBlocks();
+      Assert.assertEquals(2, recordBatches.size());
+      long previousOffset = 0;
+      for (ArrowBlock rbBlock : recordBatches) {
+        Assert.assertTrue(rbBlock.getOffset() + " > " + previousOffset, rbBlock.getOffset() > previousOffset);
+        previousOffset = rbBlock.getOffset();
+        arrowReader.loadRecordBatch(rbBlock);
+        Assert.assertEquals("RB #" + i, counts[i], root.getRowCount());
+        validateContent(counts[i], root);
+        ++i;
       }
     }
 
     // read stream
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
-        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)
-        ) {
-      arrowReader.init();
-      Schema schema = arrowReader.getSchema();
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
       int i = 0;
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator);) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-        for (int n = 0; n < 2; n++) {
-          try (ArrowRecordBatch recordBatch = arrowReader.nextRecordBatch()) {
-            assertTrue(recordBatch != null);
-            Assert.assertEquals("RB #" + i, counts[i], recordBatch.getLength());
-            List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
-            for (ArrowBuffer arrowBuffer : buffersLayout) {
-              Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
-            }
-            vectorLoader.load(recordBatch);
-            validateContent(counts[i], root);
-          }
-          ++i;
-        }
+
+      for (int n = 0; n < 2; n++) {
+        arrowReader.loadNextBatch();
+        Assert.assertEquals("RB #" + i, counts[i], root.getRowCount());
+        validateContent(counts[i], root);
+        ++i;
       }
+      arrowReader.loadNextBatch();
+      Assert.assertEquals(0, root.getRowCount());
     }
   }
 
@@ -319,90 +276,326 @@ public void testWriteReadUnion() throws IOException {
     File file = new File("target/mytest_write_union.arrow");
     ByteArrayOutputStream stream = new ByteArrayOutputStream();
     int count = COUNT;
-    try (
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
 
+    // write
+    try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
       writeUnionData(count, parent);
-
-      printVectors(parent.getChildrenFromFields());
-
       validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
-
       write(parent.getChild("root"), file, stream);
     }
-    // read
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        FileInputStream fileInputStream = new FileInputStream(file);
-        ArrowReader arrowReader = new ArrowReader(fileInputStream.getChannel(), readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        ) {
-      ArrowFooter footer = arrowReader.readFooter();
-      Schema schema = footer.getSchema();
+
+    // read file
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      validateUnionData(count, root);
+    }
+
+    // Read from stream.
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      validateUnionData(count, root);
+    }
+  }
 
-      // initialize vectors
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator);) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-        List<ArrowBlock> recordBatches = footer.getRecordBatches();
-        for (ArrowBlock rbBlock : recordBatches) {
-          try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
-            vectorLoader.load(recordBatch);
-          }
-          validateUnionData(count, root);
-        }
+  @Test
+  public void testWriteReadTiny() throws IOException {
+    File file = new File("target/mytest_write_tiny.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+
+    try (VectorSchemaRoot root = VectorSchemaRoot.create(MessageSerializerTest.testSchema(), allocator)) {
+      root.getFieldVectors().get(0).allocateNew();
+      NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator();
+      for (int i = 0; i < 16; i++) {
+        mutator.set(i, i < 8 ? 1 : 0, (byte)(i + 1));
+      }
+      mutator.setValueCount(16);
+      root.setRowCount(16);
+
+      // write file
+      try (FileOutputStream fileOutputStream = new FileOutputStream(file);
+           ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel())) {
+        LOGGER.debug("writing schema: " + root.getSchema());
+        arrowWriter.start();
+        arrowWriter.writeBatch();
+        arrowWriter.end();
+      }
+      // write stream
+      try (ArrowStreamWriter arrowWriter = new ArrowStreamWriter(root, null, stream)) {
+        arrowWriter.start();
+        arrowWriter.writeBatch();
+        arrowWriter.end();
       }
     }
 
+    // read file
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("fileReader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      validateTinyData(root);
+    }
+
     // Read from stream.
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
-        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)
-        ) {
-      arrowReader.init();
-      Schema schema = arrowReader.getSchema();
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("streamReader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      validateTinyData(root);
+    }
+  }
+
+  private void validateTinyData(VectorSchemaRoot root) {
+    Assert.assertEquals(16, root.getRowCount());
+    NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
+    for (int i = 0; i < 16; i++) {
+      if (i < 8) {
+        Assert.assertEquals((byte)(i + 1), vector.getAccessor().get(i));
+      } else {
+        Assert.assertTrue(vector.getAccessor().isNull(i));
+      }
+    }
+  }
+
+  @Test
+  public void testWriteReadDictionary() throws IOException {
+    File file = new File("target/mytest_dict.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+
+    // write
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         NullableVarCharVector vector = new NullableVarCharVector("varchar", originalVectorAllocator, null);
+         NullableVarCharVector dictionaryVector = new NullableVarCharVector("dict", originalVectorAllocator, null)) {
+      vector.allocateNewSafe();
+      NullableVarCharVector.Mutator mutator = vector.getMutator();
+      mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+      mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+      mutator.set(3, "baz".getBytes(StandardCharsets.UTF_8));
+      mutator.set(4, "bar".getBytes(StandardCharsets.UTF_8));
+      mutator.set(5, "baz".getBytes(StandardCharsets.UTF_8));
+      mutator.setValueCount(6);
+
+      dictionaryVector.allocateNewSafe();
+      mutator = dictionaryVector.getMutator();
+      mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+      mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+      mutator.set(2, "baz".getBytes(StandardCharsets.UTF_8));
+      mutator.setValueCount(3);
+
+      Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null));
+      MapDictionaryProvider provider = new MapDictionaryProvider();
+      provider.put(dictionary);
+
+      FieldVector encodedVector = (FieldVector) DictionaryEncoder.encode(vector, dictionary);
+
+      List<Field> fields = ImmutableList.of(encodedVector.getField());
+      List<FieldVector> vectors = ImmutableList.of(encodedVector);
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 6);
+
+      try (FileOutputStream fileOutputStream = new FileOutputStream(file);
+           ArrowFileWriter fileWriter = new ArrowFileWriter(root, provider, fileOutputStream.getChannel());
+           ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, provider, stream)) {
+        LOGGER.debug("writing schema: " + root.getSchema());
+        fileWriter.start();
+        streamWriter.start();
+        fileWriter.writeBatch();
+        streamWriter.writeBatch();
+        fileWriter.end();
+        streamWriter.end();
+      }
+
+      dictionaryVector.close();
+      encodedVector.close();
+    }
+
+    // read from file
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      validateFlatDictionary(root.getFieldVectors().get(0), arrowReader);
+    }
+
+    // Read from stream
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      validateFlatDictionary(root.getFieldVectors().get(0), arrowReader);
+    }
+  }
+
+  private void validateFlatDictionary(FieldVector vector, DictionaryProvider provider) {
+    Assert.assertNotNull(vector);
+
+    DictionaryEncoding encoding = vector.getField().getDictionary();
+    Assert.assertNotNull(encoding);
+    Assert.assertEquals(1L, encoding.getId());
+
+    FieldVector.Accessor accessor = vector.getAccessor();
+    Assert.assertEquals(6, accessor.getValueCount());
+    Assert.assertEquals(0, accessor.getObject(0));
+    Assert.assertEquals(1, accessor.getObject(1));
+    Assert.assertEquals(null, accessor.getObject(2));
+    Assert.assertEquals(2, accessor.getObject(3));
+    Assert.assertEquals(1, accessor.getObject(4));
+    Assert.assertEquals(2, accessor.getObject(5));
+
+    Dictionary dictionary = provider.lookup(1L);
+    Assert.assertNotNull(dictionary);
+    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
+    Assert.assertEquals(3, dictionaryAccessor.getValueCount());
+    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
+    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+    Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2));
+  }
 
-      try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator)) {
-        VectorLoader vectorLoader = new VectorLoader(root);
-        while (true) {
-          try (ArrowRecordBatch recordBatch = arrowReader.nextRecordBatch()) {
-            if (recordBatch == null) break;
-            vectorLoader.load(recordBatch);
-          }
-        }
-        validateUnionData(count, root);
+  @Test
+  public void testWriteReadNestedDictionary() throws IOException {
+    File file = new File("target/mytest_dict_nested.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+
+    DictionaryEncoding encoding = new DictionaryEncoding(2L, false, null);
+
+    // data being written:
+    // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
+
+    // write
+    try (NullableVarCharVector dictionaryVector = new NullableVarCharVector("dictionary", allocator, null);
+         ListVector listVector = new ListVector("list", allocator, null, null)) {
+
+      Dictionary dictionary = new Dictionary(dictionaryVector, encoding);
+      MapDictionaryProvider provider = new MapDictionaryProvider();
+      provider.put(dictionary);
+
+      dictionaryVector.allocateNew();
+      dictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8));
+      dictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
+      dictionaryVector.getMutator().setValueCount(2);
+
+      listVector.addOrGetVector(MinorType.INT, encoding);
+      listVector.allocateNew();
+      UnionListWriter listWriter = new UnionListWriter(listVector);
+      listWriter.startList();
+      listWriter.writeInt(0);
+      listWriter.writeInt(1);
+      listWriter.endList();
+      listWriter.startList();
+      listWriter.writeInt(0);
+      listWriter.endList();
+      listWriter.startList();
+      listWriter.writeInt(1);
+      listWriter.endList();
+      listWriter.setValueCount(3);
+
+      List<Field> fields = ImmutableList.of(listVector.getField());
+      List<FieldVector> vectors = ImmutableList.of((FieldVector) listVector);
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 3);
+
+      try (
+           FileOutputStream fileOutputStream = new FileOutputStream(file);
+           ArrowFileWriter fileWriter = new ArrowFileWriter(root, provider, fileOutputStream.getChannel());
+           ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, provider, stream)) {
+        LOGGER.debug("writing schema: " + root.getSchema());
+        fileWriter.start();
+        streamWriter.start();
+        fileWriter.writeBatch();
+        streamWriter.writeBatch();
+        fileWriter.end();
+        streamWriter.end();
       }
     }
+
+    // read from file
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      validateNestedDictionary((ListVector) root.getFieldVectors().get(0), arrowReader);
+    }
+
+    // Read from stream
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      validateNestedDictionary((ListVector) root.getFieldVectors().get(0), arrowReader);
+    }
+  }
+
+  private void validateNestedDictionary(ListVector vector, DictionaryProvider provider) {
+    Assert.assertNotNull(vector);
+    Assert.assertNull(vector.getField().getDictionary());
+    Field nestedField = vector.getField().getChildren().get(0);
+
+    DictionaryEncoding encoding = nestedField.getDictionary();
+    Assert.assertNotNull(encoding);
+    Assert.assertEquals(2L, encoding.getId());
+    Assert.assertEquals(new Int(32, true), encoding.getIndexType());
+
+    ListVector.Accessor accessor = vector.getAccessor();
+    Assert.assertEquals(3, accessor.getValueCount());
+    Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
+    Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
+    Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
+
+    Dictionary dictionary = provider.lookup(2L);
+    Assert.assertNotNull(dictionary);
+    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
+    Assert.assertEquals(2, dictionaryAccessor.getValueCount());
+    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
+    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
   }
 
   /**
    * Writes the contents of parents to file. If outStream is non-null, also writes it
    * to outStream in the streaming serialized format.
    */
-  private void write(FieldVector parent, File file, OutputStream outStream) throws FileNotFoundException, IOException {
-    VectorUnloader vectorUnloader = newVectorUnloader(parent);
-    Schema schema = vectorUnloader.getSchema();
-    LOGGER.debug("writing schema: " + schema);
-    try (
-        FileOutputStream fileOutputStream = new FileOutputStream(file);
-        ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);
-        ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
-            ) {
-      arrowWriter.writeRecordBatch(recordBatch);
+  private void write(FieldVector parent, File file, OutputStream outStream) throws IOException {
+    VectorSchemaRoot root = new VectorSchemaRoot(parent);
+
+    try (FileOutputStream fileOutputStream = new FileOutputStream(file);
+         ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel());) {
+      LOGGER.debug("writing schema: " + root.getSchema());
+      arrowWriter.start();
+      arrowWriter.writeBatch();
+      arrowWriter.end();
     }
 
     // Also try serializing to the stream writer.
     if (outStream != null) {
-      try (
-          ArrowStreamWriter arrowWriter = new ArrowStreamWriter(outStream, schema);
-          ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
-          ) {
-        arrowWriter.writeRecordBatch(recordBatch);
+      try (ArrowStreamWriter arrowWriter = new ArrowStreamWriter(root, null, outStream)) {
+        arrowWriter.start();
+        arrowWriter.writeBatch();
+        arrowWriter.end();
       }
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index 13b04de68fa62..914dfe4319db3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -17,12 +17,15 @@
  */
 package org.apache.arrow.vector.file;
 
+import static java.nio.channels.Channels.newChannel;
 import static java.util.Arrays.asList;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
@@ -34,8 +37,14 @@
 import org.apache.arrow.flatbuf.RecordBatch;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.NullableIntVector;
+import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -69,12 +78,17 @@ byte[] array(ArrowBuf buf) {
   @Test
   public void test() throws IOException {
     Schema schema = new Schema(asList(new Field("testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
-    byte[] validity = new byte[] { (byte)255, 0};
+    MinorType minorType = Types.getMinorTypeForArrowType(schema.getFields().get(0).getType());
+    FieldVector vector = minorType.getNewVector("testField", allocator, null,null);
+    vector.initializeChildrenFromFields(schema.getFields().get(0).getChildren());
+
+    byte[] validity = new byte[] { (byte) 255, 0};
     // second half is "undefined"
     byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
 
     ByteArrayOutputStream out = new ByteArrayOutputStream();
-    try (ArrowWriter writer = new ArrowWriter(Channels.newChannel(out), schema)) {
+    try (VectorSchemaRoot root = new VectorSchemaRoot(schema.getFields(), asList(vector), 16);
+         ArrowFileWriter writer = new ArrowFileWriter(root, null, newChannel(out))) {
       ArrowBuf validityb = buf(validity);
       ArrowBuf valuesb =  buf(values);
       writer.writeRecordBatch(new ArrowRecordBatch(16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)));
@@ -82,15 +96,15 @@ public void test() throws IOException {
 
     byte[] byteArray = out.toByteArray();
 
-    try (ArrowReader reader = new ArrowReader(new ByteArrayReadableSeekableByteChannel(byteArray), allocator)) {
-      ArrowFooter footer = reader.readFooter();
-      Schema readSchema = footer.getSchema();
+    SeekableReadChannel channel = new SeekableReadChannel(new ByteArrayReadableSeekableByteChannel(byteArray));
+    try (ArrowFileReader reader = new ArrowFileReader(channel, allocator)) {
+      Schema readSchema = reader.getVectorSchemaRoot().getSchema();
       assertEquals(schema, readSchema);
       assertTrue(readSchema.getFields().get(0).getTypeLayout().getVectorTypes().toString(), readSchema.getFields().get(0).getTypeLayout().getVectors().size() > 0);
       // TODO: dictionaries
-      List<ArrowBlock> recordBatches = footer.getRecordBatches();
+      List<ArrowBlock> recordBatches = reader.getRecordBlocks();
       assertEquals(1, recordBatches.size());
-      ArrowRecordBatch recordBatch = reader.readRecordBatch(recordBatches.get(0));
+      ArrowRecordBatch recordBatch = (ArrowRecordBatch) reader.readMessage(channel, allocator);
       List<ArrowFieldNode> nodes = recordBatch.getNodes();
       assertEquals(1, nodes.size());
       ArrowFieldNode node = nodes.get(0);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
new file mode 100644
index 0000000000000..e7cdf3fea4b8b
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowMessage;
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.stream.ArrowStreamReader;
+import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+public class TestArrowStream extends BaseFileTest {
+  @Test
+  public void testEmptyStream() throws IOException {
+    Schema schema = MessageSerializerTest.testSchema();
+    VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
+
+    // Write the stream.
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, out)) {
+    }
+
+    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+    try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) {
+      assertEquals(schema, reader.getVectorSchemaRoot().getSchema());
+      // Empty should return nothing. Can be called repeatedly.
+      reader.loadNextBatch();
+      assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
+      reader.loadNextBatch();
+      assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
+    }
+  }
+
+  @Test
+  public void testReadWrite() throws IOException {
+    Schema schema = MessageSerializerTest.testSchema();
+    try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+      int numBatches = 1;
+
+      root.getFieldVectors().get(0).allocateNew();
+      NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator();
+      for (int i = 0; i < 16; i++) {
+        mutator.set(i, i < 8 ? 1 : 0, (byte)(i + 1));
+      }
+      mutator.setValueCount(16);
+      root.setRowCount(16);
+
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      long bytesWritten = 0;
+      try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, out)) {
+        writer.start();
+        for (int i = 0; i < numBatches; i++) {
+          writer.writeBatch();
+        }
+        writer.end();
+        bytesWritten = writer.bytesWritten();
+      }
+
+      ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+      try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) {
+        Schema readSchema = reader.getVectorSchemaRoot().getSchema();
+        assertEquals(schema, readSchema);
+        for (int i = 0; i < numBatches; i++) {
+          reader.loadNextBatch();
+        }
+        // TODO figure out why reader isn't getting padding bytes
+        assertEquals(bytesWritten, reader.bytesRead() + 4);
+        reader.loadNextBatch();
+        assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
+      }
+    }
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
new file mode 100644
index 0000000000000..46d46794bbefa
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.file;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.channels.Pipe;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.schema.ArrowMessage;
+import org.apache.arrow.vector.stream.ArrowStreamReader;
+import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestArrowStreamPipe {
+  Schema schema = MessageSerializerTest.testSchema();
+  BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+
+  private final class WriterThread extends Thread {
+
+    private final int numBatches;
+    private final ArrowStreamWriter writer;
+    private final VectorSchemaRoot root;
+
+    public WriterThread(int numBatches, WritableByteChannel sinkChannel)
+        throws IOException {
+      this.numBatches = numBatches;
+      BufferAllocator allocator = alloc.newChildAllocator("writer thread", 0, Integer.MAX_VALUE);
+      root = VectorSchemaRoot.create(schema, allocator);
+      writer = new ArrowStreamWriter(root, null, sinkChannel);
+    }
+
+    @Override
+    public void run() {
+      try {
+        writer.start();
+        for (int j = 0; j < numBatches; j++) {
+          root.getFieldVectors().get(0).allocateNew();
+          NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator();
+          // Send a changing batch id first
+          mutator.set(0, j);
+          for (int i = 1; i < 16; i++) {
+            mutator.set(i, i < 8 ? 1 : 0, (byte)(i + 1));
+          }
+          mutator.setValueCount(16);
+          root.setRowCount(16);
+
+          writer.writeBatch();
+        }
+        writer.close();
+        root.close();
+      } catch (IOException e) {
+        e.printStackTrace();
+        Assert.fail(e.toString()); // have to explicitly fail since we're in a separate thread
+      }
+    }
+
+    public long bytesWritten() { return writer.bytesWritten(); }
+  }
+
+  private final class ReaderThread extends Thread {
+    private int batchesRead = 0;
+    private final ArrowStreamReader reader;
+    private final BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+    private boolean done = false;
+
+    public ReaderThread(ReadableByteChannel sourceChannel)
+        throws IOException {
+      reader = new ArrowStreamReader(sourceChannel, alloc) {
+        @Override
+        protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
+          // Read all the batches. Each batch contains an incrementing id and then some
+          // constant data. Verify both.
+          ArrowMessage message = super.readMessage(in, allocator);
+          if (message == null) {
+            done = true;
+          } else {
+            batchesRead++;
+          }
+          return message;
+        }
+        @Override
+        public void loadNextBatch() throws IOException {
+          super.loadNextBatch();
+          if (!done) {
+            VectorSchemaRoot root = getVectorSchemaRoot();
+            Assert.assertEquals(16, root.getRowCount());
+            NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
+            Assert.assertEquals((byte)(batchesRead - 1), vector.getAccessor().get(0));
+            for (int i = 1; i < 16; i++) {
+              if (i < 8) {
+                Assert.assertEquals((byte)(i + 1), vector.getAccessor().get(i));
+              } else {
+                Assert.assertTrue(vector.getAccessor().isNull(i));
+              }
+            }
+          }
+        }
+      };
+    }
+
+    @Override
+    public void run() {
+      try {
+        assertEquals(schema, reader.getVectorSchemaRoot().getSchema());
+        assertTrue(
+            reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectorTypes().toString(),
+            reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectors().size() > 0);
+        while (!done) {
+          reader.loadNextBatch();
+        }
+      } catch (IOException e) {
+        e.printStackTrace();
+        Assert.fail(e.toString()); // have to explicitly fail since we're in a separate thread
+      }
+    }
+
+    public int getBatchesRead() { return batchesRead; }
+    public long bytesRead() { return reader.bytesRead(); }
+  }
+
+  // Starts up a producer and consumer thread to read/write batches.
+  @Test
+  public void pipeTest() throws IOException, InterruptedException {
+    int NUM_BATCHES = 10;
+    Pipe pipe = Pipe.open();
+    WriterThread writer = new WriterThread(NUM_BATCHES, pipe.sink());
+    ReaderThread reader = new ReaderThread(pipe.source());
+
+    writer.start();
+    reader.start();
+    reader.join();
+    writer.join();
+
+    assertEquals(NUM_BATCHES, reader.getBatchesRead());
+    assertEquals(writer.bytesWritten(), reader.bytesRead());
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 3720a13b0fce5..c88958cbf2c9c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -70,7 +70,7 @@ public void testWriteComplexJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
       writeComplexData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateComplexContent(root.getRowCount(), root);
@@ -92,7 +92,7 @@ public void testWriteReadUnionJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null)) {
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
 
       writeUnionData(count, parent);
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
index 7b4de80ee03ea..bb2ccf8cbb5f6 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
@@ -34,6 +34,7 @@
 import org.apache.arrow.vector.file.ReadChannel;
 import org.apache.arrow.vector.file.WriteChannel;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.schema.ArrowMessage;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -88,9 +89,10 @@ public void testSerializeRecordBatch() throws IOException {
     MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), batch);
 
     ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-    ArrowRecordBatch deserialized = MessageSerializer.deserializeRecordBatch(
-        new ReadChannel(Channels.newChannel(in)), alloc);
-    verifyBatch(deserialized, validity, values);
+    ReadChannel channel = new ReadChannel(Channels.newChannel(in));
+    ArrowMessage deserialized = MessageSerializer.deserializeMessageBatch(channel, alloc);
+    assertEquals(ArrowRecordBatch.class, deserialized.getClass());
+    verifyBatch((ArrowRecordBatch) deserialized, validity, values);
   }
 
   public static Schema testSchema() {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
deleted file mode 100644
index 725272a0f072e..0000000000000
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStream.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.stream;
-
-import static java.util.Arrays.asList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.file.BaseFileTest;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.pojo.Schema;
-import org.junit.Test;
-
-import io.netty.buffer.ArrowBuf;
-
-public class TestArrowStream extends BaseFileTest {
-  @Test
-  public void testEmptyStream() throws IOException {
-    Schema schema = MessageSerializerTest.testSchema();
-
-    // Write the stream.
-    ByteArrayOutputStream out = new ByteArrayOutputStream();
-    try (ArrowStreamWriter writer = new ArrowStreamWriter(out, schema)) {
-    }
-
-    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-    try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) {
-      reader.init();
-      assertEquals(schema, reader.getSchema());
-      // Empty should return null. Can be called repeatedly.
-      assertTrue(reader.nextRecordBatch() == null);
-      assertTrue(reader.nextRecordBatch() == null);
-    }
-  }
-
-  @Test
-  public void testReadWrite() throws IOException {
-    Schema schema = MessageSerializerTest.testSchema();
-    byte[] validity = new byte[] { (byte)255, 0};
-    // second half is "undefined"
-    byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
-
-    int numBatches = 5;
-    BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
-    ByteArrayOutputStream out = new ByteArrayOutputStream();
-    long bytesWritten = 0;
-    try (ArrowStreamWriter writer = new ArrowStreamWriter(out, schema)) {
-      ArrowBuf validityb = MessageSerializerTest.buf(alloc, validity);
-      ArrowBuf valuesb =  MessageSerializerTest.buf(alloc, values);
-      for (int i = 0; i < numBatches; i++) {
-        writer.writeRecordBatch(new ArrowRecordBatch(
-            16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)));
-      }
-      bytesWritten = writer.bytesWritten();
-    }
-
-    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-    try (ArrowStreamReader reader = new ArrowStreamReader(in, alloc)) {
-      reader.init();
-      Schema readSchema = reader.getSchema();
-      for (int i = 0; i < numBatches; i++) {
-        assertEquals(schema, readSchema);
-        assertTrue(
-            readSchema.getFields().get(0).getTypeLayout().getVectorTypes().toString(),
-            readSchema.getFields().get(0).getTypeLayout().getVectors().size() > 0);
-        ArrowRecordBatch recordBatch = reader.nextRecordBatch();
-        MessageSerializerTest.verifyBatch(recordBatch, validity, values);
-        assertTrue(recordBatch != null);
-      }
-      assertTrue(reader.nextRecordBatch() == null);
-      assertEquals(bytesWritten, reader.bytesRead());
-    }
-  }
-}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
deleted file mode 100644
index aa0b77e46a392..0000000000000
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/TestArrowStreamPipe.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.stream;
-
-import static java.util.Arrays.asList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.nio.channels.Pipe;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.pojo.Schema;
-import org.junit.Test;
-
-import io.netty.buffer.ArrowBuf;
-
-public class TestArrowStreamPipe {
-  Schema schema = MessageSerializerTest.testSchema();
-  // second half is "undefined"
-  byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
-
-  private final class WriterThread extends Thread {
-    private final int numBatches;
-    private final ArrowStreamWriter writer;
-
-    public WriterThread(int numBatches, WritableByteChannel sinkChannel)
-        throws IOException {
-      this.numBatches = numBatches;
-      writer = new ArrowStreamWriter(sinkChannel, schema);
-    }
-
-    @Override
-    public void run() {
-      BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
-      try {
-        ArrowBuf valuesb =  MessageSerializerTest.buf(alloc, values);
-        for (int i = 0; i < numBatches; i++) {
-          // Send a changing byte id first.
-          byte[] validity = new byte[] { (byte)i, 0};
-          ArrowBuf validityb = MessageSerializerTest.buf(alloc, validity);
-          writer.writeRecordBatch(new ArrowRecordBatch(
-              16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)));
-        }
-        writer.close();
-      } catch (IOException e) {
-        e.printStackTrace();
-        assertTrue(false);
-      }
-    }
-
-    public long bytesWritten() { return writer.bytesWritten(); }
-  }
-
-  private final class ReaderThread extends Thread {
-    private int batchesRead = 0;
-    private final ArrowStreamReader reader;
-    private final BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
-
-    public ReaderThread(ReadableByteChannel sourceChannel)
-        throws IOException {
-      reader = new ArrowStreamReader(sourceChannel, alloc);
-    }
-
-    @Override
-    public void run() {
-      try {
-        reader.init();
-        assertEquals(schema, reader.getSchema());
-        assertTrue(
-            reader.getSchema().getFields().get(0).getTypeLayout().getVectorTypes().toString(),
-            reader.getSchema().getFields().get(0).getTypeLayout().getVectors().size() > 0);
-
-        // Read all the batches. Each batch contains an incrementing id and then some
-        // constant data. Verify both.
-        while (true) {
-          ArrowRecordBatch batch = reader.nextRecordBatch();
-          if (batch == null) break;
-          byte[] validity = new byte[] { (byte)batchesRead, 0};
-          MessageSerializerTest.verifyBatch(batch, validity, values);
-          batchesRead++;
-        }
-      } catch (IOException e) {
-        e.printStackTrace();
-        assertTrue(false);
-      }
-    }
-
-    public int getBatchesRead() { return batchesRead; }
-    public long bytesRead() { return reader.bytesRead(); }
-  }
-
-  // Starts up a producer and consumer thread to read/write batches.
-  @Test
-  public void pipeTest() throws IOException, InterruptedException {
-    int NUM_BATCHES = 10;
-    Pipe pipe = Pipe.open();
-    WriterThread writer = new WriterThread(NUM_BATCHES, pipe.sink());
-    ReaderThread reader = new ReaderThread(pipe.source());
-
-    writer.start();
-    reader.start();
-    reader.join();
-    writer.join();
-
-    assertEquals(NUM_BATCHES, reader.getBatchesRead());
-    assertEquals(writer.bytesWritten(), reader.bytesRead());
-  }
-}

From 1c101ffe0e7a92e1fc251f9335081e64aada8b26 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 16 Mar 2017 14:17:50 -0400
Subject: [PATCH 0371/1644] ARROW-636: [C++] Update README about Boost system
 requirement

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #386 from wesm/ARROW-636 and squashes the following commits:

2dd3052 [Wes McKinney] Update README about Boost system requirement
---
 cpp/README.md | 35 ++++++++++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 542a854990250..51f1f0606fa3a 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -14,13 +14,33 @@
 
 # Arrow C++
 
-## Setup Build Environment
+## System setup
 
 Arrow uses CMake as a build configuration system. Currently, it supports in-source and
 out-of-source builds with the latter one being preferred.
 
-Arrow requires a C++11-enabled compiler. On Linux, gcc 4.8 and higher should be
-sufficient.
+Build Arrow requires:
+
+* A C++11-enabled compiler. On Linux, gcc 4.8 and higher should be sufficient.
+* CMake
+* Boost
+
+On Ubuntu/Debian you can install the requirements with:
+
+```shell
+sudo apt-get install cmake \
+     libboost-dev \
+     libboost-filesystem-dev \
+     libboost-system-dev
+```
+
+On OS X, you can use [Homebrew][1]:
+
+```shell
+brew install boost cmake
+```
+
+## Building Arrow
 
 Simple debug build:
 
@@ -50,7 +70,6 @@ and benchmarks or `make runbenchmark` to run only the benchmark tests.
 
 Benchmark logs will be placed in the build directory under `build/benchmark-logs`.
 
-
 ### Third-party environment variables
 
 To set up your own specific build toolchain, here are the relevant environment
@@ -86,10 +105,12 @@ build failures by running the following checks before submitting your pull reque
 Note that the clang-tidy target may take a while to run.  You might consider
 running clang-tidy separately on the files you have added/changed before
 invoking the make target to reduce iteration time.  Also, it might generate warnings
-that aren't valid.  To avoid these you can use add a line comment `// NOLINT`. If  
-NOLINT doesn't suppress the warnings, you add the file in question to 
-the .clang-tidy-ignore file.  This will allow `make check-clang-tidy` to pass in 
+that aren't valid.  To avoid these you can use add a line comment `// NOLINT`. If
+NOLINT doesn't suppress the warnings, you add the file in question to
+the .clang-tidy-ignore file.  This will allow `make check-clang-tidy` to pass in
 travis-CI (but still surface the potential warnings in `make clang-tidy`).   Ideally,
 both of these options would be used rarely.  Current known uses-cases whent hey are required:
 
 *  Parameterized tests in google test.
+
+[1]: https://brew.sh/
\ No newline at end of file

From 0cf2bbb2afe6006219904265b41123c2ce10715a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 16 Mar 2017 21:16:44 +0100
Subject: [PATCH 0372/1644] ARROW-642: [Java] Remove temporary file in
 java/tools

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #389 from wesm/ARROW-642 and squashes the following commits:

03771c8 [Wes McKinney] Remove temporary file from ARROW-542
---
 java/tools/tmptestfilesio | Bin 628 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 java/tools/tmptestfilesio

diff --git a/java/tools/tmptestfilesio b/java/tools/tmptestfilesio
deleted file mode 100644
index d1b6b6cdb93878637bff514fbacc2b0054dd5f4d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 628
zcmZ{hJx;?w5QU$UB`gsjgpz<n3h2234JAjQ!~sR=BDtX83S5d~a0G5)-nZV75r}6!
z&(3)M-uhG0!UV@U)l4VK<xHf}SuyhxOFqv{Y-3)U3$1N0v~p&02Wsi}RPBT9&Xf+C
z7r&?Gtw}9?zWTn;?w0+`>{BmY7e6_I-)K^zOlJyU4|WJ7$N6$-zP8=-uI=>U^QdjD
zb6weIOX<cOQFTg_?5yk@=Cb2BgW2dyWi~w?P4vLu80&^FgDwsC`6HPb;Ak-AO>Oi<
zPk<PAMDBLK-?ZJPeS5bvid^{@F6*!08Ut0}^sk6}r_eO6I4ZFJiHM~mhx(d(N30^s
JY0WuCd;|NY6es`y


From 867f92470597a41ccef44bc145c979f0d396387c Mon Sep 17 00:00:00 2001
From: Johan Mabille <johan.mabille@gmail.com>
Date: Thu, 16 Mar 2017 18:23:16 -0400
Subject: [PATCH 0373/1644] ARROW-231 [C++]: Add typed Resize to PoolBuffer

I also added a typed Reserve to be consistent. Let me know what you think.

Author: Johan Mabille <johan.mabille@gmail.com>

Closes #391 from JohanMabille/buffer_resize and squashes the following commits:

90ccbfa [Johan Mabille] typed resize
---
 cpp/src/arrow/buffer-test.cc | 19 +++++++++++++++++++
 cpp/src/arrow/buffer.h       | 10 ++++++++++
 2 files changed, 29 insertions(+)

diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index 934fcfef14856..e0a2137b9bd78 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -66,6 +66,25 @@ TEST_F(TestBuffer, Resize) {
   ASSERT_EQ(128, buf.capacity());
 }
 
+TEST_F(TestBuffer, TypedResize) {
+  PoolBuffer buf;
+
+  ASSERT_EQ(0, buf.size());
+  ASSERT_OK(buf.TypedResize<double>(100));
+  ASSERT_EQ(800, buf.size());
+  ASSERT_OK(buf.TypedResize<double>(200));
+  ASSERT_EQ(1600, buf.size());
+
+  ASSERT_OK(buf.TypedResize<double>(50, true));
+  ASSERT_EQ(400, buf.size());
+  ASSERT_EQ(448, buf.capacity());
+
+  ASSERT_OK(buf.TypedResize<double>(100));
+  ASSERT_EQ(832, buf.capacity());
+  ASSERT_OK(buf.TypedResize<double>(50, false));
+  ASSERT_EQ(832, buf.capacity());
+}
+
 TEST_F(TestBuffer, ResizeOOM) {
 // This test doesn't play nice with AddressSanitizer
 #ifndef ADDRESS_SANITIZER
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 26c8ea60214f6..1647e8601f481 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -133,6 +133,16 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
   /// It does not change buffer's reported size.
   virtual Status Reserve(int64_t new_capacity) = 0;
 
+  template <class T>
+  Status TypedResize(int64_t new_nb_elements, bool shrink_to_fit = true) {
+    return Resize(sizeof(T) * new_nb_elements, shrink_to_fit);
+  }
+
+  template <class T>
+  Status TypedReserve(int64_t new_nb_elements) {
+    return Reserve(sizeof(T) * new_nb_elements);
+  }
+
  protected:
   ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {}
 };

From 3ee3822b6dccc9c859e5a324ef01fc1e9bf75dd1 Mon Sep 17 00:00:00 2001
From: Johan Mabille <johan.mabille@gmail.com>
Date: Thu, 16 Mar 2017 20:47:09 -0400
Subject: [PATCH 0374/1644] ARROW-593 [C++]: Rename ReadableFileInterface to
 RandomAccessFile

Author: Johan Mabille <johan.mabille@gmail.com>

Closes #393 from JohanMabille/raf and squashes the following commits:

ba755a6 [Johan Mabille] type alias for backward compatibility
c9e459f [Johan Mabille] Renamed ReadableFileInterface to RandomAccessFile
---
 cpp/src/arrow/io/file.h                  |  4 ++--
 cpp/src/arrow/io/hdfs.h                  |  2 +-
 cpp/src/arrow/io/interfaces.cc           |  6 +++---
 cpp/src/arrow/io/interfaces.h            | 10 ++++++----
 cpp/src/arrow/io/memory.h                |  2 +-
 cpp/src/arrow/ipc/adapter.cc             | 14 +++++++-------
 cpp/src/arrow/ipc/adapter.h              |  8 ++++----
 cpp/src/arrow/ipc/feather.cc             |  6 +++---
 cpp/src/arrow/ipc/feather.h              |  4 ++--
 cpp/src/arrow/ipc/ipc-adapter-test.cc    |  2 +-
 cpp/src/arrow/ipc/json.h                 |  2 +-
 cpp/src/arrow/ipc/metadata.cc            |  2 +-
 cpp/src/arrow/ipc/metadata.h             |  4 ++--
 cpp/src/arrow/ipc/reader.cc              |  8 ++++----
 cpp/src/arrow/ipc/reader.h               |  6 +++---
 python/pyarrow/_feather.pyx              |  4 ++--
 python/pyarrow/_parquet.pxd              |  6 +++---
 python/pyarrow/_parquet.pyx              |  4 ++--
 python/pyarrow/includes/libarrow_io.pxd  | 10 +++++-----
 python/pyarrow/includes/libarrow_ipc.pxd |  6 +++---
 python/pyarrow/includes/pyarrow.pxd      |  2 +-
 python/pyarrow/io.pxd                    |  8 ++++----
 python/pyarrow/io.pyx                    | 16 ++++++++--------
 python/src/pyarrow/io.h                  |  2 +-
 24 files changed, 70 insertions(+), 68 deletions(-)

diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index fe55e968e05d7..f687fadc299bd 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -64,7 +64,7 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
 };
 
 // Operating system file
-class ARROW_EXPORT ReadableFile : public ReadableFileInterface {
+class ARROW_EXPORT ReadableFile : public RandomAccessFile {
  public:
   ~ReadableFile();
 
@@ -115,7 +115,7 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
 
   Status Seek(int64_t position) override;
 
-  // Required by ReadableFileInterface, copies memory into out. Not thread-safe
+  // Required by RandomAccessFile, copies memory into out. Not thread-safe
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
 
   // Zero copy read. Not thread-safe
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index fbf1d758afb99..e3f5442f48ead 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -159,7 +159,7 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   DISALLOW_COPY_AND_ASSIGN(HdfsClient);
 };
 
-class ARROW_EXPORT HdfsReadableFile : public ReadableFileInterface {
+class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
  public:
   ~HdfsReadableFile();
 
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 51ed0693e2c57..06957d4de560d 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -29,18 +29,18 @@ namespace io {
 
 FileInterface::~FileInterface() {}
 
-ReadableFileInterface::ReadableFileInterface() {
+RandomAccessFile::RandomAccessFile() {
   set_mode(FileMode::READ);
 }
 
-Status ReadableFileInterface::ReadAt(
+Status RandomAccessFile::ReadAt(
     int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
   std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
 }
 
-Status ReadableFileInterface::ReadAt(
+Status RandomAccessFile::ReadAt(
     int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
   std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 9862a67aed0cd..258a3155743bf 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -97,7 +97,7 @@ class ARROW_EXPORT InputStream : virtual public FileInterface, public Readable {
   InputStream() {}
 };
 
-class ARROW_EXPORT ReadableFileInterface : public InputStream, public Seekable {
+class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
  public:
   virtual Status GetSize(int64_t* size) = 0;
 
@@ -118,7 +118,7 @@ class ARROW_EXPORT ReadableFileInterface : public InputStream, public Seekable {
  protected:
   std::mutex lock_;
 
-  ReadableFileInterface();
+  RandomAccessFile();
 };
 
 class ARROW_EXPORT WriteableFileInterface : public OutputStream, public Seekable {
@@ -129,12 +129,14 @@ class ARROW_EXPORT WriteableFileInterface : public OutputStream, public Seekable
   WriteableFileInterface() { set_mode(FileMode::READ); }
 };
 
-class ARROW_EXPORT ReadWriteFileInterface : public ReadableFileInterface,
+class ARROW_EXPORT ReadWriteFileInterface : public RandomAccessFile,
                                             public WriteableFileInterface {
  protected:
-  ReadWriteFileInterface() { ReadableFileInterface::set_mode(FileMode::READWRITE); }
+  ReadWriteFileInterface() { RandomAccessFile::set_mode(FileMode::READWRITE); }
 };
 
+using ReadableFileInterface = RandomAccessFile;
+
 }  // namespace io
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 82807508417d7..eb2a50912889e 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -66,7 +66,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   uint8_t* mutable_data_;
 };
 
-class ARROW_EXPORT BufferReader : public ReadableFileInterface {
+class ARROW_EXPORT BufferReader : public RandomAccessFile {
  public:
   explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
   BufferReader(const uint8_t* data, int64_t size);
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index 406ce249eec32..db9f63ca18cbd 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -523,7 +523,7 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
 
 class IpcComponentSource : public ArrayComponentSource {
  public:
-  IpcComponentSource(const RecordBatchMetadata& metadata, io::ReadableFileInterface* file)
+  IpcComponentSource(const RecordBatchMetadata& metadata, io::RandomAccessFile* file)
       : metadata_(metadata), file_(file) {}
 
   Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
@@ -547,14 +547,14 @@ class IpcComponentSource : public ArrayComponentSource {
 
  private:
   const RecordBatchMetadata& metadata_;
-  io::ReadableFileInterface* file_;
+  io::RandomAccessFile* file_;
 };
 
 class RecordBatchReader {
  public:
   RecordBatchReader(const RecordBatchMetadata& metadata,
       const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-      io::ReadableFileInterface* file)
+      io::RandomAccessFile* file)
       : metadata_(metadata),
         schema_(schema),
         max_recursion_depth_(max_recursion_depth),
@@ -582,24 +582,24 @@ class RecordBatchReader {
   const RecordBatchMetadata& metadata_;
   std::shared_ptr<Schema> schema_;
   int max_recursion_depth_;
-  io::ReadableFileInterface* file_;
+  io::RandomAccessFile* file_;
 };
 
 Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
+    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
     std::shared_ptr<RecordBatch>* out) {
   return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
 }
 
 Status ReadRecordBatch(const RecordBatchMetadata& metadata,
     const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-    io::ReadableFileInterface* file, std::shared_ptr<RecordBatch>* out) {
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
   RecordBatchReader reader(metadata, schema, max_recursion_depth, file);
   return reader.Read(out);
 }
 
 Status ReadDictionary(const DictionaryBatchMetadata& metadata,
-    const DictionaryTypeMap& dictionary_types, io::ReadableFileInterface* file,
+    const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
     std::shared_ptr<Array>* out) {
   int64_t id = metadata.id();
   auto it = dictionary_types.find(id);
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 21d814db86530..cea4686077486 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -39,7 +39,7 @@ class Status;
 
 namespace io {
 
-class ReadableFileInterface;
+class RandomAccessFile;
 class OutputStream;
 
 }  // namespace io
@@ -87,15 +87,15 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 // "Read" path; does not copy data if the input supports zero copy reads
 
 Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
+    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
     std::shared_ptr<RecordBatch>* out);
 
 Status ReadRecordBatch(const RecordBatchMetadata& metadata,
     const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-    io::ReadableFileInterface* file, std::shared_ptr<RecordBatch>* out);
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
 Status ReadDictionary(const DictionaryBatchMetadata& metadata,
-    const DictionaryTypeMap& dictionary_types, io::ReadableFileInterface* file,
+    const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
     std::shared_ptr<Array>* out);
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 1d165acccbd04..72bbaa4da3571 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -218,7 +218,7 @@ class TableReader::TableReaderImpl {
  public:
   TableReaderImpl() {}
 
-  Status Open(const std::shared_ptr<io::ReadableFileInterface>& source) {
+  Status Open(const std::shared_ptr<io::RandomAccessFile>& source) {
     source_ = source;
 
     int magic_size = static_cast<int>(strlen(kFeatherMagicBytes));
@@ -386,7 +386,7 @@ class TableReader::TableReaderImpl {
   }
 
  private:
-  std::shared_ptr<io::ReadableFileInterface> source_;
+  std::shared_ptr<io::RandomAccessFile> source_;
   std::unique_ptr<TableMetadata> metadata_;
 
   std::shared_ptr<Schema> schema_;
@@ -401,7 +401,7 @@ TableReader::TableReader() {
 
 TableReader::~TableReader() {}
 
-Status TableReader::Open(const std::shared_ptr<io::ReadableFileInterface>& source) {
+Status TableReader::Open(const std::shared_ptr<io::RandomAccessFile>& source) {
   return impl_->Open(source);
 }
 
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
index 3d370dfe02bd0..1e4ba58255456 100644
--- a/cpp/src/arrow/ipc/feather.h
+++ b/cpp/src/arrow/ipc/feather.h
@@ -37,7 +37,7 @@ class Status;
 namespace io {
 
 class OutputStream;
-class ReadableFileInterface;
+class RandomAccessFile;
 
 }  // namespace io
 
@@ -54,7 +54,7 @@ class ARROW_EXPORT TableReader {
   TableReader();
   ~TableReader();
 
-  Status Open(const std::shared_ptr<io::ReadableFileInterface>& source);
+  Status Open(const std::shared_ptr<io::RandomAccessFile>& source);
 
   static Status OpenFile(const std::string& abspath, std::unique_ptr<TableReader>* out);
 
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 36a675f5f94f7..638d98af8244d 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -61,7 +61,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
     auto metadata = std::make_shared<RecordBatchMetadata>(message);
 
     // The buffer offsets start at 0, so we must construct a
-    // ReadableFileInterface according to that frame of reference
+    // RandomAccessFile according to that frame of reference
     std::shared_ptr<Buffer> buffer_payload;
     RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
     io::BufferReader buffer_reader(buffer_payload);
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
index 88afdfaa5ff3b..0d88cef9a4d7b 100644
--- a/cpp/src/arrow/ipc/json.h
+++ b/cpp/src/arrow/ipc/json.h
@@ -31,7 +31,7 @@ namespace arrow {
 namespace io {
 
 class OutputStream;
-class ReadableFileInterface;
+class RandomAccessFile;
 
 }  // namespace io
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 695e7886e3124..71bc5c9eb3207 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -359,7 +359,7 @@ const RecordBatchMetadata& DictionaryBatchMetadata::record_batch() const {
 // Conveniences
 
 Status ReadMessage(int64_t offset, int32_t metadata_length,
-    io::ReadableFileInterface* file, std::shared_ptr<Message>* message) {
+    io::RandomAccessFile* file, std::shared_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index f6a0a3a073faa..4eb0186d3a467 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -41,7 +41,7 @@ class Status;
 namespace io {
 
 class OutputStream;
-class ReadableFileInterface;
+class RandomAccessFile;
 
 }  // namespace io
 
@@ -219,7 +219,7 @@ class ARROW_EXPORT Message {
 /// \param[out] message the message read
 /// \return Status success or failure
 Status ReadMessage(int64_t offset, int32_t metadata_length,
-    io::ReadableFileInterface* file, std::shared_ptr<Message>* message);
+    io::RandomAccessFile* file, std::shared_ptr<Message>* message);
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 4cb5f6cccc4c8..95753643c6513 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -308,7 +308,7 @@ class FileReader::FileReaderImpl {
   }
 
   Status Open(
-      const std::shared_ptr<io::ReadableFileInterface>& file, int64_t footer_offset) {
+      const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset) {
     file_ = file;
     footer_offset_ = footer_offset;
     RETURN_NOT_OK(ReadFooter());
@@ -318,7 +318,7 @@ class FileReader::FileReaderImpl {
   std::shared_ptr<Schema> schema() const { return schema_; }
 
  private:
-  std::shared_ptr<io::ReadableFileInterface> file_;
+  std::shared_ptr<io::RandomAccessFile> file_;
 
   // The location where the Arrow file layout ends. May be the end of the file
   // or some other location if embedded in a larger file.
@@ -342,14 +342,14 @@ FileReader::FileReader() {
 
 FileReader::~FileReader() {}
 
-Status FileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+Status FileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
     std::shared_ptr<FileReader>* reader) {
   int64_t footer_offset;
   RETURN_NOT_OK(file->GetSize(&footer_offset));
   return Open(file, footer_offset, reader);
 }
 
-Status FileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+Status FileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
     int64_t footer_offset, std::shared_ptr<FileReader>* reader) {
   *reader = std::shared_ptr<FileReader>(new FileReader());
   return (*reader)->impl_->Open(file, footer_offset);
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 6f143e1a1265e..ca91765edbac1 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -37,7 +37,7 @@ class Status;
 namespace io {
 
 class InputStream;
-class ReadableFileInterface;
+class RandomAccessFile;
 
 }  // namespace io
 
@@ -72,7 +72,7 @@ class ARROW_EXPORT FileReader {
   // can be any amount of data preceding the Arrow-formatted data, because we
   // need only locate the end of the Arrow file stream to discover the metadata
   // and then proceed to read the data into memory.
-  static Status Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+  static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
       std::shared_ptr<FileReader>* reader);
 
   // If the file is embedded within some larger file or memory region, you can
@@ -82,7 +82,7 @@ class ARROW_EXPORT FileReader {
   //
   // @param file: the data source
   // @param footer_offset: the position of the end of the Arrow "file"
-  static Status Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+  static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
       int64_t footer_offset, std::shared_ptr<FileReader>* reader);
 
   /// The schema includes any dictionaries
diff --git a/python/pyarrow/_feather.pyx b/python/pyarrow/_feather.pyx
index 67f734f6ed77c..beb4aaad44618 100644
--- a/python/pyarrow/_feather.pyx
+++ b/python/pyarrow/_feather.pyx
@@ -23,7 +23,7 @@ from cython.operator cimport dereference as deref
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport CArray, CColumn, CSchema, CStatus
-from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream
+from pyarrow.includes.libarrow_io cimport RandomAccessFile, OutputStream
 
 from libcpp.string cimport string
 from libcpp cimport bool as c_bool
@@ -53,7 +53,7 @@ cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
         CStatus Finalize()
 
     cdef cppclass TableReader:
-        TableReader(const shared_ptr[ReadableFileInterface]& source)
+        TableReader(const shared_ptr[RandomAccessFile]& source)
 
         @staticmethod
         CStatus OpenFile(const string& abspath, unique_ptr[TableReader]* out)
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index e106252189f42..cf9ec8e787661 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -20,7 +20,7 @@
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
                                         CTable, CMemoryPool)
-from pyarrow.includes.libarrow_io cimport ReadableFileInterface, OutputStream
+from pyarrow.includes.libarrow_io cimport RandomAccessFile, OutputStream
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
@@ -173,7 +173,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
     cdef cppclass ParquetFileReader:
         @staticmethod
         unique_ptr[ParquetFileReader] Open(
-            const shared_ptr[ReadableFileInterface]& file,
+            const shared_ptr[RandomAccessFile]& file,
             const ReaderProperties& props,
             const shared_ptr[CFileMetaData]& metadata)
 
@@ -203,7 +203,7 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
 
 
 cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
-    CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
+    CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
                      CMemoryPool* allocator,
                      const ReaderProperties& properties,
                      const shared_ptr[CFileMetaData]& metadata,
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 08c7bb5d8b1bc..8e67da9f75a6e 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -23,7 +23,7 @@ from cython.operator cimport dereference as deref
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.libarrow_io cimport (ReadableFileInterface, OutputStream,
+from pyarrow.includes.libarrow_io cimport (RandomAccessFile, OutputStream,
                                            FileOutputStream)
 cimport pyarrow.includes.pyarrow as pyarrow
 
@@ -354,7 +354,7 @@ cdef class ParquetReader:
 
     def open(self, object source, FileMetaData metadata=None):
         cdef:
-            shared_ptr[ReadableFileInterface] rd_handle
+            shared_ptr[RandomAccessFile] rd_handle
             shared_ptr[CFileMetaData] c_metadata
             ReaderProperties properties = default_reader_properties()
             c_string path
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
index 8d0d5248b4db0..5992c737df512 100644
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ b/python/pyarrow/includes/libarrow_io.pxd
@@ -51,7 +51,7 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
     cdef cppclass InputStream(FileInterface, Readable):
         pass
 
-    cdef cppclass ReadableFileInterface(InputStream, Seekable):
+    cdef cppclass RandomAccessFile(InputStream, Seekable):
         CStatus GetSize(int64_t* size)
 
         CStatus ReadAt(int64_t position, int64_t nbytes,
@@ -63,7 +63,7 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
         CStatus WriteAt(int64_t position, const uint8_t* data,
                         int64_t nbytes)
 
-    cdef cppclass ReadWriteFileInterface(ReadableFileInterface,
+    cdef cppclass ReadWriteFileInterface(RandomAccessFile,
                                          WriteableFileInterface):
         pass
 
@@ -77,7 +77,7 @@ cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
 
         int file_descriptor()
 
-    cdef cppclass ReadableFile(ReadableFileInterface):
+    cdef cppclass ReadableFile(RandomAccessFile):
         @staticmethod
         CStatus Open(const c_string& path, shared_ptr[ReadableFile]* file)
 
@@ -123,7 +123,7 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
         int64_t block_size
         int16_t permissions
 
-    cdef cppclass HdfsReadableFile(ReadableFileInterface):
+    cdef cppclass HdfsReadableFile(RandomAccessFile):
         pass
 
     cdef cppclass HdfsOutputStream(OutputStream):
@@ -163,7 +163,7 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
 
 cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
     cdef cppclass CBufferReader" arrow::io::BufferReader"\
-        (ReadableFileInterface):
+        (RandomAccessFile):
         CBufferReader(const shared_ptr[CBuffer]& buffer)
         CBufferReader(const uint8_t* data, int64_t nbytes)
 
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
index 10c70a96b0ab2..8b7d705afd4e7 100644
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -20,7 +20,7 @@
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CSchema, CRecordBatch)
 from pyarrow.includes.libarrow_io cimport (InputStream, OutputStream,
-                                           ReadableFileInterface)
+                                           RandomAccessFile)
 
 
 cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
@@ -51,11 +51,11 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
     cdef cppclass CFileReader " arrow::ipc::FileReader":
 
         @staticmethod
-        CStatus Open(const shared_ptr[ReadableFileInterface]& file,
+        CStatus Open(const shared_ptr[RandomAccessFile]& file,
                      shared_ptr[CFileReader]* out)
 
         @staticmethod
-        CStatus Open2" Open"(const shared_ptr[ReadableFileInterface]& file,
+        CStatus Open2" Open"(const shared_ptr[RandomAccessFile]& file,
                      int64_t footer_offset, shared_ptr[CFileReader]* out)
 
         shared_ptr[CSchema] schema()
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 9fbddba3d10c5..805950bd1476a 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -60,7 +60,7 @@ cdef extern from "pyarrow/common.h" namespace "arrow::py" nogil:
 
 
 cdef extern from "pyarrow/io.h" namespace "arrow::py" nogil:
-    cdef cppclass PyReadableFile(arrow_io.ReadableFileInterface):
+    cdef cppclass PyReadableFile(arrow_io.RandomAccessFile):
         PyReadableFile(object fo)
 
     cdef cppclass PyOutputStream(arrow_io.OutputStream):
diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
index 3d73e1143e15a..cffd29ab39111 100644
--- a/python/pyarrow/io.pxd
+++ b/python/pyarrow/io.pxd
@@ -19,7 +19,7 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.libarrow_io cimport (ReadableFileInterface,
+from pyarrow.includes.libarrow_io cimport (RandomAccessFile,
                                            OutputStream)
 
 cdef class Buffer:
@@ -32,7 +32,7 @@ cdef class Buffer:
 
 cdef class NativeFile:
     cdef:
-        shared_ptr[ReadableFileInterface] rd_file
+        shared_ptr[RandomAccessFile] rd_file
         shared_ptr[OutputStream] wr_file
         bint is_readable
         bint is_writeable
@@ -43,8 +43,8 @@ cdef class NativeFile:
     # extension classes are technically virtual in the C++ sense) we can expose
     # the arrow::io abstract file interfaces to other components throughout the
     # suite of Arrow C++ libraries
-    cdef read_handle(self, shared_ptr[ReadableFileInterface]* file)
+    cdef read_handle(self, shared_ptr[RandomAccessFile]* file)
     cdef write_handle(self, shared_ptr[OutputStream]* file)
 
-cdef get_reader(object source, shared_ptr[ReadableFileInterface]* reader)
+cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader)
 cdef get_writer(object source, shared_ptr[OutputStream]* writer)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 240ea240c3abe..17b43dedb0a5f 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -81,9 +81,9 @@ cdef class NativeFile:
                     check_status(self.wr_file.get().Close())
         self.is_open = False
 
-    cdef read_handle(self, shared_ptr[ReadableFileInterface]* file):
+    cdef read_handle(self, shared_ptr[RandomAccessFile]* file):
         self._assert_readable()
-        file[0] = <shared_ptr[ReadableFileInterface]> self.rd_file
+        file[0] = <shared_ptr[RandomAccessFile]> self.rd_file
 
     cdef write_handle(self, shared_ptr[OutputStream]* file):
         self._assert_writeable()
@@ -361,7 +361,7 @@ cdef class MemoryMappedFile(NativeFile):
         check_status(CMemoryMappedFile.Open(c_path, c_mode, &handle))
 
         self.wr_file = <shared_ptr[OutputStream]> handle
-        self.rd_file = <shared_ptr[ReadableFileInterface]> handle
+        self.rd_file = <shared_ptr[RandomAccessFile]> handle
         self.is_open = True
 
 
@@ -398,7 +398,7 @@ cdef class OSFile(NativeFile):
             check_status(ReadableFile.Open(path, pool, &handle))
 
         self.is_readable = 1
-        self.rd_file = <shared_ptr[ReadableFileInterface]> handle
+        self.rd_file = <shared_ptr[RandomAccessFile]> handle
 
     cdef _open_writeable(self, c_string path):
         cdef shared_ptr[FileOutputStream] handle
@@ -536,7 +536,7 @@ cdef Buffer wrap_buffer(const shared_ptr[CBuffer]& buf):
     return result
 
 
-cdef get_reader(object source, shared_ptr[ReadableFileInterface]* reader):
+cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
     cdef NativeFile nf
 
     if isinstance(source, six.string_types):
@@ -815,7 +815,7 @@ cdef class _HdfsClient:
                 check_status(self.client.get()
                              .OpenReadable(c_path, &rd_handle))
 
-            out.rd_file = <shared_ptr[ReadableFileInterface]> rd_handle
+            out.rd_file = <shared_ptr[RandomAccessFile]> rd_handle
             out.is_readable = True
             out.is_writeable = 0
 
@@ -924,7 +924,7 @@ cdef class _StreamReader:
 
     def _open(self, source):
         cdef:
-            shared_ptr[ReadableFileInterface] reader
+            shared_ptr[RandomAccessFile] reader
             shared_ptr[InputStream] in_stream
 
         get_reader(source, &reader)
@@ -996,7 +996,7 @@ cdef class _FileReader:
         pass
 
     def _open(self, source, footer_offset=None):
-        cdef shared_ptr[ReadableFileInterface] reader
+        cdef shared_ptr[RandomAccessFile] reader
         get_reader(source, &reader)
 
         cdef int64_t offset = 0
diff --git a/python/src/pyarrow/io.h b/python/src/pyarrow/io.h
index a603e81622545..e38cd81775255 100644
--- a/python/src/pyarrow/io.h
+++ b/python/src/pyarrow/io.h
@@ -49,7 +49,7 @@ class PythonFile {
   PyObject* file_;
 };
 
-class ARROW_EXPORT PyReadableFile : public io::ReadableFileInterface {
+class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile {
  public:
   explicit PyReadableFile(PyObject* file);
   virtual ~PyReadableFile();

From c13d671b15cc701e396b11f17e6d36a339dd5210 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 16 Mar 2017 20:50:10 -0400
Subject: [PATCH 0375/1644] ARROW-644: Python: Cython should be a setup-only
 requirement

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #392 from xhochy/ARROW-644 and squashes the following commits:

5d99895 [Uwe L. Korn] ARROW-644: Python: Cython should be a setup-only requirement
---
 python/setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index a0573fe1fccff..9abf9854af2a8 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -321,8 +321,8 @@ def get_outputs(self):
         'build_ext': build_ext
     },
     use_scm_version = {"root": "..", "relative_to": __file__},
-    setup_requires=['setuptools_scm'],
-    install_requires=['cython >= 0.23', 'numpy >= 1.9', 'six >= 1.0.0'],
+    setup_requires=['setuptools_scm', 'cython >= 0.23'],
+    install_requires=['numpy >= 1.9', 'six >= 1.0.0'],
     test_requires=['pytest'],
     description="Python library for Apache Arrow",
     long_description=long_description,

From 39c7274fc36b5f405f1dbfa48067dde52abec5ce Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 16 Mar 2017 21:09:38 -0400
Subject: [PATCH 0376/1644] ARROW-631: [GLib] Import

See also https://issues.apache.org/jira/browse/ARROW-631 and `glib/README.md` in this change.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #382 from kou/glib-import and squashes the following commits:

67a5d24 [Kouhei Sutou] [GLib] Rename directory to c_glib/ from glib/
24cd605 [Kouhei Sutou] [GLib] Import
---
 .travis.yml                                 |   5 +
 c_glib/.gitignore                           |  44 ++
 c_glib/Makefile.am                          |  26 ++
 c_glib/README.md                            | 114 +++++
 c_glib/arrow-glib/Makefile.am               | 494 ++++++++++++++++++++
 c_glib/arrow-glib/array-builder.cpp         | 229 +++++++++
 c_glib/arrow-glib/array-builder.h           |  70 +++
 c_glib/arrow-glib/array-builder.hpp         |  26 ++
 c_glib/arrow-glib/array.cpp                 | 268 +++++++++++
 c_glib/arrow-glib/array.h                   |  67 +++
 c_glib/arrow-glib/array.hpp                 |  27 ++
 c_glib/arrow-glib/arrow-glib.h              |  80 ++++
 c_glib/arrow-glib/arrow-glib.hpp            |  37 ++
 c_glib/arrow-glib/arrow-glib.pc.in          |  28 ++
 c_glib/arrow-glib/arrow-io-glib.h           |  32 ++
 c_glib/arrow-glib/arrow-io-glib.hpp         |  30 ++
 c_glib/arrow-glib/arrow-io-glib.pc.in       |  28 ++
 c_glib/arrow-glib/arrow-ipc-glib.h          |  27 ++
 c_glib/arrow-glib/arrow-ipc-glib.hpp        |  30 ++
 c_glib/arrow-glib/arrow-ipc-glib.pc.in      |  28 ++
 c_glib/arrow-glib/binary-array-builder.cpp  | 122 +++++
 c_glib/arrow-glib/binary-array-builder.h    |  77 +++
 c_glib/arrow-glib/binary-array.cpp          |  73 +++
 c_glib/arrow-glib/binary-array.h            |  72 +++
 c_glib/arrow-glib/binary-data-type.cpp      |  67 +++
 c_glib/arrow-glib/binary-data-type.h        |  69 +++
 c_glib/arrow-glib/boolean-array-builder.cpp | 120 +++++
 c_glib/arrow-glib/boolean-array-builder.h   |  76 +++
 c_glib/arrow-glib/boolean-array.cpp         |  69 +++
 c_glib/arrow-glib/boolean-array.h           |  70 +++
 c_glib/arrow-glib/boolean-data-type.cpp     |  67 +++
 c_glib/arrow-glib/boolean-data-type.h       |  69 +++
 c_glib/arrow-glib/chunked-array.cpp         | 241 ++++++++++
 c_glib/arrow-glib/chunked-array.h           |  78 ++++
 c_glib/arrow-glib/chunked-array.hpp         |  27 ++
 c_glib/arrow-glib/column.cpp                | 262 +++++++++++
 c_glib/arrow-glib/column.h                  |  82 ++++
 c_glib/arrow-glib/column.hpp                |  27 ++
 c_glib/arrow-glib/data-type.cpp             | 260 +++++++++++
 c_glib/arrow-glib/data-type.h               |  72 +++
 c_glib/arrow-glib/data-type.hpp             |  27 ++
 c_glib/arrow-glib/double-array-builder.cpp  | 120 +++++
 c_glib/arrow-glib/double-array-builder.h    |  76 +++
 c_glib/arrow-glib/double-array.cpp          |  69 +++
 c_glib/arrow-glib/double-array.h            |  71 +++
 c_glib/arrow-glib/double-data-type.cpp      |  68 +++
 c_glib/arrow-glib/double-data-type.h        |  70 +++
 c_glib/arrow-glib/enums.c.template          |  56 +++
 c_glib/arrow-glib/enums.h.template          |  41 ++
 c_glib/arrow-glib/error.cpp                 |  81 ++++
 c_glib/arrow-glib/error.h                   |  54 +++
 c_glib/arrow-glib/error.hpp                 |  28 ++
 c_glib/arrow-glib/field.cpp                 | 250 ++++++++++
 c_glib/arrow-glib/field.h                   |  83 ++++
 c_glib/arrow-glib/field.hpp                 |  27 ++
 c_glib/arrow-glib/float-array-builder.cpp   | 120 +++++
 c_glib/arrow-glib/float-array-builder.h     |  76 +++
 c_glib/arrow-glib/float-array.cpp           |  69 +++
 c_glib/arrow-glib/float-array.h             |  71 +++
 c_glib/arrow-glib/float-data-type.cpp       |  68 +++
 c_glib/arrow-glib/float-data-type.h         |  69 +++
 c_glib/arrow-glib/int16-array-builder.cpp   | 120 +++++
 c_glib/arrow-glib/int16-array-builder.h     |  76 +++
 c_glib/arrow-glib/int16-array.cpp           |  69 +++
 c_glib/arrow-glib/int16-array.h             |  71 +++
 c_glib/arrow-glib/int16-data-type.cpp       |  67 +++
 c_glib/arrow-glib/int16-data-type.h         |  69 +++
 c_glib/arrow-glib/int32-array-builder.cpp   | 120 +++++
 c_glib/arrow-glib/int32-array-builder.h     |  76 +++
 c_glib/arrow-glib/int32-array.cpp           |  69 +++
 c_glib/arrow-glib/int32-array.h             |  71 +++
 c_glib/arrow-glib/int32-data-type.cpp       |  67 +++
 c_glib/arrow-glib/int32-data-type.h         |  69 +++
 c_glib/arrow-glib/int64-array-builder.cpp   | 120 +++++
 c_glib/arrow-glib/int64-array-builder.h     |  76 +++
 c_glib/arrow-glib/int64-array.cpp           |  69 +++
 c_glib/arrow-glib/int64-array.h             |  71 +++
 c_glib/arrow-glib/int64-data-type.cpp       |  67 +++
 c_glib/arrow-glib/int64-data-type.h         |  69 +++
 c_glib/arrow-glib/int8-array-builder.cpp    | 120 +++++
 c_glib/arrow-glib/int8-array-builder.h      |  76 +++
 c_glib/arrow-glib/int8-array.cpp            |  69 +++
 c_glib/arrow-glib/int8-array.h              |  71 +++
 c_glib/arrow-glib/int8-data-type.cpp        |  67 +++
 c_glib/arrow-glib/int8-data-type.h          |  69 +++
 c_glib/arrow-glib/io-enums.c.template       |  56 +++
 c_glib/arrow-glib/io-enums.h.template       |  41 ++
 c_glib/arrow-glib/io-file-mode.cpp          |  63 +++
 c_glib/arrow-glib/io-file-mode.h            |  40 ++
 c_glib/arrow-glib/io-file-mode.hpp          |  27 ++
 c_glib/arrow-glib/io-file-output-stream.cpp | 231 +++++++++
 c_glib/arrow-glib/io-file-output-stream.h   |  72 +++
 c_glib/arrow-glib/io-file-output-stream.hpp |  28 ++
 c_glib/arrow-glib/io-file.cpp               | 116 +++++
 c_glib/arrow-glib/io-file.h                 |  51 ++
 c_glib/arrow-glib/io-file.hpp               |  38 ++
 c_glib/arrow-glib/io-input-stream.cpp       |  56 +++
 c_glib/arrow-glib/io-input-stream.h         |  45 ++
 c_glib/arrow-glib/io-input-stream.hpp       |  38 ++
 c_glib/arrow-glib/io-memory-mapped-file.cpp | 287 ++++++++++++
 c_glib/arrow-glib/io-memory-mapped-file.h   |  72 +++
 c_glib/arrow-glib/io-memory-mapped-file.hpp |  28 ++
 c_glib/arrow-glib/io-output-stream.cpp      |  56 +++
 c_glib/arrow-glib/io-output-stream.h        |  45 ++
 c_glib/arrow-glib/io-output-stream.hpp      |  38 ++
 c_glib/arrow-glib/io-readable-file.cpp      | 127 +++++
 c_glib/arrow-glib/io-readable-file.h        |  55 +++
 c_glib/arrow-glib/io-readable-file.hpp      |  38 ++
 c_glib/arrow-glib/io-readable.cpp           |  84 ++++
 c_glib/arrow-glib/io-readable.h             |  51 ++
 c_glib/arrow-glib/io-readable.hpp           |  38 ++
 c_glib/arrow-glib/io-writeable-file.cpp     |  84 ++++
 c_glib/arrow-glib/io-writeable-file.h       |  51 ++
 c_glib/arrow-glib/io-writeable-file.hpp     |  38 ++
 c_glib/arrow-glib/io-writeable.cpp          | 106 +++++
 c_glib/arrow-glib/io-writeable.h            |  52 +++
 c_glib/arrow-glib/io-writeable.hpp          |  38 ++
 c_glib/arrow-glib/ipc-enums.c.template      |  56 +++
 c_glib/arrow-glib/ipc-enums.h.template      |  41 ++
 c_glib/arrow-glib/ipc-file-reader.cpp       | 247 ++++++++++
 c_glib/arrow-glib/ipc-file-reader.h         |  83 ++++
 c_glib/arrow-glib/ipc-file-reader.hpp       |  28 ++
 c_glib/arrow-glib/ipc-file-writer.cpp       | 158 +++++++
 c_glib/arrow-glib/ipc-file-writer.h         |  78 ++++
 c_glib/arrow-glib/ipc-file-writer.hpp       |  28 ++
 c_glib/arrow-glib/ipc-metadata-version.cpp  |  59 +++
 c_glib/arrow-glib/ipc-metadata-version.h    |  39 ++
 c_glib/arrow-glib/ipc-metadata-version.hpp  |  27 ++
 c_glib/arrow-glib/ipc-stream-reader.cpp     | 221 +++++++++
 c_glib/arrow-glib/ipc-stream-reader.h       |  80 ++++
 c_glib/arrow-glib/ipc-stream-reader.hpp     |  28 ++
 c_glib/arrow-glib/ipc-stream-writer.cpp     | 232 +++++++++
 c_glib/arrow-glib/ipc-stream-writer.h       |  82 ++++
 c_glib/arrow-glib/ipc-stream-writer.hpp     |  28 ++
 c_glib/arrow-glib/list-array-builder.cpp    | 173 +++++++
 c_glib/arrow-glib/list-array-builder.h      |  77 +++
 c_glib/arrow-glib/list-array.cpp            |  92 ++++
 c_glib/arrow-glib/list-array.h              |  73 +++
 c_glib/arrow-glib/list-data-type.cpp        |  91 ++++
 c_glib/arrow-glib/list-data-type.h          |  73 +++
 c_glib/arrow-glib/null-array.cpp            |  69 +++
 c_glib/arrow-glib/null-array.h              |  70 +++
 c_glib/arrow-glib/null-data-type.cpp        |  67 +++
 c_glib/arrow-glib/null-data-type.h          |  69 +++
 c_glib/arrow-glib/record-batch.cpp          | 288 ++++++++++++
 c_glib/arrow-glib/record-batch.h            |  85 ++++
 c_glib/arrow-glib/record-batch.hpp          |  27 ++
 c_glib/arrow-glib/schema.cpp                | 245 ++++++++++
 c_glib/arrow-glib/schema.h                  |  80 ++++
 c_glib/arrow-glib/schema.hpp                |  27 ++
 c_glib/arrow-glib/string-array-builder.cpp  |  97 ++++
 c_glib/arrow-glib/string-array-builder.h    |  74 +++
 c_glib/arrow-glib/string-array.cpp          |  74 +++
 c_glib/arrow-glib/string-array.h            |  71 +++
 c_glib/arrow-glib/string-data-type.cpp      |  68 +++
 c_glib/arrow-glib/string-data-type.h        |  69 +++
 c_glib/arrow-glib/struct-array-builder.cpp  | 187 ++++++++
 c_glib/arrow-glib/struct-array-builder.h    |  81 ++++
 c_glib/arrow-glib/struct-array.cpp          |  97 ++++
 c_glib/arrow-glib/struct-array.h            |  73 +++
 c_glib/arrow-glib/struct-data-type.cpp      |  75 +++
 c_glib/arrow-glib/struct-data-type.h        |  71 +++
 c_glib/arrow-glib/table.cpp                 | 240 ++++++++++
 c_glib/arrow-glib/table.h                   |  80 ++++
 c_glib/arrow-glib/table.hpp                 |  27 ++
 c_glib/arrow-glib/type.cpp                  |  90 ++++
 c_glib/arrow-glib/type.h                    |  84 ++++
 c_glib/arrow-glib/type.hpp                  |  26 ++
 c_glib/arrow-glib/uint16-array-builder.cpp  | 120 +++++
 c_glib/arrow-glib/uint16-array-builder.h    |  76 +++
 c_glib/arrow-glib/uint16-array.cpp          |  69 +++
 c_glib/arrow-glib/uint16-array.h            |  71 +++
 c_glib/arrow-glib/uint16-data-type.cpp      |  67 +++
 c_glib/arrow-glib/uint16-data-type.h        |  69 +++
 c_glib/arrow-glib/uint32-array-builder.cpp  | 120 +++++
 c_glib/arrow-glib/uint32-array-builder.h    |  76 +++
 c_glib/arrow-glib/uint32-array.cpp          |  69 +++
 c_glib/arrow-glib/uint32-array.h            |  71 +++
 c_glib/arrow-glib/uint32-data-type.cpp      |  67 +++
 c_glib/arrow-glib/uint32-data-type.h        |  69 +++
 c_glib/arrow-glib/uint64-array-builder.cpp  | 120 +++++
 c_glib/arrow-glib/uint64-array-builder.h    |  76 +++
 c_glib/arrow-glib/uint64-array.cpp          |  69 +++
 c_glib/arrow-glib/uint64-array.h            |  71 +++
 c_glib/arrow-glib/uint64-data-type.cpp      |  67 +++
 c_glib/arrow-glib/uint64-data-type.h        |  69 +++
 c_glib/arrow-glib/uint8-array-builder.cpp   | 120 +++++
 c_glib/arrow-glib/uint8-array-builder.h     |  76 +++
 c_glib/arrow-glib/uint8-array.cpp           |  69 +++
 c_glib/arrow-glib/uint8-array.h             |  71 +++
 c_glib/arrow-glib/uint8-data-type.cpp       |  67 +++
 c_glib/arrow-glib/uint8-data-type.h         |  69 +++
 c_glib/autogen.sh                           |  31 ++
 c_glib/configure.ac                         |  76 +++
 c_glib/doc/Makefile.am                      |  19 +
 c_glib/doc/reference/Makefile.am            |  63 +++
 c_glib/doc/reference/arrow-glib-docs.sgml   | 171 +++++++
 c_glib/example/Makefile.am                  |  34 ++
 c_glib/example/build.c                      |  71 +++
 c_glib/test/helper/buildable.rb             |  77 +++
 c_glib/test/run-test.rb                     |  41 ++
 c_glib/test/run-test.sh                     |  29 ++
 c_glib/test/test-array.rb                   |  44 ++
 c_glib/test/test-binary-array.rb            |  25 +
 c_glib/test/test-binary-data-type.rb        |  28 ++
 c_glib/test/test-boolean-array.rb           |  25 +
 c_glib/test/test-boolean-data-type.rb       |  28 ++
 c_glib/test/test-chunked-array.rb           |  67 +++
 c_glib/test/test-column.rb                  |  86 ++++
 c_glib/test/test-double-array.rb            |  25 +
 c_glib/test/test-double-data-type.rb        |  28 ++
 c_glib/test/test-field.rb                   |  41 ++
 c_glib/test/test-float-array.rb             |  25 +
 c_glib/test/test-float-data-type.rb         |  28 ++
 c_glib/test/test-int16-array.rb             |  25 +
 c_glib/test/test-int16-data-type.rb         |  28 ++
 c_glib/test/test-int32-array.rb             |  25 +
 c_glib/test/test-int32-data-type.rb         |  28 ++
 c_glib/test/test-int64-array.rb             |  25 +
 c_glib/test/test-int64-data-type.rb         |  28 ++
 c_glib/test/test-int8-array.rb              |  25 +
 c_glib/test/test-int8-data-type.rb          |  28 ++
 c_glib/test/test-io-file-output-stream.rb   |  38 ++
 c_glib/test/test-io-memory-mapped-file.rb   | 138 ++++++
 c_glib/test/test-ipc-file-writer.rb         |  45 ++
 c_glib/test/test-ipc-stream-writer.rb       |  53 +++
 c_glib/test/test-list-array.rb              |  43 ++
 c_glib/test/test-list-data-type.rb          |  36 ++
 c_glib/test/test-null-array.rb              |  33 ++
 c_glib/test/test-null-data-type.rb          |  28 ++
 c_glib/test/test-record-batch.rb            |  80 ++++
 c_glib/test/test-schema.rb                  |  69 +++
 c_glib/test/test-string-array.rb            |  25 +
 c_glib/test/test-string-data-type.rb        |  28 ++
 c_glib/test/test-struct-array.rb            |  58 +++
 c_glib/test/test-table.rb                   |  72 +++
 c_glib/test/test-uint16-array.rb            |  25 +
 c_glib/test/test-uint16-data-type.rb        |  28 ++
 c_glib/test/test-uint32-array.rb            |  25 +
 c_glib/test/test-uint32-data-type.rb        |  28 ++
 c_glib/test/test-uint64-array.rb            |  25 +
 c_glib/test/test-uint64-data-type.rb        |  28 ++
 c_glib/test/test-uint8-array.rb             |  25 +
 c_glib/test/test-uint8-data-type.rb         |  28 ++
 ci/travis_before_script_c_glib.sh           |  40 ++
 ci/travis_script_c_glib.sh                  |  24 +
 246 files changed, 18251 insertions(+)
 create mode 100644 c_glib/.gitignore
 create mode 100644 c_glib/Makefile.am
 create mode 100644 c_glib/README.md
 create mode 100644 c_glib/arrow-glib/Makefile.am
 create mode 100644 c_glib/arrow-glib/array-builder.cpp
 create mode 100644 c_glib/arrow-glib/array-builder.h
 create mode 100644 c_glib/arrow-glib/array-builder.hpp
 create mode 100644 c_glib/arrow-glib/array.cpp
 create mode 100644 c_glib/arrow-glib/array.h
 create mode 100644 c_glib/arrow-glib/array.hpp
 create mode 100644 c_glib/arrow-glib/arrow-glib.h
 create mode 100644 c_glib/arrow-glib/arrow-glib.hpp
 create mode 100644 c_glib/arrow-glib/arrow-glib.pc.in
 create mode 100644 c_glib/arrow-glib/arrow-io-glib.h
 create mode 100644 c_glib/arrow-glib/arrow-io-glib.hpp
 create mode 100644 c_glib/arrow-glib/arrow-io-glib.pc.in
 create mode 100644 c_glib/arrow-glib/arrow-ipc-glib.h
 create mode 100644 c_glib/arrow-glib/arrow-ipc-glib.hpp
 create mode 100644 c_glib/arrow-glib/arrow-ipc-glib.pc.in
 create mode 100644 c_glib/arrow-glib/binary-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/binary-array-builder.h
 create mode 100644 c_glib/arrow-glib/binary-array.cpp
 create mode 100644 c_glib/arrow-glib/binary-array.h
 create mode 100644 c_glib/arrow-glib/binary-data-type.cpp
 create mode 100644 c_glib/arrow-glib/binary-data-type.h
 create mode 100644 c_glib/arrow-glib/boolean-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/boolean-array-builder.h
 create mode 100644 c_glib/arrow-glib/boolean-array.cpp
 create mode 100644 c_glib/arrow-glib/boolean-array.h
 create mode 100644 c_glib/arrow-glib/boolean-data-type.cpp
 create mode 100644 c_glib/arrow-glib/boolean-data-type.h
 create mode 100644 c_glib/arrow-glib/chunked-array.cpp
 create mode 100644 c_glib/arrow-glib/chunked-array.h
 create mode 100644 c_glib/arrow-glib/chunked-array.hpp
 create mode 100644 c_glib/arrow-glib/column.cpp
 create mode 100644 c_glib/arrow-glib/column.h
 create mode 100644 c_glib/arrow-glib/column.hpp
 create mode 100644 c_glib/arrow-glib/data-type.cpp
 create mode 100644 c_glib/arrow-glib/data-type.h
 create mode 100644 c_glib/arrow-glib/data-type.hpp
 create mode 100644 c_glib/arrow-glib/double-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/double-array-builder.h
 create mode 100644 c_glib/arrow-glib/double-array.cpp
 create mode 100644 c_glib/arrow-glib/double-array.h
 create mode 100644 c_glib/arrow-glib/double-data-type.cpp
 create mode 100644 c_glib/arrow-glib/double-data-type.h
 create mode 100644 c_glib/arrow-glib/enums.c.template
 create mode 100644 c_glib/arrow-glib/enums.h.template
 create mode 100644 c_glib/arrow-glib/error.cpp
 create mode 100644 c_glib/arrow-glib/error.h
 create mode 100644 c_glib/arrow-glib/error.hpp
 create mode 100644 c_glib/arrow-glib/field.cpp
 create mode 100644 c_glib/arrow-glib/field.h
 create mode 100644 c_glib/arrow-glib/field.hpp
 create mode 100644 c_glib/arrow-glib/float-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/float-array-builder.h
 create mode 100644 c_glib/arrow-glib/float-array.cpp
 create mode 100644 c_glib/arrow-glib/float-array.h
 create mode 100644 c_glib/arrow-glib/float-data-type.cpp
 create mode 100644 c_glib/arrow-glib/float-data-type.h
 create mode 100644 c_glib/arrow-glib/int16-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/int16-array-builder.h
 create mode 100644 c_glib/arrow-glib/int16-array.cpp
 create mode 100644 c_glib/arrow-glib/int16-array.h
 create mode 100644 c_glib/arrow-glib/int16-data-type.cpp
 create mode 100644 c_glib/arrow-glib/int16-data-type.h
 create mode 100644 c_glib/arrow-glib/int32-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/int32-array-builder.h
 create mode 100644 c_glib/arrow-glib/int32-array.cpp
 create mode 100644 c_glib/arrow-glib/int32-array.h
 create mode 100644 c_glib/arrow-glib/int32-data-type.cpp
 create mode 100644 c_glib/arrow-glib/int32-data-type.h
 create mode 100644 c_glib/arrow-glib/int64-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/int64-array-builder.h
 create mode 100644 c_glib/arrow-glib/int64-array.cpp
 create mode 100644 c_glib/arrow-glib/int64-array.h
 create mode 100644 c_glib/arrow-glib/int64-data-type.cpp
 create mode 100644 c_glib/arrow-glib/int64-data-type.h
 create mode 100644 c_glib/arrow-glib/int8-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/int8-array-builder.h
 create mode 100644 c_glib/arrow-glib/int8-array.cpp
 create mode 100644 c_glib/arrow-glib/int8-array.h
 create mode 100644 c_glib/arrow-glib/int8-data-type.cpp
 create mode 100644 c_glib/arrow-glib/int8-data-type.h
 create mode 100644 c_glib/arrow-glib/io-enums.c.template
 create mode 100644 c_glib/arrow-glib/io-enums.h.template
 create mode 100644 c_glib/arrow-glib/io-file-mode.cpp
 create mode 100644 c_glib/arrow-glib/io-file-mode.h
 create mode 100644 c_glib/arrow-glib/io-file-mode.hpp
 create mode 100644 c_glib/arrow-glib/io-file-output-stream.cpp
 create mode 100644 c_glib/arrow-glib/io-file-output-stream.h
 create mode 100644 c_glib/arrow-glib/io-file-output-stream.hpp
 create mode 100644 c_glib/arrow-glib/io-file.cpp
 create mode 100644 c_glib/arrow-glib/io-file.h
 create mode 100644 c_glib/arrow-glib/io-file.hpp
 create mode 100644 c_glib/arrow-glib/io-input-stream.cpp
 create mode 100644 c_glib/arrow-glib/io-input-stream.h
 create mode 100644 c_glib/arrow-glib/io-input-stream.hpp
 create mode 100644 c_glib/arrow-glib/io-memory-mapped-file.cpp
 create mode 100644 c_glib/arrow-glib/io-memory-mapped-file.h
 create mode 100644 c_glib/arrow-glib/io-memory-mapped-file.hpp
 create mode 100644 c_glib/arrow-glib/io-output-stream.cpp
 create mode 100644 c_glib/arrow-glib/io-output-stream.h
 create mode 100644 c_glib/arrow-glib/io-output-stream.hpp
 create mode 100644 c_glib/arrow-glib/io-readable-file.cpp
 create mode 100644 c_glib/arrow-glib/io-readable-file.h
 create mode 100644 c_glib/arrow-glib/io-readable-file.hpp
 create mode 100644 c_glib/arrow-glib/io-readable.cpp
 create mode 100644 c_glib/arrow-glib/io-readable.h
 create mode 100644 c_glib/arrow-glib/io-readable.hpp
 create mode 100644 c_glib/arrow-glib/io-writeable-file.cpp
 create mode 100644 c_glib/arrow-glib/io-writeable-file.h
 create mode 100644 c_glib/arrow-glib/io-writeable-file.hpp
 create mode 100644 c_glib/arrow-glib/io-writeable.cpp
 create mode 100644 c_glib/arrow-glib/io-writeable.h
 create mode 100644 c_glib/arrow-glib/io-writeable.hpp
 create mode 100644 c_glib/arrow-glib/ipc-enums.c.template
 create mode 100644 c_glib/arrow-glib/ipc-enums.h.template
 create mode 100644 c_glib/arrow-glib/ipc-file-reader.cpp
 create mode 100644 c_glib/arrow-glib/ipc-file-reader.h
 create mode 100644 c_glib/arrow-glib/ipc-file-reader.hpp
 create mode 100644 c_glib/arrow-glib/ipc-file-writer.cpp
 create mode 100644 c_glib/arrow-glib/ipc-file-writer.h
 create mode 100644 c_glib/arrow-glib/ipc-file-writer.hpp
 create mode 100644 c_glib/arrow-glib/ipc-metadata-version.cpp
 create mode 100644 c_glib/arrow-glib/ipc-metadata-version.h
 create mode 100644 c_glib/arrow-glib/ipc-metadata-version.hpp
 create mode 100644 c_glib/arrow-glib/ipc-stream-reader.cpp
 create mode 100644 c_glib/arrow-glib/ipc-stream-reader.h
 create mode 100644 c_glib/arrow-glib/ipc-stream-reader.hpp
 create mode 100644 c_glib/arrow-glib/ipc-stream-writer.cpp
 create mode 100644 c_glib/arrow-glib/ipc-stream-writer.h
 create mode 100644 c_glib/arrow-glib/ipc-stream-writer.hpp
 create mode 100644 c_glib/arrow-glib/list-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/list-array-builder.h
 create mode 100644 c_glib/arrow-glib/list-array.cpp
 create mode 100644 c_glib/arrow-glib/list-array.h
 create mode 100644 c_glib/arrow-glib/list-data-type.cpp
 create mode 100644 c_glib/arrow-glib/list-data-type.h
 create mode 100644 c_glib/arrow-glib/null-array.cpp
 create mode 100644 c_glib/arrow-glib/null-array.h
 create mode 100644 c_glib/arrow-glib/null-data-type.cpp
 create mode 100644 c_glib/arrow-glib/null-data-type.h
 create mode 100644 c_glib/arrow-glib/record-batch.cpp
 create mode 100644 c_glib/arrow-glib/record-batch.h
 create mode 100644 c_glib/arrow-glib/record-batch.hpp
 create mode 100644 c_glib/arrow-glib/schema.cpp
 create mode 100644 c_glib/arrow-glib/schema.h
 create mode 100644 c_glib/arrow-glib/schema.hpp
 create mode 100644 c_glib/arrow-glib/string-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/string-array-builder.h
 create mode 100644 c_glib/arrow-glib/string-array.cpp
 create mode 100644 c_glib/arrow-glib/string-array.h
 create mode 100644 c_glib/arrow-glib/string-data-type.cpp
 create mode 100644 c_glib/arrow-glib/string-data-type.h
 create mode 100644 c_glib/arrow-glib/struct-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/struct-array-builder.h
 create mode 100644 c_glib/arrow-glib/struct-array.cpp
 create mode 100644 c_glib/arrow-glib/struct-array.h
 create mode 100644 c_glib/arrow-glib/struct-data-type.cpp
 create mode 100644 c_glib/arrow-glib/struct-data-type.h
 create mode 100644 c_glib/arrow-glib/table.cpp
 create mode 100644 c_glib/arrow-glib/table.h
 create mode 100644 c_glib/arrow-glib/table.hpp
 create mode 100644 c_glib/arrow-glib/type.cpp
 create mode 100644 c_glib/arrow-glib/type.h
 create mode 100644 c_glib/arrow-glib/type.hpp
 create mode 100644 c_glib/arrow-glib/uint16-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/uint16-array-builder.h
 create mode 100644 c_glib/arrow-glib/uint16-array.cpp
 create mode 100644 c_glib/arrow-glib/uint16-array.h
 create mode 100644 c_glib/arrow-glib/uint16-data-type.cpp
 create mode 100644 c_glib/arrow-glib/uint16-data-type.h
 create mode 100644 c_glib/arrow-glib/uint32-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/uint32-array-builder.h
 create mode 100644 c_glib/arrow-glib/uint32-array.cpp
 create mode 100644 c_glib/arrow-glib/uint32-array.h
 create mode 100644 c_glib/arrow-glib/uint32-data-type.cpp
 create mode 100644 c_glib/arrow-glib/uint32-data-type.h
 create mode 100644 c_glib/arrow-glib/uint64-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/uint64-array-builder.h
 create mode 100644 c_glib/arrow-glib/uint64-array.cpp
 create mode 100644 c_glib/arrow-glib/uint64-array.h
 create mode 100644 c_glib/arrow-glib/uint64-data-type.cpp
 create mode 100644 c_glib/arrow-glib/uint64-data-type.h
 create mode 100644 c_glib/arrow-glib/uint8-array-builder.cpp
 create mode 100644 c_glib/arrow-glib/uint8-array-builder.h
 create mode 100644 c_glib/arrow-glib/uint8-array.cpp
 create mode 100644 c_glib/arrow-glib/uint8-array.h
 create mode 100644 c_glib/arrow-glib/uint8-data-type.cpp
 create mode 100644 c_glib/arrow-glib/uint8-data-type.h
 create mode 100755 c_glib/autogen.sh
 create mode 100644 c_glib/configure.ac
 create mode 100644 c_glib/doc/Makefile.am
 create mode 100644 c_glib/doc/reference/Makefile.am
 create mode 100644 c_glib/doc/reference/arrow-glib-docs.sgml
 create mode 100644 c_glib/example/Makefile.am
 create mode 100644 c_glib/example/build.c
 create mode 100644 c_glib/test/helper/buildable.rb
 create mode 100755 c_glib/test/run-test.rb
 create mode 100755 c_glib/test/run-test.sh
 create mode 100644 c_glib/test/test-array.rb
 create mode 100644 c_glib/test/test-binary-array.rb
 create mode 100644 c_glib/test/test-binary-data-type.rb
 create mode 100644 c_glib/test/test-boolean-array.rb
 create mode 100644 c_glib/test/test-boolean-data-type.rb
 create mode 100644 c_glib/test/test-chunked-array.rb
 create mode 100644 c_glib/test/test-column.rb
 create mode 100644 c_glib/test/test-double-array.rb
 create mode 100644 c_glib/test/test-double-data-type.rb
 create mode 100644 c_glib/test/test-field.rb
 create mode 100644 c_glib/test/test-float-array.rb
 create mode 100644 c_glib/test/test-float-data-type.rb
 create mode 100644 c_glib/test/test-int16-array.rb
 create mode 100644 c_glib/test/test-int16-data-type.rb
 create mode 100644 c_glib/test/test-int32-array.rb
 create mode 100644 c_glib/test/test-int32-data-type.rb
 create mode 100644 c_glib/test/test-int64-array.rb
 create mode 100644 c_glib/test/test-int64-data-type.rb
 create mode 100644 c_glib/test/test-int8-array.rb
 create mode 100644 c_glib/test/test-int8-data-type.rb
 create mode 100644 c_glib/test/test-io-file-output-stream.rb
 create mode 100644 c_glib/test/test-io-memory-mapped-file.rb
 create mode 100644 c_glib/test/test-ipc-file-writer.rb
 create mode 100644 c_glib/test/test-ipc-stream-writer.rb
 create mode 100644 c_glib/test/test-list-array.rb
 create mode 100644 c_glib/test/test-list-data-type.rb
 create mode 100644 c_glib/test/test-null-array.rb
 create mode 100644 c_glib/test/test-null-data-type.rb
 create mode 100644 c_glib/test/test-record-batch.rb
 create mode 100644 c_glib/test/test-schema.rb
 create mode 100644 c_glib/test/test-string-array.rb
 create mode 100644 c_glib/test/test-string-data-type.rb
 create mode 100644 c_glib/test/test-struct-array.rb
 create mode 100644 c_glib/test/test-table.rb
 create mode 100644 c_glib/test/test-uint16-array.rb
 create mode 100644 c_glib/test/test-uint16-data-type.rb
 create mode 100644 c_glib/test/test-uint32-array.rb
 create mode 100644 c_glib/test/test-uint32-data-type.rb
 create mode 100644 c_glib/test/test-uint64-array.rb
 create mode 100644 c_glib/test/test-uint64-data-type.rb
 create mode 100644 c_glib/test/test-uint8-array.rb
 create mode 100644 c_glib/test/test-uint8-data-type.rb
 create mode 100755 ci/travis_before_script_c_glib.sh
 create mode 100755 ci/travis_script_c_glib.sh

diff --git a/.travis.yml b/.travis.yml
index e8d91045c2254..b219b03e0eb2b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,9 @@ addons:
     - libboost-filesystem-dev
     - libboost-system-dev
     - libjemalloc-dev
+    - gtk-doc-tools
+    - autoconf-archive
+    - libgirepository1.0-dev
 
 matrix:
   fast_finish: true
@@ -30,9 +33,11 @@ matrix:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
   - compiler: clang
     osx_image: xcode6.4
     os: osx
diff --git a/c_glib/.gitignore b/c_glib/.gitignore
new file mode 100644
index 0000000000000..38e33a2cd88e7
--- /dev/null
+++ b/c_glib/.gitignore
@@ -0,0 +1,44 @@
+Makefile
+Makefile.in
+.deps/
+.libs/
+*.gir
+*.typelib
+*.o
+*.lo
+*.la
+*~
+/*.tar.gz
+/aclocal.m4
+/autom4te.cache/
+/config.h
+/config.h.in
+/config.log
+/config.status
+/config/
+/configure
+/doc/reference/*.txt
+/doc/reference/*.txt.bak
+/doc/reference/*.args
+/doc/reference/*.hierarchy
+/doc/reference/*.interfaces
+/doc/reference/*.prerequisites
+/doc/reference/*.signals
+/doc/reference/*.types
+/doc/reference/gtk-doc.make
+/doc/reference/*.stamp
+/doc/reference/html/
+/doc/reference/xml/
+/libtool
+/m4/
+/stamp-h1
+/version
+/arrow-glib/enums.c
+/arrow-glib/enums.h
+/arrow-glib/io-enums.c
+/arrow-glib/io-enums.h
+/arrow-glib/ipc-enums.c
+/arrow-glib/ipc-enums.h
+/arrow-glib/stamp-*
+/arrow-glib/*.pc
+/example/build
diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am
new file mode 100644
index 0000000000000..076f9be08524b
--- /dev/null
+++ b/c_glib/Makefile.am
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
+
+SUBDIRS =					\
+	arrow-glib				\
+	doc					\
+	example
+
+EXTRA_DIST =					\
+	version
diff --git a/c_glib/README.md b/c_glib/README.md
new file mode 100644
index 0000000000000..4008015a56438
--- /dev/null
+++ b/c_glib/README.md
@@ -0,0 +1,114 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Arrow GLib
+
+Arrow GLib is a wrapper library for Arrow C++. Arrow GLib provides C
+API.
+
+Arrow GLib supports
+[GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection).
+It means that you can create language bindings at runtime or compile time.
+
+For example, you can use Apache Arrow from Ruby by Arrow GLib and
+[gobject-introspection gem](https://rubygems.org/gems/gobject-introspection)
+with the following code:
+
+```ruby
+# Generate bindings at runtime
+require "gi"
+Arrow = GI.load("Arrow")
+
+# Now, you can access arrow::BooleanArray in Arrow C++ by
+# Arrow::BooleanArray
+p Arrow::BooleanArray
+```
+
+In Ruby case, you should use
+[red-arrow gem](https://rubygems.org/gems/red-arrow). It's based on
+gobject-introspection gem. It adds many convenient features to raw
+gobject-introspection gem based bindings.
+
+## Install
+
+### Package
+
+TODO
+
+### Build
+
+You need to install Arrow C++ before you install Arrow GLib. See Arrow
+C++ document about how to install Arrow C++.
+
+You need [GTK-Doc](https://www.gtk.org/gtk-doc/) and
+[GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection)
+to build Arrow GLib. You can install them by the followings:
+
+On Debian GNU/Linux or Ubuntu:
+
+```text
+% sudo apt install -y -V gtk-doc-tools libgirepository1.0-dev
+```
+
+On CentOS 7 or later:
+
+```text
+% sudo yum install -y gtk-doc gobject-introspection-devel
+```
+
+On macOS with [Homebrew](https://brew.sh/):
+
+```text
+% brew install -y gtk-doc gobject-introspection
+```
+
+Now, you can build Arrow GLib:
+
+```text
+% cd glib
+% ./configure --enable-gtk-doc
+% make
+% sudo make install
+```
+
+## Usage
+
+You can use Arrow GLib with C or other languages. If you use Arrow
+GLib with C, you use C API. If you use Arrow GLib with other
+languages, you use GObject Introspection based bindings.
+
+### C
+
+You can find API reference in the
+`/usr/local/share/gtk-doc/html/arrow-glib/` directory. If you specify
+`--prefix` to `configure`, the directory will be different.
+
+You can find example codes in the `example/` directory.
+
+### Language bindings
+
+You can use Arrow GLib with non C languages with GObject Introspection
+based bindings. Here are languages that support GObject Introspection:
+
+  * Ruby: [red-arrow gem](https://rubygems.org/gems/red-arrow) should be used.
+
+  * Python: [PyGObject](https://wiki.gnome.org/Projects/PyGObject) should be used. (Note that you should use PyArrow than Arrow GLib.)
+
+  * Lua: [LGI](https://github.com/pavouk/lgi) should be used.
+
+  * Go: [Go-gir-generator](https://github.com/linuxdeepin/go-gir-generator) should be used.
+
+See also
+[Projects/GObjectIntrospection/Users - GNOME Wiki!](https://wiki.gnome.org/Projects/GObjectIntrospection/Users)
+for other languages.
diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
new file mode 100644
index 0000000000000..61137a075f601
--- /dev/null
+++ b/c_glib/arrow-glib/Makefile.am
@@ -0,0 +1,494 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+CLEANFILES =
+
+EXTRA_DIST =
+
+AM_CPPFLAGS =					\
+	-I$(top_builddir)			\
+	-I$(top_srcdir)
+
+AM_CFLAGS =					\
+	$(GLIB_CFLAGS)				\
+	$(GARROW_CFLAGS)
+
+# libarrow-glib
+lib_LTLIBRARIES =				\
+	libarrow-glib.la
+
+libarrow_glib_la_CXXFLAGS =			\
+	$(GLIB_CFLAGS)				\
+	$(ARROW_CFLAGS)				\
+	$(GARROW_CXXFLAGS)
+
+libarrow_glib_la_LIBADD =			\
+	$(GLIB_LIBS)				\
+	$(ARROW_LIBS)
+
+libarrow_glib_la_headers =			\
+	array.h					\
+	array-builder.h				\
+	arrow-glib.h				\
+	chunked-array.h				\
+	column.h				\
+	binary-array.h				\
+	binary-array-builder.h			\
+	binary-data-type.h			\
+	boolean-array.h				\
+	boolean-array-builder.h			\
+	boolean-data-type.h			\
+	data-type.h				\
+	double-array.h				\
+	double-array-builder.h			\
+	double-data-type.h			\
+	error.h					\
+	field.h					\
+	float-array.h				\
+	float-array-builder.h			\
+	float-data-type.h			\
+	int8-array.h				\
+	int8-array-builder.h			\
+	int8-data-type.h			\
+	int16-array.h				\
+	int16-array-builder.h			\
+	int16-data-type.h			\
+	int32-array.h				\
+	int32-array-builder.h			\
+	int32-data-type.h			\
+	int64-array.h				\
+	int64-array-builder.h			\
+	int64-data-type.h			\
+	list-array.h				\
+	list-array-builder.h			\
+	list-data-type.h			\
+	null-array.h				\
+	null-data-type.h			\
+	record-batch.h				\
+	schema.h				\
+	string-array.h				\
+	string-array-builder.h			\
+	string-data-type.h			\
+	struct-array.h				\
+	struct-array-builder.h			\
+	struct-data-type.h			\
+	table.h					\
+	type.h					\
+	uint8-array.h				\
+	uint8-array-builder.h			\
+	uint8-data-type.h			\
+	uint16-array.h				\
+	uint16-array-builder.h			\
+	uint16-data-type.h			\
+	uint32-array.h				\
+	uint32-array-builder.h			\
+	uint32-data-type.h			\
+	uint64-array.h				\
+	uint64-array-builder.h			\
+	uint64-data-type.h
+
+libarrow_glib_la_generated_headers =		\
+	enums.h
+
+libarrow_glib_la_generated_sources =		\
+	enums.c					\
+	$(libarrow_glib_la_generated_headers)
+
+libarrow_glib_la_sources =			\
+	array.cpp				\
+	array-builder.cpp			\
+	binary-array.cpp			\
+	binary-array-builder.cpp		\
+	binary-data-type.cpp			\
+	boolean-array.cpp			\
+	boolean-array-builder.cpp		\
+	boolean-data-type.cpp			\
+	chunked-array.cpp			\
+	column.cpp				\
+	data-type.cpp				\
+	double-array.cpp			\
+	double-array-builder.cpp		\
+	double-data-type.cpp			\
+	error.cpp				\
+	field.cpp				\
+	float-array.cpp				\
+	float-array-builder.cpp			\
+	float-data-type.cpp			\
+	int8-array.cpp				\
+	int8-array-builder.cpp			\
+	int8-data-type.cpp			\
+	int16-array.cpp				\
+	int16-array-builder.cpp			\
+	int16-data-type.cpp			\
+	int32-array.cpp				\
+	int32-array-builder.cpp			\
+	int32-data-type.cpp			\
+	int64-array.cpp				\
+	int64-array-builder.cpp			\
+	int64-data-type.cpp			\
+	list-array.cpp				\
+	list-array-builder.cpp			\
+	list-data-type.cpp			\
+	null-array.cpp				\
+	null-data-type.cpp			\
+	record-batch.cpp			\
+	schema.cpp				\
+	string-array.cpp			\
+	string-array-builder.cpp		\
+	string-data-type.cpp			\
+	struct-array.cpp			\
+	struct-array-builder.cpp		\
+	struct-data-type.cpp			\
+	table.cpp				\
+	type.cpp				\
+	uint8-array.cpp				\
+	uint8-array-builder.cpp			\
+	uint8-data-type.cpp			\
+	uint16-array.cpp			\
+	uint16-array-builder.cpp		\
+	uint16-data-type.cpp			\
+	uint32-array.cpp			\
+	uint32-array-builder.cpp		\
+	uint32-data-type.cpp			\
+	uint64-array.cpp			\
+	uint64-array-builder.cpp		\
+	uint64-data-type.cpp			\
+	$(libarrow_glib_la_headers)		\
+	$(libarrow_glib_la_generated_sources)
+
+libarrow_glib_la_cpp_headers =			\
+	array.hpp				\
+	array-builder.hpp			\
+	arrow-glib.hpp				\
+	chunked-array.hpp			\
+	column.hpp				\
+	data-type.hpp				\
+	error.hpp				\
+	field.hpp				\
+	record-batch.hpp			\
+	schema.hpp				\
+	table.hpp				\
+	type.hpp
+
+libarrow_glib_la_SOURCES =			\
+	$(libarrow_glib_la_sources)		\
+	$(libarrow_glib_la_cpp_headers)
+
+BUILT_SOURCES =					\
+	$(libarrow_glib_la_genearted_sources)	\
+	stamp-enums.c				\
+	stamp-enums.h
+
+EXTRA_DIST +=					\
+	enums.c.template			\
+	enums.h.template
+
+enums.h: stamp-enums.h
+	@true
+stamp-enums.h: $(libarrow_glib_la_headers) enums.h.template
+	$(AM_V_GEN)					\
+	  (cd $(srcdir) &&				\
+	   $(GLIB_MKENUMS)				\
+	     --identifier-prefix GArrow			\
+	     --symbol-prefix garrow			\
+	     --template enums.h.template		\
+	     $(libarrow_glib_la_headers)) > enums.h
+	touch $@
+
+enums.c: stamp-enums.c
+	@true
+stamp-enums.c: $(libarrow_glib_la_headers) enums.c.template
+	$(AM_V_GEN)					\
+	  (cd $(srcdir) &&				\
+	   $(GLIB_MKENUMS)				\
+	     --identifier-prefix GArrow			\
+	     --symbol-prefix garrow			\
+	     --template enums.c.template		\
+	     $(libarrow_glib_la_headers)) > enums.c
+	touch $@
+
+# libarrow-io-glib
+lib_LTLIBRARIES +=				\
+	libarrow-io-glib.la
+
+libarrow_io_glib_la_CXXFLAGS =			\
+	$(GLIB_CFLAGS)				\
+	$(ARROW_IO_CFLAGS)			\
+	$(GARROW_CXXFLAGS)
+
+libarrow_io_glib_la_LIBADD =			\
+	$(GLIB_LIBS)				\
+	$(ARROW_IO_LIBS)
+
+libarrow_io_glib_la_headers =			\
+	arrow-io-glib.h				\
+	io-file.h				\
+	io-file-mode.h				\
+	io-file-output-stream.h			\
+	io-input-stream.h			\
+	io-memory-mapped-file.h			\
+	io-output-stream.h			\
+	io-readable.h				\
+	io-readable-file.h			\
+	io-writeable.h				\
+	io-writeable-file.h
+
+libarrow_io_glib_la_generated_headers =		\
+	io-enums.h
+
+libarrow_io_glib_la_generated_sources =			\
+	io-enums.c					\
+	$(libarrow_io_glib_la_generated_headers)
+
+libarrow_io_glib_la_sources =			\
+	io-file.cpp				\
+	io-file-mode.cpp			\
+	io-file-output-stream.cpp		\
+	io-input-stream.cpp			\
+	io-memory-mapped-file.cpp		\
+	io-output-stream.cpp			\
+	io-readable.cpp				\
+	io-readable-file.cpp			\
+	io-writeable.cpp			\
+	io-writeable-file.cpp			\
+	$(libarrow_io_glib_la_headers)		\
+	$(libarrow_io_glib_la_generated_sources)
+
+libarrow_io_glib_la_cpp_headers =		\
+	arrow-io-glib.hpp			\
+	io-file.hpp				\
+	io-file-mode.hpp			\
+	io-file-output-stream.hpp		\
+	io-input-stream.hpp			\
+	io-memory-mapped-file.hpp		\
+	io-output-stream.hpp			\
+	io-readable.hpp				\
+	io-readable-file.hpp			\
+	io-writeable.hpp			\
+	io-writeable-file.hpp
+
+libarrow_io_glib_la_SOURCES =			\
+	$(libarrow_io_glib_la_sources)		\
+	$(libarrow_io_glib_la_cpp_headers)
+
+BUILT_SOURCES +=					\
+	$(libarrow_io_glib_la_genearted_sources)	\
+	stamp-io-enums.c				\
+	stamp-io-enums.h
+
+EXTRA_DIST +=					\
+	io-enums.c.template			\
+	io-enums.h.template
+
+io-enums.h: stamp-io-enums.h
+	@true
+stamp-io-enums.h: $(libarrow_io_glib_la_headers) io-enums.h.template
+	$(AM_V_GEN)					\
+	  (cd $(srcdir) &&				\
+	   $(GLIB_MKENUMS)				\
+	     --identifier-prefix GArrowIO		\
+	     --symbol-prefix garrow_io			\
+	     --template io-enums.h.template		\
+	     $(libarrow_io_glib_la_headers)) > io-enums.h
+	touch $@
+
+io-enums.c: stamp-io-enums.c
+	@true
+stamp-io-enums.c: $(libarrow_io_glib_la_headers) io-enums.c.template
+	$(AM_V_GEN)					\
+	  (cd $(srcdir) &&				\
+	   $(GLIB_MKENUMS)				\
+	     --identifier-prefix GArrowIO		\
+	     --symbol-prefix garrow_io			\
+	     --template io-enums.c.template		\
+	     $(libarrow_io_glib_la_headers)) > io-enums.c
+	touch $@
+
+# libarrow-ipc-glib
+lib_LTLIBRARIES +=				\
+	libarrow-ipc-glib.la
+
+libarrow_ipc_glib_la_CXXFLAGS =			\
+	$(GLIB_CFLAGS)				\
+	$(ARROW_IPC_CFLAGS)			\
+	$(GARROW_CXXFLAGS)
+
+libarrow_ipc_glib_la_LIBADD =			\
+	$(GLIB_LIBS)				\
+	$(ARROW_IPC_LIBS)
+
+libarrow_ipc_glib_la_headers =			\
+	arrow-ipc-glib.h			\
+	ipc-file-reader.h			\
+	ipc-file-writer.h			\
+	ipc-stream-reader.h			\
+	ipc-stream-writer.h			\
+	ipc-metadata-version.h
+
+libarrow_ipc_glib_la_generated_headers =	\
+	ipc-enums.h
+
+libarrow_ipc_glib_la_generated_sources =		\
+	ipc-enums.c					\
+	$(libarrow_ipc_glib_la_generated_headers)
+
+libarrow_ipc_glib_la_sources =				\
+	ipc-file-reader.cpp				\
+	ipc-file-writer.cpp				\
+	ipc-metadata-version.cpp			\
+	ipc-stream-reader.cpp				\
+	ipc-stream-writer.cpp				\
+	$(libarrow_ipc_glib_la_headers)			\
+	$(libarrow_ipc_glib_la_generated_sources)
+
+libarrow_ipc_glib_la_cpp_headers =		\
+	arrow-ipc-glib.hpp			\
+	ipc-file-reader.hpp			\
+	ipc-file-writer.hpp			\
+	ipc-metadata-version.hpp		\
+	ipc-stream-reader.hpp			\
+	ipc-stream-writer.hpp
+
+libarrow_ipc_glib_la_SOURCES =			\
+	$(libarrow_ipc_glib_la_sources)		\
+	$(libarrow_ipc_glib_la_cpp_headers)
+
+BUILT_SOURCES +=					\
+	$(libarrow_ipc_glib_la_genearted_sources)	\
+	stamp-ipc-enums.c				\
+	stamp-ipc-enums.h
+
+EXTRA_DIST +=					\
+	ipc-enums.c.template			\
+	ipc-enums.h.template
+
+ipc-enums.h: stamp-ipc-enums.h
+	@true
+stamp-ipc-enums.h: $(libarrow_ipc_glib_la_headers) ipc-enums.h.template
+	$(AM_V_GEN)						\
+	  (cd $(srcdir) &&					\
+	   $(GLIB_MKENUMS)					\
+	     --identifier-prefix GArrowIPC			\
+	     --symbol-prefix garrow_ipc				\
+	     --template ipc-enums.h.template			\
+	     $(libarrow_ipc_glib_la_headers)) > ipc-enums.h
+	touch $@
+
+ipc-enums.c: stamp-ipc-enums.c
+	@true
+stamp-ipc-enums.c: $(libarrow_ipc_glib_la_headers) ipc-enums.c.template
+	$(AM_V_GEN)						\
+	  (cd $(srcdir) &&					\
+	   $(GLIB_MKENUMS)					\
+	     --identifier-prefix GArrowIPC			\
+	     --symbol-prefix garrow_ipc				\
+	     --template ipc-enums.c.template			\
+	     $(libarrow_ipc_glib_la_headers)) > ipc-enums.c
+	touch $@
+
+pkginclude_HEADERS =					\
+	$(libarrow_glib_la_headers)			\
+	$(libarrow_glib_la_cpp_headers)			\
+	$(libarrow_glib_la_generated_headers)		\
+	$(libarrow_io_glib_la_headers)			\
+	$(libarrow_io_glib_la_cpp_headers)		\
+	$(libarrow_io_glib_la_generated_headers)	\
+	$(libarrow_ipc_glib_la_headers)			\
+	$(libarrow_ipc_glib_la_cpp_headers)		\
+	$(libarrow_ipc_glib_la_generated_headers)
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA =				\
+	arrow-glib.pc				\
+	arrow-io-glib.pc			\
+	arrow-ipc-glib.pc
+
+# GObject Introspection
+-include $(INTROSPECTION_MAKEFILE)
+INTROSPECTION_GIRS =
+INTROSPECTION_SCANNER_ARGS =
+INTROSPECTION_COMPILER_ARGS =
+
+if HAVE_INTROSPECTION
+Arrow-1.0.gir: libarrow-glib.la
+Arrow_1_0_gir_PACKAGES =			\
+	gobject-2.0
+Arrow_1_0_gir_EXPORT_PACKAGES = arrow
+Arrow_1_0_gir_INCLUDES = GObject-2.0
+Arrow_1_0_gir_CFLAGS =				\
+	$(AM_CPPFLAGS)
+Arrow_1_0_gir_LIBS = libarrow-glib.la
+Arrow_1_0_gir_FILES = $(libarrow_glib_la_sources)
+Arrow_1_0_gir_SCANNERFLAGS =			\
+	--warn-all				\
+	--identifier-prefix=GArrow		\
+	--symbol-prefix=garrow
+INTROSPECTION_GIRS += Arrow-1.0.gir
+
+ArrowIO-1.0.gir: libarrow-io-glib.la
+ArrowIO-1.0.gir: Arrow-1.0.gir
+ArrowIO_1_0_gir_PACKAGES =			\
+	gobject-2.0
+ArrowIO_1_0_gir_EXPORT_PACKAGES = arrow-io
+ArrowIO_1_0_gir_INCLUDES =			\
+	GObject-2.0
+ArrowIO_1_0_gir_CFLAGS =			\
+	$(AM_CPPFLAGS)
+ArrowIO_1_0_gir_LIBS =				\
+	libarrow-io-glib.la			\
+	libarrow-glib.la
+ArrowIO_1_0_gir_FILES = $(libarrow_io_glib_la_sources)
+ArrowIO_1_0_gir_SCANNERFLAGS =				\
+	--include-uninstalled=$(builddir)/Arrow-1.0.gir	\
+	--warn-all					\
+	--identifier-prefix=GArrowIO			\
+	--symbol-prefix=garrow_io
+INTROSPECTION_GIRS += ArrowIO-1.0.gir
+
+ArrowIPC-1.0.gir: libarrow-ipc-glib.la
+ArrowIPC-1.0.gir: Arrow-1.0.gir
+ArrowIPC-1.0.gir: ArrowIO-1.0.gir
+ArrowIPC_1_0_gir_PACKAGES =			\
+	gobject-2.0
+ArrowIPC_1_0_gir_EXPORT_PACKAGES = arrow-ipc
+ArrowIPC_1_0_gir_INCLUDES =			\
+	GObject-2.0
+ArrowIPC_1_0_gir_CFLAGS =			\
+	$(AM_CPPFLAGS)
+ArrowIPC_1_0_gir_LIBS =				\
+	libarrow-ipc-glib.la			\
+	libarrow-io-glib.la			\
+	libarrow-glib.la
+ArrowIPC_1_0_gir_FILES = $(libarrow_ipc_glib_la_sources)
+ArrowIPC_1_0_gir_SCANNERFLAGS =					\
+	--include-uninstalled=$(builddir)/Arrow-1.0.gir		\
+	--include-uninstalled=$(builddir)/ArrowIO-1.0.gir	\
+	--warn-all						\
+	--identifier-prefix=GArrowIPC				\
+	--symbol-prefix=garrow_ipc
+INTROSPECTION_GIRS += ArrowIPC-1.0.gir
+
+girdir = $(datadir)/gir-1.0
+gir_DATA = $(INTROSPECTION_GIRS)
+
+typelibdir = $(libdir)/girepository-1.0
+typelib_DATA = $(INTROSPECTION_GIRS:.gir=.typelib)
+
+CLEANFILES +=					\
+	$(gir_DATA)				\
+	$(typelib_DATA)
+endif
diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
new file mode 100644
index 0000000000000..0f038c8f66cee
--- /dev/null
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/binary-array-builder.h>
+#include <arrow-glib/boolean-array-builder.h>
+#include <arrow-glib/double-array-builder.h>
+#include <arrow-glib/float-array-builder.h>
+#include <arrow-glib/int8-array-builder.h>
+#include <arrow-glib/int16-array-builder.h>
+#include <arrow-glib/int32-array-builder.h>
+#include <arrow-glib/int64-array-builder.h>
+#include <arrow-glib/list-array-builder.h>
+#include <arrow-glib/string-array-builder.h>
+#include <arrow-glib/struct-array-builder.h>
+#include <arrow-glib/uint8-array-builder.h>
+#include <arrow-glib/uint16-array-builder.h>
+#include <arrow-glib/uint32-array-builder.h>
+#include <arrow-glib/uint64-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: array-builder
+ * @short_description: Base class for all array builder classes.
+ *
+ * #GArrowArrayBuilder is a base class for all array builder classes
+ * such as #GArrowBooleanArrayBuilder.
+ *
+ * You need to use array builder class to create a new array.
+ */
+
+typedef struct GArrowArrayBuilderPrivate_ {
+  std::shared_ptr<arrow::ArrayBuilder> array_builder;
+} GArrowArrayBuilderPrivate;
+
+enum {
+  PROP_0,
+  PROP_ARRAY_BUILDER
+};
+
+G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE(GArrowArrayBuilder,
+                                    garrow_array_builder,
+                                    G_TYPE_OBJECT)
+
+#define GARROW_ARRAY_BUILDER_GET_PRIVATE(obj)                           \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
+                               GARROW_TYPE_ARRAY_BUILDER,               \
+                               GArrowArrayBuilderPrivate))
+
+static void
+garrow_array_builder_finalize(GObject *object)
+{
+  GArrowArrayBuilderPrivate *priv;
+
+  priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(object);
+
+  priv->array_builder = nullptr;
+
+  G_OBJECT_CLASS(garrow_array_builder_parent_class)->finalize(object);
+}
+
+static void
+garrow_array_builder_set_property(GObject *object,
+                                  guint prop_id,
+                                  const GValue *value,
+                                  GParamSpec *pspec)
+{
+  GArrowArrayBuilderPrivate *priv;
+
+  priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_ARRAY_BUILDER:
+    priv->array_builder =
+      *static_cast<std::shared_ptr<arrow::ArrayBuilder> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_array_builder_get_property(GObject *object,
+                                  guint prop_id,
+                                  GValue *value,
+                                  GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_array_builder_init(GArrowArrayBuilder *builder)
+{
+}
+
+static void
+garrow_array_builder_class_init(GArrowArrayBuilderClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_array_builder_finalize;
+  gobject_class->set_property = garrow_array_builder_set_property;
+  gobject_class->get_property = garrow_array_builder_get_property;
+
+  spec = g_param_spec_pointer("array-builder",
+                              "Array builder",
+                              "The raw std::shared<arrow::ArrayBuilder> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_ARRAY_BUILDER, spec);
+}
+
+/**
+ * garrow_array_builder_finish:
+ * @builder: A #GArrowArrayBuilder.
+ *
+ * Returns: (transfer full): The built #GArrowArray.
+ */
+GArrowArray *
+garrow_array_builder_finish(GArrowArrayBuilder *builder)
+{
+  auto arrow_builder = garrow_array_builder_get_raw(builder);
+  std::shared_ptr<arrow::Array> arrow_array;
+  arrow_builder->Finish(&arrow_array);
+  return garrow_array_new_raw(&arrow_array);
+}
+
+G_END_DECLS
+
+GArrowArrayBuilder *
+garrow_array_builder_new_raw(std::shared_ptr<arrow::ArrayBuilder> *arrow_builder)
+{
+  GType type;
+
+  switch ((*arrow_builder)->type()->type) {
+  case arrow::Type::type::BOOL:
+    type = GARROW_TYPE_BOOLEAN_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::UINT8:
+    type = GARROW_TYPE_UINT8_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::INT8:
+    type = GARROW_TYPE_INT8_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::UINT16:
+    type = GARROW_TYPE_UINT16_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::INT16:
+    type = GARROW_TYPE_INT16_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::UINT32:
+    type = GARROW_TYPE_UINT32_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::INT32:
+    type = GARROW_TYPE_INT32_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::UINT64:
+    type = GARROW_TYPE_UINT64_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::INT64:
+    type = GARROW_TYPE_INT64_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::FLOAT:
+    type = GARROW_TYPE_FLOAT_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::DOUBLE:
+    type = GARROW_TYPE_DOUBLE_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::BINARY:
+    type = GARROW_TYPE_BINARY_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::STRING:
+    type = GARROW_TYPE_STRING_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::LIST:
+    type = GARROW_TYPE_LIST_ARRAY_BUILDER;
+    break;
+  case arrow::Type::type::STRUCT:
+    type = GARROW_TYPE_STRUCT_ARRAY_BUILDER;
+    break;
+  default:
+    type = GARROW_TYPE_ARRAY_BUILDER;
+    break;
+  }
+
+  auto builder =
+    GARROW_ARRAY_BUILDER(g_object_new(type,
+                                      "array-builder", arrow_builder,
+                                      NULL));
+  return builder;
+}
+
+std::shared_ptr<arrow::ArrayBuilder>
+garrow_array_builder_get_raw(GArrowArrayBuilder *builder)
+{
+  GArrowArrayBuilderPrivate *priv;
+
+  priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(builder);
+  return priv->array_builder;
+}
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
new file mode 100644
index 0000000000000..3717aef04a2f4
--- /dev/null
+++ b/c_glib/arrow-glib/array-builder.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_ARRAY_BUILDER               \
+  (garrow_array_builder_get_type())
+#define GARROW_ARRAY_BUILDER(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_ARRAY_BUILDER,        \
+                              GArrowArrayBuilder))
+#define GARROW_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_ARRAY_BUILDER,   \
+                           GArrowArrayBuilderClass))
+#define GARROW_IS_ARRAY_BUILDER(obj)            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),            \
+                              GARROW_TYPE_ARRAY_BUILDER))
+#define GARROW_IS_ARRAY_BUILDER_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_ARRAY_BUILDER))
+#define GARROW_ARRAY_BUILDER_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_ARRAY_BUILDER, \
+                             GArrowArrayBuilderClass))
+
+typedef struct _GArrowArrayBuilder         GArrowArrayBuilder;
+typedef struct _GArrowArrayBuilderClass    GArrowArrayBuilderClass;
+
+/**
+ * GArrowArrayBuilder:
+ *
+ * It wraps `arrow::ArrayBuilder`.
+ */
+struct _GArrowArrayBuilder
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowArrayBuilderClass
+{
+  GObjectClass parent_class;
+};
+
+GType               garrow_array_builder_get_type (void) G_GNUC_CONST;
+
+GArrowArray        *garrow_array_builder_finish   (GArrowArrayBuilder *builder);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/array-builder.hpp b/c_glib/arrow-glib/array-builder.hpp
new file mode 100644
index 0000000000000..becebb23f9bb0
--- /dev/null
+++ b/c_glib/arrow-glib/array-builder.hpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/array-builder.h>
+
+GArrowArrayBuilder *garrow_array_builder_new_raw(std::shared_ptr<arrow::ArrayBuilder> *arrow_builder);
+std::shared_ptr<arrow::ArrayBuilder> garrow_array_builder_get_raw(GArrowArrayBuilder *builder);
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
new file mode 100644
index 0000000000000..5dacb07ba8710
--- /dev/null
+++ b/c_glib/arrow-glib/array.cpp
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/binary-array.h>
+#include <arrow-glib/boolean-array.h>
+#include <arrow-glib/double-array.h>
+#include <arrow-glib/float-array.h>
+#include <arrow-glib/int8-array.h>
+#include <arrow-glib/int16-array.h>
+#include <arrow-glib/int32-array.h>
+#include <arrow-glib/int64-array.h>
+#include <arrow-glib/list-array.h>
+#include <arrow-glib/null-array.h>
+#include <arrow-glib/string-array.h>
+#include <arrow-glib/struct-array.h>
+#include <arrow-glib/uint8-array.h>
+#include <arrow-glib/uint16-array.h>
+#include <arrow-glib/uint32-array.h>
+#include <arrow-glib/uint64-array.h>
+
+#include <iostream>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: array
+ * @short_description: Base class for all array classes
+ *
+ * #GArrowArray is a base class for all array classes such as
+ * #GArrowBooleanArray.
+ *
+ * Array is immutable. You need to use array builder class such as
+ * #GArrowBooleanArrayBuilder to create a new array.
+ */
+
+typedef struct GArrowArrayPrivate_ {
+  std::shared_ptr<arrow::Array> array;
+} GArrowArrayPrivate;
+
+enum {
+  PROP_0,
+  PROP_ARRAY
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowArray, garrow_array, G_TYPE_OBJECT)
+
+#define GARROW_ARRAY_GET_PRIVATE(obj)                                   \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj), GARROW_TYPE_ARRAY, GArrowArrayPrivate))
+
+static void
+garrow_array_finalize(GObject *object)
+{
+  auto priv = GARROW_ARRAY_GET_PRIVATE(object);
+
+  priv->array = nullptr;
+
+  G_OBJECT_CLASS(garrow_array_parent_class)->finalize(object);
+}
+
+static void
+garrow_array_set_property(GObject *object,
+                          guint prop_id,
+                          const GValue *value,
+                          GParamSpec *pspec)
+{
+  auto priv = GARROW_ARRAY_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_ARRAY:
+    priv->array =
+      *static_cast<std::shared_ptr<arrow::Array> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_array_get_property(GObject *object,
+                          guint prop_id,
+                          GValue *value,
+                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_array_init(GArrowArray *object)
+{
+}
+
+static void
+garrow_array_class_init(GArrowArrayClass *klass)
+{
+  GParamSpec *spec;
+
+  auto gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_array_finalize;
+  gobject_class->set_property = garrow_array_set_property;
+  gobject_class->get_property = garrow_array_get_property;
+
+  spec = g_param_spec_pointer("array",
+                              "Array",
+                              "The raw std::shared<arrow::Array> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_ARRAY, spec);
+}
+
+/**
+ * garrow_array_get_length:
+ * @array: A #GArrowArray.
+ *
+ * Returns: The number of rows in the array.
+ */
+gint64
+garrow_array_get_length(GArrowArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  return arrow_array->length();
+}
+
+/**
+ * garrow_array_get_offset:
+ * @array: A #GArrowArray.
+ *
+ * Returns: The number of values in the array.
+ */
+gint64
+garrow_array_get_offset(GArrowArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  return arrow_array->offset();
+}
+
+/**
+ * garrow_array_get_n_nulls:
+ * @array: A #GArrowArray.
+ *
+ * Returns: The number of NULLs in the array.
+ */
+gint64
+garrow_array_get_n_nulls(GArrowArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  return arrow_array->null_count();
+}
+
+/**
+ * garrow_array_slice:
+ * @array: A #GArrowArray.
+ * @offset: The offset of sub #GArrowArray.
+ * @length: The length of sub #GArrowArray.
+ *
+ * Returns: (transfer full): The sub #GArrowArray. It covers only from
+ *   `offset` to `offset + length` range. The sub #GArrowArray shares
+ *   values with the base #GArrowArray.
+ */
+GArrowArray *
+garrow_array_slice(GArrowArray *array,
+                   gint64 offset,
+                   gint64 length)
+{
+  const auto arrow_array = garrow_array_get_raw(array);
+  auto arrow_sub_array = arrow_array->Slice(offset, length);
+  return garrow_array_new_raw(&arrow_sub_array);
+}
+
+G_END_DECLS
+
+GArrowArray *
+garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array)
+{
+  GType type;
+  GArrowArray *array;
+
+  switch ((*arrow_array)->type_enum()) {
+  case arrow::Type::type::NA:
+    type = GARROW_TYPE_NULL_ARRAY;
+    break;
+  case arrow::Type::type::BOOL:
+    type = GARROW_TYPE_BOOLEAN_ARRAY;
+    break;
+  case arrow::Type::type::UINT8:
+    type = GARROW_TYPE_UINT8_ARRAY;
+    break;
+  case arrow::Type::type::INT8:
+    type = GARROW_TYPE_INT8_ARRAY;
+    break;
+  case arrow::Type::type::UINT16:
+    type = GARROW_TYPE_UINT16_ARRAY;
+    break;
+  case arrow::Type::type::INT16:
+    type = GARROW_TYPE_INT16_ARRAY;
+    break;
+  case arrow::Type::type::UINT32:
+    type = GARROW_TYPE_UINT32_ARRAY;
+    break;
+  case arrow::Type::type::INT32:
+    type = GARROW_TYPE_INT32_ARRAY;
+    break;
+  case arrow::Type::type::UINT64:
+    type = GARROW_TYPE_UINT64_ARRAY;
+    break;
+  case arrow::Type::type::INT64:
+    type = GARROW_TYPE_INT64_ARRAY;
+    break;
+  case arrow::Type::type::FLOAT:
+    type = GARROW_TYPE_FLOAT_ARRAY;
+    break;
+  case arrow::Type::type::DOUBLE:
+    type = GARROW_TYPE_DOUBLE_ARRAY;
+    break;
+  case arrow::Type::type::BINARY:
+    type = GARROW_TYPE_BINARY_ARRAY;
+    break;
+  case arrow::Type::type::STRING:
+    type = GARROW_TYPE_STRING_ARRAY;
+    break;
+  case arrow::Type::type::LIST:
+    type = GARROW_TYPE_LIST_ARRAY;
+    break;
+  case arrow::Type::type::STRUCT:
+    type = GARROW_TYPE_STRUCT_ARRAY;
+    break;
+  default:
+    type = GARROW_TYPE_ARRAY;
+    break;
+  }
+  array = GARROW_ARRAY(g_object_new(type,
+                                    "array", arrow_array,
+                                    NULL));
+  return array;
+}
+
+std::shared_ptr<arrow::Array>
+garrow_array_get_raw(GArrowArray *array)
+{
+  auto priv = GARROW_ARRAY_GET_PRIVATE(array);
+  return priv->array;
+}
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
new file mode 100644
index 0000000000000..9b1fa7e1e4a31
--- /dev/null
+++ b/c_glib/arrow-glib/array.h
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_ARRAY \
+  (garrow_array_get_type())
+#define GARROW_ARRAY(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj), GARROW_TYPE_ARRAY, GArrowArray))
+#define GARROW_ARRAY_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST((klass), GARROW_TYPE_ARRAY, GArrowArrayClass))
+#define GARROW_IS_ARRAY(obj) \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_ARRAY))
+#define GARROW_IS_ARRAY_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_ARRAY))
+#define GARROW_ARRAY_GET_CLASS(obj) \
+  (G_TYPE_INSTANCE_GET_CLASS((obj), GARROW_TYPE_ARRAY, GArrowArrayClass))
+
+typedef struct _GArrowArray         GArrowArray;
+typedef struct _GArrowArrayClass    GArrowArrayClass;
+
+/**
+ * GArrowArray:
+ *
+ * It wraps `arrow::Array`.
+ */
+struct _GArrowArray
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowArrayClass
+{
+  GObjectClass parent_class;
+};
+
+GType          garrow_array_get_type    (void) G_GNUC_CONST;
+
+gint64         garrow_array_get_length  (GArrowArray *array);
+gint64         garrow_array_get_offset  (GArrowArray *array);
+gint64         garrow_array_get_n_nulls (GArrowArray *array);
+GArrowArray   *garrow_array_slice       (GArrowArray *array,
+                                         gint64 offset,
+                                         gint64 length);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/array.hpp b/c_glib/arrow-glib/array.hpp
new file mode 100644
index 0000000000000..d2dff22c48cf9
--- /dev/null
+++ b/c_glib/arrow-glib/array.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/array.h>
+
+GArrowArray *garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array);
+std::shared_ptr<arrow::Array> garrow_array_get_raw(GArrowArray *array);
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
new file mode 100644
index 0000000000000..4356234a4a63d
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+#include <arrow-glib/array-builder.h>
+#include <arrow-glib/binary-array.h>
+#include <arrow-glib/binary-array-builder.h>
+#include <arrow-glib/binary-data-type.h>
+#include <arrow-glib/boolean-array.h>
+#include <arrow-glib/boolean-array-builder.h>
+#include <arrow-glib/boolean-data-type.h>
+#include <arrow-glib/chunked-array.h>
+#include <arrow-glib/column.h>
+#include <arrow-glib/data-type.h>
+#include <arrow-glib/double-array.h>
+#include <arrow-glib/double-array-builder.h>
+#include <arrow-glib/double-data-type.h>
+#include <arrow-glib/enums.h>
+#include <arrow-glib/error.h>
+#include <arrow-glib/field.h>
+#include <arrow-glib/float-array.h>
+#include <arrow-glib/float-array-builder.h>
+#include <arrow-glib/float-data-type.h>
+#include <arrow-glib/int8-array.h>
+#include <arrow-glib/int8-array-builder.h>
+#include <arrow-glib/int8-data-type.h>
+#include <arrow-glib/int16-array.h>
+#include <arrow-glib/int16-array-builder.h>
+#include <arrow-glib/int16-data-type.h>
+#include <arrow-glib/int32-array.h>
+#include <arrow-glib/int32-array-builder.h>
+#include <arrow-glib/int32-data-type.h>
+#include <arrow-glib/int64-array.h>
+#include <arrow-glib/int64-array-builder.h>
+#include <arrow-glib/int64-data-type.h>
+#include <arrow-glib/list-array.h>
+#include <arrow-glib/list-array-builder.h>
+#include <arrow-glib/list-data-type.h>
+#include <arrow-glib/null-array.h>
+#include <arrow-glib/null-data-type.h>
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+#include <arrow-glib/string-array.h>
+#include <arrow-glib/string-array-builder.h>
+#include <arrow-glib/string-data-type.h>
+#include <arrow-glib/struct-array.h>
+#include <arrow-glib/struct-array-builder.h>
+#include <arrow-glib/struct-data-type.h>
+#include <arrow-glib/table.h>
+#include <arrow-glib/type.h>
+#include <arrow-glib/uint8-array.h>
+#include <arrow-glib/uint8-array-builder.h>
+#include <arrow-glib/uint8-data-type.h>
+#include <arrow-glib/uint16-array.h>
+#include <arrow-glib/uint16-array-builder.h>
+#include <arrow-glib/uint16-data-type.h>
+#include <arrow-glib/uint32-array.h>
+#include <arrow-glib/uint32-array-builder.h>
+#include <arrow-glib/uint32-data-type.h>
+#include <arrow-glib/uint64-array.h>
+#include <arrow-glib/uint64-array-builder.h>
+#include <arrow-glib/uint64-data-type.h>
diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
new file mode 100644
index 0000000000000..70fda8da7c526
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/arrow-glib.h>
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/boolean-array.hpp>
+#include <arrow-glib/boolean-data-type.hpp>
+#include <arrow-glib/chunked-array.hpp>
+#include <arrow-glib/column.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/double-array.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/field.hpp>
+#include <arrow-glib/float-array.hpp>
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.hpp>
+#include <arrow-glib/table.hpp>
+#include <arrow-glib/type.hpp>
diff --git a/c_glib/arrow-glib/arrow-glib.pc.in b/c_glib/arrow-glib/arrow-glib.pc.in
new file mode 100644
index 0000000000000..f9f27b2499057
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-glib.pc.in
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Apache Arrow GLib
+Description: C API for Apache Arrow based on GLib
+Version: @VERSION@
+Libs: -L${libdir} -larrow-glib
+Cflags: -I${includedir}
+Requires: gobject-2.0 arrow
diff --git a/c_glib/arrow-glib/arrow-io-glib.h b/c_glib/arrow-glib/arrow-io-glib.h
new file mode 100644
index 0000000000000..e02aa9b96982b
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-io-glib.h
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/io-enums.h>
+#include <arrow-glib/io-file.h>
+#include <arrow-glib/io-file-mode.h>
+#include <arrow-glib/io-file-output-stream.h>
+#include <arrow-glib/io-input-stream.h>
+#include <arrow-glib/io-memory-mapped-file.h>
+#include <arrow-glib/io-output-stream.h>
+#include <arrow-glib/io-readable.h>
+#include <arrow-glib/io-readable-file.h>
+#include <arrow-glib/io-writeable.h>
+#include <arrow-glib/io-writeable-file.h>
diff --git a/c_glib/arrow-glib/arrow-io-glib.hpp b/c_glib/arrow-glib/arrow-io-glib.hpp
new file mode 100644
index 0000000000000..378f20216b6a1
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-io-glib.hpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/io-file.hpp>
+#include <arrow-glib/io-file-mode.hpp>
+#include <arrow-glib/io-file-output-stream.hpp>
+#include <arrow-glib/io-input-stream.hpp>
+#include <arrow-glib/io-memory-mapped-file.hpp>
+#include <arrow-glib/io-output-stream.hpp>
+#include <arrow-glib/io-readable.hpp>
+#include <arrow-glib/io-readable-file.hpp>
+#include <arrow-glib/io-writeable.hpp>
diff --git a/c_glib/arrow-glib/arrow-io-glib.pc.in b/c_glib/arrow-glib/arrow-io-glib.pc.in
new file mode 100644
index 0000000000000..4256184cf7348
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-io-glib.pc.in
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Apache Arrow I/O GLib
+Description: C API for Apache Arrow I/O based on GLib
+Version: @VERSION@
+Libs: -L${libdir} -larrow-glib-io
+Cflags: -I${includedir}
+Requires: arrow-glib arrow-io
diff --git a/c_glib/arrow-glib/arrow-ipc-glib.h b/c_glib/arrow-glib/arrow-ipc-glib.h
new file mode 100644
index 0000000000000..4954d83cd0728
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-ipc-glib.h
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/ipc-enums.h>
+#include <arrow-glib/ipc-file-reader.h>
+#include <arrow-glib/ipc-file-writer.h>
+#include <arrow-glib/ipc-metadata-version.h>
+#include <arrow-glib/ipc-stream-reader.h>
+#include <arrow-glib/ipc-stream-writer.h>
diff --git a/c_glib/arrow-glib/arrow-ipc-glib.hpp b/c_glib/arrow-glib/arrow-ipc-glib.hpp
new file mode 100644
index 0000000000000..d32bc052b98e5
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-ipc-glib.hpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/arrow-glib.hpp>
+
+#include <arrow-glib/arrow-io-glib.hpp>
+
+#include <arrow-glib/ipc-file-reader.hpp>
+#include <arrow-glib/ipc-file-writer.hpp>
+#include <arrow-glib/ipc-metadata-version.hpp>
+#include <arrow-glib/ipc-stream-reader.hpp>
+#include <arrow-glib/ipc-stream-writer.hpp>
diff --git a/c_glib/arrow-glib/arrow-ipc-glib.pc.in b/c_glib/arrow-glib/arrow-ipc-glib.pc.in
new file mode 100644
index 0000000000000..0b04c4a808ff1
--- /dev/null
+++ b/c_glib/arrow-glib/arrow-ipc-glib.pc.in
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Apache Arrow IPC GLib
+Description: C API for Apache Arrow IPC based on GLib
+Version: @VERSION@
+Libs: -L${libdir} -larrow-glib-ipc
+Cflags: -I${includedir}
+Requires: arrow-glib-io arrow-ipc
diff --git a/c_glib/arrow-glib/binary-array-builder.cpp b/c_glib/arrow-glib/binary-array-builder.cpp
new file mode 100644
index 0000000000000..ab11535eb8595
--- /dev/null
+++ b/c_glib/arrow-glib/binary-array-builder.cpp
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/binary-array-builder.h>
+#include <arrow-glib/error.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: binary-array-builder
+ * @short_description: Binary array builder class
+ *
+ * #GArrowBinaryArrayBuilder is the class to create a new
+ * #GArrowBinaryArray.
+ */
+
+G_DEFINE_TYPE(GArrowBinaryArrayBuilder,
+              garrow_binary_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_binary_array_builder_init(GArrowBinaryArrayBuilder *builder)
+{
+}
+
+static void
+garrow_binary_array_builder_class_init(GArrowBinaryArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_binary_array_builder_new:
+ *
+ * Returns: A newly created #GArrowBinaryArrayBuilder.
+ */
+GArrowBinaryArrayBuilder *
+garrow_binary_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::BinaryBuilder>(memory_pool, arrow::binary());
+  auto builder =
+    GARROW_BINARY_ARRAY_BUILDER(g_object_new(GARROW_TYPE_BINARY_ARRAY_BUILDER,
+                                             "array-builder", &arrow_builder,
+                                             NULL));
+  return builder;
+}
+
+/**
+ * garrow_binary_array_builder_append:
+ * @builder: A #GArrowBinaryArrayBuilder.
+ * @value: (array length=length): A binary value.
+ * @length: A value length.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_binary_array_builder_append(GArrowBinaryArrayBuilder *builder,
+                                   const guint8 *value,
+                                   gint32 length,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::BinaryBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value, length);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[binary-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_binary_array_builder_append_null:
+ * @builder: A #GArrowBinaryArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
+                                        GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::BinaryBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[binary-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/binary-array-builder.h b/c_glib/arrow-glib/binary-array-builder.h
new file mode 100644
index 0000000000000..111a83a3a09b0
--- /dev/null
+++ b/c_glib/arrow-glib/binary-array-builder.h
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_BINARY_ARRAY_BUILDER        \
+  (garrow_binary_array_builder_get_type())
+#define GARROW_BINARY_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BINARY_ARRAY_BUILDER, \
+                              GArrowBinaryArrayBuilder))
+#define GARROW_BINARY_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BINARY_ARRAY_BUILDER,    \
+                           GArrowBinaryArrayBuilderClass))
+#define GARROW_IS_BINARY_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_BINARY_ARRAY_BUILDER))
+#define GARROW_IS_BINARY_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BINARY_ARRAY_BUILDER))
+#define GARROW_BINARY_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BINARY_ARRAY_BUILDER,  \
+                             GArrowBinaryArrayBuilderClass))
+
+typedef struct _GArrowBinaryArrayBuilder         GArrowBinaryArrayBuilder;
+typedef struct _GArrowBinaryArrayBuilderClass    GArrowBinaryArrayBuilderClass;
+
+/**
+ * GArrowBinaryArrayBuilder:
+ *
+ * It wraps `arrow::BinaryBuilder`.
+ */
+struct _GArrowBinaryArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowBinaryArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_binary_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowBinaryArrayBuilder *garrow_binary_array_builder_new(void);
+
+gboolean garrow_binary_array_builder_append(GArrowBinaryArrayBuilder *builder,
+                                            const guint8 *value,
+                                            gint32 length,
+                                            GError **error);
+gboolean garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
+                                                 GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/binary-array.cpp b/c_glib/arrow-glib/binary-array.cpp
new file mode 100644
index 0000000000000..c149d14025ae7
--- /dev/null
+++ b/c_glib/arrow-glib/binary-array.cpp
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/binary-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: binary-array
+ * @short_description: Binary array class
+ *
+ * #GArrowBinaryArray is a class for binary array. It can store zero
+ * or more binary data.
+ *
+ * #GArrowBinaryArray is immutable. You need to use
+ * #GArrowBinaryArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowBinaryArray,               \
+              garrow_binary_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_binary_array_init(GArrowBinaryArray *object)
+{
+}
+
+static void
+garrow_binary_array_class_init(GArrowBinaryArrayClass *klass)
+{
+}
+
+/**
+ * garrow_binary_array_get_value:
+ * @array: A #GArrowBinaryArray.
+ * @i: The index of the target value.
+ * @length: (out): The length of the value.
+ *
+ * Returns: (array length=length): The i-th value.
+ */
+const guint8 *
+garrow_binary_array_get_value(GArrowBinaryArray *array,
+                              gint64 i,
+                              gint32 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_binary_array =
+    static_cast<arrow::BinaryArray *>(arrow_array.get());
+  return arrow_binary_array->GetValue(i, length);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/binary-array.h b/c_glib/arrow-glib/binary-array.h
new file mode 100644
index 0000000000000..ab63ece9844f8
--- /dev/null
+++ b/c_glib/arrow-glib/binary-array.h
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_BINARY_ARRAY                \
+  (garrow_binary_array_get_type())
+#define GARROW_BINARY_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_BINARY_ARRAY, \
+                              GArrowBinaryArray))
+#define GARROW_BINARY_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_BINARY_ARRAY,    \
+                           GArrowBinaryArrayClass))
+#define GARROW_IS_BINARY_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BINARY_ARRAY))
+#define GARROW_IS_BINARY_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_BINARY_ARRAY))
+#define GARROW_BINARY_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_BINARY_ARRAY,  \
+                             GArrowBinaryArrayClass))
+
+typedef struct _GArrowBinaryArray         GArrowBinaryArray;
+typedef struct _GArrowBinaryArrayClass    GArrowBinaryArrayClass;
+
+/**
+ * GArrowBinaryArray:
+ *
+ * It wraps `arrow::BinaryArray`.
+ */
+struct _GArrowBinaryArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowBinaryArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_binary_array_get_type(void) G_GNUC_CONST;
+
+const guint8 *garrow_binary_array_get_value(GArrowBinaryArray *array,
+                                            gint64 i,
+                                            gint32 *length);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/binary-data-type.cpp b/c_glib/arrow-glib/binary-data-type.cpp
new file mode 100644
index 0000000000000..e5187f7d94efe
--- /dev/null
+++ b/c_glib/arrow-glib/binary-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/binary-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: binary-data-type
+ * @short_description: Binary data type
+ *
+ * #GArrowBinaryDataType is a class for binary data type.
+ */
+
+G_DEFINE_TYPE(GArrowBinaryDataType,                \
+              garrow_binary_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_binary_data_type_init(GArrowBinaryDataType *object)
+{
+}
+
+static void
+garrow_binary_data_type_class_init(GArrowBinaryDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_binary_data_type_new:
+ *
+ * Returns: The newly created binary data type.
+ */
+GArrowBinaryDataType *
+garrow_binary_data_type_new(void)
+{
+  auto arrow_data_type = arrow::binary();
+
+  GArrowBinaryDataType *data_type =
+    GARROW_BINARY_DATA_TYPE(g_object_new(GARROW_TYPE_BINARY_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/binary-data-type.h b/c_glib/arrow-glib/binary-data-type.h
new file mode 100644
index 0000000000000..9654fe216376e
--- /dev/null
+++ b/c_glib/arrow-glib/binary-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_BINARY_DATA_TYPE            \
+  (garrow_binary_data_type_get_type())
+#define GARROW_BINARY_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BINARY_DATA_TYPE,     \
+                              GArrowBinaryDataType))
+#define GARROW_BINARY_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BINARY_DATA_TYPE,        \
+                           GArrowBinaryDataTypeClass))
+#define GARROW_IS_BINARY_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BINARY_DATA_TYPE))
+#define GARROW_IS_BINARY_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BINARY_DATA_TYPE))
+#define GARROW_BINARY_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BINARY_DATA_TYPE,      \
+                             GArrowBinaryDataTypeClass))
+
+typedef struct _GArrowBinaryDataType         GArrowBinaryDataType;
+typedef struct _GArrowBinaryDataTypeClass    GArrowBinaryDataTypeClass;
+
+/**
+ * GArrowBinaryDataType:
+ *
+ * It wraps `arrow::BinaryType`.
+ */
+struct _GArrowBinaryDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowBinaryDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_binary_data_type_get_type (void) G_GNUC_CONST;
+GArrowBinaryDataType *garrow_binary_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-array-builder.cpp b/c_glib/arrow-glib/boolean-array-builder.cpp
new file mode 100644
index 0000000000000..1a4c1f9fd8f7e
--- /dev/null
+++ b/c_glib/arrow-glib/boolean-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/boolean-array-builder.h>
+#include <arrow-glib/error.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: boolean-array-builder
+ * @short_description: Boolean array builder class
+ *
+ * #GArrowBooleanArrayBuilder is the class to create a new
+ * #GArrowBooleanArray.
+ */
+
+G_DEFINE_TYPE(GArrowBooleanArrayBuilder,
+              garrow_boolean_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_boolean_array_builder_init(GArrowBooleanArrayBuilder *builder)
+{
+}
+
+static void
+garrow_boolean_array_builder_class_init(GArrowBooleanArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_boolean_array_builder_new:
+ *
+ * Returns: A newly created #GArrowBooleanArrayBuilder.
+ */
+GArrowBooleanArrayBuilder *
+garrow_boolean_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::BooleanBuilder>(memory_pool);
+  auto builder =
+    GARROW_BOOLEAN_ARRAY_BUILDER(g_object_new(GARROW_TYPE_BOOLEAN_ARRAY_BUILDER,
+                                              "array-builder", &arrow_builder,
+                                              NULL));
+  return builder;
+}
+
+/**
+ * garrow_boolean_array_builder_append:
+ * @builder: A #GArrowBooleanArrayBuilder.
+ * @value: A boolean value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
+                                    gboolean value,
+                                    GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::BooleanBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[boolean-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_boolean_array_builder_append_null:
+ * @builder: A #GArrowBooleanArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
+                                         GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::BooleanBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[boolean-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-array-builder.h b/c_glib/arrow-glib/boolean-array-builder.h
new file mode 100644
index 0000000000000..ca50e9797d41c
--- /dev/null
+++ b/c_glib/arrow-glib/boolean-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_BOOLEAN_ARRAY_BUILDER \
+  (garrow_boolean_array_builder_get_type())
+#define GARROW_BOOLEAN_ARRAY_BUILDER(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_BOOLEAN_ARRAY_BUILDER,        \
+                              GArrowBooleanArrayBuilder))
+#define GARROW_BOOLEAN_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BOOLEAN_ARRAY_BUILDER,   \
+                           GArrowBooleanArrayBuilderClass))
+#define GARROW_IS_BOOLEAN_ARRAY_BUILDER(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_BOOLEAN_ARRAY_BUILDER))
+#define GARROW_IS_BOOLEAN_ARRAY_BUILDER_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BOOLEAN_ARRAY_BUILDER))
+#define GARROW_BOOLEAN_ARRAY_BUILDER_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BOOLEAN_ARRAY_BUILDER, \
+                             GArrowBooleanArrayBuilderClass))
+
+typedef struct _GArrowBooleanArrayBuilder         GArrowBooleanArrayBuilder;
+typedef struct _GArrowBooleanArrayBuilderClass    GArrowBooleanArrayBuilderClass;
+
+/**
+ * GArrowBooleanArrayBuilder:
+ *
+ * It wraps `arrow::BooleanBuilder`.
+ */
+struct _GArrowBooleanArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowBooleanArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_boolean_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowBooleanArrayBuilder *garrow_boolean_array_builder_new(void);
+
+gboolean garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
+                                             gboolean value,
+                                             GError **error);
+gboolean garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
+                                                  GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-array.cpp b/c_glib/arrow-glib/boolean-array.cpp
new file mode 100644
index 0000000000000..62fc40fd54112
--- /dev/null
+++ b/c_glib/arrow-glib/boolean-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/boolean-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: boolean-array
+ * @short_description: Boolean array class
+ *
+ * #GArrowBooleanArray is a class for binary array. It can store zero
+ * or more boolean data.
+ *
+ * #GArrowBooleanArray is immutable. You need to use
+ * #GArrowBooleanArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowBooleanArray,               \
+              garrow_boolean_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_boolean_array_init(GArrowBooleanArray *object)
+{
+}
+
+static void
+garrow_boolean_array_class_init(GArrowBooleanArrayClass *klass)
+{
+}
+
+/**
+ * garrow_boolean_array_get_value:
+ * @array: A #GArrowBooleanArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gboolean
+garrow_boolean_array_get_value(GArrowBooleanArray *array,
+                               gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::BooleanArray *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-array.h b/c_glib/arrow-glib/boolean-array.h
new file mode 100644
index 0000000000000..9899fdf0ceca8
--- /dev/null
+++ b/c_glib/arrow-glib/boolean-array.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_BOOLEAN_ARRAY               \
+  (garrow_boolean_array_get_type())
+#define GARROW_BOOLEAN_ARRAY(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BOOLEAN_ARRAY,        \
+                              GArrowBooleanArray))
+#define GARROW_BOOLEAN_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_BOOLEAN_ARRAY,   \
+                           GArrowBooleanArrayClass))
+#define GARROW_IS_BOOLEAN_ARRAY(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BOOLEAN_ARRAY))
+#define GARROW_IS_BOOLEAN_ARRAY_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_BOOLEAN_ARRAY))
+#define GARROW_BOOLEAN_ARRAY_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_BOOLEAN_ARRAY, \
+                             GArrowBooleanArrayClass))
+
+typedef struct _GArrowBooleanArray         GArrowBooleanArray;
+typedef struct _GArrowBooleanArrayClass    GArrowBooleanArrayClass;
+
+/**
+ * GArrowBooleanArray:
+ *
+ * It wraps `arrow::BooleanArray`.
+ */
+struct _GArrowBooleanArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowBooleanArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType          garrow_boolean_array_get_type  (void) G_GNUC_CONST;
+gboolean       garrow_boolean_array_get_value (GArrowBooleanArray *array,
+                                               gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-data-type.cpp b/c_glib/arrow-glib/boolean-data-type.cpp
new file mode 100644
index 0000000000000..99c73d9ff8873
--- /dev/null
+++ b/c_glib/arrow-glib/boolean-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/boolean-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: boolean-data-type
+ * @short_description: Boolean data type
+ *
+ * #GArrowBooleanDataType is a class for boolean data type.
+ */
+
+G_DEFINE_TYPE(GArrowBooleanDataType,                \
+              garrow_boolean_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_boolean_data_type_init(GArrowBooleanDataType *object)
+{
+}
+
+static void
+garrow_boolean_data_type_class_init(GArrowBooleanDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_boolean_data_type_new:
+ *
+ * Returns: The newly created boolean data type.
+ */
+GArrowBooleanDataType *
+garrow_boolean_data_type_new(void)
+{
+  auto arrow_data_type = arrow::boolean();
+
+  GArrowBooleanDataType *data_type =
+    GARROW_BOOLEAN_DATA_TYPE(g_object_new(GARROW_TYPE_BOOLEAN_DATA_TYPE,
+                                          "data-type", &arrow_data_type,
+                                          NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-data-type.h b/c_glib/arrow-glib/boolean-data-type.h
new file mode 100644
index 0000000000000..ad30c99960a8e
--- /dev/null
+++ b/c_glib/arrow-glib/boolean-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_BOOLEAN_DATA_TYPE           \
+  (garrow_boolean_data_type_get_type())
+#define GARROW_BOOLEAN_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BOOLEAN_DATA_TYPE,    \
+                              GArrowBooleanDataType))
+#define GARROW_BOOLEAN_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BOOLEAN_DATA_TYPE,       \
+                           GArrowBooleanDataTypeClass))
+#define GARROW_IS_BOOLEAN_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BOOLEAN_DATA_TYPE))
+#define GARROW_IS_BOOLEAN_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BOOLEAN_DATA_TYPE))
+#define GARROW_BOOLEAN_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BOOLEAN_DATA_TYPE,     \
+                             GArrowBooleanDataTypeClass))
+
+typedef struct _GArrowBooleanDataType         GArrowBooleanDataType;
+typedef struct _GArrowBooleanDataTypeClass    GArrowBooleanDataTypeClass;
+
+/**
+ * GArrowBooleanDataType:
+ *
+ * It wraps `arrow::BooleanType`.
+ */
+struct _GArrowBooleanDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowBooleanDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                  garrow_boolean_data_type_get_type (void) G_GNUC_CONST;
+GArrowBooleanDataType *garrow_boolean_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/chunked-array.cpp b/c_glib/arrow-glib/chunked-array.cpp
new file mode 100644
index 0000000000000..e732ece73c7f9
--- /dev/null
+++ b/c_glib/arrow-glib/chunked-array.cpp
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/chunked-array.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: chunked-array
+ * @short_description: Chunked array class
+ *
+ * #GArrowChunkedArray is a class for chunked array. Chunked array
+ * makes a list of #GArrowArrays one logical large array.
+ */
+
+typedef struct GArrowChunkedArrayPrivate_ {
+  std::shared_ptr<arrow::ChunkedArray> chunked_array;
+} GArrowChunkedArrayPrivate;
+
+enum {
+  PROP_0,
+  PROP_CHUNKED_ARRAY
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowChunkedArray,
+                           garrow_chunked_array,
+                           G_TYPE_OBJECT)
+
+#define GARROW_CHUNKED_ARRAY_GET_PRIVATE(obj)                   \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_TYPE_CHUNKED_ARRAY,       \
+                               GArrowChunkedArrayPrivate))
+
+static void
+garrow_chunked_array_finalize(GObject *object)
+{
+  GArrowChunkedArrayPrivate *priv;
+
+  priv = GARROW_CHUNKED_ARRAY_GET_PRIVATE(object);
+
+  priv->chunked_array = nullptr;
+
+  G_OBJECT_CLASS(garrow_chunked_array_parent_class)->finalize(object);
+}
+
+static void
+garrow_chunked_array_set_property(GObject *object,
+                                  guint prop_id,
+                                  const GValue *value,
+                                  GParamSpec *pspec)
+{
+  GArrowChunkedArrayPrivate *priv;
+
+  priv = GARROW_CHUNKED_ARRAY_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_CHUNKED_ARRAY:
+    priv->chunked_array =
+      *static_cast<std::shared_ptr<arrow::ChunkedArray> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_chunked_array_get_property(GObject *object,
+                                  guint prop_id,
+                                  GValue *value,
+                                  GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_chunked_array_init(GArrowChunkedArray *object)
+{
+}
+
+static void
+garrow_chunked_array_class_init(GArrowChunkedArrayClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_chunked_array_finalize;
+  gobject_class->set_property = garrow_chunked_array_set_property;
+  gobject_class->get_property = garrow_chunked_array_get_property;
+
+  spec = g_param_spec_pointer("chunked-array",
+                              "Chunked array",
+                              "The raw std::shared<arrow::ChunkedArray> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_CHUNKED_ARRAY, spec);
+}
+
+/**
+ * garrow_chunked_array_new:
+ * @chunks: (element-type GArrowArray): The array chunks.
+ *
+ * Returns: A newly created #GArrowChunkedArray.
+ */
+GArrowChunkedArray *
+garrow_chunked_array_new(GList *chunks)
+{
+  std::vector<std::shared_ptr<arrow::Array>> arrow_chunks;
+  for (GList *node = chunks; node; node = node->next) {
+    GArrowArray *chunk = GARROW_ARRAY(node->data);
+    arrow_chunks.push_back(garrow_array_get_raw(chunk));
+  }
+
+  auto arrow_chunked_array =
+    std::make_shared<arrow::ChunkedArray>(arrow_chunks);
+  return garrow_chunked_array_new_raw(&arrow_chunked_array);
+}
+
+/**
+ * garrow_chunked_array_get_length:
+ * @chunked_array: A #GArrowChunkedArray.
+ *
+ * Returns: The total number of rows in the chunked array.
+ */
+guint64
+garrow_chunked_array_get_length(GArrowChunkedArray *chunked_array)
+{
+  const auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+  return arrow_chunked_array->length();
+}
+
+/**
+ * garrow_chunked_array_get_n_nulls:
+ * @chunked_array: A #GArrowChunkedArray.
+ *
+ * Returns: The total number of NULL in the chunked array.
+ */
+guint64
+garrow_chunked_array_get_n_nulls(GArrowChunkedArray *chunked_array)
+{
+  const auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+  return arrow_chunked_array->null_count();
+}
+
+/**
+ * garrow_chunked_array_get_n_chunks:
+ * @chunked_array: A #GArrowChunkedArray.
+ *
+ * Returns: The total number of chunks in the chunked array.
+ */
+guint
+garrow_chunked_array_get_n_chunks(GArrowChunkedArray *chunked_array)
+{
+  const auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+  return arrow_chunked_array->num_chunks();
+}
+
+/**
+ * garrow_chunked_array_get_chunk:
+ * @chunked_array: A #GArrowChunkedArray.
+ * @i: The index of the target chunk.
+ *
+ * Returns: (transfer full): The i-th chunk of the chunked array.
+ */
+GArrowArray *
+garrow_chunked_array_get_chunk(GArrowChunkedArray *chunked_array,
+                               guint i)
+{
+  const auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+  auto arrow_chunk = arrow_chunked_array->chunk(i);
+  return garrow_array_new_raw(&arrow_chunk);
+}
+
+/**
+ * garrow_chunked_array_get_chunks:
+ * @chunked_array: A #GArrowChunkedArray.
+ *
+ * Returns: (element-type GArrowArray) (transfer full):
+ *   The chunks in the chunked array.
+ */
+GList *
+garrow_chunked_array_get_chunks(GArrowChunkedArray *chunked_array)
+{
+  const auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+
+  GList *chunks = NULL;
+  for (auto arrow_chunk : arrow_chunked_array->chunks()) {
+    GArrowArray *chunk = garrow_array_new_raw(&arrow_chunk);
+    chunks = g_list_prepend(chunks, chunk);
+  }
+
+  return g_list_reverse(chunks);
+}
+
+G_END_DECLS
+
+GArrowChunkedArray *
+garrow_chunked_array_new_raw(std::shared_ptr<arrow::ChunkedArray> *arrow_chunked_array)
+{
+  auto chunked_array =
+    GARROW_CHUNKED_ARRAY(g_object_new(GARROW_TYPE_CHUNKED_ARRAY,
+                                      "chunked-array", arrow_chunked_array,
+                                      NULL));
+  return chunked_array;
+}
+
+std::shared_ptr<arrow::ChunkedArray>
+garrow_chunked_array_get_raw(GArrowChunkedArray *chunked_array)
+{
+  GArrowChunkedArrayPrivate *priv;
+
+  priv = GARROW_CHUNKED_ARRAY_GET_PRIVATE(chunked_array);
+  return priv->chunked_array;
+}
diff --git a/c_glib/arrow-glib/chunked-array.h b/c_glib/arrow-glib/chunked-array.h
new file mode 100644
index 0000000000000..338930b9bd84a
--- /dev/null
+++ b/c_glib/arrow-glib/chunked-array.h
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_CHUNKED_ARRAY               \
+  (garrow_chunked_array_get_type())
+#define GARROW_CHUNKED_ARRAY(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_CHUNKED_ARRAY,        \
+                              GArrowChunkedArray))
+#define GARROW_CHUNKED_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_CHUNKED_ARRAY,   \
+                           GArrowChunkedArrayClass))
+#define GARROW_IS_CHUNKED_ARRAY(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_CHUNKED_ARRAY))
+#define GARROW_IS_CHUNKED_ARRAY_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_CHUNKED_ARRAY))
+#define GARROW_CHUNKED_ARRAY_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_CHUNKED_ARRAY, \
+                             GArrowChunkedArrayClass))
+
+typedef struct _GArrowChunkedArray         GArrowChunkedArray;
+typedef struct _GArrowChunkedArrayClass    GArrowChunkedArrayClass;
+
+/**
+ * GArrowChunkedArray:
+ *
+ * It wraps `arrow::ChunkedArray`.
+ */
+struct _GArrowChunkedArray
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowChunkedArrayClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_chunked_array_get_type(void) G_GNUC_CONST;
+
+GArrowChunkedArray *garrow_chunked_array_new(GList *chunks);
+
+guint64 garrow_chunked_array_get_length (GArrowChunkedArray *chunked_array);
+guint64 garrow_chunked_array_get_n_nulls(GArrowChunkedArray *chunked_array);
+guint   garrow_chunked_array_get_n_chunks (GArrowChunkedArray *chunked_array);
+
+GArrowArray *garrow_chunked_array_get_chunk(GArrowChunkedArray *chunked_array,
+                                            guint i);
+GList *garrow_chunked_array_get_chunks(GArrowChunkedArray *chunked_array);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/chunked-array.hpp b/c_glib/arrow-glib/chunked-array.hpp
new file mode 100644
index 0000000000000..ec5068adc0741
--- /dev/null
+++ b/c_glib/arrow-glib/chunked-array.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/chunked-array.h>
+
+GArrowChunkedArray *garrow_chunked_array_new_raw(std::shared_ptr<arrow::ChunkedArray> *arrow_chunked_array);
+std::shared_ptr<arrow::ChunkedArray> garrow_chunked_array_get_raw(GArrowChunkedArray *chunked_array);
diff --git a/c_glib/arrow-glib/column.cpp b/c_glib/arrow-glib/column.cpp
new file mode 100644
index 0000000000000..94df640d6b2b5
--- /dev/null
+++ b/c_glib/arrow-glib/column.cpp
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/chunked-array.hpp>
+#include <arrow-glib/column.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/field.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: column
+ * @short_description: Column class
+ *
+ * #GArrowColumn is a class for column. Column has a #GArrowField and
+ * zero or more values. Values are #GArrowChunkedArray.
+ */
+
+typedef struct GArrowColumnPrivate_ {
+  std::shared_ptr<arrow::Column> column;
+} GArrowColumnPrivate;
+
+enum {
+  PROP_0,
+  PROP_COLUMN
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowColumn,
+                           garrow_column,
+                           G_TYPE_OBJECT)
+
+#define GARROW_COLUMN_GET_PRIVATE(obj)                  \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                   \
+                               GARROW_TYPE_COLUMN,      \
+                               GArrowColumnPrivate))
+
+static void
+garrow_column_dispose(GObject *object)
+{
+  GArrowColumnPrivate *priv;
+
+  priv = GARROW_COLUMN_GET_PRIVATE(object);
+
+  priv->column = nullptr;
+
+  G_OBJECT_CLASS(garrow_column_parent_class)->dispose(object);
+}
+
+static void
+garrow_column_set_property(GObject *object,
+                           guint prop_id,
+                           const GValue *value,
+                           GParamSpec *pspec)
+{
+  GArrowColumnPrivate *priv;
+
+  priv = GARROW_COLUMN_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_COLUMN:
+    priv->column =
+      *static_cast<std::shared_ptr<arrow::Column> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_column_get_property(GObject *object,
+                           guint prop_id,
+                           GValue *value,
+                           GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_column_init(GArrowColumn *object)
+{
+}
+
+static void
+garrow_column_class_init(GArrowColumnClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->dispose      = garrow_column_dispose;
+  gobject_class->set_property = garrow_column_set_property;
+  gobject_class->get_property = garrow_column_get_property;
+
+  spec = g_param_spec_pointer("column",
+                              "Column",
+                              "The raw std::shared<arrow::Column> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_COLUMN, spec);
+}
+
+/**
+ * garrow_column_new_array:
+ * @field: The metadata of the column.
+ * @array: The data of the column.
+ *
+ * Returns: A newly created #GArrowColumn.
+ */
+GArrowColumn *
+garrow_column_new_array(GArrowField *field,
+                        GArrowArray *array)
+{
+  auto arrow_column =
+    std::make_shared<arrow::Column>(garrow_field_get_raw(field),
+                                    garrow_array_get_raw(array));
+  return garrow_column_new_raw(&arrow_column);
+}
+
+/**
+ * garrow_column_new_chunked_array:
+ * @field: The metadata of the column.
+ * @chunked_array: The data of the column.
+ *
+ * Returns: A newly created #GArrowColumn.
+ */
+GArrowColumn *
+garrow_column_new_chunked_array(GArrowField *field,
+                                GArrowChunkedArray *chunked_array)
+{
+  auto arrow_column =
+    std::make_shared<arrow::Column>(garrow_field_get_raw(field),
+                                    garrow_chunked_array_get_raw(chunked_array));
+  return garrow_column_new_raw(&arrow_column);
+}
+
+/**
+ * garrow_column_get_length:
+ * @column: A #GArrowColumn.
+ *
+ * Returns: The number of data of the column.
+ */
+guint64
+garrow_column_get_length(GArrowColumn *column)
+{
+  const auto arrow_column = garrow_column_get_raw(column);
+  return arrow_column->length();
+}
+
+/**
+ * garrow_column_get_n_nulls:
+ * @column: A #GArrowColumn.
+ *
+ * Returns: The number of nulls of the column.
+ */
+guint64
+garrow_column_get_n_nulls(GArrowColumn *column)
+{
+  const auto arrow_column = garrow_column_get_raw(column);
+  return arrow_column->null_count();
+}
+
+/**
+ * garrow_column_get_field:
+ * @column: A #GArrowColumn.
+ *
+ * Returns: (transfer full): The metadata of the column.
+ */
+GArrowField *
+garrow_column_get_field(GArrowColumn *column)
+{
+  const auto arrow_column = garrow_column_get_raw(column);
+  auto arrow_field = arrow_column->field();
+  return garrow_field_new_raw(&arrow_field);
+}
+
+/**
+ * garrow_column_get_name:
+ * @column: A #GArrowColumn.
+ *
+ * Returns: The name of the column.
+ */
+const gchar *
+garrow_column_get_name(GArrowColumn *column)
+{
+  const auto arrow_column = garrow_column_get_raw(column);
+  return arrow_column->name().c_str();
+}
+
+/**
+ * garrow_column_get_data_type:
+ * @column: A #GArrowColumn.
+ *
+ * Returns: (transfer full): The data type of the column.
+ */
+GArrowDataType *
+garrow_column_get_data_type(GArrowColumn *column)
+{
+  const auto arrow_column = garrow_column_get_raw(column);
+  auto arrow_data_type = arrow_column->type();
+  return garrow_data_type_new_raw(&arrow_data_type);
+}
+
+/**
+ * garrow_column_get_data:
+ * @column: A #GArrowColumn.
+ *
+ * Returns: (transfer full): The data of the column.
+ */
+GArrowChunkedArray *
+garrow_column_get_data(GArrowColumn *column)
+{
+  const auto arrow_column = garrow_column_get_raw(column);
+  auto arrow_chunked_array = arrow_column->data();
+  return garrow_chunked_array_new_raw(&arrow_chunked_array);
+}
+
+G_END_DECLS
+
+GArrowColumn *
+garrow_column_new_raw(std::shared_ptr<arrow::Column> *arrow_column)
+{
+  auto column = GARROW_COLUMN(g_object_new(GARROW_TYPE_COLUMN,
+                                           "column", arrow_column,
+                                           NULL));
+  return column;
+}
+
+std::shared_ptr<arrow::Column>
+garrow_column_get_raw(GArrowColumn *column)
+{
+  GArrowColumnPrivate *priv;
+
+  priv = GARROW_COLUMN_GET_PRIVATE(column);
+  return priv->column;
+}
diff --git a/c_glib/arrow-glib/column.h b/c_glib/arrow-glib/column.h
new file mode 100644
index 0000000000000..fba3c26b2f08f
--- /dev/null
+++ b/c_glib/arrow-glib/column.h
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+#include <arrow-glib/chunked-array.h>
+#include <arrow-glib/field.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_COLUMN                      \
+  (garrow_column_get_type())
+#define GARROW_COLUMN(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_COLUMN,       \
+                              GArrowColumn))
+#define GARROW_COLUMN_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),             \
+                           GARROW_TYPE_COLUMN,  \
+                           GArrowColumnClass))
+#define GARROW_IS_COLUMN(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_COLUMN))
+#define GARROW_IS_COLUMN_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),             \
+                           GARROW_TYPE_COLUMN))
+#define GARROW_COLUMN_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_COLUMN,        \
+                             GArrowColumnClass))
+
+typedef struct _GArrowColumn         GArrowColumn;
+typedef struct _GArrowColumnClass    GArrowColumnClass;
+
+/**
+ * GArrowColumn:
+ *
+ * It wraps `arrow::Column`.
+ */
+struct _GArrowColumn
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowColumnClass
+{
+  GObjectClass parent_class;
+};
+
+GType               garrow_column_get_type      (void) G_GNUC_CONST;
+
+GArrowColumn *garrow_column_new_array(GArrowField *field,
+                                      GArrowArray *array);
+GArrowColumn *garrow_column_new_chunked_array(GArrowField *field,
+                                              GArrowChunkedArray *chunked_array);
+
+guint64             garrow_column_get_length    (GArrowColumn *column);
+guint64             garrow_column_get_n_nulls   (GArrowColumn *column);
+GArrowField        *garrow_column_get_field     (GArrowColumn *column);
+const gchar        *garrow_column_get_name      (GArrowColumn *column);
+GArrowDataType     *garrow_column_get_data_type (GArrowColumn *column);
+GArrowChunkedArray *garrow_column_get_data      (GArrowColumn *column);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/column.hpp b/c_glib/arrow-glib/column.hpp
new file mode 100644
index 0000000000000..4ebb742bb5051
--- /dev/null
+++ b/c_glib/arrow-glib/column.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/column.h>
+
+GArrowColumn *garrow_column_new_raw(std::shared_ptr<arrow::Column> *arrow_column);
+std::shared_ptr<arrow::Column> garrow_column_get_raw(GArrowColumn *column);
diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/data-type.cpp
new file mode 100644
index 0000000000000..2df9e7a38da91
--- /dev/null
+++ b/c_glib/arrow-glib/data-type.cpp
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/boolean-data-type.h>
+#include <arrow-glib/binary-data-type.h>
+#include <arrow-glib/boolean-data-type.h>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/double-data-type.h>
+#include <arrow-glib/float-data-type.h>
+#include <arrow-glib/int8-data-type.h>
+#include <arrow-glib/int16-data-type.h>
+#include <arrow-glib/int32-data-type.h>
+#include <arrow-glib/int64-data-type.h>
+#include <arrow-glib/list-data-type.h>
+#include <arrow-glib/null-data-type.h>
+#include <arrow-glib/string-data-type.h>
+#include <arrow-glib/struct-data-type.h>
+#include <arrow-glib/type.hpp>
+#include <arrow-glib/uint8-data-type.h>
+#include <arrow-glib/uint16-data-type.h>
+#include <arrow-glib/uint32-data-type.h>
+#include <arrow-glib/uint64-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: data-type
+ * @short_description: Base class for all data type classes
+ *
+ * #GArrowDataType is a base class for all data type classes such as
+ * #GArrowBooleanDataType.
+ */
+
+typedef struct GArrowDataTypePrivate_ {
+  std::shared_ptr<arrow::DataType> data_type;
+} GArrowDataTypePrivate;
+
+enum {
+  PROP_0,
+  PROP_DATA_TYPE
+};
+
+G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE(GArrowDataType,
+                                    garrow_data_type,
+                                    G_TYPE_OBJECT)
+
+#define GARROW_DATA_TYPE_GET_PRIVATE(obj)               \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                   \
+                               GARROW_TYPE_DATA_TYPE,   \
+                               GArrowDataTypePrivate))
+
+static void
+garrow_data_type_finalize(GObject *object)
+{
+  GArrowDataTypePrivate *priv;
+
+  priv = GARROW_DATA_TYPE_GET_PRIVATE(object);
+
+  priv->data_type = nullptr;
+
+  G_OBJECT_CLASS(garrow_data_type_parent_class)->finalize(object);
+}
+
+static void
+garrow_data_type_set_property(GObject *object,
+                              guint prop_id,
+                              const GValue *value,
+                              GParamSpec *pspec)
+{
+  GArrowDataTypePrivate *priv;
+
+  priv = GARROW_DATA_TYPE_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_DATA_TYPE:
+    priv->data_type =
+      *static_cast<std::shared_ptr<arrow::DataType> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_data_type_get_property(GObject *object,
+                              guint prop_id,
+                              GValue *value,
+                              GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_data_type_init(GArrowDataType *object)
+{
+}
+
+static void
+garrow_data_type_class_init(GArrowDataTypeClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_data_type_finalize;
+  gobject_class->set_property = garrow_data_type_set_property;
+  gobject_class->get_property = garrow_data_type_get_property;
+
+  spec = g_param_spec_pointer("data-type",
+                              "DataType",
+                              "The raw std::shared<arrow::DataType> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_DATA_TYPE, spec);
+}
+
+/**
+ * garrow_data_type_equal:
+ * @data_type: A #GArrowDataType.
+ * @other_data_type: A #GArrowDataType.
+ *
+ * Returns: Whether they are equal or not.
+ */
+gboolean
+garrow_data_type_equal(GArrowDataType *data_type,
+                       GArrowDataType *other_data_type)
+{
+  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  const auto arrow_other_data_type = garrow_data_type_get_raw(other_data_type);
+  return arrow_data_type->Equals(arrow_other_data_type);
+}
+
+/**
+ * garrow_data_type_to_string:
+ * @data_type: A #GArrowDataType.
+ *
+ * Returns: The string representation of the data type. The caller
+ *   must free it by g_free() when the caller doesn't need it anymore.
+ */
+gchar *
+garrow_data_type_to_string(GArrowDataType *data_type)
+{
+  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  return g_strdup(arrow_data_type->ToString().c_str());
+}
+
+/**
+ * garrow_data_type_type:
+ * @data_type: A #GArrowDataType.
+ *
+ * Returns: The type of the data type.
+ */
+GArrowType
+garrow_data_type_type(GArrowDataType *data_type)
+{
+  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  return garrow_type_from_raw(arrow_data_type->type);
+}
+
+G_END_DECLS
+
+GArrowDataType *
+garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type)
+{
+  GType type;
+  GArrowDataType *data_type;
+
+  switch ((*arrow_data_type)->type) {
+  case arrow::Type::type::NA:
+    type = GARROW_TYPE_NULL_DATA_TYPE;
+    break;
+  case arrow::Type::type::BOOL:
+    type = GARROW_TYPE_BOOLEAN_DATA_TYPE;
+    break;
+  case arrow::Type::type::UINT8:
+    type = GARROW_TYPE_UINT8_DATA_TYPE;
+    break;
+  case arrow::Type::type::INT8:
+    type = GARROW_TYPE_INT8_DATA_TYPE;
+    break;
+  case arrow::Type::type::UINT16:
+    type = GARROW_TYPE_UINT16_DATA_TYPE;
+    break;
+  case arrow::Type::type::INT16:
+    type = GARROW_TYPE_INT16_DATA_TYPE;
+    break;
+  case arrow::Type::type::UINT32:
+    type = GARROW_TYPE_UINT32_DATA_TYPE;
+    break;
+  case arrow::Type::type::INT32:
+    type = GARROW_TYPE_INT32_DATA_TYPE;
+    break;
+  case arrow::Type::type::UINT64:
+    type = GARROW_TYPE_UINT64_DATA_TYPE;
+    break;
+  case arrow::Type::type::INT64:
+    type = GARROW_TYPE_INT64_DATA_TYPE;
+    break;
+  case arrow::Type::type::FLOAT:
+    type = GARROW_TYPE_FLOAT_DATA_TYPE;
+    break;
+  case arrow::Type::type::DOUBLE:
+    type = GARROW_TYPE_DOUBLE_DATA_TYPE;
+    break;
+  case arrow::Type::type::BINARY:
+    type = GARROW_TYPE_BINARY_DATA_TYPE;
+    break;
+  case arrow::Type::type::STRING:
+    type = GARROW_TYPE_STRING_DATA_TYPE;
+    break;
+  case arrow::Type::type::LIST:
+    type = GARROW_TYPE_LIST_DATA_TYPE;
+    break;
+  case arrow::Type::type::STRUCT:
+    type = GARROW_TYPE_STRUCT_DATA_TYPE;
+    break;
+  default:
+    type = GARROW_TYPE_DATA_TYPE;
+    break;
+  }
+  data_type = GARROW_DATA_TYPE(g_object_new(type,
+                                            "data_type", arrow_data_type,
+                                            NULL));
+  return data_type;
+}
+
+std::shared_ptr<arrow::DataType>
+garrow_data_type_get_raw(GArrowDataType *data_type)
+{
+  GArrowDataTypePrivate *priv;
+
+  priv = GARROW_DATA_TYPE_GET_PRIVATE(data_type);
+  return priv->data_type;
+}
diff --git a/c_glib/arrow-glib/data-type.h b/c_glib/arrow-glib/data-type.h
new file mode 100644
index 0000000000000..3203d09b5c651
--- /dev/null
+++ b/c_glib/arrow-glib/data-type.h
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_DATA_TYPE                   \
+  (garrow_data_type_get_type())
+#define GARROW_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DATA_TYPE,    \
+                              GArrowDataType))
+#define GARROW_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DATA_TYPE,       \
+                           GArrowDataTypeClass))
+#define GARROW_IS_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_DATA_TYPE))
+#define GARROW_IS_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DATA_TYPE))
+#define GARROW_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DATA_TYPE,     \
+                             GArrowDataTypeClass))
+
+typedef struct _GArrowDataType         GArrowDataType;
+typedef struct _GArrowDataTypeClass    GArrowDataTypeClass;
+
+/**
+ * GArrowDataType:
+ *
+ * It wraps `arrow::DataType`.
+ */
+struct _GArrowDataType
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowDataTypeClass
+{
+  GObjectClass parent_class;
+};
+
+GType      garrow_data_type_get_type  (void) G_GNUC_CONST;
+gboolean   garrow_data_type_equal     (GArrowDataType *data_type,
+                                       GArrowDataType *other_data_type);
+gchar     *garrow_data_type_to_string (GArrowDataType *data_type);
+GArrowType garrow_data_type_type      (GArrowDataType *data_type);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/data-type.hpp b/c_glib/arrow-glib/data-type.hpp
new file mode 100644
index 0000000000000..fddcb2eb1ac59
--- /dev/null
+++ b/c_glib/arrow-glib/data-type.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/data-type.h>
+
+GArrowDataType *garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type);
+std::shared_ptr<arrow::DataType> garrow_data_type_get_raw(GArrowDataType *data_type);
diff --git a/c_glib/arrow-glib/double-array-builder.cpp b/c_glib/arrow-glib/double-array-builder.cpp
new file mode 100644
index 0000000000000..cc44eeabfb686
--- /dev/null
+++ b/c_glib/arrow-glib/double-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/double-array-builder.h>
+#include <arrow-glib/error.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: double-array-builder
+ * @short_description: 64-bit floating point array builder class
+ *
+ * #GArrowDoubleArrayBuilder is the class to create a new
+ * #GArrowDoubleArray.
+ */
+
+G_DEFINE_TYPE(GArrowDoubleArrayBuilder,
+              garrow_double_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_double_array_builder_init(GArrowDoubleArrayBuilder *builder)
+{
+}
+
+static void
+garrow_double_array_builder_class_init(GArrowDoubleArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_double_array_builder_new:
+ *
+ * Returns: A newly created #GArrowDoubleArrayBuilder.
+ */
+GArrowDoubleArrayBuilder *
+garrow_double_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::DoubleBuilder>(memory_pool, arrow::float64());
+  auto builder =
+    GARROW_DOUBLE_ARRAY_BUILDER(g_object_new(GARROW_TYPE_DOUBLE_ARRAY_BUILDER,
+                                             "array-builder", &arrow_builder,
+                                             NULL));
+  return builder;
+}
+
+/**
+ * garrow_double_array_builder_append:
+ * @builder: A #GArrowDoubleArrayBuilder.
+ * @value: A double value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
+                                   gdouble value,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::DoubleBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[double-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_double_array_builder_append_null:
+ * @builder: A #GArrowDoubleArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
+                                        GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::DoubleBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[double-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/double-array-builder.h b/c_glib/arrow-glib/double-array-builder.h
new file mode 100644
index 0000000000000..5d95c898bc8a7
--- /dev/null
+++ b/c_glib/arrow-glib/double-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_DOUBLE_ARRAY_BUILDER        \
+  (garrow_double_array_builder_get_type())
+#define GARROW_DOUBLE_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DOUBLE_ARRAY_BUILDER, \
+                              GArrowDoubleArrayBuilder))
+#define GARROW_DOUBLE_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DOUBLE_ARRAY_BUILDER,    \
+                           GArrowDoubleArrayBuilderClass))
+#define GARROW_IS_DOUBLE_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_DOUBLE_ARRAY_BUILDER))
+#define GARROW_IS_DOUBLE_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DOUBLE_ARRAY_BUILDER))
+#define GARROW_DOUBLE_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DOUBLE_ARRAY_BUILDER,  \
+                             GArrowDoubleArrayBuilderClass))
+
+typedef struct _GArrowDoubleArrayBuilder         GArrowDoubleArrayBuilder;
+typedef struct _GArrowDoubleArrayBuilderClass    GArrowDoubleArrayBuilderClass;
+
+/**
+ * GArrowDoubleArrayBuilder:
+ *
+ * It wraps `arrow::DoubleBuilder`.
+ */
+struct _GArrowDoubleArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowDoubleArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_double_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowDoubleArrayBuilder *garrow_double_array_builder_new(void);
+
+gboolean garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
+                                            gdouble value,
+                                            GError **error);
+gboolean garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
+                                                 GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/double-array.cpp b/c_glib/arrow-glib/double-array.cpp
new file mode 100644
index 0000000000000..ecc55d7541689
--- /dev/null
+++ b/c_glib/arrow-glib/double-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/double-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: double-array
+ * @short_description: 64-bit floating point array class
+ *
+ * #GArrowDoubleArray is a class for 64-bit floating point array. It
+ * can store zero or more 64-bit floating data.
+ *
+ * #GArrowDoubleArray is immutable. You need to use
+ * #GArrowDoubleArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowDoubleArray,               \
+              garrow_double_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_double_array_init(GArrowDoubleArray *object)
+{
+}
+
+static void
+garrow_double_array_class_init(GArrowDoubleArrayClass *klass)
+{
+}
+
+/**
+ * garrow_double_array_get_value:
+ * @array: A #GArrowDoubleArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gdouble
+garrow_double_array_get_value(GArrowDoubleArray *array,
+                              gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::DoubleArray *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/double-array.h b/c_glib/arrow-glib/double-array.h
new file mode 100644
index 0000000000000..b9a236532e3bf
--- /dev/null
+++ b/c_glib/arrow-glib/double-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_DOUBLE_ARRAY                \
+  (garrow_double_array_get_type())
+#define GARROW_DOUBLE_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DOUBLE_ARRAY, \
+                              GArrowDoubleArray))
+#define GARROW_DOUBLE_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DOUBLE_ARRAY,    \
+                           GArrowDoubleArrayClass))
+#define GARROW_IS_DOUBLE_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DOUBLE_ARRAY))
+#define GARROW_IS_DOUBLE_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DOUBLE_ARRAY))
+#define GARROW_DOUBLE_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DOUBLE_ARRAY,  \
+                             GArrowDoubleArrayClass))
+
+typedef struct _GArrowDoubleArray         GArrowDoubleArray;
+typedef struct _GArrowDoubleArrayClass    GArrowDoubleArrayClass;
+
+/**
+ * GArrowDoubleArray:
+ *
+ * It wraps `arrow::DoubleArray`.
+ */
+struct _GArrowDoubleArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowDoubleArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_double_array_get_type(void) G_GNUC_CONST;
+
+gdouble garrow_double_array_get_value(GArrowDoubleArray *array,
+                                      gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/double-data-type.cpp b/c_glib/arrow-glib/double-data-type.cpp
new file mode 100644
index 0000000000000..c132f97ebe58f
--- /dev/null
+++ b/c_glib/arrow-glib/double-data-type.cpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/double-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: double-data-type
+ * @short_description: 64-bit floating point data type
+ *
+ * #GArrowDoubleDataType is a class for 64-bit floating point data
+ * type.
+ */
+
+G_DEFINE_TYPE(GArrowDoubleDataType,                \
+              garrow_double_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_double_data_type_init(GArrowDoubleDataType *object)
+{
+}
+
+static void
+garrow_double_data_type_class_init(GArrowDoubleDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_double_data_type_new:
+ *
+ * Returns: The newly created 64-bit floating point data type.
+ */
+GArrowDoubleDataType *
+garrow_double_data_type_new(void)
+{
+  auto arrow_data_type = arrow::float64();
+
+  GArrowDoubleDataType *data_type =
+    GARROW_DOUBLE_DATA_TYPE(g_object_new(GARROW_TYPE_DOUBLE_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/double-data-type.h b/c_glib/arrow-glib/double-data-type.h
new file mode 100644
index 0000000000000..ec725cbed3ba2
--- /dev/null
+++ b/c_glib/arrow-glib/double-data-type.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_DOUBLE_DATA_TYPE           \
+  (garrow_double_data_type_get_type())
+#define GARROW_DOUBLE_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DOUBLE_DATA_TYPE,    \
+                              GArrowDoubleDataType))
+#define GARROW_DOUBLE_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DOUBLE_DATA_TYPE,       \
+                           GArrowDoubleDataTypeClass))
+#define GARROW_IS_DOUBLE_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DOUBLE_DATA_TYPE))
+#define GARROW_IS_DOUBLE_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DOUBLE_DATA_TYPE))
+#define GARROW_DOUBLE_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DOUBLE_DATA_TYPE,     \
+                             GArrowDoubleDataTypeClass))
+
+typedef struct _GArrowDoubleDataType         GArrowDoubleDataType;
+typedef struct _GArrowDoubleDataTypeClass    GArrowDoubleDataTypeClass;
+
+/**
+ * GArrowDoubleDataType:
+ *
+ * It wraps `arrow::DoubleType`.
+ */
+struct _GArrowDoubleDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowDoubleDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType garrow_double_data_type_get_type(void) G_GNUC_CONST;
+
+GArrowDoubleDataType *garrow_double_data_type_new(void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/enums.c.template b/c_glib/arrow-glib/enums.c.template
new file mode 100644
index 0000000000000..6becbd565d516
--- /dev/null
+++ b/c_glib/arrow-glib/enums.c.template
@@ -0,0 +1,56 @@
+/*** BEGIN file-header ***/
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/arrow-glib.h>
+/*** END file-header ***/
+
+/*** BEGIN file-production ***/
+
+/* enumerations from "@filename@" */
+/*** END file-production ***/
+
+/*** BEGIN value-header ***/
+GType
+@enum_name@_get_type(void)
+{
+  static GType etype = 0;
+  if (G_UNLIKELY(etype == 0)) {
+    static const G@Type@Value values[] = {
+/*** END value-header ***/
+
+/*** BEGIN value-production ***/
+      {@VALUENAME@, "@VALUENAME@", "@valuenick@"},
+/*** END value-production ***/
+
+/*** BEGIN value-tail ***/
+      {0, NULL, NULL}
+    };
+    etype = g_@type@_register_static(g_intern_static_string("@EnumName@"), values);
+  }
+  return etype;
+}
+/*** END value-tail ***/
+
+/*** BEGIN file-tail ***/
+/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/enums.h.template b/c_glib/arrow-glib/enums.h.template
new file mode 100644
index 0000000000000..3509ed2e90db4
--- /dev/null
+++ b/c_glib/arrow-glib/enums.h.template
@@ -0,0 +1,41 @@
+/*** BEGIN file-header ***/
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/error.h>
+
+G_BEGIN_DECLS
+/*** END file-header ***/
+
+/*** BEGIN file-production ***/
+
+/* enumerations from "@filename@" */
+/*** END file-production ***/
+
+/*** BEGIN value-header ***/
+GType @enum_name@_get_type(void) G_GNUC_CONST;
+#define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
+/*** END value-header ***/
+
+/*** BEGIN file-tail ***/
+
+G_END_DECLS
+/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/error.cpp b/c_glib/arrow-glib/error.cpp
new file mode 100644
index 0000000000000..efbc6ae60452a
--- /dev/null
+++ b/c_glib/arrow-glib/error.cpp
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/error.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: error
+ * @title: GArrowError
+ * @short_description: Error code mapping between Arrow and arrow-glib
+ *
+ * #GArrowError provides error codes corresponding to `arrow::Status`
+ * values.
+ */
+
+G_DEFINE_QUARK(garrow-error-quark, garrow_error)
+
+static GArrowError
+garrow_error_code(const arrow::Status &status)
+{
+  switch (status.code()) {
+  case arrow::StatusCode::OK:
+    return GARROW_ERROR_UNKNOWN;
+  case arrow::StatusCode::OutOfMemory:
+    return GARROW_ERROR_OUT_OF_MEMORY;
+  case arrow::StatusCode::KeyError:
+    return GARROW_ERROR_KEY;
+  case arrow::StatusCode::TypeError:
+    return GARROW_ERROR_TYPE;
+  case arrow::StatusCode::Invalid:
+    return GARROW_ERROR_INVALID;
+  case arrow::StatusCode::IOError:
+    return GARROW_ERROR_IO;
+  case arrow::StatusCode::UnknownError:
+    return GARROW_ERROR_UNKNOWN;
+  case arrow::StatusCode::NotImplemented:
+    return GARROW_ERROR_NOT_IMPLEMENTED;
+  default:
+    return GARROW_ERROR_UNKNOWN;
+  }
+}
+
+G_END_DECLS
+
+void
+garrow_error_set(GError **error,
+                 const arrow::Status &status,
+                 const char *context)
+{
+  if (status.ok()) {
+    return;
+  }
+
+  g_set_error(error,
+              GARROW_ERROR,
+              garrow_error_code(status),
+              "%s: %s",
+              context,
+              status.ToString().c_str());
+}
diff --git a/c_glib/arrow-glib/error.h b/c_glib/arrow-glib/error.h
new file mode 100644
index 0000000000000..b4a4fac39cd73
--- /dev/null
+++ b/c_glib/arrow-glib/error.h
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+/**
+ * GArrowError:
+ * @GARROW_ERROR_OUT_OF_MEMORY: Out of memory error.
+ * @GARROW_ERROR_KEY: Key error.
+ * @GARROW_ERROR_TYPE: Type error.
+ * @GARROW_ERROR_INVALID: Invalid value error.
+ * @GARROW_ERROR_IO: IO error.
+ * @GARROW_ERROR_UNKNOWN: Unknown error.
+ * @GARROW_ERROR_NOT_IMPLEMENTED: The feature is not implemented.
+ *
+ * The error codes are used by all arrow-glib functions.
+ *
+ * They are corresponding to `arrow::Status` values.
+ */
+typedef enum {
+  GARROW_ERROR_OUT_OF_MEMORY = 1,
+  GARROW_ERROR_KEY,
+  GARROW_ERROR_TYPE,
+  GARROW_ERROR_INVALID,
+  GARROW_ERROR_IO,
+  GARROW_ERROR_UNKNOWN = 9,
+  GARROW_ERROR_NOT_IMPLEMENTED = 10
+} GArrowError;
+
+#define GARROW_ERROR garrow_error_quark()
+
+GQuark garrow_error_quark(void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/error.hpp b/c_glib/arrow-glib/error.hpp
new file mode 100644
index 0000000000000..357d293c4f127
--- /dev/null
+++ b/c_glib/arrow-glib/error.hpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.h>
+
+void garrow_error_set(GError **error,
+                      const arrow::Status &status,
+                      const char *context);
diff --git a/c_glib/arrow-glib/field.cpp b/c_glib/arrow-glib/field.cpp
new file mode 100644
index 0000000000000..0dcaf0a009a6d
--- /dev/null
+++ b/c_glib/arrow-glib/field.cpp
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/field.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: field
+ * @short_description: Field class
+ *
+ * #GArrowField is a class for field. Field is metadata of a
+ * column. It has name, data type (#GArrowDataType) and nullable
+ * information of the column.
+ */
+
+typedef struct GArrowFieldPrivate_ {
+  std::shared_ptr<arrow::Field> field;
+} GArrowFieldPrivate;
+
+enum {
+  PROP_0,
+  PROP_FIELD
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowField,
+                           garrow_field,
+                           G_TYPE_OBJECT)
+
+#define GARROW_FIELD_GET_PRIVATE(obj)               \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),               \
+                               GARROW_TYPE_FIELD,   \
+                               GArrowFieldPrivate))
+
+static void
+garrow_field_finalize(GObject *object)
+{
+  GArrowFieldPrivate *priv;
+
+  priv = GARROW_FIELD_GET_PRIVATE(object);
+
+  priv->field = nullptr;
+
+  G_OBJECT_CLASS(garrow_field_parent_class)->finalize(object);
+}
+
+static void
+garrow_field_set_property(GObject *object,
+                          guint prop_id,
+                          const GValue *value,
+                          GParamSpec *pspec)
+{
+  GArrowFieldPrivate *priv;
+
+  priv = GARROW_FIELD_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_FIELD:
+    priv->field =
+      *static_cast<std::shared_ptr<arrow::Field> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_field_get_property(GObject *object,
+                          guint prop_id,
+                          GValue *value,
+                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_field_init(GArrowField *object)
+{
+}
+
+static void
+garrow_field_class_init(GArrowFieldClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_field_finalize;
+  gobject_class->set_property = garrow_field_set_property;
+  gobject_class->get_property = garrow_field_get_property;
+
+  spec = g_param_spec_pointer("field",
+                              "Field",
+                              "The raw std::shared<arrow::Field> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_FIELD, spec);
+}
+
+/**
+ * garrow_field_new:
+ * @name: The name of the field.
+ * @data_type: The data type of the field.
+ *
+ * Returns: A newly created #GArrowField.
+ */
+GArrowField *
+garrow_field_new(const gchar *name,
+                 GArrowDataType *data_type)
+{
+  auto arrow_field =
+    std::make_shared<arrow::Field>(name,
+                                   garrow_data_type_get_raw(data_type));
+  return garrow_field_new_raw(&arrow_field);
+}
+
+/**
+ * garrow_field_new_full:
+ * @name: The name of the field.
+ * @data_type: The data type of the field.
+ * @nullable: Whether null may be included or not.
+ *
+ * Returns: A newly created #GArrowField.
+ */
+GArrowField *
+garrow_field_new_full(const gchar *name,
+                      GArrowDataType *data_type,
+                      gboolean nullable)
+{
+  auto arrow_field =
+    std::make_shared<arrow::Field>(name,
+                                   garrow_data_type_get_raw(data_type),
+                                   nullable);
+  return garrow_field_new_raw(&arrow_field);
+}
+
+/**
+ * garrow_field_get_name:
+ * @field: A #GArrowField.
+ *
+ * Returns: The name of the field.
+ */
+const gchar *
+garrow_field_get_name(GArrowField *field)
+{
+  const auto arrow_field = garrow_field_get_raw(field);
+  return arrow_field->name.c_str();
+}
+
+/**
+ * garrow_field_get_data_type:
+ * @field: A #GArrowField.
+ *
+ * Returns: (transfer full): The data type of the field.
+ */
+GArrowDataType *
+garrow_field_get_data_type(GArrowField *field)
+{
+  const auto arrow_field = garrow_field_get_raw(field);
+  return garrow_data_type_new_raw(&arrow_field->type);
+}
+
+/**
+ * garrow_field_is_nullable:
+ * @field: A #GArrowField.
+ *
+ * Returns: Whether the filed may include null or not.
+ */
+gboolean
+garrow_field_is_nullable(GArrowField *field)
+{
+  const auto arrow_field = garrow_field_get_raw(field);
+  return arrow_field->nullable;
+}
+
+/**
+ * garrow_field_equal:
+ * @field: A #GArrowField.
+ * @other_field: A #GArrowField.
+ *
+ * Returns: Whether they are equal or not.
+ */
+gboolean
+garrow_field_equal(GArrowField *field,
+                   GArrowField *other_field)
+{
+  const auto arrow_field = garrow_field_get_raw(field);
+  const auto arrow_other_field = garrow_field_get_raw(other_field);
+  return arrow_field->Equals(arrow_other_field);
+}
+
+/**
+ * garrow_field_to_string:
+ * @field: A #GArrowField.
+ *
+ * Returns: The string representation of the field.
+ */
+gchar *
+garrow_field_to_string(GArrowField *field)
+{
+  const auto arrow_field = garrow_field_get_raw(field);
+  return g_strdup(arrow_field->ToString().c_str());
+}
+
+G_END_DECLS
+
+GArrowField *
+garrow_field_new_raw(std::shared_ptr<arrow::Field> *arrow_field)
+{
+  auto field = GARROW_FIELD(g_object_new(GARROW_TYPE_FIELD,
+                                         "field", arrow_field,
+                                         NULL));
+  return field;
+}
+
+std::shared_ptr<arrow::Field>
+garrow_field_get_raw(GArrowField *field)
+{
+  GArrowFieldPrivate *priv;
+
+  priv = GARROW_FIELD_GET_PRIVATE(field);
+  return priv->field;
+}
diff --git a/c_glib/arrow-glib/field.h b/c_glib/arrow-glib/field.h
new file mode 100644
index 0000000000000..e724dce49da5c
--- /dev/null
+++ b/c_glib/arrow-glib/field.h
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_FIELD                       \
+  (garrow_field_get_type())
+#define GARROW_FIELD(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_FIELD,        \
+                              GArrowField))
+#define GARROW_FIELD_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),             \
+                           GARROW_TYPE_FIELD,   \
+                           GArrowFieldClass))
+#define GARROW_IS_FIELD(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_FIELD))
+#define GARROW_IS_FIELD_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),             \
+                           GARROW_TYPE_FIELD))
+#define GARROW_FIELD_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),             \
+                             GARROW_TYPE_FIELD, \
+                             GArrowFieldClass))
+
+typedef struct _GArrowField         GArrowField;
+typedef struct _GArrowFieldClass    GArrowFieldClass;
+
+/**
+ * GArrowField:
+ *
+ * It wraps `arrow::Field`.
+ */
+struct _GArrowField
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowFieldClass
+{
+  GObjectClass parent_class;
+};
+
+GType           garrow_field_get_type      (void) G_GNUC_CONST;
+
+GArrowField    *garrow_field_new           (const gchar *name,
+                                            GArrowDataType *data_type);
+GArrowField    *garrow_field_new_full      (const gchar *name,
+                                            GArrowDataType *data_type,
+                                            gboolean nullable);
+
+const gchar    *garrow_field_get_name      (GArrowField *field);
+GArrowDataType *garrow_field_get_data_type (GArrowField *field);
+gboolean        garrow_field_is_nullable   (GArrowField *field);
+
+gboolean        garrow_field_equal         (GArrowField *field,
+                                            GArrowField *other_field);
+
+gchar          *garrow_field_to_string     (GArrowField *field);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/field.hpp b/c_glib/arrow-glib/field.hpp
new file mode 100644
index 0000000000000..e130ad5992409
--- /dev/null
+++ b/c_glib/arrow-glib/field.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/field.h>
+
+GArrowField *garrow_field_new_raw(std::shared_ptr<arrow::Field> *arrow_field);
+std::shared_ptr<arrow::Field> garrow_field_get_raw(GArrowField *field);
diff --git a/c_glib/arrow-glib/float-array-builder.cpp b/c_glib/arrow-glib/float-array-builder.cpp
new file mode 100644
index 0000000000000..77a9a0bb75a05
--- /dev/null
+++ b/c_glib/arrow-glib/float-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/float-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: float-array-builder
+ * @short_description: 32-bit floating point array builder class
+ *
+ * #GArrowFloatArrayBuilder is the class to creating a new
+ * #GArrowFloatArray.
+ */
+
+G_DEFINE_TYPE(GArrowFloatArrayBuilder,
+              garrow_float_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_float_array_builder_init(GArrowFloatArrayBuilder *builder)
+{
+}
+
+static void
+garrow_float_array_builder_class_init(GArrowFloatArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_float_array_builder_new:
+ *
+ * Returns: A newly created #GArrowFloatArrayBuilder.
+ */
+GArrowFloatArrayBuilder *
+garrow_float_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::FloatBuilder>(memory_pool, arrow::float32());
+  auto builder =
+    GARROW_FLOAT_ARRAY_BUILDER(g_object_new(GARROW_TYPE_FLOAT_ARRAY_BUILDER,
+                                            "array-builder", &arrow_builder,
+                                            NULL));
+  return builder;
+}
+
+/**
+ * garrow_float_array_builder_append:
+ * @builder: A #GArrowFloatArrayBuilder.
+ * @value: A float value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
+                                  gfloat value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::FloatBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[float-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_float_array_builder_append_null:
+ * @builder: A #GArrowFloatArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::FloatBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[float-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/float-array-builder.h b/c_glib/arrow-glib/float-array-builder.h
new file mode 100644
index 0000000000000..003900313cca4
--- /dev/null
+++ b/c_glib/arrow-glib/float-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_FLOAT_ARRAY_BUILDER         \
+  (garrow_float_array_builder_get_type())
+#define GARROW_FLOAT_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_FLOAT_ARRAY_BUILDER,  \
+                              GArrowFloatArrayBuilder))
+#define GARROW_FLOAT_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_FLOAT_ARRAY_BUILDER,     \
+                           GArrowFloatArrayBuilderClass))
+#define GARROW_IS_FLOAT_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_FLOAT_ARRAY_BUILDER))
+#define GARROW_IS_FLOAT_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_FLOAT_ARRAY_BUILDER))
+#define GARROW_FLOAT_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_FLOAT_ARRAY_BUILDER,   \
+                             GArrowFloatArrayBuilderClass))
+
+typedef struct _GArrowFloatArrayBuilder         GArrowFloatArrayBuilder;
+typedef struct _GArrowFloatArrayBuilderClass    GArrowFloatArrayBuilderClass;
+
+/**
+ * GArrowFloatArrayBuilder:
+ *
+ * It wraps `arrow::FloatBuilder`.
+ */
+struct _GArrowFloatArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowFloatArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_float_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowFloatArrayBuilder *garrow_float_array_builder_new(void);
+
+gboolean garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
+                                           gfloat value,
+                                           GError **error);
+gboolean garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
+                                                GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/float-array.cpp b/c_glib/arrow-glib/float-array.cpp
new file mode 100644
index 0000000000000..28e8047652f7e
--- /dev/null
+++ b/c_glib/arrow-glib/float-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/float-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: float-array
+ * @short_description: 32-bit floating point array class
+ *
+ * #GArrowFloatArray is a class for 32-bit floating point array. It
+ * can store zero or more 32-bit floating data.
+ *
+ * #GArrowFloatArray is immutable. You need to use
+ * #GArrowFloatArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowFloatArray,               \
+              garrow_float_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_float_array_init(GArrowFloatArray *object)
+{
+}
+
+static void
+garrow_float_array_class_init(GArrowFloatArrayClass *klass)
+{
+}
+
+/**
+ * garrow_float_array_get_value:
+ * @array: A #GArrowFloatArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gfloat
+garrow_float_array_get_value(GArrowFloatArray *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::FloatArray *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/float-array.h b/c_glib/arrow-glib/float-array.h
new file mode 100644
index 0000000000000..d113f9757a511
--- /dev/null
+++ b/c_glib/arrow-glib/float-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_FLOAT_ARRAY                 \
+  (garrow_float_array_get_type())
+#define GARROW_FLOAT_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_FLOAT_ARRAY,  \
+                              GArrowFloatArray))
+#define GARROW_FLOAT_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_FLOAT_ARRAY,     \
+                           GArrowFloatArrayClass))
+#define GARROW_IS_FLOAT_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_FLOAT_ARRAY))
+#define GARROW_IS_FLOAT_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_FLOAT_ARRAY))
+#define GARROW_FLOAT_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_FLOAT_ARRAY,   \
+                             GArrowFloatArrayClass))
+
+typedef struct _GArrowFloatArray         GArrowFloatArray;
+typedef struct _GArrowFloatArrayClass    GArrowFloatArrayClass;
+
+/**
+ * GArrowFloatArray:
+ *
+ * It wraps `arrow::FloatArray`.
+ */
+struct _GArrowFloatArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowFloatArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_float_array_get_type(void) G_GNUC_CONST;
+
+gfloat garrow_float_array_get_value(GArrowFloatArray *array,
+                                    gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/float-data-type.cpp b/c_glib/arrow-glib/float-data-type.cpp
new file mode 100644
index 0000000000000..ce7f28acfcb45
--- /dev/null
+++ b/c_glib/arrow-glib/float-data-type.cpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/float-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: float-data-type
+ * @short_description: 32-bit floating point data type
+ *
+ * #GArrowFloatDataType is a class for 32-bit floating point data
+ * type.
+ */
+
+G_DEFINE_TYPE(GArrowFloatDataType,                \
+              garrow_float_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_float_data_type_init(GArrowFloatDataType *object)
+{
+}
+
+static void
+garrow_float_data_type_class_init(GArrowFloatDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_float_data_type_new:
+ *
+ * Returns: The newly created float data type.
+ */
+GArrowFloatDataType *
+garrow_float_data_type_new(void)
+{
+  auto arrow_data_type = arrow::float32();
+
+  GArrowFloatDataType *data_type =
+    GARROW_FLOAT_DATA_TYPE(g_object_new(GARROW_TYPE_FLOAT_DATA_TYPE,
+                                        "data-type", &arrow_data_type,
+                                        NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/float-data-type.h b/c_glib/arrow-glib/float-data-type.h
new file mode 100644
index 0000000000000..dcb6c2ab13d25
--- /dev/null
+++ b/c_glib/arrow-glib/float-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_FLOAT_DATA_TYPE           \
+  (garrow_float_data_type_get_type())
+#define GARROW_FLOAT_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
+                              GARROW_TYPE_FLOAT_DATA_TYPE,    \
+                              GArrowFloatDataType))
+#define GARROW_FLOAT_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
+                           GARROW_TYPE_FLOAT_DATA_TYPE,       \
+                           GArrowFloatDataTypeClass))
+#define GARROW_IS_FLOAT_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
+                              GARROW_TYPE_FLOAT_DATA_TYPE))
+#define GARROW_IS_FLOAT_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
+                           GARROW_TYPE_FLOAT_DATA_TYPE))
+#define GARROW_FLOAT_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
+                             GARROW_TYPE_FLOAT_DATA_TYPE,     \
+                             GArrowFloatDataTypeClass))
+
+typedef struct _GArrowFloatDataType         GArrowFloatDataType;
+typedef struct _GArrowFloatDataTypeClass    GArrowFloatDataTypeClass;
+
+/**
+ * GArrowFloatDataType:
+ *
+ * It wraps `arrow::FloatType`.
+ */
+struct _GArrowFloatDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowFloatDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                garrow_float_data_type_get_type (void) G_GNUC_CONST;
+GArrowFloatDataType *garrow_float_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-array-builder.cpp b/c_glib/arrow-glib/int16-array-builder.cpp
new file mode 100644
index 0000000000000..fbf18ef1e6ce7
--- /dev/null
+++ b/c_glib/arrow-glib/int16-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/int16-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int16-array-builder
+ * @short_description: 16-bit integer array builder class
+ *
+ * #GArrowInt16ArrayBuilder is the class to create a new
+ * #GArrowInt16Array.
+ */
+
+G_DEFINE_TYPE(GArrowInt16ArrayBuilder,
+              garrow_int16_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int16_array_builder_init(GArrowInt16ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int16_array_builder_class_init(GArrowInt16ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int16_array_builder_new:
+ *
+ * Returns: A newly created #GArrowInt16ArrayBuilder.
+ */
+GArrowInt16ArrayBuilder *
+garrow_int16_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::Int16Builder>(memory_pool, arrow::int16());
+  auto builder =
+    GARROW_INT16_ARRAY_BUILDER(g_object_new(GARROW_TYPE_INT16_ARRAY_BUILDER,
+                                           "array-builder", &arrow_builder,
+                                           NULL));
+  return builder;
+}
+
+/**
+ * garrow_int16_array_builder_append:
+ * @builder: A #GArrowInt16ArrayBuilder.
+ * @value: A int16 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
+                                 gint16 value,
+                                 GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int16Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int16-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_int16_array_builder_append_null:
+ * @builder: A #GArrowInt16ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
+                                      GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int16Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int16-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-array-builder.h b/c_glib/arrow-glib/int16-array-builder.h
new file mode 100644
index 0000000000000..f222cfdccc9b7
--- /dev/null
+++ b/c_glib/arrow-glib/int16-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT16_ARRAY_BUILDER          \
+  (garrow_int16_array_builder_get_type())
+#define GARROW_INT16_ARRAY_BUILDER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT16_ARRAY_BUILDER,   \
+                              GArrowInt16ArrayBuilder))
+#define GARROW_INT16_ARRAY_BUILDER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT16_ARRAY_BUILDER,      \
+                           GArrowInt16ArrayBuilderClass))
+#define GARROW_IS_INT16_ARRAY_BUILDER(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT16_ARRAY_BUILDER))
+#define GARROW_IS_INT16_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT16_ARRAY_BUILDER))
+#define GARROW_INT16_ARRAY_BUILDER_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT16_ARRAY_BUILDER,    \
+                             GArrowInt16ArrayBuilderClass))
+
+typedef struct _GArrowInt16ArrayBuilder         GArrowInt16ArrayBuilder;
+typedef struct _GArrowInt16ArrayBuilderClass    GArrowInt16ArrayBuilderClass;
+
+/**
+ * GArrowInt16ArrayBuilder:
+ *
+ * It wraps `arrow::Int16Builder`.
+ */
+struct _GArrowInt16ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowInt16ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int16_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowInt16ArrayBuilder *garrow_int16_array_builder_new(void);
+
+gboolean garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
+                                          gint16 value,
+                                          GError **error);
+gboolean garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
+                                               GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-array.cpp b/c_glib/arrow-glib/int16-array.cpp
new file mode 100644
index 0000000000000..456d085a3449a
--- /dev/null
+++ b/c_glib/arrow-glib/int16-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/int16-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int16-array
+ * @short_description: 16-bit integer array class
+ *
+ * #GArrowInt16Array is a class for 16-bit integer array. It can store
+ * zero or more 16-bit integer data.
+ *
+ * #GArrowInt16Array is immutable. You need to use
+ * #GArrowInt16ArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowInt16Array,               \
+              garrow_int16_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_int16_array_init(GArrowInt16Array *object)
+{
+}
+
+static void
+garrow_int16_array_class_init(GArrowInt16ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_int16_array_get_value:
+ * @array: A #GArrowInt16Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gint16
+garrow_int16_array_get_value(GArrowInt16Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Int16Array *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-array.h b/c_glib/arrow-glib/int16-array.h
new file mode 100644
index 0000000000000..d37144cef51f2
--- /dev/null
+++ b/c_glib/arrow-glib/int16-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT16_ARRAY                  \
+  (garrow_int16_array_get_type())
+#define GARROW_INT16_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT16_ARRAY,   \
+                              GArrowInt16Array))
+#define GARROW_INT16_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT16_ARRAY,      \
+                           GArrowInt16ArrayClass))
+#define GARROW_IS_INT16_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT16_ARRAY))
+#define GARROW_IS_INT16_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT16_ARRAY))
+#define GARROW_INT16_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT16_ARRAY,    \
+                             GArrowInt16ArrayClass))
+
+typedef struct _GArrowInt16Array         GArrowInt16Array;
+typedef struct _GArrowInt16ArrayClass    GArrowInt16ArrayClass;
+
+/**
+ * GArrowInt16Array:
+ *
+ * It wraps `arrow::Int16Array`.
+ */
+struct _GArrowInt16Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowInt16ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_int16_array_get_type(void) G_GNUC_CONST;
+
+gint16 garrow_int16_array_get_value(GArrowInt16Array *array,
+                                  gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-data-type.cpp b/c_glib/arrow-glib/int16-data-type.cpp
new file mode 100644
index 0000000000000..45e109e1759dc
--- /dev/null
+++ b/c_glib/arrow-glib/int16-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/int16-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int16-data-type
+ * @short_description: 16-bit integer data type
+ *
+ * #GArrowInt16DataType is a class for 16-bit integer data type.
+ */
+
+G_DEFINE_TYPE(GArrowInt16DataType,                \
+              garrow_int16_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_int16_data_type_init(GArrowInt16DataType *object)
+{
+}
+
+static void
+garrow_int16_data_type_class_init(GArrowInt16DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_int16_data_type_new:
+ *
+ * Returns: The newly created 16-bit integer data type.
+ */
+GArrowInt16DataType *
+garrow_int16_data_type_new(void)
+{
+  auto arrow_data_type = arrow::int16();
+
+  GArrowInt16DataType *data_type =
+    GARROW_INT16_DATA_TYPE(g_object_new(GARROW_TYPE_INT16_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-data-type.h b/c_glib/arrow-glib/int16-data-type.h
new file mode 100644
index 0000000000000..eaa199c4fc7f8
--- /dev/null
+++ b/c_glib/arrow-glib/int16-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT16_DATA_TYPE            \
+  (garrow_int16_data_type_get_type())
+#define GARROW_INT16_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT16_DATA_TYPE,     \
+                              GArrowInt16DataType))
+#define GARROW_INT16_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT16_DATA_TYPE,        \
+                           GArrowInt16DataTypeClass))
+#define GARROW_IS_INT16_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT16_DATA_TYPE))
+#define GARROW_IS_INT16_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT16_DATA_TYPE))
+#define GARROW_INT16_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT16_DATA_TYPE,      \
+                             GArrowInt16DataTypeClass))
+
+typedef struct _GArrowInt16DataType         GArrowInt16DataType;
+typedef struct _GArrowInt16DataTypeClass    GArrowInt16DataTypeClass;
+
+/**
+ * GArrowInt16DataType:
+ *
+ * It wraps `arrow::Int16Type`.
+ */
+struct _GArrowInt16DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt16DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int16_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt16DataType *garrow_int16_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-array-builder.cpp b/c_glib/arrow-glib/int32-array-builder.cpp
new file mode 100644
index 0000000000000..30cc4702f68fb
--- /dev/null
+++ b/c_glib/arrow-glib/int32-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/int32-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int32-array-builder
+ * @short_description: 32-bit integer array builder class
+ *
+ * #GArrowInt32ArrayBuilder is the class to create a new
+ * #GArrowInt32Array.
+ */
+
+G_DEFINE_TYPE(GArrowInt32ArrayBuilder,
+              garrow_int32_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int32_array_builder_init(GArrowInt32ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int32_array_builder_class_init(GArrowInt32ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int32_array_builder_new:
+ *
+ * Returns: A newly created #GArrowInt32ArrayBuilder.
+ */
+GArrowInt32ArrayBuilder *
+garrow_int32_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::Int32Builder>(memory_pool, arrow::int32());
+  auto builder =
+    GARROW_INT32_ARRAY_BUILDER(g_object_new(GARROW_TYPE_INT32_ARRAY_BUILDER,
+                                           "array-builder", &arrow_builder,
+                                           NULL));
+  return builder;
+}
+
+/**
+ * garrow_int32_array_builder_append:
+ * @builder: A #GArrowInt32ArrayBuilder.
+ * @value: A int32 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
+                                 gint32 value,
+                                 GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int32Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int32-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_int32_array_builder_append_null:
+ * @builder: A #GArrowInt32ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
+                                      GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int32Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int32-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-array-builder.h b/c_glib/arrow-glib/int32-array-builder.h
new file mode 100644
index 0000000000000..bdb380d6070b0
--- /dev/null
+++ b/c_glib/arrow-glib/int32-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT32_ARRAY_BUILDER         \
+  (garrow_int32_array_builder_get_type())
+#define GARROW_INT32_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT32_ARRAY_BUILDER,  \
+                              GArrowInt32ArrayBuilder))
+#define GARROW_INT32_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT32_ARRAY_BUILDER,     \
+                           GArrowInt32ArrayBuilderClass))
+#define GARROW_IS_INT32_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT32_ARRAY_BUILDER))
+#define GARROW_IS_INT32_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT32_ARRAY_BUILDER))
+#define GARROW_INT32_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT32_ARRAY_BUILDER,   \
+                             GArrowInt32ArrayBuilderClass))
+
+typedef struct _GArrowInt32ArrayBuilder         GArrowInt32ArrayBuilder;
+typedef struct _GArrowInt32ArrayBuilderClass    GArrowInt32ArrayBuilderClass;
+
+/**
+ * GArrowInt32ArrayBuilder:
+ *
+ * It wraps `arrow::Int32Builder`.
+ */
+struct _GArrowInt32ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowInt32ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int32_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowInt32ArrayBuilder *garrow_int32_array_builder_new(void);
+
+gboolean garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
+                                           gint32 value,
+                                           GError **error);
+gboolean garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
+                                                GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-array.cpp b/c_glib/arrow-glib/int32-array.cpp
new file mode 100644
index 0000000000000..8bd6f35fd6431
--- /dev/null
+++ b/c_glib/arrow-glib/int32-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/int32-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int32-array
+ * @short_description: 32-bit integer array class
+ *
+ * #GArrowInt32Array is a class for 32-bit integer array. It can store
+ * zero or more 32-bit integer data.
+ *
+ * #GArrowInt32Array is immutable. You need to use
+ * #GArrowInt32ArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowInt32Array,               \
+              garrow_int32_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_int32_array_init(GArrowInt32Array *object)
+{
+}
+
+static void
+garrow_int32_array_class_init(GArrowInt32ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_int32_array_get_value:
+ * @array: A #GArrowInt32Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gint32
+garrow_int32_array_get_value(GArrowInt32Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Int32Array *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-array.h b/c_glib/arrow-glib/int32-array.h
new file mode 100644
index 0000000000000..cce2b41aafe26
--- /dev/null
+++ b/c_glib/arrow-glib/int32-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT32_ARRAY                  \
+  (garrow_int32_array_get_type())
+#define GARROW_INT32_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT32_ARRAY,   \
+                              GArrowInt32Array))
+#define GARROW_INT32_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT32_ARRAY,      \
+                           GArrowInt32ArrayClass))
+#define GARROW_IS_INT32_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT32_ARRAY))
+#define GARROW_IS_INT32_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT32_ARRAY))
+#define GARROW_INT32_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT32_ARRAY,    \
+                             GArrowInt32ArrayClass))
+
+typedef struct _GArrowInt32Array         GArrowInt32Array;
+typedef struct _GArrowInt32ArrayClass    GArrowInt32ArrayClass;
+
+/**
+ * GArrowInt32Array:
+ *
+ * It wraps `arrow::Int32Array`.
+ */
+struct _GArrowInt32Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowInt32ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_int32_array_get_type(void) G_GNUC_CONST;
+
+gint32 garrow_int32_array_get_value(GArrowInt32Array *array,
+                                  gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-data-type.cpp b/c_glib/arrow-glib/int32-data-type.cpp
new file mode 100644
index 0000000000000..add21135364f9
--- /dev/null
+++ b/c_glib/arrow-glib/int32-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/int32-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int32-data-type
+ * @short_description: 32-bit integer data type
+ *
+ * #GArrowInt32DataType is a class for 32-bit integer data type.
+ */
+
+G_DEFINE_TYPE(GArrowInt32DataType,                \
+              garrow_int32_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_int32_data_type_init(GArrowInt32DataType *object)
+{
+}
+
+static void
+garrow_int32_data_type_class_init(GArrowInt32DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_int32_data_type_new:
+ *
+ * Returns: The newly created 32-bit integer data type.
+ */
+GArrowInt32DataType *
+garrow_int32_data_type_new(void)
+{
+  auto arrow_data_type = arrow::int32();
+
+  GArrowInt32DataType *data_type =
+    GARROW_INT32_DATA_TYPE(g_object_new(GARROW_TYPE_INT32_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-data-type.h b/c_glib/arrow-glib/int32-data-type.h
new file mode 100644
index 0000000000000..75cccbd40560d
--- /dev/null
+++ b/c_glib/arrow-glib/int32-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT32_DATA_TYPE            \
+  (garrow_int32_data_type_get_type())
+#define GARROW_INT32_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT32_DATA_TYPE,     \
+                              GArrowInt32DataType))
+#define GARROW_INT32_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT32_DATA_TYPE,        \
+                           GArrowInt32DataTypeClass))
+#define GARROW_IS_INT32_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT32_DATA_TYPE))
+#define GARROW_IS_INT32_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT32_DATA_TYPE))
+#define GARROW_INT32_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT32_DATA_TYPE,      \
+                             GArrowInt32DataTypeClass))
+
+typedef struct _GArrowInt32DataType         GArrowInt32DataType;
+typedef struct _GArrowInt32DataTypeClass    GArrowInt32DataTypeClass;
+
+/**
+ * GArrowInt32DataType:
+ *
+ * It wraps `arrow::Int32Type`.
+ */
+struct _GArrowInt32DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt32DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int32_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt32DataType *garrow_int32_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-array-builder.cpp b/c_glib/arrow-glib/int64-array-builder.cpp
new file mode 100644
index 0000000000000..b5eff114f92c9
--- /dev/null
+++ b/c_glib/arrow-glib/int64-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/int64-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int64-array-builder
+ * @short_description: 64-bit integer array builder class
+ *
+ * #GArrowInt64ArrayBuilder is the class to create a new
+ * #GArrowInt64Array.
+ */
+
+G_DEFINE_TYPE(GArrowInt64ArrayBuilder,
+              garrow_int64_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int64_array_builder_init(GArrowInt64ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int64_array_builder_class_init(GArrowInt64ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int64_array_builder_new:
+ *
+ * Returns: A newly created #GArrowInt64ArrayBuilder.
+ */
+GArrowInt64ArrayBuilder *
+garrow_int64_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::Int64Builder>(memory_pool, arrow::int64());
+  auto builder =
+    GARROW_INT64_ARRAY_BUILDER(g_object_new(GARROW_TYPE_INT64_ARRAY_BUILDER,
+                                           "array-builder", &arrow_builder,
+                                           NULL));
+  return builder;
+}
+
+/**
+ * garrow_int64_array_builder_append:
+ * @builder: A #GArrowInt64ArrayBuilder.
+ * @value: A int64 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
+                                  gint64 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int64Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int64-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_int64_array_builder_append_null:
+ * @builder: A #GArrowInt64ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
+                                      GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int64Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int64-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-array-builder.h b/c_glib/arrow-glib/int64-array-builder.h
new file mode 100644
index 0000000000000..8f4947eb7d9b1
--- /dev/null
+++ b/c_glib/arrow-glib/int64-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT64_ARRAY_BUILDER          \
+  (garrow_int64_array_builder_get_type())
+#define GARROW_INT64_ARRAY_BUILDER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT64_ARRAY_BUILDER,   \
+                              GArrowInt64ArrayBuilder))
+#define GARROW_INT64_ARRAY_BUILDER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT64_ARRAY_BUILDER,      \
+                           GArrowInt64ArrayBuilderClass))
+#define GARROW_IS_INT64_ARRAY_BUILDER(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT64_ARRAY_BUILDER))
+#define GARROW_IS_INT64_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT64_ARRAY_BUILDER))
+#define GARROW_INT64_ARRAY_BUILDER_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT64_ARRAY_BUILDER,    \
+                             GArrowInt64ArrayBuilderClass))
+
+typedef struct _GArrowInt64ArrayBuilder         GArrowInt64ArrayBuilder;
+typedef struct _GArrowInt64ArrayBuilderClass    GArrowInt64ArrayBuilderClass;
+
+/**
+ * GArrowInt64ArrayBuilder:
+ *
+ * It wraps `arrow::Int64Builder`.
+ */
+struct _GArrowInt64ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowInt64ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int64_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowInt64ArrayBuilder *garrow_int64_array_builder_new(void);
+
+gboolean garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
+                                          gint64 value,
+                                          GError **error);
+gboolean garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
+                                               GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-array.cpp b/c_glib/arrow-glib/int64-array.cpp
new file mode 100644
index 0000000000000..be49d5bf35251
--- /dev/null
+++ b/c_glib/arrow-glib/int64-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/int64-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int64-array
+ * @short_description: 64-bit integer array class
+ *
+ * #GArrowInt64Array is a class for 64-bit integer array. It can store
+ * zero or more 64-bit integer data.
+ *
+ * #GArrowInt64Array is immutable. You need to use
+ * #GArrowInt64ArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowInt64Array,               \
+              garrow_int64_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_int64_array_init(GArrowInt64Array *object)
+{
+}
+
+static void
+garrow_int64_array_class_init(GArrowInt64ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_int64_array_get_value:
+ * @array: A #GArrowInt64Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gint64
+garrow_int64_array_get_value(GArrowInt64Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Int64Array *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-array.h b/c_glib/arrow-glib/int64-array.h
new file mode 100644
index 0000000000000..73d4c6453a6d5
--- /dev/null
+++ b/c_glib/arrow-glib/int64-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT64_ARRAY                  \
+  (garrow_int64_array_get_type())
+#define GARROW_INT64_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT64_ARRAY,   \
+                              GArrowInt64Array))
+#define GARROW_INT64_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT64_ARRAY,      \
+                           GArrowInt64ArrayClass))
+#define GARROW_IS_INT64_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT64_ARRAY))
+#define GARROW_IS_INT64_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT64_ARRAY))
+#define GARROW_INT64_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT64_ARRAY,    \
+                             GArrowInt64ArrayClass))
+
+typedef struct _GArrowInt64Array         GArrowInt64Array;
+typedef struct _GArrowInt64ArrayClass    GArrowInt64ArrayClass;
+
+/**
+ * GArrowInt64Array:
+ *
+ * It wraps `arrow::Int64Array`.
+ */
+struct _GArrowInt64Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowInt64ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_int64_array_get_type(void) G_GNUC_CONST;
+
+gint64 garrow_int64_array_get_value(GArrowInt64Array *array,
+                                    gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-data-type.cpp b/c_glib/arrow-glib/int64-data-type.cpp
new file mode 100644
index 0000000000000..8e85b9d2ab922
--- /dev/null
+++ b/c_glib/arrow-glib/int64-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/int64-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int64-data-type
+ * @short_description: 64-bit integer data type
+ *
+ * #GArrowInt64DataType is a class for 64-bit integer data type.
+ */
+
+G_DEFINE_TYPE(GArrowInt64DataType,                \
+              garrow_int64_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_int64_data_type_init(GArrowInt64DataType *object)
+{
+}
+
+static void
+garrow_int64_data_type_class_init(GArrowInt64DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_int64_data_type_new:
+ *
+ * Returns: The newly created 64-bit integer data type.
+ */
+GArrowInt64DataType *
+garrow_int64_data_type_new(void)
+{
+  auto arrow_data_type = arrow::int64();
+
+  GArrowInt64DataType *data_type =
+    GARROW_INT64_DATA_TYPE(g_object_new(GARROW_TYPE_INT64_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-data-type.h b/c_glib/arrow-glib/int64-data-type.h
new file mode 100644
index 0000000000000..499e79f7ab7a7
--- /dev/null
+++ b/c_glib/arrow-glib/int64-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT64_DATA_TYPE            \
+  (garrow_int64_data_type_get_type())
+#define GARROW_INT64_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT64_DATA_TYPE,     \
+                              GArrowInt64DataType))
+#define GARROW_INT64_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT64_DATA_TYPE,        \
+                           GArrowInt64DataTypeClass))
+#define GARROW_IS_INT64_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT64_DATA_TYPE))
+#define GARROW_IS_INT64_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT64_DATA_TYPE))
+#define GARROW_INT64_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT64_DATA_TYPE,      \
+                             GArrowInt64DataTypeClass))
+
+typedef struct _GArrowInt64DataType         GArrowInt64DataType;
+typedef struct _GArrowInt64DataTypeClass    GArrowInt64DataTypeClass;
+
+/**
+ * GArrowInt64DataType:
+ *
+ * It wraps `arrow::Int64Type`.
+ */
+struct _GArrowInt64DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt64DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int64_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt64DataType *garrow_int64_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-array-builder.cpp b/c_glib/arrow-glib/int8-array-builder.cpp
new file mode 100644
index 0000000000000..5107a6fae1f6a
--- /dev/null
+++ b/c_glib/arrow-glib/int8-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/int8-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int8-array-builder
+ * @short_description: 8-bit integer array builder class
+ *
+ * #GArrowInt8ArrayBuilder is the class to create a new
+ * #GArrowInt8Array.
+ */
+
+G_DEFINE_TYPE(GArrowInt8ArrayBuilder,
+              garrow_int8_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int8_array_builder_init(GArrowInt8ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int8_array_builder_class_init(GArrowInt8ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int8_array_builder_new:
+ *
+ * Returns: A newly created #GArrowInt8ArrayBuilder.
+ */
+GArrowInt8ArrayBuilder *
+garrow_int8_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::Int8Builder>(memory_pool, arrow::int8());
+  auto builder =
+    GARROW_INT8_ARRAY_BUILDER(g_object_new(GARROW_TYPE_INT8_ARRAY_BUILDER,
+                                           "array-builder", &arrow_builder,
+                                           NULL));
+  return builder;
+}
+
+/**
+ * garrow_int8_array_builder_append:
+ * @builder: A #GArrowInt8ArrayBuilder.
+ * @value: A int8 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
+                                 gint8 value,
+                                 GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int8Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int8-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_int8_array_builder_append_null:
+ * @builder: A #GArrowInt8ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
+                                      GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int8Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int8-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-array-builder.h b/c_glib/arrow-glib/int8-array-builder.h
new file mode 100644
index 0000000000000..321e9310a6447
--- /dev/null
+++ b/c_glib/arrow-glib/int8-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT8_ARRAY_BUILDER          \
+  (garrow_int8_array_builder_get_type())
+#define GARROW_INT8_ARRAY_BUILDER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT8_ARRAY_BUILDER,   \
+                              GArrowInt8ArrayBuilder))
+#define GARROW_INT8_ARRAY_BUILDER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT8_ARRAY_BUILDER,      \
+                           GArrowInt8ArrayBuilderClass))
+#define GARROW_IS_INT8_ARRAY_BUILDER(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT8_ARRAY_BUILDER))
+#define GARROW_IS_INT8_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT8_ARRAY_BUILDER))
+#define GARROW_INT8_ARRAY_BUILDER_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT8_ARRAY_BUILDER,    \
+                             GArrowInt8ArrayBuilderClass))
+
+typedef struct _GArrowInt8ArrayBuilder         GArrowInt8ArrayBuilder;
+typedef struct _GArrowInt8ArrayBuilderClass    GArrowInt8ArrayBuilderClass;
+
+/**
+ * GArrowInt8ArrayBuilder:
+ *
+ * It wraps `arrow::Int8Builder`.
+ */
+struct _GArrowInt8ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowInt8ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int8_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowInt8ArrayBuilder *garrow_int8_array_builder_new(void);
+
+gboolean garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
+                                          gint8 value,
+                                          GError **error);
+gboolean garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
+                                               GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-array.cpp b/c_glib/arrow-glib/int8-array.cpp
new file mode 100644
index 0000000000000..d3f12ece9bbf7
--- /dev/null
+++ b/c_glib/arrow-glib/int8-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/int8-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int8-array
+ * @short_description: 8-bit integer array class
+ *
+ * #GArrowInt8Array is a class for 8-bit integer array. It can store
+ * zero or more 8-bit integer data.
+ *
+ * #GArrowInt8Array is immutable. You need to use
+ * #GArrowInt8ArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowInt8Array,               \
+              garrow_int8_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_int8_array_init(GArrowInt8Array *object)
+{
+}
+
+static void
+garrow_int8_array_class_init(GArrowInt8ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_int8_array_get_value:
+ * @array: A #GArrowInt8Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gint8
+garrow_int8_array_get_value(GArrowInt8Array *array,
+                            gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Int8Array *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-array.h b/c_glib/arrow-glib/int8-array.h
new file mode 100644
index 0000000000000..0e1e901f4fdb6
--- /dev/null
+++ b/c_glib/arrow-glib/int8-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT8_ARRAY                  \
+  (garrow_int8_array_get_type())
+#define GARROW_INT8_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT8_ARRAY,   \
+                              GArrowInt8Array))
+#define GARROW_INT8_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT8_ARRAY,      \
+                           GArrowInt8ArrayClass))
+#define GARROW_IS_INT8_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT8_ARRAY))
+#define GARROW_IS_INT8_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT8_ARRAY))
+#define GARROW_INT8_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT8_ARRAY,    \
+                             GArrowInt8ArrayClass))
+
+typedef struct _GArrowInt8Array         GArrowInt8Array;
+typedef struct _GArrowInt8ArrayClass    GArrowInt8ArrayClass;
+
+/**
+ * GArrowInt8Array:
+ *
+ * It wraps `arrow::Int8Array`.
+ */
+struct _GArrowInt8Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowInt8ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_int8_array_get_type(void) G_GNUC_CONST;
+
+gint8 garrow_int8_array_get_value(GArrowInt8Array *array,
+                                  gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-data-type.cpp b/c_glib/arrow-glib/int8-data-type.cpp
new file mode 100644
index 0000000000000..55b1ebc852d10
--- /dev/null
+++ b/c_glib/arrow-glib/int8-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/int8-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int8-data-type
+ * @short_description: 8-bit integer data type
+ *
+ * #GArrowInt8DataType is a class for 8-bit integer data type.
+ */
+
+G_DEFINE_TYPE(GArrowInt8DataType,                \
+              garrow_int8_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_int8_data_type_init(GArrowInt8DataType *object)
+{
+}
+
+static void
+garrow_int8_data_type_class_init(GArrowInt8DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_int8_data_type_new:
+ *
+ * Returns: The newly created 8-bit integer data type.
+ */
+GArrowInt8DataType *
+garrow_int8_data_type_new(void)
+{
+  auto arrow_data_type = arrow::int8();
+
+  GArrowInt8DataType *data_type =
+    GARROW_INT8_DATA_TYPE(g_object_new(GARROW_TYPE_INT8_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-data-type.h b/c_glib/arrow-glib/int8-data-type.h
new file mode 100644
index 0000000000000..4343bd17a725b
--- /dev/null
+++ b/c_glib/arrow-glib/int8-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT8_DATA_TYPE            \
+  (garrow_int8_data_type_get_type())
+#define GARROW_INT8_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT8_DATA_TYPE,     \
+                              GArrowInt8DataType))
+#define GARROW_INT8_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT8_DATA_TYPE,        \
+                           GArrowInt8DataTypeClass))
+#define GARROW_IS_INT8_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT8_DATA_TYPE))
+#define GARROW_IS_INT8_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT8_DATA_TYPE))
+#define GARROW_INT8_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT8_DATA_TYPE,      \
+                             GArrowInt8DataTypeClass))
+
+typedef struct _GArrowInt8DataType         GArrowInt8DataType;
+typedef struct _GArrowInt8DataTypeClass    GArrowInt8DataTypeClass;
+
+/**
+ * GArrowInt8DataType:
+ *
+ * It wraps `arrow::Int8Type`.
+ */
+struct _GArrowInt8DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt8DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int8_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt8DataType *garrow_int8_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-enums.c.template b/c_glib/arrow-glib/io-enums.c.template
new file mode 100644
index 0000000000000..10ee77588d98b
--- /dev/null
+++ b/c_glib/arrow-glib/io-enums.c.template
@@ -0,0 +1,56 @@
+/*** BEGIN file-header ***/
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/arrow-io-glib.h>
+/*** END file-header ***/
+
+/*** BEGIN file-production ***/
+
+/* enumerations from "@filename@" */
+/*** END file-production ***/
+
+/*** BEGIN value-header ***/
+GType
+@enum_name@_get_type(void)
+{
+  static GType etype = 0;
+  if (G_UNLIKELY(etype == 0)) {
+    static const G@Type@Value values[] = {
+/*** END value-header ***/
+
+/*** BEGIN value-production ***/
+      {@VALUENAME@, "@VALUENAME@", "@valuenick@"},
+/*** END value-production ***/
+
+/*** BEGIN value-tail ***/
+      {0, NULL, NULL}
+    };
+    etype = g_@type@_register_static(g_intern_static_string("@EnumName@"), values);
+  }
+  return etype;
+}
+/*** END value-tail ***/
+
+/*** BEGIN file-tail ***/
+/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/io-enums.h.template b/c_glib/arrow-glib/io-enums.h.template
new file mode 100644
index 0000000000000..429141dc76a60
--- /dev/null
+++ b/c_glib/arrow-glib/io-enums.h.template
@@ -0,0 +1,41 @@
+/*** BEGIN file-header ***/
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/io-file-mode.h>
+
+G_BEGIN_DECLS
+/*** END file-header ***/
+
+/*** BEGIN file-production ***/
+
+/* enumerations from "@filename@" */
+/*** END file-production ***/
+
+/*** BEGIN value-header ***/
+GType @enum_name@_get_type(void) G_GNUC_CONST;
+#define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
+/*** END value-header ***/
+
+/*** BEGIN file-tail ***/
+
+G_END_DECLS
+/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/io-file-mode.cpp b/c_glib/arrow-glib/io-file-mode.cpp
new file mode 100644
index 0000000000000..7998d3f5bb061
--- /dev/null
+++ b/c_glib/arrow-glib/io-file-mode.cpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/io-file-mode.hpp>
+
+/**
+ * SECTION: io-file-mode
+ * @title: GArrowIOFileMode
+ * @short_description: File mode mapping between Arrow and arrow-glib
+ *
+ * #GArrowIOFileMode provides file modes corresponding to
+ * `arrow::io::FileMode::type` values.
+ */
+
+GArrowIOFileMode
+garrow_io_file_mode_from_raw(arrow::io::FileMode::type mode)
+{
+  switch (mode) {
+  case arrow::io::FileMode::type::READ:
+    return GARROW_IO_FILE_MODE_READ;
+  case arrow::io::FileMode::type::WRITE:
+    return GARROW_IO_FILE_MODE_WRITE;
+  case arrow::io::FileMode::type::READWRITE:
+    return GARROW_IO_FILE_MODE_READWRITE;
+  default:
+    return GARROW_IO_FILE_MODE_READ;
+  }
+}
+
+arrow::io::FileMode::type
+garrow_io_file_mode_to_raw(GArrowIOFileMode mode)
+{
+  switch (mode) {
+  case GARROW_IO_FILE_MODE_READ:
+    return arrow::io::FileMode::type::READ;
+  case GARROW_IO_FILE_MODE_WRITE:
+    return arrow::io::FileMode::type::WRITE;
+  case GARROW_IO_FILE_MODE_READWRITE:
+    return arrow::io::FileMode::type::READWRITE;
+  default:
+    return arrow::io::FileMode::type::READ;
+  }
+}
diff --git a/c_glib/arrow-glib/io-file-mode.h b/c_glib/arrow-glib/io-file-mode.h
new file mode 100644
index 0000000000000..03eca353bbdbb
--- /dev/null
+++ b/c_glib/arrow-glib/io-file-mode.h
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+/**
+ * GArrowIOFileMode:
+ * @GARROW_IO_FILE_MODE_READ: For read.
+ * @GARROW_IO_FILE_MODE_WRITE: For write.
+ * @GARROW_IO_FILE_MODE_READWRITE: For read-write.
+ *
+ * They are corresponding to `arrow::io::FileMode::type` values.
+ */
+typedef enum {
+  GARROW_IO_FILE_MODE_READ,
+  GARROW_IO_FILE_MODE_WRITE,
+  GARROW_IO_FILE_MODE_READWRITE
+} GArrowIOFileMode;
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-file-mode.hpp b/c_glib/arrow-glib/io-file-mode.hpp
new file mode 100644
index 0000000000000..b3d8ac6d8e053
--- /dev/null
+++ b/c_glib/arrow-glib/io-file-mode.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/io/interfaces.h>
+
+#include <arrow-glib/io-file-mode.h>
+
+GArrowIOFileMode garrow_io_file_mode_from_raw(arrow::io::FileMode::type mode);
+arrow::io::FileMode::type garrow_io_file_mode_to_raw(GArrowIOFileMode mode);
diff --git a/c_glib/arrow-glib/io-file-output-stream.cpp b/c_glib/arrow-glib/io-file-output-stream.cpp
new file mode 100644
index 0000000000000..673e8cd36a60a
--- /dev/null
+++ b/c_glib/arrow-glib/io-file-output-stream.cpp
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/io/file.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-file.hpp>
+#include <arrow-glib/io-file-output-stream.hpp>
+#include <arrow-glib/io-output-stream.hpp>
+#include <arrow-glib/io-writeable.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-file-output-stream
+ * @short_description: A file output stream.
+ *
+ * The #GArrowIOFileOutputStream is a class for file output stream.
+ */
+
+typedef struct GArrowIOFileOutputStreamPrivate_ {
+  std::shared_ptr<arrow::io::FileOutputStream> file_output_stream;
+} GArrowIOFileOutputStreamPrivate;
+
+enum {
+  PROP_0,
+  PROP_FILE_OUTPUT_STREAM
+};
+
+static std::shared_ptr<arrow::io::FileInterface>
+garrow_io_file_output_stream_get_raw_file_interface(GArrowIOFile *file)
+{
+  auto file_output_stream = GARROW_IO_FILE_OUTPUT_STREAM(file);
+  auto arrow_file_output_stream =
+    garrow_io_file_output_stream_get_raw(file_output_stream);
+  return arrow_file_output_stream;
+}
+
+static void
+garrow_io_file_interface_init(GArrowIOFileInterface *iface)
+{
+  iface->get_raw = garrow_io_file_output_stream_get_raw_file_interface;
+}
+
+static std::shared_ptr<arrow::io::Writeable>
+garrow_io_file_output_stream_get_raw_writeable_interface(GArrowIOWriteable *writeable)
+{
+  auto file_output_stream = GARROW_IO_FILE_OUTPUT_STREAM(writeable);
+  auto arrow_file_output_stream =
+    garrow_io_file_output_stream_get_raw(file_output_stream);
+  return arrow_file_output_stream;
+}
+
+static void
+garrow_io_writeable_interface_init(GArrowIOWriteableInterface *iface)
+{
+  iface->get_raw = garrow_io_file_output_stream_get_raw_writeable_interface;
+}
+
+static std::shared_ptr<arrow::io::OutputStream>
+garrow_io_file_output_stream_get_raw_output_stream_interface(GArrowIOOutputStream *output_stream)
+{
+  auto file_output_stream = GARROW_IO_FILE_OUTPUT_STREAM(output_stream);
+  auto arrow_file_output_stream =
+    garrow_io_file_output_stream_get_raw(file_output_stream);
+  return arrow_file_output_stream;
+}
+
+static void
+garrow_io_output_stream_interface_init(GArrowIOOutputStreamInterface *iface)
+{
+  iface->get_raw = garrow_io_file_output_stream_get_raw_output_stream_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(GArrowIOFileOutputStream,
+                        garrow_io_file_output_stream,
+                        G_TYPE_OBJECT,
+                        G_ADD_PRIVATE(GArrowIOFileOutputStream)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_FILE,
+                                              garrow_io_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_WRITEABLE,
+                                              garrow_io_writeable_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_OUTPUT_STREAM,
+                                              garrow_io_output_stream_interface_init));
+
+#define GARROW_IO_FILE_OUTPUT_STREAM_GET_PRIVATE(obj)                   \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
+                               GARROW_IO_TYPE_FILE_OUTPUT_STREAM,       \
+                               GArrowIOFileOutputStreamPrivate))
+
+static void
+garrow_io_file_output_stream_finalize(GObject *object)
+{
+  GArrowIOFileOutputStreamPrivate *priv;
+
+  priv = GARROW_IO_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
+
+  priv->file_output_stream = nullptr;
+
+  G_OBJECT_CLASS(garrow_io_file_output_stream_parent_class)->finalize(object);
+}
+
+static void
+garrow_io_file_output_stream_set_property(GObject *object,
+                                          guint prop_id,
+                                          const GValue *value,
+                                          GParamSpec *pspec)
+{
+  GArrowIOFileOutputStreamPrivate *priv;
+
+  priv = GARROW_IO_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_FILE_OUTPUT_STREAM:
+    priv->file_output_stream =
+      *static_cast<std::shared_ptr<arrow::io::FileOutputStream> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_io_file_output_stream_get_property(GObject *object,
+                                          guint prop_id,
+                                          GValue *value,
+                                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_io_file_output_stream_init(GArrowIOFileOutputStream *object)
+{
+}
+
+static void
+garrow_io_file_output_stream_class_init(GArrowIOFileOutputStreamClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_io_file_output_stream_finalize;
+  gobject_class->set_property = garrow_io_file_output_stream_set_property;
+  gobject_class->get_property = garrow_io_file_output_stream_get_property;
+
+  spec = g_param_spec_pointer("file-output-stream",
+                              "io::FileOutputStream",
+                              "The raw std::shared<arrow::io::FileOutputStream> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_FILE_OUTPUT_STREAM, spec);
+}
+
+/**
+ * garrow_io_file_output_stream_open:
+ * @path: The path of the file output stream.
+ * @append: Whether the path is opened as append mode or recreate mode.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): A newly opened
+ *   #GArrowIOFileOutputStream or %NULL on error.
+ */
+GArrowIOFileOutputStream *
+garrow_io_file_output_stream_open(const gchar *path,
+                                  gboolean append,
+                                  GError **error)
+{
+  std::shared_ptr<arrow::io::FileOutputStream> arrow_file_output_stream;
+  auto status =
+    arrow::io::FileOutputStream::Open(std::string(path),
+                                      append,
+                                      &arrow_file_output_stream);
+  if (status.ok()) {
+    return garrow_io_file_output_stream_new_raw(&arrow_file_output_stream);
+  } else {
+    std::string context("[io][file-output-stream][open]: <");
+    context += path;
+    context += ">";
+    garrow_error_set(error, status, context.c_str());
+    return NULL;
+  }
+}
+
+G_END_DECLS
+
+GArrowIOFileOutputStream *
+garrow_io_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream)
+{
+  auto file_output_stream =
+    GARROW_IO_FILE_OUTPUT_STREAM(g_object_new(GARROW_IO_TYPE_FILE_OUTPUT_STREAM,
+                                              "file-output-stream", arrow_file_output_stream,
+                                              NULL));
+  return file_output_stream;
+}
+
+std::shared_ptr<arrow::io::FileOutputStream>
+garrow_io_file_output_stream_get_raw(GArrowIOFileOutputStream *file_output_stream)
+{
+  GArrowIOFileOutputStreamPrivate *priv;
+
+  priv = GARROW_IO_FILE_OUTPUT_STREAM_GET_PRIVATE(file_output_stream);
+  return priv->file_output_stream;
+}
diff --git a/c_glib/arrow-glib/io-file-output-stream.h b/c_glib/arrow-glib/io-file-output-stream.h
new file mode 100644
index 0000000000000..032b125544e77
--- /dev/null
+++ b/c_glib/arrow-glib/io-file-output-stream.h
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_FILE_OUTPUT_STREAM       \
+  (garrow_io_file_output_stream_get_type())
+#define GARROW_IO_FILE_OUTPUT_STREAM(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_IO_TYPE_FILE_OUTPUT_STREAM,        \
+                              GArrowIOFileOutputStream))
+#define GARROW_IO_FILE_OUTPUT_STREAM_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_IO_TYPE_FILE_OUTPUT_STREAM,   \
+                           GArrowIOFileOutputStreamClass))
+#define GARROW_IO_IS_FILE_OUTPUT_STREAM(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_IO_TYPE_FILE_OUTPUT_STREAM))
+#define GARROW_IO_IS_FILE_OUTPUT_STREAM_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_IO_TYPE_FILE_OUTPUT_STREAM))
+#define GARROW_IO_FILE_OUTPUT_STREAM_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_IO_TYPE_FILE_OUTPUT_STREAM, \
+                             GArrowIOFileOutputStreamClass))
+
+typedef struct _GArrowIOFileOutputStream         GArrowIOFileOutputStream;
+typedef struct _GArrowIOFileOutputStreamClass    GArrowIOFileOutputStreamClass;
+
+/**
+ * GArrowIOFileOutputStream:
+ *
+ * It wraps `arrow::io::FileOutputStream`.
+ */
+struct _GArrowIOFileOutputStream
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowIOFileOutputStreamClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_io_file_output_stream_get_type(void) G_GNUC_CONST;
+
+GArrowIOFileOutputStream *garrow_io_file_output_stream_open(const gchar *path,
+                                                            gboolean append,
+                                                            GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-file-output-stream.hpp b/c_glib/arrow-glib/io-file-output-stream.hpp
new file mode 100644
index 0000000000000..76b8e91f6cf43
--- /dev/null
+++ b/c_glib/arrow-glib/io-file-output-stream.hpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+#include <arrow/io/file.h>
+
+#include <arrow-glib/io-file-output-stream.h>
+
+GArrowIOFileOutputStream *garrow_io_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream);
+std::shared_ptr<arrow::io::FileOutputStream> garrow_io_file_output_stream_get_raw(GArrowIOFileOutputStream *file_output_stream);
diff --git a/c_glib/arrow-glib/io-file.cpp b/c_glib/arrow-glib/io-file.cpp
new file mode 100644
index 0000000000000..536ae3e705f59
--- /dev/null
+++ b/c_glib/arrow-glib/io-file.cpp
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-file.hpp>
+#include <arrow-glib/io-file-mode.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-file
+ * @title: GArrowIOFile
+ * @short_description: File interface
+ *
+ * #GArrowIOFile is an interface for file.
+ */
+
+G_DEFINE_INTERFACE(GArrowIOFile,
+                   garrow_io_file,
+                   G_TYPE_OBJECT)
+
+static void
+garrow_io_file_default_init (GArrowIOFileInterface *iface)
+{
+}
+
+/**
+ * garrow_io_file_close:
+ * @file: A #GArrowIOFile.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_io_file_close(GArrowIOFile *file,
+                     GError **error)
+{
+  auto arrow_file = garrow_io_file_get_raw(file);
+
+  auto status = arrow_file->Close();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[io][file][close]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_io_file_tell:
+ * @file: A #GArrowIOFile.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: The current offset on success, -1 if there was an error.
+ */
+gint64
+garrow_io_file_tell(GArrowIOFile *file,
+                    GError **error)
+{
+  auto arrow_file = garrow_io_file_get_raw(file);
+
+  gint64 position;
+  auto status = arrow_file->Tell(&position);
+  if (status.ok()) {
+    return position;
+  } else {
+    garrow_error_set(error, status, "[io][file][tell]");
+    return -1;
+  }
+}
+
+/**
+ * garrow_io_file_get_mode:
+ * @file: A #GArrowIOFile.
+ *
+ * Returns: The mode of the file.
+ */
+GArrowIOFileMode
+garrow_io_file_get_mode(GArrowIOFile *file)
+{
+  auto arrow_file = garrow_io_file_get_raw(file);
+
+  auto arrow_mode = arrow_file->mode();
+  return garrow_io_file_mode_from_raw(arrow_mode);
+}
+
+G_END_DECLS
+
+std::shared_ptr<arrow::io::FileInterface>
+garrow_io_file_get_raw(GArrowIOFile *file)
+{
+  auto *iface = GARROW_IO_FILE_GET_IFACE(file);
+  return iface->get_raw(file);
+}
diff --git a/c_glib/arrow-glib/io-file.h b/c_glib/arrow-glib/io-file.h
new file mode 100644
index 0000000000000..9fa0ec137566f
--- /dev/null
+++ b/c_glib/arrow-glib/io-file.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/io-file-mode.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_FILE                     \
+  (garrow_io_file_get_type())
+#define GARROW_IO_FILE(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_IO_TYPE_FILE,      \
+                              GArrowIOFileInterface))
+#define GARROW_IO_IS_FILE(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_IO_TYPE_FILE))
+#define GARROW_IO_FILE_GET_IFACE(obj)                           \
+  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
+                                 GARROW_IO_TYPE_FILE,           \
+                                 GArrowIOFileInterface))
+
+typedef struct _GArrowIOFile          GArrowIOFile;
+typedef struct _GArrowIOFileInterface GArrowIOFileInterface;
+
+GType garrow_io_file_get_type(void) G_GNUC_CONST;
+
+gboolean garrow_io_file_close(GArrowIOFile *file,
+                              GError **error);
+gint64 garrow_io_file_tell(GArrowIOFile *file,
+                           GError **error);
+GArrowIOFileMode garrow_io_file_get_mode(GArrowIOFile *file);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-file.hpp b/c_glib/arrow-glib/io-file.hpp
new file mode 100644
index 0000000000000..afaca90a10fa3
--- /dev/null
+++ b/c_glib/arrow-glib/io-file.hpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/io/interfaces.h>
+
+#include <arrow-glib/io-file.h>
+
+/**
+ * GArrowIOFileInterface:
+ *
+ * It wraps `arrow::io::FileInterface`.
+ */
+struct _GArrowIOFileInterface
+{
+  GTypeInterface parent_iface;
+
+  std::shared_ptr<arrow::io::FileInterface> (*get_raw)(GArrowIOFile *file);
+};
+
+std::shared_ptr<arrow::io::FileInterface> garrow_io_file_get_raw(GArrowIOFile *file);
diff --git a/c_glib/arrow-glib/io-input-stream.cpp b/c_glib/arrow-glib/io-input-stream.cpp
new file mode 100644
index 0000000000000..a28b9c6556ccd
--- /dev/null
+++ b/c_glib/arrow-glib/io-input-stream.cpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-input-stream.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-input-stream
+ * @title: GArrowIOInputStream
+ * @short_description: Stream input interface
+ *
+ * #GArrowIOInputStream is an interface for stream input. Stream input
+ * is file based and readable.
+ */
+
+G_DEFINE_INTERFACE(GArrowIOInputStream,
+                   garrow_io_input_stream,
+                   G_TYPE_OBJECT)
+
+static void
+garrow_io_input_stream_default_init (GArrowIOInputStreamInterface *iface)
+{
+}
+
+G_END_DECLS
+
+std::shared_ptr<arrow::io::InputStream>
+garrow_io_input_stream_get_raw(GArrowIOInputStream *input_stream)
+{
+  auto *iface = GARROW_IO_INPUT_STREAM_GET_IFACE(input_stream);
+  return iface->get_raw(input_stream);
+}
diff --git a/c_glib/arrow-glib/io-input-stream.h b/c_glib/arrow-glib/io-input-stream.h
new file mode 100644
index 0000000000000..a7f06819b4f97
--- /dev/null
+++ b/c_glib/arrow-glib/io-input-stream.h
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_INPUT_STREAM             \
+  (garrow_io_input_stream_get_type())
+#define GARROW_IO_INPUT_STREAM(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IO_TYPE_INPUT_STREAM,      \
+                              GArrowIOInputStreamInterface))
+#define GARROW_IO_IS_INPUT_STREAM(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IO_TYPE_INPUT_STREAM))
+#define GARROW_IO_INPUT_STREAM_GET_IFACE(obj)                   \
+  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
+                                 GARROW_IO_TYPE_INPUT_STREAM,   \
+                                 GArrowIOInputStreamInterface))
+
+typedef struct _GArrowIOInputStream          GArrowIOInputStream;
+typedef struct _GArrowIOInputStreamInterface GArrowIOInputStreamInterface;
+
+GType garrow_io_input_stream_get_type(void) G_GNUC_CONST;
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-input-stream.hpp b/c_glib/arrow-glib/io-input-stream.hpp
new file mode 100644
index 0000000000000..3b1de5da5c226
--- /dev/null
+++ b/c_glib/arrow-glib/io-input-stream.hpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/io/interfaces.h>
+
+#include <arrow-glib/io-input-stream.h>
+
+/**
+ * GArrowIOInputStreamInterface:
+ *
+ * It wraps `arrow::io::InputStream`.
+ */
+struct _GArrowIOInputStreamInterface
+{
+  GTypeInterface parent_iface;
+
+  std::shared_ptr<arrow::io::InputStream> (*get_raw)(GArrowIOInputStream *file);
+};
+
+std::shared_ptr<arrow::io::InputStream> garrow_io_input_stream_get_raw(GArrowIOInputStream *input_stream);
diff --git a/c_glib/arrow-glib/io-memory-mapped-file.cpp b/c_glib/arrow-glib/io-memory-mapped-file.cpp
new file mode 100644
index 0000000000000..aa6ae2afd6e78
--- /dev/null
+++ b/c_glib/arrow-glib/io-memory-mapped-file.cpp
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/io/file.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-file.hpp>
+#include <arrow-glib/io-file-mode.hpp>
+#include <arrow-glib/io-input-stream.hpp>
+#include <arrow-glib/io-memory-mapped-file.hpp>
+#include <arrow-glib/io-readable.hpp>
+#include <arrow-glib/io-readable-file.hpp>
+#include <arrow-glib/io-writeable.hpp>
+#include <arrow-glib/io-writeable-file.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-memory-mapped-file
+ * @short_description: Memory mapped file class
+ *
+ * #GArrowIOMemoryMappedFile is a class for memory mapped file. It's
+ * readable and writeable. It supports zero copy.
+ */
+
+typedef struct GArrowIOMemoryMappedFilePrivate_ {
+  std::shared_ptr<arrow::io::MemoryMappedFile> memory_mapped_file;
+} GArrowIOMemoryMappedFilePrivate;
+
+enum {
+  PROP_0,
+  PROP_MEMORY_MAPPED_FILE
+};
+
+static std::shared_ptr<arrow::io::FileInterface>
+garrow_io_memory_mapped_file_get_raw_file_interface(GArrowIOFile *file)
+{
+  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(file);
+  auto arrow_memory_mapped_file =
+    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_io_file_interface_init(GArrowIOFileInterface *iface)
+{
+  iface->get_raw = garrow_io_memory_mapped_file_get_raw_file_interface;
+}
+
+static std::shared_ptr<arrow::io::Readable>
+garrow_io_memory_mapped_file_get_raw_readable_interface(GArrowIOReadable *readable)
+{
+  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(readable);
+  auto arrow_memory_mapped_file =
+    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_io_readable_interface_init(GArrowIOReadableInterface *iface)
+{
+  iface->get_raw = garrow_io_memory_mapped_file_get_raw_readable_interface;
+}
+
+static std::shared_ptr<arrow::io::InputStream>
+garrow_io_memory_mapped_file_get_raw_input_stream_interface(GArrowIOInputStream *input_stream)
+{
+  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(input_stream);
+  auto arrow_memory_mapped_file =
+    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_io_input_stream_interface_init(GArrowIOInputStreamInterface *iface)
+{
+  iface->get_raw = garrow_io_memory_mapped_file_get_raw_input_stream_interface;
+}
+
+static std::shared_ptr<arrow::io::ReadableFileInterface>
+garrow_io_memory_mapped_file_get_raw_readable_file_interface(GArrowIOReadableFile *file)
+{
+  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(file);
+  auto arrow_memory_mapped_file =
+    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_io_readable_file_interface_init(GArrowIOReadableFileInterface *iface)
+{
+  iface->get_raw = garrow_io_memory_mapped_file_get_raw_readable_file_interface;
+}
+
+static std::shared_ptr<arrow::io::Writeable>
+garrow_io_memory_mapped_file_get_raw_writeable_interface(GArrowIOWriteable *writeable)
+{
+  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(writeable);
+  auto arrow_memory_mapped_file =
+    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_io_writeable_interface_init(GArrowIOWriteableInterface *iface)
+{
+  iface->get_raw = garrow_io_memory_mapped_file_get_raw_writeable_interface;
+}
+
+static std::shared_ptr<arrow::io::WriteableFileInterface>
+garrow_io_memory_mapped_file_get_raw_writeable_file_interface(GArrowIOWriteableFile *file)
+{
+  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(file);
+  auto arrow_memory_mapped_file =
+    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_io_writeable_file_interface_init(GArrowIOWriteableFileInterface *iface)
+{
+  iface->get_raw = garrow_io_memory_mapped_file_get_raw_writeable_file_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(GArrowIOMemoryMappedFile,
+                        garrow_io_memory_mapped_file,
+                        G_TYPE_OBJECT,
+                        G_ADD_PRIVATE(GArrowIOMemoryMappedFile)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_FILE,
+                                              garrow_io_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_READABLE,
+                                              garrow_io_readable_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_INPUT_STREAM,
+                                              garrow_io_input_stream_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_READABLE_FILE,
+                                              garrow_io_readable_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_WRITEABLE,
+                                              garrow_io_writeable_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_WRITEABLE_FILE,
+                                              garrow_io_writeable_file_interface_init));
+
+#define GARROW_IO_MEMORY_MAPPED_FILE_GET_PRIVATE(obj)                   \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
+                               GARROW_IO_TYPE_MEMORY_MAPPED_FILE,       \
+                               GArrowIOMemoryMappedFilePrivate))
+
+static void
+garrow_io_memory_mapped_file_finalize(GObject *object)
+{
+  GArrowIOMemoryMappedFilePrivate *priv;
+
+  priv = GARROW_IO_MEMORY_MAPPED_FILE_GET_PRIVATE(object);
+
+  priv->memory_mapped_file = nullptr;
+
+  G_OBJECT_CLASS(garrow_io_memory_mapped_file_parent_class)->finalize(object);
+}
+
+static void
+garrow_io_memory_mapped_file_set_property(GObject *object,
+                                          guint prop_id,
+                                          const GValue *value,
+                                          GParamSpec *pspec)
+{
+  GArrowIOMemoryMappedFilePrivate *priv;
+
+  priv = GARROW_IO_MEMORY_MAPPED_FILE_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_MEMORY_MAPPED_FILE:
+    priv->memory_mapped_file =
+      *static_cast<std::shared_ptr<arrow::io::MemoryMappedFile> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_io_memory_mapped_file_get_property(GObject *object,
+                                          guint prop_id,
+                                          GValue *value,
+                                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_io_memory_mapped_file_init(GArrowIOMemoryMappedFile *object)
+{
+}
+
+static void
+garrow_io_memory_mapped_file_class_init(GArrowIOMemoryMappedFileClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_io_memory_mapped_file_finalize;
+  gobject_class->set_property = garrow_io_memory_mapped_file_set_property;
+  gobject_class->get_property = garrow_io_memory_mapped_file_get_property;
+
+  spec = g_param_spec_pointer("memory-mapped-file",
+                              "io::MemoryMappedFile",
+                              "The raw std::shared<arrow::io::MemoryMappedFile> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_MEMORY_MAPPED_FILE, spec);
+}
+
+/**
+ * garrow_io_memory_mapped_file_open:
+ * @path: The path of the memory mapped file.
+ * @mode: The mode of the memory mapped file.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): A newly opened
+ *   #GArrowIOMemoryMappedFile or %NULL on error.
+ */
+GArrowIOMemoryMappedFile *
+garrow_io_memory_mapped_file_open(const gchar *path,
+                                  GArrowIOFileMode mode,
+                                  GError **error)
+{
+  std::shared_ptr<arrow::io::MemoryMappedFile> arrow_memory_mapped_file;
+  auto status =
+    arrow::io::MemoryMappedFile::Open(std::string(path),
+                                      garrow_io_file_mode_to_raw(mode),
+                                      &arrow_memory_mapped_file);
+  if (status.ok()) {
+    return garrow_io_memory_mapped_file_new_raw(&arrow_memory_mapped_file);
+  } else {
+    std::string context("[io][memory-mapped-file][open]: <");
+    context += path;
+    context += ">";
+    garrow_error_set(error, status, context.c_str());
+    return NULL;
+  }
+}
+
+G_END_DECLS
+
+GArrowIOMemoryMappedFile *
+garrow_io_memory_mapped_file_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file)
+{
+  auto memory_mapped_file =
+    GARROW_IO_MEMORY_MAPPED_FILE(g_object_new(GARROW_IO_TYPE_MEMORY_MAPPED_FILE,
+                                              "memory-mapped-file", arrow_memory_mapped_file,
+                                              NULL));
+  return memory_mapped_file;
+}
+
+std::shared_ptr<arrow::io::MemoryMappedFile>
+garrow_io_memory_mapped_file_get_raw(GArrowIOMemoryMappedFile *memory_mapped_file)
+{
+  GArrowIOMemoryMappedFilePrivate *priv;
+
+  priv = GARROW_IO_MEMORY_MAPPED_FILE_GET_PRIVATE(memory_mapped_file);
+  return priv->memory_mapped_file;
+}
diff --git a/c_glib/arrow-glib/io-memory-mapped-file.h b/c_glib/arrow-glib/io-memory-mapped-file.h
new file mode 100644
index 0000000000000..0d2d6c2f835de
--- /dev/null
+++ b/c_glib/arrow-glib/io-memory-mapped-file.h
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/io-file-mode.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_MEMORY_MAPPED_FILE       \
+  (garrow_io_memory_mapped_file_get_type())
+#define GARROW_IO_MEMORY_MAPPED_FILE(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_IO_TYPE_MEMORY_MAPPED_FILE,        \
+                              GArrowIOMemoryMappedFile))
+#define GARROW_IO_MEMORY_MAPPED_FILE_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_IO_TYPE_MEMORY_MAPPED_FILE,   \
+                           GArrowIOMemoryMappedFileClass))
+#define GARROW_IO_IS_MEMORY_MAPPED_FILE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_IO_TYPE_MEMORY_MAPPED_FILE))
+#define GARROW_IO_IS_MEMORY_MAPPED_FILE_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_IO_TYPE_MEMORY_MAPPED_FILE))
+#define GARROW_IO_MEMORY_MAPPED_FILE_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_IO_TYPE_MEMORY_MAPPED_FILE, \
+                             GArrowIOMemoryMappedFileClass))
+
+typedef struct _GArrowIOMemoryMappedFile         GArrowIOMemoryMappedFile;
+typedef struct _GArrowIOMemoryMappedFileClass    GArrowIOMemoryMappedFileClass;
+
+/**
+ * GArrowIOMemoryMappedFile:
+ *
+ * It wraps `arrow::io::MemoryMappedFile`.
+ */
+struct _GArrowIOMemoryMappedFile
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowIOMemoryMappedFileClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_io_memory_mapped_file_get_type(void) G_GNUC_CONST;
+
+GArrowIOMemoryMappedFile *garrow_io_memory_mapped_file_open(const gchar *path,
+                                                            GArrowIOFileMode mode,
+                                                            GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-memory-mapped-file.hpp b/c_glib/arrow-glib/io-memory-mapped-file.hpp
new file mode 100644
index 0000000000000..b48e05f2f9e7b
--- /dev/null
+++ b/c_glib/arrow-glib/io-memory-mapped-file.hpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+#include <arrow/io/file.h>
+
+#include <arrow-glib/io-memory-mapped-file.h>
+
+GArrowIOMemoryMappedFile *garrow_io_memory_mapped_file_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file);
+std::shared_ptr<arrow::io::MemoryMappedFile> garrow_io_memory_mapped_file_get_raw(GArrowIOMemoryMappedFile *memory_mapped_file);
diff --git a/c_glib/arrow-glib/io-output-stream.cpp b/c_glib/arrow-glib/io-output-stream.cpp
new file mode 100644
index 0000000000000..bdf5587ba1c07
--- /dev/null
+++ b/c_glib/arrow-glib/io-output-stream.cpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-output-stream.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-output-stream
+ * @title: GArrowIOOutputStream
+ * @short_description: Stream output interface
+ *
+ * #GArrowIOOutputStream is an interface for stream output. Stream
+ * output is file based and writeable
+ */
+
+G_DEFINE_INTERFACE(GArrowIOOutputStream,
+                   garrow_io_output_stream,
+                   G_TYPE_OBJECT)
+
+static void
+garrow_io_output_stream_default_init (GArrowIOOutputStreamInterface *iface)
+{
+}
+
+G_END_DECLS
+
+std::shared_ptr<arrow::io::OutputStream>
+garrow_io_output_stream_get_raw(GArrowIOOutputStream *output_stream)
+{
+  auto *iface = GARROW_IO_OUTPUT_STREAM_GET_IFACE(output_stream);
+  return iface->get_raw(output_stream);
+}
diff --git a/c_glib/arrow-glib/io-output-stream.h b/c_glib/arrow-glib/io-output-stream.h
new file mode 100644
index 0000000000000..c4079d50233cd
--- /dev/null
+++ b/c_glib/arrow-glib/io-output-stream.h
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_OUTPUT_STREAM            \
+  (garrow_io_output_stream_get_type())
+#define GARROW_IO_OUTPUT_STREAM(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IO_TYPE_OUTPUT_STREAM,     \
+                              GArrowIOOutputStreamInterface))
+#define GARROW_IO_IS_OUTPUT_STREAM(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IO_TYPE_OUTPUT_STREAM))
+#define GARROW_IO_OUTPUT_STREAM_GET_IFACE(obj)                          \
+  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
+                                 GARROW_IO_TYPE_OUTPUT_STREAM,          \
+                                 GArrowIOOutputStreamInterface))
+
+typedef struct _GArrowIOOutputStream          GArrowIOOutputStream;
+typedef struct _GArrowIOOutputStreamInterface GArrowIOOutputStreamInterface;
+
+GType garrow_io_output_stream_get_type(void) G_GNUC_CONST;
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-output-stream.hpp b/c_glib/arrow-glib/io-output-stream.hpp
new file mode 100644
index 0000000000000..f144130b1420e
--- /dev/null
+++ b/c_glib/arrow-glib/io-output-stream.hpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/io/interfaces.h>
+
+#include <arrow-glib/io-output-stream.h>
+
+/**
+ * GArrowIOOutputStreamInterface:
+ *
+ * It wraps `arrow::io::OutputStream`.
+ */
+struct _GArrowIOOutputStreamInterface
+{
+  GTypeInterface parent_iface;
+
+  std::shared_ptr<arrow::io::OutputStream> (*get_raw)(GArrowIOOutputStream *file);
+};
+
+std::shared_ptr<arrow::io::OutputStream> garrow_io_output_stream_get_raw(GArrowIOOutputStream *output_stream);
diff --git a/c_glib/arrow-glib/io-readable-file.cpp b/c_glib/arrow-glib/io-readable-file.cpp
new file mode 100644
index 0000000000000..014fd7a1c7d32
--- /dev/null
+++ b/c_glib/arrow-glib/io-readable-file.cpp
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-readable-file.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-readable-file
+ * @title: GArrowIOReadableFile
+ * @short_description: File input interface
+ *
+ * #GArrowIOReadableFile is an interface for file input.
+ */
+
+G_DEFINE_INTERFACE(GArrowIOReadableFile,
+                   garrow_io_readable_file,
+                   G_TYPE_OBJECT)
+
+static void
+garrow_io_readable_file_default_init (GArrowIOReadableFileInterface *iface)
+{
+}
+
+/**
+ * garrow_io_readable_file_get_size:
+ * @file: A #GArrowIOReadableFile.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: The size of the file.
+ */
+guint64
+garrow_io_readable_file_get_size(GArrowIOReadableFile *file,
+                                 GError **error)
+{
+  auto *iface = GARROW_IO_READABLE_FILE_GET_IFACE(file);
+  auto arrow_readable_file = iface->get_raw(file);
+  int64_t size;
+
+  auto status = arrow_readable_file->GetSize(&size);
+  if (status.ok()) {
+    return size;
+  } else {
+    garrow_error_set(error, status, "[io][readable-file][get-size]");
+    return 0;
+  }
+}
+
+/**
+ * garrow_io_readable_file_get_support_zero_copy:
+ * @file: A #GArrowIOReadableFile.
+ *
+ * Returns: Whether zero copy read is supported or not.
+ */
+gboolean
+garrow_io_readable_file_get_support_zero_copy(GArrowIOReadableFile *file)
+{
+  auto *iface = GARROW_IO_READABLE_FILE_GET_IFACE(file);
+  auto arrow_readable_file = iface->get_raw(file);
+
+  return arrow_readable_file->supports_zero_copy();
+}
+
+/**
+ * garrow_io_readable_file_read_at:
+ * @file: A #GArrowIOReadableFile.
+ * @position: The read start position.
+ * @n_bytes: The number of bytes to be read.
+ * @n_read_bytes: (out): The read number of bytes.
+ * @buffer: (array length=n_bytes): The buffer to be read data.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_io_readable_file_read_at(GArrowIOReadableFile *file,
+                                gint64 position,
+                                gint64 n_bytes,
+                                gint64 *n_read_bytes,
+                                guint8 *buffer,
+                                GError **error)
+{
+  const auto arrow_readable_file = garrow_io_readable_file_get_raw(file);
+
+  auto status = arrow_readable_file->ReadAt(position,
+                                            n_bytes,
+                                            n_read_bytes,
+                                            buffer);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[io][readable-file][read-at]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
+
+std::shared_ptr<arrow::io::ReadableFileInterface>
+garrow_io_readable_file_get_raw(GArrowIOReadableFile *readable_file)
+{
+  auto *iface = GARROW_IO_READABLE_FILE_GET_IFACE(readable_file);
+  return iface->get_raw(readable_file);
+}
diff --git a/c_glib/arrow-glib/io-readable-file.h b/c_glib/arrow-glib/io-readable-file.h
new file mode 100644
index 0000000000000..1dcb13e04969c
--- /dev/null
+++ b/c_glib/arrow-glib/io-readable-file.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_READABLE_FILE                     \
+  (garrow_io_readable_file_get_type())
+#define GARROW_IO_READABLE_FILE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IO_TYPE_READABLE_FILE,     \
+                              GArrowIOReadableFileInterface))
+#define GARROW_IO_IS_READABLE_FILE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IO_TYPE_READABLE_FILE))
+#define GARROW_IO_READABLE_FILE_GET_IFACE(obj)                          \
+  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
+                                 GARROW_IO_TYPE_READABLE_FILE,          \
+                                 GArrowIOReadableFileInterface))
+
+typedef struct _GArrowIOReadableFile          GArrowIOReadableFile;
+typedef struct _GArrowIOReadableFileInterface GArrowIOReadableFileInterface;
+
+GType garrow_io_readable_file_get_type(void) G_GNUC_CONST;
+
+guint64 garrow_io_readable_file_get_size(GArrowIOReadableFile *file,
+                                         GError **error);
+gboolean garrow_io_readable_file_get_support_zero_copy(GArrowIOReadableFile *file);
+gboolean garrow_io_readable_file_read_at(GArrowIOReadableFile *file,
+                                         gint64 position,
+                                         gint64 n_bytes,
+                                         gint64 *n_read_bytes,
+                                         guint8 *buffer,
+                                         GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-readable-file.hpp b/c_glib/arrow-glib/io-readable-file.hpp
new file mode 100644
index 0000000000000..83d8628f48b62
--- /dev/null
+++ b/c_glib/arrow-glib/io-readable-file.hpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/io/interfaces.h>
+
+#include <arrow-glib/io-readable-file.h>
+
+/**
+ * GArrowIOReadableFileInterface:
+ *
+ * It wraps `arrow::io::ReadableFileInterface`.
+ */
+struct _GArrowIOReadableFileInterface
+{
+  GTypeInterface parent_iface;
+
+  std::shared_ptr<arrow::io::ReadableFileInterface> (*get_raw)(GArrowIOReadableFile *file);
+};
+
+std::shared_ptr<arrow::io::ReadableFileInterface> garrow_io_readable_file_get_raw(GArrowIOReadableFile *readable_file);
diff --git a/c_glib/arrow-glib/io-readable.cpp b/c_glib/arrow-glib/io-readable.cpp
new file mode 100644
index 0000000000000..b372a66090ceb
--- /dev/null
+++ b/c_glib/arrow-glib/io-readable.cpp
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-readable.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-readable
+ * @title: GArrowIOReadable
+ * @short_description: Input interface
+ *
+ * #GArrowIOReadable is an interface for input. Input must be
+ * readable.
+ */
+
+G_DEFINE_INTERFACE(GArrowIOReadable,
+                   garrow_io_readable,
+                   G_TYPE_OBJECT)
+
+static void
+garrow_io_readable_default_init (GArrowIOReadableInterface *iface)
+{
+}
+
+/**
+ * garrow_io_readable_read:
+ * @readable: A #GArrowIOReadable.
+ * @n_bytes: The number of bytes to be read.
+ * @n_read_bytes: (out): The read number of bytes.
+ * @buffer: (array length=n_bytes): The buffer to be read data.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_io_readable_read(GArrowIOReadable *readable,
+                        gint64 n_bytes,
+                        gint64 *n_read_bytes,
+                        guint8 *buffer,
+                        GError **error)
+{
+  const auto arrow_readable = garrow_io_readable_get_raw(readable);
+
+  auto status = arrow_readable->Read(n_bytes, n_read_bytes, buffer);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[io][readable][read]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
+
+std::shared_ptr<arrow::io::Readable>
+garrow_io_readable_get_raw(GArrowIOReadable *readable)
+{
+  auto *iface = GARROW_IO_READABLE_GET_IFACE(readable);
+  return iface->get_raw(readable);
+}
diff --git a/c_glib/arrow-glib/io-readable.h b/c_glib/arrow-glib/io-readable.h
new file mode 100644
index 0000000000000..d24b46c50df4c
--- /dev/null
+++ b/c_glib/arrow-glib/io-readable.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_READABLE                 \
+  (garrow_io_readable_get_type())
+#define GARROW_IO_READABLE(obj)                                 \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IO_TYPE_READABLE,          \
+                              GArrowIOReadableInterface))
+#define GARROW_IO_IS_READABLE(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_IO_TYPE_READABLE))
+#define GARROW_IO_READABLE_GET_IFACE(obj)                       \
+  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
+                                 GARROW_IO_TYPE_READABLE,       \
+                                 GArrowIOReadableInterface))
+
+typedef struct _GArrowIOReadable          GArrowIOReadable;
+typedef struct _GArrowIOReadableInterface GArrowIOReadableInterface;
+
+GType garrow_io_readable_get_type(void) G_GNUC_CONST;
+
+gboolean garrow_io_readable_read(GArrowIOReadable *readable,
+                                 gint64 n_bytes,
+                                 gint64 *n_read_bytes,
+                                 guint8 *buffer,
+                                 GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-readable.hpp b/c_glib/arrow-glib/io-readable.hpp
new file mode 100644
index 0000000000000..3d27b3f92ba78
--- /dev/null
+++ b/c_glib/arrow-glib/io-readable.hpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/io/interfaces.h>
+
+#include <arrow-glib/io-readable.h>
+
+/**
+ * GArrowIOReadableInterface:
+ *
+ * It wraps `arrow::io::Readable`.
+ */
+struct _GArrowIOReadableInterface
+{
+  GTypeInterface parent_iface;
+
+  std::shared_ptr<arrow::io::Readable> (*get_raw)(GArrowIOReadable *file);
+};
+
+std::shared_ptr<arrow::io::Readable> garrow_io_readable_get_raw(GArrowIOReadable *readable);
diff --git a/c_glib/arrow-glib/io-writeable-file.cpp b/c_glib/arrow-glib/io-writeable-file.cpp
new file mode 100644
index 0000000000000..3de42dd60a971
--- /dev/null
+++ b/c_glib/arrow-glib/io-writeable-file.cpp
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-writeable-file.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-writeable-file
+ * @title: GArrowIOWriteableFile
+ * @short_description: File output interface
+ *
+ * #GArrowIOWriteableFile is an interface for file output.
+ */
+
+G_DEFINE_INTERFACE(GArrowIOWriteableFile,
+                   garrow_io_writeable_file,
+                   G_TYPE_OBJECT)
+
+static void
+garrow_io_writeable_file_default_init (GArrowIOWriteableFileInterface *iface)
+{
+}
+
+/**
+ * garrow_io_writeable_file_write_at:
+ * @writeable_file: A #GArrowIOWriteableFile.
+ * @position: The write start position.
+ * @data: (array length=n_bytes): The data to be written.
+ * @n_bytes: The number of bytes to be written.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_io_writeable_file_write_at(GArrowIOWriteableFile *writeable_file,
+                                  gint64 position,
+                                  const guint8 *data,
+                                  gint64 n_bytes,
+                                  GError **error)
+{
+  const auto arrow_writeable_file =
+    garrow_io_writeable_file_get_raw(writeable_file);
+
+  auto status = arrow_writeable_file->WriteAt(position, data, n_bytes);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[io][writeable-file][write-at]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
+
+std::shared_ptr<arrow::io::WriteableFileInterface>
+garrow_io_writeable_file_get_raw(GArrowIOWriteableFile *writeable_file)
+{
+  auto *iface = GARROW_IO_WRITEABLE_FILE_GET_IFACE(writeable_file);
+  return iface->get_raw(writeable_file);
+}
diff --git a/c_glib/arrow-glib/io-writeable-file.h b/c_glib/arrow-glib/io-writeable-file.h
new file mode 100644
index 0000000000000..4a4dee5111f5f
--- /dev/null
+++ b/c_glib/arrow-glib/io-writeable-file.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_WRITEABLE_FILE           \
+  (garrow_io_writeable_file_get_type())
+#define GARROW_IO_WRITEABLE_FILE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IO_TYPE_WRITEABLE_FILE,    \
+                              GArrowIOWriteableFileInterface))
+#define GARROW_IO_IS_WRITEABLE_FILE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IO_TYPE_WRITEABLE_FILE))
+#define GARROW_IO_WRITEABLE_FILE_GET_IFACE(obj)                         \
+  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
+                                 GARROW_IO_TYPE_WRITEABLE_FILE,         \
+                                 GArrowIOWriteableFileInterface))
+
+typedef struct _GArrowIOWriteableFile          GArrowIOWriteableFile;
+typedef struct _GArrowIOWriteableFileInterface GArrowIOWriteableFileInterface;
+
+GType garrow_io_writeable_file_get_type(void) G_GNUC_CONST;
+
+gboolean garrow_io_writeable_file_write_at(GArrowIOWriteableFile *writeable_file,
+                                           gint64 position,
+                                           const guint8 *data,
+                                           gint64 n_bytes,
+                                           GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-writeable-file.hpp b/c_glib/arrow-glib/io-writeable-file.hpp
new file mode 100644
index 0000000000000..2043007ad58e3
--- /dev/null
+++ b/c_glib/arrow-glib/io-writeable-file.hpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/io/interfaces.h>
+
+#include <arrow-glib/io-writeable-file.h>
+
+/**
+ * GArrowIOWriteableFileInterface:
+ *
+ * It wraps `arrow::io::WriteableFileInterface`.
+ */
+struct _GArrowIOWriteableFileInterface
+{
+  GTypeInterface parent_iface;
+
+  std::shared_ptr<arrow::io::WriteableFileInterface> (*get_raw)(GArrowIOWriteableFile *file);
+};
+
+std::shared_ptr<arrow::io::WriteableFileInterface> garrow_io_writeable_file_get_raw(GArrowIOWriteableFile *writeable_file);
diff --git a/c_glib/arrow-glib/io-writeable.cpp b/c_glib/arrow-glib/io-writeable.cpp
new file mode 100644
index 0000000000000..9ea69e3adccde
--- /dev/null
+++ b/c_glib/arrow-glib/io-writeable.cpp
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-writeable.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-writeable
+ * @title: GArrowIOWriteable
+ * @short_description: Output interface
+ *
+ * #GArrowIOWriteable is an interface for output. Output must be
+ * writeable.
+ */
+
+G_DEFINE_INTERFACE(GArrowIOWriteable,
+                   garrow_io_writeable,
+                   G_TYPE_OBJECT)
+
+static void
+garrow_io_writeable_default_init (GArrowIOWriteableInterface *iface)
+{
+}
+
+/**
+ * garrow_io_writeable_write:
+ * @writeable: A #GArrowIOWriteable.
+ * @data: (array length=n_bytes): The data to be written.
+ * @n_bytes: The number of bytes to be written.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_io_writeable_write(GArrowIOWriteable *writeable,
+                          const guint8 *data,
+                          gint64 n_bytes,
+                          GError **error)
+{
+  const auto arrow_writeable = garrow_io_writeable_get_raw(writeable);
+
+  auto status = arrow_writeable->Write(data, n_bytes);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[io][writeable][write]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_io_writeable_flush:
+ * @writeable: A #GArrowIOWriteable.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * It ensures writing all data on memory to storage.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_io_writeable_flush(GArrowIOWriteable *writeable,
+                          GError **error)
+{
+  const auto arrow_writeable = garrow_io_writeable_get_raw(writeable);
+
+  auto status = arrow_writeable->Flush();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[io][writeable][flush]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
+
+std::shared_ptr<arrow::io::Writeable>
+garrow_io_writeable_get_raw(GArrowIOWriteable *writeable)
+{
+  auto *iface = GARROW_IO_WRITEABLE_GET_IFACE(writeable);
+  return iface->get_raw(writeable);
+}
diff --git a/c_glib/arrow-glib/io-writeable.h b/c_glib/arrow-glib/io-writeable.h
new file mode 100644
index 0000000000000..f5c5e9129f8be
--- /dev/null
+++ b/c_glib/arrow-glib/io-writeable.h
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_WRITEABLE                \
+  (garrow_io_writeable_get_type())
+#define GARROW_IO_WRITEABLE(obj)                                \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IO_TYPE_WRITEABLE,         \
+                              GArrowIOWriteableInterface))
+#define GARROW_IO_IS_WRITEABLE(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IO_TYPE_WRITEABLE))
+#define GARROW_IO_WRITEABLE_GET_IFACE(obj)                      \
+  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
+                                 GARROW_IO_TYPE_WRITEABLE,      \
+                                 GArrowIOWriteableInterface))
+
+typedef struct _GArrowIOWriteable          GArrowIOWriteable;
+typedef struct _GArrowIOWriteableInterface GArrowIOWriteableInterface;
+
+GType garrow_io_writeable_get_type(void) G_GNUC_CONST;
+
+gboolean garrow_io_writeable_write(GArrowIOWriteable *writeable,
+                                   const guint8 *data,
+                                   gint64 n_bytes,
+                                   GError **error);
+gboolean garrow_io_writeable_flush(GArrowIOWriteable *writeable,
+                                   GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-writeable.hpp b/c_glib/arrow-glib/io-writeable.hpp
new file mode 100644
index 0000000000000..f833924a61ae8
--- /dev/null
+++ b/c_glib/arrow-glib/io-writeable.hpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/io/interfaces.h>
+
+#include <arrow-glib/io-writeable.h>
+
+/**
+ * GArrowIOWriteableInterface:
+ *
+ * It wraps `arrow::io::Writeable`.
+ */
+struct _GArrowIOWriteableInterface
+{
+  GTypeInterface parent_iface;
+
+  std::shared_ptr<arrow::io::Writeable> (*get_raw)(GArrowIOWriteable *file);
+};
+
+std::shared_ptr<arrow::io::Writeable> garrow_io_writeable_get_raw(GArrowIOWriteable *writeable);
diff --git a/c_glib/arrow-glib/ipc-enums.c.template b/c_glib/arrow-glib/ipc-enums.c.template
new file mode 100644
index 0000000000000..c938f77477172
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-enums.c.template
@@ -0,0 +1,56 @@
+/*** BEGIN file-header ***/
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/arrow-ipc-glib.h>
+/*** END file-header ***/
+
+/*** BEGIN file-production ***/
+
+/* enumerations from "@filename@" */
+/*** END file-production ***/
+
+/*** BEGIN value-header ***/
+GType
+@enum_name@_get_type(void)
+{
+  static GType etype = 0;
+  if (G_UNLIKELY(etype == 0)) {
+    static const G@Type@Value values[] = {
+/*** END value-header ***/
+
+/*** BEGIN value-production ***/
+      {@VALUENAME@, "@VALUENAME@", "@valuenick@"},
+/*** END value-production ***/
+
+/*** BEGIN value-tail ***/
+      {0, NULL, NULL}
+    };
+    etype = g_@type@_register_static(g_intern_static_string("@EnumName@"), values);
+  }
+  return etype;
+}
+/*** END value-tail ***/
+
+/*** BEGIN file-tail ***/
+/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/ipc-enums.h.template b/c_glib/arrow-glib/ipc-enums.h.template
new file mode 100644
index 0000000000000..e103c5bfeb985
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-enums.h.template
@@ -0,0 +1,41 @@
+/*** BEGIN file-header ***/
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/ipc-metadata-version.h>
+
+G_BEGIN_DECLS
+/*** END file-header ***/
+
+/*** BEGIN file-production ***/
+
+/* enumerations from "@filename@" */
+/*** END file-production ***/
+
+/*** BEGIN value-header ***/
+GType @enum_name@_get_type(void) G_GNUC_CONST;
+#define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
+/*** END value-header ***/
+
+/*** BEGIN file-tail ***/
+
+G_END_DECLS
+/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/ipc-file-reader.cpp b/c_glib/arrow-glib/ipc-file-reader.cpp
new file mode 100644
index 0000000000000..b9e408c4e9464
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-file-reader.cpp
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/record-batch.hpp>
+#include <arrow-glib/schema.hpp>
+
+#include <arrow-glib/io-readable-file.hpp>
+
+#include <arrow-glib/ipc-file-reader.hpp>
+#include <arrow-glib/ipc-metadata-version.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: ipc-file-reader
+ * @short_description: File reader class
+ *
+ * #GArrowIPCFileReader is a class for receiving data by file based IPC.
+ */
+
+typedef struct GArrowIPCFileReaderPrivate_ {
+  std::shared_ptr<arrow::ipc::FileReader> file_reader;
+} GArrowIPCFileReaderPrivate;
+
+enum {
+  PROP_0,
+  PROP_FILE_READER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowIPCFileReader,
+                           garrow_ipc_file_reader,
+                           G_TYPE_OBJECT);
+
+#define GARROW_IPC_FILE_READER_GET_PRIVATE(obj)                         \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
+                               GARROW_IPC_TYPE_FILE_READER,             \
+                               GArrowIPCFileReaderPrivate))
+
+static void
+garrow_ipc_file_reader_finalize(GObject *object)
+{
+  GArrowIPCFileReaderPrivate *priv;
+
+  priv = GARROW_IPC_FILE_READER_GET_PRIVATE(object);
+
+  priv->file_reader = nullptr;
+
+  G_OBJECT_CLASS(garrow_ipc_file_reader_parent_class)->finalize(object);
+}
+
+static void
+garrow_ipc_file_reader_set_property(GObject *object,
+                                    guint prop_id,
+                                    const GValue *value,
+                                    GParamSpec *pspec)
+{
+  GArrowIPCFileReaderPrivate *priv;
+
+  priv = GARROW_IPC_FILE_READER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_FILE_READER:
+    priv->file_reader =
+      *static_cast<std::shared_ptr<arrow::ipc::FileReader> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_ipc_file_reader_get_property(GObject *object,
+                                          guint prop_id,
+                                          GValue *value,
+                                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_ipc_file_reader_init(GArrowIPCFileReader *object)
+{
+}
+
+static void
+garrow_ipc_file_reader_class_init(GArrowIPCFileReaderClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_ipc_file_reader_finalize;
+  gobject_class->set_property = garrow_ipc_file_reader_set_property;
+  gobject_class->get_property = garrow_ipc_file_reader_get_property;
+
+  spec = g_param_spec_pointer("file-reader",
+                              "ipc::FileReader",
+                              "The raw std::shared<arrow::ipc::FileReader> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_FILE_READER, spec);
+}
+
+/**
+ * garrow_ipc_file_reader_open:
+ * @file: The file to be read.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): A newly opened
+ *   #GArrowIPCFileReader or %NULL on error.
+ */
+GArrowIPCFileReader *
+garrow_ipc_file_reader_open(GArrowIOReadableFile *file,
+                            GError **error)
+{
+  std::shared_ptr<arrow::ipc::FileReader> arrow_file_reader;
+  auto status =
+    arrow::ipc::FileReader::Open(garrow_io_readable_file_get_raw(file),
+                                 &arrow_file_reader);
+  if (status.ok()) {
+    return garrow_ipc_file_reader_new_raw(&arrow_file_reader);
+  } else {
+    garrow_error_set(error, status, "[ipc][file-reader][open]");
+    return NULL;
+  }
+}
+
+/**
+ * garrow_ipc_file_reader_get_schema:
+ * @file_reader: A #GArrowIPCFileReader.
+ *
+ * Returns: (transfer full): The schema in the file.
+ */
+GArrowSchema *
+garrow_ipc_file_reader_get_schema(GArrowIPCFileReader *file_reader)
+{
+  auto arrow_file_reader =
+    garrow_ipc_file_reader_get_raw(file_reader);
+  auto arrow_schema = arrow_file_reader->schema();
+  return garrow_schema_new_raw(&arrow_schema);
+}
+
+/**
+ * garrow_ipc_file_reader_get_n_record_batches:
+ * @file_reader: A #GArrowIPCFileReader.
+ *
+ * Returns: The number of record batches in the file.
+ */
+guint
+garrow_ipc_file_reader_get_n_record_batches(GArrowIPCFileReader *file_reader)
+{
+  auto arrow_file_reader =
+    garrow_ipc_file_reader_get_raw(file_reader);
+  return arrow_file_reader->num_record_batches();
+}
+
+/**
+ * garrow_ipc_file_reader_get_version:
+ * @file_reader: A #GArrowIPCFileReader.
+ *
+ * Returns: The format version in the file.
+ */
+GArrowIPCMetadataVersion
+garrow_ipc_file_reader_get_version(GArrowIPCFileReader *file_reader)
+{
+  auto arrow_file_reader =
+    garrow_ipc_file_reader_get_raw(file_reader);
+  auto arrow_version = arrow_file_reader->version();
+  return garrow_ipc_metadata_version_from_raw(arrow_version);
+}
+
+/**
+ * garrow_ipc_file_reader_get_record_batch:
+ * @file_reader: A #GArrowIPCFileReader.
+ * @i: The index of the target record batch.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The i-th record batch in the file or %NULL on error.
+ */
+GArrowRecordBatch *
+garrow_ipc_file_reader_get_record_batch(GArrowIPCFileReader *file_reader,
+                                        guint i,
+                                        GError **error)
+{
+  auto arrow_file_reader =
+    garrow_ipc_file_reader_get_raw(file_reader);
+  std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
+  auto status = arrow_file_reader->GetRecordBatch(i, &arrow_record_batch);
+
+  if (status.ok()) {
+    return garrow_record_batch_new_raw(&arrow_record_batch);
+  } else {
+    garrow_error_set(error, status, "[ipc][file-reader][get-record-batch]");
+    return NULL;
+  }
+}
+
+G_END_DECLS
+
+GArrowIPCFileReader *
+garrow_ipc_file_reader_new_raw(std::shared_ptr<arrow::ipc::FileReader> *arrow_file_reader)
+{
+  auto file_reader =
+    GARROW_IPC_FILE_READER(g_object_new(GARROW_IPC_TYPE_FILE_READER,
+                                        "file-reader", arrow_file_reader,
+                                        NULL));
+  return file_reader;
+}
+
+std::shared_ptr<arrow::ipc::FileReader>
+garrow_ipc_file_reader_get_raw(GArrowIPCFileReader *file_reader)
+{
+  GArrowIPCFileReaderPrivate *priv;
+
+  priv = GARROW_IPC_FILE_READER_GET_PRIVATE(file_reader);
+  return priv->file_reader;
+}
diff --git a/c_glib/arrow-glib/ipc-file-reader.h b/c_glib/arrow-glib/ipc-file-reader.h
new file mode 100644
index 0000000000000..22915f8ae6e68
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-file-reader.h
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+
+#include <arrow-glib/io-readable-file.h>
+
+#include <arrow-glib/ipc-metadata-version.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IPC_TYPE_FILE_READER      \
+  (garrow_ipc_file_reader_get_type())
+#define GARROW_IPC_FILE_READER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IPC_TYPE_FILE_READER,      \
+                              GArrowIPCFileReader))
+#define GARROW_IPC_FILE_READER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_IPC_TYPE_FILE_READER, \
+                           GArrowIPCFileReaderClass))
+#define GARROW_IPC_IS_FILE_READER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IPC_TYPE_FILE_READER))
+#define GARROW_IPC_IS_FILE_READER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_IPC_TYPE_FILE_READER))
+#define GARROW_IPC_FILE_READER_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_IPC_TYPE_FILE_READER,       \
+                             GArrowIPCFileReaderClass))
+
+typedef struct _GArrowIPCFileReader         GArrowIPCFileReader;
+typedef struct _GArrowIPCFileReaderClass    GArrowIPCFileReaderClass;
+
+/**
+ * GArrowIPCFileReader:
+ *
+ * It wraps `arrow::ipc::FileReader`.
+ */
+struct _GArrowIPCFileReader
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowIPCFileReaderClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_ipc_file_reader_get_type(void) G_GNUC_CONST;
+
+GArrowIPCFileReader *garrow_ipc_file_reader_open(GArrowIOReadableFile *file,
+                                                 GError **error);
+
+GArrowSchema *garrow_ipc_file_reader_get_schema(GArrowIPCFileReader *file_reader);
+guint garrow_ipc_file_reader_get_n_record_batches(GArrowIPCFileReader *file_reader);
+GArrowIPCMetadataVersion garrow_ipc_file_reader_get_version(GArrowIPCFileReader *file_reader);
+GArrowRecordBatch *garrow_ipc_file_reader_get_record_batch(GArrowIPCFileReader *file_reader,
+                                                           guint i,
+                                                           GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-file-reader.hpp b/c_glib/arrow-glib/ipc-file-reader.hpp
new file mode 100644
index 0000000000000..66cd45d51ddf5
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-file-reader.hpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/ipc-file-reader.h>
+
+GArrowIPCFileReader *garrow_ipc_file_reader_new_raw(std::shared_ptr<arrow::ipc::FileReader> *arrow_file_reader);
+std::shared_ptr<arrow::ipc::FileReader> garrow_ipc_file_reader_get_raw(GArrowIPCFileReader *file_reader);
diff --git a/c_glib/arrow-glib/ipc-file-writer.cpp b/c_glib/arrow-glib/ipc-file-writer.cpp
new file mode 100644
index 0000000000000..d8b3c2e72fa31
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-file-writer.cpp
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/record-batch.hpp>
+#include <arrow-glib/schema.hpp>
+
+#include <arrow-glib/io-output-stream.hpp>
+
+#include <arrow-glib/ipc-stream-writer.hpp>
+#include <arrow-glib/ipc-file-writer.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: ipc-file-writer
+ * @short_description: File writer class
+ *
+ * #GArrowIPCFileWriter is a class for sending data by file based IPC.
+ */
+
+G_DEFINE_TYPE(GArrowIPCFileWriter,
+              garrow_ipc_file_writer,
+              GARROW_IPC_TYPE_STREAM_WRITER);
+
+static void
+garrow_ipc_file_writer_init(GArrowIPCFileWriter *object)
+{
+}
+
+static void
+garrow_ipc_file_writer_class_init(GArrowIPCFileWriterClass *klass)
+{
+}
+
+/**
+ * garrow_ipc_file_writer_open:
+ * @sink: The output of the writer.
+ * @schema: The schema of the writer.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): A newly opened
+ *   #GArrowIPCFileWriter or %NULL on error.
+ */
+GArrowIPCFileWriter *
+garrow_ipc_file_writer_open(GArrowIOOutputStream *sink,
+                            GArrowSchema *schema,
+                            GError **error)
+{
+  std::shared_ptr<arrow::ipc::FileWriter> arrow_file_writer;
+  auto status =
+    arrow::ipc::FileWriter::Open(garrow_io_output_stream_get_raw(sink).get(),
+                                 garrow_schema_get_raw(schema),
+                                 &arrow_file_writer);
+  if (status.ok()) {
+    return garrow_ipc_file_writer_new_raw(&arrow_file_writer);
+  } else {
+    garrow_error_set(error, status, "[ipc][file-writer][open]");
+    return NULL;
+  }
+}
+
+/**
+ * garrow_ipc_file_writer_write_record_batch:
+ * @file_writer: A #GArrowIPCFileWriter.
+ * @record_batch: The record batch to be written.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_ipc_file_writer_write_record_batch(GArrowIPCFileWriter *file_writer,
+                                          GArrowRecordBatch *record_batch,
+                                          GError **error)
+{
+  auto arrow_file_writer =
+    garrow_ipc_file_writer_get_raw(file_writer);
+  auto arrow_record_batch =
+    garrow_record_batch_get_raw(record_batch);
+  auto arrow_record_batch_raw =
+    arrow_record_batch.get();
+
+  auto status = arrow_file_writer->WriteRecordBatch(*arrow_record_batch_raw);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[ipc][file-writer][write-record-batch]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_ipc_file_writer_close:
+ * @file_writer: A #GArrowIPCFileWriter.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_ipc_file_writer_close(GArrowIPCFileWriter *file_writer,
+                             GError **error)
+{
+  auto arrow_file_writer =
+    garrow_ipc_file_writer_get_raw(file_writer);
+
+  auto status = arrow_file_writer->Close();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[ipc][file-writer][close]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
+
+GArrowIPCFileWriter *
+garrow_ipc_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer)
+{
+  auto file_writer =
+    GARROW_IPC_FILE_WRITER(g_object_new(GARROW_IPC_TYPE_FILE_WRITER,
+                                        "stream-writer", arrow_file_writer,
+                                        NULL));
+  return file_writer;
+}
+
+arrow::ipc::FileWriter *
+garrow_ipc_file_writer_get_raw(GArrowIPCFileWriter *file_writer)
+{
+  auto arrow_stream_writer =
+    garrow_ipc_stream_writer_get_raw(GARROW_IPC_STREAM_WRITER(file_writer));
+  auto arrow_file_writer_raw =
+    dynamic_cast<arrow::ipc::FileWriter *>(arrow_stream_writer.get());
+  return arrow_file_writer_raw;
+}
diff --git a/c_glib/arrow-glib/ipc-file-writer.h b/c_glib/arrow-glib/ipc-file-writer.h
new file mode 100644
index 0000000000000..732d9426aec8e
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-file-writer.h
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/ipc-stream-writer.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IPC_TYPE_FILE_WRITER             \
+  (garrow_ipc_file_writer_get_type())
+#define GARROW_IPC_FILE_WRITER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IPC_TYPE_FILE_WRITER,      \
+                              GArrowIPCFileWriter))
+#define GARROW_IPC_FILE_WRITER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_IPC_TYPE_FILE_WRITER, \
+                           GArrowIPCFileWriterClass))
+#define GARROW_IPC_IS_FILE_WRITER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IPC_TYPE_FILE_WRITER))
+#define GARROW_IPC_IS_FILE_WRITER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_IPC_TYPE_FILE_WRITER))
+#define GARROW_IPC_FILE_WRITER_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_IPC_TYPE_FILE_WRITER,       \
+                             GArrowIPCFileWriterClass))
+
+typedef struct _GArrowIPCFileWriter         GArrowIPCFileWriter;
+typedef struct _GArrowIPCFileWriterClass    GArrowIPCFileWriterClass;
+
+/**
+ * GArrowIPCFileWriter:
+ *
+ * It wraps `arrow::ipc::FileWriter`.
+ */
+struct _GArrowIPCFileWriter
+{
+  /*< private >*/
+  GArrowIPCStreamWriter parent_instance;
+};
+
+struct _GArrowIPCFileWriterClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_ipc_file_writer_get_type(void) G_GNUC_CONST;
+
+GArrowIPCFileWriter *garrow_ipc_file_writer_open(GArrowIOOutputStream *sink,
+                                                 GArrowSchema *schema,
+                                                 GError **error);
+
+gboolean garrow_ipc_file_writer_write_record_batch(GArrowIPCFileWriter *file_writer,
+                                                   GArrowRecordBatch *record_batch,
+                                                   GError **error);
+gboolean garrow_ipc_file_writer_close(GArrowIPCFileWriter *file_writer,
+                                      GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-file-writer.hpp b/c_glib/arrow-glib/ipc-file-writer.hpp
new file mode 100644
index 0000000000000..b8ae1137a99ad
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-file-writer.hpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/ipc-file-writer.h>
+
+GArrowIPCFileWriter *garrow_ipc_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer);
+arrow::ipc::FileWriter *garrow_ipc_file_writer_get_raw(GArrowIPCFileWriter *file_writer);
diff --git a/c_glib/arrow-glib/ipc-metadata-version.cpp b/c_glib/arrow-glib/ipc-metadata-version.cpp
new file mode 100644
index 0000000000000..c5cc8d379843c
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-metadata-version.cpp
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/ipc-metadata-version.hpp>
+
+/**
+ * SECTION: ipc-metadata-version
+ * @title: GArrowIPCMetadataVersion
+ * @short_description: Metadata version mapgging between Arrow and arrow-glib
+ *
+ * #GArrowIPCMetadataVersion provides metadata versions corresponding
+ * to `arrow::ipc::MetadataVersion::type` values.
+ */
+
+GArrowIPCMetadataVersion
+garrow_ipc_metadata_version_from_raw(arrow::ipc::MetadataVersion::type version)
+{
+  switch (version) {
+  case arrow::ipc::MetadataVersion::type::V1:
+    return GARROW_IPC_METADATA_VERSION_V1;
+  case arrow::ipc::MetadataVersion::type::V2:
+    return GARROW_IPC_METADATA_VERSION_V2;
+  default:
+    return GARROW_IPC_METADATA_VERSION_V2;
+  }
+}
+
+arrow::ipc::MetadataVersion::type
+garrow_ipc_metadata_version_to_raw(GArrowIPCMetadataVersion version)
+{
+  switch (version) {
+  case GARROW_IPC_METADATA_VERSION_V1:
+    return arrow::ipc::MetadataVersion::type::V1;
+  case GARROW_IPC_METADATA_VERSION_V2:
+    return arrow::ipc::MetadataVersion::type::V2;
+  default:
+    return arrow::ipc::MetadataVersion::type::V2;
+  }
+}
diff --git a/c_glib/arrow-glib/ipc-metadata-version.h b/c_glib/arrow-glib/ipc-metadata-version.h
new file mode 100644
index 0000000000000..ccfd52a81639f
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-metadata-version.h
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+/**
+ * GArrowIPCMetadataVersion:
+ * @GARROW_IPC_METADATA_VERSION_V1: Version 1.
+ * @GARROW_IPC_METADATA_VERSION_V2: Version 2.
+ *
+ * They are corresponding to `arrow::ipc::MetadataVersion::type`
+ * values.
+ */
+typedef enum {
+  GARROW_IPC_METADATA_VERSION_V1,
+  GARROW_IPC_METADATA_VERSION_V2
+} GArrowIPCMetadataVersion;
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-metadata-version.hpp b/c_glib/arrow-glib/ipc-metadata-version.hpp
new file mode 100644
index 0000000000000..2a7e8cffa8917
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-metadata-version.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/ipc-metadata-version.h>
+
+GArrowIPCMetadataVersion garrow_ipc_metadata_version_from_raw(arrow::ipc::MetadataVersion::type version);
+arrow::ipc::MetadataVersion::type garrow_ipc_metadata_version_to_raw(GArrowIPCMetadataVersion version);
diff --git a/c_glib/arrow-glib/ipc-stream-reader.cpp b/c_glib/arrow-glib/ipc-stream-reader.cpp
new file mode 100644
index 0000000000000..48047842aaac6
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-stream-reader.cpp
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/record-batch.hpp>
+#include <arrow-glib/schema.hpp>
+
+#include <arrow-glib/io-input-stream.hpp>
+
+#include <arrow-glib/ipc-metadata-version.hpp>
+#include <arrow-glib/ipc-stream-reader.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: ipc-stream-reader
+ * @short_description: Stream reader class
+ *
+ * #GArrowIPCStreamReader is a class for receiving data by stream
+ * based IPC.
+ */
+
+typedef struct GArrowIPCStreamReaderPrivate_ {
+  std::shared_ptr<arrow::ipc::StreamReader> stream_reader;
+} GArrowIPCStreamReaderPrivate;
+
+enum {
+  PROP_0,
+  PROP_STREAM_READER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowIPCStreamReader,
+                           garrow_ipc_stream_reader,
+                           G_TYPE_OBJECT);
+
+#define GARROW_IPC_STREAM_READER_GET_PRIVATE(obj)               \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_IPC_TYPE_STREAM_READER,   \
+                               GArrowIPCStreamReaderPrivate))
+
+static void
+garrow_ipc_stream_reader_finalize(GObject *object)
+{
+  GArrowIPCStreamReaderPrivate *priv;
+
+  priv = GARROW_IPC_STREAM_READER_GET_PRIVATE(object);
+
+  priv->stream_reader = nullptr;
+
+  G_OBJECT_CLASS(garrow_ipc_stream_reader_parent_class)->finalize(object);
+}
+
+static void
+garrow_ipc_stream_reader_set_property(GObject *object,
+                                    guint prop_id,
+                                    const GValue *value,
+                                    GParamSpec *pspec)
+{
+  GArrowIPCStreamReaderPrivate *priv;
+
+  priv = GARROW_IPC_STREAM_READER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_STREAM_READER:
+    priv->stream_reader =
+      *static_cast<std::shared_ptr<arrow::ipc::StreamReader> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_ipc_stream_reader_get_property(GObject *object,
+                                          guint prop_id,
+                                          GValue *value,
+                                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_ipc_stream_reader_init(GArrowIPCStreamReader *object)
+{
+}
+
+static void
+garrow_ipc_stream_reader_class_init(GArrowIPCStreamReaderClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_ipc_stream_reader_finalize;
+  gobject_class->set_property = garrow_ipc_stream_reader_set_property;
+  gobject_class->get_property = garrow_ipc_stream_reader_get_property;
+
+  spec = g_param_spec_pointer("stream-reader",
+                              "ipc::StreamReader",
+                              "The raw std::shared<arrow::ipc::StreamReader> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_STREAM_READER, spec);
+}
+
+/**
+ * garrow_ipc_stream_reader_open:
+ * @stream: The stream to be read.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): A newly opened
+ *   #GArrowIPCStreamReader or %NULL on error.
+ */
+GArrowIPCStreamReader *
+garrow_ipc_stream_reader_open(GArrowIOInputStream *stream,
+                              GError **error)
+{
+  std::shared_ptr<arrow::ipc::StreamReader> arrow_stream_reader;
+  auto status =
+    arrow::ipc::StreamReader::Open(garrow_io_input_stream_get_raw(stream),
+                                   &arrow_stream_reader);
+  if (status.ok()) {
+    return garrow_ipc_stream_reader_new_raw(&arrow_stream_reader);
+  } else {
+    garrow_error_set(error, status, "[ipc][stream-reader][open]");
+    return NULL;
+  }
+}
+
+/**
+ * garrow_ipc_stream_reader_get_schema:
+ * @stream_reader: A #GArrowIPCStreamReader.
+ *
+ * Returns: (transfer full): The schema in the stream.
+ */
+GArrowSchema *
+garrow_ipc_stream_reader_get_schema(GArrowIPCStreamReader *stream_reader)
+{
+  auto arrow_stream_reader =
+    garrow_ipc_stream_reader_get_raw(stream_reader);
+  auto arrow_schema = arrow_stream_reader->schema();
+  return garrow_schema_new_raw(&arrow_schema);
+}
+
+/**
+ * garrow_ipc_stream_reader_get_next_record_batch:
+ * @stream_reader: A #GArrowIPCStreamReader.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The next record batch in the stream or %NULL on end of stream.
+ */
+GArrowRecordBatch *
+garrow_ipc_stream_reader_get_next_record_batch(GArrowIPCStreamReader *stream_reader,
+                                               GError **error)
+{
+  auto arrow_stream_reader =
+    garrow_ipc_stream_reader_get_raw(stream_reader);
+  std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
+  auto status = arrow_stream_reader->GetNextRecordBatch(&arrow_record_batch);
+
+  if (status.ok()) {
+    if (arrow_record_batch == nullptr) {
+      return NULL;
+    } else {
+      return garrow_record_batch_new_raw(&arrow_record_batch);
+    }
+  } else {
+    garrow_error_set(error, status, "[ipc][stream-reader][get-next-record-batch]");
+    return NULL;
+  }
+}
+
+G_END_DECLS
+
+GArrowIPCStreamReader *
+garrow_ipc_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_stream_reader)
+{
+  auto stream_reader =
+    GARROW_IPC_STREAM_READER(g_object_new(GARROW_IPC_TYPE_STREAM_READER,
+                                          "stream-reader", arrow_stream_reader,
+                                          NULL));
+  return stream_reader;
+}
+
+std::shared_ptr<arrow::ipc::StreamReader>
+garrow_ipc_stream_reader_get_raw(GArrowIPCStreamReader *stream_reader)
+{
+  GArrowIPCStreamReaderPrivate *priv;
+
+  priv = GARROW_IPC_STREAM_READER_GET_PRIVATE(stream_reader);
+  return priv->stream_reader;
+}
diff --git a/c_glib/arrow-glib/ipc-stream-reader.h b/c_glib/arrow-glib/ipc-stream-reader.h
new file mode 100644
index 0000000000000..993cd85003bb9
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-stream-reader.h
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+
+#include <arrow-glib/io-input-stream.h>
+
+#include <arrow-glib/ipc-metadata-version.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IPC_TYPE_STREAM_READER           \
+  (garrow_ipc_stream_reader_get_type())
+#define GARROW_IPC_STREAM_READER(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IPC_TYPE_STREAM_READER,    \
+                              GArrowIPCStreamReader))
+#define GARROW_IPC_STREAM_READER_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_IPC_TYPE_STREAM_READER,       \
+                           GArrowIPCStreamReaderClass))
+#define GARROW_IPC_IS_STREAM_READER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IPC_TYPE_STREAM_READER))
+#define GARROW_IPC_IS_STREAM_READER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_IPC_TYPE_STREAM_READER))
+#define GARROW_IPC_STREAM_READER_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_IPC_TYPE_STREAM_READER,     \
+                             GArrowIPCStreamReaderClass))
+
+typedef struct _GArrowIPCStreamReader         GArrowIPCStreamReader;
+typedef struct _GArrowIPCStreamReaderClass    GArrowIPCStreamReaderClass;
+
+/**
+ * GArrowIPCStreamReader:
+ *
+ * It wraps `arrow::ipc::StreamReader`.
+ */
+struct _GArrowIPCStreamReader
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowIPCStreamReaderClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_ipc_stream_reader_get_type(void) G_GNUC_CONST;
+
+GArrowIPCStreamReader *garrow_ipc_stream_reader_open(GArrowIOInputStream *stream,
+                                                     GError **error);
+
+GArrowSchema *garrow_ipc_stream_reader_get_schema(GArrowIPCStreamReader *stream_reader);
+GArrowRecordBatch *garrow_ipc_stream_reader_get_next_record_batch(GArrowIPCStreamReader *stream_reader,
+                                                                  GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-stream-reader.hpp b/c_glib/arrow-glib/ipc-stream-reader.hpp
new file mode 100644
index 0000000000000..a35bdab7e69d4
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-stream-reader.hpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/ipc-stream-reader.h>
+
+GArrowIPCStreamReader *garrow_ipc_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_stream_reader);
+std::shared_ptr<arrow::ipc::StreamReader> garrow_ipc_stream_reader_get_raw(GArrowIPCStreamReader *stream_reader);
diff --git a/c_glib/arrow-glib/ipc-stream-writer.cpp b/c_glib/arrow-glib/ipc-stream-writer.cpp
new file mode 100644
index 0000000000000..e2455a4a9c61c
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-stream-writer.cpp
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/record-batch.hpp>
+#include <arrow-glib/schema.hpp>
+
+#include <arrow-glib/io-output-stream.hpp>
+
+#include <arrow-glib/ipc-stream-writer.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: ipc-stream-writer
+ * @short_description: Stream writer class
+ *
+ * #GArrowIPCStreamWriter is a class for sending data by stream based
+ * IPC.
+ */
+
+typedef struct GArrowIPCStreamWriterPrivate_ {
+  std::shared_ptr<arrow::ipc::StreamWriter> stream_writer;
+} GArrowIPCStreamWriterPrivate;
+
+enum {
+  PROP_0,
+  PROP_STREAM_WRITER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowIPCStreamWriter,
+                           garrow_ipc_stream_writer,
+                           G_TYPE_OBJECT);
+
+#define GARROW_IPC_STREAM_WRITER_GET_PRIVATE(obj)               \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_IPC_TYPE_STREAM_WRITER,   \
+                               GArrowIPCStreamWriterPrivate))
+
+static void
+garrow_ipc_stream_writer_finalize(GObject *object)
+{
+  GArrowIPCStreamWriterPrivate *priv;
+
+  priv = GARROW_IPC_STREAM_WRITER_GET_PRIVATE(object);
+
+  priv->stream_writer = nullptr;
+
+  G_OBJECT_CLASS(garrow_ipc_stream_writer_parent_class)->finalize(object);
+}
+
+static void
+garrow_ipc_stream_writer_set_property(GObject *object,
+                                    guint prop_id,
+                                    const GValue *value,
+                                    GParamSpec *pspec)
+{
+  GArrowIPCStreamWriterPrivate *priv;
+
+  priv = GARROW_IPC_STREAM_WRITER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_STREAM_WRITER:
+    priv->stream_writer =
+      *static_cast<std::shared_ptr<arrow::ipc::StreamWriter> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_ipc_stream_writer_get_property(GObject *object,
+                                          guint prop_id,
+                                          GValue *value,
+                                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_ipc_stream_writer_init(GArrowIPCStreamWriter *object)
+{
+}
+
+static void
+garrow_ipc_stream_writer_class_init(GArrowIPCStreamWriterClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_ipc_stream_writer_finalize;
+  gobject_class->set_property = garrow_ipc_stream_writer_set_property;
+  gobject_class->get_property = garrow_ipc_stream_writer_get_property;
+
+  spec = g_param_spec_pointer("stream-writer",
+                              "ipc::StreamWriter",
+                              "The raw std::shared<arrow::ipc::StreamWriter> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_STREAM_WRITER, spec);
+}
+
+/**
+ * garrow_ipc_stream_writer_open:
+ * @sink: The output of the writer.
+ * @schema: The schema of the writer.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): A newly opened
+ *   #GArrowIPCStreamWriter or %NULL on error.
+ */
+GArrowIPCStreamWriter *
+garrow_ipc_stream_writer_open(GArrowIOOutputStream *sink,
+                              GArrowSchema *schema,
+                              GError **error)
+{
+  std::shared_ptr<arrow::ipc::StreamWriter> arrow_stream_writer;
+  auto status =
+    arrow::ipc::StreamWriter::Open(garrow_io_output_stream_get_raw(sink).get(),
+                                 garrow_schema_get_raw(schema),
+                                 &arrow_stream_writer);
+  if (status.ok()) {
+    return garrow_ipc_stream_writer_new_raw(&arrow_stream_writer);
+  } else {
+    garrow_error_set(error, status, "[ipc][stream-writer][open]");
+    return NULL;
+  }
+}
+
+/**
+ * garrow_ipc_stream_writer_write_record_batch:
+ * @stream_writer: A #GArrowIPCStreamWriter.
+ * @record_batch: The record batch to be written.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_ipc_stream_writer_write_record_batch(GArrowIPCStreamWriter *stream_writer,
+                                            GArrowRecordBatch *record_batch,
+                                            GError **error)
+{
+  auto arrow_stream_writer =
+    garrow_ipc_stream_writer_get_raw(stream_writer);
+  auto arrow_record_batch =
+    garrow_record_batch_get_raw(record_batch);
+  auto arrow_record_batch_raw =
+    arrow_record_batch.get();
+
+  auto status = arrow_stream_writer->WriteRecordBatch(*arrow_record_batch_raw);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[ipc][stream-writer][write-record-batch]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_ipc_stream_writer_close:
+ * @stream_writer: A #GArrowIPCStreamWriter.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_ipc_stream_writer_close(GArrowIPCStreamWriter *stream_writer,
+                               GError **error)
+{
+  auto arrow_stream_writer =
+    garrow_ipc_stream_writer_get_raw(stream_writer);
+
+  auto status = arrow_stream_writer->Close();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[ipc][stream-writer][close]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
+
+GArrowIPCStreamWriter *
+garrow_ipc_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer)
+{
+  auto stream_writer =
+    GARROW_IPC_STREAM_WRITER(g_object_new(GARROW_IPC_TYPE_STREAM_WRITER,
+                                        "stream-writer", arrow_stream_writer,
+                                        NULL));
+  return stream_writer;
+}
+
+std::shared_ptr<arrow::ipc::StreamWriter>
+garrow_ipc_stream_writer_get_raw(GArrowIPCStreamWriter *stream_writer)
+{
+  GArrowIPCStreamWriterPrivate *priv;
+
+  priv = GARROW_IPC_STREAM_WRITER_GET_PRIVATE(stream_writer);
+  return priv->stream_writer;
+}
diff --git a/c_glib/arrow-glib/ipc-stream-writer.h b/c_glib/arrow-glib/ipc-stream-writer.h
new file mode 100644
index 0000000000000..4488204736d51
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-stream-writer.h
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+
+#include <arrow-glib/io-output-stream.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IPC_TYPE_STREAM_WRITER           \
+  (garrow_ipc_stream_writer_get_type())
+#define GARROW_IPC_STREAM_WRITER(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_IPC_TYPE_STREAM_WRITER,    \
+                              GArrowIPCStreamWriter))
+#define GARROW_IPC_STREAM_WRITER_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_IPC_TYPE_STREAM_WRITER,       \
+                           GArrowIPCStreamWriterClass))
+#define GARROW_IPC_IS_STREAM_WRITER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_IPC_TYPE_STREAM_WRITER))
+#define GARROW_IPC_IS_STREAM_WRITER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_IPC_TYPE_STREAM_WRITER))
+#define GARROW_IPC_STREAM_WRITER_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_IPC_TYPE_STREAM_WRITER,     \
+                             GArrowIPCStreamWriterClass))
+
+typedef struct _GArrowIPCStreamWriter         GArrowIPCStreamWriter;
+typedef struct _GArrowIPCStreamWriterClass    GArrowIPCStreamWriterClass;
+
+/**
+ * GArrowIPCStreamWriter:
+ *
+ * It wraps `arrow::ipc::StreamWriter`.
+ */
+struct _GArrowIPCStreamWriter
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowIPCStreamWriterClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_ipc_stream_writer_get_type(void) G_GNUC_CONST;
+
+GArrowIPCStreamWriter *garrow_ipc_stream_writer_open(GArrowIOOutputStream *sink,
+                                                     GArrowSchema *schema,
+                                                     GError **error);
+
+gboolean garrow_ipc_stream_writer_write_record_batch(GArrowIPCStreamWriter *stream_writer,
+                                                     GArrowRecordBatch *record_batch,
+                                                     GError **error);
+gboolean garrow_ipc_stream_writer_close(GArrowIPCStreamWriter *stream_writer,
+                                        GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-stream-writer.hpp b/c_glib/arrow-glib/ipc-stream-writer.hpp
new file mode 100644
index 0000000000000..9d097404582a9
--- /dev/null
+++ b/c_glib/arrow-glib/ipc-stream-writer.hpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/ipc-stream-writer.h>
+
+GArrowIPCStreamWriter *garrow_ipc_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer);
+std::shared_ptr<arrow::ipc::StreamWriter> garrow_ipc_stream_writer_get_raw(GArrowIPCStreamWriter *stream_writer);
diff --git a/c_glib/arrow-glib/list-array-builder.cpp b/c_glib/arrow-glib/list-array-builder.cpp
new file mode 100644
index 0000000000000..6c8f53da1fc98
--- /dev/null
+++ b/c_glib/arrow-glib/list-array-builder.cpp
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/list-array-builder.h>
+#include <arrow-glib/error.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: list-array-builder
+ * @short_description: List array builder class
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowListArrayBuilder is the class to create a new
+ * #GArrowListArray.
+ */
+
+G_DEFINE_TYPE(GArrowListArrayBuilder,
+              garrow_list_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_list_array_builder_init(GArrowListArrayBuilder *builder)
+{
+}
+
+static void
+garrow_list_array_builder_class_init(GArrowListArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_list_array_builder_new:
+ * @value_builder: A #GArrowArrayBuilder for value array.
+ *
+ * Returns: A newly created #GArrowListArrayBuilder.
+ */
+GArrowListArrayBuilder *
+garrow_list_array_builder_new(GArrowArrayBuilder *value_builder)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_value_builder = garrow_array_builder_get_raw(value_builder);
+  auto arrow_list_builder =
+    std::make_shared<arrow::ListBuilder>(memory_pool, arrow_value_builder);
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_list_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_LIST_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_list_array_builder_append:
+ * @builder: A #GArrowListArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * It appends a new list element. To append a new list element, you
+ * need to call this function then append list element values to
+ * `value_builder`. `value_builder` is the #GArrowArrayBuilder
+ * specified to constructor. You can get `value_builder` by
+ * garrow_list_array_builder_get_value_builder().
+ *
+ * |[<!-- language="C" -->
+ * GArrowInt8ArrayBuilder *value_builder;
+ * GArrowListArrayBuilder *builder;
+ *
+ * value_builder = garrow_int8_array_builder_new();
+ * builder = garrow_list_array_builder_new(value_builder, NULL);
+ *
+ * // Start 0th list element: [1, 0, -1]
+ * garrow_list_array_builder_append(builder, NULL);
+ * garrow_int8_array_builder_append(value_builder, 1);
+ * garrow_int8_array_builder_append(value_builder, 0);
+ * garrow_int8_array_builder_append(value_builder, -1);
+ *
+ * // Start 1st list element: [-29, 29]
+ * garrow_list_array_builder_append(builder, NULL);
+ * garrow_int8_array_builder_append(value_builder, -29);
+ * garrow_int8_array_builder_append(value_builder, 29);
+ *
+ * {
+ *   // [[1, 0, -1], [-29, 29]]
+ *   GArrowArray *array = garrow_array_builder_finish(builder);
+ *   // Now, builder is needless.
+ *   g_object_unref(builder);
+ *   g_object_unref(value_builder);
+ *
+ *   // Use array...
+ *   g_object_unref(array);
+ * }
+ * ]|
+ */
+gboolean
+garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
+                                 GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::ListBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[list-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_list_array_builder_append_null:
+ * @builder: A #GArrowListArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * It appends a new NULL element.
+ */
+gboolean
+garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
+                                      GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::ListBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[list-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_list_array_builder_get_value_builder:
+ * @builder: A #GArrowListArrayBuilder.
+ *
+ * Returns: (transfer full): The #GArrowArrayBuilder for values.
+ */
+GArrowArrayBuilder *
+garrow_list_array_builder_get_value_builder(GArrowListArrayBuilder *builder)
+{
+  auto arrow_builder =
+    static_cast<arrow::ListBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+  auto arrow_value_builder = arrow_builder->value_builder();
+  return garrow_array_builder_new_raw(&arrow_value_builder);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/list-array-builder.h b/c_glib/arrow-glib/list-array-builder.h
new file mode 100644
index 0000000000000..2c2e58e54309b
--- /dev/null
+++ b/c_glib/arrow-glib/list-array-builder.h
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_LIST_ARRAY_BUILDER          \
+  (garrow_list_array_builder_get_type())
+#define GARROW_LIST_ARRAY_BUILDER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_LIST_ARRAY_BUILDER,   \
+                              GArrowListArrayBuilder))
+#define GARROW_LIST_ARRAY_BUILDER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_LIST_ARRAY_BUILDER,      \
+                           GArrowListArrayBuilderClass))
+#define GARROW_IS_LIST_ARRAY_BUILDER(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_LIST_ARRAY_BUILDER))
+#define GARROW_IS_LIST_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_LIST_ARRAY_BUILDER))
+#define GARROW_LIST_ARRAY_BUILDER_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_LIST_ARRAY_BUILDER,    \
+                             GArrowListArrayBuilderClass))
+
+typedef struct _GArrowListArrayBuilder         GArrowListArrayBuilder;
+typedef struct _GArrowListArrayBuilderClass    GArrowListArrayBuilderClass;
+
+/**
+ * GArrowListArrayBuilder:
+ *
+ * It wraps `arrow::ListBuilder`.
+ */
+struct _GArrowListArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowListArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_list_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowListArrayBuilder *garrow_list_array_builder_new(GArrowArrayBuilder *value_builder);
+
+gboolean garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
+                                          GError **error);
+gboolean garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
+                                               GError **error);
+
+GArrowArrayBuilder *garrow_list_array_builder_get_value_builder(GArrowListArrayBuilder *builder);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/list-array.cpp b/c_glib/arrow-glib/list-array.cpp
new file mode 100644
index 0000000000000..2b3fb311280d0
--- /dev/null
+++ b/c_glib/arrow-glib/list-array.cpp
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/list-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: list-array
+ * @short_description: List array class
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowListArray is a class for list array. It can store zero
+ * or more list data.
+ *
+ * #GArrowListArray is immutable. You need to use
+ * #GArrowListArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowListArray,               \
+              garrow_list_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_list_array_init(GArrowListArray *object)
+{
+}
+
+static void
+garrow_list_array_class_init(GArrowListArrayClass *klass)
+{
+}
+
+/**
+ * garrow_list_array_get_value_type:
+ * @array: A #GArrowListArray.
+ *
+ * Returns: (transfer full): The data type of value in each list.
+ */
+GArrowDataType *
+garrow_list_array_get_value_type(GArrowListArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_list_array =
+    static_cast<arrow::ListArray *>(arrow_array.get());
+  auto arrow_value_type = arrow_list_array->value_type();
+  return garrow_data_type_new_raw(&arrow_value_type);
+}
+
+/**
+ * garrow_list_array_get_value:
+ * @array: A #GArrowListArray.
+ * @i: The index of the target value.
+ *
+ * Returns: (transfer full): The i-th list.
+ */
+GArrowArray *
+garrow_list_array_get_value(GArrowListArray *array,
+                            gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_list_array =
+    static_cast<arrow::ListArray *>(arrow_array.get());
+  auto arrow_list =
+    arrow_list_array->values()->Slice(arrow_list_array->value_offset(i),
+                                      arrow_list_array->value_length(i));
+  return garrow_array_new_raw(&arrow_list);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/list-array.h b/c_glib/arrow-glib/list-array.h
new file mode 100644
index 0000000000000..c49aed1b9599e
--- /dev/null
+++ b/c_glib/arrow-glib/list-array.h
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_LIST_ARRAY                  \
+  (garrow_list_array_get_type())
+#define GARROW_LIST_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_LIST_ARRAY,   \
+                              GArrowListArray))
+#define GARROW_LIST_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_LIST_ARRAY,      \
+                           GArrowListArrayClass))
+#define GARROW_IS_LIST_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_LIST_ARRAY))
+#define GARROW_IS_LIST_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_LIST_ARRAY))
+#define GARROW_LIST_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_LIST_ARRAY,    \
+                             GArrowListArrayClass))
+
+typedef struct _GArrowListArray         GArrowListArray;
+typedef struct _GArrowListArrayClass    GArrowListArrayClass;
+
+/**
+ * GArrowListArray:
+ *
+ * It wraps `arrow::ListArray`.
+ */
+struct _GArrowListArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowListArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_list_array_get_type(void) G_GNUC_CONST;
+
+GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array);
+GArrowArray *garrow_list_array_get_value(GArrowListArray *array,
+                                         gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/list-data-type.cpp b/c_glib/arrow-glib/list-data-type.cpp
new file mode 100644
index 0000000000000..e82e6fdee48ba
--- /dev/null
+++ b/c_glib/arrow-glib/list-data-type.cpp
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/field.hpp>
+#include <arrow-glib/list-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: list-data-type
+ * @short_description: List data type
+ *
+ * #GArrowListDataType is a class for list data type.
+ */
+
+G_DEFINE_TYPE(GArrowListDataType,                \
+              garrow_list_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_list_data_type_init(GArrowListDataType *object)
+{
+}
+
+static void
+garrow_list_data_type_class_init(GArrowListDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_list_data_type_new:
+ * @field: The field of elements
+ *
+ * Returns: The newly created list data type.
+ */
+GArrowListDataType *
+garrow_list_data_type_new(GArrowField *field)
+{
+  auto arrow_field = garrow_field_get_raw(field);
+  auto arrow_data_type =
+    std::make_shared<arrow::ListType>(arrow_field);
+
+  GArrowListDataType *data_type =
+    GARROW_LIST_DATA_TYPE(g_object_new(GARROW_TYPE_LIST_DATA_TYPE,
+                                       "data-type", &arrow_data_type,
+                                       NULL));
+  return data_type;
+}
+
+/**
+ * garrow_list_data_type_get_value_field:
+ * @list_data_type: A #GArrowListDataType.
+ *
+ * Returns: (transfer full): The field of value.
+ */
+GArrowField *
+garrow_list_data_type_get_value_field(GArrowListDataType *list_data_type)
+{
+  auto arrow_data_type =
+    garrow_data_type_get_raw(GARROW_DATA_TYPE(list_data_type));
+  auto arrow_list_data_type =
+    static_cast<arrow::ListType *>(arrow_data_type.get());
+
+  auto arrow_field = arrow_list_data_type->value_field();
+  auto field = garrow_field_new_raw(&arrow_field);
+
+  return field;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/list-data-type.h b/c_glib/arrow-glib/list-data-type.h
new file mode 100644
index 0000000000000..bb406e2c62074
--- /dev/null
+++ b/c_glib/arrow-glib/list-data-type.h
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+#include <arrow-glib/field.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_LIST_DATA_TYPE              \
+  (garrow_list_data_type_get_type())
+#define GARROW_LIST_DATA_TYPE(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_LIST_DATA_TYPE,       \
+                              GArrowListDataType))
+#define GARROW_LIST_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_LIST_DATA_TYPE,  \
+                           GArrowListDataTypeClass))
+#define GARROW_IS_LIST_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_LIST_DATA_TYPE))
+#define GARROW_IS_LIST_DATA_TYPE_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_LIST_DATA_TYPE))
+#define GARROW_LIST_DATA_TYPE_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_LIST_DATA_TYPE,        \
+                             GArrowListDataTypeClass))
+
+typedef struct _GArrowListDataType         GArrowListDataType;
+typedef struct _GArrowListDataTypeClass    GArrowListDataTypeClass;
+
+/**
+ * GArrowListDataType:
+ *
+ * It wraps `arrow::ListType`.
+ */
+struct _GArrowListDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowListDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType               garrow_list_data_type_get_type (void) G_GNUC_CONST;
+
+GArrowListDataType *garrow_list_data_type_new      (GArrowField *field);
+
+GArrowField *garrow_list_data_type_get_value_field (GArrowListDataType *list_data_type);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/null-array.cpp b/c_glib/arrow-glib/null-array.cpp
new file mode 100644
index 0000000000000..0e0ea51e24c04
--- /dev/null
+++ b/c_glib/arrow-glib/null-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/null-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: null-array
+ * @short_description: Null array class
+ *
+ * #GArrowNullArray is a class for null array. It can store zero
+ * or more null values.
+ *
+ * #GArrowNullArray is immutable. You need to specify an array length
+ * to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowNullArray,               \
+              garrow_null_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_null_array_init(GArrowNullArray *object)
+{
+}
+
+static void
+garrow_null_array_class_init(GArrowNullArrayClass *klass)
+{
+}
+
+/**
+ * garrow_null_array_new:
+ * @length: An array length.
+ *
+ * Returns: A newly created #GArrowNullArray.
+ */
+GArrowNullArray *
+garrow_null_array_new(gint64 length)
+{
+  auto arrow_null_array = std::make_shared<arrow::NullArray>(length);
+  std::shared_ptr<arrow::Array> arrow_array = arrow_null_array;
+  auto array = garrow_array_new_raw(&arrow_array);
+  return GARROW_NULL_ARRAY(array);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/null-array.h b/c_glib/arrow-glib/null-array.h
new file mode 100644
index 0000000000000..e25f3054843e4
--- /dev/null
+++ b/c_glib/arrow-glib/null-array.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_NULL_ARRAY                  \
+  (garrow_null_array_get_type())
+#define GARROW_NULL_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_NULL_ARRAY,   \
+                              GArrowNullArray))
+#define GARROW_NULL_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_NULL_ARRAY,      \
+                           GArrowNullArrayClass))
+#define GARROW_IS_NULL_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_NULL_ARRAY))
+#define GARROW_IS_NULL_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_NULL_ARRAY))
+#define GARROW_NULL_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_NULL_ARRAY,    \
+                             GArrowNullArrayClass))
+
+typedef struct _GArrowNullArray         GArrowNullArray;
+typedef struct _GArrowNullArrayClass    GArrowNullArrayClass;
+
+/**
+ * GArrowNullArray:
+ *
+ * It wraps `arrow::NullArray`.
+ */
+struct _GArrowNullArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowNullArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_null_array_get_type(void) G_GNUC_CONST;
+
+GArrowNullArray *garrow_null_array_new(gint64 length);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/null-data-type.cpp b/c_glib/arrow-glib/null-data-type.cpp
new file mode 100644
index 0000000000000..1f75d3bb88c37
--- /dev/null
+++ b/c_glib/arrow-glib/null-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/null-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: null-data-type
+ * @short_description: Null data type
+ *
+ * #GArrowNullDataType is a class for null data type.
+ */
+
+G_DEFINE_TYPE(GArrowNullDataType,                \
+              garrow_null_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_null_data_type_init(GArrowNullDataType *object)
+{
+}
+
+static void
+garrow_null_data_type_class_init(GArrowNullDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_null_data_type_new:
+ *
+ * Returns: The newly created null data type.
+ */
+GArrowNullDataType *
+garrow_null_data_type_new(void)
+{
+  auto arrow_data_type = arrow::null();
+
+  GArrowNullDataType *data_type =
+    GARROW_NULL_DATA_TYPE(g_object_new(GARROW_TYPE_NULL_DATA_TYPE,
+                                       "data-type", &arrow_data_type,
+                                       NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/null-data-type.h b/c_glib/arrow-glib/null-data-type.h
new file mode 100644
index 0000000000000..006b76c961f3b
--- /dev/null
+++ b/c_glib/arrow-glib/null-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_NULL_DATA_TYPE              \
+  (garrow_null_data_type_get_type())
+#define GARROW_NULL_DATA_TYPE(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_NULL_DATA_TYPE,       \
+                              GArrowNullDataType))
+#define GARROW_NULL_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_NULL_DATA_TYPE,  \
+                           GArrowNullDataTypeClass))
+#define GARROW_IS_NULL_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_NULL_DATA_TYPE))
+#define GARROW_IS_NULL_DATA_TYPE_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_NULL_DATA_TYPE))
+#define GARROW_NULL_DATA_TYPE_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_NULL_DATA_TYPE,        \
+                             GArrowNullDataTypeClass))
+
+typedef struct _GArrowNullDataType         GArrowNullDataType;
+typedef struct _GArrowNullDataTypeClass    GArrowNullDataTypeClass;
+
+/**
+ * GArrowNullDataType:
+ *
+ * It wraps `arrow::NullType`.
+ */
+struct _GArrowNullDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowNullDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType               garrow_null_data_type_get_type (void) G_GNUC_CONST;
+GArrowNullDataType *garrow_null_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp
new file mode 100644
index 0000000000000..8ac1791feef8c
--- /dev/null
+++ b/c_glib/arrow-glib/record-batch.cpp
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/record-batch.hpp>
+#include <arrow-glib/schema.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: record-batch
+ * @short_description: Record batch class
+ *
+ * #GArrowRecordBatch is a class for record batch. Record batch is
+ * similar to #GArrowTable. Record batch also has also zero or more
+ * columns and zero or more records.
+ *
+ * Record batch is used for shared memory IPC.
+ */
+
+typedef struct GArrowRecordBatchPrivate_ {
+  std::shared_ptr<arrow::RecordBatch> record_batch;
+} GArrowRecordBatchPrivate;
+
+enum {
+  PROP_0,
+  PROP_RECORD_BATCH
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowRecordBatch,
+                           garrow_record_batch,
+                           G_TYPE_OBJECT)
+
+#define GARROW_RECORD_BATCH_GET_PRIVATE(obj)               \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),               \
+                               GARROW_TYPE_RECORD_BATCH,   \
+                               GArrowRecordBatchPrivate))
+
+static void
+garrow_record_batch_finalize(GObject *object)
+{
+  GArrowRecordBatchPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_GET_PRIVATE(object);
+
+  priv->record_batch = nullptr;
+
+  G_OBJECT_CLASS(garrow_record_batch_parent_class)->finalize(object);
+}
+
+static void
+garrow_record_batch_set_property(GObject *object,
+                          guint prop_id,
+                          const GValue *value,
+                          GParamSpec *pspec)
+{
+  GArrowRecordBatchPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_RECORD_BATCH:
+    priv->record_batch =
+      *static_cast<std::shared_ptr<arrow::RecordBatch> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_record_batch_get_property(GObject *object,
+                          guint prop_id,
+                          GValue *value,
+                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_record_batch_init(GArrowRecordBatch *object)
+{
+}
+
+static void
+garrow_record_batch_class_init(GArrowRecordBatchClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_record_batch_finalize;
+  gobject_class->set_property = garrow_record_batch_set_property;
+  gobject_class->get_property = garrow_record_batch_get_property;
+
+  spec = g_param_spec_pointer("record-batch",
+                              "RecordBatch",
+                              "The raw std::shared<arrow::RecordBatch> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_RECORD_BATCH, spec);
+}
+
+/**
+ * garrow_record_batch_new:
+ * @schema: The schema of the record batch.
+ * @n_rows: The number of the rows in the record batch.
+ * @columns: (element-type GArrowArray): The columns in the record batch.
+ *
+ * Returns: A newly created #GArrowRecordBatch.
+ */
+GArrowRecordBatch *
+garrow_record_batch_new(GArrowSchema *schema,
+                        guint32 n_rows,
+                        GList *columns)
+{
+  std::vector<std::shared_ptr<arrow::Array>> arrow_columns;
+  for (GList *node = columns; node; node = node->next) {
+    GArrowArray *column = GARROW_ARRAY(node->data);
+    arrow_columns.push_back(garrow_array_get_raw(column));
+  }
+
+  auto arrow_record_batch =
+    std::make_shared<arrow::RecordBatch>(garrow_schema_get_raw(schema),
+                                         n_rows,
+                                         arrow_columns);
+  return garrow_record_batch_new_raw(&arrow_record_batch);
+}
+
+/**
+ * garrow_record_batch_get_schema:
+ * @record_batch: A #GArrowRecordBatch.
+ *
+ * Returns: (transfer full): The schema of the record batch.
+ */
+GArrowSchema *
+garrow_record_batch_get_schema(GArrowRecordBatch *record_batch)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  auto arrow_schema = arrow_record_batch->schema();
+  return garrow_schema_new_raw(&arrow_schema);
+}
+
+/**
+ * garrow_record_batch_get_column:
+ * @record_batch: A #GArrowRecordBatch.
+ * @i: The index of the target column.
+ *
+ * Returns: (transfer full): The i-th column in the record batch.
+ */
+GArrowArray *
+garrow_record_batch_get_column(GArrowRecordBatch *record_batch,
+                               guint i)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  auto arrow_column = arrow_record_batch->column(i);
+  return garrow_array_new_raw(&arrow_column);
+}
+
+/**
+ * garrow_record_batch_get_columns:
+ * @record_batch: A #GArrowRecordBatch.
+ *
+ * Returns: (element-type GArrowArray) (transfer full):
+ *   The columns in the record batch.
+ */
+GList *
+garrow_record_batch_get_columns(GArrowRecordBatch *record_batch)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+
+  GList *columns = NULL;
+  for (auto arrow_column : arrow_record_batch->columns()) {
+    GArrowArray *column = garrow_array_new_raw(&arrow_column);
+    columns = g_list_prepend(columns, column);
+  }
+
+  return g_list_reverse(columns);
+}
+
+/**
+ * garrow_record_batch_get_column_name:
+ * @record_batch: A #GArrowRecordBatch.
+ * @i: The index of the target column.
+ *
+ * Returns: The name of the i-th column in the record batch.
+ */
+const gchar *
+garrow_record_batch_get_column_name(GArrowRecordBatch *record_batch,
+                                    guint i)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  return arrow_record_batch->column_name(i).c_str();
+}
+
+/**
+ * garrow_record_batch_get_n_columns:
+ * @record_batch: A #GArrowRecordBatch.
+ *
+ * Returns: The number of columns in the record batch.
+ */
+guint
+garrow_record_batch_get_n_columns(GArrowRecordBatch *record_batch)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  return arrow_record_batch->num_columns();
+}
+
+/**
+ * garrow_record_batch_get_n_rows:
+ * @record_batch: A #GArrowRecordBatch.
+ *
+ * Returns: The number of rows in the record batch.
+ */
+gint64
+garrow_record_batch_get_n_rows(GArrowRecordBatch *record_batch)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  return arrow_record_batch->num_rows();
+}
+
+/**
+ * garrow_record_batch_slice:
+ * @record_batch: A #GArrowRecordBatch.
+ * @offset: The offset of sub #GArrowRecordBatch.
+ * @length: The length of sub #GArrowRecordBatch.
+ *
+ * Returns: (transfer full): The sub #GArrowRecordBatch. It covers
+ *   only from `offset` to `offset + length` range. The sub
+ *   #GArrowRecordBatch shares values with the base
+ *   #GArrowRecordBatch.
+ */
+GArrowRecordBatch *
+garrow_record_batch_slice(GArrowRecordBatch *record_batch,
+                          gint64 offset,
+                          gint64 length)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  auto arrow_sub_record_batch = arrow_record_batch->Slice(offset, length);
+  return garrow_record_batch_new_raw(&arrow_sub_record_batch);
+}
+
+G_END_DECLS
+
+GArrowRecordBatch *
+garrow_record_batch_new_raw(std::shared_ptr<arrow::RecordBatch> *arrow_record_batch)
+{
+  auto record_batch =
+    GARROW_RECORD_BATCH(g_object_new(GARROW_TYPE_RECORD_BATCH,
+                                     "record-batch", arrow_record_batch,
+                                     NULL));
+  return record_batch;
+}
+
+std::shared_ptr<arrow::RecordBatch>
+garrow_record_batch_get_raw(GArrowRecordBatch *record_batch)
+{
+  GArrowRecordBatchPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_GET_PRIVATE(record_batch);
+  return priv->record_batch;
+}
diff --git a/c_glib/arrow-glib/record-batch.h b/c_glib/arrow-glib/record-batch.h
new file mode 100644
index 0000000000000..92eee4d9af973
--- /dev/null
+++ b/c_glib/arrow-glib/record-batch.h
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+#include <arrow-glib/schema.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_RECORD_BATCH                \
+  (garrow_record_batch_get_type())
+#define GARROW_RECORD_BATCH(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_RECORD_BATCH, \
+                              GArrowRecordBatch))
+#define GARROW_RECORD_BATCH_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_RECORD_BATCH,    \
+                           GArrowRecordBatchClass))
+#define GARROW_IS_RECORD_BATCH(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_RECORD_BATCH))
+#define GARROW_IS_RECORD_BATCH_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_RECORD_BATCH))
+#define GARROW_RECORD_BATCH_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_RECORD_BATCH,  \
+                             GArrowRecordBatchClass))
+
+typedef struct _GArrowRecordBatch         GArrowRecordBatch;
+typedef struct _GArrowRecordBatchClass    GArrowRecordBatchClass;
+
+/**
+ * GArrowRecordBatch:
+ *
+ * It wraps `arrow::RecordBatch`.
+ */
+struct _GArrowRecordBatch
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowRecordBatchClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_record_batch_get_type(void) G_GNUC_CONST;
+
+GArrowRecordBatch *garrow_record_batch_new(GArrowSchema *schema,
+                                           guint32 n_rows,
+                                           GList *columns);
+
+GArrowSchema *garrow_record_batch_get_schema     (GArrowRecordBatch *record_batch);
+GArrowArray  *garrow_record_batch_get_column     (GArrowRecordBatch *record_batch,
+                                                  guint i);
+GList        *garrow_record_batch_get_columns    (GArrowRecordBatch *record_batch);
+const gchar  *garrow_record_batch_get_column_name(GArrowRecordBatch *record_batch,
+                                                  guint i);
+guint         garrow_record_batch_get_n_columns  (GArrowRecordBatch *record_batch);
+gint64        garrow_record_batch_get_n_rows     (GArrowRecordBatch *record_batch);
+GArrowRecordBatch *garrow_record_batch_slice     (GArrowRecordBatch *record_batch,
+                                                  gint64 offset,
+                                                  gint64 length);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/record-batch.hpp b/c_glib/arrow-glib/record-batch.hpp
new file mode 100644
index 0000000000000..2e4fe039b4fc5
--- /dev/null
+++ b/c_glib/arrow-glib/record-batch.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/record-batch.h>
+
+GArrowRecordBatch *garrow_record_batch_new_raw(std::shared_ptr<arrow::RecordBatch> *arrow_record_batch);
+std::shared_ptr<arrow::RecordBatch> garrow_record_batch_get_raw(GArrowRecordBatch *record_batch);
diff --git a/c_glib/arrow-glib/schema.cpp b/c_glib/arrow-glib/schema.cpp
new file mode 100644
index 0000000000000..4d5ae5af4fb4a
--- /dev/null
+++ b/c_glib/arrow-glib/schema.cpp
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/field.hpp>
+#include <arrow-glib/schema.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: schema
+ * @short_description: Schema class
+ *
+ * #GArrowSchema is a class for schema. Schema is metadata of a
+ * table. It has zero or more #GArrowFields.
+ */
+
+typedef struct GArrowSchemaPrivate_ {
+  std::shared_ptr<arrow::Schema> schema;
+} GArrowSchemaPrivate;
+
+enum {
+  PROP_0,
+  PROP_SCHEMA
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowSchema,
+                           garrow_schema,
+                           G_TYPE_OBJECT)
+
+#define GARROW_SCHEMA_GET_PRIVATE(obj)                  \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                   \
+                               GARROW_TYPE_SCHEMA,      \
+                               GArrowSchemaPrivate))
+
+static void
+garrow_schema_finalize(GObject *object)
+{
+  GArrowSchemaPrivate *priv;
+
+  priv = GARROW_SCHEMA_GET_PRIVATE(object);
+
+  priv->schema = nullptr;
+
+  G_OBJECT_CLASS(garrow_schema_parent_class)->finalize(object);
+}
+
+static void
+garrow_schema_set_property(GObject *object,
+                           guint prop_id,
+                           const GValue *value,
+                           GParamSpec *pspec)
+{
+  GArrowSchemaPrivate *priv;
+
+  priv = GARROW_SCHEMA_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_SCHEMA:
+    priv->schema =
+      *static_cast<std::shared_ptr<arrow::Schema> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_schema_get_property(GObject *object,
+                           guint prop_id,
+                           GValue *value,
+                           GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_schema_init(GArrowSchema *object)
+{
+}
+
+static void
+garrow_schema_class_init(GArrowSchemaClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_schema_finalize;
+  gobject_class->set_property = garrow_schema_set_property;
+  gobject_class->get_property = garrow_schema_get_property;
+
+  spec = g_param_spec_pointer("schema",
+                              "Schema",
+                              "The raw std::shared<arrow::Schema> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_SCHEMA, spec);
+}
+
+/**
+ * garrow_schema_new:
+ * @fields: (element-type GArrowField): The fields of the schema.
+ *
+ * Returns: A newly created #GArrowSchema.
+ */
+GArrowSchema *
+garrow_schema_new(GList *fields)
+{
+  std::vector<std::shared_ptr<arrow::Field>> arrow_fields;
+  for (GList *node = fields; node; node = node->next) {
+    GArrowField *field = GARROW_FIELD(node->data);
+    arrow_fields.push_back(garrow_field_get_raw(field));
+  }
+
+  auto arrow_schema = std::make_shared<arrow::Schema>(arrow_fields);
+  return garrow_schema_new_raw(&arrow_schema);
+}
+
+/**
+ * garrow_schema_get_field:
+ * @schema: A #GArrowSchema.
+ * @i: The index of the target field.
+ *
+ * Returns: (transfer full): The i-th field of the schema.
+ */
+GArrowField *
+garrow_schema_get_field(GArrowSchema *schema, guint i)
+{
+  const auto arrow_schema = garrow_schema_get_raw(schema);
+  auto arrow_field = arrow_schema->field(i);
+  return garrow_field_new_raw(&arrow_field);
+}
+
+/**
+ * garrow_schema_get_field_by_name:
+ * @schema: A #GArrowSchema.
+ * @name: The name of the field to be found.
+ *
+ * Returns: (transfer full): The found field or %NULL.
+ */
+GArrowField *
+garrow_schema_get_field_by_name(GArrowSchema *schema,
+                                const gchar *name)
+{
+  const auto arrow_schema = garrow_schema_get_raw(schema);
+  auto arrow_field = arrow_schema->GetFieldByName(std::string(name));
+  if (arrow_field == nullptr) {
+    return NULL;
+  } else {
+    return garrow_field_new_raw(&arrow_field);
+  }
+}
+
+/**
+ * garrow_schema_n_fields:
+ * @schema: A #GArrowSchema.
+ *
+ * Returns: The number of fields of the schema.
+ */
+guint
+garrow_schema_n_fields(GArrowSchema *schema)
+{
+  const auto arrow_schema = garrow_schema_get_raw(schema);
+  return arrow_schema->num_fields();
+}
+
+/**
+ * garrow_schema_get_fields:
+ * @schema: A #GArrowSchema.
+ *
+ * Returns: (element-type GArrowField) (transfer full):
+ *   The fields of the schema.
+ */
+GList *
+garrow_schema_get_fields(GArrowSchema *schema)
+{
+  const auto arrow_schema = garrow_schema_get_raw(schema);
+
+  GList *fields = NULL;
+  for (auto arrow_field : arrow_schema->fields()) {
+    GArrowField *field = garrow_field_new_raw(&arrow_field);
+    fields = g_list_prepend(fields, field);
+  }
+
+  return g_list_reverse(fields);
+}
+
+/**
+ * garrow_schema_to_string:
+ * @schema: A #GArrowSchema.
+ *
+ * Returns: The string representation of the schema.
+ */
+gchar *
+garrow_schema_to_string(GArrowSchema *schema)
+{
+  const auto arrow_schema = garrow_schema_get_raw(schema);
+  return g_strdup(arrow_schema->ToString().c_str());
+}
+
+G_END_DECLS
+
+GArrowSchema *
+garrow_schema_new_raw(std::shared_ptr<arrow::Schema> *arrow_schema)
+{
+  auto schema = GARROW_SCHEMA(g_object_new(GARROW_TYPE_SCHEMA,
+                                           "schema", arrow_schema,
+                                           NULL));
+  return schema;
+}
+
+std::shared_ptr<arrow::Schema>
+garrow_schema_get_raw(GArrowSchema *schema)
+{
+  GArrowSchemaPrivate *priv;
+
+  priv = GARROW_SCHEMA_GET_PRIVATE(schema);
+  return priv->schema;
+}
diff --git a/c_glib/arrow-glib/schema.h b/c_glib/arrow-glib/schema.h
new file mode 100644
index 0000000000000..7615634021bc3
--- /dev/null
+++ b/c_glib/arrow-glib/schema.h
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/field.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_SCHEMA                      \
+  (garrow_schema_get_type())
+#define GARROW_SCHEMA(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_SCHEMA,       \
+                              GArrowSchema))
+#define GARROW_SCHEMA_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),             \
+                           GARROW_TYPE_SCHEMA,  \
+                           GArrowSchemaClass))
+#define GARROW_IS_SCHEMA(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_SCHEMA))
+#define GARROW_IS_SCHEMA_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),             \
+                           GARROW_TYPE_SCHEMA))
+#define GARROW_SCHEMA_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_SCHEMA,        \
+                             GArrowSchemaClass))
+
+typedef struct _GArrowSchema         GArrowSchema;
+typedef struct _GArrowSchemaClass    GArrowSchemaClass;
+
+/**
+ * GArrowSchema:
+ *
+ * It wraps `arrow::Schema`.
+ */
+struct _GArrowSchema
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowSchemaClass
+{
+  GObjectClass parent_class;
+};
+
+GType            garrow_schema_get_type         (void) G_GNUC_CONST;
+
+GArrowSchema    *garrow_schema_new              (GList *fields);
+
+GArrowField     *garrow_schema_get_field        (GArrowSchema *schema,
+                                                 guint i);
+GArrowField     *garrow_schema_get_field_by_name(GArrowSchema *schema,
+                                                 const gchar *name);
+
+guint            garrow_schema_n_fields         (GArrowSchema *schema);
+GList           *garrow_schema_get_fields       (GArrowSchema *schema);
+
+gchar           *garrow_schema_to_string        (GArrowSchema *schema);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/schema.hpp b/c_glib/arrow-glib/schema.hpp
new file mode 100644
index 0000000000000..0d025340844d3
--- /dev/null
+++ b/c_glib/arrow-glib/schema.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/schema.h>
+
+GArrowSchema *garrow_schema_new_raw(std::shared_ptr<arrow::Schema> *arrow_schema);
+std::shared_ptr<arrow::Schema> garrow_schema_get_raw(GArrowSchema *schema);
diff --git a/c_glib/arrow-glib/string-array-builder.cpp b/c_glib/arrow-glib/string-array-builder.cpp
new file mode 100644
index 0000000000000..ebad53a18704a
--- /dev/null
+++ b/c_glib/arrow-glib/string-array-builder.cpp
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/string-array-builder.h>
+#include <arrow-glib/error.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: string-array-builder
+ * @short_description: UTF-8 encoded string array builder class
+ *
+ * #GArrowStringArrayBuilder is the class to create a new
+ * #GArrowStringArray.
+ */
+
+G_DEFINE_TYPE(GArrowStringArrayBuilder,
+              garrow_string_array_builder,
+              GARROW_TYPE_BINARY_ARRAY_BUILDER)
+
+static void
+garrow_string_array_builder_init(GArrowStringArrayBuilder *builder)
+{
+}
+
+static void
+garrow_string_array_builder_class_init(GArrowStringArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_string_array_builder_new:
+ *
+ * Returns: A newly created #GArrowStringArrayBuilder.
+ */
+GArrowStringArrayBuilder *
+garrow_string_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::StringBuilder>(memory_pool);
+  auto builder =
+    GARROW_STRING_ARRAY_BUILDER(g_object_new(GARROW_TYPE_STRING_ARRAY_BUILDER,
+                                             "array-builder", &arrow_builder,
+                                             NULL));
+  return builder;
+}
+
+/**
+ * garrow_string_array_builder_append:
+ * @builder: A #GArrowStringArrayBuilder.
+ * @value: A string value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
+                                   const gchar *value,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::StringBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value,
+                                      static_cast<gint32>(strlen(value)));
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[string-array-builder][append]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/string-array-builder.h b/c_glib/arrow-glib/string-array-builder.h
new file mode 100644
index 0000000000000..f370ed9edec9d
--- /dev/null
+++ b/c_glib/arrow-glib/string-array-builder.h
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/binary-array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_STRING_ARRAY_BUILDER        \
+  (garrow_string_array_builder_get_type())
+#define GARROW_STRING_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STRING_ARRAY_BUILDER, \
+                              GArrowStringArrayBuilder))
+#define GARROW_STRING_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_STRING_ARRAY_BUILDER,    \
+                           GArrowStringArrayBuilderClass))
+#define GARROW_IS_STRING_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_STRING_ARRAY_BUILDER))
+#define GARROW_IS_STRING_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_STRING_ARRAY_BUILDER))
+#define GARROW_STRING_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_STRING_ARRAY_BUILDER,  \
+                             GArrowStringArrayBuilderClass))
+
+typedef struct _GArrowStringArrayBuilder         GArrowStringArrayBuilder;
+typedef struct _GArrowStringArrayBuilderClass    GArrowStringArrayBuilderClass;
+
+/**
+ * GArrowStringArrayBuilder:
+ *
+ * It wraps `arrow::StringBuilder`.
+ */
+struct _GArrowStringArrayBuilder
+{
+  /*< private >*/
+  GArrowBinaryArrayBuilder parent_instance;
+};
+
+struct _GArrowStringArrayBuilderClass
+{
+  GArrowBinaryArrayBuilderClass parent_class;
+};
+
+GType garrow_string_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowStringArrayBuilder *garrow_string_array_builder_new(void);
+
+gboolean garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
+                                            const gchar *value,
+                                            GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/string-array.cpp b/c_glib/arrow-glib/string-array.cpp
new file mode 100644
index 0000000000000..329c742ccafe1
--- /dev/null
+++ b/c_glib/arrow-glib/string-array.cpp
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/string-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: string-array
+ * @short_description: UTF-8 encoded string array class
+ *
+ * #GArrowStringArray is a class for UTF-8 encoded string array. It
+ * can store zero or more UTF-8 encoded string data.
+ *
+ * #GArrowStringArray is immutable. You need to use
+ * #GArrowStringArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowStringArray,               \
+              garrow_string_array,             \
+              GARROW_TYPE_BINARY_ARRAY)
+
+static void
+garrow_string_array_init(GArrowStringArray *object)
+{
+}
+
+static void
+garrow_string_array_class_init(GArrowStringArrayClass *klass)
+{
+}
+
+/**
+ * garrow_string_array_get_string:
+ * @array: A #GArrowStringArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th UTF-8 encoded string.
+ */
+gchar *
+garrow_string_array_get_string(GArrowStringArray *array,
+                               gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_string_array =
+    static_cast<arrow::StringArray *>(arrow_array.get());
+  gint32 length;
+  auto value =
+    reinterpret_cast<const gchar *>(arrow_string_array->GetValue(i, &length));
+  return g_strndup(value, length);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/string-array.h b/c_glib/arrow-glib/string-array.h
new file mode 100644
index 0000000000000..41a53cd5f1d4a
--- /dev/null
+++ b/c_glib/arrow-glib/string-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/binary-array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_STRING_ARRAY                \
+  (garrow_string_array_get_type())
+#define GARROW_STRING_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_STRING_ARRAY, \
+                              GArrowStringArray))
+#define GARROW_STRING_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_STRING_ARRAY,    \
+                           GArrowStringArrayClass))
+#define GARROW_IS_STRING_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRING_ARRAY))
+#define GARROW_IS_STRING_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_STRING_ARRAY))
+#define GARROW_STRING_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_STRING_ARRAY,  \
+                             GArrowStringArrayClass))
+
+typedef struct _GArrowStringArray         GArrowStringArray;
+typedef struct _GArrowStringArrayClass    GArrowStringArrayClass;
+
+/**
+ * GArrowStringArray:
+ *
+ * It wraps `arrow::StringArray`.
+ */
+struct _GArrowStringArray
+{
+  /*< private >*/
+  GArrowBinaryArray parent_instance;
+};
+
+struct _GArrowStringArrayClass
+{
+  GArrowBinaryArrayClass parent_class;
+};
+
+GType garrow_string_array_get_type(void) G_GNUC_CONST;
+
+gchar *garrow_string_array_get_string(GArrowStringArray *array,
+                                      gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/string-data-type.cpp b/c_glib/arrow-glib/string-data-type.cpp
new file mode 100644
index 0000000000000..96a31bf2f906a
--- /dev/null
+++ b/c_glib/arrow-glib/string-data-type.cpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/string-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: string-data-type
+ * @short_description: UTF-8 encoded string data type
+ *
+ * #GArrowStringDataType is a class for UTF-8 encoded string data
+ * type.
+ */
+
+G_DEFINE_TYPE(GArrowStringDataType,                \
+              garrow_string_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_string_data_type_init(GArrowStringDataType *object)
+{
+}
+
+static void
+garrow_string_data_type_class_init(GArrowStringDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_string_data_type_new:
+ *
+ * Returns: The newly created UTF-8 encoded string data type.
+ */
+GArrowStringDataType *
+garrow_string_data_type_new(void)
+{
+  auto arrow_data_type = arrow::utf8();
+
+  GArrowStringDataType *data_type =
+    GARROW_STRING_DATA_TYPE(g_object_new(GARROW_TYPE_STRING_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/string-data-type.h b/c_glib/arrow-glib/string-data-type.h
new file mode 100644
index 0000000000000..d10a325e1bb6c
--- /dev/null
+++ b/c_glib/arrow-glib/string-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_STRING_DATA_TYPE            \
+  (garrow_string_data_type_get_type())
+#define GARROW_STRING_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
+                              GARROW_TYPE_STRING_DATA_TYPE,    \
+                              GArrowStringDataType))
+#define GARROW_STRING_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
+                           GARROW_TYPE_STRING_DATA_TYPE,       \
+                           GArrowStringDataTypeClass))
+#define GARROW_IS_STRING_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
+                              GARROW_TYPE_STRING_DATA_TYPE))
+#define GARROW_IS_STRING_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
+                           GARROW_TYPE_STRING_DATA_TYPE))
+#define GARROW_STRING_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
+                             GARROW_TYPE_STRING_DATA_TYPE,     \
+                             GArrowStringDataTypeClass))
+
+typedef struct _GArrowStringDataType         GArrowStringDataType;
+typedef struct _GArrowStringDataTypeClass    GArrowStringDataTypeClass;
+
+/**
+ * GArrowStringDataType:
+ *
+ * It wraps `arrow::StringType`.
+ */
+struct _GArrowStringDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowStringDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_string_data_type_get_type (void) G_GNUC_CONST;
+GArrowStringDataType *garrow_string_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array-builder.cpp b/c_glib/arrow-glib/struct-array-builder.cpp
new file mode 100644
index 0000000000000..2453a5baf2ec8
--- /dev/null
+++ b/c_glib/arrow-glib/struct-array-builder.cpp
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/field.hpp>
+#include <arrow-glib/struct-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: struct-array-builder
+ * @short_description: Struct array builder class
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowStructArrayBuilder is the class to create a new
+ * #GArrowStructArray.
+ */
+
+G_DEFINE_TYPE(GArrowStructArrayBuilder,
+              garrow_struct_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_struct_array_builder_init(GArrowStructArrayBuilder *builder)
+{
+}
+
+static void
+garrow_struct_array_builder_class_init(GArrowStructArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_struct_array_builder_new:
+ * @data_type: #GArrowStructDataType for the struct.
+ * @field_builders: (element-type GArrowArray): #GArrowArrayBuilders
+ *   for fields.
+ *
+ * Returns: A newly created #GArrowStructArrayBuilder.
+ */
+GArrowStructArrayBuilder *
+garrow_struct_array_builder_new(GArrowStructDataType *data_type,
+                                GList *field_builders)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  std::vector<std::shared_ptr<arrow::ArrayBuilder>> arrow_field_builders;
+  for (GList *node = field_builders; node; node = g_list_next(node)) {
+    auto field_builder = static_cast<GArrowArrayBuilder *>(node->data);
+    auto arrow_field_builder = garrow_array_builder_get_raw(field_builder);
+    arrow_field_builders.push_back(arrow_field_builder);
+  }
+
+  auto arrow_struct_builder =
+    std::make_shared<arrow::StructBuilder>(memory_pool,
+                                           arrow_data_type,
+                                           arrow_field_builders);
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_struct_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_STRUCT_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_struct_array_builder_append:
+ * @builder: A #GArrowStructArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * It appends a new struct element. To append a new struct element,
+ * you need to call this function then append struct element field
+ * values to all `field_builder`s. `field_value`s are the
+ * #GArrowArrayBuilder specified to constructor. You can get
+ * `field_builder` by garrow_struct_array_builder_get_field_builder()
+ * or garrow_struct_array_builder_get_field_builders().
+ *
+ * |[<!-- language="C" -->
+ * // TODO
+ * ]|
+ */
+gboolean
+garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::StructBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[struct-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_struct_array_builder_append_null:
+ * @builder: A #GArrowStructArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * It appends a new NULL element.
+ */
+gboolean
+garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
+                                        GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::StructBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[struct-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_struct_array_builder_get_field_builder:
+ * @builder: A #GArrowStructArrayBuilder.
+ * @i: The index of the field in the struct.
+ *
+ * Returns: (transfer full): The #GArrowArrayBuilder for the i-th field.
+ */
+GArrowArrayBuilder *
+garrow_struct_array_builder_get_field_builder(GArrowStructArrayBuilder *builder,
+                                              gint i)
+{
+  auto arrow_builder =
+    static_cast<arrow::StructBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+  auto arrow_field_builder = arrow_builder->field_builder(i);
+  return garrow_array_builder_new_raw(&arrow_field_builder);
+}
+
+/**
+ * garrow_struct_array_builder_get_field_builders:
+ * @builder: A #GArrowStructArrayBuilder.
+ *
+ * Returns: (element-type GArrowArray) (transfer full):
+ *   The #GArrowArrayBuilder for all fields.
+ */
+GList *
+garrow_struct_array_builder_get_field_builders(GArrowStructArrayBuilder *builder)
+{
+  auto arrow_struct_builder =
+    static_cast<arrow::StructBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  GList *field_builders = NULL;
+  for (auto arrow_field_builder : arrow_struct_builder->field_builders()) {
+    auto field_builder = garrow_array_builder_new_raw(&arrow_field_builder);
+    field_builders = g_list_prepend(field_builders, field_builder);
+  }
+
+  return g_list_reverse(field_builders);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array-builder.h b/c_glib/arrow-glib/struct-array-builder.h
new file mode 100644
index 0000000000000..7dd86625616e3
--- /dev/null
+++ b/c_glib/arrow-glib/struct-array-builder.h
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+#include <arrow-glib/struct-data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_STRUCT_ARRAY_BUILDER        \
+  (garrow_struct_array_builder_get_type())
+#define GARROW_STRUCT_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STRUCT_ARRAY_BUILDER, \
+                              GArrowStructArrayBuilder))
+#define GARROW_STRUCT_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_STRUCT_ARRAY_BUILDER,    \
+                           GArrowStructArrayBuilderClass))
+#define GARROW_IS_STRUCT_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_STRUCT_ARRAY_BUILDER))
+#define GARROW_IS_STRUCT_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_STRUCT_ARRAY_BUILDER))
+#define GARROW_STRUCT_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_STRUCT_ARRAY_BUILDER,  \
+                             GArrowStructArrayBuilderClass))
+
+typedef struct _GArrowStructArrayBuilder         GArrowStructArrayBuilder;
+typedef struct _GArrowStructArrayBuilderClass    GArrowStructArrayBuilderClass;
+
+/**
+ * GArrowStructArrayBuilder:
+ *
+ * It wraps `arrow::StructBuilder`.
+ */
+struct _GArrowStructArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowStructArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_struct_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowStructArrayBuilder *garrow_struct_array_builder_new(GArrowStructDataType *data_type,
+                                                          GList *field_builders);
+
+gboolean garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
+                                            GError **error);
+gboolean garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
+                                                 GError **error);
+
+GArrowArrayBuilder *garrow_struct_array_builder_get_field_builder(GArrowStructArrayBuilder *builder,
+                                                                  gint i);
+GList *garrow_struct_array_builder_get_field_builders(GArrowStructArrayBuilder *builder);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array.cpp b/c_glib/arrow-glib/struct-array.cpp
new file mode 100644
index 0000000000000..14c2d17cdd737
--- /dev/null
+++ b/c_glib/arrow-glib/struct-array.cpp
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/struct-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: struct-array
+ * @short_description: Struct array class
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowStructArray is a class for struct array. It can store zero
+ * or more structs. One struct has zero or more fields.
+ *
+ * #GArrowStructArray is immutable. You need to use
+ * #GArrowStructArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowStructArray,               \
+              garrow_struct_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_struct_array_init(GArrowStructArray *object)
+{
+}
+
+static void
+garrow_struct_array_class_init(GArrowStructArrayClass *klass)
+{
+}
+
+/**
+ * garrow_struct_array_get_field
+ * @array: A #GArrowStructArray.
+ * @i: The index of the field in the struct.
+ *
+ * Returns: (transfer full): The i-th field.
+ */
+GArrowArray *
+garrow_struct_array_get_field(GArrowStructArray *array,
+                              gint i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_struct_array =
+    static_cast<arrow::StructArray *>(arrow_array.get());
+  auto arrow_field = arrow_struct_array->field(i);
+  return garrow_array_new_raw(&arrow_field);
+}
+
+/**
+ * garrow_struct_array_get_fields
+ * @array: A #GArrowStructArray.
+ *
+ * Returns: (element-type GArrowArray) (transfer full):
+ *   The fields in the struct.
+ */
+GList *
+garrow_struct_array_get_fields(GArrowStructArray *array)
+{
+  const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  const auto arrow_struct_array =
+    static_cast<const arrow::StructArray *>(arrow_array.get());
+
+  GList *fields = NULL;
+  for (auto arrow_field : arrow_struct_array->fields()) {
+    GArrowArray *field = garrow_array_new_raw(&arrow_field);
+    fields = g_list_prepend(fields, field);
+  }
+
+  return g_list_reverse(fields);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array.h b/c_glib/arrow-glib/struct-array.h
new file mode 100644
index 0000000000000..f96e9d468f350
--- /dev/null
+++ b/c_glib/arrow-glib/struct-array.h
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_STRUCT_ARRAY                \
+  (garrow_struct_array_get_type())
+#define GARROW_STRUCT_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_STRUCT_ARRAY, \
+                              GArrowStructArray))
+#define GARROW_STRUCT_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_STRUCT_ARRAY,    \
+                           GArrowStructArrayClass))
+#define GARROW_IS_STRUCT_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRUCT_ARRAY))
+#define GARROW_IS_STRUCT_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_STRUCT_ARRAY))
+#define GARROW_STRUCT_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_STRUCT_ARRAY,  \
+                             GArrowStructArrayClass))
+
+typedef struct _GArrowStructArray         GArrowStructArray;
+typedef struct _GArrowStructArrayClass    GArrowStructArrayClass;
+
+/**
+ * GArrowStructArray:
+ *
+ * It wraps `arrow::StructArray`.
+ */
+struct _GArrowStructArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowStructArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_struct_array_get_type(void) G_GNUC_CONST;
+
+GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
+                                           gint i);
+GList *garrow_struct_array_get_fields(GArrowStructArray *array);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-data-type.cpp b/c_glib/arrow-glib/struct-data-type.cpp
new file mode 100644
index 0000000000000..9a4f2a2deead0
--- /dev/null
+++ b/c_glib/arrow-glib/struct-data-type.cpp
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/field.hpp>
+#include <arrow-glib/struct-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: struct-data-type
+ * @short_description: Struct data type
+ *
+ * #GArrowStructDataType is a class for struct data type.
+ */
+
+G_DEFINE_TYPE(GArrowStructDataType,                \
+              garrow_struct_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_struct_data_type_init(GArrowStructDataType *object)
+{
+}
+
+static void
+garrow_struct_data_type_class_init(GArrowStructDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_struct_data_type_new:
+ * @fields: (element-type GArrowField): The fields of the struct.
+ *
+ * Returns: The newly created struct data type.
+ */
+GArrowStructDataType *
+garrow_struct_data_type_new(GList *fields)
+{
+  std::vector<std::shared_ptr<arrow::Field>> arrow_fields;
+  for (GList *node = fields; node; node = g_list_next(node)) {
+    auto field = GARROW_FIELD(node->data);
+    auto arrow_field = garrow_field_get_raw(field);
+    arrow_fields.push_back(arrow_field);
+  }
+
+  auto arrow_data_type = std::make_shared<arrow::StructType>(arrow_fields);
+  GArrowStructDataType *data_type =
+    GARROW_STRUCT_DATA_TYPE(g_object_new(GARROW_TYPE_STRUCT_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-data-type.h b/c_glib/arrow-glib/struct-data-type.h
new file mode 100644
index 0000000000000..0a2c743e280b7
--- /dev/null
+++ b/c_glib/arrow-glib/struct-data-type.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+#include <arrow-glib/field.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_STRUCT_DATA_TYPE            \
+  (garrow_struct_data_type_get_type())
+#define GARROW_STRUCT_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STRUCT_DATA_TYPE,     \
+                              GArrowStructDataType))
+#define GARROW_STRUCT_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_STRUCT_DATA_TYPE,        \
+                           GArrowStructDataTypeClass))
+#define GARROW_IS_STRUCT_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRUCT_DATA_TYPE))
+#define GARROW_IS_STRUCT_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_STRUCT_DATA_TYPE))
+#define GARROW_STRUCT_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_STRUCT_DATA_TYPE,      \
+                             GArrowStructDataTypeClass))
+
+typedef struct _GArrowStructDataType         GArrowStructDataType;
+typedef struct _GArrowStructDataTypeClass    GArrowStructDataTypeClass;
+
+/**
+ * GArrowStructDataType:
+ *
+ * It wraps `arrow::StructType`.
+ */
+struct _GArrowStructDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowStructDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType               garrow_struct_data_type_get_type (void) G_GNUC_CONST;
+
+GArrowStructDataType *garrow_struct_data_type_new(GList *fields);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/table.cpp b/c_glib/arrow-glib/table.cpp
new file mode 100644
index 0000000000000..2410e76c921fb
--- /dev/null
+++ b/c_glib/arrow-glib/table.cpp
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/column.hpp>
+#include <arrow-glib/schema.hpp>
+#include <arrow-glib/table.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: table
+ * @short_description: Table class
+ *
+ * #GArrowTable is a class for table. Table has zero or more
+ * #GArrowColumns and zero or more records.
+ */
+
+typedef struct GArrowTablePrivate_ {
+  std::shared_ptr<arrow::Table> table;
+} GArrowTablePrivate;
+
+enum {
+  PROP_0,
+  PROP_TABLE
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowTable,
+                           garrow_table,
+                           G_TYPE_OBJECT)
+
+#define GARROW_TABLE_GET_PRIVATE(obj)               \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),               \
+                               GARROW_TYPE_TABLE,   \
+                               GArrowTablePrivate))
+
+static void
+garrow_table_dispose(GObject *object)
+{
+  GArrowTablePrivate *priv;
+
+  priv = GARROW_TABLE_GET_PRIVATE(object);
+
+  priv->table = nullptr;
+
+  G_OBJECT_CLASS(garrow_table_parent_class)->dispose(object);
+}
+
+static void
+garrow_table_set_property(GObject *object,
+                          guint prop_id,
+                          const GValue *value,
+                          GParamSpec *pspec)
+{
+  GArrowTablePrivate *priv;
+
+  priv = GARROW_TABLE_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_TABLE:
+    priv->table =
+      *static_cast<std::shared_ptr<arrow::Table> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_table_get_property(GObject *object,
+                          guint prop_id,
+                          GValue *value,
+                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_table_init(GArrowTable *object)
+{
+}
+
+static void
+garrow_table_class_init(GArrowTableClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->dispose      = garrow_table_dispose;
+  gobject_class->set_property = garrow_table_set_property;
+  gobject_class->get_property = garrow_table_get_property;
+
+  spec = g_param_spec_pointer("table",
+                              "Table",
+                              "The raw std::shared<arrow::Table> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_TABLE, spec);
+}
+
+/**
+ * garrow_table_new:
+ * @name: The name of the table.
+ * @schema: The schema of the table.
+ * @columns: (element-type GArrowColumn): The columns of the table.
+ *
+ * Returns: A newly created #GArrowTable.
+ */
+GArrowTable *
+garrow_table_new(const gchar *name,
+                 GArrowSchema *schema,
+                 GList *columns)
+{
+  std::vector<std::shared_ptr<arrow::Column>> arrow_columns;
+  for (GList *node = columns; node; node = node->next) {
+    GArrowColumn *column = GARROW_COLUMN(node->data);
+    arrow_columns.push_back(garrow_column_get_raw(column));
+  }
+
+  auto arrow_table =
+    std::make_shared<arrow::Table>(name,
+                                   garrow_schema_get_raw(schema),
+                                   arrow_columns);
+  return garrow_table_new_raw(&arrow_table);
+}
+
+/**
+ * garrow_table_get_name:
+ * @table: A #GArrowTable.
+ *
+ * Returns: The name of the table.
+ */
+const gchar *
+garrow_table_get_name(GArrowTable *table)
+{
+  const auto arrow_table = garrow_table_get_raw(table);
+  return arrow_table->name().c_str();
+}
+
+/**
+ * garrow_table_get_schema:
+ * @table: A #GArrowTable.
+ *
+ * Returns: (transfer full): The schema of the table.
+ */
+GArrowSchema *
+garrow_table_get_schema(GArrowTable *table)
+{
+  const auto arrow_table = garrow_table_get_raw(table);
+  auto arrow_schema = arrow_table->schema();
+  return garrow_schema_new_raw(&arrow_schema);
+}
+
+/**
+ * garrow_table_get_column:
+ * @table: A #GArrowTable.
+ * @i: The index of the target column.
+ *
+ * Returns: (transfer full): The i-th column in the table.
+ */
+GArrowColumn *
+garrow_table_get_column(GArrowTable *table,
+                        guint i)
+{
+  const auto arrow_table = garrow_table_get_raw(table);
+  auto arrow_column = arrow_table->column(i);
+  return garrow_column_new_raw(&arrow_column);
+}
+
+/**
+ * garrow_table_get_n_columns:
+ * @table: A #GArrowTable.
+ *
+ * Returns: The number of columns in the table.
+ */
+guint
+garrow_table_get_n_columns(GArrowTable *table)
+{
+  const auto arrow_table = garrow_table_get_raw(table);
+  return arrow_table->num_columns();
+}
+
+/**
+ * garrow_table_get_n_rows:
+ * @table: A #GArrowTable.
+ *
+ * Returns: The number of rows in the table.
+ */
+guint64
+garrow_table_get_n_rows(GArrowTable *table)
+{
+  const auto arrow_table = garrow_table_get_raw(table);
+  return arrow_table->num_rows();
+}
+
+G_END_DECLS
+
+GArrowTable *
+garrow_table_new_raw(std::shared_ptr<arrow::Table> *arrow_table)
+{
+  auto table = GARROW_TABLE(g_object_new(GARROW_TYPE_TABLE,
+                                         "table", arrow_table,
+                                         NULL));
+  return table;
+}
+
+std::shared_ptr<arrow::Table>
+garrow_table_get_raw(GArrowTable *table)
+{
+  GArrowTablePrivate *priv;
+
+  priv = GARROW_TABLE_GET_PRIVATE(table);
+  return priv->table;
+}
diff --git a/c_glib/arrow-glib/table.h b/c_glib/arrow-glib/table.h
new file mode 100644
index 0000000000000..34a89a78abcbb
--- /dev/null
+++ b/c_glib/arrow-glib/table.h
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/column.h>
+#include <arrow-glib/schema.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_TABLE                       \
+  (garrow_table_get_type())
+#define GARROW_TABLE(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_TABLE,        \
+                              GArrowTable))
+#define GARROW_TABLE_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),             \
+                           GARROW_TYPE_TABLE,   \
+                           GArrowTableClass))
+#define GARROW_IS_TABLE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_TABLE))
+#define GARROW_IS_TABLE_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),             \
+                           GARROW_TYPE_TABLE))
+#define GARROW_TABLE_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),             \
+                             GARROW_TYPE_TABLE, \
+                             GArrowTableClass))
+
+typedef struct _GArrowTable         GArrowTable;
+typedef struct _GArrowTableClass    GArrowTableClass;
+
+/**
+ * GArrowTable:
+ *
+ * It wraps `arrow::Table`.
+ */
+struct _GArrowTable
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowTableClass
+{
+  GObjectClass parent_class;
+};
+
+GType           garrow_table_get_type      (void) G_GNUC_CONST;
+
+GArrowTable    *garrow_table_new           (const gchar *name,
+                                            GArrowSchema *schema,
+                                            GList *columns);
+
+const gchar    *garrow_table_get_name      (GArrowTable *table);
+GArrowSchema   *garrow_table_get_schema    (GArrowTable *table);
+GArrowColumn   *garrow_table_get_column    (GArrowTable *table,
+                                            guint i);
+guint           garrow_table_get_n_columns (GArrowTable *table);
+guint64         garrow_table_get_n_rows    (GArrowTable *table);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/table.hpp b/c_glib/arrow-glib/table.hpp
new file mode 100644
index 0000000000000..22b0fad502456
--- /dev/null
+++ b/c_glib/arrow-glib/table.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/table.h>
+
+GArrowTable *garrow_table_new_raw(std::shared_ptr<arrow::Table> *arrow_table);
+std::shared_ptr<arrow::Table> garrow_table_get_raw(GArrowTable *table);
diff --git a/c_glib/arrow-glib/type.cpp b/c_glib/arrow-glib/type.cpp
new file mode 100644
index 0000000000000..56cbc212211eb
--- /dev/null
+++ b/c_glib/arrow-glib/type.cpp
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/type.hpp>
+
+/**
+ * SECTION: type
+ * @title: GArrowType
+ * @short_description: Type mapping between Arrow and arrow-glib
+ *
+ * #GArrowType provides types corresponding to `arrow::Type::type`
+ * values.
+ */
+
+GArrowType
+garrow_type_from_raw(arrow::Type::type type)
+{
+  switch (type) {
+  case arrow::Type::type::NA:
+    return GARROW_TYPE_NA;
+  case arrow::Type::type::BOOL:
+    return GARROW_TYPE_BOOL;
+  case arrow::Type::type::UINT8:
+    return GARROW_TYPE_UINT8;
+  case arrow::Type::type::INT8:
+    return GARROW_TYPE_INT8;
+  case arrow::Type::type::UINT16:
+    return GARROW_TYPE_UINT16;
+  case arrow::Type::type::INT16:
+    return GARROW_TYPE_INT16;
+  case arrow::Type::type::UINT32:
+    return GARROW_TYPE_UINT32;
+  case arrow::Type::type::INT32:
+    return GARROW_TYPE_INT32;
+  case arrow::Type::type::UINT64:
+    return GARROW_TYPE_UINT64;
+  case arrow::Type::type::INT64:
+    return GARROW_TYPE_INT64;
+  case arrow::Type::type::HALF_FLOAT:
+    return GARROW_TYPE_HALF_FLOAT;
+  case arrow::Type::type::FLOAT:
+    return GARROW_TYPE_FLOAT;
+  case arrow::Type::type::DOUBLE:
+    return GARROW_TYPE_DOUBLE;
+  case arrow::Type::type::STRING:
+    return GARROW_TYPE_STRING;
+  case arrow::Type::type::BINARY:
+    return GARROW_TYPE_BINARY;
+  case arrow::Type::type::DATE:
+    return GARROW_TYPE_DATE;
+  case arrow::Type::type::TIMESTAMP:
+    return GARROW_TYPE_TIMESTAMP;
+  case arrow::Type::type::TIME:
+    return GARROW_TYPE_TIME;
+  case arrow::Type::type::INTERVAL:
+    return GARROW_TYPE_INTERVAL;
+  case arrow::Type::type::DECIMAL:
+    return GARROW_TYPE_DECIMAL;
+  case arrow::Type::type::LIST:
+    return GARROW_TYPE_LIST;
+  case arrow::Type::type::STRUCT:
+    return GARROW_TYPE_STRUCT;
+  case arrow::Type::type::UNION:
+    return GARROW_TYPE_UNION;
+  case arrow::Type::type::DICTIONARY:
+    return GARROW_TYPE_DICTIONARY;
+  default:
+    return GARROW_TYPE_NA;
+  }
+}
diff --git a/c_glib/arrow-glib/type.h b/c_glib/arrow-glib/type.h
new file mode 100644
index 0000000000000..48d2801dad42c
--- /dev/null
+++ b/c_glib/arrow-glib/type.h
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+/**
+ * GArrowType:
+ * @GARROW_TYPE_NA: A degenerate NULL type represented as 0 bytes/bits.
+ * @GARROW_TYPE_BOOL: A boolean value represented as 1 bit.
+ * @GARROW_TYPE_UINT8: Little-endian 8bit unsigned integer.
+ * @GARROW_TYPE_INT8: Little-endian 8bit signed integer.
+ * @GARROW_TYPE_UINT16: Little-endian 16bit unsigned integer.
+ * @GARROW_TYPE_INT16: Little-endian 16bit signed integer.
+ * @GARROW_TYPE_UINT32: Little-endian 32bit unsigned integer.
+ * @GARROW_TYPE_INT32: Little-endian 32bit signed integer.
+ * @GARROW_TYPE_UINT64: Little-endian 64bit unsigned integer.
+ * @GARROW_TYPE_INT64: Little-endian 64bit signed integer.
+ * @GARROW_TYPE_HALF_FLOAT: 2-byte floating point value.
+ * @GARROW_TYPE_FLOAT: 4-byte floating point value.
+ * @GARROW_TYPE_DOUBLE: 8-byte floating point value.
+ * @GARROW_TYPE_STRING: UTF-8 variable-length string.
+ * @GARROW_TYPE_BINARY: Variable-length bytes (no guarantee of UTF-8-ness).
+ * @GARROW_TYPE_DATE: By default, int32 days since the UNIX epoch.
+ * @GARROW_TYPE_TIMESTAMP: Exact timestamp encoded with int64 since UNIX epoch.
+ *   Default unit millisecond.
+ * @GARROW_TYPE_TIME: Exact time encoded with int64, default unit millisecond.
+ * @GARROW_TYPE_INTERVAL: YEAR_MONTH or DAY_TIME interval in SQL style.
+ * @GARROW_TYPE_DECIMAL: Precision- and scale-based decimal
+ *   type. Storage type depends on the parameters.
+ * @GARROW_TYPE_LIST: A list of some logical data type.
+ * @GARROW_TYPE_STRUCT: Struct of logical types.
+ * @GARROW_TYPE_UNION: Unions of logical types.
+ * @GARROW_TYPE_DICTIONARY: Dictionary aka Category type.
+ *
+ * They are corresponding to `arrow::Type::type` values.
+ */
+typedef enum {
+  GARROW_TYPE_NA,
+  GARROW_TYPE_BOOL,
+  GARROW_TYPE_UINT8,
+  GARROW_TYPE_INT8,
+  GARROW_TYPE_UINT16,
+  GARROW_TYPE_INT16,
+  GARROW_TYPE_UINT32,
+  GARROW_TYPE_INT32,
+  GARROW_TYPE_UINT64,
+  GARROW_TYPE_INT64,
+  GARROW_TYPE_HALF_FLOAT,
+  GARROW_TYPE_FLOAT,
+  GARROW_TYPE_DOUBLE,
+  GARROW_TYPE_STRING,
+  GARROW_TYPE_BINARY,
+  GARROW_TYPE_DATE,
+  GARROW_TYPE_TIMESTAMP,
+  GARROW_TYPE_TIME,
+  GARROW_TYPE_INTERVAL,
+  GARROW_TYPE_DECIMAL,
+  GARROW_TYPE_LIST,
+  GARROW_TYPE_STRUCT,
+  GARROW_TYPE_UNION,
+  GARROW_TYPE_DICTIONARY
+} GArrowType;
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/type.hpp b/c_glib/arrow-glib/type.hpp
new file mode 100644
index 0000000000000..2a452be6dd854
--- /dev/null
+++ b/c_glib/arrow-glib/type.hpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/type.h>
+
+GArrowType garrow_type_from_raw(arrow::Type::type type);
diff --git a/c_glib/arrow-glib/uint16-array-builder.cpp b/c_glib/arrow-glib/uint16-array-builder.cpp
new file mode 100644
index 0000000000000..bfade2de7a84d
--- /dev/null
+++ b/c_glib/arrow-glib/uint16-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/uint16-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint16-array-builder
+ * @short_description: 16-bit unsigned integer array builder class
+ *
+ * #GArrowUInt16ArrayBuilder is the class to create a new
+ * #GArrowUInt16Array.
+ */
+
+G_DEFINE_TYPE(GArrowUInt16ArrayBuilder,
+              garrow_uint16_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint16_array_builder_init(GArrowUInt16ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint16_array_builder_class_init(GArrowUInt16ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint16_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUInt16ArrayBuilder.
+ */
+GArrowUInt16ArrayBuilder *
+garrow_uint16_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::UInt16Builder>(memory_pool, arrow::uint16());
+  auto builder =
+    GARROW_UINT16_ARRAY_BUILDER(g_object_new(GARROW_TYPE_UINT16_ARRAY_BUILDER,
+                                            "array-builder", &arrow_builder,
+                                            NULL));
+  return builder;
+}
+
+/**
+ * garrow_uint16_array_builder_append:
+ * @builder: A #GArrowUInt16ArrayBuilder.
+ * @value: An uint16 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
+                                  guint16 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt16Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint16-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_uint16_array_builder_append_null:
+ * @builder: A #GArrowUInt16ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt16Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint16-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-array-builder.h b/c_glib/arrow-glib/uint16-array-builder.h
new file mode 100644
index 0000000000000..c08966ecc1d91
--- /dev/null
+++ b/c_glib/arrow-glib/uint16-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT16_ARRAY_BUILDER         \
+  (garrow_uint16_array_builder_get_type())
+#define GARROW_UINT16_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT16_ARRAY_BUILDER,  \
+                              GArrowUInt16ArrayBuilder))
+#define GARROW_UINT16_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT16_ARRAY_BUILDER,     \
+                           GArrowUInt16ArrayBuilderClass))
+#define GARROW_IS_UINT16_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT16_ARRAY_BUILDER))
+#define GARROW_IS_UINT16_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT16_ARRAY_BUILDER))
+#define GARROW_UINT16_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT16_ARRAY_BUILDER,   \
+                             GArrowUInt16ArrayBuilderClass))
+
+typedef struct _GArrowUInt16ArrayBuilder         GArrowUInt16ArrayBuilder;
+typedef struct _GArrowUInt16ArrayBuilderClass    GArrowUInt16ArrayBuilderClass;
+
+/**
+ * GArrowUInt16ArrayBuilder:
+ *
+ * It wraps `arrow::UInt16Builder`.
+ */
+struct _GArrowUInt16ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowUInt16ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_uint16_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowUInt16ArrayBuilder *garrow_uint16_array_builder_new(void);
+
+gboolean garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
+                                           guint16 value,
+                                           GError **error);
+gboolean garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
+                                                GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-array.cpp b/c_glib/arrow-glib/uint16-array.cpp
new file mode 100644
index 0000000000000..6c416c6592935
--- /dev/null
+++ b/c_glib/arrow-glib/uint16-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/uint16-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint16-array
+ * @short_description: 16-bit unsigned integer array class
+ *
+ * #GArrowUInt16Array is a class for 16-bit unsigned integer array. It
+ * can store zero or more 16-bit unsigned integer data.
+ *
+ * #GArrowUInt16Array is immutable. You need to use
+ * #GArrowUInt16ArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowUInt16Array,               \
+              garrow_uint16_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_uint16_array_init(GArrowUInt16Array *object)
+{
+}
+
+static void
+garrow_uint16_array_class_init(GArrowUInt16ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_uint16_array_get_value:
+ * @array: A #GArrowUInt16Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+guint16
+garrow_uint16_array_get_value(GArrowUInt16Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::UInt16Array *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-array.h b/c_glib/arrow-glib/uint16-array.h
new file mode 100644
index 0000000000000..44725510062c8
--- /dev/null
+++ b/c_glib/arrow-glib/uint16-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT16_ARRAY                 \
+  (garrow_uint16_array_get_type())
+#define GARROW_UINT16_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT16_ARRAY,  \
+                              GArrowUInt16Array))
+#define GARROW_UINT16_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT16_ARRAY,     \
+                           GArrowUInt16ArrayClass))
+#define GARROW_IS_UINT16_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_UINT16_ARRAY))
+#define GARROW_IS_UINT16_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT16_ARRAY))
+#define GARROW_UINT16_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT16_ARRAY,   \
+                             GArrowUInt16ArrayClass))
+
+typedef struct _GArrowUInt16Array         GArrowUInt16Array;
+typedef struct _GArrowUInt16ArrayClass    GArrowUInt16ArrayClass;
+
+/**
+ * GArrowUInt16Array:
+ *
+ * It wraps `arrow::UInt16Array`.
+ */
+struct _GArrowUInt16Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowUInt16ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_uint16_array_get_type(void) G_GNUC_CONST;
+
+guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array,
+                                    gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-data-type.cpp b/c_glib/arrow-glib/uint16-data-type.cpp
new file mode 100644
index 0000000000000..918b75d61c3eb
--- /dev/null
+++ b/c_glib/arrow-glib/uint16-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/uint16-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint16-data-type
+ * @short_description: 16-bit unsigned integer data type
+ *
+ * #GArrowUInt16DataType is a class for 16-bit unsigned integer data type.
+ */
+
+G_DEFINE_TYPE(GArrowUInt16DataType,                \
+              garrow_uint16_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_uint16_data_type_init(GArrowUInt16DataType *object)
+{
+}
+
+static void
+garrow_uint16_data_type_class_init(GArrowUInt16DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_uint16_data_type_new:
+ *
+ * Returns: The newly created 16-bit unsigned integer data type.
+ */
+GArrowUInt16DataType *
+garrow_uint16_data_type_new(void)
+{
+  auto arrow_data_type = arrow::uint16();
+
+  GArrowUInt16DataType *data_type =
+    GARROW_UINT16_DATA_TYPE(g_object_new(GARROW_TYPE_UINT16_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-data-type.h b/c_glib/arrow-glib/uint16-data-type.h
new file mode 100644
index 0000000000000..b65189d888fcd
--- /dev/null
+++ b/c_glib/arrow-glib/uint16-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT16_DATA_TYPE            \
+  (garrow_uint16_data_type_get_type())
+#define GARROW_UINT16_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT16_DATA_TYPE,     \
+                              GArrowUInt16DataType))
+#define GARROW_UINT16_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT16_DATA_TYPE,        \
+                           GArrowUInt16DataTypeClass))
+#define GARROW_IS_UINT16_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT16_DATA_TYPE))
+#define GARROW_IS_UINT16_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT16_DATA_TYPE))
+#define GARROW_UINT16_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT16_DATA_TYPE,      \
+                             GArrowUInt16DataTypeClass))
+
+typedef struct _GArrowUInt16DataType         GArrowUInt16DataType;
+typedef struct _GArrowUInt16DataTypeClass    GArrowUInt16DataTypeClass;
+
+/**
+ * GArrowUInt16DataType:
+ *
+ * It wraps `arrow::UInt16Type`.
+ */
+struct _GArrowUInt16DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt16DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint16_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt16DataType *garrow_uint16_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-array-builder.cpp b/c_glib/arrow-glib/uint32-array-builder.cpp
new file mode 100644
index 0000000000000..35b1893619fa5
--- /dev/null
+++ b/c_glib/arrow-glib/uint32-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/uint32-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint32-array-builder
+ * @short_description: 32-bit unsigned integer array builder class
+ *
+ * #GArrowUInt32ArrayBuilder is the class to create a new
+ * #GArrowUInt32Array.
+ */
+
+G_DEFINE_TYPE(GArrowUInt32ArrayBuilder,
+              garrow_uint32_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint32_array_builder_init(GArrowUInt32ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint32_array_builder_class_init(GArrowUInt32ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint32_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUInt32ArrayBuilder.
+ */
+GArrowUInt32ArrayBuilder *
+garrow_uint32_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::UInt32Builder>(memory_pool, arrow::uint32());
+  auto builder =
+    GARROW_UINT32_ARRAY_BUILDER(g_object_new(GARROW_TYPE_UINT32_ARRAY_BUILDER,
+                                            "array-builder", &arrow_builder,
+                                            NULL));
+  return builder;
+}
+
+/**
+ * garrow_uint32_array_builder_append:
+ * @builder: A #GArrowUInt32ArrayBuilder.
+ * @value: An uint32 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
+                                  guint32 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt32Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint32-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_uint32_array_builder_append_null:
+ * @builder: A #GArrowUInt32ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt32Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint32-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-array-builder.h b/c_glib/arrow-glib/uint32-array-builder.h
new file mode 100644
index 0000000000000..4881d3b17ff0d
--- /dev/null
+++ b/c_glib/arrow-glib/uint32-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT32_ARRAY_BUILDER         \
+  (garrow_uint32_array_builder_get_type())
+#define GARROW_UINT32_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT32_ARRAY_BUILDER,  \
+                              GArrowUInt32ArrayBuilder))
+#define GARROW_UINT32_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT32_ARRAY_BUILDER,     \
+                           GArrowUInt32ArrayBuilderClass))
+#define GARROW_IS_UINT32_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT32_ARRAY_BUILDER))
+#define GARROW_IS_UINT32_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT32_ARRAY_BUILDER))
+#define GARROW_UINT32_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT32_ARRAY_BUILDER,   \
+                             GArrowUInt32ArrayBuilderClass))
+
+typedef struct _GArrowUInt32ArrayBuilder         GArrowUInt32ArrayBuilder;
+typedef struct _GArrowUInt32ArrayBuilderClass    GArrowUInt32ArrayBuilderClass;
+
+/**
+ * GArrowUInt32ArrayBuilder:
+ *
+ * It wraps `arrow::UInt32Builder`.
+ */
+struct _GArrowUInt32ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowUInt32ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_uint32_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowUInt32ArrayBuilder *garrow_uint32_array_builder_new(void);
+
+gboolean garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
+                                           guint32 value,
+                                           GError **error);
+gboolean garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
+                                                GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-array.cpp b/c_glib/arrow-glib/uint32-array.cpp
new file mode 100644
index 0000000000000..d10f10005f9be
--- /dev/null
+++ b/c_glib/arrow-glib/uint32-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/uint32-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint32-array
+ * @short_description: 32-bit unsigned integer array class
+ *
+ * #GArrowUInt32Array is a class for 32-bit unsigned integer array. It
+ * can store zero or more 32-bit unsigned integer data.
+ *
+ * #GArrowUInt32Array is immutable. You need to use
+ * #GArrowUInt32ArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowUInt32Array,               \
+              garrow_uint32_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_uint32_array_init(GArrowUInt32Array *object)
+{
+}
+
+static void
+garrow_uint32_array_class_init(GArrowUInt32ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_uint32_array_get_value:
+ * @array: A #GArrowUInt32Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+guint32
+garrow_uint32_array_get_value(GArrowUInt32Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::UInt32Array *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-array.h b/c_glib/arrow-glib/uint32-array.h
new file mode 100644
index 0000000000000..57d4beaee6186
--- /dev/null
+++ b/c_glib/arrow-glib/uint32-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT32_ARRAY                 \
+  (garrow_uint32_array_get_type())
+#define GARROW_UINT32_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT32_ARRAY,  \
+                              GArrowUInt32Array))
+#define GARROW_UINT32_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT32_ARRAY,     \
+                           GArrowUInt32ArrayClass))
+#define GARROW_IS_UINT32_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_UINT32_ARRAY))
+#define GARROW_IS_UINT32_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT32_ARRAY))
+#define GARROW_UINT32_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT32_ARRAY,   \
+                             GArrowUInt32ArrayClass))
+
+typedef struct _GArrowUInt32Array         GArrowUInt32Array;
+typedef struct _GArrowUInt32ArrayClass    GArrowUInt32ArrayClass;
+
+/**
+ * GArrowUInt32Array:
+ *
+ * It wraps `arrow::UInt32Array`.
+ */
+struct _GArrowUInt32Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowUInt32ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_uint32_array_get_type(void) G_GNUC_CONST;
+
+guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array,
+                                    gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-data-type.cpp b/c_glib/arrow-glib/uint32-data-type.cpp
new file mode 100644
index 0000000000000..fde14f3274174
--- /dev/null
+++ b/c_glib/arrow-glib/uint32-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/uint32-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint32-data-type
+ * @short_description: 32-bit unsigned integer data type
+ *
+ * #GArrowUInt32DataType is a class for 32-bit unsigned integer data type.
+ */
+
+G_DEFINE_TYPE(GArrowUInt32DataType,                \
+              garrow_uint32_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_uint32_data_type_init(GArrowUInt32DataType *object)
+{
+}
+
+static void
+garrow_uint32_data_type_class_init(GArrowUInt32DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_uint32_data_type_new:
+ *
+ * Returns: The newly created 32-bit unsigned integer data type.
+ */
+GArrowUInt32DataType *
+garrow_uint32_data_type_new(void)
+{
+  auto arrow_data_type = arrow::uint32();
+
+  GArrowUInt32DataType *data_type =
+    GARROW_UINT32_DATA_TYPE(g_object_new(GARROW_TYPE_UINT32_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-data-type.h b/c_glib/arrow-glib/uint32-data-type.h
new file mode 100644
index 0000000000000..4fe60cd850ba8
--- /dev/null
+++ b/c_glib/arrow-glib/uint32-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT32_DATA_TYPE            \
+  (garrow_uint32_data_type_get_type())
+#define GARROW_UINT32_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT32_DATA_TYPE,     \
+                              GArrowUInt32DataType))
+#define GARROW_UINT32_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT32_DATA_TYPE,        \
+                           GArrowUInt32DataTypeClass))
+#define GARROW_IS_UINT32_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT32_DATA_TYPE))
+#define GARROW_IS_UINT32_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT32_DATA_TYPE))
+#define GARROW_UINT32_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT32_DATA_TYPE,      \
+                             GArrowUInt32DataTypeClass))
+
+typedef struct _GArrowUInt32DataType         GArrowUInt32DataType;
+typedef struct _GArrowUInt32DataTypeClass    GArrowUInt32DataTypeClass;
+
+/**
+ * GArrowUInt32DataType:
+ *
+ * It wraps `arrow::UInt32Type`.
+ */
+struct _GArrowUInt32DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt32DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint32_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt32DataType *garrow_uint32_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-array-builder.cpp b/c_glib/arrow-glib/uint64-array-builder.cpp
new file mode 100644
index 0000000000000..85d24ca54ab8b
--- /dev/null
+++ b/c_glib/arrow-glib/uint64-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/uint64-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint64-array-builder
+ * @short_description: 64-bit unsigned integer array builder class
+ *
+ * #GArrowUInt64ArrayBuilder is the class to create a new
+ * #GArrowUInt64Array.
+ */
+
+G_DEFINE_TYPE(GArrowUInt64ArrayBuilder,
+              garrow_uint64_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint64_array_builder_init(GArrowUInt64ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint64_array_builder_class_init(GArrowUInt64ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint64_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUInt64ArrayBuilder.
+ */
+GArrowUInt64ArrayBuilder *
+garrow_uint64_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::UInt64Builder>(memory_pool, arrow::uint64());
+  auto builder =
+    GARROW_UINT64_ARRAY_BUILDER(g_object_new(GARROW_TYPE_UINT64_ARRAY_BUILDER,
+                                            "array-builder", &arrow_builder,
+                                            NULL));
+  return builder;
+}
+
+/**
+ * garrow_uint64_array_builder_append:
+ * @builder: A #GArrowUInt64ArrayBuilder.
+ * @value: An uint64 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
+                                  guint64 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt64Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint64-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_uint64_array_builder_append_null:
+ * @builder: A #GArrowUInt64ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt64Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint64-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-array-builder.h b/c_glib/arrow-glib/uint64-array-builder.h
new file mode 100644
index 0000000000000..c51d1e2485d6f
--- /dev/null
+++ b/c_glib/arrow-glib/uint64-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT64_ARRAY_BUILDER         \
+  (garrow_uint64_array_builder_get_type())
+#define GARROW_UINT64_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT64_ARRAY_BUILDER,  \
+                              GArrowUInt64ArrayBuilder))
+#define GARROW_UINT64_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT64_ARRAY_BUILDER,     \
+                           GArrowUInt64ArrayBuilderClass))
+#define GARROW_IS_UINT64_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT64_ARRAY_BUILDER))
+#define GARROW_IS_UINT64_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT64_ARRAY_BUILDER))
+#define GARROW_UINT64_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT64_ARRAY_BUILDER,   \
+                             GArrowUInt64ArrayBuilderClass))
+
+typedef struct _GArrowUInt64ArrayBuilder         GArrowUInt64ArrayBuilder;
+typedef struct _GArrowUInt64ArrayBuilderClass    GArrowUInt64ArrayBuilderClass;
+
+/**
+ * GArrowUInt64ArrayBuilder:
+ *
+ * It wraps `arrow::UInt64Builder`.
+ */
+struct _GArrowUInt64ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowUInt64ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_uint64_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowUInt64ArrayBuilder *garrow_uint64_array_builder_new(void);
+
+gboolean garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
+                                           guint64 value,
+                                           GError **error);
+gboolean garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
+                                                GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-array.cpp b/c_glib/arrow-glib/uint64-array.cpp
new file mode 100644
index 0000000000000..1f900842674b8
--- /dev/null
+++ b/c_glib/arrow-glib/uint64-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/uint64-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint64-array
+ * @short_description: 64-bit unsigned integer array class
+ *
+ * #GArrowUInt64Array is a class for 64-bit unsigned integer array. It
+ * can store zero or more 64-bit unsigned integer data.
+ *
+ * #GArrowUInt64Array is immutable. You need to use
+ * #GArrowUInt64ArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowUInt64Array,               \
+              garrow_uint64_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_uint64_array_init(GArrowUInt64Array *object)
+{
+}
+
+static void
+garrow_uint64_array_class_init(GArrowUInt64ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_uint64_array_get_value:
+ * @array: A #GArrowUInt64Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+guint64
+garrow_uint64_array_get_value(GArrowUInt64Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::UInt64Array *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-array.h b/c_glib/arrow-glib/uint64-array.h
new file mode 100644
index 0000000000000..b5abde52bd263
--- /dev/null
+++ b/c_glib/arrow-glib/uint64-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT64_ARRAY                 \
+  (garrow_uint64_array_get_type())
+#define GARROW_UINT64_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT64_ARRAY,  \
+                              GArrowUInt64Array))
+#define GARROW_UINT64_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT64_ARRAY,     \
+                           GArrowUInt64ArrayClass))
+#define GARROW_IS_UINT64_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_UINT64_ARRAY))
+#define GARROW_IS_UINT64_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT64_ARRAY))
+#define GARROW_UINT64_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT64_ARRAY,   \
+                             GArrowUInt64ArrayClass))
+
+typedef struct _GArrowUInt64Array         GArrowUInt64Array;
+typedef struct _GArrowUInt64ArrayClass    GArrowUInt64ArrayClass;
+
+/**
+ * GArrowUInt64Array:
+ *
+ * It wraps `arrow::UInt64Array`.
+ */
+struct _GArrowUInt64Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowUInt64ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_uint64_array_get_type(void) G_GNUC_CONST;
+
+guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array,
+                                    gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-data-type.cpp b/c_glib/arrow-glib/uint64-data-type.cpp
new file mode 100644
index 0000000000000..7c18b36a01b3b
--- /dev/null
+++ b/c_glib/arrow-glib/uint64-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/uint64-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint64-data-type
+ * @short_description: 64-bit unsigned integer data type
+ *
+ * #GArrowUInt64DataType is a class for 64-bit unsigned integer data type.
+ */
+
+G_DEFINE_TYPE(GArrowUInt64DataType,                \
+              garrow_uint64_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_uint64_data_type_init(GArrowUInt64DataType *object)
+{
+}
+
+static void
+garrow_uint64_data_type_class_init(GArrowUInt64DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_uint64_data_type_new:
+ *
+ * Returns: The newly created 64-bit unsigned integer data type.
+ */
+GArrowUInt64DataType *
+garrow_uint64_data_type_new(void)
+{
+  auto arrow_data_type = arrow::uint64();
+
+  GArrowUInt64DataType *data_type =
+    GARROW_UINT64_DATA_TYPE(g_object_new(GARROW_TYPE_UINT64_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-data-type.h b/c_glib/arrow-glib/uint64-data-type.h
new file mode 100644
index 0000000000000..221023c863818
--- /dev/null
+++ b/c_glib/arrow-glib/uint64-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT64_DATA_TYPE            \
+  (garrow_uint64_data_type_get_type())
+#define GARROW_UINT64_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT64_DATA_TYPE,     \
+                              GArrowUInt64DataType))
+#define GARROW_UINT64_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT64_DATA_TYPE,        \
+                           GArrowUInt64DataTypeClass))
+#define GARROW_IS_UINT64_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT64_DATA_TYPE))
+#define GARROW_IS_UINT64_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT64_DATA_TYPE))
+#define GARROW_UINT64_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT64_DATA_TYPE,      \
+                             GArrowUInt64DataTypeClass))
+
+typedef struct _GArrowUInt64DataType         GArrowUInt64DataType;
+typedef struct _GArrowUInt64DataTypeClass    GArrowUInt64DataTypeClass;
+
+/**
+ * GArrowUInt64DataType:
+ *
+ * It wraps `arrow::UInt64Type`.
+ */
+struct _GArrowUInt64DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt64DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint64_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt64DataType *garrow_uint64_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-array-builder.cpp b/c_glib/arrow-glib/uint8-array-builder.cpp
new file mode 100644
index 0000000000000..2f49693236b24
--- /dev/null
+++ b/c_glib/arrow-glib/uint8-array-builder.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/uint8-array-builder.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint8-array-builder
+ * @short_description: 8-bit unsigned integer array builder class
+ *
+ * #GArrowUInt8ArrayBuilder is the class to create a new
+ * #GArrowUInt8Array.
+ */
+
+G_DEFINE_TYPE(GArrowUInt8ArrayBuilder,
+              garrow_uint8_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint8_array_builder_init(GArrowUInt8ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint8_array_builder_class_init(GArrowUInt8ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint8_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUInt8ArrayBuilder.
+ */
+GArrowUInt8ArrayBuilder *
+garrow_uint8_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder =
+    std::make_shared<arrow::UInt8Builder>(memory_pool, arrow::uint8());
+  auto builder =
+    GARROW_UINT8_ARRAY_BUILDER(g_object_new(GARROW_TYPE_UINT8_ARRAY_BUILDER,
+                                            "array-builder", &arrow_builder,
+                                            NULL));
+  return builder;
+}
+
+/**
+ * garrow_uint8_array_builder_append:
+ * @builder: A #GArrowUInt8ArrayBuilder.
+ * @value: An uint8 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
+                                  guint8 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt8Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint8-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_uint8_array_builder_append_null:
+ * @builder: A #GArrowUInt8ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt8Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint8-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-array-builder.h b/c_glib/arrow-glib/uint8-array-builder.h
new file mode 100644
index 0000000000000..e7216931a511c
--- /dev/null
+++ b/c_glib/arrow-glib/uint8-array-builder.h
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT8_ARRAY_BUILDER         \
+  (garrow_uint8_array_builder_get_type())
+#define GARROW_UINT8_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT8_ARRAY_BUILDER,  \
+                              GArrowUInt8ArrayBuilder))
+#define GARROW_UINT8_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT8_ARRAY_BUILDER,     \
+                           GArrowUInt8ArrayBuilderClass))
+#define GARROW_IS_UINT8_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT8_ARRAY_BUILDER))
+#define GARROW_IS_UINT8_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT8_ARRAY_BUILDER))
+#define GARROW_UINT8_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT8_ARRAY_BUILDER,   \
+                             GArrowUInt8ArrayBuilderClass))
+
+typedef struct _GArrowUInt8ArrayBuilder         GArrowUInt8ArrayBuilder;
+typedef struct _GArrowUInt8ArrayBuilderClass    GArrowUInt8ArrayBuilderClass;
+
+/**
+ * GArrowUInt8ArrayBuilder:
+ *
+ * It wraps `arrow::UInt8Builder`.
+ */
+struct _GArrowUInt8ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowUInt8ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_uint8_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowUInt8ArrayBuilder *garrow_uint8_array_builder_new(void);
+
+gboolean garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
+                                           guint8 value,
+                                           GError **error);
+gboolean garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
+                                                GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-array.cpp b/c_glib/arrow-glib/uint8-array.cpp
new file mode 100644
index 0000000000000..b5a2595b1ef09
--- /dev/null
+++ b/c_glib/arrow-glib/uint8-array.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/uint8-array.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint8-array
+ * @short_description: 8-bit unsigned integer array class
+ *
+ * #GArrowUInt8Array is a class for 8-bit unsigned integer array. It
+ * can store zero or more 8-bit unsigned integer data.
+ *
+ * #GArrowUInt8Array is immutable. You need to use
+ * #GArrowUInt8ArrayBuilder to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowUInt8Array,               \
+              garrow_uint8_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_uint8_array_init(GArrowUInt8Array *object)
+{
+}
+
+static void
+garrow_uint8_array_class_init(GArrowUInt8ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_uint8_array_get_value:
+ * @array: A #GArrowUInt8Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+guint8
+garrow_uint8_array_get_value(GArrowUInt8Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::UInt8Array *>(arrow_array.get())->Value(i);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-array.h b/c_glib/arrow-glib/uint8-array.h
new file mode 100644
index 0000000000000..a572bc549670e
--- /dev/null
+++ b/c_glib/arrow-glib/uint8-array.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT8_ARRAY                 \
+  (garrow_uint8_array_get_type())
+#define GARROW_UINT8_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT8_ARRAY,  \
+                              GArrowUInt8Array))
+#define GARROW_UINT8_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT8_ARRAY,     \
+                           GArrowUInt8ArrayClass))
+#define GARROW_IS_UINT8_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_UINT8_ARRAY))
+#define GARROW_IS_UINT8_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT8_ARRAY))
+#define GARROW_UINT8_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT8_ARRAY,   \
+                             GArrowUInt8ArrayClass))
+
+typedef struct _GArrowUInt8Array         GArrowUInt8Array;
+typedef struct _GArrowUInt8ArrayClass    GArrowUInt8ArrayClass;
+
+/**
+ * GArrowUInt8Array:
+ *
+ * It wraps `arrow::UInt8Array`.
+ */
+struct _GArrowUInt8Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowUInt8ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_uint8_array_get_type(void) G_GNUC_CONST;
+
+guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array,
+                                    gint64 i);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-data-type.cpp b/c_glib/arrow-glib/uint8-data-type.cpp
new file mode 100644
index 0000000000000..7c93e455a4e96
--- /dev/null
+++ b/c_glib/arrow-glib/uint8-data-type.cpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/uint8-data-type.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint8-data-type
+ * @short_description: 8-bit unsigned integer data type
+ *
+ * #GArrowUInt8DataType is a class for 8-bit unsigned integer data type.
+ */
+
+G_DEFINE_TYPE(GArrowUInt8DataType,                \
+              garrow_uint8_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_uint8_data_type_init(GArrowUInt8DataType *object)
+{
+}
+
+static void
+garrow_uint8_data_type_class_init(GArrowUInt8DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_uint8_data_type_new:
+ *
+ * Returns: The newly created 8-bit unsigned integer data type.
+ */
+GArrowUInt8DataType *
+garrow_uint8_data_type_new(void)
+{
+  auto arrow_data_type = arrow::uint8();
+
+  GArrowUInt8DataType *data_type =
+    GARROW_UINT8_DATA_TYPE(g_object_new(GARROW_TYPE_UINT8_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-data-type.h b/c_glib/arrow-glib/uint8-data-type.h
new file mode 100644
index 0000000000000..6e058524f4b10
--- /dev/null
+++ b/c_glib/arrow-glib/uint8-data-type.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT8_DATA_TYPE            \
+  (garrow_uint8_data_type_get_type())
+#define GARROW_UINT8_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT8_DATA_TYPE,     \
+                              GArrowUInt8DataType))
+#define GARROW_UINT8_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT8_DATA_TYPE,        \
+                           GArrowUInt8DataTypeClass))
+#define GARROW_IS_UINT8_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT8_DATA_TYPE))
+#define GARROW_IS_UINT8_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT8_DATA_TYPE))
+#define GARROW_UINT8_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT8_DATA_TYPE,      \
+                             GArrowUInt8DataTypeClass))
+
+typedef struct _GArrowUInt8DataType         GArrowUInt8DataType;
+typedef struct _GArrowUInt8DataTypeClass    GArrowUInt8DataTypeClass;
+
+/**
+ * GArrowUInt8DataType:
+ *
+ * It wraps `arrow::UInt8Type`.
+ */
+struct _GArrowUInt8DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt8DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint8_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt8DataType *garrow_uint8_data_type_new      (void);
+
+G_END_DECLS
diff --git a/c_glib/autogen.sh b/c_glib/autogen.sh
new file mode 100755
index 0000000000000..08e33e6ca07c0
--- /dev/null
+++ b/c_glib/autogen.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -u
+set -e
+
+ruby \
+    -e 'print ARGF.read.scan(/^  <version>(.+?)<\/version>/)[0][0]' \
+    ../java/pom.xml > \
+    version
+
+mkdir -p m4
+
+gtkdocize --copy --docdir doc/reference
+autoreconf --install
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
new file mode 100644
index 0000000000000..85f7eec3cb557
--- /dev/null
+++ b/c_glib/configure.ac
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+AC_PREREQ(2.65)
+
+m4_define([arrow_glib_version], m4_include(version))
+AC_INIT([arrow-glib], arrow_glib_version, [kou@clear-code.com])
+AC_CONFIG_AUX_DIR([config])
+AC_CONFIG_MACRO_DIR([m4])
+
+AC_CONFIG_SRCDIR([arrow-glib/arrow-glib.h])
+AC_CONFIG_HEADERS([config.h])
+
+AM_INIT_AUTOMAKE([1.13 foreign])
+AM_SILENT_RULES([yes])
+
+AC_PROG_CC
+AC_PROG_CXX
+AX_CXX_COMPILE_STDCXX_11([ext], [mandatory])
+LT_INIT
+
+GARROW_CFLAGS="-Wall -Wconversion"
+GARROW_CXXFLAGS="-Wall -Wconversion"
+AC_ARG_ENABLE(debug,
+  [AS_HELP_STRING([--enable-debug],
+                  [Use debug flags (default=no)])],
+  [GARROW_DEBUG="$enableval"],
+  [GARROW_DEBUG="no"])
+if test "x$GARROW_DEBUG" != "xno"; then
+  GARROW_DEBUG="yes"
+  if test "$CLANG" = "yes"; then
+    CFLAGS="$CFLAGS -O0 -g"
+    CXXFLAGS="$CXXFLAGS -O0 -g"
+  elif test "$GCC" = "yes"; then
+    CFLAGS="$CFLAGS -O0 -g3"
+    CXXFLAGS="$CXXFLAGS -O0 -g3"
+  fi
+fi
+AC_SUBST(GARROW_CFLAGS)
+AC_SUBST(GARROW_CXXFLAGS)
+
+AM_PATH_GLIB_2_0([2.32.4], [], [], [gobject])
+
+GOBJECT_INTROSPECTION_REQUIRE([1.32.1])
+GTK_DOC_CHECK([1.18-2])
+
+PKG_CHECK_MODULES([ARROW], [arrow])
+PKG_CHECK_MODULES([ARROW_IO], [arrow-io])
+PKG_CHECK_MODULES([ARROW_IPC], [arrow-ipc])
+
+AC_CONFIG_FILES([
+  Makefile
+  arrow-glib/Makefile
+  arrow-glib/arrow-glib.pc
+  arrow-glib/arrow-io-glib.pc
+  arrow-glib/arrow-ipc-glib.pc
+  doc/Makefile
+  doc/reference/Makefile
+  example/Makefile
+])
+
+AC_OUTPUT
diff --git a/c_glib/doc/Makefile.am b/c_glib/doc/Makefile.am
new file mode 100644
index 0000000000000..85c1d5126097c
--- /dev/null
+++ b/c_glib/doc/Makefile.am
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+SUBDIRS =                                       \
+        reference
diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am
new file mode 100644
index 0000000000000..d1c8e01c299a0
--- /dev/null
+++ b/c_glib/doc/reference/Makefile.am
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DOC_MODULE = arrow-glib
+
+DOC_MAIN_SGML_FILE = $(DOC_MODULE)-docs.sgml
+
+DOC_SOURCE_DIR =				\
+	$(top_srcdir)/arrow-glib
+
+SCAN_OPTIONS =						\
+	--deprecated-guards="GARROW_DISABLE_DEPRECATED"
+
+MKDB_OPTIONS =					\
+	--name-space=arrow			\
+	--source-suffixes="c,cpp,h"
+
+HFILE_GLOB =					\
+	$(top_srcdir)/arrow-glib/*.h
+
+IGNORE_HFILES =					\
+	enums.h					\
+	io-enums.h				\
+	ipc-enums.h
+
+CFILE_GLOB =					\
+	$(top_srcdir)/arrow-glib/*.cpp
+
+AM_CPPFLAGS =					\
+	-I$(top_builddir)			\
+	-I$(top_srcdir)
+
+AM_CFLAGS =					\
+	$(GLIB_CFLAGS)				\
+	$(ARROW_CFLAGS)
+
+GTKDOC_LIBS =						\
+	$(top_builddir)/arrow-glib/libarrow-glib.la	\
+	$(top_builddir)/arrow-glib/libarrow-io-glib.la	\
+	$(top_builddir)/arrow-glib/libarrow-ipc-glib.la
+
+include $(srcdir)/gtk-doc.make
+
+CLEANFILES +=					\
+	$(DOC_MODULE)-decl-list.txt		\
+	$(DOC_MODULE)-decl.txt			\
+	$(DOC_MODULE)-overrides.txt		\
+	$(DOC_MODULE)-sections.txt		\
+	$(DOC_MODULE).types
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
new file mode 100644
index 0000000000000..9f504bec7ad53
--- /dev/null
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -0,0 +1,171 @@
+<?xml version="1.0"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
+               "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd"
+[
+  <!ENTITY % local.common.attrib "xmlns:xi  CDATA  #FIXED 'http://www.w3.org/2003/XInclude'">
+  <!ENTITY % gtkdocentities SYSTEM "xml/gtkdocentities.ent">
+  %gtkdocentities;
+]>
+<book id="index">
+  <bookinfo>
+    <title>&package_name; Reference Manual</title>
+    <releaseinfo>
+      for &package_string;.
+      <!--
+      The latest version of this documentation can be found on-line at
+      <ulink role="online-location" url="http://[SERVER]/&package_name;/">http://[SERVER]/&package_name;/</ulink>.
+      -->
+    </releaseinfo>
+  </bookinfo>
+
+  <part id="arrow">
+    <title>GArrow</title>
+    <chapter id="array">
+      <title>Array</title>
+      <xi:include href="xml/array.xml"/>
+      <xi:include href="xml/null-array.xml"/>
+      <xi:include href="xml/boolean-array.xml"/>
+      <xi:include href="xml/int8-array.xml"/>
+      <xi:include href="xml/uint8-array.xml"/>
+      <xi:include href="xml/int16-array.xml"/>
+      <xi:include href="xml/uint16-array.xml"/>
+      <xi:include href="xml/int32-array.xml"/>
+      <xi:include href="xml/uint32-array.xml"/>
+      <xi:include href="xml/int64-array.xml"/>
+      <xi:include href="xml/uint64-array.xml"/>
+      <xi:include href="xml/float-array.xml"/>
+      <xi:include href="xml/double-array.xml"/>
+      <xi:include href="xml/binary-array.xml"/>
+      <xi:include href="xml/string-array.xml"/>
+      <xi:include href="xml/list-array.xml"/>
+      <xi:include href="xml/struct-array.xml"/>
+    </chapter>
+    <chapter id="array-builder">
+      <title>Array builder</title>
+      <xi:include href="xml/array-builder.xml"/>
+      <xi:include href="xml/boolean-array-builder.xml"/>
+      <xi:include href="xml/int8-array-builder.xml"/>
+      <xi:include href="xml/uint8-array-builder.xml"/>
+      <xi:include href="xml/int16-array-builder.xml"/>
+      <xi:include href="xml/uint16-array-builder.xml"/>
+      <xi:include href="xml/int32-array-builder.xml"/>
+      <xi:include href="xml/uint32-array-builder.xml"/>
+      <xi:include href="xml/int64-array-builder.xml"/>
+      <xi:include href="xml/uint64-array-builder.xml"/>
+      <xi:include href="xml/float-array-builder.xml"/>
+      <xi:include href="xml/double-array-builder.xml"/>
+      <xi:include href="xml/binary-array-builder.xml"/>
+      <xi:include href="xml/string-array-builder.xml"/>
+      <xi:include href="xml/list-array-builder.xml"/>
+      <xi:include href="xml/struct-array-builder.xml"/>
+    </chapter>
+    <chapter id="type">
+      <title>Type</title>
+      <xi:include href="xml/type.xml"/>
+      <xi:include href="xml/data-type.xml"/>
+      <xi:include href="xml/null-data-type.xml"/>
+      <xi:include href="xml/boolean-data-type.xml"/>
+      <xi:include href="xml/int8-data-type.xml"/>
+      <xi:include href="xml/uint8-data-type.xml"/>
+      <xi:include href="xml/int16-data-type.xml"/>
+      <xi:include href="xml/uint16-data-type.xml"/>
+      <xi:include href="xml/int32-data-type.xml"/>
+      <xi:include href="xml/uint32-data-type.xml"/>
+      <xi:include href="xml/int64-data-type.xml"/>
+      <xi:include href="xml/uint64-data-type.xml"/>
+      <xi:include href="xml/float-data-type.xml"/>
+      <xi:include href="xml/double-data-type.xml"/>
+      <xi:include href="xml/binary-data-type.xml"/>
+      <xi:include href="xml/string-data-type.xml"/>
+      <xi:include href="xml/list-data-type.xml"/>
+      <xi:include href="xml/struct-data-type.xml"/>
+    </chapter>
+    <chapter id="schema">
+      <title>Schema</title>
+      <xi:include href="xml/field.xml"/>
+      <xi:include href="xml/schema.xml"/>
+    </chapter>
+    <chapter id="table">
+      <title>Table</title>
+      <xi:include href="xml/table.xml"/>
+      <xi:include href="xml/record-batch.xml"/>
+      <xi:include href="xml/column.xml"/>
+      <xi:include href="xml/chunked-array.xml"/>
+    </chapter>
+    <chapter id="error">
+      <title>Error</title>
+      <xi:include href="xml/error.xml"/>
+    </chapter>
+  </part>
+
+  <part id="arrow-io">
+    <title>GArrowIO</title>
+    <chapter id="io-mode">
+      <title>Enums</title>
+      <xi:include href="xml/io-file-mode.xml"/>
+    </chapter>
+    <chapter id="io-input">
+      <title>Input</title>
+      <xi:include href="xml/io-readable.xml"/>
+      <xi:include href="xml/io-input-stream.xml"/>
+      <xi:include href="xml/io-readable-file.xml"/>
+    </chapter>
+    <chapter id="io-output">
+      <title>Output</title>
+      <xi:include href="xml/io-writeable.xml"/>
+      <xi:include href="xml/io-output-stream.xml"/>
+      <xi:include href="xml/io-writeable-file.xml"/>
+      <xi:include href="xml/io-file-output-stream.xml"/>
+    </chapter>
+    <chapter id="io-input-output">
+      <title>Input and output</title>
+      <xi:include href="xml/io-file.xml"/>
+      <xi:include href="xml/io-memory-mapped-file.xml"/>
+    </chapter>
+  </part>
+
+  <part id="arrow-ipc">
+    <title>GArrowIPC</title>
+    <chapter id="ipc-metadata">
+      <title>Enums</title>
+      <xi:include href="xml/ipc-metadata-version.xml"/>
+    </chapter>
+    <chapter id="ipc-reader">
+      <title>Reader</title>
+      <xi:include href="xml/ipc-file-reader.xml"/>
+      <xi:include href="xml/ipc-stream-reader.xml"/>
+    </chapter>
+    <chapter id="ipc-writer">
+      <title>Input</title>
+      <xi:include href="xml/ipc-file-writer.xml"/>
+      <xi:include href="xml/ipc-stream-writer.xml"/>
+    </chapter>
+  </part>
+
+  <chapter id="object-tree">
+    <title>Object Hierarchy</title>
+    <xi:include href="xml/tree_index.sgml"/>
+  </chapter>
+  <index id="api-index-full">
+    <title>API Index</title>
+    <xi:include href="xml/api-index-full.xml"><xi:fallback /></xi:include>
+  </index>
+  <index id="deprecated-api-index" role="deprecated">
+    <title>Index of deprecated API</title>
+    <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
+  </index>
+  <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
+</book>
diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am
new file mode 100644
index 0000000000000..3d456d7844231
--- /dev/null
+++ b/c_glib/example/Makefile.am
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+AM_CPPFLAGS =					\
+	-I$(top_builddir)			\
+	-I$(top_srcdir)
+
+AM_CFLAGS =					\
+	$(GLIB_CFLAGS)				\
+	$(GARROW_CFLAGS)
+
+AM_LDFLAGS =						\
+	$(GLIB_LIBS)					\
+	$(builddir)/../arrow-glib/libarrow-glib.la
+
+noinst_PROGRAMS =				\
+	build
+
+build_SOURCES =					\
+	build.c
diff --git a/c_glib/example/build.c b/c_glib/example/build.c
new file mode 100644
index 0000000000000..2722458acd5c4
--- /dev/null
+++ b/c_glib/example/build.c
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <stdlib.h>
+
+#include <arrow-glib/arrow-glib.h>
+
+int
+main(int argc, char **argv)
+{
+  GArrowArray *array;
+
+  {
+    GArrowInt32ArrayBuilder *builder;
+    gboolean success = TRUE;
+    GError *error = NULL;
+
+    builder = garrow_int32_array_builder_new();
+    if (success) {
+      success = garrow_int32_array_builder_append(builder, 29, &error);
+    }
+    if (success) {
+      success = garrow_int32_array_builder_append(builder, 2929, &error);
+    }
+    if (success) {
+      success = garrow_int32_array_builder_append(builder, 292929, &error);
+    }
+    if (!success) {
+      g_print("failed to append: %s\n", error->message);
+      g_error_free(error);
+      g_object_unref(builder);
+      return EXIT_FAILURE;
+    }
+    array = garrow_array_builder_finish(GARROW_ARRAY_BUILDER(builder));
+    g_object_unref(builder);
+  }
+
+  {
+    gint64 i, n;
+
+    n = garrow_array_get_length(array);
+    g_print("length: %" G_GINT64_FORMAT "\n", n);
+    for (i = 0; i < n; i++) {
+      gint32 value;
+
+      value = garrow_int32_array_get_value(GARROW_INT32_ARRAY(array), i);
+      g_print("array[%" G_GINT64_FORMAT "] = %d\n",
+              i, value);
+    }
+  }
+
+  g_object_unref(array);
+
+  return EXIT_SUCCESS;
+}
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
new file mode 100644
index 0000000000000..900e180675b45
--- /dev/null
+++ b/c_glib/test/helper/buildable.rb
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module Buildable
+    def build_boolean_array(values)
+      build_array(Arrow::BooleanArrayBuilder, values)
+    end
+
+    def build_int8_array(values)
+      build_array(Arrow::Int8ArrayBuilder, values)
+    end
+
+    def build_uint8_array(values)
+      build_array(Arrow::UInt8ArrayBuilder, values)
+    end
+
+    def build_int16_array(values)
+      build_array(Arrow::Int16ArrayBuilder, values)
+    end
+
+    def build_uint16_array(values)
+      build_array(Arrow::UInt16ArrayBuilder, values)
+    end
+
+    def build_int32_array(values)
+      build_array(Arrow::Int32ArrayBuilder, values)
+    end
+
+    def build_uint32_array(values)
+      build_array(Arrow::UInt32ArrayBuilder, values)
+    end
+
+    def build_int64_array(values)
+      build_array(Arrow::Int64ArrayBuilder, values)
+    end
+
+    def build_uint64_array(values)
+      build_array(Arrow::UInt64ArrayBuilder, values)
+    end
+
+    def build_float_array(values)
+      build_array(Arrow::FloatArrayBuilder, values)
+    end
+
+    def build_double_array(values)
+      build_array(Arrow::DoubleArrayBuilder, values)
+    end
+
+    private
+    def build_array(builder_class, values)
+      builder = builder_class.new
+      values.each do |value|
+        if value.nil?
+          builder.append_null
+        else
+          builder.append(value)
+        end
+      end
+      builder.finish
+    end
+  end
+end
diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb
new file mode 100755
index 0000000000000..32ceb4ad61d2e
--- /dev/null
+++ b/c_glib/test/run-test.rb
@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "pathname"
+require "test-unit"
+
+base_dir = Pathname(__dir__).parent
+typelib_dir = base_dir + "arrow-glib"
+test_dir = base_dir + "test"
+
+ENV["GI_TYPELIB_PATH"] = [
+  typelib_dir.to_s,
+  ENV["GI_TYPELIB_PATH"],
+].compact.join(File::PATH_SEPARATOR)
+
+require "gi"
+
+Arrow = GI.load("Arrow")
+ArrowIO = GI.load("ArrowIO")
+ArrowIPC = GI.load("ArrowIPC")
+
+require "tempfile"
+require_relative "helper/buildable"
+
+exit(Test::Unit::AutoRunner.run(true, test_dir.to_s))
diff --git a/c_glib/test/run-test.sh b/c_glib/test/run-test.sh
new file mode 100755
index 0000000000000..9b0ec8e45f52f
--- /dev/null
+++ b/c_glib/test/run-test.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+base_dir="$(cd .; pwd)"
+lib_dir="${base_dir}/arrow-glib/.libs"
+
+LD_LIBRARY_PATH="${lib_dir}:${LD_LIBRARY_PATH}"
+
+if [ "${NO_MAKE}" != "yes" ]; then
+  make -j8 > /dev/null || exit $?
+fi
+
+${GDB} ruby ${base_dir}/test/run-test.rb "$@"
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
new file mode 100644
index 0000000000000..d68827cb85b1d
--- /dev/null
+++ b/c_glib/test/test-array.rb
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestArray < Test::Unit::TestCase
+  def test_length
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append(true)
+    array = builder.finish
+    assert_equal(1, array.length)
+  end
+
+  def test_n_nulls
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append_null
+    builder.append_null
+    array = builder.finish
+    assert_equal(2, array.n_nulls)
+  end
+
+  def test_slice
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append(true)
+    builder.append(false)
+    builder.append(true)
+    array = builder.finish
+    sub_array = array.slice(1, 2)
+    assert_equal([false, true],
+                 sub_array.length.times.collect {|i| sub_array.get_value(i)})
+  end
+end
diff --git a/c_glib/test/test-binary-array.rb b/c_glib/test/test-binary-array.rb
new file mode 100644
index 0000000000000..82a537ef29e9e
--- /dev/null
+++ b/c_glib/test/test-binary-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBinaryArray < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::BinaryArrayBuilder.new
+    builder.append("\x00\x01\x02")
+    array = builder.finish
+    assert_equal([0, 1, 2], array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-binary-data-type.rb b/c_glib/test/test-binary-data-type.rb
new file mode 100644
index 0000000000000..3d4095c1b0648
--- /dev/null
+++ b/c_glib/test/test-binary-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBinaryDataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::BinaryDataType.new
+    assert_equal(Arrow::Type::BINARY, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::BinaryDataType.new
+    assert_equal("binary", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-boolean-array.rb b/c_glib/test/test-boolean-array.rb
new file mode 100644
index 0000000000000..9cc3c94d554bf
--- /dev/null
+++ b/c_glib/test/test-boolean-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBooleanArray < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append(true)
+    array = builder.finish
+    assert_equal(true, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-boolean-data-type.rb b/c_glib/test/test-boolean-data-type.rb
new file mode 100644
index 0000000000000..ac5667140fb8e
--- /dev/null
+++ b/c_glib/test/test-boolean-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBooleanDataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::BooleanDataType.new
+    assert_equal(Arrow::Type::BOOL, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::BooleanDataType.new
+    assert_equal("bool", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-chunked-array.rb b/c_glib/test/test-chunked-array.rb
new file mode 100644
index 0000000000000..167d5d1033e42
--- /dev/null
+++ b/c_glib/test/test-chunked-array.rb
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestChunkedArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_length
+    chunks = [
+      build_boolean_array([true, false]),
+      build_boolean_array([true]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(chunks)
+    assert_equal(3, chunked_array.length)
+  end
+
+  def test_n_nulls
+    chunks = [
+      build_boolean_array([true, nil, false]),
+      build_boolean_array([nil, nil, true]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(chunks)
+    assert_equal(3, chunked_array.n_nulls)
+  end
+
+
+  def test_n_chunks
+    chunks = [
+      build_boolean_array([true]),
+      build_boolean_array([false]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(chunks)
+    assert_equal(2, chunked_array.n_chunks)
+  end
+
+  def test_chunk
+    chunks = [
+      build_boolean_array([true, false]),
+      build_boolean_array([false]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(chunks)
+    assert_equal(2, chunked_array.get_chunk(0).length)
+  end
+
+  def test_chunks
+    chunks = [
+      build_boolean_array([true, false]),
+      build_boolean_array([false]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(chunks)
+    assert_equal([2, 1],
+                 chunked_array.chunks.collect(&:length))
+  end
+end
diff --git a/c_glib/test/test-column.rb b/c_glib/test/test-column.rb
new file mode 100644
index 0000000000000..ec75194edb830
--- /dev/null
+++ b/c_glib/test/test-column.rb
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestColumn < Test::Unit::TestCase
+  include Helper::Buildable
+
+  sub_test_case(".new") do
+    def test_array
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      array = build_boolean_array([true])
+      column = Arrow::Column.new(field, array)
+      assert_equal(1, column.length)
+    end
+
+    def test_chunked_array
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      chunks = [
+        build_boolean_array([true]),
+        build_boolean_array([false, true]),
+      ]
+      chunked_array = Arrow::ChunkedArray.new(chunks)
+      column = Arrow::Column.new(field, chunked_array)
+      assert_equal(3, column.length)
+    end
+  end
+
+  def test_length
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    array = build_boolean_array([true, false])
+    column = Arrow::Column.new(field, array)
+    assert_equal(2, column.length)
+  end
+
+  def test_n_nulls
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    array = build_boolean_array([true, nil, nil])
+    column = Arrow::Column.new(field, array)
+    assert_equal(2, column.n_nulls)
+  end
+
+  def test_field
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    array = build_boolean_array([true])
+    column = Arrow::Column.new(field, array)
+    assert_equal("enabled", column.field.name)
+  end
+
+  def test_name
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    array = build_boolean_array([true])
+    column = Arrow::Column.new(field, array)
+    assert_equal("enabled", column.name)
+  end
+
+  def test_data_type
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    array = build_boolean_array([true])
+    column = Arrow::Column.new(field, array)
+    assert_equal("bool", column.data_type.to_s)
+  end
+
+  def test_data
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    chunks = [
+      build_boolean_array([true]),
+      build_boolean_array([false, true]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(chunks)
+    column = Arrow::Column.new(field, chunked_array)
+    assert_equal(3, column.data.length)
+  end
+end
diff --git a/c_glib/test/test-double-array.rb b/c_glib/test/test-double-array.rb
new file mode 100644
index 0000000000000..f9c000d23f173
--- /dev/null
+++ b/c_glib/test/test-double-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDoubleArray < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::DoubleArrayBuilder.new
+    builder.append(1.5)
+    array = builder.finish
+    assert_in_delta(1.5, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-double-data-type.rb b/c_glib/test/test-double-data-type.rb
new file mode 100644
index 0000000000000..18c870cb9e62b
--- /dev/null
+++ b/c_glib/test/test-double-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDoubleDataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::DoubleDataType.new
+    assert_equal(Arrow::Type::DOUBLE, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::DoubleDataType.new
+    assert_equal("double", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-field.rb b/c_glib/test/test-field.rb
new file mode 100644
index 0000000000000..a20802c2ac653
--- /dev/null
+++ b/c_glib/test/test-field.rb
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestField < Test::Unit::TestCase
+  def test_name
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    assert_equal("enabled", field.name)
+  end
+
+  def test_data_type
+    data_type = Arrow::BooleanDataType.new
+    field = Arrow::Field.new("enabled", data_type)
+    assert_equal(data_type.to_s, field.data_type.to_s)
+  end
+
+  def test_nullable?
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    assert do
+      field.nullable?
+    end
+  end
+
+  def test_to_s
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    assert_equal("enabled: bool", field.to_s)
+  end
+end
diff --git a/c_glib/test/test-float-array.rb b/c_glib/test/test-float-array.rb
new file mode 100644
index 0000000000000..020c705aad241
--- /dev/null
+++ b/c_glib/test/test-float-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestFloatArray < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::FloatArrayBuilder.new
+    builder.append(1.5)
+    array = builder.finish
+    assert_in_delta(1.5, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-float-data-type.rb b/c_glib/test/test-float-data-type.rb
new file mode 100644
index 0000000000000..ab315fd336b84
--- /dev/null
+++ b/c_glib/test/test-float-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestFloatDataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::FloatDataType.new
+    assert_equal(Arrow::Type::FLOAT, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::FloatDataType.new
+    assert_equal("float", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-int16-array.rb b/c_glib/test/test-int16-array.rb
new file mode 100644
index 0000000000000..2aa5b0c054563
--- /dev/null
+++ b/c_glib/test/test-int16-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt16Array < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::Int16ArrayBuilder.new
+    builder.append(-1)
+    array = builder.finish
+    assert_equal(-1, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-int16-data-type.rb b/c_glib/test/test-int16-data-type.rb
new file mode 100644
index 0000000000000..273ec809c198e
--- /dev/null
+++ b/c_glib/test/test-int16-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt16DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::Int16DataType.new
+    assert_equal(Arrow::Type::INT16, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::Int16DataType.new
+    assert_equal("int16", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-int32-array.rb b/c_glib/test/test-int32-array.rb
new file mode 100644
index 0000000000000..9dd6b3afc8676
--- /dev/null
+++ b/c_glib/test/test-int32-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt32Array < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::Int32ArrayBuilder.new
+    builder.append(-1)
+    array = builder.finish
+    assert_equal(-1, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-int32-data-type.rb b/c_glib/test/test-int32-data-type.rb
new file mode 100644
index 0000000000000..f6b9b34e1d827
--- /dev/null
+++ b/c_glib/test/test-int32-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt32DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::Int32DataType.new
+    assert_equal(Arrow::Type::INT32, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::Int32DataType.new
+    assert_equal("int32", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-int64-array.rb b/c_glib/test/test-int64-array.rb
new file mode 100644
index 0000000000000..612a8b4f69276
--- /dev/null
+++ b/c_glib/test/test-int64-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt64Array < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::Int64ArrayBuilder.new
+    builder.append(-1)
+    array = builder.finish
+    assert_equal(-1, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-int64-data-type.rb b/c_glib/test/test-int64-data-type.rb
new file mode 100644
index 0000000000000..032b24dac3ecc
--- /dev/null
+++ b/c_glib/test/test-int64-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt64DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::Int64DataType.new
+    assert_equal(Arrow::Type::INT64, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::Int64DataType.new
+    assert_equal("int64", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-int8-array.rb b/c_glib/test/test-int8-array.rb
new file mode 100644
index 0000000000000..ab009964ab16f
--- /dev/null
+++ b/c_glib/test/test-int8-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt8Array < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::Int8ArrayBuilder.new
+    builder.append(-1)
+    array = builder.finish
+    assert_equal(-1, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-int8-data-type.rb b/c_glib/test/test-int8-data-type.rb
new file mode 100644
index 0000000000000..d33945614db8e
--- /dev/null
+++ b/c_glib/test/test-int8-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt8DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::Int8DataType.new
+    assert_equal(Arrow::Type::INT8, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::Int8DataType.new
+    assert_equal("int8", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-io-file-output-stream.rb b/c_glib/test/test-io-file-output-stream.rb
new file mode 100644
index 0000000000000..1f2ae5fa10fd1
--- /dev/null
+++ b/c_glib/test/test-io-file-output-stream.rb
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestIOFileOutputStream < Test::Unit::TestCase
+  sub_test_case(".open") do
+    def test_create
+      tempfile = Tempfile.open("arrow-io-file-output-stream")
+      tempfile.write("Hello")
+      tempfile.close
+      file = ArrowIO::FileOutputStream.open(tempfile.path, false)
+      file.close
+      assert_equal("", File.read(tempfile.path))
+    end
+
+    def test_append
+      tempfile = Tempfile.open("arrow-io-file-output-stream")
+      tempfile.write("Hello")
+      tempfile.close
+      file = ArrowIO::FileOutputStream.open(tempfile.path, true)
+      file.close
+      assert_equal("Hello", File.read(tempfile.path))
+    end
+  end
+end
diff --git a/c_glib/test/test-io-memory-mapped-file.rb b/c_glib/test/test-io-memory-mapped-file.rb
new file mode 100644
index 0000000000000..609819833614f
--- /dev/null
+++ b/c_glib/test/test-io-memory-mapped-file.rb
@@ -0,0 +1,138 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestIOMemoryMappedFile < Test::Unit::TestCase
+  def test_open
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    begin
+      buffer = " " * 5
+      file.read(buffer)
+      assert_equal("Hello", buffer)
+    ensure
+      file.close
+    end
+  end
+
+  def test_size
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    begin
+      assert_equal(5, file.size)
+    ensure
+      file.close
+    end
+  end
+
+  def test_read
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello World")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    begin
+      buffer = " " * 5
+      _success, n_read_bytes = file.read(buffer)
+      assert_equal("Hello", buffer.byteslice(0, n_read_bytes))
+    ensure
+      file.close
+    end
+  end
+
+  def test_read_at
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello World")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    begin
+      buffer = " " * 5
+      _success, n_read_bytes = file.read_at(6, buffer)
+      assert_equal("World", buffer.byteslice(0, n_read_bytes))
+    ensure
+      file.close
+    end
+  end
+
+  def test_write
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :readwrite)
+    begin
+      file.write("World")
+    ensure
+      file.close
+    end
+    assert_equal("World", File.read(tempfile.path))
+  end
+
+  def test_write_at
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :readwrite)
+    begin
+      file.write_at(2, "rld")
+    ensure
+      file.close
+    end
+    assert_equal("Herld", File.read(tempfile.path))
+  end
+
+  def test_flush
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :readwrite)
+    begin
+      file.write("World")
+      file.flush
+      assert_equal("World", File.read(tempfile.path))
+    ensure
+      file.close
+    end
+  end
+
+  def test_tell
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello World")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    begin
+      buffer = " " * 5
+      file.read(buffer)
+      assert_equal(5, file.tell)
+    ensure
+      file.close
+    end
+  end
+
+  def test_mode
+    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
+    tempfile.write("Hello World")
+    tempfile.close
+    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :readwrite)
+    begin
+      assert_equal(ArrowIO::FileMode::READWRITE, file.mode)
+    ensure
+      file.close
+    end
+  end
+end
diff --git a/c_glib/test/test-ipc-file-writer.rb b/c_glib/test/test-ipc-file-writer.rb
new file mode 100644
index 0000000000000..369bff324e6d9
--- /dev/null
+++ b/c_glib/test/test-ipc-file-writer.rb
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestIPCFileWriter < Test::Unit::TestCase
+  def test_write_record_batch
+    tempfile = Tempfile.open("arrow-ipc-file-writer")
+    output = ArrowIO::FileOutputStream.open(tempfile.path, false)
+    begin
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      schema = Arrow::Schema.new([field])
+      file_writer = ArrowIPC::FileWriter.open(output, schema)
+      begin
+        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        file_writer.write_record_batch(record_batch)
+      ensure
+        file_writer.close
+      end
+    ensure
+      output.close
+    end
+
+    input = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    begin
+      file_reader = ArrowIPC::FileReader.open(input)
+      assert_equal(["enabled"],
+                   file_reader.schema.fields.collect(&:name))
+    ensure
+      input.close
+    end
+  end
+end
diff --git a/c_glib/test/test-ipc-stream-writer.rb b/c_glib/test/test-ipc-stream-writer.rb
new file mode 100644
index 0000000000000..62ac45dce2c79
--- /dev/null
+++ b/c_glib/test/test-ipc-stream-writer.rb
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestIPCStreamWriter < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_write_record_batch
+    tempfile = Tempfile.open("arrow-ipc-stream-writer")
+    output = ArrowIO::FileOutputStream.open(tempfile.path, false)
+    begin
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      schema = Arrow::Schema.new([field])
+      stream_writer = ArrowIPC::StreamWriter.open(output, schema)
+      begin
+        columns = [
+          build_boolean_array([true]),
+        ]
+        record_batch = Arrow::RecordBatch.new(schema, 1, columns)
+        stream_writer.write_record_batch(record_batch)
+      ensure
+        stream_writer.close
+      end
+    ensure
+      output.close
+    end
+
+    input = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    begin
+      stream_reader = ArrowIPC::StreamReader.open(input)
+      assert_equal(["enabled"],
+                   stream_reader.schema.fields.collect(&:name))
+      assert_equal(true,
+                   stream_reader.next_record_batch.get_column(0).get_value(0))
+      assert_nil(stream_reader.next_record_batch)
+    ensure
+      input.close
+    end
+  end
+end
diff --git a/c_glib/test/test-list-array.rb b/c_glib/test/test-list-array.rb
new file mode 100644
index 0000000000000..34177de9dcdeb
--- /dev/null
+++ b/c_glib/test/test-list-array.rb
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestListArray < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::ListArrayBuilder.new(Arrow::Int8ArrayBuilder.new)
+    value_builder = builder.value_builder
+
+    builder.append
+    value_builder.append(-29)
+    value_builder.append(29)
+
+    builder.append
+    value_builder.append(-1)
+    value_builder.append(0)
+    value_builder.append(1)
+
+    array = builder.finish
+    value = array.get_value(1)
+    assert_equal([-1, 0, 1],
+                 value.length.times.collect {|i| value.get_value(i)})
+  end
+
+  def test_value_type
+    builder = Arrow::ListArrayBuilder.new(Arrow::Int8ArrayBuilder.new)
+    array = builder.finish
+    assert_equal(Arrow::Int8DataType.new, array.value_type)
+  end
+end
diff --git a/c_glib/test/test-list-data-type.rb b/c_glib/test/test-list-data-type.rb
new file mode 100644
index 0000000000000..6fde203517684
--- /dev/null
+++ b/c_glib/test/test-list-data-type.rb
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestListDataType < Test::Unit::TestCase
+  def test_type
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    data_type = Arrow::ListDataType.new(field)
+    assert_equal(Arrow::Type::LIST, data_type.type)
+  end
+
+  def test_to_s
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    data_type = Arrow::ListDataType.new(field)
+    assert_equal("list<enabled: bool>", data_type.to_s)
+  end
+
+  def test_value_field
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    data_type = Arrow::ListDataType.new(field)
+    assert_equal(field, data_type.value_field)
+  end
+end
diff --git a/c_glib/test/test-null-array.rb b/c_glib/test/test-null-array.rb
new file mode 100644
index 0000000000000..6aa8c037c17ee
--- /dev/null
+++ b/c_glib/test/test-null-array.rb
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestNullArray < Test::Unit::TestCase
+  def test_length
+    array = Arrow::NullArray.new(3)
+    assert_equal(3, array.length)
+  end
+
+  def test_n_nulls
+    array = Arrow::NullArray.new(3)
+    assert_equal(3, array.n_nulls)
+  end
+
+  def test_slice
+    array = Arrow::NullArray.new(3)
+    assert_equal(2, array.slice(1, 2).length)
+  end
+end
diff --git a/c_glib/test/test-null-data-type.rb b/c_glib/test/test-null-data-type.rb
new file mode 100644
index 0000000000000..95e54833b0896
--- /dev/null
+++ b/c_glib/test/test-null-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestNullDataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::NullDataType.new
+    assert_equal(Arrow::Type::NA, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::NullDataType.new
+    assert_equal("null", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-record-batch.rb b/c_glib/test/test-record-batch.rb
new file mode 100644
index 0000000000000..941ff35060154
--- /dev/null
+++ b/c_glib/test/test-record-batch.rb
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTable < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    fields = [
+      Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+      Arrow::Field.new("valid", Arrow::BooleanDataType.new),
+    ]
+    schema = Arrow::Schema.new(fields)
+    columns = [
+      build_boolean_array([true]),
+      build_boolean_array([false]),
+    ]
+    record_batch = Arrow::RecordBatch.new(schema, 1, columns)
+    assert_equal(1, record_batch.n_rows)
+  end
+
+  sub_test_case("instance methods") do
+    def setup
+      fields = [
+        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
+      ]
+      schema = Arrow::Schema.new(fields)
+      columns = [
+        build_boolean_array([true, false, true, false, true, false]),
+        build_boolean_array([false, true, false, true, false]),
+      ]
+      @record_batch = Arrow::RecordBatch.new(schema, 5, columns)
+    end
+
+    def test_schema
+      assert_equal(["visible", "valid"],
+                   @record_batch.schema.fields.collect(&:name))
+    end
+
+    def test_column
+      assert_equal(5, @record_batch.get_column(1).length)
+    end
+
+    def test_columns
+      assert_equal([6, 5],
+                   @record_batch.columns.collect(&:length))
+    end
+
+    def test_n_columns
+      assert_equal(2, @record_batch.n_columns)
+    end
+
+    def test_n_rows
+      assert_equal(5, @record_batch.n_rows)
+    end
+
+    def test_slice
+      sub_record_batch = @record_batch.slice(3, 2)
+      sub_visible_values = sub_record_batch.n_rows.times.collect do |i|
+        sub_record_batch.get_column(0).get_value(i)
+      end
+      assert_equal([false, true],
+                   sub_visible_values)
+    end
+  end
+end
diff --git a/c_glib/test/test-schema.rb b/c_glib/test/test-schema.rb
new file mode 100644
index 0000000000000..c9cbb756944bb
--- /dev/null
+++ b/c_glib/test/test-schema.rb
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestSchema < Test::Unit::TestCase
+  def test_field
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+    assert_equal("enabled", schema.get_field(0).name)
+  end
+
+  sub_test_case("#get_field_by_name") do
+    def test_found
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      schema = Arrow::Schema.new([field])
+      assert_equal("enabled", schema.get_field_by_name("enabled").name)
+    end
+
+    def test_not_found
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      schema = Arrow::Schema.new([field])
+      assert_nil(schema.get_field_by_name("nonexistent"))
+    end
+  end
+
+  def test_n_fields
+    fields = [
+      Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
+      Arrow::Field.new("required", Arrow::BooleanDataType.new),
+    ]
+    schema = Arrow::Schema.new(fields)
+    assert_equal(2, schema.n_fields)
+  end
+
+  def test_fields
+    fields = [
+      Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
+      Arrow::Field.new("required", Arrow::BooleanDataType.new),
+    ]
+    schema = Arrow::Schema.new(fields)
+    assert_equal(["enabled", "required"],
+                 schema.fields.collect(&:name))
+  end
+
+  def test_to_s
+    fields = [
+      Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
+      Arrow::Field.new("required", Arrow::BooleanDataType.new),
+    ]
+    schema = Arrow::Schema.new(fields)
+    assert_equal(<<-SCHEMA.chomp, schema.to_s)
+enabled: bool
+required: bool
+    SCHEMA
+  end
+end
diff --git a/c_glib/test/test-string-array.rb b/c_glib/test/test-string-array.rb
new file mode 100644
index 0000000000000..a0f5a7b6b0fda
--- /dev/null
+++ b/c_glib/test/test-string-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestStringArray < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::StringArrayBuilder.new
+    builder.append("Hello")
+    array = builder.finish
+    assert_equal("Hello", array.get_string(0))
+  end
+end
diff --git a/c_glib/test/test-string-data-type.rb b/c_glib/test/test-string-data-type.rb
new file mode 100644
index 0000000000000..daba7fd9ec768
--- /dev/null
+++ b/c_glib/test/test-string-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestStringDataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::StringDataType.new
+    assert_equal(Arrow::Type::STRING, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::StringDataType.new
+    assert_equal("string", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-struct-array.rb b/c_glib/test/test-struct-array.rb
new file mode 100644
index 0000000000000..cf450f52d299a
--- /dev/null
+++ b/c_glib/test/test-struct-array.rb
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestStructArray < Test::Unit::TestCase
+  def test_fields
+    fields = [
+      Arrow::Field.new("score", Arrow::Int8DataType.new),
+      Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
+    ]
+    data_type = Arrow::StructDataType.new(fields)
+    field_builders = [
+      Arrow::Int8ArrayBuilder.new,
+      Arrow::BooleanArrayBuilder.new,
+    ]
+    builder = Arrow::StructArrayBuilder.new(data_type, field_builders)
+
+    builder.append
+    builder.get_field_builder(0).append(-29)
+    builder.get_field_builder(1).append(true)
+
+    builder.append
+    builder.field_builders[0].append(2)
+    builder.field_builders[1].append(false)
+
+    array = builder.finish
+    values = array.length.times.collect do |i|
+      if i.zero?
+        [
+          array.get_field(0).get_value(i),
+          array.get_field(1).get_value(i),
+        ]
+      else
+        array.fields.collect do |field|
+          field.get_value(i)
+        end
+      end
+    end
+    assert_equal([
+                   [-29, true],
+                   [2, false],
+                 ],
+                 values)
+  end
+end
diff --git a/c_glib/test/test-table.rb b/c_glib/test/test-table.rb
new file mode 100644
index 0000000000000..1687d2f6e3ff6
--- /dev/null
+++ b/c_glib/test/test-table.rb
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTable < Test::Unit::TestCase
+  include Helper::Buildable
+
+  sub_test_case(".new") do
+    def test_columns
+      fields = [
+        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
+      ]
+      schema = Arrow::Schema.new(fields)
+      columns = [
+        Arrow::Column.new(fields[0], build_boolean_array([true])),
+        Arrow::Column.new(fields[1], build_boolean_array([false])),
+      ]
+      table = Arrow::Table.new("memos", schema, columns)
+      assert_equal("memos", table.name)
+    end
+  end
+
+  sub_test_case("instance methods") do
+    def setup
+      fields = [
+        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
+      ]
+      schema = Arrow::Schema.new(fields)
+      columns = [
+        Arrow::Column.new(fields[0], build_boolean_array([true])),
+        Arrow::Column.new(fields[1], build_boolean_array([false])),
+      ]
+      @table = Arrow::Table.new("memos", schema, columns)
+    end
+
+    def test_name
+      assert_equal("memos", @table.name)
+    end
+
+    def test_schema
+      assert_equal(["visible", "valid"],
+                   @table.schema.fields.collect(&:name))
+    end
+
+    def test_column
+      assert_equal("valid", @table.get_column(1).name)
+    end
+
+    def test_n_columns
+      assert_equal(2, @table.n_columns)
+    end
+
+    def test_n_rows
+      assert_equal(1, @table.n_rows)
+    end
+  end
+end
diff --git a/c_glib/test/test-uint16-array.rb b/c_glib/test/test-uint16-array.rb
new file mode 100644
index 0000000000000..ad85f09326bd3
--- /dev/null
+++ b/c_glib/test/test-uint16-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt16Array < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::UInt16ArrayBuilder.new
+    builder.append(1)
+    array = builder.finish
+    assert_equal(1, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-uint16-data-type.rb b/c_glib/test/test-uint16-data-type.rb
new file mode 100644
index 0000000000000..f5a6cc0be28bb
--- /dev/null
+++ b/c_glib/test/test-uint16-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt16DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::UInt16DataType.new
+    assert_equal(Arrow::Type::UINT16, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::UInt16DataType.new
+    assert_equal("uint16", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-uint32-array.rb b/c_glib/test/test-uint32-array.rb
new file mode 100644
index 0000000000000..59e19f3ed796f
--- /dev/null
+++ b/c_glib/test/test-uint32-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt32Array < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::UInt32ArrayBuilder.new
+    builder.append(1)
+    array = builder.finish
+    assert_equal(1, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-uint32-data-type.rb b/c_glib/test/test-uint32-data-type.rb
new file mode 100644
index 0000000000000..7a50257d6d3b9
--- /dev/null
+++ b/c_glib/test/test-uint32-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt32DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::UInt32DataType.new
+    assert_equal(Arrow::Type::UINT32, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::UInt32DataType.new
+    assert_equal("uint32", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-uint64-array.rb b/c_glib/test/test-uint64-array.rb
new file mode 100644
index 0000000000000..e0195c1d49817
--- /dev/null
+++ b/c_glib/test/test-uint64-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt64Array < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::UInt64ArrayBuilder.new
+    builder.append(1)
+    array = builder.finish
+    assert_equal(1, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-uint64-data-type.rb b/c_glib/test/test-uint64-data-type.rb
new file mode 100644
index 0000000000000..403fc9acdfcfa
--- /dev/null
+++ b/c_glib/test/test-uint64-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt64DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::UInt64DataType.new
+    assert_equal(Arrow::Type::UINT64, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::UInt64DataType.new
+    assert_equal("uint64", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-uint8-array.rb b/c_glib/test/test-uint8-array.rb
new file mode 100644
index 0000000000000..02f3470774c10
--- /dev/null
+++ b/c_glib/test/test-uint8-array.rb
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt8Array < Test::Unit::TestCase
+  def test_value
+    builder = Arrow::UInt8ArrayBuilder.new
+    builder.append(1)
+    array = builder.finish
+    assert_equal(1, array.get_value(0))
+  end
+end
diff --git a/c_glib/test/test-uint8-data-type.rb b/c_glib/test/test-uint8-data-type.rb
new file mode 100644
index 0000000000000..eb91da2761efe
--- /dev/null
+++ b/c_glib/test/test-uint8-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt8DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::UInt8DataType.new
+    assert_equal(Arrow::Type::UINT8, data_type.type)
+  end
+
+  def test_to_s
+    data_type = Arrow::UInt8DataType.new
+    assert_equal("uint8", data_type.to_s)
+  end
+end
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
new file mode 100755
index 0000000000000..1a828e7659bd9
--- /dev/null
+++ b/ci/travis_before_script_c_glib.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+
+set -ex
+
+if [ $TRAVIS_OS_NAME == "osx" ]; then
+    brew install gtk-doc autoconf-archive gobject-introspection
+fi
+
+gem install gobject-introspection
+
+ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
+
+pushd $ARROW_C_GLIB_DIR
+
+: ${ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install}
+
+./autogen.sh
+
+export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
+
+./configure --prefix=${ARROW_C_GLIB_INSTALL} --enable-gtk-doc
+
+make -j4
+make install
+
+popd
diff --git a/ci/travis_script_c_glib.sh b/ci/travis_script_c_glib.sh
new file mode 100755
index 0000000000000..1492354405810
--- /dev/null
+++ b/ci/travis_script_c_glib.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+set -e
+
+ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
+
+pushd $ARROW_C_GLIB_DIR
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
+NO_MAKE=yes test/run-test.sh
+
+popd

From 57b537a3ce54698c60addb9d193ecfc3b88397aa Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 17 Mar 2017 17:20:48 -0700
Subject: [PATCH 0377/1644] ARROW-637: [Format] Add timezone to Timestamp
 metadata, comments describing the semantics

cc @julianhyde @julienledem for comment. This makes Arrow semantically equivalent to datetime objects in Python, and I believe the SQL standard

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #388 from wesm/ARROW-637 and squashes the following commits:

e4661a4 [Wes McKinney] Allow for absolute time zone offsets
3fc10d6 [Wes McKinney] typo
a25967a [Wes McKinney] Add timezone to Timestamp metadata, comments describing the semantics
---
 format/Message.fbs | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/format/Message.fbs b/format/Message.fbs
index fb3478de5d2a0..f2d5eba75e65b 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -93,6 +93,28 @@ table Time {
 /// time from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
 table Timestamp {
   unit: TimeUnit;
+
+  /// The time zone is a string indicating the name of a time zone, one of:
+  ///
+  /// * As used in the Olson time zone database (the "tz database" or
+  ///   "tzdata"), such as "America/New_York"
+  /// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
+  ///
+  /// Whether a timezone string is present indicates different semantics about
+  /// the data:
+  ///
+  /// * If the time zone is null or equal to an empty string, the data is "time
+  ///   zone naive" and shall be displayed *as is* to the user, not localized
+  ///   to the locale of the user. This data can be though of as UTC but
+  ///   without having "UTC" as the time zone, it is not considered to be
+  ///   localized to any time zone
+  ///
+  /// * If the time zone is set to a valid value, values can be displayed as
+  ///   "localized" to that time zone, even though the underlying 64-bit
+  ///   integers are identical to the same data stored in UTC. Converting
+  ///   between time zones is a metadata-only operation and does not change the
+  ///   underlying values
+  timezone: string;
 }
 
 enum IntervalUnit: short { YEAR_MONTH, DAY_TIME}

From 16dd87164d7ab756dc6c5eaabd22ef767edca037 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 18 Mar 2017 18:14:49 +0100
Subject: [PATCH 0378/1644] ARROW-650: [GLib] Follow ReadableFileInterface ->
 RnadomAccessFile change

Author: Kouhei Sutou <kou@clear-code.com>

Closes #399 from kou/glib-follow-random-access-change and squashes the following commits:

d46a1cb [Kouhei Sutou] [GLib] Follow ReadableFileInterface -> RnadomAccessFile change
---
 c_glib/arrow-glib/Makefile.am                 |   6 +-
 c_glib/arrow-glib/arrow-io-glib.h             |   2 +-
 c_glib/arrow-glib/arrow-io-glib.hpp           |   2 +-
 c_glib/arrow-glib/io-memory-mapped-file.cpp   |  14 +-
 c_glib/arrow-glib/io-random-access-file.cpp   | 128 ++++++++++++++++++
 c_glib/arrow-glib/io-random-access-file.h     |  55 ++++++++
 ...ble-file.hpp => io-random-access-file.hpp} |  12 +-
 c_glib/arrow-glib/io-readable-file.cpp        | 127 -----------------
 c_glib/arrow-glib/io-readable-file.h          |  55 --------
 c_glib/arrow-glib/ipc-file-reader.cpp         |   6 +-
 c_glib/arrow-glib/ipc-file-reader.h           |   4 +-
 c_glib/doc/reference/arrow-glib-docs.sgml     |   2 +-
 12 files changed, 207 insertions(+), 206 deletions(-)
 create mode 100644 c_glib/arrow-glib/io-random-access-file.cpp
 create mode 100644 c_glib/arrow-glib/io-random-access-file.h
 rename c_glib/arrow-glib/{io-readable-file.hpp => io-random-access-file.hpp} (69%)
 delete mode 100644 c_glib/arrow-glib/io-readable-file.cpp
 delete mode 100644 c_glib/arrow-glib/io-readable-file.h

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 61137a075f601..7699594b7ade7 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -242,8 +242,8 @@ libarrow_io_glib_la_headers =			\
 	io-input-stream.h			\
 	io-memory-mapped-file.h			\
 	io-output-stream.h			\
+	io-random-access-file.h			\
 	io-readable.h				\
-	io-readable-file.h			\
 	io-writeable.h				\
 	io-writeable-file.h
 
@@ -261,8 +261,8 @@ libarrow_io_glib_la_sources =			\
 	io-input-stream.cpp			\
 	io-memory-mapped-file.cpp		\
 	io-output-stream.cpp			\
+	io-random-access-file.cpp		\
 	io-readable.cpp				\
-	io-readable-file.cpp			\
 	io-writeable.cpp			\
 	io-writeable-file.cpp			\
 	$(libarrow_io_glib_la_headers)		\
@@ -276,8 +276,8 @@ libarrow_io_glib_la_cpp_headers =		\
 	io-input-stream.hpp			\
 	io-memory-mapped-file.hpp		\
 	io-output-stream.hpp			\
+	io-random-access-file.hpp		\
 	io-readable.hpp				\
-	io-readable-file.hpp			\
 	io-writeable.hpp			\
 	io-writeable-file.hpp
 
diff --git a/c_glib/arrow-glib/arrow-io-glib.h b/c_glib/arrow-glib/arrow-io-glib.h
index e02aa9b96982b..4d49a9859d82a 100644
--- a/c_glib/arrow-glib/arrow-io-glib.h
+++ b/c_glib/arrow-glib/arrow-io-glib.h
@@ -26,7 +26,7 @@
 #include <arrow-glib/io-input-stream.h>
 #include <arrow-glib/io-memory-mapped-file.h>
 #include <arrow-glib/io-output-stream.h>
+#include <arrow-glib/io-random-access-file.h>
 #include <arrow-glib/io-readable.h>
-#include <arrow-glib/io-readable-file.h>
 #include <arrow-glib/io-writeable.h>
 #include <arrow-glib/io-writeable-file.h>
diff --git a/c_glib/arrow-glib/arrow-io-glib.hpp b/c_glib/arrow-glib/arrow-io-glib.hpp
index 378f20216b6a1..3e7636cc7ef99 100644
--- a/c_glib/arrow-glib/arrow-io-glib.hpp
+++ b/c_glib/arrow-glib/arrow-io-glib.hpp
@@ -25,6 +25,6 @@
 #include <arrow-glib/io-input-stream.hpp>
 #include <arrow-glib/io-memory-mapped-file.hpp>
 #include <arrow-glib/io-output-stream.hpp>
+#include <arrow-glib/io-random-access-file.hpp>
 #include <arrow-glib/io-readable.hpp>
-#include <arrow-glib/io-readable-file.hpp>
 #include <arrow-glib/io-writeable.hpp>
diff --git a/c_glib/arrow-glib/io-memory-mapped-file.cpp b/c_glib/arrow-glib/io-memory-mapped-file.cpp
index aa6ae2afd6e78..12c9a6c95ac12 100644
--- a/c_glib/arrow-glib/io-memory-mapped-file.cpp
+++ b/c_glib/arrow-glib/io-memory-mapped-file.cpp
@@ -29,7 +29,7 @@
 #include <arrow-glib/io-input-stream.hpp>
 #include <arrow-glib/io-memory-mapped-file.hpp>
 #include <arrow-glib/io-readable.hpp>
-#include <arrow-glib/io-readable-file.hpp>
+#include <arrow-glib/io-random-access-file.hpp>
 #include <arrow-glib/io-writeable.hpp>
 #include <arrow-glib/io-writeable-file.hpp>
 
@@ -97,8 +97,8 @@ garrow_io_input_stream_interface_init(GArrowIOInputStreamInterface *iface)
   iface->get_raw = garrow_io_memory_mapped_file_get_raw_input_stream_interface;
 }
 
-static std::shared_ptr<arrow::io::ReadableFileInterface>
-garrow_io_memory_mapped_file_get_raw_readable_file_interface(GArrowIOReadableFile *file)
+static std::shared_ptr<arrow::io::RandomAccessFile>
+garrow_io_memory_mapped_file_get_raw_random_access_file_interface(GArrowIORandomAccessFile *file)
 {
   auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(file);
   auto arrow_memory_mapped_file =
@@ -107,9 +107,9 @@ garrow_io_memory_mapped_file_get_raw_readable_file_interface(GArrowIOReadableFil
 }
 
 static void
-garrow_io_readable_file_interface_init(GArrowIOReadableFileInterface *iface)
+garrow_io_random_access_file_interface_init(GArrowIORandomAccessFileInterface *iface)
 {
-  iface->get_raw = garrow_io_memory_mapped_file_get_raw_readable_file_interface;
+  iface->get_raw = garrow_io_memory_mapped_file_get_raw_random_access_file_interface;
 }
 
 static std::shared_ptr<arrow::io::Writeable>
@@ -152,8 +152,8 @@ G_DEFINE_TYPE_WITH_CODE(GArrowIOMemoryMappedFile,
                                               garrow_io_readable_interface_init)
                         G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_INPUT_STREAM,
                                               garrow_io_input_stream_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_READABLE_FILE,
-                                              garrow_io_readable_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_RANDOM_ACCESS_FILE,
+                                              garrow_io_random_access_file_interface_init)
                         G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_WRITEABLE,
                                               garrow_io_writeable_interface_init)
                         G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_WRITEABLE_FILE,
diff --git a/c_glib/arrow-glib/io-random-access-file.cpp b/c_glib/arrow-glib/io-random-access-file.cpp
new file mode 100644
index 0000000000000..552b879c19794
--- /dev/null
+++ b/c_glib/arrow-glib/io-random-access-file.cpp
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/io-random-access-file.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: io-random-access-file
+ * @title: GArrowIORandomAccessFile
+ * @short_description: File input interface
+ *
+ * #GArrowIORandomAccessFile is an interface for file input.
+ */
+
+G_DEFINE_INTERFACE(GArrowIORandomAccessFile,
+                   garrow_io_random_access_file,
+                   G_TYPE_OBJECT)
+
+static void
+garrow_io_random_access_file_default_init (GArrowIORandomAccessFileInterface *iface)
+{
+}
+
+/**
+ * garrow_io_random_access_file_get_size:
+ * @file: A #GArrowIORandomAccessFile.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: The size of the file.
+ */
+guint64
+garrow_io_random_access_file_get_size(GArrowIORandomAccessFile *file,
+                                 GError **error)
+{
+  auto *iface = GARROW_IO_RANDOM_ACCESS_FILE_GET_IFACE(file);
+  auto arrow_random_access_file = iface->get_raw(file);
+  int64_t size;
+
+  auto status = arrow_random_access_file->GetSize(&size);
+  if (status.ok()) {
+    return size;
+  } else {
+    garrow_error_set(error, status, "[io][random-access-file][get-size]");
+    return 0;
+  }
+}
+
+/**
+ * garrow_io_random_access_file_get_support_zero_copy:
+ * @file: A #GArrowIORandomAccessFile.
+ *
+ * Returns: Whether zero copy read is supported or not.
+ */
+gboolean
+garrow_io_random_access_file_get_support_zero_copy(GArrowIORandomAccessFile *file)
+{
+  auto *iface = GARROW_IO_RANDOM_ACCESS_FILE_GET_IFACE(file);
+  auto arrow_random_access_file = iface->get_raw(file);
+
+  return arrow_random_access_file->supports_zero_copy();
+}
+
+/**
+ * garrow_io_random_access_file_read_at:
+ * @file: A #GArrowIORandomAccessFile.
+ * @position: The read start position.
+ * @n_bytes: The number of bytes to be read.
+ * @n_read_bytes: (out): The read number of bytes.
+ * @buffer: (array length=n_bytes): The buffer to be read data.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_io_random_access_file_read_at(GArrowIORandomAccessFile *file,
+                                     gint64 position,
+                                     gint64 n_bytes,
+                                     gint64 *n_read_bytes,
+                                     guint8 *buffer,
+                                     GError **error)
+{
+  const auto arrow_random_access_file =
+    garrow_io_random_access_file_get_raw(file);
+
+  auto status = arrow_random_access_file->ReadAt(position,
+                                                 n_bytes,
+                                                 n_read_bytes,
+                                                 buffer);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[io][random-access-file][read-at]");
+    return FALSE;
+  }
+}
+
+G_END_DECLS
+
+std::shared_ptr<arrow::io::RandomAccessFile>
+garrow_io_random_access_file_get_raw(GArrowIORandomAccessFile *random_access_file)
+{
+  auto *iface = GARROW_IO_RANDOM_ACCESS_FILE_GET_IFACE(random_access_file);
+  return iface->get_raw(random_access_file);
+}
diff --git a/c_glib/arrow-glib/io-random-access-file.h b/c_glib/arrow-glib/io-random-access-file.h
new file mode 100644
index 0000000000000..e980ab2e3c8e5
--- /dev/null
+++ b/c_glib/arrow-glib/io-random-access-file.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_IO_TYPE_RANDOM_ACCESS_FILE       \
+  (garrow_io_random_access_file_get_type())
+#define GARROW_IO_RANDOM_ACCESS_FILE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                 \
+                              GARROW_IO_TYPE_RANDOM_ACCESS_FILE,     \
+                              GArrowIORandomAccessFileInterface))
+#define GARROW_IO_IS_RANDOM_ACCESS_FILE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_IO_TYPE_RANDOM_ACCESS_FILE))
+#define GARROW_IO_RANDOM_ACCESS_FILE_GET_IFACE(obj)                     \
+  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
+                                 GARROW_IO_TYPE_RANDOM_ACCESS_FILE,     \
+                                 GArrowIORandomAccessFileInterface))
+
+typedef struct _GArrowIORandomAccessFile          GArrowIORandomAccessFile;
+typedef struct _GArrowIORandomAccessFileInterface GArrowIORandomAccessFileInterface;
+
+GType garrow_io_random_access_file_get_type(void) G_GNUC_CONST;
+
+guint64 garrow_io_random_access_file_get_size(GArrowIORandomAccessFile *file,
+                                              GError **error);
+gboolean garrow_io_random_access_file_get_support_zero_copy(GArrowIORandomAccessFile *file);
+gboolean garrow_io_random_access_file_read_at(GArrowIORandomAccessFile *file,
+                                              gint64 position,
+                                              gint64 n_bytes,
+                                              gint64 *n_read_bytes,
+                                              guint8 *buffer,
+                                              GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/io-readable-file.hpp b/c_glib/arrow-glib/io-random-access-file.hpp
similarity index 69%
rename from c_glib/arrow-glib/io-readable-file.hpp
rename to c_glib/arrow-glib/io-random-access-file.hpp
index 83d8628f48b62..7c97c9ecedb5b 100644
--- a/c_glib/arrow-glib/io-readable-file.hpp
+++ b/c_glib/arrow-glib/io-random-access-file.hpp
@@ -21,18 +21,18 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-readable-file.h>
+#include <arrow-glib/io-random-access-file.h>
 
 /**
- * GArrowIOReadableFileInterface:
+ * GArrowIORandomAccessFileInterface:
  *
- * It wraps `arrow::io::ReadableFileInterface`.
+ * It wraps `arrow::io::RandomAccessFile`.
  */
-struct _GArrowIOReadableFileInterface
+struct _GArrowIORandomAccessFileInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::ReadableFileInterface> (*get_raw)(GArrowIOReadableFile *file);
+  std::shared_ptr<arrow::io::RandomAccessFile> (*get_raw)(GArrowIORandomAccessFile *file);
 };
 
-std::shared_ptr<arrow::io::ReadableFileInterface> garrow_io_readable_file_get_raw(GArrowIOReadableFile *readable_file);
+std::shared_ptr<arrow::io::RandomAccessFile> garrow_io_random_access_file_get_raw(GArrowIORandomAccessFile *random_access_file);
diff --git a/c_glib/arrow-glib/io-readable-file.cpp b/c_glib/arrow-glib/io-readable-file.cpp
deleted file mode 100644
index 014fd7a1c7d32..0000000000000
--- a/c_glib/arrow-glib/io-readable-file.cpp
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow/api.h>
-
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/io-readable-file.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: io-readable-file
- * @title: GArrowIOReadableFile
- * @short_description: File input interface
- *
- * #GArrowIOReadableFile is an interface for file input.
- */
-
-G_DEFINE_INTERFACE(GArrowIOReadableFile,
-                   garrow_io_readable_file,
-                   G_TYPE_OBJECT)
-
-static void
-garrow_io_readable_file_default_init (GArrowIOReadableFileInterface *iface)
-{
-}
-
-/**
- * garrow_io_readable_file_get_size:
- * @file: A #GArrowIOReadableFile.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: The size of the file.
- */
-guint64
-garrow_io_readable_file_get_size(GArrowIOReadableFile *file,
-                                 GError **error)
-{
-  auto *iface = GARROW_IO_READABLE_FILE_GET_IFACE(file);
-  auto arrow_readable_file = iface->get_raw(file);
-  int64_t size;
-
-  auto status = arrow_readable_file->GetSize(&size);
-  if (status.ok()) {
-    return size;
-  } else {
-    garrow_error_set(error, status, "[io][readable-file][get-size]");
-    return 0;
-  }
-}
-
-/**
- * garrow_io_readable_file_get_support_zero_copy:
- * @file: A #GArrowIOReadableFile.
- *
- * Returns: Whether zero copy read is supported or not.
- */
-gboolean
-garrow_io_readable_file_get_support_zero_copy(GArrowIOReadableFile *file)
-{
-  auto *iface = GARROW_IO_READABLE_FILE_GET_IFACE(file);
-  auto arrow_readable_file = iface->get_raw(file);
-
-  return arrow_readable_file->supports_zero_copy();
-}
-
-/**
- * garrow_io_readable_file_read_at:
- * @file: A #GArrowIOReadableFile.
- * @position: The read start position.
- * @n_bytes: The number of bytes to be read.
- * @n_read_bytes: (out): The read number of bytes.
- * @buffer: (array length=n_bytes): The buffer to be read data.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_io_readable_file_read_at(GArrowIOReadableFile *file,
-                                gint64 position,
-                                gint64 n_bytes,
-                                gint64 *n_read_bytes,
-                                guint8 *buffer,
-                                GError **error)
-{
-  const auto arrow_readable_file = garrow_io_readable_file_get_raw(file);
-
-  auto status = arrow_readable_file->ReadAt(position,
-                                            n_bytes,
-                                            n_read_bytes,
-                                            buffer);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[io][readable-file][read-at]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
-
-std::shared_ptr<arrow::io::ReadableFileInterface>
-garrow_io_readable_file_get_raw(GArrowIOReadableFile *readable_file)
-{
-  auto *iface = GARROW_IO_READABLE_FILE_GET_IFACE(readable_file);
-  return iface->get_raw(readable_file);
-}
diff --git a/c_glib/arrow-glib/io-readable-file.h b/c_glib/arrow-glib/io-readable-file.h
deleted file mode 100644
index 1dcb13e04969c..0000000000000
--- a/c_glib/arrow-glib/io-readable-file.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <glib-object.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_IO_TYPE_READABLE_FILE                     \
-  (garrow_io_readable_file_get_type())
-#define GARROW_IO_READABLE_FILE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IO_TYPE_READABLE_FILE,     \
-                              GArrowIOReadableFileInterface))
-#define GARROW_IO_IS_READABLE_FILE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IO_TYPE_READABLE_FILE))
-#define GARROW_IO_READABLE_FILE_GET_IFACE(obj)                          \
-  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
-                                 GARROW_IO_TYPE_READABLE_FILE,          \
-                                 GArrowIOReadableFileInterface))
-
-typedef struct _GArrowIOReadableFile          GArrowIOReadableFile;
-typedef struct _GArrowIOReadableFileInterface GArrowIOReadableFileInterface;
-
-GType garrow_io_readable_file_get_type(void) G_GNUC_CONST;
-
-guint64 garrow_io_readable_file_get_size(GArrowIOReadableFile *file,
-                                         GError **error);
-gboolean garrow_io_readable_file_get_support_zero_copy(GArrowIOReadableFile *file);
-gboolean garrow_io_readable_file_read_at(GArrowIOReadableFile *file,
-                                         gint64 position,
-                                         gint64 n_bytes,
-                                         gint64 *n_read_bytes,
-                                         guint8 *buffer,
-                                         GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-file-reader.cpp b/c_glib/arrow-glib/ipc-file-reader.cpp
index b9e408c4e9464..223be857d9beb 100644
--- a/c_glib/arrow-glib/ipc-file-reader.cpp
+++ b/c_glib/arrow-glib/ipc-file-reader.cpp
@@ -27,7 +27,7 @@
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 
-#include <arrow-glib/io-readable-file.hpp>
+#include <arrow-glib/io-random-access-file.hpp>
 
 #include <arrow-glib/ipc-file-reader.hpp>
 #include <arrow-glib/ipc-metadata-version.hpp>
@@ -139,12 +139,12 @@ garrow_ipc_file_reader_class_init(GArrowIPCFileReaderClass *klass)
  *   #GArrowIPCFileReader or %NULL on error.
  */
 GArrowIPCFileReader *
-garrow_ipc_file_reader_open(GArrowIOReadableFile *file,
+garrow_ipc_file_reader_open(GArrowIORandomAccessFile *file,
                             GError **error)
 {
   std::shared_ptr<arrow::ipc::FileReader> arrow_file_reader;
   auto status =
-    arrow::ipc::FileReader::Open(garrow_io_readable_file_get_raw(file),
+    arrow::ipc::FileReader::Open(garrow_io_random_access_file_get_raw(file),
                                  &arrow_file_reader);
   if (status.ok()) {
     return garrow_ipc_file_reader_new_raw(&arrow_file_reader);
diff --git a/c_glib/arrow-glib/ipc-file-reader.h b/c_glib/arrow-glib/ipc-file-reader.h
index 22915f8ae6e68..15eba8e35a273 100644
--- a/c_glib/arrow-glib/ipc-file-reader.h
+++ b/c_glib/arrow-glib/ipc-file-reader.h
@@ -22,7 +22,7 @@
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
 
-#include <arrow-glib/io-readable-file.h>
+#include <arrow-glib/io-random-access-file.h>
 
 #include <arrow-glib/ipc-metadata-version.h>
 
@@ -70,7 +70,7 @@ struct _GArrowIPCFileReaderClass
 
 GType garrow_ipc_file_reader_get_type(void) G_GNUC_CONST;
 
-GArrowIPCFileReader *garrow_ipc_file_reader_open(GArrowIOReadableFile *file,
+GArrowIPCFileReader *garrow_ipc_file_reader_open(GArrowIORandomAccessFile *file,
                                                  GError **error);
 
 GArrowSchema *garrow_ipc_file_reader_get_schema(GArrowIPCFileReader *file_reader);
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 9f504bec7ad53..a732e09df1269 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -121,7 +121,7 @@
       <title>Input</title>
       <xi:include href="xml/io-readable.xml"/>
       <xi:include href="xml/io-input-stream.xml"/>
-      <xi:include href="xml/io-readable-file.xml"/>
+      <xi:include href="xml/io-random-access-file.xml"/>
     </chapter>
     <chapter id="io-output">
       <title>Output</title>

From f5157a0af7046a618f159a5d0693a664f45658d7 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 18 Mar 2017 18:29:12 +0100
Subject: [PATCH 0379/1644] ARROW-648: [C++] Support multiarch on Debian

On multiarch enabled Debian, we need to install libraries into
${CMAKE_INSTALL_PREFIX}/lib/${ARCH}/ instead of
${CMAKE_INSTALL_PREFIX}/lib/.

See also: https://wiki.debian.org/Multiarch/HOWTO

Author: Kouhei Sutou <kou@clear-code.com>

Closes #398 from kou/debian-support-multiarch and squashes the following commits:

f5c8495 [Kouhei Sutou] [C++] Fix missing "${prefix}/" in .pc.in
8da48f6 [Kouhei Sutou] [C++] Support multiarch on Debian
---
 cpp/CMakeLists.txt                          | 1 +
 cpp/cmake_modules/BuildUtils.cmake          | 8 ++++----
 cpp/src/arrow/CMakeLists.txt                | 4 ++--
 cpp/src/arrow/arrow.pc.in                   | 2 +-
 cpp/src/arrow/io/CMakeLists.txt             | 2 +-
 cpp/src/arrow/io/arrow-io.pc.in             | 2 +-
 cpp/src/arrow/ipc/CMakeLists.txt            | 2 +-
 cpp/src/arrow/ipc/arrow-ipc.pc.in           | 2 +-
 cpp/src/arrow/jemalloc/CMakeLists.txt       | 2 +-
 cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in | 2 +-
 10 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5ecc34e8a5fc6..b39646ed45b0a 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -28,6 +28,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
 include(CMakeParseArguments)
 include(ExternalProject)
+include(GNUInstallDirs)
 
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 2da8a05c9c42a..9e14838cef2e1 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -68,8 +68,8 @@ function(ADD_ARROW_LIB LIB_NAME)
     endif()
   
     install(TARGETS ${LIB_NAME}_shared
-      LIBRARY DESTINATION lib
-      ARCHIVE DESTINATION lib)
+      LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+      ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
   endif()
   
   if (ARROW_BUILD_STATIC)
@@ -84,8 +84,8 @@ function(ADD_ARROW_LIB LIB_NAME)
       LINK_PRIVATE ${ARG_STATIC_PRIVATE_LINK_LIBS})
   
   install(TARGETS ${LIB_NAME}_static
-      LIBRARY DESTINATION lib
-      ARCHIVE DESTINATION lib)
+      LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+      ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
   endif()
   
   if (APPLE)
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 0abd4b9c34b0a..24a95475b14e0 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -34,7 +34,7 @@ install(FILES
   type_fwd.h
   type_traits.h
   test-util.h
-  DESTINATION include/arrow)
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow")
 
 # pkg-config support
 configure_file(arrow.pc.in
@@ -42,7 +42,7 @@ configure_file(arrow.pc.in
   @ONLY)
 install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow.pc"
-  DESTINATION "lib/pkgconfig/")
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
 
 #######################################
 # Unit tests
diff --git a/cpp/src/arrow/arrow.pc.in b/cpp/src/arrow/arrow.pc.in
index 5ad429b714893..1c3f65d661101 100644
--- a/cpp/src/arrow/arrow.pc.in
+++ b/cpp/src/arrow/arrow.pc.in
@@ -16,7 +16,7 @@
 # under the License.
 
 prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/lib
+libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
 includedir=${prefix}/include
 
 Name: Apache Arrow
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index ceb7b7379322a..69621d36f9ee1 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -123,4 +123,4 @@ configure_file(arrow-io.pc.in
   @ONLY)
 install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-io.pc"
-  DESTINATION "lib/pkgconfig/")
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
diff --git a/cpp/src/arrow/io/arrow-io.pc.in b/cpp/src/arrow/io/arrow-io.pc.in
index 4b4abdd62df42..af28aae6736fe 100644
--- a/cpp/src/arrow/io/arrow-io.pc.in
+++ b/cpp/src/arrow/io/arrow-io.pc.in
@@ -16,7 +16,7 @@
 # under the License.
 
 prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/lib
+libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
 includedir=${prefix}/include
 
 Name: Apache Arrow I/O
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 09a959ba69b51..4a5e319edf8ec 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -163,7 +163,7 @@ configure_file(arrow-ipc.pc.in
   @ONLY)
 install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-ipc.pc"
-  DESTINATION "lib/pkgconfig/")
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
 
 
 set(UTIL_LINK_LIBS
diff --git a/cpp/src/arrow/ipc/arrow-ipc.pc.in b/cpp/src/arrow/ipc/arrow-ipc.pc.in
index 73b44c99f0430..cbc226abf1ff5 100644
--- a/cpp/src/arrow/ipc/arrow-ipc.pc.in
+++ b/cpp/src/arrow/ipc/arrow-ipc.pc.in
@@ -16,7 +16,7 @@
 # under the License.
 
 prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/lib
+libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
 includedir=${prefix}/include
 
 Name: Apache Arrow IPC
diff --git a/cpp/src/arrow/jemalloc/CMakeLists.txt b/cpp/src/arrow/jemalloc/CMakeLists.txt
index 5d5482ab653bf..c7e6c6af97cff 100644
--- a/cpp/src/arrow/jemalloc/CMakeLists.txt
+++ b/cpp/src/arrow/jemalloc/CMakeLists.txt
@@ -112,4 +112,4 @@ configure_file(arrow-jemalloc.pc.in
   @ONLY)
 install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-jemalloc.pc"
-  DESTINATION "lib/pkgconfig/")
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
diff --git a/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in b/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
index 0b300fec0b2bf..18085aaf715d4 100644
--- a/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
+++ b/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
@@ -16,7 +16,7 @@
 # under the License.
 
 prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/lib
+libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
 includedir=${prefix}/include
 
 Name: Apache Arrow jemalloc-based allocator

From 5ef684003cfa268d5532564e43f2589d9fe3ca43 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 18 Mar 2017 14:54:38 -0400
Subject: [PATCH 0380/1644] ARROW-652: Remove trailing f in merge script output

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #402 from xhochy/ARROW-652 and squashes the following commits:

005488c [Uwe L. Korn] ARROW-652: Remove trailing f in merge script output
---
 dev/merge_arrow_pr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index f7e7a37c36e5c..39db254a9f25d 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -253,7 +253,7 @@ def resolve_jira(title, merge_branches, comment):
     if cur_status == "Resolved" or cur_status == "Closed":
         fail("JIRA issue %s already has status '%s'" % (jira_id, cur_status))
     print("=== JIRA %s ===" % jira_id)
-    print("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%sf\n"
+    print("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n"
           % (cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))
 
     resolve = [x for x in asf_jira.transitions(jira_id)

From 019f90d75a21f6ff7b00d657310ad1c61e5ace01 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 18 Mar 2017 16:40:37 -0400
Subject: [PATCH 0381/1644] ARROW-647: [C++] Use Boost shared libraries for
 tests and utilities

Boost shared libraries are used when ARROW_BOOST_USE_SHARED is true.
Without this change, tests and utilities use Boost static libraries
event when ARROW_BOOST_USE_SHARED.

CentOS 7 provides boost-system and boost-filesystem packages but they
include only shared libraries. They don't include static libraries.

Apache Arrow C++ requires Boost static libraries only for tests and
utilities. We can support CentOS 7 by making Boost static libraries
optional.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #400 from kou/use-boost-shared-libraries-for-tests-and-utilities and squashes the following commits:

094b69f [Kouhei Sutou] [C++] Use Boost shared libraries for tests and utilities
---
 cpp/CMakeLists.txt               | 46 ++++++++++++++++++--------------
 cpp/src/arrow/io/CMakeLists.txt  | 16 ++++-------
 cpp/src/arrow/ipc/CMakeLists.txt | 19 +++++--------
 3 files changed, 37 insertions(+), 44 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index b39646ed45b0a..197aa9c7cb636 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -410,29 +410,35 @@ endfunction()
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
-# Find static boost headers and libs
-# TODO Differentiate here between release and debug builds
 set(Boost_DEBUG TRUE)
 set(Boost_USE_MULTITHREADED ON)
-set(Boost_USE_STATIC_LIBS ON)
-find_package(Boost COMPONENTS system filesystem REQUIRED)
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-  set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-  set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
-else()
-  set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-  set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
-endif()
-
-# Find shared Boost libraries.
-set(Boost_USE_STATIC_LIBS OFF)
-find_package(Boost COMPONENTS system filesystem REQUIRED)
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-  set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-  set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+if (ARROW_BOOST_USE_SHARED)
+  # Find shared Boost libraries.
+  set(Boost_USE_STATIC_LIBS OFF)
+  find_package(Boost COMPONENTS system filesystem REQUIRED)
+  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+    set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+    set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+  else()
+    set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+    set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+  endif()
+  set(BOOST_SYSTEM_LIBRARY boost_system_shared)
+  set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 else()
-  set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-  set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+  # Find static boost headers and libs
+  # TODO Differentiate here between release and debug builds
+  set(Boost_USE_STATIC_LIBS ON)
+  find_package(Boost COMPONENTS system filesystem REQUIRED)
+  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+    set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+    set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+  else()
+    set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+    set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+  endif()
+  set(BOOST_SYSTEM_LIBRARY boost_system_static)
+  set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 endif()
 
 message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 69621d36f9ee1..af3acbf06d1ef 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -56,19 +56,13 @@ else()
   )
 endif()
 
-if (ARROW_BOOST_USE_SHARED)
-  set(ARROW_IO_SHARED_PRIVATE_LINK_LIBS
-    boost_system_shared
-    boost_filesystem_shared)
-else()
-  set(ARROW_IO_SHARED_PRIVATE_LINK_LIBS
-    boost_system_static
-    boost_filesystem_static)
-endif()
+set(ARROW_IO_SHARED_PRIVATE_LINK_LIBS
+  ${BOOST_SYSTEM_LIBRARY}
+  ${BOOST_FILESYSTEM_LIBRARY})
 
 set(ARROW_IO_STATIC_PRIVATE_LINK_LIBS
-  boost_system_static
-  boost_filesystem_static)
+  ${BOOST_SYSTEM_LIBRARY}
+  ${BOOST_FILESYSTEM_LIBRARY})
 
 set(ARROW_IO_TEST_LINK_LIBS
   arrow_io_static)
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 4a5e319edf8ec..c73af63285bcd 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -90,8 +90,8 @@ if (ARROW_BUILD_TESTS)
       arrow_static
       gflags
       gtest
-      boost_filesystem_static
-      boost_system_static
+      ${BOOST_FILESYSTEM_LIBRARY}
+      ${BOOST_SYSTEM_LIBRARY}
       dl)
     set_target_properties(json-integration-test
       PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
@@ -103,8 +103,8 @@ if (ARROW_BUILD_TESTS)
       gflags
       gtest
       pthread
-      boost_filesystem_static
-      boost_system_static
+      ${BOOST_FILESYSTEM_LIBRARY}
+      ${BOOST_SYSTEM_LIBRARY}
       dl)
   endif()
 endif()
@@ -170,17 +170,10 @@ set(UTIL_LINK_LIBS
   arrow_ipc_static
   arrow_io_static
   arrow_static
-  boost_filesystem_static
-  boost_system_static
+  ${BOOST_FILESYSTEM_LIBRARY}
+  ${BOOST_SYSTEM_LIBRARY}
   dl)
 
-if (NOT APPLE)
-  set(UTIL_LINK_LIBS
-    ${UTIL_LINK_LIBS}
-    boost_filesystem_static
-    boost_system_static)
-endif()
-
 if (ARROW_BUILD_UTILITIES)
   add_executable(file-to-stream file-to-stream.cc)
   target_link_libraries(file-to-stream ${UTIL_LINK_LIBS})

From 98c9490180aed2b24be395e80f50e7f606fadcd5 Mon Sep 17 00:00:00 2001
From: Miki Tebeka <miki.tebeka@gmail.com>
Date: Sat, 18 Mar 2017 16:47:13 -0400
Subject: [PATCH 0382/1644] ARROW-639: [C++] Invalid offset in slices

Fix incrementing offest_ twice in Slice

Author: Miki Tebeka <miki.tebeka@gmail.com>

Closes #387 from tebeka/ARROW-639 and squashes the following commits:

6520f4c [Miki Tebeka] fix lint error
95fca13 [Miki Tebeka] [ARROW-639] [C++] Invalid offset in slices
---
 cpp/src/arrow/array-string-test.cc | 14 ++++++++++++++
 cpp/src/arrow/array.h              |  4 ++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index cf2ff416032c6..ed38acd010329 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -165,6 +165,20 @@ TEST_F(TestStringArray, CompareNullByteSlots) {
       equal_array.Array::Slice(1)->RangeEquals(0, 2, 0, equal_array2.Array::Slice(1)));
 }
 
+TEST_F(TestStringArray, TestSliceGetString) {
+  StringBuilder builder(default_memory_pool());
+
+  builder.Append("a");
+  builder.Append("b");
+  builder.Append("c");
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+  auto s = array->Slice(1, 10);
+  auto arr = std::dynamic_pointer_cast<StringArray>(s);
+  ASSERT_EQ(arr->GetString(0), "b");
+}
+
 // ----------------------------------------------------------------------
 // String builder tests
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index ecc8ce540b1dd..50faf0892e8c0 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -322,8 +322,8 @@ class ARROW_EXPORT BinaryArray : public Array {
     // Account for base offset
     i += offset_;
 
-    const int32_t pos = raw_value_offsets_[i + offset_];
-    *out_length = raw_value_offsets_[i + offset_ + 1] - pos;
+    const int32_t pos = raw_value_offsets_[i];
+    *out_length = raw_value_offsets_[i + 1] - pos;
     return raw_data_ + pos;
   }
 

From a9f0c63ad9b8942f3287da2b7109d486d92731b0 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 18 Mar 2017 16:52:45 -0400
Subject: [PATCH 0383/1644] ARROW-651: [C++] Set version to shared library

.deb package builder assumes that shared libraries have version.
See also: https://www.debian.org/doc/debian-policy/ch-sharedlibs.html

Author: Kouhei Sutou <kou@clear-code.com>

Closes #401 from kou/debian-set-version-to-shared-library and squashes the following commits:

2da1442 [Kouhei Sutou] [C++] Set version to shared library
---
 cpp/CMakeLists.txt                 | 3 +++
 cpp/cmake_modules/BuildUtils.cmake | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 197aa9c7cb636..956658a82524c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -30,6 +30,9 @@ include(CMakeParseArguments)
 include(ExternalProject)
 include(GNUInstallDirs)
 
+set(ARROW_SO_VERSION "0")
+set(ARROW_ABI_VERSION "${ARROW_SO_VERSION}.0.0")
+
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 9e14838cef2e1..78b514c2295ae 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -52,7 +52,9 @@ function(ADD_ARROW_LIB LIB_NAME)
       PROPERTIES
       LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
       LINK_FLAGS "${ARG_SHARED_LINK_FLAGS}"
-      OUTPUT_NAME ${LIB_NAME})
+      OUTPUT_NAME ${LIB_NAME}
+      VERSION "${ARROW_ABI_VERSION}"
+      SOVERSION "${ARROW_SO_VERSION}")
     target_link_libraries(${LIB_NAME}_shared
       LINK_PUBLIC ${ARG_SHARED_LINK_LIBS}
       LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS})

From 4c5f79c39c2b22afe68becf1d1e7a93ca781f88d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 19 Mar 2017 01:03:47 -0400
Subject: [PATCH 0384/1644] ARROW-617: [Format] Add additional Time metadata
 and comments based on discussion in ARROW-617

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #385 from wesm/ARROW-617 and squashes the following commits:

f9f0571 [Wes McKinney] Add metadata and comments based on discussion in ARROW-617
---
 format/Message.fbs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index f2d5eba75e65b..8fdcc804d4706 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -81,16 +81,21 @@ table Decimal {
   scale: int;
 }
 
+/// Date is a 64-bit type representing milliseconds since the UNIX epoch
 table Date {
 }
 
 enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }
 
+/// Time type. The physical storage type depends on the unit
+/// - SECOND and MILLISECOND: 32 bits
+/// - MICROSECOND and NANOSECOND: 64 bits
 table Time {
   unit: TimeUnit;
+  bitWidth: int;
 }
 
-/// time from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
+/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
 table Timestamp {
   unit: TimeUnit;
 

From df2220f350282925a454ed911eed6618e4d53969 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 20 Mar 2017 10:48:34 +0100
Subject: [PATCH 0385/1644] ARROW-661: [C++] Add LargeRecordBatch metadata
 type, IPC support, associated refactoring

This patch enables the following code for writing record batches exceeding 2^31 - 1

```c++
RETURN_NOT_OK(WriteLargeRecordBatch(
    batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
return ReadLargeRecordBatch(batch.schema(), 0, mmap_.get(), result);
```

This also does a fair amount of refactoring and code consolidation related to ongoing code cleaning in arrow_ipc.

These APIs are marked experimental. This does add `LargeRecordBatch` flatbuffer type to the Message union, but I've indicated that Arrow implementations (e.g. Java) are not required to implement this type. It's strictly to enable C++ users to write very large datasets that have been embedded for convenience in Arrow's structured data model.

cc @pcmoritz @robertnishihara

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #404 from wesm/ARROW-661 and squashes the following commits:

9c18a95 [Wes McKinney] Fix import ordering
d7811f2 [Wes McKinney] cpplint
179a1e3 [Wes McKinney] Add unit test for large record batches. Use bytewise comparisons with aligned bitmaps
36c3862 [Wes McKinney] Get LargeRecordBatch round trip working. Add to Message union for now
4c1d08c [Wes McKinney] Refactoring, failing test fixture for large record batch
f4c8830 [Wes McKinney] Consolidate ipc-metadata-test and ipc-read-write-test and draft large record batch read/write path
85d1a1c [Wes McKinney] Add (untested) metadata writer for LargeRecordBatch
0f2722c [Wes McKinney] Consolidate metadata-internal.h into metadata.h. Use own Arrow structs for IPC metadata and convert to flatbuffers later
e8f8973 [Wes McKinney] Split adapter.h/cc into reader.h/writer.h. Draft LargeRecordBatch type
---
 cpp/src/arrow/allocator-test.cc          |   1 +
 cpp/src/arrow/allocator.h                |   1 +
 cpp/src/arrow/io/test-common.h           |  18 +
 cpp/src/arrow/ipc/CMakeLists.txt         |  15 +-
 cpp/src/arrow/ipc/adapter.cc             | 630 ---------------------
 cpp/src/arrow/ipc/adapter.h              | 104 ----
 cpp/src/arrow/ipc/api.h                  |   1 -
 cpp/src/arrow/ipc/ipc-adapter-test.cc    | 320 -----------
 cpp/src/arrow/ipc/ipc-file-test.cc       | 228 --------
 cpp/src/arrow/ipc/ipc-metadata-test.cc   | 100 ----
 cpp/src/arrow/ipc/ipc-read-write-test.cc | 608 ++++++++++++++++++++
 cpp/src/arrow/ipc/metadata-internal.cc   | 597 -------------------
 cpp/src/arrow/ipc/metadata-internal.h    |  83 ---
 cpp/src/arrow/ipc/metadata.cc            | 692 ++++++++++++++++++++++-
 cpp/src/arrow/ipc/metadata.h             |  40 +-
 cpp/src/arrow/ipc/reader.cc              | 171 +++++-
 cpp/src/arrow/ipc/reader.h               |  22 +
 cpp/src/arrow/ipc/test-common.h          |   2 +-
 cpp/src/arrow/ipc/writer.cc              | 544 ++++++++++++++++--
 cpp/src/arrow/ipc/writer.h               |  46 +-
 cpp/src/arrow/loader.h                   |  25 +
 cpp/src/arrow/type.h                     |   1 +
 cpp/src/arrow/util/bit-util.cc           |  16 +-
 format/Message.fbs                       |  22 +-
 24 files changed, 2131 insertions(+), 2156 deletions(-)
 delete mode 100644 cpp/src/arrow/ipc/adapter.cc
 delete mode 100644 cpp/src/arrow/ipc/adapter.h
 delete mode 100644 cpp/src/arrow/ipc/ipc-adapter-test.cc
 delete mode 100644 cpp/src/arrow/ipc/ipc-file-test.cc
 delete mode 100644 cpp/src/arrow/ipc/ipc-metadata-test.cc
 create mode 100644 cpp/src/arrow/ipc/ipc-read-write-test.cc
 delete mode 100644 cpp/src/arrow/ipc/metadata-internal.cc
 delete mode 100644 cpp/src/arrow/ipc/metadata-internal.h

diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
index 0b242674bf175..811ef5a79c2dd 100644
--- a/cpp/src/arrow/allocator-test.cc
+++ b/cpp/src/arrow/allocator-test.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include "gtest/gtest.h"
+
 #include "arrow/allocator.h"
 #include "arrow/test-util.h"
 
diff --git a/cpp/src/arrow/allocator.h b/cpp/src/arrow/allocator.h
index c976ba96b8d03..e00023dc460fb 100644
--- a/cpp/src/arrow/allocator.h
+++ b/cpp/src/arrow/allocator.h
@@ -21,6 +21,7 @@
 #include <cstddef>
 #include <memory>
 #include <utility>
+
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index 8355714540e95..4c114760e9a4b 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -41,6 +41,24 @@
 namespace arrow {
 namespace io {
 
+static inline Status ZeroMemoryMap(MemoryMappedFile* file) {
+  constexpr int64_t kBufferSize = 512;
+  static constexpr uint8_t kZeroBytes[kBufferSize] = {0};
+
+  RETURN_NOT_OK(file->Seek(0));
+  int64_t position = 0;
+  int64_t file_size;
+  RETURN_NOT_OK(file->GetSize(&file_size));
+
+  int64_t chunksize;
+  while (position < file_size) {
+    chunksize = std::min(kBufferSize, file_size - position);
+    RETURN_NOT_OK(file->Write(kZeroBytes, chunksize));
+    position += chunksize;
+  }
+  return Status::OK();
+}
+
 class MemoryMapFixture {
  public:
   void TearDown() {
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index c73af63285bcd..5d470df0309b3 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -29,12 +29,10 @@ set(ARROW_IPC_TEST_LINK_LIBS
   arrow_io_static)
 
 set(ARROW_IPC_SRCS
-  adapter.cc
   feather.cc
   json.cc
   json-internal.cc
   metadata.cc
-  metadata-internal.cc
   reader.cc
   writer.cc
 )
@@ -64,16 +62,8 @@ ADD_ARROW_TEST(feather-test)
 ARROW_TEST_LINK_LIBRARIES(feather-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
-ADD_ARROW_TEST(ipc-adapter-test)
-ARROW_TEST_LINK_LIBRARIES(ipc-adapter-test
-  ${ARROW_IPC_TEST_LINK_LIBS})
-
-ADD_ARROW_TEST(ipc-file-test)
-ARROW_TEST_LINK_LIBRARIES(ipc-file-test
-  ${ARROW_IPC_TEST_LINK_LIBS})
-
-ADD_ARROW_TEST(ipc-metadata-test)
-ARROW_TEST_LINK_LIBRARIES(ipc-metadata-test
+ADD_ARROW_TEST(ipc-read-write-test)
+ARROW_TEST_LINK_LIBRARIES(ipc-read-write-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
 ADD_ARROW_TEST(ipc-json-test)
@@ -148,7 +138,6 @@ add_dependencies(arrow_ipc_objlib metadata_fbs)
 
 # Headers: top level
 install(FILES
-  adapter.h
   api.h
   feather.h
   json.h
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
deleted file mode 100644
index db9f63ca18cbd..0000000000000
--- a/cpp/src/arrow/ipc/adapter.cc
+++ /dev/null
@@ -1,630 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/ipc/adapter.h"
-
-#include <algorithm>
-#include <cstdint>
-#include <cstring>
-#include <limits>
-#include <sstream>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/io/interfaces.h"
-#include "arrow/io/memory.h"
-#include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/metadata-internal.h"
-#include "arrow/ipc/metadata.h"
-#include "arrow/ipc/util.h"
-#include "arrow/loader.h"
-#include "arrow/memory_pool.h"
-#include "arrow/schema.h"
-#include "arrow/status.h"
-#include "arrow/table.h"
-#include "arrow/type.h"
-#include "arrow/type_fwd.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/logging.h"
-
-namespace arrow {
-
-namespace flatbuf = org::apache::arrow::flatbuf;
-
-namespace ipc {
-
-// ----------------------------------------------------------------------
-// Record batch write path
-
-class RecordBatchWriter : public ArrayVisitor {
- public:
-  RecordBatchWriter(
-      MemoryPool* pool, int64_t buffer_start_offset, int max_recursion_depth)
-      : pool_(pool),
-        max_recursion_depth_(max_recursion_depth),
-        buffer_start_offset_(buffer_start_offset) {
-    DCHECK_GT(max_recursion_depth, 0);
-  }
-
-  virtual ~RecordBatchWriter() = default;
-
-  Status VisitArray(const Array& arr) {
-    if (max_recursion_depth_ <= 0) {
-      return Status::Invalid("Max recursion depth reached");
-    }
-
-    if (arr.length() > std::numeric_limits<int32_t>::max()) {
-      return Status::Invalid("Cannot write arrays larger than 2^31 - 1 in length");
-    }
-
-    // push back all common elements
-    field_nodes_.push_back(flatbuf::FieldNode(
-        static_cast<int32_t>(arr.length()), static_cast<int32_t>(arr.null_count())));
-    if (arr.null_count() > 0) {
-      std::shared_ptr<Buffer> bitmap = arr.null_bitmap();
-
-      if (arr.offset() != 0) {
-        // With a sliced array / non-zero offset, we must copy the bitmap
-        RETURN_NOT_OK(
-            CopyBitmap(pool_, bitmap->data(), arr.offset(), arr.length(), &bitmap));
-      }
-
-      buffers_.push_back(bitmap);
-    } else {
-      // Push a dummy zero-length buffer, not to be copied
-      buffers_.push_back(std::make_shared<Buffer>(nullptr, 0));
-    }
-    return arr.Accept(this);
-  }
-
-  Status Assemble(const RecordBatch& batch, int64_t* body_length) {
-    if (field_nodes_.size() > 0) {
-      field_nodes_.clear();
-      buffer_meta_.clear();
-      buffers_.clear();
-    }
-
-    // Perform depth-first traversal of the row-batch
-    for (int i = 0; i < batch.num_columns(); ++i) {
-      RETURN_NOT_OK(VisitArray(*batch.column(i)));
-    }
-
-    // The position for the start of a buffer relative to the passed frame of
-    // reference. May be 0 or some other position in an address space
-    int64_t offset = buffer_start_offset_;
-
-    // Construct the buffer metadata for the record batch header
-    for (size_t i = 0; i < buffers_.size(); ++i) {
-      const Buffer* buffer = buffers_[i].get();
-      int64_t size = 0;
-      int64_t padding = 0;
-
-      // The buffer might be null if we are handling zero row lengths.
-      if (buffer) {
-        size = buffer->size();
-        padding = BitUtil::RoundUpToMultipleOf64(size) - size;
-      }
-
-      // TODO(wesm): We currently have no notion of shared memory page id's,
-      // but we've included it in the metadata IDL for when we have it in the
-      // future. Use page = -1 for now
-      //
-      // Note that page ids are a bespoke notion for Arrow and not a feature we
-      // are using from any OS-level shared memory. The thought is that systems
-      // may (in the future) associate integer page id's with physical memory
-      // pages (according to whatever is the desired shared memory mechanism)
-      buffer_meta_.push_back(flatbuf::Buffer(-1, offset, size + padding));
-      offset += size + padding;
-    }
-
-    *body_length = offset - buffer_start_offset_;
-    DCHECK(BitUtil::IsMultipleOf64(*body_length));
-
-    return Status::OK();
-  }
-
-  // Override this for writing dictionary metadata
-  virtual Status WriteMetadataMessage(
-      int32_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) {
-    return WriteRecordBatchMessage(
-        num_rows, body_length, field_nodes_, buffer_meta_, out);
-  }
-
-  Status WriteMetadata(int32_t num_rows, int64_t body_length, io::OutputStream* dst,
-      int32_t* metadata_length) {
-    // Now that we have computed the locations of all of the buffers in shared
-    // memory, the data header can be converted to a flatbuffer and written out
-    //
-    // Note: The memory written here is prefixed by the size of the flatbuffer
-    // itself as an int32_t.
-    std::shared_ptr<Buffer> metadata_fb;
-    RETURN_NOT_OK(WriteMetadataMessage(num_rows, body_length, &metadata_fb));
-
-    // Need to write 4 bytes (metadata size), the metadata, plus padding to
-    // end on an 8-byte offset
-    int64_t start_offset;
-    RETURN_NOT_OK(dst->Tell(&start_offset));
-
-    int32_t padded_metadata_length = static_cast<int32_t>(metadata_fb->size()) + 4;
-    const int32_t remainder =
-        (padded_metadata_length + static_cast<int32_t>(start_offset)) % 8;
-    if (remainder != 0) { padded_metadata_length += 8 - remainder; }
-
-    // The returned metadata size includes the length prefix, the flatbuffer,
-    // plus padding
-    *metadata_length = padded_metadata_length;
-
-    // Write the flatbuffer size prefix including padding
-    int32_t flatbuffer_size = padded_metadata_length - 4;
-    RETURN_NOT_OK(
-        dst->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
-
-    // Write the flatbuffer
-    RETURN_NOT_OK(dst->Write(metadata_fb->data(), metadata_fb->size()));
-
-    // Write any padding
-    int32_t padding =
-        padded_metadata_length - static_cast<int32_t>(metadata_fb->size()) - 4;
-    if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
-
-    return Status::OK();
-  }
-
-  Status Write(const RecordBatch& batch, io::OutputStream* dst, int32_t* metadata_length,
-      int64_t* body_length) {
-    RETURN_NOT_OK(Assemble(batch, body_length));
-
-#ifndef NDEBUG
-    int64_t start_position, current_position;
-    RETURN_NOT_OK(dst->Tell(&start_position));
-#endif
-
-    RETURN_NOT_OK(WriteMetadata(
-        static_cast<int32_t>(batch.num_rows()), *body_length, dst, metadata_length));
-
-#ifndef NDEBUG
-    RETURN_NOT_OK(dst->Tell(&current_position));
-    DCHECK(BitUtil::IsMultipleOf8(current_position));
-#endif
-
-    // Now write the buffers
-    for (size_t i = 0; i < buffers_.size(); ++i) {
-      const Buffer* buffer = buffers_[i].get();
-      int64_t size = 0;
-      int64_t padding = 0;
-
-      // The buffer might be null if we are handling zero row lengths.
-      if (buffer) {
-        size = buffer->size();
-        padding = BitUtil::RoundUpToMultipleOf64(size) - size;
-      }
-
-      if (size > 0) { RETURN_NOT_OK(dst->Write(buffer->data(), size)); }
-
-      if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
-    }
-
-#ifndef NDEBUG
-    RETURN_NOT_OK(dst->Tell(&current_position));
-    DCHECK(BitUtil::IsMultipleOf8(current_position));
-#endif
-
-    return Status::OK();
-  }
-
-  Status GetTotalSize(const RecordBatch& batch, int64_t* size) {
-    // emulates the behavior of Write without actually writing
-    int32_t metadata_length = 0;
-    int64_t body_length = 0;
-    MockOutputStream dst;
-    RETURN_NOT_OK(Write(batch, &dst, &metadata_length, &body_length));
-    *size = dst.GetExtentBytesWritten();
-    return Status::OK();
-  }
-
- protected:
-  template <typename ArrayType>
-  Status VisitFixedWidth(const ArrayType& array) {
-    std::shared_ptr<Buffer> data_buffer = array.data();
-
-    if (array.offset() != 0) {
-      // Non-zero offset, slice the buffer
-      const auto& fw_type = static_cast<const FixedWidthType&>(*array.type());
-      const int type_width = fw_type.bit_width() / 8;
-      const int64_t byte_offset = array.offset() * type_width;
-
-      // Send padding if it's available
-      const int64_t buffer_length =
-          std::min(BitUtil::RoundUpToMultipleOf64(array.length() * type_width),
-              data_buffer->size() - byte_offset);
-      data_buffer = SliceBuffer(data_buffer, byte_offset, buffer_length);
-    }
-    buffers_.push_back(data_buffer);
-    return Status::OK();
-  }
-
-  template <typename ArrayType>
-  Status GetZeroBasedValueOffsets(
-      const ArrayType& array, std::shared_ptr<Buffer>* value_offsets) {
-    // Share slicing logic between ListArray and BinaryArray
-
-    auto offsets = array.value_offsets();
-
-    if (array.offset() != 0) {
-      // If we have a non-zero offset, then the value offsets do not start at
-      // zero. We must a) create a new offsets array with shifted offsets and
-      // b) slice the values array accordingly
-
-      std::shared_ptr<MutableBuffer> shifted_offsets;
-      RETURN_NOT_OK(AllocateBuffer(
-          pool_, sizeof(int32_t) * (array.length() + 1), &shifted_offsets));
-
-      int32_t* dest_offsets = reinterpret_cast<int32_t*>(shifted_offsets->mutable_data());
-      const int32_t start_offset = array.value_offset(0);
-
-      for (int i = 0; i < array.length(); ++i) {
-        dest_offsets[i] = array.value_offset(i) - start_offset;
-      }
-      // Final offset
-      dest_offsets[array.length()] = array.value_offset(array.length()) - start_offset;
-      offsets = shifted_offsets;
-    }
-
-    *value_offsets = offsets;
-    return Status::OK();
-  }
-
-  Status VisitBinary(const BinaryArray& array) {
-    std::shared_ptr<Buffer> value_offsets;
-    RETURN_NOT_OK(GetZeroBasedValueOffsets<BinaryArray>(array, &value_offsets));
-    auto data = array.data();
-
-    if (array.offset() != 0) {
-      // Slice the data buffer to include only the range we need now
-      data = SliceBuffer(data, array.value_offset(0), array.value_offset(array.length()));
-    }
-
-    buffers_.push_back(value_offsets);
-    buffers_.push_back(data);
-    return Status::OK();
-  }
-
-  Status Visit(const FixedWidthBinaryArray& array) override {
-    auto data = array.data();
-    int32_t width = array.byte_width();
-
-    if (array.offset() != 0) {
-      data = SliceBuffer(data, array.offset() * width, width * array.length());
-    }
-    buffers_.push_back(data);
-    return Status::OK();
-  }
-
-  Status Visit(const BooleanArray& array) override {
-    buffers_.push_back(array.data());
-    return Status::OK();
-  }
-
-#define VISIT_FIXED_WIDTH(TYPE) \
-  Status Visit(const TYPE& array) override { return VisitFixedWidth<TYPE>(array); }
-
-  VISIT_FIXED_WIDTH(Int8Array);
-  VISIT_FIXED_WIDTH(Int16Array);
-  VISIT_FIXED_WIDTH(Int32Array);
-  VISIT_FIXED_WIDTH(Int64Array);
-  VISIT_FIXED_WIDTH(UInt8Array);
-  VISIT_FIXED_WIDTH(UInt16Array);
-  VISIT_FIXED_WIDTH(UInt32Array);
-  VISIT_FIXED_WIDTH(UInt64Array);
-  VISIT_FIXED_WIDTH(HalfFloatArray);
-  VISIT_FIXED_WIDTH(FloatArray);
-  VISIT_FIXED_WIDTH(DoubleArray);
-  VISIT_FIXED_WIDTH(DateArray);
-  VISIT_FIXED_WIDTH(Date32Array);
-  VISIT_FIXED_WIDTH(TimeArray);
-  VISIT_FIXED_WIDTH(TimestampArray);
-
-#undef VISIT_FIXED_WIDTH
-
-  Status Visit(const StringArray& array) override { return VisitBinary(array); }
-
-  Status Visit(const BinaryArray& array) override { return VisitBinary(array); }
-
-  Status Visit(const ListArray& array) override {
-    std::shared_ptr<Buffer> value_offsets;
-    RETURN_NOT_OK(GetZeroBasedValueOffsets<ListArray>(array, &value_offsets));
-    buffers_.push_back(value_offsets);
-
-    --max_recursion_depth_;
-    std::shared_ptr<Array> values = array.values();
-
-    if (array.offset() != 0) {
-      // For non-zero offset, we slice the values array accordingly
-      const int32_t offset = array.value_offset(0);
-      const int32_t length = array.value_offset(array.length()) - offset;
-      values = values->Slice(offset, length);
-    }
-    RETURN_NOT_OK(VisitArray(*values));
-    ++max_recursion_depth_;
-    return Status::OK();
-  }
-
-  Status Visit(const StructArray& array) override {
-    --max_recursion_depth_;
-    for (std::shared_ptr<Array> field : array.fields()) {
-      if (array.offset() != 0) {
-        // If offset is non-zero, slice the child array
-        field = field->Slice(array.offset(), array.length());
-      }
-      RETURN_NOT_OK(VisitArray(*field));
-    }
-    ++max_recursion_depth_;
-    return Status::OK();
-  }
-
-  Status Visit(const UnionArray& array) override {
-    auto type_ids = array.type_ids();
-    if (array.offset() != 0) {
-      type_ids = SliceBuffer(type_ids, array.offset() * sizeof(UnionArray::type_id_t),
-          array.length() * sizeof(UnionArray::type_id_t));
-    }
-
-    buffers_.push_back(type_ids);
-
-    --max_recursion_depth_;
-    if (array.mode() == UnionMode::DENSE) {
-      const auto& type = static_cast<const UnionType&>(*array.type());
-      auto value_offsets = array.value_offsets();
-
-      // The Union type codes are not necessary 0-indexed
-      uint8_t max_code = 0;
-      for (uint8_t code : type.type_codes) {
-        if (code > max_code) { max_code = code; }
-      }
-
-      // Allocate an array of child offsets. Set all to -1 to indicate that we
-      // haven't observed a first occurrence of a particular child yet
-      std::vector<int32_t> child_offsets(max_code + 1);
-      std::vector<int32_t> child_lengths(max_code + 1, 0);
-
-      if (array.offset() != 0) {
-        // This is an unpleasant case. Because the offsets are different for
-        // each child array, when we have a sliced array, we need to "rebase"
-        // the value_offsets for each array
-
-        const int32_t* unshifted_offsets = array.raw_value_offsets();
-        const uint8_t* type_ids = array.raw_type_ids();
-
-        // Allocate the shifted offsets
-        std::shared_ptr<MutableBuffer> shifted_offsets_buffer;
-        RETURN_NOT_OK(AllocateBuffer(
-            pool_, array.length() * sizeof(int32_t), &shifted_offsets_buffer));
-        int32_t* shifted_offsets =
-            reinterpret_cast<int32_t*>(shifted_offsets_buffer->mutable_data());
-
-        for (int64_t i = 0; i < array.length(); ++i) {
-          const uint8_t code = type_ids[i];
-          int32_t shift = child_offsets[code];
-          if (shift == -1) { child_offsets[code] = shift = unshifted_offsets[i]; }
-          shifted_offsets[i] = unshifted_offsets[i] - shift;
-
-          // Update the child length to account for observed value
-          ++child_lengths[code];
-        }
-
-        value_offsets = shifted_offsets_buffer;
-      }
-      buffers_.push_back(value_offsets);
-
-      // Visit children and slice accordingly
-      for (int i = 0; i < type.num_children(); ++i) {
-        std::shared_ptr<Array> child = array.child(i);
-        if (array.offset() != 0) {
-          const uint8_t code = type.type_codes[i];
-          child = child->Slice(child_offsets[code], child_lengths[code]);
-        }
-        RETURN_NOT_OK(VisitArray(*child));
-      }
-    } else {
-      for (std::shared_ptr<Array> child : array.children()) {
-        // Sparse union, slicing is simpler
-        if (array.offset() != 0) {
-          // If offset is non-zero, slice the child array
-          child = child->Slice(array.offset(), array.length());
-        }
-        RETURN_NOT_OK(VisitArray(*child));
-      }
-    }
-    ++max_recursion_depth_;
-    return Status::OK();
-  }
-
-  Status Visit(const DictionaryArray& array) override {
-    // Dictionary written out separately. Slice offset contained in the indices
-    return array.indices()->Accept(this);
-  }
-
-  // In some cases, intermediate buffers may need to be allocated (with sliced arrays)
-  MemoryPool* pool_;
-
-  std::vector<flatbuf::FieldNode> field_nodes_;
-  std::vector<flatbuf::Buffer> buffer_meta_;
-  std::vector<std::shared_ptr<Buffer>> buffers_;
-
-  int64_t max_recursion_depth_;
-  int64_t buffer_start_offset_;
-};
-
-class DictionaryWriter : public RecordBatchWriter {
- public:
-  using RecordBatchWriter::RecordBatchWriter;
-
-  Status WriteMetadataMessage(
-      int32_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
-    return WriteDictionaryMessage(
-        dictionary_id_, num_rows, body_length, field_nodes_, buffer_meta_, out);
-  }
-
-  Status Write(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
-      io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
-    dictionary_id_ = dictionary_id;
-
-    // Make a dummy record batch. A bit tedious as we have to make a schema
-    std::vector<std::shared_ptr<Field>> fields = {
-        arrow::field("dictionary", dictionary->type())};
-    auto schema = std::make_shared<Schema>(fields);
-    RecordBatch batch(schema, dictionary->length(), {dictionary});
-
-    return RecordBatchWriter::Write(batch, dst, metadata_length, body_length);
-  }
-
- private:
-  // TODO(wesm): Setting this in Write is a bit unclean, but it works
-  int64_t dictionary_id_;
-};
-
-Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
-    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth) {
-  RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
-  return writer.Write(batch, dst, metadata_length, body_length);
-}
-
-Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
-    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length, MemoryPool* pool) {
-  DictionaryWriter writer(pool, buffer_start_offset, kMaxNestingDepth);
-  return writer.Write(dictionary_id, dictionary, dst, metadata_length, body_length);
-}
-
-Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
-  RecordBatchWriter writer(default_memory_pool(), 0, kMaxNestingDepth);
-  RETURN_NOT_OK(writer.GetTotalSize(batch, size));
-  return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// Record batch read path
-
-class IpcComponentSource : public ArrayComponentSource {
- public:
-  IpcComponentSource(const RecordBatchMetadata& metadata, io::RandomAccessFile* file)
-      : metadata_(metadata), file_(file) {}
-
-  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
-    BufferMetadata buffer_meta = metadata_.buffer(buffer_index);
-    if (buffer_meta.length == 0) {
-      *out = nullptr;
-      return Status::OK();
-    } else {
-      return file_->ReadAt(buffer_meta.offset, buffer_meta.length, out);
-    }
-  }
-
-  Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
-    // pop off a field
-    if (field_index >= metadata_.num_fields()) {
-      return Status::Invalid("Ran out of field metadata, likely malformed");
-    }
-    *metadata = metadata_.field(field_index);
-    return Status::OK();
-  }
-
- private:
-  const RecordBatchMetadata& metadata_;
-  io::RandomAccessFile* file_;
-};
-
-class RecordBatchReader {
- public:
-  RecordBatchReader(const RecordBatchMetadata& metadata,
-      const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-      io::RandomAccessFile* file)
-      : metadata_(metadata),
-        schema_(schema),
-        max_recursion_depth_(max_recursion_depth),
-        file_(file) {}
-
-  Status Read(std::shared_ptr<RecordBatch>* out) {
-    std::vector<std::shared_ptr<Array>> arrays(schema_->num_fields());
-
-    IpcComponentSource source(metadata_, file_);
-    ArrayLoaderContext context;
-    context.source = &source;
-    context.field_index = 0;
-    context.buffer_index = 0;
-    context.max_recursion_depth = max_recursion_depth_;
-
-    for (int i = 0; i < schema_->num_fields(); ++i) {
-      RETURN_NOT_OK(LoadArray(schema_->field(i)->type, &context, &arrays[i]));
-    }
-
-    *out = std::make_shared<RecordBatch>(schema_, metadata_.length(), arrays);
-    return Status::OK();
-  }
-
- private:
-  const RecordBatchMetadata& metadata_;
-  std::shared_ptr<Schema> schema_;
-  int max_recursion_depth_;
-  io::RandomAccessFile* file_;
-};
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
-    std::shared_ptr<RecordBatch>* out) {
-  return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
-}
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
-  RecordBatchReader reader(metadata, schema, max_recursion_depth, file);
-  return reader.Read(out);
-}
-
-Status ReadDictionary(const DictionaryBatchMetadata& metadata,
-    const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
-    std::shared_ptr<Array>* out) {
-  int64_t id = metadata.id();
-  auto it = dictionary_types.find(id);
-  if (it == dictionary_types.end()) {
-    std::stringstream ss;
-    ss << "Do not have type metadata for dictionary with id: " << id;
-    return Status::KeyError(ss.str());
-  }
-
-  std::vector<std::shared_ptr<Field>> fields = {it->second};
-
-  // We need a schema for the record batch
-  auto dummy_schema = std::make_shared<Schema>(fields);
-
-  // The dictionary is embedded in a record batch with a single column
-  std::shared_ptr<RecordBatch> batch;
-  RETURN_NOT_OK(ReadRecordBatch(metadata.record_batch(), dummy_schema, file, &batch));
-
-  if (batch->num_columns() != 1) {
-    return Status::Invalid("Dictionary record batch must only contain one field");
-  }
-
-  *out = batch->column(0);
-  return Status::OK();
-}
-
-}  // namespace ipc
-}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
deleted file mode 100644
index cea4686077486..0000000000000
--- a/cpp/src/arrow/ipc/adapter.h
+++ /dev/null
@@ -1,104 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Public API for writing and accessing (with zero copy, if possible) Arrow
-// IPC binary formatted data (e.g. in shared memory, or from some other IO source)
-
-#ifndef ARROW_IPC_ADAPTER_H
-#define ARROW_IPC_ADAPTER_H
-
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include "arrow/ipc/metadata.h"
-#include "arrow/loader.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Array;
-class MemoryPool;
-class RecordBatch;
-class Schema;
-class Status;
-
-namespace io {
-
-class RandomAccessFile;
-class OutputStream;
-
-}  // namespace io
-
-namespace ipc {
-
-// ----------------------------------------------------------------------
-// Write path
-
-// Write the RecordBatch (collection of equal-length Arrow arrays) to the
-// output stream in a contiguous block. The record batch metadata is written as
-// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
-// prefixed by its size, followed by each of the memory buffers in the batch
-// written end to end (with appropriate alignment and padding):
-//
-// <int32: metadata size> <uint8*: metadata> <buffers>
-//
-// Finally, the absolute offsets (relative to the start of the output stream)
-// to the end of the body and end of the metadata / data header (suffixed by
-// the header size) is returned in out-variables
-//
-// @param(in) buffer_start_offset: the start offset to use in the buffer metadata,
-// default should be 0
-//
-// @param(out) metadata_length: the size of the length-prefixed flatbuffer
-// including padding to a 64-byte boundary
-//
-// @param(out) body_length: the size of the contiguous buffer block plus
-// padding bytes
-Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
-    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
-
-// Write Array as a DictionaryBatch message
-Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
-    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length, MemoryPool* pool);
-
-// Compute the precise number of bytes needed in a contiguous memory segment to
-// write the record batch. This involves generating the complete serialized
-// Flatbuffers metadata.
-Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
-
-// ----------------------------------------------------------------------
-// "Read" path; does not copy data if the input supports zero copy reads
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
-    std::shared_ptr<RecordBatch>* out);
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
-
-Status ReadDictionary(const DictionaryBatchMetadata& metadata,
-    const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
-    std::shared_ptr<Array>* out);
-
-}  // namespace ipc
-}  // namespace arrow
-
-#endif  // ARROW_IPC_MEMORY_H
diff --git a/cpp/src/arrow/ipc/api.h b/cpp/src/arrow/ipc/api.h
index ad7cd84e9f986..3f05e69d5843d 100644
--- a/cpp/src/arrow/ipc/api.h
+++ b/cpp/src/arrow/ipc/api.h
@@ -18,7 +18,6 @@
 #ifndef ARROW_IPC_API_H
 #define ARROW_IPC_API_H
 
-#include "arrow/ipc/adapter.h"
 #include "arrow/ipc/feather.h"
 #include "arrow/ipc/json.h"
 #include "arrow/ipc/metadata.h"
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
deleted file mode 100644
index 638d98af8244d..0000000000000
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ /dev/null
@@ -1,320 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdio>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/io/memory.h"
-#include "arrow/io/test-common.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/metadata.h"
-#include "arrow/ipc/test-common.h"
-#include "arrow/ipc/util.h"
-
-#include "arrow/buffer.h"
-#include "arrow/memory_pool.h"
-#include "arrow/pretty_print.h"
-#include "arrow/status.h"
-#include "arrow/test-util.h"
-#include "arrow/util/bit-util.h"
-
-namespace arrow {
-namespace ipc {
-
-class IpcTestFixture : public io::MemoryMapFixture {
- public:
-  Status RoundTripHelper(const RecordBatch& batch, int memory_map_size,
-      std::shared_ptr<RecordBatch>* batch_result) {
-    std::string path = "test-write-row-batch";
-    io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
-
-    int32_t metadata_length;
-    int64_t body_length;
-
-    const int64_t buffer_offset = 0;
-
-    RETURN_NOT_OK(WriteRecordBatch(
-        batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
-
-    std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
-    auto metadata = std::make_shared<RecordBatchMetadata>(message);
-
-    // The buffer offsets start at 0, so we must construct a
-    // RandomAccessFile according to that frame of reference
-    std::shared_ptr<Buffer> buffer_payload;
-    RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
-    io::BufferReader buffer_reader(buffer_payload);
-
-    return ReadRecordBatch(*metadata, batch.schema(), &buffer_reader, batch_result);
-  }
-
-  void CheckRoundtrip(const RecordBatch& batch, int64_t buffer_size) {
-    std::shared_ptr<RecordBatch> batch_result;
-
-    ASSERT_OK(RoundTripHelper(batch, 1 << 16, &batch_result));
-    EXPECT_EQ(batch.num_rows(), batch_result->num_rows());
-
-    ASSERT_TRUE(batch.schema()->Equals(batch_result->schema()));
-    ASSERT_EQ(batch.num_columns(), batch_result->num_columns())
-        << batch.schema()->ToString()
-        << " result: " << batch_result->schema()->ToString();
-
-    for (int i = 0; i < batch.num_columns(); ++i) {
-      const auto& left = *batch.column(i);
-      const auto& right = *batch_result->column(i);
-      if (!left.Equals(right)) {
-        std::stringstream pp_result;
-        std::stringstream pp_expected;
-
-        ASSERT_OK(PrettyPrint(left, 0, &pp_expected));
-        ASSERT_OK(PrettyPrint(right, 0, &pp_result));
-
-        FAIL() << "Index: " << i << " Expected: " << pp_expected.str()
-               << "\nGot: " << pp_result.str();
-      }
-    }
-  }
-
-  void CheckRoundtrip(const std::shared_ptr<Array>& array, int64_t buffer_size) {
-    auto f0 = arrow::field("f0", array->type());
-    std::vector<std::shared_ptr<Field>> fields = {f0};
-    auto schema = std::make_shared<Schema>(fields);
-
-    RecordBatch batch(schema, 0, {array});
-    CheckRoundtrip(batch, buffer_size);
-  }
-
- protected:
-  std::shared_ptr<io::MemoryMappedFile> mmap_;
-  MemoryPool* pool_;
-};
-
-class TestWriteRecordBatch : public ::testing::Test, public IpcTestFixture {
- public:
-  void SetUp() { pool_ = default_memory_pool(); }
-  void TearDown() { io::MemoryMapFixture::TearDown(); }
-};
-
-class TestRecordBatchParam : public ::testing::TestWithParam<MakeRecordBatch*>,
-                             public IpcTestFixture {
- public:
-  void SetUp() { pool_ = default_memory_pool(); }
-  void TearDown() { io::MemoryMapFixture::TearDown(); }
-  using IpcTestFixture::RoundTripHelper;
-  using IpcTestFixture::CheckRoundtrip;
-};
-
-TEST_P(TestRecordBatchParam, RoundTrip) {
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
-
-  CheckRoundtrip(*batch, 1 << 20);
-}
-
-TEST_P(TestRecordBatchParam, SliceRoundTrip) {
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
-
-  // Skip the zero-length case
-  if (batch->num_rows() < 2) { return; }
-
-  auto sliced_batch = batch->Slice(2, 10);
-  CheckRoundtrip(*sliced_batch, 1 << 20);
-}
-
-TEST_P(TestRecordBatchParam, ZeroLengthArrays) {
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
-
-  std::shared_ptr<RecordBatch> zero_length_batch;
-  if (batch->num_rows() > 2) {
-    zero_length_batch = batch->Slice(2, 0);
-  } else {
-    zero_length_batch = batch->Slice(0, 0);
-  }
-
-  CheckRoundtrip(*zero_length_batch, 1 << 20);
-
-  // ARROW-544: check binary array
-  std::shared_ptr<MutableBuffer> value_offsets;
-  ASSERT_OK(AllocateBuffer(pool_, sizeof(int32_t), &value_offsets));
-  *reinterpret_cast<int32_t*>(value_offsets->mutable_data()) = 0;
-
-  std::shared_ptr<Array> bin_array = std::make_shared<BinaryArray>(0, value_offsets,
-      std::make_shared<Buffer>(nullptr, 0), std::make_shared<Buffer>(nullptr, 0));
-
-  // null value_offsets
-  std::shared_ptr<Array> bin_array2 = std::make_shared<BinaryArray>(0, nullptr, nullptr);
-
-  CheckRoundtrip(bin_array, 1 << 20);
-  CheckRoundtrip(bin_array2, 1 << 20);
-}
-
-INSTANTIATE_TEST_CASE_P(
-    RoundTripTests, TestRecordBatchParam,
-    ::testing::Values(&MakeIntRecordBatch, &MakeStringTypesRecordBatch,
-        &MakeNonNullRecordBatch, &MakeZeroLengthRecordBatch, &MakeListRecordBatch,
-        &MakeDeeplyNestedList, &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate,
-        &MakeTimestamps, &MakeTimes, &MakeFWBinary));
-
-void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
-  ipc::MockOutputStream mock;
-  int32_t mock_metadata_length = -1;
-  int64_t mock_body_length = -1;
-  int64_t size = -1;
-  ASSERT_OK(WriteRecordBatch(
-      *batch, 0, &mock, &mock_metadata_length, &mock_body_length, default_memory_pool()));
-  ASSERT_OK(GetRecordBatchSize(*batch, &size));
-  ASSERT_EQ(mock.GetExtentBytesWritten(), size);
-}
-
-TEST_F(TestWriteRecordBatch, IntegerGetRecordBatchSize) {
-  std::shared_ptr<RecordBatch> batch;
-
-  ASSERT_OK(MakeIntRecordBatch(&batch));
-  TestGetRecordBatchSize(batch);
-
-  ASSERT_OK(MakeListRecordBatch(&batch));
-  TestGetRecordBatchSize(batch);
-
-  ASSERT_OK(MakeZeroLengthRecordBatch(&batch));
-  TestGetRecordBatchSize(batch);
-
-  ASSERT_OK(MakeNonNullRecordBatch(&batch));
-  TestGetRecordBatchSize(batch);
-
-  ASSERT_OK(MakeDeeplyNestedList(&batch));
-  TestGetRecordBatchSize(batch);
-}
-
-class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
- public:
-  void SetUp() { pool_ = default_memory_pool(); }
-  void TearDown() { io::MemoryMapFixture::TearDown(); }
-
-  Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
-      int64_t* body_length, std::shared_ptr<RecordBatch>* batch,
-      std::shared_ptr<Schema>* schema) {
-    const int batch_length = 5;
-    TypePtr type = int32();
-    std::shared_ptr<Array> array;
-    const bool include_nulls = true;
-    RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
-    for (int i = 0; i < recursion_level; ++i) {
-      type = list(type);
-      RETURN_NOT_OK(
-          MakeRandomListArray(array, batch_length, include_nulls, pool_, &array));
-    }
-
-    auto f0 = field("f0", type);
-
-    *schema = std::shared_ptr<Schema>(new Schema({f0}));
-
-    std::vector<std::shared_ptr<Array>> arrays = {array};
-    *batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
-
-    std::string path = "test-write-past-max-recursion";
-    const int memory_map_size = 1 << 20;
-    io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
-
-    if (override_level) {
-      return WriteRecordBatch(**batch, 0, mmap_.get(), metadata_length, body_length,
-          pool_, recursion_level + 1);
-    } else {
-      return WriteRecordBatch(
-          **batch, 0, mmap_.get(), metadata_length, body_length, pool_);
-    }
-  }
-
- protected:
-  std::shared_ptr<io::MemoryMappedFile> mmap_;
-  MemoryPool* pool_;
-};
-
-TEST_F(RecursionLimits, WriteLimit) {
-  int32_t metadata_length = -1;
-  int64_t body_length = -1;
-  std::shared_ptr<Schema> schema;
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_RAISES(Invalid,
-      WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &batch, &schema));
-}
-
-TEST_F(RecursionLimits, ReadLimit) {
-  int32_t metadata_length = -1;
-  int64_t body_length = -1;
-  std::shared_ptr<Schema> schema;
-
-  const int recursion_depth = 64;
-
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK(WriteToMmap(
-      recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
-
-  std::shared_ptr<Message> message;
-  ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
-  auto metadata = std::make_shared<RecordBatchMetadata>(message);
-
-  std::shared_ptr<Buffer> payload;
-  ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
-
-  io::BufferReader reader(payload);
-
-  std::shared_ptr<RecordBatch> result;
-  ASSERT_RAISES(Invalid, ReadRecordBatch(*metadata, schema, &reader, &result));
-}
-
-TEST_F(RecursionLimits, StressLimit) {
-  auto CheckDepth = [this](int recursion_depth, bool* it_works) {
-    int32_t metadata_length = -1;
-    int64_t body_length = -1;
-    std::shared_ptr<Schema> schema;
-    std::shared_ptr<RecordBatch> batch;
-    ASSERT_OK(WriteToMmap(
-        recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
-
-    std::shared_ptr<Message> message;
-    ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
-    auto metadata = std::make_shared<RecordBatchMetadata>(message);
-
-    std::shared_ptr<Buffer> payload;
-    ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
-
-    io::BufferReader reader(payload);
-
-    std::shared_ptr<RecordBatch> result;
-    ASSERT_OK(ReadRecordBatch(*metadata, schema, recursion_depth + 1, &reader, &result));
-    *it_works = result->Equals(*batch);
-  };
-
-  bool it_works = false;
-  CheckDepth(100, &it_works);
-  ASSERT_TRUE(it_works);
-
-  CheckDepth(500, &it_works);
-  ASSERT_TRUE(it_works);
-}
-
-}  // namespace ipc
-}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
deleted file mode 100644
index b45782220e478..0000000000000
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ /dev/null
@@ -1,228 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdio>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/io/memory.h"
-#include "arrow/io/test-common.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/reader.h"
-#include "arrow/ipc/test-common.h"
-#include "arrow/ipc/util.h"
-#include "arrow/ipc/writer.h"
-
-#include "arrow/buffer.h"
-#include "arrow/memory_pool.h"
-#include "arrow/status.h"
-#include "arrow/test-util.h"
-#include "arrow/util/bit-util.h"
-
-namespace arrow {
-namespace ipc {
-
-void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
-  if (!left.schema()->Equals(right.schema())) {
-    FAIL() << "Left schema: " << left.schema()->ToString()
-           << "\nRight schema: " << right.schema()->ToString();
-  }
-  ASSERT_EQ(left.num_columns(), right.num_columns())
-      << left.schema()->ToString() << " result: " << right.schema()->ToString();
-  EXPECT_EQ(left.num_rows(), right.num_rows());
-  for (int i = 0; i < left.num_columns(); ++i) {
-    EXPECT_TRUE(left.column(i)->Equals(right.column(i)))
-        << "Idx: " << i << " Name: " << left.column_name(i);
-  }
-}
-
-using BatchVector = std::vector<std::shared_ptr<RecordBatch>>;
-
-class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
- public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-    buffer_ = std::make_shared<PoolBuffer>(pool_);
-    sink_.reset(new io::BufferOutputStream(buffer_));
-  }
-  void TearDown() {}
-
-  Status RoundTripHelper(const BatchVector& in_batches, BatchVector* out_batches) {
-    // Write the file
-    std::shared_ptr<FileWriter> writer;
-    RETURN_NOT_OK(FileWriter::Open(sink_.get(), in_batches[0]->schema(), &writer));
-
-    const int num_batches = static_cast<int>(in_batches.size());
-
-    for (const auto& batch : in_batches) {
-      RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
-    }
-    RETURN_NOT_OK(writer->Close());
-    RETURN_NOT_OK(sink_->Close());
-
-    // Current offset into stream is the end of the file
-    int64_t footer_offset;
-    RETURN_NOT_OK(sink_->Tell(&footer_offset));
-
-    // Open the file
-    auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
-    std::shared_ptr<FileReader> reader;
-    RETURN_NOT_OK(FileReader::Open(buf_reader, footer_offset, &reader));
-
-    EXPECT_EQ(num_batches, reader->num_record_batches());
-    for (int i = 0; i < num_batches; ++i) {
-      std::shared_ptr<RecordBatch> chunk;
-      RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
-      out_batches->emplace_back(chunk);
-    }
-
-    return Status::OK();
-  }
-
- protected:
-  MemoryPool* pool_;
-
-  std::unique_ptr<io::BufferOutputStream> sink_;
-  std::shared_ptr<PoolBuffer> buffer_;
-};
-
-TEST_P(TestFileFormat, RoundTrip) {
-  std::shared_ptr<RecordBatch> batch1;
-  std::shared_ptr<RecordBatch> batch2;
-  ASSERT_OK((*GetParam())(&batch1));  // NOLINT clang-tidy gtest issue
-  ASSERT_OK((*GetParam())(&batch2));  // NOLINT clang-tidy gtest issue
-
-  std::vector<std::shared_ptr<RecordBatch>> in_batches = {batch1, batch2};
-  std::vector<std::shared_ptr<RecordBatch>> out_batches;
-
-  ASSERT_OK(RoundTripHelper(in_batches, &out_batches));
-
-  // Compare batches
-  for (size_t i = 0; i < in_batches.size(); ++i) {
-    CompareBatch(*in_batches[i], *out_batches[i]);
-  }
-}
-
-class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
- public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-    buffer_ = std::make_shared<PoolBuffer>(pool_);
-    sink_.reset(new io::BufferOutputStream(buffer_));
-  }
-  void TearDown() {}
-
-  Status RoundTripHelper(
-      const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
-    // Write the file
-    std::shared_ptr<StreamWriter> writer;
-    RETURN_NOT_OK(StreamWriter::Open(sink_.get(), batch.schema(), &writer));
-    int num_batches = 5;
-    for (int i = 0; i < num_batches; ++i) {
-      RETURN_NOT_OK(writer->WriteRecordBatch(batch));
-    }
-    RETURN_NOT_OK(writer->Close());
-    RETURN_NOT_OK(sink_->Close());
-
-    // Open the file
-    auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
-
-    std::shared_ptr<StreamReader> reader;
-    RETURN_NOT_OK(StreamReader::Open(buf_reader, &reader));
-
-    std::shared_ptr<RecordBatch> chunk;
-    while (true) {
-      RETURN_NOT_OK(reader->GetNextRecordBatch(&chunk));
-      if (chunk == nullptr) { break; }
-      out_batches->emplace_back(chunk);
-    }
-    return Status::OK();
-  }
-
- protected:
-  MemoryPool* pool_;
-
-  std::unique_ptr<io::BufferOutputStream> sink_;
-  std::shared_ptr<PoolBuffer> buffer_;
-};
-
-TEST_P(TestStreamFormat, RoundTrip) {
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
-
-  std::vector<std::shared_ptr<RecordBatch>> out_batches;
-
-  ASSERT_OK(RoundTripHelper(*batch, &out_batches));
-
-  // Compare batches. Same
-  for (size_t i = 0; i < out_batches.size(); ++i) {
-    CompareBatch(*batch, *out_batches[i]);
-  }
-}
-
-#define BATCH_CASES()                                                                   \
-  ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
-      &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
-      &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate, &MakeTimestamps, &MakeTimes, \
-      &MakeFWBinary);
-
-INSTANTIATE_TEST_CASE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
-INSTANTIATE_TEST_CASE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
-
-void CheckBatchDictionaries(const RecordBatch& batch) {
-  // Check that dictionaries that should be the same are the same
-  auto schema = batch.schema();
-
-  const auto& t0 = static_cast<const DictionaryType&>(*schema->field(0)->type);
-  const auto& t1 = static_cast<const DictionaryType&>(*schema->field(1)->type);
-
-  ASSERT_EQ(t0.dictionary().get(), t1.dictionary().get());
-
-  // Same dictionary used for list values
-  const auto& t3 = static_cast<const ListType&>(*schema->field(3)->type);
-  const auto& t3_value = static_cast<const DictionaryType&>(*t3.value_type());
-  ASSERT_EQ(t0.dictionary().get(), t3_value.dictionary().get());
-}
-
-TEST_F(TestStreamFormat, DictionaryRoundTrip) {
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK(MakeDictionary(&batch));
-
-  std::vector<std::shared_ptr<RecordBatch>> out_batches;
-  ASSERT_OK(RoundTripHelper(*batch, &out_batches));
-
-  CheckBatchDictionaries(*out_batches[0]);
-}
-
-TEST_F(TestFileFormat, DictionaryRoundTrip) {
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK(MakeDictionary(&batch));
-
-  std::vector<std::shared_ptr<RecordBatch>> out_batches;
-  ASSERT_OK(RoundTripHelper({batch}, &out_batches));
-
-  CheckBatchDictionaries(*out_batches[0]);
-}
-
-}  // namespace ipc
-}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
deleted file mode 100644
index 4fb3204a5b6d2..0000000000000
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ /dev/null
@@ -1,100 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <memory>
-#include <sstream>
-#include <string>
-
-#include "gtest/gtest.h"
-
-#include "arrow/io/memory.h"
-#include "arrow/ipc/metadata-internal.h"
-#include "arrow/ipc/metadata.h"
-#include "arrow/ipc/test-common.h"
-#include "arrow/schema.h"
-#include "arrow/status.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-class Buffer;
-
-namespace ipc {
-
-class TestSchemaMetadata : public ::testing::Test {
- public:
-  void SetUp() {}
-
-  void CheckRoundtrip(const Schema& schema, DictionaryMemo* memo) {
-    std::shared_ptr<Buffer> buffer;
-    ASSERT_OK(WriteSchemaMessage(schema, memo, &buffer));
-
-    std::shared_ptr<Message> message;
-    ASSERT_OK(Message::Open(buffer, 0, &message));
-
-    ASSERT_EQ(Message::SCHEMA, message->type());
-
-    auto schema_msg = std::make_shared<SchemaMetadata>(message);
-    ASSERT_EQ(schema.num_fields(), schema_msg->num_fields());
-
-    DictionaryMemo empty_memo;
-
-    std::shared_ptr<Schema> schema2;
-    ASSERT_OK(schema_msg->GetSchema(empty_memo, &schema2));
-
-    AssertSchemaEqual(schema, *schema2);
-  }
-};
-
-const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
-
-TEST_F(TestSchemaMetadata, PrimitiveFields) {
-  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>(), false);
-  auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
-  auto f3 = std::make_shared<Field>("f3", std::make_shared<Int64Type>());
-  auto f4 = std::make_shared<Field>("f4", std::make_shared<UInt8Type>());
-  auto f5 = std::make_shared<Field>("f5", std::make_shared<UInt16Type>());
-  auto f6 = std::make_shared<Field>("f6", std::make_shared<UInt32Type>());
-  auto f7 = std::make_shared<Field>("f7", std::make_shared<UInt64Type>());
-  auto f8 = std::make_shared<Field>("f8", std::make_shared<FloatType>());
-  auto f9 = std::make_shared<Field>("f9", std::make_shared<DoubleType>(), false);
-  auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
-
-  Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
-  DictionaryMemo memo;
-
-  CheckRoundtrip(schema, &memo);
-}
-
-TEST_F(TestSchemaMetadata, NestedFields) {
-  auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
-  auto f0 = std::make_shared<Field>("f0", type);
-
-  std::shared_ptr<StructType> type2(new StructType({std::make_shared<Field>("k1", INT32),
-      std::make_shared<Field>("k2", INT32), std::make_shared<Field>("k3", INT32)}));
-  auto f1 = std::make_shared<Field>("f1", type2);
-
-  Schema schema({f0, f1});
-  DictionaryMemo memo;
-
-  CheckRoundtrip(schema, &memo);
-}
-
-}  // namespace ipc
-}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
new file mode 100644
index 0000000000000..261ca1d0e52d8
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -0,0 +1,608 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/io/memory.h"
+#include "arrow/io/test-common.h"
+#include "arrow/ipc/api.h"
+#include "arrow/ipc/test-common.h"
+#include "arrow/ipc/util.h"
+
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/pretty_print.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+#include "arrow/util/bit-util.h"
+
+namespace arrow {
+namespace ipc {
+
+void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
+  if (!left.schema()->Equals(right.schema())) {
+    FAIL() << "Left schema: " << left.schema()->ToString()
+           << "\nRight schema: " << right.schema()->ToString();
+  }
+  ASSERT_EQ(left.num_columns(), right.num_columns())
+      << left.schema()->ToString() << " result: " << right.schema()->ToString();
+  EXPECT_EQ(left.num_rows(), right.num_rows());
+  for (int i = 0; i < left.num_columns(); ++i) {
+    EXPECT_TRUE(left.column(i)->Equals(right.column(i)))
+        << "Idx: " << i << " Name: " << left.column_name(i);
+  }
+}
+
+using BatchVector = std::vector<std::shared_ptr<RecordBatch>>;
+
+class TestSchemaMetadata : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  void CheckRoundtrip(const Schema& schema, DictionaryMemo* memo) {
+    std::shared_ptr<Buffer> buffer;
+    ASSERT_OK(WriteSchemaMessage(schema, memo, &buffer));
+
+    std::shared_ptr<Message> message;
+    ASSERT_OK(Message::Open(buffer, 0, &message));
+
+    ASSERT_EQ(Message::SCHEMA, message->type());
+
+    auto schema_msg = std::make_shared<SchemaMetadata>(message);
+    ASSERT_EQ(schema.num_fields(), schema_msg->num_fields());
+
+    DictionaryMemo empty_memo;
+
+    std::shared_ptr<Schema> schema2;
+    ASSERT_OK(schema_msg->GetSchema(empty_memo, &schema2));
+
+    AssertSchemaEqual(schema, *schema2);
+  }
+};
+
+const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
+
+TEST_F(TestSchemaMetadata, PrimitiveFields) {
+  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
+  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>(), false);
+  auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
+  auto f3 = std::make_shared<Field>("f3", std::make_shared<Int64Type>());
+  auto f4 = std::make_shared<Field>("f4", std::make_shared<UInt8Type>());
+  auto f5 = std::make_shared<Field>("f5", std::make_shared<UInt16Type>());
+  auto f6 = std::make_shared<Field>("f6", std::make_shared<UInt32Type>());
+  auto f7 = std::make_shared<Field>("f7", std::make_shared<UInt64Type>());
+  auto f8 = std::make_shared<Field>("f8", std::make_shared<FloatType>());
+  auto f9 = std::make_shared<Field>("f9", std::make_shared<DoubleType>(), false);
+  auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
+
+  Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
+  DictionaryMemo memo;
+
+  CheckRoundtrip(schema, &memo);
+}
+
+TEST_F(TestSchemaMetadata, NestedFields) {
+  auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
+  auto f0 = std::make_shared<Field>("f0", type);
+
+  std::shared_ptr<StructType> type2(new StructType({std::make_shared<Field>("k1", INT32),
+      std::make_shared<Field>("k2", INT32), std::make_shared<Field>("k3", INT32)}));
+  auto f1 = std::make_shared<Field>("f1", type2);
+
+  Schema schema({f0, f1});
+  DictionaryMemo memo;
+
+  CheckRoundtrip(schema, &memo);
+}
+
+#define BATCH_CASES()                                                                   \
+  ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
+      &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
+      &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate, &MakeTimestamps, &MakeTimes, \
+      &MakeFWBinary);
+
+class IpcTestFixture : public io::MemoryMapFixture {
+ public:
+  Status DoStandardRoundTrip(const RecordBatch& batch, bool zero_data,
+      std::shared_ptr<RecordBatch>* batch_result) {
+    int32_t metadata_length;
+    int64_t body_length;
+
+    const int64_t buffer_offset = 0;
+
+    if (zero_data) { RETURN_NOT_OK(ZeroMemoryMap(mmap_.get())); }
+    RETURN_NOT_OK(mmap_->Seek(0));
+
+    RETURN_NOT_OK(WriteRecordBatch(
+        batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
+
+    std::shared_ptr<Message> message;
+    RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+    auto metadata = std::make_shared<RecordBatchMetadata>(message);
+
+    // The buffer offsets start at 0, so we must construct a
+    // RandomAccessFile according to that frame of reference
+    std::shared_ptr<Buffer> buffer_payload;
+    RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
+    io::BufferReader buffer_reader(buffer_payload);
+
+    return ReadRecordBatch(*metadata, batch.schema(), &buffer_reader, batch_result);
+  }
+
+  Status DoLargeRoundTrip(
+      const RecordBatch& batch, bool zero_data, std::shared_ptr<RecordBatch>* result) {
+    int32_t metadata_length;
+    int64_t body_length;
+
+    const int64_t buffer_offset = 0;
+
+    if (zero_data) { RETURN_NOT_OK(ZeroMemoryMap(mmap_.get())); }
+    RETURN_NOT_OK(mmap_->Seek(0));
+
+    RETURN_NOT_OK(WriteLargeRecordBatch(
+        batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
+    return ReadLargeRecordBatch(batch.schema(), 0, mmap_.get(), result);
+  }
+
+  void CheckReadResult(const RecordBatch& result, const RecordBatch& expected) {
+    EXPECT_EQ(expected.num_rows(), result.num_rows());
+
+    ASSERT_TRUE(expected.schema()->Equals(result.schema()));
+    ASSERT_EQ(expected.num_columns(), result.num_columns())
+        << expected.schema()->ToString() << " result: " << result.schema()->ToString();
+
+    for (int i = 0; i < expected.num_columns(); ++i) {
+      const auto& left = *expected.column(i);
+      const auto& right = *result.column(i);
+      if (!left.Equals(right)) {
+        std::stringstream pp_result;
+        std::stringstream pp_expected;
+
+        ASSERT_OK(PrettyPrint(left, 0, &pp_expected));
+        ASSERT_OK(PrettyPrint(right, 0, &pp_result));
+
+        FAIL() << "Index: " << i << " Expected: " << pp_expected.str()
+               << "\nGot: " << pp_result.str();
+      }
+    }
+  }
+
+  void CheckRoundtrip(const RecordBatch& batch, int64_t buffer_size) {
+    std::string path = "test-write-row-batch";
+    ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(buffer_size, path, &mmap_));
+
+    std::shared_ptr<RecordBatch> result;
+    ASSERT_OK(DoStandardRoundTrip(batch, true, &result));
+    CheckReadResult(*result, batch);
+
+    ASSERT_OK(DoLargeRoundTrip(batch, true, &result));
+    CheckReadResult(*result, batch);
+  }
+
+  void CheckRoundtrip(const std::shared_ptr<Array>& array, int64_t buffer_size) {
+    auto f0 = arrow::field("f0", array->type());
+    std::vector<std::shared_ptr<Field>> fields = {f0};
+    auto schema = std::make_shared<Schema>(fields);
+
+    RecordBatch batch(schema, 0, {array});
+    CheckRoundtrip(batch, buffer_size);
+  }
+
+ protected:
+  std::shared_ptr<io::MemoryMappedFile> mmap_;
+  MemoryPool* pool_;
+};
+
+class TestWriteRecordBatch : public ::testing::Test, public IpcTestFixture {
+ public:
+  void SetUp() { pool_ = default_memory_pool(); }
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
+};
+
+class TestIpcRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*>,
+                         public IpcTestFixture {
+ public:
+  void SetUp() { pool_ = default_memory_pool(); }
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
+};
+
+TEST_P(TestIpcRoundTrip, RoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
+
+  CheckRoundtrip(*batch, 1 << 20);
+}
+
+TEST_P(TestIpcRoundTrip, SliceRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
+
+  // Skip the zero-length case
+  if (batch->num_rows() < 2) { return; }
+
+  auto sliced_batch = batch->Slice(2, 10);
+  CheckRoundtrip(*sliced_batch, 1 << 20);
+}
+
+TEST_P(TestIpcRoundTrip, ZeroLengthArrays) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
+
+  std::shared_ptr<RecordBatch> zero_length_batch;
+  if (batch->num_rows() > 2) {
+    zero_length_batch = batch->Slice(2, 0);
+  } else {
+    zero_length_batch = batch->Slice(0, 0);
+  }
+
+  CheckRoundtrip(*zero_length_batch, 1 << 20);
+
+  // ARROW-544: check binary array
+  std::shared_ptr<MutableBuffer> value_offsets;
+  ASSERT_OK(AllocateBuffer(pool_, sizeof(int32_t), &value_offsets));
+  *reinterpret_cast<int32_t*>(value_offsets->mutable_data()) = 0;
+
+  std::shared_ptr<Array> bin_array = std::make_shared<BinaryArray>(0, value_offsets,
+      std::make_shared<Buffer>(nullptr, 0), std::make_shared<Buffer>(nullptr, 0));
+
+  // null value_offsets
+  std::shared_ptr<Array> bin_array2 = std::make_shared<BinaryArray>(0, nullptr, nullptr);
+
+  CheckRoundtrip(bin_array, 1 << 20);
+  CheckRoundtrip(bin_array2, 1 << 20);
+}
+
+void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
+  ipc::MockOutputStream mock;
+  int32_t mock_metadata_length = -1;
+  int64_t mock_body_length = -1;
+  int64_t size = -1;
+  ASSERT_OK(WriteRecordBatch(
+      *batch, 0, &mock, &mock_metadata_length, &mock_body_length, default_memory_pool()));
+  ASSERT_OK(GetRecordBatchSize(*batch, &size));
+  ASSERT_EQ(mock.GetExtentBytesWritten(), size);
+}
+
+TEST_F(TestWriteRecordBatch, IntegerGetRecordBatchSize) {
+  std::shared_ptr<RecordBatch> batch;
+
+  ASSERT_OK(MakeIntRecordBatch(&batch));
+  TestGetRecordBatchSize(batch);
+
+  ASSERT_OK(MakeListRecordBatch(&batch));
+  TestGetRecordBatchSize(batch);
+
+  ASSERT_OK(MakeZeroLengthRecordBatch(&batch));
+  TestGetRecordBatchSize(batch);
+
+  ASSERT_OK(MakeNonNullRecordBatch(&batch));
+  TestGetRecordBatchSize(batch);
+
+  ASSERT_OK(MakeDeeplyNestedList(&batch));
+  TestGetRecordBatchSize(batch);
+}
+
+class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
+ public:
+  void SetUp() { pool_ = default_memory_pool(); }
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
+
+  Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
+      int64_t* body_length, std::shared_ptr<RecordBatch>* batch,
+      std::shared_ptr<Schema>* schema) {
+    const int batch_length = 5;
+    TypePtr type = int32();
+    std::shared_ptr<Array> array;
+    const bool include_nulls = true;
+    RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
+    for (int i = 0; i < recursion_level; ++i) {
+      type = list(type);
+      RETURN_NOT_OK(
+          MakeRandomListArray(array, batch_length, include_nulls, pool_, &array));
+    }
+
+    auto f0 = field("f0", type);
+
+    *schema = std::shared_ptr<Schema>(new Schema({f0}));
+
+    std::vector<std::shared_ptr<Array>> arrays = {array};
+    *batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
+
+    std::string path = "test-write-past-max-recursion";
+    const int memory_map_size = 1 << 20;
+    io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
+
+    if (override_level) {
+      return WriteRecordBatch(**batch, 0, mmap_.get(), metadata_length, body_length,
+          pool_, recursion_level + 1);
+    } else {
+      return WriteRecordBatch(
+          **batch, 0, mmap_.get(), metadata_length, body_length, pool_);
+    }
+  }
+
+ protected:
+  std::shared_ptr<io::MemoryMappedFile> mmap_;
+  MemoryPool* pool_;
+};
+
+TEST_F(RecursionLimits, WriteLimit) {
+  int32_t metadata_length = -1;
+  int64_t body_length = -1;
+  std::shared_ptr<Schema> schema;
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_RAISES(Invalid,
+      WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &batch, &schema));
+}
+
+TEST_F(RecursionLimits, ReadLimit) {
+  int32_t metadata_length = -1;
+  int64_t body_length = -1;
+  std::shared_ptr<Schema> schema;
+
+  const int recursion_depth = 64;
+
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(WriteToMmap(
+      recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
+
+  std::shared_ptr<Message> message;
+  ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+  auto metadata = std::make_shared<RecordBatchMetadata>(message);
+
+  std::shared_ptr<Buffer> payload;
+  ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
+
+  io::BufferReader reader(payload);
+
+  std::shared_ptr<RecordBatch> result;
+  ASSERT_RAISES(Invalid, ReadRecordBatch(*metadata, schema, &reader, &result));
+}
+
+TEST_F(RecursionLimits, StressLimit) {
+  auto CheckDepth = [this](int recursion_depth, bool* it_works) {
+    int32_t metadata_length = -1;
+    int64_t body_length = -1;
+    std::shared_ptr<Schema> schema;
+    std::shared_ptr<RecordBatch> batch;
+    ASSERT_OK(WriteToMmap(
+        recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
+
+    std::shared_ptr<Message> message;
+    ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+    auto metadata = std::make_shared<RecordBatchMetadata>(message);
+
+    std::shared_ptr<Buffer> payload;
+    ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
+
+    io::BufferReader reader(payload);
+
+    std::shared_ptr<RecordBatch> result;
+    ASSERT_OK(ReadRecordBatch(*metadata, schema, recursion_depth + 1, &reader, &result));
+    *it_works = result->Equals(*batch);
+  };
+
+  bool it_works = false;
+  CheckDepth(100, &it_works);
+  ASSERT_TRUE(it_works);
+
+  CheckDepth(500, &it_works);
+  ASSERT_TRUE(it_works);
+}
+
+class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+    buffer_ = std::make_shared<PoolBuffer>(pool_);
+    sink_.reset(new io::BufferOutputStream(buffer_));
+  }
+  void TearDown() {}
+
+  Status RoundTripHelper(const BatchVector& in_batches, BatchVector* out_batches) {
+    // Write the file
+    std::shared_ptr<FileWriter> writer;
+    RETURN_NOT_OK(FileWriter::Open(sink_.get(), in_batches[0]->schema(), &writer));
+
+    const int num_batches = static_cast<int>(in_batches.size());
+
+    for (const auto& batch : in_batches) {
+      RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
+    }
+    RETURN_NOT_OK(writer->Close());
+    RETURN_NOT_OK(sink_->Close());
+
+    // Current offset into stream is the end of the file
+    int64_t footer_offset;
+    RETURN_NOT_OK(sink_->Tell(&footer_offset));
+
+    // Open the file
+    auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+    std::shared_ptr<FileReader> reader;
+    RETURN_NOT_OK(FileReader::Open(buf_reader, footer_offset, &reader));
+
+    EXPECT_EQ(num_batches, reader->num_record_batches());
+    for (int i = 0; i < num_batches; ++i) {
+      std::shared_ptr<RecordBatch> chunk;
+      RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
+      out_batches->emplace_back(chunk);
+    }
+
+    return Status::OK();
+  }
+
+ protected:
+  MemoryPool* pool_;
+
+  std::unique_ptr<io::BufferOutputStream> sink_;
+  std::shared_ptr<PoolBuffer> buffer_;
+};
+
+TEST_P(TestFileFormat, RoundTrip) {
+  std::shared_ptr<RecordBatch> batch1;
+  std::shared_ptr<RecordBatch> batch2;
+  ASSERT_OK((*GetParam())(&batch1));  // NOLINT clang-tidy gtest issue
+  ASSERT_OK((*GetParam())(&batch2));  // NOLINT clang-tidy gtest issue
+
+  std::vector<std::shared_ptr<RecordBatch>> in_batches = {batch1, batch2};
+  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+
+  ASSERT_OK(RoundTripHelper(in_batches, &out_batches));
+
+  // Compare batches
+  for (size_t i = 0; i < in_batches.size(); ++i) {
+    CompareBatch(*in_batches[i], *out_batches[i]);
+  }
+}
+
+class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+    buffer_ = std::make_shared<PoolBuffer>(pool_);
+    sink_.reset(new io::BufferOutputStream(buffer_));
+  }
+  void TearDown() {}
+
+  Status RoundTripHelper(
+      const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
+    // Write the file
+    std::shared_ptr<StreamWriter> writer;
+    RETURN_NOT_OK(StreamWriter::Open(sink_.get(), batch.schema(), &writer));
+    int num_batches = 5;
+    for (int i = 0; i < num_batches; ++i) {
+      RETURN_NOT_OK(writer->WriteRecordBatch(batch));
+    }
+    RETURN_NOT_OK(writer->Close());
+    RETURN_NOT_OK(sink_->Close());
+
+    // Open the file
+    auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+
+    std::shared_ptr<StreamReader> reader;
+    RETURN_NOT_OK(StreamReader::Open(buf_reader, &reader));
+
+    std::shared_ptr<RecordBatch> chunk;
+    while (true) {
+      RETURN_NOT_OK(reader->GetNextRecordBatch(&chunk));
+      if (chunk == nullptr) { break; }
+      out_batches->emplace_back(chunk);
+    }
+    return Status::OK();
+  }
+
+ protected:
+  MemoryPool* pool_;
+
+  std::unique_ptr<io::BufferOutputStream> sink_;
+  std::shared_ptr<PoolBuffer> buffer_;
+};
+
+TEST_P(TestStreamFormat, RoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
+
+  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+
+  ASSERT_OK(RoundTripHelper(*batch, &out_batches));
+
+  // Compare batches. Same
+  for (size_t i = 0; i < out_batches.size(); ++i) {
+    CompareBatch(*batch, *out_batches[i]);
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(GenericIpcRoundTripTests, TestIpcRoundTrip, BATCH_CASES());
+INSTANTIATE_TEST_CASE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
+INSTANTIATE_TEST_CASE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
+
+TEST_F(TestIpcRoundTrip, LargeRecordBatch) {
+  const int64_t length = static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1;
+
+  BooleanBuilder builder(default_memory_pool());
+  ASSERT_OK(builder.Reserve(length));
+  ASSERT_OK(builder.Advance(length));
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  auto f0 = arrow::field("f0", array->type());
+  std::vector<std::shared_ptr<Field>> fields = {f0};
+  auto schema = std::make_shared<Schema>(fields);
+
+  RecordBatch batch(schema, 0, {array});
+
+  std::string path = "test-write-large-record_batch";
+
+  // 512 MB
+  constexpr int64_t kBufferSize = 1 << 29;
+
+  ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_));
+
+  std::shared_ptr<RecordBatch> result;
+  ASSERT_OK(DoLargeRoundTrip(batch, false, &result));
+  CheckReadResult(*result, batch);
+
+  // Fails if we try to write this with the normal code path
+  ASSERT_RAISES(Invalid, DoStandardRoundTrip(batch, false, &result));
+}
+
+void CheckBatchDictionaries(const RecordBatch& batch) {
+  // Check that dictionaries that should be the same are the same
+  auto schema = batch.schema();
+
+  const auto& t0 = static_cast<const DictionaryType&>(*schema->field(0)->type);
+  const auto& t1 = static_cast<const DictionaryType&>(*schema->field(1)->type);
+
+  ASSERT_EQ(t0.dictionary().get(), t1.dictionary().get());
+
+  // Same dictionary used for list values
+  const auto& t3 = static_cast<const ListType&>(*schema->field(3)->type);
+  const auto& t3_value = static_cast<const DictionaryType&>(*t3.value_type());
+  ASSERT_EQ(t0.dictionary().get(), t3_value.dictionary().get());
+}
+
+TEST_F(TestStreamFormat, DictionaryRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeDictionary(&batch));
+
+  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+  ASSERT_OK(RoundTripHelper(*batch, &out_batches));
+
+  CheckBatchDictionaries(*out_batches[0]);
+}
+
+TEST_F(TestFileFormat, DictionaryRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeDictionary(&batch));
+
+  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+  ASSERT_OK(RoundTripHelper({batch}, &out_batches));
+
+  CheckBatchDictionaries(*out_batches[0]);
+}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
deleted file mode 100644
index be0d282f21bbf..0000000000000
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ /dev/null
@@ -1,597 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/ipc/metadata-internal.h"
-
-#include <cstdint>
-#include <cstring>
-#include <memory>
-#include <sstream>
-#include <string>
-
-#include "flatbuffers/flatbuffers.h"
-
-#include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/ipc/Message_generated.h"
-#include "arrow/schema.h"
-#include "arrow/status.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-namespace flatbuf = org::apache::arrow::flatbuf;
-
-namespace ipc {
-
-static Status IntFromFlatbuffer(
-    const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
-  if (int_data->bitWidth() > 64) {
-    return Status::NotImplemented("Integers with more than 64 bits not implemented");
-  }
-  if (int_data->bitWidth() < 8) {
-    return Status::NotImplemented("Integers with less than 8 bits not implemented");
-  }
-
-  switch (int_data->bitWidth()) {
-    case 8:
-      *out = int_data->is_signed() ? int8() : uint8();
-      break;
-    case 16:
-      *out = int_data->is_signed() ? int16() : uint16();
-      break;
-    case 32:
-      *out = int_data->is_signed() ? int32() : uint32();
-      break;
-    case 64:
-      *out = int_data->is_signed() ? int64() : uint64();
-      break;
-    default:
-      return Status::NotImplemented("Integers not in cstdint are not implemented");
-  }
-  return Status::OK();
-}
-
-static Status FloatFromFlatuffer(
-    const flatbuf::FloatingPoint* float_data, std::shared_ptr<DataType>* out) {
-  if (float_data->precision() == flatbuf::Precision_HALF) {
-    *out = float16();
-  } else if (float_data->precision() == flatbuf::Precision_SINGLE) {
-    *out = float32();
-  } else {
-    *out = float64();
-  }
-  return Status::OK();
-}
-
-// Forward declaration
-static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
-    DictionaryMemo* dictionary_memo, FieldOffset* offset);
-
-static Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
-  return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
-}
-
-static Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
-  return flatbuf::CreateFloatingPoint(fbb, precision).Union();
-}
-
-static Status AppendChildFields(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo) {
-  FieldOffset field;
-  for (int i = 0; i < type->num_children(); ++i) {
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), dictionary_memo, &field));
-    out_children->push_back(field);
-  }
-  return Status::OK();
-}
-
-static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
-    Offset* offset) {
-  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
-  *offset = flatbuf::CreateList(fbb).Union();
-  return Status::OK();
-}
-
-static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
-    Offset* offset) {
-  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
-  *offset = flatbuf::CreateStruct_(fbb).Union();
-  return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// Union implementation
-
-static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
-    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
-  UnionMode mode = union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE
-                                                                   : UnionMode::DENSE;
-
-  std::vector<uint8_t> type_codes;
-
-  const flatbuffers::Vector<int32_t>* fb_type_ids = union_data->typeIds();
-  if (fb_type_ids == nullptr) {
-    for (uint8_t i = 0; i < children.size(); ++i) {
-      type_codes.push_back(i);
-    }
-  } else {
-    for (int32_t id : (*fb_type_ids)) {
-      // TODO(wesm): can these values exceed 255?
-      type_codes.push_back(static_cast<uint8_t>(id));
-    }
-  }
-
-  *out = union_(children, type_codes, mode);
-  return Status::OK();
-}
-
-static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
-    Offset* offset) {
-  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
-
-  const auto& union_type = static_cast<const UnionType&>(*type);
-
-  flatbuf::UnionMode mode = union_type.mode == UnionMode::SPARSE
-                                ? flatbuf::UnionMode_Sparse
-                                : flatbuf::UnionMode_Dense;
-
-  std::vector<int32_t> type_ids;
-  type_ids.reserve(union_type.type_codes.size());
-  for (uint8_t code : union_type.type_codes) {
-    type_ids.push_back(code);
-  }
-
-  auto fb_type_ids = fbb.CreateVector(type_ids);
-
-  *offset = flatbuf::CreateUnion(fbb, mode, fb_type_ids).Union();
-  return Status::OK();
-}
-
-#define INT_TO_FB_CASE(BIT_WIDTH, IS_SIGNED)            \
-  *out_type = flatbuf::Type_Int;                        \
-  *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
-  break;
-
-static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) {
-  switch (unit) {
-    case TimeUnit::SECOND:
-      return flatbuf::TimeUnit_SECOND;
-    case TimeUnit::MILLI:
-      return flatbuf::TimeUnit_MILLISECOND;
-    case TimeUnit::MICRO:
-      return flatbuf::TimeUnit_MICROSECOND;
-    case TimeUnit::NANO:
-      return flatbuf::TimeUnit_NANOSECOND;
-    default:
-      break;
-  }
-  return flatbuf::TimeUnit_MIN;
-}
-
-static inline TimeUnit FromFlatbufferUnit(flatbuf::TimeUnit unit) {
-  switch (unit) {
-    case flatbuf::TimeUnit_SECOND:
-      return TimeUnit::SECOND;
-    case flatbuf::TimeUnit_MILLISECOND:
-      return TimeUnit::MILLI;
-    case flatbuf::TimeUnit_MICROSECOND:
-      return TimeUnit::MICRO;
-    case flatbuf::TimeUnit_NANOSECOND:
-      return TimeUnit::NANO;
-    default:
-      break;
-  }
-  // cannot reach
-  return TimeUnit::SECOND;
-}
-
-static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
-    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
-  switch (type) {
-    case flatbuf::Type_NONE:
-      return Status::Invalid("Type metadata cannot be none");
-    case flatbuf::Type_Int:
-      return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
-    case flatbuf::Type_FloatingPoint:
-      return FloatFromFlatuffer(
-          static_cast<const flatbuf::FloatingPoint*>(type_data), out);
-    case flatbuf::Type_Binary:
-      *out = binary();
-      return Status::OK();
-    case flatbuf::Type_FixedWidthBinary: {
-      auto fw_binary = static_cast<const flatbuf::FixedWidthBinary*>(type_data);
-      *out = fixed_width_binary(fw_binary->byteWidth());
-      return Status::OK();
-    }
-    case flatbuf::Type_Utf8:
-      *out = utf8();
-      return Status::OK();
-    case flatbuf::Type_Bool:
-      *out = boolean();
-      return Status::OK();
-    case flatbuf::Type_Decimal:
-      return Status::NotImplemented("Decimal");
-    case flatbuf::Type_Date:
-      *out = date();
-      return Status::OK();
-    case flatbuf::Type_Time: {
-      auto time_type = static_cast<const flatbuf::Time*>(type_data);
-      *out = time(FromFlatbufferUnit(time_type->unit()));
-      return Status::OK();
-    }
-    case flatbuf::Type_Timestamp: {
-      auto ts_type = static_cast<const flatbuf::Timestamp*>(type_data);
-      *out = timestamp(FromFlatbufferUnit(ts_type->unit()));
-      return Status::OK();
-    }
-    case flatbuf::Type_Interval:
-      return Status::NotImplemented("Interval");
-    case flatbuf::Type_List:
-      if (children.size() != 1) {
-        return Status::Invalid("List must have exactly 1 child field");
-      }
-      *out = std::make_shared<ListType>(children[0]);
-      return Status::OK();
-    case flatbuf::Type_Struct_:
-      *out = std::make_shared<StructType>(children);
-      return Status::OK();
-    case flatbuf::Type_Union:
-      return UnionFromFlatbuffer(
-          static_cast<const flatbuf::Union*>(type_data), children, out);
-    default:
-      return Status::Invalid("Unrecognized type");
-  }
-}
-
-// TODO(wesm): Convert this to visitor pattern
-static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
-    flatbuf::Type* out_type, DictionaryMemo* dictionary_memo, Offset* offset) {
-  if (type->type == Type::DICTIONARY) {
-    // In this library, the dictionary "type" is a logical construct. Here we
-    // pass through to the value type, as we've already captured the index
-    // type in the DictionaryEncoding metadata in the parent field
-    const auto& dict_type = static_cast<const DictionaryType&>(*type);
-    return TypeToFlatbuffer(fbb, dict_type.dictionary()->type(), children, layout,
-        out_type, dictionary_memo, offset);
-  }
-
-  std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
-  for (const BufferDescr& descr : buffer_layout) {
-    flatbuf::VectorType vector_type;
-    switch (descr.type()) {
-      case BufferType::OFFSET:
-        vector_type = flatbuf::VectorType_OFFSET;
-        break;
-      case BufferType::DATA:
-        vector_type = flatbuf::VectorType_DATA;
-        break;
-      case BufferType::VALIDITY:
-        vector_type = flatbuf::VectorType_VALIDITY;
-        break;
-      case BufferType::TYPE:
-        vector_type = flatbuf::VectorType_TYPE;
-        break;
-      default:
-        vector_type = flatbuf::VectorType_DATA;
-        break;
-    }
-    auto offset = flatbuf::CreateVectorLayout(
-        fbb, static_cast<int16_t>(descr.bit_width()), vector_type);
-    layout->push_back(offset);
-  }
-
-  switch (type->type) {
-    case Type::BOOL:
-      *out_type = flatbuf::Type_Bool;
-      *offset = flatbuf::CreateBool(fbb).Union();
-      break;
-    case Type::UINT8:
-      INT_TO_FB_CASE(8, false);
-    case Type::INT8:
-      INT_TO_FB_CASE(8, true);
-    case Type::UINT16:
-      INT_TO_FB_CASE(16, false);
-    case Type::INT16:
-      INT_TO_FB_CASE(16, true);
-    case Type::UINT32:
-      INT_TO_FB_CASE(32, false);
-    case Type::INT32:
-      INT_TO_FB_CASE(32, true);
-    case Type::UINT64:
-      INT_TO_FB_CASE(64, false);
-    case Type::INT64:
-      INT_TO_FB_CASE(64, true);
-    case Type::FLOAT:
-      *out_type = flatbuf::Type_FloatingPoint;
-      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_SINGLE);
-      break;
-    case Type::DOUBLE:
-      *out_type = flatbuf::Type_FloatingPoint;
-      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
-      break;
-    case Type::FIXED_WIDTH_BINARY: {
-      const auto& fw_type = static_cast<const FixedWidthBinaryType&>(*type);
-      *out_type = flatbuf::Type_FixedWidthBinary;
-      *offset = flatbuf::CreateFixedWidthBinary(fbb, fw_type.byte_width()).Union();
-    } break;
-    case Type::BINARY:
-      *out_type = flatbuf::Type_Binary;
-      *offset = flatbuf::CreateBinary(fbb).Union();
-      break;
-    case Type::STRING:
-      *out_type = flatbuf::Type_Utf8;
-      *offset = flatbuf::CreateUtf8(fbb).Union();
-      break;
-    case Type::DATE:
-      *out_type = flatbuf::Type_Date;
-      *offset = flatbuf::CreateDate(fbb).Union();
-      break;
-    case Type::TIME: {
-      const auto& time_type = static_cast<const TimeType&>(*type);
-      *out_type = flatbuf::Type_Time;
-      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
-    } break;
-    case Type::TIMESTAMP: {
-      const auto& ts_type = static_cast<const TimestampType&>(*type);
-      *out_type = flatbuf::Type_Timestamp;
-      *offset = flatbuf::CreateTimestamp(fbb, ToFlatbufferUnit(ts_type.unit)).Union();
-    } break;
-    case Type::LIST:
-      *out_type = flatbuf::Type_List;
-      return ListToFlatbuffer(fbb, type, children, dictionary_memo, offset);
-    case Type::STRUCT:
-      *out_type = flatbuf::Type_Struct_;
-      return StructToFlatbuffer(fbb, type, children, dictionary_memo, offset);
-    case Type::UNION:
-      *out_type = flatbuf::Type_Union;
-      return UnionToFlatBuffer(fbb, type, children, dictionary_memo, offset);
-    default:
-      *out_type = flatbuf::Type_NONE;  // Make clang-tidy happy
-      std::stringstream ss;
-      ss << "Unable to convert type: " << type->ToString() << std::endl;
-      return Status::NotImplemented(ss.str());
-  }
-  return Status::OK();
-}
-
-using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
-
-static DictionaryOffset GetDictionaryEncoding(
-    FBB& fbb, const DictionaryType& type, DictionaryMemo* memo) {
-  int64_t dictionary_id = memo->GetId(type.dictionary());
-
-  // We assume that the dictionary index type (as an integer) has already been
-  // validated elsewhere, and can safely assume we are dealing with signed
-  // integers
-  const auto& fw_index_type = static_cast<const FixedWidthType&>(*type.index_type());
-
-  auto index_type_offset = flatbuf::CreateInt(fbb, fw_index_type.bit_width(), true);
-
-  // TODO(wesm): ordered dictionaries
-  return flatbuf::CreateDictionaryEncoding(fbb, dictionary_id, index_type_offset);
-}
-
-static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
-    DictionaryMemo* dictionary_memo, FieldOffset* offset) {
-  auto fb_name = fbb.CreateString(field->name);
-
-  flatbuf::Type type_enum;
-  Offset type_offset;
-  Offset type_layout;
-  std::vector<FieldOffset> children;
-  std::vector<VectorLayoutOffset> layout;
-
-  RETURN_NOT_OK(TypeToFlatbuffer(
-      fbb, field->type, &children, &layout, &type_enum, dictionary_memo, &type_offset));
-  auto fb_children = fbb.CreateVector(children);
-  auto fb_layout = fbb.CreateVector(layout);
-
-  DictionaryOffset dictionary = 0;
-  if (field->type->type == Type::DICTIONARY) {
-    dictionary = GetDictionaryEncoding(
-        fbb, static_cast<const DictionaryType&>(*field->type), dictionary_memo);
-  }
-
-  // TODO: produce the list of VectorTypes
-  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_offset,
-      dictionary, fb_children, fb_layout);
-
-  return Status::OK();
-}
-
-Status FieldFromFlatbufferDictionary(
-    const flatbuf::Field* field, std::shared_ptr<Field>* out) {
-  // Need an empty memo to pass down for constructing children
-  DictionaryMemo dummy_memo;
-
-  // Any DictionaryEncoding set is ignored here
-
-  std::shared_ptr<DataType> type;
-  auto children = field->children();
-  std::vector<std::shared_ptr<Field>> child_fields(children->size());
-  for (int i = 0; i < static_cast<int>(children->size()); ++i) {
-    RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), dummy_memo, &child_fields[i]));
-  }
-
-  RETURN_NOT_OK(
-      TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
-
-  *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
-  return Status::OK();
-}
-
-Status FieldFromFlatbuffer(const flatbuf::Field* field,
-    const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out) {
-  std::shared_ptr<DataType> type;
-
-  const flatbuf::DictionaryEncoding* encoding = field->dictionary();
-
-  if (encoding == nullptr) {
-    // The field is not dictionary encoded. We must potentially visit its
-    // children to fully reconstruct the data type
-    auto children = field->children();
-    std::vector<std::shared_ptr<Field>> child_fields(children->size());
-    for (int i = 0; i < static_cast<int>(children->size()); ++i) {
-      RETURN_NOT_OK(
-          FieldFromFlatbuffer(children->Get(i), dictionary_memo, &child_fields[i]));
-    }
-    RETURN_NOT_OK(
-        TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
-  } else {
-    // The field is dictionary encoded. The type of the dictionary values has
-    // been determined elsewhere, and is stored in the DictionaryMemo. Here we
-    // construct the logical DictionaryType object
-
-    std::shared_ptr<Array> dictionary;
-    RETURN_NOT_OK(dictionary_memo.GetDictionary(encoding->id(), &dictionary));
-
-    std::shared_ptr<DataType> index_type;
-    RETURN_NOT_OK(IntFromFlatbuffer(encoding->indexType(), &index_type));
-    type = std::make_shared<DictionaryType>(index_type, dictionary);
-  }
-  *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
-  return Status::OK();
-}
-
-// Implement MessageBuilder
-
-// will return the endianness of the system we are running on
-// based the NUMPY_API function. See NOTICE.txt
-flatbuf::Endianness endianness() {
-  union {
-    uint32_t i;
-    char c[4];
-  } bint = {0x01020304};
-
-  return bint.c[0] == 1 ? flatbuf::Endianness_Big : flatbuf::Endianness_Little;
-}
-
-Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema, DictionaryMemo* dictionary_memo,
-    flatbuffers::Offset<flatbuf::Schema>* out) {
-  std::vector<FieldOffset> field_offsets;
-  for (int i = 0; i < schema.num_fields(); ++i) {
-    std::shared_ptr<Field> field = schema.field(i);
-    FieldOffset offset;
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, dictionary_memo, &offset));
-    field_offsets.push_back(offset);
-  }
-
-  *out = flatbuf::CreateSchema(fbb, endianness(), fbb.CreateVector(field_offsets));
-  return Status::OK();
-}
-
-class MessageBuilder {
- public:
-  Status SetSchema(const Schema& schema, DictionaryMemo* dictionary_memo) {
-    flatbuffers::Offset<flatbuf::Schema> fb_schema;
-    RETURN_NOT_OK(SchemaToFlatbuffer(fbb_, schema, dictionary_memo, &fb_schema));
-
-    header_type_ = flatbuf::MessageHeader_Schema;
-    header_ = fb_schema.Union();
-    body_length_ = 0;
-    return Status::OK();
-  }
-
-  Status SetRecordBatch(int32_t length, int64_t body_length,
-      const std::vector<flatbuf::FieldNode>& nodes,
-      const std::vector<flatbuf::Buffer>& buffers) {
-    header_type_ = flatbuf::MessageHeader_RecordBatch;
-    header_ = flatbuf::CreateRecordBatch(fbb_, length, fbb_.CreateVectorOfStructs(nodes),
-                  fbb_.CreateVectorOfStructs(buffers))
-                  .Union();
-    body_length_ = body_length;
-
-    return Status::OK();
-  }
-
-  Status SetDictionary(int64_t id, int32_t length, int64_t body_length,
-      const std::vector<flatbuf::FieldNode>& nodes,
-      const std::vector<flatbuf::Buffer>& buffers) {
-    header_type_ = flatbuf::MessageHeader_DictionaryBatch;
-
-    auto record_batch = flatbuf::CreateRecordBatch(fbb_, length,
-        fbb_.CreateVectorOfStructs(nodes), fbb_.CreateVectorOfStructs(buffers));
-
-    header_ = flatbuf::CreateDictionaryBatch(fbb_, id, record_batch).Union();
-    body_length_ = body_length;
-    return Status::OK();
-  }
-
-  Status Finish();
-
-  Status GetBuffer(std::shared_ptr<Buffer>* out);
-
- private:
-  flatbuf::MessageHeader header_type_;
-  flatbuffers::Offset<void> header_;
-  int64_t body_length_;
-  flatbuffers::FlatBufferBuilder fbb_;
-};
-
-Status WriteSchemaMessage(
-    const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out) {
-  MessageBuilder message;
-  RETURN_NOT_OK(message.SetSchema(schema, dictionary_memo));
-  RETURN_NOT_OK(message.Finish());
-  return message.GetBuffer(out);
-}
-
-Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
-    const std::vector<flatbuf::FieldNode>& nodes,
-    const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
-  MessageBuilder builder;
-  RETURN_NOT_OK(builder.SetRecordBatch(length, body_length, nodes, buffers));
-  RETURN_NOT_OK(builder.Finish());
-  return builder.GetBuffer(out);
-}
-
-Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
-    const std::vector<flatbuf::FieldNode>& nodes,
-    const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
-  MessageBuilder builder;
-  RETURN_NOT_OK(builder.SetDictionary(id, length, body_length, nodes, buffers));
-  RETURN_NOT_OK(builder.Finish());
-  return builder.GetBuffer(out);
-}
-
-Status MessageBuilder::Finish() {
-  auto message =
-      flatbuf::CreateMessage(fbb_, kMetadataVersion, header_type_, header_, body_length_);
-  fbb_.Finish(message);
-  return Status::OK();
-}
-
-Status MessageBuilder::GetBuffer(std::shared_ptr<Buffer>* out) {
-  int32_t size = fbb_.GetSize();
-
-  auto result = std::make_shared<PoolBuffer>();
-  RETURN_NOT_OK(result->Resize(size));
-
-  uint8_t* dst = result->mutable_data();
-  memcpy(dst, fbb_.GetBufferPointer(), size);
-
-  *out = result;
-  return Status::OK();
-}
-
-}  // namespace ipc
-}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
deleted file mode 100644
index 59afecbcbd27e..0000000000000
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_IPC_METADATA_INTERNAL_H
-#define ARROW_IPC_METADATA_INTERNAL_H
-
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include "flatbuffers/flatbuffers.h"
-
-#include "arrow/ipc/File_generated.h"
-#include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/metadata.h"
-
-namespace arrow {
-
-namespace flatbuf = org::apache::arrow::flatbuf;
-
-class Buffer;
-struct Field;
-class Schema;
-class Status;
-
-namespace ipc {
-
-using FBB = flatbuffers::FlatBufferBuilder;
-using FieldOffset = flatbuffers::Offset<arrow::flatbuf::Field>;
-using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
-using Offset = flatbuffers::Offset<void>;
-
-static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
-
-// Construct a field with type for a dictionary-encoded field. None of its
-// children or children's descendents can be dictionary encoded
-Status FieldFromFlatbufferDictionary(
-    const flatbuf::Field* field, std::shared_ptr<Field>* out);
-
-// Construct a field for a non-dictionary-encoded field. Its children may be
-// dictionary encoded
-Status FieldFromFlatbuffer(const flatbuf::Field* field,
-    const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out);
-
-Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema, DictionaryMemo* dictionary_memo,
-    flatbuffers::Offset<flatbuf::Schema>* out);
-
-// Serialize arrow::Schema as a Flatbuffer
-//
-// \param[in] schema a Schema instance
-// \param[inout] dictionary_memo class for tracking dictionaries and assigning
-// dictionary ids
-// \param[out] out the serialized arrow::Buffer
-// \return Status outcome
-Status WriteSchemaMessage(
-    const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out);
-
-Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
-    const std::vector<flatbuf::FieldNode>& nodes,
-    const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
-
-Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
-    const std::vector<flatbuf::FieldNode>& nodes,
-    const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
-
-}  // namespace ipc
-}  // namespace arrow
-
-#endif  // ARROW_IPC_METADATA_INTERNAL_H
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 71bc5c9eb3207..a418d4893dd40 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -24,14 +24,14 @@
 
 #include "flatbuffers/flatbuffers.h"
 
+#include "arrow/array.h"
+#include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/ipc/File_generated.h"
 #include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/metadata-internal.h"
-
-#include "arrow/buffer.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
+#include "arrow/type.h"
 
 namespace arrow {
 
@@ -39,6 +39,643 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
+using FBB = flatbuffers::FlatBufferBuilder;
+using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
+using FieldOffset = flatbuffers::Offset<flatbuf::Field>;
+using LargeRecordBatchOffset = flatbuffers::Offset<flatbuf::LargeRecordBatch>;
+using RecordBatchOffset = flatbuffers::Offset<flatbuf::RecordBatch>;
+using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
+using Offset = flatbuffers::Offset<void>;
+
+static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
+
+static Status IntFromFlatbuffer(
+    const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
+  if (int_data->bitWidth() > 64) {
+    return Status::NotImplemented("Integers with more than 64 bits not implemented");
+  }
+  if (int_data->bitWidth() < 8) {
+    return Status::NotImplemented("Integers with less than 8 bits not implemented");
+  }
+
+  switch (int_data->bitWidth()) {
+    case 8:
+      *out = int_data->is_signed() ? int8() : uint8();
+      break;
+    case 16:
+      *out = int_data->is_signed() ? int16() : uint16();
+      break;
+    case 32:
+      *out = int_data->is_signed() ? int32() : uint32();
+      break;
+    case 64:
+      *out = int_data->is_signed() ? int64() : uint64();
+      break;
+    default:
+      return Status::NotImplemented("Integers not in cstdint are not implemented");
+  }
+  return Status::OK();
+}
+
+static Status FloatFromFlatuffer(
+    const flatbuf::FloatingPoint* float_data, std::shared_ptr<DataType>* out) {
+  if (float_data->precision() == flatbuf::Precision_HALF) {
+    *out = float16();
+  } else if (float_data->precision() == flatbuf::Precision_SINGLE) {
+    *out = float32();
+  } else {
+    *out = float64();
+  }
+  return Status::OK();
+}
+
+// Forward declaration
+static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+    DictionaryMemo* dictionary_memo, FieldOffset* offset);
+
+static Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
+  return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
+}
+
+static Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
+  return flatbuf::CreateFloatingPoint(fbb, precision).Union();
+}
+
+static Status AppendChildFields(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo) {
+  FieldOffset field;
+  for (int i = 0; i < type->num_children(); ++i) {
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), dictionary_memo, &field));
+    out_children->push_back(field);
+  }
+  return Status::OK();
+}
+
+static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+    Offset* offset) {
+  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
+  *offset = flatbuf::CreateList(fbb).Union();
+  return Status::OK();
+}
+
+static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+    Offset* offset) {
+  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
+  *offset = flatbuf::CreateStruct_(fbb).Union();
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Union implementation
+
+static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
+    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
+  UnionMode mode = union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE
+                                                                   : UnionMode::DENSE;
+
+  std::vector<uint8_t> type_codes;
+
+  const flatbuffers::Vector<int32_t>* fb_type_ids = union_data->typeIds();
+  if (fb_type_ids == nullptr) {
+    for (uint8_t i = 0; i < children.size(); ++i) {
+      type_codes.push_back(i);
+    }
+  } else {
+    for (int32_t id : (*fb_type_ids)) {
+      // TODO(wesm): can these values exceed 255?
+      type_codes.push_back(static_cast<uint8_t>(id));
+    }
+  }
+
+  *out = union_(children, type_codes, mode);
+  return Status::OK();
+}
+
+static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+    Offset* offset) {
+  RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
+
+  const auto& union_type = static_cast<const UnionType&>(*type);
+
+  flatbuf::UnionMode mode = union_type.mode == UnionMode::SPARSE
+                                ? flatbuf::UnionMode_Sparse
+                                : flatbuf::UnionMode_Dense;
+
+  std::vector<int32_t> type_ids;
+  type_ids.reserve(union_type.type_codes.size());
+  for (uint8_t code : union_type.type_codes) {
+    type_ids.push_back(code);
+  }
+
+  auto fb_type_ids = fbb.CreateVector(type_ids);
+
+  *offset = flatbuf::CreateUnion(fbb, mode, fb_type_ids).Union();
+  return Status::OK();
+}
+
+#define INT_TO_FB_CASE(BIT_WIDTH, IS_SIGNED)            \
+  *out_type = flatbuf::Type_Int;                        \
+  *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
+  break;
+
+static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) {
+  switch (unit) {
+    case TimeUnit::SECOND:
+      return flatbuf::TimeUnit_SECOND;
+    case TimeUnit::MILLI:
+      return flatbuf::TimeUnit_MILLISECOND;
+    case TimeUnit::MICRO:
+      return flatbuf::TimeUnit_MICROSECOND;
+    case TimeUnit::NANO:
+      return flatbuf::TimeUnit_NANOSECOND;
+    default:
+      break;
+  }
+  return flatbuf::TimeUnit_MIN;
+}
+
+static inline TimeUnit FromFlatbufferUnit(flatbuf::TimeUnit unit) {
+  switch (unit) {
+    case flatbuf::TimeUnit_SECOND:
+      return TimeUnit::SECOND;
+    case flatbuf::TimeUnit_MILLISECOND:
+      return TimeUnit::MILLI;
+    case flatbuf::TimeUnit_MICROSECOND:
+      return TimeUnit::MICRO;
+    case flatbuf::TimeUnit_NANOSECOND:
+      return TimeUnit::NANO;
+    default:
+      break;
+  }
+  // cannot reach
+  return TimeUnit::SECOND;
+}
+
+static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
+    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
+  switch (type) {
+    case flatbuf::Type_NONE:
+      return Status::Invalid("Type metadata cannot be none");
+    case flatbuf::Type_Int:
+      return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
+    case flatbuf::Type_FloatingPoint:
+      return FloatFromFlatuffer(
+          static_cast<const flatbuf::FloatingPoint*>(type_data), out);
+    case flatbuf::Type_Binary:
+      *out = binary();
+      return Status::OK();
+    case flatbuf::Type_FixedWidthBinary: {
+      auto fw_binary = static_cast<const flatbuf::FixedWidthBinary*>(type_data);
+      *out = fixed_width_binary(fw_binary->byteWidth());
+      return Status::OK();
+    }
+    case flatbuf::Type_Utf8:
+      *out = utf8();
+      return Status::OK();
+    case flatbuf::Type_Bool:
+      *out = boolean();
+      return Status::OK();
+    case flatbuf::Type_Decimal:
+      return Status::NotImplemented("Decimal");
+    case flatbuf::Type_Date:
+      *out = date();
+      return Status::OK();
+    case flatbuf::Type_Time: {
+      auto time_type = static_cast<const flatbuf::Time*>(type_data);
+      *out = time(FromFlatbufferUnit(time_type->unit()));
+      return Status::OK();
+    }
+    case flatbuf::Type_Timestamp: {
+      auto ts_type = static_cast<const flatbuf::Timestamp*>(type_data);
+      *out = timestamp(FromFlatbufferUnit(ts_type->unit()));
+      return Status::OK();
+    }
+    case flatbuf::Type_Interval:
+      return Status::NotImplemented("Interval");
+    case flatbuf::Type_List:
+      if (children.size() != 1) {
+        return Status::Invalid("List must have exactly 1 child field");
+      }
+      *out = std::make_shared<ListType>(children[0]);
+      return Status::OK();
+    case flatbuf::Type_Struct_:
+      *out = std::make_shared<StructType>(children);
+      return Status::OK();
+    case flatbuf::Type_Union:
+      return UnionFromFlatbuffer(
+          static_cast<const flatbuf::Union*>(type_data), children, out);
+    default:
+      return Status::Invalid("Unrecognized type");
+  }
+}
+
+// TODO(wesm): Convert this to visitor pattern
+static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
+    flatbuf::Type* out_type, DictionaryMemo* dictionary_memo, Offset* offset) {
+  if (type->type == Type::DICTIONARY) {
+    // In this library, the dictionary "type" is a logical construct. Here we
+    // pass through to the value type, as we've already captured the index
+    // type in the DictionaryEncoding metadata in the parent field
+    const auto& dict_type = static_cast<const DictionaryType&>(*type);
+    return TypeToFlatbuffer(fbb, dict_type.dictionary()->type(), children, layout,
+        out_type, dictionary_memo, offset);
+  }
+
+  std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
+  for (const BufferDescr& descr : buffer_layout) {
+    flatbuf::VectorType vector_type;
+    switch (descr.type()) {
+      case BufferType::OFFSET:
+        vector_type = flatbuf::VectorType_OFFSET;
+        break;
+      case BufferType::DATA:
+        vector_type = flatbuf::VectorType_DATA;
+        break;
+      case BufferType::VALIDITY:
+        vector_type = flatbuf::VectorType_VALIDITY;
+        break;
+      case BufferType::TYPE:
+        vector_type = flatbuf::VectorType_TYPE;
+        break;
+      default:
+        vector_type = flatbuf::VectorType_DATA;
+        break;
+    }
+    auto offset = flatbuf::CreateVectorLayout(
+        fbb, static_cast<int16_t>(descr.bit_width()), vector_type);
+    layout->push_back(offset);
+  }
+
+  switch (type->type) {
+    case Type::BOOL:
+      *out_type = flatbuf::Type_Bool;
+      *offset = flatbuf::CreateBool(fbb).Union();
+      break;
+    case Type::UINT8:
+      INT_TO_FB_CASE(8, false);
+    case Type::INT8:
+      INT_TO_FB_CASE(8, true);
+    case Type::UINT16:
+      INT_TO_FB_CASE(16, false);
+    case Type::INT16:
+      INT_TO_FB_CASE(16, true);
+    case Type::UINT32:
+      INT_TO_FB_CASE(32, false);
+    case Type::INT32:
+      INT_TO_FB_CASE(32, true);
+    case Type::UINT64:
+      INT_TO_FB_CASE(64, false);
+    case Type::INT64:
+      INT_TO_FB_CASE(64, true);
+    case Type::FLOAT:
+      *out_type = flatbuf::Type_FloatingPoint;
+      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_SINGLE);
+      break;
+    case Type::DOUBLE:
+      *out_type = flatbuf::Type_FloatingPoint;
+      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
+      break;
+    case Type::FIXED_WIDTH_BINARY: {
+      const auto& fw_type = static_cast<const FixedWidthBinaryType&>(*type);
+      *out_type = flatbuf::Type_FixedWidthBinary;
+      *offset = flatbuf::CreateFixedWidthBinary(fbb, fw_type.byte_width()).Union();
+    } break;
+    case Type::BINARY:
+      *out_type = flatbuf::Type_Binary;
+      *offset = flatbuf::CreateBinary(fbb).Union();
+      break;
+    case Type::STRING:
+      *out_type = flatbuf::Type_Utf8;
+      *offset = flatbuf::CreateUtf8(fbb).Union();
+      break;
+    case Type::DATE:
+      *out_type = flatbuf::Type_Date;
+      *offset = flatbuf::CreateDate(fbb).Union();
+      break;
+    case Type::TIME: {
+      const auto& time_type = static_cast<const TimeType&>(*type);
+      *out_type = flatbuf::Type_Time;
+      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
+    } break;
+    case Type::TIMESTAMP: {
+      const auto& ts_type = static_cast<const TimestampType&>(*type);
+      *out_type = flatbuf::Type_Timestamp;
+      *offset = flatbuf::CreateTimestamp(fbb, ToFlatbufferUnit(ts_type.unit)).Union();
+    } break;
+    case Type::LIST:
+      *out_type = flatbuf::Type_List;
+      return ListToFlatbuffer(fbb, type, children, dictionary_memo, offset);
+    case Type::STRUCT:
+      *out_type = flatbuf::Type_Struct_;
+      return StructToFlatbuffer(fbb, type, children, dictionary_memo, offset);
+    case Type::UNION:
+      *out_type = flatbuf::Type_Union;
+      return UnionToFlatBuffer(fbb, type, children, dictionary_memo, offset);
+    default:
+      *out_type = flatbuf::Type_NONE;  // Make clang-tidy happy
+      std::stringstream ss;
+      ss << "Unable to convert type: " << type->ToString() << std::endl;
+      return Status::NotImplemented(ss.str());
+  }
+  return Status::OK();
+}
+
+static DictionaryOffset GetDictionaryEncoding(
+    FBB& fbb, const DictionaryType& type, DictionaryMemo* memo) {
+  int64_t dictionary_id = memo->GetId(type.dictionary());
+
+  // We assume that the dictionary index type (as an integer) has already been
+  // validated elsewhere, and can safely assume we are dealing with signed
+  // integers
+  const auto& fw_index_type = static_cast<const FixedWidthType&>(*type.index_type());
+
+  auto index_type_offset = flatbuf::CreateInt(fbb, fw_index_type.bit_width(), true);
+
+  // TODO(wesm): ordered dictionaries
+  return flatbuf::CreateDictionaryEncoding(fbb, dictionary_id, index_type_offset);
+}
+
+static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+    DictionaryMemo* dictionary_memo, FieldOffset* offset) {
+  auto fb_name = fbb.CreateString(field->name);
+
+  flatbuf::Type type_enum;
+  Offset type_offset;
+  Offset type_layout;
+  std::vector<FieldOffset> children;
+  std::vector<VectorLayoutOffset> layout;
+
+  RETURN_NOT_OK(TypeToFlatbuffer(
+      fbb, field->type, &children, &layout, &type_enum, dictionary_memo, &type_offset));
+  auto fb_children = fbb.CreateVector(children);
+  auto fb_layout = fbb.CreateVector(layout);
+
+  DictionaryOffset dictionary = 0;
+  if (field->type->type == Type::DICTIONARY) {
+    dictionary = GetDictionaryEncoding(
+        fbb, static_cast<const DictionaryType&>(*field->type), dictionary_memo);
+  }
+
+  // TODO: produce the list of VectorTypes
+  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_offset,
+      dictionary, fb_children, fb_layout);
+
+  return Status::OK();
+}
+
+static Status FieldFromFlatbuffer(const flatbuf::Field* field,
+    const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out) {
+  std::shared_ptr<DataType> type;
+
+  const flatbuf::DictionaryEncoding* encoding = field->dictionary();
+
+  if (encoding == nullptr) {
+    // The field is not dictionary encoded. We must potentially visit its
+    // children to fully reconstruct the data type
+    auto children = field->children();
+    std::vector<std::shared_ptr<Field>> child_fields(children->size());
+    for (int i = 0; i < static_cast<int>(children->size()); ++i) {
+      RETURN_NOT_OK(
+          FieldFromFlatbuffer(children->Get(i), dictionary_memo, &child_fields[i]));
+    }
+    RETURN_NOT_OK(
+        TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
+  } else {
+    // The field is dictionary encoded. The type of the dictionary values has
+    // been determined elsewhere, and is stored in the DictionaryMemo. Here we
+    // construct the logical DictionaryType object
+
+    std::shared_ptr<Array> dictionary;
+    RETURN_NOT_OK(dictionary_memo.GetDictionary(encoding->id(), &dictionary));
+
+    std::shared_ptr<DataType> index_type;
+    RETURN_NOT_OK(IntFromFlatbuffer(encoding->indexType(), &index_type));
+    type = std::make_shared<DictionaryType>(index_type, dictionary);
+  }
+  *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
+  return Status::OK();
+}
+
+static Status FieldFromFlatbufferDictionary(
+    const flatbuf::Field* field, std::shared_ptr<Field>* out) {
+  // Need an empty memo to pass down for constructing children
+  DictionaryMemo dummy_memo;
+
+  // Any DictionaryEncoding set is ignored here
+
+  std::shared_ptr<DataType> type;
+  auto children = field->children();
+  std::vector<std::shared_ptr<Field>> child_fields(children->size());
+  for (int i = 0; i < static_cast<int>(children->size()); ++i) {
+    RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), dummy_memo, &child_fields[i]));
+  }
+
+  RETURN_NOT_OK(
+      TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
+
+  *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
+  return Status::OK();
+}
+
+// will return the endianness of the system we are running on
+// based the NUMPY_API function. See NOTICE.txt
+flatbuf::Endianness endianness() {
+  union {
+    uint32_t i;
+    char c[4];
+  } bint = {0x01020304};
+
+  return bint.c[0] == 1 ? flatbuf::Endianness_Big : flatbuf::Endianness_Little;
+}
+
+static Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
+    DictionaryMemo* dictionary_memo, flatbuffers::Offset<flatbuf::Schema>* out) {
+  std::vector<FieldOffset> field_offsets;
+  for (int i = 0; i < schema.num_fields(); ++i) {
+    std::shared_ptr<Field> field = schema.field(i);
+    FieldOffset offset;
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, dictionary_memo, &offset));
+    field_offsets.push_back(offset);
+  }
+
+  *out = flatbuf::CreateSchema(fbb, endianness(), fbb.CreateVector(field_offsets));
+  return Status::OK();
+}
+
+static Status WriteFlatbufferBuilder(FBB& fbb, std::shared_ptr<Buffer>* out) {
+  int32_t size = fbb.GetSize();
+
+  auto result = std::make_shared<PoolBuffer>();
+  RETURN_NOT_OK(result->Resize(size));
+
+  uint8_t* dst = result->mutable_data();
+  memcpy(dst, fbb.GetBufferPointer(), size);
+  *out = result;
+  return Status::OK();
+}
+
+static Status WriteMessage(FBB& fbb, flatbuf::MessageHeader header_type,
+    flatbuffers::Offset<void> header, int64_t body_length, std::shared_ptr<Buffer>* out) {
+  auto message =
+      flatbuf::CreateMessage(fbb, kMetadataVersion, header_type, header, body_length);
+  fbb.Finish(message);
+  return WriteFlatbufferBuilder(fbb, out);
+}
+
+Status WriteSchemaMessage(
+    const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out) {
+  FBB fbb;
+  flatbuffers::Offset<flatbuf::Schema> fb_schema;
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
+  return WriteMessage(fbb, flatbuf::MessageHeader_Schema, fb_schema.Union(), 0, out);
+}
+
+using FieldNodeVector =
+    flatbuffers::Offset<flatbuffers::Vector<const flatbuf::FieldNode*>>;
+using LargeFieldNodeVector =
+    flatbuffers::Offset<flatbuffers::Vector<const flatbuf::LargeFieldNode*>>;
+using BufferVector = flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Buffer*>>;
+
+static Status WriteFieldNodes(
+    FBB& fbb, const std::vector<FieldMetadata>& nodes, FieldNodeVector* out) {
+  std::vector<flatbuf::FieldNode> fb_nodes;
+  fb_nodes.reserve(nodes.size());
+
+  for (size_t i = 0; i < nodes.size(); ++i) {
+    const FieldMetadata& node = nodes[i];
+    if (node.offset != 0) {
+      return Status::Invalid("Field metadata for IPC must have offset 0");
+    }
+    fb_nodes.emplace_back(
+        static_cast<int32_t>(node.length), static_cast<int32_t>(node.null_count));
+  }
+  *out = fbb.CreateVectorOfStructs(fb_nodes);
+  return Status::OK();
+}
+
+static Status WriteLargeFieldNodes(
+    FBB& fbb, const std::vector<FieldMetadata>& nodes, LargeFieldNodeVector* out) {
+  std::vector<flatbuf::LargeFieldNode> fb_nodes;
+  fb_nodes.reserve(nodes.size());
+
+  for (size_t i = 0; i < nodes.size(); ++i) {
+    const FieldMetadata& node = nodes[i];
+    if (node.offset != 0) {
+      return Status::Invalid("Field metadata for IPC must have offset 0");
+    }
+    fb_nodes.emplace_back(node.length, node.null_count);
+  }
+  *out = fbb.CreateVectorOfStructs(fb_nodes);
+  return Status::OK();
+}
+
+static Status WriteBuffers(
+    FBB& fbb, const std::vector<BufferMetadata>& buffers, BufferVector* out) {
+  std::vector<flatbuf::Buffer> fb_buffers;
+  fb_buffers.reserve(buffers.size());
+
+  for (size_t i = 0; i < buffers.size(); ++i) {
+    const BufferMetadata& buffer = buffers[i];
+    fb_buffers.emplace_back(buffer.page, buffer.offset, buffer.length);
+  }
+  *out = fbb.CreateVectorOfStructs(fb_buffers);
+  return Status::OK();
+}
+
+static Status MakeRecordBatch(FBB& fbb, int32_t length, int64_t body_length,
+    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+    RecordBatchOffset* offset) {
+  FieldNodeVector fb_nodes;
+  BufferVector fb_buffers;
+
+  RETURN_NOT_OK(WriteFieldNodes(fbb, nodes, &fb_nodes));
+  RETURN_NOT_OK(WriteBuffers(fbb, buffers, &fb_buffers));
+
+  *offset = flatbuf::CreateRecordBatch(fbb, length, fb_nodes, fb_buffers);
+  return Status::OK();
+}
+
+static Status MakeLargeRecordBatch(FBB& fbb, int64_t length, int64_t body_length,
+    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+    LargeRecordBatchOffset* offset) {
+  LargeFieldNodeVector fb_nodes;
+  BufferVector fb_buffers;
+
+  RETURN_NOT_OK(WriteLargeFieldNodes(fbb, nodes, &fb_nodes));
+  RETURN_NOT_OK(WriteBuffers(fbb, buffers, &fb_buffers));
+
+  *offset = flatbuf::CreateLargeRecordBatch(fbb, length, fb_nodes, fb_buffers);
+  return Status::OK();
+}
+
+Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
+    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+    std::shared_ptr<Buffer>* out) {
+  FBB fbb;
+  RecordBatchOffset record_batch;
+  RETURN_NOT_OK(MakeRecordBatch(fbb, length, body_length, nodes, buffers, &record_batch));
+  return WriteMessage(
+      fbb, flatbuf::MessageHeader_RecordBatch, record_batch.Union(), body_length, out);
+}
+
+Status WriteLargeRecordBatchMessage(int64_t length, int64_t body_length,
+    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+    std::shared_ptr<Buffer>* out) {
+  FBB fbb;
+  LargeRecordBatchOffset large_batch;
+  RETURN_NOT_OK(
+      MakeLargeRecordBatch(fbb, length, body_length, nodes, buffers, &large_batch));
+  return WriteMessage(fbb, flatbuf::MessageHeader_LargeRecordBatch, large_batch.Union(),
+      body_length, out);
+}
+
+Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
+    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+    std::shared_ptr<Buffer>* out) {
+  FBB fbb;
+  RecordBatchOffset record_batch;
+  RETURN_NOT_OK(MakeRecordBatch(fbb, length, body_length, nodes, buffers, &record_batch));
+  auto dictionary_batch = flatbuf::CreateDictionaryBatch(fbb, id, record_batch).Union();
+  return WriteMessage(
+      fbb, flatbuf::MessageHeader_DictionaryBatch, dictionary_batch, body_length, out);
+}
+
+static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
+FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
+  std::vector<flatbuf::Block> fb_blocks;
+
+  for (const FileBlock& block : blocks) {
+    fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
+  }
+
+  return fbb.CreateVectorOfStructs(fb_blocks);
+}
+
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
+    io::OutputStream* out) {
+  FBB fbb;
+
+  flatbuffers::Offset<flatbuf::Schema> fb_schema;
+  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
+
+  auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
+  auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
+
+  auto footer = flatbuf::CreateFooter(
+      fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
+
+  fbb.Finish(footer);
+
+  int32_t size = fbb.GetSize();
+
+  return out->Write(fbb.GetBufferPointer(), size);
+}
+
 // ----------------------------------------------------------------------
 // Memoization data structure for handling shared dictionaries
 
@@ -158,7 +795,18 @@ int64_t Message::body_length() const {
 // ----------------------------------------------------------------------
 // SchemaMetadata
 
-class SchemaMetadata::SchemaMetadataImpl {
+class MessageHolder {
+ public:
+  void set_message(const std::shared_ptr<Message>& message) { message_ = message; }
+  void set_buffer(const std::shared_ptr<Buffer>& buffer) { buffer_ = buffer; }
+
+ protected:
+  // Possible parents, owns the flatbuffer data
+  std::shared_ptr<Message> message_;
+  std::shared_ptr<Buffer> buffer_;
+};
+
+class SchemaMetadata::SchemaMetadataImpl : public MessageHolder {
  public:
   explicit SchemaMetadataImpl(const void* schema)
       : schema_(static_cast<const flatbuf::Schema*>(schema)) {}
@@ -196,15 +844,19 @@ class SchemaMetadata::SchemaMetadataImpl {
   const flatbuf::Schema* schema_;
 };
 
-SchemaMetadata::SchemaMetadata(
-    const std::shared_ptr<Message>& message, const void* flatbuf) {
-  message_ = message;
-  impl_.reset(new SchemaMetadataImpl(flatbuf));
+SchemaMetadata::SchemaMetadata(const std::shared_ptr<Message>& message)
+    : SchemaMetadata(message->impl_->header()) {
+  impl_->set_message(message);
 }
 
-SchemaMetadata::SchemaMetadata(const std::shared_ptr<Message>& message) {
-  message_ = message;
-  impl_.reset(new SchemaMetadataImpl(message->impl_->header()));
+SchemaMetadata::SchemaMetadata(const void* header) {
+  impl_.reset(new SchemaMetadataImpl(header));
+}
+
+SchemaMetadata::SchemaMetadata(const std::shared_ptr<Buffer>& buffer, int64_t offset)
+    : SchemaMetadata(buffer->data() + offset) {
+  // Preserve ownership
+  impl_->set_buffer(buffer);
 }
 
 SchemaMetadata::~SchemaMetadata() {}
@@ -231,7 +883,7 @@ Status SchemaMetadata::GetSchema(
 // ----------------------------------------------------------------------
 // RecordBatchMetadata
 
-class RecordBatchMetadata::RecordBatchMetadataImpl {
+class RecordBatchMetadata::RecordBatchMetadataImpl : public MessageHolder {
  public:
   explicit RecordBatchMetadataImpl(const void* batch)
       : batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
@@ -249,22 +901,14 @@ class RecordBatchMetadata::RecordBatchMetadataImpl {
 
   int num_fields() const { return batch_->nodes()->size(); }
 
-  void set_message(const std::shared_ptr<Message>& message) { message_ = message; }
-
-  void set_buffer(const std::shared_ptr<Buffer>& buffer) { buffer_ = buffer; }
-
  private:
   const flatbuf::RecordBatch* batch_;
   const flatbuffers::Vector<const flatbuf::FieldNode*>* nodes_;
   const flatbuffers::Vector<const flatbuf::Buffer*>* buffers_;
-
-  // Possible parents, owns the flatbuffer data
-  std::shared_ptr<Message> message_;
-  std::shared_ptr<Buffer> buffer_;
 };
 
-RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message) {
-  impl_.reset(new RecordBatchMetadataImpl(message->impl_->header()));
+RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message)
+    : RecordBatchMetadata(message->impl_->header()) {
   impl_->set_message(message);
 }
 
@@ -358,8 +1002,8 @@ const RecordBatchMetadata& DictionaryBatchMetadata::record_batch() const {
 // ----------------------------------------------------------------------
 // Conveniences
 
-Status ReadMessage(int64_t offset, int32_t metadata_length,
-    io::RandomAccessFile* file, std::shared_ptr<Message>* message) {
+Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
+    std::shared_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 4eb0186d3a467..41e6c5e9f19ea 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -107,10 +107,9 @@ class Message;
 // Container for serialized Schema metadata contained in an IPC message
 class ARROW_EXPORT SchemaMetadata {
  public:
+  explicit SchemaMetadata(const void* header);
   explicit SchemaMetadata(const std::shared_ptr<Message>& message);
-
-  // Accepts an opaque flatbuffer pointer
-  SchemaMetadata(const std::shared_ptr<Message>& message, const void* schema);
+  SchemaMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
 
   ~SchemaMetadata();
 
@@ -127,9 +126,6 @@ class ARROW_EXPORT SchemaMetadata {
       const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out) const;
 
  private:
-  // Parent, owns the flatbuffer data
-  std::shared_ptr<Message> message_;
-
   class SchemaMetadataImpl;
   std::unique_ptr<SchemaMetadataImpl> impl_;
 
@@ -145,8 +141,6 @@ struct ARROW_EXPORT BufferMetadata {
 // Container for serialized record batch metadata contained in an IPC message
 class ARROW_EXPORT RecordBatchMetadata {
  public:
-  // Instantiate from opaque pointer. Memory ownership must be preserved
-  // elsewhere (e.g. in a dictionary batch)
   explicit RecordBatchMetadata(const void* header);
   explicit RecordBatchMetadata(const std::shared_ptr<Message>& message);
   RecordBatchMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
@@ -218,8 +212,34 @@ class ARROW_EXPORT Message {
 /// \param[in] file the seekable file interface to read from
 /// \param[out] message the message read
 /// \return Status success or failure
-Status ReadMessage(int64_t offset, int32_t metadata_length,
-    io::RandomAccessFile* file, std::shared_ptr<Message>* message);
+Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
+    std::shared_ptr<Message>* message);
+
+// Serialize arrow::Schema as a Flatbuffer
+//
+// \param[in] schema a Schema instance
+// \param[inout] dictionary_memo class for tracking dictionaries and assigning
+// dictionary ids
+// \param[out] out the serialized arrow::Buffer
+// \return Status outcome
+Status WriteSchemaMessage(
+    const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out);
+
+Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
+    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+    std::shared_ptr<Buffer>* out);
+
+Status WriteLargeRecordBatchMessage(int64_t length, int64_t body_length,
+    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+    std::shared_ptr<Buffer>* out);
+
+Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
+    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+    std::shared_ptr<Buffer>* out);
+
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
+    io::OutputStream* out);
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 95753643c6513..a2b20a901a69e 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -26,16 +26,114 @@
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/metadata-internal.h"
+#include "arrow/ipc/File_generated.h"
+#include "arrow/ipc/Message_generated.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
+#include "arrow/schema.h"
 #include "arrow/status.h"
+#include "arrow/table.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
+
+namespace flatbuf = org::apache::arrow::flatbuf;
+
 namespace ipc {
 
+// ----------------------------------------------------------------------
+// Record batch read path
+
+class IpcComponentSource : public ArrayComponentSource {
+ public:
+  IpcComponentSource(const RecordBatchMetadata& metadata, io::RandomAccessFile* file)
+      : metadata_(metadata), file_(file) {}
+
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
+    BufferMetadata buffer_meta = metadata_.buffer(buffer_index);
+    if (buffer_meta.length == 0) {
+      *out = nullptr;
+      return Status::OK();
+    } else {
+      return file_->ReadAt(buffer_meta.offset, buffer_meta.length, out);
+    }
+  }
+
+  Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
+    // pop off a field
+    if (field_index >= metadata_.num_fields()) {
+      return Status::Invalid("Ran out of field metadata, likely malformed");
+    }
+    *metadata = metadata_.field(field_index);
+    return Status::OK();
+  }
+
+ private:
+  const RecordBatchMetadata& metadata_;
+  io::RandomAccessFile* file_;
+};
+
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
+    std::shared_ptr<RecordBatch>* out) {
+  return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
+}
+
+static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
+    int64_t num_rows, int max_recursion_depth, ArrayComponentSource* source,
+    std::shared_ptr<RecordBatch>* out) {
+  std::vector<std::shared_ptr<Array>> arrays(schema->num_fields());
+
+  ArrayLoaderContext context;
+  context.source = source;
+  context.field_index = 0;
+  context.buffer_index = 0;
+  context.max_recursion_depth = max_recursion_depth;
+
+  for (int i = 0; i < schema->num_fields(); ++i) {
+    RETURN_NOT_OK(LoadArray(schema->field(i)->type, &context, &arrays[i]));
+  }
+
+  *out = std::make_shared<RecordBatch>(schema, num_rows, arrays);
+  return Status::OK();
+}
+
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+  IpcComponentSource source(metadata, file);
+  return LoadRecordBatchFromSource(
+      schema, metadata.length(), max_recursion_depth, &source, out);
+}
+
+Status ReadDictionary(const DictionaryBatchMetadata& metadata,
+    const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
+    std::shared_ptr<Array>* out) {
+  int64_t id = metadata.id();
+  auto it = dictionary_types.find(id);
+  if (it == dictionary_types.end()) {
+    std::stringstream ss;
+    ss << "Do not have type metadata for dictionary with id: " << id;
+    return Status::KeyError(ss.str());
+  }
+
+  std::vector<std::shared_ptr<Field>> fields = {it->second};
+
+  // We need a schema for the record batch
+  auto dummy_schema = std::make_shared<Schema>(fields);
+
+  // The dictionary is embedded in a record batch with a single column
+  std::shared_ptr<RecordBatch> batch;
+  RETURN_NOT_OK(ReadRecordBatch(metadata.record_batch(), dummy_schema, file, &batch));
+
+  if (batch->num_columns() != 1) {
+    return Status::Invalid("Dictionary record batch must only contain one field");
+  }
+
+  *out = batch->column(0);
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // StreamReader implementation
 
@@ -228,7 +326,7 @@ class FileReader::FileReaderImpl {
 
     // TODO(wesm): Verify the footer
     footer_ = flatbuf::GetFooter(footer_buffer_->data());
-    schema_metadata_.reset(new SchemaMetadata(nullptr, footer_->schema()));
+    schema_metadata_.reset(new SchemaMetadata(footer_->schema()));
 
     return Status::OK();
   }
@@ -307,8 +405,7 @@ class FileReader::FileReaderImpl {
     return schema_metadata_->GetSchema(*dictionary_memo_, &schema_);
   }
 
-  Status Open(
-      const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset) {
+  Status Open(const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset) {
     file_ = file;
     footer_offset_ = footer_offset;
     RETURN_NOT_OK(ReadFooter());
@@ -371,5 +468,69 @@ Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
   return impl_->GetRecordBatch(i, batch);
 }
 
+// ----------------------------------------------------------------------
+// Read LargeRecordBatch
+
+class LargeRecordBatchSource : public ArrayComponentSource {
+ public:
+  LargeRecordBatchSource(
+      const flatbuf::LargeRecordBatch* metadata, io::RandomAccessFile* file)
+      : metadata_(metadata), file_(file) {}
+
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
+    if (buffer_index >= static_cast<int>(metadata_->buffers()->size())) {
+      return Status::Invalid("Ran out of buffer metadata, likely malformed");
+    }
+    const flatbuf::Buffer* buffer = metadata_->buffers()->Get(buffer_index);
+
+    if (buffer->length() == 0) {
+      *out = nullptr;
+      return Status::OK();
+    } else {
+      return file_->ReadAt(buffer->offset(), buffer->length(), out);
+    }
+  }
+
+  Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
+    // pop off a field
+    if (field_index >= static_cast<int>(metadata_->nodes()->size())) {
+      return Status::Invalid("Ran out of field metadata, likely malformed");
+    }
+    const flatbuf::LargeFieldNode* node = metadata_->nodes()->Get(field_index);
+
+    metadata->length = node->length();
+    metadata->null_count = node->null_count();
+    metadata->offset = 0;
+    return Status::OK();
+  }
+
+ private:
+  const flatbuf::LargeRecordBatch* metadata_;
+  io::RandomAccessFile* file_;
+};
+
+Status ReadLargeRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+  std::shared_ptr<Buffer> buffer;
+  RETURN_NOT_OK(file->Seek(offset));
+
+  RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
+  int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
+
+  RETURN_NOT_OK(file->Read(flatbuffer_size, &buffer));
+  auto message = flatbuf::GetMessage(buffer->data());
+  auto batch = reinterpret_cast<const flatbuf::LargeRecordBatch*>(message->header());
+
+  // TODO(ARROW-388): The buffer offsets start at 0, so we must construct a
+  // RandomAccessFile according to that frame of reference
+  std::shared_ptr<Buffer> buffer_payload;
+  RETURN_NOT_OK(file->Read(message->bodyLength(), &buffer_payload));
+  io::BufferReader buffer_reader(buffer_payload);
+
+  LargeRecordBatchSource source(batch, &buffer_reader);
+  return LoadRecordBatchFromSource(
+      schema, batch->length(), kMaxNestingDepth, &source, out);
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index ca91765edbac1..1c1314a040bef 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -43,6 +43,20 @@ class RandomAccessFile;
 
 namespace ipc {
 
+// Generic read functionsh; does not copy data if the input supports zero copy reads
+
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
+    std::shared_ptr<RecordBatch>* out);
+
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
+
+Status ReadDictionary(const DictionaryBatchMetadata& metadata,
+    const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
+    std::shared_ptr<Array>* out);
+
 class ARROW_EXPORT StreamReader {
  public:
   ~StreamReader();
@@ -106,6 +120,14 @@ class ARROW_EXPORT FileReader {
   std::unique_ptr<FileReaderImpl> impl_;
 };
 
+// ----------------------------------------------------------------------
+//
+
+/// EXPERIMENTAL: Read length-prefixed LargeRecordBatch metadata (64-bit array
+/// lengths) at offset and reconstruct RecordBatch
+Status ARROW_EXPORT ReadLargeRecordBatch(const std::shared_ptr<Schema>& schema,
+    int64_t offset, io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 66a5e09362cf5..ba203b090b3b7 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -103,7 +103,7 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
 typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
 
 Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
-  const int length = 1000;
+  const int length = 10;
 
   // Make the schema
   auto f0 = field("f0", int32());
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 58402b588404c..82c119ef53e9a 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -17,27 +17,509 @@
 
 #include "arrow/ipc/writer.h"
 
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
+#include <limits>
 #include <sstream>
 #include <vector>
 
+#include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
+#include "arrow/loader.h"
 #include "arrow/memory_pool.h"
 #include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
+#include "arrow/type.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
 namespace ipc {
 
+// ----------------------------------------------------------------------
+// Record batch write path
+
+class RecordBatchWriter : public ArrayVisitor {
+ public:
+  RecordBatchWriter(
+      MemoryPool* pool, int64_t buffer_start_offset, int max_recursion_depth)
+      : pool_(pool),
+        max_recursion_depth_(max_recursion_depth),
+        buffer_start_offset_(buffer_start_offset) {
+    DCHECK_GT(max_recursion_depth, 0);
+  }
+
+  virtual ~RecordBatchWriter() = default;
+
+  virtual Status CheckArrayMetadata(const Array& arr) {
+    if (arr.length() > std::numeric_limits<int32_t>::max()) {
+      return Status::Invalid("Cannot write arrays larger than 2^31 - 1 in length");
+    }
+    return Status::OK();
+  }
+
+  Status VisitArray(const Array& arr) {
+    if (max_recursion_depth_ <= 0) {
+      return Status::Invalid("Max recursion depth reached");
+    }
+
+    RETURN_NOT_OK(CheckArrayMetadata(arr));
+
+    // push back all common elements
+    field_nodes_.emplace_back(arr.length(), arr.null_count(), 0);
+
+    if (arr.null_count() > 0) {
+      std::shared_ptr<Buffer> bitmap = arr.null_bitmap();
+
+      if (arr.offset() != 0) {
+        // With a sliced array / non-zero offset, we must copy the bitmap
+        RETURN_NOT_OK(
+            CopyBitmap(pool_, bitmap->data(), arr.offset(), arr.length(), &bitmap));
+      }
+
+      buffers_.push_back(bitmap);
+    } else {
+      // Push a dummy zero-length buffer, not to be copied
+      buffers_.push_back(std::make_shared<Buffer>(nullptr, 0));
+    }
+    return arr.Accept(this);
+  }
+
+  Status Assemble(const RecordBatch& batch, int64_t* body_length) {
+    if (field_nodes_.size() > 0) {
+      field_nodes_.clear();
+      buffer_meta_.clear();
+      buffers_.clear();
+    }
+
+    // Perform depth-first traversal of the row-batch
+    for (int i = 0; i < batch.num_columns(); ++i) {
+      RETURN_NOT_OK(VisitArray(*batch.column(i)));
+    }
+
+    // The position for the start of a buffer relative to the passed frame of
+    // reference. May be 0 or some other position in an address space
+    int64_t offset = buffer_start_offset_;
+
+    buffer_meta_.reserve(buffers_.size());
+
+    const int32_t kNoPageId = -1;
+
+    // Construct the buffer metadata for the record batch header
+    for (size_t i = 0; i < buffers_.size(); ++i) {
+      const Buffer* buffer = buffers_[i].get();
+      int64_t size = 0;
+      int64_t padding = 0;
+
+      // The buffer might be null if we are handling zero row lengths.
+      if (buffer) {
+        size = buffer->size();
+        padding = BitUtil::RoundUpToMultipleOf64(size) - size;
+      }
+
+      // TODO(wesm): We currently have no notion of shared memory page id's,
+      // but we've included it in the metadata IDL for when we have it in the
+      // future. Use page = -1 for now
+      //
+      // Note that page ids are a bespoke notion for Arrow and not a feature we
+      // are using from any OS-level shared memory. The thought is that systems
+      // may (in the future) associate integer page id's with physical memory
+      // pages (according to whatever is the desired shared memory mechanism)
+      buffer_meta_.push_back({kNoPageId, offset, size + padding});
+      offset += size + padding;
+    }
+
+    *body_length = offset - buffer_start_offset_;
+    DCHECK(BitUtil::IsMultipleOf64(*body_length));
+
+    return Status::OK();
+  }
+
+  // Override this for writing dictionary metadata
+  virtual Status WriteMetadataMessage(
+      int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) {
+    return WriteRecordBatchMessage(
+        static_cast<int32_t>(num_rows), body_length, field_nodes_, buffer_meta_, out);
+  }
+
+  Status WriteMetadata(int64_t num_rows, int64_t body_length, io::OutputStream* dst,
+      int32_t* metadata_length) {
+    // Now that we have computed the locations of all of the buffers in shared
+    // memory, the data header can be converted to a flatbuffer and written out
+    //
+    // Note: The memory written here is prefixed by the size of the flatbuffer
+    // itself as an int32_t.
+    std::shared_ptr<Buffer> metadata_fb;
+    RETURN_NOT_OK(WriteMetadataMessage(num_rows, body_length, &metadata_fb));
+
+    // Need to write 4 bytes (metadata size), the metadata, plus padding to
+    // end on an 8-byte offset
+    int64_t start_offset;
+    RETURN_NOT_OK(dst->Tell(&start_offset));
+
+    int32_t padded_metadata_length = static_cast<int32_t>(metadata_fb->size()) + 4;
+    const int32_t remainder =
+        (padded_metadata_length + static_cast<int32_t>(start_offset)) % 8;
+    if (remainder != 0) { padded_metadata_length += 8 - remainder; }
+
+    // The returned metadata size includes the length prefix, the flatbuffer,
+    // plus padding
+    *metadata_length = padded_metadata_length;
+
+    // Write the flatbuffer size prefix including padding
+    int32_t flatbuffer_size = padded_metadata_length - 4;
+    RETURN_NOT_OK(
+        dst->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
+
+    // Write the flatbuffer
+    RETURN_NOT_OK(dst->Write(metadata_fb->data(), metadata_fb->size()));
+
+    // Write any padding
+    int32_t padding =
+        padded_metadata_length - static_cast<int32_t>(metadata_fb->size()) - 4;
+    if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
+
+    return Status::OK();
+  }
+
+  Status Write(const RecordBatch& batch, io::OutputStream* dst, int32_t* metadata_length,
+      int64_t* body_length) {
+    RETURN_NOT_OK(Assemble(batch, body_length));
+
+#ifndef NDEBUG
+    int64_t start_position, current_position;
+    RETURN_NOT_OK(dst->Tell(&start_position));
+#endif
+
+    RETURN_NOT_OK(WriteMetadata(batch.num_rows(), *body_length, dst, metadata_length));
+
+#ifndef NDEBUG
+    RETURN_NOT_OK(dst->Tell(&current_position));
+    DCHECK(BitUtil::IsMultipleOf8(current_position));
+#endif
+
+    // Now write the buffers
+    for (size_t i = 0; i < buffers_.size(); ++i) {
+      const Buffer* buffer = buffers_[i].get();
+      int64_t size = 0;
+      int64_t padding = 0;
+
+      // The buffer might be null if we are handling zero row lengths.
+      if (buffer) {
+        size = buffer->size();
+        padding = BitUtil::RoundUpToMultipleOf64(size) - size;
+      }
+
+      if (size > 0) { RETURN_NOT_OK(dst->Write(buffer->data(), size)); }
+
+      if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
+    }
+
+#ifndef NDEBUG
+    RETURN_NOT_OK(dst->Tell(&current_position));
+    DCHECK(BitUtil::IsMultipleOf8(current_position));
+#endif
+
+    return Status::OK();
+  }
+
+  Status GetTotalSize(const RecordBatch& batch, int64_t* size) {
+    // emulates the behavior of Write without actually writing
+    int32_t metadata_length = 0;
+    int64_t body_length = 0;
+    MockOutputStream dst;
+    RETURN_NOT_OK(Write(batch, &dst, &metadata_length, &body_length));
+    *size = dst.GetExtentBytesWritten();
+    return Status::OK();
+  }
+
+ protected:
+  template <typename ArrayType>
+  Status VisitFixedWidth(const ArrayType& array) {
+    std::shared_ptr<Buffer> data_buffer = array.data();
+
+    if (array.offset() != 0) {
+      // Non-zero offset, slice the buffer
+      const auto& fw_type = static_cast<const FixedWidthType&>(*array.type());
+      const int type_width = fw_type.bit_width() / 8;
+      const int64_t byte_offset = array.offset() * type_width;
+
+      // Send padding if it's available
+      const int64_t buffer_length =
+          std::min(BitUtil::RoundUpToMultipleOf64(array.length() * type_width),
+              data_buffer->size() - byte_offset);
+      data_buffer = SliceBuffer(data_buffer, byte_offset, buffer_length);
+    }
+    buffers_.push_back(data_buffer);
+    return Status::OK();
+  }
+
+  template <typename ArrayType>
+  Status GetZeroBasedValueOffsets(
+      const ArrayType& array, std::shared_ptr<Buffer>* value_offsets) {
+    // Share slicing logic between ListArray and BinaryArray
+
+    auto offsets = array.value_offsets();
+
+    if (array.offset() != 0) {
+      // If we have a non-zero offset, then the value offsets do not start at
+      // zero. We must a) create a new offsets array with shifted offsets and
+      // b) slice the values array accordingly
+
+      std::shared_ptr<MutableBuffer> shifted_offsets;
+      RETURN_NOT_OK(AllocateBuffer(
+          pool_, sizeof(int32_t) * (array.length() + 1), &shifted_offsets));
+
+      int32_t* dest_offsets = reinterpret_cast<int32_t*>(shifted_offsets->mutable_data());
+      const int32_t start_offset = array.value_offset(0);
+
+      for (int i = 0; i < array.length(); ++i) {
+        dest_offsets[i] = array.value_offset(i) - start_offset;
+      }
+      // Final offset
+      dest_offsets[array.length()] = array.value_offset(array.length()) - start_offset;
+      offsets = shifted_offsets;
+    }
+
+    *value_offsets = offsets;
+    return Status::OK();
+  }
+
+  Status VisitBinary(const BinaryArray& array) {
+    std::shared_ptr<Buffer> value_offsets;
+    RETURN_NOT_OK(GetZeroBasedValueOffsets<BinaryArray>(array, &value_offsets));
+    auto data = array.data();
+
+    if (array.offset() != 0) {
+      // Slice the data buffer to include only the range we need now
+      data = SliceBuffer(data, array.value_offset(0), array.value_offset(array.length()));
+    }
+
+    buffers_.push_back(value_offsets);
+    buffers_.push_back(data);
+    return Status::OK();
+  }
+
+  Status Visit(const FixedWidthBinaryArray& array) override {
+    auto data = array.data();
+    int32_t width = array.byte_width();
+
+    if (array.offset() != 0) {
+      data = SliceBuffer(data, array.offset() * width, width * array.length());
+    }
+    buffers_.push_back(data);
+    return Status::OK();
+  }
+
+  Status Visit(const BooleanArray& array) override {
+    buffers_.push_back(array.data());
+    return Status::OK();
+  }
+
+#define VISIT_FIXED_WIDTH(TYPE) \
+  Status Visit(const TYPE& array) override { return VisitFixedWidth<TYPE>(array); }
+
+  VISIT_FIXED_WIDTH(Int8Array);
+  VISIT_FIXED_WIDTH(Int16Array);
+  VISIT_FIXED_WIDTH(Int32Array);
+  VISIT_FIXED_WIDTH(Int64Array);
+  VISIT_FIXED_WIDTH(UInt8Array);
+  VISIT_FIXED_WIDTH(UInt16Array);
+  VISIT_FIXED_WIDTH(UInt32Array);
+  VISIT_FIXED_WIDTH(UInt64Array);
+  VISIT_FIXED_WIDTH(HalfFloatArray);
+  VISIT_FIXED_WIDTH(FloatArray);
+  VISIT_FIXED_WIDTH(DoubleArray);
+  VISIT_FIXED_WIDTH(DateArray);
+  VISIT_FIXED_WIDTH(Date32Array);
+  VISIT_FIXED_WIDTH(TimeArray);
+  VISIT_FIXED_WIDTH(TimestampArray);
+
+#undef VISIT_FIXED_WIDTH
+
+  Status Visit(const StringArray& array) override { return VisitBinary(array); }
+
+  Status Visit(const BinaryArray& array) override { return VisitBinary(array); }
+
+  Status Visit(const ListArray& array) override {
+    std::shared_ptr<Buffer> value_offsets;
+    RETURN_NOT_OK(GetZeroBasedValueOffsets<ListArray>(array, &value_offsets));
+    buffers_.push_back(value_offsets);
+
+    --max_recursion_depth_;
+    std::shared_ptr<Array> values = array.values();
+
+    if (array.offset() != 0) {
+      // For non-zero offset, we slice the values array accordingly
+      const int32_t offset = array.value_offset(0);
+      const int32_t length = array.value_offset(array.length()) - offset;
+      values = values->Slice(offset, length);
+    }
+    RETURN_NOT_OK(VisitArray(*values));
+    ++max_recursion_depth_;
+    return Status::OK();
+  }
+
+  Status Visit(const StructArray& array) override {
+    --max_recursion_depth_;
+    for (std::shared_ptr<Array> field : array.fields()) {
+      if (array.offset() != 0) {
+        // If offset is non-zero, slice the child array
+        field = field->Slice(array.offset(), array.length());
+      }
+      RETURN_NOT_OK(VisitArray(*field));
+    }
+    ++max_recursion_depth_;
+    return Status::OK();
+  }
+
+  Status Visit(const UnionArray& array) override {
+    auto type_ids = array.type_ids();
+    if (array.offset() != 0) {
+      type_ids = SliceBuffer(type_ids, array.offset() * sizeof(UnionArray::type_id_t),
+          array.length() * sizeof(UnionArray::type_id_t));
+    }
+
+    buffers_.push_back(type_ids);
+
+    --max_recursion_depth_;
+    if (array.mode() == UnionMode::DENSE) {
+      const auto& type = static_cast<const UnionType&>(*array.type());
+      auto value_offsets = array.value_offsets();
+
+      // The Union type codes are not necessary 0-indexed
+      uint8_t max_code = 0;
+      for (uint8_t code : type.type_codes) {
+        if (code > max_code) { max_code = code; }
+      }
+
+      // Allocate an array of child offsets. Set all to -1 to indicate that we
+      // haven't observed a first occurrence of a particular child yet
+      std::vector<int32_t> child_offsets(max_code + 1);
+      std::vector<int32_t> child_lengths(max_code + 1, 0);
+
+      if (array.offset() != 0) {
+        // This is an unpleasant case. Because the offsets are different for
+        // each child array, when we have a sliced array, we need to "rebase"
+        // the value_offsets for each array
+
+        const int32_t* unshifted_offsets = array.raw_value_offsets();
+        const uint8_t* type_ids = array.raw_type_ids();
+
+        // Allocate the shifted offsets
+        std::shared_ptr<MutableBuffer> shifted_offsets_buffer;
+        RETURN_NOT_OK(AllocateBuffer(
+            pool_, array.length() * sizeof(int32_t), &shifted_offsets_buffer));
+        int32_t* shifted_offsets =
+            reinterpret_cast<int32_t*>(shifted_offsets_buffer->mutable_data());
+
+        for (int64_t i = 0; i < array.length(); ++i) {
+          const uint8_t code = type_ids[i];
+          int32_t shift = child_offsets[code];
+          if (shift == -1) { child_offsets[code] = shift = unshifted_offsets[i]; }
+          shifted_offsets[i] = unshifted_offsets[i] - shift;
+
+          // Update the child length to account for observed value
+          ++child_lengths[code];
+        }
+
+        value_offsets = shifted_offsets_buffer;
+      }
+      buffers_.push_back(value_offsets);
+
+      // Visit children and slice accordingly
+      for (int i = 0; i < type.num_children(); ++i) {
+        std::shared_ptr<Array> child = array.child(i);
+        if (array.offset() != 0) {
+          const uint8_t code = type.type_codes[i];
+          child = child->Slice(child_offsets[code], child_lengths[code]);
+        }
+        RETURN_NOT_OK(VisitArray(*child));
+      }
+    } else {
+      for (std::shared_ptr<Array> child : array.children()) {
+        // Sparse union, slicing is simpler
+        if (array.offset() != 0) {
+          // If offset is non-zero, slice the child array
+          child = child->Slice(array.offset(), array.length());
+        }
+        RETURN_NOT_OK(VisitArray(*child));
+      }
+    }
+    ++max_recursion_depth_;
+    return Status::OK();
+  }
+
+  Status Visit(const DictionaryArray& array) override {
+    // Dictionary written out separately. Slice offset contained in the indices
+    return array.indices()->Accept(this);
+  }
+
+  // In some cases, intermediate buffers may need to be allocated (with sliced arrays)
+  MemoryPool* pool_;
+
+  std::vector<FieldMetadata> field_nodes_;
+  std::vector<BufferMetadata> buffer_meta_;
+  std::vector<std::shared_ptr<Buffer>> buffers_;
+
+  int64_t max_recursion_depth_;
+  int64_t buffer_start_offset_;
+};
+
+class DictionaryWriter : public RecordBatchWriter {
+ public:
+  using RecordBatchWriter::RecordBatchWriter;
+
+  Status WriteMetadataMessage(
+      int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
+    return WriteDictionaryMessage(dictionary_id_, static_cast<int32_t>(num_rows),
+        body_length, field_nodes_, buffer_meta_, out);
+  }
+
+  Status Write(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
+      io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
+    dictionary_id_ = dictionary_id;
+
+    // Make a dummy record batch. A bit tedious as we have to make a schema
+    std::vector<std::shared_ptr<Field>> fields = {
+        arrow::field("dictionary", dictionary->type())};
+    auto schema = std::make_shared<Schema>(fields);
+    RecordBatch batch(schema, dictionary->length(), {dictionary});
+
+    return RecordBatchWriter::Write(batch, dst, metadata_length, body_length);
+  }
+
+ private:
+  // TODO(wesm): Setting this in Write is a bit unclean, but it works
+  int64_t dictionary_id_;
+};
+
+Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+    MemoryPool* pool, int max_recursion_depth) {
+  RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
+  return writer.Write(batch, dst, metadata_length, body_length);
+}
+
+Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
+    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
+    int64_t* body_length, MemoryPool* pool) {
+  DictionaryWriter writer(pool, buffer_start_offset, kMaxNestingDepth);
+  return writer.Write(dictionary_id, dictionary, dst, metadata_length, body_length);
+}
+
+Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
+  RecordBatchWriter writer(default_memory_pool(), 0, kMaxNestingDepth);
+  RETURN_NOT_OK(writer.GetTotalSize(batch, size));
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // Stream writer implementation
 
@@ -199,38 +681,6 @@ Status StreamWriter::Close() {
 // ----------------------------------------------------------------------
 // File writer implementation
 
-static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
-FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
-  std::vector<flatbuf::Block> fb_blocks;
-
-  for (const FileBlock& block : blocks) {
-    fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
-  }
-
-  return fbb.CreateVectorOfStructs(fb_blocks);
-}
-
-Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
-    io::OutputStream* out) {
-  FBB fbb;
-
-  flatbuffers::Offset<flatbuf::Schema> fb_schema;
-  RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
-
-  auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
-  auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
-
-  auto footer = flatbuf::CreateFooter(
-      fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
-
-  fbb.Finish(footer);
-
-  int32_t size = fbb.GetSize();
-
-  return out->Write(fbb.GetBufferPointer(), size);
-}
-
 class FileWriter::FileWriterImpl : public StreamWriter::StreamWriterImpl {
  public:
   using BASE = StreamWriter::StreamWriterImpl;
@@ -283,5 +733,31 @@ Status FileWriter::Close() {
   return impl_->Close();
 }
 
+// ----------------------------------------------------------------------
+// Write record batches with 64-bit size metadata
+
+class LargeRecordBatchWriter : public RecordBatchWriter {
+ public:
+  using RecordBatchWriter::RecordBatchWriter;
+
+  Status CheckArrayMetadata(const Array& arr) override {
+    // No < INT32_MAX length check
+    return Status::OK();
+  }
+
+  Status WriteMetadataMessage(
+      int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
+    return WriteLargeRecordBatchMessage(
+        num_rows, body_length, field_nodes_, buffer_meta_, out);
+  }
+};
+
+Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+    MemoryPool* pool, int max_recursion_depth) {
+  LargeRecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
+  return writer.Write(batch, dst, metadata_length, body_length);
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 7aff71e18e486..1271652a35c78 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -45,6 +45,40 @@ class OutputStream;
 
 namespace ipc {
 
+// Write the RecordBatch (collection of equal-length Arrow arrays) to the
+// output stream in a contiguous block. The record batch metadata is written as
+// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
+// prefixed by its size, followed by each of the memory buffers in the batch
+// written end to end (with appropriate alignment and padding):
+//
+// <int32: metadata size> <uint8*: metadata> <buffers>
+//
+// Finally, the absolute offsets (relative to the start of the output stream)
+// to the end of the body and end of the metadata / data header (suffixed by
+// the header size) is returned in out-variables
+//
+// @param(in) buffer_start_offset: the start offset to use in the buffer metadata,
+// default should be 0
+//
+// @param(out) metadata_length: the size of the length-prefixed flatbuffer
+// including padding to a 64-byte boundary
+//
+// @param(out) body_length: the size of the contiguous buffer block plus
+// padding bytes
+Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+    MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
+
+// Write Array as a DictionaryBatch message
+Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
+    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
+    int64_t* body_length, MemoryPool* pool);
+
+// Compute the precise number of bytes needed in a contiguous memory segment to
+// write the record batch. This involves generating the complete serialized
+// Flatbuffers metadata.
+Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
+
 class ARROW_EXPORT StreamWriter {
  public:
   virtual ~StreamWriter() = default;
@@ -68,10 +102,6 @@ class ARROW_EXPORT StreamWriter {
   std::unique_ptr<StreamWriterImpl> impl_;
 };
 
-Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
-    io::OutputStream* out);
-
 class ARROW_EXPORT FileWriter : public StreamWriter {
  public:
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
@@ -86,6 +116,14 @@ class ARROW_EXPORT FileWriter : public StreamWriter {
   std::unique_ptr<FileWriterImpl> impl_;
 };
 
+// ----------------------------------------------------------------------
+
+/// EXPERIMENTAL: Write record batch using LargeRecordBatch IPC metadata. This
+/// data may not be readable by all Arrow implementations
+Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+    MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/loader.h b/cpp/src/arrow/loader.h
index f116d64f5c0c1..9b650e2da7426 100644
--- a/cpp/src/arrow/loader.h
+++ b/cpp/src/arrow/loader.h
@@ -41,11 +41,36 @@ struct DataType;
 constexpr int kMaxNestingDepth = 64;
 
 struct ARROW_EXPORT FieldMetadata {
+  FieldMetadata() {}
+  FieldMetadata(int64_t length, int64_t null_count, int64_t offset)
+      : length(length), null_count(null_count), offset(offset) {}
+
+  FieldMetadata(const FieldMetadata& other) {
+    this->length = other.length;
+    this->null_count = other.null_count;
+    this->offset = other.offset;
+  }
+
   int64_t length;
   int64_t null_count;
   int64_t offset;
 };
 
+struct ARROW_EXPORT BufferMetadata {
+  BufferMetadata() {}
+  BufferMetadata(int32_t page, int64_t offset, int64_t length)
+      : page(page), offset(offset), length(length) {}
+
+  /// The shared memory page id where to find this. Set to -1 if unused
+  int32_t page;
+
+  /// The relative offset into the memory page to the starting byte of the buffer
+  int64_t offset;
+
+  /// Absolute length in bytes of the buffer
+  int64_t length;
+};
+
 /// Implement this to create new types of Arrow data loaders
 class ARROW_EXPORT ArrayComponentSource {
  public:
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index a143d79013fb1..adc3161e9551a 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -222,6 +222,7 @@ struct ARROW_EXPORT Field {
 
   std::string ToString() const;
 };
+
 typedef std::shared_ptr<Field> FieldPtr;
 
 struct ARROW_EXPORT PrimitiveCType : public FixedWidthType {
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 3767ba9e62f4a..ba0bfd7a9e387 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -112,7 +112,21 @@ Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t
 
 bool BitmapEquals(const uint8_t* left, int64_t left_offset, const uint8_t* right,
     int64_t right_offset, int64_t bit_length) {
-  // TODO(wesm): Make this faster using word-wise comparisons
+  if (left_offset % 8 == 0 && right_offset % 8 == 0) {
+    // byte aligned, can use memcmp
+    bool bytes_equal = std::memcmp(left + left_offset / 8, right + right_offset / 8,
+                           bit_length / 8) == 0;
+    if (!bytes_equal) { return false; }
+    for (int64_t i = (bit_length / 8) * 8; i < bit_length; ++i) {
+      if (BitUtil::GetBit(left, left_offset + i) !=
+          BitUtil::GetBit(right, right_offset + i)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  // Unaligned slow case
   for (int64_t i = 0; i < bit_length; ++i) {
     if (BitUtil::GetBit(left, left_offset + i) !=
         BitUtil::GetBit(right, right_offset + i)) {
diff --git a/format/Message.fbs b/format/Message.fbs
index 8fdcc804d4706..2af26d4dc54f9 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -307,6 +307,22 @@ table RecordBatch {
   buffers: [Buffer];
 }
 
+/// ----------------------------------------------------------------------
+/// EXPERIMENTAL: A RecordBatch type that supports data with more than 2^31 - 1
+/// elements. Arrow implementations do not need to implement this type to be
+/// compliant
+
+struct LargeFieldNode {
+  length: long;
+  null_count: long;
+}
+
+table LargeRecordBatch {
+  length: long;
+  nodes: [LargeFieldNode];
+  buffers: [Buffer];
+}
+
 /// ----------------------------------------------------------------------
 /// For sending dictionary encoding information. Any Field can be
 /// dictionary-encoded, but in this case none of its children may be
@@ -324,8 +340,12 @@ table DictionaryBatch {
 
 /// This union enables us to easily send different message types without
 /// redundant storage, and in the future we can easily add new message types.
+///
+/// Arrow implementations do not need to implement all of the message types,
+/// which may include experimental metadata types. For maximum compatibility,
+/// it is best to send data using RecordBatch
 union MessageHeader {
-  Schema, DictionaryBatch, RecordBatch
+  Schema, DictionaryBatch, RecordBatch, LargeRecordBatch
 }
 
 table Message {

From cd4544df89b60641f49bbb3104043c0ae07ef8a9 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 20 Mar 2017 10:54:57 +0100
Subject: [PATCH 0386/1644] ARROW-664: [C++] Make C++ Arrow serialization
 deterministic

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #405 from pcmoritz/init-buffer-builder and squashes the following commits:

10a897f [Philipp Moritz] Initialize memory obtained by BufferBuilder to zero
---
 cpp/src/arrow/buffer.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 1647e8601f481..70c16a2dafc86 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -170,9 +170,13 @@ class ARROW_EXPORT BufferBuilder {
     // Resize(0) is a no-op
     if (elements == 0) { return Status::OK(); }
     if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
+    int64_t old_capacity = capacity_;
     RETURN_NOT_OK(buffer_->Resize(elements));
     capacity_ = buffer_->capacity();
     data_ = buffer_->mutable_data();
+    if (capacity_ > old_capacity) {
+      memset(data_ + old_capacity, 0, capacity_ - old_capacity);
+    }
     return Status::OK();
   }
 

From 02bdbf48a483b224ebfd61cf9be69cb0807e6e50 Mon Sep 17 00:00:00 2001
From: Johan Mabille <johan.mabille@gmail.com>
Date: Mon, 20 Mar 2017 10:57:57 +0100
Subject: [PATCH 0387/1644] ARROW-502 [C++/Python]: Logging memory pool

This is a simple decorator on MemoryPool that logs it call to ``std::cout``. I can improve it later if you need to log to other supports. Are you ok with the current logging format ?

Also, I'm not a cython expert so I hope the implementation of ``CLoggingMemoryPool`` is correct.

Author: Johan Mabille <johan.mabille@gmail.com>

Closes #395 from JohanMabille/memory_pool and squashes the following commits:

aa8ad5f [Johan Mabille] cython fix
f70e78a [Johan Mabille] python logging memory pool
9d1d144 [Johan Mabille] formatting
8f9164c [Johan Mabille] Logging memory pool
---
 cpp/src/arrow/memory_pool-test.cc    | 17 +++++++++++++++
 cpp/src/arrow/memory_pool.cc         | 32 ++++++++++++++++++++++++++++
 cpp/src/arrow/memory_pool.h          | 18 ++++++++++++++++
 python/pyarrow/includes/libarrow.pxd |  3 +++
 python/pyarrow/memory.pxd            |  5 ++++-
 python/pyarrow/memory.pyx            |  5 ++++-
 6 files changed, 78 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 6ab73fb103f50..8a185abca71cc 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -78,4 +78,21 @@ TEST(DefaultMemoryPoolDeathTest, MaxMemory) {
 
 #endif  // ARROW_VALGRIND
 
+TEST(LoggingMemoryPool, Logging) {
+  DefaultMemoryPool pool;
+  LoggingMemoryPool lp(&pool);
+
+  ASSERT_EQ(0, lp.max_memory());
+
+  uint8_t* data;
+  ASSERT_OK(pool.Allocate(100, &data));
+
+  uint8_t* data2;
+  ASSERT_OK(pool.Allocate(100, &data2));
+
+  pool.Free(data, 100);
+  pool.Free(data2, 100);
+
+  ASSERT_EQ(200, pool.max_memory());
+}
 }  // namespace arrow
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 5a630271a7da7..cf01a02938385 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -22,6 +22,7 @@
 #include <mutex>
 #include <sstream>
 #include <stdlib.h>
+#include <iostream>
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
@@ -134,4 +135,35 @@ MemoryPool* default_memory_pool() {
   return &default_memory_pool_;
 }
 
+LoggingMemoryPool::LoggingMemoryPool(MemoryPool* pool) : pool_(pool) {}
+
+Status LoggingMemoryPool::Allocate(int64_t size, uint8_t** out) {
+  Status s = pool_->Allocate(size, out);
+  std::cout << "Allocate: size = " << size << " - out = " << *out << std::endl;
+  return s;
+}
+
+Status LoggingMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
+  Status s = pool_->Reallocate(old_size, new_size, ptr);
+  std::cout << "Reallocate: old_size = " << old_size << " - new_size = " << new_size
+            << " - ptr = " << *ptr << std::endl;
+  return s;
+}
+
+void LoggingMemoryPool::Free(uint8_t* buffer, int64_t size) {
+  pool_->Free(buffer, size);
+  std::cout << "Free: buffer = " << buffer << " - size = " << size << std::endl;
+}
+
+int64_t LoggingMemoryPool::bytes_allocated() const {
+  int64_t nb_bytes = pool_->bytes_allocated();
+  std::cout << "bytes_allocated: " << nb_bytes << std::endl;
+  return nb_bytes;
+}
+
+int64_t LoggingMemoryPool::max_memory() const {
+  int64_t mem = pool_->max_memory();
+  std::cout << "max_memory: " << mem << std::endl;
+  return mem;
+}
 }  // namespace arrow
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 0edfda635d0e8..90bc593ab71fe 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -89,6 +89,24 @@ class ARROW_EXPORT DefaultMemoryPool : public MemoryPool {
   std::atomic<int64_t> max_memory_;
 };
 
+class ARROW_EXPORT LoggingMemoryPool : public MemoryPool {
+ public:
+  explicit LoggingMemoryPool(MemoryPool* pool);
+  virtual ~LoggingMemoryPool() = default;
+
+  Status Allocate(int64_t size, uint8_t** out) override;
+  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
+
+  void Free(uint8_t* buffer, int64_t size) override;
+
+  int64_t bytes_allocated() const override;
+
+  int64_t max_memory() const override;
+
+ private:
+  MemoryPool* pool_;
+};
+
 ARROW_EXPORT MemoryPool* default_memory_pool();
 
 }  // namespace arrow
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index dee7fd4f8e4e5..705fe6b4a55ca 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -104,6 +104,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CMemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
 
+    cdef cppclass CLoggingMemoryPool" arrow::LoggingMemoryPool"(CMemoryPool):
+        CLoggingMemoryPool(CMemoryPool*)
+        
     cdef cppclass CBuffer" arrow::Buffer":
         uint8_t* data()
         int64_t size()
diff --git a/python/pyarrow/memory.pxd b/python/pyarrow/memory.pxd
index 3079ccb807b0d..bb1af85c8ea65 100644
--- a/python/pyarrow/memory.pxd
+++ b/python/pyarrow/memory.pxd
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.includes.libarrow cimport CMemoryPool
+from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
 
 
 cdef class MemoryPool:
@@ -24,4 +24,7 @@ cdef class MemoryPool:
 
     cdef init(self, CMemoryPool* pool)
 
+cdef class LoggingMemoryPool(MemoryPool):
+    pass
+
 cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool)
diff --git a/python/pyarrow/memory.pyx b/python/pyarrow/memory.pyx
index 18a6de4f15392..98dbf66c8e0af 100644
--- a/python/pyarrow/memory.pyx
+++ b/python/pyarrow/memory.pyx
@@ -19,7 +19,7 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from pyarrow.includes.libarrow cimport CMemoryPool
+from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
 from pyarrow.includes.pyarrow cimport set_default_memory_pool, get_memory_pool
 
 cdef class MemoryPool:
@@ -35,6 +35,9 @@ cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool):
     else:
         return memory_pool.pool
 
+cdef class LoggingMemoryPool(MemoryPool):
+    pass
+
 def default_pool():
     cdef: 
         MemoryPool pool = MemoryPool()

From 6cd82c2a294562d1d16a4767b32f072056f396a3 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 20 Mar 2017 17:55:56 +0100
Subject: [PATCH 0388/1644] ARROW-671: [GLib] Install missing license file

Author: Kouhei Sutou <kou@clear-code.com>

Closes #406 from kou/glib-install-missing-license-file and squashes the following commits:

8e452d4 [Kouhei Sutou] [GLib] Install missing license file
---
 c_glib/.gitignore  | 1 +
 c_glib/Makefile.am | 6 ++++++
 c_glib/autogen.sh  | 2 ++
 3 files changed, 9 insertions(+)

diff --git a/c_glib/.gitignore b/c_glib/.gitignore
index 38e33a2cd88e7..e57a0594c1af3 100644
--- a/c_glib/.gitignore
+++ b/c_glib/.gitignore
@@ -8,6 +8,7 @@ Makefile.in
 *.lo
 *.la
 *~
+/LICENSE.txt
 /*.tar.gz
 /aclocal.m4
 /autom4te.cache/
diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am
index 076f9be08524b..c078b0889d4ff 100644
--- a/c_glib/Makefile.am
+++ b/c_glib/Makefile.am
@@ -23,4 +23,10 @@ SUBDIRS =					\
 	example
 
 EXTRA_DIST =					\
+	README.md				\
+	LICENSE.txt				\
 	version
+
+doc_DATA =					\
+	README.md				\
+	LICENSE.txt
diff --git a/c_glib/autogen.sh b/c_glib/autogen.sh
index 08e33e6ca07c0..6e2036da6406b 100755
--- a/c_glib/autogen.sh
+++ b/c_glib/autogen.sh
@@ -25,6 +25,8 @@ ruby \
     ../java/pom.xml > \
     version
 
+cp ../LICENSE.txt ./
+
 mkdir -p m4
 
 gtkdocize --copy --docdir doc/reference

From 98a52b4823f3cd0880eaef066dc932f533170292 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 20 Mar 2017 10:44:15 -0700
Subject: [PATCH 0389/1644] ARROW-316: [Format] Changes to Date metadata format
 per discussion in ARROW-316

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #390 from wesm/ARROW-316 and squashes the following commits:

6828e05 [Wes McKinney] Format changes for Date per discussion in ARROW-316
---
 format/Message.fbs | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/format/Message.fbs b/format/Message.fbs
index 2af26d4dc54f9..e56366d436eb9 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -81,8 +81,19 @@ table Decimal {
   scale: int;
 }
 
-/// Date is a 64-bit type representing milliseconds since the UNIX epoch
+enum DateUnit: short {
+  DAY,
+  MILLISECOND
+}
+
+/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
+/// epoch (1970-01-01), stored in either of two units:
+///
+/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
+///   leap seconds), where the values are evenly divisible by 86400000
+/// * Days (32 bits) since the UNIX epoch
 table Date {
+  unit: DateUnit;
 }
 
 enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }

From a8bf0fbc832fef3e2a6a9ec075db03007a26442a Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 21 Mar 2017 15:21:44 -0700
Subject: [PATCH 0390/1644] ARROW-673: [Java] Support additional Time metadata

Author: Julien Le Dem <julien@dremio.com>

Closes #407 from julienledem/time_md and squashes the following commits:

3f721e2 [Julien Le Dem] ARROW-673: [Java] Support additional Time metadata
---
 .../src/main/codegen/data/ArrowTypes.tdd      |  2 +-
 .../templates/NullableValueVectors.java       | 39 +------------------
 .../arrow/vector/schema/TypeLayout.java       |  2 +-
 .../org/apache/arrow/vector/types/Types.java  |  7 +++-
 .../arrow/vector/types/pojo/TestSchema.java   |  2 +-
 5 files changed, 10 insertions(+), 42 deletions(-)

diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 01465e585dad2..8f997524fccfc 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -58,7 +58,7 @@
     },
     {
       name: "Time",
-      fields: []
+      fields: [{name: "unit", type: short, valueType: TimeUnit}, {name: "bitWidth", type: int}]
     },
     {
       name: "Timestamp",
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index b3e10e3fa87a2..ec2ce7930cf5d 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -84,43 +84,8 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     values = new ${valuesName}(valuesField, allocator);
     mutator = new Mutator();
     accessor = new Accessor();
-  <#if minor.class == "TinyInt" ||
-        minor.class == "SmallInt" ||
-        minor.class == "Int" ||
-        minor.class == "BigInt">
-    field = new Field(name, true, new Int(${type.width} * 8, true), dictionary, null);
-  <#elseif minor.class == "UInt1" ||
-        minor.class == "UInt2" ||
-        minor.class == "UInt4" ||
-        minor.class == "UInt8">
-    field = new Field(name, true, new Int(${type.width} * 8, false), dictionary, null);
-  <#elseif minor.class == "Date">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Date(), dictionary, null);
-  <#elseif minor.class == "Time">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), dictionary, null);
-  <#elseif minor.class == "Float4">
-    field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), dictionary, null);
-  <#elseif minor.class == "Float8">
-    field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), dictionary, null);
-  <#elseif minor.class == "TimeStampSec">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND), dictionary, null);
-  <#elseif minor.class == "TimeStampMilli">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), dictionary, null);
-  <#elseif minor.class == "TimeStampMicro">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND), dictionary, null);
-  <#elseif minor.class == "TimeStampNano">
-    field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND), dictionary, null);
-  <#elseif minor.class == "IntervalDay">
-    field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), dictionary, null);
-  <#elseif minor.class == "IntervalYear">
-    field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.YEAR_MONTH), dictionary, null);
-  <#elseif minor.class == "VarChar">
-    field = new Field(name, true, new Utf8(), dictionary, null);
-  <#elseif minor.class == "VarBinary">
-    field = new Field(name, true, new Binary(), dictionary, null);
-  <#elseif minor.class == "Bit">
-    field = new Field(name, true, new Bool(), dictionary, null);
-  </#if>
+    ArrowType type = Types.MinorType.${minor.class?upper_case}.getType();
+    field = new Field(name, true, type, dictionary, null);
     innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
         bits,
         <#if type.major = "VarLen">
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 0b586914bdf85..69d550fc9f799 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -164,7 +164,7 @@ public TypeLayout visit(Date type) {
 
       @Override
       public TypeLayout visit(Time type) {
-        return newFixedWidthTypeLayout(dataVector(64));
+        return newFixedWidthTypeLayout(dataVector(type.getBitWidth()));
       }
 
       @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 8f2d04224c0fd..7cbf3c5bb5e36 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -108,7 +108,7 @@ public class Types {
   private static final Field UINT4_FIELD = new Field("", true, new Int(32, false), null);
   private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
   private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
-  private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
+  private static final Field TIME_FIELD = new Field("", true, new Time(TimeUnit.MILLISECOND, 32), null);
   private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND), null);
   private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null);
   private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND), null);
@@ -235,7 +235,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new DateWriterImpl((NullableDateVector) vector);
       }
     },
-    TIME(Time.INSTANCE) {
+    TIME(new Time(TimeUnit.MILLISECOND, 32)) {
       @Override
       public Field getField() {
         return TIME_FIELD;
@@ -639,6 +639,9 @@ public MinorType visit(FloatingPoint type) {
       }
 
       @Override public MinorType visit(Time type) {
+        if (type.getUnit() != TimeUnit.MILLISECOND || type.getBitWidth() != 32) {
+          throw new IllegalArgumentException("Only milliseconds on 32 bits supported for now: " + type);
+        }
         return MinorType.TIME;
       }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index f04c78ec45d97..5b74c54c9159f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -71,7 +71,7 @@ public void testAll() throws IOException {
         field("i", new ArrowType.Bool()),
         field("j", new ArrowType.Decimal(5, 5)),
         field("k", new ArrowType.Date()),
-        field("l", new ArrowType.Time()),
+        field("l", new ArrowType.Time(TimeUnit.MILLISECOND, 32)),
         field("m", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
         field("n", new ArrowType.Interval(IntervalUnit.DAY_TIME))
         ));

From a9a570139966593ed84ddd842da73b60ace89e1e Mon Sep 17 00:00:00 2001
From: Tsuyoshi Ozawa <ozawa@apache.org>
Date: Tue, 21 Mar 2017 15:24:19 -0700
Subject: [PATCH 0391/1644] ARROW-208: Add checkstyle policy to java project

Author: Tsuyoshi Ozawa <ozawa@apache.org>

Closes #96 from oza/ARROW-208 and squashes the following commits:

809e729 [Tsuyoshi Ozawa] reformatted code in memory and tools dir with IDE
40ee6a3 [Tsuyoshi Ozawa] ARROW-208: Add checkstyle policy to java project
---
 .../main/java/io/netty/buffer/ArrowBuf.java   | 219 +++----
 .../io/netty/buffer/ExpandableByteBuf.java    |   8 +-
 .../java/io/netty/buffer/LargeBuffer.java     |   9 +-
 .../netty/buffer/MutableWrappedByteBuf.java   |  18 +-
 .../netty/buffer/PooledByteBufAllocatorL.java |  84 +--
 .../buffer/UnsafeDirectLittleEndian.java      |  52 +-
 .../org/apache/arrow/memory/Accountant.java   | 102 ++--
 .../arrow/memory/AllocationListener.java      |   4 +-
 .../arrow/memory/AllocationManager.java       | 177 +++---
 .../arrow/memory/AllocationReservation.java   |  20 +-
 .../memory/AllocatorClosedException.java      |   6 +-
 .../arrow/memory/ArrowByteBufAllocator.java   |  14 +-
 .../apache/arrow/memory/BaseAllocator.java    | 539 +++++++++---------
 .../apache/arrow/memory/BoundsChecking.java   |   7 +-
 .../apache/arrow/memory/BufferAllocator.java  |  80 +--
 .../apache/arrow/memory/BufferManager.java    |  15 +-
 .../apache/arrow/memory/ChildAllocator.java   |  18 +-
 .../arrow/memory/OutOfMemoryException.java    |  13 +-
 .../apache/arrow/memory/RootAllocator.java    |   6 +-
 .../org/apache/arrow/memory/package-info.java |  49 +-
 .../arrow/memory/util/AssertionUtil.java      |  15 +-
 .../arrow/memory/util/AutoCloseableLock.java  |   5 +-
 .../arrow/memory/util/HistoricalLog.java      |  85 +--
 .../apache/arrow/memory/util/StackTrace.java  |  15 +-
 java/pom.xml                                  |  55 ++
 .../org/apache/arrow/tools/EchoServer.java    | 102 ++--
 .../org/apache/arrow/tools/FileRoundtrip.java |  29 +-
 .../org/apache/arrow/tools/FileToStream.java  |  17 +-
 .../org/apache/arrow/tools/Integration.java   | 133 +++--
 .../org/apache/arrow/tools/StreamToFile.java  |  17 +-
 .../arrow/tools/ArrowFileTestFixtures.java    |  28 +-
 .../apache/arrow/tools/EchoServerTest.java    |  66 ++-
 .../apache/arrow/tools/TestFileRoundtrip.java |  15 +-
 .../apache/arrow/tools/TestIntegration.java   | 159 +++---
 34 files changed, 1218 insertions(+), 963 deletions(-)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index 95d2be5a43a36..e777b5a6a5d58 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -6,27 +6,21 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package io.netty.buffer;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-import java.nio.charset.Charset;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.base.Preconditions;
+
+import io.netty.util.internal.PlatformDependent;
 
 import org.apache.arrow.memory.AllocationManager.BufferLedger;
 import org.apache.arrow.memory.ArrowByteBufAllocator;
@@ -37,15 +31,23 @@
 import org.apache.arrow.memory.BufferManager;
 import org.apache.arrow.memory.util.HistoricalLog;
 
-import com.google.common.base.Preconditions;
-
-import io.netty.util.internal.PlatformDependent;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.nio.charset.Charset;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
+
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ArrowBuf.class);
 
   private static final AtomicLong idGenerator = new AtomicLong(0);
-
+  private static final int LOG_BYTES_PER_ROW = 10;
   private final long id = idGenerator.incrementAndGet();
   private final AtomicInteger refCnt;
   private final UnsafeDirectLittleEndian udle;
@@ -55,9 +57,9 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
   private final BufferManager bufManager;
   private final ArrowByteBufAllocator alloc;
   private final boolean isEmpty;
-  private volatile int length;
   private final HistoricalLog historicalLog = BaseAllocator.DEBUG ?
       new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", id) : null;
+  private volatile int length;
 
   public ArrowBuf(
       final AtomicInteger refCnt,
@@ -85,6 +87,17 @@ public ArrowBuf(
 
   }
 
+  public static String bufferState(final ByteBuf buf) {
+    final int cap = buf.capacity();
+    final int mcap = buf.maxCapacity();
+    final int ri = buf.readerIndex();
+    final int rb = buf.readableBytes();
+    final int wi = buf.writerIndex();
+    final int wb = buf.writableBytes();
+    return String.format("cap/max: %d/%d, ri: %d, rb: %d, wi: %d, wb: %d",
+        cap, mcap, ri, rb, wi, wb);
+  }
+
   public ArrowBuf reallocIfNeeded(final int size) {
     Preconditions.checkArgument(size >= 0, "reallocation size must be non-negative");
 
@@ -95,7 +108,8 @@ public ArrowBuf reallocIfNeeded(final int size) {
     if (bufManager != null) {
       return bufManager.replace(this, size);
     } else {
-      throw new UnsupportedOperationException("Realloc is only available in the context of an operator's UDFs");
+      throw new UnsupportedOperationException("Realloc is only available in the context of an " +
+          "operator's UDFs");
     }
   }
 
@@ -128,14 +142,13 @@ private final void checkIndexD(int index, int fieldLength) {
 
   /**
    * Allows a function to determine whether not reading a particular string of bytes is valid.
-   *
-   * Will throw an exception if the memory is not readable for some reason. Only doesn't something in the case that
+   * <p>
+   * Will throw an exception if the memory is not readable for some reason. Only doesn't
+   * something in the case that
    * AssertionUtil.BOUNDS_CHECKING_ENABLED is true.
    *
-   * @param start
-   *          The starting position of the bytes to be read.
-   * @param end
-   *          The exclusive endpoint of the bytes to be read.
+   * @param start The starting position of the bytes to be read.
+   * @param end   The exclusive endpoint of the bytes to be read.
    */
   public void checkBytes(int start, int end) {
     if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
@@ -156,17 +169,21 @@ private void ensure(int width) {
   }
 
   /**
-   * Create a new ArrowBuf that is associated with an alternative allocator for the purposes of memory ownership and
-   * accounting. This has no impact on the reference counting for the current ArrowBuf except in the situation where the
+   * Create a new ArrowBuf that is associated with an alternative allocator for the purposes of
+   * memory ownership and
+   * accounting. This has no impact on the reference counting for the current ArrowBuf except in
+   * the situation where the
    * passed in Allocator is the same as the current buffer.
-   *
-   * This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a
-   * reference count of 1 (in the case that this is the first time this memory is being associated with the new
-   * allocator) or the current value of the reference count + 1 for the other AllocationManager/BufferLedger combination
+   * <p>
+   * This operation has no impact on the reference count of this ArrowBuf. The newly created
+   * ArrowBuf with either have a
+   * reference count of 1 (in the case that this is the first time this memory is being
+   * associated with the new
+   * allocator) or the current value of the reference count + 1 for the other
+   * AllocationManager/BufferLedger combination
    * in the case that the provided allocator already had an association to this underlying memory.
    *
-   * @param target
-   *          The target allocator to create an association with.
+   * @param target The target allocator to create an association with.
    * @return A new ArrowBuf which shares the same underlying memory as this ArrowBuf.
    */
   public ArrowBuf retain(BufferAllocator target) {
@@ -186,28 +203,39 @@ public ArrowBuf retain(BufferAllocator target) {
   }
 
   /**
-   * Transfer the memory accounting ownership of this ArrowBuf to another allocator. This will generate a new ArrowBuf
-   * that carries an association with the underlying memory of this ArrowBuf. If this ArrowBuf is connected to the
-   * owning BufferLedger of this memory, that memory ownership/accounting will be transferred to the taret allocator. If
-   * this ArrowBuf does not currently own the memory underlying it (and is only associated with it), this does not
+   * Transfer the memory accounting ownership of this ArrowBuf to another allocator. This will
+   * generate a new ArrowBuf
+   * that carries an association with the underlying memory of this ArrowBuf. If this ArrowBuf is
+   * connected to the
+   * owning BufferLedger of this memory, that memory ownership/accounting will be transferred to
+   * the taret allocator. If
+   * this ArrowBuf does not currently own the memory underlying it (and is only associated with
+   * it), this does not
    * transfer any ownership to the newly created ArrowBuf.
-   *
-   * This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a
-   * reference count of 1 (in the case that this is the first time this memory is being associated with the new
-   * allocator) or the current value of the reference count for the other AllocationManager/BufferLedger combination in
+   * <p>
+   * This operation has no impact on the reference count of this ArrowBuf. The newly created
+   * ArrowBuf with either have a
+   * reference count of 1 (in the case that this is the first time this memory is being
+   * associated with the new
+   * allocator) or the current value of the reference count for the other
+   * AllocationManager/BufferLedger combination in
    * the case that the provided allocator already had an association to this underlying memory.
-   *
-   * Transfers will always succeed, even if that puts the other allocator into an overlimit situation. This is possible
-   * due to the fact that the original owning allocator may have allocated this memory out of a local reservation
-   * whereas the target allocator may need to allocate new memory from a parent or RootAllocator. This operation is done
-   * in a mostly-lockless but consistent manner. As such, the overlimit==true situation could occur slightly prematurely
-   * to an actual overlimit==true condition. This is simply conservative behavior which means we may return overlimit
+   * <p>
+   * Transfers will always succeed, even if that puts the other allocator into an overlimit
+   * situation. This is possible
+   * due to the fact that the original owning allocator may have allocated this memory out of a
+   * local reservation
+   * whereas the target allocator may need to allocate new memory from a parent or RootAllocator.
+   * This operation is done
+   * in a mostly-lockless but consistent manner. As such, the overlimit==true situation could
+   * occur slightly prematurely
+   * to an actual overlimit==true condition. This is simply conservative behavior which means we
+   * may return overlimit
    * slightly sooner than is necessary.
    *
-   * @param target
-   *          The allocator to transfer ownership to.
-   * @return A new transfer result with the impact of the transfer (whether it was overlimit) as well as the newly
-   *         created ArrowBuf.
+   * @param target The allocator to transfer ownership to.
+   * @return A new transfer result with the impact of the transfer (whether it was overlimit) as
+   * well as the newly created ArrowBuf.
    */
   public TransferResult transferOwnership(BufferAllocator target) {
 
@@ -223,28 +251,6 @@ public TransferResult transferOwnership(BufferAllocator target) {
     return new TransferResult(allocationFit, newBuf);
   }
 
-  /**
-   * The outcome of a Transfer.
-   */
-  public class TransferResult {
-
-    /**
-     * Whether this transfer fit within the target allocator's capacity.
-     */
-    public final boolean allocationFit;
-
-    /**
-     * The newly created buffer associated with the target allocator.
-     */
-    public final ArrowBuf buffer;
-
-    private TransferResult(boolean allocationFit, ArrowBuf buffer) {
-      this.allocationFit = allocationFit;
-      this.buffer = buffer;
-    }
-
-  }
-
   @Override
   public boolean release() {
     return release(1);
@@ -261,7 +267,8 @@ public boolean release(int decrement) {
     }
 
     if (decrement < 1) {
-      throw new IllegalStateException(String.format("release(%d) argument is not positive. Buffer Info: %s",
+      throw new IllegalStateException(String.format("release(%d) argument is not positive. Buffer" +
+              " Info: %s",
           decrement, toVerboseString()));
     }
 
@@ -273,7 +280,8 @@ public boolean release(int decrement) {
 
     if (refCnt < 0) {
       throw new IllegalStateException(
-          String.format("ArrowBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString()));
+          String.format("ArrowBuf[%d] refCnt has gone negative. Buffer Info: %s", id,
+              toVerboseString()));
     }
 
     return refCnt == 0;
@@ -299,7 +307,8 @@ public synchronized ArrowBuf capacity(int newCapacity) {
       return this;
     }
 
-    throw new UnsupportedOperationException("Buffers don't support resizing that increases the size.");
+    throw new UnsupportedOperationException("Buffers don't support resizing that increases the " +
+        "size.");
   }
 
   @Override
@@ -354,17 +363,6 @@ public ArrowBuf slice() {
     return slice(readerIndex(), readableBytes());
   }
 
-  public static String bufferState(final ByteBuf buf) {
-    final int cap = buf.capacity();
-    final int mcap = buf.maxCapacity();
-    final int ri = buf.readerIndex();
-    final int rb = buf.readableBytes();
-    final int wi = buf.writerIndex();
-    final int wb = buf.writableBytes();
-    return String.format("cap/max: %d/%d, ri: %d, rb: %d, wi: %d, wb: %d",
-        cap, mcap, ri, rb, wi, wb);
-  }
-
   @Override
   public ArrowBuf slice(int index, int length) {
 
@@ -373,7 +371,8 @@ public ArrowBuf slice(int index, int length) {
     }
 
     /*
-     * Re the behavior of reference counting, see http://netty.io/wiki/reference-counted-objects.html#wiki-h3-5, which
+     * Re the behavior of reference counting, see http://netty.io/wiki/reference-counted-objects
+     * .html#wiki-h3-5, which
      * explains that derived buffers share their reference count with their parent
      */
     final ArrowBuf newBuf = ledger.newArrowBuf(offset + index, length);
@@ -408,12 +407,12 @@ public ByteBuffer internalNioBuffer(int index, int length) {
 
   @Override
   public ByteBuffer[] nioBuffers() {
-    return new ByteBuffer[] { nioBuffer() };
+    return new ByteBuffer[]{nioBuffer()};
   }
 
   @Override
   public ByteBuffer[] nioBuffers(int index, int length) {
-    return new ByteBuffer[] { nioBuffer(index, length) };
+    return new ByteBuffer[]{nioBuffer(index, length)};
   }
 
   @Override
@@ -443,7 +442,8 @@ public long memoryAddress() {
 
   @Override
   public String toString() {
-    return String.format("ArrowBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity());
+    return String.format("ArrowBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset +
+        capacity());
   }
 
   @Override
@@ -738,7 +738,8 @@ public ArrowBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
   public ArrowBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
     if (src.isDirect()) {
       checkIndex(index, length);
-      PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this.memoryAddress() + index,
+      PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this
+              .memoryAddress() + index,
           length);
     } else {
       if (srcIndex == 0 && src.capacity() == length) {
@@ -788,7 +789,8 @@ public void close() {
   }
 
   /**
-   * Returns the possible memory consumed by this ArrowBuf in the worse case scenario. (not shared, connected to larger
+   * Returns the possible memory consumed by this ArrowBuf in the worse case scenario. (not
+   * shared, connected to larger
    * underlying buffer of allocated memory)
    *
    * @return Size in bytes.
@@ -798,7 +800,8 @@ public int getPossibleMemoryConsumed() {
   }
 
   /**
-   * Return that is Accounted for by this buffer (and its potentially shared siblings within the context of the
+   * Return that is Accounted for by this buffer (and its potentially shared siblings within the
+   * context of the
    * associated allocator).
    *
    * @return Size in bytes.
@@ -807,15 +810,11 @@ public int getActualMemoryConsumed() {
     return ledger.getAccountedSize();
   }
 
-  private final static int LOG_BYTES_PER_ROW = 10;
-
   /**
    * Return the buffer's byte contents in the form of a hex dump.
    *
-   * @param start
-   *          the starting byte index
-   * @param length
-   *          how many bytes to log
+   * @param start  the starting byte index
+   * @param length how many bytes to log
    * @return A hex dump in a String.
    */
   public String toHexString(final int start, final int length) {
@@ -878,5 +877,27 @@ public ArrowBuf writerIndex(int writerIndex) {
     return this;
   }
 
+  /**
+   * The outcome of a Transfer.
+   */
+  public class TransferResult {
+
+    /**
+     * Whether this transfer fit within the target allocator's capacity.
+     */
+    public final boolean allocationFit;
+
+    /**
+     * The newly created buffer associated with the target allocator.
+     */
+    public final ArrowBuf buffer;
+
+    private TransferResult(boolean allocationFit, ArrowBuf buffer) {
+      this.allocationFit = allocationFit;
+      this.buffer = buffer;
+    }
+
+  }
+
 
 }
diff --git a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java b/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
index 7fb884daa3952..9f8af93109739 100644
--- a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java
@@ -6,21 +6,23 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package io.netty.buffer;
 
 import org.apache.arrow.memory.BufferAllocator;
 
 /**
- * Allows us to decorate ArrowBuf to make it expandable so that we can use them in the context of the Netty framework
+ * Allows us to decorate ArrowBuf to make it expandable so that we can use them in the context of
+ * the Netty framework
  * (thus supporting RPC level memory accounting).
  */
 public class ExpandableByteBuf extends MutableWrappedByteBuf {
diff --git a/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java b/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
index c026e430d77f3..9a6e402dad53e 100644
--- a/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
+++ b/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java
@@ -6,21 +6,24 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package io.netty.buffer;
 
 /**
- * A MutableWrappedByteBuf that also maintains a metric of the number of huge buffer bytes and counts.
+ * A MutableWrappedByteBuf that also maintains a metric of the number of huge buffer bytes and
+ * counts.
  */
 public class LargeBuffer extends MutableWrappedByteBuf {
+
   public LargeBuffer(ByteBuf buffer) {
     super(buffer);
   }
diff --git a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
index 5709473135e4b..a5683adccbc32 100644
--- a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
@@ -6,15 +6,16 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package io.netty.buffer;
 
 import java.io.IOException;
@@ -26,16 +27,12 @@
 import java.nio.channels.ScatteringByteChannel;
 
 /**
- * This is basically a complete copy of DuplicatedByteBuf. We copy because we want to override some behaviors and make
+ * This is basically a complete copy of DuplicatedByteBuf. We copy because we want to override
+ * some behaviors and make
  * buffer mutable.
  */
 abstract class MutableWrappedByteBuf extends AbstractByteBuf {
 
-  @Override
-  public ByteBuffer nioBuffer(int index, int length) {
-    return unwrap().nioBuffer(index, length);
-  }
-
   ByteBuf buffer;
 
   public MutableWrappedByteBuf(ByteBuf buffer) {
@@ -50,6 +47,11 @@ public MutableWrappedByteBuf(ByteBuf buffer) {
     setIndex(buffer.readerIndex(), buffer.writerIndex());
   }
 
+  @Override
+  public ByteBuffer nioBuffer(int index, int length) {
+    return unwrap().nioBuffer(index, length);
+  }
+
   @Override
   public ByteBuf unwrap() {
     return buffer;
diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index a843ac5586e79..b6de2e3aa2acb 100644
--- a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -6,42 +6,44 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package io.netty.buffer;
 
-import static org.apache.arrow.memory.util.AssertionUtil.ASSERT_ENABLED;
+import io.netty.util.internal.StringUtil;
+
+import org.apache.arrow.memory.OutOfMemoryException;
 
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.arrow.memory.OutOfMemoryException;
-
-import io.netty.util.internal.StringUtil;
+import static org.apache.arrow.memory.util.AssertionUtil.ASSERT_ENABLED;
 
 /**
- * The base allocator that we use for all of Arrow's memory management. Returns UnsafeDirectLittleEndian buffers.
+ * The base allocator that we use for all of Arrow's memory management. Returns
+ * UnsafeDirectLittleEndian buffers.
  */
 public class PooledByteBufAllocatorL {
-  private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("arrow.allocator");
 
-  private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
+  private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("arrow" +
+      ".allocator");
 
+  private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
+  public final UnsafeDirectLittleEndian empty;
   private final AtomicLong hugeBufferSize = new AtomicLong(0);
   private final AtomicLong hugeBufferCount = new AtomicLong(0);
   private final AtomicLong normalBufferSize = new AtomicLong(0);
   private final AtomicLong normalBufferCount = new AtomicLong(0);
-
   private final InnerAllocator allocator;
-  public final UnsafeDirectLittleEndian empty;
 
   public PooledByteBufAllocatorL() {
     allocator = new InnerAllocator();
@@ -78,6 +80,7 @@ public long getNormalBufferCount() {
   }
 
   private static class AccountedUnsafeDirectLittleEndian extends UnsafeDirectLittleEndian {
+
     private final long initialCapacity;
     private final AtomicLong count;
     private final AtomicLong size;
@@ -89,7 +92,8 @@ private AccountedUnsafeDirectLittleEndian(LargeBuffer buf, AtomicLong count, Ato
       this.size = size;
     }
 
-    private AccountedUnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong count, AtomicLong size) {
+    private AccountedUnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong count,
+                                              AtomicLong size) {
       super(buf);
       this.initialCapacity = buf.capacity();
       this.count = count;
@@ -119,6 +123,7 @@ public boolean release(int decrement) {
   }
 
   private class InnerAllocator extends PooledByteBufAllocator {
+
     private final PoolArena<ByteBuffer>[] directArenas;
     private final MemoryStatusThread statusThread;
     private final int chunkSize;
@@ -131,7 +136,8 @@ public InnerAllocator() {
         f.setAccessible(true);
         this.directArenas = (PoolArena<ByteBuffer>[]) f.get(this);
       } catch (Exception e) {
-        throw new RuntimeException("Failure while initializing allocator.  Unable to retrieve direct arenas field.", e);
+        throw new RuntimeException("Failure while initializing allocator.  Unable to retrieve " +
+            "direct arenas field.", e);
       }
 
       this.chunkSize = directArenas[0].chunkSize;
@@ -158,7 +164,8 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa
           hugeBufferCount.incrementAndGet();
 
           // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
-          return new AccountedUnsafeDirectLittleEndian(new LargeBuffer(buf), hugeBufferCount, hugeBufferSize);
+          return new AccountedUnsafeDirectLittleEndian(new LargeBuffer(buf), hugeBufferCount,
+              hugeBufferSize);
         } else {
           // within chunk, use arena.
           ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity);
@@ -173,7 +180,8 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa
           normalBufferSize.addAndGet(buf.capacity());
           normalBufferCount.incrementAndGet();
 
-          return new AccountedUnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount, normalBufferSize);
+          return new AccountedUnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf,
+              normalBufferCount, normalBufferSize);
         }
 
       } else {
@@ -183,7 +191,8 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa
 
     private UnsupportedOperationException fail() {
       return new UnsupportedOperationException(
-          "Arrow requires that the JVM used supports access sun.misc.Unsafe.  This platform didn't provide that functionality.");
+          "Arrow requires that the JVM used supports access sun.misc.Unsafe.  This platform " +
+              "didn't provide that functionality.");
     }
 
     @Override
@@ -203,7 +212,8 @@ public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
 
     private void validate(int initialCapacity, int maxCapacity) {
       if (initialCapacity < 0) {
-        throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)");
+        throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: " +
+            "0+)");
       }
       if (initialCapacity > maxCapacity) {
         throw new IllegalArgumentException(String.format(
@@ -212,26 +222,6 @@ private void validate(int initialCapacity, int maxCapacity) {
       }
     }
 
-    private class MemoryStatusThread extends Thread {
-
-      public MemoryStatusThread() {
-        super("allocation.logger");
-        this.setDaemon(true);
-      }
-
-      @Override
-      public void run() {
-        while (true) {
-          memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString());
-          try {
-            Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000);
-          } catch (InterruptedException e) {
-            return;
-          }
-        }
-      }
-    }
-
     @Override
     public String toString() {
       StringBuilder buf = new StringBuilder();
@@ -256,6 +246,26 @@ public String toString() {
       return buf.toString();
     }
 
+    private class MemoryStatusThread extends Thread {
+
+      public MemoryStatusThread() {
+        super("allocation.logger");
+        this.setDaemon(true);
+      }
+
+      @Override
+      public void run() {
+        while (true) {
+          memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString());
+          try {
+            Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000);
+          } catch (InterruptedException e) {
+            return;
+          }
+        }
+      }
+    }
+
 
   }
 }
diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
index 5ea176745f25e..87d822f58a315 100644
--- a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,22 +18,31 @@
 
 package io.netty.buffer;
 
+import io.netty.util.internal.PlatformDependent;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteOrder;
 import java.util.concurrent.atomic.AtomicLong;
 
-import io.netty.util.internal.PlatformDependent;
-
 /**
- * The underlying class we use for little-endian access to memory. Is used underneath ArrowBufs to abstract away the
+ * The underlying class we use for little-endian access to memory. Is used underneath ArrowBufs
+ * to abstract away the
  * Netty classes and underlying Netty memory management.
  */
 public class UnsafeDirectLittleEndian extends WrappedByteBuf {
+
+  public static final boolean ASSERT_ENABLED;
   private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
   private static final AtomicLong ID_GENERATOR = new AtomicLong(0);
 
+  static {
+    boolean isAssertEnabled = false;
+    assert isAssertEnabled = true;
+    ASSERT_ENABLED = isAssertEnabled;
+  }
+
   public final long id = ID_GENERATOR.incrementAndGet();
   private final AbstractByteBuf wrapped;
   private final long memoryAddress;
@@ -60,21 +69,22 @@ private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake) {
     this.wrapped = buf;
     this.memoryAddress = buf.memoryAddress();
   }
-    private long addr(int index) {
-        return memoryAddress + index;
-    }
 
-    @Override
-    public long getLong(int index) {
+  private long addr(int index) {
+    return memoryAddress + index;
+  }
+
+  @Override
+  public long getLong(int index) {
 //        wrapped.checkIndex(index, 8);
-        long v = PlatformDependent.getLong(addr(index));
-        return v;
-    }
+    long v = PlatformDependent.getLong(addr(index));
+    return v;
+  }
 
-    @Override
-    public float getFloat(int index) {
-        return Float.intBitsToFloat(getInt(index));
-    }
+  @Override
+  public float getFloat(int index) {
+    return Float.intBitsToFloat(getInt(index));
+  }
 
   @Override
   public ByteBuf slice() {
@@ -259,12 +269,4 @@ public int hashCode() {
     return System.identityHashCode(this);
   }
 
-  public static final boolean ASSERT_ENABLED;
-
-  static {
-    boolean isAssertEnabled = false;
-    assert isAssertEnabled = true;
-    ASSERT_ENABLED = isAssertEnabled;
-  }
-
 }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
index 37c598ad89ece..6ddc8f784bc4a 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -6,30 +6,33 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
+import com.google.common.base.Preconditions;
+
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.annotation.concurrent.ThreadSafe;
 
-import com.google.common.base.Preconditions;
-
 /**
- * Provides a concurrent way to manage account for memory usage without locking. Used as basis for Allocators. All
+ * Provides a concurrent way to manage account for memory usage without locking. Used as basis
+ * for Allocators. All
  * operations are threadsafe (except for close).
  */
 @ThreadSafe
 class Accountant implements AutoCloseable {
-  // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Accountant.class);
+  // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Accountant
+  // .class);
 
   /**
    * The parent allocator
@@ -37,7 +40,8 @@ class Accountant implements AutoCloseable {
   protected final Accountant parent;
 
   /**
-   * The amount of memory reserved for this allocator. Releases below this amount of memory will not be returned to the
+   * The amount of memory reserved for this allocator. Releases below this amount of memory will
+   * not be returned to the
    * parent Accountant until this Accountant is closed.
    */
   protected final long reservation;
@@ -45,7 +49,8 @@ class Accountant implements AutoCloseable {
   private final AtomicLong peakAllocation = new AtomicLong();
 
   /**
-   * Maximum local memory that can be held. This can be externally updated. Changing it won't cause past memory to
+   * Maximum local memory that can be held. This can be externally updated. Changing it won't
+   * cause past memory to
    * change but will change responses to future allocation efforts
    */
   private final AtomicLong allocationLimit = new AtomicLong();
@@ -56,11 +61,14 @@ class Accountant implements AutoCloseable {
   private final AtomicLong locallyHeldMemory = new AtomicLong();
 
   public Accountant(Accountant parent, long reservation, long maxAllocation) {
-    Preconditions.checkArgument(reservation >= 0, "The initial reservation size must be non-negative.");
-    Preconditions.checkArgument(maxAllocation >= 0, "The maximum allocation limit must be non-negative.");
+    Preconditions.checkArgument(reservation >= 0, "The initial reservation size must be " +
+        "non-negative.");
+    Preconditions.checkArgument(maxAllocation >= 0, "The maximum allocation limit must be " +
+        "non-negative.");
     Preconditions.checkArgument(reservation <= maxAllocation,
         "The initial reservation size must be <= the maximum allocation.");
-    Preconditions.checkArgument(reservation == 0 || parent != null, "The root accountant can't reserve memory.");
+    Preconditions.checkArgument(reservation == 0 || parent != null, "The root accountant can't " +
+        "reserve memory.");
 
     this.parent = parent;
     this.reservation = reservation;
@@ -72,19 +80,20 @@ public Accountant(Accountant parent, long reservation, long maxAllocation) {
       if (!outcome.isOk()) {
         throw new OutOfMemoryException(String.format(
             "Failure trying to allocate initial reservation for Allocator. "
-                + "Attempted to allocate %d bytes and received an outcome of %s.", reservation, outcome.name()));
+                + "Attempted to allocate %d bytes and received an outcome of %s.", reservation,
+            outcome.name()));
       }
     }
   }
 
   /**
-   * Attempt to allocate the requested amount of memory. Either completely succeeds or completely fails. Constructs a a
+   * Attempt to allocate the requested amount of memory. Either completely succeeds or completely
+   * fails. Constructs a a
    * log of delta
-   *
+   * <p>
    * If it fails, no changes are made to accounting.
    *
-   * @param size
-   *          The amount of memory to reserve in bytes.
+   * @param size The amount of memory to reserve in bytes.
    * @return True if the allocation was successful, false if the allocation failed.
    */
   AllocationOutcome allocateBytes(long size) {
@@ -116,8 +125,7 @@ private void updatePeak() {
   /**
    * Increase the accounting. Returns whether the allocation fit within limits.
    *
-   * @param size
-   *          to increase
+   * @param size to increase
    * @return Whether the allocation fit within limits.
    */
   boolean forceAllocate(long size) {
@@ -126,24 +134,29 @@ boolean forceAllocate(long size) {
   }
 
   /**
-   * Internal method for allocation. This takes a forced approach to allocation to ensure that we manage reservation
-   * boundary issues consistently. Allocation is always done through the entire tree. The two options that we influence
-   * are whether the allocation should be forced and whether or not the peak memory allocation should be updated. If at
-   * some point during allocation escalation we determine that the allocation is no longer possible, we will continue to
-   * do a complete and consistent allocation but we will stop updating the peak allocation. We do this because we know
-   * that we will be directly unwinding this allocation (and thus never actually making the allocation). If force
-   * allocation is passed, then we continue to update the peak limits since we now know that this allocation will occur
+   * Internal method for allocation. This takes a forced approach to allocation to ensure that we
+   * manage reservation
+   * boundary issues consistently. Allocation is always done through the entire tree. The two
+   * options that we influence
+   * are whether the allocation should be forced and whether or not the peak memory allocation
+   * should be updated. If at
+   * some point during allocation escalation we determine that the allocation is no longer
+   * possible, we will continue to
+   * do a complete and consistent allocation but we will stop updating the peak allocation. We do
+   * this because we know
+   * that we will be directly unwinding this allocation (and thus never actually making the
+   * allocation). If force
+   * allocation is passed, then we continue to update the peak limits since we now know that this
+   * allocation will occur
    * despite our moving past one or more limits.
    *
-   * @param size
-   *          The size of the allocation.
-   * @param incomingUpdatePeak
-   *          Whether we should update the local peak for this allocation.
-   * @param forceAllocation
-   *          Whether we should force the allocation.
+   * @param size               The size of the allocation.
+   * @param incomingUpdatePeak Whether we should update the local peak for this allocation.
+   * @param forceAllocation    Whether we should force the allocation.
    * @return The outcome of the allocation.
    */
-  private AllocationOutcome allocate(final long size, final boolean incomingUpdatePeak, final boolean forceAllocation) {
+  private AllocationOutcome allocate(final long size, final boolean incomingUpdatePeak, final
+  boolean forceAllocation) {
     final long newLocal = locallyHeldMemory.addAndGet(size);
     final long beyondReservation = newLocal - reservation;
     final boolean beyondLimit = newLocal > allocationLimit.get();
@@ -173,7 +186,7 @@ public void releaseBytes(long size) {
     Preconditions.checkArgument(newSize >= 0, "Accounted size went negative.");
 
     final long originalSize = newSize + size;
-    if(originalSize > reservation && parent != null){
+    if (originalSize > reservation && parent != null) {
       // we deallocated memory that we should release to our parent.
       final long possibleAmountToReleaseToParent = originalSize - reservation;
       final long actualToReleaseToParent = Math.min(size, possibleAmountToReleaseToParent);
@@ -182,16 +195,6 @@ public void releaseBytes(long size) {
 
   }
 
-  /**
-   * Set the maximum amount of memory that can be allocated in the this Accountant before failing an allocation.
-   *
-   * @param newLimit
-   *          The limit in bytes.
-   */
-  public void setLimit(long newLimit) {
-    allocationLimit.set(newLimit);
-  }
-
   public boolean isOverLimit() {
     return getAllocatedMemory() > getLimit() || (parent != null && parent.isOverLimit());
   }
@@ -216,7 +219,18 @@ public long getLimit() {
   }
 
   /**
-   * Return the current amount of allocated memory that this Accountant is managing accounting for. Note this does not
+   * Set the maximum amount of memory that can be allocated in the this Accountant before failing
+   * an allocation.
+   *
+   * @param newLimit The limit in bytes.
+   */
+  public void setLimit(long newLimit) {
+    allocationLimit.set(newLimit);
+  }
+
+  /**
+   * Return the current amount of allocated memory that this Accountant is managing accounting
+   * for. Note this does not
    * include reservation memory that hasn't been allocated.
    *
    * @return Currently allocate memory in bytes.
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
index 1b127f8181222..d36cb37fc2e24 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
@@ -15,15 +15,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 /**
  * An allocation listener being notified for allocation/deallocation
- *
+ * <p>
  * It is expected to be called from multiple threads and as such,
  * provider should take care of making the implementation thread-safe
  */
 public interface AllocationListener {
+
   public static final AllocationListener NOOP = new AllocationListener() {
     @Override
     public void onAllocation(long size) {
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index f15bb8a40fa01..683752e6a4980 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -6,53 +6,62 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
-import static org.apache.arrow.memory.BaseAllocator.indent;
+import com.google.common.base.Preconditions;
 
-import java.util.IdentityHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.PooledByteBufAllocatorL;
+import io.netty.buffer.UnsafeDirectLittleEndian;
 
 import org.apache.arrow.memory.BaseAllocator.Verbosity;
 import org.apache.arrow.memory.util.AutoCloseableLock;
 import org.apache.arrow.memory.util.HistoricalLog;
 
-import com.google.common.base.Preconditions;
+import java.util.IdentityHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import io.netty.buffer.ArrowBuf;
-import io.netty.buffer.PooledByteBufAllocatorL;
-import io.netty.buffer.UnsafeDirectLittleEndian;
+import static org.apache.arrow.memory.BaseAllocator.indent;
 
 /**
- * Manages the relationship between one or more allocators and a particular UDLE. Ensures that one allocator owns the
- * memory that multiple allocators may be referencing. Manages a BufferLedger between each of its associated allocators.
- * This class is also responsible for managing when memory is allocated and returned to the Netty-based
+ * Manages the relationship between one or more allocators and a particular UDLE. Ensures that
+ * one allocator owns the
+ * memory that multiple allocators may be referencing. Manages a BufferLedger between each of its
+ * associated allocators.
+ * This class is also responsible for managing when memory is allocated and returned to the
+ * Netty-based
  * PooledByteBufAllocatorL.
- *
- * The only reason that this isn't package private is we're forced to put ArrowBuf in Netty's package which need access
+ * <p>
+ * The only reason that this isn't package private is we're forced to put ArrowBuf in Netty's
+ * package which need access
  * to these objects or methods.
- *
- * Threading: AllocationManager manages thread-safety internally. Operations within the context of a single BufferLedger
- * are lockless in nature and can be leveraged by multiple threads. Operations that cross the context of two ledgers
- * will acquire a lock on the AllocationManager instance. Important note, there is one AllocationManager per
- * UnsafeDirectLittleEndian buffer allocation. As such, there will be thousands of these in a typical query. The
+ * <p>
+ * Threading: AllocationManager manages thread-safety internally. Operations within the context
+ * of a single BufferLedger
+ * are lockless in nature and can be leveraged by multiple threads. Operations that cross the
+ * context of two ledgers
+ * will acquire a lock on the AllocationManager instance. Important note, there is one
+ * AllocationManager per
+ * UnsafeDirectLittleEndian buffer allocation. As such, there will be thousands of these in a
+ * typical query. The
  * contention of acquiring a lock on AllocationManager should be very low.
- *
  */
 public class AllocationManager {
-  // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationManager.class);
+  // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger
+  // (AllocationManager.class);
 
   private static final AtomicLong MANAGER_ID_GENERATOR = new AtomicLong(0);
   private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0);
@@ -81,17 +90,19 @@ public class AllocationManager {
     this.root = accountingAllocator.root;
     this.underlying = INNER_ALLOCATOR.allocate(size);
 
-    // we do a no retain association since our creator will want to retrieve the newly created ledger and will create a
+    // we do a no retain association since our creator will want to retrieve the newly created
+    // ledger and will create a
     // reference count at that point
     this.owningLedger = associate(accountingAllocator, false);
     this.size = underlying.capacity();
   }
 
   /**
-   * Associate the existing underlying buffer with a new allocator. This will increase the reference count to the
+   * Associate the existing underlying buffer with a new allocator. This will increase the
+   * reference count to the
    * provided ledger by 1.
-   * @param allocator
-   *          The target allocator to associate this buffer with.
+   *
+   * @param allocator The target allocator to associate this buffer with.
    * @return The Ledger (new or existing) that associates the underlying buffer to this new ledger.
    */
   BufferLedger associate(final BaseAllocator allocator) {
@@ -118,7 +129,8 @@ private BufferLedger associate(final BaseAllocator allocator, final boolean reta
 
     }
     try (AutoCloseableLock write = writeLock.open()) {
-      // we have to recheck existing ledger since a second reader => writer could be competing with us.
+      // we have to recheck existing ledger since a second reader => writer could be competing
+      // with us.
 
       final BufferLedger existingLedger = map.get(allocator);
       if (existingLedger != null) {
@@ -141,7 +153,8 @@ private BufferLedger associate(final BaseAllocator allocator, final boolean reta
 
 
   /**
-   * The way that a particular BufferLedger communicates back to the AllocationManager that it now longer needs to hold
+   * The way that a particular BufferLedger communicates back to the AllocationManager that it
+   * now longer needs to hold
    * a reference to particular piece of memory.
    */
   private class ReleaseListener {
@@ -169,16 +182,19 @@ public void release() {
           amDestructionTime = System.nanoTime();
           owningLedger = null;
         } else {
-          // we need to change the owning allocator. we've been removed so we'll get whatever is top of list
+          // we need to change the owning allocator. we've been removed so we'll get whatever is
+          // top of list
           BufferLedger newLedger = map.values().iterator().next();
 
-          // we'll forcefully transfer the ownership and not worry about whether we exceeded the limit
+          // we'll forcefully transfer the ownership and not worry about whether we exceeded the
+          // limit
           // since this consumer can't do anything with this.
           oldLedger.transferBalance(newLedger);
         }
       } else {
         if (map.isEmpty()) {
-          throw new IllegalStateException("The final removal of a ledger should be connected to the owning ledger.");
+          throw new IllegalStateException("The final removal of a ledger should be connected to " +
+              "the owning ledger.");
         }
       }
 
@@ -187,25 +203,30 @@ public void release() {
   }
 
   /**
-   * The reference manager that binds an allocator manager to a particular BaseAllocator. Also responsible for creating
+   * The reference manager that binds an allocator manager to a particular BaseAllocator. Also
+   * responsible for creating
    * a set of ArrowBufs that share a common fate and set of reference counts.
-   * As with AllocationManager, the only reason this is public is due to ArrowBuf being in io.netty.buffer package.
+   * As with AllocationManager, the only reason this is public is due to ArrowBuf being in io
+   * .netty.buffer package.
    */
   public class BufferLedger {
 
     private final IdentityHashMap<ArrowBuf, Object> buffers =
         BaseAllocator.DEBUG ? new IdentityHashMap<ArrowBuf, Object>() : null;
 
-    private final long ledgerId = LEDGER_ID_GENERATOR.incrementAndGet(); // unique ID assigned to each ledger
-    private final AtomicInteger bufRefCnt = new AtomicInteger(0); // start at zero so we can manage request for retain
-                                                                  // correctly
+    private final long ledgerId = LEDGER_ID_GENERATOR.incrementAndGet(); // unique ID assigned to
+    // each ledger
+    private final AtomicInteger bufRefCnt = new AtomicInteger(0); // start at zero so we can
+    // manage request for retain
+    // correctly
     private final long lCreationTime = System.nanoTime();
-    private volatile long lDestructionTime = 0;
     private final BaseAllocator allocator;
     private final ReleaseListener listener;
-    private final HistoricalLog historicalLog = BaseAllocator.DEBUG ? new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH,
-        "BufferLedger[%d]", 1)
+    private final HistoricalLog historicalLog = BaseAllocator.DEBUG ? new HistoricalLog
+        (BaseAllocator.DEBUG_LOG_LENGTH,
+            "BufferLedger[%d]", 1)
         : null;
+    private volatile long lDestructionTime = 0;
 
     private BufferLedger(BaseAllocator allocator, ReleaseListener listener) {
       this.allocator = allocator;
@@ -213,10 +234,11 @@ private BufferLedger(BaseAllocator allocator, ReleaseListener listener) {
     }
 
     /**
-     * Transfer any balance the current ledger has to the target ledger. In the case that the current ledger holds no
+     * Transfer any balance the current ledger has to the target ledger. In the case that the
+     * current ledger holds no
      * memory, no transfer is made to the new ledger.
-     * @param target
-     *          The ledger to transfer ownership account to.
+     *
+     * @param target The ledger to transfer ownership account to.
      * @return Whether transfer fit within target ledgers limits.
      */
     public boolean transferBalance(final BufferLedger target) {
@@ -231,7 +253,8 @@ public boolean transferBalance(final BufferLedger target) {
         return true;
       }
 
-      // since two balance transfers out from the allocator manager could cause incorrect accounting, we need to ensure
+      // since two balance transfers out from the allocator manager could cause incorrect
+      // accounting, we need to ensure
       // that this won't happen by synchronizing on the allocator manager instance.
       try (AutoCloseableLock write = writeLock.open()) {
         if (owningLedger != this) {
@@ -253,12 +276,10 @@ public boolean transferBalance(final BufferLedger target) {
 
     /**
      * Print the current ledger state to a the provided StringBuilder.
-     * @param sb
-     *          The StringBuilder to populate.
-     * @param indent
-     *          The level of indentation to position the data.
-     * @param verbosity
-     *          The level of verbosity to print.
+     *
+     * @param sb        The StringBuilder to populate.
+     * @param indent    The level of indentation to position the data.
+     * @param verbosity The level of verbosity to print.
      */
     public void print(StringBuilder sb, int indent, Verbosity verbosity) {
       indent(sb, indent)
@@ -304,7 +325,8 @@ private void inc() {
     }
 
     /**
-     * Decrement the ledger's reference count. If the ledger is decremented to zero, this ledger should release its
+     * Decrement the ledger's reference count. If the ledger is decremented to zero, this ledger
+     * should release its
      * ownership back to the AllocationManager
      */
     public int decrement(int decrement) {
@@ -323,15 +345,19 @@ public int decrement(int decrement) {
     }
 
     /**
-     * Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator doesn't currently have a
-     * ledger associated with this AllocationManager, a new one is created. This is placed on BufferLedger rather than
-     * AllocationManager directly because ArrowBufs don't have access to AllocationManager and they are the ones
-     * responsible for exposing the ability to associate multiple allocators with a particular piece of underlying
-     * memory. Note that this will increment the reference count of this ledger by one to ensure the ledger isn't
+     * Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator
+     * doesn't currently have a
+     * ledger associated with this AllocationManager, a new one is created. This is placed on
+     * BufferLedger rather than
+     * AllocationManager directly because ArrowBufs don't have access to AllocationManager and
+     * they are the ones
+     * responsible for exposing the ability to associate multiple allocators with a particular
+     * piece of underlying
+     * memory. Note that this will increment the reference count of this ledger by one to ensure
+     * the ledger isn't
      * destroyed before use.
      *
-     * @param allocator
-     *          A BufferAllocator.
+     * @param allocator A BufferAllocator.
      * @return The ledger associated with the BufferAllocator.
      */
     public BufferLedger getLedgerForAllocator(BufferAllocator allocator) {
@@ -339,13 +365,14 @@ public BufferLedger getLedgerForAllocator(BufferAllocator allocator) {
     }
 
     /**
-     * Create a new ArrowBuf associated with this AllocationManager and memory. Does not impact reference count.
+     * Create a new ArrowBuf associated with this AllocationManager and memory. Does not impact
+     * reference count.
      * Typically used for slicing.
-     * @param offset
-     *          The offset in bytes to start this new ArrowBuf.
-     * @param length
-     *          The length in bytes that this ArrowBuf will provide access to.
-     * @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
+     *
+     * @param offset The offset in bytes to start this new ArrowBuf.
+     * @param length The length in bytes that this ArrowBuf will provide access to.
+     * @return A new ArrowBuf that shares references with all ArrowBufs associated with this
+     * BufferLedger
      */
     public ArrowBuf newArrowBuf(int offset, int length) {
       allocator.assertOpen();
@@ -354,13 +381,13 @@ public ArrowBuf newArrowBuf(int offset, int length) {
 
     /**
      * Create a new ArrowBuf associated with this AllocationManager and memory.
-     * @param offset
-     *          The offset in bytes to start this new ArrowBuf.
-     * @param length
-     *          The length in bytes that this ArrowBuf will provide access to.
-     * @param manager
-     *          An optional BufferManager argument that can be used to manage expansion of this ArrowBuf
-     * @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
+     *
+     * @param offset  The offset in bytes to start this new ArrowBuf.
+     * @param length  The length in bytes that this ArrowBuf will provide access to.
+     * @param manager An optional BufferManager argument that can be used to manage expansion of
+     *                this ArrowBuf
+     * @return A new ArrowBuf that shares references with all ArrowBufs associated with this
+     * BufferLedger
      */
     public ArrowBuf newArrowBuf(int offset, int length, BufferManager manager) {
       allocator.assertOpen();
@@ -377,7 +404,8 @@ public ArrowBuf newArrowBuf(int offset, int length, BufferManager manager) {
 
       if (BaseAllocator.DEBUG) {
         historicalLog.recordEvent(
-            "ArrowBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == "
+            "ArrowBuf(BufferLedger, BufferAllocator[%s], " +
+                "UnsafeDirectLittleEndian[identityHashCode == "
                 + "%d](%s)) => ledger hc == %d",
             allocator.name, System.identityHashCode(buf), buf.toString(),
             System.identityHashCode(this));
@@ -401,7 +429,8 @@ public int getSize() {
     }
 
     /**
-     * How much memory is accounted for by this ledger. This is either getSize() if this is the owning ledger for the
+     * How much memory is accounted for by this ledger. This is either getSize() if this is the
+     * owning ledger for the
      * memory or zero in the case that this is not the owning ledger associated with this memory.
      *
      * @return Amount of accounted(owned) memory associated with this ledger.
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
index 68d1244d1e328..7f5aa313779a7 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
@@ -6,32 +6,36 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 import io.netty.buffer.ArrowBuf;
 
 /**
- * Supports cumulative allocation reservation. Clients may increase the size of the reservation repeatedly until they
- * call for an allocation of the current total size. The reservation can only be used once, and will throw an exception
+ * Supports cumulative allocation reservation. Clients may increase the size of the reservation
+ * repeatedly until they
+ * call for an allocation of the current total size. The reservation can only be used once, and
+ * will throw an exception
  * if it is used more than once.
  * <p>
- * For the purposes of airtight memory accounting, the reservation must be close()d whether it is used or not.
+ * For the purposes of airtight memory accounting, the reservation must be close()d whether it is
+ * used or not.
  * This is not threadsafe.
  */
 public interface AllocationReservation extends AutoCloseable {
 
   /**
    * Add to the current reservation.
-   *
+   * <p>
    * <p>Adding may fail if the allocator is not allowed to consume any more space.
    *
    * @param nBytes the number of bytes to add
@@ -42,7 +46,7 @@ public interface AllocationReservation extends AutoCloseable {
 
   /**
    * Requests a reservation of additional space.
-   *
+   * <p>
    * <p>The implementation of the allocator's inner class provides this.
    *
    * @param nBytes the amount to reserve
@@ -52,7 +56,7 @@ public interface AllocationReservation extends AutoCloseable {
 
   /**
    * Allocate a buffer whose size is the total of all the add()s made.
-   *
+   * <p>
    * <p>The allocation request can still fail, even if the amount of space
    * requested is available, if the allocation cannot be made contiguously.
    *
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
index 3274642dedd59..d5b638e1ed298 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java
@@ -6,15 +6,16 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 /**
@@ -23,6 +24,7 @@
  */
 @SuppressWarnings("serial")
 public class AllocatorClosedException extends RuntimeException {
+
   /**
    * @param message string associated with the cause
    */
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
index 5dc5ac397bd93..b8b5283423c82 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
@@ -6,15 +6,16 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 import io.netty.buffer.ByteBuf;
@@ -23,9 +24,12 @@
 import io.netty.buffer.ExpandableByteBuf;
 
 /**
- * An implementation of ByteBufAllocator that wraps a Arrow BufferAllocator. This allows the RPC layer to be accounted
- * and managed using Arrow's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is
- * the signature and the fact that this Allocator returns ExpandableByteBufs which enable otherwise non-expandable
+ * An implementation of ByteBufAllocator that wraps a Arrow BufferAllocator. This allows the RPC
+ * layer to be accounted
+ * and managed using Arrow's BufferAllocator infrastructure. The only thin different from a
+ * typical BufferAllocator is
+ * the signature and the fact that this Allocator returns ExpandableByteBufs which enable
+ * otherwise non-expandable
  * ArrowBufs to be expandable.
  */
 public class ArrowByteBufAllocator implements ByteBufAllocator {
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 9edafbce082cb..aaa7ce804c3e5 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -6,57 +6,54 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
-import java.util.Arrays;
-import java.util.IdentityHashMap;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.base.Preconditions;
+
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.UnsafeDirectLittleEndian;
 
 import org.apache.arrow.memory.AllocationManager.BufferLedger;
 import org.apache.arrow.memory.util.AssertionUtil;
 import org.apache.arrow.memory.util.HistoricalLog;
 
-import com.google.common.base.Preconditions;
-
-import io.netty.buffer.ArrowBuf;
-import io.netty.buffer.UnsafeDirectLittleEndian;
+import java.util.Arrays;
+import java.util.IdentityHashMap;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 
 public abstract class BaseAllocator extends Accountant implements BufferAllocator {
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseAllocator.class);
 
   public static final String DEBUG_ALLOCATOR = "arrow.memory.debug.allocator";
-
   public static final int DEBUG_LOG_LENGTH = 6;
   public static final boolean DEBUG = AssertionUtil.isAssertionsEnabled()
       || Boolean.parseBoolean(System.getProperty(DEBUG_ALLOCATOR, "false"));
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseAllocator
+      .class);
+  // Package exposed for sharing between AllocatorManger and BaseAllocator objects
+  final String name;
+  final RootAllocator root;
   private final Object DEBUG_LOCK = DEBUG ? new Object() : null;
-
   private final AllocationListener listener;
   private final BaseAllocator parentAllocator;
   private final ArrowByteBufAllocator thisAsByteBufAllocator;
   private final IdentityHashMap<BaseAllocator, Object> childAllocators;
   private final ArrowBuf empty;
-
-  private volatile boolean isClosed = false; // the allocator has been closed
-
-  // Package exposed for sharing between AllocatorManger and BaseAllocator objects
-  final String name;
-  final RootAllocator root;
-
   // members used purely for debugging
   private final IdentityHashMap<BufferLedger, Object> childLedgers;
   private final IdentityHashMap<Reservation, Object> reservations;
   private final HistoricalLog historicalLog;
+  private volatile boolean isClosed = false; // the allocator has been closed
 
   protected BaseAllocator(
       final AllocationListener listener,
@@ -91,7 +88,8 @@ private BaseAllocator(
       this.root = (RootAllocator) this;
       empty = createEmpty();
     } else {
-      throw new IllegalStateException("An parent allocator must either carry a root or be the root.");
+      throw new IllegalStateException("An parent allocator must either carry a root or be the " +
+          "root.");
     }
 
     this.parentAllocator = parentAllocator;
@@ -114,11 +112,52 @@ private BaseAllocator(
 
   }
 
+  private static String createErrorMsg(final BufferAllocator allocator, final int rounded, final
+  int requested) {
+    if (rounded != requested) {
+      return String.format(
+          "Unable to allocate buffer of size %d (rounded from %d) due to memory limit. Current " +
+              "allocation: %d",
+          rounded, requested, allocator.getAllocatedMemory());
+    } else {
+      return String.format("Unable to allocate buffer of size %d due to memory limit. Current " +
+              "allocation: %d",
+          rounded, allocator.getAllocatedMemory());
+    }
+  }
+
+  /**
+   * Rounds up the provided value to the nearest power of two.
+   *
+   * @param val An integer value.
+   * @return The closest power of two of that value.
+   */
+  static int nextPowerOfTwo(int val) {
+    int highestBit = Integer.highestOneBit(val);
+    if (highestBit == val) {
+      return val;
+    } else {
+      return highestBit << 1;
+    }
+  }
+
+  public static StringBuilder indent(StringBuilder sb, int indent) {
+    final char[] indentation = new char[indent * 2];
+    Arrays.fill(indentation, ' ');
+    sb.append(indentation);
+    return sb;
+  }
+
+  public static boolean isDebug() {
+    return DEBUG;
+  }
+
   @Override
   public void assertOpen() {
     if (AssertionUtil.ASSERT_ENABLED) {
       if (isClosed) {
-        throw new IllegalStateException("Attempting operation on allocator when allocator is closed.\n"
+        throw new IllegalStateException("Attempting operation on allocator when allocator is " +
+            "closed.\n"
             + toVerboseString());
       }
     }
@@ -136,7 +175,8 @@ public ArrowBuf getEmpty() {
   }
 
   /**
-   * For debug/verification purposes only. Allows an AllocationManager to tell the allocator that we have a new ledger
+   * For debug/verification purposes only. Allows an AllocationManager to tell the allocator that
+   * we have a new ledger
    * associated with this allocator.
    */
   void associateLedger(BufferLedger ledger) {
@@ -149,7 +189,8 @@ void associateLedger(BufferLedger ledger) {
   }
 
   /**
-   * For debug/verification purposes only. Allows an AllocationManager to tell the allocator that we are removing a
+   * For debug/verification purposes only. Allows an AllocationManager to tell the allocator that
+   * we are removing a
    * ledger associated with this allocator
    */
   void dissociateLedger(BufferLedger ledger) {
@@ -167,8 +208,7 @@ void dissociateLedger(BufferLedger ledger) {
   /**
    * Track when a ChildAllocator of this BaseAllocator is closed. Used for debugging purposes.
    *
-   * @param childAllocator
-   *          The child allocator that has been closed.
+   * @param childAllocator The child allocator that has been closed.
    */
   private void childClosed(final BaseAllocator childAllocator) {
     assertOpen();
@@ -187,17 +227,6 @@ private void childClosed(final BaseAllocator childAllocator) {
     }
   }
 
-  private static String createErrorMsg(final BufferAllocator allocator, final int rounded, final int requested) {
-    if (rounded != requested) {
-      return String.format(
-          "Unable to allocate buffer of size %d (rounded from %d) due to memory limit. Current allocation: %d",
-          rounded, requested, allocator.getAllocatedMemory());
-    } else {
-      return String.format("Unable to allocate buffer of size %d due to memory limit. Current allocation: %d",
-          rounded, allocator.getAllocatedMemory());
-    }
-  }
-
   @Override
   public ArrowBuf buffer(final int initialRequestSize) {
     assertOpen();
@@ -205,7 +234,7 @@ public ArrowBuf buffer(final int initialRequestSize) {
     return buffer(initialRequestSize, null);
   }
 
-  private ArrowBuf createEmpty(){
+  private ArrowBuf createEmpty() {
     assertOpen();
 
     return new ArrowBuf(new AtomicInteger(), null, AllocationManager.EMPTY, null, null, 0, 0, true);
@@ -221,7 +250,8 @@ public ArrowBuf buffer(final int initialRequestSize, BufferManager manager) {
       return empty;
     }
 
-    // round to next largest power of two if we're within a chunk since that is how our allocator operates
+    // round to next largest power of two if we're within a chunk since that is how our allocator
+    // operates
     final int actualRequestSize = initialRequestSize < AllocationManager.CHUNK_SIZE ?
         nextPowerOfTwo(initialRequestSize)
         : initialRequestSize;
@@ -245,10 +275,12 @@ public ArrowBuf buffer(final int initialRequestSize, BufferManager manager) {
   }
 
   /**
-   * Used by usual allocation as well as for allocating a pre-reserved buffer. Skips the typical accounting associated
+   * Used by usual allocation as well as for allocating a pre-reserved buffer. Skips the typical
+   * accounting associated
    * with creating a new buffer.
    */
-  private ArrowBuf bufferWithoutReservation(final int size, BufferManager bufferManager) throws OutOfMemoryException {
+  private ArrowBuf bufferWithoutReservation(final int size, BufferManager bufferManager) throws
+      OutOfMemoryException {
     assertOpen();
 
     final AllocationManager manager = new AllocationManager(this, size);
@@ -274,185 +306,20 @@ public BufferAllocator newChildAllocator(
       final long maxAllocation) {
     assertOpen();
 
-    final ChildAllocator childAllocator = new ChildAllocator(this, name, initReservation, maxAllocation);
+    final ChildAllocator childAllocator = new ChildAllocator(this, name, initReservation,
+        maxAllocation);
 
     if (DEBUG) {
       synchronized (DEBUG_LOCK) {
         childAllocators.put(childAllocator, childAllocator);
-        historicalLog.recordEvent("allocator[%s] created new child allocator[%s]", name, childAllocator.name);
+        historicalLog.recordEvent("allocator[%s] created new child allocator[%s]", name,
+            childAllocator.name);
       }
     }
 
     return childAllocator;
   }
 
-  public class Reservation implements AllocationReservation {
-    private int nBytes = 0;
-    private boolean used = false;
-    private boolean closed = false;
-    private final HistoricalLog historicalLog;
-
-    public Reservation() {
-      if (DEBUG) {
-        historicalLog = new HistoricalLog("Reservation[allocator[%s], %d]", name, System.identityHashCode(this));
-        historicalLog.recordEvent("created");
-        synchronized (DEBUG_LOCK) {
-          reservations.put(this, this);
-        }
-      } else {
-        historicalLog = null;
-      }
-    }
-
-    @Override
-    public boolean add(final int nBytes) {
-      assertOpen();
-
-      Preconditions.checkArgument(nBytes >= 0, "nBytes(%d) < 0", nBytes);
-      Preconditions.checkState(!closed, "Attempt to increase reservation after reservation has been closed");
-      Preconditions.checkState(!used, "Attempt to increase reservation after reservation has been used");
-
-      // we round up to next power of two since all reservations are done in powers of two. This may overestimate the
-      // preallocation since someone may perceive additions to be power of two. If this becomes a problem, we can look
-      // at
-      // modifying this behavior so that we maintain what we reserve and what the user asked for and make sure to only
-      // round to power of two as necessary.
-      final int nBytesTwo = BaseAllocator.nextPowerOfTwo(nBytes);
-      if (!reserve(nBytesTwo)) {
-        return false;
-      }
-
-      this.nBytes += nBytesTwo;
-      return true;
-    }
-
-    @Override
-    public ArrowBuf allocateBuffer() {
-      assertOpen();
-
-      Preconditions.checkState(!closed, "Attempt to allocate after closed");
-      Preconditions.checkState(!used, "Attempt to allocate more than once");
-
-      final ArrowBuf arrowBuf = allocate(nBytes);
-      used = true;
-      return arrowBuf;
-    }
-
-    @Override
-    public int getSize() {
-      return nBytes;
-    }
-
-    @Override
-    public boolean isUsed() {
-      return used;
-    }
-
-    @Override
-    public boolean isClosed() {
-      return closed;
-    }
-
-    @Override
-    public void close() {
-      assertOpen();
-
-      if (closed) {
-        return;
-      }
-
-      if (DEBUG) {
-        if (!isClosed()) {
-          final Object object;
-          synchronized (DEBUG_LOCK) {
-            object = reservations.remove(this);
-          }
-          if (object == null) {
-            final StringBuilder sb = new StringBuilder();
-            print(sb, 0, Verbosity.LOG_WITH_STACKTRACE);
-            logger.debug(sb.toString());
-            throw new IllegalStateException(
-                String.format("Didn't find closing reservation[%d]", System.identityHashCode(this)));
-          }
-
-          historicalLog.recordEvent("closed");
-        }
-      }
-
-      if (!used) {
-        releaseReservation(nBytes);
-      }
-
-      closed = true;
-    }
-
-    @Override
-    public boolean reserve(int nBytes) {
-      assertOpen();
-
-      final AllocationOutcome outcome = BaseAllocator.this.allocateBytes(nBytes);
-
-      if (DEBUG) {
-        historicalLog.recordEvent("reserve(%d) => %s", nBytes, Boolean.toString(outcome.isOk()));
-      }
-
-      return outcome.isOk();
-    }
-
-    /**
-     * Allocate the a buffer of the requested size.
-     *
-     * <p>
-     * The implementation of the allocator's inner class provides this.
-     *
-     * @param nBytes
-     *          the size of the buffer requested
-     * @return the buffer, or null, if the request cannot be satisfied
-     */
-    private ArrowBuf allocate(int nBytes) {
-      assertOpen();
-
-      boolean success = false;
-
-      /*
-       * The reservation already added the requested bytes to the allocators owned and allocated bytes via reserve().
-       * This ensures that they can't go away. But when we ask for the buffer here, that will add to the allocated bytes
-       * as well, so we need to return the same number back to avoid double-counting them.
-       */
-      try {
-        final ArrowBuf arrowBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);
-
-        listener.onAllocation(nBytes);
-        if (DEBUG) {
-          historicalLog.recordEvent("allocate() => %s", String.format("ArrowBuf[%d]", arrowBuf.getId()));
-        }
-        success = true;
-        return arrowBuf;
-      } finally {
-        if (!success) {
-          releaseBytes(nBytes);
-        }
-      }
-    }
-
-    /**
-     * Return the reservation back to the allocator without having used it.
-     *
-     * @param nBytes
-     *          the size of the reservation
-     */
-    private void releaseReservation(int nBytes) {
-      assertOpen();
-
-      releaseBytes(nBytes);
-
-      if (DEBUG) {
-        historicalLog.recordEvent("releaseReservation(%d)", nBytes);
-      }
-    }
-
-  }
-
   @Override
   public AllocationReservation newReservation() {
     assertOpen();
@@ -460,7 +327,6 @@ public AllocationReservation newReservation() {
     return new Reservation();
   }
 
-
   @Override
   public synchronized void close() {
     /*
@@ -474,7 +340,7 @@ public synchronized void close() {
     isClosed = true;
 
     if (DEBUG) {
-      synchronized(DEBUG_LOCK) {
+      synchronized (DEBUG_LOCK) {
         verifyAllocator();
 
         // are there outstanding child allocators?
@@ -488,7 +354,8 @@ public synchronized void close() {
           }
 
           throw new IllegalStateException(
-              String.format("Allocator[%s] closed with outstanding child allocators.\n%s", name, toString()));
+              String.format("Allocator[%s] closed with outstanding child allocators.\n%s", name,
+                  toString()));
         }
 
         // are there outstanding buffers?
@@ -501,7 +368,8 @@ public synchronized void close() {
 
         if (reservations.size() != 0) {
           throw new IllegalStateException(
-              String.format("Allocator[%s] closed with outstanding reservations (%d).\n%s", name, reservations.size(),
+              String.format("Allocator[%s] closed with outstanding reservations (%d).\n%s", name,
+                  reservations.size(),
                   toString()));
         }
 
@@ -512,7 +380,8 @@ public synchronized void close() {
     final long allocated = getAllocatedMemory();
     if (allocated > 0) {
       throw new IllegalStateException(
-          String.format("Memory was leaked by query. Memory leaked: (%d)\n%s", allocated, toString()));
+          String.format("Memory was leaked by query. Memory leaked: (%d)\n%s", allocated,
+              toString()));
     }
 
     // we need to release our memory to our parent before we tell it we've closed.
@@ -543,7 +412,8 @@ public String toString() {
   }
 
   /**
-   * Provide a verbose string of the current allocator state. Includes the state of all child allocators, along with
+   * Provide a verbose string of the current allocator state. Includes the state of all child
+   * allocators, along with
    * historical logs of each object and including stacktraces.
    *
    * @return A Verbose string of current allocator state.
@@ -559,48 +429,32 @@ private void hist(String noteFormat, Object... args) {
     historicalLog.recordEvent(noteFormat, args);
   }
 
-  /**
-   * Rounds up the provided value to the nearest power of two.
-   *
-   * @param val
-   *          An integer value.
-   * @return The closest power of two of that value.
-   */
-  static int nextPowerOfTwo(int val) {
-    int highestBit = Integer.highestOneBit(val);
-    if (highestBit == val) {
-      return val;
-    } else {
-      return highestBit << 1;
-    }
-  }
-
-
   /**
    * Verifies the accounting state of the allocator. Only works for DEBUG.
    *
-   * @throws IllegalStateException
-   *           when any problems are found
+   * @throws IllegalStateException when any problems are found
    */
   void verifyAllocator() {
-    final IdentityHashMap<UnsafeDirectLittleEndian, BaseAllocator> buffersSeen = new IdentityHashMap<>();
+    final IdentityHashMap<UnsafeDirectLittleEndian, BaseAllocator> buffersSeen = new
+        IdentityHashMap<>();
     verifyAllocator(buffersSeen);
   }
 
   /**
    * Verifies the accounting state of the allocator. Only works for DEBUG.
-   *
    * <p>
-   * This overload is used for recursive calls, allowing for checking that ArrowBufs are unique across all allocators
+   * <p>
+   * This overload is used for recursive calls, allowing for checking that ArrowBufs are unique
+   * across all allocators
    * that are checked.
    * </p>
    *
-   * @param buffersSeen
-   *          a map of buffers that have already been seen when walking a tree of allocators
-   * @throws IllegalStateException
-   *           when any problems are found
+   * @param buffersSeen a map of buffers that have already been seen when walking a tree of
+   *                    allocators
+   * @throws IllegalStateException when any problems are found
    */
-  private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, BaseAllocator> buffersSeen) {
+  private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, BaseAllocator>
+                                   buffersSeen) {
     // The remaining tests can only be performed if we're in debug mode.
     if (!DEBUG) {
       return;
@@ -618,7 +472,8 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas
       /*
        * Verify my relationships with my descendants.
        *
-       * The sum of direct child allocators' owned memory must be <= my allocated memory; my allocated memory also
+       * The sum of direct child allocators' owned memory must be <= my allocated memory; my
+       * allocated memory also
        * includes ArrowBuf's directly allocated by me.
        */
       long childTotal = 0;
@@ -648,11 +503,13 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas
 
         final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
         /*
-         * Even when shared, ArrowBufs are rewrapped, so we should never see the same instance twice.
+         * Even when shared, ArrowBufs are rewrapped, so we should never see the same instance
+         * twice.
          */
         final BaseAllocator otherOwner = buffersSeen.get(udle);
         if (otherOwner != null) {
-          throw new IllegalStateException("This allocator's ArrowBuf already owned by another allocator");
+          throw new IllegalStateException("This allocator's ArrowBuf already owned by another " +
+              "allocator");
         }
         buffersSeen.put(udle, this);
 
@@ -713,12 +570,14 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas
 
         if (allocated2 != allocated) {
           throw new IllegalStateException(String.format(
-              "allocator[%s]: allocated t1 (%d) + allocated t2 (%d). Someone released memory while in verification.",
+              "allocator[%s]: allocated t1 (%d) + allocated t2 (%d). Someone released memory " +
+                  "while in verification.",
               name, allocated, allocated2));
 
         }
         throw new IllegalStateException(String.format(
-            "allocator[%s]: buffer space (%d) + prealloc space (%d) + child space (%d) != allocated (%d)",
+            "allocator[%s]: buffer space (%d) + prealloc space (%d) + child space (%d) != " +
+                "allocated (%d)",
             name, bufferTotal, reservedTotal, childTotal, allocated));
       }
     }
@@ -777,14 +636,6 @@ private void dumpBuffers(final StringBuilder sb, final Set<BufferLedger> ledgerS
     }
   }
 
-
-  public static StringBuilder indent(StringBuilder sb, int indent) {
-    final char[] indentation = new char[indent * 2];
-    Arrays.fill(indentation, ' ');
-    sb.append(indentation);
-    return sb;
-  }
-
   public static enum Verbosity {
     BASIC(false, false), // only include basic information
     LOG(true, false), // include basic
@@ -800,7 +651,179 @@ public static enum Verbosity {
     }
   }
 
-  public static boolean isDebug() {
-    return DEBUG;
+  public class Reservation implements AllocationReservation {
+
+    private final HistoricalLog historicalLog;
+    private int nBytes = 0;
+    private boolean used = false;
+    private boolean closed = false;
+
+    public Reservation() {
+      if (DEBUG) {
+        historicalLog = new HistoricalLog("Reservation[allocator[%s], %d]", name, System
+            .identityHashCode(this));
+        historicalLog.recordEvent("created");
+        synchronized (DEBUG_LOCK) {
+          reservations.put(this, this);
+        }
+      } else {
+        historicalLog = null;
+      }
+    }
+
+    @Override
+    public boolean add(final int nBytes) {
+      assertOpen();
+
+      Preconditions.checkArgument(nBytes >= 0, "nBytes(%d) < 0", nBytes);
+      Preconditions.checkState(!closed, "Attempt to increase reservation after reservation has " +
+          "been closed");
+      Preconditions.checkState(!used, "Attempt to increase reservation after reservation has been" +
+          " used");
+
+      // we round up to next power of two since all reservations are done in powers of two. This
+      // may overestimate the
+      // preallocation since someone may perceive additions to be power of two. If this becomes a
+      // problem, we can look
+      // at
+      // modifying this behavior so that we maintain what we reserve and what the user asked for
+      // and make sure to only
+      // round to power of two as necessary.
+      final int nBytesTwo = BaseAllocator.nextPowerOfTwo(nBytes);
+      if (!reserve(nBytesTwo)) {
+        return false;
+      }
+
+      this.nBytes += nBytesTwo;
+      return true;
+    }
+
+    @Override
+    public ArrowBuf allocateBuffer() {
+      assertOpen();
+
+      Preconditions.checkState(!closed, "Attempt to allocate after closed");
+      Preconditions.checkState(!used, "Attempt to allocate more than once");
+
+      final ArrowBuf arrowBuf = allocate(nBytes);
+      used = true;
+      return arrowBuf;
+    }
+
+    @Override
+    public int getSize() {
+      return nBytes;
+    }
+
+    @Override
+    public boolean isUsed() {
+      return used;
+    }
+
+    @Override
+    public boolean isClosed() {
+      return closed;
+    }
+
+    @Override
+    public void close() {
+      assertOpen();
+
+      if (closed) {
+        return;
+      }
+
+      if (DEBUG) {
+        if (!isClosed()) {
+          final Object object;
+          synchronized (DEBUG_LOCK) {
+            object = reservations.remove(this);
+          }
+          if (object == null) {
+            final StringBuilder sb = new StringBuilder();
+            print(sb, 0, Verbosity.LOG_WITH_STACKTRACE);
+            logger.debug(sb.toString());
+            throw new IllegalStateException(
+                String.format("Didn't find closing reservation[%d]", System.identityHashCode
+                    (this)));
+          }
+
+          historicalLog.recordEvent("closed");
+        }
+      }
+
+      if (!used) {
+        releaseReservation(nBytes);
+      }
+
+      closed = true;
+    }
+
+    @Override
+    public boolean reserve(int nBytes) {
+      assertOpen();
+
+      final AllocationOutcome outcome = BaseAllocator.this.allocateBytes(nBytes);
+
+      if (DEBUG) {
+        historicalLog.recordEvent("reserve(%d) => %s", nBytes, Boolean.toString(outcome.isOk()));
+      }
+
+      return outcome.isOk();
+    }
+
+    /**
+     * Allocate the a buffer of the requested size.
+     * <p>
+     * <p>
+     * The implementation of the allocator's inner class provides this.
+     *
+     * @param nBytes the size of the buffer requested
+     * @return the buffer, or null, if the request cannot be satisfied
+     */
+    private ArrowBuf allocate(int nBytes) {
+      assertOpen();
+
+      boolean success = false;
+
+      /*
+       * The reservation already added the requested bytes to the allocators owned and allocated
+       * bytes via reserve().
+       * This ensures that they can't go away. But when we ask for the buffer here, that will add
+        * to the allocated bytes
+       * as well, so we need to return the same number back to avoid double-counting them.
+       */
+      try {
+        final ArrowBuf arrowBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);
+
+        listener.onAllocation(nBytes);
+        if (DEBUG) {
+          historicalLog.recordEvent("allocate() => %s", String.format("ArrowBuf[%d]", arrowBuf
+              .getId()));
+        }
+        success = true;
+        return arrowBuf;
+      } finally {
+        if (!success) {
+          releaseBytes(nBytes);
+        }
+      }
+    }
+
+    /**
+     * Return the reservation back to the allocator without having used it.
+     *
+     * @param nBytes the size of the reservation
+     */
+    private void releaseReservation(int nBytes) {
+      assertOpen();
+
+      releaseBytes(nBytes);
+
+      if (DEBUG) {
+        historicalLog.recordEvent("releaseReservation(%d)", nBytes);
+      }
+    }
+
   }
 }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java b/java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java
index 4e88c734ab4be..b0e9cd8c1a0e9 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java
@@ -6,21 +6,22 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 public class BoundsChecking {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BoundsChecking.class);
 
   public static final boolean BOUNDS_CHECKING_ENABLED;
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BoundsChecking.class);
 
   static {
     boolean isAssertEnabled = false;
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
index 356a3416cbf85..81ffb1bec780e 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -6,47 +6,48 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
-import io.netty.buffer.ByteBufAllocator;
 import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.ByteBufAllocator;
 
 /**
  * Wrapper class to deal with byte buffer allocation. Ensures users only use designated methods.
  */
 public interface BufferAllocator extends AutoCloseable {
+
   /**
-   * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
-   * requested size for rounding purposes. However, the buffer's capacity will be set to the configured size.
+   * Allocate a new or reused buffer of the provided size. Note that the buffer may technically
+   * be larger than the
+   * requested size for rounding purposes. However, the buffer's capacity will be set to the
+   * configured size.
    *
-   * @param size
-   *          The size in bytes.
+   * @param size The size in bytes.
    * @return a new ArrowBuf, or null if the request can't be satisfied
-   * @throws OutOfMemoryException
-   *           if buffer cannot be allocated
+   * @throws OutOfMemoryException if buffer cannot be allocated
    */
   public ArrowBuf buffer(int size);
 
   /**
-   * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
-   * requested size for rounding purposes. However, the buffer's capacity will be set to the configured size.
+   * Allocate a new or reused buffer of the provided size. Note that the buffer may technically
+   * be larger than the
+   * requested size for rounding purposes. However, the buffer's capacity will be set to the
+   * configured size.
    *
-   * @param size
-   *          The size in bytes.
-   * @param manager
-   *          A buffer manager to manage reallocation.
+   * @param size    The size in bytes.
+   * @param manager A buffer manager to manage reallocation.
    * @return a new ArrowBuf, or null if the request can't be satisfied
-   * @throws OutOfMemoryException
-   *           if buffer cannot be allocated
+   * @throws OutOfMemoryException if buffer cannot be allocated
    */
   public ArrowBuf buffer(int size, BufferManager manager);
 
@@ -60,19 +61,16 @@ public interface BufferAllocator extends AutoCloseable {
   /**
    * Create a new child allocator.
    *
-   * @param name
-   *          the name of the allocator.
-   * @param initReservation
-   *          the initial space reservation (obtained from this allocator)
-   * @param maxAllocation
-   *          maximum amount of space the new allocator can allocate
+   * @param name            the name of the allocator.
+   * @param initReservation the initial space reservation (obtained from this allocator)
+   * @param maxAllocation   maximum amount of space the new allocator can allocate
    * @return the new allocator, or null if it can't be created
    */
   public BufferAllocator newChildAllocator(String name, long initReservation, long maxAllocation);
 
   /**
    * Close and release all buffers generated from this buffer pool.
-   *
+   * <p>
    * <p>When assertions are on, complains if there are any outstanding buffers; to avoid
    * that, release all buffers before the allocator is closed.
    */
@@ -87,19 +85,18 @@ public interface BufferAllocator extends AutoCloseable {
   public long getAllocatedMemory();
 
   /**
-   * Set the maximum amount of memory this allocator is allowed to allocate.
+   * Return the current maximum limit this allocator imposes.
    *
-   * @param newLimit
-   *          The new Limit to apply to allocations
+   * @return Limit in number of bytes.
    */
-  public void setLimit(long newLimit);
+  public long getLimit();
 
   /**
-   * Return the current maximum limit this allocator imposes.
+   * Set the maximum amount of memory this allocator is allowed to allocate.
    *
-   * @return Limit in number of bytes.
+   * @param newLimit The new Limit to apply to allocations
    */
-  public long getLimit();
+  public void setLimit(long newLimit);
 
   /**
    * Returns the peak amount of memory allocated from this allocator.
@@ -118,25 +115,31 @@ public interface BufferAllocator extends AutoCloseable {
   public AllocationReservation newReservation();
 
   /**
-   * Get a reference to the empty buffer associated with this allocator. Empty buffers are special because we don't
-   * worry about them leaking or managing reference counts on them since they don't actually point to any memory.
+   * Get a reference to the empty buffer associated with this allocator. Empty buffers are
+   * special because we don't
+   * worry about them leaking or managing reference counts on them since they don't actually
+   * point to any memory.
    */
   public ArrowBuf getEmpty();
 
   /**
-   * Return the name of this allocator. This is a human readable name that can help debugging. Typically provides
+   * Return the name of this allocator. This is a human readable name that can help debugging.
+   * Typically provides
    * coordinates about where this allocator was created
    */
   public String getName();
 
   /**
-   * Return whether or not this allocator (or one if its parents) is over its limits. In the case that an allocator is
-   * over its limit, all consumers of that allocator should aggressively try to addrss the overlimit situation.
+   * Return whether or not this allocator (or one if its parents) is over its limits. In the case
+   * that an allocator is
+   * over its limit, all consumers of that allocator should aggressively try to addrss the
+   * overlimit situation.
    */
   public boolean isOverLimit();
 
   /**
-   * Return a verbose string describing this allocator. If in DEBUG mode, this will also include relevant stacktraces
+   * Return a verbose string describing this allocator. If in DEBUG mode, this will also include
+   * relevant stacktraces
    * and historical logs for underlying objects
    *
    * @return A very verbose description of the allocator hierarchy.
@@ -144,7 +147,8 @@ public interface BufferAllocator extends AutoCloseable {
   public String toVerboseString();
 
   /**
-   * Asserts (using java assertions) that the provided allocator is currently open. If assertions are disabled, this is
+   * Asserts (using java assertions) that the provided allocator is currently open. If assertions
+   * are disabled, this is
    * a no-op.
    */
   public void assertOpen();
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
index 8969434791012..2fe763e10aff9 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.memory;
 
 import io.netty.buffer.ArrowBuf;
@@ -24,7 +25,7 @@
  * re-allocation the old buffer will be freed. Managing a list of these buffers
  * prevents some parts of the system from needing to define a correct location
  * to place the final call to free them.
- *
+ * <p>
  * The current uses of these types of buffers are within the pluggable components of Drill.
  * In UDFs, memory management should not be a concern. We provide access to re-allocatable
  * ArrowBufs to give UDF writers general purpose buffers we can account for. To prevent the need
@@ -38,12 +39,9 @@ public interface BufferManager extends AutoCloseable {
   /**
    * Replace an old buffer with a new version at least of the provided size. Does not copy data.
    *
-   * @param old
-   *          Old Buffer that the user is no longer going to use.
-   * @param newSize
-   *          Size of new replacement buffer.
-   * @return
-   *          A new version of the buffer.
+   * @param old     Old Buffer that the user is no longer going to use.
+   * @param newSize Size of new replacement buffer.
+   * @return A new version of the buffer.
    */
   public ArrowBuf replace(ArrowBuf old, int newSize);
 
@@ -57,8 +55,7 @@ public interface BufferManager extends AutoCloseable {
   /**
    * Get a managed buffer of at least a certain size.
    *
-   * @param size
-   *          The desired size
+   * @param size The desired size
    * @return A buffer
    */
   public ArrowBuf getManagedBuffer(int size);
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
index 11c9063fc9c69..f9a6dc72ece8c 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
@@ -6,15 +6,16 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 
@@ -22,21 +23,22 @@
  * Child allocator class. Only slightly different from the {@see RootAllocator},
  * in that these can't be created directly, but must be obtained from
  * {@see BufferAllocator#newChildAllocator(AllocatorOwner, long, long, int)}.
-
+ * <p>
  * <p>Child allocators can only be created by the root, or other children, so
  * this class is package private.</p>
  */
 class ChildAllocator extends BaseAllocator {
+
   /**
    * Constructor.
    *
    * @param parentAllocator parent allocator -- the one creating this child
-   * @param name the name of this child allocator
+   * @param name            the name of this child allocator
    * @param initReservation initial amount of space to reserve (obtained from the parent)
-   * @param maxAllocation maximum amount of space that can be obtained from this allocator;
-   *   note this includes direct allocations (via {@see BufferAllocator#buffer(int, int)}
-   *   et al) and requests from descendant allocators. Depending on the allocation policy in
-   *   force, even less memory may be available
+   * @param maxAllocation   maximum amount of space that can be obtained from this allocator; note
+   *                        this includes direct allocations (via {@see BufferAllocator#buffer(int,
+   *int)} et al) and requests from descendant allocators. Depending on the
+   *                        allocation policy in force, even less memory may be available
    */
   ChildAllocator(
       BaseAllocator parentAllocator,
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java b/java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java
index 6ba0284d8d449..c36584c9538b0 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java
@@ -6,28 +6,31 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 
 public class OutOfMemoryException extends RuntimeException {
-  private static final long serialVersionUID = -6858052345185793382L;
 
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutOfMemoryException.class);
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutOfMemoryException
+      .class);
+  private static final long serialVersionUID = -6858052345185793382L;
 
   public OutOfMemoryException() {
     super();
   }
 
-  public OutOfMemoryException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+  public OutOfMemoryException(String message, Throwable cause, boolean enableSuppression, boolean
+      writableStackTrace) {
     super(message, cause, enableSuppression, writableStackTrace);
   }
 
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
index 57a2c0cdae8d8..1dc6bf0c92fa0 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
@@ -6,15 +6,16 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -24,6 +25,7 @@
  * tree of descendant child allocators.
  */
 public class RootAllocator extends BaseAllocator {
+
   public RootAllocator(final long limit) {
     this(AllocationListener.NOOP, limit);
   }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/package-info.java b/java/memory/src/main/java/org/apache/arrow/memory/package-info.java
index 40d25cada4519..cef382d1e044e 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/package-info.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/package-info.java
@@ -1,24 +1,43 @@
 /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements.  See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership.  The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License.  You may obtain
+ * a copy of the License at
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * <p>
+ * Memory Allocation, Account and Management
+ * <p>
+ * See the README.md file in this directory for detailed information about Arrow's memory allocation
+ * subsystem.
+ * <p>
+ * Memory Allocation, Account and Management
+ * <p>
+ * See the README.md file in this directory for detailed information about Arrow's memory
+ * allocation subsystem.
+ * <p>
+ * Memory Allocation, Account and Management
+ * <p>
+ * See the README.md file in this directory for detailed information about Arrow's memory
+ * allocation subsystem.
+ * <p>
+ * Memory Allocation, Account and Management
+ * <p>
+ * See the README.md file in this directory for detailed information about Arrow's memory
+ * allocation subsystem.
  */
 /**
  *  Memory Allocation, Account and Management
  *
- *  See the README.md file in this directory for detailed information about Arrow's memory allocation subsystem.
+ *  See the README.md file in this directory for detailed information about Arrow's memory
+ *  allocation subsystem.
  *
  */
+
 package org.apache.arrow.memory;
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java b/java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java
index 28d078528974e..710f572e06027 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java
@@ -6,32 +6,33 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory.util;
 
 public class AssertionUtil {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
 
   public static final boolean ASSERT_ENABLED;
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
 
-  static{
+  static {
     boolean isAssertEnabled = false;
     assert isAssertEnabled = true;
     ASSERT_ENABLED = isAssertEnabled;
   }
 
-  public static boolean isAssertionsEnabled(){
-    return ASSERT_ENABLED;
+  private AssertionUtil() {
   }
 
-  private AssertionUtil() {
+  public static boolean isAssertionsEnabled() {
+    return ASSERT_ENABLED;
   }
 }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java b/java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java
index 94e5cc5fded4f..8d9008c894ac8 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java
@@ -6,15 +6,16 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory.util;
 
 import java.util.concurrent.locks.Lock;
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
index c9b5c5385c596..c464598bfb856 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
@@ -6,53 +6,43 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory.util;
 
+import org.slf4j.Logger;
+
 import java.util.Arrays;
 import java.util.LinkedList;
 
-import org.slf4j.Logger;
-
 /**
  * Utility class that can be used to log activity within a class
  * for later logging and debugging. Supports recording events and
  * recording the stack at the time they occur.
  */
 public class HistoricalLog {
-  private static class Event {
-    private final String note; // the event text
-    private final StackTrace stackTrace; // where the event occurred
-    private final long time;
-
-    public Event(final String note) {
-      this.note = note;
-      this.time = System.nanoTime();
-      stackTrace = new StackTrace();
-    }
-  }
 
   private final LinkedList<Event> history = new LinkedList<>();
   private final String idString; // the formatted id string
-  private Event firstEvent; // the first stack trace recorded
   private final int limit; // the limit on the number of events kept
+  private Event firstEvent; // the first stack trace recorded
 
   /**
    * Constructor. The format string will be formatted and have its arguments
    * substituted at the time this is called.
    *
-   * @param idStringFormat {@link String#format} format string that can be used
-   *     to identify this object in a log. Including some kind of unique identifier
-   *     that can be associated with the object instance is best.
-   * @param args for the format string, or nothing if none are required
+   * @param idStringFormat {@link String#format} format string that can be used to identify this
+   *                       object in a log. Including some kind of unique identifier that can be
+   *                       associated with the object instance is best.
+   * @param args           for the format string, or nothing if none are required
    */
   public HistoricalLog(final String idStringFormat, Object... args) {
     this(Integer.MAX_VALUE, idStringFormat, args);
@@ -61,7 +51,7 @@ public HistoricalLog(final String idStringFormat, Object... args) {
   /**
    * Constructor. The format string will be formatted and have its arguments
    * substituted at the time this is called.
-   *
+   * <p>
    * <p>This form supports the specification of a limit that will limit the
    * number of historical entries kept (which keeps down the amount of memory
    * used). With the limit, the first entry made is always kept (under the
@@ -70,12 +60,12 @@ public HistoricalLog(final String idStringFormat, Object... args) {
    * Each time a new entry is made, the oldest that is not the first is dropped.
    * </p>
    *
-   * @param limit the maximum number of historical entries that will be kept,
-   *   not including the first entry made
-   * @param idStringFormat {@link String#format} format string that can be used
-   *     to identify this object in a log. Including some kind of unique identifier
-   *     that can be associated with the object instance is best.
-   * @param args for the format string, or nothing if none are required
+   * @param limit          the maximum number of historical entries that will be kept, not including
+   *                       the first entry made
+   * @param idStringFormat {@link String#format} format string that can be used to identify this
+   *                       object in a log. Including some kind of unique identifier that can be
+   *                       associated with the object instance is best.
+   * @param args           for the format string, or nothing if none are required
    */
   public HistoricalLog(final int limit, final String idStringFormat, Object... args) {
     this.limit = limit;
@@ -88,7 +78,7 @@ public HistoricalLog(final int limit, final String idStringFormat, Object... arg
    * at the time this is called.
    *
    * @param noteFormat {@link String#format} format string that describes the event
-   * @param args for the format string, or nothing if none are required
+   * @param args       for the format string, or nothing if none are required
    */
   public synchronized void recordEvent(final String noteFormat, Object... args) {
     final String note = String.format(noteFormat, args);
@@ -113,23 +103,14 @@ public void buildHistory(final StringBuilder sb, boolean includeStackTrace) {
     buildHistory(sb, 0, includeStackTrace);
   }
 
-  /**
-   * Write the history of this object to the given {@link StringBuilder}. The history
-   * includes the identifying string provided at construction time, and all the recorded
-   * events with their stack traces.
-   *
-   * @param sb {@link StringBuilder} to write to
-   * @param additional an extra string that will be written between the identifying
-   *     information and the history; often used for a current piece of state
-   */
-
   /**
    *
    * @param sb
    * @param indent
    * @param includeStackTrace
    */
-  public synchronized void buildHistory(final StringBuilder sb, int indent, boolean includeStackTrace) {
+  public synchronized void buildHistory(final StringBuilder sb, int indent, boolean
+      includeStackTrace) {
     final char[] indentation = new char[indent];
     final char[] innerIndentation = new char[indent + 2];
     Arrays.fill(indentation, ' ');
@@ -140,7 +121,6 @@ public synchronized void buildHistory(final StringBuilder sb, int indent, boolea
         .append(idString)
         .append('\n');
 
-
     if (firstEvent != null) {
       sb.append(innerIndentation)
           .append(firstEvent.time)
@@ -151,7 +131,7 @@ public synchronized void buildHistory(final StringBuilder sb, int indent, boolea
         firstEvent.stackTrace.writeToBuilder(sb, indent + 2);
       }
 
-      for(final Event event : history) {
+      for (final Event event : history) {
         if (event == firstEvent) {
           continue;
         }
@@ -170,6 +150,16 @@ public synchronized void buildHistory(final StringBuilder sb, int indent, boolea
     }
   }
 
+  /**
+   * Write the history of this object to the given {@link StringBuilder}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param sb {@link StringBuilder} to write to
+   * @param additional an extra string that will be written between the identifying
+   *     information and the history; often used for a current piece of state
+   */
+
   /**
    * Write the history of this object to the given {@link Logger}. The history
    * includes the identifying string provided at construction time, and all the recorded
@@ -182,4 +172,17 @@ public void logHistory(final Logger logger) {
     buildHistory(sb, 0, true);
     logger.debug(sb.toString());
   }
+
+  private static class Event {
+
+    private final String note; // the event text
+    private final StackTrace stackTrace; // where the event occurred
+    private final long time;
+
+    public Event(final String note) {
+      this.note = note;
+      this.time = System.nanoTime();
+      stackTrace = new StackTrace();
+    }
+  }
 }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java b/java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java
index 638c2fb9a959e..bb4ea6c46179e 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java
@@ -6,15 +6,16 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory.util;
 
 import java.util.Arrays;
@@ -23,6 +24,7 @@
  * Convenient way of obtaining and manipulating stack traces for debugging.
  */
 public class StackTrace {
+
   private final StackTraceElement[] stackTraceElements;
 
   /**
@@ -36,10 +38,9 @@ public StackTrace() {
 
   /**
    * Write the stack trace to a StringBuilder.
-   * @param sb
-   *          where to write it
-   * @param indent
-   *          how many double spaces to indent each line
+   *
+   * @param sb     where to write it
+   * @param indent how many double spaces to indent each line
    */
   public void writeToBuilder(final StringBuilder sb, final int indent) {
     // create the indentation string
@@ -47,7 +48,7 @@ public void writeToBuilder(final StringBuilder sb, final int indent) {
     Arrays.fill(indentation, ' ');
 
     // write the stack trace in standard Java format
-    for(StackTraceElement ste : stackTraceElements) {
+    for (StackTraceElement ste : stackTraceElements) {
       sb.append(indentation)
           .append("at ")
           .append(ste.getClassName())
diff --git a/java/pom.xml b/java/pom.xml
index fa03783396ffb..774761f0c1e66 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -35,6 +35,7 @@
     <forkCount>2</forkCount>
     <jackson.version>2.7.1</jackson.version>
     <hadoop.version>2.7.1</hadoop.version>
+    <checkstyle.failOnViolation>false</checkstyle.failOnViolation>
   </properties>
 
   <scm>
@@ -269,6 +270,47 @@
           </gitDescribe>
         </configuration>
       </plugin>
+
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-checkstyle-plugin</artifactId>
+          <version>2.17</version>
+          <dependencies>
+            <dependency>
+              <groupId>com.puppycrawl.tools</groupId>
+              <artifactId>checkstyle</artifactId>
+              <version>6.15</version>
+            </dependency>
+            <dependency>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+              <version>${dep.guava.version}</version>
+            </dependency>
+          </dependencies>
+          <executions>
+            <execution>
+              <id>validate</id>
+              <phase>validate</phase>
+              <goals>
+                <goal>check</goal>
+              </goals>
+            </execution>
+          </executions>
+          <configuration>
+            <configLocation>google_checks.xml</configLocation>
+            <encoding>UTF-8</encoding>
+            <consoleOutput>true</consoleOutput>
+            <failsOnError>${checkstyle.failOnViolation}</failsOnError>
+            <failOnViolation>${checkstyle.failOnViolation}</failOnViolation>
+            <violationSeverity>warning</violationSeverity>
+            <format>xml</format>
+            <format>html</format>
+            <outputFile>${project.build.directory}/test/checkstyle-errors.xml</outputFile>
+            <linkXRef>false</linkXRef>
+          </configuration>
+        </plugin>
+
+
     </plugins>
     <pluginManagement>
 
@@ -382,6 +424,19 @@
                     <ignore />
                   </action>
                 </pluginExecution>
+                <pluginExecution>
+                  <pluginExecutionFilter>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-checkstyle-plugin</artifactId>
+                    <versionRange>[0,)</versionRange>
+                    <goals>
+                      <goal>check</goal>
+                    </goals>
+                  </pluginExecutionFilter>
+                  <action>
+                    <ignore />
+                  </action>
+                </pluginExecution>
               </pluginExecutions>
             </lifecycleMappingMetadata>
           </configuration>
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
index 7c0cadd9d77dd..24079b62da919 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
@@ -6,20 +6,17 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.arrow.tools;
 
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.net.Socket;
+package org.apache.arrow.tools;
 
 import com.google.common.base.Preconditions;
 
@@ -31,11 +28,14 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+
 public class EchoServer {
   private static final Logger LOGGER = LoggerFactory.getLogger(EchoServer.class);
-
-  private boolean closed = false;
   private final ServerSocket serverSocket;
+  private boolean closed = false;
 
   public EchoServer(int port) throws IOException {
     LOGGER.info("Starting echo server.");
@@ -43,22 +43,64 @@ public EchoServer(int port) throws IOException {
     LOGGER.info("Running echo server on port: " + port());
   }
 
-  public int port() { return serverSocket.getLocalPort(); }
+  public static void main(String[] args) throws Exception {
+    int port;
+    if (args.length > 0) {
+      port = Integer.parseInt(args[0]);
+    } else {
+      port = 8080;
+    }
+    new EchoServer(port).run();
+  }
+
+  public int port() {
+    return serverSocket.getLocalPort();
+  }
+
+  public void run() throws IOException {
+    try {
+      while (!closed) {
+        LOGGER.info("Waiting to accept new client connection.");
+        Socket clientSocket = serverSocket.accept();
+        LOGGER.info("Accepted new client connection.");
+        try (ClientConnection client = new ClientConnection(clientSocket)) {
+          try {
+            client.run();
+          } catch (IOException e) {
+            LOGGER.warn("Error handling client connection.", e);
+          }
+        }
+        LOGGER.info("Closed connection with client");
+      }
+    } catch (java.net.SocketException ex) {
+      if (!closed) throw ex;
+    } finally {
+      serverSocket.close();
+      LOGGER.info("Server closed.");
+    }
+  }
+
+  public void close() throws IOException {
+    closed = true;
+    serverSocket.close();
+  }
 
   public static class ClientConnection implements AutoCloseable {
     public final Socket socket;
+
     public ClientConnection(Socket socket) {
       this.socket = socket;
     }
 
     public void run() throws IOException {
-      BufferAllocator  allocator = new RootAllocator(Long.MAX_VALUE);
+      BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
       // Read the entire input stream and write it back
       try (ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
         VectorSchemaRoot root = reader.getVectorSchemaRoot();
         // load the first batch before instantiating the writer so that we have any dictionaries
         reader.loadNextBatch();
-        try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream())) {
+        try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
+            .getOutputStream())) {
           writer.start();
           int echoed = 0;
           while (true) {
@@ -83,42 +125,4 @@ public void close() throws IOException {
       socket.close();
     }
   }
-
-  public void run() throws IOException {
-    try {
-      while (!closed) {
-        LOGGER.info("Waiting to accept new client connection.");
-        Socket clientSocket = serverSocket.accept();
-        LOGGER.info("Accepted new client connection.");
-        try (ClientConnection client = new ClientConnection(clientSocket)) {
-          try {
-            client.run();
-          } catch (IOException e) {
-            LOGGER.warn("Error handling client connection.", e);
-          }
-        }
-        LOGGER.info("Closed connection with client");
-      }
-    } catch (java.net.SocketException ex) {
-      if (!closed) throw ex;
-    } finally {
-      serverSocket.close();
-      LOGGER.info("Server closed.");
-    }
-  }
-
-  public void close() throws IOException {
-    closed = true;
-    serverSocket.close();
-  }
-
-  public static void main(String[] args) throws Exception {
-    int port;
-    if (args.length > 0) {
-      port = Integer.parseInt(args[0]);
-    } else {
-      port = 8080;
-    }
-    new EchoServer(port).run();
-  }
 }
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
index 9fa7b761a5772..b8621920d3348 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
@@ -16,13 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.arrow.tools;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
+package org.apache.arrow.tools;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -38,17 +33,17 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
 public class FileRoundtrip {
   private static final Logger LOGGER = LoggerFactory.getLogger(FileRoundtrip.class);
-
-  public static void main(String[] args) {
-    System.exit(new FileRoundtrip(System.out, System.err).run(args));
-  }
-
   private final Options options;
   private final PrintStream out;
   private final PrintStream err;
-
   FileRoundtrip(PrintStream out, PrintStream err) {
     this.out = out;
     this.err = err;
@@ -58,6 +53,10 @@ public static void main(String[] args) {
 
   }
 
+  public static void main(String[] args) {
+    System.exit(new FileRoundtrip(System.out, System.err).run(args));
+  }
+
   private File validateFile(String type, String fileName) {
     if (fileName == null) {
       throw new IllegalArgumentException("missing " + type + " file parameter");
@@ -81,7 +80,8 @@ int run(String[] args) {
       File outFile = validateFile("output", outFileName);
       BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); // TODO: close
       try (FileInputStream fileInputStream = new FileInputStream(inFile);
-           ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), allocator)) {
+           ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(),
+               allocator)) {
 
         VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
         Schema schema = root.getSchema();
@@ -89,7 +89,8 @@ int run(String[] args) {
         LOGGER.debug("Found schema: " + schema);
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(outFile);
-             ArrowFileWriter arrowWriter = new ArrowFileWriter(root, arrowReader, fileOutputStream.getChannel())) {
+             ArrowFileWriter arrowWriter = new ArrowFileWriter(root, arrowReader,
+                 fileOutputStream.getChannel())) {
           arrowWriter.start();
           while (true) {
             arrowReader.loadNextBatch();
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
index d5345535d19dc..be404fd4c5950 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
@@ -6,22 +6,17 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.arrow.tools;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
+package org.apache.arrow.tools;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -29,6 +24,12 @@
 import org.apache.arrow.vector.file.ArrowFileReader;
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
 /**
  * Converts an Arrow file to an Arrow stream. The file should be specified as the
  * first argument and the output is written to standard out.
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index 5d4849c234383..453693d7fa489 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -16,15 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.arrow.tools;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
+package org.apache.arrow.tools;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -44,8 +37,25 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
 public class Integration {
   private static final Logger LOGGER = LoggerFactory.getLogger(Integration.class);
+  private final Options options;
+
+  Integration() {
+    this.options = new Options();
+    this.options.addOption("a", "arrow", true, "arrow file");
+    this.options.addOption("j", "json", true, "json file");
+    this.options.addOption("c", "command", true, "command to execute: " + Arrays.toString(Command
+        .values()));
+  }
 
   public static void main(String[] args) {
     try {
@@ -59,20 +69,61 @@ public static void main(String[] args) {
     }
   }
 
-  private final Options options;
+  private static void fatalError(String message, Throwable e) {
+    System.err.println(message);
+    System.err.println(e.getMessage());
+    LOGGER.error(message, e);
+    System.exit(1);
+  }
+
+  private File validateFile(String type, String fileName, boolean shouldExist) {
+    if (fileName == null) {
+      throw new IllegalArgumentException("missing " + type + " file parameter");
+    }
+    File f = new File(fileName);
+    if (shouldExist && (!f.exists() || f.isDirectory())) {
+      throw new IllegalArgumentException(type + " file not found: " + f.getAbsolutePath());
+    }
+    if (!shouldExist && f.exists()) {
+      throw new IllegalArgumentException(type + " file already exists: " + f.getAbsolutePath());
+    }
+    return f;
+  }
+
+  void run(String[] args) throws ParseException, IOException {
+    CommandLineParser parser = new PosixParser();
+    CommandLine cmd = parser.parse(options, args, false);
+
+
+    Command command = toCommand(cmd.getOptionValue("command"));
+    File arrowFile = validateFile("arrow", cmd.getOptionValue("arrow"), command.arrowExists);
+    File jsonFile = validateFile("json", cmd.getOptionValue("json"), command.jsonExists);
+    command.execute(arrowFile, jsonFile);
+  }
+
+  private Command toCommand(String commandName) {
+    try {
+      return Command.valueOf(commandName);
+    } catch (IllegalArgumentException e) {
+      throw new IllegalArgumentException("Unknown command: " + commandName + " expected one of "
+          + Arrays.toString(Command.values()));
+    }
+  }
 
   enum Command {
     ARROW_TO_JSON(true, false) {
       @Override
       public void execute(File arrowFile, File jsonFile) throws IOException {
-        try(BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
-            FileInputStream fileInputStream = new FileInputStream(arrowFile);
-            ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), allocator)) {
+        try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+             FileInputStream fileInputStream = new FileInputStream(arrowFile);
+             ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(),
+                 allocator)) {
           VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
           Schema schema = root.getSchema();
           LOGGER.debug("Input file size: " + arrowFile.length());
           LOGGER.debug("Found schema: " + schema);
-          try (JsonFileWriter writer = new JsonFileWriter(jsonFile, JsonFileWriter.config().pretty(true))) {
+          try (JsonFileWriter writer = new JsonFileWriter(jsonFile, JsonFileWriter.config()
+              .pretty(true))) {
             writer.start(schema);
             for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
               arrowReader.loadRecordBatch(rbBlock);
@@ -94,7 +145,8 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
           try (FileOutputStream fileOutputStream = new FileOutputStream(arrowFile);
                VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
                // TODO json dictionaries
-               ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel())) {
+               ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream
+                   .getChannel())) {
             arrowWriter.start();
             reader.read(root);
             while (root.getRowCount() != 0) {
@@ -113,7 +165,8 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
         try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
              JsonFileReader jsonReader = new JsonFileReader(jsonFile, allocator);
              FileInputStream fileInputStream = new FileInputStream(arrowFile);
-             ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), allocator)) {
+             ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(),
+                 allocator)) {
           Schema jsonSchema = jsonReader.start();
           VectorSchemaRoot arrowRoot = arrowReader.getVectorSchemaRoot();
           Schema arrowSchema = arrowRoot.getSchema();
@@ -135,7 +188,8 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
           boolean hasMoreJSON = jsonRoot != null;
           boolean hasMoreArrow = iterator.hasNext();
           if (hasMoreJSON || hasMoreArrow) {
-            throw new IllegalArgumentException("Unexpected RecordBatches. J:" + hasMoreJSON + " A:" + hasMoreArrow);
+            throw new IllegalArgumentException("Unexpected RecordBatches. J:" + hasMoreJSON + " "
+                + "A:" + hasMoreArrow);
           }
         }
       }
@@ -153,51 +207,4 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
 
   }
 
-  Integration() {
-    this.options = new Options();
-    this.options.addOption("a", "arrow", true, "arrow file");
-    this.options.addOption("j", "json", true, "json file");
-    this.options.addOption("c", "command", true, "command to execute: " + Arrays.toString(Command.values()));
-  }
-
-  private File validateFile(String type, String fileName, boolean shouldExist) {
-    if (fileName == null) {
-      throw new IllegalArgumentException("missing " + type + " file parameter");
-    }
-    File f = new File(fileName);
-    if (shouldExist && (!f.exists() || f.isDirectory())) {
-      throw new IllegalArgumentException(type + " file not found: " + f.getAbsolutePath());
-    }
-    if (!shouldExist && f.exists()) {
-      throw new IllegalArgumentException(type + " file already exists: " + f.getAbsolutePath());
-    }
-    return f;
-  }
-
-  void run(String[] args) throws ParseException, IOException {
-    CommandLineParser parser = new PosixParser();
-    CommandLine cmd = parser.parse(options, args, false);
-
-
-    Command command = toCommand(cmd.getOptionValue("command"));
-    File arrowFile = validateFile("arrow", cmd.getOptionValue("arrow"), command.arrowExists);
-    File jsonFile = validateFile("json", cmd.getOptionValue("json"), command.jsonExists);
-    command.execute(arrowFile, jsonFile);
-  }
-
-  private Command toCommand(String commandName) {
-    try {
-      return Command.valueOf(commandName);
-    } catch (IllegalArgumentException e) {
-      throw new IllegalArgumentException("Unknown command: " + commandName + " expected one of " + Arrays.toString(Command.values()));
-    }
-  }
-
-  private static void fatalError(String message, Throwable e) {
-    System.err.println(message);
-    System.err.println(e.getMessage());
-    LOGGER.error(message, e);
-    System.exit(1);
-  }
-
 }
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
index 3b79d5b05e116..41dfd347be579 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
@@ -6,17 +6,24 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.tools;
 
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.file.ArrowFileWriter;
+import org.apache.arrow.vector.stream.ArrowStreamReader;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -25,12 +32,6 @@
 import java.io.OutputStream;
 import java.nio.channels.Channels;
 
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.file.ArrowFileWriter;
-import org.apache.arrow.vector.stream.ArrowStreamReader;
-
 /**
  * Converts an Arrow stream to an Arrow file.
  */
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
index f752f7eaa74b9..1a389098b4f47 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -16,13 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.arrow.tools;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
+package org.apache.arrow.tools;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
@@ -39,6 +34,12 @@
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Assert;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
 public class ArrowFileTestFixtures {
   static final int COUNT = 10;
 
@@ -58,9 +59,11 @@ static void writeData(int count, MapVector parent) {
 
   static void validateOutput(File testOutFile, BufferAllocator allocator) throws Exception {
     // read
-    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer
+        .MAX_VALUE);
          FileInputStream fileInputStream = new FileInputStream(testOutFile);
-         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(),
+             readerAllocator)) {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
@@ -81,16 +84,19 @@ static void validateContent(int count, VectorSchemaRoot root) {
   static void write(FieldVector parent, File file) throws FileNotFoundException, IOException {
     VectorSchemaRoot root = new VectorSchemaRoot(parent);
     try (FileOutputStream fileOutputStream = new FileOutputStream(file);
-         ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel())) {
+         ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream
+             .getChannel())) {
       arrowWriter.writeBatch();
     }
   }
 
 
-  static void writeInput(File testInFile, BufferAllocator allocator) throws FileNotFoundException, IOException {
+  static void writeInput(File testInFile, BufferAllocator allocator) throws
+      FileNotFoundException, IOException {
     int count = ArrowFileTestFixtures.COUNT;
     try (
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
+            Integer.MAX_VALUE);
         MapVector parent = new MapVector("parent", vectorAllocator, null)) {
       writeData(count, parent);
       write(parent.getChild("root"), testInFile);
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 706f8e2ca4d36..5970c57f46583 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -6,28 +6,17 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.arrow.tools;
 
-import static java.util.Arrays.asList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.net.Socket;
-import java.net.UnknownHostException;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
+package org.apache.arrow.tools;
 
 import com.google.common.collect.ImmutableList;
 
@@ -57,6 +46,18 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.io.IOException;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 public class EchoServerTest {
 
   private static EchoServer server;
@@ -94,8 +95,8 @@ private void testEchoServer(int serverPort,
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
     VectorSchemaRoot root = new VectorSchemaRoot(asList(field), asList((FieldVector) vector), 0);
     try (Socket socket = new Socket("localhost", serverPort);
-        ArrowStreamWriter writer = new ArrowStreamWriter(root, null, socket.getOutputStream());
-        ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), alloc)) {
+         ArrowStreamWriter writer = new ArrowStreamWriter(root, null, socket.getOutputStream());
+         ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), alloc)) {
       writer.start();
       for (int i = 0; i < batches; i++) {
         vector.allocateNew(16);
@@ -111,7 +112,8 @@ private void testEchoServer(int serverPort,
 
       assertEquals(new Schema(asList(field)), reader.getVectorSchemaRoot().getSchema());
 
-      NullableTinyIntVector readVector = (NullableTinyIntVector) reader.getVectorSchemaRoot().getFieldVectors().get(0);
+      NullableTinyIntVector readVector = (NullableTinyIntVector) reader.getVectorSchemaRoot()
+          .getFieldVectors().get(0);
       for (int i = 0; i < batches; i++) {
         reader.loadNextBatch();
         assertEquals(16, reader.getVectorSchemaRoot().getRowCount());
@@ -131,7 +133,8 @@ private void testEchoServer(int serverPort,
   public void basicTest() throws InterruptedException, IOException {
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
 
-    Field field = new Field("testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList());
+    Field field = new Field("testField", true, new ArrowType.Int(8, true), Collections
+        .<Field>emptyList());
     NullableTinyIntVector vector = new NullableTinyIntVector("testField", alloc, null);
     Schema schema = new Schema(asList(field));
 
@@ -150,7 +153,8 @@ public void testFlatDictionary() throws IOException {
     DictionaryEncoding writeEncoding = new DictionaryEncoding(1L, false, null);
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
          NullableIntVector writeVector = new NullableIntVector("varchar", allocator, writeEncoding);
-         NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dict", allocator, null)) {
+         NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dict",
+             allocator, null)) {
       writeVector.allocateNewSafe();
       NullableIntVector.Mutator mutator = writeVector.getMutator();
       mutator.set(0, 0);
@@ -171,10 +175,12 @@ public void testFlatDictionary() throws IOException {
       List<FieldVector> vectors = ImmutableList.of((FieldVector) writeVector);
       VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 6);
 
-      DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary(writeDictionaryVector, writeEncoding));
+      DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary
+          (writeDictionaryVector, writeEncoding));
 
       try (Socket socket = new Socket("localhost", serverPort);
-           ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket.getOutputStream());
+           ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket
+               .getOutputStream());
            ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
         writer.start();
         writer.writeBatch();
@@ -202,7 +208,8 @@ public void testFlatDictionary() throws IOException {
 
         Dictionary dictionary = reader.lookup(1L);
         Assert.assertNotNull(dictionary);
-        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
+        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary
+            .getVector()).getAccessor();
         Assert.assertEquals(3, dictionaryAccessor.getValueCount());
         Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
         Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
@@ -215,7 +222,8 @@ public void testFlatDictionary() throws IOException {
   public void testNestedDictionary() throws IOException {
     DictionaryEncoding writeEncoding = new DictionaryEncoding(2L, false, null);
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
-         NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dictionary", allocator, null);
+         NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dictionary",
+             allocator, null);
          ListVector writeVector = new ListVector("list", allocator, null, null)) {
 
       // data being written:
@@ -245,10 +253,12 @@ public void testNestedDictionary() throws IOException {
       List<FieldVector> vectors = ImmutableList.of((FieldVector) writeVector);
       VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 3);
 
-      DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary(writeDictionaryVector, writeEncoding));
+      DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary
+          (writeDictionaryVector, writeEncoding));
 
       try (Socket socket = new Socket("localhost", serverPort);
-           ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket.getOutputStream());
+           ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket
+               .getOutputStream());
            ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
         writer.start();
         writer.writeBatch();
@@ -262,7 +272,8 @@ public void testNestedDictionary() throws IOException {
         Assert.assertNotNull(readVector);
 
         Assert.assertNull(readVector.getField().getDictionary());
-        DictionaryEncoding readEncoding = readVector.getField().getChildren().get(0).getDictionary();
+        DictionaryEncoding readEncoding = readVector.getField().getChildren().get(0)
+            .getDictionary();
         Assert.assertNotNull(readEncoding);
         Assert.assertEquals(2L, readEncoding.getId());
 
@@ -281,7 +292,8 @@ public void testNestedDictionary() throws IOException {
 
         Dictionary readDictionary = reader.lookup(2L);
         Assert.assertNotNull(readDictionary);
-        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) readDictionary.getVector()).getAccessor();
+        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector)
+            readDictionary.getVector()).getAccessor();
         Assert.assertEquals(2, dictionaryAccessor.getValueCount());
         Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
         Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
index ee39f5e92c7b0..78021f8ad076c 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
@@ -16,13 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.arrow.tools;
 
-import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput;
-import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
+package org.apache.arrow.tools;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -32,6 +27,12 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.File;
+
+import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput;
+import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
+import static org.junit.Assert.assertEquals;
+
 public class TestFileRoundtrip {
 
   @Rule
@@ -56,7 +57,7 @@ public void test() throws Exception {
 
     writeInput(testInFile, allocator);
 
-    String[] args = { "-i", testInFile.getAbsolutePath(), "-o",  testOutFile.getAbsolutePath()};
+    String[] args = {"-i", testInFile.getAbsolutePath(), "-o", testOutFile.getAbsolutePath()};
     int result = new FileRoundtrip(System.out, System.err).run(args);
     assertEquals(0, result);
 
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
index 9d4ef5c26505b..7d9a41985bbe3 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -16,22 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.arrow.tools;
-
-import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput;
-import static org.apache.arrow.tools.ArrowFileTestFixtures.write;
-import static org.apache.arrow.tools.ArrowFileTestFixtures.writeData;
-import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.Map;
+package org.apache.arrow.tools;
 
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
@@ -54,12 +40,75 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Map;
+
+import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput;
+import static org.apache.arrow.tools.ArrowFileTestFixtures.write;
+import static org.apache.arrow.tools.ArrowFileTestFixtures.writeData;
+import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 public class TestIntegration {
 
   @Rule
   public TemporaryFolder testFolder = new TemporaryFolder();
 
   private BufferAllocator allocator;
+  private ObjectMapper om = new ObjectMapper();
+
+  {
+    DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
+    prettyPrinter.indentArraysWith(NopIndenter.instance);
+    om.setDefaultPrettyPrinter(prettyPrinter);
+    om.enable(SerializationFeature.INDENT_OUTPUT);
+    om.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
+  }
+
+  static void writeInputFloat(File testInFile, BufferAllocator allocator, double... f) throws
+      FileNotFoundException, IOException {
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
+            Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+      ComplexWriter writer = new ComplexWriterImpl("root", parent);
+      MapWriter rootWriter = writer.rootAsMap();
+      Float8Writer floatWriter = rootWriter.float8("float");
+      for (int i = 0; i < f.length; i++) {
+        floatWriter.setPosition(i);
+        floatWriter.writeFloat8(f[i]);
+      }
+      writer.setValueCount(f.length);
+      write(parent.getChild("root"), testInFile);
+    }
+  }
+
+  static void writeInput2(File testInFile, BufferAllocator allocator) throws
+      FileNotFoundException, IOException {
+    int count = ArrowFileTestFixtures.COUNT;
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
+            Integer.MAX_VALUE);
+        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+      writeData(count, parent);
+      ComplexWriter writer = new ComplexWriterImpl("root", parent);
+      MapWriter rootWriter = writer.rootAsMap();
+      IntWriter intWriter = rootWriter.integer("int");
+      BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+      intWriter.setPosition(5);
+      intWriter.writeInt(999);
+      bigIntWriter.setPosition(4);
+      bigIntWriter.writeBigInt(777L);
+      writer.setValueCount(count);
+      write(parent.getChild("root"), testInFile);
+    }
+  }
 
   @Before
   public void init() {
@@ -85,18 +134,21 @@ public void testValid() throws Exception {
     Integration integration = new Integration();
 
     // convert it to json
-    String[] args1 = { "-arrow", testInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    String[] args1 = {"-arrow", testInFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
     integration.run(args1);
 
     // convert back to arrow
-    String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
+    String[] args2 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
     integration.run(args2);
 
     // check it is the same
     validateOutput(testOutFile, allocator);
 
     // validate arrow against json
-    String[] args3 = { "-arrow", testInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()};
+    String[] args3 = {"-arrow", testInFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.VALIDATE.name()};
     integration.run(args3);
   }
 
@@ -111,11 +163,13 @@ public void testJSONRoundTripWithVariableWidth() throws Exception {
     Integration integration = new Integration();
 
     // convert to arrow
-    String[] args1 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
+    String[] args1 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
     integration.run(args1);
 
     // convert back to json
-    String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testRoundTripJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    String[] args2 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testRoundTripJSONFile
+        .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
     integration.run(args2);
 
     BufferedReader orig = readNormalized(testJSONFile);
@@ -139,11 +193,13 @@ public void testJSONRoundTripWithStruct() throws Exception {
     Integration integration = new Integration();
 
     // convert to arrow
-    String[] args1 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
+    String[] args1 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
     integration.run(args1);
 
     // convert back to json
-    String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json",  testRoundTripJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    String[] args2 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testRoundTripJSONFile
+        .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
     integration.run(args2);
 
     BufferedReader orig = readNormalized(testJSONFile);
@@ -156,22 +212,12 @@ public void testJSONRoundTripWithStruct() throws Exception {
     }
   }
 
-  private ObjectMapper om = new ObjectMapper();
-  {
-    DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
-    prettyPrinter.indentArraysWith(NopIndenter.instance);
-    om.setDefaultPrettyPrinter(prettyPrinter);
-    om.enable(SerializationFeature.INDENT_OUTPUT);
-    om.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
-  }
-
   private BufferedReader readNormalized(File f) throws IOException {
-    Map<?,?> tree = om.readValue(f, Map.class);
+    Map<?, ?> tree = om.readValue(f, Map.class);
     String normalized = om.writeValueAsString(tree);
     return new BufferedReader(new StringReader(normalized));
   }
 
-
   /**
    * the test should not be sensitive to small variations in float representation
    */
@@ -190,11 +236,13 @@ public void testFloat() throws Exception {
     Integration integration = new Integration();
 
     // convert the "valid" file to json
-    String[] args1 = { "-arrow", testValidInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    String[] args1 = {"-arrow", testValidInFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
     integration.run(args1);
 
     // compare the "invalid" file to the "valid" json
-    String[] args3 = { "-arrow", testInvalidInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()};
+    String[] args3 = {"-arrow", testInvalidInFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.VALIDATE.name()};
     // this should fail
     integration.run(args3);
   }
@@ -214,11 +262,13 @@ public void testInvalid() throws Exception {
     Integration integration = new Integration();
 
     // convert the "valid" file to json
-    String[] args1 = { "-arrow", testValidInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
+    String[] args1 = {"-arrow", testValidInFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
     integration.run(args1);
 
     // compare the "invalid" file to the "valid" json
-    String[] args3 = { "-arrow", testInvalidInFile.getAbsolutePath(), "-json",  testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()};
+    String[] args3 = {"-arrow", testInvalidInFile.getAbsolutePath(), "-json", testJSONFile
+        .getAbsolutePath(), "-command", Command.VALIDATE.name()};
     // this should fail
     try {
       integration.run(args3);
@@ -229,39 +279,4 @@ public void testInvalid() throws Exception {
     }
 
   }
-
-  static void writeInputFloat(File testInFile, BufferAllocator allocator, double... f) throws FileNotFoundException, IOException {
-    try (
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
-      ComplexWriter writer = new ComplexWriterImpl("root", parent);
-      MapWriter rootWriter = writer.rootAsMap();
-      Float8Writer floatWriter = rootWriter.float8("float");
-      for (int i = 0; i < f.length; i++) {
-        floatWriter.setPosition(i);
-        floatWriter.writeFloat8(f[i]);
-      }
-      writer.setValueCount(f.length);
-      write(parent.getChild("root"), testInFile);
-    }
-  }
-
-  static void writeInput2(File testInFile, BufferAllocator allocator) throws FileNotFoundException, IOException {
-    int count = ArrowFileTestFixtures.COUNT;
-    try (
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
-      writeData(count, parent);
-      ComplexWriter writer = new ComplexWriterImpl("root", parent);
-      MapWriter rootWriter = writer.rootAsMap();
-      IntWriter intWriter = rootWriter.integer("int");
-      BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
-      intWriter.setPosition(5);
-      intWriter.writeInt(999);
-      bigIntWriter.setPosition(4);
-      bigIntWriter.writeBigInt(777L);
-      writer.setValueCount(count);
-      write(parent.getChild("root"), testInFile);
-    }
-  }
 }

From 55d8f99c351c22c2357924b4e70fcef7c8fd119a Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Tue, 21 Mar 2017 18:15:52 -0700
Subject: [PATCH 0392/1644] ARROW-677: [java] Fix checkstyle jcl-over-slf4j
 conflict issue

Author: Julien Le Dem <julien@dremio.com>

Closes #412 from julienledem/checkstyle_slf4j and squashes the following commits:

2fda6b8 [Julien Le Dem] ARROW-677: [java] Fix checkstyle jcl-over-slf4j conflict issue
---
 java/pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/java/pom.xml b/java/pom.xml
index 774761f0c1e66..5edd605e8eedb 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -286,6 +286,11 @@
               <artifactId>guava</artifactId>
               <version>${dep.guava.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>jcl-over-slf4j</artifactId>
+                <version>1.7.5</version>
+            </dependency>
           </dependencies>
           <executions>
             <execution>

From 82b15a4c38d5bc3bf0e2e1ff27a0dfc7c8929551 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 22 Mar 2017 00:15:19 -0400
Subject: [PATCH 0393/1644] ARROW-678: [GLib] Fix dependencies

libarrow-glib-io.so should link to libarrow-glib.so.

libarrow-glib-ipc.so should link to libarrow-glib.so and
libarrow-io-glib.so.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #413 from kou/glib-fix-dependencies and squashes the following commits:

f67c04e [Kouhei Sutou] [GLib] Fix dependencies
---
 c_glib/arrow-glib/Makefile.am | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 7699594b7ade7..a948007741cbd 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -232,7 +232,8 @@ libarrow_io_glib_la_CXXFLAGS =			\
 
 libarrow_io_glib_la_LIBADD =			\
 	$(GLIB_LIBS)				\
-	$(ARROW_IO_LIBS)
+	$(ARROW_IO_LIBS)			\
+	libarrow-glib.la
 
 libarrow_io_glib_la_headers =			\
 	arrow-io-glib.h				\
@@ -329,7 +330,9 @@ libarrow_ipc_glib_la_CXXFLAGS =			\
 
 libarrow_ipc_glib_la_LIBADD =			\
 	$(GLIB_LIBS)				\
-	$(ARROW_IPC_LIBS)
+	$(ARROW_IPC_LIBS)			\
+	libarrow-glib.la			\
+	libarrow-io-glib.la
 
 libarrow_ipc_glib_la_headers =			\
 	arrow-ipc-glib.h			\
@@ -448,9 +451,7 @@ ArrowIO_1_0_gir_INCLUDES =			\
 	GObject-2.0
 ArrowIO_1_0_gir_CFLAGS =			\
 	$(AM_CPPFLAGS)
-ArrowIO_1_0_gir_LIBS =				\
-	libarrow-io-glib.la			\
-	libarrow-glib.la
+ArrowIO_1_0_gir_LIBS = libarrow-io-glib.la
 ArrowIO_1_0_gir_FILES = $(libarrow_io_glib_la_sources)
 ArrowIO_1_0_gir_SCANNERFLAGS =				\
 	--include-uninstalled=$(builddir)/Arrow-1.0.gir	\
@@ -469,10 +470,7 @@ ArrowIPC_1_0_gir_INCLUDES =			\
 	GObject-2.0
 ArrowIPC_1_0_gir_CFLAGS =			\
 	$(AM_CPPFLAGS)
-ArrowIPC_1_0_gir_LIBS =				\
-	libarrow-ipc-glib.la			\
-	libarrow-io-glib.la			\
-	libarrow-glib.la
+ArrowIPC_1_0_gir_LIBS = libarrow-ipc-glib.la
 ArrowIPC_1_0_gir_FILES = $(libarrow_ipc_glib_la_sources)
 ArrowIPC_1_0_gir_SCANNERFLAGS =					\
 	--include-uninstalled=$(builddir)/Arrow-1.0.gir		\

From d25286718c283bec0b1fd4cbe47ddb3f159c29b5 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 22 Mar 2017 00:17:01 -0400
Subject: [PATCH 0394/1644] ARROW-675: [GLib] Update package metadata

Author: Kouhei Sutou <kou@clear-code.com>

Closes #411 from kou/glib-update-metadata and squashes the following commits:

da7bba6 [Kouhei Sutou] [GLib] Update package metadata
---
 c_glib/configure.ac | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index 85f7eec3cb557..c6913437d93f8 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -18,7 +18,10 @@
 AC_PREREQ(2.65)
 
 m4_define([arrow_glib_version], m4_include(version))
-AC_INIT([arrow-glib], arrow_glib_version, [kou@clear-code.com])
+AC_INIT([arrow-glib],
+        arrow_glib_version,
+        [https://issues.apache.org/jira/browse/ARROW],
+        [apache-arrow-glib])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
 

From 96734efb73852f2d8372f72d7c56e8fb3ab4e516 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 22 Mar 2017 09:26:09 -0400
Subject: [PATCH 0395/1644] ARROW-654: [C++] Serialize timezone in IPC metadata

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #416 from wesm/ARROW-654 and squashes the following commits:

001708e [Wes McKinney] Fix API change in Python bindings
3729cf9 [Wes McKinney] Serialize timezone in IPC metadata
---
 cpp/src/arrow/ipc/feather-test.cc    |  2 +-
 cpp/src/arrow/ipc/feather.cc         |  2 +-
 cpp/src/arrow/ipc/metadata.cc        | 16 ++++++++++++++--
 cpp/src/arrow/ipc/test-common.h      |  2 +-
 cpp/src/arrow/memory_pool.cc         |  2 +-
 cpp/src/arrow/type-test.cc           |  2 +-
 cpp/src/arrow/type.cc                |  4 ++--
 cpp/src/arrow/type.h                 |  4 ++--
 python/pyarrow/includes/libarrow.pxd |  4 ++--
 python/pyarrow/schema.pyx            |  2 +-
 10 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index 078c3e10aff29..2513887f75903 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -355,7 +355,7 @@ TEST_F(TestTableWriter, TimeTypes) {
   auto f0 = field("f0", date32());
   auto f1 = field("f1", time(TimeUnit::MILLI));
   auto f2 = field("f2", timestamp(TimeUnit::NANO));
-  auto f3 = field("f3", timestamp("US/Los_Angeles", TimeUnit::SECOND));
+  auto f3 = field("f3", timestamp(TimeUnit::SECOND, "US/Los_Angeles"));
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2, f3}));
 
   std::vector<int64_t> values_vec = {0, 1, 2, 3, 4, 5, 6};
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 72bbaa4da3571..0dd9a8183fdc2 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -287,7 +287,7 @@ class TableReader::TableReaderImpl {
         } else {
           tz = "";
         }
-        *out = std::make_shared<TimestampType>(tz, unit);
+        *out = timestamp(unit, tz);
       } break;
       case fbs::TypeMetadata_DateMetadata:
         *out = date32();
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index a418d4893dd40..4dfda543ebf6b 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -46,6 +46,7 @@ using LargeRecordBatchOffset = flatbuffers::Offset<flatbuf::LargeRecordBatch>;
 using RecordBatchOffset = flatbuffers::Offset<flatbuf::RecordBatch>;
 using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
+using FBString = flatbuffers::Offset<flatbuffers::String>;
 
 static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
 
@@ -250,7 +251,12 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     }
     case flatbuf::Type_Timestamp: {
       auto ts_type = static_cast<const flatbuf::Timestamp*>(type_data);
-      *out = timestamp(FromFlatbufferUnit(ts_type->unit()));
+      TimeUnit unit = FromFlatbufferUnit(ts_type->unit());
+      if (ts_type->timezone() != 0 && ts_type->timezone()->Length() > 0) {
+        *out = timestamp(unit, ts_type->timezone()->str());
+      } else {
+        *out = timestamp(unit);
+      }
       return Status::OK();
     }
     case flatbuf::Type_Interval:
@@ -364,7 +370,13 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     case Type::TIMESTAMP: {
       const auto& ts_type = static_cast<const TimestampType&>(*type);
       *out_type = flatbuf::Type_Timestamp;
-      *offset = flatbuf::CreateTimestamp(fbb, ToFlatbufferUnit(ts_type.unit)).Union();
+
+      flatbuf::TimeUnit fb_unit = ToFlatbufferUnit(ts_type.unit);
+      FBString fb_timezone = 0;
+      if (ts_type.timezone.size() > 0) {
+        fb_timezone = fbb.CreateString(ts_type.timezone);
+      }
+      *offset = flatbuf::CreateTimestamp(fbb, fb_unit, fb_timezone).Union();
     } break;
     case Type::LIST:
       *out_type = flatbuf::Type_List;
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index ba203b090b3b7..330af0c6ced20 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -497,7 +497,7 @@ Status MakeDate32(std::shared_ptr<RecordBatch>* out) {
 Status MakeTimestamps(std::shared_ptr<RecordBatch>* out) {
   std::vector<bool> is_valid = {true, true, true, false, true, true, true};
   auto f0 = field("f0", timestamp(TimeUnit::MILLI));
-  auto f1 = field("f1", timestamp(TimeUnit::NANO));
+  auto f1 = field("f1", timestamp(TimeUnit::NANO, "America/New_York"));
   auto f2 = field("f2", timestamp(TimeUnit::SECOND));
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index cf01a02938385..7992f229862bf 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -19,10 +19,10 @@
 
 #include <algorithm>
 #include <cstdlib>
+#include <iostream>
 #include <mutex>
 #include <sstream>
 #include <stdlib.h>
-#include <iostream>
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index ddfff8745b97e..22aa7eba8a3e8 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -209,7 +209,7 @@ TEST(TestTimestampType, Equals) {
 
 TEST(TestTimestampType, ToString) {
   auto t1 = timestamp(TimeUnit::MILLI);
-  auto t2 = timestamp("US/Eastern", TimeUnit::NANO);
+  auto t2 = timestamp(TimeUnit::NANO, "US/Eastern");
   auto t3 = timestamp(TimeUnit::SECOND);
   auto t4 = timestamp(TimeUnit::MICRO);
 
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index ee0a89ab8abea..64070cb13abd0 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -244,8 +244,8 @@ std::shared_ptr<DataType> timestamp(TimeUnit unit) {
   return std::make_shared<TimestampType>(unit);
 }
 
-std::shared_ptr<DataType> timestamp(const std::string& timezone, TimeUnit unit) {
-  return std::make_shared<TimestampType>(timezone, unit);
+std::shared_ptr<DataType> timestamp(TimeUnit unit, const std::string& timezone) {
+  return std::make_shared<TimestampType>(unit, timezone);
 }
 
 std::shared_ptr<DataType> time(TimeUnit unit) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index adc3161e9551a..27b28d2f42bc0 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -532,7 +532,7 @@ struct ARROW_EXPORT TimestampType : public FixedWidthType {
   explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
       : FixedWidthType(Type::TIMESTAMP), unit(unit) {}
 
-  explicit TimestampType(const std::string& timezone, TimeUnit unit = TimeUnit::MILLI)
+  explicit TimestampType(TimeUnit unit, const std::string& timezone)
       : FixedWidthType(Type::TIMESTAMP), unit(unit), timezone(timezone) {}
 
   TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
@@ -603,7 +603,7 @@ std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& val
 
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit unit);
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(
-    const std::string& timezone, TimeUnit unit);
+    TimeUnit unit, const std::string& timezone);
 std::shared_ptr<DataType> ARROW_EXPORT time(TimeUnit unit);
 
 std::shared_ptr<DataType> ARROW_EXPORT struct_(
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 705fe6b4a55ca..2d698d35b1b84 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -99,14 +99,14 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CArray] dictionary()
 
     shared_ptr[CDataType] timestamp(TimeUnit unit)
-    shared_ptr[CDataType] timestamp(const c_string& timezone, TimeUnit unit)
+    shared_ptr[CDataType] timestamp(TimeUnit unit, const c_string& timezone)
 
     cdef cppclass CMemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
 
     cdef cppclass CLoggingMemoryPool" arrow::LoggingMemoryPool"(CMemoryPool):
         CLoggingMemoryPool(CMemoryPool*)
-        
+
     cdef cppclass CBuffer" arrow::Buffer":
         uint8_t* data()
         int64_t size()
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 4bc938df668f8..ee38144e6e3db 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -314,7 +314,7 @@ def timestamp(unit_str, tz=None):
             tz = tz.zone
 
         c_timezone = tobytes(tz)
-        out.init(la.timestamp(c_timezone, unit))
+        out.init(la.timestamp(unit, c_timezone))
 
     return out
 

From b179ad2d80c3f3c1ab81bfa9ff0c343fb47b148a Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Wed, 22 Mar 2017 10:05:19 -0400
Subject: [PATCH 0396/1644] =?UTF-8?q?ARROW-681:=20[C++]=20Disable=20boost'?=
 =?UTF-8?q?s=20autolinking=20if=20shared=20boost=20is=20used=20=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…on Windows; Correct linking with IMPORTED_IMPLIB of 3rd party shared libs on WIndows.

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #415 from MaxRis/master and squashes the following commits:

9fd851d [Max Risuhin] ARROW-681: [C++] Disable boost's autolinking if shared boost is used on Windows; Correct linking with IMPORTED_IMPLIB of 3rd party shared libs on WIndows.
---
 cpp/CMakeLists.txt | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 956658a82524c..84158cc008132 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -379,8 +379,15 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
 
     SET(AUG_LIB_NAME "${LIB_NAME}_shared")
     add_library(${AUG_LIB_NAME} SHARED IMPORTED)
-    set_target_properties(${AUG_LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+
+    if(MSVC)
+        # Mark the ”.lib” location as part of a Windows DLL
+        set_target_properties(${AUG_LIB_NAME}
+            PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
+    else()
+        set_target_properties(${AUG_LIB_NAME}
+            PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    endif()
     message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
   elseif(ARG_STATIC_LIB)
     add_library(${LIB_NAME} STATIC IMPORTED)
@@ -397,8 +404,15 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
       PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     SET(AUG_LIB_NAME "${LIB_NAME}_shared")
     add_library(${AUG_LIB_NAME} SHARED IMPORTED)
-    set_target_properties(${AUG_LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+
+    if(MSVC)
+        # Mark the ”.lib” location as part of a Windows DLL
+        set_target_properties(${AUG_LIB_NAME}
+            PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
+    else()
+        set_target_properties(${AUG_LIB_NAME}
+            PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    endif()
     message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
   else()
     message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
@@ -418,6 +432,15 @@ set(Boost_USE_MULTITHREADED ON)
 if (ARROW_BOOST_USE_SHARED)
   # Find shared Boost libraries.
   set(Boost_USE_STATIC_LIBS OFF)
+
+  if(MSVC)
+    # disable autolinking in boost
+    add_definitions(-DBOOST_ALL_NO_LIB)
+
+    # force all boost libraries to dynamic link
+    add_definitions(-DBOOST_ALL_DYN_LINK)
+  endif()
+
   find_package(Boost COMPONENTS system filesystem REQUIRED)
   if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})

From 5fda24776d82cf120525d298ba261ddd02e5fcc8 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 22 Mar 2017 11:04:42 -0400
Subject: [PATCH 0397/1644] ARROW-680: [C++] Support CMake 2 or older again

GNUInstallDirs in CMake 2 always uses multiarch cared library directory.

See also: https://github.com/Kitware/CMake/commit/620939e4e6f5a61cd5c0fac2704de4bfda0eb7ef

Author: Kouhei Sutou <kou@clear-code.com>

Closes #419 from kou/cpp-support-cmake-2 and squashes the following commits:

684cb2b [Kouhei Sutou] [C++] Support CMake 2 or older again
---
 cpp/CMakeLists.txt | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 84158cc008132..61e645da20e75 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -28,7 +28,13 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
 include(CMakeParseArguments)
 include(ExternalProject)
-include(GNUInstallDirs)
+
+if(CMAKE_MAJOR_VERSION LESS 3)
+  set(CMAKE_INSTALL_INCLUDEDIR "include")
+  set(CMAKE_INSTALL_LIBDIR "lib")
+else()
+  include(GNUInstallDirs)
+endif()
 
 set(ARROW_SO_VERSION "0")
 set(ARROW_ABI_VERSION "${ARROW_SO_VERSION}.0.0")

From 1b957dcf1a025a45a858091e51138b4a75c3826a Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 22 Mar 2017 11:48:41 -0400
Subject: [PATCH 0398/1644] ARROW-688: [C++] Use CMAKE_INSTALL_INCLUDEDIR for
 consistency

Using CMAKE_INSTALL_INCLUDEDIR isn't required. It's just for
consistency. We already used CMAKE_INSTALL_INCLUDEDIR at
cpp/src/arrow/CMakeLists.txt.

Or we can revert CMAKE_INSTALL_INCLUDEDIR change at cpp/src/arrow/CMakeLists.txt.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #420 from kou/cpp-use-cmake-install-includedir-for-consistency and squashes the following commits:

ba7da0d [Kouhei Sutou] [C++] Use CMAKE_INSTALL_INCLUDEDIR for consistency
---
 cpp/src/arrow/io/CMakeLists.txt       | 2 +-
 cpp/src/arrow/ipc/CMakeLists.txt      | 2 +-
 cpp/src/arrow/jemalloc/CMakeLists.txt | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index af3acbf06d1ef..8aabf6496f8f7 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -109,7 +109,7 @@ install(FILES
   hdfs.h
   interfaces.h
   memory.h
-  DESTINATION include/arrow/io)
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/io")
 
 # pkg-config support
 configure_file(arrow-io.pc.in
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 5d470df0309b3..3a98a380e7019 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -144,7 +144,7 @@ install(FILES
   metadata.h
   reader.h
   writer.h
-  DESTINATION include/arrow/ipc)
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/ipc")
 
 # pkg-config support
 configure_file(arrow-ipc.pc.in
diff --git a/cpp/src/arrow/jemalloc/CMakeLists.txt b/cpp/src/arrow/jemalloc/CMakeLists.txt
index c7e6c6af97cff..b8e6e231a3dca 100644
--- a/cpp/src/arrow/jemalloc/CMakeLists.txt
+++ b/cpp/src/arrow/jemalloc/CMakeLists.txt
@@ -104,7 +104,7 @@ ARROW_BENCHMARK_LINK_LIBRARIES(jemalloc-builder-benchmark
 # Headers: top level
 install(FILES
   memory_pool.h
-  DESTINATION include/arrow/jemalloc)
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/jemalloc")
 
 # pkg-config support
 configure_file(arrow-jemalloc.pc.in

From 36103143b5975138522f4e54f8b21565a34f6504 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 22 Mar 2017 18:34:52 +0100
Subject: [PATCH 0399/1644] ARROW-689: [GLib] Fix install directories

Header files should be installed into
`${PREFIX}/include/arrow-glib/` instead of
`${PREFIX}/include/apache-arrow-glib/`.

Documents should be installed into
`${PREFIX}/share/doc/arrow-glib/` instead of
`${PREFIX}/share/doc/apache-arrow-glib/`.

We needed to change install directories when we changed `AC_INIT()`'s 3rd
argument to apache-arrow-glib...

Author: Kouhei Sutou <kou@clear-code.com>

Closes #421 from kou/glib-fix-install-directory and squashes the following commits:

65e5cee [Kouhei Sutou] [GLib] Fix install directories
---
 c_glib/Makefile.am            | 3 ++-
 c_glib/arrow-glib/Makefile.am | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am
index c078b0889d4ff..40e8395a56824 100644
--- a/c_glib/Makefile.am
+++ b/c_glib/Makefile.am
@@ -27,6 +27,7 @@ EXTRA_DIST =					\
 	LICENSE.txt				\
 	version
 
-doc_DATA =					\
+arrow_glib_docdir = ${datarootdir}/doc/arrow-glib
+arrow_glib_doc_DATA =				\
 	README.md				\
 	LICENSE.txt
diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index a948007741cbd..a72d1e874402a 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -403,7 +403,8 @@ stamp-ipc-enums.c: $(libarrow_ipc_glib_la_headers) ipc-enums.c.template
 	     $(libarrow_ipc_glib_la_headers)) > ipc-enums.c
 	touch $@
 
-pkginclude_HEADERS =					\
+arrow_glib_includedir = $(includedir)/arrow-glib
+arrow_glib_include_HEADERS =				\
 	$(libarrow_glib_la_headers)			\
 	$(libarrow_glib_la_cpp_headers)			\
 	$(libarrow_glib_la_generated_headers)		\

From 71424c20d31addb37cf7db56561790ca69db0430 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 22 Mar 2017 14:03:40 -0400
Subject: [PATCH 0400/1644] ARROW-683: [C++/Python] Refactor to make Date32 and
 Date64 types for new metadata. Test IPC roundtrip

Maintains existing Python behavior (datetime.date getting converted to milliseconds)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #418 from wesm/ARROW-683 and squashes the following commits:

69f156b [Wes McKinney] Add autoconf-archive to README for system requirements
10988ad [Wes McKinney] Remove hacks for ax_cxx_compile_stdcxx_11
334558d [Wes McKinney] Fix glib for date32/date64. Add ax_cxx_compile_stdcxx_11.m4 macro for older autoconf
93cf8d6 [Wes McKinney] Refactor to make Date32 and Date64 types for new metadata. Test IPC roundtrips, maintain existing Python behavior
---
 c_glib/README.md                             |  5 ++-
 c_glib/arrow-glib/type.cpp                   |  6 ++-
 c_glib/arrow-glib/type.h                     |  6 ++-
 cpp/src/arrow/array.cc                       |  4 +-
 cpp/src/arrow/array.h                        |  4 +-
 cpp/src/arrow/builder.cc                     |  5 ++-
 cpp/src/arrow/builder.h                      |  2 +-
 cpp/src/arrow/compare.cc                     | 10 +++--
 cpp/src/arrow/ipc/ipc-json-test.cc           |  8 ++--
 cpp/src/arrow/ipc/ipc-read-write-test.cc     |  8 ++--
 cpp/src/arrow/ipc/json-internal.cc           | 16 ++++----
 cpp/src/arrow/ipc/metadata.cc                | 18 +++++++--
 cpp/src/arrow/ipc/test-common.h              | 29 +++++---------
 cpp/src/arrow/ipc/writer.cc                  |  2 +-
 cpp/src/arrow/loader.cc                      |  2 +-
 cpp/src/arrow/pretty_print.cc                |  4 +-
 cpp/src/arrow/type-test.cc                   |  8 ++++
 cpp/src/arrow/type.cc                        | 12 +++---
 cpp/src/arrow/type.h                         | 40 ++++++++++----------
 cpp/src/arrow/type_fwd.h                     |  8 ++--
 cpp/src/arrow/type_traits.h                  |  8 ++--
 python/pyarrow/__init__.py                   |  2 +-
 python/pyarrow/array.pyx                     |  9 ++++-
 python/pyarrow/includes/libarrow.pxd         |  8 +++-
 python/pyarrow/scalar.pyx                    | 19 +++++++---
 python/pyarrow/schema.pyx                    | 13 +++++--
 python/pyarrow/tests/test_convert_builtin.py |  2 +-
 python/pyarrow/tests/test_convert_pandas.py  |  2 +-
 python/src/pyarrow/adapters/builtin.cc       |  6 +--
 python/src/pyarrow/adapters/pandas.cc        | 18 ++++-----
 python/src/pyarrow/helpers.cc                |  3 +-
 python/src/pyarrow/type_traits.h             |  2 +-
 32 files changed, 166 insertions(+), 123 deletions(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index 4008015a56438..84027bf2cb3db 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -58,7 +58,7 @@ to build Arrow GLib. You can install them by the followings:
 On Debian GNU/Linux or Ubuntu:
 
 ```text
-% sudo apt install -y -V gtk-doc-tools libgirepository1.0-dev
+% sudo apt install -y -V gtk-doc-tools autoconf-archive libgirepository1.0-dev
 ```
 
 On CentOS 7 or later:
@@ -76,7 +76,8 @@ On macOS with [Homebrew](https://brew.sh/):
 Now, you can build Arrow GLib:
 
 ```text
-% cd glib
+% cd c_glib
+% ./autogen.sh
 % ./configure --enable-gtk-doc
 % make
 % sudo make install
diff --git a/c_glib/arrow-glib/type.cpp b/c_glib/arrow-glib/type.cpp
index 56cbc212211eb..2e59647884551 100644
--- a/c_glib/arrow-glib/type.cpp
+++ b/c_glib/arrow-glib/type.cpp
@@ -66,8 +66,10 @@ garrow_type_from_raw(arrow::Type::type type)
     return GARROW_TYPE_STRING;
   case arrow::Type::type::BINARY:
     return GARROW_TYPE_BINARY;
-  case arrow::Type::type::DATE:
-    return GARROW_TYPE_DATE;
+  case arrow::Type::type::DATE32:
+    return GARROW_TYPE_DATE32;
+  case arrow::Type::type::DATE64:
+    return GARROW_TYPE_DATE64;
   case arrow::Type::type::TIMESTAMP:
     return GARROW_TYPE_TIMESTAMP;
   case arrow::Type::type::TIME:
diff --git a/c_glib/arrow-glib/type.h b/c_glib/arrow-glib/type.h
index 48d2801dad42c..cd6137cb5ba5f 100644
--- a/c_glib/arrow-glib/type.h
+++ b/c_glib/arrow-glib/type.h
@@ -40,7 +40,8 @@ G_BEGIN_DECLS
  * @GARROW_TYPE_DOUBLE: 8-byte floating point value.
  * @GARROW_TYPE_STRING: UTF-8 variable-length string.
  * @GARROW_TYPE_BINARY: Variable-length bytes (no guarantee of UTF-8-ness).
- * @GARROW_TYPE_DATE: By default, int32 days since the UNIX epoch.
+ * @GARROW_TYPE_DATE32: int32 days since the UNIX epoch.
+ * @GARROW_TYPE_DATE64: int64 milliseconds since the UNIX epoch.
  * @GARROW_TYPE_TIMESTAMP: Exact timestamp encoded with int64 since UNIX epoch.
  *   Default unit millisecond.
  * @GARROW_TYPE_TIME: Exact time encoded with int64, default unit millisecond.
@@ -70,7 +71,8 @@ typedef enum {
   GARROW_TYPE_DOUBLE,
   GARROW_TYPE_STRING,
   GARROW_TYPE_BINARY,
-  GARROW_TYPE_DATE,
+  GARROW_TYPE_DATE32,
+  GARROW_TYPE_DATE64,
   GARROW_TYPE_TIMESTAMP,
   GARROW_TYPE_TIME,
   GARROW_TYPE_INTERVAL,
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 36b3fccf79ed0..4fa2b2b521f59 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -489,8 +489,8 @@ ARRAY_VISITOR_DEFAULT(DoubleArray);
 ARRAY_VISITOR_DEFAULT(BinaryArray);
 ARRAY_VISITOR_DEFAULT(StringArray);
 ARRAY_VISITOR_DEFAULT(FixedWidthBinaryArray);
-ARRAY_VISITOR_DEFAULT(DateArray);
 ARRAY_VISITOR_DEFAULT(Date32Array);
+ARRAY_VISITOR_DEFAULT(Date64Array);
 ARRAY_VISITOR_DEFAULT(TimeArray);
 ARRAY_VISITOR_DEFAULT(TimestampArray);
 ARRAY_VISITOR_DEFAULT(IntervalArray);
@@ -515,8 +515,8 @@ template class NumericArray<Int16Type>;
 template class NumericArray<Int32Type>;
 template class NumericArray<Int64Type>;
 template class NumericArray<TimestampType>;
-template class NumericArray<DateType>;
 template class NumericArray<Date32Type>;
+template class NumericArray<Date64Type>;
 template class NumericArray<TimeType>;
 template class NumericArray<HalfFloatType>;
 template class NumericArray<FloatType>;
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 50faf0892e8c0..e66ac505d5dbf 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -58,8 +58,8 @@ class ARROW_EXPORT ArrayVisitor {
   virtual Status Visit(const StringArray& array);
   virtual Status Visit(const BinaryArray& array);
   virtual Status Visit(const FixedWidthBinaryArray& array);
-  virtual Status Visit(const DateArray& array);
   virtual Status Visit(const Date32Array& array);
+  virtual Status Visit(const Date64Array& array);
   virtual Status Visit(const TimeArray& array);
   virtual Status Visit(const TimestampArray& array);
   virtual Status Visit(const IntervalArray& array);
@@ -559,8 +559,8 @@ extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
 extern template class ARROW_EXPORT NumericArray<FloatType>;
 extern template class ARROW_EXPORT NumericArray<DoubleType>;
 extern template class ARROW_EXPORT NumericArray<TimestampType>;
-extern template class ARROW_EXPORT NumericArray<DateType>;
 extern template class ARROW_EXPORT NumericArray<Date32Type>;
+extern template class ARROW_EXPORT NumericArray<Date64Type>;
 extern template class ARROW_EXPORT NumericArray<TimeType>;
 
 #if defined(__GNUC__) && !defined(__clang__)
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index b65a4928ec999..483d6f0a425ea 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -238,8 +238,8 @@ template class PrimitiveBuilder<Int8Type>;
 template class PrimitiveBuilder<Int16Type>;
 template class PrimitiveBuilder<Int32Type>;
 template class PrimitiveBuilder<Int64Type>;
-template class PrimitiveBuilder<DateType>;
 template class PrimitiveBuilder<Date32Type>;
+template class PrimitiveBuilder<Date64Type>;
 template class PrimitiveBuilder<TimestampType>;
 template class PrimitiveBuilder<TimeType>;
 template class PrimitiveBuilder<HalfFloatType>;
@@ -531,7 +531,8 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(INT32, Int32Builder);
     BUILDER_CASE(UINT64, UInt64Builder);
     BUILDER_CASE(INT64, Int64Builder);
-    BUILDER_CASE(DATE, DateBuilder);
+    BUILDER_CASE(DATE32, Date32Builder);
+    BUILDER_CASE(DATE64, Date64Builder);
     case Type::TIMESTAMP:
       out->reset(new TimestampBuilder(pool, type));
       return Status::OK();
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 07b7cfcb3a964..7cefa649cbf71 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -232,8 +232,8 @@ using Int32Builder = NumericBuilder<Int32Type>;
 using Int64Builder = NumericBuilder<Int64Type>;
 using TimestampBuilder = NumericBuilder<TimestampType>;
 using TimeBuilder = NumericBuilder<TimeType>;
-using DateBuilder = NumericBuilder<DateType>;
 using Date32Builder = NumericBuilder<Date32Type>;
+using Date64Builder = NumericBuilder<Date64Type>;
 
 using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
 using FloatBuilder = NumericBuilder<FloatType>;
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 86ed8ccecd1ea..3e6ecefc5ca5b 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -169,12 +169,14 @@ class RangeEqualsVisitor : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DateArray& left) override { return CompareValues<DateArray>(left); }
-
   Status Visit(const Date32Array& left) override {
     return CompareValues<Date32Array>(left);
   }
 
+  Status Visit(const Date64Array& left) override {
+    return CompareValues<Date64Array>(left);
+  }
+
   Status Visit(const TimeArray& left) override { return CompareValues<TimeArray>(left); }
 
   Status Visit(const TimestampArray& left) override {
@@ -409,10 +411,10 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
 
   Status Visit(const DoubleArray& left) override { return ComparePrimitive(left); }
 
-  Status Visit(const DateArray& left) override { return ComparePrimitive(left); }
-
   Status Visit(const Date32Array& left) override { return ComparePrimitive(left); }
 
+  Status Visit(const Date64Array& left) override { return ComparePrimitive(left); }
+
   Status Visit(const TimeArray& left) override { return ComparePrimitive(left); }
 
   Status Visit(const TimestampArray& left) override { return ComparePrimitive(left); }
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 4c18a496f4c80..fd35182751948 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -96,15 +96,17 @@ void CheckPrimitive(const std::shared_ptr<DataType>& type,
 }
 
 TEST(TestJsonSchemaWriter, FlatTypes) {
+  // TODO
+  // field("f14", date32())
   std::vector<std::shared_ptr<Field>> fields = {field("f0", int8()),
       field("f1", int16(), false), field("f2", int32()), field("f3", int64(), false),
       field("f4", uint8()), field("f5", uint16()), field("f6", uint32()),
       field("f7", uint64()), field("f8", float32()), field("f9", float64()),
       field("f10", utf8()), field("f11", binary()), field("f12", list(int32())),
       field("f13", struct_({field("s1", int32()), field("s2", utf8())})),
-      field("f14", date()), field("f15", timestamp(TimeUnit::NANO)),
-      field("f16", time(TimeUnit::MICRO)),
-      field("f17", union_({field("u1", int8()), field("u2", time(TimeUnit::MILLI))},
+      field("f15", date64()), field("f16", timestamp(TimeUnit::NANO)),
+      field("f17", time(TimeUnit::MICRO)),
+      field("f18", union_({field("u1", int8()), field("u2", time(TimeUnit::MILLI))},
                        {0, 1}, UnionMode::DENSE))};
 
   Schema schema(fields);
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 261ca1d0e52d8..00118448ff044 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -117,10 +117,10 @@ TEST_F(TestSchemaMetadata, NestedFields) {
   CheckRoundtrip(schema, &memo);
 }
 
-#define BATCH_CASES()                                                                   \
-  ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
-      &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
-      &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate, &MakeTimestamps, &MakeTimes, \
+#define BATCH_CASES()                                                                    \
+  ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch,  \
+      &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,    \
+      &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes, \
       &MakeFWBinary);
 
 class IpcTestFixture : public io::MemoryMapFixture {
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 549b26bfe8201..08f0bdc3a023e 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -133,10 +133,7 @@ class JsonSchemaWriter : public TypeVisitor {
   }
 
   template <typename T>
-  typename std::enable_if<
-      std::is_base_of<NoExtraMeta, T>::value || std::is_base_of<BooleanType, T>::value ||
-          std::is_base_of<DateType, T>::value || std::is_base_of<NullType, T>::value,
-      void>::type
+  typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value, void>::type
   WriteTypeMetadata(const T& type) {}
 
   template <typename T>
@@ -303,7 +300,10 @@ class JsonSchemaWriter : public TypeVisitor {
 
   Status Visit(const BinaryType& type) override { return WriteVarBytes("binary", type); }
 
-  Status Visit(const DateType& type) override { return WritePrimitive("date", type); }
+  // TODO
+  Status Visit(const Date32Type& type) override { return WritePrimitive("date", type); }
+
+  Status Visit(const Date64Type& type) override { return WritePrimitive("date", type); }
 
   Status Visit(const TimeType& type) override { return WritePrimitive("time", type); }
 
@@ -733,7 +733,8 @@ class JsonSchemaReader {
     } else if (type_name == "null") {
       *type = null();
     } else if (type_name == "date") {
-      *type = date();
+      // TODO
+      *type = date64();
     } else if (type_name == "time") {
       return GetTimeLike<TimeType>(json_type, type);
     } else if (type_name == "timestamp") {
@@ -1059,7 +1060,8 @@ class JsonArrayReader {
       TYPE_CASE(DoubleType);
       TYPE_CASE(StringType);
       TYPE_CASE(BinaryType);
-      NOT_IMPLEMENTED_CASE(DATE);
+      NOT_IMPLEMENTED_CASE(DATE32);
+      NOT_IMPLEMENTED_CASE(DATE64);
       NOT_IMPLEMENTED_CASE(TIMESTAMP);
       NOT_IMPLEMENTED_CASE(TIME);
       NOT_IMPLEMENTED_CASE(INTERVAL);
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 4dfda543ebf6b..c091bac5a09e2 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -241,9 +241,15 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       return Status::OK();
     case flatbuf::Type_Decimal:
       return Status::NotImplemented("Decimal");
-    case flatbuf::Type_Date:
-      *out = date();
+    case flatbuf::Type_Date: {
+      auto date_type = static_cast<const flatbuf::Date*>(type_data);
+      if (date_type->unit() == flatbuf::DateUnit_DAY) {
+        *out = date32();
+      } else {
+        *out = date64();
+      }
       return Status::OK();
+    }
     case flatbuf::Type_Time: {
       auto time_type = static_cast<const flatbuf::Time*>(type_data);
       *out = time(FromFlatbufferUnit(time_type->unit()));
@@ -358,9 +364,13 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *out_type = flatbuf::Type_Utf8;
       *offset = flatbuf::CreateUtf8(fbb).Union();
       break;
-    case Type::DATE:
+    case Type::DATE32:
+      *out_type = flatbuf::Type_Date;
+      *offset = flatbuf::CreateDate(fbb, flatbuf::DateUnit_DAY).Union();
+      break;
+    case Type::DATE64:
       *out_type = flatbuf::Type_Date;
-      *offset = flatbuf::CreateDate(fbb).Union();
+      *offset = flatbuf::CreateDate(fbb, flatbuf::DateUnit_MILLISECOND).Union();
       break;
     case Type::TIME: {
       const auto& time_type = static_cast<const TimeType&>(*type);
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 330af0c6ced20..4085ecf9e3da9 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -463,33 +463,22 @@ Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
-Status MakeDate(std::shared_ptr<RecordBatch>* out) {
-  std::vector<bool> is_valid = {true, true, true, false, true, true, true};
-  auto f1 = field("f1", date());
-  std::shared_ptr<Schema> schema(new Schema({f1}));
-
-  std::vector<int64_t> date_values = {1489269000000, 1489270000000, 1489271000000,
-      1489272000000, 1489272000000, 1489273000000};
-
-  std::shared_ptr<Array> date_array;
-  ArrayFromVector<DateType, int64_t>(is_valid, date_values, &date_array);
-
-  std::vector<std::shared_ptr<Array>> arrays = {date_array};
-  *out = std::make_shared<RecordBatch>(schema, date_array->length(), arrays);
-  return Status::OK();
-}
-
-Status MakeDate32(std::shared_ptr<RecordBatch>* out) {
+Status MakeDates(std::shared_ptr<RecordBatch>* out) {
   std::vector<bool> is_valid = {true, true, true, false, true, true, true};
   auto f0 = field("f0", date32());
-  std::shared_ptr<Schema> schema(new Schema({f0}));
+  auto f1 = field("f1", date64());
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
   std::vector<int32_t> date32_values = {0, 1, 2, 3, 4, 5, 6};
-
   std::shared_ptr<Array> date32_array;
   ArrayFromVector<Date32Type, int32_t>(is_valid, date32_values, &date32_array);
 
-  std::vector<std::shared_ptr<Array>> arrays = {date32_array};
+  std::vector<int64_t> date64_values = {1489269000000, 1489270000000, 1489271000000,
+      1489272000000, 1489272000000, 1489273000000};
+  std::shared_ptr<Array> date64_array;
+  ArrayFromVector<Date64Type, int64_t>(is_valid, date64_values, &date64_array);
+
+  std::vector<std::shared_ptr<Array>> arrays = {date32_array, date64_array};
   *out = std::make_shared<RecordBatch>(schema, date32_array->length(), arrays);
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 82c119ef53e9a..ef59471e3c7c9 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -336,8 +336,8 @@ class RecordBatchWriter : public ArrayVisitor {
   VISIT_FIXED_WIDTH(HalfFloatArray);
   VISIT_FIXED_WIDTH(FloatArray);
   VISIT_FIXED_WIDTH(DoubleArray);
-  VISIT_FIXED_WIDTH(DateArray);
   VISIT_FIXED_WIDTH(Date32Array);
+  VISIT_FIXED_WIDTH(Date64Array);
   VISIT_FIXED_WIDTH(TimeArray);
   VISIT_FIXED_WIDTH(TimestampArray);
 
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
index fc373715105e1..bc506be572625 100644
--- a/cpp/src/arrow/loader.cc
+++ b/cpp/src/arrow/loader.cc
@@ -146,8 +146,8 @@ class ArrayLoader : public TypeVisitor {
   VISIT_PRIMITIVE(HalfFloatType);
   VISIT_PRIMITIVE(FloatType);
   VISIT_PRIMITIVE(DoubleType);
-  VISIT_PRIMITIVE(DateType);
   VISIT_PRIMITIVE(Date32Type);
+  VISIT_PRIMITIVE(Date64Type);
   VISIT_PRIMITIVE(TimeType);
   VISIT_PRIMITIVE(TimestampType);
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 87c1a1cf9d9c5..fc5eed18d8776 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -171,10 +171,10 @@ class ArrayPrinter : public ArrayVisitor {
 
   Status Visit(const FixedWidthBinaryArray& array) override { return WriteArray(array); }
 
-  Status Visit(const DateArray& array) override { return WriteArray(array); }
-
   Status Visit(const Date32Array& array) override { return WriteArray(array); }
 
+  Status Visit(const Date64Array& array) override { return WriteArray(array); }
+
   Status Visit(const TimeArray& array) override { return WriteArray(array); }
 
   Status Visit(const TimestampArray& array) override {
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 22aa7eba8a3e8..c2d115ccbfe6f 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -173,6 +173,14 @@ TEST(TestListType, Basics) {
   ASSERT_EQ("list<item: list<item: string>>", lt2.ToString());
 }
 
+TEST(TestDateTypes, ToString) {
+  auto t1 = date32();
+  auto t2 = date64();
+
+  ASSERT_EQ("date32[day]", t1->ToString());
+  ASSERT_EQ("date64[ms]", t2->ToString());
+}
+
 TEST(TestTimeType, Equals) {
   TimeType t1;
   TimeType t2;
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 64070cb13abd0..937cbc5a7669d 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -110,12 +110,12 @@ std::string StructType::ToString() const {
   return s.str();
 }
 
-std::string DateType::ToString() const {
-  return std::string("date");
+std::string Date64Type::ToString() const {
+  return std::string("date64[ms]");
 }
 
 std::string Date32Type::ToString() const {
-  return std::string("date32");
+  return std::string("date32[day]");
 }
 
 std::string TimeType::ToString() const {
@@ -205,7 +205,7 @@ ACCEPT_VISITOR(ListType);
 ACCEPT_VISITOR(StructType);
 ACCEPT_VISITOR(DecimalType);
 ACCEPT_VISITOR(UnionType);
-ACCEPT_VISITOR(DateType);
+ACCEPT_VISITOR(Date64Type);
 ACCEPT_VISITOR(Date32Type);
 ACCEPT_VISITOR(TimeType);
 ACCEPT_VISITOR(TimestampType);
@@ -233,7 +233,7 @@ TYPE_FACTORY(float32, FloatType);
 TYPE_FACTORY(float64, DoubleType);
 TYPE_FACTORY(utf8, StringType);
 TYPE_FACTORY(binary, BinaryType);
-TYPE_FACTORY(date, DateType);
+TYPE_FACTORY(date64, Date64Type);
 TYPE_FACTORY(date32, Date32Type);
 
 std::shared_ptr<DataType> fixed_width_binary(int32_t byte_width) {
@@ -355,7 +355,7 @@ TYPE_VISITOR_DEFAULT(DoubleType);
 TYPE_VISITOR_DEFAULT(StringType);
 TYPE_VISITOR_DEFAULT(BinaryType);
 TYPE_VISITOR_DEFAULT(FixedWidthBinaryType);
-TYPE_VISITOR_DEFAULT(DateType);
+TYPE_VISITOR_DEFAULT(Date64Type);
 TYPE_VISITOR_DEFAULT(Date32Type);
 TYPE_VISITOR_DEFAULT(TimeType);
 TYPE_VISITOR_DEFAULT(TimestampType);
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 27b28d2f42bc0..c179bf336987b 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -71,12 +71,12 @@ struct Type {
     // Fixed-width binary. Each value occupies the same number of bytes
     FIXED_WIDTH_BINARY,
 
-    // int64_t milliseconds since the UNIX epoch
-    DATE,
-
     // int32_t days since the UNIX epoch
     DATE32,
 
+    // int64_t milliseconds since the UNIX epoch
+    DATE64,
+
     // Exact timestamp encoded with int64 since UNIX epoch
     // Default unit millisecond
     TIMESTAMP,
@@ -139,7 +139,7 @@ class ARROW_EXPORT TypeVisitor {
   virtual Status Visit(const StringType& type);
   virtual Status Visit(const BinaryType& type);
   virtual Status Visit(const FixedWidthBinaryType& type);
-  virtual Status Visit(const DateType& type);
+  virtual Status Visit(const Date64Type& type);
   virtual Status Visit(const Date32Type& type);
   virtual Status Visit(const TimeType& type);
   virtual Status Visit(const TimestampType& type);
@@ -245,7 +245,7 @@ struct ARROW_EXPORT CTypeImpl : public PrimitiveCType {
   std::string ToString() const override { return std::string(DERIVED::name()); }
 };
 
-struct ARROW_EXPORT NullType : public DataType {
+struct ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
   static constexpr Type::type type_id = Type::NA;
 
   NullType() : DataType(Type::NA) {}
@@ -263,7 +263,7 @@ struct IntegerTypeImpl : public CTypeImpl<DERIVED, TYPE_ID, C_TYPE>, public Inte
   bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
 };
 
-struct ARROW_EXPORT BooleanType : public FixedWidthType {
+struct ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
   static constexpr Type::type type_id = Type::BOOL;
 
   BooleanType() : FixedWidthType(Type::BOOL) {}
@@ -455,33 +455,33 @@ struct ARROW_EXPORT UnionType : public DataType {
 // ----------------------------------------------------------------------
 // Date and time types
 
-/// Date as int64_t milliseconds since UNIX epoch
-struct ARROW_EXPORT DateType : public FixedWidthType {
-  static constexpr Type::type type_id = Type::DATE;
+/// Date as int32_t days since UNIX epoch
+struct ARROW_EXPORT Date32Type : public FixedWidthType, public NoExtraMeta {
+  static constexpr Type::type type_id = Type::DATE32;
 
-  using c_type = int64_t;
+  using c_type = int32_t;
 
-  DateType() : FixedWidthType(Type::DATE) {}
+  Date32Type() : FixedWidthType(Type::DATE32) {}
 
-  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "date"; }
 };
 
-/// Date as int32_t days since UNIX epoch
-struct ARROW_EXPORT Date32Type : public FixedWidthType {
-  static constexpr Type::type type_id = Type::DATE32;
+/// Date as int64_t milliseconds since UNIX epoch
+struct ARROW_EXPORT Date64Type : public FixedWidthType, public NoExtraMeta {
+  static constexpr Type::type type_id = Type::DATE64;
 
-  using c_type = int32_t;
+  using c_type = int64_t;
 
-  Date32Type() : FixedWidthType(Type::DATE32) {}
+  Date64Type() : FixedWidthType(Type::DATE64) {}
 
-  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+  static std::string name() { return "date"; }
 };
 
 enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
@@ -666,8 +666,8 @@ static inline bool is_primitive(Type::type type_id) {
     case Type::HALF_FLOAT:
     case Type::FLOAT:
     case Type::DOUBLE:
-    case Type::DATE:
     case Type::DATE32:
+    case Type::DATE64:
     case Type::TIMESTAMP:
     case Type::TIME:
     case Type::INTERVAL:
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 7fc36c4bde06b..ae85593cf4546 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -95,9 +95,9 @@ _NUMERIC_TYPE_DECL(Double);
 
 #undef _NUMERIC_TYPE_DECL
 
-struct DateType;
-using DateArray = NumericArray<DateType>;
-using DateBuilder = NumericBuilder<DateType>;
+struct Date64Type;
+using Date64Array = NumericArray<Date64Type>;
+using Date64Builder = NumericBuilder<Date64Type>;
 
 struct Date32Type;
 using Date32Array = NumericArray<Date32Type>;
@@ -132,8 +132,8 @@ std::shared_ptr<DataType> ARROW_EXPORT float32();
 std::shared_ptr<DataType> ARROW_EXPORT float64();
 std::shared_ptr<DataType> ARROW_EXPORT utf8();
 std::shared_ptr<DataType> ARROW_EXPORT binary();
-std::shared_ptr<DataType> ARROW_EXPORT date();
 std::shared_ptr<DataType> ARROW_EXPORT date32();
+std::shared_ptr<DataType> ARROW_EXPORT date64();
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 242e59d10fce4..e731913bbd226 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -119,15 +119,15 @@ struct TypeTraits<Int64Type> {
 };
 
 template <>
-struct TypeTraits<DateType> {
-  using ArrayType = DateArray;
-  using BuilderType = DateBuilder;
+struct TypeTraits<Date64Type> {
+  using ArrayType = Date64Array;
+  using BuilderType = Date64Builder;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(int64_t);
   }
   constexpr static bool is_parameter_free = true;
-  static inline std::shared_ptr<DataType> type_singleton() { return date(); }
+  static inline std::shared_ptr<DataType> type_singleton() { return date64(); }
 };
 
 template <>
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index a4aac443fae82..c6f0be04e8d0d 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -61,7 +61,7 @@
 from pyarrow.schema import (null, bool_,
                             int8, int16, int32, int64,
                             uint8, uint16, uint32, uint64,
-                            timestamp, date,
+                            timestamp, date32, date64,
                             float_, double, binary, string,
                             list_, struct, dictionary, field,
                             DataType, Field, Schema, schema)
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 11244e7836058..6afeaa0a7332b 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -372,7 +372,11 @@ cdef class UInt64Array(IntegerArray):
     pass
 
 
-cdef class DateArray(NumericArray):
+cdef class Date32Array(NumericArray):
+    pass
+
+
+cdef class Date64Array(NumericArray):
     pass
 
 
@@ -459,7 +463,8 @@ cdef dict _array_classes = {
     Type_INT16: Int16Array,
     Type_INT32: Int32Array,
     Type_INT64: Int64Array,
-    Type_DATE: DateArray,
+    Type_DATE32: Date32Array,
+    Type_DATE64: Date64Array,
     Type_FLOAT: FloatArray,
     Type_DOUBLE: DoubleArray,
     Type_LIST: ListArray,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 2d698d35b1b84..1d9c38e48cfe9 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -39,7 +39,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         Type_DOUBLE" arrow::Type::DOUBLE"
 
         Type_TIMESTAMP" arrow::Type::TIMESTAMP"
-        Type_DATE" arrow::Type::DATE"
+        Type_DATE32" arrow::Type::DATE32"
+        Type_DATE64" arrow::Type::DATE64"
         Type_BINARY" arrow::Type::BINARY"
         Type_STRING" arrow::Type::STRING"
 
@@ -177,7 +178,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CInt64Array" arrow::Int64Array"(CArray):
         int64_t Value(int i)
 
-    cdef cppclass CDateArray" arrow::DateArray"(CArray):
+    cdef cppclass CDate32Array" arrow::Date32Array"(CArray):
+        int32_t Value(int i)
+
+    cdef cppclass CDate64Array" arrow::Date64Array"(CArray):
         int64_t Value(int i)
 
     cdef cppclass CTimestampArray" arrow::TimestampArray"(CArray):
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 1337b2b2cb198..8c88f90422fac 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -124,11 +124,18 @@ cdef class UInt64Value(ArrayValue):
         return ap.Value(self.index)
 
 
-cdef class DateValue(ArrayValue):
+cdef class Date32Value(ArrayValue):
 
     def as_py(self):
-        cdef CDateArray* ap = <CDateArray*> self.sp_array.get()
-        return datetime.datetime.utcfromtimestamp(ap.Value(self.index) / 1000).date()
+        raise NotImplementedError
+
+
+cdef class Date64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
+        return datetime.datetime.utcfromtimestamp(
+            ap.Value(self.index) / 1000).date()
 
 
 cdef class TimestampValue(ArrayValue):
@@ -147,7 +154,8 @@ cdef class TimestampValue(ArrayValue):
             return datetime.datetime.utcfromtimestamp(float(val) / 1000000)
         else:
             # TimeUnit_NANO
-            raise NotImplementedError("Cannot convert nanosecond timestamps to datetime.datetime")
+            raise NotImplementedError("Cannot convert nanosecond timestamps "
+                                      "to datetime.datetime")
 
 
 cdef class FloatValue(ArrayValue):
@@ -226,7 +234,8 @@ cdef dict _scalar_classes = {
     Type_INT16: Int16Value,
     Type_INT32: Int32Value,
     Type_INT64: Int64Value,
-    Type_DATE: DateValue,
+    Type_DATE32: Date32Value,
+    Type_DATE64: Date64Value,
     Type_TIMESTAMP: TimestampValue,
     Type_FLOAT: FloatValue,
     Type_DOUBLE: DoubleValue,
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index ee38144e6e3db..ab5ae5fa2a3f2 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -228,8 +228,9 @@ cdef set PRIMITIVE_TYPES = set([
     la.Type_UINT16, la.Type_INT16,
     la.Type_UINT32, la.Type_INT32,
     la.Type_UINT64, la.Type_INT64,
-    la.Type_TIMESTAMP, la.Type_DATE,
-    la.Type_FLOAT, la.Type_DOUBLE])
+    la.Type_TIMESTAMP, la.Type_DATE32,
+    la.Type_DATE64, la.Type_FLOAT,
+    la.Type_DOUBLE])
 
 
 def null():
@@ -319,8 +320,12 @@ def timestamp(unit_str, tz=None):
     return out
 
 
-def date():
-    return primitive_type(la.Type_DATE)
+def date32():
+    return primitive_type(la.Type_DATE32)
+
+
+def date64():
+    return primitive_type(la.Type_DATE64)
 
 
 def float_():
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index c06d18d19c049..7915f9766bf67 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -97,7 +97,7 @@ def test_date(self):
                 datetime.date(2040, 2, 26)]
         arr = pyarrow.from_pylist(data)
         assert len(arr) == 4
-        assert arr.type == pyarrow.date()
+        assert arr.type == pyarrow.date64()
         assert arr.null_count == 1
         assert arr[0].as_py() == datetime.date(2000, 1, 1)
         assert arr[1].as_py() is None
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 6b89444b3e824..ea7a892a6f2a4 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -326,7 +326,7 @@ def test_date(self):
                      datetime.date(1970, 1, 1),
                      datetime.date(2040, 2, 26)]})
         table = A.Table.from_pandas(df)
-        field = A.Field.from_py('date', A.date())
+        field = A.Field.from_py('date', A.date64())
         schema = A.Schema.from_fields([field])
         assert table.schema.equals(schema)
         result = table.to_pandas()
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index b197f5845c020..06e098a80369e 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -82,7 +82,7 @@ class ScalarVisitor {
       // TODO(wesm): tighter type later
       return int64();
     } else if (date_count_) {
-      return date();
+      return date64();
     } else if (timestamp_count_) {
       return timestamp(TimeUnit::MICRO);
     } else if (bool_count_) {
@@ -291,7 +291,7 @@ class Int64Converter : public TypedConverter<Int64Builder> {
   }
 };
 
-class DateConverter : public TypedConverter<DateBuilder> {
+class DateConverter : public TypedConverter<Date64Builder> {
  public:
   Status AppendData(PyObject* seq) override {
     Py_ssize_t size = PySequence_Size(seq);
@@ -457,7 +457,7 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
       return std::make_shared<BoolConverter>();
     case Type::INT64:
       return std::make_shared<Int64Converter>();
-    case Type::DATE:
+    case Type::DATE64:
       return std::make_shared<DateConverter>();
     case Type::TIMESTAMP:
       return std::make_shared<TimestampConverter>();
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index 863cf54c9aa1c..a7386cefcdbbf 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -379,7 +379,7 @@ Status PandasConverter::ConvertDates(std::shared_ptr<Array>* out) {
   PyAcquireGIL lock;
 
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  DateBuilder date_builder(pool_);
+  Date64Builder date_builder(pool_);
   RETURN_NOT_OK(date_builder.Resize(length_));
 
   Status s;
@@ -477,7 +477,7 @@ Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
         return ConvertObjectStrings(out);
       case Type::BOOL:
         return ConvertBooleans(out);
-      case Type::DATE:
+      case Type::DATE64:
         return ConvertDates(out);
       case Type::LIST: {
         const auto& list_field = static_cast<const ListType&>(*type_);
@@ -725,7 +725,7 @@ inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out)
           break;
       }
     } else {
-      // datatype->type == Type::DATE
+      // datatype->type == Type::DATE64
       date_dtype->meta.base = NPY_FR_D;
     }
   }
@@ -1245,8 +1245,8 @@ class DatetimeBlock : public PandasBlock {
 
     const ChunkedArray& data = *col.get()->data();
 
-    if (type == Type::DATE) {
-      // DateType is millisecond timestamp stored as int64_t
+    if (type == Type::DATE64) {
+      // Date64Type is millisecond timestamp stored as int64_t
       // TODO(wesm): Do we want to make sure to zero out the milliseconds?
       ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
     } else if (type == Type::TIMESTAMP) {
@@ -1490,7 +1490,7 @@ class DataFrameBlockCreator {
         case Type::BINARY:
           output_type = PandasBlock::OBJECT;
           break;
-        case Type::DATE:
+        case Type::DATE64:
           output_type = PandasBlock::DATETIME;
           break;
         case Type::TIMESTAMP: {
@@ -1752,7 +1752,7 @@ class ArrowDeserializer {
       CONVERT_CASE(DOUBLE);
       CONVERT_CASE(BINARY);
       CONVERT_CASE(STRING);
-      CONVERT_CASE(DATE);
+      CONVERT_CASE(DATE64);
       CONVERT_CASE(TIMESTAMP);
       CONVERT_CASE(DICTIONARY);
       CONVERT_CASE(LIST);
@@ -1771,7 +1771,7 @@ class ArrowDeserializer {
 
   template <int TYPE>
   inline typename std::enable_if<
-      (TYPE != Type::DATE) & arrow_traits<TYPE>::is_numeric_nullable, Status>::type
+      (TYPE != Type::DATE64) & arrow_traits<TYPE>::is_numeric_nullable, Status>::type
   ConvertValues() {
     typedef typename arrow_traits<TYPE>::T T;
     int npy_type = arrow_traits<TYPE>::npy_type;
@@ -1788,7 +1788,7 @@ class ArrowDeserializer {
   }
 
   template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::DATE, Status>::type ConvertValues() {
+  inline typename std::enable_if<TYPE == Type::DATE64, Status>::type ConvertValues() {
     typedef typename arrow_traits<TYPE>::T T;
 
     RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
diff --git a/python/src/pyarrow/helpers.cc b/python/src/pyarrow/helpers.cc
index edebea6d97c95..43edf8af17fa2 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/python/src/pyarrow/helpers.cc
@@ -39,7 +39,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
       GET_PRIMITIVE_TYPE(INT32, int32);
       GET_PRIMITIVE_TYPE(UINT64, uint64);
       GET_PRIMITIVE_TYPE(INT64, int64);
-      GET_PRIMITIVE_TYPE(DATE, date);
+      GET_PRIMITIVE_TYPE(DATE32, date32);
+      GET_PRIMITIVE_TYPE(DATE64, date64);
       GET_PRIMITIVE_TYPE(BOOL, boolean);
       GET_PRIMITIVE_TYPE(FLOAT, float32);
       GET_PRIMITIVE_TYPE(DOUBLE, float64);
diff --git a/python/src/pyarrow/type_traits.h b/python/src/pyarrow/type_traits.h
index f4604d7a9894d..cc65d5ceed9c1 100644
--- a/python/src/pyarrow/type_traits.h
+++ b/python/src/pyarrow/type_traits.h
@@ -180,7 +180,7 @@ struct arrow_traits<Type::TIMESTAMP> {
 };
 
 template <>
-struct arrow_traits<Type::DATE> {
+struct arrow_traits<Type::DATE64> {
   static constexpr int npy_type = NPY_DATETIME;
   static constexpr bool supports_nulls = true;
   static constexpr int64_t na_value = kPandasTimestampNull;

From ced9d766d70e84c4d0542c6f5d9bd57faf10781d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 22 Mar 2017 14:05:33 -0400
Subject: [PATCH 0401/1644] ARROW-679: [Format] Change FieldNode, RecordBatch
 lengths to long, remove LargeRecordBatch. Refactoring

This enables me to delete a bunch of code without losing functionality. C++ users must explicitly opt-in to writing size over INT32_MAX.

cc @julienledem. I have not added checks in Java about sizes over INT32_MAX, wasn't sure where you might want to do that.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #417 from wesm/ARROW-679 and squashes the following commits:

ea237b1 [Wes McKinney] Document allow_64bit for WriteRecordBatch
e237d4a [Wes McKinney] Change FieldNode, RecordBatch lengths to long, remove LargeRecordBatch. Refactoring
---
 cpp/src/arrow/ipc/ipc-read-write-test.cc      |  2 +-
 cpp/src/arrow/ipc/metadata.cc                 | 48 +---------------
 cpp/src/arrow/ipc/metadata.h                  |  6 +-
 cpp/src/arrow/ipc/reader.cc                   | 55 +++----------------
 cpp/src/arrow/ipc/reader.h                    |  7 +--
 cpp/src/arrow/ipc/writer.cc                   | 52 ++++++------------
 cpp/src/arrow/ipc/writer.h                    | 51 +++++++++--------
 format/Message.fbs                            | 24 ++------
 .../arrow/vector/schema/ArrowFieldNode.java   |  2 +-
 .../vector/stream/MessageSerializer.java      |  4 +-
 10 files changed, 61 insertions(+), 190 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 00118448ff044..6919aebbe8d6d 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -163,7 +163,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
 
     RETURN_NOT_OK(WriteLargeRecordBatch(
         batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
-    return ReadLargeRecordBatch(batch.schema(), 0, mmap_.get(), result);
+    return ReadRecordBatch(batch.schema(), 0, mmap_.get(), result);
   }
 
   void CheckReadResult(const RecordBatch& result, const RecordBatch& expected) {
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index c091bac5a09e2..b10ccec9e7c4e 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -42,7 +42,6 @@ namespace ipc {
 using FBB = flatbuffers::FlatBufferBuilder;
 using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
 using FieldOffset = flatbuffers::Offset<flatbuf::Field>;
-using LargeRecordBatchOffset = flatbuffers::Offset<flatbuf::LargeRecordBatch>;
 using RecordBatchOffset = flatbuffers::Offset<flatbuf::RecordBatch>;
 using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
@@ -558,8 +557,6 @@ Status WriteSchemaMessage(
 
 using FieldNodeVector =
     flatbuffers::Offset<flatbuffers::Vector<const flatbuf::FieldNode*>>;
-using LargeFieldNodeVector =
-    flatbuffers::Offset<flatbuffers::Vector<const flatbuf::LargeFieldNode*>>;
 using BufferVector = flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Buffer*>>;
 
 static Status WriteFieldNodes(
@@ -567,23 +564,6 @@ static Status WriteFieldNodes(
   std::vector<flatbuf::FieldNode> fb_nodes;
   fb_nodes.reserve(nodes.size());
 
-  for (size_t i = 0; i < nodes.size(); ++i) {
-    const FieldMetadata& node = nodes[i];
-    if (node.offset != 0) {
-      return Status::Invalid("Field metadata for IPC must have offset 0");
-    }
-    fb_nodes.emplace_back(
-        static_cast<int32_t>(node.length), static_cast<int32_t>(node.null_count));
-  }
-  *out = fbb.CreateVectorOfStructs(fb_nodes);
-  return Status::OK();
-}
-
-static Status WriteLargeFieldNodes(
-    FBB& fbb, const std::vector<FieldMetadata>& nodes, LargeFieldNodeVector* out) {
-  std::vector<flatbuf::LargeFieldNode> fb_nodes;
-  fb_nodes.reserve(nodes.size());
-
   for (size_t i = 0; i < nodes.size(); ++i) {
     const FieldMetadata& node = nodes[i];
     if (node.offset != 0) {
@@ -621,19 +601,6 @@ static Status MakeRecordBatch(FBB& fbb, int32_t length, int64_t body_length,
   return Status::OK();
 }
 
-static Status MakeLargeRecordBatch(FBB& fbb, int64_t length, int64_t body_length,
-    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
-    LargeRecordBatchOffset* offset) {
-  LargeFieldNodeVector fb_nodes;
-  BufferVector fb_buffers;
-
-  RETURN_NOT_OK(WriteLargeFieldNodes(fbb, nodes, &fb_nodes));
-  RETURN_NOT_OK(WriteBuffers(fbb, buffers, &fb_buffers));
-
-  *offset = flatbuf::CreateLargeRecordBatch(fbb, length, fb_nodes, fb_buffers);
-  return Status::OK();
-}
-
 Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out) {
@@ -644,17 +611,6 @@ Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
       fbb, flatbuf::MessageHeader_RecordBatch, record_batch.Union(), body_length, out);
 }
 
-Status WriteLargeRecordBatchMessage(int64_t length, int64_t body_length,
-    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
-    std::shared_ptr<Buffer>* out) {
-  FBB fbb;
-  LargeRecordBatchOffset large_batch;
-  RETURN_NOT_OK(
-      MakeLargeRecordBatch(fbb, length, body_length, nodes, buffers, &large_batch));
-  return WriteMessage(fbb, flatbuf::MessageHeader_LargeRecordBatch, large_batch.Union(),
-      body_length, out);
-}
-
 Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out) {
@@ -917,7 +873,7 @@ class RecordBatchMetadata::RecordBatchMetadataImpl : public MessageHolder {
 
   const flatbuf::Buffer* buffer(int i) const { return buffers_->Get(i); }
 
-  int32_t length() const { return batch_->length(); }
+  int64_t length() const { return batch_->length(); }
 
   int num_buffers() const { return batch_->buffers()->size(); }
 
@@ -969,7 +925,7 @@ BufferMetadata RecordBatchMetadata::buffer(int i) const {
   return result;
 }
 
-int32_t RecordBatchMetadata::length() const {
+int64_t RecordBatchMetadata::length() const {
   return impl_->length();
 }
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 41e6c5e9f19ea..dc07c7a1bd9b7 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -150,7 +150,7 @@ class ARROW_EXPORT RecordBatchMetadata {
   FieldMetadata field(int i) const;
   BufferMetadata buffer(int i) const;
 
-  int32_t length() const;
+  int64_t length() const;
   int num_buffers() const;
   int num_fields() const;
 
@@ -229,10 +229,6 @@ Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out);
 
-Status WriteLargeRecordBatchMessage(int64_t length, int64_t body_length,
-    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
-    std::shared_ptr<Buffer>* out);
-
 Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out);
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index a2b20a901a69e..71ba951111999 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -468,48 +468,7 @@ Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
   return impl_->GetRecordBatch(i, batch);
 }
 
-// ----------------------------------------------------------------------
-// Read LargeRecordBatch
-
-class LargeRecordBatchSource : public ArrayComponentSource {
- public:
-  LargeRecordBatchSource(
-      const flatbuf::LargeRecordBatch* metadata, io::RandomAccessFile* file)
-      : metadata_(metadata), file_(file) {}
-
-  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
-    if (buffer_index >= static_cast<int>(metadata_->buffers()->size())) {
-      return Status::Invalid("Ran out of buffer metadata, likely malformed");
-    }
-    const flatbuf::Buffer* buffer = metadata_->buffers()->Get(buffer_index);
-
-    if (buffer->length() == 0) {
-      *out = nullptr;
-      return Status::OK();
-    } else {
-      return file_->ReadAt(buffer->offset(), buffer->length(), out);
-    }
-  }
-
-  Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
-    // pop off a field
-    if (field_index >= static_cast<int>(metadata_->nodes()->size())) {
-      return Status::Invalid("Ran out of field metadata, likely malformed");
-    }
-    const flatbuf::LargeFieldNode* node = metadata_->nodes()->Get(field_index);
-
-    metadata->length = node->length();
-    metadata->null_count = node->null_count();
-    metadata->offset = 0;
-    return Status::OK();
-  }
-
- private:
-  const flatbuf::LargeRecordBatch* metadata_;
-  io::RandomAccessFile* file_;
-};
-
-Status ReadLargeRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->Seek(offset));
@@ -517,19 +476,19 @@ Status ReadLargeRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offse
   RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
   int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
 
+  std::shared_ptr<Message> message;
   RETURN_NOT_OK(file->Read(flatbuffer_size, &buffer));
-  auto message = flatbuf::GetMessage(buffer->data());
-  auto batch = reinterpret_cast<const flatbuf::LargeRecordBatch*>(message->header());
+  RETURN_NOT_OK(Message::Open(buffer, 0, &message));
+
+  RecordBatchMetadata metadata(message);
 
   // TODO(ARROW-388): The buffer offsets start at 0, so we must construct a
   // RandomAccessFile according to that frame of reference
   std::shared_ptr<Buffer> buffer_payload;
-  RETURN_NOT_OK(file->Read(message->bodyLength(), &buffer_payload));
+  RETURN_NOT_OK(file->Read(message->body_length(), &buffer_payload));
   io::BufferReader buffer_reader(buffer_payload);
 
-  LargeRecordBatchSource source(batch, &buffer_reader);
-  return LoadRecordBatchFromSource(
-      schema, batch->length(), kMaxNestingDepth, &source, out);
+  return ReadRecordBatch(metadata, schema, kMaxNestingDepth, &buffer_reader, out);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 1c1314a040bef..1e8636c1efcce 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -120,12 +120,9 @@ class ARROW_EXPORT FileReader {
   std::unique_ptr<FileReaderImpl> impl_;
 };
 
-// ----------------------------------------------------------------------
-//
 
-/// EXPERIMENTAL: Read length-prefixed LargeRecordBatch metadata (64-bit array
-/// lengths) at offset and reconstruct RecordBatch
-Status ARROW_EXPORT ReadLargeRecordBatch(const std::shared_ptr<Schema>& schema,
+/// Read encapsulated message and RecordBatch
+Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<Schema>& schema,
     int64_t offset, io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index ef59471e3c7c9..0f55f8e33e71d 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -48,28 +48,25 @@ namespace ipc {
 class RecordBatchWriter : public ArrayVisitor {
  public:
   RecordBatchWriter(
-      MemoryPool* pool, int64_t buffer_start_offset, int max_recursion_depth)
+      MemoryPool* pool, int64_t buffer_start_offset, int max_recursion_depth,
+      bool allow_64bit)
       : pool_(pool),
         max_recursion_depth_(max_recursion_depth),
-        buffer_start_offset_(buffer_start_offset) {
+        buffer_start_offset_(buffer_start_offset),
+        allow_64bit_(allow_64bit) {
     DCHECK_GT(max_recursion_depth, 0);
   }
 
   virtual ~RecordBatchWriter() = default;
 
-  virtual Status CheckArrayMetadata(const Array& arr) {
-    if (arr.length() > std::numeric_limits<int32_t>::max()) {
-      return Status::Invalid("Cannot write arrays larger than 2^31 - 1 in length");
-    }
-    return Status::OK();
-  }
-
   Status VisitArray(const Array& arr) {
     if (max_recursion_depth_ <= 0) {
       return Status::Invalid("Max recursion depth reached");
     }
 
-    RETURN_NOT_OK(CheckArrayMetadata(arr));
+    if (!allow_64bit_ && arr.length() > std::numeric_limits<int32_t>::max()) {
+      return Status::Invalid("Cannot write arrays larger than 2^31 - 1 in length");
+    }
 
     // push back all common elements
     field_nodes_.emplace_back(arr.length(), arr.null_count(), 0);
@@ -470,6 +467,7 @@ class RecordBatchWriter : public ArrayVisitor {
 
   int64_t max_recursion_depth_;
   int64_t buffer_start_offset_;
+  bool allow_64bit_;
 };
 
 class DictionaryWriter : public RecordBatchWriter {
@@ -502,20 +500,21 @@ class DictionaryWriter : public RecordBatchWriter {
 
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth) {
-  RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
+    MemoryPool* pool, int max_recursion_depth, bool allow_64bit) {
+  RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth,
+      allow_64bit);
   return writer.Write(batch, dst, metadata_length, body_length);
 }
 
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
     int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
     int64_t* body_length, MemoryPool* pool) {
-  DictionaryWriter writer(pool, buffer_start_offset, kMaxNestingDepth);
+  DictionaryWriter writer(pool, buffer_start_offset, kMaxNestingDepth, false);
   return writer.Write(dictionary_id, dictionary, dst, metadata_length, body_length);
 }
 
 Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
-  RecordBatchWriter writer(default_memory_pool(), 0, kMaxNestingDepth);
+  RecordBatchWriter writer(default_memory_pool(), 0, kMaxNestingDepth, true);
   RETURN_NOT_OK(writer.GetTotalSize(batch, size));
   return Status::OK();
 }
@@ -733,30 +732,11 @@ Status FileWriter::Close() {
   return impl_->Close();
 }
 
-// ----------------------------------------------------------------------
-// Write record batches with 64-bit size metadata
-
-class LargeRecordBatchWriter : public RecordBatchWriter {
- public:
-  using RecordBatchWriter::RecordBatchWriter;
-
-  Status CheckArrayMetadata(const Array& arr) override {
-    // No < INT32_MAX length check
-    return Status::OK();
-  }
-
-  Status WriteMetadataMessage(
-      int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
-    return WriteLargeRecordBatchMessage(
-        num_rows, body_length, field_nodes_, buffer_meta_, out);
-  }
-};
-
 Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth) {
-  LargeRecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
-  return writer.Write(batch, dst, metadata_length, body_length);
+    MemoryPool* pool) {
+  return WriteRecordBatch(batch, buffer_start_offset, dst, metadata_length, body_length,
+      pool, kMaxNestingDepth, true);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 1271652a35c78..3b7e710c124cb 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -45,29 +45,30 @@ class OutputStream;
 
 namespace ipc {
 
-// Write the RecordBatch (collection of equal-length Arrow arrays) to the
-// output stream in a contiguous block. The record batch metadata is written as
-// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
-// prefixed by its size, followed by each of the memory buffers in the batch
-// written end to end (with appropriate alignment and padding):
-//
-// <int32: metadata size> <uint8*: metadata> <buffers>
-//
-// Finally, the absolute offsets (relative to the start of the output stream)
-// to the end of the body and end of the metadata / data header (suffixed by
-// the header size) is returned in out-variables
-//
-// @param(in) buffer_start_offset: the start offset to use in the buffer metadata,
-// default should be 0
-//
-// @param(out) metadata_length: the size of the length-prefixed flatbuffer
-// including padding to a 64-byte boundary
-//
-// @param(out) body_length: the size of the contiguous buffer block plus
-// padding bytes
+/// Write the RecordBatch (collection of equal-length Arrow arrays) to the
+/// output stream in a contiguous block. The record batch metadata is written as
+/// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
+/// prefixed by its size, followed by each of the memory buffers in the batch
+/// written end to end (with appropriate alignment and padding):
+///
+/// <int32: metadata size> <uint8*: metadata> <buffers>
+///
+/// Finally, the absolute offsets (relative to the start of the output stream)
+/// to the end of the body and end of the metadata / data header (suffixed by
+/// the header size) is returned in out-variables
+///
+/// @param(in) buffer_start_offset the start offset to use in the buffer metadata,
+/// default should be 0
+/// @param(in) allow_64bit permit field lengths exceeding INT32_MAX. May not be
+/// readable by other Arrow implementations
+/// @param(out) metadata_length: the size of the length-prefixed flatbuffer
+/// including padding to a 64-byte boundary
+/// @param(out) body_length: the size of the contiguous buffer block plus
+/// padding bytes
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
+    MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth,
+    bool allow_64bit = false);
 
 // Write Array as a DictionaryBatch message
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
@@ -116,13 +117,11 @@ class ARROW_EXPORT FileWriter : public StreamWriter {
   std::unique_ptr<FileWriterImpl> impl_;
 };
 
-// ----------------------------------------------------------------------
-
-/// EXPERIMENTAL: Write record batch using LargeRecordBatch IPC metadata. This
-/// data may not be readable by all Arrow implementations
+/// EXPERIMENTAL: Write RecordBatch allowing lengths over INT32_MAX. This data
+/// may not be readable by all Arrow implementations
 Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
+    MemoryPool* pool);
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/format/Message.fbs b/format/Message.fbs
index e56366d436eb9..ff30aceeda4f3 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -290,12 +290,12 @@ struct Buffer {
 struct FieldNode {
   /// The number of value slots in the Arrow array at this level of a nested
   /// tree
-  length: int;
+  length: long;
 
   /// The number of observed nulls. Fields with null_count == 0 may choose not
   /// to write their physical validity bitmap out as a materialized buffer,
   /// instead setting the length of the bitmap buffer to 0.
-  null_count: int;
+  null_count: long;
 }
 
 /// A data header describing the shared memory layout of a "record" or "row"
@@ -304,7 +304,7 @@ struct FieldNode {
 table RecordBatch {
   /// number of records / rows. The arrays in the batch should all have this
   /// length
-  length: int;
+  length: long;
 
   /// Nodes correspond to the pre-ordered flattened logical schema
   nodes: [FieldNode];
@@ -318,22 +318,6 @@ table RecordBatch {
   buffers: [Buffer];
 }
 
-/// ----------------------------------------------------------------------
-/// EXPERIMENTAL: A RecordBatch type that supports data with more than 2^31 - 1
-/// elements. Arrow implementations do not need to implement this type to be
-/// compliant
-
-struct LargeFieldNode {
-  length: long;
-  null_count: long;
-}
-
-table LargeRecordBatch {
-  length: long;
-  nodes: [LargeFieldNode];
-  buffers: [Buffer];
-}
-
 /// ----------------------------------------------------------------------
 /// For sending dictionary encoding information. Any Field can be
 /// dictionary-encoded, but in this case none of its children may be
@@ -356,7 +340,7 @@ table DictionaryBatch {
 /// which may include experimental metadata types. For maximum compatibility,
 /// it is best to send data using RecordBatch
 union MessageHeader {
-  Schema, DictionaryBatch, RecordBatch, LargeRecordBatch
+  Schema, DictionaryBatch, RecordBatch
 }
 
 table Message {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
index 71dd0abc6bcef..72ce982f2e7ee 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
@@ -34,7 +34,7 @@ public ArrowFieldNode(int length, int nullCount) {
 
   @Override
   public int writeTo(FlatBufferBuilder builder) {
-    return FieldNode.createFieldNode(builder, length, nullCount);
+    return FieldNode.createFieldNode(builder, (long)length, (long)nullCount);
   }
 
   public int getNullCount() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 92a6c0c26ba6e..f85fb51710bde 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -207,7 +207,7 @@ private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB
     List<ArrowFieldNode> nodes = new ArrayList<>();
     for (int i = 0; i < nodesLength; ++i) {
       FieldNode node = recordBatchFB.nodes(i);
-      nodes.add(new ArrowFieldNode(node.length(), node.nullCount()));
+      nodes.add(new ArrowFieldNode((int)node.length(), (int)node.nullCount()));
     }
     List<ArrowBuf> buffers = new ArrayList<>();
     for (int i = 0; i < recordBatchFB.buffersLength(); ++i) {
@@ -216,7 +216,7 @@ private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB
       buffers.add(vectorBuffer);
     }
     ArrowRecordBatch arrowRecordBatch =
-        new ArrowRecordBatch(recordBatchFB.length(), nodes, buffers);
+        new ArrowRecordBatch((int)recordBatchFB.length(), nodes, buffers);
     body.release();
     return arrowRecordBatch;
   }

From 2406d4eed9af41b1ef60c53834aced036a933327 Mon Sep 17 00:00:00 2001
From: Miki Tebeka <miki.tebeka@gmail.com>
Date: Wed, 22 Mar 2017 14:06:42 -0400
Subject: [PATCH 0402/1644] ARROW-552: [Python] Implement getitem for
 DictionaryArray by returning a value from the dictionary

Author: Miki Tebeka <miki.tebeka@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #414 from wesm/ARROW-552 and squashes the following commits:

8a039b5 [Wes McKinney] Implement DictionaryArray.getitem by indexing into the dictionary. Add indices and dictionary properties
e700b45 [Miki Tebeka] ARROW-552: [Python] Add scalar value support for Dictionary type (WIP)
---
 python/pyarrow/array.pxd             |  4 +++-
 python/pyarrow/array.pyx             | 25 +++++++++++++++++++++++++
 python/pyarrow/scalar.pyx            |  2 +-
 python/pyarrow/tests/test_scalars.py | 13 +++++++++++++
 4 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index 56bb53d5c97dc..c3e7997aa823c 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -109,7 +109,9 @@ cdef class BinaryArray(Array):
 
 
 cdef class DictionaryArray(Array):
-    pass
+    cdef:
+        object _indices, _dictionary
+
 
 
 cdef wrap_array_output(PyObject* output)
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 6afeaa0a7332b..795076cfccb7e 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -406,6 +406,31 @@ cdef class BinaryArray(Array):
 
 cdef class DictionaryArray(Array):
 
+    cdef getitem(self, int64_t i):
+        cdef Array dictionary = self.dictionary
+        cdef int64_t index = self.indices[i].as_py()
+        return scalar.box_scalar(dictionary.type, dictionary.sp_array, index)
+
+    property dictionary:
+
+        def __get__(self):
+            cdef CDictionaryArray* darr = <CDictionaryArray*>(self.ap)
+
+            if self._dictionary is None:
+                self._dictionary = box_array(darr.dictionary())
+
+            return self._dictionary
+
+    property indices:
+
+        def __get__(self):
+            cdef CDictionaryArray* darr = <CDictionaryArray*>(self.ap)
+
+            if self._indices is None:
+                self._indices = box_array(darr.indices())
+
+            return self._indices
+
     @staticmethod
     def from_arrays(indices, dictionary, mask=None,
                     MemoryPool memory_pool=None):
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 8c88f90422fac..1b7e67b356a2f 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -241,7 +241,7 @@ cdef dict _scalar_classes = {
     Type_DOUBLE: DoubleValue,
     Type_LIST: ListValue,
     Type_BINARY: BinaryValue,
-    Type_STRING: StringValue,
+    Type_STRING: StringValue
 }
 
 cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index ef600a06296cb..d56481c06d0f8 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -16,6 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import pandas as pd
+
 from pyarrow.compat import unittest, u, unicode_type
 import pyarrow as A
 
@@ -100,3 +102,14 @@ def test_list(self):
 
         v = arr[3]
         assert len(v) == 0
+
+    def test_dictionary(self):
+        colors = ['red', 'green', 'blue']
+        values = pd.Series(colors * 4)
+
+        categorical = pd.Categorical(values, categories=colors)
+
+        v = A.DictionaryArray.from_arrays(categorical.codes,
+                                          categorical.categories)
+        for i, c in enumerate(values):
+            assert v[i].as_py() == c

From bf2acf6cb22b8d2bf6d0fb98a6117e78e92b81fe Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 22 Mar 2017 23:07:05 -0400
Subject: [PATCH 0403/1644] ARROW-454: pojo.Field doesn't implement hashCode()

Author: Julien Le Dem <julien@dremio.com>

Closes #423 from julienledem/field_hashcode and squashes the following commits:

192a689 [Julien Le Dem] ARROW-454: pojo.Field doesn't implement hashCode()
---
 .../src/main/codegen/templates/ArrowType.java |  2 +-
 .../apache/arrow/vector/types/pojo/Field.java | 29 ++++++++++---------
 .../arrow/vector/types/pojo/TestSchema.java   | 22 ++++++++++++++
 3 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index 85ea3898e09c6..91cbe98196b81 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -164,7 +164,7 @@ public String toString() {
 
     @Override
     public int hashCode() {
-      return Objects.hash(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+      return java.util.Arrays.deepHashCode(new Object[] {<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>});
     }
 
     @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index bbbd559f10a3d..c310b9082f78f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -24,14 +24,6 @@
 import java.util.List;
 import java.util.Objects;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
-import com.google.flatbuffers.FlatBufferBuilder;
-
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.schema.TypeLayout;
@@ -40,6 +32,14 @@
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+
 public class Field {
   private final String name;
   private final boolean nullable;
@@ -176,6 +176,11 @@ public TypeLayout getTypeLayout() {
     return typeLayout;
   }
 
+  @Override
+  public int hashCode() {
+    return Objects.hash(name, nullable, type, dictionary, children);
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (!(obj instanceof Field)) {
@@ -183,12 +188,10 @@ public boolean equals(Object obj) {
     }
     Field that = (Field) obj;
     return Objects.equals(this.name, that.name) &&
-            Objects.equals(this.nullable, that.nullable) &&
-            Objects.equals(this.type, that.type) &&
+           Objects.equals(this.nullable, that.nullable) &&
+           Objects.equals(this.type, that.type) &&
            Objects.equals(this.dictionary, that.dictionary) &&
-            (Objects.equals(this.children, that.children) ||
-                    (this.children == null || this.children.size() == 0) &&
-                    (that.children == null || that.children.size() == 0));
+           Objects.equals(this.children, that.children);
   }
 
   @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 5b74c54c9159f..a7d1cce917747 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -22,6 +22,7 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.IntervalUnit;
@@ -125,6 +126,27 @@ private void roundTrip(Schema schema) throws IOException {
     Schema actual = Schema.fromJSON(json);
     assertEquals(schema.toJson(), actual.toJson());
     assertEquals(schema, actual);
+    validateFieldsHashcode(schema.getFields(), actual.getFields());
+    assertEquals(schema.hashCode(), actual.hashCode());
+  }
+
+  private void validateFieldsHashcode(List<Field> schemaFields, List<Field> actualFields) {
+    assertEquals(schemaFields.size(), actualFields.size());
+    if (schemaFields.size() == 0) {
+      return;
+    }
+    for (int i = 0; i < schemaFields.size(); i++) {
+      Field schemaField = schemaFields.get(i);
+      Field actualField = actualFields.get(i);
+      validateFieldsHashcode(schemaField.getChildren(), actualField.getChildren());
+      validateHashCode(schemaField.getType(), actualField.getType());
+      validateHashCode(schemaField, actualField);
+    }
+  }
+
+  private void validateHashCode(Object o1, Object o2) {
+    assertEquals(o1, o2);
+    assertEquals(o1 + " == " + o2, o1.hashCode(), o2.hashCode());
   }
 
   private void contains(Schema schema, String... s) throws IOException {

From 990e2bde758ac8bc6e4497ae1bc37f89b71bb5cf Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Wed, 22 Mar 2017 23:08:01 -0400
Subject: [PATCH 0404/1644] ARROW-691: [Java] Encode dictionary type in message
 format

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #422 from elahrvivaz/ARROW-691 and squashes the following commits:

c1adad1 [Emilio Lahr-Vivaz] ARROW-691 Encode dictionary type in message format
---
 .../vector/types/pojo/DictionaryEncoding.java  | 18 ++++++++++++++++++
 .../apache/arrow/vector/types/pojo/Field.java  |  3 ++-
 .../vector/stream/MessageSerializerTest.java   | 15 +++++++++++++++
 3 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
index 6d35cdef832f9..32568d34ba495 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
@@ -18,6 +18,8 @@
  ******************************************************************************/
 package org.apache.arrow.vector.types.pojo;
 
+import java.util.Objects;
+
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 
 public class DictionaryEncoding {
@@ -48,4 +50,20 @@ public Int getIndexType() {
   public String toString() {
     return "DictionaryEncoding[id=" + id + ",ordered=" + ordered + ",indexType=" + indexType + "]";
   }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    } else if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    DictionaryEncoding that = (DictionaryEncoding) o;
+    return id == that.id && ordered == that.ordered && Objects.equals(indexType, that.indexType);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(id, ordered, indexType);
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index c310b9082f78f..011f0e6e446a8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -121,10 +121,11 @@ public int getField(FlatBufferBuilder builder) {
     int typeOffset = type.getType(builder);
     int dictionaryOffset = -1;
     if (dictionary != null) {
-      // TODO encode dictionary type - currently type is only signed 32 bit int (default null)
+      int dictionaryType = dictionary.getIndexType().getType(builder);
       org.apache.arrow.flatbuf.DictionaryEncoding.startDictionaryEncoding(builder);
       org.apache.arrow.flatbuf.DictionaryEncoding.addId(builder, dictionary.getId());
       org.apache.arrow.flatbuf.DictionaryEncoding.addIsOrdered(builder, dictionary.isOrdered());
+      org.apache.arrow.flatbuf.DictionaryEncoding.addIndexType(builder, dictionaryType);
       dictionaryOffset = org.apache.arrow.flatbuf.DictionaryEncoding.endDictionaryEncoding(builder);
     }
     int[] childrenData = new int[children.size()];
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
index bb2ccf8cbb5f6..d3d49d5fb8096 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
@@ -37,6 +37,7 @@
 import org.apache.arrow.vector.schema.ArrowMessage;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Test;
@@ -72,6 +73,20 @@ public void testSchemaMessageSerialization() throws IOException {
     assertEquals(1, deserialized.getFields().size());
   }
 
+  @Test
+  public void testSchemaDictionaryMessageSerialization() throws IOException {
+    DictionaryEncoding dictionary = new DictionaryEncoding(9L, false, new ArrowType.Int(8, true));
+    Field field = new Field("test", true, ArrowType.Utf8.INSTANCE, dictionary, null);
+    Schema schema = new Schema(Collections.singletonList(field));
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    long size = MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), schema);
+    assertEquals(size, out.toByteArray().length);
+
+    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+    Schema deserialized = MessageSerializer.deserializeSchema(new ReadChannel(Channels.newChannel(in)));
+    assertEquals(schema, deserialized);
+  }
+
   @Test
   public void testSerializeRecordBatch() throws IOException {
     byte[] validity = new byte[] { (byte)255, 0};

From 2926183276e69390bd84569c364b4e8fb316db53 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 22 Mar 2017 23:09:10 -0400
Subject: [PATCH 0405/1644] ARROW-347: Add method to pass CallBack when
 creating a transfer pair

supersedes and closes #182

Author: Julien Le Dem <julien@dremio.com>

Closes #425 from julienledem/arrow_347 and squashes the following commits:

3c47b82 [Julien Le Dem] ARROW-347: Add method to pass CallBack when creating a transfer pair
---
 .../main/codegen/templates/UnionVector.java   | 13 ++-
 .../arrow/vector/BaseDataValueVector.java     |  7 ++
 .../org/apache/arrow/vector/ValueVector.java  |  3 +
 .../org/apache/arrow/vector/ZeroVector.java   |  6 ++
 .../complex/BaseRepeatedValueVector.java      | 14 ++-
 .../arrow/vector/complex/ListVector.java      | 34 +++++---
 .../arrow/vector/complex/MapVector.java       |  5 ++
 .../vector/complex/NullableMapVector.java     |  7 +-
 .../vector/complex/impl/PromotableWriter.java |  2 +-
 .../complex/writer/TestComplexWriter.java     | 86 +++++++++++++++++--
 10 files changed, 146 insertions(+), 31 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 076ed93999623..d17935b08eefc 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -236,12 +236,17 @@ public Field getField() {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new TransferImpl(name, allocator);
+    return getTransferPair(name, allocator);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new TransferImpl(ref, allocator);
+    return getTransferPair(ref, allocator, null);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return new org.apache.arrow.vector.complex.UnionVector.TransferImpl(ref, allocator, callBack);
   }
 
   @Override
@@ -276,8 +281,8 @@ private class TransferImpl implements TransferPair {
     private final TransferPair typeVectorTransferPair;
     private final UnionVector to;
 
-    public TransferImpl(String name, BufferAllocator allocator) {
-      to = new UnionVector(name, allocator, null);
+    public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
+      to = new UnionVector(name, allocator, callBack);
       internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap);
       typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector);
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 7fe1615da5a27..6d7d3f04a6d04 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -24,6 +24,8 @@
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 
 import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.TransferPair;
 
 
 public abstract class BaseDataValueVector extends BaseValueVector implements BufferBacked {
@@ -87,6 +89,11 @@ public void close() {
     super.close();
   }
 
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return getTransferPair(ref, allocator);
+  }
+
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
     ArrowBuf[] out;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index ff7b94c34d80d..8e35398b9394b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -24,6 +24,7 @@
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.TransferPair;
 
 import io.netty.buffer.ArrowBuf;
@@ -106,6 +107,8 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   TransferPair getTransferPair(String ref, BufferAllocator allocator);
 
+  TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack);
+
   /**
    * Returns a new {@link org.apache.arrow.vector.util.TransferPair transfer pair} that is used to transfer underlying
    * buffers into the target vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index e163b4fa9398f..73f858e4d35a0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -29,6 +29,7 @@
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.TransferPair;
 
 import io.netty.buffer.ArrowBuf;
@@ -159,6 +160,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
     return defaultPair;
   }
 
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return defaultPair;
+  }
+
   @Override
   public TransferPair makeTransferPair(ValueVector target) {
     return defaultPair;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index eeb8f5830f404..eda1f3bc80a96 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -29,6 +29,7 @@
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
 import com.google.common.base.Preconditions;
@@ -44,15 +45,17 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
 
   protected final UInt4Vector offsets;
   protected FieldVector vector;
+  protected final CallBack callBack;
 
-  protected BaseRepeatedValueVector(String name, BufferAllocator allocator) {
-    this(name, allocator, DEFAULT_DATA_VECTOR);
+  protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) {
+    this(name, allocator, DEFAULT_DATA_VECTOR, callBack);
   }
 
-  protected BaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector) {
+  protected BaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) {
     super(name, allocator);
     this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
     this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
+    this.callBack = callBack;
   }
 
   @Override
@@ -154,9 +157,12 @@ public int size() {
   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType, DictionaryEncoding dictionary) {
     boolean created = false;
     if (vector instanceof ZeroVector) {
-      vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, dictionary, null);
+      vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, dictionary, callBack);
       // returned vector must have the same field
       created = true;
+      if (callBack != null) {
+        callBack.doWork();
+      }
     }
 
     if (vector.getField().getType().getTypeID() != minorType.getType().getTypeID()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index a12440e39e8fe..54b051b9781e5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -24,10 +24,6 @@
 import java.util.Collections;
 import java.util.List;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
@@ -52,6 +48,11 @@
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.TransferPair;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+
 public class ListVector extends BaseRepeatedValueVector implements FieldVector {
 
   final UInt4Vector offsets;
@@ -59,17 +60,15 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector {
   private final List<BufferBacked> innerVectors;
   private Mutator mutator = new Mutator();
   private Accessor accessor = new Accessor();
-  private UnionListWriter writer;
   private UnionListReader reader;
   private CallBack callBack;
   private final DictionaryEncoding dictionary;
 
   public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
-    super(name, allocator);
+    super(name, allocator, callBack);
     this.bits = new BitVector("$bits$", allocator);
     this.offsets = getOffsetVector();
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
-    this.writer = new UnionListWriter(this);
     this.reader = new UnionListReader(this);
     this.dictionary = dictionary;
     this.callBack = callBack;
@@ -86,6 +85,8 @@ public void initializeChildrenFromFields(List<Field> children) {
     if (!addOrGetVector.isCreated()) {
       throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
     }
+
+    addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren());
   }
 
   @Override
@@ -111,7 +112,7 @@ public List<BufferBacked> getFieldInnerVectors() {
   }
 
   public UnionListWriter getWriter() {
-    return writer;
+    return new UnionListWriter(this);
   }
 
   @Override
@@ -139,7 +140,12 @@ public FieldVector getDataVector() {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new TransferImpl(ref, allocator);
+    return getTransferPair(ref, allocator, null);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return new TransferImpl(ref, allocator, callBack);
   }
 
   @Override
@@ -152,8 +158,8 @@ private class TransferImpl implements TransferPair {
     ListVector to;
     TransferPair pairs[] = new TransferPair[3];
 
-    public TransferImpl(String name, BufferAllocator allocator) {
-      this(new ListVector(name, allocator, dictionary, null));
+    public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
+      this(new ListVector(name, allocator, dictionary, callBack));
     }
 
     public TransferImpl(ListVector to) {
@@ -172,6 +178,7 @@ public void transfer() {
       for (TransferPair pair : pairs) {
         pair.transfer();
       }
+      to.lastSet = lastSet;
     }
 
     @Override
@@ -282,9 +289,12 @@ public ArrowBuf[] getBuffers(boolean clear) {
   }
 
   public UnionVector promoteToUnion() {
-    UnionVector vector = new UnionVector(name, allocator, null);
+    UnionVector vector = new UnionVector(name, allocator, callBack);
     replaceDataVector(vector);
     reader = new UnionListReader(this);
+    if (callBack != null) {
+      callBack.doWork();
+    }
     return vector;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 4d750cad264db..cb67537c446c6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -115,6 +115,11 @@ public int getBufferSizeFor(final int valueCount) {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
+    return getTransferPair(name, allocator, null);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
     return new MapTransferPair(this, new MapVector(name, allocator, callBack), false);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index bb1fdf841a305..de1d1857370b0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -86,7 +86,7 @@ public FieldReader getReader() {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, dictionary, callBack), false);
+    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, dictionary, null), false);
   }
 
   @Override
@@ -96,6 +96,11 @@ public TransferPair makeTransferPair(ValueVector to) {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, null), false);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
     return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, callBack), false);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index e33319a2270b1..1880c9b490c27 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -142,7 +142,7 @@ public boolean isEmptyMap() {
   }
 
   protected FieldWriter getWriter() {
-    return getWriter(type);
+    return writer;
   }
 
   private FieldWriter promoteToUnion() {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index a8a2d512c09ec..99ba19bec80e7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -29,8 +29,10 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.SchemaChangeCallBack;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
@@ -49,7 +51,11 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.Text;
+import org.apache.arrow.vector.util.TransferPair;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.Assert;
@@ -65,7 +71,38 @@ public class TestComplexWriter {
 
   @Test
   public void simpleNestedTypes() {
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = populateMapVector(null);
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < COUNT; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
+      Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+    }
+
+    parent.close();
+  }
+
+  @Test
+  public void transferPairSchemaChange() {
+    SchemaChangeCallBack callBack1 = new SchemaChangeCallBack();
+    SchemaChangeCallBack callBack2 = new SchemaChangeCallBack();
+    MapVector parent = populateMapVector(callBack1);
+
+    TransferPair tp = parent.getTransferPair("newVector", allocator, callBack2);
+
+    ComplexWriter writer = new ComplexWriterImpl("newWriter", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("newInt");
+    intWriter.writeInt(1);
+    writer.setValueCount(1);
+
+    assertTrue(callBack1.getSchemaChangedAndReset());
+    // The second vector should not have registered a schema change
+    assertFalse(callBack1.getSchemaChangedAndReset());
+  }
+
+  private MapVector populateMapVector(CallBack callBack) {
+    MapVector parent = new MapVector("parent", allocator, callBack);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
     IntWriter intWriter = rootWriter.integer("int");
@@ -77,14 +114,7 @@ public void simpleNestedTypes() {
       rootWriter.end();
     }
     writer.setValueCount(COUNT);
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
-    for (int i = 0; i < COUNT; i++) {
-      rootReader.setPosition(i);
-      Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
-      Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
-    }
-
-    parent.close();
+    return parent;
   }
 
   @Test
@@ -646,4 +676,42 @@ public void timeStampWriters() throws Exception {
     long nanoLong = nanoReader.readLong();
     Assert.assertEquals(expectedNanos, nanoLong);
   }
+
+  @Test
+  public void complexCopierWithList() {
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    ListWriter listWriter = rootWriter.list("list");
+    MapWriter innerMapWriter = listWriter.map();
+    IntWriter outerIntWriter = listWriter.integer();
+    rootWriter.start();
+    listWriter.startList();
+    outerIntWriter.writeInt(1);
+    outerIntWriter.writeInt(2);
+    innerMapWriter.start();
+    IntWriter intWriter = innerMapWriter.integer("a");
+    intWriter.writeInt(1);
+    innerMapWriter.end();
+    innerMapWriter.start();
+    intWriter = innerMapWriter.integer("a");
+    intWriter.writeInt(2);
+    innerMapWriter.end();
+    listWriter.endList();
+    rootWriter.end();
+    writer.setValueCount(1);
+
+    NullableMapVector mapVector = (NullableMapVector) parent.getChild("root");
+    TransferPair tp = mapVector.getTransferPair(allocator);
+    tp.splitAndTransfer(0, 1);
+    MapVector toMapVector = (MapVector) tp.getTo();
+    JsonStringHashMap<?,?> toMapValue = (JsonStringHashMap<?,?>) toMapVector.getAccessor().getObject(0);
+    JsonStringArrayList<?> object = (JsonStringArrayList<?>) toMapValue.get("list");
+    assertEquals(1, object.get(0));
+    assertEquals(2, object.get(1));
+    JsonStringHashMap<?,?> innerMap = (JsonStringHashMap<?,?>) object.get(2);
+    assertEquals(1, innerMap.get("a"));
+    innerMap = (JsonStringHashMap<?,?>) object.get(3);
+    assertEquals(2, innerMap.get("a"));
+  }
 }
\ No newline at end of file

From f67974b190349c781509d2b1657331935f458f9b Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 22 Mar 2017 23:10:01 -0400
Subject: [PATCH 0406/1644] ARROW-700: Add headroom interface for allocator

Author: Julien Le Dem <julien@dremio.com>

Closes #424 from julienledem/headroom and squashes the following commits:

2aab160 [Julien Le Dem] ARROW-700: Add headroom interface for allocator
---
 .../java/org/apache/arrow/memory/Accountant.java   | 14 ++++++++++++--
 .../org/apache/arrow/memory/BufferAllocator.java   |  8 ++++++++
 .../org/apache/arrow/memory/TestAccountant.java    | 13 +++++++++++--
 3 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
index 6ddc8f784bc4a..89329b2766357 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -18,12 +18,12 @@
 
 package org.apache.arrow.memory;
 
-import com.google.common.base.Preconditions;
-
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.annotation.concurrent.ThreadSafe;
 
+import com.google.common.base.Preconditions;
+
 /**
  * Provides a concurrent way to manage account for memory usage without locking. Used as basis
  * for Allocators. All
@@ -202,6 +202,7 @@ public boolean isOverLimit() {
   /**
    * Close this Accountant. This will release any reservation bytes back to a parent Accountant.
    */
+  @Override
   public void close() {
     // return memory reservation to parent allocator.
     if (parent != null) {
@@ -248,6 +249,15 @@ public long getPeakMemoryAllocation() {
     return peakAllocation.get();
   }
 
+  public long getHeadroom(){
+    long localHeadroom = allocationLimit.get() - locallyHeldMemory.get();
+    if(parent == null){
+      return localHeadroom;
+    }
+
+    return Math.min(localHeadroom, parent.getHeadroom());
+  }
+
   /**
    * Describes the type of outcome that occurred when trying to account for allocation of memory.
    */
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
index 81ffb1bec780e..c05e9acb0aa96 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -105,6 +105,14 @@ public interface BufferAllocator extends AutoCloseable {
    */
   public long getPeakMemoryAllocation();
 
+  /**
+   * Returns the amount of memory that can probably be allocated at this moment
+   * without exceeding this or any parents allocation maximum.
+   *
+   * @return Headroom in bytes
+   */
+  public long getHeadroom();
+
   /**
    * Create an allocation reservation. A reservation is a way of building up
    * a request for a buffer whose size is not known in advance. See
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java b/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
index 86bccf5064a60..2624a4a047e7e 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
@@ -19,7 +19,6 @@
 
 import static org.junit.Assert.assertEquals;
 
-import org.apache.arrow.memory.Accountant;
 import org.apache.arrow.memory.Accountant.AllocationOutcome;
 import org.junit.Assert;
 import org.junit.Test;
@@ -36,6 +35,7 @@ public void nested() {
     final Accountant parent = new Accountant(null, 0, Long.MAX_VALUE);
     ensureAccurateReservations(parent);
     assertEquals(0, parent.getAllocatedMemory());
+    assertEquals(parent.getLimit() - parent.getAllocatedMemory(), parent.getHeadroom());
   }
 
   @Test
@@ -71,6 +71,7 @@ public void run() {
     }
 
     assertEquals(0, parent.getAllocatedMemory());
+    assertEquals(parent.getLimit() - parent.getAllocatedMemory(), parent.getHeadroom());
   }
 
   private void ensureAccurateReservations(Accountant outsideParent) {
@@ -121,6 +122,9 @@ private void ensureAccurateReservations(Accountant outsideParent) {
     // went beyond reservation, now in parent accountant
     assertEquals(3, parent.getAllocatedMemory());
 
+    assertEquals(7, child.getHeadroom());
+    assertEquals(7, parent.getHeadroom());
+
     {
       AllocationOutcome first = child.allocateBytes(7);
       assertEquals(AllocationOutcome.SUCCESS, first);
@@ -135,9 +139,11 @@ private void ensureAccurateReservations(Accountant outsideParent) {
     child.releaseBytes(9);
 
     assertEquals(1, child.getAllocatedMemory());
+    assertEquals(8, child.getHeadroom());
 
     // back to reservation size
     assertEquals(2, parent.getAllocatedMemory());
+    assertEquals(8, parent.getHeadroom());
 
     AllocationOutcome first = child.allocateBytes(10);
     assertEquals(AllocationOutcome.FAILED_PARENT, first);
@@ -152,11 +158,14 @@ private void ensureAccurateReservations(Accountant outsideParent) {
     // at new limit
     assertEquals(child.getAllocatedMemory(), 11);
     assertEquals(parent.getAllocatedMemory(), 11);
-
+    assertEquals(-1, child.getHeadroom());
+    assertEquals(-1, parent.getHeadroom());
 
     child.releaseBytes(11);
     assertEquals(child.getAllocatedMemory(), 0);
     assertEquals(parent.getAllocatedMemory(), 2);
+    assertEquals(8, child.getHeadroom());
+    assertEquals(8, parent.getHeadroom());
 
     child.close();
     parent.close();

From e8f6a492d30d32cd67fe3a537b3aec4cbae566c9 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Wed, 22 Mar 2017 20:15:55 -0700
Subject: [PATCH 0407/1644] ARROW-674: [Java] Support additional Timestamp
 timezone metadata

Author: Julien Le Dem <julien@dremio.com>

Closes #408 from julienledem/timestamp_md and squashes the following commits:

e394526 [Julien Le Dem] ARROW-674: [Java] Support additional Timestamp timezone metadata
---
 .../src/main/codegen/data/ArrowTypes.tdd      |  2 +-
 .../org/apache/arrow/vector/types/Types.java  | 16 ++--
 .../apache/arrow/vector/pojo/TestConvert.java |  2 +-
 .../arrow/vector/types/pojo/TestSchema.java   | 90 ++++++++++++-------
 4 files changed, 66 insertions(+), 44 deletions(-)

diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 8f997524fccfc..94fe31e8dc0d8 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -62,7 +62,7 @@
     },
     {
       name: "Timestamp",
-      fields: [{name: "unit", type: short, valueType: TimeUnit}]
+      fields: [{name: "unit", type: short, valueType: TimeUnit}, {name: "timezone", type: String}]
     },
     {
       name: "Interval",
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 7cbf3c5bb5e36..81743b51917a1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -109,10 +109,10 @@ public class Types {
   private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
   private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
   private static final Field TIME_FIELD = new Field("", true, new Time(TimeUnit.MILLISECOND, 32), null);
-  private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND), null);
-  private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null);
-  private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND), null);
-  private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND), null);
+  private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND, "UTC"), null);
+  private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null);
+  private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND, "UTC"), null);
+  private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND, "UTC"), null);
   private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
   private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
   private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);
@@ -252,7 +252,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
-    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND)) {
+    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, "UTC")) {
       @Override
       public Field getField() {
         return TIMESTAMPSEC_FIELD;
@@ -269,7 +269,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
-    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) {
+    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, "UTC")) {
       @Override
       public Field getField() {
         return TIMESTAMPMILLI_FIELD;
@@ -286,7 +286,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
-    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND)) {
+    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC")) {
       @Override
       public Field getField() {
         return TIMESTAMPMICRO_FIELD;
@@ -303,7 +303,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
-    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND)) {
+    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC")) {
       @Override
       public Field getField() {
         return TIMESTAMPNANO_FIELD;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 65823e2a821a1..824c62aa5fbf3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -81,7 +81,7 @@ public void nestedSchema() {
         new Field("child4.1", true, Utf8.INSTANCE, null)
         )));
     childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
-        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null),
+        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null),
         new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));
     Schema initialSchema = new Schema(childrenBuilder.build());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index a7d1cce917747..9f1b2e014b860 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -28,6 +28,20 @@
 import org.apache.arrow.vector.types.IntervalUnit;
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.UnionMode;
+import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
+import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
+import org.apache.arrow.vector.types.pojo.ArrowType.Date;
+import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
+import org.apache.arrow.vector.types.pojo.ArrowType.List;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.ArrowType.Time;
+import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.junit.Test;
 
 public class TestSchema {
@@ -43,38 +57,40 @@ private static Field field(String name, ArrowType type, Field... children) {
   @Test
   public void testComplex() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", false, new ArrowType.Int(8, true)),
-        field("b", new ArrowType.Struct(),
-            field("c", new ArrowType.Int(16, true)),
-            field("d", new ArrowType.Utf8())),
-        field("e", new ArrowType.List(), field(null, new ArrowType.Date())),
-        field("f", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
-        field("g", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
-        field("h", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        field("a", false, new Int(8, true)),
+        field("b", new Struct(),
+            field("c", new Int(16, true)),
+            field("d", new Utf8())),
+        field("e", new List(), field(null, new Date())),
+        field("f", new FloatingPoint(FloatingPointPrecision.SINGLE)),
+        field("g", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
+        field("h", new Timestamp(TimeUnit.MICROSECOND, null)),
+        field("i", new Interval(IntervalUnit.DAY_TIME))
         ));
     roundTrip(schema);
     assertEquals(
-        "Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>, e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND), h: Interval(DAY_TIME)>",
+        "Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>, e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND, UTC), h: Timestamp(MICROSECOND, null), i: Interval(DAY_TIME)>",
         schema.toString());
   }
 
   @Test
   public void testAll() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", false, new ArrowType.Null()),
-        field("b", new ArrowType.Struct(), field("ba", new ArrowType.Null())),
-        field("c", new ArrowType.List(), field("ca", new ArrowType.Null())),
-        field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new ArrowType.Null())),
-        field("e", new ArrowType.Int(8, true)),
-        field("f", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
-        field("g", new ArrowType.Utf8()),
-        field("h", new ArrowType.Binary()),
-        field("i", new ArrowType.Bool()),
-        field("j", new ArrowType.Decimal(5, 5)),
-        field("k", new ArrowType.Date()),
-        field("l", new ArrowType.Time(TimeUnit.MILLISECOND, 32)),
-        field("m", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
-        field("n", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        field("a", false, new Null()),
+        field("b", new Struct(), field("ba", new Null())),
+        field("c", new List(), field("ca", new Null())),
+        field("d", new Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new Null())),
+        field("e", new Int(8, true)),
+        field("f", new FloatingPoint(FloatingPointPrecision.SINGLE)),
+        field("g", new Utf8()),
+        field("h", new Binary()),
+        field("i", new Bool()),
+        field("j", new Decimal(5, 5)),
+        field("k", new Date()),
+        field("l", new Time(TimeUnit.MILLISECOND, 32)),
+        field("m", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
+        field("n", new Timestamp(TimeUnit.MICROSECOND, null)),
+        field("o", new Interval(IntervalUnit.DAY_TIME))
         ));
     roundTrip(schema);
   }
@@ -82,7 +98,7 @@ public void testAll() throws IOException {
   @Test
   public void testUnion() throws IOException {
     Schema schema = new Schema(asList(
-        field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new ArrowType.Null()))
+        field("d", new Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new Null()))
         ));
     roundTrip(schema);
     contains(schema, "Sparse");
@@ -91,20 +107,26 @@ public void testUnion() throws IOException {
   @Test
   public void testTS() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", new ArrowType.Timestamp(TimeUnit.SECOND)),
-        field("b", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
-        field("c", new ArrowType.Timestamp(TimeUnit.MICROSECOND)),
-        field("d", new ArrowType.Timestamp(TimeUnit.NANOSECOND))
+        field("a", new Timestamp(TimeUnit.SECOND, "UTC")),
+        field("b", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
+        field("c", new Timestamp(TimeUnit.MICROSECOND, "UTC")),
+        field("d", new Timestamp(TimeUnit.NANOSECOND, "UTC")),
+        field("e", new Timestamp(TimeUnit.SECOND, null)),
+        field("f", new Timestamp(TimeUnit.MILLISECOND, null)),
+        field("g", new Timestamp(TimeUnit.MICROSECOND, null)),
+        field("h", new Timestamp(TimeUnit.NANOSECOND, null))
         ));
     roundTrip(schema);
-    contains(schema, "SECOND", "MILLISECOND", "MICROSECOND", "NANOSECOND");
+    assertEquals(
+        "Schema<a: Timestamp(SECOND, UTC), b: Timestamp(MILLISECOND, UTC), c: Timestamp(MICROSECOND, UTC), d: Timestamp(NANOSECOND, UTC), e: Timestamp(SECOND, null), f: Timestamp(MILLISECOND, null), g: Timestamp(MICROSECOND, null), h: Timestamp(NANOSECOND, null)>",
+        schema.toString());
   }
 
   @Test
   public void testInterval() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", new ArrowType.Interval(IntervalUnit.YEAR_MONTH)),
-        field("b", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        field("a", new Interval(IntervalUnit.YEAR_MONTH)),
+        field("b", new Interval(IntervalUnit.DAY_TIME))
         ));
     roundTrip(schema);
     contains(schema, "YEAR_MONTH", "DAY_TIME");
@@ -113,9 +135,9 @@ public void testInterval() throws IOException {
   @Test
   public void testFP() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)),
-        field("b", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
-        field("c", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))
+        field("a", new FloatingPoint(FloatingPointPrecision.HALF)),
+        field("b", new FloatingPoint(FloatingPointPrecision.SINGLE)),
+        field("c", new FloatingPoint(FloatingPointPrecision.DOUBLE))
         ));
     roundTrip(schema);
     contains(schema, "HALF", "SINGLE", "DOUBLE");

From 7594492d5105e86d3388c8bac94dab8dbfa5226a Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Thu, 23 Mar 2017 17:18:35 +0100
Subject: [PATCH 0408/1644] ARROW-704: Fix bad import caused by conflicting
 changes

Author: Julien Le Dem <julien@dremio.com>

Closes #430 from julienledem/ARROW-704 and squashes the following commits:

2e42330 [Julien Le Dem] ARROW-704: Fix bad import caused by conflicting changes
---
 .../java/org/apache/arrow/vector/types/pojo/TestSchema.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 9f1b2e014b860..57af9528c5933 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -22,7 +22,6 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
-import java.util.List;
 
 import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.IntervalUnit;
@@ -152,7 +151,7 @@ private void roundTrip(Schema schema) throws IOException {
     assertEquals(schema.hashCode(), actual.hashCode());
   }
 
-  private void validateFieldsHashcode(List<Field> schemaFields, List<Field> actualFields) {
+  private void validateFieldsHashcode(java.util.List<Field> schemaFields, java.util.List<Field> actualFields) {
     assertEquals(schemaFields.size(), actualFields.size());
     if (schemaFields.size() == 0) {
       return;

From 2a568f093670daba7b5dab8c096669bcfdd09a5f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 23 Mar 2017 12:30:44 -0400
Subject: [PATCH 0409/1644] ARROW-662: [Format] Move Schema flatbuffers into
 their own file that can be included

@julienledem for some reason the Java build is failing for me locally (also on master):

```
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.2:testCompile (default-testCompile) on project arrow-vector: Compilation failure: Compilation failure:
[ERROR] /home/wesm/code/arrow/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java:[38] error: a type with the same simple name is already defined by the single-type-import of List
[ERROR] /home/wesm/code/arrow/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java:[64,19] error: List is abstract; cannot be instantiated
[ERROR] /home/wesm/code/arrow/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java:[81,19] error: List is abstract; cannot be instantiated
[ERROR] -> [Help 1]
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #429 from wesm/ARROW-662 and squashes the following commits:

b588f81 [Wes McKinney] Move Schema flatbuffers into their own file that can be included
---
 cpp/src/arrow/ipc/CMakeLists.txt |   1 +
 format/File.fbs                  |   2 +-
 format/Message.fbs               | 264 +----------------------------
 format/Schema.fbs                | 280 +++++++++++++++++++++++++++++++
 java/format/pom.xml              |  20 +--
 5 files changed, 295 insertions(+), 272 deletions(-)
 create mode 100644 format/Schema.fbs

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 3a98a380e7019..629cc5bbed055 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -113,6 +113,7 @@ set(FBS_OUTPUT_FILES
 set(FBS_SRC
   ${CMAKE_SOURCE_DIR}/../format/Message.fbs
   ${CMAKE_SOURCE_DIR}/../format/File.fbs
+  ${CMAKE_SOURCE_DIR}/../format/Schema.fbs
   ${CMAKE_CURRENT_SOURCE_DIR}/feather.fbs)
 
 foreach(FIL ${FBS_SRC})
diff --git a/format/File.fbs b/format/File.fbs
index e8d6da4f848ff..3a27ca67caf5f 100644
--- a/format/File.fbs
+++ b/format/File.fbs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-include "Message.fbs";
+include "Schema.fbs";
 
 namespace org.apache.arrow.flatbuf;
 
diff --git a/format/Message.fbs b/format/Message.fbs
index ff30aceeda4f3..2cb60953c6a79 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -15,272 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-namespace org.apache.arrow.flatbuf;
-
-enum MetadataVersion:short {
-  V1,
-  V2
-}
-
-/// ----------------------------------------------------------------------
-/// Logical types and their metadata (if any)
-///
-/// These are stored in the flatbuffer in the Type union below
-
-table Null {
-}
-
-/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct
-/// (according to the physical memory layout). We used Struct_ here as
-/// Struct is a reserved word in Flatbuffers
-table Struct_ {
-}
-
-table List {
-}
-
-enum UnionMode:short { Sparse, Dense }
-
-/// A union is a complex type with children in Field
-/// By default ids in the type vector refer to the offsets in the children
-/// optionally typeIds provides an indirection between the child offset and the type id
-/// for each child typeIds[offset] is the id used in the type vector
-table Union {
-  mode: UnionMode;
-  typeIds: [ int ]; // optional, describes typeid of each child.
-}
-
-table Int {
-  bitWidth: int; // restricted to 8, 16, 32, and 64 in v1
-  is_signed: bool;
-}
-
-enum Precision:short {HALF, SINGLE, DOUBLE}
-
-table FloatingPoint {
-  precision: Precision;
-}
-
-/// Unicode with UTF-8 encoding
-table Utf8 {
-}
-
-table Binary {
-}
-
-table FixedWidthBinary {
-  /// Number of bytes per value
-  byteWidth: int;
-}
-
-table Bool {
-}
-
-table Decimal {
-  precision: int;
-  scale: int;
-}
-
-enum DateUnit: short {
-  DAY,
-  MILLISECOND
-}
-
-/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
-/// epoch (1970-01-01), stored in either of two units:
-///
-/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
-///   leap seconds), where the values are evenly divisible by 86400000
-/// * Days (32 bits) since the UNIX epoch
-table Date {
-  unit: DateUnit;
-}
-
-enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }
-
-/// Time type. The physical storage type depends on the unit
-/// - SECOND and MILLISECOND: 32 bits
-/// - MICROSECOND and NANOSECOND: 64 bits
-table Time {
-  unit: TimeUnit;
-  bitWidth: int;
-}
-
-/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
-table Timestamp {
-  unit: TimeUnit;
-
-  /// The time zone is a string indicating the name of a time zone, one of:
-  ///
-  /// * As used in the Olson time zone database (the "tz database" or
-  ///   "tzdata"), such as "America/New_York"
-  /// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
-  ///
-  /// Whether a timezone string is present indicates different semantics about
-  /// the data:
-  ///
-  /// * If the time zone is null or equal to an empty string, the data is "time
-  ///   zone naive" and shall be displayed *as is* to the user, not localized
-  ///   to the locale of the user. This data can be though of as UTC but
-  ///   without having "UTC" as the time zone, it is not considered to be
-  ///   localized to any time zone
-  ///
-  /// * If the time zone is set to a valid value, values can be displayed as
-  ///   "localized" to that time zone, even though the underlying 64-bit
-  ///   integers are identical to the same data stored in UTC. Converting
-  ///   between time zones is a metadata-only operation and does not change the
-  ///   underlying values
-  timezone: string;
-}
-
-enum IntervalUnit: short { YEAR_MONTH, DAY_TIME}
-table Interval {
-  unit: IntervalUnit;
-}
-
-/// ----------------------------------------------------------------------
-/// Top-level Type value, enabling extensible type-specific metadata. We can
-/// add new logical types to Type without breaking backwards compatibility
-
-union Type {
-  Null,
-  Int,
-  FloatingPoint,
-  Binary,
-  Utf8,
-  Bool,
-  Decimal,
-  Date,
-  Time,
-  Timestamp,
-  Interval,
-  List,
-  Struct_,
-  Union,
-  FixedWidthBinary
-}
-
-/// ----------------------------------------------------------------------
-/// The possible types of a vector
-
-enum VectorType: short {
-  /// used in List type, Dense Union and variable length primitive types (String, Binary)
-  OFFSET,
-  /// actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
-  DATA,
-  /// Bit vector indicating if each value is null
-  VALIDITY,
-  /// Type vector used in Union type
-  TYPE
-}
-
-/// ----------------------------------------------------------------------
-/// represents the physical layout of a buffer
-/// buffers have fixed width slots of a given type
-
-table VectorLayout {
-  /// the width of a slot in the buffer (typically 1, 8, 16, 32 or 64)
-  bit_width: short;
-  /// the purpose of the vector
-  type: VectorType;
-}
-
-
-/// ----------------------------------------------------------------------
-/// user defined key value pairs to add custom metadata to arrow
-/// key namespacing is the responsibility of the user
-
-table KeyValue {
-  key: string;
-  value: [ubyte];
-}
-
-/// ----------------------------------------------------------------------
-/// Dictionary encoding metadata
-
-table DictionaryEncoding {
-  /// The known dictionary id in the application where this data is used. In
-  /// the file or streaming formats, the dictionary ids are found in the
-  /// DictionaryBatch messages
-  id: long;
-
-  /// The dictionary indices are constrained to be positive integers. If this
-  /// field is null, the indices must be signed int32
-  indexType: Int;
+include "Schema.fbs";
 
-  /// By default, dictionaries are not ordered, or the order does not have
-  /// semantic meaning. In some statistical, applications, dictionary-encoding
-  /// is used to represent ordered categorical data, and we provide a way to
-  /// preserve that metadata here
-  isOrdered: bool;
-}
-
-/// ----------------------------------------------------------------------
-/// A field represents a named column in a record / row batch or child of a
-/// nested type.
-///
-/// - children is only for nested Arrow arrays
-/// - For primitive types, children will have length 0
-/// - nullable should default to true in general
-
-table Field {
-  // Name is not required, in i.e. a List
-  name: string;
-  nullable: bool;
-  type: Type;
-
-  // Present only if the field is dictionary encoded
-  dictionary: DictionaryEncoding;
-
-  // children apply only to Nested data types like Struct, List and Union
-  children: [Field];
-  /// layout of buffers produced for this type (as derived from the Type)
-  /// does not include children
-  /// each recordbatch will return instances of those Buffers.
-  layout: [ VectorLayout ];
-  // User-defined metadata
-  custom_metadata: [ KeyValue ];
-}
-
-/// ----------------------------------------------------------------------
-/// Endianness of the platform that produces the RecordBatch
-
-enum Endianness:short { Little, Big }
-
-/// ----------------------------------------------------------------------
-/// A Schema describes the columns in a row batch
-
-table Schema {
-
-  /// endianness of the buffer
-  /// it is Little Endian by default
-  /// if endianness doesn't match the underlying system then the vectors need to be converted
-  endianness: Endianness=Little;
-
-  fields: [Field];
-  // User-defined metadata
-  custom_metadata: [ KeyValue ];
-}
+namespace org.apache.arrow.flatbuf;
 
 /// ----------------------------------------------------------------------
 /// Data structures for describing a table row batch (a collection of
 /// equal-length Arrow arrays)
 
-/// A Buffer represents a single contiguous memory segment
-struct Buffer {
-  /// The shared memory page id where this buffer is located. Currently this is
-  /// not used
-  page: int;
-
-  /// The relative offset into the shared memory page where the bytes for this
-  /// buffer starts
-  offset: long;
-
-  /// The absolute length (in bytes) of the memory buffer. The memory is found
-  /// from offset (inclusive) to offset + length (non-inclusive).
-  length: long;
-}
-
 /// Metadata about a field at some level of a nested type tree (but not
 /// its children).
 ///
@@ -349,4 +91,4 @@ table Message {
   bodyLength: long;
 }
 
-root_type Message;
+root_type Message;
\ No newline at end of file
diff --git a/format/Schema.fbs b/format/Schema.fbs
new file mode 100644
index 0000000000000..5268bf95cfdc8
--- /dev/null
+++ b/format/Schema.fbs
@@ -0,0 +1,280 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Logical types, vector layouts, and schemas
+
+namespace org.apache.arrow.flatbuf;
+
+enum MetadataVersion:short {
+  V1,
+  V2
+}
+
+/// These are stored in the flatbuffer in the Type union below
+
+table Null {
+}
+
+/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct
+/// (according to the physical memory layout). We used Struct_ here as
+/// Struct is a reserved word in Flatbuffers
+table Struct_ {
+}
+
+table List {
+}
+
+enum UnionMode:short { Sparse, Dense }
+
+/// A union is a complex type with children in Field
+/// By default ids in the type vector refer to the offsets in the children
+/// optionally typeIds provides an indirection between the child offset and the type id
+/// for each child typeIds[offset] is the id used in the type vector
+table Union {
+  mode: UnionMode;
+  typeIds: [ int ]; // optional, describes typeid of each child.
+}
+
+table Int {
+  bitWidth: int; // restricted to 8, 16, 32, and 64 in v1
+  is_signed: bool;
+}
+
+enum Precision:short {HALF, SINGLE, DOUBLE}
+
+table FloatingPoint {
+  precision: Precision;
+}
+
+/// Unicode with UTF-8 encoding
+table Utf8 {
+}
+
+table Binary {
+}
+
+table FixedWidthBinary {
+  /// Number of bytes per value
+  byteWidth: int;
+}
+
+table Bool {
+}
+
+table Decimal {
+  precision: int;
+  scale: int;
+}
+
+enum DateUnit: short {
+  DAY,
+  MILLISECOND
+}
+
+/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
+/// epoch (1970-01-01), stored in either of two units:
+///
+/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
+///   leap seconds), where the values are evenly divisible by 86400000
+/// * Days (32 bits) since the UNIX epoch
+table Date {
+  unit: DateUnit;
+}
+
+enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }
+
+/// Time type. The physical storage type depends on the unit
+/// - SECOND and MILLISECOND: 32 bits
+/// - MICROSECOND and NANOSECOND: 64 bits
+table Time {
+  unit: TimeUnit;
+  bitWidth: int;
+}
+
+/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
+table Timestamp {
+  unit: TimeUnit;
+
+  /// The time zone is a string indicating the name of a time zone, one of:
+  ///
+  /// * As used in the Olson time zone database (the "tz database" or
+  ///   "tzdata"), such as "America/New_York"
+  /// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
+  ///
+  /// Whether a timezone string is present indicates different semantics about
+  /// the data:
+  ///
+  /// * If the time zone is null or equal to an empty string, the data is "time
+  ///   zone naive" and shall be displayed *as is* to the user, not localized
+  ///   to the locale of the user. This data can be though of as UTC but
+  ///   without having "UTC" as the time zone, it is not considered to be
+  ///   localized to any time zone
+  ///
+  /// * If the time zone is set to a valid value, values can be displayed as
+  ///   "localized" to that time zone, even though the underlying 64-bit
+  ///   integers are identical to the same data stored in UTC. Converting
+  ///   between time zones is a metadata-only operation and does not change the
+  ///   underlying values
+  timezone: string;
+}
+
+enum IntervalUnit: short { YEAR_MONTH, DAY_TIME}
+table Interval {
+  unit: IntervalUnit;
+}
+
+/// ----------------------------------------------------------------------
+/// Top-level Type value, enabling extensible type-specific metadata. We can
+/// add new logical types to Type without breaking backwards compatibility
+
+union Type {
+  Null,
+  Int,
+  FloatingPoint,
+  Binary,
+  Utf8,
+  Bool,
+  Decimal,
+  Date,
+  Time,
+  Timestamp,
+  Interval,
+  List,
+  Struct_,
+  Union,
+  FixedWidthBinary
+}
+
+/// ----------------------------------------------------------------------
+/// The possible types of a vector
+
+enum VectorType: short {
+  /// used in List type, Dense Union and variable length primitive types (String, Binary)
+  OFFSET,
+  /// actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
+  DATA,
+  /// Bit vector indicating if each value is null
+  VALIDITY,
+  /// Type vector used in Union type
+  TYPE
+}
+
+/// ----------------------------------------------------------------------
+/// represents the physical layout of a buffer
+/// buffers have fixed width slots of a given type
+
+table VectorLayout {
+  /// the width of a slot in the buffer (typically 1, 8, 16, 32 or 64)
+  bit_width: short;
+  /// the purpose of the vector
+  type: VectorType;
+}
+
+
+/// ----------------------------------------------------------------------
+/// user defined key value pairs to add custom metadata to arrow
+/// key namespacing is the responsibility of the user
+
+table KeyValue {
+  key: string;
+  value: [ubyte];
+}
+
+/// ----------------------------------------------------------------------
+/// Dictionary encoding metadata
+
+table DictionaryEncoding {
+  /// The known dictionary id in the application where this data is used. In
+  /// the file or streaming formats, the dictionary ids are found in the
+  /// DictionaryBatch messages
+  id: long;
+
+  /// The dictionary indices are constrained to be positive integers. If this
+  /// field is null, the indices must be signed int32
+  indexType: Int;
+
+  /// By default, dictionaries are not ordered, or the order does not have
+  /// semantic meaning. In some statistical, applications, dictionary-encoding
+  /// is used to represent ordered categorical data, and we provide a way to
+  /// preserve that metadata here
+  isOrdered: bool;
+}
+
+/// ----------------------------------------------------------------------
+/// A field represents a named column in a record / row batch or child of a
+/// nested type.
+///
+/// - children is only for nested Arrow arrays
+/// - For primitive types, children will have length 0
+/// - nullable should default to true in general
+
+table Field {
+  // Name is not required, in i.e. a List
+  name: string;
+  nullable: bool;
+  type: Type;
+
+  // Present only if the field is dictionary encoded
+  dictionary: DictionaryEncoding;
+
+  // children apply only to Nested data types like Struct, List and Union
+  children: [Field];
+  /// layout of buffers produced for this type (as derived from the Type)
+  /// does not include children
+  /// each recordbatch will return instances of those Buffers.
+  layout: [ VectorLayout ];
+  // User-defined metadata
+  custom_metadata: [ KeyValue ];
+}
+
+/// ----------------------------------------------------------------------
+/// Endianness of the platform producing the data
+
+enum Endianness:short { Little, Big }
+
+/// ----------------------------------------------------------------------
+/// A Buffer represents a single contiguous memory segment
+struct Buffer {
+  /// The shared memory page id where this buffer is located. Currently this is
+  /// not used
+  page: int;
+
+  /// The relative offset into the shared memory page where the bytes for this
+  /// buffer starts
+  offset: long;
+
+  /// The absolute length (in bytes) of the memory buffer. The memory is found
+  /// from offset (inclusive) to offset + length (non-inclusive).
+  length: long;
+}
+
+/// ----------------------------------------------------------------------
+/// A Schema describes the columns in a row batch
+
+table Schema {
+
+  /// endianness of the buffer
+  /// it is Little Endian by default
+  /// if endianness doesn't match the underlying system then the vectors need to be converted
+  endianness: Endianness=Little;
+
+  fields: [Field];
+  // User-defined metadata
+  custom_metadata: [ KeyValue ];
+}
+
+root_type Schema;
diff --git a/java/format/pom.xml b/java/format/pom.xml
index c65a7bc3de197..e7a58a4172fe2 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -1,13 +1,13 @@
 <?xml version="1.0"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
-  license agreements. See the NOTICE file distributed with this work for additional 
-  information regarding copyright ownership. The ASF licenses this file to 
-  You under the Apache License, Version 2.0 (the "License"); you may not use 
-  this file except in compliance with the License. You may obtain a copy of 
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
-  by applicable law or agreed to in writing, software distributed under the 
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
-  OF ANY KIND, either express or implied. See the License for the specific 
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
   language governing permissions and limitations under the License. -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 <modelVersion>4.0.0</modelVersion>
@@ -109,6 +109,7 @@
               <argument>-j</argument>
               <argument>-o</argument>
               <argument>${flatc.generated.files}</argument>
+              <argument>../../format/Schema.fbs</argument>
               <argument>../../format/Message.fbs</argument>
               <argument>../../format/File.fbs</argument>
             </arguments>
@@ -165,4 +166,3 @@
 
 </build>
 </project>
-

From e968ca6e30209abeb90e099eb95de59655be73a8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 23 Mar 2017 12:38:40 -0400
Subject: [PATCH 0410/1644] ARROW-621: [C++] Start IPC benchmark suite for
 record batches, implement "inline" visitor. Code reorg

From the benchmarks, the difference between virtual functions and an "inline" switch statement is very small, but it serves to reduce some boilerplate when many of the visit functions are the same

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #427 from wesm/ARROW-621 and squashes the following commits:

b975053 [Wes McKinney] cpplint
782636a [Wes McKinney] Mark template inline
3ae494e [Wes McKinney] Inline visitor, remove code duplication in loader.cc in favor of templates / std::enable_if
1b2d253 [Wes McKinney] Tweak benchmark params
b126ca8 [Wes McKinney] Draft IPC roundtrip benchmark for wide record batches. Some test code refactoring
---
 cpp/CMakeLists.txt                            |   1 +
 cpp/src/arrow/CMakeLists.txt                  |   1 +
 cpp/src/arrow/api.h                           |   1 +
 cpp/src/arrow/array-list-test.cc              |   1 +
 cpp/src/arrow/array-primitive-test.cc         |   1 +
 cpp/src/arrow/array-string-test.cc            |   1 +
 cpp/src/arrow/array-struct-test.cc            |   1 +
 cpp/src/arrow/array.cc                        |  36 +----
 cpp/src/arrow/array.h                         |  33 +----
 cpp/src/arrow/column-test.cc                  |   1 +
 cpp/src/arrow/ipc/CMakeLists.txt              |   4 +
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc | 134 ++++++++++++++++++
 cpp/src/arrow/ipc/reader.h                    |   5 +-
 cpp/src/arrow/ipc/writer.cc                   |   8 +-
 cpp/src/arrow/loader.cc                       |  65 ++++-----
 cpp/src/arrow/table-test.cc                   |   1 +
 cpp/src/arrow/test-common.h                   |  84 +++++++++++
 cpp/src/arrow/test-util.h                     |  45 +-----
 cpp/src/arrow/type.cc                         |  36 +----
 cpp/src/arrow/type.h                          |  33 +----
 cpp/src/arrow/type_fwd.h                      |   2 +
 cpp/src/arrow/visitor.cc                      |  96 +++++++++++++
 cpp/src/arrow/visitor.h                       |  93 ++++++++++++
 cpp/src/arrow/visitor_inline.h                |  67 +++++++++
 24 files changed, 528 insertions(+), 222 deletions(-)
 create mode 100644 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
 create mode 100644 cpp/src/arrow/test-common.h
 create mode 100644 cpp/src/arrow/visitor.cc
 create mode 100644 cpp/src/arrow/visitor.h
 create mode 100644 cpp/src/arrow/visitor_inline.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 61e645da20e75..c04afe47030a5 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -845,6 +845,7 @@ set(ARROW_SRCS
   src/arrow/status.cc
   src/arrow/table.cc
   src/arrow/type.cc
+  src/arrow/visitor.cc
 
   src/arrow/util/bit-util.cc
 )
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 24a95475b14e0..0e83aacaadab5 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -34,6 +34,7 @@ install(FILES
   type_fwd.h
   type_traits.h
   test-util.h
+  visitor.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow")
 
 # pkg-config support
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 3bc86662613ed..ea818b62931d6 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -32,5 +32,6 @@
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
+#include "arrow/visitor.h"
 
 #endif  // ARROW_API_H
diff --git a/cpp/src/arrow/array-list-test.cc b/cpp/src/arrow/array-list-test.cc
index 87dfdaaed33a4..1cfa77f684868 100644
--- a/cpp/src/arrow/array-list-test.cc
+++ b/cpp/src/arrow/array-list-test.cc
@@ -26,6 +26,7 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/status.h"
+#include "arrow/test-common.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
index dfa37a8063767..6863e58df05d2 100644
--- a/cpp/src/arrow/array-primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -26,6 +26,7 @@
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
 #include "arrow/status.h"
+#include "arrow/test-common.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
index ed38acd010329..6c2c1516c8f3c 100644
--- a/cpp/src/arrow/array-string-test.cc
+++ b/cpp/src/arrow/array-string-test.cc
@@ -25,6 +25,7 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/test-common.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
diff --git a/cpp/src/arrow/array-struct-test.cc b/cpp/src/arrow/array-struct-test.cc
index f4e7409a6232a..4eb1eab13fbc6 100644
--- a/cpp/src/arrow/array-struct-test.cc
+++ b/cpp/src/arrow/array-struct-test.cc
@@ -24,6 +24,7 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/status.h"
+#include "arrow/test-common.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 4fa2b2b521f59..20b732ab114da 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -28,6 +28,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
+#include "arrow/visitor.h"
 
 namespace arrow {
 
@@ -468,41 +469,6 @@ Status DictionaryArray::Accept(ArrayVisitor* visitor) const {
   return visitor->Visit(*this);
 }
 
-#define ARRAY_VISITOR_DEFAULT(ARRAY_CLASS)                   \
-  Status ArrayVisitor::Visit(const ARRAY_CLASS& array) {     \
-    return Status::NotImplemented(array.type()->ToString()); \
-  }
-
-ARRAY_VISITOR_DEFAULT(NullArray);
-ARRAY_VISITOR_DEFAULT(BooleanArray);
-ARRAY_VISITOR_DEFAULT(Int8Array);
-ARRAY_VISITOR_DEFAULT(Int16Array);
-ARRAY_VISITOR_DEFAULT(Int32Array);
-ARRAY_VISITOR_DEFAULT(Int64Array);
-ARRAY_VISITOR_DEFAULT(UInt8Array);
-ARRAY_VISITOR_DEFAULT(UInt16Array);
-ARRAY_VISITOR_DEFAULT(UInt32Array);
-ARRAY_VISITOR_DEFAULT(UInt64Array);
-ARRAY_VISITOR_DEFAULT(HalfFloatArray);
-ARRAY_VISITOR_DEFAULT(FloatArray);
-ARRAY_VISITOR_DEFAULT(DoubleArray);
-ARRAY_VISITOR_DEFAULT(BinaryArray);
-ARRAY_VISITOR_DEFAULT(StringArray);
-ARRAY_VISITOR_DEFAULT(FixedWidthBinaryArray);
-ARRAY_VISITOR_DEFAULT(Date32Array);
-ARRAY_VISITOR_DEFAULT(Date64Array);
-ARRAY_VISITOR_DEFAULT(TimeArray);
-ARRAY_VISITOR_DEFAULT(TimestampArray);
-ARRAY_VISITOR_DEFAULT(IntervalArray);
-ARRAY_VISITOR_DEFAULT(ListArray);
-ARRAY_VISITOR_DEFAULT(StructArray);
-ARRAY_VISITOR_DEFAULT(UnionArray);
-ARRAY_VISITOR_DEFAULT(DictionaryArray);
-
-Status ArrayVisitor::Visit(const DecimalArray& array) {
-  return Status::NotImplemented("decimal");
-}
-
 // ----------------------------------------------------------------------
 // Instantiate templates
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index e66ac505d5dbf..2a072dbf25ec0 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -31,6 +31,7 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
+#include "arrow/visitor.h"
 
 namespace arrow {
 
@@ -38,38 +39,6 @@ class MemoryPool;
 class MutableBuffer;
 class Status;
 
-class ARROW_EXPORT ArrayVisitor {
- public:
-  virtual ~ArrayVisitor() = default;
-
-  virtual Status Visit(const NullArray& array);
-  virtual Status Visit(const BooleanArray& array);
-  virtual Status Visit(const Int8Array& array);
-  virtual Status Visit(const Int16Array& array);
-  virtual Status Visit(const Int32Array& array);
-  virtual Status Visit(const Int64Array& array);
-  virtual Status Visit(const UInt8Array& array);
-  virtual Status Visit(const UInt16Array& array);
-  virtual Status Visit(const UInt32Array& array);
-  virtual Status Visit(const UInt64Array& array);
-  virtual Status Visit(const HalfFloatArray& array);
-  virtual Status Visit(const FloatArray& array);
-  virtual Status Visit(const DoubleArray& array);
-  virtual Status Visit(const StringArray& array);
-  virtual Status Visit(const BinaryArray& array);
-  virtual Status Visit(const FixedWidthBinaryArray& array);
-  virtual Status Visit(const Date32Array& array);
-  virtual Status Visit(const Date64Array& array);
-  virtual Status Visit(const TimeArray& array);
-  virtual Status Visit(const TimestampArray& array);
-  virtual Status Visit(const IntervalArray& array);
-  virtual Status Visit(const DecimalArray& array);
-  virtual Status Visit(const ListArray& array);
-  virtual Status Visit(const StructArray& array);
-  virtual Status Visit(const UnionArray& array);
-  virtual Status Visit(const DictionaryArray& type);
-};
-
 /// Immutable data array with some logical type and some length.
 ///
 /// Any memory is owned by the respective Buffer instance (or its parents).
diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
index 24d58c80b9fae..872fcb95c08e1 100644
--- a/cpp/src/arrow/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -25,6 +25,7 @@
 #include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/schema.h"
+#include "arrow/test-common.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 629cc5bbed055..056e7dba53830 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -170,3 +170,7 @@ if (ARROW_BUILD_UTILITIES)
   add_executable(stream-to-file stream-to-file.cc)
   target_link_libraries(stream-to-file ${UTIL_LINK_LIBS})
 endif()
+
+ADD_ARROW_BENCHMARK(ipc-read-write-benchmark)
+ARROW_TEST_LINK_LIBRARIES(ipc-read-write-benchmark
+  ${ARROW_IPC_TEST_LINK_LIBS})
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
new file mode 100644
index 0000000000000..e27e5136a0d5a
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include <cstdint>
+#include <sstream>
+#include <string>
+
+#include "arrow/api.h"
+#include "arrow/io/memory.h"
+#include "arrow/ipc/api.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+template <typename TYPE>
+std::shared_ptr<RecordBatch> MakeRecordBatch(int64_t total_size, int64_t num_fields) {
+  using T = typename TYPE::c_type;
+  size_t itemsize = sizeof(T);
+  int64_t length = total_size / num_fields / itemsize;
+
+  auto type = TypeTraits<TYPE>::type_singleton();
+
+  std::vector<bool> is_valid;
+  test::random_is_valid(length, 0.1, &is_valid);
+
+  std::vector<T> values;
+  test::randint<T>(length, 0, 100, &values);
+
+  MemoryPool* pool = default_memory_pool();
+  typename TypeTraits<TYPE>::BuilderType builder(pool, type);
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid[i]) {
+      builder.Append(values[i]);
+    } else {
+      builder.AppendNull();
+    }
+  }
+  std::shared_ptr<Array> array;
+  builder.Finish(&array);
+
+  ArrayVector arrays;
+  std::vector<std::shared_ptr<Field>> fields;
+  for (int64_t i = 0; i < num_fields; ++i) {
+    std::stringstream ss;
+    ss << "f" << i;
+    fields.push_back(field(ss.str(), type));
+    arrays.push_back(array);
+  }
+
+  auto schema = std::make_shared<Schema>(fields);
+  return std::make_shared<RecordBatch>(schema, length, arrays);
+}
+
+static void BM_WriteRecordBatch(benchmark::State& state) {  // NOLINT non-const reference
+  // 1MB
+  constexpr int64_t kTotalSize = 1 << 20;
+
+  auto buffer = std::make_shared<PoolBuffer>(default_memory_pool());
+  buffer->Resize(kTotalSize & 2);
+  auto record_batch = MakeRecordBatch<Int64Type>(kTotalSize, state.range(0));
+
+  while (state.KeepRunning()) {
+    io::BufferOutputStream stream(buffer);
+    int32_t metadata_length;
+    int64_t body_length;
+    if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
+             default_memory_pool())
+             .ok()) {
+      state.SkipWithError("Failed to write!");
+    }
+  }
+  state.SetBytesProcessed(int64_t(state.iterations()) * kTotalSize);
+}
+
+static void BM_ReadRecordBatch(benchmark::State& state) {  // NOLINT non-const reference
+  // 1MB
+  constexpr int64_t kTotalSize = 1 << 20;
+
+  auto buffer = std::make_shared<PoolBuffer>(default_memory_pool());
+  buffer->Resize(kTotalSize & 2);
+  auto record_batch = MakeRecordBatch<Int64Type>(kTotalSize, state.range(0));
+
+  io::BufferOutputStream stream(buffer);
+
+  int32_t metadata_length;
+  int64_t body_length;
+  if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
+           default_memory_pool())
+           .ok()) {
+    state.SkipWithError("Failed to write!");
+  }
+
+  while (state.KeepRunning()) {
+    std::shared_ptr<RecordBatch> result;
+    io::BufferReader reader(buffer);
+
+    if (!ipc::ReadRecordBatch(record_batch->schema(), 0, &reader, &result).ok()) {
+      state.SkipWithError("Failed to read!");
+    }
+  }
+  state.SetBytesProcessed(int64_t(state.iterations()) * kTotalSize);
+}
+
+BENCHMARK(BM_WriteRecordBatch)
+    ->RangeMultiplier(4)
+    ->Range(1, 1 << 13)
+    ->MinTime(1.0)
+    ->Unit(benchmark::kMicrosecond)
+    ->UseRealTime();
+
+BENCHMARK(BM_ReadRecordBatch)
+    ->RangeMultiplier(4)
+    ->Range(1, 1 << 13)
+    ->MinTime(1.0)
+    ->Unit(benchmark::kMicrosecond)
+    ->UseRealTime();
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 1e8636c1efcce..ffd0a111d604b 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -120,10 +120,9 @@ class ARROW_EXPORT FileReader {
   std::unique_ptr<FileReaderImpl> impl_;
 };
 
-
 /// Read encapsulated message and RecordBatch
-Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<Schema>& schema,
-    int64_t offset, io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
+Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 0f55f8e33e71d..dc991aba79795 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -47,9 +47,8 @@ namespace ipc {
 
 class RecordBatchWriter : public ArrayVisitor {
  public:
-  RecordBatchWriter(
-      MemoryPool* pool, int64_t buffer_start_offset, int max_recursion_depth,
-      bool allow_64bit)
+  RecordBatchWriter(MemoryPool* pool, int64_t buffer_start_offset,
+      int max_recursion_depth, bool allow_64bit)
       : pool_(pool),
         max_recursion_depth_(max_recursion_depth),
         buffer_start_offset_(buffer_start_offset),
@@ -501,8 +500,7 @@ class DictionaryWriter : public RecordBatchWriter {
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
     MemoryPool* pool, int max_recursion_depth, bool allow_64bit) {
-  RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth,
-      allow_64bit);
+  RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth, allow_64bit);
   return writer.Write(batch, dst, metadata_length, body_length);
 }
 
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
index bc506be572625..a67a3e94bd5f2 100644
--- a/cpp/src/arrow/loader.cc
+++ b/cpp/src/arrow/loader.cc
@@ -28,6 +28,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/visibility.h"
+#include "arrow/visitor_inline.h"
 
 namespace arrow {
 
@@ -35,7 +36,7 @@ class Array;
 struct DataType;
 class Status;
 
-class ArrayLoader : public TypeVisitor {
+class ArrayLoader {
  public:
   ArrayLoader(const std::shared_ptr<DataType>& type, ArrayLoaderContext* context)
       : type_(type), context_(context) {}
@@ -45,8 +46,7 @@ class ArrayLoader : public TypeVisitor {
       return Status::Invalid("Max recursion depth reached");
     }
 
-    // Load the array
-    RETURN_NOT_OK(type_->Accept(this));
+    RETURN_NOT_OK(VisitTypeInline(*type_, this));
 
     *out = std::move(result_);
     return Status::OK();
@@ -92,8 +92,10 @@ class ArrayLoader : public TypeVisitor {
     return Status::OK();
   }
 
-  template <typename CONTAINER>
+  template <typename TYPE>
   Status LoadBinary() {
+    using CONTAINER = typename TypeTraits<TYPE>::ArrayType;
+
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap, offsets, values;
 
@@ -131,33 +133,24 @@ class ArrayLoader : public TypeVisitor {
     return Status::OK();
   }
 
-#define VISIT_PRIMITIVE(TYPE) \
-  Status Visit(const TYPE& type) override { return LoadPrimitive<TYPE>(); }
-
-  VISIT_PRIMITIVE(BooleanType);
-  VISIT_PRIMITIVE(Int8Type);
-  VISIT_PRIMITIVE(Int16Type);
-  VISIT_PRIMITIVE(Int32Type);
-  VISIT_PRIMITIVE(Int64Type);
-  VISIT_PRIMITIVE(UInt8Type);
-  VISIT_PRIMITIVE(UInt16Type);
-  VISIT_PRIMITIVE(UInt32Type);
-  VISIT_PRIMITIVE(UInt64Type);
-  VISIT_PRIMITIVE(HalfFloatType);
-  VISIT_PRIMITIVE(FloatType);
-  VISIT_PRIMITIVE(DoubleType);
-  VISIT_PRIMITIVE(Date32Type);
-  VISIT_PRIMITIVE(Date64Type);
-  VISIT_PRIMITIVE(TimeType);
-  VISIT_PRIMITIVE(TimestampType);
-
-#undef VISIT_PRIMITIVE
-
-  Status Visit(const StringType& type) override { return LoadBinary<StringArray>(); }
-
-  Status Visit(const BinaryType& type) override { return LoadBinary<BinaryArray>(); }
-
-  Status Visit(const FixedWidthBinaryType& type) override {
+  Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
+                              !std::is_base_of<FixedWidthBinaryType, T>::value &&
+                              !std::is_base_of<DictionaryType, T>::value,
+      Status>::type
+  Visit(const T& type) {
+    return LoadPrimitive<T>();
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BinaryType, T>::value, Status>::type Visit(
+      const T& type) {
+    return LoadBinary<T>();
+  }
+
+  Status Visit(const FixedWidthBinaryType& type) {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap, data;
 
@@ -169,7 +162,7 @@ class ArrayLoader : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const ListType& type) override {
+  Status Visit(const ListType& type) {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap, offsets;
 
@@ -196,7 +189,7 @@ class ArrayLoader : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const StructType& type) override {
+  Status Visit(const StructType& type) {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap;
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
@@ -209,7 +202,7 @@ class ArrayLoader : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const UnionType& type) override {
+  Status Visit(const UnionType& type) {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap, type_ids, offsets;
 
@@ -230,12 +223,12 @@ class ArrayLoader : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DictionaryType& type) override {
+  Status Visit(const DictionaryType& type) {
     std::shared_ptr<Array> indices;
     RETURN_NOT_OK(LoadArray(type.index_type(), context_, &indices));
     result_ = std::make_shared<DictionaryArray>(type_, indices);
     return Status::OK();
-  };
+  }
 
   std::shared_ptr<Array> result() const { return result_; }
 
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 36374731cbb49..6bb31638ecbbf 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -26,6 +26,7 @@
 #include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
+#include "arrow/test-common.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 
diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h
new file mode 100644
index 0000000000000..f704b6b545b7d
--- /dev/null
+++ b/cpp/src/arrow/test-common.h
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TEST_COMMON_H
+#define ARROW_TEST_COMMON_H
+
+#include <cstdint>
+#include <memory>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/column.h"
+#include "arrow/memory_pool.h"
+#include "arrow/table.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+class TestBase : public ::testing::Test {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+    random_seed_ = 0;
+  }
+
+  template <typename ArrayType>
+  std::shared_ptr<Array> MakePrimitive(int64_t length, int64_t null_count = 0) {
+    auto data = std::make_shared<PoolBuffer>(pool_);
+    const int64_t data_nbytes = length * sizeof(typename ArrayType::value_type);
+    EXPECT_OK(data->Resize(data_nbytes));
+
+    // Fill with random data
+    test::random_bytes(data_nbytes, random_seed_++, data->mutable_data());
+
+    auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
+    EXPECT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
+    return std::make_shared<ArrayType>(length, data, null_bitmap, null_count);
+  }
+
+ protected:
+  uint32_t random_seed_;
+  MemoryPool* pool_;
+};
+
+class TestBuilder : public ::testing::Test {
+ public:
+  void SetUp() {
+    pool_ = default_memory_pool();
+    type_ = TypePtr(new UInt8Type());
+    builder_.reset(new UInt8Builder(pool_));
+    builder_nn_.reset(new UInt8Builder(pool_));
+  }
+
+ protected:
+  MemoryPool* pool_;
+
+  TypePtr type_;
+  std::unique_ptr<ArrayBuilder> builder_;
+  std::unique_ptr<ArrayBuilder> builder_nn_;
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_TEST_COMMON_H_
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index f05a54168b631..bed555984fb68 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -25,7 +25,7 @@
 #include <string>
 #include <vector>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
@@ -208,32 +208,6 @@ Status MakeRandomBytePoolBuffer(int64_t length, MemoryPool* pool,
 
 }  // namespace test
 
-class TestBase : public ::testing::Test {
- public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-    random_seed_ = 0;
-  }
-
-  template <typename ArrayType>
-  std::shared_ptr<Array> MakePrimitive(int64_t length, int64_t null_count = 0) {
-    auto data = std::make_shared<PoolBuffer>(pool_);
-    const int64_t data_nbytes = length * sizeof(typename ArrayType::value_type);
-    EXPECT_OK(data->Resize(data_nbytes));
-
-    // Fill with random data
-    test::random_bytes(data_nbytes, random_seed_++, data->mutable_data());
-
-    auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
-    EXPECT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
-    return std::make_shared<ArrayType>(length, data, null_bitmap, null_count);
-  }
-
- protected:
-  uint32_t random_seed_;
-  MemoryPool* pool_;
-};
-
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::shared_ptr<DataType>& type,
     const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
@@ -275,23 +249,6 @@ void ArrayFromVector(const std::vector<C_TYPE>& values, std::shared_ptr<Array>*
   ASSERT_OK(builder.Finish(out));
 }
 
-class TestBuilder : public ::testing::Test {
- public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-    type_ = TypePtr(new UInt8Type());
-    builder_.reset(new UInt8Builder(pool_));
-    builder_nn_.reset(new UInt8Builder(pool_));
-  }
-
- protected:
-  MemoryPool* pool_;
-
-  TypePtr type_;
-  std::unique_ptr<ArrayBuilder> builder_;
-  std::unique_ptr<ArrayBuilder> builder_nn_;
-};
-
 template <class T, class Builder>
 Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
     int64_t size, Builder* builder, std::shared_ptr<Array>* out) {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 937cbc5a7669d..1c61eb61abea0 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -24,6 +24,7 @@
 #include "arrow/compare.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
+#include "arrow/visitor.h"
 
 namespace arrow {
 
@@ -331,39 +332,4 @@ std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
   return {};
 }
 
-// ----------------------------------------------------------------------
-// Default implementations of TypeVisitor methods
-
-#define TYPE_VISITOR_DEFAULT(TYPE_CLASS)              \
-  Status TypeVisitor::Visit(const TYPE_CLASS& type) { \
-    return Status::NotImplemented(type.ToString());   \
-  }
-
-TYPE_VISITOR_DEFAULT(NullType);
-TYPE_VISITOR_DEFAULT(BooleanType);
-TYPE_VISITOR_DEFAULT(Int8Type);
-TYPE_VISITOR_DEFAULT(Int16Type);
-TYPE_VISITOR_DEFAULT(Int32Type);
-TYPE_VISITOR_DEFAULT(Int64Type);
-TYPE_VISITOR_DEFAULT(UInt8Type);
-TYPE_VISITOR_DEFAULT(UInt16Type);
-TYPE_VISITOR_DEFAULT(UInt32Type);
-TYPE_VISITOR_DEFAULT(UInt64Type);
-TYPE_VISITOR_DEFAULT(HalfFloatType);
-TYPE_VISITOR_DEFAULT(FloatType);
-TYPE_VISITOR_DEFAULT(DoubleType);
-TYPE_VISITOR_DEFAULT(StringType);
-TYPE_VISITOR_DEFAULT(BinaryType);
-TYPE_VISITOR_DEFAULT(FixedWidthBinaryType);
-TYPE_VISITOR_DEFAULT(Date64Type);
-TYPE_VISITOR_DEFAULT(Date32Type);
-TYPE_VISITOR_DEFAULT(TimeType);
-TYPE_VISITOR_DEFAULT(TimestampType);
-TYPE_VISITOR_DEFAULT(IntervalType);
-TYPE_VISITOR_DEFAULT(DecimalType);
-TYPE_VISITOR_DEFAULT(ListType);
-TYPE_VISITOR_DEFAULT(StructType);
-TYPE_VISITOR_DEFAULT(UnionType);
-TYPE_VISITOR_DEFAULT(DictionaryType);
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index c179bf336987b..40c00a4bac1b1 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -28,6 +28,7 @@
 #include "arrow/type_fwd.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
+#include "arrow/visitor.h"
 
 namespace arrow {
 
@@ -119,38 +120,6 @@ class BufferDescr {
   int bit_width_;
 };
 
-class ARROW_EXPORT TypeVisitor {
- public:
-  virtual ~TypeVisitor() = default;
-
-  virtual Status Visit(const NullType& type);
-  virtual Status Visit(const BooleanType& type);
-  virtual Status Visit(const Int8Type& type);
-  virtual Status Visit(const Int16Type& type);
-  virtual Status Visit(const Int32Type& type);
-  virtual Status Visit(const Int64Type& type);
-  virtual Status Visit(const UInt8Type& type);
-  virtual Status Visit(const UInt16Type& type);
-  virtual Status Visit(const UInt32Type& type);
-  virtual Status Visit(const UInt64Type& type);
-  virtual Status Visit(const HalfFloatType& type);
-  virtual Status Visit(const FloatType& type);
-  virtual Status Visit(const DoubleType& type);
-  virtual Status Visit(const StringType& type);
-  virtual Status Visit(const BinaryType& type);
-  virtual Status Visit(const FixedWidthBinaryType& type);
-  virtual Status Visit(const Date64Type& type);
-  virtual Status Visit(const Date32Type& type);
-  virtual Status Visit(const TimeType& type);
-  virtual Status Visit(const TimestampType& type);
-  virtual Status Visit(const IntervalType& type);
-  virtual Status Visit(const DecimalType& type);
-  virtual Status Visit(const ListType& type);
-  virtual Status Visit(const StructType& type);
-  virtual Status Visit(const UnionType& type);
-  virtual Status Visit(const DictionaryType& type);
-};
-
 struct ARROW_EXPORT DataType {
   Type::type type;
 
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index ae85593cf4546..f62c0314a4620 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -18,6 +18,8 @@
 #ifndef ARROW_TYPE_FWD_H
 #define ARROW_TYPE_FWD_H
 
+#include <memory>
+
 #include "arrow/util/visibility.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
new file mode 100644
index 0000000000000..181e932eeebf6
--- /dev/null
+++ b/cpp/src/arrow/visitor.cc
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/visitor.h"
+
+#include "arrow/array.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+#define ARRAY_VISITOR_DEFAULT(ARRAY_CLASS)                   \
+  Status ArrayVisitor::Visit(const ARRAY_CLASS& array) {     \
+    return Status::NotImplemented(array.type()->ToString()); \
+  }
+
+ARRAY_VISITOR_DEFAULT(NullArray);
+ARRAY_VISITOR_DEFAULT(BooleanArray);
+ARRAY_VISITOR_DEFAULT(Int8Array);
+ARRAY_VISITOR_DEFAULT(Int16Array);
+ARRAY_VISITOR_DEFAULT(Int32Array);
+ARRAY_VISITOR_DEFAULT(Int64Array);
+ARRAY_VISITOR_DEFAULT(UInt8Array);
+ARRAY_VISITOR_DEFAULT(UInt16Array);
+ARRAY_VISITOR_DEFAULT(UInt32Array);
+ARRAY_VISITOR_DEFAULT(UInt64Array);
+ARRAY_VISITOR_DEFAULT(HalfFloatArray);
+ARRAY_VISITOR_DEFAULT(FloatArray);
+ARRAY_VISITOR_DEFAULT(DoubleArray);
+ARRAY_VISITOR_DEFAULT(BinaryArray);
+ARRAY_VISITOR_DEFAULT(StringArray);
+ARRAY_VISITOR_DEFAULT(FixedWidthBinaryArray);
+ARRAY_VISITOR_DEFAULT(Date32Array);
+ARRAY_VISITOR_DEFAULT(Date64Array);
+ARRAY_VISITOR_DEFAULT(TimeArray);
+ARRAY_VISITOR_DEFAULT(TimestampArray);
+ARRAY_VISITOR_DEFAULT(IntervalArray);
+ARRAY_VISITOR_DEFAULT(ListArray);
+ARRAY_VISITOR_DEFAULT(StructArray);
+ARRAY_VISITOR_DEFAULT(UnionArray);
+ARRAY_VISITOR_DEFAULT(DictionaryArray);
+
+Status ArrayVisitor::Visit(const DecimalArray& array) {
+  return Status::NotImplemented("decimal");
+}
+
+// ----------------------------------------------------------------------
+// Default implementations of TypeVisitor methods
+
+#define TYPE_VISITOR_DEFAULT(TYPE_CLASS)              \
+  Status TypeVisitor::Visit(const TYPE_CLASS& type) { \
+    return Status::NotImplemented(type.ToString());   \
+  }
+
+TYPE_VISITOR_DEFAULT(NullType);
+TYPE_VISITOR_DEFAULT(BooleanType);
+TYPE_VISITOR_DEFAULT(Int8Type);
+TYPE_VISITOR_DEFAULT(Int16Type);
+TYPE_VISITOR_DEFAULT(Int32Type);
+TYPE_VISITOR_DEFAULT(Int64Type);
+TYPE_VISITOR_DEFAULT(UInt8Type);
+TYPE_VISITOR_DEFAULT(UInt16Type);
+TYPE_VISITOR_DEFAULT(UInt32Type);
+TYPE_VISITOR_DEFAULT(UInt64Type);
+TYPE_VISITOR_DEFAULT(HalfFloatType);
+TYPE_VISITOR_DEFAULT(FloatType);
+TYPE_VISITOR_DEFAULT(DoubleType);
+TYPE_VISITOR_DEFAULT(StringType);
+TYPE_VISITOR_DEFAULT(BinaryType);
+TYPE_VISITOR_DEFAULT(FixedWidthBinaryType);
+TYPE_VISITOR_DEFAULT(Date64Type);
+TYPE_VISITOR_DEFAULT(Date32Type);
+TYPE_VISITOR_DEFAULT(TimeType);
+TYPE_VISITOR_DEFAULT(TimestampType);
+TYPE_VISITOR_DEFAULT(IntervalType);
+TYPE_VISITOR_DEFAULT(DecimalType);
+TYPE_VISITOR_DEFAULT(ListType);
+TYPE_VISITOR_DEFAULT(StructType);
+TYPE_VISITOR_DEFAULT(UnionType);
+TYPE_VISITOR_DEFAULT(DictionaryType);
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h
new file mode 100644
index 0000000000000..a9c59c8f762fe
--- /dev/null
+++ b/cpp/src/arrow/visitor.h
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_VISITOR_H
+#define ARROW_VISITOR_H
+
+#include "arrow/status.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class ARROW_EXPORT ArrayVisitor {
+ public:
+  virtual ~ArrayVisitor() = default;
+
+  virtual Status Visit(const NullArray& array);
+  virtual Status Visit(const BooleanArray& array);
+  virtual Status Visit(const Int8Array& array);
+  virtual Status Visit(const Int16Array& array);
+  virtual Status Visit(const Int32Array& array);
+  virtual Status Visit(const Int64Array& array);
+  virtual Status Visit(const UInt8Array& array);
+  virtual Status Visit(const UInt16Array& array);
+  virtual Status Visit(const UInt32Array& array);
+  virtual Status Visit(const UInt64Array& array);
+  virtual Status Visit(const HalfFloatArray& array);
+  virtual Status Visit(const FloatArray& array);
+  virtual Status Visit(const DoubleArray& array);
+  virtual Status Visit(const StringArray& array);
+  virtual Status Visit(const BinaryArray& array);
+  virtual Status Visit(const FixedWidthBinaryArray& array);
+  virtual Status Visit(const Date32Array& array);
+  virtual Status Visit(const Date64Array& array);
+  virtual Status Visit(const TimeArray& array);
+  virtual Status Visit(const TimestampArray& array);
+  virtual Status Visit(const IntervalArray& array);
+  virtual Status Visit(const DecimalArray& array);
+  virtual Status Visit(const ListArray& array);
+  virtual Status Visit(const StructArray& array);
+  virtual Status Visit(const UnionArray& array);
+  virtual Status Visit(const DictionaryArray& type);
+};
+
+class ARROW_EXPORT TypeVisitor {
+ public:
+  virtual ~TypeVisitor() = default;
+
+  virtual Status Visit(const NullType& type);
+  virtual Status Visit(const BooleanType& type);
+  virtual Status Visit(const Int8Type& type);
+  virtual Status Visit(const Int16Type& type);
+  virtual Status Visit(const Int32Type& type);
+  virtual Status Visit(const Int64Type& type);
+  virtual Status Visit(const UInt8Type& type);
+  virtual Status Visit(const UInt16Type& type);
+  virtual Status Visit(const UInt32Type& type);
+  virtual Status Visit(const UInt64Type& type);
+  virtual Status Visit(const HalfFloatType& type);
+  virtual Status Visit(const FloatType& type);
+  virtual Status Visit(const DoubleType& type);
+  virtual Status Visit(const StringType& type);
+  virtual Status Visit(const BinaryType& type);
+  virtual Status Visit(const FixedWidthBinaryType& type);
+  virtual Status Visit(const Date64Type& type);
+  virtual Status Visit(const Date32Type& type);
+  virtual Status Visit(const TimeType& type);
+  virtual Status Visit(const TimestampType& type);
+  virtual Status Visit(const IntervalType& type);
+  virtual Status Visit(const DecimalType& type);
+  virtual Status Visit(const ListType& type);
+  virtual Status Visit(const StructType& type);
+  virtual Status Visit(const UnionType& type);
+  virtual Status Visit(const DictionaryType& type);
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_VISITOR_H
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
new file mode 100644
index 0000000000000..b69468d17eebe
--- /dev/null
+++ b/cpp/src/arrow/visitor_inline.h
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Private header, not to be exported
+
+#ifndef ARROW_VISITOR_INLINE_H
+#define ARROW_VISITOR_INLINE_H
+
+#include "arrow/array.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+#define TYPE_VISIT_INLINE(TYPE_CLASS) \
+  case TYPE_CLASS::type_id:           \
+    return visitor->Visit(static_cast<const TYPE_CLASS&>(type));
+
+template <typename VISITOR>
+inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
+  switch (type.type) {
+    TYPE_VISIT_INLINE(NullType);
+    TYPE_VISIT_INLINE(BooleanType);
+    TYPE_VISIT_INLINE(Int8Type);
+    TYPE_VISIT_INLINE(UInt8Type);
+    TYPE_VISIT_INLINE(Int16Type);
+    TYPE_VISIT_INLINE(UInt16Type);
+    TYPE_VISIT_INLINE(Int32Type);
+    TYPE_VISIT_INLINE(UInt32Type);
+    TYPE_VISIT_INLINE(Int64Type);
+    TYPE_VISIT_INLINE(UInt64Type);
+    TYPE_VISIT_INLINE(FloatType);
+    TYPE_VISIT_INLINE(DoubleType);
+    TYPE_VISIT_INLINE(StringType);
+    TYPE_VISIT_INLINE(BinaryType);
+    TYPE_VISIT_INLINE(FixedWidthBinaryType);
+    TYPE_VISIT_INLINE(Date32Type);
+    TYPE_VISIT_INLINE(Date64Type);
+    TYPE_VISIT_INLINE(TimestampType);
+    TYPE_VISIT_INLINE(TimeType);
+    TYPE_VISIT_INLINE(ListType);
+    TYPE_VISIT_INLINE(StructType);
+    TYPE_VISIT_INLINE(UnionType);
+    TYPE_VISIT_INLINE(DictionaryType);
+    default:
+      break;
+  }
+  return Status::NotImplemented("Type not implemented");
+}
+
+}  // namespace arrow
+
+#endif  // ARROW_VISITOR_INLINE_H

From dcaa8e5d7ef1353c657e016bf271495042825a91 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Thu, 23 Mar 2017 12:44:27 -0700
Subject: [PATCH 0411/1644] ARROW-702: fix BitVector.copyFromSafe to reAllocate
 instead of returning false

Author: Julien Le Dem <julien@dremio.com>

Closes #426 from julienledem/arrow_702 and squashes the following commits:

4c77b95 [Julien Le Dem] add license
7ab84aa [Julien Le Dem] Thanks Hakim for the test case
ba8aa8e [Julien Le Dem] ARROW-702: fix BitVector.copyFromSafe to reAllocate instead of returning false
---
 .../arrow/memory/TestBaseAllocator.java       |  2 +-
 .../org/apache/arrow/vector/BitVector.java    |  8 +--
 .../apache/arrow/vector/TestBitVector.java    | 66 +++++++++++++++++++
 3 files changed, 70 insertions(+), 6 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java

diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index 3c96d57f4e64d..59b7be87e17be 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -381,7 +381,7 @@ public void testAllocator_sliceRanges() throws Exception {
         assertEquals((byte) i, slice1.getByte(i));
       }
 
-      final ArrowBuf slice2 = (ArrowBuf) arrowBuf.slice(25, 25);
+      final ArrowBuf slice2 = arrowBuf.slice(25, 25);
       assertEquals(0, slice2.readerIndex());
       assertEquals(25, slice2.readableBytes());
       for(int i = 25; i < 50; ++i) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 179f2ee879f43..ed574333beacd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -216,13 +216,11 @@ public void copyFrom(int inIndex, int outIndex, BitVector from) {
     this.mutator.set(outIndex, from.accessor.get(inIndex));
   }
 
-  public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) {
+  public void copyFromSafe(int inIndex, int outIndex, BitVector from) {
     if (outIndex >= this.getValueCapacity()) {
-      decrementAllocationMonitor();
-      return false;
+      reAlloc();
     }
     copyFrom(inIndex, outIndex, from);
-    return true;
   }
 
   @Override
@@ -273,7 +271,7 @@ public void splitAndTransferTo(int startIndex, int length, BitVector target) {
       if (target.data != null) {
         target.data.release();
       }
-      target.data = (ArrowBuf) data.slice(firstByte, byteSize);
+      target.data = data.slice(firstByte, byteSize);
       target.data.retain(1);
     } else {
       // Copy data
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
new file mode 100644
index 0000000000000..f2343c88e70a5
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestBitVector {
+  private final static String EMPTY_SCHEMA_PATH = "";
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testBitVectorCopyFromSafe() {
+    final int size = 20;
+    try (final BitVector src = new BitVector(EMPTY_SCHEMA_PATH, allocator);
+         final BitVector dst = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
+      src.allocateNew(size);
+      dst.allocateNew(10);
+
+      for (int i = 0; i < size; i++) {
+        src.getMutator().set(i, i % 2);
+      }
+      src.getMutator().setValueCount(size);
+
+      for (int i = 0; i < size; i++) {
+        dst.copyFromSafe(i, i, src);
+      }
+      dst.getMutator().setValueCount(size);
+
+      for (int i = 0; i < size; i++) {
+        assertEquals(src.getAccessor().getObject(i), dst.getAccessor().getObject(i));
+      }
+    }
+  }
+
+}

From 13c12c6ea5e23928268b5c2c7b962d223cca7bd4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 24 Mar 2017 11:54:18 +0100
Subject: [PATCH 0412/1644] ARROW-682: [Integration] Check implementations
 against themselves

This adds an additional layer of internal consistency checks

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #433 from wesm/ARROW-682 and squashes the following commits:

b33ac7a [Wes McKinney] Run integration tests with same implementation producing and consuming to validate internal consistency
---
 integration/integration_test.py | 56 +++++++++++++++++----------------
 1 file changed, 29 insertions(+), 27 deletions(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 5cd63c502bd20..ec2a38d840d0b 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -34,10 +34,12 @@
 # Control for flakiness
 np.random.seed(12345)
 
+
 def load_version_from_pom():
     import xml.etree.ElementTree as ET
     tree = ET.parse(os.path.join(ARROW_HOME, 'java', 'pom.xml'))
-    version_tag = list(tree.getroot().findall('{http://maven.apache.org/POM/4.0.0}version'))[0]
+    tag_pattern = '{http://maven.apache.org/POM/4.0.0}version'
+    version_tag = list(tree.getroot().findall(tag_pattern))[0]
     return version_tag.text
 
 
@@ -596,32 +598,32 @@ def __init__(self, json_files, testers, debug=False):
     def run(self):
         for producer, consumer in itertools.product(self.testers,
                                                     self.testers):
-            if producer is consumer:
-                continue
-
-            print('-- {0} producing, {1} consuming'.format(producer.name,
-                                                           consumer.name))
-
-            for json_path in self.json_files:
-                print('Testing file {0}'.format(json_path))
-
-                # Make the random access file
-                print('-- Creating binary inputs')
-                producer_file_path = os.path.join(self.temp_dir, guid())
-                producer.json_to_file(json_path, producer_file_path)
-
-                # Validate the file
-                print('-- Validating file')
-                consumer.validate(json_path, producer_file_path)
-
-                print('-- Validating stream')
-                producer_stream_path = os.path.join(self.temp_dir, guid())
-                consumer_file_path = os.path.join(self.temp_dir, guid())
-                producer.file_to_stream(producer_file_path,
-                                        producer_stream_path)
-                consumer.stream_to_file(producer_stream_path,
-                                        consumer_file_path)
-                consumer.validate(json_path, consumer_file_path)
+            self._compare_implementations(producer, consumer)
+
+    def _compare_implementations(self, producer, consumer):
+        print('-- {0} producing, {1} consuming'.format(producer.name,
+                                                       consumer.name))
+
+        for json_path in self.json_files:
+            print('Testing file {0}'.format(json_path))
+
+            # Make the random access file
+            print('-- Creating binary inputs')
+            producer_file_path = os.path.join(self.temp_dir, guid())
+            producer.json_to_file(json_path, producer_file_path)
+
+            # Validate the file
+            print('-- Validating file')
+            consumer.validate(json_path, producer_file_path)
+
+            print('-- Validating stream')
+            producer_stream_path = os.path.join(self.temp_dir, guid())
+            consumer_file_path = os.path.join(self.temp_dir, guid())
+            producer.file_to_stream(producer_file_path,
+                                    producer_stream_path)
+            consumer.stream_to_file(producer_stream_path,
+                                    consumer_file_path)
+            consumer.validate(json_path, consumer_file_path)
 
 
 class Tester(object):

From bc185a41a239181d255e72bf255a354da4f5dae6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 24 Mar 2017 11:58:30 +0100
Subject: [PATCH 0413/1644] ARROW-595: [Python] Set schema attribute on
 StreamReader

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #434 from wesm/ARROW-595 and squashes the following commits:

484cc7b [Wes McKinney] Set schema attribute on StreamReader
---
 python/pyarrow/io.pyx            | 4 ++--
 python/pyarrow/tests/test_ipc.py | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 17b43dedb0a5f..72e0e0ff01512 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -933,8 +933,8 @@ cdef class _StreamReader:
         with nogil:
             check_status(CStreamReader.Open(in_stream, &self.reader))
 
-        schema = Schema()
-        schema.init_schema(self.reader.get().schema())
+        self.schema = Schema()
+        self.schema.init_schema(self.reader.get().schema())
 
     def get_next_batch(self):
         """
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 665a63b6d5a38..4c9dad1b840a8 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -104,6 +104,8 @@ def test_simple_roundtrip(self):
         file_contents = self._get_source()
         reader = pa.StreamReader(file_contents)
 
+        assert reader.schema.equals(batches[0].schema)
+
         total = 0
         for i, next_batch in enumerate(reader):
             assert next_batch.equals(batches[i])

From 016a209815465c3161ac357a316efa55061da983 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 24 Mar 2017 11:30:27 -0400
Subject: [PATCH 0414/1644] ARROW-706: [GLib] Add package install document

Author: Kouhei Sutou <kou@clear-code.com>

Closes #436 from kou/glib-add-package-install and squashes the following commits:

fa8dc04 [Kouhei Sutou] [GLib] Add a note about "unofficial"
d23c34d [Kouhei Sutou] [GLib] Add package install document
---
 c_glib/README.md | 70 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 69 insertions(+), 1 deletion(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index 84027bf2cb3db..95cc9a65c5bd8 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -42,9 +42,77 @@ gobject-introspection gem based bindings.
 
 ## Install
 
+You can use packages or build by yourself to install Arrow GLib. It's
+recommended that you use packages.
+
+Note that the packages are "unofficial". "Official" packages will be
+released in the future.
+
 ### Package
 
-TODO
+There are supported platforms:
+
+  * Debian GNU/Linux Jessie
+  * Ubuntu 16.04 LTS
+  * Ubuntu 16.10
+  * CentOS 7
+
+You can feedback to https://github.com/kou/arrow-packages about
+packages things.
+
+#### Debian GNU/Linux jessie
+
+You need to add the following apt-lines to
+`/etc/apt/sources.list.d/groonga.list`:
+
+```text
+deb http://packages.groonga.org/debian/ jessie main
+deb-src http://packages.groonga.org/debian/ jessie main
+```
+
+Then you need to run the following command lines:
+
+```text
+% sudo apt update
+% sudo apt install -y --allow-unauthenticated groonga-keyring
+% sudo apt update
+```
+
+Now you can install Arrow GLib packages:
+
+```text
+% sudo apt install -y libarrow-glib-dev
+```
+
+#### Ubuntu 16.04 LTS and Ubuntu 16.10
+
+You need to add an APT repository:
+
+```text
+% sudo apt install -y software-properties-common
+% sudo add-apt-repository -y ppa:groonga/ppa
+% sudo apt update
+```
+
+Now you can install Arrow GLib packages:
+
+```text
+% sudo apt install -y libarrow-glib-dev
+```
+
+#### CentOS 7
+
+You need to add a Yum repository:
+
+```text
+% sudo yum install -y http://packages.groonga.org/centos/groonga-release-1.2.0-1.noarch.rpm
+```
+
+Now you can install Arrow GLib packages:
+
+```text
+% sudo yum install -y --enablerepo=epel arrow-glib-devel
+```
 
 ### Build
 

From dc3cb30b90ca818cdd053dacdc9badb7cfc9214a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 24 Mar 2017 11:53:42 -0400
Subject: [PATCH 0415/1644] ARROW-550: [Format] Draft experimental Tensor
 flatbuffer message type

Tensor-like data occurs very frequently in scientific computing and machine learning applications that are mostly implemented in C and C++. Arrow's C++ memory management and shared memory utilities can help serve these use cases for zero-copy data transfer to other tensor-like data structures (like NumPy ndarrays, or the tensor objects used in machine learning libraries like TensorFlow or Torch).

The Tensor data structure is loosely modeled after NumPy's ndarray object and TensorFlow's tensor protocol buffers type (https://github.com/tensorflow/tensorflow/blob/754048a0453a04a761e112ae5d99c149eb9910dd/tensorflow/core/framework/tensor.proto).

cc @pcmoritz @robertnishihara @sylvaincorlay @JohanMabille

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #435 from wesm/ARROW-550 and squashes the following commits:

afac56e [Wes McKinney] Change TensorOrder enum to byte
249a9d5 [Wes McKinney] Replace strides with TensorOrder enum for row major / column major
d7d6407 [Wes McKinney] Draft Tensor flatbuffer type
---
 cpp/src/arrow/ipc/CMakeLists.txt |  1 +
 format/Message.fbs               |  3 +-
 format/Tensor.fbs                | 60 ++++++++++++++++++++++++++++++++
 java/format/pom.xml              |  3 +-
 4 files changed, 65 insertions(+), 2 deletions(-)
 create mode 100644 format/Tensor.fbs

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 056e7dba53830..d6ee9309b44d8 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -114,6 +114,7 @@ set(FBS_SRC
   ${CMAKE_SOURCE_DIR}/../format/Message.fbs
   ${CMAKE_SOURCE_DIR}/../format/File.fbs
   ${CMAKE_SOURCE_DIR}/../format/Schema.fbs
+  ${CMAKE_SOURCE_DIR}/../format/Tensor.fbs
   ${CMAKE_CURRENT_SOURCE_DIR}/feather.fbs)
 
 foreach(FIL ${FBS_SRC})
diff --git a/format/Message.fbs b/format/Message.fbs
index 2cb60953c6a79..f4a95713cea93 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -16,6 +16,7 @@
 // under the License.
 
 include "Schema.fbs";
+include "Tensor.fbs";
 
 namespace org.apache.arrow.flatbuf;
 
@@ -82,7 +83,7 @@ table DictionaryBatch {
 /// which may include experimental metadata types. For maximum compatibility,
 /// it is best to send data using RecordBatch
 union MessageHeader {
-  Schema, DictionaryBatch, RecordBatch
+  Schema, DictionaryBatch, RecordBatch, Tensor
 }
 
 table Message {
diff --git a/format/Tensor.fbs b/format/Tensor.fbs
new file mode 100644
index 0000000000000..bc5b6d1289b2f
--- /dev/null
+++ b/format/Tensor.fbs
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// EXPERIMENTAL: Metadata for n-dimensional arrays, aka "tensors" or
+/// "ndarrays". Arrow implementations in general are not required to implement
+/// this type
+
+include "Schema.fbs";
+
+namespace org.apache.arrow.flatbuf;
+
+/// Shape data for a single axis in a tensor
+table TensorDim {
+  /// Length of dimension
+  size: long;
+
+  /// Name of the dimension, optional
+  name: string;
+}
+
+enum TensorOrder : byte {
+  /// Higher dimensions vary first when traversing data in byte-contiguous
+  /// order, aka "C order"
+  ROW_MAJOR,
+
+  /// Lower dimensions vary first when traversing data in byte-contiguous
+  /// order, aka "Fortran order"
+  COLUMN_MAJOR
+}
+
+table Tensor {
+  /// The type of data contained in a value cell. Currently only fixed-width
+  /// value types are supported, no strings or nested types
+  type: Type;
+
+  /// The dimensions of the tensor, optionally named
+  shape: [TensorDim];
+
+  /// The memory order of the tensor's data
+  order: TensorOrder;
+
+  /// The location and size of the tensor's data
+  data: Buffer;
+}
+
+root_type Tensor;
diff --git a/java/format/pom.xml b/java/format/pom.xml
index e7a58a4172fe2..98a113a30cf78 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -110,8 +110,9 @@
               <argument>-o</argument>
               <argument>${flatc.generated.files}</argument>
               <argument>../../format/Schema.fbs</argument>
-              <argument>../../format/Message.fbs</argument>
+              <argument>../../format/Tensor.fbs</argument>
               <argument>../../format/File.fbs</argument>
+              <argument>../../format/Message.fbs</argument>
             </arguments>
           </configuration>
         </execution>

From 5ad498833fe6cd5519b8d652d4bf620add5a7eed Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 24 Mar 2017 14:10:42 -0400
Subject: [PATCH 0416/1644] ARROW-708: [C++] Simplify metadata APIs to all use
 the Message class, perf analysis

This doesn't produce a meaningful perf improvement, but it does remove a fair amount of code which is nice.

Here is an interactive FlameGraph SVG: https://www.dropbox.com/s/kp8i5r3j7i0em02/ipc-perf-20170324.svg?dl=0

![screenshot from 2017-03-24 12 52 54](https://cloud.githubusercontent.com/assets/329591/24304760/f283960a-1090-11e7-9bc5-4cb26f7ca0ae.png)

So it appears that out of the few hundred nanoseconds spent constructing each Array object, the time is mostly spent in object constructors. One thing that shows up is the RecordBatch constructor which is spending a bunch of time copying the `vector<shared_ptr<Array>>` passed, so I added a move constructor.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #437 from wesm/record-batch-read-perf and squashes the following commits:

95fdbc7 [Wes McKinney] Add RecordBatch constructor with rvalue-reference for the columns
793b3be [Wes McKinney] Inline SliceBuffer
212f17f [Wes McKinney] Benchmark in nanoseconds
a295aae [Wes McKinney] Remove record batch / dictionary PIMPL interfaces, handle flatbuffer details internally
---
 cpp/src/arrow/buffer.cc                       |   7 -
 cpp/src/arrow/buffer.h                        |   6 +-
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc |   2 -
 cpp/src/arrow/ipc/ipc-read-write-test.cc      |   9 +-
 cpp/src/arrow/ipc/metadata.cc                 | 123 +-----------------
 cpp/src/arrow/ipc/metadata.h                  |  41 +-----
 cpp/src/arrow/ipc/reader.cc                   |  85 +++++++-----
 cpp/src/arrow/ipc/reader.h                    |  16 +--
 cpp/src/arrow/table.cc                        |   4 +
 cpp/src/arrow/table.h                         |   3 +
 10 files changed, 76 insertions(+), 220 deletions(-)

diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index a0b78ac0b9f20..28edf5e824c1f 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -68,13 +68,6 @@ bool Buffer::Equals(const Buffer& other) const {
                                                             static_cast<size_t>(size_))));
 }
 
-std::shared_ptr<Buffer> SliceBuffer(
-    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
-  DCHECK_LE(offset, buffer->size());
-  DCHECK_LE(length, buffer->size() - offset);
-  return std::make_shared<Buffer>(buffer, offset, length);
-}
-
 std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
   return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
 }
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 70c16a2dafc86..449bb537d9caa 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -96,8 +96,10 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
 
 /// Construct a view on passed buffer at the indicated offset and length. This
 /// function cannot fail and does not error checking (except in debug builds)
-std::shared_ptr<Buffer> ARROW_EXPORT SliceBuffer(
-    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
+static inline std::shared_ptr<Buffer> SliceBuffer(
+    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
+  return std::make_shared<Buffer>(buffer, offset, length);
+}
 
 /// A Buffer whose contents can be mutated. May or may not own its data.
 class ARROW_EXPORT MutableBuffer : public Buffer {
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
index e27e5136a0d5a..1aecdbc633190 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -121,14 +121,12 @@ BENCHMARK(BM_WriteRecordBatch)
     ->RangeMultiplier(4)
     ->Range(1, 1 << 13)
     ->MinTime(1.0)
-    ->Unit(benchmark::kMicrosecond)
     ->UseRealTime();
 
 BENCHMARK(BM_ReadRecordBatch)
     ->RangeMultiplier(4)
     ->Range(1, 1 << 13)
     ->MinTime(1.0)
-    ->Unit(benchmark::kMicrosecond)
     ->UseRealTime();
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 6919aebbe8d6d..086cc68176783 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -140,7 +140,6 @@ class IpcTestFixture : public io::MemoryMapFixture {
 
     std::shared_ptr<Message> message;
     RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
-    auto metadata = std::make_shared<RecordBatchMetadata>(message);
 
     // The buffer offsets start at 0, so we must construct a
     // RandomAccessFile according to that frame of reference
@@ -148,7 +147,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
     RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
     io::BufferReader buffer_reader(buffer_payload);
 
-    return ReadRecordBatch(*metadata, batch.schema(), &buffer_reader, batch_result);
+    return ReadRecordBatch(*message, batch.schema(), &buffer_reader, batch_result);
   }
 
   Status DoLargeRoundTrip(
@@ -370,7 +369,6 @@ TEST_F(RecursionLimits, ReadLimit) {
 
   std::shared_ptr<Message> message;
   ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
-  auto metadata = std::make_shared<RecordBatchMetadata>(message);
 
   std::shared_ptr<Buffer> payload;
   ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
@@ -378,7 +376,7 @@ TEST_F(RecursionLimits, ReadLimit) {
   io::BufferReader reader(payload);
 
   std::shared_ptr<RecordBatch> result;
-  ASSERT_RAISES(Invalid, ReadRecordBatch(*metadata, schema, &reader, &result));
+  ASSERT_RAISES(Invalid, ReadRecordBatch(*message, schema, &reader, &result));
 }
 
 TEST_F(RecursionLimits, StressLimit) {
@@ -392,7 +390,6 @@ TEST_F(RecursionLimits, StressLimit) {
 
     std::shared_ptr<Message> message;
     ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
-    auto metadata = std::make_shared<RecordBatchMetadata>(message);
 
     std::shared_ptr<Buffer> payload;
     ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
@@ -400,7 +397,7 @@ TEST_F(RecursionLimits, StressLimit) {
     io::BufferReader reader(payload);
 
     std::shared_ptr<RecordBatch> result;
-    ASSERT_OK(ReadRecordBatch(*metadata, schema, recursion_depth + 1, &reader, &result));
+    ASSERT_OK(ReadRecordBatch(*message, schema, recursion_depth + 1, &reader, &result));
     *it_works = result->Equals(*batch);
   };
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index b10ccec9e7c4e..14cb627982343 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -770,6 +770,10 @@ int64_t Message::body_length() const {
   return impl_->body_length();
 }
 
+const void* Message::header() const {
+  return impl_->header();
+}
+
 // ----------------------------------------------------------------------
 // SchemaMetadata
 
@@ -858,125 +862,6 @@ Status SchemaMetadata::GetSchema(
   return Status::OK();
 }
 
-// ----------------------------------------------------------------------
-// RecordBatchMetadata
-
-class RecordBatchMetadata::RecordBatchMetadataImpl : public MessageHolder {
- public:
-  explicit RecordBatchMetadataImpl(const void* batch)
-      : batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
-    nodes_ = batch_->nodes();
-    buffers_ = batch_->buffers();
-  }
-
-  const flatbuf::FieldNode* field(int i) const { return nodes_->Get(i); }
-
-  const flatbuf::Buffer* buffer(int i) const { return buffers_->Get(i); }
-
-  int64_t length() const { return batch_->length(); }
-
-  int num_buffers() const { return batch_->buffers()->size(); }
-
-  int num_fields() const { return batch_->nodes()->size(); }
-
- private:
-  const flatbuf::RecordBatch* batch_;
-  const flatbuffers::Vector<const flatbuf::FieldNode*>* nodes_;
-  const flatbuffers::Vector<const flatbuf::Buffer*>* buffers_;
-};
-
-RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message)
-    : RecordBatchMetadata(message->impl_->header()) {
-  impl_->set_message(message);
-}
-
-RecordBatchMetadata::RecordBatchMetadata(const void* header) {
-  impl_.reset(new RecordBatchMetadataImpl(header));
-}
-
-RecordBatchMetadata::RecordBatchMetadata(
-    const std::shared_ptr<Buffer>& buffer, int64_t offset)
-    : RecordBatchMetadata(buffer->data() + offset) {
-  // Preserve ownership
-  impl_->set_buffer(buffer);
-}
-
-RecordBatchMetadata::~RecordBatchMetadata() {}
-
-// TODO(wesm): Copying the flatbuffer data isn't great, but this will do for
-// now
-FieldMetadata RecordBatchMetadata::field(int i) const {
-  const flatbuf::FieldNode* node = impl_->field(i);
-
-  FieldMetadata result;
-  result.length = node->length();
-  result.null_count = node->null_count();
-  result.offset = 0;
-  return result;
-}
-
-BufferMetadata RecordBatchMetadata::buffer(int i) const {
-  const flatbuf::Buffer* buffer = impl_->buffer(i);
-
-  BufferMetadata result;
-  result.page = buffer->page();
-  result.offset = buffer->offset();
-  result.length = buffer->length();
-  return result;
-}
-
-int64_t RecordBatchMetadata::length() const {
-  return impl_->length();
-}
-
-int RecordBatchMetadata::num_buffers() const {
-  return impl_->num_buffers();
-}
-
-int RecordBatchMetadata::num_fields() const {
-  return impl_->num_fields();
-}
-
-// ----------------------------------------------------------------------
-// DictionaryBatchMetadata
-
-class DictionaryBatchMetadata::DictionaryBatchMetadataImpl {
- public:
-  explicit DictionaryBatchMetadataImpl(const void* dictionary)
-      : metadata_(static_cast<const flatbuf::DictionaryBatch*>(dictionary)) {
-    record_batch_.reset(new RecordBatchMetadata(metadata_->data()));
-  }
-
-  int64_t id() const { return metadata_->id(); }
-  const RecordBatchMetadata& record_batch() const { return *record_batch_; }
-
-  void set_message(const std::shared_ptr<Message>& message) { message_ = message; }
-
- private:
-  const flatbuf::DictionaryBatch* metadata_;
-
-  std::unique_ptr<RecordBatchMetadata> record_batch_;
-
-  // Parent, owns the flatbuffer data
-  std::shared_ptr<Message> message_;
-};
-
-DictionaryBatchMetadata::DictionaryBatchMetadata(
-    const std::shared_ptr<Message>& message) {
-  impl_.reset(new DictionaryBatchMetadataImpl(message->impl_->header()));
-  impl_->set_message(message);
-}
-
-DictionaryBatchMetadata::~DictionaryBatchMetadata() {}
-
-int64_t DictionaryBatchMetadata::id() const {
-  return impl_->id();
-}
-
-const RecordBatchMetadata& DictionaryBatchMetadata::record_batch() const {
-  return impl_->record_batch();
-}
-
 // ----------------------------------------------------------------------
 // Conveniences
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index dc07c7a1bd9b7..6e903c0a18ef6 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -138,44 +138,6 @@ struct ARROW_EXPORT BufferMetadata {
   int64_t length;
 };
 
-// Container for serialized record batch metadata contained in an IPC message
-class ARROW_EXPORT RecordBatchMetadata {
- public:
-  explicit RecordBatchMetadata(const void* header);
-  explicit RecordBatchMetadata(const std::shared_ptr<Message>& message);
-  RecordBatchMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
-
-  ~RecordBatchMetadata();
-
-  FieldMetadata field(int i) const;
-  BufferMetadata buffer(int i) const;
-
-  int64_t length() const;
-  int num_buffers() const;
-  int num_fields() const;
-
- private:
-  class RecordBatchMetadataImpl;
-  std::unique_ptr<RecordBatchMetadataImpl> impl_;
-
-  DISALLOW_COPY_AND_ASSIGN(RecordBatchMetadata);
-};
-
-class ARROW_EXPORT DictionaryBatchMetadata {
- public:
-  explicit DictionaryBatchMetadata(const std::shared_ptr<Message>& message);
-  ~DictionaryBatchMetadata();
-
-  int64_t id() const;
-  const RecordBatchMetadata& record_batch() const;
-
- private:
-  class DictionaryBatchMetadataImpl;
-  std::unique_ptr<DictionaryBatchMetadataImpl> impl_;
-
-  DISALLOW_COPY_AND_ASSIGN(DictionaryBatchMetadata);
-};
-
 class ARROW_EXPORT Message {
  public:
   enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH };
@@ -187,11 +149,12 @@ class ARROW_EXPORT Message {
 
   Type type() const;
 
+  const void* header() const;
+
  private:
   Message(const std::shared_ptr<Buffer>& buffer, int64_t offset);
 
   friend class DictionaryBatchMetadata;
-  friend class RecordBatchMetadata;
   friend class SchemaMetadata;
 
   // Hide serialization details from user API
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 71ba951111999..83e03aa0b36b4 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -46,36 +46,41 @@ namespace ipc {
 
 class IpcComponentSource : public ArrayComponentSource {
  public:
-  IpcComponentSource(const RecordBatchMetadata& metadata, io::RandomAccessFile* file)
+  IpcComponentSource(const flatbuf::RecordBatch* metadata, io::RandomAccessFile* file)
       : metadata_(metadata), file_(file) {}
 
   Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
-    BufferMetadata buffer_meta = metadata_.buffer(buffer_index);
-    if (buffer_meta.length == 0) {
+    const flatbuf::Buffer* buffer = metadata_->buffers()->Get(buffer_index);
+
+    if (buffer->length() == 0) {
       *out = nullptr;
       return Status::OK();
     } else {
-      return file_->ReadAt(buffer_meta.offset, buffer_meta.length, out);
+      return file_->ReadAt(buffer->offset(), buffer->length(), out);
     }
   }
 
-  Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
+  Status GetFieldMetadata(int field_index, FieldMetadata* field) override {
+    auto nodes = metadata_->nodes();
     // pop off a field
-    if (field_index >= metadata_.num_fields()) {
+    if (field_index >= static_cast<int>(nodes->size())) {
       return Status::Invalid("Ran out of field metadata, likely malformed");
     }
-    *metadata = metadata_.field(field_index);
+    const flatbuf::FieldNode* node = nodes->Get(field_index);
+
+    field->length = node->length();
+    field->null_count = node->null_count();
+    field->offset = 0;
     return Status::OK();
   }
 
  private:
-  const RecordBatchMetadata& metadata_;
+  const flatbuf::RecordBatch* metadata_;
   io::RandomAccessFile* file_;
 };
 
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
-    std::shared_ptr<RecordBatch>* out) {
+Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
   return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
 }
 
@@ -94,22 +99,32 @@ static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
     RETURN_NOT_OK(LoadArray(schema->field(i)->type, &context, &arrays[i]));
   }
 
-  *out = std::make_shared<RecordBatch>(schema, num_rows, arrays);
+  *out = std::make_shared<RecordBatch>(schema, num_rows, std::move(arrays));
   return Status::OK();
 }
 
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+static inline Status ReadRecordBatch(const flatbuf::RecordBatch* metadata,
     const std::shared_ptr<Schema>& schema, int max_recursion_depth,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
   IpcComponentSource source(metadata, file);
   return LoadRecordBatchFromSource(
-      schema, metadata.length(), max_recursion_depth, &source, out);
+      schema, metadata->length(), max_recursion_depth, &source, out);
 }
 
-Status ReadDictionary(const DictionaryBatchMetadata& metadata,
-    const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
-    std::shared_ptr<Array>* out) {
-  int64_t id = metadata.id();
+Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
+    int max_recursion_depth, io::RandomAccessFile* file,
+    std::shared_ptr<RecordBatch>* out) {
+  DCHECK_EQ(metadata.type(), Message::RECORD_BATCH);
+  auto batch = reinterpret_cast<const flatbuf::RecordBatch*>(metadata.header());
+  return ReadRecordBatch(batch, schema, max_recursion_depth, file, out);
+}
+
+Status ReadDictionary(const Message& metadata, const DictionaryTypeMap& dictionary_types,
+    io::RandomAccessFile* file, int64_t* dictionary_id, std::shared_ptr<Array>* out) {
+  auto dictionary_batch =
+      reinterpret_cast<const flatbuf::DictionaryBatch*>(metadata.header());
+
+  int64_t id = *dictionary_id = dictionary_batch->id();
   auto it = dictionary_types.find(id);
   if (it == dictionary_types.end()) {
     std::stringstream ss;
@@ -124,7 +139,10 @@ Status ReadDictionary(const DictionaryBatchMetadata& metadata,
 
   // The dictionary is embedded in a record batch with a single column
   std::shared_ptr<RecordBatch> batch;
-  RETURN_NOT_OK(ReadRecordBatch(metadata.record_batch(), dummy_schema, file, &batch));
+  auto batch_meta =
+      reinterpret_cast<const flatbuf::RecordBatch*>(dictionary_batch->data());
+  RETURN_NOT_OK(
+      ReadRecordBatch(batch_meta, dummy_schema, kMaxNestingDepth, file, &batch));
 
   if (batch->num_columns() != 1) {
     return Status::Invalid("Dictionary record batch must only contain one field");
@@ -211,15 +229,14 @@ class StreamReader::StreamReaderImpl {
     std::shared_ptr<Message> message;
     RETURN_NOT_OK(ReadNextMessage(Message::DICTIONARY_BATCH, &message));
 
-    DictionaryBatchMetadata metadata(message);
-
     std::shared_ptr<Buffer> batch_body;
     RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body))
     io::BufferReader reader(batch_body);
 
     std::shared_ptr<Array> dictionary;
-    RETURN_NOT_OK(ReadDictionary(metadata, dictionary_types_, &reader, &dictionary));
-    return dictionary_memo_.AddDictionary(metadata.id(), dictionary);
+    int64_t id;
+    RETURN_NOT_OK(ReadDictionary(*message, dictionary_types_, &reader, &id, &dictionary));
+    return dictionary_memo_.AddDictionary(id, dictionary);
   }
 
   Status ReadSchema() {
@@ -249,12 +266,10 @@ class StreamReader::StreamReaderImpl {
       return Status::OK();
     }
 
-    RecordBatchMetadata batch_metadata(message);
-
     std::shared_ptr<Buffer> batch_body;
     RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body));
     io::BufferReader reader(batch_body);
-    return ReadRecordBatch(batch_metadata, schema_, &reader, batch);
+    return ReadRecordBatch(*message, schema_, &reader, batch);
   }
 
   std::shared_ptr<Schema> schema() const { return schema_; }
@@ -365,7 +380,6 @@ class FileReader::FileReaderImpl {
     std::shared_ptr<Message> message;
     RETURN_NOT_OK(
         ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
-    auto metadata = std::make_shared<RecordBatchMetadata>(message);
 
     // TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
     // ARROW-384).
@@ -373,7 +387,7 @@ class FileReader::FileReaderImpl {
     RETURN_NOT_OK(file_->Read(block.body_length, &buffer_block));
     io::BufferReader reader(buffer_block);
 
-    return ReadRecordBatch(*metadata, schema_, &reader, batch);
+    return ReadRecordBatch(*message, schema_, &reader, batch);
   }
 
   Status ReadSchema() {
@@ -386,9 +400,8 @@ class FileReader::FileReaderImpl {
       RETURN_NOT_OK(
           ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
 
-      // TODO(wesm): ARROW-577: This code is duplicated, can be fixed with a more
-      // invasive refactor
-      DictionaryBatchMetadata metadata(message);
+      // TODO(wesm): ARROW-577: This code is a bit duplicated, can be fixed
+      // with a more invasive refactor
 
       // TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
       // ARROW-384).
@@ -397,8 +410,10 @@ class FileReader::FileReaderImpl {
       io::BufferReader reader(buffer_block);
 
       std::shared_ptr<Array> dictionary;
-      RETURN_NOT_OK(ReadDictionary(metadata, dictionary_fields_, &reader, &dictionary));
-      RETURN_NOT_OK(dictionary_memo_->AddDictionary(metadata.id(), dictionary));
+      int64_t dictionary_id;
+      RETURN_NOT_OK(ReadDictionary(
+          *message, dictionary_fields_, &reader, &dictionary_id, &dictionary));
+      RETURN_NOT_OK(dictionary_memo_->AddDictionary(dictionary_id, dictionary));
     }
 
     // Get the schema
@@ -480,15 +495,13 @@ Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
   RETURN_NOT_OK(file->Read(flatbuffer_size, &buffer));
   RETURN_NOT_OK(Message::Open(buffer, 0, &message));
 
-  RecordBatchMetadata metadata(message);
-
   // TODO(ARROW-388): The buffer offsets start at 0, so we must construct a
   // RandomAccessFile according to that frame of reference
   std::shared_ptr<Buffer> buffer_payload;
   RETURN_NOT_OK(file->Read(message->body_length(), &buffer_payload));
   io::BufferReader buffer_reader(buffer_payload);
 
-  return ReadRecordBatch(metadata, schema, kMaxNestingDepth, &buffer_reader, out);
+  return ReadRecordBatch(*message, schema, kMaxNestingDepth, &buffer_reader, out);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index ffd0a111d604b..6d9e6ca7b0ab7 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -45,17 +45,15 @@ namespace ipc {
 
 // Generic read functionsh; does not copy data if the input supports zero copy reads
 
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
-    std::shared_ptr<RecordBatch>* out);
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
-    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
-Status ReadDictionary(const DictionaryBatchMetadata& metadata,
-    const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
-    std::shared_ptr<Array>* out);
+Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
+    int max_recursion_depth, io::RandomAccessFile* file,
+    std::shared_ptr<RecordBatch>* out);
+
+Status ReadDictionary(const Message& metadata, const DictionaryTypeMap& dictionary_types,
+    io::RandomAccessFile* file, std::shared_ptr<Array>* out);
 
 class ARROW_EXPORT StreamReader {
  public:
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 6b957c081e502..3f254aae6d3fa 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -33,6 +33,10 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
     const std::vector<std::shared_ptr<Array>>& columns)
     : schema_(schema), num_rows_(num_rows), columns_(columns) {}
 
+RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+    std::vector<std::shared_ptr<Array>>&& columns)
+    : schema_(schema), num_rows_(num_rows), columns_(std::move(columns)) {}
+
 const std::string& RecordBatch::column_name(int i) const {
   return schema_->field(i)->name;
 }
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 68f664b38a365..bf0d99c4e9d2b 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -43,6 +43,9 @@ class ARROW_EXPORT RecordBatch {
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
       const std::vector<std::shared_ptr<Array>>& columns);
 
+  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+      std::vector<std::shared_ptr<Array>>&& columns);
+
   bool Equals(const RecordBatch& other) const;
 
   bool ApproxEquals(const RecordBatch& other) const;

From 60b5832e4c75457e98b5caf7a8622c2201de2cd5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 24 Mar 2017 18:27:16 -0400
Subject: [PATCH 0417/1644] ARROW-686: [C++] Account for time metadata changes,
 add Time32 and Time64 types

This also has a little visitor refactoring

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #432 from wesm/ARROW-686 and squashes the following commits:

300c7f2 [Wes McKinney] Fix glib for time32/64 changes
be4c976 [Wes McKinney] Remove JSON time todo
504059a [Wes McKinney] Remove copy ctors to fix MSVC linker error
ae574ce [Wes McKinney] Some cleaning
cf9783c [Wes McKinney] Add new time types to Python bindings
95f5a05 [Wes McKinney] Implement Time32 and Time64 types, IPC roundtrip
---
 c_glib/arrow-glib/type.cpp            |   6 +-
 c_glib/arrow-glib/type.h              |   6 +-
 cpp/src/arrow/array-decimal-test.cc   |   6 -
 cpp/src/arrow/array-primitive-test.cc |   3 -
 cpp/src/arrow/array.cc                |   3 +-
 cpp/src/arrow/array.h                 |   5 +-
 cpp/src/arrow/builder.cc              |  28 ++---
 cpp/src/arrow/builder.h               |   4 +-
 cpp/src/arrow/compare.cc              |  60 ++++++----
 cpp/src/arrow/ipc/feather-test.cc     |   2 +-
 cpp/src/arrow/ipc/feather.cc          |  14 ++-
 cpp/src/arrow/ipc/ipc-json-test.cc    |   8 +-
 cpp/src/arrow/ipc/json-internal.cc    |  47 ++++++--
 cpp/src/arrow/ipc/metadata.cc         |  20 +++-
 cpp/src/arrow/ipc/test-common.h       |  26 ++--
 cpp/src/arrow/ipc/writer.cc           |   3 +-
 cpp/src/arrow/pretty_print.cc         | 164 +++++++++++---------------
 cpp/src/arrow/type-test.cc            |  36 +++---
 cpp/src/arrow/type.cc                 |  41 +++++--
 cpp/src/arrow/type.h                  |  63 +++++++---
 cpp/src/arrow/type_fwd.h              |  11 +-
 cpp/src/arrow/type_traits.h           |  49 +++++++-
 cpp/src/arrow/visitor.cc              |   6 +-
 cpp/src/arrow/visitor.h               |   6 +-
 cpp/src/arrow/visitor_inline.h        |  41 ++++++-
 python/pyarrow/array.pyx              |  10 +-
 python/pyarrow/includes/libarrow.pxd  |  15 ++-
 27 files changed, 446 insertions(+), 237 deletions(-)

diff --git a/c_glib/arrow-glib/type.cpp b/c_glib/arrow-glib/type.cpp
index 2e59647884551..8adbaa90a58c7 100644
--- a/c_glib/arrow-glib/type.cpp
+++ b/c_glib/arrow-glib/type.cpp
@@ -72,8 +72,10 @@ garrow_type_from_raw(arrow::Type::type type)
     return GARROW_TYPE_DATE64;
   case arrow::Type::type::TIMESTAMP:
     return GARROW_TYPE_TIMESTAMP;
-  case arrow::Type::type::TIME:
-    return GARROW_TYPE_TIME;
+  case arrow::Type::type::TIME32:
+    return GARROW_TYPE_TIME32;
+  case arrow::Type::type::TIME64:
+    return GARROW_TYPE_TIME64;
   case arrow::Type::type::INTERVAL:
     return GARROW_TYPE_INTERVAL;
   case arrow::Type::type::DECIMAL:
diff --git a/c_glib/arrow-glib/type.h b/c_glib/arrow-glib/type.h
index cd6137cb5ba5f..e171aa3220f05 100644
--- a/c_glib/arrow-glib/type.h
+++ b/c_glib/arrow-glib/type.h
@@ -44,7 +44,8 @@ G_BEGIN_DECLS
  * @GARROW_TYPE_DATE64: int64 milliseconds since the UNIX epoch.
  * @GARROW_TYPE_TIMESTAMP: Exact timestamp encoded with int64 since UNIX epoch.
  *   Default unit millisecond.
- * @GARROW_TYPE_TIME: Exact time encoded with int64, default unit millisecond.
+ * @GARROW_TYPE_TIME32: Exact time encoded with int32, supporting seconds or milliseconds
+ * @GARROW_TYPE_TIME64: Exact time encoded with int64, supporting micro- or nanoseconds
  * @GARROW_TYPE_INTERVAL: YEAR_MONTH or DAY_TIME interval in SQL style.
  * @GARROW_TYPE_DECIMAL: Precision- and scale-based decimal
  *   type. Storage type depends on the parameters.
@@ -74,7 +75,8 @@ typedef enum {
   GARROW_TYPE_DATE32,
   GARROW_TYPE_DATE64,
   GARROW_TYPE_TIMESTAMP,
-  GARROW_TYPE_TIME,
+  GARROW_TYPE_TIME32,
+  GARROW_TYPE_TIME64,
   GARROW_TYPE_INTERVAL,
   GARROW_TYPE_DECIMAL,
   GARROW_TYPE_LIST,
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 9e00fd9a7dd49..b64023bbc6a1e 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -29,12 +29,6 @@ TEST(TypesTest, TestDecimalType) {
   ASSERT_EQ(t1.scale, 4);
 
   ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
-
-  // Test copy constructor
-  DecimalType t2 = t1;
-  ASSERT_EQ(t2.type, Type::DECIMAL);
-  ASSERT_EQ(t2.precision, 8);
-  ASSERT_EQ(t2.scale, 4);
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
index 6863e58df05d2..fe60170cc5cc4 100644
--- a/cpp/src/arrow/array-primitive-test.cc
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -47,9 +47,6 @@ class Array;
                                             \
     ASSERT_EQ(tp.type, Type::ENUM);         \
     ASSERT_EQ(tp.ToString(), string(NAME)); \
-                                            \
-    KLASS tp_copy = tp;                     \
-    ASSERT_EQ(tp_copy.type, Type::ENUM);    \
   }
 
 PRIMITIVE_TEST(Int8Type, INT8, "int8");
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 20b732ab114da..f1c8bd42c476d 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -483,7 +483,8 @@ template class NumericArray<Int64Type>;
 template class NumericArray<TimestampType>;
 template class NumericArray<Date32Type>;
 template class NumericArray<Date64Type>;
-template class NumericArray<TimeType>;
+template class NumericArray<Time32Type>;
+template class NumericArray<Time64Type>;
 template class NumericArray<HalfFloatType>;
 template class NumericArray<FloatType>;
 template class NumericArray<DoubleType>;
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 2a072dbf25ec0..c73b7a87a4f50 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -527,10 +527,11 @@ extern template class ARROW_EXPORT NumericArray<UInt64Type>;
 extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
 extern template class ARROW_EXPORT NumericArray<FloatType>;
 extern template class ARROW_EXPORT NumericArray<DoubleType>;
-extern template class ARROW_EXPORT NumericArray<TimestampType>;
 extern template class ARROW_EXPORT NumericArray<Date32Type>;
 extern template class ARROW_EXPORT NumericArray<Date64Type>;
-extern template class ARROW_EXPORT NumericArray<TimeType>;
+extern template class ARROW_EXPORT NumericArray<Time32Type>;
+extern template class ARROW_EXPORT NumericArray<Time64Type>;
+extern template class ARROW_EXPORT NumericArray<TimestampType>;
 
 #if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic pop
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 483d6f0a425ea..52a785d086117 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -240,8 +240,9 @@ template class PrimitiveBuilder<Int32Type>;
 template class PrimitiveBuilder<Int64Type>;
 template class PrimitiveBuilder<Date32Type>;
 template class PrimitiveBuilder<Date64Type>;
+template class PrimitiveBuilder<Time32Type>;
+template class PrimitiveBuilder<Time64Type>;
 template class PrimitiveBuilder<TimestampType>;
-template class PrimitiveBuilder<TimeType>;
 template class PrimitiveBuilder<HalfFloatType>;
 template class PrimitiveBuilder<FloatType>;
 template class PrimitiveBuilder<DoubleType>;
@@ -511,9 +512,9 @@ std::shared_ptr<ArrayBuilder> StructBuilder::field_builder(int pos) const {
 // ----------------------------------------------------------------------
 // Helper functions
 
-#define BUILDER_CASE(ENUM, BuilderType) \
-  case Type::ENUM:                      \
-    out->reset(new BuilderType(pool));  \
+#define BUILDER_CASE(ENUM, BuilderType)      \
+  case Type::ENUM:                           \
+    out->reset(new BuilderType(pool, type)); \
     return Status::OK();
 
 // Initially looked at doing this with vtables, but shared pointers makes it
@@ -533,17 +534,14 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(INT64, Int64Builder);
     BUILDER_CASE(DATE32, Date32Builder);
     BUILDER_CASE(DATE64, Date64Builder);
-    case Type::TIMESTAMP:
-      out->reset(new TimestampBuilder(pool, type));
-      return Status::OK();
-    case Type::TIME:
-      out->reset(new TimeBuilder(pool, type));
-      return Status::OK();
-      BUILDER_CASE(BOOL, BooleanBuilder);
-      BUILDER_CASE(FLOAT, FloatBuilder);
-      BUILDER_CASE(DOUBLE, DoubleBuilder);
-      BUILDER_CASE(STRING, StringBuilder);
-      BUILDER_CASE(BINARY, BinaryBuilder);
+    BUILDER_CASE(TIME32, Time32Builder);
+    BUILDER_CASE(TIME64, Time64Builder);
+    BUILDER_CASE(TIMESTAMP, TimestampBuilder);
+    BUILDER_CASE(BOOL, BooleanBuilder);
+    BUILDER_CASE(FLOAT, FloatBuilder);
+    BUILDER_CASE(DOUBLE, DoubleBuilder);
+    BUILDER_CASE(STRING, StringBuilder);
+    BUILDER_CASE(BINARY, BinaryBuilder);
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 7cefa649cbf71..bd957b38280da 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -231,7 +231,8 @@ using Int16Builder = NumericBuilder<Int16Type>;
 using Int32Builder = NumericBuilder<Int32Type>;
 using Int64Builder = NumericBuilder<Int64Type>;
 using TimestampBuilder = NumericBuilder<TimestampType>;
-using TimeBuilder = NumericBuilder<TimeType>;
+using Time32Builder = NumericBuilder<Time32Type>;
+using Time64Builder = NumericBuilder<Time64Type>;
 using Date32Builder = NumericBuilder<Date32Type>;
 using Date64Builder = NumericBuilder<Date64Type>;
 
@@ -378,6 +379,7 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
 // String builder
 class ARROW_EXPORT StringBuilder : public BinaryBuilder {
  public:
+  using BinaryBuilder::BinaryBuilder;
   explicit StringBuilder(MemoryPool* pool);
 
   using BinaryBuilder::Append;
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 3e6ecefc5ca5b..13511cf0f11be 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -29,6 +29,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
+#include "arrow/visitor_inline.h"
 
 namespace arrow {
 
@@ -177,7 +178,13 @@ class RangeEqualsVisitor : public ArrayVisitor {
     return CompareValues<Date64Array>(left);
   }
 
-  Status Visit(const TimeArray& left) override { return CompareValues<TimeArray>(left); }
+  Status Visit(const Time32Array& left) override {
+    return CompareValues<Time32Array>(left);
+  }
+
+  Status Visit(const Time64Array& left) override {
+    return CompareValues<Time64Array>(left);
+  }
 
   Status Visit(const TimestampArray& left) override {
     return CompareValues<TimestampArray>(left);
@@ -415,7 +422,9 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
 
   Status Visit(const Date64Array& left) override { return ComparePrimitive(left); }
 
-  Status Visit(const TimeArray& left) override { return ComparePrimitive(left); }
+  Status Visit(const Time32Array& left) override { return ComparePrimitive(left); }
+
+  Status Visit(const Time64Array& left) override { return ComparePrimitive(left); }
 
   Status Visit(const TimestampArray& left) override { return ComparePrimitive(left); }
 
@@ -628,7 +637,7 @@ Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal)
 // ----------------------------------------------------------------------
 // Implement TypeEquals
 
-class TypeEqualsVisitor : public TypeVisitor {
+class TypeEqualsVisitor {
  public:
   explicit TypeEqualsVisitor(const DataType& right) : right_(right), result_(false) {}
 
@@ -648,29 +657,44 @@ class TypeEqualsVisitor : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const TimeType& left) override {
-    const auto& right = static_cast<const TimeType&>(right_);
+  template <typename T>
+  typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
+                              std::is_base_of<PrimitiveCType, T>::value,
+      Status>::type
+  Visit(const T& type) {
+    result_ = true;
+    return Status::OK();
+  }
+
+  Status Visit(const Time32Type& left) {
+    const auto& right = static_cast<const Time32Type&>(right_);
     result_ = left.unit == right.unit;
     return Status::OK();
   }
 
-  Status Visit(const TimestampType& left) override {
+  Status Visit(const Time64Type& left) {
+    const auto& right = static_cast<const Time64Type&>(right_);
+    result_ = left.unit == right.unit;
+    return Status::OK();
+  }
+
+  Status Visit(const TimestampType& left) {
     const auto& right = static_cast<const TimestampType&>(right_);
     result_ = left.unit == right.unit && left.timezone == right.timezone;
     return Status::OK();
   }
 
-  Status Visit(const FixedWidthBinaryType& left) override {
+  Status Visit(const FixedWidthBinaryType& left) {
     const auto& right = static_cast<const FixedWidthBinaryType&>(right_);
     result_ = left.byte_width() == right.byte_width();
     return Status::OK();
   }
 
-  Status Visit(const ListType& left) override { return VisitChildren(left); }
+  Status Visit(const ListType& left) { return VisitChildren(left); }
 
-  Status Visit(const StructType& left) override { return VisitChildren(left); }
+  Status Visit(const StructType& left) { return VisitChildren(left); }
 
-  Status Visit(const UnionType& left) override {
+  Status Visit(const UnionType& left) {
     const auto& right = static_cast<const UnionType&>(right_);
 
     if (left.mode != right.mode || left.type_codes.size() != right.type_codes.size()) {
@@ -691,7 +715,7 @@ class TypeEqualsVisitor : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DictionaryType& left) override {
+  Status Visit(const DictionaryType& left) {
     const auto& right = static_cast<const DictionaryType&>(right_);
     result_ = left.index_type()->Equals(right.index_type()) &&
               left.dictionary()->Equals(right.dictionary());
@@ -713,18 +737,8 @@ Status TypeEquals(const DataType& left, const DataType& right, bool* are_equal)
     *are_equal = false;
   } else {
     TypeEqualsVisitor visitor(right);
-    Status s = left.Accept(&visitor);
-
-    // We do not implement any type visitors where there is no additional
-    // metadata to compare.
-    if (s.IsNotImplemented()) {
-      // Not implemented means there is no additional metadata to compare
-      *are_equal = true;
-    } else if (!s.ok()) {
-      return s;
-    } else {
-      *are_equal = visitor.result();
-    }
+    RETURN_NOT_OK(VisitTypeInline(left, &visitor));
+    *are_equal = visitor.result();
   }
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index 2513887f75903..e181f6933541b 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -353,7 +353,7 @@ TEST_F(TestTableWriter, CategoryRoundtrip) {
 TEST_F(TestTableWriter, TimeTypes) {
   std::vector<bool> is_valid = {true, true, true, false, true, true, true};
   auto f0 = field("f0", date32());
-  auto f1 = field("f1", time(TimeUnit::MILLI));
+  auto f1 = field("f1", time32(TimeUnit::MILLI));
   auto f2 = field("f2", timestamp(TimeUnit::NANO));
   auto f3 = field("f3", timestamp(TimeUnit::SECOND, "US/Los_Angeles"));
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2, f3}));
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 0dd9a8183fdc2..000bba9cce03b 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -294,7 +294,7 @@ class TableReader::TableReaderImpl {
         break;
       case fbs::TypeMetadata_TimeMetadata: {
         auto meta = static_cast<const fbs::TimeMetadata*>(metadata);
-        *out = std::make_shared<TimeType>(FromFlatbufferEnum(meta->unit()));
+        *out = time32(FromFlatbufferEnum(meta->unit()));
       } break;
       default:
         switch (values->type()) {
@@ -476,7 +476,9 @@ fbs::Type ToFlatbufferType(Type::type type) {
       return fbs::Type_DATE;
     case Type::TIMESTAMP:
       return fbs::Type_TIMESTAMP;
-    case Type::TIME:
+    case Type::TIME32:
+      return fbs::Type_TIME;
+    case Type::TIME64:
       return fbs::Type_TIME;
     case Type::DICTIONARY:
       return fbs::Type_CATEGORY;
@@ -646,13 +648,17 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status Visit(const TimeArray& values) override {
+  Status Visit(const Time32Array& values) override {
     RETURN_NOT_OK(WritePrimitiveValues(values));
-    auto unit = static_cast<const TimeType&>(*values.type()).unit;
+    auto unit = static_cast<const Time32Type&>(*values.type()).unit;
     current_column_->SetTime(unit);
     return Status::OK();
   }
 
+  Status Visit(const Time64Array& values) override {
+    return Status::NotImplemented("time64");
+  }
+
   Status Append(const std::string& name, const Array& values) {
     current_column_ = metadata_.AddColumn(name);
     RETURN_NOT_OK(values.Accept(this));
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index fd35182751948..e943ef1558a75 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -52,7 +52,9 @@ void TestSchemaRoundTrip(const Schema& schema) {
   std::shared_ptr<Schema> out;
   ASSERT_OK(ReadJsonSchema(d, &out));
 
-  ASSERT_TRUE(schema.Equals(out));
+  if (!schema.Equals(out)) {
+    FAIL() << "In schema: " << schema.ToString() << "\nOut schema: " << out->ToString();
+  }
 }
 
 void TestArrayRoundTrip(const Array& array) {
@@ -105,8 +107,8 @@ TEST(TestJsonSchemaWriter, FlatTypes) {
       field("f10", utf8()), field("f11", binary()), field("f12", list(int32())),
       field("f13", struct_({field("s1", int32()), field("s2", utf8())})),
       field("f15", date64()), field("f16", timestamp(TimeUnit::NANO)),
-      field("f17", time(TimeUnit::MICRO)),
-      field("f18", union_({field("u1", int8()), field("u2", time(TimeUnit::MILLI))},
+      field("f17", time64(TimeUnit::MICRO)),
+      field("f18", union_({field("u1", int8()), field("u2", time32(TimeUnit::MILLI))},
                        {0, 1}, UnionMode::DENSE))};
 
   Schema schema(fields);
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 08f0bdc3a023e..348468006d0b5 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -133,7 +133,10 @@ class JsonSchemaWriter : public TypeVisitor {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value, void>::type
+  typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
+                              std::is_base_of<ListType, T>::value ||
+                              std::is_base_of<StructType, T>::value,
+      void>::type
   WriteTypeMetadata(const T& type) {}
 
   template <typename T>
@@ -167,7 +170,8 @@ class JsonSchemaWriter : public TypeVisitor {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<TimeType, T>::value ||
+  typename std::enable_if<std::is_base_of<Time32Type, T>::value ||
+                              std::is_base_of<Time64Type, T>::value ||
                               std::is_base_of<TimestampType, T>::value,
       void>::type
   WriteTypeMetadata(const T& type) {
@@ -305,7 +309,9 @@ class JsonSchemaWriter : public TypeVisitor {
 
   Status Visit(const Date64Type& type) override { return WritePrimitive("date", type); }
 
-  Status Visit(const TimeType& type) override { return WritePrimitive("time", type); }
+  Status Visit(const Time32Type& type) override { return WritePrimitive("time", type); }
+
+  Status Visit(const Time64Type& type) override { return WritePrimitive("time", type); }
 
   Status Visit(const TimestampType& type) override {
     return WritePrimitive("timestamp", type);
@@ -650,15 +656,35 @@ class JsonSchemaReader {
     return Status::OK();
   }
 
-  template <typename T>
-  Status GetTimeLike(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type) {
     const auto& json_unit = json_type.FindMember("unit");
     RETURN_NOT_STRING("unit", json_unit, json_type);
 
     std::string unit_str = json_unit->value.GetString();
 
-    TimeUnit unit;
+    if (unit_str == "SECOND") {
+      *type = time32(TimeUnit::SECOND);
+    } else if (unit_str == "MILLISECOND") {
+      *type = time32(TimeUnit::MILLI);
+    } else if (unit_str == "MICROSECOND") {
+      *type = time64(TimeUnit::MICRO);
+    } else if (unit_str == "NANOSECOND") {
+      *type = time64(TimeUnit::NANO);
+    } else {
+      std::stringstream ss;
+      ss << "Invalid time unit: " << unit_str;
+      return Status::Invalid(ss.str());
+    }
+    return Status::OK();
+  }
+
+  Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+    const auto& json_unit = json_type.FindMember("unit");
+    RETURN_NOT_STRING("unit", json_unit, json_type);
 
+    std::string unit_str = json_unit->value.GetString();
+
+    TimeUnit unit;
     if (unit_str == "SECOND") {
       unit = TimeUnit::SECOND;
     } else if (unit_str == "MILLISECOND") {
@@ -673,7 +699,7 @@ class JsonSchemaReader {
       return Status::Invalid(ss.str());
     }
 
-    *type = std::make_shared<T>(unit);
+    *type = timestamp(unit);
 
     return Status::OK();
   }
@@ -736,9 +762,9 @@ class JsonSchemaReader {
       // TODO
       *type = date64();
     } else if (type_name == "time") {
-      return GetTimeLike<TimeType>(json_type, type);
+      return GetTime(json_type, type);
     } else if (type_name == "timestamp") {
-      return GetTimeLike<TimestampType>(json_type, type);
+      return GetTimestamp(json_type, type);
     } else if (type_name == "list") {
       *type = list(children[0]);
     } else if (type_name == "struct") {
@@ -1063,7 +1089,8 @@ class JsonArrayReader {
       NOT_IMPLEMENTED_CASE(DATE32);
       NOT_IMPLEMENTED_CASE(DATE64);
       NOT_IMPLEMENTED_CASE(TIMESTAMP);
-      NOT_IMPLEMENTED_CASE(TIME);
+      NOT_IMPLEMENTED_CASE(TIME32);
+      NOT_IMPLEMENTED_CASE(TIME64);
       NOT_IMPLEMENTED_CASE(INTERVAL);
       TYPE_CASE(ListType);
       TYPE_CASE(StructType);
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 14cb627982343..17af563805792 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -251,7 +251,16 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     }
     case flatbuf::Type_Time: {
       auto time_type = static_cast<const flatbuf::Time*>(type_data);
-      *out = time(FromFlatbufferUnit(time_type->unit()));
+      TimeUnit unit = FromFlatbufferUnit(time_type->unit());
+      switch (unit) {
+        case TimeUnit::SECOND:
+        case TimeUnit::MILLI:
+          *out = time32(unit);
+          break;
+        default:
+          *out = time64(unit);
+          break;
+      }
       return Status::OK();
     }
     case flatbuf::Type_Timestamp: {
@@ -371,8 +380,13 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *out_type = flatbuf::Type_Date;
       *offset = flatbuf::CreateDate(fbb, flatbuf::DateUnit_MILLISECOND).Union();
       break;
-    case Type::TIME: {
-      const auto& time_type = static_cast<const TimeType&>(*type);
+    case Type::TIME32: {
+      const auto& time_type = static_cast<const Time32Type&>(*type);
+      *out_type = flatbuf::Type_Time;
+      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
+    } break;
+    case Type::TIME64: {
+      const auto& time_type = static_cast<const Time64Type&>(*type);
       *out_type = flatbuf::Type_Time;
       *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
     } break;
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 4085ecf9e3da9..7ee57d2152c1b 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -505,20 +505,24 @@ Status MakeTimestamps(std::shared_ptr<RecordBatch>* out) {
 
 Status MakeTimes(std::shared_ptr<RecordBatch>* out) {
   std::vector<bool> is_valid = {true, true, true, false, true, true, true};
-  auto f0 = field("f0", time(TimeUnit::MILLI));
-  auto f1 = field("f1", time(TimeUnit::NANO));
-  auto f2 = field("f2", time(TimeUnit::SECOND));
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
-
-  std::vector<int64_t> ts_values = {1489269000000, 1489270000000, 1489271000000,
+  auto f0 = field("f0", time32(TimeUnit::MILLI));
+  auto f1 = field("f1", time64(TimeUnit::NANO));
+  auto f2 = field("f2", time32(TimeUnit::SECOND));
+  auto f3 = field("f3", time64(TimeUnit::NANO));
+  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2, f3}));
+
+  std::vector<int32_t> t32_values = {
+      1489269000, 1489270000, 1489271000, 1489272000, 1489272000, 1489273000};
+  std::vector<int64_t> t64_values = {1489269000000, 1489270000000, 1489271000000,
       1489272000000, 1489272000000, 1489273000000};
 
-  std::shared_ptr<Array> a0, a1, a2;
-  ArrayFromVector<TimeType, int64_t>(f0->type, is_valid, ts_values, &a0);
-  ArrayFromVector<TimeType, int64_t>(f1->type, is_valid, ts_values, &a1);
-  ArrayFromVector<TimeType, int64_t>(f2->type, is_valid, ts_values, &a2);
+  std::shared_ptr<Array> a0, a1, a2, a3;
+  ArrayFromVector<Time32Type, int32_t>(f0->type, is_valid, t32_values, &a0);
+  ArrayFromVector<Time64Type, int64_t>(f1->type, is_valid, t64_values, &a1);
+  ArrayFromVector<Time32Type, int32_t>(f2->type, is_valid, t32_values, &a2);
+  ArrayFromVector<Time64Type, int64_t>(f3->type, is_valid, t64_values, &a3);
 
-  ArrayVector arrays = {a0, a1, a2};
+  ArrayVector arrays = {a0, a1, a2, a3};
   *out = std::make_shared<RecordBatch>(schema, a0->length(), arrays);
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index dc991aba79795..e795ef961cb64 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -334,8 +334,9 @@ class RecordBatchWriter : public ArrayVisitor {
   VISIT_FIXED_WIDTH(DoubleArray);
   VISIT_FIXED_WIDTH(Date32Array);
   VISIT_FIXED_WIDTH(Date64Array);
-  VISIT_FIXED_WIDTH(TimeArray);
   VISIT_FIXED_WIDTH(TimestampArray);
+  VISIT_FIXED_WIDTH(Time32Array);
+  VISIT_FIXED_WIDTH(Time64Array);
 
 #undef VISIT_FIXED_WIDTH
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index fc5eed18d8776..0f67fe5bc52a7 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -27,20 +27,17 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/string.h"
+#include "arrow/visitor_inline.h"
 
 namespace arrow {
 
-class ArrayPrinter : public ArrayVisitor {
+class ArrayPrinter {
  public:
   ArrayPrinter(const Array& array, int indent, std::ostream* sink)
       : array_(array), indent_(indent), sink_(sink) {}
 
-  Status Print() { return VisitArray(array_); }
-
-  Status VisitArray(const Array& array) { return array.Accept(this); }
-
   template <typename T>
-  typename std::enable_if<IsInteger<T>::value, void>::type WriteDataValues(
+  inline typename std::enable_if<IsInteger<T>::value, void>::type WriteDataValues(
       const T& array) {
     const auto data = array.raw_data();
     for (int i = 0; i < array.length(); ++i) {
@@ -54,7 +51,7 @@ class ArrayPrinter : public ArrayVisitor {
   }
 
   template <typename T>
-  typename std::enable_if<IsFloatingPoint<T>::value, void>::type WriteDataValues(
+  inline typename std::enable_if<IsFloatingPoint<T>::value, void>::type WriteDataValues(
       const T& array) {
     const auto data = array.raw_data();
     for (int i = 0; i < array.length(); ++i) {
@@ -69,7 +66,7 @@ class ArrayPrinter : public ArrayVisitor {
 
   // String (Utf8)
   template <typename T>
-  typename std::enable_if<std::is_same<StringArray, T>::value, void>::type
+  inline typename std::enable_if<std::is_same<StringArray, T>::value, void>::type
   WriteDataValues(const T& array) {
     int32_t length;
     for (int i = 0; i < array.length(); ++i) {
@@ -85,7 +82,7 @@ class ArrayPrinter : public ArrayVisitor {
 
   // Binary
   template <typename T>
-  typename std::enable_if<std::is_same<BinaryArray, T>::value, void>::type
+  inline typename std::enable_if<std::is_same<BinaryArray, T>::value, void>::type
   WriteDataValues(const T& array) {
     int32_t length;
     for (int i = 0; i < array.length(); ++i) {
@@ -100,8 +97,9 @@ class ArrayPrinter : public ArrayVisitor {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_same<FixedWidthBinaryArray, T>::value, void>::type
-  WriteDataValues(const T& array) {
+  inline
+      typename std::enable_if<std::is_same<FixedWidthBinaryArray, T>::value, void>::type
+      WriteDataValues(const T& array) {
     int32_t width = array.byte_width();
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) { (*sink_) << ", "; }
@@ -115,7 +113,7 @@ class ArrayPrinter : public ArrayVisitor {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
+  inline typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
   WriteDataValues(const T& array) {
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) { (*sink_) << ", "; }
@@ -127,83 +125,34 @@ class ArrayPrinter : public ArrayVisitor {
     }
   }
 
-  void OpenArray() { (*sink_) << "["; }
+  void Write(const char* data);
+  void Write(const std::string& data);
+  void Newline();
+  void Indent();
+  void OpenArray();
+  void CloseArray();
 
-  void CloseArray() { (*sink_) << "]"; }
+  Status Visit(const NullArray& array) { return Status::OK(); }
 
   template <typename T>
-  Status WriteArray(const T& array) {
+  typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value ||
+                              std::is_base_of<FixedWidthBinaryArray, T>::value ||
+                              std::is_base_of<BinaryArray, T>::value,
+      Status>::type
+  Visit(const T& array) {
     OpenArray();
     WriteDataValues(array);
     CloseArray();
     return Status::OK();
   }
 
-  Status Visit(const NullArray& array) override { return Status::OK(); }
-
-  Status Visit(const BooleanArray& array) override { return WriteArray(array); }
-
-  Status Visit(const Int8Array& array) override { return WriteArray(array); }
-
-  Status Visit(const Int16Array& array) override { return WriteArray(array); }
-
-  Status Visit(const Int32Array& array) override { return WriteArray(array); }
-
-  Status Visit(const Int64Array& array) override { return WriteArray(array); }
-
-  Status Visit(const UInt8Array& array) override { return WriteArray(array); }
-
-  Status Visit(const UInt16Array& array) override { return WriteArray(array); }
-
-  Status Visit(const UInt32Array& array) override { return WriteArray(array); }
-
-  Status Visit(const UInt64Array& array) override { return WriteArray(array); }
-
-  Status Visit(const HalfFloatArray& array) override { return WriteArray(array); }
-
-  Status Visit(const FloatArray& array) override { return WriteArray(array); }
-
-  Status Visit(const DoubleArray& array) override { return WriteArray(array); }
-
-  Status Visit(const StringArray& array) override { return WriteArray(array); }
-
-  Status Visit(const BinaryArray& array) override { return WriteArray(array); }
-
-  Status Visit(const FixedWidthBinaryArray& array) override { return WriteArray(array); }
-
-  Status Visit(const Date32Array& array) override { return WriteArray(array); }
-
-  Status Visit(const Date64Array& array) override { return WriteArray(array); }
-
-  Status Visit(const TimeArray& array) override { return WriteArray(array); }
-
-  Status Visit(const TimestampArray& array) override {
-    return Status::NotImplemented("timestamp");
-  }
-
-  Status Visit(const IntervalArray& array) override {
-    return Status::NotImplemented("interval");
-  }
+  Status Visit(const IntervalArray& array) { return Status::NotImplemented("interval"); }
 
-  Status Visit(const DecimalArray& array) override {
-    return Status::NotImplemented("decimal");
-  }
+  Status Visit(const DecimalArray& array) { return Status::NotImplemented("decimal"); }
 
-  Status WriteValidityBitmap(const Array& array) {
-    Newline();
-    Write("-- is_valid: ");
-
-    if (array.null_count() > 0) {
-      BooleanArray is_valid(
-          array.length(), array.null_bitmap(), nullptr, 0, array.offset());
-      return PrettyPrint(is_valid, indent_ + 2, sink_);
-    } else {
-      Write("all not null");
-      return Status::OK();
-    }
-  }
+  Status WriteValidityBitmap(const Array& array);
 
-  Status Visit(const ListArray& array) override {
+  Status Visit(const ListArray& array) {
     RETURN_NOT_OK(WriteValidityBitmap(array));
 
     Newline();
@@ -239,12 +188,12 @@ class ArrayPrinter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status Visit(const StructArray& array) override {
+  Status Visit(const StructArray& array) {
     RETURN_NOT_OK(WriteValidityBitmap(array));
     return PrintChildren(array.fields(), array.offset(), array.length());
   }
 
-  Status Visit(const UnionArray& array) override {
+  Status Visit(const UnionArray& array) {
     RETURN_NOT_OK(WriteValidityBitmap(array));
 
     Newline();
@@ -264,7 +213,7 @@ class ArrayPrinter : public ArrayVisitor {
     return PrintChildren(array.children(), 0, array.length() + array.offset());
   }
 
-  Status Visit(const DictionaryArray& array) override {
+  Status Visit(const DictionaryArray& array) {
     RETURN_NOT_OK(WriteValidityBitmap(array));
 
     Newline();
@@ -276,20 +225,7 @@ class ArrayPrinter : public ArrayVisitor {
     return PrettyPrint(*array.indices(), indent_ + 2, sink_);
   }
 
-  void Write(const char* data) { (*sink_) << data; }
-
-  void Write(const std::string& data) { (*sink_) << data; }
-
-  void Newline() {
-    (*sink_) << "\n";
-    Indent();
-  }
-
-  void Indent() {
-    for (int i = 0; i < indent_; ++i) {
-      (*sink_) << " ";
-    }
-  }
+  Status Print() { return VisitArrayInline(array_, this); }
 
  private:
   const Array& array_;
@@ -298,6 +234,46 @@ class ArrayPrinter : public ArrayVisitor {
   std::ostream* sink_;
 };
 
+Status ArrayPrinter::WriteValidityBitmap(const Array& array) {
+  Newline();
+  Write("-- is_valid: ");
+
+  if (array.null_count() > 0) {
+    BooleanArray is_valid(
+        array.length(), array.null_bitmap(), nullptr, 0, array.offset());
+    return PrettyPrint(is_valid, indent_ + 2, sink_);
+  } else {
+    Write("all not null");
+    return Status::OK();
+  }
+}
+
+void ArrayPrinter::OpenArray() {
+  (*sink_) << "[";
+}
+void ArrayPrinter::CloseArray() {
+  (*sink_) << "]";
+}
+
+void ArrayPrinter::Write(const char* data) {
+  (*sink_) << data;
+}
+
+void ArrayPrinter::Write(const std::string& data) {
+  (*sink_) << data;
+}
+
+void ArrayPrinter::Newline() {
+  (*sink_) << "\n";
+  Indent();
+}
+
+void ArrayPrinter::Indent() {
+  for (int i = 0; i < indent_; ++i) {
+    (*sink_) << " ";
+  }
+}
+
 Status PrettyPrint(const Array& arr, int indent, std::ostream* sink) {
   ArrayPrinter printer(arr, indent, sink);
   return printer.Print();
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index c2d115ccbfe6f..b6a84df339e6e 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -182,26 +182,30 @@ TEST(TestDateTypes, ToString) {
 }
 
 TEST(TestTimeType, Equals) {
-  TimeType t1;
-  TimeType t2;
-  TimeType t3(TimeUnit::NANO);
-  TimeType t4(TimeUnit::NANO);
-
-  ASSERT_TRUE(t1.Equals(t2));
+  Time32Type t0;
+  Time32Type t1(TimeUnit::SECOND);
+  Time32Type t2(TimeUnit::MILLI);
+  Time64Type t3(TimeUnit::MICRO);
+  Time64Type t4(TimeUnit::NANO);
+  Time64Type t5(TimeUnit::MICRO);
+
+  ASSERT_TRUE(t0.Equals(t2));
+  ASSERT_TRUE(t1.Equals(t1));
   ASSERT_FALSE(t1.Equals(t3));
-  ASSERT_TRUE(t3.Equals(t4));
+  ASSERT_FALSE(t3.Equals(t4));
+  ASSERT_TRUE(t3.Equals(t5));
 }
 
 TEST(TestTimeType, ToString) {
-  auto t1 = time(TimeUnit::MILLI);
-  auto t2 = time(TimeUnit::NANO);
-  auto t3 = time(TimeUnit::SECOND);
-  auto t4 = time(TimeUnit::MICRO);
-
-  ASSERT_EQ("time[ms]", t1->ToString());
-  ASSERT_EQ("time[ns]", t2->ToString());
-  ASSERT_EQ("time[s]", t3->ToString());
-  ASSERT_EQ("time[us]", t4->ToString());
+  auto t1 = time32(TimeUnit::MILLI);
+  auto t2 = time64(TimeUnit::NANO);
+  auto t3 = time32(TimeUnit::SECOND);
+  auto t4 = time64(TimeUnit::MICRO);
+
+  ASSERT_EQ("time32[ms]", t1->ToString());
+  ASSERT_EQ("time64[ns]", t2->ToString());
+  ASSERT_EQ("time32[s]", t3->ToString());
+  ASSERT_EQ("time64[us]", t4->ToString());
 }
 
 TEST(TestTimestampType, Equals) {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 1c61eb61abea0..388502214e733 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -119,12 +119,34 @@ std::string Date32Type::ToString() const {
   return std::string("date32[day]");
 }
 
-std::string TimeType::ToString() const {
+// ----------------------------------------------------------------------
+// Time types
+
+Time32Type::Time32Type(TimeUnit unit) : FixedWidthType(Type::TIME32), unit(unit) {
+  DCHECK(unit == TimeUnit::SECOND || unit == TimeUnit::MILLI)
+      << "Must be seconds or milliseconds";
+}
+
+std::string Time32Type::ToString() const {
+  std::stringstream ss;
+  ss << "time32[" << this->unit << "]";
+  return ss.str();
+}
+
+Time64Type::Time64Type(TimeUnit unit) : FixedWidthType(Type::TIME64), unit(unit) {
+  DCHECK(unit == TimeUnit::MICRO || unit == TimeUnit::NANO)
+      << "Must be microseconds or nanoseconds";
+}
+
+std::string Time64Type::ToString() const {
   std::stringstream ss;
-  ss << "time[" << this->unit << "]";
+  ss << "time64[" << this->unit << "]";
   return ss.str();
 }
 
+// ----------------------------------------------------------------------
+// Timestamp types
+
 std::string TimestampType::ToString() const {
   std::stringstream ss;
   ss << "timestamp[" << this->unit;
@@ -138,7 +160,7 @@ std::string TimestampType::ToString() const {
 
 UnionType::UnionType(const std::vector<std::shared_ptr<Field>>& fields,
     const std::vector<uint8_t>& type_codes, UnionMode mode)
-    : DataType(Type::UNION), mode(mode), type_codes(type_codes) {
+    : NestedType(Type::UNION), mode(mode), type_codes(type_codes) {
   children_ = fields;
 }
 
@@ -206,9 +228,10 @@ ACCEPT_VISITOR(ListType);
 ACCEPT_VISITOR(StructType);
 ACCEPT_VISITOR(DecimalType);
 ACCEPT_VISITOR(UnionType);
-ACCEPT_VISITOR(Date64Type);
 ACCEPT_VISITOR(Date32Type);
-ACCEPT_VISITOR(TimeType);
+ACCEPT_VISITOR(Date64Type);
+ACCEPT_VISITOR(Time32Type);
+ACCEPT_VISITOR(Time64Type);
 ACCEPT_VISITOR(TimestampType);
 ACCEPT_VISITOR(IntervalType);
 ACCEPT_VISITOR(DictionaryType);
@@ -249,8 +272,12 @@ std::shared_ptr<DataType> timestamp(TimeUnit unit, const std::string& timezone)
   return std::make_shared<TimestampType>(unit, timezone);
 }
 
-std::shared_ptr<DataType> time(TimeUnit unit) {
-  return std::make_shared<TimeType>(unit);
+std::shared_ptr<DataType> time32(TimeUnit unit) {
+  return std::make_shared<Time32Type>(unit);
+}
+
+std::shared_ptr<DataType> time64(TimeUnit unit) {
+  return std::make_shared<Time64Type>(unit);
 }
 
 std::shared_ptr<DataType> list(const std::shared_ptr<DataType>& value_type) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 40c00a4bac1b1..7ae5ae3c4b72e 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -82,8 +82,13 @@ struct Type {
     // Default unit millisecond
     TIMESTAMP,
 
-    // Exact time encoded with int64, default unit millisecond
-    TIME,
+    // Time as signed 32-bit integer, representing either seconds or
+    // milliseconds since midnight
+    TIME32,
+
+    // Time as signed 64-bit integer, representing either microseconds or
+    // nanoseconds since midnight
+    TIME64,
 
     // YEAR_MONTH or DAY_TIME interval in SQL style
     INTERVAL,
@@ -147,6 +152,9 @@ struct ARROW_EXPORT DataType {
   virtual std::string ToString() const = 0;
 
   virtual std::vector<BufferDescr> GetBufferLayout() const = 0;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(DataType);
 };
 
 typedef std::shared_ptr<DataType> TypePtr;
@@ -168,6 +176,10 @@ struct ARROW_EXPORT FloatingPointMeta {
   virtual Precision precision() const = 0;
 };
 
+struct ARROW_EXPORT NestedType : public DataType {
+  using DataType::DataType;
+};
+
 struct NoExtraMeta {};
 
 // A field is a piece of metadata that includes (for now) a name and a data
@@ -298,14 +310,14 @@ struct ARROW_EXPORT DoubleType : public CTypeImpl<DoubleType, Type::DOUBLE, doub
   static std::string name() { return "double"; }
 };
 
-struct ARROW_EXPORT ListType : public DataType, public NoExtraMeta {
+struct ARROW_EXPORT ListType : public NestedType {
   static constexpr Type::type type_id = Type::LIST;
 
   // List can contain any other logical value type
   explicit ListType(const std::shared_ptr<DataType>& value_type)
       : ListType(std::make_shared<Field>("item", value_type)) {}
 
-  explicit ListType(const std::shared_ptr<Field>& value_field) : DataType(Type::LIST) {
+  explicit ListType(const std::shared_ptr<Field>& value_field) : NestedType(Type::LIST) {
     children_ = {value_field};
   }
 
@@ -369,11 +381,11 @@ struct ARROW_EXPORT StringType : public BinaryType {
   static std::string name() { return "utf8"; }
 };
 
-struct ARROW_EXPORT StructType : public DataType, public NoExtraMeta {
+struct ARROW_EXPORT StructType : public NestedType {
   static constexpr Type::type type_id = Type::STRUCT;
 
   explicit StructType(const std::vector<std::shared_ptr<Field>>& fields)
-      : DataType(Type::STRUCT) {
+      : NestedType(Type::STRUCT) {
     children_ = fields;
   }
 
@@ -401,7 +413,7 @@ struct ARROW_EXPORT DecimalType : public DataType {
 
 enum class UnionMode : char { SPARSE, DENSE };
 
-struct ARROW_EXPORT UnionType : public DataType {
+struct ARROW_EXPORT UnionType : public NestedType {
   static constexpr Type::type type_id = Type::UNION;
 
   UnionType(const std::vector<std::shared_ptr<Field>>& fields,
@@ -473,8 +485,23 @@ static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) {
   return os;
 }
 
-struct ARROW_EXPORT TimeType : public FixedWidthType {
-  static constexpr Type::type type_id = Type::TIME;
+struct ARROW_EXPORT Time32Type : public FixedWidthType {
+  static constexpr Type::type type_id = Type::TIME32;
+  using Unit = TimeUnit;
+  using c_type = int32_t;
+
+  TimeUnit unit;
+
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
+
+  explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);
+
+  Status Accept(TypeVisitor* visitor) const override;
+  std::string ToString() const override;
+};
+
+struct ARROW_EXPORT Time64Type : public FixedWidthType {
+  static constexpr Type::type type_id = Type::TIME64;
   using Unit = TimeUnit;
   using c_type = int64_t;
 
@@ -482,9 +509,7 @@ struct ARROW_EXPORT TimeType : public FixedWidthType {
 
   int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
 
-  explicit TimeType(TimeUnit unit = TimeUnit::MILLI)
-      : FixedWidthType(Type::TIME), unit(unit) {}
-  TimeType(const TimeType& other) : TimeType(other.unit) {}
+  explicit Time64Type(TimeUnit unit = TimeUnit::MILLI);
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -504,8 +529,6 @@ struct ARROW_EXPORT TimestampType : public FixedWidthType {
   explicit TimestampType(TimeUnit unit, const std::string& timezone)
       : FixedWidthType(Type::TIMESTAMP), unit(unit), timezone(timezone) {}
 
-  TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
-
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "timestamp"; }
@@ -527,8 +550,6 @@ struct ARROW_EXPORT IntervalType : public FixedWidthType {
   explicit IntervalType(Unit unit = Unit::YEAR_MONTH)
       : FixedWidthType(Type::INTERVAL), unit(unit) {}
 
-  IntervalType(const IntervalType& other) : IntervalType(other.unit) {}
-
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override { return name(); }
   static std::string name() { return "date"; }
@@ -573,7 +594,12 @@ std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& val
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit unit);
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(
     TimeUnit unit, const std::string& timezone);
-std::shared_ptr<DataType> ARROW_EXPORT time(TimeUnit unit);
+
+/// Unit can be either SECOND or MILLI
+std::shared_ptr<DataType> ARROW_EXPORT time32(TimeUnit unit);
+
+/// Unit can be either MICRO or NANO
+std::shared_ptr<DataType> ARROW_EXPORT time64(TimeUnit unit);
 
 std::shared_ptr<DataType> ARROW_EXPORT struct_(
     const std::vector<std::shared_ptr<Field>>& fields);
@@ -637,8 +663,9 @@ static inline bool is_primitive(Type::type type_id) {
     case Type::DOUBLE:
     case Type::DATE32:
     case Type::DATE64:
+    case Type::TIME32:
+    case Type::TIME64:
     case Type::TIMESTAMP:
-    case Type::TIME:
     case Type::INTERVAL:
       return true;
     default:
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index f62c0314a4620..201f4e92bb00d 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -105,9 +105,13 @@ struct Date32Type;
 using Date32Array = NumericArray<Date32Type>;
 using Date32Builder = NumericBuilder<Date32Type>;
 
-struct TimeType;
-using TimeArray = NumericArray<TimeType>;
-using TimeBuilder = NumericBuilder<TimeType>;
+struct Time32Type;
+using Time32Array = NumericArray<Time32Type>;
+using Time32Builder = NumericBuilder<Time32Type>;
+
+struct Time64Type;
+using Time64Array = NumericArray<Time64Type>;
+using Time64Builder = NumericBuilder<Time64Type>;
 
 struct TimestampType;
 using TimestampArray = NumericArray<TimestampType>;
@@ -134,6 +138,7 @@ std::shared_ptr<DataType> ARROW_EXPORT float32();
 std::shared_ptr<DataType> ARROW_EXPORT float64();
 std::shared_ptr<DataType> ARROW_EXPORT utf8();
 std::shared_ptr<DataType> ARROW_EXPORT binary();
+
 std::shared_ptr<DataType> ARROW_EXPORT date32();
 std::shared_ptr<DataType> ARROW_EXPORT date64();
 
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index e731913bbd226..f735d2706e5a9 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -28,6 +28,12 @@ namespace arrow {
 template <typename T>
 struct TypeTraits {};
 
+template <>
+struct TypeTraits<NullType> {
+  using ArrayType = NullArray;
+  constexpr static bool is_parameter_free = false;
+};
+
 template <>
 struct TypeTraits<UInt8Type> {
   using ArrayType = UInt8Array;
@@ -154,9 +160,20 @@ struct TypeTraits<TimestampType> {
 };
 
 template <>
-struct TypeTraits<TimeType> {
-  using ArrayType = TimeArray;
-  using BuilderType = TimeBuilder;
+struct TypeTraits<Time32Type> {
+  using ArrayType = Time32Array;
+  using BuilderType = Time32Builder;
+
+  static inline int64_t bytes_required(int64_t elements) {
+    return elements * sizeof(int32_t);
+  }
+  constexpr static bool is_parameter_free = false;
+};
+
+template <>
+struct TypeTraits<Time64Type> {
+  using ArrayType = Time64Array;
+  using BuilderType = Time64Builder;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(int64_t);
@@ -235,6 +252,32 @@ struct TypeTraits<FixedWidthBinaryType> {
   constexpr static bool is_parameter_free = false;
 };
 
+template <>
+struct TypeTraits<ListType> {
+  using ArrayType = ListArray;
+  using BuilderType = ListBuilder;
+  constexpr static bool is_parameter_free = false;
+};
+
+template <>
+struct TypeTraits<StructType> {
+  using ArrayType = StructArray;
+  using BuilderType = StructBuilder;
+  constexpr static bool is_parameter_free = false;
+};
+
+template <>
+struct TypeTraits<UnionType> {
+  using ArrayType = UnionArray;
+  constexpr static bool is_parameter_free = false;
+};
+
+template <>
+struct TypeTraits<DictionaryType> {
+  using ArrayType = DictionaryArray;
+  constexpr static bool is_parameter_free = false;
+};
+
 // Not all type classes have a c_type
 template <typename T>
 struct as_void {
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index 181e932eeebf6..9200e0ff228a3 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -46,7 +46,8 @@ ARRAY_VISITOR_DEFAULT(StringArray);
 ARRAY_VISITOR_DEFAULT(FixedWidthBinaryArray);
 ARRAY_VISITOR_DEFAULT(Date32Array);
 ARRAY_VISITOR_DEFAULT(Date64Array);
-ARRAY_VISITOR_DEFAULT(TimeArray);
+ARRAY_VISITOR_DEFAULT(Time32Array);
+ARRAY_VISITOR_DEFAULT(Time64Array);
 ARRAY_VISITOR_DEFAULT(TimestampArray);
 ARRAY_VISITOR_DEFAULT(IntervalArray);
 ARRAY_VISITOR_DEFAULT(ListArray);
@@ -84,7 +85,8 @@ TYPE_VISITOR_DEFAULT(BinaryType);
 TYPE_VISITOR_DEFAULT(FixedWidthBinaryType);
 TYPE_VISITOR_DEFAULT(Date64Type);
 TYPE_VISITOR_DEFAULT(Date32Type);
-TYPE_VISITOR_DEFAULT(TimeType);
+TYPE_VISITOR_DEFAULT(Time32Type);
+TYPE_VISITOR_DEFAULT(Time64Type);
 TYPE_VISITOR_DEFAULT(TimestampType);
 TYPE_VISITOR_DEFAULT(IntervalType);
 TYPE_VISITOR_DEFAULT(DecimalType);
diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h
index a9c59c8f762fe..d44dcf6b97676 100644
--- a/cpp/src/arrow/visitor.h
+++ b/cpp/src/arrow/visitor.h
@@ -46,7 +46,8 @@ class ARROW_EXPORT ArrayVisitor {
   virtual Status Visit(const FixedWidthBinaryArray& array);
   virtual Status Visit(const Date32Array& array);
   virtual Status Visit(const Date64Array& array);
-  virtual Status Visit(const TimeArray& array);
+  virtual Status Visit(const Time32Array& array);
+  virtual Status Visit(const Time64Array& array);
   virtual Status Visit(const TimestampArray& array);
   virtual Status Visit(const IntervalArray& array);
   virtual Status Visit(const DecimalArray& array);
@@ -78,7 +79,8 @@ class ARROW_EXPORT TypeVisitor {
   virtual Status Visit(const FixedWidthBinaryType& type);
   virtual Status Visit(const Date64Type& type);
   virtual Status Visit(const Date32Type& type);
-  virtual Status Visit(const TimeType& type);
+  virtual Status Visit(const Time32Type& type);
+  virtual Status Visit(const Time64Type& type);
   virtual Status Visit(const TimestampType& type);
   virtual Status Visit(const IntervalType& type);
   virtual Status Visit(const DecimalType& type);
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index b69468d17eebe..0ea16bcc73366 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -51,7 +51,8 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
     TYPE_VISIT_INLINE(Date32Type);
     TYPE_VISIT_INLINE(Date64Type);
     TYPE_VISIT_INLINE(TimestampType);
-    TYPE_VISIT_INLINE(TimeType);
+    TYPE_VISIT_INLINE(Time32Type);
+    TYPE_VISIT_INLINE(Time64Type);
     TYPE_VISIT_INLINE(ListType);
     TYPE_VISIT_INLINE(StructType);
     TYPE_VISIT_INLINE(UnionType);
@@ -62,6 +63,44 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
   return Status::NotImplemented("Type not implemented");
 }
 
+#define ARRAY_VISIT_INLINE(TYPE_CLASS) \
+  case TYPE_CLASS::type_id:            \
+    return visitor->Visit(             \
+        static_cast<const typename TypeTraits<TYPE_CLASS>::ArrayType&>(array));
+
+template <typename VISITOR>
+inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
+  switch (array.type_enum()) {
+    ARRAY_VISIT_INLINE(NullType);
+    ARRAY_VISIT_INLINE(BooleanType);
+    ARRAY_VISIT_INLINE(Int8Type);
+    ARRAY_VISIT_INLINE(UInt8Type);
+    ARRAY_VISIT_INLINE(Int16Type);
+    ARRAY_VISIT_INLINE(UInt16Type);
+    ARRAY_VISIT_INLINE(Int32Type);
+    ARRAY_VISIT_INLINE(UInt32Type);
+    ARRAY_VISIT_INLINE(Int64Type);
+    ARRAY_VISIT_INLINE(UInt64Type);
+    ARRAY_VISIT_INLINE(FloatType);
+    ARRAY_VISIT_INLINE(DoubleType);
+    ARRAY_VISIT_INLINE(StringType);
+    ARRAY_VISIT_INLINE(BinaryType);
+    ARRAY_VISIT_INLINE(FixedWidthBinaryType);
+    ARRAY_VISIT_INLINE(Date32Type);
+    ARRAY_VISIT_INLINE(Date64Type);
+    ARRAY_VISIT_INLINE(TimestampType);
+    ARRAY_VISIT_INLINE(Time32Type);
+    ARRAY_VISIT_INLINE(Time64Type);
+    ARRAY_VISIT_INLINE(ListType);
+    ARRAY_VISIT_INLINE(StructType);
+    ARRAY_VISIT_INLINE(UnionType);
+    ARRAY_VISIT_INLINE(DictionaryType);
+    default:
+      break;
+  }
+  return Status::NotImplemented("Type not implemented");
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_VISITOR_INLINE_H
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 795076cfccb7e..654f5ab527043 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -383,6 +383,12 @@ cdef class Date64Array(NumericArray):
 cdef class TimestampArray(NumericArray):
     pass
 
+cdef class Time32Array(NumericArray):
+    pass
+
+
+cdef class Time64Array(NumericArray):
+    pass
 
 cdef class FloatArray(FloatingPointArray):
     pass
@@ -490,12 +496,14 @@ cdef dict _array_classes = {
     Type_INT64: Int64Array,
     Type_DATE32: Date32Array,
     Type_DATE64: Date64Array,
+    Type_TIMESTAMP: TimestampArray,
+    Type_TIME32: Time32Array,
+    Type_TIME64: Time64Array,
     Type_FLOAT: FloatArray,
     Type_DOUBLE: DoubleArray,
     Type_LIST: ListArray,
     Type_BINARY: BinaryArray,
     Type_STRING: StringArray,
-    Type_TIMESTAMP: TimestampArray,
     Type_DICTIONARY: DictionaryArray
 }
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 1d9c38e48cfe9..bdbd18bce01df 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -38,9 +38,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         Type_FLOAT" arrow::Type::FLOAT"
         Type_DOUBLE" arrow::Type::DOUBLE"
 
-        Type_TIMESTAMP" arrow::Type::TIMESTAMP"
         Type_DATE32" arrow::Type::DATE32"
         Type_DATE64" arrow::Type::DATE64"
+        Type_TIMESTAMP" arrow::Type::TIMESTAMP"
+        Type_TIME32" arrow::Type::TIME32"
+        Type_TIME64" arrow::Type::TIME64"
         Type_BINARY" arrow::Type::BINARY"
         Type_STRING" arrow::Type::STRING"
 
@@ -85,11 +87,20 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CArray] indices()
         shared_ptr[CArray] dictionary()
 
+    cdef cppclass CDate32Type" arrow::Date32Type"(CFixedWidthType):
+        pass
+
+    cdef cppclass CDate64Type" arrow::Date64Type"(CFixedWidthType):
+        pass
+
     cdef cppclass CTimestampType" arrow::TimestampType"(CFixedWidthType):
         TimeUnit unit
         c_string timezone
 
-    cdef cppclass CTimeType" arrow::TimeType"(CFixedWidthType):
+    cdef cppclass CTime32Type" arrow::Time32Type"(CFixedWidthType):
+        TimeUnit unit
+
+    cdef cppclass CTime64Type" arrow::Time64Type"(CFixedWidthType):
         TimeUnit unit
 
     cdef cppclass CDictionaryType" arrow::DictionaryType"(CFixedWidthType):

From c7947dc2d08a0a2295016d34db201cc38a38360c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 24 Mar 2017 22:10:36 -0400
Subject: [PATCH 0418/1644] ARROW-709: [C++] Restore type comparator for
 DecimalType

Needed for PARQUET-923.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #439 from wesm/ARROW-709 and squashes the following commits:

55d23a3 [Wes McKinney] Restore type comparator for DecimalType
---
 cpp/src/arrow/compare.cc       | 6 ++++++
 cpp/src/arrow/loader.cc        | 2 ++
 cpp/src/arrow/type_traits.h    | 6 ++++++
 cpp/src/arrow/visitor_inline.h | 2 ++
 4 files changed, 16 insertions(+)

diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 13511cf0f11be..8274e0f80dc50 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -690,6 +690,12 @@ class TypeEqualsVisitor {
     return Status::OK();
   }
 
+  Status Visit(const DecimalType& left) {
+    const auto& right = static_cast<const DecimalType&>(right_);
+    result_ = left.precision == right.precision && left.scale == right.scale;
+    return Status::OK();
+  }
+
   Status Visit(const ListType& left) { return VisitChildren(left); }
 
   Status Visit(const StructType& left) { return VisitChildren(left); }
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
index a67a3e94bd5f2..cc64c4d8264f7 100644
--- a/cpp/src/arrow/loader.cc
+++ b/cpp/src/arrow/loader.cc
@@ -135,6 +135,8 @@ class ArrayLoader {
 
   Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
 
+  Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
+
   template <typename T>
   typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
                               !std::is_base_of<FixedWidthBinaryType, T>::value &&
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index f735d2706e5a9..1270aee1622ea 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -278,6 +278,12 @@ struct TypeTraits<DictionaryType> {
   constexpr static bool is_parameter_free = false;
 };
 
+template <>
+struct TypeTraits<DecimalType> {
+  // using ArrayType = DecimalArray;
+  constexpr static bool is_parameter_free = false;
+};
+
 // Not all type classes have a c_type
 template <typename T>
 struct as_void {
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index 0ea16bcc73366..586b123e67cfb 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -53,6 +53,7 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
     TYPE_VISIT_INLINE(TimestampType);
     TYPE_VISIT_INLINE(Time32Type);
     TYPE_VISIT_INLINE(Time64Type);
+    TYPE_VISIT_INLINE(DecimalType);
     TYPE_VISIT_INLINE(ListType);
     TYPE_VISIT_INLINE(StructType);
     TYPE_VISIT_INLINE(UnionType);
@@ -91,6 +92,7 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
     ARRAY_VISIT_INLINE(TimestampType);
     ARRAY_VISIT_INLINE(Time32Type);
     ARRAY_VISIT_INLINE(Time64Type);
+    // ARRAY_VISIT_INLINE(DecimalType);
     ARRAY_VISIT_INLINE(ListType);
     ARRAY_VISIT_INLINE(StructType);
     ARRAY_VISIT_INLINE(UnionType);

From 685ebf49001ef02134e404dddae2bfd032dc4a65 Mon Sep 17 00:00:00 2001
From: Jeff Knupp <jeff.knupp@enigma.io>
Date: Sat, 25 Mar 2017 12:15:02 -0400
Subject: [PATCH 0419/1644] ARROW-626: [Python] Replace PyBytesBuffer with
 zero-copy, memoryview-based PyBuffer

WIP, but tests all pass

Author: Jeff Knupp <jeff.knupp@enigma.io>

Closes #410 from jeffknupp/master and squashes the following commits:

bfba71d [Jeff Knupp] Fix typo in test
0a39f24 [Jeff Knupp] Fix some logical issues with initialization; add bytearray test
fb2cfa3 [Jeff Knupp] Add proper reference counting regardless of if buf is memoryview
26f8b74 [Jeff Knupp] Need to investigate why test failed travis but not locally. For now, revert
f7d21ac [Jeff Knupp] ARROW-626: [Python] Replace PyBytesBuffer with zero-copy, memoryview-based PyBuffer
---
 python/pyarrow/includes/pyarrow.pxd |  4 ++--
 python/pyarrow/io.pyx               | 19 ++++++++-----------
 python/pyarrow/tests/test_io.py     | 20 +++++++++++++++-----
 python/src/pyarrow/common.cc        | 24 +++++++++++++++---------
 python/src/pyarrow/common.h         | 10 +++++++---
 python/src/pyarrow/io.cc            |  6 +++---
 python/src/pyarrow/io.h             |  2 +-
 7 files changed, 51 insertions(+), 34 deletions(-)

diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 805950bd1476a..3fdbebc9293cd 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -55,8 +55,8 @@ cdef extern from "pyarrow/api.h" namespace "arrow::py" nogil:
 
 
 cdef extern from "pyarrow/common.h" namespace "arrow::py" nogil:
-    cdef cppclass PyBytesBuffer(CBuffer):
-        PyBytesBuffer(object o)
+    cdef cppclass PyBuffer(CBuffer):
+        PyBuffer(object o)
 
 
 cdef extern from "pyarrow/io.h" namespace "arrow::py" nogil:
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 72e0e0ff01512..cb44ce8816338 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -501,16 +501,11 @@ cdef class BufferReader(NativeFile):
         Buffer buffer
 
     def __cinit__(self, object obj):
-        cdef shared_ptr[CBuffer] buf
 
         if isinstance(obj, Buffer):
             self.buffer = obj
-        elif isinstance(obj, bytes):
-            buf.reset(new pyarrow.PyBytesBuffer(obj))
-            self.buffer = wrap_buffer(buf)
         else:
-            raise ValueError('Unable to convert value to buffer: {0}'
-                             .format(type(obj)))
+            self.buffer = build_arrow_buffer(obj)
 
         self.rd_file.reset(new CBufferReader(self.buffer.buffer))
         self.is_readable = 1
@@ -518,16 +513,18 @@ cdef class BufferReader(NativeFile):
         self.is_open = True
 
 
-def buffer_from_bytes(object obj):
+def build_arrow_buffer(object obj):
     """
     Construct an Arrow buffer from a Python bytes object
     """
     cdef shared_ptr[CBuffer] buf
-    if not isinstance(obj, bytes):
-        raise ValueError('Must pass bytes object')
+    try:
+        memoryview(obj)
+        buf.reset(new pyarrow.PyBuffer(obj))
+        return wrap_buffer(buf)
+    except TypeError:
+        raise ValueError('Must pass object that implements buffer protocol')
 
-    buf.reset(new pyarrow.PyBytesBuffer(obj))
-    return wrap_buffer(buf)
 
 
 cdef Buffer wrap_buffer(const shared_ptr[CBuffer]& buf):
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index c6caba5ce641a..9cd15c4a76cef 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -82,7 +82,6 @@ def test_bytes_reader():
     # Like a BytesIO, but zero-copy underneath for C++ consumers
     data = b'some sample data'
     f = io.BufferReader(data)
-
     assert f.tell() == 0
 
     assert f.size() == len(data)
@@ -128,7 +127,7 @@ def get_buffer():
 def test_buffer_bytes():
     val = b'some data'
 
-    buf = io.buffer_from_bytes(val)
+    buf = io.build_arrow_buffer(val)
     assert isinstance(buf, io.Buffer)
 
     result = buf.to_pybytes()
@@ -138,18 +137,29 @@ def test_buffer_bytes():
 def test_buffer_memoryview():
     val = b'some data'
 
-    buf = io.buffer_from_bytes(val)
+    buf = io.build_arrow_buffer(val)
     assert isinstance(buf, io.Buffer)
 
     result = memoryview(buf)
 
     assert result == val
 
+def test_buffer_bytearray():
+    val = bytearray(b'some data')
+
+
+    buf = io.build_arrow_buffer(val)
+    assert isinstance(buf, io.Buffer)
+
+    result = bytearray(buf)
+
+    assert result == val
+
 
 def test_buffer_memoryview_is_immutable():
     val = b'some data'
 
-    buf = io.buffer_from_bytes(val)
+    buf = io.build_arrow_buffer(val)
     assert isinstance(buf, io.Buffer)
 
     result = memoryview(buf)
@@ -192,7 +202,7 @@ def test_buffer_protocol_ref_counting():
     import gc
 
     def make_buffer(bytes_obj):
-        return bytearray(io.buffer_from_bytes(bytes_obj))
+        return bytearray(io.build_arrow_buffer(bytes_obj))
 
     buf = make_buffer(b'foo')
     gc.collect()
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index c898f634aedbb..792aa4775d4f0 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -45,18 +45,24 @@ MemoryPool* get_memory_pool() {
 }
 
 // ----------------------------------------------------------------------
-// PyBytesBuffer
+// PyBuffer
 
-PyBytesBuffer::PyBytesBuffer(PyObject* obj)
-    : Buffer(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)),
-          PyBytes_GET_SIZE(obj)),
-      obj_(obj) {
-  Py_INCREF(obj_);
+PyBuffer::PyBuffer(PyObject* obj)
+    : Buffer(nullptr, 0) {
+    if (PyObject_CheckBuffer(obj)) {
+        obj_ = PyMemoryView_FromObject(obj);
+        Py_buffer* buffer = PyMemoryView_GET_BUFFER(obj_);
+        data_ = reinterpret_cast<const uint8_t*>(buffer->buf);
+        size_ = buffer->len;
+        capacity_ = buffer->len;
+        is_mutable_ = false;
+        Py_INCREF(obj_);
+    } 
 }
 
-PyBytesBuffer::~PyBytesBuffer() {
-  PyAcquireGIL lock;
-  Py_DECREF(obj_);
+PyBuffer::~PyBuffer() {
+    PyAcquireGIL lock;
+    Py_DECREF(obj_);
 }
 
 }  // namespace py
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 0b4c6bebcfe79..b4e4ea6d2b908 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -118,10 +118,14 @@ class ARROW_EXPORT NumPyBuffer : public Buffer {
   PyArrayObject* arr_;
 };
 
-class ARROW_EXPORT PyBytesBuffer : public Buffer {
+class ARROW_EXPORT PyBuffer : public Buffer {
  public:
-  PyBytesBuffer(PyObject* obj);
-  ~PyBytesBuffer();
+  /// Note that the GIL must be held when calling the PyBuffer constructor.
+  ///
+  /// While memoryview objects support multi-demensional buffers, PyBuffer only supports
+  /// one-dimensional byte buffers.
+  PyBuffer(PyObject* obj);
+  ~PyBuffer();
 
  private:
   PyObject* obj_;
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index 0aa61dc811f5c..c66155b946a64 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -156,7 +156,7 @@ Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   PyObject* bytes_obj;
   ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
 
-  *out = std::make_shared<PyBytesBuffer>(bytes_obj);
+  *out = std::make_shared<PyBuffer>(bytes_obj);
   Py_DECREF(bytes_obj);
 
   return Status::OK();
@@ -210,10 +210,10 @@ Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
 }
 
 // ----------------------------------------------------------------------
-// A readable file that is backed by a PyBytes
+// A readable file that is backed by a PyBuffer
 
 PyBytesReader::PyBytesReader(PyObject* obj)
-    : io::BufferReader(std::make_shared<PyBytesBuffer>(obj)) {}
+    : io::BufferReader(std::make_shared<PyBuffer>(obj)) {}
 
 PyBytesReader::~PyBytesReader() {}
 
diff --git a/python/src/pyarrow/io.h b/python/src/pyarrow/io.h
index e38cd81775255..89af60926ad94 100644
--- a/python/src/pyarrow/io.h
+++ b/python/src/pyarrow/io.h
@@ -84,7 +84,7 @@ class ARROW_EXPORT PyOutputStream : public io::OutputStream {
   std::unique_ptr<PythonFile> file_;
 };
 
-// A zero-copy reader backed by a PyBytes object
+// A zero-copy reader backed by a PyBuffer object
 class ARROW_EXPORT PyBytesReader : public io::BufferReader {
  public:
   explicit PyBytesReader(PyObject* obj);

From ab848f0eab053eeea62d1cf0c0f285db6460da54 Mon Sep 17 00:00:00 2001
From: Jeff Knupp <jeff.knupp@enigma.io>
Date: Sun, 26 Mar 2017 09:19:44 +0200
Subject: [PATCH 0420/1644] ARROW-713: [C++] Fix cmake linking issue in new IPC
 benchmark

Author: Jeff Knupp <jeff.knupp@enigma.io>

Closes #444 from jeffknupp/master and squashes the following commits:

37aa10f [Jeff Knupp] [C++] ARROW-713: Fix cmake linking issue in new IPC benchmark
---
 cpp/src/arrow/ipc/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index d6ee9309b44d8..030cba93f5fc0 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -173,5 +173,5 @@ if (ARROW_BUILD_UTILITIES)
 endif()
 
 ADD_ARROW_BENCHMARK(ipc-read-write-benchmark)
-ARROW_TEST_LINK_LIBRARIES(ipc-read-write-benchmark
+ARROW_BENCHMARK_LINK_LIBRARIES(ipc-read-write-benchmark
   ${ARROW_IPC_TEST_LINK_LIBS})

From fd876697fc37a270a978117f020bf9e07a6c1bad Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 26 Mar 2017 09:21:15 +0200
Subject: [PATCH 0421/1644] ARROW-684: [Python] More helpful error message if
 libparquet_arrow not built

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #443 from wesm/ARROW-684 and squashes the following commits:

c18ca81 [Wes McKinney] More helpful error message if libparquet_arrow not built
---
 python/cmake_modules/FindParquet.cmake | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index 7445e0919acb6..a20b651e2b3c6 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -68,13 +68,21 @@ else ()
   set(PARQUET_ARROW_FOUND FALSE)
 endif ()
 
-if (PARQUET_FOUND)
+if (PARQUET_FOUND AND PARQUET_ARROW_FOUND)
   if (NOT Parquet_FIND_QUIETLY)
     message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}")
+    message(STATUS "Found the Parquet Arrow library: ${PARQUET_ARROW_LIBS}")
   endif ()
 else ()
   if (NOT Parquet_FIND_QUIETLY)
-    set(PARQUET_ERR_MSG "Could not find the Parquet library. Looked in ")
+    if (NOT PARQUET_FOUND)
+      set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet library.")
+    endif()
+
+    if (NOT PARQUET_ARROW_FOUND)
+      set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet_arrow library. Did you build with -DPARQUET_ARROW=on?")
+    endif()
+    set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Looked in ")
     if ( _parquet_roots )
       set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} in ${_parquet_roots}.")
     else ()
@@ -88,12 +96,6 @@ else ()
   endif ()
 endif ()
 
-if (PARQUET_ARROW_FOUND)
-  if (NOT Parquet_FIND_QUIETLY)
-    message(STATUS "Found the Parquet Arrow library: ${PARQUET_ARROW_LIBS}")
-  endif()
-endif()
-
 mark_as_advanced(
   PARQUET_FOUND
   PARQUET_INCLUDE_DIR

From 6d4e862902fd56c93ae394d6bc2938a1c4d1d949 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 26 Mar 2017 11:01:14 -0400
Subject: [PATCH 0422/1644] ARROW-712: [C++] Reimplement Array::Accept as
 inline visitor

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #442 from wesm/more-inline-visitors and squashes the following commits:

69af01a [Wes McKinney] Remove unused member
362dd7e [Wes McKinney] cpplint
1e56f0f [Wes McKinney] Reimplement Array::Accept as inline visitor
---
 cpp/src/arrow/array.cc | 58 +++++++++++++-----------------------------
 cpp/src/arrow/array.h  | 22 +---------------
 2 files changed, 18 insertions(+), 62 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index f1c8bd42c476d..cff0126647647 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -29,6 +29,7 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
 #include "arrow/visitor.h"
+#include "arrow/visitor_inline.h"
 
 namespace arrow {
 
@@ -103,6 +104,10 @@ bool Array::RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
   return are_equal;
 }
 
+Status Array::Validate() const {
+  return Status::OK();
+}
+
 // Last two parameters are in-out parameters
 static inline void ConformSliceParams(
     int64_t array_offset, int64_t array_length, int64_t* offset, int64_t* length) {
@@ -117,10 +122,6 @@ std::shared_ptr<Array> Array::Slice(int64_t offset) const {
   return Slice(offset, slice_length);
 }
 
-Status Array::Validate() const {
-  return Status::OK();
-}
-
 NullArray::NullArray(int64_t length) : Array(null(), length, nullptr, length) {}
 
 std::shared_ptr<Array> NullArray::Slice(int64_t offset, int64_t length) const {
@@ -426,47 +427,22 @@ std::shared_ptr<Array> DictionaryArray::Slice(int64_t offset, int64_t length) co
 }
 
 // ----------------------------------------------------------------------
-// Implement ArrayVisitor methods
-
-Status NullArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
+// Implement Array::Accept as inline visitor
 
-Status BooleanArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
+struct AcceptVirtualVisitor {
+  explicit AcceptVirtualVisitor(ArrayVisitor* visitor) : visitor(visitor) {}
 
-template <typename T>
-Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
+  ArrayVisitor* visitor;
 
-Status BinaryArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-Status StringArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-Status FixedWidthBinaryArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-Status ListArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-Status StructArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
-
-Status UnionArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
-}
+  template <typename T>
+  Status Visit(const T& array) {
+    return visitor->Visit(array);
+  }
+};
 
-Status DictionaryArray::Accept(ArrayVisitor* visitor) const {
-  return visitor->Visit(*this);
+Status Array::Accept(ArrayVisitor* visitor) const {
+  AcceptVirtualVisitor inline_visitor(visitor);
+  return VisitArrayInline(*this, visitor);
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index c73b7a87a4f50..cc0cf98092a8c 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -110,7 +110,7 @@ class ARROW_EXPORT Array {
   /// Defaults to always returning Status::OK. This can be an expensive check.
   virtual Status Validate() const;
 
-  virtual Status Accept(ArrayVisitor* visitor) const = 0;
+  Status Accept(ArrayVisitor* visitor) const;
 
   /// Construct a zero-copy slice of the array with the indicated offset and
   /// length
@@ -151,8 +151,6 @@ class ARROW_EXPORT NullArray : public Array {
 
   explicit NullArray(int64_t length);
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
@@ -196,8 +194,6 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
     return reinterpret_cast<const value_type*>(raw_data_) + offset_;
   }
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
   value_type Value(int64_t i) const { return raw_data()[i]; }
@@ -213,8 +209,6 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0);
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
   bool Value(int64_t i) const {
@@ -262,8 +256,6 @@ class ARROW_EXPORT ListArray : public Array {
     return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
   }
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
@@ -313,8 +305,6 @@ class ARROW_EXPORT BinaryArray : public Array {
 
   Status Validate() const override;
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
@@ -351,8 +341,6 @@ class ARROW_EXPORT StringArray : public BinaryArray {
 
   Status Validate() const override;
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
@@ -379,8 +367,6 @@ class ARROW_EXPORT FixedWidthBinaryArray : public Array {
 
   const uint8_t* raw_data() const { return raw_data_; }
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
@@ -409,8 +395,6 @@ class ARROW_EXPORT StructArray : public Array {
 
   const std::vector<std::shared_ptr<Array>>& fields() const { return children_; }
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
@@ -450,8 +434,6 @@ class ARROW_EXPORT UnionArray : public Array {
 
   const std::vector<std::shared_ptr<Array>>& children() const { return children_; }
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
@@ -496,8 +478,6 @@ class ARROW_EXPORT DictionaryArray : public Array {
 
   const DictionaryType* dict_type() { return dict_type_; }
 
-  Status Accept(ArrayVisitor* visitor) const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:

From 3aac4adef11345f211e4c66467ff758cbc397e43 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 26 Mar 2017 11:45:38 -0400
Subject: [PATCH 0423/1644] ARROW-341: [Python] Move pyarrow's C++ code to the
 main C++ source tree, install libarrow_python and headers

This will enable third parties to link to `libarrow_python`.

For now, the pyarrow build system continues to use CMake -- for the purpose of resolving the thirdparty toolchain we may or may not want to go completely to distutils, but we can sort that out later.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #440 from wesm/ARROW-341 and squashes the following commits:

193bc51 [Wes McKinney] Ensure that '-undefined dynamic_lookup' is passed when linking shared library on OS X
a93496b [Wes McKinney] Add missing backslash
7620f50 [Wes McKinney] Fix cpplint issues
0617c69 [Wes McKinney] Fix LD_LIBRARY_PATH, ARROW_HOME
090c78c [Wes McKinney] Build Arrow library stack specific to active Python version
10e4626 [Wes McKinney] Get Python test suite passing again
cfb7f44 [Wes McKinney] Remove print statement
c1e63dc [Wes McKinney] Scrubbing python/CMakeLists.txt
b80b153 [Wes McKinney] Cleanup, build pandas-test within main test suite
7ef1f81 [Wes McKinney] Start moving python/src/pyarrow tp cpp/src/arrow/python
---
 ci/travis_script_python.sh                    |  26 ++-
 cpp/CMakeLists.txt                            | 115 ++++------
 cpp/cmake_modules/BuildUtils.cmake            |  88 ++++++-
 {python => cpp}/cmake_modules/FindNumPy.cmake |   0
 .../cmake_modules/FindPythonLibsNew.cmake     |   0
 cpp/src/arrow/python/CMakeLists.txt           |  93 ++++++++
 .../pyarrow => cpp/src/arrow/python}/api.h    |  15 +-
 .../src/arrow/python/builtin_convert.cc       |   6 +-
 .../src/arrow/python/builtin_convert.h        |   8 +-
 .../src/arrow/python}/common.cc               |  35 ++-
 .../pyarrow => cpp/src/arrow/python}/common.h |  18 +-
 .../src/arrow/python}/config.cc               |   6 +-
 .../pyarrow => cpp/src/arrow/python}/config.h |  13 +-
 .../src/arrow/python}/do_import_numpy.h       |   0
 .../src/arrow/python}/helpers.cc              |   2 +-
 .../src/arrow/python}/helpers.h               |   0
 .../pyarrow => cpp/src/arrow/python}/io.cc    |   7 +-
 .../src/pyarrow => cpp/src/arrow/python}/io.h |   6 +-
 .../src/arrow/python}/numpy_interop.h         |   2 +-
 .../src/arrow/python}/pandas-test.cc          |   4 +-
 .../src/arrow/python/pandas_convert.cc        |  26 +--
 .../src/arrow/python/pandas_convert.h         |   6 +-
 .../src/arrow/python}/type_traits.h           |   3 +-
 .../src/arrow/python}/util/CMakeLists.txt     |  10 +-
 .../src/arrow/python}/util/datetime.h         |   0
 .../src/arrow/python}/util/test_main.cc       |   4 +-
 python/CMakeLists.txt                         | 215 ++----------------
 python/cmake_modules/FindArrow.cmake          |   9 +
 python/pyarrow/config.pyx                     |  14 +-
 python/pyarrow/includes/pyarrow.pxd           |   6 +-
 python/setup.py                               |  11 +-
 python/src/pyarrow/CMakeLists.txt             |  22 --
 32 files changed, 359 insertions(+), 411 deletions(-)
 rename {python => cpp}/cmake_modules/FindNumPy.cmake (100%)
 rename {python => cpp}/cmake_modules/FindPythonLibsNew.cmake (100%)
 create mode 100644 cpp/src/arrow/python/CMakeLists.txt
 rename {python/src/pyarrow => cpp/src/arrow/python}/api.h (75%)
 rename python/src/pyarrow/adapters/builtin.cc => cpp/src/arrow/python/builtin_convert.cc (99%)
 rename python/src/pyarrow/adapters/builtin.h => cpp/src/arrow/python/builtin_convert.h (90%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/common.cc (69%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/common.h (90%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/config.cc (91%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/config.h (85%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/do_import_numpy.h (100%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/helpers.cc (98%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/helpers.h (100%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/io.cc (98%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/io.h (96%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/numpy_interop.h (97%)
 rename {python/src/pyarrow/adapters => cpp/src/arrow/python}/pandas-test.cc (95%)
 rename python/src/pyarrow/adapters/pandas.cc => cpp/src/arrow/python/pandas_convert.cc (99%)
 rename python/src/pyarrow/adapters/pandas.h => cpp/src/arrow/python/pandas_convert.h (95%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/type_traits.h (99%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/util/CMakeLists.txt (83%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/util/datetime.h (100%)
 rename {python/src/pyarrow => cpp/src/arrow/python}/util/test_main.cc (92%)
 delete mode 100644 python/src/pyarrow/CMakeLists.txt

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 6f4b8e9a090a7..df11209e7c49b 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -23,7 +23,6 @@ export MINICONDA=$HOME/miniconda
 export PATH="$MINICONDA/bin:$PATH"
 
 export ARROW_HOME=$ARROW_CPP_INSTALL
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
 
 pushd $PYTHON_DIR
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
@@ -70,11 +69,31 @@ build_parquet_cpp() {
 
 build_parquet_cpp
 
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PARQUET_HOME/lib
+function build_arrow_libraries() {
+  CPP_BUILD_DIR=$1
+  CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+
+  mkdir $CPP_BUILD_DIR
+  pushd $CPP_BUILD_DIR
+
+  cmake -DARROW_BUILD_TESTS=off \
+        -DARROW_PYTHON=on \
+        -DCMAKE_INSTALL_PREFIX=$2 \
+        $CPP_DIR
+
+  make -j4
+  make install
+
+  popd
+}
 
 python_version_tests() {
   PYTHON_VERSION=$1
   CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
+
+  export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION
+  export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib
+
   conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION
   source activate $CONDA_ENV_DIR
 
@@ -87,6 +106,9 @@ python_version_tests() {
   # Expensive dependencies install from Continuum package repo
   conda install -y pip numpy pandas cython
 
+  # Build C++ libraries
+  build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME
+
   # Other stuff pip install
   pip install -r requirements.txt
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index c04afe47030a5..c77cf601cbd46 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -106,6 +106,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Rely on boost shared libraries where relevant"
     ON)
 
+  option(ARROW_PYTHON
+    "Build the Arrow CPython extensions"
+    OFF)
+
   option(ARROW_SSE3
     "Build Arrow with SSE3"
     ON)
@@ -133,6 +137,7 @@ if(NOT ARROW_BUILD_BENCHMARKS)
   set(NO_BENCHMARKS 1)
 endif()
 
+include(BuildUtils)
 
 ############################################################
 # Compiler flags
@@ -303,6 +308,14 @@ endfunction()
 #
 # Arguments after the test name will be passed to set_tests_properties().
 function(ADD_ARROW_TEST REL_TEST_NAME)
+  set(options)
+  set(single_value_args)
+  set(multi_value_args STATIC_LINK_LIBS)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
   if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
     return()
   endif()
@@ -312,7 +325,13 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
     # This test has a corresponding .cc file, set it up as an executable.
     set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
     add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
-    target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+
+    if (ARG_STATIC_LINK_LIBS)
+      # Customize link libraries
+      target_link_libraries(${TEST_NAME} ${ARG_STATIC_LINK_LIBS})
+    else()
+      target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+    endif()
     add_dependencies(unittest ${TEST_NAME})
   else()
     # No executable, just invoke the test (probably a script) directly.
@@ -332,10 +351,6 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
       ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
   endif()
   set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
-
-  if(ARGN)
-    set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
-  endif()
 endfunction()
 
 # A wrapper for add_dependencies() that is compatible with NO_TESTS.
@@ -363,72 +378,6 @@ enable_testing()
 ############################################################
 # Dependencies
 ############################################################
-function(ADD_THIRDPARTY_LIB LIB_NAME)
-  set(options)
-  set(one_value_args SHARED_LIB STATIC_LIB)
-  set(multi_value_args DEPS)
-  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
-  if(ARG_UNPARSED_ARGUMENTS)
-    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
-  endif()
-
-  if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
-    if(NOT ARG_STATIC_LIB)
-      message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
-    endif()
-
-    SET(AUG_LIB_NAME "${LIB_NAME}_static")
-    add_library(${AUG_LIB_NAME} STATIC IMPORTED)
-    set_target_properties(${AUG_LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
-    message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
-
-    SET(AUG_LIB_NAME "${LIB_NAME}_shared")
-    add_library(${AUG_LIB_NAME} SHARED IMPORTED)
-
-    if(MSVC)
-        # Mark the ”.lib” location as part of a Windows DLL
-        set_target_properties(${AUG_LIB_NAME}
-            PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
-    else()
-        set_target_properties(${AUG_LIB_NAME}
-            PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
-    endif()
-    message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
-  elseif(ARG_STATIC_LIB)
-    add_library(${LIB_NAME} STATIC IMPORTED)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
-    SET(AUG_LIB_NAME "${LIB_NAME}_static")
-    add_library(${AUG_LIB_NAME} STATIC IMPORTED)
-    set_target_properties(${AUG_LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
-    message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
-  elseif(ARG_SHARED_LIB)
-    add_library(${LIB_NAME} SHARED IMPORTED)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
-    SET(AUG_LIB_NAME "${LIB_NAME}_shared")
-    add_library(${AUG_LIB_NAME} SHARED IMPORTED)
-
-    if(MSVC)
-        # Mark the ”.lib” location as part of a Windows DLL
-        set_target_properties(${AUG_LIB_NAME}
-            PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
-    else()
-        set_target_properties(${AUG_LIB_NAME}
-            PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
-    endif()
-    message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
-  else()
-    message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
-  endif()
-
-  if(ARG_DEPS)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
-  endif()
-endfunction()
 
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
@@ -798,8 +747,7 @@ if (${CLANG_FORMAT_FOUND})
   add_custom_target(format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1
     `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h |
     sed -e '/_generated/g' |
-    sed -e '/windows_compatibility.h/g'`
-    `find ${CMAKE_CURRENT_SOURCE_DIR}/../python -name \\*.cc -or -name \\*.h`)
+    sed -e '/windows_compatibility.h/g'`)
 
   # runs clang format and exits with a non-zero exit code if any files need to be reformatted
   add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0
@@ -857,11 +805,9 @@ if(NOT APPLE)
   set(ARROW_SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map")
 endif()
 
-include(BuildUtils)
-
 ADD_ARROW_LIB(arrow
-    SOURCES ${ARROW_SRCS}
-    SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
+  SOURCES ${ARROW_SRCS}
+  SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
 )
 
 add_subdirectory(src/arrow)
@@ -875,6 +821,10 @@ endif()
 #----------------------------------------------------------------------
 # IPC library
 
+if(ARROW_PYTHON)
+  set(ARROW_IPC on)
+endif()
+
 ## Flatbuffers
 if(ARROW_IPC)
   if("$ENV{FLATBUFFERS_HOME}" STREQUAL "")
@@ -908,3 +858,14 @@ if(ARROW_IPC)
 
   add_subdirectory(src/arrow/ipc)
 endif()
+
+if(ARROW_PYTHON)
+  find_package(PythonLibsNew REQUIRED)
+  find_package(NumPy REQUIRED)
+
+  include_directories(SYSTEM
+    ${NUMPY_INCLUDE_DIRS}
+    ${PYTHON_INCLUDE_DIRS})
+
+  add_subdirectory(src/arrow/python)
+endif()
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 78b514c2295ae..c9930418185c7 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -15,6 +15,73 @@
 # specific language governing permissions and limitations
 # under the License.
 
+function(ADD_THIRDPARTY_LIB LIB_NAME)
+  set(options)
+  set(one_value_args SHARED_LIB STATIC_LIB)
+  set(multi_value_args DEPS)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
+    if(NOT ARG_STATIC_LIB)
+      message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
+    endif()
+
+    SET(AUG_LIB_NAME "${LIB_NAME}_static")
+    add_library(${AUG_LIB_NAME} STATIC IMPORTED)
+    set_target_properties(${AUG_LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
+
+    SET(AUG_LIB_NAME "${LIB_NAME}_shared")
+    add_library(${AUG_LIB_NAME} SHARED IMPORTED)
+
+    if(MSVC)
+        # Mark the ”.lib” location as part of a Windows DLL
+        set_target_properties(${AUG_LIB_NAME}
+            PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
+    else()
+        set_target_properties(${AUG_LIB_NAME}
+            PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    endif()
+    message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+  elseif(ARG_STATIC_LIB)
+    add_library(${LIB_NAME} STATIC IMPORTED)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    SET(AUG_LIB_NAME "${LIB_NAME}_static")
+    add_library(${AUG_LIB_NAME} STATIC IMPORTED)
+    set_target_properties(${AUG_LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
+  elseif(ARG_SHARED_LIB)
+    add_library(${LIB_NAME} SHARED IMPORTED)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    SET(AUG_LIB_NAME "${LIB_NAME}_shared")
+    add_library(${AUG_LIB_NAME} SHARED IMPORTED)
+
+    if(MSVC)
+        # Mark the ”.lib” location as part of a Windows DLL
+        set_target_properties(${AUG_LIB_NAME}
+            PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
+    else()
+        set_target_properties(${AUG_LIB_NAME}
+            PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+    endif()
+    message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+  else()
+    message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
+  endif()
+
+  if(ARG_DEPS)
+    set_target_properties(${LIB_NAME}
+      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+  endif()
+endfunction()
+
 function(ADD_ARROW_LIB LIB_NAME)
   set(options)
   set(one_value_args SHARED_LINK_FLAGS)
@@ -45,9 +112,16 @@ function(ADD_ARROW_LIB LIB_NAME)
 
   if (ARROW_BUILD_SHARED)
     add_library(${LIB_NAME}_shared SHARED $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
+
     if(APPLE)
-      set_target_properties(${LIB_NAME}_shared PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+      # On OS X, you can avoid linking at library load time and instead
+      # expecting that the symbols have been loaded separately. This happens
+      # with libpython* where there can be conflicts between system Python and
+      # the Python from a thirdparty distribution
+      set(ARG_SHARED_LINK_FLAGS
+        "-undefined dynamic_lookup ${ARG_SHARED_LINK_FLAGS}")
     endif()
+
     set_target_properties(${LIB_NAME}_shared
       PROPERTIES
       LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
@@ -55,6 +129,7 @@ function(ADD_ARROW_LIB LIB_NAME)
       OUTPUT_NAME ${LIB_NAME}
       VERSION "${ARROW_ABI_VERSION}"
       SOVERSION "${ARROW_SO_VERSION}")
+
     target_link_libraries(${LIB_NAME}_shared
       LINK_PUBLIC ${ARG_SHARED_LINK_LIBS}
       LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS})
@@ -68,28 +143,28 @@ function(ADD_ARROW_LIB LIB_NAME)
         set_target_properties(${LIB_NAME}_shared PROPERTIES
             INSTALL_RPATH ${_lib_install_rpath})
     endif()
-  
+
     install(TARGETS ${LIB_NAME}_shared
       LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
       ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
   endif()
-  
+
   if (ARROW_BUILD_STATIC)
       add_library(${LIB_NAME}_static STATIC $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
     set_target_properties(${LIB_NAME}_static
       PROPERTIES
       LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
       OUTPUT_NAME ${LIB_NAME})
-  
+
   target_link_libraries(${LIB_NAME}_static
       LINK_PUBLIC ${ARG_STATIC_LINK_LIBS}
       LINK_PRIVATE ${ARG_STATIC_PRIVATE_LINK_LIBS})
-  
+
   install(TARGETS ${LIB_NAME}_static
       LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
       ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
   endif()
-  
+
   if (APPLE)
       set_target_properties(${LIB_NAME}_shared
       PROPERTIES
@@ -98,4 +173,3 @@ function(ADD_ARROW_LIB LIB_NAME)
   endif()
 
 endfunction()
-
diff --git a/python/cmake_modules/FindNumPy.cmake b/cpp/cmake_modules/FindNumPy.cmake
similarity index 100%
rename from python/cmake_modules/FindNumPy.cmake
rename to cpp/cmake_modules/FindNumPy.cmake
diff --git a/python/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake
similarity index 100%
rename from python/cmake_modules/FindPythonLibsNew.cmake
rename to cpp/cmake_modules/FindPythonLibsNew.cmake
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
new file mode 100644
index 0000000000000..03f5afc624b34
--- /dev/null
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+
+#######################################
+# arrow_python
+#######################################
+
+if (ARROW_BUILD_TESTS)
+  add_library(arrow_python_test_main STATIC
+	util/test_main.cc)
+
+  if (APPLE)
+	target_link_libraries(arrow_python_test_main
+      gtest
+      dl)
+	set_target_properties(arrow_python_test_main
+      PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  else()
+	target_link_libraries(arrow_python_test_main
+      gtest
+      pthread
+      dl
+	  )
+  endif()
+endif()
+
+set(ARROW_PYTHON_MIN_TEST_LIBS
+  arrow_python_test_main
+  arrow_python_static
+  arrow_ipc_static
+  arrow_io_static
+  arrow_static)
+
+if(NOT APPLE AND ARROW_BUILD_TESTS)
+  ADD_THIRDPARTY_LIB(python
+    SHARED_LIB "${PYTHON_LIBRARIES}")
+  list(APPEND ARROW_PYTHON_MIN_TEST_LIBS python)
+endif()
+
+set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
+
+# ----------------------------------------------------------------------
+
+set(ARROW_PYTHON_SRCS
+  builtin_convert.cc
+  common.cc
+  config.cc
+  helpers.cc
+  io.cc
+  pandas_convert.cc
+)
+
+set(ARROW_PYTHON_SHARED_LINK_LIBS
+  arrow_io_shared
+  arrow_ipc_shared
+  arrow_shared
+)
+
+ADD_ARROW_LIB(arrow_python
+  SOURCES ${ARROW_PYTHON_SRCS}
+  SHARED_LINK_FLAGS ""
+  SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS}
+  STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
+)
+
+install(FILES
+  api.h
+  builtin_convert.h
+  common.h
+  config.h
+  do_import_numpy.h
+  helpers.h
+  io.h
+  numpy_interop.h
+  pandas_convert.h
+  type_traits.h
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python")
+
+# set_target_properties(arrow_python_shared PROPERTIES
+#     INSTALL_RPATH "\$ORIGIN")
+
+if (ARROW_BUILD_TESTS)
+  ADD_ARROW_TEST(pandas-test
+    STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}")
+endif()
diff --git a/python/src/pyarrow/api.h b/cpp/src/arrow/python/api.h
similarity index 75%
rename from python/src/pyarrow/api.h
rename to cpp/src/arrow/python/api.h
index f65cc097f548f..f4f1c0cf9a5d6 100644
--- a/python/src/pyarrow/api.h
+++ b/cpp/src/arrow/python/api.h
@@ -15,12 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef PYARROW_API_H
-#define PYARROW_API_H
+#ifndef ARROW_PYTHON_API_H
+#define ARROW_PYTHON_API_H
 
-#include "pyarrow/helpers.h"
+#include "arrow/python/builtin_convert.h"
+#include "arrow/python/common.h"
+#include "arrow/python/helpers.h"
+#include "arrow/python/io.h"
+#include "arrow/python/pandas_convert.h"
 
-#include "pyarrow/adapters/builtin.h"
-#include "pyarrow/adapters/pandas.h"
-
-#endif  // PYARROW_API_H
+#endif  // ARROW_PYTHON_API_H
diff --git a/python/src/pyarrow/adapters/builtin.cc b/cpp/src/arrow/python/builtin_convert.cc
similarity index 99%
rename from python/src/pyarrow/adapters/builtin.cc
rename to cpp/src/arrow/python/builtin_convert.cc
index 06e098a80369e..9acccc149664b 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -19,13 +19,13 @@
 #include <datetime.h>
 #include <sstream>
 
-#include "pyarrow/adapters/builtin.h"
+#include "arrow/python/builtin_convert.h"
 
 #include "arrow/api.h"
 #include "arrow/status.h"
 
-#include "pyarrow/helpers.h"
-#include "pyarrow/util/datetime.h"
+#include "arrow/python/helpers.h"
+#include "arrow/python/util/datetime.h"
 
 namespace arrow {
 namespace py {
diff --git a/python/src/pyarrow/adapters/builtin.h b/cpp/src/arrow/python/builtin_convert.h
similarity index 90%
rename from python/src/pyarrow/adapters/builtin.h
rename to cpp/src/arrow/python/builtin_convert.h
index 2d45e670628b5..7b50990dd557c 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -18,8 +18,8 @@
 // Functions for converting between CPython built-in data structures and Arrow
 // data structures
 
-#ifndef PYARROW_ADAPTERS_BUILTIN_H
-#define PYARROW_ADAPTERS_BUILTIN_H
+#ifndef ARROW_PYTHON_ADAPTERS_BUILTIN_H
+#define ARROW_PYTHON_ADAPTERS_BUILTIN_H
 
 #include <Python.h>
 
@@ -29,7 +29,7 @@
 
 #include "arrow/util/visibility.h"
 
-#include "pyarrow/common.h"
+#include "arrow/python/common.h"
 
 namespace arrow {
 
@@ -51,4 +51,4 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
 }  // namespace py
 }  // namespace arrow
 
-#endif  // PYARROW_ADAPTERS_BUILTIN_H
+#endif  // ARROW_PYTHON_ADAPTERS_BUILTIN_H
diff --git a/python/src/pyarrow/common.cc b/cpp/src/arrow/python/common.cc
similarity index 69%
rename from python/src/pyarrow/common.cc
rename to cpp/src/arrow/python/common.cc
index 792aa4775d4f0..a5aea30884468 100644
--- a/python/src/pyarrow/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "pyarrow/common.h"
+#include "arrow/python/common.h"
 
 #include <cstdlib>
 #include <mutex>
@@ -28,17 +28,17 @@ namespace arrow {
 namespace py {
 
 static std::mutex memory_pool_mutex;
-static MemoryPool* default_pyarrow_pool = nullptr;
+static MemoryPool* default_python_pool = nullptr;
 
 void set_default_memory_pool(MemoryPool* pool) {
   std::lock_guard<std::mutex> guard(memory_pool_mutex);
-  default_pyarrow_pool = pool;
+  default_python_pool = pool;
 }
 
 MemoryPool* get_memory_pool() {
   std::lock_guard<std::mutex> guard(memory_pool_mutex);
-  if (default_pyarrow_pool) {
-    return default_pyarrow_pool;
+  if (default_python_pool) {
+    return default_python_pool;
   } else {
     return default_memory_pool();
   }
@@ -47,22 +47,21 @@ MemoryPool* get_memory_pool() {
 // ----------------------------------------------------------------------
 // PyBuffer
 
-PyBuffer::PyBuffer(PyObject* obj)
-    : Buffer(nullptr, 0) {
-    if (PyObject_CheckBuffer(obj)) {
-        obj_ = PyMemoryView_FromObject(obj);
-        Py_buffer* buffer = PyMemoryView_GET_BUFFER(obj_);
-        data_ = reinterpret_cast<const uint8_t*>(buffer->buf);
-        size_ = buffer->len;
-        capacity_ = buffer->len;
-        is_mutable_ = false;
-        Py_INCREF(obj_);
-    } 
+PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0) {
+  if (PyObject_CheckBuffer(obj)) {
+    obj_ = PyMemoryView_FromObject(obj);
+    Py_buffer* buffer = PyMemoryView_GET_BUFFER(obj_);
+    data_ = reinterpret_cast<const uint8_t*>(buffer->buf);
+    size_ = buffer->len;
+    capacity_ = buffer->len;
+    is_mutable_ = false;
+    Py_INCREF(obj_);
+  }
 }
 
 PyBuffer::~PyBuffer() {
-    PyAcquireGIL lock;
-    Py_DECREF(obj_);
+  PyAcquireGIL lock;
+  Py_DECREF(obj_);
 }
 
 }  // namespace py
diff --git a/python/src/pyarrow/common.h b/cpp/src/arrow/python/common.h
similarity index 90%
rename from python/src/pyarrow/common.h
rename to cpp/src/arrow/python/common.h
index b4e4ea6d2b908..f1be471cd3a83 100644
--- a/python/src/pyarrow/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -15,10 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef PYARROW_COMMON_H
-#define PYARROW_COMMON_H
+#ifndef ARROW_PYTHON_COMMON_H
+#define ARROW_PYTHON_COMMON_H
 
-#include "pyarrow/config.h"
+#include <string>
+
+#include "arrow/python/config.h"
 
 #include "arrow/buffer.h"
 #include "arrow/util/macros.h"
@@ -47,7 +49,7 @@ class OwnedRef {
  public:
   OwnedRef() : obj_(nullptr) {}
 
-  OwnedRef(PyObject* obj) : obj_(obj) {}
+  explicit OwnedRef(PyObject* obj) : obj_(obj) {}
 
   ~OwnedRef() {
     PyAcquireGIL lock;
@@ -71,7 +73,7 @@ struct PyObjectStringify {
   OwnedRef tmp_obj;
   const char* bytes;
 
-  PyObjectStringify(PyObject* obj) {
+  explicit PyObjectStringify(PyObject* obj) {
     PyObject* bytes_obj;
     if (PyUnicode_Check(obj)) {
       bytes_obj = PyUnicode_AsUTF8String(obj);
@@ -103,7 +105,7 @@ ARROW_EXPORT MemoryPool* get_memory_pool();
 
 class ARROW_EXPORT NumPyBuffer : public Buffer {
  public:
-  NumPyBuffer(PyArrayObject* arr) : Buffer(nullptr, 0) {
+  explicit NumPyBuffer(PyArrayObject* arr) : Buffer(nullptr, 0) {
     arr_ = arr;
     Py_INCREF(arr);
 
@@ -124,7 +126,7 @@ class ARROW_EXPORT PyBuffer : public Buffer {
   ///
   /// While memoryview objects support multi-demensional buffers, PyBuffer only supports
   /// one-dimensional byte buffers.
-  PyBuffer(PyObject* obj);
+  explicit PyBuffer(PyObject* obj);
   ~PyBuffer();
 
  private:
@@ -134,4 +136,4 @@ class ARROW_EXPORT PyBuffer : public Buffer {
 }  // namespace py
 }  // namespace arrow
 
-#endif  // PYARROW_COMMON_H
+#endif  // ARROW_PYTHON_COMMON_H
diff --git a/python/src/pyarrow/config.cc b/cpp/src/arrow/python/config.cc
similarity index 91%
rename from python/src/pyarrow/config.cc
rename to cpp/src/arrow/python/config.cc
index 0be6d962b55ab..2abc4dda6ee17 100644
--- a/python/src/pyarrow/config.cc
+++ b/cpp/src/arrow/python/config.cc
@@ -17,16 +17,16 @@
 
 #include <Python.h>
 
-#include "pyarrow/config.h"
+#include "arrow/python/config.h"
 
 namespace arrow {
 namespace py {
 
-void pyarrow_init() {}
+void Init() {}
 
 PyObject* numpy_nan = nullptr;
 
-void pyarrow_set_numpy_nan(PyObject* obj) {
+void set_numpy_nan(PyObject* obj) {
   Py_INCREF(obj);
   numpy_nan = obj;
 }
diff --git a/python/src/pyarrow/config.h b/cpp/src/arrow/python/config.h
similarity index 85%
rename from python/src/pyarrow/config.h
rename to cpp/src/arrow/python/config.h
index 87fc5c2b290f6..dd554e05b9379 100644
--- a/python/src/pyarrow/config.h
+++ b/cpp/src/arrow/python/config.h
@@ -15,15 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef PYARROW_CONFIG_H
-#define PYARROW_CONFIG_H
+#ifndef ARROW_PYTHON_CONFIG_H
+#define ARROW_PYTHON_CONFIG_H
 
 #include <Python.h>
 
+#include "arrow/python/numpy_interop.h"
 #include "arrow/util/visibility.h"
 
-#include "pyarrow/numpy_interop.h"
-
 #if PY_MAJOR_VERSION >= 3
 #define PyString_Check PyUnicode_Check
 #endif
@@ -35,12 +34,12 @@ ARROW_EXPORT
 extern PyObject* numpy_nan;
 
 ARROW_EXPORT
-void pyarrow_init();
+void Init();
 
 ARROW_EXPORT
-void pyarrow_set_numpy_nan(PyObject* obj);
+void set_numpy_nan(PyObject* obj);
 
 }  // namespace py
 }  // namespace arrow
 
-#endif  // PYARROW_CONFIG_H
+#endif  // ARROW_PYTHON_CONFIG_H
diff --git a/python/src/pyarrow/do_import_numpy.h b/cpp/src/arrow/python/do_import_numpy.h
similarity index 100%
rename from python/src/pyarrow/do_import_numpy.h
rename to cpp/src/arrow/python/do_import_numpy.h
diff --git a/python/src/pyarrow/helpers.cc b/cpp/src/arrow/python/helpers.cc
similarity index 98%
rename from python/src/pyarrow/helpers.cc
rename to cpp/src/arrow/python/helpers.cc
index 43edf8af17fa2..add2d9a222adf 100644
--- a/python/src/pyarrow/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "pyarrow/helpers.h"
+#include "arrow/python/helpers.h"
 
 #include <arrow/api.h>
 
diff --git a/python/src/pyarrow/helpers.h b/cpp/src/arrow/python/helpers.h
similarity index 100%
rename from python/src/pyarrow/helpers.h
rename to cpp/src/arrow/python/helpers.h
diff --git a/python/src/pyarrow/io.cc b/cpp/src/arrow/python/io.cc
similarity index 98%
rename from python/src/pyarrow/io.cc
rename to cpp/src/arrow/python/io.cc
index c66155b946a64..ba82a45411c4c 100644
--- a/python/src/pyarrow/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -15,16 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "pyarrow/io.h"
+#include "arrow/python/io.h"
 
 #include <cstdint>
 #include <cstdlib>
+#include <string>
 
 #include "arrow/io/memory.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 
-#include "pyarrow/common.h"
+#include "arrow/python/common.h"
 
 namespace arrow {
 namespace py {
@@ -166,7 +167,7 @@ Status PyReadableFile::GetSize(int64_t* size) {
   PyAcquireGIL lock;
 
   int64_t current_position;
-  ;
+
   ARROW_RETURN_NOT_OK(file_->Tell(&current_position));
 
   ARROW_RETURN_NOT_OK(file_->Seek(0, 2));
diff --git a/python/src/pyarrow/io.h b/cpp/src/arrow/python/io.h
similarity index 96%
rename from python/src/pyarrow/io.h
rename to cpp/src/arrow/python/io.h
index 89af60926ad94..905bd6c7a6aed 100644
--- a/python/src/pyarrow/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -22,9 +22,9 @@
 #include "arrow/io/memory.h"
 #include "arrow/util/visibility.h"
 
-#include "pyarrow/config.h"
+#include "arrow/python/config.h"
 
-#include "pyarrow/common.h"
+#include "arrow/python/common.h"
 
 namespace arrow {
 
@@ -36,7 +36,7 @@ namespace py {
 // calling any methods
 class PythonFile {
  public:
-  PythonFile(PyObject* file);
+  explicit PythonFile(PyObject* file);
   ~PythonFile();
 
   Status Close();
diff --git a/python/src/pyarrow/numpy_interop.h b/cpp/src/arrow/python/numpy_interop.h
similarity index 97%
rename from python/src/pyarrow/numpy_interop.h
rename to cpp/src/arrow/python/numpy_interop.h
index 57f3328e87078..0a4b425e734f7 100644
--- a/python/src/pyarrow/numpy_interop.h
+++ b/cpp/src/arrow/python/numpy_interop.h
@@ -34,7 +34,7 @@
 
 // This is required to be able to access the NumPy C API properly in C++ files
 // other than this main one
-#define PY_ARRAY_UNIQUE_SYMBOL pyarrow_ARRAY_API
+#define PY_ARRAY_UNIQUE_SYMBOL arrow_ARRAY_API
 #ifndef NUMPY_IMPORT_ARRAY
 #define NO_IMPORT_ARRAY
 #endif
diff --git a/python/src/pyarrow/adapters/pandas-test.cc b/cpp/src/arrow/python/pandas-test.cc
similarity index 95%
rename from python/src/pyarrow/adapters/pandas-test.cc
rename to cpp/src/arrow/python/pandas-test.cc
index e694e790a38d1..ae2527e19c00e 100644
--- a/python/src/pyarrow/adapters/pandas-test.cc
+++ b/cpp/src/arrow/python/pandas-test.cc
@@ -24,17 +24,17 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/python/pandas_convert.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "pyarrow/adapters/pandas.h"
 
 namespace arrow {
 namespace py {
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
-  StringBuilder builder;
+  StringBuilder builder(default_memory_pool());
   const char value[] = {'\xf1', '\0'};
 
   for (int i = 0; i < 1000; ++i) {
diff --git a/python/src/pyarrow/adapters/pandas.cc b/cpp/src/arrow/python/pandas_convert.cc
similarity index 99%
rename from python/src/pyarrow/adapters/pandas.cc
rename to cpp/src/arrow/python/pandas_convert.cc
index a7386cefcdbbf..f2c2415ed2793 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -19,8 +19,8 @@
 
 #include <Python.h>
 
-#include "pyarrow/adapters/pandas.h"
-#include "pyarrow/numpy_interop.h"
+#include "arrow/python/numpy_interop.h"
+#include "arrow/python/pandas_convert.h"
 
 #include <algorithm>
 #include <atomic>
@@ -32,10 +32,16 @@
 #include <string>
 #include <thread>
 #include <unordered_map>
+#include <vector>
 
 #include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/loader.h"
+#include "arrow/python/builtin_convert.h"
+#include "arrow/python/common.h"
+#include "arrow/python/config.h"
+#include "arrow/python/type_traits.h"
+#include "arrow/python/util/datetime.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type_fwd.h"
@@ -43,12 +49,6 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 
-#include "pyarrow/adapters/builtin.h"
-#include "pyarrow/common.h"
-#include "pyarrow/config.h"
-#include "pyarrow/type_traits.h"
-#include "pyarrow/util/datetime.h"
-
 namespace arrow {
 namespace py {
 
@@ -125,7 +125,7 @@ static int64_t ValuesToValidBytes(
 
   // TODO(wesm): striding
   for (int i = 0; i < length; ++i) {
-    valid_bytes[i] = not traits::isnull(values[i]);
+    valid_bytes[i] = !traits::isnull(values[i]);
     if (traits::isnull(values[i])) null_count++;
   }
 
@@ -226,7 +226,7 @@ class PandasConverter : public TypeVisitor {
         type_(type),
         arr_(reinterpret_cast<PyArrayObject*>(ao)),
         mask_(nullptr) {
-    if (mo != nullptr and mo != Py_None) { mask_ = reinterpret_cast<PyArrayObject*>(mo); }
+    if (mo != nullptr && mo != Py_None) { mask_ = reinterpret_cast<PyArrayObject*>(mo); }
     length_ = PyArray_SIZE(arr_);
   }
 
@@ -820,6 +820,7 @@ class PandasBlock {
   OwnedRef placement_arr_;
   int64_t* placement_data_;
 
+ private:
   DISALLOW_COPY_AND_ASSIGN(PandasBlock);
 };
 
@@ -947,7 +948,6 @@ inline Status ConvertListsLike(
   for (int c = 0; c < data.num_chunks(); c++) {
     auto arr = std::static_pointer_cast<ListArray>(data.chunk(c));
 
-    const uint8_t* data_ptr;
     const bool has_nulls = data.null_count() > 0;
     for (int64_t i = 0; i < arr->length(); ++i) {
       if (has_nulls && arr->IsNull(i)) {
@@ -1304,7 +1304,7 @@ class DatetimeTZBlock : public DatetimeBlock {
 template <int ARROW_INDEX_TYPE>
 class CategoricalBlock : public PandasBlock {
  public:
-  CategoricalBlock(int64_t num_rows) : PandasBlock(num_rows, 1) {}
+  explicit CategoricalBlock(int64_t num_rows) : PandasBlock(num_rows, 1) {}
 
   Status Allocate() override {
     constexpr int npy_type = arrow_traits<ARROW_INDEX_TYPE>::npy_type;
@@ -1432,7 +1432,7 @@ using BlockMap = std::unordered_map<int, std::shared_ptr<PandasBlock>>;
 // * placement arrays as we go
 class DataFrameBlockCreator {
  public:
-  DataFrameBlockCreator(const std::shared_ptr<Table>& table) : table_(table) {}
+  explicit DataFrameBlockCreator(const std::shared_ptr<Table>& table) : table_(table) {}
 
   Status Convert(int nthreads, PyObject** output) {
     column_types_.resize(table_->num_columns());
diff --git a/python/src/pyarrow/adapters/pandas.h b/cpp/src/arrow/python/pandas_convert.h
similarity index 95%
rename from python/src/pyarrow/adapters/pandas.h
rename to cpp/src/arrow/python/pandas_convert.h
index 6862339d89baf..a33741efaa492 100644
--- a/python/src/pyarrow/adapters/pandas.h
+++ b/cpp/src/arrow/python/pandas_convert.h
@@ -18,8 +18,8 @@
 // Functions for converting between pandas's NumPy-based data representation
 // and Arrow data structures
 
-#ifndef PYARROW_ADAPTERS_PANDAS_H
-#define PYARROW_ADAPTERS_PANDAS_H
+#ifndef ARROW_PYTHON_ADAPTERS_PANDAS_H
+#define ARROW_PYTHON_ADAPTERS_PANDAS_H
 
 #include <Python.h>
 
@@ -76,4 +76,4 @@ Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
 }  // namespace py
 }  // namespace arrow
 
-#endif  // PYARROW_ADAPTERS_PANDAS_H
+#endif  // ARROW_PYTHON_ADAPTERS_PANDAS_H
diff --git a/python/src/pyarrow/type_traits.h b/cpp/src/arrow/python/type_traits.h
similarity index 99%
rename from python/src/pyarrow/type_traits.h
rename to cpp/src/arrow/python/type_traits.h
index cc65d5ceed9c1..f78dc360095dc 100644
--- a/python/src/pyarrow/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -18,8 +18,9 @@
 #include <Python.h>
 
 #include <cstdint>
+#include <limits>
 
-#include "pyarrow/numpy_interop.h"
+#include "arrow/python/numpy_interop.h"
 
 #include "arrow/builder.h"
 #include "arrow/type.h"
diff --git a/python/src/pyarrow/util/CMakeLists.txt b/cpp/src/arrow/python/util/CMakeLists.txt
similarity index 83%
rename from python/src/pyarrow/util/CMakeLists.txt
rename to cpp/src/arrow/python/util/CMakeLists.txt
index 6cd49cb75a4fb..4cc20f6f4b47e 100644
--- a/python/src/pyarrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/python/util/CMakeLists.txt
@@ -16,21 +16,21 @@
 # under the License.
 
 #######################################
-# pyarrow_test_main
+# arrow/python_test_main
 #######################################
 
 if (PYARROW_BUILD_TESTS)
-  add_library(pyarrow_test_main STATIC
+  add_library(arrow/python_test_main STATIC
 	test_main.cc)
 
   if (APPLE)
-	target_link_libraries(pyarrow_test_main
+	target_link_libraries(arrow/python_test_main
       gtest
       dl)
-	set_target_properties(pyarrow_test_main
+	set_target_properties(arrow/python_test_main
       PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
   else()
-	target_link_libraries(pyarrow_test_main
+	target_link_libraries(arrow/python_test_main
       gtest
       pthread
       dl
diff --git a/python/src/pyarrow/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
similarity index 100%
rename from python/src/pyarrow/util/datetime.h
rename to cpp/src/arrow/python/util/datetime.h
diff --git a/python/src/pyarrow/util/test_main.cc b/cpp/src/arrow/python/util/test_main.cc
similarity index 92%
rename from python/src/pyarrow/util/test_main.cc
rename to cpp/src/arrow/python/util/test_main.cc
index d8d1d030f8f97..c83514d0dbd37 100644
--- a/python/src/pyarrow/util/test_main.cc
+++ b/cpp/src/arrow/python/util/test_main.cc
@@ -19,8 +19,8 @@
 
 #include <gtest/gtest.h>
 
-#include "pyarrow/do_import_numpy.h"
-#include "pyarrow/numpy_interop.h"
+#include "arrow/python/do_import_numpy.h"
+#include "arrow/python/numpy_interop.h"
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index ef874e3d07959..35a1a89ef3164 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -47,9 +47,6 @@ endif()
 
 # Top level cmake dir
 if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
-  option(PYARROW_BUILD_TESTS
-    "Build the PyArrow C++ googletest unit tests"
-    OFF)
   option(PYARROW_BUILD_PARQUET
     "Build the PyArrow Parquet integration"
     OFF)
@@ -57,7 +54,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the PyArrow jemalloc integration"
     OFF)
   option(PYARROW_BUNDLE_ARROW_CPP
-    "Bundle the Arrow C++ libraries" 
+    "Bundle the Arrow C++ libraries"
     OFF)
 endif()
 
@@ -75,6 +72,8 @@ endif(CCACHE_FOUND)
 # Compiler flags
 ############################################################
 
+include(BuildUtils)
+include(CompilerInfo)
 include(SetupCxxFlags)
 
 # Add common flags
@@ -86,8 +85,6 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
 # Suppress Cython warnings
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
 
-# Determine compiler version
-include(CompilerInfo)
 
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
   # Using Clang with ccache causes a bunch of spurious warnings that are
@@ -215,116 +212,9 @@ include_directories(SYSTEM
   ${PYTHON_INCLUDE_DIRS}
   src)
 
-############################################################
-# Testing
-############################################################
-
-# Add a new test case, with or without an executable that should be built.
-#
-# REL_TEST_NAME is the name of the test. It may be a single component
-# (e.g. monotime-test) or contain additional components (e.g.
-# net/net_util-test). Either way, the last component must be a globally
-# unique name.
-#
-# Arguments after the test name will be passed to set_tests_properties().
-function(ADD_PYARROW_TEST REL_TEST_NAME)
-  if(NO_TESTS)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
-    # This test has a corresponding .cc file, set it up as an executable.
-    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-    add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
-    target_link_libraries(${TEST_NAME} ${PYARROW_TEST_LINK_LIBS})
-  else()
-    # No executable, just invoke the test (probably a script) directly.
-    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
-  endif()
-
-  add_test(${TEST_NAME}
-    ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
-  if(ARGN)
-    set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
-  endif()
-endfunction()
-
-# A wrapper for add_dependencies() that is compatible with NO_TESTS.
-function(ADD_PYARROW_TEST_DEPENDENCIES REL_TEST_NAME)
-  if(NO_TESTS)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  add_dependencies(${TEST_NAME} ${ARGN})
-endfunction()
-
-enable_testing()
-
 ############################################################
 # Dependencies
 ############################################################
-function(ADD_THIRDPARTY_LIB LIB_NAME)
-  set(options)
-  set(one_value_args SHARED_LIB STATIC_LIB)
-  set(multi_value_args DEPS)
-  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
-  if(ARG_UNPARSED_ARGUMENTS)
-    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
-  endif()
-
-  if(("${PYARROW_LINK}" STREQUAL "s" AND ARG_STATIC_LIB) OR (NOT ARG_SHARED_LIB))
-    if(NOT ARG_STATIC_LIB)
-      message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
-    endif()
-    add_library(${LIB_NAME} STATIC IMPORTED)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
-    message(STATUS "Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
-  else()
-    add_library(${LIB_NAME} SHARED IMPORTED)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
-    message(STATUS "Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
-  endif()
-
-  if(ARG_DEPS)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
-  endif()
-
-  # Set up an "exported variant" for this thirdparty library (see "Visibility"
-  # above). It's the same as the real target, just with an "_exported" suffix.
-  # We prefer the static archive if it exists (as it's akin to an "internal"
-  # library), but we'll settle for the shared object if we must.
-  #
-  # A shared object exported variant will force any "leaf" library that
-  # transitively depends on it to also depend on it at runtime; this is
-  # desirable for some libraries (e.g. cyrus_sasl).
-  set(LIB_NAME_EXPORTED ${LIB_NAME}_exported)
-  if(ARG_STATIC_LIB)
-    add_library(${LIB_NAME_EXPORTED} STATIC IMPORTED)
-    set_target_properties(${LIB_NAME_EXPORTED}
-      PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
-  else()
-    add_library(${LIB_NAME_EXPORTED} SHARED IMPORTED)
-    set_target_properties(${LIB_NAME_EXPORTED}
-      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
-  endif()
-  if(ARG_DEPS)
-    set_target_properties(${LIB_NAME_EXPORTED}
-      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
-  endif()
-endfunction()
-
-## GMock
-if (PYARROW_BUILD_TESTS)
-  find_package(GTest REQUIRED)
-  include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(gtest
-    STATIC_LIB ${GTEST_STATIC_LIB})
-endif()
 
 ## Parquet
 find_package(Parquet)
@@ -352,6 +242,8 @@ if (PYARROW_BUNDLE_ARROW_CPP)
       COPYONLY)
   SET(ARROW_IPC_SHARED_LIB
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX})
+  SET(ARROW_PYTHON_SHARED_LIB
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
 endif()
 
 ADD_THIRDPARTY_LIB(arrow
@@ -360,66 +252,8 @@ ADD_THIRDPARTY_LIB(arrow_io
   SHARED_LIB ${ARROW_IO_SHARED_LIB})
 ADD_THIRDPARTY_LIB(arrow_ipc
   SHARED_LIB ${ARROW_IPC_SHARED_LIB})
-
-############################################################
-# Linker setup
-############################################################
-
-set(PYARROW_MIN_TEST_LIBS
-  pyarrow_test_main
-  pyarrow)
-
-set(PYARROW_MIN_TEST_LIBS
-  pyarrow_test_main
-  pyarrow
-  ${PYARROW_BASE_LIBS})
-
-if(NOT APPLE AND PYARROW_BUILD_TESTS)
-  ADD_THIRDPARTY_LIB(python
-    SHARED_LIB "${PYTHON_LIBRARIES}")
-  list(APPEND PYARROW_MIN_TEST_LIBS python)
-endif()
-
-set(PYARROW_TEST_LINK_LIBS ${PYARROW_MIN_TEST_LIBS})
-
-############################################################
-# "make ctags" target
-############################################################
-if (UNIX)
-  add_custom_target(ctags ctags -R --languages=c++,c --exclude=thirdparty/installed)
-endif (UNIX)
-
-############################################################
-# "make etags" target
-############################################################
-if (UNIX)
-  add_custom_target(tags etags --members --declarations
-  `find ${CMAKE_CURRENT_SOURCE_DIR}/src
-   -name \\*.cc -or -name \\*.hh -or -name \\*.cpp -or -name \\*.h -or -name \\*.c -or
-   -name \\*.f`)
-  add_custom_target(etags DEPENDS tags)
-endif (UNIX)
-
-############################################################
-# "make cscope" target
-############################################################
-if (UNIX)
-  add_custom_target(cscope find ${CMAKE_CURRENT_SOURCE_DIR}
-  ( -name \\*.cc -or -name \\*.hh -or -name \\*.cpp -or
-    -name \\*.h -or -name \\*.c -or -name \\*.f )
-  -exec echo \"{}\" \; > cscope.files && cscope -q -b VERBATIM)
-endif (UNIX)
-
-############################################################
-# "make lint" target
-############################################################
-if (UNIX)
-  # Full lint
-  add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
-  --verbose=2
-  --filter=-whitespace/comments,-readability/todo,-build/header_guard
-    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h`)
-endif (UNIX)
+ADD_THIRDPARTY_LIB(arrow_python
+  SHARED_LIB ${ARROW_PYTHON_SHARED_LIB})
 
 ############################################################
 # Subdirectories
@@ -429,9 +263,6 @@ if (UNIX)
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 endif()
 
-add_subdirectory(src/pyarrow)
-add_subdirectory(src/pyarrow/util)
-
 set(CYTHON_EXTENSIONS
   array
   config
@@ -444,19 +275,11 @@ set(CYTHON_EXTENSIONS
   table
 )
 
-set(PYARROW_SRCS
-  src/pyarrow/common.cc
-  src/pyarrow/config.cc
-  src/pyarrow/helpers.cc
-  src/pyarrow/io.cc
-  src/pyarrow/adapters/builtin.cc
-  src/pyarrow/adapters/pandas.cc
-)
-
 set(LINK_LIBS
-  arrow
-  arrow_io
-  arrow_ipc
+  arrow_shared
+  arrow_io_shared
+  arrow_ipc_shared
+  arrow_python_shared
 )
 
 if (PYARROW_BUILD_PARQUET)
@@ -497,24 +320,12 @@ if (PYARROW_BUILD_JEMALLOC)
     SHARED_LIB ${ARROW_JEMALLOC_SHARED_LIB})
   set(LINK_LIBS
     ${LINK_LIBS}
-    arrow_jemalloc)
+    arrow_jemalloc_shared)
   set(CYTHON_EXTENSIONS
     ${CYTHON_EXTENSIONS}
     jemalloc)
 endif()
 
-add_library(pyarrow SHARED
-  ${PYARROW_SRCS})
-if (PYARROW_BUNDLE_ARROW_CPP)
-  set_target_properties(pyarrow PROPERTIES
-      INSTALL_RPATH "\$ORIGIN")
-endif()
-target_link_libraries(pyarrow ${LINK_LIBS})
-
-if(APPLE)
-  set_target_properties(pyarrow PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-endif()
-
 ############################################################
 # Setup and build Cython modules
 ############################################################
@@ -555,5 +366,5 @@ foreach(module ${CYTHON_EXTENSIONS})
 
     set_target_properties(${module_name} PROPERTIES
       INSTALL_RPATH ${module_install_rpath})
-    target_link_libraries(${module_name} pyarrow)
+    target_link_libraries(${module_name} ${LINK_LIBS})
 endforeach(module)
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 5d0207d7c7769..5030c9c8ce900 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -57,12 +57,18 @@ find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
+find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+
 if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
   set(ARROW_IO_LIB_NAME libarrow_io)
   set(ARROW_IPC_LIB_NAME libarrow_ipc)
   set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc)
+  set(ARROW_PYTHON_LIB_NAME libarrow_python)
 
   set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
@@ -77,6 +83,9 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_JEMALLOC_LIB_NAME}.a)
   set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
+  set(ARROW_PYTHON_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PYTHON_LIB_NAME}.a)
+  set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
     message(STATUS "Found the Arrow IO library: ${ARROW_IO_LIB_PATH}")
diff --git a/python/pyarrow/config.pyx b/python/pyarrow/config.pyx
index 5ad7cf53261e3..536f27839ae91 100644
--- a/python/pyarrow/config.pyx
+++ b/python/pyarrow/config.pyx
@@ -14,21 +14,21 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-cdef extern from 'pyarrow/do_import_numpy.h':
+cdef extern from 'arrow/python/do_import_numpy.h':
     pass
 
-cdef extern from 'pyarrow/numpy_interop.h' namespace 'arrow::py':
+cdef extern from 'arrow/python/numpy_interop.h' namespace 'arrow::py':
     int import_numpy()
 
-cdef extern from 'pyarrow/config.h' namespace 'arrow::py':
-    void pyarrow_init()
-    void pyarrow_set_numpy_nan(object o)
+cdef extern from 'arrow/python/config.h' namespace 'arrow::py':
+    void Init()
+    void set_numpy_nan(object o)
 
 import_numpy()
-pyarrow_init()
+Init()
 
 import numpy as np
-pyarrow_set_numpy_nan(np.nan)
+set_numpy_nan(np.nan)
 
 import multiprocessing
 import os
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 3fdbebc9293cd..c3fdf4b070ee0 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -25,7 +25,7 @@ from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn,
 cimport pyarrow.includes.libarrow_io as arrow_io
 
 
-cdef extern from "pyarrow/api.h" namespace "arrow::py" nogil:
+cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     shared_ptr[CDataType] GetPrimitiveType(Type type)
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
     CStatus ConvertPySequence(object obj, CMemoryPool* pool,
@@ -53,13 +53,9 @@ cdef extern from "pyarrow/api.h" namespace "arrow::py" nogil:
     void set_default_memory_pool(CMemoryPool* pool)
     CMemoryPool* get_memory_pool()
 
-
-cdef extern from "pyarrow/common.h" namespace "arrow::py" nogil:
     cdef cppclass PyBuffer(CBuffer):
         PyBuffer(object o)
 
-
-cdef extern from "pyarrow/io.h" namespace "arrow::py" nogil:
     cdef cppclass PyReadableFile(arrow_io.RandomAccessFile):
         PyReadableFile(object fo)
 
diff --git a/python/setup.py b/python/setup.py
index 9abf9854af2a8..dae6cb2f078f6 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -186,7 +186,7 @@ def _run_cmake(self):
             # a bit hacky
             build_lib = saved_cwd
 
-        # Move the built libpyarrow library to the place expected by the Python
+        # Move the libraries to the place expected by the Python
         # build
         shared_library_prefix = 'lib'
         if sys.platform == 'darwin':
@@ -203,15 +203,16 @@ def _run_cmake(self):
             pass
 
         def move_lib(lib_name):
-            lib_filename = shared_library_prefix + lib_name + shared_library_suffix
+            lib_filename = (shared_library_prefix + lib_name +
+                            shared_library_suffix)
             shutil.move(pjoin(self.build_type, lib_filename),
                         pjoin(build_lib, 'pyarrow', lib_filename))
 
-        move_lib("pyarrow")
         if self.bundle_arrow_cpp:
             move_lib("arrow")
             move_lib("arrow_io")
             move_lib("arrow_ipc")
+            move_lib("arrow_python")
             if self.with_jemalloc:
                 move_lib("arrow_jemalloc")
             if self.with_parquet:
@@ -227,14 +228,14 @@ def move_lib(lib_name):
                 if self._failure_permitted(name):
                     print('Cython module {0} failure permitted'.format(name))
                     continue
-                raise RuntimeError('libpyarrow C-extension failed to build:',
+                raise RuntimeError('pyarrow C-extension failed to build:',
                                    os.path.abspath(built_path))
 
             ext_path = pjoin(build_lib, self._get_cmake_ext_path(name))
             if os.path.exists(ext_path):
                 os.remove(ext_path)
             self.mkpath(os.path.dirname(ext_path))
-            print('Moving built libpyarrow C-extension', built_path,
+            print('Moving built C-extension', built_path,
                   'to build path', ext_path)
             shutil.move(self.get_ext_built(name), ext_path)
             self._found_names.append(name)
diff --git a/python/src/pyarrow/CMakeLists.txt b/python/src/pyarrow/CMakeLists.txt
deleted file mode 100644
index 9e69718dfa7c7..0000000000000
--- a/python/src/pyarrow/CMakeLists.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#######################################
-# Unit tests
-#######################################
-
-ADD_PYARROW_TEST(adapters/pandas-test)

From d2d27555b4b2f3f0ba26539211bfe8b4d1b52481 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Mar 2017 10:43:56 -0400
Subject: [PATCH 0424/1644] ARROW-658: [C++] Implement a prototype in-memory
 arrow::Tensor type

I haven't implemented much beyond the data container and automatically computing row major strides. If we agree on the basics, then I will implement IPC read/writes of this data structure in a follow up patch.

cc @pcmoritz @robertnishihara @JohanMabille @sylvaincorlay

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #438 from wesm/ARROW-658 and squashes the following commits:

7f82028 [Wes McKinney] Include numeric STL header
8160393 [Wes McKinney] std::accumulate is in algorithm header
bdd4c55 [Wes McKinney] No need to special case 0-dim
471c719 [Wes McKinney] Add test for 0-d tensor. Use std::accumulate in Tensor::size
8d4a13a [Wes McKinney] Make std::vector args const-refs
8bd9716 [Wes McKinney] Add extern templates for numeric tensors
7d805bf [Wes McKinney] cpplint
8b65aea [Wes McKinney] Implement a prototype in-memory arrow::Tensor type
---
 cpp/CMakeLists.txt           |   1 +
 cpp/src/arrow/CMakeLists.txt |   1 +
 cpp/src/arrow/buffer.cc      |   4 -
 cpp/src/arrow/buffer.h       |   7 +-
 cpp/src/arrow/tensor-test.cc |  73 ++++++++++++++++
 cpp/src/arrow/tensor.cc      | 116 +++++++++++++++++++++++++
 cpp/src/arrow/tensor.h       | 158 +++++++++++++++++++++++++++++++++++
 cpp/src/arrow/type_fwd.h     |  13 ++-
 8 files changed, 359 insertions(+), 14 deletions(-)
 create mode 100644 cpp/src/arrow/tensor-test.cc
 create mode 100644 cpp/src/arrow/tensor.cc
 create mode 100644 cpp/src/arrow/tensor.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index c77cf601cbd46..e4c18ca86e4d7 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -792,6 +792,7 @@ set(ARROW_SRCS
   src/arrow/schema.cc
   src/arrow/status.cc
   src/arrow/table.cc
+  src/arrow/tensor.cc
   src/arrow/type.cc
   src/arrow/visitor.cc
 
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 0e83aacaadab5..f965f1d07feef 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -65,6 +65,7 @@ ADD_ARROW_TEST(pretty_print-test)
 ADD_ARROW_TEST(status-test)
 ADD_ARROW_TEST(type-test)
 ADD_ARROW_TEST(table-test)
+ADD_ARROW_TEST(tensor-test)
 
 ADD_ARROW_BENCHMARK(builder-benchmark)
 ADD_ARROW_BENCHMARK(column-benchmark)
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 28edf5e824c1f..be747e1d49504 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -68,10 +68,6 @@ bool Buffer::Equals(const Buffer& other) const {
                                                             static_cast<size_t>(size_))));
 }
 
-std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
-  return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
-}
-
 PoolBuffer::PoolBuffer(MemoryPool* pool) : ResizableBuffer(nullptr, 0) {
   if (pool == nullptr) { pool = default_memory_pool(); }
   pool_ = pool;
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 449bb537d9caa..713d57a1f101d 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -43,7 +43,7 @@ class Status;
 /// of bytes that where allocated for the buffer in total.
 ///
 /// The following invariant is always true: Size < Capacity
-class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
+class ARROW_EXPORT Buffer {
  public:
   Buffer(const uint8_t* data, int64_t size)
       : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
@@ -58,8 +58,6 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
   /// we might add utility methods to help determine if a buffer satisfies this contract.
   Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
 
-  std::shared_ptr<Buffer> get_shared_ptr() { return shared_from_this(); }
-
   bool is_mutable() const { return is_mutable_; }
 
   /// Return true if both buffers are the same size and contain the same bytes
@@ -111,9 +109,6 @@ class ARROW_EXPORT MutableBuffer : public Buffer {
 
   uint8_t* mutable_data() { return mutable_data_; }
 
-  /// Get a read-only view of this buffer
-  std::shared_ptr<Buffer> GetImmutableView();
-
  protected:
   MutableBuffer() : Buffer(nullptr, 0), mutable_data_(nullptr) {}
 
diff --git a/cpp/src/arrow/tensor-test.cc b/cpp/src/arrow/tensor-test.cc
new file mode 100644
index 0000000000000..99a94934c7990
--- /dev/null
+++ b/cpp/src/arrow/tensor-test.cc
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Unit tests for DataType (and subclasses), Field, and Schema
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/buffer.h"
+#include "arrow/tensor.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+TEST(TestTensor, ZeroDim) {
+  const int64_t values = 1;
+  std::vector<int64_t> shape = {};
+
+  using T = int64_t;
+
+  std::shared_ptr<MutableBuffer> buffer;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer));
+
+  Int64Tensor t0(buffer, shape);
+
+  ASSERT_EQ(1, t0.size());
+}
+
+TEST(TestTensor, BasicCtors) {
+  const int64_t values = 24;
+  std::vector<int64_t> shape = {4, 6};
+  std::vector<int64_t> strides = {48, 8};
+  std::vector<std::string> dim_names = {"foo", "bar"};
+
+  using T = int64_t;
+
+  std::shared_ptr<MutableBuffer> buffer;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer));
+
+  Int64Tensor t1(buffer, shape);
+  Int64Tensor t2(buffer, shape, strides);
+  Int64Tensor t3(buffer, shape, strides, dim_names);
+
+  ASSERT_EQ(24, t1.size());
+  ASSERT_TRUE(t1.is_mutable());
+  ASSERT_FALSE(t1.has_dim_names());
+
+  ASSERT_EQ(strides, t1.strides());
+  ASSERT_EQ(strides, t2.strides());
+
+  ASSERT_EQ("foo", t3.dim_name(0));
+  ASSERT_EQ("bar", t3.dim_name(1));
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
new file mode 100644
index 0000000000000..c0d128f563906
--- /dev/null
+++ b/cpp/src/arrow/tensor.cc
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/tensor.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+void ComputeRowMajorStrides(const FixedWidthType& type, const std::vector<int64_t>& shape,
+    std::vector<int64_t>* strides) {
+  int64_t remaining = type.bit_width() / 8;
+  for (int64_t dimsize : shape) {
+    remaining *= dimsize;
+  }
+
+  for (int64_t dimsize : shape) {
+    remaining /= dimsize;
+    strides->push_back(remaining);
+  }
+}
+
+/// Constructor with strides and dimension names
+Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
+    const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
+    const std::vector<std::string>& dim_names)
+    : type_(type), data_(data), shape_(shape), strides_(strides), dim_names_(dim_names) {
+  DCHECK(is_tensor_supported(type->type));
+  if (shape.size() > 0 && strides.size() == 0) {
+    ComputeRowMajorStrides(static_cast<const FixedWidthType&>(*type_), shape, &strides_);
+  }
+}
+
+Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
+    const std::vector<int64_t>& shape, const std::vector<int64_t>& strides)
+    : Tensor(type, data, shape, strides, {}) {}
+
+Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
+    const std::vector<int64_t>& shape)
+    : Tensor(type, data, shape, {}, {}) {}
+
+const std::string& Tensor::dim_name(int i) const {
+  DCHECK_LT(i, static_cast<int>(dim_names_.size()));
+  return dim_names_[i];
+}
+
+int64_t Tensor::size() const {
+  return std::accumulate(
+      shape_.begin(), shape_.end(), 1, std::multiplies<int64_t>());
+}
+
+template <typename T>
+NumericTensor<T>::NumericTensor(const std::shared_ptr<Buffer>& data,
+    const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
+    const std::vector<std::string>& dim_names)
+    : Tensor(TypeTraits<T>::type_singleton(), data, shape, strides, dim_names),
+      raw_data_(nullptr),
+      mutable_raw_data_(nullptr) {
+  if (data_) {
+    raw_data_ = reinterpret_cast<const value_type*>(data_->data());
+    if (data_->is_mutable()) {
+      auto mut_buf = static_cast<MutableBuffer*>(data_.get());
+      mutable_raw_data_ = reinterpret_cast<value_type*>(mut_buf->mutable_data());
+    }
+  }
+}
+
+template <typename T>
+NumericTensor<T>::NumericTensor(
+    const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape)
+    : NumericTensor(data, shape, {}, {}) {}
+
+template <typename T>
+NumericTensor<T>::NumericTensor(const std::shared_ptr<Buffer>& data,
+    const std::vector<int64_t>& shape, const std::vector<int64_t>& strides)
+    : NumericTensor(data, shape, strides, {}) {}
+
+template class NumericTensor<Int8Type>;
+template class NumericTensor<UInt8Type>;
+template class NumericTensor<Int16Type>;
+template class NumericTensor<UInt16Type>;
+template class NumericTensor<Int32Type>;
+template class NumericTensor<UInt32Type>;
+template class NumericTensor<Int64Type>;
+template class NumericTensor<UInt64Type>;
+template class NumericTensor<HalfFloatType>;
+template class NumericTensor<FloatType>;
+template class NumericTensor<DoubleType>;
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
new file mode 100644
index 0000000000000..0059368f7b2d8
--- /dev/null
+++ b/cpp/src/arrow/tensor.h
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TENSOR_H
+#define ARROW_TENSOR_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/buffer.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Buffer;
+class MemoryPool;
+class MutableBuffer;
+class Status;
+
+static inline bool is_tensor_supported(Type::type type_id) {
+  switch (type_id) {
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::UINT64:
+    case Type::INT64:
+    case Type::HALF_FLOAT:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
+class ARROW_EXPORT Tensor {
+ public:
+  virtual ~Tensor() = default;
+
+  /// Constructor with no dimension names or strides, data assumed to be row-major
+  Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
+      const std::vector<int64_t>& shape);
+
+  /// Constructor with non-negative strides
+  Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
+      const std::vector<int64_t>& shape, const std::vector<int64_t>& strides);
+
+  /// Constructor with strides and dimension names
+  Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
+      const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
+      const std::vector<std::string>& dim_names);
+
+  std::shared_ptr<Buffer> data() const { return data_; }
+  const std::vector<int64_t>& shape() const { return shape_; }
+  const std::vector<int64_t>& strides() const { return strides_; }
+
+  const std::string& dim_name(int i) const;
+  bool has_dim_names() const { return shape_.size() > 0 && dim_names_.size() > 0; }
+
+  /// Total number of value cells in the tensor
+  int64_t size() const;
+
+  /// Return true if the underlying data buffer is mutable
+  bool is_mutable() const { return data_->is_mutable(); }
+
+ protected:
+  Tensor() {}
+
+  std::shared_ptr<DataType> type_;
+
+  std::shared_ptr<Buffer> data_;
+
+  std::vector<int64_t> shape_;
+  std::vector<int64_t> strides_;
+
+  /// These names are optional
+  std::vector<std::string> dim_names_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(Tensor);
+};
+
+template <typename T>
+class ARROW_EXPORT NumericTensor : public Tensor {
+ public:
+  using value_type = typename T::c_type;
+
+  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape);
+
+  /// Constructor with non-negative strides
+  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
+      const std::vector<int64_t>& strides);
+
+  /// Constructor with strides and dimension names
+  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
+      const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names);
+
+  const value_type* raw_data() const { return raw_data_; }
+  value_type* raw_data() { return mutable_raw_data_; }
+
+ private:
+  const value_type* raw_data_;
+  value_type* mutable_raw_data_;
+};
+
+// ----------------------------------------------------------------------
+// extern templates and other details
+
+// gcc and clang disagree about how to handle template visibility when you have
+// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif
+
+// Only instantiate these templates once
+extern template class ARROW_EXPORT NumericTensor<Int8Type>;
+extern template class ARROW_EXPORT NumericTensor<UInt8Type>;
+extern template class ARROW_EXPORT NumericTensor<Int16Type>;
+extern template class ARROW_EXPORT NumericTensor<UInt16Type>;
+extern template class ARROW_EXPORT NumericTensor<Int32Type>;
+extern template class ARROW_EXPORT NumericTensor<UInt32Type>;
+extern template class ARROW_EXPORT NumericTensor<Int64Type>;
+extern template class ARROW_EXPORT NumericTensor<UInt64Type>;
+extern template class ARROW_EXPORT NumericTensor<HalfFloatType>;
+extern template class ARROW_EXPORT NumericTensor<FloatType>;
+extern template class ARROW_EXPORT NumericTensor<DoubleType>;
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+}  // namespace arrow
+
+#endif  // ARROW_TENSOR_H
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 201f4e92bb00d..04ddf7e74dd1d 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -30,6 +30,7 @@ struct DataType;
 class Array;
 class ArrayBuilder;
 struct Field;
+class Tensor;
 
 class Buffer;
 class MemoryPool;
@@ -78,10 +79,14 @@ class NumericArray;
 template <typename TypeClass>
 class NumericBuilder;
 
-#define _NUMERIC_TYPE_DECL(KLASS)                 \
-  struct KLASS##Type;                             \
-  using KLASS##Array = NumericArray<KLASS##Type>; \
-  using KLASS##Builder = NumericBuilder<KLASS##Type>;
+template <typename TypeClass>
+class NumericTensor;
+
+#define _NUMERIC_TYPE_DECL(KLASS)                     \
+  struct KLASS##Type;                                 \
+  using KLASS##Array = NumericArray<KLASS##Type>;     \
+  using KLASS##Builder = NumericBuilder<KLASS##Type>; \
+  using KLASS##Tensor = NumericTensor<KLASS##Type>;
 
 _NUMERIC_TYPE_DECL(Int8);
 _NUMERIC_TYPE_DECL(Int16);

From e717d47865038a65a23d80d6d5d6df782d9a8e43 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Mar 2017 23:13:33 -0400
Subject: [PATCH 0425/1644] ARROW-716: [Python] Update README build
 instructions  after moving libpyarrow to C++ tree

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #445 from wesm/ARROW-716 and squashes the following commits:

2608d2b [Wes McKinney] Update README after moving libpyarrow to main C++ source tree
---
 cpp/README.md    | 10 ++++++++++
 python/README.md | 33 +++++++++++++++++++--------------
 2 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 51f1f0606fa3a..b6f0fa0e3531b 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -81,6 +81,16 @@ variables
 * Hadoop: `HADOOP_HOME` (only required for the HDFS I/O extensions)
 * jemalloc: `JEMALLOC_HOME` (only required for the jemalloc-based memory pool)
 
+### Building Python integration library
+
+The `arrow_python` shared library can be built by passing `-DARROW_PYTHON=on`
+to CMake. This must be installed or in your library load path to be able to
+build pyarrow, the Arrow Python bindings.
+
+The Python library must be built against the same Python version for which you
+are building pyarrow, e.g. Python 2.7 or Python 3.6. NumPy must also be
+installed.
+
 ### API documentation
 
 To generate the (html) API documentation, run the following command in the apidoc
diff --git a/python/README.md b/python/README.md
index 88ab17e71730f..25a3a67b83b03 100644
--- a/python/README.md
+++ b/python/README.md
@@ -22,25 +22,30 @@ other traditional Python scientific computing packages.
 
 This project is layered in two pieces:
 
-* pyarrow, a C++ library for easier interoperability between Arrow C++, NumPy,
-  and pandas
-* Cython extensions and pure Python code under arrow/ which expose Arrow C++
+* arrow_python, a library part of the main Arrow C++ project for Python,
+  pandas, and NumPy interoperability
+* Cython extensions and pure Python code under pyarrow/ which expose Arrow C++
   and pyarrow to pure Python users
 
 #### PyArrow Dependencies:
-These are the various projects that PyArrow depends on.
 
-1. **g++ and gcc Version >= 4.8**
-2. **cmake > 2.8.6**
-3. **boost**
-4. **Arrow-cpp and its dependencies**
-
-The Arrow C++ library must be built with all options enabled and installed with
-``ARROW_HOME`` environment variable set to the installation location. Look at
-(https://github.com/apache/arrow/blob/master/cpp/README.md) for instructions.
+To build pyarrow, first build and install Arrow C++ with the Python component
+enabled using `-DARROW_PYTHON=on`, see
+(https://github.com/apache/arrow/blob/master/cpp/README.md) . These components
+must be installed either in the default system location (e.g. `/usr/local`) or
+in a custom `$ARROW_HOME` location.
+
+```shell
+mkdir cpp/build
+pushd cpp/build
+cmake -DARROW_PYTHON=on -DCMAKE_INSTALL_PREFIX=$ARROW_HOME ..
+make -j4
+make install
+```
 
-Ensure PyArrow can locate the Arrow-cpp shared libraries by setting the
-LD_LIBRARY_PATH environment variable.
+If you build with a custom `CMAKE_INSTALL_PREFIX`, during development, you must
+set `ARROW_HOME` as an environment variable and add it to your
+`LD_LIBRARY_PATH` on Linux and OS X:
 
 ```bash
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_HOME/lib

From 3b71d87c5e2a79cc5955e6cb73fa4a5cc906458f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 28 Mar 2017 10:07:44 -0400
Subject: [PATCH 0426/1644] ARROW-620: [C++] Implement JSON integration test
 support for date, time, timestamp, fixed width binary

This also contains some code scrubbing, and uses inline visitors in the JSON reader/writer path

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #446 from wesm/ARROW-620 and squashes the following commits:

46978aa [Wes McKinney] Fix compiler warning
cd714f8 [Wes McKinney] No underscores. Fix bug slicing null buffer
d26ad14 [Wes McKinney] Implement FixedWidthBinary support in JSON reader/writer. Make FWBinaryArray subclass of PrimitiveArray
bd5652e [Wes McKinney] Get date/time/timestamp JSON tests passing. Cleanup, fix large metadata issues
fcbf64a [Wes McKinney] Refactoring, implement record batch round trip fixture for JSON, failing unit tests
---
 cpp/src/arrow/array.cc                   |   5 +-
 cpp/src/arrow/array.h                    |   7 +-
 cpp/src/arrow/compare.cc                 |  14 +-
 cpp/src/arrow/ipc/ipc-json-test.cc       |  43 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  63 +--
 cpp/src/arrow/ipc/json-internal.cc       | 693 ++++++++++++-----------
 cpp/src/arrow/ipc/metadata.cc            |   6 +-
 cpp/src/arrow/ipc/metadata.h             |   4 +-
 cpp/src/arrow/ipc/reader.cc              |   2 +
 cpp/src/arrow/ipc/test-common.h          |  40 +-
 cpp/src/arrow/ipc/writer.cc              |   8 +-
 cpp/src/arrow/tensor.cc                  |   3 +-
 cpp/src/arrow/type.cc                    |  31 +-
 cpp/src/arrow/type.h                     |  71 ++-
 14 files changed, 545 insertions(+), 445 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index cff0126647647..3ea033376fca3 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -283,12 +283,9 @@ std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const
 FixedWidthBinaryArray::FixedWidthBinaryArray(const std::shared_ptr<DataType>& type,
     int64_t length, const std::shared_ptr<Buffer>& data,
     const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
-    : Array(type, length, null_bitmap, null_count, offset),
-      data_(data),
-      raw_data_(nullptr) {
+    : PrimitiveArray(type, length, data, null_bitmap, null_count, offset) {
   DCHECK(type->type == Type::FIXED_WIDTH_BINARY);
   byte_width_ = static_cast<const FixedWidthBinaryType&>(*type).byte_width();
-  if (data) { raw_data_ = data->data(); }
 }
 
 std::shared_ptr<Array> FixedWidthBinaryArray::Slice(
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index cc0cf98092a8c..c0ec571e45983 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -347,7 +347,7 @@ class ARROW_EXPORT StringArray : public BinaryArray {
 // ----------------------------------------------------------------------
 // Fixed width binary
 
-class ARROW_EXPORT FixedWidthBinaryArray : public Array {
+class ARROW_EXPORT FixedWidthBinaryArray : public PrimitiveArray {
  public:
   using TypeClass = FixedWidthBinaryType;
 
@@ -360,9 +360,6 @@ class ARROW_EXPORT FixedWidthBinaryArray : public Array {
     return raw_data_ + (i + offset_) * byte_width_;
   }
 
-  /// Note that this buffer does not account for any slice offset
-  std::shared_ptr<Buffer> data() const { return data_; }
-
   int32_t byte_width() const { return byte_width_; }
 
   const uint8_t* raw_data() const { return raw_data_; }
@@ -371,8 +368,6 @@ class ARROW_EXPORT FixedWidthBinaryArray : public Array {
 
  protected:
   int32_t byte_width_;
-  std::shared_ptr<Buffer> data_;
-  const uint8_t* raw_data_;
 };
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 8274e0f80dc50..3e282f8886623 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -666,14 +666,12 @@ class TypeEqualsVisitor {
     return Status::OK();
   }
 
-  Status Visit(const Time32Type& left) {
-    const auto& right = static_cast<const Time32Type&>(right_);
-    result_ = left.unit == right.unit;
-    return Status::OK();
-  }
-
-  Status Visit(const Time64Type& left) {
-    const auto& right = static_cast<const Time64Type&>(right_);
+  template <typename T>
+  typename std::enable_if<std::is_base_of<TimeType, T>::value ||
+                              std::is_base_of<DateType, T>::value,
+      Status>::type
+  Visit(const T& left) {
+    const auto& right = static_cast<const T&>(right_);
     result_ = left.unit == right.unit;
     return Status::OK();
   }
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index e943ef1558a75..68261ab25a43a 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -75,7 +75,8 @@ void TestArrayRoundTrip(const Array& array) {
   std::shared_ptr<Array> out;
   ASSERT_OK(ReadJsonArray(default_memory_pool(), d, array.type(), &out));
 
-  ASSERT_TRUE(array.Equals(out)) << array_as_json;
+  // std::cout << array_as_json << std::endl;
+  CompareArraysDetailed(0, *out, array);
 }
 
 template <typename T, typename ValueType>
@@ -351,5 +352,45 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
   ASSERT_TRUE(batch->column(1)->Equals(bar));
 }
 
+#define BATCH_CASES()                                                                   \
+  ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
+      &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
+      &MakeStruct, &MakeUnion, &MakeDates, &MakeTimestamps, &MakeTimes, &MakeFWBinary);
+
+class TestJsonRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*> {
+ public:
+  void SetUp() {}
+  void TearDown() {}
+};
+
+void CheckRoundtrip(const RecordBatch& batch) {
+  std::unique_ptr<JsonWriter> writer;
+  ASSERT_OK(JsonWriter::Open(batch.schema(), &writer));
+  ASSERT_OK(writer->WriteRecordBatch(batch));
+
+  std::string result;
+  ASSERT_OK(writer->Finish(&result));
+
+  auto buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(result.c_str()),
+      static_cast<int64_t>(result.size()));
+
+  std::unique_ptr<JsonReader> reader;
+  ASSERT_OK(JsonReader::Open(buffer, &reader));
+
+  std::shared_ptr<RecordBatch> result_batch;
+  ASSERT_OK(reader->GetRecordBatch(0, &result_batch));
+
+  CompareBatch(batch, *result_batch);
+}
+
+TEST_P(TestJsonRoundTrip, RoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
+
+  CheckRoundtrip(*batch);
+}
+
+INSTANTIATE_TEST_CASE_P(TestJsonRoundTrip, TestJsonRoundTrip, BATCH_CASES());
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 086cc68176783..cd3f190fe4a27 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -41,20 +41,6 @@
 namespace arrow {
 namespace ipc {
 
-void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
-  if (!left.schema()->Equals(right.schema())) {
-    FAIL() << "Left schema: " << left.schema()->ToString()
-           << "\nRight schema: " << right.schema()->ToString();
-  }
-  ASSERT_EQ(left.num_columns(), right.num_columns())
-      << left.schema()->ToString() << " result: " << right.schema()->ToString();
-  EXPECT_EQ(left.num_rows(), right.num_rows());
-  for (int i = 0; i < left.num_columns(); ++i) {
-    EXPECT_TRUE(left.column(i)->Equals(right.column(i)))
-        << "Idx: " << i << " Name: " << left.column_name(i);
-  }
-}
-
 using BatchVector = std::vector<std::shared_ptr<RecordBatch>>;
 
 class TestSchemaMetadata : public ::testing::Test {
@@ -85,17 +71,17 @@ class TestSchemaMetadata : public ::testing::Test {
 const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
 
 TEST_F(TestSchemaMetadata, PrimitiveFields) {
-  auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
-  auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>(), false);
-  auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
-  auto f3 = std::make_shared<Field>("f3", std::make_shared<Int64Type>());
-  auto f4 = std::make_shared<Field>("f4", std::make_shared<UInt8Type>());
-  auto f5 = std::make_shared<Field>("f5", std::make_shared<UInt16Type>());
-  auto f6 = std::make_shared<Field>("f6", std::make_shared<UInt32Type>());
-  auto f7 = std::make_shared<Field>("f7", std::make_shared<UInt64Type>());
-  auto f8 = std::make_shared<Field>("f8", std::make_shared<FloatType>());
-  auto f9 = std::make_shared<Field>("f9", std::make_shared<DoubleType>(), false);
-  auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
+  auto f0 = field("f0", std::make_shared<Int8Type>());
+  auto f1 = field("f1", std::make_shared<Int16Type>(), false);
+  auto f2 = field("f2", std::make_shared<Int32Type>());
+  auto f3 = field("f3", std::make_shared<Int64Type>());
+  auto f4 = field("f4", std::make_shared<UInt8Type>());
+  auto f5 = field("f5", std::make_shared<UInt16Type>());
+  auto f6 = field("f6", std::make_shared<UInt32Type>());
+  auto f7 = field("f7", std::make_shared<UInt64Type>());
+  auto f8 = field("f8", std::make_shared<FloatType>());
+  auto f9 = field("f9", std::make_shared<DoubleType>(), false);
+  auto f10 = field("f10", std::make_shared<BooleanType>());
 
   Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
   DictionaryMemo memo;
@@ -105,11 +91,11 @@ TEST_F(TestSchemaMetadata, PrimitiveFields) {
 
 TEST_F(TestSchemaMetadata, NestedFields) {
   auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
-  auto f0 = std::make_shared<Field>("f0", type);
+  auto f0 = field("f0", type);
 
-  std::shared_ptr<StructType> type2(new StructType({std::make_shared<Field>("k1", INT32),
-      std::make_shared<Field>("k2", INT32), std::make_shared<Field>("k3", INT32)}));
-  auto f1 = std::make_shared<Field>("f1", type2);
+  std::shared_ptr<StructType> type2(
+      new StructType({field("k1", INT32), field("k2", INT32), field("k3", INT32)}));
+  auto f1 = field("f1", type2);
 
   Schema schema({f0, f1});
   DictionaryMemo memo;
@@ -172,20 +158,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
     ASSERT_EQ(expected.num_columns(), result.num_columns())
         << expected.schema()->ToString() << " result: " << result.schema()->ToString();
 
-    for (int i = 0; i < expected.num_columns(); ++i) {
-      const auto& left = *expected.column(i);
-      const auto& right = *result.column(i);
-      if (!left.Equals(right)) {
-        std::stringstream pp_result;
-        std::stringstream pp_expected;
-
-        ASSERT_OK(PrettyPrint(left, 0, &pp_expected));
-        ASSERT_OK(PrettyPrint(right, 0, &pp_result));
-
-        FAIL() << "Index: " << i << " Expected: " << pp_expected.str()
-               << "\nGot: " << pp_result.str();
-      }
-    }
+    CompareBatchColumnsDetailed(result, expected);
   }
 
   void CheckRoundtrip(const RecordBatch& batch, int64_t buffer_size) {
@@ -549,7 +522,7 @@ TEST_F(TestIpcRoundTrip, LargeRecordBatch) {
   std::vector<std::shared_ptr<Field>> fields = {f0};
   auto schema = std::make_shared<Schema>(fields);
 
-  RecordBatch batch(schema, 0, {array});
+  RecordBatch batch(schema, length, {array});
 
   std::string path = "test-write-large-record_batch";
 
@@ -562,6 +535,8 @@ TEST_F(TestIpcRoundTrip, LargeRecordBatch) {
   ASSERT_OK(DoLargeRoundTrip(batch, false, &result));
   CheckReadResult(*result, batch);
 
+  ASSERT_EQ(length, result->num_rows());
+
   // Fails if we try to write this with the normal code path
   ASSERT_RAISES(Invalid, DoStandardRoundTrip(batch, false, &result));
 }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 348468006d0b5..95ab011bd087f 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -40,6 +40,7 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/string.h"
+#include "arrow/visitor_inline.h"
 
 namespace arrow {
 namespace ipc {
@@ -63,13 +64,13 @@ static std::string GetBufferTypeName(BufferType type) {
   return "UNKNOWN";
 }
 
-static std::string GetFloatingPrecisionName(FloatingPointMeta::Precision precision) {
+static std::string GetFloatingPrecisionName(FloatingPoint::Precision precision) {
   switch (precision) {
-    case FloatingPointMeta::HALF:
+    case FloatingPoint::HALF:
       return "HALF";
-    case FloatingPointMeta::SINGLE:
+    case FloatingPoint::SINGLE:
       return "SINGLE";
-    case FloatingPointMeta::DOUBLE:
+    case FloatingPoint::DOUBLE:
       return "DOUBLE";
     default:
       break;
@@ -93,7 +94,7 @@ static std::string GetTimeUnitName(TimeUnit unit) {
   return "UNKNOWN";
 }
 
-class JsonSchemaWriter : public TypeVisitor {
+class JsonSchemaWriter {
  public:
   explicit JsonSchemaWriter(const Schema& schema, RjWriter* writer)
       : schema_(schema), writer_(writer) {}
@@ -120,7 +121,7 @@ class JsonSchemaWriter : public TypeVisitor {
     writer_->Bool(field.nullable);
 
     // Visit the type
-    RETURN_NOT_OK(field.type->Accept(this));
+    RETURN_NOT_OK(VisitTypeInline(*field.type, this));
     writer_->EndObject();
 
     return Status::OK();
@@ -139,25 +140,19 @@ class JsonSchemaWriter : public TypeVisitor {
       void>::type
   WriteTypeMetadata(const T& type) {}
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<IntegerMeta, T>::value, void>::type
-  WriteTypeMetadata(const T& type) {
+  void WriteTypeMetadata(const Integer& type) {
     writer_->Key("bitWidth");
     writer_->Int(type.bit_width());
     writer_->Key("isSigned");
     writer_->Bool(type.is_signed());
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<FloatingPointMeta, T>::value, void>::type
-  WriteTypeMetadata(const T& type) {
+  void WriteTypeMetadata(const FloatingPoint& type) {
     writer_->Key("precision");
     writer_->String(GetFloatingPrecisionName(type.precision()));
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<IntervalType, T>::value, void>::type
-  WriteTypeMetadata(const T& type) {
+  void WriteTypeMetadata(const IntervalType& type) {
     writer_->Key("unit");
     switch (type.unit) {
       case IntervalType::Unit::YEAR_MONTH:
@@ -169,28 +164,45 @@ class JsonSchemaWriter : public TypeVisitor {
     }
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<Time32Type, T>::value ||
-                              std::is_base_of<Time64Type, T>::value ||
-                              std::is_base_of<TimestampType, T>::value,
-      void>::type
-  WriteTypeMetadata(const T& type) {
+  void WriteTypeMetadata(const TimestampType& type) {
+    writer_->Key("unit");
+    writer_->String(GetTimeUnitName(type.unit));
+    if (type.timezone.size() > 0) {
+      writer_->Key("timezone");
+      writer_->String(type.timezone);
+    }
+  }
+
+  void WriteTypeMetadata(const TimeType& type) {
     writer_->Key("unit");
     writer_->String(GetTimeUnitName(type.unit));
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<DecimalType, T>::value, void>::type
-  WriteTypeMetadata(const T& type) {
+  void WriteTypeMetadata(const DateType& type) {
+    writer_->Key("unit");
+    switch (type.unit) {
+      case DateUnit::DAY:
+        writer_->String("DAY");
+        break;
+      case DateUnit::MILLI:
+        writer_->String("MILLISECOND");
+        break;
+    }
+  }
+
+  void WriteTypeMetadata(const FixedWidthBinaryType& type) {
+    writer_->Key("byteWidth");
+    writer_->Int(type.byte_width());
+  }
+
+  void WriteTypeMetadata(const DecimalType& type) {
     writer_->Key("precision");
     writer_->Int(type.precision);
     writer_->Key("scale");
     writer_->Int(type.scale);
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<UnionType, T>::value, void>::type
-  WriteTypeMetadata(const T& type) {
+  void WriteTypeMetadata(const UnionType& type) {
     writer_->Key("mode");
     switch (type.mode) {
       case UnionMode::SPARSE:
@@ -268,86 +280,65 @@ class JsonSchemaWriter : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const NullType& type) override { return WritePrimitive("null", type); }
-
-  Status Visit(const BooleanType& type) override { return WritePrimitive("bool", type); }
-
-  Status Visit(const Int8Type& type) override { return WritePrimitive("int", type); }
-
-  Status Visit(const Int16Type& type) override { return WritePrimitive("int", type); }
+  Status Visit(const NullType& type) { return WritePrimitive("null", type); }
 
-  Status Visit(const Int32Type& type) override { return WritePrimitive("int", type); }
+  Status Visit(const BooleanType& type) { return WritePrimitive("bool", type); }
 
-  Status Visit(const Int64Type& type) override { return WritePrimitive("int", type); }
+  Status Visit(const Integer& type) { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt8Type& type) override { return WritePrimitive("int", type); }
-
-  Status Visit(const UInt16Type& type) override { return WritePrimitive("int", type); }
-
-  Status Visit(const UInt32Type& type) override { return WritePrimitive("int", type); }
-
-  Status Visit(const UInt64Type& type) override { return WritePrimitive("int", type); }
-
-  Status Visit(const HalfFloatType& type) override {
-    return WritePrimitive("floatingpoint", type);
-  }
-
-  Status Visit(const FloatType& type) override {
-    return WritePrimitive("floatingpoint", type);
-  }
-
-  Status Visit(const DoubleType& type) override {
+  Status Visit(const FloatingPoint& type) {
     return WritePrimitive("floatingpoint", type);
   }
 
-  Status Visit(const StringType& type) override { return WriteVarBytes("utf8", type); }
-
-  Status Visit(const BinaryType& type) override { return WriteVarBytes("binary", type); }
-
-  // TODO
-  Status Visit(const Date32Type& type) override { return WritePrimitive("date", type); }
+  Status Visit(const DateType& type) { return WritePrimitive("date", type); }
 
-  Status Visit(const Date64Type& type) override { return WritePrimitive("date", type); }
+  Status Visit(const TimeType& type) { return WritePrimitive("time", type); }
 
-  Status Visit(const Time32Type& type) override { return WritePrimitive("time", type); }
+  Status Visit(const StringType& type) { return WriteVarBytes("utf8", type); }
 
-  Status Visit(const Time64Type& type) override { return WritePrimitive("time", type); }
+  Status Visit(const BinaryType& type) { return WriteVarBytes("binary", type); }
 
-  Status Visit(const TimestampType& type) override {
-    return WritePrimitive("timestamp", type);
+  Status Visit(const FixedWidthBinaryType& type) {
+    return WritePrimitive("fixedwidthbinary", type);
   }
 
-  Status Visit(const IntervalType& type) override {
-    return WritePrimitive("interval", type);
-  }
+  Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); }
+
+  Status Visit(const IntervalType& type) { return WritePrimitive("interval", type); }
 
-  Status Visit(const ListType& type) override {
+  Status Visit(const ListType& type) {
     WriteName("list", type);
     RETURN_NOT_OK(WriteChildren(type.children()));
     WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
-  Status Visit(const StructType& type) override {
+  Status Visit(const StructType& type) {
     WriteName("struct", type);
     WriteChildren(type.children());
     WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
-  Status Visit(const UnionType& type) override {
+  Status Visit(const UnionType& type) {
     WriteName("union", type);
     WriteChildren(type.children());
     WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
+  Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
+
+  Status Visit(const DictionaryType& type) {
+    return Status::NotImplemented("dictionary");
+  }
+
  private:
   const Schema& schema_;
   RjWriter* writer_;
 };
 
-class JsonArrayWriter : public ArrayVisitor {
+class JsonArrayWriter {
  public:
   JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
       : name_(name), array_(array), writer_(writer) {}
@@ -362,7 +353,7 @@ class JsonArrayWriter : public ArrayVisitor {
     writer_->Key("count");
     writer_->Int(static_cast<int32_t>(arr.length()));
 
-    RETURN_NOT_OK(arr.Accept(this));
+    RETURN_NOT_OK(VisitArrayInline(arr, this));
 
     writer_->EndObject();
     return Status::OK();
@@ -411,9 +402,15 @@ class JsonArrayWriter : public ArrayVisitor {
     }
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
-  WriteDataValues(const T& arr) {
+  void WriteDataValues(const FixedWidthBinaryArray& arr) {
+    int32_t width = arr.byte_width();
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      const char* buf = reinterpret_cast<const char*>(arr.GetValue(i));
+      writer_->String(HexEncode(buf, width));
+    }
+  }
+
+  void WriteDataValues(const BooleanArray& arr) {
     for (int i = 0; i < arr.length(); ++i) {
       writer_->Bool(arr.Value(i));
     }
@@ -458,23 +455,6 @@ class JsonArrayWriter : public ArrayVisitor {
     writer_->EndArray();
   }
 
-  template <typename T>
-  Status WritePrimitive(const T& array) {
-    WriteValidityField(array);
-    WriteDataField(array);
-    SetNoChildren();
-    return Status::OK();
-  }
-
-  template <typename T>
-  Status WriteVarBytes(const T& array) {
-    WriteValidityField(array);
-    WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length() + 1);
-    WriteDataField(array);
-    SetNoChildren();
-    return Status::OK();
-  }
-
   Status WriteChildren(const std::vector<std::shared_ptr<Field>>& fields,
       const std::vector<std::shared_ptr<Array>>& arrays) {
     writer_->Key("children");
@@ -486,53 +466,48 @@ class JsonArrayWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status Visit(const NullArray& array) override {
+  Status Visit(const NullArray& array) {
     SetNoChildren();
     return Status::OK();
   }
 
-  Status Visit(const BooleanArray& array) override { return WritePrimitive(array); }
-
-  Status Visit(const Int8Array& array) override { return WritePrimitive(array); }
-
-  Status Visit(const Int16Array& array) override { return WritePrimitive(array); }
-
-  Status Visit(const Int32Array& array) override { return WritePrimitive(array); }
-
-  Status Visit(const Int64Array& array) override { return WritePrimitive(array); }
-
-  Status Visit(const UInt8Array& array) override { return WritePrimitive(array); }
-
-  Status Visit(const UInt16Array& array) override { return WritePrimitive(array); }
-
-  Status Visit(const UInt32Array& array) override { return WritePrimitive(array); }
-
-  Status Visit(const UInt64Array& array) override { return WritePrimitive(array); }
-
-  Status Visit(const HalfFloatArray& array) override { return WritePrimitive(array); }
-
-  Status Visit(const FloatArray& array) override { return WritePrimitive(array); }
+  template <typename T>
+  typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value, Status>::type Visit(
+      const T& array) {
+    WriteValidityField(array);
+    WriteDataField(array);
+    SetNoChildren();
+    return Status::OK();
+  }
 
-  Status Visit(const DoubleArray& array) override { return WritePrimitive(array); }
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BinaryArray, T>::value, Status>::type Visit(
+      const T& array) {
+    WriteValidityField(array);
+    WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length() + 1);
+    WriteDataField(array);
+    SetNoChildren();
+    return Status::OK();
+  }
 
-  Status Visit(const StringArray& array) override { return WriteVarBytes(array); }
+  Status Visit(const DecimalArray& array) { return Status::NotImplemented("decimal"); }
 
-  Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); }
+  Status Visit(const DictionaryArray& array) { return Status::NotImplemented("decimal"); }
 
-  Status Visit(const ListArray& array) override {
+  Status Visit(const ListArray& array) {
     WriteValidityField(array);
     WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length() + 1);
     auto type = static_cast<const ListType*>(array.type().get());
     return WriteChildren(type->children(), {array.values()});
   }
 
-  Status Visit(const StructArray& array) override {
+  Status Visit(const StructArray& array) {
     WriteValidityField(array);
     auto type = static_cast<const StructType*>(array.type().get());
     return WriteChildren(type->children(), array.fields());
   }
 
-  Status Visit(const UnionArray& array) override {
+  Status Visit(const UnionArray& array) {
     WriteValidityField(array);
     auto type = static_cast<const UnionType*>(array.type().get());
 
@@ -549,240 +524,256 @@ class JsonArrayWriter : public ArrayVisitor {
   RjWriter* writer_;
 };
 
-class JsonSchemaReader {
- public:
-  explicit JsonSchemaReader(const rj::Value& json_schema) : json_schema_(json_schema) {}
-
-  Status GetSchema(std::shared_ptr<Schema>* schema) {
-    const auto& obj_schema = json_schema_.GetObject();
+static Status GetInteger(
+    const rj::Value::ConstObject& json_type, std::shared_ptr<DataType>* type) {
+  const auto& json_bit_width = json_type.FindMember("bitWidth");
+  RETURN_NOT_INT("bitWidth", json_bit_width, json_type);
 
-    const auto& json_fields = obj_schema.FindMember("fields");
-    RETURN_NOT_ARRAY("fields", json_fields, obj_schema);
+  const auto& json_is_signed = json_type.FindMember("isSigned");
+  RETURN_NOT_BOOL("isSigned", json_is_signed, json_type);
 
-    std::vector<std::shared_ptr<Field>> fields;
-    RETURN_NOT_OK(GetFieldsFromArray(json_fields->value, &fields));
+  bool is_signed = json_is_signed->value.GetBool();
+  int bit_width = json_bit_width->value.GetInt();
 
-    *schema = std::make_shared<Schema>(fields);
-    return Status::OK();
+  switch (bit_width) {
+    case 8:
+      *type = is_signed ? int8() : uint8();
+      break;
+    case 16:
+      *type = is_signed ? int16() : uint16();
+      break;
+    case 32:
+      *type = is_signed ? int32() : uint32();
+      break;
+    case 64:
+      *type = is_signed ? int64() : uint64();
+      break;
+    default:
+      std::stringstream ss;
+      ss << "Invalid bit width: " << bit_width;
+      return Status::Invalid(ss.str());
   }
+  return Status::OK();
+}
 
-  Status GetFieldsFromArray(
-      const rj::Value& obj, std::vector<std::shared_ptr<Field>>* fields) {
-    const auto& values = obj.GetArray();
+static Status GetFloatingPoint(
+    const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  const auto& json_precision = json_type.FindMember("precision");
+  RETURN_NOT_STRING("precision", json_precision, json_type);
 
-    fields->resize(values.Size());
-    for (rj::SizeType i = 0; i < fields->size(); ++i) {
-      RETURN_NOT_OK(GetField(values[i], &(*fields)[i]));
-    }
-    return Status::OK();
+  std::string precision = json_precision->value.GetString();
+
+  if (precision == "DOUBLE") {
+    *type = float64();
+  } else if (precision == "SINGLE") {
+    *type = float32();
+  } else if (precision == "HALF") {
+    *type = float16();
+  } else {
+    std::stringstream ss;
+    ss << "Invalid precision: " << precision;
+    return Status::Invalid(ss.str());
   }
+  return Status::OK();
+}
 
-  Status GetField(const rj::Value& obj, std::shared_ptr<Field>* field) {
-    if (!obj.IsObject()) { return Status::Invalid("Field was not a JSON object"); }
-    const auto& json_field = obj.GetObject();
+static Status GetFixedWidthBinary(
+    const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  const auto& json_byte_width = json_type.FindMember("byteWidth");
+  RETURN_NOT_INT("byteWidth", json_byte_width, json_type);
 
-    const auto& json_name = json_field.FindMember("name");
-    RETURN_NOT_STRING("name", json_name, json_field);
+  int32_t byte_width = json_byte_width->value.GetInt();
+  *type = fixed_width_binary(byte_width);
+  return Status::OK();
+}
 
-    const auto& json_nullable = json_field.FindMember("nullable");
-    RETURN_NOT_BOOL("nullable", json_nullable, json_field);
+static Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  const auto& json_unit = json_type.FindMember("unit");
+  RETURN_NOT_STRING("unit", json_unit, json_type);
 
-    const auto& json_type = json_field.FindMember("type");
-    RETURN_NOT_OBJECT("type", json_type, json_field);
+  std::string unit_str = json_unit->value.GetString();
 
-    const auto& json_children = json_field.FindMember("children");
-    RETURN_NOT_ARRAY("children", json_children, json_field);
+  if (unit_str == "DAY") {
+    *type = date32();
+  } else if (unit_str == "MILLISECOND") {
+    *type = date64();
+  } else {
+    std::stringstream ss;
+    ss << "Invalid date unit: " << unit_str;
+    return Status::Invalid(ss.str());
+  }
+  return Status::OK();
+}
 
-    std::vector<std::shared_ptr<Field>> children;
-    RETURN_NOT_OK(GetFieldsFromArray(json_children->value, &children));
+static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  const auto& json_unit = json_type.FindMember("unit");
+  RETURN_NOT_STRING("unit", json_unit, json_type);
+
+  std::string unit_str = json_unit->value.GetString();
+
+  if (unit_str == "SECOND") {
+    *type = time32(TimeUnit::SECOND);
+  } else if (unit_str == "MILLISECOND") {
+    *type = time32(TimeUnit::MILLI);
+  } else if (unit_str == "MICROSECOND") {
+    *type = time64(TimeUnit::MICRO);
+  } else if (unit_str == "NANOSECOND") {
+    *type = time64(TimeUnit::NANO);
+  } else {
+    std::stringstream ss;
+    ss << "Invalid time unit: " << unit_str;
+    return Status::Invalid(ss.str());
+  }
+  return Status::OK();
+}
 
-    std::shared_ptr<DataType> type;
-    RETURN_NOT_OK(GetType(json_type->value.GetObject(), children, &type));
+static Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  const auto& json_unit = json_type.FindMember("unit");
+  RETURN_NOT_STRING("unit", json_unit, json_type);
+
+  std::string unit_str = json_unit->value.GetString();
+
+  TimeUnit unit;
+  if (unit_str == "SECOND") {
+    unit = TimeUnit::SECOND;
+  } else if (unit_str == "MILLISECOND") {
+    unit = TimeUnit::MILLI;
+  } else if (unit_str == "MICROSECOND") {
+    unit = TimeUnit::MICRO;
+  } else if (unit_str == "NANOSECOND") {
+    unit = TimeUnit::NANO;
+  } else {
+    std::stringstream ss;
+    ss << "Invalid time unit: " << unit_str;
+    return Status::Invalid(ss.str());
+  }
 
-    *field = std::make_shared<Field>(
-        json_name->value.GetString(), type, json_nullable->value.GetBool());
-    return Status::OK();
+  const auto& json_tz = json_type.FindMember("timezone");
+  if (json_tz == json_type.MemberEnd()) {
+    *type = timestamp(unit);
+  } else {
+    *type = timestamp(unit, json_tz->value.GetString());
   }
 
-  Status GetInteger(
-      const rj::Value::ConstObject& json_type, std::shared_ptr<DataType>* type) {
-    const auto& json_bit_width = json_type.FindMember("bitWidth");
-    RETURN_NOT_INT("bitWidth", json_bit_width, json_type);
+  return Status::OK();
+}
 
-    const auto& json_is_signed = json_type.FindMember("isSigned");
-    RETURN_NOT_BOOL("isSigned", json_is_signed, json_type);
+static Status GetUnion(const RjObject& json_type,
+    const std::vector<std::shared_ptr<Field>>& children,
+    std::shared_ptr<DataType>* type) {
+  const auto& json_mode = json_type.FindMember("mode");
+  RETURN_NOT_STRING("mode", json_mode, json_type);
 
-    bool is_signed = json_is_signed->value.GetBool();
-    int bit_width = json_bit_width->value.GetInt();
+  std::string mode_str = json_mode->value.GetString();
+  UnionMode mode;
 
-    switch (bit_width) {
-      case 8:
-        *type = is_signed ? int8() : uint8();
-        break;
-      case 16:
-        *type = is_signed ? int16() : uint16();
-        break;
-      case 32:
-        *type = is_signed ? int32() : uint32();
-        break;
-      case 64:
-        *type = is_signed ? int64() : uint64();
-        break;
-      default:
-        std::stringstream ss;
-        ss << "Invalid bit width: " << bit_width;
-        return Status::Invalid(ss.str());
-    }
-    return Status::OK();
+  if (mode_str == "SPARSE") {
+    mode = UnionMode::SPARSE;
+  } else if (mode_str == "DENSE") {
+    mode = UnionMode::DENSE;
+  } else {
+    std::stringstream ss;
+    ss << "Invalid union mode: " << mode_str;
+    return Status::Invalid(ss.str());
   }
 
-  Status GetFloatingPoint(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-    const auto& json_precision = json_type.FindMember("precision");
-    RETURN_NOT_STRING("precision", json_precision, json_type);
-
-    std::string precision = json_precision->value.GetString();
+  const auto& json_type_codes = json_type.FindMember("typeIds");
+  RETURN_NOT_ARRAY("typeIds", json_type_codes, json_type);
 
-    if (precision == "DOUBLE") {
-      *type = float64();
-    } else if (precision == "SINGLE") {
-      *type = float32();
-    } else if (precision == "HALF") {
-      *type = float16();
-    } else {
-      std::stringstream ss;
-      ss << "Invalid precision: " << precision;
-      return Status::Invalid(ss.str());
-    }
-    return Status::OK();
+  std::vector<uint8_t> type_codes;
+  const auto& id_array = json_type_codes->value.GetArray();
+  for (const rj::Value& val : id_array) {
+    DCHECK(val.IsUint());
+    type_codes.push_back(static_cast<uint8_t>(val.GetUint()));
   }
 
-  Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-    const auto& json_unit = json_type.FindMember("unit");
-    RETURN_NOT_STRING("unit", json_unit, json_type);
-
-    std::string unit_str = json_unit->value.GetString();
-
-    if (unit_str == "SECOND") {
-      *type = time32(TimeUnit::SECOND);
-    } else if (unit_str == "MILLISECOND") {
-      *type = time32(TimeUnit::MILLI);
-    } else if (unit_str == "MICROSECOND") {
-      *type = time64(TimeUnit::MICRO);
-    } else if (unit_str == "NANOSECOND") {
-      *type = time64(TimeUnit::NANO);
-    } else {
-      std::stringstream ss;
-      ss << "Invalid time unit: " << unit_str;
-      return Status::Invalid(ss.str());
-    }
-    return Status::OK();
-  }
+  *type = union_(children, type_codes, mode);
 
-  Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-    const auto& json_unit = json_type.FindMember("unit");
-    RETURN_NOT_STRING("unit", json_unit, json_type);
+  return Status::OK();
+}
 
-    std::string unit_str = json_unit->value.GetString();
+static Status GetType(const RjObject& json_type,
+    const std::vector<std::shared_ptr<Field>>& children,
+    std::shared_ptr<DataType>* type) {
+  const auto& json_type_name = json_type.FindMember("name");
+  RETURN_NOT_STRING("name", json_type_name, json_type);
+
+  std::string type_name = json_type_name->value.GetString();
+
+  if (type_name == "int") {
+    return GetInteger(json_type, type);
+  } else if (type_name == "floatingpoint") {
+    return GetFloatingPoint(json_type, type);
+  } else if (type_name == "bool") {
+    *type = boolean();
+  } else if (type_name == "utf8") {
+    *type = utf8();
+  } else if (type_name == "binary") {
+    *type = binary();
+  } else if (type_name == "fixedwidthbinary") {
+    return GetFixedWidthBinary(json_type, type);
+  } else if (type_name == "null") {
+    *type = null();
+  } else if (type_name == "date") {
+    return GetDate(json_type, type);
+  } else if (type_name == "time") {
+    return GetTime(json_type, type);
+  } else if (type_name == "timestamp") {
+    return GetTimestamp(json_type, type);
+  } else if (type_name == "list") {
+    *type = list(children[0]);
+  } else if (type_name == "struct") {
+    *type = struct_(children);
+  } else {
+    return GetUnion(json_type, children, type);
+  }
+  return Status::OK();
+}
 
-    TimeUnit unit;
-    if (unit_str == "SECOND") {
-      unit = TimeUnit::SECOND;
-    } else if (unit_str == "MILLISECOND") {
-      unit = TimeUnit::MILLI;
-    } else if (unit_str == "MICROSECOND") {
-      unit = TimeUnit::MICRO;
-    } else if (unit_str == "NANOSECOND") {
-      unit = TimeUnit::NANO;
-    } else {
-      std::stringstream ss;
-      ss << "Invalid time unit: " << unit_str;
-      return Status::Invalid(ss.str());
-    }
+static Status GetField(const rj::Value& obj, std::shared_ptr<Field>* field);
 
-    *type = timestamp(unit);
+static Status GetFieldsFromArray(
+    const rj::Value& obj, std::vector<std::shared_ptr<Field>>* fields) {
+  const auto& values = obj.GetArray();
 
-    return Status::OK();
+  fields->resize(values.Size());
+  for (rj::SizeType i = 0; i < fields->size(); ++i) {
+    RETURN_NOT_OK(GetField(values[i], &(*fields)[i]));
   }
+  return Status::OK();
+}
 
-  Status GetUnion(const RjObject& json_type,
-      const std::vector<std::shared_ptr<Field>>& children,
-      std::shared_ptr<DataType>* type) {
-    const auto& json_mode = json_type.FindMember("mode");
-    RETURN_NOT_STRING("mode", json_mode, json_type);
-
-    std::string mode_str = json_mode->value.GetString();
-    UnionMode mode;
+static Status GetField(const rj::Value& obj, std::shared_ptr<Field>* field) {
+  if (!obj.IsObject()) { return Status::Invalid("Field was not a JSON object"); }
+  const auto& json_field = obj.GetObject();
 
-    if (mode_str == "SPARSE") {
-      mode = UnionMode::SPARSE;
-    } else if (mode_str == "DENSE") {
-      mode = UnionMode::DENSE;
-    } else {
-      std::stringstream ss;
-      ss << "Invalid union mode: " << mode_str;
-      return Status::Invalid(ss.str());
-    }
+  const auto& json_name = json_field.FindMember("name");
+  RETURN_NOT_STRING("name", json_name, json_field);
 
-    const auto& json_type_codes = json_type.FindMember("typeIds");
-    RETURN_NOT_ARRAY("typeIds", json_type_codes, json_type);
+  const auto& json_nullable = json_field.FindMember("nullable");
+  RETURN_NOT_BOOL("nullable", json_nullable, json_field);
 
-    std::vector<uint8_t> type_codes;
-    const auto& id_array = json_type_codes->value.GetArray();
-    for (const rj::Value& val : id_array) {
-      DCHECK(val.IsUint());
-      type_codes.push_back(static_cast<uint8_t>(val.GetUint()));
-    }
+  const auto& json_type = json_field.FindMember("type");
+  RETURN_NOT_OBJECT("type", json_type, json_field);
 
-    *type = union_(children, type_codes, mode);
+  const auto& json_children = json_field.FindMember("children");
+  RETURN_NOT_ARRAY("children", json_children, json_field);
 
-    return Status::OK();
-  }
+  std::vector<std::shared_ptr<Field>> children;
+  RETURN_NOT_OK(GetFieldsFromArray(json_children->value, &children));
 
-  Status GetType(const RjObject& json_type,
-      const std::vector<std::shared_ptr<Field>>& children,
-      std::shared_ptr<DataType>* type) {
-    const auto& json_type_name = json_type.FindMember("name");
-    RETURN_NOT_STRING("name", json_type_name, json_type);
-
-    std::string type_name = json_type_name->value.GetString();
-
-    if (type_name == "int") {
-      return GetInteger(json_type, type);
-    } else if (type_name == "floatingpoint") {
-      return GetFloatingPoint(json_type, type);
-    } else if (type_name == "bool") {
-      *type = boolean();
-    } else if (type_name == "utf8") {
-      *type = utf8();
-    } else if (type_name == "binary") {
-      *type = binary();
-    } else if (type_name == "null") {
-      *type = null();
-    } else if (type_name == "date") {
-      // TODO
-      *type = date64();
-    } else if (type_name == "time") {
-      return GetTime(json_type, type);
-    } else if (type_name == "timestamp") {
-      return GetTimestamp(json_type, type);
-    } else if (type_name == "list") {
-      *type = list(children[0]);
-    } else if (type_name == "struct") {
-      *type = struct_(children);
-    } else {
-      return GetUnion(json_type, children, type);
-    }
-    return Status::OK();
-  }
+  std::shared_ptr<DataType> type;
+  RETURN_NOT_OK(GetType(json_type->value.GetObject(), children, &type));
 
- private:
-  const rj::Value& json_schema_;
-};
+  *field = std::make_shared<Field>(
+      json_name->value.GetString(), type, json_nullable->value.GetBool());
+  return Status::OK();
+}
 
 template <typename T>
 inline typename std::enable_if<IsSignedInt<T>::value, typename T::c_type>::type
 UnboxValue(const rj::Value& val) {
-  DCHECK(val.IsInt());
+  DCHECK(val.IsInt64());
   return static_cast<typename T::c_type>(val.GetInt64());
 }
 
@@ -833,8 +824,10 @@ class JsonArrayReader {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
-                              std::is_base_of<BooleanType, T>::value,
+  typename std::enable_if<
+      std::is_base_of<PrimitiveCType, T>::value || std::is_base_of<DateType, T>::value ||
+          std::is_base_of<TimestampType, T>::value ||
+          std::is_base_of<TimeType, T>::value || std::is_base_of<BooleanType, T>::value,
       Status>::type
   ReadArray(const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
       const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
@@ -903,6 +896,47 @@ class JsonArrayReader {
     return builder.Finish(array);
   }
 
+  template <typename T>
+  typename std::enable_if<std::is_base_of<FixedWidthBinaryType, T>::value, Status>::type
+  ReadArray(const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+    FixedWidthBinaryBuilder builder(pool_, type);
+
+    const auto& json_data = json_array.FindMember("DATA");
+    RETURN_NOT_ARRAY("DATA", json_data, json_array);
+
+    const auto& json_data_arr = json_data->value.GetArray();
+
+    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length);
+
+    int32_t byte_width = static_cast<const FixedWidthBinaryType&>(*type).byte_width();
+
+    // Allocate space for parsed values
+    std::shared_ptr<MutableBuffer> byte_buffer;
+    RETURN_NOT_OK(AllocateBuffer(pool_, byte_width, &byte_buffer));
+    uint8_t* byte_buffer_data = byte_buffer->mutable_data();
+
+    for (int i = 0; i < length; ++i) {
+      if (!is_valid[i]) {
+        builder.AppendNull();
+        continue;
+      }
+
+      const rj::Value& val = json_data_arr[i];
+      DCHECK(val.IsString());
+      std::string hex_string = val.GetString();
+      DCHECK_EQ(static_cast<int32_t>(hex_string.size()), byte_width * 2)
+          << "Expected size: " << byte_width * 2 << " got: " << hex_string.size();
+      const char* hex_data = hex_string.c_str();
+
+      for (int32_t j = 0; j < byte_width; ++j) {
+        RETURN_NOT_OK(ParseHexValue(hex_data + j * 2, &byte_buffer_data[j]));
+      }
+      RETURN_NOT_OK(builder.Append(byte_buffer_data));
+    }
+    return builder.Finish(array);
+  }
+
   template <typename T>
   Status GetIntArray(
       const RjArray& json_array, const int32_t length, std::shared_ptr<Buffer>* out) {
@@ -1063,13 +1097,6 @@ class JsonArrayReader {
   case TYPE::type_id:   \
     return ReadArray<TYPE>(json_array, length, is_valid, type, array);
 
-#define NOT_IMPLEMENTED_CASE(TYPE_ENUM)      \
-  case Type::TYPE_ENUM: {                    \
-    std::stringstream ss;                    \
-    ss << type->ToString();                  \
-    return Status::NotImplemented(ss.str()); \
-  }
-
     switch (type->type) {
       TYPE_CASE(NullType);
       TYPE_CASE(BooleanType);
@@ -1086,16 +1113,15 @@ class JsonArrayReader {
       TYPE_CASE(DoubleType);
       TYPE_CASE(StringType);
       TYPE_CASE(BinaryType);
-      NOT_IMPLEMENTED_CASE(DATE32);
-      NOT_IMPLEMENTED_CASE(DATE64);
-      NOT_IMPLEMENTED_CASE(TIMESTAMP);
-      NOT_IMPLEMENTED_CASE(TIME32);
-      NOT_IMPLEMENTED_CASE(TIME64);
-      NOT_IMPLEMENTED_CASE(INTERVAL);
+      TYPE_CASE(FixedWidthBinaryType);
+      TYPE_CASE(Date32Type);
+      TYPE_CASE(Date64Type);
+      TYPE_CASE(TimestampType);
+      TYPE_CASE(Time32Type);
+      TYPE_CASE(Time64Type);
       TYPE_CASE(ListType);
       TYPE_CASE(StructType);
       TYPE_CASE(UnionType);
-      NOT_IMPLEMENTED_CASE(DICTIONARY);
       default:
         std::stringstream ss;
         ss << type->ToString();
@@ -1103,7 +1129,6 @@ class JsonArrayReader {
     }
 
 #undef TYPE_CASE
-#undef NOT_IMPLEMENTED_CASE
 
     return Status::OK();
   }
@@ -1118,8 +1143,16 @@ Status WriteJsonSchema(const Schema& schema, RjWriter* json_writer) {
 }
 
 Status ReadJsonSchema(const rj::Value& json_schema, std::shared_ptr<Schema>* schema) {
-  JsonSchemaReader converter(json_schema);
-  return converter.GetSchema(schema);
+  const auto& obj_schema = json_schema.GetObject();
+
+  const auto& json_fields = obj_schema.FindMember("fields");
+  RETURN_NOT_ARRAY("fields", json_fields, obj_schema);
+
+  std::vector<std::shared_ptr<Field>> fields;
+  RETURN_NOT_OK(GetFieldsFromArray(json_fields->value, &fields));
+
+  *schema = std::make_shared<Schema>(fields);
+  return Status::OK();
 }
 
 Status WriteJsonArray(
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 17af563805792..85dc8b321c41d 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -602,7 +602,7 @@ static Status WriteBuffers(
   return Status::OK();
 }
 
-static Status MakeRecordBatch(FBB& fbb, int32_t length, int64_t body_length,
+static Status MakeRecordBatch(FBB& fbb, int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     RecordBatchOffset* offset) {
   FieldNodeVector fb_nodes;
@@ -615,7 +615,7 @@ static Status MakeRecordBatch(FBB& fbb, int32_t length, int64_t body_length,
   return Status::OK();
 }
 
-Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
+Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out) {
   FBB fbb;
@@ -625,7 +625,7 @@ Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
       fbb, flatbuf::MessageHeader_RecordBatch, record_batch.Union(), body_length, out);
 }
 
-Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
+Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out) {
   FBB fbb;
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 6e903c0a18ef6..f60fb770c3696 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -188,11 +188,11 @@ Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile
 Status WriteSchemaMessage(
     const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out);
 
-Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
+Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out);
 
-Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
+Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out);
 
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 83e03aa0b36b4..03c678ab7e280 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -97,6 +97,8 @@ static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
 
   for (int i = 0; i < schema->num_fields(); ++i) {
     RETURN_NOT_OK(LoadArray(schema->field(i)->type, &context, &arrays[i]));
+    DCHECK_EQ(num_rows, arrays[i]->length())
+        << "Array length did not match record batch length";
   }
 
   *out = std::make_shared<RecordBatch>(schema, num_rows, std::move(arrays));
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 7ee57d2152c1b..994e1283004a9 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -29,6 +29,7 @@
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
 #include "arrow/memory_pool.h"
+#include "arrow/pretty_print.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
@@ -47,6 +48,41 @@ static inline void AssertSchemaEqual(const Schema& lhs, const Schema& rhs) {
   }
 }
 
+static inline void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
+  if (!left.schema()->Equals(right.schema())) {
+    FAIL() << "Left schema: " << left.schema()->ToString()
+           << "\nRight schema: " << right.schema()->ToString();
+  }
+  ASSERT_EQ(left.num_columns(), right.num_columns())
+      << left.schema()->ToString() << " result: " << right.schema()->ToString();
+  EXPECT_EQ(left.num_rows(), right.num_rows());
+  for (int i = 0; i < left.num_columns(); ++i) {
+    EXPECT_TRUE(left.column(i)->Equals(right.column(i)))
+        << "Idx: " << i << " Name: " << left.column_name(i);
+  }
+}
+
+static inline void CompareArraysDetailed(
+    int index, const Array& result, const Array& expected) {
+  if (!expected.Equals(result)) {
+    std::stringstream pp_result;
+    std::stringstream pp_expected;
+
+    ASSERT_OK(PrettyPrint(expected, 0, &pp_expected));
+    ASSERT_OK(PrettyPrint(result, 0, &pp_result));
+
+    FAIL() << "Index: " << index << " Expected: " << pp_expected.str()
+           << "\nGot: " << pp_result.str();
+  }
+}
+
+static inline void CompareBatchColumnsDetailed(
+    const RecordBatch& result, const RecordBatch& expected) {
+  for (int i = 0; i < expected.num_columns(); ++i) {
+    CompareArraysDetailed(i, *result.column(i), *expected.column(i));
+  }
+}
+
 const auto kListInt32 = list(int32());
 const auto kListListInt32 = list(kListInt32);
 
@@ -474,7 +510,7 @@ Status MakeDates(std::shared_ptr<RecordBatch>* out) {
   ArrayFromVector<Date32Type, int32_t>(is_valid, date32_values, &date32_array);
 
   std::vector<int64_t> date64_values = {1489269000000, 1489270000000, 1489271000000,
-      1489272000000, 1489272000000, 1489273000000};
+      1489272000000, 1489272000000, 1489273000000, 1489274000000};
   std::shared_ptr<Array> date64_array;
   ArrayFromVector<Date64Type, int64_t>(is_valid, date64_values, &date64_array);
 
@@ -548,7 +584,7 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Array> a1, a2;
 
   FixedWidthBinaryBuilder b1(default_memory_pool(), f0->type);
-  FixedWidthBinaryBuilder b2(default_memory_pool(), f0->type);
+  FixedWidthBinaryBuilder b2(default_memory_pool(), f1->type);
 
   std::vector<std::string> values1 = {"foo1", "foo2", "foo3", "foo4"};
   AppendValues(is_valid, values1, &b1);
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index e795ef961cb64..da360f31641b8 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -141,7 +141,7 @@ class RecordBatchWriter : public ArrayVisitor {
   virtual Status WriteMetadataMessage(
       int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) {
     return WriteRecordBatchMessage(
-        static_cast<int32_t>(num_rows), body_length, field_nodes_, buffer_meta_, out);
+        num_rows, body_length, field_nodes_, buffer_meta_, out);
   }
 
   Status WriteMetadata(int64_t num_rows, int64_t body_length, io::OutputStream* dst,
@@ -306,7 +306,7 @@ class RecordBatchWriter : public ArrayVisitor {
     auto data = array.data();
     int32_t width = array.byte_width();
 
-    if (array.offset() != 0) {
+    if (data && array.offset() != 0) {
       data = SliceBuffer(data, array.offset() * width, width * array.length());
     }
     buffers_.push_back(data);
@@ -476,8 +476,8 @@ class DictionaryWriter : public RecordBatchWriter {
 
   Status WriteMetadataMessage(
       int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
-    return WriteDictionaryMessage(dictionary_id_, static_cast<int32_t>(num_rows),
-        body_length, field_nodes_, buffer_meta_, out);
+    return WriteDictionaryMessage(
+        dictionary_id_, num_rows, body_length, field_nodes_, buffer_meta_, out);
   }
 
   Status Write(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index c0d128f563906..6489cd01d4c80 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -71,8 +71,7 @@ const std::string& Tensor::dim_name(int i) const {
 }
 
 int64_t Tensor::size() const {
-  return std::accumulate(
-      shape_.begin(), shape_.end(), 1, std::multiplies<int64_t>());
+  return std::accumulate(shape_.begin(), shape_.end(), 1, std::multiplies<int64_t>());
 }
 
 template <typename T>
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 388502214e733..c790f6e5a4345 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -45,8 +45,6 @@ std::string Field::ToString() const {
   return ss.str();
 }
 
-DataType::~DataType() {}
-
 bool DataType::Equals(const DataType& other) const {
   bool are_equal = false;
   Status error = TypeEquals(*this, other, &are_equal);
@@ -63,16 +61,16 @@ std::string BooleanType::ToString() const {
   return name();
 }
 
-FloatingPointMeta::Precision HalfFloatType::precision() const {
-  return FloatingPointMeta::HALF;
+FloatingPoint::Precision HalfFloatType::precision() const {
+  return FloatingPoint::HALF;
 }
 
-FloatingPointMeta::Precision FloatType::precision() const {
-  return FloatingPointMeta::SINGLE;
+FloatingPoint::Precision FloatType::precision() const {
+  return FloatingPoint::SINGLE;
 }
 
-FloatingPointMeta::Precision DoubleType::precision() const {
-  return FloatingPointMeta::DOUBLE;
+FloatingPoint::Precision DoubleType::precision() const {
+  return FloatingPoint::DOUBLE;
 }
 
 std::string StringType::ToString() const {
@@ -111,6 +109,16 @@ std::string StructType::ToString() const {
   return s.str();
 }
 
+// ----------------------------------------------------------------------
+// Date types
+
+DateType::DateType(Type::type type_id, DateUnit unit)
+    : FixedWidthType(type_id), unit(unit) {}
+
+Date32Type::Date32Type() : DateType(Type::DATE32, DateUnit::DAY) {}
+
+Date64Type::Date64Type() : DateType(Type::DATE64, DateUnit::MILLI) {}
+
 std::string Date64Type::ToString() const {
   return std::string("date64[ms]");
 }
@@ -122,7 +130,10 @@ std::string Date32Type::ToString() const {
 // ----------------------------------------------------------------------
 // Time types
 
-Time32Type::Time32Type(TimeUnit unit) : FixedWidthType(Type::TIME32), unit(unit) {
+TimeType::TimeType(Type::type type_id, TimeUnit unit)
+    : FixedWidthType(type_id), unit(unit) {}
+
+Time32Type::Time32Type(TimeUnit unit) : TimeType(Type::TIME32, unit) {
   DCHECK(unit == TimeUnit::SECOND || unit == TimeUnit::MILLI)
       << "Must be seconds or milliseconds";
 }
@@ -133,7 +144,7 @@ std::string Time32Type::ToString() const {
   return ss.str();
 }
 
-Time64Type::Time64Type(TimeUnit unit) : FixedWidthType(Type::TIME64), unit(unit) {
+Time64Type::Time64Type(TimeUnit unit) : TimeType(Type::TIME64, unit) {
   DCHECK(unit == TimeUnit::MICRO || unit == TimeUnit::NANO)
       << "Must be microseconds or nanoseconds";
 }
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 7ae5ae3c4b72e..dc50ecd669cae 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -132,7 +132,7 @@ struct ARROW_EXPORT DataType {
 
   explicit DataType(Type::type type) : type(type) {}
 
-  virtual ~DataType();
+  virtual ~DataType() = default;
 
   // Return whether the types are equal
   //
@@ -167,11 +167,17 @@ struct ARROW_EXPORT FixedWidthType : public DataType {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-struct ARROW_EXPORT IntegerMeta {
+struct ARROW_EXPORT PrimitiveCType : public FixedWidthType {
+  using FixedWidthType::FixedWidthType;
+};
+
+struct ARROW_EXPORT Integer : public PrimitiveCType {
+  using PrimitiveCType::PrimitiveCType;
   virtual bool is_signed() const = 0;
 };
 
-struct ARROW_EXPORT FloatingPointMeta {
+struct ARROW_EXPORT FloatingPoint : public PrimitiveCType {
+  using PrimitiveCType::PrimitiveCType;
   enum Precision { HALF, SINGLE, DOUBLE };
   virtual Precision precision() const = 0;
 };
@@ -206,16 +212,12 @@ struct ARROW_EXPORT Field {
 
 typedef std::shared_ptr<Field> FieldPtr;
 
-struct ARROW_EXPORT PrimitiveCType : public FixedWidthType {
-  using FixedWidthType::FixedWidthType;
-};
-
-template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
-struct ARROW_EXPORT CTypeImpl : public PrimitiveCType {
+template <typename DERIVED, typename BASE, Type::type TYPE_ID, typename C_TYPE>
+struct ARROW_EXPORT CTypeImpl : public BASE {
   using c_type = C_TYPE;
   static constexpr Type::type type_id = TYPE_ID;
 
-  CTypeImpl() : PrimitiveCType(TYPE_ID) {}
+  CTypeImpl() : BASE(TYPE_ID) {}
 
   int bit_width() const override { return static_cast<int>(sizeof(C_TYPE) * 8); }
 
@@ -240,7 +242,7 @@ struct ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
 };
 
 template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
-struct IntegerTypeImpl : public CTypeImpl<DERIVED, TYPE_ID, C_TYPE>, public IntegerMeta {
+struct IntegerTypeImpl : public CTypeImpl<DERIVED, Integer, TYPE_ID, C_TYPE> {
   bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
 };
 
@@ -292,20 +294,19 @@ struct ARROW_EXPORT Int64Type : public IntegerTypeImpl<Int64Type, Type::INT64, i
 };
 
 struct ARROW_EXPORT HalfFloatType
-    : public CTypeImpl<HalfFloatType, Type::HALF_FLOAT, uint16_t>,
-      public FloatingPointMeta {
+    : public CTypeImpl<HalfFloatType, FloatingPoint, Type::HALF_FLOAT, uint16_t> {
   Precision precision() const override;
   static std::string name() { return "halffloat"; }
 };
 
-struct ARROW_EXPORT FloatType : public CTypeImpl<FloatType, Type::FLOAT, float>,
-                                public FloatingPointMeta {
+struct ARROW_EXPORT FloatType
+    : public CTypeImpl<FloatType, FloatingPoint, Type::FLOAT, float> {
   Precision precision() const override;
   static std::string name() { return "float"; }
 };
 
-struct ARROW_EXPORT DoubleType : public CTypeImpl<DoubleType, Type::DOUBLE, double>,
-                                 public FloatingPointMeta {
+struct ARROW_EXPORT DoubleType
+    : public CTypeImpl<DoubleType, FloatingPoint, Type::DOUBLE, double> {
   Precision precision() const override;
   static std::string name() { return "double"; }
 };
@@ -436,13 +437,23 @@ struct ARROW_EXPORT UnionType : public NestedType {
 // ----------------------------------------------------------------------
 // Date and time types
 
+enum class DateUnit : char { DAY = 0, MILLI = 1 };
+
+struct DateType : public FixedWidthType {
+ public:
+  DateUnit unit;
+
+ protected:
+  DateType(Type::type type_id, DateUnit unit);
+};
+
 /// Date as int32_t days since UNIX epoch
-struct ARROW_EXPORT Date32Type : public FixedWidthType, public NoExtraMeta {
+struct ARROW_EXPORT Date32Type : public DateType {
   static constexpr Type::type type_id = Type::DATE32;
 
   using c_type = int32_t;
 
-  Date32Type() : FixedWidthType(Type::DATE32) {}
+  Date32Type();
 
   int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 
@@ -451,12 +462,12 @@ struct ARROW_EXPORT Date32Type : public FixedWidthType, public NoExtraMeta {
 };
 
 /// Date as int64_t milliseconds since UNIX epoch
-struct ARROW_EXPORT Date64Type : public FixedWidthType, public NoExtraMeta {
+struct ARROW_EXPORT Date64Type : public DateType {
   static constexpr Type::type type_id = Type::DATE64;
 
   using c_type = int64_t;
 
-  Date64Type() : FixedWidthType(Type::DATE64) {}
+  Date64Type();
 
   int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
 
@@ -485,13 +496,18 @@ static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) {
   return os;
 }
 
-struct ARROW_EXPORT Time32Type : public FixedWidthType {
+struct TimeType : public FixedWidthType {
+ public:
+  TimeUnit unit;
+
+ protected:
+  TimeType(Type::type type_id, TimeUnit unit);
+};
+
+struct ARROW_EXPORT Time32Type : public TimeType {
   static constexpr Type::type type_id = Type::TIME32;
-  using Unit = TimeUnit;
   using c_type = int32_t;
 
-  TimeUnit unit;
-
   int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
 
   explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);
@@ -500,13 +516,10 @@ struct ARROW_EXPORT Time32Type : public FixedWidthType {
   std::string ToString() const override;
 };
 
-struct ARROW_EXPORT Time64Type : public FixedWidthType {
+struct ARROW_EXPORT Time64Type : public TimeType {
   static constexpr Type::type type_id = Type::TIME64;
-  using Unit = TimeUnit;
   using c_type = int64_t;
 
-  TimeUnit unit;
-
   int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
 
   explicit Time64Type(TimeUnit unit = TimeUnit::MILLI);

From dac648db7053bc3cd71e9c64b69edc8959d8ec62 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Tue, 28 Mar 2017 14:21:59 -0700
Subject: [PATCH 0427/1644] ARROW-701: [Java] Support Additional Date Type
 Metadata

The format for Date type now includes metadata for units as DAYS or MILLISECONDS.  This change adds DateUnit and support for usage in metadata.  Includes round-trip JSON testing.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #431 from BryanCutler/java-date_unit-metadata-ARROW-701 and squashes the following commits:

cdbcbfd [Bryan Cutler] Added support for DateUnit metadata
---
 .../src/main/codegen/data/ArrowTypes.tdd      |  2 +-
 .../apache/arrow/vector/types/DateUnit.java   | 44 +++++++++++++++++++
 .../org/apache/arrow/vector/types/Types.java  |  4 +-
 .../arrow/vector/types/pojo/TestSchema.java   |  7 +--
 4 files changed, 51 insertions(+), 6 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/DateUnit.java

diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 94fe31e8dc0d8..67785ad6b4d19 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -54,7 +54,7 @@
     },
     {
       name: "Date",
-      fields: []
+      fields: [{name: "unit", type: short, valueType: DateUnit}]
     },
     {
       name: "Time",
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/DateUnit.java b/java/vector/src/main/java/org/apache/arrow/vector/types/DateUnit.java
new file mode 100644
index 0000000000000..e5beebffde9e4
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/DateUnit.java
@@ -0,0 +1,44 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.types;
+
+public enum DateUnit {
+  DAY(org.apache.arrow.flatbuf.DateUnit.DAY),
+  MILLISECOND(org.apache.arrow.flatbuf.DateUnit.MILLISECOND);
+
+  private static final DateUnit[] valuesByFlatbufId = new DateUnit[DateUnit.values().length];
+  static {
+    for (DateUnit v : DateUnit.values()) {
+      valuesByFlatbufId[v.flatbufID] = v;
+    }
+  }
+
+  private final short flatbufID;
+
+  DateUnit(short flatbufID) {
+    this.flatbufID = flatbufID;
+  }
+
+  public short getFlatbufID() {
+    return flatbufID;
+  }
+
+  public static DateUnit fromFlatbufID(short id) {
+    return valuesByFlatbufId[id];
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 81743b51917a1..2f070237101d8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -107,7 +107,7 @@ public class Types {
   private static final Field UINT2_FIELD = new Field("", true, new Int(16, false), null);
   private static final Field UINT4_FIELD = new Field("", true, new Int(32, false), null);
   private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
-  private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
+  private static final Field DATE_FIELD = new Field("", true, new Date(DateUnit.MILLISECOND), null);
   private static final Field TIME_FIELD = new Field("", true, new Time(TimeUnit.MILLISECOND, 32), null);
   private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND, "UTC"), null);
   private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null);
@@ -219,7 +219,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new BigIntWriterImpl((NullableBigIntVector) vector);
       }
     },
-    DATE(Date.INSTANCE) {
+    DATE(new Date(DateUnit.MILLISECOND)) {
       @Override
       public Field getField() {
         return DATE_FIELD;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 57af9528c5933..45f3b5656d861 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 
+import org.apache.arrow.vector.types.DateUnit;
 import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.IntervalUnit;
 import org.apache.arrow.vector.types.TimeUnit;
@@ -60,7 +61,7 @@ public void testComplex() throws IOException {
         field("b", new Struct(),
             field("c", new Int(16, true)),
             field("d", new Utf8())),
-        field("e", new List(), field(null, new Date())),
+        field("e", new List(), field(null, new Date(DateUnit.MILLISECOND))),
         field("f", new FloatingPoint(FloatingPointPrecision.SINGLE)),
         field("g", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
         field("h", new Timestamp(TimeUnit.MICROSECOND, null)),
@@ -68,7 +69,7 @@ public void testComplex() throws IOException {
         ));
     roundTrip(schema);
     assertEquals(
-        "Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>, e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND, UTC), h: Timestamp(MICROSECOND, null), i: Interval(DAY_TIME)>",
+        "Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>, e: List<Date(MILLISECOND)>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND, UTC), h: Timestamp(MICROSECOND, null), i: Interval(DAY_TIME)>",
         schema.toString());
   }
 
@@ -85,7 +86,7 @@ public void testAll() throws IOException {
         field("h", new Binary()),
         field("i", new Bool()),
         field("j", new Decimal(5, 5)),
-        field("k", new Date()),
+        field("k", new Date(DateUnit.MILLISECOND)),
         field("l", new Time(TimeUnit.MILLISECOND, 32)),
         field("m", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
         field("n", new Timestamp(TimeUnit.MICROSECOND, null)),

From b03236360a5ba04078d5ec1129a13f9e905f0626 Mon Sep 17 00:00:00 2001
From: Itai Incze <itai.in@gmail.com>
Date: Wed, 29 Mar 2017 14:55:39 -0400
Subject: [PATCH 0428/1644] ARROW-732: [C++] Schema comparison bugs in struct
 and union types

Found 2 small bugs in comparing the nested subfields in compare.cc

Fixed  and added a new test to type-test

Author: Itai Incze <itai.in@gmail.com>

Closes #450 from itaiin/master and squashes the following commits:

fd6e5cf [Itai Incze] Fixed schema comparison bug for union types
44a068c [Itai Incze] Fixed: nested schema comparison bug
---
 cpp/src/arrow/compare.cc   | 12 ++++++++++--
 cpp/src/arrow/type-test.cc | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 3e282f8886623..f786222f7e4f2 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -650,7 +650,7 @@ class TypeEqualsVisitor {
     for (int i = 0; i < left.num_children(); ++i) {
       if (!left.child(i)->Equals(right_.child(i))) {
         result_ = false;
-        break;
+        return Status::OK();
       }
     }
     result_ = true;
@@ -712,9 +712,17 @@ class TypeEqualsVisitor {
     for (size_t i = 0; i < left_codes.size(); ++i) {
       if (left_codes[i] != right_codes[i]) {
         result_ = false;
-        break;
+        return Status::OK();
+      }
+    }
+
+    for (int i = 0; i < left.num_children(); ++i) {
+      if (!left.child(i)->Equals(right_.child(i))) {
+        result_ = false;
+        return Status::OK();
       }
     }
+
     result_ = true;
     return Status::OK();
   }
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index b6a84df339e6e..7f13f8ba480b4 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -231,4 +231,41 @@ TEST(TestTimestampType, ToString) {
   ASSERT_EQ("timestamp[us]", t4->ToString());
 }
 
+TEST(TestNestedType, Equals) {
+  auto create_struct =
+      [](std::string inner_name, std::string struct_name) -> shared_ptr<Field> {
+    auto f_type = field(inner_name, int32());
+    vector<shared_ptr<Field>> fields = {f_type};
+    auto s_type = std::make_shared<StructType>(fields);
+    return field(struct_name, s_type);
+  };
+
+  auto create_union =
+      [](std::string inner_name, std::string union_name) -> shared_ptr<Field> {
+    auto f_type = field(inner_name, int32());
+    vector<shared_ptr<Field>> fields = {f_type};
+    vector<uint8_t> codes = {Type::INT32};
+    auto u_type = std::make_shared<UnionType>(fields, codes, UnionMode::SPARSE);
+    return field(union_name, u_type);
+  };
+
+  auto s0 = create_struct("f0", "s0");
+  auto s0_other = create_struct("f0", "s0");
+  auto s0_bad = create_struct("f1", "s0");
+  auto s1 = create_struct("f1", "s1");
+
+  ASSERT_TRUE(s0->Equals(s0_other));
+  ASSERT_FALSE(s0->Equals(s1));
+  ASSERT_FALSE(s0->Equals(s0_bad));
+
+  auto u0 = create_union("f0", "u0");
+  auto u0_other = create_union("f0", "u0");
+  auto u0_bad = create_union("f1", "u0");
+  auto u1 = create_union("f1", "u1");
+
+  ASSERT_TRUE(u0->Equals(u0_other));
+  ASSERT_FALSE(u0->Equals(u1));
+  ASSERT_FALSE(u0->Equals(u0_bad));
+}
+
 }  // namespace arrow

From 8f386374eca26d0eebe562beac52fc75459f352c Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 29 Mar 2017 15:03:20 -0400
Subject: [PATCH 0429/1644] ARROW-731: [C++] Add shared library related
 versions to .pc

They can be used to find real shared library path in parquet-cpp.

See also https://github.com/apache/parquet-cpp/pull/276#issuecomment-289816148

Author: Kouhei Sutou <kou@clear-code.com>

Closes #451 from kou/cpp-add-soversion-to-pc and squashes the following commits:

f657a88 [Kouhei Sutou] [C++] Add shared library related versions to .pc
---
 cpp/src/arrow/arrow.pc.in                   | 3 +++
 cpp/src/arrow/io/arrow-io.pc.in             | 3 +++
 cpp/src/arrow/ipc/arrow-ipc.pc.in           | 3 +++
 cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in | 3 +++
 4 files changed, 12 insertions(+)

diff --git a/cpp/src/arrow/arrow.pc.in b/cpp/src/arrow/arrow.pc.in
index 1c3f65d661101..0debee32a243a 100644
--- a/cpp/src/arrow/arrow.pc.in
+++ b/cpp/src/arrow/arrow.pc.in
@@ -19,6 +19,9 @@ prefix=@CMAKE_INSTALL_PREFIX@
 libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
 includedir=${prefix}/include
 
+so_version=@ARROW_SO_VERSION@
+abi_version=@ARROW_ABI_VERSION@
+
 Name: Apache Arrow
 Description: Arrow is a set of technologies that enable big-data systems to process and move data fast.
 Version: @ARROW_VERSION@
diff --git a/cpp/src/arrow/io/arrow-io.pc.in b/cpp/src/arrow/io/arrow-io.pc.in
index af28aae6736fe..61af3577f5a38 100644
--- a/cpp/src/arrow/io/arrow-io.pc.in
+++ b/cpp/src/arrow/io/arrow-io.pc.in
@@ -19,6 +19,9 @@ prefix=@CMAKE_INSTALL_PREFIX@
 libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
 includedir=${prefix}/include
 
+so_version=@ARROW_SO_VERSION@
+abi_version=@ARROW_ABI_VERSION@
+
 Name: Apache Arrow I/O
 Description: I/O interface for Arrow.
 Version: @ARROW_VERSION@
diff --git a/cpp/src/arrow/ipc/arrow-ipc.pc.in b/cpp/src/arrow/ipc/arrow-ipc.pc.in
index cbc226abf1ff5..29a942acf0331 100644
--- a/cpp/src/arrow/ipc/arrow-ipc.pc.in
+++ b/cpp/src/arrow/ipc/arrow-ipc.pc.in
@@ -19,6 +19,9 @@ prefix=@CMAKE_INSTALL_PREFIX@
 libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
 includedir=${prefix}/include
 
+so_version=@ARROW_SO_VERSION@
+abi_version=@ARROW_ABI_VERSION@
+
 Name: Apache Arrow IPC
 Description: IPC extension for Arrow.
 Version: @ARROW_VERSION@
diff --git a/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in b/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
index 18085aaf715d4..8e946d17d8601 100644
--- a/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
+++ b/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
@@ -19,6 +19,9 @@ prefix=@CMAKE_INSTALL_PREFIX@
 libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
 includedir=${prefix}/include
 
+so_version=@ARROW_SO_VERSION@
+abi_version=@ARROW_ABI_VERSION@
+
 Name: Apache Arrow jemalloc-based allocator
 Description: jemalloc allocator for Arrow.
 Version: @ARROW_VERSION@

From f7b287a28d62c6b246665da7eee50fe222ebaaeb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 29 Mar 2017 19:53:08 -0400
Subject: [PATCH 0430/1644] ARROW-627: [C++] Add compatibility macros for
 exported extern templates

This should also reduce compiler warnings on MSVC

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #447 from wesm/ARROW-627 and squashes the following commits:

3f6277d [Wes McKinney] Wrong define for msvc
b53a400 [Wes McKinney] MSVC needs export annotation when instantiating templates
8a9fcb4 [Wes McKinney] Add compatibility macros for exported extern templates, also to reduce compiler warnings in MSVC
---
 cpp/src/arrow/array.cc          | 32 ++++++++++++------------
 cpp/src/arrow/array.h           | 43 ++++++++++++---------------------
 cpp/src/arrow/tensor.cc         | 22 ++++++++---------
 cpp/src/arrow/tensor.h          | 33 +++++++++----------------
 cpp/src/arrow/type.h            |  4 +--
 cpp/src/arrow/util/visibility.h | 17 +++++++++++++
 6 files changed, 73 insertions(+), 78 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 3ea033376fca3..b25411a1c5938 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -445,21 +445,21 @@ Status Array::Accept(ArrayVisitor* visitor) const {
 // ----------------------------------------------------------------------
 // Instantiate templates
 
-template class NumericArray<UInt8Type>;
-template class NumericArray<UInt16Type>;
-template class NumericArray<UInt32Type>;
-template class NumericArray<UInt64Type>;
-template class NumericArray<Int8Type>;
-template class NumericArray<Int16Type>;
-template class NumericArray<Int32Type>;
-template class NumericArray<Int64Type>;
-template class NumericArray<TimestampType>;
-template class NumericArray<Date32Type>;
-template class NumericArray<Date64Type>;
-template class NumericArray<Time32Type>;
-template class NumericArray<Time64Type>;
-template class NumericArray<HalfFloatType>;
-template class NumericArray<FloatType>;
-template class NumericArray<DoubleType>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<UInt8Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<UInt16Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<UInt32Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<UInt64Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<Int8Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<Int16Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<Int32Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<Int64Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<TimestampType>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<Date32Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<Date64Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<Time32Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<Time64Type>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<HalfFloatType>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<FloatType>;
+template class ARROW_TEMPLATE_EXPORT NumericArray<DoubleType>;
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index c0ec571e45983..53b640853d5a6 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -483,34 +483,23 @@ class ARROW_EXPORT DictionaryArray : public Array {
 // ----------------------------------------------------------------------
 // extern templates and other details
 
-// gcc and clang disagree about how to handle template visibility when you have
-// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-#endif
-
 // Only instantiate these templates once
-extern template class ARROW_EXPORT NumericArray<Int8Type>;
-extern template class ARROW_EXPORT NumericArray<UInt8Type>;
-extern template class ARROW_EXPORT NumericArray<Int16Type>;
-extern template class ARROW_EXPORT NumericArray<UInt16Type>;
-extern template class ARROW_EXPORT NumericArray<Int32Type>;
-extern template class ARROW_EXPORT NumericArray<UInt32Type>;
-extern template class ARROW_EXPORT NumericArray<Int64Type>;
-extern template class ARROW_EXPORT NumericArray<UInt64Type>;
-extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
-extern template class ARROW_EXPORT NumericArray<FloatType>;
-extern template class ARROW_EXPORT NumericArray<DoubleType>;
-extern template class ARROW_EXPORT NumericArray<Date32Type>;
-extern template class ARROW_EXPORT NumericArray<Date64Type>;
-extern template class ARROW_EXPORT NumericArray<Time32Type>;
-extern template class ARROW_EXPORT NumericArray<Time64Type>;
-extern template class ARROW_EXPORT NumericArray<TimestampType>;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
+ARROW_EXTERN_TEMPLATE NumericArray<Int8Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<UInt8Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<Int16Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<UInt16Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<Int32Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<UInt32Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<Int64Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<UInt64Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<HalfFloatType>;
+ARROW_EXTERN_TEMPLATE NumericArray<FloatType>;
+ARROW_EXTERN_TEMPLATE NumericArray<DoubleType>;
+ARROW_EXTERN_TEMPLATE NumericArray<Date32Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<Date64Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<Time32Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<Time64Type>;
+ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index 6489cd01d4c80..7c4593fc40e66 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -100,16 +100,16 @@ NumericTensor<T>::NumericTensor(const std::shared_ptr<Buffer>& data,
     const std::vector<int64_t>& shape, const std::vector<int64_t>& strides)
     : NumericTensor(data, shape, strides, {}) {}
 
-template class NumericTensor<Int8Type>;
-template class NumericTensor<UInt8Type>;
-template class NumericTensor<Int16Type>;
-template class NumericTensor<UInt16Type>;
-template class NumericTensor<Int32Type>;
-template class NumericTensor<UInt32Type>;
-template class NumericTensor<Int64Type>;
-template class NumericTensor<UInt64Type>;
-template class NumericTensor<HalfFloatType>;
-template class NumericTensor<FloatType>;
-template class NumericTensor<DoubleType>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<Int8Type>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt8Type>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<Int16Type>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt16Type>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<Int32Type>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt32Type>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<Int64Type>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt64Type>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<HalfFloatType>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<FloatType>;
+template class ARROW_TEMPLATE_EXPORT NumericTensor<DoubleType>;
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index 0059368f7b2d8..7bee867a9b33a 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -129,29 +129,18 @@ class ARROW_EXPORT NumericTensor : public Tensor {
 // ----------------------------------------------------------------------
 // extern templates and other details
 
-// gcc and clang disagree about how to handle template visibility when you have
-// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-#endif
-
 // Only instantiate these templates once
-extern template class ARROW_EXPORT NumericTensor<Int8Type>;
-extern template class ARROW_EXPORT NumericTensor<UInt8Type>;
-extern template class ARROW_EXPORT NumericTensor<Int16Type>;
-extern template class ARROW_EXPORT NumericTensor<UInt16Type>;
-extern template class ARROW_EXPORT NumericTensor<Int32Type>;
-extern template class ARROW_EXPORT NumericTensor<UInt32Type>;
-extern template class ARROW_EXPORT NumericTensor<Int64Type>;
-extern template class ARROW_EXPORT NumericTensor<UInt64Type>;
-extern template class ARROW_EXPORT NumericTensor<HalfFloatType>;
-extern template class ARROW_EXPORT NumericTensor<FloatType>;
-extern template class ARROW_EXPORT NumericTensor<DoubleType>;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
+ARROW_EXTERN_TEMPLATE NumericTensor<Int8Type>;
+ARROW_EXTERN_TEMPLATE NumericTensor<UInt8Type>;
+ARROW_EXTERN_TEMPLATE NumericTensor<Int16Type>;
+ARROW_EXTERN_TEMPLATE NumericTensor<UInt16Type>;
+ARROW_EXTERN_TEMPLATE NumericTensor<Int32Type>;
+ARROW_EXTERN_TEMPLATE NumericTensor<UInt32Type>;
+ARROW_EXTERN_TEMPLATE NumericTensor<Int64Type>;
+ARROW_EXTERN_TEMPLATE NumericTensor<UInt64Type>;
+ARROW_EXTERN_TEMPLATE NumericTensor<HalfFloatType>;
+ARROW_EXTERN_TEMPLATE NumericTensor<FloatType>;
+ARROW_EXTERN_TEMPLATE NumericTensor<DoubleType>;
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index dc50ecd669cae..2a73f6be934eb 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -439,7 +439,7 @@ struct ARROW_EXPORT UnionType : public NestedType {
 
 enum class DateUnit : char { DAY = 0, MILLI = 1 };
 
-struct DateType : public FixedWidthType {
+struct ARROW_EXPORT DateType : public FixedWidthType {
  public:
   DateUnit unit;
 
@@ -496,7 +496,7 @@ static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) {
   return os;
 }
 
-struct TimeType : public FixedWidthType {
+struct ARROW_EXPORT TimeType : public FixedWidthType {
  public:
   TimeUnit unit;
 
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
index 4819a0061e75f..6382f7f63180c 100644
--- a/cpp/src/arrow/util/visibility.h
+++ b/cpp/src/arrow/util/visibility.h
@@ -35,4 +35,21 @@
 #endif
 #endif  // Non-Windows
 
+// gcc and clang disagree about how to handle template visibility when you have
+// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
+
+#if defined(__clang__)
+  #define ARROW_EXTERN_TEMPLATE extern template class ARROW_EXPORT
+#else
+  #define ARROW_EXTERN_TEMPLATE extern template class
+#endif
+
+// This is a complicated topic, some reading on it:
+// http://www.codesynthesis.com/~boris/blog/2010/01/18/dll-export-cxx-templates/
+#if defined(_MSC_VER)
+  #define ARROW_TEMPLATE_EXPORT ARROW_EXPORT
+#else
+  #define ARROW_TEMPLATE_EXPORT
+#endif
+
 #endif  // ARROW_UTIL_VISIBILITY_H

From 642b753a49a3fcb5d53946c773cd70ab2a3ece88 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 30 Mar 2017 10:19:50 -0400
Subject: [PATCH 0431/1644] ARROW-698: Add flag to FileWriter::WriteRecordBatch
 for writing record batches with lengths over INT32_MAX

cc @pcmoritz

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #455 from wesm/ARROW-698 and squashes the following commits:

42c100c [Wes McKinney] Add allow_64bit option to FileWriter::WriteRecordBatch
---
 cpp/src/arrow/ipc/ipc-read-write-test.cc | 20 ++++++++++++--------
 cpp/src/arrow/ipc/writer.cc              | 18 ++++++++++--------
 cpp/src/arrow/ipc/writer.h               |  4 ++--
 cpp/src/arrow/type-test.cc               |  8 ++++----
 cpp/src/arrow/util/visibility.h          |  8 ++++----
 5 files changed, 32 insertions(+), 26 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index cd3f190fe4a27..48e546eed12f5 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -138,17 +138,21 @@ class IpcTestFixture : public io::MemoryMapFixture {
 
   Status DoLargeRoundTrip(
       const RecordBatch& batch, bool zero_data, std::shared_ptr<RecordBatch>* result) {
-    int32_t metadata_length;
-    int64_t body_length;
-
-    const int64_t buffer_offset = 0;
-
     if (zero_data) { RETURN_NOT_OK(ZeroMemoryMap(mmap_.get())); }
     RETURN_NOT_OK(mmap_->Seek(0));
 
-    RETURN_NOT_OK(WriteLargeRecordBatch(
-        batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
-    return ReadRecordBatch(batch.schema(), 0, mmap_.get(), result);
+    std::shared_ptr<FileWriter> file_writer;
+    RETURN_NOT_OK(FileWriter::Open(mmap_.get(), batch.schema(), &file_writer));
+    RETURN_NOT_OK(file_writer->WriteRecordBatch(batch, true));
+    RETURN_NOT_OK(file_writer->Close());
+
+    int64_t offset;
+    RETURN_NOT_OK(mmap_->Tell(&offset));
+
+    std::shared_ptr<FileReader> file_reader;
+    RETURN_NOT_OK(FileReader::Open(mmap_, offset, &file_reader));
+
+    return file_reader->GetRecordBatch(0, result);
   }
 
   void CheckReadResult(const RecordBatch& result, const RecordBatch& expected) {
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index da360f31641b8..92e61941937a6 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -591,7 +591,7 @@ class StreamWriter::StreamWriterImpl {
     return Status::OK();
   }
 
-  Status WriteRecordBatch(const RecordBatch& batch, FileBlock* block) {
+  Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit, FileBlock* block) {
     RETURN_NOT_OK(CheckStarted());
 
     block->offset = position_;
@@ -599,7 +599,8 @@ class StreamWriter::StreamWriterImpl {
     // Frame of reference in file format is 0, see ARROW-384
     const int64_t buffer_start_offset = 0;
     RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(batch, buffer_start_offset, sink_,
-        &block->metadata_length, &block->body_length, pool_));
+        &block->metadata_length, &block->body_length, pool_, kMaxNestingDepth,
+        allow_64bit));
     RETURN_NOT_OK(UpdatePosition());
 
     DCHECK(position_ % 8 == 0) << "WriteRecordBatch did not perform aligned writes";
@@ -607,10 +608,11 @@ class StreamWriter::StreamWriterImpl {
     return Status::OK();
   }
 
-  Status WriteRecordBatch(const RecordBatch& batch) {
+  Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit) {
     // Push an empty FileBlock. Can be written in the footer later
     record_batches_.emplace_back(0, 0, 0);
-    return WriteRecordBatch(batch, &record_batches_[record_batches_.size() - 1]);
+    return WriteRecordBatch(
+        batch, allow_64bit, &record_batches_[record_batches_.size() - 1]);
   }
 
   // Adds padding bytes if necessary to ensure all memory blocks are written on
@@ -657,8 +659,8 @@ StreamWriter::StreamWriter() {
   impl_.reset(new StreamWriterImpl());
 }
 
-Status StreamWriter::WriteRecordBatch(const RecordBatch& batch) {
-  return impl_->WriteRecordBatch(batch);
+Status StreamWriter::WriteRecordBatch(const RecordBatch& batch, bool allow_64bit) {
+  return impl_->WriteRecordBatch(batch, allow_64bit);
 }
 
 void StreamWriter::set_memory_pool(MemoryPool* pool) {
@@ -723,8 +725,8 @@ Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& s
   return (*out)->impl_->Open(sink, schema);
 }
 
-Status FileWriter::WriteRecordBatch(const RecordBatch& batch) {
-  return impl_->WriteRecordBatch(batch);
+Status FileWriter::WriteRecordBatch(const RecordBatch& batch, bool allow_64bit) {
+  return impl_->WriteRecordBatch(batch, allow_64bit);
 }
 
 Status FileWriter::Close() {
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 3b7e710c124cb..25b5ad62726d9 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -87,7 +87,7 @@ class ARROW_EXPORT StreamWriter {
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
       std::shared_ptr<StreamWriter>* out);
 
-  virtual Status WriteRecordBatch(const RecordBatch& batch);
+  virtual Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false);
 
   /// Perform any logic necessary to finish the stream. User is responsible for
   /// closing the actual OutputStream
@@ -108,7 +108,7 @@ class ARROW_EXPORT FileWriter : public StreamWriter {
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
       std::shared_ptr<FileWriter>* out);
 
-  Status WriteRecordBatch(const RecordBatch& batch) override;
+  Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
   Status Close() override;
 
  private:
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 7f13f8ba480b4..ed8654314ee6d 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -232,16 +232,16 @@ TEST(TestTimestampType, ToString) {
 }
 
 TEST(TestNestedType, Equals) {
-  auto create_struct =
-      [](std::string inner_name, std::string struct_name) -> shared_ptr<Field> {
+  auto create_struct = [](
+      std::string inner_name, std::string struct_name) -> shared_ptr<Field> {
     auto f_type = field(inner_name, int32());
     vector<shared_ptr<Field>> fields = {f_type};
     auto s_type = std::make_shared<StructType>(fields);
     return field(struct_name, s_type);
   };
 
-  auto create_union =
-      [](std::string inner_name, std::string union_name) -> shared_ptr<Field> {
+  auto create_union = [](
+      std::string inner_name, std::string union_name) -> shared_ptr<Field> {
     auto f_type = field(inner_name, int32());
     vector<shared_ptr<Field>> fields = {f_type};
     vector<uint8_t> codes = {Type::INT32};
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
index 6382f7f63180c..e84cc45aadf01 100644
--- a/cpp/src/arrow/util/visibility.h
+++ b/cpp/src/arrow/util/visibility.h
@@ -39,17 +39,17 @@
 // explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
 
 #if defined(__clang__)
-  #define ARROW_EXTERN_TEMPLATE extern template class ARROW_EXPORT
+#define ARROW_EXTERN_TEMPLATE extern template class ARROW_EXPORT
 #else
-  #define ARROW_EXTERN_TEMPLATE extern template class
+#define ARROW_EXTERN_TEMPLATE extern template class
 #endif
 
 // This is a complicated topic, some reading on it:
 // http://www.codesynthesis.com/~boris/blog/2010/01/18/dll-export-cxx-templates/
 #if defined(_MSC_VER)
-  #define ARROW_TEMPLATE_EXPORT ARROW_EXPORT
+#define ARROW_TEMPLATE_EXPORT ARROW_EXPORT
 #else
-  #define ARROW_TEMPLATE_EXPORT
+#define ARROW_TEMPLATE_EXPORT
 #endif
 
 #endif  // ARROW_UTIL_VISIBILITY_H

From 47fad3f42c05bd4139796b93375dfb3cba74e87b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 30 Mar 2017 15:04:07 -0400
Subject: [PATCH 0432/1644] ARROW-728: [C++/Python] Add Table::RemoveColumn
 method, remove name member, some other code cleaning

* Consolidated column.h and table.h
* Consolidated schema.h and type.h
* Removed some `Equals(const std::shared_ptr<T>&)` methods, better to use `const T&` methods

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #457 from wesm/ARROW-728 and squashes the following commits:

961783d [Wes McKinney] Fix glib test suite
1645ea2 [Wes McKinney] Return new vector from DeleteVectorElement
ea36a8c [Wes McKinney] Fix GLib bindings for removal of name Table member
77d363c [Wes McKinney] Incorporate API changes in pyarrow, add Table.remove_column function. Make nicer repr
b73c4d7 [Wes McKinney] Remove Table name attribute, implement and test Table::RemoveColumn
6a7f022 [Wes McKinney] Move Schema to type.h, remove Equals with shared_ptr function
818c46f [Wes McKinney] Consolidate column.h into table.h
---
 c_glib/arrow-glib/table.cpp                |  20 +-
 c_glib/arrow-glib/table.h                  |   4 +-
 c_glib/test/test-table.rb                  |   9 +-
 cpp/CMakeLists.txt                         |   2 -
 cpp/src/arrow/CMakeLists.txt               |   3 -
 cpp/src/arrow/api.h                        |   2 -
 cpp/src/arrow/column-test.cc               | 191 ----------------
 cpp/src/arrow/column.cc                    | 132 -----------
 cpp/src/arrow/column.h                     | 104 ---------
 cpp/src/arrow/ipc/feather.cc               |   2 +-
 cpp/src/arrow/ipc/ipc-json-test.cc         |   6 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc   |   2 +-
 cpp/src/arrow/ipc/json-integration-test.cc |   6 +-
 cpp/src/arrow/ipc/json-internal.cc         |   1 -
 cpp/src/arrow/ipc/json.cc                  |   1 -
 cpp/src/arrow/ipc/metadata.cc              |   1 -
 cpp/src/arrow/ipc/reader.cc                |   2 +-
 cpp/src/arrow/ipc/test-common.h            |   2 +-
 cpp/src/arrow/ipc/writer.cc                |   1 -
 cpp/src/arrow/python/pandas-test.cc        |   3 +-
 cpp/src/arrow/python/pandas_convert.cc     |   1 -
 cpp/src/arrow/schema.cc                    |  72 ------
 cpp/src/arrow/schema.h                     |  59 -----
 cpp/src/arrow/table-test.cc                | 246 +++++++++++++++++----
 cpp/src/arrow/table.cc                     | 149 +++++++++++--
 cpp/src/arrow/table.h                      |  90 +++++++-
 cpp/src/arrow/test-common.h                |   1 -
 cpp/src/arrow/test-util.h                  |   2 -
 cpp/src/arrow/type-test.cc                 |   8 +-
 cpp/src/arrow/type.cc                      |  53 +++++
 cpp/src/arrow/type.h                       |  33 ++-
 cpp/src/arrow/util/stl.h                   |  40 ++++
 python/pyarrow/array.pyx                   |   4 +-
 python/pyarrow/includes/libarrow.pxd       |  17 +-
 python/pyarrow/io.pyx                      |   6 +-
 python/pyarrow/schema.pyx                  |   9 +-
 python/pyarrow/table.pyx                   |  65 +++---
 python/pyarrow/tests/test_parquet.py       |   2 +-
 python/pyarrow/tests/test_table.py         |  33 ++-
 39 files changed, 623 insertions(+), 761 deletions(-)
 delete mode 100644 cpp/src/arrow/column-test.cc
 delete mode 100644 cpp/src/arrow/column.cc
 delete mode 100644 cpp/src/arrow/column.h
 delete mode 100644 cpp/src/arrow/schema.cc
 delete mode 100644 cpp/src/arrow/schema.h
 create mode 100644 cpp/src/arrow/util/stl.h

diff --git a/c_glib/arrow-glib/table.cpp b/c_glib/arrow-glib/table.cpp
index 2410e76c921fb..2f82ffa4320e0 100644
--- a/c_glib/arrow-glib/table.cpp
+++ b/c_glib/arrow-glib/table.cpp
@@ -126,15 +126,13 @@ garrow_table_class_init(GArrowTableClass *klass)
 
 /**
  * garrow_table_new:
- * @name: The name of the table.
  * @schema: The schema of the table.
  * @columns: (element-type GArrowColumn): The columns of the table.
  *
  * Returns: A newly created #GArrowTable.
  */
 GArrowTable *
-garrow_table_new(const gchar *name,
-                 GArrowSchema *schema,
+garrow_table_new(GArrowSchema *schema,
                  GList *columns)
 {
   std::vector<std::shared_ptr<arrow::Column>> arrow_columns;
@@ -144,25 +142,11 @@ garrow_table_new(const gchar *name,
   }
 
   auto arrow_table =
-    std::make_shared<arrow::Table>(name,
-                                   garrow_schema_get_raw(schema),
+    std::make_shared<arrow::Table>(garrow_schema_get_raw(schema),
                                    arrow_columns);
   return garrow_table_new_raw(&arrow_table);
 }
 
-/**
- * garrow_table_get_name:
- * @table: A #GArrowTable.
- *
- * Returns: The name of the table.
- */
-const gchar *
-garrow_table_get_name(GArrowTable *table)
-{
-  const auto arrow_table = garrow_table_get_raw(table);
-  return arrow_table->name().c_str();
-}
-
 /**
  * garrow_table_get_schema:
  * @table: A #GArrowTable.
diff --git a/c_glib/arrow-glib/table.h b/c_glib/arrow-glib/table.h
index 34a89a78abcbb..4dbb8c587a2ec 100644
--- a/c_glib/arrow-glib/table.h
+++ b/c_glib/arrow-glib/table.h
@@ -66,11 +66,9 @@ struct _GArrowTableClass
 
 GType           garrow_table_get_type      (void) G_GNUC_CONST;
 
-GArrowTable    *garrow_table_new           (const gchar *name,
-                                            GArrowSchema *schema,
+GArrowTable    *garrow_table_new           (GArrowSchema *schema,
                                             GList *columns);
 
-const gchar    *garrow_table_get_name      (GArrowTable *table);
 GArrowSchema   *garrow_table_get_schema    (GArrowTable *table);
 GArrowColumn   *garrow_table_get_column    (GArrowTable *table,
                                             guint i);
diff --git a/c_glib/test/test-table.rb b/c_glib/test/test-table.rb
index 1687d2f6e3ff6..0583e8139e47a 100644
--- a/c_glib/test/test-table.rb
+++ b/c_glib/test/test-table.rb
@@ -29,8 +29,7 @@ def test_columns
         Arrow::Column.new(fields[0], build_boolean_array([true])),
         Arrow::Column.new(fields[1], build_boolean_array([false])),
       ]
-      table = Arrow::Table.new("memos", schema, columns)
-      assert_equal("memos", table.name)
+      table = Arrow::Table.new(schema, columns)
     end
   end
 
@@ -45,11 +44,7 @@ def setup
         Arrow::Column.new(fields[0], build_boolean_array([true])),
         Arrow::Column.new(fields[1], build_boolean_array([false])),
       ]
-      @table = Arrow::Table.new("memos", schema, columns)
-    end
-
-    def test_name
-      assert_equal("memos", @table.name)
+      @table = Arrow::Table.new(schema, columns)
     end
 
     def test_schema
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index e4c18ca86e4d7..e11de1b4fb0da 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -784,12 +784,10 @@ set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/buffer.cc
   src/arrow/builder.cc
-  src/arrow/column.cc
   src/arrow/compare.cc
   src/arrow/loader.cc
   src/arrow/memory_pool.cc
   src/arrow/pretty_print.cc
-  src/arrow/schema.cc
   src/arrow/status.cc
   src/arrow/table.cc
   src/arrow/tensor.cc
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index f965f1d07feef..5c9aadf9ee79b 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -22,12 +22,10 @@ install(FILES
   array.h
   buffer.h
   builder.h
-  column.h
   compare.h
   loader.h
   memory_pool.h
   pretty_print.h
-  schema.h
   status.h
   table.h
   type.h
@@ -59,7 +57,6 @@ ADD_ARROW_TEST(array-string-test)
 ADD_ARROW_TEST(array-struct-test)
 ADD_ARROW_TEST(array-union-test)
 ADD_ARROW_TEST(buffer-test)
-ADD_ARROW_TEST(column-test)
 ADD_ARROW_TEST(memory_pool-test)
 ADD_ARROW_TEST(pretty_print-test)
 ADD_ARROW_TEST(status-test)
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index ea818b62931d6..50a09515297ff 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -23,12 +23,10 @@
 #include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
-#include "arrow/column.h"
 #include "arrow/compare.h"
 #include "arrow/loader.h"
 #include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
deleted file mode 100644
index 872fcb95c08e1..0000000000000
--- a/cpp/src/arrow/column-test.cc
+++ /dev/null
@@ -1,191 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/column.h"
-#include "arrow/schema.h"
-#include "arrow/test-common.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-
-using std::shared_ptr;
-using std::vector;
-
-namespace arrow {
-
-class TestChunkedArray : public TestBase {
- protected:
-  virtual void Construct() {
-    one_ = std::make_shared<ChunkedArray>(arrays_one_);
-    another_ = std::make_shared<ChunkedArray>(arrays_another_);
-  }
-
-  ArrayVector arrays_one_;
-  ArrayVector arrays_another_;
-
-  std::shared_ptr<ChunkedArray> one_;
-  std::shared_ptr<ChunkedArray> another_;
-};
-
-TEST_F(TestChunkedArray, BasicEquals) {
-  std::vector<bool> null_bitmap(100, true);
-  std::vector<int32_t> data(100, 1);
-  std::shared_ptr<Array> array;
-  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data, &array);
-  arrays_one_.push_back(array);
-  arrays_another_.push_back(array);
-
-  Construct();
-  ASSERT_TRUE(one_->Equals(one_));
-  ASSERT_FALSE(one_->Equals(nullptr));
-  ASSERT_TRUE(one_->Equals(another_));
-  ASSERT_TRUE(one_->Equals(*another_.get()));
-}
-
-TEST_F(TestChunkedArray, EqualsDifferingTypes) {
-  std::vector<bool> null_bitmap(100, true);
-  std::vector<int32_t> data32(100, 1);
-  std::vector<int64_t> data64(100, 1);
-  std::shared_ptr<Array> array;
-  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data32, &array);
-  arrays_one_.push_back(array);
-  ArrayFromVector<Int64Type, int64_t>(null_bitmap, data64, &array);
-  arrays_another_.push_back(array);
-
-  Construct();
-  ASSERT_FALSE(one_->Equals(another_));
-  ASSERT_FALSE(one_->Equals(*another_.get()));
-}
-
-TEST_F(TestChunkedArray, EqualsDifferingLengths) {
-  std::vector<bool> null_bitmap100(100, true);
-  std::vector<bool> null_bitmap101(101, true);
-  std::vector<int32_t> data100(100, 1);
-  std::vector<int32_t> data101(101, 1);
-  std::shared_ptr<Array> array;
-  ArrayFromVector<Int32Type, int32_t>(null_bitmap100, data100, &array);
-  arrays_one_.push_back(array);
-  ArrayFromVector<Int32Type, int32_t>(null_bitmap101, data101, &array);
-  arrays_another_.push_back(array);
-
-  Construct();
-  ASSERT_FALSE(one_->Equals(another_));
-  ASSERT_FALSE(one_->Equals(*another_.get()));
-
-  std::vector<bool> null_bitmap1(1, true);
-  std::vector<int32_t> data1(1, 1);
-  ArrayFromVector<Int32Type, int32_t>(null_bitmap1, data1, &array);
-  arrays_one_.push_back(array);
-
-  Construct();
-  ASSERT_TRUE(one_->Equals(another_));
-  ASSERT_TRUE(one_->Equals(*another_.get()));
-}
-
-class TestColumn : public TestChunkedArray {
- protected:
-  void Construct() override {
-    TestChunkedArray::Construct();
-
-    one_col_ = std::make_shared<Column>(one_field_, one_);
-    another_col_ = std::make_shared<Column>(another_field_, another_);
-  }
-
-  std::shared_ptr<ChunkedArray> data_;
-  std::unique_ptr<Column> column_;
-
-  std::shared_ptr<Field> one_field_;
-  std::shared_ptr<Field> another_field_;
-
-  std::shared_ptr<Column> one_col_;
-  std::shared_ptr<Column> another_col_;
-};
-
-TEST_F(TestColumn, BasicAPI) {
-  ArrayVector arrays;
-  arrays.push_back(MakePrimitive<Int32Array>(100));
-  arrays.push_back(MakePrimitive<Int32Array>(100, 10));
-  arrays.push_back(MakePrimitive<Int32Array>(100, 20));
-
-  auto field = std::make_shared<Field>("c0", int32());
-  column_.reset(new Column(field, arrays));
-
-  ASSERT_EQ("c0", column_->name());
-  ASSERT_TRUE(column_->type()->Equals(int32()));
-  ASSERT_EQ(300, column_->length());
-  ASSERT_EQ(30, column_->null_count());
-  ASSERT_EQ(3, column_->data()->num_chunks());
-
-  // nullptr array should not break
-  column_.reset(new Column(field, std::shared_ptr<Array>(nullptr)));
-  ASSERT_NE(column_.get(), nullptr);
-}
-
-TEST_F(TestColumn, ChunksInhomogeneous) {
-  ArrayVector arrays;
-  arrays.push_back(MakePrimitive<Int32Array>(100));
-  arrays.push_back(MakePrimitive<Int32Array>(100, 10));
-
-  auto field = std::make_shared<Field>("c0", int32());
-  column_.reset(new Column(field, arrays));
-
-  ASSERT_OK(column_->ValidateData());
-
-  arrays.push_back(MakePrimitive<Int16Array>(100, 10));
-  column_.reset(new Column(field, arrays));
-  ASSERT_RAISES(Invalid, column_->ValidateData());
-}
-
-TEST_F(TestColumn, Equals) {
-  std::vector<bool> null_bitmap(100, true);
-  std::vector<int32_t> data(100, 1);
-  std::shared_ptr<Array> array;
-  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data, &array);
-  arrays_one_.push_back(array);
-  arrays_another_.push_back(array);
-
-  one_field_ = std::make_shared<Field>("column", int32());
-  another_field_ = std::make_shared<Field>("column", int32());
-
-  Construct();
-  ASSERT_TRUE(one_col_->Equals(one_col_));
-  ASSERT_FALSE(one_col_->Equals(nullptr));
-  ASSERT_TRUE(one_col_->Equals(another_col_));
-  ASSERT_TRUE(one_col_->Equals(*another_col_.get()));
-
-  // Field is different
-  another_field_ = std::make_shared<Field>("two", int32());
-  Construct();
-  ASSERT_FALSE(one_col_->Equals(another_col_));
-  ASSERT_FALSE(one_col_->Equals(*another_col_.get()));
-
-  // ChunkedArray is different
-  another_field_ = std::make_shared<Field>("column", int32());
-  arrays_another_.push_back(array);
-  Construct();
-  ASSERT_FALSE(one_col_->Equals(another_col_));
-  ASSERT_FALSE(one_col_->Equals(*another_col_.get()));
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
deleted file mode 100644
index 78501f9393e22..0000000000000
--- a/cpp/src/arrow/column.cc
+++ /dev/null
@@ -1,132 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/column.h"
-
-#include <memory>
-#include <sstream>
-
-#include "arrow/array.h"
-#include "arrow/status.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-ChunkedArray::ChunkedArray(const ArrayVector& chunks) : chunks_(chunks) {
-  length_ = 0;
-  null_count_ = 0;
-  for (const std::shared_ptr<Array>& chunk : chunks) {
-    length_ += chunk->length();
-    null_count_ += chunk->null_count();
-  }
-}
-
-bool ChunkedArray::Equals(const ChunkedArray& other) const {
-  if (length_ != other.length()) { return false; }
-  if (null_count_ != other.null_count()) { return false; }
-
-  // Check contents of the underlying arrays. This checks for equality of
-  // the underlying data independently of the chunk size.
-  int this_chunk_idx = 0;
-  int64_t this_start_idx = 0;
-  int other_chunk_idx = 0;
-  int64_t other_start_idx = 0;
-
-  int64_t elements_compared = 0;
-  while (elements_compared < length_) {
-    const std::shared_ptr<Array> this_array = chunks_[this_chunk_idx];
-    const std::shared_ptr<Array> other_array = other.chunk(other_chunk_idx);
-    int64_t common_length = std::min(
-        this_array->length() - this_start_idx, other_array->length() - other_start_idx);
-    if (!this_array->RangeEquals(this_start_idx, this_start_idx + common_length,
-            other_start_idx, other_array)) {
-      return false;
-    }
-
-    elements_compared += common_length;
-
-    // If we have exhausted the current chunk, proceed to the next one individually.
-    if (this_start_idx + common_length == this_array->length()) {
-      this_chunk_idx++;
-      this_start_idx = 0;
-    } else {
-      this_start_idx += common_length;
-    }
-
-    if (other_start_idx + common_length == other_array->length()) {
-      other_chunk_idx++;
-      other_start_idx = 0;
-    } else {
-      other_start_idx += common_length;
-    }
-  }
-  return true;
-}
-
-bool ChunkedArray::Equals(const std::shared_ptr<ChunkedArray>& other) const {
-  if (this == other.get()) { return true; }
-  if (!other) { return false; }
-  return Equals(*other.get());
-}
-
-Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks)
-    : field_(field) {
-  data_ = std::make_shared<ChunkedArray>(chunks);
-}
-
-Column::Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data)
-    : field_(field) {
-  if (data) {
-    data_ = std::make_shared<ChunkedArray>(ArrayVector({data}));
-  } else {
-    data_ = std::make_shared<ChunkedArray>(ArrayVector({}));
-  }
-}
-
-Column::Column(const std::string& name, const std::shared_ptr<Array>& data)
-    : Column(::arrow::field(name, data->type()), data) {}
-
-Column::Column(
-    const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data)
-    : field_(field), data_(data) {}
-
-bool Column::Equals(const Column& other) const {
-  if (!field_->Equals(other.field())) { return false; }
-  return data_->Equals(other.data());
-}
-
-bool Column::Equals(const std::shared_ptr<Column>& other) const {
-  if (this == other.get()) { return true; }
-  if (!other) { return false; }
-
-  return Equals(*other.get());
-}
-
-Status Column::ValidateData() {
-  for (int i = 0; i < data_->num_chunks(); ++i) {
-    std::shared_ptr<DataType> type = data_->chunk(i)->type();
-    if (!this->type()->Equals(type)) {
-      std::stringstream ss;
-      ss << "In chunk " << i << " expected type " << this->type()->ToString()
-         << " but saw " << type->ToString();
-      return Status::Invalid(ss.str());
-    }
-  }
-  return Status::OK();
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h
deleted file mode 100644
index bfcfd8ee459c0..0000000000000
--- a/cpp/src/arrow/column.h
+++ /dev/null
@@ -1,104 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_COLUMN_H
-#define ARROW_COLUMN_H
-
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Array;
-class Status;
-
-typedef std::vector<std::shared_ptr<Array>> ArrayVector;
-
-// A data structure managing a list of primitive Arrow arrays logically as one
-// large array
-class ARROW_EXPORT ChunkedArray {
- public:
-  explicit ChunkedArray(const ArrayVector& chunks);
-
-  // @returns: the total length of the chunked array; computed on construction
-  int64_t length() const { return length_; }
-
-  int64_t null_count() const { return null_count_; }
-
-  int num_chunks() const { return static_cast<int>(chunks_.size()); }
-
-  std::shared_ptr<Array> chunk(int i) const { return chunks_[i]; }
-
-  const ArrayVector& chunks() const { return chunks_; }
-
-  bool Equals(const ChunkedArray& other) const;
-  bool Equals(const std::shared_ptr<ChunkedArray>& other) const;
-
- protected:
-  ArrayVector chunks_;
-  int64_t length_;
-  int64_t null_count_;
-};
-
-// An immutable column data structure consisting of a field (type metadata) and
-// a logical chunked data array (which can be validated as all being the same
-// type).
-class ARROW_EXPORT Column {
- public:
-  Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
-  Column(const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data);
-
-  Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data);
-
-  /// Construct from name and array
-  Column(const std::string& name, const std::shared_ptr<Array>& data);
-
-  int64_t length() const { return data_->length(); }
-
-  int64_t null_count() const { return data_->null_count(); }
-
-  std::shared_ptr<Field> field() const { return field_; }
-
-  // @returns: the column's name in the passed metadata
-  const std::string& name() const { return field_->name; }
-
-  // @returns: the column's type according to the metadata
-  std::shared_ptr<DataType> type() const { return field_->type; }
-
-  // @returns: the column's data as a chunked logical array
-  std::shared_ptr<ChunkedArray> data() const { return data_; }
-
-  bool Equals(const Column& other) const;
-  bool Equals(const std::shared_ptr<Column>& other) const;
-
-  // Verify that the column's array data is consistent with the passed field's
-  // metadata
-  Status ValidateData();
-
- protected:
-  std::shared_ptr<Field> field_;
-  std::shared_ptr<ChunkedArray> data_;
-};
-
-}  // namespace arrow
-
-#endif  // ARROW_COLUMN_H
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 000bba9cce03b..5820563b43834 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -30,12 +30,12 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
-#include "arrow/column.h"
 #include "arrow/io/file.h"
 #include "arrow/ipc/feather-internal.h"
 #include "arrow/ipc/feather_generated.h"
 #include "arrow/loader.h"
 #include "arrow/status.h"
+#include "arrow/table.h"
 #include "arrow/util/bit-util.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 68261ab25a43a..9cf6a88a6f3f1 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -52,7 +52,7 @@ void TestSchemaRoundTrip(const Schema& schema) {
   std::shared_ptr<Schema> out;
   ASSERT_OK(ReadJsonSchema(d, &out));
 
-  if (!schema.Equals(out)) {
+  if (!schema.Equals(*out)) {
     FAIL() << "In schema: " << schema.ToString() << "\nOut schema: " << out->ToString();
   }
 }
@@ -263,14 +263,14 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
       reinterpret_cast<const uint8_t*>(result.c_str()), static_cast<int>(result.size()));
 
   ASSERT_OK(JsonReader::Open(buffer, &reader));
-  ASSERT_TRUE(reader->schema()->Equals(*schema.get()));
+  ASSERT_TRUE(reader->schema()->Equals(*schema));
 
   ASSERT_EQ(nbatches, reader->num_record_batches());
 
   for (int i = 0; i < nbatches; ++i) {
     std::shared_ptr<RecordBatch> batch;
     ASSERT_OK(reader->GetRecordBatch(i, &batch));
-    ASSERT_TRUE(batch->Equals(*batches[i].get()));
+    ASSERT_TRUE(batch->Equals(*batches[i]));
   }
 }
 
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 48e546eed12f5..6ddda3f339641 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -158,7 +158,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
   void CheckReadResult(const RecordBatch& result, const RecordBatch& expected) {
     EXPECT_EQ(expected.num_rows(), result.num_rows());
 
-    ASSERT_TRUE(expected.schema()->Equals(result.schema()));
+    ASSERT_TRUE(expected.schema()->Equals(*result.schema()));
     ASSERT_EQ(expected.num_columns(), result.num_columns())
         << expected.schema()->ToString() << " result: " << result.schema()->ToString();
 
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index c16074ee32dc6..aa95500003ec0 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -33,10 +33,10 @@
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
 #include "arrow/pretty_print.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
+#include "arrow/type.h"
 
 DEFINE_string(arrow, "", "Arrow file name");
 DEFINE_string(json, "", "JSON file name");
@@ -143,7 +143,7 @@ static Status ValidateArrowVsJson(
   auto json_schema = json_reader->schema();
   auto arrow_schema = arrow_reader->schema();
 
-  if (!json_schema->Equals(arrow_schema)) {
+  if (!json_schema->Equals(*arrow_schema)) {
     std::stringstream ss;
     ss << "JSON schema: \n"
        << json_schema->ToString() << "\n"
@@ -170,7 +170,7 @@ static Status ValidateArrowVsJson(
     RETURN_NOT_OK(json_reader->GetRecordBatch(i, &json_batch));
     RETURN_NOT_OK(arrow_reader->GetRecordBatch(i, &arrow_batch));
 
-    if (!json_batch->ApproxEquals(*arrow_batch.get())) {
+    if (!json_batch->ApproxEquals(*arrow_batch)) {
       std::stringstream ss;
       ss << "Record batch " << i << " did not match";
 
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 95ab011bd087f..9572a0a81898d 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -33,7 +33,6 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/memory_pool.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index a01be191aa8ad..8056b6f3e758e 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -26,7 +26,6 @@
 #include "arrow/buffer.h"
 #include "arrow/ipc/json-internal.h"
 #include "arrow/memory_pool.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 85dc8b321c41d..36ba4b26042a8 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -29,7 +29,6 @@
 #include "arrow/io/interfaces.h"
 #include "arrow/ipc/File_generated.h"
 #include "arrow/ipc/Message_generated.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 03c678ab7e280..28320d98df9d1 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -30,9 +30,9 @@
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
+#include "arrow/type.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 994e1283004a9..583f909d071e6 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -49,7 +49,7 @@ static inline void AssertSchemaEqual(const Schema& lhs, const Schema& rhs) {
 }
 
 static inline void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
-  if (!left.schema()->Equals(right.schema())) {
+  if (!left.schema()->Equals(*right.schema())) {
     FAIL() << "Left schema: " << left.schema()->ToString()
            << "\nRight schema: " << right.schema()->ToString();
   }
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 92e61941937a6..db5f0829f92f7 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -32,7 +32,6 @@
 #include "arrow/ipc/util.h"
 #include "arrow/loader.h"
 #include "arrow/memory_pool.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/python/pandas-test.cc b/cpp/src/arrow/python/pandas-test.cc
index ae2527e19c00e..0d643df2e9f38 100644
--- a/cpp/src/arrow/python/pandas-test.cc
+++ b/cpp/src/arrow/python/pandas-test.cc
@@ -25,7 +25,6 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/python/pandas_convert.h"
-#include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
@@ -52,7 +51,7 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) {
       std::make_shared<Column>(f2, arr), std::make_shared<Column>(f3, arr)};
 
   auto schema = std::make_shared<Schema>(fields);
-  auto table = std::make_shared<Table>("", schema, cols);
+  auto table = std::make_shared<Table>(schema, cols);
 
   PyObject* out;
   Py_BEGIN_ALLOW_THREADS;
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index f2c2415ed2793..685b1f421c457 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -35,7 +35,6 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/column.h"
 #include "arrow/loader.h"
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
diff --git a/cpp/src/arrow/schema.cc b/cpp/src/arrow/schema.cc
deleted file mode 100644
index aa38fd3dd9260..0000000000000
--- a/cpp/src/arrow/schema.cc
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/schema.h"
-
-#include <memory>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-
-namespace arrow {
-
-Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields) : fields_(fields) {}
-
-bool Schema::Equals(const Schema& other) const {
-  if (this == &other) { return true; }
-
-  if (num_fields() != other.num_fields()) { return false; }
-  for (int i = 0; i < num_fields(); ++i) {
-    if (!field(i)->Equals(*other.field(i).get())) { return false; }
-  }
-  return true;
-}
-
-bool Schema::Equals(const std::shared_ptr<Schema>& other) const {
-  return Equals(*other.get());
-}
-
-std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
-  if (fields_.size() > 0 && name_to_index_.size() == 0) {
-    for (size_t i = 0; i < fields_.size(); ++i) {
-      name_to_index_[fields_[i]->name] = static_cast<int>(i);
-    }
-  }
-
-  auto it = name_to_index_.find(name);
-  if (it == name_to_index_.end()) {
-    return nullptr;
-  } else {
-    return fields_[it->second];
-  }
-}
-
-std::string Schema::ToString() const {
-  std::stringstream buffer;
-
-  int i = 0;
-  for (auto field : fields_) {
-    if (i > 0) { buffer << std::endl; }
-    buffer << field->ToString();
-    ++i;
-  }
-  return buffer.str();
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/schema.h b/cpp/src/arrow/schema.h
deleted file mode 100644
index 37cdbf7d786a4..0000000000000
--- a/cpp/src/arrow/schema.h
+++ /dev/null
@@ -1,59 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_SCHEMA_H
-#define ARROW_SCHEMA_H
-
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "arrow/type.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class ARROW_EXPORT Schema {
- public:
-  explicit Schema(const std::vector<std::shared_ptr<Field>>& fields);
-
-  // Returns true if all of the schema fields are equal
-  bool Equals(const Schema& other) const;
-  bool Equals(const std::shared_ptr<Schema>& other) const;
-
-  // Return the ith schema element. Does not boundscheck
-  std::shared_ptr<Field> field(int i) const { return fields_[i]; }
-
-  // Returns nullptr if name not found
-  std::shared_ptr<Field> GetFieldByName(const std::string& name);
-
-  const std::vector<std::shared_ptr<Field>>& fields() const { return fields_; }
-
-  // Render a string representation of the schema suitable for debugging
-  std::string ToString() const;
-
-  int num_fields() const { return static_cast<int>(fields_.size()); }
-
- private:
-  std::vector<std::shared_ptr<Field>> fields_;
-  std::unordered_map<std::string, int> name_to_index_;
-};
-
-}  // namespace arrow
-
-#endif  // ARROW_FIELD_H
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 6bb31638ecbbf..38533063cbc07 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -22,8 +22,6 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
-#include "arrow/column.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/test-common.h"
@@ -35,6 +33,160 @@ using std::vector;
 
 namespace arrow {
 
+class TestChunkedArray : public TestBase {
+ protected:
+  virtual void Construct() {
+    one_ = std::make_shared<ChunkedArray>(arrays_one_);
+    another_ = std::make_shared<ChunkedArray>(arrays_another_);
+  }
+
+  ArrayVector arrays_one_;
+  ArrayVector arrays_another_;
+
+  std::shared_ptr<ChunkedArray> one_;
+  std::shared_ptr<ChunkedArray> another_;
+};
+
+TEST_F(TestChunkedArray, BasicEquals) {
+  std::vector<bool> null_bitmap(100, true);
+  std::vector<int32_t> data(100, 1);
+  std::shared_ptr<Array> array;
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data, &array);
+  arrays_one_.push_back(array);
+  arrays_another_.push_back(array);
+
+  Construct();
+  ASSERT_TRUE(one_->Equals(one_));
+  ASSERT_FALSE(one_->Equals(nullptr));
+  ASSERT_TRUE(one_->Equals(another_));
+  ASSERT_TRUE(one_->Equals(*another_.get()));
+}
+
+TEST_F(TestChunkedArray, EqualsDifferingTypes) {
+  std::vector<bool> null_bitmap(100, true);
+  std::vector<int32_t> data32(100, 1);
+  std::vector<int64_t> data64(100, 1);
+  std::shared_ptr<Array> array;
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data32, &array);
+  arrays_one_.push_back(array);
+  ArrayFromVector<Int64Type, int64_t>(null_bitmap, data64, &array);
+  arrays_another_.push_back(array);
+
+  Construct();
+  ASSERT_FALSE(one_->Equals(another_));
+  ASSERT_FALSE(one_->Equals(*another_.get()));
+}
+
+TEST_F(TestChunkedArray, EqualsDifferingLengths) {
+  std::vector<bool> null_bitmap100(100, true);
+  std::vector<bool> null_bitmap101(101, true);
+  std::vector<int32_t> data100(100, 1);
+  std::vector<int32_t> data101(101, 1);
+  std::shared_ptr<Array> array;
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap100, data100, &array);
+  arrays_one_.push_back(array);
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap101, data101, &array);
+  arrays_another_.push_back(array);
+
+  Construct();
+  ASSERT_FALSE(one_->Equals(another_));
+  ASSERT_FALSE(one_->Equals(*another_.get()));
+
+  std::vector<bool> null_bitmap1(1, true);
+  std::vector<int32_t> data1(1, 1);
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap1, data1, &array);
+  arrays_one_.push_back(array);
+
+  Construct();
+  ASSERT_TRUE(one_->Equals(another_));
+  ASSERT_TRUE(one_->Equals(*another_.get()));
+}
+
+class TestColumn : public TestChunkedArray {
+ protected:
+  void Construct() override {
+    TestChunkedArray::Construct();
+
+    one_col_ = std::make_shared<Column>(one_field_, one_);
+    another_col_ = std::make_shared<Column>(another_field_, another_);
+  }
+
+  std::shared_ptr<ChunkedArray> data_;
+  std::unique_ptr<Column> column_;
+
+  std::shared_ptr<Field> one_field_;
+  std::shared_ptr<Field> another_field_;
+
+  std::shared_ptr<Column> one_col_;
+  std::shared_ptr<Column> another_col_;
+};
+
+TEST_F(TestColumn, BasicAPI) {
+  ArrayVector arrays;
+  arrays.push_back(MakePrimitive<Int32Array>(100));
+  arrays.push_back(MakePrimitive<Int32Array>(100, 10));
+  arrays.push_back(MakePrimitive<Int32Array>(100, 20));
+
+  auto field = std::make_shared<Field>("c0", int32());
+  column_.reset(new Column(field, arrays));
+
+  ASSERT_EQ("c0", column_->name());
+  ASSERT_TRUE(column_->type()->Equals(int32()));
+  ASSERT_EQ(300, column_->length());
+  ASSERT_EQ(30, column_->null_count());
+  ASSERT_EQ(3, column_->data()->num_chunks());
+
+  // nullptr array should not break
+  column_.reset(new Column(field, std::shared_ptr<Array>(nullptr)));
+  ASSERT_NE(column_.get(), nullptr);
+}
+
+TEST_F(TestColumn, ChunksInhomogeneous) {
+  ArrayVector arrays;
+  arrays.push_back(MakePrimitive<Int32Array>(100));
+  arrays.push_back(MakePrimitive<Int32Array>(100, 10));
+
+  auto field = std::make_shared<Field>("c0", int32());
+  column_.reset(new Column(field, arrays));
+
+  ASSERT_OK(column_->ValidateData());
+
+  arrays.push_back(MakePrimitive<Int16Array>(100, 10));
+  column_.reset(new Column(field, arrays));
+  ASSERT_RAISES(Invalid, column_->ValidateData());
+}
+
+TEST_F(TestColumn, Equals) {
+  std::vector<bool> null_bitmap(100, true);
+  std::vector<int32_t> data(100, 1);
+  std::shared_ptr<Array> array;
+  ArrayFromVector<Int32Type, int32_t>(null_bitmap, data, &array);
+  arrays_one_.push_back(array);
+  arrays_another_.push_back(array);
+
+  one_field_ = std::make_shared<Field>("column", int32());
+  another_field_ = std::make_shared<Field>("column", int32());
+
+  Construct();
+  ASSERT_TRUE(one_col_->Equals(one_col_));
+  ASSERT_FALSE(one_col_->Equals(nullptr));
+  ASSERT_TRUE(one_col_->Equals(another_col_));
+  ASSERT_TRUE(one_col_->Equals(*another_col_.get()));
+
+  // Field is different
+  another_field_ = std::make_shared<Field>("two", int32());
+  Construct();
+  ASSERT_FALSE(one_col_->Equals(another_col_));
+  ASSERT_FALSE(one_col_->Equals(*another_col_.get()));
+
+  // ChunkedArray is different
+  another_field_ = std::make_shared<Field>("column", int32());
+  arrays_another_.push_back(array);
+  Construct();
+  ASSERT_FALSE(one_col_->Equals(another_col_));
+  ASSERT_FALSE(one_col_->Equals(*another_col_.get()));
+}
+
 class TestTable : public TestBase {
  public:
   void MakeExample1(int length) {
@@ -63,7 +215,7 @@ class TestTable : public TestBase {
 
 TEST_F(TestTable, EmptySchema) {
   auto empty_schema = shared_ptr<Schema>(new Schema({}));
-  table_.reset(new Table("data", empty_schema, columns_));
+  table_.reset(new Table(empty_schema, columns_));
   ASSERT_OK(table_->ValidateColumns());
   ASSERT_EQ(0, table_->num_rows());
   ASSERT_EQ(0, table_->num_columns());
@@ -73,17 +225,13 @@ TEST_F(TestTable, Ctors) {
   const int length = 100;
   MakeExample1(length);
 
-  std::string name = "data";
-
-  table_.reset(new Table(name, schema_, columns_));
+  table_.reset(new Table(schema_, columns_));
   ASSERT_OK(table_->ValidateColumns());
-  ASSERT_EQ(name, table_->name());
   ASSERT_EQ(length, table_->num_rows());
   ASSERT_EQ(3, table_->num_columns());
 
-  table_.reset(new Table(name, schema_, columns_, length));
+  table_.reset(new Table(schema_, columns_, length));
   ASSERT_OK(table_->ValidateColumns());
-  ASSERT_EQ(name, table_->name());
   ASSERT_EQ(length, table_->num_rows());
 }
 
@@ -91,10 +239,9 @@ TEST_F(TestTable, Metadata) {
   const int length = 100;
   MakeExample1(length);
 
-  std::string name = "data";
-  table_.reset(new Table(name, schema_, columns_));
+  table_.reset(new Table(schema_, columns_));
 
-  ASSERT_TRUE(table_->schema()->Equals(schema_));
+  ASSERT_TRUE(table_->schema()->Equals(*schema_));
 
   auto col = table_->column(0);
   ASSERT_EQ(schema_->field(0)->name, col->name());
@@ -106,13 +253,13 @@ TEST_F(TestTable, InvalidColumns) {
   const int length = 100;
   MakeExample1(length);
 
-  table_.reset(new Table("data", schema_, columns_, length - 1));
+  table_.reset(new Table(schema_, columns_, length - 1));
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
 
   columns_.clear();
 
   // Wrong number of columns
-  table_.reset(new Table("data", schema_, columns_, length));
+  table_.reset(new Table(schema_, columns_, length));
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
 
   columns_ = {
@@ -120,7 +267,7 @@ TEST_F(TestTable, InvalidColumns) {
       std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
       std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length - 1))};
 
-  table_.reset(new Table("data", schema_, columns_, length));
+  table_.reset(new Table(schema_, columns_, length));
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
 }
 
@@ -128,26 +275,22 @@ TEST_F(TestTable, Equals) {
   const int length = 100;
   MakeExample1(length);
 
-  std::string name = "data";
-  table_.reset(new Table(name, schema_, columns_));
+  table_.reset(new Table(schema_, columns_));
 
-  ASSERT_TRUE(table_->Equals(table_));
-  ASSERT_FALSE(table_->Equals(nullptr));
-  // Differing name
-  ASSERT_FALSE(table_->Equals(std::make_shared<Table>("other_name", schema_, columns_)));
+  ASSERT_TRUE(table_->Equals(*table_));
   // Differing schema
   auto f0 = std::make_shared<Field>("f3", int32());
   auto f1 = std::make_shared<Field>("f4", uint8());
   auto f2 = std::make_shared<Field>("f5", int16());
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto other_schema = std::make_shared<Schema>(fields);
-  ASSERT_FALSE(table_->Equals(std::make_shared<Table>(name, other_schema, columns_)));
+  ASSERT_FALSE(table_->Equals(Table(other_schema, columns_)));
   // Differing columns
   std::vector<std::shared_ptr<Column>> other_columns = {
       std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length, 10)),
       std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length, 10)),
       std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length, 10))};
-  ASSERT_FALSE(table_->Equals(std::make_shared<Table>(name, schema_, other_columns)));
+  ASSERT_FALSE(table_->Equals(Table(schema_, other_columns)));
 }
 
 TEST_F(TestTable, FromRecordBatches) {
@@ -157,10 +300,10 @@ TEST_F(TestTable, FromRecordBatches) {
   auto batch1 = std::make_shared<RecordBatch>(schema_, length, arrays_);
 
   std::shared_ptr<Table> result, expected;
-  ASSERT_OK(Table::FromRecordBatches("foo", {batch1}, &result));
+  ASSERT_OK(Table::FromRecordBatches({batch1}, &result));
 
-  expected = std::make_shared<Table>("foo", schema_, columns_);
-  ASSERT_TRUE(result->Equals(expected));
+  expected = std::make_shared<Table>(schema_, columns_);
+  ASSERT_TRUE(result->Equals(*expected));
 
   std::vector<std::shared_ptr<Column>> other_columns;
   for (int i = 0; i < schema_->num_fields(); ++i) {
@@ -168,20 +311,20 @@ TEST_F(TestTable, FromRecordBatches) {
     other_columns.push_back(std::make_shared<Column>(schema_->field(i), col_arrays));
   }
 
-  ASSERT_OK(Table::FromRecordBatches("foo", {batch1, batch1}, &result));
-  expected = std::make_shared<Table>("foo", schema_, other_columns);
-  ASSERT_TRUE(result->Equals(expected));
+  ASSERT_OK(Table::FromRecordBatches({batch1, batch1}, &result));
+  expected = std::make_shared<Table>(schema_, other_columns);
+  ASSERT_TRUE(result->Equals(*expected));
 
   // Error states
   std::vector<std::shared_ptr<RecordBatch>> empty_batches;
-  ASSERT_RAISES(Invalid, Table::FromRecordBatches("", empty_batches, &result));
+  ASSERT_RAISES(Invalid, Table::FromRecordBatches(empty_batches, &result));
 
   std::vector<std::shared_ptr<Field>> fields = {schema_->field(0), schema_->field(1)};
   auto other_schema = std::make_shared<Schema>(fields);
 
   std::vector<std::shared_ptr<Array>> other_arrays = {arrays_[0], arrays_[1]};
   auto batch2 = std::make_shared<RecordBatch>(other_schema, length, other_arrays);
-  ASSERT_RAISES(Invalid, Table::FromRecordBatches("", {batch1, batch2}, &result));
+  ASSERT_RAISES(Invalid, Table::FromRecordBatches({batch1, batch2}, &result));
 }
 
 TEST_F(TestTable, ConcatenateTables) {
@@ -195,25 +338,50 @@ TEST_F(TestTable, ConcatenateTables) {
   auto batch2 = std::make_shared<RecordBatch>(schema_, length, arrays_);
 
   std::shared_ptr<Table> t1, t2, t3, result, expected;
-  ASSERT_OK(Table::FromRecordBatches("foo", {batch1}, &t1));
-  ASSERT_OK(Table::FromRecordBatches("foo", {batch2}, &t2));
+  ASSERT_OK(Table::FromRecordBatches({batch1}, &t1));
+  ASSERT_OK(Table::FromRecordBatches({batch2}, &t2));
 
-  ASSERT_OK(ConcatenateTables("bar", {t1, t2}, &result));
-  ASSERT_OK(Table::FromRecordBatches("bar", {batch1, batch2}, &expected));
-  ASSERT_TRUE(result->Equals(expected));
+  ASSERT_OK(ConcatenateTables({t1, t2}, &result));
+  ASSERT_OK(Table::FromRecordBatches({batch1, batch2}, &expected));
+  ASSERT_TRUE(result->Equals(*expected));
 
   // Error states
   std::vector<std::shared_ptr<Table>> empty_tables;
-  ASSERT_RAISES(Invalid, ConcatenateTables("", empty_tables, &result));
+  ASSERT_RAISES(Invalid, ConcatenateTables(empty_tables, &result));
 
   std::vector<std::shared_ptr<Field>> fields = {schema_->field(0), schema_->field(1)};
   auto other_schema = std::make_shared<Schema>(fields);
 
   std::vector<std::shared_ptr<Array>> other_arrays = {arrays_[0], arrays_[1]};
   auto batch3 = std::make_shared<RecordBatch>(other_schema, length, other_arrays);
-  ASSERT_OK(Table::FromRecordBatches("", {batch3}, &t3));
+  ASSERT_OK(Table::FromRecordBatches({batch3}, &t3));
+
+  ASSERT_RAISES(Invalid, ConcatenateTables({t1, t3}, &result));
+}
+
+TEST_F(TestTable, RemoveColumn) {
+  const int64_t length = 10;
+  MakeExample1(length);
+
+  Table table(schema_, columns_);
+
+  std::shared_ptr<Table> result;
+  ASSERT_OK(table.RemoveColumn(0, &result));
+
+  auto ex_schema =
+      std::shared_ptr<Schema>(new Schema({schema_->field(1), schema_->field(2)}));
+  std::vector<std::shared_ptr<Column>> ex_columns = {table.column(1), table.column(2)};
+  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+
+  ASSERT_OK(table.RemoveColumn(1, &result));
+  ex_schema = std::shared_ptr<Schema>(new Schema({schema_->field(0), schema_->field(2)}));
+  ex_columns = {table.column(0), table.column(2)};
+  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
-  ASSERT_RAISES(Invalid, ConcatenateTables("foo", {t1, t3}, &result));
+  ASSERT_OK(table.RemoveColumn(2, &result));
+  ex_schema = std::shared_ptr<Schema>(new Schema({schema_->field(0), schema_->field(1)}));
+  ex_columns = {table.column(0), table.column(1)};
+  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 }
 
 class TestRecordBatch : public TestBase {};
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 3f254aae6d3fa..8e283f4da9bb7 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -23,12 +23,122 @@
 #include <sstream>
 
 #include "arrow/array.h"
-#include "arrow/column.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/stl.h"
 
 namespace arrow {
 
+// ----------------------------------------------------------------------
+// ChunkedArray and Column methods
+
+ChunkedArray::ChunkedArray(const ArrayVector& chunks) : chunks_(chunks) {
+  length_ = 0;
+  null_count_ = 0;
+  for (const std::shared_ptr<Array>& chunk : chunks) {
+    length_ += chunk->length();
+    null_count_ += chunk->null_count();
+  }
+}
+
+bool ChunkedArray::Equals(const ChunkedArray& other) const {
+  if (length_ != other.length()) { return false; }
+  if (null_count_ != other.null_count()) { return false; }
+
+  // Check contents of the underlying arrays. This checks for equality of
+  // the underlying data independently of the chunk size.
+  int this_chunk_idx = 0;
+  int64_t this_start_idx = 0;
+  int other_chunk_idx = 0;
+  int64_t other_start_idx = 0;
+
+  int64_t elements_compared = 0;
+  while (elements_compared < length_) {
+    const std::shared_ptr<Array> this_array = chunks_[this_chunk_idx];
+    const std::shared_ptr<Array> other_array = other.chunk(other_chunk_idx);
+    int64_t common_length = std::min(
+        this_array->length() - this_start_idx, other_array->length() - other_start_idx);
+    if (!this_array->RangeEquals(this_start_idx, this_start_idx + common_length,
+            other_start_idx, other_array)) {
+      return false;
+    }
+
+    elements_compared += common_length;
+
+    // If we have exhausted the current chunk, proceed to the next one individually.
+    if (this_start_idx + common_length == this_array->length()) {
+      this_chunk_idx++;
+      this_start_idx = 0;
+    } else {
+      this_start_idx += common_length;
+    }
+
+    if (other_start_idx + common_length == other_array->length()) {
+      other_chunk_idx++;
+      other_start_idx = 0;
+    } else {
+      other_start_idx += common_length;
+    }
+  }
+  return true;
+}
+
+bool ChunkedArray::Equals(const std::shared_ptr<ChunkedArray>& other) const {
+  if (this == other.get()) { return true; }
+  if (!other) { return false; }
+  return Equals(*other.get());
+}
+
+Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks)
+    : field_(field) {
+  data_ = std::make_shared<ChunkedArray>(chunks);
+}
+
+Column::Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data)
+    : field_(field) {
+  if (data) {
+    data_ = std::make_shared<ChunkedArray>(ArrayVector({data}));
+  } else {
+    data_ = std::make_shared<ChunkedArray>(ArrayVector({}));
+  }
+}
+
+Column::Column(const std::string& name, const std::shared_ptr<Array>& data)
+    : Column(::arrow::field(name, data->type()), data) {}
+
+Column::Column(
+    const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data)
+    : field_(field), data_(data) {}
+
+bool Column::Equals(const Column& other) const {
+  if (!field_->Equals(other.field())) { return false; }
+  return data_->Equals(other.data());
+}
+
+bool Column::Equals(const std::shared_ptr<Column>& other) const {
+  if (this == other.get()) { return true; }
+  if (!other) { return false; }
+
+  return Equals(*other.get());
+}
+
+Status Column::ValidateData() {
+  for (int i = 0; i < data_->num_chunks(); ++i) {
+    std::shared_ptr<DataType> type = data_->chunk(i)->type();
+    if (!this->type()->Equals(type)) {
+      std::stringstream ss;
+      ss << "In chunk " << i << " expected type " << this->type()->ToString()
+         << " but saw " << type->ToString();
+      return Status::Invalid(ss.str());
+    }
+  }
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// RecordBatch methods
+
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
     const std::vector<std::shared_ptr<Array>>& columns)
     : schema_(schema), num_rows_(num_rows), columns_(columns) {}
@@ -83,9 +193,9 @@ std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length)
 // ----------------------------------------------------------------------
 // Table methods
 
-Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
+Table::Table(const std::shared_ptr<Schema>& schema,
     const std::vector<std::shared_ptr<Column>>& columns)
-    : name_(name), schema_(schema), columns_(columns) {
+    : schema_(schema), columns_(columns) {
   if (columns.size() == 0) {
     num_rows_ = 0;
   } else {
@@ -93,12 +203,11 @@ Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
   }
 }
 
-Table::Table(const std::string& name, const std::shared_ptr<Schema>& schema,
+Table::Table(const std::shared_ptr<Schema>& schema,
     const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
-    : name_(name), schema_(schema), columns_(columns), num_rows_(num_rows) {}
+    : schema_(schema), columns_(columns), num_rows_(num_rows) {}
 
-Status Table::FromRecordBatches(const std::string& name,
-    const std::vector<std::shared_ptr<RecordBatch>>& batches,
+Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>& batches,
     std::shared_ptr<Table>* table) {
   if (batches.size() == 0) {
     return Status::Invalid("Must pass at least one record batch");
@@ -110,7 +219,7 @@ Status Table::FromRecordBatches(const std::string& name,
   const int ncolumns = static_cast<int>(schema->num_fields());
 
   for (int i = 1; i < nbatches; ++i) {
-    if (!batches[i]->schema()->Equals(schema)) {
+    if (!batches[i]->schema()->Equals(*schema)) {
       std::stringstream ss;
       ss << "Schema at index " << static_cast<int>(i) << " was different: \n"
          << schema->ToString() << "\nvs\n"
@@ -129,11 +238,11 @@ Status Table::FromRecordBatches(const std::string& name,
     columns[i] = std::make_shared<Column>(schema->field(i), column_arrays);
   }
 
-  *table = std::make_shared<Table>(name, schema, columns);
+  *table = std::make_shared<Table>(schema, columns);
   return Status::OK();
 }
 
-Status ConcatenateTables(const std::string& output_name,
+Status ConcatenateTables(
     const std::vector<std::shared_ptr<Table>>& tables, std::shared_ptr<Table>* table) {
   if (tables.size() == 0) { return Status::Invalid("Must pass at least one table"); }
 
@@ -143,7 +252,7 @@ Status ConcatenateTables(const std::string& output_name,
   const int ncolumns = static_cast<int>(schema->num_fields());
 
   for (int i = 1; i < ntables; ++i) {
-    if (!tables[i]->schema()->Equals(schema)) {
+    if (!tables[i]->schema()->Equals(*schema)) {
       std::stringstream ss;
       ss << "Schema at index " << static_cast<int>(i) << " was different: \n"
          << schema->ToString() << "\nvs\n"
@@ -164,13 +273,13 @@ Status ConcatenateTables(const std::string& output_name,
     }
     columns[i] = std::make_shared<Column>(schema->field(i), column_arrays);
   }
-  *table = std::make_shared<Table>(output_name, schema, columns);
+  *table = std::make_shared<Table>(schema, columns);
   return Status::OK();
 }
 
 bool Table::Equals(const Table& other) const {
-  if (name_ != other.name()) { return false; }
-  if (!schema_->Equals(other.schema())) { return false; }
+  if (this == &other) { return true; }
+  if (!schema_->Equals(*other.schema())) { return false; }
   if (static_cast<int64_t>(columns_.size()) != other.num_columns()) { return false; }
 
   for (int i = 0; i < static_cast<int>(columns_.size()); i++) {
@@ -179,10 +288,12 @@ bool Table::Equals(const Table& other) const {
   return true;
 }
 
-bool Table::Equals(const std::shared_ptr<Table>& other) const {
-  if (this == other.get()) { return true; }
-  if (!other) { return false; }
-  return Equals(*other.get());
+Status Table::RemoveColumn(int i, std::shared_ptr<Table>* out) const {
+  std::shared_ptr<Schema> new_schema;
+  RETURN_NOT_OK(schema_->RemoveField(i, &new_schema));
+
+  *out = std::make_shared<Table>(new_schema, DeleteVectorElement(columns_, i));
+  return Status::OK();
 }
 
 Status Table::ValidateColumns() const {
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index bf0d99c4e9d2b..7b739c9a1b314 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+#include "arrow/type.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -32,6 +33,74 @@ class Column;
 class Schema;
 class Status;
 
+using ArrayVector = std::vector<std::shared_ptr<Array>>;
+
+// A data structure managing a list of primitive Arrow arrays logically as one
+// large array
+class ARROW_EXPORT ChunkedArray {
+ public:
+  explicit ChunkedArray(const ArrayVector& chunks);
+
+  // @returns: the total length of the chunked array; computed on construction
+  int64_t length() const { return length_; }
+
+  int64_t null_count() const { return null_count_; }
+
+  int num_chunks() const { return static_cast<int>(chunks_.size()); }
+
+  std::shared_ptr<Array> chunk(int i) const { return chunks_[i]; }
+
+  const ArrayVector& chunks() const { return chunks_; }
+
+  bool Equals(const ChunkedArray& other) const;
+  bool Equals(const std::shared_ptr<ChunkedArray>& other) const;
+
+ protected:
+  ArrayVector chunks_;
+  int64_t length_;
+  int64_t null_count_;
+};
+
+// An immutable column data structure consisting of a field (type metadata) and
+// a logical chunked data array (which can be validated as all being the same
+// type).
+class ARROW_EXPORT Column {
+ public:
+  Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
+  Column(const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data);
+
+  Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data);
+
+  /// Construct from name and array
+  Column(const std::string& name, const std::shared_ptr<Array>& data);
+
+  int64_t length() const { return data_->length(); }
+
+  int64_t null_count() const { return data_->null_count(); }
+
+  std::shared_ptr<Field> field() const { return field_; }
+
+  // @returns: the column's name in the passed metadata
+  const std::string& name() const { return field_->name; }
+
+  // @returns: the column's type according to the metadata
+  std::shared_ptr<DataType> type() const { return field_->type; }
+
+  // @returns: the column's data as a chunked logical array
+  std::shared_ptr<ChunkedArray> data() const { return data_; }
+
+  bool Equals(const Column& other) const;
+  bool Equals(const std::shared_ptr<Column>& other) const;
+
+  // Verify that the column's array data is consistent with the passed field's
+  // metadata
+  Status ValidateData();
+
+ protected:
+  std::shared_ptr<Field> field_;
+  std::shared_ptr<ChunkedArray> data_;
+};
+
 // A record batch is a simpler and more rigid table data structure intended for
 // use primarily in shared memory IPC. It contains a schema (metadata) and a
 // corresponding sequence of equal-length Arrow arrays
@@ -81,25 +150,22 @@ class ARROW_EXPORT RecordBatch {
 class ARROW_EXPORT Table {
  public:
   // If columns is zero-length, the table's number of rows is zero
-  Table(const std::string& name, const std::shared_ptr<Schema>& schema,
+  Table(const std::shared_ptr<Schema>& schema,
       const std::vector<std::shared_ptr<Column>>& columns);
 
   // num_rows is a parameter to allow for tables of a particular size not
   // having any materialized columns. Each column should therefore have the
   // same length as num_rows -- you can validate this using
   // Table::ValidateColumns
-  Table(const std::string& name, const std::shared_ptr<Schema>& schema,
-      const std::vector<std::shared_ptr<Column>>& columns, int64_t nubm_rows);
+  Table(const std::shared_ptr<Schema>& schema,
+      const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows);
 
   // Construct table from RecordBatch, but only if all of the batch schemas are
   // equal. Returns Status::Invalid if there is some problem
-  static Status FromRecordBatches(const std::string& name,
+  static Status FromRecordBatches(
       const std::vector<std::shared_ptr<RecordBatch>>& batches,
       std::shared_ptr<Table>* table);
 
-  // @returns: the table's name, if any (may be length 0)
-  const std::string& name() const { return name_; }
-
   // @returns: the table's schema
   std::shared_ptr<Schema> schema() const { return schema_; }
 
@@ -107,6 +173,10 @@ class ARROW_EXPORT Table {
   // @returns: the i-th column
   std::shared_ptr<Column> column(int i) const { return columns_[i]; }
 
+  /// Remove column from the table, producing a new Table (because tables and
+  /// schemas are immutable)
+  Status RemoveColumn(int i, std::shared_ptr<Table>* out) const;
+
   // @returns: the number of columns in the table
   int num_columns() const { return static_cast<int>(columns_.size()); }
 
@@ -114,15 +184,11 @@ class ARROW_EXPORT Table {
   int64_t num_rows() const { return num_rows_; }
 
   bool Equals(const Table& other) const;
-  bool Equals(const std::shared_ptr<Table>& other) const;
 
   // After construction, perform any checks to validate the input arguments
   Status ValidateColumns() const;
 
  private:
-  // The table's name, optional
-  std::string name_;
-
   std::shared_ptr<Schema> schema_;
   std::vector<std::shared_ptr<Column>> columns_;
 
@@ -131,7 +197,7 @@ class ARROW_EXPORT Table {
 
 // Construct table from multiple input tables. Return Status::Invalid if
 // schemas are not equal
-Status ARROW_EXPORT ConcatenateTables(const std::string& output_name,
+Status ARROW_EXPORT ConcatenateTables(
     const std::vector<std::shared_ptr<Table>>& tables, std::shared_ptr<Table>* table);
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h
index f704b6b545b7d..dc11e76edf465 100644
--- a/cpp/src/arrow/test-common.h
+++ b/cpp/src/arrow/test-common.h
@@ -29,7 +29,6 @@
 #include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
-#include "arrow/column.h"
 #include "arrow/memory_pool.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index bed555984fb68..94937b592cc33 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -30,9 +30,7 @@
 #include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
-#include "arrow/column.h"
 #include "arrow/memory_pool.h"
-#include "arrow/schema.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index ed8654314ee6d..70c173432a960 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -23,7 +23,6 @@
 
 #include "gtest/gtest.h"
 
-#include "arrow/schema.h"
 #include "arrow/type.h"
 
 using std::shared_ptr;
@@ -75,11 +74,8 @@ TEST_F(TestSchema, Basics) {
 
   vector<shared_ptr<Field>> fields3 = {f0, f1_optional, f2};
   auto schema3 = std::make_shared<Schema>(fields3);
-  ASSERT_TRUE(schema->Equals(schema2));
-  ASSERT_FALSE(schema->Equals(schema3));
-
-  ASSERT_TRUE(schema->Equals(*schema2.get()));
-  ASSERT_FALSE(schema->Equals(*schema3.get()));
+  ASSERT_TRUE(schema->Equals(*schema2));
+  ASSERT_FALSE(schema->Equals(*schema3));
 }
 
 TEST_F(TestSchema, ToString) {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index c790f6e5a4345..e6e6f5c3e8bc7 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -24,6 +24,7 @@
 #include "arrow/compare.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/stl.h"
 #include "arrow/visitor.h"
 
 namespace arrow {
@@ -45,6 +46,8 @@ std::string Field::ToString() const {
   return ss.str();
 }
 
+DataType::~DataType() {}
+
 bool DataType::Equals(const DataType& other) const {
   bool are_equal = false;
   Status error = TypeEquals(*this, other, &are_equal);
@@ -224,6 +227,56 @@ std::string NullType::ToString() const {
   return name();
 }
 
+// ----------------------------------------------------------------------
+// Schema implementation
+
+Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields) : fields_(fields) {}
+
+bool Schema::Equals(const Schema& other) const {
+  if (this == &other) { return true; }
+
+  if (num_fields() != other.num_fields()) { return false; }
+  for (int i = 0; i < num_fields(); ++i) {
+    if (!field(i)->Equals(*other.field(i).get())) { return false; }
+  }
+  return true;
+}
+
+std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
+  if (fields_.size() > 0 && name_to_index_.size() == 0) {
+    for (size_t i = 0; i < fields_.size(); ++i) {
+      name_to_index_[fields_[i]->name] = static_cast<int>(i);
+    }
+  }
+
+  auto it = name_to_index_.find(name);
+  if (it == name_to_index_.end()) {
+    return nullptr;
+  } else {
+    return fields_[it->second];
+  }
+}
+
+Status Schema::RemoveField(int i, std::shared_ptr<Schema>* out) const {
+  DCHECK_GE(i, 0);
+  DCHECK_LT(i, this->num_fields());
+
+  *out = std::make_shared<Schema>(DeleteVectorElement(fields_, i));
+  return Status::OK();
+}
+
+std::string Schema::ToString() const {
+  std::stringstream buffer;
+
+  int i = 0;
+  for (auto field : fields_) {
+    if (i > 0) { buffer << std::endl; }
+    buffer << field->ToString();
+    ++i;
+  }
+  return buffer.str();
+}
+
 // ----------------------------------------------------------------------
 // Visitors and factory functions
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 2a73f6be934eb..4f931907ee79f 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -22,6 +22,7 @@
 #include <memory>
 #include <ostream>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #include "arrow/status.h"
@@ -132,7 +133,7 @@ struct ARROW_EXPORT DataType {
 
   explicit DataType(Type::type type) : type(type) {}
 
-  virtual ~DataType() = default;
+  virtual ~DataType();
 
   // Return whether the types are equal
   //
@@ -596,6 +597,36 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
   bool ordered_;
 };
 
+// ----------------------------------------------------------------------
+// Schema
+
+class ARROW_EXPORT Schema {
+ public:
+  explicit Schema(const std::vector<std::shared_ptr<Field>>& fields);
+
+  // Returns true if all of the schema fields are equal
+  bool Equals(const Schema& other) const;
+
+  // Return the ith schema element. Does not boundscheck
+  std::shared_ptr<Field> field(int i) const { return fields_[i]; }
+
+  // Returns nullptr if name not found
+  std::shared_ptr<Field> GetFieldByName(const std::string& name);
+
+  const std::vector<std::shared_ptr<Field>>& fields() const { return fields_; }
+
+  // Render a string representation of the schema suitable for debugging
+  std::string ToString() const;
+
+  Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
+
+  int num_fields() const { return static_cast<int>(fields_.size()); }
+
+ private:
+  std::vector<std::shared_ptr<Field>> fields_;
+  std::unordered_map<std::string, int> name_to_index_;
+};
+
 // ----------------------------------------------------------------------
 // Factory functions
 
diff --git a/cpp/src/arrow/util/stl.h b/cpp/src/arrow/util/stl.h
new file mode 100644
index 0000000000000..3ec535d62b920
--- /dev/null
+++ b/cpp/src/arrow/util/stl.h
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_STL_H
+#define ARROW_UTIL_STL_H
+
+#include <vector>
+
+namespace arrow {
+
+template <typename T>
+inline std::vector<T> DeleteVectorElement(const std::vector<T>& values, size_t index) {
+  std::vector<T> out;
+  out.reserve(values.size() - 1);
+  for (size_t i = 0; i < index; ++i) {
+    out.push_back(values[i]);
+  }
+  for (size_t i = index + 1; i < values.size(); ++i) {
+    out.push_back(values[i]);
+  }
+  return out;
+}
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_STL_H
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 654f5ab527043..6cae1966cb16e 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -19,6 +19,8 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+from cython.operator cimport dereference as deref
+
 import numpy as np
 
 from pyarrow.includes.libarrow cimport *
@@ -216,7 +218,7 @@ cdef class Array:
         return '{0}\n{1}'.format(type_format, values)
 
     def equals(Array self, Array other):
-        return self.ap.Equals(other.sp_array)
+        return self.ap.Equals(deref(other.ap))
 
     def __len__(self):
         if self.sp_array.get():
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index bdbd18bce01df..8e428b40b8f8b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -59,7 +59,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CDataType" arrow::DataType":
         Type type
 
-        c_bool Equals(const shared_ptr[CDataType]& other)
         c_bool Equals(const CDataType& other)
 
         c_string ToString()
@@ -71,7 +70,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int64_t null_count()
         Type type_enum()
 
-        c_bool Equals(const shared_ptr[CArray]& arr)
+        c_bool Equals(const CArray& arr)
         c_bool IsNull(int i)
 
         shared_ptr[CArray] Slice(int64_t offset)
@@ -155,7 +154,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CSchema" arrow::Schema":
         CSchema(const vector[shared_ptr[CField]]& fields)
 
-        c_bool Equals(const shared_ptr[CSchema]& other)
+        c_bool Equals(const CSchema& other)
 
         shared_ptr[CField] field(int i)
         shared_ptr[CField] GetFieldByName(c_string& name)
@@ -231,7 +230,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
                 const vector[shared_ptr[CArray]]& chunks)
 
         c_bool Equals(const CColumn& other)
-        c_bool Equals(const shared_ptr[CColumn]& other)
 
         int64_t length()
         int64_t null_count()
@@ -258,12 +256,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CRecordBatch] Slice(int64_t offset, int64_t length)
 
     cdef cppclass CTable" arrow::Table":
-        CTable(const c_string& name, const shared_ptr[CSchema]& schema,
+        CTable(const shared_ptr[CSchema]& schema,
                const vector[shared_ptr[CColumn]]& columns)
 
         @staticmethod
         CStatus FromRecordBatches(
-            const c_string& name,
             const vector[shared_ptr[CRecordBatch]]& batches,
             shared_ptr[CTable]* table)
 
@@ -271,15 +268,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int num_rows()
 
         c_bool Equals(const CTable& other)
-        c_bool Equals(const shared_ptr[CTable]& other)
-
-        const c_string& name()
 
         shared_ptr[CSchema] schema()
         shared_ptr[CColumn] column(int i)
 
-    CStatus ConcatenateTables(const c_string& output_name,
-                              const vector[shared_ptr[CTable]]& tables,
+        CStatus RemoveColumn(int i, shared_ptr[CTable]* out)
+
+    CStatus ConcatenateTables(const vector[shared_ptr[CTable]]& tables,
                               shared_ptr[CTable]* result)
 
 
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index cb44ce8816338..d528bdc495208 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -956,7 +956,6 @@ cdef class _StreamReader:
             vector[shared_ptr[CRecordBatch]] batches
             shared_ptr[CRecordBatch] batch
             shared_ptr[CTable] table
-            c_string name = b''
 
         with nogil:
             while True:
@@ -965,7 +964,7 @@ cdef class _StreamReader:
                     break
                 batches.push_back(batch)
 
-            check_status(CTable.FromRecordBatches(name, batches, &table))
+            check_status(CTable.FromRecordBatches(batches, &table))
 
         return table_from_ctable(table)
 
@@ -1033,7 +1032,6 @@ cdef class _FileReader:
         cdef:
             vector[shared_ptr[CRecordBatch]] batches
             shared_ptr[CTable] table
-            c_string name = b''
             int i, nbatches
 
         nbatches = self.num_record_batches
@@ -1042,6 +1040,6 @@ cdef class _FileReader:
         with nogil:
             for i in range(nbatches):
                 check_status(self.reader.get().GetRecordBatch(i, &batches[i]))
-            check_status(CTable.FromRecordBatches(name, batches, &table))
+            check_status(CTable.FromRecordBatches(batches, &table))
 
         return table_from_ctable(table)
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index ab5ae5fa2a3f2..4f02901cc9a11 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -166,7 +166,7 @@ cdef class Schema:
         cdef Schema _other
         _other = other
 
-        return self.sp_schema.get().Equals(_other.sp_schema)
+        return self.sp_schema.get().Equals(deref(_other.schema))
 
     def field_by_name(self, name):
         """
@@ -200,11 +200,16 @@ cdef class Schema:
 
         return result
 
-    def __repr__(self):
+    def __str__(self):
         return frombytes(self.schema.ToString())
 
+    def __repr__(self):
+        return self.__str__()
+
+
 cdef dict _type_cache = {}
 
+
 cdef DataType primitive_type(Type type):
     if type in _type_cache:
         return _type_cache[type]
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 58f5d680393f7..e6fddbd0cfbbd 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -298,7 +298,7 @@ cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
 
 
 
-cdef _dataframe_to_arrays(df, name, timestamps_to_ms, Schema schema):
+cdef _dataframe_to_arrays(df, timestamps_to_ms, Schema schema):
     cdef:
         list names = []
         list arrays = []
@@ -474,7 +474,7 @@ cdef class RecordBatch:
         -------
         pyarrow.table.RecordBatch
         """
-        names, arrays = _dataframe_to_arrays(df, None, False, schema)
+        names, arrays = _dataframe_to_arrays(df, False, schema)
         return cls.from_arrays(arrays, names)
 
     @staticmethod
@@ -573,6 +573,9 @@ cdef class Table:
     def __cinit__(self):
         self.table = NULL
 
+    def __repr__(self):
+        return 'pyarrow.Table\n{0}'.format(str(self.schema))
+
     cdef init(self, const shared_ptr[CTable]& table):
         self.sp_table = table
         self.table = table.get()
@@ -608,7 +611,7 @@ cdef class Table:
         return result
 
     @classmethod
-    def from_pandas(cls, df, name=None, timestamps_to_ms=False, schema=None):
+    def from_pandas(cls, df, timestamps_to_ms=False, schema=None):
         """
         Convert pandas.DataFrame to an Arrow Table
 
@@ -616,8 +619,6 @@ cdef class Table:
         ----------
         df: pandas.DataFrame
 
-        name: str
-
         timestamps_to_ms: bool
             Convert datetime columns to ms resolution. This is needed for
             compability with other functionality like Parquet I/O which
@@ -643,13 +644,13 @@ cdef class Table:
         >>> pa.Table.from_pandas(df)
         <pyarrow.table.Table object at 0x7f05d1fb1b40>
         """
-        names, arrays = _dataframe_to_arrays(df, name=name,
+        names, arrays = _dataframe_to_arrays(df,
                                              timestamps_to_ms=timestamps_to_ms,
                                              schema=schema)
-        return cls.from_arrays(arrays, names=names, name=name)
+        return cls.from_arrays(arrays, names=names)
 
     @staticmethod
-    def from_arrays(arrays, names=None, name=None):
+    def from_arrays(arrays, names=None):
         """
         Construct a Table from Arrow arrays or columns
 
@@ -660,8 +661,6 @@ cdef class Table:
         names: list of str, optional
             Names for the table columns. If Columns passed, will be
             inferred. If Arrays passed, this argument is required
-        name: str, optional
-            name for the Table
 
         Returns
         -------
@@ -669,7 +668,6 @@ cdef class Table:
 
         """
         cdef:
-            c_string c_name
             vector[shared_ptr[CField]] fields
             vector[shared_ptr[CColumn]] columns
             shared_ptr[CSchema] schema
@@ -689,16 +687,11 @@ cdef class Table:
             else:
                 raise ValueError(type(arrays[i]))
 
-        if name is None:
-            c_name = ''
-        else:
-            c_name = tobytes(name)
-
-        table.reset(new CTable(c_name, schema, columns))
+        table.reset(new CTable(schema, columns))
         return table_from_ctable(table)
 
     @staticmethod
-    def from_batches(batches, name=None):
+    def from_batches(batches):
         """
         Construct a Table from a list of Arrow RecordBatches
 
@@ -712,16 +705,12 @@ cdef class Table:
             vector[shared_ptr[CRecordBatch]] c_batches
             shared_ptr[CTable] c_table
             RecordBatch batch
-            Table table
-            c_string c_name
-
-        c_name = b'' if name is None else tobytes(name)
 
         for batch in batches:
             c_batches.push_back(batch.sp_batch)
 
         with nogil:
-            check_status(CTable.FromRecordBatches(c_name, c_batches, &c_table))
+            check_status(CTable.FromRecordBatches(c_batches, &c_table))
 
         return table_from_ctable(c_table)
 
@@ -761,18 +750,6 @@ cdef class Table:
             entries.append((name, column))
         return OrderedDict(entries)
 
-    @property
-    def name(self):
-        """
-        Label of the table
-
-        Returns
-        -------
-        str
-        """
-        self._check_nullptr()
-        return frombytes(self.table.name())
-
     @property
     def schema(self):
         """
@@ -851,8 +828,19 @@ cdef class Table:
         """
         return (self.num_rows, self.num_columns)
 
+    def remove_column(self, int i):
+        """
+        Create new Table with the indicated column removed
+        """
+        cdef shared_ptr[CTable] c_table
 
-def concat_tables(tables, output_name=None):
+        with nogil:
+            check_status(self.table.RemoveColumn(i, &c_table))
+
+        return table_from_ctable(c_table)
+
+
+def concat_tables(tables):
     """
     Perform zero-copy concatenation of pyarrow.Table objects. Raises exception
     if all of the Table schemas are not the same
@@ -867,15 +855,12 @@ def concat_tables(tables, output_name=None):
         vector[shared_ptr[CTable]] c_tables
         shared_ptr[CTable] c_result
         Table table
-        c_string c_name
-
-    c_name = b'' if output_name is None else tobytes(output_name)
 
     for table in tables:
         c_tables.push_back(table.sp_table)
 
     with nogil:
-        check_status(ConcatenateTables(c_name, c_tables, &c_result))
+        check_status(ConcatenateTables(c_tables, &c_result))
 
     return table_from_ctable(c_result)
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index c72ff9e862b76..fc32b9fac8b98 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -47,7 +47,7 @@ def test_single_pylist_column_roundtrip(tmpdir):
         filename = tmpdir.join('single_{}_column.parquet'
                                .format(dtype.__name__))
         data = [pa.from_pylist(list(map(dtype, range(5))))]
-        table = pa.Table.from_arrays(data, names=('a', 'b'), name='table_name')
+        table = pa.Table.from_arrays(data, names=('a', 'b'))
         pq.write_table(table, filename.strpath)
         table_read = pq.read_table(filename.strpath)
         for col_written, col_read in zip(table.itercolumns(),
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 67f1892a9987b..548f4782a7030 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -31,7 +31,7 @@ def test_basics(self):
         data = [
             pa.from_pylist([-10, -5, 0, 5, 10])
         ]
-        table = pa.Table.from_arrays(data, names=['a'], name='table_name')
+        table = pa.Table.from_arrays(data, names=['a'])
         column = table.column(0)
         assert column.name == 'a'
         assert column.length() == 5
@@ -43,7 +43,7 @@ def test_pandas(self):
         data = [
             pa.from_pylist([-10, -5, 0, 5, 10])
         ]
-        table = pa.Table.from_arrays(data, names=['a'], name='table_name')
+        table = pa.Table.from_arrays(data, names=['a'])
         column = table.column(0)
         series = column.to_pandas()
         assert series.name == 'a'
@@ -154,8 +154,7 @@ def test_table_basics():
         pa.from_pylist(range(5)),
         pa.from_pylist([-10, -5, 0, 5, 10])
     ]
-    table = pa.Table.from_arrays(data, names=('a', 'b'), name='table_name')
-    assert table.name == 'table_name'
+    table = pa.Table.from_arrays(data, names=('a', 'b'))
     assert len(table) == 5
     assert table.num_rows == 5
     assert table.num_columns == 2
@@ -170,6 +169,19 @@ def test_table_basics():
             assert chunk is not None
 
 
+def test_table_remove_column():
+    data = [
+        pa.from_pylist(range(5)),
+        pa.from_pylist([-10, -5, 0, 5, 10]),
+        pa.from_pylist(range(5, 10))
+    ]
+    table = pa.Table.from_arrays(data, names=('a', 'b', 'c'))
+
+    t2 = table.remove_column(0)
+    expected = pa.Table.from_arrays(data[1:], names=('b', 'c'))
+    assert t2.equals(expected)
+
+
 def test_concat_tables():
     data = [
         list(range(5)),
@@ -181,18 +193,16 @@ def test_concat_tables():
     ]
 
     t1 = pa.Table.from_arrays([pa.from_pylist(x) for x in data],
-                              names=('a', 'b'), name='table_name')
+                              names=('a', 'b'))
     t2 = pa.Table.from_arrays([pa.from_pylist(x) for x in data2],
-                              names=('a', 'b'), name='table_name')
+                              names=('a', 'b'))
 
-    result = pa.concat_tables([t1, t2], output_name='foo')
-    assert result.name == 'foo'
+    result = pa.concat_tables([t1, t2])
     assert len(result) == 10
 
     expected = pa.Table.from_arrays([pa.from_pylist(x + y)
                                      for x, y in zip(data, data2)],
-                                    names=('a', 'b'),
-                                    name='foo')
+                                    names=('a', 'b'))
 
     assert result.equals(expected)
 
@@ -202,8 +212,7 @@ def test_table_pandas():
         pa.from_pylist(range(5)),
         pa.from_pylist([-10, -5, 0, 5, 10])
     ]
-    table = pa.Table.from_arrays(data, names=('a', 'b'),
-                                 name='table_name')
+    table = pa.Table.from_arrays(data, names=('a', 'b'))
 
     # TODO: Use this part once from_pandas is implemented
     # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}

From 15b874e47e3975c5240290ec7ed105bf8d1b56bc Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Thu, 30 Mar 2017 15:13:39 -0400
Subject: [PATCH 0433/1644] ARROW-699: [C++] Resolve Arrow and Arrow IPC build
 issues on Windows;

Resolve Arrow and Arrow IPC build issues on Windows; Running unit tests in Appveyor.

Changes description:

- Current file.cc implementation ( https://github.com/apache/arrow/compare/master...MaxRis:ARROW-699?expand=1#diff-1b2fb57add5bb8f21e28a707f24462b0L161 ) assumes that input file name is encoded with utf-16 inside std::string. But unit tests are passing just utf-8 compatible C-strings.
Util method Utf8ToUtf16 introduced ( https://github.com/apache/arrow/compare/master...MaxRis:ARROW-699?expand=1#diff-1b2fb57add5bb8f21e28a707f24462b0R156 ) to convert utf-8 to utf-16 (std::wstring).

- io-file-test has FIleIsClosed method which uses method _close(FILE_HANDLE) to test if file handle valid or invalid. Result is interpreted as file was closed or not. MSVC C runtime implementation by default crashes application if input param is invalid. To overwrite this behavior it's needed to set custom hander (https://github.com/apache/arrow/compare/master...MaxRis:ARROW-699?expand=1#diff-05724c5d85bf64720fa85ef3012e470dR61). More info here: https://msdn.microsoft.com/en-us/library/ksazx244.aspx

- Message and FileWriter classes keeps their internal implementation as private member of unique_ptr of FORWARD declared type, for example:

```
class MessageImpl;
std::unique_ptr<MessageImpl> impl_;
```

MSVC compiler requires constructor and destructor of Message class be defined. Currently, they are defined by default, and because of this, compiler places auto generated code into the .hpp file, which is not visible for others libs during the linking (We got unresolved linking issues). The solution is to define destructors explicitly. More there http://stackoverflow.com/a/42158611/2266412 and there http://stackoverflow.com/a/6089065/2266412

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #449 from MaxRis/ARROW-699 and squashes the following commits:

2d5383f [Max Risuhin] ARROW-699: [C++] Resolve Arrow and Arrow IPC build issues on Windows; Running unit tests in Appveyor.
---
 appveyor.yml                       | 12 +++++----
 cpp/CMakeLists.txt                 | 11 +++++++-
 cpp/cmake_modules/BuildUtils.cmake |  2 ++
 cpp/src/arrow/io/file.cc           | 42 +++++++++++++++++-------------
 cpp/src/arrow/io/io-file-test.cc   | 23 +++++++++++++---
 cpp/src/arrow/io/io-hdfs-test.cc   |  5 ++--
 cpp/src/arrow/io/test-common.h     | 10 ++++---
 cpp/src/arrow/ipc/CMakeLists.txt   | 33 +++++++++++++++++------
 cpp/src/arrow/ipc/metadata.cc      |  2 ++
 cpp/src/arrow/ipc/metadata.h       |  1 +
 cpp/src/arrow/ipc/writer.cc        |  4 +++
 cpp/src/arrow/ipc/writer.h         |  4 ++-
 12 files changed, 106 insertions(+), 43 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 17362c993d053..9f3594907d17e 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -23,8 +23,8 @@ environment:
     - GENERATOR: Visual Studio 14 2015 Win64
     # - GENERATOR: Visual Studio 14 2015
   MSVC_DEFAULT_OPTIONS: ON
-  BOOST_ROOT: C:\Libraries\boost_1_59_0
-  BOOST_LIBRARYDIR: C:\Libraries\boost_1_59_0\lib64-msvc-14.0
+  BOOST_ROOT: C:\Libraries\boost_1_63_0
+  BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
 
 build_script:
  - cd cpp
@@ -32,8 +32,10 @@ build_script:
  - cd build
  # A lot of features are still deactivated as they do not build on Windows
  #  * gbenchmark doesn't build with MSVC
- - cmake -G "%GENERATOR%" -DARROW_BOOST_USE_SHARED=OFF -DARROW_IPC=OFF -DARROW_HDFS=OFF -DARROW_BUILD_BENCHMARKS=OFF -DARROW_JEMALLOC=OFF ..
- - cmake --build . --config Debug
+ - cmake -G "%GENERATOR%" -DARROW_BOOST_USE_SHARED=OFF -DARROW_BUILD_BENCHMARKS=OFF -DARROW_JEMALLOC=OFF -DCMAKE_BUILD_TYPE=Release ..
+ - cmake --build . --config Release
 
 # test_script:
-#  - ctest -VV
+ - ctest -VV
+
+
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index e11de1b4fb0da..aa8ea31b831e3 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -500,7 +500,11 @@ if(ARROW_BUILD_TESTS)
     set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
     set(GFLAGS_HOME "${GFLAGS_PREFIX}")
     set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
-    set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
+    if(MSVC)
+      set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/gflags_static.lib")
+    else()
+      set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
+    endif()
     set(GFLAGS_VENDORED 1)
     set(GFLAGS_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                           -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
@@ -536,6 +540,11 @@ if(ARROW_BUILD_TESTS)
   include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(gflags
     STATIC_LIB ${GFLAGS_STATIC_LIB})
+  if(MSVC)
+    set_target_properties(gflags
+      PROPERTIES
+      IMPORTED_LINK_INTERFACE_LIBRARIES "shlwapi.lib")
+  endif()
 
   if(GFLAGS_VENDORED)
     add_dependencies(gflags gflags_ep)
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index c9930418185c7..43d984045eb20 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -125,6 +125,8 @@ function(ADD_ARROW_LIB LIB_NAME)
     set_target_properties(${LIB_NAME}_shared
       PROPERTIES
       LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+      RUNTIME_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+      PDB_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
       LINK_FLAGS "${ARG_SHARED_LINK_FLAGS}"
       OUTPUT_NAME ${LIB_NAME}
       VERSION "${ARROW_ABI_VERSION}"
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 7c14238e8fda4..0aa2c92a07281 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -152,20 +152,30 @@ static inline int64_t lseek64_compat(int fd, int64_t pos, int whence) {
 #endif
 }
 
+#if defined(_MSC_VER)
+static inline Status ConvertToUtf16(const std::string& input, std::wstring* result) {
+  if (result == nullptr) { return Status::Invalid("Pointer to result is not valid"); }
+
+  if (input.empty()) {
+    *result = std::wstring();
+    return Status::OK();
+  }
+
+  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> utf16_converter;
+  *result = utf16_converter.from_bytes(input);
+  return Status::OK();
+}
+#endif
+
 static inline Status FileOpenReadable(const std::string& filename, int* fd) {
   int ret;
   errno_t errno_actual = 0;
 #if defined(_MSC_VER)
-  // https://msdn.microsoft.com/en-us/library/w64k0ytk.aspx
-
-  // See GH #209. Here we are assuming that the filename has been encoded in
-  // utf-16le so that unicode filenames can be supported
-  const int nwchars = static_cast<int>(filename.size()) / sizeof(wchar_t);
-  std::vector<wchar_t> wpath(nwchars + 1);
-  memcpy(wpath.data(), filename.data(), filename.size());
-  memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
+  std::wstring wide_filename;
+  RETURN_NOT_OK(ConvertToUtf16(filename, &wide_filename));
 
-  errno_actual = _wsopen_s(fd, wpath.data(), _O_RDONLY | _O_BINARY, _SH_DENYNO, _S_IREAD);
+  errno_actual =
+      _wsopen_s(fd, wide_filename.c_str(), _O_RDONLY | _O_BINARY, _SH_DENYNO, _S_IREAD);
   ret = *fd;
 #else
   ret = *fd = open(filename.c_str(), O_RDONLY | O_BINARY);
@@ -181,16 +191,12 @@ static inline Status FileOpenWriteable(
   errno_t errno_actual = 0;
 
 #if defined(_MSC_VER)
-  // https://msdn.microsoft.com/en-us/library/w64k0ytk.aspx
-  // Same story with wchar_t as above
-  const int nwchars = static_cast<int>(filename.size()) / sizeof(wchar_t);
-  std::vector<wchar_t> wpath(nwchars + 1);
-  memcpy(wpath.data(), filename.data(), filename.size());
-  memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
+  std::wstring wide_filename;
+  RETURN_NOT_OK(ConvertToUtf16(filename, &wide_filename));
 
   int oflag = _O_CREAT | _O_BINARY;
-  int sh_flag = _S_IWRITE;
-  if (!write_only) { sh_flag |= _S_IREAD; }
+  int pmode = _S_IWRITE;
+  if (!write_only) { pmode |= _S_IREAD; }
 
   if (truncate) { oflag |= _O_TRUNC; }
 
@@ -200,7 +206,7 @@ static inline Status FileOpenWriteable(
     oflag |= _O_RDWR;
   }
 
-  errno_actual = _wsopen_s(fd, wpath.data(), oflag, _SH_DENYNO, sh_flag);
+  errno_actual = _wsopen_s(fd, wide_filename.c_str(), oflag, _SH_DENYNO, pmode);
   ret = *fd;
 
 #else
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 5810c820f6dd7..348be17d89341 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -41,14 +41,29 @@ static bool FileExists(const std::string& path) {
   return std::ifstream(path.c_str()).good();
 }
 
+#if defined(_MSC_VER)
+void InvalidParamHandler(const wchar_t* expr, const wchar_t* func,
+    const wchar_t* source_file, unsigned int source_line, uintptr_t reserved) {
+  wprintf(L"Invalid parameter in funcion %s. Source: %s line %d expression %s", func,
+      source_file, source_line, expr);
+}
+#endif
+
 static bool FileIsClosed(int fd) {
-#ifdef _MSC_VER
-  // Close file a second time, this should set errno to EBADF
-  close(fd);
+#if defined(_MSC_VER)
+  // Disables default behavior on wrong params which causes the application to crash
+  // https://msdn.microsoft.com/en-us/library/ksazx244.aspx
+  _set_invalid_parameter_handler(InvalidParamHandler);
+
+  // Disables possible assertion alert box on invalid input arguments
+  _CrtSetReportMode(_CRT_ASSERT, 0);
+
+  int ret = static_cast<int>(_close(fd));
+  return (ret == -1);
 #else
   if (-1 != fcntl(fd, F_GETFD)) { return false; }
-#endif
   return errno == EBADF;
+#endif
 }
 
 class FileTestFixture : public ::testing::Test {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 648d4baac9b6f..f3140be0b2dac 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -78,8 +78,9 @@ class TestHdfsClient : public ::testing::Test {
     LibHdfsShim* driver_shim;
 
     client_ = nullptr;
-    scratch_dir_ =
-        boost::filesystem::unique_path("/tmp/arrow-hdfs/scratch-%%%%").string();
+    scratch_dir_ = boost::filesystem::unique_path(
+        boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
+                       .string();
 
     loaded_driver_ = false;
 
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index 4c114760e9a4b..db5bcc1b4f49b 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -69,13 +69,15 @@ class MemoryMapFixture {
 
   void CreateFile(const std::string path, int64_t size) {
     FILE* file = fopen(path.c_str(), "w");
-    if (file != nullptr) { tmp_files_.push_back(path); }
+    if (file != nullptr) {
+      tmp_files_.push_back(path);
 #ifdef _MSC_VER
-    _chsize(fileno(file), static_cast<size_t>(size));
+      _chsize(fileno(file), static_cast<size_t>(size));
 #else
-    ftruncate(fileno(file), static_cast<size_t>(size));
+      ftruncate(fileno(file), static_cast<size_t>(size));
 #endif
-    fclose(file);
+      fclose(file);
+    }
   }
 
   Status InitMemoryMap(
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 030cba93f5fc0..31a04dfc07818 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -85,6 +85,15 @@ if (ARROW_BUILD_TESTS)
       dl)
     set_target_properties(json-integration-test
       PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  elseif (MSVC)
+    target_link_libraries(json-integration-test
+      arrow_ipc_static
+      arrow_io_static
+      arrow_static
+      gflags
+      gtest
+      ${BOOST_FILESYSTEM_LIBRARY}
+      ${BOOST_SYSTEM_LIBRARY})
   else()
     target_link_libraries(json-integration-test
       arrow_ipc_static
@@ -156,14 +165,22 @@ install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-ipc.pc"
   DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
 
-
-set(UTIL_LINK_LIBS
-  arrow_ipc_static
-  arrow_io_static
-  arrow_static
-  ${BOOST_FILESYSTEM_LIBRARY}
-  ${BOOST_SYSTEM_LIBRARY}
-  dl)
+if(MSVC)
+  set(UTIL_LINK_LIBS
+    arrow_ipc_static
+    arrow_io_static
+    arrow_static
+    ${BOOST_FILESYSTEM_LIBRARY}
+    ${BOOST_SYSTEM_LIBRARY})
+else()
+  set(UTIL_LINK_LIBS
+    arrow_ipc_static
+    arrow_io_static
+    arrow_static
+    ${BOOST_FILESYSTEM_LIBRARY}
+    ${BOOST_SYSTEM_LIBRARY}
+    dl)
+endif()
 
 if (ARROW_BUILD_UTILITIES)
   add_executable(file-to-stream file-to-stream.cc)
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 36ba4b26042a8..6d9fabdc920f9 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -767,6 +767,8 @@ Message::Message(const std::shared_ptr<Buffer>& buffer, int64_t offset) {
   impl_.reset(new MessageImpl(buffer, offset));
 }
 
+Message::~Message() {}
+
 Status Message::Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
     std::shared_ptr<Message>* out) {
   // ctor is private
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index f60fb770c3696..798abdcdf9db7 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -140,6 +140,7 @@ struct ARROW_EXPORT BufferMetadata {
 
 class ARROW_EXPORT Message {
  public:
+  ~Message();
   enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH };
 
   static Status Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index db5f0829f92f7..0a19f69d27d8c 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -662,6 +662,8 @@ Status StreamWriter::WriteRecordBatch(const RecordBatch& batch, bool allow_64bit
   return impl_->WriteRecordBatch(batch, allow_64bit);
 }
 
+StreamWriter::~StreamWriter() {}
+
 void StreamWriter::set_memory_pool(MemoryPool* pool) {
   impl_->set_memory_pool(pool);
 }
@@ -718,6 +720,8 @@ FileWriter::FileWriter() {
   impl_.reset(new FileWriterImpl());
 }
 
+FileWriter::~FileWriter() {}
+
 Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
     std::shared_ptr<FileWriter>* out) {
   *out = std::shared_ptr<FileWriter>(new FileWriter());  // ctor is private
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 25b5ad62726d9..c572157b465a6 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -82,7 +82,7 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 
 class ARROW_EXPORT StreamWriter {
  public:
-  virtual ~StreamWriter() = default;
+  virtual ~StreamWriter();
 
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
       std::shared_ptr<StreamWriter>* out);
@@ -105,6 +105,8 @@ class ARROW_EXPORT StreamWriter {
 
 class ARROW_EXPORT FileWriter : public StreamWriter {
  public:
+  virtual ~FileWriter();
+
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
       std::shared_ptr<FileWriter>* out);
 

From 957a0e67836b66f8ff4fc3fdae343553c589b53f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 30 Mar 2017 18:03:26 -0400
Subject: [PATCH 0434/1644] ARROW-717: [C++] Implement IPC zero-copy round trip
 for tensors

This patch provides:

```python
WriteTensor(tensor, file, &metadata_length, &body_length));
std::shared_ptr<Tensor> result;
ReadTensor(offset, file, &result));
```

Also implemented `Tensor::Equals` and did some refactoring / code simplification in compare.cc

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #454 from wesm/ARROW-717 and squashes the following commits:

6c15481 [Wes McKinney] Tensor IPC read/write, and refactoring / code scrubbing
---
 cpp/src/arrow/buffer.cc                  |   6 +-
 cpp/src/arrow/compare.cc                 | 330 ++++++++++-------------
 cpp/src/arrow/compare.h                  |   4 +
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  54 +++-
 cpp/src/arrow/ipc/metadata.cc            | 266 ++++++++++++------
 cpp/src/arrow/ipc/metadata.h             |  67 +++--
 cpp/src/arrow/ipc/reader.cc              |  79 +++---
 cpp/src/arrow/ipc/reader.h               |  32 +--
 cpp/src/arrow/ipc/writer.cc              |  79 +++---
 cpp/src/arrow/ipc/writer.h               |  12 +-
 cpp/src/arrow/tensor-test.cc             |  25 +-
 cpp/src/arrow/tensor.cc                  |  67 ++++-
 cpp/src/arrow/tensor.h                   |  18 +-
 cpp/src/arrow/type_traits.h              |  11 +
 cpp/src/arrow/visitor_inline.h           |  26 ++
 format/Tensor.fbs                        |  14 +-
 16 files changed, 656 insertions(+), 434 deletions(-)

diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index be747e1d49504..59623403e5c5e 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -27,11 +27,9 @@
 
 namespace arrow {
 
-Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size) {
-  data_ = parent->data() + offset;
-  size_ = size;
+Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size)
+    : Buffer(parent->data() + offset, size) {
   parent_ = parent;
-  capacity_ = size;
 }
 
 Buffer::~Buffer() {}
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index f786222f7e4f2..c2580b4f54109 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -25,6 +25,7 @@
 
 #include "arrow/array.h"
 #include "arrow/status.h"
+#include "arrow/tensor.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -36,7 +37,7 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Public method implementations
 
-class RangeEqualsVisitor : public ArrayVisitor {
+class RangeEqualsVisitor {
  public:
   RangeEqualsVisitor(const Array& right, int64_t left_start_idx, int64_t left_end_idx,
       int64_t right_start_idx)
@@ -46,12 +47,6 @@ class RangeEqualsVisitor : public ArrayVisitor {
         right_start_idx_(right_start_idx),
         result_(false) {}
 
-  Status Visit(const NullArray& left) override {
-    UNUSED(left);
-    result_ = true;
-    return Status::OK();
-  }
-
   template <typename ArrayType>
   inline Status CompareValues(const ArrayType& left) {
     const auto& right = static_cast<const ArrayType&>(right_);
@@ -96,108 +91,6 @@ class RangeEqualsVisitor : public ArrayVisitor {
     return true;
   }
 
-  Status Visit(const BooleanArray& left) override {
-    return CompareValues<BooleanArray>(left);
-  }
-
-  Status Visit(const Int8Array& left) override { return CompareValues<Int8Array>(left); }
-
-  Status Visit(const Int16Array& left) override {
-    return CompareValues<Int16Array>(left);
-  }
-  Status Visit(const Int32Array& left) override {
-    return CompareValues<Int32Array>(left);
-  }
-  Status Visit(const Int64Array& left) override {
-    return CompareValues<Int64Array>(left);
-  }
-  Status Visit(const UInt8Array& left) override {
-    return CompareValues<UInt8Array>(left);
-  }
-  Status Visit(const UInt16Array& left) override {
-    return CompareValues<UInt16Array>(left);
-  }
-  Status Visit(const UInt32Array& left) override {
-    return CompareValues<UInt32Array>(left);
-  }
-  Status Visit(const UInt64Array& left) override {
-    return CompareValues<UInt64Array>(left);
-  }
-  Status Visit(const FloatArray& left) override {
-    return CompareValues<FloatArray>(left);
-  }
-  Status Visit(const DoubleArray& left) override {
-    return CompareValues<DoubleArray>(left);
-  }
-
-  Status Visit(const HalfFloatArray& left) override {
-    return Status::NotImplemented("Half float type");
-  }
-
-  Status Visit(const StringArray& left) override {
-    result_ = CompareBinaryRange(left);
-    return Status::OK();
-  }
-
-  Status Visit(const BinaryArray& left) override {
-    result_ = CompareBinaryRange(left);
-    return Status::OK();
-  }
-
-  Status Visit(const FixedWidthBinaryArray& left) override {
-    const auto& right = static_cast<const FixedWidthBinaryArray&>(right_);
-
-    int32_t width = left.byte_width();
-
-    const uint8_t* left_data = left.raw_data() + left.offset() * width;
-    const uint8_t* right_data = right.raw_data() + right.offset() * width;
-
-    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
-         ++i, ++o_i) {
-      const bool is_null = left.IsNull(i);
-      if (is_null != right.IsNull(o_i)) {
-        result_ = false;
-        return Status::OK();
-      }
-      if (is_null) continue;
-
-      if (std::memcmp(left_data + width * i, right_data + width * o_i, width)) {
-        result_ = false;
-        return Status::OK();
-      }
-    }
-    result_ = true;
-    return Status::OK();
-  }
-
-  Status Visit(const Date32Array& left) override {
-    return CompareValues<Date32Array>(left);
-  }
-
-  Status Visit(const Date64Array& left) override {
-    return CompareValues<Date64Array>(left);
-  }
-
-  Status Visit(const Time32Array& left) override {
-    return CompareValues<Time32Array>(left);
-  }
-
-  Status Visit(const Time64Array& left) override {
-    return CompareValues<Time64Array>(left);
-  }
-
-  Status Visit(const TimestampArray& left) override {
-    return CompareValues<TimestampArray>(left);
-  }
-
-  Status Visit(const IntervalArray& left) override {
-    return CompareValues<IntervalArray>(left);
-  }
-
-  Status Visit(const DecimalArray& left) override {
-    return Status::NotImplemented("Decimal type");
-  }
-
   bool CompareLists(const ListArray& left) {
     const auto& right = static_cast<const ListArray&>(right_);
 
@@ -225,11 +118,6 @@ class RangeEqualsVisitor : public ArrayVisitor {
     return true;
   }
 
-  Status Visit(const ListArray& left) override {
-    result_ = CompareLists(left);
-    return Status::OK();
-  }
-
   bool CompareStructs(const StructArray& left) {
     const auto& right = static_cast<const StructArray&>(right_);
     bool equal_fields = true;
@@ -251,11 +139,6 @@ class RangeEqualsVisitor : public ArrayVisitor {
     return true;
   }
 
-  Status Visit(const StructArray& left) override {
-    result_ = CompareStructs(left);
-    return Status::OK();
-  }
-
   bool CompareUnions(const UnionArray& left) const {
     const auto& right = static_cast<const UnionArray&>(right_);
 
@@ -314,12 +197,73 @@ class RangeEqualsVisitor : public ArrayVisitor {
     return true;
   }
 
-  Status Visit(const UnionArray& left) override {
+  Status Visit(const BinaryArray& left) {
+    result_ = CompareBinaryRange(left);
+    return Status::OK();
+  }
+
+  Status Visit(const FixedWidthBinaryArray& left) {
+    const auto& right = static_cast<const FixedWidthBinaryArray&>(right_);
+
+    int32_t width = left.byte_width();
+
+    const uint8_t* left_data = nullptr;
+    const uint8_t* right_data = nullptr;
+
+    if (left.data()) { left_data = left.raw_data() + left.offset() * width; }
+
+    if (right.data()) { right_data = right.raw_data() + right.offset() * width; }
+
+    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+         ++i, ++o_i) {
+      const bool is_null = left.IsNull(i);
+      if (is_null != right.IsNull(o_i)) {
+        result_ = false;
+        return Status::OK();
+      }
+      if (is_null) continue;
+
+      if (std::memcmp(left_data + width * i, right_data + width * o_i, width)) {
+        result_ = false;
+        return Status::OK();
+      }
+    }
+    result_ = true;
+    return Status::OK();
+  }
+
+  Status Visit(const NullArray& left) {
+    UNUSED(left);
+    result_ = true;
+    return Status::OK();
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value, Status>::type Visit(
+      const T& left) {
+    return CompareValues<T>(left);
+  }
+
+  Status Visit(const DecimalArray& left) {
+    return Status::NotImplemented("Decimal type");
+  }
+
+  Status Visit(const ListArray& left) {
+    result_ = CompareLists(left);
+    return Status::OK();
+  }
+
+  Status Visit(const StructArray& left) {
+    result_ = CompareStructs(left);
+    return Status::OK();
+  }
+
+  Status Visit(const UnionArray& left) {
     result_ = CompareUnions(left);
     return Status::OK();
   }
 
-  Status Visit(const DictionaryArray& left) override {
+  Status Visit(const DictionaryArray& left) {
     const auto& right = static_cast<const DictionaryArray&>(right_);
     if (!left.dictionary()->Equals(right.dictionary())) {
       result_ = false;
@@ -346,9 +290,9 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
   explicit ArrayEqualsVisitor(const Array& right)
       : RangeEqualsVisitor(right, 0, right.length(), 0) {}
 
-  Status Visit(const NullArray& left) override { return Status::OK(); }
+  Status Visit(const NullArray& left) { return Status::OK(); }
 
-  Status Visit(const BooleanArray& left) override {
+  Status Visit(const BooleanArray& left) {
     const auto& right = static_cast<const BooleanArray&>(right_);
     if (left.null_count() > 0) {
       const uint8_t* left_data = left.data()->data();
@@ -372,64 +316,39 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
   bool IsEqualPrimitive(const PrimitiveArray& left) {
     const auto& right = static_cast<const PrimitiveArray&>(right_);
     const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
-    const int value_byte_size = size_meta.bit_width() / 8;
-    DCHECK_GT(value_byte_size, 0);
+    const int byte_width = size_meta.bit_width() / 8;
+
+    const uint8_t* left_data = nullptr;
+    const uint8_t* right_data = nullptr;
+
+    if (left.data()) { left_data = left.data()->data() + left.offset() * byte_width; }
 
-    const uint8_t* left_data = left.data()->data() + left.offset() * value_byte_size;
-    const uint8_t* right_data = right.data()->data() + right.offset() * value_byte_size;
+    if (right.data()) { right_data = right.data()->data() + right.offset() * byte_width; }
 
     if (left.null_count() > 0) {
       for (int64_t i = 0; i < left.length(); ++i) {
-        if (!left.IsNull(i) && memcmp(left_data, right_data, value_byte_size)) {
+        if (!left.IsNull(i) && memcmp(left_data, right_data, byte_width)) {
           return false;
         }
-        left_data += value_byte_size;
-        right_data += value_byte_size;
+        left_data += byte_width;
+        right_data += byte_width;
       }
       return true;
     } else {
       return memcmp(left_data, right_data,
-                 static_cast<size_t>(value_byte_size * left.length())) == 0;
+                 static_cast<size_t>(byte_width * left.length())) == 0;
     }
   }
 
-  Status ComparePrimitive(const PrimitiveArray& left) {
+  template <typename T>
+  typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value &&
+                              !std::is_base_of<BooleanArray, T>::value,
+      Status>::type
+  Visit(const T& left) {
     result_ = IsEqualPrimitive(left);
     return Status::OK();
   }
 
-  Status Visit(const Int8Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const Int16Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const Int32Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const Int64Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const UInt8Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const UInt16Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const UInt32Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const UInt64Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const FloatArray& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const DoubleArray& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const Date32Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const Date64Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const Time32Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const Time64Array& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const TimestampArray& left) override { return ComparePrimitive(left); }
-
-  Status Visit(const IntervalArray& left) override { return ComparePrimitive(left); }
-
   template <typename ArrayType>
   bool ValueOffsetsEqual(const ArrayType& left) {
     const auto& right = static_cast<const ArrayType&>(right_);
@@ -494,17 +413,12 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     }
   }
 
-  Status Visit(const StringArray& left) override {
-    result_ = CompareBinary(left);
-    return Status::OK();
-  }
-
-  Status Visit(const BinaryArray& left) override {
+  Status Visit(const BinaryArray& left) {
     result_ = CompareBinary(left);
     return Status::OK();
   }
 
-  Status Visit(const ListArray& left) override {
+  Status Visit(const ListArray& left) {
     const auto& right = static_cast<const ListArray&>(right_);
     bool equal_offsets = ValueOffsetsEqual<ListArray>(left);
     if (!equal_offsets) {
@@ -523,7 +437,7 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DictionaryArray& left) override {
+  Status Visit(const DictionaryArray& left) {
     const auto& right = static_cast<const DictionaryArray&>(right_);
     if (!left.dictionary()->Equals(right.dictionary())) {
       result_ = false;
@@ -532,6 +446,13 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     }
     return Status::OK();
   }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<NestedType, typename T::TypeClass>::value,
+      Status>::type
+  Visit(const T& left) {
+    return RangeEqualsVisitor::Visit(left);
+  }
 };
 
 template <typename TYPE>
@@ -560,14 +481,15 @@ inline bool FloatingApproxEquals(
 class ApproxEqualsVisitor : public ArrayEqualsVisitor {
  public:
   using ArrayEqualsVisitor::ArrayEqualsVisitor;
+  using ArrayEqualsVisitor::Visit;
 
-  Status Visit(const FloatArray& left) override {
+  Status Visit(const FloatArray& left) {
     result_ =
         FloatingApproxEquals<FloatType>(left, static_cast<const FloatArray&>(right_));
     return Status::OK();
   }
 
-  Status Visit(const DoubleArray& left) override {
+  Status Visit(const DoubleArray& left) {
     result_ =
         FloatingApproxEquals<DoubleType>(left, static_cast<const DoubleArray&>(right_));
     return Status::OK();
@@ -586,7 +508,8 @@ static bool BaseDataEquals(const Array& left, const Array& right) {
   return true;
 }
 
-Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
+template <typename VISITOR>
+inline Status ArrayEqualsImpl(const Array& left, const Array& right, bool* are_equal) {
   // The arrays are the same object
   if (&left == &right) {
     *are_equal = true;
@@ -595,13 +518,21 @@ Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
   } else if (left.length() == 0) {
     *are_equal = true;
   } else {
-    ArrayEqualsVisitor visitor(right);
-    RETURN_NOT_OK(left.Accept(&visitor));
+    VISITOR visitor(right);
+    RETURN_NOT_OK(VisitArrayInline(left, &visitor));
     *are_equal = visitor.result();
   }
   return Status::OK();
 }
 
+Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
+  return ArrayEqualsImpl<ArrayEqualsVisitor>(left, right, are_equal);
+}
+
+Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal) {
+  return ArrayEqualsImpl<ApproxEqualsVisitor>(left, right, are_equal);
+}
+
 Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_start_idx,
     int64_t left_end_idx, int64_t right_start_idx, bool* are_equal) {
   if (&left == &right) {
@@ -612,23 +543,56 @@ Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_star
     *are_equal = true;
   } else {
     RangeEqualsVisitor visitor(right, left_start_idx, left_end_idx, right_start_idx);
-    RETURN_NOT_OK(left.Accept(&visitor));
+    RETURN_NOT_OK(VisitArrayInline(left, &visitor));
     *are_equal = visitor.result();
   }
   return Status::OK();
 }
 
-Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal) {
+// ----------------------------------------------------------------------
+// Implement TensorEquals
+
+class TensorEqualsVisitor {
+ public:
+  explicit TensorEqualsVisitor(const Tensor& right) : right_(right) {}
+
+  template <typename TensorType>
+  Status Visit(const TensorType& left) {
+    const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
+    const int byte_width = size_meta.bit_width() / 8;
+    DCHECK_GT(byte_width, 0);
+
+    const uint8_t* left_data = left.data()->data();
+    const uint8_t* right_data = right_.data()->data();
+
+    result_ =
+        memcmp(left_data, right_data, static_cast<size_t>(byte_width * left.size())) == 0;
+    return Status::OK();
+  }
+
+  bool result() const { return result_; }
+
+ protected:
+  const Tensor& right_;
+  bool result_;
+};
+
+Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
   // The arrays are the same object
   if (&left == &right) {
     *are_equal = true;
-  } else if (!BaseDataEquals(left, right)) {
+  } else if (left.type_enum() != right.type_enum()) {
     *are_equal = false;
-  } else if (left.length() == 0) {
+  } else if (left.size() == 0) {
     *are_equal = true;
   } else {
-    ApproxEqualsVisitor visitor(right);
-    RETURN_NOT_OK(left.Accept(&visitor));
+    if (!left.is_contiguous() || !right.is_contiguous()) {
+      return Status::NotImplemented(
+          "Comparison not implemented for non-contiguous tensors");
+    }
+
+    TensorEqualsVisitor visitor(right);
+    RETURN_NOT_OK(VisitTensorInline(left, &visitor));
     *are_equal = visitor.result();
   }
   return Status::OK();
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index 1ddf0497dd3d9..522b11dadec47 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -29,10 +29,14 @@ namespace arrow {
 class Array;
 struct DataType;
 class Status;
+class Tensor;
 
 /// Returns true if the arrays are exactly equal
 Status ARROW_EXPORT ArrayEquals(const Array& left, const Array& right, bool* are_equal);
 
+Status ARROW_EXPORT TensorEquals(
+    const Tensor& left, const Tensor& right, bool* are_equal);
+
 /// Returns true if the arrays are approximately equal. For non-floating point
 /// types, this is equivalent to ArrayEquals(left, right)
 Status ARROW_EXPORT ArrayApproxEquals(
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 6ddda3f339641..74ca017df5cf1 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -25,16 +25,16 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
+#include "arrow/buffer.h"
 #include "arrow/io/memory.h"
 #include "arrow/io/test-common.h"
 #include "arrow/ipc/api.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/ipc/util.h"
-
-#include "arrow/buffer.h"
 #include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
 #include "arrow/status.h"
+#include "arrow/tensor.h"
 #include "arrow/test-util.h"
 #include "arrow/util/bit-util.h"
 
@@ -56,13 +56,10 @@ class TestSchemaMetadata : public ::testing::Test {
 
     ASSERT_EQ(Message::SCHEMA, message->type());
 
-    auto schema_msg = std::make_shared<SchemaMetadata>(message);
-    ASSERT_EQ(schema.num_fields(), schema_msg->num_fields());
-
     DictionaryMemo empty_memo;
 
     std::shared_ptr<Schema> schema2;
-    ASSERT_OK(schema_msg->GetSchema(empty_memo, &schema2));
+    ASSERT_OK(GetSchema(message->header(), empty_memo, &schema2));
 
     AssertSchemaEqual(schema, *schema2);
   }
@@ -90,7 +87,7 @@ TEST_F(TestSchemaMetadata, PrimitiveFields) {
 }
 
 TEST_F(TestSchemaMetadata, NestedFields) {
-  auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
+  auto type = list(int32());
   auto f0 = field("f0", type);
 
   std::shared_ptr<StructType> type2(
@@ -532,7 +529,6 @@ TEST_F(TestIpcRoundTrip, LargeRecordBatch) {
 
   // 512 MB
   constexpr int64_t kBufferSize = 1 << 29;
-
   ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_));
 
   std::shared_ptr<RecordBatch> result;
@@ -580,5 +576,47 @@ TEST_F(TestFileFormat, DictionaryRoundTrip) {
   CheckBatchDictionaries(*out_batches[0]);
 }
 
+class TestTensorRoundTrip : public ::testing::Test, public IpcTestFixture {
+ public:
+  void SetUp() { pool_ = default_memory_pool(); }
+  void TearDown() { io::MemoryMapFixture::TearDown(); }
+
+  void CheckTensorRoundTrip(const Tensor& tensor) {
+    int32_t metadata_length;
+    int64_t body_length;
+
+    ASSERT_OK(mmap_->Seek(0));
+
+    ASSERT_OK(WriteTensor(tensor, mmap_.get(), &metadata_length, &body_length));
+
+    std::shared_ptr<Tensor> result;
+    ASSERT_OK(ReadTensor(0, mmap_.get(), &result));
+
+    ASSERT_TRUE(tensor.Equals(*result));
+  }
+};
+
+TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
+  std::string path = "test-write-tensor";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_));
+
+  std::vector<int64_t> shape = {4, 6};
+  std::vector<int64_t> strides = {48, 8};
+  std::vector<std::string> dim_names = {"foo", "bar"};
+  int64_t size = 24;
+
+  std::vector<int64_t> values;
+  test::randint<int64_t>(size, 0, 100, &values);
+
+  auto data = test::GetBufferFromVector(values);
+
+  Int64Tensor t0(data, shape, strides, dim_names);
+  Int64Tensor tzero(data, {}, {}, {});
+
+  CheckTensorRoundTrip(t0);
+  CheckTensorRoundTrip(tzero);
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 6d9fabdc920f9..076a6e792ba40 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -20,6 +20,7 @@
 #include <cstdint>
 #include <memory>
 #include <sstream>
+#include <string>
 #include <vector>
 
 #include "flatbuffers/flatbuffers.h"
@@ -29,7 +30,10 @@
 #include "arrow/io/interfaces.h"
 #include "arrow/ipc/File_generated.h"
 #include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/Tensor_generated.h"
+#include "arrow/ipc/util.h"
 #include "arrow/status.h"
+#include "arrow/tensor.h"
 #include "arrow/type.h"
 
 namespace arrow {
@@ -418,6 +422,46 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
   return Status::OK();
 }
 
+static Status TensorTypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+    flatbuf::Type* out_type, Offset* offset) {
+  switch (type->type) {
+    case Type::UINT8:
+      INT_TO_FB_CASE(8, false);
+    case Type::INT8:
+      INT_TO_FB_CASE(8, true);
+    case Type::UINT16:
+      INT_TO_FB_CASE(16, false);
+    case Type::INT16:
+      INT_TO_FB_CASE(16, true);
+    case Type::UINT32:
+      INT_TO_FB_CASE(32, false);
+    case Type::INT32:
+      INT_TO_FB_CASE(32, true);
+    case Type::UINT64:
+      INT_TO_FB_CASE(64, false);
+    case Type::INT64:
+      INT_TO_FB_CASE(64, true);
+    case Type::HALF_FLOAT:
+      *out_type = flatbuf::Type_FloatingPoint;
+      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_HALF);
+      break;
+    case Type::FLOAT:
+      *out_type = flatbuf::Type_FloatingPoint;
+      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_SINGLE);
+      break;
+    case Type::DOUBLE:
+      *out_type = flatbuf::Type_FloatingPoint;
+      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
+      break;
+    default:
+      *out_type = flatbuf::Type_NONE;  // Make clang-tidy happy
+      std::stringstream ss;
+      ss << "Unable to convert type: " << type->ToString() << std::endl;
+      return Status::NotImplemented(ss.str());
+  }
+  return Status::OK();
+}
+
 static DictionaryOffset GetDictionaryEncoding(
     FBB& fbb, const DictionaryType& type, DictionaryMemo* memo) {
   int64_t dictionary_id = memo->GetId(type.dictionary());
@@ -552,7 +596,7 @@ static Status WriteFlatbufferBuilder(FBB& fbb, std::shared_ptr<Buffer>* out) {
   return Status::OK();
 }
 
-static Status WriteMessage(FBB& fbb, flatbuf::MessageHeader header_type,
+static Status WriteFBMessage(FBB& fbb, flatbuf::MessageHeader header_type,
     flatbuffers::Offset<void> header, int64_t body_length, std::shared_ptr<Buffer>* out) {
   auto message =
       flatbuf::CreateMessage(fbb, kMetadataVersion, header_type, header, body_length);
@@ -565,7 +609,7 @@ Status WriteSchemaMessage(
   FBB fbb;
   flatbuffers::Offset<flatbuf::Schema> fb_schema;
   RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
-  return WriteMessage(fbb, flatbuf::MessageHeader_Schema, fb_schema.Union(), 0, out);
+  return WriteFBMessage(fbb, flatbuf::MessageHeader_Schema, fb_schema.Union(), 0, out);
 }
 
 using FieldNodeVector =
@@ -620,10 +664,39 @@ Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
   FBB fbb;
   RecordBatchOffset record_batch;
   RETURN_NOT_OK(MakeRecordBatch(fbb, length, body_length, nodes, buffers, &record_batch));
-  return WriteMessage(
+  return WriteFBMessage(
       fbb, flatbuf::MessageHeader_RecordBatch, record_batch.Union(), body_length, out);
 }
 
+Status WriteTensorMessage(
+    const Tensor& tensor, int64_t buffer_start_offset, std::shared_ptr<Buffer>* out) {
+  using TensorDimOffset = flatbuffers::Offset<flatbuf::TensorDim>;
+  using TensorOffset = flatbuffers::Offset<flatbuf::Tensor>;
+
+  FBB fbb;
+
+  flatbuf::Type fb_type_type;
+  Offset fb_type;
+  RETURN_NOT_OK(TensorTypeToFlatbuffer(fbb, tensor.type(), &fb_type_type, &fb_type));
+
+  std::vector<TensorDimOffset> dims;
+  for (int i = 0; i < tensor.ndim(); ++i) {
+    FBString name = fbb.CreateString(tensor.dim_name(i));
+    dims.push_back(flatbuf::CreateTensorDim(fbb, tensor.shape()[i], name));
+  }
+
+  auto fb_shape = fbb.CreateVector(dims);
+  auto fb_strides = fbb.CreateVector(tensor.strides());
+  int64_t body_length = tensor.data()->size();
+  flatbuf::Buffer buffer(-1, buffer_start_offset, body_length);
+
+  TensorOffset fb_tensor =
+      flatbuf::CreateTensor(fbb, fb_type_type, fb_type, fb_shape, fb_strides, &buffer);
+
+  return WriteFBMessage(
+      fbb, flatbuf::MessageHeader_Tensor, fb_tensor.Union(), body_length, out);
+}
+
 Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out) {
@@ -631,7 +704,7 @@ Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
   RecordBatchOffset record_batch;
   RETURN_NOT_OK(MakeRecordBatch(fbb, length, body_length, nodes, buffers, &record_batch));
   auto dictionary_batch = flatbuf::CreateDictionaryBatch(fbb, id, record_batch).Union();
-  return WriteMessage(
+  return WriteFBMessage(
       fbb, flatbuf::MessageHeader_DictionaryBatch, dictionary_batch, body_length, out);
 }
 
@@ -746,6 +819,8 @@ class Message::MessageImpl {
         return Message::DICTIONARY_BATCH;
       case flatbuf::MessageHeader_RecordBatch:
         return Message::RECORD_BATCH;
+      case flatbuf::MessageHeader_Tensor:
+        return Message::TENSOR;
       default:
         return Message::NONE;
     }
@@ -790,95 +865,78 @@ const void* Message::header() const {
 }
 
 // ----------------------------------------------------------------------
-// SchemaMetadata
-
-class MessageHolder {
- public:
-  void set_message(const std::shared_ptr<Message>& message) { message_ = message; }
-  void set_buffer(const std::shared_ptr<Buffer>& buffer) { buffer_ = buffer; }
-
- protected:
-  // Possible parents, owns the flatbuffer data
-  std::shared_ptr<Message> message_;
-  std::shared_ptr<Buffer> buffer_;
-};
-
-class SchemaMetadata::SchemaMetadataImpl : public MessageHolder {
- public:
-  explicit SchemaMetadataImpl(const void* schema)
-      : schema_(static_cast<const flatbuf::Schema*>(schema)) {}
-
-  const flatbuf::Field* get_field(int i) const { return schema_->fields()->Get(i); }
 
-  int num_fields() const { return schema_->fields()->size(); }
-
-  Status VisitField(const flatbuf::Field* field, DictionaryTypeMap* id_to_field) const {
-    const flatbuf::DictionaryEncoding* dict_metadata = field->dictionary();
-    if (dict_metadata == nullptr) {
-      // Field is not dictionary encoded. Visit children
-      auto children = field->children();
-      for (flatbuffers::uoffset_t i = 0; i < children->size(); ++i) {
-        RETURN_NOT_OK(VisitField(children->Get(i), id_to_field));
-      }
-    } else {
-      // Field is dictionary encoded. Construct the data type for the
-      // dictionary (no descendents can be dictionary encoded)
-      std::shared_ptr<Field> dictionary_field;
-      RETURN_NOT_OK(FieldFromFlatbufferDictionary(field, &dictionary_field));
-      (*id_to_field)[dict_metadata->id()] = dictionary_field;
+static Status VisitField(const flatbuf::Field* field, DictionaryTypeMap* id_to_field) {
+  const flatbuf::DictionaryEncoding* dict_metadata = field->dictionary();
+  if (dict_metadata == nullptr) {
+    // Field is not dictionary encoded. Visit children
+    auto children = field->children();
+    for (flatbuffers::uoffset_t i = 0; i < children->size(); ++i) {
+      RETURN_NOT_OK(VisitField(children->Get(i), id_to_field));
     }
-    return Status::OK();
+  } else {
+    // Field is dictionary encoded. Construct the data type for the
+    // dictionary (no descendents can be dictionary encoded)
+    std::shared_ptr<Field> dictionary_field;
+    RETURN_NOT_OK(FieldFromFlatbufferDictionary(field, &dictionary_field));
+    (*id_to_field)[dict_metadata->id()] = dictionary_field;
   }
+  return Status::OK();
+}
 
-  Status GetDictionaryTypes(DictionaryTypeMap* id_to_field) const {
-    for (int i = 0; i < num_fields(); ++i) {
-      RETURN_NOT_OK(VisitField(get_field(i), id_to_field));
-    }
-    return Status::OK();
+Status GetDictionaryTypes(const void* opaque_schema, DictionaryTypeMap* id_to_field) {
+  auto schema = static_cast<const flatbuf::Schema*>(opaque_schema);
+  int num_fields = static_cast<int>(schema->fields()->size());
+  for (int i = 0; i < num_fields; ++i) {
+    RETURN_NOT_OK(VisitField(schema->fields()->Get(i), id_to_field));
   }
-
- private:
-  const flatbuf::Schema* schema_;
-};
-
-SchemaMetadata::SchemaMetadata(const std::shared_ptr<Message>& message)
-    : SchemaMetadata(message->impl_->header()) {
-  impl_->set_message(message);
+  return Status::OK();
 }
 
-SchemaMetadata::SchemaMetadata(const void* header) {
-  impl_.reset(new SchemaMetadataImpl(header));
-}
+Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_memo,
+    std::shared_ptr<Schema>* out) {
+  auto schema = static_cast<const flatbuf::Schema*>(opaque_schema);
+  int num_fields = static_cast<int>(schema->fields()->size());
 
-SchemaMetadata::SchemaMetadata(const std::shared_ptr<Buffer>& buffer, int64_t offset)
-    : SchemaMetadata(buffer->data() + offset) {
-  // Preserve ownership
-  impl_->set_buffer(buffer);
+  std::vector<std::shared_ptr<Field>> fields(num_fields);
+  for (int i = 0; i < num_fields; ++i) {
+    const flatbuf::Field* field = schema->fields()->Get(i);
+    RETURN_NOT_OK(FieldFromFlatbuffer(field, dictionary_memo, &fields[i]));
+  }
+  *out = std::make_shared<Schema>(fields);
+  return Status::OK();
 }
 
-SchemaMetadata::~SchemaMetadata() {}
+Status GetTensorMetadata(const void* opaque_tensor, std::shared_ptr<DataType>* type,
+    std::vector<int64_t>* shape, std::vector<int64_t>* strides,
+    std::vector<std::string>* dim_names) {
+  auto tensor = static_cast<const flatbuf::Tensor*>(opaque_tensor);
 
-int SchemaMetadata::num_fields() const {
-  return impl_->num_fields();
-}
+  int ndim = static_cast<int>(tensor->shape()->size());
 
-Status SchemaMetadata::GetDictionaryTypes(DictionaryTypeMap* id_to_field) const {
-  return impl_->GetDictionaryTypes(id_to_field);
-}
+  for (int i = 0; i < ndim; ++i) {
+    auto dim = tensor->shape()->Get(i);
 
-Status SchemaMetadata::GetSchema(
-    const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out) const {
-  std::vector<std::shared_ptr<Field>> fields(num_fields());
-  for (int i = 0; i < this->num_fields(); ++i) {
-    const flatbuf::Field* field = impl_->get_field(i);
-    RETURN_NOT_OK(FieldFromFlatbuffer(field, dictionary_memo, &fields[i]));
+    shape->push_back(dim->size());
+    auto fb_name = dim->name();
+    if (fb_name == 0) {
+      dim_names->push_back("");
+    } else {
+      dim_names->push_back(fb_name->str());
+    }
   }
-  *out = std::make_shared<Schema>(fields);
-  return Status::OK();
+
+  if (tensor->strides()->size() > 0) {
+    for (int i = 0; i < ndim; ++i) {
+      strides->push_back(tensor->strides()->Get(i));
+    }
+  }
+
+  return TypeFromFlatbuffer(tensor->type_type(), tensor->type(), {}, type);
 }
 
 // ----------------------------------------------------------------------
-// Conveniences
+// Read and write messages
 
 Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
     std::shared_ptr<Message>* message) {
@@ -896,5 +954,61 @@ Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile
   return Message::Open(buffer, 4, message);
 }
 
+Status ReadMessage(io::InputStream* file, std::shared_ptr<Message>* message) {
+  std::shared_ptr<Buffer> buffer;
+  RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
+
+  if (buffer->size() != sizeof(int32_t)) {
+    *message = nullptr;
+    return Status::OK();
+  }
+
+  int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
+
+  if (message_length == 0) {
+    // Optional 0 EOS control message
+    *message = nullptr;
+    return Status::OK();
+  }
+
+  RETURN_NOT_OK(file->Read(message_length, &buffer));
+  if (buffer->size() != message_length) {
+    return Status::IOError("Unexpected end of stream trying to read message");
+  }
+
+  return Message::Open(buffer, 0, message);
+}
+
+Status WriteMessage(
+    const Buffer& message, io::OutputStream* file, int32_t* message_length) {
+  // Need to write 4 bytes (message size), the message, plus padding to
+  // end on an 8-byte offset
+  int64_t start_offset;
+  RETURN_NOT_OK(file->Tell(&start_offset));
+
+  int32_t padded_message_length = static_cast<int32_t>(message.size()) + 4;
+  const int32_t remainder =
+      (padded_message_length + static_cast<int32_t>(start_offset)) % 8;
+  if (remainder != 0) { padded_message_length += 8 - remainder; }
+
+  // The returned message size includes the length prefix, the flatbuffer,
+  // plus padding
+  *message_length = padded_message_length;
+
+  // Write the flatbuffer size prefix including padding
+  int32_t flatbuffer_size = padded_message_length - 4;
+  RETURN_NOT_OK(
+      file->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
+
+  // Write the flatbuffer
+  RETURN_NOT_OK(file->Write(message.data(), message.size()));
+
+  // Write any padding
+  int32_t padding = padded_message_length - static_cast<int32_t>(message.size()) - 4;
+  if (padding > 0) { RETURN_NOT_OK(file->Write(kPaddingBytes, padding)); }
+
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 798abdcdf9db7..fac4a70aada8d 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -22,6 +22,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <string>
 #include <unordered_map>
 #include <vector>
 
@@ -37,9 +38,11 @@ struct DataType;
 struct Field;
 class Schema;
 class Status;
+class Tensor;
 
 namespace io {
 
+class InputStream;
 class OutputStream;
 class RandomAccessFile;
 
@@ -53,7 +56,7 @@ struct MetadataVersion {
 
 static constexpr const char* kArrowMagicBytes = "ARROW1";
 
-struct ARROW_EXPORT FileBlock {
+struct FileBlock {
   FileBlock() {}
   FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
       : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
@@ -104,44 +107,25 @@ class DictionaryMemo {
 
 class Message;
 
-// Container for serialized Schema metadata contained in an IPC message
-class ARROW_EXPORT SchemaMetadata {
- public:
-  explicit SchemaMetadata(const void* header);
-  explicit SchemaMetadata(const std::shared_ptr<Message>& message);
-  SchemaMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
-
-  ~SchemaMetadata();
-
-  int num_fields() const;
-
-  // Retrieve a list of all the dictionary ids and types required by the schema for
-  // reconstruction. The presumption is that these will be loaded either from
-  // the stream or file (or they may already be somewhere else in memory)
-  Status GetDictionaryTypes(DictionaryTypeMap* id_to_field) const;
+// Retrieve a list of all the dictionary ids and types required by the schema for
+// reconstruction. The presumption is that these will be loaded either from
+// the stream or file (or they may already be somewhere else in memory)
+Status GetDictionaryTypes(const void* opaque_schema, DictionaryTypeMap* id_to_field);
 
-  // Construct a complete Schema from the message. May be expensive for very
-  // large schemas if you are only interested in a few fields
-  Status GetSchema(
-      const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out) const;
-
- private:
-  class SchemaMetadataImpl;
-  std::unique_ptr<SchemaMetadataImpl> impl_;
-
-  DISALLOW_COPY_AND_ASSIGN(SchemaMetadata);
-};
+// Construct a complete Schema from the message. May be expensive for very
+// large schemas if you are only interested in a few fields
+Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_memo,
+    std::shared_ptr<Schema>* out);
 
-struct ARROW_EXPORT BufferMetadata {
-  int32_t page;
-  int64_t offset;
-  int64_t length;
-};
+Status GetTensorMetadata(const void* opaque_tensor, std::shared_ptr<DataType>* type,
+    std::vector<int64_t>* shape, std::vector<int64_t>* strides,
+    std::vector<std::string>* dim_names);
 
 class ARROW_EXPORT Message {
  public:
+  enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH, TENSOR };
+
   ~Message();
-  enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH };
 
   static Status Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
       std::shared_ptr<Message>* out);
@@ -155,9 +139,6 @@ class ARROW_EXPORT Message {
  private:
   Message(const std::shared_ptr<Buffer>& buffer, int64_t offset);
 
-  friend class DictionaryBatchMetadata;
-  friend class SchemaMetadata;
-
   // Hide serialization details from user API
   class MessageImpl;
   std::unique_ptr<MessageImpl> impl_;
@@ -179,6 +160,17 @@ class ARROW_EXPORT Message {
 Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
     std::shared_ptr<Message>* message);
 
+/// Read length-prefixed message with as-yet unknown length. Returns nullptr if
+/// there are not enough bytes available or the message length is 0 (e.g. EOS
+/// in a stream)
+Status ReadMessage(io::InputStream* stream, std::shared_ptr<Message>* message);
+
+/// Write a serialized message with a length-prefix and padding to an 8-byte offset
+///
+/// <message_size: int32><message: const void*><padding>
+Status WriteMessage(
+    const Buffer& message, io::OutputStream* file, int32_t* message_length);
+
 // Serialize arrow::Schema as a Flatbuffer
 //
 // \param[in] schema a Schema instance
@@ -193,6 +185,9 @@ Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out);
 
+Status WriteTensorMessage(
+    const Tensor& tensor, int64_t buffer_start_offset, std::shared_ptr<Buffer>* out);
+
 Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out);
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 28320d98df9d1..b47b773192774 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -33,6 +33,7 @@
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
+#include "arrow/tensor.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -186,28 +187,9 @@ class StreamReader::StreamReaderImpl {
   }
 
   Status ReadNextMessage(Message::Type expected_type, std::shared_ptr<Message>* message) {
-    std::shared_ptr<Buffer> buffer;
-    RETURN_NOT_OK(stream_->Read(sizeof(int32_t), &buffer));
-
-    if (buffer->size() != sizeof(int32_t)) {
-      *message = nullptr;
-      return Status::OK();
-    }
-
-    int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
-
-    if (message_length == 0) {
-      // Optional 0 EOS control message
-      *message = nullptr;
-      return Status::OK();
-    }
-
-    RETURN_NOT_OK(stream_->Read(message_length, &buffer));
-    if (buffer->size() != message_length) {
-      return Status::IOError("Unexpected end of stream trying to read message");
-    }
+    RETURN_NOT_OK(ReadMessage(stream_.get(), message));
 
-    RETURN_NOT_OK(Message::Open(buffer, 0, message));
+    if ((*message) == nullptr) { return Status::OK(); }
 
     if ((*message)->type() != expected_type) {
       std::stringstream ss;
@@ -245,8 +227,7 @@ class StreamReader::StreamReaderImpl {
     std::shared_ptr<Message> message;
     RETURN_NOT_OK(ReadNextMessage(Message::SCHEMA, &message));
 
-    SchemaMetadata schema_meta(message);
-    RETURN_NOT_OK(schema_meta.GetDictionaryTypes(&dictionary_types_));
+    RETURN_NOT_OK(GetDictionaryTypes(message->header(), &dictionary_types_));
 
     // TODO(wesm): In future, we may want to reconcile the ids in the stream with
     // those found in the schema
@@ -255,7 +236,7 @@ class StreamReader::StreamReaderImpl {
       RETURN_NOT_OK(ReadNextDictionary());
     }
 
-    return schema_meta.GetSchema(dictionary_memo_, &schema_);
+    return GetSchema(message->header(), dictionary_memo_, &schema_);
   }
 
   Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
@@ -343,7 +324,6 @@ class FileReader::FileReaderImpl {
 
     // TODO(wesm): Verify the footer
     footer_ = flatbuf::GetFooter(footer_buffer_->data());
-    schema_metadata_.reset(new SchemaMetadata(footer_->schema()));
 
     return Status::OK();
   }
@@ -372,8 +352,6 @@ class FileReader::FileReaderImpl {
     return FileBlockFromFlatbuffer(footer_->dictionaries()->Get(i));
   }
 
-  const SchemaMetadata& schema_metadata() const { return *schema_metadata_; }
-
   Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
     DCHECK_GE(i, 0);
     DCHECK_LT(i, num_record_batches());
@@ -393,7 +371,7 @@ class FileReader::FileReaderImpl {
   }
 
   Status ReadSchema() {
-    RETURN_NOT_OK(schema_metadata_->GetDictionaryTypes(&dictionary_fields_));
+    RETURN_NOT_OK(GetDictionaryTypes(footer_->schema(), &dictionary_fields_));
 
     // Read all the dictionaries
     for (int i = 0; i < num_dictionaries(); ++i) {
@@ -419,7 +397,7 @@ class FileReader::FileReaderImpl {
     }
 
     // Get the schema
-    return schema_metadata_->GetSchema(*dictionary_memo_, &schema_);
+    return GetSchema(footer_->schema(), *dictionary_memo_, &schema_);
   }
 
   Status Open(const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset) {
@@ -441,7 +419,6 @@ class FileReader::FileReaderImpl {
   // Footer metadata
   std::shared_ptr<Buffer> footer_buffer_;
   const flatbuf::Footer* footer_;
-  std::unique_ptr<SchemaMetadata> schema_metadata_;
 
   DictionaryTypeMap dictionary_fields_;
   std::shared_ptr<DictionaryMemo> dictionary_memo_;
@@ -485,26 +462,46 @@ Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
   return impl_->GetRecordBatch(i, batch);
 }
 
-Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+static Status ReadContiguousPayload(int64_t offset, io::RandomAccessFile* file,
+    std::shared_ptr<Message>* message, std::shared_ptr<Buffer>* payload) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->Seek(offset));
+  RETURN_NOT_OK(ReadMessage(file, message));
 
-  RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
-  int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
-
-  std::shared_ptr<Message> message;
-  RETURN_NOT_OK(file->Read(flatbuffer_size, &buffer));
-  RETURN_NOT_OK(Message::Open(buffer, 0, &message));
+  if (*message == nullptr) {
+    return Status::Invalid("Unable to read metadata at offset");
+  }
 
   // TODO(ARROW-388): The buffer offsets start at 0, so we must construct a
   // RandomAccessFile according to that frame of reference
-  std::shared_ptr<Buffer> buffer_payload;
-  RETURN_NOT_OK(file->Read(message->body_length(), &buffer_payload));
-  io::BufferReader buffer_reader(buffer_payload);
+  RETURN_NOT_OK(file->Read((*message)->body_length(), payload));
+  return Status::OK();
+}
 
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+  std::shared_ptr<Buffer> payload;
+  std::shared_ptr<Message> message;
+
+  RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message, &payload));
+  io::BufferReader buffer_reader(payload);
   return ReadRecordBatch(*message, schema, kMaxNestingDepth, &buffer_reader, out);
 }
 
+Status ReadTensor(
+    int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out) {
+  std::shared_ptr<Message> message;
+  std::shared_ptr<Buffer> data;
+  RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message, &data));
+
+  std::shared_ptr<DataType> type;
+  std::vector<int64_t> shape;
+  std::vector<int64_t> strides;
+  std::vector<std::string> dim_names;
+  RETURN_NOT_OK(
+      GetTensorMetadata(message->header(), &type, &shape, &strides, &dim_names));
+  return MakeTensor(type, data, shape, strides, dim_names, out);
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 6d9e6ca7b0ab7..b62f0527e0ca0 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -17,8 +17,8 @@
 
 // Implement Arrow file layout for IPC/RPC purposes and short-lived storage
 
-#ifndef ARROW_IPC_FILE_H
-#define ARROW_IPC_FILE_H
+#ifndef ARROW_IPC_READER_H
+#define ARROW_IPC_READER_H
 
 #include <cstdint>
 #include <memory>
@@ -33,6 +33,7 @@ class Buffer;
 class RecordBatch;
 class Schema;
 class Status;
+class Tensor;
 
 namespace io {
 
@@ -43,18 +44,6 @@ class RandomAccessFile;
 
 namespace ipc {
 
-// Generic read functionsh; does not copy data if the input supports zero copy reads
-
-Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
-
-Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
-    int max_recursion_depth, io::RandomAccessFile* file,
-    std::shared_ptr<RecordBatch>* out);
-
-Status ReadDictionary(const Message& metadata, const DictionaryTypeMap& dictionary_types,
-    io::RandomAccessFile* file, std::shared_ptr<Array>* out);
-
 class ARROW_EXPORT StreamReader {
  public:
   ~StreamReader();
@@ -118,11 +107,24 @@ class ARROW_EXPORT FileReader {
   std::unique_ptr<FileReaderImpl> impl_;
 };
 
+// Generic read functionsh; does not copy data if the input supports zero copy reads
+Status ARROW_EXPORT ReadRecordBatch(const Message& metadata,
+    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
+    std::shared_ptr<RecordBatch>* out);
+
+Status ARROW_EXPORT ReadRecordBatch(const Message& metadata,
+    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
+
 /// Read encapsulated message and RecordBatch
 Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
+/// EXPERIMENTAL: Read arrow::Tensor from a contiguous message
+Status ARROW_EXPORT ReadTensor(
+    int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out);
+
 }  // namespace ipc
 }  // namespace arrow
 
-#endif  // ARROW_IPC_FILE_H
+#endif  // ARROW_IPC_READER_H
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 0a19f69d27d8c..249ef201c66bb 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -34,6 +34,7 @@
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
+#include "arrow/tensor.h"
 #include "arrow/type.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
@@ -143,46 +144,6 @@ class RecordBatchWriter : public ArrayVisitor {
         num_rows, body_length, field_nodes_, buffer_meta_, out);
   }
 
-  Status WriteMetadata(int64_t num_rows, int64_t body_length, io::OutputStream* dst,
-      int32_t* metadata_length) {
-    // Now that we have computed the locations of all of the buffers in shared
-    // memory, the data header can be converted to a flatbuffer and written out
-    //
-    // Note: The memory written here is prefixed by the size of the flatbuffer
-    // itself as an int32_t.
-    std::shared_ptr<Buffer> metadata_fb;
-    RETURN_NOT_OK(WriteMetadataMessage(num_rows, body_length, &metadata_fb));
-
-    // Need to write 4 bytes (metadata size), the metadata, plus padding to
-    // end on an 8-byte offset
-    int64_t start_offset;
-    RETURN_NOT_OK(dst->Tell(&start_offset));
-
-    int32_t padded_metadata_length = static_cast<int32_t>(metadata_fb->size()) + 4;
-    const int32_t remainder =
-        (padded_metadata_length + static_cast<int32_t>(start_offset)) % 8;
-    if (remainder != 0) { padded_metadata_length += 8 - remainder; }
-
-    // The returned metadata size includes the length prefix, the flatbuffer,
-    // plus padding
-    *metadata_length = padded_metadata_length;
-
-    // Write the flatbuffer size prefix including padding
-    int32_t flatbuffer_size = padded_metadata_length - 4;
-    RETURN_NOT_OK(
-        dst->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
-
-    // Write the flatbuffer
-    RETURN_NOT_OK(dst->Write(metadata_fb->data(), metadata_fb->size()));
-
-    // Write any padding
-    int32_t padding =
-        padded_metadata_length - static_cast<int32_t>(metadata_fb->size()) - 4;
-    if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
-
-    return Status::OK();
-  }
-
   Status Write(const RecordBatch& batch, io::OutputStream* dst, int32_t* metadata_length,
       int64_t* body_length) {
     RETURN_NOT_OK(Assemble(batch, body_length));
@@ -192,7 +153,14 @@ class RecordBatchWriter : public ArrayVisitor {
     RETURN_NOT_OK(dst->Tell(&start_position));
 #endif
 
-    RETURN_NOT_OK(WriteMetadata(batch.num_rows(), *body_length, dst, metadata_length));
+    // Now that we have computed the locations of all of the buffers in shared
+    // memory, the data header can be converted to a flatbuffer and written out
+    //
+    // Note: The memory written here is prefixed by the size of the flatbuffer
+    // itself as an int32_t.
+    std::shared_ptr<Buffer> metadata_fb;
+    RETURN_NOT_OK(WriteMetadataMessage(batch.num_rows(), *body_length, &metadata_fb));
+    RETURN_NOT_OK(WriteMessage(*metadata_fb, dst, metadata_length));
 
 #ifndef NDEBUG
     RETURN_NOT_OK(dst->Tell(&current_position));
@@ -504,6 +472,28 @@ Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
   return writer.Write(batch, dst, metadata_length, body_length);
 }
 
+Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+    MemoryPool* pool) {
+  return WriteRecordBatch(batch, buffer_start_offset, dst, metadata_length, body_length,
+      pool, kMaxNestingDepth, true);
+}
+
+Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
+    int64_t* body_length) {
+  std::shared_ptr<Buffer> metadata;
+  RETURN_NOT_OK(WriteTensorMessage(tensor, 0, &metadata));
+  RETURN_NOT_OK(WriteMessage(*metadata, dst, metadata_length));
+  auto data = tensor.data();
+  if (data) {
+    *body_length = data->size();
+    return dst->Write(data->data(), *body_length);
+  } else {
+    *body_length = 0;
+    return Status::OK();
+  }
+}
+
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
     int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
     int64_t* body_length, MemoryPool* pool) {
@@ -736,12 +726,5 @@ Status FileWriter::Close() {
   return impl_->Close();
 }
 
-Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
-    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool) {
-  return WriteRecordBatch(batch, buffer_start_offset, dst, metadata_length, body_length,
-      pool, kMaxNestingDepth, true);
-}
-
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index c572157b465a6..8b2dc9cd48788 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -17,8 +17,8 @@
 
 // Implement Arrow streaming binary format
 
-#ifndef ARROW_IPC_STREAM_H
-#define ARROW_IPC_STREAM_H
+#ifndef ARROW_IPC_WRITER_H
+#define ARROW_IPC_WRITER_H
 
 #include <cstdint>
 #include <memory>
@@ -36,6 +36,7 @@ class MemoryPool;
 class RecordBatch;
 class Schema;
 class Status;
+class Tensor;
 
 namespace io {
 
@@ -125,7 +126,12 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
     MemoryPool* pool);
 
+/// EXPERIMENTAL: Write arrow::Tensor as a contiguous message
+/// <metadata size><metadata><tensor data>
+Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
+    int64_t* body_length);
+
 }  // namespace ipc
 }  // namespace arrow
 
-#endif  // ARROW_IPC_STREAM_H
+#endif  // ARROW_IPC_WRITER_H
diff --git a/cpp/src/arrow/tensor-test.cc b/cpp/src/arrow/tensor-test.cc
index 99a94934c7990..336905c21ae81 100644
--- a/cpp/src/arrow/tensor-test.cc
+++ b/cpp/src/arrow/tensor-test.cc
@@ -61,13 +61,36 @@ TEST(TestTensor, BasicCtors) {
 
   ASSERT_EQ(24, t1.size());
   ASSERT_TRUE(t1.is_mutable());
-  ASSERT_FALSE(t1.has_dim_names());
 
   ASSERT_EQ(strides, t1.strides());
   ASSERT_EQ(strides, t2.strides());
 
   ASSERT_EQ("foo", t3.dim_name(0));
   ASSERT_EQ("bar", t3.dim_name(1));
+  ASSERT_EQ("", t1.dim_name(0));
+  ASSERT_EQ("", t1.dim_name(1));
+}
+
+TEST(TestTensor, IsContiguous) {
+  const int64_t values = 24;
+  std::vector<int64_t> shape = {4, 6};
+  std::vector<int64_t> strides = {48, 8};
+
+  using T = int64_t;
+
+  std::shared_ptr<MutableBuffer> buffer;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer));
+
+  std::vector<int64_t> c_strides = {48, 8};
+  std::vector<int64_t> f_strides = {8, 32};
+  std::vector<int64_t> noncontig_strides = {8, 8};
+  Int64Tensor t1(buffer, shape, c_strides);
+  Int64Tensor t2(buffer, shape, f_strides);
+  Int64Tensor t3(buffer, shape, noncontig_strides);
+
+  ASSERT_TRUE(t1.is_contiguous());
+  ASSERT_TRUE(t2.is_contiguous());
+  ASSERT_FALSE(t3.is_contiguous());
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index 7c4593fc40e66..9a8de5119ea58 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -27,14 +27,15 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
+#include "arrow/compare.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
 
-void ComputeRowMajorStrides(const FixedWidthType& type, const std::vector<int64_t>& shape,
-    std::vector<int64_t>* strides) {
+static void ComputeRowMajorStrides(const FixedWidthType& type,
+    const std::vector<int64_t>& shape, std::vector<int64_t>* strides) {
   int64_t remaining = type.bit_width() / 8;
   for (int64_t dimsize : shape) {
     remaining *= dimsize;
@@ -46,6 +47,15 @@ void ComputeRowMajorStrides(const FixedWidthType& type, const std::vector<int64_
   }
 }
 
+static void ComputeColumnMajorStrides(const FixedWidthType& type,
+    const std::vector<int64_t>& shape, std::vector<int64_t>* strides) {
+  int64_t total = type.bit_width() / 8;
+  for (int64_t dimsize : shape) {
+    strides->push_back(total);
+    total *= dimsize;
+  }
+}
+
 /// Constructor with strides and dimension names
 Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
     const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
@@ -66,14 +76,36 @@ Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buff
     : Tensor(type, data, shape, {}, {}) {}
 
 const std::string& Tensor::dim_name(int i) const {
-  DCHECK_LT(i, static_cast<int>(dim_names_.size()));
-  return dim_names_[i];
+  static const std::string kEmpty = "";
+  if (dim_names_.size() == 0) {
+    return kEmpty;
+  } else {
+    DCHECK_LT(i, static_cast<int>(dim_names_.size()));
+    return dim_names_[i];
+  }
 }
 
 int64_t Tensor::size() const {
   return std::accumulate(shape_.begin(), shape_.end(), 1, std::multiplies<int64_t>());
 }
 
+bool Tensor::is_contiguous() const {
+  std::vector<int64_t> c_strides;
+  std::vector<int64_t> f_strides;
+
+  const auto& fw_type = static_cast<const FixedWidthType&>(*type_);
+  ComputeRowMajorStrides(fw_type, shape_, &c_strides);
+  ComputeColumnMajorStrides(fw_type, shape_, &f_strides);
+  return strides_ == c_strides || strides_ == f_strides;
+}
+
+bool Tensor::Equals(const Tensor& other) const {
+  bool are_equal = false;
+  Status error = TensorEquals(*this, other, &are_equal);
+  if (!error.ok()) { DCHECK(false) << "Tensors not comparable: " << error.ToString(); }
+  return are_equal;
+}
+
 template <typename T>
 NumericTensor<T>::NumericTensor(const std::shared_ptr<Buffer>& data,
     const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
@@ -112,4 +144,31 @@ template class ARROW_TEMPLATE_EXPORT NumericTensor<HalfFloatType>;
 template class ARROW_TEMPLATE_EXPORT NumericTensor<FloatType>;
 template class ARROW_TEMPLATE_EXPORT NumericTensor<DoubleType>;
 
+#define TENSOR_CASE(TYPE, TENSOR_TYPE)                                        \
+  case Type::TYPE:                                                            \
+    *tensor = std::make_shared<TENSOR_TYPE>(data, shape, strides, dim_names); \
+    break;
+
+Status ARROW_EXPORT MakeTensor(const std::shared_ptr<DataType>& type,
+    const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
+    const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names,
+    std::shared_ptr<Tensor>* tensor) {
+  switch (type->type) {
+    TENSOR_CASE(INT8, Int8Tensor);
+    TENSOR_CASE(INT16, Int16Tensor);
+    TENSOR_CASE(INT32, Int32Tensor);
+    TENSOR_CASE(INT64, Int64Tensor);
+    TENSOR_CASE(UINT8, UInt8Tensor);
+    TENSOR_CASE(UINT16, UInt16Tensor);
+    TENSOR_CASE(UINT32, UInt32Tensor);
+    TENSOR_CASE(UINT64, UInt64Tensor);
+    TENSOR_CASE(HALF_FLOAT, HalfFloatTensor);
+    TENSOR_CASE(FLOAT, FloatTensor);
+    TENSOR_CASE(DOUBLE, DoubleTensor);
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index 7bee867a9b33a..eeb5c3e8e5536 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -73,12 +73,15 @@ class ARROW_EXPORT Tensor {
       const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
       const std::vector<std::string>& dim_names);
 
+  std::shared_ptr<DataType> type() const { return type_; }
   std::shared_ptr<Buffer> data() const { return data_; }
+
   const std::vector<int64_t>& shape() const { return shape_; }
   const std::vector<int64_t>& strides() const { return strides_; }
 
+  int ndim() const { return static_cast<int>(shape_.size()); }
+
   const std::string& dim_name(int i) const;
-  bool has_dim_names() const { return shape_.size() > 0 && dim_names_.size() > 0; }
 
   /// Total number of value cells in the tensor
   int64_t size() const;
@@ -86,13 +89,17 @@ class ARROW_EXPORT Tensor {
   /// Return true if the underlying data buffer is mutable
   bool is_mutable() const { return data_->is_mutable(); }
 
+  bool is_contiguous() const;
+
+  Type::type type_enum() const { return type_->type; }
+
+  bool Equals(const Tensor& other) const;
+
  protected:
   Tensor() {}
 
   std::shared_ptr<DataType> type_;
-
   std::shared_ptr<Buffer> data_;
-
   std::vector<int64_t> shape_;
   std::vector<int64_t> strides_;
 
@@ -126,6 +133,11 @@ class ARROW_EXPORT NumericTensor : public Tensor {
   value_type* mutable_raw_data_;
 };
 
+Status ARROW_EXPORT MakeTensor(const std::shared_ptr<DataType>& type,
+    const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
+    const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names,
+    std::shared_ptr<Tensor>* tensor);
+
 // ----------------------------------------------------------------------
 // extern templates and other details
 
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 1270aee1622ea..b73d5a68d257e 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -38,6 +38,7 @@ template <>
 struct TypeTraits<UInt8Type> {
   using ArrayType = UInt8Array;
   using BuilderType = UInt8Builder;
+  using TensorType = UInt8Tensor;
   static inline int64_t bytes_required(int64_t elements) { return elements; }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return uint8(); }
@@ -47,6 +48,7 @@ template <>
 struct TypeTraits<Int8Type> {
   using ArrayType = Int8Array;
   using BuilderType = Int8Builder;
+  using TensorType = Int8Tensor;
   static inline int64_t bytes_required(int64_t elements) { return elements; }
   constexpr static bool is_parameter_free = true;
   static inline std::shared_ptr<DataType> type_singleton() { return int8(); }
@@ -56,6 +58,7 @@ template <>
 struct TypeTraits<UInt16Type> {
   using ArrayType = UInt16Array;
   using BuilderType = UInt16Builder;
+  using TensorType = UInt16Tensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(uint16_t);
@@ -68,6 +71,7 @@ template <>
 struct TypeTraits<Int16Type> {
   using ArrayType = Int16Array;
   using BuilderType = Int16Builder;
+  using TensorType = Int16Tensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(int16_t);
@@ -80,6 +84,7 @@ template <>
 struct TypeTraits<UInt32Type> {
   using ArrayType = UInt32Array;
   using BuilderType = UInt32Builder;
+  using TensorType = UInt32Tensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(uint32_t);
@@ -92,6 +97,7 @@ template <>
 struct TypeTraits<Int32Type> {
   using ArrayType = Int32Array;
   using BuilderType = Int32Builder;
+  using TensorType = Int32Tensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(int32_t);
@@ -104,6 +110,7 @@ template <>
 struct TypeTraits<UInt64Type> {
   using ArrayType = UInt64Array;
   using BuilderType = UInt64Builder;
+  using TensorType = UInt64Tensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(uint64_t);
@@ -116,6 +123,7 @@ template <>
 struct TypeTraits<Int64Type> {
   using ArrayType = Int64Array;
   using BuilderType = Int64Builder;
+  using TensorType = Int64Tensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(int64_t);
@@ -185,6 +193,7 @@ template <>
 struct TypeTraits<HalfFloatType> {
   using ArrayType = HalfFloatArray;
   using BuilderType = HalfFloatBuilder;
+  using TensorType = HalfFloatTensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return elements * sizeof(uint16_t);
@@ -197,6 +206,7 @@ template <>
 struct TypeTraits<FloatType> {
   using ArrayType = FloatArray;
   using BuilderType = FloatBuilder;
+  using TensorType = FloatTensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return static_cast<int64_t>(elements * sizeof(float));
@@ -209,6 +219,7 @@ template <>
 struct TypeTraits<DoubleType> {
   using ArrayType = DoubleArray;
   using BuilderType = DoubleBuilder;
+  using TensorType = DoubleTensor;
 
   static inline int64_t bytes_required(int64_t elements) {
     return static_cast<int64_t>(elements * sizeof(double));
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index 586b123e67cfb..cbc4d5acdb8cf 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -22,6 +22,7 @@
 
 #include "arrow/array.h"
 #include "arrow/status.h"
+#include "arrow/tensor.h"
 #include "arrow/type.h"
 
 namespace arrow {
@@ -103,6 +104,31 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
   return Status::NotImplemented("Type not implemented");
 }
 
+#define TENSOR_VISIT_INLINE(TYPE_CLASS) \
+  case TYPE_CLASS::type_id:             \
+    return visitor->Visit(              \
+        static_cast<const typename TypeTraits<TYPE_CLASS>::TensorType&>(array));
+
+template <typename VISITOR>
+inline Status VisitTensorInline(const Tensor& array, VISITOR* visitor) {
+  switch (array.type_enum()) {
+    TENSOR_VISIT_INLINE(Int8Type);
+    TENSOR_VISIT_INLINE(UInt8Type);
+    TENSOR_VISIT_INLINE(Int16Type);
+    TENSOR_VISIT_INLINE(UInt16Type);
+    TENSOR_VISIT_INLINE(Int32Type);
+    TENSOR_VISIT_INLINE(UInt32Type);
+    TENSOR_VISIT_INLINE(Int64Type);
+    TENSOR_VISIT_INLINE(UInt64Type);
+    TENSOR_VISIT_INLINE(HalfFloatType);
+    TENSOR_VISIT_INLINE(FloatType);
+    TENSOR_VISIT_INLINE(DoubleType);
+    default:
+      break;
+  }
+  return Status::NotImplemented("Type not implemented");
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_VISITOR_INLINE_H
diff --git a/format/Tensor.fbs b/format/Tensor.fbs
index bc5b6d1289b2f..18b614c3bde62 100644
--- a/format/Tensor.fbs
+++ b/format/Tensor.fbs
@@ -32,16 +32,6 @@ table TensorDim {
   name: string;
 }
 
-enum TensorOrder : byte {
-  /// Higher dimensions vary first when traversing data in byte-contiguous
-  /// order, aka "C order"
-  ROW_MAJOR,
-
-  /// Lower dimensions vary first when traversing data in byte-contiguous
-  /// order, aka "Fortran order"
-  COLUMN_MAJOR
-}
-
 table Tensor {
   /// The type of data contained in a value cell. Currently only fixed-width
   /// value types are supported, no strings or nested types
@@ -50,8 +40,8 @@ table Tensor {
   /// The dimensions of the tensor, optionally named
   shape: [TensorDim];
 
-  /// The memory order of the tensor's data
-  order: TensorOrder;
+  /// Non-negative byte offsets to advance one value cell along each dimension
+  strides: [long];
 
   /// The location and size of the tensor's data
   data: Buffer;

From 4938d8d7cea65d039650f684afaa29a74510c3e0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 30 Mar 2017 18:22:11 -0400
Subject: [PATCH 0435/1644] ARROW-726: [C++] Fix segfault caused when passing
 non-buffer object to arrow::py::PyBuffer

This leads to calling `Py_DECREF` on a null pointer

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #459 from wesm/ARROW-726 and squashes the following commits:

a764134 [Wes McKinney] Fix segfault caused when passing non-buffer object to arrow::py::PyBuffer. Fix some compiler warnings
---
 cpp/src/arrow/python/builtin_convert.cc |  4 ++--
 cpp/src/arrow/python/common.cc          |  4 ++--
 cpp/src/arrow/python/pandas-test.cc     | 10 ++++++++--
 cpp/src/arrow/python/pandas_convert.cc  | 10 +++++-----
 cpp/src/arrow/python/pandas_convert.h   |  2 +-
 5 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 9acccc149664b..6e59845dea76a 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -390,7 +390,7 @@ class BytesConverter : public TypedConverter<BinaryBuilder> {
       // No error checking
       length = PyBytes_GET_SIZE(bytes_obj);
       bytes = PyBytes_AS_STRING(bytes_obj);
-      RETURN_NOT_OK(typed_builder_->Append(bytes, length));
+      RETURN_NOT_OK(typed_builder_->Append(bytes, static_cast<int32_t>(length)));
     }
     return Status::OK();
   }
@@ -422,7 +422,7 @@ class UTF8Converter : public TypedConverter<StringBuilder> {
       // No error checking
       length = PyBytes_GET_SIZE(bytes_obj);
       bytes = PyBytes_AS_STRING(bytes_obj);
-      RETURN_NOT_OK(typed_builder_->Append(bytes, length));
+      RETURN_NOT_OK(typed_builder_->Append(bytes, static_cast<int32_t>(length)));
     }
     return Status::OK();
   }
diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
index a5aea30884468..717cb5c5cc122 100644
--- a/cpp/src/arrow/python/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -47,7 +47,7 @@ MemoryPool* get_memory_pool() {
 // ----------------------------------------------------------------------
 // PyBuffer
 
-PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0) {
+PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0), obj_(nullptr) {
   if (PyObject_CheckBuffer(obj)) {
     obj_ = PyMemoryView_FromObject(obj);
     Py_buffer* buffer = PyMemoryView_GET_BUFFER(obj_);
@@ -61,7 +61,7 @@ PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0) {
 
 PyBuffer::~PyBuffer() {
   PyAcquireGIL lock;
-  Py_DECREF(obj_);
+  Py_XDECREF(obj_);
 }
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/pandas-test.cc b/cpp/src/arrow/python/pandas-test.cc
index 0d643df2e9f38..a4e640b83718b 100644
--- a/cpp/src/arrow/python/pandas-test.cc
+++ b/cpp/src/arrow/python/pandas-test.cc
@@ -24,20 +24,26 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
-#include "arrow/python/pandas_convert.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 
+#include "arrow/python/common.h"
+#include "arrow/python/pandas_convert.h"
+
 namespace arrow {
 namespace py {
 
+TEST(PyBuffer, InvalidInputObject) {
+  PyBuffer buffer(Py_None);
+}
+
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   StringBuilder builder(default_memory_pool());
   const char value[] = {'\xf1', '\0'};
 
   for (int i = 0; i < 1000; ++i) {
-    builder.Append(value, strlen(value));
+    builder.Append(value, static_cast<int32_t>(strlen(value)));
   }
 
   std::shared_ptr<Array> arr;
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 685b1f421c457..db2e90eb8b0ff 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -159,13 +159,13 @@ Status AppendObjectStrings(StringBuilder& string_builder, PyObject** objects,
         PyErr_Clear();
         return Status::TypeError("failed converting unicode to UTF8");
       }
-      const int64_t length = PyBytes_GET_SIZE(obj);
+      const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
       Status s = string_builder.Append(PyBytes_AS_STRING(obj), length);
       Py_DECREF(obj);
       if (!s.ok()) { return s; }
     } else if (PyBytes_Check(obj)) {
       *have_bytes = true;
-      const int64_t length = PyBytes_GET_SIZE(obj);
+      const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
       RETURN_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
     } else {
       string_builder.AppendNull();
@@ -235,7 +235,7 @@ class PandasConverter : public TypeVisitor {
   }
 
   Status InitNullBitmap() {
-    int null_bytes = BitUtil::BytesForBits(length_);
+    int64_t null_bytes = BitUtil::BytesForBits(length_);
 
     null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
     RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
@@ -357,7 +357,7 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
 
 template <>
 inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>* data) {
-  int nbytes = BitUtil::BytesForBits(length_);
+  int64_t nbytes = BitUtil::BytesForBits(length_);
   auto buffer = std::make_shared<PoolBuffer>(pool_);
   RETURN_NOT_OK(buffer->Resize(nbytes));
 
@@ -423,7 +423,7 @@ Status PandasConverter::ConvertBooleans(std::shared_ptr<Array>* out) {
 
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
 
-  int nbytes = BitUtil::BytesForBits(length_);
+  int64_t nbytes = BitUtil::BytesForBits(length_);
   auto data = std::make_shared<PoolBuffer>(pool_);
   RETURN_NOT_OK(data->Resize(nbytes));
   uint8_t* bitmap = data->mutable_data();
diff --git a/cpp/src/arrow/python/pandas_convert.h b/cpp/src/arrow/python/pandas_convert.h
index a33741efaa492..12644d98da156 100644
--- a/cpp/src/arrow/python/pandas_convert.h
+++ b/cpp/src/arrow/python/pandas_convert.h
@@ -31,7 +31,7 @@ namespace arrow {
 
 class Array;
 class Column;
-class DataType;
+struct DataType;
 class MemoryPool;
 class Status;
 class Table;

From ae2da980b94c73719f659071537e40570981adf4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 30 Mar 2017 18:31:23 -0400
Subject: [PATCH 0436/1644] ARROW-743: [C++] Consolidate all but decimal array
 tests into array-test, collect some tests in type-test.cc

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #463 from wesm/ARROW-743 and squashes the following commits:

49df9f7 [Wes McKinney] Consolidate all but decimal array tests into array-test, move some type tests to type-test
---
 cpp/src/arrow/CMakeLists.txt           |    6 -
 cpp/src/arrow/array-dictionary-test.cc |  150 --
 cpp/src/arrow/array-list-test.cc       |  238 ----
 cpp/src/arrow/array-primitive-test.cc  |  543 -------
 cpp/src/arrow/array-string-test.cc     |  654 ---------
 cpp/src/arrow/array-struct-test.cc     |  410 ------
 cpp/src/arrow/array-test.cc            | 1812 +++++++++++++++++++++++-
 cpp/src/arrow/array-union-test.cc      |   67 -
 cpp/src/arrow/io/io-hdfs-test.cc       |    7 +-
 cpp/src/arrow/type-test.cc             |   45 +
 10 files changed, 1858 insertions(+), 2074 deletions(-)
 delete mode 100644 cpp/src/arrow/array-dictionary-test.cc
 delete mode 100644 cpp/src/arrow/array-list-test.cc
 delete mode 100644 cpp/src/arrow/array-primitive-test.cc
 delete mode 100644 cpp/src/arrow/array-string-test.cc
 delete mode 100644 cpp/src/arrow/array-struct-test.cc
 delete mode 100644 cpp/src/arrow/array-union-test.cc

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 5c9aadf9ee79b..bd33bf5b8296e 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -50,12 +50,6 @@ install(
 ADD_ARROW_TEST(allocator-test)
 ADD_ARROW_TEST(array-test)
 ADD_ARROW_TEST(array-decimal-test)
-ADD_ARROW_TEST(array-dictionary-test)
-ADD_ARROW_TEST(array-list-test)
-ADD_ARROW_TEST(array-primitive-test)
-ADD_ARROW_TEST(array-string-test)
-ADD_ARROW_TEST(array-struct-test)
-ADD_ARROW_TEST(array-union-test)
 ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(memory_pool-test)
 ADD_ARROW_TEST(pretty_print-test)
diff --git a/cpp/src/arrow/array-dictionary-test.cc b/cpp/src/arrow/array-dictionary-test.cc
deleted file mode 100644
index 0c4e628111a15..0000000000000
--- a/cpp/src/arrow/array-dictionary-test.cc
+++ /dev/null
@@ -1,150 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdlib>
-#include <memory>
-#include <numeric>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/memory_pool.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-TEST(TestDictionary, Basics) {
-  std::vector<int32_t> values = {100, 1000, 10000, 100000};
-  std::shared_ptr<Array> dict;
-  ArrayFromVector<Int32Type, int32_t>(values, &dict);
-
-  std::shared_ptr<DictionaryType> type1 =
-      std::dynamic_pointer_cast<DictionaryType>(dictionary(int16(), dict));
-  DictionaryType type2(int16(), dict);
-
-  ASSERT_TRUE(int16()->Equals(type1->index_type()));
-  ASSERT_TRUE(type1->dictionary()->Equals(dict));
-
-  ASSERT_TRUE(int16()->Equals(type2.index_type()));
-  ASSERT_TRUE(type2.dictionary()->Equals(dict));
-
-  ASSERT_EQ("dictionary<values=int32, indices=int16>", type1->ToString());
-}
-
-TEST(TestDictionary, Equals) {
-  std::vector<bool> is_valid = {true, true, false, true, true, true};
-
-  std::shared_ptr<Array> dict;
-  std::vector<std::string> dict_values = {"foo", "bar", "baz"};
-  ArrayFromVector<StringType, std::string>(dict_values, &dict);
-  std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
-
-  std::shared_ptr<Array> dict2;
-  std::vector<std::string> dict2_values = {"foo", "bar", "baz", "qux"};
-  ArrayFromVector<StringType, std::string>(dict2_values, &dict2);
-  std::shared_ptr<DataType> dict2_type = dictionary(int16(), dict2);
-
-  std::shared_ptr<Array> indices;
-  std::vector<int16_t> indices_values = {1, 2, -1, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(is_valid, indices_values, &indices);
-
-  std::shared_ptr<Array> indices2;
-  std::vector<int16_t> indices2_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(is_valid, indices2_values, &indices2);
-
-  std::shared_ptr<Array> indices3;
-  std::vector<int16_t> indices3_values = {1, 1, 0, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(is_valid, indices3_values, &indices3);
-
-  auto array = std::make_shared<DictionaryArray>(dict_type, indices);
-  auto array2 = std::make_shared<DictionaryArray>(dict_type, indices2);
-  auto array3 = std::make_shared<DictionaryArray>(dict2_type, indices);
-  auto array4 = std::make_shared<DictionaryArray>(dict_type, indices3);
-
-  ASSERT_TRUE(array->Equals(array));
-
-  // Equal, because the unequal index is masked by null
-  ASSERT_TRUE(array->Equals(array2));
-
-  // Unequal dictionaries
-  ASSERT_FALSE(array->Equals(array3));
-
-  // Unequal indices
-  ASSERT_FALSE(array->Equals(array4));
-
-  // RangeEquals
-  ASSERT_TRUE(array->RangeEquals(3, 6, 3, array4));
-  ASSERT_FALSE(array->RangeEquals(1, 3, 1, array4));
-
-  // ARROW-33 Test slices
-  const int64_t size = array->length();
-
-  std::shared_ptr<Array> slice, slice2;
-  slice = array->Array::Slice(2);
-  slice2 = array->Array::Slice(2);
-  ASSERT_EQ(size - 2, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(2, array->length(), 0, slice));
-
-  // Chained slices
-  slice2 = array->Array::Slice(1)->Array::Slice(1);
-  ASSERT_TRUE(slice->Equals(slice2));
-
-  slice = array->Slice(1, 3);
-  slice2 = array->Slice(1, 3);
-  ASSERT_EQ(3, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(1, 4, 0, slice));
-}
-
-TEST(TestDictionary, Validate) {
-  std::vector<bool> is_valid = {true, true, false, true, true, true};
-
-  std::shared_ptr<Array> dict;
-  std::vector<std::string> dict_values = {"foo", "bar", "baz"};
-  ArrayFromVector<StringType, std::string>(dict_values, &dict);
-  std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
-
-  std::shared_ptr<Array> indices;
-  std::vector<uint8_t> indices_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<UInt8Type, uint8_t>(is_valid, indices_values, &indices);
-
-  std::shared_ptr<Array> indices2;
-  std::vector<float> indices2_values = {1., 2., 0., 0., 2., 0.};
-  ArrayFromVector<FloatType, float>(is_valid, indices2_values, &indices2);
-
-  std::shared_ptr<Array> indices3;
-  std::vector<int64_t> indices3_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<Int64Type, int64_t>(is_valid, indices3_values, &indices3);
-
-  std::shared_ptr<Array> arr = std::make_shared<DictionaryArray>(dict_type, indices);
-  std::shared_ptr<Array> arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
-  std::shared_ptr<Array> arr3 = std::make_shared<DictionaryArray>(dict_type, indices3);
-
-  // Only checking index type for now
-  ASSERT_OK(arr->Validate());
-  ASSERT_RAISES(Invalid, arr2->Validate());
-  ASSERT_OK(arr3->Validate());
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/array-list-test.cc b/cpp/src/arrow/array-list-test.cc
deleted file mode 100644
index 1cfa77f684868..0000000000000
--- a/cpp/src/arrow/array-list-test.cc
+++ /dev/null
@@ -1,238 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdlib>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/status.h"
-#include "arrow/test-common.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-
-using std::shared_ptr;
-using std::string;
-using std::unique_ptr;
-using std::vector;
-
-namespace arrow {
-
-// ----------------------------------------------------------------------
-// List tests
-
-class TestListBuilder : public TestBuilder {
- public:
-  void SetUp() {
-    TestBuilder::SetUp();
-
-    value_type_ = TypePtr(new Int32Type());
-    type_ = TypePtr(new ListType(value_type_));
-
-    std::shared_ptr<ArrayBuilder> tmp;
-    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-    builder_ = std::dynamic_pointer_cast<ListBuilder>(tmp);
-  }
-
-  void Done() {
-    std::shared_ptr<Array> out;
-    EXPECT_OK(builder_->Finish(&out));
-    result_ = std::dynamic_pointer_cast<ListArray>(out);
-  }
-
- protected:
-  TypePtr value_type_;
-  TypePtr type_;
-
-  shared_ptr<ListBuilder> builder_;
-  shared_ptr<ListArray> result_;
-};
-
-TEST_F(TestListBuilder, Equality) {
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
-
-  std::shared_ptr<Array> array, equal_array, unequal_array;
-  vector<int32_t> equal_offsets = {0, 1, 2, 5, 6, 7, 8, 10};
-  vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2, 5, 6};
-  vector<int32_t> unequal_offsets = {0, 1, 4, 7};
-  vector<int32_t> unequal_values = {1, 2, 2, 2, 3, 4, 5};
-
-  // setup two equal arrays
-  ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
-  ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
-
-  ASSERT_OK(builder_->Finish(&array));
-  ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
-  ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
-
-  ASSERT_OK(builder_->Finish(&equal_array));
-  // now an unequal one
-  ASSERT_OK(builder_->Append(unequal_offsets.data(), unequal_offsets.size()));
-  ASSERT_OK(vb->Append(unequal_values.data(), unequal_values.size()));
-
-  ASSERT_OK(builder_->Finish(&unequal_array));
-
-  // Test array equality
-  EXPECT_TRUE(array->Equals(array));
-  EXPECT_TRUE(array->Equals(equal_array));
-  EXPECT_TRUE(equal_array->Equals(array));
-  EXPECT_FALSE(equal_array->Equals(unequal_array));
-  EXPECT_FALSE(unequal_array->Equals(equal_array));
-
-  // Test range equality
-  EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_array));
-  EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_array));
-  EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
-  EXPECT_TRUE(array->RangeEquals(2, 3, 2, unequal_array));
-
-  // Check with slices, ARROW-33
-  std::shared_ptr<Array> slice, slice2;
-
-  slice = array->Slice(2);
-  slice2 = array->Slice(2);
-  ASSERT_EQ(array->length() - 2, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(2, slice->length(), 0, slice));
-
-  // Chained slices
-  slice2 = array->Slice(1)->Slice(1);
-  ASSERT_TRUE(slice->Equals(slice2));
-
-  slice = array->Slice(1, 4);
-  slice2 = array->Slice(1, 4);
-  ASSERT_EQ(4, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(1, 5, 0, slice));
-}
-
-TEST_F(TestListBuilder, TestResize) {}
-
-TEST_F(TestListBuilder, TestAppendNull) {
-  ASSERT_OK(builder_->AppendNull());
-  ASSERT_OK(builder_->AppendNull());
-
-  Done();
-
-  ASSERT_OK(result_->Validate());
-  ASSERT_TRUE(result_->IsNull(0));
-  ASSERT_TRUE(result_->IsNull(1));
-
-  ASSERT_EQ(0, result_->raw_value_offsets()[0]);
-  ASSERT_EQ(0, result_->value_offset(1));
-  ASSERT_EQ(0, result_->value_offset(2));
-
-  Int32Array* values = static_cast<Int32Array*>(result_->values().get());
-  ASSERT_EQ(0, values->length());
-}
-
-void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& values,
-    const vector<uint8_t>& is_valid) {
-  ASSERT_OK(result->Validate());
-  ASSERT_EQ(1, result->null_count());
-  ASSERT_EQ(0, result->values()->null_count());
-
-  ASSERT_EQ(3, result->length());
-  vector<int32_t> ex_offsets = {0, 3, 3, 7};
-  for (size_t i = 0; i < ex_offsets.size(); ++i) {
-    ASSERT_EQ(ex_offsets[i], result->value_offset(i));
-  }
-
-  for (int i = 0; i < result->length(); ++i) {
-    ASSERT_EQ(!static_cast<bool>(is_valid[i]), result->IsNull(i));
-  }
-
-  ASSERT_EQ(7, result->values()->length());
-  Int32Array* varr = static_cast<Int32Array*>(result->values().get());
-
-  for (size_t i = 0; i < values.size(); ++i) {
-    ASSERT_EQ(values[i], varr->Value(i));
-  }
-}
-
-TEST_F(TestListBuilder, TestBasics) {
-  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
-  vector<int> lengths = {3, 0, 4};
-  vector<uint8_t> is_valid = {1, 0, 1};
-
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
-
-  ASSERT_OK(builder_->Reserve(lengths.size()));
-  ASSERT_OK(vb->Reserve(values.size()));
-
-  int pos = 0;
-  for (size_t i = 0; i < lengths.size(); ++i) {
-    ASSERT_OK(builder_->Append(is_valid[i] > 0));
-    for (int j = 0; j < lengths[i]; ++j) {
-      vb->Append(values[pos++]);
-    }
-  }
-
-  Done();
-  ValidateBasicListArray(result_.get(), values, is_valid);
-}
-
-TEST_F(TestListBuilder, BulkAppend) {
-  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
-  vector<int> lengths = {3, 0, 4};
-  vector<uint8_t> is_valid = {1, 0, 1};
-  vector<int32_t> offsets = {0, 3, 3};
-
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
-  ASSERT_OK(vb->Reserve(values.size()));
-
-  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
-  for (int32_t value : values) {
-    vb->Append(value);
-  }
-  Done();
-  ValidateBasicListArray(result_.get(), values, is_valid);
-}
-
-TEST_F(TestListBuilder, BulkAppendInvalid) {
-  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
-  vector<int> lengths = {3, 0, 4};
-  vector<uint8_t> is_null = {0, 1, 0};
-  vector<uint8_t> is_valid = {1, 0, 1};
-  vector<int32_t> offsets = {0, 2, 4};  // should be 0, 3, 3 given the is_null array
-
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
-  ASSERT_OK(vb->Reserve(values.size()));
-
-  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
-  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
-  for (int32_t value : values) {
-    vb->Append(value);
-  }
-
-  Done();
-  ASSERT_RAISES(Invalid, result_->Validate());
-}
-
-TEST_F(TestListBuilder, TestZeroLength) {
-  // All buffers are null
-  Done();
-  ASSERT_OK(result_->Validate());
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
deleted file mode 100644
index fe60170cc5cc4..0000000000000
--- a/cpp/src/arrow/array-primitive-test.cc
+++ /dev/null
@@ -1,543 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/builder.h"
-#include "arrow/status.h"
-#include "arrow/test-common.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/type_traits.h"
-#include "arrow/util/bit-util.h"
-
-using std::string;
-using std::shared_ptr;
-using std::unique_ptr;
-using std::vector;
-
-namespace arrow {
-
-class Array;
-
-#define PRIMITIVE_TEST(KLASS, ENUM, NAME)   \
-  TEST(TypesTest, TestPrimitive_##ENUM) {   \
-    KLASS tp;                               \
-                                            \
-    ASSERT_EQ(tp.type, Type::ENUM);         \
-    ASSERT_EQ(tp.ToString(), string(NAME)); \
-  }
-
-PRIMITIVE_TEST(Int8Type, INT8, "int8");
-PRIMITIVE_TEST(Int16Type, INT16, "int16");
-PRIMITIVE_TEST(Int32Type, INT32, "int32");
-PRIMITIVE_TEST(Int64Type, INT64, "int64");
-PRIMITIVE_TEST(UInt8Type, UINT8, "uint8");
-PRIMITIVE_TEST(UInt16Type, UINT16, "uint16");
-PRIMITIVE_TEST(UInt32Type, UINT32, "uint32");
-PRIMITIVE_TEST(UInt64Type, UINT64, "uint64");
-
-PRIMITIVE_TEST(FloatType, FLOAT, "float");
-PRIMITIVE_TEST(DoubleType, DOUBLE, "double");
-
-PRIMITIVE_TEST(BooleanType, BOOL, "bool");
-
-// ----------------------------------------------------------------------
-// Primitive type tests
-
-TEST_F(TestBuilder, TestReserve) {
-  builder_->Init(10);
-  ASSERT_EQ(2, builder_->null_bitmap()->size());
-
-  builder_->Reserve(30);
-  ASSERT_EQ(4, builder_->null_bitmap()->size());
-}
-
-template <typename Attrs>
-class TestPrimitiveBuilder : public TestBuilder {
- public:
-  typedef typename Attrs::ArrayType ArrayType;
-  typedef typename Attrs::BuilderType BuilderType;
-  typedef typename Attrs::T T;
-  typedef typename Attrs::Type Type;
-
-  virtual void SetUp() {
-    TestBuilder::SetUp();
-
-    type_ = Attrs::type();
-
-    std::shared_ptr<ArrayBuilder> tmp;
-    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-    builder_ = std::dynamic_pointer_cast<BuilderType>(tmp);
-
-    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-    builder_nn_ = std::dynamic_pointer_cast<BuilderType>(tmp);
-  }
-
-  void RandomData(int64_t N, double pct_null = 0.1) {
-    Attrs::draw(N, &draws_);
-
-    valid_bytes_.resize(static_cast<size_t>(N));
-    test::random_null_bytes(N, pct_null, valid_bytes_.data());
-  }
-
-  void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
-    int64_t size = builder->length();
-
-    auto ex_data = std::make_shared<Buffer>(
-        reinterpret_cast<uint8_t*>(draws_.data()), size * sizeof(T));
-
-    std::shared_ptr<Buffer> ex_null_bitmap;
-    int64_t ex_null_count = 0;
-
-    if (nullable) {
-      ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
-      ex_null_count = test::null_count(valid_bytes_);
-    } else {
-      ex_null_bitmap = nullptr;
-    }
-
-    auto expected =
-        std::make_shared<ArrayType>(size, ex_data, ex_null_bitmap, ex_null_count);
-
-    std::shared_ptr<Array> out;
-    ASSERT_OK(builder->Finish(&out));
-
-    std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(out);
-
-    // Builder is now reset
-    ASSERT_EQ(0, builder->length());
-    ASSERT_EQ(0, builder->capacity());
-    ASSERT_EQ(0, builder->null_count());
-    ASSERT_EQ(nullptr, builder->data());
-
-    ASSERT_EQ(ex_null_count, result->null_count());
-    ASSERT_TRUE(result->Equals(*expected));
-  }
-
- protected:
-  std::shared_ptr<DataType> type_;
-  shared_ptr<BuilderType> builder_;
-  shared_ptr<BuilderType> builder_nn_;
-
-  vector<T> draws_;
-  vector<uint8_t> valid_bytes_;
-};
-
-#define PTYPE_DECL(CapType, c_type)               \
-  typedef CapType##Array ArrayType;               \
-  typedef CapType##Builder BuilderType;           \
-  typedef CapType##Type Type;                     \
-  typedef c_type T;                               \
-                                                  \
-  static std::shared_ptr<DataType> type() {       \
-    return std::shared_ptr<DataType>(new Type()); \
-  }
-
-#define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
-  struct P##CapType {                               \
-    PTYPE_DECL(CapType, c_type);                    \
-    static void draw(int64_t N, vector<T>* draws) { \
-      test::randint<T>(N, LOWER, UPPER, draws);     \
-    }                                               \
-  }
-
-#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)     \
-  struct P##CapType {                                  \
-    PTYPE_DECL(CapType, c_type);                       \
-    static void draw(int64_t N, vector<T>* draws) {    \
-      test::random_real<T>(N, 0, LOWER, UPPER, draws); \
-    }                                                  \
-  }
-
-PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX);
-PINT_DECL(UInt16, uint16_t, 0, UINT16_MAX);
-PINT_DECL(UInt32, uint32_t, 0, UINT32_MAX);
-PINT_DECL(UInt64, uint64_t, 0, UINT64_MAX);
-
-PINT_DECL(Int8, int8_t, INT8_MIN, INT8_MAX);
-PINT_DECL(Int16, int16_t, INT16_MIN, INT16_MAX);
-PINT_DECL(Int32, int32_t, INT32_MIN, INT32_MAX);
-PINT_DECL(Int64, int64_t, INT64_MIN, INT64_MAX);
-
-PFLOAT_DECL(Float, float, -1000, 1000);
-PFLOAT_DECL(Double, double, -1000, 1000);
-
-struct PBoolean {
-  PTYPE_DECL(Boolean, uint8_t);
-};
-
-template <>
-void TestPrimitiveBuilder<PBoolean>::RandomData(int64_t N, double pct_null) {
-  draws_.resize(static_cast<size_t>(N));
-  valid_bytes_.resize(static_cast<size_t>(N));
-
-  test::random_null_bytes(N, 0.5, draws_.data());
-  test::random_null_bytes(N, pct_null, valid_bytes_.data());
-}
-
-template <>
-void TestPrimitiveBuilder<PBoolean>::Check(
-    const std::shared_ptr<BooleanBuilder>& builder, bool nullable) {
-  int64_t size = builder->length();
-
-  auto ex_data = test::bytes_to_null_buffer(draws_);
-
-  std::shared_ptr<Buffer> ex_null_bitmap;
-  int64_t ex_null_count = 0;
-
-  if (nullable) {
-    ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
-    ex_null_count = test::null_count(valid_bytes_);
-  } else {
-    ex_null_bitmap = nullptr;
-  }
-
-  auto expected =
-      std::make_shared<BooleanArray>(size, ex_data, ex_null_bitmap, ex_null_count);
-
-  std::shared_ptr<Array> out;
-  ASSERT_OK(builder->Finish(&out));
-  std::shared_ptr<BooleanArray> result = std::dynamic_pointer_cast<BooleanArray>(out);
-
-  // Builder is now reset
-  ASSERT_EQ(0, builder->length());
-  ASSERT_EQ(0, builder->capacity());
-  ASSERT_EQ(0, builder->null_count());
-  ASSERT_EQ(nullptr, builder->data());
-
-  ASSERT_EQ(ex_null_count, result->null_count());
-
-  ASSERT_EQ(expected->length(), result->length());
-
-  for (int64_t i = 0; i < result->length(); ++i) {
-    if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
-    bool actual = BitUtil::GetBit(result->data()->data(), i);
-    ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
-  }
-  ASSERT_TRUE(result->Equals(*expected));
-}
-
-typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
-    PInt32, PInt64, PFloat, PDouble>
-    Primitives;
-
-TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
-
-#define DECL_T() typedef typename TestFixture::T T;
-
-#define DECL_TYPE() typedef typename TestFixture::Type Type;
-
-#define DECL_ARRAYTYPE() typedef typename TestFixture::ArrayType ArrayType;
-
-TYPED_TEST(TestPrimitiveBuilder, TestInit) {
-  DECL_TYPE();
-
-  int64_t n = 1000;
-  ASSERT_OK(this->builder_->Reserve(n));
-  ASSERT_EQ(BitUtil::NextPower2(n), this->builder_->capacity());
-  ASSERT_EQ(BitUtil::NextPower2(TypeTraits<Type>::bytes_required(n)),
-      this->builder_->data()->size());
-
-  // unsure if this should go in all builder classes
-  ASSERT_EQ(0, this->builder_->num_children());
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
-  int64_t size = 1000;
-  for (int64_t i = 0; i < size; ++i) {
-    ASSERT_OK(this->builder_->AppendNull());
-  }
-
-  std::shared_ptr<Array> result;
-  ASSERT_OK(this->builder_->Finish(&result));
-
-  for (int64_t i = 0; i < size; ++i) {
-    ASSERT_TRUE(result->IsNull(i)) << i;
-  }
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
-  DECL_T();
-
-  int64_t size = 1000;
-
-  vector<T>& draws = this->draws_;
-  vector<uint8_t>& valid_bytes = this->valid_bytes_;
-
-  int64_t memory_before = this->pool_->bytes_allocated();
-
-  this->RandomData(size);
-
-  this->builder_->Reserve(size);
-
-  int64_t i;
-  for (i = 0; i < size; ++i) {
-    if (valid_bytes[i] > 0) {
-      this->builder_->Append(draws[i]);
-    } else {
-      this->builder_->AppendNull();
-    }
-  }
-
-  do {
-    std::shared_ptr<Array> result;
-    ASSERT_OK(this->builder_->Finish(&result));
-  } while (false);
-
-  ASSERT_EQ(memory_before, this->pool_->bytes_allocated());
-}
-
-TYPED_TEST(TestPrimitiveBuilder, Equality) {
-  DECL_T();
-
-  const int64_t size = 1000;
-  this->RandomData(size);
-  vector<T>& draws = this->draws_;
-  vector<uint8_t>& valid_bytes = this->valid_bytes_;
-  std::shared_ptr<Array> array, equal_array, unequal_array;
-  auto builder = this->builder_.get();
-  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
-  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &equal_array));
-
-  // Make the not equal array by negating the first valid element with itself.
-  const auto first_valid = std::find_if(
-      valid_bytes.begin(), valid_bytes.end(), [](uint8_t valid) { return valid > 0; });
-  const int64_t first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
-  // This should be true with a very high probability, but might introduce flakiness
-  ASSERT_LT(first_valid_idx, size - 1);
-  draws[first_valid_idx] =
-      static_cast<T>(~*reinterpret_cast<int64_t*>(&draws[first_valid_idx]));
-  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &unequal_array));
-
-  // test normal equality
-  EXPECT_TRUE(array->Equals(array));
-  EXPECT_TRUE(array->Equals(equal_array));
-  EXPECT_TRUE(equal_array->Equals(array));
-  EXPECT_FALSE(equal_array->Equals(unequal_array));
-  EXPECT_FALSE(unequal_array->Equals(equal_array));
-
-  // Test range equality
-  EXPECT_FALSE(array->RangeEquals(0, first_valid_idx + 1, 0, unequal_array));
-  EXPECT_FALSE(array->RangeEquals(first_valid_idx, size, first_valid_idx, unequal_array));
-  EXPECT_TRUE(array->RangeEquals(0, first_valid_idx, 0, unequal_array));
-  EXPECT_TRUE(
-      array->RangeEquals(first_valid_idx + 1, size, first_valid_idx + 1, unequal_array));
-}
-
-TYPED_TEST(TestPrimitiveBuilder, SliceEquality) {
-  DECL_T();
-
-  const int64_t size = 1000;
-  this->RandomData(size);
-  vector<T>& draws = this->draws_;
-  vector<uint8_t>& valid_bytes = this->valid_bytes_;
-  auto builder = this->builder_.get();
-
-  std::shared_ptr<Array> array;
-  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
-
-  std::shared_ptr<Array> slice, slice2;
-
-  slice = array->Slice(5);
-  slice2 = array->Slice(5);
-  ASSERT_EQ(size - 5, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(5, array->length(), 0, slice));
-
-  // Chained slices
-  slice2 = array->Slice(2)->Slice(3);
-  ASSERT_TRUE(slice->Equals(slice2));
-
-  slice = array->Slice(5, 10);
-  slice2 = array->Slice(5, 10);
-  ASSERT_EQ(10, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(5, 15, 0, slice));
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
-  DECL_T();
-
-  const int64_t size = 10000;
-
-  vector<T>& draws = this->draws_;
-  vector<uint8_t>& valid_bytes = this->valid_bytes_;
-
-  this->RandomData(size);
-
-  this->builder_->Reserve(1000);
-  this->builder_nn_->Reserve(1000);
-
-  int64_t null_count = 0;
-  // Append the first 1000
-  for (size_t i = 0; i < 1000; ++i) {
-    if (valid_bytes[i] > 0) {
-      this->builder_->Append(draws[i]);
-    } else {
-      this->builder_->AppendNull();
-      ++null_count;
-    }
-    this->builder_nn_->Append(draws[i]);
-  }
-
-  ASSERT_EQ(null_count, this->builder_->null_count());
-
-  ASSERT_EQ(1000, this->builder_->length());
-  ASSERT_EQ(1024, this->builder_->capacity());
-
-  ASSERT_EQ(1000, this->builder_nn_->length());
-  ASSERT_EQ(1024, this->builder_nn_->capacity());
-
-  this->builder_->Reserve(size - 1000);
-  this->builder_nn_->Reserve(size - 1000);
-
-  // Append the next 9000
-  for (size_t i = 1000; i < size; ++i) {
-    if (valid_bytes[i] > 0) {
-      this->builder_->Append(draws[i]);
-    } else {
-      this->builder_->AppendNull();
-    }
-    this->builder_nn_->Append(draws[i]);
-  }
-
-  ASSERT_EQ(size, this->builder_->length());
-  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
-
-  ASSERT_EQ(size, this->builder_nn_->length());
-  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_nn_->capacity());
-
-  this->Check(this->builder_, true);
-  this->Check(this->builder_nn_, false);
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
-  DECL_T();
-
-  int64_t size = 10000;
-  this->RandomData(size);
-
-  vector<T>& draws = this->draws_;
-  vector<uint8_t>& valid_bytes = this->valid_bytes_;
-
-  // first slug
-  int64_t K = 1000;
-
-  ASSERT_OK(this->builder_->Append(draws.data(), K, valid_bytes.data()));
-  ASSERT_OK(this->builder_nn_->Append(draws.data(), K));
-
-  ASSERT_EQ(1000, this->builder_->length());
-  ASSERT_EQ(1024, this->builder_->capacity());
-
-  ASSERT_EQ(1000, this->builder_nn_->length());
-  ASSERT_EQ(1024, this->builder_nn_->capacity());
-
-  // Append the next 9000
-  ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, valid_bytes.data() + K));
-  ASSERT_OK(this->builder_nn_->Append(draws.data() + K, size - K));
-
-  ASSERT_EQ(size, this->builder_->length());
-  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
-
-  this->Check(this->builder_, true);
-  this->Check(this->builder_nn_, false);
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
-  int64_t n = 1000;
-  ASSERT_OK(this->builder_->Reserve(n));
-
-  ASSERT_OK(this->builder_->Advance(100));
-  ASSERT_EQ(100, this->builder_->length());
-
-  ASSERT_OK(this->builder_->Advance(900));
-
-  int64_t too_many = this->builder_->capacity() - 1000 + 1;
-  ASSERT_RAISES(Invalid, this->builder_->Advance(too_many));
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestResize) {
-  DECL_TYPE();
-
-  int64_t cap = kMinBuilderCapacity * 2;
-
-  ASSERT_OK(this->builder_->Reserve(cap));
-  ASSERT_EQ(cap, this->builder_->capacity());
-
-  ASSERT_EQ(TypeTraits<Type>::bytes_required(cap), this->builder_->data()->size());
-  ASSERT_EQ(BitUtil::BytesForBits(cap), this->builder_->null_bitmap()->size());
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
-  ASSERT_OK(this->builder_->Reserve(10));
-  ASSERT_EQ(0, this->builder_->length());
-  ASSERT_EQ(kMinBuilderCapacity, this->builder_->capacity());
-
-  ASSERT_OK(this->builder_->Reserve(90));
-  ASSERT_OK(this->builder_->Advance(100));
-  ASSERT_OK(this->builder_->Reserve(kMinBuilderCapacity));
-
-  ASSERT_EQ(BitUtil::NextPower2(kMinBuilderCapacity + 100), this->builder_->capacity());
-}
-
-template <typename TYPE>
-void CheckSliceApproxEquals() {
-  using T = typename TYPE::c_type;
-
-  const int64_t kSize = 50;
-  std::vector<T> draws1;
-  std::vector<T> draws2;
-
-  const uint32_t kSeed = 0;
-  test::random_real<T>(kSize, kSeed, 0, 100, &draws1);
-  test::random_real<T>(kSize, kSeed + 1, 0, 100, &draws2);
-
-  // Make the draws equal in the sliced segment, but unequal elsewhere (to
-  // catch not using the slice offset)
-  for (int64_t i = 10; i < 30; ++i) {
-    draws2[i] = draws1[i];
-  }
-
-  std::vector<bool> is_valid;
-  test::random_is_valid(kSize, 0.1, &is_valid);
-
-  std::shared_ptr<Array> array1, array2;
-  ArrayFromVector<TYPE, T>(is_valid, draws1, &array1);
-  ArrayFromVector<TYPE, T>(is_valid, draws2, &array2);
-
-  std::shared_ptr<Array> slice1 = array1->Slice(10, 20);
-  std::shared_ptr<Array> slice2 = array2->Slice(10, 20);
-
-  ASSERT_TRUE(slice1->ApproxEquals(slice2));
-}
-
-TEST(TestPrimitiveAdHoc, FloatingSliceApproxEquals) {
-  CheckSliceApproxEquals<FloatType>();
-  CheckSliceApproxEquals<DoubleType>();
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
deleted file mode 100644
index 6c2c1516c8f3c..0000000000000
--- a/cpp/src/arrow/array-string-test.cc
+++ /dev/null
@@ -1,654 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdlib>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/test-common.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/type_traits.h"
-
-namespace arrow {
-
-class Buffer;
-
-// ----------------------------------------------------------------------
-// String container
-
-class TestStringArray : public ::testing::Test {
- public:
-  void SetUp() {
-    chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
-    offsets_ = {0, 1, 1, 1, 3, 6};
-    valid_bytes_ = {1, 1, 0, 1, 1};
-    expected_ = {"a", "", "", "bb", "ccc"};
-
-    MakeArray();
-  }
-
-  void MakeArray() {
-    length_ = static_cast<int64_t>(offsets_.size()) - 1;
-    value_buf_ = test::GetBufferFromVector(chars_);
-    offsets_buf_ = test::GetBufferFromVector(offsets_);
-    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
-    null_count_ = test::null_count(valid_bytes_);
-
-    strings_ = std::make_shared<StringArray>(
-        length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
-  }
-
- protected:
-  std::vector<int32_t> offsets_;
-  std::vector<char> chars_;
-  std::vector<uint8_t> valid_bytes_;
-
-  std::vector<std::string> expected_;
-
-  std::shared_ptr<Buffer> value_buf_;
-  std::shared_ptr<Buffer> offsets_buf_;
-  std::shared_ptr<Buffer> null_bitmap_;
-
-  int64_t null_count_;
-  int64_t length_;
-
-  std::shared_ptr<StringArray> strings_;
-};
-
-TEST_F(TestStringArray, TestArrayBasics) {
-  ASSERT_EQ(length_, strings_->length());
-  ASSERT_EQ(1, strings_->null_count());
-  ASSERT_OK(strings_->Validate());
-}
-
-TEST_F(TestStringArray, TestType) {
-  TypePtr type = strings_->type();
-
-  ASSERT_EQ(Type::STRING, type->type);
-  ASSERT_EQ(Type::STRING, strings_->type_enum());
-}
-
-TEST_F(TestStringArray, TestListFunctions) {
-  int pos = 0;
-  for (size_t i = 0; i < expected_.size(); ++i) {
-    ASSERT_EQ(pos, strings_->value_offset(i));
-    ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
-    pos += static_cast<int>(expected_[i].size());
-  }
-}
-
-TEST_F(TestStringArray, TestDestructor) {
-  auto arr = std::make_shared<StringArray>(
-      length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
-}
-
-TEST_F(TestStringArray, TestGetString) {
-  for (size_t i = 0; i < expected_.size(); ++i) {
-    if (valid_bytes_[i] == 0) {
-      ASSERT_TRUE(strings_->IsNull(i));
-    } else {
-      ASSERT_EQ(expected_[i], strings_->GetString(i));
-    }
-  }
-}
-
-TEST_F(TestStringArray, TestEmptyStringComparison) {
-  offsets_ = {0, 0, 0, 0, 0, 0};
-  offsets_buf_ = test::GetBufferFromVector(offsets_);
-  length_ = static_cast<int64_t>(offsets_.size() - 1);
-
-  auto strings_a = std::make_shared<StringArray>(
-      length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
-  auto strings_b = std::make_shared<StringArray>(
-      length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
-  ASSERT_TRUE(strings_a->Equals(strings_b));
-}
-
-TEST_F(TestStringArray, CompareNullByteSlots) {
-  StringBuilder builder(default_memory_pool());
-  StringBuilder builder2(default_memory_pool());
-  StringBuilder builder3(default_memory_pool());
-
-  builder.Append("foo");
-  builder2.Append("foo");
-  builder3.Append("foo");
-
-  builder.Append("bar");
-  builder2.AppendNull();
-
-  // same length, but different
-  builder3.Append("xyz");
-
-  builder.Append("baz");
-  builder2.Append("baz");
-  builder3.Append("baz");
-
-  std::shared_ptr<Array> array, array2, array3;
-  ASSERT_OK(builder.Finish(&array));
-  ASSERT_OK(builder2.Finish(&array2));
-  ASSERT_OK(builder3.Finish(&array3));
-
-  const auto& a1 = static_cast<const StringArray&>(*array);
-  const auto& a2 = static_cast<const StringArray&>(*array2);
-  const auto& a3 = static_cast<const StringArray&>(*array3);
-
-  // The validity bitmaps are the same, the data is different, but the unequal
-  // portion is masked out
-  StringArray equal_array(3, a1.value_offsets(), a1.data(), a2.null_bitmap(), 1);
-  StringArray equal_array2(3, a3.value_offsets(), a3.data(), a2.null_bitmap(), 1);
-
-  ASSERT_TRUE(equal_array.Equals(equal_array2));
-  ASSERT_TRUE(a2.RangeEquals(equal_array2, 0, 3, 0));
-
-  ASSERT_TRUE(equal_array.Array::Slice(1)->Equals(equal_array2.Array::Slice(1)));
-  ASSERT_TRUE(
-      equal_array.Array::Slice(1)->RangeEquals(0, 2, 0, equal_array2.Array::Slice(1)));
-}
-
-TEST_F(TestStringArray, TestSliceGetString) {
-  StringBuilder builder(default_memory_pool());
-
-  builder.Append("a");
-  builder.Append("b");
-  builder.Append("c");
-
-  std::shared_ptr<Array> array;
-  ASSERT_OK(builder.Finish(&array));
-  auto s = array->Slice(1, 10);
-  auto arr = std::dynamic_pointer_cast<StringArray>(s);
-  ASSERT_EQ(arr->GetString(0), "b");
-}
-
-// ----------------------------------------------------------------------
-// String builder tests
-
-class TestStringBuilder : public TestBuilder {
- public:
-  void SetUp() {
-    TestBuilder::SetUp();
-    builder_.reset(new StringBuilder(pool_));
-  }
-
-  void Done() {
-    std::shared_ptr<Array> out;
-    EXPECT_OK(builder_->Finish(&out));
-
-    result_ = std::dynamic_pointer_cast<StringArray>(out);
-    result_->Validate();
-  }
-
- protected:
-  std::unique_ptr<StringBuilder> builder_;
-  std::shared_ptr<StringArray> result_;
-};
-
-TEST_F(TestStringBuilder, TestScalarAppend) {
-  std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
-  std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
-
-  int N = static_cast<int>(strings.size());
-  int reps = 1000;
-
-  for (int j = 0; j < reps; ++j) {
-    for (int i = 0; i < N; ++i) {
-      if (is_null[i]) {
-        builder_->AppendNull();
-      } else {
-        builder_->Append(strings[i]);
-      }
-    }
-  }
-  Done();
-
-  ASSERT_EQ(reps * N, result_->length());
-  ASSERT_EQ(reps, result_->null_count());
-  ASSERT_EQ(reps * 6, result_->data()->size());
-
-  int32_t length;
-  int32_t pos = 0;
-  for (int i = 0; i < N * reps; ++i) {
-    if (is_null[i % N]) {
-      ASSERT_TRUE(result_->IsNull(i));
-    } else {
-      ASSERT_FALSE(result_->IsNull(i));
-      result_->GetValue(i, &length);
-      ASSERT_EQ(pos, result_->value_offset(i));
-      ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
-      ASSERT_EQ(strings[i % N], result_->GetString(i));
-
-      pos += length;
-    }
-  }
-}
-
-TEST_F(TestStringBuilder, TestZeroLength) {
-  // All buffers are null
-  Done();
-}
-
-// Binary container type
-// TODO(emkornfield) there should be some way to refactor these to avoid code duplicating
-// with String
-class TestBinaryArray : public ::testing::Test {
- public:
-  void SetUp() {
-    chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
-    offsets_ = {0, 1, 1, 1, 3, 6};
-    valid_bytes_ = {1, 1, 0, 1, 1};
-    expected_ = {"a", "", "", "bb", "ccc"};
-
-    MakeArray();
-  }
-
-  void MakeArray() {
-    length_ = static_cast<int64_t>(offsets_.size() - 1);
-    value_buf_ = test::GetBufferFromVector(chars_);
-    offsets_buf_ = test::GetBufferFromVector(offsets_);
-
-    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
-    null_count_ = test::null_count(valid_bytes_);
-
-    strings_ = std::make_shared<BinaryArray>(
-        length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
-  }
-
- protected:
-  std::vector<int32_t> offsets_;
-  std::vector<char> chars_;
-  std::vector<uint8_t> valid_bytes_;
-
-  std::vector<std::string> expected_;
-
-  std::shared_ptr<Buffer> value_buf_;
-  std::shared_ptr<Buffer> offsets_buf_;
-  std::shared_ptr<Buffer> null_bitmap_;
-
-  int64_t null_count_;
-  int64_t length_;
-
-  std::shared_ptr<BinaryArray> strings_;
-};
-
-TEST_F(TestBinaryArray, TestArrayBasics) {
-  ASSERT_EQ(length_, strings_->length());
-  ASSERT_EQ(1, strings_->null_count());
-  ASSERT_OK(strings_->Validate());
-}
-
-TEST_F(TestBinaryArray, TestType) {
-  TypePtr type = strings_->type();
-
-  ASSERT_EQ(Type::BINARY, type->type);
-  ASSERT_EQ(Type::BINARY, strings_->type_enum());
-}
-
-TEST_F(TestBinaryArray, TestListFunctions) {
-  size_t pos = 0;
-  for (size_t i = 0; i < expected_.size(); ++i) {
-    ASSERT_EQ(pos, strings_->value_offset(i));
-    ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
-    pos += expected_[i].size();
-  }
-}
-
-TEST_F(TestBinaryArray, TestDestructor) {
-  auto arr = std::make_shared<BinaryArray>(
-      length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
-}
-
-TEST_F(TestBinaryArray, TestGetValue) {
-  for (size_t i = 0; i < expected_.size(); ++i) {
-    if (valid_bytes_[i] == 0) {
-      ASSERT_TRUE(strings_->IsNull(i));
-    } else {
-      int32_t len = -1;
-      const uint8_t* bytes = strings_->GetValue(i, &len);
-      ASSERT_EQ(0, std::memcmp(expected_[i].data(), bytes, len));
-    }
-  }
-}
-
-TEST_F(TestBinaryArray, TestEqualsEmptyStrings) {
-  BinaryBuilder builder(default_memory_pool(), arrow::binary());
-
-  std::string empty_string("");
-
-  builder.Append(empty_string);
-  builder.Append(empty_string);
-  builder.Append(empty_string);
-  builder.Append(empty_string);
-  builder.Append(empty_string);
-
-  std::shared_ptr<Array> left_arr;
-  ASSERT_OK(builder.Finish(&left_arr));
-
-  const BinaryArray& left = static_cast<const BinaryArray&>(*left_arr);
-  std::shared_ptr<Array> right = std::make_shared<BinaryArray>(left.length(),
-      left.value_offsets(), nullptr, left.null_bitmap(), left.null_count());
-
-  ASSERT_TRUE(left.Equals(right));
-  ASSERT_TRUE(left.RangeEquals(0, left.length(), 0, right));
-}
-
-class TestBinaryBuilder : public TestBuilder {
- public:
-  void SetUp() {
-    TestBuilder::SetUp();
-    builder_.reset(new BinaryBuilder(pool_));
-  }
-
-  void Done() {
-    std::shared_ptr<Array> out;
-    EXPECT_OK(builder_->Finish(&out));
-
-    result_ = std::dynamic_pointer_cast<BinaryArray>(out);
-    result_->Validate();
-  }
-
- protected:
-  std::unique_ptr<BinaryBuilder> builder_;
-  std::shared_ptr<BinaryArray> result_;
-};
-
-TEST_F(TestBinaryBuilder, TestScalarAppend) {
-  std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
-  std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
-
-  int N = static_cast<int>(strings.size());
-  int reps = 1000;
-
-  for (int j = 0; j < reps; ++j) {
-    for (int i = 0; i < N; ++i) {
-      if (is_null[i]) {
-        builder_->AppendNull();
-      } else {
-        builder_->Append(strings[i]);
-      }
-    }
-  }
-  Done();
-  ASSERT_OK(result_->Validate());
-  ASSERT_EQ(reps * N, result_->length());
-  ASSERT_EQ(reps, result_->null_count());
-  ASSERT_EQ(reps * 6, result_->data()->size());
-
-  int32_t length;
-  for (int i = 0; i < N * reps; ++i) {
-    if (is_null[i % N]) {
-      ASSERT_TRUE(result_->IsNull(i));
-    } else {
-      ASSERT_FALSE(result_->IsNull(i));
-      const uint8_t* vals = result_->GetValue(i, &length);
-      ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
-      ASSERT_EQ(0, std::memcmp(vals, strings[i % N].data(), length));
-    }
-  }
-}
-
-TEST_F(TestBinaryBuilder, TestZeroLength) {
-  // All buffers are null
-  Done();
-}
-
-// ----------------------------------------------------------------------
-// Slice tests
-
-template <typename TYPE>
-void CheckSliceEquality() {
-  using Traits = TypeTraits<TYPE>;
-  using BuilderType = typename Traits::BuilderType;
-
-  BuilderType builder(default_memory_pool());
-
-  std::vector<std::string> strings = {"foo", "", "bar", "baz", "qux", ""};
-  std::vector<uint8_t> is_null = {0, 1, 0, 1, 0, 0};
-
-  int N = static_cast<int>(strings.size());
-  int reps = 10;
-
-  for (int j = 0; j < reps; ++j) {
-    for (int i = 0; i < N; ++i) {
-      if (is_null[i]) {
-        builder.AppendNull();
-      } else {
-        builder.Append(strings[i]);
-      }
-    }
-  }
-
-  std::shared_ptr<Array> array;
-  ASSERT_OK(builder.Finish(&array));
-
-  std::shared_ptr<Array> slice, slice2;
-
-  slice = array->Slice(5);
-  slice2 = array->Slice(5);
-  ASSERT_EQ(N * reps - 5, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(5, slice->length(), 0, slice));
-
-  // Chained slices
-  slice2 = array->Slice(2)->Slice(3);
-  ASSERT_TRUE(slice->Equals(slice2));
-
-  slice = array->Slice(5, 20);
-  slice2 = array->Slice(5, 20);
-  ASSERT_EQ(20, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(5, 25, 0, slice));
-}
-
-TEST_F(TestBinaryArray, TestSliceEquality) {
-  CheckSliceEquality<BinaryType>();
-}
-
-TEST_F(TestStringArray, TestSliceEquality) {
-  CheckSliceEquality<BinaryType>();
-}
-
-TEST_F(TestBinaryArray, LengthZeroCtor) {
-  BinaryArray array(0, nullptr, nullptr);
-}
-
-// ----------------------------------------------------------------------
-// FixedWidthBinary tests
-
-class TestFWBinaryArray : public ::testing::Test {
- public:
-  void SetUp() {}
-
-  void InitBuilder(int byte_width) {
-    auto type = fixed_width_binary(byte_width);
-    builder_.reset(new FixedWidthBinaryBuilder(default_memory_pool(), type));
-  }
-
- protected:
-  std::unique_ptr<FixedWidthBinaryBuilder> builder_;
-};
-
-TEST_F(TestFWBinaryArray, Builder) {
-  const int32_t byte_width = 10;
-  int64_t length = 4096;
-
-  int64_t nbytes = length * byte_width;
-
-  std::vector<uint8_t> data(nbytes);
-  test::random_bytes(nbytes, 0, data.data());
-
-  std::vector<uint8_t> is_valid(length);
-  test::random_null_bytes(length, 0.1, is_valid.data());
-
-  const uint8_t* raw_data = data.data();
-
-  std::shared_ptr<Array> result;
-
-  auto CheckResult = [this, &length, &is_valid, &raw_data, &byte_width](
-      const Array& result) {
-    // Verify output
-    const auto& fw_result = static_cast<const FixedWidthBinaryArray&>(result);
-
-    ASSERT_EQ(length, result.length());
-
-    for (int64_t i = 0; i < result.length(); ++i) {
-      if (is_valid[i]) {
-        ASSERT_EQ(
-            0, memcmp(raw_data + byte_width * i, fw_result.GetValue(i), byte_width));
-      } else {
-        ASSERT_TRUE(fw_result.IsNull(i));
-      }
-    }
-  };
-
-  // Build using iterative API
-  InitBuilder(byte_width);
-  for (int64_t i = 0; i < length; ++i) {
-    if (is_valid[i]) {
-      builder_->Append(raw_data + byte_width * i);
-    } else {
-      builder_->AppendNull();
-    }
-  }
-
-  ASSERT_OK(builder_->Finish(&result));
-  CheckResult(*result);
-
-  // Build using batch API
-  InitBuilder(byte_width);
-
-  const uint8_t* raw_is_valid = is_valid.data();
-
-  ASSERT_OK(builder_->Append(raw_data, 50, raw_is_valid));
-  ASSERT_OK(builder_->Append(raw_data + 50 * byte_width, length - 50, raw_is_valid + 50));
-  ASSERT_OK(builder_->Finish(&result));
-  CheckResult(*result);
-
-  // Build from std::string
-  InitBuilder(byte_width);
-  for (int64_t i = 0; i < length; ++i) {
-    if (is_valid[i]) {
-      builder_->Append(std::string(
-          reinterpret_cast<const char*>(raw_data + byte_width * i), byte_width));
-    } else {
-      builder_->AppendNull();
-    }
-  }
-
-  ASSERT_OK(builder_->Finish(&result));
-  CheckResult(*result);
-}
-
-TEST_F(TestFWBinaryArray, EqualsRangeEquals) {
-  // Check that we don't compare data in null slots
-
-  auto type = fixed_width_binary(4);
-  FixedWidthBinaryBuilder builder1(default_memory_pool(), type);
-  FixedWidthBinaryBuilder builder2(default_memory_pool(), type);
-
-  ASSERT_OK(builder1.Append("foo1"));
-  ASSERT_OK(builder1.AppendNull());
-
-  ASSERT_OK(builder2.Append("foo1"));
-  ASSERT_OK(builder2.Append("foo2"));
-
-  std::shared_ptr<Array> array1, array2;
-  ASSERT_OK(builder1.Finish(&array1));
-  ASSERT_OK(builder2.Finish(&array2));
-
-  const auto& a1 = static_cast<const FixedWidthBinaryArray&>(*array1);
-  const auto& a2 = static_cast<const FixedWidthBinaryArray&>(*array2);
-
-  FixedWidthBinaryArray equal1(type, 2, a1.data(), a1.null_bitmap(), 1);
-  FixedWidthBinaryArray equal2(type, 2, a2.data(), a1.null_bitmap(), 1);
-
-  ASSERT_TRUE(equal1.Equals(equal2));
-  ASSERT_TRUE(equal1.RangeEquals(equal2, 0, 2, 0));
-}
-
-TEST_F(TestFWBinaryArray, ZeroSize) {
-  auto type = fixed_width_binary(0);
-  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
-
-  ASSERT_OK(builder.Append(nullptr));
-  ASSERT_OK(builder.Append(nullptr));
-  ASSERT_OK(builder.Append(nullptr));
-  ASSERT_OK(builder.AppendNull());
-  ASSERT_OK(builder.AppendNull());
-  ASSERT_OK(builder.AppendNull());
-
-  std::shared_ptr<Array> array;
-  ASSERT_OK(builder.Finish(&array));
-
-  const auto& fw_array = static_cast<const FixedWidthBinaryArray&>(*array);
-
-  // data is never allocated
-  ASSERT_TRUE(fw_array.data() == nullptr);
-  ASSERT_EQ(0, fw_array.byte_width());
-
-  ASSERT_EQ(6, array->length());
-  ASSERT_EQ(3, array->null_count());
-}
-
-TEST_F(TestFWBinaryArray, Slice) {
-  auto type = fixed_width_binary(4);
-  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
-
-  std::vector<std::string> strings = {"foo1", "foo2", "foo3", "foo4", "foo5"};
-  std::vector<uint8_t> is_null = {0, 1, 0, 0, 0};
-
-  for (int i = 0; i < 5; ++i) {
-    if (is_null[i]) {
-      builder.AppendNull();
-    } else {
-      builder.Append(strings[i]);
-    }
-  }
-
-  std::shared_ptr<Array> array;
-  ASSERT_OK(builder.Finish(&array));
-
-  std::shared_ptr<Array> slice, slice2;
-
-  slice = array->Slice(1);
-  slice2 = array->Slice(1);
-  ASSERT_EQ(4, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(1, slice->length(), 0, slice));
-
-  // Chained slices
-  slice = array->Slice(2);
-  slice2 = array->Slice(1)->Slice(1);
-  ASSERT_TRUE(slice->Equals(slice2));
-
-  slice = array->Slice(1, 3);
-  ASSERT_EQ(3, slice->length());
-
-  slice2 = array->Slice(1, 3);
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(1, 3, 0, slice));
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/array-struct-test.cc b/cpp/src/arrow/array-struct-test.cc
deleted file mode 100644
index 4eb1eab13fbc6..0000000000000
--- a/cpp/src/arrow/array-struct-test.cc
+++ /dev/null
@@ -1,410 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/status.h"
-#include "arrow/test-common.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-namespace arrow {
-
-TEST(TestStructType, Basics) {
-  TypePtr f0_type = TypePtr(new Int32Type());
-  auto f0 = std::make_shared<Field>("f0", f0_type);
-
-  TypePtr f1_type = TypePtr(new StringType());
-  auto f1 = std::make_shared<Field>("f1", f1_type);
-
-  TypePtr f2_type = TypePtr(new UInt8Type());
-  auto f2 = std::make_shared<Field>("f2", f2_type);
-
-  vector<shared_ptr<Field>> fields = {f0, f1, f2};
-
-  StructType struct_type(fields);
-
-  ASSERT_TRUE(struct_type.child(0)->Equals(f0));
-  ASSERT_TRUE(struct_type.child(1)->Equals(f1));
-  ASSERT_TRUE(struct_type.child(2)->Equals(f2));
-
-  ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
-
-  // TODO(wesm): out of bounds for field(...)
-}
-
-void ValidateBasicStructArray(const StructArray* result,
-    const vector<uint8_t>& struct_is_valid, const vector<char>& list_values,
-    const vector<uint8_t>& list_is_valid, const vector<int>& list_lengths,
-    const vector<int>& list_offsets, const vector<int32_t>& int_values) {
-  ASSERT_EQ(4, result->length());
-  ASSERT_OK(result->Validate());
-
-  auto list_char_arr = static_cast<ListArray*>(result->field(0).get());
-  auto char_arr = static_cast<Int8Array*>(list_char_arr->values().get());
-  auto int32_arr = static_cast<Int32Array*>(result->field(1).get());
-
-  ASSERT_EQ(0, result->null_count());
-  ASSERT_EQ(1, list_char_arr->null_count());
-  ASSERT_EQ(0, int32_arr->null_count());
-
-  // List<char>
-  ASSERT_EQ(4, list_char_arr->length());
-  ASSERT_EQ(10, list_char_arr->values()->length());
-  for (size_t i = 0; i < list_offsets.size(); ++i) {
-    ASSERT_EQ(list_offsets[i], list_char_arr->raw_value_offsets()[i]);
-  }
-  for (size_t i = 0; i < list_values.size(); ++i) {
-    ASSERT_EQ(list_values[i], char_arr->Value(i));
-  }
-
-  // Int32
-  ASSERT_EQ(4, int32_arr->length());
-  for (size_t i = 0; i < int_values.size(); ++i) {
-    ASSERT_EQ(int_values[i], int32_arr->Value(i));
-  }
-}
-
-// ----------------------------------------------------------------------------------
-// Struct test
-class TestStructBuilder : public TestBuilder {
- public:
-  void SetUp() {
-    TestBuilder::SetUp();
-
-    auto int32_type = TypePtr(new Int32Type());
-    auto char_type = TypePtr(new Int8Type());
-    auto list_type = TypePtr(new ListType(char_type));
-
-    std::vector<TypePtr> types = {list_type, int32_type};
-    std::vector<FieldPtr> fields;
-    fields.push_back(FieldPtr(new Field("list", list_type)));
-    fields.push_back(FieldPtr(new Field("int", int32_type)));
-
-    type_ = TypePtr(new StructType(fields));
-    value_fields_ = fields;
-
-    std::shared_ptr<ArrayBuilder> tmp;
-    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-
-    builder_ = std::dynamic_pointer_cast<StructBuilder>(tmp);
-    ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
-  }
-
-  void Done() {
-    std::shared_ptr<Array> out;
-    ASSERT_OK(builder_->Finish(&out));
-    result_ = std::dynamic_pointer_cast<StructArray>(out);
-  }
-
- protected:
-  std::vector<FieldPtr> value_fields_;
-  TypePtr type_;
-
-  std::shared_ptr<StructBuilder> builder_;
-  std::shared_ptr<StructArray> result_;
-};
-
-TEST_F(TestStructBuilder, TestAppendNull) {
-  ASSERT_OK(builder_->AppendNull());
-  ASSERT_OK(builder_->AppendNull());
-  ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
-
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  ASSERT_OK(list_vb->AppendNull());
-  ASSERT_OK(list_vb->AppendNull());
-  ASSERT_EQ(2, list_vb->length());
-
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-  ASSERT_OK(int_vb->AppendNull());
-  ASSERT_OK(int_vb->AppendNull());
-  ASSERT_EQ(2, int_vb->length());
-
-  Done();
-
-  ASSERT_OK(result_->Validate());
-
-  ASSERT_EQ(2, static_cast<int>(result_->fields().size()));
-  ASSERT_EQ(2, result_->length());
-  ASSERT_EQ(2, result_->field(0)->length());
-  ASSERT_EQ(2, result_->field(1)->length());
-  ASSERT_TRUE(result_->IsNull(0));
-  ASSERT_TRUE(result_->IsNull(1));
-  ASSERT_TRUE(result_->field(0)->IsNull(0));
-  ASSERT_TRUE(result_->field(0)->IsNull(1));
-  ASSERT_TRUE(result_->field(1)->IsNull(0));
-  ASSERT_TRUE(result_->field(1)->IsNull(1));
-
-  ASSERT_EQ(Type::LIST, result_->field(0)->type_enum());
-  ASSERT_EQ(Type::INT32, result_->field(1)->type_enum());
-}
-
-TEST_F(TestStructBuilder, TestBasics) {
-  vector<int32_t> int_values = {1, 2, 3, 4};
-  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
-  vector<int> list_lengths = {3, 0, 3, 4};
-  vector<int> list_offsets = {0, 3, 3, 6, 10};
-  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
-  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
-
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-  ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
-
-  EXPECT_OK(builder_->Resize(list_lengths.size()));
-  EXPECT_OK(char_vb->Resize(list_values.size()));
-  EXPECT_OK(int_vb->Resize(int_values.size()));
-
-  int pos = 0;
-  for (size_t i = 0; i < list_lengths.size(); ++i) {
-    ASSERT_OK(list_vb->Append(list_is_valid[i] > 0));
-    int_vb->UnsafeAppend(int_values[i]);
-    for (int j = 0; j < list_lengths[i]; ++j) {
-      char_vb->UnsafeAppend(list_values[pos++]);
-    }
-  }
-
-  for (size_t i = 0; i < struct_is_valid.size(); ++i) {
-    ASSERT_OK(builder_->Append(struct_is_valid[i] > 0));
-  }
-
-  Done();
-
-  ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
-      list_lengths, list_offsets, int_values);
-}
-
-TEST_F(TestStructBuilder, BulkAppend) {
-  vector<int32_t> int_values = {1, 2, 3, 4};
-  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
-  vector<int> list_lengths = {3, 0, 3, 4};
-  vector<int> list_offsets = {0, 3, 3, 6};
-  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
-  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
-
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-
-  ASSERT_OK(builder_->Resize(list_lengths.size()));
-  ASSERT_OK(char_vb->Resize(list_values.size()));
-  ASSERT_OK(int_vb->Resize(int_values.size()));
-
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
-  for (int8_t value : list_values) {
-    char_vb->UnsafeAppend(value);
-  }
-  for (int32_t value : int_values) {
-    int_vb->UnsafeAppend(value);
-  }
-
-  Done();
-  ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
-      list_lengths, list_offsets, int_values);
-}
-
-TEST_F(TestStructBuilder, BulkAppendInvalid) {
-  vector<int32_t> int_values = {1, 2, 3, 4};
-  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
-  vector<int> list_lengths = {3, 0, 3, 4};
-  vector<int> list_offsets = {0, 3, 3, 6};
-  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
-  vector<uint8_t> struct_is_valid = {1, 0, 1, 1};  // should be 1, 1, 1, 1
-
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-
-  ASSERT_OK(builder_->Reserve(list_lengths.size()));
-  ASSERT_OK(char_vb->Reserve(list_values.size()));
-  ASSERT_OK(int_vb->Reserve(int_values.size()));
-
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
-  for (int8_t value : list_values) {
-    char_vb->UnsafeAppend(value);
-  }
-  for (int32_t value : int_values) {
-    int_vb->UnsafeAppend(value);
-  }
-
-  Done();
-  // Even null bitmap of the parent Struct is not valid, Validate() will ignore it.
-  ASSERT_OK(result_->Validate());
-}
-
-TEST_F(TestStructBuilder, TestEquality) {
-  std::shared_ptr<Array> array, equal_array;
-  std::shared_ptr<Array> unequal_bitmap_array, unequal_offsets_array,
-      unequal_values_array;
-
-  vector<int32_t> int_values = {1, 2, 3, 4};
-  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
-  vector<int> list_lengths = {3, 0, 3, 4};
-  vector<int> list_offsets = {0, 3, 3, 6};
-  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
-  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
-
-  vector<int32_t> unequal_int_values = {4, 2, 3, 1};
-  vector<char> unequal_list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'l', 'u', 'c', 'y'};
-  vector<int> unequal_list_offsets = {0, 3, 4, 6};
-  vector<uint8_t> unequal_list_is_valid = {1, 1, 1, 1};
-  vector<uint8_t> unequal_struct_is_valid = {1, 0, 0, 1};
-
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-  ASSERT_OK(builder_->Reserve(list_lengths.size()));
-  ASSERT_OK(char_vb->Reserve(list_values.size()));
-  ASSERT_OK(int_vb->Reserve(int_values.size()));
-
-  // setup two equal arrays, one of which takes an unequal bitmap
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
-  for (int8_t value : list_values) {
-    char_vb->UnsafeAppend(value);
-  }
-  for (int32_t value : int_values) {
-    int_vb->UnsafeAppend(value);
-  }
-
-  ASSERT_OK(builder_->Finish(&array));
-
-  ASSERT_OK(builder_->Resize(list_lengths.size()));
-  ASSERT_OK(char_vb->Resize(list_values.size()));
-  ASSERT_OK(int_vb->Resize(int_values.size()));
-
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
-  for (int8_t value : list_values) {
-    char_vb->UnsafeAppend(value);
-  }
-  for (int32_t value : int_values) {
-    int_vb->UnsafeAppend(value);
-  }
-
-  ASSERT_OK(builder_->Finish(&equal_array));
-
-  ASSERT_OK(builder_->Resize(list_lengths.size()));
-  ASSERT_OK(char_vb->Resize(list_values.size()));
-  ASSERT_OK(int_vb->Resize(int_values.size()));
-
-  // setup an unequal one with the unequal bitmap
-  builder_->Append(unequal_struct_is_valid.size(), unequal_struct_is_valid.data());
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
-  for (int8_t value : list_values) {
-    char_vb->UnsafeAppend(value);
-  }
-  for (int32_t value : int_values) {
-    int_vb->UnsafeAppend(value);
-  }
-
-  ASSERT_OK(builder_->Finish(&unequal_bitmap_array));
-
-  ASSERT_OK(builder_->Resize(list_lengths.size()));
-  ASSERT_OK(char_vb->Resize(list_values.size()));
-  ASSERT_OK(int_vb->Resize(int_values.size()));
-
-  // setup an unequal one with unequal offsets
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-  list_vb->Append(unequal_list_offsets.data(), unequal_list_offsets.size(),
-      unequal_list_is_valid.data());
-  for (int8_t value : list_values) {
-    char_vb->UnsafeAppend(value);
-  }
-  for (int32_t value : int_values) {
-    int_vb->UnsafeAppend(value);
-  }
-
-  ASSERT_OK(builder_->Finish(&unequal_offsets_array));
-
-  ASSERT_OK(builder_->Resize(list_lengths.size()));
-  ASSERT_OK(char_vb->Resize(list_values.size()));
-  ASSERT_OK(int_vb->Resize(int_values.size()));
-
-  // setup anunequal one with unequal values
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
-  for (int8_t value : unequal_list_values) {
-    char_vb->UnsafeAppend(value);
-  }
-  for (int32_t value : unequal_int_values) {
-    int_vb->UnsafeAppend(value);
-  }
-
-  ASSERT_OK(builder_->Finish(&unequal_values_array));
-
-  // Test array equality
-  EXPECT_TRUE(array->Equals(array));
-  EXPECT_TRUE(array->Equals(equal_array));
-  EXPECT_TRUE(equal_array->Equals(array));
-  EXPECT_FALSE(equal_array->Equals(unequal_bitmap_array));
-  EXPECT_FALSE(unequal_bitmap_array->Equals(equal_array));
-  EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_values_array));
-  EXPECT_FALSE(unequal_values_array->Equals(unequal_bitmap_array));
-  EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_offsets_array));
-  EXPECT_FALSE(unequal_offsets_array->Equals(unequal_bitmap_array));
-
-  // Test range equality
-  EXPECT_TRUE(array->RangeEquals(0, 4, 0, equal_array));
-  EXPECT_TRUE(array->RangeEquals(3, 4, 3, unequal_bitmap_array));
-  EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_offsets_array));
-  EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_offsets_array));
-  EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_offsets_array));
-  EXPECT_FALSE(array->RangeEquals(0, 1, 0, unequal_values_array));
-  EXPECT_TRUE(array->RangeEquals(1, 3, 1, unequal_values_array));
-  EXPECT_FALSE(array->RangeEquals(3, 4, 3, unequal_values_array));
-
-  // ARROW-33 Slice / equality
-  std::shared_ptr<Array> slice, slice2;
-
-  slice = array->Slice(2);
-  slice2 = array->Slice(2);
-  ASSERT_EQ(array->length() - 2, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(2, slice->length(), 0, slice));
-
-  slice = array->Slice(1, 2);
-  slice2 = array->Slice(1, 2);
-  ASSERT_EQ(2, slice->length());
-
-  ASSERT_TRUE(slice->Equals(slice2));
-  ASSERT_TRUE(array->RangeEquals(1, 3, 0, slice));
-}
-
-TEST_F(TestStructBuilder, TestZeroLength) {
-  // All buffers are null
-  Done();
-  ASSERT_OK(result_->Validate());
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 854ebb20f53ed..52f3727d46a15 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -25,12 +25,20 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/ipc/test-common.h"
 #include "arrow/memory_pool.h"
+#include "arrow/status.h"
+#include "arrow/test-common.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
+#include "arrow/type_traits.h"
 
 namespace arrow {
 
+using std::string;
+using std::vector;
+
 class TestArray : public ::testing::Test {
  public:
   void SetUp() { pool_ = default_memory_pool(); }
@@ -57,7 +65,7 @@ TEST_F(TestArray, TestLength) {
 }
 
 std::shared_ptr<Array> MakeArrayFromValidBytes(
-    const std::vector<uint8_t>& v, MemoryPool* pool) {
+    const vector<uint8_t>& v, MemoryPool* pool) {
   int64_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
   std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(v);
 
@@ -88,7 +96,7 @@ TEST_F(TestArray, TestEquality) {
 }
 
 TEST_F(TestArray, SliceRecomputeNullCount) {
-  std::vector<uint8_t> valid_bytes = {1, 0, 1, 1, 0, 1, 0, 0, 0};
+  vector<uint8_t> valid_bytes = {1, 0, 1, 1, 0, 1, 0, 0, 0};
 
   auto array = MakeArrayFromValidBytes(valid_bytes, pool_);
 
@@ -115,7 +123,7 @@ TEST_F(TestArray, SliceRecomputeNullCount) {
 
 TEST_F(TestArray, TestIsNull) {
   // clang-format off
-  std::vector<uint8_t> null_bitmap = {1, 0, 1, 1, 0, 1, 0, 0,
+  vector<uint8_t> null_bitmap = {1, 0, 1, 1, 0, 1, 0, 0,
                                       1, 0, 1, 1, 0, 1, 0, 0,
                                       1, 0, 1, 1, 0, 1, 0, 0,
                                       1, 0, 1, 1, 0, 1, 0, 0,
@@ -155,4 +163,1802 @@ TEST_F(TestArray, BuildLargeInMemoryArray) {
 
 TEST_F(TestArray, TestCopy) {}
 
+// ----------------------------------------------------------------------
+// Primitive type tests
+
+TEST_F(TestBuilder, TestReserve) {
+  builder_->Init(10);
+  ASSERT_EQ(2, builder_->null_bitmap()->size());
+
+  builder_->Reserve(30);
+  ASSERT_EQ(4, builder_->null_bitmap()->size());
+}
+
+template <typename Attrs>
+class TestPrimitiveBuilder : public TestBuilder {
+ public:
+  typedef typename Attrs::ArrayType ArrayType;
+  typedef typename Attrs::BuilderType BuilderType;
+  typedef typename Attrs::T T;
+  typedef typename Attrs::Type Type;
+
+  virtual void SetUp() {
+    TestBuilder::SetUp();
+
+    type_ = Attrs::type();
+
+    std::shared_ptr<ArrayBuilder> tmp;
+    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+    builder_ = std::dynamic_pointer_cast<BuilderType>(tmp);
+
+    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+    builder_nn_ = std::dynamic_pointer_cast<BuilderType>(tmp);
+  }
+
+  void RandomData(int64_t N, double pct_null = 0.1) {
+    Attrs::draw(N, &draws_);
+
+    valid_bytes_.resize(static_cast<size_t>(N));
+    test::random_null_bytes(N, pct_null, valid_bytes_.data());
+  }
+
+  void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
+    int64_t size = builder->length();
+
+    auto ex_data = std::make_shared<Buffer>(
+        reinterpret_cast<uint8_t*>(draws_.data()), size * sizeof(T));
+
+    std::shared_ptr<Buffer> ex_null_bitmap;
+    int64_t ex_null_count = 0;
+
+    if (nullable) {
+      ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
+      ex_null_count = test::null_count(valid_bytes_);
+    } else {
+      ex_null_bitmap = nullptr;
+    }
+
+    auto expected =
+        std::make_shared<ArrayType>(size, ex_data, ex_null_bitmap, ex_null_count);
+
+    std::shared_ptr<Array> out;
+    ASSERT_OK(builder->Finish(&out));
+
+    std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(out);
+
+    // Builder is now reset
+    ASSERT_EQ(0, builder->length());
+    ASSERT_EQ(0, builder->capacity());
+    ASSERT_EQ(0, builder->null_count());
+    ASSERT_EQ(nullptr, builder->data());
+
+    ASSERT_EQ(ex_null_count, result->null_count());
+    ASSERT_TRUE(result->Equals(*expected));
+  }
+
+ protected:
+  std::shared_ptr<DataType> type_;
+  std::shared_ptr<BuilderType> builder_;
+  std::shared_ptr<BuilderType> builder_nn_;
+
+  vector<T> draws_;
+  vector<uint8_t> valid_bytes_;
+};
+
+#define PTYPE_DECL(CapType, c_type)               \
+  typedef CapType##Array ArrayType;               \
+  typedef CapType##Builder BuilderType;           \
+  typedef CapType##Type Type;                     \
+  typedef c_type T;                               \
+                                                  \
+  static std::shared_ptr<DataType> type() {       \
+    return std::shared_ptr<DataType>(new Type()); \
+  }
+
+#define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
+  struct P##CapType {                               \
+    PTYPE_DECL(CapType, c_type);                    \
+    static void draw(int64_t N, vector<T>* draws) { \
+      test::randint<T>(N, LOWER, UPPER, draws);     \
+    }                                               \
+  }
+
+#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)     \
+  struct P##CapType {                                  \
+    PTYPE_DECL(CapType, c_type);                       \
+    static void draw(int64_t N, vector<T>* draws) {    \
+      test::random_real<T>(N, 0, LOWER, UPPER, draws); \
+    }                                                  \
+  }
+
+PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX);
+PINT_DECL(UInt16, uint16_t, 0, UINT16_MAX);
+PINT_DECL(UInt32, uint32_t, 0, UINT32_MAX);
+PINT_DECL(UInt64, uint64_t, 0, UINT64_MAX);
+
+PINT_DECL(Int8, int8_t, INT8_MIN, INT8_MAX);
+PINT_DECL(Int16, int16_t, INT16_MIN, INT16_MAX);
+PINT_DECL(Int32, int32_t, INT32_MIN, INT32_MAX);
+PINT_DECL(Int64, int64_t, INT64_MIN, INT64_MAX);
+
+PFLOAT_DECL(Float, float, -1000, 1000);
+PFLOAT_DECL(Double, double, -1000, 1000);
+
+struct PBoolean {
+  PTYPE_DECL(Boolean, uint8_t);
+};
+
+template <>
+void TestPrimitiveBuilder<PBoolean>::RandomData(int64_t N, double pct_null) {
+  draws_.resize(static_cast<size_t>(N));
+  valid_bytes_.resize(static_cast<size_t>(N));
+
+  test::random_null_bytes(N, 0.5, draws_.data());
+  test::random_null_bytes(N, pct_null, valid_bytes_.data());
+}
+
+template <>
+void TestPrimitiveBuilder<PBoolean>::Check(
+    const std::shared_ptr<BooleanBuilder>& builder, bool nullable) {
+  int64_t size = builder->length();
+
+  auto ex_data = test::bytes_to_null_buffer(draws_);
+
+  std::shared_ptr<Buffer> ex_null_bitmap;
+  int64_t ex_null_count = 0;
+
+  if (nullable) {
+    ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
+    ex_null_count = test::null_count(valid_bytes_);
+  } else {
+    ex_null_bitmap = nullptr;
+  }
+
+  auto expected =
+      std::make_shared<BooleanArray>(size, ex_data, ex_null_bitmap, ex_null_count);
+
+  std::shared_ptr<Array> out;
+  ASSERT_OK(builder->Finish(&out));
+  std::shared_ptr<BooleanArray> result = std::dynamic_pointer_cast<BooleanArray>(out);
+
+  // Builder is now reset
+  ASSERT_EQ(0, builder->length());
+  ASSERT_EQ(0, builder->capacity());
+  ASSERT_EQ(0, builder->null_count());
+  ASSERT_EQ(nullptr, builder->data());
+
+  ASSERT_EQ(ex_null_count, result->null_count());
+
+  ASSERT_EQ(expected->length(), result->length());
+
+  for (int64_t i = 0; i < result->length(); ++i) {
+    if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
+    bool actual = BitUtil::GetBit(result->data()->data(), i);
+    ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
+  }
+  ASSERT_TRUE(result->Equals(*expected));
+}
+
+typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
+    PInt32, PInt64, PFloat, PDouble>
+    Primitives;
+
+TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
+
+#define DECL_T() typedef typename TestFixture::T T;
+
+#define DECL_TYPE() typedef typename TestFixture::Type Type;
+
+#define DECL_ARRAYTYPE() typedef typename TestFixture::ArrayType ArrayType;
+
+TYPED_TEST(TestPrimitiveBuilder, TestInit) {
+  DECL_TYPE();
+
+  int64_t n = 1000;
+  ASSERT_OK(this->builder_->Reserve(n));
+  ASSERT_EQ(BitUtil::NextPower2(n), this->builder_->capacity());
+  ASSERT_EQ(BitUtil::NextPower2(TypeTraits<Type>::bytes_required(n)),
+      this->builder_->data()->size());
+
+  // unsure if this should go in all builder classes
+  ASSERT_EQ(0, this->builder_->num_children());
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
+  int64_t size = 1000;
+  for (int64_t i = 0; i < size; ++i) {
+    ASSERT_OK(this->builder_->AppendNull());
+  }
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(this->builder_->Finish(&result));
+
+  for (int64_t i = 0; i < size; ++i) {
+    ASSERT_TRUE(result->IsNull(i)) << i;
+  }
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
+  DECL_T();
+
+  int64_t size = 1000;
+
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
+
+  int64_t memory_before = this->pool_->bytes_allocated();
+
+  this->RandomData(size);
+
+  this->builder_->Reserve(size);
+
+  int64_t i;
+  for (i = 0; i < size; ++i) {
+    if (valid_bytes[i] > 0) {
+      this->builder_->Append(draws[i]);
+    } else {
+      this->builder_->AppendNull();
+    }
+  }
+
+  do {
+    std::shared_ptr<Array> result;
+    ASSERT_OK(this->builder_->Finish(&result));
+  } while (false);
+
+  ASSERT_EQ(memory_before, this->pool_->bytes_allocated());
+}
+
+TYPED_TEST(TestPrimitiveBuilder, Equality) {
+  DECL_T();
+
+  const int64_t size = 1000;
+  this->RandomData(size);
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
+  std::shared_ptr<Array> array, equal_array, unequal_array;
+  auto builder = this->builder_.get();
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &equal_array));
+
+  // Make the not equal array by negating the first valid element with itself.
+  const auto first_valid = std::find_if(
+      valid_bytes.begin(), valid_bytes.end(), [](uint8_t valid) { return valid > 0; });
+  const int64_t first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
+  // This should be true with a very high probability, but might introduce flakiness
+  ASSERT_LT(first_valid_idx, size - 1);
+  draws[first_valid_idx] =
+      static_cast<T>(~*reinterpret_cast<int64_t*>(&draws[first_valid_idx]));
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &unequal_array));
+
+  // test normal equality
+  EXPECT_TRUE(array->Equals(array));
+  EXPECT_TRUE(array->Equals(equal_array));
+  EXPECT_TRUE(equal_array->Equals(array));
+  EXPECT_FALSE(equal_array->Equals(unequal_array));
+  EXPECT_FALSE(unequal_array->Equals(equal_array));
+
+  // Test range equality
+  EXPECT_FALSE(array->RangeEquals(0, first_valid_idx + 1, 0, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(first_valid_idx, size, first_valid_idx, unequal_array));
+  EXPECT_TRUE(array->RangeEquals(0, first_valid_idx, 0, unequal_array));
+  EXPECT_TRUE(
+      array->RangeEquals(first_valid_idx + 1, size, first_valid_idx + 1, unequal_array));
+}
+
+TYPED_TEST(TestPrimitiveBuilder, SliceEquality) {
+  DECL_T();
+
+  const int64_t size = 1000;
+  this->RandomData(size);
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
+  auto builder = this->builder_.get();
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
+
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(5);
+  slice2 = array->Slice(5);
+  ASSERT_EQ(size - 5, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, array->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Slice(2)->Slice(3);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(5, 10);
+  slice2 = array->Slice(5, 10);
+  ASSERT_EQ(10, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, 15, 0, slice));
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
+  DECL_T();
+
+  const int64_t size = 10000;
+
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
+
+  this->RandomData(size);
+
+  this->builder_->Reserve(1000);
+  this->builder_nn_->Reserve(1000);
+
+  int64_t null_count = 0;
+  // Append the first 1000
+  for (size_t i = 0; i < 1000; ++i) {
+    if (valid_bytes[i] > 0) {
+      this->builder_->Append(draws[i]);
+    } else {
+      this->builder_->AppendNull();
+      ++null_count;
+    }
+    this->builder_nn_->Append(draws[i]);
+  }
+
+  ASSERT_EQ(null_count, this->builder_->null_count());
+
+  ASSERT_EQ(1000, this->builder_->length());
+  ASSERT_EQ(1024, this->builder_->capacity());
+
+  ASSERT_EQ(1000, this->builder_nn_->length());
+  ASSERT_EQ(1024, this->builder_nn_->capacity());
+
+  this->builder_->Reserve(size - 1000);
+  this->builder_nn_->Reserve(size - 1000);
+
+  // Append the next 9000
+  for (size_t i = 1000; i < size; ++i) {
+    if (valid_bytes[i] > 0) {
+      this->builder_->Append(draws[i]);
+    } else {
+      this->builder_->AppendNull();
+    }
+    this->builder_nn_->Append(draws[i]);
+  }
+
+  ASSERT_EQ(size, this->builder_->length());
+  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
+
+  ASSERT_EQ(size, this->builder_nn_->length());
+  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_nn_->capacity());
+
+  this->Check(this->builder_, true);
+  this->Check(this->builder_nn_, false);
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
+  DECL_T();
+
+  int64_t size = 10000;
+  this->RandomData(size);
+
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
+
+  // first slug
+  int64_t K = 1000;
+
+  ASSERT_OK(this->builder_->Append(draws.data(), K, valid_bytes.data()));
+  ASSERT_OK(this->builder_nn_->Append(draws.data(), K));
+
+  ASSERT_EQ(1000, this->builder_->length());
+  ASSERT_EQ(1024, this->builder_->capacity());
+
+  ASSERT_EQ(1000, this->builder_nn_->length());
+  ASSERT_EQ(1024, this->builder_nn_->capacity());
+
+  // Append the next 9000
+  ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, valid_bytes.data() + K));
+  ASSERT_OK(this->builder_nn_->Append(draws.data() + K, size - K));
+
+  ASSERT_EQ(size, this->builder_->length());
+  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
+
+  this->Check(this->builder_, true);
+  this->Check(this->builder_nn_, false);
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
+  int64_t n = 1000;
+  ASSERT_OK(this->builder_->Reserve(n));
+
+  ASSERT_OK(this->builder_->Advance(100));
+  ASSERT_EQ(100, this->builder_->length());
+
+  ASSERT_OK(this->builder_->Advance(900));
+
+  int64_t too_many = this->builder_->capacity() - 1000 + 1;
+  ASSERT_RAISES(Invalid, this->builder_->Advance(too_many));
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestResize) {
+  DECL_TYPE();
+
+  int64_t cap = kMinBuilderCapacity * 2;
+
+  ASSERT_OK(this->builder_->Reserve(cap));
+  ASSERT_EQ(cap, this->builder_->capacity());
+
+  ASSERT_EQ(TypeTraits<Type>::bytes_required(cap), this->builder_->data()->size());
+  ASSERT_EQ(BitUtil::BytesForBits(cap), this->builder_->null_bitmap()->size());
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
+  ASSERT_OK(this->builder_->Reserve(10));
+  ASSERT_EQ(0, this->builder_->length());
+  ASSERT_EQ(kMinBuilderCapacity, this->builder_->capacity());
+
+  ASSERT_OK(this->builder_->Reserve(90));
+  ASSERT_OK(this->builder_->Advance(100));
+  ASSERT_OK(this->builder_->Reserve(kMinBuilderCapacity));
+
+  ASSERT_EQ(BitUtil::NextPower2(kMinBuilderCapacity + 100), this->builder_->capacity());
+}
+
+template <typename TYPE>
+void CheckSliceApproxEquals() {
+  using T = typename TYPE::c_type;
+
+  const int64_t kSize = 50;
+  vector<T> draws1;
+  vector<T> draws2;
+
+  const uint32_t kSeed = 0;
+  test::random_real<T>(kSize, kSeed, 0, 100, &draws1);
+  test::random_real<T>(kSize, kSeed + 1, 0, 100, &draws2);
+
+  // Make the draws equal in the sliced segment, but unequal elsewhere (to
+  // catch not using the slice offset)
+  for (int64_t i = 10; i < 30; ++i) {
+    draws2[i] = draws1[i];
+  }
+
+  vector<bool> is_valid;
+  test::random_is_valid(kSize, 0.1, &is_valid);
+
+  std::shared_ptr<Array> array1, array2;
+  ArrayFromVector<TYPE, T>(is_valid, draws1, &array1);
+  ArrayFromVector<TYPE, T>(is_valid, draws2, &array2);
+
+  std::shared_ptr<Array> slice1 = array1->Slice(10, 20);
+  std::shared_ptr<Array> slice2 = array2->Slice(10, 20);
+
+  ASSERT_TRUE(slice1->ApproxEquals(slice2));
+}
+
+TEST(TestPrimitiveAdHoc, FloatingSliceApproxEquals) {
+  CheckSliceApproxEquals<FloatType>();
+  CheckSliceApproxEquals<DoubleType>();
+}
+
+// ----------------------------------------------------------------------
+// String / Binary tests
+
+class TestStringArray : public ::testing::Test {
+ public:
+  void SetUp() {
+    chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
+    offsets_ = {0, 1, 1, 1, 3, 6};
+    valid_bytes_ = {1, 1, 0, 1, 1};
+    expected_ = {"a", "", "", "bb", "ccc"};
+
+    MakeArray();
+  }
+
+  void MakeArray() {
+    length_ = static_cast<int64_t>(offsets_.size()) - 1;
+    value_buf_ = test::GetBufferFromVector(chars_);
+    offsets_buf_ = test::GetBufferFromVector(offsets_);
+    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
+    null_count_ = test::null_count(valid_bytes_);
+
+    strings_ = std::make_shared<StringArray>(
+        length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
+  }
+
+ protected:
+  vector<int32_t> offsets_;
+  vector<char> chars_;
+  vector<uint8_t> valid_bytes_;
+
+  vector<string> expected_;
+
+  std::shared_ptr<Buffer> value_buf_;
+  std::shared_ptr<Buffer> offsets_buf_;
+  std::shared_ptr<Buffer> null_bitmap_;
+
+  int64_t null_count_;
+  int64_t length_;
+
+  std::shared_ptr<StringArray> strings_;
+};
+
+TEST_F(TestStringArray, TestArrayBasics) {
+  ASSERT_EQ(length_, strings_->length());
+  ASSERT_EQ(1, strings_->null_count());
+  ASSERT_OK(strings_->Validate());
+}
+
+TEST_F(TestStringArray, TestType) {
+  std::shared_ptr<DataType> type = strings_->type();
+
+  ASSERT_EQ(Type::STRING, type->type);
+  ASSERT_EQ(Type::STRING, strings_->type_enum());
+}
+
+TEST_F(TestStringArray, TestListFunctions) {
+  int pos = 0;
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    ASSERT_EQ(pos, strings_->value_offset(i));
+    ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
+    pos += static_cast<int>(expected_[i].size());
+  }
+}
+
+TEST_F(TestStringArray, TestDestructor) {
+  auto arr = std::make_shared<StringArray>(
+      length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
+}
+
+TEST_F(TestStringArray, TestGetString) {
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    if (valid_bytes_[i] == 0) {
+      ASSERT_TRUE(strings_->IsNull(i));
+    } else {
+      ASSERT_EQ(expected_[i], strings_->GetString(i));
+    }
+  }
+}
+
+TEST_F(TestStringArray, TestEmptyStringComparison) {
+  offsets_ = {0, 0, 0, 0, 0, 0};
+  offsets_buf_ = test::GetBufferFromVector(offsets_);
+  length_ = static_cast<int64_t>(offsets_.size() - 1);
+
+  auto strings_a = std::make_shared<StringArray>(
+      length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
+  auto strings_b = std::make_shared<StringArray>(
+      length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
+  ASSERT_TRUE(strings_a->Equals(strings_b));
+}
+
+TEST_F(TestStringArray, CompareNullByteSlots) {
+  StringBuilder builder(default_memory_pool());
+  StringBuilder builder2(default_memory_pool());
+  StringBuilder builder3(default_memory_pool());
+
+  builder.Append("foo");
+  builder2.Append("foo");
+  builder3.Append("foo");
+
+  builder.Append("bar");
+  builder2.AppendNull();
+
+  // same length, but different
+  builder3.Append("xyz");
+
+  builder.Append("baz");
+  builder2.Append("baz");
+  builder3.Append("baz");
+
+  std::shared_ptr<Array> array, array2, array3;
+  ASSERT_OK(builder.Finish(&array));
+  ASSERT_OK(builder2.Finish(&array2));
+  ASSERT_OK(builder3.Finish(&array3));
+
+  const auto& a1 = static_cast<const StringArray&>(*array);
+  const auto& a2 = static_cast<const StringArray&>(*array2);
+  const auto& a3 = static_cast<const StringArray&>(*array3);
+
+  // The validity bitmaps are the same, the data is different, but the unequal
+  // portion is masked out
+  StringArray equal_array(3, a1.value_offsets(), a1.data(), a2.null_bitmap(), 1);
+  StringArray equal_array2(3, a3.value_offsets(), a3.data(), a2.null_bitmap(), 1);
+
+  ASSERT_TRUE(equal_array.Equals(equal_array2));
+  ASSERT_TRUE(a2.RangeEquals(equal_array2, 0, 3, 0));
+
+  ASSERT_TRUE(equal_array.Array::Slice(1)->Equals(equal_array2.Array::Slice(1)));
+  ASSERT_TRUE(
+      equal_array.Array::Slice(1)->RangeEquals(0, 2, 0, equal_array2.Array::Slice(1)));
+}
+
+TEST_F(TestStringArray, TestSliceGetString) {
+  StringBuilder builder(default_memory_pool());
+
+  builder.Append("a");
+  builder.Append("b");
+  builder.Append("c");
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+  auto s = array->Slice(1, 10);
+  auto arr = std::dynamic_pointer_cast<StringArray>(s);
+  ASSERT_EQ(arr->GetString(0), "b");
+}
+
+// ----------------------------------------------------------------------
+// String builder tests
+
+class TestStringBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+    builder_.reset(new StringBuilder(pool_));
+  }
+
+  void Done() {
+    std::shared_ptr<Array> out;
+    EXPECT_OK(builder_->Finish(&out));
+
+    result_ = std::dynamic_pointer_cast<StringArray>(out);
+    result_->Validate();
+  }
+
+ protected:
+  std::unique_ptr<StringBuilder> builder_;
+  std::shared_ptr<StringArray> result_;
+};
+
+TEST_F(TestStringBuilder, TestScalarAppend) {
+  vector<string> strings = {"", "bb", "a", "", "ccc"};
+  vector<uint8_t> is_null = {0, 0, 0, 1, 0};
+
+  int N = static_cast<int>(strings.size());
+  int reps = 1000;
+
+  for (int j = 0; j < reps; ++j) {
+    for (int i = 0; i < N; ++i) {
+      if (is_null[i]) {
+        builder_->AppendNull();
+      } else {
+        builder_->Append(strings[i]);
+      }
+    }
+  }
+  Done();
+
+  ASSERT_EQ(reps * N, result_->length());
+  ASSERT_EQ(reps, result_->null_count());
+  ASSERT_EQ(reps * 6, result_->data()->size());
+
+  int32_t length;
+  int32_t pos = 0;
+  for (int i = 0; i < N * reps; ++i) {
+    if (is_null[i % N]) {
+      ASSERT_TRUE(result_->IsNull(i));
+    } else {
+      ASSERT_FALSE(result_->IsNull(i));
+      result_->GetValue(i, &length);
+      ASSERT_EQ(pos, result_->value_offset(i));
+      ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
+      ASSERT_EQ(strings[i % N], result_->GetString(i));
+
+      pos += length;
+    }
+  }
+}
+
+TEST_F(TestStringBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+}
+
+// Binary container type
+// TODO(emkornfield) there should be some way to refactor these to avoid code duplicating
+// with String
+class TestBinaryArray : public ::testing::Test {
+ public:
+  void SetUp() {
+    chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
+    offsets_ = {0, 1, 1, 1, 3, 6};
+    valid_bytes_ = {1, 1, 0, 1, 1};
+    expected_ = {"a", "", "", "bb", "ccc"};
+
+    MakeArray();
+  }
+
+  void MakeArray() {
+    length_ = static_cast<int64_t>(offsets_.size() - 1);
+    value_buf_ = test::GetBufferFromVector(chars_);
+    offsets_buf_ = test::GetBufferFromVector(offsets_);
+
+    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
+    null_count_ = test::null_count(valid_bytes_);
+
+    strings_ = std::make_shared<BinaryArray>(
+        length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
+  }
+
+ protected:
+  vector<int32_t> offsets_;
+  vector<char> chars_;
+  vector<uint8_t> valid_bytes_;
+
+  vector<string> expected_;
+
+  std::shared_ptr<Buffer> value_buf_;
+  std::shared_ptr<Buffer> offsets_buf_;
+  std::shared_ptr<Buffer> null_bitmap_;
+
+  int64_t null_count_;
+  int64_t length_;
+
+  std::shared_ptr<BinaryArray> strings_;
+};
+
+TEST_F(TestBinaryArray, TestArrayBasics) {
+  ASSERT_EQ(length_, strings_->length());
+  ASSERT_EQ(1, strings_->null_count());
+  ASSERT_OK(strings_->Validate());
+}
+
+TEST_F(TestBinaryArray, TestType) {
+  std::shared_ptr<DataType> type = strings_->type();
+
+  ASSERT_EQ(Type::BINARY, type->type);
+  ASSERT_EQ(Type::BINARY, strings_->type_enum());
+}
+
+TEST_F(TestBinaryArray, TestListFunctions) {
+  size_t pos = 0;
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    ASSERT_EQ(pos, strings_->value_offset(i));
+    ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
+    pos += expected_[i].size();
+  }
+}
+
+TEST_F(TestBinaryArray, TestDestructor) {
+  auto arr = std::make_shared<BinaryArray>(
+      length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
+}
+
+TEST_F(TestBinaryArray, TestGetValue) {
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    if (valid_bytes_[i] == 0) {
+      ASSERT_TRUE(strings_->IsNull(i));
+    } else {
+      int32_t len = -1;
+      const uint8_t* bytes = strings_->GetValue(i, &len);
+      ASSERT_EQ(0, std::memcmp(expected_[i].data(), bytes, len));
+    }
+  }
+}
+
+TEST_F(TestBinaryArray, TestEqualsEmptyStrings) {
+  BinaryBuilder builder(default_memory_pool(), arrow::binary());
+
+  string empty_string("");
+
+  builder.Append(empty_string);
+  builder.Append(empty_string);
+  builder.Append(empty_string);
+  builder.Append(empty_string);
+  builder.Append(empty_string);
+
+  std::shared_ptr<Array> left_arr;
+  ASSERT_OK(builder.Finish(&left_arr));
+
+  const BinaryArray& left = static_cast<const BinaryArray&>(*left_arr);
+  std::shared_ptr<Array> right = std::make_shared<BinaryArray>(left.length(),
+      left.value_offsets(), nullptr, left.null_bitmap(), left.null_count());
+
+  ASSERT_TRUE(left.Equals(right));
+  ASSERT_TRUE(left.RangeEquals(0, left.length(), 0, right));
+}
+
+class TestBinaryBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+    builder_.reset(new BinaryBuilder(pool_));
+  }
+
+  void Done() {
+    std::shared_ptr<Array> out;
+    EXPECT_OK(builder_->Finish(&out));
+
+    result_ = std::dynamic_pointer_cast<BinaryArray>(out);
+    result_->Validate();
+  }
+
+ protected:
+  std::unique_ptr<BinaryBuilder> builder_;
+  std::shared_ptr<BinaryArray> result_;
+};
+
+TEST_F(TestBinaryBuilder, TestScalarAppend) {
+  vector<string> strings = {"", "bb", "a", "", "ccc"};
+  vector<uint8_t> is_null = {0, 0, 0, 1, 0};
+
+  int N = static_cast<int>(strings.size());
+  int reps = 1000;
+
+  for (int j = 0; j < reps; ++j) {
+    for (int i = 0; i < N; ++i) {
+      if (is_null[i]) {
+        builder_->AppendNull();
+      } else {
+        builder_->Append(strings[i]);
+      }
+    }
+  }
+  Done();
+  ASSERT_OK(result_->Validate());
+  ASSERT_EQ(reps * N, result_->length());
+  ASSERT_EQ(reps, result_->null_count());
+  ASSERT_EQ(reps * 6, result_->data()->size());
+
+  int32_t length;
+  for (int i = 0; i < N * reps; ++i) {
+    if (is_null[i % N]) {
+      ASSERT_TRUE(result_->IsNull(i));
+    } else {
+      ASSERT_FALSE(result_->IsNull(i));
+      const uint8_t* vals = result_->GetValue(i, &length);
+      ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
+      ASSERT_EQ(0, std::memcmp(vals, strings[i % N].data(), length));
+    }
+  }
+}
+
+TEST_F(TestBinaryBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+}
+
+// ----------------------------------------------------------------------
+// Slice tests
+
+template <typename TYPE>
+void CheckSliceEquality() {
+  using Traits = TypeTraits<TYPE>;
+  using BuilderType = typename Traits::BuilderType;
+
+  BuilderType builder(default_memory_pool());
+
+  vector<string> strings = {"foo", "", "bar", "baz", "qux", ""};
+  vector<uint8_t> is_null = {0, 1, 0, 1, 0, 0};
+
+  int N = static_cast<int>(strings.size());
+  int reps = 10;
+
+  for (int j = 0; j < reps; ++j) {
+    for (int i = 0; i < N; ++i) {
+      if (is_null[i]) {
+        builder.AppendNull();
+      } else {
+        builder.Append(strings[i]);
+      }
+    }
+  }
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(5);
+  slice2 = array->Slice(5);
+  ASSERT_EQ(N * reps - 5, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, slice->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Slice(2)->Slice(3);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(5, 20);
+  slice2 = array->Slice(5, 20);
+  ASSERT_EQ(20, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, 25, 0, slice));
+}
+
+TEST_F(TestBinaryArray, TestSliceEquality) {
+  CheckSliceEquality<BinaryType>();
+}
+
+TEST_F(TestStringArray, TestSliceEquality) {
+  CheckSliceEquality<BinaryType>();
+}
+
+TEST_F(TestBinaryArray, LengthZeroCtor) {
+  BinaryArray array(0, nullptr, nullptr);
+}
+
+// ----------------------------------------------------------------------
+// FixedWidthBinary tests
+
+class TestFWBinaryArray : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  void InitBuilder(int byte_width) {
+    auto type = fixed_width_binary(byte_width);
+    builder_.reset(new FixedWidthBinaryBuilder(default_memory_pool(), type));
+  }
+
+ protected:
+  std::unique_ptr<FixedWidthBinaryBuilder> builder_;
+};
+
+TEST_F(TestFWBinaryArray, Builder) {
+  const int32_t byte_width = 10;
+  int64_t length = 4096;
+
+  int64_t nbytes = length * byte_width;
+
+  vector<uint8_t> data(nbytes);
+  test::random_bytes(nbytes, 0, data.data());
+
+  vector<uint8_t> is_valid(length);
+  test::random_null_bytes(length, 0.1, is_valid.data());
+
+  const uint8_t* raw_data = data.data();
+
+  std::shared_ptr<Array> result;
+
+  auto CheckResult = [this, &length, &is_valid, &raw_data, &byte_width](
+      const Array& result) {
+    // Verify output
+    const auto& fw_result = static_cast<const FixedWidthBinaryArray&>(result);
+
+    ASSERT_EQ(length, result.length());
+
+    for (int64_t i = 0; i < result.length(); ++i) {
+      if (is_valid[i]) {
+        ASSERT_EQ(
+            0, memcmp(raw_data + byte_width * i, fw_result.GetValue(i), byte_width));
+      } else {
+        ASSERT_TRUE(fw_result.IsNull(i));
+      }
+    }
+  };
+
+  // Build using iterative API
+  InitBuilder(byte_width);
+  for (int64_t i = 0; i < length; ++i) {
+    if (is_valid[i]) {
+      builder_->Append(raw_data + byte_width * i);
+    } else {
+      builder_->AppendNull();
+    }
+  }
+
+  ASSERT_OK(builder_->Finish(&result));
+  CheckResult(*result);
+
+  // Build using batch API
+  InitBuilder(byte_width);
+
+  const uint8_t* raw_is_valid = is_valid.data();
+
+  ASSERT_OK(builder_->Append(raw_data, 50, raw_is_valid));
+  ASSERT_OK(builder_->Append(raw_data + 50 * byte_width, length - 50, raw_is_valid + 50));
+  ASSERT_OK(builder_->Finish(&result));
+  CheckResult(*result);
+
+  // Build from std::string
+  InitBuilder(byte_width);
+  for (int64_t i = 0; i < length; ++i) {
+    if (is_valid[i]) {
+      builder_->Append(
+          string(reinterpret_cast<const char*>(raw_data + byte_width * i), byte_width));
+    } else {
+      builder_->AppendNull();
+    }
+  }
+
+  ASSERT_OK(builder_->Finish(&result));
+  CheckResult(*result);
+}
+
+TEST_F(TestFWBinaryArray, EqualsRangeEquals) {
+  // Check that we don't compare data in null slots
+
+  auto type = fixed_width_binary(4);
+  FixedWidthBinaryBuilder builder1(default_memory_pool(), type);
+  FixedWidthBinaryBuilder builder2(default_memory_pool(), type);
+
+  ASSERT_OK(builder1.Append("foo1"));
+  ASSERT_OK(builder1.AppendNull());
+
+  ASSERT_OK(builder2.Append("foo1"));
+  ASSERT_OK(builder2.Append("foo2"));
+
+  std::shared_ptr<Array> array1, array2;
+  ASSERT_OK(builder1.Finish(&array1));
+  ASSERT_OK(builder2.Finish(&array2));
+
+  const auto& a1 = static_cast<const FixedWidthBinaryArray&>(*array1);
+  const auto& a2 = static_cast<const FixedWidthBinaryArray&>(*array2);
+
+  FixedWidthBinaryArray equal1(type, 2, a1.data(), a1.null_bitmap(), 1);
+  FixedWidthBinaryArray equal2(type, 2, a2.data(), a1.null_bitmap(), 1);
+
+  ASSERT_TRUE(equal1.Equals(equal2));
+  ASSERT_TRUE(equal1.RangeEquals(equal2, 0, 2, 0));
+}
+
+TEST_F(TestFWBinaryArray, ZeroSize) {
+  auto type = fixed_width_binary(0);
+  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
+
+  ASSERT_OK(builder.Append(nullptr));
+  ASSERT_OK(builder.Append(nullptr));
+  ASSERT_OK(builder.Append(nullptr));
+  ASSERT_OK(builder.AppendNull());
+  ASSERT_OK(builder.AppendNull());
+  ASSERT_OK(builder.AppendNull());
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  const auto& fw_array = static_cast<const FixedWidthBinaryArray&>(*array);
+
+  // data is never allocated
+  ASSERT_TRUE(fw_array.data() == nullptr);
+  ASSERT_EQ(0, fw_array.byte_width());
+
+  ASSERT_EQ(6, array->length());
+  ASSERT_EQ(3, array->null_count());
+}
+
+TEST_F(TestFWBinaryArray, Slice) {
+  auto type = fixed_width_binary(4);
+  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
+
+  vector<string> strings = {"foo1", "foo2", "foo3", "foo4", "foo5"};
+  vector<uint8_t> is_null = {0, 1, 0, 0, 0};
+
+  for (int i = 0; i < 5; ++i) {
+    if (is_null[i]) {
+      builder.AppendNull();
+    } else {
+      builder.Append(strings[i]);
+    }
+  }
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(1);
+  slice2 = array->Slice(1);
+  ASSERT_EQ(4, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, slice->length(), 0, slice));
+
+  // Chained slices
+  slice = array->Slice(2);
+  slice2 = array->Slice(1)->Slice(1);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(1, 3);
+  ASSERT_EQ(3, slice->length());
+
+  slice2 = array->Slice(1, 3);
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 3, 0, slice));
+}
+
+// ----------------------------------------------------------------------
+// List tests
+
+class TestListBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+
+    value_type_ = int32();
+    type_ = list(value_type_);
+
+    std::shared_ptr<ArrayBuilder> tmp;
+    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+    builder_ = std::dynamic_pointer_cast<ListBuilder>(tmp);
+  }
+
+  void Done() {
+    std::shared_ptr<Array> out;
+    EXPECT_OK(builder_->Finish(&out));
+    result_ = std::dynamic_pointer_cast<ListArray>(out);
+  }
+
+ protected:
+  std::shared_ptr<DataType> value_type_;
+  std::shared_ptr<DataType> type_;
+
+  std::shared_ptr<ListBuilder> builder_;
+  std::shared_ptr<ListArray> result_;
+};
+
+TEST_F(TestListBuilder, Equality) {
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+
+  std::shared_ptr<Array> array, equal_array, unequal_array;
+  vector<int32_t> equal_offsets = {0, 1, 2, 5, 6, 7, 8, 10};
+  vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2, 5, 6};
+  vector<int32_t> unequal_offsets = {0, 1, 4, 7};
+  vector<int32_t> unequal_values = {1, 2, 2, 2, 3, 4, 5};
+
+  // setup two equal arrays
+  ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
+  ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
+
+  ASSERT_OK(builder_->Finish(&array));
+  ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
+  ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
+
+  ASSERT_OK(builder_->Finish(&equal_array));
+  // now an unequal one
+  ASSERT_OK(builder_->Append(unequal_offsets.data(), unequal_offsets.size()));
+  ASSERT_OK(vb->Append(unequal_values.data(), unequal_values.size()));
+
+  ASSERT_OK(builder_->Finish(&unequal_array));
+
+  // Test array equality
+  EXPECT_TRUE(array->Equals(array));
+  EXPECT_TRUE(array->Equals(equal_array));
+  EXPECT_TRUE(equal_array->Equals(array));
+  EXPECT_FALSE(equal_array->Equals(unequal_array));
+  EXPECT_FALSE(unequal_array->Equals(equal_array));
+
+  // Test range equality
+  EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_array));
+  EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
+  EXPECT_TRUE(array->RangeEquals(2, 3, 2, unequal_array));
+
+  // Check with slices, ARROW-33
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(2);
+  slice2 = array->Slice(2);
+  ASSERT_EQ(array->length() - 2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(2, slice->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Slice(1)->Slice(1);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(1, 4);
+  slice2 = array->Slice(1, 4);
+  ASSERT_EQ(4, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 5, 0, slice));
+}
+
+TEST_F(TestListBuilder, TestResize) {}
+
+TEST_F(TestListBuilder, TestAppendNull) {
+  ASSERT_OK(builder_->AppendNull());
+  ASSERT_OK(builder_->AppendNull());
+
+  Done();
+
+  ASSERT_OK(result_->Validate());
+  ASSERT_TRUE(result_->IsNull(0));
+  ASSERT_TRUE(result_->IsNull(1));
+
+  ASSERT_EQ(0, result_->raw_value_offsets()[0]);
+  ASSERT_EQ(0, result_->value_offset(1));
+  ASSERT_EQ(0, result_->value_offset(2));
+
+  Int32Array* values = static_cast<Int32Array*>(result_->values().get());
+  ASSERT_EQ(0, values->length());
+}
+
+void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& values,
+    const vector<uint8_t>& is_valid) {
+  ASSERT_OK(result->Validate());
+  ASSERT_EQ(1, result->null_count());
+  ASSERT_EQ(0, result->values()->null_count());
+
+  ASSERT_EQ(3, result->length());
+  vector<int32_t> ex_offsets = {0, 3, 3, 7};
+  for (size_t i = 0; i < ex_offsets.size(); ++i) {
+    ASSERT_EQ(ex_offsets[i], result->value_offset(i));
+  }
+
+  for (int i = 0; i < result->length(); ++i) {
+    ASSERT_EQ(!static_cast<bool>(is_valid[i]), result->IsNull(i));
+  }
+
+  ASSERT_EQ(7, result->values()->length());
+  Int32Array* varr = static_cast<Int32Array*>(result->values().get());
+
+  for (size_t i = 0; i < values.size(); ++i) {
+    ASSERT_EQ(values[i], varr->Value(i));
+  }
+}
+
+TEST_F(TestListBuilder, TestBasics) {
+  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+  vector<int> lengths = {3, 0, 4};
+  vector<uint8_t> is_valid = {1, 0, 1};
+
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+
+  ASSERT_OK(builder_->Reserve(lengths.size()));
+  ASSERT_OK(vb->Reserve(values.size()));
+
+  int pos = 0;
+  for (size_t i = 0; i < lengths.size(); ++i) {
+    ASSERT_OK(builder_->Append(is_valid[i] > 0));
+    for (int j = 0; j < lengths[i]; ++j) {
+      vb->Append(values[pos++]);
+    }
+  }
+
+  Done();
+  ValidateBasicListArray(result_.get(), values, is_valid);
+}
+
+TEST_F(TestListBuilder, BulkAppend) {
+  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+  vector<int> lengths = {3, 0, 4};
+  vector<uint8_t> is_valid = {1, 0, 1};
+  vector<int32_t> offsets = {0, 3, 3};
+
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+  ASSERT_OK(vb->Reserve(values.size()));
+
+  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+  for (int32_t value : values) {
+    vb->Append(value);
+  }
+  Done();
+  ValidateBasicListArray(result_.get(), values, is_valid);
+}
+
+TEST_F(TestListBuilder, BulkAppendInvalid) {
+  vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+  vector<int> lengths = {3, 0, 4};
+  vector<uint8_t> is_null = {0, 1, 0};
+  vector<uint8_t> is_valid = {1, 0, 1};
+  vector<int32_t> offsets = {0, 2, 4};  // should be 0, 3, 3 given the is_null array
+
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+  ASSERT_OK(vb->Reserve(values.size()));
+
+  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+  for (int32_t value : values) {
+    vb->Append(value);
+  }
+
+  Done();
+  ASSERT_RAISES(Invalid, result_->Validate());
+}
+
+TEST_F(TestListBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+  ASSERT_OK(result_->Validate());
+}
+
+// ----------------------------------------------------------------------
+// DictionaryArray tests
+
+TEST(TestDictionary, Basics) {
+  vector<int32_t> values = {100, 1000, 10000, 100000};
+  std::shared_ptr<Array> dict;
+  ArrayFromVector<Int32Type, int32_t>(values, &dict);
+
+  std::shared_ptr<DictionaryType> type1 =
+      std::dynamic_pointer_cast<DictionaryType>(dictionary(int16(), dict));
+  DictionaryType type2(int16(), dict);
+
+  ASSERT_TRUE(int16()->Equals(type1->index_type()));
+  ASSERT_TRUE(type1->dictionary()->Equals(dict));
+
+  ASSERT_TRUE(int16()->Equals(type2.index_type()));
+  ASSERT_TRUE(type2.dictionary()->Equals(dict));
+
+  ASSERT_EQ("dictionary<values=int32, indices=int16>", type1->ToString());
+}
+
+TEST(TestDictionary, Equals) {
+  vector<bool> is_valid = {true, true, false, true, true, true};
+
+  std::shared_ptr<Array> dict;
+  vector<string> dict_values = {"foo", "bar", "baz"};
+  ArrayFromVector<StringType, string>(dict_values, &dict);
+  std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
+
+  std::shared_ptr<Array> dict2;
+  vector<string> dict2_values = {"foo", "bar", "baz", "qux"};
+  ArrayFromVector<StringType, string>(dict2_values, &dict2);
+  std::shared_ptr<DataType> dict2_type = dictionary(int16(), dict2);
+
+  std::shared_ptr<Array> indices;
+  vector<int16_t> indices_values = {1, 2, -1, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices_values, &indices);
+
+  std::shared_ptr<Array> indices2;
+  vector<int16_t> indices2_values = {1, 2, 0, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices2_values, &indices2);
+
+  std::shared_ptr<Array> indices3;
+  vector<int16_t> indices3_values = {1, 1, 0, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices3_values, &indices3);
+
+  auto array = std::make_shared<DictionaryArray>(dict_type, indices);
+  auto array2 = std::make_shared<DictionaryArray>(dict_type, indices2);
+  auto array3 = std::make_shared<DictionaryArray>(dict2_type, indices);
+  auto array4 = std::make_shared<DictionaryArray>(dict_type, indices3);
+
+  ASSERT_TRUE(array->Equals(array));
+
+  // Equal, because the unequal index is masked by null
+  ASSERT_TRUE(array->Equals(array2));
+
+  // Unequal dictionaries
+  ASSERT_FALSE(array->Equals(array3));
+
+  // Unequal indices
+  ASSERT_FALSE(array->Equals(array4));
+
+  // RangeEquals
+  ASSERT_TRUE(array->RangeEquals(3, 6, 3, array4));
+  ASSERT_FALSE(array->RangeEquals(1, 3, 1, array4));
+
+  // ARROW-33 Test slices
+  const int64_t size = array->length();
+
+  std::shared_ptr<Array> slice, slice2;
+  slice = array->Array::Slice(2);
+  slice2 = array->Array::Slice(2);
+  ASSERT_EQ(size - 2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(2, array->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Array::Slice(1)->Array::Slice(1);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(1, 3);
+  slice2 = array->Slice(1, 3);
+  ASSERT_EQ(3, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 4, 0, slice));
+}
+
+TEST(TestDictionary, Validate) {
+  vector<bool> is_valid = {true, true, false, true, true, true};
+
+  std::shared_ptr<Array> dict;
+  vector<string> dict_values = {"foo", "bar", "baz"};
+  ArrayFromVector<StringType, string>(dict_values, &dict);
+  std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
+
+  std::shared_ptr<Array> indices;
+  vector<uint8_t> indices_values = {1, 2, 0, 0, 2, 0};
+  ArrayFromVector<UInt8Type, uint8_t>(is_valid, indices_values, &indices);
+
+  std::shared_ptr<Array> indices2;
+  vector<float> indices2_values = {1., 2., 0., 0., 2., 0.};
+  ArrayFromVector<FloatType, float>(is_valid, indices2_values, &indices2);
+
+  std::shared_ptr<Array> indices3;
+  vector<int64_t> indices3_values = {1, 2, 0, 0, 2, 0};
+  ArrayFromVector<Int64Type, int64_t>(is_valid, indices3_values, &indices3);
+
+  std::shared_ptr<Array> arr = std::make_shared<DictionaryArray>(dict_type, indices);
+  std::shared_ptr<Array> arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
+  std::shared_ptr<Array> arr3 = std::make_shared<DictionaryArray>(dict_type, indices3);
+
+  // Only checking index type for now
+  ASSERT_OK(arr->Validate());
+  ASSERT_RAISES(Invalid, arr2->Validate());
+  ASSERT_OK(arr3->Validate());
+}
+
+// ----------------------------------------------------------------------
+// Struct tests
+
+void ValidateBasicStructArray(const StructArray* result,
+    const vector<uint8_t>& struct_is_valid, const vector<char>& list_values,
+    const vector<uint8_t>& list_is_valid, const vector<int>& list_lengths,
+    const vector<int>& list_offsets, const vector<int32_t>& int_values) {
+  ASSERT_EQ(4, result->length());
+  ASSERT_OK(result->Validate());
+
+  auto list_char_arr = static_cast<ListArray*>(result->field(0).get());
+  auto char_arr = static_cast<Int8Array*>(list_char_arr->values().get());
+  auto int32_arr = static_cast<Int32Array*>(result->field(1).get());
+
+  ASSERT_EQ(0, result->null_count());
+  ASSERT_EQ(1, list_char_arr->null_count());
+  ASSERT_EQ(0, int32_arr->null_count());
+
+  // List<char>
+  ASSERT_EQ(4, list_char_arr->length());
+  ASSERT_EQ(10, list_char_arr->values()->length());
+  for (size_t i = 0; i < list_offsets.size(); ++i) {
+    ASSERT_EQ(list_offsets[i], list_char_arr->raw_value_offsets()[i]);
+  }
+  for (size_t i = 0; i < list_values.size(); ++i) {
+    ASSERT_EQ(list_values[i], char_arr->Value(i));
+  }
+
+  // Int32
+  ASSERT_EQ(4, int32_arr->length());
+  for (size_t i = 0; i < int_values.size(); ++i) {
+    ASSERT_EQ(int_values[i], int32_arr->Value(i));
+  }
+}
+
+// ----------------------------------------------------------------------------------
+// Struct test
+class TestStructBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+
+    auto int32_type = int32();
+    auto char_type = int8();
+    auto list_type = list(char_type);
+
+    vector<std::shared_ptr<DataType>> types = {list_type, int32_type};
+    vector<FieldPtr> fields;
+    fields.push_back(FieldPtr(new Field("list", list_type)));
+    fields.push_back(FieldPtr(new Field("int", int32_type)));
+
+    type_ = struct_(fields);
+    value_fields_ = fields;
+
+    std::shared_ptr<ArrayBuilder> tmp;
+    ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+
+    builder_ = std::dynamic_pointer_cast<StructBuilder>(tmp);
+    ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+  }
+
+  void Done() {
+    std::shared_ptr<Array> out;
+    ASSERT_OK(builder_->Finish(&out));
+    result_ = std::dynamic_pointer_cast<StructArray>(out);
+  }
+
+ protected:
+  vector<FieldPtr> value_fields_;
+  std::shared_ptr<DataType> type_;
+
+  std::shared_ptr<StructBuilder> builder_;
+  std::shared_ptr<StructArray> result_;
+};
+
+TEST_F(TestStructBuilder, TestAppendNull) {
+  ASSERT_OK(builder_->AppendNull());
+  ASSERT_OK(builder_->AppendNull());
+  ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  ASSERT_OK(list_vb->AppendNull());
+  ASSERT_OK(list_vb->AppendNull());
+  ASSERT_EQ(2, list_vb->length());
+
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ASSERT_OK(int_vb->AppendNull());
+  ASSERT_OK(int_vb->AppendNull());
+  ASSERT_EQ(2, int_vb->length());
+
+  Done();
+
+  ASSERT_OK(result_->Validate());
+
+  ASSERT_EQ(2, static_cast<int>(result_->fields().size()));
+  ASSERT_EQ(2, result_->length());
+  ASSERT_EQ(2, result_->field(0)->length());
+  ASSERT_EQ(2, result_->field(1)->length());
+  ASSERT_TRUE(result_->IsNull(0));
+  ASSERT_TRUE(result_->IsNull(1));
+  ASSERT_TRUE(result_->field(0)->IsNull(0));
+  ASSERT_TRUE(result_->field(0)->IsNull(1));
+  ASSERT_TRUE(result_->field(1)->IsNull(0));
+  ASSERT_TRUE(result_->field(1)->IsNull(1));
+
+  ASSERT_EQ(Type::LIST, result_->field(0)->type_enum());
+  ASSERT_EQ(Type::INT32, result_->field(1)->type_enum());
+}
+
+TEST_F(TestStructBuilder, TestBasics) {
+  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6, 10};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+
+  EXPECT_OK(builder_->Resize(list_lengths.size()));
+  EXPECT_OK(char_vb->Resize(list_values.size()));
+  EXPECT_OK(int_vb->Resize(int_values.size()));
+
+  int pos = 0;
+  for (size_t i = 0; i < list_lengths.size(); ++i) {
+    ASSERT_OK(list_vb->Append(list_is_valid[i] > 0));
+    int_vb->UnsafeAppend(int_values[i]);
+    for (int j = 0; j < list_lengths[i]; ++j) {
+      char_vb->UnsafeAppend(list_values[pos++]);
+    }
+  }
+
+  for (size_t i = 0; i < struct_is_valid.size(); ++i) {
+    ASSERT_OK(builder_->Append(struct_is_valid[i] > 0));
+  }
+
+  Done();
+
+  ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
+      list_lengths, list_offsets, int_values);
+}
+
+TEST_F(TestStructBuilder, BulkAppend) {
+  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  Done();
+  ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
+      list_lengths, list_offsets, int_values);
+}
+
+TEST_F(TestStructBuilder, BulkAppendInvalid) {
+  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 0, 1, 1};  // should be 1, 1, 1, 1
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+
+  ASSERT_OK(builder_->Reserve(list_lengths.size()));
+  ASSERT_OK(char_vb->Reserve(list_values.size()));
+  ASSERT_OK(int_vb->Reserve(int_values.size()));
+
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  Done();
+  // Even null bitmap of the parent Struct is not valid, Validate() will ignore it.
+  ASSERT_OK(result_->Validate());
+}
+
+TEST_F(TestStructBuilder, TestEquality) {
+  std::shared_ptr<Array> array, equal_array;
+  std::shared_ptr<Array> unequal_bitmap_array, unequal_offsets_array,
+      unequal_values_array;
+
+  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+  vector<int32_t> unequal_int_values = {4, 2, 3, 1};
+  vector<char> unequal_list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'l', 'u', 'c', 'y'};
+  vector<int> unequal_list_offsets = {0, 3, 4, 6};
+  vector<uint8_t> unequal_list_is_valid = {1, 1, 1, 1};
+  vector<uint8_t> unequal_struct_is_valid = {1, 0, 0, 1};
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ASSERT_OK(builder_->Reserve(list_lengths.size()));
+  ASSERT_OK(char_vb->Reserve(list_values.size()));
+  ASSERT_OK(int_vb->Reserve(int_values.size()));
+
+  // setup two equal arrays, one of which takes an unequal bitmap
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  ASSERT_OK(builder_->Finish(&array));
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  ASSERT_OK(builder_->Finish(&equal_array));
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  // setup an unequal one with the unequal bitmap
+  builder_->Append(unequal_struct_is_valid.size(), unequal_struct_is_valid.data());
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  ASSERT_OK(builder_->Finish(&unequal_bitmap_array));
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  // setup an unequal one with unequal offsets
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  list_vb->Append(unequal_list_offsets.data(), unequal_list_offsets.size(),
+      unequal_list_is_valid.data());
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  ASSERT_OK(builder_->Finish(&unequal_offsets_array));
+
+  ASSERT_OK(builder_->Resize(list_lengths.size()));
+  ASSERT_OK(char_vb->Resize(list_values.size()));
+  ASSERT_OK(int_vb->Resize(int_values.size()));
+
+  // setup anunequal one with unequal values
+  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  for (int8_t value : unequal_list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : unequal_int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+
+  ASSERT_OK(builder_->Finish(&unequal_values_array));
+
+  // Test array equality
+  EXPECT_TRUE(array->Equals(array));
+  EXPECT_TRUE(array->Equals(equal_array));
+  EXPECT_TRUE(equal_array->Equals(array));
+  EXPECT_FALSE(equal_array->Equals(unequal_bitmap_array));
+  EXPECT_FALSE(unequal_bitmap_array->Equals(equal_array));
+  EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_values_array));
+  EXPECT_FALSE(unequal_values_array->Equals(unequal_bitmap_array));
+  EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_offsets_array));
+  EXPECT_FALSE(unequal_offsets_array->Equals(unequal_bitmap_array));
+
+  // Test range equality
+  EXPECT_TRUE(array->RangeEquals(0, 4, 0, equal_array));
+  EXPECT_TRUE(array->RangeEquals(3, 4, 3, unequal_bitmap_array));
+  EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_offsets_array));
+  EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_offsets_array));
+  EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_offsets_array));
+  EXPECT_FALSE(array->RangeEquals(0, 1, 0, unequal_values_array));
+  EXPECT_TRUE(array->RangeEquals(1, 3, 1, unequal_values_array));
+  EXPECT_FALSE(array->RangeEquals(3, 4, 3, unequal_values_array));
+
+  // ARROW-33 Slice / equality
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(2);
+  slice2 = array->Slice(2);
+  ASSERT_EQ(array->length() - 2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(2, slice->length(), 0, slice));
+
+  slice = array->Slice(1, 2);
+  slice2 = array->Slice(1, 2);
+  ASSERT_EQ(2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 3, 0, slice));
+}
+
+TEST_F(TestStructBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+  ASSERT_OK(result_->Validate());
+}
+
+// ----------------------------------------------------------------------
+// Union tests
+
+TEST(TestUnionArrayAdHoc, TestSliceEquals) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(ipc::MakeUnion(&batch));
+
+  const int64_t size = batch->num_rows();
+
+  auto CheckUnion = [&size](std::shared_ptr<Array> array) {
+    std::shared_ptr<Array> slice, slice2;
+    slice = array->Slice(2);
+    slice2 = array->Slice(2);
+    ASSERT_EQ(size - 2, slice->length());
+
+    ASSERT_TRUE(slice->Equals(slice2));
+    ASSERT_TRUE(array->RangeEquals(2, array->length(), 0, slice));
+
+    // Chained slices
+    slice2 = array->Slice(1)->Slice(1);
+    ASSERT_TRUE(slice->Equals(slice2));
+
+    slice = array->Slice(1, 5);
+    slice2 = array->Slice(1, 5);
+    ASSERT_EQ(5, slice->length());
+
+    ASSERT_TRUE(slice->Equals(slice2));
+    ASSERT_TRUE(array->RangeEquals(1, 6, 0, slice));
+  };
+
+  CheckUnion(batch->column(1));
+  CheckUnion(batch->column(2));
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array-union-test.cc b/cpp/src/arrow/array-union-test.cc
deleted file mode 100644
index 83c3196cab74b..0000000000000
--- a/cpp/src/arrow/array-union-test.cc
+++ /dev/null
@@ -1,67 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Tests for UnionArray
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/ipc/test-common.h"
-#include "arrow/status.h"
-#include "arrow/table.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-TEST(TestUnionArrayAdHoc, TestSliceEquals) {
-  std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK(ipc::MakeUnion(&batch));
-
-  const int64_t size = batch->num_rows();
-
-  auto CheckUnion = [&size](std::shared_ptr<Array> array) {
-    std::shared_ptr<Array> slice, slice2;
-    slice = array->Slice(2);
-    slice2 = array->Slice(2);
-    ASSERT_EQ(size - 2, slice->length());
-
-    ASSERT_TRUE(slice->Equals(slice2));
-    ASSERT_TRUE(array->RangeEquals(2, array->length(), 0, slice));
-
-    // Chained slices
-    slice2 = array->Slice(1)->Slice(1);
-    ASSERT_TRUE(slice->Equals(slice2));
-
-    slice = array->Slice(1, 5);
-    slice2 = array->Slice(1, 5);
-    ASSERT_EQ(5, slice->length());
-
-    ASSERT_TRUE(slice->Equals(slice2));
-    ASSERT_TRUE(array->RangeEquals(1, 6, 0, slice));
-  };
-
-  CheckUnion(batch->column(1));
-  CheckUnion(batch->column(2));
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index f3140be0b2dac..af59e96a1448f 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -78,9 +78,10 @@ class TestHdfsClient : public ::testing::Test {
     LibHdfsShim* driver_shim;
 
     client_ = nullptr;
-    scratch_dir_ = boost::filesystem::unique_path(
-        boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
-                       .string();
+    scratch_dir_ =
+        boost::filesystem::unique_path(
+            boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
+            .string();
 
     loaded_driver_ = false;
 
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 70c173432a960..b221c80391cde 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -117,6 +117,28 @@ TEST_F(TestSchema, GetFieldByName) {
   ASSERT_TRUE(result == nullptr);
 }
 
+#define PRIMITIVE_TEST(KLASS, ENUM, NAME)        \
+  TEST(TypesTest, TestPrimitive_##ENUM) {        \
+    KLASS tp;                                    \
+                                                 \
+    ASSERT_EQ(tp.type, Type::ENUM);              \
+    ASSERT_EQ(tp.ToString(), std::string(NAME)); \
+  }
+
+PRIMITIVE_TEST(Int8Type, INT8, "int8");
+PRIMITIVE_TEST(Int16Type, INT16, "int16");
+PRIMITIVE_TEST(Int32Type, INT32, "int32");
+PRIMITIVE_TEST(Int64Type, INT64, "int64");
+PRIMITIVE_TEST(UInt8Type, UINT8, "uint8");
+PRIMITIVE_TEST(UInt16Type, UINT16, "uint16");
+PRIMITIVE_TEST(UInt32Type, UINT32, "uint32");
+PRIMITIVE_TEST(UInt64Type, UINT64, "uint64");
+
+PRIMITIVE_TEST(FloatType, FLOAT, "float");
+PRIMITIVE_TEST(DoubleType, DOUBLE, "double");
+
+PRIMITIVE_TEST(BooleanType, BOOL, "bool");
+
 TEST(TestBinaryType, ToString) {
   BinaryType t1;
   BinaryType e1;
@@ -264,4 +286,27 @@ TEST(TestNestedType, Equals) {
   ASSERT_FALSE(u0->Equals(u0_bad));
 }
 
+TEST(TestStructType, Basics) {
+  auto f0_type = int32();
+  auto f0 = field("f0", f0_type);
+
+  auto f1_type = utf8();
+  auto f1 = field("f1", f1_type);
+
+  auto f2_type = uint8();
+  auto f2 = field("f2", f2_type);
+
+  vector<std::shared_ptr<Field>> fields = {f0, f1, f2};
+
+  StructType struct_type(fields);
+
+  ASSERT_TRUE(struct_type.child(0)->Equals(f0));
+  ASSERT_TRUE(struct_type.child(1)->Equals(f1));
+  ASSERT_TRUE(struct_type.child(2)->Equals(f2));
+
+  ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
+
+  // TODO(wesm): out of bounds for field(...)
+}
+
 }  // namespace arrow

From ba4f478e7c651f43e5e605b3fa6818f2e9f7cd3d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 30 Mar 2017 18:41:35 -0400
Subject: [PATCH 0437/1644] ARROW-715: [Python] Make pandas not a hard
 requirement, flake8 fixes

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #462 from wesm/ARROW-715 and squashes the following commits:

21fe8eb [Wes McKinney] Make pandas not a hard requirement, flake8 fixes
---
 python/pyarrow/compat.py | 19 +++++++++++--------
 python/setup.py          | 21 +++++++++++++--------
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 74d7ca2827bc9..b9206aacbc9f1 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -21,7 +21,6 @@
 import itertools
 
 import numpy as np
-import pandas as pd
 
 import sys
 import six
@@ -31,6 +30,17 @@
 PY26 = sys.version_info[:2] == (2, 6)
 PY2 = sys.version_info[0] == 2
 
+try:
+    import pandas as pd
+    if LooseVersion(pd.__version__) < '0.19.0':
+        pdapi = pd.core.common
+        from pandas.core.dtypes import DatetimeTZDtype
+    else:
+        from pandas.types.dtypes import DatetimeTZDtype
+        pdapi = pd.api.types
+    HAVE_PANDAS = True
+except:
+    HAVE_PANDAS = False
 
 if PY26:
     import unittest2 as unittest
@@ -117,13 +127,6 @@ def encode_file_path(path):
     return encoded_path
 
 
-if LooseVersion(pd.__version__) < '0.19.0':
-    pdapi = pd.core.common
-    from pandas.core.dtypes import DatetimeTZDtype
-else:
-    from pandas.types.dtypes import DatetimeTZDtype
-    pdapi = pd.api.types
-
 integer_types = six.integer_types + (np.integer,)
 
 __all__ = []
diff --git a/python/setup.py b/python/setup.py
index dae6cb2f078f6..9ff091819c760 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -17,7 +17,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import glob
 import os.path as osp
 import re
 import shutil
@@ -83,16 +82,20 @@ def run(self):
                      ('build-type=', None, 'build type (debug or release)'),
                      ('with-parquet', None, 'build the Parquet extension'),
                      ('with-jemalloc', None, 'build the jemalloc extension'),
-                     ('bundle-arrow-cpp', None, 'bundle the Arrow C++ libraries')] +
+                     ('bundle-arrow-cpp', None,
+                      'bundle the Arrow C++ libraries')] +
                     _build_ext.user_options)
 
     def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
-        self.with_parquet = strtobool(os.environ.get('PYARROW_WITH_PARQUET', '0'))
-        self.with_jemalloc = strtobool(os.environ.get('PYARROW_WITH_JEMALLOC', '0'))
-        self.bundle_arrow_cpp = strtobool(os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
+        self.with_parquet = strtobool(
+            os.environ.get('PYARROW_WITH_PARQUET', '0'))
+        self.with_jemalloc = strtobool(
+            os.environ.get('PYARROW_WITH_JEMALLOC', '0'))
+        self.bundle_arrow_cpp = strtobool(
+            os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
 
     CYTHON_MODULE_NAMES = [
         'array',
@@ -300,8 +303,10 @@ def get_outputs(self):
 if not os.path.exists('../.git') and os.path.exists('../java/pom.xml'):
     import xml.etree.ElementTree as ET
     tree = ET.parse('../java/pom.xml')
-    version_tag = list(tree.getroot().findall('{http://maven.apache.org/POM/4.0.0}version'))[0]
-    os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version_tag.text.replace("-SNAPSHOT", "a0")
+    version_tag = list(tree.getroot().findall(
+        '{http://maven.apache.org/POM/4.0.0}version'))[0]
+    os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version_tag.text.replace(
+        "-SNAPSHOT", "a0")
 
 long_description = """Apache Arrow is a columnar in-memory analytics layer
 designed to accelerate big data. It houses a set of canonical in-memory
@@ -321,7 +326,7 @@ def get_outputs(self):
         'clean': clean,
         'build_ext': build_ext
     },
-    use_scm_version = {"root": "..", "relative_to": __file__},
+    use_scm_version={"root": "..", "relative_to": __file__},
     setup_requires=['setuptools_scm', 'cython >= 0.23'],
     install_requires=['numpy >= 1.9', 'six >= 1.0.0'],
     test_requires=['pytest'],

From edd6cfcd9bfc02b2ed093f22acf830a57422f7b3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 30 Mar 2017 18:42:52 -0400
Subject: [PATCH 0438/1644] ARROW-727: [Python] Ensure that NativeFile.write
 accepts any bytes, unicode, or object providing buffer protocol. Rename
 build_arrow_buffer to pyarrow.frombuffer

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #464 from wesm/ARROW-727 and squashes the following commits:

c93edb0 [Wes McKinney] Rename build_arrow_buffer to pyarrow.frombuffer. Ensure that NativeFile.write accepts any bytes, unicode, or object providing buffer protocol
---
 python/pyarrow/__init__.py      |  3 ++-
 python/pyarrow/io.pyx           | 16 ++++++++++------
 python/pyarrow/tests/test_io.py | 29 +++++++++++++++++++++++------
 3 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index c6f0be04e8d0d..dce438910151b 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -43,7 +43,8 @@
 
 from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
 from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
-                        Buffer, InMemoryOutputStream, BufferReader)
+                        Buffer, InMemoryOutputStream, BufferReader,
+                        frombuffer)
 
 from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
 
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index d528bdc495208..d64427aa36ef5 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -126,14 +126,18 @@ cdef class NativeFile:
 
     def write(self, data):
         """
-        Write bytes-like (unicode, encoded to UTF-8) to file
+        Write byte from any object implementing buffer protocol (bytes,
+        bytearray, ndarray, pyarrow.Buffer)
         """
         self._assert_writeable()
 
-        data = tobytes(data)
+        if isinstance(data, six.string_types):
+            data = tobytes(data)
 
-        cdef const uint8_t* buf = <const uint8_t*> cp.PyBytes_AS_STRING(data)
-        cdef int64_t bufsize = len(data)
+        cdef Buffer arrow_buffer = frombuffer(data)
+
+        cdef const uint8_t* buf = arrow_buffer.buffer.get().data()
+        cdef int64_t bufsize = len(arrow_buffer)
         with nogil:
             check_status(self.wr_file.get().Write(buf, bufsize))
 
@@ -505,7 +509,7 @@ cdef class BufferReader(NativeFile):
         if isinstance(obj, Buffer):
             self.buffer = obj
         else:
-            self.buffer = build_arrow_buffer(obj)
+            self.buffer = frombuffer(obj)
 
         self.rd_file.reset(new CBufferReader(self.buffer.buffer))
         self.is_readable = 1
@@ -513,7 +517,7 @@ cdef class BufferReader(NativeFile):
         self.is_open = True
 
 
-def build_arrow_buffer(object obj):
+def frombuffer(object obj):
     """
     Construct an Arrow buffer from a Python bytes object
     """
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 9cd15c4a76cef..15c5e6b924385 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -23,6 +23,7 @@
 import numpy as np
 
 from pyarrow.compat import u, guid
+import pyarrow as pa
 import pyarrow.io as io
 
 # ----------------------------------------------------------------------
@@ -127,28 +128,29 @@ def get_buffer():
 def test_buffer_bytes():
     val = b'some data'
 
-    buf = io.build_arrow_buffer(val)
+    buf = pa.frombuffer(val)
     assert isinstance(buf, io.Buffer)
 
     result = buf.to_pybytes()
 
     assert result == val
 
+
 def test_buffer_memoryview():
     val = b'some data'
 
-    buf = io.build_arrow_buffer(val)
+    buf = pa.frombuffer(val)
     assert isinstance(buf, io.Buffer)
 
     result = memoryview(buf)
 
     assert result == val
 
+
 def test_buffer_bytearray():
     val = bytearray(b'some data')
 
-
-    buf = io.build_arrow_buffer(val)
+    buf = pa.frombuffer(val)
     assert isinstance(buf, io.Buffer)
 
     result = bytearray(buf)
@@ -159,7 +161,7 @@ def test_buffer_bytearray():
 def test_buffer_memoryview_is_immutable():
     val = b'some data'
 
-    buf = io.build_arrow_buffer(val)
+    buf = pa.frombuffer(val)
     assert isinstance(buf, io.Buffer)
 
     result = memoryview(buf)
@@ -198,21 +200,36 @@ def test_inmemory_write_after_closed():
     with pytest.raises(IOError):
         f.write(b'not ok')
 
+
 def test_buffer_protocol_ref_counting():
     import gc
 
     def make_buffer(bytes_obj):
-        return bytearray(io.build_arrow_buffer(bytes_obj))
+        return bytearray(pa.frombuffer(bytes_obj))
 
     buf = make_buffer(b'foo')
     gc.collect()
     assert buf == b'foo'
 
 
+def test_nativefile_write_memoryview():
+    f = io.InMemoryOutputStream()
+    data = b'ok'
+
+    arr = np.frombuffer(data, dtype='S1')
+
+    f.write(arr)
+    f.write(bytearray(data))
+
+    buf = f.get_result()
+
+    assert buf.to_pybytes() == data * 2
+
 
 # ----------------------------------------------------------------------
 # OS files and memory maps
 
+
 @pytest.fixture
 def sample_disk_data(request):
     SIZE = 4096

From 4915ecf1e1dba625d916604d30f2575e4ddb6439 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 30 Mar 2017 19:12:49 -0400
Subject: [PATCH 0439/1644] ARROW-632: [Python] Add support for
 FixedWidthBinary type

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #461 from cpcloud/ARROW-632 and squashes the following commits:

134644a [Phillip Cloud] ARROW-632: [Python] Add support for FixedWidthBinary type
---
 .gitignore                                    |   3 +
 cpp/src/arrow/builder.cc                      |   1 +
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc |   4 +-
 cpp/src/arrow/ipc/reader.cc                   |   2 +-
 cpp/src/arrow/python/builtin_convert.cc       | 101 +++++++++++---
 cpp/src/arrow/python/builtin_convert.h        |  17 ++-
 cpp/src/arrow/python/pandas_convert.cc        | 131 +++++++++++++++---
 cpp/src/arrow/util/logging.h                  |   7 +-
 python/pyarrow/__init__.py                    |   5 +-
 python/pyarrow/array.pxd                      |   8 ++
 python/pyarrow/array.pyx                      |  16 ++-
 python/pyarrow/includes/libarrow.pxd          |   8 ++
 python/pyarrow/includes/pyarrow.pxd           |   3 +
 python/pyarrow/scalar.pxd                     |   5 +
 python/pyarrow/scalar.pyx                     |  19 ++-
 python/pyarrow/schema.pxd                     |   6 +
 python/pyarrow/schema.pyx                     |  42 ++++--
 python/pyarrow/tests/test_convert_builtin.py  |  13 ++
 python/pyarrow/tests/test_convert_pandas.py   |  17 +++
 python/pyarrow/tests/test_scalars.py          |  14 ++
 20 files changed, 367 insertions(+), 55 deletions(-)

diff --git a/.gitignore b/.gitignore
index a00cbba065a03..5e28b3685e465 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,6 @@
 *.dylib
 .build_cache_dir
 MANIFEST
+
+cpp/.idea/
+python/.eggs/
\ No newline at end of file
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 52a785d086117..82b62146b0f98 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -542,6 +542,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(DOUBLE, DoubleBuilder);
     BUILDER_CASE(STRING, StringBuilder);
     BUILDER_CASE(BINARY, BinaryBuilder);
+    BUILDER_CASE(FIXED_WIDTH_BINARY, FixedWidthBinaryBuilder);
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
index 1aecdbc633190..b385929d8b10a 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -80,7 +80,7 @@ static void BM_WriteRecordBatch(benchmark::State& state) {  // NOLINT non-const
     int32_t metadata_length;
     int64_t body_length;
     if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
-             default_memory_pool())
+            default_memory_pool())
              .ok()) {
       state.SkipWithError("Failed to write!");
     }
@@ -101,7 +101,7 @@ static void BM_ReadRecordBatch(benchmark::State& state) {  // NOLINT non-const r
   int32_t metadata_length;
   int64_t body_length;
   if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
-           default_memory_pool())
+          default_memory_pool())
            .ok()) {
     state.SkipWithError("Failed to write!");
   }
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index b47b773192774..00ea20cf5dfb1 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -32,8 +32,8 @@
 #include "arrow/ipc/util.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
-#include "arrow/type.h"
 #include "arrow/tensor.h"
+#include "arrow/type.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 6e59845dea76a..72e86774fcca7 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -23,6 +23,7 @@
 
 #include "arrow/api.h"
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 #include "arrow/python/helpers.h"
 #include "arrow/python/util/datetime.h"
@@ -200,18 +201,25 @@ class SeqVisitor {
   int nesting_histogram_[MAX_NESTING_LEVELS];
 };
 
-// Non-exhaustive type inference
-Status InferArrowType(PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
-  *size = PySequence_Size(obj);
+Status InferArrowSize(PyObject* obj, int64_t* size) {
+  *size = static_cast<int64_t>(PySequence_Size(obj));
   if (PyErr_Occurred()) {
     // Not a sequence
     PyErr_Clear();
     return Status::TypeError("Object is not a sequence");
   }
+  return Status::OK();
+}
+
+// Non-exhaustive type inference
+Status InferArrowTypeAndSize(
+    PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
+  RETURN_NOT_OK(InferArrowSize(obj, size));
 
   // For 0-length sequences, refuse to guess
   if (*size == 0) { *out_type = null(); }
 
+  PyDateTime_IMPORT;
   SeqVisitor seq_visitor;
   RETURN_NOT_OK(seq_visitor.Visit(obj));
   RETURN_NOT_OK(seq_visitor.Validate());
@@ -253,7 +261,7 @@ class TypedConverter : public SeqConverter {
 class BoolConverter : public TypedConverter<BooleanBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
-    Py_ssize_t size = PySequence_Size(seq);
+    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
     RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
@@ -275,14 +283,14 @@ class Int64Converter : public TypedConverter<Int64Builder> {
  public:
   Status AppendData(PyObject* seq) override {
     int64_t val;
-    Py_ssize_t size = PySequence_Size(seq);
+    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
     RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
       if (item.obj() == Py_None) {
         typed_builder_->AppendNull();
       } else {
-        val = PyLong_AsLongLong(item.obj());
+        val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
         RETURN_IF_PYERROR();
         typed_builder_->Append(val);
       }
@@ -294,7 +302,7 @@ class Int64Converter : public TypedConverter<Int64Builder> {
 class DateConverter : public TypedConverter<Date64Builder> {
  public:
   Status AppendData(PyObject* seq) override {
-    Py_ssize_t size = PySequence_Size(seq);
+    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
     RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
@@ -312,7 +320,7 @@ class DateConverter : public TypedConverter<Date64Builder> {
 class TimestampConverter : public TypedConverter<TimestampBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
-    Py_ssize_t size = PySequence_Size(seq);
+    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
     RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
@@ -334,7 +342,8 @@ class TimestampConverter : public TypedConverter<TimestampBuilder> {
         epoch.tm_year = 70;
         epoch.tm_mday = 1;
         // Microseconds since the epoch
-        int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000 + us;
+        int64_t val = static_cast<int64_t>(
+            lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000 + us);
         typed_builder_->Append(val);
       }
     }
@@ -346,7 +355,7 @@ class DoubleConverter : public TypedConverter<DoubleBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     double val;
-    Py_ssize_t size = PySequence_Size(seq);
+    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
     RETURN_NOT_OK(typed_builder_->Reserve(size));
     for (int64_t i = 0; i < size; ++i) {
       OwnedRef item(PySequence_GetItem(seq, i));
@@ -369,7 +378,7 @@ class BytesConverter : public TypedConverter<BinaryBuilder> {
     PyObject* bytes_obj;
     OwnedRef tmp;
     const char* bytes;
-    int64_t length;
+    Py_ssize_t length;
     Py_ssize_t size = PySequence_Size(seq);
     for (int64_t i = 0; i < size; ++i) {
       item = PySequence_GetItem(seq, i);
@@ -385,7 +394,8 @@ class BytesConverter : public TypedConverter<BinaryBuilder> {
       } else if (PyBytes_Check(item)) {
         bytes_obj = item;
       } else {
-        return Status::TypeError("Non-string value encountered");
+        return Status::TypeError(
+            "Value that cannot be converted to bytes was encountered");
       }
       // No error checking
       length = PyBytes_GET_SIZE(bytes_obj);
@@ -396,6 +406,41 @@ class BytesConverter : public TypedConverter<BinaryBuilder> {
   }
 };
 
+class FixedWidthBytesConverter : public TypedConverter<FixedWidthBinaryBuilder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    PyObject* item;
+    PyObject* bytes_obj;
+    OwnedRef tmp;
+    Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedWidthBinaryType>(
+        typed_builder_->type())->byte_width();
+    Py_ssize_t size = PySequence_Size(seq);
+    for (int64_t i = 0; i < size; ++i) {
+      item = PySequence_GetItem(seq, i);
+      OwnedRef holder(item);
+
+      if (item == Py_None) {
+        RETURN_NOT_OK(typed_builder_->AppendNull());
+        continue;
+      } else if (PyUnicode_Check(item)) {
+        tmp.reset(PyUnicode_AsUTF8String(item));
+        RETURN_IF_PYERROR();
+        bytes_obj = tmp.obj();
+      } else if (PyBytes_Check(item)) {
+        bytes_obj = item;
+      } else {
+        return Status::TypeError(
+            "Value that cannot be converted to bytes was encountered");
+      }
+      // No error checking
+      RETURN_NOT_OK(CheckPythonBytesAreFixedLength(bytes_obj, expected_length));
+      RETURN_NOT_OK(typed_builder_->Append(
+          reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(bytes_obj))));
+    }
+    return Status::OK();
+  }
+};
+
 class UTF8Converter : public TypedConverter<StringBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
@@ -403,7 +448,7 @@ class UTF8Converter : public TypedConverter<StringBuilder> {
     PyObject* bytes_obj;
     OwnedRef tmp;
     const char* bytes;
-    int64_t length;
+    Py_ssize_t length;
     Py_ssize_t size = PySequence_Size(seq);
     for (int64_t i = 0; i < size; ++i) {
       item = PySequence_GetItem(seq, i);
@@ -465,6 +510,8 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
       return std::make_shared<DoubleConverter>();
     case Type::BINARY:
       return std::make_shared<BytesConverter>();
+    case Type::FIXED_WIDTH_BINARY:
+      return std::make_shared<FixedWidthBytesConverter>();
     case Type::STRING:
       return std::make_shared<UTF8Converter>();
     case Type::LIST:
@@ -472,7 +519,6 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
     case Type::STRUCT:
     default:
       return nullptr;
-      break;
   }
 }
 
@@ -492,6 +538,7 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
 
 Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
     const std::shared_ptr<ArrayBuilder>& builder) {
+  PyDateTime_IMPORT;
   std::shared_ptr<SeqConverter> converter = GetConverter(type);
   if (converter == nullptr) {
     std::stringstream ss;
@@ -506,9 +553,12 @@ Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out) {
   std::shared_ptr<DataType> type;
   int64_t size;
-  PyDateTime_IMPORT;
-  RETURN_NOT_OK(InferArrowType(obj, &size, &type));
+  RETURN_NOT_OK(InferArrowTypeAndSize(obj, &size, &type));
+  return ConvertPySequence(obj, pool, out, type, size);
+}
 
+Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
+    const std::shared_ptr<DataType>& type, int64_t size) {
   // Handle NA / NullType case
   if (type->type == Type::NA) {
     out->reset(new NullArray(size));
@@ -519,9 +569,26 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
   std::shared_ptr<ArrayBuilder> builder;
   RETURN_NOT_OK(MakeBuilder(pool, type, &builder));
   RETURN_NOT_OK(AppendPySequence(obj, type, builder));
-
   return builder->Finish(out);
 }
 
+Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
+    const std::shared_ptr<DataType>& type) {
+  int64_t size;
+  RETURN_NOT_OK(InferArrowSize(obj, &size));
+  return ConvertPySequence(obj, pool, out, type, size);
+}
+
+Status CheckPythonBytesAreFixedLength(PyObject* obj, Py_ssize_t expected_length) {
+  const Py_ssize_t length = PyBytes_GET_SIZE(obj);
+  if (length != expected_length) {
+    std::stringstream ss;
+    ss << "Found byte string of length " << length << ", expected length is "
+       << expected_length;
+    return Status::TypeError(ss.str());
+  }
+  return Status::OK();
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index 7b50990dd557c..00ff0fd8236fc 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -38,16 +38,31 @@ class Status;
 
 namespace py {
 
-ARROW_EXPORT arrow::Status InferArrowType(
+ARROW_EXPORT arrow::Status InferArrowTypeAndSize(
     PyObject* obj, int64_t* size, std::shared_ptr<arrow::DataType>* out_type);
+ARROW_EXPORT arrow::Status InferArrowSize(PyObject* obj, int64_t* size);
 
 ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj,
     const std::shared_ptr<arrow::DataType>& type,
     const std::shared_ptr<arrow::ArrayBuilder>& builder);
 
+// Type and size inference
 ARROW_EXPORT
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out);
 
+// Size inference
+ARROW_EXPORT
+Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
+    const std::shared_ptr<DataType>& type);
+
+// No inference
+ARROW_EXPORT
+Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
+    const std::shared_ptr<DataType>& type, int64_t size);
+
+ARROW_EXPORT Status CheckPythonBytesAreFixedLength(
+    PyObject* obj, Py_ssize_t expected_length);
+
 }  // namespace py
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index db2e90eb8b0ff..68a8d7d7afcf5 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -147,8 +147,8 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
   return Status::OK();
 }
 
-Status AppendObjectStrings(StringBuilder& string_builder, PyObject** objects,
-    int64_t objects_length, bool* have_bytes) {
+Status AppendObjectStrings(int64_t objects_length, StringBuilder* builder,
+    PyObject** objects, bool* have_bytes) {
   PyObject* obj;
 
   for (int64_t i = 0; i < objects_length; ++i) {
@@ -160,15 +160,45 @@ Status AppendObjectStrings(StringBuilder& string_builder, PyObject** objects,
         return Status::TypeError("failed converting unicode to UTF8");
       }
       const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
-      Status s = string_builder.Append(PyBytes_AS_STRING(obj), length);
+      Status s = builder->Append(PyBytes_AS_STRING(obj), length);
       Py_DECREF(obj);
       if (!s.ok()) { return s; }
     } else if (PyBytes_Check(obj)) {
       *have_bytes = true;
       const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
-      RETURN_NOT_OK(string_builder.Append(PyBytes_AS_STRING(obj), length));
+      RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
     } else {
-      string_builder.AppendNull();
+      builder->AppendNull();
+    }
+  }
+
+  return Status::OK();
+}
+
+static Status AppendObjectFixedWidthBytes(int64_t objects_length, int byte_width,
+    FixedWidthBinaryBuilder* builder, PyObject** objects) {
+  PyObject* obj;
+
+  for (int64_t i = 0; i < objects_length; ++i) {
+    obj = objects[i];
+    if (PyUnicode_Check(obj)) {
+      obj = PyUnicode_AsUTF8String(obj);
+      if (obj == NULL) {
+        PyErr_Clear();
+        return Status::TypeError("failed converting unicode to UTF8");
+      }
+
+      RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
+      Status s =
+          builder->Append(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)));
+      Py_DECREF(obj);
+      RETURN_NOT_OK(s);
+    } else if (PyBytes_Check(obj)) {
+      RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
+      RETURN_NOT_OK(
+          builder->Append(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj))));
+    } else {
+      builder->AppendNull();
     }
   }
 
@@ -192,6 +222,13 @@ struct WrapBytes<BinaryArray> {
   }
 };
 
+template <>
+struct WrapBytes<FixedWidthBinaryArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
+  }
+};
+
 static inline bool ListTypeSupported(const Type::type type_id) {
   switch (type_id) {
     case Type::UINT8:
@@ -226,7 +263,7 @@ class PandasConverter : public TypeVisitor {
         arr_(reinterpret_cast<PyArrayObject*>(ao)),
         mask_(nullptr) {
     if (mo != nullptr && mo != Py_None) { mask_ = reinterpret_cast<PyArrayObject*>(mo); }
-    length_ = PyArray_SIZE(arr_);
+    length_ = static_cast<int64_t>(PyArray_SIZE(arr_));
   }
 
   bool is_strided() const {
@@ -241,7 +278,7 @@ class PandasConverter : public TypeVisitor {
     RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
 
     null_bitmap_data_ = null_bitmap_->mutable_data();
-    memset(null_bitmap_data_, 0, null_bytes);
+    memset(null_bitmap_data_, 0, static_cast<size_t>(null_bytes));
 
     return Status::OK();
   }
@@ -321,6 +358,8 @@ class PandasConverter : public TypeVisitor {
       const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
 
   Status ConvertObjectStrings(std::shared_ptr<Array>* out);
+  Status ConvertObjectFixedWidthBytes(
+      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
   Status ConvertBooleans(std::shared_ptr<Array>* out);
   Status ConvertDates(std::shared_ptr<Array>* out);
   Status ConvertLists(const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
@@ -402,13 +441,13 @@ Status PandasConverter::ConvertObjectStrings(std::shared_ptr<Array>* out) {
   // and unicode mixed in the object array
 
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  StringBuilder string_builder(pool_);
-  RETURN_NOT_OK(string_builder.Resize(length_));
+  StringBuilder builder(pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
 
   Status s;
   bool have_bytes = false;
-  RETURN_NOT_OK(AppendObjectStrings(string_builder, objects, length_, &have_bytes));
-  RETURN_NOT_OK(string_builder.Finish(out));
+  RETURN_NOT_OK(AppendObjectStrings(length_, &builder, objects, &have_bytes));
+  RETURN_NOT_OK(builder.Finish(out));
 
   if (have_bytes) {
     const auto& arr = static_cast<const StringArray&>(*out->get());
@@ -418,6 +457,20 @@ Status PandasConverter::ConvertObjectStrings(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
+Status PandasConverter::ConvertObjectFixedWidthBytes(
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+  PyAcquireGIL lock;
+
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  FixedWidthBinaryBuilder builder(pool_, type);
+  RETURN_NOT_OK(builder.Resize(length_));
+  RETURN_NOT_OK(AppendObjectFixedWidthBytes(length_,
+      std::dynamic_pointer_cast<FixedWidthBinaryType>(builder.type())->byte_width(),
+      &builder, objects));
+  RETURN_NOT_OK(builder.Finish(out));
+  return Status::OK();
+}
+
 Status PandasConverter::ConvertBooleans(std::shared_ptr<Array>* out) {
   PyAcquireGIL lock;
 
@@ -474,6 +527,8 @@ Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
     switch (type_->type) {
       case Type::STRING:
         return ConvertObjectStrings(out);
+      case Type::FIXED_WIDTH_BINARY:
+        return ConvertObjectFixedWidthBytes(type_, out);
       case Type::BOOL:
         return ConvertBooleans(out);
       case Type::DATE64:
@@ -543,7 +598,7 @@ inline Status PandasConverter::ConvertTypedLists(
       int64_t size;
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(list_builder.Append(true));
-      RETURN_NOT_OK(InferArrowType(objects[i], &size, &inferred_type));
+      RETURN_NOT_OK(InferArrowTypeAndSize(objects[i], &size, &inferred_type));
       if (inferred_type->type != type->type) {
         std::stringstream ss;
         ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
@@ -577,14 +632,14 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
       // TODO(uwe): Support more complex numpy array structures
       RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
 
-      int64_t size = PyArray_DIM(numpy_array, 0);
+      int64_t size = static_cast<int64_t>(PyArray_DIM(numpy_array, 0));
       auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
-      RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
+      RETURN_NOT_OK(AppendObjectStrings(size, value_builder.get(), data, &have_bytes));
     } else if (PyList_Check(objects[i])) {
       int64_t size;
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(list_builder.Append(true));
-      RETURN_NOT_OK(InferArrowType(objects[i], &size, &inferred_type));
+      RETURN_NOT_OK(InferArrowTypeAndSize(objects[i], &size, &inferred_type));
       if (inferred_type->type != Type::STRING) {
         std::stringstream ss;
         ss << inferred_type->ToString() << " cannot be converted to STRING.";
@@ -832,7 +887,7 @@ inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values
     // Upcast to double, set NaN as appropriate
 
     for (int i = 0; i < arr->length(); ++i) {
-      *out_values++ = prim_arr->IsNull(i) ? NAN : in_values[i];
+      *out_values++ = prim_arr->IsNull(i) ? NAN : static_cast<double>(in_values[i]);
     }
   }
 }
@@ -924,6 +979,36 @@ inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values)
   return Status::OK();
 }
 
+inline Status ConvertFixedWidthBinary(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = static_cast<FixedWidthBinaryArray*>(data.chunk(c).get());
+
+    const uint8_t* data_ptr;
+    int32_t length =
+        std::dynamic_pointer_cast<FixedWidthBinaryType>(arr->type())->byte_width();
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (has_nulls && arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        data_ptr = arr->GetValue(i);
+        *out_values = WrapBytes<FixedWidthBinaryArray>::Wrap(data_ptr, length);
+        if (*out_values == nullptr) {
+          PyErr_Clear();
+          std::stringstream ss;
+          ss << "Wrapping "
+             << std::string(reinterpret_cast<const char*>(data_ptr), length) << " failed";
+          return Status::UnknownError(ss.str());
+        }
+      }
+      ++out_values;
+    }
+  }
+  return Status::OK();
+}
+
 template <typename ArrowType>
 inline Status ConvertListsLike(
     const std::shared_ptr<Column>& col, PyObject** out_values) {
@@ -1058,6 +1143,8 @@ class ObjectBlock : public PandasBlock {
       RETURN_NOT_OK(ConvertBinaryLike<BinaryArray>(data, out_buffer));
     } else if (type == Type::STRING) {
       RETURN_NOT_OK(ConvertBinaryLike<StringArray>(data, out_buffer));
+    } else if (type == Type::FIXED_WIDTH_BINARY) {
+      RETURN_NOT_OK(ConvertFixedWidthBinary(data, out_buffer));
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
       switch (list_type->value_type()->type) {
@@ -1487,6 +1574,7 @@ class DataFrameBlockCreator {
           break;
         case Type::STRING:
         case Type::BINARY:
+        case Type::FIXED_WIDTH_BINARY:
           output_type = PandasBlock::OBJECT;
           break;
         case Type::DATE64:
@@ -1751,6 +1839,7 @@ class ArrowDeserializer {
       CONVERT_CASE(DOUBLE);
       CONVERT_CASE(BINARY);
       CONVERT_CASE(STRING);
+      CONVERT_CASE(FIXED_WIDTH_BINARY);
       CONVERT_CASE(DATE64);
       CONVERT_CASE(TIMESTAMP);
       CONVERT_CASE(DICTIONARY);
@@ -1845,6 +1934,7 @@ class ArrowDeserializer {
     return ConvertBinaryLike<StringArray>(data_, out_values);
   }
 
+  // Binary strings
   template <int T2>
   inline typename std::enable_if<T2 == Type::BINARY, Status>::type ConvertValues() {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
@@ -1852,6 +1942,15 @@ class ArrowDeserializer {
     return ConvertBinaryLike<BinaryArray>(data_, out_values);
   }
 
+  // Fixed length binary strings
+  template <int TYPE>
+  inline typename std::enable_if<TYPE == Type::FIXED_WIDTH_BINARY, Status>::type
+  ConvertValues() {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    return ConvertFixedWidthBinary(data_, out_values);
+  }
+
 #define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \
   case Type::ArrowEnum:                                    \
     return ConvertListsLike<ArrowType>(col_, out_values);
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index b22f07dd6345f..697d47c541003 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -38,9 +38,10 @@ namespace arrow {
 #define ARROW_LOG_INTERNAL(level) ::arrow::internal::CerrLog(level)
 #define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level)
 
-#define ARROW_CHECK(condition)                               \
-  (condition) ? 0 : ::arrow::internal::FatalLog(ARROW_FATAL) \
-                        << __FILE__ << __LINE__ << " Check failed: " #condition " "
+#define ARROW_CHECK(condition)                           \
+  (condition) ? 0                                        \
+              : ::arrow::internal::FatalLog(ARROW_FATAL) \
+                    << __FILE__ << __LINE__ << " Check failed: " #condition " "
 
 #ifdef NDEBUG
 #define ARROW_DFATAL ARROW_WARNING
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index dce438910151b..66b6038617944 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -55,7 +55,7 @@
                             Int8Value, Int16Value, Int32Value, Int64Value,
                             UInt8Value, UInt16Value, UInt32Value, UInt64Value,
                             FloatValue, DoubleValue, ListValue,
-                            BinaryValue, StringValue)
+                            BinaryValue, StringValue, FixedWidthBinaryValue)
 
 import pyarrow.schema as _schema
 
@@ -65,7 +65,8 @@
                             timestamp, date32, date64,
                             float_, double, binary, string,
                             list_, struct, dictionary, field,
-                            DataType, Field, Schema, schema)
+                            DataType, FixedWidthBinaryType,
+                            Field, Schema, schema)
 
 
 from pyarrow.table import Column, RecordBatch, Table, concat_tables
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index c3e7997aa823c..a7241c6a47e31 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -24,9 +24,11 @@ from pyarrow.schema cimport DataType
 
 from cpython cimport PyObject
 
+
 cdef extern from "Python.h":
     int PySlice_Check(object)
 
+
 cdef class Array:
     cdef:
         shared_ptr[CArray] sp_array
@@ -38,6 +40,7 @@ cdef class Array:
     cdef init(self, const shared_ptr[CArray]& sp_array)
     cdef getitem(self, int64_t i)
 
+
 cdef object box_array(const shared_ptr[CArray]& sp_array)
 
 
@@ -52,6 +55,7 @@ cdef class NumericArray(Array):
 cdef class IntegerArray(NumericArray):
     pass
 
+
 cdef class FloatingPointArray(NumericArray):
     pass
 
@@ -96,6 +100,10 @@ cdef class DoubleArray(FloatingPointArray):
     pass
 
 
+cdef class FixedWidthBinaryArray(Array):
+    pass
+
+
 cdef class ListArray(Array):
     pass
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 6cae1966cb16e..289baf2993081 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -37,6 +37,7 @@ cimport pyarrow.scalar as scalar
 from pyarrow.scalar import NA
 
 from pyarrow.schema cimport (DataType, Field, Schema, DictionaryType,
+                             FixedWidthBinaryType,
                              box_data_type)
 import pyarrow.schema as schema
 
@@ -197,7 +198,11 @@ cdef class Array:
         if type is None:
             check_status(pyarrow.ConvertPySequence(list_obj, pool, &sp_array))
         else:
-            raise NotImplementedError()
+            check_status(
+                pyarrow.ConvertPySequence(
+                    list_obj, pool, &sp_array, type.sp_type
+                )
+            )
 
         return box_array(sp_array)
 
@@ -385,6 +390,7 @@ cdef class Date64Array(NumericArray):
 cdef class TimestampArray(NumericArray):
     pass
 
+
 cdef class Time32Array(NumericArray):
     pass
 
@@ -392,6 +398,7 @@ cdef class Time32Array(NumericArray):
 cdef class Time64Array(NumericArray):
     pass
 
+
 cdef class FloatArray(FloatingPointArray):
     pass
 
@@ -400,6 +407,10 @@ cdef class DoubleArray(FloatingPointArray):
     pass
 
 
+cdef class FixedWidthBinaryArray(Array):
+    pass
+
+
 cdef class ListArray(Array):
     pass
 
@@ -506,7 +517,8 @@ cdef dict _array_classes = {
     Type_LIST: ListArray,
     Type_BINARY: BinaryArray,
     Type_STRING: StringArray,
-    Type_DICTIONARY: DictionaryArray
+    Type_DICTIONARY: DictionaryArray,
+    Type_FIXED_WIDTH_BINARY: FixedWidthBinaryArray,
 }
 
 cdef object box_array(const shared_ptr[CArray]& sp_array):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 8e428b40b8f8b..b44ade5298eb3 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -45,6 +45,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         Type_TIME64" arrow::Type::TIME64"
         Type_BINARY" arrow::Type::BINARY"
         Type_STRING" arrow::Type::STRING"
+        Type_FIXED_WIDTH_BINARY" arrow::Type::FIXED_WIDTH_BINARY"
 
         Type_LIST" arrow::Type::LIST"
         Type_STRUCT" arrow::Type::STRUCT"
@@ -139,6 +140,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
 
+    cdef cppclass CFixedWidthBinaryType" arrow::FixedWidthBinaryType"(CFixedWidthType):
+        CFixedWidthBinaryType(int byte_width)
+        int byte_width()
+
     cdef cppclass CField" arrow::Field":
         c_string name
         shared_ptr[CDataType] type
@@ -203,6 +208,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CDoubleArray" arrow::DoubleArray"(CArray):
         double Value(int i)
 
+    cdef cppclass CFixedWidthBinaryArray" arrow::FixedWidthBinaryArray"(CArray):
+        const uint8_t* GetValue(int i)
+
     cdef cppclass CListArray" arrow::ListArray"(CArray):
         const int32_t* raw_value_offsets()
         int32_t value_offset(int i)
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index c3fdf4b070ee0..8142c1c06ff75 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -30,6 +30,9 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
     CStatus ConvertPySequence(object obj, CMemoryPool* pool,
                               shared_ptr[CArray]* out)
+    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
+                              shared_ptr[CArray]* out,
+                              const shared_ptr[CDataType]& type)
 
     CStatus PandasDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
 
diff --git a/python/pyarrow/scalar.pxd b/python/pyarrow/scalar.pxd
index 551aeb9697bf7..e9cc3cb487cbc 100644
--- a/python/pyarrow/scalar.pxd
+++ b/python/pyarrow/scalar.pxd
@@ -61,6 +61,11 @@ cdef class ListValue(ArrayValue):
 cdef class StringValue(ArrayValue):
     pass
 
+
+cdef class FixedWidthBinaryValue(ArrayValue):
+    pass
+
+
 cdef object box_scalar(DataType type,
                        const shared_ptr[CArray]& sp_array,
                        int64_t index)
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 1b7e67b356a2f..f4a1c9e08eb64 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -224,6 +224,22 @@ cdef class ListValue(ArrayValue):
         return result
 
 
+cdef class FixedWidthBinaryValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CFixedWidthBinaryArray* ap
+            CFixedWidthBinaryType* ap_type
+            int32_t length
+            const char* data
+        ap = <CFixedWidthBinaryArray*> self.sp_array.get()
+        ap_type = <CFixedWidthBinaryType*> ap.type().get()
+        length = ap_type.byte_width()
+        data = <const char*> ap.GetValue(self.index)
+        return cp.PyBytes_FromStringAndSize(data, length)
+
+
+
 cdef dict _scalar_classes = {
     Type_BOOL: BooleanValue,
     Type_UINT8: Int8Value,
@@ -241,7 +257,8 @@ cdef dict _scalar_classes = {
     Type_DOUBLE: DoubleValue,
     Type_LIST: ListValue,
     Type_BINARY: BinaryValue,
-    Type_STRING: StringValue
+    Type_STRING: StringValue,
+    Type_FIXED_WIDTH_BINARY: FixedWidthBinaryValue,
 }
 
 cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
index 15ee5f19ee5d9..c0c2c709b2744 100644
--- a/python/pyarrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -19,6 +19,7 @@ from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CDataType,
                                         CDictionaryType,
                                         CTimestampType,
+                                        CFixedWidthBinaryType,
                                         CField, CSchema)
 
 cdef class DataType:
@@ -39,6 +40,11 @@ cdef class TimestampType(DataType):
         const CTimestampType* ts_type
 
 
+cdef class FixedWidthBinaryType(DataType):
+    cdef:
+        const CFixedWidthBinaryType* fixed_width_binary_type
+
+
 cdef class Field:
     cdef:
         shared_ptr[CField] sp_field
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 4f02901cc9a11..532a318840caf 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -28,6 +28,7 @@ from pyarrow.compat import frombytes, tobytes
 from pyarrow.array cimport Array
 from pyarrow.error cimport check_status
 from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
+                                        CFixedWidthBinaryType,
                                         TimeUnit_SECOND, TimeUnit_MILLI,
                                         TimeUnit_MICRO, TimeUnit_NANO,
                                         Type, TimeUnit)
@@ -52,7 +53,7 @@ cdef class DataType:
         return frombytes(self.type.ToString())
 
     def __repr__(self):
-        return 'DataType({0})'.format(str(self))
+        return '{0.__class__.__name__}({0})'.format(self)
 
     def __richcmp__(DataType self, DataType other, int op):
         if op == cpython.Py_EQ:
@@ -69,9 +70,6 @@ cdef class DictionaryType(DataType):
         DataType.init(self, type)
         self.dict_type = <const CDictionaryType*> type.get()
 
-    def __repr__(self):
-        return 'DictionaryType({0})'.format(str(self))
-
 
 cdef class TimestampType(DataType):
 
@@ -92,8 +90,17 @@ cdef class TimestampType(DataType):
             else:
                 return None
 
-    def __repr__(self):
-        return 'TimestampType({0})'.format(str(self))
+
+cdef class FixedWidthBinaryType(DataType):
+
+    cdef init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.fixed_width_binary_type = <const CFixedWidthBinaryType*> type.get()
+
+    property byte_width:
+
+        def __get__(self):
+            return self.fixed_width_binary_type.byte_width()
 
 
 cdef class Field:
@@ -348,11 +355,24 @@ def string():
     return primitive_type(la.Type_STRING)
 
 
-def binary():
-    """
-    Binary (PyBytes-like) type
+def binary(int length=-1):
+    """Binary (PyBytes-like) type
+
+    Parameters
+    ----------
+    length : int, optional, default -1
+        If length == -1 then return a variable length binary type. If length is
+        greater than or equal to 0 then return a fixed width binary type of
+        width `length`.
     """
-    return primitive_type(la.Type_BINARY)
+    if length == -1:
+        return primitive_type(la.Type_BINARY)
+
+    cdef FixedWidthBinaryType out = FixedWidthBinaryType()
+    cdef shared_ptr[CDataType] fixed_width_binary_type
+    fixed_width_binary_type.reset(new CFixedWidthBinaryType(length))
+    out.init(fixed_width_binary_type)
+    return out
 
 
 def list_(DataType value_type):
@@ -408,6 +428,8 @@ cdef DataType box_data_type(const shared_ptr[CDataType]& type):
         out = DictionaryType()
     elif type.get().type == la.Type_TIMESTAMP:
         out = TimestampType()
+    elif type.get().type == la.Type_FIXED_WIDTH_BINARY:
+        out = FixedWidthBinaryType()
     else:
         out = DataType()
 
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 7915f9766bf67..99251250499d2 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -92,6 +92,19 @@ def test_bytes(self):
         assert arr.type == pyarrow.binary()
         assert arr.to_pylist() == [b'foo', u1, None]
 
+    def test_fixed_width_bytes(self):
+        data = [b'foof', None, b'barb', b'2346']
+        arr = pyarrow.from_pylist(data, type=pyarrow.binary(4))
+        assert len(arr) == 4
+        assert arr.null_count == 1
+        assert arr.type == pyarrow.binary(4)
+        assert arr.to_pylist() == data
+
+    def test_fixed_width_bytes_does_not_accept_varying_lengths(self):
+        data = [b'foo', None, b'barb', b'2346']
+        with self.assertRaises(pyarrow.error.ArrowException):
+            pyarrow.from_pylist(data, type=pyarrow.binary(4))
+
     def test_date(self):
         data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1),
                 datetime.date(2040, 2, 26)]
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index ea7a892a6f2a4..f7cb47f685590 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -244,6 +244,23 @@ def test_bytes_to_binary(self):
         expected = pd.DataFrame({'strings': values2})
         self._check_pandas_roundtrip(df, expected)
 
+    def test_fixed_width_bytes(self):
+        values = [b'foo', None, b'bar', None, None, b'hey']
+        df = pd.DataFrame({'strings': values})
+        schema = A.Schema.from_fields([A.field('strings', A.binary(3))])
+        table = A.Table.from_pandas(df, schema=schema)
+        assert table.schema[0].type == schema[0].type
+        assert table.schema[0].name == schema[0].name
+        result = table.to_pandas()
+        tm.assert_frame_equal(result, df)
+
+    def test_fixed_width_bytes_does_not_accept_varying_lengths(self):
+        values = [b'foo', None, b'ba', None, None, b'hey']
+        df = pd.DataFrame({'strings': values})
+        schema = A.Schema.from_fields([A.field('strings', A.binary(3))])
+        with self.assertRaises(A.error.ArrowException):
+            A.Table.from_pandas(df, schema=schema)
+
     def test_timestamps_notimezone_no_nulls(self):
         df = pd.DataFrame({
             'datetime64': np.array([
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index d56481c06d0f8..265ce8d3a58a1 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -87,6 +87,20 @@ def test_bytes(self):
         assert v == b'bar'
         assert isinstance(v, bytes)
 
+    def test_fixed_width_bytes(self):
+        data = [b'foof', None, b'barb']
+        arr = A.from_pylist(data, type=A.binary(4))
+
+        v = arr[0]
+        assert isinstance(v, A.FixedWidthBinaryValue)
+        assert v.as_py() == b'foof'
+
+        assert arr[1] is A.NA
+
+        v = arr[2].as_py()
+        assert v == b'barb'
+        assert isinstance(v, bytes)
+
     def test_list(self):
         arr = A.from_pylist([['foo', None], None, ['bar'], []])
 

From f5967ed682e63dd752d0120573bb33f42dd56e27 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 31 Mar 2017 10:20:20 -0400
Subject: [PATCH 0440/1644] ARROW-603: [C++] Add RecordBatch::Validate method,
 call in RecordBatch ctor in debug builds

This function will help catch malformed RecordBatch objects during development

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #466 from wesm/ARROW-603 and squashes the following commits:

dfdb048 [Wes McKinney] Fix incorrect clang-tidy name
5a51f69 [Wes McKinney] Add RecordBatch::Validate method, call in RecordBatch ctor in debug builds
---
 cpp/cmake_modules/FindClangTools.cmake | 29 ++++++----
 cpp/src/arrow/io/io-hdfs-test.cc       |  7 +--
 cpp/src/arrow/table-test.cc            | 76 ++++++++++++++++++--------
 cpp/src/arrow/table.cc                 | 25 +++++++++
 cpp/src/arrow/table.h                  |  4 ++
 5 files changed, 104 insertions(+), 37 deletions(-)

diff --git a/cpp/cmake_modules/FindClangTools.cmake b/cpp/cmake_modules/FindClangTools.cmake
index c07c7d244493e..0e9430ba29195 100644
--- a/cpp/cmake_modules/FindClangTools.cmake
+++ b/cpp/cmake_modules/FindClangTools.cmake
@@ -27,16 +27,21 @@
 # This module defines
 #  CLANG_TIDY_BIN, The  path to the clang tidy binary
 #  CLANG_TIDY_FOUND, Whether clang tidy was found
-#  CLANG_FORMAT_BIN, The path to the clang format binary 
+#  CLANG_FORMAT_BIN, The path to the clang format binary
 #  CLANG_TIDY_FOUND, Whether clang format was found
 
-find_program(CLANG_TIDY_BIN 
-  NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6  clang-tidy
-  PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin 
+find_program(CLANG_TIDY_BIN
+  NAMES clang-tidy-4.0
+  clang-tidy-3.9
+  clang-tidy-3.8
+  clang-tidy-3.7
+  clang-tidy-3.6
+  clang-tidy
+  PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
         NO_DEFAULT_PATH
 )
 
-if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" ) 
+if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
   set(CLANG_TIDY_FOUND 0)
   message("clang-tidy not found")
 else()
@@ -44,17 +49,21 @@ else()
   message("clang-tidy found at ${CLANG_TIDY_BIN}")
 endif()
 
-find_program(CLANG_FORMAT_BIN 
-  NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6  clang-format
-  PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin 
+find_program(CLANG_FORMAT_BIN
+  NAMES clang-format-4.0
+  clang-format-3.9
+  clang-format-3.8
+  clang-format-3.7
+  clang-format-3.6
+  clang-format
+  PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
         NO_DEFAULT_PATH
 )
 
-if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" ) 
+if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
   set(CLANG_FORMAT_FOUND 0)
   message("clang-format not found")
 else()
   set(CLANG_FORMAT_FOUND 1)
   message("clang-format found at ${CLANG_FORMAT_BIN}")
 endif()
-
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index af59e96a1448f..f3140be0b2dac 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -78,10 +78,9 @@ class TestHdfsClient : public ::testing::Test {
     LibHdfsShim* driver_shim;
 
     client_ = nullptr;
-    scratch_dir_ =
-        boost::filesystem::unique_path(
-            boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
-            .string();
+    scratch_dir_ = boost::filesystem::unique_path(
+        boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
+                       .string();
 
     loaded_driver_ = false;
 
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 38533063cbc07..cd32f4a387290 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -127,8 +127,8 @@ TEST_F(TestColumn, BasicAPI) {
   arrays.push_back(MakePrimitive<Int32Array>(100, 10));
   arrays.push_back(MakePrimitive<Int32Array>(100, 20));
 
-  auto field = std::make_shared<Field>("c0", int32());
-  column_.reset(new Column(field, arrays));
+  auto f0 = field("c0", int32());
+  column_.reset(new Column(f0, arrays));
 
   ASSERT_EQ("c0", column_->name());
   ASSERT_TRUE(column_->type()->Equals(int32()));
@@ -137,7 +137,7 @@ TEST_F(TestColumn, BasicAPI) {
   ASSERT_EQ(3, column_->data()->num_chunks());
 
   // nullptr array should not break
-  column_.reset(new Column(field, std::shared_ptr<Array>(nullptr)));
+  column_.reset(new Column(f0, std::shared_ptr<Array>(nullptr)));
   ASSERT_NE(column_.get(), nullptr);
 }
 
@@ -146,13 +146,13 @@ TEST_F(TestColumn, ChunksInhomogeneous) {
   arrays.push_back(MakePrimitive<Int32Array>(100));
   arrays.push_back(MakePrimitive<Int32Array>(100, 10));
 
-  auto field = std::make_shared<Field>("c0", int32());
-  column_.reset(new Column(field, arrays));
+  auto f0 = field("c0", int32());
+  column_.reset(new Column(f0, arrays));
 
   ASSERT_OK(column_->ValidateData());
 
   arrays.push_back(MakePrimitive<Int16Array>(100, 10));
-  column_.reset(new Column(field, arrays));
+  column_.reset(new Column(f0, arrays));
   ASSERT_RAISES(Invalid, column_->ValidateData());
 }
 
@@ -164,8 +164,8 @@ TEST_F(TestColumn, Equals) {
   arrays_one_.push_back(array);
   arrays_another_.push_back(array);
 
-  one_field_ = std::make_shared<Field>("column", int32());
-  another_field_ = std::make_shared<Field>("column", int32());
+  one_field_ = field("column", int32());
+  another_field_ = field("column", int32());
 
   Construct();
   ASSERT_TRUE(one_col_->Equals(one_col_));
@@ -174,13 +174,13 @@ TEST_F(TestColumn, Equals) {
   ASSERT_TRUE(one_col_->Equals(*another_col_.get()));
 
   // Field is different
-  another_field_ = std::make_shared<Field>("two", int32());
+  another_field_ = field("two", int32());
   Construct();
   ASSERT_FALSE(one_col_->Equals(another_col_));
   ASSERT_FALSE(one_col_->Equals(*another_col_.get()));
 
   // ChunkedArray is different
-  another_field_ = std::make_shared<Field>("column", int32());
+  another_field_ = field("column", int32());
   arrays_another_.push_back(array);
   Construct();
   ASSERT_FALSE(one_col_->Equals(another_col_));
@@ -190,9 +190,9 @@ TEST_F(TestColumn, Equals) {
 class TestTable : public TestBase {
  public:
   void MakeExample1(int length) {
-    auto f0 = std::make_shared<Field>("f0", int32());
-    auto f1 = std::make_shared<Field>("f1", uint8());
-    auto f2 = std::make_shared<Field>("f2", int16());
+    auto f0 = field("f0", int32());
+    auto f1 = field("f1", uint8());
+    auto f2 = field("f2", int16());
 
     vector<shared_ptr<Field>> fields = {f0, f1, f2};
     schema_ = std::make_shared<Schema>(fields);
@@ -279,9 +279,9 @@ TEST_F(TestTable, Equals) {
 
   ASSERT_TRUE(table_->Equals(*table_));
   // Differing schema
-  auto f0 = std::make_shared<Field>("f3", int32());
-  auto f1 = std::make_shared<Field>("f4", uint8());
-  auto f2 = std::make_shared<Field>("f5", int16());
+  auto f0 = field("f3", int32());
+  auto f1 = field("f4", uint8());
+  auto f2 = field("f5", int16());
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto other_schema = std::make_shared<Schema>(fields);
   ASSERT_FALSE(table_->Equals(Table(other_schema, columns_)));
@@ -389,9 +389,9 @@ class TestRecordBatch : public TestBase {};
 TEST_F(TestRecordBatch, Equals) {
   const int length = 10;
 
-  auto f0 = std::make_shared<Field>("f0", int32());
-  auto f1 = std::make_shared<Field>("f1", uint8());
-  auto f2 = std::make_shared<Field>("f2", int16());
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8());
+  auto f2 = field("f2", int16());
 
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto schema = std::make_shared<Schema>(fields);
@@ -401,21 +401,51 @@ TEST_F(TestRecordBatch, Equals) {
   auto a2 = MakePrimitive<Int16Array>(length);
 
   RecordBatch b1(schema, length, {a0, a1, a2});
-  RecordBatch b2(schema, 5, {a0, a1, a2});
   RecordBatch b3(schema, length, {a0, a1});
   RecordBatch b4(schema, length, {a0, a1, a1});
 
   ASSERT_TRUE(b1.Equals(b1));
-  ASSERT_FALSE(b1.Equals(b2));
   ASSERT_FALSE(b1.Equals(b3));
   ASSERT_FALSE(b1.Equals(b4));
 }
 
+#ifdef NDEBUG
+// In debug builds, RecordBatch ctor aborts if you construct an invalid one
+
+TEST_F(TestRecordBatch, Validate) {
+  const int length = 10;
+
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8());
+  auto f2 = field("f2", int16());
+
+  auto schema = std::shared_ptr<Schema>(new Schema({f0, f1, f2}));
+
+  auto a0 = MakePrimitive<Int32Array>(length);
+  auto a1 = MakePrimitive<UInt8Array>(length);
+  auto a2 = MakePrimitive<Int16Array>(length);
+  auto a3 = MakePrimitive<Int16Array>(5);
+
+  RecordBatch b1(schema, length, {a0, a1, a2});
+
+  ASSERT_OK(b1.Validate());
+
+  // Length mismatch
+  RecordBatch b2(schema, length, {a0, a1, a3});
+  ASSERT_RAISES(Invalid, b2.Validate());
+
+  // Type mismatch
+  RecordBatch b3(schema, length, {a0, a1, a0});
+  ASSERT_RAISES(Invalid, b3.Validate());
+}
+
+#endif
+
 TEST_F(TestRecordBatch, Slice) {
   const int length = 10;
 
-  auto f0 = std::make_shared<Field>("f0", int32());
-  auto f1 = std::make_shared<Field>("f1", uint8());
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8());
 
   vector<shared_ptr<Field>> fields = {f0, f1};
   auto schema = std::make_shared<Schema>(fields);
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 8e283f4da9bb7..da61fbb9a6daf 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -139,6 +139,11 @@ Status Column::ValidateData() {
 // ----------------------------------------------------------------------
 // RecordBatch methods
 
+void AssertBatchValid(const RecordBatch& batch) {
+  Status s = batch.Validate();
+  if (!s.ok()) { DCHECK(false) << s.ToString(); }
+}
+
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
     const std::vector<std::shared_ptr<Array>>& columns)
     : schema_(schema), num_rows_(num_rows), columns_(columns) {}
@@ -190,6 +195,26 @@ std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length)
   return std::make_shared<RecordBatch>(schema_, num_rows, arrays);
 }
 
+Status RecordBatch::Validate() const {
+  for (int i = 0; i < num_columns(); ++i) {
+    const Array& arr = *columns_[i];
+    if (arr.length() != num_rows_) {
+      std::stringstream ss;
+      ss << "Number of rows in column " << i << " did not match batch: " << arr.length()
+         << " vs " << num_rows_;
+      return Status::Invalid(ss.str());
+    }
+    const auto& schema_type = *schema_->field(i)->type;
+    if (!arr.type()->Equals(schema_type)) {
+      std::stringstream ss;
+      ss << "Column " << i << " type not match schema: " << arr.type()->ToString()
+         << " vs " << schema_type.ToString();
+      return Status::Invalid(ss.str());
+    }
+  }
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // Table methods
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 7b739c9a1b314..0f35dd888fe2f 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -140,6 +140,10 @@ class ARROW_EXPORT RecordBatch {
   std::shared_ptr<RecordBatch> Slice(int64_t offset);
   std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length);
 
+  /// Returns error status is there is something wrong with the record batch
+  /// contents, like a schema/array mismatch or inconsistent lengths
+  Status Validate() const;
+
  private:
   std::shared_ptr<Schema> schema_;
   int64_t num_rows_;

From 005826f804f7db668e8e165fad45c9c1cd1de0cf Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 31 Mar 2017 10:27:20 -0400
Subject: [PATCH 0441/1644] ARROW-719: [GLib] Release source archive

I don't know about his approach is good but it will help you to consider what approach is better.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #448 from kou/glib-release-source-archive and squashes the following commits:

76b0110 [Kouhei Sutou] [GLib] Release source archive
---
 dev/release/02-source.sh | 39 ++++++++++++++++++++++++++++++++++++++-
 dev/release/run-rat.sh   | 19 +++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 dev/release/02-source.sh

diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
old mode 100644
new mode 100755
index bdaa5cc9340fe..924b94fd6caa0
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -54,9 +54,46 @@ echo "Using commit $release_hash"
 
 tarball=$tag.tar.gz
 
+extract_dir=tmp-apache-arrow
+rm -rf $extract_dir
 # be conservative and use the release hash, even though git produces the same
 # archive (identical hashes) using the scm tag
-git archive $release_hash --prefix $tag/ -o $tarball
+git archive $release_hash --prefix $extract_dir/ | tar xf -
+
+# build Apache Arrow C++ before building Apache Arrow GLib because
+# Apache Arrow GLib requires Apache Arrow C++.
+mkdir -p $extract_dir/cpp/build
+cpp_install_dir=$PWD/$extract_dir/cpp/install
+cd $extract_dir/cpp/build
+cmake .. \
+  -DCMAKE_INSTALL_PREFIX=$cpp_install_dir \
+  -DARROW_BUILD_TESTS=no
+make -j8
+make install
+cd -
+
+# build source archive for Apache Arrow GLib by "make dist".
+cd $extract_dir/c_glib
+./autogen.sh
+./configure \
+  PKG_CONFIG_PATH=$cpp_install_dir/lib/pkgconfig \
+  --enable-gtk-doc
+LD_LIBRARY_PATH=$cpp_install_dir/lib make -j8
+make dist
+tar xzf *.tar.gz
+rm *.tar.gz
+cd -
+rm -rf tmp-c_glib/
+mv $extract_dir/c_glib/apache-arrow-glib-* tmp-c_glib/
+rm -rf $extract_dir
+
+# replace c_glib/ by tar.gz generated by "make dist"
+rm -rf $tag
+git archive $release_hash --prefix $tag/ | tar xf -
+rm -rf $tag/c_glib
+mv tmp-c_glib $tag/c_glib
+tar czf $tarball $tag
+rm -rf $tag
 
 ${SOURCE_DIR}/run-rat.sh $tarball
 
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index e26dd589695b1..a3c12a0ce8a92 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -40,6 +40,25 @@ $RAT $1 \
   -e __init__.pxd \
   -e __init__.py \
   -e requirements.txt \
+  -e version \
+  -e "*.m4" \
+  -e configure \
+  -e config.sub \
+  -e config.h.in \
+  -e compile \
+  -e missing \
+  -e install-sh \
+  -e config.guess \
+  -e depcomp \
+  -e ltmain.sh \
+  -e arrow-glib.types \
+  -e arrow-glib-sections.txt \
+  -e arrow-glib-overrides.txt \
+  -e gtk-doc.make \
+  -e "*.html" \
+  -e "*.css" \
+  -e "*.png" \
+  -e "*.devhelp2" \
   > rat.txt
 cat rat.txt
 UNAPPROVED=`cat rat.txt  | grep "Unknown Licenses" | head -n 1 | cut -d " " -f 1`

From ad8a0cfeced7f86e21fcaa63de3e55ce42b8f962 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 31 Mar 2017 10:30:08 -0400
Subject: [PATCH 0442/1644] ARROW-739: Don't install jemalloc in parallel

Alternative fix proposal. Couldn't trigger it locally though.

Author: Uwe L. Korn <uwelk@xhochy.com>
Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #456 from xhochy/ARROW-739 and squashes the following commits:

c1cad56 [Robert Nishihara] Replace MAKE -> CMAKE_MAKE_PROGRAM in CMakeLists.txt.
f121072 [Uwe L. Korn] Add install to install command
e8803b8 [Uwe L. Korn] ARROW-739: Don't install jemalloc in parallel
---
 cpp/CMakeLists.txt | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index aa8ea31b831e3..5dcf58c0f232d 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -637,14 +637,16 @@ if (ARROW_JEMALLOC)
         URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
         CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
         BUILD_IN_SOURCE 1
-        BUILD_COMMAND ${MAKE}
-        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}")
+        BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
+        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
+        INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j1 install)
     else()
       ExternalProject_Add(jemalloc_ep
         URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
         CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
         BUILD_IN_SOURCE 1
-        BUILD_COMMAND ${MAKE})
+        BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
+        INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j1 install)
     endif()
   else()
     set(JEMALLOC_VENDORED 0)

From e5b682760614a2a51e9587afbb4b9b676e59e5a9 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 31 Mar 2017 12:57:46 -0400
Subject: [PATCH 0443/1644] ARROW-744: [GLib] Re-add an assertion for
 garrow_table_new() test

Author: Kouhei Sutou <kou@clear-code.com>

Closes #469 from kou/glib-re-add-assertion-to-garrow-table-new-test and squashes the following commits:

64c2e50 [Kouhei Sutou] [GLib] Re-add an assertion for garrow_table_new() test
---
 c_glib/test/test-table.rb | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/c_glib/test/test-table.rb b/c_glib/test/test-table.rb
index 0583e8139e47a..e2b71b31e44c0 100644
--- a/c_glib/test/test-table.rb
+++ b/c_glib/test/test-table.rb
@@ -30,6 +30,25 @@ def test_columns
         Arrow::Column.new(fields[1], build_boolean_array([false])),
       ]
       table = Arrow::Table.new(schema, columns)
+
+      data = table.n_columns.times.collect do |i|
+        column = table.get_column(i)
+        values = []
+        column.data.chunks.each do |chunk|
+          chunk.length.times do |j|
+            values << chunk.get_value(j)
+          end
+        end
+        [
+          column.name,
+          values,
+        ]
+      end
+      assert_equal([
+                     ["visible", [true]],
+                     ["valid", [false]],
+                   ],
+                   data)
     end
   end
 

From 4e77d3382f6cc6450c79b1ebefea0bbd1f2dd379 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 31 Mar 2017 12:58:55 -0400
Subject: [PATCH 0444/1644] ARROW-746: [GLib] Add garrow_array_get_data_type()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #470 from kou/glib-add-garrow-array-get-data-type and squashes the following commits:

3f4de67 [Kouhei Sutou] [GLib] Add garrow_array_get_data_type()
---
 c_glib/arrow-glib/array.cpp | 15 +++++++++++++++
 c_glib/arrow-glib/array.h   |  3 ++-
 c_glib/test/test-array.rb   |  6 ++++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 5dacb07ba8710..b084054f9af87 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -24,6 +24,7 @@
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/binary-array.h>
 #include <arrow-glib/boolean-array.h>
+#include <arrow-glib/data-type.hpp>
 #include <arrow-glib/double-array.h>
 #include <arrow-glib/float-array.h>
 #include <arrow-glib/int8-array.h>
@@ -173,6 +174,20 @@ garrow_array_get_n_nulls(GArrowArray *array)
   return arrow_array->null_count();
 }
 
+/**
+ * garrow_array_get_data_type:
+ * @array: A #GArrowArray.
+ *
+ * Returns: (transfer full): The #GArrowDataType for the array.
+ */
+GArrowDataType *
+garrow_array_get_data_type(GArrowArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto arrow_data_type = arrow_array->type();
+  return garrow_data_type_new_raw(&arrow_data_type);
+}
+
 /**
  * garrow_array_slice:
  * @array: A #GArrowArray.
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 9b1fa7e1e4a31..6467db5ff45db 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <glib-object.h>
+#include <arrow-glib/data-type.h>
 
 G_BEGIN_DECLS
 
@@ -60,6 +60,7 @@ GType          garrow_array_get_type    (void) G_GNUC_CONST;
 gint64         garrow_array_get_length  (GArrowArray *array);
 gint64         garrow_array_get_offset  (GArrowArray *array);
 gint64         garrow_array_get_n_nulls (GArrowArray *array);
+GArrowDataType *garrow_array_get_data_type(GArrowArray *array);
 GArrowArray   *garrow_array_slice       (GArrowArray *array,
                                          gint64 offset,
                                          gint64 length);
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
index d68827cb85b1d..c427f0200ef02 100644
--- a/c_glib/test/test-array.rb
+++ b/c_glib/test/test-array.rb
@@ -31,6 +31,12 @@ def test_n_nulls
     assert_equal(2, array.n_nulls)
   end
 
+  def test_data_type
+    builder = Arrow::BooleanArrayBuilder.new
+    array = builder.finish
+    assert_equal(Arrow::BooleanDataType.new, array.data_type)
+  end
+
   def test_slice
     builder = Arrow::BooleanArrayBuilder.new
     builder.append(true)

From 067cd4ebfbd9be9b607658a2a249017cc6db84f9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 31 Mar 2017 13:00:11 -0400
Subject: [PATCH 0445/1644] ARROW-630: [C++] Create boolean batches for IPC
 testing, properly account for nonzero offset

This fixes a couple bugs; boolean IPC was not being tested directly like the other types (it was implicitly by integration tests, though)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #460 from wesm/ARROW-630 and squashes the following commits:

f9448a7 [Wes McKinney] Create boolean batches for IPC testing, properly account for offset in unloading, comparison
---
 cpp/src/arrow/compare.cc                 |  4 +++-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  2 +-
 cpp/src/arrow/ipc/test-common.h          | 22 ++++++++++++++++++++++
 cpp/src/arrow/ipc/writer.cc              |  7 ++++++-
 4 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index c2580b4f54109..4cd617e6021df 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -294,13 +294,15 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
 
   Status Visit(const BooleanArray& left) {
     const auto& right = static_cast<const BooleanArray&>(right_);
+
     if (left.null_count() > 0) {
       const uint8_t* left_data = left.data()->data();
       const uint8_t* right_data = right.data()->data();
 
       for (int64_t i = 0; i < left.length(); ++i) {
         if (!left.IsNull(i) &&
-            BitUtil::GetBit(left_data, i) != BitUtil::GetBit(right_data, i)) {
+            BitUtil::GetBit(left_data, i + left.offset()) !=
+                BitUtil::GetBit(right_data, i + right.offset())) {
           result_ = false;
           return Status::OK();
         }
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 74ca017df5cf1..c900d0ba37ed2 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -104,7 +104,7 @@ TEST_F(TestSchemaMetadata, NestedFields) {
   ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch,  \
       &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,    \
       &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes, \
-      &MakeFWBinary);
+      &MakeFWBinary, &MakeBooleanBatch);
 
 class IpcTestFixture : public io::MemoryMapFixture {
  public:
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 583f909d071e6..134a5caee8ec4 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -138,6 +138,28 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
 
 typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
 
+Status MakeBooleanBatch(std::shared_ptr<RecordBatch>* out) {
+  const int length = 1000;
+
+  // Make the schema
+  auto f0 = field("f0", boolean());
+  auto f1 = field("f1", boolean());
+  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+
+  std::vector<uint8_t> values(length);
+  std::vector<uint8_t> valid_bytes(length);
+  test::random_null_bytes(length, 0.5, values.data());
+  test::random_null_bytes(length, 0.1, valid_bytes.data());
+
+  auto data = test::bytes_to_null_buffer(values);
+  auto null_bitmap = test::bytes_to_null_buffer(valid_bytes);
+
+  auto a0 = std::make_shared<BooleanArray>(length, data, null_bitmap, -1);
+  auto a1 = std::make_shared<BooleanArray>(length, data, nullptr, 0);
+  out->reset(new RecordBatch(schema, length, {a0, a1}));
+  return Status::OK();
+}
+
 Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
   const int length = 10;
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 249ef201c66bb..0867382e6b1b0 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -281,7 +281,12 @@ class RecordBatchWriter : public ArrayVisitor {
   }
 
   Status Visit(const BooleanArray& array) override {
-    buffers_.push_back(array.data());
+    std::shared_ptr<Buffer> bits = array.data();
+    if (array.offset() != 0) {
+      RETURN_NOT_OK(
+          CopyBitmap(pool_, bits->data(), array.offset(), array.length(), &bits));
+    }
+    buffers_.push_back(bits);
     return Status::OK();
   }
 

From d75d7a96ca21bb2c1cfcf3bce8d09c2f24a5b8a6 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sat, 1 Apr 2017 00:00:24 -0400
Subject: [PATCH 0446/1644] =?UTF-8?q?ARROW-736:=20[Python]=20Mixed-type=20?=
 =?UTF-8?q?object=20DataFrame=20columns=20should=20not=20silently=20co?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…erce to an Arrow type by default

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #465 from cpcloud/ARROW-736 and squashes the following commits:

fd09def [Phillip Cloud] Update cmake
bcf6236 [Phillip Cloud] Rename and move
4a18014 [Phillip Cloud] Move test
e80efe1 [Phillip Cloud] Use OwnedRef instead of horror
b2df3e9 [Phillip Cloud] Fix python error handling and make compatible with python27
84d33b4 [Phillip Cloud] ARROW-736: Mixed-type object DataFrame columns should not silently coerce to an Arrow type by default
---
 cpp/src/arrow/python/CMakeLists.txt           |  2 +-
 cpp/src/arrow/python/pandas_convert.cc        | 52 ++++++++++++++++---
 cpp/src/arrow/python/pandas_convert.h         |  4 ++
 .../python/{pandas-test.cc => python-test.cc} | 35 +++++++++++--
 python/pyarrow/tests/test_convert_builtin.py  |  5 ++
 python/pyarrow/tests/test_convert_pandas.py   |  5 ++
 6 files changed, 91 insertions(+), 12 deletions(-)
 rename cpp/src/arrow/python/{pandas-test.cc => python-test.cc} (70%)

diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 03f5afc624b34..faaad89656f92 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -88,6 +88,6 @@ install(FILES
 #     INSTALL_RPATH "\$ORIGIN")
 
 if (ARROW_BUILD_TESTS)
-  ADD_ARROW_TEST(pandas-test
+  ADD_ARROW_TEST(python-test
     STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}")
 endif()
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 68a8d7d7afcf5..ae9b17ca9ac86 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -46,6 +46,7 @@
 #include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 
 namespace arrow {
@@ -167,8 +168,10 @@ Status AppendObjectStrings(int64_t objects_length, StringBuilder* builder,
       *have_bytes = true;
       const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
       RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
+    } else if (PyObject_is_null(obj)) {
+      RETURN_NOT_OK(builder->AppendNull());
     } else {
-      builder->AppendNull();
+      return InvalidConversion(obj, "string or bytes");
     }
   }
 
@@ -197,8 +200,10 @@ static Status AppendObjectFixedWidthBytes(int64_t objects_length, int byte_width
       RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
       RETURN_NOT_OK(
           builder->Append(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj))));
+    } else if (PyObject_is_null(obj)) {
+      RETURN_NOT_OK(builder->AppendNull());
     } else {
-      builder->AppendNull();
+      return InvalidConversion(obj, "string or bytes");
     }
   }
 
@@ -413,6 +418,32 @@ inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>*
   return Status::OK();
 }
 
+Status InvalidConversion(PyObject* obj, const std::string& expected_type_name) {
+  OwnedRef type(PyObject_Type(obj));
+  RETURN_IF_PYERROR();
+  DCHECK_NE(type.obj(), nullptr);
+
+  OwnedRef type_name(PyObject_GetAttrString(type.obj(), "__name__"));
+  RETURN_IF_PYERROR();
+  DCHECK_NE(type_name.obj(), nullptr);
+
+  OwnedRef bytes_obj(PyUnicode_AsUTF8String(type_name.obj()));
+  RETURN_IF_PYERROR();
+  DCHECK_NE(bytes_obj.obj(), nullptr);
+
+  Py_ssize_t size = PyBytes_GET_SIZE(bytes_obj.obj());
+  const char* bytes = PyBytes_AS_STRING(bytes_obj.obj());
+
+  DCHECK_NE(bytes, nullptr) << "bytes from type(...).__name__ were null";
+
+  std::string cpp_type_name(bytes, size);
+
+  std::stringstream ss;
+  ss << "Python object of type " << cpp_type_name << " is not None and is not a "
+     << expected_type_name << " object";
+  return Status::TypeError(ss.str());
+}
+
 Status PandasConverter::ConvertDates(std::shared_ptr<Array>* out) {
   PyAcquireGIL lock;
 
@@ -427,8 +458,10 @@ Status PandasConverter::ConvertDates(std::shared_ptr<Array>* out) {
     if (PyDate_CheckExact(obj)) {
       PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(obj);
       date_builder.Append(PyDate_to_ms(pydate));
-    } else {
+    } else if (PyObject_is_null(obj)) {
       date_builder.AppendNull();
+    } else {
+      return InvalidConversion(obj, "date");
     }
   }
   return date_builder.Finish(out);
@@ -483,14 +516,18 @@ Status PandasConverter::ConvertBooleans(std::shared_ptr<Array>* out) {
   memset(bitmap, 0, nbytes);
 
   int64_t null_count = 0;
+  PyObject* obj;
   for (int64_t i = 0; i < length_; ++i) {
-    if (objects[i] == Py_True) {
+    obj = objects[i];
+    if (obj == Py_True) {
       BitUtil::SetBit(bitmap, i);
       BitUtil::SetBit(null_bitmap_data_, i);
-    } else if (objects[i] != Py_False) {
+    } else if (obj == Py_False) {
+      BitUtil::SetBit(null_bitmap_data_, i);
+    } else if (PyObject_is_null(obj)) {
       ++null_count;
     } else {
-      BitUtil::SetBit(null_bitmap_data_, i);
+      return InvalidConversion(obj, "bool");
     }
   }
 
@@ -551,7 +588,8 @@ Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
       } else if (PyDate_CheckExact(objects[i])) {
         return ConvertDates(out);
       } else {
-        return Status::TypeError("unhandled python type");
+        return InvalidConversion(
+            const_cast<PyObject*>(objects[i]), "string, bool, or date");
       }
     }
   }
diff --git a/cpp/src/arrow/python/pandas_convert.h b/cpp/src/arrow/python/pandas_convert.h
index 12644d98da156..105c1598d3936 100644
--- a/cpp/src/arrow/python/pandas_convert.h
+++ b/cpp/src/arrow/python/pandas_convert.h
@@ -24,6 +24,7 @@
 #include <Python.h>
 
 #include <memory>
+#include <string>
 
 #include "arrow/util/visibility.h"
 
@@ -73,6 +74,9 @@ ARROW_EXPORT
 Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
 
+ARROW_EXPORT
+Status InvalidConversion(PyObject* obj, const std::string& expected_type_name);
+
 }  // namespace py
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/python/pandas-test.cc b/cpp/src/arrow/python/python-test.cc
similarity index 70%
rename from cpp/src/arrow/python/pandas-test.cc
rename to cpp/src/arrow/python/python-test.cc
index a4e640b83718b..01e30f5a36ce8 100644
--- a/cpp/src/arrow/python/pandas-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -17,19 +17,18 @@
 
 #include "gtest/gtest.h"
 
-#include <cstdint>
 #include <memory>
-#include <string>
-#include <vector>
+
+#include <Python.h>
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
-#include "arrow/type.h"
 
 #include "arrow/python/common.h"
 #include "arrow/python/pandas_convert.h"
+#include "arrow/python/builtin_convert.h"
 
 namespace arrow {
 namespace py {
@@ -65,5 +64,33 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   Py_END_ALLOW_THREADS;
 }
 
+TEST(BuiltinConversionTest, TestMixedTypeFails) {
+  PyAcquireGIL lock;
+  MemoryPool* pool = default_memory_pool();
+  std::shared_ptr<Array> arr;
+
+  OwnedRef list_ref(PyList_New(3));
+  PyObject* list = list_ref.obj();
+
+  ASSERT_NE(list, nullptr);
+
+  PyObject* str = PyUnicode_FromString("abc");
+  ASSERT_NE(str, nullptr);
+
+  PyObject* integer = PyLong_FromLong(1234L);
+  ASSERT_NE(integer, nullptr);
+
+  PyObject* doub = PyFloat_FromDouble(123.0234);
+  ASSERT_NE(doub, nullptr);
+
+  // This steals a reference to each object, so we don't need to decref them later
+  // just the list
+  ASSERT_EQ(PyList_SetItem(list, 0, str), 0);
+  ASSERT_EQ(PyList_SetItem(list, 1, integer), 0);
+  ASSERT_EQ(PyList_SetItem(list, 2, doub), 0);
+
+  ASSERT_RAISES(UnknownError, ConvertPySequence(list, pool, &arr));
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 99251250499d2..3309ba018628d 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -157,3 +157,8 @@ def test_list_of_int(self):
         assert arr.null_count == 1
         assert arr.type == pyarrow.list_(pyarrow.int64())
         assert arr.to_pylist() == data
+
+    def test_mixed_types_fails(self):
+        data = ['a', 1, 2.0]
+        with self.assertRaises(pyarrow.error.ArrowException):
+            pyarrow.from_pylist(data)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index f7cb47f685590..3f19b68fe0a03 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -398,3 +398,8 @@ def test_category(self):
         ]
         for values in arrays:
             self._check_array_roundtrip(values)
+
+    def test_mixed_types_fails(self):
+        data = pd.DataFrame({'a': ['a', 1, 2.0]})
+        with self.assertRaises(A.error.ArrowException):
+            A.Table.from_pandas(data)

From 9f5e17448f984a709be36bfd6f731852a775e1b0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 1 Apr 2017 11:18:30 -0400
Subject: [PATCH 0447/1644] ARROW-733: [C++/Python] Rename FixedWidthBinary to
 FixedSizeBinary for consistency with FixedSizeList

As discussed on JIRA

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #473 from wesm/ARROW-733 and squashes the following commits:

0e30af3 [Wes McKinney] Rename FixedWidthBinary to FixedSizeBinary for consistency with FixedSizeList type
---
 cpp/src/arrow/array-test.cc                  | 34 ++++++++++----------
 cpp/src/arrow/array.cc                       | 11 +++----
 cpp/src/arrow/array.h                        |  6 ++--
 cpp/src/arrow/builder.cc                     | 24 +++++++-------
 cpp/src/arrow/builder.h                      |  6 ++--
 cpp/src/arrow/compare.cc                     |  8 ++---
 cpp/src/arrow/ipc/json-internal.cc           | 24 +++++++-------
 cpp/src/arrow/ipc/metadata.cc                | 14 ++++----
 cpp/src/arrow/ipc/test-common.h              |  8 ++---
 cpp/src/arrow/ipc/writer.cc                  |  2 +-
 cpp/src/arrow/loader.cc                      |  6 ++--
 cpp/src/arrow/pretty_print-test.cc           |  6 ++--
 cpp/src/arrow/pretty_print.cc                |  7 ++--
 cpp/src/arrow/python/builtin_convert.cc      |  6 ++--
 cpp/src/arrow/python/pandas_convert.cc       | 30 ++++++++---------
 cpp/src/arrow/type-test.cc                   | 16 ++++-----
 cpp/src/arrow/type.cc                        | 14 ++++----
 cpp/src/arrow/type.h                         | 14 ++++----
 cpp/src/arrow/type_fwd.h                     |  6 ++--
 cpp/src/arrow/type_traits.h                  |  6 ++--
 cpp/src/arrow/visitor.cc                     |  4 +--
 cpp/src/arrow/visitor.h                      |  4 +--
 cpp/src/arrow/visitor_inline.h               |  4 +--
 format/Schema.fbs                            |  4 +--
 python/pyarrow/__init__.py                   |  4 +--
 python/pyarrow/array.pxd                     |  2 +-
 python/pyarrow/array.pyx                     |  6 ++--
 python/pyarrow/includes/libarrow.pxd         |  8 ++---
 python/pyarrow/scalar.pxd                    |  2 +-
 python/pyarrow/scalar.pyx                    | 12 +++----
 python/pyarrow/schema.pxd                    |  6 ++--
 python/pyarrow/schema.pyx                    | 22 ++++++-------
 python/pyarrow/tests/test_convert_builtin.py |  4 +--
 python/pyarrow/tests/test_convert_pandas.py  |  4 +--
 python/pyarrow/tests/test_scalars.py         |  4 +--
 35 files changed, 168 insertions(+), 170 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 52f3727d46a15..68b9864301d20 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1080,19 +1080,19 @@ TEST_F(TestBinaryArray, LengthZeroCtor) {
 }
 
 // ----------------------------------------------------------------------
-// FixedWidthBinary tests
+// FixedSizeBinary tests
 
 class TestFWBinaryArray : public ::testing::Test {
  public:
   void SetUp() {}
 
   void InitBuilder(int byte_width) {
-    auto type = fixed_width_binary(byte_width);
-    builder_.reset(new FixedWidthBinaryBuilder(default_memory_pool(), type));
+    auto type = fixed_size_binary(byte_width);
+    builder_.reset(new FixedSizeBinaryBuilder(default_memory_pool(), type));
   }
 
  protected:
-  std::unique_ptr<FixedWidthBinaryBuilder> builder_;
+  std::unique_ptr<FixedSizeBinaryBuilder> builder_;
 };
 
 TEST_F(TestFWBinaryArray, Builder) {
@@ -1114,7 +1114,7 @@ TEST_F(TestFWBinaryArray, Builder) {
   auto CheckResult = [this, &length, &is_valid, &raw_data, &byte_width](
       const Array& result) {
     // Verify output
-    const auto& fw_result = static_cast<const FixedWidthBinaryArray&>(result);
+    const auto& fw_result = static_cast<const FixedSizeBinaryArray&>(result);
 
     ASSERT_EQ(length, result.length());
 
@@ -1169,9 +1169,9 @@ TEST_F(TestFWBinaryArray, Builder) {
 TEST_F(TestFWBinaryArray, EqualsRangeEquals) {
   // Check that we don't compare data in null slots
 
-  auto type = fixed_width_binary(4);
-  FixedWidthBinaryBuilder builder1(default_memory_pool(), type);
-  FixedWidthBinaryBuilder builder2(default_memory_pool(), type);
+  auto type = fixed_size_binary(4);
+  FixedSizeBinaryBuilder builder1(default_memory_pool(), type);
+  FixedSizeBinaryBuilder builder2(default_memory_pool(), type);
 
   ASSERT_OK(builder1.Append("foo1"));
   ASSERT_OK(builder1.AppendNull());
@@ -1183,19 +1183,19 @@ TEST_F(TestFWBinaryArray, EqualsRangeEquals) {
   ASSERT_OK(builder1.Finish(&array1));
   ASSERT_OK(builder2.Finish(&array2));
 
-  const auto& a1 = static_cast<const FixedWidthBinaryArray&>(*array1);
-  const auto& a2 = static_cast<const FixedWidthBinaryArray&>(*array2);
+  const auto& a1 = static_cast<const FixedSizeBinaryArray&>(*array1);
+  const auto& a2 = static_cast<const FixedSizeBinaryArray&>(*array2);
 
-  FixedWidthBinaryArray equal1(type, 2, a1.data(), a1.null_bitmap(), 1);
-  FixedWidthBinaryArray equal2(type, 2, a2.data(), a1.null_bitmap(), 1);
+  FixedSizeBinaryArray equal1(type, 2, a1.data(), a1.null_bitmap(), 1);
+  FixedSizeBinaryArray equal2(type, 2, a2.data(), a1.null_bitmap(), 1);
 
   ASSERT_TRUE(equal1.Equals(equal2));
   ASSERT_TRUE(equal1.RangeEquals(equal2, 0, 2, 0));
 }
 
 TEST_F(TestFWBinaryArray, ZeroSize) {
-  auto type = fixed_width_binary(0);
-  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
+  auto type = fixed_size_binary(0);
+  FixedSizeBinaryBuilder builder(default_memory_pool(), type);
 
   ASSERT_OK(builder.Append(nullptr));
   ASSERT_OK(builder.Append(nullptr));
@@ -1207,7 +1207,7 @@ TEST_F(TestFWBinaryArray, ZeroSize) {
   std::shared_ptr<Array> array;
   ASSERT_OK(builder.Finish(&array));
 
-  const auto& fw_array = static_cast<const FixedWidthBinaryArray&>(*array);
+  const auto& fw_array = static_cast<const FixedSizeBinaryArray&>(*array);
 
   // data is never allocated
   ASSERT_TRUE(fw_array.data() == nullptr);
@@ -1218,8 +1218,8 @@ TEST_F(TestFWBinaryArray, ZeroSize) {
 }
 
 TEST_F(TestFWBinaryArray, Slice) {
-  auto type = fixed_width_binary(4);
-  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
+  auto type = fixed_size_binary(4);
+  FixedSizeBinaryBuilder builder(default_memory_pool(), type);
 
   vector<string> strings = {"foo1", "foo2", "foo3", "foo4", "foo5"};
   vector<uint8_t> is_null = {0, 1, 0, 0, 0};
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index b25411a1c5938..bd20654bc87d4 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -280,18 +280,17 @@ std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const
 // ----------------------------------------------------------------------
 // Fixed width binary
 
-FixedWidthBinaryArray::FixedWidthBinaryArray(const std::shared_ptr<DataType>& type,
+FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<DataType>& type,
     int64_t length, const std::shared_ptr<Buffer>& data,
     const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
     : PrimitiveArray(type, length, data, null_bitmap, null_count, offset) {
-  DCHECK(type->type == Type::FIXED_WIDTH_BINARY);
-  byte_width_ = static_cast<const FixedWidthBinaryType&>(*type).byte_width();
+  DCHECK(type->type == Type::FIXED_SIZE_BINARY);
+  byte_width_ = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
 }
 
-std::shared_ptr<Array> FixedWidthBinaryArray::Slice(
-    int64_t offset, int64_t length) const {
+std::shared_ptr<Array> FixedSizeBinaryArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<FixedWidthBinaryArray>(
+  return std::make_shared<FixedSizeBinaryArray>(
       type_, length, data_, null_bitmap_, kUnknownNullCount, offset);
 }
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 53b640853d5a6..9f0e73914da84 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -347,11 +347,11 @@ class ARROW_EXPORT StringArray : public BinaryArray {
 // ----------------------------------------------------------------------
 // Fixed width binary
 
-class ARROW_EXPORT FixedWidthBinaryArray : public PrimitiveArray {
+class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
  public:
-  using TypeClass = FixedWidthBinaryType;
+  using TypeClass = FixedSizeBinaryType;
 
-  FixedWidthBinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
+  FixedSizeBinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& data,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0);
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 82b62146b0f98..40b81cf015ab4 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -438,51 +438,51 @@ Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
 // ----------------------------------------------------------------------
 // Fixed width binary
 
-FixedWidthBinaryBuilder::FixedWidthBinaryBuilder(
+FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(
     MemoryPool* pool, const std::shared_ptr<DataType>& type)
     : ArrayBuilder(pool, type), byte_builder_(pool) {
-  DCHECK(type->type == Type::FIXED_WIDTH_BINARY);
-  byte_width_ = static_cast<const FixedWidthBinaryType&>(*type).byte_width();
+  DCHECK(type->type == Type::FIXED_SIZE_BINARY);
+  byte_width_ = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
 }
 
-Status FixedWidthBinaryBuilder::Append(const uint8_t* value) {
+Status FixedSizeBinaryBuilder::Append(const uint8_t* value) {
   RETURN_NOT_OK(Reserve(1));
   UnsafeAppendToBitmap(true);
   return byte_builder_.Append(value, byte_width_);
 }
 
-Status FixedWidthBinaryBuilder::Append(
+Status FixedSizeBinaryBuilder::Append(
     const uint8_t* data, int64_t length, const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
   UnsafeAppendToBitmap(valid_bytes, length);
   return byte_builder_.Append(data, length * byte_width_);
 }
 
-Status FixedWidthBinaryBuilder::Append(const std::string& value) {
+Status FixedSizeBinaryBuilder::Append(const std::string& value) {
   return Append(reinterpret_cast<const uint8_t*>(value.c_str()));
 }
 
-Status FixedWidthBinaryBuilder::AppendNull() {
+Status FixedSizeBinaryBuilder::AppendNull() {
   RETURN_NOT_OK(Reserve(1));
   UnsafeAppendToBitmap(false);
   return byte_builder_.Advance(byte_width_);
 }
 
-Status FixedWidthBinaryBuilder::Init(int64_t elements) {
+Status FixedSizeBinaryBuilder::Init(int64_t elements) {
   DCHECK_LT(elements, std::numeric_limits<int64_t>::max());
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
   return byte_builder_.Resize(elements * byte_width_);
 }
 
-Status FixedWidthBinaryBuilder::Resize(int64_t capacity) {
+Status FixedSizeBinaryBuilder::Resize(int64_t capacity) {
   DCHECK_LT(capacity, std::numeric_limits<int64_t>::max());
   RETURN_NOT_OK(byte_builder_.Resize(capacity * byte_width_));
   return ArrayBuilder::Resize(capacity);
 }
 
-Status FixedWidthBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
+Status FixedSizeBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<Buffer> data = byte_builder_.Finish();
-  *out = std::make_shared<FixedWidthBinaryArray>(
+  *out = std::make_shared<FixedSizeBinaryArray>(
       type_, length_, data, null_bitmap_, null_count_);
   return Status::OK();
 }
@@ -542,7 +542,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(DOUBLE, DoubleBuilder);
     BUILDER_CASE(STRING, StringBuilder);
     BUILDER_CASE(BINARY, BinaryBuilder);
-    BUILDER_CASE(FIXED_WIDTH_BINARY, FixedWidthBinaryBuilder);
+    BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder);
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index bd957b38280da..61207a334db32 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -390,11 +390,11 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
 };
 
 // ----------------------------------------------------------------------
-// FixedWidthBinaryBuilder
+// FixedSizeBinaryBuilder
 
-class ARROW_EXPORT FixedWidthBinaryBuilder : public ArrayBuilder {
+class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
  public:
-  FixedWidthBinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+  FixedSizeBinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 
   Status Append(const uint8_t* value);
   Status Append(
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 4cd617e6021df..7451439a875d6 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -202,8 +202,8 @@ class RangeEqualsVisitor {
     return Status::OK();
   }
 
-  Status Visit(const FixedWidthBinaryArray& left) {
-    const auto& right = static_cast<const FixedWidthBinaryArray&>(right_);
+  Status Visit(const FixedSizeBinaryArray& left) {
+    const auto& right = static_cast<const FixedSizeBinaryArray&>(right_);
 
     int32_t width = left.byte_width();
 
@@ -648,8 +648,8 @@ class TypeEqualsVisitor {
     return Status::OK();
   }
 
-  Status Visit(const FixedWidthBinaryType& left) {
-    const auto& right = static_cast<const FixedWidthBinaryType&>(right_);
+  Status Visit(const FixedSizeBinaryType& left) {
+    const auto& right = static_cast<const FixedSizeBinaryType&>(right_);
     result_ = left.byte_width() == right.byte_width();
     return Status::OK();
   }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 9572a0a81898d..1e2385b73f82c 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -189,7 +189,7 @@ class JsonSchemaWriter {
     }
   }
 
-  void WriteTypeMetadata(const FixedWidthBinaryType& type) {
+  void WriteTypeMetadata(const FixedSizeBinaryType& type) {
     writer_->Key("byteWidth");
     writer_->Int(type.byte_width());
   }
@@ -297,8 +297,8 @@ class JsonSchemaWriter {
 
   Status Visit(const BinaryType& type) { return WriteVarBytes("binary", type); }
 
-  Status Visit(const FixedWidthBinaryType& type) {
-    return WritePrimitive("fixedwidthbinary", type);
+  Status Visit(const FixedSizeBinaryType& type) {
+    return WritePrimitive("fixedsizebinary", type);
   }
 
   Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); }
@@ -401,7 +401,7 @@ class JsonArrayWriter {
     }
   }
 
-  void WriteDataValues(const FixedWidthBinaryArray& arr) {
+  void WriteDataValues(const FixedSizeBinaryArray& arr) {
     int32_t width = arr.byte_width();
     for (int64_t i = 0; i < arr.length(); ++i) {
       const char* buf = reinterpret_cast<const char*>(arr.GetValue(i));
@@ -576,13 +576,13 @@ static Status GetFloatingPoint(
   return Status::OK();
 }
 
-static Status GetFixedWidthBinary(
+static Status GetFixedSizeBinary(
     const RjObject& json_type, std::shared_ptr<DataType>* type) {
   const auto& json_byte_width = json_type.FindMember("byteWidth");
   RETURN_NOT_INT("byteWidth", json_byte_width, json_type);
 
   int32_t byte_width = json_byte_width->value.GetInt();
-  *type = fixed_width_binary(byte_width);
+  *type = fixed_size_binary(byte_width);
   return Status::OK();
 }
 
@@ -709,8 +709,8 @@ static Status GetType(const RjObject& json_type,
     *type = utf8();
   } else if (type_name == "binary") {
     *type = binary();
-  } else if (type_name == "fixedwidthbinary") {
-    return GetFixedWidthBinary(json_type, type);
+  } else if (type_name == "fixedsizebinary") {
+    return GetFixedSizeBinary(json_type, type);
   } else if (type_name == "null") {
     *type = null();
   } else if (type_name == "date") {
@@ -896,10 +896,10 @@ class JsonArrayReader {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<FixedWidthBinaryType, T>::value, Status>::type
+  typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value, Status>::type
   ReadArray(const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
       const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-    FixedWidthBinaryBuilder builder(pool_, type);
+    FixedSizeBinaryBuilder builder(pool_, type);
 
     const auto& json_data = json_array.FindMember("DATA");
     RETURN_NOT_ARRAY("DATA", json_data, json_array);
@@ -908,7 +908,7 @@ class JsonArrayReader {
 
     DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length);
 
-    int32_t byte_width = static_cast<const FixedWidthBinaryType&>(*type).byte_width();
+    int32_t byte_width = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
 
     // Allocate space for parsed values
     std::shared_ptr<MutableBuffer> byte_buffer;
@@ -1112,7 +1112,7 @@ class JsonArrayReader {
       TYPE_CASE(DoubleType);
       TYPE_CASE(StringType);
       TYPE_CASE(BinaryType);
-      TYPE_CASE(FixedWidthBinaryType);
+      TYPE_CASE(FixedSizeBinaryType);
       TYPE_CASE(Date32Type);
       TYPE_CASE(Date64Type);
       TYPE_CASE(TimestampType);
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 076a6e792ba40..5007f1309087d 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -230,9 +230,9 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     case flatbuf::Type_Binary:
       *out = binary();
       return Status::OK();
-    case flatbuf::Type_FixedWidthBinary: {
-      auto fw_binary = static_cast<const flatbuf::FixedWidthBinary*>(type_data);
-      *out = fixed_width_binary(fw_binary->byteWidth());
+    case flatbuf::Type_FixedSizeBinary: {
+      auto fw_binary = static_cast<const flatbuf::FixedSizeBinary*>(type_data);
+      *out = fixed_size_binary(fw_binary->byteWidth());
       return Status::OK();
     }
     case flatbuf::Type_Utf8:
@@ -362,10 +362,10 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *out_type = flatbuf::Type_FloatingPoint;
       *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
       break;
-    case Type::FIXED_WIDTH_BINARY: {
-      const auto& fw_type = static_cast<const FixedWidthBinaryType&>(*type);
-      *out_type = flatbuf::Type_FixedWidthBinary;
-      *offset = flatbuf::CreateFixedWidthBinary(fbb, fw_type.byte_width()).Union();
+    case Type::FIXED_SIZE_BINARY: {
+      const auto& fw_type = static_cast<const FixedSizeBinaryType&>(*type);
+      *out_type = flatbuf::Type_FixedSizeBinary;
+      *offset = flatbuf::CreateFixedSizeBinary(fbb, fw_type.byte_width()).Union();
     } break;
     case Type::BINARY:
       *out_type = flatbuf::Type_Binary;
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 134a5caee8ec4..d113531822c96 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -599,14 +599,14 @@ void AppendValues(const std::vector<bool>& is_valid, const std::vector<T>& value
 
 Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
   std::vector<bool> is_valid = {true, true, true, false};
-  auto f0 = field("f0", fixed_width_binary(4));
-  auto f1 = field("f1", fixed_width_binary(0));
+  auto f0 = field("f0", fixed_size_binary(4));
+  auto f1 = field("f1", fixed_size_binary(0));
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
   std::shared_ptr<Array> a1, a2;
 
-  FixedWidthBinaryBuilder b1(default_memory_pool(), f0->type);
-  FixedWidthBinaryBuilder b2(default_memory_pool(), f1->type);
+  FixedSizeBinaryBuilder b1(default_memory_pool(), f0->type);
+  FixedSizeBinaryBuilder b2(default_memory_pool(), f1->type);
 
   std::vector<std::string> values1 = {"foo1", "foo2", "foo3", "foo4"};
   AppendValues(is_valid, values1, &b1);
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 0867382e6b1b0..5330206480928 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -269,7 +269,7 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status Visit(const FixedWidthBinaryArray& array) override {
+  Status Visit(const FixedSizeBinaryArray& array) override {
     auto data = array.data();
     int32_t width = array.byte_width();
 
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
index cc64c4d8264f7..f3347f92e6d87 100644
--- a/cpp/src/arrow/loader.cc
+++ b/cpp/src/arrow/loader.cc
@@ -139,7 +139,7 @@ class ArrayLoader {
 
   template <typename T>
   typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
-                              !std::is_base_of<FixedWidthBinaryType, T>::value &&
+                              !std::is_base_of<FixedSizeBinaryType, T>::value &&
                               !std::is_base_of<DictionaryType, T>::value,
       Status>::type
   Visit(const T& type) {
@@ -152,14 +152,14 @@ class ArrayLoader {
     return LoadBinary<T>();
   }
 
-  Status Visit(const FixedWidthBinaryType& type) {
+  Status Visit(const FixedSizeBinaryType& type) {
     FieldMetadata field_meta;
     std::shared_ptr<Buffer> null_bitmap, data;
 
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
     RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &data));
 
-    result_ = std::make_shared<FixedWidthBinaryArray>(
+    result_ = std::make_shared<FixedSizeBinaryArray>(
         type_, field_meta.length, data, null_bitmap, field_meta.null_count);
     return Status::OK();
   }
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index f21383f0cb06f..80cd9cfe6ac6d 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -78,14 +78,14 @@ TEST_F(TestPrettyPrint, BinaryType) {
   CheckPrimitive<BinaryType, std::string>(0, is_valid, values, ex);
 }
 
-TEST_F(TestPrettyPrint, FixedWidthBinaryType) {
+TEST_F(TestPrettyPrint, FixedSizeBinaryType) {
   std::vector<bool> is_valid = {true, true, false, true, false};
   std::vector<std::string> values = {"foo", "bar", "baz"};
   static const char* ex = R"expected([666F6F, 626172, 62617A])expected";
 
   std::shared_ptr<Array> array;
-  auto type = fixed_width_binary(3);
-  FixedWidthBinaryBuilder builder(default_memory_pool(), type);
+  auto type = fixed_size_binary(3);
+  FixedSizeBinaryBuilder builder(default_memory_pool(), type);
 
   builder.Append(values[0]);
   builder.Append(values[1]);
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 0f67fe5bc52a7..0f46f0306fe08 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -97,9 +97,8 @@ class ArrayPrinter {
   }
 
   template <typename T>
-  inline
-      typename std::enable_if<std::is_same<FixedWidthBinaryArray, T>::value, void>::type
-      WriteDataValues(const T& array) {
+  inline typename std::enable_if<std::is_same<FixedSizeBinaryArray, T>::value, void>::type
+  WriteDataValues(const T& array) {
     int32_t width = array.byte_width();
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) { (*sink_) << ", "; }
@@ -136,7 +135,7 @@ class ArrayPrinter {
 
   template <typename T>
   typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value ||
-                              std::is_base_of<FixedWidthBinaryArray, T>::value ||
+                              std::is_base_of<FixedSizeBinaryArray, T>::value ||
                               std::is_base_of<BinaryArray, T>::value,
       Status>::type
   Visit(const T& array) {
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 72e86774fcca7..6a13fdccdeaff 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -406,13 +406,13 @@ class BytesConverter : public TypedConverter<BinaryBuilder> {
   }
 };
 
-class FixedWidthBytesConverter : public TypedConverter<FixedWidthBinaryBuilder> {
+class FixedWidthBytesConverter : public TypedConverter<FixedSizeBinaryBuilder> {
  public:
   Status AppendData(PyObject* seq) override {
     PyObject* item;
     PyObject* bytes_obj;
     OwnedRef tmp;
-    Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedWidthBinaryType>(
+    Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedSizeBinaryType>(
         typed_builder_->type())->byte_width();
     Py_ssize_t size = PySequence_Size(seq);
     for (int64_t i = 0; i < size; ++i) {
@@ -510,7 +510,7 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
       return std::make_shared<DoubleConverter>();
     case Type::BINARY:
       return std::make_shared<BytesConverter>();
-    case Type::FIXED_WIDTH_BINARY:
+    case Type::FIXED_SIZE_BINARY:
       return std::make_shared<FixedWidthBytesConverter>();
     case Type::STRING:
       return std::make_shared<UTF8Converter>();
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index ae9b17ca9ac86..ddfec1bf45a2e 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -179,7 +179,7 @@ Status AppendObjectStrings(int64_t objects_length, StringBuilder* builder,
 }
 
 static Status AppendObjectFixedWidthBytes(int64_t objects_length, int byte_width,
-    FixedWidthBinaryBuilder* builder, PyObject** objects) {
+    FixedSizeBinaryBuilder* builder, PyObject** objects) {
   PyObject* obj;
 
   for (int64_t i = 0; i < objects_length; ++i) {
@@ -228,7 +228,7 @@ struct WrapBytes<BinaryArray> {
 };
 
 template <>
-struct WrapBytes<FixedWidthBinaryArray> {
+struct WrapBytes<FixedSizeBinaryArray> {
   static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
     return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
   }
@@ -495,10 +495,10 @@ Status PandasConverter::ConvertObjectFixedWidthBytes(
   PyAcquireGIL lock;
 
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  FixedWidthBinaryBuilder builder(pool_, type);
+  FixedSizeBinaryBuilder builder(pool_, type);
   RETURN_NOT_OK(builder.Resize(length_));
   RETURN_NOT_OK(AppendObjectFixedWidthBytes(length_,
-      std::dynamic_pointer_cast<FixedWidthBinaryType>(builder.type())->byte_width(),
+      std::dynamic_pointer_cast<FixedSizeBinaryType>(builder.type())->byte_width(),
       &builder, objects));
   RETURN_NOT_OK(builder.Finish(out));
   return Status::OK();
@@ -564,7 +564,7 @@ Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
     switch (type_->type) {
       case Type::STRING:
         return ConvertObjectStrings(out);
-      case Type::FIXED_WIDTH_BINARY:
+      case Type::FIXED_SIZE_BINARY:
         return ConvertObjectFixedWidthBytes(type_, out);
       case Type::BOOL:
         return ConvertBooleans(out);
@@ -1017,14 +1017,14 @@ inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values)
   return Status::OK();
 }
 
-inline Status ConvertFixedWidthBinary(const ChunkedArray& data, PyObject** out_values) {
+inline Status ConvertFixedSizeBinary(const ChunkedArray& data, PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = static_cast<FixedWidthBinaryArray*>(data.chunk(c).get());
+    auto arr = static_cast<FixedSizeBinaryArray*>(data.chunk(c).get());
 
     const uint8_t* data_ptr;
     int32_t length =
-        std::dynamic_pointer_cast<FixedWidthBinaryType>(arr->type())->byte_width();
+        std::dynamic_pointer_cast<FixedSizeBinaryType>(arr->type())->byte_width();
     const bool has_nulls = data.null_count() > 0;
     for (int64_t i = 0; i < arr->length(); ++i) {
       if (has_nulls && arr->IsNull(i)) {
@@ -1032,7 +1032,7 @@ inline Status ConvertFixedWidthBinary(const ChunkedArray& data, PyObject** out_v
         *out_values = Py_None;
       } else {
         data_ptr = arr->GetValue(i);
-        *out_values = WrapBytes<FixedWidthBinaryArray>::Wrap(data_ptr, length);
+        *out_values = WrapBytes<FixedSizeBinaryArray>::Wrap(data_ptr, length);
         if (*out_values == nullptr) {
           PyErr_Clear();
           std::stringstream ss;
@@ -1181,8 +1181,8 @@ class ObjectBlock : public PandasBlock {
       RETURN_NOT_OK(ConvertBinaryLike<BinaryArray>(data, out_buffer));
     } else if (type == Type::STRING) {
       RETURN_NOT_OK(ConvertBinaryLike<StringArray>(data, out_buffer));
-    } else if (type == Type::FIXED_WIDTH_BINARY) {
-      RETURN_NOT_OK(ConvertFixedWidthBinary(data, out_buffer));
+    } else if (type == Type::FIXED_SIZE_BINARY) {
+      RETURN_NOT_OK(ConvertFixedSizeBinary(data, out_buffer));
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
       switch (list_type->value_type()->type) {
@@ -1612,7 +1612,7 @@ class DataFrameBlockCreator {
           break;
         case Type::STRING:
         case Type::BINARY:
-        case Type::FIXED_WIDTH_BINARY:
+        case Type::FIXED_SIZE_BINARY:
           output_type = PandasBlock::OBJECT;
           break;
         case Type::DATE64:
@@ -1877,7 +1877,7 @@ class ArrowDeserializer {
       CONVERT_CASE(DOUBLE);
       CONVERT_CASE(BINARY);
       CONVERT_CASE(STRING);
-      CONVERT_CASE(FIXED_WIDTH_BINARY);
+      CONVERT_CASE(FIXED_SIZE_BINARY);
       CONVERT_CASE(DATE64);
       CONVERT_CASE(TIMESTAMP);
       CONVERT_CASE(DICTIONARY);
@@ -1982,11 +1982,11 @@ class ArrowDeserializer {
 
   // Fixed length binary strings
   template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::FIXED_WIDTH_BINARY, Status>::type
+  inline typename std::enable_if<TYPE == Type::FIXED_SIZE_BINARY, Status>::type
   ConvertValues() {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertFixedWidthBinary(data_, out_values);
+    return ConvertFixedSizeBinary(data_, out_values);
   }
 
 #define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index b221c80391cde..dafadc168c191 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -155,16 +155,16 @@ TEST(TestStringType, ToString) {
   ASSERT_EQ(str.ToString(), std::string("string"));
 }
 
-TEST(TestFixedWidthBinaryType, ToString) {
-  auto t = fixed_width_binary(10);
-  ASSERT_EQ(t->type, Type::FIXED_WIDTH_BINARY);
-  ASSERT_EQ("fixed_width_binary[10]", t->ToString());
+TEST(TestFixedSizeBinaryType, ToString) {
+  auto t = fixed_size_binary(10);
+  ASSERT_EQ(t->type, Type::FIXED_SIZE_BINARY);
+  ASSERT_EQ("fixed_size_binary[10]", t->ToString());
 }
 
-TEST(TestFixedWidthBinaryType, Equals) {
-  auto t1 = fixed_width_binary(10);
-  auto t2 = fixed_width_binary(10);
-  auto t3 = fixed_width_binary(3);
+TEST(TestFixedSizeBinaryType, Equals) {
+  auto t1 = fixed_size_binary(10);
+  auto t2 = fixed_size_binary(10);
+  auto t3 = fixed_size_binary(3);
 
   ASSERT_TRUE(t1->Equals(t1));
   ASSERT_TRUE(t1->Equals(t2));
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index e6e6f5c3e8bc7..d99551d661d69 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -90,13 +90,13 @@ std::string BinaryType::ToString() const {
   return std::string("binary");
 }
 
-int FixedWidthBinaryType::bit_width() const {
+int FixedSizeBinaryType::bit_width() const {
   return 8 * byte_width();
 }
 
-std::string FixedWidthBinaryType::ToString() const {
+std::string FixedSizeBinaryType::ToString() const {
   std::stringstream ss;
-  ss << "fixed_width_binary[" << byte_width_ << "]";
+  ss << "fixed_size_binary[" << byte_width_ << "]";
   return ss.str();
 }
 
@@ -286,7 +286,7 @@ std::string Schema::ToString() const {
 ACCEPT_VISITOR(NullType);
 ACCEPT_VISITOR(BooleanType);
 ACCEPT_VISITOR(BinaryType);
-ACCEPT_VISITOR(FixedWidthBinaryType);
+ACCEPT_VISITOR(FixedSizeBinaryType);
 ACCEPT_VISITOR(StringType);
 ACCEPT_VISITOR(ListType);
 ACCEPT_VISITOR(StructType);
@@ -324,8 +324,8 @@ TYPE_FACTORY(binary, BinaryType);
 TYPE_FACTORY(date64, Date64Type);
 TYPE_FACTORY(date32, Date32Type);
 
-std::shared_ptr<DataType> fixed_width_binary(int32_t byte_width) {
-  return std::make_shared<FixedWidthBinaryType>(byte_width);
+std::shared_ptr<DataType> fixed_size_binary(int32_t byte_width) {
+  return std::make_shared<FixedSizeBinaryType>(byte_width);
 }
 
 std::shared_ptr<DataType> timestamp(TimeUnit unit) {
@@ -392,7 +392,7 @@ std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
   return {kValidityBuffer, kOffsetBuffer, kValues8};
 }
 
-std::vector<BufferDescr> FixedWidthBinaryType::GetBufferLayout() const {
+std::vector<BufferDescr> FixedSizeBinaryType::GetBufferLayout() const {
   return {kValidityBuffer, BufferDescr(BufferType::DATA, byte_width_ * 8)};
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 4f931907ee79f..6b936f348d4de 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -70,8 +70,8 @@ struct Type {
     // Variable-length bytes (no guarantee of UTF8-ness)
     BINARY,
 
-    // Fixed-width binary. Each value occupies the same number of bytes
-    FIXED_WIDTH_BINARY,
+    // Fixed-size binary. Each value occupies the same number of bytes
+    FIXED_SIZE_BINARY,
 
     // int32_t days since the UNIX epoch
     DATE32,
@@ -353,12 +353,12 @@ struct ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
 };
 
 // BinaryType type is represents lists of 1-byte values.
-class ARROW_EXPORT FixedWidthBinaryType : public FixedWidthType {
+class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType {
  public:
-  static constexpr Type::type type_id = Type::FIXED_WIDTH_BINARY;
+  static constexpr Type::type type_id = Type::FIXED_SIZE_BINARY;
 
-  explicit FixedWidthBinaryType(int32_t byte_width)
-      : FixedWidthType(Type::FIXED_WIDTH_BINARY), byte_width_(byte_width) {}
+  explicit FixedSizeBinaryType(int32_t byte_width)
+      : FixedWidthType(Type::FIXED_SIZE_BINARY), byte_width_(byte_width) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -630,7 +630,7 @@ class ARROW_EXPORT Schema {
 // ----------------------------------------------------------------------
 // Factory functions
 
-std::shared_ptr<DataType> ARROW_EXPORT fixed_width_binary(int32_t byte_width);
+std::shared_ptr<DataType> ARROW_EXPORT fixed_size_binary(int32_t byte_width);
 
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type);
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 04ddf7e74dd1d..2e27ce9858964 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -51,9 +51,9 @@ struct BinaryType;
 class BinaryArray;
 class BinaryBuilder;
 
-class FixedWidthBinaryType;
-class FixedWidthBinaryArray;
-class FixedWidthBinaryBuilder;
+class FixedSizeBinaryType;
+class FixedSizeBinaryArray;
+class FixedSizeBinaryBuilder;
 
 struct StringType;
 class StringArray;
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index b73d5a68d257e..353b638fed894 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -257,9 +257,9 @@ struct TypeTraits<BinaryType> {
 };
 
 template <>
-struct TypeTraits<FixedWidthBinaryType> {
-  using ArrayType = FixedWidthBinaryArray;
-  using BuilderType = FixedWidthBinaryBuilder;
+struct TypeTraits<FixedSizeBinaryType> {
+  using ArrayType = FixedSizeBinaryArray;
+  using BuilderType = FixedSizeBinaryBuilder;
   constexpr static bool is_parameter_free = false;
 };
 
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index 9200e0ff228a3..117578965ccc4 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -43,7 +43,7 @@ ARRAY_VISITOR_DEFAULT(FloatArray);
 ARRAY_VISITOR_DEFAULT(DoubleArray);
 ARRAY_VISITOR_DEFAULT(BinaryArray);
 ARRAY_VISITOR_DEFAULT(StringArray);
-ARRAY_VISITOR_DEFAULT(FixedWidthBinaryArray);
+ARRAY_VISITOR_DEFAULT(FixedSizeBinaryArray);
 ARRAY_VISITOR_DEFAULT(Date32Array);
 ARRAY_VISITOR_DEFAULT(Date64Array);
 ARRAY_VISITOR_DEFAULT(Time32Array);
@@ -82,7 +82,7 @@ TYPE_VISITOR_DEFAULT(FloatType);
 TYPE_VISITOR_DEFAULT(DoubleType);
 TYPE_VISITOR_DEFAULT(StringType);
 TYPE_VISITOR_DEFAULT(BinaryType);
-TYPE_VISITOR_DEFAULT(FixedWidthBinaryType);
+TYPE_VISITOR_DEFAULT(FixedSizeBinaryType);
 TYPE_VISITOR_DEFAULT(Date64Type);
 TYPE_VISITOR_DEFAULT(Date32Type);
 TYPE_VISITOR_DEFAULT(Time32Type);
diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h
index d44dcf6b97676..6c36e465ec436 100644
--- a/cpp/src/arrow/visitor.h
+++ b/cpp/src/arrow/visitor.h
@@ -43,7 +43,7 @@ class ARROW_EXPORT ArrayVisitor {
   virtual Status Visit(const DoubleArray& array);
   virtual Status Visit(const StringArray& array);
   virtual Status Visit(const BinaryArray& array);
-  virtual Status Visit(const FixedWidthBinaryArray& array);
+  virtual Status Visit(const FixedSizeBinaryArray& array);
   virtual Status Visit(const Date32Array& array);
   virtual Status Visit(const Date64Array& array);
   virtual Status Visit(const Time32Array& array);
@@ -76,7 +76,7 @@ class ARROW_EXPORT TypeVisitor {
   virtual Status Visit(const DoubleType& type);
   virtual Status Visit(const StringType& type);
   virtual Status Visit(const BinaryType& type);
-  virtual Status Visit(const FixedWidthBinaryType& type);
+  virtual Status Visit(const FixedSizeBinaryType& type);
   virtual Status Visit(const Date64Type& type);
   virtual Status Visit(const Date32Type& type);
   virtual Status Visit(const Time32Type& type);
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index cbc4d5acdb8cf..c61c9f59f7ab2 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -48,7 +48,7 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
     TYPE_VISIT_INLINE(DoubleType);
     TYPE_VISIT_INLINE(StringType);
     TYPE_VISIT_INLINE(BinaryType);
-    TYPE_VISIT_INLINE(FixedWidthBinaryType);
+    TYPE_VISIT_INLINE(FixedSizeBinaryType);
     TYPE_VISIT_INLINE(Date32Type);
     TYPE_VISIT_INLINE(Date64Type);
     TYPE_VISIT_INLINE(TimestampType);
@@ -87,7 +87,7 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
     ARRAY_VISIT_INLINE(DoubleType);
     ARRAY_VISIT_INLINE(StringType);
     ARRAY_VISIT_INLINE(BinaryType);
-    ARRAY_VISIT_INLINE(FixedWidthBinaryType);
+    ARRAY_VISIT_INLINE(FixedSizeBinaryType);
     ARRAY_VISIT_INLINE(Date32Type);
     ARRAY_VISIT_INLINE(Date64Type);
     ARRAY_VISIT_INLINE(TimestampType);
diff --git a/format/Schema.fbs b/format/Schema.fbs
index 5268bf95cfdc8..958f09181bfa6 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -67,7 +67,7 @@ table Utf8 {
 table Binary {
 }
 
-table FixedWidthBinary {
+table FixedSizeBinary {
   /// Number of bytes per value
   byteWidth: int;
 }
@@ -156,7 +156,7 @@ union Type {
   List,
   Struct_,
   Union,
-  FixedWidthBinary
+  FixedSizeBinary
 }
 
 /// ----------------------------------------------------------------------
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 66b6038617944..3df2a1d445549 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -55,7 +55,7 @@
                             Int8Value, Int16Value, Int32Value, Int64Value,
                             UInt8Value, UInt16Value, UInt32Value, UInt64Value,
                             FloatValue, DoubleValue, ListValue,
-                            BinaryValue, StringValue, FixedWidthBinaryValue)
+                            BinaryValue, StringValue, FixedSizeBinaryValue)
 
 import pyarrow.schema as _schema
 
@@ -65,7 +65,7 @@
                             timestamp, date32, date64,
                             float_, double, binary, string,
                             list_, struct, dictionary, field,
-                            DataType, FixedWidthBinaryType,
+                            DataType, FixedSizeBinaryType,
                             Field, Schema, schema)
 
 
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index a7241c6a47e31..0b5f33d0d2db6 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -100,7 +100,7 @@ cdef class DoubleArray(FloatingPointArray):
     pass
 
 
-cdef class FixedWidthBinaryArray(Array):
+cdef class FixedSizeBinaryArray(Array):
     pass
 
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 289baf2993081..b9799f15bf3e7 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -37,7 +37,7 @@ cimport pyarrow.scalar as scalar
 from pyarrow.scalar import NA
 
 from pyarrow.schema cimport (DataType, Field, Schema, DictionaryType,
-                             FixedWidthBinaryType,
+                             FixedSizeBinaryType,
                              box_data_type)
 import pyarrow.schema as schema
 
@@ -407,7 +407,7 @@ cdef class DoubleArray(FloatingPointArray):
     pass
 
 
-cdef class FixedWidthBinaryArray(Array):
+cdef class FixedSizeBinaryArray(Array):
     pass
 
 
@@ -518,7 +518,7 @@ cdef dict _array_classes = {
     Type_BINARY: BinaryArray,
     Type_STRING: StringArray,
     Type_DICTIONARY: DictionaryArray,
-    Type_FIXED_WIDTH_BINARY: FixedWidthBinaryArray,
+    Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
 }
 
 cdef object box_array(const shared_ptr[CArray]& sp_array):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index b44ade5298eb3..f549884d175fa 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -45,7 +45,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         Type_TIME64" arrow::Type::TIME64"
         Type_BINARY" arrow::Type::BINARY"
         Type_STRING" arrow::Type::STRING"
-        Type_FIXED_WIDTH_BINARY" arrow::Type::FIXED_WIDTH_BINARY"
+        Type_FIXED_SIZE_BINARY" arrow::Type::FIXED_SIZE_BINARY"
 
         Type_LIST" arrow::Type::LIST"
         Type_STRUCT" arrow::Type::STRUCT"
@@ -140,8 +140,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
 
-    cdef cppclass CFixedWidthBinaryType" arrow::FixedWidthBinaryType"(CFixedWidthType):
-        CFixedWidthBinaryType(int byte_width)
+    cdef cppclass CFixedSizeBinaryType" arrow::FixedSizeBinaryType"(CFixedWidthType):
+        CFixedSizeBinaryType(int byte_width)
         int byte_width()
 
     cdef cppclass CField" arrow::Field":
@@ -208,7 +208,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CDoubleArray" arrow::DoubleArray"(CArray):
         double Value(int i)
 
-    cdef cppclass CFixedWidthBinaryArray" arrow::FixedWidthBinaryArray"(CArray):
+    cdef cppclass CFixedSizeBinaryArray" arrow::FixedSizeBinaryArray"(CArray):
         const uint8_t* GetValue(int i)
 
     cdef cppclass CListArray" arrow::ListArray"(CArray):
diff --git a/python/pyarrow/scalar.pxd b/python/pyarrow/scalar.pxd
index e9cc3cb487cbc..d6c3b35160c12 100644
--- a/python/pyarrow/scalar.pxd
+++ b/python/pyarrow/scalar.pxd
@@ -62,7 +62,7 @@ cdef class StringValue(ArrayValue):
     pass
 
 
-cdef class FixedWidthBinaryValue(ArrayValue):
+cdef class FixedSizeBinaryValue(ArrayValue):
     pass
 
 
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index f4a1c9e08eb64..983a9a7334044 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -224,16 +224,16 @@ cdef class ListValue(ArrayValue):
         return result
 
 
-cdef class FixedWidthBinaryValue(ArrayValue):
+cdef class FixedSizeBinaryValue(ArrayValue):
 
     def as_py(self):
         cdef:
-            CFixedWidthBinaryArray* ap
-            CFixedWidthBinaryType* ap_type
+            CFixedSizeBinaryArray* ap
+            CFixedSizeBinaryType* ap_type
             int32_t length
             const char* data
-        ap = <CFixedWidthBinaryArray*> self.sp_array.get()
-        ap_type = <CFixedWidthBinaryType*> ap.type().get()
+        ap = <CFixedSizeBinaryArray*> self.sp_array.get()
+        ap_type = <CFixedSizeBinaryType*> ap.type().get()
         length = ap_type.byte_width()
         data = <const char*> ap.GetValue(self.index)
         return cp.PyBytes_FromStringAndSize(data, length)
@@ -258,7 +258,7 @@ cdef dict _scalar_classes = {
     Type_LIST: ListValue,
     Type_BINARY: BinaryValue,
     Type_STRING: StringValue,
-    Type_FIXED_WIDTH_BINARY: FixedWidthBinaryValue,
+    Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
 }
 
 cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
index c0c2c709b2744..94d65bfc157a1 100644
--- a/python/pyarrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -19,7 +19,7 @@ from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CDataType,
                                         CDictionaryType,
                                         CTimestampType,
-                                        CFixedWidthBinaryType,
+                                        CFixedSizeBinaryType,
                                         CField, CSchema)
 
 cdef class DataType:
@@ -40,9 +40,9 @@ cdef class TimestampType(DataType):
         const CTimestampType* ts_type
 
 
-cdef class FixedWidthBinaryType(DataType):
+cdef class FixedSizeBinaryType(DataType):
     cdef:
-        const CFixedWidthBinaryType* fixed_width_binary_type
+        const CFixedSizeBinaryType* fixed_size_binary_type
 
 
 cdef class Field:
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 532a318840caf..06df64461ae22 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -28,7 +28,7 @@ from pyarrow.compat import frombytes, tobytes
 from pyarrow.array cimport Array
 from pyarrow.error cimport check_status
 from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
-                                        CFixedWidthBinaryType,
+                                        CFixedSizeBinaryType,
                                         TimeUnit_SECOND, TimeUnit_MILLI,
                                         TimeUnit_MICRO, TimeUnit_NANO,
                                         Type, TimeUnit)
@@ -91,16 +91,16 @@ cdef class TimestampType(DataType):
                 return None
 
 
-cdef class FixedWidthBinaryType(DataType):
+cdef class FixedSizeBinaryType(DataType):
 
     cdef init(self, const shared_ptr[CDataType]& type):
         DataType.init(self, type)
-        self.fixed_width_binary_type = <const CFixedWidthBinaryType*> type.get()
+        self.fixed_size_binary_type = <const CFixedSizeBinaryType*> type.get()
 
     property byte_width:
 
         def __get__(self):
-            return self.fixed_width_binary_type.byte_width()
+            return self.fixed_size_binary_type.byte_width()
 
 
 cdef class Field:
@@ -362,16 +362,16 @@ def binary(int length=-1):
     ----------
     length : int, optional, default -1
         If length == -1 then return a variable length binary type. If length is
-        greater than or equal to 0 then return a fixed width binary type of
+        greater than or equal to 0 then return a fixed size binary type of
         width `length`.
     """
     if length == -1:
         return primitive_type(la.Type_BINARY)
 
-    cdef FixedWidthBinaryType out = FixedWidthBinaryType()
-    cdef shared_ptr[CDataType] fixed_width_binary_type
-    fixed_width_binary_type.reset(new CFixedWidthBinaryType(length))
-    out.init(fixed_width_binary_type)
+    cdef FixedSizeBinaryType out = FixedSizeBinaryType()
+    cdef shared_ptr[CDataType] fixed_size_binary_type
+    fixed_size_binary_type.reset(new CFixedSizeBinaryType(length))
+    out.init(fixed_size_binary_type)
     return out
 
 
@@ -428,8 +428,8 @@ cdef DataType box_data_type(const shared_ptr[CDataType]& type):
         out = DictionaryType()
     elif type.get().type == la.Type_TIMESTAMP:
         out = TimestampType()
-    elif type.get().type == la.Type_FIXED_WIDTH_BINARY:
-        out = FixedWidthBinaryType()
+    elif type.get().type == la.Type_FIXED_SIZE_BINARY:
+        out = FixedSizeBinaryType()
     else:
         out = DataType()
 
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 3309ba018628d..bb6d2d17d5f0c 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -92,7 +92,7 @@ def test_bytes(self):
         assert arr.type == pyarrow.binary()
         assert arr.to_pylist() == [b'foo', u1, None]
 
-    def test_fixed_width_bytes(self):
+    def test_fixed_size_bytes(self):
         data = [b'foof', None, b'barb', b'2346']
         arr = pyarrow.from_pylist(data, type=pyarrow.binary(4))
         assert len(arr) == 4
@@ -100,7 +100,7 @@ def test_fixed_width_bytes(self):
         assert arr.type == pyarrow.binary(4)
         assert arr.to_pylist() == data
 
-    def test_fixed_width_bytes_does_not_accept_varying_lengths(self):
+    def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
         data = [b'foo', None, b'barb', b'2346']
         with self.assertRaises(pyarrow.error.ArrowException):
             pyarrow.from_pylist(data, type=pyarrow.binary(4))
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 3f19b68fe0a03..c472ee69034c8 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -244,7 +244,7 @@ def test_bytes_to_binary(self):
         expected = pd.DataFrame({'strings': values2})
         self._check_pandas_roundtrip(df, expected)
 
-    def test_fixed_width_bytes(self):
+    def test_fixed_size_bytes(self):
         values = [b'foo', None, b'bar', None, None, b'hey']
         df = pd.DataFrame({'strings': values})
         schema = A.Schema.from_fields([A.field('strings', A.binary(3))])
@@ -254,7 +254,7 @@ def test_fixed_width_bytes(self):
         result = table.to_pandas()
         tm.assert_frame_equal(result, df)
 
-    def test_fixed_width_bytes_does_not_accept_varying_lengths(self):
+    def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
         values = [b'foo', None, b'ba', None, None, b'hey']
         df = pd.DataFrame({'strings': values})
         schema = A.Schema.from_fields([A.field('strings', A.binary(3))])
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 265ce8d3a58a1..a5db7e0835607 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -87,12 +87,12 @@ def test_bytes(self):
         assert v == b'bar'
         assert isinstance(v, bytes)
 
-    def test_fixed_width_bytes(self):
+    def test_fixed_size_bytes(self):
         data = [b'foof', None, b'barb']
         arr = A.from_pylist(data, type=A.binary(4))
 
         v = arr[0]
-        assert isinstance(v, A.FixedWidthBinaryValue)
+        assert isinstance(v, A.FixedSizeBinaryValue)
         assert v.as_py() == b'foof'
 
         assert arr[1] is A.NA

From fd000964d218b355e725d8eced1d1301f36dc092 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 1 Apr 2017 11:19:09 -0400
Subject: [PATCH 0448/1644] ARROW-723: [Python] Ensure that passing
 chunk_size=0 when writing Parquet file does not enter infinite loop

This should also be fixed in parquet-cpp, will open a JIRA.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #468 from wesm/ARROW-723 and squashes the following commits:

f938703 [Wes McKinney] Raise if row group size is 0, use default if -1
5f83850 [Wes McKinney] Ensure that passing chunk_size=0 when writing Parquet file does not enter infinite loop
---
 python/pyarrow/_parquet.pyx          |  5 ++++-
 python/pyarrow/parquet.py            |  2 +-
 python/pyarrow/tests/test_parquet.py | 17 +++++++++++++++++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 8e67da9f75a6e..c4cbd28e85dab 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -538,10 +538,13 @@ cdef class ParquetWriter:
     def write_table(self, Table table, row_group_size=None):
         cdef CTable* ctable = table.table
 
-        if row_group_size is None:
+        if row_group_size is None or row_group_size == -1:
             row_group_size = ctable.num_rows()
+        elif row_group_size == 0:
+            raise ValueError('Row group size cannot be 0')
 
         cdef int c_row_group_size = row_group_size
+
         with nogil:
             check_status(WriteTable(deref(ctable), self.allocator,
                                     self.sink, c_row_group_size,
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index fa96f95698013..2985316f35f01 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -187,7 +187,7 @@ def write_table(table, sink, chunk_size=None, version='1.0',
     ----------
     table : pyarrow.Table
     sink: string or pyarrow.io.NativeFile
-    chunk_size : int
+    chunk_size : int, default None
         The maximum number of rows in each Parquet RowGroup. As a default,
         we will write a single RowGroup per file.
     version : {"1.0", "2.0"}, default "1.0"
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index fc32b9fac8b98..b8b2800259caf 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -365,6 +365,23 @@ def test_multithreaded_read():
     assert table1.equals(table2)
 
 
+@parquet
+def test_min_chunksize():
+    data = pd.DataFrame([np.arange(4)], columns=['A', 'B', 'C', 'D'])
+    table = pa.Table.from_pandas(data.reset_index())
+
+    buf = io.BytesIO()
+    pq.write_table(table, buf, chunk_size=-1)
+
+    buf.seek(0)
+    result = pq.read_table(buf)
+
+    assert result.equals(table)
+
+    with pytest.raises(ValueError):
+        pq.write_table(table, buf, chunk_size=0)
+
+
 @parquet
 def test_pass_separate_metadata():
     # ARROW-471

From 31a1f53f4990d07a337ea0b000e04df2917b6d73 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 1 Apr 2017 11:19:40 -0400
Subject: [PATCH 0449/1644] ARROW-710: [Python] Read/write with file-like
 Python objects from read_feather/write_feather

cc @jreback

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #474 from wesm/ARROW-710 and squashes the following commits:

61d7218 [Wes McKinney] Do not close OutputStream in Feather writer. Read and write to file-like Python objects
---
 cpp/src/arrow/ipc/feather-test.cc        |   3 +-
 cpp/src/arrow/ipc/feather.cc             |  25 +---
 cpp/src/arrow/ipc/feather.h              |   7 +-
 python/CMakeLists.txt                    |   1 -
 python/pyarrow/_feather.pyx              | 158 -----------------------
 python/pyarrow/feather.py                |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |  31 ++++-
 python/pyarrow/io.pyx                    | 101 ++++++++++++++-
 python/pyarrow/tests/test_feather.py     |  17 ++-
 python/setup.py                          |   1 -
 10 files changed, 160 insertions(+), 198 deletions(-)
 delete mode 100644 python/pyarrow/_feather.pyx

diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index e181f6933541b..077a44b896fc1 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -272,8 +272,7 @@ class TestTableWriter : public ::testing::Test {
     ASSERT_OK(stream_->Finish(&output_));
 
     std::shared_ptr<io::BufferReader> buffer(new io::BufferReader(output_));
-    reader_.reset(new TableReader());
-    ASSERT_OK(reader_->Open(buffer));
+    ASSERT_OK(TableReader::Open(buffer, &reader_));
   }
 
   void CheckBatch(const RecordBatch& batch) {
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 5820563b43834..e838e1fdbcd61 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -401,16 +401,10 @@ TableReader::TableReader() {
 
 TableReader::~TableReader() {}
 
-Status TableReader::Open(const std::shared_ptr<io::RandomAccessFile>& source) {
-  return impl_->Open(source);
-}
-
-Status TableReader::OpenFile(
-    const std::string& abspath, std::unique_ptr<TableReader>* out) {
-  std::shared_ptr<io::MemoryMappedFile> file;
-  RETURN_NOT_OK(io::MemoryMappedFile::Open(abspath, io::FileMode::READ, &file));
+Status TableReader::Open(const std::shared_ptr<io::RandomAccessFile>& source,
+    std::unique_ptr<TableReader>* out) {
   out->reset(new TableReader());
-  return (*out)->Open(file);
+  return (*out)->impl_->Open(source);
 }
 
 bool TableReader::HasDescription() const {
@@ -517,9 +511,8 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     // Footer: metadata length, magic bytes
     RETURN_NOT_OK(
         stream_->Write(reinterpret_cast<const uint8_t*>(&buffer_size), sizeof(uint32_t)));
-    RETURN_NOT_OK(stream_->Write(reinterpret_cast<const uint8_t*>(kFeatherMagicBytes),
-        strlen(kFeatherMagicBytes)));
-    return stream_->Close();
+    return stream_->Write(
+        reinterpret_cast<const uint8_t*>(kFeatherMagicBytes), strlen(kFeatherMagicBytes));
   }
 
   Status LoadArrayMetadata(const Array& values, ArrayMetadata* meta) {
@@ -700,14 +693,6 @@ Status TableWriter::Open(
   return (*out)->impl_->Open(stream);
 }
 
-Status TableWriter::OpenFile(
-    const std::string& abspath, std::unique_ptr<TableWriter>* out) {
-  std::shared_ptr<io::FileOutputStream> file;
-  RETURN_NOT_OK(io::FileOutputStream::Open(abspath, &file));
-  out->reset(new TableWriter());
-  return (*out)->impl_->Open(file);
-}
-
 void TableWriter::SetDescription(const std::string& desc) {
   impl_->SetDescription(desc);
 }
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
index 1e4ba58255456..8cc8ca092a1b2 100644
--- a/cpp/src/arrow/ipc/feather.h
+++ b/cpp/src/arrow/ipc/feather.h
@@ -54,9 +54,8 @@ class ARROW_EXPORT TableReader {
   TableReader();
   ~TableReader();
 
-  Status Open(const std::shared_ptr<io::RandomAccessFile>& source);
-
-  static Status OpenFile(const std::string& abspath, std::unique_ptr<TableReader>* out);
+  static Status Open(const std::shared_ptr<io::RandomAccessFile>& source,
+      std::unique_ptr<TableReader>* out);
 
   // Optional table description
   //
@@ -86,8 +85,6 @@ class ARROW_EXPORT TableWriter {
   static Status Open(
       const std::shared_ptr<io::OutputStream>& stream, std::unique_ptr<TableWriter>* out);
 
-  static Status OpenFile(const std::string& abspath, std::unique_ptr<TableWriter>* out);
-
   void SetDescription(const std::string& desc);
   void SetNumRows(int64_t num_rows);
 
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 35a1a89ef3164..f315d019bb4de 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -268,7 +268,6 @@ set(CYTHON_EXTENSIONS
   config
   error
   io
-  _feather
   memory
   scalar
   schema
diff --git a/python/pyarrow/_feather.pyx b/python/pyarrow/_feather.pyx
deleted file mode 100644
index beb4aaad44618..0000000000000
--- a/python/pyarrow/_feather.pyx
+++ /dev/null
@@ -1,158 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from cython.operator cimport dereference as deref
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport CArray, CColumn, CSchema, CStatus
-from pyarrow.includes.libarrow_io cimport RandomAccessFile, OutputStream
-
-from libcpp.string cimport string
-from libcpp cimport bool as c_bool
-
-cimport cpython
-
-from pyarrow.compat import frombytes, tobytes, encode_file_path
-
-from pyarrow.array cimport Array
-from pyarrow.error cimport check_status
-from pyarrow.table cimport Column
-
-cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
-
-    cdef cppclass TableWriter:
-        @staticmethod
-        CStatus Open(const shared_ptr[OutputStream]& stream,
-                     unique_ptr[TableWriter]* out)
-
-        @staticmethod
-        CStatus OpenFile(const string& abspath, unique_ptr[TableWriter]* out)
-
-        void SetDescription(const string& desc)
-        void SetNumRows(int64_t num_rows)
-
-        CStatus Append(const string& name, const CArray& values)
-        CStatus Finalize()
-
-    cdef cppclass TableReader:
-        TableReader(const shared_ptr[RandomAccessFile]& source)
-
-        @staticmethod
-        CStatus OpenFile(const string& abspath, unique_ptr[TableReader]* out)
-
-        string GetDescription()
-        c_bool HasDescription()
-
-        int64_t num_rows()
-        int64_t num_columns()
-
-        shared_ptr[CSchema] schema()
-
-        CStatus GetColumn(int i, shared_ptr[CColumn]* out)
-        c_string GetColumnName(int i)
-
-
-class FeatherError(Exception):
-    pass
-
-
-cdef class FeatherWriter:
-    cdef:
-        unique_ptr[TableWriter] writer
-
-    cdef public:
-        int64_t num_rows
-
-    def __cinit__(self):
-        self.num_rows = -1
-
-    def open(self, object dest):
-        cdef:
-            string c_name = encode_file_path(dest)
-
-        check_status(TableWriter.OpenFile(c_name, &self.writer))
-
-    def close(self):
-        if self.num_rows < 0:
-            self.num_rows = 0
-        self.writer.get().SetNumRows(self.num_rows)
-        check_status(self.writer.get().Finalize())
-
-    def write_array(self, object name, object col, object mask=None):
-        cdef Array arr
-
-        if self.num_rows >= 0:
-            if len(col) != self.num_rows:
-                raise ValueError('prior column had a different number of rows')
-        else:
-            self.num_rows = len(col)
-
-        if isinstance(col, Array):
-            arr = col
-        else:
-            arr = Array.from_pandas(col, mask=mask)
-
-        cdef c_string c_name = tobytes(name)
-
-        with nogil:
-            check_status(
-                self.writer.get().Append(c_name, deref(arr.sp_array)))
-
-
-cdef class FeatherReader:
-    cdef:
-        unique_ptr[TableReader] reader
-
-    def __cinit__(self):
-        pass
-
-    def open(self, source):
-        cdef:
-            string c_name = encode_file_path(source)
-
-        check_status(TableReader.OpenFile(c_name, &self.reader))
-
-    property num_rows:
-
-        def __get__(self):
-            return self.reader.get().num_rows()
-
-    property num_columns:
-
-        def __get__(self):
-            return self.reader.get().num_columns()
-
-    def get_column_name(self, int i):
-        cdef c_string name = self.reader.get().GetColumnName(i)
-        return frombytes(name)
-
-    def get_column(self, int i):
-        if i < 0 or i >= self.num_columns:
-            raise IndexError(i)
-
-        cdef shared_ptr[CColumn] sp_column
-        with nogil:
-            check_status(self.reader.get()
-                         .GetColumn(i, &sp_column))
-
-        cdef Column col = Column()
-        col.init(sp_column)
-        return col
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 28424afb093b5..f87c7f3a95ee4 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -20,9 +20,9 @@
 import pandas as pd
 
 from pyarrow.compat import pdapi
-from pyarrow._feather import FeatherError  # noqa
+from pyarrow.io import FeatherError  # noqa
 from pyarrow.table import Table
-import pyarrow._feather as ext
+import pyarrow.io as ext
 
 
 if LooseVersion(pd.__version__) < '0.17.0':
@@ -54,12 +54,12 @@ def read(self, columns=None):
         return table.to_pandas()
 
 
-def write_feather(df, path):
+def write_feather(df, dest):
     '''
     Write a pandas.DataFrame to Feather format
     '''
     writer = ext.FeatherWriter()
-    writer.open(path)
+    writer.open(dest)
 
     if isinstance(df, pd.SparseDataFrame):
         df = df.to_dense()
@@ -95,13 +95,13 @@ def write_feather(df, path):
     writer.close()
 
 
-def read_feather(path, columns=None):
+def read_feather(source, columns=None):
     """
     Read a pandas.DataFrame from Feather format
 
     Parameters
     ----------
-    path : string, path to read from
+    source : string file path, or file-like object
     columns : sequence, optional
         Only read a specific set of columns. If not provided, all columns are
         read
@@ -110,5 +110,5 @@ def read_feather(path, columns=None):
     -------
     df : pandas.DataFrame
     """
-    reader = FeatherReader(path)
+    reader = FeatherReader(source)
     return reader.read(columns=columns)
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
index 8b7d705afd4e7..59fd90bdac7a8 100644
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ b/python/pyarrow/includes/libarrow_ipc.pxd
@@ -18,7 +18,7 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CSchema, CRecordBatch)
+from pyarrow.includes.libarrow cimport (CArray, CColumn, CSchema, CRecordBatch)
 from pyarrow.includes.libarrow_io cimport (InputStream, OutputStream,
                                            RandomAccessFile)
 
@@ -63,3 +63,32 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
         int num_record_batches()
 
         CStatus GetRecordBatch(int i, shared_ptr[CRecordBatch]* batch)
+
+cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
+
+    cdef cppclass CFeatherWriter" arrow::ipc::feather::TableWriter":
+        @staticmethod
+        CStatus Open(const shared_ptr[OutputStream]& stream,
+                     unique_ptr[CFeatherWriter]* out)
+
+        void SetDescription(const c_string& desc)
+        void SetNumRows(int64_t num_rows)
+
+        CStatus Append(const c_string& name, const CArray& values)
+        CStatus Finalize()
+
+    cdef cppclass CFeatherReader" arrow::ipc::feather::TableReader":
+        @staticmethod
+        CStatus Open(const shared_ptr[RandomAccessFile]& file,
+                     unique_ptr[CFeatherReader]* out)
+
+        c_string GetDescription()
+        c_bool HasDescription()
+
+        int64_t num_rows()
+        int64_t num_columns()
+
+        shared_ptr[CSchema] schema()
+
+        CStatus GetColumn(int i, shared_ptr[CColumn]* out)
+        c_string GetColumnName(int i)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index d64427aa36ef5..0b27379c273b0 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -32,10 +32,11 @@ from pyarrow.includes.libarrow_ipc cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.compat import frombytes, tobytes, encode_file_path
+from pyarrow.array cimport Array
 from pyarrow.error cimport check_status
 from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
 from pyarrow.schema cimport Schema
-from pyarrow.table cimport (RecordBatch, batch_from_cbatch,
+from pyarrow.table cimport (Column, RecordBatch, batch_from_cbatch,
                             table_from_ctable)
 
 cimport cpython as cp
@@ -564,7 +565,9 @@ cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
 cdef get_writer(object source, shared_ptr[OutputStream]* writer):
     cdef NativeFile nf
 
-    if not isinstance(source, NativeFile) and hasattr(source, 'write'):
+    if isinstance(source, six.string_types):
+        source = OSFile(source, mode='w')
+    elif not isinstance(source, NativeFile) and hasattr(source, 'write'):
         # Optimistically hope this is file-like
         source = PythonFileInterface(source, mode='w')
 
@@ -1047,3 +1050,97 @@ cdef class _FileReader:
             check_status(CTable.FromRecordBatches(batches, &table))
 
         return table_from_ctable(table)
+
+
+#----------------------------------------------------------------------
+# Implement legacy Feather file format
+
+
+class FeatherError(Exception):
+    pass
+
+
+cdef class FeatherWriter:
+    cdef:
+        unique_ptr[CFeatherWriter] writer
+
+    cdef public:
+        int64_t num_rows
+
+    def __cinit__(self):
+        self.num_rows = -1
+
+    def open(self, object dest):
+        cdef shared_ptr[OutputStream] sink
+        get_writer(dest, &sink)
+
+        with nogil:
+            check_status(CFeatherWriter.Open(sink, &self.writer))
+
+    def close(self):
+        if self.num_rows < 0:
+            self.num_rows = 0
+        self.writer.get().SetNumRows(self.num_rows)
+        check_status(self.writer.get().Finalize())
+
+    def write_array(self, object name, object col, object mask=None):
+        cdef Array arr
+
+        if self.num_rows >= 0:
+            if len(col) != self.num_rows:
+                raise ValueError('prior column had a different number of rows')
+        else:
+            self.num_rows = len(col)
+
+        if isinstance(col, Array):
+            arr = col
+        else:
+            arr = Array.from_pandas(col, mask=mask)
+
+        cdef c_string c_name = tobytes(name)
+
+        with nogil:
+            check_status(
+                self.writer.get().Append(c_name, deref(arr.sp_array)))
+
+
+cdef class FeatherReader:
+    cdef:
+        unique_ptr[CFeatherReader] reader
+
+    def __cinit__(self):
+        pass
+
+    def open(self, source):
+        cdef shared_ptr[RandomAccessFile] reader
+        get_reader(source, &reader)
+
+        with nogil:
+            check_status(CFeatherReader.Open(reader, &self.reader))
+
+    property num_rows:
+
+        def __get__(self):
+            return self.reader.get().num_rows()
+
+    property num_columns:
+
+        def __get__(self):
+            return self.reader.get().num_columns()
+
+    def get_column_name(self, int i):
+        cdef c_string name = self.reader.get().GetColumnName(i)
+        return frombytes(name)
+
+    def get_column(self, int i):
+        if i < 0 or i >= self.num_columns:
+            raise IndexError(i)
+
+        cdef shared_ptr[CColumn] sp_column
+        with nogil:
+            check_status(self.reader.get()
+                         .GetColumn(i, &sp_column))
+
+        cdef Column col = Column()
+        col.init(sp_column)
+        return col
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index e4b6273ffccf4..dd6888f2d1306 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -27,7 +27,7 @@
 from pyarrow.compat import guid
 from pyarrow.feather import (read_feather, write_feather,
                              FeatherReader)
-from pyarrow._feather import FeatherWriter
+from pyarrow.io import FeatherWriter
 
 
 def random_path():
@@ -347,6 +347,21 @@ def test_overwritten_file(self):
         df = pd.DataFrame({'ints': values[0: num_values//2]})
         self._check_pandas_roundtrip(df, path=path)
 
+    def test_filelike_objects(self):
+        from io import BytesIO
+
+        buf = BytesIO()
+
+        # the copy makes it non-strided
+        df = pd.DataFrame(np.arange(12).reshape(4, 3),
+                          columns=['a', 'b', 'c']).copy()
+        write_feather(df, buf)
+
+        buf.seek(0)
+
+        result = read_feather(buf)
+        assert_frame_equal(result, df)
+
     def test_sparse_dataframe(self):
         # GH #221
         data = {'A': [0, 1, 2],
diff --git a/python/setup.py b/python/setup.py
index 9ff091819c760..12b44e1bad520 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -104,7 +104,6 @@ def initialize_options(self):
         'io',
         'jemalloc',
         'memory',
-        '_feather',
         '_parquet',
         'scalar',
         'schema',

From 651ea9247c42b889da457432d4ff13b558e8bec1 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sat, 1 Apr 2017 13:18:40 -0400
Subject: [PATCH 0450/1644] ARROW-745: [C++] Allow use of system cpplint

You can do `pip install cpplint` to take advantage of this.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #476 from cpcloud/ARROW-745 and squashes the following commits:

e43d20c [Phillip Cloud] ARROW-745: [C++] Allow use of system cpplint
---
 cpp/CMakeLists.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5dcf58c0f232d..7a5a0e68874ac 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -741,8 +741,11 @@ if (UNIX)
     ENDIF()
   ENDFOREACH(item ${LINT_FILES})
 
+  find_program(CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR})
+  message(STATUS "Found cpplint executable at ${CPPLINT_BIN}")
+
   # Full lint
-  add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
+  add_custom_target(lint ${CPPLINT_BIN}
   --verbose=2
   --linelength=90
   --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11,-runtime/references,-build/include_order

From baf38e47a7d73d87017699304dcbe15f297c9284 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 2 Apr 2017 10:04:11 +0200
Subject: [PATCH 0451/1644] ARROW-747: [C++] Calling add_dependencies with dl
 causes spurious CMake warning

I added an option to make the dependency targets (e.g. external projects) in libraries more explicit.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #472 from wesm/ARROW-747 and squashes the following commits:

c60832f [Wes McKinney] Add DEPENDENCIES argument to ADD_ARROW_LIB to fix spurious dl dependency issue
---
 cpp/CMakeLists.txt                    |  9 +---
 cpp/cmake_modules/BuildUtils.cmake    | 18 ++-----
 cpp/src/arrow/io/CMakeLists.txt       |  4 +-
 cpp/src/arrow/ipc/CMakeLists.txt      | 74 ++++++++++++---------------
 cpp/src/arrow/jemalloc/CMakeLists.txt |  5 ++
 5 files changed, 46 insertions(+), 64 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 7a5a0e68874ac..aacc7a15fffc9 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -654,13 +654,8 @@ if (ARROW_JEMALLOC)
 
   include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(jemalloc
-      STATIC_LIB ${JEMALLOC_STATIC_LIB}
-      SHARED_LIB ${JEMALLOC_SHARED_LIB})
-
-  if (JEMALLOC_VENDORED)
-    add_dependencies(jemalloc_shared jemalloc_ep)
-    add_dependencies(jemalloc_static jemalloc_ep)
-  endif()
+    STATIC_LIB ${JEMALLOC_STATIC_LIB}
+    SHARED_LIB ${JEMALLOC_SHARED_LIB})
 endif()
 
 ## Google PerfTools
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 43d984045eb20..3a3b53678f6e5 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -85,26 +85,18 @@ endfunction()
 function(ADD_ARROW_LIB LIB_NAME)
   set(options)
   set(one_value_args SHARED_LINK_FLAGS)
-  set(multi_value_args SOURCES STATIC_LINK_LIBS STATIC_PRIVATE_LINK_LIBS SHARED_LINK_LIBS SHARED_PRIVATE_LINK_LIBS)
+  set(multi_value_args SOURCES STATIC_LINK_LIBS STATIC_PRIVATE_LINK_LIBS SHARED_LINK_LIBS SHARED_PRIVATE_LINK_LIBS DEPENDENCIES)
   cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
   if(ARG_UNPARSED_ARGUMENTS)
     message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
   endif()
 
   add_library(${LIB_NAME}_objlib OBJECT
-      ${ARG_SOURCES}
+    ${ARG_SOURCES}
   )
-  if (ARG_STATIC_LINK_LIBS)
-    add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_LINK_LIBS})
-  endif()
-  if (ARG_STATIC_PRIVATE_LINK_LIBS)
-    add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_PRIVATE_LINK_LIBS})
-  endif()
-  if (ARG_SHARED_LINK_LIBS)
-    add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_LINK_LIBS})
-  endif()
-  if(ARG_SHARED_PRIVATE_LINK_LIBS)
-    add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_PRIVATE_LINK_LIBS})
+
+  if (ARG_DEPENDENCIES)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_DEPENDENCIES})
   endif()
 
   # Necessary to make static linking into other shared libraries work properly
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 8aabf6496f8f7..3951eac322c6a 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -48,11 +48,11 @@ if (MSVC)
 else()
   set(ARROW_IO_STATIC_LINK_LIBS
     arrow_static
-    dl
+    ${CMAKE_DL_LIBS}
   )
   set(ARROW_IO_SHARED_LINK_LIBS
     arrow_shared
-    dl
+    ${CMAKE_DL_LIBS}
   )
 endif()
 
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 31a04dfc07818..5fa7d6125ce5e 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -26,7 +26,8 @@ set(ARROW_IPC_SHARED_LINK_LIBS
 
 set(ARROW_IPC_TEST_LINK_LIBS
   arrow_ipc_static
-  arrow_io_static)
+  arrow_io_static
+  arrow_static)
 
 set(ARROW_IPC_SRCS
   feather.cc
@@ -44,20 +45,22 @@ if(NOT APPLE)
   set(ARROW_IPC_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
 endif()
 
-ADD_ARROW_LIB(arrow_ipc
-    SOURCES ${ARROW_IPC_SRCS}
-    SHARED_LINK_FLAGS ${ARROW_IPC_LINK_FLAGS}
-    SHARED_LINK_LIBS ${ARROW_IPC_SHARED_LINK_LIBS}
-    STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
-)
-
 if(RAPIDJSON_VENDORED)
-  add_dependencies(arrow_ipc_objlib rapidjson_ep)
+  set(IPC_DEPENDENCIES ${IPC_DEPENDENCIES} rapidjson_ep)
 endif()
+
 if(FLATBUFFERS_VENDORED)
-  add_dependencies(arrow_ipc_objlib flatbuffers_ep)
+  set(IPC_DEPENDENCIES ${IPC_DEPENDENCIES} flatbuffers_ep)
 endif()
 
+ADD_ARROW_LIB(arrow_ipc
+  SOURCES ${ARROW_IPC_SRCS}
+  DEPENDENCIES ${IPC_DEPENDENCIES}
+  SHARED_LINK_FLAGS ${ARROW_IPC_LINK_FLAGS}
+  SHARED_LINK_LIBS ${ARROW_IPC_SHARED_LINK_LIBS}
+  STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
+)
+
 ADD_ARROW_TEST(feather-test)
 ARROW_TEST_LINK_LIBRARIES(feather-test
   ${ARROW_IPC_TEST_LINK_LIBS})
@@ -71,40 +74,27 @@ ARROW_TEST_LINK_LIBRARIES(ipc-json-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
 ADD_ARROW_TEST(json-integration-test)
+ARROW_TEST_LINK_LIBRARIES(json-integration-test
+  ${ARROW_IPC_TEST_LINK_LIBS})
 
 if (ARROW_BUILD_TESTS)
-  if (APPLE)
-    target_link_libraries(json-integration-test
-      arrow_ipc_static
-      arrow_io_static
-      arrow_static
-      gflags
-      gtest
-      ${BOOST_FILESYSTEM_LIBRARY}
-      ${BOOST_SYSTEM_LIBRARY}
-      dl)
-    set_target_properties(json-integration-test
-      PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-  elseif (MSVC)
-    target_link_libraries(json-integration-test
-      arrow_ipc_static
-      arrow_io_static
-      arrow_static
-      gflags
-      gtest
-      ${BOOST_FILESYSTEM_LIBRARY}
-      ${BOOST_SYSTEM_LIBRARY})
-  else()
-    target_link_libraries(json-integration-test
-      arrow_ipc_static
-      arrow_io_static
-      arrow_static
-      gflags
-      gtest
-      pthread
-      ${BOOST_FILESYSTEM_LIBRARY}
-      ${BOOST_SYSTEM_LIBRARY}
-      dl)
+  target_link_libraries(json-integration-test
+    gflags
+    gtest
+    ${BOOST_FILESYSTEM_LIBRARY}
+    ${BOOST_SYSTEM_LIBRARY})
+
+  if (UNIX)
+    if (APPLE)
+      target_link_libraries(json-integration-test
+        ${CMAKE_DL_LIBS})
+      set_target_properties(json-integration-test
+        PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+    else()
+      target_link_libraries(json-integration-test
+        pthread
+        ${CMAKE_DL_LIBS})
+    endif()
   endif()
 endif()
 
diff --git a/cpp/src/arrow/jemalloc/CMakeLists.txt b/cpp/src/arrow/jemalloc/CMakeLists.txt
index b8e6e231a3dca..7b627ac97b884 100644
--- a/cpp/src/arrow/jemalloc/CMakeLists.txt
+++ b/cpp/src/arrow/jemalloc/CMakeLists.txt
@@ -84,8 +84,13 @@ if(NOT APPLE)
   set(ARROW_JEMALLOC_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
 endif()
 
+if (JEMALLOC_VENDORED)
+  set(JEMALLOC_DEPENDENCIES jemalloc_ep)
+endif()
+
 ADD_ARROW_LIB(arrow_jemalloc
   SOURCES ${ARROW_JEMALLOC_SRCS}
+  DEPENDENCIES ${JEMALLOC_DEPENDENCIES}
   SHARED_LINK_FLAGS ${ARROW_JEMALLOC_LINK_FLAGS}
   SHARED_LINK_LIBS ${ARROW_JEMALLOC_SHARED_LINK_LIBS}
   SHARED_PRIVATE_LINK_LIBS ${ARROW_JEMALLOC_SHARED_PRIVATE_LINK_LIBS}

From 7fec7d30c75dd8910522002bb6bb640330834b90 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 2 Apr 2017 11:29:14 -0400
Subject: [PATCH 0452/1644] ARROW-754: [GLib] Add garrow_array_is_null()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #480 from kou/glib-support-array-is-null and squashes the following commits:

5c4259b [Kouhei Sutou] [GLib] Add garrow_array_is_null()
---
 c_glib/arrow-glib/array.cpp | 14 ++++++++++++++
 c_glib/arrow-glib/array.h   |  2 ++
 c_glib/test/test-array.rb   |  9 +++++++++
 3 files changed, 25 insertions(+)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index b084054f9af87..caf2eb55d6b2c 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -135,6 +135,20 @@ garrow_array_class_init(GArrowArrayClass *klass)
   g_object_class_install_property(gobject_class, PROP_ARRAY, spec);
 }
 
+/**
+ * garrow_array_is_null:
+ * @array: A #GArrowArray.
+ * @i: The index of the target value.
+ *
+ * Returns: Whether the i-th value is null or not.
+ */
+gboolean
+garrow_array_is_null(GArrowArray *array, gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  return arrow_array->IsNull(i);
+}
+
 /**
  * garrow_array_get_length:
  * @array: A #GArrowArray.
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 6467db5ff45db..957b4416fa581 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -57,6 +57,8 @@ struct _GArrowArrayClass
 
 GType          garrow_array_get_type    (void) G_GNUC_CONST;
 
+gboolean       garrow_array_is_null     (GArrowArray *array,
+                                         gint64 i);
 gint64         garrow_array_get_length  (GArrowArray *array);
 gint64         garrow_array_get_offset  (GArrowArray *array);
 gint64         garrow_array_get_n_nulls (GArrowArray *array);
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
index c427f0200ef02..08908b08961a7 100644
--- a/c_glib/test/test-array.rb
+++ b/c_glib/test/test-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestArray < Test::Unit::TestCase
+  def test_is_null
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append_null
+    builder.append(true)
+    array = builder.finish
+    assert_equal([true, false],
+                 array.length.times.collect {|i| array.null?(i)})
+  end
+
   def test_length
     builder = Arrow::BooleanArrayBuilder.new
     builder.append(true)

From e333576a0d215e97cc4e2a218ddc56ee1242986d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 2 Apr 2017 11:59:18 -0400
Subject: [PATCH 0453/1644] ARROW-718: [Python] Implement pyarrow.Tensor
 container, zero-copy NumPy roundtrips

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #477 from wesm/ARROW-718 and squashes the following commits:

2c23427 [Wes McKinney] Restore clang-format-3.9 formatting
eb21a17 [Wes McKinney] Finish basic tensor zero-copy roundtrips, simple repr. flake8 fixes
4cf6d2b [Wes McKinney] Draft tensor conversion to/from numpy
---
 cpp/src/arrow/CMakeLists.txt                 |   1 +
 cpp/src/arrow/api.h                          |   1 +
 cpp/src/arrow/python/CMakeLists.txt          |  10 +
 cpp/src/arrow/python/api.h                   |   1 +
 cpp/src/arrow/python/common.h                |  17 --
 cpp/src/arrow/python/helpers.cc              |   1 +
 cpp/src/arrow/python/numpy_convert.cc        | 267 +++++++++++++++++++
 cpp/src/arrow/python/numpy_convert.h         |  69 +++++
 cpp/src/arrow/python/pandas_convert.cc       |  88 +-----
 cpp/src/arrow/python/pandas_convert.h        |   3 -
 cpp/src/arrow/python/python-test.cc          |   2 +-
 cpp/src/arrow/tensor.cc                      |  14 +-
 cpp/src/arrow/tensor.h                       |   7 +
 python/pyarrow/__init__.py                   |   6 +-
 python/pyarrow/array.pxd                     |  13 +-
 python/pyarrow/array.pyx                     |  97 ++++++-
 python/pyarrow/includes/libarrow.pxd         |  17 ++
 python/pyarrow/includes/pyarrow.pxd          |  12 +-
 python/pyarrow/io.pyx                        |   2 +-
 python/pyarrow/schema.pyx                    |  14 +-
 python/pyarrow/table.pyx                     |   2 +-
 python/pyarrow/tests/pandas_examples.py      |   5 +-
 python/pyarrow/tests/test_convert_builtin.py |   2 +-
 python/pyarrow/tests/test_convert_pandas.py  |  15 +-
 python/pyarrow/tests/test_feather.py         |   2 -
 python/pyarrow/tests/test_jemalloc.py        |  17 +-
 26 files changed, 541 insertions(+), 144 deletions(-)
 create mode 100644 cpp/src/arrow/python/numpy_convert.cc
 create mode 100644 cpp/src/arrow/python/numpy_convert.h

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index bd33bf5b8296e..8eaa76ae9e843 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -28,6 +28,7 @@ install(FILES
   pretty_print.h
   status.h
   table.h
+  tensor.h
   type.h
   type_fwd.h
   type_traits.h
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 50a09515297ff..aa0da7580244a 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -29,6 +29,7 @@
 #include "arrow/pretty_print.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
+#include "arrow/tensor.h"
 #include "arrow/type.h"
 #include "arrow/visitor.h"
 
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index faaad89656f92..a8b4cc7ff1ded 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -55,6 +55,7 @@ set(ARROW_PYTHON_SRCS
   config.cc
   helpers.cc
   io.cc
+  numpy_convert.cc
   pandas_convert.cc
 )
 
@@ -71,6 +72,14 @@ ADD_ARROW_LIB(arrow_python
   STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
 )
 
+if ("${COMPILER_FAMILY}" STREQUAL "clang")
+  # Clang, be quiet. Python C API has lots of macros
+  set_property(SOURCE ${ARROW_PYTHON_SRCS}
+    APPEND_STRING
+    PROPERTY
+    COMPILE_FLAGS -Wno-parentheses-equality)
+endif()
+
 install(FILES
   api.h
   builtin_convert.h
@@ -79,6 +88,7 @@ install(FILES
   do_import_numpy.h
   helpers.h
   io.h
+  numpy_convert.h
   numpy_interop.h
   pandas_convert.h
   type_traits.h
diff --git a/cpp/src/arrow/python/api.h b/cpp/src/arrow/python/api.h
index f4f1c0cf9a5d6..895d1f447ff58 100644
--- a/cpp/src/arrow/python/api.h
+++ b/cpp/src/arrow/python/api.h
@@ -22,6 +22,7 @@
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/io.h"
+#include "arrow/python/numpy_convert.h"
 #include "arrow/python/pandas_convert.h"
 
 #endif  // ARROW_PYTHON_API_H
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index f1be471cd3a83..32bfa784acbd0 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -103,23 +103,6 @@ struct PyObjectStringify {
 ARROW_EXPORT void set_default_memory_pool(MemoryPool* pool);
 ARROW_EXPORT MemoryPool* get_memory_pool();
 
-class ARROW_EXPORT NumPyBuffer : public Buffer {
- public:
-  explicit NumPyBuffer(PyArrayObject* arr) : Buffer(nullptr, 0) {
-    arr_ = arr;
-    Py_INCREF(arr);
-
-    data_ = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
-    size_ = PyArray_SIZE(arr_) * PyArray_DESCR(arr_)->elsize;
-    capacity_ = size_;
-  }
-
-  virtual ~NumPyBuffer() { Py_XDECREF(arr_); }
-
- private:
-  PyArrayObject* arr_;
-};
-
 class ARROW_EXPORT PyBuffer : public Buffer {
  public:
   /// Note that the GIL must be held when calling the PyBuffer constructor.
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index add2d9a222adf..be5f412fbea1c 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -42,6 +42,7 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
       GET_PRIMITIVE_TYPE(DATE32, date32);
       GET_PRIMITIVE_TYPE(DATE64, date64);
       GET_PRIMITIVE_TYPE(BOOL, boolean);
+      GET_PRIMITIVE_TYPE(HALF_FLOAT, float16);
       GET_PRIMITIVE_TYPE(FLOAT, float32);
       GET_PRIMITIVE_TYPE(DOUBLE, float64);
       GET_PRIMITIVE_TYPE(BINARY, binary);
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
new file mode 100644
index 0000000000000..3697819120dbe
--- /dev/null
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -0,0 +1,267 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <Python.h>
+
+#include "arrow/python/numpy_convert.h"
+#include "arrow/python/numpy_interop.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "arrow/buffer.h"
+#include "arrow/tensor.h"
+#include "arrow/type.h"
+
+#include "arrow/python/common.h"
+#include "arrow/python/type_traits.h"
+
+namespace arrow {
+namespace py {
+
+bool is_contiguous(PyObject* array) {
+  if (PyArray_Check(array)) {
+    return PyArray_FLAGS(reinterpret_cast<PyArrayObject*>(array)) &
+           (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS);
+  } else {
+    return false;
+  }
+}
+
+int cast_npy_type_compat(int type_num) {
+// Both LONGLONG and INT64 can be observed in the wild, which is buggy. We set
+// U/LONGLONG to U/INT64 so things work properly.
+
+#if (NPY_INT64 == NPY_LONGLONG) && (NPY_SIZEOF_LONGLONG == 8)
+  if (type_num == NPY_LONGLONG) { type_num = NPY_INT64; }
+  if (type_num == NPY_ULONGLONG) { type_num = NPY_UINT64; }
+#endif
+
+  return type_num;
+}
+
+NumPyBuffer::NumPyBuffer(PyObject* ao) : Buffer(nullptr, 0) {
+  arr_ = ao;
+  Py_INCREF(ao);
+
+  if (PyArray_Check(ao)) {
+    PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(ao);
+    data_ = reinterpret_cast<const uint8_t*>(PyArray_DATA(ndarray));
+    size_ = PyArray_SIZE(ndarray) * PyArray_DESCR(ndarray)->elsize;
+    capacity_ = size_;
+
+    if (PyArray_FLAGS(ndarray) & NPY_ARRAY_WRITEABLE) { is_mutable_ = true; }
+  }
+}
+
+NumPyBuffer::~NumPyBuffer() {
+  Py_XDECREF(arr_);
+}
+
+#define TO_ARROW_TYPE_CASE(NPY_NAME, FACTORY) \
+  case NPY_##NPY_NAME:                        \
+    *out = FACTORY();                         \
+    break;
+
+Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out) {
+  PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
+  int type_num = cast_npy_type_compat(descr->type_num);
+
+  switch (type_num) {
+    TO_ARROW_TYPE_CASE(BOOL, uint8);
+    TO_ARROW_TYPE_CASE(INT8, int8);
+    TO_ARROW_TYPE_CASE(INT16, int16);
+    TO_ARROW_TYPE_CASE(INT32, int32);
+    TO_ARROW_TYPE_CASE(INT64, int64);
+#if (NPY_INT64 != NPY_LONGLONG)
+    TO_ARROW_TYPE_CASE(LONGLONG, int64);
+#endif
+    TO_ARROW_TYPE_CASE(UINT8, uint8);
+    TO_ARROW_TYPE_CASE(UINT16, uint16);
+    TO_ARROW_TYPE_CASE(UINT32, uint32);
+    TO_ARROW_TYPE_CASE(UINT64, uint64);
+#if (NPY_UINT64 != NPY_ULONGLONG)
+    TO_ARROW_CASE(ULONGLONG);
+#endif
+    TO_ARROW_TYPE_CASE(FLOAT16, float16);
+    TO_ARROW_TYPE_CASE(FLOAT32, float32);
+    TO_ARROW_TYPE_CASE(FLOAT64, float64);
+    default: {
+      std::stringstream ss;
+      ss << "Unsupported numpy type " << descr->type_num << std::endl;
+      return Status::NotImplemented(ss.str());
+    }
+  }
+  return Status::OK();
+}
+
+Status GetNumPyType(const DataType& type, int* type_num) {
+#define NUMPY_TYPE_CASE(ARROW_NAME, NPY_NAME) \
+  case Type::ARROW_NAME:                      \
+    *type_num = NPY_##NPY_NAME;               \
+    break;
+
+  switch (type.type) {
+    NUMPY_TYPE_CASE(UINT8, UINT8);
+    NUMPY_TYPE_CASE(INT8, INT8);
+    NUMPY_TYPE_CASE(UINT16, UINT16);
+    NUMPY_TYPE_CASE(INT16, INT16);
+    NUMPY_TYPE_CASE(UINT32, UINT32);
+    NUMPY_TYPE_CASE(INT32, INT32);
+    NUMPY_TYPE_CASE(UINT64, UINT64);
+    NUMPY_TYPE_CASE(INT64, INT64);
+    NUMPY_TYPE_CASE(HALF_FLOAT, FLOAT16);
+    NUMPY_TYPE_CASE(FLOAT, FLOAT32);
+    NUMPY_TYPE_CASE(DOUBLE, FLOAT64);
+    default: {
+      std::stringstream ss;
+      ss << "Unsupported tensor type: " << type.ToString() << std::endl;
+      return Status::NotImplemented(ss.str());
+    }
+  }
+#undef NUMPY_TYPE_CASE
+
+  return Status::OK();
+}
+
+Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
+  PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
+
+  int type_num = cast_npy_type_compat(descr->type_num);
+
+  switch (type_num) {
+    TO_ARROW_TYPE_CASE(BOOL, boolean);
+    TO_ARROW_TYPE_CASE(INT8, int8);
+    TO_ARROW_TYPE_CASE(INT16, int16);
+    TO_ARROW_TYPE_CASE(INT32, int32);
+    TO_ARROW_TYPE_CASE(INT64, int64);
+#if (NPY_INT64 != NPY_LONGLONG)
+    TO_ARROW_TYPE_CASE(LONGLONG, int64);
+#endif
+    TO_ARROW_TYPE_CASE(UINT8, uint8);
+    TO_ARROW_TYPE_CASE(UINT16, uint16);
+    TO_ARROW_TYPE_CASE(UINT32, uint32);
+    TO_ARROW_TYPE_CASE(UINT64, uint64);
+#if (NPY_UINT64 != NPY_ULONGLONG)
+    TO_ARROW_CASE(ULONGLONG);
+#endif
+    TO_ARROW_TYPE_CASE(FLOAT32, float32);
+    TO_ARROW_TYPE_CASE(FLOAT64, float64);
+    case NPY_DATETIME: {
+      auto date_dtype =
+          reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+      TimeUnit unit;
+      switch (date_dtype->meta.base) {
+        case NPY_FR_s:
+          unit = TimeUnit::SECOND;
+          break;
+        case NPY_FR_ms:
+          unit = TimeUnit::MILLI;
+          break;
+        case NPY_FR_us:
+          unit = TimeUnit::MICRO;
+          break;
+        case NPY_FR_ns:
+          unit = TimeUnit::NANO;
+          break;
+        default:
+          return Status::NotImplemented("Unsupported datetime64 time unit");
+      }
+      *out = timestamp(unit);
+    } break;
+    default: {
+      std::stringstream ss;
+      ss << "Unsupported numpy type " << descr->type_num << std::endl;
+      return Status::NotImplemented(ss.str());
+    }
+  }
+
+  return Status::OK();
+}
+
+#undef TO_ARROW_TYPE_CASE
+
+Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>* out) {
+  if (!PyArray_Check(ao)) { return Status::TypeError("Did not pass ndarray object"); }
+
+  PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(ao);
+
+  // TODO(wesm): What do we want to do with non-contiguous memory and negative strides?
+
+  int ndim = PyArray_NDIM(ndarray);
+
+  std::shared_ptr<Buffer> data = std::make_shared<NumPyBuffer>(ao);
+  std::vector<int64_t> shape(ndim);
+  std::vector<int64_t> strides(ndim);
+
+  npy_intp* array_strides = PyArray_STRIDES(ndarray);
+  npy_intp* array_shape = PyArray_SHAPE(ndarray);
+  for (int i = 0; i < ndim; ++i) {
+    if (array_strides[i] < 0) {
+      return Status::Invalid("Negative ndarray strides not supported");
+    }
+    shape[i] = array_shape[i];
+    strides[i] = array_strides[i];
+  }
+
+  std::shared_ptr<DataType> type;
+  RETURN_NOT_OK(
+      GetTensorType(reinterpret_cast<PyObject*>(PyArray_DESCR(ndarray)), &type));
+  return MakeTensor(type, data, shape, strides, {}, out);
+}
+
+Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out) {
+  int type_num;
+  RETURN_NOT_OK(GetNumPyType(*tensor.type(), &type_num));
+  PyArray_Descr* dtype = PyArray_DescrNewFromType(type_num);
+  RETURN_IF_PYERROR();
+
+  std::vector<npy_intp> npy_shape(tensor.ndim());
+  std::vector<npy_intp> npy_strides(tensor.ndim());
+
+  for (int i = 0; i < tensor.ndim(); ++i) {
+    npy_shape[i] = tensor.shape()[i];
+    npy_strides[i] = tensor.strides()[i];
+  }
+
+  const void* immutable_data = nullptr;
+  if (tensor.data()) { immutable_data = tensor.data()->data(); }
+
+  // Remove const =(
+  void* mutable_data = const_cast<void*>(immutable_data);
+
+  int array_flags = 0;
+  if (tensor.is_row_major()) { array_flags |= NPY_ARRAY_C_CONTIGUOUS; }
+  if (tensor.is_column_major()) { array_flags |= NPY_ARRAY_F_CONTIGUOUS; }
+  if (tensor.is_mutable()) { array_flags |= NPY_ARRAY_WRITEABLE; }
+
+  PyObject* result = PyArray_NewFromDescr(&PyArray_Type, dtype, tensor.ndim(),
+      npy_shape.data(), npy_strides.data(), mutable_data, array_flags, nullptr);
+  RETURN_IF_PYERROR()
+
+  if (base != Py_None) {
+    PyArray_SetBaseObject(reinterpret_cast<PyArrayObject*>(result), base);
+  }
+  *out = result;
+  return Status::OK();
+}
+
+}  // namespace py
+}  // namespace arrow
diff --git a/cpp/src/arrow/python/numpy_convert.h b/cpp/src/arrow/python/numpy_convert.h
new file mode 100644
index 0000000000000..685a626d4ca28
--- /dev/null
+++ b/cpp/src/arrow/python/numpy_convert.h
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Functions for converting between pandas's NumPy-based data representation
+// and Arrow data structures
+
+#ifndef ARROW_PYTHON_NUMPY_CONVERT_H
+#define ARROW_PYTHON_NUMPY_CONVERT_H
+
+#include <Python.h>
+
+#include <memory>
+#include <string>
+
+#include "arrow/buffer.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+struct DataType;
+class MemoryPool;
+class Status;
+class Tensor;
+
+namespace py {
+
+class ARROW_EXPORT NumPyBuffer : public Buffer {
+ public:
+  explicit NumPyBuffer(PyObject* arr);
+  virtual ~NumPyBuffer();
+
+ private:
+  PyObject* arr_;
+};
+
+// Handle misbehaved types like LONGLONG and ULONGLONG
+int cast_npy_type_compat(int type_num);
+
+bool is_contiguous(PyObject* array);
+
+ARROW_EXPORT
+Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out);
+
+Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out);
+Status GetNumPyType(const DataType& type, int* type_num);
+
+ARROW_EXPORT Status NdarrayToTensor(
+    MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>* out);
+
+ARROW_EXPORT Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out);
+
+}  // namespace py
+}  // namespace arrow
+
+#endif  // ARROW_PYTHON_NUMPY_CONVERT_H
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index ddfec1bf45a2e..01019e5669f2d 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -36,11 +36,6 @@
 
 #include "arrow/array.h"
 #include "arrow/loader.h"
-#include "arrow/python/builtin_convert.h"
-#include "arrow/python/common.h"
-#include "arrow/python/config.h"
-#include "arrow/python/type_traits.h"
-#include "arrow/python/util/datetime.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type_fwd.h"
@@ -49,24 +44,19 @@
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 
+#include "arrow/python/builtin_convert.h"
+#include "arrow/python/common.h"
+#include "arrow/python/config.h"
+#include "arrow/python/numpy_convert.h"
+#include "arrow/python/type_traits.h"
+#include "arrow/python/util/datetime.h"
+
 namespace arrow {
 namespace py {
 
 // ----------------------------------------------------------------------
 // Utility code
 
-int cast_npy_type_compat(int type_num) {
-// Both LONGLONG and INT64 can be observed in the wild, which is buggy. We set
-// U/LONGLONG to U/INT64 so things work properly.
-
-#if (NPY_INT64 == NPY_LONGLONG) && (NPY_SIZEOF_LONGLONG == 8)
-  if (type_num == NPY_LONGLONG) { type_num = NPY_INT64; }
-  if (type_num == NPY_ULONGLONG) { type_num = NPY_UINT64; }
-#endif
-
-  return type_num;
-}
-
 static inline bool PyObject_is_null(const PyObject* obj) {
   return obj == Py_None || obj == numpy_nan;
 }
@@ -395,7 +385,7 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
     return Status::NotImplemented("NumPy type casts not yet implemented");
   }
 
-  *data = std::make_shared<NumPyBuffer>(arr_);
+  *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
   return Status::OK();
 }
 
@@ -730,68 +720,6 @@ Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
   return converter.ConvertObjects(out);
 }
 
-Status PandasDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
-  PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
-
-  int type_num = cast_npy_type_compat(descr->type_num);
-
-#define TO_ARROW_TYPE_CASE(NPY_NAME, FACTORY) \
-  case NPY_##NPY_NAME:                        \
-    *out = FACTORY();                         \
-    break;
-
-  switch (type_num) {
-    TO_ARROW_TYPE_CASE(BOOL, boolean);
-    TO_ARROW_TYPE_CASE(INT8, int8);
-    TO_ARROW_TYPE_CASE(INT16, int16);
-    TO_ARROW_TYPE_CASE(INT32, int32);
-    TO_ARROW_TYPE_CASE(INT64, int64);
-#if (NPY_INT64 != NPY_LONGLONG)
-    TO_ARROW_TYPE_CASE(LONGLONG, int64);
-#endif
-    TO_ARROW_TYPE_CASE(UINT8, uint8);
-    TO_ARROW_TYPE_CASE(UINT16, uint16);
-    TO_ARROW_TYPE_CASE(UINT32, uint32);
-    TO_ARROW_TYPE_CASE(UINT64, uint64);
-#if (NPY_UINT64 != NPY_ULONGLONG)
-    TO_ARROW_CASE(ULONGLONG);
-#endif
-    TO_ARROW_TYPE_CASE(FLOAT32, float32);
-    TO_ARROW_TYPE_CASE(FLOAT64, float64);
-    case NPY_DATETIME: {
-      auto date_dtype =
-          reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-      TimeUnit unit;
-      switch (date_dtype->meta.base) {
-        case NPY_FR_s:
-          unit = TimeUnit::SECOND;
-          break;
-        case NPY_FR_ms:
-          unit = TimeUnit::MILLI;
-          break;
-        case NPY_FR_us:
-          unit = TimeUnit::MICRO;
-          break;
-        case NPY_FR_ns:
-          unit = TimeUnit::NANO;
-          break;
-        default:
-          return Status::NotImplemented("Unsupported datetime64 time unit");
-      }
-      *out = timestamp(unit);
-    } break;
-    default: {
-      std::stringstream ss;
-      ss << "Unsupported numpy type " << descr->type_num << std::endl;
-      return Status::NotImplemented(ss.str());
-    }
-  }
-
-#undef TO_ARROW_TYPE_CASE
-
-  return Status::OK();
-}
-
 // ----------------------------------------------------------------------
 // pandas 0.x DataFrame conversion internals
 
diff --git a/cpp/src/arrow/python/pandas_convert.h b/cpp/src/arrow/python/pandas_convert.h
index 105c1598d3936..8fd31076a994f 100644
--- a/cpp/src/arrow/python/pandas_convert.h
+++ b/cpp/src/arrow/python/pandas_convert.h
@@ -61,9 +61,6 @@ ARROW_EXPORT
 Status ConvertTableToPandas(
     const std::shared_ptr<Table>& table, int nthreads, PyObject** out);
 
-ARROW_EXPORT
-Status PandasDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out);
-
 ARROW_EXPORT
 Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 01e30f5a36ce8..f269ebfb642c7 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -26,9 +26,9 @@
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 
+#include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
 #include "arrow/python/pandas_convert.h"
-#include "arrow/python/builtin_convert.h"
 
 namespace arrow {
 namespace py {
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index 9a8de5119ea58..8bbb97b596e18 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -90,13 +90,21 @@ int64_t Tensor::size() const {
 }
 
 bool Tensor::is_contiguous() const {
-  std::vector<int64_t> c_strides;
-  std::vector<int64_t> f_strides;
+  return is_row_major() || is_column_major();
+}
 
+bool Tensor::is_row_major() const {
+  std::vector<int64_t> c_strides;
   const auto& fw_type = static_cast<const FixedWidthType&>(*type_);
   ComputeRowMajorStrides(fw_type, shape_, &c_strides);
+  return strides_ == c_strides;
+}
+
+bool Tensor::is_column_major() const {
+  std::vector<int64_t> f_strides;
+  const auto& fw_type = static_cast<const FixedWidthType&>(*type_);
   ComputeColumnMajorStrides(fw_type, shape_, &f_strides);
-  return strides_ == c_strides || strides_ == f_strides;
+  return strides_ == f_strides;
 }
 
 bool Tensor::Equals(const Tensor& other) const {
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index eeb5c3e8e5536..12015f14b1d3d 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -89,8 +89,15 @@ class ARROW_EXPORT Tensor {
   /// Return true if the underlying data buffer is mutable
   bool is_mutable() const { return data_->is_mutable(); }
 
+  /// Either row major or column major
   bool is_contiguous() const;
 
+  /// AKA "C order"
+  bool is_row_major() const;
+
+  /// AKA "Fortran order"
+  bool is_column_major() const;
+
   Type::type type_enum() const { return type_->type; }
 
   bool Equals(const Tensor& other) const;
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 3df2a1d445549..5215028c90f0d 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -28,8 +28,7 @@
 import pyarrow.config
 from pyarrow.config import cpu_count, set_cpu_count
 
-from pyarrow.array import (Array,
-                           from_pandas_series, from_pylist,
+from pyarrow.array import (Array, Tensor, from_pylist,
                            NumericArray, IntegerArray, FloatingPointArray,
                            BooleanArray,
                            Int8Array, UInt8Array,
@@ -63,7 +62,8 @@
                             int8, int16, int32, int64,
                             uint8, uint16, uint32, uint64,
                             timestamp, date32, date64,
-                            float_, double, binary, string,
+                            float16, float32, float64,
+                            binary, string,
                             list_, struct, dictionary, field,
                             DataType, FixedSizeBinaryType,
                             Field, Schema, schema)
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index 0b5f33d0d2db6..42675630fd51b 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -16,7 +16,7 @@
 # under the License.
 
 from pyarrow.includes.common cimport shared_ptr, int64_t
-from pyarrow.includes.libarrow cimport CArray
+from pyarrow.includes.libarrow cimport CArray, CTensor
 
 from pyarrow.scalar import NA
 
@@ -41,6 +41,17 @@ cdef class Array:
     cdef getitem(self, int64_t i)
 
 
+cdef class Tensor:
+    cdef:
+        shared_ptr[CTensor] sp_tensor
+        CTensor* tp
+
+    cdef readonly:
+        DataType type
+
+    cdef init(self, const shared_ptr[CTensor]& sp_tensor)
+
+
 cdef object box_array(const shared_ptr[CArray]& sp_array)
 
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index b9799f15bf3e7..398e4cbffa94d 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -81,7 +81,7 @@ cdef class Array:
         self.type = box_data_type(self.sp_array.get().type())
 
     @staticmethod
-    def from_pandas(obj, mask=None, DataType type=None,
+    def from_numpy(obj, mask=None, DataType type=None,
                     timestamps_to_ms=False,
                     MemoryPool memory_pool=None):
         """
@@ -116,7 +116,7 @@ cdef class Array:
 
         >>> import pandas as pd
         >>> import pyarrow as pa
-        >>> pa.Array.from_pandas(pd.Series([1, 2]))
+        >>> pa.Array.from_numpy(pd.Series([1, 2]))
         <pyarrow.array.Int64Array object at 0x7f674e4c0e10>
         [
           1,
@@ -124,7 +124,7 @@ cdef class Array:
         ]
 
         >>> import numpy as np
-        >>> pa.Array.from_pandas(pd.Series([1, 2]), np.array([0, 1],
+        >>> pa.Array.from_numpy(pd.Series([1, 2]), np.array([0, 1],
         ... dtype=bool))
         <pyarrow.array.Int64Array object at 0x7f9019e11208>
         [
@@ -166,7 +166,7 @@ cdef class Array:
                 values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
 
             if type is None:
-                check_status(pyarrow.PandasDtypeToArrow(values.dtype, &c_type))
+                check_status(pyarrow.NumPyDtypeToArrow(values.dtype, &c_type))
             else:
                 c_type = type.sp_type
 
@@ -316,6 +316,77 @@ cdef class Array:
         return [x.as_py() for x in self]
 
 
+cdef class Tensor:
+
+    cdef init(self, const shared_ptr[CTensor]& sp_tensor):
+        self.sp_tensor = sp_tensor
+        self.tp = sp_tensor.get()
+        self.type = box_data_type(self.tp.type())
+
+    def __repr__(self):
+        return """<pyarrow.Tensor>
+type: {0}
+shape: {1}
+strides: {2}""".format(self.type, self.shape, self.strides)
+
+    @staticmethod
+    def from_numpy(obj):
+        cdef shared_ptr[CTensor] ctensor
+        check_status(pyarrow.NdarrayToTensor(default_memory_pool(),
+                                             obj, &ctensor))
+        return box_tensor(ctensor)
+
+    def to_numpy(self):
+        """
+        Convert arrow::Tensor to numpy.ndarray with zero copy
+        """
+        cdef:
+            PyObject* out
+
+        check_status(pyarrow.TensorToNdarray(deref(self.tp), <PyObject*> self,
+                                             &out))
+        return PyObject_to_object(out)
+
+    property is_mutable:
+
+        def __get__(self):
+            return self.tp.is_mutable()
+
+    property is_contiguous:
+
+        def __get__(self):
+            return self.tp.is_contiguous()
+
+    property ndim:
+
+        def __get__(self):
+            return self.tp.ndim()
+
+    property size:
+
+        def __get__(self):
+            return self.tp.size()
+
+    property shape:
+
+        def __get__(self):
+            cdef size_t i
+            py_shape = []
+            for i in range(self.tp.shape().size()):
+                py_shape.append(self.tp.shape()[i])
+            return py_shape
+
+    property strides:
+
+        def __get__(self):
+            cdef size_t i
+            py_strides = []
+            for i in range(self.tp.strides().size()):
+                py_strides.append(self.tp.strides()[i])
+            return py_strides
+
+
+
 cdef wrap_array_output(PyObject* output):
     cdef object obj = PyObject_to_object(output)
 
@@ -479,10 +550,10 @@ cdef class DictionaryArray(Array):
         else:
             mask = mask | (indices == -1)
 
-        arrow_indices = Array.from_pandas(indices, mask=mask,
-                                          memory_pool=memory_pool)
-        arrow_dictionary = Array.from_pandas(dictionary,
-                                             memory_pool=memory_pool)
+        arrow_indices = Array.from_numpy(indices, mask=mask,
+                                         memory_pool=memory_pool)
+        arrow_dictionary = Array.from_numpy(dictionary,
+                                            memory_pool=memory_pool)
 
         if not isinstance(arrow_indices, IntegerArray):
             raise ValueError('Indices must be integer type')
@@ -535,6 +606,15 @@ cdef object box_array(const shared_ptr[CArray]& sp_array):
     return arr
 
 
+cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor):
+    if sp_tensor.get() == NULL:
+        raise ValueError('Tensor was NULL')
+
+    cdef Tensor tensor = Tensor()
+    tensor.init(sp_tensor)
+    return tensor
+
+
 cdef object get_series_values(object obj):
     import pandas as pd
 
@@ -549,4 +629,3 @@ cdef object get_series_values(object obj):
 
 
 from_pylist = Array.from_list
-from_pandas_series = Array.from_pandas
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index f549884d175fa..8da063cbdc364 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -35,6 +35,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         Type_UINT64" arrow::Type::UINT64"
         Type_INT64" arrow::Type::INT64"
 
+        Type_HALF_FLOAT" arrow::Type::HALF_FLOAT"
         Type_FLOAT" arrow::Type::FLOAT"
         Type_DOUBLE" arrow::Type::DOUBLE"
 
@@ -282,6 +283,22 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         CStatus RemoveColumn(int i, shared_ptr[CTable]* out)
 
+    cdef cppclass CTensor" arrow::Tensor":
+        shared_ptr[CDataType] type()
+        shared_ptr[CBuffer] data()
+
+        const vector[int64_t]& shape()
+        const vector[int64_t]& strides()
+        int64_t size()
+
+        int ndim()
+        const c_string& dim_name(int i)
+
+        c_bool is_mutable()
+        c_bool is_contiguous()
+        Type type_enum()
+        c_bool Equals(const CTensor& other)
+
     CStatus ConcatenateTables(const vector[shared_ptr[CTable]]& tables,
                               shared_ptr[CTable]* result)
 
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 8142c1c06ff75..9b64435e48d7f 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -18,8 +18,8 @@
 # distutils: language = c++
 
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn,
-                                        CTable, CDataType, CStatus, Type,
+from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CDataType,
+                                        CTable, CTensor, CStatus, Type,
                                         CMemoryPool, TimeUnit)
 
 cimport pyarrow.includes.libarrow_io as arrow_io
@@ -34,7 +34,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
                               shared_ptr[CArray]* out,
                               const shared_ptr[CDataType]& type)
 
-    CStatus PandasDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
+    CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
 
     CStatus PandasToArrow(CMemoryPool* pool, object ao, object mo,
                           const shared_ptr[CDataType]& type,
@@ -44,6 +44,12 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
                                  const shared_ptr[CDataType]& type,
                                  shared_ptr[CArray]* out)
 
+    CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
+                            shared_ptr[CTensor]* out);
+
+    CStatus TensorToNdarray(const CTensor& tensor, PyObject* base,
+                            PyObject** out)
+
     CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
                                  PyObject* py_ref, PyObject** out)
 
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 0b27379c273b0..608b20d896ae3 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -1095,7 +1095,7 @@ cdef class FeatherWriter:
         if isinstance(col, Array):
             arr = col
         else:
-            arr = Array.from_pandas(col, mask=mask)
+            arr = Array.from_numpy(col, mask=mask)
 
         cdef c_string c_name = tobytes(name)
 
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 06df64461ae22..253be4590b518 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -241,7 +241,9 @@ cdef set PRIMITIVE_TYPES = set([
     la.Type_UINT32, la.Type_INT32,
     la.Type_UINT64, la.Type_INT64,
     la.Type_TIMESTAMP, la.Type_DATE32,
-    la.Type_DATE64, la.Type_FLOAT,
+    la.Type_DATE64,
+    la.Type_HALF_FLOAT,
+    la.Type_FLOAT,
     la.Type_DOUBLE])
 
 
@@ -340,11 +342,15 @@ def date64():
     return primitive_type(la.Type_DATE64)
 
 
-def float_():
+def float16():
+    return primitive_type(la.Type_HALF_FLOAT)
+
+
+def float32():
     return primitive_type(la.Type_FLOAT)
 
 
-def double():
+def float64():
     return primitive_type(la.Type_DOUBLE)
 
 
@@ -452,6 +458,6 @@ cdef Schema box_schema(const shared_ptr[CSchema]& type):
 def type_from_numpy_dtype(object dtype):
     cdef shared_ptr[CDataType] c_type
     with nogil:
-        check_status(pyarrow.PandasDtypeToArrow(dtype, &c_type))
+        check_status(pyarrow.NumPyDtypeToArrow(dtype, &c_type))
 
     return box_data_type(c_type)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index e6fddbd0cfbbd..94389a73cc974 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -309,7 +309,7 @@ cdef _dataframe_to_arrays(df, timestamps_to_ms, Schema schema):
         if schema is not None:
             type = schema.field_by_name(name).type
 
-        arr = Array.from_pandas(col, type=type,
+        arr = Array.from_numpy(col, type=type,
                                 timestamps_to_ms=timestamps_to_ms)
         names.append(name)
         arrays.append(arr)
diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py
index c9343fce233d2..e081c38713057 100644
--- a/python/pyarrow/tests/pandas_examples.py
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -37,7 +37,7 @@ def dataframe_with_arrays():
               ('i4', pa.int32()), ('i8', pa.int64()),
               ('u1', pa.uint8()), ('u2', pa.uint16()),
               ('u4', pa.uint32()), ('u8', pa.uint64()),
-              ('f4', pa.float_()), ('f8', pa.double())]
+              ('f4', pa.float32()), ('f8', pa.float64())]
 
     arrays = OrderedDict()
     fields = []
@@ -77,6 +77,7 @@ def dataframe_with_arrays():
 
     return df, schema
 
+
 def dataframe_with_lists():
     """
     Dataframe with list columns of every possible primtive type.
@@ -97,7 +98,7 @@ def dataframe_with_lists():
         None,
         [0]
     ]
-    fields.append(pa.field('double', pa.list_(pa.double())))
+    fields.append(pa.field('double', pa.list_(pa.float64())))
     arrays['double'] = [
         [0., 1., 2., 3., 4., 5., 6., 7., 8., 9.],
         [0., 1., 2., 3., 4.],
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index bb6d2d17d5f0c..15fca560c6513 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -70,7 +70,7 @@ def test_double(self):
         arr = pyarrow.from_pylist(data)
         assert len(arr) == 6
         assert arr.null_count == 3
-        assert arr.type == pyarrow.double()
+        assert arr.type == pyarrow.float64()
         assert arr.to_pylist() == data
 
     def test_unicode(self):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index c472ee69034c8..0b3c02e9945eb 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -22,7 +22,6 @@
 import unittest
 
 import numpy as np
-import numpy.testing as npt
 
 import pandas as pd
 import pandas.util.testing as tm
@@ -78,8 +77,8 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
 
     def _check_array_roundtrip(self, values, expected=None,
                                timestamps_to_ms=False, type=None):
-        arr = A.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
-                                  type=type)
+        arr = A.Array.from_numpy(values, timestamps_to_ms=timestamps_to_ms,
+                                 type=type)
         result = arr.to_pandas()
 
         assert arr.null_count == pd.isnull(values).sum()
@@ -90,7 +89,7 @@ def _check_array_roundtrip(self, values, expected=None,
     def test_float_no_nulls(self):
         data = {}
         fields = []
-        dtypes = [('f4', A.float_()), ('f8', A.double())]
+        dtypes = [('f4', A.float32()), ('f8', A.float64())]
         num_values = 100
 
         for numpy_dtype, arrow_dtype in dtypes:
@@ -106,7 +105,7 @@ def test_float_nulls(self):
         num_values = 100
 
         null_mask = np.random.randint(0, 10, size=num_values) < 3
-        dtypes = [('f4', A.float_()), ('f8', A.double())]
+        dtypes = [('f4', A.float32()), ('f8', A.float64())]
         names = ['f4', 'f8']
         expected_cols = []
 
@@ -115,7 +114,7 @@ def test_float_nulls(self):
         for name, arrow_dtype in dtypes:
             values = np.random.randn(num_values).astype(name)
 
-            arr = A.from_pandas_series(values, null_mask)
+            arr = A.Array.from_numpy(values, null_mask)
             arrays.append(arr)
             fields.append(A.Field.from_py(name, arrow_dtype))
             values[null_mask] = np.nan
@@ -168,7 +167,7 @@ def test_integer_with_nulls(self):
         for name in int_dtypes:
             values = np.random.randint(0, 100, size=num_values)
 
-            arr = A.from_pandas_series(values, null_mask)
+            arr = A.Array.from_numpy(values, null_mask)
             arrays.append(arr)
 
             expected = values.astype('f8')
@@ -202,7 +201,7 @@ def test_boolean_nulls(self):
         mask = np.random.randint(0, 10, size=num_values) < 3
         values = np.random.randint(0, 10, size=num_values) < 5
 
-        arr = A.from_pandas_series(values, mask)
+        arr = A.Array.from_numpy(values, mask)
 
         expected = values.astype(object)
         expected[mask] = None
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index dd6888f2d1306..525da344c9951 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -15,8 +15,6 @@
 import os
 import unittest
 
-import pytest
-
 from numpy.testing import assert_array_equal
 import numpy as np
 
diff --git a/python/pyarrow/tests/test_jemalloc.py b/python/pyarrow/tests/test_jemalloc.py
index 8efd514dd0cae..c6cc2cc34a08b 100644
--- a/python/pyarrow/tests/test_jemalloc.py
+++ b/python/pyarrow/tests/test_jemalloc.py
@@ -33,11 +33,15 @@ def test_different_memory_pool():
     gc.collect()
     bytes_before_default = pyarrow.total_allocated_bytes()
     bytes_before_jemalloc = pyarrow.jemalloc.default_pool().bytes_allocated()
-    array = pyarrow.from_pylist([1, None, 3, None],
-                        memory_pool=pyarrow.jemalloc.default_pool())
+
+    # it works
+    array = pyarrow.from_pylist([1, None, 3, None],  # noqa
+                                memory_pool=pyarrow.jemalloc.default_pool())
     gc.collect()
     assert pyarrow.total_allocated_bytes() == bytes_before_default
-    assert pyarrow.jemalloc.default_pool().bytes_allocated() > bytes_before_jemalloc
+    assert (pyarrow.jemalloc.default_pool().bytes_allocated() >
+            bytes_before_jemalloc)
+
 
 @jemalloc
 def test_default_memory_pool():
@@ -47,10 +51,13 @@ def test_default_memory_pool():
 
     old_memory_pool = pyarrow.memory.default_pool()
     pyarrow.memory.set_default_pool(pyarrow.jemalloc.default_pool())
-    array = pyarrow.from_pylist([1, None, 3, None])
+
+    array = pyarrow.from_pylist([1, None, 3, None])  # noqa
+
     pyarrow.memory.set_default_pool(old_memory_pool)
     gc.collect()
 
     assert pyarrow.total_allocated_bytes() == bytes_before_default
-    assert pyarrow.jemalloc.default_pool().bytes_allocated() > bytes_before_jemalloc
 
+    assert (pyarrow.jemalloc.default_pool().bytes_allocated() >
+            bytes_before_jemalloc)

From d54ab9a23aa8d4fe52ce91b117511540cc7491bb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 2 Apr 2017 12:06:56 -0400
Subject: [PATCH 0454/1644] ARROW-737: [C++] Enable mutable buffer slices,
 SliceMutableBuffer function

This patch also results in better microperformance on my laptop.

```
Run on (4 X 1200 MHz CPU s)
2017-04-01 22:35:30
Benchmark                                                  Time           CPU Iterations
----------------------------------------------------------------------------------------
BM_WriteRecordBatch/1/min_time:1.000/real_time         53617 ns      53423 ns      26903   18.2136GB/s
BM_WriteRecordBatch/4/min_time:1.000/real_time         48118 ns      47999 ns      26823    20.295GB/s
BM_WriteRecordBatch/16/min_time:1.000/real_time        46679 ns      46562 ns      30409   20.9209GB/s
BM_WriteRecordBatch/64/min_time:1.000/real_time        50583 ns      50344 ns      27308   19.3061GB/s
BM_WriteRecordBatch/256/min_time:1.000/real_time       93474 ns      93259 ns      11720   10.4474GB/s
BM_WriteRecordBatch/1024/min_time:1.000/real_time     254056 ns     253461 ns       5434   3.84389GB/s
BM_WriteRecordBatch/4k/min_time:1.000/real_time       892292 ns     888924 ns       1210   1120.71MB/s
BM_WriteRecordBatch/8k/min_time:1.000/real_time      1799323 ns    1795023 ns        754   555.764MB/s
BM_ReadRecordBatch/1/min_time:1.000/real_time           2501 ns       2452 ns     586633   390.521GB/s
BM_ReadRecordBatch/4/min_time:1.000/real_time           6921 ns       5577 ns     227670   141.095GB/s
BM_ReadRecordBatch/16/min_time:1.000/real_time         15561 ns      14505 ns      67493    62.758GB/s
BM_ReadRecordBatch/64/min_time:1.000/real_time         48583 ns      48242 ns      30070   20.1008GB/s
BM_ReadRecordBatch/256/min_time:1.000/real_time       184637 ns     183306 ns       6660    5.2891GB/s
BM_ReadRecordBatch/1024/min_time:1.000/real_time      734128 ns     729692 ns       1905   1.33024GB/s
BM_ReadRecordBatch/4k/min_time:1.000/real_time       3027028 ns    3008020 ns        445   330.357MB/s
BM_ReadRecordBatch/8k/min_time:1.000/real_time       6472022 ns    6426801 ns        211   154.511MB/s
```

before

```
Run on (4 X 1200 MHz CPU s)
2017-04-01 22:37:36
Benchmark                                                  Time           CPU Iterations
----------------------------------------------------------------------------------------
BM_WriteRecordBatch/1/min_time:1.000/real_time         59317 ns      59202 ns      22727   16.4633GB/s
BM_WriteRecordBatch/4/min_time:1.000/real_time         56322 ns      56191 ns      24944   17.3389GB/s
BM_WriteRecordBatch/16/min_time:1.000/real_time        52027 ns      51880 ns      26682   18.7703GB/s
BM_WriteRecordBatch/64/min_time:1.000/real_time        56324 ns      56202 ns      24724   17.3384GB/s
BM_WriteRecordBatch/256/min_time:1.000/real_time      108096 ns     107868 ns      11284   9.03423GB/s
BM_WriteRecordBatch/1024/min_time:1.000/real_time     279508 ns     278730 ns       4957   3.49386GB/s
BM_WriteRecordBatch/4k/min_time:1.000/real_time      1013229 ns    1009772 ns       1191   986.944MB/s
BM_WriteRecordBatch/8k/min_time:1.000/real_time      2011309 ns    2005377 ns        661   497.189MB/s
BM_ReadRecordBatch/1/min_time:1.000/real_time           2507 ns       2501 ns     558949   389.563GB/s
BM_ReadRecordBatch/4/min_time:1.000/real_time           5120 ns       5110 ns     275798   190.735GB/s
BM_ReadRecordBatch/16/min_time:1.000/real_time         15800 ns      15706 ns      85481   61.8072GB/s
BM_ReadRecordBatch/64/min_time:1.000/real_time         55678 ns      55476 ns      25022   17.5393GB/s
BM_ReadRecordBatch/256/min_time:1.000/real_time       218083 ns     217596 ns       6163   4.47794GB/s
BM_ReadRecordBatch/1024/min_time:1.000/real_time      875861 ns     873419 ns       1591   1.11497GB/s
BM_ReadRecordBatch/4k/min_time:1.000/real_time       3545586 ns    3538141 ns        383   282.041MB/s
BM_ReadRecordBatch/8k/min_time:1.000/real_time       7118830 ns    7104433 ns        194   140.473MB/s
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #479 from wesm/ARROW-737 and squashes the following commits:

b663ca4 [Wes McKinney] Enable mutable buffer slices, SliceMutableBuffer function
---
 cpp/src/arrow/buffer-test.cc | 17 +++++++++++++++++
 cpp/src/arrow/buffer.cc      | 17 ++++++++++++-----
 cpp/src/arrow/buffer.h       | 25 +++++++++++++++++--------
 3 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index e0a2137b9bd78..5815ed17af50e 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -168,4 +168,21 @@ TEST_F(TestBuffer, SliceBuffer) {
   ASSERT_EQ(2, buf.use_count());
 }
 
+TEST_F(TestBuffer, SliceMutableBuffer) {
+  std::string data_str = "some data to slice";
+  auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
+
+  std::shared_ptr<MutableBuffer> buffer;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), 50, &buffer));
+
+  memcpy(buffer->mutable_data(), data, data_str.size());
+
+  std::shared_ptr<Buffer> slice = SliceMutableBuffer(buffer, 5, 10);
+  ASSERT_TRUE(slice->is_mutable());
+  ASSERT_EQ(10, slice->size());
+
+  Buffer expected(data + 5, 10);
+  ASSERT_TRUE(slice->Equals(expected));
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 59623403e5c5e..fb6379894c3b0 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -27,11 +27,6 @@
 
 namespace arrow {
 
-Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size)
-    : Buffer(parent->data() + offset, size) {
-  parent_ = parent;
-}
-
 Buffer::~Buffer() {}
 
 Status Buffer::Copy(
@@ -116,6 +111,18 @@ Status PoolBuffer::Resize(int64_t new_size, bool shrink_to_fit) {
   return Status::OK();
 }
 
+std::shared_ptr<Buffer> SliceMutableBuffer(
+    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
+  return std::make_shared<MutableBuffer>(buffer, offset, length);
+}
+
+MutableBuffer::MutableBuffer(
+    const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size)
+    : MutableBuffer(parent->mutable_data() + offset, size) {
+  DCHECK(parent->is_mutable()) << "Must pass mutable buffer";
+  parent_ = parent;
+}
+
 Status AllocateBuffer(
     MemoryPool* pool, int64_t size, std::shared_ptr<MutableBuffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 713d57a1f101d..3f14c964e83c1 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -46,7 +46,7 @@ class Status;
 class ARROW_EXPORT Buffer {
  public:
   Buffer(const uint8_t* data, int64_t size)
-      : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
+    : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
   virtual ~Buffer();
 
   /// An offset into data that is owned by another buffer, but we want to be
@@ -56,7 +56,10 @@ class ARROW_EXPORT Buffer {
   /// This method makes no assertions about alignment or padding of the buffer but
   /// in general we expected buffers to be aligned and padded to 64 bytes.  In the future
   /// we might add utility methods to help determine if a buffer satisfies this contract.
-  Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
+  Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size)
+    : Buffer(parent->data() + offset, size) {
+    parent_ = parent;
+  }
 
   bool is_mutable() const { return is_mutable_; }
 
@@ -74,6 +77,7 @@ class ARROW_EXPORT Buffer {
 
   int64_t capacity() const { return capacity_; }
   const uint8_t* data() const { return data_; }
+  uint8_t* mutable_data() { return mutable_data_; }
 
   int64_t size() const { return size_; }
 
@@ -82,6 +86,7 @@ class ARROW_EXPORT Buffer {
  protected:
   bool is_mutable_;
   const uint8_t* data_;
+  uint8_t* mutable_data_;
   int64_t size_;
   int64_t capacity_;
 
@@ -99,20 +104,24 @@ static inline std::shared_ptr<Buffer> SliceBuffer(
   return std::make_shared<Buffer>(buffer, offset, length);
 }
 
+/// Construct a mutable buffer slice. If the parent buffer is not mutable, this
+/// will abort in debug builds
+std::shared_ptr<Buffer> ARROW_EXPORT SliceMutableBuffer(
+    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
+
 /// A Buffer whose contents can be mutated. May or may not own its data.
 class ARROW_EXPORT MutableBuffer : public Buffer {
  public:
-  MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
-    is_mutable_ = true;
+  MutableBuffer(uint8_t* data, int64_t size)
+    : Buffer(data, size) {
     mutable_data_ = data;
+    is_mutable_ = true;
   }
 
-  uint8_t* mutable_data() { return mutable_data_; }
+  MutableBuffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
 
  protected:
-  MutableBuffer() : Buffer(nullptr, 0), mutable_data_(nullptr) {}
-
-  uint8_t* mutable_data_;
+  MutableBuffer() : Buffer(nullptr, 0) {}
 };
 
 class ARROW_EXPORT ResizableBuffer : public MutableBuffer {

From c4d535ca1ed98981c62ecceba81fa7c4efeaf66d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 2 Apr 2017 14:40:08 -0400
Subject: [PATCH 0455/1644] ARROW-753: [Python] Fix linker error for
 python-test on OS X

We don't link to libpython in the pyarrow C extensions, but we need to run the googletest unit tests.

I thought we were building and running this test in Travis CI, but we're not yet. It's not that easy to add that right now, so this triages the builds in the meantime

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #478 from wesm/ARROW-753 and squashes the following commits:

bcc2455 [Wes McKinney] Fix linker error for python-test on OS X
---
 cpp/cmake_modules/FindPythonLibsNew.cmake | 10 ++++------
 cpp/src/arrow/python/CMakeLists.txt       |  2 +-
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/cpp/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake
index 1000a957a6269..3e248a93342c5 100644
--- a/cpp/cmake_modules/FindPythonLibsNew.cmake
+++ b/cpp/cmake_modules/FindPythonLibsNew.cmake
@@ -148,12 +148,10 @@ if(CMAKE_HOST_WIN32)
         set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/libpython${PYTHON_LIBRARY_SUFFIX}.a")
     endif()
 elseif(APPLE)
-     # Seems to require "-undefined dynamic_lookup" instead of linking
-     # against the .dylib, otherwise it crashes. This flag is added
-     # below
-    set(PYTHON_LIBRARY "")
-    #set(PYTHON_LIBRARY
-    #    "${PYTHON_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
+  # In Python C extensions on OS X, the flag "-undefined dynamic_lookup" can
+  # avoid certain kinds of dynamic linking issues with portable binaries, so
+  # you should avoid targeting libpython at link time if at all possible
+  set(PYTHON_LIBRARY "${PYTHON_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
 else()
     if(${PYTHON_SIZEOF_VOID_P} MATCHES 8)
         set(_PYTHON_LIBS_SEARCH "${PYTHON_PREFIX}/lib64" "${PYTHON_PREFIX}/lib" "${PYTHON_LIBRARY_PATH}")
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index a8b4cc7ff1ded..c69d976737f91 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -39,7 +39,7 @@ set(ARROW_PYTHON_MIN_TEST_LIBS
   arrow_io_static
   arrow_static)
 
-if(NOT APPLE AND ARROW_BUILD_TESTS)
+if(ARROW_BUILD_TESTS)
   ADD_THIRDPARTY_LIB(python
     SHARED_LIB "${PYTHON_LIBRARIES}")
   list(APPEND ARROW_PYTHON_MIN_TEST_LIBS python)

From 9f720b117648f42356bbfd0a36f6275b878305d1 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 2 Apr 2017 14:49:20 -0400
Subject: [PATCH 0456/1644] ARROW-738: Fix manylinux1 build

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #471 from xhochy/fix-manylinux1-build and squashes the following commits:

5259118 [Uwe L. Korn] Instead of copying, symlink to correct ABI versions
cffcca3 [Uwe L. Korn] Don't hardcode ABI versions
98f79af [Uwe L. Korn] Set PKG_CONFIG_PATH
b97bcf4 [Uwe L. Korn] Read SOVERSION from pkg-config
a83dbc8 [Uwe L. Korn] Build arrow-cpp for each Python version
---
 python/CMakeLists.txt                | 39 ++++++++++++++--------------
 python/cmake_modules/FindArrow.cmake | 36 ++++++++++++++++---------
 python/manylinux1/Dockerfile-x86_64  |  4 ++-
 python/manylinux1/build_arrow.sh     | 10 +++++++
 python/setup.py                      | 18 ++++++++++---
 5 files changed, 71 insertions(+), 36 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index f315d019bb4de..463a29d87b711 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -226,24 +226,27 @@ endif()
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 
-if (PYARROW_BUNDLE_ARROW_CPP)
-  configure_file(${ARROW_SHARED_LIB}
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}
+function(bundle_arrow_lib library_path)
+  get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
+  configure_file(${${library_path}}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
       COPYONLY)
-  SET(ARROW_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
-  configure_file(${ARROW_IO_SHARED_LIB}
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX}
+  configure_file(${${library_path}}.${ARROW_ABI_VERSION}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
       COPYONLY)
-  SET(ARROW_IO_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX})
-  configure_file(${ARROW_IPC_SHARED_LIB}
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX}
+  configure_file(${${library_path}}.${ARROW_SO_VERSION}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}
       COPYONLY)
-  SET(ARROW_IPC_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX})
-  SET(ARROW_PYTHON_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+  SET(ARROW_SHARED_LIB
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+endfunction(bundle_arrow_lib)
+
+if (PYARROW_BUNDLE_ARROW_CPP)
+  # arrow
+  bundle_arrow_lib(ARROW_SHARED_LIB)
+  bundle_arrow_lib(ARROW_IO_SHARED_LIB)
+  bundle_arrow_lib(ARROW_IPC_SHARED_LIB)
+  bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
 endif()
 
 ADD_THIRDPARTY_LIB(arrow
@@ -309,11 +312,7 @@ endif()
 
 if (PYARROW_BUILD_JEMALLOC)
   if (PYARROW_BUNDLE_ARROW_CPP)
-    configure_file(${ARROW_JEMALLOC_SHARED_LIB}
-       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}
-       COPYONLY)
-    SET(ARROW_JEMALLOC_SHARED_LIB
-        ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
+    bundle_arrow_lib(ARROW_JEMALLOC_SHARED_LIB)
   endif()
   ADD_THIRDPARTY_LIB(arrow_jemalloc
     SHARED_LIB ${ARROW_JEMALLOC_SHARED_LIB})
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 5030c9c8ce900..c2ca0f4ad22c8 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -23,6 +23,8 @@
 #  ARROW_SHARED_LIB, path to libarrow's shared library
 #  ARROW_FOUND, whether arrow has been found
 
+include(FindPkgConfig)
+
 set(ARROW_SEARCH_HEADER_PATHS
   $ENV{ARROW_HOME}/include
 )
@@ -31,16 +33,27 @@ set(ARROW_SEARCH_LIB_PATH
   $ENV{ARROW_HOME}/lib
 )
 
-find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
-  ${ARROW_SEARCH_HEADER_PATHS}
-  # make sure we don't accidentally pick up a different version
-  NO_DEFAULT_PATH
-)
-
-find_library(ARROW_LIB_PATH NAMES arrow
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
+pkg_check_modules(ARROW arrow)
+if (ARROW_FOUND)
+  pkg_get_variable(ARROW_ABI_VERSION arrow abi_version)
+  message(STATUS "Arrow ABI version: ${ARROW_ABI_VERSION}")
+  pkg_get_variable(ARROW_SO_VERSION arrow so_version)
+  message(STATUS "Arrow SO version: ${ARROW_SO_VERSION}")
+  set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
+  set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
+else()
+  find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
+    ${ARROW_SEARCH_HEADER_PATHS}
+    # make sure we don't accidentally pick up a different version
+    NO_DEFAULT_PATH
+  )
+
+  find_library(ARROW_LIB_PATH NAMES arrow
+    PATHS
+    ${ARROW_SEARCH_LIB_PATH}
+    NO_DEFAULT_PATH)
+  get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
+endif()
 
 find_library(ARROW_IO_LIB_PATH NAMES arrow_io
   PATHS
@@ -62,7 +75,7 @@ find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
-if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
+if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
   set(ARROW_IO_LIB_NAME libarrow_io)
@@ -70,7 +83,6 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc)
   set(ARROW_PYTHON_LIB_NAME libarrow_python)
 
-  set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
   set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 820b94e306afe..56b27ad2ae808 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -52,7 +52,9 @@ RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
-RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF .
+RUN mkdir build-plain
+WORKDIR /arrow/cpp/build-plain
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF ..
 RUN make -j5 install
 
 WORKDIR /
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 576a983b11c37..8bc4e60235b49 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -39,6 +39,7 @@ export PYARROW_BUNDLE_ARROW_CPP=1
 export LDFLAGS="-Wl,--no-as-needed"
 export ARROW_HOME="/arrow-dist"
 export PARQUET_HOME="/usr"
+export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
 
 # Ensure the target directory exists
 mkdir -p /io/dist
@@ -52,6 +53,15 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     echo "=== (${PYTHON}) Installing build dependencies ==="
     $PIPI_IO "numpy==1.9.0"
     $PIPI_IO "cython==0.24"
+    $PIPI_IO "pandas==0.19.2"
+
+    echo "=== (${PYTHON}) Building Arrow C++ libraries ==="
+    ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
+    mkdir -p "${ARROW_BUILD_DIR}"
+    pushd "${ARROW_BUILD_DIR}"
+    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} ..
+    make -j5 install
+    popd
 
     # Clear output directory
     rm -rf dist/
diff --git a/python/setup.py b/python/setup.py
index 12b44e1bad520..ba77e688ae1f6 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -17,19 +17,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import glob
+import os
 import os.path as osp
 import re
 import shutil
+import sys
+
 from Cython.Distutils import build_ext as _build_ext
 import Cython
 
-import sys
 
 import pkg_resources
 from setuptools import setup, Extension
 
-import os
-
 from os.path import join as pjoin
 
 from distutils.command.clean import clean as _clean
@@ -207,8 +208,19 @@ def _run_cmake(self):
         def move_lib(lib_name):
             lib_filename = (shared_library_prefix + lib_name +
                             shared_library_suffix)
+            # Also copy libraries with ABI/SO version suffix
+            libs = glob.glob(pjoin(self.build_type, lib_filename) + '*')
+            # Longest suffix library should be copied, all others symlinked
+            libs.sort(key=lambda s: -len(s))
+            print(libs, libs[0])
+            lib_filename = os.path.basename(libs[0])
             shutil.move(pjoin(self.build_type, lib_filename),
                         pjoin(build_lib, 'pyarrow', lib_filename))
+            for lib in libs[1:]:
+                filename = os.path.basename(lib)
+                link_name = pjoin(build_lib, 'pyarrow', filename)
+                if not os.path.exists(link_name):
+                    os.symlink(lib_filename, link_name)
 
         if self.bundle_arrow_cpp:
             move_lib("arrow")

From 8f113b4d0fc344ab7d411af85fbf99154d5d1eaa Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 2 Apr 2017 23:31:19 -0400
Subject: [PATCH 0457/1644] ARROW-755: [GLib] Add garrow_array_get_value_type()

garrow_array_get_data_type() is renamed to
garrow_array_get_value_data_type() for consistency.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #481 from kou/glib-support-array-value-type and squashes the following commits:

bb4349c [Kouhei Sutou] [GLib] Add index for new symbols in 0.3.0 to API reference
7b07306 [Kouhei Sutou] [GLib] Add garrow_array_get_value_type()
---
 c_glib/arrow-glib/array.cpp               | 23 ++++++++++++++++++++---
 c_glib/arrow-glib/array.h                 |  3 ++-
 c_glib/doc/reference/arrow-glib-docs.sgml |  4 ++++
 c_glib/test/test-array.rb                 | 10 ++++++++--
 4 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index caf2eb55d6b2c..9d0e101e1b52f 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -35,6 +35,7 @@
 #include <arrow-glib/null-array.h>
 #include <arrow-glib/string-array.h>
 #include <arrow-glib/struct-array.h>
+#include <arrow-glib/type.hpp>
 #include <arrow-glib/uint8-array.h>
 #include <arrow-glib/uint16-array.h>
 #include <arrow-glib/uint32-array.h>
@@ -189,19 +190,35 @@ garrow_array_get_n_nulls(GArrowArray *array)
 }
 
 /**
- * garrow_array_get_data_type:
+ * garrow_array_get_value_data_type:
  * @array: A #GArrowArray.
  *
- * Returns: (transfer full): The #GArrowDataType for the array.
+ * Since: 0.3.0
+ * Returns: (transfer full): The #GArrowDataType for each value of the
+ *   array.
  */
 GArrowDataType *
-garrow_array_get_data_type(GArrowArray *array)
+garrow_array_get_value_data_type(GArrowArray *array)
 {
   auto arrow_array = garrow_array_get_raw(array);
   auto arrow_data_type = arrow_array->type();
   return garrow_data_type_new_raw(&arrow_data_type);
 }
 
+/**
+ * garrow_array_get_value_type:
+ * @array: A #GArrowArray.
+ *
+ * Since: 0.3.0
+ * Returns: The #GArrowType for each value of the array.
+ */
+GArrowType
+garrow_array_get_value_type(GArrowArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  return garrow_type_from_raw(arrow_array->type_enum());
+}
+
 /**
  * garrow_array_slice:
  * @array: A #GArrowArray.
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 957b4416fa581..06a37e9b43ad6 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -62,7 +62,8 @@ gboolean       garrow_array_is_null     (GArrowArray *array,
 gint64         garrow_array_get_length  (GArrowArray *array);
 gint64         garrow_array_get_offset  (GArrowArray *array);
 gint64         garrow_array_get_n_nulls (GArrowArray *array);
-GArrowDataType *garrow_array_get_data_type(GArrowArray *array);
+GArrowDataType *garrow_array_get_value_data_type(GArrowArray *array);
+GArrowType     garrow_array_get_value_type(GArrowArray *array);
 GArrowArray   *garrow_array_slice       (GArrowArray *array,
                                          gint64 offset,
                                          gint64 length);
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index a732e09df1269..06a19369640b5 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -167,5 +167,9 @@
     <title>Index of deprecated API</title>
     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
   </index>
+  <index id="api-index-0-3-0" role="0.3.0">
+    <title>Index of new symbols in 0.3.0</title>
+    <xi:include href="xml/api-index-0.3.0.xml"><xi:fallback /></xi:include>
+  </index>
   <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
 </book>
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
index 08908b08961a7..06102eb36575b 100644
--- a/c_glib/test/test-array.rb
+++ b/c_glib/test/test-array.rb
@@ -40,10 +40,16 @@ def test_n_nulls
     assert_equal(2, array.n_nulls)
   end
 
-  def test_data_type
+  def test_value_data_type
     builder = Arrow::BooleanArrayBuilder.new
     array = builder.finish
-    assert_equal(Arrow::BooleanDataType.new, array.data_type)
+    assert_equal(Arrow::BooleanDataType.new, array.value_data_type)
+  end
+
+  def test_value_type
+    builder = Arrow::BooleanArrayBuilder.new
+    array = builder.finish
+    assert_equal(Arrow::Type::BOOL, array.value_type)
   end
 
   def test_slice

From 96f3d6176d8c95717f4ff45e4226161de3168b05 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 3 Apr 2017 08:43:47 +0200
Subject: [PATCH 0458/1644] ARROW-749: [Python] Delete partially-written
 Feather file when column write fails

This is currently the only place where we are doing an atomic create-file/write-file. We should be mindful of other serialization functions which may yield unreadable files in the future.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #484 from wesm/ARROW-749 and squashes the following commits:

137e235 [Wes McKinney] Delete partially-written Feather file when column write fails
---
 python/pyarrow/feather.py            | 79 ++++++++++++++++++----------
 python/pyarrow/tests/test_feather.py | 16 ++++++
 2 files changed, 67 insertions(+), 28 deletions(-)

diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index f87c7f3a95ee4..3b5716e36be0a 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -15,8 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import six
 from distutils.version import LooseVersion
+import os
+
+import six
 import pandas as pd
 
 from pyarrow.compat import pdapi
@@ -54,45 +56,66 @@ def read(self, columns=None):
         return table.to_pandas()
 
 
-def write_feather(df, dest):
-    '''
-    Write a pandas.DataFrame to Feather format
-    '''
-    writer = ext.FeatherWriter()
-    writer.open(dest)
+class FeatherWriter(object):
 
-    if isinstance(df, pd.SparseDataFrame):
-        df = df.to_dense()
+    def __init__(self, dest):
+        self.dest = dest
+        self.writer = ext.FeatherWriter()
+        self.writer.open(dest)
 
-    if not df.columns.is_unique:
-        raise ValueError("cannot serialize duplicate column names")
+    def write(self, df):
+        if isinstance(df, pd.SparseDataFrame):
+            df = df.to_dense()
 
-    # TODO(wesm): pipeline conversion to Arrow memory layout
-    for i, name in enumerate(df.columns):
-        col = df.iloc[:, i]
+        if not df.columns.is_unique:
+            raise ValueError("cannot serialize duplicate column names")
 
-        if pdapi.is_object_dtype(col):
-            inferred_type = pd.lib.infer_dtype(col)
-            msg = ("cannot serialize column {n} "
-                   "named {name} with dtype {dtype}".format(
-                       n=i, name=name, dtype=inferred_type))
+        # TODO(wesm): pipeline conversion to Arrow memory layout
+        for i, name in enumerate(df.columns):
+            col = df.iloc[:, i]
 
-            if inferred_type in ['mixed']:
+            if pdapi.is_object_dtype(col):
+                inferred_type = pd.lib.infer_dtype(col)
+                msg = ("cannot serialize column {n} "
+                       "named {name} with dtype {dtype}".format(
+                           n=i, name=name, dtype=inferred_type))
 
-                # allow columns with nulls + an inferable type
-                inferred_type = pd.lib.infer_dtype(col[col.notnull()])
                 if inferred_type in ['mixed']:
+
+                    # allow columns with nulls + an inferable type
+                    inferred_type = pd.lib.infer_dtype(col[col.notnull()])
+                    if inferred_type in ['mixed']:
+                        raise ValueError(msg)
+
+                elif inferred_type not in ['unicode', 'string']:
                     raise ValueError(msg)
 
-            elif inferred_type not in ['unicode', 'string']:
-                raise ValueError(msg)
+            if not isinstance(name, six.string_types):
+                name = str(name)
 
-        if not isinstance(name, six.string_types):
-            name = str(name)
+            self.writer.write_array(name, col)
 
-        writer.write_array(name, col)
+        self.writer.close()
 
-    writer.close()
+
+def write_feather(df, dest):
+    '''
+    Write a pandas.DataFrame to Feather format
+    '''
+    writer = FeatherWriter(dest)
+    try:
+        writer.write(df)
+    except:
+        # Try to make sure the resource is closed
+        import gc
+        writer = None
+        gc.collect()
+        if isinstance(dest, six.string_types):
+            try:
+                os.remove(dest)
+            except os.error:
+                pass
+        raise
 
 
 def read_feather(source, columns=None):
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 525da344c9951..c7b4f1e997327 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -249,6 +249,22 @@ def test_boolean_object_nulls(self):
         df = pd.DataFrame({'bools': arr})
         self._check_pandas_roundtrip(df, null_counts=[1 * repeats])
 
+    def test_delete_partial_file_on_error(self):
+        # strings will fail
+        df = pd.DataFrame(
+            {
+                'numbers': range(5),
+                'strings': [b'foo', None, u'bar', 'qux', np.nan]},
+            columns=['numbers', 'strings'])
+
+        path = random_path()
+        try:
+            write_feather(df, path)
+        except:
+            pass
+
+        assert not os.path.exists(path)
+
     def test_strings(self):
         repeats = 1000
 

From 7232e5b5df64f9dbdc9405798644cd08a6d9db6b Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Mon, 3 Apr 2017 18:01:24 -0400
Subject: [PATCH 0459/1644] ARROW-676: move from MinorType to FieldType in
 ValueVectors to carry all the relevant type bits

I'm adding all the Type information to the vector with FieldType.
This avoids losing information and carrying around extra type metadata that can not go in the MinorType enum (for example: decimals' precision and dictionary encoding)

Author: Julien Le Dem <julien@dremio.com>

Closes #409 from julienledem/minor_field and squashes the following commits:

0407e63 [Julien Le Dem] ARROW-676: move from MinorType to FieldType in ValueVectors to carry all the relevant type bits
---
 .../apache/arrow/tools/EchoServerTest.java    |  55 ++--
 .../src/main/codegen/data/ArrowTypes.tdd      |  38 ++-
 .../src/main/codegen/templates/ArrowType.java |  81 +++++-
 .../main/codegen/templates/MapWriters.java    |  10 +-
 .../templates/NullableValueVectors.java       |  42 +--
 .../main/codegen/templates/UnionVector.java   |  20 +-
 .../org/apache/arrow/vector/FieldVector.java  |   3 +-
 .../apache/arrow/vector/VectorSchemaRoot.java |   6 +-
 .../complex/AbstractContainerVector.java      |  20 +-
 .../vector/complex/AbstractMapVector.java     |  29 +-
 .../complex/BaseRepeatedValueVector.java      |  11 +-
 .../arrow/vector/complex/ListVector.java      |  13 +-
 .../arrow/vector/complex/MapVector.java       |   6 +-
 .../vector/complex/NullableMapVector.java     |   1 +
 .../complex/impl/ComplexWriterImpl.java       |   5 +-
 .../vector/complex/impl/PromotableWriter.java |   5 +-
 .../org/apache/arrow/vector/types/Types.java  | 264 ++++--------------
 .../apache/arrow/vector/types/pojo/Field.java |  82 +++---
 .../arrow/vector/types/pojo/FieldType.java    |  60 ++++
 .../arrow/vector/TestDecimalVector.java       |  14 +-
 .../arrow/vector/TestDictionaryVector.java    |  16 +-
 .../org/apache/arrow/vector/TestUtils.java    |  39 +++
 .../apache/arrow/vector/TestValueVector.java  |  15 +-
 .../complex/impl/TestPromotableWriter.java    |   3 +-
 .../arrow/vector/file/TestArrowFile.java      |  20 +-
 .../vector/file/TestArrowReaderWriter.java    |  12 +-
 26 files changed, 461 insertions(+), 409 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java

diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 5970c57f46583..7d07588892cf9 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -18,7 +18,19 @@
 
 package org.apache.arrow.tools;
 
-import com.google.common.collect.ImmutableList;
+import static java.util.Arrays.asList;
+import static org.apache.arrow.vector.types.Types.MinorType.TINYINT;
+import static org.apache.arrow.vector.types.Types.MinorType.VARCHAR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -39,6 +51,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.Text;
 import org.junit.AfterClass;
@@ -46,17 +59,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.net.Socket;
-import java.net.UnknownHostException;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import static java.util.Arrays.asList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import com.google.common.collect.ImmutableList;
 
 public class EchoServerTest {
 
@@ -133,9 +136,12 @@ private void testEchoServer(int serverPort,
   public void basicTest() throws InterruptedException, IOException {
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
 
-    Field field = new Field("testField", true, new ArrowType.Int(8, true), Collections
-        .<Field>emptyList());
-    NullableTinyIntVector vector = new NullableTinyIntVector("testField", alloc, null);
+    Field field = new Field(
+        "testField", true,
+        new ArrowType.Int(8, true),
+        Collections.<Field>emptyList());
+    NullableTinyIntVector vector =
+        new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
     Schema schema = new Schema(asList(field));
 
     // Try an empty stream, just the header.
@@ -152,9 +158,16 @@ public void basicTest() throws InterruptedException, IOException {
   public void testFlatDictionary() throws IOException {
     DictionaryEncoding writeEncoding = new DictionaryEncoding(1L, false, null);
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
-         NullableIntVector writeVector = new NullableIntVector("varchar", allocator, writeEncoding);
-         NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dict",
-             allocator, null)) {
+        NullableIntVector writeVector =
+            new NullableIntVector(
+                "varchar",
+                new FieldType(true, MinorType.INT.getType(), writeEncoding),
+                allocator);
+        NullableVarCharVector writeDictionaryVector =
+            new NullableVarCharVector(
+                "dict",
+                FieldType.nullable(VARCHAR.getType()),
+                allocator)) {
       writeVector.allocateNewSafe();
       NullableIntVector.Mutator mutator = writeVector.getMutator();
       mutator.set(0, 0);
@@ -222,8 +235,8 @@ public void testFlatDictionary() throws IOException {
   public void testNestedDictionary() throws IOException {
     DictionaryEncoding writeEncoding = new DictionaryEncoding(2L, false, null);
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
-         NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dictionary",
-             allocator, null);
+         NullableVarCharVector writeDictionaryVector =
+             new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
          ListVector writeVector = new ListVector("list", allocator, null, null)) {
 
       // data being written:
@@ -234,7 +247,7 @@ public void testNestedDictionary() throws IOException {
       writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
       writeDictionaryVector.getMutator().setValueCount(2);
 
-      writeVector.addOrGetVector(MinorType.INT, writeEncoding);
+      writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
       writeVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(writeVector);
       listWriter.startList();
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 67785ad6b4d19..e1fb5e0619a9b 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -14,59 +14,73 @@
   types: [
     {
       name: "Null",
-      fields: []
+      fields: [],
+      complex: false
     },
     {
       name: "Struct_",
-      fields: []
+      fields: [],
+      complex: true
     },
     {
       name: "List",
-      fields: []
+      fields: [],
+      complex: true
     },
     {
       name: "Union",
-      fields: [{name: "mode", type: short, valueType: UnionMode}, {name: "typeIds", type: "int[]"}]
+      fields: [{name: "mode", type: short, valueType: UnionMode}, {name: "typeIds", type: "int[]"}],
+      complex: true
     },
     {
       name: "Int",
-      fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
+      fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}],
+      complex: false
     },
     {
       name: "FloatingPoint",
-      fields: [{name: precision, type: short, valueType: FloatingPointPrecision}]
+      fields: [{name: precision, type: short, valueType: FloatingPointPrecision}],
+      complex: false
     },
     {
       name: "Utf8",
-      fields: []
+      fields: [],
+      complex: false
     },
     {
       name: "Binary",
-      fields: []
+      fields: [],
+      complex: false
     },
     {
       name: "Bool",
-      fields: []
+      fields: [],
+      complex: false
     },
     {
       name: "Decimal",
-      fields: [{name: "precision", type: int}, {name: "scale", type: int}]
+      fields: [{name: "precision", type: int}, {name: "scale", type: int}],
+      complex: false
     },
     {
       name: "Date",
       fields: [{name: "unit", type: short, valueType: DateUnit}]
+      complex: false
     },
     {
       name: "Time",
-      fields: [{name: "unit", type: short, valueType: TimeUnit}, {name: "bitWidth", type: int}]
+      fields: [{name: "unit", type: short, valueType: TimeUnit}, {name: "bitWidth", type: int}],
+      complex: false
     },
     {
       name: "Timestamp",
       fields: [{name: "unit", type: short, valueType: TimeUnit}, {name: "timezone", type: String}]
+      complex: false
     },
     {
       name: "Interval",
-      fields: [{name: "unit", type: short, valueType: IntervalUnit}]
+      fields: [{name: "unit", type: short, valueType: IntervalUnit}],
+      complex: false
     }
   ]
 }
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index 91cbe98196b81..a9e875a2095f7 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -50,13 +50,35 @@
 })
 public abstract class ArrowType {
 
+  public static abstract class PrimitiveType extends ArrowType {
+
+    private PrimitiveType() {
+    }
+
+    @Override
+    public boolean isComplex() {
+      return false;
+    }
+  }
+
+  public static abstract class ComplexType extends ArrowType {
+
+    private ComplexType() {
+    }
+
+    @Override
+    public boolean isComplex() {
+      return true;
+    }
+  }
+
   public static enum ArrowTypeID {
     <#list arrowTypes.types as type>
     <#assign name = type.name>
     ${name?remove_ending("_")}(Type.${name}),
     </#list>
     NONE(Type.NONE);
-  
+
     private final byte flatbufType;
 
     public byte getFlatbufID() {
@@ -70,6 +92,8 @@ private ArrowTypeID(byte flatbufType) {
 
   @JsonIgnore
   public abstract ArrowTypeID getTypeID();
+  @JsonIgnore
+  public abstract boolean isComplex();
   public abstract int getType(FlatBufferBuilder builder);
   public abstract <T> T accept(ArrowTypeVisitor<T> visitor);
 
@@ -87,21 +111,56 @@ public static interface ArrowTypeVisitor<T> {
   </#list>
   }
 
+  /**
+   * to visit the Complex ArrowTypes and bundle Primitive ones in one case
+   */
+  public static abstract class ComplexTypeVisitor<T> implements ArrowTypeVisitor<T> {
+
+    public T visit(PrimitiveType type) {
+      throw new UnsupportedOperationException("Unexpected Primitive type: " + type);
+    }
+
+  <#list arrowTypes.types as type>
+    <#if !type.complex>
+    public final T visit(${type.name?remove_ending("_")} type) {
+      return visit((PrimitiveType) type);
+    }
+    </#if>
+  </#list>
+  }
+
+  /**
+   * to visit the Primitive ArrowTypes and bundle Complex ones under one case
+   */
+  public static abstract class PrimitiveTypeVisitor<T> implements ArrowTypeVisitor<T> {
+
+    public T visit(ComplexType type) {
+      throw new UnsupportedOperationException("Unexpected Complex type: " + type);
+    }
+
+  <#list arrowTypes.types as type>
+    <#if type.complex>
+    public final T visit(${type.name?remove_ending("_")} type) {
+      return visit((ComplexType) type);
+    }
+    </#if>
+  </#list>
+  }
+
   <#list arrowTypes.types as type>
   <#assign name = type.name?remove_ending("_")>
   <#assign fields = type.fields>
-  public static class ${name} extends ArrowType {
+  public static class ${name} extends <#if type.complex>ComplexType<#else>PrimitiveType</#if> {
     public static final ArrowTypeID TYPE_TYPE = ArrowTypeID.${name};
     <#if type.fields?size == 0>
     public static final ${name} INSTANCE = new ${name}();
-    </#if>
+    <#else>
 
     <#list fields as field>
     <#assign fieldType = field.valueType!field.type>
     ${fieldType} ${field.name};
     </#list>
 
-    <#if type.fields?size != 0>
     @JsonCreator
     public ${type.name}(
     <#list type.fields as field>
@@ -113,6 +172,13 @@ public static class ${name} extends ArrowType {
       this.${field.name} = ${field.name};
       </#list>
     }
+
+    <#list fields as field>
+    <#assign fieldType = field.valueType!field.type>
+    public ${fieldType} get${field.name?cap_first}() {
+      return ${field.name};
+    }
+    </#list>
     </#if>
 
     @Override
@@ -143,13 +209,6 @@ public int getType(FlatBufferBuilder builder) {
       return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
     }
 
-    <#list fields as field>
-    <#assign fieldType = field.valueType!field.type>
-    public ${fieldType} get${field.name?cap_first}() {
-      return ${field.name};
-    }
-    </#list>
-
     public String toString() {
       return "${name}"
       <#if fields?size != 0>
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 428ce0427d4b8..d3e6de9527123 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -64,7 +64,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
         list(child.getName());
         break;
       case UNION:
-        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), MinorType.UNION, UnionVector.class, null), getNullableMapWriterFactory());
+        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), FieldType.nullable(MinorType.UNION.getType()), UnionVector.class), getNullableMapWriterFactory());
         fields.put(handleCase(child.getName()), writer);
         break;
 <#list vv.types as type><#list type.minor as minor>
@@ -113,7 +113,7 @@ public MapWriter map(String name) {
     FieldWriter writer = fields.get(finalName);
     if(writer == null){
       int vectorCount=container.size();
-      NullableMapVector vector = container.addOrGet(name, MinorType.MAP, NullableMapVector.class, null);
+      NullableMapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), NullableMapVector.class);
       writer = new PromotableWriter(vector, container, getNullableMapWriterFactory());
       if(vectorCount != container.size()) {
         writer.allocate();
@@ -157,7 +157,7 @@ public ListWriter list(String name) {
     FieldWriter writer = fields.get(finalName);
     int vectorCount = container.size();
     if(writer == null) {
-      writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class, null), container, getNullableMapWriterFactory());
+      writer = new PromotableWriter(container.addOrGet(name, FieldType.nullable(MinorType.LIST.getType()), ListVector.class), container, getNullableMapWriterFactory());
       if (container.size() > vectorCount) {
         writer.allocate();
       }
@@ -222,7 +222,9 @@ public void end() {
     if(writer == null) {
       ValueVector vector;
       ValueVector currentVector = container.getChild(name);
-      ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class, null<#if minor.class == "Decimal"> , new int[] {precision, scale}</#if>);
+      ${vectName}Vector v = container.addOrGet(name, 
+          FieldType.nullable(<#if minor.class == "Decimal">new Decimal(precision, scale)<#else>MinorType.${upperName}.getType()</#if>),
+          ${vectName}Vector.class);
       writer = new PromotableWriter(v, container, getNullableMapWriterFactory());
       vector = v;
       if (currentVector == null || currentVector != vector) {
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index ec2ce7930cf5d..8e1727ca6c820 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -64,28 +64,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#if minor.class == "Decimal">
   private final int precision;
   private final int scale;
+  </#if>
 
-  public ${className}(String name, BufferAllocator allocator, DictionaryEncoding dictionary, int precision, int scale) {
+  public ${className}(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator);
-    values = new ${valuesName}(valuesField, allocator, precision, scale);
-    this.precision = precision;
-    this.scale = scale;
-    mutator = new Mutator();
-    accessor = new Accessor();
-    field = new Field(name, true, new Decimal(precision, scale), dictionary, null);
-    innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
-        bits,
-        values
-    ));
-  }
-  <#else>
-  public ${className}(String name, BufferAllocator allocator, DictionaryEncoding dictionary) {
-    super(name, allocator);
-    values = new ${valuesName}(valuesField, allocator);
-    mutator = new Mutator();
-    accessor = new Accessor();
-    ArrowType type = Types.MinorType.${minor.class?upper_case}.getType();
-    field = new Field(name, true, type, dictionary, null);
+    <#if minor.class == "Decimal">
+    Decimal decimal = (Decimal)fieldType.getType();
+    this.precision = decimal.getPrecision();
+    this.scale = decimal.getScale();
+    this.values = new ${valuesName}(valuesField, allocator, precision, scale);
+    <#else>
+    this.values = new ${valuesName}(valuesField, allocator);
+    </#if>
+    this.mutator = new Mutator();
+    this.accessor = new Accessor();
+    this.field = new Field(name, fieldType, null);
     innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
         bits,
         <#if type.major = "VarLen">
@@ -94,7 +87,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
         values
     ));
   }
-  </#if>
 
   @Override
   public BitVector getValidityVector() {
@@ -341,12 +333,8 @@ public void splitAndTransferTo(int startIndex, int length, ${className} target)
   private class TransferImpl implements TransferPair {
     ${className} to;
 
-    public TransferImpl(String name, BufferAllocator allocator){
-      <#if minor.class == "Decimal">
-      to = new ${className}(name, allocator, field.getDictionary(), precision, scale);
-      <#else>
-      to = new ${className}(name, allocator, field.getDictionary());
-      </#if>
+    public TransferImpl(String ref, BufferAllocator allocator){
+      to = new ${className}(ref, field.getFieldType(), allocator);
     }
 
     public TransferImpl(${className} to){
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index d17935b08eefc..797b29342e4c1 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -119,10 +119,22 @@ public List<BufferBacked> getFieldInnerVectors() {
      return this.innerVectors;
   }
 
+  private String fieldName(MinorType type) {
+    return type.name().toLowerCase();
+  }
+
+  private FieldType fieldType(MinorType type) {
+    return new FieldType(true, type.getType(), null);
+  }
+
+  private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
+    return internalMap.addOrGet(fieldName(minorType), fieldType(minorType), c);
+  }
+
   public NullableMapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
-      mapVector = internalMap.addOrGet("map", MinorType.MAP, NullableMapVector.class, null);
+      mapVector = addOrGet(MinorType.MAP, NullableMapVector.class);
       if (internalMap.size() > vectorCount) {
         mapVector.allocateNew();
         if (callBack != null) {
@@ -144,7 +156,7 @@ public NullableMapVector getMap() {
   public Nullable${name}Vector get${name}Vector() {
     if (${uncappedName}Vector == null) {
       int vectorCount = internalMap.size();
-      ${uncappedName}Vector = internalMap.addOrGet("${lowerCaseName}", MinorType.${name?upper_case}, Nullable${name}Vector.class, null);
+      ${uncappedName}Vector = addOrGet(MinorType.${name?upper_case}, Nullable${name}Vector.class);
       if (internalMap.size() > vectorCount) {
         ${uncappedName}Vector.allocateNew();
         if (callBack != null) {
@@ -162,7 +174,7 @@ public NullableMapVector getMap() {
   public ListVector getList() {
     if (listVector == null) {
       int vectorCount = internalMap.size();
-      listVector = internalMap.addOrGet("list", MinorType.LIST, ListVector.class, null);
+      listVector = addOrGet(MinorType.LIST, ListVector.class);
       if (internalMap.size() > vectorCount) {
         listVector.allocateNew();
         if (callBack != null) {
@@ -267,7 +279,7 @@ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
   public FieldVector addVector(FieldVector v) {
     String name = v.getMinorType().name().toLowerCase();
     Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
-    final FieldVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass(), v.getField().getDictionary());
+    final FieldVector newVector = internalMap.addOrGet(name, v.getField().getFieldType(), v.getClass());
     v.makeTransferPair(newVector).transfer();
     internalMap.putChild(name, newVector);
     if (callBack != null) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
index 0fdbc48552aaa..6c2c8302a7b8b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
@@ -19,10 +19,11 @@
 
 import java.util.List;
 
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 
+import io.netty.buffer.ArrowBuf;
+
 /**
  * A vector corresponding to a Field in the schema
  * It has inner vectors backed by buffers (validity, offsets, data, ...)
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
index 7e626fb14305e..29b96736001ce 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -23,8 +23,6 @@
 import java.util.Map;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 
@@ -60,9 +58,7 @@ public VectorSchemaRoot(List<Field> fields, List<FieldVector> fieldVectors, int
   public static VectorSchemaRoot create(Schema schema, BufferAllocator allocator) {
     List<FieldVector> fieldVectors = new ArrayList<>();
     for (Field field : schema.getFields()) {
-      MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
-      FieldVector vector = minorType.getNewVector(field.getName(), allocator, field.getDictionary(), null);
-      vector.initializeChildrenFromFields(field.getChildren());
+      FieldVector vector = field.createVector(allocator);
       fieldVectors.add(vector);
     }
     if (fieldVectors.size() != schema.getFields().size()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 86a5e82119831..71f2bea5b8fe1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -22,7 +22,9 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.ArrowType.List;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 
 /**
@@ -85,12 +87,24 @@ protected boolean supportsDirectRead() {
   // return the number of child vectors
   public abstract int size();
 
-  // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
-  public abstract <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, DictionaryEncoding dictionary, int... precisionScale);
+  // add a new vector with the input FieldType or return the existing vector if we already added one with the same name
+  public abstract <T extends FieldVector> T addOrGet(String name, FieldType fieldType, Class<T> clazz);
 
   // return the child vector with the input name
   public abstract <T extends FieldVector> T getChild(String name, Class<T> clazz);
 
   // return the child vector's ordinal in the composite container
   public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
+
+  public NullableMapVector addOrGetMap(String name) {
+    return addOrGet(name, FieldType.nullable(new Struct()), NullableMapVector.class);
+  }
+
+  public ListVector addOrGetList(String name) {
+    return addOrGet(name, FieldType.nullable(new List()), ListVector.class);
+  }
+
+  public UnionVector addOrGetUnion(String name) {
+    return addOrGet(name, FieldType.nullable(MinorType.UNION.getType()), UnionVector.class);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index baeeb07873714..dc833edbed8d0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -25,8 +25,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.MapWithOrdinal;
 
@@ -102,8 +101,8 @@ public boolean allocateNewSafe() {
    *   </li>
    * </ul>
    *
-   * @param name the name of the field
-   * @param minorType the minorType for the vector
+   * @param childName the name of the field
+   * @param fieldType the type for the vector
    * @param clazz class of expected vector type
    * @param <T> class type of expected vector type
    * @throws java.lang.IllegalStateException raised if there is a hard schema change
@@ -111,8 +110,8 @@ public boolean allocateNewSafe() {
    * @return resultant {@link org.apache.arrow.vector.ValueVector}
    */
   @Override
-  public <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, DictionaryEncoding dictionary, int... precisionScale) {
-    final ValueVector existing = getChild(name);
+  public <T extends FieldVector> T addOrGet(String childName, FieldType fieldType, Class<T> clazz) {
+    final ValueVector existing = getChild(childName);
     boolean create = false;
     if (existing == null) {
       create = true;
@@ -123,9 +122,9 @@ public <T extends FieldVector> T addOrGet(String name, MinorType minorType, Clas
       create = true;
     }
     if (create) {
-      final T vector = clazz.cast(minorType.getNewVector(name, allocator, dictionary, callBack, precisionScale));
-      putChild(name, vector);
-      if (callBack!=null) {
+      final T vector = clazz.cast(fieldType.createNewSingleVector(childName, allocator, callBack));
+      putChild(childName, vector);
+      if (callBack != null) {
         callBack.doWork();
       }
       return vector;
@@ -163,14 +162,14 @@ public <T extends FieldVector> T getChild(String name, Class<T> clazz) {
     return typeify(v, clazz);
   }
 
-  protected ValueVector add(String name, MinorType minorType, DictionaryEncoding dictionary, int... precisionScale) {
-    final ValueVector existing = getChild(name);
+  protected ValueVector add(String childName, FieldType fieldType) {
+    final ValueVector existing = getChild(childName);
     if (existing != null) {
-      throw new IllegalStateException(String.format("Vector already exists: Existing[%s], Requested[%s] ", existing.getClass().getSimpleName(), minorType));
+      throw new IllegalStateException(String.format("Vector already exists: Existing[%s], Requested[%s] ", existing.getClass().getSimpleName(), fieldType));
     }
-    FieldVector vector = minorType.getNewVector(name, allocator, dictionary, callBack, precisionScale);
-    putChild(name, vector);
-    if (callBack!=null) {
+    FieldVector vector = fieldType.createNewSingleVector(childName, allocator, callBack);
+    putChild(childName, vector);
+    if (callBack != null) {
       callBack.doWork();
     }
     return vector;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index eda1f3bc80a96..6b240c04f7124 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -27,8 +27,7 @@
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
@@ -154,10 +153,10 @@ public int size() {
     return vector == DEFAULT_DATA_VECTOR ? 0:1;
   }
 
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType, DictionaryEncoding dictionary) {
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) {
     boolean created = false;
     if (vector instanceof ZeroVector) {
-      vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, dictionary, callBack);
+      vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack);
       // returned vector must have the same field
       created = true;
       if (callBack != null) {
@@ -165,9 +164,9 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorT
       }
     }
 
-    if (vector.getField().getType().getTypeID() != minorType.getType().getTypeID()) {
+    if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) {
       final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
-          minorType.getType().getTypeID(), vector.getField().getType().getTypeID());
+          fieldType.getType().getTypeID(), vector.getField().getType().getTypeID());
       throw new SchemaChangeRuntimeException(msg);
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 54b051b9781e5..d138ca339e3cf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -40,10 +40,10 @@
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.TransferPair;
@@ -80,8 +80,7 @@ public void initializeChildrenFromFields(List<Field> children) {
       throw new IllegalArgumentException("Lists have only one child. Found: " + children);
     }
     Field field = children.get(0);
-    MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
-    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(minorType, field.getDictionary());
+    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
     if (!addOrGetVector.isCreated()) {
       throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
     }
@@ -164,11 +163,11 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 
     public TransferImpl(ListVector to) {
       this.to = to;
-      to.addOrGetVector(vector.getMinorType(), vector.getField().getDictionary());
+      to.addOrGetVector(vector.getField().getFieldType());
       pairs[0] = offsets.makeTransferPair(to.offsets);
       pairs[1] = bits.makeTransferPair(to.bits);
       if (to.getDataVector() instanceof ZeroVector) {
-        to.addOrGetVector(vector.getMinorType(), vector.getField().getDictionary());
+        to.addOrGetVector(vector.getField().getFieldType());
       }
       pairs[2] = getDataVector().makeTransferPair(to.getDataVector());
     }
@@ -241,8 +240,8 @@ public boolean allocateNewSafe() {
     return success;
   }
 
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType, DictionaryEncoding dictionary) {
-    AddOrGetResult<T> result = super.addOrGetVector(minorType, dictionary);
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) {
+    AddOrGetResult<T> result = super.addOrGetVector(fieldType);
     reader = new UnionListReader(this);
     return result;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index cb67537c446c6..997a6a38a080a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -32,7 +32,6 @@
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -165,7 +164,7 @@ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
         // (This is similar to what happens in ScanBatch where the children cannot be added till they are
         // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
         // include the hashCode of the children but is based only on MaterializedField$key.
-        final FieldVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass(), vector.getField().getDictionary());
+        final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass());
         if (allocate && to.size() != preSize) {
           newVector.allocateNew();
         }
@@ -318,8 +317,7 @@ public void close() {
 
   public void initializeChildrenFromFields(List<Field> children) {
     for (Field field : children) {
-      MinorType minorType = Types.getMinorTypeForArrowType(field.getType());
-      FieldVector vector = (FieldVector)this.add(field.getName(), minorType, field.getDictionary());
+      FieldVector vector = (FieldVector)this.add(field.getName(), field.getFieldType());
       vector.initializeChildrenFromFields(field.getChildren());
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index de1d1857370b0..7fe35e8253afb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -270,4 +270,5 @@ public Accessor getAccessor() {
   public Mutator getMutator() {
     return mutator;
   }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 6d0531678488a..6851d6d45d562 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -22,7 +22,6 @@
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.StateTool;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
 
 import com.google.common.base.Preconditions;
@@ -150,7 +149,7 @@ public MapWriter rootAsMap() {
 
     case INIT:
       // TODO allow dictionaries in complex types
-      NullableMapVector map = container.addOrGet(name, MinorType.MAP, NullableMapVector.class, null);
+      NullableMapVector map = container.addOrGetMap(name);
       mapRoot = nullableMapWriterFactory.build(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
@@ -182,7 +181,7 @@ public ListWriter rootAsList() {
     case INIT:
       int vectorCount = container.size();
       // TODO allow dictionaries in complex types
-      ListVector listVector = container.addOrGet(name, MinorType.LIST, ListVector.class, null);
+      ListVector listVector = container.addOrGetList(name);
       if (container.size() > vectorCount) {
         listVector.allocateNew();
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 1880c9b490c27..d16718e75a701 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -27,6 +27,7 @@
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
@@ -125,7 +126,7 @@ protected FieldWriter getWriter(MinorType type) {
         // ???
         return null;
       }
-      ValueVector v = listVector.addOrGetVector(type, null).getVector();
+      ValueVector v = listVector.addOrGetVector(FieldType.nullable(type.getType())).getVector();
       v.allocateNew();
       setWriter(v);
       writer.setPosition(position);
@@ -151,7 +152,7 @@ private FieldWriter promoteToUnion() {
     tp.transfer();
     if (parentContainer != null) {
       // TODO allow dictionaries in complex types
-      unionVector = parentContainer.addOrGet(name, MinorType.UNION, UnionVector.class, null);
+      unionVector = parentContainer.addOrGetUnion(name);
       unionVector.allocateNew();
     } else if (listVector != null) {
       unionVector = listVector.promoteToUnion();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 2f070237101d8..f07bb585f810c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -92,45 +92,15 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
-import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 
 public class Types {
 
-  private static final Field NULL_FIELD = new Field("", true, Null.INSTANCE, null);
-  private static final Field TINYINT_FIELD = new Field("", true, new Int(8, true), null);
-  private static final Field SMALLINT_FIELD = new Field("", true, new Int(16, true), null);
-  private static final Field INT_FIELD = new Field("", true, new Int(32, true), null);
-  private static final Field BIGINT_FIELD = new Field("", true, new Int(64, true), null);
-  private static final Field UINT1_FIELD = new Field("", true, new Int(8, false), null);
-  private static final Field UINT2_FIELD = new Field("", true, new Int(16, false), null);
-  private static final Field UINT4_FIELD = new Field("", true, new Int(32, false), null);
-  private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
-  private static final Field DATE_FIELD = new Field("", true, new Date(DateUnit.MILLISECOND), null);
-  private static final Field TIME_FIELD = new Field("", true, new Time(TimeUnit.MILLISECOND, 32), null);
-  private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND, "UTC"), null);
-  private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null);
-  private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND, "UTC"), null);
-  private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND, "UTC"), null);
-  private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
-  private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
-  private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);
-  private static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.DOUBLE), null);
-  private static final Field VARCHAR_FIELD = new Field("", true, Utf8.INSTANCE, null);
-  private static final Field VARBINARY_FIELD = new Field("", true, Binary.INSTANCE, null);
-  private static final Field BIT_FIELD = new Field("", true, Bool.INSTANCE, null);
-
-
   public enum MinorType {
     NULL(Null.INSTANCE) {
       @Override
-      public Field getField() {
-        return NULL_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
         return ZeroVector.INSTANCE;
       }
 
@@ -141,13 +111,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     MAP(Struct.INSTANCE) {
       @Override
-      public Field getField() {
-        throw new UnsupportedOperationException("Cannot get simple field for Map type");
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-         return new NullableMapVector(name, allocator, dictionary, callBack);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableMapVector(name, allocator, fieldType.getDictionary(), schemaChangeCallback);
       }
 
       @Override
@@ -157,13 +122,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     TINYINT(new Int(8, true)) {
       @Override
-      public Field getField() {
-        return TINYINT_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableTinyIntVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTinyIntVector(name, fieldType, allocator);
       }
 
       @Override
@@ -173,13 +133,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     SMALLINT(new Int(16, true)) {
       @Override
-      public Field getField() {
-        return SMALLINT_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableSmallIntVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableSmallIntVector(name, fieldType, allocator);
       }
 
       @Override
@@ -189,13 +144,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     INT(new Int(32, true)) {
       @Override
-      public Field getField() {
-        return INT_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableIntVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableIntVector(name, fieldType, allocator);
       }
 
       @Override
@@ -205,13 +155,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     BIGINT(new Int(64, true)) {
       @Override
-      public Field getField() {
-        return BIGINT_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableBigIntVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableBigIntVector(name, fieldType, allocator);
       }
 
       @Override
@@ -221,13 +166,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     DATE(new Date(DateUnit.MILLISECOND)) {
       @Override
-      public Field getField() {
-        return DATE_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableDateVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableDateVector(name, fieldType, allocator);
       }
 
       @Override
@@ -237,13 +177,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     TIME(new Time(TimeUnit.MILLISECOND, 32)) {
       @Override
-      public Field getField() {
-        return TIME_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableTimeVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeVector(name, fieldType, allocator);
       }
 
       @Override
@@ -254,13 +189,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
     TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, "UTC")) {
       @Override
-      public Field getField() {
-        return TIMESTAMPSEC_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampSecVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeStampSecVector(name, fieldType, allocator);
       }
 
       @Override
@@ -271,13 +201,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
     TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, "UTC")) {
       @Override
-      public Field getField() {
-        return TIMESTAMPMILLI_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampMilliVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeStampMilliVector(name, fieldType, allocator);
       }
 
       @Override
@@ -288,13 +213,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
     TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC")) {
       @Override
-      public Field getField() {
-        return TIMESTAMPMICRO_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampMicroVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeStampMicroVector(name, fieldType, allocator);
       }
 
       @Override
@@ -305,13 +225,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
     TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC")) {
       @Override
-      public Field getField() {
-        return TIMESTAMPNANO_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableTimeStampNanoVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeStampNanoVector(name, fieldType, allocator);
       }
 
       @Override
@@ -321,13 +236,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     INTERVALDAY(new Interval(IntervalUnit.DAY_TIME)) {
       @Override
-      public Field getField() {
-        return INTERVALDAY_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableIntervalDayVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableIntervalDayVector(name, fieldType, allocator);
       }
 
       @Override
@@ -337,13 +247,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     INTERVALYEAR(new Interval(IntervalUnit.YEAR_MONTH)) {
       @Override
-      public Field getField() {
-        return INTERVALYEAR_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableIntervalDayVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableIntervalYearVector(name, fieldType, allocator);
       }
 
       @Override
@@ -354,13 +259,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     //  4 byte ieee 754
     FLOAT4(new FloatingPoint(SINGLE)) {
       @Override
-      public Field getField() {
-        return FLOAT4_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableFloat4Vector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableFloat4Vector(name, fieldType, allocator);
       }
 
       @Override
@@ -371,13 +271,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     //  8 byte ieee 754
     FLOAT8(new FloatingPoint(DOUBLE)) {
       @Override
-      public Field getField() {
-        return FLOAT8_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableFloat8Vector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableFloat8Vector(name, fieldType, allocator);
       }
 
       @Override
@@ -387,13 +282,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     BIT(Bool.INSTANCE) {
       @Override
-      public Field getField() {
-        return BIT_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableBitVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableBitVector(name, fieldType, allocator);
       }
 
       @Override
@@ -403,13 +293,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     VARCHAR(Utf8.INSTANCE) {
       @Override
-      public Field getField() {
-        return VARCHAR_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableVarCharVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableVarCharVector(name, fieldType, allocator);
       }
 
       @Override
@@ -419,13 +304,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     VARBINARY(Binary.INSTANCE) {
       @Override
-      public Field getField() {
-        return VARBINARY_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableVarBinaryVector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableVarBinaryVector(name, fieldType, allocator);
       }
 
       @Override
@@ -438,14 +318,10 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       public ArrowType getType() {
         throw new UnsupportedOperationException("Cannot get simple type for Decimal type");
       }
-      @Override
-      public Field getField() {
-        throw new UnsupportedOperationException("Cannot get simple field for Decimal type");
-      }
 
       @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableDecimalVector(name, allocator, dictionary, precisionScale[0], precisionScale[1]);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableDecimalVector(name, fieldType, allocator);
       }
 
       @Override
@@ -455,13 +331,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     UINT1(new Int(8, false)) {
       @Override
-      public Field getField() {
-        return UINT1_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableUInt1Vector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableUInt1Vector(name, fieldType, allocator);
       }
 
       @Override
@@ -471,13 +342,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     UINT2(new Int(16, false)) {
       @Override
-      public Field getField() {
-        return UINT2_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableUInt2Vector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableUInt2Vector(name, fieldType, allocator);
       }
 
       @Override
@@ -487,13 +353,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     UINT4(new Int(32, false)) {
       @Override
-      public Field getField() {
-        return UINT4_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableUInt4Vector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableUInt4Vector(name, fieldType, allocator);
       }
 
       @Override
@@ -503,13 +364,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     UINT8(new Int(64, false)) {
       @Override
-      public Field getField() {
-        return UINT8_FIELD;
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new NullableUInt8Vector(name, allocator, dictionary);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableUInt8Vector(name, fieldType, allocator);
       }
 
       @Override
@@ -519,13 +375,8 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     LIST(List.INSTANCE) {
       @Override
-      public Field getField() {
-        throw new UnsupportedOperationException("Cannot get simple field for List type");
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        return new ListVector(name, allocator, dictionary, callBack);
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new ListVector(name, allocator, fieldType.getDictionary(), schemaChangeCallback);
       }
 
       @Override
@@ -535,16 +386,11 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     },
     UNION(new Union(Sparse, null)) {
       @Override
-      public Field getField() {
-        throw new UnsupportedOperationException("Cannot get simple field for Union type");
-      }
-
-      @Override
-      public FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale) {
-        if (dictionary != null) {
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        if (fieldType.getDictionary() != null) {
           throw new UnsupportedOperationException("Dictionary encoding not supported for complex types");
         }
-        return new UnionVector(name, allocator, callBack);
+        return new UnionVector(name, allocator, schemaChangeCallback);
       }
 
       @Override
@@ -563,9 +409,7 @@ public ArrowType getType() {
       return type;
     }
 
-    public abstract Field getField();
-
-    public abstract FieldVector getNewVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack, int... precisionScale);
+    public abstract FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback);
 
     public abstract FieldWriter getNewFieldWriter(ValueVector vector);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 011f0e6e446a8..05eb9cdceac23 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -28,11 +28,10 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.schema.TypeLayout;
 import org.apache.arrow.vector.schema.VectorLayout;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -41,10 +40,17 @@
 import com.google.flatbuffers.FlatBufferBuilder;
 
 public class Field {
+
+  public static Field nullablePrimitive(String name, ArrowType.PrimitiveType type) {
+    return nullable(name, type);
+  }
+
+  public static Field nullable(String name, ArrowType type) {
+    return new Field(name, true, type, null, null);
+  }
+
   private final String name;
-  private final boolean nullable;
-  private final ArrowType type;
-  private final DictionaryEncoding dictionary;
+  private final FieldType fieldType;
   private final List<Field> children;
   private final TypeLayout typeLayout;
 
@@ -56,29 +62,31 @@ private Field(
       @JsonProperty("dictionary") DictionaryEncoding dictionary,
       @JsonProperty("children") List<Field> children,
       @JsonProperty("typeLayout") TypeLayout typeLayout) {
+    this(name, new FieldType(nullable, type, dictionary), children, typeLayout);
+  }
+
+  private Field(String name, FieldType fieldType, List<Field> children, TypeLayout typeLayout) {
+    super();
     this.name = name;
-    this.nullable = nullable;
-    this.type = checkNotNull(type);
-    this.dictionary = dictionary;
-    if (children == null) {
-      this.children = ImmutableList.of();
-    } else {
-      this.children = children;
-    }
+    this.fieldType = checkNotNull(fieldType);
+    this.children = children == null ? ImmutableList.<Field>of() : children;
     this.typeLayout = checkNotNull(typeLayout);
   }
 
+  public Field(String name, FieldType fieldType, List<Field> children) {
+    this(name, fieldType, children, TypeLayout.getTypeLayout(fieldType.getType()));
+  }
+
   public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
-    this(name, nullable, type, null, children, TypeLayout.getTypeLayout(checkNotNull(type)));
+    this(name, nullable, type, null, children);
   }
 
   public Field(String name, boolean nullable, ArrowType type, DictionaryEncoding dictionary, List<Field> children) {
-    this(name, nullable, type, dictionary, children, TypeLayout.getTypeLayout(checkNotNull(type)));
+    this(name, new FieldType(nullable, type, dictionary), children);
   }
 
   public FieldVector createVector(BufferAllocator allocator) {
-    MinorType minorType = Types.getMinorTypeForArrowType(type);
-    FieldVector vector = minorType.getNewVector(name, allocator, dictionary, null);
+    FieldVector vector = fieldType.createNewSingleVector(name, allocator, null);
     vector.initializeChildrenFromFields(children);
     return vector;
   }
@@ -110,7 +118,7 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
   }
 
   public void validate() {
-    TypeLayout expectedLayout = TypeLayout.getTypeLayout(type);
+    TypeLayout expectedLayout = TypeLayout.getTypeLayout(getType());
     if (!expectedLayout.equals(typeLayout)) {
       throw new IllegalArgumentException("Deserialized field does not match expected vectors. expected: " + expectedLayout + " got " + typeLayout);
     }
@@ -118,8 +126,9 @@ public void validate() {
 
   public int getField(FlatBufferBuilder builder) {
     int nameOffset = name == null ? -1 : builder.createString(name);
-    int typeOffset = type.getType(builder);
+    int typeOffset = getType().getType(builder);
     int dictionaryOffset = -1;
+    DictionaryEncoding dictionary = getDictionary();
     if (dictionary != null) {
       int dictionaryType = dictionary.getIndexType().getType(builder);
       org.apache.arrow.flatbuf.DictionaryEncoding.startDictionaryEncoding(builder);
@@ -143,8 +152,8 @@ public int getField(FlatBufferBuilder builder) {
     if (name != null) {
       org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
     }
-    org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
-    org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeID().getFlatbufID());
+    org.apache.arrow.flatbuf.Field.addNullable(builder, isNullable());
+    org.apache.arrow.flatbuf.Field.addTypeType(builder, getType().getTypeID().getFlatbufID());
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
     org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
     org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
@@ -159,15 +168,22 @@ public String getName() {
   }
 
   public boolean isNullable() {
-    return nullable;
+    return fieldType.isNullable();
   }
 
   public ArrowType getType() {
-    return type;
+    return fieldType.getType();
+  }
+
+  @JsonIgnore
+  public FieldType getFieldType() {
+    return fieldType;
   }
 
   @JsonInclude(Include.NON_NULL)
-  public DictionaryEncoding getDictionary() { return dictionary; }
+  public DictionaryEncoding getDictionary() {
+    return fieldType.getDictionary();
+  }
 
   public List<Field> getChildren() {
     return children;
@@ -179,7 +195,7 @@ public TypeLayout getTypeLayout() {
 
   @Override
   public int hashCode() {
-    return Objects.hash(name, nullable, type, dictionary, children);
+    return Objects.hash(name, isNullable(), getType(), getDictionary(), children);
   }
 
   @Override
@@ -189,10 +205,10 @@ public boolean equals(Object obj) {
     }
     Field that = (Field) obj;
     return Objects.equals(this.name, that.name) &&
-           Objects.equals(this.nullable, that.nullable) &&
-           Objects.equals(this.type, that.type) &&
-           Objects.equals(this.dictionary, that.dictionary) &&
-           Objects.equals(this.children, that.children);
+            Objects.equals(this.isNullable(), that.isNullable()) &&
+            Objects.equals(this.getType(), that.getType()) &&
+           Objects.equals(this.getDictionary(), that.getDictionary()) &&
+            Objects.equals(this.children, that.children);
   }
 
   @Override
@@ -201,14 +217,14 @@ public String toString() {
     if (name != null) {
       sb.append(name).append(": ");
     }
-    sb.append(type);
-    if (dictionary != null) {
-      sb.append("[dictionary: ").append(dictionary.getId()).append("]");
+    sb.append(getType());
+    if (getDictionary() != null) {
+      sb.append("[dictionary: ").append(getDictionary().getId()).append("]");
     }
     if (!children.isEmpty()) {
       sb.append("<").append(Joiner.on(", ").join(children)).append(">");
     }
-    if (!nullable) {
+    if (!isNullable()) {
       sb.append(" not null");
     }
     return sb.toString();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
new file mode 100644
index 0000000000000..fe99e631360cc
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.types.pojo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+
+public class FieldType {
+
+  public static FieldType nullable(ArrowType type) {
+    return new FieldType(true, type, null);
+  }
+
+  private final boolean nullable;
+  private final ArrowType type;
+  private final DictionaryEncoding dictionary;
+
+  public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary) {
+    super();
+    this.nullable = nullable;
+    this.type = checkNotNull(type);
+    this.dictionary = dictionary;
+  }
+
+  public boolean isNullable() {
+    return nullable;
+  }
+  public ArrowType getType() {
+    return type;
+  }
+  public DictionaryEncoding getDictionary() {
+    return dictionary;
+  }
+
+  public FieldVector createNewSingleVector(String name, BufferAllocator allocator, CallBack schemaCallBack) {
+    MinorType minorType = Types.getMinorTypeForArrowType(type);
+    return minorType.getNewVector(name, this, allocator, schemaCallBack);
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index 20f4aa8cf643d..ee7530c8d1085 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -17,16 +17,16 @@
  */
 package org.apache.arrow.vector;
 
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.util.DecimalUtility;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
 
-import static org.junit.Assert.assertEquals;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.util.DecimalUtility;
+import org.junit.Test;
 
 public class TestDecimalVector {
 
@@ -44,7 +44,7 @@ public class TestDecimalVector {
   @Test
   public void test() {
     BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
-    NullableDecimalVector decimalVector = new NullableDecimalVector("decimal", allocator, null, 10, scale);
+    NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(10, scale), allocator);
     decimalVector.allocateNew();
     BigDecimal[] values = new BigDecimal[intValues.length];
     for (int i = 0; i < intValues.length; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
index e3087ef8c95cc..3bf3b1cedff38 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -17,19 +17,19 @@
  */
 package org.apache.arrow.vector;
 
+import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
+import static org.junit.Assert.assertEquals;
+
+import java.nio.charset.StandardCharsets;
+
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.dictionary.DictionaryEncoder;
 import org.apache.arrow.vector.dictionary.Dictionary;
-import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.dictionary.DictionaryEncoder;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.nio.charset.StandardCharsets;
-
-import static org.junit.Assert.assertEquals;
-
 public class TestDictionaryVector {
 
   private BufferAllocator allocator;
@@ -51,8 +51,8 @@ public void terminate() throws Exception {
   @Test
   public void testEncodeStrings() {
     // Create a new value vector
-    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("foo", allocator, null, null);
-         final NullableVarCharVector dictionaryVector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector("dict", allocator, null, null)) {
+    try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator);
+         final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(512, 5);
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
new file mode 100644
index 0000000000000..b79f2da9210ab
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+
+public class TestUtils {
+
+  public static NullableVarCharVector newNullableVarCharVector(String name, BufferAllocator allocator) {
+    return (NullableVarCharVector)FieldType.nullable(new ArrowType.Utf8()).createNewSingleVector(name, allocator, null);
+  }
+
+  public static <T> T newVector(Class<T> c, String name, ArrowType type, BufferAllocator allocator) {
+    return c.cast(FieldType.nullable(type).createNewSingleVector(name, allocator, null));
+  }
+
+  public static <T> T newVector(Class<T> c, String name, MinorType type, BufferAllocator allocator) {
+    return c.cast(FieldType.nullable(type.getType()).createNewSingleVector(name, allocator, null));
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 6917638d74e4d..78ca14dc406ea 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -17,6 +17,8 @@
  */
 package org.apache.arrow.vector;
 
+import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
+import static org.apache.arrow.vector.TestUtils.newVector;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -28,6 +30,7 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.schema.TypeLayout;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
 import org.junit.Assert;
@@ -86,7 +89,7 @@ public void testFixedType() {
   public void testNullableVarLen2() {
 
     // Create a new value vector for 1024 integers.
-    try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+    try (final NullableVarCharVector vector = newNullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024 * 10, 1024);
 
@@ -116,7 +119,7 @@ public void testNullableVarLen2() {
   public void testNullableFixedType() {
 
     // Create a new value vector for 1024 integers.
-    try (final NullableUInt4Vector vector = new NullableUInt4Vector(EMPTY_SCHEMA_PATH, allocator, null)) {
+    try (final NullableUInt4Vector vector = newVector(NullableUInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) {
       final NullableUInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -186,7 +189,7 @@ public void testNullableFixedType() {
   @Test
   public void testNullableFloat() {
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null, null)) {
+    try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) {
       final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -233,7 +236,7 @@ public void testNullableFloat() {
   @Test
   public void testNullableInt() {
     // Create a new value vector for 1024 integers
-    try (final NullableIntVector vector = (NullableIntVector) MinorType.INT.getNewVector(EMPTY_SCHEMA_PATH, allocator, null, null)) {
+    try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) {
       final NullableIntVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -403,7 +406,7 @@ private void validateRange(int length, int start, int count) {
   @Test
   public void testReAllocNullableFixedWidthVector() {
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null, null)) {
+    try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) {
       final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -436,7 +439,7 @@ public void testReAllocNullableFixedWidthVector() {
   @Test
   public void testReAllocNullableVariableWidthVector() {
     // Create a new value vector for 1024 integers
-    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector(EMPTY_SCHEMA_PATH, allocator, null, null)) {
+    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 2b49d8ed4b582..65b193c0aee4c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -27,7 +27,6 @@
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
@@ -53,7 +52,7 @@ public void terminate() throws Exception {
   public void testPromoteToUnion() throws Exception {
 
     try (final MapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
-         final NullableMapVector v = container.addOrGet("test", MinorType.MAP, NullableMapVector.class, null);
+         final NullableMapVector v = container.addOrGetMap("test");
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
       container.allocateNew();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 75e5d2d6e5c98..a1104ffe545d8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -17,6 +17,8 @@
  */
 package org.apache.arrow.vector.file;
 
+import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -28,8 +30,6 @@
 import java.util.Arrays;
 import java.util.List;
 
-import com.google.common.collect.ImmutableList;
-
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.NullableTinyIntVector;
@@ -40,19 +40,19 @@
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryEncoder;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
-import org.apache.arrow.vector.dictionary.DictionaryEncoder;
 import org.apache.arrow.vector.schema.ArrowBuffer;
 import org.apache.arrow.vector.schema.ArrowMessage;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.stream.ArrowStreamReader;
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
 import org.apache.arrow.vector.stream.MessageSerializerTest;
-import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.Text;
 import org.junit.Assert;
@@ -60,6 +60,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.ImmutableList;
+
 public class TestArrowFile extends BaseFileTest {
   private static final Logger LOGGER = LoggerFactory.getLogger(TestArrowFile.class);
 
@@ -380,8 +382,8 @@ public void testWriteReadDictionary() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableVarCharVector vector = new NullableVarCharVector("varchar", originalVectorAllocator, null);
-         NullableVarCharVector dictionaryVector = new NullableVarCharVector("dict", originalVectorAllocator, null)) {
+         NullableVarCharVector vector = newNullableVarCharVector("varchar", originalVectorAllocator);
+         NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", originalVectorAllocator)) {
       vector.allocateNewSafe();
       NullableVarCharVector.Mutator mutator = vector.getMutator();
       mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
@@ -483,7 +485,7 @@ public void testWriteReadNestedDictionary() throws IOException {
     // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
 
     // write
-    try (NullableVarCharVector dictionaryVector = new NullableVarCharVector("dictionary", allocator, null);
+    try (NullableVarCharVector dictionaryVector = newNullableVarCharVector("dictionary", allocator);
          ListVector listVector = new ListVector("list", allocator, null, null)) {
 
       Dictionary dictionary = new Dictionary(dictionaryVector, encoding);
@@ -495,7 +497,7 @@ public void testWriteReadNestedDictionary() throws IOException {
       dictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
       dictionaryVector.getMutator().setValueCount(2);
 
-      listVector.addOrGetVector(MinorType.INT, encoding);
+      listVector.addOrGetVector(new FieldType(true, new Int(32, true), encoding));
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       listWriter.startList();
@@ -511,7 +513,7 @@ public void testWriteReadNestedDictionary() throws IOException {
       listWriter.setValueCount(3);
 
       List<Field> fields = ImmutableList.of(listVector.getField());
-      List<FieldVector> vectors = ImmutableList.of((FieldVector) listVector);
+      List<FieldVector> vectors = ImmutableList.<FieldVector>of(listVector);
       VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 3);
 
       try (
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index 914dfe4319db3..d00cb0f8c0065 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -24,11 +24,8 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
 import java.util.Collections;
 import java.util.List;
 
@@ -38,13 +35,10 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.NullableIntVector;
-import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.TestUtils;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -78,8 +72,8 @@ byte[] array(ArrowBuf buf) {
   @Test
   public void test() throws IOException {
     Schema schema = new Schema(asList(new Field("testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
-    MinorType minorType = Types.getMinorTypeForArrowType(schema.getFields().get(0).getType());
-    FieldVector vector = minorType.getNewVector("testField", allocator, null,null);
+    ArrowType type = schema.getFields().get(0).getType();
+    FieldVector vector = TestUtils.newVector(FieldVector.class, "testField", type, allocator);
     vector.initializeChildrenFromFields(schema.getFields().get(0).getChildren());
 
     byte[] validity = new byte[] { (byte) 255, 0};

From 7d1d4e751807ac38cfe7a5c537450ede3ae9eb00 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 3 Apr 2017 18:05:04 -0400
Subject: [PATCH 0460/1644] ARROW-657: [C++/Python] Expose Tensor IPC in
 Python. Add equals method. Add pyarrow.create_memory_map/memory_map functions

This adds a `MemoryMappedFile::Create` C++ function for allocating new memory maps of a particular size, with a Python wrapper. I combined the Cython header declarations for the main libarrow libraries into a single pxd file.

I am also checking in some tests from ARROW-718 that got left off the patch.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #483 from wesm/ARROW-657 and squashes the following commits:

a5d2f00 [Wes McKinney] More readable cinit post refactor
fbf438d [Wes McKinney] clang-format
4024c22 [Wes McKinney] Fix MSVC issues, use slower SetBitTo implementation that doesn't have compiler warning
7847d0f [Wes McKinney] Make file names unique
994b83d [Wes McKinney] Try to fix MSVC
25cfc83 [Wes McKinney] Expose Tensor IPC in Python. Add equals method. Add create_memory_map function and memory_map factory
---
 cpp/CMakeLists.txt                       |   2 +-
 cpp/src/arrow/buffer.h                   |   7 +-
 cpp/src/arrow/io/CMakeLists.txt          |   2 +-
 cpp/src/arrow/io/file.cc                 |  13 ++
 cpp/src/arrow/io/file.h                  |   4 +
 cpp/src/arrow/io/io-file-test.cc         |   8 +-
 cpp/src/arrow/io/test-common.h           |  20 +-
 cpp/src/arrow/ipc/CMakeLists.txt         |   2 +-
 cpp/src/arrow/ipc/feather-internal.h     |  16 +-
 cpp/src/arrow/ipc/feather.h              |   1 +
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  12 +-
 cpp/src/arrow/ipc/metadata.h             |  27 +--
 cpp/src/arrow/ipc/writer.h               |  18 +-
 cpp/src/arrow/util/bit-util.h            |  10 +-
 python/pyarrow/__init__.py               |   6 +-
 python/pyarrow/_parquet.pxd              |   4 +-
 python/pyarrow/_parquet.pyx              |   2 -
 python/pyarrow/array.pxd                 |   1 +
 python/pyarrow/array.pyx                 |   6 +
 python/pyarrow/includes/libarrow.pxd     | 235 +++++++++++++++++++++++
 python/pyarrow/includes/libarrow_io.pxd  | 171 -----------------
 python/pyarrow/includes/libarrow_ipc.pxd |  94 ---------
 python/pyarrow/includes/pyarrow.pxd      |  12 +-
 python/pyarrow/io.pxd                    |   4 +-
 python/pyarrow/io.pyx                    | 124 +++++++++++-
 python/pyarrow/tests/test_io.py          |  16 +-
 python/pyarrow/tests/test_tensor.py      |  93 +++++++++
 27 files changed, 553 insertions(+), 357 deletions(-)
 delete mode 100644 python/pyarrow/includes/libarrow_io.pxd
 delete mode 100644 python/pyarrow/includes/libarrow_ipc.pxd
 create mode 100644 python/pyarrow/tests/test_tensor.py

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index aacc7a15fffc9..d26c847807d79 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -806,7 +806,7 @@ set(ARROW_SRCS
   src/arrow/util/bit-util.cc
 )
 
-if(NOT APPLE)
+if(NOT APPLE AND NOT MSVC)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 3f14c964e83c1..a02ce3cbe8107 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -46,7 +46,7 @@ class Status;
 class ARROW_EXPORT Buffer {
  public:
   Buffer(const uint8_t* data, int64_t size)
-    : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
+      : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
   virtual ~Buffer();
 
   /// An offset into data that is owned by another buffer, but we want to be
@@ -57,7 +57,7 @@ class ARROW_EXPORT Buffer {
   /// in general we expected buffers to be aligned and padded to 64 bytes.  In the future
   /// we might add utility methods to help determine if a buffer satisfies this contract.
   Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size)
-    : Buffer(parent->data() + offset, size) {
+      : Buffer(parent->data() + offset, size) {
     parent_ = parent;
   }
 
@@ -112,8 +112,7 @@ std::shared_ptr<Buffer> ARROW_EXPORT SliceMutableBuffer(
 /// A Buffer whose contents can be mutated. May or may not own its data.
 class ARROW_EXPORT MutableBuffer : public Buffer {
  public:
-  MutableBuffer(uint8_t* data, int64_t size)
-    : Buffer(data, size) {
+  MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
     mutable_data_ = data;
     is_mutable_ = true;
   }
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 3951eac322c6a..791c29c2797f9 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -75,7 +75,7 @@ set(ARROW_IO_SRCS
   memory.cc
 )
 
-if(NOT APPLE)
+if(NOT APPLE AND NOT MSVC)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 0aa2c92a07281..720be3d6e739c 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -604,6 +604,19 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
 MemoryMappedFile::MemoryMappedFile() {}
 MemoryMappedFile::~MemoryMappedFile() {}
 
+Status MemoryMappedFile::Create(
+    const std::string& path, int64_t size, std::shared_ptr<MemoryMappedFile>* out) {
+  std::shared_ptr<FileOutputStream> file;
+  RETURN_NOT_OK(FileOutputStream::Open(path, &file));
+#ifdef _MSC_VER
+  _chsize_s(file->file_descriptor(), static_cast<size_t>(size));
+#else
+  ftruncate(file->file_descriptor(), static_cast<size_t>(size));
+#endif
+  RETURN_NOT_OK(file->Close());
+  return MemoryMappedFile::Open(path, FileMode::READWRITE, out);
+}
+
 Status MemoryMappedFile::Open(const std::string& path, FileMode::type mode,
     std::shared_ptr<MemoryMappedFile>* out) {
   std::shared_ptr<MemoryMappedFile> result(new MemoryMappedFile());
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index f687fadc299bd..f0be3cf980162 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -106,6 +106,10 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
  public:
   ~MemoryMappedFile();
 
+  /// Create new file with indicated size, return in read/write mode
+  static Status Create(
+      const std::string& path, int64_t size, std::shared_ptr<MemoryMappedFile>* out);
+
   static Status Open(const std::string& path, FileMode::type mode,
       std::shared_ptr<MemoryMappedFile>* out);
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 348be17d89341..a5784de3752d9 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -393,10 +393,8 @@ TEST_F(TestMemoryMappedFile, WriteRead) {
   const int reps = 5;
 
   std::string path = "ipc-write-read-test";
-  CreateFile(path, reps * buffer_size);
-
   std::shared_ptr<MemoryMappedFile> result;
-  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &result));
+  ASSERT_OK(InitMemoryMap(reps * buffer_size, path, &result));
 
   int64_t position = 0;
   std::shared_ptr<Buffer> out_buffer;
@@ -419,10 +417,8 @@ TEST_F(TestMemoryMappedFile, ReadOnly) {
   const int reps = 5;
 
   std::string path = "ipc-read-only-test";
-  CreateFile(path, reps * buffer_size);
-
   std::shared_ptr<MemoryMappedFile> rwmmap;
-  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &rwmmap));
+  ASSERT_OK(InitMemoryMap(reps * buffer_size, path, &rwmmap));
 
   int64_t position = 0;
   for (int i = 0; i < reps; ++i) {
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index db5bcc1b4f49b..d6ec27048d51e 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -67,23 +67,17 @@ class MemoryMapFixture {
     }
   }
 
-  void CreateFile(const std::string path, int64_t size) {
-    FILE* file = fopen(path.c_str(), "w");
-    if (file != nullptr) {
-      tmp_files_.push_back(path);
-#ifdef _MSC_VER
-      _chsize(fileno(file), static_cast<size_t>(size));
-#else
-      ftruncate(fileno(file), static_cast<size_t>(size));
-#endif
-      fclose(file);
-    }
+  void CreateFile(const std::string& path, int64_t size) {
+    std::shared_ptr<MemoryMappedFile> file;
+    ASSERT_OK(MemoryMappedFile::Create(path, size, &file));
+    tmp_files_.push_back(path);
   }
 
   Status InitMemoryMap(
       int64_t size, const std::string& path, std::shared_ptr<MemoryMappedFile>* mmap) {
-    CreateFile(path, size);
-    return MemoryMappedFile::Open(path, FileMode::READWRITE, mmap);
+    RETURN_NOT_OK(MemoryMappedFile::Create(path, size, mmap));
+    tmp_files_.push_back(path);
+    return Status::OK();
   }
 
  private:
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 5fa7d6125ce5e..57db03311c06f 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -38,7 +38,7 @@ set(ARROW_IPC_SRCS
   writer.cc
 )
 
-if(NOT APPLE)
+if(NOT MSVC AND NOT APPLE)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
diff --git a/cpp/src/arrow/ipc/feather-internal.h b/cpp/src/arrow/ipc/feather-internal.h
index 10b0cfd5d5ea2..6847445149bb0 100644
--- a/cpp/src/arrow/ipc/feather-internal.h
+++ b/cpp/src/arrow/ipc/feather-internal.h
@@ -41,11 +41,11 @@ typedef std::vector<flatbuffers::Offset<fbs::Column>> ColumnVector;
 typedef flatbuffers::FlatBufferBuilder FBB;
 typedef flatbuffers::Offset<flatbuffers::String> FBString;
 
-struct ColumnType {
+struct ARROW_EXPORT ColumnType {
   enum type { PRIMITIVE, CATEGORY, TIMESTAMP, DATE, TIME };
 };
 
-struct ArrayMetadata {
+struct ARROW_EXPORT ArrayMetadata {
   ArrayMetadata() {}
 
   ArrayMetadata(fbs::Type type, int64_t offset, int64_t length, int64_t null_count,
@@ -69,12 +69,12 @@ struct ArrayMetadata {
   int64_t total_bytes;
 };
 
-struct CategoryMetadata {
+struct ARROW_EXPORT CategoryMetadata {
   ArrayMetadata levels;
   bool ordered;
 };
 
-struct TimestampMetadata {
+struct ARROW_EXPORT TimestampMetadata {
   TimeUnit unit;
 
   // A timezone name known to the Olson timezone database. For display purposes
@@ -82,7 +82,7 @@ struct TimestampMetadata {
   std::string timezone;
 };
 
-struct TimeMetadata {
+struct ARROW_EXPORT TimeMetadata {
   TimeUnit unit;
 };
 
@@ -91,7 +91,7 @@ static constexpr const int kFeatherDefaultAlignment = 8;
 
 class ColumnBuilder;
 
-class TableBuilder {
+class ARROW_EXPORT TableBuilder {
  public:
   explicit TableBuilder(int64_t num_rows);
   ~TableBuilder() = default;
@@ -116,7 +116,7 @@ class TableBuilder {
   int64_t num_rows_;
 };
 
-class TableMetadata {
+class ARROW_EXPORT TableMetadata {
  public:
   TableMetadata() {}
   ~TableMetadata() = default;
@@ -186,7 +186,7 @@ static inline void FromFlatbuffer(const fbs::PrimitiveArray* values, ArrayMetada
   out->total_bytes = values->total_bytes();
 }
 
-class ColumnBuilder {
+class ARROW_EXPORT ColumnBuilder {
  public:
   ColumnBuilder(TableBuilder* parent, const std::string& name);
   ~ColumnBuilder() = default;
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
index 8cc8ca092a1b2..4d59a8bbd54a9 100644
--- a/cpp/src/arrow/ipc/feather.h
+++ b/cpp/src/arrow/ipc/feather.h
@@ -27,6 +27,7 @@
 #include <vector>
 
 #include "arrow/type.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index c900d0ba37ed2..86ec7701add20 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -106,6 +106,8 @@ TEST_F(TestSchemaMetadata, NestedFields) {
       &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes, \
       &MakeFWBinary, &MakeBooleanBatch);
 
+static int g_file_number = 0;
+
 class IpcTestFixture : public io::MemoryMapFixture {
  public:
   Status DoStandardRoundTrip(const RecordBatch& batch, bool zero_data,
@@ -163,8 +165,9 @@ class IpcTestFixture : public io::MemoryMapFixture {
   }
 
   void CheckRoundtrip(const RecordBatch& batch, int64_t buffer_size) {
-    std::string path = "test-write-row-batch";
-    ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(buffer_size, path, &mmap_));
+    std::stringstream ss;
+    ss << "test-write-row-batch-" << g_file_number++;
+    ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(buffer_size, ss.str(), &mmap_));
 
     std::shared_ptr<RecordBatch> result;
     ASSERT_OK(DoStandardRoundTrip(batch, true, &result));
@@ -303,9 +306,10 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
     std::vector<std::shared_ptr<Array>> arrays = {array};
     *batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
 
-    std::string path = "test-write-past-max-recursion";
+    std::stringstream ss;
+    ss << "test-write-past-max-recursion-" << g_file_number++;
     const int memory_map_size = 1 << 20;
-    io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
+    RETURN_NOT_OK(io::MemoryMapFixture::InitMemoryMap(memory_map_size, ss.str(), &mmap_));
 
     if (override_level) {
       return WriteRecordBatch(**batch, 0, mmap_.get(), metadata_length, body_length,
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index fac4a70aada8d..451a76d5249e0 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -72,7 +72,7 @@ using DictionaryMap = std::unordered_map<int64_t, std::shared_ptr<Array>>;
 using DictionaryTypeMap = std::unordered_map<int64_t, std::shared_ptr<Field>>;
 
 // Memoization data structure for handling shared dictionaries
-class DictionaryMemo {
+class ARROW_EXPORT DictionaryMemo {
  public:
   DictionaryMemo();
 
@@ -114,12 +114,12 @@ Status GetDictionaryTypes(const void* opaque_schema, DictionaryTypeMap* id_to_fi
 
 // Construct a complete Schema from the message. May be expensive for very
 // large schemas if you are only interested in a few fields
-Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_memo,
-    std::shared_ptr<Schema>* out);
+Status ARROW_EXPORT GetSchema(const void* opaque_schema,
+    const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out);
 
-Status GetTensorMetadata(const void* opaque_tensor, std::shared_ptr<DataType>* type,
-    std::vector<int64_t>* shape, std::vector<int64_t>* strides,
-    std::vector<std::string>* dim_names);
+Status ARROW_EXPORT GetTensorMetadata(const void* opaque_tensor,
+    std::shared_ptr<DataType>* type, std::vector<int64_t>* shape,
+    std::vector<int64_t>* strides, std::vector<std::string>* dim_names);
 
 class ARROW_EXPORT Message {
  public:
@@ -157,18 +157,19 @@ class ARROW_EXPORT Message {
 /// \param[in] file the seekable file interface to read from
 /// \param[out] message the message read
 /// \return Status success or failure
-Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
-    std::shared_ptr<Message>* message);
+Status ARROW_EXPORT ReadMessage(int64_t offset, int32_t metadata_length,
+    io::RandomAccessFile* file, std::shared_ptr<Message>* message);
 
 /// Read length-prefixed message with as-yet unknown length. Returns nullptr if
 /// there are not enough bytes available or the message length is 0 (e.g. EOS
 /// in a stream)
-Status ReadMessage(io::InputStream* stream, std::shared_ptr<Message>* message);
+Status ARROW_EXPORT ReadMessage(
+    io::InputStream* stream, std::shared_ptr<Message>* message);
 
 /// Write a serialized message with a length-prefix and padding to an 8-byte offset
 ///
 /// <message_size: int32><message: const void*><padding>
-Status WriteMessage(
+Status ARROW_EXPORT WriteMessage(
     const Buffer& message, io::OutputStream* file, int32_t* message_length);
 
 // Serialize arrow::Schema as a Flatbuffer
@@ -178,14 +179,14 @@ Status WriteMessage(
 // dictionary ids
 // \param[out] out the serialized arrow::Buffer
 // \return Status outcome
-Status WriteSchemaMessage(
+Status ARROW_EXPORT WriteSchemaMessage(
     const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out);
 
-Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
+Status ARROW_EXPORT WriteRecordBatchMessage(int64_t length, int64_t body_length,
     const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
     std::shared_ptr<Buffer>* out);
 
-Status WriteTensorMessage(
+Status ARROW_EXPORT WriteTensorMessage(
     const Tensor& tensor, int64_t buffer_start_offset, std::shared_ptr<Buffer>* out);
 
 Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 8b2dc9cd48788..0b7a6e1b56be5 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -66,9 +66,9 @@ namespace ipc {
 /// including padding to a 64-byte boundary
 /// @param(out) body_length: the size of the contiguous buffer block plus
 /// padding bytes
-Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
-    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth,
+Status ARROW_EXPORT WriteRecordBatch(const RecordBatch& batch,
+    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
+    int64_t* body_length, MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth,
     bool allow_64bit = false);
 
 // Write Array as a DictionaryBatch message
@@ -79,7 +79,7 @@ Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dict
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the record batch. This involves generating the complete serialized
 // Flatbuffers metadata.
-Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
+Status ARROW_EXPORT GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 
 class ARROW_EXPORT StreamWriter {
  public:
@@ -122,14 +122,14 @@ class ARROW_EXPORT FileWriter : public StreamWriter {
 
 /// EXPERIMENTAL: Write RecordBatch allowing lengths over INT32_MAX. This data
 /// may not be readable by all Arrow implementations
-Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
-    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool);
+Status ARROW_EXPORT WriteLargeRecordBatch(const RecordBatch& batch,
+    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
+    int64_t* body_length, MemoryPool* pool);
 
 /// EXPERIMENTAL: Write arrow::Tensor as a contiguous message
 /// <metadata size><metadata><tensor data>
-Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length);
+Status ARROW_EXPORT WriteTensor(const Tensor& tensor, io::OutputStream* dst,
+    int32_t* metadata_length, int64_t* body_length);
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 6e3e8ae9f2160..42afd0705f0f9 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -52,7 +52,7 @@ static inline int64_t Ceil2Bytes(int64_t size) {
 }
 
 static inline bool GetBit(const uint8_t* bits, int64_t i) {
-  return static_cast<bool>(bits[i / 8] & kBitmask[i % 8]);
+  return (bits[i / 8] & kBitmask[i % 8]) != 0;
 }
 
 static inline bool BitNotSet(const uint8_t* bits, int64_t i) {
@@ -68,9 +68,13 @@ static inline void SetBit(uint8_t* bits, int64_t i) {
 }
 
 static inline void SetBitTo(uint8_t* bits, int64_t i, bool bit_is_set) {
-  // See https://graphics.stanford.edu/~seander/bithacks.html
+  // TODO: speed up. See https://graphics.stanford.edu/~seander/bithacks.html
   // "Conditionally set or clear bits without branching"
-  bits[i / 8] ^= static_cast<uint8_t>(-bit_is_set ^ bits[i / 8]) & kBitmask[i % 8];
+  if (bit_is_set) {
+    SetBit(bits, i);
+  } else {
+    ClearBit(bits, i);
+  }
 }
 
 static inline int64_t NextPower2(int64_t n) {
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 5215028c90f0d..6860f986fb6e8 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -42,8 +42,10 @@
 
 from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
 from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
-                        Buffer, InMemoryOutputStream, BufferReader,
-                        frombuffer)
+                        Buffer, BufferReader, InMemoryOutputStream,
+                        MemoryMappedFile, memory_map,
+                        frombuffer, read_tensor, write_tensor,
+                        memory_map, create_memory_map)
 
 from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
 
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index cf9ec8e787661..f12c86fdebc83 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -19,8 +19,8 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
-                                        CTable, CMemoryPool)
-from pyarrow.includes.libarrow_io cimport RandomAccessFile, OutputStream
+                                        CTable, CMemoryPool,
+                                        RandomAccessFile, OutputStream)
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index c4cbd28e85dab..cfd2816e2a16e 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -23,8 +23,6 @@ from cython.operator cimport dereference as deref
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.libarrow_io cimport (RandomAccessFile, OutputStream,
-                                           FileOutputStream)
 cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.array cimport Array
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index 42675630fd51b..f6aaea2582e21 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -53,6 +53,7 @@ cdef class Tensor:
 
 
 cdef object box_array(const shared_ptr[CArray]& sp_array)
+cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
 
 
 cdef class BooleanArray(Array):
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 398e4cbffa94d..e7c456d80a41f 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -347,6 +347,12 @@ strides: {2}""".format(self.type, self.shape, self.strides)
                                              &out))
         return PyObject_to_object(out)
 
+    def equals(self, Tensor other):
+        """
+        Return true if the tensors contains exactly equal data
+        """
+        return self.tp.Equals(deref(other.tp))
+
     property is_mutable:
 
         def __get__(self):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 8da063cbdc364..67d6af910c2b9 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -303,6 +303,162 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
                               shared_ptr[CTable]* result)
 
 
+cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
+    enum FileMode" arrow::io::FileMode::type":
+        FileMode_READ" arrow::io::FileMode::READ"
+        FileMode_WRITE" arrow::io::FileMode::WRITE"
+        FileMode_READWRITE" arrow::io::FileMode::READWRITE"
+
+    enum ObjectType" arrow::io::ObjectType::type":
+        ObjectType_FILE" arrow::io::ObjectType::FILE"
+        ObjectType_DIRECTORY" arrow::io::ObjectType::DIRECTORY"
+
+    cdef cppclass FileInterface:
+        CStatus Close()
+        CStatus Tell(int64_t* position)
+        FileMode mode()
+
+    cdef cppclass Readable:
+        CStatus ReadB" Read"(int64_t nbytes, shared_ptr[CBuffer]* out)
+        CStatus Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out)
+
+    cdef cppclass Seekable:
+        CStatus Seek(int64_t position)
+
+    cdef cppclass Writeable:
+        CStatus Write(const uint8_t* data, int64_t nbytes)
+
+    cdef cppclass OutputStream(FileInterface, Writeable):
+        pass
+
+    cdef cppclass InputStream(FileInterface, Readable):
+        pass
+
+    cdef cppclass RandomAccessFile(InputStream, Seekable):
+        CStatus GetSize(int64_t* size)
+
+        CStatus ReadAt(int64_t position, int64_t nbytes,
+                       int64_t* bytes_read, uint8_t* buffer)
+        CStatus ReadAt(int64_t position, int64_t nbytes,
+                       int64_t* bytes_read, shared_ptr[CBuffer]* out)
+
+    cdef cppclass WriteableFileInterface(OutputStream, Seekable):
+        CStatus WriteAt(int64_t position, const uint8_t* data,
+                        int64_t nbytes)
+
+    cdef cppclass ReadWriteFileInterface(RandomAccessFile,
+                                         WriteableFileInterface):
+        pass
+
+
+cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
+
+
+    cdef cppclass FileOutputStream(OutputStream):
+        @staticmethod
+        CStatus Open(const c_string& path, shared_ptr[FileOutputStream]* file)
+
+        int file_descriptor()
+
+    cdef cppclass ReadableFile(RandomAccessFile):
+        @staticmethod
+        CStatus Open(const c_string& path, shared_ptr[ReadableFile]* file)
+
+        @staticmethod
+        CStatus Open(const c_string& path, CMemoryPool* memory_pool,
+                     shared_ptr[ReadableFile]* file)
+
+        int file_descriptor()
+
+    cdef cppclass CMemoryMappedFile" arrow::io::MemoryMappedFile"\
+        (ReadWriteFileInterface):
+
+        @staticmethod
+        CStatus Create(const c_string& path, int64_t size,
+                     shared_ptr[CMemoryMappedFile]* file)
+
+        @staticmethod
+        CStatus Open(const c_string& path, FileMode mode,
+                     shared_ptr[CMemoryMappedFile]* file)
+
+        int file_descriptor()
+
+
+cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
+    CStatus HaveLibHdfs()
+    CStatus HaveLibHdfs3()
+
+    enum HdfsDriver" arrow::io::HdfsDriver":
+        HdfsDriver_LIBHDFS" arrow::io::HdfsDriver::LIBHDFS"
+        HdfsDriver_LIBHDFS3" arrow::io::HdfsDriver::LIBHDFS3"
+
+    cdef cppclass HdfsConnectionConfig:
+        c_string host
+        int port
+        c_string user
+        c_string kerb_ticket
+        HdfsDriver driver
+
+    cdef cppclass HdfsPathInfo:
+        ObjectType kind;
+        c_string name
+        c_string owner
+        c_string group
+        int32_t last_modified_time
+        int32_t last_access_time
+        int64_t size
+        int16_t replication
+        int64_t block_size
+        int16_t permissions
+
+    cdef cppclass HdfsReadableFile(RandomAccessFile):
+        pass
+
+    cdef cppclass HdfsOutputStream(OutputStream):
+        pass
+
+    cdef cppclass CHdfsClient" arrow::io::HdfsClient":
+        @staticmethod
+        CStatus Connect(const HdfsConnectionConfig* config,
+                        shared_ptr[CHdfsClient]* client)
+
+        CStatus CreateDirectory(const c_string& path)
+
+        CStatus Delete(const c_string& path, c_bool recursive)
+
+        CStatus Disconnect()
+
+        c_bool Exists(const c_string& path)
+
+        CStatus GetCapacity(int64_t* nbytes)
+        CStatus GetUsed(int64_t* nbytes)
+
+        CStatus ListDirectory(const c_string& path,
+                              vector[HdfsPathInfo]* listing)
+
+        CStatus GetPathInfo(const c_string& path, HdfsPathInfo* info)
+
+        CStatus Rename(const c_string& src, const c_string& dst)
+
+        CStatus OpenReadable(const c_string& path,
+                             shared_ptr[HdfsReadableFile]* handle)
+
+        CStatus OpenWriteable(const c_string& path, c_bool append,
+                              int32_t buffer_size, int16_t replication,
+                              int64_t default_block_size,
+                              shared_ptr[HdfsOutputStream]* handle)
+
+
+cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
+    cdef cppclass CBufferReader" arrow::io::BufferReader"\
+        (RandomAccessFile):
+        CBufferReader(const shared_ptr[CBuffer]& buffer)
+        CBufferReader(const uint8_t* data, int64_t nbytes)
+
+    cdef cppclass BufferOutputStream(OutputStream):
+        BufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
+
+
 cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
     cdef cppclass SchemaMessage:
         int num_fields()
@@ -335,3 +491,82 @@ cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
         shared_ptr[SchemaMessage] GetSchema()
         shared_ptr[RecordBatchMessage] GetRecordBatch()
         shared_ptr[DictionaryBatchMessage] GetDictionaryBatch()
+
+
+cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
+
+    cdef cppclass CStreamWriter " arrow::ipc::StreamWriter":
+        @staticmethod
+        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
+                     shared_ptr[CStreamWriter]* out)
+
+        CStatus Close()
+        CStatus WriteRecordBatch(const CRecordBatch& batch)
+
+    cdef cppclass CStreamReader " arrow::ipc::StreamReader":
+
+        @staticmethod
+        CStatus Open(const shared_ptr[InputStream]& stream,
+                     shared_ptr[CStreamReader]* out)
+
+        shared_ptr[CSchema] schema()
+
+        CStatus GetNextRecordBatch(shared_ptr[CRecordBatch]* batch)
+
+    cdef cppclass CFileWriter " arrow::ipc::FileWriter"(CStreamWriter):
+        @staticmethod
+        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
+                     shared_ptr[CFileWriter]* out)
+
+    cdef cppclass CFileReader " arrow::ipc::FileReader":
+
+        @staticmethod
+        CStatus Open(const shared_ptr[RandomAccessFile]& file,
+                     shared_ptr[CFileReader]* out)
+
+        @staticmethod
+        CStatus Open2" Open"(const shared_ptr[RandomAccessFile]& file,
+                     int64_t footer_offset, shared_ptr[CFileReader]* out)
+
+        shared_ptr[CSchema] schema()
+
+        int num_record_batches()
+
+        CStatus GetRecordBatch(int i, shared_ptr[CRecordBatch]* batch)
+
+    CStatus WriteTensor(const CTensor& tensor, OutputStream* dst,
+                        int32_t* metadata_length,
+                        int64_t* body_length)
+
+    CStatus ReadTensor(int64_t offset, RandomAccessFile* file,
+                       shared_ptr[CTensor]* out)
+
+
+cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
+
+    cdef cppclass CFeatherWriter" arrow::ipc::feather::TableWriter":
+        @staticmethod
+        CStatus Open(const shared_ptr[OutputStream]& stream,
+                     unique_ptr[CFeatherWriter]* out)
+
+        void SetDescription(const c_string& desc)
+        void SetNumRows(int64_t num_rows)
+
+        CStatus Append(const c_string& name, const CArray& values)
+        CStatus Finalize()
+
+    cdef cppclass CFeatherReader" arrow::ipc::feather::TableReader":
+        @staticmethod
+        CStatus Open(const shared_ptr[RandomAccessFile]& file,
+                     unique_ptr[CFeatherReader]* out)
+
+        c_string GetDescription()
+        c_bool HasDescription()
+
+        int64_t num_rows()
+        int64_t num_columns()
+
+        shared_ptr[CSchema] schema()
+
+        CStatus GetColumn(int i, shared_ptr[CColumn]* out)
+        c_string GetColumnName(int i)
diff --git a/python/pyarrow/includes/libarrow_io.pxd b/python/pyarrow/includes/libarrow_io.pxd
deleted file mode 100644
index 5992c737df512..0000000000000
--- a/python/pyarrow/includes/libarrow_io.pxd
+++ /dev/null
@@ -1,171 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# distutils: language = c++
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport *
-
-cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
-    enum FileMode" arrow::io::FileMode::type":
-        FileMode_READ" arrow::io::FileMode::READ"
-        FileMode_WRITE" arrow::io::FileMode::WRITE"
-        FileMode_READWRITE" arrow::io::FileMode::READWRITE"
-
-    enum ObjectType" arrow::io::ObjectType::type":
-        ObjectType_FILE" arrow::io::ObjectType::FILE"
-        ObjectType_DIRECTORY" arrow::io::ObjectType::DIRECTORY"
-
-    cdef cppclass FileInterface:
-        CStatus Close()
-        CStatus Tell(int64_t* position)
-        FileMode mode()
-
-    cdef cppclass Readable:
-        CStatus ReadB" Read"(int64_t nbytes, shared_ptr[CBuffer]* out)
-        CStatus Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out)
-
-    cdef cppclass Seekable:
-        CStatus Seek(int64_t position)
-
-    cdef cppclass Writeable:
-        CStatus Write(const uint8_t* data, int64_t nbytes)
-
-    cdef cppclass OutputStream(FileInterface, Writeable):
-        pass
-
-    cdef cppclass InputStream(FileInterface, Readable):
-        pass
-
-    cdef cppclass RandomAccessFile(InputStream, Seekable):
-        CStatus GetSize(int64_t* size)
-
-        CStatus ReadAt(int64_t position, int64_t nbytes,
-                       int64_t* bytes_read, uint8_t* buffer)
-        CStatus ReadAt(int64_t position, int64_t nbytes,
-                       int64_t* bytes_read, shared_ptr[CBuffer]* out)
-
-    cdef cppclass WriteableFileInterface(OutputStream, Seekable):
-        CStatus WriteAt(int64_t position, const uint8_t* data,
-                        int64_t nbytes)
-
-    cdef cppclass ReadWriteFileInterface(RandomAccessFile,
-                                         WriteableFileInterface):
-        pass
-
-
-cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
-
-
-    cdef cppclass FileOutputStream(OutputStream):
-        @staticmethod
-        CStatus Open(const c_string& path, shared_ptr[FileOutputStream]* file)
-
-        int file_descriptor()
-
-    cdef cppclass ReadableFile(RandomAccessFile):
-        @staticmethod
-        CStatus Open(const c_string& path, shared_ptr[ReadableFile]* file)
-
-        @staticmethod
-        CStatus Open(const c_string& path, CMemoryPool* memory_pool,
-                     shared_ptr[ReadableFile]* file)
-
-        int file_descriptor()
-
-    cdef cppclass CMemoryMappedFile" arrow::io::MemoryMappedFile"\
-        (ReadWriteFileInterface):
-        @staticmethod
-        CStatus Open(const c_string& path, FileMode mode,
-                     shared_ptr[CMemoryMappedFile]* file)
-
-        int file_descriptor()
-
-
-cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
-    CStatus HaveLibHdfs()
-    CStatus HaveLibHdfs3()
-
-    enum HdfsDriver" arrow::io::HdfsDriver":
-        HdfsDriver_LIBHDFS" arrow::io::HdfsDriver::LIBHDFS"
-        HdfsDriver_LIBHDFS3" arrow::io::HdfsDriver::LIBHDFS3"
-
-    cdef cppclass HdfsConnectionConfig:
-        c_string host
-        int port
-        c_string user
-        c_string kerb_ticket
-        HdfsDriver driver
-
-    cdef cppclass HdfsPathInfo:
-        ObjectType kind;
-        c_string name
-        c_string owner
-        c_string group
-        int32_t last_modified_time
-        int32_t last_access_time
-        int64_t size
-        int16_t replication
-        int64_t block_size
-        int16_t permissions
-
-    cdef cppclass HdfsReadableFile(RandomAccessFile):
-        pass
-
-    cdef cppclass HdfsOutputStream(OutputStream):
-        pass
-
-    cdef cppclass CHdfsClient" arrow::io::HdfsClient":
-        @staticmethod
-        CStatus Connect(const HdfsConnectionConfig* config,
-                        shared_ptr[CHdfsClient]* client)
-
-        CStatus CreateDirectory(const c_string& path)
-
-        CStatus Delete(const c_string& path, c_bool recursive)
-
-        CStatus Disconnect()
-
-        c_bool Exists(const c_string& path)
-
-        CStatus GetCapacity(int64_t* nbytes)
-        CStatus GetUsed(int64_t* nbytes)
-
-        CStatus ListDirectory(const c_string& path,
-                              vector[HdfsPathInfo]* listing)
-
-        CStatus GetPathInfo(const c_string& path, HdfsPathInfo* info)
-
-        CStatus Rename(const c_string& src, const c_string& dst)
-
-        CStatus OpenReadable(const c_string& path,
-                             shared_ptr[HdfsReadableFile]* handle)
-
-        CStatus OpenWriteable(const c_string& path, c_bool append,
-                              int32_t buffer_size, int16_t replication,
-                              int64_t default_block_size,
-                              shared_ptr[HdfsOutputStream]* handle)
-
-
-cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
-    cdef cppclass CBufferReader" arrow::io::BufferReader"\
-        (RandomAccessFile):
-        CBufferReader(const shared_ptr[CBuffer]& buffer)
-        CBufferReader(const uint8_t* data, int64_t nbytes)
-
-    cdef cppclass BufferOutputStream(OutputStream):
-        BufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
diff --git a/python/pyarrow/includes/libarrow_ipc.pxd b/python/pyarrow/includes/libarrow_ipc.pxd
deleted file mode 100644
index 59fd90bdac7a8..0000000000000
--- a/python/pyarrow/includes/libarrow_ipc.pxd
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# distutils: language = c++
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CColumn, CSchema, CRecordBatch)
-from pyarrow.includes.libarrow_io cimport (InputStream, OutputStream,
-                                           RandomAccessFile)
-
-
-cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
-
-    cdef cppclass CStreamWriter " arrow::ipc::StreamWriter":
-        @staticmethod
-        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
-                     shared_ptr[CStreamWriter]* out)
-
-        CStatus Close()
-        CStatus WriteRecordBatch(const CRecordBatch& batch)
-
-    cdef cppclass CStreamReader " arrow::ipc::StreamReader":
-
-        @staticmethod
-        CStatus Open(const shared_ptr[InputStream]& stream,
-                     shared_ptr[CStreamReader]* out)
-
-        shared_ptr[CSchema] schema()
-
-        CStatus GetNextRecordBatch(shared_ptr[CRecordBatch]* batch)
-
-    cdef cppclass CFileWriter " arrow::ipc::FileWriter"(CStreamWriter):
-        @staticmethod
-        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
-                     shared_ptr[CFileWriter]* out)
-
-    cdef cppclass CFileReader " arrow::ipc::FileReader":
-
-        @staticmethod
-        CStatus Open(const shared_ptr[RandomAccessFile]& file,
-                     shared_ptr[CFileReader]* out)
-
-        @staticmethod
-        CStatus Open2" Open"(const shared_ptr[RandomAccessFile]& file,
-                     int64_t footer_offset, shared_ptr[CFileReader]* out)
-
-        shared_ptr[CSchema] schema()
-
-        int num_record_batches()
-
-        CStatus GetRecordBatch(int i, shared_ptr[CRecordBatch]* batch)
-
-cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
-
-    cdef cppclass CFeatherWriter" arrow::ipc::feather::TableWriter":
-        @staticmethod
-        CStatus Open(const shared_ptr[OutputStream]& stream,
-                     unique_ptr[CFeatherWriter]* out)
-
-        void SetDescription(const c_string& desc)
-        void SetNumRows(int64_t num_rows)
-
-        CStatus Append(const c_string& name, const CArray& values)
-        CStatus Finalize()
-
-    cdef cppclass CFeatherReader" arrow::ipc::feather::TableReader":
-        @staticmethod
-        CStatus Open(const shared_ptr[RandomAccessFile]& file,
-                     unique_ptr[CFeatherReader]* out)
-
-        c_string GetDescription()
-        c_bool HasDescription()
-
-        int64_t num_rows()
-        int64_t num_columns()
-
-        shared_ptr[CSchema] schema()
-
-        CStatus GetColumn(int i, shared_ptr[CColumn]* out)
-        c_string GetColumnName(int i)
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index 9b64435e48d7f..c40df3db8a9c5 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -20,9 +20,9 @@
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CDataType,
                                         CTable, CTensor, CStatus, Type,
-                                        CMemoryPool, TimeUnit)
-
-cimport pyarrow.includes.libarrow_io as arrow_io
+                                        CMemoryPool, TimeUnit,
+                                        RandomAccessFile, OutputStream,
+                                        CBufferReader)
 
 
 cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
@@ -65,11 +65,11 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     cdef cppclass PyBuffer(CBuffer):
         PyBuffer(object o)
 
-    cdef cppclass PyReadableFile(arrow_io.RandomAccessFile):
+    cdef cppclass PyReadableFile(RandomAccessFile):
         PyReadableFile(object fo)
 
-    cdef cppclass PyOutputStream(arrow_io.OutputStream):
+    cdef cppclass PyOutputStream(OutputStream):
         PyOutputStream(object fo)
 
-    cdef cppclass PyBytesReader(arrow_io.CBufferReader):
+    cdef cppclass PyBytesReader(CBufferReader):
         PyBytesReader(object fo)
diff --git a/python/pyarrow/io.pxd b/python/pyarrow/io.pxd
index cffd29ab39111..0c37a09add574 100644
--- a/python/pyarrow/io.pxd
+++ b/python/pyarrow/io.pxd
@@ -19,8 +19,7 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.libarrow_io cimport (RandomAccessFile,
-                                           OutputStream)
+
 
 cdef class Buffer:
     cdef:
@@ -30,6 +29,7 @@ cdef class Buffer:
 
     cdef init(self, const shared_ptr[CBuffer]& buffer)
 
+
 cdef class NativeFile:
     cdef:
         shared_ptr[RandomAccessFile] rd_file
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 608b20d896ae3..98b5a62b372a2 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -27,12 +27,10 @@ from cython.operator cimport dereference as deref
 from libc.stdlib cimport malloc, free
 
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.libarrow_io cimport *
-from pyarrow.includes.libarrow_ipc cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
 
 from pyarrow.compat import frombytes, tobytes, encode_file_path
-from pyarrow.array cimport Array
+from pyarrow.array cimport Array, Tensor, box_tensor
 from pyarrow.error cimport check_status
 from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
 from pyarrow.schema cimport Schema
@@ -340,7 +338,32 @@ cdef class MemoryMappedFile(NativeFile):
     cdef:
         object path
 
-    def __cinit__(self, path, mode='r'):
+    def __cinit__(self):
+        self.is_open = False
+        self.is_readable = 0
+        self.is_writeable = 0
+
+    @staticmethod
+    def create(path, size):
+        cdef:
+            shared_ptr[CMemoryMappedFile] handle
+            c_string c_path = encode_file_path(path)
+            int64_t c_size = size
+
+        with nogil:
+            check_status(CMemoryMappedFile.Create(c_path, c_size, &handle))
+
+        cdef MemoryMappedFile result = MemoryMappedFile()
+        result.path = path
+        result.is_readable = 1
+        result.is_writeable = 1
+        result.wr_file = <shared_ptr[OutputStream]> handle
+        result.rd_file = <shared_ptr[RandomAccessFile]> handle
+        result.is_open = True
+
+        return result
+
+    def open(self, path, mode='r'):
         self.path = path
 
         cdef:
@@ -348,8 +371,6 @@ cdef class MemoryMappedFile(NativeFile):
             shared_ptr[CMemoryMappedFile] handle
             c_string c_path = encode_file_path(path)
 
-        self.is_readable = self.is_writeable = 0
-
         if mode in ('r', 'rb'):
             c_mode = FileMode_READ
             self.is_readable = 1
@@ -370,6 +391,41 @@ cdef class MemoryMappedFile(NativeFile):
         self.is_open = True
 
 
+def memory_map(path, mode='r'):
+    """
+    Open memory map at file path. Size of the memory map cannot change
+
+    Parameters
+    ----------
+    path : string
+    mode : {'r', 'w'}, default 'r'
+
+    Returns
+    -------
+    mmap : MemoryMappedFile
+    """
+    cdef MemoryMappedFile mmap = MemoryMappedFile()
+    mmap.open(path, mode)
+    return mmap
+
+
+def create_memory_map(path, size):
+    """
+    Create memory map at indicated path of the given size, return open
+    writeable file object
+
+    Parameters
+    ----------
+    path : string
+    size : int
+
+    Returns
+    -------
+    mmap : MemoryMappedFile
+    """
+    return MemoryMappedFile.create(path, size)
+
+
 cdef class OSFile(NativeFile):
     """
     Supports 'r', 'w' modes
@@ -542,7 +598,7 @@ cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
     cdef NativeFile nf
 
     if isinstance(source, six.string_types):
-        source = MemoryMappedFile(source, mode='r')
+        source = memory_map(source, mode='r')
     elif isinstance(source, Buffer):
         source = BufferReader(source)
     elif not isinstance(source, NativeFile) and hasattr(source, 'read'):
@@ -1144,3 +1200,57 @@ cdef class FeatherReader:
         cdef Column col = Column()
         col.init(sp_column)
         return col
+
+
+def write_tensor(Tensor tensor, NativeFile dest):
+    """
+    Write pyarrow.Tensor to pyarrow.NativeFile object its current position
+
+    Parameters
+    ----------
+    tensor : pyarrow.Tensor
+    dest : pyarrow.NativeFile
+
+    Returns
+    -------
+    bytes_written : int
+        Total number of bytes written to the file
+    """
+    cdef:
+        int32_t metadata_length
+        int64_t body_length
+
+    dest._assert_writeable()
+
+    with nogil:
+        check_status(
+            WriteTensor(deref(tensor.tp), dest.wr_file.get(),
+                        &metadata_length, &body_length))
+
+    return metadata_length + body_length
+
+
+def read_tensor(NativeFile source):
+    """
+    Read pyarrow.Tensor from pyarrow.NativeFile object from current
+    position. If the file source supports zero copy (e.g. a memory map), then
+    this operation does not allocate any memory
+
+    Parameters
+    ----------
+    source : pyarrow.NativeFile
+
+    Returns
+    -------
+    tensor : Tensor
+    """
+    cdef:
+        shared_ptr[CTensor] sp_tensor
+
+    source._assert_writeable()
+
+    cdef int64_t offset = source.tell()
+    with nogil:
+        check_status(ReadTensor(offset, source.rd_file.get(), &sp_tensor))
+
+    return box_tensor(sp_tensor)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 15c5e6b924385..beb6113849ac3 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -246,10 +246,10 @@ def teardown():
     return path, data
 
 
-def _check_native_file_reader(KLASS, sample_data):
+def _check_native_file_reader(FACTORY, sample_data):
     path, data = sample_data
 
-    f = KLASS(path, mode='r')
+    f = FACTORY(path, mode='r')
 
     assert f.read(10) == data[:10]
     assert f.read(0) == b''
@@ -269,14 +269,14 @@ def _check_native_file_reader(KLASS, sample_data):
 
 
 def test_memory_map_reader(sample_disk_data):
-    _check_native_file_reader(io.MemoryMappedFile, sample_disk_data)
+    _check_native_file_reader(pa.memory_map, sample_disk_data)
 
 
 def test_memory_map_retain_buffer_reference(sample_disk_data):
     path, data = sample_disk_data
 
     cases = []
-    with io.MemoryMappedFile(path, 'rb') as f:
+    with pa.memory_map(path, 'rb') as f:
         cases.append((f.read_buffer(100), data[:100]))
         cases.append((f.read_buffer(100), data[100:200]))
         cases.append((f.read_buffer(100), data[200:300]))
@@ -309,7 +309,7 @@ def test_memory_map_writer():
         with open(path, 'wb') as f:
             f.write(data)
 
-        f = io.MemoryMappedFile(path, mode='r+w')
+        f = pa.memory_map(path, mode='r+w')
 
         f.seek(10)
         f.write('peekaboo')
@@ -318,7 +318,7 @@ def test_memory_map_writer():
         f.seek(10)
         assert f.read(8) == b'peekaboo'
 
-        f2 = io.MemoryMappedFile(path, mode='r+w')
+        f2 = pa.memory_map(path, mode='r+w')
 
         f2.seek(10)
         f2.write(b'booapeak')
@@ -328,10 +328,10 @@ def test_memory_map_writer():
         assert f.read(8) == b'booapeak'
 
         # Does not truncate file
-        f3 = io.MemoryMappedFile(path, mode='w')
+        f3 = pa.memory_map(path, mode='w')
         f3.write('foo')
 
-        with io.MemoryMappedFile(path) as f4:
+        with pa.memory_map(path) as f4:
             assert f4.size() == SIZE
 
         with pytest.raises(IOError):
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
new file mode 100644
index 0000000000000..5327f1a74a33e
--- /dev/null
+++ b/python/pyarrow/tests/test_tensor.py
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import pytest
+
+import numpy as np
+import pyarrow as pa
+
+
+def test_tensor_attrs():
+    data = np.random.randn(10, 4)
+
+    tensor = pa.Tensor.from_numpy(data)
+
+    assert tensor.ndim == 2
+    assert tensor.size == 40
+    assert tensor.shape == list(data.shape)
+    assert tensor.strides == list(data.strides)
+
+    assert tensor.is_contiguous
+    assert tensor.is_mutable
+
+    # not writeable
+    data2 = data.copy()
+    data2.flags.writeable = False
+    tensor = pa.Tensor.from_numpy(data2)
+    assert not tensor.is_mutable
+
+
+@pytest.mark.parametrize('dtype_str,arrow_type', [
+    ('i1', pa.int8()),
+    ('i2', pa.int16()),
+    ('i4', pa.int32()),
+    ('i8', pa.int64()),
+    ('u1', pa.uint8()),
+    ('u2', pa.uint16()),
+    ('u4', pa.uint32()),
+    ('u8', pa.uint64()),
+    ('f2', pa.float16()),
+    ('f4', pa.float32()),
+    ('f8', pa.float64())
+])
+def test_tensor_numpy_roundtrip(dtype_str, arrow_type):
+    dtype = np.dtype(dtype_str)
+    data = (100 * np.random.randn(10, 4)).astype(dtype)
+
+    tensor = pa.Tensor.from_numpy(data)
+    assert tensor.type == arrow_type
+
+    repr(tensor)
+
+    result = tensor.to_numpy()
+    assert (data == result).all()
+
+
+def _try_delete(path):
+    try:
+        os.remove(path)
+    except os.error:
+        pass
+
+
+def test_tensor_ipc_roundtrip():
+    data = np.random.randn(10, 4)
+    tensor = pa.Tensor.from_numpy(data)
+
+    path = 'pyarrow-tensor-ipc-roundtrip'
+    try:
+        mmap = pa.create_memory_map(path, 1024)
+
+        pa.write_tensor(tensor, mmap)
+
+        mmap.seek(0)
+        result = pa.read_tensor(mmap)
+
+        assert result.equals(tensor)
+    finally:
+        _try_delete(path)

From f05b7c62cf6151a2a03292508628f8f1a8e7a1aa Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 3 Apr 2017 18:05:39 -0400
Subject: [PATCH 0461/1644] ARROW-443: [Python] Support ingest of strided NumPy
 arrays from pandas

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #482 from wesm/ARROW-443 and squashes the following commits:

d9b36c0 [Wes McKinney] Run commented out test cases, fix issue
8f0ff38 [Wes McKinney] cpplint
88eef1a [Wes McKinney] Support strided mask argument in some object conversions
22d4489 [Wes McKinney] First cut at strided NumPy import
---
 cpp/src/arrow/python/config.cc              |   1 +
 cpp/src/arrow/python/numpy-internal.h       |  66 +++++
 cpp/src/arrow/python/pandas_convert.cc      | 252 +++++++++++++-------
 python/pyarrow/array.pyx                    |   4 +-
 python/pyarrow/tests/test_convert_pandas.py |  59 ++++-
 5 files changed, 290 insertions(+), 92 deletions(-)
 create mode 100644 cpp/src/arrow/python/numpy-internal.h

diff --git a/cpp/src/arrow/python/config.cc b/cpp/src/arrow/python/config.cc
index 2abc4dda6ee17..c2a69168bb01e 100644
--- a/cpp/src/arrow/python/config.cc
+++ b/cpp/src/arrow/python/config.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include <Python.h>
+#include <datetime.h>
 
 #include "arrow/python/config.h"
 
diff --git a/cpp/src/arrow/python/numpy-internal.h b/cpp/src/arrow/python/numpy-internal.h
new file mode 100644
index 0000000000000..fcc6a58f2a347
--- /dev/null
+++ b/cpp/src/arrow/python/numpy-internal.h
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Internal utilities for dealing with NumPy
+
+#ifndef ARROW_PYTHON_NUMPY_INTERNAL_H
+#define ARROW_PYTHON_NUMPY_INTERNAL_H
+
+#include <Python.h>
+
+#include <cstdint>
+
+#include "arrow/python/numpy_convert.h"
+#include "arrow/python/numpy_interop.h"
+
+namespace arrow {
+namespace py {
+
+/// Indexing convenience for interacting with strided 1-dim ndarray objects
+template <typename T>
+class Ndarray1DIndexer {
+ public:
+  typedef int64_t size_type;
+
+  Ndarray1DIndexer() : arr_(nullptr), data_(nullptr) {}
+
+  explicit Ndarray1DIndexer(PyArrayObject* arr) : Ndarray1DIndexer() { Init(arr); }
+
+  void Init(PyArrayObject* arr) {
+    arr_ = arr;
+    DCHECK_EQ(1, PyArray_NDIM(arr)) << "Only works with 1-dimensional arrays";
+    Py_INCREF(arr);
+    data_ = reinterpret_cast<T*>(PyArray_DATA(arr));
+    stride_ = PyArray_STRIDES(arr)[0] / sizeof(T);
+  }
+
+  ~Ndarray1DIndexer() { Py_XDECREF(arr_); }
+
+  int64_t size() const { return PyArray_SIZE(arr_); }
+
+  T& operator[](size_type index) { return *(data_ + index * stride_); }
+
+ private:
+  PyArrayObject* arr_;
+  T* data_;
+  int64_t stride_;
+};
+
+}  // namespace py
+}  // namespace arrow
+
+#endif  // ARROW_PYTHON_NUMPY_INTERNAL_H
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 01019e5669f2d..9577892a55b76 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -47,6 +47,7 @@
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
 #include "arrow/python/config.h"
+#include "arrow/python/numpy-internal.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/type_traits.h"
 #include "arrow/python/util/datetime.h"
@@ -70,15 +71,16 @@ static inline bool PyObject_is_string(const PyObject* obj) {
 }
 
 template <int TYPE>
-static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap) {
+static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
   typedef npy_traits<TYPE> traits;
   typedef typename traits::value_type T;
 
   int64_t null_count = 0;
-  const T* values = reinterpret_cast<const T*>(data);
+
+  Ndarray1DIndexer<T> values(arr);
 
   // TODO(wesm): striding
-  for (int i = 0; i < length; ++i) {
+  for (int i = 0; i < values.size(); ++i) {
     if (traits::isnull(values[i])) {
       ++null_count;
     } else {
@@ -92,8 +94,8 @@ static int64_t ValuesToBitmap(const void* data, int64_t length, uint8_t* bitmap)
 // Returns null count
 static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
   int64_t null_count = 0;
-  const uint8_t* mask_values = static_cast<const uint8_t*>(PyArray_DATA(mask));
-  // TODO(wesm): strided null mask
+
+  Ndarray1DIndexer<uint8_t> mask_values(mask);
   for (int i = 0; i < length; ++i) {
     if (mask_values[i]) {
       ++null_count;
@@ -138,13 +140,24 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
   return Status::OK();
 }
 
-Status AppendObjectStrings(int64_t objects_length, StringBuilder* builder,
-    PyObject** objects, bool* have_bytes) {
+static Status AppendObjectStrings(
+    PyArrayObject* arr, PyArrayObject* mask, StringBuilder* builder, bool* have_bytes) {
   PyObject* obj;
 
-  for (int64_t i = 0; i < objects_length; ++i) {
+  Ndarray1DIndexer<PyObject*> objects(arr);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask != nullptr) {
+    mask_values.Init(mask);
+    have_mask = true;
+  }
+
+  for (int64_t i = 0; i < objects.size(); ++i) {
     obj = objects[i];
-    if (PyUnicode_Check(obj)) {
+    if ((have_mask && mask_values[i]) || PyObject_is_null(obj)) {
+      RETURN_NOT_OK(builder->AppendNull());
+    } else if (PyUnicode_Check(obj)) {
       obj = PyUnicode_AsUTF8String(obj);
       if (obj == NULL) {
         PyErr_Clear();
@@ -158,8 +171,6 @@ Status AppendObjectStrings(int64_t objects_length, StringBuilder* builder,
       *have_bytes = true;
       const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
       RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
-    } else if (PyObject_is_null(obj)) {
-      RETURN_NOT_OK(builder->AppendNull());
     } else {
       return InvalidConversion(obj, "string or bytes");
     }
@@ -168,13 +179,24 @@ Status AppendObjectStrings(int64_t objects_length, StringBuilder* builder,
   return Status::OK();
 }
 
-static Status AppendObjectFixedWidthBytes(int64_t objects_length, int byte_width,
-    FixedSizeBinaryBuilder* builder, PyObject** objects) {
+static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mask,
+    int byte_width, FixedSizeBinaryBuilder* builder) {
   PyObject* obj;
 
-  for (int64_t i = 0; i < objects_length; ++i) {
+  Ndarray1DIndexer<PyObject*> objects(arr);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask != nullptr) {
+    mask_values.Init(mask);
+    have_mask = true;
+  }
+
+  for (int64_t i = 0; i < objects.size(); ++i) {
     obj = objects[i];
-    if (PyUnicode_Check(obj)) {
+    if ((have_mask && mask_values[i]) || PyObject_is_null(obj)) {
+      RETURN_NOT_OK(builder->AppendNull());
+    } else if (PyUnicode_Check(obj)) {
       obj = PyUnicode_AsUTF8String(obj);
       if (obj == NULL) {
         PyErr_Clear();
@@ -190,8 +212,6 @@ static Status AppendObjectFixedWidthBytes(int64_t objects_length, int byte_width
       RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
       RETURN_NOT_OK(
           builder->Append(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj))));
-    } else if (PyObject_is_null(obj)) {
-      RETURN_NOT_OK(builder->AppendNull());
     } else {
       return InvalidConversion(obj, "string or bytes");
     }
@@ -299,8 +319,7 @@ class PandasConverter : public TypeVisitor {
     } else if (traits::supports_nulls) {
       // TODO(wesm): this presumes the NumPy C type and arrow C type are the
       // same
-      null_count = ValuesToBitmap<traits::npy_type>(
-          PyArray_DATA(arr_), length_, null_bitmap_data_);
+      null_count = ValuesToBitmap<traits::npy_type>(arr_, null_bitmap_data_);
     }
 
     std::vector<FieldMetadata> fields(1);
@@ -329,36 +348,33 @@ class PandasConverter : public TypeVisitor {
 
 #undef VISIT_NATIVE
 
-  Status Convert(std::shared_ptr<Array>* out) {
+  Status Convert() {
     if (PyArray_NDIM(arr_) != 1) {
       return Status::Invalid("only handle 1-dimensional arrays");
     }
-    // TODO(wesm): strided arrays
-    if (is_strided()) { return Status::Invalid("no support for strided data yet"); }
 
     if (type_ == nullptr) { return Status::Invalid("Must pass data type"); }
 
     // Visit the type to perform conversion
     RETURN_NOT_OK(type_->Accept(this));
 
-    *out = out_;
     return Status::OK();
   }
 
+  std::shared_ptr<Array> result() const { return out_; }
+
   // ----------------------------------------------------------------------
   // Conversion logic for various object dtype arrays
 
   template <int ITEM_TYPE, typename ArrowType>
-  Status ConvertTypedLists(
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
+  Status ConvertTypedLists(const std::shared_ptr<DataType>& type);
 
-  Status ConvertObjectStrings(std::shared_ptr<Array>* out);
-  Status ConvertObjectFixedWidthBytes(
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
-  Status ConvertBooleans(std::shared_ptr<Array>* out);
-  Status ConvertDates(std::shared_ptr<Array>* out);
-  Status ConvertLists(const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
-  Status ConvertObjects(std::shared_ptr<Array>* out);
+  Status ConvertObjectStrings();
+  Status ConvertObjectFixedWidthBytes(const std::shared_ptr<DataType>& type);
+  Status ConvertBooleans();
+  Status ConvertDates();
+  Status ConvertLists(const std::shared_ptr<DataType>& type);
+  Status ConvertObjects();
 
  protected:
   MemoryPool* pool_;
@@ -374,9 +390,31 @@ class PandasConverter : public TypeVisitor {
   uint8_t* null_bitmap_data_;
 };
 
+template <typename T>
+void CopyStrided(T* input_data, int64_t length, int64_t stride, T* output_data) {
+  // Passing input_data as non-const is a concession to PyObject*
+  int64_t j = 0;
+  for (int64_t i = 0; i < length; ++i) {
+    output_data[i] = input_data[j];
+    j += stride;
+  }
+}
+
+template <>
+void CopyStrided<PyObject*>(
+    PyObject** input_data, int64_t length, int64_t stride, PyObject** output_data) {
+  int64_t j = 0;
+  for (int64_t i = 0; i < length; ++i) {
+    output_data[i] = input_data[j];
+    if (output_data[i] != nullptr) { Py_INCREF(output_data[i]); }
+    j += stride;
+  }
+}
+
 template <typename ArrowType>
 inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   using traits = arrow_traits<ArrowType::type_id>;
+  using T = typename traits::T;
 
   // Handle LONGLONG->INT64 and other fun things
   int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
@@ -385,7 +423,20 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
     return Status::NotImplemented("NumPy type casts not yet implemented");
   }
 
-  *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
+  if (is_strided()) {
+    // Strided, must copy into new contiguous memory
+    const int64_t stride = PyArray_STRIDES(arr_)[0];
+    const int64_t stride_elements = stride / sizeof(T);
+
+    auto new_buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length_));
+    CopyStrided(reinterpret_cast<T*>(PyArray_DATA(arr_)), length_, stride_elements,
+        reinterpret_cast<T*>(new_buffer->mutable_data()));
+    *data = new_buffer;
+  } else {
+    // Can zero-copy
+    *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
+  }
   return Status::OK();
 }
 
@@ -395,7 +446,7 @@ inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>*
   auto buffer = std::make_shared<PoolBuffer>(pool_);
   RETURN_NOT_OK(buffer->Resize(nbytes));
 
-  const uint8_t* values = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
+  Ndarray1DIndexer<uint8_t> values(arr_);
 
   uint8_t* bitmap = buffer->mutable_data();
 
@@ -434,13 +485,22 @@ Status InvalidConversion(PyObject* obj, const std::string& expected_type_name) {
   return Status::TypeError(ss.str());
 }
 
-Status PandasConverter::ConvertDates(std::shared_ptr<Array>* out) {
+Status PandasConverter::ConvertDates() {
   PyAcquireGIL lock;
 
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+
+  if (mask_ != nullptr) {
+    return Status::NotImplemented("mask not supported in object conversions yet");
+  }
+
   Date64Builder date_builder(pool_);
   RETURN_NOT_OK(date_builder.Resize(length_));
 
+  /// We have to run this in this compilation unit, since we cannot use the
+  /// datetime API otherwise
+  PyDateTime_IMPORT;
+
   Status s;
   PyObject* obj;
   for (int64_t i = 0; i < length_; ++i) {
@@ -454,50 +514,57 @@ Status PandasConverter::ConvertDates(std::shared_ptr<Array>* out) {
       return InvalidConversion(obj, "date");
     }
   }
-  return date_builder.Finish(out);
+  return date_builder.Finish(&out_);
 }
 
-Status PandasConverter::ConvertObjectStrings(std::shared_ptr<Array>* out) {
+Status PandasConverter::ConvertObjectStrings() {
   PyAcquireGIL lock;
 
   // The output type at this point is inconclusive because there may be bytes
   // and unicode mixed in the object array
 
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
   StringBuilder builder(pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
   Status s;
   bool have_bytes = false;
-  RETURN_NOT_OK(AppendObjectStrings(length_, &builder, objects, &have_bytes));
-  RETURN_NOT_OK(builder.Finish(out));
+  RETURN_NOT_OK(AppendObjectStrings(arr_, mask_, &builder, &have_bytes));
+  RETURN_NOT_OK(builder.Finish(&out_));
 
   if (have_bytes) {
-    const auto& arr = static_cast<const StringArray&>(*out->get());
-    *out = std::make_shared<BinaryArray>(arr.length(), arr.value_offsets(), arr.data(),
+    const auto& arr = static_cast<const StringArray&>(*out_);
+    out_ = std::make_shared<BinaryArray>(arr.length(), arr.value_offsets(), arr.data(),
         arr.null_bitmap(), arr.null_count());
   }
   return Status::OK();
 }
 
 Status PandasConverter::ConvertObjectFixedWidthBytes(
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+    const std::shared_ptr<DataType>& type) {
   PyAcquireGIL lock;
 
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+
+  int32_t value_size = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
+
   FixedSizeBinaryBuilder builder(pool_, type);
   RETURN_NOT_OK(builder.Resize(length_));
-  RETURN_NOT_OK(AppendObjectFixedWidthBytes(length_,
-      std::dynamic_pointer_cast<FixedSizeBinaryType>(builder.type())->byte_width(),
-      &builder, objects));
-  RETURN_NOT_OK(builder.Finish(out));
+  RETURN_NOT_OK(AppendObjectFixedWidthBytes(arr_, mask_, value_size, &builder));
+  RETURN_NOT_OK(builder.Finish(&out_));
   return Status::OK();
 }
 
-Status PandasConverter::ConvertBooleans(std::shared_ptr<Array>* out) {
+Status PandasConverter::ConvertBooleans() {
   PyAcquireGIL lock;
 
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask_ != nullptr) {
+    mask_values.Init(mask_);
+    have_mask = true;
+  }
 
   int64_t nbytes = BitUtil::BytesForBits(length_);
   auto data = std::make_shared<PoolBuffer>(pool_);
@@ -509,24 +576,24 @@ Status PandasConverter::ConvertBooleans(std::shared_ptr<Array>* out) {
   PyObject* obj;
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
-    if (obj == Py_True) {
+    if ((have_mask && mask_values[i]) || PyObject_is_null(obj)) {
+      ++null_count;
+    } else if (obj == Py_True) {
       BitUtil::SetBit(bitmap, i);
       BitUtil::SetBit(null_bitmap_data_, i);
     } else if (obj == Py_False) {
       BitUtil::SetBit(null_bitmap_data_, i);
-    } else if (PyObject_is_null(obj)) {
-      ++null_count;
     } else {
       return InvalidConversion(obj, "bool");
     }
   }
 
-  *out = std::make_shared<BooleanArray>(length_, data, null_bitmap_, null_count);
+  out_ = std::make_shared<BooleanArray>(length_, data, null_bitmap_, null_count);
 
   return Status::OK();
 }
 
-Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
+Status PandasConverter::ConvertObjects() {
   // Python object arrays are annoying, since we could have one of:
   //
   // * Strings
@@ -538,31 +605,27 @@ Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
 
   RETURN_NOT_OK(InitNullBitmap());
 
-  // TODO: mask not supported here
-  if (mask_ != nullptr) {
-    return Status::NotImplemented("mask not supported in object conversions yet");
-  }
+  Ndarray1DIndexer<PyObject*> objects;
 
-  const PyObject** objects;
   {
     PyAcquireGIL lock;
-    objects = reinterpret_cast<const PyObject**>(PyArray_DATA(arr_));
+    objects.Init(arr_);
     PyDateTime_IMPORT;
   }
 
   if (type_) {
     switch (type_->type) {
       case Type::STRING:
-        return ConvertObjectStrings(out);
+        return ConvertObjectStrings();
       case Type::FIXED_SIZE_BINARY:
-        return ConvertObjectFixedWidthBytes(type_, out);
+        return ConvertObjectFixedWidthBytes(type_);
       case Type::BOOL:
-        return ConvertBooleans(out);
+        return ConvertBooleans();
       case Type::DATE64:
-        return ConvertDates(out);
+        return ConvertDates();
       case Type::LIST: {
         const auto& list_field = static_cast<const ListType&>(*type_);
-        return ConvertLists(list_field.value_field()->type, out);
+        return ConvertLists(list_field.value_field()->type);
       }
       default:
         return Status::TypeError("No known conversion to Arrow type");
@@ -572,11 +635,11 @@ Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
       if (PyObject_is_null(objects[i])) {
         continue;
       } else if (PyObject_is_string(objects[i])) {
-        return ConvertObjectStrings(out);
+        return ConvertObjectStrings();
       } else if (PyBool_Check(objects[i])) {
-        return ConvertBooleans(out);
+        return ConvertBooleans();
       } else if (PyDate_CheckExact(objects[i])) {
-        return ConvertDates(out);
+        return ConvertDates();
       } else {
         return InvalidConversion(
             const_cast<PyObject*>(objects[i]), "string, bool, or date");
@@ -588,14 +651,22 @@ Status PandasConverter::ConvertObjects(std::shared_ptr<Array>* out) {
 }
 
 template <int ITEM_TYPE, typename ArrowType>
-inline Status PandasConverter::ConvertTypedLists(
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>& type) {
   typedef npy_traits<ITEM_TYPE> traits;
   typedef typename traits::value_type T;
   typedef typename traits::BuilderClass BuilderT;
 
   PyAcquireGIL lock;
 
+  // TODO: mask not supported here
+  if (mask_ != nullptr) {
+    return Status::NotImplemented("mask not supported in object conversions yet");
+  }
+
+  if (is_strided()) {
+    return Status::NotImplemented("strided arrays not implemented for lists");
+  }
+
   auto value_builder = std::make_shared<BuilderT>(pool_, type);
   ListBuilder list_builder(pool_, value_builder);
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
@@ -637,16 +708,25 @@ inline Status PandasConverter::ConvertTypedLists(
       return Status::TypeError("Unsupported Python type for list items");
     }
   }
-  return list_builder.Finish(out);
+  return list_builder.Finish(&out_);
 }
 
 template <>
 inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+    const std::shared_ptr<DataType>& type) {
   PyAcquireGIL lock;
   // TODO: If there are bytes involed, convert to Binary representation
   bool have_bytes = false;
 
+  // TODO: mask not supported here
+  if (mask_ != nullptr) {
+    return Status::NotImplemented("mask not supported in object conversions yet");
+  }
+
+  if (is_strided()) {
+    return Status::NotImplemented("strided arrays not implemented for lists");
+  }
+
   auto value_builder = std::make_shared<StringBuilder>(pool_);
   ListBuilder list_builder(pool_, value_builder);
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
@@ -660,9 +740,8 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
       // TODO(uwe): Support more complex numpy array structures
       RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
 
-      int64_t size = static_cast<int64_t>(PyArray_DIM(numpy_array, 0));
-      auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
-      RETURN_NOT_OK(AppendObjectStrings(size, value_builder.get(), data, &have_bytes));
+      RETURN_NOT_OK(
+          AppendObjectStrings(numpy_array, nullptr, value_builder.get(), &have_bytes));
     } else if (PyList_Check(objects[i])) {
       int64_t size;
       std::shared_ptr<DataType> inferred_type;
@@ -678,16 +757,15 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
       return Status::TypeError("Unsupported Python type for list items");
     }
   }
-  return list_builder.Finish(out);
+  return list_builder.Finish(&out_);
 }
 
-#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                  \
-  case Type::TYPE: {                                            \
-    return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type, out); \
+#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)             \
+  case Type::TYPE: {                                       \
+    return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type); \
   }
 
-Status PandasConverter::ConvertLists(
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
   switch (type->type) {
     LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
     LIST_CASE(INT8, NPY_INT8, Int8Type)
@@ -711,13 +789,17 @@ Status PandasConverter::ConvertLists(
 Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
   PandasConverter converter(pool, ao, mo, type);
-  return converter.Convert(out);
+  RETURN_NOT_OK(converter.Convert());
+  *out = converter.result();
+  return Status::OK();
 }
 
 Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
   PandasConverter converter(pool, ao, mo, type);
-  return converter.ConvertObjects(out);
+  RETURN_NOT_OK(converter.ConvertObjects());
+  *out = converter.result();
+  return Status::OK();
 }
 
 // ----------------------------------------------------------------------
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index e7c456d80a41f..67785e34075f4 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -82,8 +82,8 @@ cdef class Array:
 
     @staticmethod
     def from_numpy(obj, mask=None, DataType type=None,
-                    timestamps_to_ms=False,
-                    MemoryPool memory_pool=None):
+                   timestamps_to_ms=False,
+                   MemoryPool memory_pool=None):
         """
         Convert pandas.Series to an Arrow Array.
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 0b3c02e9945eb..56830a88f2ec2 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -75,16 +75,25 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
             expected = df
         tm.assert_frame_equal(result, expected, check_dtype=check_dtype)
 
-    def _check_array_roundtrip(self, values, expected=None,
+    def _check_array_roundtrip(self, values, expected=None, mask=None,
                                timestamps_to_ms=False, type=None):
         arr = A.Array.from_numpy(values, timestamps_to_ms=timestamps_to_ms,
-                                 type=type)
+                                 mask=mask, type=type)
         result = arr.to_pandas()
 
-        assert arr.null_count == pd.isnull(values).sum()
+        values_nulls = pd.isnull(values)
+        if mask is None:
+            assert arr.null_count == values_nulls.sum()
+        else:
+            assert arr.null_count == (mask | values_nulls).sum()
 
-        tm.assert_series_equal(pd.Series(result), pd.Series(values),
-                               check_names=False)
+        if mask is None:
+            tm.assert_series_equal(pd.Series(result), pd.Series(values),
+                                   check_names=False)
+        else:
+            expected = pd.Series(np.ma.masked_array(values, mask=mask))
+            tm.assert_series_equal(pd.Series(result), expected,
+                                   check_names=False)
 
     def test_float_no_nulls(self):
         data = {}
@@ -402,3 +411,43 @@ def test_mixed_types_fails(self):
         data = pd.DataFrame({'a': ['a', 1, 2.0]})
         with self.assertRaises(A.error.ArrowException):
             A.Table.from_pandas(data)
+
+    def test_strided_data_import(self):
+        cases = []
+
+        columns = ['a', 'b', 'c']
+        N, K = 100, 3
+        random_numbers = np.random.randn(N, K).copy() * 100
+
+        numeric_dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8',
+                          'f4', 'f8']
+
+        for type_name in numeric_dtypes:
+            cases.append(random_numbers.astype(type_name))
+
+        # strings
+        cases.append(np.array([tm.rands(10) for i in range(N * K)],
+                              dtype=object)
+                     .reshape(N, K).copy())
+
+        # booleans
+        boolean_objects = (np.array([True, False, True] * N, dtype=object)
+                           .reshape(N, K).copy())
+
+        # add some nulls, so dtype comes back as objects
+        boolean_objects[5] = None
+        cases.append(boolean_objects)
+
+        cases.append(np.arange("2016-01-01T00:00:00.001", N * K,
+                               dtype='datetime64[ms]')
+                     .reshape(N, K).copy())
+
+        strided_mask = (random_numbers > 0).astype(bool)[:, 0]
+
+        for case in cases:
+            df = pd.DataFrame(case, columns=columns)
+            col = df['a']
+
+            self._check_pandas_roundtrip(df)
+            self._check_array_roundtrip(col)
+            self._check_array_roundtrip(col, mask=strided_mask)

From d0cd03d78547b12aaeb5e50d8c52ace60a973d4e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 3 Apr 2017 18:08:06 -0400
Subject: [PATCH 0462/1644] ARROW-763: C++: Use  to find libpythonX.X.dylib

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #485 from xhochy/ARROW-763 and squashes the following commits:

d5a475f [Uwe L. Korn] ARROW-763: C++: Use  to find libpythonX.X.dylib
---
 cpp/cmake_modules/FindPythonLibsNew.cmake | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/cpp/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake
index 3e248a93342c5..dfe5661b015b5 100644
--- a/cpp/cmake_modules/FindPythonLibsNew.cmake
+++ b/cpp/cmake_modules/FindPythonLibsNew.cmake
@@ -148,10 +148,20 @@ if(CMAKE_HOST_WIN32)
         set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/libpython${PYTHON_LIBRARY_SUFFIX}.a")
     endif()
 elseif(APPLE)
-  # In Python C extensions on OS X, the flag "-undefined dynamic_lookup" can
-  # avoid certain kinds of dynamic linking issues with portable binaries, so
-  # you should avoid targeting libpython at link time if at all possible
-  set(PYTHON_LIBRARY "${PYTHON_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
+  # In some cases libpythonX.X.dylib is not part of the PYTHON_PREFIX and we
+  # need to call `python-config --prefix` to determine the correct location.
+
+  find_program(PYTHON_CONFIG python-config
+      NO_CMAKE_SYSTEM_PATH)
+  if (PYTHON_CONFIG)
+    execute_process(
+        COMMAND "${PYTHON_CONFIG}" "--prefix"
+        OUTPUT_VARIABLE PYTHON_CONFIG_PREFIX
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+    set(PYTHON_LIBRARY "${PYTHON_CONFIG_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
+  else()
+    set(PYTHON_LIBRARY "${PYTHON_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
+  endif()
 else()
     if(${PYTHON_SIZEOF_VOID_P} MATCHES 8)
         set(_PYTHON_LIBS_SEARCH "${PYTHON_PREFIX}/lib64" "${PYTHON_PREFIX}/lib" "${PYTHON_LIBRARY_PATH}")

From d560e307749a2397810962db1a5af4fb65675f17 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 4 Apr 2017 08:40:40 +0200
Subject: [PATCH 0463/1644] ARROW-656: [C++] Add random access writer for a
 mutable buffer. Rename WriteableFileInterface to WriteableFile for better
 consistency

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #486 from wesm/ARROW-656 and squashes the following commits:

be0d4bc [Wes McKinney] Fix glib after renaming class
042f533 [Wes McKinney] Add random access writer for a mutable buffer. Rename WriteableFileInterface to WriteableFile for better consistency
---
 c_glib/arrow-glib/io-memory-mapped-file.cpp |  2 +-
 c_glib/arrow-glib/io-writeable-file.cpp     |  2 +-
 c_glib/arrow-glib/io-writeable-file.h       |  2 +-
 c_glib/arrow-glib/io-writeable-file.hpp     |  8 ++--
 cpp/src/arrow/io/interfaces.h               |  6 +--
 cpp/src/arrow/io/io-memory-test.cc          | 27 +++++++++++++
 cpp/src/arrow/io/memory.cc                  | 45 +++++++++++++++++++++
 cpp/src/arrow/io/memory.h                   | 23 +++++++++++
 python/pyarrow/includes/libarrow.pxd        |  4 +-
 9 files changed, 107 insertions(+), 12 deletions(-)

diff --git a/c_glib/arrow-glib/io-memory-mapped-file.cpp b/c_glib/arrow-glib/io-memory-mapped-file.cpp
index 12c9a6c95ac12..e2e255c039109 100644
--- a/c_glib/arrow-glib/io-memory-mapped-file.cpp
+++ b/c_glib/arrow-glib/io-memory-mapped-file.cpp
@@ -127,7 +127,7 @@ garrow_io_writeable_interface_init(GArrowIOWriteableInterface *iface)
   iface->get_raw = garrow_io_memory_mapped_file_get_raw_writeable_interface;
 }
 
-static std::shared_ptr<arrow::io::WriteableFileInterface>
+static std::shared_ptr<arrow::io::WriteableFile>
 garrow_io_memory_mapped_file_get_raw_writeable_file_interface(GArrowIOWriteableFile *file)
 {
   auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(file);
diff --git a/c_glib/arrow-glib/io-writeable-file.cpp b/c_glib/arrow-glib/io-writeable-file.cpp
index 3de42dd60a971..41b682acd1e26 100644
--- a/c_glib/arrow-glib/io-writeable-file.cpp
+++ b/c_glib/arrow-glib/io-writeable-file.cpp
@@ -76,7 +76,7 @@ garrow_io_writeable_file_write_at(GArrowIOWriteableFile *writeable_file,
 
 G_END_DECLS
 
-std::shared_ptr<arrow::io::WriteableFileInterface>
+std::shared_ptr<arrow::io::WriteableFile>
 garrow_io_writeable_file_get_raw(GArrowIOWriteableFile *writeable_file)
 {
   auto *iface = GARROW_IO_WRITEABLE_FILE_GET_IFACE(writeable_file);
diff --git a/c_glib/arrow-glib/io-writeable-file.h b/c_glib/arrow-glib/io-writeable-file.h
index 4a4dee5111f5f..d1ebdbe630ef2 100644
--- a/c_glib/arrow-glib/io-writeable-file.h
+++ b/c_glib/arrow-glib/io-writeable-file.h
@@ -28,7 +28,7 @@ G_BEGIN_DECLS
 #define GARROW_IO_WRITEABLE_FILE(obj)                           \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
                               GARROW_IO_TYPE_WRITEABLE_FILE,    \
-                              GArrowIOWriteableFileInterface))
+                              GArrowIOWriteableFile))
 #define GARROW_IO_IS_WRITEABLE_FILE(obj)                        \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
                               GARROW_IO_TYPE_WRITEABLE_FILE))
diff --git a/c_glib/arrow-glib/io-writeable-file.hpp b/c_glib/arrow-glib/io-writeable-file.hpp
index 2043007ad58e3..aba95b209d827 100644
--- a/c_glib/arrow-glib/io-writeable-file.hpp
+++ b/c_glib/arrow-glib/io-writeable-file.hpp
@@ -24,15 +24,15 @@
 #include <arrow-glib/io-writeable-file.h>
 
 /**
- * GArrowIOWriteableFileInterface:
+ * GArrowIOWriteableFile:
  *
- * It wraps `arrow::io::WriteableFileInterface`.
+ * It wraps `arrow::io::WriteableFile`.
  */
 struct _GArrowIOWriteableFileInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::WriteableFileInterface> (*get_raw)(GArrowIOWriteableFile *file);
+  std::shared_ptr<arrow::io::WriteableFile> (*get_raw)(GArrowIOWriteableFile *file);
 };
 
-std::shared_ptr<arrow::io::WriteableFileInterface> garrow_io_writeable_file_get_raw(GArrowIOWriteableFile *writeable_file);
+std::shared_ptr<arrow::io::WriteableFile> garrow_io_writeable_file_get_raw(GArrowIOWriteableFile *writeable_file);
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 258a3155743bf..b5a0bd85bf27b 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -121,16 +121,16 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
   RandomAccessFile();
 };
 
-class ARROW_EXPORT WriteableFileInterface : public OutputStream, public Seekable {
+class ARROW_EXPORT WriteableFile : public OutputStream, public Seekable {
  public:
   virtual Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) = 0;
 
  protected:
-  WriteableFileInterface() { set_mode(FileMode::READ); }
+  WriteableFile() { set_mode(FileMode::READ); }
 };
 
 class ARROW_EXPORT ReadWriteFileInterface : public RandomAccessFile,
-                                            public WriteableFileInterface {
+                                            public WriteableFile {
  protected:
   ReadWriteFileInterface() { RandomAccessFile::set_mode(FileMode::READWRITE); }
 };
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 442cd0c4bbccd..4704fe8f4d391 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -66,6 +66,33 @@ TEST_F(TestBufferOutputStream, CloseResizes) {
   ASSERT_EQ(static_cast<int64_t>(K * data.size()), buffer_->size());
 }
 
+TEST(TestFixedSizeBufferWriter, Basics) {
+  std::shared_ptr<MutableBuffer> buffer;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), 1024, &buffer));
+
+  FixedSizeBufferWriter writer(buffer);
+
+  int64_t position;
+  ASSERT_OK(writer.Tell(&position));
+  ASSERT_EQ(0, position);
+
+  std::string data = "data123456";
+  auto nbytes = static_cast<int64_t>(data.size());
+  ASSERT_OK(writer.Write(reinterpret_cast<const uint8_t*>(data.c_str()), nbytes));
+
+  ASSERT_OK(writer.Tell(&position));
+  ASSERT_EQ(nbytes, position);
+
+  ASSERT_OK(writer.Seek(4));
+  ASSERT_OK(writer.Tell(&position));
+  ASSERT_EQ(4, position);
+
+  ASSERT_RAISES(IOError, writer.Seek(-1));
+  ASSERT_RAISES(IOError, writer.Seek(1024));
+
+  ASSERT_OK(writer.Close());
+}
+
 TEST(TestBufferReader, RetainParentReference) {
   // ARROW-387
   std::string data = "data123456";
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 5b5c8649deec4..2e701e1104d1c 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -98,6 +98,51 @@ Status BufferOutputStream::Reserve(int64_t nbytes) {
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// In-memory buffer writer
+
+/// Input buffer must be mutable, will abort if not
+FixedSizeBufferWriter::FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer) {
+  buffer_ = buffer;
+  DCHECK(buffer->is_mutable()) << "Must pass mutable buffer";
+  mutable_data_ = buffer->mutable_data();
+  size_ = buffer->size();
+  position_ = 0;
+}
+
+FixedSizeBufferWriter::~FixedSizeBufferWriter() {}
+
+Status FixedSizeBufferWriter::Close() {
+  // No-op
+  return Status::OK();
+}
+
+Status FixedSizeBufferWriter::Seek(int64_t position) {
+  if (position < 0 || position >= size_) {
+    return Status::IOError("position out of bounds");
+  }
+  position_ = position;
+  return Status::OK();
+}
+
+Status FixedSizeBufferWriter::Tell(int64_t* position) {
+  *position = position_;
+  return Status::OK();
+}
+
+Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
+  std::memcpy(mutable_data_ + position_, data, nbytes);
+  position_ += nbytes;
+  return Status::OK();
+}
+
+Status FixedSizeBufferWriter::WriteAt(
+    int64_t position, const uint8_t* data, int64_t nbytes) {
+  std::lock_guard<std::mutex> guard(lock_);
+  RETURN_NOT_OK(Seek(position));
+  return Write(data, nbytes);
+}
+
 // ----------------------------------------------------------------------
 // In-memory buffer reader
 
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index eb2a50912889e..fbb186b728022 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -22,6 +22,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <mutex>
 #include <string>
 
 #include "arrow/io/interfaces.h"
@@ -66,6 +67,28 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   uint8_t* mutable_data_;
 };
 
+/// \brief Enables random writes into a fixed-size mutable buffer
+///
+class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
+ public:
+  /// Input buffer must be mutable, will abort if not
+  explicit FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer);
+  ~FixedSizeBufferWriter();
+
+  Status Close() override;
+  Status Seek(int64_t position) override;
+  Status Tell(int64_t* position) override;
+  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+
+ private:
+  std::mutex lock_;
+  std::shared_ptr<Buffer> buffer_;
+  uint8_t* mutable_data_;
+  int64_t size_;
+  int64_t position_;
+};
+
 class ARROW_EXPORT BufferReader : public RandomAccessFile {
  public:
   explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 67d6af910c2b9..2a0488f3a0139 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -342,12 +342,12 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
         CStatus ReadAt(int64_t position, int64_t nbytes,
                        int64_t* bytes_read, shared_ptr[CBuffer]* out)
 
-    cdef cppclass WriteableFileInterface(OutputStream, Seekable):
+    cdef cppclass WriteableFile(OutputStream, Seekable):
         CStatus WriteAt(int64_t position, const uint8_t* data,
                         int64_t nbytes)
 
     cdef cppclass ReadWriteFileInterface(RandomAccessFile,
-                                         WriteableFileInterface):
+                                         WriteableFile):
         pass
 
 

From ec6188efcc884e46481fe986605e3cbfc33c7e07 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 4 Apr 2017 18:24:07 +0200
Subject: [PATCH 0464/1644] ARROW-769: [GLib] Support building without
 installed Arrow C++

It doesn't require "make install"-ed Arrow C++ to build Arrow GLib.
But it requires "make"-ed Arrow C++.

This is useful to build packages.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #490 from kou/glib-support-build-without-installed-arrow-cpp and squashes the following commits:

352999b [Kouhei Sutou] [GLib] Support building without installed Arrow C++
---
 c_glib/configure.ac | 39 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index c6913437d93f8..fc24c1b3c4778 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -61,9 +61,42 @@ AM_PATH_GLIB_2_0([2.32.4], [], [], [gobject])
 GOBJECT_INTROSPECTION_REQUIRE([1.32.1])
 GTK_DOC_CHECK([1.18-2])
 
-PKG_CHECK_MODULES([ARROW], [arrow])
-PKG_CHECK_MODULES([ARROW_IO], [arrow-io])
-PKG_CHECK_MODULES([ARROW_IPC], [arrow-ipc])
+AC_ARG_WITH(arrow-cpp-build-type,
+  [AS_HELP_STRING([--with-arrow-cpp-build-type=TYPE],
+                  [-DCMAKE_BUILD_TYPE option value for Arrow C++ (default=Release)])],
+  [GARROW_ARROW_CPP_BUILD_TYPE="$withval"],
+  [GARROW_ARROW_CPP_BUILD_TYPE="Release"])
+
+AC_ARG_WITH(arrow-cpp-build-dir,
+  [AS_HELP_STRING([--with-arrow-cpp-build-dir=PATH],
+                  [Use this option to build with not installed Arrow C++])],
+  [GARROW_ARROW_CPP_BUILD_DIR="$withval"],
+  [GARROW_ARROW_CPP_BUILD_DIR=""])
+if test "x$GARROW_ARROW_CPP_BUILD_DIR" = "x"; then
+  PKG_CHECK_MODULES([ARROW], [arrow])
+  PKG_CHECK_MODULES([ARROW_IO], [arrow-io])
+  PKG_CHECK_MODULES([ARROW_IPC], [arrow-ipc])
+else
+  ARROW_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src"
+  ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}"
+
+  ARROW_CFLAGS="-I${ARROW_INCLUDE_DIR}"
+  ARROW_IO_CFLAGS="-I${ARROW_INCLUDE_DIR}"
+  ARROW_IPC_CFLAGS="-I${ARROW_INCLUDE_DIR}"
+  ARROW_LIBS="-L${ARROW_LIB_DIR} -larrow"
+  ARROW_IO_LIBS="-L${ARROW_LIB_DIR} -larrow_io"
+  ARROW_IPC_LIBS="-L${ARROW_LIB_DIR} -larrow_ipc"
+
+  AC_SUBST(ARROW_LIB_DIR)
+
+  AC_SUBST(ARROW_CFLAGS)
+  AC_SUBST(ARROW_IO_CFLAGS)
+  AC_SUBST(ARROW_IPC_CFLAGS)
+  AC_SUBST(ARROW_LIBS)
+  AC_SUBST(ARROW_IO_LIBS)
+  AC_SUBST(ARROW_IPC_LIBS)
+fi
+
 
 AC_CONFIG_FILES([
   Makefile

From 2aed7845fbc9e3d91ab9d16965ee9f6f3abc668b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 4 Apr 2017 13:18:16 -0400
Subject: [PATCH 0465/1644] ARROW-765: [Python] Add more natural Exception type
 hierarchy for thirdparty users

I also took the liberty of changing a number of error types in libarrow_python

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #489 from wesm/ARROW-765 and squashes the following commits:

74c43df [Wes McKinney] Make a nicer Exception hierachy, with more intuitive bases for thirdparty users
2a58a1b [Wes McKinney] Add a nicer exception hierarchy. Unknown errors return as ValueError
---
 cpp/src/arrow/python/builtin_convert.cc      |  8 +-
 cpp/src/arrow/python/pandas_convert.cc       |  6 +-
 cpp/src/arrow/status.h                       |  2 +-
 python/pyarrow/__init__.py                   |  8 +-
 python/pyarrow/error.pyx                     | 43 ++++++++++-
 python/pyarrow/includes/common.pxd           |  4 +-
 python/pyarrow/tests/test_convert_builtin.py | 78 ++++++++++----------
 python/pyarrow/tests/test_convert_pandas.py  |  4 +-
 python/pyarrow/tests/test_feather.py         |  2 +-
 9 files changed, 101 insertions(+), 54 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 6a13fdccdeaff..25b32ee26a06b 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -394,7 +394,7 @@ class BytesConverter : public TypedConverter<BinaryBuilder> {
       } else if (PyBytes_Check(item)) {
         bytes_obj = item;
       } else {
-        return Status::TypeError(
+        return Status::Invalid(
             "Value that cannot be converted to bytes was encountered");
       }
       // No error checking
@@ -429,7 +429,7 @@ class FixedWidthBytesConverter : public TypedConverter<FixedSizeBinaryBuilder> {
       } else if (PyBytes_Check(item)) {
         bytes_obj = item;
       } else {
-        return Status::TypeError(
+        return Status::Invalid(
             "Value that cannot be converted to bytes was encountered");
       }
       // No error checking
@@ -458,7 +458,7 @@ class UTF8Converter : public TypedConverter<StringBuilder> {
         RETURN_NOT_OK(typed_builder_->AppendNull());
         continue;
       } else if (!PyUnicode_Check(item)) {
-        return Status::TypeError("Non-unicode value encountered");
+        return Status::Invalid("Non-unicode value encountered");
       }
       tmp.reset(PyUnicode_AsUTF8String(item));
       RETURN_IF_PYERROR();
@@ -585,7 +585,7 @@ Status CheckPythonBytesAreFixedLength(PyObject* obj, Py_ssize_t expected_length)
     std::stringstream ss;
     ss << "Found byte string of length " << length << ", expected length is "
        << expected_length;
-    return Status::TypeError(ss.str());
+    return Status::Invalid(ss.str());
   }
   return Status::OK();
 }
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 9577892a55b76..48d3489bf900b 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -161,7 +161,7 @@ static Status AppendObjectStrings(
       obj = PyUnicode_AsUTF8String(obj);
       if (obj == NULL) {
         PyErr_Clear();
-        return Status::TypeError("failed converting unicode to UTF8");
+        return Status::Invalid("failed converting unicode to UTF8");
       }
       const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
       Status s = builder->Append(PyBytes_AS_STRING(obj), length);
@@ -200,7 +200,7 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
       obj = PyUnicode_AsUTF8String(obj);
       if (obj == NULL) {
         PyErr_Clear();
-        return Status::TypeError("failed converting unicode to UTF8");
+        return Status::Invalid("failed converting unicode to UTF8");
       }
 
       RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
@@ -482,7 +482,7 @@ Status InvalidConversion(PyObject* obj, const std::string& expected_type_name) {
   std::stringstream ss;
   ss << "Python object of type " << cpp_type_name << " is not None and is not a "
      << expected_type_name << " object";
-  return Status::TypeError(ss.str());
+  return Status::Invalid(ss.str());
 }
 
 Status PandasConverter::ConvertDates() {
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 05f5b749b60cb..dd65b753fef31 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -134,7 +134,7 @@ class ARROW_EXPORT Status {
   bool IsKeyError() const { return code() == StatusCode::KeyError; }
   bool IsInvalid() const { return code() == StatusCode::Invalid; }
   bool IsIOError() const { return code() == StatusCode::IOError; }
-
+  bool IsTypeError() const { return code() == StatusCode::TypeError; }
   bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
   bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 6860f986fb6e8..8c520748cf316 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -38,7 +38,13 @@
                            ListArray, StringArray,
                            DictionaryArray)
 
-from pyarrow.error import ArrowException
+from pyarrow.error import (ArrowException,
+                           ArrowKeyError,
+                           ArrowInvalid,
+                           ArrowIOError,
+                           ArrowMemoryError,
+                           ArrowNotImplementedError,
+                           ArrowTypeError)
 
 from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
 from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
diff --git a/python/pyarrow/error.pyx b/python/pyarrow/error.pyx
index b8a82b3754c1b..259aeb074e3c2 100644
--- a/python/pyarrow/error.pyx
+++ b/python/pyarrow/error.pyx
@@ -19,13 +19,52 @@ from pyarrow.includes.libarrow cimport CStatus
 from pyarrow.includes.common cimport c_string
 from pyarrow.compat import frombytes
 
+
 class ArrowException(Exception):
     pass
 
+
+class ArrowInvalid(ValueError, ArrowException):
+    pass
+
+
+class ArrowMemoryError(MemoryError, ArrowException):
+    pass
+
+
+class ArrowIOError(IOError, ArrowException):
+    pass
+
+
+class ArrowKeyError(KeyError, ArrowException):
+    pass
+
+
+class ArrowTypeError(TypeError, ArrowException):
+    pass
+
+
+class ArrowNotImplementedError(NotImplementedError, ArrowException):
+    pass
+
+
 cdef int check_status(const CStatus& status) nogil except -1:
     if status.ok():
         return 0
 
-    cdef c_string c_message = status.ToString()
     with gil:
-        raise ArrowException(frombytes(c_message))
+        message = frombytes(status.ToString())
+        if status.IsInvalid():
+            raise ArrowInvalid(message)
+        elif status.IsIOError():
+            raise ArrowIOError(message)
+        elif status.IsOutOfMemory():
+            raise ArrowMemoryError(message)
+        elif status.IsKeyError():
+            raise ArrowKeyError(message)
+        elif status.IsNotImplemented():
+            raise ArrowNotImplementedError(message)
+        elif status.IsTypeError():
+            raise ArrowTypeError(message)
+        else:
+            raise ArrowException(message)
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index f689bdc3fd819..ab38ff3084f01 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -43,10 +43,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_string ToString()
 
         c_bool ok()
+        c_bool IsIOError()
         c_bool IsOutOfMemory()
+        c_bool IsInvalid()
         c_bool IsKeyError()
         c_bool IsNotImplemented()
-        c_bool IsInvalid()
+        c_bool IsTypeError()
 
 
 cdef inline object PyObject_to_object(PyObject* o):
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 15fca560c6513..e2b03d85ecd50 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -17,7 +17,7 @@
 # under the License.
 
 from pyarrow.compat import unittest, u  # noqa
-import pyarrow
+import pyarrow as pa
 
 import datetime
 
@@ -26,32 +26,32 @@ class TestConvertList(unittest.TestCase):
 
     def test_boolean(self):
         expected = [True, None, False, None]
-        arr = pyarrow.from_pylist(expected)
+        arr = pa.from_pylist(expected)
         assert len(arr) == 4
         assert arr.null_count == 2
-        assert arr.type == pyarrow.bool_()
+        assert arr.type == pa.bool_()
         assert arr.to_pylist() == expected
 
     def test_empty_list(self):
-        arr = pyarrow.from_pylist([])
+        arr = pa.from_pylist([])
         assert len(arr) == 0
         assert arr.null_count == 0
-        assert arr.type == pyarrow.null()
+        assert arr.type == pa.null()
         assert arr.to_pylist() == []
 
     def test_all_none(self):
-        arr = pyarrow.from_pylist([None, None])
+        arr = pa.from_pylist([None, None])
         assert len(arr) == 2
         assert arr.null_count == 2
-        assert arr.type == pyarrow.null()
+        assert arr.type == pa.null()
         assert arr.to_pylist() == [None, None]
 
     def test_integer(self):
         expected = [1, None, 3, None]
-        arr = pyarrow.from_pylist(expected)
+        arr = pa.from_pylist(expected)
         assert len(arr) == 4
         assert arr.null_count == 2
-        assert arr.type == pyarrow.int64()
+        assert arr.type == pa.int64()
         assert arr.to_pylist() == expected
 
     def test_garbage_collection(self):
@@ -60,25 +60,25 @@ def test_garbage_collection(self):
         # Force the cyclic garbage collector to run
         gc.collect()
 
-        bytes_before = pyarrow.total_allocated_bytes()
-        pyarrow.from_pylist([1, None, 3, None])
+        bytes_before = pa.total_allocated_bytes()
+        pa.from_pylist([1, None, 3, None])
         gc.collect()
-        assert pyarrow.total_allocated_bytes() == bytes_before
+        assert pa.total_allocated_bytes() == bytes_before
 
     def test_double(self):
         data = [1.5, 1, None, 2.5, None, None]
-        arr = pyarrow.from_pylist(data)
+        arr = pa.from_pylist(data)
         assert len(arr) == 6
         assert arr.null_count == 3
-        assert arr.type == pyarrow.float64()
+        assert arr.type == pa.float64()
         assert arr.to_pylist() == data
 
     def test_unicode(self):
         data = [u'foo', u'bar', None, u'mañana']
-        arr = pyarrow.from_pylist(data)
+        arr = pa.from_pylist(data)
         assert len(arr) == 4
         assert arr.null_count == 1
-        assert arr.type == pyarrow.string()
+        assert arr.type == pa.string()
         assert arr.to_pylist() == data
 
     def test_bytes(self):
@@ -86,31 +86,31 @@ def test_bytes(self):
         data = [b'foo',
                 u1.decode('utf-8'),  # unicode gets encoded,
                 None]
-        arr = pyarrow.from_pylist(data)
+        arr = pa.from_pylist(data)
         assert len(arr) == 3
         assert arr.null_count == 1
-        assert arr.type == pyarrow.binary()
+        assert arr.type == pa.binary()
         assert arr.to_pylist() == [b'foo', u1, None]
 
     def test_fixed_size_bytes(self):
         data = [b'foof', None, b'barb', b'2346']
-        arr = pyarrow.from_pylist(data, type=pyarrow.binary(4))
+        arr = pa.from_pylist(data, type=pa.binary(4))
         assert len(arr) == 4
         assert arr.null_count == 1
-        assert arr.type == pyarrow.binary(4)
+        assert arr.type == pa.binary(4)
         assert arr.to_pylist() == data
 
     def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
         data = [b'foo', None, b'barb', b'2346']
-        with self.assertRaises(pyarrow.error.ArrowException):
-            pyarrow.from_pylist(data, type=pyarrow.binary(4))
+        with self.assertRaises(pa.ArrowInvalid):
+            pa.from_pylist(data, type=pa.binary(4))
 
     def test_date(self):
         data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1),
                 datetime.date(2040, 2, 26)]
-        arr = pyarrow.from_pylist(data)
+        arr = pa.from_pylist(data)
         assert len(arr) == 4
-        assert arr.type == pyarrow.date64()
+        assert arr.type == pa.date64()
         assert arr.null_count == 1
         assert arr[0].as_py() == datetime.date(2000, 1, 1)
         assert arr[1].as_py() is None
@@ -124,9 +124,9 @@ def test_timestamp(self):
             datetime.datetime(2006, 1, 13, 12, 34, 56, 432539),
             datetime.datetime(2010, 8, 13, 5, 46, 57, 437699)
         ]
-        arr = pyarrow.from_pylist(data)
+        arr = pa.from_pylist(data)
         assert len(arr) == 4
-        assert arr.type == pyarrow.timestamp('us')
+        assert arr.type == pa.timestamp('us')
         assert arr.null_count == 1
         assert arr[0].as_py() == datetime.datetime(2007, 7, 13, 1,
                                                    23, 34, 123456)
@@ -137,28 +137,28 @@ def test_timestamp(self):
                                                    46, 57, 437699)
 
     def test_mixed_nesting_levels(self):
-        pyarrow.from_pylist([1, 2, None])
-        pyarrow.from_pylist([[1], [2], None])
-        pyarrow.from_pylist([[1], [2], [None]])
+        pa.from_pylist([1, 2, None])
+        pa.from_pylist([[1], [2], None])
+        pa.from_pylist([[1], [2], [None]])
 
-        with self.assertRaises(pyarrow.ArrowException):
-            pyarrow.from_pylist([1, 2, [1]])
+        with self.assertRaises(pa.ArrowInvalid):
+            pa.from_pylist([1, 2, [1]])
 
-        with self.assertRaises(pyarrow.ArrowException):
-            pyarrow.from_pylist([1, 2, []])
+        with self.assertRaises(pa.ArrowInvalid):
+            pa.from_pylist([1, 2, []])
 
-        with self.assertRaises(pyarrow.ArrowException):
-            pyarrow.from_pylist([[1], [2], [None, [1]]])
+        with self.assertRaises(pa.ArrowInvalid):
+            pa.from_pylist([[1], [2], [None, [1]]])
 
     def test_list_of_int(self):
         data = [[1, 2, 3], [], None, [1, 2]]
-        arr = pyarrow.from_pylist(data)
+        arr = pa.from_pylist(data)
         assert len(arr) == 4
         assert arr.null_count == 1
-        assert arr.type == pyarrow.list_(pyarrow.int64())
+        assert arr.type == pa.list_(pa.int64())
         assert arr.to_pylist() == data
 
     def test_mixed_types_fails(self):
         data = ['a', 1, 2.0]
-        with self.assertRaises(pyarrow.error.ArrowException):
-            pyarrow.from_pylist(data)
+        with self.assertRaises(pa.ArrowException):
+            pa.from_pylist(data)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 56830a88f2ec2..87c9c03d7da11 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -266,7 +266,7 @@ def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
         values = [b'foo', None, b'ba', None, None, b'hey']
         df = pd.DataFrame({'strings': values})
         schema = A.Schema.from_fields([A.field('strings', A.binary(3))])
-        with self.assertRaises(A.error.ArrowException):
+        with self.assertRaises(A.ArrowInvalid):
             A.Table.from_pandas(df, schema=schema)
 
     def test_timestamps_notimezone_no_nulls(self):
@@ -409,7 +409,7 @@ def test_category(self):
 
     def test_mixed_types_fails(self):
         data = pd.DataFrame({'a': ['a', 1, 2.0]})
-        with self.assertRaises(A.error.ArrowException):
+        with self.assertRaises(A.ArrowException):
             A.Table.from_pandas(data)
 
     def test_strided_data_import(self):
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index c7b4f1e997327..cba9464354a4e 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -45,7 +45,7 @@ def tearDown(self):
                 pass
 
     def test_file_not_exist(self):
-        with self.assertRaises(pa.ArrowException):
+        with self.assertRaises(pa.ArrowIOError):
             FeatherReader('test_invalid_file')
 
     def _get_null_counts(self, path, columns=None):

From 5d6c6ad6a81be6194a4f8349a369a94ef927e18b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 4 Apr 2017 21:57:13 +0200
Subject: [PATCH 0466/1644] ARROW-770: [C++] Move .clang* files back into cpp
 source tree

After ARROW-341, we don't need these files at the top level anymore to get clang-format to work on all of our C++ code

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #491 from wesm/ARROW-770 and squashes the following commits:

1588a4f [Wes McKinney] Move .clang* files back into cpp source tree
---
 .clang-format => cpp/.clang-format           | 0
 .clang-tidy => cpp/.clang-tidy               | 0
 .clang-tidy-ignore => cpp/.clang-tidy-ignore | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename .clang-format => cpp/.clang-format (100%)
 rename .clang-tidy => cpp/.clang-tidy (100%)
 rename .clang-tidy-ignore => cpp/.clang-tidy-ignore (100%)

diff --git a/.clang-format b/cpp/.clang-format
similarity index 100%
rename from .clang-format
rename to cpp/.clang-format
diff --git a/.clang-tidy b/cpp/.clang-tidy
similarity index 100%
rename from .clang-tidy
rename to cpp/.clang-tidy
diff --git a/.clang-tidy-ignore b/cpp/.clang-tidy-ignore
similarity index 100%
rename from .clang-tidy-ignore
rename to cpp/.clang-tidy-ignore

From 360942e6171b301d5efb1686794239e3527828f3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 4 Apr 2017 16:20:56 -0400
Subject: [PATCH 0467/1644] ARROW-672: [Format] Add MetadataVersion::V3 for
 Arrow 0.3

As a matter of diligence, we increment the metadata version for Arrow 0.3 since we've changed the metadata format is various ways.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #488 from wesm/ARROW-672 and squashes the following commits:

f39733e [Wes McKinney] Add C++ unit test for read/write MetadataVersion. Change MetadataVersion to C++11 enum class
bb09ba2 [Wes McKinney] Add MetadataVersion::V3 for Arrow 0.3
---
 c_glib/arrow-glib/ipc-metadata-version.cpp    | 22 ++++++++++--------
 c_glib/arrow-glib/ipc-metadata-version.h      |  4 +++-
 c_glib/arrow-glib/ipc-metadata-version.hpp    |  4 ++--
 cpp/src/arrow/ipc/ipc-read-write-test.cc      | 20 ++++++++++++++++
 cpp/src/arrow/ipc/metadata.cc                 | 23 ++++++++++++++++++-
 cpp/src/arrow/ipc/metadata.h                  |  6 ++---
 cpp/src/arrow/ipc/reader.cc                   | 11 ++++++---
 cpp/src/arrow/ipc/reader.h                    |  2 +-
 format/Schema.fbs                             |  3 ++-
 .../vector/stream/MessageSerializer.java      |  2 +-
 10 files changed, 75 insertions(+), 22 deletions(-)

diff --git a/c_glib/arrow-glib/ipc-metadata-version.cpp b/c_glib/arrow-glib/ipc-metadata-version.cpp
index c5cc8d379843c..f591f295ec886 100644
--- a/c_glib/arrow-glib/ipc-metadata-version.cpp
+++ b/c_glib/arrow-glib/ipc-metadata-version.cpp
@@ -29,31 +29,35 @@
  * @short_description: Metadata version mapgging between Arrow and arrow-glib
  *
  * #GArrowIPCMetadataVersion provides metadata versions corresponding
- * to `arrow::ipc::MetadataVersion::type` values.
+ * to `arrow::ipc::MetadataVersion` values.
  */
 
 GArrowIPCMetadataVersion
-garrow_ipc_metadata_version_from_raw(arrow::ipc::MetadataVersion::type version)
+garrow_ipc_metadata_version_from_raw(arrow::ipc::MetadataVersion version)
 {
   switch (version) {
-  case arrow::ipc::MetadataVersion::type::V1:
+  case arrow::ipc::MetadataVersion::V1:
     return GARROW_IPC_METADATA_VERSION_V1;
-  case arrow::ipc::MetadataVersion::type::V2:
+  case arrow::ipc::MetadataVersion::V2:
     return GARROW_IPC_METADATA_VERSION_V2;
+  case arrow::ipc::MetadataVersion::V3:
+    return GARROW_IPC_METADATA_VERSION_V3;
   default:
-    return GARROW_IPC_METADATA_VERSION_V2;
+    return GARROW_IPC_METADATA_VERSION_V3;
   }
 }
 
-arrow::ipc::MetadataVersion::type
+arrow::ipc::MetadataVersion
 garrow_ipc_metadata_version_to_raw(GArrowIPCMetadataVersion version)
 {
   switch (version) {
   case GARROW_IPC_METADATA_VERSION_V1:
-    return arrow::ipc::MetadataVersion::type::V1;
+    return arrow::ipc::MetadataVersion::V1;
   case GARROW_IPC_METADATA_VERSION_V2:
-    return arrow::ipc::MetadataVersion::type::V2;
+    return arrow::ipc::MetadataVersion::V2;
+  case GARROW_IPC_METADATA_VERSION_V3:
+    return arrow::ipc::MetadataVersion::V3;
   default:
-    return arrow::ipc::MetadataVersion::type::V2;
+    return arrow::ipc::MetadataVersion::V3;
   }
 }
diff --git a/c_glib/arrow-glib/ipc-metadata-version.h b/c_glib/arrow-glib/ipc-metadata-version.h
index ccfd52a81639f..20defdb71b4f2 100644
--- a/c_glib/arrow-glib/ipc-metadata-version.h
+++ b/c_glib/arrow-glib/ipc-metadata-version.h
@@ -27,13 +27,15 @@ G_BEGIN_DECLS
  * GArrowIPCMetadataVersion:
  * @GARROW_IPC_METADATA_VERSION_V1: Version 1.
  * @GARROW_IPC_METADATA_VERSION_V2: Version 2.
+ * @GARROW_IPC_METADATA_VERSION_V3: Version 3.
  *
  * They are corresponding to `arrow::ipc::MetadataVersion::type`
  * values.
  */
 typedef enum {
   GARROW_IPC_METADATA_VERSION_V1,
-  GARROW_IPC_METADATA_VERSION_V2
+  GARROW_IPC_METADATA_VERSION_V2,
+  GARROW_IPC_METADATA_VERSION_V3
 } GArrowIPCMetadataVersion;
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-metadata-version.hpp b/c_glib/arrow-glib/ipc-metadata-version.hpp
index 2a7e8cffa8917..229565f002180 100644
--- a/c_glib/arrow-glib/ipc-metadata-version.hpp
+++ b/c_glib/arrow-glib/ipc-metadata-version.hpp
@@ -23,5 +23,5 @@
 
 #include <arrow-glib/ipc-metadata-version.h>
 
-GArrowIPCMetadataVersion garrow_ipc_metadata_version_from_raw(arrow::ipc::MetadataVersion::type version);
-arrow::ipc::MetadataVersion::type garrow_ipc_metadata_version_to_raw(GArrowIPCMetadataVersion version);
+GArrowIPCMetadataVersion garrow_ipc_metadata_version_from_raw(arrow::ipc::MetadataVersion version);
+arrow::ipc::MetadataVersion garrow_ipc_metadata_version_to_raw(GArrowIPCMetadataVersion version);
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 86ec7701add20..6807296b59a5e 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -211,6 +211,26 @@ TEST_P(TestIpcRoundTrip, RoundTrip) {
   CheckRoundtrip(*batch, 1 << 20);
 }
 
+TEST_F(TestIpcRoundTrip, MetadataVersion) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeIntRecordBatch(&batch));
+
+  ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(1 << 16, "test-metadata", &mmap_));
+
+  int32_t metadata_length;
+  int64_t body_length;
+
+  const int64_t buffer_offset = 0;
+
+  ASSERT_OK(WriteRecordBatch(
+      *batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
+
+  std::shared_ptr<Message> message;
+  ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+
+  ASSERT_EQ(MetadataVersion::V3, message->metadata_version());
+}
+
 TEST_P(TestIpcRoundTrip, SliceRoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 5007f1309087d..2ff25eeaa9213 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -50,7 +50,7 @@ using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
 using FBString = flatbuffers::Offset<flatbuffers::String>;
 
-static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
+static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V3;
 
 static Status IntFromFlatbuffer(
     const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
@@ -826,6 +826,23 @@ class Message::MessageImpl {
     }
   }
 
+  MetadataVersion version() const {
+    switch (message_->version()) {
+      case flatbuf::MetadataVersion_V1:
+        // Arrow 0.1
+        return MetadataVersion::V1;
+      case flatbuf::MetadataVersion_V2:
+        // Arrow 0.2
+        return MetadataVersion::V2;
+      case flatbuf::MetadataVersion_V3:
+        // Arrow 0.3
+        return MetadataVersion::V3;
+      // Add cases as other versions become available
+      default:
+        return MetadataVersion::V3;
+    }
+  }
+
   const void* header() const { return message_->header(); }
 
   int64_t body_length() const { return message_->bodyLength(); }
@@ -856,6 +873,10 @@ Message::Type Message::type() const {
   return impl_->type();
 }
 
+MetadataVersion Message::metadata_version() const {
+  return impl_->version();
+}
+
 int64_t Message::body_length() const {
   return impl_->body_length();
 }
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 451a76d5249e0..b042882c7cd31 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -50,9 +50,7 @@ class RandomAccessFile;
 
 namespace ipc {
 
-struct MetadataVersion {
-  enum type { V1, V2 };
-};
+enum class MetadataVersion : char { V1, V2, V3 };
 
 static constexpr const char* kArrowMagicBytes = "ARROW1";
 
@@ -134,6 +132,8 @@ class ARROW_EXPORT Message {
 
   Type type() const;
 
+  MetadataVersion metadata_version() const;
+
   const void* header() const;
 
  private:
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 00ea20cf5dfb1..55f632f306b9a 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -332,15 +332,20 @@ class FileReader::FileReaderImpl {
 
   int num_record_batches() const { return footer_->recordBatches()->size(); }
 
-  MetadataVersion::type version() const {
+  MetadataVersion version() const {
     switch (footer_->version()) {
       case flatbuf::MetadataVersion_V1:
+        // Arrow 0.1
         return MetadataVersion::V1;
       case flatbuf::MetadataVersion_V2:
+        // Arrow 0.2
         return MetadataVersion::V2;
+      case flatbuf::MetadataVersion_V3:
+        // Arrow 0.3
+        return MetadataVersion::V3;
       // Add cases as other versions become available
       default:
-        return MetadataVersion::V2;
+        return MetadataVersion::V3;
     }
   }
 
@@ -454,7 +459,7 @@ int FileReader::num_record_batches() const {
   return impl_->num_record_batches();
 }
 
-MetadataVersion::type FileReader::version() const {
+MetadataVersion FileReader::version() const {
   return impl_->version();
 }
 
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index b62f0527e0ca0..1972446743bc1 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -91,7 +91,7 @@ class ARROW_EXPORT FileReader {
 
   int num_record_batches() const;
 
-  MetadataVersion::type version() const;
+  MetadataVersion version() const;
 
   // Read a record batch from the file. Does not copy memory if the input
   // source supports zero-copy.
diff --git a/format/Schema.fbs b/format/Schema.fbs
index 958f09181bfa6..ca9c8e6c3e76c 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -21,7 +21,8 @@ namespace org.apache.arrow.flatbuf;
 
 enum MetadataVersion:short {
   V1,
-  V2
+  V2,
+  V3
 }
 
 /// These are stored in the flatbuffer in the Type union below
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index f85fb51710bde..ec7e0f2ffb115 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -329,7 +329,7 @@ private static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte heade
     Message.startMessage(builder);
     Message.addHeaderType(builder, headerType);
     Message.addHeader(builder, headerOffset);
-    Message.addVersion(builder, MetadataVersion.V2);
+    Message.addVersion(builder, MetadataVersion.V3);
     Message.addBodyLength(builder, bodyLength);
     builder.finish(Message.endMessage(builder));
     return builder.dataBuffer();

From e29a7d4cae943312a1f8598e71c5d46c1954b5fa Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 4 Apr 2017 16:22:29 -0400
Subject: [PATCH 0468/1644] ARROW-668: [Python] Box timestamp values as
 pandas.Timestamp if available, attach tzinfo

I'm not sure how to easily test the behavior if pandas is not present. I created an environment without pandas and added some fixes so that I verify the behavior, but at some point we should create a "no pandas" test suite to see what using pyarrow is like without pandas installed.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #487 from wesm/ARROW-668 and squashes the following commits:

554a647 [Wes McKinney] Remove cython from requirements.txt
649d28a [Wes McKinney] Box timestamp values as pandas.Timestamp if available, return timezone also if available
---
 python/pyarrow/array.pyx             | 25 +++------
 python/pyarrow/compat.py             | 17 +++++++
 python/pyarrow/scalar.pyx            | 47 +++++++++++++----
 python/pyarrow/tests/test_scalars.py | 76 ++++++++++++++++++----------
 4 files changed, 112 insertions(+), 53 deletions(-)

diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 67785e34075f4..1f59556e94fb8 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -29,7 +29,7 @@ cimport pyarrow.includes.pyarrow as pyarrow
 
 import pyarrow.config
 
-from pyarrow.compat import frombytes, tobytes
+from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
 from pyarrow.error cimport check_status
 from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
 
@@ -44,11 +44,6 @@ import pyarrow.schema as schema
 cimport cpython
 
 
-cdef _pandas():
-    import pandas as pd
-    return pd
-
-
 cdef maybe_coerce_datetime64(values, dtype, DataType type,
                              timestamps_to_ms=False):
 
@@ -66,7 +61,7 @@ cdef maybe_coerce_datetime64(values, dtype, DataType type,
         tz = dtype.tz
         unit = 'ms' if coerce_ms else dtype.unit
         type = schema.timestamp(unit, tz)
-    else:
+    elif type is None:
         # Trust the NumPy dtype
         type = schema.type_from_numpy_dtype(values.dtype)
 
@@ -141,15 +136,13 @@ cdef class Array:
             shared_ptr[CDataType] c_type
             CMemoryPool* pool
 
-        pd = _pandas()
-
         if mask is not None:
             mask = get_series_values(mask)
 
         values = get_series_values(obj)
         pool = maybe_unbox_memory_pool(memory_pool)
 
-        if isinstance(values, pd.Categorical):
+        if isinstance(values, Categorical):
             return DictionaryArray.from_arrays(
                 values.codes, values.categories.values,
                 mask=mask, memory_pool=memory_pool)
@@ -397,9 +390,9 @@ cdef wrap_array_output(PyObject* output):
     cdef object obj = PyObject_to_object(output)
 
     if isinstance(obj, dict):
-        return _pandas().Categorical(obj['indices'],
-                                     categories=obj['dictionary'],
-                                     fastpath=True)
+        return Categorical(obj['indices'],
+                           categories=obj['dictionary'],
+                           fastpath=True)
     else:
         return obj
 
@@ -622,14 +615,12 @@ cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor):
 
 
 cdef object get_series_values(object obj):
-    import pandas as pd
-
-    if isinstance(obj, pd.Series):
+    if isinstance(obj, PandasSeries):
         result = obj.values
     elif isinstance(obj, np.ndarray):
         result = obj
     else:
-        result = pd.Series(obj).values
+        result = PandasSeries(obj).values
 
     return result
 
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index b9206aacbc9f1..4dcc11677e7dd 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -38,9 +38,26 @@
     else:
         from pandas.types.dtypes import DatetimeTZDtype
         pdapi = pd.api.types
+
+    PandasSeries = pd.Series
+    Categorical = pd.Categorical
     HAVE_PANDAS = True
 except:
     HAVE_PANDAS = False
+    class DatetimeTZDtype(object):
+        pass
+
+    class ClassPlaceholder(object):
+
+        def __init__(self, *args, **kwargs):
+            raise NotImplementedError
+
+    class PandasSeries(ClassPlaceholder):
+        pass
+
+    class Categorical(ClassPlaceholder):
+        pass
+
 
 if PY26:
     import unittest2 as unittest
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 983a9a7334044..1c0790a4fdc3c 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -26,6 +26,12 @@ cimport cpython as cp
 
 NA = None
 
+
+cdef _pandas():
+    import pandas as pd
+    return pd
+
+
 cdef class NAType(Scalar):
 
     def __cinit__(self):
@@ -146,16 +152,37 @@ cdef class TimestampValue(ArrayValue):
             CTimestampType* dtype = <CTimestampType*>ap.type().get()
             int64_t val = ap.Value(self.index)
 
-        if dtype.unit == TimeUnit_SECOND:
-            return datetime.datetime.utcfromtimestamp(val)
-        elif dtype.unit == TimeUnit_MILLI:
-            return datetime.datetime.utcfromtimestamp(float(val) / 1000)
-        elif dtype.unit == TimeUnit_MICRO:
-            return datetime.datetime.utcfromtimestamp(float(val) / 1000000)
-        else:
-            # TimeUnit_NANO
-            raise NotImplementedError("Cannot convert nanosecond timestamps "
-                                      "to datetime.datetime")
+        timezone = None
+        tzinfo = None
+        if dtype.timezone.size() > 0:
+            timezone = frombytes(dtype.timezone)
+            import pytz
+            tzinfo = pytz.timezone(timezone)
+
+        try:
+            pd = _pandas()
+            if dtype.unit == TimeUnit_SECOND:
+                val = val * 1000000000
+            elif dtype.unit == TimeUnit_MILLI:
+                val = val * 1000000
+            elif dtype.unit == TimeUnit_MICRO:
+                val = val * 1000
+            return pd.Timestamp(val, tz=tzinfo)
+        except ImportError:
+            if dtype.unit == TimeUnit_SECOND:
+                result = datetime.datetime.utcfromtimestamp(val)
+            elif dtype.unit == TimeUnit_MILLI:
+                result = datetime.datetime.utcfromtimestamp(float(val) / 1000)
+            elif dtype.unit == TimeUnit_MICRO:
+                result = datetime.datetime.utcfromtimestamp(
+                    float(val) / 1000000)
+            else:
+                # TimeUnit_NANO
+                raise NotImplementedError("Cannot convert nanosecond "
+                                          "timestamps without pandas")
+            if timezone is not None:
+                result = result.replace(tzinfo=tzinfo)
+            return result
 
 
 cdef class FloatValue(ArrayValue):
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index a5db7e0835607..f4f275b994228 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -19,69 +19,69 @@
 import pandas as pd
 
 from pyarrow.compat import unittest, u, unicode_type
-import pyarrow as A
+import pyarrow as pa
 
 
 class TestScalars(unittest.TestCase):
 
     def test_null_singleton(self):
         with self.assertRaises(Exception):
-            A.NAType()
+            pa.NAType()
 
     def test_bool(self):
-        arr = A.from_pylist([True, None, False, None])
+        arr = pa.from_pylist([True, None, False, None])
 
         v = arr[0]
-        assert isinstance(v, A.BooleanValue)
+        assert isinstance(v, pa.BooleanValue)
         assert repr(v) == "True"
         assert v.as_py() is True
 
-        assert arr[1] is A.NA
+        assert arr[1] is pa.NA
 
     def test_int64(self):
-        arr = A.from_pylist([1, 2, None])
+        arr = pa.from_pylist([1, 2, None])
 
         v = arr[0]
-        assert isinstance(v, A.Int64Value)
+        assert isinstance(v, pa.Int64Value)
         assert repr(v) == "1"
         assert v.as_py() == 1
 
-        assert arr[2] is A.NA
+        assert arr[2] is pa.NA
 
     def test_double(self):
-        arr = A.from_pylist([1.5, None, 3])
+        arr = pa.from_pylist([1.5, None, 3])
 
         v = arr[0]
-        assert isinstance(v, A.DoubleValue)
+        assert isinstance(v, pa.DoubleValue)
         assert repr(v) == "1.5"
         assert v.as_py() == 1.5
 
-        assert arr[1] is A.NA
+        assert arr[1] is pa.NA
 
         v = arr[2]
         assert v.as_py() == 3.0
 
     def test_string_unicode(self):
-        arr = A.from_pylist([u'foo', None, u'mañana'])
+        arr = pa.from_pylist([u'foo', None, u'mañana'])
 
         v = arr[0]
-        assert isinstance(v, A.StringValue)
+        assert isinstance(v, pa.StringValue)
         assert v.as_py() == 'foo'
 
-        assert arr[1] is A.NA
+        assert arr[1] is pa.NA
 
         v = arr[2].as_py()
         assert v == u'mañana'
         assert isinstance(v, unicode_type)
 
     def test_bytes(self):
-        arr = A.from_pylist([b'foo', None, u('bar')])
+        arr = pa.from_pylist([b'foo', None, u('bar')])
 
         v = arr[0]
-        assert isinstance(v, A.BinaryValue)
+        assert isinstance(v, pa.BinaryValue)
         assert v.as_py() == b'foo'
 
-        assert arr[1] is A.NA
+        assert arr[1] is pa.NA
 
         v = arr[2].as_py()
         assert v == b'bar'
@@ -89,41 +89,65 @@ def test_bytes(self):
 
     def test_fixed_size_bytes(self):
         data = [b'foof', None, b'barb']
-        arr = A.from_pylist(data, type=A.binary(4))
+        arr = pa.from_pylist(data, type=pa.binary(4))
 
         v = arr[0]
-        assert isinstance(v, A.FixedSizeBinaryValue)
+        assert isinstance(v, pa.FixedSizeBinaryValue)
         assert v.as_py() == b'foof'
 
-        assert arr[1] is A.NA
+        assert arr[1] is pa.NA
 
         v = arr[2].as_py()
         assert v == b'barb'
         assert isinstance(v, bytes)
 
     def test_list(self):
-        arr = A.from_pylist([['foo', None], None, ['bar'], []])
+        arr = pa.from_pylist([['foo', None], None, ['bar'], []])
 
         v = arr[0]
         assert len(v) == 2
-        assert isinstance(v, A.ListValue)
+        assert isinstance(v, pa.ListValue)
         assert repr(v) == "['foo', None]"
         assert v.as_py() == ['foo', None]
         assert v[0].as_py() == 'foo'
-        assert v[1] is A.NA
+        assert v[1] is pa.NA
 
-        assert arr[1] is A.NA
+        assert arr[1] is pa.NA
 
         v = arr[3]
         assert len(v) == 0
 
+    def test_timestamp(self):
+        arr = pd.date_range('2000-01-01 12:34:56', periods=10).values
+
+        units = ['s', 'ms', 'us', 'ns']
+
+        for unit in units:
+            dtype = 'datetime64[{0}]'.format(unit)
+            arrow_arr = pa.Array.from_numpy(arr.astype(dtype))
+            expected = pd.Timestamp('2000-01-01 12:34:56')
+
+            assert arrow_arr[0].as_py() == expected
+
+            tz = 'America/New_York'
+            arrow_type = pa.timestamp(unit, tz=tz)
+
+            dtype = 'datetime64[{0}]'.format(unit)
+            arrow_arr = pa.Array.from_numpy(arr.astype(dtype),
+                                            type=arrow_type)
+            expected = (pd.Timestamp('2000-01-01 12:34:56')
+                        .tz_localize('utc')
+                        .tz_convert(tz))
+
+            assert arrow_arr[0].as_py() == expected
+
     def test_dictionary(self):
         colors = ['red', 'green', 'blue']
         values = pd.Series(colors * 4)
 
         categorical = pd.Categorical(values, categories=colors)
 
-        v = A.DictionaryArray.from_arrays(categorical.codes,
-                                          categorical.categories)
+        v = pa.DictionaryArray.from_arrays(categorical.codes,
+                                           categorical.categories)
         for i, c in enumerate(values):
             assert v[i].as_py() == c

From f4fcb42c2cb0d463db4ddeef68e4392f8d7c049f Mon Sep 17 00:00:00 2001
From: Leif Walsh <leif.walsh@gmail.com>
Date: Wed, 5 Apr 2017 13:33:25 -0400
Subject: [PATCH 0469/1644] ARROW-510 ARROW-582 ARROW-663 ARROW-729: [Java]
 Added units for Time and Date types, and integration tests

closes #366

Author: Leif Walsh <leif.walsh@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #475 from leifwalsh/feature/java-date-time-types and squashes the following commits:

2e2a4cf [Leif Walsh] ARROW-729: [Java] removed Joda DateTime getters from Date* and Time* types
47f83a8 [Wes McKinney] Integration tests for all date and time combinations
6e86422 [Wes McKinney] ARROW-733: [C++/Python] Rename FixedWidthBinary to FixedSizeBinary for consistency with FixedSizeList
2dca474 [Leif Walsh] ARROW-729: [Java] Added units for date/time types
---
 cpp/src/arrow/ipc/json-internal.cc            |  13 ++
 cpp/src/arrow/ipc/metadata.cc                 |  11 +-
 cpp/src/arrow/type-test.cc                    |   8 +-
 cpp/src/arrow/type.h                          |  15 +-
 integration/integration_test.py               | 145 ++++++++++++++----
 .../main/codegen/data/ValueVectorTypes.tdd    |  18 ++-
 .../codegen/templates/FixedValueVectors.java  |  21 +--
 .../vector/file/json/JsonFileReader.java      |  28 +++-
 .../vector/file/json/JsonFileWriter.java      |  28 +++-
 .../org/apache/arrow/vector/types/Types.java  |  95 ++++++++++--
 .../arrow/vector/types/pojo/TestSchema.java   |  40 ++++-
 11 files changed, 337 insertions(+), 85 deletions(-)

diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 1e2385b73f82c..124c21b8fc023 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -175,6 +175,8 @@ class JsonSchemaWriter {
   void WriteTypeMetadata(const TimeType& type) {
     writer_->Key("unit");
     writer_->String(GetTimeUnitName(type.unit));
+    writer_->Key("bitWidth");
+    writer_->Int(type.bit_width());
   }
 
   void WriteTypeMetadata(const DateType& type) {
@@ -608,6 +610,9 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
   const auto& json_unit = json_type.FindMember("unit");
   RETURN_NOT_STRING("unit", json_unit, json_type);
 
+  const auto& json_bit_width = json_type.FindMember("bitWidth");
+  RETURN_NOT_INT("bitWidth", json_bit_width, json_type);
+
   std::string unit_str = json_unit->value.GetString();
 
   if (unit_str == "SECOND") {
@@ -623,6 +628,14 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
     ss << "Invalid time unit: " << unit_str;
     return Status::Invalid(ss.str());
   }
+
+  const auto& fw_type = static_cast<const FixedWidthType&>(**type);
+
+  int bit_width = json_bit_width->value.GetInt();
+  if (bit_width != fw_type.bit_width()) {
+    return Status::Invalid("Indicated bit width does not match unit");
+  }
+
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 2ff25eeaa9213..d902ec296cff3 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -255,12 +255,19 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     case flatbuf::Type_Time: {
       auto time_type = static_cast<const flatbuf::Time*>(type_data);
       TimeUnit unit = FromFlatbufferUnit(time_type->unit());
+      int32_t bit_width = time_type->bitWidth();
       switch (unit) {
         case TimeUnit::SECOND:
         case TimeUnit::MILLI:
+          if (bit_width != 32) {
+            return Status::Invalid("Time is 32 bits for second/milli unit");
+          }
           *out = time32(unit);
           break;
         default:
+          if (bit_width != 64) {
+            return Status::Invalid("Time is 64 bits for micro/nano unit");
+          }
           *out = time64(unit);
           break;
       }
@@ -386,12 +393,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     case Type::TIME32: {
       const auto& time_type = static_cast<const Time32Type&>(*type);
       *out_type = flatbuf::Type_Time;
-      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
+      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
     } break;
     case Type::TIME64: {
       const auto& time_type = static_cast<const Time64Type&>(*type);
       *out_type = flatbuf::Type_Time;
-      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
+      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
     } break;
     case Type::TIMESTAMP: {
       const auto& ts_type = static_cast<const TimestampType&>(*type);
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index dafadc168c191..66164e3430913 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -191,12 +191,15 @@ TEST(TestListType, Basics) {
   ASSERT_EQ("list<item: list<item: string>>", lt2.ToString());
 }
 
-TEST(TestDateTypes, ToString) {
+TEST(TestDateTypes, Attrs) {
   auto t1 = date32();
   auto t2 = date64();
 
   ASSERT_EQ("date32[day]", t1->ToString());
   ASSERT_EQ("date64[ms]", t2->ToString());
+
+  ASSERT_EQ(32, static_cast<const FixedWidthType&>(*t1).bit_width());
+  ASSERT_EQ(64, static_cast<const FixedWidthType&>(*t2).bit_width());
 }
 
 TEST(TestTimeType, Equals) {
@@ -207,6 +210,9 @@ TEST(TestTimeType, Equals) {
   Time64Type t4(TimeUnit::NANO);
   Time64Type t5(TimeUnit::MICRO);
 
+  ASSERT_EQ(32, t0.bit_width());
+  ASSERT_EQ(64, t3.bit_width());
+
   ASSERT_TRUE(t0.Equals(t2));
   ASSERT_TRUE(t1.Equals(t1));
   ASSERT_FALSE(t1.Equals(t3));
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 6b936f348d4de..0e69133219d55 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_TYPE_H
 #define ARROW_TYPE_H
 
+#include <climits>
 #include <cstdint>
 #include <memory>
 #include <ostream>
@@ -220,7 +221,7 @@ struct ARROW_EXPORT CTypeImpl : public BASE {
 
   CTypeImpl() : BASE(TYPE_ID) {}
 
-  int bit_width() const override { return static_cast<int>(sizeof(C_TYPE) * 8); }
+  int bit_width() const override { return static_cast<int>(sizeof(C_TYPE) * CHAR_BIT); }
 
   Status Accept(TypeVisitor* visitor) const override {
     return visitor->Visit(*static_cast<const DERIVED*>(this));
@@ -456,7 +457,7 @@ struct ARROW_EXPORT Date32Type : public DateType {
 
   Date32Type();
 
-  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * CHAR_BIT); }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -470,7 +471,7 @@ struct ARROW_EXPORT Date64Type : public DateType {
 
   Date64Type();
 
-  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * CHAR_BIT); }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -509,7 +510,7 @@ struct ARROW_EXPORT Time32Type : public TimeType {
   static constexpr Type::type type_id = Type::TIME32;
   using c_type = int32_t;
 
-  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 4); }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * CHAR_BIT); }
 
   explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);
 
@@ -521,7 +522,7 @@ struct ARROW_EXPORT Time64Type : public TimeType {
   static constexpr Type::type type_id = Type::TIME64;
   using c_type = int64_t;
 
-  int bit_width() const override { return static_cast<int>(sizeof(c_type) * 8); }
+  int bit_width() const override { return static_cast<int>(sizeof(c_type) * CHAR_BIT); }
 
   explicit Time64Type(TimeUnit unit = TimeUnit::MILLI);
 
@@ -535,7 +536,7 @@ struct ARROW_EXPORT TimestampType : public FixedWidthType {
   typedef int64_t c_type;
   static constexpr Type::type type_id = Type::TIMESTAMP;
 
-  int bit_width() const override { return static_cast<int>(sizeof(int64_t) * 8); }
+  int bit_width() const override { return static_cast<int>(sizeof(int64_t) * CHAR_BIT); }
 
   explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
       : FixedWidthType(Type::TIMESTAMP), unit(unit) {}
@@ -557,7 +558,7 @@ struct ARROW_EXPORT IntervalType : public FixedWidthType {
   using c_type = int64_t;
   static constexpr Type::type type_id = Type::INTERVAL;
 
-  int bit_width() const override { return static_cast<int>(sizeof(int64_t) * 8); }
+  int bit_width() const override { return static_cast<int>(sizeof(int64_t) * CHAR_BIT); }
 
   Unit unit;
 
diff --git a/integration/integration_test.py b/integration/integration_test.py
index ec2a38d840d0b..6631dc8c2f761 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -175,10 +175,14 @@ def _get_buffers(self):
 
 class IntegerType(PrimitiveType):
 
-    def __init__(self, name, is_signed, bit_width, nullable=True):
+    def __init__(self, name, is_signed, bit_width, nullable=True,
+                 min_value=TEST_INT_MIN,
+                 max_value=TEST_INT_MAX):
         PrimitiveType.__init__(self, name, nullable=nullable)
         self.is_signed = is_signed
         self.bit_width = bit_width
+        self.min_value = min_value
+        self.max_value = max_value
 
     @property
     def numpy_type(self):
@@ -194,14 +198,80 @@ def _get_type(self):
     def generate_column(self, size):
         iinfo = np.iinfo(self.numpy_type)
         values = [int(x) for x in
-                  np.random.randint(max(iinfo.min, TEST_INT_MIN),
-                                    min(iinfo.max, TEST_INT_MAX),
+                  np.random.randint(max(iinfo.min, self.min_value),
+                                    min(iinfo.max, self.max_value),
                                     size=size)]
 
         is_valid = self._make_is_valid(size)
         return PrimitiveColumn(self.name, size, is_valid, values)
 
 
+class DateType(IntegerType):
+
+    DAY = 0
+    MILLISECOND = 1
+
+    def __init__(self, name, unit, nullable=True):
+        self.unit = unit
+        bit_width = 32 if unit == self.DAY else 64
+        IntegerType.__init__(self, name, True, bit_width, nullable=nullable)
+
+    def _get_type(self):
+        return OrderedDict([
+            ('name', 'date'),
+            ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
+        ])
+
+
+TIMEUNIT_NAMES = {
+    's': 'SECOND',
+    'ms': 'MILLISECOND',
+    'us': 'MICROSECOND',
+    'ns': 'NANOSECOND'
+}
+
+
+class TimeType(IntegerType):
+
+    BIT_WIDTHS = {
+        's': 32,
+        'ms': 32,
+        'us': 64,
+        'ns': 64
+    }
+
+    def __init__(self, name, unit='s', nullable=True):
+        self.unit = unit
+        IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
+                             nullable=nullable)
+
+    def _get_type(self):
+        return OrderedDict([
+            ('name', 'time'),
+            ('unit', TIMEUNIT_NAMES[self.unit]),
+            ('bitWidth', self.bit_width)
+        ])
+
+
+class TimestampType(IntegerType):
+
+    def __init__(self, name, unit='s', tz=None, nullable=True):
+        self.unit = unit
+        self.tz = tz
+        IntegerType.__init__(self, name, True, 64, nullable=nullable)
+
+    def _get_type(self):
+        fields = [
+            ('name', 'timestamp'),
+            ('unit', TIMEUNIT_NAMES[self.unit])
+        ]
+
+        if self.tz is not None:
+            fields.append(('timezone', self.tz))
+
+        return OrderedDict(fields)
+
+
 class FloatingPointType(PrimitiveType):
 
     def __init__(self, name, bit_width, nullable=True):
@@ -509,6 +579,20 @@ def get_field(name, type_, nullable=True):
         raise TypeError(dtype)
 
 
+def _generate_file(fields, batch_sizes):
+    schema = JSONSchema(fields)
+    batches = []
+    for size in batch_sizes:
+        columns = []
+        for field in fields:
+            col = field.generate_column(size)
+            columns.append(col)
+
+        batches.append(JSONRecordBatch(size, columns))
+
+    return JSONFile(schema, batches)
+
+
 def generate_primitive_case():
     types = ['bool', 'int8', 'int16', 'int32', 'int64',
              'uint8', 'uint16', 'uint32', 'uint64',
@@ -520,19 +604,27 @@ def generate_primitive_case():
         fields.append(get_field(type_ + "_nullable", type_, True))
         fields.append(get_field(type_ + "_nonnullable", type_, False))
 
-    schema = JSONSchema(fields)
-
     batch_sizes = [7, 10]
-    batches = []
-    for size in batch_sizes:
-        columns = []
-        for field in fields:
-            col = field.generate_column(size)
-            columns.append(col)
+    return _generate_file(fields, batch_sizes)
 
-        batches.append(JSONRecordBatch(size, columns))
 
-    return JSONFile(schema, batches)
+def generate_datetime_case():
+    fields = [
+        DateType('f0', DateType.DAY),
+        DateType('f1', DateType.MILLISECOND),
+        TimeType('f2', 's'),
+        TimeType('f3', 'ms'),
+        TimeType('f4', 'us'),
+        TimeType('f5', 'ns'),
+        TimestampType('f6', 's'),
+        TimestampType('f7', 'ms'),
+        TimestampType('f8', 'us'),
+        TimestampType('f9', 'ns'),
+        TimestampType('f10', 'ms', tz='America/New_York')
+    ]
+
+    batch_sizes = [7, 10]
+    return _generate_file(fields, batch_sizes)
 
 
 def generate_nested_case():
@@ -545,19 +637,8 @@ def generate_nested_case():
         # ListType('list_nonnullable', get_field('item', 'int32'), False),
     ]
 
-    schema = JSONSchema(fields)
-
     batch_sizes = [7, 10]
-    batches = []
-    for size in batch_sizes:
-        columns = []
-        for field in fields:
-            col = field.generate_column(size)
-            columns.append(col)
-
-        batches.append(JSONRecordBatch(size, columns))
-
-    return JSONFile(schema, batches)
+    return _generate_file(fields, batch_sizes)
 
 
 def get_generated_json_files():
@@ -566,13 +647,13 @@ def get_generated_json_files():
     def _temp_path():
         return
 
-    file_objs = []
-
-    K = 10
-    for i in range(K):
-        file_objs.append(generate_primitive_case())
-
-    file_objs.append(generate_nested_case())
+    file_objs = [
+        generate_primitive_case(),
+        generate_primitive_case(),
+        generate_primitive_case(),
+        generate_datetime_case(),
+        generate_nested_case()
+    ]
 
     generated_paths = []
     for file_obj in file_objs:
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index 2181cfdc335b4..b08c100edcac8 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -56,8 +56,10 @@
         { class: "Int", valueHolder: "IntHolder"},
         { class: "UInt4", valueHolder: "UInt4Holder" },
         { class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]},
-        { class: "IntervalYear", javaType: "int", friendlyType: "Period" }
-        { class: "Time", javaType: "int", friendlyType: "DateTime" }
+        { class: "DateDay" },
+        { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
+        { class: "TimeSec" },
+        { class: "TimeMilli" }
       ]
     },
     {
@@ -70,11 +72,13 @@
         { class: "BigInt"},
         { class: "UInt8" },
         { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
-        { class: "Date", javaType: "long", friendlyType: "DateTime" },
-        { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
-        { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
-        { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
-        { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
+        { class: "DateMilli" },
+        { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
+        { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
+        { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
+        { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
+        { class: "TimeMicro" },
+        { class: "TimeNano" }
       ]
     },
     {
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index d5265f1140ee0..947c82c74a401 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -19,6 +19,7 @@
 import org.apache.arrow.vector.util.DecimalUtility;
 
 import java.lang.Override;
+import java.util.concurrent.TimeUnit;
 
 <@pp.dropOutputFile />
 <#list vv.types as type>
@@ -482,12 +483,15 @@ public long getTwoAsLong(int index) {
 
     </#if>
 
-    <#if minor.class == "Date">
+    <#if minor.class == "DateDay" ||
+         minor.class == "DateMilli" ||
+         minor.class == "TimeSec" ||
+         minor.class == "TimeMilli" ||
+         minor.class == "TimeMicro" ||
+         minor.class == "TimeNano">
     @Override
     public ${friendlyType} getObject(int index) {
-        org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
-        return date;
+      return get(index);
     }
 
     <#elseif minor.class == "TimeStampSec">
@@ -554,15 +558,6 @@ public StringBuilder getAsStringBuilder(int index) {
              append(months).append(monthString));
     }
 
-    <#elseif minor.class == "Time">
-    @Override
-    public DateTime getObject(int index) {
-
-        org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
-        return time;
-    }
-
     <#elseif minor.class == "Decimal9" || minor.class == "Decimal18">
     @Override
     public ${friendlyType} getObject(int index) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index bdb63b92cb105..2f91205cffcbd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -32,15 +32,21 @@
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.DateDayVector;
+import org.apache.arrow.vector.DateMilliVector;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.Float4Vector;
 import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.SmallIntVector;
-import org.apache.arrow.vector.TimeStampSecVector;
-import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeMicroVector;
+import org.apache.arrow.vector.TimeMilliVector;
+import org.apache.arrow.vector.TimeNanoVector;
+import org.apache.arrow.vector.TimeSecVector;
 import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
 import org.apache.arrow.vector.TimeStampNanoVector;
+import org.apache.arrow.vector.TimeStampSecVector;
 import org.apache.arrow.vector.TinyIntVector;
 import org.apache.arrow.vector.UInt1Vector;
 import org.apache.arrow.vector.UInt2Vector;
@@ -240,6 +246,24 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
     case VARCHAR:
       ((VarCharVector)valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8));
       break;
+    case DATEDAY:
+      ((DateDayVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case DATEMILLI:
+      ((DateMilliVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case TIMESEC:
+      ((TimeSecVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case TIMEMILLI:
+      ((TimeMilliVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+      break;
+    case TIMEMICRO:
+      ((TimeMicroVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case TIMENANO:
+      ((TimeNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
     case TIMESTAMPSEC:
       ((TimeStampSecVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
       break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index 99040b67e1cd3..d86b3de3b9da3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -23,11 +23,17 @@
 
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.DateDayVector;
+import org.apache.arrow.vector.DateMilliVector;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.TimeStampSecVector;
-import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeMicroVector;
+import org.apache.arrow.vector.TimeMilliVector;
+import org.apache.arrow.vector.TimeNanoVector;
+import org.apache.arrow.vector.TimeSecVector;
 import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
 import org.apache.arrow.vector.TimeStampNanoVector;
+import org.apache.arrow.vector.TimeStampSecVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VarBinaryVector;
@@ -144,6 +150,24 @@ private void writeVector(Field field, FieldVector vector) throws IOException {
 
   private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException {
     switch (valueVector.getMinorType()) {
+      case DATEDAY:
+        generator.writeNumber(((DateDayVector)valueVector).getAccessor().get(i));
+        break;
+      case DATEMILLI:
+        generator.writeNumber(((DateMilliVector)valueVector).getAccessor().get(i));
+        break;
+      case TIMESEC:
+        generator.writeNumber(((TimeSecVector)valueVector).getAccessor().get(i));
+        break;
+      case TIMEMILLI:
+        generator.writeNumber(((TimeMilliVector)valueVector).getAccessor().get(i));
+        break;
+      case TIMEMICRO:
+        generator.writeNumber(((TimeMicroVector)valueVector).getAccessor().get(i));
+        break;
+      case TIMENANO:
+        generator.writeNumber(((TimeNanoVector)valueVector).getAccessor().get(i));
+        break;
       case TIMESTAMPSEC:
         generator.writeNumber(((TimeStampSecVector)valueVector).getAccessor().get(i));
         break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index f07bb585f810c..b0455fa14e44c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -25,7 +25,8 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.NullableBigIntVector;
 import org.apache.arrow.vector.NullableBitVector;
-import org.apache.arrow.vector.NullableDateVector;
+import org.apache.arrow.vector.NullableDateDayVector;
+import org.apache.arrow.vector.NullableDateMilliVector;
 import org.apache.arrow.vector.NullableDecimalVector;
 import org.apache.arrow.vector.NullableFloat4Vector;
 import org.apache.arrow.vector.NullableFloat8Vector;
@@ -33,11 +34,14 @@
 import org.apache.arrow.vector.NullableIntervalDayVector;
 import org.apache.arrow.vector.NullableIntervalYearVector;
 import org.apache.arrow.vector.NullableSmallIntVector;
+import org.apache.arrow.vector.NullableTimeMicroVector;
+import org.apache.arrow.vector.NullableTimeMilliVector;
+import org.apache.arrow.vector.NullableTimeNanoVector;
+import org.apache.arrow.vector.NullableTimeSecVector;
 import org.apache.arrow.vector.NullableTimeStampMicroVector;
 import org.apache.arrow.vector.NullableTimeStampMilliVector;
 import org.apache.arrow.vector.NullableTimeStampNanoVector;
 import org.apache.arrow.vector.NullableTimeStampSecVector;
-import org.apache.arrow.vector.NullableTimeVector;
 import org.apache.arrow.vector.NullableTinyIntVector;
 import org.apache.arrow.vector.NullableUInt1Vector;
 import org.apache.arrow.vector.NullableUInt2Vector;
@@ -52,7 +56,8 @@
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.BitWriterImpl;
-import org.apache.arrow.vector.complex.impl.DateWriterImpl;
+import org.apache.arrow.vector.complex.impl.DateDayWriterImpl;
+import org.apache.arrow.vector.complex.impl.DateMilliWriterImpl;
 import org.apache.arrow.vector.complex.impl.DecimalWriterImpl;
 import org.apache.arrow.vector.complex.impl.Float4WriterImpl;
 import org.apache.arrow.vector.complex.impl.Float8WriterImpl;
@@ -61,11 +66,14 @@
 import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl;
 import org.apache.arrow.vector.complex.impl.NullableMapWriter;
 import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeMicroWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeMilliWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeNanoWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeSecWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl;
-import org.apache.arrow.vector.complex.impl.TimeWriterImpl;
 import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.UInt1WriterImpl;
 import org.apache.arrow.vector.complex.impl.UInt2WriterImpl;
@@ -164,26 +172,70 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new BigIntWriterImpl((NullableBigIntVector) vector);
       }
     },
-    DATE(new Date(DateUnit.MILLISECOND)) {
+    DATEDAY(new Date(DateUnit.DAY)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableDateVector(name, fieldType, allocator);
+        return new NullableDateDayVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new DateWriterImpl((NullableDateVector) vector);
+        return new DateDayWriterImpl((NullableDateDayVector) vector);
       }
     },
-    TIME(new Time(TimeUnit.MILLISECOND, 32)) {
+    DATEMILLI(new Date(DateUnit.MILLISECOND)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeVector(name, fieldType, allocator);
+        return new NullableDateMilliVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeWriterImpl((NullableTimeVector) vector);
+        return new DateMilliWriterImpl((NullableDateMilliVector) vector);
+      }
+    },
+    TIMESEC(new Time(TimeUnit.SECOND, 32)) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeSecVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeSecWriterImpl((NullableTimeSecVector) vector);
+      }
+    },
+    TIMEMILLI(new Time(TimeUnit.MILLISECOND, 32)) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeMilliVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeMilliWriterImpl((NullableTimeMilliVector) vector);
+      }
+    },
+    TIMEMICRO(new Time(TimeUnit.MICROSECOND, 64)) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeMicroVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeMicroWriterImpl((NullableTimeMicroVector) vector);
+      }
+    },
+    TIMENANO(new Time(TimeUnit.NANOSECOND, 64)) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeNanoVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeNanoWriterImpl((NullableTimeNanoVector) vector);
       }
     },
     // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
@@ -479,14 +531,29 @@ public MinorType visit(FloatingPoint type) {
       }
 
       @Override public MinorType visit(Date type) {
-        return MinorType.DATE;
+        switch (type.getUnit()) {
+          case DAY:
+            return MinorType.DATEDAY;
+          case MILLISECOND:
+            return MinorType.DATEMILLI;
+          default:
+            throw new IllegalArgumentException("unknown unit: " + type);
+        }
       }
 
       @Override public MinorType visit(Time type) {
-        if (type.getUnit() != TimeUnit.MILLISECOND || type.getBitWidth() != 32) {
-          throw new IllegalArgumentException("Only milliseconds on 32 bits supported for now: " + type);
+        switch (type.getUnit()) {
+          case SECOND:
+            return MinorType.TIMESEC;
+          case MILLISECOND:
+            return MinorType.TIMEMILLI;
+          case MICROSECOND:
+            return MinorType.TIMEMICRO;
+          case NANOSECOND:
+            return MinorType.TIMENANO;
+          default:
+            throw new IllegalArgumentException("unknown unit: " + type);
         }
-        return MinorType.TIME;
       }
 
       @Override public MinorType visit(Timestamp type) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 45f3b5656d861..56fa73eccebf0 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -86,11 +86,15 @@ public void testAll() throws IOException {
         field("h", new Binary()),
         field("i", new Bool()),
         field("j", new Decimal(5, 5)),
-        field("k", new Date(DateUnit.MILLISECOND)),
-        field("l", new Time(TimeUnit.MILLISECOND, 32)),
-        field("m", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
-        field("n", new Timestamp(TimeUnit.MICROSECOND, null)),
-        field("o", new Interval(IntervalUnit.DAY_TIME))
+        field("k", new Date(DateUnit.DAY)),
+        field("l", new Date(DateUnit.MILLISECOND)),
+        field("m", new Time(TimeUnit.SECOND, 32)),
+        field("n", new Time(TimeUnit.MILLISECOND, 32)),
+        field("o", new Time(TimeUnit.MICROSECOND, 64)),
+        field("p", new Time(TimeUnit.NANOSECOND, 64)),
+        field("q", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
+        field("r", new Timestamp(TimeUnit.MICROSECOND, null)),
+        field("s", new Interval(IntervalUnit.DAY_TIME))
         ));
     roundTrip(schema);
   }
@@ -104,6 +108,32 @@ public void testUnion() throws IOException {
     contains(schema, "Sparse");
   }
 
+  @Test
+  public void testDate() throws IOException {
+    Schema schema = new Schema(asList(
+        field("a", new Date(DateUnit.DAY)),
+        field("b", new Date(DateUnit.MILLISECOND))
+        ));
+    roundTrip(schema);
+    assertEquals(
+        "Schema<a: Date(DAY), b: Date(MILLISECOND)>",
+        schema.toString());
+  }
+
+  @Test
+  public void testTime() throws IOException {
+    Schema schema = new Schema(asList(
+            field("a", new Time(TimeUnit.SECOND, 32)),
+            field("b", new Time(TimeUnit.MILLISECOND, 32)),
+            field("c", new Time(TimeUnit.MICROSECOND, 64)),
+            field("d", new Time(TimeUnit.NANOSECOND, 64))
+    ));
+    roundTrip(schema);
+    assertEquals(
+            "Schema<a: Time(SECOND, 32), b: Time(MILLISECOND, 32), c: Time(MICROSECOND, 64), d: Time(NANOSECOND, 64)>",
+            schema.toString());
+  }
+
   @Test
   public void testTS() throws IOException {
     Schema schema = new Schema(asList(

From ddf880b312c1b11739d09bc014d4649b8f2f26d4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 6 Apr 2017 09:10:13 -0400
Subject: [PATCH 0470/1644] ARROW-752: [Python] Support boxed Arrow arrays as
 input to DictionaryArray.from_arrays

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #496 from wesm/ARROW-752 and squashes the following commits:

2f57574 [Wes McKinney] Support boxed Arrow arrays as input to DictionaryArray.from_arrays
---
 python/pyarrow/array.pyx           | 31 ++++++++----
 python/pyarrow/tests/test_array.py | 81 ++++++++++++++++++++++++------
 2 files changed, 89 insertions(+), 23 deletions(-)

diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 1f59556e94fb8..9f302e02cdb04 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -497,8 +497,12 @@ cdef class DictionaryArray(Array):
 
     cdef getitem(self, int64_t i):
         cdef Array dictionary = self.dictionary
-        cdef int64_t index = self.indices[i].as_py()
-        return scalar.box_scalar(dictionary.type, dictionary.sp_array, index)
+        index = self.indices[i]
+        if index is NA:
+            return index
+        else:
+            return scalar.box_scalar(dictionary.type, dictionary.sp_array,
+                                     index.as_py())
 
     property dictionary:
 
@@ -544,15 +548,24 @@ cdef class DictionaryArray(Array):
             shared_ptr[CDataType] c_type
             shared_ptr[CArray] c_result
 
-        if mask is None:
-            mask = indices == -1
+        if isinstance(indices, Array):
+            if mask is not None:
+                raise NotImplementedError(
+                    "mask not implemented with Arrow array inputs yet")
+            arrow_indices = indices
         else:
-            mask = mask | (indices == -1)
+            if mask is None:
+                mask = indices == -1
+            else:
+                mask = mask | (indices == -1)
+            arrow_indices = Array.from_numpy(indices, mask=mask,
+                                             memory_pool=memory_pool)
 
-        arrow_indices = Array.from_numpy(indices, mask=mask,
-                                         memory_pool=memory_pool)
-        arrow_dictionary = Array.from_numpy(dictionary,
-                                            memory_pool=memory_pool)
+        if isinstance(dictionary, Array):
+            arrow_dictionary = dictionary
+        else:
+            arrow_dictionary = Array.from_numpy(dictionary,
+                                                memory_pool=memory_pool)
 
         if not isinstance(arrow_indices, IntegerArray):
             raise ValueError('Indices must be integer type')
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index d8b2e2f5d80d6..57b17f6cea756 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -15,30 +15,33 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import pytest
 import sys
 
-import pytest
+import numpy as np
+import pandas as pd
+import pandas.util.testing as tm
 
-import pyarrow
+import pyarrow as pa
 import pyarrow.formatting as fmt
 
 
 def test_total_bytes_allocated():
-    assert pyarrow.total_allocated_bytes() == 0
+    assert pa.total_allocated_bytes() == 0
 
 
 def test_repr_on_pre_init_array():
-    arr = pyarrow.array.Array()
+    arr = pa.Array()
     assert len(repr(arr)) > 0
 
 
 def test_getitem_NA():
-    arr = pyarrow.from_pylist([1, None, 2])
-    assert arr[1] is pyarrow.NA
+    arr = pa.from_pylist([1, None, 2])
+    assert arr[1] is pa.NA
 
 
 def test_list_format():
-    arr = pyarrow.from_pylist([[1], None, [2, 3, None]])
+    arr = pa.from_pylist([[1], None, [2, 3, None]])
     result = fmt.array_format(arr)
     expected = """\
 [
@@ -52,7 +55,7 @@ def test_list_format():
 
 
 def test_string_format():
-    arr = pyarrow.from_pylist(['', None, 'foo'])
+    arr = pa.from_pylist(['', None, 'foo'])
     result = fmt.array_format(arr)
     expected = """\
 [
@@ -64,7 +67,7 @@ def test_string_format():
 
 
 def test_long_array_format():
-    arr = pyarrow.from_pylist(range(100))
+    arr = pa.from_pylist(range(100))
     result = fmt.array_format(arr, window=2)
     expected = """\
 [
@@ -80,7 +83,7 @@ def test_long_array_format():
 def test_to_pandas_zero_copy():
     import gc
 
-    arr = pyarrow.from_pylist(range(10))
+    arr = pa.from_pylist(range(10))
 
     for i in range(10):
         np_arr = arr.to_pandas()
@@ -90,7 +93,7 @@ def test_to_pandas_zero_copy():
     assert sys.getrefcount(arr) == 2
 
     for i in range(10):
-        arr = pyarrow.from_pylist(range(10))
+        arr = pa.from_pylist(range(10))
         np_arr = arr.to_pandas()
         arr = None
         gc.collect()
@@ -105,14 +108,14 @@ def test_to_pandas_zero_copy():
 
 
 def test_array_slice():
-    arr = pyarrow.from_pylist(range(10))
+    arr = pa.from_pylist(range(10))
 
     sliced = arr.slice(2)
-    expected = pyarrow.from_pylist(range(2, 10))
+    expected = pa.from_pylist(range(2, 10))
     assert sliced.equals(expected)
 
     sliced2 = arr.slice(2, 4)
-    expected2 = pyarrow.from_pylist(range(2, 6))
+    expected2 = pa.from_pylist(range(2, 6))
     assert sliced2.equals(expected2)
 
     # 0 offset
@@ -136,3 +139,53 @@ def test_array_slice():
 
     with pytest.raises(IndexError):
         arr[::2]
+
+
+def test_dictionary_from_numpy():
+    indices = np.repeat([0, 1, 2], 2)
+    dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
+    mask = np.array([False, False, True, False, False, False])
+
+    d1 = pa.DictionaryArray.from_arrays(indices, dictionary)
+    d2 = pa.DictionaryArray.from_arrays(indices, dictionary, mask=mask)
+
+    for i in range(len(indices)):
+        assert d1[i].as_py() == dictionary[indices[i]]
+
+        if mask[i]:
+            assert d2[i] is pa.NA
+        else:
+            assert d2[i].as_py() == dictionary[indices[i]]
+
+
+def test_dictionary_from_boxed_arrays():
+    indices = np.repeat([0, 1, 2], 2)
+    dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
+
+    iarr = pa.Array.from_numpy(indices)
+    darr = pa.Array.from_numpy(dictionary)
+
+    d1 = pa.DictionaryArray.from_arrays(iarr, darr)
+
+    for i in range(len(indices)):
+        assert d1[i].as_py() == dictionary[indices[i]]
+
+
+def test_dictionary_with_pandas():
+    indices = np.repeat([0, 1, 2], 2)
+    dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
+    mask = np.array([False, False, True, False, False, False])
+
+    d1 = pa.DictionaryArray.from_arrays(indices, dictionary)
+    d2 = pa.DictionaryArray.from_arrays(indices, dictionary, mask=mask)
+
+    pandas1 = d1.to_pandas()
+    ex_pandas1 = pd.Categorical.from_codes(indices, categories=dictionary)
+
+    tm.assert_series_equal(pd.Series(pandas1), pd.Series(ex_pandas1))
+
+    pandas2 = d2.to_pandas()
+    ex_pandas2 = pd.Categorical.from_codes(np.where(mask, -1, indices),
+                                           categories=dictionary)
+
+    tm.assert_series_equal(pd.Series(pandas2), pd.Series(ex_pandas2))

From 621d52740b52af4042c4aaa3ac424c5916aa94da Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 6 Apr 2017 09:33:17 -0400
Subject: [PATCH 0471/1644] ARROW-582: [Java] Added JSON reader/writer unit
 test for date, time, and timestamp

New unit test to verify Java JSON reader/writer round-trip with date, time and timestamp types

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #495 from BryanCutler/java-json-DateTime-Test-ARROW-582 and squashes the following commits:

e80683b [Bryan Cutler] added JSON read and write support unit test
---
 .../vector/file/json/JsonFileReader.java      |  2 +-
 .../arrow/vector/file/BaseFileTest.java       | 46 +++++++++++++++++++
 .../arrow/vector/file/json/TestJSONFile.java  | 38 ++++++++++++++-
 3 files changed, 84 insertions(+), 2 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 2f91205cffcbd..fde9954d288bb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -275,7 +275,7 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
       break;
     case TIMESTAMPNANO:
       ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-    break;
+      break;
     default:
       throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 774bead3207a7..5c68a1904be70 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -32,8 +32,12 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.DateMilliWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
+import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter;
 import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.Assert;
@@ -138,6 +142,48 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) {
     }
   }
 
+  private DateTime makeDateTimeFromCount(int i) {
+    return new DateTime(2000 + i, 1 + i, 1 + i, i, i, i, i, DateTimeZone.UTC);
+  }
+
+  protected void writeDateTimeData(int count, NullableMapVector parent) {
+    Assert.assertTrue(count < 100);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    DateMilliWriter dateWriter = rootWriter.dateMilli("date");
+    TimeMilliWriter timeWriter = rootWriter.timeMilli("time");
+    TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli");
+    for (int i = 0; i < count; i++) {
+      DateTime dt = makeDateTimeFromCount(i);
+      // Number of days in milliseconds since epoch, stored as 64-bit integer, only date part is used
+      dateWriter.setPosition(i);
+      long dateLong = dt.minusMillis(dt.getMillisOfDay()).getMillis();
+      dateWriter.writeDateMilli(dateLong);
+      // Time is a value in milliseconds since midnight, stored as 32-bit integer
+      timeWriter.setPosition(i);
+      timeWriter.writeTimeMilli(dt.getMillisOfDay());
+      // Timestamp is milliseconds since the epoch, stored as 64-bit integer
+      timeStampMilliWriter.setPosition(i);
+      timeStampMilliWriter.writeTimeStampMilli(dt.getMillis());
+    }
+    writer.setValueCount(count);
+  }
+
+  protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
+    Assert.assertEquals(count, root.getRowCount());
+    printVectors(root.getFieldVectors());
+    for (int i = 0; i < count; i++) {
+      Object dateVal = root.getVector("date").getAccessor().getObject(i);
+      DateTime dt = makeDateTimeFromCount(i);
+      DateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
+      Assert.assertEquals(dateExpected.getMillis(), dateVal);
+      Object timeVal = root.getVector("time").getAccessor().getObject(i);
+      Assert.assertEquals(dt.getMillisOfDay(), timeVal);
+      Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
+      Assert.assertTrue(dt.withZoneRetainFields(DateTimeZone.getDefault()).equals(timestampMilliVal));
+    }
+  }
+
   protected void writeData(int count, MapVector parent) {
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index c88958cbf2c9c..6369c07c3205c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -103,7 +103,7 @@ public void testWriteReadUnionJSON() throws IOException {
 
       writeJSON(file, root);
     }
- // read
+    // read
     try (
         BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
         BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
@@ -119,6 +119,42 @@ public void testWriteReadUnionJSON() throws IOException {
     }
   }
 
+  @Test
+  public void testWriteReadDateTimeJSON() throws IOException {
+    File file = new File("target/mytest_datetime.json");
+    int count = COUNT;
+
+    // write
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+
+      writeDateTimeData(count, parent);
+
+      printVectors(parent.getChildrenFromFields());
+
+      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
+      validateDateTimeContent(count, root);
+
+      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")));
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+    ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateDateTimeContent(count, root);
+      }
+      reader.close();
+    }
+  }
+
   @Test
   public void testSetStructLength() throws IOException {
     File file = new File("../../integration/data/struct_example.json");

From 49b3e0e2a08f633238875d9663ad5745fbb52db1 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 6 Apr 2017 10:13:58 -0400
Subject: [PATCH 0472/1644] ARROW-774: [GLib] Remove needless LICENSE.txt copy

"make dist" result is included in source archive. LICENSE.txt is also
included the source archive.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #497 from kou/glib-remove-needless-license-copy and squashes the following commits:

a2fd0b9 [Kouhei Sutou] [GLib] Remove needless LICENSE.txt copy
---
 c_glib/Makefile.am | 4 +---
 c_glib/autogen.sh  | 2 --
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am
index 40e8395a56824..bb52ce503e04e 100644
--- a/c_glib/Makefile.am
+++ b/c_glib/Makefile.am
@@ -24,10 +24,8 @@ SUBDIRS =					\
 
 EXTRA_DIST =					\
 	README.md				\
-	LICENSE.txt				\
 	version
 
 arrow_glib_docdir = ${datarootdir}/doc/arrow-glib
 arrow_glib_doc_DATA =				\
-	README.md				\
-	LICENSE.txt
+	README.md
diff --git a/c_glib/autogen.sh b/c_glib/autogen.sh
index 6e2036da6406b..08e33e6ca07c0 100755
--- a/c_glib/autogen.sh
+++ b/c_glib/autogen.sh
@@ -25,8 +25,6 @@ ruby \
     ../java/pom.xml > \
     version
 
-cp ../LICENSE.txt ./
-
 mkdir -p m4
 
 gtkdocize --copy --docdir doc/reference

From ff744ef13c6dff42abf4a0a3ca697634f84b9bf8 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Thu, 6 Apr 2017 09:07:35 -0700
Subject: [PATCH 0473/1644] ARROW-775: add simple constructors to value vectors

Author: Julien Le Dem <julien@apache.org>

Closes #498 from julienledem/ARROW-775 and squashes the following commits:

badf8d1 [Julien Le Dem] ARROW-775: add simple constructors to value vectors
---
 .../src/main/codegen/templates/NullableValueVectors.java  | 8 ++++++++
 .../java/org/apache/arrow/vector/complex/ListVector.java  | 4 ++++
 .../apache/arrow/vector/complex/NullableMapVector.java    | 4 ++++
 .../java/org/apache/arrow/vector/util/DateUtility.java    | 8 ++++----
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 8e1727ca6c820..a50771a45a034 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -64,6 +64,14 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   <#if minor.class == "Decimal">
   private final int precision;
   private final int scale;
+
+  public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
+    this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
+  }
+  <#else>
+  public ${className}(String name, BufferAllocator allocator) {
+    this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
+  }
   </#if>
 
   public ${className}(String name, FieldType fieldType, BufferAllocator allocator) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index d138ca339e3cf..0461a8d9d285a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -64,6 +64,10 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector {
   private CallBack callBack;
   private final DictionaryEncoding dictionary;
 
+  public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
+    this(name, allocator, null, callBack);
+  }
+
   public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
     super(name, allocator, callBack);
     this.bits = new BitVector("$bits$", allocator);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 7fe35e8253afb..71fee67d49c9f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -54,6 +54,10 @@ public class NullableMapVector extends MapVector implements FieldVector {
   private final Accessor accessor;
   private final Mutator mutator;
 
+  public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+    this(name, allocator, null, callBack);
+  }
+
   public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
     super(name, checkNotNull(allocator), callBack);
     this.bits = new BitVector("$bits$", allocator);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
index f4fc1736032c0..1f8ce069cf9cf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
@@ -618,10 +618,10 @@ public class DateUtility {
       }
     }
 
-    public static final DateTimeFormatter formatDate        = DateTimeFormat.forPattern("yyyy-MM-dd");
-    public static final DateTimeFormatter formatTimeStamp    = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
-    public static final DateTimeFormatter formatTimeStampTZ = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ");
-    public static final DateTimeFormatter formatTime        = DateTimeFormat.forPattern("HH:mm:ss.SSS");
+    public static final DateTimeFormatter formatDate           = DateTimeFormat.forPattern("yyyy-MM-dd");
+    public static final DateTimeFormatter formatTimeStampMilli = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+    public static final DateTimeFormatter formatTimeStampTZ    = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ");
+    public static final DateTimeFormatter formatTime           = DateTimeFormat.forPattern("HH:mm:ss.SSS");
 
     public static DateTimeFormatter dateTimeTZFormat = null;
     public static DateTimeFormatter timeFormat = null;

From 56f1e91d2961a13b7f677785fa705bed06d9639d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 6 Apr 2017 13:49:32 -0400
Subject: [PATCH 0474/1644] ARROW-771: [Python] Add read_row_group /
 num_row_groups to ParquetFile

requires PARQUET-946 https://github.com/apache/parquet-cpp/pull/291

cc @cpcloud @jreback @mrocklin

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #494 from wesm/ARROW-771 and squashes the following commits:

126789a [Wes McKinney] Fix docstring
1009423 [Wes McKinney] Add read_row_group / num_row_groups to ParquetFile
---
 python/pyarrow/_parquet.pxd          | 17 ++++++---
 python/pyarrow/_parquet.pyx          | 37 ++++++++++++++-----
 python/pyarrow/parquet.py            | 53 ++++++++++++++++++++--------
 python/pyarrow/tests/test_parquet.py | 29 +++++++++++++++
 4 files changed, 109 insertions(+), 27 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index f12c86fdebc83..1ac1f69b033ce 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -179,7 +179,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 
         @staticmethod
         unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
-        shared_ptr[CFileMetaData] metadata();
+        shared_ptr[CFileMetaData] metadata()
 
 
 cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
@@ -211,11 +211,18 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 
     cdef cppclass FileReader:
         FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
-        CStatus ReadColumn(int i, shared_ptr[CArray]* out);
-        CStatus ReadTable(shared_ptr[CTable]* out);
+        CStatus ReadColumn(int i, shared_ptr[CArray]* out)
+
+        int num_row_groups()
+        CStatus ReadRowGroup(int i, shared_ptr[CTable]* out)
+        CStatus ReadRowGroup(int i, const vector[int]& column_indices,
+                             shared_ptr[CTable]* out)
+
+        CStatus ReadTable(shared_ptr[CTable]* out)
         CStatus ReadTable(const vector[int]& column_indices,
-                              shared_ptr[CTable]* out);
-        const ParquetFileReader* parquet_reader();
+                          shared_ptr[CTable]* out)
+
+        const ParquetFileReader* parquet_reader()
 
         void set_num_threads(int num_threads)
 
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index cfd2816e2a16e..079bf5ee5924a 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -31,7 +31,7 @@ from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
 from pyarrow.io import NativeFile
 from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
-from pyarrow.table cimport Table
+from pyarrow.table cimport Table, table_from_ctable
 
 from pyarrow.io cimport NativeFile, get_reader, get_writer
 
@@ -381,16 +381,39 @@ cdef class ParquetReader:
         result.init(metadata)
         return result
 
-    def read(self, column_indices=None, nthreads=1):
+    property num_row_groups:
+
+        def __get__(self):
+            return self.reader.get().num_row_groups()
+
+    def set_num_threads(self, int nthreads):
+        self.reader.get().set_num_threads(nthreads)
+
+    def read_row_group(self, int i, column_indices=None):
         cdef:
-            Table table = Table()
             shared_ptr[CTable] ctable
             vector[int] c_column_indices
 
-        self.reader.get().set_num_threads(nthreads)
+        if column_indices is not None:
+            for index in column_indices:
+                c_column_indices.push_back(index)
+
+            with nogil:
+                check_status(self.reader.get()
+                             .ReadRowGroup(i, c_column_indices, &ctable))
+        else:
+            # Read all columns
+            with nogil:
+                check_status(self.reader.get()
+                             .ReadRowGroup(i, &ctable))
+        return table_from_ctable(ctable)
+
+    def read_all(self, column_indices=None):
+        cdef:
+            shared_ptr[CTable] ctable
+            vector[int] c_column_indices
 
         if column_indices is not None:
-            # Read only desired column indices
             for index in column_indices:
                 c_column_indices.push_back(index)
 
@@ -402,9 +425,7 @@ cdef class ParquetReader:
             with nogil:
                 check_status(self.reader.get()
                              .ReadTable(&ctable))
-
-        table.init(ctable)
-        return table
+        return table_from_ctable(ctable)
 
     def column_name_idx(self, column_name):
         """
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 2985316f35f01..d95c3b3aecaf8 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -50,7 +50,32 @@ def metadata(self):
     def schema(self):
         return self.metadata.schema
 
-    def read(self, nrows=None, columns=None, nthreads=1):
+    @property
+    def num_row_groups(self):
+        return self.reader.num_row_groups
+
+    def read_row_group(self, i, columns=None, nthreads=1):
+        """
+        Read a single row group from a Parquet file
+
+        Parameters
+        ----------
+        columns: list
+            If not None, only these columns will be read from the row group.
+        nthreads : int, default 1
+            Number of columns to read in parallel. If > 1, requires that the
+            underlying file source is threadsafe
+
+        Returns
+        -------
+        pyarrow.table.Table
+            Content of the row group as a table (of columns)
+        """
+        column_indices = self._get_column_indices(columns)
+        self.reader.set_num_threads(nthreads)
+        return self.reader.read_row_group(i, column_indices=column_indices)
+
+    def read(self, columns=None, nthreads=1):
         """
         Read a Table from Parquet format
 
@@ -67,17 +92,16 @@ def read(self, nrows=None, columns=None, nthreads=1):
         pyarrow.table.Table
             Content of the file as a table (of columns)
         """
-        if nrows is not None:
-            raise NotImplementedError("nrows argument")
+        column_indices = self._get_column_indices(columns)
+        self.reader.set_num_threads(nthreads)
+        return self.reader.read_all(column_indices=column_indices)
 
-        if columns is None:
-            column_indices = None
+    def _get_column_indices(self, column_names):
+        if column_names is None:
+            return None
         else:
-            column_indices = [self.reader.column_name_idx(column)
-                              for column in columns]
-
-        return self.reader.read(column_indices=column_indices,
-                                nthreads=nthreads)
+            return [self.reader.column_name_idx(column)
+                    for column in column_names]
 
 
 def read_table(source, columns=None, nthreads=1, metadata=None):
@@ -178,8 +202,8 @@ def open_file(path, meta=None):
     return all_data
 
 
-def write_table(table, sink, chunk_size=None, version='1.0',
-                use_dictionary=True, compression='snappy'):
+def write_table(table, sink, row_group_size=None, version='1.0',
+                use_dictionary=True, compression='snappy', **kwargs):
     """
     Write a Table to Parquet format
 
@@ -187,7 +211,7 @@ def write_table(table, sink, chunk_size=None, version='1.0',
     ----------
     table : pyarrow.Table
     sink: string or pyarrow.io.NativeFile
-    chunk_size : int, default None
+    row_group_size : int, default None
         The maximum number of rows in each Parquet RowGroup. As a default,
         we will write a single RowGroup per file.
     version : {"1.0", "2.0"}, default "1.0"
@@ -198,7 +222,8 @@ def write_table(table, sink, chunk_size=None, version='1.0',
     compression : str or dict
         Specify the compression codec, either on a general basis or per-column.
     """
+    row_group_size = kwargs.get('chunk_size', row_group_size)
     writer = ParquetWriter(sink, use_dictionary=use_dictionary,
                            compression=compression,
                            version=version)
-    writer.write_table(table, row_group_size=chunk_size)
+    writer.write_table(table, row_group_size=row_group_size)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index b8b2800259caf..86165be7052c6 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -402,6 +402,35 @@ def test_pass_separate_metadata():
     pdt.assert_frame_equal(df, fileh.read().to_pandas())
 
 
+@parquet
+def test_read_single_row_group():
+    # ARROW-471
+    N, K = 10000, 4
+    df = alltypes_sample(size=N)
+
+    a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+
+    buf = io.BytesIO()
+    pq.write_table(a_table, buf, row_group_size=N / K,
+                   compression='snappy', version='2.0')
+
+    buf.seek(0)
+
+    pf = pq.ParquetFile(buf)
+
+    assert pf.num_row_groups == K
+
+    row_groups = [pf.read_row_group(i) for i in range(K)]
+    result = pa.concat_tables(row_groups)
+    pdt.assert_frame_equal(df, result.to_pandas())
+
+    cols = df.columns[:2]
+    row_groups = [pf.read_row_group(i, columns=cols)
+                  for i in range(K)]
+    result = pa.concat_tables(row_groups)
+    pdt.assert_frame_equal(df[cols], result.to_pandas())
+
+
 @parquet
 def test_read_multiple_files(tmpdir):
     nfiles = 10

From 58fa4c2fcc75f763a89b44eeedafade771d342e8 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 6 Apr 2017 20:36:29 +0200
Subject: [PATCH 0475/1644] ARROW-776: [GLib] Fix wrong type name

Author: Kouhei Sutou <kou@clear-code.com>

Closes #499 from kou/glib-fix-wrong-type-name and squashes the following commits:

105f2f2 [Kouhei Sutou] [GLib] Fix wrong type name
---
 c_glib/arrow-glib/io-file.h               | 2 +-
 c_glib/arrow-glib/io-input-stream.h       | 2 +-
 c_glib/arrow-glib/io-output-stream.h      | 2 +-
 c_glib/arrow-glib/io-random-access-file.h | 2 +-
 c_glib/arrow-glib/io-readable.h           | 2 +-
 c_glib/arrow-glib/io-writeable.h          | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/c_glib/arrow-glib/io-file.h b/c_glib/arrow-glib/io-file.h
index 9fa0ec137566f..7181f6d37aeb3 100644
--- a/c_glib/arrow-glib/io-file.h
+++ b/c_glib/arrow-glib/io-file.h
@@ -28,7 +28,7 @@ G_BEGIN_DECLS
 #define GARROW_IO_FILE(obj)                             \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
                               GARROW_IO_TYPE_FILE,      \
-                              GArrowIOFileInterface))
+                              GArrowIOFile))
 #define GARROW_IO_IS_FILE(obj)                          \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
                               GARROW_IO_TYPE_FILE))
diff --git a/c_glib/arrow-glib/io-input-stream.h b/c_glib/arrow-glib/io-input-stream.h
index a7f06819b4f97..57902095010c8 100644
--- a/c_glib/arrow-glib/io-input-stream.h
+++ b/c_glib/arrow-glib/io-input-stream.h
@@ -28,7 +28,7 @@ G_BEGIN_DECLS
 #define GARROW_IO_INPUT_STREAM(obj)                             \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
                               GARROW_IO_TYPE_INPUT_STREAM,      \
-                              GArrowIOInputStreamInterface))
+                              GArrowIOInputStream))
 #define GARROW_IO_IS_INPUT_STREAM(obj)                          \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
                               GARROW_IO_TYPE_INPUT_STREAM))
diff --git a/c_glib/arrow-glib/io-output-stream.h b/c_glib/arrow-glib/io-output-stream.h
index c4079d50233cd..02478ce9621eb 100644
--- a/c_glib/arrow-glib/io-output-stream.h
+++ b/c_glib/arrow-glib/io-output-stream.h
@@ -28,7 +28,7 @@ G_BEGIN_DECLS
 #define GARROW_IO_OUTPUT_STREAM(obj)                            \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
                               GARROW_IO_TYPE_OUTPUT_STREAM,     \
-                              GArrowIOOutputStreamInterface))
+                              GArrowIOOutputStream))
 #define GARROW_IO_IS_OUTPUT_STREAM(obj)                         \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
                               GARROW_IO_TYPE_OUTPUT_STREAM))
diff --git a/c_glib/arrow-glib/io-random-access-file.h b/c_glib/arrow-glib/io-random-access-file.h
index e980ab2e3c8e5..8ac63e417a3f2 100644
--- a/c_glib/arrow-glib/io-random-access-file.h
+++ b/c_glib/arrow-glib/io-random-access-file.h
@@ -28,7 +28,7 @@ G_BEGIN_DECLS
 #define GARROW_IO_RANDOM_ACCESS_FILE(obj)                            \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                                 \
                               GARROW_IO_TYPE_RANDOM_ACCESS_FILE,     \
-                              GArrowIORandomAccessFileInterface))
+                              GArrowIORandomAccessFile))
 #define GARROW_IO_IS_RANDOM_ACCESS_FILE(obj)                            \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
                               GARROW_IO_TYPE_RANDOM_ACCESS_FILE))
diff --git a/c_glib/arrow-glib/io-readable.h b/c_glib/arrow-glib/io-readable.h
index d24b46c50df4c..279984b3014a3 100644
--- a/c_glib/arrow-glib/io-readable.h
+++ b/c_glib/arrow-glib/io-readable.h
@@ -28,7 +28,7 @@ G_BEGIN_DECLS
 #define GARROW_IO_READABLE(obj)                                 \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
                               GARROW_IO_TYPE_READABLE,          \
-                              GArrowIOReadableInterface))
+                              GArrowIOReadable))
 #define GARROW_IO_IS_READABLE(obj)                      \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
                               GARROW_IO_TYPE_READABLE))
diff --git a/c_glib/arrow-glib/io-writeable.h b/c_glib/arrow-glib/io-writeable.h
index f5c5e9129f8be..ce23247497706 100644
--- a/c_glib/arrow-glib/io-writeable.h
+++ b/c_glib/arrow-glib/io-writeable.h
@@ -28,7 +28,7 @@ G_BEGIN_DECLS
 #define GARROW_IO_WRITEABLE(obj)                                \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
                               GARROW_IO_TYPE_WRITEABLE,         \
-                              GArrowIOWriteableInterface))
+                              GArrowIOWriteable))
 #define GARROW_IO_IS_WRITEABLE(obj)                             \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
                               GARROW_IO_TYPE_WRITEABLE))

From e371ebd7e16e5e5f4b14f0f578049d9246714e77 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 6 Apr 2017 15:19:59 -0400
Subject: [PATCH 0476/1644] ARROW-756: [C++] MSVC build fixes and cleanup,
 remove -fPIC flag from EP builds on Windows, Dev docs

Includes existing patch for ARROW-757 and closes #492

With this patch I'm able to build with

```
cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ..
nmake
```

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Max Risuhin <risuhin.max@gmail.com>

Closes #500 from wesm/ARROW-757-2 and squashes the following commits:

106e454 [Wes McKinney] Notes about MSVC solution file
f34adf2 [Wes McKinney] Windows developer guide
43e5f3f [Wes McKinney] More MSVC cleaning / fixes. Remove -fPIC flags from builds
ec7805e [Max Risuhin] ARROW-757: [C++] Resolve nmake build issues on Windows
---
 cpp/CMakeLists.txt                    | 40 +++++++------
 cpp/README.md                         |  5 +-
 cpp/cmake_modules/SetupCxxFlags.cmake | 25 ++++++--
 cpp/doc/Windows.md                    | 83 +++++++++++++++++++++++++++
 4 files changed, 129 insertions(+), 24 deletions(-)
 create mode 100644 cpp/doc/Windows.md

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d26c847807d79..9947a34e4e7bb 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -146,9 +146,11 @@ include(BuildUtils)
 include(SetupCxxFlags)
 
 # Add common flags
-set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMMON_FLAGS}")
 set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-set(CMAKE_CXX_FLAGS "${ARROW_CXXFLAGS} ${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}")
+
+message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
 
 # Determine compiler version
 include(CompilerInfo)
@@ -446,7 +448,7 @@ if(ARROW_BUILD_TESTS)
   if("$ENV{GTEST_HOME}" STREQUAL "")
     if(APPLE)
       set(GTEST_CMAKE_CXX_FLAGS "-fPIC -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
-    else()
+    elseif(NOT MSVC)
       set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
     endif()
     string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
@@ -456,12 +458,15 @@ if(ARROW_BUILD_TESTS)
     set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
     set(GTEST_STATIC_LIB "${GTEST_PREFIX}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GTEST_VENDORED 1)
+    set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                         -Dgtest_force_shared_crt=ON
+                         -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
 
     if (CMAKE_VERSION VERSION_GREATER "3.2")
       # BUILD_BYPRODUCTS is a 3.2+ feature
       ExternalProject_Add(googletest_ep
         URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-        CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON
+        CMAKE_ARGS ${GTEST_CMAKE_ARGS}
         # googletest doesn't define install rules, so just build in the
         # source dir and don't try to install.  See its README for
         # details.
@@ -471,7 +476,7 @@ if(ARROW_BUILD_TESTS)
     else()
       ExternalProject_Add(googletest_ep
         URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-        CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON
+        CMAKE_ARGS ${GTEST_CMAKE_ARGS}
         # googletest doesn't define install rules, so just build in the
         # source dir and don't try to install.  See its README for
         # details.
@@ -556,9 +561,9 @@ if(ARROW_BUILD_BENCHMARKS)
 
   if("$ENV{GBENCHMARK_HOME}" STREQUAL "")
     if(APPLE)
-      set(GBENCHMARK_CMAKE_CXX_FLAGS "-std=c++11 -stdlib=libc++")
-    else()
-      set(GBENCHMARK_CMAKE_CXX_FLAGS "--std=c++11")
+      set(GBENCHMARK_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++")
+    elseif(NOT MSVC)
+      set(GBENCHMARK_CMAKE_CXX_FLAGS "-fPIC --std=c++11")
     endif()
 
     set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install")
@@ -569,7 +574,7 @@ if(ARROW_BUILD_BENCHMARKS)
           "-DCMAKE_BUILD_TYPE=Release"
           "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}"
           "-DBENCHMARK_ENABLE_TESTING=OFF"
-          "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}")
+          "-DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}")
     if (APPLE)
       set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON")
     endif()
@@ -621,6 +626,13 @@ endif()
 message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
 include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
 
+#----------------------------------------------------------------------
+
+if (MSVC)
+  # jemalloc is not supported on Windows
+  set(ARROW_JEMALLOC off)
+endif()
+
 if (ARROW_JEMALLOC)
   find_package(jemalloc)
 
@@ -840,12 +852,10 @@ if(ARROW_IPC)
     ExternalProject_Add(flatbuffers_ep
       URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
       CMAKE_ARGS
-        "-DCMAKE_CXX_FLAGS=-fPIC"
         "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
         "-DFLATBUFFERS_BUILD_TESTS=OFF")
 
     set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
-    set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_PREFIX}/libflatbuffers.a")
     set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
     set(FLATBUFFERS_VENDORED 1)
   else()
@@ -854,16 +864,8 @@ if(ARROW_IPC)
   endif()
 
   message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
-  message(STATUS "Flatbuffers static library: ${FLATBUFFERS_STATIC_LIB}")
   message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
   include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(flatbuffers
-    STATIC_LIB ${FLATBUFFERS_STATIC_LIB})
-
-  if(FLATBUFFERS_VENDORED)
-    add_dependencies(flatbuffers flatbuffers_ep)
-  endif()
-
   add_subdirectory(src/arrow/ipc)
 endif()
 
diff --git a/cpp/README.md b/cpp/README.md
index b6f0fa0e3531b..b19fa001198a4 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -40,6 +40,8 @@ On OS X, you can use [Homebrew][1]:
 brew install boost cmake
 ```
 
+If you are developing on Windows, see the [Windows developer guide][2].
+
 ## Building Arrow
 
 Simple debug build:
@@ -123,4 +125,5 @@ both of these options would be used rarely.  Current known uses-cases whent hey
 
 *  Parameterized tests in google test.
 
-[1]: https://brew.sh/
\ No newline at end of file
+[1]: https://brew.sh/
+[2]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
\ No newline at end of file
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index ee672bd5f6a96..09a662ec6e583 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -24,15 +24,32 @@ CHECK_CXX_COMPILER_FLAG("-msse3" CXX_SUPPORTS_SSE3)
 CHECK_CXX_COMPILER_FLAG("-maltivec" CXX_SUPPORTS_ALTIVEC)
 
 # compiler flags that are common across debug/release builds
-#  - Wall: Enable all warnings.
-set(CXX_COMMON_FLAGS "-std=c++11 -Wall")
+
+if (MSVC)
+  if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+    # clang-cl
+    set(CXX_COMMON_FLAGS "-EHsc")
+  elseif(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 19)
+    message(FATAL_ERROR "Only MSVC 2015 (Version 19.0) and later are supported
+    by Arrow. Found version ${CMAKE_CXX_COMPILER_VERSION}.")
+  else()
+    # Fix annoying D9025 warning
+    string(REPLACE "/W3" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+
+    # Set desired warning level (e.g. set /W4 for more warnings)
+    set(CXX_COMMON_FLAGS "/W3")
+  endif()
+else()
+  set(CXX_COMMON_FLAGS "-Wall -std=c++11")
+endif()
 
 # Only enable additional instruction sets if they are supported
 if (CXX_SUPPORTS_SSE3 AND ARROW_SSE3)
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse3")
+  set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse3")
 endif()
+
 if (CXX_SUPPORTS_ALTIVEC AND ARROW_ALTIVEC)
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -maltivec")
+  set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -maltivec")
 endif()
 
 if (APPLE)
diff --git a/cpp/doc/Windows.md b/cpp/doc/Windows.md
new file mode 100644
index 0000000000000..64f6a1b98f62e
--- /dev/null
+++ b/cpp/doc/Windows.md
@@ -0,0 +1,83 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Developing Arrow C++ on Windows
+
+## System setup, conda, and conda-forge
+
+Since some of the Arrow developers work in the Python ecosystem, we are
+investing time in maintaining the thirdparty build dependencies for Arrow and
+related C++ libraries using the conda package manager. Others are free to add
+other development instructions for Windows here.
+
+### Visual Studio
+
+Microsoft provides the free Visual Studio 2017 Community edition. When doing
+development, you must launch the developer command prompt using
+
+```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64```
+
+It's easiest to configure a console emulator like [cmder][3] to automatically
+launch this when starting a new development console.
+
+### conda and package toolchain
+
+[Miniconda][1] is a minimal Python distribution including the conda package
+manager. To get started, download and install a 64-bit distribution.
+
+We recommend using packages from [conda-forge][2]
+
+```shell
+conda config --add channels conda-forge
+```
+
+Now, you can bootstrap a build environment
+
+```shell
+conda create -n arrow-dev cmake git boost
+```
+
+## Building with NMake
+
+Activate your conda build environment:
+
+```
+activate arrow-dev
+```
+
+Now, do an out of source build using `nmake`:
+
+```
+cd cpp
+mkdir build
+cd build
+cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ..
+nmake
+```
+
+When using conda, only release builds are currently supported.
+
+## Build using Visual Studio (MSVC) Solution Files
+
+To build on the command line by instead generating a MSVC solution, instead
+run:
+
+```
+cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=Release ..
+cmake --build . --config Release
+```
+
+[1]: https://conda.io/miniconda.html
+[2]: https://conda-forge.github.io/
+[3]: http://cmder.net/
\ No newline at end of file

From e53357cd610f1bdca0cbbac001e417f329d54be1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 6 Apr 2017 18:49:06 -0400
Subject: [PATCH 0477/1644] ARROW-778: Port merge tool to work on Windows

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #501 from wesm/ARROW-778 and squashes the following commits:

a554320 [Wes McKinney] Use os.path.sep for splitting paths
---
 dev/merge_arrow_pr.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index 39db254a9f25d..99ccc43394f27 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -42,8 +42,9 @@
     JIRA_IMPORTED = False
 
 # Location of your Arrow git clone
-ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
-PROJECT_NAME = ARROW_HOME.rsplit("/", 1)[1]
+SEP = os.path.sep
+ARROW_HOME = os.path.abspath(__file__).rsplit(SEP, 2)[0]
+PROJECT_NAME = ARROW_HOME.rsplit(SEP, 1)[1]
 print("ARROW_HOME = " + ARROW_HOME)
 print("PROJECT_NAME = " + PROJECT_NAME)
 

From 1c6609746aeb9584fc83284f2587fa97bdbac47a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 7 Apr 2017 10:08:08 -0400
Subject: [PATCH 0478/1644] ARROW-758: [C++] Build with /WX in Appveyor, fix
 MSVC compiler warnings

This will help keep the build clean.

cc @MaxRis

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #502 from wesm/ARROW-758 and squashes the following commits:

054c185 [Wes McKinney] Build with /WX in Appveyor, fix MSVC compiler warnings
---
 appveyor.yml                          |  6 +-----
 cpp/cmake_modules/SetupCxxFlags.cmake |  4 ++++
 cpp/src/arrow/array-test.cc           | 14 +++++++-------
 cpp/src/arrow/builder.h               |  4 ++++
 cpp/src/arrow/io/file.cc              |  6 +++---
 cpp/src/arrow/io/io-hdfs-test.cc      |  2 +-
 cpp/src/arrow/ipc/json-internal.cc    |  2 +-
 cpp/src/arrow/ipc/test-common.h       |  2 +-
 cpp/src/arrow/tensor.cc               |  2 +-
 9 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 9f3594907d17e..b8c26e6e5084c 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -30,12 +30,8 @@ build_script:
  - cd cpp
  - mkdir build
  - cd build
- # A lot of features are still deactivated as they do not build on Windows
- #  * gbenchmark doesn't build with MSVC
- - cmake -G "%GENERATOR%" -DARROW_BOOST_USE_SHARED=OFF -DARROW_BUILD_BENCHMARKS=OFF -DARROW_JEMALLOC=OFF -DCMAKE_BUILD_TYPE=Release ..
+ - cmake -G "%GENERATOR%" -DARROW_CXXFLAGS="/WX" -DARROW_BOOST_USE_SHARED=OFF -DCMAKE_BUILD_TYPE=Release ..
  - cmake --build . --config Release
 
 # test_script:
  - ctest -VV
-
-
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 09a662ec6e583..694e5a37df4ba 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -26,6 +26,10 @@ CHECK_CXX_COMPILER_FLAG("-maltivec" CXX_SUPPORTS_ALTIVEC)
 # compiler flags that are common across debug/release builds
 
 if (MSVC)
+  # TODO(wesm): Change usages of C runtime functions that MSVC says are
+  # insecure, like std::getenv
+  add_definitions(-D_CRT_SECURE_NO_WARNINGS)
+
   if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
     # clang-cl
     set(CXX_COMMON_FLAGS "-EHsc")
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 68b9864301d20..e50f4fd10b087 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -124,10 +124,10 @@ TEST_F(TestArray, SliceRecomputeNullCount) {
 TEST_F(TestArray, TestIsNull) {
   // clang-format off
   vector<uint8_t> null_bitmap = {1, 0, 1, 1, 0, 1, 0, 0,
-                                      1, 0, 1, 1, 0, 1, 0, 0,
-                                      1, 0, 1, 1, 0, 1, 0, 0,
-                                      1, 0, 1, 1, 0, 1, 0, 0,
-                                      1, 0, 0, 1};
+                                 1, 0, 1, 1, 0, 1, 0, 0,
+                                 1, 0, 1, 1, 0, 1, 0, 0,
+                                 1, 0, 1, 1, 0, 1, 0, 0,
+                                 1, 0, 0, 1};
   // clang-format on
   int64_t null_count = 0;
   for (uint8_t x : null_bitmap) {
@@ -144,7 +144,7 @@ TEST_F(TestArray, TestIsNull) {
   ASSERT_TRUE(arr->null_bitmap()->Equals(*null_buf.get()));
 
   for (size_t i = 0; i < null_bitmap.size(); ++i) {
-    EXPECT_EQ(null_bitmap[i], !arr->IsNull(i)) << i;
+    EXPECT_EQ(null_bitmap[i] != 0, !arr->IsNull(i)) << i;
   }
 }
 
@@ -334,7 +334,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
   for (int64_t i = 0; i < result->length(); ++i) {
     if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
     bool actual = BitUtil::GetBit(result->data()->data(), i);
-    ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
+    ASSERT_EQ(draws_[i] != 0, actual) << i;
   }
   ASSERT_TRUE(result->Equals(*expected));
 }
@@ -1379,7 +1379,7 @@ void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& valu
   }
 
   for (int i = 0; i < result->length(); ++i) {
-    ASSERT_EQ(!static_cast<bool>(is_valid[i]), result->IsNull(i));
+    ASSERT_EQ(is_valid[i] == 0, result->IsNull(i));
   }
 
   ASSERT_EQ(7, result->values()->length());
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 61207a334db32..60cdc4cb3a5db 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -275,6 +275,10 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
+  Status Append(uint8_t val) {
+    return Append(val != 0);
+  }
+
   /// Vector append
   ///
   /// If passed, valid_bytes is of equal length to values, and any zero byte
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 720be3d6e739c..eb4b9fc43884f 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -250,9 +250,9 @@ static inline Status FileRead(
     int fd, uint8_t* buffer, int64_t nbytes, int64_t* bytes_read) {
 #if defined(_MSC_VER)
   if (nbytes > INT32_MAX) { return Status::IOError("Unable to read > 2GB blocks yet"); }
-  *bytes_read = _read(fd, buffer, static_cast<size_t>(nbytes));
+  *bytes_read = static_cast<int64_t>(_read(fd, buffer, static_cast<uint32_t>(nbytes)));
 #else
-  *bytes_read = read(fd, buffer, static_cast<size_t>(nbytes));
+  *bytes_read = static_cast<int64_t>(read(fd, buffer, static_cast<size_t>(nbytes)));
 #endif
 
   if (*bytes_read == -1) {
@@ -269,7 +269,7 @@ static inline Status FileWrite(int fd, const uint8_t* buffer, int64_t nbytes) {
   if (nbytes > INT32_MAX) {
     return Status::IOError("Unable to write > 2GB blocks to file yet");
   }
-  ret = static_cast<int>(_write(fd, buffer, static_cast<size_t>(nbytes)));
+  ret = static_cast<int>(_write(fd, buffer, static_cast<uint32_t>(nbytes)));
 #else
   ret = static_cast<int>(write(fd, buffer, static_cast<size_t>(nbytes)));
 #endif
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index f3140be0b2dac..a2c9c5210b10d 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -107,7 +107,7 @@ class TestHdfsClient : public ::testing::Test {
     const char* port = std::getenv("ARROW_HDFS_TEST_PORT");
     const char* user = std::getenv("ARROW_HDFS_TEST_USER");
 
-    ASSERT_TRUE(user) << "Set ARROW_HDFS_TEST_USER";
+    ASSERT_TRUE(user != nullptr) << "Set ARROW_HDFS_TEST_USER";
 
     conf_.host = host == nullptr ? "localhost" : host;
     conf_.user = user;
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 124c21b8fc023..fe0a7c94226f0 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -1102,7 +1102,7 @@ class JsonArrayReader {
     std::vector<bool> is_valid;
     for (const rj::Value& val : json_validity) {
       DCHECK(val.IsInt());
-      is_valid.push_back(static_cast<bool>(val.GetInt()));
+      is_valid.push_back(val.GetInt() != 0);
     }
 
 #define TYPE_CASE(TYPE) \
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index d113531822c96..9e0480d4c3634 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -125,7 +125,7 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
     std::partial_sum(list_sizes.begin(), list_sizes.end(), ++offsets.begin());
 
     // Force invariants
-    const int64_t child_length = child_array->length();
+    const int32_t child_length = static_cast<int32_t>(child_array->length());
     offsets[0] = 0;
     std::replace_if(offsets.begin(), offsets.end(),
         [child_length](int32_t offset) { return offset > child_length; }, child_length);
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index 8bbb97b596e18..d1c4083289f96 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -86,7 +86,7 @@ const std::string& Tensor::dim_name(int i) const {
 }
 
 int64_t Tensor::size() const {
-  return std::accumulate(shape_.begin(), shape_.end(), 1, std::multiplies<int64_t>());
+  return std::accumulate(shape_.begin(), shape_.end(), 1LL, std::multiplies<int64_t>());
 }
 
 bool Tensor::is_contiguous() const {

From 027c6b8084961cf10d80927c8380cce7a23acc1f Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 7 Apr 2017 10:16:47 -0400
Subject: [PATCH 0479/1644] ARROW-781 [C++/Python] Increase reference count of
 the numpy base array?

see https://issues.apache.org/jira/browse/ARROW-781

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #503 from pcmoritz/numpy-base-object and squashes the following commits:

207e439 [Philipp Moritz] add test for numpy base object
e96c89a [Philipp Moritz] increase reference count of the numpy base array
---
 cpp/src/arrow/python/numpy_convert.cc | 1 +
 python/pyarrow/tests/test_tensor.py   | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 3697819120dbe..23470fbc41aca 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -258,6 +258,7 @@ Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out) {
 
   if (base != Py_None) {
     PyArray_SetBaseObject(reinterpret_cast<PyArrayObject*>(result), base);
+    Py_XINCREF(base);
   }
   *out = result;
   return Status::OK();
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index 5327f1a74a33e..a39064b49dfbc 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -16,6 +16,7 @@
 # under the License.
 
 import os
+import sys
 import pytest
 
 import numpy as np
@@ -41,6 +42,12 @@ def test_tensor_attrs():
     tensor = pa.Tensor.from_numpy(data2)
     assert not tensor.is_mutable
 
+def test_tensor_base_object():
+    tensor = pa.Tensor.from_numpy(np.random.randn(10, 4))
+    n = sys.getrefcount(tensor)
+    array = tensor.to_numpy()
+    assert sys.getrefcount(tensor) == n + 1
+
 
 @pytest.mark.parametrize('dtype_str,arrow_type', [
     ('i1', pa.int8()),

From 8ae3283b2ecdf6cb5a6d7e97753781128a57512d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 7 Apr 2017 16:49:39 -0400
Subject: [PATCH 0480/1644] ARROW-787: [GLib] Fix compilation error caused by
 introducing BooleanBuilder::Append overload

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #506 from wesm/ARROW-787 and squashes the following commits:

e0edb47 [Wes McKinney] Fix compilation error caused by introducing BooleanBuilder::Append overload
---
 c_glib/arrow-glib/boolean-array-builder.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/boolean-array-builder.cpp b/c_glib/arrow-glib/boolean-array-builder.cpp
index 1a4c1f9fd8f7e..146eb31e8bdf8 100644
--- a/c_glib/arrow-glib/boolean-array-builder.cpp
+++ b/c_glib/arrow-glib/boolean-array-builder.cpp
@@ -84,7 +84,7 @@ garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
     static_cast<arrow::BooleanBuilder *>(
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
-  auto status = arrow_builder->Append(value);
+  auto status = arrow_builder->Append(static_cast<bool>(value));
   if (status.ok()) {
     return TRUE;
   } else {

From 35911037031e46784f4e585ac5922642351660c1 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 8 Apr 2017 09:22:16 -0400
Subject: [PATCH 0481/1644] ARROW-793: [GLib] Fix indent

Author: Kouhei Sutou <kou@clear-code.com>

Closes #509 from kou/glib-fix-indent and squashes the following commits:

5453fb6 [Kouhei Sutou] [GLib] Fix indent
---
 c_glib/arrow-glib/uint32-array.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/uint32-array.cpp b/c_glib/arrow-glib/uint32-array.cpp
index d10f10005f9be..18a9aedc0658f 100644
--- a/c_glib/arrow-glib/uint32-array.cpp
+++ b/c_glib/arrow-glib/uint32-array.cpp
@@ -60,7 +60,7 @@ garrow_uint32_array_class_init(GArrowUInt32ArrayClass *klass)
  */
 guint32
 garrow_uint32_array_get_value(GArrowUInt32Array *array,
-                             gint64 i)
+                              gint64 i)
 {
   auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
   return static_cast<arrow::UInt32Array *>(arrow_array.get())->Value(i);

From b0e3122b904924dc86cf470edfa726e38bd14f83 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 9 Apr 2017 13:52:20 -0400
Subject: [PATCH 0482/1644] ARROW-724: Add How to Contribute section to README

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #504 from wesm/ARROW-724 and squashes the following commits:

c1e07b6 [Wes McKinney] Typo
3a6083c [Wes McKinney] Add how to contribute section modifed from parquet-mr
---
 README.md | 43 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 1eb3f86f98656..2790895878563 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@
   </tr>
 </table>
 
-#### Powering Columnar In-Memory Analytics
+### Powering Columnar In-Memory Analytics
 
 Arrow is a set of technologies that enable big-data systems to process and move data fast.
 
@@ -39,7 +39,7 @@ Initial implementations include:
 Arrow is an [Apache Software Foundation](www.apache.org) project. Learn more at
 [arrow.apache.org](http://arrow.apache.org).
 
-#### What's in the Arrow libraries?
+### What's in the Arrow libraries?
 
 The reference Arrow implementations contain a number of distinct software
 components:
@@ -59,12 +59,7 @@ components:
 - Conversions to and from other in-memory data structures (e.g. Python's pandas
   library)
 
-#### Getting involved
-
-Right now the primary audience for Apache Arrow are the developers of data
-systems; most people will use Apache Arrow indirectly through systems that use
-it for internal data handling and interoperating with other Arrow-enabled
-systems.
+### Getting involved
 
 Even if you do not plan to contribute to Apache Arrow itself or Arrow
 integrations in other projects, we'd be happy to have you involved:
@@ -76,6 +71,38 @@ integrations in other projects, we'd be happy to have you involved:
 - [Learn the format][2]
 - Contribute code to one of the reference implementations
 
+### How to Contribute
+
+We prefer to receive contributions in the form of GitHub pull requests. Please
+send pull requests against the [github.com/apache/arrow][4] repository.
+
+If you are looking for some ideas on what to contribute, check out the [JIRA
+issues][3] for the Apache Arrow project. Comment on the issue and/or contact
+[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+with your questions and ideas.
+
+If you’d like to report a bug but don’t have time to fix it, you can still post
+it on JIRA, or email the mailing list
+[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+
+To contribute a patch:
+
+1. Break your work into small, single-purpose patches if possible. It’s much
+harder to merge in a large change with a lot of disjoint features.
+2. Create a JIRA for your patch on the [Arrow Project
+JIRA](https://issues.apache.org/jira/browse/ARROW).
+3. Submit the patch as a GitHub pull request against the master branch. For a
+tutorial, see the GitHub guides on forking a repo and sending a pull
+request. Prefix your pull request name with the JIRA name (ex:
+https://github.com/apache/arrow/pull/240).
+4. Make sure that your code passes the unit tests. You can find instructions
+how to run the unit tests for each Arrow component in its respective README
+file.
+5. Add new unit tests for your code.
+
+Thank you in advance for your contributions!
+
 [1]: mailto:dev-subscribe@arrow.apache.org
 [2]: https://github.com/apache/arrow/tree/master/format
 [3]: https://issues.apache.org/jira/browse/ARROW
+[4]: https://github.com/apache/arrow
\ No newline at end of file

From 739ed82028e9efae43f00f4e19b39737adb8a348 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 9 Apr 2017 13:54:10 -0400
Subject: [PATCH 0483/1644] ARROW-762: [Python] Start docs page about files and
 filesystems, adapt C++ docs about HDFS

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #511 from wesm/ARROW-762 and squashes the following commits:

273142e [Wes McKinney] Add initial docs about configuring environment to use pyarrow.HdfsClient
---
 python/doc/filesystems.rst | 58 ++++++++++++++++++++++++++++++++++++++
 python/doc/index.rst       | 12 ++++----
 2 files changed, 64 insertions(+), 6 deletions(-)
 create mode 100644 python/doc/filesystems.rst

diff --git a/python/doc/filesystems.rst b/python/doc/filesystems.rst
new file mode 100644
index 0000000000000..9e00ddd558127
--- /dev/null
+++ b/python/doc/filesystems.rst
@@ -0,0 +1,58 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+File interfaces and Memory Maps
+===============================
+
+PyArrow features a number of file-like interfaces
+
+Hadoop File System (HDFS)
+-------------------------
+
+PyArrow comes with bindings to a C++-based interface to the Hadoop File
+System. You connect like so:
+
+.. code-block:: python
+
+   import pyarrow as pa
+   hdfs = pa.HdfsClient(host, port, user=user, kerb_ticket=ticket_cache_path)
+
+By default, ``pyarrow.HdfsClient`` uses libhdfs, a JNI-based interface to the
+Java Hadoop client. This library is loaded **at runtime** (rather than at link
+/ library load time, since the library may not be in your LD_LIBRARY_PATH), and
+relies on some environment variables.
+
+* ``HADOOP_HOME``: the root of your installed Hadoop distribution. Often has
+  `lib/native/libhdfs.so`.
+
+* ``JAVA_HOME``: the location of your Java SDK installation.
+
+* ``ARROW_LIBHDFS_DIR`` (optional): explicit location of ``libhdfs.so`` if it is
+  installed somewhere other than ``$HADOOP_HOME/lib/native``.
+
+* ``CLASSPATH``: must contain the Hadoop jars. You can set these using:
+
+.. code-block:: shell
+
+    export CLASSPATH=`$HADOOP_HOME/bin/hdfs classpath --glob`
+
+You can also use libhdfs3, a thirdparty C++ library for HDFS from Pivotal Labs:
+
+.. code-block:: python
+
+   hdfs3 = pa.HdfsClient(host, port, user=user, kerb_ticket=ticket_cache_path,
+                         driver='libhdfs3')
diff --git a/python/doc/index.rst b/python/doc/index.rst
index d64354be05520..608fff5d57ba4 100644
--- a/python/doc/index.rst
+++ b/python/doc/index.rst
@@ -34,15 +34,15 @@ structures.
    :maxdepth: 2
    :caption: Getting Started
 
-   Installing pyarrow <install.rst>
-   Pandas <pandas.rst>
-   Module Reference <modules.rst>
-   Getting Involved <getting_involved.rst>
+   install
+   pandas
+   filesystems
+   parquet
+   modules
+   getting_involved
 
 .. toctree::
    :maxdepth: 2
    :caption: Additional Features
 
-   Parquet format <parquet.rst>
    jemalloc MemoryPool <jemalloc.rst>
-

From b0863cb63d62ae7c4a429164e5a2e350d3c1f21a Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sun, 9 Apr 2017 13:56:14 -0400
Subject: [PATCH 0484/1644] ARROW-788: [C++] Align WriteTensor message

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #512 from pcmoritz/tensor-alignment and squashes the following commits:

fd20f05 [Philipp Moritz] align WriteTensor to 8 bytes
---
 cpp/src/arrow/ipc/reader.cc |  2 ++
 cpp/src/arrow/ipc/writer.cc | 11 +++++++++++
 2 files changed, 13 insertions(+)

diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 55f632f306b9a..a7c4f04a4d4cc 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -495,6 +495,8 @@ Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
 
 Status ReadTensor(
     int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out) {
+  // Respect alignment of Tensor messages (see WriteTensor)
+  offset = PaddedLength(offset);
   std::shared_ptr<Message> message;
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message, &data));
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 5330206480928..9305567e74f6b 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -470,6 +470,16 @@ class DictionaryWriter : public RecordBatchWriter {
   int64_t dictionary_id_;
 };
 
+// Adds padding bytes if necessary to ensure all memory blocks are written on
+// 8-byte boundaries.
+Status AlignStreamPosition(io::OutputStream* stream) {
+  int64_t position;
+  RETURN_NOT_OK(stream->Tell(&position));
+  int64_t remainder = PaddedLength(position) - position;
+  if (remainder > 0) { return stream->Write(kPaddingBytes, remainder); }
+  return Status::OK();
+}
+
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
     MemoryPool* pool, int max_recursion_depth, bool allow_64bit) {
@@ -486,6 +496,7 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
 
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
     int64_t* body_length) {
+  RETURN_NOT_OK(AlignStreamPosition(dst));
   std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(WriteTensorMessage(tensor, 0, &metadata));
   RETURN_NOT_OK(WriteMessage(*metadata, dst, metadata_length));

From 449f99162abab52378e2d6b2ca18099df567dc29 Mon Sep 17 00:00:00 2001
From: Nong Li <nongli@gmail.com>
Date: Sun, 9 Apr 2017 14:31:57 -0400
Subject: [PATCH 0485/1644] ARROW-773: [CPP] Add Table::AddColumn API

Author: Nong Li <nongli@gmail.com>

Closes #513 from nongli/arrow-773 and squashes the following commits:

e6f5846 [Nong Li] ARROW-773: [CPP] Add Table::AddColumn API
---
 cpp/src/arrow/table-test.cc         | 73 +++++++++++++++++++++++++++++
 cpp/src/arrow/table.cc              | 24 ++++++++++
 cpp/src/arrow/table.h               |  4 ++
 cpp/src/arrow/type.cc               |  9 ++++
 cpp/src/arrow/type.h                |  2 +
 cpp/src/arrow/util/CMakeLists.txt   |  1 +
 cpp/src/arrow/util/stl-util-test.cc | 60 ++++++++++++++++++++++++
 cpp/src/arrow/util/stl.h            | 20 ++++++++
 8 files changed, 193 insertions(+)
 create mode 100644 cpp/src/arrow/util/stl-util-test.cc

diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index cd32f4a387290..156c3d16d4db0 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -384,6 +384,79 @@ TEST_F(TestTable, RemoveColumn) {
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 }
 
+TEST_F(TestTable, AddColumn) {
+  const int64_t length = 10;
+  MakeExample1(length);
+
+  Table table(schema_, columns_);
+
+  std::shared_ptr<Table> result;
+  // Some negative tests with invalid index
+  Status status = table.AddColumn(10, columns_[0], &result);
+  ASSERT_TRUE(status.IsInvalid());
+  status = table.AddColumn(-1, columns_[0], &result);
+  ASSERT_TRUE(status.IsInvalid());
+
+  // Add column with wrong length
+  auto longer_col = std::make_shared<Column>(
+      schema_->field(0), MakePrimitive<Int32Array>(length + 1));
+  status = table.AddColumn(0, longer_col, &result);
+  ASSERT_TRUE(status.IsInvalid());
+
+  // Add column 0 in different places
+  ASSERT_OK(table.AddColumn(0, columns_[0], &result));
+  auto ex_schema = std::shared_ptr<Schema>(new Schema({
+      schema_->field(0),
+      schema_->field(0),
+      schema_->field(1),
+      schema_->field(2)}));
+  std::vector<std::shared_ptr<Column>> ex_columns = {
+      table.column(0),
+      table.column(0),
+      table.column(1),
+      table.column(2)};
+  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+
+  ASSERT_OK(table.AddColumn(1, columns_[0], &result));
+  ex_schema = std::shared_ptr<Schema>(new Schema({
+      schema_->field(0),
+      schema_->field(0),
+      schema_->field(1),
+      schema_->field(2)}));
+  ex_columns = {
+      table.column(0),
+      table.column(0),
+      table.column(1),
+      table.column(2)};
+  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+
+  ASSERT_OK(table.AddColumn(2, columns_[0], &result));
+  ex_schema = std::shared_ptr<Schema>(new Schema({
+      schema_->field(0),
+      schema_->field(1),
+      schema_->field(0),
+      schema_->field(2)}));
+  ex_columns = {
+      table.column(0),
+      table.column(1),
+      table.column(0),
+      table.column(2)};
+  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+
+  ASSERT_OK(table.AddColumn(3, columns_[0], &result));
+  ex_schema = std::shared_ptr<Schema>(new Schema({
+      schema_->field(0),
+      schema_->field(1),
+      schema_->field(2),
+      schema_->field(0)}));
+  ex_columns = {
+      table.column(0),
+      table.column(1),
+      table.column(2),
+      table.column(0)};
+  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+}
+
 class TestRecordBatch : public TestBase {};
 
 TEST_F(TestRecordBatch, Equals) {
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index da61fbb9a6daf..9b39f770a17b7 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -321,6 +321,30 @@ Status Table::RemoveColumn(int i, std::shared_ptr<Table>* out) const {
   return Status::OK();
 }
 
+Status Table::AddColumn(int i, const std::shared_ptr<Column>& col,
+    std::shared_ptr<Table>* out) const {
+  if (i < 0 || i > num_columns() + 1) {
+    return Status::Invalid("Invalid column index.");
+  }
+  if (col == nullptr) {
+    std::stringstream ss;
+    ss << "Column " << i << " was null";
+    return Status::Invalid(ss.str());
+  }
+  if (col->length() != num_rows_) {
+    std::stringstream ss;
+    ss << "Added column's length must match table's length. Expected length " << num_rows_
+        << " but got length " << col->length();
+    return Status::Invalid(ss.str());
+  }
+
+  std::shared_ptr<Schema> new_schema;
+  RETURN_NOT_OK(schema_->AddField(i, col->field(), &new_schema));
+
+  *out = std::make_shared<Table>(new_schema, AddVectorElement(columns_, i, col));
+  return Status::OK();
+}
+
 Status Table::ValidateColumns() const {
   if (num_columns() != schema_->num_fields()) {
     return Status::Invalid("Number of columns did not match schema");
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 0f35dd888fe2f..dcea53d8fb1dd 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -181,6 +181,10 @@ class ARROW_EXPORT Table {
   /// schemas are immutable)
   Status RemoveColumn(int i, std::shared_ptr<Table>* out) const;
 
+  /// Add column to the table, producing a new Table
+  Status AddColumn(int i, const std::shared_ptr<Column>& column,
+      std::shared_ptr<Table>* out) const;
+
   // @returns: the number of columns in the table
   int num_columns() const { return static_cast<int>(columns_.size()); }
 
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index d99551d661d69..abbb626e0fcb4 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -257,6 +257,15 @@ std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
   }
 }
 
+Status Schema::AddField(int i, const std::shared_ptr<Field>& field,
+    std::shared_ptr<Schema>* out) const {
+  DCHECK_GE(i, 0);
+  DCHECK_LE(i, this->num_fields());
+
+  *out = std::make_shared<Schema>(AddVectorElement(fields_, i, field));
+  return Status::OK();
+}
+
 Status Schema::RemoveField(int i, std::shared_ptr<Schema>* out) const {
   DCHECK_GE(i, 0);
   DCHECK_LT(i, this->num_fields());
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 0e69133219d55..36ab9d8b2b9d5 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -619,6 +619,8 @@ class ARROW_EXPORT Schema {
   // Render a string representation of the schema suitable for debugging
   std::string ToString() const;
 
+  Status AddField(int i, const std::shared_ptr<Field>& field,
+      std::shared_ptr<Schema>* out) const;
   Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
 
   int num_fields() const { return static_cast<int>(fields_.size()); }
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 8d9afccf867df..c1b6877a3e9ef 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -69,3 +69,4 @@ if (ARROW_BUILD_BENCHMARKS)
 endif()
 
 ADD_ARROW_TEST(bit-util-test)
+ADD_ARROW_TEST(stl-util-test)
diff --git a/cpp/src/arrow/util/stl-util-test.cc b/cpp/src/arrow/util/stl-util-test.cc
new file mode 100644
index 0000000000000..526520e7a2dec
--- /dev/null
+++ b/cpp/src/arrow/util/stl-util-test.cc
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/stl.h"
+
+#include <cstdint>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+TEST(StlUtilTest, VectorAddRemoveTest) {
+  std::vector<int> values;
+  std::vector<int> result = AddVectorElement(values, 0, 100);
+  EXPECT_EQ(values.size(), 0);
+  EXPECT_EQ(result.size(), 1);
+  EXPECT_EQ(result[0], 100);
+
+  // Add 200 at index 0 and 300 at the end.
+  std::vector<int> result2 = AddVectorElement(result, 0, 200);
+  result2 = AddVectorElement(result2, result2.size(), 300);
+  EXPECT_EQ(result.size(), 1);
+  EXPECT_EQ(result2.size(), 3);
+  EXPECT_EQ(result2[0], 200);
+  EXPECT_EQ(result2[1], 100);
+  EXPECT_EQ(result2[2], 300);
+
+  // Remove 100, 300, 200
+  std::vector<int> result3 = DeleteVectorElement(result2, 1);
+  EXPECT_EQ(result2.size(), 3);
+  EXPECT_EQ(result3.size(), 2);
+  EXPECT_EQ(result3[0], 200);
+  EXPECT_EQ(result3[1], 300);
+
+  result3 = DeleteVectorElement(result3, 1);
+  EXPECT_EQ(result3.size(), 1);
+  EXPECT_EQ(result3[0], 200);
+
+  result3 = DeleteVectorElement(result3, 0);
+  EXPECT_TRUE(result3.empty());
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/stl.h b/cpp/src/arrow/util/stl.h
index 3ec535d62b920..bd250539a8c8a 100644
--- a/cpp/src/arrow/util/stl.h
+++ b/cpp/src/arrow/util/stl.h
@@ -20,10 +20,14 @@
 
 #include <vector>
 
+#include <arrow/util/logging.h>
+
 namespace arrow {
 
 template <typename T>
 inline std::vector<T> DeleteVectorElement(const std::vector<T>& values, size_t index) {
+  DCHECK(!values.empty());
+  DCHECK_LT(index, values.size());
   std::vector<T> out;
   out.reserve(values.size() - 1);
   for (size_t i = 0; i < index; ++i) {
@@ -35,6 +39,22 @@ inline std::vector<T> DeleteVectorElement(const std::vector<T>& values, size_t i
   return out;
 }
 
+template <typename T>
+inline std::vector<T> AddVectorElement(const std::vector<T>& values, size_t index,
+    const T& new_element) {
+  DCHECK_LE(index, values.size());
+  std::vector<T> out;
+  out.reserve(values.size() + 1);
+  for (size_t i = 0; i < index; ++i) {
+    out.push_back(values[i]);
+  }
+  out.push_back(new_element);
+  for (size_t i = index; i < values.size(); ++i) {
+    out.push_back(values[i]);
+  }
+  return out;
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_STL_H

From 754bcce686ecf02e123dcf4801715bf155f15e1f Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 9 Apr 2017 15:19:53 -0400
Subject: [PATCH 0486/1644] ARROW-655: [C++/Python] Implement DecimalArray

Adds Decimal support for C++ and Python.

TODOs:
- [x] Tighten up some of the GIL acquisition. E.g., we may not need to hold it when importing the decimal module if we acquire it where we import the decimal module.
- [x] Investigate FreeBSD issue (manifesting on OS X) where typeinfo symbols for `__int128_t` are not exported: https://bugs.llvm.org//show_bug.cgi?id=26156.
- [x] See if there's a better way to visit scalar decimals, rather than keeping extra state on the class. Seems like an unacceptable hack.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #403 from cpcloud/decimal and squashes the following commits:

e5470fd [Phillip Cloud] Remove unnecessary header in helpers.h
07713a7 [Phillip Cloud] Remove more boost leakage
f764156 [Phillip Cloud] Revert "Transitively link static libs as well"
a7109b2 [Phillip Cloud] Transitively link static libs as well
bf2a7ea [Phillip Cloud] Move IsNegative to cc file
cb2c1ac [Phillip Cloud] Do not link boost regex to jemalloc
e63b766 [Phillip Cloud] Remove python extra cmake args
805bbac [Phillip Cloud] ARROW-655: [C++/Python] Implement DecimalArray
---
 .travis.yml                                  |   1 +
 cpp/CMakeLists.txt                           |  27 ++-
 cpp/cmake_modules/FindPythonLibsNew.cmake    |   3 +-
 cpp/src/arrow/array-decimal-test.cc          | 194 ++++++++++++++++++-
 cpp/src/arrow/array.cc                       |  49 ++++-
 cpp/src/arrow/array.h                        |  31 ++-
 cpp/src/arrow/builder.cc                     |  88 ++++++++-
 cpp/src/arrow/builder.h                      |  29 ++-
 cpp/src/arrow/compare.cc                     |  40 +++-
 cpp/src/arrow/ipc/CMakeLists.txt             |   7 +-
 cpp/src/arrow/python/CMakeLists.txt          |   3 +-
 cpp/src/arrow/python/builtin_convert.cc      |  62 +++++-
 cpp/src/arrow/python/builtin_convert.h       |   2 +-
 cpp/src/arrow/python/common.h                |   9 +-
 cpp/src/arrow/python/helpers.cc              |  79 ++++++++
 cpp/src/arrow/python/helpers.h               |  26 ++-
 cpp/src/arrow/python/pandas_convert.cc       | 176 ++++++++++++++++-
 cpp/src/arrow/python/python-test.cc          |  33 ++++
 cpp/src/arrow/type.cc                        |  18 +-
 cpp/src/arrow/type.h                         |  26 ++-
 cpp/src/arrow/type_fwd.h                     |   2 +
 cpp/src/arrow/type_traits.h                  |  13 +-
 cpp/src/arrow/util/CMakeLists.txt            |   2 +
 cpp/src/arrow/util/bit-util.h                |   1 -
 cpp/src/arrow/util/decimal-test.cc           | 161 +++++++++++++++
 cpp/src/arrow/util/decimal.cc                | 141 ++++++++++++++
 cpp/src/arrow/util/decimal.h                 | 144 ++++++++++++++
 cpp/src/arrow/visitor_inline.h               |   2 +-
 format/Schema.fbs                            |   2 +
 python/pyarrow/__init__.py                   |   2 +-
 python/pyarrow/array.pxd                     |   4 +
 python/pyarrow/array.pyx                     |   5 +
 python/pyarrow/includes/common.pxd           |   5 +
 python/pyarrow/includes/libarrow.pxd         |  16 ++
 python/pyarrow/scalar.pxd                    |   1 +
 python/pyarrow/scalar.pyx                    |  25 ++-
 python/pyarrow/schema.pxd                    |  10 +-
 python/pyarrow/schema.pyx                    |  28 ++-
 python/pyarrow/tests/test_convert_builtin.py |  40 ++++
 python/pyarrow/tests/test_convert_pandas.py  |  70 +++++++
 40 files changed, 1497 insertions(+), 80 deletions(-)
 create mode 100644 cpp/src/arrow/util/decimal-test.cc
 create mode 100644 cpp/src/arrow/util/decimal.cc
 create mode 100644 cpp/src/arrow/util/decimal.h

diff --git a/.travis.yml b/.travis.yml
index b219b03e0eb2b..f74a3b205c4b6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,6 +14,7 @@ addons:
     - valgrind
     - libboost-dev
     - libboost-filesystem-dev
+    - libboost-regex-dev
     - libboost-system-dev
     - libjemalloc-dev
     - gtk-doc-tools
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 9947a34e4e7bb..5852fe59da095 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -398,30 +398,36 @@ if (ARROW_BOOST_USE_SHARED)
     add_definitions(-DBOOST_ALL_DYN_LINK)
   endif()
 
-  find_package(Boost COMPONENTS system filesystem REQUIRED)
+  find_package(Boost COMPONENTS system filesystem regex REQUIRED)
   if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
     set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+    set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
   else()
     set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
     set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+    set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
   endif()
   set(BOOST_SYSTEM_LIBRARY boost_system_shared)
   set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
+  set(BOOST_REGEX_LIBRARY boost_regex_shared)
 else()
   # Find static boost headers and libs
   # TODO Differentiate here between release and debug builds
   set(Boost_USE_STATIC_LIBS ON)
-  find_package(Boost COMPONENTS system filesystem REQUIRED)
+  find_package(Boost COMPONENTS system filesystem regex REQUIRED)
   if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
     set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+    set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
   else()
     set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
     set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+    set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
   endif()
   set(BOOST_SYSTEM_LIBRARY boost_system_static)
   set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
+  set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 
 message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
@@ -435,7 +441,11 @@ ADD_THIRDPARTY_LIB(boost_filesystem
     STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
     SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
 
-SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
+ADD_THIRDPARTY_LIB(boost_regex
+        STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
+        SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")
+
+SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
 
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 
@@ -695,14 +705,16 @@ endif()
 set(ARROW_MIN_TEST_LIBS
   arrow_static
   arrow_test_main
-  ${ARROW_BASE_LIBS})
+  ${ARROW_BASE_LIBS}
+  ${BOOST_REGEX_LIBRARY})
 
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 set(ARROW_BENCHMARK_LINK_LIBS
   arrow_static
   arrow_benchmark_main
-  ${ARROW_BASE_LIBS})
+  ${ARROW_BASE_LIBS}
+  ${BOOST_REGEX_LIBRARY})
 
 ############################################################
 # "make ctags" target
@@ -796,7 +808,7 @@ endif()
 ############################################################
 
 set(ARROW_LINK_LIBS
-)
+    ${BOOST_REGEX_LIBRARY})
 
 set(ARROW_PRIVATE_LINK_LIBS
 )
@@ -816,6 +828,7 @@ set(ARROW_SRCS
   src/arrow/visitor.cc
 
   src/arrow/util/bit-util.cc
+  src/arrow/util/decimal.cc
 )
 
 if(NOT APPLE AND NOT MSVC)
@@ -825,9 +838,11 @@ if(NOT APPLE AND NOT MSVC)
   set(ARROW_SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map")
 endif()
 
+
 ADD_ARROW_LIB(arrow
   SOURCES ${ARROW_SRCS}
   SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
+  SHARED_LINK_LIBS ${ARROW_LINK_LIBS}
 )
 
 add_subdirectory(src/arrow)
diff --git a/cpp/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake
index dfe5661b015b5..d9cc4b3955734 100644
--- a/cpp/cmake_modules/FindPythonLibsNew.cmake
+++ b/cpp/cmake_modules/FindPythonLibsNew.cmake
@@ -175,7 +175,8 @@ else()
     find_library(PYTHON_LIBRARY
         NAMES "python${PYTHON_LIBRARY_SUFFIX}"
         PATHS ${_PYTHON_LIBS_SEARCH}
-        NO_SYSTEM_ENVIRONMENT_PATH)
+        NO_SYSTEM_ENVIRONMENT_PATH
+        NO_CMAKE_SYSTEM_PATH)
     message(STATUS "Found Python lib ${PYTHON_LIBRARY}")
 endif()
 
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index b64023bbc6a1e..4c01f928a6f26 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -15,13 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "arrow/type.h"
 #include "gtest/gtest.h"
 
-#include "arrow/type.h"
+#include "arrow/builder.h"
+#include "arrow/test-util.h"
+#include "arrow/util/decimal.h"
 
 namespace arrow {
 
-TEST(TypesTest, TestDecimalType) {
+TEST(TypesTest, TestDecimal32Type) {
   DecimalType t1(8, 4);
 
   ASSERT_EQ(t1.type, Type::DECIMAL);
@@ -29,6 +32,193 @@ TEST(TypesTest, TestDecimalType) {
   ASSERT_EQ(t1.scale, 4);
 
   ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
+
+  // Test properties
+  ASSERT_EQ(t1.byte_width(), 4);
+  ASSERT_EQ(t1.bit_width(), 32);
 }
 
+TEST(TypesTest, TestDecimal64Type) {
+  DecimalType t1(12, 5);
+
+  ASSERT_EQ(t1.type, Type::DECIMAL);
+  ASSERT_EQ(t1.precision, 12);
+  ASSERT_EQ(t1.scale, 5);
+
+  ASSERT_EQ(t1.ToString(), std::string("decimal(12, 5)"));
+
+  // Test properties
+  ASSERT_EQ(t1.byte_width(), 8);
+  ASSERT_EQ(t1.bit_width(), 64);
+}
+
+TEST(TypesTest, TestDecimal128Type) {
+  DecimalType t1(27, 7);
+
+  ASSERT_EQ(t1.type, Type::DECIMAL);
+  ASSERT_EQ(t1.precision, 27);
+  ASSERT_EQ(t1.scale, 7);
+
+  ASSERT_EQ(t1.ToString(), std::string("decimal(27, 7)"));
+
+  // Test properties
+  ASSERT_EQ(t1.byte_width(), 16);
+  ASSERT_EQ(t1.bit_width(), 128);
+}
+
+template <typename T>
+class DecimalTestBase {
+ public:
+  virtual std::vector<uint8_t> data(
+      const std::vector<T>& input, size_t byte_width) const = 0;
+
+  void test(int precision, const std::vector<T>& draw,
+      const std::vector<uint8_t>& valid_bytes,
+      const std::vector<uint8_t>& sign_bitmap = {}, int64_t offset = 0) const {
+    auto type = std::make_shared<DecimalType>(precision, 4);
+    int byte_width = type->byte_width();
+    auto pool = default_memory_pool();
+    auto builder = std::make_shared<DecimalBuilder>(pool, type);
+    size_t null_count = 0;
+
+    size_t size = draw.size();
+    builder->Reserve(size);
+
+    for (size_t i = 0; i < size; ++i) {
+      if (valid_bytes[i]) {
+        builder->Append(draw[i]);
+      } else {
+        builder->AppendNull();
+        ++null_count;
+      }
+    }
+
+    std::shared_ptr<Buffer> expected_sign_bitmap;
+    if (!sign_bitmap.empty()) {
+      BitUtil::BytesToBits(sign_bitmap, &expected_sign_bitmap);
+    }
+
+    auto raw_bytes = data(draw, byte_width);
+    auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), size * byte_width);
+    auto expected_null_bitmap = test::bytes_to_null_buffer(valid_bytes);
+    int64_t expected_null_count = test::null_count(valid_bytes);
+    auto expected = std::make_shared<DecimalArray>(type, size, expected_data,
+        expected_null_bitmap, expected_null_count, offset, expected_sign_bitmap);
+
+    std::shared_ptr<Array> out;
+    ASSERT_OK(builder->Finish(&out));
+    ASSERT_TRUE(out->Equals(*expected));
+  }
+};
+
+template <typename T>
+class DecimalTest : public DecimalTestBase<T> {
+ public:
+  std::vector<uint8_t> data(
+      const std::vector<T>& input, size_t byte_width) const override {
+    std::vector<uint8_t> result;
+    result.reserve(input.size() * byte_width);
+    // TODO(phillipc): There's probably a better way to do this
+    constexpr static const size_t bytes_per_element = sizeof(T);
+    for (size_t i = 0, j = 0; i < input.size(); ++i, j += bytes_per_element) {
+      *reinterpret_cast<typename T::value_type*>(&result[j]) = input[i].value;
+    }
+    return result;
+  }
+};
+
+template <>
+class DecimalTest<Decimal128> : public DecimalTestBase<Decimal128> {
+ public:
+  std::vector<uint8_t> data(
+      const std::vector<Decimal128>& input, size_t byte_width) const override {
+    std::vector<uint8_t> result;
+    result.reserve(input.size() * byte_width);
+    constexpr static const size_t bytes_per_element = 16;
+    for (size_t i = 0; i < input.size(); ++i) {
+      uint8_t stack_bytes[bytes_per_element] = {0};
+      uint8_t* bytes = stack_bytes;
+      bool is_negative;
+      ToBytes(input[i], &bytes, &is_negative);
+
+      for (size_t i = 0; i < bytes_per_element; ++i) {
+        result.push_back(bytes[i]);
+      }
+    }
+    return result;
+  }
+};
+
+class Decimal32BuilderTest : public ::testing::TestWithParam<int>,
+                             public DecimalTest<Decimal32> {};
+
+class Decimal64BuilderTest : public ::testing::TestWithParam<int>,
+                             public DecimalTest<Decimal64> {};
+
+class Decimal128BuilderTest : public ::testing::TestWithParam<int>,
+                              public DecimalTest<Decimal128> {};
+
+TEST_P(Decimal32BuilderTest, NoNulls) {
+  int precision = GetParam();
+  std::vector<Decimal32> draw = {
+      Decimal32(1), Decimal32(2), Decimal32(2389), Decimal32(4), Decimal32(-12348)};
+  std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
+  this->test(precision, draw, valid_bytes);
+}
+
+TEST_P(Decimal64BuilderTest, NoNulls) {
+  int precision = GetParam();
+  std::vector<Decimal64> draw = {
+      Decimal64(1), Decimal64(2), Decimal64(2389), Decimal64(4), Decimal64(-12348)};
+  std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
+  this->test(precision, draw, valid_bytes);
+}
+
+TEST_P(Decimal128BuilderTest, NoNulls) {
+  int precision = GetParam();
+  std::vector<Decimal128> draw = {
+      Decimal128(1), Decimal128(-2), Decimal128(2389), Decimal128(4), Decimal128(-12348)};
+  std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
+  std::vector<uint8_t> sign_bitmap = {false, true, false, false, true};
+  this->test(precision, draw, valid_bytes, sign_bitmap);
+}
+
+TEST_P(Decimal32BuilderTest, WithNulls) {
+  int precision = GetParam();
+  std::vector<Decimal32> draw = {
+      Decimal32(1), Decimal32(2), Decimal32(-1), Decimal32(4), Decimal32(-1)};
+  std::vector<uint8_t> valid_bytes = {true, true, false, true, false};
+  this->test(precision, draw, valid_bytes);
+}
+
+TEST_P(Decimal64BuilderTest, WithNulls) {
+  int precision = GetParam();
+  std::vector<Decimal64> draw = {
+      Decimal64(-1), Decimal64(2), Decimal64(-1), Decimal64(4), Decimal64(-1)};
+  std::vector<uint8_t> valid_bytes = {true, true, false, true, false};
+  this->test(precision, draw, valid_bytes);
+}
+
+TEST_P(Decimal128BuilderTest, WithNulls) {
+  int precision = GetParam();
+  std::vector<Decimal128> draw = {Decimal128(1), Decimal128(2), Decimal128(-1),
+      Decimal128(4), Decimal128(-1), Decimal128(1), Decimal128(2),
+      Decimal128("230342903942.234234"), Decimal128("-23049302932.235234")};
+  std::vector<uint8_t> valid_bytes = {
+      true, true, false, true, false, true, true, true, true};
+  std::vector<uint8_t> sign_bitmap = {
+      false, false, false, false, false, false, false, false, true};
+  this->test(precision, draw, valid_bytes, sign_bitmap);
+}
+
+INSTANTIATE_TEST_CASE_P(Decimal32BuilderTest, Decimal32BuilderTest,
+    ::testing::Range(
+        DecimalPrecision<int32_t>::minimum, DecimalPrecision<int32_t>::maximum));
+INSTANTIATE_TEST_CASE_P(Decimal64BuilderTest, Decimal64BuilderTest,
+    ::testing::Range(
+        DecimalPrecision<int64_t>::minimum, DecimalPrecision<int64_t>::maximum));
+INSTANTIATE_TEST_CASE_P(Decimal128BuilderTest, Decimal128BuilderTest,
+    ::testing::Range(
+        DecimalPrecision<int128_t>::minimum, DecimalPrecision<int128_t>::maximum));
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index bd20654bc87d4..4e73e7176fa9c 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -27,6 +27,7 @@
 #include "arrow/status.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 #include "arrow/visitor.h"
 #include "arrow/visitor_inline.h"
@@ -283,10 +284,8 @@ std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const
 FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<DataType>& type,
     int64_t length, const std::shared_ptr<Buffer>& data,
     const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
-    : PrimitiveArray(type, length, data, null_bitmap, null_count, offset) {
-  DCHECK(type->type == Type::FIXED_SIZE_BINARY);
-  byte_width_ = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
-}
+    : PrimitiveArray(type, length, data, null_bitmap, null_count, offset),
+      byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {}
 
 std::shared_ptr<Array> FixedSizeBinaryArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
@@ -294,6 +293,48 @@ std::shared_ptr<Array> FixedSizeBinaryArray::Slice(int64_t offset, int64_t lengt
       type_, length, data_, null_bitmap_, kUnknownNullCount, offset);
 }
 
+const uint8_t* FixedSizeBinaryArray::GetValue(int64_t i) const {
+  return raw_data_ + (i + offset_) * byte_width_;
+}
+
+// ----------------------------------------------------------------------
+// Decimal
+DecimalArray::DecimalArray(const std::shared_ptr<DataType>& type, int64_t length,
+    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
+    int64_t null_count, int64_t offset, const std::shared_ptr<Buffer>& sign_bitmap)
+    : FixedSizeBinaryArray(type, length, data, null_bitmap, null_count, offset),
+      sign_bitmap_(sign_bitmap),
+      sign_bitmap_data_(sign_bitmap != nullptr ? sign_bitmap->data() : nullptr) {}
+
+bool DecimalArray::IsNegative(int64_t i) const {
+  return sign_bitmap_data_ != nullptr ? BitUtil::GetBit(sign_bitmap_data_, i) : false;
+}
+
+template <typename T>
+ARROW_EXPORT Decimal<T> DecimalArray::Value(int64_t i) const {
+  Decimal<T> result;
+  FromBytes(GetValue(i), &result);
+  return result;
+}
+
+template ARROW_EXPORT Decimal32 DecimalArray::Value(int64_t i) const;
+template ARROW_EXPORT Decimal64 DecimalArray::Value(int64_t i) const;
+
+template <>
+ARROW_EXPORT Decimal128 DecimalArray::Value(int64_t i) const {
+  Decimal128 result;
+  FromBytes(GetValue(i), IsNegative(i), &result);
+  return result;
+}
+
+template ARROW_EXPORT Decimal128 DecimalArray::Value(int64_t i) const;
+
+std::shared_ptr<Array> DecimalArray::Slice(int64_t offset, int64_t length) const {
+  ConformSliceParams(offset_, length_, &offset, &length);
+  return std::make_shared<DecimalArray>(
+      type_, length, data_, null_bitmap_, kUnknownNullCount, offset, sign_bitmap_);
+}
+
 // ----------------------------------------------------------------------
 // Struct
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 9f0e73914da84..a4117facdefd0 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -39,6 +39,9 @@ class MemoryPool;
 class MutableBuffer;
 class Status;
 
+template <typename T>
+struct Decimal;
+
 /// Immutable data array with some logical type and some length.
 ///
 /// Any memory is owned by the respective Buffer instance (or its parents).
@@ -356,9 +359,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0);
 
-  const uint8_t* GetValue(int64_t i) const {
-    return raw_data_ + (i + offset_) * byte_width_;
-  }
+  const uint8_t* GetValue(int64_t i) const;
 
   int32_t byte_width() const { return byte_width_; }
 
@@ -370,6 +371,30 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
   int32_t byte_width_;
 };
 
+// ----------------------------------------------------------------------
+// DecimalArray
+class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray {
+ public:
+  using TypeClass = Type;
+
+  DecimalArray(const std::shared_ptr<DataType>& type, int64_t length,
+      const std::shared_ptr<Buffer>& data,
+      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      int64_t offset = 0, const std::shared_ptr<Buffer>& sign_bitmap = nullptr);
+
+  bool IsNegative(int64_t i) const;
+
+  template <typename T>
+  ARROW_EXPORT Decimal<T> Value(int64_t i) const;
+
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
+
+ private:
+  /// Only needed for 128 bit Decimals
+  std::shared_ptr<Buffer> sign_bitmap_;
+  const uint8_t* sign_bitmap_data_;
+};
+
 // ----------------------------------------------------------------------
 // Struct
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 40b81cf015ab4..a3677eff68669 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -27,6 +27,7 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -323,6 +324,85 @@ Status BooleanBuilder::Append(
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// DecimalBuilder
+DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
+    : FixedSizeBinaryBuilder(pool, type),
+      sign_bitmap_(nullptr),
+      sign_bitmap_data_(nullptr) {}
+
+template <typename T>
+ARROW_EXPORT Status DecimalBuilder::Append(const Decimal<T>& val) {
+  DCHECK_EQ(sign_bitmap_, nullptr) << "sign_bitmap_ is not null";
+  DCHECK_EQ(sign_bitmap_data_, nullptr) << "sign_bitmap_data_ is not null";
+
+  RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
+  return FixedSizeBinaryBuilder::Append(reinterpret_cast<const uint8_t*>(&val.value));
+}
+
+template ARROW_EXPORT Status DecimalBuilder::Append(const Decimal32& val);
+template ARROW_EXPORT Status DecimalBuilder::Append(const Decimal64& val);
+
+template <>
+ARROW_EXPORT Status DecimalBuilder::Append(const Decimal128& value) {
+  DCHECK_NE(sign_bitmap_, nullptr) << "sign_bitmap_ is null";
+  DCHECK_NE(sign_bitmap_data_, nullptr) << "sign_bitmap_data_ is null";
+
+  RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
+  uint8_t stack_bytes[16] = {0};
+  uint8_t* bytes = stack_bytes;
+  bool is_negative;
+  ToBytes(value, &bytes, &is_negative);
+  RETURN_NOT_OK(FixedSizeBinaryBuilder::Append(bytes));
+
+  // TODO(phillipc): calculate the proper storage size here (do we have a function to do
+  // this)?
+  // TODO(phillipc): Reserve number of elements
+  RETURN_NOT_OK(sign_bitmap_->Reserve(1));
+  BitUtil::SetBitTo(sign_bitmap_data_, length_ - 1, is_negative);
+  return Status::OK();
+}
+
+template ARROW_EXPORT Status DecimalBuilder::Append(const Decimal128& val);
+
+Status DecimalBuilder::Init(int64_t capacity) {
+  RETURN_NOT_OK(FixedSizeBinaryBuilder::Init(capacity));
+  if (byte_width_ == 16) {
+    AllocateResizableBuffer(pool_, null_bitmap_->size(), &sign_bitmap_);
+    sign_bitmap_data_ = sign_bitmap_->mutable_data();
+    memset(sign_bitmap_data_, 0, static_cast<size_t>(sign_bitmap_->capacity()));
+  }
+  return Status::OK();
+}
+
+Status DecimalBuilder::Resize(int64_t capacity) {
+  int64_t old_bytes = null_bitmap_ != nullptr ? null_bitmap_->size() : 0;
+  if (sign_bitmap_ == nullptr) { return Init(capacity); }
+  RETURN_NOT_OK(FixedSizeBinaryBuilder::Resize(capacity));
+
+  if (byte_width_ == 16) {
+    RETURN_NOT_OK(sign_bitmap_->Resize(null_bitmap_->size()));
+    int64_t new_bytes = sign_bitmap_->size();
+    sign_bitmap_data_ = sign_bitmap_->mutable_data();
+
+    // The buffer might be overpadded to deal with padding according to the spec
+    if (old_bytes < new_bytes) {
+      memset(sign_bitmap_data_ + old_bytes, 0,
+          static_cast<size_t>(sign_bitmap_->capacity() - old_bytes));
+    }
+  }
+  return Status::OK();
+}
+
+Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Buffer> data = byte_builder_.Finish();
+
+  /// TODO(phillipc): not sure where to get the offset argument here
+  *out = std::make_shared<DecimalArray>(
+      type_, length_, data, null_bitmap_, null_count_, 0, sign_bitmap_);
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // ListBuilder
 
@@ -440,10 +520,9 @@ Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
 
 FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(
     MemoryPool* pool, const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool, type), byte_builder_(pool) {
-  DCHECK(type->type == Type::FIXED_SIZE_BINARY);
-  byte_width_ = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
-}
+    : ArrayBuilder(pool, type),
+      byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()),
+      byte_builder_(pool) {}
 
 Status FixedSizeBinaryBuilder::Append(const uint8_t* value) {
   RETURN_NOT_OK(Reserve(1));
@@ -543,6 +622,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(STRING, StringBuilder);
     BUILDER_CASE(BINARY, BinaryBuilder);
     BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder);
+    BUILDER_CASE(DECIMAL, DecimalBuilder);
     case Type::LIST: {
       std::shared_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 60cdc4cb3a5db..d42ab5b01d1ba 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -37,6 +37,9 @@ namespace arrow {
 
 class Array;
 
+template <typename T>
+struct Decimal;
+
 static constexpr int64_t kMinBuilderCapacity = 1 << 5;
 
 /// Base class for all data array builders.
@@ -76,12 +79,12 @@ class ARROW_EXPORT ArrayBuilder {
   Status SetNotNull(int64_t length);
 
   /// Allocates initial capacity requirements for the builder.  In most
-  /// cases subclasses should override and call there parent classes
+  /// cases subclasses should override and call their parent class's
   /// method as well.
   virtual Status Init(int64_t capacity);
 
   /// Resizes the null_bitmap array.  In most
-  /// cases subclasses should override and call there parent classes
+  /// cases subclasses should override and call their parent class's
   /// method as well.
   virtual Status Resize(int64_t new_bits);
 
@@ -275,9 +278,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  Status Append(uint8_t val) {
-    return Append(val != 0);
-  }
+  Status Append(uint8_t val) { return Append(val != 0); }
 
   /// Vector append
   ///
@@ -415,6 +416,24 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
   BufferBuilder byte_builder_;
 };
 
+class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
+ public:
+  explicit DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+
+  template <typename T>
+  ARROW_EXPORT Status Append(const Decimal<T>& val);
+
+  Status Init(int64_t capacity) override;
+  Status Resize(int64_t capacity) override;
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+ private:
+  /// We only need these for 128 bit decimals, because boost stores the sign
+  /// separate from the underlying bytes.
+  std::shared_ptr<ResizableBuffer> sign_bitmap_;
+  uint8_t* sign_bitmap_data_;
+};
+
 // ----------------------------------------------------------------------
 // Struct
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 7451439a875d6..2297e4b206d1f 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -29,6 +29,7 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 #include "arrow/visitor_inline.h"
 
@@ -232,6 +233,41 @@ class RangeEqualsVisitor {
     return Status::OK();
   }
 
+  Status Visit(const DecimalArray& left) {
+    const auto& right = static_cast<const DecimalArray&>(right_);
+
+    int32_t width = left.byte_width();
+
+    const uint8_t* left_data = nullptr;
+    const uint8_t* right_data = nullptr;
+
+    if (left.data()) { left_data = left.raw_data() + left.offset() * width; }
+
+    if (right.data()) { right_data = right.raw_data() + right.offset() * width; }
+
+    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
+         ++i, ++o_i) {
+      if (left.IsNegative(i) != right.IsNegative(o_i)) {
+        result_ = false;
+        return Status::OK();
+      }
+
+      const bool is_null = left.IsNull(i);
+      if (is_null != right.IsNull(o_i)) {
+        result_ = false;
+        return Status::OK();
+      }
+      if (is_null) continue;
+
+      if (std::memcmp(left_data + width * i, right_data + width * o_i, width)) {
+        result_ = false;
+        return Status::OK();
+      }
+    }
+    result_ = true;
+    return Status::OK();
+  }
+
   Status Visit(const NullArray& left) {
     UNUSED(left);
     result_ = true;
@@ -244,10 +280,6 @@ class RangeEqualsVisitor {
     return CompareValues<T>(left);
   }
 
-  Status Visit(const DecimalArray& left) {
-    return Status::NotImplemented("Decimal type");
-  }
-
   Status Visit(const ListArray& left) {
     result_ = CompareLists(left);
     return Status::OK();
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 57db03311c06f..c6880c56e466b 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -27,7 +27,8 @@ set(ARROW_IPC_SHARED_LINK_LIBS
 set(ARROW_IPC_TEST_LINK_LIBS
   arrow_ipc_static
   arrow_io_static
-  arrow_static)
+  arrow_static
+  ${BOOST_REGEX_LIBRARY})
 
 set(ARROW_IPC_SRCS
   feather.cc
@@ -161,7 +162,8 @@ if(MSVC)
     arrow_io_static
     arrow_static
     ${BOOST_FILESYSTEM_LIBRARY}
-    ${BOOST_SYSTEM_LIBRARY})
+    ${BOOST_SYSTEM_LIBRARY}
+    ${BOOST_REGEX_LIBRARY})
 else()
   set(UTIL_LINK_LIBS
     arrow_ipc_static
@@ -169,6 +171,7 @@ else()
     arrow_static
     ${BOOST_FILESYSTEM_LIBRARY}
     ${BOOST_SYSTEM_LIBRARY}
+    ${BOOST_REGEX_LIBRARY}
     dl)
 endif()
 
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index c69d976737f91..604527f6304ac 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -37,7 +37,8 @@ set(ARROW_PYTHON_MIN_TEST_LIBS
   arrow_python_static
   arrow_ipc_static
   arrow_io_static
-  arrow_static)
+  arrow_static
+  ${BOOST_REGEX_LIBRARY})
 
 if(ARROW_BUILD_TESTS)
   ADD_THIRDPARTY_LIB(python
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 25b32ee26a06b..189ecee4fe022 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -17,12 +17,16 @@
 
 #include <Python.h>
 #include <datetime.h>
+
+#include <algorithm>
 #include <sstream>
+#include <string>
 
 #include "arrow/python/builtin_convert.h"
 
 #include "arrow/api.h"
 #include "arrow/status.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 
 #include "arrow/python/helpers.h"
@@ -109,7 +113,6 @@ class ScalarVisitor {
   int64_t float_count_;
   int64_t binary_count_;
   int64_t unicode_count_;
-
   // Place to accumulate errors
   // std::vector<Status> errors_;
 };
@@ -394,8 +397,7 @@ class BytesConverter : public TypedConverter<BinaryBuilder> {
       } else if (PyBytes_Check(item)) {
         bytes_obj = item;
       } else {
-        return Status::Invalid(
-            "Value that cannot be converted to bytes was encountered");
+        return Status::Invalid("Value that cannot be converted to bytes was encountered");
       }
       // No error checking
       length = PyBytes_GET_SIZE(bytes_obj);
@@ -429,8 +431,7 @@ class FixedWidthBytesConverter : public TypedConverter<FixedSizeBinaryBuilder> {
       } else if (PyBytes_Check(item)) {
         bytes_obj = item;
       } else {
-        return Status::Invalid(
-            "Value that cannot be converted to bytes was encountered");
+        return Status::Invalid("Value that cannot be converted to bytes was encountered");
       }
       // No error checking
       RETURN_NOT_OK(CheckPythonBytesAreFixedLength(bytes_obj, expected_length));
@@ -495,6 +496,54 @@ class ListConverter : public TypedConverter<ListBuilder> {
   std::shared_ptr<SeqConverter> value_converter_;
 };
 
+#define DECIMAL_CONVERT_CASE(bit_width, item, builder)        \
+  case bit_width: {                                           \
+    arrow::Decimal##bit_width out;                            \
+    RETURN_NOT_OK(PythonDecimalToArrowDecimal((item), &out)); \
+    RETURN_NOT_OK((builder)->Append(out));                    \
+    break;                                                    \
+  }
+
+class DecimalConverter : public TypedConverter<arrow::DecimalBuilder> {
+ public:
+  Status AppendData(PyObject* seq) override {
+    /// Ensure we've allocated enough space
+    Py_ssize_t size = PySequence_Size(seq);
+    RETURN_NOT_OK(typed_builder_->Reserve(size));
+
+    /// Can the compiler figure out that the case statement below isn't necessary
+    /// once we're running?
+    const int bit_width =
+        std::dynamic_pointer_cast<arrow::DecimalType>(typed_builder_->type())
+            ->bit_width();
+
+    OwnedRef ref;
+    PyObject* item = nullptr;
+    for (int64_t i = 0; i < size; ++i) {
+      ref.reset(PySequence_GetItem(seq, i));
+      item = ref.obj();
+
+      /// TODO(phillipc): Check for nan?
+      if (item != Py_None) {
+        switch (bit_width) {
+          DECIMAL_CONVERT_CASE(32, item, typed_builder_)
+          DECIMAL_CONVERT_CASE(64, item, typed_builder_)
+          DECIMAL_CONVERT_CASE(128, item, typed_builder_)
+          default:
+            break;
+        }
+        RETURN_IF_PYERROR();
+      } else {
+        RETURN_NOT_OK(typed_builder_->AppendNull());
+      }
+    }
+
+    return Status::OK();
+  }
+};
+
+#undef DECIMAL_CONVERT_CASE
+
 // Dynamic constructor for sequence converters
 std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type) {
   switch (type->type) {
@@ -516,6 +565,9 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
       return std::make_shared<UTF8Converter>();
     case Type::LIST:
       return std::make_shared<ListConverter>();
+    case Type::DECIMAL: {
+      return std::make_shared<DecimalConverter>();
+    }
     case Type::STRUCT:
     default:
       return nullptr;
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index 00ff0fd8236fc..3c2e350269a78 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -25,7 +25,7 @@
 
 #include <memory>
 
-#include <arrow/type.h>
+#include "arrow/type.h"
 
 #include "arrow/util/visibility.h"
 
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index 32bfa784acbd0..a6806ab95ab95 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -57,12 +57,13 @@ class OwnedRef {
   }
 
   void reset(PyObject* obj) {
-    if (obj_ != nullptr) { Py_XDECREF(obj_); }
+    /// TODO(phillipc): Should we acquire the GIL here? It definitely needs to be
+    /// acquired,
+    /// but callers have probably already acquired it
+    Py_XDECREF(obj_);
     obj_ = obj;
   }
 
-  void release() { obj_ = nullptr; }
-
   PyObject* obj() const { return obj_; }
 
  private:
@@ -72,6 +73,7 @@ class OwnedRef {
 struct PyObjectStringify {
   OwnedRef tmp_obj;
   const char* bytes;
+  Py_ssize_t size;
 
   explicit PyObjectStringify(PyObject* obj) {
     PyObject* bytes_obj;
@@ -82,6 +84,7 @@ struct PyObjectStringify {
       bytes_obj = obj;
     }
     bytes = PyBytes_AsString(bytes_obj);
+    size = PyBytes_GET_SIZE(bytes_obj);
   }
 };
 
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index be5f412fbea1c..ffba7bbc21c14 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -16,6 +16,8 @@
 // under the License.
 
 #include "arrow/python/helpers.h"
+#include "arrow/python/common.h"
+#include "arrow/util/decimal.h"
 
 #include <arrow/api.h>
 
@@ -52,5 +54,82 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
   }
 }
 
+Status ImportModule(const std::string& module_name, OwnedRef* ref) {
+  PyAcquireGIL lock;
+  PyObject* module = PyImport_ImportModule(module_name.c_str());
+  RETURN_IF_PYERROR();
+  ref->reset(module);
+  return Status::OK();
+}
+
+Status ImportFromModule(const OwnedRef& module, const std::string& name, OwnedRef* ref) {
+  /// Assumes that ImportModule was called first
+  DCHECK_NE(module.obj(), nullptr) << "Cannot import from nullptr Python module";
+
+  PyAcquireGIL lock;
+  PyObject* attr = PyObject_GetAttrString(module.obj(), name.c_str());
+  RETURN_IF_PYERROR();
+  ref->reset(attr);
+  return Status::OK();
+}
+
+template <typename T>
+Status PythonDecimalToArrowDecimal(PyObject* python_decimal, Decimal<T>* arrow_decimal) {
+  // Call Python's str(decimal_object)
+  OwnedRef str_obj(PyObject_Str(python_decimal));
+  RETURN_IF_PYERROR();
+
+  PyObjectStringify str(str_obj.obj());
+  RETURN_IF_PYERROR();
+
+  const char* bytes = str.bytes;
+  DCHECK_NE(bytes, nullptr);
+
+  Py_ssize_t size = str.size;
+
+  std::string c_string(bytes, size);
+  return FromString(c_string, arrow_decimal);
+}
+
+template Status PythonDecimalToArrowDecimal(
+    PyObject* python_decimal, Decimal32* arrow_decimal);
+template Status PythonDecimalToArrowDecimal(
+    PyObject* python_decimal, Decimal64* arrow_decimal);
+template Status PythonDecimalToArrowDecimal(
+    PyObject* python_decimal, Decimal128* arrow_decimal);
+
+Status InferDecimalPrecisionAndScale(
+    PyObject* python_decimal, int* precision, int* scale) {
+  // Call Python's str(decimal_object)
+  OwnedRef str_obj(PyObject_Str(python_decimal));
+  RETURN_IF_PYERROR();
+  PyObjectStringify str(str_obj.obj());
+
+  const char* bytes = str.bytes;
+  DCHECK_NE(bytes, nullptr);
+
+  auto size = str.size;
+
+  std::string c_string(bytes, size);
+  return FromString(c_string, static_cast<Decimal32*>(nullptr), precision, scale);
+}
+
+Status DecimalFromString(
+    PyObject* decimal_constructor, const std::string& decimal_string, PyObject** out) {
+  DCHECK_NE(decimal_constructor, nullptr);
+  DCHECK_NE(out, nullptr);
+
+  auto string_size = decimal_string.size();
+  DCHECK_GT(string_size, 0);
+
+  auto string_bytes = decimal_string.c_str();
+  DCHECK_NE(string_bytes, nullptr);
+
+  *out = PyObject_CallFunction(
+      decimal_constructor, const_cast<char*>("s#"), string_bytes, string_size);
+  RETURN_IF_PYERROR();
+  return Status::OK();
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index 611e814b7d858..a19b25f7db805 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -18,16 +18,38 @@
 #ifndef PYARROW_HELPERS_H
 #define PYARROW_HELPERS_H
 
+#include <Python.h>
+
 #include <memory>
+#include <string>
+#include <utility>
 
 #include "arrow/type.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
+
+template <typename T>
+struct Decimal;
+
 namespace py {
 
-ARROW_EXPORT
-std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
+class OwnedRef;
+
+ARROW_EXPORT std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
+
+Status ImportModule(const std::string& module_name, OwnedRef* ref);
+Status ImportFromModule(
+    const OwnedRef& module, const std::string& module_name, OwnedRef* ref);
+
+template <typename T>
+Status PythonDecimalToArrowDecimal(PyObject* python_decimal, Decimal<T>* arrow_decimal);
+
+Status InferDecimalPrecisionAndScale(
+    PyObject* python_decimal, int* precision = nullptr, int* scale = nullptr);
+
+Status DecimalFromString(
+    PyObject* decimal_constructor, const std::string& decimal_string, PyObject** out);
 
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 48d3489bf900b..f6e627e668e2d 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -41,12 +41,14 @@
 #include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
 #include "arrow/python/config.h"
+#include "arrow/python/helpers.h"
 #include "arrow/python/numpy-internal.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/type_traits.h"
@@ -375,6 +377,7 @@ class PandasConverter : public TypeVisitor {
   Status ConvertDates();
   Status ConvertLists(const std::shared_ptr<DataType>& type);
   Status ConvertObjects();
+  Status ConvertDecimals();
 
  protected:
   MemoryPool* pool_;
@@ -468,15 +471,14 @@ Status InvalidConversion(PyObject* obj, const std::string& expected_type_name) {
   RETURN_IF_PYERROR();
   DCHECK_NE(type_name.obj(), nullptr);
 
-  OwnedRef bytes_obj(PyUnicode_AsUTF8String(type_name.obj()));
+  PyObjectStringify bytestring(type_name.obj());
   RETURN_IF_PYERROR();
-  DCHECK_NE(bytes_obj.obj(), nullptr);
-
-  Py_ssize_t size = PyBytes_GET_SIZE(bytes_obj.obj());
-  const char* bytes = PyBytes_AS_STRING(bytes_obj.obj());
 
+  const char* bytes = bytestring.bytes;
   DCHECK_NE(bytes, nullptr) << "bytes from type(...).__name__ were null";
 
+  Py_ssize_t size = bytestring.size;
+
   std::string cpp_type_name(bytes, size);
 
   std::stringstream ss;
@@ -517,6 +519,59 @@ Status PandasConverter::ConvertDates() {
   return date_builder.Finish(&out_);
 }
 
+#define CONVERT_DECIMAL_CASE(bit_width, builder, object)      \
+  case bit_width: {                                           \
+    Decimal##bit_width d;                                     \
+    RETURN_NOT_OK(PythonDecimalToArrowDecimal((object), &d)); \
+    RETURN_NOT_OK((builder).Append(d));                       \
+    break;                                                    \
+  }
+
+Status PandasConverter::ConvertDecimals() {
+  PyAcquireGIL lock;
+
+  // Import the decimal module and Decimal class
+  OwnedRef decimal;
+  OwnedRef Decimal;
+  RETURN_NOT_OK(ImportModule("decimal", &decimal));
+  RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  PyObject* object = objects[0];
+
+  int precision;
+  int scale;
+
+  RETURN_NOT_OK(InferDecimalPrecisionAndScale(object, &precision, &scale));
+
+  type_ = std::make_shared<DecimalType>(precision, scale);
+
+  const int bit_width = std::dynamic_pointer_cast<DecimalType>(type_)->bit_width();
+  DecimalBuilder decimal_builder(pool_, type_);
+
+  RETURN_NOT_OK(decimal_builder.Resize(length_));
+
+  for (int64_t i = 0; i < length_; ++i) {
+    object = objects[i];
+    if (PyObject_IsInstance(object, Decimal.obj())) {
+      switch (bit_width) {
+        CONVERT_DECIMAL_CASE(32, decimal_builder, object)
+        CONVERT_DECIMAL_CASE(64, decimal_builder, object)
+        CONVERT_DECIMAL_CASE(128, decimal_builder, object)
+        default:
+          break;
+      }
+    } else if (PyObject_is_null(object)) {
+      decimal_builder.AppendNull();
+    } else {
+      return InvalidConversion(object, "decimal.Decimal");
+    }
+  }
+  return decimal_builder.Finish(&out_);
+}
+
+#undef CONVERT_DECIMAL_CASE
+
 Status PandasConverter::ConvertObjectStrings() {
   PyAcquireGIL lock;
 
@@ -554,6 +609,90 @@ Status PandasConverter::ConvertObjectFixedWidthBytes(
   return Status::OK();
 }
 
+template <typename T>
+Status validate_precision(int precision) {
+  constexpr static const int maximum_precision = DecimalPrecision<T>::maximum;
+  if (!(precision > 0 && precision <= maximum_precision)) {
+    std::stringstream ss;
+    ss << "Invalid precision: " << precision << ". Minimum is 1, maximum is "
+       << maximum_precision;
+    return Status::Invalid(ss.str());
+  }
+  return Status::OK();
+}
+
+template <typename T>
+Status RawDecimalToString(
+    const uint8_t* bytes, int precision, int scale, std::string* result) {
+  DCHECK_NE(bytes, nullptr);
+  DCHECK_NE(result, nullptr);
+  RETURN_NOT_OK(validate_precision<T>(precision));
+  Decimal<T> decimal;
+  FromBytes(bytes, &decimal);
+  *result = ToString(decimal, precision, scale);
+  return Status::OK();
+}
+
+template Status RawDecimalToString<int32_t>(
+    const uint8_t*, int, int, std::string* result);
+template Status RawDecimalToString<int64_t>(
+    const uint8_t*, int, int, std::string* result);
+
+Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
+    bool is_negative, std::string* result) {
+  DCHECK_NE(bytes, nullptr);
+  DCHECK_NE(result, nullptr);
+  RETURN_NOT_OK(validate_precision<int128_t>(precision));
+  Decimal128 decimal;
+  FromBytes(bytes, is_negative, &decimal);
+  *result = ToString(decimal, precision, scale);
+  return Status::OK();
+}
+
+static Status ConvertDecimals(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  OwnedRef decimal_ref;
+  OwnedRef Decimal_ref;
+  RETURN_NOT_OK(ImportModule("decimal", &decimal_ref));
+  RETURN_NOT_OK(ImportFromModule(decimal_ref, "Decimal", &Decimal_ref));
+  PyObject* Decimal = Decimal_ref.obj();
+
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto* arr(static_cast<arrow::DecimalArray*>(data.chunk(c).get()));
+    auto type(std::dynamic_pointer_cast<arrow::DecimalType>(arr->type()));
+    const int precision = type->precision;
+    const int scale = type->scale;
+    const int bit_width = type->bit_width();
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values++ = Py_None;
+      } else {
+        const uint8_t* raw_value = arr->GetValue(i);
+        std::string s;
+        switch (bit_width) {
+          case 32:
+            RETURN_NOT_OK(RawDecimalToString<int32_t>(raw_value, precision, scale, &s));
+            break;
+          case 64:
+            RETURN_NOT_OK(RawDecimalToString<int64_t>(raw_value, precision, scale, &s));
+            break;
+          case 128:
+            RETURN_NOT_OK(
+                RawDecimalToString(raw_value, precision, scale, arr->IsNegative(i), &s));
+            break;
+          default:
+            break;
+        }
+        RETURN_NOT_OK(DecimalFromString(Decimal, s, out_values++));
+      }
+    }
+  }
+
+  return Status::OK();
+}
+
 Status PandasConverter::ConvertBooleans() {
   PyAcquireGIL lock;
 
@@ -598,6 +737,7 @@ Status PandasConverter::ConvertObjects() {
   //
   // * Strings
   // * Booleans with nulls
+  // * decimal.Decimals
   // * Mixed type (not supported at the moment by arrow format)
   //
   // Additionally, nulls may be encoded either as np.nan or None. So we have to
@@ -613,6 +753,7 @@ Status PandasConverter::ConvertObjects() {
     PyDateTime_IMPORT;
   }
 
+  // This means we received an explicit type from the user
   if (type_) {
     switch (type_->type) {
       case Type::STRING:
@@ -627,10 +768,17 @@ Status PandasConverter::ConvertObjects() {
         const auto& list_field = static_cast<const ListType&>(*type_);
         return ConvertLists(list_field.value_field()->type);
       }
+      case Type::DECIMAL:
+        return ConvertDecimals();
       default:
         return Status::TypeError("No known conversion to Arrow type");
     }
   } else {
+    OwnedRef decimal;
+    OwnedRef Decimal;
+    RETURN_NOT_OK(ImportModule("decimal", &decimal));
+    RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+
     for (int64_t i = 0; i < length_; ++i) {
       if (PyObject_is_null(objects[i])) {
         continue;
@@ -640,6 +788,8 @@ Status PandasConverter::ConvertObjects() {
         return ConvertBooleans();
       } else if (PyDate_CheckExact(objects[i])) {
         return ConvertDates();
+      } else if (PyObject_IsInstance(const_cast<PyObject*>(objects[i]), Decimal.obj())) {
+        return ConvertDecimals();
       } else {
         return InvalidConversion(
             const_cast<PyObject*>(objects[i]), "string, bool, or date");
@@ -847,6 +997,7 @@ class PandasBlock {
     INT64,
     FLOAT,
     DOUBLE,
+    DECIMAL,
     BOOL,
     DATETIME,
     DATETIME_WITH_TZ,
@@ -1193,6 +1344,8 @@ class ObjectBlock : public PandasBlock {
       RETURN_NOT_OK(ConvertBinaryLike<StringArray>(data, out_buffer));
     } else if (type == Type::FIXED_SIZE_BINARY) {
       RETURN_NOT_OK(ConvertFixedSizeBinary(data, out_buffer));
+    } else if (type == Type::DECIMAL) {
+      RETURN_NOT_OK(ConvertDecimals(data, out_buffer));
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
       switch (list_type->value_type()->type) {
@@ -1519,6 +1672,7 @@ Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
     BLOCK_CASE(DOUBLE, Float64Block);
     BLOCK_CASE(BOOL, BoolBlock);
     BLOCK_CASE(DATETIME, DatetimeBlock);
+    BLOCK_CASE(DECIMAL, ObjectBlock);
     default:
       return Status::NotImplemented("Unsupported block type");
   }
@@ -1649,6 +1803,9 @@ class DataFrameBlockCreator {
         case Type::DICTIONARY:
           output_type = PandasBlock::CATEGORICAL;
           break;
+        case Type::DECIMAL:
+          output_type = PandasBlock::DECIMAL;
+          break;
         default:
           return Status::NotImplemented(col->type()->ToString());
       }
@@ -1892,6 +2049,7 @@ class ArrowDeserializer {
       CONVERT_CASE(TIMESTAMP);
       CONVERT_CASE(DICTIONARY);
       CONVERT_CASE(LIST);
+      CONVERT_CASE(DECIMAL);
       default: {
         std::stringstream ss;
         ss << "Arrow type reading not implemented for " << col_->type()->ToString();
@@ -1999,6 +2157,13 @@ class ArrowDeserializer {
     return ConvertFixedSizeBinary(data_, out_values);
   }
 
+  template <int TYPE>
+  inline typename std::enable_if<TYPE == Type::DECIMAL, Status>::type ConvertValues() {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    return ConvertDecimals(data_, out_values);
+  }
+
 #define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \
   case Type::ArrowEnum:                                    \
     return ConvertListsLike<ArrowType>(col_, out_values);
@@ -2021,6 +2186,7 @@ class ArrowDeserializer {
       CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT)
       CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE)
       CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING)
+      CONVERTVALUES_LISTSLIKE_CASE(DecimalType, DECIMAL)
       default: {
         std::stringstream ss;
         ss << "Not implemented type for lists: " << list_type->value_type()->ToString();
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index f269ebfb642c7..b63d2ffb1cd2c 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -28,8 +28,11 @@
 
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
+#include "arrow/python/helpers.h"
 #include "arrow/python/pandas_convert.h"
 
+#include "arrow/util/decimal.h"
+
 namespace arrow {
 namespace py {
 
@@ -37,6 +40,36 @@ TEST(PyBuffer, InvalidInputObject) {
   PyBuffer buffer(Py_None);
 }
 
+TEST(DecimalTest, TestPythonDecimalToArrowDecimal128) {
+  PyAcquireGIL lock;
+
+  OwnedRef decimal;
+  OwnedRef Decimal;
+  ASSERT_OK(ImportModule("decimal", &decimal));
+  ASSERT_NE(decimal.obj(), nullptr);
+
+  ASSERT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+  ASSERT_NE(Decimal.obj(), nullptr);
+
+  std::string decimal_string("-39402950693754869342983");
+  const char* format = "s#";
+  auto c_string = decimal_string.c_str();
+  ASSERT_NE(c_string, nullptr);
+
+  auto c_string_size = decimal_string.size();
+  ASSERT_GT(c_string_size, 0);
+  OwnedRef pydecimal(PyObject_CallFunction(
+      Decimal.obj(), const_cast<char*>(format), c_string, c_string_size));
+  ASSERT_NE(pydecimal.obj(), nullptr);
+  ASSERT_EQ(PyErr_Occurred(), nullptr);
+
+  Decimal128 arrow_decimal;
+  int128_t boost_decimal(decimal_string);
+  PyObject* obj = pydecimal.obj();
+  ASSERT_OK(PythonDecimalToArrowDecimal(obj, &arrow_decimal));
+  ASSERT_EQ(boost_decimal, arrow_decimal.value);
+}
+
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   StringBuilder builder(default_memory_pool());
   const char value[] = {'\xf1', '\0'};
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index abbb626e0fcb4..df4590f18d733 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/type.h"
 
+#include <climits>
 #include <sstream>
 #include <string>
 
@@ -91,7 +92,7 @@ std::string BinaryType::ToString() const {
 }
 
 int FixedSizeBinaryType::bit_width() const {
-  return 8 * byte_width();
+  return CHAR_BIT * byte_width();
 }
 
 std::string FixedSizeBinaryType::ToString() const {
@@ -380,6 +381,10 @@ std::shared_ptr<Field> field(
   return std::make_shared<Field>(name, type, nullable);
 }
 
+std::shared_ptr<DataType> decimal(int precision, int scale) {
+  return std::make_shared<DecimalType>(precision, scale);
+}
+
 static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1);
 static const BufferDescr kOffsetBuffer(BufferType::OFFSET, 32);
 static const BufferDescr kTypeBuffer(BufferType::TYPE, 32);
@@ -402,7 +407,11 @@ std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
 }
 
 std::vector<BufferDescr> FixedSizeBinaryType::GetBufferLayout() const {
-  return {kValidityBuffer, BufferDescr(BufferType::DATA, byte_width_ * 8)};
+  return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
+}
+
+std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
+  return {kValidityBuffer, kBooleanBuffer, BufferDescr(BufferType::DATA, bit_width())};
 }
 
 std::vector<BufferDescr> ListType::GetBufferLayout() const {
@@ -427,9 +436,4 @@ std::string DecimalType::ToString() const {
   return s.str();
 }
 
-std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
-  // TODO(wesm)
-  return {};
-}
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 36ab9d8b2b9d5..3a35f56381197 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -360,6 +360,8 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType {
 
   explicit FixedSizeBinaryType(int32_t byte_width)
       : FixedWidthType(Type::FIXED_SIZE_BINARY), byte_width_(byte_width) {}
+  explicit FixedSizeBinaryType(int32_t byte_width, Type::type type_id)
+      : FixedWidthType(type_id), byte_width_(byte_width) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -399,19 +401,31 @@ struct ARROW_EXPORT StructType : public NestedType {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-struct ARROW_EXPORT DecimalType : public DataType {
+static inline int decimal_byte_width(int precision) {
+  if (precision >= 0 && precision < 10) {
+    return 4;
+  } else if (precision >= 10 && precision < 19) {
+    return 8;
+  } else {
+    // TODO(phillipc): validate that we can't construct > 128 bit types
+    return 16;
+  }
+}
+
+struct ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
   static constexpr Type::type type_id = Type::DECIMAL;
 
   explicit DecimalType(int precision_, int scale_)
-      : DataType(Type::DECIMAL), precision(precision_), scale(scale_) {}
-  int precision;
-  int scale;
-
+      : FixedSizeBinaryType(decimal_byte_width(precision_), Type::DECIMAL),
+        precision(precision_),
+        scale(scale_) {}
+  std::vector<BufferDescr> GetBufferLayout() const override;
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "decimal"; }
 
-  std::vector<BufferDescr> GetBufferLayout() const override;
+  int precision;
+  int scale;
 };
 
 enum class UnionMode : char { SPARSE, DENSE };
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 2e27ce9858964..acf12c3d9d18e 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -69,6 +69,7 @@ class StructBuilder;
 
 struct DecimalType;
 class DecimalArray;
+class DecimalBuilder;
 
 struct UnionType;
 class UnionArray;
@@ -146,6 +147,7 @@ std::shared_ptr<DataType> ARROW_EXPORT binary();
 
 std::shared_ptr<DataType> ARROW_EXPORT date32();
 std::shared_ptr<DataType> ARROW_EXPORT date64();
+std::shared_ptr<DataType> ARROW_EXPORT decimal(int precision, int scale);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 353b638fed894..3e8ea23432b98 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -228,6 +228,13 @@ struct TypeTraits<DoubleType> {
   static inline std::shared_ptr<DataType> type_singleton() { return float64(); }
 };
 
+template <>
+struct TypeTraits<DecimalType> {
+  using ArrayType = DecimalArray;
+  using BuilderType = DecimalBuilder;
+  constexpr static bool is_parameter_free = false;
+};
+
 template <>
 struct TypeTraits<BooleanType> {
   using ArrayType = BooleanArray;
@@ -289,12 +296,6 @@ struct TypeTraits<DictionaryType> {
   constexpr static bool is_parameter_free = false;
 };
 
-template <>
-struct TypeTraits<DecimalType> {
-  // using ArrayType = DecimalArray;
-  constexpr static bool is_parameter_free = false;
-};
-
 // Not all type classes have a c_type
 template <typename T>
 struct as_void {
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index c1b6877a3e9ef..054f11055b60e 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -22,6 +22,7 @@
 # Headers: top level
 install(FILES
   bit-util.h
+  decimal.h
   logging.h
   macros.h
   random.h
@@ -70,3 +71,4 @@ endif()
 
 ADD_ARROW_TEST(bit-util-test)
 ADD_ARROW_TEST(stl-util-test)
+ADD_ARROW_TEST(decimal-test)
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 42afd0705f0f9..90a1c3eab9266 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -149,7 +149,6 @@ int64_t ARROW_EXPORT CountSetBits(
 
 bool ARROW_EXPORT BitmapEquals(const uint8_t* left, int64_t left_offset,
     const uint8_t* right, int64_t right_offset, int64_t bit_length);
-
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_BIT_UTIL_H
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
new file mode 100644
index 0000000000000..1e22643962d5b
--- /dev/null
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -0,0 +1,161 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+#include "arrow/util/decimal.h"
+
+#include "gtest/gtest.h"
+
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+template <typename T>
+class DecimalTest : public ::testing::Test {
+ public:
+  DecimalTest() : string_value("234.23445") { integer_value.value = 23423445; }
+  Decimal<T> integer_value;
+  std::string string_value;
+};
+
+typedef ::testing::Types<int32_t, int64_t, int128_t> DecimalTypes;
+TYPED_TEST_CASE(DecimalTest, DecimalTypes);
+
+TYPED_TEST(DecimalTest, TestToString) {
+  Decimal<TypeParam> decimal(this->integer_value);
+  int precision = 8;
+  int scale = 5;
+  std::string result = ToString(decimal, precision, scale);
+  ASSERT_EQ(result, this->string_value);
+}
+
+TYPED_TEST(DecimalTest, TestFromString) {
+  Decimal<TypeParam> expected(this->integer_value);
+  Decimal<TypeParam> result;
+  int precision, scale;
+  ASSERT_OK(FromString(this->string_value, &result, &precision, &scale));
+  ASSERT_EQ(result.value, expected.value);
+  ASSERT_EQ(precision, 8);
+  ASSERT_EQ(scale, 5);
+}
+
+TEST(DecimalTest, TestStringToInt32) {
+  int32_t value = 0;
+  StringToInteger("123", "456", 1, &value);
+  ASSERT_EQ(value, 123456);
+}
+
+TEST(DecimalTest, TestStringToInt64) {
+  int64_t value = 0;
+  StringToInteger("123456789", "456", -1, &value);
+  ASSERT_EQ(value, -123456789456);
+}
+
+TEST(DecimalTest, TestStringToInt128) {
+  int128_t value = 0;
+  StringToInteger("123456789", "456789123", 1, &value);
+  ASSERT_EQ(value, 123456789456789123);
+}
+
+TEST(DecimalTest, TestFromString128) {
+  static const std::string string_value("-23049223942343532412");
+  Decimal<int128_t> result(string_value);
+  int128_t expected = -230492239423435324;
+  ASSERT_EQ(result.value, expected * 100 - 12);
+
+  // Sanity check that our number is actually using more than 64 bits
+  ASSERT_NE(result.value, static_cast<int64_t>(result.value));
+}
+
+TEST(DecimalTest, TestFromDecimalString128) {
+  static const std::string string_value("-23049223942343.532412");
+  Decimal<int128_t> result(string_value);
+  int128_t expected = -230492239423435324;
+  ASSERT_EQ(result.value, expected * 100 - 12);
+
+  // Sanity check that our number is actually using more than 64 bits
+  ASSERT_NE(result.value, static_cast<int64_t>(result.value));
+}
+
+TEST(DecimalTest, TestDecimal32Precision) {
+  auto min_precision = DecimalPrecision<int32_t>::minimum;
+  auto max_precision = DecimalPrecision<int32_t>::maximum;
+  ASSERT_EQ(min_precision, 1);
+  ASSERT_EQ(max_precision, 9);
+}
+
+TEST(DecimalTest, TestDecimal64Precision) {
+  auto min_precision = DecimalPrecision<int64_t>::minimum;
+  auto max_precision = DecimalPrecision<int64_t>::maximum;
+  ASSERT_EQ(min_precision, 10);
+  ASSERT_EQ(max_precision, 18);
+}
+
+TEST(DecimalTest, TestDecimal128Precision) {
+  auto min_precision = DecimalPrecision<int128_t>::minimum;
+  auto max_precision = DecimalPrecision<int128_t>::maximum;
+  ASSERT_EQ(min_precision, 19);
+  ASSERT_EQ(max_precision, 38);
+}
+
+TEST(DecimalTest, TestDecimal32SignedRoundTrip) {
+  Decimal32 expected(std::string("-3402692"));
+
+  uint8_t stack_bytes[4] = {0};
+  uint8_t* bytes = stack_bytes;
+  ToBytes(expected, &bytes);
+
+  Decimal32 result;
+  FromBytes(bytes, &result);
+  ASSERT_EQ(expected.value, result.value);
+}
+
+TEST(DecimalTest, TestDecimal64SignedRoundTrip) {
+  Decimal64 expected(std::string("-34034293045.921"));
+
+  uint8_t stack_bytes[8] = {0};
+  uint8_t* bytes = stack_bytes;
+  ToBytes(expected, &bytes);
+
+  Decimal64 result;
+  FromBytes(bytes, &result);
+
+  ASSERT_EQ(expected.value, result.value);
+}
+
+TEST(DecimalTest, TestDecimal128StringAndBytesRoundTrip) {
+  std::string string_value("-340282366920938463463374607431.711455");
+  Decimal128 expected(string_value);
+
+  std::string expected_string_value("-340282366920938463463374607431711455");
+  int128_t expected_underlying_value(expected_string_value);
+
+  ASSERT_EQ(expected.value, expected_underlying_value);
+
+  uint8_t stack_bytes[16] = {0};
+  uint8_t* bytes = stack_bytes;
+  bool is_negative;
+  ToBytes(expected, &bytes, &is_negative);
+
+  ASSERT_TRUE(is_negative);
+
+  Decimal128 result;
+  FromBytes(bytes, is_negative, &result);
+
+  ASSERT_EQ(expected.value, result.value);
+}
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
new file mode 100644
index 0000000000000..1ac347180fec5
--- /dev/null
+++ b/cpp/src/arrow/util/decimal.cc
@@ -0,0 +1,141 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/decimal.h"
+
+#include <boost/regex.hpp>
+
+namespace arrow {
+
+static const boost::regex DECIMAL_PATTERN("(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?");
+
+template <typename T>
+ARROW_EXPORT Status FromString(
+    const std::string& s, Decimal<T>* out, int* precision, int* scale) {
+  if (s.empty()) {
+    return Status::Invalid("Empty string cannot be converted to decimal");
+  }
+  boost::smatch match;
+  if (!boost::regex_match(s, match, DECIMAL_PATTERN)) {
+    std::stringstream ss;
+    ss << "String " << s << " is not a valid decimal string";
+    return Status::Invalid(ss.str());
+  }
+  const int8_t sign = match[1].str() == "-" ? -1 : 1;
+  std::string whole_part = match[4].str();
+  std::string fractional_part = match[6].str();
+  if (scale != nullptr) { *scale = static_cast<int>(fractional_part.size()); }
+  if (precision != nullptr) {
+    *precision =
+        static_cast<int>(whole_part.size()) + static_cast<int>(fractional_part.size());
+  }
+  if (out != nullptr) { StringToInteger(whole_part, fractional_part, sign, &out->value); }
+  return Status::OK();
+}
+
+template ARROW_EXPORT Status FromString(
+    const std::string& s, Decimal32* out, int* precision, int* scale);
+template ARROW_EXPORT Status FromString(
+    const std::string& s, Decimal64* out, int* precision, int* scale);
+template ARROW_EXPORT Status FromString(
+    const std::string& s, Decimal128* out, int* precision, int* scale);
+
+void StringToInteger(
+    const std::string& whole, const std::string& fractional, int8_t sign, int32_t* out) {
+  DCHECK(sign == -1 || sign == 1);
+  DCHECK_NE(out, nullptr);
+  DCHECK(!whole.empty() || !fractional.empty());
+  if (!whole.empty()) {
+    *out = std::stoi(whole, nullptr, 10) *
+           static_cast<int32_t>(pow(10.0, static_cast<double>(fractional.size())));
+  }
+  if (!fractional.empty()) { *out += std::stoi(fractional, nullptr, 10); }
+  *out *= sign;
+}
+
+void StringToInteger(
+    const std::string& whole, const std::string& fractional, int8_t sign, int64_t* out) {
+  DCHECK(sign == -1 || sign == 1);
+  DCHECK_NE(out, nullptr);
+  DCHECK(!whole.empty() || !fractional.empty());
+  if (!whole.empty()) {
+    *out = static_cast<int64_t>(std::stoll(whole, nullptr, 10)) *
+           static_cast<int64_t>(pow(10.0, static_cast<double>(fractional.size())));
+  }
+  if (!fractional.empty()) { *out += std::stoll(fractional, nullptr, 10); }
+  *out *= sign;
+}
+
+void StringToInteger(
+    const std::string& whole, const std::string& fractional, int8_t sign, int128_t* out) {
+  DCHECK(sign == -1 || sign == 1);
+  DCHECK_NE(out, nullptr);
+  DCHECK(!whole.empty() || !fractional.empty());
+  *out = int128_t(whole + fractional) * sign;
+}
+
+void FromBytes(const uint8_t* bytes, Decimal32* decimal) {
+  DCHECK_NE(bytes, nullptr);
+  DCHECK_NE(decimal, nullptr);
+  decimal->value = *reinterpret_cast<const int32_t*>(bytes);
+}
+
+void FromBytes(const uint8_t* bytes, Decimal64* decimal) {
+  DCHECK_NE(bytes, nullptr);
+  DCHECK_NE(decimal, nullptr);
+  decimal->value = *reinterpret_cast<const int64_t*>(bytes);
+}
+
+constexpr static const size_t BYTES_IN_128_BITS = 128 / CHAR_BIT;
+constexpr static const size_t LIMB_SIZE =
+    sizeof(std::remove_pointer<int128_t::backend_type::limb_pointer>::type);
+constexpr static const size_t BYTES_PER_LIMB = BYTES_IN_128_BITS / LIMB_SIZE;
+
+void FromBytes(const uint8_t* bytes, bool is_negative, Decimal128* decimal) {
+  DCHECK_NE(bytes, nullptr);
+  DCHECK_NE(decimal, nullptr);
+
+  auto& decimal_value(decimal->value);
+  int128_t::backend_type& backend(decimal_value.backend());
+  backend.resize(BYTES_PER_LIMB, BYTES_PER_LIMB);
+  std::memcpy(backend.limbs(), bytes, BYTES_IN_128_BITS);
+  if (is_negative) { decimal->value = -decimal->value; }
+}
+
+void ToBytes(const Decimal32& value, uint8_t** bytes) {
+  DCHECK_NE(*bytes, nullptr);
+  *reinterpret_cast<int32_t*>(*bytes) = value.value;
+}
+
+void ToBytes(const Decimal64& value, uint8_t** bytes) {
+  DCHECK_NE(*bytes, nullptr);
+  *reinterpret_cast<int64_t*>(*bytes) = value.value;
+}
+
+void ToBytes(const Decimal128& decimal, uint8_t** bytes, bool* is_negative) {
+  DCHECK_NE(*bytes, nullptr);
+  DCHECK_NE(is_negative, nullptr);
+
+  /// TODO(phillipc): boost multiprecision is unreliable here, int128_t can't be
+  /// roundtripped
+  const auto& backend(decimal.value.backend());
+  auto boost_bytes = reinterpret_cast<const uint8_t*>(backend.limbs());
+  std::memcpy(*bytes, boost_bytes, BYTES_IN_128_BITS);
+  *is_negative = backend.isneg();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
new file mode 100644
index 0000000000000..46883e3de93c3
--- /dev/null
+++ b/cpp/src/arrow/util/decimal.h
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_DECIMAL_H
+#define ARROW_DECIMAL_H
+
+#include <cmath>
+#include <cstdlib>
+#include <iterator>
+#include <regex>
+#include <string>
+
+#include "arrow/status.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/logging.h"
+
+#include <boost/multiprecision/cpp_int.hpp>
+
+namespace arrow {
+
+using boost::multiprecision::int128_t;
+
+template <typename T>
+struct ARROW_EXPORT Decimal;
+
+ARROW_EXPORT void StringToInteger(
+    const std::string& whole, const std::string& fractional, int8_t sign, int32_t* out);
+ARROW_EXPORT void StringToInteger(
+    const std::string& whole, const std::string& fractional, int8_t sign, int64_t* out);
+ARROW_EXPORT void StringToInteger(
+    const std::string& whole, const std::string& fractional, int8_t sign, int128_t* out);
+
+template <typename T>
+ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out,
+    int* precision = nullptr, int* scale = nullptr);
+
+template <typename T>
+struct ARROW_EXPORT Decimal {
+  Decimal() : value() {}
+  explicit Decimal(const std::string& s) : value() { FromString(s, this); }
+  explicit Decimal(const char* s) : Decimal(std::string(s)) {}
+  explicit Decimal(const T& value) : value(value) {}
+
+  using value_type = T;
+  value_type value;
+};
+
+using Decimal32 = Decimal<int32_t>;
+using Decimal64 = Decimal<int64_t>;
+using Decimal128 = Decimal<int128_t>;
+
+template <typename T>
+struct ARROW_EXPORT DecimalPrecision {};
+
+template <>
+struct ARROW_EXPORT DecimalPrecision<int32_t> {
+  constexpr static const int minimum = 1;
+  constexpr static const int maximum = 9;
+};
+
+template <>
+struct ARROW_EXPORT DecimalPrecision<int64_t> {
+  constexpr static const int minimum = 10;
+  constexpr static const int maximum = 18;
+};
+
+template <>
+struct ARROW_EXPORT DecimalPrecision<int128_t> {
+  constexpr static const int minimum = 19;
+  constexpr static const int maximum = 38;
+};
+
+template <typename T>
+ARROW_EXPORT std::string ToString(
+    const Decimal<T>& decimal_value, int precision, int scale) {
+  T value = decimal_value.value;
+
+  // Decimal values are sent to clients as strings so in the interest of
+  // speed the string will be created without the using stringstream with the
+  // whole/fractional_part().
+  size_t last_char_idx = precision + (scale > 0)  // Add a space for decimal place
+                         + (scale == precision)   // Add a space for leading 0
+                         + (value < 0);           // Add a space for negative sign
+  std::string str = std::string(last_char_idx, '0');
+  // Start filling in the values in reverse order by taking the last digit
+  // of the value. Use a positive value and worry about the sign later. At this
+  // point the last_char_idx points to the string terminator.
+  T remaining_value = value;
+  size_t first_digit_idx = 0;
+  if (value < 0) {
+    remaining_value = -value;
+    first_digit_idx = 1;
+  }
+  if (scale > 0) {
+    int remaining_scale = scale;
+    do {
+      str[--last_char_idx] = static_cast<char>(
+          (remaining_value % 10) + static_cast<T>('0'));  // Ascii offset
+      remaining_value /= 10;
+    } while (--remaining_scale > 0);
+    str[--last_char_idx] = '.';
+    DCHECK_GT(last_char_idx, first_digit_idx) << "Not enough space remaining";
+  }
+  do {
+    str[--last_char_idx] =
+        static_cast<char>((remaining_value % 10) + static_cast<T>('0'));  // Ascii offset
+    remaining_value /= 10;
+    if (remaining_value == 0) {
+      // Trim any extra leading 0's.
+      if (last_char_idx > first_digit_idx) str.erase(0, last_char_idx - first_digit_idx);
+      break;
+    }
+    // For safety, enforce string length independent of remaining_value.
+  } while (last_char_idx > first_digit_idx);
+  if (value < 0) str[0] = '-';
+  return str;
+}
+
+/// Conversion from raw bytes to a Decimal value
+ARROW_EXPORT void FromBytes(const uint8_t* bytes, Decimal32* value);
+ARROW_EXPORT void FromBytes(const uint8_t* bytes, Decimal64* value);
+ARROW_EXPORT void FromBytes(const uint8_t* bytes, bool is_negative, Decimal128* decimal);
+
+/// Conversion from a Decimal value to raw bytes
+ARROW_EXPORT void ToBytes(const Decimal32& value, uint8_t** bytes);
+ARROW_EXPORT void ToBytes(const Decimal64& value, uint8_t** bytes);
+ARROW_EXPORT void ToBytes(const Decimal128& decimal, uint8_t** bytes, bool* is_negative);
+
+}  // namespace arrow
+#endif  // ARROW_DECIMAL_H
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index c61c9f59f7ab2..29b3db60cadf8 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -93,7 +93,7 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
     ARRAY_VISIT_INLINE(TimestampType);
     ARRAY_VISIT_INLINE(Time32Type);
     ARRAY_VISIT_INLINE(Time64Type);
-    // ARRAY_VISIT_INLINE(DecimalType);
+    ARRAY_VISIT_INLINE(DecimalType);
     ARRAY_VISIT_INLINE(ListType);
     ARRAY_VISIT_INLINE(StructType);
     ARRAY_VISIT_INLINE(UnionType);
diff --git a/format/Schema.fbs b/format/Schema.fbs
index ca9c8e6c3e76c..badc7ea8befbf 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -77,7 +77,9 @@ table Bool {
 }
 
 table Decimal {
+  /// Total number of decimal digits
   precision: int;
+  /// Number of digits after the decimal point "."
   scale: int;
 }
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 8c520748cf316..7b23cf66c6f7e 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -71,7 +71,7 @@
                             uint8, uint16, uint32, uint64,
                             timestamp, date32, date64,
                             float16, float32, float64,
-                            binary, string,
+                            binary, string, decimal,
                             list_, struct, dictionary, field,
                             DataType, FixedSizeBinaryType,
                             Field, Schema, schema)
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/array.pxd
index f6aaea2582e21..3ba48718265db 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/array.pxd
@@ -116,6 +116,10 @@ cdef class FixedSizeBinaryArray(Array):
     pass
 
 
+cdef class DecimalArray(FixedSizeBinaryArray):
+    pass
+
+
 cdef class ListArray(Array):
     pass
 
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index 9f302e02cdb04..ee500e6812974 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -481,6 +481,10 @@ cdef class FixedSizeBinaryArray(Array):
     pass
 
 
+cdef class DecimalArray(FixedSizeBinaryArray):
+    pass
+
+
 cdef class ListArray(Array):
     pass
 
@@ -602,6 +606,7 @@ cdef dict _array_classes = {
     Type_STRING: StringArray,
     Type_DICTIONARY: DictionaryArray,
     Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
+    Type_DECIMAL: DecimalArray,
 }
 
 cdef object box_array(const shared_ptr[CArray]& sp_array):
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index ab38ff3084f01..4860334a9213c 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -51,6 +51,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsTypeError()
 
 
+cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil:
+    cdef cppclass int128_t:
+        pass
+
+
 cdef inline object PyObject_to_object(PyObject* o):
     # Cast to "object" increments reference count
     cdef object result = <object> o
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 2a0488f3a0139..73d96b25f521b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -39,6 +39,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         Type_FLOAT" arrow::Type::FLOAT"
         Type_DOUBLE" arrow::Type::DOUBLE"
 
+        Type_DECIMAL" arrow::Type::DECIMAL"
+
         Type_DATE32" arrow::Type::DATE32"
         Type_DATE64" arrow::Type::DATE64"
         Type_TIMESTAMP" arrow::Type::TIMESTAMP"
@@ -58,6 +60,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         TimeUnit_MICRO" arrow::TimeUnit::MICRO"
         TimeUnit_NANO" arrow::TimeUnit::NANO"
 
+    cdef cppclass Decimal[T]:
+        Decimal(const T&)
+
+    cdef c_string ToString[T](const Decimal[T]&, int, int)
+
     cdef cppclass CDataType" arrow::DataType":
         Type type
 
@@ -144,6 +151,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CFixedSizeBinaryType" arrow::FixedSizeBinaryType"(CFixedWidthType):
         CFixedSizeBinaryType(int byte_width)
         int byte_width()
+        int bit_width()
+
+    cdef cppclass CDecimalType" arrow::DecimalType"(CFixedSizeBinaryType):
+        int precision
+        int scale
+        CDecimalType(int precision, int scale)
 
     cdef cppclass CField" arrow::Field":
         c_string name
@@ -212,6 +225,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CFixedSizeBinaryArray" arrow::FixedSizeBinaryArray"(CArray):
         const uint8_t* GetValue(int i)
 
+    cdef cppclass CDecimalArray" arrow::DecimalArray"(CFixedSizeBinaryArray):
+        Decimal[T] Value[T](int i)
+
     cdef cppclass CListArray" arrow::ListArray"(CArray):
         const int32_t* raw_value_offsets()
         int32_t value_offset(int i)
diff --git a/python/pyarrow/scalar.pxd b/python/pyarrow/scalar.pxd
index d6c3b35160c12..62a5664e57eb4 100644
--- a/python/pyarrow/scalar.pxd
+++ b/python/pyarrow/scalar.pxd
@@ -20,6 +20,7 @@ from pyarrow.includes.libarrow cimport *
 
 from pyarrow.schema cimport DataType
 
+
 cdef class Scalar:
     cdef readonly:
         DataType type
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 1c0790a4fdc3c..f3d9321326964 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -17,9 +17,10 @@
 
 from pyarrow.schema cimport DataType, box_data_type
 
+from pyarrow.includes.common cimport int128_t
 from pyarrow.compat import frombytes
 import pyarrow.schema as schema
-
+import decimal
 import datetime
 
 cimport cpython as cp
@@ -64,7 +65,7 @@ cdef class ArrayValue(Scalar):
         if hasattr(self, 'as_py'):
             return repr(self.as_py())
         else:
-            return Scalar.__repr__(self)
+            return super(Scalar, self).__repr__()
 
 
 cdef class BooleanValue(ArrayValue):
@@ -199,6 +200,25 @@ cdef class DoubleValue(ArrayValue):
         return ap.Value(self.index)
 
 
+cdef class DecimalValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CDecimalArray* ap = <CDecimalArray*> self.sp_array.get()
+            CDecimalType* t = <CDecimalType*> ap.type().get()
+            int bit_width = t.bit_width()
+            int precision = t.precision
+            int scale = t.scale
+            c_string s
+        if bit_width == 32:
+            s = ToString[int32_t](ap.Value[int32_t](self.index), precision, scale)
+        elif bit_width == 64:
+            s = ToString[int64_t](ap.Value[int64_t](self.index), precision, scale)
+        elif bit_width == 128:
+            s = ToString[int128_t](ap.Value[int128_t](self.index), precision, scale)
+        return decimal.Decimal(s.decode('utf8'))
+
+
 cdef class StringValue(ArrayValue):
 
     def as_py(self):
@@ -286,6 +306,7 @@ cdef dict _scalar_classes = {
     Type_BINARY: BinaryValue,
     Type_STRING: StringValue,
     Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
+    Type_DECIMAL: DecimalValue,
 }
 
 cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
index 94d65bfc157a1..eceedbad0ba0d 100644
--- a/python/pyarrow/schema.pxd
+++ b/python/pyarrow/schema.pxd
@@ -20,6 +20,7 @@ from pyarrow.includes.libarrow cimport (CDataType,
                                         CDictionaryType,
                                         CTimestampType,
                                         CFixedSizeBinaryType,
+                                        CDecimalType,
                                         CField, CSchema)
 
 cdef class DataType:
@@ -27,7 +28,7 @@ cdef class DataType:
         shared_ptr[CDataType] sp_type
         CDataType* type
 
-    cdef init(self, const shared_ptr[CDataType]& type)
+    cdef void init(self, const shared_ptr[CDataType]& type)
 
 
 cdef class DictionaryType(DataType):
@@ -45,6 +46,11 @@ cdef class FixedSizeBinaryType(DataType):
         const CFixedSizeBinaryType* fixed_size_binary_type
 
 
+cdef class DecimalType(FixedSizeBinaryType):
+    cdef:
+        const CDecimalType* decimal_type
+
+
 cdef class Field:
     cdef:
         shared_ptr[CField] sp_field
@@ -55,6 +61,7 @@ cdef class Field:
 
     cdef init(self, const shared_ptr[CField]& field)
 
+
 cdef class Schema:
     cdef:
         shared_ptr[CSchema] sp_schema
@@ -63,6 +70,7 @@ cdef class Schema:
     cdef init(self, const vector[shared_ptr[CField]]& fields)
     cdef init_schema(self, const shared_ptr[CSchema]& schema)
 
+
 cdef DataType box_data_type(const shared_ptr[CDataType]& type)
 cdef Field box_field(const shared_ptr[CField]& field)
 cdef Schema box_schema(const shared_ptr[CSchema]& schema)
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 253be4590b518..4b931bf452239 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -29,6 +29,7 @@ from pyarrow.array cimport Array
 from pyarrow.error cimport check_status
 from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
                                         CFixedSizeBinaryType,
+                                        CDecimalType,
                                         TimeUnit_SECOND, TimeUnit_MILLI,
                                         TimeUnit_MICRO, TimeUnit_NANO,
                                         Type, TimeUnit)
@@ -45,7 +46,7 @@ cdef class DataType:
     def __cinit__(self):
         pass
 
-    cdef init(self, const shared_ptr[CDataType]& type):
+    cdef void init(self, const shared_ptr[CDataType]& type):
         self.sp_type = type
         self.type = type.get()
 
@@ -66,14 +67,14 @@ cdef class DataType:
 
 cdef class DictionaryType(DataType):
 
-    cdef init(self, const shared_ptr[CDataType]& type):
+    cdef void init(self, const shared_ptr[CDataType]& type):
         DataType.init(self, type)
         self.dict_type = <const CDictionaryType*> type.get()
 
 
 cdef class TimestampType(DataType):
 
-    cdef init(self, const shared_ptr[CDataType]& type):
+    cdef void init(self, const shared_ptr[CDataType]& type):
         DataType.init(self, type)
         self.ts_type = <const CTimestampType*> type.get()
 
@@ -93,7 +94,7 @@ cdef class TimestampType(DataType):
 
 cdef class FixedSizeBinaryType(DataType):
 
-    cdef init(self, const shared_ptr[CDataType]& type):
+    cdef void init(self, const shared_ptr[CDataType]& type):
         DataType.init(self, type)
         self.fixed_size_binary_type = <const CFixedSizeBinaryType*> type.get()
 
@@ -103,6 +104,13 @@ cdef class FixedSizeBinaryType(DataType):
             return self.fixed_size_binary_type.byte_width()
 
 
+cdef class DecimalType(FixedSizeBinaryType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.decimal_type = <const CDecimalType*> type.get()
+
+
 cdef class Field:
 
     def __cinit__(self):
@@ -354,6 +362,12 @@ def float64():
     return primitive_type(la.Type_DOUBLE)
 
 
+cpdef DataType decimal(int precision, int scale=0):
+    cdef shared_ptr[CDataType] decimal_type
+    decimal_type.reset(new CDecimalType(precision, scale))
+    return box_data_type(decimal_type)
+
+
 def string():
     """
     UTF8 string
@@ -374,11 +388,9 @@ def binary(int length=-1):
     if length == -1:
         return primitive_type(la.Type_BINARY)
 
-    cdef FixedSizeBinaryType out = FixedSizeBinaryType()
     cdef shared_ptr[CDataType] fixed_size_binary_type
     fixed_size_binary_type.reset(new CFixedSizeBinaryType(length))
-    out.init(fixed_size_binary_type)
-    return out
+    return box_data_type(fixed_size_binary_type)
 
 
 def list_(DataType value_type):
@@ -436,6 +448,8 @@ cdef DataType box_data_type(const shared_ptr[CDataType]& type):
         out = TimestampType()
     elif type.get().type == la.Type_FIXED_SIZE_BINARY:
         out = FixedSizeBinaryType()
+    elif type.get().type == la.Type_DECIMAL:
+        out = DecimalType()
     else:
         out = DataType()
 
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index e2b03d85ecd50..d89a8e0c54ceb 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -20,6 +20,7 @@
 import pyarrow as pa
 
 import datetime
+import decimal
 
 
 class TestConvertList(unittest.TestCase):
@@ -162,3 +163,42 @@ def test_mixed_types_fails(self):
         data = ['a', 1, 2.0]
         with self.assertRaises(pa.ArrowException):
             pa.from_pylist(data)
+
+    def test_decimal(self):
+        data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
+        type = pa.decimal(precision=7, scale=3)
+        arr = pa.from_pylist(data, type=type)
+        assert arr.to_pylist() == data
+
+    def test_decimal_different_precisions(self):
+        data = [
+            decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
+        ]
+        type = pa.decimal(precision=13, scale=3)
+        arr = pa.from_pylist(data, type=type)
+        assert arr.to_pylist() == data
+
+    def test_decimal_no_scale(self):
+        data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
+        type = pa.decimal(precision=10)
+        arr = pa.from_pylist(data, type=type)
+        assert arr.to_pylist() == data
+
+    def test_decimal_negative(self):
+        data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
+        type = pa.decimal(precision=10, scale=6)
+        arr = pa.from_pylist(data, type=type)
+        assert arr.to_pylist() == data
+
+    def test_decimal_no_whole_part(self):
+        data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
+        type = pa.decimal(precision=7, scale=7)
+        arr = pa.from_pylist(data, type=type)
+        assert arr.to_pylist() == data
+
+    def test_decimal_large_integer(self):
+        data = [decimal.Decimal('-394029506937548693.42983'),
+                decimal.Decimal('32358695912932.01033')]
+        type = pa.decimal(precision=23, scale=5)
+        arr = pa.from_pylist(data, type=type)
+        assert arr.to_pylist() == data
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 87c9c03d7da11..0504e1ddb4f53 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -20,6 +20,7 @@
 
 import datetime
 import unittest
+import decimal
 
 import numpy as np
 
@@ -451,3 +452,72 @@ def test_strided_data_import(self):
             self._check_pandas_roundtrip(df)
             self._check_array_roundtrip(col)
             self._check_array_roundtrip(col, mask=strided_mask)
+
+    def test_decimal_32_from_pandas(self):
+        expected = pd.DataFrame({
+            'decimals': [
+                decimal.Decimal('-1234.123'),
+                decimal.Decimal('1234.439'),
+            ]
+        })
+        converted = A.Table.from_pandas(expected)
+        field = A.Field.from_py('decimals', A.decimal(7, 3))
+        schema = A.Schema.from_fields([field])
+        assert converted.schema.equals(schema)
+
+    def test_decimal_32_to_pandas(self):
+        expected = pd.DataFrame({
+            'decimals': [
+                decimal.Decimal('-1234.123'),
+                decimal.Decimal('1234.439'),
+            ]
+        })
+        converted = A.Table.from_pandas(expected)
+        df = converted.to_pandas()
+        tm.assert_frame_equal(df, expected)
+
+    def test_decimal_64_from_pandas(self):
+        expected = pd.DataFrame({
+            'decimals': [
+                decimal.Decimal('-129934.123331'),
+                decimal.Decimal('129534.123731'),
+            ]
+        })
+        converted = A.Table.from_pandas(expected)
+        field = A.Field.from_py('decimals', A.decimal(12, 6))
+        schema = A.Schema.from_fields([field])
+        assert converted.schema.equals(schema)
+
+    def test_decimal_64_to_pandas(self):
+        expected = pd.DataFrame({
+            'decimals': [
+                decimal.Decimal('-129934.123331'),
+                decimal.Decimal('129534.123731'),
+            ]
+        })
+        converted = A.Table.from_pandas(expected)
+        df = converted.to_pandas()
+        tm.assert_frame_equal(df, expected)
+
+    def test_decimal_128_from_pandas(self):
+        expected = pd.DataFrame({
+            'decimals': [
+                decimal.Decimal('394092382910493.12341234678'),
+                -decimal.Decimal('314292388910493.12343437128'),
+            ]
+        })
+        converted = A.Table.from_pandas(expected)
+        field = A.Field.from_py('decimals', A.decimal(26, 11))
+        schema = A.Schema.from_fields([field])
+        assert converted.schema.equals(schema)
+
+    def test_decimal_128_to_pandas(self):
+        expected = pd.DataFrame({
+            'decimals': [
+                decimal.Decimal('394092382910493.12341234678'),
+                -decimal.Decimal('314292388910493.12343437128'),
+            ]
+        })
+        converted = A.Table.from_pandas(expected)
+        df = converted.to_pandas()
+        tm.assert_frame_equal(df, expected)

From 137aade404cf53a7dbe0eaa31a868d1c376654b3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 9 Apr 2017 20:00:30 -0400
Subject: [PATCH 0487/1644] ARROW-722: [Python] Support additional date/time
 types and metadata, conversion to/from NumPy and pandas.DataFrame

Would appreciate a close look from @xhochy, @cpcloud. Also did some inline visitor cleaning for nicer code reuse

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #510 from wesm/ARROW-722 and squashes the following commits:

3e1fda3 [Wes McKinney] cpplint
cb32a6b [Wes McKinney] Nicer error message. Run clang-format
854f470 [Wes McKinney] First cut refactor
06dce15 [Wes McKinney] Rebase conflicts
d1dc342 [Wes McKinney] Bring Python bindings to date/time types up to spec. Handle zero-copy creation from same-size int32/64. Use inline visitor in PandasConverter
---
 cpp/CMakeLists.txt                          |   6 +
 cpp/src/arrow/CMakeLists.txt                |   1 +
 cpp/src/arrow/python/builtin_convert.cc     |  29 +-
 cpp/src/arrow/python/builtin_convert.h      |   4 +
 cpp/src/arrow/python/pandas_convert.cc      | 345 ++++++++++----------
 cpp/src/arrow/python/pandas_convert.h       |   3 -
 cpp/src/arrow/python/type_traits.h          |  63 ++--
 cpp/src/arrow/python/util/datetime.h        |   6 +-
 cpp/src/arrow/table-test.cc                 |  55 +---
 cpp/src/arrow/table.cc                      |  10 +-
 cpp/src/arrow/table.h                       |   4 +-
 cpp/src/arrow/type.cc                       |   4 +-
 cpp/src/arrow/type.h                        |   4 +-
 cpp/src/arrow/util/stl.h                    |   4 +-
 python/pyarrow/scalar.pyx                   |   6 +-
 python/pyarrow/tests/test_convert_pandas.py | 182 +++++++----
 16 files changed, 399 insertions(+), 327 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5852fe59da095..b29cb7b075a94 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -386,6 +386,12 @@ enable_testing()
 
 set(Boost_DEBUG TRUE)
 set(Boost_USE_MULTITHREADED ON)
+set(Boost_ADDITIONAL_VERSIONS
+  "1.63.0" "1.63"
+  "1.62.0" "1.61"
+  "1.61.0" "1.62"
+  "1.60.0" "1.60")
+
 if (ARROW_BOOST_USE_SHARED)
   # Find shared Boost libraries.
   set(Boost_USE_STATIC_LIBS OFF)
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 8eaa76ae9e843..cb5282cbf1eff 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -34,6 +34,7 @@ install(FILES
   type_traits.h
   test-util.h
   visitor.h
+  visitor_inline.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow")
 
 # pkg-config support
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 189ecee4fe022..a064a3daf970d 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -43,6 +43,31 @@ static inline bool IsPyInteger(PyObject* obj) {
 #endif
 }
 
+Status InvalidConversion(PyObject* obj, const std::string& expected_type_name) {
+  OwnedRef type(PyObject_Type(obj));
+  RETURN_IF_PYERROR();
+  DCHECK_NE(type.obj(), nullptr);
+
+  OwnedRef type_name(PyObject_GetAttrString(type.obj(), "__name__"));
+  RETURN_IF_PYERROR();
+  DCHECK_NE(type_name.obj(), nullptr);
+
+  PyObjectStringify bytestring(type_name.obj());
+  RETURN_IF_PYERROR();
+
+  const char* bytes = bytestring.bytes;
+  DCHECK_NE(bytes, nullptr) << "bytes from type(...).__name__ were null";
+
+  Py_ssize_t size = bytestring.size;
+
+  std::string cpp_type_name(bytes, size);
+
+  std::stringstream ss;
+  ss << "Python object of type " << cpp_type_name << " is not None and is not a "
+     << expected_type_name << " object";
+  return Status::Invalid(ss.str());
+}
+
 class ScalarVisitor {
  public:
   ScalarVisitor()
@@ -397,7 +422,7 @@ class BytesConverter : public TypedConverter<BinaryBuilder> {
       } else if (PyBytes_Check(item)) {
         bytes_obj = item;
       } else {
-        return Status::Invalid("Value that cannot be converted to bytes was encountered");
+        return InvalidConversion(item, "bytes");
       }
       // No error checking
       length = PyBytes_GET_SIZE(bytes_obj);
@@ -431,7 +456,7 @@ class FixedWidthBytesConverter : public TypedConverter<FixedSizeBinaryBuilder> {
       } else if (PyBytes_Check(item)) {
         bytes_obj = item;
       } else {
-        return Status::Invalid("Value that cannot be converted to bytes was encountered");
+        return InvalidConversion(item, "bytes");
       }
       // No error checking
       RETURN_NOT_OK(CheckPythonBytesAreFixedLength(bytes_obj, expected_length));
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index 3c2e350269a78..2141c25e95ef0 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -24,6 +24,7 @@
 #include <Python.h>
 
 #include <memory>
+#include <string>
 
 #include "arrow/type.h"
 
@@ -60,6 +61,9 @@ ARROW_EXPORT
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
     const std::shared_ptr<DataType>& type, int64_t size);
 
+ARROW_EXPORT
+Status InvalidConversion(PyObject* obj, const std::string& expected_type_name);
+
 ARROW_EXPORT Status CheckPythonBytesAreFixedLength(
     PyObject* obj, Py_ssize_t expected_length);
 
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index f6e627e668e2d..5bb8e45e191a9 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -44,6 +44,7 @@
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
+#include "arrow/visitor_inline.h"
 
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
@@ -271,7 +272,7 @@ static inline bool ListTypeSupported(const Type::type type_id) {
 // ----------------------------------------------------------------------
 // Conversion from NumPy-in-Pandas to Arrow
 
-class PandasConverter : public TypeVisitor {
+class PandasConverter {
  public:
   PandasConverter(
       MemoryPool* pool, PyObject* ao, PyObject* mo, const std::shared_ptr<DataType>& type)
@@ -332,23 +333,37 @@ class PandasConverter : public TypeVisitor {
     return LoadArray(type_, fields, {null_bitmap_, data}, &out_);
   }
 
-#define VISIT_NATIVE(TYPE) \
-  Status Visit(const TYPE& type) override { return VisitNative<TYPE>(); }
+  template <typename T>
+  typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
+                              std::is_same<BooleanType, T>::value,
+      Status>::type
+  Visit(const T& type) {
+    return VisitNative<T>();
+  }
+
+  Status Visit(const Date32Type& type) { return VisitNative<Int32Type>(); }
+  Status Visit(const Date64Type& type) { return VisitNative<Int64Type>(); }
+  Status Visit(const TimestampType& type) { return VisitNative<TimestampType>(); }
+  Status Visit(const Time32Type& type) { return VisitNative<Int32Type>(); }
+  Status Visit(const Time64Type& type) { return VisitNative<Int64Type>(); }
+
+  Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
+
+  Status Visit(const BinaryType& type) { return Status::NotImplemented(type.ToString()); }
+
+  Status Visit(const FixedSizeBinaryType& type) {
+    return Status::NotImplemented(type.ToString());
+  }
 
-  VISIT_NATIVE(BooleanType);
-  VISIT_NATIVE(Int8Type);
-  VISIT_NATIVE(Int16Type);
-  VISIT_NATIVE(Int32Type);
-  VISIT_NATIVE(Int64Type);
-  VISIT_NATIVE(UInt8Type);
-  VISIT_NATIVE(UInt16Type);
-  VISIT_NATIVE(UInt32Type);
-  VISIT_NATIVE(UInt64Type);
-  VISIT_NATIVE(FloatType);
-  VISIT_NATIVE(DoubleType);
-  VISIT_NATIVE(TimestampType);
+  Status Visit(const DecimalType& type) {
+    return Status::NotImplemented(type.ToString());
+  }
 
-#undef VISIT_NATIVE
+  Status Visit(const DictionaryType& type) {
+    return Status::NotImplemented(type.ToString());
+  }
+
+  Status Visit(const NestedType& type) { return Status::NotImplemented(type.ToString()); }
 
   Status Convert() {
     if (PyArray_NDIM(arr_) != 1) {
@@ -358,9 +373,7 @@ class PandasConverter : public TypeVisitor {
     if (type_ == nullptr) { return Status::Invalid("Must pass data type"); }
 
     // Visit the type to perform conversion
-    RETURN_NOT_OK(type_->Accept(this));
-
-    return Status::OK();
+    return VisitTypeInline(*type_, this);
   }
 
   std::shared_ptr<Array> result() const { return out_; }
@@ -371,10 +384,12 @@ class PandasConverter : public TypeVisitor {
   template <int ITEM_TYPE, typename ArrowType>
   Status ConvertTypedLists(const std::shared_ptr<DataType>& type);
 
+  template <typename ArrowType>
+  Status ConvertDates();
+
   Status ConvertObjectStrings();
   Status ConvertObjectFixedWidthBytes(const std::shared_ptr<DataType>& type);
   Status ConvertBooleans();
-  Status ConvertDates();
   Status ConvertLists(const std::shared_ptr<DataType>& type);
   Status ConvertObjects();
   Status ConvertDecimals();
@@ -462,41 +477,36 @@ inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>*
   return Status::OK();
 }
 
-Status InvalidConversion(PyObject* obj, const std::string& expected_type_name) {
-  OwnedRef type(PyObject_Type(obj));
-  RETURN_IF_PYERROR();
-  DCHECK_NE(type.obj(), nullptr);
-
-  OwnedRef type_name(PyObject_GetAttrString(type.obj(), "__name__"));
-  RETURN_IF_PYERROR();
-  DCHECK_NE(type_name.obj(), nullptr);
-
-  PyObjectStringify bytestring(type_name.obj());
-  RETURN_IF_PYERROR();
-
-  const char* bytes = bytestring.bytes;
-  DCHECK_NE(bytes, nullptr) << "bytes from type(...).__name__ were null";
-
-  Py_ssize_t size = bytestring.size;
+template <typename T>
+struct UnboxDate {};
 
-  std::string cpp_type_name(bytes, size);
+template <>
+struct UnboxDate<Date32Type> {
+  static int64_t Unbox(PyObject* obj) {
+    return PyDate_to_days(reinterpret_cast<PyDateTime_Date*>(obj));
+  }
+};
 
-  std::stringstream ss;
-  ss << "Python object of type " << cpp_type_name << " is not None and is not a "
-     << expected_type_name << " object";
-  return Status::Invalid(ss.str());
-}
+template <>
+struct UnboxDate<Date64Type> {
+  static int64_t Unbox(PyObject* obj) {
+    return PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(obj));
+  }
+};
 
+template <typename ArrowType>
 Status PandasConverter::ConvertDates() {
   PyAcquireGIL lock;
 
+  using BuilderType = typename TypeTraits<ArrowType>::BuilderType;
+
   Ndarray1DIndexer<PyObject*> objects(arr_);
 
   if (mask_ != nullptr) {
     return Status::NotImplemented("mask not supported in object conversions yet");
   }
 
-  Date64Builder date_builder(pool_);
+  BuilderType date_builder(pool_);
   RETURN_NOT_OK(date_builder.Resize(length_));
 
   /// We have to run this in this compilation unit, since we cannot use the
@@ -508,8 +518,7 @@ Status PandasConverter::ConvertDates() {
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
     if (PyDate_CheckExact(obj)) {
-      PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(obj);
-      date_builder.Append(PyDate_to_ms(pydate));
+      date_builder.Append(UnboxDate<ArrowType>::Unbox(obj));
     } else if (PyObject_is_null(obj)) {
       date_builder.AppendNull();
     } else {
@@ -762,8 +771,10 @@ Status PandasConverter::ConvertObjects() {
         return ConvertObjectFixedWidthBytes(type_);
       case Type::BOOL:
         return ConvertBooleans();
+      case Type::DATE32:
+        return ConvertDates<Date32Type>();
       case Type::DATE64:
-        return ConvertDates();
+        return ConvertDates<Date64Type>();
       case Type::LIST: {
         const auto& list_field = static_cast<const ListType&>(*type_);
         return ConvertLists(list_field.value_field()->type);
@@ -787,7 +798,8 @@ Status PandasConverter::ConvertObjects() {
       } else if (PyBool_Check(objects[i])) {
         return ConvertBooleans();
       } else if (PyDate_CheckExact(objects[i])) {
-        return ConvertDates();
+        // We could choose Date32 or Date64
+        return ConvertDates<Date32Type>();
       } else if (PyObject_IsInstance(const_cast<PyObject*>(objects[i]), Decimal.obj())) {
         return ConvertDecimals();
       } else {
@@ -955,34 +967,6 @@ Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
 // ----------------------------------------------------------------------
 // pandas 0.x DataFrame conversion internals
 
-inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
-  if (type == NPY_DATETIME) {
-    PyArray_Descr* descr = PyArray_DESCR(out);
-    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-    if (datatype->type == Type::TIMESTAMP) {
-      auto timestamp_type = static_cast<TimestampType*>(datatype);
-
-      switch (timestamp_type->unit) {
-        case TimestampType::Unit::SECOND:
-          date_dtype->meta.base = NPY_FR_s;
-          break;
-        case TimestampType::Unit::MILLI:
-          date_dtype->meta.base = NPY_FR_ms;
-          break;
-        case TimestampType::Unit::MICRO:
-          date_dtype->meta.base = NPY_FR_us;
-          break;
-        case TimestampType::Unit::NANO:
-          date_dtype->meta.base = NPY_FR_ns;
-          break;
-      }
-    } else {
-      // datatype->type == Type::DATE64
-      date_dtype->meta.base = NPY_FR_D;
-    }
-  }
-}
-
 class PandasBlock {
  public:
   enum type {
@@ -1148,8 +1132,9 @@ static void ConvertBooleanNoNulls(const ChunkedArray& data, uint8_t* out_values)
   }
 }
 
-template <typename ArrayType>
+template <typename Type>
 inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values) {
+  using ArrayType = typename TypeTraits<Type>::ArrayType;
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
     auto arr = static_cast<ArrayType*>(data.chunk(c).get());
@@ -1287,21 +1272,7 @@ inline void ConvertNumericNullableCast(
   }
 }
 
-template <typename T>
-inline void ConvertDates(const ChunkedArray& data, T na_value, T* out_values) {
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
-      *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / 86400000;
-    }
-  }
-}
-
-template <typename InType, int SHIFT>
+template <typename InType, int64_t SHIFT>
 inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
@@ -1339,9 +1310,9 @@ class ObjectBlock : public PandasBlock {
     if (type == Type::BOOL) {
       RETURN_NOT_OK(ConvertBooleanWithNulls(data, out_buffer));
     } else if (type == Type::BINARY) {
-      RETURN_NOT_OK(ConvertBinaryLike<BinaryArray>(data, out_buffer));
+      RETURN_NOT_OK(ConvertBinaryLike<BinaryType>(data, out_buffer));
     } else if (type == Type::STRING) {
-      RETURN_NOT_OK(ConvertBinaryLike<StringArray>(data, out_buffer));
+      RETURN_NOT_OK(ConvertBinaryLike<StringType>(data, out_buffer));
     } else if (type == Type::FIXED_SIZE_BINARY) {
       RETURN_NOT_OK(ConvertFixedSizeBinary(data, out_buffer));
     } else if (type == Type::DECIMAL) {
@@ -1532,7 +1503,11 @@ class DatetimeBlock : public PandasBlock {
 
     const ChunkedArray& data = *col.get()->data();
 
-    if (type == Type::DATE64) {
+    if (type == Type::DATE32) {
+      // Date64Type is millisecond timestamp stored as int64_t
+      // TODO(wesm): Do we want to make sure to zero out the milliseconds?
+      ConvertDatetimeNanos<int32_t, kNanosecondsInDay>(data, out_buffer);
+    } else if (type == Type::DATE64) {
       // Date64Type is millisecond timestamp stored as int64_t
       // TODO(wesm): Do we want to make sure to zero out the milliseconds?
       ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
@@ -1779,6 +1754,9 @@ class DataFrameBlockCreator {
         case Type::FIXED_SIZE_BINARY:
           output_type = PandasBlock::OBJECT;
           break;
+        case Type::DATE32:
+          output_type = PandasBlock::DATETIME;
+          break;
         case Type::DATE64:
           output_type = PandasBlock::DATETIME;
           break;
@@ -1960,6 +1938,34 @@ class DataFrameBlockCreator {
   BlockMap datetimetz_blocks_;
 };
 
+inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
+  if (type == NPY_DATETIME) {
+    PyArray_Descr* descr = PyArray_DESCR(out);
+    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
+    if (datatype->type == Type::TIMESTAMP) {
+      auto timestamp_type = static_cast<TimestampType*>(datatype);
+
+      switch (timestamp_type->unit) {
+        case TimestampType::Unit::SECOND:
+          date_dtype->meta.base = NPY_FR_s;
+          break;
+        case TimestampType::Unit::MILLI:
+          date_dtype->meta.base = NPY_FR_ms;
+          break;
+        case TimestampType::Unit::MICRO:
+          date_dtype->meta.base = NPY_FR_us;
+          break;
+        case TimestampType::Unit::NANO:
+          date_dtype->meta.base = NPY_FR_ns;
+          break;
+      }
+    } else {
+      // datatype->type == Type::DATE64
+      date_dtype->meta.base = NPY_FR_D;
+    }
+  }
+}
+
 class ArrowDeserializer {
  public:
   ArrowDeserializer(const std::shared_ptr<Column>& col, PyObject* py_ref)
@@ -2024,51 +2030,14 @@ class ArrowDeserializer {
   // Allocate new array and deserialize. Can do a zero copy conversion for some
   // types
 
-  Status Convert(PyObject** out) {
-#define CONVERT_CASE(TYPE)                      \
-  case Type::TYPE: {                            \
-    RETURN_NOT_OK(ConvertValues<Type::TYPE>()); \
-  } break;
-
-    switch (col_->type()->type) {
-      CONVERT_CASE(BOOL);
-      CONVERT_CASE(INT8);
-      CONVERT_CASE(INT16);
-      CONVERT_CASE(INT32);
-      CONVERT_CASE(INT64);
-      CONVERT_CASE(UINT8);
-      CONVERT_CASE(UINT16);
-      CONVERT_CASE(UINT32);
-      CONVERT_CASE(UINT64);
-      CONVERT_CASE(FLOAT);
-      CONVERT_CASE(DOUBLE);
-      CONVERT_CASE(BINARY);
-      CONVERT_CASE(STRING);
-      CONVERT_CASE(FIXED_SIZE_BINARY);
-      CONVERT_CASE(DATE64);
-      CONVERT_CASE(TIMESTAMP);
-      CONVERT_CASE(DICTIONARY);
-      CONVERT_CASE(LIST);
-      CONVERT_CASE(DECIMAL);
-      default: {
-        std::stringstream ss;
-        ss << "Arrow type reading not implemented for " << col_->type()->ToString();
-        return Status::NotImplemented(ss.str());
-      }
-    }
-
-#undef CONVERT_CASE
-
-    *out = result_;
-    return Status::OK();
-  }
+  template <typename Type>
+  typename std::enable_if<std::is_base_of<FloatingPoint, Type>::value, Status>::type
+  Visit(const Type& type) {
+    constexpr int TYPE = Type::type_id;
+    using traits = arrow_traits<TYPE>;
 
-  template <int TYPE>
-  inline typename std::enable_if<
-      (TYPE != Type::DATE64) & arrow_traits<TYPE>::is_numeric_nullable, Status>::type
-  ConvertValues() {
-    typedef typename arrow_traits<TYPE>::T T;
-    int npy_type = arrow_traits<TYPE>::npy_type;
+    typedef typename traits::T T;
+    int npy_type = traits::npy_type;
 
     if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
       return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
@@ -2076,31 +2045,56 @@ class ArrowDeserializer {
 
     RETURN_NOT_OK(AllocateOutput(npy_type));
     auto out_values = reinterpret_cast<T*>(PyArray_DATA(arr_));
-    ConvertNumericNullable<T>(data_, arrow_traits<TYPE>::na_value, out_values);
+    ConvertNumericNullable<T>(data_, traits::na_value, out_values);
 
     return Status::OK();
   }
 
-  template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::DATE64, Status>::type ConvertValues() {
-    typedef typename arrow_traits<TYPE>::T T;
+  template <typename Type>
+  typename std::enable_if<std::is_base_of<DateType, Type>::value ||
+                              std::is_base_of<TimestampType, Type>::value,
+      Status>::type
+  Visit(const Type& type) {
+    constexpr int TYPE = Type::type_id;
+    using traits = arrow_traits<TYPE>;
+
+    typedef typename traits::T T;
 
-    RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+    RETURN_NOT_OK(AllocateOutput(traits::npy_type));
     auto out_values = reinterpret_cast<T*>(PyArray_DATA(arr_));
-    ConvertDates<T>(data_, arrow_traits<TYPE>::na_value, out_values);
+
+    constexpr T na_value = traits::na_value;
+    constexpr int64_t kShift = traits::npy_shift;
+
+    for (int c = 0; c < data_.num_chunks(); c++) {
+      const std::shared_ptr<Array> arr = data_.chunk(c);
+      auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
+      auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+
+      for (int64_t i = 0; i < arr->length(); ++i) {
+        *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / kShift;
+      }
+    }
     return Status::OK();
   }
 
+  template <typename Type>
+  typename std::enable_if<std::is_base_of<TimeType, Type>::value, Status>::type Visit(
+      const Type& type) {
+    return Status::NotImplemented("Don't know how to serialize Arrow time type to NumPy");
+  }
+
   // Integer specialization
-  template <int TYPE>
-  inline
-      typename std::enable_if<arrow_traits<TYPE>::is_numeric_not_nullable, Status>::type
-      ConvertValues() {
-    typedef typename arrow_traits<TYPE>::T T;
-    int npy_type = arrow_traits<TYPE>::npy_type;
+  template <typename Type>
+  typename std::enable_if<std::is_base_of<Integer, Type>::value, Status>::type Visit(
+      const Type& type) {
+    constexpr int TYPE = Type::type_id;
+    using traits = arrow_traits<TYPE>;
+
+    typedef typename traits::T T;
 
     if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
-      return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
+      return ConvertValuesZeroCopy<TYPE>(traits::npy_type, data_.chunk(0));
     }
 
     if (data_.null_count() > 0) {
@@ -2108,7 +2102,7 @@ class ArrowDeserializer {
       auto out_values = reinterpret_cast<double*>(PyArray_DATA(arr_));
       ConvertIntegerWithNulls<T>(data_, out_values);
     } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+      RETURN_NOT_OK(AllocateOutput(traits::npy_type));
       auto out_values = reinterpret_cast<T*>(PyArray_DATA(arr_));
       ConvertIntegerNoNullsSameType<T>(data_, out_values);
     }
@@ -2117,15 +2111,13 @@ class ArrowDeserializer {
   }
 
   // Boolean specialization
-  template <int TYPE>
-  inline typename std::enable_if<arrow_traits<TYPE>::is_boolean, Status>::type
-  ConvertValues() {
+  Status Visit(const BooleanType& type) {
     if (data_.null_count() > 0) {
       RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
       auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
       RETURN_NOT_OK(ConvertBooleanWithNulls(data_, out_values));
     } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<Type::BOOL>::npy_type));
       auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(arr_));
       ConvertBooleanNoNulls(data_, out_values);
     }
@@ -2133,43 +2125,32 @@ class ArrowDeserializer {
   }
 
   // UTF8 strings
-  template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::STRING, Status>::type ConvertValues() {
+  template <typename Type>
+  typename std::enable_if<std::is_base_of<BinaryType, Type>::value, Status>::type Visit(
+      const Type& type) {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertBinaryLike<StringArray>(data_, out_values);
-  }
-
-  // Binary strings
-  template <int T2>
-  inline typename std::enable_if<T2 == Type::BINARY, Status>::type ConvertValues() {
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertBinaryLike<BinaryArray>(data_, out_values);
+    return ConvertBinaryLike<Type>(data_, out_values);
   }
 
   // Fixed length binary strings
-  template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::FIXED_SIZE_BINARY, Status>::type
-  ConvertValues() {
+  Status Visit(const FixedSizeBinaryType& type) {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     return ConvertFixedSizeBinary(data_, out_values);
   }
 
-  template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::DECIMAL, Status>::type ConvertValues() {
+  Status Visit(const DecimalType& type) {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     return ConvertDecimals(data_, out_values);
   }
 
+  Status Visit(const ListType& type) {
 #define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \
   case Type::ArrowEnum:                                    \
     return ConvertListsLike<ArrowType>(col_, out_values);
 
-  template <int T2>
-  inline typename std::enable_if<T2 == Type::LIST, Status>::type ConvertValues() {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     auto list_type = std::static_pointer_cast<ListType>(col_->type());
@@ -2193,10 +2174,10 @@ class ArrowDeserializer {
         return Status::NotImplemented(ss.str());
       }
     }
+#undef CONVERTVALUES_LISTSLIKE_CASE
   }
 
-  template <int TYPE>
-  inline typename std::enable_if<TYPE == Type::DICTIONARY, Status>::type ConvertValues() {
+  Status Visit(const DictionaryType& type) {
     std::shared_ptr<PandasBlock> block;
     RETURN_NOT_OK(MakeCategoricalBlock(col_->type(), col_->length(), &block));
     RETURN_NOT_OK(block->Write(col_, 0, 0));
@@ -2216,6 +2197,18 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
+  Status Visit(const NullType& type) { return Status::NotImplemented("null type"); }
+
+  Status Visit(const StructType& type) { return Status::NotImplemented("struct type"); }
+
+  Status Visit(const UnionType& type) { return Status::NotImplemented("union type"); }
+
+  Status Convert(PyObject** out) {
+    RETURN_NOT_OK(VisitTypeInline(*col_->type(), this));
+    *out = result_;
+    return Status::OK();
+  }
+
  private:
   std::shared_ptr<Column> col_;
   const ChunkedArray& data_;
diff --git a/cpp/src/arrow/python/pandas_convert.h b/cpp/src/arrow/python/pandas_convert.h
index 8fd31076a994f..4d32c8b86cf50 100644
--- a/cpp/src/arrow/python/pandas_convert.h
+++ b/cpp/src/arrow/python/pandas_convert.h
@@ -71,9 +71,6 @@ ARROW_EXPORT
 Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
 
-ARROW_EXPORT
-Status InvalidConversion(PyObject* obj, const std::string& expected_type_name);
-
 }  // namespace py
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h
index f78dc360095dc..c464d65a4946c 100644
--- a/cpp/src/arrow/python/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -119,9 +119,6 @@ template <>
 struct arrow_traits<Type::BOOL> {
   static constexpr int npy_type = NPY_BOOL;
   static constexpr bool supports_nulls = false;
-  static constexpr bool is_boolean = true;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
 };
 
 #define INT_DECL(TYPE)                                     \
@@ -130,9 +127,6 @@ struct arrow_traits<Type::BOOL> {
     static constexpr int npy_type = NPY_##TYPE;            \
     static constexpr bool supports_nulls = false;          \
     static constexpr double na_value = NAN;                \
-    static constexpr bool is_boolean = false;              \
-    static constexpr bool is_numeric_not_nullable = true;  \
-    static constexpr bool is_numeric_nullable = false;     \
     typedef typename npy_traits<NPY_##TYPE>::value_type T; \
   };
 
@@ -150,9 +144,6 @@ struct arrow_traits<Type::FLOAT> {
   static constexpr int npy_type = NPY_FLOAT32;
   static constexpr bool supports_nulls = true;
   static constexpr float na_value = NAN;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
   typedef typename npy_traits<NPY_FLOAT32>::value_type T;
 };
 
@@ -161,33 +152,63 @@ struct arrow_traits<Type::DOUBLE> {
   static constexpr int npy_type = NPY_FLOAT64;
   static constexpr bool supports_nulls = true;
   static constexpr double na_value = NAN;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
   typedef typename npy_traits<NPY_FLOAT64>::value_type T;
 };
 
 static constexpr int64_t kPandasTimestampNull = std::numeric_limits<int64_t>::min();
 
+constexpr int64_t kNanosecondsInDay = 86400000000000LL;
+
 template <>
 struct arrow_traits<Type::TIMESTAMP> {
   static constexpr int npy_type = NPY_DATETIME;
+  static constexpr int64_t npy_shift = 1;
+
   static constexpr bool supports_nulls = true;
   static constexpr int64_t na_value = kPandasTimestampNull;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
   typedef typename npy_traits<NPY_DATETIME>::value_type T;
 };
 
+template <>
+struct arrow_traits<Type::DATE32> {
+  // Data stores as FR_D day unit
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr int64_t npy_shift = 1;
+
+  static constexpr bool supports_nulls = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+
+  static constexpr int64_t na_value = kPandasTimestampNull;
+  static inline bool isnull(int64_t v) { return npy_traits<NPY_DATETIME>::isnull(v); }
+};
+
 template <>
 struct arrow_traits<Type::DATE64> {
+  // Data stores as FR_D day unit
   static constexpr int npy_type = NPY_DATETIME;
+
+  // There are 1000 * 60 * 60 * 24 = 86400000ms in a day
+  static constexpr int64_t npy_shift = 86400000;
+
+  static constexpr bool supports_nulls = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+
+  static constexpr int64_t na_value = kPandasTimestampNull;
+  static inline bool isnull(int64_t v) { return npy_traits<NPY_DATETIME>::isnull(v); }
+};
+
+template <>
+struct arrow_traits<Type::TIME32> {
+  static constexpr int npy_type = NPY_OBJECT;
   static constexpr bool supports_nulls = true;
   static constexpr int64_t na_value = kPandasTimestampNull;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
+
+template <>
+struct arrow_traits<Type::TIME64> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
   typedef typename npy_traits<NPY_DATETIME>::value_type T;
 };
 
@@ -195,18 +216,12 @@ template <>
 struct arrow_traits<Type::STRING> {
   static constexpr int npy_type = NPY_OBJECT;
   static constexpr bool supports_nulls = true;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
 };
 
 template <>
 struct arrow_traits<Type::BINARY> {
   static constexpr int npy_type = NPY_OBJECT;
   static constexpr bool supports_nulls = true;
-  static constexpr bool is_boolean = false;
-  static constexpr bool is_numeric_not_nullable = false;
-  static constexpr bool is_numeric_nullable = false;
 };
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index f704a96d91bba..82cf6fc48cad4 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -24,7 +24,7 @@
 namespace arrow {
 namespace py {
 
-inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
+static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   struct tm date = {0};
   date.tm_year = PyDateTime_GET_YEAR(pydate) - 1900;
   date.tm_mon = PyDateTime_GET_MONTH(pydate) - 1;
@@ -36,6 +36,10 @@ inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
 }
 
+static inline int32_t PyDate_to_days(PyDateTime_Date* pydate) {
+  return static_cast<int32_t>(PyDate_to_ms(pydate) / 86400000LL);
+}
+
 }  // namespace py
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 156c3d16d4db0..cdc0238cf4ab8 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -398,62 +398,35 @@ TEST_F(TestTable, AddColumn) {
   ASSERT_TRUE(status.IsInvalid());
 
   // Add column with wrong length
-  auto longer_col = std::make_shared<Column>(
-      schema_->field(0), MakePrimitive<Int32Array>(length + 1));
+  auto longer_col =
+      std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length + 1));
   status = table.AddColumn(0, longer_col, &result);
   ASSERT_TRUE(status.IsInvalid());
 
   // Add column 0 in different places
   ASSERT_OK(table.AddColumn(0, columns_[0], &result));
-  auto ex_schema = std::shared_ptr<Schema>(new Schema({
-      schema_->field(0),
-      schema_->field(0),
-      schema_->field(1),
-      schema_->field(2)}));
+  auto ex_schema = std::shared_ptr<Schema>(new Schema(
+      {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)}));
   std::vector<std::shared_ptr<Column>> ex_columns = {
-      table.column(0),
-      table.column(0),
-      table.column(1),
-      table.column(2)};
+      table.column(0), table.column(0), table.column(1), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.AddColumn(1, columns_[0], &result));
-  ex_schema = std::shared_ptr<Schema>(new Schema({
-      schema_->field(0),
-      schema_->field(0),
-      schema_->field(1),
-      schema_->field(2)}));
-  ex_columns = {
-      table.column(0),
-      table.column(0),
-      table.column(1),
-      table.column(2)};
+  ex_schema = std::shared_ptr<Schema>(new Schema(
+      {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)}));
+  ex_columns = {table.column(0), table.column(0), table.column(1), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.AddColumn(2, columns_[0], &result));
-  ex_schema = std::shared_ptr<Schema>(new Schema({
-      schema_->field(0),
-      schema_->field(1),
-      schema_->field(0),
-      schema_->field(2)}));
-  ex_columns = {
-      table.column(0),
-      table.column(1),
-      table.column(0),
-      table.column(2)};
+  ex_schema = std::shared_ptr<Schema>(new Schema(
+      {schema_->field(0), schema_->field(1), schema_->field(0), schema_->field(2)}));
+  ex_columns = {table.column(0), table.column(1), table.column(0), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.AddColumn(3, columns_[0], &result));
-  ex_schema = std::shared_ptr<Schema>(new Schema({
-      schema_->field(0),
-      schema_->field(1),
-      schema_->field(2),
-      schema_->field(0)}));
-  ex_columns = {
-      table.column(0),
-      table.column(1),
-      table.column(2),
-      table.column(0)};
+  ex_schema = std::shared_ptr<Schema>(new Schema(
+      {schema_->field(0), schema_->field(1), schema_->field(2), schema_->field(0)}));
+  ex_columns = {table.column(0), table.column(1), table.column(2), table.column(0)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 }
 
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 9b39f770a17b7..4c5257b92c033 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -321,11 +321,9 @@ Status Table::RemoveColumn(int i, std::shared_ptr<Table>* out) const {
   return Status::OK();
 }
 
-Status Table::AddColumn(int i, const std::shared_ptr<Column>& col,
-    std::shared_ptr<Table>* out) const {
-  if (i < 0 || i > num_columns() + 1) {
-    return Status::Invalid("Invalid column index.");
-  }
+Status Table::AddColumn(
+    int i, const std::shared_ptr<Column>& col, std::shared_ptr<Table>* out) const {
+  if (i < 0 || i > num_columns() + 1) { return Status::Invalid("Invalid column index."); }
   if (col == nullptr) {
     std::stringstream ss;
     ss << "Column " << i << " was null";
@@ -334,7 +332,7 @@ Status Table::AddColumn(int i, const std::shared_ptr<Column>& col,
   if (col->length() != num_rows_) {
     std::stringstream ss;
     ss << "Added column's length must match table's length. Expected length " << num_rows_
-        << " but got length " << col->length();
+       << " but got length " << col->length();
     return Status::Invalid(ss.str());
   }
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index dcea53d8fb1dd..b15d31b23a872 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -182,8 +182,8 @@ class ARROW_EXPORT Table {
   Status RemoveColumn(int i, std::shared_ptr<Table>* out) const;
 
   /// Add column to the table, producing a new Table
-  Status AddColumn(int i, const std::shared_ptr<Column>& column,
-      std::shared_ptr<Table>* out) const;
+  Status AddColumn(
+      int i, const std::shared_ptr<Column>& column, std::shared_ptr<Table>* out) const;
 
   // @returns: the number of columns in the table
   int num_columns() const { return static_cast<int>(columns_.size()); }
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index df4590f18d733..93cab14d797c3 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -258,8 +258,8 @@ std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
   }
 }
 
-Status Schema::AddField(int i, const std::shared_ptr<Field>& field,
-    std::shared_ptr<Schema>* out) const {
+Status Schema::AddField(
+    int i, const std::shared_ptr<Field>& field, std::shared_ptr<Schema>* out) const {
   DCHECK_GE(i, 0);
   DCHECK_LE(i, this->num_fields());
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 3a35f56381197..730cbed8f4d67 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -633,8 +633,8 @@ class ARROW_EXPORT Schema {
   // Render a string representation of the schema suitable for debugging
   std::string ToString() const;
 
-  Status AddField(int i, const std::shared_ptr<Field>& field,
-      std::shared_ptr<Schema>* out) const;
+  Status AddField(
+      int i, const std::shared_ptr<Field>& field, std::shared_ptr<Schema>* out) const;
   Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
 
   int num_fields() const { return static_cast<int>(fields_.size()); }
diff --git a/cpp/src/arrow/util/stl.h b/cpp/src/arrow/util/stl.h
index bd250539a8c8a..bfce111ff8a22 100644
--- a/cpp/src/arrow/util/stl.h
+++ b/cpp/src/arrow/util/stl.h
@@ -40,8 +40,8 @@ inline std::vector<T> DeleteVectorElement(const std::vector<T>& values, size_t i
 }
 
 template <typename T>
-inline std::vector<T> AddVectorElement(const std::vector<T>& values, size_t index,
-    const T& new_element) {
+inline std::vector<T> AddVectorElement(
+    const std::vector<T>& values, size_t index, const T& new_element) {
   DCHECK_LE(index, values.size());
   std::vector<T> out;
   out.reserve(values.size() + 1);
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index f3d9321326964..196deedefa959 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -134,7 +134,11 @@ cdef class UInt64Value(ArrayValue):
 cdef class Date32Value(ArrayValue):
 
     def as_py(self):
-        raise NotImplementedError
+        cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+
+        # Shift to seconds since epoch
+        return datetime.datetime.utcfromtimestamp(
+            int(ap.Value(self.index)) * 86400).date()
 
 
 cdef class Date64Value(ArrayValue):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 0504e1ddb4f53..d1bea0b3e32f0 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -28,7 +28,7 @@
 import pandas.util.testing as tm
 
 from pyarrow.compat import u
-import pyarrow as A
+import pyarrow as pa
 
 from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
@@ -67,7 +67,7 @@ def tearDown(self):
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
                                 timestamps_to_ms=False, expected_schema=None,
                                 check_dtype=True, schema=None):
-        table = A.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms,
+        table = pa.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms,
                                     schema=schema)
         result = table.to_pandas(nthreads=nthreads)
         if expected_schema:
@@ -78,7 +78,7 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
 
     def _check_array_roundtrip(self, values, expected=None, mask=None,
                                timestamps_to_ms=False, type=None):
-        arr = A.Array.from_numpy(values, timestamps_to_ms=timestamps_to_ms,
+        arr = pa.Array.from_numpy(values, timestamps_to_ms=timestamps_to_ms,
                                  mask=mask, type=type)
         result = arr.to_pandas()
 
@@ -99,23 +99,23 @@ def _check_array_roundtrip(self, values, expected=None, mask=None,
     def test_float_no_nulls(self):
         data = {}
         fields = []
-        dtypes = [('f4', A.float32()), ('f8', A.float64())]
+        dtypes = [('f4', pa.float32()), ('f8', pa.float64())]
         num_values = 100
 
         for numpy_dtype, arrow_dtype in dtypes:
             values = np.random.randn(num_values)
             data[numpy_dtype] = values.astype(numpy_dtype)
-            fields.append(A.Field.from_py(numpy_dtype, arrow_dtype))
+            fields.append(pa.Field.from_py(numpy_dtype, arrow_dtype))
 
         df = pd.DataFrame(data)
-        schema = A.Schema.from_fields(fields)
+        schema = pa.Schema.from_fields(fields)
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_float_nulls(self):
         num_values = 100
 
         null_mask = np.random.randint(0, 10, size=num_values) < 3
-        dtypes = [('f4', A.float32()), ('f8', A.float64())]
+        dtypes = [('f4', pa.float32()), ('f8', pa.float64())]
         names = ['f4', 'f8']
         expected_cols = []
 
@@ -124,9 +124,9 @@ def test_float_nulls(self):
         for name, arrow_dtype in dtypes:
             values = np.random.randn(num_values).astype(name)
 
-            arr = A.Array.from_numpy(values, null_mask)
+            arr = pa.Array.from_numpy(values, null_mask)
             arrays.append(arr)
-            fields.append(A.Field.from_py(name, arrow_dtype))
+            fields.append(pa.Field.from_py(name, arrow_dtype))
             values[null_mask] = np.nan
 
             expected_cols.append(values)
@@ -134,8 +134,8 @@ def test_float_nulls(self):
         ex_frame = pd.DataFrame(dict(zip(names, expected_cols)),
                                 columns=names)
 
-        table = A.Table.from_arrays(arrays, names)
-        assert table.schema.equals(A.Schema.from_fields(fields))
+        table = pa.Table.from_arrays(arrays, names)
+        assert table.schema.equals(pa.Schema.from_fields(fields))
         result = table.to_pandas()
         tm.assert_frame_equal(result, ex_frame)
 
@@ -144,11 +144,11 @@ def test_integer_no_nulls(self):
         fields = []
 
         numpy_dtypes = [
-            ('i1', A.int8()), ('i2', A.int16()),
-            ('i4', A.int32()), ('i8', A.int64()),
-            ('u1', A.uint8()), ('u2', A.uint16()),
-            ('u4', A.uint32()), ('u8', A.uint64()),
-            ('longlong', A.int64()), ('ulonglong', A.uint64())
+            ('i1', pa.int8()), ('i2', pa.int16()),
+            ('i4', pa.int32()), ('i8', pa.int64()),
+            ('u1', pa.uint8()), ('u2', pa.uint16()),
+            ('u4', pa.uint32()), ('u8', pa.uint64()),
+            ('longlong', pa.int64()), ('ulonglong', pa.uint64())
         ]
         num_values = 100
 
@@ -158,10 +158,10 @@ def test_integer_no_nulls(self):
                                        min(info.max, np.iinfo('i8').max),
                                        size=num_values)
             data[dtype] = values.astype(dtype)
-            fields.append(A.Field.from_py(dtype, arrow_dtype))
+            fields.append(pa.Field.from_py(dtype, arrow_dtype))
 
         df = pd.DataFrame(data)
-        schema = A.Schema.from_fields(fields)
+        schema = pa.Schema.from_fields(fields)
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_integer_with_nulls(self):
@@ -177,7 +177,7 @@ def test_integer_with_nulls(self):
         for name in int_dtypes:
             values = np.random.randint(0, 100, size=num_values)
 
-            arr = A.Array.from_numpy(values, null_mask)
+            arr = pa.Array.from_numpy(values, null_mask)
             arrays.append(arr)
 
             expected = values.astype('f8')
@@ -188,7 +188,7 @@ def test_integer_with_nulls(self):
         ex_frame = pd.DataFrame(dict(zip(int_dtypes, expected_cols)),
                                 columns=int_dtypes)
 
-        table = A.Table.from_arrays(arrays, int_dtypes)
+        table = pa.Table.from_arrays(arrays, int_dtypes)
         result = table.to_pandas()
 
         tm.assert_frame_equal(result, ex_frame)
@@ -199,8 +199,8 @@ def test_boolean_no_nulls(self):
         np.random.seed(0)
 
         df = pd.DataFrame({'bools': np.random.randn(num_values) > 0})
-        field = A.Field.from_py('bools', A.bool_())
-        schema = A.Schema.from_fields([field])
+        field = pa.Field.from_py('bools', pa.bool_())
+        schema = pa.Schema.from_fields([field])
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_boolean_nulls(self):
@@ -211,16 +211,16 @@ def test_boolean_nulls(self):
         mask = np.random.randint(0, 10, size=num_values) < 3
         values = np.random.randint(0, 10, size=num_values) < 5
 
-        arr = A.Array.from_numpy(values, mask)
+        arr = pa.Array.from_numpy(values, mask)
 
         expected = values.astype(object)
         expected[mask] = None
 
-        field = A.Field.from_py('bools', A.bool_())
-        schema = A.Schema.from_fields([field])
+        field = pa.Field.from_py('bools', pa.bool_())
+        schema = pa.Schema.from_fields([field])
         ex_frame = pd.DataFrame({'bools': expected})
 
-        table = A.Table.from_arrays([arr], ['bools'])
+        table = pa.Table.from_arrays([arr], ['bools'])
         assert table.schema.equals(schema)
         result = table.to_pandas()
 
@@ -229,16 +229,16 @@ def test_boolean_nulls(self):
     def test_boolean_object_nulls(self):
         arr = np.array([False, None, True] * 100, dtype=object)
         df = pd.DataFrame({'bools': arr})
-        field = A.Field.from_py('bools', A.bool_())
-        schema = A.Schema.from_fields([field])
+        field = pa.Field.from_py('bools', pa.bool_())
+        schema = pa.Schema.from_fields([field])
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_unicode(self):
         repeats = 1000
         values = [u'foo', None, u'bar', u'mañana', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
-        field = A.Field.from_py('strings', A.string())
-        schema = A.Schema.from_fields([field])
+        field = pa.Field.from_py('strings', pa.string())
+        schema = pa.Schema.from_fields([field])
 
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
@@ -246,8 +246,8 @@ def test_bytes_to_binary(self):
         values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan]
         df = pd.DataFrame({'strings': values})
 
-        table = A.Table.from_pandas(df)
-        assert table[0].type == A.binary()
+        table = pa.Table.from_pandas(df)
+        assert table[0].type == pa.binary()
 
         values2 = [b'qux', b'foo', None, b'bar', b'qux', np.nan]
         expected = pd.DataFrame({'strings': values2})
@@ -256,8 +256,8 @@ def test_bytes_to_binary(self):
     def test_fixed_size_bytes(self):
         values = [b'foo', None, b'bar', None, None, b'hey']
         df = pd.DataFrame({'strings': values})
-        schema = A.Schema.from_fields([A.field('strings', A.binary(3))])
-        table = A.Table.from_pandas(df, schema=schema)
+        schema = pa.Schema.from_fields([pa.field('strings', pa.binary(3))])
+        table = pa.Table.from_pandas(df, schema=schema)
         assert table.schema[0].type == schema[0].type
         assert table.schema[0].name == schema[0].name
         result = table.to_pandas()
@@ -266,9 +266,9 @@ def test_fixed_size_bytes(self):
     def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
         values = [b'foo', None, b'ba', None, None, b'hey']
         df = pd.DataFrame({'strings': values})
-        schema = A.Schema.from_fields([A.field('strings', A.binary(3))])
-        with self.assertRaises(A.ArrowInvalid):
-            A.Table.from_pandas(df, schema=schema)
+        schema = pa.Schema.from_fields([pa.field('strings', pa.binary(3))])
+        with self.assertRaises(pa.ArrowInvalid):
+            pa.Table.from_pandas(df, schema=schema)
 
     def test_timestamps_notimezone_no_nulls(self):
         df = pd.DataFrame({
@@ -278,8 +278,8 @@ def test_timestamps_notimezone_no_nulls(self):
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
             })
-        field = A.Field.from_py('datetime64', A.timestamp('ms'))
-        schema = A.Schema.from_fields([field])
+        field = pa.Field.from_py('datetime64', pa.timestamp('ms'))
+        schema = pa.Schema.from_fields([field])
         self._check_pandas_roundtrip(df, timestamps_to_ms=True,
                                      expected_schema=schema)
 
@@ -290,8 +290,8 @@ def test_timestamps_notimezone_no_nulls(self):
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
             })
-        field = A.Field.from_py('datetime64', A.timestamp('ns'))
-        schema = A.Schema.from_fields([field])
+        field = pa.Field.from_py('datetime64', pa.timestamp('ns'))
+        schema = pa.Schema.from_fields([field])
         self._check_pandas_roundtrip(df, timestamps_to_ms=False,
                                      expected_schema=schema)
 
@@ -303,8 +303,8 @@ def test_timestamps_notimezone_nulls(self):
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
             })
-        field = A.Field.from_py('datetime64', A.timestamp('ms'))
-        schema = A.Schema.from_fields([field])
+        field = pa.Field.from_py('datetime64', pa.timestamp('ms'))
+        schema = pa.Schema.from_fields([field])
         self._check_pandas_roundtrip(df, timestamps_to_ms=True,
                                      expected_schema=schema)
 
@@ -315,8 +315,8 @@ def test_timestamps_notimezone_nulls(self):
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
             })
-        field = A.Field.from_py('datetime64', A.timestamp('ns'))
-        schema = A.Schema.from_fields([field])
+        field = pa.Field.from_py('datetime64', pa.timestamp('ns'))
+        schema = pa.Schema.from_fields([field])
         self._check_pandas_roundtrip(df, timestamps_to_ms=False,
                                      expected_schema=schema)
 
@@ -345,25 +345,77 @@ def test_timestamps_with_timezone(self):
                             .to_frame())
         self._check_pandas_roundtrip(df, timestamps_to_ms=False)
 
-    def test_date(self):
+    def test_date_infer(self):
         df = pd.DataFrame({
             'date': [datetime.date(2000, 1, 1),
                      None,
                      datetime.date(1970, 1, 1),
                      datetime.date(2040, 2, 26)]})
-        table = A.Table.from_pandas(df)
-        field = A.Field.from_py('date', A.date64())
-        schema = A.Schema.from_fields([field])
+        table = pa.Table.from_pandas(df)
+        field = pa.Field.from_py('date', pa.date32())
+        schema = pa.Schema.from_fields([field])
         assert table.schema.equals(schema)
         result = table.to_pandas()
         expected = df.copy()
         expected['date'] = pd.to_datetime(df['date'])
         tm.assert_frame_equal(result, expected)
 
+    def test_date_objects_typed(self):
+        arr = np.array([
+            datetime.date(2017, 4, 3),
+            None,
+            datetime.date(2017, 4, 4),
+            datetime.date(2017, 4, 5)], dtype=object)
+
+        arr_i4 = np.array([17259, -1, 17260, 17261], dtype='int32')
+        arr_i8 = arr_i4.astype('int64') * 86400000
+        mask = np.array([False, True, False, False])
+
+        t32 = pa.date32()
+        t64 = pa.date64()
+
+        a32 = pa.Array.from_numpy(arr, type=t32)
+        a64 = pa.Array.from_numpy(arr, type=t64)
+
+        a32_expected = pa.Array.from_numpy(arr_i4, mask=mask, type=t32)
+        a64_expected = pa.Array.from_numpy(arr_i8, mask=mask, type=t64)
+
+        assert a32.equals(a32_expected)
+        assert a64.equals(a64_expected)
+
+        # Test converting back to pandas
+        colnames = ['date32', 'date64']
+        table = pa.Table.from_arrays([a32, a64], colnames)
+        table_pandas = table.to_pandas()
+
+        ex_values = (np.array(['2017-04-03', '2017-04-04', '2017-04-04',
+                              '2017-04-05'],
+                              dtype='datetime64[D]')
+                     .astype('datetime64[ns]'))
+        ex_values[1] = pd.NaT.value
+        expected_pandas = pd.DataFrame({'date32': ex_values,
+                                        'date64': ex_values},
+                                       columns=colnames)
+        tm.assert_frame_equal(table_pandas, expected_pandas)
+
+    def test_dates_from_integers(self):
+        t1 = pa.date32()
+        t2 = pa.date64()
+
+        arr = np.array([17259, 17260, 17261], dtype='int32')
+        arr2 = arr.astype('int64') * 86400000
+
+        a1 = pa.Array.from_numpy(arr, type=t1)
+        a2 = pa.Array.from_numpy(arr2, type=t2)
+
+        expected = datetime.date(2017, 4, 3)
+        assert a1[0].as_py() == expected
+        assert a2[0].as_py() == expected
+
     def test_column_of_arrays(self):
         df, schema = dataframe_with_arrays()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
-        table = A.Table.from_pandas(df, schema=schema)
+        table = pa.Table.from_pandas(df, schema=schema)
         assert table.schema.equals(schema)
 
         for column in df.columns:
@@ -373,7 +425,7 @@ def test_column_of_arrays(self):
     def test_column_of_lists(self):
         df, schema = dataframe_with_lists()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
-        table = A.Table.from_pandas(df, schema=schema)
+        table = pa.Table.from_pandas(df, schema=schema)
         assert table.schema.equals(schema)
 
         for column in df.columns:
@@ -410,8 +462,8 @@ def test_category(self):
 
     def test_mixed_types_fails(self):
         data = pd.DataFrame({'a': ['a', 1, 2.0]})
-        with self.assertRaises(A.ArrowException):
-            A.Table.from_pandas(data)
+        with self.assertRaises(pa.ArrowException):
+            pa.Table.from_pandas(data)
 
     def test_strided_data_import(self):
         cases = []
@@ -460,9 +512,9 @@ def test_decimal_32_from_pandas(self):
                 decimal.Decimal('1234.439'),
             ]
         })
-        converted = A.Table.from_pandas(expected)
-        field = A.Field.from_py('decimals', A.decimal(7, 3))
-        schema = A.Schema.from_fields([field])
+        converted = pa.Table.from_pandas(expected)
+        field = pa.Field.from_py('decimals', pa.decimal(7, 3))
+        schema = pa.Schema.from_fields([field])
         assert converted.schema.equals(schema)
 
     def test_decimal_32_to_pandas(self):
@@ -472,7 +524,7 @@ def test_decimal_32_to_pandas(self):
                 decimal.Decimal('1234.439'),
             ]
         })
-        converted = A.Table.from_pandas(expected)
+        converted = pa.Table.from_pandas(expected)
         df = converted.to_pandas()
         tm.assert_frame_equal(df, expected)
 
@@ -483,9 +535,9 @@ def test_decimal_64_from_pandas(self):
                 decimal.Decimal('129534.123731'),
             ]
         })
-        converted = A.Table.from_pandas(expected)
-        field = A.Field.from_py('decimals', A.decimal(12, 6))
-        schema = A.Schema.from_fields([field])
+        converted = pa.Table.from_pandas(expected)
+        field = pa.Field.from_py('decimals', pa.decimal(12, 6))
+        schema = pa.Schema.from_fields([field])
         assert converted.schema.equals(schema)
 
     def test_decimal_64_to_pandas(self):
@@ -495,7 +547,7 @@ def test_decimal_64_to_pandas(self):
                 decimal.Decimal('129534.123731'),
             ]
         })
-        converted = A.Table.from_pandas(expected)
+        converted = pa.Table.from_pandas(expected)
         df = converted.to_pandas()
         tm.assert_frame_equal(df, expected)
 
@@ -506,9 +558,9 @@ def test_decimal_128_from_pandas(self):
                 -decimal.Decimal('314292388910493.12343437128'),
             ]
         })
-        converted = A.Table.from_pandas(expected)
-        field = A.Field.from_py('decimals', A.decimal(26, 11))
-        schema = A.Schema.from_fields([field])
+        converted = pa.Table.from_pandas(expected)
+        field = pa.Field.from_py('decimals', pa.decimal(26, 11))
+        schema = pa.Schema.from_fields([field])
         assert converted.schema.equals(schema)
 
     def test_decimal_128_to_pandas(self):
@@ -518,6 +570,6 @@ def test_decimal_128_to_pandas(self):
                 -decimal.Decimal('314292388910493.12343437128'),
             ]
         })
-        converted = A.Table.from_pandas(expected)
+        converted = pa.Table.from_pandas(expected)
         df = converted.to_pandas()
         tm.assert_frame_equal(df, expected)

From 72e1e08754003a56b413f49a107d55d61519f7ef Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 9 Apr 2017 21:11:33 -0400
Subject: [PATCH 0488/1644] ARROW-800: [C++] Boost headers being transitively
 included in pyarrow

thanks to @wesm for suggesting the idea of returning `std::string` and doing the dispatching in c++.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #518 from cpcloud/ARROW-800 and squashes the following commits:

a983841 [Phillip Cloud] Formatting
ba46502 [Phillip Cloud] decimal namespace and change to FormatValue
f326f3a [Phillip Cloud] Const things
9001432 [Phillip Cloud] Remove ARROW_EXPORT of method inside ARROW_EXPORTed class
0c300ec [Phillip Cloud] ARROW-800: [C++] Boost headers being transitively included in pyarrow
---
 cpp/src/arrow/array-decimal-test.cc     |  2 ++
 cpp/src/arrow/array.cc                  | 44 +++++++++++++++----------
 cpp/src/arrow/array.h                   |  3 +-
 cpp/src/arrow/builder.cc                | 12 +++----
 cpp/src/arrow/builder.h                 |  6 +++-
 cpp/src/arrow/python/builtin_convert.cc |  2 +-
 cpp/src/arrow/python/helpers.cc         | 12 ++++---
 cpp/src/arrow/python/helpers.h          |  7 +++-
 cpp/src/arrow/python/pandas_convert.cc  | 10 +++---
 cpp/src/arrow/python/python-test.cc     |  4 +--
 cpp/src/arrow/type_fwd.h                |  2 +-
 cpp/src/arrow/util/CMakeLists.txt       |  1 -
 cpp/src/arrow/util/decimal-test.cc      |  2 ++
 cpp/src/arrow/util/decimal.cc           |  2 ++
 cpp/src/arrow/util/decimal.h            |  2 ++
 python/pyarrow/includes/common.pxd      |  5 ---
 python/pyarrow/includes/libarrow.pxd    |  7 +---
 python/pyarrow/scalar.pyx               |  9 +----
 18 files changed, 71 insertions(+), 61 deletions(-)

diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 4c01f928a6f26..8353acc454f40 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -23,6 +23,7 @@
 #include "arrow/util/decimal.h"
 
 namespace arrow {
+namespace decimal {
 
 TEST(TypesTest, TestDecimal32Type) {
   DecimalType t1(8, 4);
@@ -221,4 +222,5 @@ INSTANTIATE_TEST_CASE_P(Decimal128BuilderTest, Decimal128BuilderTest,
     ::testing::Range(
         DecimalPrecision<int128_t>::minimum, DecimalPrecision<int128_t>::maximum));
 
+}  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 4e73e7176fa9c..c4a78f3b2e400 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -310,25 +310,35 @@ bool DecimalArray::IsNegative(int64_t i) const {
   return sign_bitmap_data_ != nullptr ? BitUtil::GetBit(sign_bitmap_data_, i) : false;
 }
 
-template <typename T>
-ARROW_EXPORT Decimal<T> DecimalArray::Value(int64_t i) const {
-  Decimal<T> result;
-  FromBytes(GetValue(i), &result);
-  return result;
-}
-
-template ARROW_EXPORT Decimal32 DecimalArray::Value(int64_t i) const;
-template ARROW_EXPORT Decimal64 DecimalArray::Value(int64_t i) const;
-
-template <>
-ARROW_EXPORT Decimal128 DecimalArray::Value(int64_t i) const {
-  Decimal128 result;
-  FromBytes(GetValue(i), IsNegative(i), &result);
-  return result;
+std::string DecimalArray::FormatValue(int64_t i) const {
+  const auto type_ = std::dynamic_pointer_cast<DecimalType>(type());
+  const int precision = type_->precision;
+  const int scale = type_->scale;
+  const int byte_width = byte_width_;
+  const uint8_t* bytes = GetValue(i);
+  switch (byte_width) {
+    case 4: {
+      decimal::Decimal32 value;
+      decimal::FromBytes(bytes, &value);
+      return decimal::ToString(value, precision, scale);
+    }
+    case 8: {
+      decimal::Decimal64 value;
+      decimal::FromBytes(bytes, &value);
+      return decimal::ToString(value, precision, scale);
+    }
+    case 16: {
+      decimal::Decimal128 value;
+      decimal::FromBytes(bytes, IsNegative(i), &value);
+      return decimal::ToString(value, precision, scale);
+    }
+    default: {
+      DCHECK(false) << "Invalid byte width: " << byte_width;
+      return "";
+    }
+  }
 }
 
-template ARROW_EXPORT Decimal128 DecimalArray::Value(int64_t i) const;
-
 std::shared_ptr<Array> DecimalArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<DecimalArray>(
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index a4117facdefd0..4f8b22e31b4eb 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -384,8 +384,7 @@ class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray {
 
   bool IsNegative(int64_t i) const;
 
-  template <typename T>
-  ARROW_EXPORT Decimal<T> Value(int64_t i) const;
+  std::string FormatValue(int64_t i) const;
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index a3677eff68669..4281a61474cce 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -332,7 +332,7 @@ DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>
       sign_bitmap_data_(nullptr) {}
 
 template <typename T>
-ARROW_EXPORT Status DecimalBuilder::Append(const Decimal<T>& val) {
+ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal<T>& val) {
   DCHECK_EQ(sign_bitmap_, nullptr) << "sign_bitmap_ is not null";
   DCHECK_EQ(sign_bitmap_data_, nullptr) << "sign_bitmap_data_ is not null";
 
@@ -340,11 +340,11 @@ ARROW_EXPORT Status DecimalBuilder::Append(const Decimal<T>& val) {
   return FixedSizeBinaryBuilder::Append(reinterpret_cast<const uint8_t*>(&val.value));
 }
 
-template ARROW_EXPORT Status DecimalBuilder::Append(const Decimal32& val);
-template ARROW_EXPORT Status DecimalBuilder::Append(const Decimal64& val);
+template ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal32& val);
+template ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal64& val);
 
 template <>
-ARROW_EXPORT Status DecimalBuilder::Append(const Decimal128& value) {
+ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal128& value) {
   DCHECK_NE(sign_bitmap_, nullptr) << "sign_bitmap_ is null";
   DCHECK_NE(sign_bitmap_data_, nullptr) << "sign_bitmap_data_ is null";
 
@@ -352,7 +352,7 @@ ARROW_EXPORT Status DecimalBuilder::Append(const Decimal128& value) {
   uint8_t stack_bytes[16] = {0};
   uint8_t* bytes = stack_bytes;
   bool is_negative;
-  ToBytes(value, &bytes, &is_negative);
+  decimal::ToBytes(value, &bytes, &is_negative);
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Append(bytes));
 
   // TODO(phillipc): calculate the proper storage size here (do we have a function to do
@@ -363,7 +363,7 @@ ARROW_EXPORT Status DecimalBuilder::Append(const Decimal128& value) {
   return Status::OK();
 }
 
-template ARROW_EXPORT Status DecimalBuilder::Append(const Decimal128& val);
+template ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal128& val);
 
 Status DecimalBuilder::Init(int64_t capacity) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Init(capacity));
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index d42ab5b01d1ba..68769165b02c0 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -37,9 +37,13 @@ namespace arrow {
 
 class Array;
 
+namespace decimal {
+
 template <typename T>
 struct Decimal;
 
+}  // namespace decimal
+
 static constexpr int64_t kMinBuilderCapacity = 1 << 5;
 
 /// Base class for all data array builders.
@@ -421,7 +425,7 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
   explicit DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 
   template <typename T>
-  ARROW_EXPORT Status Append(const Decimal<T>& val);
+  ARROW_EXPORT Status Append(const decimal::Decimal<T>& val);
 
   Status Init(int64_t capacity) override;
   Status Resize(int64_t capacity) override;
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index a064a3daf970d..1ae13f3db061c 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -523,7 +523,7 @@ class ListConverter : public TypedConverter<ListBuilder> {
 
 #define DECIMAL_CONVERT_CASE(bit_width, item, builder)        \
   case bit_width: {                                           \
-    arrow::Decimal##bit_width out;                            \
+    arrow::decimal::Decimal##bit_width out;                   \
     RETURN_NOT_OK(PythonDecimalToArrowDecimal((item), &out)); \
     RETURN_NOT_OK((builder)->Append(out));                    \
     break;                                                    \
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index ffba7bbc21c14..3d3d07a515833 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -74,7 +74,8 @@ Status ImportFromModule(const OwnedRef& module, const std::string& name, OwnedRe
 }
 
 template <typename T>
-Status PythonDecimalToArrowDecimal(PyObject* python_decimal, Decimal<T>* arrow_decimal) {
+Status PythonDecimalToArrowDecimal(
+    PyObject* python_decimal, decimal::Decimal<T>* arrow_decimal) {
   // Call Python's str(decimal_object)
   OwnedRef str_obj(PyObject_Str(python_decimal));
   RETURN_IF_PYERROR();
@@ -92,11 +93,11 @@ Status PythonDecimalToArrowDecimal(PyObject* python_decimal, Decimal<T>* arrow_d
 }
 
 template Status PythonDecimalToArrowDecimal(
-    PyObject* python_decimal, Decimal32* arrow_decimal);
+    PyObject* python_decimal, decimal::Decimal32* arrow_decimal);
 template Status PythonDecimalToArrowDecimal(
-    PyObject* python_decimal, Decimal64* arrow_decimal);
+    PyObject* python_decimal, decimal::Decimal64* arrow_decimal);
 template Status PythonDecimalToArrowDecimal(
-    PyObject* python_decimal, Decimal128* arrow_decimal);
+    PyObject* python_decimal, decimal::Decimal128* arrow_decimal);
 
 Status InferDecimalPrecisionAndScale(
     PyObject* python_decimal, int* precision, int* scale) {
@@ -111,7 +112,8 @@ Status InferDecimalPrecisionAndScale(
   auto size = str.size;
 
   std::string c_string(bytes, size);
-  return FromString(c_string, static_cast<Decimal32*>(nullptr), precision, scale);
+  return FromString(
+      c_string, static_cast<decimal::Decimal32*>(nullptr), precision, scale);
 }
 
 Status DecimalFromString(
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index a19b25f7db805..77fde263de7e0 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -29,9 +29,13 @@
 
 namespace arrow {
 
+namespace decimal {
+
 template <typename T>
 struct Decimal;
 
+}  // namespace decimal
+
 namespace py {
 
 class OwnedRef;
@@ -43,7 +47,8 @@ Status ImportFromModule(
     const OwnedRef& module, const std::string& module_name, OwnedRef* ref);
 
 template <typename T>
-Status PythonDecimalToArrowDecimal(PyObject* python_decimal, Decimal<T>* arrow_decimal);
+Status PythonDecimalToArrowDecimal(
+    PyObject* python_decimal, decimal::Decimal<T>* arrow_decimal);
 
 Status InferDecimalPrecisionAndScale(
     PyObject* python_decimal, int* precision = nullptr, int* scale = nullptr);
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 5bb8e45e191a9..1a250e83c5093 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -530,7 +530,7 @@ Status PandasConverter::ConvertDates() {
 
 #define CONVERT_DECIMAL_CASE(bit_width, builder, object)      \
   case bit_width: {                                           \
-    Decimal##bit_width d;                                     \
+    decimal::Decimal##bit_width d;                            \
     RETURN_NOT_OK(PythonDecimalToArrowDecimal((object), &d)); \
     RETURN_NOT_OK((builder).Append(d));                       \
     break;                                                    \
@@ -620,7 +620,7 @@ Status PandasConverter::ConvertObjectFixedWidthBytes(
 
 template <typename T>
 Status validate_precision(int precision) {
-  constexpr static const int maximum_precision = DecimalPrecision<T>::maximum;
+  constexpr static const int maximum_precision = decimal::DecimalPrecision<T>::maximum;
   if (!(precision > 0 && precision <= maximum_precision)) {
     std::stringstream ss;
     ss << "Invalid precision: " << precision << ". Minimum is 1, maximum is "
@@ -636,7 +636,7 @@ Status RawDecimalToString(
   DCHECK_NE(bytes, nullptr);
   DCHECK_NE(result, nullptr);
   RETURN_NOT_OK(validate_precision<T>(precision));
-  Decimal<T> decimal;
+  decimal::Decimal<T> decimal;
   FromBytes(bytes, &decimal);
   *result = ToString(decimal, precision, scale);
   return Status::OK();
@@ -651,8 +651,8 @@ Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
     bool is_negative, std::string* result) {
   DCHECK_NE(bytes, nullptr);
   DCHECK_NE(result, nullptr);
-  RETURN_NOT_OK(validate_precision<int128_t>(precision));
-  Decimal128 decimal;
+  RETURN_NOT_OK(validate_precision<boost::multiprecision::int128_t>(precision));
+  decimal::Decimal128 decimal;
   FromBytes(bytes, is_negative, &decimal);
   *result = ToString(decimal, precision, scale);
   return Status::OK();
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index b63d2ffb1cd2c..a4a11c039b60c 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -63,8 +63,8 @@ TEST(DecimalTest, TestPythonDecimalToArrowDecimal128) {
   ASSERT_NE(pydecimal.obj(), nullptr);
   ASSERT_EQ(PyErr_Occurred(), nullptr);
 
-  Decimal128 arrow_decimal;
-  int128_t boost_decimal(decimal_string);
+  decimal::Decimal128 arrow_decimal;
+  boost::multiprecision::int128_t boost_decimal(decimal_string);
   PyObject* obj = pydecimal.obj();
   ASSERT_OK(PythonDecimalToArrowDecimal(obj, &arrow_decimal));
   ASSERT_EQ(boost_decimal, arrow_decimal.value);
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index acf12c3d9d18e..2bb05f853a094 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -147,7 +147,7 @@ std::shared_ptr<DataType> ARROW_EXPORT binary();
 
 std::shared_ptr<DataType> ARROW_EXPORT date32();
 std::shared_ptr<DataType> ARROW_EXPORT date64();
-std::shared_ptr<DataType> ARROW_EXPORT decimal(int precision, int scale);
+std::shared_ptr<DataType> ARROW_EXPORT decimal_type(int precision, int scale);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 054f11055b60e..9aa8bae273fb8 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -22,7 +22,6 @@
 # Headers: top level
 install(FILES
   bit-util.h
-  decimal.h
   logging.h
   macros.h
   random.h
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 1e22643962d5b..dcaa9afd8724a 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -23,6 +23,7 @@
 #include "arrow/test-util.h"
 
 namespace arrow {
+namespace decimal {
 
 template <typename T>
 class DecimalTest : public ::testing::Test {
@@ -158,4 +159,5 @@ TEST(DecimalTest, TestDecimal128StringAndBytesRoundTrip) {
 
   ASSERT_EQ(expected.value, result.value);
 }
+}  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 1ac347180fec5..3b8a3ff0398b5 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -20,6 +20,7 @@
 #include <boost/regex.hpp>
 
 namespace arrow {
+namespace decimal {
 
 static const boost::regex DECIMAL_PATTERN("(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?");
 
@@ -138,4 +139,5 @@ void ToBytes(const Decimal128& decimal, uint8_t** bytes, bool* is_negative) {
   *is_negative = backend.isneg();
 }
 
+}  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 46883e3de93c3..c73bae1b4c995 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -31,6 +31,7 @@
 #include <boost/multiprecision/cpp_int.hpp>
 
 namespace arrow {
+namespace decimal {
 
 using boost::multiprecision::int128_t;
 
@@ -140,5 +141,6 @@ ARROW_EXPORT void ToBytes(const Decimal32& value, uint8_t** bytes);
 ARROW_EXPORT void ToBytes(const Decimal64& value, uint8_t** bytes);
 ARROW_EXPORT void ToBytes(const Decimal128& decimal, uint8_t** bytes, bool* is_negative);
 
+}  // namespace decimal
 }  // namespace arrow
 #endif  // ARROW_DECIMAL_H
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 4860334a9213c..ab38ff3084f01 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -51,11 +51,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsTypeError()
 
 
-cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil:
-    cdef cppclass int128_t:
-        pass
-
-
 cdef inline object PyObject_to_object(PyObject* o):
     # Cast to "object" increments reference count
     cdef object result = <object> o
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 73d96b25f521b..e719e185b7b13 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -60,11 +60,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         TimeUnit_MICRO" arrow::TimeUnit::MICRO"
         TimeUnit_NANO" arrow::TimeUnit::NANO"
 
-    cdef cppclass Decimal[T]:
-        Decimal(const T&)
-
-    cdef c_string ToString[T](const Decimal[T]&, int, int)
-
     cdef cppclass CDataType" arrow::DataType":
         Type type
 
@@ -226,7 +221,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         const uint8_t* GetValue(int i)
 
     cdef cppclass CDecimalArray" arrow::DecimalArray"(CFixedSizeBinaryArray):
-        Decimal[T] Value[T](int i)
+        c_string FormatValue(int i)
 
     cdef cppclass CListArray" arrow::ListArray"(CArray):
         const int32_t* raw_value_offsets()
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 196deedefa959..7591ae880da3d 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -17,7 +17,6 @@
 
 from pyarrow.schema cimport DataType, box_data_type
 
-from pyarrow.includes.common cimport int128_t
 from pyarrow.compat import frombytes
 import pyarrow.schema as schema
 import decimal
@@ -213,13 +212,7 @@ cdef class DecimalValue(ArrayValue):
             int bit_width = t.bit_width()
             int precision = t.precision
             int scale = t.scale
-            c_string s
-        if bit_width == 32:
-            s = ToString[int32_t](ap.Value[int32_t](self.index), precision, scale)
-        elif bit_width == 64:
-            s = ToString[int64_t](ap.Value[int64_t](self.index), precision, scale)
-        elif bit_width == 128:
-            s = ToString[int128_t](ap.Value[int128_t](self.index), precision, scale)
+            c_string s = ap.FormatValue(self.index)
         return decimal.Decimal(s.decode('utf8'))
 
 

From acbda1893c55c68b3afd6c4cde1ee11e6926bb75 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 10 Apr 2017 08:11:25 -0400
Subject: [PATCH 0489/1644] ARROW-794: [C++/Python] Disallow strided tensors in
 ipc::WriteTensor

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #519 from wesm/ARROW-794 and squashes the following commits:

ab82ba7 [Wes McKinney] Typo
6f0a350 [Wes McKinney] Typo
e945298 [Wes McKinney] Raise ValueError if writing strided ndarray with WriteTensor and pyarrow.write_tensor
---
 cpp/src/arrow/ipc/ipc-read-write-test.cc | 18 ++++++++++++++++++
 cpp/src/arrow/ipc/writer.cc              |  4 ++++
 python/pyarrow/tests/test_tensor.py      | 13 +++++++++++++
 3 files changed, 35 insertions(+)

diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 6807296b59a5e..1a91ec39ca1fc 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -642,5 +642,23 @@ TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
   CheckTensorRoundTrip(tzero);
 }
 
+TEST_F(TestTensorRoundTrip, NonContiguous) {
+  std::string path = "test-write-tensor-strided";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_));
+
+  std::vector<int64_t> values;
+  test::randint<int64_t>(24, 0, 100, &values);
+
+  auto data = test::GetBufferFromVector(values);
+  Int64Tensor tensor(data, {4, 3}, {48, 16});
+
+  int32_t metadata_length;
+  int64_t body_length;
+  ASSERT_OK(mmap_->Seek(0));
+  ASSERT_RAISES(
+      Invalid, WriteTensor(tensor, mmap_.get(), &metadata_length, &body_length));
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 9305567e74f6b..d38a65c983d98 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -496,6 +496,10 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
 
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
     int64_t* body_length) {
+  if (!tensor.is_contiguous()) {
+    return Status::Invalid("No support yet for writing non-contiguous tensors");
+  }
+
   RETURN_NOT_OK(AlignStreamPosition(dst));
   std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(WriteTensorMessage(tensor, 0, &metadata));
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index a39064b49dfbc..327b7f08a37f1 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -98,3 +98,16 @@ def test_tensor_ipc_roundtrip():
         assert result.equals(tensor)
     finally:
         _try_delete(path)
+
+
+def test_tensor_ipc_strided():
+    data = np.random.randn(10, 4)
+    tensor = pa.Tensor.from_numpy(data[::2])
+
+    path = 'pyarrow-tensor-ipc-strided'
+    try:
+        with pytest.raises(ValueError):
+            mmap = pa.create_memory_map(path, 1024)
+            pa.write_tensor(tensor, mmap)
+    finally:
+        _try_delete(path)

From ddda3039e6fb6a9d4f2c5b1189369204bfe1ea93 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 10 Apr 2017 08:30:27 -0400
Subject: [PATCH 0490/1644] ARROW-526: [Format] Revise Format documents for
 evolution in IPC stream / file / tensor formats

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #515 from wesm/ARROW-526 and squashes the following commits:

6a38432 [Wes McKinney] Typo
5d564a6 [Wes McKinney] Revise Format documents for evolution in IPC stream / file / tensor formats
---
 format/IPC.md      | 131 ++++++++++++++++++++++++++++++++-------------
 format/Metadata.md |  57 +++++++++++++++++---
 2 files changed, 146 insertions(+), 42 deletions(-)

diff --git a/format/IPC.md b/format/IPC.md
index d386e6048cf12..f0a67e292186c 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -14,65 +14,106 @@
 
 # Interprocess messaging / communication (IPC)
 
-## File format
+## Encapsulated message format
+
+Data components in the stream and file formats are represented as encapsulated
+*messages* consisting of:
 
-We define a self-contained "file format" containing an Arrow schema along with
-one or more record batches defining a dataset. See [format/File.fbs][1] for the
-precise details of the file metadata.
+* A length prefix indicating the metadata size
+* The message metadata as a [Flatbuffer][3]
+* Padding bytes to an 8-byte boundary
+* The message body
 
-In general, the file looks like:
+Schematically, we have:
 
 ```
-<magic number "ARROW1">
-<empty padding bytes [to 8 byte boundary]>
+<metadata_size: int32>
+<metadata_flatbuffer: bytes>
+<padding>
+<message body>
+```
+
+The `metadata_size` includes the size of the flatbuffer plus padding. The
+`Message` flatbuffer includes a version number, the particular message (as a
+flatbuffer union), and the size of the message body:
+
+```
+table Message {
+  version: org.apache.arrow.flatbuf.MetadataVersion;
+  header: MessageHeader;
+  bodyLength: long;
+}
+```
+
+Currently, we support 4 types of messages:
+
+* Schema
+* RecordBatch
+* DictionaryBatch
+* Tensor
+
+## Streaming format
+
+We provide a streaming format for record batches. It is presented as a sequence
+of encapsulated messages, each of which follows the format above. The schema
+comes first in the stream, and it is the same for all of the record batches
+that follow. If any fields in the schema are dictionary-encoded, one or more
+`DictionaryBatch` messages will follow the schema.
+
+```
+<SCHEMA>
 <DICTIONARY 0>
 ...
 <DICTIONARY k - 1>
 <RECORD BATCH 0>
 ...
 <RECORD BATCH n - 1>
-<METADATA org.apache.arrow.flatbuf.Footer>
-<metadata_size: int32>
-<magic number "ARROW1">
+<EOS [optional]: int32>
 ```
 
-See the File.fbs document for details about the Flatbuffers metadata. The
-record batches have a particular structure, defined next.
+When a stream reader implementation is reading a stream, after each message, it
+may read the next 4 bytes to know how large the message metadata that follows
+is. Once the message flatbuffer is read, you can then read the message body.
+
+The stream writer can signal end-of-stream (EOS) either by writing a 0 length
+as an `int32` or simply closing the stream interface.
+
+## File format
 
-### Record batches
+We define a "file format" supporting random access in a very similar format to
+the streaming format. The file starts and ends with a magic string `ARROW1`
+(plus padding). What follows in the file is identical to the stream format. At
+the end of the file, we write a *footer* including offsets and sizes for each
+of the data blocks in the file, so that random access is possible. See
+[format/File.fbs][1] for the precise details of the file footer.
 
-The record batch metadata is written as a flatbuffer (see
-[format/Message.fbs][2] -- the RecordBatch message type) prefixed by its size,
-followed by each of the memory buffers in the batch written end to end (with
-appropriate alignment and padding):
+Schematically we have:
 
 ```
-<int32: metadata flatbuffer size>
-<metadata: org.apache.arrow.flatbuf.RecordBatch>
-<padding bytes [to 8-byte boundary]>
-<body: buffers end to end>
+<magic number "ARROW1">
+<empty padding bytes [to 8 byte boundary]>
+<STREAMING FORMAT>
+<FOOTER>
+<FOOTER SIZE: int32>
+<magic number "ARROW1">
 ```
 
+### RecordBatch body structure
+
 The `RecordBatch` metadata contains a depth-first (pre-order) flattened set of
 field metadata and physical memory buffers (some comments from [Message.fbs][2]
 have been shortened / removed):
 
 ```
 table RecordBatch {
-  length: int;
+  length: long;
   nodes: [FieldNode];
   buffers: [Buffer];
 }
 
 struct FieldNode {
-  /// The number of value slots in the Arrow array at this level of a nested
-  /// tree
-  length: int;
-
-  /// The number of observed nulls. Fields with null_count == 0 may choose not
-  /// to write their physical validity bitmap out as a materialized buffer,
-  /// instead setting the length of the bitmap buffer to 0.
-  null_count: int;
+  length: long;
+  null_count: long;
 }
 
 struct Buffer {
@@ -91,9 +132,9 @@ struct Buffer {
 ```
 
 In the context of a file, the `page` is not used, and the `Buffer` offsets use
-as a frame of reference the start of the segment where they are written in the
-file. So, while in a general IPC setting these offsets may be anyplace in one
-or more shared memory regions, in the file format the offsets start from 0.
+as a frame of reference the start of the message body. So, while in a general
+IPC setting these offsets may be anyplace in one or more shared memory regions,
+in the file format the offsets start from 0.
 
 The location of a record batch and the size of the metadata block as well as
 the body of buffers is stored in the file footer:
@@ -112,12 +153,30 @@ Some notes about this
 * The metadata length includes the flatbuffer size, the record batch metadata
   flatbuffer, and any padding bytes
 
-
-### Dictionary batches
+### Dictionary Batches
 
 Dictionary batches have not yet been implemented, while they are provided for
 in the metadata. For the time being, the `DICTIONARY` segments shown above in
 the file do not appear in any of the file implementations.
 
+### Tensor (Multi-dimensional Array) Message Format
+
+The `Tensor` message types provides a way to write a multidimensional array of
+fixed-size values (such as a NumPy ndarray) using Arrow's shared memory
+tools. Arrow implementations in general are not required to implement this data
+format, though we provide a reference implementation in C++.
+
+When writing a standalone encapsulated tensor message, we use the format as
+indicated above, but additionally align the starting offset (if writing to a
+shared memory region) to be a multiple of 8:
+
+```
+<PADDING>
+<metadata size: int32>
+<metadata>
+<tensor body>
+```
+
 [1]: https://github.com/apache/arrow/blob/master/format/File.fbs
-[1]: https://github.com/apache/arrow/blob/master/format/Message.fbs
\ No newline at end of file
+[2]: https://github.com/apache/arrow/blob/master/format/Message.fbs
+[3]: https://github.com/google]/flatbuffers
diff --git a/format/Metadata.md b/format/Metadata.md
index a4878f347073f..18fac527470d5 100644
--- a/format/Metadata.md
+++ b/format/Metadata.md
@@ -86,8 +86,8 @@ VectorLayout:
 Type:
 ```
 {
-  "name" : "null|struct|list|union|int|floatingpoint|utf8|binary|bool|decimal|date|time|timestamp|interval"
-  // fields as defined in the flatbuff depending on the type name
+  "name" : "null|struct|list|union|int|floatingpoint|utf8|binary|fixedsizebinary|bool|decimal|date|time|timestamp|interval"
+  // fields as defined in the Flatbuffer depending on the type name
 }
 ```
 Union:
@@ -126,14 +126,37 @@ Decimal:
   "scale" : /* integer */
 }
 ```
+
 Timestamp:
+
 ```
 {
   "name" : "timestamp",
   "unit" : "SECOND|MILLISECOND|MICROSECOND|NANOSECOND"
 }
 ```
+
+Date:
+
+```
+{
+  "name" : "date",
+  "unit" : "DAY|MILLISECOND"
+}
+```
+
+Time:
+
+```
+{
+  "name" : "time",
+  "unit" : "SECOND|MILLISECOND|MICROSECOND|NANOSECOND",
+  "bitWidth": /* integer: 32 or 64 */
+}
+```
+
 Interval:
+
 ```
 {
   "name" : "interval",
@@ -161,12 +184,16 @@ Flatbuffers IDL for a record batch data header
 
 ```
 table RecordBatch {
-  length: int;
+  length: long;
   nodes: [FieldNode];
   buffers: [Buffer];
 }
 ```
 
+The `RecordBatch` metadata provides for record batches with length exceeding
+2^31 - 1, but Arrow implementations are not required to implement support
+beyond this size.
+
 The `nodes` and `buffers` fields are produced by a depth-first traversal /
 flattening of a schema (possibly containing nested types) for a given in-memory
 data set.
@@ -205,13 +232,17 @@ hierarchy.
 struct FieldNode {
   /// The number of value slots in the Arrow array at this level of a nested
   /// tree
-  length: int;
+  length: long;
 
   /// The number of observed nulls.
-  null_count: int;
+  null_count: lohng;
 }
 ```
 
+The `FieldNode` metadata provides for fields with length exceeding 2^31 - 1,
+but Arrow implementations are not required to implement support for large
+arrays.
+
 ## Flattening of nested data
 
 Nested types are flattened in the record batch in depth-first order. When
@@ -359,7 +390,21 @@ TBD
 
 ### Timestamp
 
-TBD
+All timestamps are stored as a 64-bit integer, with one of four unit
+resolutions: second, millisecond, microsecond, and nanosecond.
+
+### Date
+
+We support two different date types:
+
+* Days since the UNIX epoch as a 32-bit integer
+* Milliseconds since the UNIX epoch as a 64-bit integer
+
+### Time
+
+Time supports the same unit resolutions: second, millisecond, microsecond, and
+nanosecond. We represent time as the smallest integer accommodating the
+indicated unit. For second and millisecond: 32-bit, for the others 64-bit.
 
 ## Dictionary encoding
 

From d1a9aff2937efe54fe3a5c80f7fbe19851cb71f3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 10 Apr 2017 09:23:12 -0400
Subject: [PATCH 0491/1644] ARROW-795: [C++] Consolidate
 arrow/arrow_io/arrow_ipc into a single shared and static library

This leaves c_glib in a broken state, since I'd rather let @kou fix up things the way he prefers (I started refactoring the automake files and realized I would probably just make a mess).

I'm going to submit a patch to make parquet-cpp work on top of this.

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Kouhei Sutou <kou@clear-code.com>

Closes #516 from wesm/ARROW-795 and squashes the following commits:

7221e66 [Kouhei Sutou] Update arrow-glib after codebase consolidation
207749c [Wes McKinney] Consolidate arrow/arrow_io/arrow_ipc into a single shared and static library
---
 c_glib/arrow-glib/Makefile.am             | 295 +++++-----------------
 c_glib/arrow-glib/arrow-glib.h            |  17 ++
 c_glib/arrow-glib/arrow-glib.hpp          |  16 ++
 c_glib/arrow-glib/arrow-io-glib.h         |  32 ---
 c_glib/arrow-glib/arrow-io-glib.hpp       |  30 ---
 c_glib/arrow-glib/arrow-io-glib.pc.in     |  28 --
 c_glib/arrow-glib/arrow-ipc-glib.h        |  27 --
 c_glib/arrow-glib/arrow-ipc-glib.hpp      |  30 ---
 c_glib/arrow-glib/arrow-ipc-glib.pc.in    |  28 --
 c_glib/arrow-glib/io-enums.c.template     |  56 ----
 c_glib/arrow-glib/io-enums.h.template     |  41 ---
 c_glib/arrow-glib/ipc-enums.c.template    |  56 ----
 c_glib/arrow-glib/ipc-enums.h.template    |  41 ---
 c_glib/configure.ac                       |  12 -
 c_glib/doc/reference/Makefile.am          |   8 +-
 c_glib/test/run-test.rb                   |   2 -
 c_glib/test/test-io-file-output-stream.rb |   4 +-
 c_glib/test/test-io-memory-mapped-file.rb |  20 +-
 c_glib/test/test-ipc-file-writer.rb       |   8 +-
 c_glib/test/test-ipc-stream-writer.rb     |   8 +-
 cpp/CMakeLists.txt                        | 130 +++++++---
 cpp/src/arrow/io/CMakeLists.txt           |  89 -------
 cpp/src/arrow/io/arrow-io.pc.in           |  30 ---
 cpp/src/arrow/io/symbols.map              |  30 ---
 cpp/src/arrow/ipc/CMakeLists.txt          |  83 +-----
 cpp/src/arrow/ipc/arrow-ipc.pc.in         |  30 ---
 cpp/src/arrow/ipc/symbols.map             |  30 ---
 cpp/src/arrow/python/CMakeLists.txt       |   4 -
 python/CMakeLists.txt                     |   8 -
 python/cmake_modules/FindArrow.cmake      |  24 --
 python/setup.py                           |   2 -
 31 files changed, 214 insertions(+), 1005 deletions(-)
 delete mode 100644 c_glib/arrow-glib/arrow-io-glib.h
 delete mode 100644 c_glib/arrow-glib/arrow-io-glib.hpp
 delete mode 100644 c_glib/arrow-glib/arrow-io-glib.pc.in
 delete mode 100644 c_glib/arrow-glib/arrow-ipc-glib.h
 delete mode 100644 c_glib/arrow-glib/arrow-ipc-glib.hpp
 delete mode 100644 c_glib/arrow-glib/arrow-ipc-glib.pc.in
 delete mode 100644 c_glib/arrow-glib/io-enums.c.template
 delete mode 100644 c_glib/arrow-glib/io-enums.h.template
 delete mode 100644 c_glib/arrow-glib/ipc-enums.c.template
 delete mode 100644 c_glib/arrow-glib/ipc-enums.h.template
 delete mode 100644 cpp/src/arrow/io/arrow-io.pc.in
 delete mode 100644 cpp/src/arrow/io/symbols.map
 delete mode 100644 cpp/src/arrow/ipc/arrow-ipc.pc.in
 delete mode 100644 cpp/src/arrow/ipc/symbols.map

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index a72d1e874402a..e719cccfa85ab 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -101,6 +101,25 @@ libarrow_glib_la_headers =			\
 	uint64-array-builder.h			\
 	uint64-data-type.h
 
+libarrow_glib_la_headers +=			\
+	io-file.h				\
+	io-file-mode.h				\
+	io-file-output-stream.h			\
+	io-input-stream.h			\
+	io-memory-mapped-file.h			\
+	io-output-stream.h			\
+	io-random-access-file.h			\
+	io-readable.h				\
+	io-writeable.h				\
+	io-writeable-file.h
+
+libarrow_glib_la_headers +=			\
+	ipc-file-reader.h			\
+	ipc-file-writer.h			\
+	ipc-stream-reader.h			\
+	ipc-stream-writer.h			\
+	ipc-metadata-version.h
+
 libarrow_glib_la_generated_headers =		\
 	enums.h
 
@@ -170,6 +189,25 @@ libarrow_glib_la_sources =			\
 	$(libarrow_glib_la_headers)		\
 	$(libarrow_glib_la_generated_sources)
 
+libarrow_glib_la_sources +=			\
+	io-file.cpp				\
+	io-file-mode.cpp			\
+	io-file-output-stream.cpp		\
+	io-input-stream.cpp			\
+	io-memory-mapped-file.cpp		\
+	io-output-stream.cpp			\
+	io-random-access-file.cpp		\
+	io-readable.cpp				\
+	io-writeable.cpp			\
+	io-writeable-file.cpp
+
+libarrow_glib_la_sources +=			\
+	ipc-file-reader.cpp			\
+	ipc-file-writer.cpp			\
+	ipc-metadata-version.cpp		\
+	ipc-stream-reader.cpp			\
+	ipc-stream-writer.cpp
+
 libarrow_glib_la_cpp_headers =			\
 	array.hpp				\
 	array-builder.hpp			\
@@ -184,6 +222,25 @@ libarrow_glib_la_cpp_headers =			\
 	table.hpp				\
 	type.hpp
 
+libarrow_glib_la_cpp_headers +=			\
+	io-file.hpp				\
+	io-file-mode.hpp			\
+	io-file-output-stream.hpp		\
+	io-input-stream.hpp			\
+	io-memory-mapped-file.hpp		\
+	io-output-stream.hpp			\
+	io-random-access-file.hpp		\
+	io-readable.hpp				\
+	io-writeable.hpp			\
+	io-writeable-file.hpp
+
+libarrow_glib_la_cpp_headers +=			\
+	ipc-file-reader.hpp			\
+	ipc-file-writer.hpp			\
+	ipc-metadata-version.hpp		\
+	ipc-stream-reader.hpp			\
+	ipc-stream-writer.hpp
+
 libarrow_glib_la_SOURCES =			\
 	$(libarrow_glib_la_sources)		\
 	$(libarrow_glib_la_cpp_headers)
@@ -221,205 +278,15 @@ stamp-enums.c: $(libarrow_glib_la_headers) enums.c.template
 	     $(libarrow_glib_la_headers)) > enums.c
 	touch $@
 
-# libarrow-io-glib
-lib_LTLIBRARIES +=				\
-	libarrow-io-glib.la
-
-libarrow_io_glib_la_CXXFLAGS =			\
-	$(GLIB_CFLAGS)				\
-	$(ARROW_IO_CFLAGS)			\
-	$(GARROW_CXXFLAGS)
-
-libarrow_io_glib_la_LIBADD =			\
-	$(GLIB_LIBS)				\
-	$(ARROW_IO_LIBS)			\
-	libarrow-glib.la
-
-libarrow_io_glib_la_headers =			\
-	arrow-io-glib.h				\
-	io-file.h				\
-	io-file-mode.h				\
-	io-file-output-stream.h			\
-	io-input-stream.h			\
-	io-memory-mapped-file.h			\
-	io-output-stream.h			\
-	io-random-access-file.h			\
-	io-readable.h				\
-	io-writeable.h				\
-	io-writeable-file.h
-
-libarrow_io_glib_la_generated_headers =		\
-	io-enums.h
-
-libarrow_io_glib_la_generated_sources =			\
-	io-enums.c					\
-	$(libarrow_io_glib_la_generated_headers)
-
-libarrow_io_glib_la_sources =			\
-	io-file.cpp				\
-	io-file-mode.cpp			\
-	io-file-output-stream.cpp		\
-	io-input-stream.cpp			\
-	io-memory-mapped-file.cpp		\
-	io-output-stream.cpp			\
-	io-random-access-file.cpp		\
-	io-readable.cpp				\
-	io-writeable.cpp			\
-	io-writeable-file.cpp			\
-	$(libarrow_io_glib_la_headers)		\
-	$(libarrow_io_glib_la_generated_sources)
-
-libarrow_io_glib_la_cpp_headers =		\
-	arrow-io-glib.hpp			\
-	io-file.hpp				\
-	io-file-mode.hpp			\
-	io-file-output-stream.hpp		\
-	io-input-stream.hpp			\
-	io-memory-mapped-file.hpp		\
-	io-output-stream.hpp			\
-	io-random-access-file.hpp		\
-	io-readable.hpp				\
-	io-writeable.hpp			\
-	io-writeable-file.hpp
-
-libarrow_io_glib_la_SOURCES =			\
-	$(libarrow_io_glib_la_sources)		\
-	$(libarrow_io_glib_la_cpp_headers)
-
-BUILT_SOURCES +=					\
-	$(libarrow_io_glib_la_genearted_sources)	\
-	stamp-io-enums.c				\
-	stamp-io-enums.h
-
-EXTRA_DIST +=					\
-	io-enums.c.template			\
-	io-enums.h.template
-
-io-enums.h: stamp-io-enums.h
-	@true
-stamp-io-enums.h: $(libarrow_io_glib_la_headers) io-enums.h.template
-	$(AM_V_GEN)					\
-	  (cd $(srcdir) &&				\
-	   $(GLIB_MKENUMS)				\
-	     --identifier-prefix GArrowIO		\
-	     --symbol-prefix garrow_io			\
-	     --template io-enums.h.template		\
-	     $(libarrow_io_glib_la_headers)) > io-enums.h
-	touch $@
-
-io-enums.c: stamp-io-enums.c
-	@true
-stamp-io-enums.c: $(libarrow_io_glib_la_headers) io-enums.c.template
-	$(AM_V_GEN)					\
-	  (cd $(srcdir) &&				\
-	   $(GLIB_MKENUMS)				\
-	     --identifier-prefix GArrowIO		\
-	     --symbol-prefix garrow_io			\
-	     --template io-enums.c.template		\
-	     $(libarrow_io_glib_la_headers)) > io-enums.c
-	touch $@
-
-# libarrow-ipc-glib
-lib_LTLIBRARIES +=				\
-	libarrow-ipc-glib.la
-
-libarrow_ipc_glib_la_CXXFLAGS =			\
-	$(GLIB_CFLAGS)				\
-	$(ARROW_IPC_CFLAGS)			\
-	$(GARROW_CXXFLAGS)
-
-libarrow_ipc_glib_la_LIBADD =			\
-	$(GLIB_LIBS)				\
-	$(ARROW_IPC_LIBS)			\
-	libarrow-glib.la			\
-	libarrow-io-glib.la
-
-libarrow_ipc_glib_la_headers =			\
-	arrow-ipc-glib.h			\
-	ipc-file-reader.h			\
-	ipc-file-writer.h			\
-	ipc-stream-reader.h			\
-	ipc-stream-writer.h			\
-	ipc-metadata-version.h
-
-libarrow_ipc_glib_la_generated_headers =	\
-	ipc-enums.h
-
-libarrow_ipc_glib_la_generated_sources =		\
-	ipc-enums.c					\
-	$(libarrow_ipc_glib_la_generated_headers)
-
-libarrow_ipc_glib_la_sources =				\
-	ipc-file-reader.cpp				\
-	ipc-file-writer.cpp				\
-	ipc-metadata-version.cpp			\
-	ipc-stream-reader.cpp				\
-	ipc-stream-writer.cpp				\
-	$(libarrow_ipc_glib_la_headers)			\
-	$(libarrow_ipc_glib_la_generated_sources)
-
-libarrow_ipc_glib_la_cpp_headers =		\
-	arrow-ipc-glib.hpp			\
-	ipc-file-reader.hpp			\
-	ipc-file-writer.hpp			\
-	ipc-metadata-version.hpp		\
-	ipc-stream-reader.hpp			\
-	ipc-stream-writer.hpp
-
-libarrow_ipc_glib_la_SOURCES =			\
-	$(libarrow_ipc_glib_la_sources)		\
-	$(libarrow_ipc_glib_la_cpp_headers)
-
-BUILT_SOURCES +=					\
-	$(libarrow_ipc_glib_la_genearted_sources)	\
-	stamp-ipc-enums.c				\
-	stamp-ipc-enums.h
-
-EXTRA_DIST +=					\
-	ipc-enums.c.template			\
-	ipc-enums.h.template
-
-ipc-enums.h: stamp-ipc-enums.h
-	@true
-stamp-ipc-enums.h: $(libarrow_ipc_glib_la_headers) ipc-enums.h.template
-	$(AM_V_GEN)						\
-	  (cd $(srcdir) &&					\
-	   $(GLIB_MKENUMS)					\
-	     --identifier-prefix GArrowIPC			\
-	     --symbol-prefix garrow_ipc				\
-	     --template ipc-enums.h.template			\
-	     $(libarrow_ipc_glib_la_headers)) > ipc-enums.h
-	touch $@
-
-ipc-enums.c: stamp-ipc-enums.c
-	@true
-stamp-ipc-enums.c: $(libarrow_ipc_glib_la_headers) ipc-enums.c.template
-	$(AM_V_GEN)						\
-	  (cd $(srcdir) &&					\
-	   $(GLIB_MKENUMS)					\
-	     --identifier-prefix GArrowIPC			\
-	     --symbol-prefix garrow_ipc				\
-	     --template ipc-enums.c.template			\
-	     $(libarrow_ipc_glib_la_headers)) > ipc-enums.c
-	touch $@
-
 arrow_glib_includedir = $(includedir)/arrow-glib
-arrow_glib_include_HEADERS =				\
-	$(libarrow_glib_la_headers)			\
-	$(libarrow_glib_la_cpp_headers)			\
-	$(libarrow_glib_la_generated_headers)		\
-	$(libarrow_io_glib_la_headers)			\
-	$(libarrow_io_glib_la_cpp_headers)		\
-	$(libarrow_io_glib_la_generated_headers)	\
-	$(libarrow_ipc_glib_la_headers)			\
-	$(libarrow_ipc_glib_la_cpp_headers)		\
-	$(libarrow_ipc_glib_la_generated_headers)
+arrow_glib_include_HEADERS =			\
+	$(libarrow_glib_la_headers)		\
+	$(libarrow_glib_la_cpp_headers)		\
+	$(libarrow_glib_la_generated_headers)
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA =				\
-	arrow-glib.pc				\
-	arrow-io-glib.pc			\
-	arrow-ipc-glib.pc
+	arrow-glib.pc
 
 # GObject Introspection
 -include $(INTROSPECTION_MAKEFILE)
@@ -443,44 +310,6 @@ Arrow_1_0_gir_SCANNERFLAGS =			\
 	--symbol-prefix=garrow
 INTROSPECTION_GIRS += Arrow-1.0.gir
 
-ArrowIO-1.0.gir: libarrow-io-glib.la
-ArrowIO-1.0.gir: Arrow-1.0.gir
-ArrowIO_1_0_gir_PACKAGES =			\
-	gobject-2.0
-ArrowIO_1_0_gir_EXPORT_PACKAGES = arrow-io
-ArrowIO_1_0_gir_INCLUDES =			\
-	GObject-2.0
-ArrowIO_1_0_gir_CFLAGS =			\
-	$(AM_CPPFLAGS)
-ArrowIO_1_0_gir_LIBS = libarrow-io-glib.la
-ArrowIO_1_0_gir_FILES = $(libarrow_io_glib_la_sources)
-ArrowIO_1_0_gir_SCANNERFLAGS =				\
-	--include-uninstalled=$(builddir)/Arrow-1.0.gir	\
-	--warn-all					\
-	--identifier-prefix=GArrowIO			\
-	--symbol-prefix=garrow_io
-INTROSPECTION_GIRS += ArrowIO-1.0.gir
-
-ArrowIPC-1.0.gir: libarrow-ipc-glib.la
-ArrowIPC-1.0.gir: Arrow-1.0.gir
-ArrowIPC-1.0.gir: ArrowIO-1.0.gir
-ArrowIPC_1_0_gir_PACKAGES =			\
-	gobject-2.0
-ArrowIPC_1_0_gir_EXPORT_PACKAGES = arrow-ipc
-ArrowIPC_1_0_gir_INCLUDES =			\
-	GObject-2.0
-ArrowIPC_1_0_gir_CFLAGS =			\
-	$(AM_CPPFLAGS)
-ArrowIPC_1_0_gir_LIBS = libarrow-ipc-glib.la
-ArrowIPC_1_0_gir_FILES = $(libarrow_ipc_glib_la_sources)
-ArrowIPC_1_0_gir_SCANNERFLAGS =					\
-	--include-uninstalled=$(builddir)/Arrow-1.0.gir		\
-	--include-uninstalled=$(builddir)/ArrowIO-1.0.gir	\
-	--warn-all						\
-	--identifier-prefix=GArrowIPC				\
-	--symbol-prefix=garrow_ipc
-INTROSPECTION_GIRS += ArrowIPC-1.0.gir
-
 girdir = $(datadir)/gir-1.0
 gir_DATA = $(INTROSPECTION_GIRS)
 
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index 4356234a4a63d..9b03175799f44 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -78,3 +78,20 @@
 #include <arrow-glib/uint64-array.h>
 #include <arrow-glib/uint64-array-builder.h>
 #include <arrow-glib/uint64-data-type.h>
+
+#include <arrow-glib/io-file.h>
+#include <arrow-glib/io-file-mode.h>
+#include <arrow-glib/io-file-output-stream.h>
+#include <arrow-glib/io-input-stream.h>
+#include <arrow-glib/io-memory-mapped-file.h>
+#include <arrow-glib/io-output-stream.h>
+#include <arrow-glib/io-random-access-file.h>
+#include <arrow-glib/io-readable.h>
+#include <arrow-glib/io-writeable.h>
+#include <arrow-glib/io-writeable-file.h>
+
+#include <arrow-glib/ipc-file-reader.h>
+#include <arrow-glib/ipc-file-writer.h>
+#include <arrow-glib/ipc-metadata-version.h>
+#include <arrow-glib/ipc-stream-reader.h>
+#include <arrow-glib/ipc-stream-writer.h>
diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
index 70fda8da7c526..fd59d4a1a9240 100644
--- a/c_glib/arrow-glib/arrow-glib.hpp
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -35,3 +35,19 @@
 #include <arrow-glib/schema.hpp>
 #include <arrow-glib/table.hpp>
 #include <arrow-glib/type.hpp>
+
+#include <arrow-glib/io-file.hpp>
+#include <arrow-glib/io-file-mode.hpp>
+#include <arrow-glib/io-file-output-stream.hpp>
+#include <arrow-glib/io-input-stream.hpp>
+#include <arrow-glib/io-memory-mapped-file.hpp>
+#include <arrow-glib/io-output-stream.hpp>
+#include <arrow-glib/io-random-access-file.hpp>
+#include <arrow-glib/io-readable.hpp>
+#include <arrow-glib/io-writeable.hpp>
+
+#include <arrow-glib/ipc-file-reader.hpp>
+#include <arrow-glib/ipc-file-writer.hpp>
+#include <arrow-glib/ipc-metadata-version.hpp>
+#include <arrow-glib/ipc-stream-reader.hpp>
+#include <arrow-glib/ipc-stream-writer.hpp>
diff --git a/c_glib/arrow-glib/arrow-io-glib.h b/c_glib/arrow-glib/arrow-io-glib.h
deleted file mode 100644
index 4d49a9859d82a..0000000000000
--- a/c_glib/arrow-glib/arrow-io-glib.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/io-enums.h>
-#include <arrow-glib/io-file.h>
-#include <arrow-glib/io-file-mode.h>
-#include <arrow-glib/io-file-output-stream.h>
-#include <arrow-glib/io-input-stream.h>
-#include <arrow-glib/io-memory-mapped-file.h>
-#include <arrow-glib/io-output-stream.h>
-#include <arrow-glib/io-random-access-file.h>
-#include <arrow-glib/io-readable.h>
-#include <arrow-glib/io-writeable.h>
-#include <arrow-glib/io-writeable-file.h>
diff --git a/c_glib/arrow-glib/arrow-io-glib.hpp b/c_glib/arrow-glib/arrow-io-glib.hpp
deleted file mode 100644
index 3e7636cc7ef99..0000000000000
--- a/c_glib/arrow-glib/arrow-io-glib.hpp
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/io-file.hpp>
-#include <arrow-glib/io-file-mode.hpp>
-#include <arrow-glib/io-file-output-stream.hpp>
-#include <arrow-glib/io-input-stream.hpp>
-#include <arrow-glib/io-memory-mapped-file.hpp>
-#include <arrow-glib/io-output-stream.hpp>
-#include <arrow-glib/io-random-access-file.hpp>
-#include <arrow-glib/io-readable.hpp>
-#include <arrow-glib/io-writeable.hpp>
diff --git a/c_glib/arrow-glib/arrow-io-glib.pc.in b/c_glib/arrow-glib/arrow-io-glib.pc.in
deleted file mode 100644
index 4256184cf7348..0000000000000
--- a/c_glib/arrow-glib/arrow-io-glib.pc.in
+++ /dev/null
@@ -1,28 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: Apache Arrow I/O GLib
-Description: C API for Apache Arrow I/O based on GLib
-Version: @VERSION@
-Libs: -L${libdir} -larrow-glib-io
-Cflags: -I${includedir}
-Requires: arrow-glib arrow-io
diff --git a/c_glib/arrow-glib/arrow-ipc-glib.h b/c_glib/arrow-glib/arrow-ipc-glib.h
deleted file mode 100644
index 4954d83cd0728..0000000000000
--- a/c_glib/arrow-glib/arrow-ipc-glib.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/ipc-enums.h>
-#include <arrow-glib/ipc-file-reader.h>
-#include <arrow-glib/ipc-file-writer.h>
-#include <arrow-glib/ipc-metadata-version.h>
-#include <arrow-glib/ipc-stream-reader.h>
-#include <arrow-glib/ipc-stream-writer.h>
diff --git a/c_glib/arrow-glib/arrow-ipc-glib.hpp b/c_glib/arrow-glib/arrow-ipc-glib.hpp
deleted file mode 100644
index d32bc052b98e5..0000000000000
--- a/c_glib/arrow-glib/arrow-ipc-glib.hpp
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/arrow-glib.hpp>
-
-#include <arrow-glib/arrow-io-glib.hpp>
-
-#include <arrow-glib/ipc-file-reader.hpp>
-#include <arrow-glib/ipc-file-writer.hpp>
-#include <arrow-glib/ipc-metadata-version.hpp>
-#include <arrow-glib/ipc-stream-reader.hpp>
-#include <arrow-glib/ipc-stream-writer.hpp>
diff --git a/c_glib/arrow-glib/arrow-ipc-glib.pc.in b/c_glib/arrow-glib/arrow-ipc-glib.pc.in
deleted file mode 100644
index 0b04c4a808ff1..0000000000000
--- a/c_glib/arrow-glib/arrow-ipc-glib.pc.in
+++ /dev/null
@@ -1,28 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: Apache Arrow IPC GLib
-Description: C API for Apache Arrow IPC based on GLib
-Version: @VERSION@
-Libs: -L${libdir} -larrow-glib-ipc
-Cflags: -I${includedir}
-Requires: arrow-glib-io arrow-ipc
diff --git a/c_glib/arrow-glib/io-enums.c.template b/c_glib/arrow-glib/io-enums.c.template
deleted file mode 100644
index 10ee77588d98b..0000000000000
--- a/c_glib/arrow-glib/io-enums.c.template
+++ /dev/null
@@ -1,56 +0,0 @@
-/*** BEGIN file-header ***/
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/arrow-io-glib.h>
-/*** END file-header ***/
-
-/*** BEGIN file-production ***/
-
-/* enumerations from "@filename@" */
-/*** END file-production ***/
-
-/*** BEGIN value-header ***/
-GType
-@enum_name@_get_type(void)
-{
-  static GType etype = 0;
-  if (G_UNLIKELY(etype == 0)) {
-    static const G@Type@Value values[] = {
-/*** END value-header ***/
-
-/*** BEGIN value-production ***/
-      {@VALUENAME@, "@VALUENAME@", "@valuenick@"},
-/*** END value-production ***/
-
-/*** BEGIN value-tail ***/
-      {0, NULL, NULL}
-    };
-    etype = g_@type@_register_static(g_intern_static_string("@EnumName@"), values);
-  }
-  return etype;
-}
-/*** END value-tail ***/
-
-/*** BEGIN file-tail ***/
-/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/io-enums.h.template b/c_glib/arrow-glib/io-enums.h.template
deleted file mode 100644
index 429141dc76a60..0000000000000
--- a/c_glib/arrow-glib/io-enums.h.template
+++ /dev/null
@@ -1,41 +0,0 @@
-/*** BEGIN file-header ***/
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/io-file-mode.h>
-
-G_BEGIN_DECLS
-/*** END file-header ***/
-
-/*** BEGIN file-production ***/
-
-/* enumerations from "@filename@" */
-/*** END file-production ***/
-
-/*** BEGIN value-header ***/
-GType @enum_name@_get_type(void) G_GNUC_CONST;
-#define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
-/*** END value-header ***/
-
-/*** BEGIN file-tail ***/
-
-G_END_DECLS
-/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/ipc-enums.c.template b/c_glib/arrow-glib/ipc-enums.c.template
deleted file mode 100644
index c938f77477172..0000000000000
--- a/c_glib/arrow-glib/ipc-enums.c.template
+++ /dev/null
@@ -1,56 +0,0 @@
-/*** BEGIN file-header ***/
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/arrow-ipc-glib.h>
-/*** END file-header ***/
-
-/*** BEGIN file-production ***/
-
-/* enumerations from "@filename@" */
-/*** END file-production ***/
-
-/*** BEGIN value-header ***/
-GType
-@enum_name@_get_type(void)
-{
-  static GType etype = 0;
-  if (G_UNLIKELY(etype == 0)) {
-    static const G@Type@Value values[] = {
-/*** END value-header ***/
-
-/*** BEGIN value-production ***/
-      {@VALUENAME@, "@VALUENAME@", "@valuenick@"},
-/*** END value-production ***/
-
-/*** BEGIN value-tail ***/
-      {0, NULL, NULL}
-    };
-    etype = g_@type@_register_static(g_intern_static_string("@EnumName@"), values);
-  }
-  return etype;
-}
-/*** END value-tail ***/
-
-/*** BEGIN file-tail ***/
-/*** END file-tail ***/
diff --git a/c_glib/arrow-glib/ipc-enums.h.template b/c_glib/arrow-glib/ipc-enums.h.template
deleted file mode 100644
index e103c5bfeb985..0000000000000
--- a/c_glib/arrow-glib/ipc-enums.h.template
+++ /dev/null
@@ -1,41 +0,0 @@
-/*** BEGIN file-header ***/
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/ipc-metadata-version.h>
-
-G_BEGIN_DECLS
-/*** END file-header ***/
-
-/*** BEGIN file-production ***/
-
-/* enumerations from "@filename@" */
-/*** END file-production ***/
-
-/*** BEGIN value-header ***/
-GType @enum_name@_get_type(void) G_GNUC_CONST;
-#define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
-/*** END value-header ***/
-
-/*** BEGIN file-tail ***/
-
-G_END_DECLS
-/*** END file-tail ***/
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index fc24c1b3c4778..d63132e6f293c 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -74,27 +74,17 @@ AC_ARG_WITH(arrow-cpp-build-dir,
   [GARROW_ARROW_CPP_BUILD_DIR=""])
 if test "x$GARROW_ARROW_CPP_BUILD_DIR" = "x"; then
   PKG_CHECK_MODULES([ARROW], [arrow])
-  PKG_CHECK_MODULES([ARROW_IO], [arrow-io])
-  PKG_CHECK_MODULES([ARROW_IPC], [arrow-ipc])
 else
   ARROW_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src"
   ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}"
 
   ARROW_CFLAGS="-I${ARROW_INCLUDE_DIR}"
-  ARROW_IO_CFLAGS="-I${ARROW_INCLUDE_DIR}"
-  ARROW_IPC_CFLAGS="-I${ARROW_INCLUDE_DIR}"
   ARROW_LIBS="-L${ARROW_LIB_DIR} -larrow"
-  ARROW_IO_LIBS="-L${ARROW_LIB_DIR} -larrow_io"
-  ARROW_IPC_LIBS="-L${ARROW_LIB_DIR} -larrow_ipc"
 
   AC_SUBST(ARROW_LIB_DIR)
 
   AC_SUBST(ARROW_CFLAGS)
-  AC_SUBST(ARROW_IO_CFLAGS)
-  AC_SUBST(ARROW_IPC_CFLAGS)
   AC_SUBST(ARROW_LIBS)
-  AC_SUBST(ARROW_IO_LIBS)
-  AC_SUBST(ARROW_IPC_LIBS)
 fi
 
 
@@ -102,8 +92,6 @@ AC_CONFIG_FILES([
   Makefile
   arrow-glib/Makefile
   arrow-glib/arrow-glib.pc
-  arrow-glib/arrow-io-glib.pc
-  arrow-glib/arrow-ipc-glib.pc
   doc/Makefile
   doc/reference/Makefile
   example/Makefile
diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am
index d1c8e01c299a0..116bc6ce1b9a6 100644
--- a/c_glib/doc/reference/Makefile.am
+++ b/c_glib/doc/reference/Makefile.am
@@ -33,9 +33,7 @@ HFILE_GLOB =					\
 	$(top_srcdir)/arrow-glib/*.h
 
 IGNORE_HFILES =					\
-	enums.h					\
-	io-enums.h				\
-	ipc-enums.h
+	enums.h
 
 CFILE_GLOB =					\
 	$(top_srcdir)/arrow-glib/*.cpp
@@ -49,9 +47,7 @@ AM_CFLAGS =					\
 	$(ARROW_CFLAGS)
 
 GTKDOC_LIBS =						\
-	$(top_builddir)/arrow-glib/libarrow-glib.la	\
-	$(top_builddir)/arrow-glib/libarrow-io-glib.la	\
-	$(top_builddir)/arrow-glib/libarrow-ipc-glib.la
+	$(top_builddir)/arrow-glib/libarrow-glib.la
 
 include $(srcdir)/gtk-doc.make
 
diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb
index 32ceb4ad61d2e..53805caef374f 100755
--- a/c_glib/test/run-test.rb
+++ b/c_glib/test/run-test.rb
@@ -32,8 +32,6 @@
 require "gi"
 
 Arrow = GI.load("Arrow")
-ArrowIO = GI.load("ArrowIO")
-ArrowIPC = GI.load("ArrowIPC")
 
 require "tempfile"
 require_relative "helper/buildable"
diff --git a/c_glib/test/test-io-file-output-stream.rb b/c_glib/test/test-io-file-output-stream.rb
index 1f2ae5fa10fd1..e35a18361aab6 100644
--- a/c_glib/test/test-io-file-output-stream.rb
+++ b/c_glib/test/test-io-file-output-stream.rb
@@ -21,7 +21,7 @@ def test_create
       tempfile = Tempfile.open("arrow-io-file-output-stream")
       tempfile.write("Hello")
       tempfile.close
-      file = ArrowIO::FileOutputStream.open(tempfile.path, false)
+      file = Arrow::IOFileOutputStream.open(tempfile.path, false)
       file.close
       assert_equal("", File.read(tempfile.path))
     end
@@ -30,7 +30,7 @@ def test_append
       tempfile = Tempfile.open("arrow-io-file-output-stream")
       tempfile.write("Hello")
       tempfile.close
-      file = ArrowIO::FileOutputStream.open(tempfile.path, true)
+      file = Arrow::IOFileOutputStream.open(tempfile.path, true)
       file.close
       assert_equal("Hello", File.read(tempfile.path))
     end
diff --git a/c_glib/test/test-io-memory-mapped-file.rb b/c_glib/test/test-io-memory-mapped-file.rb
index 609819833614f..197d1886f1e86 100644
--- a/c_glib/test/test-io-memory-mapped-file.rb
+++ b/c_glib/test/test-io-memory-mapped-file.rb
@@ -20,7 +20,7 @@ def test_open
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
     begin
       buffer = " " * 5
       file.read(buffer)
@@ -34,7 +34,7 @@ def test_size
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
     begin
       assert_equal(5, file.size)
     ensure
@@ -46,7 +46,7 @@ def test_read
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello World")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
     begin
       buffer = " " * 5
       _success, n_read_bytes = file.read(buffer)
@@ -60,7 +60,7 @@ def test_read_at
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello World")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
     begin
       buffer = " " * 5
       _success, n_read_bytes = file.read_at(6, buffer)
@@ -74,7 +74,7 @@ def test_write
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :readwrite)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :readwrite)
     begin
       file.write("World")
     ensure
@@ -87,7 +87,7 @@ def test_write_at
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :readwrite)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :readwrite)
     begin
       file.write_at(2, "rld")
     ensure
@@ -100,7 +100,7 @@ def test_flush
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :readwrite)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :readwrite)
     begin
       file.write("World")
       file.flush
@@ -114,7 +114,7 @@ def test_tell
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello World")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
     begin
       buffer = " " * 5
       file.read(buffer)
@@ -128,9 +128,9 @@ def test_mode
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello World")
     tempfile.close
-    file = ArrowIO::MemoryMappedFile.open(tempfile.path, :readwrite)
+    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :readwrite)
     begin
-      assert_equal(ArrowIO::FileMode::READWRITE, file.mode)
+      assert_equal(Arrow::IOFileMode::READWRITE, file.mode)
     ensure
       file.close
     end
diff --git a/c_glib/test/test-ipc-file-writer.rb b/c_glib/test/test-ipc-file-writer.rb
index 369bff324e6d9..1c33ccc1919e7 100644
--- a/c_glib/test/test-ipc-file-writer.rb
+++ b/c_glib/test/test-ipc-file-writer.rb
@@ -18,11 +18,11 @@
 class TestIPCFileWriter < Test::Unit::TestCase
   def test_write_record_batch
     tempfile = Tempfile.open("arrow-ipc-file-writer")
-    output = ArrowIO::FileOutputStream.open(tempfile.path, false)
+    output = Arrow::IOFileOutputStream.open(tempfile.path, false)
     begin
       field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
       schema = Arrow::Schema.new([field])
-      file_writer = ArrowIPC::FileWriter.open(output, schema)
+      file_writer = Arrow::IPCFileWriter.open(output, schema)
       begin
         record_batch = Arrow::RecordBatch.new(schema, 0, [])
         file_writer.write_record_batch(record_batch)
@@ -33,9 +33,9 @@ def test_write_record_batch
       output.close
     end
 
-    input = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    input = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
     begin
-      file_reader = ArrowIPC::FileReader.open(input)
+      file_reader = Arrow::IPCFileReader.open(input)
       assert_equal(["enabled"],
                    file_reader.schema.fields.collect(&:name))
     ensure
diff --git a/c_glib/test/test-ipc-stream-writer.rb b/c_glib/test/test-ipc-stream-writer.rb
index 62ac45dce2c79..78bb4a7c1743c 100644
--- a/c_glib/test/test-ipc-stream-writer.rb
+++ b/c_glib/test/test-ipc-stream-writer.rb
@@ -20,11 +20,11 @@ class TestIPCStreamWriter < Test::Unit::TestCase
 
   def test_write_record_batch
     tempfile = Tempfile.open("arrow-ipc-stream-writer")
-    output = ArrowIO::FileOutputStream.open(tempfile.path, false)
+    output = Arrow::IOFileOutputStream.open(tempfile.path, false)
     begin
       field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
       schema = Arrow::Schema.new([field])
-      stream_writer = ArrowIPC::StreamWriter.open(output, schema)
+      stream_writer = Arrow::IPCStreamWriter.open(output, schema)
       begin
         columns = [
           build_boolean_array([true]),
@@ -38,9 +38,9 @@ def test_write_record_batch
       output.close
     end
 
-    input = ArrowIO::MemoryMappedFile.open(tempfile.path, :read)
+    input = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
     begin
-      stream_reader = ArrowIPC::StreamReader.open(input)
+      stream_reader = Arrow::IPCStreamReader.open(input)
       assert_equal(["enabled"],
                    stream_reader.schema.fields.collect(&:name))
       assert_equal(true,
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index b29cb7b075a94..0e4a4bbf34b67 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -705,6 +705,49 @@ endif()
 #   set(ARROW_TCMALLOC_AVAILABLE 1)
 # endif()
 
+## Flatbuffers
+
+if("$ENV{FLATBUFFERS_HOME}" STREQUAL "")
+  set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
+  ExternalProject_Add(flatbuffers_ep
+    URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
+    CMAKE_ARGS
+    "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
+    "-DFLATBUFFERS_BUILD_TESTS=OFF")
+
+  set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
+  set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
+  set(FLATBUFFERS_VENDORED 1)
+else()
+  find_package(Flatbuffers REQUIRED)
+  set(FLATBUFFERS_VENDORED 0)
+endif()
+
+message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
+message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
+include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
+
+########################################################################
+# HDFS thirdparty setup
+
+if (DEFINED ENV{HADOOP_HOME})
+  set(HADOOP_HOME $ENV{HADOOP_HOME})
+  if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
+    message(STATUS "Did not find hdfs.h in expected location, using vendored one")
+    set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+  endif()
+else()
+  set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+endif()
+
+set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
+if (NOT EXISTS ${HDFS_H_PATH})
+  message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
+endif()
+message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
+
+include_directories(SYSTEM "${HADOOP_HOME}/include")
+
 ############################################################
 # Linker setup
 ############################################################
@@ -814,10 +857,37 @@ endif()
 ############################################################
 
 set(ARROW_LINK_LIBS
-    ${BOOST_REGEX_LIBRARY})
+  ${BOOST_REGEX_LIBRARY})
 
-set(ARROW_PRIVATE_LINK_LIBS
-)
+set(ARROW_STATIC_LINK_LIBS)
+
+set(ARROW_SHARED_PRIVATE_LINK_LIBS
+  ${BOOST_SYSTEM_LIBRARY}
+  ${BOOST_FILESYSTEM_LIBRARY})
+
+set(ARROW_STATIC_PRIVATE_LINK_LIBS
+  ${BOOST_SYSTEM_LIBRARY}
+  ${BOOST_FILESYSTEM_LIBRARY})
+
+if (NOT MSVC)
+  set(ARROW_LINK_LIBS
+    ${ARROW_LINK_LIBS}
+    ${CMAKE_DL_LIBS})
+endif()
+
+if(RAPIDJSON_VENDORED)
+  set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} rapidjson_ep)
+endif()
+
+if(FLATBUFFERS_VENDORED)
+  set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} flatbuffers_ep)
+endif()
+
+add_subdirectory(src/arrow)
+add_subdirectory(src/arrow/io)
+add_subdirectory(src/arrow/ipc)
+
+set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} metadata_fbs)
 
 set(ARROW_SRCS
   src/arrow/array.cc
@@ -833,6 +903,19 @@ set(ARROW_SRCS
   src/arrow/type.cc
   src/arrow/visitor.cc
 
+  src/arrow/io/file.cc
+  src/arrow/io/hdfs.cc
+  src/arrow/io/hdfs-internal.cc
+  src/arrow/io/interfaces.cc
+  src/arrow/io/memory.cc
+
+  src/arrow/ipc/feather.cc
+  src/arrow/ipc/json.cc
+  src/arrow/ipc/json-internal.cc
+  src/arrow/ipc/metadata.cc
+  src/arrow/ipc/reader.cc
+  src/arrow/ipc/writer.cc
+
   src/arrow/util/bit-util.cc
   src/arrow/util/decimal.cc
 )
@@ -844,52 +927,25 @@ if(NOT APPLE AND NOT MSVC)
   set(ARROW_SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map")
 endif()
 
+set(ARROW_ALL_SRCS
+  ${ARROW_SRCS})
 
 ADD_ARROW_LIB(arrow
-  SOURCES ${ARROW_SRCS}
+  SOURCES ${ARROW_ALL_SRCS}
+  DEPENDENCIES ${ARROW_DEPENDENCIES}
   SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
   SHARED_LINK_LIBS ${ARROW_LINK_LIBS}
+  SHARED_PRIVATE_LINK_LIBS ${ARROW_SHARED_PRIVATE_LINK_LIBS}
+  STATIC_LINK_LIBS ${ARROW_STATIC_LINK_LIBS}
+  STATIC_PRIVATE_LINK_LIBS ${ARROW_STATIC_PRIVATE_LINK_LIBS}
 )
 
-add_subdirectory(src/arrow)
-add_subdirectory(src/arrow/io)
 add_subdirectory(src/arrow/util)
 
 if(ARROW_JEMALLOC)
   add_subdirectory(src/arrow/jemalloc)
 endif()
 
-#----------------------------------------------------------------------
-# IPC library
-
-if(ARROW_PYTHON)
-  set(ARROW_IPC on)
-endif()
-
-## Flatbuffers
-if(ARROW_IPC)
-  if("$ENV{FLATBUFFERS_HOME}" STREQUAL "")
-    set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
-    ExternalProject_Add(flatbuffers_ep
-      URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
-      CMAKE_ARGS
-        "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
-        "-DFLATBUFFERS_BUILD_TESTS=OFF")
-
-    set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
-    set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
-    set(FLATBUFFERS_VENDORED 1)
-  else()
-    find_package(Flatbuffers REQUIRED)
-    set(FLATBUFFERS_VENDORED 0)
-  endif()
-
-  message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
-  message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
-  include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
-  add_subdirectory(src/arrow/ipc)
-endif()
-
 if(ARROW_PYTHON)
   find_package(PythonLibsNew REQUIRED)
   find_package(NumPy REQUIRED)
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 791c29c2797f9..c0199d7ef2599 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -18,90 +18,9 @@
 # ----------------------------------------------------------------------
 # arrow_io : Arrow IO interfaces
 
-# HDFS thirdparty setup
-if (DEFINED ENV{HADOOP_HOME})
-  set(HADOOP_HOME $ENV{HADOOP_HOME})
-  if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
-    message(STATUS "Did not find hdfs.h in expected location, using vendored one")
-    set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
-  endif()
-else()
-  set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
-endif()
-
-set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
-if (NOT EXISTS ${HDFS_H_PATH})
-  message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
-endif()
-message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
-
-include_directories(SYSTEM "${HADOOP_HOME}/include")
-
-# arrow_io library
-if (MSVC)
-  set(ARROW_IO_STATIC_LINK_LIBS
-    arrow_static
-  )
-  set(ARROW_IO_SHARED_LINK_LIBS
-    arrow_shared
-  )
-else()
-  set(ARROW_IO_STATIC_LINK_LIBS
-    arrow_static
-    ${CMAKE_DL_LIBS}
-  )
-  set(ARROW_IO_SHARED_LINK_LIBS
-    arrow_shared
-    ${CMAKE_DL_LIBS}
-  )
-endif()
-
-set(ARROW_IO_SHARED_PRIVATE_LINK_LIBS
-  ${BOOST_SYSTEM_LIBRARY}
-  ${BOOST_FILESYSTEM_LIBRARY})
-
-set(ARROW_IO_STATIC_PRIVATE_LINK_LIBS
-  ${BOOST_SYSTEM_LIBRARY}
-  ${BOOST_FILESYSTEM_LIBRARY})
-
-set(ARROW_IO_TEST_LINK_LIBS
-  arrow_io_static)
-
-set(ARROW_IO_SRCS
-  file.cc
-  hdfs.cc
-  hdfs-internal.cc
-  interfaces.cc
-  memory.cc
-)
-
-if(NOT APPLE AND NOT MSVC)
-  # Localize thirdparty symbols using a linker version script. This hides them
-  # from the client application. The OS X linker does not support the
-  # version-script option.
-  set(ARROW_IO_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
-endif()
-
-ADD_ARROW_LIB(arrow_io
-  SOURCES ${ARROW_IO_SRCS}
-  SHARED_LINK_FLAGS ${ARROW_IO_LINK_FLAGS}
-  SHARED_LINK_LIBS ${ARROW_IO_SHARED_LINK_LIBS}
-  SHARED_PRIVATE_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
-  STATIC_LINK_LIBS ${ARROW_IO_STATIC_LINK_LIBS}
-  STATIC_PRIVATE_LINK_LIBS ${ARROW_IO_STATIC_PRIVATE_LINK_LIBS}
-)
-
 ADD_ARROW_TEST(io-file-test)
-ARROW_TEST_LINK_LIBRARIES(io-file-test
-  ${ARROW_IO_TEST_LINK_LIBS})
-
 ADD_ARROW_TEST(io-hdfs-test)
-ARROW_TEST_LINK_LIBRARIES(io-hdfs-test
-  ${ARROW_IO_TEST_LINK_LIBS})
-
 ADD_ARROW_TEST(io-memory-test)
-ARROW_TEST_LINK_LIBRARIES(io-memory-test
-  ${ARROW_IO_TEST_LINK_LIBS})
 
 # Headers: top level
 install(FILES
@@ -110,11 +29,3 @@ install(FILES
   interfaces.h
   memory.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/io")
-
-# pkg-config support
-configure_file(arrow-io.pc.in
-  "${CMAKE_CURRENT_BINARY_DIR}/arrow-io.pc"
-  @ONLY)
-install(
-  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-io.pc"
-  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
diff --git a/cpp/src/arrow/io/arrow-io.pc.in b/cpp/src/arrow/io/arrow-io.pc.in
deleted file mode 100644
index 61af3577f5a38..0000000000000
--- a/cpp/src/arrow/io/arrow-io.pc.in
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
-includedir=${prefix}/include
-
-so_version=@ARROW_SO_VERSION@
-abi_version=@ARROW_ABI_VERSION@
-
-Name: Apache Arrow I/O
-Description: I/O interface for Arrow.
-Version: @ARROW_VERSION@
-Libs: -L${libdir} -larrow_io
-Cflags: -I${includedir}
-Requires: arrow
diff --git a/cpp/src/arrow/io/symbols.map b/cpp/src/arrow/io/symbols.map
deleted file mode 100644
index 1e87caef9c8c1..0000000000000
--- a/cpp/src/arrow/io/symbols.map
+++ /dev/null
@@ -1,30 +0,0 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-{
-  # Symbols marked as 'local' are not exported by the DSO and thus may not
-  # be used by client applications.
-  local:
-    # devtoolset / static-libstdc++ symbols
-    __cxa_*;
-
-    extern "C++" {
-      # boost
-      boost::*;
-
-      # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
-      # links c++11 symbols into binaries so that the result may be executed on
-      # a system with an older libstdc++ which doesn't include the necessary
-      # c++11 symbols.
-      std::*;
-    };
-};
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index c6880c56e466b..37b455395644f 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -16,85 +16,24 @@
 # under the License.
 
 #######################################
-# arrow_ipc
-#######################################
-
-set(ARROW_IPC_SHARED_LINK_LIBS
-  arrow_io_shared
-  arrow_shared
-)
-
-set(ARROW_IPC_TEST_LINK_LIBS
-  arrow_ipc_static
-  arrow_io_static
-  arrow_static
-  ${BOOST_REGEX_LIBRARY})
-
-set(ARROW_IPC_SRCS
-  feather.cc
-  json.cc
-  json-internal.cc
-  metadata.cc
-  reader.cc
-  writer.cc
-)
-
-if(NOT MSVC AND NOT APPLE)
-  # Localize thirdparty symbols using a linker version script. This hides them
-  # from the client application. The OS X linker does not support the
-  # version-script option.
-  set(ARROW_IPC_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
-endif()
-
-if(RAPIDJSON_VENDORED)
-  set(IPC_DEPENDENCIES ${IPC_DEPENDENCIES} rapidjson_ep)
-endif()
-
-if(FLATBUFFERS_VENDORED)
-  set(IPC_DEPENDENCIES ${IPC_DEPENDENCIES} flatbuffers_ep)
-endif()
-
-ADD_ARROW_LIB(arrow_ipc
-  SOURCES ${ARROW_IPC_SRCS}
-  DEPENDENCIES ${IPC_DEPENDENCIES}
-  SHARED_LINK_FLAGS ${ARROW_IPC_LINK_FLAGS}
-  SHARED_LINK_LIBS ${ARROW_IPC_SHARED_LINK_LIBS}
-  STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
-)
+# Messaging and interprocess communication
 
 ADD_ARROW_TEST(feather-test)
-ARROW_TEST_LINK_LIBRARIES(feather-test
-  ${ARROW_IPC_TEST_LINK_LIBS})
-
 ADD_ARROW_TEST(ipc-read-write-test)
-ARROW_TEST_LINK_LIBRARIES(ipc-read-write-test
-  ${ARROW_IPC_TEST_LINK_LIBS})
-
 ADD_ARROW_TEST(ipc-json-test)
-ARROW_TEST_LINK_LIBRARIES(ipc-json-test
-  ${ARROW_IPC_TEST_LINK_LIBS})
-
 ADD_ARROW_TEST(json-integration-test)
-ARROW_TEST_LINK_LIBRARIES(json-integration-test
-  ${ARROW_IPC_TEST_LINK_LIBS})
 
 if (ARROW_BUILD_TESTS)
   target_link_libraries(json-integration-test
-    gflags
-    gtest
-    ${BOOST_FILESYSTEM_LIBRARY}
-    ${BOOST_SYSTEM_LIBRARY})
+    gflags)
 
   if (UNIX)
     if (APPLE)
-      target_link_libraries(json-integration-test
-        ${CMAKE_DL_LIBS})
       set_target_properties(json-integration-test
         PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
     else()
       target_link_libraries(json-integration-test
-        pthread
-        ${CMAKE_DL_LIBS})
+        pthread)
     endif()
   endif()
 endif()
@@ -127,6 +66,7 @@ if(FLATBUFFERS_VENDORED)
 else()
   set(FBS_DEPENDS ${ABS_FBS_SRC})
 endif()
+
 add_custom_command(
   OUTPUT ${FBS_OUTPUT_FILES}
   COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC}
@@ -136,7 +76,6 @@ add_custom_command(
 )
 
 add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES})
-add_dependencies(arrow_ipc_objlib metadata_fbs)
 
 # Headers: top level
 install(FILES
@@ -148,26 +87,14 @@ install(FILES
   writer.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/ipc")
 
-# pkg-config support
-configure_file(arrow-ipc.pc.in
-  "${CMAKE_CURRENT_BINARY_DIR}/arrow-ipc.pc"
-  @ONLY)
-install(
-  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-ipc.pc"
-  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
-
 if(MSVC)
   set(UTIL_LINK_LIBS
-    arrow_ipc_static
-    arrow_io_static
     arrow_static
     ${BOOST_FILESYSTEM_LIBRARY}
     ${BOOST_SYSTEM_LIBRARY}
     ${BOOST_REGEX_LIBRARY})
 else()
   set(UTIL_LINK_LIBS
-    arrow_ipc_static
-    arrow_io_static
     arrow_static
     ${BOOST_FILESYSTEM_LIBRARY}
     ${BOOST_SYSTEM_LIBRARY}
@@ -183,5 +110,3 @@ if (ARROW_BUILD_UTILITIES)
 endif()
 
 ADD_ARROW_BENCHMARK(ipc-read-write-benchmark)
-ARROW_BENCHMARK_LINK_LIBRARIES(ipc-read-write-benchmark
-  ${ARROW_IPC_TEST_LINK_LIBS})
diff --git a/cpp/src/arrow/ipc/arrow-ipc.pc.in b/cpp/src/arrow/ipc/arrow-ipc.pc.in
deleted file mode 100644
index 29a942acf0331..0000000000000
--- a/cpp/src/arrow/ipc/arrow-ipc.pc.in
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
-includedir=${prefix}/include
-
-so_version=@ARROW_SO_VERSION@
-abi_version=@ARROW_ABI_VERSION@
-
-Name: Apache Arrow IPC
-Description: IPC extension for Arrow.
-Version: @ARROW_VERSION@
-Libs: -L${libdir} -larrow_ipc
-Cflags: -I${includedir}
-Requires: arrow-io
diff --git a/cpp/src/arrow/ipc/symbols.map b/cpp/src/arrow/ipc/symbols.map
deleted file mode 100644
index 1e87caef9c8c1..0000000000000
--- a/cpp/src/arrow/ipc/symbols.map
+++ /dev/null
@@ -1,30 +0,0 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-{
-  # Symbols marked as 'local' are not exported by the DSO and thus may not
-  # be used by client applications.
-  local:
-    # devtoolset / static-libstdc++ symbols
-    __cxa_*;
-
-    extern "C++" {
-      # boost
-      boost::*;
-
-      # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
-      # links c++11 symbols into binaries so that the result may be executed on
-      # a system with an older libstdc++ which doesn't include the necessary
-      # c++11 symbols.
-      std::*;
-    };
-};
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 604527f6304ac..8f7991e7f6832 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -35,8 +35,6 @@ endif()
 set(ARROW_PYTHON_MIN_TEST_LIBS
   arrow_python_test_main
   arrow_python_static
-  arrow_ipc_static
-  arrow_io_static
   arrow_static
   ${BOOST_REGEX_LIBRARY})
 
@@ -61,8 +59,6 @@ set(ARROW_PYTHON_SRCS
 )
 
 set(ARROW_PYTHON_SHARED_LINK_LIBS
-  arrow_io_shared
-  arrow_ipc_shared
   arrow_shared
 )
 
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 463a29d87b711..3e86521757342 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -244,17 +244,11 @@ endfunction(bundle_arrow_lib)
 if (PYARROW_BUNDLE_ARROW_CPP)
   # arrow
   bundle_arrow_lib(ARROW_SHARED_LIB)
-  bundle_arrow_lib(ARROW_IO_SHARED_LIB)
-  bundle_arrow_lib(ARROW_IPC_SHARED_LIB)
   bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
 endif()
 
 ADD_THIRDPARTY_LIB(arrow
   SHARED_LIB ${ARROW_SHARED_LIB})
-ADD_THIRDPARTY_LIB(arrow_io
-  SHARED_LIB ${ARROW_IO_SHARED_LIB})
-ADD_THIRDPARTY_LIB(arrow_ipc
-  SHARED_LIB ${ARROW_IPC_SHARED_LIB})
 ADD_THIRDPARTY_LIB(arrow_python
   SHARED_LIB ${ARROW_PYTHON_SHARED_LIB})
 
@@ -279,8 +273,6 @@ set(CYTHON_EXTENSIONS
 
 set(LINK_LIBS
   arrow_shared
-  arrow_io_shared
-  arrow_ipc_shared
   arrow_python_shared
 )
 
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index c2ca0f4ad22c8..51a887189ccd4 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -55,16 +55,6 @@ else()
   get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
 endif()
 
-find_library(ARROW_IO_LIB_PATH NAMES arrow_io
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
-
-find_library(ARROW_IPC_LIB_PATH NAMES arrow_ipc
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
-
 find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
   PATHS
   ${ARROW_SEARCH_LIB_PATH}
@@ -78,20 +68,12 @@ find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
 if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
-  set(ARROW_IO_LIB_NAME libarrow_io)
-  set(ARROW_IPC_LIB_NAME libarrow_ipc)
   set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc)
   set(ARROW_PYTHON_LIB_NAME libarrow_python)
 
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
   set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
-  set(ARROW_IO_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IO_LIB_NAME}.a)
-  set(ARROW_IO_SHARED_LIB ${ARROW_LIBS}/${ARROW_IO_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-
-  set(ARROW_IPC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IPC_LIB_NAME}.a)
-  set(ARROW_IPC_SHARED_LIB ${ARROW_LIBS}/${ARROW_IPC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-
   set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_JEMALLOC_LIB_NAME}.a)
   set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
@@ -100,8 +82,6 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
 
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
-    message(STATUS "Found the Arrow IO library: ${ARROW_IO_LIB_PATH}")
-    message(STATUS "Found the Arrow IPC library: ${ARROW_IPC_LIB_PATH}")
     message(STATUS "Found the Arrow jemalloc library: ${ARROW_JEMALLOC_LIB_PATH}")
   endif ()
 else ()
@@ -123,10 +103,6 @@ mark_as_advanced(
   ARROW_LIBS
   ARROW_STATIC_LIB
   ARROW_SHARED_LIB
-  ARROW_IO_STATIC_LIB
-  ARROW_IO_SHARED_LIB
-  ARROW_IPC_STATIC_LIB
-  ARROW_IPC_SHARED_LIB
   ARROW_JEMALLOC_STATIC_LIB
   ARROW_JEMALLOC_SHARED_LIB
 )
diff --git a/python/setup.py b/python/setup.py
index ba77e688ae1f6..99bac15c779e6 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -224,8 +224,6 @@ def move_lib(lib_name):
 
         if self.bundle_arrow_cpp:
             move_lib("arrow")
-            move_lib("arrow_io")
-            move_lib("arrow_ipc")
             move_lib("arrow_python")
             if self.with_jemalloc:
                 move_lib("arrow_jemalloc")

From 793f4e0c51e320247ba71c9ccc7970e3eac1d01e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 10 Apr 2017 09:25:17 -0400
Subject: [PATCH 0492/1644] ARROW-782: [C++] API cleanup, change public member
 access in DataType classes to functions, use class instead of struct

Breaking APIs isn't ideal, but this one is fairly long overdue. The DataType classes are more than passive data carriers, and Google's C++ guide recommends using class instead of struct for this. That means we should put members in protected or private scope, and access them.

I also renamed a couple of things to help with code clarity

* `DataType::type` is now `DataType::id()`
* `Array::type_enum` is not `Array::type_id`

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #520 from wesm/ARROW-782 and squashes the following commits:

f8dd131 [Wes McKinney] Revert changes with garrow_data_type_new_raw
40de60e [Wes McKinney] Fix glib usages of changed APIs
0097122 [Wes McKinney] Update post rebase
f725655 [Wes McKinney] cpplint
e77f6a5 [Wes McKinney] Change public member access in DataType classes to functions, use class instead of struct
---
 c_glib/arrow-glib/array-builder.cpp      |   2 +-
 c_glib/arrow-glib/array.cpp              |   4 +-
 c_glib/arrow-glib/data-type.cpp          |   4 +-
 c_glib/arrow-glib/field.cpp              |   7 +-
 cpp/src/arrow/array-decimal-test.cc      |  42 -----
 cpp/src/arrow/array-test.cc              |  12 +-
 cpp/src/arrow/array.cc                   |   8 +-
 cpp/src/arrow/array.h                    |   4 +-
 cpp/src/arrow/builder.cc                 |   8 +-
 cpp/src/arrow/compare.cc                 |  23 +--
 cpp/src/arrow/compare.h                  |   2 +-
 cpp/src/arrow/ipc/feather-test.cc        |   2 +-
 cpp/src/arrow/ipc/feather.cc             |  16 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |   6 +-
 cpp/src/arrow/ipc/json-internal.cc       |  44 ++---
 cpp/src/arrow/ipc/json.cc                |   4 +-
 cpp/src/arrow/ipc/metadata.cc            |  32 ++--
 cpp/src/arrow/ipc/metadata.h             |   4 +-
 cpp/src/arrow/ipc/reader.cc              |   2 +-
 cpp/src/arrow/ipc/test-common.h          |  18 +-
 cpp/src/arrow/ipc/writer.cc              |   4 +-
 cpp/src/arrow/ipc/writer.h               |   2 +-
 cpp/src/arrow/loader.cc                  |  11 +-
 cpp/src/arrow/loader.h                   |   2 +-
 cpp/src/arrow/python/builtin_convert.cc  |   4 +-
 cpp/src/arrow/python/numpy_convert.cc    |   2 +-
 cpp/src/arrow/python/numpy_convert.h     |   2 +-
 cpp/src/arrow/python/pandas_convert.cc   |  52 +++---
 cpp/src/arrow/python/pandas_convert.h    |   2 +-
 cpp/src/arrow/table-test.cc              |   4 +-
 cpp/src/arrow/table.cc                   |   4 +-
 cpp/src/arrow/table.h                    |   4 +-
 cpp/src/arrow/tensor.cc                  |   8 +-
 cpp/src/arrow/tensor.h                   |   2 +-
 cpp/src/arrow/type-test.cc               |  64 ++++++--
 cpp/src/arrow/type.cc                    |  34 ++--
 cpp/src/arrow/type.h                     | 200 +++++++++++++++--------
 cpp/src/arrow/type_fwd.h                 |  34 ++--
 cpp/src/arrow/visitor_inline.h           |   6 +-
 python/pyarrow/array.pyx                 |   2 +-
 python/pyarrow/includes/libarrow.pxd     |  25 ++-
 python/pyarrow/scalar.pyx                |  24 ++-
 python/pyarrow/schema.pyx                |  22 +--
 43 files changed, 407 insertions(+), 351 deletions(-)

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 0f038c8f66cee..aea93d00bafe4 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -161,7 +161,7 @@ garrow_array_builder_new_raw(std::shared_ptr<arrow::ArrayBuilder> *arrow_builder
 {
   GType type;
 
-  switch ((*arrow_builder)->type()->type) {
+  switch ((*arrow_builder)->type()->id()) {
   case arrow::Type::type::BOOL:
     type = GARROW_TYPE_BOOLEAN_ARRAY_BUILDER;
     break;
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 9d0e101e1b52f..e016ba9728dec 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -216,7 +216,7 @@ GArrowType
 garrow_array_get_value_type(GArrowArray *array)
 {
   auto arrow_array = garrow_array_get_raw(array);
-  return garrow_type_from_raw(arrow_array->type_enum());
+  return garrow_type_from_raw(arrow_array->type_id());
 }
 
 /**
@@ -247,7 +247,7 @@ garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array)
   GType type;
   GArrowArray *array;
 
-  switch ((*arrow_array)->type_enum()) {
+  switch ((*arrow_array)->type_id()) {
   case arrow::Type::type::NA:
     type = GARROW_TYPE_NULL_ARRAY;
     break;
diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/data-type.cpp
index 2df9e7a38da91..12932a16269e8 100644
--- a/c_glib/arrow-glib/data-type.cpp
+++ b/c_glib/arrow-glib/data-type.cpp
@@ -180,7 +180,7 @@ GArrowType
 garrow_data_type_type(GArrowDataType *data_type)
 {
   const auto arrow_data_type = garrow_data_type_get_raw(data_type);
-  return garrow_type_from_raw(arrow_data_type->type);
+  return garrow_type_from_raw(arrow_data_type->id());
 }
 
 G_END_DECLS
@@ -191,7 +191,7 @@ garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type)
   GType type;
   GArrowDataType *data_type;
 
-  switch ((*arrow_data_type)->type) {
+  switch ((*arrow_data_type)->id()) {
   case arrow::Type::type::NA:
     type = GARROW_TYPE_NULL_DATA_TYPE;
     break;
diff --git a/c_glib/arrow-glib/field.cpp b/c_glib/arrow-glib/field.cpp
index 0dcaf0a009a6d..5fd0c4d221bba 100644
--- a/c_glib/arrow-glib/field.cpp
+++ b/c_glib/arrow-glib/field.cpp
@@ -171,7 +171,7 @@ const gchar *
 garrow_field_get_name(GArrowField *field)
 {
   const auto arrow_field = garrow_field_get_raw(field);
-  return arrow_field->name.c_str();
+  return arrow_field->name().c_str();
 }
 
 /**
@@ -184,7 +184,8 @@ GArrowDataType *
 garrow_field_get_data_type(GArrowField *field)
 {
   const auto arrow_field = garrow_field_get_raw(field);
-  return garrow_data_type_new_raw(&arrow_field->type);
+  auto type = arrow_field->type();
+  return garrow_data_type_new_raw(&type);
 }
 
 /**
@@ -197,7 +198,7 @@ gboolean
 garrow_field_is_nullable(GArrowField *field)
 {
   const auto arrow_field = garrow_field_get_raw(field);
-  return arrow_field->nullable;
+  return arrow_field->nullable();
 }
 
 /**
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 8353acc454f40..4bde7abd9221a 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -25,48 +25,6 @@
 namespace arrow {
 namespace decimal {
 
-TEST(TypesTest, TestDecimal32Type) {
-  DecimalType t1(8, 4);
-
-  ASSERT_EQ(t1.type, Type::DECIMAL);
-  ASSERT_EQ(t1.precision, 8);
-  ASSERT_EQ(t1.scale, 4);
-
-  ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
-
-  // Test properties
-  ASSERT_EQ(t1.byte_width(), 4);
-  ASSERT_EQ(t1.bit_width(), 32);
-}
-
-TEST(TypesTest, TestDecimal64Type) {
-  DecimalType t1(12, 5);
-
-  ASSERT_EQ(t1.type, Type::DECIMAL);
-  ASSERT_EQ(t1.precision, 12);
-  ASSERT_EQ(t1.scale, 5);
-
-  ASSERT_EQ(t1.ToString(), std::string("decimal(12, 5)"));
-
-  // Test properties
-  ASSERT_EQ(t1.byte_width(), 8);
-  ASSERT_EQ(t1.bit_width(), 64);
-}
-
-TEST(TypesTest, TestDecimal128Type) {
-  DecimalType t1(27, 7);
-
-  ASSERT_EQ(t1.type, Type::DECIMAL);
-  ASSERT_EQ(t1.precision, 27);
-  ASSERT_EQ(t1.scale, 7);
-
-  ASSERT_EQ(t1.ToString(), std::string("decimal(27, 7)"));
-
-  // Test properties
-  ASSERT_EQ(t1.byte_width(), 16);
-  ASSERT_EQ(t1.bit_width(), 128);
-}
-
 template <typename T>
 class DecimalTestBase {
  public:
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index e50f4fd10b087..99279f3a8bb65 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -691,8 +691,8 @@ TEST_F(TestStringArray, TestArrayBasics) {
 TEST_F(TestStringArray, TestType) {
   std::shared_ptr<DataType> type = strings_->type();
 
-  ASSERT_EQ(Type::STRING, type->type);
-  ASSERT_EQ(Type::STRING, strings_->type_enum());
+  ASSERT_EQ(Type::STRING, type->id());
+  ASSERT_EQ(Type::STRING, strings_->type_id());
 }
 
 TEST_F(TestStringArray, TestListFunctions) {
@@ -905,8 +905,8 @@ TEST_F(TestBinaryArray, TestArrayBasics) {
 TEST_F(TestBinaryArray, TestType) {
   std::shared_ptr<DataType> type = strings_->type();
 
-  ASSERT_EQ(Type::BINARY, type->type);
-  ASSERT_EQ(Type::BINARY, strings_->type_enum());
+  ASSERT_EQ(Type::BINARY, type->id());
+  ASSERT_EQ(Type::BINARY, strings_->type_id());
 }
 
 TEST_F(TestBinaryArray, TestListFunctions) {
@@ -1679,8 +1679,8 @@ TEST_F(TestStructBuilder, TestAppendNull) {
   ASSERT_TRUE(result_->field(1)->IsNull(0));
   ASSERT_TRUE(result_->field(1)->IsNull(1));
 
-  ASSERT_EQ(Type::LIST, result_->field(0)->type_enum());
-  ASSERT_EQ(Type::INT32, result_->field(1)->type_enum());
+  ASSERT_EQ(Type::LIST, result_->field(0)->type_id());
+  ASSERT_EQ(Type::INT32, result_->field(1)->type_id());
 }
 
 TEST_F(TestStructBuilder, TestBasics) {
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index c4a78f3b2e400..e640bbd4a206e 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -312,8 +312,8 @@ bool DecimalArray::IsNegative(int64_t i) const {
 
 std::string DecimalArray::FormatValue(int64_t i) const {
   const auto type_ = std::dynamic_pointer_cast<DecimalType>(type());
-  const int precision = type_->precision;
-  const int scale = type_->scale;
+  const int precision = type_->precision();
+  const int scale = type_->scale();
   const int byte_width = byte_width_;
   const uint8_t* bytes = GetValue(i);
   switch (byte_width) {
@@ -453,11 +453,11 @@ DictionaryArray::DictionaryArray(
           indices->offset()),
       dict_type_(static_cast<const DictionaryType*>(type.get())),
       indices_(indices) {
-  DCHECK_EQ(type->type, Type::DICTIONARY);
+  DCHECK_EQ(type->id(), Type::DICTIONARY);
 }
 
 Status DictionaryArray::Validate() const {
-  Type::type index_type_id = indices_->type()->type;
+  Type::type index_type_id = indices_->type()->id();
   if (!is_integer(index_type_id)) {
     return Status::Invalid("Dictionary indices must be integer type");
   }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 4f8b22e31b4eb..071d4e30f52dd 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -80,7 +80,7 @@ class ARROW_EXPORT Array {
   int64_t null_count() const;
 
   std::shared_ptr<DataType> type() const { return type_; }
-  Type::type type_enum() const { return type_->type; }
+  Type::type type_id() const { return type_->id(); }
 
   /// Buffer for the null bitmap.
   ///
@@ -447,7 +447,7 @@ class ARROW_EXPORT UnionArray : public Array {
   const type_id_t* raw_type_ids() const { return raw_type_ids_ + offset_; }
   const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
 
-  UnionMode mode() const { return static_cast<const UnionType&>(*type_.get()).mode; }
+  UnionMode mode() const { return static_cast<const UnionType&>(*type_.get()).mode(); }
 
   std::shared_ptr<Array> child(int pos) const;
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 4281a61474cce..d85eb32652c47 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -253,7 +253,7 @@ BooleanBuilder::BooleanBuilder(MemoryPool* pool)
 
 BooleanBuilder::BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
     : BooleanBuilder(pool) {
-  DCHECK_EQ(Type::BOOL, type->type);
+  DCHECK_EQ(Type::BOOL, type->id());
 }
 
 Status BooleanBuilder::Init(int64_t capacity) {
@@ -602,7 +602,7 @@ std::shared_ptr<ArrayBuilder> StructBuilder::field_builder(int pos) const {
 // TODO(wesm): come up with a less monolithic strategy
 Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     std::shared_ptr<ArrayBuilder>* out) {
-  switch (type->type) {
+  switch (type->id()) {
     BUILDER_CASE(UINT8, UInt8Builder);
     BUILDER_CASE(INT8, Int8Builder);
     BUILDER_CASE(UINT16, UInt16Builder);
@@ -633,12 +633,12 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     }
 
     case Type::STRUCT: {
-      std::vector<FieldPtr>& fields = type->children_;
+      const std::vector<FieldPtr>& fields = type->children();
       std::vector<std::shared_ptr<ArrayBuilder>> values_builder;
 
       for (auto it : fields) {
         std::shared_ptr<ArrayBuilder> builder;
-        RETURN_NOT_OK(MakeBuilder(pool, it->type, &builder));
+        RETURN_NOT_OK(MakeBuilder(pool, it->type(), &builder));
         values_builder.push_back(builder);
       }
       out->reset(new StructBuilder(pool, type, values_builder));
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 2297e4b206d1f..e02f3f0a9a69c 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -151,7 +151,7 @@ class RangeEqualsVisitor {
     // Define a mapping from the type id to child number
     uint8_t max_code = 0;
 
-    const std::vector<uint8_t> type_codes = left_type.type_codes;
+    const std::vector<uint8_t>& type_codes = left_type.type_codes();
     for (size_t i = 0; i < type_codes.size(); ++i) {
       const uint8_t code = type_codes[i];
       if (code > max_code) { max_code = code; }
@@ -532,7 +532,7 @@ class ApproxEqualsVisitor : public ArrayEqualsVisitor {
 
 static bool BaseDataEquals(const Array& left, const Array& right) {
   if (left.length() != right.length() || left.null_count() != right.null_count() ||
-      left.type_enum() != right.type_enum()) {
+      left.type_id() != right.type_id()) {
     return false;
   }
   if (left.null_count() > 0) {
@@ -571,7 +571,7 @@ Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_star
     int64_t left_end_idx, int64_t right_start_idx, bool* are_equal) {
   if (&left == &right) {
     *are_equal = true;
-  } else if (left.type_enum() != right.type_enum()) {
+  } else if (left.type_id() != right.type_id()) {
     *are_equal = false;
   } else if (left.length() == 0) {
     *are_equal = true;
@@ -615,7 +615,7 @@ Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
   // The arrays are the same object
   if (&left == &right) {
     *are_equal = true;
-  } else if (left.type_enum() != right.type_enum()) {
+  } else if (left.type_id() != right.type_id()) {
     *are_equal = false;
   } else if (left.size() == 0) {
     *are_equal = true;
@@ -670,13 +670,13 @@ class TypeEqualsVisitor {
       Status>::type
   Visit(const T& left) {
     const auto& right = static_cast<const T&>(right_);
-    result_ = left.unit == right.unit;
+    result_ = left.unit() == right.unit();
     return Status::OK();
   }
 
   Status Visit(const TimestampType& left) {
     const auto& right = static_cast<const TimestampType&>(right_);
-    result_ = left.unit == right.unit && left.timezone == right.timezone;
+    result_ = left.unit() == right.unit() && left.timezone() == right.timezone();
     return Status::OK();
   }
 
@@ -688,7 +688,7 @@ class TypeEqualsVisitor {
 
   Status Visit(const DecimalType& left) {
     const auto& right = static_cast<const DecimalType&>(right_);
-    result_ = left.precision == right.precision && left.scale == right.scale;
+    result_ = left.precision() == right.precision() && left.scale() == right.scale();
     return Status::OK();
   }
 
@@ -699,13 +699,14 @@ class TypeEqualsVisitor {
   Status Visit(const UnionType& left) {
     const auto& right = static_cast<const UnionType&>(right_);
 
-    if (left.mode != right.mode || left.type_codes.size() != right.type_codes.size()) {
+    if (left.mode() != right.mode() ||
+        left.type_codes().size() != right.type_codes().size()) {
       result_ = false;
       return Status::OK();
     }
 
-    const std::vector<uint8_t> left_codes = left.type_codes;
-    const std::vector<uint8_t> right_codes = right.type_codes;
+    const std::vector<uint8_t>& left_codes = left.type_codes();
+    const std::vector<uint8_t>& right_codes = right.type_codes();
 
     for (size_t i = 0; i < left_codes.size(); ++i) {
       if (left_codes[i] != right_codes[i]) {
@@ -743,7 +744,7 @@ Status TypeEquals(const DataType& left, const DataType& right, bool* are_equal)
   // The arrays are the same object
   if (&left == &right) {
     *are_equal = true;
-  } else if (left.type != right.type) {
+  } else if (left.id() != right.id()) {
     *are_equal = false;
   } else {
     TypeEqualsVisitor visitor(right);
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index 522b11dadec47..96a6435c5df33 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -27,7 +27,7 @@
 namespace arrow {
 
 class Array;
-struct DataType;
+class DataType;
 class Status;
 class Tensor;
 
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index 077a44b896fc1..fb26df6e130f2 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -379,7 +379,7 @@ TEST_F(TestTableWriter, TimeTypes) {
 
   for (int i = 1; i < schema->num_fields(); ++i) {
     std::shared_ptr<Array> arr;
-    LoadArray(schema->field(i)->type, fields, buffers, &arr);
+    LoadArray(schema->field(i)->type(), fields, buffers, &arr);
     arrays.push_back(arr);
   }
 
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index e838e1fdbcd61..5dc039662ce9d 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -349,7 +349,7 @@ class TableReader::TableReaderImpl {
       buffers.push_back(nullptr);
     }
 
-    if (is_binary_like(type->type)) {
+    if (is_binary_like(type->id())) {
       int64_t offsets_size = GetOutputLength((meta->length() + 1) * sizeof(int32_t));
       buffers.push_back(SliceBuffer(buffer, offset, offsets_size));
       offset += offsets_size;
@@ -516,13 +516,13 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
   }
 
   Status LoadArrayMetadata(const Array& values, ArrayMetadata* meta) {
-    if (!(is_primitive(values.type_enum()) || is_binary_like(values.type_enum()))) {
+    if (!(is_primitive(values.type_id()) || is_binary_like(values.type_id()))) {
       std::stringstream ss;
       ss << "Array is not primitive type: " << values.type()->ToString();
       return Status::Invalid(ss.str());
     }
 
-    meta->type = ToFlatbufferType(values.type_enum());
+    meta->type = ToFlatbufferType(values.type_id());
 
     RETURN_NOT_OK(stream_->Tell(&meta->offset));
 
@@ -552,7 +552,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
 
     const uint8_t* values_buffer = nullptr;
 
-    if (is_binary_like(values.type_enum())) {
+    if (is_binary_like(values.type_id())) {
       const auto& bin_values = static_cast<const BinaryArray&>(values);
 
       int64_t offset_bytes = sizeof(int32_t) * (values.length() + 1);
@@ -570,7 +570,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
       const auto& prim_values = static_cast<const PrimitiveArray&>(values);
       const auto& fw_type = static_cast<const FixedWidthType&>(*values.type());
 
-      if (values.type_enum() == Type::BOOL) {
+      if (values.type_id() == Type::BOOL) {
         // Booleans are bit-packed
         values_bytes = BitUtil::BytesForBits(values.length());
       } else {
@@ -616,7 +616,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
   Status Visit(const DictionaryArray& values) override {
     const auto& dict_type = static_cast<const DictionaryType&>(*values.type());
 
-    if (!is_integer(values.indices()->type_enum())) {
+    if (!is_integer(values.indices()->type_id())) {
       return Status::Invalid("Category values must be integers");
     }
 
@@ -631,7 +631,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
   Status Visit(const TimestampArray& values) override {
     RETURN_NOT_OK(WritePrimitiveValues(values));
     const auto& ts_type = static_cast<const TimestampType&>(*values.type());
-    current_column_->SetTimestamp(ts_type.unit, ts_type.timezone);
+    current_column_->SetTimestamp(ts_type.unit(), ts_type.timezone());
     return Status::OK();
   }
 
@@ -643,7 +643,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
 
   Status Visit(const Time32Array& values) override {
     RETURN_NOT_OK(WritePrimitiveValues(values));
-    auto unit = static_cast<const Time32Type&>(*values.type()).unit;
+    auto unit = static_cast<const Time32Type&>(*values.type()).unit();
     current_column_->SetTime(unit);
     return Status::OK();
   }
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 1a91ec39ca1fc..98a7c3dd58a6b 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -569,13 +569,13 @@ void CheckBatchDictionaries(const RecordBatch& batch) {
   // Check that dictionaries that should be the same are the same
   auto schema = batch.schema();
 
-  const auto& t0 = static_cast<const DictionaryType&>(*schema->field(0)->type);
-  const auto& t1 = static_cast<const DictionaryType&>(*schema->field(1)->type);
+  const auto& t0 = static_cast<const DictionaryType&>(*schema->field(0)->type());
+  const auto& t1 = static_cast<const DictionaryType&>(*schema->field(1)->type());
 
   ASSERT_EQ(t0.dictionary().get(), t1.dictionary().get());
 
   // Same dictionary used for list values
-  const auto& t3 = static_cast<const ListType&>(*schema->field(3)->type);
+  const auto& t3 = static_cast<const ListType&>(*schema->field(3)->type());
   const auto& t3_value = static_cast<const DictionaryType&>(*t3.value_type());
   ASSERT_EQ(t0.dictionary().get(), t3_value.dictionary().get());
 }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index fe0a7c94226f0..18ee8349da66a 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -114,13 +114,13 @@ class JsonSchemaWriter {
     writer_->StartObject();
 
     writer_->Key("name");
-    writer_->String(field.name.c_str());
+    writer_->String(field.name().c_str());
 
     writer_->Key("nullable");
-    writer_->Bool(field.nullable);
+    writer_->Bool(field.nullable());
 
     // Visit the type
-    RETURN_NOT_OK(VisitTypeInline(*field.type, this));
+    RETURN_NOT_OK(VisitTypeInline(*field.type(), this));
     writer_->EndObject();
 
     return Status::OK();
@@ -153,7 +153,7 @@ class JsonSchemaWriter {
 
   void WriteTypeMetadata(const IntervalType& type) {
     writer_->Key("unit");
-    switch (type.unit) {
+    switch (type.unit()) {
       case IntervalType::Unit::YEAR_MONTH:
         writer_->String("YEAR_MONTH");
         break;
@@ -165,23 +165,23 @@ class JsonSchemaWriter {
 
   void WriteTypeMetadata(const TimestampType& type) {
     writer_->Key("unit");
-    writer_->String(GetTimeUnitName(type.unit));
-    if (type.timezone.size() > 0) {
+    writer_->String(GetTimeUnitName(type.unit()));
+    if (type.timezone().size() > 0) {
       writer_->Key("timezone");
-      writer_->String(type.timezone);
+      writer_->String(type.timezone());
     }
   }
 
   void WriteTypeMetadata(const TimeType& type) {
     writer_->Key("unit");
-    writer_->String(GetTimeUnitName(type.unit));
+    writer_->String(GetTimeUnitName(type.unit()));
     writer_->Key("bitWidth");
     writer_->Int(type.bit_width());
   }
 
   void WriteTypeMetadata(const DateType& type) {
     writer_->Key("unit");
-    switch (type.unit) {
+    switch (type.unit()) {
       case DateUnit::DAY:
         writer_->String("DAY");
         break;
@@ -198,14 +198,14 @@ class JsonSchemaWriter {
 
   void WriteTypeMetadata(const DecimalType& type) {
     writer_->Key("precision");
-    writer_->Int(type.precision);
+    writer_->Int(type.precision());
     writer_->Key("scale");
-    writer_->Int(type.scale);
+    writer_->Int(type.scale());
   }
 
   void WriteTypeMetadata(const UnionType& type) {
     writer_->Key("mode");
-    switch (type.mode) {
+    switch (type.mode()) {
       case UnionMode::SPARSE:
         writer_->String("SPARSE");
         break;
@@ -217,8 +217,8 @@ class JsonSchemaWriter {
     // Write type ids
     writer_->Key("typeIds");
     writer_->StartArray();
-    for (size_t i = 0; i < type.type_codes.size(); ++i) {
-      writer_->Uint(type.type_codes[i]);
+    for (size_t i = 0; i < type.type_codes().size(); ++i) {
+      writer_->Uint(type.type_codes()[i]);
     }
     writer_->EndArray();
   }
@@ -461,7 +461,7 @@ class JsonArrayWriter {
     writer_->Key("children");
     writer_->StartArray();
     for (size_t i = 0; i < fields.size(); ++i) {
-      RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
+      RETURN_NOT_OK(VisitArray(fields[i]->name(), *arrays[i].get()));
     }
     writer_->EndArray();
     return Status::OK();
@@ -513,7 +513,7 @@ class JsonArrayWriter {
     auto type = static_cast<const UnionType*>(array.type().get());
 
     WriteIntegerField("TYPE_ID", array.raw_type_ids(), array.length());
-    if (type->mode == UnionMode::DENSE) {
+    if (type->mode() == UnionMode::DENSE) {
       WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length());
     }
     return WriteChildren(type->children(), array.children());
@@ -1026,7 +1026,7 @@ class JsonArrayReader {
     RETURN_NOT_OK(
         GetIntArray<uint8_t>(json_type_ids->value.GetArray(), length, &type_id_buffer));
 
-    if (union_type.mode == UnionMode::DENSE) {
+    if (union_type.mode() == UnionMode::DENSE) {
       const auto& json_offsets = json_array.FindMember("OFFSET");
       RETURN_NOT_ARRAY("OFFSET", json_offsets, json_array);
       RETURN_NOT_OK(
@@ -1072,9 +1072,9 @@ class JsonArrayReader {
       auto it = json_child.FindMember("name");
       RETURN_NOT_STRING("name", it, json_child);
 
-      DCHECK_EQ(it->value.GetString(), child_field->name);
+      DCHECK_EQ(it->value.GetString(), child_field->name());
       std::shared_ptr<Array> child;
-      RETURN_NOT_OK(GetArray(json_children_arr[i], child_field->type, &child));
+      RETURN_NOT_OK(GetArray(json_children_arr[i], child_field->type(), &child));
       array->emplace_back(child);
     }
 
@@ -1109,7 +1109,7 @@ class JsonArrayReader {
   case TYPE::type_id:   \
     return ReadArray<TYPE>(json_array, length, is_valid, type, array);
 
-    switch (type->type) {
+    switch (type->id()) {
       TYPE_CASE(NullType);
       TYPE_CASE(BooleanType);
       TYPE_CASE(UInt8Type);
@@ -1192,7 +1192,7 @@ Status ReadJsonArray(MemoryPool* pool, const rj::Value& json_array, const Schema
 
   std::shared_ptr<Field> result = nullptr;
   for (const std::shared_ptr<Field>& field : schema.fields()) {
-    if (field->name == name) {
+    if (field->name() == name) {
       result = field;
       break;
     }
@@ -1204,7 +1204,7 @@ Status ReadJsonArray(MemoryPool* pool, const rj::Value& json_array, const Schema
     return Status::KeyError(ss.str());
   }
 
-  return ReadJsonArray(pool, json_array, result->type, array);
+  return ReadJsonArray(pool, json_array, result->type(), array);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index 8056b6f3e758e..0abd6d7ffe3df 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -79,7 +79,7 @@ class JsonWriter::JsonWriterImpl {
       DCHECK_EQ(batch.num_rows(), column->length())
           << "Array length did not match record batch length";
 
-      RETURN_NOT_OK(WriteJsonArray(schema_->field(i)->name, *column, writer_.get()));
+      RETURN_NOT_OK(WriteJsonArray(schema_->field(i)->name(), *column, writer_.get()));
     }
 
     writer_->EndArray();
@@ -158,7 +158,7 @@ class JsonReader::JsonReaderImpl {
 
     std::vector<std::shared_ptr<Array>> columns(json_columns.Size());
     for (int i = 0; i < static_cast<int>(columns.size()); ++i) {
-      const std::shared_ptr<DataType>& type = schema_->field(i)->type;
+      const std::shared_ptr<DataType>& type = schema_->field(i)->type();
       RETURN_NOT_OK(ReadJsonArray(pool_, json_columns[i], type, &columns[i]));
     }
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index d902ec296cff3..84f8883ffb949 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -163,13 +163,13 @@ static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 
   const auto& union_type = static_cast<const UnionType&>(*type);
 
-  flatbuf::UnionMode mode = union_type.mode == UnionMode::SPARSE
+  flatbuf::UnionMode mode = union_type.mode() == UnionMode::SPARSE
                                 ? flatbuf::UnionMode_Sparse
                                 : flatbuf::UnionMode_Dense;
 
   std::vector<int32_t> type_ids;
-  type_ids.reserve(union_type.type_codes.size());
-  for (uint8_t code : union_type.type_codes) {
+  type_ids.reserve(union_type.type_codes().size());
+  for (uint8_t code : union_type.type_codes()) {
     type_ids.push_back(code);
   }
 
@@ -306,7 +306,7 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
     flatbuf::Type* out_type, DictionaryMemo* dictionary_memo, Offset* offset) {
-  if (type->type == Type::DICTIONARY) {
+  if (type->id() == Type::DICTIONARY) {
     // In this library, the dictionary "type" is a logical construct. Here we
     // pass through to the value type, as we've already captured the index
     // type in the DictionaryEncoding metadata in the parent field
@@ -340,7 +340,7 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     layout->push_back(offset);
   }
 
-  switch (type->type) {
+  switch (type->id()) {
     case Type::BOOL:
       *out_type = flatbuf::Type_Bool;
       *offset = flatbuf::CreateBool(fbb).Union();
@@ -393,21 +393,21 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     case Type::TIME32: {
       const auto& time_type = static_cast<const Time32Type&>(*type);
       *out_type = flatbuf::Type_Time;
-      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 32).Union();
+      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit()), 32).Union();
     } break;
     case Type::TIME64: {
       const auto& time_type = static_cast<const Time64Type&>(*type);
       *out_type = flatbuf::Type_Time;
-      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit), 64).Union();
+      *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit()), 64).Union();
     } break;
     case Type::TIMESTAMP: {
       const auto& ts_type = static_cast<const TimestampType&>(*type);
       *out_type = flatbuf::Type_Timestamp;
 
-      flatbuf::TimeUnit fb_unit = ToFlatbufferUnit(ts_type.unit);
+      flatbuf::TimeUnit fb_unit = ToFlatbufferUnit(ts_type.unit());
       FBString fb_timezone = 0;
-      if (ts_type.timezone.size() > 0) {
-        fb_timezone = fbb.CreateString(ts_type.timezone);
+      if (ts_type.timezone().size() > 0) {
+        fb_timezone = fbb.CreateString(ts_type.timezone());
       }
       *offset = flatbuf::CreateTimestamp(fbb, fb_unit, fb_timezone).Union();
     } break;
@@ -431,7 +431,7 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 
 static Status TensorTypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     flatbuf::Type* out_type, Offset* offset) {
-  switch (type->type) {
+  switch (type->id()) {
     case Type::UINT8:
       INT_TO_FB_CASE(8, false);
     case Type::INT8:
@@ -486,7 +486,7 @@ static DictionaryOffset GetDictionaryEncoding(
 
 static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
     DictionaryMemo* dictionary_memo, FieldOffset* offset) {
-  auto fb_name = fbb.CreateString(field->name);
+  auto fb_name = fbb.CreateString(field->name());
 
   flatbuf::Type type_enum;
   Offset type_offset;
@@ -495,18 +495,18 @@ static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
   std::vector<VectorLayoutOffset> layout;
 
   RETURN_NOT_OK(TypeToFlatbuffer(
-      fbb, field->type, &children, &layout, &type_enum, dictionary_memo, &type_offset));
+      fbb, field->type(), &children, &layout, &type_enum, dictionary_memo, &type_offset));
   auto fb_children = fbb.CreateVector(children);
   auto fb_layout = fbb.CreateVector(layout);
 
   DictionaryOffset dictionary = 0;
-  if (field->type->type == Type::DICTIONARY) {
+  if (field->type()->id() == Type::DICTIONARY) {
     dictionary = GetDictionaryEncoding(
-        fbb, static_cast<const DictionaryType&>(*field->type), dictionary_memo);
+        fbb, static_cast<const DictionaryType&>(*field->type()), dictionary_memo);
   }
 
   // TODO: produce the list of VectorTypes
-  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_offset,
+  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable(), type_enum, type_offset,
       dictionary, fb_children, fb_layout);
 
   return Status::OK();
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index b042882c7cd31..84026c452ad27 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -34,8 +34,8 @@ namespace arrow {
 
 class Array;
 class Buffer;
-struct DataType;
-struct Field;
+class DataType;
+class Field;
 class Schema;
 class Status;
 class Tensor;
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index a7c4f04a4d4cc..69fde1783d7d3 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -97,7 +97,7 @@ static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
   context.max_recursion_depth = max_recursion_depth;
 
   for (int i = 0; i < schema->num_fields(); ++i) {
-    RETURN_NOT_OK(LoadArray(schema->field(i)->type, &context, &arrays[i]));
+    RETURN_NOT_OK(LoadArray(schema->field(i)->type(), &context, &arrays[i]));
     DCHECK_EQ(num_rows, arrays[i]->length())
         << "Array length did not match record batch length";
   }
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 9e0480d4c3634..a17b609bbcba2 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -552,9 +552,9 @@ Status MakeTimestamps(std::shared_ptr<RecordBatch>* out) {
       1489272000000, 1489272000000, 1489273000000};
 
   std::shared_ptr<Array> a0, a1, a2;
-  ArrayFromVector<TimestampType, int64_t>(f0->type, is_valid, ts_values, &a0);
-  ArrayFromVector<TimestampType, int64_t>(f1->type, is_valid, ts_values, &a1);
-  ArrayFromVector<TimestampType, int64_t>(f2->type, is_valid, ts_values, &a2);
+  ArrayFromVector<TimestampType, int64_t>(f0->type(), is_valid, ts_values, &a0);
+  ArrayFromVector<TimestampType, int64_t>(f1->type(), is_valid, ts_values, &a1);
+  ArrayFromVector<TimestampType, int64_t>(f2->type(), is_valid, ts_values, &a2);
 
   ArrayVector arrays = {a0, a1, a2};
   *out = std::make_shared<RecordBatch>(schema, a0->length(), arrays);
@@ -575,10 +575,10 @@ Status MakeTimes(std::shared_ptr<RecordBatch>* out) {
       1489272000000, 1489272000000, 1489273000000};
 
   std::shared_ptr<Array> a0, a1, a2, a3;
-  ArrayFromVector<Time32Type, int32_t>(f0->type, is_valid, t32_values, &a0);
-  ArrayFromVector<Time64Type, int64_t>(f1->type, is_valid, t64_values, &a1);
-  ArrayFromVector<Time32Type, int32_t>(f2->type, is_valid, t32_values, &a2);
-  ArrayFromVector<Time64Type, int64_t>(f3->type, is_valid, t64_values, &a3);
+  ArrayFromVector<Time32Type, int32_t>(f0->type(), is_valid, t32_values, &a0);
+  ArrayFromVector<Time64Type, int64_t>(f1->type(), is_valid, t64_values, &a1);
+  ArrayFromVector<Time32Type, int32_t>(f2->type(), is_valid, t32_values, &a2);
+  ArrayFromVector<Time64Type, int64_t>(f3->type(), is_valid, t64_values, &a3);
 
   ArrayVector arrays = {a0, a1, a2, a3};
   *out = std::make_shared<RecordBatch>(schema, a0->length(), arrays);
@@ -605,8 +605,8 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
 
   std::shared_ptr<Array> a1, a2;
 
-  FixedSizeBinaryBuilder b1(default_memory_pool(), f0->type);
-  FixedSizeBinaryBuilder b2(default_memory_pool(), f1->type);
+  FixedSizeBinaryBuilder b1(default_memory_pool(), f0->type());
+  FixedSizeBinaryBuilder b2(default_memory_pool(), f1->type());
 
   std::vector<std::string> values1 = {"foo1", "foo2", "foo3", "foo4"};
   AppendValues(is_valid, values1, &b1);
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index d38a65c983d98..18a585599a31b 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -364,7 +364,7 @@ class RecordBatchWriter : public ArrayVisitor {
 
       // The Union type codes are not necessary 0-indexed
       uint8_t max_code = 0;
-      for (uint8_t code : type.type_codes) {
+      for (uint8_t code : type.type_codes()) {
         if (code > max_code) { max_code = code; }
       }
 
@@ -406,7 +406,7 @@ class RecordBatchWriter : public ArrayVisitor {
       for (int i = 0; i < type.num_children(); ++i) {
         std::shared_ptr<Array> child = array.child(i);
         if (array.offset() != 0) {
-          const uint8_t code = type.type_codes[i];
+          const uint8_t code = type.type_codes()[i];
           child = child->Slice(child_offsets[code], child_lengths[code]);
         }
         RETURN_NOT_OK(VisitArray(*child));
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 0b7a6e1b56be5..629bcb9c6c980 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -31,7 +31,7 @@ namespace arrow {
 
 class Array;
 class Buffer;
-struct Field;
+class Field;
 class MemoryPool;
 class RecordBatch;
 class Schema;
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
index f3347f92e6d87..f9f6e6fcac826 100644
--- a/cpp/src/arrow/loader.cc
+++ b/cpp/src/arrow/loader.cc
@@ -24,6 +24,7 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
+#include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/logging.h"
@@ -32,10 +33,6 @@
 
 namespace arrow {
 
-class Array;
-struct DataType;
-class Status;
-
 class ArrayLoader {
  public:
   ArrayLoader(const std::shared_ptr<DataType>& type, ArrayLoaderContext* context)
@@ -114,7 +111,7 @@ class ArrayLoader {
   }
 
   Status LoadChild(const Field& field, std::shared_ptr<Array>* out) {
-    ArrayLoader loader(field.type, context_);
+    ArrayLoader loader(field.type(), context_);
     --context_->max_recursion_depth;
     RETURN_NOT_OK(loader.Load(out));
     ++context_->max_recursion_depth;
@@ -211,11 +208,11 @@ class ArrayLoader {
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
     if (field_meta.length > 0) {
       RETURN_NOT_OK(GetBuffer(context_->buffer_index, &type_ids));
-      if (type.mode == UnionMode::DENSE) {
+      if (type.mode() == UnionMode::DENSE) {
         RETURN_NOT_OK(GetBuffer(context_->buffer_index + 1, &offsets));
       }
     }
-    context_->buffer_index += type.mode == UnionMode::DENSE ? 2 : 1;
+    context_->buffer_index += type.mode() == UnionMode::DENSE ? 2 : 1;
 
     std::vector<std::shared_ptr<Array>> fields;
     RETURN_NOT_OK(LoadChildren(type.children(), &fields));
diff --git a/cpp/src/arrow/loader.h b/cpp/src/arrow/loader.h
index 9b650e2da7426..f5e399537fd7b 100644
--- a/cpp/src/arrow/loader.h
+++ b/cpp/src/arrow/loader.h
@@ -33,7 +33,7 @@ namespace arrow {
 
 class Array;
 class Buffer;
-struct DataType;
+class DataType;
 
 // ARROW-109: We set this number arbitrarily to help catch user mistakes. For
 // deeply nested schemas, it is expected the user will indicate explicitly the
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 1ae13f3db061c..8cc9876fa9fc5 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -571,7 +571,7 @@ class DecimalConverter : public TypedConverter<arrow::DecimalBuilder> {
 
 // Dynamic constructor for sequence converters
 std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type) {
-  switch (type->type) {
+  switch (type->id()) {
     case Type::BOOL:
       return std::make_shared<BoolConverter>();
     case Type::INT64:
@@ -637,7 +637,7 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
     const std::shared_ptr<DataType>& type, int64_t size) {
   // Handle NA / NullType case
-  if (type->type == Type::NA) {
+  if (type->id() == Type::NA) {
     out->reset(new NullArray(size));
     return Status::OK();
   }
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 23470fbc41aca..ab79e179c7ea5 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -118,7 +118,7 @@ Status GetNumPyType(const DataType& type, int* type_num) {
     *type_num = NPY_##NPY_NAME;               \
     break;
 
-  switch (type.type) {
+  switch (type.id()) {
     NUMPY_TYPE_CASE(UINT8, UINT8);
     NUMPY_TYPE_CASE(INT8, INT8);
     NUMPY_TYPE_CASE(UINT16, UINT16);
diff --git a/cpp/src/arrow/python/numpy_convert.h b/cpp/src/arrow/python/numpy_convert.h
index 685a626d4ca28..c2526403213b1 100644
--- a/cpp/src/arrow/python/numpy_convert.h
+++ b/cpp/src/arrow/python/numpy_convert.h
@@ -31,7 +31,7 @@
 
 namespace arrow {
 
-struct DataType;
+class DataType;
 class MemoryPool;
 class Status;
 class Tensor;
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 1a250e83c5093..643c5fb8796a0 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -669,8 +669,8 @@ static Status ConvertDecimals(const ChunkedArray& data, PyObject** out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     auto* arr(static_cast<arrow::DecimalArray*>(data.chunk(c).get()));
     auto type(std::dynamic_pointer_cast<arrow::DecimalType>(arr->type()));
-    const int precision = type->precision;
-    const int scale = type->scale;
+    const int precision = type->precision();
+    const int scale = type->scale();
     const int bit_width = type->bit_width();
 
     for (int64_t i = 0; i < arr->length(); ++i) {
@@ -764,7 +764,7 @@ Status PandasConverter::ConvertObjects() {
 
   // This means we received an explicit type from the user
   if (type_) {
-    switch (type_->type) {
+    switch (type_->id()) {
       case Type::STRING:
         return ConvertObjectStrings();
       case Type::FIXED_SIZE_BINARY:
@@ -777,7 +777,7 @@ Status PandasConverter::ConvertObjects() {
         return ConvertDates<Date64Type>();
       case Type::LIST: {
         const auto& list_field = static_cast<const ListType&>(*type_);
-        return ConvertLists(list_field.value_field()->type);
+        return ConvertLists(list_field.value_field()->type());
       }
       case Type::DECIMAL:
         return ConvertDecimals();
@@ -860,7 +860,7 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(list_builder.Append(true));
       RETURN_NOT_OK(InferArrowTypeAndSize(objects[i], &size, &inferred_type));
-      if (inferred_type->type != type->type) {
+      if (inferred_type->id() != type->id()) {
         std::stringstream ss;
         ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
         return Status::TypeError(ss.str());
@@ -909,7 +909,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(list_builder.Append(true));
       RETURN_NOT_OK(InferArrowTypeAndSize(objects[i], &size, &inferred_type));
-      if (inferred_type->type != Type::STRING) {
+      if (inferred_type->id() != Type::STRING) {
         std::stringstream ss;
         ss << inferred_type->ToString() << " cannot be converted to STRING.";
         return Status::TypeError(ss.str());
@@ -928,7 +928,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
   }
 
 Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
-  switch (type->type) {
+  switch (type->id()) {
     LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
     LIST_CASE(INT8, NPY_INT8, Int8Type)
     LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
@@ -1300,7 +1300,7 @@ class ObjectBlock : public PandasBlock {
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
       int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+    Type::type type = col->type()->id();
 
     PyObject** out_buffer =
         reinterpret_cast<PyObject**>(block_data_) + rel_placement * num_rows_;
@@ -1319,7 +1319,7 @@ class ObjectBlock : public PandasBlock {
       RETURN_NOT_OK(ConvertDecimals(data, out_buffer));
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
-      switch (list_type->value_type()->type) {
+      switch (list_type->value_type()->id()) {
         CONVERTLISTSLIKE_CASE(UInt8Type, UINT8)
         CONVERTLISTSLIKE_CASE(Int8Type, INT8)
         CONVERTLISTSLIKE_CASE(UInt16Type, UINT16)
@@ -1360,7 +1360,7 @@ class IntBlock : public PandasBlock {
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
       int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+    Type::type type = col->type()->id();
 
     C_TYPE* out_buffer =
         reinterpret_cast<C_TYPE*>(block_data_) + rel_placement * num_rows_;
@@ -1392,7 +1392,7 @@ class Float32Block : public PandasBlock {
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
       int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+    Type::type type = col->type()->id();
 
     if (type != Type::FLOAT) { return Status::NotImplemented(col->type()->ToString()); }
 
@@ -1412,7 +1412,7 @@ class Float64Block : public PandasBlock {
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
       int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+    Type::type type = col->type()->id();
 
     double* out_buffer =
         reinterpret_cast<double*>(block_data_) + rel_placement * num_rows_;
@@ -1465,7 +1465,7 @@ class BoolBlock : public PandasBlock {
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
       int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+    Type::type type = col->type()->id();
 
     if (type != Type::BOOL) { return Status::NotImplemented(col->type()->ToString()); }
 
@@ -1496,7 +1496,7 @@ class DatetimeBlock : public PandasBlock {
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
       int64_t rel_placement) override {
-    Type::type type = col->type()->type;
+    Type::type type = col->type()->id();
 
     int64_t* out_buffer =
         reinterpret_cast<int64_t*>(block_data_) + rel_placement * num_rows_;
@@ -1514,13 +1514,13 @@ class DatetimeBlock : public PandasBlock {
     } else if (type == Type::TIMESTAMP) {
       auto ts_type = static_cast<TimestampType*>(col->type().get());
 
-      if (ts_type->unit == TimeUnit::NANO) {
+      if (ts_type->unit() == TimeUnit::NANO) {
         ConvertNumericNullable<int64_t>(data, kPandasTimestampNull, out_buffer);
-      } else if (ts_type->unit == TimeUnit::MICRO) {
+      } else if (ts_type->unit() == TimeUnit::MICRO) {
         ConvertDatetimeNanos<int64_t, 1000L>(data, out_buffer);
-      } else if (ts_type->unit == TimeUnit::MILLI) {
+      } else if (ts_type->unit() == TimeUnit::MILLI) {
         ConvertDatetimeNanos<int64_t, 1000000L>(data, out_buffer);
-      } else if (ts_type->unit == TimeUnit::SECOND) {
+      } else if (ts_type->unit() == TimeUnit::SECOND) {
         ConvertDatetimeNanos<int64_t, 1000000000L>(data, out_buffer);
       } else {
         return Status::NotImplemented("Unsupported time unit");
@@ -1661,7 +1661,7 @@ static inline Status MakeCategoricalBlock(const std::shared_ptr<DataType>& type,
     int64_t num_rows, std::shared_ptr<PandasBlock>* block) {
   // All categoricals become a block with a single column
   auto dict_type = static_cast<const DictionaryType*>(type.get());
-  switch (dict_type->index_type()->type) {
+  switch (dict_type->index_type()->id()) {
     case Type::INT8:
       *block = std::make_shared<CategoricalBlock<Type::INT8>>(num_rows);
       break;
@@ -1714,7 +1714,7 @@ class DataFrameBlockCreator {
       std::shared_ptr<Column> col = table_->column(i);
       PandasBlock::type output_type;
 
-      Type::type column_type = col->type()->type;
+      Type::type column_type = col->type()->id();
       switch (column_type) {
         case Type::BOOL:
           output_type = col->null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL;
@@ -1762,7 +1762,7 @@ class DataFrameBlockCreator {
           break;
         case Type::TIMESTAMP: {
           const auto& ts_type = static_cast<const TimestampType&>(*col->type());
-          if (ts_type.timezone != "") {
+          if (ts_type.timezone() != "") {
             output_type = PandasBlock::DATETIME_WITH_TZ;
           } else {
             output_type = PandasBlock::DATETIME;
@@ -1770,7 +1770,7 @@ class DataFrameBlockCreator {
         } break;
         case Type::LIST: {
           auto list_type = std::static_pointer_cast<ListType>(col->type());
-          if (!ListTypeSupported(list_type->value_type()->type)) {
+          if (!ListTypeSupported(list_type->value_type()->id())) {
             std::stringstream ss;
             ss << "Not implemented type for lists: "
                << list_type->value_type()->ToString();
@@ -1795,7 +1795,7 @@ class DataFrameBlockCreator {
         categorical_blocks_[i] = block;
       } else if (output_type == PandasBlock::DATETIME_WITH_TZ) {
         const auto& ts_type = static_cast<const TimestampType&>(*col->type());
-        block = std::make_shared<DatetimeTZBlock>(ts_type.timezone, table_->num_rows());
+        block = std::make_shared<DatetimeTZBlock>(ts_type.timezone(), table_->num_rows());
         RETURN_NOT_OK(block->Allocate());
         datetimetz_blocks_[i] = block;
       } else {
@@ -1942,10 +1942,10 @@ inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out)
   if (type == NPY_DATETIME) {
     PyArray_Descr* descr = PyArray_DESCR(out);
     auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-    if (datatype->type == Type::TIMESTAMP) {
+    if (datatype->id() == Type::TIMESTAMP) {
       auto timestamp_type = static_cast<TimestampType*>(datatype);
 
-      switch (timestamp_type->unit) {
+      switch (timestamp_type->unit()) {
         case TimestampType::Unit::SECOND:
           date_dtype->meta.base = NPY_FR_s;
           break;
@@ -2154,7 +2154,7 @@ class ArrowDeserializer {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     auto list_type = std::static_pointer_cast<ListType>(col_->type());
-    switch (list_type->value_type()->type) {
+    switch (list_type->value_type()->id()) {
       CONVERTVALUES_LISTSLIKE_CASE(UInt8Type, UINT8)
       CONVERTVALUES_LISTSLIKE_CASE(Int8Type, INT8)
       CONVERTVALUES_LISTSLIKE_CASE(UInt16Type, UINT16)
diff --git a/cpp/src/arrow/python/pandas_convert.h b/cpp/src/arrow/python/pandas_convert.h
index 4d32c8b86cf50..fd901d8f09fce 100644
--- a/cpp/src/arrow/python/pandas_convert.h
+++ b/cpp/src/arrow/python/pandas_convert.h
@@ -32,7 +32,7 @@ namespace arrow {
 
 class Array;
 class Column;
-struct DataType;
+class DataType;
 class MemoryPool;
 class Status;
 class Table;
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index cdc0238cf4ab8..0da4c0f9641a3 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -244,8 +244,8 @@ TEST_F(TestTable, Metadata) {
   ASSERT_TRUE(table_->schema()->Equals(*schema_));
 
   auto col = table_->column(0);
-  ASSERT_EQ(schema_->field(0)->name, col->name());
-  ASSERT_EQ(schema_->field(0)->type, col->type());
+  ASSERT_EQ(schema_->field(0)->name(), col->name());
+  ASSERT_EQ(schema_->field(0)->type(), col->type());
 }
 
 TEST_F(TestTable, InvalidColumns) {
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 4c5257b92c033..eabd98bda1893 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -153,7 +153,7 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
     : schema_(schema), num_rows_(num_rows), columns_(std::move(columns)) {}
 
 const std::string& RecordBatch::column_name(int i) const {
-  return schema_->field(i)->name;
+  return schema_->field(i)->name();
 }
 
 bool RecordBatch::Equals(const RecordBatch& other) const {
@@ -204,7 +204,7 @@ Status RecordBatch::Validate() const {
          << " vs " << num_rows_;
       return Status::Invalid(ss.str());
     }
-    const auto& schema_type = *schema_->field(i)->type;
+    const auto& schema_type = *schema_->field(i)->type();
     if (!arr.type()->Equals(schema_type)) {
       std::stringstream ss;
       ss << "Column " << i << " type not match schema: " << arr.type()->ToString()
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index b15d31b23a872..cfd1f366b039f 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -81,10 +81,10 @@ class ARROW_EXPORT Column {
   std::shared_ptr<Field> field() const { return field_; }
 
   // @returns: the column's name in the passed metadata
-  const std::string& name() const { return field_->name; }
+  const std::string& name() const { return field_->name(); }
 
   // @returns: the column's type according to the metadata
-  std::shared_ptr<DataType> type() const { return field_->type; }
+  std::shared_ptr<DataType> type() const { return field_->type(); }
 
   // @returns: the column's data as a chunked logical array
   std::shared_ptr<ChunkedArray> data() const { return data_; }
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index d1c4083289f96..fa3e203c998ba 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -61,7 +61,7 @@ Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buff
     const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
     const std::vector<std::string>& dim_names)
     : type_(type), data_(data), shape_(shape), strides_(strides), dim_names_(dim_names) {
-  DCHECK(is_tensor_supported(type->type));
+  DCHECK(is_tensor_supported(type->id()));
   if (shape.size() > 0 && strides.size() == 0) {
     ComputeRowMajorStrides(static_cast<const FixedWidthType&>(*type_), shape, &strides_);
   }
@@ -107,6 +107,10 @@ bool Tensor::is_column_major() const {
   return strides_ == f_strides;
 }
 
+Type::type Tensor::type_id() const {
+  return type_->id();
+}
+
 bool Tensor::Equals(const Tensor& other) const {
   bool are_equal = false;
   Status error = TensorEquals(*this, other, &are_equal);
@@ -161,7 +165,7 @@ Status ARROW_EXPORT MakeTensor(const std::shared_ptr<DataType>& type,
     const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
     const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names,
     std::shared_ptr<Tensor>* tensor) {
-  switch (type->type) {
+  switch (type->id()) {
     TENSOR_CASE(INT8, Int8Tensor);
     TENSOR_CASE(INT16, Int16Tensor);
     TENSOR_CASE(INT32, Int32Tensor);
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index 12015f14b1d3d..7741c305f870d 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -98,7 +98,7 @@ class ARROW_EXPORT Tensor {
   /// AKA "Fortran order"
   bool is_column_major() const;
 
-  Type::type type_enum() const { return type_->type; }
+  Type::type type_id() const;
 
   bool Equals(const Tensor& other) const;
 
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 66164e3430913..dec7268a5a8b5 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -34,11 +34,11 @@ TEST(TestField, Basics) {
   Field f0("f0", int32());
   Field f0_nn("f0", int32(), false);
 
-  ASSERT_EQ(f0.name, "f0");
-  ASSERT_EQ(f0.type->ToString(), int32()->ToString());
+  ASSERT_EQ(f0.name(), "f0");
+  ASSERT_EQ(f0.type()->ToString(), int32()->ToString());
 
-  ASSERT_TRUE(f0.nullable);
-  ASSERT_FALSE(f0_nn.nullable);
+  ASSERT_TRUE(f0.nullable());
+  ASSERT_FALSE(f0_nn.nullable());
 }
 
 TEST(TestField, Equals) {
@@ -121,7 +121,7 @@ TEST_F(TestSchema, GetFieldByName) {
   TEST(TypesTest, TestPrimitive_##ENUM) {        \
     KLASS tp;                                    \
                                                  \
-    ASSERT_EQ(tp.type, Type::ENUM);              \
+    ASSERT_EQ(tp.id(), Type::ENUM);              \
     ASSERT_EQ(tp.ToString(), std::string(NAME)); \
   }
 
@@ -145,19 +145,19 @@ TEST(TestBinaryType, ToString) {
   StringType t2;
   EXPECT_TRUE(t1.Equals(e1));
   EXPECT_FALSE(t1.Equals(t2));
-  ASSERT_EQ(t1.type, Type::BINARY);
+  ASSERT_EQ(t1.id(), Type::BINARY);
   ASSERT_EQ(t1.ToString(), std::string("binary"));
 }
 
 TEST(TestStringType, ToString) {
   StringType str;
-  ASSERT_EQ(str.type, Type::STRING);
+  ASSERT_EQ(str.id(), Type::STRING);
   ASSERT_EQ(str.ToString(), std::string("string"));
 }
 
 TEST(TestFixedSizeBinaryType, ToString) {
   auto t = fixed_size_binary(10);
-  ASSERT_EQ(t->type, Type::FIXED_SIZE_BINARY);
+  ASSERT_EQ(t->id(), Type::FIXED_SIZE_BINARY);
   ASSERT_EQ("fixed_size_binary[10]", t->ToString());
 }
 
@@ -175,13 +175,13 @@ TEST(TestListType, Basics) {
   std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
 
   ListType list_type(vt);
-  ASSERT_EQ(list_type.type, Type::LIST);
+  ASSERT_EQ(list_type.id(), Type::LIST);
 
   ASSERT_EQ("list", list_type.name());
   ASSERT_EQ("list<item: uint8>", list_type.ToString());
 
-  ASSERT_EQ(list_type.value_type()->type, vt->type);
-  ASSERT_EQ(list_type.value_type()->type, vt->type);
+  ASSERT_EQ(list_type.value_type()->id(), vt->id());
+  ASSERT_EQ(list_type.value_type()->id(), vt->id());
 
   std::shared_ptr<DataType> st = std::make_shared<StringType>();
   std::shared_ptr<DataType> lt = std::make_shared<ListType>(st);
@@ -315,4 +315,46 @@ TEST(TestStructType, Basics) {
   // TODO(wesm): out of bounds for field(...)
 }
 
+TEST(TypesTest, TestDecimal32Type) {
+  DecimalType t1(8, 4);
+
+  ASSERT_EQ(t1.id(), Type::DECIMAL);
+  ASSERT_EQ(t1.precision(), 8);
+  ASSERT_EQ(t1.scale(), 4);
+
+  ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
+
+  // Test properties
+  ASSERT_EQ(t1.byte_width(), 4);
+  ASSERT_EQ(t1.bit_width(), 32);
+}
+
+TEST(TypesTest, TestDecimal64Type) {
+  DecimalType t1(12, 5);
+
+  ASSERT_EQ(t1.id(), Type::DECIMAL);
+  ASSERT_EQ(t1.precision(), 12);
+  ASSERT_EQ(t1.scale(), 5);
+
+  ASSERT_EQ(t1.ToString(), std::string("decimal(12, 5)"));
+
+  // Test properties
+  ASSERT_EQ(t1.byte_width(), 8);
+  ASSERT_EQ(t1.bit_width(), 64);
+}
+
+TEST(TypesTest, TestDecimal128Type) {
+  DecimalType t1(27, 7);
+
+  ASSERT_EQ(t1.id(), Type::DECIMAL);
+  ASSERT_EQ(t1.precision(), 27);
+  ASSERT_EQ(t1.scale(), 7);
+
+  ASSERT_EQ(t1.ToString(), std::string("decimal(27, 7)"));
+
+  // Test properties
+  ASSERT_EQ(t1.byte_width(), 16);
+  ASSERT_EQ(t1.bit_width(), 128);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 93cab14d797c3..a2300d6029e39 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -32,8 +32,8 @@ namespace arrow {
 
 bool Field::Equals(const Field& other) const {
   return (this == &other) ||
-         (this->name == other.name && this->nullable == other.nullable &&
-             this->type->Equals(*other.type.get()));
+         (this->name_ == other.name_ && this->nullable_ == other.nullable_ &&
+             this->type_->Equals(*other.type_.get()));
 }
 
 bool Field::Equals(const std::shared_ptr<Field>& other) const {
@@ -42,8 +42,8 @@ bool Field::Equals(const std::shared_ptr<Field>& other) const {
 
 std::string Field::ToString() const {
   std::stringstream ss;
-  ss << this->name << ": " << this->type->ToString();
-  if (!this->nullable) { ss << " not null"; }
+  ss << this->name_ << ": " << this->type_->ToString();
+  if (!this->nullable_) { ss << " not null"; }
   return ss.str();
 }
 
@@ -107,7 +107,7 @@ std::string StructType::ToString() const {
   for (int i = 0; i < this->num_children(); ++i) {
     if (i > 0) { s << ", "; }
     std::shared_ptr<Field> field = this->child(i);
-    s << field->name << ": " << field->type->ToString();
+    s << field->name() << ": " << field->type()->ToString();
   }
   s << ">";
   return s.str();
@@ -117,7 +117,7 @@ std::string StructType::ToString() const {
 // Date types
 
 DateType::DateType(Type::type type_id, DateUnit unit)
-    : FixedWidthType(type_id), unit(unit) {}
+    : FixedWidthType(type_id), unit_(unit) {}
 
 Date32Type::Date32Type() : DateType(Type::DATE32, DateUnit::DAY) {}
 
@@ -135,7 +135,7 @@ std::string Date32Type::ToString() const {
 // Time types
 
 TimeType::TimeType(Type::type type_id, TimeUnit unit)
-    : FixedWidthType(type_id), unit(unit) {}
+    : FixedWidthType(type_id), unit_(unit) {}
 
 Time32Type::Time32Type(TimeUnit unit) : TimeType(Type::TIME32, unit) {
   DCHECK(unit == TimeUnit::SECOND || unit == TimeUnit::MILLI)
@@ -144,7 +144,7 @@ Time32Type::Time32Type(TimeUnit unit) : TimeType(Type::TIME32, unit) {
 
 std::string Time32Type::ToString() const {
   std::stringstream ss;
-  ss << "time32[" << this->unit << "]";
+  ss << "time32[" << this->unit_ << "]";
   return ss.str();
 }
 
@@ -155,7 +155,7 @@ Time64Type::Time64Type(TimeUnit unit) : TimeType(Type::TIME64, unit) {
 
 std::string Time64Type::ToString() const {
   std::stringstream ss;
-  ss << "time64[" << this->unit << "]";
+  ss << "time64[" << this->unit_ << "]";
   return ss.str();
 }
 
@@ -164,8 +164,8 @@ std::string Time64Type::ToString() const {
 
 std::string TimestampType::ToString() const {
   std::stringstream ss;
-  ss << "timestamp[" << this->unit;
-  if (this->timezone.size() > 0) { ss << ", tz=" << this->timezone; }
+  ss << "timestamp[" << this->unit_;
+  if (this->timezone_.size() > 0) { ss << ", tz=" << this->timezone_; }
   ss << "]";
   return ss.str();
 }
@@ -175,14 +175,14 @@ std::string TimestampType::ToString() const {
 
 UnionType::UnionType(const std::vector<std::shared_ptr<Field>>& fields,
     const std::vector<uint8_t>& type_codes, UnionMode mode)
-    : NestedType(Type::UNION), mode(mode), type_codes(type_codes) {
+    : NestedType(Type::UNION), mode_(mode), type_codes_(type_codes) {
   children_ = fields;
 }
 
 std::string UnionType::ToString() const {
   std::stringstream s;
 
-  if (mode == UnionMode::SPARSE) {
+  if (mode_ == UnionMode::SPARSE) {
     s << "union[sparse]<";
   } else {
     s << "union[dense]<";
@@ -190,7 +190,7 @@ std::string UnionType::ToString() const {
 
   for (size_t i = 0; i < children_.size(); ++i) {
     if (i) { s << ", "; }
-    s << children_[i]->ToString() << "=" << static_cast<int>(type_codes[i]);
+    s << children_[i]->ToString() << "=" << static_cast<int>(type_codes_[i]);
   }
   s << ">";
   return s.str();
@@ -246,7 +246,7 @@ bool Schema::Equals(const Schema& other) const {
 std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
   if (fields_.size() > 0 && name_to_index_.size() == 0) {
     for (size_t i = 0; i < fields_.size(); ++i) {
-      name_to_index_[fields_[i]->name] = static_cast<int>(i);
+      name_to_index_[fields_[i]->name()] = static_cast<int>(i);
     }
   }
 
@@ -423,7 +423,7 @@ std::vector<BufferDescr> StructType::GetBufferLayout() const {
 }
 
 std::vector<BufferDescr> UnionType::GetBufferLayout() const {
-  if (mode == UnionMode::SPARSE) {
+  if (mode_ == UnionMode::SPARSE) {
     return {kValidityBuffer, kTypeBuffer};
   } else {
     return {kValidityBuffer, kTypeBuffer, kOffsetBuffer};
@@ -432,7 +432,7 @@ std::vector<BufferDescr> UnionType::GetBufferLayout() const {
 
 std::string DecimalType::ToString() const {
   std::stringstream s;
-  s << "decimal(" << precision << ", " << scale << ")";
+  s << "decimal(" << precision_ << ", " << scale_ << ")";
   return s.str();
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 730cbed8f4d67..6810b35f05b70 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -127,13 +127,9 @@ class BufferDescr {
   int bit_width_;
 };
 
-struct ARROW_EXPORT DataType {
-  Type::type type;
-
-  std::vector<std::shared_ptr<Field>> children_;
-
-  explicit DataType(Type::type type) : type(type) {}
-
+class ARROW_EXPORT DataType {
+ public:
+  explicit DataType(Type::type id) : id_(id) {}
   virtual ~DataType();
 
   // Return whether the types are equal
@@ -155,13 +151,20 @@ struct ARROW_EXPORT DataType {
 
   virtual std::vector<BufferDescr> GetBufferLayout() const = 0;
 
+  Type::type id() const { return id_; }
+
+ protected:
+  Type::type id_;
+  std::vector<std::shared_ptr<Field>> children_;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(DataType);
 };
 
 typedef std::shared_ptr<DataType> TypePtr;
 
-struct ARROW_EXPORT FixedWidthType : public DataType {
+class ARROW_EXPORT FixedWidthType : public DataType {
+ public:
   using DataType::DataType;
 
   virtual int bit_width() const = 0;
@@ -169,53 +172,64 @@ struct ARROW_EXPORT FixedWidthType : public DataType {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-struct ARROW_EXPORT PrimitiveCType : public FixedWidthType {
+class ARROW_EXPORT PrimitiveCType : public FixedWidthType {
+ public:
   using FixedWidthType::FixedWidthType;
 };
 
-struct ARROW_EXPORT Integer : public PrimitiveCType {
+class ARROW_EXPORT Integer : public PrimitiveCType {
+ public:
   using PrimitiveCType::PrimitiveCType;
   virtual bool is_signed() const = 0;
 };
 
-struct ARROW_EXPORT FloatingPoint : public PrimitiveCType {
+class ARROW_EXPORT FloatingPoint : public PrimitiveCType {
+ public:
   using PrimitiveCType::PrimitiveCType;
   enum Precision { HALF, SINGLE, DOUBLE };
   virtual Precision precision() const = 0;
 };
 
-struct ARROW_EXPORT NestedType : public DataType {
+class ARROW_EXPORT NestedType : public DataType {
+ public:
   using DataType::DataType;
 };
 
-struct NoExtraMeta {};
+class NoExtraMeta {};
 
 // A field is a piece of metadata that includes (for now) a name and a data
 // type
-struct ARROW_EXPORT Field {
-  // Field name
-  std::string name;
-
-  // The field's data type
-  std::shared_ptr<DataType> type;
-
-  // Fields can be nullable
-  bool nullable;
-
+class ARROW_EXPORT Field {
+ public:
   Field(const std::string& name, const std::shared_ptr<DataType>& type,
       bool nullable = true)
-      : name(name), type(type), nullable(nullable) {}
+      : name_(name), type_(type), nullable_(nullable) {}
 
   bool Equals(const Field& other) const;
   bool Equals(const std::shared_ptr<Field>& other) const;
 
   std::string ToString() const;
+
+  const std::string& name() const { return name_; }
+  std::shared_ptr<DataType> type() const { return type_; }
+  bool nullable() const { return nullable_; }
+
+ private:
+  // Field name
+  std::string name_;
+
+  // The field's data type
+  std::shared_ptr<DataType> type_;
+
+  // Fields can be nullable
+  bool nullable_;
 };
 
 typedef std::shared_ptr<Field> FieldPtr;
 
 template <typename DERIVED, typename BASE, Type::type TYPE_ID, typename C_TYPE>
-struct ARROW_EXPORT CTypeImpl : public BASE {
+class ARROW_EXPORT CTypeImpl : public BASE {
+ public:
   using c_type = C_TYPE;
   static constexpr Type::type type_id = TYPE_ID;
 
@@ -230,7 +244,8 @@ struct ARROW_EXPORT CTypeImpl : public BASE {
   std::string ToString() const override { return std::string(DERIVED::name()); }
 };
 
-struct ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
+class ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
+ public:
   static constexpr Type::type type_id = Type::NA;
 
   NullType() : DataType(Type::NA) {}
@@ -244,11 +259,12 @@ struct ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
 };
 
 template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
-struct IntegerTypeImpl : public CTypeImpl<DERIVED, Integer, TYPE_ID, C_TYPE> {
+class IntegerTypeImpl : public CTypeImpl<DERIVED, Integer, TYPE_ID, C_TYPE> {
   bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
 };
 
-struct ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
+class ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
+ public:
   static constexpr Type::type type_id = Type::BOOL;
 
   BooleanType() : FixedWidthType(Type::BOOL) {}
@@ -260,60 +276,72 @@ struct ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
   static std::string name() { return "bool"; }
 };
 
-struct ARROW_EXPORT UInt8Type : public IntegerTypeImpl<UInt8Type, Type::UINT8, uint8_t> {
+class ARROW_EXPORT UInt8Type : public IntegerTypeImpl<UInt8Type, Type::UINT8, uint8_t> {
+ public:
   static std::string name() { return "uint8"; }
 };
 
-struct ARROW_EXPORT Int8Type : public IntegerTypeImpl<Int8Type, Type::INT8, int8_t> {
+class ARROW_EXPORT Int8Type : public IntegerTypeImpl<Int8Type, Type::INT8, int8_t> {
+ public:
   static std::string name() { return "int8"; }
 };
 
-struct ARROW_EXPORT UInt16Type
+class ARROW_EXPORT UInt16Type
     : public IntegerTypeImpl<UInt16Type, Type::UINT16, uint16_t> {
+ public:
   static std::string name() { return "uint16"; }
 };
 
-struct ARROW_EXPORT Int16Type : public IntegerTypeImpl<Int16Type, Type::INT16, int16_t> {
+class ARROW_EXPORT Int16Type : public IntegerTypeImpl<Int16Type, Type::INT16, int16_t> {
+ public:
   static std::string name() { return "int16"; }
 };
 
-struct ARROW_EXPORT UInt32Type
+class ARROW_EXPORT UInt32Type
     : public IntegerTypeImpl<UInt32Type, Type::UINT32, uint32_t> {
+ public:
   static std::string name() { return "uint32"; }
 };
 
-struct ARROW_EXPORT Int32Type : public IntegerTypeImpl<Int32Type, Type::INT32, int32_t> {
+class ARROW_EXPORT Int32Type : public IntegerTypeImpl<Int32Type, Type::INT32, int32_t> {
+ public:
   static std::string name() { return "int32"; }
 };
 
-struct ARROW_EXPORT UInt64Type
+class ARROW_EXPORT UInt64Type
     : public IntegerTypeImpl<UInt64Type, Type::UINT64, uint64_t> {
+ public:
   static std::string name() { return "uint64"; }
 };
 
-struct ARROW_EXPORT Int64Type : public IntegerTypeImpl<Int64Type, Type::INT64, int64_t> {
+class ARROW_EXPORT Int64Type : public IntegerTypeImpl<Int64Type, Type::INT64, int64_t> {
+ public:
   static std::string name() { return "int64"; }
 };
 
-struct ARROW_EXPORT HalfFloatType
+class ARROW_EXPORT HalfFloatType
     : public CTypeImpl<HalfFloatType, FloatingPoint, Type::HALF_FLOAT, uint16_t> {
+ public:
   Precision precision() const override;
   static std::string name() { return "halffloat"; }
 };
 
-struct ARROW_EXPORT FloatType
+class ARROW_EXPORT FloatType
     : public CTypeImpl<FloatType, FloatingPoint, Type::FLOAT, float> {
+ public:
   Precision precision() const override;
   static std::string name() { return "float"; }
 };
 
-struct ARROW_EXPORT DoubleType
+class ARROW_EXPORT DoubleType
     : public CTypeImpl<DoubleType, FloatingPoint, Type::DOUBLE, double> {
+ public:
   Precision precision() const override;
   static std::string name() { return "double"; }
 };
 
-struct ARROW_EXPORT ListType : public NestedType {
+class ARROW_EXPORT ListType : public NestedType {
+ public:
   static constexpr Type::type type_id = Type::LIST;
 
   // List can contain any other logical value type
@@ -326,7 +354,7 @@ struct ARROW_EXPORT ListType : public NestedType {
 
   std::shared_ptr<Field> value_field() const { return children_[0]; }
 
-  std::shared_ptr<DataType> value_type() const { return children_[0]->type; }
+  std::shared_ptr<DataType> value_type() const { return children_[0]->type(); }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -337,7 +365,8 @@ struct ARROW_EXPORT ListType : public NestedType {
 };
 
 // BinaryType type is represents lists of 1-byte values.
-struct ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
+class ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
+ public:
   static constexpr Type::type type_id = Type::BINARY;
 
   BinaryType() : BinaryType(Type::BINARY) {}
@@ -376,7 +405,8 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType {
 };
 
 // UTF-8 encoded strings
-struct ARROW_EXPORT StringType : public BinaryType {
+class ARROW_EXPORT StringType : public BinaryType {
+ public:
   static constexpr Type::type type_id = Type::STRING;
 
   StringType() : BinaryType(Type::STRING) {}
@@ -386,7 +416,8 @@ struct ARROW_EXPORT StringType : public BinaryType {
   static std::string name() { return "utf8"; }
 };
 
-struct ARROW_EXPORT StructType : public NestedType {
+class ARROW_EXPORT StructType : public NestedType {
+ public:
   static constexpr Type::type type_id = Type::STRUCT;
 
   explicit StructType(const std::vector<std::shared_ptr<Field>>& fields)
@@ -412,25 +443,32 @@ static inline int decimal_byte_width(int precision) {
   }
 }
 
-struct ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
+class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
+ public:
   static constexpr Type::type type_id = Type::DECIMAL;
 
-  explicit DecimalType(int precision_, int scale_)
-      : FixedSizeBinaryType(decimal_byte_width(precision_), Type::DECIMAL),
-        precision(precision_),
-        scale(scale_) {}
+  explicit DecimalType(int precision, int scale)
+      : FixedSizeBinaryType(decimal_byte_width(precision), Type::DECIMAL),
+        precision_(precision),
+        scale_(scale) {}
+
   std::vector<BufferDescr> GetBufferLayout() const override;
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "decimal"; }
 
-  int precision;
-  int scale;
+  int precision() const { return precision_; }
+  int scale() const { return scale_; }
+
+ private:
+  int precision_;
+  int scale_;
 };
 
 enum class UnionMode : char { SPARSE, DENSE };
 
-struct ARROW_EXPORT UnionType : public NestedType {
+class ARROW_EXPORT UnionType : public NestedType {
+ public:
   static constexpr Type::type type_id = Type::UNION;
 
   UnionType(const std::vector<std::shared_ptr<Field>>& fields,
@@ -442,12 +480,17 @@ struct ARROW_EXPORT UnionType : public NestedType {
 
   std::vector<BufferDescr> GetBufferLayout() const override;
 
-  UnionMode mode;
+  const std::vector<uint8_t>& type_codes() const { return type_codes_; }
+
+  UnionMode mode() const { return mode_; }
+
+ private:
+  UnionMode mode_;
 
   // The type id used in the data to indicate each data type in the union. For
   // example, the first type in the union might be denoted by the id 5 (instead
   // of 0).
-  std::vector<uint8_t> type_codes;
+  std::vector<uint8_t> type_codes_;
 };
 
 // ----------------------------------------------------------------------
@@ -455,16 +498,18 @@ struct ARROW_EXPORT UnionType : public NestedType {
 
 enum class DateUnit : char { DAY = 0, MILLI = 1 };
 
-struct ARROW_EXPORT DateType : public FixedWidthType {
+class ARROW_EXPORT DateType : public FixedWidthType {
  public:
-  DateUnit unit;
+  DateUnit unit() const { return unit_; }
 
  protected:
   DateType(Type::type type_id, DateUnit unit);
+  DateUnit unit_;
 };
 
 /// Date as int32_t days since UNIX epoch
-struct ARROW_EXPORT Date32Type : public DateType {
+class ARROW_EXPORT Date32Type : public DateType {
+ public:
   static constexpr Type::type type_id = Type::DATE32;
 
   using c_type = int32_t;
@@ -478,7 +523,8 @@ struct ARROW_EXPORT Date32Type : public DateType {
 };
 
 /// Date as int64_t milliseconds since UNIX epoch
-struct ARROW_EXPORT Date64Type : public DateType {
+class ARROW_EXPORT Date64Type : public DateType {
+ public:
   static constexpr Type::type type_id = Type::DATE64;
 
   using c_type = int64_t;
@@ -512,15 +558,17 @@ static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) {
   return os;
 }
 
-struct ARROW_EXPORT TimeType : public FixedWidthType {
+class ARROW_EXPORT TimeType : public FixedWidthType {
  public:
-  TimeUnit unit;
+  TimeUnit unit() const { return unit_; }
 
  protected:
   TimeType(Type::type type_id, TimeUnit unit);
+  TimeUnit unit_;
 };
 
-struct ARROW_EXPORT Time32Type : public TimeType {
+class ARROW_EXPORT Time32Type : public TimeType {
+ public:
   static constexpr Type::type type_id = Type::TIME32;
   using c_type = int32_t;
 
@@ -532,7 +580,8 @@ struct ARROW_EXPORT Time32Type : public TimeType {
   std::string ToString() const override;
 };
 
-struct ARROW_EXPORT Time64Type : public TimeType {
+class ARROW_EXPORT Time64Type : public TimeType {
+ public:
   static constexpr Type::type type_id = Type::TIME64;
   using c_type = int64_t;
 
@@ -544,7 +593,8 @@ struct ARROW_EXPORT Time64Type : public TimeType {
   std::string ToString() const override;
 };
 
-struct ARROW_EXPORT TimestampType : public FixedWidthType {
+class ARROW_EXPORT TimestampType : public FixedWidthType {
+ public:
   using Unit = TimeUnit;
 
   typedef int64_t c_type;
@@ -553,20 +603,25 @@ struct ARROW_EXPORT TimestampType : public FixedWidthType {
   int bit_width() const override { return static_cast<int>(sizeof(int64_t) * CHAR_BIT); }
 
   explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
-      : FixedWidthType(Type::TIMESTAMP), unit(unit) {}
+      : FixedWidthType(Type::TIMESTAMP), unit_(unit) {}
 
   explicit TimestampType(TimeUnit unit, const std::string& timezone)
-      : FixedWidthType(Type::TIMESTAMP), unit(unit), timezone(timezone) {}
+      : FixedWidthType(Type::TIMESTAMP), unit_(unit), timezone_(timezone) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "timestamp"; }
 
-  TimeUnit unit;
-  std::string timezone;
+  TimeUnit unit() const { return unit_; }
+  const std::string& timezone() const { return timezone_; }
+
+ private:
+  TimeUnit unit_;
+  std::string timezone_;
 };
 
-struct ARROW_EXPORT IntervalType : public FixedWidthType {
+class ARROW_EXPORT IntervalType : public FixedWidthType {
+ public:
   enum class Unit : char { YEAR_MONTH = 0, DAY_TIME = 1 };
 
   using c_type = int64_t;
@@ -574,14 +629,17 @@ struct ARROW_EXPORT IntervalType : public FixedWidthType {
 
   int bit_width() const override { return static_cast<int>(sizeof(int64_t) * CHAR_BIT); }
 
-  Unit unit;
-
   explicit IntervalType(Unit unit = Unit::YEAR_MONTH)
-      : FixedWidthType(Type::INTERVAL), unit(unit) {}
+      : FixedWidthType(Type::INTERVAL), unit_(unit) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override { return name(); }
   static std::string name() { return "date"; }
+
+  Unit unit() const { return unit_; }
+
+ private:
+  Unit unit_;
 };
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 2bb05f853a094..99c09bd6b7dca 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -26,10 +26,10 @@ namespace arrow {
 
 class Status;
 
-struct DataType;
+class DataType;
 class Array;
 class ArrayBuilder;
-struct Field;
+class Field;
 class Tensor;
 
 class Buffer;
@@ -40,14 +40,14 @@ class Schema;
 class DictionaryType;
 class DictionaryArray;
 
-struct NullType;
+class NullType;
 class NullArray;
 
-struct BooleanType;
+class BooleanType;
 class BooleanArray;
 class BooleanBuilder;
 
-struct BinaryType;
+class BinaryType;
 class BinaryArray;
 class BinaryBuilder;
 
@@ -55,23 +55,23 @@ class FixedSizeBinaryType;
 class FixedSizeBinaryArray;
 class FixedSizeBinaryBuilder;
 
-struct StringType;
+class StringType;
 class StringArray;
 class StringBuilder;
 
-struct ListType;
+class ListType;
 class ListArray;
 class ListBuilder;
 
-struct StructType;
+class StructType;
 class StructArray;
 class StructBuilder;
 
-struct DecimalType;
+class DecimalType;
 class DecimalArray;
 class DecimalBuilder;
 
-struct UnionType;
+class UnionType;
 class UnionArray;
 
 template <typename TypeClass>
@@ -84,7 +84,7 @@ template <typename TypeClass>
 class NumericTensor;
 
 #define _NUMERIC_TYPE_DECL(KLASS)                     \
-  struct KLASS##Type;                                 \
+  class KLASS##Type;                                  \
   using KLASS##Array = NumericArray<KLASS##Type>;     \
   using KLASS##Builder = NumericBuilder<KLASS##Type>; \
   using KLASS##Tensor = NumericTensor<KLASS##Type>;
@@ -103,27 +103,27 @@ _NUMERIC_TYPE_DECL(Double);
 
 #undef _NUMERIC_TYPE_DECL
 
-struct Date64Type;
+class Date64Type;
 using Date64Array = NumericArray<Date64Type>;
 using Date64Builder = NumericBuilder<Date64Type>;
 
-struct Date32Type;
+class Date32Type;
 using Date32Array = NumericArray<Date32Type>;
 using Date32Builder = NumericBuilder<Date32Type>;
 
-struct Time32Type;
+class Time32Type;
 using Time32Array = NumericArray<Time32Type>;
 using Time32Builder = NumericBuilder<Time32Type>;
 
-struct Time64Type;
+class Time64Type;
 using Time64Array = NumericArray<Time64Type>;
 using Time64Builder = NumericBuilder<Time64Type>;
 
-struct TimestampType;
+class TimestampType;
 using TimestampArray = NumericArray<TimestampType>;
 using TimestampBuilder = NumericBuilder<TimestampType>;
 
-struct IntervalType;
+class IntervalType;
 using IntervalArray = NumericArray<IntervalType>;
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index 29b3db60cadf8..bc5f493fa1f9a 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -33,7 +33,7 @@ namespace arrow {
 
 template <typename VISITOR>
 inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
-  switch (type.type) {
+  switch (type.id()) {
     TYPE_VISIT_INLINE(NullType);
     TYPE_VISIT_INLINE(BooleanType);
     TYPE_VISIT_INLINE(Int8Type);
@@ -72,7 +72,7 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
 
 template <typename VISITOR>
 inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
-  switch (array.type_enum()) {
+  switch (array.type_id()) {
     ARRAY_VISIT_INLINE(NullType);
     ARRAY_VISIT_INLINE(BooleanType);
     ARRAY_VISIT_INLINE(Int8Type);
@@ -111,7 +111,7 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
 
 template <typename VISITOR>
 inline Status VisitTensorInline(const Tensor& array, VISITOR* visitor) {
-  switch (array.type_enum()) {
+  switch (array.type_id()) {
     TENSOR_VISIT_INLINE(Int8Type);
     TENSOR_VISIT_INLINE(UInt8Type);
     TENSOR_VISIT_INLINE(Int16Type);
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
index ee500e6812974..1c4253eebe46a 100644
--- a/python/pyarrow/array.pyx
+++ b/python/pyarrow/array.pyx
@@ -618,7 +618,7 @@ cdef object box_array(const shared_ptr[CArray]& sp_array):
     if data_type == NULL:
         raise ValueError('Array data type was NULL')
 
-    cdef Array arr = _array_classes[data_type.type]()
+    cdef Array arr = _array_classes[data_type.id()]()
     arr.init(sp_array)
     return arr
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index e719e185b7b13..71b5c8d2172dc 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -61,7 +61,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         TimeUnit_NANO" arrow::TimeUnit::NANO"
 
     cdef cppclass CDataType" arrow::DataType":
-        Type type
+        Type id()
 
         c_bool Equals(const CDataType& other)
 
@@ -72,7 +72,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         int64_t length()
         int64_t null_count()
-        Type type_enum()
+        Type type_id()
 
         c_bool Equals(const CArray& arr)
         c_bool IsNull(int i)
@@ -97,14 +97,14 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         pass
 
     cdef cppclass CTimestampType" arrow::TimestampType"(CFixedWidthType):
-        TimeUnit unit
-        c_string timezone
+        TimeUnit unit()
+        const c_string& timezone()
 
     cdef cppclass CTime32Type" arrow::Time32Type"(CFixedWidthType):
-        TimeUnit unit
+        TimeUnit unit()
 
     cdef cppclass CTime64Type" arrow::Time64Type"(CFixedWidthType):
-        TimeUnit unit
+        TimeUnit unit()
 
     cdef cppclass CDictionaryType" arrow::DictionaryType"(CFixedWidthType):
         CDictionaryType(const shared_ptr[CDataType]& index_type,
@@ -149,15 +149,14 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int bit_width()
 
     cdef cppclass CDecimalType" arrow::DecimalType"(CFixedSizeBinaryType):
-        int precision
-        int scale
+        int precision()
+        int scale()
         CDecimalType(int precision, int scale)
 
     cdef cppclass CField" arrow::Field":
-        c_string name
-        shared_ptr[CDataType] type
-
-        c_bool nullable
+        const c_string& name()
+        shared_ptr[CDataType] type()
+        c_bool nullable()
 
         CField(const c_string& name, const shared_ptr[CDataType]& type,
                c_bool nullable)
@@ -307,7 +306,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_bool is_mutable()
         c_bool is_contiguous()
-        Type type_enum()
+        Type type_id()
         c_bool Equals(const CTensor& other)
 
     CStatus ConcatenateTables(const vector[shared_ptr[CTable]]& tables,
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
index 7591ae880da3d..2b6746a3cf815 100644
--- a/python/pyarrow/scalar.pyx
+++ b/python/pyarrow/scalar.pyx
@@ -158,26 +158,26 @@ cdef class TimestampValue(ArrayValue):
 
         timezone = None
         tzinfo = None
-        if dtype.timezone.size() > 0:
-            timezone = frombytes(dtype.timezone)
+        if dtype.timezone().size() > 0:
+            timezone = frombytes(dtype.timezone())
             import pytz
             tzinfo = pytz.timezone(timezone)
 
         try:
             pd = _pandas()
-            if dtype.unit == TimeUnit_SECOND:
+            if dtype.unit() == TimeUnit_SECOND:
                 val = val * 1000000000
-            elif dtype.unit == TimeUnit_MILLI:
+            elif dtype.unit() == TimeUnit_MILLI:
                 val = val * 1000000
-            elif dtype.unit == TimeUnit_MICRO:
+            elif dtype.unit() == TimeUnit_MICRO:
                 val = val * 1000
             return pd.Timestamp(val, tz=tzinfo)
         except ImportError:
-            if dtype.unit == TimeUnit_SECOND:
+            if dtype.unit() == TimeUnit_SECOND:
                 result = datetime.datetime.utcfromtimestamp(val)
-            elif dtype.unit == TimeUnit_MILLI:
+            elif dtype.unit() == TimeUnit_MILLI:
                 result = datetime.datetime.utcfromtimestamp(float(val) / 1000)
-            elif dtype.unit == TimeUnit_MICRO:
+            elif dtype.unit() == TimeUnit_MICRO:
                 result = datetime.datetime.utcfromtimestamp(
                     float(val) / 1000000)
             else:
@@ -208,10 +208,6 @@ cdef class DecimalValue(ArrayValue):
     def as_py(self):
         cdef:
             CDecimalArray* ap = <CDecimalArray*> self.sp_array.get()
-            CDecimalType* t = <CDecimalType*> ap.type().get()
-            int bit_width = t.bit_width()
-            int precision = t.precision
-            int scale = t.scale
             c_string s = ap.FormatValue(self.index)
         return decimal.Decimal(s.decode('utf8'))
 
@@ -309,11 +305,11 @@ cdef dict _scalar_classes = {
 cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
                        int64_t index):
     cdef ArrayValue val
-    if type.type.type == Type_NA:
+    if type.type.id() == Type_NA:
         return NA
     elif sp_array.get().IsNull(index):
         return NA
     else:
-        val = _scalar_classes[type.type.type]()
+        val = _scalar_classes[type.type.id()]()
         val.init(type, sp_array, index)
         return val
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index 4b931bf452239..474980973959f 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -81,13 +81,13 @@ cdef class TimestampType(DataType):
     property unit:
 
         def __get__(self):
-            return timeunit_to_string(self.ts_type.unit)
+            return timeunit_to_string(self.ts_type.unit())
 
     property tz:
 
         def __get__(self):
-            if self.ts_type.timezone.size() > 0:
-                return frombytes(self.ts_type.timezone)
+            if self.ts_type.timezone().size() > 0:
+                return frombytes(self.ts_type.timezone())
             else:
                 return None
 
@@ -119,7 +119,7 @@ cdef class Field:
     cdef init(self, const shared_ptr[CField]& field):
         self.sp_field = field
         self.field = field.get()
-        self.type = box_data_type(field.get().type)
+        self.type = box_data_type(field.get().type())
 
     @classmethod
     def from_py(cls, object name, DataType type, bint nullable=True):
@@ -137,7 +137,7 @@ cdef class Field:
     property nullable:
 
         def __get__(self):
-            return self.field.nullable
+            return self.field.nullable()
 
     property name:
 
@@ -145,7 +145,7 @@ cdef class Field:
             if box_field(self.sp_field) is None:
                 raise ReferenceError(
                     'Field not initialized (references NULL pointer)')
-            return frombytes(self.field.name)
+            return frombytes(self.field.name())
 
 
 cdef class Schema:
@@ -162,7 +162,7 @@ cdef class Schema:
 
         cdef Field result = Field()
         result.init(self.schema.field(i))
-        result.type = box_data_type(result.field.type)
+        result.type = box_data_type(result.field.type())
 
         return result
 
@@ -442,13 +442,13 @@ cdef DataType box_data_type(const shared_ptr[CDataType]& type):
     if type.get() == NULL:
         return None
 
-    if type.get().type == la.Type_DICTIONARY:
+    if type.get().id() == la.Type_DICTIONARY:
         out = DictionaryType()
-    elif type.get().type == la.Type_TIMESTAMP:
+    elif type.get().id() == la.Type_TIMESTAMP:
         out = TimestampType()
-    elif type.get().type == la.Type_FIXED_SIZE_BINARY:
+    elif type.get().id() == la.Type_FIXED_SIZE_BINARY:
         out = FixedSizeBinaryType()
-    elif type.get().type == la.Type_DECIMAL:
+    elif type.get().id() == la.Type_DECIMAL:
         out = DecimalType()
     else:
         out = DataType()

From e327c2e08d51ee13b3cf3b8801cd3adfe88b3f7c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 10 Apr 2017 09:47:08 -0400
Subject: [PATCH 0493/1644] ARROW-761: [C++/Python] Add GetTensorSize method,
 Python bindings

This computes the memory footprint of a serialized `arrow::Tensor` so that an appropriate memory region can be allocated.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #521 from wesm/ARROW-761 and squashes the following commits:

983177e [Wes McKinney] Fix sign comparison warning
0d787ad [Wes McKinney] Add GetTensorSize method, Python bindings
---
 cpp/src/arrow/ipc/ipc-read-write-test.cc    |  4 +++
 cpp/src/arrow/ipc/writer.cc                 | 29 ++++++++++++---------
 cpp/src/arrow/ipc/writer.h                  |  4 +++
 python/pyarrow/__init__.py                  |  3 ++-
 python/pyarrow/includes/libarrow.pxd        |  3 +++
 python/pyarrow/io.pyx                       | 20 ++++++++++++++
 python/pyarrow/tests/test_convert_pandas.py |  4 +--
 python/pyarrow/tests/test_ipc.py            |  9 +++++++
 python/pyarrow/tests/test_tensor.py         |  9 ++++++-
 9 files changed, 69 insertions(+), 16 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 98a7c3dd58a6b..cfba0d0a95106 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -640,6 +640,10 @@ TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
 
   CheckTensorRoundTrip(t0);
   CheckTensorRoundTrip(tzero);
+
+  int64_t serialized_size;
+  ASSERT_OK(GetTensorSize(t0, &serialized_size));
+  ASSERT_TRUE(serialized_size > static_cast<int64_t>(size * sizeof(int64_t)));
 }
 
 TEST_F(TestTensorRoundTrip, NonContiguous) {
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 18a585599a31b..8ba00a6ffd599 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -192,16 +192,6 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status GetTotalSize(const RecordBatch& batch, int64_t* size) {
-    // emulates the behavior of Write without actually writing
-    int32_t metadata_length = 0;
-    int64_t body_length = 0;
-    MockOutputStream dst;
-    RETURN_NOT_OK(Write(batch, &dst, &metadata_length, &body_length));
-    *size = dst.GetExtentBytesWritten();
-    return Status::OK();
-  }
-
  protected:
   template <typename ArrayType>
   Status VisitFixedWidth(const ArrayType& array) {
@@ -522,8 +512,23 @@ Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dict
 }
 
 Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
-  RecordBatchWriter writer(default_memory_pool(), 0, kMaxNestingDepth, true);
-  RETURN_NOT_OK(writer.GetTotalSize(batch, size));
+  // emulates the behavior of Write without actually writing
+  int32_t metadata_length = 0;
+  int64_t body_length = 0;
+  MockOutputStream dst;
+  RETURN_NOT_OK(WriteRecordBatch(batch, 0, &dst, &metadata_length, &body_length,
+      default_memory_pool(), kMaxNestingDepth, true));
+  *size = dst.GetExtentBytesWritten();
+  return Status::OK();
+}
+
+Status GetTensorSize(const Tensor& tensor, int64_t* size) {
+  // emulates the behavior of Write without actually writing
+  int32_t metadata_length = 0;
+  int64_t body_length = 0;
+  MockOutputStream dst;
+  RETURN_NOT_OK(WriteTensor(tensor, &dst, &metadata_length, &body_length));
+  *size = dst.GetExtentBytesWritten();
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 629bcb9c6c980..b71becb8c73b8 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -81,6 +81,10 @@ Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dict
 // Flatbuffers metadata.
 Status ARROW_EXPORT GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 
+// Compute the precise number of bytes needed in a contiguous memory segment to
+// write the tensor including metadata, padding, and data
+Status ARROW_EXPORT GetTensorSize(const Tensor& tensor, int64_t* size);
+
 class ARROW_EXPORT StreamWriter {
  public:
   virtual ~StreamWriter();
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 7b23cf66c6f7e..df615b428c1c1 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -51,7 +51,8 @@
                         Buffer, BufferReader, InMemoryOutputStream,
                         MemoryMappedFile, memory_map,
                         frombuffer, read_tensor, write_tensor,
-                        memory_map, create_memory_map)
+                        memory_map, create_memory_map,
+                        get_record_batch_size, get_tensor_size)
 
 from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 71b5c8d2172dc..40dd83776b82d 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -544,6 +544,9 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
 
         CStatus GetRecordBatch(int i, shared_ptr[CRecordBatch]* batch)
 
+    CStatus GetRecordBatchSize(const CRecordBatch& batch, int64_t* size)
+    CStatus GetTensorSize(const CTensor& tensor, int64_t* size)
+
     CStatus WriteTensor(const CTensor& tensor, OutputStream* dst,
                         int32_t* metadata_length,
                         int64_t* body_length)
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/io.pyx
index 98b5a62b372a2..4eb0816ecbdea 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/io.pyx
@@ -1202,6 +1202,26 @@ cdef class FeatherReader:
         return col
 
 
+def get_tensor_size(Tensor tensor):
+    """
+    Return total size of serialized Tensor including metadata and padding
+    """
+    cdef int64_t size
+    with nogil:
+        check_status(GetTensorSize(deref(tensor.tp), &size))
+    return size
+
+
+def get_record_batch_size(RecordBatch batch):
+    """
+    Return total size of serialized RecordBatch including metadata and padding
+    """
+    cdef int64_t size
+    with nogil:
+        check_status(GetRecordBatchSize(deref(batch.batch), &size))
+    return size
+
+
 def write_tensor(Tensor tensor, NativeFile dest):
     """
     Write pyarrow.Tensor to pyarrow.NativeFile object its current position
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index d1bea0b3e32f0..4a57e4ba1d4fb 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -68,7 +68,7 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
                                 timestamps_to_ms=False, expected_schema=None,
                                 check_dtype=True, schema=None):
         table = pa.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms,
-                                    schema=schema)
+                                     schema=schema)
         result = table.to_pandas(nthreads=nthreads)
         if expected_schema:
             assert table.schema.equals(expected_schema)
@@ -79,7 +79,7 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
     def _check_array_roundtrip(self, values, expected=None, mask=None,
                                timestamps_to_ms=False, type=None):
         arr = pa.Array.from_numpy(values, timestamps_to_ms=timestamps_to_ms,
-                                 mask=mask, type=type)
+                                  mask=mask, type=type)
         result = arr.to_pandas()
 
         values_nulls = pd.isnull(values)
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 4c9dad1b840a8..31d418d5150ac 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -151,6 +151,15 @@ def test_ipc_zero_copy_numpy():
     assert_frame_equal(df, rdf)
 
 
+def test_get_record_batch_size():
+    N = 10
+    itemsize = 8
+    df = pd.DataFrame({'foo': np.random.randn(N)})
+
+    batch = pa.RecordBatch.from_pandas(df)
+    assert pa.get_record_batch_size(batch) > (N * itemsize)
+
+
 def write_file(batch, sink):
     writer = pa.FileWriter(sink, batch.schema)
     writer.write_batch(batch)
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index 327b7f08a37f1..ec71735b2a540 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -42,10 +42,11 @@ def test_tensor_attrs():
     tensor = pa.Tensor.from_numpy(data2)
     assert not tensor.is_mutable
 
+
 def test_tensor_base_object():
     tensor = pa.Tensor.from_numpy(np.random.randn(10, 4))
     n = sys.getrefcount(tensor)
-    array = tensor.to_numpy()
+    array = tensor.to_numpy()  # noqa
     assert sys.getrefcount(tensor) == n + 1
 
 
@@ -111,3 +112,9 @@ def test_tensor_ipc_strided():
             pa.write_tensor(tensor, mmap)
     finally:
         _try_delete(path)
+
+
+def test_tensor_size():
+    data = np.random.randn(10, 4)
+    tensor = pa.Tensor.from_numpy(data)
+    assert pa.get_tensor_size(tensor) > (data.size * 8)

From c2f28cd07413e262fa0b741c286f86d5c7277c56 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 10 Apr 2017 09:47:49 -0400
Subject: [PATCH 0494/1644] ARROW-741: [Python] Switch Travis CI to use Python
 3.6 instead of 3.5

I'm OK with not building Python 3.5 in Travis CI anymore because 3.5 and 3.6 are essentially the same at the C API level. Other opinions?

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #514 from wesm/ARROW-741 and squashes the following commits:

3aee721 [Wes McKinney] Remove apache channel
116b229 [Wes McKinney] Switch Travis CI to use Python 3.6 instead of 3.5
---
 ci/travis_install_conda.sh | 1 -
 ci/travis_script_python.sh | 8 ++++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index 9c13b1bc0f079..e064317f12303 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -32,7 +32,6 @@ conda info -a
 conda config --set show_channel_urls True
 conda config --add channels https://repo.continuum.io/pkgs/free
 conda config --add channels conda-forge
-conda config --add channels apache
 conda info -a
 
 conda install --yes conda-build jinja2 anaconda-client
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index df11209e7c49b..604cd13916299 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -28,7 +28,7 @@ pushd $PYTHON_DIR
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 
 build_parquet_cpp() {
-  conda create -y -q -p $PARQUET_HOME python=3.5
+  conda create -y -q -p $PARQUET_HOME python=3.6
   source activate $PARQUET_HOME
 
   # In case some package wants to download the MKL
@@ -120,15 +120,15 @@ python_version_tests() {
   python -m pytest -vv -r sxX pyarrow
 
   # Build documentation once
-  if [[ "$PYTHON_VERSION" == "3.5" ]]
+  if [[ "$PYTHON_VERSION" == "3.6" ]]
   then
       pip install -r doc/requirements.txt
       python setup.py build_sphinx
   fi
 }
 
-# run tests for python 2.7 and 3.5
+# run tests for python 2.7 and 3.6
 python_version_tests 2.7
-python_version_tests 3.5
+python_version_tests 3.6
 
 popd

From 06d92bbab426d8b343d238e3e61166353da11877 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 10 Apr 2017 17:35:18 -0400
Subject: [PATCH 0495/1644] ARROW-779: [C++] Check for old metadata and raise
 exception if found

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #507 from wesm/ARROW-779 and squashes the following commits:

dad42f7 [Wes McKinney] Check for old metadata and raise exception if found
---
 cpp/src/arrow/ipc/metadata.cc | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 84f8883ffb949..ee21156c08c67 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -50,7 +50,11 @@ using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
 using FBString = flatbuffers::Offset<flatbuffers::String>;
 
-static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V3;
+static constexpr flatbuf::MetadataVersion kCurrentMetadataVersion =
+    flatbuf::MetadataVersion_V3;
+
+static constexpr flatbuf::MetadataVersion kMinMetadataVersion =
+    flatbuf::MetadataVersion_V3;
 
 static Status IntFromFlatbuffer(
     const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
@@ -605,8 +609,8 @@ static Status WriteFlatbufferBuilder(FBB& fbb, std::shared_ptr<Buffer>* out) {
 
 static Status WriteFBMessage(FBB& fbb, flatbuf::MessageHeader header_type,
     flatbuffers::Offset<void> header, int64_t body_length, std::shared_ptr<Buffer>* out) {
-  auto message =
-      flatbuf::CreateMessage(fbb, kMetadataVersion, header_type, header, body_length);
+  auto message = flatbuf::CreateMessage(
+      fbb, kCurrentMetadataVersion, header_type, header, body_length);
   fbb.Finish(message);
   return WriteFlatbufferBuilder(fbb, out);
 }
@@ -738,7 +742,7 @@ Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dicti
   auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
 
   auto footer = flatbuf::CreateFooter(
-      fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
+      fbb, kCurrentMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
 
   fbb.Finish(footer);
 
@@ -814,7 +818,11 @@ class Message::MessageImpl {
   Status Open() {
     message_ = flatbuf::GetMessage(buffer_->data() + offset_);
 
-    // TODO(wesm): verify the message
+    // Check that the metadata version is supported
+    if (message_->version() < kMinMetadataVersion) {
+      return Status::Invalid("Old metadata version not supported");
+    }
+
     return Status::OK();
   }
 

From 85b870e72803641568ff260af2306d9fc993a6d4 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 11 Apr 2017 10:17:43 -0400
Subject: [PATCH 0496/1644] ARROW-802: [GLib] Add read examples

Author: Kouhei Sutou <kou@clear-code.com>

Closes #522 from kou/glib-add-read-examples and squashes the following commits:

3fd5a2f [Kouhei Sutou] [GLib] Add read examples
---
 c_glib/.gitignore            |   2 +
 c_glib/configure.ac          |   2 +
 c_glib/example/Makefile.am   |  15 +++-
 c_glib/example/README.md     |  42 ++++++++++
 c_glib/example/read-batch.c  | 144 +++++++++++++++++++++++++++++++++++
 c_glib/example/read-stream.c | 143 ++++++++++++++++++++++++++++++++++
 6 files changed, 347 insertions(+), 1 deletion(-)
 create mode 100644 c_glib/example/README.md
 create mode 100644 c_glib/example/read-batch.c
 create mode 100644 c_glib/example/read-stream.c

diff --git a/c_glib/.gitignore b/c_glib/.gitignore
index e57a0594c1af3..8928158f6ca3a 100644
--- a/c_glib/.gitignore
+++ b/c_glib/.gitignore
@@ -43,3 +43,5 @@ Makefile.in
 /arrow-glib/stamp-*
 /arrow-glib/*.pc
 /example/build
+/example/read-batch
+/example/read-stream
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index d63132e6f293c..f36719284711b 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -87,6 +87,8 @@ else
   AC_SUBST(ARROW_LIBS)
 fi
 
+exampledir="\$(datadir)/arrow-glib/example"
+AC_SUBST(exampledir)
 
 AC_CONFIG_FILES([
   Makefile
diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am
index 3d456d7844231..8bf3c15526759 100644
--- a/c_glib/example/Makefile.am
+++ b/c_glib/example/Makefile.am
@@ -28,7 +28,20 @@ AM_LDFLAGS =						\
 	$(builddir)/../arrow-glib/libarrow-glib.la
 
 noinst_PROGRAMS =				\
-	build
+	build					\
+	read-batch				\
+	read-stream
 
 build_SOURCES =					\
 	build.c
+
+read_batch_SOURCES =				\
+	read-batch.c
+
+read_stream_SOURCES =				\
+	read-stream.c
+
+example_DATA =					\
+	$(build_SOURCES)			\
+	$(read_batch_SOURCES)			\
+	$(read_stream_SOURCES)
diff --git a/c_glib/example/README.md b/c_glib/example/README.md
new file mode 100644
index 0000000000000..b1ba259534cb1
--- /dev/null
+++ b/c_glib/example/README.md
@@ -0,0 +1,42 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Arrow GLib example
+
+There are example codes in this directory.
+
+C example codes exist in this directory.
+
+## C example codes
+
+Here are example codes in this directory:
+
+  * `build.c`: It shows how to create an array by array builder.
+
+<!---
+  * `write-batch.c`: It shows how to write Arrow array to file in batch
+    mode.
+-->
+
+  * `read-batch.c`: It shows how to read Arrow array from file in batch
+    mode.
+
+<!---
+  * `write-stream.c`: It shows how to write Arrow array to file in
+    stream mode.
+-->
+
+  * `read-stream.c`: It shows how to read Arrow array from file in
+    stream mode.
+
diff --git a/c_glib/example/read-batch.c b/c_glib/example/read-batch.c
new file mode 100644
index 0000000000000..a55b085d961d1
--- /dev/null
+++ b/c_glib/example/read-batch.c
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <stdlib.h>
+
+#include <arrow-glib/arrow-glib.h>
+
+static void
+print_array(GArrowArray *array)
+{
+  GArrowType value_type;
+  gint64 i, n;
+
+  value_type = garrow_array_get_value_type(array);
+
+  g_print("[");
+  n = garrow_array_get_length(array);
+
+#define ARRAY_CASE(type, Type, TYPE, format)                            \
+  case GARROW_TYPE_ ## TYPE:                                            \
+    {                                                                   \
+      GArrow ## Type ## Array *real_array;                              \
+      real_array = GARROW_ ## TYPE ## _ARRAY(array);                    \
+      for (i = 0; i < n; i++) {                                         \
+        if (i > 0) {                                                    \
+          g_print(", ");                                                \
+        }                                                               \
+        g_print(format,                                                 \
+                garrow_ ## type ## _array_get_value(real_array, i));    \
+      }                                                                 \
+    }                                                                   \
+    break
+
+  switch (value_type) {
+    ARRAY_CASE(uint8,  UInt8,  UINT8,  "%hhu");
+    ARRAY_CASE(uint16, UInt16, UINT16, "%" G_GUINT16_FORMAT);
+    ARRAY_CASE(uint32, UInt32, UINT32, "%" G_GUINT32_FORMAT);
+    ARRAY_CASE(uint64, UInt64, UINT64, "%" G_GUINT64_FORMAT);
+    ARRAY_CASE( int8,   Int8,   INT8,  "%hhd");
+    ARRAY_CASE( int16,  Int16,  INT16, "%" G_GINT16_FORMAT);
+    ARRAY_CASE( int32,  Int32,  INT32, "%" G_GINT32_FORMAT);
+    ARRAY_CASE( int64,  Int64,  INT64, "%" G_GINT64_FORMAT);
+    ARRAY_CASE( float,  Float,  FLOAT, "%g");
+    ARRAY_CASE(double, Double, DOUBLE, "%g");
+  default:
+    break;
+  }
+#undef ARRAY_CASE
+
+  g_print("]\n");
+}
+
+static void
+print_record_batch(GArrowRecordBatch *record_batch)
+{
+  guint nth_column, n_columns;
+
+  n_columns = garrow_record_batch_get_n_columns(record_batch);
+  for (nth_column = 0; nth_column < n_columns; nth_column++) {
+    GArrowArray *array;
+
+    g_print("columns[%u](%s): ",
+            nth_column,
+            garrow_record_batch_get_column_name(record_batch, nth_column));
+    array = garrow_record_batch_get_column(record_batch, nth_column);
+    print_array(array);
+  }
+}
+
+int
+main(int argc, char **argv)
+{
+  const char *input_path = "/tmp/batch.arrow";
+  GArrowIOMemoryMappedFile *input;
+  GError *error = NULL;
+
+  if (argc > 1)
+    input_path = argv[1];
+  input = garrow_io_memory_mapped_file_open(input_path,
+                                            GARROW_IO_FILE_MODE_READ,
+                                            &error);
+  if (!input) {
+    g_print("failed to open file: %s\n", error->message);
+    g_error_free(error);
+    return EXIT_FAILURE;
+  }
+
+  {
+    GArrowIPCFileReader *reader;
+
+    reader = garrow_ipc_file_reader_open(GARROW_IO_RANDOM_ACCESS_FILE(input),
+                                         &error);
+    if (!reader) {
+      g_print("failed to open file reader: %s\n", error->message);
+      g_error_free(error);
+      g_object_unref(input);
+      return EXIT_FAILURE;
+    }
+
+    {
+      guint i, n;
+
+      n = garrow_ipc_file_reader_get_n_record_batches(reader);
+      for (i = 0; i < n; i++) {
+        GArrowRecordBatch *record_batch;
+
+        record_batch =
+          garrow_ipc_file_reader_get_record_batch(reader, i, &error);
+        if (!record_batch) {
+          g_print("failed to open file reader: %s\n", error->message);
+          g_error_free(error);
+          g_object_unref(reader);
+          g_object_unref(input);
+          return EXIT_FAILURE;
+        }
+
+        print_record_batch(record_batch);
+        g_object_unref(record_batch);
+      }
+    }
+
+    g_object_unref(reader);
+  }
+
+  g_object_unref(input);
+
+  return EXIT_SUCCESS;
+}
diff --git a/c_glib/example/read-stream.c b/c_glib/example/read-stream.c
new file mode 100644
index 0000000000000..c56942c7770c5
--- /dev/null
+++ b/c_glib/example/read-stream.c
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <stdlib.h>
+
+#include <arrow-glib/arrow-glib.h>
+
+static void
+print_array(GArrowArray *array)
+{
+  GArrowType value_type;
+  gint64 i, n;
+
+  value_type = garrow_array_get_value_type(array);
+
+  g_print("[");
+  n = garrow_array_get_length(array);
+
+#define ARRAY_CASE(type, Type, TYPE, format)                            \
+  case GARROW_TYPE_ ## TYPE:                                            \
+    {                                                                   \
+      GArrow ## Type ## Array *real_array;                              \
+      real_array = GARROW_ ## TYPE ## _ARRAY(array);                    \
+      for (i = 0; i < n; i++) {                                         \
+        if (i > 0) {                                                    \
+          g_print(", ");                                                \
+        }                                                               \
+        g_print(format,                                                 \
+                garrow_ ## type ## _array_get_value(real_array, i));    \
+      }                                                                 \
+    }                                                                   \
+    break
+
+  switch (value_type) {
+    ARRAY_CASE(uint8,  UInt8,  UINT8,  "%hhu");
+    ARRAY_CASE(uint16, UInt16, UINT16, "%" G_GUINT16_FORMAT);
+    ARRAY_CASE(uint32, UInt32, UINT32, "%" G_GUINT32_FORMAT);
+    ARRAY_CASE(uint64, UInt64, UINT64, "%" G_GUINT64_FORMAT);
+    ARRAY_CASE( int8,   Int8,   INT8,  "%hhd");
+    ARRAY_CASE( int16,  Int16,  INT16, "%" G_GINT16_FORMAT);
+    ARRAY_CASE( int32,  Int32,  INT32, "%" G_GINT32_FORMAT);
+    ARRAY_CASE( int64,  Int64,  INT64, "%" G_GINT64_FORMAT);
+    ARRAY_CASE( float,  Float,  FLOAT, "%g");
+    ARRAY_CASE(double, Double, DOUBLE, "%g");
+  default:
+    break;
+  }
+#undef ARRAY_CASE
+
+  g_print("]\n");
+}
+
+static void
+print_record_batch(GArrowRecordBatch *record_batch)
+{
+  guint nth_column, n_columns;
+
+  n_columns = garrow_record_batch_get_n_columns(record_batch);
+  for (nth_column = 0; nth_column < n_columns; nth_column++) {
+    GArrowArray *array;
+
+    g_print("columns[%u](%s): ",
+            nth_column,
+            garrow_record_batch_get_column_name(record_batch, nth_column));
+    array = garrow_record_batch_get_column(record_batch, nth_column);
+    print_array(array);
+  }
+}
+
+int
+main(int argc, char **argv)
+{
+  const char *input_path = "/tmp/stream.arrow";
+  GArrowIOMemoryMappedFile *input;
+  GError *error = NULL;
+
+  if (argc > 1)
+    input_path = argv[1];
+  input = garrow_io_memory_mapped_file_open(input_path,
+                                            GARROW_IO_FILE_MODE_READ,
+                                            &error);
+  if (!input) {
+    g_print("failed to open file: %s\n", error->message);
+    g_error_free(error);
+    return EXIT_FAILURE;
+  }
+
+  {
+    GArrowIPCStreamReader *reader;
+
+    reader = garrow_ipc_stream_reader_open(GARROW_IO_INPUT_STREAM(input),
+                                           &error);
+    if (!reader) {
+      g_print("failed to open stream reader: %s\n", error->message);
+      g_error_free(error);
+      g_object_unref(input);
+      return EXIT_FAILURE;
+    }
+
+    while (TRUE) {
+      GArrowRecordBatch *record_batch;
+
+      record_batch =
+        garrow_ipc_stream_reader_get_next_record_batch(reader, &error);
+      if (error) {
+        g_print("failed to get record batch: %s\n", error->message);
+        g_error_free(error);
+        g_object_unref(reader);
+        g_object_unref(input);
+        return EXIT_FAILURE;
+      }
+
+      if (!record_batch) {
+        break;
+      }
+
+      print_record_batch(record_batch);
+      g_object_unref(record_batch);
+    }
+
+    g_object_unref(reader);
+  }
+
+  g_object_unref(input);
+
+  return EXIT_SUCCESS;
+}

From b7423a63cc53ad7d86a5e003a0bd48f855622354 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 11 Apr 2017 10:20:24 -0400
Subject: [PATCH 0497/1644] ARROW-803: [GLib] Update package repository URL

Author: Kouhei Sutou <kou@clear-code.com>

Closes #523 from kou/glib-update-package-repository-url and squashes the following commits:

d130478 [Kouhei Sutou] [GLib] Update package repository URL
---
 c_glib/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index 95cc9a65c5bd8..0137d9059ee1e 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -66,8 +66,8 @@ You need to add the following apt-lines to
 `/etc/apt/sources.list.d/groonga.list`:
 
 ```text
-deb http://packages.groonga.org/debian/ jessie main
-deb-src http://packages.groonga.org/debian/ jessie main
+deb https://packages.groonga.org/debian/ jessie main
+deb-src https://packages.groonga.org/debian/ jessie main
 ```
 
 Then you need to run the following command lines:
@@ -105,7 +105,7 @@ Now you can install Arrow GLib packages:
 You need to add a Yum repository:
 
 ```text
-% sudo yum install -y http://packages.groonga.org/centos/groonga-release-1.2.0-1.noarch.rpm
+% sudo yum install -y https://packages.groonga.org/centos/groonga-release-1.3.0-1.noarch.rpm
 ```
 
 Now you can install Arrow GLib packages:

From b3cec804bbd1b2626ff55e1a733deca9b7ba032b Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 11 Apr 2017 10:22:11 -0400
Subject: [PATCH 0498/1644] ARROW-804: [GLib] Update build document

Author: Kouhei Sutou <kou@clear-code.com>

Closes #524 from kou/glib-update-build-document and squashes the following commits:

07085e6 [Kouhei Sutou] [GLib] Update build document
---
 c_glib/README.md | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index 0137d9059ee1e..b253d32b266d4 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -114,7 +114,30 @@ Now you can install Arrow GLib packages:
 % sudo yum install -y --enablerepo=epel arrow-glib-devel
 ```
 
-### Build
+### How to build by users
+
+Arrow GLib users should use released source archive to build Arrow
+GLib:
+
+```text
+% wget https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
+% tar xf apache-arrow-0.3.0.tar.gz
+% cd apache-arrow-0.3.0
+```
+
+You need to build and install Arrow C++ before you build and install
+Arrow GLib. See Arrow C++ document about how to install Arrow C++.
+
+You can build and install Arrow GLib after you install Arrow C++:
+
+```text
+% cd c_glib
+% ./configure
+% make
+% sudo make install
+```
+
+### How to build by developers
 
 You need to install Arrow C++ before you install Arrow GLib. See Arrow
 C++ document about how to install Arrow C++.

From f5245cc6b1811217df78acfb7bf6163d9dd09f32 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 11 Apr 2017 13:55:11 -0400
Subject: [PATCH 0499/1644] ARROW-806: [GLib] Support add/remove a column from
 table

Author: Kouhei Sutou <kou@clear-code.com>

Closes #525 from kou/glib-add-remove-column and squashes the following commits:

72d495a [Kouhei Sutou] [GLib] Support add/remove a column from table
---
 c_glib/arrow-glib/table.cpp | 58 +++++++++++++++++++++++++++++++++++++
 c_glib/arrow-glib/table.h   |  8 +++++
 c_glib/test/test-table.rb   | 14 +++++++++
 3 files changed, 80 insertions(+)

diff --git a/c_glib/arrow-glib/table.cpp b/c_glib/arrow-glib/table.cpp
index 2f82ffa4320e0..1d743b70731bb 100644
--- a/c_glib/arrow-glib/table.cpp
+++ b/c_glib/arrow-glib/table.cpp
@@ -22,6 +22,7 @@
 #endif
 
 #include <arrow-glib/column.hpp>
+#include <arrow-glib/error.hpp>
 #include <arrow-glib/schema.hpp>
 #include <arrow-glib/table.hpp>
 
@@ -203,6 +204,63 @@ garrow_table_get_n_rows(GArrowTable *table)
   return arrow_table->num_rows();
 }
 
+/**
+ * garrow_table_add_column:
+ * @table: A #GArrowTable.
+ * @i: The index of the new column.
+ * @column: The column to be added.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): The newly allocated
+ *   #GArrowTable that has a new column or %NULL on error.
+ *
+ * Since: 0.3.0
+ */
+GArrowTable *
+garrow_table_add_column(GArrowTable *table,
+                        guint i,
+                        GArrowColumn *column,
+                        GError **error)
+{
+  const auto arrow_table = garrow_table_get_raw(table);
+  const auto arrow_column = garrow_column_get_raw(column);
+  std::shared_ptr<arrow::Table> arrow_new_table;
+  auto status = arrow_table->AddColumn(i, arrow_column, &arrow_new_table);
+  if (status.ok()) {
+    return garrow_table_new_raw(&arrow_new_table);
+  } else {
+    garrow_error_set(error, status, "[table][add-column]");
+    return NULL;
+  }
+}
+
+/**
+ * garrow_table_remove_column:
+ * @table: A #GArrowTable.
+ * @i: The index of the column to be removed.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): The newly allocated
+ *   #GArrowTable that doesn't have the column or %NULL on error.
+ *
+ * Since: 0.3.0
+ */
+GArrowTable *
+garrow_table_remove_column(GArrowTable *table,
+                           guint i,
+                           GError **error)
+{
+  const auto arrow_table = garrow_table_get_raw(table);
+  std::shared_ptr<arrow::Table> arrow_new_table;
+  auto status = arrow_table->RemoveColumn(i, &arrow_new_table);
+  if (status.ok()) {
+    return garrow_table_new_raw(&arrow_new_table);
+  } else {
+    garrow_error_set(error, status, "[table][remove-column]");
+    return NULL;
+  }
+}
+
 G_END_DECLS
 
 GArrowTable *
diff --git a/c_glib/arrow-glib/table.h b/c_glib/arrow-glib/table.h
index 4dbb8c587a2ec..9ae0cce1b7d9d 100644
--- a/c_glib/arrow-glib/table.h
+++ b/c_glib/arrow-glib/table.h
@@ -75,4 +75,12 @@ GArrowColumn   *garrow_table_get_column    (GArrowTable *table,
 guint           garrow_table_get_n_columns (GArrowTable *table);
 guint64         garrow_table_get_n_rows    (GArrowTable *table);
 
+GArrowTable    *garrow_table_add_column    (GArrowTable *table,
+                                            guint i,
+                                            GArrowColumn *column,
+                                            GError **error);
+GArrowTable    *garrow_table_remove_column (GArrowTable *table,
+                                            guint i,
+                                            GError **error);
+
 G_END_DECLS
diff --git a/c_glib/test/test-table.rb b/c_glib/test/test-table.rb
index e2b71b31e44c0..da6871ec1d090 100644
--- a/c_glib/test/test-table.rb
+++ b/c_glib/test/test-table.rb
@@ -82,5 +82,19 @@ def test_n_columns
     def test_n_rows
       assert_equal(1, @table.n_rows)
     end
+
+    def test_add_column
+      field = Arrow::Field.new("added", Arrow::BooleanDataType.new)
+      column = Arrow::Column.new(field, build_boolean_array([true]))
+      new_table = @table.add_column(1, column)
+      assert_equal(["visible", "added", "valid"],
+                   new_table.schema.fields.collect(&:name))
+    end
+
+    def test_remove_column
+      new_table = @table.remove_column(0)
+      assert_equal(["valid"],
+                   new_table.schema.fields.collect(&:name))
+    end
   end
 end

From 7b4723b2b4f259ac27b959d108fdc65c734d7359 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 11 Apr 2017 13:55:57 -0400
Subject: [PATCH 0500/1644] ARROW-807: [GLib] Update "Since" tag

Author: Kouhei Sutou <kou@clear-code.com>

Closes #526 from kou/glib-update-since-tag and squashes the following commits:

2ad64cc [Kouhei Sutou] [GLib] Update "Since" tag
---
 c_glib/arrow-glib/array.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index e016ba9728dec..3bd7b40ff9493 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -142,6 +142,8 @@ garrow_array_class_init(GArrowArrayClass *klass)
  * @i: The index of the target value.
  *
  * Returns: Whether the i-th value is null or not.
+ *
+ * Since: 0.3.0
  */
 gboolean
 garrow_array_is_null(GArrowArray *array, gint64 i)
@@ -193,9 +195,10 @@ garrow_array_get_n_nulls(GArrowArray *array)
  * garrow_array_get_value_data_type:
  * @array: A #GArrowArray.
  *
- * Since: 0.3.0
  * Returns: (transfer full): The #GArrowDataType for each value of the
  *   array.
+ *
+ * Since: 0.3.0
  */
 GArrowDataType *
 garrow_array_get_value_data_type(GArrowArray *array)
@@ -209,8 +212,9 @@ garrow_array_get_value_data_type(GArrowArray *array)
  * garrow_array_get_value_type:
  * @array: A #GArrowArray.
  *
- * Since: 0.3.0
  * Returns: The #GArrowType for each value of the array.
+ *
+ * Since: 0.3.0
  */
 GArrowType
 garrow_array_get_value_type(GArrowArray *array)

From ab520cbc7a1e3fe14a2290322ca2e392af30d612 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 11 Apr 2017 13:56:41 -0400
Subject: [PATCH 0501/1644] ARROW-808: [GLib] Remove needless ignore entries

Author: Kouhei Sutou <kou@clear-code.com>

Closes #527 from kou/glib-remove-needless-ignore-entries and squashes the following commits:

57f734c [Kouhei Sutou] [GLib] Remove needless ignore entries
---
 c_glib/.gitignore | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/c_glib/.gitignore b/c_glib/.gitignore
index 8928158f6ca3a..6f2de80d4f79e 100644
--- a/c_glib/.gitignore
+++ b/c_glib/.gitignore
@@ -36,10 +36,6 @@ Makefile.in
 /version
 /arrow-glib/enums.c
 /arrow-glib/enums.h
-/arrow-glib/io-enums.c
-/arrow-glib/io-enums.h
-/arrow-glib/ipc-enums.c
-/arrow-glib/ipc-enums.h
 /arrow-glib/stamp-*
 /arrow-glib/*.pc
 /example/build

From 5e5a5878d7be62e0ae26ca0b45b4aafd761eb43d Mon Sep 17 00:00:00 2001
From: Leif Walsh <leif.walsh@gmail.com>
Date: Tue, 11 Apr 2017 19:04:21 -0400
Subject: [PATCH 0502/1644] ARROW-805: [C++] Don't throw IOError when listing
 empty HDFS dir

Author: Leif Walsh <leif.walsh@gmail.com>

Closes #528 from leifwalsh/ARROW-805 and squashes the following commits:

4e1bb05 [Leif Walsh] ARROW-805: [C++] Don't throw IOError when listing empty HDFS dir
---
 cpp/src/arrow/io/hdfs.cc         | 7 +++++--
 cpp/src/arrow/io/io-hdfs-test.cc | 4 ++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 408b85f8daeb7..3510ba183d8e4 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -406,8 +406,11 @@ class HdfsClient::HdfsClientImpl {
       // errno indicates error
       //
       // Note: errno is thread-locala
-      if (errno == 0) { num_entries = 0; }
-      { return Status::IOError("HDFS: list directory failed"); }
+      if (errno == 0) {
+        num_entries = 0;
+      } else {
+        return Status::IOError("HDFS: list directory failed");
+      }
     }
 
     // Allocate additional space for elements
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index a2c9c5210b10d..0a9f5d9885e19 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -170,8 +170,12 @@ TYPED_TEST(TestHdfsClient, CreateDirectory) {
 
   ASSERT_OK(this->client_->CreateDirectory(path));
   ASSERT_TRUE(this->client_->Exists(path));
+  std::vector<HdfsPathInfo> listing;
+  EXPECT_OK(this->client_->ListDirectory(path, &listing));
+  ASSERT_EQ(0, listing.size());
   EXPECT_OK(this->client_->Delete(path, true));
   ASSERT_FALSE(this->client_->Exists(path));
+  ASSERT_RAISES(IOError, this->client_->ListDirectory(path, &listing));
 }
 
 TYPED_TEST(TestHdfsClient, GetCapacityUsed) {

From 6443b82878489ed6a308d1e5ace33088788a060e Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 12 Apr 2017 10:54:42 -0400
Subject: [PATCH 0503/1644] ARROW-810: [GLib] Remove io/ipc prefix

Author: Kouhei Sutou <kou@clear-code.com>

Closes #530 from kou/glib-remove-io-ipc-prefix and squashes the following commits:

adfad7c [Kouhei Sutou] [GLib] Remove io/ipc prefix
---
 c_glib/arrow-glib/Makefile.am                 |  90 +++---
 c_glib/arrow-glib/arrow-glib.h                |  30 +-
 c_glib/arrow-glib/arrow-glib.hpp              |  28 +-
 .../{io-file-mode.cpp => file-mode.cpp}       |  28 +-
 .../{io-file-mode.h => file-mode.h}           |  16 +-
 .../{io-file-mode.hpp => file-mode.hpp}       |   6 +-
 ...tput-stream.cpp => file-output-stream.cpp} | 120 ++++----
 ...e-output-stream.h => file-output-stream.h} |  44 +--
 ...mapped-file.hpp => file-output-stream.hpp} |   6 +-
 .../{ipc-file-reader.cpp => file-reader.cpp}  | 110 +++----
 c_glib/arrow-glib/file-reader.h               |  83 +++++
 .../{ipc-file-writer.hpp => file-reader.hpp}  |   6 +-
 .../{ipc-file-writer.cpp => file-writer.cpp}  |  58 ++--
 .../{ipc-file-writer.h => file-writer.h}      |  52 ++--
 .../{ipc-file-reader.hpp => file-writer.hpp}  |   6 +-
 c_glib/arrow-glib/{io-file.cpp => file.cpp}   |  48 +--
 c_glib/arrow-glib/{io-file.h => file.h}       |  34 +--
 c_glib/arrow-glib/{io-file.hpp => file.hpp}   |  10 +-
 .../{io-input-stream.cpp => input-stream.cpp} |  18 +-
 .../{io-input-stream.h => input-stream.h}     |  26 +-
 .../{io-input-stream.hpp => input-stream.hpp} |  10 +-
 c_glib/arrow-glib/io-memory-mapped-file.cpp   | 287 ------------------
 c_glib/arrow-glib/ipc-file-reader.h           |  83 -----
 c_glib/arrow-glib/ipc-stream-reader.h         |  80 -----
 c_glib/arrow-glib/memory-mapped-file.cpp      | 287 ++++++++++++++++++
 ...ory-mapped-file.h => memory-mapped-file.h} |  48 +--
 ...tput-stream.hpp => memory-mapped-file.hpp} |   6 +-
 ...adata-version.cpp => metadata-version.cpp} |  28 +-
 ...-metadata-version.h => metadata-version.h} |  16 +-
 ...adata-version.hpp => metadata-version.hpp} |   6 +-
 ...io-output-stream.cpp => output-stream.cpp} |  18 +-
 .../{io-output-stream.h => output-stream.h}   |  26 +-
 ...io-output-stream.hpp => output-stream.hpp} |  10 +-
 ...access-file.cpp => random-access-file.cpp} |  42 +--
 ...dom-access-file.h => random-access-file.h} |  32 +-
 ...access-file.hpp => random-access-file.hpp} |  10 +-
 .../{io-readable.cpp => readable.cpp}         |  26 +-
 .../arrow-glib/{io-readable.h => readable.h}  |  28 +-
 .../{io-readable.hpp => readable.hpp}         |  10 +-
 ...pc-stream-reader.cpp => stream-reader.cpp} |  90 +++---
 c_glib/arrow-glib/stream-reader.h             |  80 +++++
 ...pc-stream-reader.hpp => stream-reader.hpp} |   6 +-
 ...pc-stream-writer.cpp => stream-writer.cpp} |  88 +++---
 .../{ipc-stream-writer.h => stream-writer.h}  |  50 +--
 ...pc-stream-writer.hpp => stream-writer.hpp} |   6 +-
 ...-writeable-file.cpp => writeable-file.cpp} |  26 +-
 .../{io-writeable-file.h => writeable-file.h} |  28 +-
 ...-writeable-file.hpp => writeable-file.hpp} |  10 +-
 .../{io-writeable.cpp => writeable.cpp}       |  34 +--
 .../{io-writeable.h => writeable.h}           |  30 +-
 .../{io-writeable.hpp => writeable.hpp}       |  10 +-
 c_glib/doc/reference/Makefile.am              |   2 +-
 c_glib/doc/reference/arrow-glib-docs.sgml     |  62 ++--
 c_glib/example/read-batch.c                   |  18 +-
 c_glib/example/read-stream.c                  |  16 +-
 ...t-stream.rb => test-file-output-stream.rb} |   6 +-
 ...ipc-file-writer.rb => test-file-writer.rb} |  10 +-
 ...ped-file.rb => test-memory-mapped-file.rb} |  22 +-
 ...stream-writer.rb => test-stream-writer.rb} |  10 +-
 59 files changed, 1238 insertions(+), 1238 deletions(-)
 rename c_glib/arrow-glib/{io-file-mode.cpp => file-mode.cpp} (72%)
 rename c_glib/arrow-glib/{io-file-mode.h => file-mode.h} (78%)
 rename c_glib/arrow-glib/{io-file-mode.hpp => file-mode.hpp} (81%)
 rename c_glib/arrow-glib/{io-file-output-stream.cpp => file-output-stream.cpp} (52%)
 rename c_glib/arrow-glib/{io-file-output-stream.h => file-output-stream.h} (52%)
 rename c_glib/arrow-glib/{io-memory-mapped-file.hpp => file-output-stream.hpp} (73%)
 rename c_glib/arrow-glib/{ipc-file-reader.cpp => file-reader.cpp} (60%)
 create mode 100644 c_glib/arrow-glib/file-reader.h
 rename c_glib/arrow-glib/{ipc-file-writer.hpp => file-reader.hpp} (78%)
 rename c_glib/arrow-glib/{ipc-file-writer.cpp => file-writer.cpp} (68%)
 rename c_glib/arrow-glib/{ipc-file-writer.h => file-writer.h} (52%)
 rename c_glib/arrow-glib/{ipc-file-reader.hpp => file-writer.hpp} (77%)
 rename c_glib/arrow-glib/{io-file.cpp => file.cpp} (68%)
 rename c_glib/arrow-glib/{io-file.h => file.h} (55%)
 rename c_glib/arrow-glib/{io-file.hpp => file.hpp} (79%)
 rename c_glib/arrow-glib/{io-input-stream.cpp => input-stream.cpp} (71%)
 rename c_glib/arrow-glib/{io-input-stream.h => input-stream.h} (57%)
 rename c_glib/arrow-glib/{io-input-stream.hpp => input-stream.hpp} (76%)
 delete mode 100644 c_glib/arrow-glib/io-memory-mapped-file.cpp
 delete mode 100644 c_glib/arrow-glib/ipc-file-reader.h
 delete mode 100644 c_glib/arrow-glib/ipc-stream-reader.h
 create mode 100644 c_glib/arrow-glib/memory-mapped-file.cpp
 rename c_glib/arrow-glib/{io-memory-mapped-file.h => memory-mapped-file.h} (51%)
 rename c_glib/arrow-glib/{io-file-output-stream.hpp => memory-mapped-file.hpp} (73%)
 rename c_glib/arrow-glib/{ipc-metadata-version.cpp => metadata-version.cpp} (68%)
 rename c_glib/arrow-glib/{ipc-metadata-version.h => metadata-version.h} (76%)
 rename c_glib/arrow-glib/{ipc-metadata-version.hpp => metadata-version.hpp} (77%)
 rename c_glib/arrow-glib/{io-output-stream.cpp => output-stream.cpp} (71%)
 rename c_glib/arrow-glib/{io-output-stream.h => output-stream.h} (57%)
 rename c_glib/arrow-glib/{io-output-stream.hpp => output-stream.hpp} (75%)
 rename c_glib/arrow-glib/{io-random-access-file.cpp => random-access-file.cpp} (70%)
 rename c_glib/arrow-glib/{io-random-access-file.h => random-access-file.h} (57%)
 rename c_glib/arrow-glib/{io-random-access-file.hpp => random-access-file.hpp} (78%)
 rename c_glib/arrow-glib/{io-readable.cpp => readable.cpp} (75%)
 rename c_glib/arrow-glib/{io-readable.h => readable.h} (60%)
 rename c_glib/arrow-glib/{io-readable.hpp => readable.hpp} (77%)
 rename c_glib/arrow-glib/{ipc-stream-reader.cpp => stream-reader.cpp} (63%)
 create mode 100644 c_glib/arrow-glib/stream-reader.h
 rename c_glib/arrow-glib/{ipc-stream-reader.hpp => stream-reader.hpp} (75%)
 rename c_glib/arrow-glib/{ipc-stream-writer.cpp => stream-writer.cpp} (65%)
 rename c_glib/arrow-glib/{ipc-stream-writer.h => stream-writer.h} (54%)
 rename c_glib/arrow-glib/{ipc-stream-writer.hpp => stream-writer.hpp} (75%)
 rename c_glib/arrow-glib/{io-writeable-file.cpp => writeable-file.cpp} (73%)
 rename c_glib/arrow-glib/{io-writeable-file.h => writeable-file.h} (59%)
 rename c_glib/arrow-glib/{io-writeable-file.hpp => writeable-file.hpp} (75%)
 rename c_glib/arrow-glib/{io-writeable.cpp => writeable.cpp} (72%)
 rename c_glib/arrow-glib/{io-writeable.h => writeable.h} (58%)
 rename c_glib/arrow-glib/{io-writeable.hpp => writeable.hpp} (77%)
 rename c_glib/test/{test-io-file-output-stream.rb => test-file-output-stream.rb} (87%)
 rename c_glib/test/{test-ipc-file-writer.rb => test-file-writer.rb} (82%)
 rename c_glib/test/{test-io-memory-mapped-file.rb => test-memory-mapped-file.rb} (81%)
 rename c_glib/test/{test-ipc-stream-writer.rb => test-stream-writer.rb} (84%)

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index e719cccfa85ab..387707c7d5897 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -102,23 +102,23 @@ libarrow_glib_la_headers =			\
 	uint64-data-type.h
 
 libarrow_glib_la_headers +=			\
-	io-file.h				\
-	io-file-mode.h				\
-	io-file-output-stream.h			\
-	io-input-stream.h			\
-	io-memory-mapped-file.h			\
-	io-output-stream.h			\
-	io-random-access-file.h			\
-	io-readable.h				\
-	io-writeable.h				\
-	io-writeable-file.h
+	file.h					\
+	file-mode.h				\
+	file-output-stream.h			\
+	input-stream.h				\
+	memory-mapped-file.h			\
+	output-stream.h				\
+	random-access-file.h			\
+	readable.h				\
+	writeable.h				\
+	writeable-file.h
 
 libarrow_glib_la_headers +=			\
-	ipc-file-reader.h			\
-	ipc-file-writer.h			\
-	ipc-stream-reader.h			\
-	ipc-stream-writer.h			\
-	ipc-metadata-version.h
+	file-reader.h				\
+	file-writer.h				\
+	stream-reader.h				\
+	stream-writer.h				\
+	metadata-version.h
 
 libarrow_glib_la_generated_headers =		\
 	enums.h
@@ -190,23 +190,23 @@ libarrow_glib_la_sources =			\
 	$(libarrow_glib_la_generated_sources)
 
 libarrow_glib_la_sources +=			\
-	io-file.cpp				\
-	io-file-mode.cpp			\
-	io-file-output-stream.cpp		\
-	io-input-stream.cpp			\
-	io-memory-mapped-file.cpp		\
-	io-output-stream.cpp			\
-	io-random-access-file.cpp		\
-	io-readable.cpp				\
-	io-writeable.cpp			\
-	io-writeable-file.cpp
+	file.cpp				\
+	file-mode.cpp				\
+	file-output-stream.cpp			\
+	input-stream.cpp			\
+	memory-mapped-file.cpp			\
+	output-stream.cpp			\
+	random-access-file.cpp			\
+	readable.cpp				\
+	writeable.cpp				\
+	writeable-file.cpp
 
 libarrow_glib_la_sources +=			\
-	ipc-file-reader.cpp			\
-	ipc-file-writer.cpp			\
-	ipc-metadata-version.cpp		\
-	ipc-stream-reader.cpp			\
-	ipc-stream-writer.cpp
+	file-reader.cpp				\
+	file-writer.cpp				\
+	metadata-version.cpp			\
+	stream-reader.cpp			\
+	stream-writer.cpp
 
 libarrow_glib_la_cpp_headers =			\
 	array.hpp				\
@@ -223,23 +223,23 @@ libarrow_glib_la_cpp_headers =			\
 	type.hpp
 
 libarrow_glib_la_cpp_headers +=			\
-	io-file.hpp				\
-	io-file-mode.hpp			\
-	io-file-output-stream.hpp		\
-	io-input-stream.hpp			\
-	io-memory-mapped-file.hpp		\
-	io-output-stream.hpp			\
-	io-random-access-file.hpp		\
-	io-readable.hpp				\
-	io-writeable.hpp			\
-	io-writeable-file.hpp
+	file.hpp				\
+	file-mode.hpp				\
+	file-output-stream.hpp			\
+	input-stream.hpp			\
+	memory-mapped-file.hpp			\
+	output-stream.hpp			\
+	random-access-file.hpp			\
+	readable.hpp				\
+	writeable.hpp				\
+	writeable-file.hpp
 
 libarrow_glib_la_cpp_headers +=			\
-	ipc-file-reader.hpp			\
-	ipc-file-writer.hpp			\
-	ipc-metadata-version.hpp		\
-	ipc-stream-reader.hpp			\
-	ipc-stream-writer.hpp
+	file-reader.hpp				\
+	file-writer.hpp				\
+	metadata-version.hpp			\
+	stream-reader.hpp			\
+	stream-writer.hpp
 
 libarrow_glib_la_SOURCES =			\
 	$(libarrow_glib_la_sources)		\
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index 9b03175799f44..b15c56f7bb486 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -79,19 +79,19 @@
 #include <arrow-glib/uint64-array-builder.h>
 #include <arrow-glib/uint64-data-type.h>
 
-#include <arrow-glib/io-file.h>
-#include <arrow-glib/io-file-mode.h>
-#include <arrow-glib/io-file-output-stream.h>
-#include <arrow-glib/io-input-stream.h>
-#include <arrow-glib/io-memory-mapped-file.h>
-#include <arrow-glib/io-output-stream.h>
-#include <arrow-glib/io-random-access-file.h>
-#include <arrow-glib/io-readable.h>
-#include <arrow-glib/io-writeable.h>
-#include <arrow-glib/io-writeable-file.h>
+#include <arrow-glib/file.h>
+#include <arrow-glib/file-mode.h>
+#include <arrow-glib/file-output-stream.h>
+#include <arrow-glib/input-stream.h>
+#include <arrow-glib/memory-mapped-file.h>
+#include <arrow-glib/output-stream.h>
+#include <arrow-glib/random-access-file.h>
+#include <arrow-glib/readable.h>
+#include <arrow-glib/writeable.h>
+#include <arrow-glib/writeable-file.h>
 
-#include <arrow-glib/ipc-file-reader.h>
-#include <arrow-glib/ipc-file-writer.h>
-#include <arrow-glib/ipc-metadata-version.h>
-#include <arrow-glib/ipc-stream-reader.h>
-#include <arrow-glib/ipc-stream-writer.h>
+#include <arrow-glib/file-reader.h>
+#include <arrow-glib/file-writer.h>
+#include <arrow-glib/metadata-version.h>
+#include <arrow-glib/stream-reader.h>
+#include <arrow-glib/stream-writer.h>
diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
index fd59d4a1a9240..3404d4d212917 100644
--- a/c_glib/arrow-glib/arrow-glib.hpp
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -36,18 +36,18 @@
 #include <arrow-glib/table.hpp>
 #include <arrow-glib/type.hpp>
 
-#include <arrow-glib/io-file.hpp>
-#include <arrow-glib/io-file-mode.hpp>
-#include <arrow-glib/io-file-output-stream.hpp>
-#include <arrow-glib/io-input-stream.hpp>
-#include <arrow-glib/io-memory-mapped-file.hpp>
-#include <arrow-glib/io-output-stream.hpp>
-#include <arrow-glib/io-random-access-file.hpp>
-#include <arrow-glib/io-readable.hpp>
-#include <arrow-glib/io-writeable.hpp>
+#include <arrow-glib/file.hpp>
+#include <arrow-glib/file-mode.hpp>
+#include <arrow-glib/file-output-stream.hpp>
+#include <arrow-glib/input-stream.hpp>
+#include <arrow-glib/memory-mapped-file.hpp>
+#include <arrow-glib/output-stream.hpp>
+#include <arrow-glib/random-access-file.hpp>
+#include <arrow-glib/readable.hpp>
+#include <arrow-glib/writeable.hpp>
 
-#include <arrow-glib/ipc-file-reader.hpp>
-#include <arrow-glib/ipc-file-writer.hpp>
-#include <arrow-glib/ipc-metadata-version.hpp>
-#include <arrow-glib/ipc-stream-reader.hpp>
-#include <arrow-glib/ipc-stream-writer.hpp>
+#include <arrow-glib/file-reader.hpp>
+#include <arrow-glib/file-writer.hpp>
+#include <arrow-glib/metadata-version.hpp>
+#include <arrow-glib/stream-reader.hpp>
+#include <arrow-glib/stream-writer.hpp>
diff --git a/c_glib/arrow-glib/io-file-mode.cpp b/c_glib/arrow-glib/file-mode.cpp
similarity index 72%
rename from c_glib/arrow-glib/io-file-mode.cpp
rename to c_glib/arrow-glib/file-mode.cpp
index 7998d3f5bb061..1fb17062ab2c9 100644
--- a/c_glib/arrow-glib/io-file-mode.cpp
+++ b/c_glib/arrow-glib/file-mode.cpp
@@ -21,41 +21,41 @@
 #  include <config.h>
 #endif
 
-#include <arrow-glib/io-file-mode.hpp>
+#include <arrow-glib/file-mode.hpp>
 
 /**
- * SECTION: io-file-mode
- * @title: GArrowIOFileMode
+ * SECTION: file-mode
+ * @title: GArrowFileMode
  * @short_description: File mode mapping between Arrow and arrow-glib
  *
- * #GArrowIOFileMode provides file modes corresponding to
+ * #GArrowFileMode provides file modes corresponding to
  * `arrow::io::FileMode::type` values.
  */
 
-GArrowIOFileMode
-garrow_io_file_mode_from_raw(arrow::io::FileMode::type mode)
+GArrowFileMode
+garrow_file_mode_from_raw(arrow::io::FileMode::type mode)
 {
   switch (mode) {
   case arrow::io::FileMode::type::READ:
-    return GARROW_IO_FILE_MODE_READ;
+    return GARROW_FILE_MODE_READ;
   case arrow::io::FileMode::type::WRITE:
-    return GARROW_IO_FILE_MODE_WRITE;
+    return GARROW_FILE_MODE_WRITE;
   case arrow::io::FileMode::type::READWRITE:
-    return GARROW_IO_FILE_MODE_READWRITE;
+    return GARROW_FILE_MODE_READWRITE;
   default:
-    return GARROW_IO_FILE_MODE_READ;
+    return GARROW_FILE_MODE_READ;
   }
 }
 
 arrow::io::FileMode::type
-garrow_io_file_mode_to_raw(GArrowIOFileMode mode)
+garrow_file_mode_to_raw(GArrowFileMode mode)
 {
   switch (mode) {
-  case GARROW_IO_FILE_MODE_READ:
+  case GARROW_FILE_MODE_READ:
     return arrow::io::FileMode::type::READ;
-  case GARROW_IO_FILE_MODE_WRITE:
+  case GARROW_FILE_MODE_WRITE:
     return arrow::io::FileMode::type::WRITE;
-  case GARROW_IO_FILE_MODE_READWRITE:
+  case GARROW_FILE_MODE_READWRITE:
     return arrow::io::FileMode::type::READWRITE;
   default:
     return arrow::io::FileMode::type::READ;
diff --git a/c_glib/arrow-glib/io-file-mode.h b/c_glib/arrow-glib/file-mode.h
similarity index 78%
rename from c_glib/arrow-glib/io-file-mode.h
rename to c_glib/arrow-glib/file-mode.h
index 03eca353bbdbb..8812af805abd5 100644
--- a/c_glib/arrow-glib/io-file-mode.h
+++ b/c_glib/arrow-glib/file-mode.h
@@ -24,17 +24,17 @@
 G_BEGIN_DECLS
 
 /**
- * GArrowIOFileMode:
- * @GARROW_IO_FILE_MODE_READ: For read.
- * @GARROW_IO_FILE_MODE_WRITE: For write.
- * @GARROW_IO_FILE_MODE_READWRITE: For read-write.
+ * GArrowFileMode:
+ * @GARROW_FILE_MODE_READ: For read.
+ * @GARROW_FILE_MODE_WRITE: For write.
+ * @GARROW_FILE_MODE_READWRITE: For read-write.
  *
  * They are corresponding to `arrow::io::FileMode::type` values.
  */
 typedef enum {
-  GARROW_IO_FILE_MODE_READ,
-  GARROW_IO_FILE_MODE_WRITE,
-  GARROW_IO_FILE_MODE_READWRITE
-} GArrowIOFileMode;
+  GARROW_FILE_MODE_READ,
+  GARROW_FILE_MODE_WRITE,
+  GARROW_FILE_MODE_READWRITE
+} GArrowFileMode;
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/io-file-mode.hpp b/c_glib/arrow-glib/file-mode.hpp
similarity index 81%
rename from c_glib/arrow-glib/io-file-mode.hpp
rename to c_glib/arrow-glib/file-mode.hpp
index b3d8ac6d8e053..2b67379421d5a 100644
--- a/c_glib/arrow-glib/io-file-mode.hpp
+++ b/c_glib/arrow-glib/file-mode.hpp
@@ -21,7 +21,7 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-file-mode.h>
+#include <arrow-glib/file-mode.h>
 
-GArrowIOFileMode garrow_io_file_mode_from_raw(arrow::io::FileMode::type mode);
-arrow::io::FileMode::type garrow_io_file_mode_to_raw(GArrowIOFileMode mode);
+GArrowFileMode garrow_file_mode_from_raw(arrow::io::FileMode::type mode);
+arrow::io::FileMode::type garrow_file_mode_to_raw(GArrowFileMode mode);
diff --git a/c_glib/arrow-glib/io-file-output-stream.cpp b/c_glib/arrow-glib/file-output-stream.cpp
similarity index 52%
rename from c_glib/arrow-glib/io-file-output-stream.cpp
rename to c_glib/arrow-glib/file-output-stream.cpp
index 673e8cd36a60a..b6ca42a1d59da 100644
--- a/c_glib/arrow-glib/io-file-output-stream.cpp
+++ b/c_glib/arrow-glib/file-output-stream.cpp
@@ -24,23 +24,23 @@
 #include <arrow/io/file.h>
 
 #include <arrow-glib/error.hpp>
-#include <arrow-glib/io-file.hpp>
-#include <arrow-glib/io-file-output-stream.hpp>
-#include <arrow-glib/io-output-stream.hpp>
-#include <arrow-glib/io-writeable.hpp>
+#include <arrow-glib/file.hpp>
+#include <arrow-glib/file-output-stream.hpp>
+#include <arrow-glib/output-stream.hpp>
+#include <arrow-glib/writeable.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: io-file-output-stream
+ * SECTION: file-output-stream
  * @short_description: A file output stream.
  *
- * The #GArrowIOFileOutputStream is a class for file output stream.
+ * The #GArrowFileOutputStream is a class for file output stream.
  */
 
-typedef struct GArrowIOFileOutputStreamPrivate_ {
+typedef struct GArrowFileOutputStreamPrivate_ {
   std::shared_ptr<arrow::io::FileOutputStream> file_output_stream;
-} GArrowIOFileOutputStreamPrivate;
+} GArrowFileOutputStreamPrivate;
 
 enum {
   PROP_0,
@@ -48,87 +48,87 @@ enum {
 };
 
 static std::shared_ptr<arrow::io::FileInterface>
-garrow_io_file_output_stream_get_raw_file_interface(GArrowIOFile *file)
+garrow_file_output_stream_get_raw_file_interface(GArrowFile *file)
 {
-  auto file_output_stream = GARROW_IO_FILE_OUTPUT_STREAM(file);
+  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(file);
   auto arrow_file_output_stream =
-    garrow_io_file_output_stream_get_raw(file_output_stream);
+    garrow_file_output_stream_get_raw(file_output_stream);
   return arrow_file_output_stream;
 }
 
 static void
-garrow_io_file_interface_init(GArrowIOFileInterface *iface)
+garrow_file_interface_init(GArrowFileInterface *iface)
 {
-  iface->get_raw = garrow_io_file_output_stream_get_raw_file_interface;
+  iface->get_raw = garrow_file_output_stream_get_raw_file_interface;
 }
 
 static std::shared_ptr<arrow::io::Writeable>
-garrow_io_file_output_stream_get_raw_writeable_interface(GArrowIOWriteable *writeable)
+garrow_file_output_stream_get_raw_writeable_interface(GArrowWriteable *writeable)
 {
-  auto file_output_stream = GARROW_IO_FILE_OUTPUT_STREAM(writeable);
+  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(writeable);
   auto arrow_file_output_stream =
-    garrow_io_file_output_stream_get_raw(file_output_stream);
+    garrow_file_output_stream_get_raw(file_output_stream);
   return arrow_file_output_stream;
 }
 
 static void
-garrow_io_writeable_interface_init(GArrowIOWriteableInterface *iface)
+garrow_writeable_interface_init(GArrowWriteableInterface *iface)
 {
-  iface->get_raw = garrow_io_file_output_stream_get_raw_writeable_interface;
+  iface->get_raw = garrow_file_output_stream_get_raw_writeable_interface;
 }
 
 static std::shared_ptr<arrow::io::OutputStream>
-garrow_io_file_output_stream_get_raw_output_stream_interface(GArrowIOOutputStream *output_stream)
+garrow_file_output_stream_get_raw_output_stream_interface(GArrowOutputStream *output_stream)
 {
-  auto file_output_stream = GARROW_IO_FILE_OUTPUT_STREAM(output_stream);
+  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(output_stream);
   auto arrow_file_output_stream =
-    garrow_io_file_output_stream_get_raw(file_output_stream);
+    garrow_file_output_stream_get_raw(file_output_stream);
   return arrow_file_output_stream;
 }
 
 static void
-garrow_io_output_stream_interface_init(GArrowIOOutputStreamInterface *iface)
+garrow_output_stream_interface_init(GArrowOutputStreamInterface *iface)
 {
-  iface->get_raw = garrow_io_file_output_stream_get_raw_output_stream_interface;
+  iface->get_raw = garrow_file_output_stream_get_raw_output_stream_interface;
 }
 
-G_DEFINE_TYPE_WITH_CODE(GArrowIOFileOutputStream,
-                        garrow_io_file_output_stream,
+G_DEFINE_TYPE_WITH_CODE(GArrowFileOutputStream,
+                        garrow_file_output_stream,
                         G_TYPE_OBJECT,
-                        G_ADD_PRIVATE(GArrowIOFileOutputStream)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_FILE,
-                                              garrow_io_file_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_WRITEABLE,
-                                              garrow_io_writeable_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_OUTPUT_STREAM,
-                                              garrow_io_output_stream_interface_init));
-
-#define GARROW_IO_FILE_OUTPUT_STREAM_GET_PRIVATE(obj)                   \
+                        G_ADD_PRIVATE(GArrowFileOutputStream)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_FILE,
+                                              garrow_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE,
+                                              garrow_writeable_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_OUTPUT_STREAM,
+                                              garrow_output_stream_interface_init));
+
+#define GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(obj)                   \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
-                               GARROW_IO_TYPE_FILE_OUTPUT_STREAM,       \
-                               GArrowIOFileOutputStreamPrivate))
+                               GARROW_TYPE_FILE_OUTPUT_STREAM,       \
+                               GArrowFileOutputStreamPrivate))
 
 static void
-garrow_io_file_output_stream_finalize(GObject *object)
+garrow_file_output_stream_finalize(GObject *object)
 {
-  GArrowIOFileOutputStreamPrivate *priv;
+  GArrowFileOutputStreamPrivate *priv;
 
-  priv = GARROW_IO_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
+  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
 
   priv->file_output_stream = nullptr;
 
-  G_OBJECT_CLASS(garrow_io_file_output_stream_parent_class)->finalize(object);
+  G_OBJECT_CLASS(garrow_file_output_stream_parent_class)->finalize(object);
 }
 
 static void
-garrow_io_file_output_stream_set_property(GObject *object,
+garrow_file_output_stream_set_property(GObject *object,
                                           guint prop_id,
                                           const GValue *value,
                                           GParamSpec *pspec)
 {
-  GArrowIOFileOutputStreamPrivate *priv;
+  GArrowFileOutputStreamPrivate *priv;
 
-  priv = GARROW_IO_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
+  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
 
   switch (prop_id) {
   case PROP_FILE_OUTPUT_STREAM:
@@ -142,7 +142,7 @@ garrow_io_file_output_stream_set_property(GObject *object,
 }
 
 static void
-garrow_io_file_output_stream_get_property(GObject *object,
+garrow_file_output_stream_get_property(GObject *object,
                                           guint prop_id,
                                           GValue *value,
                                           GParamSpec *pspec)
@@ -155,21 +155,21 @@ garrow_io_file_output_stream_get_property(GObject *object,
 }
 
 static void
-garrow_io_file_output_stream_init(GArrowIOFileOutputStream *object)
+garrow_file_output_stream_init(GArrowFileOutputStream *object)
 {
 }
 
 static void
-garrow_io_file_output_stream_class_init(GArrowIOFileOutputStreamClass *klass)
+garrow_file_output_stream_class_init(GArrowFileOutputStreamClass *klass)
 {
   GObjectClass *gobject_class;
   GParamSpec *spec;
 
   gobject_class = G_OBJECT_CLASS(klass);
 
-  gobject_class->finalize     = garrow_io_file_output_stream_finalize;
-  gobject_class->set_property = garrow_io_file_output_stream_set_property;
-  gobject_class->get_property = garrow_io_file_output_stream_get_property;
+  gobject_class->finalize     = garrow_file_output_stream_finalize;
+  gobject_class->set_property = garrow_file_output_stream_set_property;
+  gobject_class->get_property = garrow_file_output_stream_get_property;
 
   spec = g_param_spec_pointer("file-output-stream",
                               "io::FileOutputStream",
@@ -180,16 +180,16 @@ garrow_io_file_output_stream_class_init(GArrowIOFileOutputStreamClass *klass)
 }
 
 /**
- * garrow_io_file_output_stream_open:
+ * garrow_file_output_stream_open:
  * @path: The path of the file output stream.
  * @append: Whether the path is opened as append mode or recreate mode.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowIOFileOutputStream or %NULL on error.
+ *   #GArrowFileOutputStream or %NULL on error.
  */
-GArrowIOFileOutputStream *
-garrow_io_file_output_stream_open(const gchar *path,
+GArrowFileOutputStream *
+garrow_file_output_stream_open(const gchar *path,
                                   gboolean append,
                                   GError **error)
 {
@@ -199,7 +199,7 @@ garrow_io_file_output_stream_open(const gchar *path,
                                       append,
                                       &arrow_file_output_stream);
   if (status.ok()) {
-    return garrow_io_file_output_stream_new_raw(&arrow_file_output_stream);
+    return garrow_file_output_stream_new_raw(&arrow_file_output_stream);
   } else {
     std::string context("[io][file-output-stream][open]: <");
     context += path;
@@ -211,21 +211,21 @@ garrow_io_file_output_stream_open(const gchar *path,
 
 G_END_DECLS
 
-GArrowIOFileOutputStream *
-garrow_io_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream)
+GArrowFileOutputStream *
+garrow_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream)
 {
   auto file_output_stream =
-    GARROW_IO_FILE_OUTPUT_STREAM(g_object_new(GARROW_IO_TYPE_FILE_OUTPUT_STREAM,
+    GARROW_FILE_OUTPUT_STREAM(g_object_new(GARROW_TYPE_FILE_OUTPUT_STREAM,
                                               "file-output-stream", arrow_file_output_stream,
                                               NULL));
   return file_output_stream;
 }
 
 std::shared_ptr<arrow::io::FileOutputStream>
-garrow_io_file_output_stream_get_raw(GArrowIOFileOutputStream *file_output_stream)
+garrow_file_output_stream_get_raw(GArrowFileOutputStream *file_output_stream)
 {
-  GArrowIOFileOutputStreamPrivate *priv;
+  GArrowFileOutputStreamPrivate *priv;
 
-  priv = GARROW_IO_FILE_OUTPUT_STREAM_GET_PRIVATE(file_output_stream);
+  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(file_output_stream);
   return priv->file_output_stream;
 }
diff --git a/c_glib/arrow-glib/io-file-output-stream.h b/c_glib/arrow-glib/file-output-stream.h
similarity index 52%
rename from c_glib/arrow-glib/io-file-output-stream.h
rename to c_glib/arrow-glib/file-output-stream.h
index 032b125544e77..bef3700039921 100644
--- a/c_glib/arrow-glib/io-file-output-stream.h
+++ b/c_glib/arrow-glib/file-output-stream.h
@@ -23,49 +23,49 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_FILE_OUTPUT_STREAM       \
-  (garrow_io_file_output_stream_get_type())
-#define GARROW_IO_FILE_OUTPUT_STREAM(obj)                               \
+#define GARROW_TYPE_FILE_OUTPUT_STREAM       \
+  (garrow_file_output_stream_get_type())
+#define GARROW_FILE_OUTPUT_STREAM(obj)                               \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
-                              GARROW_IO_TYPE_FILE_OUTPUT_STREAM,        \
-                              GArrowIOFileOutputStream))
-#define GARROW_IO_FILE_OUTPUT_STREAM_CLASS(klass)               \
+                              GARROW_TYPE_FILE_OUTPUT_STREAM,        \
+                              GArrowFileOutputStream))
+#define GARROW_FILE_OUTPUT_STREAM_CLASS(klass)               \
   (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_IO_TYPE_FILE_OUTPUT_STREAM,   \
-                           GArrowIOFileOutputStreamClass))
-#define GARROW_IO_IS_FILE_OUTPUT_STREAM(obj)                            \
+                           GARROW_TYPE_FILE_OUTPUT_STREAM,   \
+                           GArrowFileOutputStreamClass))
+#define GARROW_IS_FILE_OUTPUT_STREAM(obj)                            \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_IO_TYPE_FILE_OUTPUT_STREAM))
-#define GARROW_IO_IS_FILE_OUTPUT_STREAM_CLASS(klass)            \
+                              GARROW_TYPE_FILE_OUTPUT_STREAM))
+#define GARROW_IS_FILE_OUTPUT_STREAM_CLASS(klass)            \
   (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_IO_TYPE_FILE_OUTPUT_STREAM))
-#define GARROW_IO_FILE_OUTPUT_STREAM_GET_CLASS(obj)             \
+                           GARROW_TYPE_FILE_OUTPUT_STREAM))
+#define GARROW_FILE_OUTPUT_STREAM_GET_CLASS(obj)             \
   (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_IO_TYPE_FILE_OUTPUT_STREAM, \
-                             GArrowIOFileOutputStreamClass))
+                             GARROW_TYPE_FILE_OUTPUT_STREAM, \
+                             GArrowFileOutputStreamClass))
 
-typedef struct _GArrowIOFileOutputStream         GArrowIOFileOutputStream;
-typedef struct _GArrowIOFileOutputStreamClass    GArrowIOFileOutputStreamClass;
+typedef struct _GArrowFileOutputStream         GArrowFileOutputStream;
+typedef struct _GArrowFileOutputStreamClass    GArrowFileOutputStreamClass;
 
 /**
- * GArrowIOFileOutputStream:
+ * GArrowFileOutputStream:
  *
  * It wraps `arrow::io::FileOutputStream`.
  */
-struct _GArrowIOFileOutputStream
+struct _GArrowFileOutputStream
 {
   /*< private >*/
   GObject parent_instance;
 };
 
-struct _GArrowIOFileOutputStreamClass
+struct _GArrowFileOutputStreamClass
 {
   GObjectClass parent_class;
 };
 
-GType garrow_io_file_output_stream_get_type(void) G_GNUC_CONST;
+GType garrow_file_output_stream_get_type(void) G_GNUC_CONST;
 
-GArrowIOFileOutputStream *garrow_io_file_output_stream_open(const gchar *path,
+GArrowFileOutputStream *garrow_file_output_stream_open(const gchar *path,
                                                             gboolean append,
                                                             GError **error);
 
diff --git a/c_glib/arrow-glib/io-memory-mapped-file.hpp b/c_glib/arrow-glib/file-output-stream.hpp
similarity index 73%
rename from c_glib/arrow-glib/io-memory-mapped-file.hpp
rename to c_glib/arrow-glib/file-output-stream.hpp
index b48e05f2f9e7b..0b10418cdf176 100644
--- a/c_glib/arrow-glib/io-memory-mapped-file.hpp
+++ b/c_glib/arrow-glib/file-output-stream.hpp
@@ -22,7 +22,7 @@
 #include <arrow/api.h>
 #include <arrow/io/file.h>
 
-#include <arrow-glib/io-memory-mapped-file.h>
+#include <arrow-glib/file-output-stream.h>
 
-GArrowIOMemoryMappedFile *garrow_io_memory_mapped_file_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file);
-std::shared_ptr<arrow::io::MemoryMappedFile> garrow_io_memory_mapped_file_get_raw(GArrowIOMemoryMappedFile *memory_mapped_file);
+GArrowFileOutputStream *garrow_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream);
+std::shared_ptr<arrow::io::FileOutputStream> garrow_file_output_stream_get_raw(GArrowFileOutputStream *file_output_stream);
diff --git a/c_glib/arrow-glib/ipc-file-reader.cpp b/c_glib/arrow-glib/file-reader.cpp
similarity index 60%
rename from c_glib/arrow-glib/ipc-file-reader.cpp
rename to c_glib/arrow-glib/file-reader.cpp
index 223be857d9beb..c2aeabe5eed21 100644
--- a/c_glib/arrow-glib/ipc-file-reader.cpp
+++ b/c_glib/arrow-glib/file-reader.cpp
@@ -27,59 +27,59 @@
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 
-#include <arrow-glib/io-random-access-file.hpp>
+#include <arrow-glib/random-access-file.hpp>
 
-#include <arrow-glib/ipc-file-reader.hpp>
-#include <arrow-glib/ipc-metadata-version.hpp>
+#include <arrow-glib/file-reader.hpp>
+#include <arrow-glib/metadata-version.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: ipc-file-reader
+ * SECTION: file-reader
  * @short_description: File reader class
  *
- * #GArrowIPCFileReader is a class for receiving data by file based IPC.
+ * #GArrowFileReader is a class for receiving data by file based IPC.
  */
 
-typedef struct GArrowIPCFileReaderPrivate_ {
+typedef struct GArrowFileReaderPrivate_ {
   std::shared_ptr<arrow::ipc::FileReader> file_reader;
-} GArrowIPCFileReaderPrivate;
+} GArrowFileReaderPrivate;
 
 enum {
   PROP_0,
   PROP_FILE_READER
 };
 
-G_DEFINE_TYPE_WITH_PRIVATE(GArrowIPCFileReader,
-                           garrow_ipc_file_reader,
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowFileReader,
+                           garrow_file_reader,
                            G_TYPE_OBJECT);
 
-#define GARROW_IPC_FILE_READER_GET_PRIVATE(obj)                         \
+#define GARROW_FILE_READER_GET_PRIVATE(obj)                         \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
-                               GARROW_IPC_TYPE_FILE_READER,             \
-                               GArrowIPCFileReaderPrivate))
+                               GARROW_TYPE_FILE_READER,             \
+                               GArrowFileReaderPrivate))
 
 static void
-garrow_ipc_file_reader_finalize(GObject *object)
+garrow_file_reader_finalize(GObject *object)
 {
-  GArrowIPCFileReaderPrivate *priv;
+  GArrowFileReaderPrivate *priv;
 
-  priv = GARROW_IPC_FILE_READER_GET_PRIVATE(object);
+  priv = GARROW_FILE_READER_GET_PRIVATE(object);
 
   priv->file_reader = nullptr;
 
-  G_OBJECT_CLASS(garrow_ipc_file_reader_parent_class)->finalize(object);
+  G_OBJECT_CLASS(garrow_file_reader_parent_class)->finalize(object);
 }
 
 static void
-garrow_ipc_file_reader_set_property(GObject *object,
+garrow_file_reader_set_property(GObject *object,
                                     guint prop_id,
                                     const GValue *value,
                                     GParamSpec *pspec)
 {
-  GArrowIPCFileReaderPrivate *priv;
+  GArrowFileReaderPrivate *priv;
 
-  priv = GARROW_IPC_FILE_READER_GET_PRIVATE(object);
+  priv = GARROW_FILE_READER_GET_PRIVATE(object);
 
   switch (prop_id) {
   case PROP_FILE_READER:
@@ -93,7 +93,7 @@ garrow_ipc_file_reader_set_property(GObject *object,
 }
 
 static void
-garrow_ipc_file_reader_get_property(GObject *object,
+garrow_file_reader_get_property(GObject *object,
                                           guint prop_id,
                                           GValue *value,
                                           GParamSpec *pspec)
@@ -106,21 +106,21 @@ garrow_ipc_file_reader_get_property(GObject *object,
 }
 
 static void
-garrow_ipc_file_reader_init(GArrowIPCFileReader *object)
+garrow_file_reader_init(GArrowFileReader *object)
 {
 }
 
 static void
-garrow_ipc_file_reader_class_init(GArrowIPCFileReaderClass *klass)
+garrow_file_reader_class_init(GArrowFileReaderClass *klass)
 {
   GObjectClass *gobject_class;
   GParamSpec *spec;
 
   gobject_class = G_OBJECT_CLASS(klass);
 
-  gobject_class->finalize     = garrow_ipc_file_reader_finalize;
-  gobject_class->set_property = garrow_ipc_file_reader_set_property;
-  gobject_class->get_property = garrow_ipc_file_reader_get_property;
+  gobject_class->finalize     = garrow_file_reader_finalize;
+  gobject_class->set_property = garrow_file_reader_set_property;
+  gobject_class->get_property = garrow_file_reader_get_property;
 
   spec = g_param_spec_pointer("file-reader",
                               "ipc::FileReader",
@@ -131,23 +131,23 @@ garrow_ipc_file_reader_class_init(GArrowIPCFileReaderClass *klass)
 }
 
 /**
- * garrow_ipc_file_reader_open:
+ * garrow_file_reader_open:
  * @file: The file to be read.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowIPCFileReader or %NULL on error.
+ *   #GArrowFileReader or %NULL on error.
  */
-GArrowIPCFileReader *
-garrow_ipc_file_reader_open(GArrowIORandomAccessFile *file,
+GArrowFileReader *
+garrow_file_reader_open(GArrowRandomAccessFile *file,
                             GError **error)
 {
   std::shared_ptr<arrow::ipc::FileReader> arrow_file_reader;
   auto status =
-    arrow::ipc::FileReader::Open(garrow_io_random_access_file_get_raw(file),
+    arrow::ipc::FileReader::Open(garrow_random_access_file_get_raw(file),
                                  &arrow_file_reader);
   if (status.ok()) {
-    return garrow_ipc_file_reader_new_raw(&arrow_file_reader);
+    return garrow_file_reader_new_raw(&arrow_file_reader);
   } else {
     garrow_error_set(error, status, "[ipc][file-reader][open]");
     return NULL;
@@ -155,52 +155,52 @@ garrow_ipc_file_reader_open(GArrowIORandomAccessFile *file,
 }
 
 /**
- * garrow_ipc_file_reader_get_schema:
- * @file_reader: A #GArrowIPCFileReader.
+ * garrow_file_reader_get_schema:
+ * @file_reader: A #GArrowFileReader.
  *
  * Returns: (transfer full): The schema in the file.
  */
 GArrowSchema *
-garrow_ipc_file_reader_get_schema(GArrowIPCFileReader *file_reader)
+garrow_file_reader_get_schema(GArrowFileReader *file_reader)
 {
   auto arrow_file_reader =
-    garrow_ipc_file_reader_get_raw(file_reader);
+    garrow_file_reader_get_raw(file_reader);
   auto arrow_schema = arrow_file_reader->schema();
   return garrow_schema_new_raw(&arrow_schema);
 }
 
 /**
- * garrow_ipc_file_reader_get_n_record_batches:
- * @file_reader: A #GArrowIPCFileReader.
+ * garrow_file_reader_get_n_record_batches:
+ * @file_reader: A #GArrowFileReader.
  *
  * Returns: The number of record batches in the file.
  */
 guint
-garrow_ipc_file_reader_get_n_record_batches(GArrowIPCFileReader *file_reader)
+garrow_file_reader_get_n_record_batches(GArrowFileReader *file_reader)
 {
   auto arrow_file_reader =
-    garrow_ipc_file_reader_get_raw(file_reader);
+    garrow_file_reader_get_raw(file_reader);
   return arrow_file_reader->num_record_batches();
 }
 
 /**
- * garrow_ipc_file_reader_get_version:
- * @file_reader: A #GArrowIPCFileReader.
+ * garrow_file_reader_get_version:
+ * @file_reader: A #GArrowFileReader.
  *
  * Returns: The format version in the file.
  */
-GArrowIPCMetadataVersion
-garrow_ipc_file_reader_get_version(GArrowIPCFileReader *file_reader)
+GArrowMetadataVersion
+garrow_file_reader_get_version(GArrowFileReader *file_reader)
 {
   auto arrow_file_reader =
-    garrow_ipc_file_reader_get_raw(file_reader);
+    garrow_file_reader_get_raw(file_reader);
   auto arrow_version = arrow_file_reader->version();
-  return garrow_ipc_metadata_version_from_raw(arrow_version);
+  return garrow_metadata_version_from_raw(arrow_version);
 }
 
 /**
- * garrow_ipc_file_reader_get_record_batch:
- * @file_reader: A #GArrowIPCFileReader.
+ * garrow_file_reader_get_record_batch:
+ * @file_reader: A #GArrowFileReader.
  * @i: The index of the target record batch.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
@@ -208,12 +208,12 @@ garrow_ipc_file_reader_get_version(GArrowIPCFileReader *file_reader)
  *   The i-th record batch in the file or %NULL on error.
  */
 GArrowRecordBatch *
-garrow_ipc_file_reader_get_record_batch(GArrowIPCFileReader *file_reader,
+garrow_file_reader_get_record_batch(GArrowFileReader *file_reader,
                                         guint i,
                                         GError **error)
 {
   auto arrow_file_reader =
-    garrow_ipc_file_reader_get_raw(file_reader);
+    garrow_file_reader_get_raw(file_reader);
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
   auto status = arrow_file_reader->GetRecordBatch(i, &arrow_record_batch);
 
@@ -227,21 +227,21 @@ garrow_ipc_file_reader_get_record_batch(GArrowIPCFileReader *file_reader,
 
 G_END_DECLS
 
-GArrowIPCFileReader *
-garrow_ipc_file_reader_new_raw(std::shared_ptr<arrow::ipc::FileReader> *arrow_file_reader)
+GArrowFileReader *
+garrow_file_reader_new_raw(std::shared_ptr<arrow::ipc::FileReader> *arrow_file_reader)
 {
   auto file_reader =
-    GARROW_IPC_FILE_READER(g_object_new(GARROW_IPC_TYPE_FILE_READER,
+    GARROW_FILE_READER(g_object_new(GARROW_TYPE_FILE_READER,
                                         "file-reader", arrow_file_reader,
                                         NULL));
   return file_reader;
 }
 
 std::shared_ptr<arrow::ipc::FileReader>
-garrow_ipc_file_reader_get_raw(GArrowIPCFileReader *file_reader)
+garrow_file_reader_get_raw(GArrowFileReader *file_reader)
 {
-  GArrowIPCFileReaderPrivate *priv;
+  GArrowFileReaderPrivate *priv;
 
-  priv = GARROW_IPC_FILE_READER_GET_PRIVATE(file_reader);
+  priv = GARROW_FILE_READER_GET_PRIVATE(file_reader);
   return priv->file_reader;
 }
diff --git a/c_glib/arrow-glib/file-reader.h b/c_glib/arrow-glib/file-reader.h
new file mode 100644
index 0000000000000..084f7148ed903
--- /dev/null
+++ b/c_glib/arrow-glib/file-reader.h
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+
+#include <arrow-glib/random-access-file.h>
+
+#include <arrow-glib/metadata-version.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_FILE_READER      \
+  (garrow_file_reader_get_type())
+#define GARROW_FILE_READER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_FILE_READER,      \
+                              GArrowFileReader))
+#define GARROW_FILE_READER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_FILE_READER, \
+                           GArrowFileReaderClass))
+#define GARROW_IS_FILE_READER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_FILE_READER))
+#define GARROW_IS_FILE_READER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_FILE_READER))
+#define GARROW_FILE_READER_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_FILE_READER,       \
+                             GArrowFileReaderClass))
+
+typedef struct _GArrowFileReader         GArrowFileReader;
+typedef struct _GArrowFileReaderClass    GArrowFileReaderClass;
+
+/**
+ * GArrowFileReader:
+ *
+ * It wraps `arrow::ipc::FileReader`.
+ */
+struct _GArrowFileReader
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowFileReaderClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_file_reader_get_type(void) G_GNUC_CONST;
+
+GArrowFileReader *garrow_file_reader_open(GArrowRandomAccessFile *file,
+                                                 GError **error);
+
+GArrowSchema *garrow_file_reader_get_schema(GArrowFileReader *file_reader);
+guint garrow_file_reader_get_n_record_batches(GArrowFileReader *file_reader);
+GArrowMetadataVersion garrow_file_reader_get_version(GArrowFileReader *file_reader);
+GArrowRecordBatch *garrow_file_reader_get_record_batch(GArrowFileReader *file_reader,
+                                                           guint i,
+                                                           GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-file-writer.hpp b/c_glib/arrow-glib/file-reader.hpp
similarity index 78%
rename from c_glib/arrow-glib/ipc-file-writer.hpp
rename to c_glib/arrow-glib/file-reader.hpp
index b8ae1137a99ad..152379bbda4ff 100644
--- a/c_glib/arrow-glib/ipc-file-writer.hpp
+++ b/c_glib/arrow-glib/file-reader.hpp
@@ -22,7 +22,7 @@
 #include <arrow/api.h>
 #include <arrow/ipc/api.h>
 
-#include <arrow-glib/ipc-file-writer.h>
+#include <arrow-glib/file-reader.h>
 
-GArrowIPCFileWriter *garrow_ipc_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer);
-arrow::ipc::FileWriter *garrow_ipc_file_writer_get_raw(GArrowIPCFileWriter *file_writer);
+GArrowFileReader *garrow_file_reader_new_raw(std::shared_ptr<arrow::ipc::FileReader> *arrow_file_reader);
+std::shared_ptr<arrow::ipc::FileReader> garrow_file_reader_get_raw(GArrowFileReader *file_reader);
diff --git a/c_glib/arrow-glib/ipc-file-writer.cpp b/c_glib/arrow-glib/file-writer.cpp
similarity index 68%
rename from c_glib/arrow-glib/ipc-file-writer.cpp
rename to c_glib/arrow-glib/file-writer.cpp
index d8b3c2e72fa31..68eca2edf77c1 100644
--- a/c_glib/arrow-glib/ipc-file-writer.cpp
+++ b/c_glib/arrow-glib/file-writer.cpp
@@ -28,55 +28,55 @@
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 
-#include <arrow-glib/io-output-stream.hpp>
+#include <arrow-glib/output-stream.hpp>
 
-#include <arrow-glib/ipc-stream-writer.hpp>
-#include <arrow-glib/ipc-file-writer.hpp>
+#include <arrow-glib/stream-writer.hpp>
+#include <arrow-glib/file-writer.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: ipc-file-writer
+ * SECTION: file-writer
  * @short_description: File writer class
  *
- * #GArrowIPCFileWriter is a class for sending data by file based IPC.
+ * #GArrowFileWriter is a class for sending data by file based IPC.
  */
 
-G_DEFINE_TYPE(GArrowIPCFileWriter,
-              garrow_ipc_file_writer,
-              GARROW_IPC_TYPE_STREAM_WRITER);
+G_DEFINE_TYPE(GArrowFileWriter,
+              garrow_file_writer,
+              GARROW_TYPE_STREAM_WRITER);
 
 static void
-garrow_ipc_file_writer_init(GArrowIPCFileWriter *object)
+garrow_file_writer_init(GArrowFileWriter *object)
 {
 }
 
 static void
-garrow_ipc_file_writer_class_init(GArrowIPCFileWriterClass *klass)
+garrow_file_writer_class_init(GArrowFileWriterClass *klass)
 {
 }
 
 /**
- * garrow_ipc_file_writer_open:
+ * garrow_file_writer_open:
  * @sink: The output of the writer.
  * @schema: The schema of the writer.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowIPCFileWriter or %NULL on error.
+ *   #GArrowFileWriter or %NULL on error.
  */
-GArrowIPCFileWriter *
-garrow_ipc_file_writer_open(GArrowIOOutputStream *sink,
+GArrowFileWriter *
+garrow_file_writer_open(GArrowOutputStream *sink,
                             GArrowSchema *schema,
                             GError **error)
 {
   std::shared_ptr<arrow::ipc::FileWriter> arrow_file_writer;
   auto status =
-    arrow::ipc::FileWriter::Open(garrow_io_output_stream_get_raw(sink).get(),
+    arrow::ipc::FileWriter::Open(garrow_output_stream_get_raw(sink).get(),
                                  garrow_schema_get_raw(schema),
                                  &arrow_file_writer);
   if (status.ok()) {
-    return garrow_ipc_file_writer_new_raw(&arrow_file_writer);
+    return garrow_file_writer_new_raw(&arrow_file_writer);
   } else {
     garrow_error_set(error, status, "[ipc][file-writer][open]");
     return NULL;
@@ -84,20 +84,20 @@ garrow_ipc_file_writer_open(GArrowIOOutputStream *sink,
 }
 
 /**
- * garrow_ipc_file_writer_write_record_batch:
- * @file_writer: A #GArrowIPCFileWriter.
+ * garrow_file_writer_write_record_batch:
+ * @file_writer: A #GArrowFileWriter.
  * @record_batch: The record batch to be written.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_ipc_file_writer_write_record_batch(GArrowIPCFileWriter *file_writer,
+garrow_file_writer_write_record_batch(GArrowFileWriter *file_writer,
                                           GArrowRecordBatch *record_batch,
                                           GError **error)
 {
   auto arrow_file_writer =
-    garrow_ipc_file_writer_get_raw(file_writer);
+    garrow_file_writer_get_raw(file_writer);
   auto arrow_record_batch =
     garrow_record_batch_get_raw(record_batch);
   auto arrow_record_batch_raw =
@@ -113,18 +113,18 @@ garrow_ipc_file_writer_write_record_batch(GArrowIPCFileWriter *file_writer,
 }
 
 /**
- * garrow_ipc_file_writer_close:
- * @file_writer: A #GArrowIPCFileWriter.
+ * garrow_file_writer_close:
+ * @file_writer: A #GArrowFileWriter.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_ipc_file_writer_close(GArrowIPCFileWriter *file_writer,
+garrow_file_writer_close(GArrowFileWriter *file_writer,
                              GError **error)
 {
   auto arrow_file_writer =
-    garrow_ipc_file_writer_get_raw(file_writer);
+    garrow_file_writer_get_raw(file_writer);
 
   auto status = arrow_file_writer->Close();
   if (status.ok()) {
@@ -137,21 +137,21 @@ garrow_ipc_file_writer_close(GArrowIPCFileWriter *file_writer,
 
 G_END_DECLS
 
-GArrowIPCFileWriter *
-garrow_ipc_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer)
+GArrowFileWriter *
+garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer)
 {
   auto file_writer =
-    GARROW_IPC_FILE_WRITER(g_object_new(GARROW_IPC_TYPE_FILE_WRITER,
+    GARROW_FILE_WRITER(g_object_new(GARROW_TYPE_FILE_WRITER,
                                         "stream-writer", arrow_file_writer,
                                         NULL));
   return file_writer;
 }
 
 arrow::ipc::FileWriter *
-garrow_ipc_file_writer_get_raw(GArrowIPCFileWriter *file_writer)
+garrow_file_writer_get_raw(GArrowFileWriter *file_writer)
 {
   auto arrow_stream_writer =
-    garrow_ipc_stream_writer_get_raw(GARROW_IPC_STREAM_WRITER(file_writer));
+    garrow_stream_writer_get_raw(GARROW_STREAM_WRITER(file_writer));
   auto arrow_file_writer_raw =
     dynamic_cast<arrow::ipc::FileWriter *>(arrow_stream_writer.get());
   return arrow_file_writer_raw;
diff --git a/c_glib/arrow-glib/ipc-file-writer.h b/c_glib/arrow-glib/file-writer.h
similarity index 52%
rename from c_glib/arrow-glib/ipc-file-writer.h
rename to c_glib/arrow-glib/file-writer.h
index 732d9426aec8e..7f9a4f0399454 100644
--- a/c_glib/arrow-glib/ipc-file-writer.h
+++ b/c_glib/arrow-glib/file-writer.h
@@ -19,60 +19,60 @@
 
 #pragma once
 
-#include <arrow-glib/ipc-stream-writer.h>
+#include <arrow-glib/stream-writer.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_IPC_TYPE_FILE_WRITER             \
-  (garrow_ipc_file_writer_get_type())
-#define GARROW_IPC_FILE_WRITER(obj)                             \
+#define GARROW_TYPE_FILE_WRITER             \
+  (garrow_file_writer_get_type())
+#define GARROW_FILE_WRITER(obj)                             \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IPC_TYPE_FILE_WRITER,      \
-                              GArrowIPCFileWriter))
-#define GARROW_IPC_FILE_WRITER_CLASS(klass)             \
+                              GARROW_TYPE_FILE_WRITER,      \
+                              GArrowFileWriter))
+#define GARROW_FILE_WRITER_CLASS(klass)             \
   (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_IPC_TYPE_FILE_WRITER, \
-                           GArrowIPCFileWriterClass))
-#define GARROW_IPC_IS_FILE_WRITER(obj)                          \
+                           GARROW_TYPE_FILE_WRITER, \
+                           GArrowFileWriterClass))
+#define GARROW_IS_FILE_WRITER(obj)                          \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IPC_TYPE_FILE_WRITER))
-#define GARROW_IPC_IS_FILE_WRITER_CLASS(klass)                  \
+                              GARROW_TYPE_FILE_WRITER))
+#define GARROW_IS_FILE_WRITER_CLASS(klass)                  \
   (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_IPC_TYPE_FILE_WRITER))
-#define GARROW_IPC_FILE_WRITER_GET_CLASS(obj)                   \
+                           GARROW_TYPE_FILE_WRITER))
+#define GARROW_FILE_WRITER_GET_CLASS(obj)                   \
   (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_IPC_TYPE_FILE_WRITER,       \
-                             GArrowIPCFileWriterClass))
+                             GARROW_TYPE_FILE_WRITER,       \
+                             GArrowFileWriterClass))
 
-typedef struct _GArrowIPCFileWriter         GArrowIPCFileWriter;
-typedef struct _GArrowIPCFileWriterClass    GArrowIPCFileWriterClass;
+typedef struct _GArrowFileWriter         GArrowFileWriter;
+typedef struct _GArrowFileWriterClass    GArrowFileWriterClass;
 
 /**
- * GArrowIPCFileWriter:
+ * GArrowFileWriter:
  *
  * It wraps `arrow::ipc::FileWriter`.
  */
-struct _GArrowIPCFileWriter
+struct _GArrowFileWriter
 {
   /*< private >*/
-  GArrowIPCStreamWriter parent_instance;
+  GArrowStreamWriter parent_instance;
 };
 
-struct _GArrowIPCFileWriterClass
+struct _GArrowFileWriterClass
 {
   GObjectClass parent_class;
 };
 
-GType garrow_ipc_file_writer_get_type(void) G_GNUC_CONST;
+GType garrow_file_writer_get_type(void) G_GNUC_CONST;
 
-GArrowIPCFileWriter *garrow_ipc_file_writer_open(GArrowIOOutputStream *sink,
+GArrowFileWriter *garrow_file_writer_open(GArrowOutputStream *sink,
                                                  GArrowSchema *schema,
                                                  GError **error);
 
-gboolean garrow_ipc_file_writer_write_record_batch(GArrowIPCFileWriter *file_writer,
+gboolean garrow_file_writer_write_record_batch(GArrowFileWriter *file_writer,
                                                    GArrowRecordBatch *record_batch,
                                                    GError **error);
-gboolean garrow_ipc_file_writer_close(GArrowIPCFileWriter *file_writer,
+gboolean garrow_file_writer_close(GArrowFileWriter *file_writer,
                                       GError **error);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-file-reader.hpp b/c_glib/arrow-glib/file-writer.hpp
similarity index 77%
rename from c_glib/arrow-glib/ipc-file-reader.hpp
rename to c_glib/arrow-glib/file-writer.hpp
index 66cd45d51ddf5..f6a720a6cde7e 100644
--- a/c_glib/arrow-glib/ipc-file-reader.hpp
+++ b/c_glib/arrow-glib/file-writer.hpp
@@ -22,7 +22,7 @@
 #include <arrow/api.h>
 #include <arrow/ipc/api.h>
 
-#include <arrow-glib/ipc-file-reader.h>
+#include <arrow-glib/file-writer.h>
 
-GArrowIPCFileReader *garrow_ipc_file_reader_new_raw(std::shared_ptr<arrow::ipc::FileReader> *arrow_file_reader);
-std::shared_ptr<arrow::ipc::FileReader> garrow_ipc_file_reader_get_raw(GArrowIPCFileReader *file_reader);
+GArrowFileWriter *garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer);
+arrow::ipc::FileWriter *garrow_file_writer_get_raw(GArrowFileWriter *file_writer);
diff --git a/c_glib/arrow-glib/io-file.cpp b/c_glib/arrow-glib/file.cpp
similarity index 68%
rename from c_glib/arrow-glib/io-file.cpp
rename to c_glib/arrow-glib/file.cpp
index 536ae3e705f59..0d0fe1d8b9c83 100644
--- a/c_glib/arrow-glib/io-file.cpp
+++ b/c_glib/arrow-glib/file.cpp
@@ -24,40 +24,40 @@
 #include <arrow/api.h>
 
 #include <arrow-glib/error.hpp>
-#include <arrow-glib/io-file.hpp>
-#include <arrow-glib/io-file-mode.hpp>
+#include <arrow-glib/file.hpp>
+#include <arrow-glib/file-mode.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: io-file
- * @title: GArrowIOFile
+ * SECTION: file
+ * @title: GArrowFile
  * @short_description: File interface
  *
- * #GArrowIOFile is an interface for file.
+ * #GArrowFile is an interface for file.
  */
 
-G_DEFINE_INTERFACE(GArrowIOFile,
-                   garrow_io_file,
+G_DEFINE_INTERFACE(GArrowFile,
+                   garrow_file,
                    G_TYPE_OBJECT)
 
 static void
-garrow_io_file_default_init (GArrowIOFileInterface *iface)
+garrow_file_default_init (GArrowFileInterface *iface)
 {
 }
 
 /**
- * garrow_io_file_close:
- * @file: A #GArrowIOFile.
+ * garrow_file_close:
+ * @file: A #GArrowFile.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_io_file_close(GArrowIOFile *file,
+garrow_file_close(GArrowFile *file,
                      GError **error)
 {
-  auto arrow_file = garrow_io_file_get_raw(file);
+  auto arrow_file = garrow_file_get_raw(file);
 
   auto status = arrow_file->Close();
   if (status.ok()) {
@@ -69,17 +69,17 @@ garrow_io_file_close(GArrowIOFile *file,
 }
 
 /**
- * garrow_io_file_tell:
- * @file: A #GArrowIOFile.
+ * garrow_file_tell:
+ * @file: A #GArrowFile.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: The current offset on success, -1 if there was an error.
  */
 gint64
-garrow_io_file_tell(GArrowIOFile *file,
+garrow_file_tell(GArrowFile *file,
                     GError **error)
 {
-  auto arrow_file = garrow_io_file_get_raw(file);
+  auto arrow_file = garrow_file_get_raw(file);
 
   gint64 position;
   auto status = arrow_file->Tell(&position);
@@ -92,25 +92,25 @@ garrow_io_file_tell(GArrowIOFile *file,
 }
 
 /**
- * garrow_io_file_get_mode:
- * @file: A #GArrowIOFile.
+ * garrow_file_get_mode:
+ * @file: A #GArrowFile.
  *
  * Returns: The mode of the file.
  */
-GArrowIOFileMode
-garrow_io_file_get_mode(GArrowIOFile *file)
+GArrowFileMode
+garrow_file_get_mode(GArrowFile *file)
 {
-  auto arrow_file = garrow_io_file_get_raw(file);
+  auto arrow_file = garrow_file_get_raw(file);
 
   auto arrow_mode = arrow_file->mode();
-  return garrow_io_file_mode_from_raw(arrow_mode);
+  return garrow_file_mode_from_raw(arrow_mode);
 }
 
 G_END_DECLS
 
 std::shared_ptr<arrow::io::FileInterface>
-garrow_io_file_get_raw(GArrowIOFile *file)
+garrow_file_get_raw(GArrowFile *file)
 {
-  auto *iface = GARROW_IO_FILE_GET_IFACE(file);
+  auto *iface = GARROW_FILE_GET_IFACE(file);
   return iface->get_raw(file);
 }
diff --git a/c_glib/arrow-glib/io-file.h b/c_glib/arrow-glib/file.h
similarity index 55%
rename from c_glib/arrow-glib/io-file.h
rename to c_glib/arrow-glib/file.h
index 7181f6d37aeb3..68054aa5b6217 100644
--- a/c_glib/arrow-glib/io-file.h
+++ b/c_glib/arrow-glib/file.h
@@ -19,33 +19,33 @@
 
 #pragma once
 
-#include <arrow-glib/io-file-mode.h>
+#include <arrow-glib/file-mode.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_FILE                     \
-  (garrow_io_file_get_type())
-#define GARROW_IO_FILE(obj)                             \
+#define GARROW_TYPE_FILE                     \
+  (garrow_file_get_type())
+#define GARROW_FILE(obj)                             \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_IO_TYPE_FILE,      \
-                              GArrowIOFile))
-#define GARROW_IO_IS_FILE(obj)                          \
+                              GARROW_TYPE_FILE,      \
+                              GArrowFile))
+#define GARROW_IS_FILE(obj)                          \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_IO_TYPE_FILE))
-#define GARROW_IO_FILE_GET_IFACE(obj)                           \
+                              GARROW_TYPE_FILE))
+#define GARROW_FILE_GET_IFACE(obj)                           \
   (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
-                                 GARROW_IO_TYPE_FILE,           \
-                                 GArrowIOFileInterface))
+                                 GARROW_TYPE_FILE,           \
+                                 GArrowFileInterface))
 
-typedef struct _GArrowIOFile          GArrowIOFile;
-typedef struct _GArrowIOFileInterface GArrowIOFileInterface;
+typedef struct _GArrowFile          GArrowFile;
+typedef struct _GArrowFileInterface GArrowFileInterface;
 
-GType garrow_io_file_get_type(void) G_GNUC_CONST;
+GType garrow_file_get_type(void) G_GNUC_CONST;
 
-gboolean garrow_io_file_close(GArrowIOFile *file,
+gboolean garrow_file_close(GArrowFile *file,
                               GError **error);
-gint64 garrow_io_file_tell(GArrowIOFile *file,
+gint64 garrow_file_tell(GArrowFile *file,
                            GError **error);
-GArrowIOFileMode garrow_io_file_get_mode(GArrowIOFile *file);
+GArrowFileMode garrow_file_get_mode(GArrowFile *file);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/io-file.hpp b/c_glib/arrow-glib/file.hpp
similarity index 79%
rename from c_glib/arrow-glib/io-file.hpp
rename to c_glib/arrow-glib/file.hpp
index afaca90a10fa3..c4cc78747cf6a 100644
--- a/c_glib/arrow-glib/io-file.hpp
+++ b/c_glib/arrow-glib/file.hpp
@@ -21,18 +21,18 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-file.h>
+#include <arrow-glib/file.h>
 
 /**
- * GArrowIOFileInterface:
+ * GArrowFileInterface:
  *
  * It wraps `arrow::io::FileInterface`.
  */
-struct _GArrowIOFileInterface
+struct _GArrowFileInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::FileInterface> (*get_raw)(GArrowIOFile *file);
+  std::shared_ptr<arrow::io::FileInterface> (*get_raw)(GArrowFile *file);
 };
 
-std::shared_ptr<arrow::io::FileInterface> garrow_io_file_get_raw(GArrowIOFile *file);
+std::shared_ptr<arrow::io::FileInterface> garrow_file_get_raw(GArrowFile *file);
diff --git a/c_glib/arrow-glib/io-input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
similarity index 71%
rename from c_glib/arrow-glib/io-input-stream.cpp
rename to c_glib/arrow-glib/input-stream.cpp
index a28b9c6556ccd..36bef80422489 100644
--- a/c_glib/arrow-glib/io-input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -24,33 +24,33 @@
 #include <arrow/api.h>
 
 #include <arrow-glib/error.hpp>
-#include <arrow-glib/io-input-stream.hpp>
+#include <arrow-glib/input-stream.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: io-input-stream
- * @title: GArrowIOInputStream
+ * SECTION: input-stream
+ * @title: GArrowInputStream
  * @short_description: Stream input interface
  *
- * #GArrowIOInputStream is an interface for stream input. Stream input
+ * #GArrowInputStream is an interface for stream input. Stream input
  * is file based and readable.
  */
 
-G_DEFINE_INTERFACE(GArrowIOInputStream,
-                   garrow_io_input_stream,
+G_DEFINE_INTERFACE(GArrowInputStream,
+                   garrow_input_stream,
                    G_TYPE_OBJECT)
 
 static void
-garrow_io_input_stream_default_init (GArrowIOInputStreamInterface *iface)
+garrow_input_stream_default_init (GArrowInputStreamInterface *iface)
 {
 }
 
 G_END_DECLS
 
 std::shared_ptr<arrow::io::InputStream>
-garrow_io_input_stream_get_raw(GArrowIOInputStream *input_stream)
+garrow_input_stream_get_raw(GArrowInputStream *input_stream)
 {
-  auto *iface = GARROW_IO_INPUT_STREAM_GET_IFACE(input_stream);
+  auto *iface = GARROW_INPUT_STREAM_GET_IFACE(input_stream);
   return iface->get_raw(input_stream);
 }
diff --git a/c_glib/arrow-glib/io-input-stream.h b/c_glib/arrow-glib/input-stream.h
similarity index 57%
rename from c_glib/arrow-glib/io-input-stream.h
rename to c_glib/arrow-glib/input-stream.h
index 57902095010c8..4b331b93fb27f 100644
--- a/c_glib/arrow-glib/io-input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -23,23 +23,23 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_INPUT_STREAM             \
-  (garrow_io_input_stream_get_type())
-#define GARROW_IO_INPUT_STREAM(obj)                             \
+#define GARROW_TYPE_INPUT_STREAM             \
+  (garrow_input_stream_get_type())
+#define GARROW_INPUT_STREAM(obj)                             \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IO_TYPE_INPUT_STREAM,      \
-                              GArrowIOInputStream))
-#define GARROW_IO_IS_INPUT_STREAM(obj)                          \
+                              GARROW_TYPE_INPUT_STREAM,      \
+                              GArrowInputStream))
+#define GARROW_IS_INPUT_STREAM(obj)                          \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IO_TYPE_INPUT_STREAM))
-#define GARROW_IO_INPUT_STREAM_GET_IFACE(obj)                   \
+                              GARROW_TYPE_INPUT_STREAM))
+#define GARROW_INPUT_STREAM_GET_IFACE(obj)                   \
   (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
-                                 GARROW_IO_TYPE_INPUT_STREAM,   \
-                                 GArrowIOInputStreamInterface))
+                                 GARROW_TYPE_INPUT_STREAM,   \
+                                 GArrowInputStreamInterface))
 
-typedef struct _GArrowIOInputStream          GArrowIOInputStream;
-typedef struct _GArrowIOInputStreamInterface GArrowIOInputStreamInterface;
+typedef struct _GArrowInputStream          GArrowInputStream;
+typedef struct _GArrowInputStreamInterface GArrowInputStreamInterface;
 
-GType garrow_io_input_stream_get_type(void) G_GNUC_CONST;
+GType garrow_input_stream_get_type(void) G_GNUC_CONST;
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/io-input-stream.hpp b/c_glib/arrow-glib/input-stream.hpp
similarity index 76%
rename from c_glib/arrow-glib/io-input-stream.hpp
rename to c_glib/arrow-glib/input-stream.hpp
index 3b1de5da5c226..7958df1585887 100644
--- a/c_glib/arrow-glib/io-input-stream.hpp
+++ b/c_glib/arrow-glib/input-stream.hpp
@@ -21,18 +21,18 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-input-stream.h>
+#include <arrow-glib/input-stream.h>
 
 /**
- * GArrowIOInputStreamInterface:
+ * GArrowInputStreamInterface:
  *
  * It wraps `arrow::io::InputStream`.
  */
-struct _GArrowIOInputStreamInterface
+struct _GArrowInputStreamInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::InputStream> (*get_raw)(GArrowIOInputStream *file);
+  std::shared_ptr<arrow::io::InputStream> (*get_raw)(GArrowInputStream *file);
 };
 
-std::shared_ptr<arrow::io::InputStream> garrow_io_input_stream_get_raw(GArrowIOInputStream *input_stream);
+std::shared_ptr<arrow::io::InputStream> garrow_input_stream_get_raw(GArrowInputStream *input_stream);
diff --git a/c_glib/arrow-glib/io-memory-mapped-file.cpp b/c_glib/arrow-glib/io-memory-mapped-file.cpp
deleted file mode 100644
index e2e255c039109..0000000000000
--- a/c_glib/arrow-glib/io-memory-mapped-file.cpp
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow/io/file.h>
-
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/io-file.hpp>
-#include <arrow-glib/io-file-mode.hpp>
-#include <arrow-glib/io-input-stream.hpp>
-#include <arrow-glib/io-memory-mapped-file.hpp>
-#include <arrow-glib/io-readable.hpp>
-#include <arrow-glib/io-random-access-file.hpp>
-#include <arrow-glib/io-writeable.hpp>
-#include <arrow-glib/io-writeable-file.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: io-memory-mapped-file
- * @short_description: Memory mapped file class
- *
- * #GArrowIOMemoryMappedFile is a class for memory mapped file. It's
- * readable and writeable. It supports zero copy.
- */
-
-typedef struct GArrowIOMemoryMappedFilePrivate_ {
-  std::shared_ptr<arrow::io::MemoryMappedFile> memory_mapped_file;
-} GArrowIOMemoryMappedFilePrivate;
-
-enum {
-  PROP_0,
-  PROP_MEMORY_MAPPED_FILE
-};
-
-static std::shared_ptr<arrow::io::FileInterface>
-garrow_io_memory_mapped_file_get_raw_file_interface(GArrowIOFile *file)
-{
-  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(file);
-  auto arrow_memory_mapped_file =
-    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_io_file_interface_init(GArrowIOFileInterface *iface)
-{
-  iface->get_raw = garrow_io_memory_mapped_file_get_raw_file_interface;
-}
-
-static std::shared_ptr<arrow::io::Readable>
-garrow_io_memory_mapped_file_get_raw_readable_interface(GArrowIOReadable *readable)
-{
-  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(readable);
-  auto arrow_memory_mapped_file =
-    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_io_readable_interface_init(GArrowIOReadableInterface *iface)
-{
-  iface->get_raw = garrow_io_memory_mapped_file_get_raw_readable_interface;
-}
-
-static std::shared_ptr<arrow::io::InputStream>
-garrow_io_memory_mapped_file_get_raw_input_stream_interface(GArrowIOInputStream *input_stream)
-{
-  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(input_stream);
-  auto arrow_memory_mapped_file =
-    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_io_input_stream_interface_init(GArrowIOInputStreamInterface *iface)
-{
-  iface->get_raw = garrow_io_memory_mapped_file_get_raw_input_stream_interface;
-}
-
-static std::shared_ptr<arrow::io::RandomAccessFile>
-garrow_io_memory_mapped_file_get_raw_random_access_file_interface(GArrowIORandomAccessFile *file)
-{
-  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(file);
-  auto arrow_memory_mapped_file =
-    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_io_random_access_file_interface_init(GArrowIORandomAccessFileInterface *iface)
-{
-  iface->get_raw = garrow_io_memory_mapped_file_get_raw_random_access_file_interface;
-}
-
-static std::shared_ptr<arrow::io::Writeable>
-garrow_io_memory_mapped_file_get_raw_writeable_interface(GArrowIOWriteable *writeable)
-{
-  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(writeable);
-  auto arrow_memory_mapped_file =
-    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_io_writeable_interface_init(GArrowIOWriteableInterface *iface)
-{
-  iface->get_raw = garrow_io_memory_mapped_file_get_raw_writeable_interface;
-}
-
-static std::shared_ptr<arrow::io::WriteableFile>
-garrow_io_memory_mapped_file_get_raw_writeable_file_interface(GArrowIOWriteableFile *file)
-{
-  auto memory_mapped_file = GARROW_IO_MEMORY_MAPPED_FILE(file);
-  auto arrow_memory_mapped_file =
-    garrow_io_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_io_writeable_file_interface_init(GArrowIOWriteableFileInterface *iface)
-{
-  iface->get_raw = garrow_io_memory_mapped_file_get_raw_writeable_file_interface;
-}
-
-G_DEFINE_TYPE_WITH_CODE(GArrowIOMemoryMappedFile,
-                        garrow_io_memory_mapped_file,
-                        G_TYPE_OBJECT,
-                        G_ADD_PRIVATE(GArrowIOMemoryMappedFile)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_FILE,
-                                              garrow_io_file_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_READABLE,
-                                              garrow_io_readable_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_INPUT_STREAM,
-                                              garrow_io_input_stream_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_RANDOM_ACCESS_FILE,
-                                              garrow_io_random_access_file_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_WRITEABLE,
-                                              garrow_io_writeable_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_IO_TYPE_WRITEABLE_FILE,
-                                              garrow_io_writeable_file_interface_init));
-
-#define GARROW_IO_MEMORY_MAPPED_FILE_GET_PRIVATE(obj)                   \
-  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
-                               GARROW_IO_TYPE_MEMORY_MAPPED_FILE,       \
-                               GArrowIOMemoryMappedFilePrivate))
-
-static void
-garrow_io_memory_mapped_file_finalize(GObject *object)
-{
-  GArrowIOMemoryMappedFilePrivate *priv;
-
-  priv = GARROW_IO_MEMORY_MAPPED_FILE_GET_PRIVATE(object);
-
-  priv->memory_mapped_file = nullptr;
-
-  G_OBJECT_CLASS(garrow_io_memory_mapped_file_parent_class)->finalize(object);
-}
-
-static void
-garrow_io_memory_mapped_file_set_property(GObject *object,
-                                          guint prop_id,
-                                          const GValue *value,
-                                          GParamSpec *pspec)
-{
-  GArrowIOMemoryMappedFilePrivate *priv;
-
-  priv = GARROW_IO_MEMORY_MAPPED_FILE_GET_PRIVATE(object);
-
-  switch (prop_id) {
-  case PROP_MEMORY_MAPPED_FILE:
-    priv->memory_mapped_file =
-      *static_cast<std::shared_ptr<arrow::io::MemoryMappedFile> *>(g_value_get_pointer(value));
-    break;
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_io_memory_mapped_file_get_property(GObject *object,
-                                          guint prop_id,
-                                          GValue *value,
-                                          GParamSpec *pspec)
-{
-  switch (prop_id) {
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_io_memory_mapped_file_init(GArrowIOMemoryMappedFile *object)
-{
-}
-
-static void
-garrow_io_memory_mapped_file_class_init(GArrowIOMemoryMappedFileClass *klass)
-{
-  GObjectClass *gobject_class;
-  GParamSpec *spec;
-
-  gobject_class = G_OBJECT_CLASS(klass);
-
-  gobject_class->finalize     = garrow_io_memory_mapped_file_finalize;
-  gobject_class->set_property = garrow_io_memory_mapped_file_set_property;
-  gobject_class->get_property = garrow_io_memory_mapped_file_get_property;
-
-  spec = g_param_spec_pointer("memory-mapped-file",
-                              "io::MemoryMappedFile",
-                              "The raw std::shared<arrow::io::MemoryMappedFile> *",
-                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
-                                                       G_PARAM_CONSTRUCT_ONLY));
-  g_object_class_install_property(gobject_class, PROP_MEMORY_MAPPED_FILE, spec);
-}
-
-/**
- * garrow_io_memory_mapped_file_open:
- * @path: The path of the memory mapped file.
- * @mode: The mode of the memory mapped file.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowIOMemoryMappedFile or %NULL on error.
- */
-GArrowIOMemoryMappedFile *
-garrow_io_memory_mapped_file_open(const gchar *path,
-                                  GArrowIOFileMode mode,
-                                  GError **error)
-{
-  std::shared_ptr<arrow::io::MemoryMappedFile> arrow_memory_mapped_file;
-  auto status =
-    arrow::io::MemoryMappedFile::Open(std::string(path),
-                                      garrow_io_file_mode_to_raw(mode),
-                                      &arrow_memory_mapped_file);
-  if (status.ok()) {
-    return garrow_io_memory_mapped_file_new_raw(&arrow_memory_mapped_file);
-  } else {
-    std::string context("[io][memory-mapped-file][open]: <");
-    context += path;
-    context += ">";
-    garrow_error_set(error, status, context.c_str());
-    return NULL;
-  }
-}
-
-G_END_DECLS
-
-GArrowIOMemoryMappedFile *
-garrow_io_memory_mapped_file_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file)
-{
-  auto memory_mapped_file =
-    GARROW_IO_MEMORY_MAPPED_FILE(g_object_new(GARROW_IO_TYPE_MEMORY_MAPPED_FILE,
-                                              "memory-mapped-file", arrow_memory_mapped_file,
-                                              NULL));
-  return memory_mapped_file;
-}
-
-std::shared_ptr<arrow::io::MemoryMappedFile>
-garrow_io_memory_mapped_file_get_raw(GArrowIOMemoryMappedFile *memory_mapped_file)
-{
-  GArrowIOMemoryMappedFilePrivate *priv;
-
-  priv = GARROW_IO_MEMORY_MAPPED_FILE_GET_PRIVATE(memory_mapped_file);
-  return priv->memory_mapped_file;
-}
diff --git a/c_glib/arrow-glib/ipc-file-reader.h b/c_glib/arrow-glib/ipc-file-reader.h
deleted file mode 100644
index 15eba8e35a273..0000000000000
--- a/c_glib/arrow-glib/ipc-file-reader.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/record-batch.h>
-#include <arrow-glib/schema.h>
-
-#include <arrow-glib/io-random-access-file.h>
-
-#include <arrow-glib/ipc-metadata-version.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_IPC_TYPE_FILE_READER      \
-  (garrow_ipc_file_reader_get_type())
-#define GARROW_IPC_FILE_READER(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IPC_TYPE_FILE_READER,      \
-                              GArrowIPCFileReader))
-#define GARROW_IPC_FILE_READER_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_IPC_TYPE_FILE_READER, \
-                           GArrowIPCFileReaderClass))
-#define GARROW_IPC_IS_FILE_READER(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IPC_TYPE_FILE_READER))
-#define GARROW_IPC_IS_FILE_READER_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_IPC_TYPE_FILE_READER))
-#define GARROW_IPC_FILE_READER_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_IPC_TYPE_FILE_READER,       \
-                             GArrowIPCFileReaderClass))
-
-typedef struct _GArrowIPCFileReader         GArrowIPCFileReader;
-typedef struct _GArrowIPCFileReaderClass    GArrowIPCFileReaderClass;
-
-/**
- * GArrowIPCFileReader:
- *
- * It wraps `arrow::ipc::FileReader`.
- */
-struct _GArrowIPCFileReader
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowIPCFileReaderClass
-{
-  GObjectClass parent_class;
-};
-
-GType garrow_ipc_file_reader_get_type(void) G_GNUC_CONST;
-
-GArrowIPCFileReader *garrow_ipc_file_reader_open(GArrowIORandomAccessFile *file,
-                                                 GError **error);
-
-GArrowSchema *garrow_ipc_file_reader_get_schema(GArrowIPCFileReader *file_reader);
-guint garrow_ipc_file_reader_get_n_record_batches(GArrowIPCFileReader *file_reader);
-GArrowIPCMetadataVersion garrow_ipc_file_reader_get_version(GArrowIPCFileReader *file_reader);
-GArrowRecordBatch *garrow_ipc_file_reader_get_record_batch(GArrowIPCFileReader *file_reader,
-                                                           guint i,
-                                                           GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-stream-reader.h b/c_glib/arrow-glib/ipc-stream-reader.h
deleted file mode 100644
index 993cd85003bb9..0000000000000
--- a/c_glib/arrow-glib/ipc-stream-reader.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/record-batch.h>
-#include <arrow-glib/schema.h>
-
-#include <arrow-glib/io-input-stream.h>
-
-#include <arrow-glib/ipc-metadata-version.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_IPC_TYPE_STREAM_READER           \
-  (garrow_ipc_stream_reader_get_type())
-#define GARROW_IPC_STREAM_READER(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IPC_TYPE_STREAM_READER,    \
-                              GArrowIPCStreamReader))
-#define GARROW_IPC_STREAM_READER_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_IPC_TYPE_STREAM_READER,       \
-                           GArrowIPCStreamReaderClass))
-#define GARROW_IPC_IS_STREAM_READER(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IPC_TYPE_STREAM_READER))
-#define GARROW_IPC_IS_STREAM_READER_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_IPC_TYPE_STREAM_READER))
-#define GARROW_IPC_STREAM_READER_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_IPC_TYPE_STREAM_READER,     \
-                             GArrowIPCStreamReaderClass))
-
-typedef struct _GArrowIPCStreamReader         GArrowIPCStreamReader;
-typedef struct _GArrowIPCStreamReaderClass    GArrowIPCStreamReaderClass;
-
-/**
- * GArrowIPCStreamReader:
- *
- * It wraps `arrow::ipc::StreamReader`.
- */
-struct _GArrowIPCStreamReader
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowIPCStreamReaderClass
-{
-  GObjectClass parent_class;
-};
-
-GType garrow_ipc_stream_reader_get_type(void) G_GNUC_CONST;
-
-GArrowIPCStreamReader *garrow_ipc_stream_reader_open(GArrowIOInputStream *stream,
-                                                     GError **error);
-
-GArrowSchema *garrow_ipc_stream_reader_get_schema(GArrowIPCStreamReader *stream_reader);
-GArrowRecordBatch *garrow_ipc_stream_reader_get_next_record_batch(GArrowIPCStreamReader *stream_reader,
-                                                                  GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/memory-mapped-file.cpp b/c_glib/arrow-glib/memory-mapped-file.cpp
new file mode 100644
index 0000000000000..a3e1d0c45f142
--- /dev/null
+++ b/c_glib/arrow-glib/memory-mapped-file.cpp
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/io/file.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/file.hpp>
+#include <arrow-glib/file-mode.hpp>
+#include <arrow-glib/input-stream.hpp>
+#include <arrow-glib/memory-mapped-file.hpp>
+#include <arrow-glib/readable.hpp>
+#include <arrow-glib/random-access-file.hpp>
+#include <arrow-glib/writeable.hpp>
+#include <arrow-glib/writeable-file.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: memory-mapped-file
+ * @short_description: Memory mapped file class
+ *
+ * #GArrowMemoryMappedFile is a class for memory mapped file. It's
+ * readable and writeable. It supports zero copy.
+ */
+
+typedef struct GArrowMemoryMappedFilePrivate_ {
+  std::shared_ptr<arrow::io::MemoryMappedFile> memory_mapped_file;
+} GArrowMemoryMappedFilePrivate;
+
+enum {
+  PROP_0,
+  PROP_MEMORY_MAPPED_FILE
+};
+
+static std::shared_ptr<arrow::io::FileInterface>
+garrow_memory_mapped_file_get_raw_file_interface(GArrowFile *file)
+{
+  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(file);
+  auto arrow_memory_mapped_file =
+    garrow_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_file_interface_init(GArrowFileInterface *iface)
+{
+  iface->get_raw = garrow_memory_mapped_file_get_raw_file_interface;
+}
+
+static std::shared_ptr<arrow::io::Readable>
+garrow_memory_mapped_file_get_raw_readable_interface(GArrowReadable *readable)
+{
+  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(readable);
+  auto arrow_memory_mapped_file =
+    garrow_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_readable_interface_init(GArrowReadableInterface *iface)
+{
+  iface->get_raw = garrow_memory_mapped_file_get_raw_readable_interface;
+}
+
+static std::shared_ptr<arrow::io::InputStream>
+garrow_memory_mapped_file_get_raw_input_stream_interface(GArrowInputStream *input_stream)
+{
+  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(input_stream);
+  auto arrow_memory_mapped_file =
+    garrow_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_input_stream_interface_init(GArrowInputStreamInterface *iface)
+{
+  iface->get_raw = garrow_memory_mapped_file_get_raw_input_stream_interface;
+}
+
+static std::shared_ptr<arrow::io::RandomAccessFile>
+garrow_memory_mapped_file_get_raw_random_access_file_interface(GArrowRandomAccessFile *file)
+{
+  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(file);
+  auto arrow_memory_mapped_file =
+    garrow_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_random_access_file_interface_init(GArrowRandomAccessFileInterface *iface)
+{
+  iface->get_raw = garrow_memory_mapped_file_get_raw_random_access_file_interface;
+}
+
+static std::shared_ptr<arrow::io::Writeable>
+garrow_memory_mapped_file_get_raw_writeable_interface(GArrowWriteable *writeable)
+{
+  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(writeable);
+  auto arrow_memory_mapped_file =
+    garrow_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_writeable_interface_init(GArrowWriteableInterface *iface)
+{
+  iface->get_raw = garrow_memory_mapped_file_get_raw_writeable_interface;
+}
+
+static std::shared_ptr<arrow::io::WriteableFile>
+garrow_memory_mapped_file_get_raw_writeable_file_interface(GArrowWriteableFile *file)
+{
+  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(file);
+  auto arrow_memory_mapped_file =
+    garrow_memory_mapped_file_get_raw(memory_mapped_file);
+  return arrow_memory_mapped_file;
+}
+
+static void
+garrow_writeable_file_interface_init(GArrowWriteableFileInterface *iface)
+{
+  iface->get_raw = garrow_memory_mapped_file_get_raw_writeable_file_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(GArrowMemoryMappedFile,
+                        garrow_memory_mapped_file,
+                        G_TYPE_OBJECT,
+                        G_ADD_PRIVATE(GArrowMemoryMappedFile)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_FILE,
+                                              garrow_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_READABLE,
+                                              garrow_readable_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_INPUT_STREAM,
+                                              garrow_input_stream_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_RANDOM_ACCESS_FILE,
+                                              garrow_random_access_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE,
+                                              garrow_writeable_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE_FILE,
+                                              garrow_writeable_file_interface_init));
+
+#define GARROW_MEMORY_MAPPED_FILE_GET_PRIVATE(obj)                   \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
+                               GARROW_TYPE_MEMORY_MAPPED_FILE,       \
+                               GArrowMemoryMappedFilePrivate))
+
+static void
+garrow_memory_mapped_file_finalize(GObject *object)
+{
+  GArrowMemoryMappedFilePrivate *priv;
+
+  priv = GARROW_MEMORY_MAPPED_FILE_GET_PRIVATE(object);
+
+  priv->memory_mapped_file = nullptr;
+
+  G_OBJECT_CLASS(garrow_memory_mapped_file_parent_class)->finalize(object);
+}
+
+static void
+garrow_memory_mapped_file_set_property(GObject *object,
+                                          guint prop_id,
+                                          const GValue *value,
+                                          GParamSpec *pspec)
+{
+  GArrowMemoryMappedFilePrivate *priv;
+
+  priv = GARROW_MEMORY_MAPPED_FILE_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_MEMORY_MAPPED_FILE:
+    priv->memory_mapped_file =
+      *static_cast<std::shared_ptr<arrow::io::MemoryMappedFile> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_memory_mapped_file_get_property(GObject *object,
+                                          guint prop_id,
+                                          GValue *value,
+                                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_memory_mapped_file_init(GArrowMemoryMappedFile *object)
+{
+}
+
+static void
+garrow_memory_mapped_file_class_init(GArrowMemoryMappedFileClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_memory_mapped_file_finalize;
+  gobject_class->set_property = garrow_memory_mapped_file_set_property;
+  gobject_class->get_property = garrow_memory_mapped_file_get_property;
+
+  spec = g_param_spec_pointer("memory-mapped-file",
+                              "io::MemoryMappedFile",
+                              "The raw std::shared<arrow::io::MemoryMappedFile> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_MEMORY_MAPPED_FILE, spec);
+}
+
+/**
+ * garrow_memory_mapped_file_open:
+ * @path: The path of the memory mapped file.
+ * @mode: The mode of the memory mapped file.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): A newly opened
+ *   #GArrowMemoryMappedFile or %NULL on error.
+ */
+GArrowMemoryMappedFile *
+garrow_memory_mapped_file_open(const gchar *path,
+                                  GArrowFileMode mode,
+                                  GError **error)
+{
+  std::shared_ptr<arrow::io::MemoryMappedFile> arrow_memory_mapped_file;
+  auto status =
+    arrow::io::MemoryMappedFile::Open(std::string(path),
+                                      garrow_file_mode_to_raw(mode),
+                                      &arrow_memory_mapped_file);
+  if (status.ok()) {
+    return garrow_memory_mapped_file_new_raw(&arrow_memory_mapped_file);
+  } else {
+    std::string context("[io][memory-mapped-file][open]: <");
+    context += path;
+    context += ">";
+    garrow_error_set(error, status, context.c_str());
+    return NULL;
+  }
+}
+
+G_END_DECLS
+
+GArrowMemoryMappedFile *
+garrow_memory_mapped_file_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file)
+{
+  auto memory_mapped_file =
+    GARROW_MEMORY_MAPPED_FILE(g_object_new(GARROW_TYPE_MEMORY_MAPPED_FILE,
+                                              "memory-mapped-file", arrow_memory_mapped_file,
+                                              NULL));
+  return memory_mapped_file;
+}
+
+std::shared_ptr<arrow::io::MemoryMappedFile>
+garrow_memory_mapped_file_get_raw(GArrowMemoryMappedFile *memory_mapped_file)
+{
+  GArrowMemoryMappedFilePrivate *priv;
+
+  priv = GARROW_MEMORY_MAPPED_FILE_GET_PRIVATE(memory_mapped_file);
+  return priv->memory_mapped_file;
+}
diff --git a/c_glib/arrow-glib/io-memory-mapped-file.h b/c_glib/arrow-glib/memory-mapped-file.h
similarity index 51%
rename from c_glib/arrow-glib/io-memory-mapped-file.h
rename to c_glib/arrow-glib/memory-mapped-file.h
index 0d2d6c2f835de..40b8de04a5a75 100644
--- a/c_glib/arrow-glib/io-memory-mapped-file.h
+++ b/c_glib/arrow-glib/memory-mapped-file.h
@@ -19,54 +19,54 @@
 
 #pragma once
 
-#include <arrow-glib/io-file-mode.h>
+#include <arrow-glib/file-mode.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_MEMORY_MAPPED_FILE       \
-  (garrow_io_memory_mapped_file_get_type())
-#define GARROW_IO_MEMORY_MAPPED_FILE(obj)                               \
+#define GARROW_TYPE_MEMORY_MAPPED_FILE       \
+  (garrow_memory_mapped_file_get_type())
+#define GARROW_MEMORY_MAPPED_FILE(obj)                               \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
-                              GARROW_IO_TYPE_MEMORY_MAPPED_FILE,        \
-                              GArrowIOMemoryMappedFile))
-#define GARROW_IO_MEMORY_MAPPED_FILE_CLASS(klass)               \
+                              GARROW_TYPE_MEMORY_MAPPED_FILE,        \
+                              GArrowMemoryMappedFile))
+#define GARROW_MEMORY_MAPPED_FILE_CLASS(klass)               \
   (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_IO_TYPE_MEMORY_MAPPED_FILE,   \
-                           GArrowIOMemoryMappedFileClass))
-#define GARROW_IO_IS_MEMORY_MAPPED_FILE(obj)                            \
+                           GARROW_TYPE_MEMORY_MAPPED_FILE,   \
+                           GArrowMemoryMappedFileClass))
+#define GARROW_IS_MEMORY_MAPPED_FILE(obj)                            \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_IO_TYPE_MEMORY_MAPPED_FILE))
-#define GARROW_IO_IS_MEMORY_MAPPED_FILE_CLASS(klass)            \
+                              GARROW_TYPE_MEMORY_MAPPED_FILE))
+#define GARROW_IS_MEMORY_MAPPED_FILE_CLASS(klass)            \
   (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_IO_TYPE_MEMORY_MAPPED_FILE))
-#define GARROW_IO_MEMORY_MAPPED_FILE_GET_CLASS(obj)             \
+                           GARROW_TYPE_MEMORY_MAPPED_FILE))
+#define GARROW_MEMORY_MAPPED_FILE_GET_CLASS(obj)             \
   (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_IO_TYPE_MEMORY_MAPPED_FILE, \
-                             GArrowIOMemoryMappedFileClass))
+                             GARROW_TYPE_MEMORY_MAPPED_FILE, \
+                             GArrowMemoryMappedFileClass))
 
-typedef struct _GArrowIOMemoryMappedFile         GArrowIOMemoryMappedFile;
-typedef struct _GArrowIOMemoryMappedFileClass    GArrowIOMemoryMappedFileClass;
+typedef struct _GArrowMemoryMappedFile         GArrowMemoryMappedFile;
+typedef struct _GArrowMemoryMappedFileClass    GArrowMemoryMappedFileClass;
 
 /**
- * GArrowIOMemoryMappedFile:
+ * GArrowMemoryMappedFile:
  *
  * It wraps `arrow::io::MemoryMappedFile`.
  */
-struct _GArrowIOMemoryMappedFile
+struct _GArrowMemoryMappedFile
 {
   /*< private >*/
   GObject parent_instance;
 };
 
-struct _GArrowIOMemoryMappedFileClass
+struct _GArrowMemoryMappedFileClass
 {
   GObjectClass parent_class;
 };
 
-GType garrow_io_memory_mapped_file_get_type(void) G_GNUC_CONST;
+GType garrow_memory_mapped_file_get_type(void) G_GNUC_CONST;
 
-GArrowIOMemoryMappedFile *garrow_io_memory_mapped_file_open(const gchar *path,
-                                                            GArrowIOFileMode mode,
+GArrowMemoryMappedFile *garrow_memory_mapped_file_open(const gchar *path,
+                                                            GArrowFileMode mode,
                                                             GError **error);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/io-file-output-stream.hpp b/c_glib/arrow-glib/memory-mapped-file.hpp
similarity index 73%
rename from c_glib/arrow-glib/io-file-output-stream.hpp
rename to c_glib/arrow-glib/memory-mapped-file.hpp
index 76b8e91f6cf43..522e43d117f39 100644
--- a/c_glib/arrow-glib/io-file-output-stream.hpp
+++ b/c_glib/arrow-glib/memory-mapped-file.hpp
@@ -22,7 +22,7 @@
 #include <arrow/api.h>
 #include <arrow/io/file.h>
 
-#include <arrow-glib/io-file-output-stream.h>
+#include <arrow-glib/memory-mapped-file.h>
 
-GArrowIOFileOutputStream *garrow_io_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream);
-std::shared_ptr<arrow::io::FileOutputStream> garrow_io_file_output_stream_get_raw(GArrowIOFileOutputStream *file_output_stream);
+GArrowMemoryMappedFile *garrow_memory_mapped_file_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file);
+std::shared_ptr<arrow::io::MemoryMappedFile> garrow_memory_mapped_file_get_raw(GArrowMemoryMappedFile *memory_mapped_file);
diff --git a/c_glib/arrow-glib/ipc-metadata-version.cpp b/c_glib/arrow-glib/metadata-version.cpp
similarity index 68%
rename from c_glib/arrow-glib/ipc-metadata-version.cpp
rename to c_glib/arrow-glib/metadata-version.cpp
index f591f295ec886..ee458ebfcea4f 100644
--- a/c_glib/arrow-glib/ipc-metadata-version.cpp
+++ b/c_glib/arrow-glib/metadata-version.cpp
@@ -21,41 +21,41 @@
 #  include <config.h>
 #endif
 
-#include <arrow-glib/ipc-metadata-version.hpp>
+#include <arrow-glib/metadata-version.hpp>
 
 /**
- * SECTION: ipc-metadata-version
- * @title: GArrowIPCMetadataVersion
+ * SECTION: metadata-version
+ * @title: GArrowMetadataVersion
  * @short_description: Metadata version mapgging between Arrow and arrow-glib
  *
- * #GArrowIPCMetadataVersion provides metadata versions corresponding
+ * #GArrowMetadataVersion provides metadata versions corresponding
  * to `arrow::ipc::MetadataVersion` values.
  */
 
-GArrowIPCMetadataVersion
-garrow_ipc_metadata_version_from_raw(arrow::ipc::MetadataVersion version)
+GArrowMetadataVersion
+garrow_metadata_version_from_raw(arrow::ipc::MetadataVersion version)
 {
   switch (version) {
   case arrow::ipc::MetadataVersion::V1:
-    return GARROW_IPC_METADATA_VERSION_V1;
+    return GARROW_METADATA_VERSION_V1;
   case arrow::ipc::MetadataVersion::V2:
-    return GARROW_IPC_METADATA_VERSION_V2;
+    return GARROW_METADATA_VERSION_V2;
   case arrow::ipc::MetadataVersion::V3:
-    return GARROW_IPC_METADATA_VERSION_V3;
+    return GARROW_METADATA_VERSION_V3;
   default:
-    return GARROW_IPC_METADATA_VERSION_V3;
+    return GARROW_METADATA_VERSION_V3;
   }
 }
 
 arrow::ipc::MetadataVersion
-garrow_ipc_metadata_version_to_raw(GArrowIPCMetadataVersion version)
+garrow_metadata_version_to_raw(GArrowMetadataVersion version)
 {
   switch (version) {
-  case GARROW_IPC_METADATA_VERSION_V1:
+  case GARROW_METADATA_VERSION_V1:
     return arrow::ipc::MetadataVersion::V1;
-  case GARROW_IPC_METADATA_VERSION_V2:
+  case GARROW_METADATA_VERSION_V2:
     return arrow::ipc::MetadataVersion::V2;
-  case GARROW_IPC_METADATA_VERSION_V3:
+  case GARROW_METADATA_VERSION_V3:
     return arrow::ipc::MetadataVersion::V3;
   default:
     return arrow::ipc::MetadataVersion::V3;
diff --git a/c_glib/arrow-glib/ipc-metadata-version.h b/c_glib/arrow-glib/metadata-version.h
similarity index 76%
rename from c_glib/arrow-glib/ipc-metadata-version.h
rename to c_glib/arrow-glib/metadata-version.h
index 20defdb71b4f2..d902a3949e69a 100644
--- a/c_glib/arrow-glib/ipc-metadata-version.h
+++ b/c_glib/arrow-glib/metadata-version.h
@@ -24,18 +24,18 @@
 G_BEGIN_DECLS
 
 /**
- * GArrowIPCMetadataVersion:
- * @GARROW_IPC_METADATA_VERSION_V1: Version 1.
- * @GARROW_IPC_METADATA_VERSION_V2: Version 2.
- * @GARROW_IPC_METADATA_VERSION_V3: Version 3.
+ * GArrowMetadataVersion:
+ * @GARROW_METADATA_VERSION_V1: Version 1.
+ * @GARROW_METADATA_VERSION_V2: Version 2.
+ * @GARROW_METADATA_VERSION_V3: Version 3.
  *
  * They are corresponding to `arrow::ipc::MetadataVersion::type`
  * values.
  */
 typedef enum {
-  GARROW_IPC_METADATA_VERSION_V1,
-  GARROW_IPC_METADATA_VERSION_V2,
-  GARROW_IPC_METADATA_VERSION_V3
-} GArrowIPCMetadataVersion;
+  GARROW_METADATA_VERSION_V1,
+  GARROW_METADATA_VERSION_V2,
+  GARROW_METADATA_VERSION_V3
+} GArrowMetadataVersion;
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-metadata-version.hpp b/c_glib/arrow-glib/metadata-version.hpp
similarity index 77%
rename from c_glib/arrow-glib/ipc-metadata-version.hpp
rename to c_glib/arrow-glib/metadata-version.hpp
index 229565f002180..7b3865e59216b 100644
--- a/c_glib/arrow-glib/ipc-metadata-version.hpp
+++ b/c_glib/arrow-glib/metadata-version.hpp
@@ -21,7 +21,7 @@
 
 #include <arrow/ipc/api.h>
 
-#include <arrow-glib/ipc-metadata-version.h>
+#include <arrow-glib/metadata-version.h>
 
-GArrowIPCMetadataVersion garrow_ipc_metadata_version_from_raw(arrow::ipc::MetadataVersion version);
-arrow::ipc::MetadataVersion garrow_ipc_metadata_version_to_raw(GArrowIPCMetadataVersion version);
+GArrowMetadataVersion garrow_metadata_version_from_raw(arrow::ipc::MetadataVersion version);
+arrow::ipc::MetadataVersion garrow_metadata_version_to_raw(GArrowMetadataVersion version);
diff --git a/c_glib/arrow-glib/io-output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
similarity index 71%
rename from c_glib/arrow-glib/io-output-stream.cpp
rename to c_glib/arrow-glib/output-stream.cpp
index bdf5587ba1c07..bbc29b794f7c6 100644
--- a/c_glib/arrow-glib/io-output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -24,33 +24,33 @@
 #include <arrow/api.h>
 
 #include <arrow-glib/error.hpp>
-#include <arrow-glib/io-output-stream.hpp>
+#include <arrow-glib/output-stream.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: io-output-stream
- * @title: GArrowIOOutputStream
+ * SECTION: output-stream
+ * @title: GArrowOutputStream
  * @short_description: Stream output interface
  *
- * #GArrowIOOutputStream is an interface for stream output. Stream
+ * #GArrowOutputStream is an interface for stream output. Stream
  * output is file based and writeable
  */
 
-G_DEFINE_INTERFACE(GArrowIOOutputStream,
-                   garrow_io_output_stream,
+G_DEFINE_INTERFACE(GArrowOutputStream,
+                   garrow_output_stream,
                    G_TYPE_OBJECT)
 
 static void
-garrow_io_output_stream_default_init (GArrowIOOutputStreamInterface *iface)
+garrow_output_stream_default_init (GArrowOutputStreamInterface *iface)
 {
 }
 
 G_END_DECLS
 
 std::shared_ptr<arrow::io::OutputStream>
-garrow_io_output_stream_get_raw(GArrowIOOutputStream *output_stream)
+garrow_output_stream_get_raw(GArrowOutputStream *output_stream)
 {
-  auto *iface = GARROW_IO_OUTPUT_STREAM_GET_IFACE(output_stream);
+  auto *iface = GARROW_OUTPUT_STREAM_GET_IFACE(output_stream);
   return iface->get_raw(output_stream);
 }
diff --git a/c_glib/arrow-glib/io-output-stream.h b/c_glib/arrow-glib/output-stream.h
similarity index 57%
rename from c_glib/arrow-glib/io-output-stream.h
rename to c_glib/arrow-glib/output-stream.h
index 02478ce9621eb..3481072c27d8b 100644
--- a/c_glib/arrow-glib/io-output-stream.h
+++ b/c_glib/arrow-glib/output-stream.h
@@ -23,23 +23,23 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_OUTPUT_STREAM            \
-  (garrow_io_output_stream_get_type())
-#define GARROW_IO_OUTPUT_STREAM(obj)                            \
+#define GARROW_TYPE_OUTPUT_STREAM            \
+  (garrow_output_stream_get_type())
+#define GARROW_OUTPUT_STREAM(obj)                            \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IO_TYPE_OUTPUT_STREAM,     \
-                              GArrowIOOutputStream))
-#define GARROW_IO_IS_OUTPUT_STREAM(obj)                         \
+                              GARROW_TYPE_OUTPUT_STREAM,     \
+                              GArrowOutputStream))
+#define GARROW_IS_OUTPUT_STREAM(obj)                         \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IO_TYPE_OUTPUT_STREAM))
-#define GARROW_IO_OUTPUT_STREAM_GET_IFACE(obj)                          \
+                              GARROW_TYPE_OUTPUT_STREAM))
+#define GARROW_OUTPUT_STREAM_GET_IFACE(obj)                          \
   (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
-                                 GARROW_IO_TYPE_OUTPUT_STREAM,          \
-                                 GArrowIOOutputStreamInterface))
+                                 GARROW_TYPE_OUTPUT_STREAM,          \
+                                 GArrowOutputStreamInterface))
 
-typedef struct _GArrowIOOutputStream          GArrowIOOutputStream;
-typedef struct _GArrowIOOutputStreamInterface GArrowIOOutputStreamInterface;
+typedef struct _GArrowOutputStream          GArrowOutputStream;
+typedef struct _GArrowOutputStreamInterface GArrowOutputStreamInterface;
 
-GType garrow_io_output_stream_get_type(void) G_GNUC_CONST;
+GType garrow_output_stream_get_type(void) G_GNUC_CONST;
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/io-output-stream.hpp b/c_glib/arrow-glib/output-stream.hpp
similarity index 75%
rename from c_glib/arrow-glib/io-output-stream.hpp
rename to c_glib/arrow-glib/output-stream.hpp
index f144130b1420e..635da10e24766 100644
--- a/c_glib/arrow-glib/io-output-stream.hpp
+++ b/c_glib/arrow-glib/output-stream.hpp
@@ -21,18 +21,18 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-output-stream.h>
+#include <arrow-glib/output-stream.h>
 
 /**
- * GArrowIOOutputStreamInterface:
+ * GArrowOutputStreamInterface:
  *
  * It wraps `arrow::io::OutputStream`.
  */
-struct _GArrowIOOutputStreamInterface
+struct _GArrowOutputStreamInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::OutputStream> (*get_raw)(GArrowIOOutputStream *file);
+  std::shared_ptr<arrow::io::OutputStream> (*get_raw)(GArrowOutputStream *file);
 };
 
-std::shared_ptr<arrow::io::OutputStream> garrow_io_output_stream_get_raw(GArrowIOOutputStream *output_stream);
+std::shared_ptr<arrow::io::OutputStream> garrow_output_stream_get_raw(GArrowOutputStream *output_stream);
diff --git a/c_glib/arrow-glib/io-random-access-file.cpp b/c_glib/arrow-glib/random-access-file.cpp
similarity index 70%
rename from c_glib/arrow-glib/io-random-access-file.cpp
rename to c_glib/arrow-glib/random-access-file.cpp
index 552b879c19794..71f315ec7efaa 100644
--- a/c_glib/arrow-glib/io-random-access-file.cpp
+++ b/c_glib/arrow-glib/random-access-file.cpp
@@ -24,39 +24,39 @@
 #include <arrow/api.h>
 
 #include <arrow-glib/error.hpp>
-#include <arrow-glib/io-random-access-file.hpp>
+#include <arrow-glib/random-access-file.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: io-random-access-file
- * @title: GArrowIORandomAccessFile
+ * SECTION: random-access-file
+ * @title: GArrowRandomAccessFile
  * @short_description: File input interface
  *
- * #GArrowIORandomAccessFile is an interface for file input.
+ * #GArrowRandomAccessFile is an interface for file input.
  */
 
-G_DEFINE_INTERFACE(GArrowIORandomAccessFile,
-                   garrow_io_random_access_file,
+G_DEFINE_INTERFACE(GArrowRandomAccessFile,
+                   garrow_random_access_file,
                    G_TYPE_OBJECT)
 
 static void
-garrow_io_random_access_file_default_init (GArrowIORandomAccessFileInterface *iface)
+garrow_random_access_file_default_init (GArrowRandomAccessFileInterface *iface)
 {
 }
 
 /**
- * garrow_io_random_access_file_get_size:
- * @file: A #GArrowIORandomAccessFile.
+ * garrow_random_access_file_get_size:
+ * @file: A #GArrowRandomAccessFile.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: The size of the file.
  */
 guint64
-garrow_io_random_access_file_get_size(GArrowIORandomAccessFile *file,
+garrow_random_access_file_get_size(GArrowRandomAccessFile *file,
                                  GError **error)
 {
-  auto *iface = GARROW_IO_RANDOM_ACCESS_FILE_GET_IFACE(file);
+  auto *iface = GARROW_RANDOM_ACCESS_FILE_GET_IFACE(file);
   auto arrow_random_access_file = iface->get_raw(file);
   int64_t size;
 
@@ -70,23 +70,23 @@ garrow_io_random_access_file_get_size(GArrowIORandomAccessFile *file,
 }
 
 /**
- * garrow_io_random_access_file_get_support_zero_copy:
- * @file: A #GArrowIORandomAccessFile.
+ * garrow_random_access_file_get_support_zero_copy:
+ * @file: A #GArrowRandomAccessFile.
  *
  * Returns: Whether zero copy read is supported or not.
  */
 gboolean
-garrow_io_random_access_file_get_support_zero_copy(GArrowIORandomAccessFile *file)
+garrow_random_access_file_get_support_zero_copy(GArrowRandomAccessFile *file)
 {
-  auto *iface = GARROW_IO_RANDOM_ACCESS_FILE_GET_IFACE(file);
+  auto *iface = GARROW_RANDOM_ACCESS_FILE_GET_IFACE(file);
   auto arrow_random_access_file = iface->get_raw(file);
 
   return arrow_random_access_file->supports_zero_copy();
 }
 
 /**
- * garrow_io_random_access_file_read_at:
- * @file: A #GArrowIORandomAccessFile.
+ * garrow_random_access_file_read_at:
+ * @file: A #GArrowRandomAccessFile.
  * @position: The read start position.
  * @n_bytes: The number of bytes to be read.
  * @n_read_bytes: (out): The read number of bytes.
@@ -96,7 +96,7 @@ garrow_io_random_access_file_get_support_zero_copy(GArrowIORandomAccessFile *fil
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_io_random_access_file_read_at(GArrowIORandomAccessFile *file,
+garrow_random_access_file_read_at(GArrowRandomAccessFile *file,
                                      gint64 position,
                                      gint64 n_bytes,
                                      gint64 *n_read_bytes,
@@ -104,7 +104,7 @@ garrow_io_random_access_file_read_at(GArrowIORandomAccessFile *file,
                                      GError **error)
 {
   const auto arrow_random_access_file =
-    garrow_io_random_access_file_get_raw(file);
+    garrow_random_access_file_get_raw(file);
 
   auto status = arrow_random_access_file->ReadAt(position,
                                                  n_bytes,
@@ -121,8 +121,8 @@ garrow_io_random_access_file_read_at(GArrowIORandomAccessFile *file,
 G_END_DECLS
 
 std::shared_ptr<arrow::io::RandomAccessFile>
-garrow_io_random_access_file_get_raw(GArrowIORandomAccessFile *random_access_file)
+garrow_random_access_file_get_raw(GArrowRandomAccessFile *random_access_file)
 {
-  auto *iface = GARROW_IO_RANDOM_ACCESS_FILE_GET_IFACE(random_access_file);
+  auto *iface = GARROW_RANDOM_ACCESS_FILE_GET_IFACE(random_access_file);
   return iface->get_raw(random_access_file);
 }
diff --git a/c_glib/arrow-glib/io-random-access-file.h b/c_glib/arrow-glib/random-access-file.h
similarity index 57%
rename from c_glib/arrow-glib/io-random-access-file.h
rename to c_glib/arrow-glib/random-access-file.h
index 8ac63e417a3f2..8a7f6b4218a31 100644
--- a/c_glib/arrow-glib/io-random-access-file.h
+++ b/c_glib/arrow-glib/random-access-file.h
@@ -23,29 +23,29 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_RANDOM_ACCESS_FILE       \
-  (garrow_io_random_access_file_get_type())
-#define GARROW_IO_RANDOM_ACCESS_FILE(obj)                            \
+#define GARROW_TYPE_RANDOM_ACCESS_FILE       \
+  (garrow_random_access_file_get_type())
+#define GARROW_RANDOM_ACCESS_FILE(obj)                            \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                                 \
-                              GARROW_IO_TYPE_RANDOM_ACCESS_FILE,     \
-                              GArrowIORandomAccessFile))
-#define GARROW_IO_IS_RANDOM_ACCESS_FILE(obj)                            \
+                              GARROW_TYPE_RANDOM_ACCESS_FILE,     \
+                              GArrowRandomAccessFile))
+#define GARROW_IS_RANDOM_ACCESS_FILE(obj)                            \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_IO_TYPE_RANDOM_ACCESS_FILE))
-#define GARROW_IO_RANDOM_ACCESS_FILE_GET_IFACE(obj)                     \
+                              GARROW_TYPE_RANDOM_ACCESS_FILE))
+#define GARROW_RANDOM_ACCESS_FILE_GET_IFACE(obj)                     \
   (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
-                                 GARROW_IO_TYPE_RANDOM_ACCESS_FILE,     \
-                                 GArrowIORandomAccessFileInterface))
+                                 GARROW_TYPE_RANDOM_ACCESS_FILE,     \
+                                 GArrowRandomAccessFileInterface))
 
-typedef struct _GArrowIORandomAccessFile          GArrowIORandomAccessFile;
-typedef struct _GArrowIORandomAccessFileInterface GArrowIORandomAccessFileInterface;
+typedef struct _GArrowRandomAccessFile          GArrowRandomAccessFile;
+typedef struct _GArrowRandomAccessFileInterface GArrowRandomAccessFileInterface;
 
-GType garrow_io_random_access_file_get_type(void) G_GNUC_CONST;
+GType garrow_random_access_file_get_type(void) G_GNUC_CONST;
 
-guint64 garrow_io_random_access_file_get_size(GArrowIORandomAccessFile *file,
+guint64 garrow_random_access_file_get_size(GArrowRandomAccessFile *file,
                                               GError **error);
-gboolean garrow_io_random_access_file_get_support_zero_copy(GArrowIORandomAccessFile *file);
-gboolean garrow_io_random_access_file_read_at(GArrowIORandomAccessFile *file,
+gboolean garrow_random_access_file_get_support_zero_copy(GArrowRandomAccessFile *file);
+gboolean garrow_random_access_file_read_at(GArrowRandomAccessFile *file,
                                               gint64 position,
                                               gint64 n_bytes,
                                               gint64 *n_read_bytes,
diff --git a/c_glib/arrow-glib/io-random-access-file.hpp b/c_glib/arrow-glib/random-access-file.hpp
similarity index 78%
rename from c_glib/arrow-glib/io-random-access-file.hpp
rename to c_glib/arrow-glib/random-access-file.hpp
index 7c97c9ecedb5b..6d6fed70b4b62 100644
--- a/c_glib/arrow-glib/io-random-access-file.hpp
+++ b/c_glib/arrow-glib/random-access-file.hpp
@@ -21,18 +21,18 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-random-access-file.h>
+#include <arrow-glib/random-access-file.h>
 
 /**
- * GArrowIORandomAccessFileInterface:
+ * GArrowRandomAccessFileInterface:
  *
  * It wraps `arrow::io::RandomAccessFile`.
  */
-struct _GArrowIORandomAccessFileInterface
+struct _GArrowRandomAccessFileInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::RandomAccessFile> (*get_raw)(GArrowIORandomAccessFile *file);
+  std::shared_ptr<arrow::io::RandomAccessFile> (*get_raw)(GArrowRandomAccessFile *file);
 };
 
-std::shared_ptr<arrow::io::RandomAccessFile> garrow_io_random_access_file_get_raw(GArrowIORandomAccessFile *random_access_file);
+std::shared_ptr<arrow::io::RandomAccessFile> garrow_random_access_file_get_raw(GArrowRandomAccessFile *random_access_file);
diff --git a/c_glib/arrow-glib/io-readable.cpp b/c_glib/arrow-glib/readable.cpp
similarity index 75%
rename from c_glib/arrow-glib/io-readable.cpp
rename to c_glib/arrow-glib/readable.cpp
index b372a66090ceb..b8c0cd99df06a 100644
--- a/c_glib/arrow-glib/io-readable.cpp
+++ b/c_glib/arrow-glib/readable.cpp
@@ -24,31 +24,31 @@
 #include <arrow/api.h>
 
 #include <arrow-glib/error.hpp>
-#include <arrow-glib/io-readable.hpp>
+#include <arrow-glib/readable.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: io-readable
- * @title: GArrowIOReadable
+ * SECTION: readable
+ * @title: GArrowReadable
  * @short_description: Input interface
  *
- * #GArrowIOReadable is an interface for input. Input must be
+ * #GArrowReadable is an interface for input. Input must be
  * readable.
  */
 
-G_DEFINE_INTERFACE(GArrowIOReadable,
-                   garrow_io_readable,
+G_DEFINE_INTERFACE(GArrowReadable,
+                   garrow_readable,
                    G_TYPE_OBJECT)
 
 static void
-garrow_io_readable_default_init (GArrowIOReadableInterface *iface)
+garrow_readable_default_init (GArrowReadableInterface *iface)
 {
 }
 
 /**
- * garrow_io_readable_read:
- * @readable: A #GArrowIOReadable.
+ * garrow_readable_read:
+ * @readable: A #GArrowReadable.
  * @n_bytes: The number of bytes to be read.
  * @n_read_bytes: (out): The read number of bytes.
  * @buffer: (array length=n_bytes): The buffer to be read data.
@@ -57,13 +57,13 @@ garrow_io_readable_default_init (GArrowIOReadableInterface *iface)
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_io_readable_read(GArrowIOReadable *readable,
+garrow_readable_read(GArrowReadable *readable,
                         gint64 n_bytes,
                         gint64 *n_read_bytes,
                         guint8 *buffer,
                         GError **error)
 {
-  const auto arrow_readable = garrow_io_readable_get_raw(readable);
+  const auto arrow_readable = garrow_readable_get_raw(readable);
 
   auto status = arrow_readable->Read(n_bytes, n_read_bytes, buffer);
   if (status.ok()) {
@@ -77,8 +77,8 @@ garrow_io_readable_read(GArrowIOReadable *readable,
 G_END_DECLS
 
 std::shared_ptr<arrow::io::Readable>
-garrow_io_readable_get_raw(GArrowIOReadable *readable)
+garrow_readable_get_raw(GArrowReadable *readable)
 {
-  auto *iface = GARROW_IO_READABLE_GET_IFACE(readable);
+  auto *iface = GARROW_READABLE_GET_IFACE(readable);
   return iface->get_raw(readable);
 }
diff --git a/c_glib/arrow-glib/io-readable.h b/c_glib/arrow-glib/readable.h
similarity index 60%
rename from c_glib/arrow-glib/io-readable.h
rename to c_glib/arrow-glib/readable.h
index 279984b3014a3..bde4b01ee1f15 100644
--- a/c_glib/arrow-glib/io-readable.h
+++ b/c_glib/arrow-glib/readable.h
@@ -23,26 +23,26 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_READABLE                 \
-  (garrow_io_readable_get_type())
-#define GARROW_IO_READABLE(obj)                                 \
+#define GARROW_TYPE_READABLE                 \
+  (garrow_readable_get_type())
+#define GARROW_READABLE(obj)                                 \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IO_TYPE_READABLE,          \
-                              GArrowIOReadable))
-#define GARROW_IO_IS_READABLE(obj)                      \
+                              GARROW_TYPE_READABLE,          \
+                              GArrowReadable))
+#define GARROW_IS_READABLE(obj)                      \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_IO_TYPE_READABLE))
-#define GARROW_IO_READABLE_GET_IFACE(obj)                       \
+                              GARROW_TYPE_READABLE))
+#define GARROW_READABLE_GET_IFACE(obj)                       \
   (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
-                                 GARROW_IO_TYPE_READABLE,       \
-                                 GArrowIOReadableInterface))
+                                 GARROW_TYPE_READABLE,       \
+                                 GArrowReadableInterface))
 
-typedef struct _GArrowIOReadable          GArrowIOReadable;
-typedef struct _GArrowIOReadableInterface GArrowIOReadableInterface;
+typedef struct _GArrowReadable          GArrowReadable;
+typedef struct _GArrowReadableInterface GArrowReadableInterface;
 
-GType garrow_io_readable_get_type(void) G_GNUC_CONST;
+GType garrow_readable_get_type(void) G_GNUC_CONST;
 
-gboolean garrow_io_readable_read(GArrowIOReadable *readable,
+gboolean garrow_readable_read(GArrowReadable *readable,
                                  gint64 n_bytes,
                                  gint64 *n_read_bytes,
                                  guint8 *buffer,
diff --git a/c_glib/arrow-glib/io-readable.hpp b/c_glib/arrow-glib/readable.hpp
similarity index 77%
rename from c_glib/arrow-glib/io-readable.hpp
rename to c_glib/arrow-glib/readable.hpp
index 3d27b3f92ba78..c241c77aa0329 100644
--- a/c_glib/arrow-glib/io-readable.hpp
+++ b/c_glib/arrow-glib/readable.hpp
@@ -21,18 +21,18 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-readable.h>
+#include <arrow-glib/readable.h>
 
 /**
- * GArrowIOReadableInterface:
+ * GArrowReadableInterface:
  *
  * It wraps `arrow::io::Readable`.
  */
-struct _GArrowIOReadableInterface
+struct _GArrowReadableInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::Readable> (*get_raw)(GArrowIOReadable *file);
+  std::shared_ptr<arrow::io::Readable> (*get_raw)(GArrowReadable *file);
 };
 
-std::shared_ptr<arrow::io::Readable> garrow_io_readable_get_raw(GArrowIOReadable *readable);
+std::shared_ptr<arrow::io::Readable> garrow_readable_get_raw(GArrowReadable *readable);
diff --git a/c_glib/arrow-glib/ipc-stream-reader.cpp b/c_glib/arrow-glib/stream-reader.cpp
similarity index 63%
rename from c_glib/arrow-glib/ipc-stream-reader.cpp
rename to c_glib/arrow-glib/stream-reader.cpp
index 48047842aaac6..c4ccebe56f6ba 100644
--- a/c_glib/arrow-glib/ipc-stream-reader.cpp
+++ b/c_glib/arrow-glib/stream-reader.cpp
@@ -27,60 +27,60 @@
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 
-#include <arrow-glib/io-input-stream.hpp>
+#include <arrow-glib/input-stream.hpp>
 
-#include <arrow-glib/ipc-metadata-version.hpp>
-#include <arrow-glib/ipc-stream-reader.hpp>
+#include <arrow-glib/metadata-version.hpp>
+#include <arrow-glib/stream-reader.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: ipc-stream-reader
+ * SECTION: stream-reader
  * @short_description: Stream reader class
  *
- * #GArrowIPCStreamReader is a class for receiving data by stream
+ * #GArrowStreamReader is a class for receiving data by stream
  * based IPC.
  */
 
-typedef struct GArrowIPCStreamReaderPrivate_ {
+typedef struct GArrowStreamReaderPrivate_ {
   std::shared_ptr<arrow::ipc::StreamReader> stream_reader;
-} GArrowIPCStreamReaderPrivate;
+} GArrowStreamReaderPrivate;
 
 enum {
   PROP_0,
   PROP_STREAM_READER
 };
 
-G_DEFINE_TYPE_WITH_PRIVATE(GArrowIPCStreamReader,
-                           garrow_ipc_stream_reader,
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowStreamReader,
+                           garrow_stream_reader,
                            G_TYPE_OBJECT);
 
-#define GARROW_IPC_STREAM_READER_GET_PRIVATE(obj)               \
+#define GARROW_STREAM_READER_GET_PRIVATE(obj)               \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
-                               GARROW_IPC_TYPE_STREAM_READER,   \
-                               GArrowIPCStreamReaderPrivate))
+                               GARROW_TYPE_STREAM_READER,   \
+                               GArrowStreamReaderPrivate))
 
 static void
-garrow_ipc_stream_reader_finalize(GObject *object)
+garrow_stream_reader_finalize(GObject *object)
 {
-  GArrowIPCStreamReaderPrivate *priv;
+  GArrowStreamReaderPrivate *priv;
 
-  priv = GARROW_IPC_STREAM_READER_GET_PRIVATE(object);
+  priv = GARROW_STREAM_READER_GET_PRIVATE(object);
 
   priv->stream_reader = nullptr;
 
-  G_OBJECT_CLASS(garrow_ipc_stream_reader_parent_class)->finalize(object);
+  G_OBJECT_CLASS(garrow_stream_reader_parent_class)->finalize(object);
 }
 
 static void
-garrow_ipc_stream_reader_set_property(GObject *object,
+garrow_stream_reader_set_property(GObject *object,
                                     guint prop_id,
                                     const GValue *value,
                                     GParamSpec *pspec)
 {
-  GArrowIPCStreamReaderPrivate *priv;
+  GArrowStreamReaderPrivate *priv;
 
-  priv = GARROW_IPC_STREAM_READER_GET_PRIVATE(object);
+  priv = GARROW_STREAM_READER_GET_PRIVATE(object);
 
   switch (prop_id) {
   case PROP_STREAM_READER:
@@ -94,7 +94,7 @@ garrow_ipc_stream_reader_set_property(GObject *object,
 }
 
 static void
-garrow_ipc_stream_reader_get_property(GObject *object,
+garrow_stream_reader_get_property(GObject *object,
                                           guint prop_id,
                                           GValue *value,
                                           GParamSpec *pspec)
@@ -107,21 +107,21 @@ garrow_ipc_stream_reader_get_property(GObject *object,
 }
 
 static void
-garrow_ipc_stream_reader_init(GArrowIPCStreamReader *object)
+garrow_stream_reader_init(GArrowStreamReader *object)
 {
 }
 
 static void
-garrow_ipc_stream_reader_class_init(GArrowIPCStreamReaderClass *klass)
+garrow_stream_reader_class_init(GArrowStreamReaderClass *klass)
 {
   GObjectClass *gobject_class;
   GParamSpec *spec;
 
   gobject_class = G_OBJECT_CLASS(klass);
 
-  gobject_class->finalize     = garrow_ipc_stream_reader_finalize;
-  gobject_class->set_property = garrow_ipc_stream_reader_set_property;
-  gobject_class->get_property = garrow_ipc_stream_reader_get_property;
+  gobject_class->finalize     = garrow_stream_reader_finalize;
+  gobject_class->set_property = garrow_stream_reader_set_property;
+  gobject_class->get_property = garrow_stream_reader_get_property;
 
   spec = g_param_spec_pointer("stream-reader",
                               "ipc::StreamReader",
@@ -132,23 +132,23 @@ garrow_ipc_stream_reader_class_init(GArrowIPCStreamReaderClass *klass)
 }
 
 /**
- * garrow_ipc_stream_reader_open:
+ * garrow_stream_reader_open:
  * @stream: The stream to be read.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowIPCStreamReader or %NULL on error.
+ *   #GArrowStreamReader or %NULL on error.
  */
-GArrowIPCStreamReader *
-garrow_ipc_stream_reader_open(GArrowIOInputStream *stream,
+GArrowStreamReader *
+garrow_stream_reader_open(GArrowInputStream *stream,
                               GError **error)
 {
   std::shared_ptr<arrow::ipc::StreamReader> arrow_stream_reader;
   auto status =
-    arrow::ipc::StreamReader::Open(garrow_io_input_stream_get_raw(stream),
+    arrow::ipc::StreamReader::Open(garrow_input_stream_get_raw(stream),
                                    &arrow_stream_reader);
   if (status.ok()) {
-    return garrow_ipc_stream_reader_new_raw(&arrow_stream_reader);
+    return garrow_stream_reader_new_raw(&arrow_stream_reader);
   } else {
     garrow_error_set(error, status, "[ipc][stream-reader][open]");
     return NULL;
@@ -156,34 +156,34 @@ garrow_ipc_stream_reader_open(GArrowIOInputStream *stream,
 }
 
 /**
- * garrow_ipc_stream_reader_get_schema:
- * @stream_reader: A #GArrowIPCStreamReader.
+ * garrow_stream_reader_get_schema:
+ * @stream_reader: A #GArrowStreamReader.
  *
  * Returns: (transfer full): The schema in the stream.
  */
 GArrowSchema *
-garrow_ipc_stream_reader_get_schema(GArrowIPCStreamReader *stream_reader)
+garrow_stream_reader_get_schema(GArrowStreamReader *stream_reader)
 {
   auto arrow_stream_reader =
-    garrow_ipc_stream_reader_get_raw(stream_reader);
+    garrow_stream_reader_get_raw(stream_reader);
   auto arrow_schema = arrow_stream_reader->schema();
   return garrow_schema_new_raw(&arrow_schema);
 }
 
 /**
- * garrow_ipc_stream_reader_get_next_record_batch:
- * @stream_reader: A #GArrowIPCStreamReader.
+ * garrow_stream_reader_get_next_record_batch:
+ * @stream_reader: A #GArrowStreamReader.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: (nullable) (transfer full):
  *   The next record batch in the stream or %NULL on end of stream.
  */
 GArrowRecordBatch *
-garrow_ipc_stream_reader_get_next_record_batch(GArrowIPCStreamReader *stream_reader,
+garrow_stream_reader_get_next_record_batch(GArrowStreamReader *stream_reader,
                                                GError **error)
 {
   auto arrow_stream_reader =
-    garrow_ipc_stream_reader_get_raw(stream_reader);
+    garrow_stream_reader_get_raw(stream_reader);
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
   auto status = arrow_stream_reader->GetNextRecordBatch(&arrow_record_batch);
 
@@ -201,21 +201,21 @@ garrow_ipc_stream_reader_get_next_record_batch(GArrowIPCStreamReader *stream_rea
 
 G_END_DECLS
 
-GArrowIPCStreamReader *
-garrow_ipc_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_stream_reader)
+GArrowStreamReader *
+garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_stream_reader)
 {
   auto stream_reader =
-    GARROW_IPC_STREAM_READER(g_object_new(GARROW_IPC_TYPE_STREAM_READER,
+    GARROW_STREAM_READER(g_object_new(GARROW_TYPE_STREAM_READER,
                                           "stream-reader", arrow_stream_reader,
                                           NULL));
   return stream_reader;
 }
 
 std::shared_ptr<arrow::ipc::StreamReader>
-garrow_ipc_stream_reader_get_raw(GArrowIPCStreamReader *stream_reader)
+garrow_stream_reader_get_raw(GArrowStreamReader *stream_reader)
 {
-  GArrowIPCStreamReaderPrivate *priv;
+  GArrowStreamReaderPrivate *priv;
 
-  priv = GARROW_IPC_STREAM_READER_GET_PRIVATE(stream_reader);
+  priv = GARROW_STREAM_READER_GET_PRIVATE(stream_reader);
   return priv->stream_reader;
 }
diff --git a/c_glib/arrow-glib/stream-reader.h b/c_glib/arrow-glib/stream-reader.h
new file mode 100644
index 0000000000000..16a7f57bf801b
--- /dev/null
+++ b/c_glib/arrow-glib/stream-reader.h
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+
+#include <arrow-glib/input-stream.h>
+
+#include <arrow-glib/metadata-version.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_STREAM_READER           \
+  (garrow_stream_reader_get_type())
+#define GARROW_STREAM_READER(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STREAM_READER,    \
+                              GArrowStreamReader))
+#define GARROW_STREAM_READER_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_STREAM_READER,       \
+                           GArrowStreamReaderClass))
+#define GARROW_IS_STREAM_READER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STREAM_READER))
+#define GARROW_IS_STREAM_READER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_STREAM_READER))
+#define GARROW_STREAM_READER_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_STREAM_READER,     \
+                             GArrowStreamReaderClass))
+
+typedef struct _GArrowStreamReader         GArrowStreamReader;
+typedef struct _GArrowStreamReaderClass    GArrowStreamReaderClass;
+
+/**
+ * GArrowStreamReader:
+ *
+ * It wraps `arrow::ipc::StreamReader`.
+ */
+struct _GArrowStreamReader
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowStreamReaderClass
+{
+  GObjectClass parent_class;
+};
+
+GType garrow_stream_reader_get_type(void) G_GNUC_CONST;
+
+GArrowStreamReader *garrow_stream_reader_open(GArrowInputStream *stream,
+                                                     GError **error);
+
+GArrowSchema *garrow_stream_reader_get_schema(GArrowStreamReader *stream_reader);
+GArrowRecordBatch *garrow_stream_reader_get_next_record_batch(GArrowStreamReader *stream_reader,
+                                                                  GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-stream-reader.hpp b/c_glib/arrow-glib/stream-reader.hpp
similarity index 75%
rename from c_glib/arrow-glib/ipc-stream-reader.hpp
rename to c_glib/arrow-glib/stream-reader.hpp
index a35bdab7e69d4..ca8e6895a4fd6 100644
--- a/c_glib/arrow-glib/ipc-stream-reader.hpp
+++ b/c_glib/arrow-glib/stream-reader.hpp
@@ -22,7 +22,7 @@
 #include <arrow/api.h>
 #include <arrow/ipc/api.h>
 
-#include <arrow-glib/ipc-stream-reader.h>
+#include <arrow-glib/stream-reader.h>
 
-GArrowIPCStreamReader *garrow_ipc_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_stream_reader);
-std::shared_ptr<arrow::ipc::StreamReader> garrow_ipc_stream_reader_get_raw(GArrowIPCStreamReader *stream_reader);
+GArrowStreamReader *garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_stream_reader);
+std::shared_ptr<arrow::ipc::StreamReader> garrow_stream_reader_get_raw(GArrowStreamReader *stream_reader);
diff --git a/c_glib/arrow-glib/ipc-stream-writer.cpp b/c_glib/arrow-glib/stream-writer.cpp
similarity index 65%
rename from c_glib/arrow-glib/ipc-stream-writer.cpp
rename to c_glib/arrow-glib/stream-writer.cpp
index e2455a4a9c61c..016ce93759c87 100644
--- a/c_glib/arrow-glib/ipc-stream-writer.cpp
+++ b/c_glib/arrow-glib/stream-writer.cpp
@@ -28,59 +28,59 @@
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 
-#include <arrow-glib/io-output-stream.hpp>
+#include <arrow-glib/output-stream.hpp>
 
-#include <arrow-glib/ipc-stream-writer.hpp>
+#include <arrow-glib/stream-writer.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: ipc-stream-writer
+ * SECTION: stream-writer
  * @short_description: Stream writer class
  *
- * #GArrowIPCStreamWriter is a class for sending data by stream based
+ * #GArrowStreamWriter is a class for sending data by stream based
  * IPC.
  */
 
-typedef struct GArrowIPCStreamWriterPrivate_ {
+typedef struct GArrowStreamWriterPrivate_ {
   std::shared_ptr<arrow::ipc::StreamWriter> stream_writer;
-} GArrowIPCStreamWriterPrivate;
+} GArrowStreamWriterPrivate;
 
 enum {
   PROP_0,
   PROP_STREAM_WRITER
 };
 
-G_DEFINE_TYPE_WITH_PRIVATE(GArrowIPCStreamWriter,
-                           garrow_ipc_stream_writer,
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowStreamWriter,
+                           garrow_stream_writer,
                            G_TYPE_OBJECT);
 
-#define GARROW_IPC_STREAM_WRITER_GET_PRIVATE(obj)               \
+#define GARROW_STREAM_WRITER_GET_PRIVATE(obj)               \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
-                               GARROW_IPC_TYPE_STREAM_WRITER,   \
-                               GArrowIPCStreamWriterPrivate))
+                               GARROW_TYPE_STREAM_WRITER,   \
+                               GArrowStreamWriterPrivate))
 
 static void
-garrow_ipc_stream_writer_finalize(GObject *object)
+garrow_stream_writer_finalize(GObject *object)
 {
-  GArrowIPCStreamWriterPrivate *priv;
+  GArrowStreamWriterPrivate *priv;
 
-  priv = GARROW_IPC_STREAM_WRITER_GET_PRIVATE(object);
+  priv = GARROW_STREAM_WRITER_GET_PRIVATE(object);
 
   priv->stream_writer = nullptr;
 
-  G_OBJECT_CLASS(garrow_ipc_stream_writer_parent_class)->finalize(object);
+  G_OBJECT_CLASS(garrow_stream_writer_parent_class)->finalize(object);
 }
 
 static void
-garrow_ipc_stream_writer_set_property(GObject *object,
+garrow_stream_writer_set_property(GObject *object,
                                     guint prop_id,
                                     const GValue *value,
                                     GParamSpec *pspec)
 {
-  GArrowIPCStreamWriterPrivate *priv;
+  GArrowStreamWriterPrivate *priv;
 
-  priv = GARROW_IPC_STREAM_WRITER_GET_PRIVATE(object);
+  priv = GARROW_STREAM_WRITER_GET_PRIVATE(object);
 
   switch (prop_id) {
   case PROP_STREAM_WRITER:
@@ -94,7 +94,7 @@ garrow_ipc_stream_writer_set_property(GObject *object,
 }
 
 static void
-garrow_ipc_stream_writer_get_property(GObject *object,
+garrow_stream_writer_get_property(GObject *object,
                                           guint prop_id,
                                           GValue *value,
                                           GParamSpec *pspec)
@@ -107,21 +107,21 @@ garrow_ipc_stream_writer_get_property(GObject *object,
 }
 
 static void
-garrow_ipc_stream_writer_init(GArrowIPCStreamWriter *object)
+garrow_stream_writer_init(GArrowStreamWriter *object)
 {
 }
 
 static void
-garrow_ipc_stream_writer_class_init(GArrowIPCStreamWriterClass *klass)
+garrow_stream_writer_class_init(GArrowStreamWriterClass *klass)
 {
   GObjectClass *gobject_class;
   GParamSpec *spec;
 
   gobject_class = G_OBJECT_CLASS(klass);
 
-  gobject_class->finalize     = garrow_ipc_stream_writer_finalize;
-  gobject_class->set_property = garrow_ipc_stream_writer_set_property;
-  gobject_class->get_property = garrow_ipc_stream_writer_get_property;
+  gobject_class->finalize     = garrow_stream_writer_finalize;
+  gobject_class->set_property = garrow_stream_writer_set_property;
+  gobject_class->get_property = garrow_stream_writer_get_property;
 
   spec = g_param_spec_pointer("stream-writer",
                               "ipc::StreamWriter",
@@ -132,26 +132,26 @@ garrow_ipc_stream_writer_class_init(GArrowIPCStreamWriterClass *klass)
 }
 
 /**
- * garrow_ipc_stream_writer_open:
+ * garrow_stream_writer_open:
  * @sink: The output of the writer.
  * @schema: The schema of the writer.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowIPCStreamWriter or %NULL on error.
+ *   #GArrowStreamWriter or %NULL on error.
  */
-GArrowIPCStreamWriter *
-garrow_ipc_stream_writer_open(GArrowIOOutputStream *sink,
+GArrowStreamWriter *
+garrow_stream_writer_open(GArrowOutputStream *sink,
                               GArrowSchema *schema,
                               GError **error)
 {
   std::shared_ptr<arrow::ipc::StreamWriter> arrow_stream_writer;
   auto status =
-    arrow::ipc::StreamWriter::Open(garrow_io_output_stream_get_raw(sink).get(),
+    arrow::ipc::StreamWriter::Open(garrow_output_stream_get_raw(sink).get(),
                                  garrow_schema_get_raw(schema),
                                  &arrow_stream_writer);
   if (status.ok()) {
-    return garrow_ipc_stream_writer_new_raw(&arrow_stream_writer);
+    return garrow_stream_writer_new_raw(&arrow_stream_writer);
   } else {
     garrow_error_set(error, status, "[ipc][stream-writer][open]");
     return NULL;
@@ -159,20 +159,20 @@ garrow_ipc_stream_writer_open(GArrowIOOutputStream *sink,
 }
 
 /**
- * garrow_ipc_stream_writer_write_record_batch:
- * @stream_writer: A #GArrowIPCStreamWriter.
+ * garrow_stream_writer_write_record_batch:
+ * @stream_writer: A #GArrowStreamWriter.
  * @record_batch: The record batch to be written.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_ipc_stream_writer_write_record_batch(GArrowIPCStreamWriter *stream_writer,
+garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
                                             GArrowRecordBatch *record_batch,
                                             GError **error)
 {
   auto arrow_stream_writer =
-    garrow_ipc_stream_writer_get_raw(stream_writer);
+    garrow_stream_writer_get_raw(stream_writer);
   auto arrow_record_batch =
     garrow_record_batch_get_raw(record_batch);
   auto arrow_record_batch_raw =
@@ -188,18 +188,18 @@ garrow_ipc_stream_writer_write_record_batch(GArrowIPCStreamWriter *stream_writer
 }
 
 /**
- * garrow_ipc_stream_writer_close:
- * @stream_writer: A #GArrowIPCStreamWriter.
+ * garrow_stream_writer_close:
+ * @stream_writer: A #GArrowStreamWriter.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_ipc_stream_writer_close(GArrowIPCStreamWriter *stream_writer,
+garrow_stream_writer_close(GArrowStreamWriter *stream_writer,
                                GError **error)
 {
   auto arrow_stream_writer =
-    garrow_ipc_stream_writer_get_raw(stream_writer);
+    garrow_stream_writer_get_raw(stream_writer);
 
   auto status = arrow_stream_writer->Close();
   if (status.ok()) {
@@ -212,21 +212,21 @@ garrow_ipc_stream_writer_close(GArrowIPCStreamWriter *stream_writer,
 
 G_END_DECLS
 
-GArrowIPCStreamWriter *
-garrow_ipc_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer)
+GArrowStreamWriter *
+garrow_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer)
 {
   auto stream_writer =
-    GARROW_IPC_STREAM_WRITER(g_object_new(GARROW_IPC_TYPE_STREAM_WRITER,
+    GARROW_STREAM_WRITER(g_object_new(GARROW_TYPE_STREAM_WRITER,
                                         "stream-writer", arrow_stream_writer,
                                         NULL));
   return stream_writer;
 }
 
 std::shared_ptr<arrow::ipc::StreamWriter>
-garrow_ipc_stream_writer_get_raw(GArrowIPCStreamWriter *stream_writer)
+garrow_stream_writer_get_raw(GArrowStreamWriter *stream_writer)
 {
-  GArrowIPCStreamWriterPrivate *priv;
+  GArrowStreamWriterPrivate *priv;
 
-  priv = GARROW_IPC_STREAM_WRITER_GET_PRIVATE(stream_writer);
+  priv = GARROW_STREAM_WRITER_GET_PRIVATE(stream_writer);
   return priv->stream_writer;
 }
diff --git a/c_glib/arrow-glib/ipc-stream-writer.h b/c_glib/arrow-glib/stream-writer.h
similarity index 54%
rename from c_glib/arrow-glib/ipc-stream-writer.h
rename to c_glib/arrow-glib/stream-writer.h
index 4488204736d51..6e773f1fc316e 100644
--- a/c_glib/arrow-glib/ipc-stream-writer.h
+++ b/c_glib/arrow-glib/stream-writer.h
@@ -23,60 +23,60 @@
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
 
-#include <arrow-glib/io-output-stream.h>
+#include <arrow-glib/output-stream.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_IPC_TYPE_STREAM_WRITER           \
-  (garrow_ipc_stream_writer_get_type())
-#define GARROW_IPC_STREAM_WRITER(obj)                           \
+#define GARROW_TYPE_STREAM_WRITER           \
+  (garrow_stream_writer_get_type())
+#define GARROW_STREAM_WRITER(obj)                           \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IPC_TYPE_STREAM_WRITER,    \
-                              GArrowIPCStreamWriter))
-#define GARROW_IPC_STREAM_WRITER_CLASS(klass)                   \
+                              GARROW_TYPE_STREAM_WRITER,    \
+                              GArrowStreamWriter))
+#define GARROW_STREAM_WRITER_CLASS(klass)                   \
   (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_IPC_TYPE_STREAM_WRITER,       \
-                           GArrowIPCStreamWriterClass))
-#define GARROW_IPC_IS_STREAM_WRITER(obj)                        \
+                           GARROW_TYPE_STREAM_WRITER,       \
+                           GArrowStreamWriterClass))
+#define GARROW_IS_STREAM_WRITER(obj)                        \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IPC_TYPE_STREAM_WRITER))
-#define GARROW_IPC_IS_STREAM_WRITER_CLASS(klass)                \
+                              GARROW_TYPE_STREAM_WRITER))
+#define GARROW_IS_STREAM_WRITER_CLASS(klass)                \
   (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_IPC_TYPE_STREAM_WRITER))
-#define GARROW_IPC_STREAM_WRITER_GET_CLASS(obj)                 \
+                           GARROW_TYPE_STREAM_WRITER))
+#define GARROW_STREAM_WRITER_GET_CLASS(obj)                 \
   (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_IPC_TYPE_STREAM_WRITER,     \
-                             GArrowIPCStreamWriterClass))
+                             GARROW_TYPE_STREAM_WRITER,     \
+                             GArrowStreamWriterClass))
 
-typedef struct _GArrowIPCStreamWriter         GArrowIPCStreamWriter;
-typedef struct _GArrowIPCStreamWriterClass    GArrowIPCStreamWriterClass;
+typedef struct _GArrowStreamWriter         GArrowStreamWriter;
+typedef struct _GArrowStreamWriterClass    GArrowStreamWriterClass;
 
 /**
- * GArrowIPCStreamWriter:
+ * GArrowStreamWriter:
  *
  * It wraps `arrow::ipc::StreamWriter`.
  */
-struct _GArrowIPCStreamWriter
+struct _GArrowStreamWriter
 {
   /*< private >*/
   GObject parent_instance;
 };
 
-struct _GArrowIPCStreamWriterClass
+struct _GArrowStreamWriterClass
 {
   GObjectClass parent_class;
 };
 
-GType garrow_ipc_stream_writer_get_type(void) G_GNUC_CONST;
+GType garrow_stream_writer_get_type(void) G_GNUC_CONST;
 
-GArrowIPCStreamWriter *garrow_ipc_stream_writer_open(GArrowIOOutputStream *sink,
+GArrowStreamWriter *garrow_stream_writer_open(GArrowOutputStream *sink,
                                                      GArrowSchema *schema,
                                                      GError **error);
 
-gboolean garrow_ipc_stream_writer_write_record_batch(GArrowIPCStreamWriter *stream_writer,
+gboolean garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
                                                      GArrowRecordBatch *record_batch,
                                                      GError **error);
-gboolean garrow_ipc_stream_writer_close(GArrowIPCStreamWriter *stream_writer,
+gboolean garrow_stream_writer_close(GArrowStreamWriter *stream_writer,
                                         GError **error);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/ipc-stream-writer.hpp b/c_glib/arrow-glib/stream-writer.hpp
similarity index 75%
rename from c_glib/arrow-glib/ipc-stream-writer.hpp
rename to c_glib/arrow-glib/stream-writer.hpp
index 9d097404582a9..994c83b8f4ad5 100644
--- a/c_glib/arrow-glib/ipc-stream-writer.hpp
+++ b/c_glib/arrow-glib/stream-writer.hpp
@@ -22,7 +22,7 @@
 #include <arrow/api.h>
 #include <arrow/ipc/api.h>
 
-#include <arrow-glib/ipc-stream-writer.h>
+#include <arrow-glib/stream-writer.h>
 
-GArrowIPCStreamWriter *garrow_ipc_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer);
-std::shared_ptr<arrow::ipc::StreamWriter> garrow_ipc_stream_writer_get_raw(GArrowIPCStreamWriter *stream_writer);
+GArrowStreamWriter *garrow_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer);
+std::shared_ptr<arrow::ipc::StreamWriter> garrow_stream_writer_get_raw(GArrowStreamWriter *stream_writer);
diff --git a/c_glib/arrow-glib/io-writeable-file.cpp b/c_glib/arrow-glib/writeable-file.cpp
similarity index 73%
rename from c_glib/arrow-glib/io-writeable-file.cpp
rename to c_glib/arrow-glib/writeable-file.cpp
index 41b682acd1e26..d0937ea2612d2 100644
--- a/c_glib/arrow-glib/io-writeable-file.cpp
+++ b/c_glib/arrow-glib/writeable-file.cpp
@@ -24,30 +24,30 @@
 #include <arrow/api.h>
 
 #include <arrow-glib/error.hpp>
-#include <arrow-glib/io-writeable-file.hpp>
+#include <arrow-glib/writeable-file.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: io-writeable-file
- * @title: GArrowIOWriteableFile
+ * SECTION: writeable-file
+ * @title: GArrowWriteableFile
  * @short_description: File output interface
  *
- * #GArrowIOWriteableFile is an interface for file output.
+ * #GArrowWriteableFile is an interface for file output.
  */
 
-G_DEFINE_INTERFACE(GArrowIOWriteableFile,
-                   garrow_io_writeable_file,
+G_DEFINE_INTERFACE(GArrowWriteableFile,
+                   garrow_writeable_file,
                    G_TYPE_OBJECT)
 
 static void
-garrow_io_writeable_file_default_init (GArrowIOWriteableFileInterface *iface)
+garrow_writeable_file_default_init (GArrowWriteableFileInterface *iface)
 {
 }
 
 /**
- * garrow_io_writeable_file_write_at:
- * @writeable_file: A #GArrowIOWriteableFile.
+ * garrow_writeable_file_write_at:
+ * @writeable_file: A #GArrowWriteableFile.
  * @position: The write start position.
  * @data: (array length=n_bytes): The data to be written.
  * @n_bytes: The number of bytes to be written.
@@ -56,14 +56,14 @@ garrow_io_writeable_file_default_init (GArrowIOWriteableFileInterface *iface)
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_io_writeable_file_write_at(GArrowIOWriteableFile *writeable_file,
+garrow_writeable_file_write_at(GArrowWriteableFile *writeable_file,
                                   gint64 position,
                                   const guint8 *data,
                                   gint64 n_bytes,
                                   GError **error)
 {
   const auto arrow_writeable_file =
-    garrow_io_writeable_file_get_raw(writeable_file);
+    garrow_writeable_file_get_raw(writeable_file);
 
   auto status = arrow_writeable_file->WriteAt(position, data, n_bytes);
   if (status.ok()) {
@@ -77,8 +77,8 @@ garrow_io_writeable_file_write_at(GArrowIOWriteableFile *writeable_file,
 G_END_DECLS
 
 std::shared_ptr<arrow::io::WriteableFile>
-garrow_io_writeable_file_get_raw(GArrowIOWriteableFile *writeable_file)
+garrow_writeable_file_get_raw(GArrowWriteableFile *writeable_file)
 {
-  auto *iface = GARROW_IO_WRITEABLE_FILE_GET_IFACE(writeable_file);
+  auto *iface = GARROW_WRITEABLE_FILE_GET_IFACE(writeable_file);
   return iface->get_raw(writeable_file);
 }
diff --git a/c_glib/arrow-glib/io-writeable-file.h b/c_glib/arrow-glib/writeable-file.h
similarity index 59%
rename from c_glib/arrow-glib/io-writeable-file.h
rename to c_glib/arrow-glib/writeable-file.h
index d1ebdbe630ef2..7f4c186379b7e 100644
--- a/c_glib/arrow-glib/io-writeable-file.h
+++ b/c_glib/arrow-glib/writeable-file.h
@@ -23,26 +23,26 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_WRITEABLE_FILE           \
-  (garrow_io_writeable_file_get_type())
-#define GARROW_IO_WRITEABLE_FILE(obj)                           \
+#define GARROW_TYPE_WRITEABLE_FILE           \
+  (garrow_writeable_file_get_type())
+#define GARROW_WRITEABLE_FILE(obj)                           \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IO_TYPE_WRITEABLE_FILE,    \
-                              GArrowIOWriteableFile))
-#define GARROW_IO_IS_WRITEABLE_FILE(obj)                        \
+                              GARROW_TYPE_WRITEABLE_FILE,    \
+                              GArrowWriteableFile))
+#define GARROW_IS_WRITEABLE_FILE(obj)                        \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IO_TYPE_WRITEABLE_FILE))
-#define GARROW_IO_WRITEABLE_FILE_GET_IFACE(obj)                         \
+                              GARROW_TYPE_WRITEABLE_FILE))
+#define GARROW_WRITEABLE_FILE_GET_IFACE(obj)                         \
   (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
-                                 GARROW_IO_TYPE_WRITEABLE_FILE,         \
-                                 GArrowIOWriteableFileInterface))
+                                 GARROW_TYPE_WRITEABLE_FILE,         \
+                                 GArrowWriteableFileInterface))
 
-typedef struct _GArrowIOWriteableFile          GArrowIOWriteableFile;
-typedef struct _GArrowIOWriteableFileInterface GArrowIOWriteableFileInterface;
+typedef struct _GArrowWriteableFile          GArrowWriteableFile;
+typedef struct _GArrowWriteableFileInterface GArrowWriteableFileInterface;
 
-GType garrow_io_writeable_file_get_type(void) G_GNUC_CONST;
+GType garrow_writeable_file_get_type(void) G_GNUC_CONST;
 
-gboolean garrow_io_writeable_file_write_at(GArrowIOWriteableFile *writeable_file,
+gboolean garrow_writeable_file_write_at(GArrowWriteableFile *writeable_file,
                                            gint64 position,
                                            const guint8 *data,
                                            gint64 n_bytes,
diff --git a/c_glib/arrow-glib/io-writeable-file.hpp b/c_glib/arrow-glib/writeable-file.hpp
similarity index 75%
rename from c_glib/arrow-glib/io-writeable-file.hpp
rename to c_glib/arrow-glib/writeable-file.hpp
index aba95b209d827..aa3cc5082d0c5 100644
--- a/c_glib/arrow-glib/io-writeable-file.hpp
+++ b/c_glib/arrow-glib/writeable-file.hpp
@@ -21,18 +21,18 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-writeable-file.h>
+#include <arrow-glib/writeable-file.h>
 
 /**
- * GArrowIOWriteableFile:
+ * GArrowWriteableFile:
  *
  * It wraps `arrow::io::WriteableFile`.
  */
-struct _GArrowIOWriteableFileInterface
+struct _GArrowWriteableFileInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::WriteableFile> (*get_raw)(GArrowIOWriteableFile *file);
+  std::shared_ptr<arrow::io::WriteableFile> (*get_raw)(GArrowWriteableFile *file);
 };
 
-std::shared_ptr<arrow::io::WriteableFile> garrow_io_writeable_file_get_raw(GArrowIOWriteableFile *writeable_file);
+std::shared_ptr<arrow::io::WriteableFile> garrow_writeable_file_get_raw(GArrowWriteableFile *writeable_file);
diff --git a/c_glib/arrow-glib/io-writeable.cpp b/c_glib/arrow-glib/writeable.cpp
similarity index 72%
rename from c_glib/arrow-glib/io-writeable.cpp
rename to c_glib/arrow-glib/writeable.cpp
index 9ea69e3adccde..6f4c63008ae49 100644
--- a/c_glib/arrow-glib/io-writeable.cpp
+++ b/c_glib/arrow-glib/writeable.cpp
@@ -24,31 +24,31 @@
 #include <arrow/api.h>
 
 #include <arrow-glib/error.hpp>
-#include <arrow-glib/io-writeable.hpp>
+#include <arrow-glib/writeable.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: io-writeable
- * @title: GArrowIOWriteable
+ * SECTION: writeable
+ * @title: GArrowWriteable
  * @short_description: Output interface
  *
- * #GArrowIOWriteable is an interface for output. Output must be
+ * #GArrowWriteable is an interface for output. Output must be
  * writeable.
  */
 
-G_DEFINE_INTERFACE(GArrowIOWriteable,
-                   garrow_io_writeable,
+G_DEFINE_INTERFACE(GArrowWriteable,
+                   garrow_writeable,
                    G_TYPE_OBJECT)
 
 static void
-garrow_io_writeable_default_init (GArrowIOWriteableInterface *iface)
+garrow_writeable_default_init (GArrowWriteableInterface *iface)
 {
 }
 
 /**
- * garrow_io_writeable_write:
- * @writeable: A #GArrowIOWriteable.
+ * garrow_writeable_write:
+ * @writeable: A #GArrowWriteable.
  * @data: (array length=n_bytes): The data to be written.
  * @n_bytes: The number of bytes to be written.
  * @error: (nullable): Return location for a #GError or %NULL.
@@ -56,12 +56,12 @@ garrow_io_writeable_default_init (GArrowIOWriteableInterface *iface)
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_io_writeable_write(GArrowIOWriteable *writeable,
+garrow_writeable_write(GArrowWriteable *writeable,
                           const guint8 *data,
                           gint64 n_bytes,
                           GError **error)
 {
-  const auto arrow_writeable = garrow_io_writeable_get_raw(writeable);
+  const auto arrow_writeable = garrow_writeable_get_raw(writeable);
 
   auto status = arrow_writeable->Write(data, n_bytes);
   if (status.ok()) {
@@ -73,8 +73,8 @@ garrow_io_writeable_write(GArrowIOWriteable *writeable,
 }
 
 /**
- * garrow_io_writeable_flush:
- * @writeable: A #GArrowIOWriteable.
+ * garrow_writeable_flush:
+ * @writeable: A #GArrowWriteable.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
  * It ensures writing all data on memory to storage.
@@ -82,10 +82,10 @@ garrow_io_writeable_write(GArrowIOWriteable *writeable,
  * Returns: %TRUE on success, %FALSE if there was an error.
  */
 gboolean
-garrow_io_writeable_flush(GArrowIOWriteable *writeable,
+garrow_writeable_flush(GArrowWriteable *writeable,
                           GError **error)
 {
-  const auto arrow_writeable = garrow_io_writeable_get_raw(writeable);
+  const auto arrow_writeable = garrow_writeable_get_raw(writeable);
 
   auto status = arrow_writeable->Flush();
   if (status.ok()) {
@@ -99,8 +99,8 @@ garrow_io_writeable_flush(GArrowIOWriteable *writeable,
 G_END_DECLS
 
 std::shared_ptr<arrow::io::Writeable>
-garrow_io_writeable_get_raw(GArrowIOWriteable *writeable)
+garrow_writeable_get_raw(GArrowWriteable *writeable)
 {
-  auto *iface = GARROW_IO_WRITEABLE_GET_IFACE(writeable);
+  auto *iface = GARROW_WRITEABLE_GET_IFACE(writeable);
   return iface->get_raw(writeable);
 }
diff --git a/c_glib/arrow-glib/io-writeable.h b/c_glib/arrow-glib/writeable.h
similarity index 58%
rename from c_glib/arrow-glib/io-writeable.h
rename to c_glib/arrow-glib/writeable.h
index ce23247497706..66d6922360ae4 100644
--- a/c_glib/arrow-glib/io-writeable.h
+++ b/c_glib/arrow-glib/writeable.h
@@ -23,30 +23,30 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_IO_TYPE_WRITEABLE                \
-  (garrow_io_writeable_get_type())
-#define GARROW_IO_WRITEABLE(obj)                                \
+#define GARROW_TYPE_WRITEABLE                \
+  (garrow_writeable_get_type())
+#define GARROW_WRITEABLE(obj)                                \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_IO_TYPE_WRITEABLE,         \
-                              GArrowIOWriteable))
-#define GARROW_IO_IS_WRITEABLE(obj)                             \
+                              GARROW_TYPE_WRITEABLE,         \
+                              GArrowWriteable))
+#define GARROW_IS_WRITEABLE(obj)                             \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_IO_TYPE_WRITEABLE))
-#define GARROW_IO_WRITEABLE_GET_IFACE(obj)                      \
+                              GARROW_TYPE_WRITEABLE))
+#define GARROW_WRITEABLE_GET_IFACE(obj)                      \
   (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
-                                 GARROW_IO_TYPE_WRITEABLE,      \
-                                 GArrowIOWriteableInterface))
+                                 GARROW_TYPE_WRITEABLE,      \
+                                 GArrowWriteableInterface))
 
-typedef struct _GArrowIOWriteable          GArrowIOWriteable;
-typedef struct _GArrowIOWriteableInterface GArrowIOWriteableInterface;
+typedef struct _GArrowWriteable          GArrowWriteable;
+typedef struct _GArrowWriteableInterface GArrowWriteableInterface;
 
-GType garrow_io_writeable_get_type(void) G_GNUC_CONST;
+GType garrow_writeable_get_type(void) G_GNUC_CONST;
 
-gboolean garrow_io_writeable_write(GArrowIOWriteable *writeable,
+gboolean garrow_writeable_write(GArrowWriteable *writeable,
                                    const guint8 *data,
                                    gint64 n_bytes,
                                    GError **error);
-gboolean garrow_io_writeable_flush(GArrowIOWriteable *writeable,
+gboolean garrow_writeable_flush(GArrowWriteable *writeable,
                                    GError **error);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/io-writeable.hpp b/c_glib/arrow-glib/writeable.hpp
similarity index 77%
rename from c_glib/arrow-glib/io-writeable.hpp
rename to c_glib/arrow-glib/writeable.hpp
index f833924a61ae8..2b398f8b507c1 100644
--- a/c_glib/arrow-glib/io-writeable.hpp
+++ b/c_glib/arrow-glib/writeable.hpp
@@ -21,18 +21,18 @@
 
 #include <arrow/io/interfaces.h>
 
-#include <arrow-glib/io-writeable.h>
+#include <arrow-glib/writeable.h>
 
 /**
- * GArrowIOWriteableInterface:
+ * GArrowWriteableInterface:
  *
  * It wraps `arrow::io::Writeable`.
  */
-struct _GArrowIOWriteableInterface
+struct _GArrowWriteableInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::Writeable> (*get_raw)(GArrowIOWriteable *file);
+  std::shared_ptr<arrow::io::Writeable> (*get_raw)(GArrowWriteable *file);
 };
 
-std::shared_ptr<arrow::io::Writeable> garrow_io_writeable_get_raw(GArrowIOWriteable *writeable);
+std::shared_ptr<arrow::io::Writeable> garrow_writeable_get_raw(GArrowWriteable *writeable);
diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am
index 116bc6ce1b9a6..d3389dc2ae81e 100644
--- a/c_glib/doc/reference/Makefile.am
+++ b/c_glib/doc/reference/Makefile.am
@@ -26,7 +26,7 @@ SCAN_OPTIONS =						\
 	--deprecated-guards="GARROW_DISABLE_DEPRECATED"
 
 MKDB_OPTIONS =					\
-	--name-space=arrow			\
+	--name-space=garrow			\
 	--source-suffixes="c,cpp,h"
 
 HFILE_GLOB =					\
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 06a19369640b5..396dce5049837 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -31,8 +31,8 @@
     </releaseinfo>
   </bookinfo>
 
-  <part id="arrow">
-    <title>GArrow</title>
+  <part id="data">
+    <title>Data</title>
     <chapter id="array">
       <title>Array</title>
       <xi:include href="xml/array.xml"/>
@@ -111,47 +111,47 @@
     </chapter>
   </part>
 
-  <part id="arrow-io">
-    <title>GArrowIO</title>
-    <chapter id="io-mode">
-      <title>Enums</title>
-      <xi:include href="xml/io-file-mode.xml"/>
+  <part id="io">
+    <title>IO</title>
+    <chapter id="mode">
+      <title>Mode</title>
+      <xi:include href="xml/file-mode.xml"/>
     </chapter>
-    <chapter id="io-input">
+    <chapter id="input">
       <title>Input</title>
-      <xi:include href="xml/io-readable.xml"/>
-      <xi:include href="xml/io-input-stream.xml"/>
-      <xi:include href="xml/io-random-access-file.xml"/>
+      <xi:include href="xml/readable.xml"/>
+      <xi:include href="xml/input-stream.xml"/>
+      <xi:include href="xml/random-access-file.xml"/>
     </chapter>
-    <chapter id="io-output">
+    <chapter id="output">
       <title>Output</title>
-      <xi:include href="xml/io-writeable.xml"/>
-      <xi:include href="xml/io-output-stream.xml"/>
-      <xi:include href="xml/io-writeable-file.xml"/>
-      <xi:include href="xml/io-file-output-stream.xml"/>
+      <xi:include href="xml/writeable.xml"/>
+      <xi:include href="xml/output-stream.xml"/>
+      <xi:include href="xml/writeable-file.xml"/>
+      <xi:include href="xml/file-output-stream.xml"/>
     </chapter>
-    <chapter id="io-input-output">
+    <chapter id="input-output">
       <title>Input and output</title>
-      <xi:include href="xml/io-file.xml"/>
-      <xi:include href="xml/io-memory-mapped-file.xml"/>
+      <xi:include href="xml/file.xml"/>
+      <xi:include href="xml/memory-mapped-file.xml"/>
     </chapter>
   </part>
 
-  <part id="arrow-ipc">
-    <title>GArrowIPC</title>
-    <chapter id="ipc-metadata">
-      <title>Enums</title>
-      <xi:include href="xml/ipc-metadata-version.xml"/>
+  <part id="ipc">
+    <title>IPC</title>
+    <chapter id="metadata">
+      <title>Metadata</title>
+      <xi:include href="xml/metadata-version.xml"/>
     </chapter>
-    <chapter id="ipc-reader">
+    <chapter id="reader">
       <title>Reader</title>
-      <xi:include href="xml/ipc-file-reader.xml"/>
-      <xi:include href="xml/ipc-stream-reader.xml"/>
+      <xi:include href="xml/file-reader.xml"/>
+      <xi:include href="xml/stream-reader.xml"/>
     </chapter>
-    <chapter id="ipc-writer">
-      <title>Input</title>
-      <xi:include href="xml/ipc-file-writer.xml"/>
-      <xi:include href="xml/ipc-stream-writer.xml"/>
+    <chapter id="writer">
+      <title>Writer</title>
+      <xi:include href="xml/file-writer.xml"/>
+      <xi:include href="xml/stream-writer.xml"/>
     </chapter>
   </part>
 
diff --git a/c_glib/example/read-batch.c b/c_glib/example/read-batch.c
index a55b085d961d1..dce96b8713362 100644
--- a/c_glib/example/read-batch.c
+++ b/c_glib/example/read-batch.c
@@ -87,14 +87,14 @@ int
 main(int argc, char **argv)
 {
   const char *input_path = "/tmp/batch.arrow";
-  GArrowIOMemoryMappedFile *input;
+  GArrowMemoryMappedFile *input;
   GError *error = NULL;
 
   if (argc > 1)
     input_path = argv[1];
-  input = garrow_io_memory_mapped_file_open(input_path,
-                                            GARROW_IO_FILE_MODE_READ,
-                                            &error);
+  input = garrow_memory_mapped_file_open(input_path,
+                                         GARROW_FILE_MODE_READ,
+                                         &error);
   if (!input) {
     g_print("failed to open file: %s\n", error->message);
     g_error_free(error);
@@ -102,10 +102,10 @@ main(int argc, char **argv)
   }
 
   {
-    GArrowIPCFileReader *reader;
+    GArrowFileReader *reader;
 
-    reader = garrow_ipc_file_reader_open(GARROW_IO_RANDOM_ACCESS_FILE(input),
-                                         &error);
+    reader = garrow_file_reader_open(GARROW_RANDOM_ACCESS_FILE(input),
+                                     &error);
     if (!reader) {
       g_print("failed to open file reader: %s\n", error->message);
       g_error_free(error);
@@ -116,12 +116,12 @@ main(int argc, char **argv)
     {
       guint i, n;
 
-      n = garrow_ipc_file_reader_get_n_record_batches(reader);
+      n = garrow_file_reader_get_n_record_batches(reader);
       for (i = 0; i < n; i++) {
         GArrowRecordBatch *record_batch;
 
         record_batch =
-          garrow_ipc_file_reader_get_record_batch(reader, i, &error);
+          garrow_file_reader_get_record_batch(reader, i, &error);
         if (!record_batch) {
           g_print("failed to open file reader: %s\n", error->message);
           g_error_free(error);
diff --git a/c_glib/example/read-stream.c b/c_glib/example/read-stream.c
index c56942c7770c5..ba461e3ad6aed 100644
--- a/c_glib/example/read-stream.c
+++ b/c_glib/example/read-stream.c
@@ -87,14 +87,14 @@ int
 main(int argc, char **argv)
 {
   const char *input_path = "/tmp/stream.arrow";
-  GArrowIOMemoryMappedFile *input;
+  GArrowMemoryMappedFile *input;
   GError *error = NULL;
 
   if (argc > 1)
     input_path = argv[1];
-  input = garrow_io_memory_mapped_file_open(input_path,
-                                            GARROW_IO_FILE_MODE_READ,
-                                            &error);
+  input = garrow_memory_mapped_file_open(input_path,
+                                         GARROW_FILE_MODE_READ,
+                                         &error);
   if (!input) {
     g_print("failed to open file: %s\n", error->message);
     g_error_free(error);
@@ -102,10 +102,10 @@ main(int argc, char **argv)
   }
 
   {
-    GArrowIPCStreamReader *reader;
+    GArrowStreamReader *reader;
 
-    reader = garrow_ipc_stream_reader_open(GARROW_IO_INPUT_STREAM(input),
-                                           &error);
+    reader = garrow_stream_reader_open(GARROW_INPUT_STREAM(input),
+                                       &error);
     if (!reader) {
       g_print("failed to open stream reader: %s\n", error->message);
       g_error_free(error);
@@ -117,7 +117,7 @@ main(int argc, char **argv)
       GArrowRecordBatch *record_batch;
 
       record_batch =
-        garrow_ipc_stream_reader_get_next_record_batch(reader, &error);
+        garrow_stream_reader_get_next_record_batch(reader, &error);
       if (error) {
         g_print("failed to get record batch: %s\n", error->message);
         g_error_free(error);
diff --git a/c_glib/test/test-io-file-output-stream.rb b/c_glib/test/test-file-output-stream.rb
similarity index 87%
rename from c_glib/test/test-io-file-output-stream.rb
rename to c_glib/test/test-file-output-stream.rb
index e35a18361aab6..26737c0c87b38 100644
--- a/c_glib/test/test-io-file-output-stream.rb
+++ b/c_glib/test/test-file-output-stream.rb
@@ -15,13 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class TestIOFileOutputStream < Test::Unit::TestCase
+class TestFileOutputStream < Test::Unit::TestCase
   sub_test_case(".open") do
     def test_create
       tempfile = Tempfile.open("arrow-io-file-output-stream")
       tempfile.write("Hello")
       tempfile.close
-      file = Arrow::IOFileOutputStream.open(tempfile.path, false)
+      file = Arrow::FileOutputStream.open(tempfile.path, false)
       file.close
       assert_equal("", File.read(tempfile.path))
     end
@@ -30,7 +30,7 @@ def test_append
       tempfile = Tempfile.open("arrow-io-file-output-stream")
       tempfile.write("Hello")
       tempfile.close
-      file = Arrow::IOFileOutputStream.open(tempfile.path, true)
+      file = Arrow::FileOutputStream.open(tempfile.path, true)
       file.close
       assert_equal("Hello", File.read(tempfile.path))
     end
diff --git a/c_glib/test/test-ipc-file-writer.rb b/c_glib/test/test-file-writer.rb
similarity index 82%
rename from c_glib/test/test-ipc-file-writer.rb
rename to c_glib/test/test-file-writer.rb
index 1c33ccc1919e7..31c094dd3bd29 100644
--- a/c_glib/test/test-ipc-file-writer.rb
+++ b/c_glib/test/test-file-writer.rb
@@ -15,14 +15,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class TestIPCFileWriter < Test::Unit::TestCase
+class TestFileWriter < Test::Unit::TestCase
   def test_write_record_batch
     tempfile = Tempfile.open("arrow-ipc-file-writer")
-    output = Arrow::IOFileOutputStream.open(tempfile.path, false)
+    output = Arrow::FileOutputStream.open(tempfile.path, false)
     begin
       field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
       schema = Arrow::Schema.new([field])
-      file_writer = Arrow::IPCFileWriter.open(output, schema)
+      file_writer = Arrow::FileWriter.open(output, schema)
       begin
         record_batch = Arrow::RecordBatch.new(schema, 0, [])
         file_writer.write_record_batch(record_batch)
@@ -33,9 +33,9 @@ def test_write_record_batch
       output.close
     end
 
-    input = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
+    input = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
-      file_reader = Arrow::IPCFileReader.open(input)
+      file_reader = Arrow::FileReader.open(input)
       assert_equal(["enabled"],
                    file_reader.schema.fields.collect(&:name))
     ensure
diff --git a/c_glib/test/test-io-memory-mapped-file.rb b/c_glib/test/test-memory-mapped-file.rb
similarity index 81%
rename from c_glib/test/test-io-memory-mapped-file.rb
rename to c_glib/test/test-memory-mapped-file.rb
index 197d1886f1e86..e78d07a72d3b8 100644
--- a/c_glib/test/test-io-memory-mapped-file.rb
+++ b/c_glib/test/test-memory-mapped-file.rb
@@ -15,12 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class TestIOMemoryMappedFile < Test::Unit::TestCase
+class TestMemoryMappedFile < Test::Unit::TestCase
   def test_open
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
       buffer = " " * 5
       file.read(buffer)
@@ -34,7 +34,7 @@ def test_size
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
       assert_equal(5, file.size)
     ensure
@@ -46,7 +46,7 @@ def test_read
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello World")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
       buffer = " " * 5
       _success, n_read_bytes = file.read(buffer)
@@ -60,7 +60,7 @@ def test_read_at
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello World")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
       buffer = " " * 5
       _success, n_read_bytes = file.read_at(6, buffer)
@@ -74,7 +74,7 @@ def test_write
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :readwrite)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :readwrite)
     begin
       file.write("World")
     ensure
@@ -87,7 +87,7 @@ def test_write_at
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :readwrite)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :readwrite)
     begin
       file.write_at(2, "rld")
     ensure
@@ -100,7 +100,7 @@ def test_flush
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :readwrite)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :readwrite)
     begin
       file.write("World")
       file.flush
@@ -114,7 +114,7 @@ def test_tell
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello World")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
       buffer = " " * 5
       file.read(buffer)
@@ -128,9 +128,9 @@ def test_mode
     tempfile = Tempfile.open("arrow-io-memory-mapped-file")
     tempfile.write("Hello World")
     tempfile.close
-    file = Arrow::IOMemoryMappedFile.open(tempfile.path, :readwrite)
+    file = Arrow::MemoryMappedFile.open(tempfile.path, :readwrite)
     begin
-      assert_equal(Arrow::IOFileMode::READWRITE, file.mode)
+      assert_equal(Arrow::FileMode::READWRITE, file.mode)
     ensure
       file.close
     end
diff --git a/c_glib/test/test-ipc-stream-writer.rb b/c_glib/test/test-stream-writer.rb
similarity index 84%
rename from c_glib/test/test-ipc-stream-writer.rb
rename to c_glib/test/test-stream-writer.rb
index 78bb4a7c1743c..306115ee78925 100644
--- a/c_glib/test/test-ipc-stream-writer.rb
+++ b/c_glib/test/test-stream-writer.rb
@@ -15,16 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class TestIPCStreamWriter < Test::Unit::TestCase
+class TestStreamWriter < Test::Unit::TestCase
   include Helper::Buildable
 
   def test_write_record_batch
     tempfile = Tempfile.open("arrow-ipc-stream-writer")
-    output = Arrow::IOFileOutputStream.open(tempfile.path, false)
+    output = Arrow::FileOutputStream.open(tempfile.path, false)
     begin
       field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
       schema = Arrow::Schema.new([field])
-      stream_writer = Arrow::IPCStreamWriter.open(output, schema)
+      stream_writer = Arrow::StreamWriter.open(output, schema)
       begin
         columns = [
           build_boolean_array([true]),
@@ -38,9 +38,9 @@ def test_write_record_batch
       output.close
     end
 
-    input = Arrow::IOMemoryMappedFile.open(tempfile.path, :read)
+    input = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
-      stream_reader = Arrow::IPCStreamReader.open(input)
+      stream_reader = Arrow::StreamReader.open(input)
       assert_equal(["enabled"],
                    stream_reader.schema.fields.collect(&:name))
       assert_equal(true,

From 9db96fea4e5de59860a481da3036b3129eb97e3b Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 12 Apr 2017 12:23:22 -0400
Subject: [PATCH 0504/1644] ARROW-811: [GLib] Add GArrowBuffer

Author: Kouhei Sutou <kou@clear-code.com>

Closes #531 from kou/glib-add-buffer and squashes the following commits:

1954c95 [Kouhei Sutou] [GLib] Add GArrowBuffer
---
 c_glib/arrow-glib/Makefile.am             |   7 +-
 c_glib/arrow-glib/buffer.cpp              | 289 ++++++++++++++++++++++
 c_glib/arrow-glib/buffer.h                |  77 ++++++
 c_glib/arrow-glib/buffer.hpp              |  27 ++
 c_glib/doc/reference/arrow-glib-docs.sgml |   4 +
 c_glib/test/test-buffer.rb                |  55 ++++
 6 files changed, 457 insertions(+), 2 deletions(-)
 create mode 100644 c_glib/arrow-glib/buffer.cpp
 create mode 100644 c_glib/arrow-glib/buffer.h
 create mode 100644 c_glib/arrow-glib/buffer.hpp
 create mode 100644 c_glib/test/test-buffer.rb

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 387707c7d5897..2e7a9a0e439eb 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -44,14 +44,15 @@ libarrow_glib_la_headers =			\
 	array.h					\
 	array-builder.h				\
 	arrow-glib.h				\
-	chunked-array.h				\
-	column.h				\
 	binary-array.h				\
 	binary-array-builder.h			\
 	binary-data-type.h			\
 	boolean-array.h				\
 	boolean-array-builder.h			\
 	boolean-data-type.h			\
+	buffer.h				\
+	chunked-array.h				\
+	column.h				\
 	data-type.h				\
 	double-array.h				\
 	double-array-builder.h			\
@@ -136,6 +137,7 @@ libarrow_glib_la_sources =			\
 	boolean-array.cpp			\
 	boolean-array-builder.cpp		\
 	boolean-data-type.cpp			\
+	buffer.cpp				\
 	chunked-array.cpp			\
 	column.cpp				\
 	data-type.cpp				\
@@ -212,6 +214,7 @@ libarrow_glib_la_cpp_headers =			\
 	array.hpp				\
 	array-builder.hpp			\
 	arrow-glib.hpp				\
+	buffer.hpp				\
 	chunked-array.hpp			\
 	column.hpp				\
 	data-type.hpp				\
diff --git a/c_glib/arrow-glib/buffer.cpp b/c_glib/arrow-glib/buffer.cpp
new file mode 100644
index 0000000000000..0ec52df0aee67
--- /dev/null
+++ b/c_glib/arrow-glib/buffer.cpp
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/buffer.hpp>
+#include <arrow-glib/error.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: buffer
+ * @short_description: Buffer class
+ *
+ * #GArrowBuffer is a class for keeping data. Other classes such as
+ * #GArrowArray and #GArrowTensor can use data in buffer.
+ */
+
+typedef struct GArrowBufferPrivate_ {
+  std::shared_ptr<arrow::Buffer> buffer;
+} GArrowBufferPrivate;
+
+enum {
+  PROP_0,
+  PROP_BUFFER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowBuffer, garrow_buffer, G_TYPE_OBJECT)
+
+#define GARROW_BUFFER_GET_PRIVATE(obj) \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj), GARROW_TYPE_BUFFER, GArrowBufferPrivate))
+
+static void
+garrow_buffer_finalize(GObject *object)
+{
+  auto priv = GARROW_BUFFER_GET_PRIVATE(object);
+
+  priv->buffer = nullptr;
+
+  G_OBJECT_CLASS(garrow_buffer_parent_class)->finalize(object);
+}
+
+static void
+garrow_buffer_set_property(GObject *object,
+                          guint prop_id,
+                          const GValue *value,
+                          GParamSpec *pspec)
+{
+  auto priv = GARROW_BUFFER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_BUFFER:
+    priv->buffer =
+      *static_cast<std::shared_ptr<arrow::Buffer> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_buffer_get_property(GObject *object,
+                          guint prop_id,
+                          GValue *value,
+                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_buffer_init(GArrowBuffer *object)
+{
+}
+
+static void
+garrow_buffer_class_init(GArrowBufferClass *klass)
+{
+  GParamSpec *spec;
+
+  auto gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_buffer_finalize;
+  gobject_class->set_property = garrow_buffer_set_property;
+  gobject_class->get_property = garrow_buffer_get_property;
+
+  spec = g_param_spec_pointer("buffer",
+                              "Buffer",
+                              "The raw std::shared<arrow::Buffer> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_BUFFER, spec);
+}
+
+/**
+ * garrow_buffer_new:
+ * @data: (array length=size): Data for the buffer.
+ *   They aren't owned by the new buffer.
+ *   You must not free the data while the new buffer is alive.
+ * @size: The number of bytes of the data.
+ *
+ * Returns: A newly created #GArrowBuffer.
+ *
+ * Since: 0.3.0
+ */
+GArrowBuffer *
+garrow_buffer_new(const guint8 *data, gint64 size)
+{
+  auto arrow_buffer = std::make_shared<arrow::Buffer>(data, size);
+  return garrow_buffer_new_raw(&arrow_buffer);
+
+}
+
+/**
+ * garrow_buffer_is_mutable:
+ * @buffer: A #GArrowBuffer.
+ *
+ * Returns: %TRUE if the buffer is mutable, %FALSE otherwise.
+ *
+ * Since: 0.3.0
+ */
+gboolean
+garrow_buffer_is_mutable(GArrowBuffer *buffer)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  return arrow_buffer->is_mutable();
+}
+
+/**
+ * garrow_buffer_get_capacity:
+ * @buffer: A #GArrowBuffer.
+ *
+ * Returns: The number of bytes that where allocated for the buffer in
+ *   total.
+ *
+ * Since: 0.3.0
+ */
+gint64
+garrow_buffer_get_capacity(GArrowBuffer *buffer)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  return arrow_buffer->capacity();
+}
+
+/**
+ * garrow_buffer_get_data:
+ * @buffer: A #GArrowBuffer.
+ * @size: (out): The number of bytes of the data.
+ *
+ * Returns: (array length=size): The data of the buffer.
+ *
+ * Since: 0.3.0
+ */
+const guint8 *
+garrow_buffer_get_data(GArrowBuffer *buffer, gint64 *size)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  *size = arrow_buffer->size();
+  return arrow_buffer->data();
+}
+
+/**
+ * garrow_buffer_get_size:
+ * @buffer: A #GArrowBuffer.
+ *
+ * Returns: The number of bytes that might have valid data.
+ *
+ * Since: 0.3.0
+ */
+gint64
+garrow_buffer_get_size(GArrowBuffer *buffer)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  return arrow_buffer->size();
+}
+
+/**
+ * garrow_buffer_get_parent:
+ * @buffer: A #GArrowBuffer.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The parent #GArrowBuffer or %NULL.
+ *
+ * Since: 0.3.0
+ */
+GArrowBuffer *
+garrow_buffer_get_parent(GArrowBuffer *buffer)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  auto arrow_parent_buffer = arrow_buffer->parent();
+
+  if (arrow_parent_buffer) {
+    return garrow_buffer_new_raw(&arrow_parent_buffer);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_buffer_copy:
+ * @buffer: A #GArrowBuffer.
+ * @start: An offset of data to be copied in byte.
+ * @size: The number of bytes to be copied from the start.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   A newly copied #GArrowBuffer on success, %NULL on error.
+ *
+ * Since: 0.3.0
+ */
+GArrowBuffer *
+garrow_buffer_copy(GArrowBuffer *buffer,
+                   gint64 start,
+                   gint64 size,
+                   GError **error)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  std::shared_ptr<arrow::Buffer> arrow_copied_buffer;
+  auto status = arrow_buffer->Copy(start, size, &arrow_copied_buffer);
+  if (status.ok()) {
+    return garrow_buffer_new_raw(&arrow_copied_buffer);
+  } else {
+    garrow_error_set(error, status, "[buffer][copy]");
+    return NULL;
+  }
+}
+
+/**
+ * garrow_buffer_slice:
+ * @buffer: A #GArrowBuffer.
+ * @offset: An offset in the buffer data in byte.
+ * @size: The number of bytes of the sliced data.
+ *
+ * Returns: (transfer full): A newly created #GArrowBuffer that shares
+ *   data of the base #GArrowBuffer. The created #GArrowBuffer has data
+ *   start with offset from the base buffer data and are the specified
+ *   bytes size.
+ *
+ * Since: 0.3.0
+ */
+GArrowBuffer *
+garrow_buffer_slice(GArrowBuffer *buffer, gint64 offset, gint64 size)
+{
+  auto arrow_parent_buffer = garrow_buffer_get_raw(buffer);
+  auto arrow_buffer = std::make_shared<arrow::Buffer>(arrow_parent_buffer,
+                                                      offset,
+                                                      size);
+  return garrow_buffer_new_raw(&arrow_buffer);
+}
+
+G_END_DECLS
+
+GArrowBuffer *
+garrow_buffer_new_raw(std::shared_ptr<arrow::Buffer> *arrow_buffer)
+{
+  auto buffer = GARROW_BUFFER(g_object_new(GARROW_TYPE_BUFFER,
+                                           "buffer", arrow_buffer,
+                                           NULL));
+  return buffer;
+}
+
+std::shared_ptr<arrow::Buffer>
+garrow_buffer_get_raw(GArrowBuffer *buffer)
+{
+  auto priv = GARROW_BUFFER_GET_PRIVATE(buffer);
+  return priv->buffer;
+}
diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h
new file mode 100644
index 0000000000000..1e7d55182fd1d
--- /dev/null
+++ b/c_glib/arrow-glib/buffer.h
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_BUFFER \
+  (garrow_buffer_get_type())
+#define GARROW_BUFFER(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj), GARROW_TYPE_BUFFER, GArrowBuffer))
+#define GARROW_BUFFER_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST((klass), GARROW_TYPE_BUFFER, GArrowBufferClass))
+#define GARROW_IS_BUFFER(obj) \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_BUFFER))
+#define GARROW_IS_BUFFER_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_BUFFER))
+#define GARROW_BUFFER_GET_CLASS(obj) \
+  (G_TYPE_INSTANCE_GET_CLASS((obj), GARROW_TYPE_BUFFER, GArrowBufferClass))
+
+typedef struct _GArrowBuffer         GArrowBuffer;
+typedef struct _GArrowBufferClass    GArrowBufferClass;
+
+/**
+ * GArrowBuffer:
+ *
+ * It wraps `arrow::Buffer`.
+ */
+struct _GArrowBuffer
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowBufferClass
+{
+  GObjectClass parent_class;
+};
+
+GType          garrow_buffer_get_type     (void) G_GNUC_CONST;
+
+GArrowBuffer  *garrow_buffer_new          (const guint8 *data,
+                                           gint64 size);
+gboolean       garrow_buffer_is_mutable   (GArrowBuffer *buffer);
+gint64         garrow_buffer_get_capacity (GArrowBuffer *buffer);
+const guint8  *garrow_buffer_get_data     (GArrowBuffer *buffer,
+                                           gint64 *size);
+gint64         garrow_buffer_get_size     (GArrowBuffer *buffer);
+GArrowBuffer  *garrow_buffer_get_parent   (GArrowBuffer *buffer);
+
+GArrowBuffer  *garrow_buffer_copy         (GArrowBuffer *buffer,
+                                           gint64 start,
+                                           gint64 size,
+                                           GError **error);
+GArrowBuffer  *garrow_buffer_slice        (GArrowBuffer *buffer,
+                                           gint64 offset,
+                                           gint64 size);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/buffer.hpp b/c_glib/arrow-glib/buffer.hpp
new file mode 100644
index 0000000000000..00dd3de3bfd50
--- /dev/null
+++ b/c_glib/arrow-glib/buffer.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/buffer.h>
+
+GArrowBuffer *garrow_buffer_new_raw(std::shared_ptr<arrow::Buffer> *arrow_buffer);
+std::shared_ptr<arrow::Buffer> garrow_buffer_get_raw(GArrowBuffer *buffer);
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 396dce5049837..3c1d8d161179c 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -105,6 +105,10 @@
       <xi:include href="xml/column.xml"/>
       <xi:include href="xml/chunked-array.xml"/>
     </chapter>
+    <chapter id="buffer">
+      <title>Buffer</title>
+      <xi:include href="xml/buffer.xml"/>
+    </chapter>
     <chapter id="error">
       <title>Error</title>
       <xi:include href="xml/error.xml"/>
diff --git a/c_glib/test/test-buffer.rb b/c_glib/test/test-buffer.rb
new file mode 100644
index 0000000000000..1ea26f24ce873
--- /dev/null
+++ b/c_glib/test/test-buffer.rb
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBuffer < Test::Unit::TestCase
+  def setup
+    @data = "Hello"
+    @buffer = Arrow::Buffer.new(@data)
+  end
+
+  def test_mutable?
+    assert do
+      not @buffer.mutable?
+    end
+  end
+
+  def test_capacity
+    assert_equal(@data.bytesize, @buffer.capacity)
+  end
+
+  def test_data
+    assert_equal(@data, @buffer.data.pack("C*"))
+  end
+
+  def test_size
+    assert_equal(@data.bytesize, @buffer.size)
+  end
+
+  def test_parent
+    assert_nil(@buffer.parent)
+  end
+
+  def test_copy
+    copied_buffer = @buffer.copy(1, 3)
+    assert_equal(@data[1, 3], copied_buffer.data.pack("C*"))
+  end
+
+  def test_slice
+    sliced_buffer = @buffer.slice(1, 3)
+    assert_equal(@data[1, 3], sliced_buffer.data.pack("C*"))
+  end
+end

From 9d532c49d563ec22f73af3cc49549eb2e5cb6898 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 12 Apr 2017 13:05:21 -0400
Subject: [PATCH 0505/1644] ARROW-539: [Python] Add support for reading
 partitioned Parquet files with Hive-like directory schemes

I probably didn't get all the use cases, but this should be a good start.

First, the directory structure is walked to determine the distinct partition keys. These keys are later used as the dictionary for `arrow::DictionaryArray` objects which are constructed.

I also created the `ParquetDatasetPiece` class to enable distributed processing of file components in frameworks like Dask. We may need to address pickling of the `ParquetPartitions` object (which must be passed to `ParquetDatasetPiece.read` so the right array metadata can be constructed.

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Miki Tebeka <miki.tebeka@gmail.com>

Closes #529 from wesm/ARROW-539 and squashes the following commits:

a0451fa [Wes McKinney] Code review comments
deb6d82 [Wes McKinney] Don't make file-like Python object on LocalFilesystem
04dc691 [Wes McKinney] Complete initial partitioned reads, supporting unit tests. Expose arrow::Table::AddColumn
7d33755 [Wes McKinney] Untested draft of ParquetManifest for partitioned directory structures. Get test suite passing again
ba8825f [Wes McKinney] Prototyping
18fe639 [Wes McKinney] Refactoring, add ParquetDataset, ParquetDatasetPiece
016b445 [Miki Tebeka] [ARROW-539] [Python] Support reading Parquet datasets with standard partition directory schemes
---
 python/pyarrow/filesystem.py         |  25 +-
 python/pyarrow/includes/libarrow.pxd |   2 +
 python/pyarrow/parquet.py            | 547 +++++++++++++++++++++++----
 python/pyarrow/table.pxd             |   1 +
 python/pyarrow/table.pyx             |  40 +-
 python/pyarrow/tests/test_parquet.py | 156 +++++++-
 python/pyarrow/tests/test_table.py   |  31 ++
 7 files changed, 692 insertions(+), 110 deletions(-)

diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index e820806ab4e68..269cf1c8ffa12 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -87,20 +87,10 @@ def read_parquet(self, path, columns=None, metadata=None, schema=None,
         -------
         table : pyarrow.Table
         """
-        from pyarrow.parquet import read_multiple_files
-
-        if self.isdir(path):
-            paths_to_read = []
-            for path in self.ls(path):
-                if path.endswith('parq') or path.endswith('parquet'):
-                    paths_to_read.append(path)
-        else:
-            paths_to_read = [path]
-
-        return read_multiple_files(paths_to_read, columns=columns,
-                                   filesystem=self, schema=schema,
-                                   metadata=metadata,
-                                   nthreads=nthreads)
+        from pyarrow.parquet import ParquetDataset
+        dataset = ParquetDataset(path, schema=schema, metadata=metadata,
+                                 filesystem=self)
+        return dataset.read(columns=columns, nthreads=nthreads)
 
 
 class LocalFilesystem(Filesystem):
@@ -117,6 +107,13 @@ def get_instance(cls):
     def ls(self, path):
         return sorted(pjoin(path, x) for x in os.listdir(path))
 
+    @implements(Filesystem.mkdir)
+    def mkdir(self, path, create_parents=True):
+        if create_parents:
+            os.makedirs(path)
+        else:
+            os.mkdir(path)
+
     @implements(Filesystem.isdir)
     def isdir(self, path):
         return os.path.isdir(path)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 40dd83776b82d..ae2b45fbdb212 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -291,6 +291,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CSchema] schema()
         shared_ptr[CColumn] column(int i)
 
+        CStatus AddColumn(int i, const shared_ptr[CColumn]& column,
+                          shared_ptr[CTable]* out)
         CStatus RemoveColumn(int i, shared_ptr[CTable]* out)
 
     cdef cppclass CTensor" arrow::Tensor":
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index d95c3b3aecaf8..f81b6c24c691f 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -17,18 +17,23 @@
 
 import six
 
+import numpy as np
+
+from pyarrow.filesystem import LocalFilesystem
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
                               RowGroupMetaData, Schema, ParquetWriter)
 import pyarrow._parquet as _parquet  # noqa
-from pyarrow.table import concat_tables
+import pyarrow.array as _array
+import pyarrow.table as _table
 
 
-EXCLUDED_PARQUET_PATHS = {'_metadata', '_common_metadata', '_SUCCESS'}
+# ----------------------------------------------------------------------
+# Reading a single Parquet file
 
 
 class ParquetFile(object):
     """
-    Open a Parquet binary file for reading
+    Reader interface for a single Parquet file
 
     Parameters
     ----------
@@ -72,7 +77,8 @@ def read_row_group(self, i, columns=None, nthreads=1):
             Content of the row group as a table (of columns)
         """
         column_indices = self._get_column_indices(columns)
-        self.reader.set_num_threads(nthreads)
+        if nthreads is not None:
+            self.reader.set_num_threads(nthreads)
         return self.reader.read_row_group(i, column_indices=column_indices)
 
     def read(self, columns=None, nthreads=1):
@@ -93,7 +99,8 @@ def read(self, columns=None, nthreads=1):
             Content of the file as a table (of columns)
         """
         column_indices = self._get_column_indices(columns)
-        self.reader.set_num_threads(nthreads)
+        if nthreads is not None:
+            self.reader.set_num_threads(nthreads)
         return self.reader.read_all(column_indices=column_indices)
 
     def _get_column_indices(self, column_names):
@@ -104,6 +111,463 @@ def _get_column_indices(self, column_names):
                     for column in column_names]
 
 
+# ----------------------------------------------------------------------
+# Metadata container providing instructions about reading a single Parquet
+# file, possibly part of a partitioned dataset
+
+
+class ParquetDatasetPiece(object):
+    """
+    A single chunk of a potentially larger Parquet dataset to read. The
+    arguments will indicate to read either a single row group or all row
+    groups, and whether to add partition keys to the resulting pyarrow.Table
+
+    Parameters
+    ----------
+    path : str
+        Path to file in the file system where this piece is located
+    partition_keys : list of tuples
+      [(column name, ordinal index)]
+    row_group : int, default None
+        Row group to load. By default, reads all row groups
+    """
+
+    def __init__(self, path, row_group=None, partition_keys=None):
+        self.path = path
+        self.row_group = row_group
+        self.partition_keys = partition_keys or []
+
+    def __eq__(self, other):
+        if not isinstance(other, ParquetDatasetPiece):
+            return False
+        return (self.path == other.path and
+                self.row_group == other.row_group and
+                self.partition_keys == other.partition_keys)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return ('{0}({1!r}, row_group={2!r}, partition_keys={3!r})'
+                .format(type(self).__name__, self.path,
+                        self.row_group,
+                        self.partition_keys))
+
+    def __str__(self):
+        result = ''
+
+        if len(self.partition_keys) > 0:
+            partition_str = ', '.join('{0}={1}'.format(name, index)
+                                      for name, index in self.partition_keys)
+            result += 'partition[{0}] '.format(partition_str)
+
+        result += self.path
+
+        if self.row_group is not None:
+            result += ' | row_group={0}'.format(self.row_group)
+
+        return result
+
+    def get_metadata(self, open_file_func=None):
+        """
+        Given a function that can create an open ParquetFile object, return the
+        file's metadata
+        """
+        return self._open(open_file_func).metadata
+
+    def _open(self, open_file_func=None):
+        """
+        Returns instance of ParquetFile
+        """
+        if open_file_func is None:
+            def simple_opener(path):
+                return ParquetFile(path)
+            open_file_func = simple_opener
+        return open_file_func(self.path)
+
+    def read(self, columns=None, nthreads=1, partitions=None,
+             open_file_func=None):
+        """
+        Read this piece as a pyarrow.Table
+
+        Parameters
+        ----------
+        columns : list of column names, default None
+        nthreads : int, default 1
+            For multithreaded file reads
+        partitions : ParquetPartitions, default None
+        open_file_func : function, default None
+            A function that knows how to construct a ParquetFile object given
+            the file path in this piece
+
+        Returns
+        -------
+        table : pyarrow.Table
+        """
+        reader = self._open(open_file_func)
+
+        if self.row_group is not None:
+            table = reader.read_row_group(self.row_group, columns=columns,
+                                          nthreads=nthreads)
+        else:
+            table = reader.read(columns=columns, nthreads=nthreads)
+
+        if len(self.partition_keys) > 0:
+            if partitions is None:
+                raise ValueError('Must pass partition sets')
+
+            # Here, the index is the categorical code of the partition where
+            # this piece is located. Suppose we had
+            #
+            # /foo=a/0.parq
+            # /foo=b/0.parq
+            # /foo=c/0.parq
+            #
+            # Then we assign a=0, b=1, c=2. And the resulting Table pieces will
+            # have a DictionaryArray column named foo having the constant index
+            # value as indicated. The distinct categories of the partition have
+            # been computed in the ParquetManifest
+            for i, (name, index) in enumerate(self.partition_keys):
+                # The partition code is the same for all values in this piece
+                indices = np.array([index], dtype='i4').repeat(len(table))
+
+                # This is set of all partition values, computed as part of the
+                # manifest, so ['a', 'b', 'c'] as in our example above.
+                dictionary = partitions.levels[i].dictionary
+
+                arr = _array.DictionaryArray.from_arrays(indices, dictionary)
+                col = _table.Column.from_array(name, arr)
+                table = table.append_column(col)
+
+        return table
+
+
+def _is_parquet_file(path):
+    return path.endswith('parq') or path.endswith('parquet')
+
+
+class PartitionSet(object):
+    """A data structure for cataloguing the observed Parquet partitions at a
+    particular level. So if we have
+
+    /foo=a/bar=0
+    /foo=a/bar=1
+    /foo=a/bar=2
+    /foo=b/bar=0
+    /foo=b/bar=1
+    /foo=b/bar=2
+
+    Then we have two partition sets, one for foo, another for bar. As we visit
+    levels of the partition hierarchy, a PartitionSet tracks the distinct
+    values and assigns categorical codes to use when reading the pieces
+    """
+
+    def __init__(self, name, keys=None):
+        self.name = name
+        self.keys = keys or []
+        self.key_indices = {k: i for i, k in enumerate(self.keys)}
+        self._dictionary = None
+
+    def get_index(self, key):
+        """
+        Get the index of the partition value if it is known, otherwise assign
+        one
+        """
+        if key in self.key_indices:
+            return self.key_indices[key]
+        else:
+            index = len(self.key_indices)
+            self.keys.append(key)
+            self.key_indices[key] = index
+            return index
+
+    @property
+    def dictionary(self):
+        if self._dictionary is not None:
+            return self._dictionary
+
+        if len(self.keys) == 0:
+            raise ValueError('No known partition keys')
+
+        # Only integer and string partition types are supported right now
+        try:
+            integer_keys = [int(x) for x in self.keys]
+            dictionary = _array.from_pylist(integer_keys)
+        except ValueError:
+            dictionary = _array.from_pylist(self.keys)
+
+        self._dictionary = dictionary
+        return dictionary
+
+    @property
+    def is_sorted(self):
+        return list(self.keys) == sorted(self.keys)
+
+
+class ParquetPartitions(object):
+
+    def __init__(self):
+        self.levels = []
+        self.partition_names = set()
+
+    def __len__(self):
+        return len(self.levels)
+
+    def __getitem__(self, i):
+        return self.levels[i]
+
+    def get_index(self, level, name, key):
+        """
+        Record a partition value at a particular level, returning the distinct
+        code for that value at that level. Example:
+
+        partitions.get_index(1, 'foo', 'a') returns 0
+        partitions.get_index(1, 'foo', 'b') returns 1
+        partitions.get_index(1, 'foo', 'c') returns 2
+        partitions.get_index(1, 'foo', 'a') returns 0
+
+        Parameters
+        ----------
+        level : int
+            The nesting level of the partition we are observing
+        name : string
+            The partition name
+        key : string or int
+            The partition value
+        """
+        if level == len(self.levels):
+            if name in self.partition_names:
+                raise ValueError('{0} was the name of the partition in '
+                                 'another level'.format(name))
+
+            part_set = PartitionSet(name)
+            self.levels.append(part_set)
+            self.partition_names.add(name)
+
+        return self.levels[level].get_index(key)
+
+
+def is_string(x):
+    return isinstance(x, six.string_types)
+
+
+class ParquetManifest(object):
+    """
+
+    """
+    def __init__(self, dirpath, filesystem=None, pathsep='/',
+                 partition_scheme='hive'):
+        self.filesystem = filesystem or LocalFilesystem.get_instance()
+        self.pathsep = pathsep
+        self.dirpath = dirpath
+        self.partition_scheme = partition_scheme
+        self.partitions = ParquetPartitions()
+        self.pieces = []
+
+        self.common_metadata_path = None
+        self.metadata_path = None
+
+        self._visit_level(0, self.dirpath, [])
+
+    def _visit_level(self, level, base_path, part_keys):
+        directories = []
+        files = []
+        fs = self.filesystem
+
+        if not fs.isdir(base_path):
+            raise ValueError('"{0}" is not a directory'.format(base_path))
+
+        for path in sorted(fs.ls(base_path)):
+            if fs.isfile(path):
+                if _is_parquet_file(path):
+                    files.append(path)
+                elif path.endswith('_common_metadata'):
+                    self.common_metadata_path = path
+                elif path.endswith('_metadata'):
+                    self.metadata_path = path
+                elif not self._should_silently_exclude(path):
+                    print('Ignoring path: {0}'.format(path))
+            elif fs.isdir(path):
+                directories.append(path)
+
+        if len(files) > 0 and len(directories) > 0:
+            raise ValueError('Found files in an intermediate '
+                             'directory: {0}'.format(base_path))
+        elif len(directories) > 0:
+            self._visit_directories(level, directories, part_keys)
+        else:
+            self._push_pieces(files, part_keys)
+
+    def _should_silently_exclude(self, path):
+        _, tail = path.rsplit(self.pathsep, 1)
+        return tail.endswith('.crc') or tail in EXCLUDED_PARQUET_PATHS
+
+    def _visit_directories(self, level, directories, part_keys):
+        for path in directories:
+            head, tail = _path_split(path, self.pathsep)
+            name, key = _parse_hive_partition(tail)
+
+            index = self.partitions.get_index(level, name, key)
+            dir_part_keys = part_keys + [(name, index)]
+            self._visit_level(level + 1, path, dir_part_keys)
+
+    def _parse_partition(self, dirname):
+        if self.partition_scheme == 'hive':
+            return _parse_hive_partition(dirname)
+        else:
+            raise NotImplementedError('partition schema: {0}'
+                                      .format(self.partition_scheme))
+
+    def _push_pieces(self, files, part_keys):
+        self.pieces.extend([
+            ParquetDatasetPiece(path, partition_keys=part_keys)
+            for path in files
+        ])
+
+
+def _parse_hive_partition(value):
+    if '=' not in value:
+        raise ValueError('Directory name did not appear to be a '
+                         'partition: {0}'.format(value))
+    return value.split('=', 1)
+
+
+def _path_split(path, sep):
+    i = path.rfind(sep) + 1
+    head, tail = path[:i], path[i:]
+    head = head.rstrip(sep)
+    return head, tail
+
+
+EXCLUDED_PARQUET_PATHS = {'_SUCCESS'}
+
+
+class ParquetDataset(object):
+    """
+    Encapsulates details of reading a complete Parquet dataset possibly
+    consisting of multiple files and partitions in subdirectories
+
+    Parameters
+    ----------
+    path_or_paths : str or List[str]
+        A directory name, single file name, or list of file names
+    filesystem : Filesystem, default None
+        If nothing passed, paths assumed to be found in the local on-disk
+        filesystem
+    metadata : pyarrow.parquet.FileMetaData
+        Use metadata obtained elsewhere to validate file schemas
+    schema : pyarrow.parquet.Schema
+        Use schema obtained elsewhere to validate file schemas. Alternative to
+        metadata parameter
+    split_row_groups : boolean, default False
+        Divide files into pieces for each row group in the file
+    validate_schema : boolean, default True
+        Check that individual file schemas are all the same / compatible
+    """
+    def __init__(self, path_or_paths, filesystem=None, schema=None,
+                 metadata=None, split_row_groups=False, validate_schema=True):
+        if filesystem is None:
+            self.fs = LocalFilesystem.get_instance()
+        else:
+            self.fs = filesystem
+
+        self.pieces, self.partitions = _make_manifest(path_or_paths, self.fs)
+
+        self.metadata = metadata
+        self.schema = schema
+
+        self.split_row_groups = split_row_groups
+
+        if split_row_groups:
+            raise NotImplementedError("split_row_groups not yet implemented")
+
+        if validate_schema:
+            self.validate_schemas()
+
+    def validate_schemas(self):
+        open_file = self._get_open_file_func()
+
+        if self.metadata is None and self.schema is None:
+            self.schema = self.pieces[0].get_metadata(open_file).schema
+        elif self.schema is None:
+            self.schema = self.metadata.schema
+
+        # Verify schemas are all equal
+        for piece in self.pieces:
+            file_metadata = piece.get_metadata(open_file)
+            if not self.schema.equals(file_metadata.schema):
+                raise ValueError('Schema in {0!s} was different. '
+                                 '{1!s} vs {2!s}'
+                                 .format(piece, file_metadata.schema,
+                                         self.schema))
+
+    def read(self, columns=None, nthreads=1):
+        """
+        Read multiple Parquet files as a single pyarrow.Table
+
+        Parameters
+        ----------
+        columns : List[str]
+            Names of columns to read from the file
+        nthreads : int, default 1
+            Number of columns to read in parallel. Requires that the underlying
+            file source is threadsafe
+
+        Returns
+        -------
+        pyarrow.Table
+            Content of the file as a table (of columns)
+        """
+        open_file = self._get_open_file_func()
+
+        tables = []
+        for piece in self.pieces:
+            table = piece.read(columns=columns, nthreads=nthreads,
+                               partitions=self.partitions,
+                               open_file_func=open_file)
+            tables.append(table)
+
+        all_data = _table.concat_tables(tables)
+        return all_data
+
+    def _get_open_file_func(self):
+        if self.fs is None or isinstance(self.fs, LocalFilesystem):
+            def open_file(path, meta=None):
+                return ParquetFile(path, metadata=meta)
+        else:
+            def open_file(path, meta=None):
+                return ParquetFile(self.fs.open(path, mode='rb'),
+                                   metadata=meta)
+        return open_file
+
+
+def _make_manifest(path_or_paths, fs, pathsep='/'):
+    partitions = None
+
+    if is_string(path_or_paths) and fs.isdir(path_or_paths):
+        manifest = ParquetManifest(path_or_paths, filesystem=fs,
+                                   pathsep=pathsep)
+        pieces = manifest.pieces
+        partitions = manifest.partitions
+    else:
+        if not isinstance(path_or_paths, list):
+            path_or_paths = [path_or_paths]
+
+        # List of paths
+        if len(path_or_paths) == 0:
+            raise ValueError('Must pass at least one file path')
+
+        pieces = []
+        for path in path_or_paths:
+            if not fs.isfile(path):
+                raise IOError('Passed non-file path: {0}'
+                              .format(path))
+            piece = ParquetDatasetPiece(path)
+            pieces.append(piece)
+
+    return pieces, partitions
+
+
 def read_table(source, columns=None, nthreads=1, metadata=None):
     """
     Read a Table from Parquet format
@@ -127,9 +591,7 @@ def read_table(source, columns=None, nthreads=1, metadata=None):
     pyarrow.Table
         Content of the file as a table (of columns)
     """
-    from pyarrow.filesystem import LocalFilesystem
-
-    if isinstance(source, six.string_types):
+    if is_string(source):
         fs = LocalFilesystem.get_instance()
         if fs.isdir(source):
             return fs.read_parquet(source, columns=columns,
@@ -139,70 +601,7 @@ def read_table(source, columns=None, nthreads=1, metadata=None):
     return pf.read(columns=columns, nthreads=nthreads)
 
 
-def read_multiple_files(paths, columns=None, filesystem=None, nthreads=1,
-                        metadata=None, schema=None):
-    """
-    Read multiple Parquet files as a single pyarrow.Table
-
-    Parameters
-    ----------
-    paths : List[str]
-        List of file paths
-    columns : List[str]
-        Names of columns to read from the file
-    filesystem : Filesystem, default None
-        If nothing passed, paths assumed to be found in the local on-disk
-        filesystem
-    nthreads : int, default 1
-        Number of columns to read in parallel. Requires that the underlying
-        file source is threadsafe
-    metadata : pyarrow.parquet.FileMetaData
-        Use metadata obtained elsewhere to validate file schemas
-    schema : pyarrow.parquet.Schema
-        Use schema obtained elsewhere to validate file schemas. Alternative to
-        metadata parameter
-
-    Returns
-    -------
-    pyarrow.Table
-        Content of the file as a table (of columns)
-    """
-    if filesystem is None:
-        def open_file(path, meta=None):
-            return ParquetFile(path, metadata=meta)
-    else:
-        def open_file(path, meta=None):
-            return ParquetFile(filesystem.open(path, mode='rb'), metadata=meta)
-
-    if len(paths) == 0:
-        raise ValueError('Must pass at least one file path')
-
-    if metadata is None and schema is None:
-        schema = open_file(paths[0]).schema
-    elif schema is None:
-        schema = metadata.schema
-
-    # Verify schemas are all equal
-    all_file_metadata = []
-    for path in paths:
-        file_metadata = open_file(path).metadata
-        if not schema.equals(file_metadata.schema):
-            raise ValueError('Schema in {0} was different. {1!s} vs {2!s}'
-                             .format(path, file_metadata.schema, schema))
-        all_file_metadata.append(file_metadata)
-
-    # Read the tables
-    tables = []
-    for path, path_metadata in zip(paths, all_file_metadata):
-        reader = open_file(path, meta=path_metadata)
-        table = reader.read(columns=columns, nthreads=nthreads)
-        tables.append(table)
-
-    all_data = concat_tables(tables)
-    return all_data
-
-
-def write_table(table, sink, row_group_size=None, version='1.0',
+def write_table(table, where, row_group_size=None, version='1.0',
                 use_dictionary=True, compression='snappy', **kwargs):
     """
     Write a Table to Parquet format
@@ -210,7 +609,7 @@ def write_table(table, sink, row_group_size=None, version='1.0',
     Parameters
     ----------
     table : pyarrow.Table
-    sink: string or pyarrow.io.NativeFile
+    where: string or pyarrow.io.NativeFile
     row_group_size : int, default None
         The maximum number of rows in each Parquet RowGroup. As a default,
         we will write a single RowGroup per file.
@@ -223,7 +622,7 @@ def write_table(table, sink, row_group_size=None, version='1.0',
         Specify the compression codec, either on a general basis or per-column.
     """
     row_group_size = kwargs.get('chunk_size', row_group_size)
-    writer = ParquetWriter(sink, use_dictionary=use_dictionary,
+    writer = ParquetWriter(where, use_dictionary=use_dictionary,
                            compression=compression,
                            version=version)
     writer.write_table(table, row_group_size=row_group_size)
diff --git a/python/pyarrow/table.pxd b/python/pyarrow/table.pxd
index 389727b4dc1d7..f564042b62d7b 100644
--- a/python/pyarrow/table.pxd
+++ b/python/pyarrow/table.pxd
@@ -58,5 +58,6 @@ cdef class RecordBatch:
     cdef init(self, const shared_ptr[CRecordBatch]& table)
     cdef _check_nullptr(self)
 
+cdef object box_column(const shared_ptr[CColumn]& ccolumn)
 cdef api object table_from_ctable(const shared_ptr[CTable]& ctable)
 cdef api object batch_from_cbatch(const shared_ptr[CRecordBatch]& cbatch)
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/table.pyx
index 94389a73cc974..3972bda4ee425 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/table.pyx
@@ -30,8 +30,9 @@ import pyarrow.config
 from pyarrow.array cimport Array, box_array, wrap_array_output
 from pyarrow.error import ArrowException
 from pyarrow.error cimport check_status
-from pyarrow.schema cimport box_data_type, box_schema, DataType
+from pyarrow.schema cimport box_data_type, box_schema, DataType, Field
 
+from pyarrow.schema import field
 from pyarrow.compat import frombytes, tobytes
 
 cimport cpython
@@ -141,6 +142,19 @@ cdef class Column:
         self.sp_column = column
         self.column = column.get()
 
+    @staticmethod
+    def from_array(object field_or_name, Array arr):
+        cdef Field boxed_field
+
+        if isinstance(field_or_name, Field):
+            boxed_field = field_or_name
+        else:
+            boxed_field = field(field_or_name, arr.type)
+
+        cdef shared_ptr[CColumn] sp_column
+        sp_column.reset(new CColumn(boxed_field.sp_field, arr.sp_array))
+        return box_column(sp_column)
+
     def to_pandas(self):
         """
         Convert the arrow::Column to a pandas.Series
@@ -828,6 +842,24 @@ cdef class Table:
         """
         return (self.num_rows, self.num_columns)
 
+    def add_column(self, int i, Column column):
+        """
+        Add column to Table at position. Returns new table
+        """
+        cdef:
+            shared_ptr[CTable] c_table
+
+        with nogil:
+            check_status(self.table.AddColumn(i, column.sp_column, &c_table))
+
+        return table_from_ctable(c_table)
+
+    def append_column(self, Column column):
+        """
+        Append column at end of columns. Returns new table
+        """
+        return self.add_column(self.num_columns, column)
+
     def remove_column(self, int i):
         """
         Create new Table with the indicated column removed
@@ -865,6 +897,12 @@ def concat_tables(tables):
     return table_from_ctable(c_result)
 
 
+cdef object box_column(const shared_ptr[CColumn]& ccolumn):
+    cdef Column column = Column()
+    column.init(ccolumn)
+    return column
+
+
 cdef api object table_from_ctable(const shared_ptr[CTable]& ctable):
     cdef Table table = Table()
     table.init(ctable)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 86165be7052c6..de1b1488c1475 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -16,11 +16,13 @@
 # under the License.
 
 from os.path import join as pjoin
+import datetime
 import io
 import os
 import pytest
 
-from pyarrow.compat import guid
+from pyarrow.compat import guid, u
+from pyarrow.filesystem import LocalFilesystem
 import pyarrow as pa
 import pyarrow.io as paio
 from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
@@ -28,7 +30,7 @@
 import numpy as np
 import pandas as pd
 
-import pandas.util.testing as pdt
+import pandas.util.testing as tm
 
 try:
     import pyarrow.parquet as pq
@@ -93,7 +95,7 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
     pq.write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
-    pdt.assert_frame_equal(df, df_read)
+    tm.assert_frame_equal(df, df_read)
 
 
 @parquet
@@ -125,7 +127,7 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
     # We pass uint32_t as int64_t if we write Parquet version 1.0
     df['uint32'] = df['uint32'].values.astype(np.int64)
 
-    pdt.assert_frame_equal(df, df_read)
+    tm.assert_frame_equal(df, df_read)
 
 
 @parquet
@@ -142,7 +144,7 @@ def test_pandas_column_selection(tmpdir):
     table_read = pq.read_table(filename.strpath, columns=['uint8'])
     df_read = table_read.to_pandas()
 
-    pdt.assert_frame_equal(df[['uint8']], df_read)
+    tm.assert_frame_equal(df[['uint8']], df_read)
 
 
 def _random_integers(size, dtype):
@@ -169,7 +171,7 @@ def _test_dataframe(size=10000, seed=0):
         'float64': np.random.randn(size),
         'float64': np.arange(size, dtype=np.float64),
         'bool': np.random.randn(size) > 0,
-        'strings': [pdt.rands(10) for i in range(size)]
+        'strings': [tm.rands(10) for i in range(size)]
     })
     return df
 
@@ -183,7 +185,7 @@ def test_pandas_parquet_native_file_roundtrip(tmpdir):
     buf = imos.get_result()
     reader = paio.BufferReader(buf)
     df_read = pq.read_table(reader).to_pandas()
-    pdt.assert_frame_equal(df, df_read)
+    tm.assert_frame_equal(df, df_read)
 
 
 @parquet
@@ -207,7 +209,7 @@ def test_pandas_parquet_pyfile_roundtrip(tmpdir):
 
     table_read = pq.read_table(data)
     df_read = table_read.to_pandas()
-    pdt.assert_frame_equal(df, df_read)
+    tm.assert_frame_equal(df, df_read)
 
 
 @parquet
@@ -236,7 +238,7 @@ def test_pandas_parquet_configuration_options(tmpdir):
                        use_dictionary=use_dictionary)
         table_read = pq.read_table(filename.strpath)
         df_read = table_read.to_pandas()
-        pdt.assert_frame_equal(df, df_read)
+        tm.assert_frame_equal(df, df_read)
 
     for compression in ['NONE', 'SNAPPY', 'GZIP']:
         pq.write_table(arrow_table, filename.strpath,
@@ -244,7 +246,7 @@ def test_pandas_parquet_configuration_options(tmpdir):
                        compression=compression)
         table_read = pq.read_table(filename.strpath)
         df_read = table_read.to_pandas()
-        pdt.assert_frame_equal(df, df_read)
+        tm.assert_frame_equal(df, df_read)
 
 
 def make_sample_file(df):
@@ -331,7 +333,7 @@ def test_column_of_arrays(tmpdir):
     pq.write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
-    pdt.assert_frame_equal(df, df_read)
+    tm.assert_frame_equal(df, df_read)
 
 
 @parquet
@@ -344,7 +346,7 @@ def test_column_of_lists(tmpdir):
     pq.write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pq.read_table(filename.strpath)
     df_read = table_read.to_pandas()
-    pdt.assert_frame_equal(df, df_read)
+    tm.assert_frame_equal(df, df_read)
 
 
 @parquet
@@ -399,7 +401,7 @@ def test_pass_separate_metadata():
 
     fileh = pq.ParquetFile(buf, metadata=metadata)
 
-    pdt.assert_frame_equal(df, fileh.read().to_pandas())
+    tm.assert_frame_equal(df, fileh.read().to_pandas())
 
 
 @parquet
@@ -422,13 +424,121 @@ def test_read_single_row_group():
 
     row_groups = [pf.read_row_group(i) for i in range(K)]
     result = pa.concat_tables(row_groups)
-    pdt.assert_frame_equal(df, result.to_pandas())
+    tm.assert_frame_equal(df, result.to_pandas())
 
     cols = df.columns[:2]
     row_groups = [pf.read_row_group(i, columns=cols)
                   for i in range(K)]
     result = pa.concat_tables(row_groups)
-    pdt.assert_frame_equal(df[cols], result.to_pandas())
+    tm.assert_frame_equal(df[cols], result.to_pandas())
+
+
+@parquet
+def test_parquet_piece_basics():
+    path = '/baz.parq'
+
+    piece1 = pq.ParquetDatasetPiece(path)
+    piece2 = pq.ParquetDatasetPiece(path, row_group=1)
+    piece3 = pq.ParquetDatasetPiece(
+        path, row_group=1, partition_keys=[('foo', 0), ('bar', 1)])
+
+    assert str(piece1) == path
+    assert str(piece2) == '/baz.parq | row_group=1'
+    assert str(piece3) == 'partition[foo=0, bar=1] /baz.parq | row_group=1'
+
+    assert piece1 == piece1
+    assert piece2 == piece2
+    assert piece3 == piece3
+    assert piece1 != piece3
+
+
+@parquet
+def test_partition_set_dictionary_type():
+    set1 = pq.PartitionSet('key1', [u('foo'), u('bar'), u('baz')])
+    set2 = pq.PartitionSet('key2', [2007, 2008, 2009])
+
+    assert isinstance(set1.dictionary, pa.StringArray)
+    assert isinstance(set2.dictionary, pa.IntegerArray)
+
+    set3 = pq.PartitionSet('key2', [datetime.datetime(2007, 1, 1)])
+    with pytest.raises(TypeError):
+        set3.dictionary
+
+
+@parquet
+def test_read_partitioned_directory(tmpdir):
+    foo_keys = [0, 1]
+    bar_keys = ['a', 'b', 'c']
+    partition_spec = [
+        ['foo', foo_keys],
+        ['bar', bar_keys]
+    ]
+    N = 30
+
+    df = pd.DataFrame({
+        'index': np.arange(N),
+        'foo': np.array(foo_keys, dtype='i4').repeat(15),
+        'bar': np.tile(np.tile(np.array(bar_keys, dtype=object), 5), 2),
+        'values': np.random.randn(N)
+    }, columns=['index', 'foo', 'bar', 'values'])
+
+    base_path = str(tmpdir)
+    _generate_partition_directories(base_path, partition_spec, df)
+
+    dataset = pq.ParquetDataset(base_path)
+    table = dataset.read()
+    result_df = (table.to_pandas()
+                 .sort_values(by='index')
+                 .reset_index(drop=True))
+
+    expected_df = (df.sort_values(by='index')
+                   .reset_index(drop=True)
+                   .reindex(columns=result_df.columns))
+    expected_df['foo'] = pd.Categorical(df['foo'], categories=foo_keys)
+    expected_df['bar'] = pd.Categorical(df['bar'], categories=bar_keys)
+
+    assert (result_df.columns == ['index', 'values', 'foo', 'bar']).all()
+
+    tm.assert_frame_equal(result_df, expected_df)
+
+
+def _generate_partition_directories(base_dir, partition_spec, df):
+    # partition_spec : list of lists, e.g. [['foo', [0, 1, 2],
+    #                                       ['bar', ['a', 'b', 'c']]
+    # part_table : a pyarrow.Table to write to each partition
+    DEPTH = len(partition_spec)
+    fs = LocalFilesystem.get_instance()
+
+    def _visit_level(base_dir, level, part_keys):
+        name, values = partition_spec[level]
+        for value in values:
+            this_part_keys = part_keys + [(name, value)]
+
+            level_dir = pjoin(base_dir, '{0}={1}'.format(name, value))
+            fs.mkdir(level_dir)
+
+            if level == DEPTH - 1:
+                # Generate example data
+                file_path = pjoin(level_dir, 'data.parq')
+
+                filtered_df = _filter_partition(df, this_part_keys)
+                part_table = pa.Table.from_pandas(filtered_df)
+                pq.write_table(part_table, file_path)
+            else:
+                _visit_level(level_dir, level + 1, this_part_keys)
+
+    _visit_level(base_dir, 0, [])
+
+
+def _filter_partition(df, part_keys):
+    predicate = np.ones(len(df), dtype=bool)
+
+    to_drop = []
+    for name, value in part_keys:
+        to_drop.append(name)
+        predicate &= df[name] == value
+
+    return df[predicate].drop(to_drop, axis=1)
 
 
 @parquet
@@ -459,7 +569,11 @@ def test_read_multiple_files(tmpdir):
     with open(pjoin(dirpath, '_SUCCESS.crc'), 'wb') as f:
         f.write(b'0')
 
-    result = pq.read_multiple_files(paths)
+    def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
+        dataset = pq.ParquetDataset(paths, **kwargs)
+        return dataset.read(columns=columns, nthreads=nthreads)
+
+    result = read_multiple_files(paths)
     expected = pa.concat_tables(test_data)
 
     assert result.equals(expected)
@@ -467,7 +581,7 @@ def test_read_multiple_files(tmpdir):
     # Read with provided metadata
     metadata = pq.ParquetFile(paths[0]).metadata
 
-    result2 = pq.read_multiple_files(paths, metadata=metadata)
+    result2 = read_multiple_files(paths, metadata=metadata)
     assert result2.equals(expected)
 
     result3 = pa.localfs.read_parquet(dirpath, schema=metadata.schema)
@@ -493,15 +607,15 @@ def test_read_multiple_files(tmpdir):
     bad_meta = pq.ParquetFile(bad_apple_path).metadata
 
     with pytest.raises(ValueError):
-        pq.read_multiple_files(paths + [bad_apple_path])
+        read_multiple_files(paths + [bad_apple_path])
 
     with pytest.raises(ValueError):
-        pq.read_multiple_files(paths, metadata=bad_meta)
+        read_multiple_files(paths, metadata=bad_meta)
 
     mixed_paths = [bad_apple_path, paths[0]]
 
     with pytest.raises(ValueError):
-        pq.read_multiple_files(mixed_paths, schema=bad_meta.schema)
+        read_multiple_files(mixed_paths, schema=bad_meta.schema)
 
     with pytest.raises(ValueError):
-        pq.read_multiple_files(mixed_paths)
+        read_multiple_files(mixed_paths)
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 548f4782a7030..79b4c159fd10a 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -39,6 +39,14 @@ def test_basics(self):
         assert column.shape == (5,)
         assert column.to_pylist() == [-10, -5, 0, 5, 10]
 
+    def test_from_array(self):
+        arr = pa.from_pylist([0, 1, 2, 3, 4])
+
+        col1 = pa.Column.from_array('foo', arr)
+        col2 = pa.Column.from_array(pa.field('foo', arr.type), arr)
+
+        assert col1.equals(col2)
+
     def test_pandas(self):
         data = [
             pa.from_pylist([-10, -5, 0, 5, 10])
@@ -169,6 +177,29 @@ def test_table_basics():
             assert chunk is not None
 
 
+def test_table_add_column():
+    data = [
+        pa.from_pylist(range(5)),
+        pa.from_pylist([-10, -5, 0, 5, 10]),
+        pa.from_pylist(range(5, 10))
+    ]
+    table = pa.Table.from_arrays(data, names=('a', 'b', 'c'))
+
+    col = pa.Column.from_array('d', data[1])
+    t2 = table.add_column(3, col)
+    t3 = table.append_column(col)
+
+    expected = pa.Table.from_arrays(data + [data[1]],
+                                    names=('a', 'b', 'c', 'd'))
+    assert t2.equals(expected)
+    assert t3.equals(expected)
+
+    t4 = table.add_column(0, col)
+    expected = pa.Table.from_arrays([data[1]] + data,
+                                    names=('d', 'a', 'b', 'c'))
+    assert t4.equals(expected)
+
+
 def test_table_remove_column():
     data = [
         pa.from_pylist(range(5)),

From 3d9bfc2aeb376c994ca9b257cb9156d08b870455 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 12 Apr 2017 15:51:47 -0400
Subject: [PATCH 0506/1644] ARROW-646: [Python] Conda s3 robustness, set
 CONDA_PKGS_DIR env variable and add Travis CI caching

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #532 from wesm/ARROW-646 and squashes the following commits:

2f27123 [Wes McKinney] Fix env variable name
7ead593 [Wes McKinney] Set CONDA_PKGS_DIR env variable and add to Travis CI cache. Change some other conda settings
---
 .travis.yml                | 6 ++++++
 ci/travis_install_conda.sh | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index f74a3b205c4b6..4a49c717bf75d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,6 +21,12 @@ addons:
     - autoconf-archive
     - libgirepository1.0-dev
 
+cache:
+  ccache: true
+  directories:
+    - $HOME/.conda_packages
+    - $HOME/.ccache
+
 matrix:
   fast_finish: true
   allow_failures:
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index e064317f12303..c036e925427a9 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -23,13 +23,20 @@ fi
 wget -O miniconda.sh $MINICONDA_URL
 
 export MINICONDA=$HOME/miniconda
+export CONDA_PKGS_DIRS=$HOME/.conda_packages
+mkdir -p $CONDA_PKGS_DIRS
 
 bash miniconda.sh -b -p $MINICONDA
 export PATH="$MINICONDA/bin:$PATH"
 conda update -y -q conda
+conda config --set auto_update_conda false
 conda info -a
 
 conda config --set show_channel_urls True
+
+# Help with SSL timeouts to S3
+conda config --set remote_connect_timeout_secs 12
+
 conda config --add channels https://repo.continuum.io/pkgs/free
 conda config --add channels conda-forge
 conda info -a

From e9343650355b1820562bfa85d370cac2070b7c92 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 13 Apr 2017 12:46:58 +0200
Subject: [PATCH 0507/1644] ARROW-797: [Python] Make more explicitly curated
 public API page, sphinx cleanup

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #535 from wesm/ARROW-797 and squashes the following commits:

bc344a8 [Wes McKinney] rat warning
fb1d916 [Wes McKinney] build_sphinx target needs extra options
00c6a03 [Wes McKinney] Remove sphinxext until it's actually needed. Add some ASF license headers
60d6ab6 [Wes McKinney] Update gitignore
2b9f3f9 [Wes McKinney] Add _static stub
80e4a4b [Wes McKinney] Remove unused options
b662b85 [Wes McKinney] Remove unused options
30ebd05 [Wes McKinney] Cleaning, explicit API index
83e31d5 [Wes McKinney] Initial API doc
d7f4ed7 [Wes McKinney] Add NumPy extensions from pandas
---
 ci/travis_script_python.sh                   |   2 +-
 python/cmake_modules/UseCython.cmake         |   5 +-
 python/doc/.gitignore                        |  22 ++-
 python/doc/Makefile                          |   4 +-
 python/doc/source/_static/stub               |  18 +++
 python/doc/source/api.rst                    | 153 +++++++++++++++++++
 python/doc/{ => source}/conf.py              |  22 ++-
 python/doc/{ => source}/filesystems.rst      |   0
 python/doc/{ => source}/getting_involved.rst |   0
 python/doc/{ => source}/index.rst            |   2 +-
 python/doc/{ => source}/install.rst          |   0
 python/doc/{ => source}/jemalloc.rst         |   0
 python/doc/{ => source}/pandas.rst           |   0
 python/doc/{ => source}/parquet.rst          |   0
 14 files changed, 207 insertions(+), 21 deletions(-)
 create mode 100644 python/doc/source/_static/stub
 create mode 100644 python/doc/source/api.rst
 rename python/doc/{ => source}/conf.py (96%)
 rename python/doc/{ => source}/filesystems.rst (100%)
 rename python/doc/{ => source}/getting_involved.rst (100%)
 rename python/doc/{ => source}/index.rst (99%)
 rename python/doc/{ => source}/install.rst (100%)
 rename python/doc/{ => source}/jemalloc.rst (100%)
 rename python/doc/{ => source}/pandas.rst (100%)
 rename python/doc/{ => source}/parquet.rst (100%)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 604cd13916299..680eb01158d8f 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -123,7 +123,7 @@ python_version_tests() {
   if [[ "$PYTHON_VERSION" == "3.6" ]]
   then
       pip install -r doc/requirements.txt
-      python setup.py build_sphinx
+      python setup.py build_sphinx -s doc/source
   fi
 }
 
diff --git a/python/cmake_modules/UseCython.cmake b/python/cmake_modules/UseCython.cmake
index cee6066d31de0..7c06b023db2cb 100644
--- a/python/cmake_modules/UseCython.cmake
+++ b/python/cmake_modules/UseCython.cmake
@@ -64,7 +64,7 @@ set( CYTHON_NO_DOCSTRINGS OFF
   CACHE BOOL "Strip docstrings from the compiled module." )
 set( CYTHON_FLAGS "" CACHE STRING
   "Extra flags to the cython compiler." )
-mark_as_advanced( CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS )
+mark_as_advanced( CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS)
 
 find_package( Cython REQUIRED )
 find_package( PythonLibsNew REQUIRED )
@@ -131,7 +131,8 @@ function( compile_pyx _name pyx_target_name generated_files pyx_file)
   # Add the command to run the compiler.
   add_custom_target(${pyx_target_name}
     COMMAND ${CYTHON_EXECUTABLE} ${cxx_arg} ${include_directory_arg}
-    ${annotate_arg} ${no_docstrings_arg} ${cython_debug_arg} ${CYTHON_FLAGS}
+    ${annotate_arg} ${no_docstrings_arg} ${cython_debug_arg}
+    ${CYTHON_FLAGS}
     --output-file "${_name}.${extension}" ${pyx_location}
     DEPENDS ${pyx_location}
     # do not specify byproducts for now since they don't work with the older
diff --git a/python/doc/.gitignore b/python/doc/.gitignore
index 87d04134d6fc3..3bee39fa36fe4 100644
--- a/python/doc/.gitignore
+++ b/python/doc/.gitignore
@@ -1,3 +1,19 @@
-# auto-generated module documentation
-pyarrow*.rst
-modules.rst
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+_build
+source/generated
\ No newline at end of file
diff --git a/python/doc/Makefile b/python/doc/Makefile
index 7257583952481..65d6a4df3b20f 100644
--- a/python/doc/Makefile
+++ b/python/doc/Makefile
@@ -22,9 +22,9 @@ BUILDDIR      = _build
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 # the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
 .PHONY: help
 help:
diff --git a/python/doc/source/_static/stub b/python/doc/source/_static/stub
new file mode 100644
index 0000000000000..765c78f7bc0d2
--- /dev/null
+++ b/python/doc/source/_static/stub
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
\ No newline at end of file
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
new file mode 100644
index 0000000000000..514dcf966f8cc
--- /dev/null
+++ b/python/doc/source/api.rst
@@ -0,0 +1,153 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+.. _api:
+
+*************
+API Reference
+*************
+
+.. _api.functions:
+
+Type Metadata and Schemas
+-------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   null
+   bool_
+   int8
+   int16
+   int32
+   int64
+   uint8
+   uint16
+   uint32
+   uint64
+   float16
+   float32
+   float64
+   timestamp
+   date32
+   date64
+   binary
+   string
+   decimal
+   list_
+   struct
+   dictionary
+   field
+   DataType
+   Field
+   Schema
+   schema
+
+Scalar Value Types
+------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   NA
+   NAType
+   Scalar
+   ArrayValue
+   Int8Value
+   Int16Value
+   Int32Value
+   Int64Value
+   UInt8Value
+   UInt16Value
+   UInt32Value
+   UInt64Value
+   FloatValue
+   DoubleValue
+   ListValue
+   BinaryValue
+   StringValue
+   FixedSizeBinaryValue
+
+Array Types
+-----------
+
+.. autosummary::
+   :toctree: generated/
+
+   Array
+   NumericArray
+   IntegerArray
+   FloatingPointArray
+   BooleanArray
+   Int8Array
+   Int16Array
+   Int32Array
+   Int64Array
+   UInt8Array
+   UInt16Array
+   UInt32Array
+   UInt64Array
+   DictionaryArray
+   StringArray
+
+Tables and Record Batches
+-------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   Column
+   RecordBatch
+   Table
+
+Tensor type and Functions
+-------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   Tensor
+   write_tensor
+   get_tensor_size
+   read_tensor
+
+Input / Output and Shared Memory
+--------------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   Buffer
+   BufferReader
+   InMemoryOutputStream
+   NativeFile
+   MemoryMappedFile
+   memory_map
+   create_memory_map
+   PythonFileInterface
+
+Interprocess Communication and Messaging
+----------------------------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   FileReader
+   FileWriter
+   StreamReader
+   StreamWriter
diff --git a/python/doc/conf.py b/python/doc/source/conf.py
similarity index 96%
rename from python/doc/conf.py
rename to python/doc/source/conf.py
index e817bbdd42bd3..a9262bf7db3dd 100644
--- a/python/doc/conf.py
+++ b/python/doc/source/conf.py
@@ -29,19 +29,8 @@
 import os
 import sys
 
-from sphinx import apidoc
-
 import sphinx_rtd_theme
 
-
-__location__ = os.path.join(os.getcwd(), os.path.dirname(
-        inspect.getfile(inspect.currentframe())))
-output_dir = os.path.join(__location__)
-module_dir = os.path.join(__location__, "..", "pyarrow")
-cmd_line_template = "sphinx-apidoc -f -e -o {outputdir} {moduledir}"
-cmd_line = cmd_line_template.format(outputdir=output_dir, moduledir=module_dir)
-apidoc.main(cmd_line.split(" "))
-
 on_rtd = os.environ.get('READTHEDOCS') == 'True'
 
 if not on_rtd:
@@ -49,6 +38,12 @@
     # build pyarrow there.
     sys.path.insert(0, os.path.abspath('..'))
 
+sys.path.extend([
+    os.path.join(os.path.dirname(__file__),
+                 '..', '../..')
+
+])
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -64,7 +59,7 @@
     'sphinx.ext.doctest',
     'sphinx.ext.mathjax',
     'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon'
+    'sphinx.ext.napoleon',
 ]
 
 # numpydoc configuration
@@ -79,6 +74,9 @@
 # source_suffix = ['.rst', '.md']
 source_suffix = '.rst'
 
+import glob
+autosummary_generate = glob.glob("*.rst")
+
 # The encoding of source files.
 #
 # source_encoding = 'utf-8-sig'
diff --git a/python/doc/filesystems.rst b/python/doc/source/filesystems.rst
similarity index 100%
rename from python/doc/filesystems.rst
rename to python/doc/source/filesystems.rst
diff --git a/python/doc/getting_involved.rst b/python/doc/source/getting_involved.rst
similarity index 100%
rename from python/doc/getting_involved.rst
rename to python/doc/source/getting_involved.rst
diff --git a/python/doc/index.rst b/python/doc/source/index.rst
similarity index 99%
rename from python/doc/index.rst
rename to python/doc/source/index.rst
index 608fff5d57ba4..ecb8e8f4830f3 100644
--- a/python/doc/index.rst
+++ b/python/doc/source/index.rst
@@ -38,7 +38,7 @@ structures.
    pandas
    filesystems
    parquet
-   modules
+   api
    getting_involved
 
 .. toctree::
diff --git a/python/doc/install.rst b/python/doc/source/install.rst
similarity index 100%
rename from python/doc/install.rst
rename to python/doc/source/install.rst
diff --git a/python/doc/jemalloc.rst b/python/doc/source/jemalloc.rst
similarity index 100%
rename from python/doc/jemalloc.rst
rename to python/doc/source/jemalloc.rst
diff --git a/python/doc/pandas.rst b/python/doc/source/pandas.rst
similarity index 100%
rename from python/doc/pandas.rst
rename to python/doc/source/pandas.rst
diff --git a/python/doc/parquet.rst b/python/doc/source/parquet.rst
similarity index 100%
rename from python/doc/parquet.rst
rename to python/doc/source/parquet.rst

From 8b64a4fb2d3973813e2094e108021606034d27f4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 13 Apr 2017 12:51:47 +0200
Subject: [PATCH 0508/1644] ARROW-751: [Python] Make all Cython modules
 private. Some code tidying

I also combined schema/array/scalar, as they are all interrelated.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #533 from wesm/ARROW-751 and squashes the following commits:

63b479b [Wes McKinney] jemalloc is now private
0f46116 [Wes McKinney] Fix APIs in Parquet
1074e7c [Wes McKinney] Make all Cython modules private. Code cleaning
---
 ci/travis_script_python.sh                    |    2 +-
 python/CMakeLists.txt                         |   16 +-
 python/pyarrow/__init__.py                    |   84 +-
 python/pyarrow/{array.pxd => _array.pxd}      |  116 +-
 python/pyarrow/_array.pyx                     | 1368 +++++++++++++++++
 python/pyarrow/{config.pyx => _config.pyx}    |    0
 python/pyarrow/{error.pxd => _error.pxd}      |    0
 python/pyarrow/{error.pyx => _error.pyx}      |    0
 python/pyarrow/{io.pxd => _io.pxd}            |    0
 python/pyarrow/{io.pyx => _io.pyx}            |   17 +-
 .../pyarrow/{jemalloc.pyx => _jemalloc.pyx}   |    2 +-
 python/pyarrow/{memory.pxd => _memory.pxd}    |    0
 python/pyarrow/{memory.pyx => _memory.pyx}    |    0
 python/pyarrow/_parquet.pyx                   |   16 +-
 python/pyarrow/{table.pxd => _table.pxd}      |    3 +-
 python/pyarrow/{table.pyx => _table.pyx}      |   18 +-
 python/pyarrow/array.pyx                      |  646 --------
 python/pyarrow/feather.py                     |    6 +-
 python/pyarrow/filesystem.py                  |    2 +-
 python/pyarrow/formatting.py                  |    4 +-
 python/pyarrow/includes/libarrow.pxd          |    5 +-
 python/pyarrow/ipc.py                         |   10 +-
 python/pyarrow/parquet.py                     |    4 +-
 python/pyarrow/scalar.pxd                     |   72 -
 python/pyarrow/scalar.pyx                     |  315 ----
 python/pyarrow/schema.pxd                     |   76 -
 python/pyarrow/schema.pyx                     |  477 ------
 python/pyarrow/tests/test_feather.py          |    2 +-
 python/pyarrow/tests/test_hdfs.py             |    8 +-
 python/pyarrow/tests/test_io.py               |   31 +-
 python/pyarrow/tests/test_parquet.py          |    5 +-
 python/pyarrow/tests/test_schema.py           |    8 +-
 python/setup.py                               |   18 +-
 33 files changed, 1591 insertions(+), 1740 deletions(-)
 rename python/pyarrow/{array.pxd => _array.pxd} (54%)
 create mode 100644 python/pyarrow/_array.pyx
 rename python/pyarrow/{config.pyx => _config.pyx} (100%)
 rename python/pyarrow/{error.pxd => _error.pxd} (100%)
 rename python/pyarrow/{error.pyx => _error.pyx} (100%)
 rename python/pyarrow/{io.pxd => _io.pxd} (100%)
 rename python/pyarrow/{io.pyx => _io.pyx} (99%)
 rename python/pyarrow/{jemalloc.pyx => _jemalloc.pyx} (96%)
 rename python/pyarrow/{memory.pxd => _memory.pxd} (100%)
 rename python/pyarrow/{memory.pyx => _memory.pyx} (100%)
 rename python/pyarrow/{table.pxd => _table.pxd} (98%)
 rename python/pyarrow/{table.pyx => _table.pyx} (98%)
 delete mode 100644 python/pyarrow/array.pyx
 delete mode 100644 python/pyarrow/scalar.pxd
 delete mode 100644 python/pyarrow/scalar.pyx
 delete mode 100644 python/pyarrow/schema.pxd
 delete mode 100644 python/pyarrow/schema.pyx

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 680eb01158d8f..549fe1141cfb1 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -115,7 +115,7 @@ python_version_tests() {
   python setup.py build_ext --inplace --with-parquet --with-jemalloc
 
   python -c "import pyarrow.parquet"
-  python -c "import pyarrow.jemalloc"
+  python -c "import pyarrow._jemalloc"
 
   python -m pytest -vv -r sxX pyarrow
 
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 3e86521757342..36052bc257232 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -261,14 +261,12 @@ if (UNIX)
 endif()
 
 set(CYTHON_EXTENSIONS
-  array
-  config
-  error
-  io
-  memory
-  scalar
-  schema
-  table
+  _array
+  _config
+  _error
+  _io
+  _memory
+  _table
 )
 
 set(LINK_LIBS
@@ -313,7 +311,7 @@ if (PYARROW_BUILD_JEMALLOC)
     arrow_jemalloc_shared)
   set(CYTHON_EXTENSIONS
     ${CYTHON_EXTENSIONS}
-    jemalloc)
+    _jemalloc)
 endif()
 
 ############################################################
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index df615b428c1c1..66bde4933ee2d 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -25,49 +25,10 @@
    pass
 
 
-import pyarrow.config
-from pyarrow.config import cpu_count, set_cpu_count
+import pyarrow._config
+from pyarrow._config import cpu_count, set_cpu_count
 
-from pyarrow.array import (Array, Tensor, from_pylist,
-                           NumericArray, IntegerArray, FloatingPointArray,
-                           BooleanArray,
-                           Int8Array, UInt8Array,
-                           Int16Array, UInt16Array,
-                           Int32Array, UInt32Array,
-                           Int64Array, UInt64Array,
-                           ListArray, StringArray,
-                           DictionaryArray)
-
-from pyarrow.error import (ArrowException,
-                           ArrowKeyError,
-                           ArrowInvalid,
-                           ArrowIOError,
-                           ArrowMemoryError,
-                           ArrowNotImplementedError,
-                           ArrowTypeError)
-
-from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
-from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface,
-                        Buffer, BufferReader, InMemoryOutputStream,
-                        MemoryMappedFile, memory_map,
-                        frombuffer, read_tensor, write_tensor,
-                        memory_map, create_memory_map,
-                        get_record_batch_size, get_tensor_size)
-
-from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
-
-from pyarrow.memory import MemoryPool, total_allocated_bytes
-
-from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType,
-                            BooleanValue,
-                            Int8Value, Int16Value, Int32Value, Int64Value,
-                            UInt8Value, UInt16Value, UInt32Value, UInt64Value,
-                            FloatValue, DoubleValue, ListValue,
-                            BinaryValue, StringValue, FixedSizeBinaryValue)
-
-import pyarrow.schema as _schema
-
-from pyarrow.schema import (null, bool_,
+from pyarrow._array import (null, bool_,
                             int8, int16, int32, int64,
                             uint8, uint16, uint32, uint64,
                             timestamp, date32, date64,
@@ -75,10 +36,45 @@
                             binary, string, decimal,
                             list_, struct, dictionary, field,
                             DataType, FixedSizeBinaryType,
-                            Field, Schema, schema)
+                            Field, Schema, schema,
+                            Array, Tensor,
+                            from_pylist,
+                            from_numpy_dtype,
+                            NumericArray, IntegerArray, FloatingPointArray,
+                            BooleanArray,
+                            Int8Array, UInt8Array,
+                            Int16Array, UInt16Array,
+                            Int32Array, UInt32Array,
+                            Int64Array, UInt64Array,
+                            ListArray, StringArray,
+                            DictionaryArray,
+                            ArrayValue, Scalar, NA, NAType,
+                            BooleanValue,
+                            Int8Value, Int16Value, Int32Value, Int64Value,
+                            UInt8Value, UInt16Value, UInt32Value, UInt64Value,
+                            FloatValue, DoubleValue, ListValue,
+                            BinaryValue, StringValue, FixedSizeBinaryValue)
 
+from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
+                         Buffer, BufferReader, InMemoryOutputStream,
+                         OSFile, MemoryMappedFile, memory_map,
+                         frombuffer, read_tensor, write_tensor,
+                         memory_map, create_memory_map,
+                         get_record_batch_size, get_tensor_size)
+
+from pyarrow._memory import MemoryPool, total_allocated_bytes
+from pyarrow._table import Column, RecordBatch, Table, concat_tables
+from pyarrow._error import (ArrowException,
+                            ArrowKeyError,
+                            ArrowInvalid,
+                            ArrowIOError,
+                            ArrowMemoryError,
+                            ArrowNotImplementedError,
+                            ArrowTypeError)
 
-from pyarrow.table import Column, RecordBatch, Table, concat_tables
+from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
+
+from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
 
 
 localfs = LocalFilesystem.get_instance()
diff --git a/python/pyarrow/array.pxd b/python/pyarrow/_array.pxd
similarity index 54%
rename from python/pyarrow/array.pxd
rename to python/pyarrow/_array.pxd
index 3ba48718265db..40413746fc94b 100644
--- a/python/pyarrow/array.pxd
+++ b/python/pyarrow/_array.pxd
@@ -15,20 +15,109 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.includes.common cimport shared_ptr, int64_t
-from pyarrow.includes.libarrow cimport CArray, CTensor
-
-from pyarrow.scalar import NA
-
-from pyarrow.schema cimport DataType
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport *
 
 from cpython cimport PyObject
 
-
 cdef extern from "Python.h":
     int PySlice_Check(object)
 
 
+cdef class DataType:
+    cdef:
+        shared_ptr[CDataType] sp_type
+        CDataType* type
+
+    cdef void init(self, const shared_ptr[CDataType]& type)
+
+
+cdef class DictionaryType(DataType):
+    cdef:
+        const CDictionaryType* dict_type
+
+
+cdef class TimestampType(DataType):
+    cdef:
+        const CTimestampType* ts_type
+
+
+cdef class FixedSizeBinaryType(DataType):
+    cdef:
+        const CFixedSizeBinaryType* fixed_size_binary_type
+
+
+cdef class DecimalType(FixedSizeBinaryType):
+    cdef:
+        const CDecimalType* decimal_type
+
+
+cdef class Field:
+    cdef:
+        shared_ptr[CField] sp_field
+        CField* field
+
+    cdef readonly:
+        DataType type
+
+    cdef init(self, const shared_ptr[CField]& field)
+
+
+cdef class Schema:
+    cdef:
+        shared_ptr[CSchema] sp_schema
+        CSchema* schema
+
+    cdef init(self, const vector[shared_ptr[CField]]& fields)
+    cdef init_schema(self, const shared_ptr[CSchema]& schema)
+
+
+cdef class Scalar:
+    cdef readonly:
+        DataType type
+
+
+cdef class NAType(Scalar):
+    pass
+
+
+cdef class ArrayValue(Scalar):
+    cdef:
+        shared_ptr[CArray] sp_array
+        int64_t index
+
+    cdef void init(self, DataType type,
+                   const shared_ptr[CArray]& sp_array, int64_t index)
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array)
+
+
+cdef class Int8Value(ArrayValue):
+    pass
+
+
+cdef class Int64Value(ArrayValue):
+    pass
+
+
+cdef class ListValue(ArrayValue):
+    cdef readonly:
+        DataType value_type
+
+    cdef:
+        CListArray* ap
+
+    cdef getitem(self, int64_t i)
+
+
+cdef class StringValue(ArrayValue):
+    pass
+
+
+cdef class FixedSizeBinaryValue(ArrayValue):
+    pass
+
+
 cdef class Array:
     cdef:
         shared_ptr[CArray] sp_array
@@ -52,10 +141,6 @@ cdef class Tensor:
     cdef init(self, const shared_ptr[CTensor]& sp_tensor)
 
 
-cdef object box_array(const shared_ptr[CArray]& sp_array)
-cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
-
-
 cdef class BooleanArray(Array):
     pass
 
@@ -137,5 +222,12 @@ cdef class DictionaryArray(Array):
         object _indices, _dictionary
 
 
-
 cdef wrap_array_output(PyObject* output)
+cdef DataType box_data_type(const shared_ptr[CDataType]& type)
+cdef Field box_field(const shared_ptr[CField]& field)
+cdef Schema box_schema(const shared_ptr[CSchema]& schema)
+cdef object box_array(const shared_ptr[CArray]& sp_array)
+cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
+cdef object box_scalar(DataType type,
+                       const shared_ptr[CArray]& sp_array,
+                       int64_t index)
diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/_array.pyx
new file mode 100644
index 0000000000000..7ef8e5867a1a2
--- /dev/null
+++ b/python/pyarrow/_array.pyx
@@ -0,0 +1,1368 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from cython.operator cimport dereference as deref
+from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.common cimport PyObject_to_object
+cimport pyarrow.includes.pyarrow as pyarrow
+from pyarrow._error cimport check_status
+from pyarrow._memory cimport MemoryPool, maybe_unbox_memory_pool
+cimport cpython as cp
+
+
+import datetime
+import decimal as _pydecimal
+import numpy as np
+import six
+import pyarrow._config
+from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
+
+
+cdef _pandas():
+    import pandas as pd
+    return pd
+
+
+cdef class DataType:
+
+    def __cinit__(self):
+        pass
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        self.sp_type = type
+        self.type = type.get()
+
+    def __str__(self):
+        return frombytes(self.type.ToString())
+
+    def __repr__(self):
+        return '{0.__class__.__name__}({0})'.format(self)
+
+    def __richcmp__(DataType self, DataType other, int op):
+        if op == cp.Py_EQ:
+            return self.type.Equals(deref(other.type))
+        elif op == cp.Py_NE:
+            return not self.type.Equals(deref(other.type))
+        else:
+            raise TypeError('Invalid comparison')
+
+
+cdef class DictionaryType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.dict_type = <const CDictionaryType*> type.get()
+
+
+cdef class TimestampType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.ts_type = <const CTimestampType*> type.get()
+
+    property unit:
+
+        def __get__(self):
+            return timeunit_to_string(self.ts_type.unit())
+
+    property tz:
+
+        def __get__(self):
+            if self.ts_type.timezone().size() > 0:
+                return frombytes(self.ts_type.timezone())
+            else:
+                return None
+
+
+cdef class FixedSizeBinaryType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.fixed_size_binary_type = (
+            <const CFixedSizeBinaryType*> type.get())
+
+    property byte_width:
+
+        def __get__(self):
+            return self.fixed_size_binary_type.byte_width()
+
+
+cdef class DecimalType(FixedSizeBinaryType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.decimal_type = <const CDecimalType*> type.get()
+
+
+cdef class Field:
+
+    def __cinit__(self):
+        pass
+
+    cdef init(self, const shared_ptr[CField]& field):
+        self.sp_field = field
+        self.field = field.get()
+        self.type = box_data_type(field.get().type())
+
+    @classmethod
+    def from_py(cls, object name, DataType type, bint nullable=True):
+        cdef Field result = Field()
+        result.type = type
+        result.sp_field.reset(new CField(tobytes(name), type.sp_type,
+                                         nullable))
+        result.field = result.sp_field.get()
+
+        return result
+
+    def __repr__(self):
+        return 'Field({0!r}, type={1})'.format(self.name, str(self.type))
+
+    property nullable:
+
+        def __get__(self):
+            return self.field.nullable()
+
+    property name:
+
+        def __get__(self):
+            if box_field(self.sp_field) is None:
+                raise ReferenceError(
+                    'Field not initialized (references NULL pointer)')
+            return frombytes(self.field.name())
+
+
+cdef class Schema:
+
+    def __cinit__(self):
+        pass
+
+    def __len__(self):
+        return self.schema.num_fields()
+
+    def __getitem__(self, i):
+        if i < 0 or i >= len(self):
+            raise IndexError("{0} is out of bounds".format(i))
+
+        cdef Field result = Field()
+        result.init(self.schema.field(i))
+        result.type = box_data_type(result.field.type())
+
+        return result
+
+    cdef init(self, const vector[shared_ptr[CField]]& fields):
+        self.schema = new CSchema(fields)
+        self.sp_schema.reset(self.schema)
+
+    cdef init_schema(self, const shared_ptr[CSchema]& schema):
+        self.schema = schema.get()
+        self.sp_schema = schema
+
+    def equals(self, other):
+        """
+        Test if this schema is equal to the other
+        """
+        cdef Schema _other
+        _other = other
+
+        return self.sp_schema.get().Equals(deref(_other.schema))
+
+    def field_by_name(self, name):
+        """
+        Access a field by its name rather than the column index.
+
+        Parameters
+        ----------
+        name: str
+
+        Returns
+        -------
+        field: pyarrow.Field
+        """
+        return box_field(self.schema.GetFieldByName(tobytes(name)))
+
+    @classmethod
+    def from_fields(cls, fields):
+        cdef:
+            Schema result
+            Field field
+            vector[shared_ptr[CField]] c_fields
+
+        c_fields.resize(len(fields))
+
+        for i in range(len(fields)):
+            field = fields[i]
+            c_fields[i] = field.sp_field
+
+        result = Schema()
+        result.init(c_fields)
+
+        return result
+
+    def __str__(self):
+        return frombytes(self.schema.ToString())
+
+    def __repr__(self):
+        return self.__str__()
+
+
+cdef dict _type_cache = {}
+
+
+cdef DataType primitive_type(Type type):
+    if type in _type_cache:
+        return _type_cache[type]
+
+    cdef DataType out = DataType()
+    out.init(pyarrow.GetPrimitiveType(type))
+
+    _type_cache[type] = out
+    return out
+
+#------------------------------------------------------------
+# Type factory functions
+
+def field(name, type, bint nullable=True):
+    return Field.from_py(name, type, nullable)
+
+
+cdef set PRIMITIVE_TYPES = set([
+    Type_NA, Type_BOOL,
+    Type_UINT8, Type_INT8,
+    Type_UINT16, Type_INT16,
+    Type_UINT32, Type_INT32,
+    Type_UINT64, Type_INT64,
+    Type_TIMESTAMP, Type_DATE32,
+    Type_DATE64,
+    Type_HALF_FLOAT,
+    Type_FLOAT,
+    Type_DOUBLE])
+
+
+def null():
+    return primitive_type(Type_NA)
+
+
+def bool_():
+    return primitive_type(Type_BOOL)
+
+
+def uint8():
+    return primitive_type(Type_UINT8)
+
+
+def int8():
+    return primitive_type(Type_INT8)
+
+
+def uint16():
+    return primitive_type(Type_UINT16)
+
+
+def int16():
+    return primitive_type(Type_INT16)
+
+
+def uint32():
+    return primitive_type(Type_UINT32)
+
+
+def int32():
+    return primitive_type(Type_INT32)
+
+
+def uint64():
+    return primitive_type(Type_UINT64)
+
+
+def int64():
+    return primitive_type(Type_INT64)
+
+
+cdef dict _timestamp_type_cache = {}
+
+
+cdef timeunit_to_string(TimeUnit unit):
+    if unit == TimeUnit_SECOND:
+        return 's'
+    elif unit == TimeUnit_MILLI:
+        return 'ms'
+    elif unit == TimeUnit_MICRO:
+        return 'us'
+    elif unit == TimeUnit_NANO:
+        return 'ns'
+
+
+def timestamp(unit_str, tz=None):
+    cdef:
+        TimeUnit unit
+        c_string c_timezone
+
+    if unit_str == "s":
+        unit = TimeUnit_SECOND
+    elif unit_str == 'ms':
+        unit = TimeUnit_MILLI
+    elif unit_str == 'us':
+        unit = TimeUnit_MICRO
+    elif unit_str == 'ns':
+        unit = TimeUnit_NANO
+    else:
+        raise TypeError('Invalid TimeUnit string')
+
+    cdef TimestampType out = TimestampType()
+
+    if tz is None:
+        out.init(ctimestamp(unit))
+        if unit in _timestamp_type_cache:
+            return _timestamp_type_cache[unit]
+        _timestamp_type_cache[unit] = out
+    else:
+        if not isinstance(tz, six.string_types):
+            tz = tz.zone
+
+        c_timezone = tobytes(tz)
+        out.init(ctimestamp(unit, c_timezone))
+
+    return out
+
+
+def date32():
+    return primitive_type(Type_DATE32)
+
+
+def date64():
+    return primitive_type(Type_DATE64)
+
+
+def float16():
+    return primitive_type(Type_HALF_FLOAT)
+
+
+def float32():
+    return primitive_type(Type_FLOAT)
+
+
+def float64():
+    return primitive_type(Type_DOUBLE)
+
+
+cpdef DataType decimal(int precision, int scale=0):
+    cdef shared_ptr[CDataType] decimal_type
+    decimal_type.reset(new CDecimalType(precision, scale))
+    return box_data_type(decimal_type)
+
+
+def string():
+    """
+    UTF8 string
+    """
+    return primitive_type(Type_STRING)
+
+
+def binary(int length=-1):
+    """Binary (PyBytes-like) type
+
+    Parameters
+    ----------
+    length : int, optional, default -1
+        If length == -1 then return a variable length binary type. If length is
+        greater than or equal to 0 then return a fixed size binary type of
+        width `length`.
+    """
+    if length == -1:
+        return primitive_type(Type_BINARY)
+
+    cdef shared_ptr[CDataType] fixed_size_binary_type
+    fixed_size_binary_type.reset(new CFixedSizeBinaryType(length))
+    return box_data_type(fixed_size_binary_type)
+
+
+def list_(DataType value_type):
+    cdef DataType out = DataType()
+    cdef shared_ptr[CDataType] list_type
+    list_type.reset(new CListType(value_type.sp_type))
+    out.init(list_type)
+    return out
+
+
+def dictionary(DataType index_type, Array dictionary):
+    """
+    Dictionary (categorical, or simply encoded) type
+    """
+    cdef DictionaryType out = DictionaryType()
+    cdef shared_ptr[CDataType] dict_type
+    dict_type.reset(new CDictionaryType(index_type.sp_type,
+                                        dictionary.sp_array))
+    out.init(dict_type)
+    return out
+
+
+def struct(fields):
+    """
+
+    """
+    cdef:
+        DataType out = DataType()
+        Field field
+        vector[shared_ptr[CField]] c_fields
+        cdef shared_ptr[CDataType] struct_type
+
+    for field in fields:
+        c_fields.push_back(field.sp_field)
+
+    struct_type.reset(new CStructType(c_fields))
+    out.init(struct_type)
+    return out
+
+
+def schema(fields):
+    return Schema.from_fields(fields)
+
+
+cdef DataType box_data_type(const shared_ptr[CDataType]& type):
+    cdef:
+        DataType out
+
+    if type.get() == NULL:
+        return None
+
+    if type.get().id() == Type_DICTIONARY:
+        out = DictionaryType()
+    elif type.get().id() == Type_TIMESTAMP:
+        out = TimestampType()
+    elif type.get().id() == Type_FIXED_SIZE_BINARY:
+        out = FixedSizeBinaryType()
+    elif type.get().id() == Type_DECIMAL:
+        out = DecimalType()
+    else:
+        out = DataType()
+
+    out.init(type)
+    return out
+
+cdef Field box_field(const shared_ptr[CField]& field):
+    if field.get() == NULL:
+        return None
+    cdef Field out = Field()
+    out.init(field)
+    return out
+
+cdef Schema box_schema(const shared_ptr[CSchema]& type):
+    cdef Schema out = Schema()
+    out.init_schema(type)
+    return out
+
+
+def from_numpy_dtype(object dtype):
+    cdef shared_ptr[CDataType] c_type
+    with nogil:
+        check_status(pyarrow.NumPyDtypeToArrow(dtype, &c_type))
+
+    return box_data_type(c_type)
+
+
+NA = None
+
+
+cdef class NAType(Scalar):
+
+    def __cinit__(self):
+        global NA
+        if NA is not None:
+            raise Exception('Cannot create multiple NAType instances')
+
+        self.type = null()
+
+    def __repr__(self):
+        return 'NA'
+
+    def as_py(self):
+        return None
+
+
+NA = NAType()
+
+
+cdef class ArrayValue(Scalar):
+
+    cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
+                   int64_t index):
+        self.type = type
+        self.index = index
+        self._set_array(sp_array)
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+
+    def __repr__(self):
+        if hasattr(self, 'as_py'):
+            return repr(self.as_py())
+        else:
+            return super(Scalar, self).__repr__()
+
+
+cdef class BooleanValue(ArrayValue):
+
+    def as_py(self):
+        cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int8Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt8Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int16Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt16Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int32Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt32Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Date32Value(ArrayValue):
+
+    def as_py(self):
+        cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+
+        # Shift to seconds since epoch
+        return datetime.datetime.utcfromtimestamp(
+            int(ap.Value(self.index)) * 86400).date()
+
+
+cdef class Date64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
+        return datetime.datetime.utcfromtimestamp(
+            ap.Value(self.index) / 1000).date()
+
+
+cdef class TimestampValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
+            CTimestampType* dtype = <CTimestampType*>ap.type().get()
+            int64_t val = ap.Value(self.index)
+
+        timezone = None
+        tzinfo = None
+        if dtype.timezone().size() > 0:
+            timezone = frombytes(dtype.timezone())
+            import pytz
+            tzinfo = pytz.timezone(timezone)
+
+        try:
+            pd = _pandas()
+            if dtype.unit() == TimeUnit_SECOND:
+                val = val * 1000000000
+            elif dtype.unit() == TimeUnit_MILLI:
+                val = val * 1000000
+            elif dtype.unit() == TimeUnit_MICRO:
+                val = val * 1000
+            return pd.Timestamp(val, tz=tzinfo)
+        except ImportError:
+            if dtype.unit() == TimeUnit_SECOND:
+                result = datetime.datetime.utcfromtimestamp(val)
+            elif dtype.unit() == TimeUnit_MILLI:
+                result = datetime.datetime.utcfromtimestamp(float(val) / 1000)
+            elif dtype.unit() == TimeUnit_MICRO:
+                result = datetime.datetime.utcfromtimestamp(
+                    float(val) / 1000000)
+            else:
+                # TimeUnit_NANO
+                raise NotImplementedError("Cannot convert nanosecond "
+                                          "timestamps without pandas")
+            if timezone is not None:
+                result = result.replace(tzinfo=tzinfo)
+            return result
+
+
+cdef class FloatValue(ArrayValue):
+
+    def as_py(self):
+        cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class DoubleValue(ArrayValue):
+
+    def as_py(self):
+        cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class DecimalValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CDecimalArray* ap = <CDecimalArray*> self.sp_array.get()
+            c_string s = ap.FormatValue(self.index)
+        return _pydecimal.Decimal(s.decode('utf8'))
+
+
+cdef class StringValue(ArrayValue):
+
+    def as_py(self):
+        cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
+        return ap.GetString(self.index).decode('utf-8')
+
+
+cdef class BinaryValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            const uint8_t* ptr
+            int32_t length
+            CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+
+        ptr = ap.GetValue(self.index, &length)
+        return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+
+
+cdef class ListValue(ArrayValue):
+
+    def __len__(self):
+        return self.ap.value_length(self.index)
+
+    def __getitem__(self, i):
+        return self.getitem(i)
+
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self.getitem(i)
+        raise StopIteration
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+        self.ap = <CListArray*> sp_array.get()
+        self.value_type = box_data_type(self.ap.value_type())
+
+    cdef getitem(self, int64_t i):
+        cdef int64_t j = self.ap.value_offset(self.index) + i
+        return box_scalar(self.value_type, self.ap.values(), j)
+
+    def as_py(self):
+        cdef:
+            int64_t j
+            list result = []
+
+        for j in range(len(self)):
+            result.append(self.getitem(j).as_py())
+
+        return result
+
+
+cdef class FixedSizeBinaryValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CFixedSizeBinaryArray* ap
+            CFixedSizeBinaryType* ap_type
+            int32_t length
+            const char* data
+        ap = <CFixedSizeBinaryArray*> self.sp_array.get()
+        ap_type = <CFixedSizeBinaryType*> ap.type().get()
+        length = ap_type.byte_width()
+        data = <const char*> ap.GetValue(self.index)
+        return cp.PyBytes_FromStringAndSize(data, length)
+
+
+
+cdef dict _scalar_classes = {
+    Type_BOOL: BooleanValue,
+    Type_UINT8: Int8Value,
+    Type_UINT16: Int16Value,
+    Type_UINT32: Int32Value,
+    Type_UINT64: Int64Value,
+    Type_INT8: Int8Value,
+    Type_INT16: Int16Value,
+    Type_INT32: Int32Value,
+    Type_INT64: Int64Value,
+    Type_DATE32: Date32Value,
+    Type_DATE64: Date64Value,
+    Type_TIMESTAMP: TimestampValue,
+    Type_FLOAT: FloatValue,
+    Type_DOUBLE: DoubleValue,
+    Type_LIST: ListValue,
+    Type_BINARY: BinaryValue,
+    Type_STRING: StringValue,
+    Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
+    Type_DECIMAL: DecimalValue,
+}
+
+cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
+                       int64_t index):
+    cdef ArrayValue val
+    if type.type.id() == Type_NA:
+        return NA
+    elif sp_array.get().IsNull(index):
+        return NA
+    else:
+        val = _scalar_classes[type.type.id()]()
+        val.init(type, sp_array, index)
+        return val
+
+
+cdef maybe_coerce_datetime64(values, dtype, DataType type,
+                             timestamps_to_ms=False):
+
+    from pyarrow.compat import DatetimeTZDtype
+
+    if values.dtype.type != np.datetime64:
+        return values, type
+
+    coerce_ms = timestamps_to_ms and values.dtype != 'datetime64[ms]'
+
+    if coerce_ms:
+        values = values.astype('datetime64[ms]')
+
+    if isinstance(dtype, DatetimeTZDtype):
+        tz = dtype.tz
+        unit = 'ms' if coerce_ms else dtype.unit
+        type = timestamp(unit, tz)
+    elif type is None:
+        # Trust the NumPy dtype
+        type = from_numpy_dtype(values.dtype)
+
+    return values, type
+
+
+cdef class Array:
+
+    cdef init(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+        self.ap = sp_array.get()
+        self.type = box_data_type(self.sp_array.get().type())
+
+    @staticmethod
+    def from_numpy(obj, mask=None, DataType type=None,
+                   timestamps_to_ms=False,
+                   MemoryPool memory_pool=None):
+        """
+        Convert pandas.Series to an Arrow Array.
+
+        Parameters
+        ----------
+        series : pandas.Series or numpy.ndarray
+
+        mask : pandas.Series or numpy.ndarray, optional
+            boolean mask if the object is valid or null
+
+        type : pyarrow.DataType
+            Explicit type to attempt to coerce to
+
+        timestamps_to_ms : bool, optional
+            Convert datetime columns to ms resolution. This is needed for
+            compatibility with other functionality like Parquet I/O which
+            only supports milliseconds.
+
+        memory_pool: MemoryPool, optional
+            Specific memory pool to use to allocate the resulting Arrow array.
+
+        Notes
+        -----
+        Localized timestamps will currently be returned as UTC (pandas's native
+        representation).  Timezone-naive data will be implicitly interpreted as
+        UTC.
+
+        Examples
+        --------
+
+        >>> import pandas as pd
+        >>> import pyarrow as pa
+        >>> pa.Array.from_numpy(pd.Series([1, 2]))
+        <pyarrow.array.Int64Array object at 0x7f674e4c0e10>
+        [
+          1,
+          2
+        ]
+
+        >>> import numpy as np
+        >>> pa.Array.from_numpy(pd.Series([1, 2]), np.array([0, 1],
+        ... dtype=bool))
+        <pyarrow.array.Int64Array object at 0x7f9019e11208>
+        [
+          1,
+          NA
+        ]
+
+        Returns
+        -------
+        pyarrow.array.Array
+        """
+        cdef:
+            shared_ptr[CArray] out
+            shared_ptr[CDataType] c_type
+            CMemoryPool* pool
+
+        if mask is not None:
+            mask = get_series_values(mask)
+
+        values = get_series_values(obj)
+        pool = maybe_unbox_memory_pool(memory_pool)
+
+        if isinstance(values, Categorical):
+            return DictionaryArray.from_arrays(
+                values.codes, values.categories.values,
+                mask=mask, memory_pool=memory_pool)
+        elif values.dtype == object:
+            # Object dtype undergoes a different conversion path as more type
+            # inference may be needed
+            if type is not None:
+                c_type = type.sp_type
+            with nogil:
+                check_status(pyarrow.PandasObjectsToArrow(
+                    pool, values, mask, c_type, &out))
+        else:
+            values, type = maybe_coerce_datetime64(
+                values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
+
+            if type is None:
+                check_status(pyarrow.NumPyDtypeToArrow(values.dtype, &c_type))
+            else:
+                c_type = type.sp_type
+
+            with nogil:
+                check_status(pyarrow.PandasToArrow(
+                    pool, values, mask, c_type, &out))
+
+        return box_array(out)
+
+    @staticmethod
+    def from_list(object list_obj, DataType type=None,
+                  MemoryPool memory_pool=None):
+        """
+        Convert Python list to Arrow array
+
+        Parameters
+        ----------
+        list_obj : array_like
+
+        Returns
+        -------
+        pyarrow.array.Array
+        """
+        cdef:
+           shared_ptr[CArray] sp_array
+           CMemoryPool* pool
+
+        pool = maybe_unbox_memory_pool(memory_pool)
+        if type is None:
+            check_status(pyarrow.ConvertPySequence(list_obj, pool, &sp_array))
+        else:
+            check_status(
+                pyarrow.ConvertPySequence(
+                    list_obj, pool, &sp_array, type.sp_type
+                )
+            )
+
+        return box_array(sp_array)
+
+    property null_count:
+
+        def __get__(self):
+            return self.sp_array.get().null_count()
+
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self.getitem(i)
+        raise StopIteration
+
+    def __repr__(self):
+        from pyarrow.formatting import array_format
+        type_format = object.__repr__(self)
+        values = array_format(self, window=10)
+        return '{0}\n{1}'.format(type_format, values)
+
+    def equals(Array self, Array other):
+        return self.ap.Equals(deref(other.ap))
+
+    def __len__(self):
+        if self.sp_array.get():
+            return self.sp_array.get().length()
+        else:
+            return 0
+
+    def isnull(self):
+        raise NotImplemented
+
+    def __getitem__(self, key):
+        cdef:
+            Py_ssize_t n = len(self)
+
+        if PySlice_Check(key):
+            start = key.start or 0
+            while start < 0:
+                start += n
+
+            stop = key.stop if key.stop is not None else n
+            while stop < 0:
+                stop += n
+
+            step = key.step or 1
+            if step != 1:
+                raise IndexError('only slices with step 1 supported')
+            else:
+                return self.slice(start, stop - start)
+
+        while key < 0:
+            key += len(self)
+
+        return self.getitem(key)
+
+    cdef getitem(self, int64_t i):
+        return box_scalar(self.type, self.sp_array, i)
+
+    def slice(self, offset=0, length=None):
+        """
+        Compute zero-copy slice of this array
+
+        Parameters
+        ----------
+        offset : int, default 0
+            Offset from start of array to slice
+        length : int, default None
+            Length of slice (default is until end of Array starting from
+            offset)
+
+        Returns
+        -------
+        sliced : RecordBatch
+        """
+        cdef:
+            shared_ptr[CArray] result
+
+        if offset < 0:
+            raise IndexError('Offset must be non-negative')
+
+        if length is None:
+            result = self.ap.Slice(offset)
+        else:
+            result = self.ap.Slice(offset, length)
+
+        return box_array(result)
+
+    def to_pandas(self):
+        """
+        Convert to an array object suitable for use in pandas
+
+        See also
+        --------
+        Column.to_pandas
+        Table.to_pandas
+        RecordBatch.to_pandas
+        """
+        cdef:
+            PyObject* out
+
+        with nogil:
+            check_status(
+                pyarrow.ConvertArrayToPandas(self.sp_array, <PyObject*> self,
+                                             &out))
+        return wrap_array_output(out)
+
+    def to_pylist(self):
+        """
+        Convert to an list of native Python objects.
+        """
+        return [x.as_py() for x in self]
+
+
+cdef class Tensor:
+
+    cdef init(self, const shared_ptr[CTensor]& sp_tensor):
+        self.sp_tensor = sp_tensor
+        self.tp = sp_tensor.get()
+        self.type = box_data_type(self.tp.type())
+
+    def __repr__(self):
+        return """<pyarrow.Tensor>
+type: {0}
+shape: {1}
+strides: {2}""".format(self.type, self.shape, self.strides)
+
+    @staticmethod
+    def from_numpy(obj):
+        cdef shared_ptr[CTensor] ctensor
+        check_status(pyarrow.NdarrayToTensor(default_memory_pool(),
+                                             obj, &ctensor))
+        return box_tensor(ctensor)
+
+    def to_numpy(self):
+        """
+        Convert arrow::Tensor to numpy.ndarray with zero copy
+        """
+        cdef:
+            PyObject* out
+
+        check_status(pyarrow.TensorToNdarray(deref(self.tp), <PyObject*> self,
+                                             &out))
+        return PyObject_to_object(out)
+
+    def equals(self, Tensor other):
+        """
+        Return true if the tensors contains exactly equal data
+        """
+        return self.tp.Equals(deref(other.tp))
+
+    property is_mutable:
+
+        def __get__(self):
+            return self.tp.is_mutable()
+
+    property is_contiguous:
+
+        def __get__(self):
+            return self.tp.is_contiguous()
+
+    property ndim:
+
+        def __get__(self):
+            return self.tp.ndim()
+
+    property size:
+
+        def __get__(self):
+            return self.tp.size()
+
+    property shape:
+
+        def __get__(self):
+            cdef size_t i
+            py_shape = []
+            for i in range(self.tp.shape().size()):
+                py_shape.append(self.tp.shape()[i])
+            return py_shape
+
+    property strides:
+
+        def __get__(self):
+            cdef size_t i
+            py_strides = []
+            for i in range(self.tp.strides().size()):
+                py_strides.append(self.tp.strides()[i])
+            return py_strides
+
+
+
+cdef wrap_array_output(PyObject* output):
+    cdef object obj = PyObject_to_object(output)
+
+    if isinstance(obj, dict):
+        return Categorical(obj['indices'],
+                           categories=obj['dictionary'],
+                           fastpath=True)
+    else:
+        return obj
+
+
+cdef class NullArray(Array):
+    pass
+
+
+cdef class BooleanArray(Array):
+    pass
+
+
+cdef class NumericArray(Array):
+    pass
+
+
+cdef class IntegerArray(NumericArray):
+    pass
+
+
+cdef class FloatingPointArray(NumericArray):
+    pass
+
+
+cdef class Int8Array(IntegerArray):
+    pass
+
+
+cdef class UInt8Array(IntegerArray):
+    pass
+
+
+cdef class Int16Array(IntegerArray):
+    pass
+
+
+cdef class UInt16Array(IntegerArray):
+    pass
+
+
+cdef class Int32Array(IntegerArray):
+    pass
+
+
+cdef class UInt32Array(IntegerArray):
+    pass
+
+
+cdef class Int64Array(IntegerArray):
+    pass
+
+
+cdef class UInt64Array(IntegerArray):
+    pass
+
+
+cdef class Date32Array(NumericArray):
+    pass
+
+
+cdef class Date64Array(NumericArray):
+    pass
+
+
+cdef class TimestampArray(NumericArray):
+    pass
+
+
+cdef class Time32Array(NumericArray):
+    pass
+
+
+cdef class Time64Array(NumericArray):
+    pass
+
+
+cdef class FloatArray(FloatingPointArray):
+    pass
+
+
+cdef class DoubleArray(FloatingPointArray):
+    pass
+
+
+cdef class FixedSizeBinaryArray(Array):
+    pass
+
+
+cdef class DecimalArray(FixedSizeBinaryArray):
+    pass
+
+
+cdef class ListArray(Array):
+    pass
+
+
+cdef class StringArray(Array):
+    pass
+
+
+cdef class BinaryArray(Array):
+    pass
+
+
+cdef class DictionaryArray(Array):
+
+    cdef getitem(self, int64_t i):
+        cdef Array dictionary = self.dictionary
+        index = self.indices[i]
+        if index is NA:
+            return index
+        else:
+            return box_scalar(dictionary.type, dictionary.sp_array,
+                              index.as_py())
+
+    property dictionary:
+
+        def __get__(self):
+            cdef CDictionaryArray* darr = <CDictionaryArray*>(self.ap)
+
+            if self._dictionary is None:
+                self._dictionary = box_array(darr.dictionary())
+
+            return self._dictionary
+
+    property indices:
+
+        def __get__(self):
+            cdef CDictionaryArray* darr = <CDictionaryArray*>(self.ap)
+
+            if self._indices is None:
+                self._indices = box_array(darr.indices())
+
+            return self._indices
+
+    @staticmethod
+    def from_arrays(indices, dictionary, mask=None,
+                    MemoryPool memory_pool=None):
+        """
+        Construct Arrow DictionaryArray from array of indices (must be
+        non-negative integers) and corresponding array of dictionary values
+
+        Parameters
+        ----------
+        indices : ndarray or pandas.Series, integer type
+        dictionary : ndarray or pandas.Series
+        mask : ndarray or pandas.Series, boolean type
+            True values indicate that indices are actually null
+
+        Returns
+        -------
+        dict_array : DictionaryArray
+        """
+        cdef:
+            Array arrow_indices, arrow_dictionary
+            DictionaryArray result
+            shared_ptr[CDataType] c_type
+            shared_ptr[CArray] c_result
+
+        if isinstance(indices, Array):
+            if mask is not None:
+                raise NotImplementedError(
+                    "mask not implemented with Arrow array inputs yet")
+            arrow_indices = indices
+        else:
+            if mask is None:
+                mask = indices == -1
+            else:
+                mask = mask | (indices == -1)
+            arrow_indices = Array.from_numpy(indices, mask=mask,
+                                             memory_pool=memory_pool)
+
+        if isinstance(dictionary, Array):
+            arrow_dictionary = dictionary
+        else:
+            arrow_dictionary = Array.from_numpy(dictionary,
+                                                memory_pool=memory_pool)
+
+        if not isinstance(arrow_indices, IntegerArray):
+            raise ValueError('Indices must be integer type')
+
+        c_type.reset(new CDictionaryType(arrow_indices.type.sp_type,
+                                         arrow_dictionary.sp_array))
+        c_result.reset(new CDictionaryArray(c_type, arrow_indices.sp_array))
+
+        result = DictionaryArray()
+        result.init(c_result)
+        return result
+
+
+cdef dict _array_classes = {
+    Type_NA: NullArray,
+    Type_BOOL: BooleanArray,
+    Type_UINT8: UInt8Array,
+    Type_UINT16: UInt16Array,
+    Type_UINT32: UInt32Array,
+    Type_UINT64: UInt64Array,
+    Type_INT8: Int8Array,
+    Type_INT16: Int16Array,
+    Type_INT32: Int32Array,
+    Type_INT64: Int64Array,
+    Type_DATE32: Date32Array,
+    Type_DATE64: Date64Array,
+    Type_TIMESTAMP: TimestampArray,
+    Type_TIME32: Time32Array,
+    Type_TIME64: Time64Array,
+    Type_FLOAT: FloatArray,
+    Type_DOUBLE: DoubleArray,
+    Type_LIST: ListArray,
+    Type_BINARY: BinaryArray,
+    Type_STRING: StringArray,
+    Type_DICTIONARY: DictionaryArray,
+    Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
+    Type_DECIMAL: DecimalArray,
+}
+
+cdef object box_array(const shared_ptr[CArray]& sp_array):
+    if sp_array.get() == NULL:
+        raise ValueError('Array was NULL')
+
+    cdef CDataType* data_type = sp_array.get().type().get()
+
+    if data_type == NULL:
+        raise ValueError('Array data type was NULL')
+
+    cdef Array arr = _array_classes[data_type.id()]()
+    arr.init(sp_array)
+    return arr
+
+
+cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor):
+    if sp_tensor.get() == NULL:
+        raise ValueError('Tensor was NULL')
+
+    cdef Tensor tensor = Tensor()
+    tensor.init(sp_tensor)
+    return tensor
+
+
+cdef object get_series_values(object obj):
+    if isinstance(obj, PandasSeries):
+        result = obj.values
+    elif isinstance(obj, np.ndarray):
+        result = obj
+    else:
+        result = PandasSeries(obj).values
+
+    return result
+
+
+from_pylist = Array.from_list
diff --git a/python/pyarrow/config.pyx b/python/pyarrow/_config.pyx
similarity index 100%
rename from python/pyarrow/config.pyx
rename to python/pyarrow/_config.pyx
diff --git a/python/pyarrow/error.pxd b/python/pyarrow/_error.pxd
similarity index 100%
rename from python/pyarrow/error.pxd
rename to python/pyarrow/_error.pxd
diff --git a/python/pyarrow/error.pyx b/python/pyarrow/_error.pyx
similarity index 100%
rename from python/pyarrow/error.pyx
rename to python/pyarrow/_error.pyx
diff --git a/python/pyarrow/io.pxd b/python/pyarrow/_io.pxd
similarity index 100%
rename from python/pyarrow/io.pxd
rename to python/pyarrow/_io.pxd
diff --git a/python/pyarrow/io.pyx b/python/pyarrow/_io.pyx
similarity index 99%
rename from python/pyarrow/io.pyx
rename to python/pyarrow/_io.pyx
index 4eb0816ecbdea..9f067fb2166c6 100644
--- a/python/pyarrow/io.pyx
+++ b/python/pyarrow/_io.pyx
@@ -23,21 +23,18 @@
 # cython: embedsignature = True
 
 from cython.operator cimport dereference as deref
-
 from libc.stdlib cimport malloc, free
-
 from pyarrow.includes.libarrow cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
+from pyarrow._array cimport Array, Tensor, box_tensor, Schema
+from pyarrow._error cimport check_status
+from pyarrow._memory cimport MemoryPool, maybe_unbox_memory_pool
+from pyarrow._table cimport (Column, RecordBatch, batch_from_cbatch,
+                             table_from_ctable)
+cimport cpython as cp
 
+import pyarrow._config
 from pyarrow.compat import frombytes, tobytes, encode_file_path
-from pyarrow.array cimport Array, Tensor, box_tensor
-from pyarrow.error cimport check_status
-from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
-from pyarrow.schema cimport Schema
-from pyarrow.table cimport (Column, RecordBatch, batch_from_cbatch,
-                            table_from_ctable)
-
-cimport cpython as cp
 
 import re
 import six
diff --git a/python/pyarrow/jemalloc.pyx b/python/pyarrow/_jemalloc.pyx
similarity index 96%
rename from python/pyarrow/jemalloc.pyx
rename to python/pyarrow/_jemalloc.pyx
index 97583f4b0da95..3b41964a39cb6 100644
--- a/python/pyarrow/jemalloc.pyx
+++ b/python/pyarrow/_jemalloc.pyx
@@ -20,7 +20,7 @@
 # cython: embedsignature = True
 
 from pyarrow.includes.libarrow_jemalloc cimport CJemallocMemoryPool
-from pyarrow.memory cimport MemoryPool
+from pyarrow._memory cimport MemoryPool
 
 def default_pool():
     cdef MemoryPool pool = MemoryPool()
diff --git a/python/pyarrow/memory.pxd b/python/pyarrow/_memory.pxd
similarity index 100%
rename from python/pyarrow/memory.pxd
rename to python/pyarrow/_memory.pxd
diff --git a/python/pyarrow/memory.pyx b/python/pyarrow/_memory.pyx
similarity index 100%
rename from python/pyarrow/memory.pyx
rename to python/pyarrow/_memory.pyx
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 079bf5ee5924a..5418e1dc82730 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -20,20 +20,18 @@
 # cython: embedsignature = True
 
 from cython.operator cimport dereference as deref
-
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
+from pyarrow._array cimport Array
+from pyarrow._error cimport check_status
+from pyarrow._memory cimport MemoryPool, maybe_unbox_memory_pool
+from pyarrow._table cimport Table, table_from_ctable
+from pyarrow._io cimport NativeFile, get_reader, get_writer
 
-from pyarrow.array cimport Array
 from pyarrow.compat import tobytes, frombytes
-from pyarrow.error import ArrowException
-from pyarrow.error cimport check_status
-from pyarrow.io import NativeFile
-from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
-from pyarrow.table cimport Table, table_from_ctable
-
-from pyarrow.io cimport NativeFile, get_reader, get_writer
+from pyarrow._error import ArrowException
+from pyarrow._io import NativeFile
 
 import six
 
diff --git a/python/pyarrow/table.pxd b/python/pyarrow/_table.pxd
similarity index 98%
rename from python/pyarrow/table.pxd
rename to python/pyarrow/_table.pxd
index f564042b62d7b..e61e90d7462f9 100644
--- a/python/pyarrow/table.pxd
+++ b/python/pyarrow/_table.pxd
@@ -18,8 +18,7 @@
 from pyarrow.includes.common cimport shared_ptr
 from pyarrow.includes.libarrow cimport (CChunkedArray, CColumn, CTable,
                                         CRecordBatch)
-
-from pyarrow.schema cimport Schema
+from pyarrow._array cimport Schema
 
 
 cdef class ChunkedArray:
diff --git a/python/pyarrow/table.pyx b/python/pyarrow/_table.pyx
similarity index 98%
rename from python/pyarrow/table.pyx
rename to python/pyarrow/_table.pyx
index 3972bda4ee425..6558b2ea463fa 100644
--- a/python/pyarrow/table.pyx
+++ b/python/pyarrow/_table.pyx
@@ -24,18 +24,16 @@ from cython.operator cimport dereference as deref
 from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.common cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
+from pyarrow._array cimport (Array, box_array, wrap_array_output,
+                             box_data_type, box_schema, DataType, Field)
+from pyarrow._error cimport check_status
+cimport cpython
 
-import pyarrow.config
-
-from pyarrow.array cimport Array, box_array, wrap_array_output
-from pyarrow.error import ArrowException
-from pyarrow.error cimport check_status
-from pyarrow.schema cimport box_data_type, box_schema, DataType, Field
-
-from pyarrow.schema import field
+import pyarrow._config
+from pyarrow._error import ArrowException
+from pyarrow._array import field
 from pyarrow.compat import frombytes, tobytes
 
-cimport cpython
 
 from collections import OrderedDict
 
@@ -744,7 +742,7 @@ cdef class Table:
         pandas.DataFrame
         """
         if nthreads is None:
-            nthreads = pyarrow.config.cpu_count()
+            nthreads = pyarrow._config.cpu_count()
 
         mgr = table_to_blockmanager(self.sp_table, nthreads)
         return _pandas().DataFrame(mgr)
diff --git a/python/pyarrow/array.pyx b/python/pyarrow/array.pyx
deleted file mode 100644
index 1c4253eebe46a..0000000000000
--- a/python/pyarrow/array.pyx
+++ /dev/null
@@ -1,646 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from cython.operator cimport dereference as deref
-
-import numpy as np
-
-from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.common cimport PyObject_to_object
-cimport pyarrow.includes.pyarrow as pyarrow
-
-import pyarrow.config
-
-from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
-from pyarrow.error cimport check_status
-from pyarrow.memory cimport MemoryPool, maybe_unbox_memory_pool
-
-cimport pyarrow.scalar as scalar
-from pyarrow.scalar import NA
-
-from pyarrow.schema cimport (DataType, Field, Schema, DictionaryType,
-                             FixedSizeBinaryType,
-                             box_data_type)
-import pyarrow.schema as schema
-
-cimport cpython
-
-
-cdef maybe_coerce_datetime64(values, dtype, DataType type,
-                             timestamps_to_ms=False):
-
-    from pyarrow.compat import DatetimeTZDtype
-
-    if values.dtype.type != np.datetime64:
-        return values, type
-
-    coerce_ms = timestamps_to_ms and values.dtype != 'datetime64[ms]'
-
-    if coerce_ms:
-        values = values.astype('datetime64[ms]')
-
-    if isinstance(dtype, DatetimeTZDtype):
-        tz = dtype.tz
-        unit = 'ms' if coerce_ms else dtype.unit
-        type = schema.timestamp(unit, tz)
-    elif type is None:
-        # Trust the NumPy dtype
-        type = schema.type_from_numpy_dtype(values.dtype)
-
-    return values, type
-
-
-cdef class Array:
-
-    cdef init(self, const shared_ptr[CArray]& sp_array):
-        self.sp_array = sp_array
-        self.ap = sp_array.get()
-        self.type = box_data_type(self.sp_array.get().type())
-
-    @staticmethod
-    def from_numpy(obj, mask=None, DataType type=None,
-                   timestamps_to_ms=False,
-                   MemoryPool memory_pool=None):
-        """
-        Convert pandas.Series to an Arrow Array.
-
-        Parameters
-        ----------
-        series : pandas.Series or numpy.ndarray
-
-        mask : pandas.Series or numpy.ndarray, optional
-            boolean mask if the object is valid or null
-
-        type : pyarrow.DataType
-            Explicit type to attempt to coerce to
-
-        timestamps_to_ms : bool, optional
-            Convert datetime columns to ms resolution. This is needed for
-            compatibility with other functionality like Parquet I/O which
-            only supports milliseconds.
-
-        memory_pool: MemoryPool, optional
-            Specific memory pool to use to allocate the resulting Arrow array.
-
-        Notes
-        -----
-        Localized timestamps will currently be returned as UTC (pandas's native
-        representation).  Timezone-naive data will be implicitly interpreted as
-        UTC.
-
-        Examples
-        --------
-
-        >>> import pandas as pd
-        >>> import pyarrow as pa
-        >>> pa.Array.from_numpy(pd.Series([1, 2]))
-        <pyarrow.array.Int64Array object at 0x7f674e4c0e10>
-        [
-          1,
-          2
-        ]
-
-        >>> import numpy as np
-        >>> pa.Array.from_numpy(pd.Series([1, 2]), np.array([0, 1],
-        ... dtype=bool))
-        <pyarrow.array.Int64Array object at 0x7f9019e11208>
-        [
-          1,
-          NA
-        ]
-
-        Returns
-        -------
-        pyarrow.array.Array
-        """
-        cdef:
-            shared_ptr[CArray] out
-            shared_ptr[CDataType] c_type
-            CMemoryPool* pool
-
-        if mask is not None:
-            mask = get_series_values(mask)
-
-        values = get_series_values(obj)
-        pool = maybe_unbox_memory_pool(memory_pool)
-
-        if isinstance(values, Categorical):
-            return DictionaryArray.from_arrays(
-                values.codes, values.categories.values,
-                mask=mask, memory_pool=memory_pool)
-        elif values.dtype == object:
-            # Object dtype undergoes a different conversion path as more type
-            # inference may be needed
-            if type is not None:
-                c_type = type.sp_type
-            with nogil:
-                check_status(pyarrow.PandasObjectsToArrow(
-                    pool, values, mask, c_type, &out))
-        else:
-            values, type = maybe_coerce_datetime64(
-                values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
-
-            if type is None:
-                check_status(pyarrow.NumPyDtypeToArrow(values.dtype, &c_type))
-            else:
-                c_type = type.sp_type
-
-            with nogil:
-                check_status(pyarrow.PandasToArrow(
-                    pool, values, mask, c_type, &out))
-
-        return box_array(out)
-
-    @staticmethod
-    def from_list(object list_obj, DataType type=None,
-                  MemoryPool memory_pool=None):
-        """
-        Convert Python list to Arrow array
-
-        Parameters
-        ----------
-        list_obj : array_like
-
-        Returns
-        -------
-        pyarrow.array.Array
-        """
-        cdef:
-           shared_ptr[CArray] sp_array
-           CMemoryPool* pool
-
-        pool = maybe_unbox_memory_pool(memory_pool)
-        if type is None:
-            check_status(pyarrow.ConvertPySequence(list_obj, pool, &sp_array))
-        else:
-            check_status(
-                pyarrow.ConvertPySequence(
-                    list_obj, pool, &sp_array, type.sp_type
-                )
-            )
-
-        return box_array(sp_array)
-
-    property null_count:
-
-        def __get__(self):
-            return self.sp_array.get().null_count()
-
-    def __iter__(self):
-        for i in range(len(self)):
-            yield self.getitem(i)
-        raise StopIteration
-
-    def __repr__(self):
-        from pyarrow.formatting import array_format
-        type_format = object.__repr__(self)
-        values = array_format(self, window=10)
-        return '{0}\n{1}'.format(type_format, values)
-
-    def equals(Array self, Array other):
-        return self.ap.Equals(deref(other.ap))
-
-    def __len__(self):
-        if self.sp_array.get():
-            return self.sp_array.get().length()
-        else:
-            return 0
-
-    def isnull(self):
-        raise NotImplemented
-
-    def __getitem__(self, key):
-        cdef:
-            Py_ssize_t n = len(self)
-
-        if PySlice_Check(key):
-            start = key.start or 0
-            while start < 0:
-                start += n
-
-            stop = key.stop if key.stop is not None else n
-            while stop < 0:
-                stop += n
-
-            step = key.step or 1
-            if step != 1:
-                raise IndexError('only slices with step 1 supported')
-            else:
-                return self.slice(start, stop - start)
-
-        while key < 0:
-            key += len(self)
-
-        return self.getitem(key)
-
-    cdef getitem(self, int64_t i):
-        return scalar.box_scalar(self.type, self.sp_array, i)
-
-    def slice(self, offset=0, length=None):
-        """
-        Compute zero-copy slice of this array
-
-        Parameters
-        ----------
-        offset : int, default 0
-            Offset from start of array to slice
-        length : int, default None
-            Length of slice (default is until end of Array starting from
-            offset)
-
-        Returns
-        -------
-        sliced : RecordBatch
-        """
-        cdef:
-            shared_ptr[CArray] result
-
-        if offset < 0:
-            raise IndexError('Offset must be non-negative')
-
-        if length is None:
-            result = self.ap.Slice(offset)
-        else:
-            result = self.ap.Slice(offset, length)
-
-        return box_array(result)
-
-    def to_pandas(self):
-        """
-        Convert to an array object suitable for use in pandas
-
-        See also
-        --------
-        Column.to_pandas
-        Table.to_pandas
-        RecordBatch.to_pandas
-        """
-        cdef:
-            PyObject* out
-
-        with nogil:
-            check_status(
-                pyarrow.ConvertArrayToPandas(self.sp_array, <PyObject*> self,
-                                             &out))
-        return wrap_array_output(out)
-
-    def to_pylist(self):
-        """
-        Convert to an list of native Python objects.
-        """
-        return [x.as_py() for x in self]
-
-
-cdef class Tensor:
-
-    cdef init(self, const shared_ptr[CTensor]& sp_tensor):
-        self.sp_tensor = sp_tensor
-        self.tp = sp_tensor.get()
-        self.type = box_data_type(self.tp.type())
-
-    def __repr__(self):
-        return """<pyarrow.Tensor>
-type: {0}
-shape: {1}
-strides: {2}""".format(self.type, self.shape, self.strides)
-
-    @staticmethod
-    def from_numpy(obj):
-        cdef shared_ptr[CTensor] ctensor
-        check_status(pyarrow.NdarrayToTensor(default_memory_pool(),
-                                             obj, &ctensor))
-        return box_tensor(ctensor)
-
-    def to_numpy(self):
-        """
-        Convert arrow::Tensor to numpy.ndarray with zero copy
-        """
-        cdef:
-            PyObject* out
-
-        check_status(pyarrow.TensorToNdarray(deref(self.tp), <PyObject*> self,
-                                             &out))
-        return PyObject_to_object(out)
-
-    def equals(self, Tensor other):
-        """
-        Return true if the tensors contains exactly equal data
-        """
-        return self.tp.Equals(deref(other.tp))
-
-    property is_mutable:
-
-        def __get__(self):
-            return self.tp.is_mutable()
-
-    property is_contiguous:
-
-        def __get__(self):
-            return self.tp.is_contiguous()
-
-    property ndim:
-
-        def __get__(self):
-            return self.tp.ndim()
-
-    property size:
-
-        def __get__(self):
-            return self.tp.size()
-
-    property shape:
-
-        def __get__(self):
-            cdef size_t i
-            py_shape = []
-            for i in range(self.tp.shape().size()):
-                py_shape.append(self.tp.shape()[i])
-            return py_shape
-
-    property strides:
-
-        def __get__(self):
-            cdef size_t i
-            py_strides = []
-            for i in range(self.tp.strides().size()):
-                py_strides.append(self.tp.strides()[i])
-            return py_strides
-
-
-
-cdef wrap_array_output(PyObject* output):
-    cdef object obj = PyObject_to_object(output)
-
-    if isinstance(obj, dict):
-        return Categorical(obj['indices'],
-                           categories=obj['dictionary'],
-                           fastpath=True)
-    else:
-        return obj
-
-
-cdef class NullArray(Array):
-    pass
-
-
-cdef class BooleanArray(Array):
-    pass
-
-
-cdef class NumericArray(Array):
-    pass
-
-
-cdef class IntegerArray(NumericArray):
-    pass
-
-
-cdef class FloatingPointArray(NumericArray):
-    pass
-
-
-cdef class Int8Array(IntegerArray):
-    pass
-
-
-cdef class UInt8Array(IntegerArray):
-    pass
-
-
-cdef class Int16Array(IntegerArray):
-    pass
-
-
-cdef class UInt16Array(IntegerArray):
-    pass
-
-
-cdef class Int32Array(IntegerArray):
-    pass
-
-
-cdef class UInt32Array(IntegerArray):
-    pass
-
-
-cdef class Int64Array(IntegerArray):
-    pass
-
-
-cdef class UInt64Array(IntegerArray):
-    pass
-
-
-cdef class Date32Array(NumericArray):
-    pass
-
-
-cdef class Date64Array(NumericArray):
-    pass
-
-
-cdef class TimestampArray(NumericArray):
-    pass
-
-
-cdef class Time32Array(NumericArray):
-    pass
-
-
-cdef class Time64Array(NumericArray):
-    pass
-
-
-cdef class FloatArray(FloatingPointArray):
-    pass
-
-
-cdef class DoubleArray(FloatingPointArray):
-    pass
-
-
-cdef class FixedSizeBinaryArray(Array):
-    pass
-
-
-cdef class DecimalArray(FixedSizeBinaryArray):
-    pass
-
-
-cdef class ListArray(Array):
-    pass
-
-
-cdef class StringArray(Array):
-    pass
-
-
-cdef class BinaryArray(Array):
-    pass
-
-
-cdef class DictionaryArray(Array):
-
-    cdef getitem(self, int64_t i):
-        cdef Array dictionary = self.dictionary
-        index = self.indices[i]
-        if index is NA:
-            return index
-        else:
-            return scalar.box_scalar(dictionary.type, dictionary.sp_array,
-                                     index.as_py())
-
-    property dictionary:
-
-        def __get__(self):
-            cdef CDictionaryArray* darr = <CDictionaryArray*>(self.ap)
-
-            if self._dictionary is None:
-                self._dictionary = box_array(darr.dictionary())
-
-            return self._dictionary
-
-    property indices:
-
-        def __get__(self):
-            cdef CDictionaryArray* darr = <CDictionaryArray*>(self.ap)
-
-            if self._indices is None:
-                self._indices = box_array(darr.indices())
-
-            return self._indices
-
-    @staticmethod
-    def from_arrays(indices, dictionary, mask=None,
-                    MemoryPool memory_pool=None):
-        """
-        Construct Arrow DictionaryArray from array of indices (must be
-        non-negative integers) and corresponding array of dictionary values
-
-        Parameters
-        ----------
-        indices : ndarray or pandas.Series, integer type
-        dictionary : ndarray or pandas.Series
-        mask : ndarray or pandas.Series, boolean type
-            True values indicate that indices are actually null
-
-        Returns
-        -------
-        dict_array : DictionaryArray
-        """
-        cdef:
-            Array arrow_indices, arrow_dictionary
-            DictionaryArray result
-            shared_ptr[CDataType] c_type
-            shared_ptr[CArray] c_result
-
-        if isinstance(indices, Array):
-            if mask is not None:
-                raise NotImplementedError(
-                    "mask not implemented with Arrow array inputs yet")
-            arrow_indices = indices
-        else:
-            if mask is None:
-                mask = indices == -1
-            else:
-                mask = mask | (indices == -1)
-            arrow_indices = Array.from_numpy(indices, mask=mask,
-                                             memory_pool=memory_pool)
-
-        if isinstance(dictionary, Array):
-            arrow_dictionary = dictionary
-        else:
-            arrow_dictionary = Array.from_numpy(dictionary,
-                                                memory_pool=memory_pool)
-
-        if not isinstance(arrow_indices, IntegerArray):
-            raise ValueError('Indices must be integer type')
-
-        c_type.reset(new CDictionaryType(arrow_indices.type.sp_type,
-                                         arrow_dictionary.sp_array))
-        c_result.reset(new CDictionaryArray(c_type, arrow_indices.sp_array))
-
-        result = DictionaryArray()
-        result.init(c_result)
-        return result
-
-
-cdef dict _array_classes = {
-    Type_NA: NullArray,
-    Type_BOOL: BooleanArray,
-    Type_UINT8: UInt8Array,
-    Type_UINT16: UInt16Array,
-    Type_UINT32: UInt32Array,
-    Type_UINT64: UInt64Array,
-    Type_INT8: Int8Array,
-    Type_INT16: Int16Array,
-    Type_INT32: Int32Array,
-    Type_INT64: Int64Array,
-    Type_DATE32: Date32Array,
-    Type_DATE64: Date64Array,
-    Type_TIMESTAMP: TimestampArray,
-    Type_TIME32: Time32Array,
-    Type_TIME64: Time64Array,
-    Type_FLOAT: FloatArray,
-    Type_DOUBLE: DoubleArray,
-    Type_LIST: ListArray,
-    Type_BINARY: BinaryArray,
-    Type_STRING: StringArray,
-    Type_DICTIONARY: DictionaryArray,
-    Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
-    Type_DECIMAL: DecimalArray,
-}
-
-cdef object box_array(const shared_ptr[CArray]& sp_array):
-    if sp_array.get() == NULL:
-        raise ValueError('Array was NULL')
-
-    cdef CDataType* data_type = sp_array.get().type().get()
-
-    if data_type == NULL:
-        raise ValueError('Array data type was NULL')
-
-    cdef Array arr = _array_classes[data_type.id()]()
-    arr.init(sp_array)
-    return arr
-
-
-cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor):
-    if sp_tensor.get() == NULL:
-        raise ValueError('Tensor was NULL')
-
-    cdef Tensor tensor = Tensor()
-    tensor.init(sp_tensor)
-    return tensor
-
-
-cdef object get_series_values(object obj):
-    if isinstance(obj, PandasSeries):
-        result = obj.values
-    elif isinstance(obj, np.ndarray):
-        result = obj
-    else:
-        result = PandasSeries(obj).values
-
-    return result
-
-
-from_pylist = Array.from_list
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 3b5716e36be0a..c7b118e60a46d 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -22,9 +22,9 @@
 import pandas as pd
 
 from pyarrow.compat import pdapi
-from pyarrow.io import FeatherError  # noqa
-from pyarrow.table import Table
-import pyarrow.io as ext
+from pyarrow._io import FeatherError  # noqa
+from pyarrow._table import Table
+import pyarrow._io as ext
 
 
 if LooseVersion(pd.__version__) < '0.17.0':
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 269cf1c8ffa12..92dd91ce9de07 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -19,7 +19,7 @@
 import os
 
 from pyarrow.util import implements
-import pyarrow.io as io
+import pyarrow._io as io
 
 
 class Filesystem(object):
diff --git a/python/pyarrow/formatting.py b/python/pyarrow/formatting.py
index 5fe0611f8450b..c3583448d6e17 100644
--- a/python/pyarrow/formatting.py
+++ b/python/pyarrow/formatting.py
@@ -17,7 +17,7 @@
 
 # Pretty-printing and other formatting utilities for Arrow data structures
 
-import pyarrow.scalar as scalar
+import pyarrow._array as _array
 
 
 def array_format(arr, window=None):
@@ -42,7 +42,7 @@ def array_format(arr, window=None):
 
 
 def value_format(x, indent_level=0):
-    if isinstance(x, scalar.ListValue):
+    if isinstance(x, _array.ListValue):
         contents = ',\n'.join(value_format(item) for item in x)
         return '[{0}]'.format(_indent(contents, 1).strip())
     else:
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index ae2b45fbdb212..2444f3fd0683e 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -113,8 +113,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CDataType] index_type()
         shared_ptr[CArray] dictionary()
 
-    shared_ptr[CDataType] timestamp(TimeUnit unit)
-    shared_ptr[CDataType] timestamp(TimeUnit unit, const c_string& timezone)
+    shared_ptr[CDataType] ctimestamp" arrow::timestamp"(TimeUnit unit)
+    shared_ptr[CDataType] ctimestamp" arrow::timestamp"(
+        TimeUnit unit, const c_string& timezone)
 
     cdef cppclass CMemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index 5a5616564324c..f96ead3b92346 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -17,10 +17,10 @@
 
 # Arrow file and stream reader/writer classes, and other messaging tools
 
-import pyarrow.io as io
+import pyarrow._io as _io
 
 
-class StreamReader(io._StreamReader):
+class StreamReader(_io._StreamReader):
     """
     Reader for the Arrow streaming binary format
 
@@ -37,7 +37,7 @@ def __iter__(self):
             yield self.get_next_batch()
 
 
-class StreamWriter(io._StreamWriter):
+class StreamWriter(_io._StreamWriter):
     """
     Writer for the Arrow streaming binary format
 
@@ -52,7 +52,7 @@ def __init__(self, sink, schema):
         self._open(sink, schema)
 
 
-class FileReader(io._FileReader):
+class FileReader(_io._FileReader):
     """
     Class for reading Arrow record batch data from the Arrow binary file format
 
@@ -68,7 +68,7 @@ def __init__(self, source, footer_offset=None):
         self._open(source, footer_offset=footer_offset)
 
 
-class FileWriter(io._FileWriter):
+class FileWriter(_io._FileWriter):
     """
     Writer to create the Arrow binary file format
 
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index f81b6c24c691f..aaec43ab06027 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -23,8 +23,8 @@
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
                               RowGroupMetaData, Schema, ParquetWriter)
 import pyarrow._parquet as _parquet  # noqa
-import pyarrow.array as _array
-import pyarrow.table as _table
+import pyarrow._array as _array
+import pyarrow._table as _table
 
 
 # ----------------------------------------------------------------------
diff --git a/python/pyarrow/scalar.pxd b/python/pyarrow/scalar.pxd
deleted file mode 100644
index 62a5664e57eb4..0000000000000
--- a/python/pyarrow/scalar.pxd
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport *
-
-from pyarrow.schema cimport DataType
-
-
-cdef class Scalar:
-    cdef readonly:
-        DataType type
-
-
-cdef class NAType(Scalar):
-    pass
-
-
-cdef class ArrayValue(Scalar):
-    cdef:
-        shared_ptr[CArray] sp_array
-        int64_t index
-
-    cdef void init(self, DataType type,
-                   const shared_ptr[CArray]& sp_array, int64_t index)
-
-    cdef void _set_array(self, const shared_ptr[CArray]& sp_array)
-
-
-cdef class Int8Value(ArrayValue):
-    pass
-
-
-cdef class Int64Value(ArrayValue):
-    pass
-
-
-cdef class ListValue(ArrayValue):
-    cdef readonly:
-        DataType value_type
-
-    cdef:
-        CListArray* ap
-
-    cdef getitem(self, int64_t i)
-
-
-cdef class StringValue(ArrayValue):
-    pass
-
-
-cdef class FixedSizeBinaryValue(ArrayValue):
-    pass
-
-
-cdef object box_scalar(DataType type,
-                       const shared_ptr[CArray]& sp_array,
-                       int64_t index)
diff --git a/python/pyarrow/scalar.pyx b/python/pyarrow/scalar.pyx
deleted file mode 100644
index 2b6746a3cf815..0000000000000
--- a/python/pyarrow/scalar.pyx
+++ /dev/null
@@ -1,315 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyarrow.schema cimport DataType, box_data_type
-
-from pyarrow.compat import frombytes
-import pyarrow.schema as schema
-import decimal
-import datetime
-
-cimport cpython as cp
-
-NA = None
-
-
-cdef _pandas():
-    import pandas as pd
-    return pd
-
-
-cdef class NAType(Scalar):
-
-    def __cinit__(self):
-        global NA
-        if NA is not None:
-            raise Exception('Cannot create multiple NAType instances')
-
-        self.type = schema.null()
-
-    def __repr__(self):
-        return 'NA'
-
-    def as_py(self):
-        return None
-
-NA = NAType()
-
-cdef class ArrayValue(Scalar):
-
-    cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
-                   int64_t index):
-        self.type = type
-        self.index = index
-        self._set_array(sp_array)
-
-    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
-        self.sp_array = sp_array
-
-    def __repr__(self):
-        if hasattr(self, 'as_py'):
-            return repr(self.as_py())
-        else:
-            return super(Scalar, self).__repr__()
-
-
-cdef class BooleanValue(ArrayValue):
-
-    def as_py(self):
-        cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Int8Value(ArrayValue):
-
-    def as_py(self):
-        cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class UInt8Value(ArrayValue):
-
-    def as_py(self):
-        cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Int16Value(ArrayValue):
-
-    def as_py(self):
-        cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class UInt16Value(ArrayValue):
-
-    def as_py(self):
-        cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Int32Value(ArrayValue):
-
-    def as_py(self):
-        cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class UInt32Value(ArrayValue):
-
-    def as_py(self):
-        cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Int64Value(ArrayValue):
-
-    def as_py(self):
-        cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class UInt64Value(ArrayValue):
-
-    def as_py(self):
-        cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Date32Value(ArrayValue):
-
-    def as_py(self):
-        cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
-
-        # Shift to seconds since epoch
-        return datetime.datetime.utcfromtimestamp(
-            int(ap.Value(self.index)) * 86400).date()
-
-
-cdef class Date64Value(ArrayValue):
-
-    def as_py(self):
-        cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
-        return datetime.datetime.utcfromtimestamp(
-            ap.Value(self.index) / 1000).date()
-
-
-cdef class TimestampValue(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
-            CTimestampType* dtype = <CTimestampType*>ap.type().get()
-            int64_t val = ap.Value(self.index)
-
-        timezone = None
-        tzinfo = None
-        if dtype.timezone().size() > 0:
-            timezone = frombytes(dtype.timezone())
-            import pytz
-            tzinfo = pytz.timezone(timezone)
-
-        try:
-            pd = _pandas()
-            if dtype.unit() == TimeUnit_SECOND:
-                val = val * 1000000000
-            elif dtype.unit() == TimeUnit_MILLI:
-                val = val * 1000000
-            elif dtype.unit() == TimeUnit_MICRO:
-                val = val * 1000
-            return pd.Timestamp(val, tz=tzinfo)
-        except ImportError:
-            if dtype.unit() == TimeUnit_SECOND:
-                result = datetime.datetime.utcfromtimestamp(val)
-            elif dtype.unit() == TimeUnit_MILLI:
-                result = datetime.datetime.utcfromtimestamp(float(val) / 1000)
-            elif dtype.unit() == TimeUnit_MICRO:
-                result = datetime.datetime.utcfromtimestamp(
-                    float(val) / 1000000)
-            else:
-                # TimeUnit_NANO
-                raise NotImplementedError("Cannot convert nanosecond "
-                                          "timestamps without pandas")
-            if timezone is not None:
-                result = result.replace(tzinfo=tzinfo)
-            return result
-
-
-cdef class FloatValue(ArrayValue):
-
-    def as_py(self):
-        cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class DoubleValue(ArrayValue):
-
-    def as_py(self):
-        cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class DecimalValue(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            CDecimalArray* ap = <CDecimalArray*> self.sp_array.get()
-            c_string s = ap.FormatValue(self.index)
-        return decimal.Decimal(s.decode('utf8'))
-
-
-cdef class StringValue(ArrayValue):
-
-    def as_py(self):
-        cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
-        return ap.GetString(self.index).decode('utf-8')
-
-
-cdef class BinaryValue(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            const uint8_t* ptr
-            int32_t length
-            CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
-
-        ptr = ap.GetValue(self.index, &length)
-        return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
-
-cdef class ListValue(ArrayValue):
-
-    def __len__(self):
-        return self.ap.value_length(self.index)
-
-    def __getitem__(self, i):
-        return self.getitem(i)
-
-    def __iter__(self):
-        for i in range(len(self)):
-            yield self.getitem(i)
-        raise StopIteration
-
-    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
-        self.sp_array = sp_array
-        self.ap = <CListArray*> sp_array.get()
-        self.value_type = box_data_type(self.ap.value_type())
-
-    cdef getitem(self, int64_t i):
-        cdef int64_t j = self.ap.value_offset(self.index) + i
-        return box_scalar(self.value_type, self.ap.values(), j)
-
-    def as_py(self):
-        cdef:
-            int64_t j
-            list result = []
-
-        for j in range(len(self)):
-            result.append(self.getitem(j).as_py())
-
-        return result
-
-
-cdef class FixedSizeBinaryValue(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            CFixedSizeBinaryArray* ap
-            CFixedSizeBinaryType* ap_type
-            int32_t length
-            const char* data
-        ap = <CFixedSizeBinaryArray*> self.sp_array.get()
-        ap_type = <CFixedSizeBinaryType*> ap.type().get()
-        length = ap_type.byte_width()
-        data = <const char*> ap.GetValue(self.index)
-        return cp.PyBytes_FromStringAndSize(data, length)
-
-
-
-cdef dict _scalar_classes = {
-    Type_BOOL: BooleanValue,
-    Type_UINT8: Int8Value,
-    Type_UINT16: Int16Value,
-    Type_UINT32: Int32Value,
-    Type_UINT64: Int64Value,
-    Type_INT8: Int8Value,
-    Type_INT16: Int16Value,
-    Type_INT32: Int32Value,
-    Type_INT64: Int64Value,
-    Type_DATE32: Date32Value,
-    Type_DATE64: Date64Value,
-    Type_TIMESTAMP: TimestampValue,
-    Type_FLOAT: FloatValue,
-    Type_DOUBLE: DoubleValue,
-    Type_LIST: ListValue,
-    Type_BINARY: BinaryValue,
-    Type_STRING: StringValue,
-    Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
-    Type_DECIMAL: DecimalValue,
-}
-
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
-                       int64_t index):
-    cdef ArrayValue val
-    if type.type.id() == Type_NA:
-        return NA
-    elif sp_array.get().IsNull(index):
-        return NA
-    else:
-        val = _scalar_classes[type.type.id()]()
-        val.init(type, sp_array, index)
-        return val
diff --git a/python/pyarrow/schema.pxd b/python/pyarrow/schema.pxd
deleted file mode 100644
index eceedbad0ba0d..0000000000000
--- a/python/pyarrow/schema.pxd
+++ /dev/null
@@ -1,76 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CDataType,
-                                        CDictionaryType,
-                                        CTimestampType,
-                                        CFixedSizeBinaryType,
-                                        CDecimalType,
-                                        CField, CSchema)
-
-cdef class DataType:
-    cdef:
-        shared_ptr[CDataType] sp_type
-        CDataType* type
-
-    cdef void init(self, const shared_ptr[CDataType]& type)
-
-
-cdef class DictionaryType(DataType):
-    cdef:
-        const CDictionaryType* dict_type
-
-
-cdef class TimestampType(DataType):
-    cdef:
-        const CTimestampType* ts_type
-
-
-cdef class FixedSizeBinaryType(DataType):
-    cdef:
-        const CFixedSizeBinaryType* fixed_size_binary_type
-
-
-cdef class DecimalType(FixedSizeBinaryType):
-    cdef:
-        const CDecimalType* decimal_type
-
-
-cdef class Field:
-    cdef:
-        shared_ptr[CField] sp_field
-        CField* field
-
-    cdef readonly:
-        DataType type
-
-    cdef init(self, const shared_ptr[CField]& field)
-
-
-cdef class Schema:
-    cdef:
-        shared_ptr[CSchema] sp_schema
-        CSchema* schema
-
-    cdef init(self, const vector[shared_ptr[CField]]& fields)
-    cdef init_schema(self, const shared_ptr[CSchema]& schema)
-
-
-cdef DataType box_data_type(const shared_ptr[CDataType]& type)
-cdef Field box_field(const shared_ptr[CField]& field)
-cdef Schema box_schema(const shared_ptr[CSchema]& schema)
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
deleted file mode 100644
index 474980973959f..0000000000000
--- a/python/pyarrow/schema.pyx
+++ /dev/null
@@ -1,477 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-########################################
-# Data types, fields, schemas, and so forth
-
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from cython.operator cimport dereference as deref
-
-from pyarrow.compat import frombytes, tobytes
-from pyarrow.array cimport Array
-from pyarrow.error cimport check_status
-from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
-                                        CFixedSizeBinaryType,
-                                        CDecimalType,
-                                        TimeUnit_SECOND, TimeUnit_MILLI,
-                                        TimeUnit_MICRO, TimeUnit_NANO,
-                                        Type, TimeUnit)
-cimport pyarrow.includes.pyarrow as pyarrow
-cimport pyarrow.includes.libarrow as la
-
-cimport cpython
-
-import six
-
-
-cdef class DataType:
-
-    def __cinit__(self):
-        pass
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        self.sp_type = type
-        self.type = type.get()
-
-    def __str__(self):
-        return frombytes(self.type.ToString())
-
-    def __repr__(self):
-        return '{0.__class__.__name__}({0})'.format(self)
-
-    def __richcmp__(DataType self, DataType other, int op):
-        if op == cpython.Py_EQ:
-            return self.type.Equals(deref(other.type))
-        elif op == cpython.Py_NE:
-            return not self.type.Equals(deref(other.type))
-        else:
-            raise TypeError('Invalid comparison')
-
-
-cdef class DictionaryType(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.dict_type = <const CDictionaryType*> type.get()
-
-
-cdef class TimestampType(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.ts_type = <const CTimestampType*> type.get()
-
-    property unit:
-
-        def __get__(self):
-            return timeunit_to_string(self.ts_type.unit())
-
-    property tz:
-
-        def __get__(self):
-            if self.ts_type.timezone().size() > 0:
-                return frombytes(self.ts_type.timezone())
-            else:
-                return None
-
-
-cdef class FixedSizeBinaryType(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.fixed_size_binary_type = <const CFixedSizeBinaryType*> type.get()
-
-    property byte_width:
-
-        def __get__(self):
-            return self.fixed_size_binary_type.byte_width()
-
-
-cdef class DecimalType(FixedSizeBinaryType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.decimal_type = <const CDecimalType*> type.get()
-
-
-cdef class Field:
-
-    def __cinit__(self):
-        pass
-
-    cdef init(self, const shared_ptr[CField]& field):
-        self.sp_field = field
-        self.field = field.get()
-        self.type = box_data_type(field.get().type())
-
-    @classmethod
-    def from_py(cls, object name, DataType type, bint nullable=True):
-        cdef Field result = Field()
-        result.type = type
-        result.sp_field.reset(new CField(tobytes(name), type.sp_type,
-                                         nullable))
-        result.field = result.sp_field.get()
-
-        return result
-
-    def __repr__(self):
-        return 'Field({0!r}, type={1})'.format(self.name, str(self.type))
-
-    property nullable:
-
-        def __get__(self):
-            return self.field.nullable()
-
-    property name:
-
-        def __get__(self):
-            if box_field(self.sp_field) is None:
-                raise ReferenceError(
-                    'Field not initialized (references NULL pointer)')
-            return frombytes(self.field.name())
-
-
-cdef class Schema:
-
-    def __cinit__(self):
-        pass
-
-    def __len__(self):
-        return self.schema.num_fields()
-
-    def __getitem__(self, i):
-        if i < 0 or i >= len(self):
-            raise IndexError("{0} is out of bounds".format(i))
-
-        cdef Field result = Field()
-        result.init(self.schema.field(i))
-        result.type = box_data_type(result.field.type())
-
-        return result
-
-    cdef init(self, const vector[shared_ptr[CField]]& fields):
-        self.schema = new CSchema(fields)
-        self.sp_schema.reset(self.schema)
-
-    cdef init_schema(self, const shared_ptr[CSchema]& schema):
-        self.schema = schema.get()
-        self.sp_schema = schema
-
-    def equals(self, other):
-        """
-        Test if this schema is equal to the other
-        """
-        cdef Schema _other
-        _other = other
-
-        return self.sp_schema.get().Equals(deref(_other.schema))
-
-    def field_by_name(self, name):
-        """
-        Access a field by its name rather than the column index.
-
-        Parameters
-        ----------
-        name: str
-
-        Returns
-        -------
-        field: pyarrow.Field
-        """
-        return box_field(self.schema.GetFieldByName(tobytes(name)))
-
-    @classmethod
-    def from_fields(cls, fields):
-        cdef:
-            Schema result
-            Field field
-            vector[shared_ptr[CField]] c_fields
-
-        c_fields.resize(len(fields))
-
-        for i in range(len(fields)):
-            field = fields[i]
-            c_fields[i] = field.sp_field
-
-        result = Schema()
-        result.init(c_fields)
-
-        return result
-
-    def __str__(self):
-        return frombytes(self.schema.ToString())
-
-    def __repr__(self):
-        return self.__str__()
-
-
-cdef dict _type_cache = {}
-
-
-cdef DataType primitive_type(Type type):
-    if type in _type_cache:
-        return _type_cache[type]
-
-    cdef DataType out = DataType()
-    out.init(pyarrow.GetPrimitiveType(type))
-
-    _type_cache[type] = out
-    return out
-
-#------------------------------------------------------------
-# Type factory functions
-
-def field(name, type, bint nullable=True):
-    return Field.from_py(name, type, nullable)
-
-
-cdef set PRIMITIVE_TYPES = set([
-    la.Type_NA, la.Type_BOOL,
-    la.Type_UINT8, la.Type_INT8,
-    la.Type_UINT16, la.Type_INT16,
-    la.Type_UINT32, la.Type_INT32,
-    la.Type_UINT64, la.Type_INT64,
-    la.Type_TIMESTAMP, la.Type_DATE32,
-    la.Type_DATE64,
-    la.Type_HALF_FLOAT,
-    la.Type_FLOAT,
-    la.Type_DOUBLE])
-
-
-def null():
-    return primitive_type(la.Type_NA)
-
-
-def bool_():
-    return primitive_type(la.Type_BOOL)
-
-
-def uint8():
-    return primitive_type(la.Type_UINT8)
-
-
-def int8():
-    return primitive_type(la.Type_INT8)
-
-
-def uint16():
-    return primitive_type(la.Type_UINT16)
-
-
-def int16():
-    return primitive_type(la.Type_INT16)
-
-
-def uint32():
-    return primitive_type(la.Type_UINT32)
-
-
-def int32():
-    return primitive_type(la.Type_INT32)
-
-
-def uint64():
-    return primitive_type(la.Type_UINT64)
-
-
-def int64():
-    return primitive_type(la.Type_INT64)
-
-
-cdef dict _timestamp_type_cache = {}
-
-
-cdef timeunit_to_string(TimeUnit unit):
-    if unit == TimeUnit_SECOND:
-        return 's'
-    elif unit == TimeUnit_MILLI:
-        return 'ms'
-    elif unit == TimeUnit_MICRO:
-        return 'us'
-    elif unit == TimeUnit_NANO:
-        return 'ns'
-
-
-def timestamp(unit_str, tz=None):
-    cdef:
-        TimeUnit unit
-        c_string c_timezone
-
-    if unit_str == "s":
-        unit = TimeUnit_SECOND
-    elif unit_str == 'ms':
-        unit = TimeUnit_MILLI
-    elif unit_str == 'us':
-        unit = TimeUnit_MICRO
-    elif unit_str == 'ns':
-        unit = TimeUnit_NANO
-    else:
-        raise TypeError('Invalid TimeUnit string')
-
-    cdef TimestampType out = TimestampType()
-
-    if tz is None:
-        out.init(la.timestamp(unit))
-        if unit in _timestamp_type_cache:
-            return _timestamp_type_cache[unit]
-        _timestamp_type_cache[unit] = out
-    else:
-        if not isinstance(tz, six.string_types):
-            tz = tz.zone
-
-        c_timezone = tobytes(tz)
-        out.init(la.timestamp(unit, c_timezone))
-
-    return out
-
-
-def date32():
-    return primitive_type(la.Type_DATE32)
-
-
-def date64():
-    return primitive_type(la.Type_DATE64)
-
-
-def float16():
-    return primitive_type(la.Type_HALF_FLOAT)
-
-
-def float32():
-    return primitive_type(la.Type_FLOAT)
-
-
-def float64():
-    return primitive_type(la.Type_DOUBLE)
-
-
-cpdef DataType decimal(int precision, int scale=0):
-    cdef shared_ptr[CDataType] decimal_type
-    decimal_type.reset(new CDecimalType(precision, scale))
-    return box_data_type(decimal_type)
-
-
-def string():
-    """
-    UTF8 string
-    """
-    return primitive_type(la.Type_STRING)
-
-
-def binary(int length=-1):
-    """Binary (PyBytes-like) type
-
-    Parameters
-    ----------
-    length : int, optional, default -1
-        If length == -1 then return a variable length binary type. If length is
-        greater than or equal to 0 then return a fixed size binary type of
-        width `length`.
-    """
-    if length == -1:
-        return primitive_type(la.Type_BINARY)
-
-    cdef shared_ptr[CDataType] fixed_size_binary_type
-    fixed_size_binary_type.reset(new CFixedSizeBinaryType(length))
-    return box_data_type(fixed_size_binary_type)
-
-
-def list_(DataType value_type):
-    cdef DataType out = DataType()
-    cdef shared_ptr[CDataType] list_type
-    list_type.reset(new CListType(value_type.sp_type))
-    out.init(list_type)
-    return out
-
-
-def dictionary(DataType index_type, Array dictionary):
-    """
-    Dictionary (categorical, or simply encoded) type
-    """
-    cdef DictionaryType out = DictionaryType()
-    cdef shared_ptr[CDataType] dict_type
-    dict_type.reset(new CDictionaryType(index_type.sp_type,
-                                        dictionary.sp_array))
-    out.init(dict_type)
-    return out
-
-
-def struct(fields):
-    """
-
-    """
-    cdef:
-        DataType out = DataType()
-        Field field
-        vector[shared_ptr[CField]] c_fields
-        cdef shared_ptr[CDataType] struct_type
-
-    for field in fields:
-        c_fields.push_back(field.sp_field)
-
-    struct_type.reset(new CStructType(c_fields))
-    out.init(struct_type)
-    return out
-
-
-def schema(fields):
-    return Schema.from_fields(fields)
-
-
-cdef DataType box_data_type(const shared_ptr[CDataType]& type):
-    cdef:
-        DataType out
-
-    if type.get() == NULL:
-        return None
-
-    if type.get().id() == la.Type_DICTIONARY:
-        out = DictionaryType()
-    elif type.get().id() == la.Type_TIMESTAMP:
-        out = TimestampType()
-    elif type.get().id() == la.Type_FIXED_SIZE_BINARY:
-        out = FixedSizeBinaryType()
-    elif type.get().id() == la.Type_DECIMAL:
-        out = DecimalType()
-    else:
-        out = DataType()
-
-    out.init(type)
-    return out
-
-cdef Field box_field(const shared_ptr[CField]& field):
-    if field.get() == NULL:
-        return None
-    cdef Field out = Field()
-    out.init(field)
-    return out
-
-cdef Schema box_schema(const shared_ptr[CSchema]& type):
-    cdef Schema out = Schema()
-    out.init_schema(type)
-    return out
-
-
-def type_from_numpy_dtype(object dtype):
-    cdef shared_ptr[CDataType] c_type
-    with nogil:
-        check_status(pyarrow.NumPyDtypeToArrow(dtype, &c_type))
-
-    return box_data_type(c_type)
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index cba9464354a4e..6f8040fd483c9 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -25,7 +25,7 @@
 from pyarrow.compat import guid
 from pyarrow.feather import (read_feather, write_feather,
                              FeatherReader)
-from pyarrow.io import FeatherWriter
+from pyarrow._io import FeatherWriter
 
 
 def random_path():
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index b8f7e25233421..d2a54790668d5 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -26,8 +26,6 @@
 import pytest
 
 from pyarrow.compat import guid
-from pyarrow.filesystem import HdfsClient
-import pyarrow.io as io
 import pyarrow as pa
 
 import pyarrow.tests.test_parquet as test_parquet
@@ -45,7 +43,7 @@ def hdfs_test_client(driver='libhdfs'):
         raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
                          'an integer')
 
-    return HdfsClient(host, port, user, driver=driver)
+    return pa.HdfsClient(host, port, user, driver=driver)
 
 
 @pytest.mark.hdfs
@@ -190,7 +188,7 @@ class TestLibHdfs(HdfsTestCases, unittest.TestCase):
 
     @classmethod
     def check_driver(cls):
-        if not io.have_libhdfs():
+        if not pa.have_libhdfs():
             pytest.fail('No libhdfs available on system')
 
     def test_hdfs_orphaned_file(self):
@@ -209,5 +207,5 @@ class TestLibHdfs3(HdfsTestCases, unittest.TestCase):
 
     @classmethod
     def check_driver(cls):
-        if not io.have_libhdfs3():
+        if not pa.have_libhdfs3():
             pytest.fail('No libhdfs3 available on system')
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index beb6113849ac3..c5d3708d6a9ac 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -24,7 +24,6 @@
 
 from pyarrow.compat import u, guid
 import pyarrow as pa
-import pyarrow.io as io
 
 # ----------------------------------------------------------------------
 # Python file-like objects
@@ -33,7 +32,7 @@
 def test_python_file_write():
     buf = BytesIO()
 
-    f = io.PythonFileInterface(buf)
+    f = pa.PythonFileInterface(buf)
 
     assert f.tell() == 0
 
@@ -57,7 +56,7 @@ def test_python_file_read():
     data = b'some sample data'
 
     buf = BytesIO(data)
-    f = io.PythonFileInterface(buf, mode='r')
+    f = pa.PythonFileInterface(buf, mode='r')
 
     assert f.size() == len(data)
 
@@ -82,7 +81,7 @@ def test_python_file_read():
 def test_bytes_reader():
     # Like a BytesIO, but zero-copy underneath for C++ consumers
     data = b'some sample data'
-    f = io.BufferReader(data)
+    f = pa.BufferReader(data)
     assert f.tell() == 0
 
     assert f.size() == len(data)
@@ -103,7 +102,7 @@ def test_bytes_reader():
 
 def test_bytes_reader_non_bytes():
     with pytest.raises(ValueError):
-        io.BufferReader(u('some sample data'))
+        pa.BufferReader(u('some sample data'))
 
 
 def test_bytes_reader_retains_parent_reference():
@@ -112,7 +111,7 @@ def test_bytes_reader_retains_parent_reference():
     # ARROW-421
     def get_buffer():
         data = b'some sample data' * 1000
-        reader = io.BufferReader(data)
+        reader = pa.BufferReader(data)
         reader.seek(5)
         return reader.read_buffer(6)
 
@@ -129,7 +128,7 @@ def test_buffer_bytes():
     val = b'some data'
 
     buf = pa.frombuffer(val)
-    assert isinstance(buf, io.Buffer)
+    assert isinstance(buf, pa.Buffer)
 
     result = buf.to_pybytes()
 
@@ -140,7 +139,7 @@ def test_buffer_memoryview():
     val = b'some data'
 
     buf = pa.frombuffer(val)
-    assert isinstance(buf, io.Buffer)
+    assert isinstance(buf, pa.Buffer)
 
     result = memoryview(buf)
 
@@ -151,7 +150,7 @@ def test_buffer_bytearray():
     val = bytearray(b'some data')
 
     buf = pa.frombuffer(val)
-    assert isinstance(buf, io.Buffer)
+    assert isinstance(buf, pa.Buffer)
 
     result = bytearray(buf)
 
@@ -162,7 +161,7 @@ def test_buffer_memoryview_is_immutable():
     val = b'some data'
 
     buf = pa.frombuffer(val)
-    assert isinstance(buf, io.Buffer)
+    assert isinstance(buf, pa.Buffer)
 
     result = memoryview(buf)
 
@@ -180,7 +179,7 @@ def test_memory_output_stream():
     # 10 bytes
     val = b'dataabcdef'
 
-    f = io.InMemoryOutputStream()
+    f = pa.InMemoryOutputStream()
 
     K = 1000
     for i in range(K):
@@ -193,7 +192,7 @@ def test_memory_output_stream():
 
 
 def test_inmemory_write_after_closed():
-    f = io.InMemoryOutputStream()
+    f = pa.InMemoryOutputStream()
     f.write(b'ok')
     f.get_result()
 
@@ -213,7 +212,7 @@ def make_buffer(bytes_obj):
 
 
 def test_nativefile_write_memoryview():
-    f = io.InMemoryOutputStream()
+    f = pa.InMemoryOutputStream()
     data = b'ok'
 
     arr = np.frombuffer(data, dtype='S1')
@@ -289,7 +288,7 @@ def test_memory_map_retain_buffer_reference(sample_disk_data):
 
 
 def test_os_file_reader(sample_disk_data):
-    _check_native_file_reader(io.OSFile, sample_disk_data)
+    _check_native_file_reader(pa.OSFile, sample_disk_data)
 
 
 def _try_delete(path):
@@ -354,10 +353,10 @@ def test_os_file_writer():
             f.write(data)
 
         # Truncates file
-        f2 = io.OSFile(path, mode='w')
+        f2 = pa.OSFile(path, mode='w')
         f2.write('foo')
 
-        with io.OSFile(path) as f3:
+        with pa.OSFile(path) as f3:
             assert f3.size() == 3
 
         with pytest.raises(IOError):
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index de1b1488c1475..a5c70aa16225f 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -24,7 +24,6 @@
 from pyarrow.compat import guid, u
 from pyarrow.filesystem import LocalFilesystem
 import pyarrow as pa
-import pyarrow.io as paio
 from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
 import numpy as np
@@ -180,10 +179,10 @@ def _test_dataframe(size=10000, seed=0):
 def test_pandas_parquet_native_file_roundtrip(tmpdir):
     df = _test_dataframe(10000)
     arrow_table = pa.Table.from_pandas(df)
-    imos = paio.InMemoryOutputStream()
+    imos = pa.InMemoryOutputStream()
     pq.write_table(arrow_table, imos, version="2.0")
     buf = imos.get_result()
-    reader = paio.BufferReader(buf)
+    reader = pa.BufferReader(buf)
     df_read = pq.read_table(reader).to_pandas()
     tm.assert_frame_equal(df, df_read)
 
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 5588840cceb1f..53b6b68cfde3c 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -16,13 +16,9 @@
 # under the License.
 
 import pytest
-
-import pyarrow as pa
-
 import numpy as np
 
-# XXX: pyarrow.schema.schema masks the module on imports
-sch = pa._schema
+import pyarrow as pa
 
 
 def test_type_integers():
@@ -62,7 +58,7 @@ def test_type_from_numpy_dtype_timestamps():
     ]
 
     for dt, pt in cases:
-        result = sch.type_from_numpy_dtype(dt)
+        result = pa.from_numpy_dtype(dt)
         assert result == pt
 
 
diff --git a/python/setup.py b/python/setup.py
index 99bac15c779e6..3991856404bc8 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -99,16 +99,14 @@ def initialize_options(self):
             os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
 
     CYTHON_MODULE_NAMES = [
-        'array',
-        'config',
-        'error',
-        'io',
-        'jemalloc',
-        'memory',
+        '_array',
+        '_config',
+        '_error',
+        '_io',
+        '_jemalloc',
+        '_memory',
         '_parquet',
-        'scalar',
-        'schema',
-        'table']
+        '_table']
 
     def _run_cmake(self):
         # The directory containing this setup.py
@@ -261,7 +259,7 @@ def move_lib(lib_name):
     def _failure_permitted(self, name):
         if name == '_parquet' and not self.with_parquet:
             return True
-        if name == 'jemalloc' and not self.with_jemalloc:
+        if name == '_jemalloc' and not self.with_jemalloc:
             return True
         return False
 

From 19da86ab96fa839786eef768ff4521f46acaa3a4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 13 Apr 2017 11:16:38 -0400
Subject: [PATCH 0509/1644] ARROW-817: [Python] Fix comment in date32
 conversion

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #536 from wesm/ARROW-817 and squashes the following commits:

3982948 [Wes McKinney] Fix comment
---
 cpp/src/arrow/python/pandas_convert.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 643c5fb8796a0..b33aea4565817 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -1504,8 +1504,7 @@ class DatetimeBlock : public PandasBlock {
     const ChunkedArray& data = *col.get()->data();
 
     if (type == Type::DATE32) {
-      // Date64Type is millisecond timestamp stored as int64_t
-      // TODO(wesm): Do we want to make sure to zero out the milliseconds?
+      // Convert from days since epoch to datetime64[ns]
       ConvertDatetimeNanos<int32_t, kNanosecondsInDay>(data, out_buffer);
     } else if (type == Type::DATE64) {
       // Date64Type is millisecond timestamp stored as int64_t

From 874666a61c4c7bf9f1242d8bb05274b7d1bbe2bd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 13 Apr 2017 14:01:56 -0400
Subject: [PATCH 0510/1644] ARROW-816: [C++] Travis CI script cleanup, add C++
 toolchain env with Flatbuffers, RapidJSON

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #537 from wesm/ARROW-816 and squashes the following commits:

16992b6 [Wes McKinney] Disable Travis CI cache on OS X. brew install ccache
4621d2d [Wes McKinney] Fix variable name
dc86821 [Wes McKinney] Fixes for integration tests Travis script
5e2c226 [Wes McKinney] Change file mode
ed4be57 [Wes McKinney] Travis CI script cleanup, add C++ toolchain env with flatbuffers, rapidjson
---
 .travis.yml                       |  1 +
 ci/travis_before_script_c_glib.sh |  4 ++--
 ci/travis_before_script_cpp.sh    | 19 ++++++++++---------
 ci/travis_env_common.sh           | 31 +++++++++++++++++++++++++++++++
 ci/travis_install_conda.sh        |  3 +--
 ci/travis_script_integration.sh   | 15 ++++-----------
 ci/travis_script_python.sh        | 14 ++++----------
 7 files changed, 53 insertions(+), 34 deletions(-)
 create mode 100755 ci/travis_env_common.sh

diff --git a/.travis.yml b/.travis.yml
index 4a49c717bf75d..824f62bccaab9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,6 +48,7 @@ matrix:
   - compiler: clang
     osx_image: xcode6.4
     os: osx
+    cache:
     addons:
     before_script:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 1a828e7659bd9..74bdd94b96a8b 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -15,14 +15,14 @@
 
 set -ex
 
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
 if [ $TRAVIS_OS_NAME == "osx" ]; then
     brew install gtk-doc autoconf-archive gobject-introspection
 fi
 
 gem install gobject-introspection
 
-ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
-
 pushd $ARROW_C_GLIB_DIR
 
 : ${ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install}
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index f804a38e76484..3f9f67c359289 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -15,19 +15,20 @@
 
 set -ex
 
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
+# Set up C++ toolchain from conda-forge packages for faster builds
+conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
   brew update > /dev/null
   brew install jemalloc
+  brew install ccache
 fi
 
-mkdir $CPP_BUILD_DIR
-pushd $CPP_BUILD_DIR
-
-CPP_DIR=$TRAVIS_BUILD_DIR/cpp
-
-: ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
+mkdir $ARROW_CPP_BUILD_DIR
+pushd $ARROW_CPP_BUILD_DIR
 
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
@@ -37,11 +38,11 @@ if [ $TRAVIS_OS_NAME == "linux" ]; then
     cmake -DARROW_TEST_MEMCHECK=on \
           $CMAKE_COMMON_FLAGS \
           -DARROW_CXXFLAGS="-Wconversion -Werror" \
-          $CPP_DIR
+          $ARROW_CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
           -DARROW_CXXFLAGS=-Werror \
-          $CPP_DIR
+          $ARROW_CPP_DIR
 fi
 
 make -j4
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
new file mode 100755
index 0000000000000..5593f0079f411
--- /dev/null
+++ b/ci/travis_env_common.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+export MINICONDA=$HOME/miniconda
+export PATH="$MINICONDA/bin:$PATH"
+export CONDA_PKGS_DIRS=$HOME/.conda_packages
+
+# C++ toolchain
+export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
+export RAPIDJSON_HOME=$CPP_TOOLCHAIN
+
+export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python
+export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
+export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
+
+export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
+export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index c036e925427a9..7d185ee82275b 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -22,8 +22,7 @@ fi
 
 wget -O miniconda.sh $MINICONDA_URL
 
-export MINICONDA=$HOME/miniconda
-export CONDA_PKGS_DIRS=$HOME/.conda_packages
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 mkdir -p $CONDA_PKGS_DIRS
 
 bash miniconda.sh -b -p $MINICONDA
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 8ddd89b1639b0..56f5ab7d2d35e 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -14,23 +14,16 @@
 
 set -e
 
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-JAVA_DIR=${TRAVIS_BUILD_DIR}/java
-
-pushd $JAVA_DIR
+pushd $ARROW_JAVA_DIR
 
 mvn package
 
 popd
 
-pushd $TRAVIS_BUILD_DIR/integration
-
-export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug
-
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-export MINICONDA=$HOME/miniconda
-export PATH="$MINICONDA/bin:$PATH"
+pushd $ARROW_INTEGRATION_DIR
+export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug
 
 CONDA_ENV_NAME=arrow-integration-test
 conda create -y -q -n $CONDA_ENV_NAME python=3.5
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 549fe1141cfb1..bde1fd7e249ec 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -14,17 +14,11 @@
 
 set -e
 
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-
-PYTHON_DIR=$TRAVIS_BUILD_DIR/python
-
-# Re-use conda installation from C++
-export MINICONDA=$HOME/miniconda
-export PATH="$MINICONDA/bin:$PATH"
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 export ARROW_HOME=$ARROW_CPP_INSTALL
 
-pushd $PYTHON_DIR
+pushd $ARROW_PYTHON_DIR
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 
 build_parquet_cpp() {
@@ -101,10 +95,10 @@ python_version_tests() {
   which python
 
   # faster builds, please
-  conda install -y nomkl
+  conda install -y -q nomkl
 
   # Expensive dependencies install from Continuum package repo
-  conda install -y pip numpy pandas cython
+  conda install -y -q pip numpy pandas cython
 
   # Build C++ libraries
   build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME

From b4892fd9fb676a678a966da51407b3ce4ba3ec65 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 14 Apr 2017 12:15:57 -0400
Subject: [PATCH 0511/1644] ARROW-528: [Python] Utilize improved Parquet writer
 C++ API, add write_metadata function, test _metadata files

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #539 from wesm/ARROW-528 and squashes the following commits:

848ff93 [Wes McKinney] Add test for _metadata file
8b8f333 [Wes McKinney] Refactor to use APIs introduced in PARQUET-953. Add write_metadata function
---
 python/pyarrow/_parquet.pxd          | 16 ++++++---
 python/pyarrow/_parquet.pyx          | 52 ++++++++++++++++------------
 python/pyarrow/parquet.py            | 34 +++++++++++++++---
 python/pyarrow/tests/test_parquet.py | 24 +++++++++++++
 4 files changed, 94 insertions(+), 32 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 1ac1f69b033ce..9f6edc0b31dc6 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -235,8 +235,14 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
 
 
 cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
-    cdef CStatus WriteTable(
-        const CTable& table, CMemoryPool* pool,
-        const shared_ptr[OutputStream]& sink,
-        int64_t chunk_size,
-        const shared_ptr[WriterProperties]& properties)
+    cdef cppclass FileWriter:
+
+        @staticmethod
+        CStatus Open(const CSchema& schema, CMemoryPool* pool,
+                     const shared_ptr[OutputStream]& sink,
+                     const shared_ptr[WriterProperties]& properties,
+                     unique_ptr[FileWriter]* writer)
+
+        CStatus WriteTable(const CTable& table, int64_t chunk_size)
+        CStatus NewRowGroup(int64_t chunk_size)
+        CStatus Close()
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 5418e1dc82730..b7358a6a47386 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -23,7 +23,7 @@ from cython.operator cimport dereference as deref
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
-from pyarrow._array cimport Array
+from pyarrow._array cimport Array, Schema
 from pyarrow._error cimport check_status
 from pyarrow._memory cimport MemoryPool, maybe_unbox_memory_pool
 from pyarrow._table cimport Table, table_from_ctable
@@ -108,7 +108,7 @@ cdef class FileMetaData:
         if self._schema is not None:
             return self._schema
 
-        cdef Schema schema = Schema()
+        cdef ParquetSchema schema = ParquetSchema()
         schema.init_from_filemeta(self)
         self._schema = schema
         return schema
@@ -160,7 +160,7 @@ cdef class FileMetaData:
         return result
 
 
-cdef class Schema:
+cdef class ParquetSchema:
     cdef:
         object parent  # the FileMetaData owning the SchemaDescriptor
         const SchemaDescriptor* schema
@@ -194,7 +194,7 @@ cdef class Schema:
     def __getitem__(self, i):
         return self.column(i)
 
-    def equals(self, Schema other):
+    def equals(self, ParquetSchema other):
         """
         Returns True if the Parquet schemas are equal
         """
@@ -217,7 +217,7 @@ cdef class ColumnSchema:
     def __cinit__(self):
         self.descr = NULL
 
-    cdef init_from_schema(self, Schema schema, int i):
+    cdef init_from_schema(self, ParquetSchema schema, int i):
         self.parent = schema
         self.descr = schema.schema.Column(i)
 
@@ -373,7 +373,8 @@ cdef class ParquetReader:
         if self._metadata is not None:
             return self._metadata
 
-        metadata = self.reader.get().parquet_reader().metadata()
+        with nogil:
+            metadata = self.reader.get().parquet_reader().metadata()
 
         self._metadata = result = FileMetaData()
         result.init(metadata)
@@ -487,9 +488,7 @@ cdef ParquetCompression compression_from_name(object name):
 
 cdef class ParquetWriter:
     cdef:
-        shared_ptr[WriterProperties] properties
-        shared_ptr[OutputStream] sink
-        CMemoryPool* allocator
+        unique_ptr[FileWriter] writer
 
     cdef readonly:
         object use_dictionary
@@ -497,28 +496,34 @@ cdef class ParquetWriter:
         object version
         int row_group_size
 
-    def __cinit__(self, where, use_dictionary=None, compression=None,
-                  version=None, MemoryPool memory_pool=None):
-        cdef shared_ptr[FileOutputStream] filestream
+    def __cinit__(self, where, Schema schema, use_dictionary=None,
+                  compression=None, version=None,
+                  MemoryPool memory_pool=None):
+        cdef:
+            shared_ptr[FileOutputStream] filestream
+            shared_ptr[OutputStream] sink
+            shared_ptr[WriterProperties] properties
 
         if isinstance(where, six.string_types):
             check_status(FileOutputStream.Open(tobytes(where), &filestream))
-            self.sink = <shared_ptr[OutputStream]> filestream
+            sink = <shared_ptr[OutputStream]> filestream
         else:
-            get_writer(where, &self.sink)
-        self.allocator = maybe_unbox_memory_pool(memory_pool)
+            get_writer(where, &sink)
 
         self.use_dictionary = use_dictionary
         self.compression = compression
         self.version = version
-        self._setup_properties()
 
-    cdef _setup_properties(self):
         cdef WriterProperties.Builder properties_builder
         self._set_version(&properties_builder)
         self._set_compression_props(&properties_builder)
         self._set_dictionary_props(&properties_builder)
-        self.properties = properties_builder.build()
+        properties = properties_builder.build()
+
+        check_status(
+            FileWriter.Open(deref(schema.schema),
+                            maybe_unbox_memory_pool(memory_pool),
+                            sink, properties, &self.writer))
 
     cdef _set_version(self, WriterProperties.Builder* props):
         if self.version is not None:
@@ -546,12 +551,16 @@ cdef class ParquetWriter:
                 props.enable_dictionary()
             else:
                 props.disable_dictionary()
-        else:
+        elif self.use_dictionary is not None:
             # Deactivate dictionary encoding by default
             props.disable_dictionary()
             for column in self.use_dictionary:
                 props.enable_dictionary(column)
 
+    def close(self):
+        with nogil:
+            check_status(self.writer.get().Close())
+
     def write_table(self, Table table, row_group_size=None):
         cdef CTable* ctable = table.table
 
@@ -563,6 +572,5 @@ cdef class ParquetWriter:
         cdef int c_row_group_size = row_group_size
 
         with nogil:
-            check_status(WriteTable(deref(ctable), self.allocator,
-                                    self.sink, c_row_group_size,
-                                    self.properties))
+            check_status(self.writer.get()
+                         .WriteTable(deref(ctable), c_row_group_size))
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index aaec43ab06027..4ff7e038b5e6c 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -21,7 +21,8 @@
 
 from pyarrow.filesystem import LocalFilesystem
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
-                              RowGroupMetaData, Schema, ParquetWriter)
+                              RowGroupMetaData, ParquetSchema,
+                              ParquetWriter)
 import pyarrow._parquet as _parquet  # noqa
 import pyarrow._array as _array
 import pyarrow._table as _table
@@ -471,7 +472,8 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
         else:
             self.fs = filesystem
 
-        self.pieces, self.partitions = _make_manifest(path_or_paths, self.fs)
+        (self.pieces, self.partitions,
+         self.metadata_path) = _make_manifest(path_or_paths, self.fs)
 
         self.metadata = metadata
         self.schema = schema
@@ -488,7 +490,10 @@ def validate_schemas(self):
         open_file = self._get_open_file_func()
 
         if self.metadata is None and self.schema is None:
-            self.schema = self.pieces[0].get_metadata(open_file).schema
+            if self.metadata_path is not None:
+                self.schema = open_file(self.metadata_path).schema
+            else:
+                self.schema = self.pieces[0].get_metadata(open_file).schema
         elif self.schema is None:
             self.schema = self.metadata.schema
 
@@ -543,10 +548,12 @@ def open_file(path, meta=None):
 
 def _make_manifest(path_or_paths, fs, pathsep='/'):
     partitions = None
+    metadata_path = None
 
     if is_string(path_or_paths) and fs.isdir(path_or_paths):
         manifest = ParquetManifest(path_or_paths, filesystem=fs,
                                    pathsep=pathsep)
+        metadata_path = manifest.metadata_path
         pieces = manifest.pieces
         partitions = manifest.partitions
     else:
@@ -565,7 +572,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
             piece = ParquetDatasetPiece(path)
             pieces.append(piece)
 
-    return pieces, partitions
+    return pieces, partitions, metadata_path
 
 
 def read_table(source, columns=None, nthreads=1, metadata=None):
@@ -622,7 +629,24 @@ def write_table(table, where, row_group_size=None, version='1.0',
         Specify the compression codec, either on a general basis or per-column.
     """
     row_group_size = kwargs.get('chunk_size', row_group_size)
-    writer = ParquetWriter(where, use_dictionary=use_dictionary,
+    writer = ParquetWriter(where, table.schema,
+                           use_dictionary=use_dictionary,
                            compression=compression,
                            version=version)
     writer.write_table(table, row_group_size=row_group_size)
+    writer.close()
+
+
+def write_metadata(schema, where, version='1.0'):
+    """
+    Write metadata-only Parquet file from schema
+
+    Parameters
+    ----------
+    schema : pyarrow.Schema
+    where: string or pyarrow.io.NativeFile
+    version : {"1.0", "2.0"}, default "1.0"
+        The Parquet format version, defaults to 1.0
+    """
+    writer = ParquetWriter(where, schema, version=version)
+    writer.close()
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index a5c70aa16225f..ca6ae2d0b3be0 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -529,6 +529,30 @@ def _visit_level(base_dir, level, part_keys):
     _visit_level(base_dir, 0, [])
 
 
+@parquet
+def test_read_common_metadata_files(tmpdir):
+    N = 100
+    df = pd.DataFrame({
+        'index': np.arange(N),
+        'values': np.random.randn(N)
+    }, columns=['index', 'values'])
+
+    base_path = str(tmpdir)
+    data_path = pjoin(base_path, 'data.parquet')
+
+    table = pa.Table.from_pandas(df)
+    pq.write_table(table, data_path)
+
+    metadata_path = pjoin(base_path, '_metadata')
+    pq.write_metadata(table.schema, metadata_path)
+
+    dataset = pq.ParquetDataset(base_path)
+    assert dataset.metadata_path == metadata_path
+
+    pf = pq.ParquetFile(data_path)
+    assert dataset.schema.equals(pf.schema)
+
+
 def _filter_partition(df, part_keys):
     predicate = np.ones(len(df), dtype=bool)
 

From 01114d831b1cd0cdb9a7f28958d181dcece2537f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 14 Apr 2017 16:20:06 -0400
Subject: [PATCH 0512/1644] ARROW-783: [Java/C++] Fixes for 0-length record
 batches

@StevenMPhillips @nongli @julienledem I found a number of issues in both C++ and Java around the handling of 0-length vectors. It seems that preserving a single inconsequential offset for a length-0 variable length vector can be a bit difficult, so I relaxed a restruction in `loadFieldVectors` about this. Let me know if there's anything concerning about the other changes around EOS signaling

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #505 from wesm/ARROW-783 and squashes the following commits:

28ddcab [Wes McKinney] * Have loadNextBatch return true/false for EOS to accommodate 0-length record batches * Relax n + 1 restruction for 0-length vectors
---
 cpp/src/arrow/loader.cc                       | 16 +++---------
 integration/integration_test.py               |  8 +++---
 .../org/apache/arrow/tools/FileRoundtrip.java |  4 +--
 .../org/apache/arrow/tools/FileToStream.java  | 10 +++++---
 .../org/apache/arrow/tools/Integration.java   | 17 ++++++++-----
 .../org/apache/arrow/tools/StreamToFile.java  | 10 +++++---
 .../arrow/tools/ArrowFileTestFixtures.java    |  4 ++-
 .../apache/arrow/tools/EchoServerTest.java    |  4 +--
 .../templates/NullableValueVectors.java       |  4 ++-
 .../arrow/vector/file/ArrowFileReader.java    |  4 +--
 .../apache/arrow/vector/file/ArrowReader.java | 14 +++++++++--
 .../vector/file/json/JsonFileReader.java      |  4 ++-
 .../arrow/vector/file/TestArrowFile.java      | 25 +++++++++----------
 .../arrow/vector/file/TestArrowStream.java    | 12 +++++----
 .../vector/file/TestArrowStreamPipe.java      |  9 ++++---
 15 files changed, 82 insertions(+), 63 deletions(-)

diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
index f9f6e6fcac826..e4e1ba42ff600 100644
--- a/cpp/src/arrow/loader.cc
+++ b/cpp/src/arrow/loader.cc
@@ -97,13 +97,8 @@ class ArrayLoader {
     std::shared_ptr<Buffer> null_bitmap, offsets, values;
 
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    if (field_meta.length > 0) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
-    } else {
-      context_->buffer_index += 2;
-      offsets = values = nullptr;
-    }
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
 
     result_ = std::make_shared<CONTAINER>(
         field_meta.length, offsets, values, null_bitmap, field_meta.null_count);
@@ -166,12 +161,7 @@ class ArrayLoader {
     std::shared_ptr<Buffer> null_bitmap, offsets;
 
     RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    if (field_meta.length > 0) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &offsets));
-    } else {
-      offsets = nullptr;
-    }
-    ++context_->buffer_index;
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
 
     const int num_children = type.num_children();
     if (num_children != 1) {
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 6631dc8c2f761..661f5c97770d9 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -593,7 +593,7 @@ def _generate_file(fields, batch_sizes):
     return JSONFile(schema, batches)
 
 
-def generate_primitive_case():
+def generate_primitive_case(batch_sizes):
     types = ['bool', 'int8', 'int16', 'int32', 'int64',
              'uint8', 'uint16', 'uint32', 'uint64',
              'float32', 'float64', 'binary', 'utf8']
@@ -604,7 +604,6 @@ def generate_primitive_case():
         fields.append(get_field(type_ + "_nullable", type_, True))
         fields.append(get_field(type_ + "_nonnullable", type_, False))
 
-    batch_sizes = [7, 10]
     return _generate_file(fields, batch_sizes)
 
 
@@ -648,9 +647,8 @@ def _temp_path():
         return
 
     file_objs = [
-        generate_primitive_case(),
-        generate_primitive_case(),
-        generate_primitive_case(),
+        generate_primitive_case([7, 10]),
+        generate_primitive_case([0, 0, 0]),
         generate_datetime_case(),
         generate_nested_case()
     ]
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
index b8621920d3348..135d4921ed128 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
@@ -93,9 +93,7 @@ int run(String[] args) {
                  fileOutputStream.getChannel())) {
           arrowWriter.start();
           while (true) {
-            arrowReader.loadNextBatch();
-            int loaded = root.getRowCount();
-            if (loaded == 0) {
+            if (!arrowReader.loadNextBatch()) {
               break;
             } else {
               arrowWriter.writeBatch();
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
index be404fd4c5950..6722b30fa7f50 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
@@ -41,12 +41,16 @@ public static void convert(FileInputStream in, OutputStream out) throws IOExcept
     try (ArrowFileReader reader = new ArrowFileReader(in.getChannel(), allocator)) {
       VectorSchemaRoot root = reader.getVectorSchemaRoot();
       // load the first batch before instantiating the writer so that we have any dictionaries
-      reader.loadNextBatch();
+      if (!reader.loadNextBatch()) {
+        throw new IOException("Unable to read first record batch");
+      }
       try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, out)) {
         writer.start();
-        while (root.getRowCount() > 0) {
+        while (true) {
           writer.writeBatch();
-          reader.loadNextBatch();
+          if (!reader.loadNextBatch()) {
+            break;
+          }
         }
         writer.end();
       }
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index 453693d7fa489..e8266d50786d3 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -126,7 +126,9 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
               .pretty(true))) {
             writer.start(schema);
             for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
-              arrowReader.loadRecordBatch(rbBlock);
+              if (!arrowReader.loadRecordBatch(rbBlock)) {
+                throw new IOException("Expected to load record batch");
+              }
               writer.write(root);
             }
           }
@@ -148,10 +150,8 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
                ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream
                    .getChannel())) {
             arrowWriter.start();
-            reader.read(root);
-            while (root.getRowCount() != 0) {
+            while (reader.read(root)) {
               arrowWriter.writeBatch();
-              reader.read(root);
             }
             arrowWriter.end();
           }
@@ -179,16 +179,21 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
           List<ArrowBlock> recordBatches = arrowReader.getRecordBlocks();
           Iterator<ArrowBlock> iterator = recordBatches.iterator();
           VectorSchemaRoot jsonRoot;
+          int totalBatches = 0;
           while ((jsonRoot = jsonReader.read()) != null && iterator.hasNext()) {
             ArrowBlock rbBlock = iterator.next();
-            arrowReader.loadRecordBatch(rbBlock);
+            if (!arrowReader.loadRecordBatch(rbBlock)) {
+              throw new IOException("Expected to load record batch");
+            }
             Validator.compareVectorSchemaRoot(arrowRoot, jsonRoot);
             jsonRoot.close();
+            totalBatches++;
           }
           boolean hasMoreJSON = jsonRoot != null;
           boolean hasMoreArrow = iterator.hasNext();
           if (hasMoreJSON || hasMoreArrow) {
-            throw new IllegalArgumentException("Unexpected RecordBatches. J:" + hasMoreJSON + " "
+            throw new IllegalArgumentException("Unexpected RecordBatches. Total: " + totalBatches
+                                               + " J:" + hasMoreJSON + " "
                 + "A:" + hasMoreArrow);
           }
         }
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
index 41dfd347be579..ef1a11f6bfac8 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
@@ -41,12 +41,16 @@ public static void convert(InputStream in, OutputStream out) throws IOException
     try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) {
       VectorSchemaRoot root = reader.getVectorSchemaRoot();
       // load the first batch before instantiating the writer so that we have any dictionaries
-      reader.loadNextBatch();
+      if (!reader.loadNextBatch()) {
+        throw new IOException("Unable to read first record batch");
+      }
       try (ArrowFileWriter writer = new ArrowFileWriter(root, reader, Channels.newChannel(out))) {
         writer.start();
-        while (root.getRowCount() > 0) {
+        while (true) {
           writer.writeBatch();
-          reader.loadNextBatch();
+          if (!reader.loadNextBatch()) {
+            break;
+          }
         }
         writer.end();
       }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
index 1a389098b4f47..34c93ed232c80 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -67,7 +67,9 @@ static void validateOutput(File testOutFile, BufferAllocator allocator) throws E
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
-        arrowReader.loadRecordBatch(rbBlock);
+        if (!arrowReader.loadRecordBatch(rbBlock)) {
+          throw new IOException("Expected to read record batch");
+        }
         validateContent(COUNT, root);
       }
     }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 7d07588892cf9..7cca33955d93a 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -118,7 +118,7 @@ private void testEchoServer(int serverPort,
       NullableTinyIntVector readVector = (NullableTinyIntVector) reader.getVectorSchemaRoot()
           .getFieldVectors().get(0);
       for (int i = 0; i < batches; i++) {
-        reader.loadNextBatch();
+        Assert.assertTrue(reader.loadNextBatch());
         assertEquals(16, reader.getVectorSchemaRoot().getRowCount());
         assertEquals(16, readVector.getAccessor().getValueCount());
         for (int j = 0; j < 8; j++) {
@@ -126,7 +126,7 @@ private void testEchoServer(int serverPort,
           assertTrue(readVector.getAccessor().isNull(j + 8));
         }
       }
-      reader.loadNextBatch();
+      Assert.assertFalse(reader.loadNextBatch());
       assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
       assertEquals(reader.bytesRead(), writer.bytesWritten());
     }
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index a50771a45a034..e5257ce554e3b 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -122,7 +122,9 @@ public List<FieldVector> getChildrenFromFields() {
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
     <#if type.major = "VarLen">
     // variable width values: truncate offset vector buffer to size (#1)
-    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.offsetVector.getBufferSizeFor(fieldNode.getLength() + 1));
+    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1,
+        values.offsetVector.getBufferSizeFor(
+        fieldNode.getLength() == 0? 0 : fieldNode.getLength() + 1));
     <#else>
     // fixed width values truncate value vector to size (#1)
     org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.getBufferSizeFor(fieldNode.getLength()));
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
index 28440a190ad43..f4d6ada932494 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
@@ -103,14 +103,14 @@ public List<ArrowBlock> getRecordBlocks() throws IOException {
     return footer.getRecordBatches();
   }
 
-  public void loadRecordBatch(ArrowBlock block) throws IOException {
+  public boolean loadRecordBatch(ArrowBlock block) throws IOException {
     ensureInitialized();
     int blockIndex = footer.getRecordBatches().indexOf(block);
     if (blockIndex == -1) {
       throw new IllegalArgumentException("Arrow bock does not exist in record batches: " + block);
     }
     currentRecordBatch = blockIndex;
-    loadNextBatch();
+    return loadNextBatch();
   }
 
   private ArrowDictionaryBatch readDictionaryBatch(SeekableReadChannel in,
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index 1646fbe803687..1d33913f71a95 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -89,7 +89,8 @@ public Dictionary lookup(long id) {
     }
   }
 
-  public void loadNextBatch() throws IOException {
+  // Returns true if a batch was read, false on EOS
+  public boolean loadNextBatch() throws IOException {
     ensureInitialized();
     // read in all dictionary batches, then stop after our first record batch
     ArrowMessageVisitor<Boolean> visitor = new ArrowMessageVisitor<Boolean>() {
@@ -106,9 +107,18 @@ public Boolean visit(ArrowRecordBatch message) {
     };
     root.setRowCount(0);
     ArrowMessage message = readMessage(in, allocator);
-    while (message != null && message.accepts(visitor)) {
+
+    boolean readBatch = false;
+    while (message != null) {
+      if (!message.accepts(visitor)) {
+        readBatch = true;
+        break;
+      }
+      // else read a dictionary
       message = readMessage(in, allocator);
     }
+
+    return readBatch;
   }
 
   public long bytesRead() { return in.bytesRead(); }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index fde9954d288bb..21aa0372c6b38 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -94,7 +94,7 @@ public Schema start() throws JsonParseException, IOException {
     }
   }
 
-  public void read(VectorSchemaRoot root) throws IOException {
+  public boolean read(VectorSchemaRoot root) throws IOException {
     JsonToken t = parser.nextToken();
     if (t == START_OBJECT) {
       {
@@ -111,8 +111,10 @@ public void read(VectorSchemaRoot root) throws IOException {
         readToken(END_ARRAY);
       }
       readToken(END_OBJECT);
+      return true;
     } else if (t == END_ARRAY) {
       root.setRowCount(0);
+      return false;
     } else {
       throw new IllegalArgumentException("Invalid token: " + t);
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index a1104ffe545d8..11730afd55406 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -152,7 +152,7 @@ protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) th
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       Assert.assertEquals(count, root.getRowCount());
       validateContent(count, root);
     }
@@ -193,7 +193,7 @@ public void testWriteReadComplex() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       Assert.assertEquals(count, root.getRowCount());
       validateComplexContent(count, root);
     }
@@ -263,13 +263,12 @@ public void testWriteReadMultipleRBs() throws IOException {
       int i = 0;
 
       for (int n = 0; n < 2; n++) {
-        arrowReader.loadNextBatch();
+        Assert.assertTrue(arrowReader.loadNextBatch());
         Assert.assertEquals("RB #" + i, counts[i], root.getRowCount());
         validateContent(counts[i], root);
         ++i;
       }
-      arrowReader.loadNextBatch();
-      Assert.assertEquals(0, root.getRowCount());
+      Assert.assertFalse(arrowReader.loadNextBatch());
     }
   }
 
@@ -294,7 +293,7 @@ public void testWriteReadUnion() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       validateUnionData(count, root);
     }
 
@@ -305,7 +304,7 @@ public void testWriteReadUnion() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       validateUnionData(count, root);
     }
   }
@@ -347,7 +346,7 @@ public void testWriteReadTiny() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       validateTinyData(root);
     }
 
@@ -358,7 +357,7 @@ public void testWriteReadTiny() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       validateTinyData(root);
     }
   }
@@ -433,7 +432,7 @@ public void testWriteReadDictionary() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       validateFlatDictionary(root.getFieldVectors().get(0), arrowReader);
     }
 
@@ -444,7 +443,7 @@ public void testWriteReadDictionary() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       validateFlatDictionary(root.getFieldVectors().get(0), arrowReader);
     }
   }
@@ -537,7 +536,7 @@ public void testWriteReadNestedDictionary() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       validateNestedDictionary((ListVector) root.getFieldVectors().get(0), arrowReader);
     }
 
@@ -548,7 +547,7 @@ public void testWriteReadNestedDictionary() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      arrowReader.loadNextBatch();
+      Assert.assertTrue(arrowReader.loadNextBatch());
       validateNestedDictionary((ListVector) root.getFieldVectors().get(0), arrowReader);
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
index e7cdf3fea4b8b..7e9afd381c181 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
@@ -19,6 +19,7 @@
 
 import static java.util.Arrays.asList;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayInputStream;
@@ -36,6 +37,7 @@
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
 import org.apache.arrow.vector.stream.MessageSerializerTest;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestArrowStream extends BaseFileTest {
@@ -52,10 +54,10 @@ public void testEmptyStream() throws IOException {
     ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
     try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) {
       assertEquals(schema, reader.getVectorSchemaRoot().getSchema());
-      // Empty should return nothing. Can be called repeatedly.
-      reader.loadNextBatch();
+      // Empty should return false
+      Assert.assertFalse(reader.loadNextBatch());
       assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
-      reader.loadNextBatch();
+      Assert.assertFalse(reader.loadNextBatch());
       assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
     }
   }
@@ -90,11 +92,11 @@ public void testReadWrite() throws IOException {
         Schema readSchema = reader.getVectorSchemaRoot().getSchema();
         assertEquals(schema, readSchema);
         for (int i = 0; i < numBatches; i++) {
-          reader.loadNextBatch();
+          assertTrue(reader.loadNextBatch());
         }
         // TODO figure out why reader isn't getting padding bytes
         assertEquals(bytesWritten, reader.bytesRead() + 4);
-        reader.loadNextBatch();
+        assertFalse(reader.loadNextBatch());
         assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
       }
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
index 46d46794bbefa..20d4482da7c98 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
@@ -105,8 +105,10 @@ protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) th
           return message;
         }
         @Override
-        public void loadNextBatch() throws IOException {
-          super.loadNextBatch();
+        public boolean loadNextBatch() throws IOException {
+          if (!super.loadNextBatch()) {
+            return false;
+          }
           if (!done) {
             VectorSchemaRoot root = getVectorSchemaRoot();
             Assert.assertEquals(16, root.getRowCount());
@@ -120,6 +122,7 @@ public void loadNextBatch() throws IOException {
               }
             }
           }
+          return true;
         }
       };
     }
@@ -132,7 +135,7 @@ public void run() {
             reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectorTypes().toString(),
             reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectors().size() > 0);
         while (!done) {
-          reader.loadNextBatch();
+          assertTrue(reader.loadNextBatch());
         }
       } catch (IOException e) {
         e.printStackTrace();

From b6033378c2533ed7b396f111cc5aed10450907fb Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Fri, 14 Apr 2017 16:37:25 -0400
Subject: [PATCH 0513/1644] ARROW-815 [Java] Exposing reAlloc for ValueVector

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #534 from elahrvivaz/ARROW-815 and squashes the following commits:

cf14944 [Emilio Lahr-Vivaz] unit test
45fa773 [Emilio Lahr-Vivaz] ARROW-815 [Java] Exposing reAlloc for ValueVector
---
 .../templates/NullableValueVectors.java       |   6 +
 .../main/codegen/templates/UnionVector.java   |   6 +
 .../org/apache/arrow/vector/ValueVector.java  |   6 +
 .../org/apache/arrow/vector/ZeroVector.java   |   3 +
 .../vector/complex/AbstractMapVector.java     |   7 +
 .../complex/BaseRepeatedValueVector.java      |  14 +-
 .../arrow/vector/complex/ListVector.java      |   6 +
 .../vector/complex/NullableMapVector.java     |   7 +
 .../arrow/vector/TestVectorReAlloc.java       | 144 ++++++++++++++++++
 9 files changed, 194 insertions(+), 5 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index e5257ce554e3b..acee6cb738d76 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -242,6 +242,12 @@ public boolean allocateNewSafe() {
     return success;
   }
 
+  @Override
+  public void reAlloc() {
+    bits.reAlloc();
+    values.reAlloc();
+  }
+
   <#if type.major == "VarLen">
   @Override
   public void allocateNew(int totalBytes, int valueCount) {
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 797b29342e4c1..d70cbae02bf33 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
     return safe;
   }
 
+  @Override
+  public void reAlloc() {
+    internalMap.reAlloc();
+    typeVector.reAlloc();
+  }
+
   @Override
   public void setInitialCapacity(int numRecords) {
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 8e35398b9394b..685b0be010a08 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
    */
   boolean allocateNewSafe();
 
+  /**
+   * Allocate new buffer with double capacity, and copy data into the new buffer.
+   * Replace vector's buffer with new buffer, and release old one
+   */
+  void reAlloc();
+
   BufferAllocator getAllocator();
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 73f858e4d35a0..e48788c6ae7c0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
     return true;
   }
 
+  @Override
+  public void reAlloc() {}
+
   @Override
   public BufferAllocator getAllocator() {
     throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index dc833edbed8d0..15e8a5bc624ac 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
     return true;
   }
 
+  @Override
+  public void reAlloc() {
+    for (final ValueVector v: vectors.values()) {
+      v.reAlloc();
+    }
+  }
+
   /**
    * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
    * {@link org.apache.arrow.vector.ValueVector}.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 6b240c04f7124..da221e33013d1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
   public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
     return success;
   }
 
+  @Override
+  public void reAlloc() {
+    offsets.reAlloc();
+    vector.reAlloc();
+  }
 
   @Override
   public UInt4Vector getOffsetVector() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 0461a8d9d285a..63235dfda87df 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
     bits.allocateNewSafe();
   }
 
+  @Override
+  public void reAlloc() {
+    super.reAlloc();
+    bits.reAlloc();
+  }
+
   public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
     copyFrom(inIndex, outIndex, from);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 71fee67d49c9f..647ab28352f0d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
     bits.zeroVector();
     return success;
   }
+
+  @Override
+  public void reAlloc() {
+    bits.reAlloc();
+    super.reAlloc();
+  }
+
   public final class Accessor extends MapVector.Accessor  {
     final BitVector.Accessor bAccessor = bits.getAccessor();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
new file mode 100644
index 0000000000000..a7c35b6363cf1
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testFixedType() {
+    try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+      final UInt4Vector.Mutator m = vector.getMutator();
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+
+      assertEquals(512, vector.getValueCapacity());
+
+      try {
+        m.set(512, 0);
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(1024, vector.getValueCapacity());
+
+      m.set(512, 100);
+      assertEquals(100, vector.getAccessor().get(512));
+    }
+  }
+
+  @Test
+  public void testNullableType() {
+    try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+
+      assertEquals(512, vector.getValueCapacity());
+
+      try {
+        m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+      m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+      assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+    }
+  }
+
+  @Test
+  public void testListType() {
+    try (final ListVector vector = new ListVector("", allocator, null)) {
+      vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+
+      assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+      try {
+        vector.getOffsetVector().getAccessor().get(2014);
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+      assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+    }
+  }
+
+  @Test
+  public void testMapType() {
+    try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+      vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+
+      assertEquals(512, vector.getValueCapacity());
+
+      try {
+        vector.getAccessor().getObject(513);
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(1024, vector.getValueCapacity());
+      assertNull(vector.getAccessor().getObject(513));
+    }
+  }
+}

From 794d0204cb33bc98bce418785b4643ee4f1083d8 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Fri, 14 Apr 2017 20:14:02 -0400
Subject: [PATCH 0514/1644] ARROW-777: restore getObject behavior on Date and
 Time

Author: Julien Le Dem <julien@apache.org>

Closes #542 from julienledem/ARROW-777 and squashes the following commits:

c77f5a0 [Julien Le Dem] ARROW-777: restore getObject behavior on Date and Time
---
 .../src/main/codegen/data/ValueVectorTypes.tdd |  4 ++--
 .../codegen/templates/FixedValueVectors.java   | 18 ++++++++++++++++--
 .../apache/arrow/vector/file/BaseFileTest.java |  6 ++++--
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index b08c100edcac8..d472b559347f0 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -59,7 +59,7 @@
         { class: "DateDay" },
         { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
         { class: "TimeSec" },
-        { class: "TimeMilli" }
+        { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
       ]
     },
     {
@@ -72,7 +72,7 @@
         { class: "BigInt"},
         { class: "UInt8" },
         { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
-        { class: "DateMilli" },
+        { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
         { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
         { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
         { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 947c82c74a401..5c09e30c71487 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -484,9 +484,7 @@ public long getTwoAsLong(int index) {
     </#if>
 
     <#if minor.class == "DateDay" ||
-         minor.class == "DateMilli" ||
          minor.class == "TimeSec" ||
-         minor.class == "TimeMilli" ||
          minor.class == "TimeMicro" ||
          minor.class == "TimeNano">
     @Override
@@ -494,6 +492,22 @@ public long getTwoAsLong(int index) {
       return get(index);
     }
 
+    <#elseif minor.class == "DateMilli">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      return date;
+    }
+
+    <#elseif minor.class == "TimeMilli">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+      time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      return time;
+    }
+
     <#elseif minor.class == "TimeStampSec">
     @Override
     public ${friendlyType} getObject(int index) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 5c68a1904be70..5ca083aa2dfab 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -22,6 +22,8 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.NullableDateMilliVector;
+import org.apache.arrow.vector.NullableTimeMilliVector;
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.MapVector;
@@ -173,11 +175,11 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
     Assert.assertEquals(count, root.getRowCount());
     printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
-      Object dateVal = root.getVector("date").getAccessor().getObject(i);
+      long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i);
       DateTime dt = makeDateTimeFromCount(i);
       DateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
       Assert.assertEquals(dateExpected.getMillis(), dateVal);
-      Object timeVal = root.getVector("time").getAccessor().getObject(i);
+      long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i);
       Assert.assertEquals(dt.getMillisOfDay(), timeVal);
       Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
       Assert.assertTrue(dt.withZoneRetainFields(DateTimeZone.getDefault()).equals(timestampMilliVal));

From 88c351abc24179ae1b1fa76450c2c8a4d6e4f04e Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Fri, 14 Apr 2017 20:14:36 -0400
Subject: [PATCH 0515/1644] =?UTF-8?q?ARROW-720:=20arrow=20should=20not=20h?=
 =?UTF-8?q?ave=20a=20dependency=20on=20slf4j=20bridges=20in=20com=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…pile scope

Author: Julien Le Dem <julien@apache.org>

Closes #541 from julienledem/ARROW-720 and squashes the following commits:

fa63e20 [Julien Le Dem] ARROW-720: arrow should not have a dependency on slf4j bridges in compile scope
---
 java/pom.xml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/java/pom.xml b/java/pom.xml
index 5edd605e8eedb..5d07186e3e714 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -469,25 +469,28 @@
       <version>${dep.slf4j.version}</version>
     </dependency>
 
+    <!-- Test Dependencies -->
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>jul-to-slf4j</artifactId>
       <version>${dep.slf4j.version}</version>
+      <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>jcl-over-slf4j</artifactId>
       <version>${dep.slf4j.version}</version>
+      <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>log4j-over-slf4j</artifactId>
       <version>${dep.slf4j.version}</version>
+      <scope>test</scope>
     </dependency>
 
-    <!-- Test Dependencies -->
     <dependency>
       <!-- JMockit needs to be on class path before JUnit. -->
       <groupId>com.googlecode.jmockit</groupId>

From 4b030dd0ea193eeb60644518f897ec966eb6b720 Mon Sep 17 00:00:00 2001
From: Jeff Knupp <jeff@jeffknupp.com>
Date: Sat, 15 Apr 2017 11:09:51 +0200
Subject: [PATCH 0516/1644] ARROW-828: [C++] Add new dependency to README

`libboost-regex-dev` is required to build on Ubuntu; added to `apt` install command.

Author: Jeff Knupp <jeff@jeffknupp.com>

Closes #545 from jeffknupp/master and squashes the following commits:

b527ebb [Jeff Knupp] Add new dependency to README
---
 cpp/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cpp/README.md b/cpp/README.md
index b19fa001198a4..339b6b47533cb 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -31,6 +31,7 @@ On Ubuntu/Debian you can install the requirements with:
 sudo apt-get install cmake \
      libboost-dev \
      libboost-filesystem-dev \
+     libboost-regex-dev \
      libboost-system-dev
 ```
 
@@ -126,4 +127,4 @@ both of these options would be used rarely.  Current known uses-cases whent hey
 *  Parameterized tests in google test.
 
 [1]: https://brew.sh/
-[2]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
\ No newline at end of file
+[2]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md

From ce5b98e1d8254219419220c42e45959ca1aeac21 Mon Sep 17 00:00:00 2001
From: Deepak Majeti <deepak.majeti@hpe.com>
Date: Sat, 15 Apr 2017 11:27:46 +0200
Subject: [PATCH 0517/1644] =?UTF-8?q?ARROW-820:=20[C++]=20Build=20dependen?=
 =?UTF-8?q?cies=20for=20Parquet=20library=20without=20arrow=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… support

Author: Deepak Majeti <deepak.majeti@hpe.com>

Closes #538 from majetideepak/ARROW-820 and squashes the following commits:

10ca617 [Deepak Majeti] Revert HDFS change
f399ab5 [Deepak Majeti] Add flags for ARROW_IPC and ARROW_HDFS
add683a [Deepak Majeti] ARROW-820: [C++] Build dependencies for Parquet library without arrow support
---
 cpp/CMakeLists.txt | 107 ++++++++++++++++++++++++---------------------
 1 file changed, 57 insertions(+), 50 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0e4a4bbf34b67..83610d33e6af1 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -621,27 +621,49 @@ if(ARROW_BUILD_BENCHMARKS)
   endif()
 endif()
 
-# RapidJSON, header only dependency
-if("$ENV{RAPIDJSON_HOME}" STREQUAL "")
-  ExternalProject_Add(rapidjson_ep
-    PREFIX "${CMAKE_BINARY_DIR}"
-    URL "https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz"
-    URL_MD5 "badd12c511e081fec6c89c43a7027bce"
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND ""
-    BUILD_IN_SOURCE 1
-    INSTALL_COMMAND "")
-
-  ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR)
-  set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include")
-  set(RAPIDJSON_VENDORED 1)
-else()
-  set(RAPIDJSON_INCLUDE_DIR "$ENV{RAPIDJSON_HOME}/include")
-  set(RAPIDJSON_VENDORED 0)
-endif()
-message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
-include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
+if (ARROW_IPC)
+  # RapidJSON, header only dependency
+  if("$ENV{RAPIDJSON_HOME}" STREQUAL "")
+    ExternalProject_Add(rapidjson_ep
+      PREFIX "${CMAKE_BINARY_DIR}"
+      URL "https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz"
+      URL_MD5 "badd12c511e081fec6c89c43a7027bce"
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND ""
+      BUILD_IN_SOURCE 1
+      INSTALL_COMMAND "")
+
+    ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR)
+    set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include")
+    set(RAPIDJSON_VENDORED 1)
+  else()
+    set(RAPIDJSON_INCLUDE_DIR "$ENV{RAPIDJSON_HOME}/include")
+    set(RAPIDJSON_VENDORED 0)
+  endif()
+  message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
+  include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
+
+  ## Flatbuffers
+  if("$ENV{FLATBUFFERS_HOME}" STREQUAL "")
+    set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
+    ExternalProject_Add(flatbuffers_ep
+      URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
+      CMAKE_ARGS
+      "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
+      "-DFLATBUFFERS_BUILD_TESTS=OFF")
+
+    set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
+    set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
+    set(FLATBUFFERS_VENDORED 1)
+  else()
+    find_package(Flatbuffers REQUIRED)
+    set(FLATBUFFERS_VENDORED 0)
+  endif()
 
+  message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
+  message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
+  include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
+endif()
 #----------------------------------------------------------------------
 
 if (MSVC)
@@ -705,28 +727,6 @@ endif()
 #   set(ARROW_TCMALLOC_AVAILABLE 1)
 # endif()
 
-## Flatbuffers
-
-if("$ENV{FLATBUFFERS_HOME}" STREQUAL "")
-  set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
-  ExternalProject_Add(flatbuffers_ep
-    URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
-    CMAKE_ARGS
-    "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
-    "-DFLATBUFFERS_BUILD_TESTS=OFF")
-
-  set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
-  set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
-  set(FLATBUFFERS_VENDORED 1)
-else()
-  find_package(Flatbuffers REQUIRED)
-  set(FLATBUFFERS_VENDORED 0)
-endif()
-
-message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
-message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
-include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
-
 ########################################################################
 # HDFS thirdparty setup
 
@@ -885,7 +885,9 @@ endif()
 
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
-add_subdirectory(src/arrow/ipc)
+if (ARROW_IPC)
+  add_subdirectory(src/arrow/ipc)
+endif()
 
 set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} metadata_fbs)
 
@@ -909,17 +911,22 @@ set(ARROW_SRCS
   src/arrow/io/interfaces.cc
   src/arrow/io/memory.cc
 
-  src/arrow/ipc/feather.cc
-  src/arrow/ipc/json.cc
-  src/arrow/ipc/json-internal.cc
-  src/arrow/ipc/metadata.cc
-  src/arrow/ipc/reader.cc
-  src/arrow/ipc/writer.cc
-
   src/arrow/util/bit-util.cc
   src/arrow/util/decimal.cc
 )
 
+if (ARROW_IPC)
+  set(ARROW_SRCS ${ARROW_SRCS}
+    src/arrow/ipc/feather.cc
+    src/arrow/ipc/json.cc
+    src/arrow/ipc/json-internal.cc
+    src/arrow/ipc/metadata.cc
+    src/arrow/ipc/reader.cc
+    src/arrow/ipc/writer.cc
+  )
+endif()
+
+
 if(NOT APPLE AND NOT MSVC)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the

From 4d2ac871c9126ba431ebb193ea19bd5eb7ef8ab3 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 15 Apr 2017 09:35:41 -0400
Subject: [PATCH 0518/1644] ARROW-826: [C++/Python] Fix compilation error on
 Mac with -DARROW_PYTHON=on

This fixes https://github.com/ray-project/ray/issues/461

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #544 from pcmoritz/fix-python-macos and squashes the following commits:

cf59732 [Philipp Moritz] include <iostream> before <Python.h>
---
 cpp/src/arrow/python/config.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cpp/src/arrow/python/config.h b/cpp/src/arrow/python/config.h
index dd554e05b9379..c13272667540a 100644
--- a/cpp/src/arrow/python/config.h
+++ b/cpp/src/arrow/python/config.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_PYTHON_CONFIG_H
 #define ARROW_PYTHON_CONFIG_H
 
+#include <iostream>
 #include <Python.h>
 
 #include "arrow/python/numpy_interop.h"

From edb8252c7534b787cb4dc0234080765e9bd6a045 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 15 Apr 2017 12:43:20 -0400
Subject: [PATCH 0519/1644] =?UTF-8?q?ARROW-829:=20Don't=20deactivate=20Par?=
 =?UTF-8?q?quet=20dictionary=20encoding=20on=20column-wis=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…e compression

Change-Id: Icae5494babc7cbac2e1c3e405e440ff42b2b6ae5

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #546 from xhochy/ARROW-829 and squashes the following commits:

7962877 [Uwe L. Korn] ARROW-829: Don't deactivate Parquet dictionary encoding on column-wise compression
---
 python/manylinux1/build_arrow.sh | 2 +-
 python/pyarrow/_parquet.pyx      | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 8bc4e60235b49..3df322581b54c 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -72,7 +72,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     echo "=== (${PYTHON}) Test the existence of optional modules ==="
     $PIPI_IO -r requirements.txt
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.jemalloc"
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow._jemalloc"
 
     echo "=== (${PYTHON}) Tag the wheel with manylinux1 ==="
     mkdir -p repaired_wheels/
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index b7358a6a47386..dafcdaff9bfee 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -539,8 +539,6 @@ cdef class ParquetWriter:
             check_compression_name(self.compression)
             props.compression(compression_from_name(self.compression))
         elif self.compression is not None:
-            # Deactivate dictionary encoding by default
-            props.disable_dictionary()
             for column, codec in self.compression.iteritems():
                 check_compression_name(codec)
                 props.compression(column, compression_from_name(codec))

From 0f9c88f71bc64ec3288e381c8a4edb48c696b182 Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Sat, 15 Apr 2017 17:15:07 -0400
Subject: [PATCH 0520/1644] ARROW-725: [Formats/Java] FixedSizeList message and
 java implementation

~Currently only added minor type for 2-tuples~

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #452 from elahrvivaz/ARROW-725 and squashes the following commits:

b139d3d [Emilio Lahr-Vivaz] adding reAlloc to FixedSizeListVector
229e24a [Emilio Lahr-Vivaz] re-ordering imports
594c0a2 [Emilio Lahr-Vivaz] simplifying writing of list vectors through mutator
7cb2324 [Emilio Lahr-Vivaz] reverting writer changes, adding examples of writing fixed size list using vector mutators
756dc8a [Emilio Lahr-Vivaz] ARROW-725: [Formats/Java] FixedSizeList message and java implementation
---
 format/Schema.fbs                             |   8 +-
 .../src/main/codegen/data/ArrowTypes.tdd      |   5 +
 .../main/codegen/templates/ComplexCopier.java |   2 +
 .../complex/BaseRepeatedValueVector.java      |   6 +-
 .../vector/complex/FixedSizeListVector.java   | 387 ++++++++++++++++++
 .../arrow/vector/complex/ListVector.java      |  18 +-
 .../vector/complex/NullableMapVector.java     |   8 +-
 .../arrow/vector/complex/Positionable.java    |   1 +
 .../vector/complex/PromotableVector.java      |  32 ++
 .../vector/complex/RepeatedValueVector.java   |   6 +-
 .../complex/impl/AbstractBaseReader.java      |   5 +
 .../complex/impl/AbstractBaseWriter.java      |   5 +
 .../impl/UnionFixedSizeListReader.java        | 103 +++++
 .../arrow/vector/schema/TypeLayout.java       |   8 +
 .../org/apache/arrow/vector/types/Types.java  |  23 ++
 .../vector/util/JsonStringArrayList.java      |   8 +
 .../arrow/vector/TestFixedSizeListVector.java | 156 +++++++
 .../arrow/vector/file/TestArrowFile.java      |  69 +++-
 18 files changed, 838 insertions(+), 12 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/PromotableVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java

diff --git a/format/Schema.fbs b/format/Schema.fbs
index badc7ea8befbf..ff6119931dd34 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -39,6 +39,11 @@ table Struct_ {
 table List {
 }
 
+table FixedSizeList {
+  /// Number of list items per value
+  listSize: int;
+}
+
 enum UnionMode:short { Sparse, Dense }
 
 /// A union is a complex type with children in Field
@@ -159,7 +164,8 @@ union Type {
   List,
   Struct_,
   Union,
-  FixedSizeBinary
+  FixedSizeBinary,
+  FixedSizeList
 }
 
 /// ----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index e1fb5e0619a9b..ce92c1333a501 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -27,6 +27,11 @@
       fields: [],
       complex: true
     },
+    {
+      name: "FixedSizeList",
+      fields: [{name: "listSize", type: int}],
+      complex: true
+    },
     {
       name: "Union",
       fields: [{name: "mode", type: short, valueType: UnionMode}, {name: "typeIds", type: "int[]"}],
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index 0dffe5e30bea0..89368ce6e0b96 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -55,6 +55,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
           writer.endList();
         }
         break;
+      case FIXED_SIZE_LIST:
+        throw new UnsupportedOperationException("Copy fixed size list");
       case MAP:
         if (reader.isSet()) {
           writer.start();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index da221e33013d1..c9a9319c69154 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -213,12 +213,14 @@ public boolean isEmpty(int index) {
   public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
 
     @Override
-    public void startNewValue(int index) {
+    public int startNewValue(int index) {
       while (offsets.getValueCapacity() <= index) {
         offsets.reAlloc();
       }
-      offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
+      int offset = offsets.getAccessor().get(index);
+      offsets.getMutator().setSafe(index+1, offset);
       setValueCount(index+1);
+      return offset;
     }
 
     @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
new file mode 100644
index 0000000000000..7ac9f3bd5137f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -0,0 +1,387 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex;
+
+import static java.util.Collections.singletonList;
+import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BaseDataValueVector;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {
+
+  private FieldVector vector;
+  private final BitVector bits;
+  private final int listSize;
+  private final DictionaryEncoding dictionary;
+  private final List<BufferBacked> innerVectors;
+
+  private UnionFixedSizeListReader reader;
+
+  private Mutator mutator = new Mutator();
+  private Accessor accessor = new Accessor();
+
+  public FixedSizeListVector(String name,
+                             BufferAllocator allocator,
+                             int listSize,
+                             DictionaryEncoding dictionary,
+                             CallBack schemaChangeCallback) {
+    super(name, allocator);
+    Preconditions.checkArgument(listSize > 0, "list size must be positive");
+    this.bits = new BitVector("$bits$", allocator);
+    this.vector = ZeroVector.INSTANCE;
+    this.listSize = listSize;
+    this.dictionary = dictionary;
+    this.innerVectors = Collections.singletonList((BufferBacked) bits);
+    this.reader = new UnionFixedSizeListReader(this);
+  }
+
+  @Override
+  public Field getField() {
+    List<Field> children = ImmutableList.of(getDataVector().getField());
+    return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.FIXED_SIZE_LIST;
+  }
+
+  public int getListSize() {
+    return listSize;
+  }
+
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    if (children.size() != 1) {
+      throw new IllegalArgumentException("Lists have only one child. Found: " + children);
+    }
+    Field field = children.get(0);
+    FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
+    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
+    if (!addOrGetVector.isCreated()) {
+      throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
+    }
+    addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren());
+  }
+
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return singletonList(vector);
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    BaseDataValueVector.load(fieldNode, innerVectors, ownBuffers);
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    return BaseDataValueVector.unload(innerVectors);
+  }
+
+  @Override
+  public List<BufferBacked> getFieldInnerVectors() {
+    return innerVectors;
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public UnionFixedSizeListReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    allocateNewSafe();
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      success = bits.allocateNewSafe() && vector.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    if (success) {
+      bits.zeroVector();
+    }
+    return success;
+  }
+
+  @Override
+  public void reAlloc() {
+    bits.reAlloc();
+    vector.reAlloc();
+  }
+
+  public FieldVector getDataVector() {
+    return vector;
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    bits.setInitialCapacity(numRecords);
+    vector.setInitialCapacity(numRecords * listSize);
+  }
+
+  @Override
+  public int getValueCapacity() {
+    if (vector == ZeroVector.INSTANCE) {
+      return 0;
+    }
+    return vector.getValueCapacity() / listSize;
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (accessor.getValueCount() == 0) {
+      return 0;
+    }
+    return bits.getBufferSize() + vector.getBufferSize();
+  }
+
+  @Override
+  public int getBufferSizeFor(int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+    return bits.getBufferSizeFor(valueCount) + vector.getBufferSizeFor(valueCount * listSize);
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return Collections.<ValueVector>singleton(vector).iterator();
+  }
+
+  @Override
+  public void clear() {
+    bits.clear();
+    vector.clear();
+    super.clear();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
+    if (clear) {
+      for (ArrowBuf buffer: buffers) {
+        buffer.retain();
+      }
+      clear();
+    }
+    return buffers;
+  }
+
+  /**
+   * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
+   */
+  public int size() {
+    return vector == ZeroVector.INSTANCE ? 0 : 1;
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType type) {
+    boolean created = false;
+    if (vector instanceof ZeroVector) {
+      vector = type.createNewSingleVector(DATA_VECTOR_NAME, allocator, null);
+      this.reader = new UnionFixedSizeListReader(this);
+      created = true;
+    }
+    // returned vector must have the same field
+    if (!Objects.equals(vector.getField().getType(), type.getType())) {
+      final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
+        type.getType(), vector.getField().getType());
+      throw new SchemaChangeRuntimeException(msg);
+    }
+
+    return new AddOrGetResult<>((T) vector, created);
+  }
+
+  public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) {
+    copyFrom(inIndex, outIndex, from);
+  }
+
+  public void copyFrom(int inIndex, int outIndex, FixedSizeListVector from) {
+    throw new UnsupportedOperationException("FixedSizeListVector.copyFrom");
+  }
+
+  @Override
+  public UnionVector promoteToUnion() {
+    UnionVector vector = new UnionVector(name, allocator, null);
+    this.vector.clear();
+    this.vector = vector;
+    this.reader = new UnionFixedSizeListReader(this);
+    return vector;
+  }
+
+  public class Accessor extends BaseValueVector.BaseAccessor {
+
+    @Override
+    public Object getObject(int index) {
+      if (isNull(index)) {
+        return null;
+      }
+      final List<Object> vals = new JsonStringArrayList<>(listSize);
+      final ValueVector.Accessor valuesAccessor = vector.getAccessor();
+      for(int i = 0; i < listSize; i++) {
+        vals.add(valuesAccessor.getObject(index * listSize + i));
+      }
+      return vals;
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return bits.getAccessor().get(index) == 0;
+    }
+
+    @Override
+    public int getNullCount() {
+      return bits.getAccessor().getNullCount();
+    }
+
+    @Override
+    public int getValueCount() {
+      return bits.getAccessor().getValueCount();
+    }
+  }
+
+  public class Mutator extends BaseValueVector.BaseMutator {
+
+    public void setNull(int index) {
+      bits.getMutator().setSafe(index, 0);
+    }
+
+    public void setNotNull(int index) {
+      bits.getMutator().setSafe(index, 1);
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      bits.getMutator().setValueCount(valueCount);
+      vector.getMutator().setValueCount(valueCount * listSize);
+    }
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return getTransferPair(ref, allocator, null);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return new TransferImpl(ref, allocator, callBack);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector target) {
+    return new TransferImpl((FixedSizeListVector) target);
+  }
+
+  private class TransferImpl implements TransferPair {
+
+    FixedSizeListVector to;
+    TransferPair pairs[] = new TransferPair[2];
+
+    public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
+      this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
+    }
+
+    public TransferImpl(FixedSizeListVector to) {
+      this.to = to;
+      Field field = vector.getField();
+      FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
+      to.addOrGetVector(type);
+      pairs[0] = bits.makeTransferPair(to.bits);
+      pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
+    }
+
+    @Override
+    public void transfer() {
+      for (TransferPair pair : pairs) {
+        pair.transfer();
+      }
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      to.allocateNew();
+      for (int i = 0; i < length; i++) {
+        copyValueSafe(startIndex + i, i);
+      }
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      this.to.copyFrom(from, to, FixedSizeListVector.this);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 63235dfda87df..9392afbccdaa8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -53,7 +53,7 @@
 
 import io.netty.buffer.ArrowBuf;
 
-public class ListVector extends BaseRepeatedValueVector implements FieldVector {
+public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {
 
   final UInt4Vector offsets;
   final BitVector bits;
@@ -220,7 +220,7 @@ public Mutator getMutator() {
   }
 
   @Override
-  public FieldReader getReader() {
+  public UnionListReader getReader() {
     return reader;
   }
 
@@ -297,6 +297,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
     return buffers;
   }
 
+  @Override
   public UnionVector promoteToUnion() {
     UnionVector vector = new UnionVector(name, allocator, callBack);
     replaceDataVector(vector);
@@ -345,12 +346,23 @@ public void setNotNull(int index) {
     }
 
     @Override
-    public void startNewValue(int index) {
+    public int startNewValue(int index) {
       for (int i = lastSet; i <= index; i++) {
         offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
       }
       setNotNull(index);
       lastSet = index + 1;
+      return offsets.getAccessor().get(lastSet);
+    }
+
+    /**
+     * End the current value
+     *
+     * @param index index of the value to end
+     * @param size number of elements in the list that was written
+     */
+    public void endValue(int index, int size) {
+      offsets.getMutator().set(index + 1, offsets.getAccessor().get(index + 1) + size);
     }
 
     @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 647ab28352f0d..6456efba0dcb4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -31,6 +31,7 @@
 import org.apache.arrow.vector.NullableVectorDefinitionSetter;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.NullableMapWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
@@ -45,6 +46,7 @@
 public class NullableMapVector extends MapVector implements FieldVector {
 
   private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
+  private final NullableMapWriter writer = new NullableMapWriter(this);
 
   protected final BitVector bits;
 
@@ -84,10 +86,14 @@ public List<BufferBacked> getFieldInnerVectors() {
   }
 
   @Override
-  public FieldReader getReader() {
+  public NullableMapReaderImpl getReader() {
     return reader;
   }
 
+  public NullableMapWriter getWriter() {
+    return writer;
+  }
+
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
     return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, dictionary, null), false);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
index 93451181ca949..e1a4f36296987 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
@@ -18,5 +18,6 @@
 package org.apache.arrow.vector.complex;
 
 public interface Positionable {
+  public int getPosition();
   public void setPosition(int index);
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/PromotableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/PromotableVector.java
new file mode 100644
index 0000000000000..8b528b4ccab9b
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/PromotableVector.java
@@ -0,0 +1,32 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.FieldType;
+
+public interface PromotableVector {
+
+  <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType type);
+
+  UnionVector promoteToUnion();
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index 54db393e8310d..b01a4e7cf49d4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -73,13 +73,13 @@ interface RepeatedAccessor extends ValueVector.Accessor {
   }
 
   interface RepeatedMutator extends ValueVector.Mutator {
+
     /**
      * Starts a new value that is a container of cells.
      *
      * @param index  index of new value to start
+     * @return index into the child vector
      */
-    void startNewValue(int index);
-
-
+    int startNewValue(int index);
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
index e7c3c8c7e4b42..7c73c27ecff41 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
@@ -35,6 +35,11 @@ public AbstractBaseReader() {
     super();
   }
 
+  @Override
+  public int getPosition() {
+    return index;
+  }
+
   public void setPosition(int index){
     this.index = index;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
index e6cf098f16f59..13a0a6bd9e28f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
@@ -34,6 +34,11 @@ int idx() {
     return index;
   }
 
+  @Override
+  public int getPosition() {
+    return index;
+  }
+
   @Override
   public void setPosition(int index) {
     this.index = index;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
new file mode 100644
index 0000000000000..515d4ab8ce907
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
@@ -0,0 +1,103 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex.impl;
+
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.holders.UnionHolder;
+import org.apache.arrow.vector.types.Types.MinorType;
+
+/**
+ * Reader for fixed size list vectors
+ */
+public class UnionFixedSizeListReader extends AbstractFieldReader {
+
+  private final FixedSizeListVector vector;
+  private final ValueVector data;
+  private final int listSize;
+
+  private int currentOffset;
+
+  public UnionFixedSizeListReader(FixedSizeListVector vector) {
+    this.vector = vector;
+    this.data = vector.getDataVector();
+    this.listSize = vector.getListSize();
+  }
+
+  @Override
+  public boolean isSet() {
+    return !vector.getAccessor().isNull(idx());
+  }
+
+  @Override
+  public FieldReader reader() {
+    return data.getReader();
+  }
+
+  @Override
+  public Object readObject() {
+    return vector.getAccessor().getObject(idx());
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return vector.getMinorType();
+  }
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    data.getReader().setPosition(index * listSize);
+    currentOffset = 0;
+  }
+
+  @Override
+  public void read(int index, UnionHolder holder) {
+    setPosition(idx());
+    for (int i = -1; i < index; i++) {
+      if (!next()) {
+        throw new IndexOutOfBoundsException("Requested " + index + ", size " + listSize);
+      }
+    }
+    holder.reader = data.getReader();
+    holder.isSet = vector.getAccessor().isNull(idx()) ? 0 : 1;
+  }
+
+  @Override
+  public int size() {
+    return listSize;
+  }
+
+  @Override
+  public boolean next() {
+    if (currentOffset < listSize) {
+      data.getReader().setPosition(idx() * listSize + currentOffset++);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  public void copyAsValue(ListWriter writer) {
+    ComplexCopier.copy(this, (FieldWriter) writer);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 69d550fc9f799..24840ec988ac3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -35,6 +35,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
 import org.apache.arrow.vector.types.pojo.ArrowType.Date;
 import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
@@ -105,6 +106,13 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
         return new TypeLayout(vectors);
       }
 
+      @Override public TypeLayout visit(FixedSizeList type) {
+        List<VectorLayout> vectors = asList(
+            validityVector()
+            );
+        return new TypeLayout(vectors);
+      }
+
       @Override public TypeLayout visit(FloatingPoint type) {
         int bitWidth;
         switch (type.getPrecision()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index b0455fa14e44c..6023f1c9500e7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -51,6 +51,7 @@
 import org.apache.arrow.vector.NullableVarCharVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
@@ -90,6 +91,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
 import org.apache.arrow.vector.types.pojo.ArrowType.Date;
 import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
@@ -436,6 +438,23 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UnionListWriter((ListVector) vector);
       }
     },
+    FIXED_SIZE_LIST(null) {
+      @Override
+      public ArrowType getType() {
+        throw new UnsupportedOperationException("Cannot get simple type for FixedSizeList type");
+      }
+
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        int size = ((FixedSizeList)fieldType.getType()).getListSize();
+        return new FixedSizeListVector(name, allocator, size, fieldType.getDictionary(), schemaChangeCallback);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        throw new UnsupportedOperationException("FieldWriter not implemented for FixedSizeList type");
+      }
+    },
     UNION(new Union(Sparse, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
@@ -480,6 +499,10 @@ public static MinorType getMinorTypeForArrowType(ArrowType arrowType) {
         return MinorType.LIST;
       }
 
+      @Override public MinorType visit(FixedSizeList type) {
+        return MinorType.FIXED_SIZE_LIST;
+      }
+
       @Override public MinorType visit(Union type) {
         return MinorType.UNION;
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
index 6291bfeaee666..c598069c2c309 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
@@ -31,6 +31,14 @@ public class JsonStringArrayList<E> extends ArrayList<E> {
     mapper = new ObjectMapper();
   }
 
+  public JsonStringArrayList() {
+    super();
+  }
+
+  public JsonStringArrayList(int size) {
+    super(size);
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (this == obj) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
new file mode 100644
index 0000000000000..cfb7b3d2a26ac
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import com.google.common.collect.Lists;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestFixedSizeListVector {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testIntType() {
+    try (FixedSizeListVector vector = new FixedSizeListVector("list", allocator, 2, null, null)) {
+      NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
+      NullableIntVector.Mutator mutator = nested.getMutator();
+      vector.allocateNew();
+
+      for (int i = 0; i < 10; i++) {
+        vector.getMutator().setNotNull(i);
+        mutator.set(i * 2, i);
+        mutator.set(i * 2 + 1, i + 10);
+      }
+      vector.getMutator().setValueCount(10);
+
+      UnionFixedSizeListReader reader = vector.getReader();
+      for (int i = 0; i < 10; i++) {
+        reader.setPosition(i);
+        Assert.assertTrue(reader.isSet());
+        Assert.assertTrue(reader.next());
+        Assert.assertEquals(i, reader.reader().readInteger().intValue());
+        Assert.assertTrue(reader.next());
+        Assert.assertEquals(i + 10, reader.reader().readInteger().intValue());
+        Assert.assertFalse(reader.next());
+        Assert.assertEquals(Lists.newArrayList(i, i + 10), reader.readObject());
+      }
+    }
+  }
+
+  @Test
+  public void testFloatTypeNullable() {
+    try (FixedSizeListVector vector = new FixedSizeListVector("list", allocator, 2, null, null)) {
+      NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
+      NullableFloat4Vector.Mutator mutator = nested.getMutator();
+      vector.allocateNew();
+
+      for (int i = 0; i < 10; i++) {
+        if (i % 2 == 0) {
+          vector.getMutator().setNotNull(i);
+          mutator.set(i * 2, i + 0.1f);
+          mutator.set(i * 2 + 1, i + 10.1f);
+        }
+      }
+      vector.getMutator().setValueCount(10);
+
+      UnionFixedSizeListReader reader = vector.getReader();
+      for (int i = 0; i < 10; i++) {
+        reader.setPosition(i);
+        if (i % 2 == 0) {
+          Assert.assertTrue(reader.isSet());
+          Assert.assertTrue(reader.next());
+          Assert.assertEquals(i + 0.1f, reader.reader().readFloat(), 0.00001);
+          Assert.assertTrue(reader.next());
+          Assert.assertEquals(i + 10.1f, reader.reader().readFloat(), 0.00001);
+          Assert.assertFalse(reader.next());
+          Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), reader.readObject());
+        } else {
+          Assert.assertFalse(reader.isSet());
+          Assert.assertNull(reader.readObject());
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testNestedInList() {
+    try (ListVector vector = new ListVector("list", allocator, null, null)) {
+      ListVector.Mutator mutator = vector.getMutator();
+      FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector();
+      FixedSizeListVector.Mutator tupleMutator = tuples.getMutator();
+      NullableIntVector.Mutator innerMutator = (NullableIntVector.Mutator) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector().getMutator();
+      vector.allocateNew();
+
+      for (int i = 0; i < 10; i++) {
+        if (i % 2 == 0) {
+          int position = mutator.startNewValue(i);
+          for (int j = 0; j < i % 7; j++) {
+            tupleMutator.setNotNull(position + j);
+            innerMutator.set((position + j) * 2, j);
+            innerMutator.set((position + j) * 2 + 1, j + 1);
+          }
+          mutator.endValue(i, i % 7);
+        }
+      }
+      mutator.setValueCount(10);
+
+      UnionListReader reader = vector.getReader();
+      for (int i = 0; i < 10; i++) {
+        reader.setPosition(i);
+        if (i % 2 == 0) {
+          for (int j = 0; j < i % 7; j++) {
+            Assert.assertTrue(reader.next());
+            FieldReader innerListReader = reader.reader();
+            for (int k = 0; k < 2; k++) {
+              Assert.assertTrue(innerListReader.next());
+              Assert.assertEquals(k + j, innerListReader.reader().readInteger().intValue());
+            }
+            Assert.assertFalse(innerListReader.next());
+          }
+          Assert.assertFalse(reader.next());
+        } else {
+          Assert.assertFalse(reader.isSet());
+          Assert.assertNull(reader.readObject());
+        }
+      }
+    }
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 11730afd55406..3bed45361fc20 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -30,11 +30,17 @@
 import java.util.Arrays;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.NullableFloat4Vector;
+import org.apache.arrow.vector.NullableIntVector;
 import org.apache.arrow.vector.NullableTinyIntVector;
 import org.apache.arrow.vector.NullableVarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
@@ -49,6 +55,8 @@
 import org.apache.arrow.vector.stream.ArrowStreamReader;
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
 import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -60,8 +68,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.ImmutableList;
-
 public class TestArrowFile extends BaseFileTest {
   private static final Logger LOGGER = LoggerFactory.getLogger(TestArrowFile.class);
 
@@ -576,6 +582,65 @@ private void validateNestedDictionary(ListVector vector, DictionaryProvider prov
     Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
   }
 
+  @Test
+  public void testWriteReadFixedSizeList() throws IOException {
+    File file = new File("target/mytest_fixed_list.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+    int count = COUNT;
+
+    // write
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         NullableMapVector parent = new NullableMapVector("parent", originalVectorAllocator, null, null)) {
+      FixedSizeListVector tuples = parent.addOrGet("float-pairs", new FieldType(true, new FixedSizeList(2), null), FixedSizeListVector.class);
+      NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(new FieldType(true, MinorType.FLOAT4.getType(), null)).getVector();
+      NullableIntVector ints = parent.addOrGet("ints", new FieldType(true, new Int(32, true), null), NullableIntVector.class);
+      parent.allocateNew();
+
+      for (int i = 0; i < 10; i++) {
+        tuples.getMutator().setNotNull(i);
+        floats.getMutator().set(i * 2, i + 0.1f);
+        floats.getMutator().set(i * 2 + 1, i + 10.1f);
+        ints.getMutator().set(i, i);
+      }
+
+      parent.getMutator().setValueCount(10);
+      write(parent, file, stream);
+    }
+
+    // read
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+
+      for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
+        arrowReader.loadRecordBatch(rbBlock);
+        Assert.assertEquals(count, root.getRowCount());
+        for (int i = 0; i < 10; i++) {
+          Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i));
+          Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i));
+        }
+      }
+    }
+
+    // read from stream
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      Assert.assertEquals(count, root.getRowCount());
+      for (int i = 0; i < 10; i++) {
+        Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i));
+        Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i));
+      }
+    }
+  }
+
   /**
    * Writes the contents of parents to file. If outStream is non-null, also writes it
    * to outStream in the streaming serialized format.

From 30e03a90718971c2a1d773145fb042d0c2857036 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@dremio.com>
Date: Sat, 15 Apr 2017 18:26:19 -0400
Subject: [PATCH 0521/1644] =?UTF-8?q?ARROW-703:=20Fix=20issue=20where=20se?=
 =?UTF-8?q?tValueCount(0)=20doesn=E2=80=99t=20work=20in=20the=20case=20tha?=
 =?UTF-8?q?t=20we=E2=80=99ve=20shipped=20vectors=20across=20the=20wire?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Julien Le Dem <julien@dremio.com>

Closes #428 from julienledem/arrow_703 and squashes the following commits:

72b0f79 [Julien Le Dem] ARROW-703: Fix issue where setValueCount(0) doesn’t work in the case that we’ve shipped vectors across the wire
---
 .../templates/VariableLengthVectors.java      | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index bcd639ab8c30c..4a460c5475323 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -613,16 +613,21 @@ protected void set(int index, ${minor.class}Holder holder){
 
     @Override
     public void setValueCount(int valueCount) {
-      final int currentByteCapacity = getByteCapacity();
-      final int idx = offsetVector.getAccessor().get(valueCount);
-      data.writerIndex(idx);
-      if (valueCount > 0 && currentByteCapacity > idx * 2) {
-        incrementAllocationMonitor();
-      } else if (allocationMonitor > 0) {
-        allocationMonitor = 0;
+      if (valueCount == 0) {
+        // if no values in vector, don't try to retrieve the current value count.
+        offsetVector.getMutator().setValueCount(0);
+      } else {
+        final int currentByteCapacity = getByteCapacity();
+        final int idx = offsetVector.getAccessor().get(valueCount);
+        data.writerIndex(idx);
+        if (currentByteCapacity > idx * 2) {
+          incrementAllocationMonitor();
+        } else if (allocationMonitor > 0) {
+          allocationMonitor = 0;
+        }
+        VectorTrimmer.trim(data, idx);
+        offsetVector.getMutator().setValueCount(valueCount+1);
       }
-      VectorTrimmer.trim(data, idx);
-      offsetVector.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
     }
 
     @Override

From ee5cb2ad171f0f4c7673f2937dc226d62aad972c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 16 Apr 2017 09:28:34 -0400
Subject: [PATCH 0522/1644] ARROW-830: [Python] Expose jemalloc memory pool and
 other memory pool functions in public pyarrow API

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #550 from wesm/ARROW-830 and squashes the following commits:

c1ca9fb [Wes McKinney] Expose jemalloc memory pool and other memory pool functions in public pyarrow API
---
 python/README.md                      |  2 +-
 python/doc/source/api.rst             | 12 ++++++++++
 python/doc/source/jemalloc.rst        |  8 ++-----
 python/pyarrow/__init__.py            | 13 +++++++++-
 python/pyarrow/_memory.pyx            | 12 +++++++---
 python/pyarrow/tests/test_jemalloc.py | 34 +++++++++++++++------------
 6 files changed, 55 insertions(+), 26 deletions(-)

diff --git a/python/README.md b/python/README.md
index 25a3a67b83b03..ed008ea975d21 100644
--- a/python/README.md
+++ b/python/README.md
@@ -89,7 +89,7 @@ export PYARROW_CMAKE_OPTIONS=-DPYARROW_BUILD_PARQUET=on
 
 ```bash
 pip install -r doc/requirements.txt
-python setup.py build_sphinx
+python setup.py build_sphinx -s doc/source
 ```
 
 [1]: https://github.com/apache/parquet-cpp
\ No newline at end of file
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 514dcf966f8cc..801ab34126c7c 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -151,3 +151,15 @@ Interprocess Communication and Messaging
    FileWriter
    StreamReader
    StreamWriter
+
+Memory Pools
+------------
+
+.. autosummary::
+   :toctree: generated/
+
+   MemoryPool
+   default_memory_pool
+   jemalloc_memory_pool
+   total_allocated_bytes
+   set_memory_pool
diff --git a/python/doc/source/jemalloc.rst b/python/doc/source/jemalloc.rst
index 33fe61729c1e9..8d7a5dc4a82ec 100644
--- a/python/doc/source/jemalloc.rst
+++ b/python/doc/source/jemalloc.rst
@@ -35,18 +35,14 @@ operations.
 .. code:: python
 
     import pyarrow as pa
-    import pyarrow.jemalloc
-    import pyarrow.memory
 
-    jemalloc_pool = pyarrow.jemalloc.default_pool()
+    jemalloc_pool = pyarrow.jemalloc_memory_pool()
 
     # Explicitly use jemalloc for allocating memory for an Arrow Table object
     array = pa.Array.from_pylist([1, 2, 3], memory_pool=jemalloc_pool)
 
     # Set the global pool
-    pyarrow.memory.set_default_pool(jemalloc_pool)
+    pyarrow.set_memory_pool(jemalloc_pool)
     # This operation has no explicit MemoryPool specified and will thus will
     # also use jemalloc for its allocations.
     array = pa.Array.from_pylist([1, 2, 3])
-
-
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 66bde4933ee2d..506d567b0c508 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -62,7 +62,8 @@
                          memory_map, create_memory_map,
                          get_record_batch_size, get_tensor_size)
 
-from pyarrow._memory import MemoryPool, total_allocated_bytes
+from pyarrow._memory import (MemoryPool, total_allocated_bytes,
+                             set_memory_pool, default_memory_pool)
 from pyarrow._table import Column, RecordBatch, Table, concat_tables
 from pyarrow._error import (ArrowException,
                             ArrowKeyError,
@@ -72,6 +73,16 @@
                             ArrowNotImplementedError,
                             ArrowTypeError)
 
+
+def jemalloc_memory_pool():
+    """
+    Returns a jemalloc-based memory allocator, which can be passed to
+    pyarrow.set_memory_pool
+    """
+    from pyarrow._jemalloc import default_pool
+    return default_pool()
+
+
 from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
 
 from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
diff --git a/python/pyarrow/_memory.pyx b/python/pyarrow/_memory.pyx
index 98dbf66c8e0af..8b73a17553edf 100644
--- a/python/pyarrow/_memory.pyx
+++ b/python/pyarrow/_memory.pyx
@@ -22,6 +22,7 @@
 from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
 from pyarrow.includes.pyarrow cimport set_default_memory_pool, get_memory_pool
 
+
 cdef class MemoryPool:
     cdef init(self, CMemoryPool* pool):
         self.pool = pool
@@ -29,24 +30,29 @@ cdef class MemoryPool:
     def bytes_allocated(self):
         return self.pool.bytes_allocated()
 
+
 cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool):
     if memory_pool is None:
         return get_memory_pool()
     else:
         return memory_pool.pool
 
+
 cdef class LoggingMemoryPool(MemoryPool):
     pass
 
-def default_pool():
-    cdef: 
+
+def default_memory_pool():
+    cdef:
         MemoryPool pool = MemoryPool()
     pool.init(get_memory_pool())
     return pool
 
-def set_default_pool(MemoryPool pool):
+
+def set_memory_pool(MemoryPool pool):
     set_default_memory_pool(pool.pool)
 
+
 def total_allocated_bytes():
     cdef CMemoryPool* pool = get_memory_pool()
     return pool.bytes_allocated()
diff --git a/python/pyarrow/tests/test_jemalloc.py b/python/pyarrow/tests/test_jemalloc.py
index c6cc2cc34a08b..0a4d8a63ad2d2 100644
--- a/python/pyarrow/tests/test_jemalloc.py
+++ b/python/pyarrow/tests/test_jemalloc.py
@@ -18,12 +18,16 @@
 import gc
 import pytest
 
+import pyarrow as pa
+
+
 try:
-    import pyarrow.jemalloc
+    pa.jemalloc_memory_pool()
     HAVE_JEMALLOC = True
 except ImportError:
     HAVE_JEMALLOC = False
 
+
 jemalloc = pytest.mark.skipif(not HAVE_JEMALLOC,
                               reason='jemalloc support not built')
 
@@ -31,33 +35,33 @@
 @jemalloc
 def test_different_memory_pool():
     gc.collect()
-    bytes_before_default = pyarrow.total_allocated_bytes()
-    bytes_before_jemalloc = pyarrow.jemalloc.default_pool().bytes_allocated()
+    bytes_before_default = pa.total_allocated_bytes()
+    bytes_before_jemalloc = pa.jemalloc_memory_pool().bytes_allocated()
 
     # it works
-    array = pyarrow.from_pylist([1, None, 3, None],  # noqa
-                                memory_pool=pyarrow.jemalloc.default_pool())
+    array = pa.from_pylist([1, None, 3, None],  # noqa
+                                memory_pool=pa.jemalloc_memory_pool())
     gc.collect()
-    assert pyarrow.total_allocated_bytes() == bytes_before_default
-    assert (pyarrow.jemalloc.default_pool().bytes_allocated() >
+    assert pa.total_allocated_bytes() == bytes_before_default
+    assert (pa.jemalloc_memory_pool().bytes_allocated() >
             bytes_before_jemalloc)
 
 
 @jemalloc
 def test_default_memory_pool():
     gc.collect()
-    bytes_before_default = pyarrow.total_allocated_bytes()
-    bytes_before_jemalloc = pyarrow.jemalloc.default_pool().bytes_allocated()
+    bytes_before_default = pa.total_allocated_bytes()
+    bytes_before_jemalloc = pa.jemalloc_memory_pool().bytes_allocated()
 
-    old_memory_pool = pyarrow.memory.default_pool()
-    pyarrow.memory.set_default_pool(pyarrow.jemalloc.default_pool())
+    old_memory_pool = pa.default_memory_pool()
+    pa.set_memory_pool(pa.jemalloc_memory_pool())
 
-    array = pyarrow.from_pylist([1, None, 3, None])  # noqa
+    array = pa.from_pylist([1, None, 3, None])  # noqa
 
-    pyarrow.memory.set_default_pool(old_memory_pool)
+    pa.set_memory_pool(old_memory_pool)
     gc.collect()
 
-    assert pyarrow.total_allocated_bytes() == bytes_before_default
+    assert pa.total_allocated_bytes() == bytes_before_default
 
-    assert (pyarrow.jemalloc.default_pool().bytes_allocated() >
+    assert (pa.jemalloc_memory_pool().bytes_allocated() >
             bytes_before_jemalloc)

From dad1a8ee3810d1584b96a5324f0d84215cd48216 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 16 Apr 2017 09:29:15 -0400
Subject: [PATCH 0523/1644] ARROW-832: [C++] Update to gtest 1.8.0, remove now
 unneeded test_main.cc

I haven't tried this out on MSVC yet.

Also includes .gitignore fix for ARROW-821

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #549 from wesm/ARROW-832 and squashes the following commits:

2f246a0 [Wes McKinney] Remove unused CMake variable
7a62cf4 [Wes McKinney] Small fix when ARROW_BUILD_BENCHMARKS=off
8eaa318 [Wes McKinney] Add dependency on gtest for benchmarks
5f692db [Wes McKinney] Update to gtest 1.8.0, remove now unneeded test_main.cc
---
 cpp/CMakeLists.txt                | 42 ++++++++++++++++++-------------
 cpp/src/arrow/util/CMakeLists.txt | 25 +++---------------
 cpp/src/arrow/util/test_main.cc   | 26 -------------------
 python/.gitignore                 |  2 +-
 4 files changed, 29 insertions(+), 66 deletions(-)
 delete mode 100644 cpp/src/arrow/util/test_main.cc

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 83610d33e6af1..08120e9ea68a5 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -43,7 +43,7 @@ set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 
 set(GFLAGS_VERSION "2.1.2")
-set(GTEST_VERSION "1.7.0")
+set(GTEST_VERSION "1.8.0")
 set(GBENCHMARK_VERSION "1.1.0")
 set(FLATBUFFERS_VERSION "1.6.0")
 set(JEMALLOC_VERSION "4.4.0")
@@ -458,7 +458,7 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 # ----------------------------------------------------------------------
 # Enable / disable tests and benchmarks
 
-if(ARROW_BUILD_TESTS)
+if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
   add_custom_target(unittest ctest -L unittest)
 
   if("$ENV{GTEST_HOME}" STREQUAL "")
@@ -472,9 +472,13 @@ if(ARROW_BUILD_TESTS)
 
     set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
     set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
-    set(GTEST_STATIC_LIB "${GTEST_PREFIX}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GTEST_STATIC_LIB
+      "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GTEST_MAIN_STATIC_LIB
+      "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GTEST_VENDORED 1)
     set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                         -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}
                          -Dgtest_force_shared_crt=ON
                          -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
 
@@ -482,22 +486,11 @@ if(ARROW_BUILD_TESTS)
       # BUILD_BYPRODUCTS is a 3.2+ feature
       ExternalProject_Add(googletest_ep
         URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-        CMAKE_ARGS ${GTEST_CMAKE_ARGS}
-        # googletest doesn't define install rules, so just build in the
-        # source dir and don't try to install.  See its README for
-        # details.
-        BUILD_IN_SOURCE 1
-        BUILD_BYPRODUCTS "${GTEST_STATIC_LIB}"
-        INSTALL_COMMAND "")
+        CMAKE_ARGS ${GTEST_CMAKE_ARGS})
     else()
       ExternalProject_Add(googletest_ep
         URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-        CMAKE_ARGS ${GTEST_CMAKE_ARGS}
-        # googletest doesn't define install rules, so just build in the
-        # source dir and don't try to install.  See its README for
-        # details.
-        BUILD_IN_SOURCE 1
-        INSTALL_COMMAND "")
+        CMAKE_ARGS ${GTEST_CMAKE_ARGS})
     endif()
   else()
     find_package(GTest REQUIRED)
@@ -509,9 +502,12 @@ if(ARROW_BUILD_TESTS)
   include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(gtest
     STATIC_LIB ${GTEST_STATIC_LIB})
+  ADD_THIRDPARTY_LIB(gtest_main
+    STATIC_LIB ${GTEST_MAIN_STATIC_LIB})
 
   if(GTEST_VENDORED)
     add_dependencies(gtest googletest_ep)
+    add_dependencies(gtest_main googletest_ep)
   endif()
 
   # gflags (formerly Googleflags) command line parsing
@@ -753,10 +749,22 @@ include_directories(SYSTEM "${HADOOP_HOME}/include")
 ############################################################
 set(ARROW_MIN_TEST_LIBS
   arrow_static
-  arrow_test_main
+  gtest
+  gtest_main
   ${ARROW_BASE_LIBS}
   ${BOOST_REGEX_LIBRARY})
 
+if (APPLE)
+  set(ARROW_MIN_TEST_LIBS
+    ${ARROW_MIN_TEST_LIBS}
+    ${CMAKE_DL_LIBS})
+elseif(NOT MSVC)
+  set(ARROW_MIN_TEST_LIBS
+    ${ARROW_MIN_TEST_LIBS}
+    pthread
+    ${CMAKE_DL_LIBS})
+endif()
+
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 set(ARROW_BENCHMARK_LINK_LIBS
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 9aa8bae273fb8..b22c8aca11c5d 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -32,28 +32,6 @@ install(FILES
 # arrow_test_main
 #######################################
 
-if (ARROW_BUILD_TESTS)
-  add_library(arrow_test_main
-	test_main.cc)
-
-  if (APPLE)
-	  target_link_libraries(arrow_test_main
-        gtest
-        dl)
-	  set_target_properties(arrow_test_main
-        PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-  elseif(MSVC)
-	  target_link_libraries(arrow_test_main
-        gtest)
-  else()
-	  target_link_libraries(arrow_test_main
-        gtest
-        pthread
-        dl
-	    )
-  endif()
-endif()
-
 if (ARROW_BUILD_BENCHMARKS)
   add_library(arrow_benchmark_main benchmark_main.cc)
   if (APPLE)
@@ -66,6 +44,9 @@ if (ARROW_BUILD_BENCHMARKS)
       pthread
 	  )
   endif()
+
+  # TODO(wesm): Some benchmarks include gtest.h
+  add_dependencies(arrow_benchmark_main gtest)
 endif()
 
 ADD_ARROW_TEST(bit-util-test)
diff --git a/cpp/src/arrow/util/test_main.cc b/cpp/src/arrow/util/test_main.cc
deleted file mode 100644
index f928047023966..0000000000000
--- a/cpp/src/arrow/util/test_main.cc
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-
-  int ret = RUN_ALL_TESTS();
-
-  return ret;
-}
diff --git a/python/.gitignore b/python/.gitignore
index 4ab802006914e..ba40c3ea88882 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -16,7 +16,7 @@ Testing/
 *.c
 *.cpp
 pyarrow/version.py
-pyarrow/table_api.h
+pyarrow/*_api.h
 # Python files
 
 # setup.py working directory

From 09e6eade166b60db95694d291ebfb074f1442ff8 Mon Sep 17 00:00:00 2001
From: Jeff Reback <jeff.reback@twosigma.com>
Date: Sun, 16 Apr 2017 13:11:38 -0400
Subject: [PATCH 0524/1644] ARROW-836: add test for pandas conversion of
 timedelta, currently unimplemented

xref https://github.com/pandas-dev/pandas/pull/16004

Author: Jeff Reback <jeff.reback@twosigma.com>

Closes #551 from jreback/timedelta and squashes the following commits:

cfd310e [Jeff Reback] TST: add test for pandas conversion of timedelta, currently unimplemented
---
 python/pyarrow/tests/test_convert_pandas.py | 13 +++++++++++++
 python/pyarrow/tests/test_feather.py        | 10 ++++++++++
 2 files changed, 23 insertions(+)

diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 4a57e4ba1d4fb..2394d638d073e 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -18,6 +18,7 @@
 
 from collections import OrderedDict
 
+import pytest
 import datetime
 import unittest
 import decimal
@@ -412,6 +413,18 @@ def test_dates_from_integers(self):
         assert a1[0].as_py() == expected
         assert a2[0].as_py() == expected
 
+    @pytest.mark.xfail(reason="not supported ATM",
+                       raises=NotImplementedError)
+    def test_timedelta(self):
+        # TODO(jreback): Pandas only support ns resolution
+        # Arrow supports ??? for resolution
+        df = pd.DataFrame({
+            'timedelta': np.arange(start=0, stop=3*86400000,
+                                   step=86400000,
+                                   dtype='timedelta64[ms]')
+            })
+        pa.Table.from_pandas(df)
+
     def test_column_of_arrays(self):
         df, schema = dataframe_with_arrays()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 6f8040fd483c9..ef73a8feeb65c 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -14,6 +14,7 @@
 
 import os
 import unittest
+import pytest
 
 from numpy.testing import assert_array_equal
 import numpy as np
@@ -320,6 +321,15 @@ def test_timestamp_with_nulls(self):
 
         self._check_pandas_roundtrip(df, null_counts=[1, 1])
 
+    @pytest.mark.xfail(reason="not supported ATM",
+                       raises=NotImplementedError)
+    def test_timedelta_with_nulls(self):
+        df = pd.DataFrame({'test': [pd.Timedelta('1 day'),
+                                    None,
+                                    pd.Timedelta('3 day')]})
+
+        self._check_pandas_roundtrip(df, null_counts=[1, 1])
+
     def test_out_of_float64_timestamp_with_nulls(self):
         df = pd.DataFrame(
             {'test': pd.DatetimeIndex([1451606400000000001,

From f51259068640af92490c0832d5d55885a510776d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Apr 2017 09:22:15 -0400
Subject: [PATCH 0525/1644] ARROW-827: [Python] Miscellaneous improvements to
 help with Dask support

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #543 from wesm/dask-improvements and squashes the following commits:

1f587e2 [Wes McKinney] Store the input Parquet paths on the dataset object
3504281 [Wes McKinney] Add some more cases
edc9b59 [Wes McKinney] Unit tests
88f4380 [Wes McKinney] Use dict for type mapping for now
7e69cab [Wes McKinney] Miscellaneous improvements to help with Dask support
---
 python/pyarrow/_array.pyx            | 193 +++++++++++++++++----------
 python/pyarrow/_parquet.pyx          |  23 +++-
 python/pyarrow/includes/libarrow.pxd |  64 ++++-----
 python/pyarrow/parquet.py            |  22 ++-
 python/pyarrow/tests/test_parquet.py |   4 +
 python/pyarrow/tests/test_schema.py  |  30 +++++
 6 files changed, 222 insertions(+), 114 deletions(-)

diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/_array.pyx
index 7ef8e5867a1a2..c5a595c6c67e5 100644
--- a/python/pyarrow/_array.pyx
+++ b/python/pyarrow/_array.pyx
@@ -41,6 +41,31 @@ cdef _pandas():
     return pd
 
 
+# These are imprecise because the type (in pandas 0.x) depends on the presence
+# of nulls
+_pandas_type_map = {
+    _Type_NA: np.float64,  # NaNs
+    _Type_BOOL: np.bool_,
+    _Type_INT8: np.int8,
+    _Type_INT16: np.int16,
+    _Type_INT32: np.int32,
+    _Type_INT64: np.int64,
+    _Type_UINT8: np.uint8,
+    _Type_UINT16: np.uint16,
+    _Type_UINT32: np.uint32,
+    _Type_UINT64: np.uint64,
+    _Type_HALF_FLOAT: np.float16,
+    _Type_FLOAT: np.float32,
+    _Type_DOUBLE: np.float64,
+    _Type_DATE32: np.dtype('datetime64[ns]'),
+    _Type_DATE64: np.dtype('datetime64[ns]'),
+    _Type_TIMESTAMP: np.dtype('datetime64[ns]'),
+    _Type_BINARY: np.object_,
+    _Type_FIXED_SIZE_BINARY: np.object_,
+    _Type_STRING: np.object_,
+    _Type_LIST: np.object_
+}
+
 cdef class DataType:
 
     def __cinit__(self):
@@ -64,6 +89,16 @@ cdef class DataType:
         else:
             raise TypeError('Invalid comparison')
 
+    def to_pandas_dtype(self):
+        """
+        Return the NumPy dtype that would be used for storing this
+        """
+        cdef Type type_id = self.type.id()
+        if type_id in _pandas_type_map:
+            return _pandas_type_map[type_id]
+        else:
+            raise NotImplementedError(str(self))
+
 
 cdef class DictionaryType(DataType):
 
@@ -167,6 +202,16 @@ cdef class Schema:
 
         return result
 
+    property names:
+
+        def __get__(self):
+            cdef int i
+            result = []
+            for i in range(self.schema.num_fields()):
+                name = frombytes(self.schema.field(i).get().name())
+                result.append(name)
+            return result
+
     cdef init(self, const vector[shared_ptr[CField]]& fields):
         self.schema = new CSchema(fields)
         self.sp_schema.reset(self.schema)
@@ -244,56 +289,56 @@ def field(name, type, bint nullable=True):
 
 
 cdef set PRIMITIVE_TYPES = set([
-    Type_NA, Type_BOOL,
-    Type_UINT8, Type_INT8,
-    Type_UINT16, Type_INT16,
-    Type_UINT32, Type_INT32,
-    Type_UINT64, Type_INT64,
-    Type_TIMESTAMP, Type_DATE32,
-    Type_DATE64,
-    Type_HALF_FLOAT,
-    Type_FLOAT,
-    Type_DOUBLE])
+    _Type_NA, _Type_BOOL,
+    _Type_UINT8, _Type_INT8,
+    _Type_UINT16, _Type_INT16,
+    _Type_UINT32, _Type_INT32,
+    _Type_UINT64, _Type_INT64,
+    _Type_TIMESTAMP, _Type_DATE32,
+    _Type_DATE64,
+    _Type_HALF_FLOAT,
+    _Type_FLOAT,
+    _Type_DOUBLE])
 
 
 def null():
-    return primitive_type(Type_NA)
+    return primitive_type(_Type_NA)
 
 
 def bool_():
-    return primitive_type(Type_BOOL)
+    return primitive_type(_Type_BOOL)
 
 
 def uint8():
-    return primitive_type(Type_UINT8)
+    return primitive_type(_Type_UINT8)
 
 
 def int8():
-    return primitive_type(Type_INT8)
+    return primitive_type(_Type_INT8)
 
 
 def uint16():
-    return primitive_type(Type_UINT16)
+    return primitive_type(_Type_UINT16)
 
 
 def int16():
-    return primitive_type(Type_INT16)
+    return primitive_type(_Type_INT16)
 
 
 def uint32():
-    return primitive_type(Type_UINT32)
+    return primitive_type(_Type_UINT32)
 
 
 def int32():
-    return primitive_type(Type_INT32)
+    return primitive_type(_Type_INT32)
 
 
 def uint64():
-    return primitive_type(Type_UINT64)
+    return primitive_type(_Type_UINT64)
 
 
 def int64():
-    return primitive_type(Type_INT64)
+    return primitive_type(_Type_INT64)
 
 
 cdef dict _timestamp_type_cache = {}
@@ -344,23 +389,23 @@ def timestamp(unit_str, tz=None):
 
 
 def date32():
-    return primitive_type(Type_DATE32)
+    return primitive_type(_Type_DATE32)
 
 
 def date64():
-    return primitive_type(Type_DATE64)
+    return primitive_type(_Type_DATE64)
 
 
 def float16():
-    return primitive_type(Type_HALF_FLOAT)
+    return primitive_type(_Type_HALF_FLOAT)
 
 
 def float32():
-    return primitive_type(Type_FLOAT)
+    return primitive_type(_Type_FLOAT)
 
 
 def float64():
-    return primitive_type(Type_DOUBLE)
+    return primitive_type(_Type_DOUBLE)
 
 
 cpdef DataType decimal(int precision, int scale=0):
@@ -373,7 +418,7 @@ def string():
     """
     UTF8 string
     """
-    return primitive_type(Type_STRING)
+    return primitive_type(_Type_STRING)
 
 
 def binary(int length=-1):
@@ -387,7 +432,7 @@ def binary(int length=-1):
         width `length`.
     """
     if length == -1:
-        return primitive_type(Type_BINARY)
+        return primitive_type(_Type_BINARY)
 
     cdef shared_ptr[CDataType] fixed_size_binary_type
     fixed_size_binary_type.reset(new CFixedSizeBinaryType(length))
@@ -443,13 +488,13 @@ cdef DataType box_data_type(const shared_ptr[CDataType]& type):
     if type.get() == NULL:
         return None
 
-    if type.get().id() == Type_DICTIONARY:
+    if type.get().id() == _Type_DICTIONARY:
         out = DictionaryType()
-    elif type.get().id() == Type_TIMESTAMP:
+    elif type.get().id() == _Type_TIMESTAMP:
         out = TimestampType()
-    elif type.get().id() == Type_FIXED_SIZE_BINARY:
+    elif type.get().id() == _Type_FIXED_SIZE_BINARY:
         out = FixedSizeBinaryType()
-    elif type.get().id() == Type_DECIMAL:
+    elif type.get().id() == _Type_DECIMAL:
         out = DecimalType()
     else:
         out = DataType()
@@ -732,31 +777,31 @@ cdef class FixedSizeBinaryValue(ArrayValue):
 
 
 cdef dict _scalar_classes = {
-    Type_BOOL: BooleanValue,
-    Type_UINT8: Int8Value,
-    Type_UINT16: Int16Value,
-    Type_UINT32: Int32Value,
-    Type_UINT64: Int64Value,
-    Type_INT8: Int8Value,
-    Type_INT16: Int16Value,
-    Type_INT32: Int32Value,
-    Type_INT64: Int64Value,
-    Type_DATE32: Date32Value,
-    Type_DATE64: Date64Value,
-    Type_TIMESTAMP: TimestampValue,
-    Type_FLOAT: FloatValue,
-    Type_DOUBLE: DoubleValue,
-    Type_LIST: ListValue,
-    Type_BINARY: BinaryValue,
-    Type_STRING: StringValue,
-    Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
-    Type_DECIMAL: DecimalValue,
+    _Type_BOOL: BooleanValue,
+    _Type_UINT8: Int8Value,
+    _Type_UINT16: Int16Value,
+    _Type_UINT32: Int32Value,
+    _Type_UINT64: Int64Value,
+    _Type_INT8: Int8Value,
+    _Type_INT16: Int16Value,
+    _Type_INT32: Int32Value,
+    _Type_INT64: Int64Value,
+    _Type_DATE32: Date32Value,
+    _Type_DATE64: Date64Value,
+    _Type_TIMESTAMP: TimestampValue,
+    _Type_FLOAT: FloatValue,
+    _Type_DOUBLE: DoubleValue,
+    _Type_LIST: ListValue,
+    _Type_BINARY: BinaryValue,
+    _Type_STRING: StringValue,
+    _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
+    _Type_DECIMAL: DecimalValue,
 }
 
 cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
                        int64_t index):
     cdef ArrayValue val
-    if type.type.id() == Type_NA:
+    if type.type.id() == _Type_NA:
         return NA
     elif sp_array.get().IsNull(index):
         return NA
@@ -1306,29 +1351,29 @@ cdef class DictionaryArray(Array):
 
 
 cdef dict _array_classes = {
-    Type_NA: NullArray,
-    Type_BOOL: BooleanArray,
-    Type_UINT8: UInt8Array,
-    Type_UINT16: UInt16Array,
-    Type_UINT32: UInt32Array,
-    Type_UINT64: UInt64Array,
-    Type_INT8: Int8Array,
-    Type_INT16: Int16Array,
-    Type_INT32: Int32Array,
-    Type_INT64: Int64Array,
-    Type_DATE32: Date32Array,
-    Type_DATE64: Date64Array,
-    Type_TIMESTAMP: TimestampArray,
-    Type_TIME32: Time32Array,
-    Type_TIME64: Time64Array,
-    Type_FLOAT: FloatArray,
-    Type_DOUBLE: DoubleArray,
-    Type_LIST: ListArray,
-    Type_BINARY: BinaryArray,
-    Type_STRING: StringArray,
-    Type_DICTIONARY: DictionaryArray,
-    Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
-    Type_DECIMAL: DecimalArray,
+    _Type_NA: NullArray,
+    _Type_BOOL: BooleanArray,
+    _Type_UINT8: UInt8Array,
+    _Type_UINT16: UInt16Array,
+    _Type_UINT32: UInt32Array,
+    _Type_UINT64: UInt64Array,
+    _Type_INT8: Int8Array,
+    _Type_INT16: Int16Array,
+    _Type_INT32: Int32Array,
+    _Type_INT64: Int64Array,
+    _Type_DATE32: Date32Array,
+    _Type_DATE64: Date64Array,
+    _Type_TIMESTAMP: TimestampArray,
+    _Type_TIME32: Time32Array,
+    _Type_TIME64: Time64Array,
+    _Type_FLOAT: FloatArray,
+    _Type_DOUBLE: DoubleArray,
+    _Type_LIST: ListArray,
+    _Type_BINARY: BinaryArray,
+    _Type_STRING: StringArray,
+    _Type_DICTIONARY: DictionaryArray,
+    _Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
+    _Type_DECIMAL: DecimalArray,
 }
 
 cdef object box_array(const shared_ptr[CArray]& sp_array):
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index dafcdaff9bfee..c06eab2630210 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -23,7 +23,7 @@ from cython.operator cimport dereference as deref
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 cimport pyarrow.includes.pyarrow as pyarrow
-from pyarrow._array cimport Array, Schema
+from pyarrow._array cimport Array, Schema, box_schema
 from pyarrow._error cimport check_status
 from pyarrow._memory cimport MemoryPool, maybe_unbox_memory_pool
 from pyarrow._table cimport Table, table_from_ctable
@@ -194,6 +194,27 @@ cdef class ParquetSchema:
     def __getitem__(self, i):
         return self.column(i)
 
+    property names:
+
+        def __get__(self):
+            return [self[i].name for i in range(len(self))]
+
+    def to_arrow_schema(self):
+        """
+        Convert Parquet schema to effective Arrow schema
+
+        Returns
+        -------
+        schema : pyarrow.Schema
+        """
+        cdef:
+            shared_ptr[CSchema] sp_arrow_schema
+
+        with nogil:
+            check_status(FromParquetSchema(self.schema, &sp_arrow_schema))
+
+        return box_schema(sp_arrow_schema)
+
     def equals(self, ParquetSchema other):
         """
         Returns True if the Parquet schemas are equal
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 2444f3fd0683e..b8aa24c65e11b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -1,4 +1,4 @@
-# Licensed to the Apache Software Foundation (ASF) under one
+#t Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
@@ -22,37 +22,37 @@ from pyarrow.includes.common cimport *
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     enum Type" arrow::Type::type":
-        Type_NA" arrow::Type::NA"
-
-        Type_BOOL" arrow::Type::BOOL"
-
-        Type_UINT8" arrow::Type::UINT8"
-        Type_INT8" arrow::Type::INT8"
-        Type_UINT16" arrow::Type::UINT16"
-        Type_INT16" arrow::Type::INT16"
-        Type_UINT32" arrow::Type::UINT32"
-        Type_INT32" arrow::Type::INT32"
-        Type_UINT64" arrow::Type::UINT64"
-        Type_INT64" arrow::Type::INT64"
-
-        Type_HALF_FLOAT" arrow::Type::HALF_FLOAT"
-        Type_FLOAT" arrow::Type::FLOAT"
-        Type_DOUBLE" arrow::Type::DOUBLE"
-
-        Type_DECIMAL" arrow::Type::DECIMAL"
-
-        Type_DATE32" arrow::Type::DATE32"
-        Type_DATE64" arrow::Type::DATE64"
-        Type_TIMESTAMP" arrow::Type::TIMESTAMP"
-        Type_TIME32" arrow::Type::TIME32"
-        Type_TIME64" arrow::Type::TIME64"
-        Type_BINARY" arrow::Type::BINARY"
-        Type_STRING" arrow::Type::STRING"
-        Type_FIXED_SIZE_BINARY" arrow::Type::FIXED_SIZE_BINARY"
-
-        Type_LIST" arrow::Type::LIST"
-        Type_STRUCT" arrow::Type::STRUCT"
-        Type_DICTIONARY" arrow::Type::DICTIONARY"
+        _Type_NA" arrow::Type::NA"
+
+        _Type_BOOL" arrow::Type::BOOL"
+
+        _Type_UINT8" arrow::Type::UINT8"
+        _Type_INT8" arrow::Type::INT8"
+        _Type_UINT16" arrow::Type::UINT16"
+        _Type_INT16" arrow::Type::INT16"
+        _Type_UINT32" arrow::Type::UINT32"
+        _Type_INT32" arrow::Type::INT32"
+        _Type_UINT64" arrow::Type::UINT64"
+        _Type_INT64" arrow::Type::INT64"
+
+        _Type_HALF_FLOAT" arrow::Type::HALF_FLOAT"
+        _Type_FLOAT" arrow::Type::FLOAT"
+        _Type_DOUBLE" arrow::Type::DOUBLE"
+
+        _Type_DECIMAL" arrow::Type::DECIMAL"
+
+        _Type_DATE32" arrow::Type::DATE32"
+        _Type_DATE64" arrow::Type::DATE64"
+        _Type_TIMESTAMP" arrow::Type::TIMESTAMP"
+        _Type_TIME32" arrow::Type::TIME32"
+        _Type_TIME64" arrow::Type::TIME64"
+        _Type_BINARY" arrow::Type::BINARY"
+        _Type_STRING" arrow::Type::STRING"
+        _Type_FIXED_SIZE_BINARY" arrow::Type::FIXED_SIZE_BINARY"
+
+        _Type_LIST" arrow::Type::LIST"
+        _Type_STRUCT" arrow::Type::STRUCT"
+        _Type_DICTIONARY" arrow::Type::DICTIONARY"
 
     enum TimeUnit" arrow::TimeUnit":
         TimeUnit_SECOND" arrow::TimeUnit::SECOND"
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 4ff7e038b5e6c..fef99d5e12a06 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -180,14 +180,13 @@ def _open(self, open_file_func=None):
         """
         Returns instance of ParquetFile
         """
-        if open_file_func is None:
-            def simple_opener(path):
-                return ParquetFile(path)
-            open_file_func = simple_opener
-        return open_file_func(self.path)
+        reader = open_file_func(self.path)
+        if not isinstance(reader, ParquetFile):
+            reader = ParquetFile(reader)
+        return reader
 
     def read(self, columns=None, nthreads=1, partitions=None,
-             open_file_func=None):
+             open_file_func=None, file=None):
         """
         Read this piece as a pyarrow.Table
 
@@ -205,7 +204,10 @@ def read(self, columns=None, nthreads=1, partitions=None,
         -------
         table : pyarrow.Table
         """
-        reader = self._open(open_file_func)
+        if open_file_func is not None:
+            reader = self._open(open_file_func)
+        elif file is not None:
+            reader = ParquetFile(file)
 
         if self.row_group is not None:
             table = reader.read_row_group(self.row_group, columns=columns,
@@ -472,6 +474,8 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
         else:
             self.fs = filesystem
 
+        self.paths = path_or_paths
+
         (self.pieces, self.partitions,
          self.metadata_path) = _make_manifest(path_or_paths, self.fs)
 
@@ -550,6 +554,10 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
     partitions = None
     metadata_path = None
 
+    if len(path_or_paths) == 1:
+        # Dask passes a directory as a list of length 1
+        path_or_paths = path_or_paths[0]
+
     if is_string(path_or_paths) and fs.isdir(path_or_paths):
         manifest = ParquetManifest(path_or_paths, filesystem=fs,
                                    pathsep=pathsep)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index ca6ae2d0b3be0..fc35781c54722 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -552,6 +552,10 @@ def test_read_common_metadata_files(tmpdir):
     pf = pq.ParquetFile(data_path)
     assert dataset.schema.equals(pf.schema)
 
+    # handle list of one directory
+    dataset2 = pq.ParquetDataset([base_path])
+    assert dataset2.schema.equals(dataset.schema)
+
 
 def _filter_partition(df, part_keys):
     predicate = np.ones(len(df), dtype=bool)
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 53b6b68cfde3c..d1107fb1faf3f 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -31,6 +31,34 @@ def test_type_integers():
         assert str(t) == name
 
 
+def test_type_to_pandas_dtype():
+    M8_ns = np.dtype('datetime64[ns]')
+    cases = [
+        (pa.null(), np.float64),
+        (pa.bool_(), np.bool_),
+        (pa.int8(), np.int8),
+        (pa.int16(), np.int16),
+        (pa.int32(), np.int32),
+        (pa.int64(), np.int64),
+        (pa.uint8(), np.uint8),
+        (pa.uint16(), np.uint16),
+        (pa.uint32(), np.uint32),
+        (pa.uint64(), np.uint64),
+        (pa.float16(), np.float16),
+        (pa.float32(), np.float32),
+        (pa.float64(), np.float64),
+        (pa.date32(), M8_ns),
+        (pa.date64(), M8_ns),
+        (pa.timestamp('ms'), M8_ns),
+        (pa.binary(), np.object_),
+        (pa.binary(12), np.object_),
+        (pa.string(), np.object_),
+        (pa.list_(pa.int8()), np.object_),
+    ]
+    for arrow_type, numpy_type in cases:
+        assert arrow_type.to_pandas_dtype() == numpy_type
+
+
 def test_type_list():
     value_type = pa.int32()
     list_type = pa.list_(value_type)
@@ -83,6 +111,8 @@ def test_schema():
     ]
     sch = pa.schema(fields)
 
+    assert sch.names == ['foo', 'bar', 'baz']
+
     assert len(sch) == 3
     assert sch[0].name == 'foo'
     assert sch[0].type == fields[0].type

From 312a665353c420452e98b6b266a5a7cb214c936f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Apr 2017 09:56:53 -0400
Subject: [PATCH 0526/1644] ARROW-707: [Python] Return NullArray for array of
 all None in Array.from_pandas. Revert from_numpy -> from_pandas

per ARROW-838, I reverted the `Array.from_numpy` name to `Array.from_pandas` to reflect that the import is specific to pandas 0.x's memory representation

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #554 from wesm/ARROW-707 and squashes the following commits:

a875257 [Wes McKinney] Rename PyObject_is_null to reflect domain-specific nature
093b057 [Wes McKinney] Check more cases of all nulls. Fix segfault for NaN that resulted from computations
7d97f28 [Wes McKinney] Return NullArray for array of all None in Array.from_pandas. Revert from_numpy -> from_pandas
---
 cpp/src/arrow/python/pandas_convert.cc      | 31 ++++++++++++-------
 python/doc/source/api.rst                   |  1 +
 python/pyarrow/__init__.py                  |  1 +
 python/pyarrow/_array.pxd                   |  4 +++
 python/pyarrow/_array.pyx                   | 18 +++++------
 python/pyarrow/_io.pyx                      |  2 +-
 python/pyarrow/_table.pyx                   |  2 +-
 python/pyarrow/tests/test_array.py          |  4 +--
 python/pyarrow/tests/test_convert_pandas.py | 34 ++++++++++++++-------
 python/pyarrow/tests/test_scalars.py        |  6 ++--
 10 files changed, 65 insertions(+), 38 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index b33aea4565817..5cdcb6fa49602 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -61,8 +61,16 @@ namespace py {
 // ----------------------------------------------------------------------
 // Utility code
 
-static inline bool PyObject_is_null(const PyObject* obj) {
-  return obj == Py_None || obj == numpy_nan;
+static inline bool PyFloat_isnan(const PyObject* obj) {
+  if (PyFloat_Check(obj)) {
+    double val = PyFloat_AS_DOUBLE(obj);
+    return val != val;
+  } else {
+    return false;
+  }
+}
+static inline bool PandasObjectIsNull(const PyObject* obj) {
+  return obj == Py_None || obj == numpy_nan || PyFloat_isnan(obj);
 }
 
 static inline bool PyObject_is_string(const PyObject* obj) {
@@ -158,7 +166,7 @@ static Status AppendObjectStrings(
 
   for (int64_t i = 0; i < objects.size(); ++i) {
     obj = objects[i];
-    if ((have_mask && mask_values[i]) || PyObject_is_null(obj)) {
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder->AppendNull());
     } else if (PyUnicode_Check(obj)) {
       obj = PyUnicode_AsUTF8String(obj);
@@ -197,7 +205,7 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
 
   for (int64_t i = 0; i < objects.size(); ++i) {
     obj = objects[i];
-    if ((have_mask && mask_values[i]) || PyObject_is_null(obj)) {
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder->AppendNull());
     } else if (PyUnicode_Check(obj)) {
       obj = PyUnicode_AsUTF8String(obj);
@@ -519,7 +527,7 @@ Status PandasConverter::ConvertDates() {
     obj = objects[i];
     if (PyDate_CheckExact(obj)) {
       date_builder.Append(UnboxDate<ArrowType>::Unbox(obj));
-    } else if (PyObject_is_null(obj)) {
+    } else if (PandasObjectIsNull(obj)) {
       date_builder.AppendNull();
     } else {
       return InvalidConversion(obj, "date");
@@ -570,7 +578,7 @@ Status PandasConverter::ConvertDecimals() {
         default:
           break;
       }
-    } else if (PyObject_is_null(object)) {
+    } else if (PandasObjectIsNull(object)) {
       decimal_builder.AppendNull();
     } else {
       return InvalidConversion(object, "decimal.Decimal");
@@ -724,7 +732,7 @@ Status PandasConverter::ConvertBooleans() {
   PyObject* obj;
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
-    if ((have_mask && mask_values[i]) || PyObject_is_null(obj)) {
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
       ++null_count;
     } else if (obj == Py_True) {
       BitUtil::SetBit(bitmap, i);
@@ -791,7 +799,7 @@ Status PandasConverter::ConvertObjects() {
     RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
 
     for (int64_t i = 0; i < length_; ++i) {
-      if (PyObject_is_null(objects[i])) {
+      if (PandasObjectIsNull(objects[i])) {
         continue;
       } else if (PyObject_is_string(objects[i])) {
         return ConvertObjectStrings();
@@ -809,7 +817,8 @@ Status PandasConverter::ConvertObjects() {
     }
   }
 
-  return Status::TypeError("Unable to infer type of object array, were all null");
+  out_ = std::make_shared<NullArray>(length_);
+  return Status::OK();
 }
 
 template <int ITEM_TYPE, typename ArrowType>
@@ -833,7 +842,7 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
   ListBuilder list_builder(pool_, value_builder);
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
   for (int64_t i = 0; i < length_; ++i) {
-    if (PyObject_is_null(objects[i])) {
+    if (PandasObjectIsNull(objects[i])) {
       RETURN_NOT_OK(list_builder.AppendNull());
     } else if (PyArray_Check(objects[i])) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
@@ -893,7 +902,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
   ListBuilder list_builder(pool_, value_builder);
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
   for (int64_t i = 0; i < length_; ++i) {
-    if (PyObject_is_null(objects[i])) {
+    if (PandasObjectIsNull(objects[i])) {
       RETURN_NOT_OK(list_builder.AppendNull());
     } else if (PyArray_Check(objects[i])) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 801ab34126c7c..1b7b9bdc8f8c8 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -90,6 +90,7 @@ Array Types
    :toctree: generated/
 
    Array
+   NullArray
    NumericArray
    IntegerArray
    FloatingPointArray
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 506d567b0c508..3db2a4f4dd0c8 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -40,6 +40,7 @@
                             Array, Tensor,
                             from_pylist,
                             from_numpy_dtype,
+                            NullArray,
                             NumericArray, IntegerArray, FloatingPointArray,
                             BooleanArray,
                             Int8Array, UInt8Array,
diff --git a/python/pyarrow/_array.pxd b/python/pyarrow/_array.pxd
index 40413746fc94b..afb0c27d4e1ef 100644
--- a/python/pyarrow/_array.pxd
+++ b/python/pyarrow/_array.pxd
@@ -141,6 +141,10 @@ cdef class Tensor:
     cdef init(self, const shared_ptr[CTensor]& sp_tensor)
 
 
+cdef class NullArray(Array):
+    pass
+
+
 cdef class BooleanArray(Array):
     pass
 
diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/_array.pyx
index c5a595c6c67e5..99ff6f28096ef 100644
--- a/python/pyarrow/_array.pyx
+++ b/python/pyarrow/_array.pyx
@@ -843,9 +843,9 @@ cdef class Array:
         self.type = box_data_type(self.sp_array.get().type())
 
     @staticmethod
-    def from_numpy(obj, mask=None, DataType type=None,
-                   timestamps_to_ms=False,
-                   MemoryPool memory_pool=None):
+    def from_pandas(obj, mask=None, DataType type=None,
+                    timestamps_to_ms=False,
+                    MemoryPool memory_pool=None):
         """
         Convert pandas.Series to an Arrow Array.
 
@@ -878,7 +878,7 @@ cdef class Array:
 
         >>> import pandas as pd
         >>> import pyarrow as pa
-        >>> pa.Array.from_numpy(pd.Series([1, 2]))
+        >>> pa.Array.from_pandas(pd.Series([1, 2]))
         <pyarrow.array.Int64Array object at 0x7f674e4c0e10>
         [
           1,
@@ -886,7 +886,7 @@ cdef class Array:
         ]
 
         >>> import numpy as np
-        >>> pa.Array.from_numpy(pd.Series([1, 2]), np.array([0, 1],
+        >>> pa.Array.from_pandas(pd.Series([1, 2]), np.array([0, 1],
         ... dtype=bool))
         <pyarrow.array.Int64Array object at 0x7f9019e11208>
         [
@@ -1329,14 +1329,14 @@ cdef class DictionaryArray(Array):
                 mask = indices == -1
             else:
                 mask = mask | (indices == -1)
-            arrow_indices = Array.from_numpy(indices, mask=mask,
-                                             memory_pool=memory_pool)
+            arrow_indices = Array.from_pandas(indices, mask=mask,
+                                              memory_pool=memory_pool)
 
         if isinstance(dictionary, Array):
             arrow_dictionary = dictionary
         else:
-            arrow_dictionary = Array.from_numpy(dictionary,
-                                                memory_pool=memory_pool)
+            arrow_dictionary = Array.from_pandas(dictionary,
+                                                 memory_pool=memory_pool)
 
         if not isinstance(arrow_indices, IntegerArray):
             raise ValueError('Indices must be integer type')
diff --git a/python/pyarrow/_io.pyx b/python/pyarrow/_io.pyx
index 9f067fb2166c6..ec37de0d72de9 100644
--- a/python/pyarrow/_io.pyx
+++ b/python/pyarrow/_io.pyx
@@ -1148,7 +1148,7 @@ cdef class FeatherWriter:
         if isinstance(col, Array):
             arr = col
         else:
-            arr = Array.from_numpy(col, mask=mask)
+            arr = Array.from_pandas(col, mask=mask)
 
         cdef c_string c_name = tobytes(name)
 
diff --git a/python/pyarrow/_table.pyx b/python/pyarrow/_table.pyx
index 6558b2ea463fa..78fec75cf3e7d 100644
--- a/python/pyarrow/_table.pyx
+++ b/python/pyarrow/_table.pyx
@@ -321,7 +321,7 @@ cdef _dataframe_to_arrays(df, timestamps_to_ms, Schema schema):
         if schema is not None:
             type = schema.field_by_name(name).type
 
-        arr = Array.from_numpy(col, type=type,
+        arr = Array.from_pandas(col, type=type,
                                 timestamps_to_ms=timestamps_to_ms)
         names.append(name)
         arrays.append(arr)
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 57b17f6cea756..a1fe842c7ab8b 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -162,8 +162,8 @@ def test_dictionary_from_boxed_arrays():
     indices = np.repeat([0, 1, 2], 2)
     dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
 
-    iarr = pa.Array.from_numpy(indices)
-    darr = pa.Array.from_numpy(dictionary)
+    iarr = pa.Array.from_pandas(indices)
+    darr = pa.Array.from_pandas(dictionary)
 
     d1 = pa.DictionaryArray.from_arrays(iarr, darr)
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 2394d638d073e..f3602347a78a6 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -79,8 +79,8 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
 
     def _check_array_roundtrip(self, values, expected=None, mask=None,
                                timestamps_to_ms=False, type=None):
-        arr = pa.Array.from_numpy(values, timestamps_to_ms=timestamps_to_ms,
-                                  mask=mask, type=type)
+        arr = pa.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
+                                   mask=mask, type=type)
         result = arr.to_pandas()
 
         values_nulls = pd.isnull(values)
@@ -125,7 +125,7 @@ def test_float_nulls(self):
         for name, arrow_dtype in dtypes:
             values = np.random.randn(num_values).astype(name)
 
-            arr = pa.Array.from_numpy(values, null_mask)
+            arr = pa.Array.from_pandas(values, null_mask)
             arrays.append(arr)
             fields.append(pa.Field.from_py(name, arrow_dtype))
             values[null_mask] = np.nan
@@ -178,7 +178,7 @@ def test_integer_with_nulls(self):
         for name in int_dtypes:
             values = np.random.randint(0, 100, size=num_values)
 
-            arr = pa.Array.from_numpy(values, null_mask)
+            arr = pa.Array.from_pandas(values, null_mask)
             arrays.append(arr)
 
             expected = values.astype('f8')
@@ -212,7 +212,7 @@ def test_boolean_nulls(self):
         mask = np.random.randint(0, 10, size=num_values) < 3
         values = np.random.randint(0, 10, size=num_values) < 5
 
-        arr = pa.Array.from_numpy(values, mask)
+        arr = pa.Array.from_pandas(values, mask)
 
         expected = values.astype(object)
         expected[mask] = None
@@ -375,11 +375,11 @@ def test_date_objects_typed(self):
         t32 = pa.date32()
         t64 = pa.date64()
 
-        a32 = pa.Array.from_numpy(arr, type=t32)
-        a64 = pa.Array.from_numpy(arr, type=t64)
+        a32 = pa.Array.from_pandas(arr, type=t32)
+        a64 = pa.Array.from_pandas(arr, type=t64)
 
-        a32_expected = pa.Array.from_numpy(arr_i4, mask=mask, type=t32)
-        a64_expected = pa.Array.from_numpy(arr_i8, mask=mask, type=t64)
+        a32_expected = pa.Array.from_pandas(arr_i4, mask=mask, type=t32)
+        a64_expected = pa.Array.from_pandas(arr_i8, mask=mask, type=t64)
 
         assert a32.equals(a32_expected)
         assert a64.equals(a64_expected)
@@ -406,8 +406,8 @@ def test_dates_from_integers(self):
         arr = np.array([17259, 17260, 17261], dtype='int32')
         arr2 = arr.astype('int64') * 86400000
 
-        a1 = pa.Array.from_numpy(arr, type=t1)
-        a2 = pa.Array.from_numpy(arr2, type=t2)
+        a1 = pa.Array.from_pandas(arr, type=t1)
+        a2 = pa.Array.from_pandas(arr2, type=t2)
 
         expected = datetime.date(2017, 4, 3)
         assert a1[0].as_py() == expected
@@ -586,3 +586,15 @@ def test_decimal_128_to_pandas(self):
         converted = pa.Table.from_pandas(expected)
         df = converted.to_pandas()
         tm.assert_frame_equal(df, expected)
+
+    def test_all_nones(self):
+        def _check_series(s):
+            converted = pa.Array.from_pandas(s)
+            assert isinstance(converted, pa.NullArray)
+            assert len(converted) == 3
+            assert converted.null_count == 3
+            assert converted[0] is pa.NA
+
+        _check_series(pd.Series([None] * 3, dtype=object))
+        _check_series(pd.Series([np.nan] * 3, dtype=object))
+        _check_series(pd.Series([np.sqrt(-1)] * 3, dtype=object))
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index f4f275b994228..df2a8980710f8 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -124,7 +124,7 @@ def test_timestamp(self):
 
         for unit in units:
             dtype = 'datetime64[{0}]'.format(unit)
-            arrow_arr = pa.Array.from_numpy(arr.astype(dtype))
+            arrow_arr = pa.Array.from_pandas(arr.astype(dtype))
             expected = pd.Timestamp('2000-01-01 12:34:56')
 
             assert arrow_arr[0].as_py() == expected
@@ -133,8 +133,8 @@ def test_timestamp(self):
             arrow_type = pa.timestamp(unit, tz=tz)
 
             dtype = 'datetime64[{0}]'.format(unit)
-            arrow_arr = pa.Array.from_numpy(arr.astype(dtype),
-                                            type=arrow_type)
+            arrow_arr = pa.Array.from_pandas(arr.astype(dtype),
+                                             type=arrow_type)
             expected = (pd.Timestamp('2000-01-01 12:34:56')
                         .tz_localize('utc')
                         .tz_convert(tz))

From 7238d544c1f0b05a393cdf68b2e2c9485bdb154e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Apr 2017 17:46:11 -0400
Subject: [PATCH 0527/1644] ARROW-734: [C++/Python] Support building PyArrow on
 MSVC

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #553 from wesm/ARROW-734 and squashes the following commits:

6e00485 [Wes McKinney] Restore -undefined,dynamic_lookup linker logic on Linux for Python extensions
5be7e31 [Wes McKinney] do_import_numpy.h no longer needed
2d00e6b [Wes McKinney] Fix Unix library names
1e6bb6e [Wes McKinney] typos
8f4928e [Wes McKinney] More build fixes. Can now import pyarrow
5162203 [Wes McKinney] Remove unneeded exports
024579e [Wes McKinney] Wow, MSVC mangles the name CreateDirectory
990fdc2 [Wes McKinney] Install DLLs fixes in FindArrow.cmake
ccf941e [Wes McKinney] Restore CompilerInfo to original state
1e657ad [Wes McKinney] More fixes. Change TimeUnit to struct-based enum
2be93f0 [Wes McKinney] NumPy initialization / build fixes
1744f83 [Wes McKinney] Add new files
68e2d5b [Wes McKinney] Move NumPy API initialization into libarrow_python
0a2d387 [Wes McKinney] WIP MSVC support for PyArrow. Linker errors
---
 cpp/cmake_modules/BuildUtils.cmake            |  4 ++
 cpp/cmake_modules/CompilerInfo.cmake          |  5 +-
 cpp/cmake_modules/FindPythonLibsNew.cmake     | 15 ++++--
 cpp/src/arrow/io/hdfs-internal.cc             |  2 +-
 cpp/src/arrow/io/hdfs-internal.h              |  2 +-
 cpp/src/arrow/io/hdfs.cc                      |  8 ++--
 cpp/src/arrow/io/hdfs.h                       |  2 +-
 cpp/src/arrow/io/io-hdfs-test.cc              |  8 ++--
 cpp/src/arrow/ipc/feather-internal.h          | 16 +++----
 cpp/src/arrow/ipc/feather.cc                  |  8 ++--
 cpp/src/arrow/ipc/json-internal.cc            |  4 +-
 cpp/src/arrow/ipc/metadata.cc                 |  8 ++--
 cpp/src/arrow/python/CMakeLists.txt           | 34 ++++++-------
 cpp/src/arrow/python/builtin_convert.cc       |  3 +-
 cpp/src/arrow/python/builtin_convert.h        |  3 +-
 cpp/src/arrow/python/common.h                 |  6 +--
 cpp/src/arrow/python/config.cc                |  4 +-
 cpp/src/arrow/python/config.h                 |  6 +--
 cpp/src/arrow/python/helpers.cc               |  6 +--
 cpp/src/arrow/python/helpers.h                | 12 ++---
 .../python/{do_import_numpy.h => init.cc}     | 15 +++++-
 cpp/src/arrow/python/init.h                   | 35 ++++++++++++++
 cpp/src/arrow/python/io.h                     |  2 +-
 cpp/src/arrow/python/numpy-internal.h         |  7 ++-
 cpp/src/arrow/python/numpy_convert.cc         |  9 ++--
 cpp/src/arrow/python/numpy_convert.h          |  7 ++-
 cpp/src/arrow/python/numpy_interop.h          |  2 +-
 cpp/src/arrow/python/pandas_convert.cc        |  5 +-
 cpp/src/arrow/python/pandas_convert.h         |  2 +-
 cpp/src/arrow/python/platform.h               | 32 +++++++++++++
 cpp/src/arrow/python/python-test.cc           |  2 +-
 cpp/src/arrow/python/type_traits.h            |  2 +-
 cpp/src/arrow/python/util/datetime.h          |  2 +-
 cpp/src/arrow/python/util/test_main.cc        |  7 ++-
 cpp/src/arrow/type.cc                         | 14 +++---
 cpp/src/arrow/type.h                          | 32 +++++++------
 python/CMakeLists.txt                         | 34 ++++++-------
 python/cmake_modules/CompilerInfo.cmake       | 48 -------------------
 python/cmake_modules/FindArrow.cmake          | 25 ++++++----
 python/cmake_modules/UseCython.cmake          |  8 ++--
 python/pyarrow/_config.pyx                    | 11 ++---
 python/pyarrow/_io.pyx                        |  2 +-
 python/pyarrow/includes/common.pxd            |  3 +-
 python/pyarrow/includes/libarrow.pxd          |  4 +-
 python/setup.py                               | 28 ++++++-----
 45 files changed, 269 insertions(+), 225 deletions(-)
 rename cpp/src/arrow/python/{do_import_numpy.h => init.cc} (79%)
 create mode 100644 cpp/src/arrow/python/init.h
 create mode 100644 cpp/src/arrow/python/platform.h
 delete mode 100644 python/cmake_modules/CompilerInfo.cmake

diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 3a3b53678f6e5..4e6532be9aa7a 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -102,6 +102,8 @@ function(ADD_ARROW_LIB LIB_NAME)
   # Necessary to make static linking into other shared libraries work properly
   set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
 
+  set(RUNTIME_INSTALL_DIR bin)
+
   if (ARROW_BUILD_SHARED)
     add_library(${LIB_NAME}_shared SHARED $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
 
@@ -139,6 +141,7 @@ function(ADD_ARROW_LIB LIB_NAME)
     endif()
 
     install(TARGETS ${LIB_NAME}_shared
+      RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
       LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
       ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
   endif()
@@ -155,6 +158,7 @@ function(ADD_ARROW_LIB LIB_NAME)
       LINK_PRIVATE ${ARG_STATIC_PRIVATE_LINK_LIBS})
 
   install(TARGETS ${LIB_NAME}_static
+      RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
       LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
       ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
   endif()
diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index 079d9d1f3270d..3c603918a82ec 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -19,8 +19,8 @@
 # Sets COMPILER_VERSION to the version
 execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
                 ERROR_VARIABLE COMPILER_VERSION_FULL)
-message(INFO " ${COMPILER_VERSION_FULL}")
-message(INFO " ${CMAKE_CXX_COMPILER_ID}")
+message(INFO "Compiler version: ${COMPILER_VERSION_FULL}")
+message(INFO "Compiler id: ${CMAKE_CXX_COMPILER_ID}")
 string(TOLOWER "${COMPILER_VERSION_FULL}" COMPILER_VERSION_FULL_LOWER)
 
 if(MSVC)
@@ -62,4 +62,3 @@ else()
   message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
 endif()
 message("Selected compiler ${COMPILER_FAMILY} ${COMPILER_VERSION}")
-
diff --git a/cpp/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake
index d9cc4b3955734..961081609cb12 100644
--- a/cpp/cmake_modules/FindPythonLibsNew.cmake
+++ b/cpp/cmake_modules/FindPythonLibsNew.cmake
@@ -233,12 +233,17 @@ FUNCTION(PYTHON_ADD_MODULE _NAME )
       # segfaults, so do this dynamic lookup instead.
       SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS
                           "-undefined dynamic_lookup")
+    ELSEIF(MSVC)
+      target_link_libraries(${_NAME} ${PYTHON_LIBRARIES})
     ELSE()
-     # In general, we should not link against libpython as we do not embed
-     # the Python interpreter. The python binary itself can then define where
-     # the symbols should loaded from.
-     SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS
-         "-Wl,-undefined,dynamic_lookup")
+      # In general, we should not link against libpython as we do not embed the
+      # Python interpreter. The python binary itself can then define where the
+      # symbols should loaded from. For being manylinux1 compliant, one is not
+      # allowed to link to libpython. Partly because not all systems ship it,
+      # also because the interpreter ABI/API was not stable between patch
+      # releases for Python < 3.5.
+      SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS
+        "-Wl,-undefined,dynamic_lookup")
     ENDIF()
     IF(PYTHON_MODULE_${_NAME}_BUILD_SHARED)
       SET_TARGET_PROPERTIES(${_NAME} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}")
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index e4b2cd55978cb..e67419b5fa501 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -409,7 +409,7 @@ int LibHdfsShim::SetWorkingDirectory(hdfsFS fs, const char* path) {
   }
 }
 
-int LibHdfsShim::CreateDirectory(hdfsFS fs, const char* path) {
+int LibHdfsShim::MakeDirectory(hdfsFS fs, const char* path) {
   return this->hdfsCreateDirectory(fs, path);
 }
 
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
index 01cf1499857d9..c5ea397af0bd5 100644
--- a/cpp/src/arrow/io/hdfs-internal.h
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -173,7 +173,7 @@ struct LibHdfsShim {
 
   int SetWorkingDirectory(hdfsFS fs, const char* path);
 
-  int CreateDirectory(hdfsFS fs, const char* path);
+  int MakeDirectory(hdfsFS fs, const char* path);
 
   int SetReplication(hdfsFS fs, const char* path, int16_t replication);
 
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 3510ba183d8e4..a27e132d155b1 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -347,8 +347,8 @@ class HdfsClient::HdfsClientImpl {
     return Status::OK();
   }
 
-  Status CreateDirectory(const std::string& path) {
-    int ret = driver_->CreateDirectory(fs_, path.c_str());
+  Status MakeDirectory(const std::string& path) {
+    int ret = driver_->MakeDirectory(fs_, path.c_str());
     CHECK_FAILURE(ret, "create directory");
     return Status::OK();
   }
@@ -505,8 +505,8 @@ Status HdfsClient::Connect(
   return Status::OK();
 }
 
-Status HdfsClient::CreateDirectory(const std::string& path) {
-  return impl_->CreateDirectory(path);
+Status HdfsClient::MakeDirectory(const std::string& path) {
+  return impl_->MakeDirectory(path);
 }
 
 Status HdfsClient::Delete(const std::string& path, bool recursive) {
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index e3f5442f48ead..f3de4a2bf174f 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -82,7 +82,7 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   //
   // @param path (in): absolute HDFS path
   // @returns Status
-  Status CreateDirectory(const std::string& path);
+  Status MakeDirectory(const std::string& path);
 
   // Delete file or directory
   // @param path: absolute path to data
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 0a9f5d9885e19..0fdb897d94410 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -45,7 +45,7 @@ class TestHdfsClient : public ::testing::Test {
     if (client_->Exists(scratch_dir_)) {
       RETURN_NOT_OK((client_->Delete(scratch_dir_, true)));
     }
-    return client_->CreateDirectory(scratch_dir_);
+    return client_->MakeDirectory(scratch_dir_);
   }
 
   Status WriteDummyFile(const std::string& path, const uint8_t* buffer, int64_t size,
@@ -161,14 +161,14 @@ TYPED_TEST(TestHdfsClient, ConnectsAgain) {
   ASSERT_OK(client->Disconnect());
 }
 
-TYPED_TEST(TestHdfsClient, CreateDirectory) {
+TYPED_TEST(TestHdfsClient, MakeDirectory) {
   SKIP_IF_NO_DRIVER();
 
   std::string path = this->ScratchPath("create-directory");
 
   if (this->client_->Exists(path)) { ASSERT_OK(this->client_->Delete(path, true)); }
 
-  ASSERT_OK(this->client_->CreateDirectory(path));
+  ASSERT_OK(this->client_->MakeDirectory(path));
   ASSERT_TRUE(this->client_->Exists(path));
   std::vector<HdfsPathInfo> listing;
   EXPECT_OK(this->client_->ListDirectory(path, &listing));
@@ -253,7 +253,7 @@ TYPED_TEST(TestHdfsClient, ListDirectory) {
   ASSERT_OK(this->MakeScratchDir());
   ASSERT_OK(this->WriteDummyFile(p1, data.data(), size));
   ASSERT_OK(this->WriteDummyFile(p2, data.data(), size / 2));
-  ASSERT_OK(this->client_->CreateDirectory(d1));
+  ASSERT_OK(this->client_->MakeDirectory(d1));
 
   std::vector<HdfsPathInfo> listing;
   ASSERT_OK(this->client_->ListDirectory(this->scratch_dir_, &listing));
diff --git a/cpp/src/arrow/ipc/feather-internal.h b/cpp/src/arrow/ipc/feather-internal.h
index 6847445149bb0..646c3b2f9f2e3 100644
--- a/cpp/src/arrow/ipc/feather-internal.h
+++ b/cpp/src/arrow/ipc/feather-internal.h
@@ -75,7 +75,7 @@ struct ARROW_EXPORT CategoryMetadata {
 };
 
 struct ARROW_EXPORT TimestampMetadata {
-  TimeUnit unit;
+  TimeUnit::type unit;
 
   // A timezone name known to the Olson timezone database. For display purposes
   // because the actual data is all UTC
@@ -83,7 +83,7 @@ struct ARROW_EXPORT TimestampMetadata {
 };
 
 struct ARROW_EXPORT TimeMetadata {
-  TimeUnit unit;
+  TimeUnit::type unit;
 };
 
 static constexpr const char* kFeatherMagicBytes = "FEA1";
@@ -156,12 +156,12 @@ static inline flatbuffers::Offset<fbs::PrimitiveArray> GetPrimitiveArray(
       array.length, array.null_count, array.total_bytes);
 }
 
-static inline fbs::TimeUnit ToFlatbufferEnum(TimeUnit unit) {
+static inline fbs::TimeUnit ToFlatbufferEnum(TimeUnit::type unit) {
   return static_cast<fbs::TimeUnit>(static_cast<int>(unit));
 }
 
-static inline TimeUnit FromFlatbufferEnum(fbs::TimeUnit unit) {
-  return static_cast<TimeUnit>(static_cast<int>(unit));
+static inline TimeUnit::type FromFlatbufferEnum(fbs::TimeUnit unit) {
+  return static_cast<TimeUnit::type>(static_cast<int>(unit));
 }
 
 // Convert Feather enums to Flatbuffer enums
@@ -197,10 +197,10 @@ class ARROW_EXPORT ColumnBuilder {
   void SetValues(const ArrayMetadata& values);
   void SetUserMetadata(const std::string& data);
   void SetCategory(const ArrayMetadata& levels, bool ordered = false);
-  void SetTimestamp(TimeUnit unit);
-  void SetTimestamp(TimeUnit unit, const std::string& timezone);
+  void SetTimestamp(TimeUnit::type unit);
+  void SetTimestamp(TimeUnit::type unit, const std::string& timezone);
   void SetDate();
-  void SetTime(TimeUnit unit);
+  void SetTime(TimeUnit::type unit);
   FBB& fbb();
 
  private:
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 5dc039662ce9d..7d0abdda1aadc 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -184,12 +184,12 @@ void ColumnBuilder::SetCategory(const ArrayMetadata& levels, bool ordered) {
   meta_category_.ordered = ordered;
 }
 
-void ColumnBuilder::SetTimestamp(TimeUnit unit) {
+void ColumnBuilder::SetTimestamp(TimeUnit::type unit) {
   type_ = ColumnType::TIMESTAMP;
   meta_timestamp_.unit = unit;
 }
 
-void ColumnBuilder::SetTimestamp(TimeUnit unit, const std::string& timezone) {
+void ColumnBuilder::SetTimestamp(TimeUnit::type unit, const std::string& timezone) {
   SetTimestamp(unit);
   meta_timestamp_.timezone = timezone;
 }
@@ -198,7 +198,7 @@ void ColumnBuilder::SetDate() {
   type_ = ColumnType::DATE;
 }
 
-void ColumnBuilder::SetTime(TimeUnit unit) {
+void ColumnBuilder::SetTime(TimeUnit::type unit) {
   type_ = ColumnType::TIME;
   meta_time_.unit = unit;
 }
@@ -279,7 +279,7 @@ class TableReader::TableReaderImpl {
       }
       case fbs::TypeMetadata_TimestampMetadata: {
         auto meta = static_cast<const fbs::TimestampMetadata*>(metadata);
-        TimeUnit unit = FromFlatbufferEnum(meta->unit());
+        TimeUnit::type unit = FromFlatbufferEnum(meta->unit());
         std::string tz;
         // flatbuffer non-null
         if (meta->timezone() != 0) {
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 18ee8349da66a..2ab3acba37f8d 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -77,7 +77,7 @@ static std::string GetFloatingPrecisionName(FloatingPoint::Precision precision)
   return "UNKNOWN";
 }
 
-static std::string GetTimeUnitName(TimeUnit unit) {
+static std::string GetTimeUnitName(TimeUnit::type unit) {
   switch (unit) {
     case TimeUnit::SECOND:
       return "SECOND";
@@ -645,7 +645,7 @@ static Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>*
 
   std::string unit_str = json_unit->value.GetString();
 
-  TimeUnit unit;
+  TimeUnit::type unit;
   if (unit_str == "SECOND") {
     unit = TimeUnit::SECOND;
   } else if (unit_str == "MILLISECOND") {
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index ee21156c08c67..791948b50b0ac 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -188,7 +188,7 @@ static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
   *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
   break;
 
-static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) {
+static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit::type unit) {
   switch (unit) {
     case TimeUnit::SECOND:
       return flatbuf::TimeUnit_SECOND;
@@ -204,7 +204,7 @@ static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) {
   return flatbuf::TimeUnit_MIN;
 }
 
-static inline TimeUnit FromFlatbufferUnit(flatbuf::TimeUnit unit) {
+static inline TimeUnit::type FromFlatbufferUnit(flatbuf::TimeUnit unit) {
   switch (unit) {
     case flatbuf::TimeUnit_SECOND:
       return TimeUnit::SECOND;
@@ -258,7 +258,7 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     }
     case flatbuf::Type_Time: {
       auto time_type = static_cast<const flatbuf::Time*>(type_data);
-      TimeUnit unit = FromFlatbufferUnit(time_type->unit());
+      TimeUnit::type unit = FromFlatbufferUnit(time_type->unit());
       int32_t bit_width = time_type->bitWidth();
       switch (unit) {
         case TimeUnit::SECOND:
@@ -279,7 +279,7 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     }
     case flatbuf::Type_Timestamp: {
       auto ts_type = static_cast<const flatbuf::Timestamp*>(type_data);
-      TimeUnit unit = FromFlatbufferUnit(ts_type->unit());
+      TimeUnit::type unit = FromFlatbufferUnit(ts_type->unit());
       if (ts_type->timezone() != 0 && ts_type->timezone()->Length() > 0) {
         *out = timestamp(unit, ts_type->timezone()->str());
       } else {
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 8f7991e7f6832..607a1c436c45d 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -17,18 +17,18 @@ if (ARROW_BUILD_TESTS)
   add_library(arrow_python_test_main STATIC
 	util/test_main.cc)
 
+  target_link_libraries(arrow_python_test_main
+    gtest)
+
   if (APPLE)
 	target_link_libraries(arrow_python_test_main
-      gtest
-      dl)
+      ${CMAKE_DL_LIBS})
 	set_target_properties(arrow_python_test_main
       PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-  else()
+  elseif(NOT MSVC)
 	target_link_libraries(arrow_python_test_main
-      gtest
       pthread
-      dl
-	  )
+      ${CMAKE_DL_LIBS})
   endif()
 endif()
 
@@ -38,12 +38,6 @@ set(ARROW_PYTHON_MIN_TEST_LIBS
   arrow_static
   ${BOOST_REGEX_LIBRARY})
 
-if(ARROW_BUILD_TESTS)
-  ADD_THIRDPARTY_LIB(python
-    SHARED_LIB "${PYTHON_LIBRARIES}")
-  list(APPEND ARROW_PYTHON_MIN_TEST_LIBS python)
-endif()
-
 set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
 
 # ----------------------------------------------------------------------
@@ -53,6 +47,7 @@ set(ARROW_PYTHON_SRCS
   common.cc
   config.cc
   helpers.cc
+  init.cc
   io.cc
   numpy_convert.cc
   pandas_convert.cc
@@ -66,9 +61,14 @@ ADD_ARROW_LIB(arrow_python
   SOURCES ${ARROW_PYTHON_SRCS}
   SHARED_LINK_FLAGS ""
   SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS}
-  STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
+  STATIC_LINK_LIBS ""
 )
 
+if (MSVC)
+  target_link_libraries(arrow_python_shared
+    ${PYTHON_LIBRARIES})
+endif()
+
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
   # Clang, be quiet. Python C API has lots of macros
   set_property(SOURCE ${ARROW_PYTHON_SRCS}
@@ -82,19 +82,19 @@ install(FILES
   builtin_convert.h
   common.h
   config.h
-  do_import_numpy.h
   helpers.h
+  init.h
   io.h
   numpy_convert.h
   numpy_interop.h
   pandas_convert.h
+  platform.h
   type_traits.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python")
 
-# set_target_properties(arrow_python_shared PROPERTIES
-#     INSTALL_RPATH "\$ORIGIN")
-
 if (ARROW_BUILD_TESTS)
   ADD_ARROW_TEST(python-test
     STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}")
+  target_link_libraries(python-test
+    ${PYTHON_LIBRARIES})
 endif()
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 8cc9876fa9fc5..137937c0946df 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -15,7 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <Python.h>
+#include "arrow/python/platform.h"
+
 #include <datetime.h>
 
 #include <algorithm>
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index 2141c25e95ef0..a6180d496a920 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -21,13 +21,12 @@
 #ifndef ARROW_PYTHON_ADAPTERS_BUILTIN_H
 #define ARROW_PYTHON_ADAPTERS_BUILTIN_H
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include <memory>
 #include <string>
 
 #include "arrow/type.h"
-
 #include "arrow/util/visibility.h"
 
 #include "arrow/python/common.h"
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index a6806ab95ab95..882bb156224c0 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -32,7 +32,7 @@ class MemoryPool;
 
 namespace py {
 
-class PyAcquireGIL {
+class ARROW_EXPORT PyAcquireGIL {
  public:
   PyAcquireGIL() { state_ = PyGILState_Ensure(); }
 
@@ -45,7 +45,7 @@ class PyAcquireGIL {
 
 #define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
 
-class OwnedRef {
+class ARROW_EXPORT OwnedRef {
  public:
   OwnedRef() : obj_(nullptr) {}
 
@@ -70,7 +70,7 @@ class OwnedRef {
   PyObject* obj_;
 };
 
-struct PyObjectStringify {
+struct ARROW_EXPORT PyObjectStringify {
   OwnedRef tmp_obj;
   const char* bytes;
   Py_ssize_t size;
diff --git a/cpp/src/arrow/python/config.cc b/cpp/src/arrow/python/config.cc
index c2a69168bb01e..3cec7c41a2f31 100644
--- a/cpp/src/arrow/python/config.cc
+++ b/cpp/src/arrow/python/config.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 #include <datetime.h>
 
 #include "arrow/python/config.h"
@@ -23,8 +23,6 @@
 namespace arrow {
 namespace py {
 
-void Init() {}
-
 PyObject* numpy_nan = nullptr;
 
 void set_numpy_nan(PyObject* obj) {
diff --git a/cpp/src/arrow/python/config.h b/cpp/src/arrow/python/config.h
index c13272667540a..c2b089d382c00 100644
--- a/cpp/src/arrow/python/config.h
+++ b/cpp/src/arrow/python/config.h
@@ -18,8 +18,7 @@
 #ifndef ARROW_PYTHON_CONFIG_H
 #define ARROW_PYTHON_CONFIG_H
 
-#include <iostream>
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include "arrow/python/numpy_interop.h"
 #include "arrow/util/visibility.h"
@@ -34,9 +33,6 @@ namespace py {
 ARROW_EXPORT
 extern PyObject* numpy_nan;
 
-ARROW_EXPORT
-void Init();
-
 ARROW_EXPORT
 void set_numpy_nan(PyObject* obj);
 
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 3d3d07a515833..f7c73a87fbf16 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -92,11 +92,11 @@ Status PythonDecimalToArrowDecimal(
   return FromString(c_string, arrow_decimal);
 }
 
-template Status PythonDecimalToArrowDecimal(
+template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal(
     PyObject* python_decimal, decimal::Decimal32* arrow_decimal);
-template Status PythonDecimalToArrowDecimal(
+template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal(
     PyObject* python_decimal, decimal::Decimal64* arrow_decimal);
-template Status PythonDecimalToArrowDecimal(
+template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal(
     PyObject* python_decimal, decimal::Decimal128* arrow_decimal);
 
 Status InferDecimalPrecisionAndScale(
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index 77fde263de7e0..c6402d8796fe2 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -18,7 +18,7 @@
 #ifndef PYARROW_HELPERS_H
 #define PYARROW_HELPERS_H
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include <memory>
 #include <string>
@@ -42,18 +42,18 @@ class OwnedRef;
 
 ARROW_EXPORT std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
 
-Status ImportModule(const std::string& module_name, OwnedRef* ref);
-Status ImportFromModule(
+Status ARROW_EXPORT ImportModule(const std::string& module_name, OwnedRef* ref);
+Status ARROW_EXPORT ImportFromModule(
     const OwnedRef& module, const std::string& module_name, OwnedRef* ref);
 
 template <typename T>
-Status PythonDecimalToArrowDecimal(
+Status ARROW_EXPORT PythonDecimalToArrowDecimal(
     PyObject* python_decimal, decimal::Decimal<T>* arrow_decimal);
 
-Status InferDecimalPrecisionAndScale(
+Status ARROW_EXPORT InferDecimalPrecisionAndScale(
     PyObject* python_decimal, int* precision = nullptr, int* scale = nullptr);
 
-Status DecimalFromString(
+Status ARROW_EXPORT DecimalFromString(
     PyObject* decimal_constructor, const std::string& decimal_string, PyObject** out);
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/do_import_numpy.h b/cpp/src/arrow/python/init.cc
similarity index 79%
rename from cpp/src/arrow/python/do_import_numpy.h
rename to cpp/src/arrow/python/init.cc
index bb4a382959102..fa70af7e44db3 100644
--- a/cpp/src/arrow/python/do_import_numpy.h
+++ b/cpp/src/arrow/python/init.cc
@@ -15,7 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// Trick borrowed from dynd-python for initializing the NumPy array API
+#include "arrow/python/platform.h"
 
 // Trigger the array import (inversion of NO_IMPORT_ARRAY)
 #define NUMPY_IMPORT_ARRAY
+
+#include "arrow/python/init.h"
+#include "arrow/python/numpy_interop.h"
+
+namespace arrow {
+namespace py {
+
+void InitNumPy() {
+  import_numpy();
+}
+
+}  // namespace py
+}  // namespace arrow
diff --git a/cpp/src/arrow/python/init.h b/cpp/src/arrow/python/init.h
new file mode 100644
index 0000000000000..a2533d8059273
--- /dev/null
+++ b/cpp/src/arrow/python/init.h
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PYTHON_INIT_H
+#define ARROW_PYTHON_INIT_H
+
+#include "arrow/python/platform.h"
+
+#include "arrow/python/numpy_interop.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+namespace py {
+
+ARROW_EXPORT
+void InitNumPy();
+
+}  // namespace py
+}  // namespace arrow
+
+#endif  // ARROW_PYTHON_INIT_H
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index 905bd6c7a6aed..bf14cd6f45dbd 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -34,7 +34,7 @@ namespace py {
 
 // A common interface to a Python file-like object. Must acquire GIL before
 // calling any methods
-class PythonFile {
+class ARROW_EXPORT PythonFile {
  public:
   explicit PythonFile(PyObject* file);
   ~PythonFile();
diff --git a/cpp/src/arrow/python/numpy-internal.h b/cpp/src/arrow/python/numpy-internal.h
index fcc6a58f2a347..f1ef7dadde084 100644
--- a/cpp/src/arrow/python/numpy-internal.h
+++ b/cpp/src/arrow/python/numpy-internal.h
@@ -20,12 +20,11 @@
 #ifndef ARROW_PYTHON_NUMPY_INTERNAL_H
 #define ARROW_PYTHON_NUMPY_INTERNAL_H
 
-#include <Python.h>
+#include "arrow/python/numpy_interop.h"
 
-#include <cstdint>
+#include "arrow/python/platform.h"
 
-#include "arrow/python/numpy_convert.h"
-#include "arrow/python/numpy_interop.h"
+#include <cstdint>
 
 namespace arrow {
 namespace py {
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index ab79e179c7ea5..2c1a5910f06d5 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -15,10 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <Python.h>
+#include "arrow/python/numpy_interop.h"
 
 #include "arrow/python/numpy_convert.h"
-#include "arrow/python/numpy_interop.h"
 
 #include <cstdint>
 #include <memory>
@@ -38,8 +37,8 @@ namespace py {
 
 bool is_contiguous(PyObject* array) {
   if (PyArray_Check(array)) {
-    return PyArray_FLAGS(reinterpret_cast<PyArrayObject*>(array)) &
-           (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS);
+    return (PyArray_FLAGS(reinterpret_cast<PyArrayObject*>(array)) &
+               (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS)) != 0;
   } else {
     return false;
   }
@@ -167,7 +166,7 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
     case NPY_DATETIME: {
       auto date_dtype =
           reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-      TimeUnit unit;
+      TimeUnit::type unit;
       switch (date_dtype->meta.base) {
         case NPY_FR_s:
           unit = TimeUnit::SECOND;
diff --git a/cpp/src/arrow/python/numpy_convert.h b/cpp/src/arrow/python/numpy_convert.h
index c2526403213b1..a486646cdec64 100644
--- a/cpp/src/arrow/python/numpy_convert.h
+++ b/cpp/src/arrow/python/numpy_convert.h
@@ -21,7 +21,7 @@
 #ifndef ARROW_PYTHON_NUMPY_CONVERT_H
 #define ARROW_PYTHON_NUMPY_CONVERT_H
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include <memory>
 #include <string>
@@ -48,14 +48,19 @@ class ARROW_EXPORT NumPyBuffer : public Buffer {
 };
 
 // Handle misbehaved types like LONGLONG and ULONGLONG
+ARROW_EXPORT
 int cast_npy_type_compat(int type_num);
 
+ARROW_EXPORT
 bool is_contiguous(PyObject* array);
 
 ARROW_EXPORT
 Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out);
 
+ARROW_EXPORT
 Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out);
+
+ARROW_EXPORT
 Status GetNumPyType(const DataType& type, int* type_num);
 
 ARROW_EXPORT Status NdarrayToTensor(
diff --git a/cpp/src/arrow/python/numpy_interop.h b/cpp/src/arrow/python/numpy_interop.h
index 0a4b425e734f7..b93200cc8972d 100644
--- a/cpp/src/arrow/python/numpy_interop.h
+++ b/cpp/src/arrow/python/numpy_interop.h
@@ -18,7 +18,7 @@
 #ifndef PYARROW_NUMPY_INTEROP_H
 #define PYARROW_NUMPY_INTEROP_H
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include <numpy/numpyconfig.h>
 
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 5cdcb6fa49602..636a3fd15c044 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -17,9 +17,8 @@
 
 // Functions for pandas conversion via NumPy
 
-#include <Python.h>
-
 #include "arrow/python/numpy_interop.h"
+
 #include "arrow/python/pandas_convert.h"
 
 #include <algorithm>
@@ -490,7 +489,7 @@ struct UnboxDate {};
 
 template <>
 struct UnboxDate<Date32Type> {
-  static int64_t Unbox(PyObject* obj) {
+  static int32_t Unbox(PyObject* obj) {
     return PyDate_to_days(reinterpret_cast<PyDateTime_Date*>(obj));
   }
 };
diff --git a/cpp/src/arrow/python/pandas_convert.h b/cpp/src/arrow/python/pandas_convert.h
index fd901d8f09fce..45c8a1a21fe20 100644
--- a/cpp/src/arrow/python/pandas_convert.h
+++ b/cpp/src/arrow/python/pandas_convert.h
@@ -21,7 +21,7 @@
 #ifndef ARROW_PYTHON_ADAPTERS_PANDAS_H
 #define ARROW_PYTHON_ADAPTERS_PANDAS_H
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include <memory>
 #include <string>
diff --git a/cpp/src/arrow/python/platform.h b/cpp/src/arrow/python/platform.h
new file mode 100644
index 0000000000000..38f8e0f1c23ff
--- /dev/null
+++ b/cpp/src/arrow/python/platform.h
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Functions for converting between pandas's NumPy-based data representation
+// and Arrow data structures
+
+#ifndef ARROW_PYTHON_PLATFORM_H
+#define ARROW_PYTHON_PLATFORM_H
+
+#include <Python.h>
+#include <iostream>
+
+// Work around C2528 error
+#if _MSC_VER >= 1900
+#undef timezone
+#endif
+
+#endif  // ARROW_PYTHON_PLATFORM_H
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index a4a11c039b60c..cbc93776f98ef 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -19,7 +19,7 @@
 
 #include <memory>
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h
index c464d65a4946c..26b15bdc9f464 100644
--- a/cpp/src/arrow/python/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include <cstdint>
 #include <limits>
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index 82cf6fc48cad4..852f426c157c2 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -18,7 +18,7 @@
 #ifndef PYARROW_UTIL_DATETIME_H
 #define PYARROW_UTIL_DATETIME_H
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 #include <datetime.h>
 
 namespace arrow {
diff --git a/cpp/src/arrow/python/util/test_main.cc b/cpp/src/arrow/python/util/test_main.cc
index c83514d0dbd37..c24da40aadcf6 100644
--- a/cpp/src/arrow/python/util/test_main.cc
+++ b/cpp/src/arrow/python/util/test_main.cc
@@ -15,18 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <Python.h>
+#include "arrow/python/platform.h"
 
 #include <gtest/gtest.h>
 
-#include "arrow/python/do_import_numpy.h"
-#include "arrow/python/numpy_interop.h"
+#include "arrow/python/init.h"
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
 
   Py_Initialize();
-  arrow::py::import_numpy();
+  arrow::py::InitNumPy();
 
   int ret = RUN_ALL_TESTS();
 
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index a2300d6029e39..2e454ae81886f 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -134,10 +134,10 @@ std::string Date32Type::ToString() const {
 // ----------------------------------------------------------------------
 // Time types
 
-TimeType::TimeType(Type::type type_id, TimeUnit unit)
+TimeType::TimeType(Type::type type_id, TimeUnit::type unit)
     : FixedWidthType(type_id), unit_(unit) {}
 
-Time32Type::Time32Type(TimeUnit unit) : TimeType(Type::TIME32, unit) {
+Time32Type::Time32Type(TimeUnit::type unit) : TimeType(Type::TIME32, unit) {
   DCHECK(unit == TimeUnit::SECOND || unit == TimeUnit::MILLI)
       << "Must be seconds or milliseconds";
 }
@@ -148,7 +148,7 @@ std::string Time32Type::ToString() const {
   return ss.str();
 }
 
-Time64Type::Time64Type(TimeUnit unit) : TimeType(Type::TIME64, unit) {
+Time64Type::Time64Type(TimeUnit::type unit) : TimeType(Type::TIME64, unit) {
   DCHECK(unit == TimeUnit::MICRO || unit == TimeUnit::NANO)
       << "Must be microseconds or nanoseconds";
 }
@@ -338,19 +338,19 @@ std::shared_ptr<DataType> fixed_size_binary(int32_t byte_width) {
   return std::make_shared<FixedSizeBinaryType>(byte_width);
 }
 
-std::shared_ptr<DataType> timestamp(TimeUnit unit) {
+std::shared_ptr<DataType> timestamp(TimeUnit::type unit) {
   return std::make_shared<TimestampType>(unit);
 }
 
-std::shared_ptr<DataType> timestamp(TimeUnit unit, const std::string& timezone) {
+std::shared_ptr<DataType> timestamp(TimeUnit::type unit, const std::string& timezone) {
   return std::make_shared<TimestampType>(unit, timezone);
 }
 
-std::shared_ptr<DataType> time32(TimeUnit unit) {
+std::shared_ptr<DataType> time32(TimeUnit::type unit) {
   return std::make_shared<Time32Type>(unit);
 }
 
-std::shared_ptr<DataType> time64(TimeUnit unit) {
+std::shared_ptr<DataType> time64(TimeUnit::type unit) {
   return std::make_shared<Time64Type>(unit);
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 6810b35f05b70..ea4ea03ff569a 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -538,9 +538,11 @@ class ARROW_EXPORT Date64Type : public DateType {
   static std::string name() { return "date"; }
 };
 
-enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
+struct TimeUnit {
+  enum type { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
+};
 
-static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) {
+static inline std::ostream& operator<<(std::ostream& os, TimeUnit::type unit) {
   switch (unit) {
     case TimeUnit::SECOND:
       os << "s";
@@ -560,11 +562,11 @@ static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) {
 
 class ARROW_EXPORT TimeType : public FixedWidthType {
  public:
-  TimeUnit unit() const { return unit_; }
+  TimeUnit::type unit() const { return unit_; }
 
  protected:
-  TimeType(Type::type type_id, TimeUnit unit);
-  TimeUnit unit_;
+  TimeType(Type::type type_id, TimeUnit::type unit);
+  TimeUnit::type unit_;
 };
 
 class ARROW_EXPORT Time32Type : public TimeType {
@@ -574,7 +576,7 @@ class ARROW_EXPORT Time32Type : public TimeType {
 
   int bit_width() const override { return static_cast<int>(sizeof(c_type) * CHAR_BIT); }
 
-  explicit Time32Type(TimeUnit unit = TimeUnit::MILLI);
+  explicit Time32Type(TimeUnit::type unit = TimeUnit::MILLI);
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -587,7 +589,7 @@ class ARROW_EXPORT Time64Type : public TimeType {
 
   int bit_width() const override { return static_cast<int>(sizeof(c_type) * CHAR_BIT); }
 
-  explicit Time64Type(TimeUnit unit = TimeUnit::MILLI);
+  explicit Time64Type(TimeUnit::type unit = TimeUnit::MILLI);
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -602,21 +604,21 @@ class ARROW_EXPORT TimestampType : public FixedWidthType {
 
   int bit_width() const override { return static_cast<int>(sizeof(int64_t) * CHAR_BIT); }
 
-  explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
+  explicit TimestampType(TimeUnit::type unit = TimeUnit::MILLI)
       : FixedWidthType(Type::TIMESTAMP), unit_(unit) {}
 
-  explicit TimestampType(TimeUnit unit, const std::string& timezone)
+  explicit TimestampType(TimeUnit::type unit, const std::string& timezone)
       : FixedWidthType(Type::TIMESTAMP), unit_(unit), timezone_(timezone) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "timestamp"; }
 
-  TimeUnit unit() const { return unit_; }
+  TimeUnit::type unit() const { return unit_; }
   const std::string& timezone() const { return timezone_; }
 
  private:
-  TimeUnit unit_;
+  TimeUnit::type unit_;
   std::string timezone_;
 };
 
@@ -710,15 +712,15 @@ std::shared_ptr<DataType> ARROW_EXPORT fixed_size_binary(int32_t byte_width);
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type);
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
 
-std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit unit);
+std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit);
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(
-    TimeUnit unit, const std::string& timezone);
+    TimeUnit::type unit, const std::string& timezone);
 
 /// Unit can be either SECOND or MILLI
-std::shared_ptr<DataType> ARROW_EXPORT time32(TimeUnit unit);
+std::shared_ptr<DataType> ARROW_EXPORT time32(TimeUnit::type unit);
 
 /// Unit can be either MICRO or NANO
-std::shared_ptr<DataType> ARROW_EXPORT time64(TimeUnit unit);
+std::shared_ptr<DataType> ARROW_EXPORT time64(TimeUnit::type unit);
 
 std::shared_ptr<DataType> ARROW_EXPORT struct_(
     const std::vector<std::shared_ptr<Field>>& fields);
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 36052bc257232..c1431af67ed55 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -28,7 +28,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../cpp/cmake_mod
 
 include(CMakeParseArguments)
 
-set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cpp/build-support)
+set(BUILD_SUPPORT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../cpp/build-support")
 
 # Allow "make install" to not depend on all targets.
 #
@@ -58,10 +58,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     OFF)
 endif()
 
-if(NOT PYARROW_BUILD_TESTS)
-  set(NO_TESTS 1)
-endif()
-
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
   set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
@@ -73,18 +69,19 @@ endif(CCACHE_FOUND)
 ############################################################
 
 include(BuildUtils)
-include(CompilerInfo)
 include(SetupCxxFlags)
+include(CompilerInfo)
 
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
 
-# Enable perf and other tools to work properly
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
-
-# Suppress Cython warnings
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
+if (NOT MSVC)
+  # Enable perf and other tools to work properly
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
 
+  # Suppress Cython warnings
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
+endif()
 
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
   # Using Clang with ccache causes a bunch of spurious warnings that are
@@ -146,9 +143,10 @@ endif()
 #
 # The gold linker is only for ELF binaries, which OSX doesn't use. We can
 # just skip.
-if (NOT APPLE)
+if (NOT APPLE AND NOT MSVC)
   execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Wl,--version OUTPUT_VARIABLE LINKER_OUTPUT)
 endif ()
+
 if (LINKER_OUTPUT MATCHES "gold")
   if ("${PYARROW_LINK}" STREQUAL "d" AND
       "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
@@ -166,9 +164,6 @@ endif()
 # act on its value.
 if ("${PYARROW_LINK}" STREQUAL "d")
   set(BUILD_SHARED_LIBS ON)
-
-  # Position independent code is only necessary when producing shared objects.
-  add_definitions(-fPIC)
 endif()
 
 # set compile output directory
@@ -188,9 +183,16 @@ if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
 EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
   ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
 else()
-  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
+  if (MSVC)
+    # MSVC makes its own output directories based on the build configuration
+    set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/")
+  else()
+    set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
+  endif()
 endif()
 
+message(STATUS "Build output directory: ${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
 # where to put generated archives (.a files)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
diff --git a/python/cmake_modules/CompilerInfo.cmake b/python/cmake_modules/CompilerInfo.cmake
deleted file mode 100644
index 8e85bdea96ea5..0000000000000
--- a/python/cmake_modules/CompilerInfo.cmake
+++ /dev/null
@@ -1,48 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Sets COMPILER_FAMILY to 'clang' or 'gcc'
-# Sets COMPILER_VERSION to the version
-execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
-                ERROR_VARIABLE COMPILER_VERSION_FULL)
-message(INFO " ${COMPILER_VERSION_FULL}")
-
-# clang on Linux and Mac OS X before 10.9
-if("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
-  set(COMPILER_FAMILY "clang")
-  string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
-    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
-# clang on Mac OS X 10.9 and later
-elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
-  set(COMPILER_FAMILY "clang")
-  string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
-    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
-
-# clang on Mac OS X, XCode 7+. No version replacement is done
-# because Apple no longer advertises the upstream LLVM version.
-elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-.*")
-  set(COMPILER_FAMILY "clang")
-
-# gcc
-elseif("${COMPILER_VERSION_FULL}" MATCHES ".*gcc version.*")
-  set(COMPILER_FAMILY "gcc")
-  string(REGEX REPLACE ".*gcc version ([0-9\\.]+).*" "\\1"
-    COMPILER_VERSION "${COMPILER_VERSION_FULL}")
-else()
-  message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
-endif()
-message("Selected compiler ${COMPILER_FAMILY} ${COMPILER_VERSION}")
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 51a887189ccd4..8e13dd66b9f0f 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -67,18 +67,23 @@ find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
 
 if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   set(ARROW_FOUND TRUE)
-  set(ARROW_LIB_NAME libarrow)
-  set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc)
-  set(ARROW_PYTHON_LIB_NAME libarrow_python)
 
-  set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
-  set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  if (MSVC)
+    set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
+    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
+    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
+    set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
+    set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
+    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
+  else()
+    set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
+    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
+    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
 
-  set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_JEMALLOC_LIB_NAME}.a)
-  set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-
-  set(ARROW_PYTHON_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PYTHON_LIB_NAME}.a)
-  set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
 
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
diff --git a/python/cmake_modules/UseCython.cmake b/python/cmake_modules/UseCython.cmake
index 7c06b023db2cb..7920940e688c7 100644
--- a/python/cmake_modules/UseCython.cmake
+++ b/python/cmake_modules/UseCython.cmake
@@ -122,9 +122,11 @@ function( compile_pyx _name pyx_target_name generated_files pyx_file)
   endif()
   set_source_files_properties( ${_generated_files} PROPERTIES GENERATED TRUE )
 
-  # Cython creates a lot of compiler warning detritus on clang
-  set_source_files_properties(${_generated_files} PROPERTIES
-    COMPILE_FLAGS -Wno-unused-function)
+  if (NOT WIN32)
+    # Cython creates a lot of compiler warning detritus on clang
+    set_source_files_properties(${_generated_files} PROPERTIES
+      COMPILE_FLAGS -Wno-unused-function)
+  endif()
 
   set( ${generated_files} ${_generated_files} PARENT_SCOPE )
 
diff --git a/python/pyarrow/_config.pyx b/python/pyarrow/_config.pyx
index 536f27839ae91..2c1e6bf3143aa 100644
--- a/python/pyarrow/_config.pyx
+++ b/python/pyarrow/_config.pyx
@@ -14,18 +14,13 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-cdef extern from 'arrow/python/do_import_numpy.h':
-    pass
-
-cdef extern from 'arrow/python/numpy_interop.h' namespace 'arrow::py':
-    int import_numpy()
+cdef extern from 'arrow/python/init.h' namespace 'arrow::py':
+    void InitNumPy()
 
 cdef extern from 'arrow/python/config.h' namespace 'arrow::py':
-    void Init()
     void set_numpy_nan(object o)
 
-import_numpy()
-Init()
+InitNumPy()
 
 import numpy as np
 set_numpy_nan(np.nan)
diff --git a/python/pyarrow/_io.pyx b/python/pyarrow/_io.pyx
index ec37de0d72de9..09e8233bcbc2f 100644
--- a/python/pyarrow/_io.pyx
+++ b/python/pyarrow/_io.pyx
@@ -807,7 +807,7 @@ cdef class _HdfsClient:
         cdef c_string c_path = tobytes(path)
         with nogil:
             check_status(self.client.get()
-                         .CreateDirectory(c_path))
+                         .MakeDirectory(c_path))
 
     def delete(self, path, bint recursive=False):
         """
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index ab38ff3084f01..44723faa7400e 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -26,8 +26,7 @@ from libcpp.vector cimport vector
 from cpython cimport PyObject
 cimport cpython
 
-# This must be included for cerr and other things to work
-cdef extern from "<iostream>":
+cdef extern from "arrow/python/platform.h":
     pass
 
 cdef extern from "<Python.h>":
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index b8aa24c65e11b..ea835f6d7bbc8 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -54,7 +54,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         _Type_STRUCT" arrow::Type::STRUCT"
         _Type_DICTIONARY" arrow::Type::DICTIONARY"
 
-    enum TimeUnit" arrow::TimeUnit":
+    enum TimeUnit" arrow::TimeUnit::type":
         TimeUnit_SECOND" arrow::TimeUnit::SECOND"
         TimeUnit_MILLI" arrow::TimeUnit::MILLI"
         TimeUnit_MICRO" arrow::TimeUnit::MICRO"
@@ -435,7 +435,7 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
         CStatus Connect(const HdfsConnectionConfig* config,
                         shared_ptr[CHdfsClient]* client)
 
-        CStatus CreateDirectory(const c_string& path)
+        CStatus MakeDirectory(const c_string& path)
 
         CStatus Delete(const c_string& path, c_bool recursive)
 
diff --git a/python/setup.py b/python/setup.py
index 3991856404bc8..ab71e7858e626 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -91,6 +91,13 @@ def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
+
+        if sys.platform == 'win32':
+            # Cannot do debug builds in Windows unless Python itself is a debug
+            # build
+            if not hasattr(sys, 'gettotalrefcount'):
+                self.build_type = 'release'
+
         self.with_parquet = strtobool(
             os.environ.get('PYARROW_WITH_PARQUET', '0'))
         self.with_jemalloc = strtobool(
@@ -132,13 +139,10 @@ def _run_cmake(self):
                 return
 
         static_lib_option = ''
-        build_tests_option = ''
 
         cmake_options = [
             '-DPYTHON_EXECUTABLE=%s' % sys.executable,
-            '-DPYARROW_BUILD_TESTS=off',
             static_lib_option,
-            build_tests_option,
         ]
 
         if self.with_parquet:
@@ -150,10 +154,10 @@ def _run_cmake(self):
         if self.bundle_arrow_cpp:
             cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
 
-        if sys.platform != 'win32':
-            cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
-                                 .format(self.build_type))
+        cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
+                             .format(self.build_type))
 
+        if sys.platform != 'win32':
             cmake_command = (['cmake', self.extra_cmake_args] +
                              cmake_options + [source])
 
@@ -167,15 +171,15 @@ def _run_cmake(self):
             self.spawn(args)
         else:
             import shlex
-            cmake_generator = 'Visual Studio 14 2015'
-            if is_64_bit:
-                cmake_generator += ' Win64'
+            cmake_generator = 'Visual Studio 14 2015 Win64'
+            if not is_64_bit:
+                raise RuntimeError('Not supported on 32-bit Windows')
+
             # Generate the build files
             extra_cmake_args = shlex.split(self.extra_cmake_args)
             cmake_command = (['cmake'] + extra_cmake_args +
                              cmake_options +
-                             [source,
-                             '-G', cmake_generator])
+                             [source, '-G', cmake_generator])
             if "-G" in self.extra_cmake_args:
                 cmake_command = cmake_command[:-2]
 
@@ -336,7 +340,7 @@ def get_outputs(self):
     use_scm_version={"root": "..", "relative_to": __file__},
     setup_requires=['setuptools_scm', 'cython >= 0.23'],
     install_requires=['numpy >= 1.9', 'six >= 1.0.0'],
-    test_requires=['pytest'],
+    tests_require=['pytest'],
     description="Python library for Apache Arrow",
     long_description=long_description,
     classifiers=[

From 84d725ba2610c778af75060d1c69a4ff8b2a2efc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Apr 2017 17:47:51 -0400
Subject: [PATCH 0528/1644] ARROW-825: [Python] Rename pyarrow.from_pylist to
 pyarrow.array, test on tuples

The idea is to make this function more semantically analogous to `numpy.array` -- convert to native data structure with optional explicit type.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #552 from wesm/ARROW-825 and squashes the following commits:

5d69c70 [Wes McKinney] Update test_jemalloc after ARROW-830
c25fdee [Wes McKinney] Update docstring
3a284b7 [Wes McKinney] Rename pyarrow.from_pylist to pyarrow.array, test on tuples
---
 python/doc/source/api.rst                    |  5 +-
 python/doc/source/install.rst                | 17 +++--
 python/pyarrow/__init__.py                   |  2 +-
 python/pyarrow/_array.pyx                    | 69 ++++++++++----------
 python/pyarrow/parquet.py                    |  4 +-
 python/pyarrow/tests/test_array.py           | 18 ++---
 python/pyarrow/tests/test_convert_builtin.py | 60 +++++++++--------
 python/pyarrow/tests/test_jemalloc.py        |  6 +-
 python/pyarrow/tests/test_parquet.py         |  2 +-
 python/pyarrow/tests/test_scalars.py         | 14 ++--
 python/pyarrow/tests/test_table.py           | 40 ++++++------
 11 files changed, 123 insertions(+), 114 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 1b7b9bdc8f8c8..92e248b686ac0 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -83,12 +83,13 @@ Scalar Value Types
    StringValue
    FixedSizeBinaryValue
 
-Array Types
------------
+Array Types and Constructors
+----------------------------
 
 .. autosummary::
    :toctree: generated/
 
+   array
    Array
    NullArray
    NumericArray
diff --git a/python/doc/source/install.rst b/python/doc/source/install.rst
index 16d19ef123135..278b466941a6f 100644
--- a/python/doc/source/install.rst
+++ b/python/doc/source/install.rst
@@ -90,7 +90,7 @@ using the default system install location will work, but for now we are being
 explicit:
 
 .. code-block:: bash
-    
+
     export ARROW_HOME=$HOME/local
 
 Now, we build Arrow:
@@ -98,18 +98,18 @@ Now, we build Arrow:
 .. code-block:: bash
 
     cd arrow/cpp
-    
+
     mkdir dev-build
     cd dev-build
-    
+
     cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME ..
-    
+
     make
-    
+
     # Use sudo here if $ARROW_HOME requires it
     make install
 
-To get the optional Parquet support, you should also build and install 
+To get the optional Parquet support, you should also build and install
 `parquet-cpp <https://github.com/apache/parquet-cpp/blob/master/README.md>`_.
 
 Install `pyarrow`
@@ -138,10 +138,10 @@ Install `pyarrow`
 
 
 .. code-block:: python
-    
+
     In [1]: import pyarrow
 
-    In [2]: pyarrow.from_pylist([1,2,3])
+    In [2]: pyarrow.array([1,2,3])
     Out[2]:
     <pyarrow.array.Int64Array object at 0x7f899f3e60e8>
     [
@@ -149,4 +149,3 @@ Install `pyarrow`
       2,
       3
     ]
-
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 3db2a4f4dd0c8..87f23524ab49f 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -38,7 +38,7 @@
                             DataType, FixedSizeBinaryType,
                             Field, Schema, schema,
                             Array, Tensor,
-                            from_pylist,
+                            array,
                             from_numpy_dtype,
                             NullArray,
                             NumericArray, IntegerArray, FloatingPointArray,
diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/_array.pyx
index 99ff6f28096ef..e41380d0a6685 100644
--- a/python/pyarrow/_array.pyx
+++ b/python/pyarrow/_array.pyx
@@ -835,6 +835,42 @@ cdef maybe_coerce_datetime64(values, dtype, DataType type,
     return values, type
 
 
+
+def array(object sequence, DataType type=None, MemoryPool memory_pool=None):
+    """
+    Create pyarrow.Array instance from a Python sequence
+
+    Parameters
+    ----------
+    sequence : sequence-like object of Python objects
+    type : pyarrow.DataType, optional
+        If not passed, will be inferred from the data
+    memory_pool : pyarrow.MemoryPool, optional
+        If not passed, will allocate memory from the currently-set default
+        memory pool
+
+    Returns
+    -------
+    array : pyarrow.Array
+    """
+    cdef:
+       shared_ptr[CArray] sp_array
+       CMemoryPool* pool
+
+    pool = maybe_unbox_memory_pool(memory_pool)
+    if type is None:
+        check_status(pyarrow.ConvertPySequence(sequence, pool, &sp_array))
+    else:
+        check_status(
+            pyarrow.ConvertPySequence(
+                sequence, pool, &sp_array, type.sp_type
+            )
+        )
+
+    return box_array(sp_array)
+
+
+
 cdef class Array:
 
     cdef init(self, const shared_ptr[CArray]& sp_array):
@@ -936,36 +972,6 @@ cdef class Array:
 
         return box_array(out)
 
-    @staticmethod
-    def from_list(object list_obj, DataType type=None,
-                  MemoryPool memory_pool=None):
-        """
-        Convert Python list to Arrow array
-
-        Parameters
-        ----------
-        list_obj : array_like
-
-        Returns
-        -------
-        pyarrow.array.Array
-        """
-        cdef:
-           shared_ptr[CArray] sp_array
-           CMemoryPool* pool
-
-        pool = maybe_unbox_memory_pool(memory_pool)
-        if type is None:
-            check_status(pyarrow.ConvertPySequence(list_obj, pool, &sp_array))
-        else:
-            check_status(
-                pyarrow.ConvertPySequence(
-                    list_obj, pool, &sp_array, type.sp_type
-                )
-            )
-
-        return box_array(sp_array)
-
     property null_count:
 
         def __get__(self):
@@ -1408,6 +1414,3 @@ cdef object get_series_values(object obj):
         result = PandasSeries(obj).values
 
     return result
-
-
-from_pylist = Array.from_list
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index fef99d5e12a06..94ad227fbefa9 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -295,9 +295,9 @@ def dictionary(self):
         # Only integer and string partition types are supported right now
         try:
             integer_keys = [int(x) for x in self.keys]
-            dictionary = _array.from_pylist(integer_keys)
+            dictionary = _array.array(integer_keys)
         except ValueError:
-            dictionary = _array.from_pylist(self.keys)
+            dictionary = _array.array(self.keys)
 
         self._dictionary = dictionary
         return dictionary
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index a1fe842c7ab8b..7c91785e12b2a 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -36,12 +36,12 @@ def test_repr_on_pre_init_array():
 
 
 def test_getitem_NA():
-    arr = pa.from_pylist([1, None, 2])
+    arr = pa.array([1, None, 2])
     assert arr[1] is pa.NA
 
 
 def test_list_format():
-    arr = pa.from_pylist([[1], None, [2, 3, None]])
+    arr = pa.array([[1], None, [2, 3, None]])
     result = fmt.array_format(arr)
     expected = """\
 [
@@ -55,7 +55,7 @@ def test_list_format():
 
 
 def test_string_format():
-    arr = pa.from_pylist(['', None, 'foo'])
+    arr = pa.array(['', None, 'foo'])
     result = fmt.array_format(arr)
     expected = """\
 [
@@ -67,7 +67,7 @@ def test_string_format():
 
 
 def test_long_array_format():
-    arr = pa.from_pylist(range(100))
+    arr = pa.array(range(100))
     result = fmt.array_format(arr, window=2)
     expected = """\
 [
@@ -83,7 +83,7 @@ def test_long_array_format():
 def test_to_pandas_zero_copy():
     import gc
 
-    arr = pa.from_pylist(range(10))
+    arr = pa.array(range(10))
 
     for i in range(10):
         np_arr = arr.to_pandas()
@@ -93,7 +93,7 @@ def test_to_pandas_zero_copy():
     assert sys.getrefcount(arr) == 2
 
     for i in range(10):
-        arr = pa.from_pylist(range(10))
+        arr = pa.array(range(10))
         np_arr = arr.to_pandas()
         arr = None
         gc.collect()
@@ -108,14 +108,14 @@ def test_to_pandas_zero_copy():
 
 
 def test_array_slice():
-    arr = pa.from_pylist(range(10))
+    arr = pa.array(range(10))
 
     sliced = arr.slice(2)
-    expected = pa.from_pylist(range(2, 10))
+    expected = pa.array(range(2, 10))
     assert sliced.equals(expected)
 
     sliced2 = arr.slice(2, 4)
-    expected2 = pa.from_pylist(range(2, 6))
+    expected2 = pa.array(range(2, 6))
     assert sliced2.equals(expected2)
 
     # 0 offset
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index d89a8e0c54ceb..d25055d828062 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -23,25 +23,31 @@
 import decimal
 
 
-class TestConvertList(unittest.TestCase):
+class TestConvertSequence(unittest.TestCase):
+
+    def test_sequence_types(self):
+        arr1 = pa.array([1, 2, 3])
+        arr2 = pa.array((1, 2, 3))
+
+        assert arr1.equals(arr2)
 
     def test_boolean(self):
         expected = [True, None, False, None]
-        arr = pa.from_pylist(expected)
+        arr = pa.array(expected)
         assert len(arr) == 4
         assert arr.null_count == 2
         assert arr.type == pa.bool_()
         assert arr.to_pylist() == expected
 
     def test_empty_list(self):
-        arr = pa.from_pylist([])
+        arr = pa.array([])
         assert len(arr) == 0
         assert arr.null_count == 0
         assert arr.type == pa.null()
         assert arr.to_pylist() == []
 
     def test_all_none(self):
-        arr = pa.from_pylist([None, None])
+        arr = pa.array([None, None])
         assert len(arr) == 2
         assert arr.null_count == 2
         assert arr.type == pa.null()
@@ -49,7 +55,7 @@ def test_all_none(self):
 
     def test_integer(self):
         expected = [1, None, 3, None]
-        arr = pa.from_pylist(expected)
+        arr = pa.array(expected)
         assert len(arr) == 4
         assert arr.null_count == 2
         assert arr.type == pa.int64()
@@ -62,13 +68,13 @@ def test_garbage_collection(self):
         gc.collect()
 
         bytes_before = pa.total_allocated_bytes()
-        pa.from_pylist([1, None, 3, None])
+        pa.array([1, None, 3, None])
         gc.collect()
         assert pa.total_allocated_bytes() == bytes_before
 
     def test_double(self):
         data = [1.5, 1, None, 2.5, None, None]
-        arr = pa.from_pylist(data)
+        arr = pa.array(data)
         assert len(arr) == 6
         assert arr.null_count == 3
         assert arr.type == pa.float64()
@@ -76,7 +82,7 @@ def test_double(self):
 
     def test_unicode(self):
         data = [u'foo', u'bar', None, u'mañana']
-        arr = pa.from_pylist(data)
+        arr = pa.array(data)
         assert len(arr) == 4
         assert arr.null_count == 1
         assert arr.type == pa.string()
@@ -87,7 +93,7 @@ def test_bytes(self):
         data = [b'foo',
                 u1.decode('utf-8'),  # unicode gets encoded,
                 None]
-        arr = pa.from_pylist(data)
+        arr = pa.array(data)
         assert len(arr) == 3
         assert arr.null_count == 1
         assert arr.type == pa.binary()
@@ -95,7 +101,7 @@ def test_bytes(self):
 
     def test_fixed_size_bytes(self):
         data = [b'foof', None, b'barb', b'2346']
-        arr = pa.from_pylist(data, type=pa.binary(4))
+        arr = pa.array(data, type=pa.binary(4))
         assert len(arr) == 4
         assert arr.null_count == 1
         assert arr.type == pa.binary(4)
@@ -104,12 +110,12 @@ def test_fixed_size_bytes(self):
     def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
         data = [b'foo', None, b'barb', b'2346']
         with self.assertRaises(pa.ArrowInvalid):
-            pa.from_pylist(data, type=pa.binary(4))
+            pa.array(data, type=pa.binary(4))
 
     def test_date(self):
         data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1),
                 datetime.date(2040, 2, 26)]
-        arr = pa.from_pylist(data)
+        arr = pa.array(data)
         assert len(arr) == 4
         assert arr.type == pa.date64()
         assert arr.null_count == 1
@@ -125,7 +131,7 @@ def test_timestamp(self):
             datetime.datetime(2006, 1, 13, 12, 34, 56, 432539),
             datetime.datetime(2010, 8, 13, 5, 46, 57, 437699)
         ]
-        arr = pa.from_pylist(data)
+        arr = pa.array(data)
         assert len(arr) == 4
         assert arr.type == pa.timestamp('us')
         assert arr.null_count == 1
@@ -138,22 +144,22 @@ def test_timestamp(self):
                                                    46, 57, 437699)
 
     def test_mixed_nesting_levels(self):
-        pa.from_pylist([1, 2, None])
-        pa.from_pylist([[1], [2], None])
-        pa.from_pylist([[1], [2], [None]])
+        pa.array([1, 2, None])
+        pa.array([[1], [2], None])
+        pa.array([[1], [2], [None]])
 
         with self.assertRaises(pa.ArrowInvalid):
-            pa.from_pylist([1, 2, [1]])
+            pa.array([1, 2, [1]])
 
         with self.assertRaises(pa.ArrowInvalid):
-            pa.from_pylist([1, 2, []])
+            pa.array([1, 2, []])
 
         with self.assertRaises(pa.ArrowInvalid):
-            pa.from_pylist([[1], [2], [None, [1]]])
+            pa.array([[1], [2], [None, [1]]])
 
     def test_list_of_int(self):
         data = [[1, 2, 3], [], None, [1, 2]]
-        arr = pa.from_pylist(data)
+        arr = pa.array(data)
         assert len(arr) == 4
         assert arr.null_count == 1
         assert arr.type == pa.list_(pa.int64())
@@ -162,12 +168,12 @@ def test_list_of_int(self):
     def test_mixed_types_fails(self):
         data = ['a', 1, 2.0]
         with self.assertRaises(pa.ArrowException):
-            pa.from_pylist(data)
+            pa.array(data)
 
     def test_decimal(self):
         data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
         type = pa.decimal(precision=7, scale=3)
-        arr = pa.from_pylist(data, type=type)
+        arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_different_precisions(self):
@@ -175,30 +181,30 @@ def test_decimal_different_precisions(self):
             decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
         ]
         type = pa.decimal(precision=13, scale=3)
-        arr = pa.from_pylist(data, type=type)
+        arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_no_scale(self):
         data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
         type = pa.decimal(precision=10)
-        arr = pa.from_pylist(data, type=type)
+        arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_negative(self):
         data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
         type = pa.decimal(precision=10, scale=6)
-        arr = pa.from_pylist(data, type=type)
+        arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_no_whole_part(self):
         data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
         type = pa.decimal(precision=7, scale=7)
-        arr = pa.from_pylist(data, type=type)
+        arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_large_integer(self):
         data = [decimal.Decimal('-394029506937548693.42983'),
                 decimal.Decimal('32358695912932.01033')]
         type = pa.decimal(precision=23, scale=5)
-        arr = pa.from_pylist(data, type=type)
+        arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
diff --git a/python/pyarrow/tests/test_jemalloc.py b/python/pyarrow/tests/test_jemalloc.py
index 0a4d8a63ad2d2..50eb74ae0e2c6 100644
--- a/python/pyarrow/tests/test_jemalloc.py
+++ b/python/pyarrow/tests/test_jemalloc.py
@@ -39,8 +39,8 @@ def test_different_memory_pool():
     bytes_before_jemalloc = pa.jemalloc_memory_pool().bytes_allocated()
 
     # it works
-    array = pa.from_pylist([1, None, 3, None],  # noqa
-                                memory_pool=pa.jemalloc_memory_pool())
+    array = pa.array([1, None, 3, None],  # noqa
+                     memory_pool=pa.jemalloc_memory_pool())
     gc.collect()
     assert pa.total_allocated_bytes() == bytes_before_default
     assert (pa.jemalloc_memory_pool().bytes_allocated() >
@@ -56,7 +56,7 @@ def test_default_memory_pool():
     old_memory_pool = pa.default_memory_pool()
     pa.set_memory_pool(pa.jemalloc_memory_pool())
 
-    array = pa.from_pylist([1, None, 3, None])  # noqa
+    array = pa.array([1, None, 3, None])  # noqa
 
     pa.set_memory_pool(old_memory_pool)
     gc.collect()
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index fc35781c54722..268e87af7dda4 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -47,7 +47,7 @@ def test_single_pylist_column_roundtrip(tmpdir):
     for dtype in [int, float]:
         filename = tmpdir.join('single_{}_column.parquet'
                                .format(dtype.__name__))
-        data = [pa.from_pylist(list(map(dtype, range(5))))]
+        data = [pa.array(list(map(dtype, range(5))))]
         table = pa.Table.from_arrays(data, names=('a', 'b'))
         pq.write_table(table, filename.strpath)
         table_read = pq.read_table(filename.strpath)
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index df2a8980710f8..149973b7831cb 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -29,7 +29,7 @@ def test_null_singleton(self):
             pa.NAType()
 
     def test_bool(self):
-        arr = pa.from_pylist([True, None, False, None])
+        arr = pa.array([True, None, False, None])
 
         v = arr[0]
         assert isinstance(v, pa.BooleanValue)
@@ -39,7 +39,7 @@ def test_bool(self):
         assert arr[1] is pa.NA
 
     def test_int64(self):
-        arr = pa.from_pylist([1, 2, None])
+        arr = pa.array([1, 2, None])
 
         v = arr[0]
         assert isinstance(v, pa.Int64Value)
@@ -49,7 +49,7 @@ def test_int64(self):
         assert arr[2] is pa.NA
 
     def test_double(self):
-        arr = pa.from_pylist([1.5, None, 3])
+        arr = pa.array([1.5, None, 3])
 
         v = arr[0]
         assert isinstance(v, pa.DoubleValue)
@@ -62,7 +62,7 @@ def test_double(self):
         assert v.as_py() == 3.0
 
     def test_string_unicode(self):
-        arr = pa.from_pylist([u'foo', None, u'mañana'])
+        arr = pa.array([u'foo', None, u'mañana'])
 
         v = arr[0]
         assert isinstance(v, pa.StringValue)
@@ -75,7 +75,7 @@ def test_string_unicode(self):
         assert isinstance(v, unicode_type)
 
     def test_bytes(self):
-        arr = pa.from_pylist([b'foo', None, u('bar')])
+        arr = pa.array([b'foo', None, u('bar')])
 
         v = arr[0]
         assert isinstance(v, pa.BinaryValue)
@@ -89,7 +89,7 @@ def test_bytes(self):
 
     def test_fixed_size_bytes(self):
         data = [b'foof', None, b'barb']
-        arr = pa.from_pylist(data, type=pa.binary(4))
+        arr = pa.array(data, type=pa.binary(4))
 
         v = arr[0]
         assert isinstance(v, pa.FixedSizeBinaryValue)
@@ -102,7 +102,7 @@ def test_fixed_size_bytes(self):
         assert isinstance(v, bytes)
 
     def test_list(self):
-        arr = pa.from_pylist([['foo', None], None, ['bar'], []])
+        arr = pa.array([['foo', None], None, ['bar'], []])
 
         v = arr[0]
         assert len(v) == 2
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 79b4c159fd10a..0567e8aba685a 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -29,7 +29,7 @@ class TestColumn(unittest.TestCase):
 
     def test_basics(self):
         data = [
-            pa.from_pylist([-10, -5, 0, 5, 10])
+            pa.array([-10, -5, 0, 5, 10])
         ]
         table = pa.Table.from_arrays(data, names=['a'])
         column = table.column(0)
@@ -40,7 +40,7 @@ def test_basics(self):
         assert column.to_pylist() == [-10, -5, 0, 5, 10]
 
     def test_from_array(self):
-        arr = pa.from_pylist([0, 1, 2, 3, 4])
+        arr = pa.array([0, 1, 2, 3, 4])
 
         col1 = pa.Column.from_array('foo', arr)
         col2 = pa.Column.from_array(pa.field('foo', arr.type), arr)
@@ -49,7 +49,7 @@ def test_from_array(self):
 
     def test_pandas(self):
         data = [
-            pa.from_pylist([-10, -5, 0, 5, 10])
+            pa.array([-10, -5, 0, 5, 10])
         ]
         table = pa.Table.from_arrays(data, names=['a'])
         column = table.column(0)
@@ -61,8 +61,8 @@ def test_pandas(self):
 
 def test_recordbatch_basics():
     data = [
-        pa.from_pylist(range(5)),
-        pa.from_pylist([-10, -5, 0, 5, 10])
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10])
     ]
 
     batch = pa.RecordBatch.from_arrays(data, ['c0', 'c1'])
@@ -78,8 +78,8 @@ def test_recordbatch_basics():
 
 def test_recordbatch_slice():
     data = [
-        pa.from_pylist(range(5)),
-        pa.from_pylist([-10, -5, 0, 5, 10])
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10])
     ]
     names = ['c0', 'c1']
 
@@ -159,8 +159,8 @@ def test_recordbatchlist_schema_equals():
 
 def test_table_basics():
     data = [
-        pa.from_pylist(range(5)),
-        pa.from_pylist([-10, -5, 0, 5, 10])
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10])
     ]
     table = pa.Table.from_arrays(data, names=('a', 'b'))
     assert len(table) == 5
@@ -179,9 +179,9 @@ def test_table_basics():
 
 def test_table_add_column():
     data = [
-        pa.from_pylist(range(5)),
-        pa.from_pylist([-10, -5, 0, 5, 10]),
-        pa.from_pylist(range(5, 10))
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10]),
+        pa.array(range(5, 10))
     ]
     table = pa.Table.from_arrays(data, names=('a', 'b', 'c'))
 
@@ -202,9 +202,9 @@ def test_table_add_column():
 
 def test_table_remove_column():
     data = [
-        pa.from_pylist(range(5)),
-        pa.from_pylist([-10, -5, 0, 5, 10]),
-        pa.from_pylist(range(5, 10))
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10]),
+        pa.array(range(5, 10))
     ]
     table = pa.Table.from_arrays(data, names=('a', 'b', 'c'))
 
@@ -223,15 +223,15 @@ def test_concat_tables():
         [1., 2., 3., 4., 5.]
     ]
 
-    t1 = pa.Table.from_arrays([pa.from_pylist(x) for x in data],
+    t1 = pa.Table.from_arrays([pa.array(x) for x in data],
                               names=('a', 'b'))
-    t2 = pa.Table.from_arrays([pa.from_pylist(x) for x in data2],
+    t2 = pa.Table.from_arrays([pa.array(x) for x in data2],
                               names=('a', 'b'))
 
     result = pa.concat_tables([t1, t2])
     assert len(result) == 10
 
-    expected = pa.Table.from_arrays([pa.from_pylist(x + y)
+    expected = pa.Table.from_arrays([pa.array(x + y)
                                      for x, y in zip(data, data2)],
                                     names=('a', 'b'))
 
@@ -240,8 +240,8 @@ def test_concat_tables():
 
 def test_table_pandas():
     data = [
-        pa.from_pylist(range(5)),
-        pa.from_pylist([-10, -5, 0, 5, 10])
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10])
     ]
     table = pa.Table.from_arrays(data, names=('a', 'b'))
 

From bb8514cc9d7068c8b62d346577370751d68221d8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Apr 2017 17:50:05 -0400
Subject: [PATCH 0529/1644] ARROW-833: [Python] Add Developer quickstart for
 conda users

cc @mrocklin @cpcloud, would you mind giving this a shot to make sure I haven't missed anything idiosyncratic from my environment?

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #548 from wesm/ARROW-833 and squashes the following commits:

a8ff608 [Wes McKinney] Don't check out parquet-cpp in the root of your arrow clone, more instructions
c60fd60 [Wes McKinney] Add build type to pyarrow build command
3cfeb04 [Wes McKinney] Mark blocks as shell
a738820 [Wes McKinney] Add Developer quickstart for conda users
---
 python/DEVELOPMENT.md | 136 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 136 insertions(+)
 create mode 100644 python/DEVELOPMENT.md

diff --git a/python/DEVELOPMENT.md b/python/DEVELOPMENT.md
new file mode 100644
index 0000000000000..280314f702f3d
--- /dev/null
+++ b/python/DEVELOPMENT.md
@@ -0,0 +1,136 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+## Developer guide for conda users
+
+First, set up your thirdparty C++ toolchain using libraries from conda-forge:
+
+```shell
+conda config --add channels conda-forge
+
+export ARROW_BUILD_TYPE=Release
+
+export CPP_TOOLCHAIN=$HOME/cpp-toolchain
+export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
+
+export BOOST_ROOT=$CPP_TOOLCHAIN
+export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
+export RAPIDJSON_HOME=$CPP_TOOLCHAIN
+export THRIFT_HOME=$CPP_TOOLCHAIN
+export ZLIB_HOME=$CPP_TOOLCHAIN
+export SNAPPY_HOME=$CPP_TOOLCHAIN
+export BROTLI_HOME=$CPP_TOOLCHAIN
+export JEMALLOC_HOME=$CPP_TOOLCHAIN
+export ARROW_HOME=$CPP_TOOLCHAIN
+export PARQUET_HOME=$CPP_TOOLCHAIN
+
+conda create -y -q -p $CPP_TOOLCHAIN \
+      flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib brotli jemalloc
+```
+
+Now, activate a conda environment containing your target Python version and
+NumPy installed:
+
+```shell
+conda create -y -q -n pyarrow-dev python=3.6 numpy
+source activate pyarrow-dev
+```
+
+Now, let's clone the Arrow and Parquet git repositories:
+
+```shell
+mkdir repos
+cd repos
+git clone https://github.com/apache/arrow.git
+git clone https://github.com/apache/parquet-cpp.git
+```
+
+You should now see
+
+```shell
+$ ls -l
+total 8
+drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 arrow/
+drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 parquet-cpp/
+```
+
+Now build and install the Arrow C++ libraries:
+
+```shell
+mkdir arrow/cpp/build
+pushd arrow/cpp/build
+
+cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX=$CPP_TOOLCHAIN \
+      -DARROW_PYTHON=on \
+      -DARROW_BUILD_TESTS=OFF \
+      ..
+make -j4
+make install
+popd
+```
+
+Now build and install the Apache Parquet libraries in your toolchain:
+
+```shell
+mkdir parquet-cpp/build
+pushd parquet-cpp/build
+
+cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX=$CPP_TOOLCHAIN \
+      -DPARQUET_BUILD_BENCHMARKS=off \
+      -DPARQUET_BUILD_EXECUTABLES=off \
+      -DPARQUET_ZLIB_VENDORED=off \
+      -DPARQUET_BUILD_TESTS=off \
+      ..
+
+make -j4
+make install
+popd
+```
+
+Now, install requisite build requirements for pyarrow, then build:
+
+```shell
+conda install -y -q six setuptools cython pandas pytest
+
+cd arrow/python
+python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --with-parquet --inplace
+```
+
+You should be able to run the unit tests with:
+
+```shell
+$ py.test pyarrow
+================================ test session starts ================================
+platform linux -- Python 3.6.1, pytest-3.0.7, py-1.4.33, pluggy-0.4.0
+rootdir: /home/wesm/arrow-clone/python, inifile:
+collected 198 items
+
+pyarrow/tests/test_array.py ...........
+pyarrow/tests/test_convert_builtin.py .....................
+pyarrow/tests/test_convert_pandas.py .............................
+pyarrow/tests/test_feather.py ..........................
+pyarrow/tests/test_hdfs.py sssssssssssssss
+pyarrow/tests/test_io.py ..................
+pyarrow/tests/test_ipc.py ........
+pyarrow/tests/test_jemalloc.py ss
+pyarrow/tests/test_parquet.py ....................
+pyarrow/tests/test_scalars.py ..........
+pyarrow/tests/test_schema.py .........
+pyarrow/tests/test_table.py .............
+pyarrow/tests/test_tensor.py ................
+
+====================== 181 passed, 17 skipped in 0.98 seconds =======================
+```

From 0bcb7852feb464790791cf5f9c4da1aaaf429970 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 18 Apr 2017 16:25:02 +0200
Subject: [PATCH 0530/1644] ARROW-839: [Python] Use mktime variant that is
 reliable on MSVC

This also reverts an unintentional regression from https://github.com/apache/arrow/pull/544 when code from config.h was moved to platform.h

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #559 from wesm/ARROW-839 and squashes the following commits:

2e9b300 [Wes McKinney] Use _mkgmtime64 on MSVC
f182bab [Wes McKinney] Restore include order in platform.h
38c29bf [Wes McKinney] Add Windows build instructions for Python
---
 cpp/CMakeLists.txt                   |  4 ++-
 cpp/src/arrow/python/platform.h      |  2 +-
 cpp/src/arrow/python/util/datetime.h |  6 ++++
 python/DEVELOPMENT.md                | 48 ++++++++++++++++++++++++++++
 4 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 08120e9ea68a5..65fb2c9b1f7ea 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -837,7 +837,9 @@ if (${CLANG_FORMAT_FOUND})
   add_custom_target(format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1
     `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h |
     sed -e '/_generated/g' |
-    sed -e '/windows_compatibility.h/g'`)
+    sed -e '/windows_compatibility.h/g' |
+    sed -e '/config.h/g' |
+    sed -e '/platform.h/g'`)
 
   # runs clang format and exits with a non-zero exit code if any files need to be reformatted
   add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0
diff --git a/cpp/src/arrow/python/platform.h b/cpp/src/arrow/python/platform.h
index 38f8e0f1c23ff..a354b38f04cea 100644
--- a/cpp/src/arrow/python/platform.h
+++ b/cpp/src/arrow/python/platform.h
@@ -21,8 +21,8 @@
 #ifndef ARROW_PYTHON_PLATFORM_H
 #define ARROW_PYTHON_PLATFORM_H
 
-#include <Python.h>
 #include <iostream>
+#include <Python.h>
 
 // Work around C2528 error
 #if _MSC_VER >= 1900
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index 852f426c157c2..bd80d9f636890 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -33,7 +33,13 @@ static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   epoch.tm_year = 70;
   epoch.tm_mday = 1;
   // Milliseconds since the epoch
+#ifdef _MSC_VER
+  const int64_t current_timestamp = static_cast<int64_t>(_mkgmtime64(&date));
+  const int64_t epoch_timestamp = static_cast<int64_t>(_mkgmtime64(&epoch));
+  return (current_timestamp - epoch_timestamp) * 1000LL;
+#else
   return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
+#endif
 }
 
 static inline int32_t PyDate_to_days(PyDateTime_Date* pydate) {
diff --git a/python/DEVELOPMENT.md b/python/DEVELOPMENT.md
index 280314f702f3d..ca744628da1b5 100644
--- a/python/DEVELOPMENT.md
+++ b/python/DEVELOPMENT.md
@@ -14,6 +14,8 @@
 
 ## Developer guide for conda users
 
+### Linux and macOS
+
 First, set up your thirdparty C++ toolchain using libraries from conda-forge:
 
 ```shell
@@ -134,3 +136,49 @@ pyarrow/tests/test_tensor.py ................
 
 ====================== 181 passed, 17 skipped in 0.98 seconds =======================
 ```
+
+### Windows
+
+First, make sure you can [build the C++ library][1].
+
+Now, we need to build and install the C++ libraries someplace.
+
+```shell
+mkdir cpp\build
+cd cpp\build
+set ARROW_HOME=C:\thirdparty
+cmake -G "Visual Studio 14 2015 Win64" ^
+      -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
+      -DCMAKE_BUILD_TYPE=Release ^
+      -DARROW_BUILD_TESTS=off ^
+      -DARROW_PYTHON=on ..
+cmake --build . --target INSTALL --config Release
+cd ..\..
+```
+
+After that, we must put the install directory's bin path in our `%PATH%`:
+
+```shell
+set PATH=%ARROW_HOME%\bin;%PATH%
+```
+
+Now, we can build pyarrow:
+
+```shell
+cd python
+python setup.py build_ext --inplace
+```
+
+#### Running C++ unit tests with Python
+
+Getting `python-test.exe` to run is a bit tricky because your `%PYTHONPATH%`
+must be configured given the active conda environment:
+
+```shell
+set CONDA_ENV=C:\Users\wesm\Miniconda\envs\arrow-test
+set PYTHONPATH=%CONDA_ENV%\Lib;%CONDA_ENV%\Lib\site-packages;%CONDA_ENV%\python35.zip;%CONDA_ENV%\DLLs;%CONDA_ENV%
+```
+
+Now `python-test.exe` or simply `ctest` (to run all tests) should work.
+
+[1]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
\ No newline at end of file

From bb287e2030c2b209edc4040099b138866e6e4692 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 18 Apr 2017 16:34:08 +0200
Subject: [PATCH 0531/1644] ARROW-845: [Python] Sync changes from PARQUET-955;
 explicit ARROW_HOME will override pkgconfig

This will avoid build failures due to a stale system-level Arrow install

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #558 from wesm/ARROW-845 and squashes the following commits:

4f89207 [Wes McKinney] Sync changes from PARQUET-955; explicit ARROW_HOME will override pkgconfig
---
 python/cmake_modules/FindArrow.cmake | 92 +++++++++++++++-------------
 1 file changed, 50 insertions(+), 42 deletions(-)

diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 8e13dd66b9f0f..fbe4545a520af 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -25,68 +25,75 @@
 
 include(FindPkgConfig)
 
-set(ARROW_SEARCH_HEADER_PATHS
-  $ENV{ARROW_HOME}/include
-)
+if ("$ENV{ARROW_HOME}" STREQUAL "")
+  pkg_check_modules(ARROW arrow)
+  if (ARROW_FOUND)
+    pkg_get_variable(ARROW_ABI_VERSION arrow abi_version)
+    message(STATUS "Arrow ABI version: ${ARROW_ABI_VERSION}")
+    pkg_get_variable(ARROW_SO_VERSION arrow so_version)
+    message(STATUS "Arrow SO version: ${ARROW_SO_VERSION}")
+    set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
+    set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
+  endif()
+else()
+  set(ARROW_HOME "$ENV{ARROW_HOME}")
 
-set(ARROW_SEARCH_LIB_PATH
-  $ENV{ARROW_HOME}/lib
-)
+  set(ARROW_SEARCH_HEADER_PATHS
+    ${ARROW_HOME}/include
+    )
+
+  set(ARROW_SEARCH_LIB_PATH
+    ${ARROW_HOME}/lib
+    )
 
-pkg_check_modules(ARROW arrow)
-if (ARROW_FOUND)
-  pkg_get_variable(ARROW_ABI_VERSION arrow abi_version)
-  message(STATUS "Arrow ABI version: ${ARROW_ABI_VERSION}")
-  pkg_get_variable(ARROW_SO_VERSION arrow so_version)
-  message(STATUS "Arrow SO version: ${ARROW_SO_VERSION}")
-  set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
-  set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
-else()
   find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
     ${ARROW_SEARCH_HEADER_PATHS}
     # make sure we don't accidentally pick up a different version
     NO_DEFAULT_PATH
-  )
+    )
 
   find_library(ARROW_LIB_PATH NAMES arrow
     PATHS
     ${ARROW_SEARCH_LIB_PATH}
     NO_DEFAULT_PATH)
   get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
-endif()
 
-find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
+  find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
+    PATHS
+    ${ARROW_SEARCH_LIB_PATH}
+    NO_DEFAULT_PATH)
 
-find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
+  find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
+    PATHS
+    ${ARROW_SEARCH_LIB_PATH}
+    NO_DEFAULT_PATH)
 
-if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
-  set(ARROW_FOUND TRUE)
+  if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
+    set(ARROW_FOUND TRUE)
 
-  if (MSVC)
-    set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
-    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
-    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
-    set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
-    set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
-    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
-  else()
-    set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
-    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
-    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
+    if (MSVC)
+      set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
+      set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
+      set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
+      set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
+      set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
+      set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
+    else()
+      set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
+      set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
+      set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
 
-    set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
-    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
-    set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+      set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+      set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
+      set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+    endif()
   endif()
+endif()
 
+if (ARROW_FOUND)
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
+    message(STATUS "Found the Arrow Python library: ${ARROW_PYTHON_LIB_PATH}")
     message(STATUS "Found the Arrow jemalloc library: ${ARROW_JEMALLOC_LIB_PATH}")
   endif ()
 else ()
@@ -105,9 +112,10 @@ endif ()
 
 mark_as_advanced(
   ARROW_INCLUDE_DIR
-  ARROW_LIBS
   ARROW_STATIC_LIB
   ARROW_SHARED_LIB
+  ARROW_PYTHON_STATIC_LIB
+  ARROW_PYTHON_SHARED_LIB
   ARROW_JEMALLOC_STATIC_LIB
   ARROW_JEMALLOC_SHARED_LIB
 )

From 7f20f6e738a2e163b0b753416ee4c4ed00998f4b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 18 Apr 2017 16:37:03 +0200
Subject: [PATCH 0532/1644] ARROW-818: [Python] Expand Sphinx API docs,
 pyarrow.* namespace. Add factory functions for time32, time64

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #557 from wesm/ARROW-818 and squashes the following commits:

96ce436 [Wes McKinney] Expand Sphinx API docs, pyarrow.* namespace. Add factory functions for time32, time64
---
 python/doc/source/api.rst            | 69 +++++++++++++++++++++-----
 python/pyarrow/__init__.py           | 33 ++++++++++---
 python/pyarrow/_array.pxd            | 10 ++++
 python/pyarrow/_array.pyx            | 74 +++++++++++++++++++++++++++-
 python/pyarrow/_io.pyx               |  6 +--
 python/pyarrow/includes/libarrow.pxd |  3 ++
 python/pyarrow/tests/test_io.py      |  4 +-
 python/pyarrow/tests/test_schema.py  | 21 ++++++++
 8 files changed, 195 insertions(+), 25 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 92e248b686ac0..08a06948a3fba 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -24,8 +24,8 @@ API Reference
 
 .. _api.functions:
 
-Type Metadata and Schemas
--------------------------
+Type and Schema Factory Functions
+---------------------------------
 
 .. autosummary::
    :toctree: generated/
@@ -43,6 +43,8 @@ Type Metadata and Schemas
    float16
    float32
    float64
+   time32
+   time64
    timestamp
    date32
    date64
@@ -53,10 +55,8 @@ Type Metadata and Schemas
    struct
    dictionary
    field
-   DataType
-   Field
-   Schema
    schema
+   from_numpy_dtype
 
 Scalar Value Types
 ------------------
@@ -68,6 +68,7 @@ Scalar Value Types
    NAType
    Scalar
    ArrayValue
+   BooleanValue
    Int8Value
    Int16Value
    Int32Value
@@ -82,6 +83,11 @@ Scalar Value Types
    BinaryValue
    StringValue
    FixedSizeBinaryValue
+   Date32Value
+   Date64Value
+   TimestampValue
+   DecimalValue
+
 
 Array Types and Constructors
 ----------------------------
@@ -91,21 +97,30 @@ Array Types and Constructors
 
    array
    Array
-   NullArray
-   NumericArray
-   IntegerArray
-   FloatingPointArray
    BooleanArray
+   DictionaryArray
+   FloatingPointArray
+   IntegerArray
    Int8Array
    Int16Array
    Int32Array
    Int64Array
+   NullArray
+   NumericArray
    UInt8Array
    UInt16Array
    UInt32Array
    UInt64Array
-   DictionaryArray
+   BinaryArray
+   FixedSizeBinaryArray
    StringArray
+   Time32Array
+   Time64Array
+   Date32Array
+   Date64Array
+   TimestampArray
+   DecimalArray
+   ListArray
 
 Tables and Record Batches
 -------------------------
@@ -113,9 +128,11 @@ Tables and Record Batches
 .. autosummary::
    :toctree: generated/
 
+   ChunkedArray
    Column
    RecordBatch
    Table
+   get_record_batch_size
 
 Tensor type and Functions
 -------------------------
@@ -141,7 +158,7 @@ Input / Output and Shared Memory
    MemoryMappedFile
    memory_map
    create_memory_map
-   PythonFileInterface
+   PythonFile
 
 Interprocess Communication and Messaging
 ----------------------------------------
@@ -165,3 +182,33 @@ Memory Pools
    jemalloc_memory_pool
    total_allocated_bytes
    set_memory_pool
+
+Type Classes
+------------
+
+.. autosummary::
+   :toctree: generated/
+
+   DataType
+   DecimalType
+   DictionaryType
+   FixedSizeBinaryType
+   Time32Type
+   Time64Type
+   TimestampType
+   Field
+   Schema
+
+.. currentmodule:: pyarrow.parquet
+
+Apache Parquet
+--------------
+
+.. autosummary::
+   :toctree: generated/
+
+   ParquetDataset
+   ParquetFile
+   read_table
+   write_metadata
+   write_table
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 87f23524ab49f..4d8da9f5a10ed 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -31,12 +31,20 @@
 from pyarrow._array import (null, bool_,
                             int8, int16, int32, int64,
                             uint8, uint16, uint32, uint64,
-                            timestamp, date32, date64,
+                            time32, time64, timestamp, date32, date64,
                             float16, float32, float64,
                             binary, string, decimal,
                             list_, struct, dictionary, field,
-                            DataType, FixedSizeBinaryType,
-                            Field, Schema, schema,
+                            DataType,
+                            DecimalType,
+                            DictionaryType,
+                            FixedSizeBinaryType,
+                            TimestampType,
+                            Time32Type,
+                            Time64Type,
+                            Field,
+                            Schema,
+                            schema,
                             Array, Tensor,
                             array,
                             from_numpy_dtype,
@@ -47,25 +55,34 @@
                             Int16Array, UInt16Array,
                             Int32Array, UInt32Array,
                             Int64Array, UInt64Array,
-                            ListArray, StringArray,
+                            ListArray,
+                            BinaryArray, StringArray,
+                            FixedSizeBinaryArray,
                             DictionaryArray,
+                            Date32Array, Date64Array,
+                            TimestampArray, Time32Array, Time64Array,
+                            DecimalArray,
                             ArrayValue, Scalar, NA, NAType,
                             BooleanValue,
                             Int8Value, Int16Value, Int32Value, Int64Value,
                             UInt8Value, UInt16Value, UInt32Value, UInt64Value,
                             FloatValue, DoubleValue, ListValue,
-                            BinaryValue, StringValue, FixedSizeBinaryValue)
+                            BinaryValue, StringValue, FixedSizeBinaryValue,
+                            DecimalValue,
+                            Date32Value, Date64Value, TimestampValue)
 
-from pyarrow._io import (HdfsFile, NativeFile, PythonFileInterface,
+from pyarrow._io import (HdfsFile, NativeFile, PythonFile,
                          Buffer, BufferReader, InMemoryOutputStream,
                          OSFile, MemoryMappedFile, memory_map,
                          frombuffer, read_tensor, write_tensor,
                          memory_map, create_memory_map,
-                         get_record_batch_size, get_tensor_size)
+                         get_record_batch_size, get_tensor_size,
+                         have_libhdfs, have_libhdfs3)
 
 from pyarrow._memory import (MemoryPool, total_allocated_bytes,
                              set_memory_pool, default_memory_pool)
-from pyarrow._table import Column, RecordBatch, Table, concat_tables
+from pyarrow._table import (ChunkedArray, Column, RecordBatch, Table,
+                            concat_tables)
 from pyarrow._error import (ArrowException,
                             ArrowKeyError,
                             ArrowInvalid,
diff --git a/python/pyarrow/_array.pxd b/python/pyarrow/_array.pxd
index afb0c27d4e1ef..464de316f0437 100644
--- a/python/pyarrow/_array.pxd
+++ b/python/pyarrow/_array.pxd
@@ -42,6 +42,16 @@ cdef class TimestampType(DataType):
         const CTimestampType* ts_type
 
 
+cdef class Time32Type(DataType):
+    cdef:
+        const CTime32Type* time_type
+
+
+cdef class Time64Type(DataType):
+    cdef:
+        const CTime64Type* time_type
+
+
 cdef class FixedSizeBinaryType(DataType):
     cdef:
         const CFixedSizeBinaryType* fixed_size_binary_type
diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/_array.pyx
index e41380d0a6685..1c571ba153dfa 100644
--- a/python/pyarrow/_array.pyx
+++ b/python/pyarrow/_array.pyx
@@ -127,6 +127,30 @@ cdef class TimestampType(DataType):
                 return None
 
 
+cdef class Time32Type(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.time_type = <const CTime32Type*> type.get()
+
+    property unit:
+
+        def __get__(self):
+            return timeunit_to_string(self.time_type.unit())
+
+
+cdef class Time64Type(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.time_type = <const CTime64Type*> type.get()
+
+    property unit:
+
+        def __get__(self):
+            return timeunit_to_string(self.time_type.unit())
+
+
 cdef class FixedSizeBinaryType(DataType):
 
     cdef void init(self, const shared_ptr[CDataType]& type):
@@ -342,6 +366,7 @@ def int64():
 
 
 cdef dict _timestamp_type_cache = {}
+cdef dict _time_type_cache = {}
 
 
 cdef timeunit_to_string(TimeUnit unit):
@@ -369,7 +394,7 @@ def timestamp(unit_str, tz=None):
     elif unit_str == 'ns':
         unit = TimeUnit_NANO
     else:
-        raise TypeError('Invalid TimeUnit string')
+        raise ValueError('Invalid TimeUnit string')
 
     cdef TimestampType out = TimestampType()
 
@@ -388,6 +413,50 @@ def timestamp(unit_str, tz=None):
     return out
 
 
+def time32(unit_str):
+    cdef:
+        TimeUnit unit
+        c_string c_timezone
+
+    if unit_str == "s":
+        unit = TimeUnit_SECOND
+    elif unit_str == 'ms':
+        unit = TimeUnit_MILLI
+    else:
+        raise ValueError('Invalid TimeUnit for time32: {}'.format(unit_str))
+
+    cdef Time32Type out
+    if unit in _time_type_cache:
+        return _time_type_cache[unit]
+    else:
+        out = Time32Type()
+        out.init(ctime32(unit))
+        _time_type_cache[unit] = out
+        return out
+
+
+def time64(unit_str):
+    cdef:
+        TimeUnit unit
+        c_string c_timezone
+
+    if unit_str == "us":
+        unit = TimeUnit_MICRO
+    elif unit_str == 'ns':
+        unit = TimeUnit_NANO
+    else:
+        raise ValueError('Invalid TimeUnit for time64: {}'.format(unit_str))
+
+    cdef Time64Type out
+    if unit in _time_type_cache:
+        return _time_type_cache[unit]
+    else:
+        out = Time64Type()
+        out.init(ctime64(unit))
+        _time_type_cache[unit] = out
+        return out
+
+
 def date32():
     return primitive_type(_Type_DATE32)
 
@@ -516,6 +585,9 @@ cdef Schema box_schema(const shared_ptr[CSchema]& type):
 
 
 def from_numpy_dtype(object dtype):
+    """
+    Convert NumPy dtype to pyarrow.DataType
+    """
     cdef shared_ptr[CDataType] c_type
     with nogil:
         check_status(pyarrow.NumPyDtypeToArrow(dtype, &c_type))
diff --git a/python/pyarrow/_io.pyx b/python/pyarrow/_io.pyx
index 09e8233bcbc2f..40c76f8363cd2 100644
--- a/python/pyarrow/_io.pyx
+++ b/python/pyarrow/_io.pyx
@@ -307,7 +307,7 @@ cdef class NativeFile:
 # Python file-like objects
 
 
-cdef class PythonFileInterface(NativeFile):
+cdef class PythonFile(NativeFile):
     cdef:
         object handle
 
@@ -600,7 +600,7 @@ cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
         source = BufferReader(source)
     elif not isinstance(source, NativeFile) and hasattr(source, 'read'):
         # Optimistically hope this is file-like
-        source = PythonFileInterface(source, mode='r')
+        source = PythonFile(source, mode='r')
 
     if isinstance(source, NativeFile):
         nf = source
@@ -622,7 +622,7 @@ cdef get_writer(object source, shared_ptr[OutputStream]* writer):
         source = OSFile(source, mode='w')
     elif not isinstance(source, NativeFile) and hasattr(source, 'write'):
         # Optimistically hope this is file-like
-        source = PythonFileInterface(source, mode='w')
+        source = PythonFile(source, mode='w')
 
     if isinstance(source, NativeFile):
         nf = source
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index ea835f6d7bbc8..473a0b9cd9b6d 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -106,6 +106,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CTime64Type" arrow::Time64Type"(CFixedWidthType):
         TimeUnit unit()
 
+    shared_ptr[CDataType] ctime32" arrow::time32"(TimeUnit unit)
+    shared_ptr[CDataType] ctime64" arrow::time64"(TimeUnit unit)
+
     cdef cppclass CDictionaryType" arrow::DictionaryType"(CFixedWidthType):
         CDictionaryType(const shared_ptr[CDataType]& index_type,
                         const shared_ptr[CArray]& dictionary)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index c5d3708d6a9ac..a14898ff2ffd0 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -32,7 +32,7 @@
 def test_python_file_write():
     buf = BytesIO()
 
-    f = pa.PythonFileInterface(buf)
+    f = pa.PythonFile(buf)
 
     assert f.tell() == 0
 
@@ -56,7 +56,7 @@ def test_python_file_read():
     data = b'some sample data'
 
     buf = BytesIO(data)
-    f = pa.PythonFileInterface(buf, mode='r')
+    f = pa.PythonFile(buf, mode='r')
 
     assert f.size() == len(data)
 
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index d1107fb1faf3f..da704f378873b 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -77,6 +77,27 @@ def test_type_timestamp_with_tz():
     assert t.tz == tz
 
 
+def test_time_types():
+    t1 = pa.time32('s')
+    t2 = pa.time32('ms')
+    t3 = pa.time64('us')
+    t4 = pa.time64('ns')
+
+    assert t1.unit == 's'
+    assert t2.unit == 'ms'
+    assert t3.unit == 'us'
+    assert t4.unit == 'ns'
+
+    assert str(t1) == 'time32[s]'
+    assert str(t4) == 'time64[ns]'
+
+    with pytest.raises(ValueError):
+        pa.time32('us')
+
+    with pytest.raises(ValueError):
+        pa.time64('s')
+
+
 def test_type_from_numpy_dtype_timestamps():
     cases = [
         (np.dtype('datetime64[s]'), pa.timestamp('s')),

From 38efabea9bbc8d6386f96a635a95c53ba70e6149 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 18 Apr 2017 11:43:13 -0400
Subject: [PATCH 0533/1644] ARROW-844: [Format] Update README documents in
 format/

Added a section reflecting specification maturity and stability.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #556 from wesm/ARROW-844 and squashes the following commits:

03dbb71 [Wes McKinney] Update README documents in format/
---
 format/README.md | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/format/README.md b/format/README.md
index 048badb12214b..3aa8fdd6d4d6e 100644
--- a/format/README.md
+++ b/format/README.md
@@ -14,16 +14,14 @@
 
 ## Arrow specification documents
 
-> **Work-in-progress specification documents**. These are discussion documents
-> created by the Arrow developers during late 2015 and in no way represents a
-> finalized specification.
-
 Currently, the Arrow specification consists of these pieces:
 
 - Metadata specification (see Metadata.md)
 - Physical memory layout specification (see Layout.md)
-- Metadata serialized representation (see Message.fbs)
+- Logical Types, Schemas, and Record Batch Metadata (see Schema.fbs)
+- Encapsulated Messages (see Message.fbs)
 - Mechanics of messaging between Arrow systems (IPC, RPC, etc.) (see IPC.md)
+- Tensor (Multi-dimensional array) Metadata (see Tensor.fbs)
 
 The metadata currently uses Google's [flatbuffers library][1] for serializing a
 couple related pieces of information:
@@ -35,4 +33,16 @@ couple related pieces of information:
    schema, and enable a system to send and receive Arrow row batches in a form
    that can be precisely disassembled or reconstructed.
 
+## Arrow Format Maturity and Stability
+
+We have made significant progress hardening the Arrow in-memory format and
+Flatbuffer metadata since the project started in February 2016. We have
+integration tests which verify binary compatibility between the Java and C++
+implementations, for example.
+
+Major versions may still include breaking changes to the memory format or
+metadata, so it is recommended to use the same released version of all
+libraries in your applications for maximum compatibility. Data stored in the
+Arrow IPC formats should not be used for long term storage.
+
 [1]: http://github.com/google/flatbuffers

From 4baaa88c3f36d92ffe44f70198c510b7b326932c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 18 Apr 2017 21:11:07 -0400
Subject: [PATCH 0534/1644] ARROW-847: Specify BUILD_BYPRODUCTS for gtest

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #561 from xhochy/ARROW-847 and squashes the following commits:

e8d5439 [Uwe L. Korn] ARROW-847: Specify BUILD_BYPRODUCTS for gtest
---
 cpp/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 65fb2c9b1f7ea..5d8a0f6f9dd45 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -486,6 +486,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
       # BUILD_BYPRODUCTS is a 3.2+ feature
       ExternalProject_Add(googletest_ep
         URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+        BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB}
         CMAKE_ARGS ${GTEST_CMAKE_ARGS})
     else()
       ExternalProject_Add(googletest_ep

From a94c03a02f1da8fa61ac86ba2d6c5e91d29c5767 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 18 Apr 2017 21:12:06 -0400
Subject: [PATCH 0535/1644] ARROW-809: [C++] Do not write excess bytes in IPC
 writer after slicing arrays

cc @itaiin

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #555 from wesm/ARROW-809 and squashes the following commits:

318c748 [Wes McKinney] Fix compiler warning
7fd6410 [Wes McKinney] Add sparse union test
290a300 [Wes McKinney] clang-format
1d14aa8 [Wes McKinney] Buffer truncation for unions
51f450f [Wes McKinney] Fix struct
7da5cac [Wes McKinney] Add List test and fix implementation. Fix list comparison bug for sliced arrays
9da3936 [Wes McKinney] Refactor to construct explicit non-nullable arrays
33eaa53 [Wes McKinney] Sliced array buffer truncation for fixed size types, string/binary
---
 cpp/CMakeLists.txt                       |   5 +-
 cpp/src/arrow/compare.cc                 |  10 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  69 ++++++++++
 cpp/src/arrow/ipc/test-common.h          |  50 ++++---
 cpp/src/arrow/ipc/writer.cc              | 158 +++++++++++++++--------
 cpp/src/arrow/pretty_print.cc            |   6 +-
 cpp/src/arrow/python/util/datetime.h     |   2 +-
 cpp/src/arrow/table.cc                   |   4 +-
 cpp/src/arrow/table.h                    |   4 +-
 9 files changed, 216 insertions(+), 92 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5d8a0f6f9dd45..f702da16e7a4e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -839,8 +839,9 @@ if (${CLANG_FORMAT_FOUND})
     `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h |
     sed -e '/_generated/g' |
     sed -e '/windows_compatibility.h/g' |
-    sed -e '/config.h/g' |
-    sed -e '/platform.h/g'`)
+    sed -e '/config.h/g' |   # python/config.h
+    sed -e '/platform.h/g'`  # python/platform.h
+    )
 
   # runs clang format and exits with a non-zero exit code if any files need to be reformatted
   add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index e02f3f0a9a69c..ccb299e53a11e 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -460,14 +460,8 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       return Status::OK();
     }
 
-    if (left.offset() == 0 && right.offset() == 0) {
-      result_ = left.values()->Equals(right.values());
-    } else {
-      // One of the arrays is sliced
-      result_ = left.values()->RangeEquals(left.value_offset(0),
-          left.value_offset(left.length()), right.value_offset(0), right.values());
-    }
-
+    result_ = left.values()->RangeEquals(left.value_offset(0),
+        left.value_offset(left.length()), right.value_offset(0), right.values());
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index cfba0d0a95106..b39136ec12d04 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -270,6 +270,75 @@ TEST_P(TestIpcRoundTrip, ZeroLengthArrays) {
   CheckRoundtrip(bin_array2, 1 << 20);
 }
 
+TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) {
+  auto CheckArray = [this](const std::shared_ptr<Array>& array) {
+    auto f0 = field("f0", array->type());
+    auto schema = std::shared_ptr<Schema>(new Schema({f0}));
+    RecordBatch batch(schema, array->length(), {array});
+    auto sliced_batch = batch.Slice(0, 5);
+
+    int64_t full_size;
+    int64_t sliced_size;
+
+    ASSERT_OK(GetRecordBatchSize(batch, &full_size));
+    ASSERT_OK(GetRecordBatchSize(*sliced_batch, &sliced_size));
+    ASSERT_TRUE(sliced_size < full_size) << sliced_size << " " << full_size;
+
+    // make sure we can write and read it
+    this->CheckRoundtrip(*sliced_batch, 1 << 20);
+  };
+
+  std::shared_ptr<Array> a0, a1;
+  auto pool = default_memory_pool();
+
+  // Integer
+  ASSERT_OK(MakeRandomInt32Array(500, false, pool, &a0));
+  CheckArray(a0);
+
+  // String / Binary
+  {
+    auto s = MakeRandomBinaryArray<StringBuilder, char>(500, false, pool, &a0);
+    ASSERT_TRUE(s.ok());
+  }
+  CheckArray(a0);
+
+  // Boolean
+  ASSERT_OK(MakeRandomBooleanArray(10000, false, &a0));
+  CheckArray(a0);
+
+  // List
+  ASSERT_OK(MakeRandomInt32Array(500, false, pool, &a0));
+  ASSERT_OK(MakeRandomListArray(a0, 200, false, pool, &a1));
+  CheckArray(a1);
+
+  // Struct
+  auto struct_type = struct_({field("f0", a0->type())});
+  std::vector<std::shared_ptr<Array>> struct_children = {a0};
+  a1 = std::make_shared<StructArray>(struct_type, a0->length(), struct_children);
+  CheckArray(a1);
+
+  // Sparse Union
+  auto union_type = union_({field("f0", a0->type())}, {0});
+  std::vector<int32_t> type_ids(a0->length());
+  std::shared_ptr<Buffer> ids_buffer;
+  ASSERT_OK(test::CopyBufferFromVector(type_ids, &ids_buffer));
+  a1 = std::make_shared<UnionArray>(
+      union_type, a0->length(), struct_children, ids_buffer);
+  CheckArray(a1);
+
+  // Dense union
+  auto dense_union_type = union_({field("f0", a0->type())}, {0}, UnionMode::DENSE);
+  std::vector<int32_t> type_offsets;
+  for (int32_t i = 0; i < a0->length(); ++i) {
+    type_offsets.push_back(i);
+  }
+  std::shared_ptr<Buffer> offsets_buffer;
+  ASSERT_OK(test::CopyBufferFromVector(type_offsets, &offsets_buffer));
+  a1 = std::make_shared<UnionArray>(
+      dense_union_type, a0->length(), struct_children, ids_buffer, offsets_buffer);
+  CheckArray(a1);
+}
+
 void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   ipc::MockOutputStream mock;
   int32_t mock_metadata_length = -1;
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index a17b609bbcba2..c8ca21cb8f17d 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -138,31 +138,41 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
 
 typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
 
-Status MakeBooleanBatch(std::shared_ptr<RecordBatch>* out) {
-  const int length = 1000;
+Status MakeRandomBooleanArray(
+    const int length, bool include_nulls, std::shared_ptr<Array>* out) {
+  std::vector<uint8_t> values(length);
+  test::random_null_bytes(length, 0.5, values.data());
+  auto data = test::bytes_to_null_buffer(values);
 
+  if (include_nulls) {
+    std::vector<uint8_t> valid_bytes(length);
+    auto null_bitmap = test::bytes_to_null_buffer(valid_bytes);
+    test::random_null_bytes(length, 0.1, valid_bytes.data());
+    *out = std::make_shared<BooleanArray>(length, data, null_bitmap, -1);
+  } else {
+    *out = std::make_shared<BooleanArray>(length, data, nullptr, 0);
+  }
+  return Status::OK();
+}
+
+Status MakeBooleanBatchSized(const int length, std::shared_ptr<RecordBatch>* out) {
   // Make the schema
   auto f0 = field("f0", boolean());
   auto f1 = field("f1", boolean());
   std::shared_ptr<Schema> schema(new Schema({f0, f1}));
 
-  std::vector<uint8_t> values(length);
-  std::vector<uint8_t> valid_bytes(length);
-  test::random_null_bytes(length, 0.5, values.data());
-  test::random_null_bytes(length, 0.1, valid_bytes.data());
-
-  auto data = test::bytes_to_null_buffer(values);
-  auto null_bitmap = test::bytes_to_null_buffer(valid_bytes);
-
-  auto a0 = std::make_shared<BooleanArray>(length, data, null_bitmap, -1);
-  auto a1 = std::make_shared<BooleanArray>(length, data, nullptr, 0);
+  std::shared_ptr<Array> a0, a1;
+  RETURN_NOT_OK(MakeRandomBooleanArray(length, true, &a0));
+  RETURN_NOT_OK(MakeRandomBooleanArray(length, false, &a1));
   out->reset(new RecordBatch(schema, length, {a0, a1}));
   return Status::OK();
 }
 
-Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
-  const int length = 10;
+Status MakeBooleanBatch(std::shared_ptr<RecordBatch>* out) {
+  return MakeBooleanBatchSized(1000, out);
+}
 
+Status MakeIntBatchSized(int length, std::shared_ptr<RecordBatch>* out) {
   // Make the schema
   auto f0 = field("f0", int32());
   auto f1 = field("f1", int32());
@@ -177,16 +187,20 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
+Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
+  return MakeIntBatchSized(10, out);
+}
+
 template <class Builder, class RawType>
 Status MakeRandomBinaryArray(
-    int64_t length, MemoryPool* pool, std::shared_ptr<Array>* out) {
+    int64_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
   const std::vector<std::string> values = {
       "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
   Builder builder(pool);
   const size_t values_len = values.size();
   for (int64_t i = 0; i < length; ++i) {
     int64_t values_index = i % values_len;
-    if (values_index == 0) {
+    if (include_nulls && values_index == 0) {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       const std::string& value = values[values_index];
@@ -210,12 +224,12 @@ Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
   // Quirk with RETURN_NOT_OK macro and templated functions
   {
-    auto s = MakeRandomBinaryArray<StringBuilder, char>(length, pool, &a0);
+    auto s = MakeRandomBinaryArray<StringBuilder, char>(length, true, pool, &a0);
     RETURN_NOT_OK(s);
   }
 
   {
-    auto s = MakeRandomBinaryArray<BinaryBuilder, uint8_t>(length, pool, &a1);
+    auto s = MakeRandomBinaryArray<BinaryBuilder, uint8_t>(length, true, pool, &a1);
     RETURN_NOT_OK(s);
   }
   out->reset(new RecordBatch(schema, length, {a0, a1}));
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 8ba00a6ffd599..61caf6403c8dc 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -45,6 +45,49 @@ namespace ipc {
 // ----------------------------------------------------------------------
 // Record batch write path
 
+static inline Status GetTruncatedBitmap(int64_t offset, int64_t length,
+    const std::shared_ptr<Buffer> input, MemoryPool* pool,
+    std::shared_ptr<Buffer>* buffer) {
+  if (!input) {
+    *buffer = input;
+    return Status::OK();
+  }
+  int64_t min_length = PaddedLength(BitUtil::BytesForBits(length));
+  if (offset != 0 || min_length < input->size()) {
+    // With a sliced array / non-zero offset, we must copy the bitmap
+    RETURN_NOT_OK(CopyBitmap(pool, input->data(), offset, length, buffer));
+  } else {
+    *buffer = input;
+  }
+  return Status::OK();
+}
+
+template <typename T>
+inline Status GetTruncatedBuffer(int64_t offset, int64_t length,
+    const std::shared_ptr<Buffer> input, MemoryPool* pool,
+    std::shared_ptr<Buffer>* buffer) {
+  if (!input) {
+    *buffer = input;
+    return Status::OK();
+  }
+  int32_t byte_width = static_cast<int32_t>(sizeof(T));
+  int64_t padded_length = PaddedLength(length * byte_width);
+  if (offset != 0 || padded_length < input->size()) {
+    *buffer =
+        SliceBuffer(input, offset * byte_width, std::min(padded_length, input->size()));
+  } else {
+    *buffer = input;
+  }
+  return Status::OK();
+}
+
+static inline bool NeedTruncate(
+    int64_t offset, const Buffer* buffer, int64_t min_length) {
+  // buffer can be NULL
+  if (buffer == nullptr) { return false; }
+  return offset != 0 || min_length < buffer->size();
+}
+
 class RecordBatchWriter : public ArrayVisitor {
  public:
   RecordBatchWriter(MemoryPool* pool, int64_t buffer_start_offset,
@@ -71,14 +114,9 @@ class RecordBatchWriter : public ArrayVisitor {
     field_nodes_.emplace_back(arr.length(), arr.null_count(), 0);
 
     if (arr.null_count() > 0) {
-      std::shared_ptr<Buffer> bitmap = arr.null_bitmap();
-
-      if (arr.offset() != 0) {
-        // With a sliced array / non-zero offset, we must copy the bitmap
-        RETURN_NOT_OK(
-            CopyBitmap(pool_, bitmap->data(), arr.offset(), arr.length(), &bitmap));
-      }
-
+      std::shared_ptr<Buffer> bitmap;
+      RETURN_NOT_OK(GetTruncatedBitmap(
+          arr.offset(), arr.length(), arr.null_bitmap(), pool_, &bitmap));
       buffers_.push_back(bitmap);
     } else {
       // Push a dummy zero-length buffer, not to be copied
@@ -195,21 +233,23 @@ class RecordBatchWriter : public ArrayVisitor {
  protected:
   template <typename ArrayType>
   Status VisitFixedWidth(const ArrayType& array) {
-    std::shared_ptr<Buffer> data_buffer = array.data();
+    std::shared_ptr<Buffer> data = array.data();
 
-    if (array.offset() != 0) {
+    const auto& fw_type = static_cast<const FixedWidthType&>(*array.type());
+    const int64_t type_width = fw_type.bit_width() / 8;
+    int64_t min_length = PaddedLength(array.length() * type_width);
+
+    if (NeedTruncate(array.offset(), data.get(), min_length)) {
       // Non-zero offset, slice the buffer
-      const auto& fw_type = static_cast<const FixedWidthType&>(*array.type());
-      const int type_width = fw_type.bit_width() / 8;
       const int64_t byte_offset = array.offset() * type_width;
 
       // Send padding if it's available
       const int64_t buffer_length =
           std::min(BitUtil::RoundUpToMultipleOf64(array.length() * type_width),
-              data_buffer->size() - byte_offset);
-      data_buffer = SliceBuffer(data_buffer, byte_offset, buffer_length);
+              data->size() - byte_offset);
+      data = SliceBuffer(data, byte_offset, buffer_length);
     }
-    buffers_.push_back(data_buffer);
+    buffers_.push_back(data);
     return Status::OK();
   }
 
@@ -249,9 +289,16 @@ class RecordBatchWriter : public ArrayVisitor {
     RETURN_NOT_OK(GetZeroBasedValueOffsets<BinaryArray>(array, &value_offsets));
     auto data = array.data();
 
-    if (array.offset() != 0) {
+    int64_t total_data_bytes = 0;
+    if (value_offsets) {
+      total_data_bytes = array.value_offset(array.length()) - array.value_offset(0);
+    }
+    if (NeedTruncate(array.offset(), data.get(), total_data_bytes)) {
       // Slice the data buffer to include only the range we need now
-      data = SliceBuffer(data, array.value_offset(0), array.value_offset(array.length()));
+      const int64_t start_offset = array.value_offset(0);
+      const int64_t slice_length =
+          std::min(PaddedLength(total_data_bytes), data->size() - start_offset);
+      data = SliceBuffer(data, start_offset, slice_length);
     }
 
     buffers_.push_back(value_offsets);
@@ -259,24 +306,11 @@ class RecordBatchWriter : public ArrayVisitor {
     return Status::OK();
   }
 
-  Status Visit(const FixedSizeBinaryArray& array) override {
-    auto data = array.data();
-    int32_t width = array.byte_width();
-
-    if (data && array.offset() != 0) {
-      data = SliceBuffer(data, array.offset() * width, width * array.length());
-    }
-    buffers_.push_back(data);
-    return Status::OK();
-  }
-
   Status Visit(const BooleanArray& array) override {
-    std::shared_ptr<Buffer> bits = array.data();
-    if (array.offset() != 0) {
-      RETURN_NOT_OK(
-          CopyBitmap(pool_, bits->data(), array.offset(), array.length(), &bits));
-    }
-    buffers_.push_back(bits);
+    std::shared_ptr<Buffer> data;
+    RETURN_NOT_OK(
+        GetTruncatedBitmap(array.offset(), array.length(), array.data(), pool_, &data));
+    buffers_.push_back(data);
     return Status::OK();
   }
 
@@ -299,6 +333,7 @@ class RecordBatchWriter : public ArrayVisitor {
   VISIT_FIXED_WIDTH(TimestampArray);
   VISIT_FIXED_WIDTH(Time32Array);
   VISIT_FIXED_WIDTH(Time64Array);
+  VISIT_FIXED_WIDTH(FixedSizeBinaryArray);
 
 #undef VISIT_FIXED_WIDTH
 
@@ -314,11 +349,16 @@ class RecordBatchWriter : public ArrayVisitor {
     --max_recursion_depth_;
     std::shared_ptr<Array> values = array.values();
 
-    if (array.offset() != 0) {
-      // For non-zero offset, we slice the values array accordingly
-      const int32_t offset = array.value_offset(0);
-      const int32_t length = array.value_offset(array.length()) - offset;
-      values = values->Slice(offset, length);
+    int32_t values_offset = 0;
+    int32_t values_length = 0;
+    if (value_offsets) {
+      values_offset = array.value_offset(0);
+      values_length = array.value_offset(array.length()) - values_offset;
+    }
+
+    if (array.offset() != 0 || values_length < values->length()) {
+      // Must also slice the values
+      values = values->Slice(values_offset, values_length);
     }
     RETURN_NOT_OK(VisitArray(*values));
     ++max_recursion_depth_;
@@ -328,7 +368,7 @@ class RecordBatchWriter : public ArrayVisitor {
   Status Visit(const StructArray& array) override {
     --max_recursion_depth_;
     for (std::shared_ptr<Array> field : array.fields()) {
-      if (array.offset() != 0) {
+      if (array.offset() != 0 || array.length() < field->length()) {
         // If offset is non-zero, slice the child array
         field = field->Slice(array.offset(), array.length());
       }
@@ -339,18 +379,21 @@ class RecordBatchWriter : public ArrayVisitor {
   }
 
   Status Visit(const UnionArray& array) override {
-    auto type_ids = array.type_ids();
-    if (array.offset() != 0) {
-      type_ids = SliceBuffer(type_ids, array.offset() * sizeof(UnionArray::type_id_t),
-          array.length() * sizeof(UnionArray::type_id_t));
-    }
+    const int64_t offset = array.offset();
+    const int64_t length = array.length();
 
+    std::shared_ptr<Buffer> type_ids;
+    RETURN_NOT_OK(GetTruncatedBuffer<UnionArray::type_id_t>(
+        offset, length, array.type_ids(), pool_, &type_ids));
     buffers_.push_back(type_ids);
 
     --max_recursion_depth_;
     if (array.mode() == UnionMode::DENSE) {
       const auto& type = static_cast<const UnionType&>(*array.type());
-      auto value_offsets = array.value_offsets();
+
+      std::shared_ptr<Buffer> value_offsets;
+      RETURN_NOT_OK(GetTruncatedBuffer<int32_t>(
+          offset, length, array.value_offsets(), pool_, &value_offsets));
 
       // The Union type codes are not necessary 0-indexed
       uint8_t max_code = 0;
@@ -363,7 +406,7 @@ class RecordBatchWriter : public ArrayVisitor {
       std::vector<int32_t> child_offsets(max_code + 1);
       std::vector<int32_t> child_lengths(max_code + 1, 0);
 
-      if (array.offset() != 0) {
+      if (offset != 0) {
         // This is an unpleasant case. Because the offsets are different for
         // each child array, when we have a sliced array, we need to "rebase"
         // the value_offsets for each array
@@ -373,12 +416,12 @@ class RecordBatchWriter : public ArrayVisitor {
 
         // Allocate the shifted offsets
         std::shared_ptr<MutableBuffer> shifted_offsets_buffer;
-        RETURN_NOT_OK(AllocateBuffer(
-            pool_, array.length() * sizeof(int32_t), &shifted_offsets_buffer));
+        RETURN_NOT_OK(
+            AllocateBuffer(pool_, length * sizeof(int32_t), &shifted_offsets_buffer));
         int32_t* shifted_offsets =
             reinterpret_cast<int32_t*>(shifted_offsets_buffer->mutable_data());
 
-        for (int64_t i = 0; i < array.length(); ++i) {
+        for (int64_t i = 0; i < length; ++i) {
           const uint8_t code = type_ids[i];
           int32_t shift = child_offsets[code];
           if (shift == -1) { child_offsets[code] = shift = unshifted_offsets[i]; }
@@ -395,18 +438,23 @@ class RecordBatchWriter : public ArrayVisitor {
       // Visit children and slice accordingly
       for (int i = 0; i < type.num_children(); ++i) {
         std::shared_ptr<Array> child = array.child(i);
-        if (array.offset() != 0) {
-          const uint8_t code = type.type_codes()[i];
-          child = child->Slice(child_offsets[code], child_lengths[code]);
+
+        // TODO: ARROW-809, for sliced unions, tricky to know how much to
+        // truncate the children. For now, we are truncating the children to be
+        // no longer than the parent union
+        const uint8_t code = type.type_codes()[i];
+        const int64_t child_length = child_lengths[code];
+        if (offset != 0 || length < child_length) {
+          child = child->Slice(child_offsets[code], std::min(length, child_length));
         }
         RETURN_NOT_OK(VisitArray(*child));
       }
     } else {
       for (std::shared_ptr<Array> child : array.children()) {
         // Sparse union, slicing is simpler
-        if (array.offset() != 0) {
+        if (offset != 0 || length < child->length()) {
           // If offset is non-zero, slice the child array
-          child = child->Slice(array.offset(), array.length());
+          child = child->Slice(offset, length);
         }
         RETURN_NOT_OK(VisitArray(*child));
       }
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 0f46f0306fe08..1f4bfa9acd357 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -162,10 +162,8 @@ class ArrayPrinter {
 
     Newline();
     Write("-- values: ");
-    auto values = array.values();
-    if (array.offset() != 0) {
-      values = values->Slice(array.value_offset(0), array.value_offset(array.length()));
-    }
+    auto values =
+        array.values()->Slice(array.value_offset(0), array.value_offset(array.length()));
     RETURN_NOT_OK(PrettyPrint(*values, indent_ + 2, sink_));
 
     return Status::OK();
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index bd80d9f636890..ad0ee0f5056da 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -32,8 +32,8 @@ static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   struct tm epoch = {0};
   epoch.tm_year = 70;
   epoch.tm_mday = 1;
-  // Milliseconds since the epoch
 #ifdef _MSC_VER
+  // Milliseconds since the epoch
   const int64_t current_timestamp = static_cast<int64_t>(_mkgmtime64(&date));
   const int64_t epoch_timestamp = static_cast<int64_t>(_mkgmtime64(&epoch));
   return (current_timestamp - epoch_timestamp) * 1000LL;
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index eabd98bda1893..db17da72a6a33 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -180,11 +180,11 @@ bool RecordBatch::ApproxEquals(const RecordBatch& other) const {
   return true;
 }
 
-std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset) {
+std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset) const {
   return Slice(offset, this->num_rows() - offset);
 }
 
-std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length) {
+std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length) const {
   std::vector<std::shared_ptr<Array>> arrays;
   arrays.reserve(num_columns());
   for (const auto& field : columns_) {
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index cfd1f366b039f..efc2077bd009a 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -137,8 +137,8 @@ class ARROW_EXPORT RecordBatch {
   int64_t num_rows() const { return num_rows_; }
 
   /// Slice each of the arrays in the record batch and construct a new RecordBatch object
-  std::shared_ptr<RecordBatch> Slice(int64_t offset);
-  std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length);
+  std::shared_ptr<RecordBatch> Slice(int64_t offset) const;
+  std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length) const;
 
   /// Returns error status is there is something wrong with the record batch
   /// contents, like a schema/array mismatch or inconsistent lengths

From 59cd801a7645783c0c33ed2435be08db4ffcd378 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 18 Apr 2017 22:32:59 -0400
Subject: [PATCH 0536/1644] ARROW-852: Also search for ARROW libs when
 pkg-config provided the path

Change-Id: Ic7fb227342782dfed5885f8fc5e73418fd31d504

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #563 from xhochy/ARROW-852 and squashes the following commits:

9630352 [Uwe L. Korn] Remove ARROW_HOME
5fc43ce [Uwe L. Korn] Always search for libs
---
 python/cmake_modules/FindArrow.cmake | 61 ++++++++++++++--------------
 python/manylinux1/build_arrow.sh     |  1 -
 2 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index fbe4545a520af..9fb1355fe1d52 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -34,6 +34,7 @@ if ("$ENV{ARROW_HOME}" STREQUAL "")
     message(STATUS "Arrow SO version: ${ARROW_SO_VERSION}")
     set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
     set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
+    set(ARROW_SEARCH_LIB_PATH ${ARROW_LIBRARY_DIRS})
   endif()
 else()
   set(ARROW_HOME "$ENV{ARROW_HOME}")
@@ -51,42 +52,42 @@ else()
     # make sure we don't accidentally pick up a different version
     NO_DEFAULT_PATH
     )
+endif()
 
-  find_library(ARROW_LIB_PATH NAMES arrow
-    PATHS
-    ${ARROW_SEARCH_LIB_PATH}
-    NO_DEFAULT_PATH)
-  get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
+find_library(ARROW_LIB_PATH NAMES arrow
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
 
-  find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
-    PATHS
-    ${ARROW_SEARCH_LIB_PATH}
-    NO_DEFAULT_PATH)
+find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
 
-  find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
-    PATHS
-    ${ARROW_SEARCH_LIB_PATH}
-    NO_DEFAULT_PATH)
+find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
 
-  if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
-    set(ARROW_FOUND TRUE)
+if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
+  set(ARROW_FOUND TRUE)
 
-    if (MSVC)
-      set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
-      set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
-      set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
-      set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
-      set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
-      set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
-    else()
-      set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
-      set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
-      set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
+  if (MSVC)
+    set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
+    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
+    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
+    set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
+    set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
+    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
+  else()
+    set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
+    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
+    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
 
-      set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
-      set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
-      set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
-    endif()
+    set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
 endif()
 
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 3df322581b54c..8ef087c7d262f 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -37,7 +37,6 @@ export PYARROW_WITH_JEMALLOC=1
 export PYARROW_BUNDLE_ARROW_CPP=1
 # Need as otherwise arrow_io is sometimes not linked
 export LDFLAGS="-Wl,--no-as-needed"
-export ARROW_HOME="/arrow-dist"
 export PARQUET_HOME="/usr"
 export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
 

From 4555ab92b174ce645ed20c7c6e15ee236a0e2f7a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 19 Apr 2017 11:22:54 -0400
Subject: [PATCH 0537/1644] ARROW-841: [Python] Add pyarrow build to Appveyor

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #566 from wesm/ARROW-841 and squashes the following commits:

4a04d57 [Wes McKinney] Set CMP0054 policy also in python/CMakeLists.txt
d2b4ffb [Wes McKinney] install cython
695e9b3 [Wes McKinney] Fix directory
36ad9e2 [Wes McKinney] Another fix for compiler id check
ba31cf5 [Wes McKinney] typo
7c32abb [Wes McKinney] Set CMP0054 policy
d27563f [Wes McKinney] Fix for CMP0054
ee883d8 [Wes McKinney] Exit early on failure of things
ac903db [Wes McKinney] Fix cmake warning
949558a [Wes McKinney] Remove conda list
54fbd48 [Wes McKinney] Add directory
86f91d2 [Wes McKinney] Write msvc build script that builds pyarrow
---
 appveyor.yml                              | 16 +++----
 ci/msvc-build.bat                         | 52 +++++++++++++++++++++++
 cpp/CMakeLists.txt                        |  6 +++
 cpp/cmake_modules/FindPythonLibsNew.cmake | 13 +++---
 cpp/cmake_modules/SetupCxxFlags.cmake     |  2 +-
 cpp/src/arrow/python/CMakeLists.txt       | 12 +++---
 python/CMakeLists.txt                     |  6 +++
 7 files changed, 86 insertions(+), 21 deletions(-)
 create mode 100644 ci/msvc-build.bat

diff --git a/appveyor.yml b/appveyor.yml
index b8c26e6e5084c..f2954a92e9e19 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -21,17 +21,15 @@ os: Visual Studio 2015
 environment:
   matrix:
     - GENERATOR: Visual Studio 14 2015 Win64
-    # - GENERATOR: Visual Studio 14 2015
+      PYTHON: "3.5"
+      ARCH: "64"
   MSVC_DEFAULT_OPTIONS: ON
   BOOST_ROOT: C:\Libraries\boost_1_63_0
   BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
 
-build_script:
- - cd cpp
- - mkdir build
- - cd build
- - cmake -G "%GENERATOR%" -DARROW_CXXFLAGS="/WX" -DARROW_BOOST_USE_SHARED=OFF -DCMAKE_BUILD_TYPE=Release ..
- - cmake --build . --config Release
+init:
+  - set MINICONDA=C:\Miniconda35-x64
+  - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
 
-# test_script:
- - ctest -VV
+build_script:
+ - call ci\msvc-build.bat
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
new file mode 100644
index 0000000000000..de428b6e46e14
--- /dev/null
+++ b/ci/msvc-build.bat
@@ -0,0 +1,52 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+set CONDA_ENV=C:\arrow-conda-env
+set ARROW_HOME=C:\arrow-install
+
+conda create -p %CONDA_ENV% -q -y python=%PYTHON% ^
+      six pytest setuptools numpy pandas cython
+call activate %CONDA_ENV%
+
+@rem Build and test Arrow C++ libraries
+
+cd cpp
+mkdir build
+cd build
+cmake -G "%GENERATOR%" ^
+      -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=Release ^
+      -DARROW_CXXFLAGS="/WX" ^
+      -DARROW_PYTHON=on ^
+      ..  || exit /B
+cmake --build . --target INSTALL --config Release  || exit /B
+
+@rem Needed so python-test.exe works
+set PYTHONPATH=%CONDA_ENV%\Lib;%CONDA_ENV%\Lib\site-packages;%CONDA_ENV%\python35.zip;%CONDA_ENV%\DLLs;%CONDA_ENV%
+
+ctest -VV  || exit /B
+
+@rem Build and import pyarrow
+
+set PATH=%ARROW_HOME%\bin;%PATH%
+
+cd ..\..\python
+python setup.py build_ext --inplace  || exit /B
+python -c "import pyarrow"  || exit /B
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f702da16e7a4e..c1cf7852a30b9 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -36,6 +36,12 @@ else()
   include(GNUInstallDirs)
 endif()
 
+# Compatibility with CMake 3.1
+if(POLICY CMP0054)
+  # http://www.cmake.org/cmake/help/v3.1/policy/CMP0054.html
+  cmake_policy(SET CMP0054 NEW)
+endif()
+
 set(ARROW_SO_VERSION "0")
 set(ARROW_ABI_VERSION "${ARROW_SO_VERSION}.0.0")
 
diff --git a/cpp/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake
index 961081609cb12..09124aa17bb9c 100644
--- a/cpp/cmake_modules/FindPythonLibsNew.cmake
+++ b/cpp/cmake_modules/FindPythonLibsNew.cmake
@@ -141,12 +141,13 @@ string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR ${PYTHON_INCLUDE_DIR})
 string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES})
 
 if(CMAKE_HOST_WIN32)
-    if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
-        set(PYTHON_LIBRARY
-            "${PYTHON_PREFIX}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib")
-    else()
-        set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/libpython${PYTHON_LIBRARY_SUFFIX}.a")
-    endif()
+  # Appease CMP0054
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    set(PYTHON_LIBRARY
+      "${PYTHON_PREFIX}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib")
+  else()
+    set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/libpython${PYTHON_LIBRARY_SUFFIX}.a")
+  endif()
 elseif(APPLE)
   # In some cases libpythonX.X.dylib is not part of the PYTHON_PREFIX and we
   # need to call `python-config --prefix` to determine the correct location.
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 694e5a37df4ba..7e229ff90a3e7 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -30,7 +30,7 @@ if (MSVC)
   # insecure, like std::getenv
   add_definitions(-D_CRT_SECURE_NO_WARNINGS)
 
-  if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+  if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     # clang-cl
     set(CXX_COMMON_FLAGS "-EHsc")
   elseif(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 19)
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 607a1c436c45d..5c2b58825294c 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -57,6 +57,13 @@ set(ARROW_PYTHON_SHARED_LINK_LIBS
   arrow_shared
 )
 
+if (MSVC)
+  set(ARROW_PYTHON_SHARED_LINK_LIBS
+    ${ARROW_PYTHON_SHARED_LINK_LIBS}
+    ${PYTHON_LIBRARIES}
+    )
+endif()
+
 ADD_ARROW_LIB(arrow_python
   SOURCES ${ARROW_PYTHON_SRCS}
   SHARED_LINK_FLAGS ""
@@ -64,11 +71,6 @@ ADD_ARROW_LIB(arrow_python
   STATIC_LINK_LIBS ""
 )
 
-if (MSVC)
-  target_link_libraries(arrow_python_shared
-    ${PYTHON_LIBRARIES})
-endif()
-
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
   # Clang, be quiet. Python C API has lots of macros
   set_property(SOURCE ${ARROW_PYTHON_SRCS}
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index c1431af67ed55..3db7b7bf83822 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -28,6 +28,12 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../cpp/cmake_mod
 
 include(CMakeParseArguments)
 
+# Compatibility with CMake 3.1
+if(POLICY CMP0054)
+  # http://www.cmake.org/cmake/help/v3.1/policy/CMP0054.html
+  cmake_policy(SET CMP0054 NEW)
+endif()
+
 set(BUILD_SUPPORT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../cpp/build-support")
 
 # Allow "make install" to not depend on all targets.

From 41a8ff9ad18a4970c16b674b56ade25b8e8986ec Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 19 Apr 2017 19:42:49 +0200
Subject: [PATCH 0538/1644] ARROW-853: [Python] Only set RPATH when bundling
 the shared libraries

See discussion in https://github.com/apache/arrow/pull/562. Modifying RPATH is no longer needed when libarrow/libarrow_python are installed someplace else in the loader path.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #564 from wesm/ARROW-853 and squashes the following commits:

262f43a [Wes McKinney] Only set RPATH when bundling the shared libraries
---
 python/CMakeLists.txt | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 3db7b7bf83822..0d34bcdfa6e49 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -346,21 +346,25 @@ foreach(module ${CYTHON_EXTENSIONS})
           LIBRARY_OUTPUT_DIRECTORY ${module_output_directory})
     endif()
 
-    if(APPLE)
+    if (PYARROW_BUNDLE_ARROW_CPP)
+      # In the event that we are bundling the shared libraries (e.g. in a
+      # manylinux1 wheel), we need to set the RPATH of the extensions to the
+      # root of the pyarrow/ package so that libarrow/libarrow_python are able
+      # to be loaded properly
+      if(APPLE)
         set(module_install_rpath "@loader_path")
-    else()
+      else()
         set(module_install_rpath "\$ORIGIN")
-    endif()
-    list(LENGTH directories i)
-    while(${i} GREATER 0)
+      endif()
+      list(LENGTH directories i)
+      while(${i} GREATER 0)
         set(module_install_rpath "${module_install_rpath}/..")
         math(EXPR i "${i} - 1" )
-    endwhile(${i} GREATER 0)
+      endwhile(${i} GREATER 0)
 
-    # for inplace development for now
-    #set(module_install_rpath "${CMAKE_SOURCE_DIR}/pyarrow/")
+      set_target_properties(${module_name} PROPERTIES
+        INSTALL_RPATH ${module_install_rpath})
+    endif()
 
-    set_target_properties(${module_name} PROPERTIES
-      INSTALL_RPATH ${module_install_rpath})
     target_link_libraries(${module_name} ${LINK_LIBS})
 endforeach(module)

From 391242a17d5bdb041b7b1b036b48e69e82ec29b8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 19 Apr 2017 15:49:00 -0400
Subject: [PATCH 0539/1644] ARROW-848: [Python] Another pass on conda dev guide

Per feedback in https://github.com/apache/arrow/commit/bb8514cc9d7068c8b62d346577370751d68221d8

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #562 from wesm/conda-quickstart-iterate and squashes the following commits:

881a44d [Wes McKinney] Add system requirements notes about gcc 4.9, use boost shared libs
8c95705 [Wes McKinney] Install cmake in conda env
8c2885e [Wes McKinney] Another pass on conda dev guide, do not require LD_LIBRARY_PATH. Install everything in a single conda environment
---
 python/DEVELOPMENT.md | 73 ++++++++++++++++++++++++++++---------------
 python/setup.py       |  2 +-
 2 files changed, 49 insertions(+), 26 deletions(-)

diff --git a/python/DEVELOPMENT.md b/python/DEVELOPMENT.md
index ca744628da1b5..7f08169d613f0 100644
--- a/python/DEVELOPMENT.md
+++ b/python/DEVELOPMENT.md
@@ -16,36 +16,41 @@
 
 ### Linux and macOS
 
-First, set up your thirdparty C++ toolchain using libraries from conda-forge:
+#### System Requirements
+
+On macOS, any modern XCode (6.4 or higher; the current version is 8.3.1) is
+sufficient.
+
+On Linux, for this guide, we recommend using gcc 4.8 or 4.9, or clang 3.7 or
+higher. You can check your version by running
 
 ```shell
-conda config --add channels conda-forge
+$ gcc --version
+```
 
-export ARROW_BUILD_TYPE=Release
+On Ubuntu 16.04 and higher, you can obtain gcc 4.9 with:
 
-export CPP_TOOLCHAIN=$HOME/cpp-toolchain
-export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
+```shell
+$ sudo apt-get install g++-4.9
+```
 
-export BOOST_ROOT=$CPP_TOOLCHAIN
-export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
-export RAPIDJSON_HOME=$CPP_TOOLCHAIN
-export THRIFT_HOME=$CPP_TOOLCHAIN
-export ZLIB_HOME=$CPP_TOOLCHAIN
-export SNAPPY_HOME=$CPP_TOOLCHAIN
-export BROTLI_HOME=$CPP_TOOLCHAIN
-export JEMALLOC_HOME=$CPP_TOOLCHAIN
-export ARROW_HOME=$CPP_TOOLCHAIN
-export PARQUET_HOME=$CPP_TOOLCHAIN
+Finally, set gcc 4.9 as the active compiler using:
 
-conda create -y -q -p $CPP_TOOLCHAIN \
-      flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib brotli jemalloc
+```shell
+export CC=gcc-4.9
+export CXX=g++-4.9
 ```
 
-Now, activate a conda environment containing your target Python version and
-NumPy installed:
+#### Environment Setup and Build
+
+First, let's create a conda environment with all the C++ build and Python
+dependencies from conda-forge:
 
 ```shell
-conda create -y -q -n pyarrow-dev python=3.6 numpy
+conda create -y -q -n pyarrow-dev \
+      python=3.6 numpy six setuptools cython pandas pytest \
+      cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
+      brotli jemalloc -c conda-forge
 source activate pyarrow-dev
 ```
 
@@ -67,6 +72,26 @@ drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 arrow/
 drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 parquet-cpp/
 ```
 
+We need to set a number of environment variables to let Arrow's build system
+know about our build toolchain:
+
+```
+export ARROW_BUILD_TYPE=release
+
+export BOOST_ROOT=$CONDA_PREFIX
+export BOOST_LIBRARYDIR=$CONDA_PREFIX/lib
+
+export FLATBUFFERS_HOME=$CONDA_PREFIX
+export RAPIDJSON_HOME=$CONDA_PREFIX
+export THRIFT_HOME=$CONDA_PREFIX
+export ZLIB_HOME=$CONDA_PREFIX
+export SNAPPY_HOME=$CONDA_PREFIX
+export BROTLI_HOME=$CONDA_PREFIX
+export JEMALLOC_HOME=$CONDA_PREFIX
+export ARROW_HOME=$CONDA_PREFIX
+export PARQUET_HOME=$CONDA_PREFIX
+```
+
 Now build and install the Arrow C++ libraries:
 
 ```shell
@@ -74,7 +99,7 @@ mkdir arrow/cpp/build
 pushd arrow/cpp/build
 
 cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-      -DCMAKE_INSTALL_PREFIX=$CPP_TOOLCHAIN \
+      -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
       -DARROW_PYTHON=on \
       -DARROW_BUILD_TESTS=OFF \
       ..
@@ -90,7 +115,7 @@ mkdir parquet-cpp/build
 pushd parquet-cpp/build
 
 cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-      -DCMAKE_INSTALL_PREFIX=$CPP_TOOLCHAIN \
+      -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
       -DPARQUET_BUILD_BENCHMARKS=off \
       -DPARQUET_BUILD_EXECUTABLES=off \
       -DPARQUET_ZLIB_VENDORED=off \
@@ -102,11 +127,9 @@ make install
 popd
 ```
 
-Now, install requisite build requirements for pyarrow, then build:
+Now, build pyarrow:
 
 ```shell
-conda install -y -q six setuptools cython pandas pytest
-
 cd arrow/python
 python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --with-parquet --inplace
 ```
diff --git a/python/setup.py b/python/setup.py
index ab71e7858e626..1c46617066925 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -155,7 +155,7 @@ def _run_cmake(self):
             cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
 
         cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
-                             .format(self.build_type))
+                             .format(self.build_type.lower()))
 
         if sys.platform != 'win32':
             cmake_command = (['cmake', self.extra_cmake_args] +

From 74f89cfbe0793043eb579ec30b3d6467b0ad9af2 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Wed, 19 Apr 2017 17:11:51 -0400
Subject: [PATCH 0540/1644] ARROW-858: Remove boost_regex from arrow
 dependencies

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #567 from cpcloud/decimal-no-regex and squashes the following commits:

b5c59bd [Phillip Cloud] ARROW-858: Remove boost_regex from arrow dependencies
---
 .travis.yml                              |  1 -
 ci/travis_script_python.sh               |  1 +
 cpp/CMakeLists.txt                       | 22 ++-----
 cpp/README.md                            |  1 -
 cpp/src/arrow/ipc/CMakeLists.txt         |  4 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  4 +-
 cpp/src/arrow/python/CMakeLists.txt      |  3 +-
 cpp/src/arrow/util/decimal-test.cc       | 40 +++++++++++++
 cpp/src/arrow/util/decimal.cc            | 73 +++++++++++++++++++-----
 9 files changed, 108 insertions(+), 41 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 824f62bccaab9..6ebebd4513fc7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,7 +14,6 @@ addons:
     - valgrind
     - libboost-dev
     - libboost-filesystem-dev
-    - libboost-regex-dev
     - libboost-system-dev
     - libjemalloc-dev
     - gtk-doc-tools
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index bde1fd7e249ec..c1426da7247b2 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -22,6 +22,7 @@ pushd $ARROW_PYTHON_DIR
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 
 build_parquet_cpp() {
+  export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
   conda create -y -q -p $PARQUET_HOME python=3.6
   source activate $PARQUET_HOME
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index c1cf7852a30b9..81e4c90c73147 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -410,19 +410,16 @@ if (ARROW_BOOST_USE_SHARED)
     add_definitions(-DBOOST_ALL_DYN_LINK)
   endif()
 
-  find_package(Boost COMPONENTS system filesystem regex REQUIRED)
+  find_package(Boost COMPONENTS system filesystem REQUIRED)
   if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
     set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
-    set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
   else()
     set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
     set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
-    set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
   endif()
   set(BOOST_SYSTEM_LIBRARY boost_system_shared)
   set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
-  set(BOOST_REGEX_LIBRARY boost_regex_shared)
 else()
   # Find static boost headers and libs
   # TODO Differentiate here between release and debug builds
@@ -431,15 +428,12 @@ else()
   if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
     set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
-    set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
   else()
     set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
     set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
-    set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
   endif()
   set(BOOST_SYSTEM_LIBRARY boost_system_static)
   set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
-  set(BOOST_REGEX_LIBRARY boost_regex_static)
 endif()
 
 message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
@@ -453,11 +447,7 @@ ADD_THIRDPARTY_LIB(boost_filesystem
     STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
     SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
 
-ADD_THIRDPARTY_LIB(boost_regex
-        STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
-        SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")
-
-SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
+SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 
@@ -758,8 +748,7 @@ set(ARROW_MIN_TEST_LIBS
   arrow_static
   gtest
   gtest_main
-  ${ARROW_BASE_LIBS}
-  ${BOOST_REGEX_LIBRARY})
+  ${ARROW_BASE_LIBS})
 
 if (APPLE)
   set(ARROW_MIN_TEST_LIBS
@@ -777,8 +766,7 @@ set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 set(ARROW_BENCHMARK_LINK_LIBS
   arrow_static
   arrow_benchmark_main
-  ${ARROW_BASE_LIBS}
-  ${BOOST_REGEX_LIBRARY})
+  ${ARROW_BASE_LIBS})
 
 ############################################################
 # "make ctags" target
@@ -875,7 +863,7 @@ endif()
 ############################################################
 
 set(ARROW_LINK_LIBS
-  ${BOOST_REGEX_LIBRARY})
+    )
 
 set(ARROW_STATIC_LINK_LIBS)
 
diff --git a/cpp/README.md b/cpp/README.md
index 339b6b47533cb..69c695020add5 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -31,7 +31,6 @@ On Ubuntu/Debian you can install the requirements with:
 sudo apt-get install cmake \
      libboost-dev \
      libboost-filesystem-dev \
-     libboost-regex-dev \
      libboost-system-dev
 ```
 
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 37b455395644f..fc1d53e18a3dc 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -91,14 +91,12 @@ if(MSVC)
   set(UTIL_LINK_LIBS
     arrow_static
     ${BOOST_FILESYSTEM_LIBRARY}
-    ${BOOST_SYSTEM_LIBRARY}
-    ${BOOST_REGEX_LIBRARY})
+    ${BOOST_SYSTEM_LIBRARY})
 else()
   set(UTIL_LINK_LIBS
     arrow_static
     ${BOOST_FILESYSTEM_LIBRARY}
     ${BOOST_SYSTEM_LIBRARY}
-    ${BOOST_REGEX_LIBRARY}
     dl)
 endif()
 
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index b39136ec12d04..cd793e08a26be 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -322,8 +322,8 @@ TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) {
   std::vector<int32_t> type_ids(a0->length());
   std::shared_ptr<Buffer> ids_buffer;
   ASSERT_OK(test::CopyBufferFromVector(type_ids, &ids_buffer));
-  a1 = std::make_shared<UnionArray>(
-      union_type, a0->length(), struct_children, ids_buffer);
+  a1 =
+      std::make_shared<UnionArray>(union_type, a0->length(), struct_children, ids_buffer);
   CheckArray(a1);
 
   // Dense union
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 5c2b58825294c..c5cbc50845de0 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -35,8 +35,7 @@ endif()
 set(ARROW_PYTHON_MIN_TEST_LIBS
   arrow_python_test_main
   arrow_python_static
-  arrow_static
-  ${BOOST_REGEX_LIBRARY})
+  arrow_static)
 
 set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
 
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index dcaa9afd8724a..5d95c2cadc107 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -159,5 +159,45 @@ TEST(DecimalTest, TestDecimal128StringAndBytesRoundTrip) {
 
   ASSERT_EQ(expected.value, result.value);
 }
+
+template <typename T>
+class DecimalZerosTest : public ::testing::Test {};
+TYPED_TEST_CASE(DecimalZerosTest, DecimalTypes);
+
+TYPED_TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
+  std::string string_value("0000000");
+  Decimal<TypeParam> d;
+  int precision;
+  int scale;
+  FromString(string_value, &d, &precision, &scale);
+  ASSERT_EQ(precision, 7);
+  ASSERT_EQ(scale, 0);
+  ASSERT_EQ(d.value, 0);
+}
+
+TYPED_TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
+  std::string string_value("000.0000");
+  Decimal<TypeParam> d;
+  int precision;
+  int scale;
+  FromString(string_value, &d, &precision, &scale);
+  // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
+  ASSERT_NE(precision, 7);
+
+  ASSERT_EQ(scale, 4);
+  ASSERT_EQ(d.value, 0);
+}
+
+TYPED_TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
+  std::string string_value(".00000");
+  Decimal<TypeParam> d;
+  int precision;
+  int scale;
+  FromString(string_value, &d, &precision, &scale);
+  ASSERT_EQ(precision, 5);
+  ASSERT_EQ(scale, 5);
+  ASSERT_EQ(d.value, 0);
+}
+
 }  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 3b8a3ff0398b5..2fe9da4aba9a2 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -17,34 +17,77 @@
 
 #include "arrow/util/decimal.h"
 
-#include <boost/regex.hpp>
-
 namespace arrow {
 namespace decimal {
 
-static const boost::regex DECIMAL_PATTERN("(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?");
-
 template <typename T>
 ARROW_EXPORT Status FromString(
     const std::string& s, Decimal<T>* out, int* precision, int* scale) {
+  // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?";
   if (s.empty()) {
     return Status::Invalid("Empty string cannot be converted to decimal");
   }
-  boost::smatch match;
-  if (!boost::regex_match(s, match, DECIMAL_PATTERN)) {
-    std::stringstream ss;
-    ss << "String " << s << " is not a valid decimal string";
-    return Status::Invalid(ss.str());
+
+  int8_t sign = 1;
+  auto charp = s.cbegin();
+  auto end = s.cend();
+
+  if (*charp == '+' || *charp == '-') {
+    if (*charp == '-') { sign = -1; }
+    ++charp;
   }
-  const int8_t sign = match[1].str() == "-" ? -1 : 1;
-  std::string whole_part = match[4].str();
-  std::string fractional_part = match[6].str();
-  if (scale != nullptr) { *scale = static_cast<int>(fractional_part.size()); }
+
+  auto numeric_string_start = charp;
+
+  // skip leading zeros
+  while (*charp == '0') {
+    ++charp;
+  }
+
+  // all zeros and no decimal point
+  if (charp == end) {
+    if (out != nullptr) { out->value = static_cast<T>(0); }
+
+    // Not sure what other libraries assign precision to for this case (this case of
+    // a string consisting only of one or more zeros)
+    if (precision != nullptr) {
+      *precision = static_cast<int>(charp - numeric_string_start);
+    }
+
+    if (scale != nullptr) { *scale = 0; }
+
+    return Status::OK();
+  }
+
+  auto whole_part_start = charp;
+  while (isdigit(*charp)) {
+    ++charp;
+  }
+  auto whole_part_end = charp;
+  std::string whole_part(whole_part_start, whole_part_end);
+
+  if (*charp == '.') {
+    ++charp;
+  } else {
+    // no decimal point
+    DCHECK_EQ(charp, end);
+  }
+
+  auto fractional_part_start = charp;
+  while (isdigit(*charp)) {
+    ++charp;
+  }
+  auto fractional_part_end = charp;
+  std::string fractional_part(fractional_part_start, fractional_part_end);
+
   if (precision != nullptr) {
-    *precision =
-        static_cast<int>(whole_part.size()) + static_cast<int>(fractional_part.size());
+    *precision = static_cast<int>(whole_part.size() + fractional_part.size());
   }
+
+  if (scale != nullptr) { *scale = static_cast<int>(fractional_part.size()); }
+
   if (out != nullptr) { StringToInteger(whole_part, fractional_part, sign, &out->value); }
+
   return Status::OK();
 }
 

From 0dc6fe8f33befaaa5fc8055b6c157ac1ccb09e6b Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 19 Apr 2017 17:17:38 -0400
Subject: [PATCH 0541/1644] ARROW-846: [GLib] Add GArrowTensor,
 GArrowInt8Tensor and GArrowUInt8Tensor

Author: Kouhei Sutou <kou@clear-code.com>

Closes #560 from kou/glib-add-tensor and squashes the following commits:

ed949d4 [Kouhei Sutou] [GLib] Support running tests on Ubuntu 14.04
39d40f0 [Kouhei Sutou] [GLib] Add GArrowTensor, GArrowInt8Tensor and GArrowUInt8Tensor
---
 c_glib/arrow-glib/Makefile.am        |   8 +
 c_glib/arrow-glib/arrow-glib.h       |   3 +
 c_glib/arrow-glib/arrow-glib.hpp     |   2 +
 c_glib/arrow-glib/int8-tensor.cpp    | 105 ++++++++
 c_glib/arrow-glib/int8-tensor.h      |  79 ++++++
 c_glib/arrow-glib/numeric-tensor.hpp |  64 +++++
 c_glib/arrow-glib/tensor.cpp         | 390 +++++++++++++++++++++++++++
 c_glib/arrow-glib/tensor.h           |  77 ++++++
 c_glib/arrow-glib/tensor.hpp         |  27 ++
 c_glib/arrow-glib/uint8-tensor.cpp   | 105 ++++++++
 c_glib/arrow-glib/uint8-tensor.h     |  79 ++++++
 c_glib/test/helper/omittable.rb      |  28 ++
 c_glib/test/run-test.rb              |   1 +
 c_glib/test/test-int8-tensor.rb      |  43 +++
 c_glib/test/test-tensor.rb           | 100 +++++++
 c_glib/test/test-uint8-tensor.rb     |  43 +++
 16 files changed, 1154 insertions(+)
 create mode 100644 c_glib/arrow-glib/int8-tensor.cpp
 create mode 100644 c_glib/arrow-glib/int8-tensor.h
 create mode 100644 c_glib/arrow-glib/numeric-tensor.hpp
 create mode 100644 c_glib/arrow-glib/tensor.cpp
 create mode 100644 c_glib/arrow-glib/tensor.h
 create mode 100644 c_glib/arrow-glib/tensor.hpp
 create mode 100644 c_glib/arrow-glib/uint8-tensor.cpp
 create mode 100644 c_glib/arrow-glib/uint8-tensor.h
 create mode 100644 c_glib/test/helper/omittable.rb
 create mode 100644 c_glib/test/test-int8-tensor.rb
 create mode 100644 c_glib/test/test-tensor.rb
 create mode 100644 c_glib/test/test-uint8-tensor.rb

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 2e7a9a0e439eb..fbfe3a4071000 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -65,6 +65,7 @@ libarrow_glib_la_headers =			\
 	int8-array.h				\
 	int8-array-builder.h			\
 	int8-data-type.h			\
+	int8-tensor.h				\
 	int16-array.h				\
 	int16-array-builder.h			\
 	int16-data-type.h			\
@@ -88,10 +89,12 @@ libarrow_glib_la_headers =			\
 	struct-array-builder.h			\
 	struct-data-type.h			\
 	table.h					\
+	tensor.h				\
 	type.h					\
 	uint8-array.h				\
 	uint8-array-builder.h			\
 	uint8-data-type.h			\
+	uint8-tensor.h				\
 	uint16-array.h				\
 	uint16-array-builder.h			\
 	uint16-data-type.h			\
@@ -152,6 +155,7 @@ libarrow_glib_la_sources =			\
 	int8-array.cpp				\
 	int8-array-builder.cpp			\
 	int8-data-type.cpp			\
+	int8-tensor.cpp				\
 	int16-array.cpp				\
 	int16-array-builder.cpp			\
 	int16-data-type.cpp			\
@@ -175,10 +179,12 @@ libarrow_glib_la_sources =			\
 	struct-array-builder.cpp		\
 	struct-data-type.cpp			\
 	table.cpp				\
+	tensor.cpp				\
 	type.cpp				\
 	uint8-array.cpp				\
 	uint8-array-builder.cpp			\
 	uint8-data-type.cpp			\
+	uint8-tensor.cpp			\
 	uint16-array.cpp			\
 	uint16-array-builder.cpp		\
 	uint16-data-type.cpp			\
@@ -220,9 +226,11 @@ libarrow_glib_la_cpp_headers =			\
 	data-type.hpp				\
 	error.hpp				\
 	field.hpp				\
+	numeric-tensor.hpp			\
 	record-batch.hpp			\
 	schema.hpp				\
 	table.hpp				\
+	tensor.hpp				\
 	type.hpp
 
 libarrow_glib_la_cpp_headers +=			\
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index b15c56f7bb486..eec9e25ebf690 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -42,6 +42,7 @@
 #include <arrow-glib/int8-array.h>
 #include <arrow-glib/int8-array-builder.h>
 #include <arrow-glib/int8-data-type.h>
+#include <arrow-glib/int8-tensor.h>
 #include <arrow-glib/int16-array.h>
 #include <arrow-glib/int16-array-builder.h>
 #include <arrow-glib/int16-data-type.h>
@@ -65,10 +66,12 @@
 #include <arrow-glib/struct-array-builder.h>
 #include <arrow-glib/struct-data-type.h>
 #include <arrow-glib/table.h>
+#include <arrow-glib/tensor.h>
 #include <arrow-glib/type.h>
 #include <arrow-glib/uint8-array.h>
 #include <arrow-glib/uint8-array-builder.h>
 #include <arrow-glib/uint8-data-type.h>
+#include <arrow-glib/uint8-tensor.h>
 #include <arrow-glib/uint16-array.h>
 #include <arrow-glib/uint16-array-builder.h>
 #include <arrow-glib/uint16-data-type.h>
diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
index 3404d4d212917..d6ef370095bdf 100644
--- a/c_glib/arrow-glib/arrow-glib.hpp
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -31,9 +31,11 @@
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/field.hpp>
 #include <arrow-glib/float-array.hpp>
+#include <arrow-glib/numeric-tensor.hpp>
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.hpp>
 #include <arrow-glib/table.hpp>
+#include <arrow-glib/tensor.hpp>
 #include <arrow-glib/type.hpp>
 
 #include <arrow-glib/file.hpp>
diff --git a/c_glib/arrow-glib/int8-tensor.cpp b/c_glib/arrow-glib/int8-tensor.cpp
new file mode 100644
index 0000000000000..06521a00997c0
--- /dev/null
+++ b/c_glib/arrow-glib/int8-tensor.cpp
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/buffer.hpp>
+#include <arrow-glib/int8-tensor.h>
+#include <arrow-glib/numeric-tensor.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: int8-tensor
+ * @short_description: 8-bit integer tensor class
+ *
+ * #GArrowInt8Tensor is a class for 8-bit integer tensor. It can store
+ * zero or more 8-bit integer data.
+ */
+
+G_DEFINE_TYPE(GArrowInt8Tensor,               \
+              garrow_int8_tensor,             \
+              GARROW_TYPE_TENSOR)
+
+static void
+garrow_int8_tensor_init(GArrowInt8Tensor *object)
+{
+}
+
+static void
+garrow_int8_tensor_class_init(GArrowInt8TensorClass *klass)
+{
+}
+
+/**
+ * garrow_int8_tensor_new:
+ * @data: A #GArrowBuffer that contains tensor data.
+ * @shape: (array length=n_dimensions): A list of dimension sizes.
+ * @n_dimensions: The number of dimensions.
+ * @strides: (array length=n_strides) (nullable): A list of the number of
+ *   bytes in each dimension.
+ * @n_strides: The number of strides.
+ * @dimention_names: (array length=n_dimention_names) (nullable): A list of
+ *   dimension names.
+ * @n_dimention_names: The number of dimension names
+ *
+ * Returns: The newly created #GArrowInt8Tensor.
+ *
+ * Since: 0.3.0
+ */
+GArrowInt8Tensor *
+garrow_int8_tensor_new(GArrowBuffer *data,
+                       gint64 *shape,
+                       gsize n_dimensions,
+                       gint64 *strides,
+                       gsize n_strides,
+                       gchar **dimension_names,
+                       gsize n_dimension_names)
+{
+  auto tensor =
+    garrow::numeric_tensor_new<arrow::Int8Tensor>(data,
+                                                  shape,
+                                                  n_dimensions,
+                                                  strides,
+                                                  n_strides,
+                                                  dimension_names,
+                                                  n_dimension_names);
+  return GARROW_INT8_TENSOR(tensor);
+}
+
+/**
+ * garrow_int8_tensor_get_raw_data:
+ * @tensor: A #GArrowInt8Tensor.
+ * @n_data: (out): The number of data.
+ *
+ * Returns: (array length=n_data): The raw data in the tensor.
+ *
+ * Since: 0.3.0
+ */
+const gint8 *
+garrow_int8_tensor_get_raw_data(GArrowInt8Tensor *tensor,
+                                gint64 *n_data)
+{
+  return garrow::numeric_tensor_get_raw_data<arrow::Int8Tensor, int8_t>(GARROW_TENSOR(tensor),
+                                                                        n_data);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-tensor.h b/c_glib/arrow-glib/int8-tensor.h
new file mode 100644
index 0000000000000..76ed3c8d7a7ee
--- /dev/null
+++ b/c_glib/arrow-glib/int8-tensor.h
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/tensor.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_INT8_TENSOR                 \
+  (garrow_int8_tensor_get_type())
+#define GARROW_INT8_TENSOR(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT8_TENSOR,  \
+                              GArrowInt8Tensor))
+#define GARROW_INT8_TENSOR_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT8_TENSOR,     \
+                           GArrowInt8TensorClass))
+#define GARROW_IS_INT8_TENSOR(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT8_TENSOR))
+#define GARROW_IS_INT8_TENSOR_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT8_TENSOR))
+#define GARROW_INT8_TENSOR_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT8_TENSOR,   \
+                             GArrowInt8TensorClass))
+
+typedef struct _GArrowInt8Tensor         GArrowInt8Tensor;
+typedef struct _GArrowInt8TensorClass    GArrowInt8TensorClass;
+
+/**
+ * GArrowInt8Tensor:
+ *
+ * It wraps `arrow::Int8Tensor`.
+ */
+struct _GArrowInt8Tensor
+{
+  /*< private >*/
+  GArrowTensor parent_instance;
+};
+
+struct _GArrowInt8TensorClass
+{
+  GArrowTensorClass parent_class;
+};
+
+GType garrow_int8_tensor_get_type(void) G_GNUC_CONST;
+
+GArrowInt8Tensor *garrow_int8_tensor_new(GArrowBuffer *data,
+                                         gint64 *shape,
+                                         gsize n_dimensions,
+                                         gint64 *strides,
+                                         gsize n_strides,
+                                         gchar **dimention_names,
+                                         gsize n_dimention_names);
+
+const gint8 *garrow_int8_tensor_get_raw_data(GArrowInt8Tensor *tensor,
+                                             gint64 *n_data);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/numeric-tensor.hpp b/c_glib/arrow-glib/numeric-tensor.hpp
new file mode 100644
index 0000000000000..07cea62bd7b25
--- /dev/null
+++ b/c_glib/arrow-glib/numeric-tensor.hpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/tensor.hpp>
+
+namespace garrow {
+  template <typename T>
+  GArrowTensor *numeric_tensor_new(GArrowBuffer *data,
+                                   gint64 *shape,
+                                   gsize n_dimensions,
+                                   gint64 *strides,
+                                   gsize n_strides,
+                                   gchar **dimention_names,
+                                   gsize n_dimention_names) {
+    auto arrow_data = garrow_buffer_get_raw(data);
+    std::vector<int64_t> arrow_shape;
+    for (gsize i = 0; i < n_dimensions; ++i) {
+      arrow_shape.push_back(shape[i]);
+    }
+    std::vector<int64_t> arrow_strides;
+    for (gsize i = 0; i < n_strides; ++i) {
+      arrow_strides.push_back(strides[i]);
+    }
+    std::vector<std::string> arrow_dimention_names;
+    for (gsize i = 0; i < n_dimention_names; ++i) {
+      arrow_dimention_names.push_back(dimention_names[i]);
+    }
+    auto arrow_numeric_tensor =
+      std::make_shared<T>(arrow_data,
+                          arrow_shape,
+                          arrow_strides,
+                          arrow_dimention_names);
+    std::shared_ptr<arrow::Tensor> arrow_tensor = arrow_numeric_tensor;
+    auto tensor = garrow_tensor_new_raw(&arrow_tensor);
+    return tensor;
+  }
+
+  template <typename T, typename value_type>
+  const value_type *numeric_tensor_get_raw_data(GArrowTensor *tensor,
+                                                gint64 *n_data) {
+    auto arrow_tensor = garrow_tensor_get_raw(tensor);
+    auto arrow_numeric_tensor = static_cast<const T *>(arrow_tensor.get());
+    *n_data = arrow_numeric_tensor->size();
+    return arrow_numeric_tensor->raw_data();
+  }
+}
diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp
new file mode 100644
index 0000000000000..cbc9d8e31fe9d
--- /dev/null
+++ b/c_glib/arrow-glib/tensor.cpp
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/buffer.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/int8-tensor.h>
+#include <arrow-glib/tensor.hpp>
+#include <arrow-glib/type.hpp>
+#include <arrow-glib/uint8-tensor.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: tensor
+ * @short_description: Base class for all tensor classes
+ *
+ * #GArrowTensor is a base class for all tensor classes such as
+ * #GArrowInt8Tensor.
+ * #GArrowBooleanTensorBuilder to create a new tensor.
+ *
+ * Since: 0.3.0
+ */
+
+typedef struct GArrowTensorPrivate_ {
+  std::shared_ptr<arrow::Tensor> tensor;
+} GArrowTensorPrivate;
+
+enum {
+  PROP_0,
+  PROP_TENSOR
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowTensor, garrow_tensor, G_TYPE_OBJECT)
+
+#define GARROW_TENSOR_GET_PRIVATE(obj)                                   \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj), GARROW_TYPE_TENSOR, GArrowTensorPrivate))
+
+static void
+garrow_tensor_finalize(GObject *object)
+{
+  auto priv = GARROW_TENSOR_GET_PRIVATE(object);
+
+  priv->tensor = nullptr;
+
+  G_OBJECT_CLASS(garrow_tensor_parent_class)->finalize(object);
+}
+
+static void
+garrow_tensor_set_property(GObject *object,
+                          guint prop_id,
+                          const GValue *value,
+                          GParamSpec *pspec)
+{
+  auto priv = GARROW_TENSOR_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_TENSOR:
+    priv->tensor =
+      *static_cast<std::shared_ptr<arrow::Tensor> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_tensor_get_property(GObject *object,
+                          guint prop_id,
+                          GValue *value,
+                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_tensor_init(GArrowTensor *object)
+{
+}
+
+static void
+garrow_tensor_class_init(GArrowTensorClass *klass)
+{
+  GParamSpec *spec;
+
+  auto gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_tensor_finalize;
+  gobject_class->set_property = garrow_tensor_set_property;
+  gobject_class->get_property = garrow_tensor_get_property;
+
+  spec = g_param_spec_pointer("tensor",
+                              "Tensor",
+                              "The raw std::shared<arrow::Tensor> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_TENSOR, spec);
+}
+
+/**
+ * garrow_tensor_get_value_data_type:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: (transfer full): The data type of each value in the tensor.
+ *
+ * Since: 0.3.0
+ */
+GArrowDataType *
+garrow_tensor_get_value_data_type(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  auto arrow_data_type = arrow_tensor->type();
+  return garrow_data_type_new_raw(&arrow_data_type);
+}
+
+/**
+ * garrow_tensor_get_value_type:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: The type of each value in the tensor.
+ *
+ * Since: 0.3.0
+ */
+GArrowType
+garrow_tensor_get_value_type(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  auto arrow_type = arrow_tensor->type_id();
+  return garrow_type_from_raw(arrow_type);
+}
+
+/**
+ * garrow_tensor_get_buffer:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: (transfer full): The data of the tensor.
+ *
+ * Since: 0.3.0
+ */
+GArrowBuffer *
+garrow_tensor_get_buffer(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  auto arrow_buffer = arrow_tensor->data();
+  return garrow_buffer_new_raw(&arrow_buffer);
+}
+
+/**
+ * garrow_tensor_get_shape:
+ * @tensor: A #GArrowTensor.
+ * @n_dimensions: (out): The number of dimensions.
+ *
+ * Returns: (array length=n_dimensions): The shape of the tensor.
+ *   It should be freed with g_free() when no longer needed.
+ *
+ * Since: 0.3.0
+ */
+gint64 *
+garrow_tensor_get_shape(GArrowTensor *tensor, gint *n_dimensions)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  auto arrow_shape = arrow_tensor->shape();
+  auto n_dimensions_raw = arrow_shape.size();
+  auto shape =
+    static_cast<gint64 *>(g_malloc_n(sizeof(gint64), n_dimensions_raw));
+  for (gsize i = 0; i < n_dimensions_raw; ++i) {
+    shape[i] = arrow_shape[i];
+  }
+  *n_dimensions = static_cast<gint>(n_dimensions_raw);
+  return shape;
+}
+
+/**
+ * garrow_tensor_get_strides:
+ * @tensor: A #GArrowTensor.
+ * @n_strides: (out): The number of strides.
+ *
+ * Returns: (array length=n_strides): The strides of the tensor.
+ *   It should be freed with g_free() when no longer needed.
+ *
+ * Since: 0.3.0
+ */
+gint64 *
+garrow_tensor_get_strides(GArrowTensor *tensor, gint *n_strides)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  auto arrow_strides = arrow_tensor->strides();
+  auto n_strides_raw = arrow_strides.size();
+  auto strides =
+    static_cast<gint64 *>(g_malloc_n(sizeof(gint64), n_strides_raw));
+  for (gsize i = 0; i < n_strides_raw; ++i) {
+    strides[i] = arrow_strides[i];
+  }
+  *n_strides = static_cast<gint>(n_strides_raw);
+  return strides;
+}
+
+/**
+ * garrow_tensor_get_n_dimensions:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: The number of dimensions of the tensor.
+ *
+ * Since: 0.3.0
+ */
+gint
+garrow_tensor_get_n_dimensions(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  return arrow_tensor->ndim();
+}
+
+/**
+ * garrow_tensor_get_dimension_name:
+ * @tensor: A #GArrowTensor.
+ * @i: The index of the target dimension.
+ *
+ * Returns: The i-th dimension name of the tensor.
+ *
+ * Since: 0.3.0
+ */
+const gchar *
+garrow_tensor_get_dimension_name(GArrowTensor *tensor, gint i)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  auto arrow_dimension_name = arrow_tensor->dim_name(i);
+  return arrow_dimension_name.c_str();
+}
+
+/**
+ * garrow_tensor_get_size:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: The number of value cells in the tensor.
+ *
+ * Since: 0.3.0
+ */
+gint64
+garrow_tensor_get_size(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  return arrow_tensor->size();
+}
+
+/**
+ * garrow_tensor_is_mutable:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: %TRUE if the tensor is mutable, %FALSE otherwise.
+ *
+ * Since: 0.3.0
+ */
+gboolean
+garrow_tensor_is_mutable(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  return arrow_tensor->is_mutable();
+}
+
+/**
+ * garrow_tensor_is_contiguous:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: %TRUE if the tensor is contiguous, %FALSE otherwise.
+ *
+ * Since: 0.3.0
+ */
+gboolean
+garrow_tensor_is_contiguous(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  return arrow_tensor->is_contiguous();
+}
+
+/**
+ * garrow_tensor_is_row_major:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: %TRUE if the tensor is row major a.k.a. C order,
+ *   %FALSE otherwise.
+ *
+ * Since: 0.3.0
+ */
+gboolean
+garrow_tensor_is_row_major(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  return arrow_tensor->is_row_major();
+}
+
+/**
+ * garrow_tensor_is_column_major:
+ * @tensor: A #GArrowTensor.
+ *
+ * Returns: %TRUE if the tensor is column major a.k.a. Fortran order,
+ *   %FALSE otherwise.
+ *
+ * Since: 0.3.0
+ */
+gboolean
+garrow_tensor_is_column_major(GArrowTensor *tensor)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  return arrow_tensor->is_column_major();
+}
+
+G_END_DECLS
+
+GArrowTensor *
+garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor)
+{
+  GType type;
+  GArrowTensor *tensor;
+
+  switch ((*arrow_tensor)->type_id()) {
+  case arrow::Type::type::UINT8:
+    type = GARROW_TYPE_UINT8_TENSOR;
+    break;
+  case arrow::Type::type::INT8:
+    type = GARROW_TYPE_INT8_TENSOR;
+    break;
+/*
+  case arrow::Type::type::UINT16:
+    type = GARROW_TYPE_UINT16_TENSOR;
+    break;
+  case arrow::Type::type::INT16:
+    type = GARROW_TYPE_INT16_TENSOR;
+    break;
+  case arrow::Type::type::UINT32:
+    type = GARROW_TYPE_UINT32_TENSOR;
+    break;
+  case arrow::Type::type::INT32:
+    type = GARROW_TYPE_INT32_TENSOR;
+    break;
+  case arrow::Type::type::UINT64:
+    type = GARROW_TYPE_UINT64_TENSOR;
+    break;
+  case arrow::Type::type::INT64:
+    type = GARROW_TYPE_INT64_TENSOR;
+    break;
+  case arrow::Type::type::HALF_FLOAT:
+    type = GARROW_TYPE_HALF_FLOAT_TENSOR;
+    break;
+  case arrow::Type::type::FLOAT:
+    type = GARROW_TYPE_FLOAT_TENSOR;
+    break;
+  case arrow::Type::type::DOUBLE:
+    type = GARROW_TYPE_DOUBLE_TENSOR;
+    break;
+*/
+  default:
+    type = GARROW_TYPE_TENSOR;
+    break;
+  }
+  tensor = GARROW_TENSOR(g_object_new(type,
+                                    "tensor", arrow_tensor,
+                                    NULL));
+  return tensor;
+}
+
+std::shared_ptr<arrow::Tensor>
+garrow_tensor_get_raw(GArrowTensor *tensor)
+{
+  auto priv = GARROW_TENSOR_GET_PRIVATE(tensor);
+  return priv->tensor;
+}
diff --git a/c_glib/arrow-glib/tensor.h b/c_glib/arrow-glib/tensor.h
new file mode 100644
index 0000000000000..bedc80324f581
--- /dev/null
+++ b/c_glib/arrow-glib/tensor.h
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/buffer.h>
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_TENSOR \
+  (garrow_tensor_get_type())
+#define GARROW_TENSOR(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj), GARROW_TYPE_TENSOR, GArrowTensor))
+#define GARROW_TENSOR_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST((klass), GARROW_TYPE_TENSOR, GArrowTensorClass))
+#define GARROW_IS_TENSOR(obj) \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_TENSOR))
+#define GARROW_IS_TENSOR_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_TENSOR))
+#define GARROW_TENSOR_GET_CLASS(obj) \
+  (G_TYPE_INSTANCE_GET_CLASS((obj), GARROW_TYPE_TENSOR, GArrowTensorClass))
+
+typedef struct _GArrowTensor         GArrowTensor;
+typedef struct _GArrowTensorClass    GArrowTensorClass;
+
+/**
+ * GArrowTensor:
+ *
+ * It wraps `arrow::Tensor`.
+ */
+struct _GArrowTensor
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowTensorClass
+{
+  GObjectClass parent_class;
+};
+
+GType           garrow_tensor_get_type           (void) G_GNUC_CONST;
+
+GArrowDataType *garrow_tensor_get_value_data_type(GArrowTensor *tensor);
+GArrowType      garrow_tensor_get_value_type     (GArrowTensor *tensor);
+GArrowBuffer   *garrow_tensor_get_buffer         (GArrowTensor *tensor);
+gint64         *garrow_tensor_get_shape          (GArrowTensor *tensor,
+                                                  gint *n_dimensions);
+gint64         *garrow_tensor_get_strides        (GArrowTensor *tensor,
+                                                  gint *n_strides);
+gint            garrow_tensor_get_n_dimensions   (GArrowTensor *tensor);
+const gchar    *garrow_tensor_get_dimension_name (GArrowTensor *tensor,
+                                                  gint i);
+gint64          garrow_tensor_get_size           (GArrowTensor *tensor);
+gboolean        garrow_tensor_is_mutable         (GArrowTensor *tensor);
+gboolean        garrow_tensor_is_contiguous      (GArrowTensor *tensor);
+gboolean        garrow_tensor_is_row_major       (GArrowTensor *tensor);
+gboolean        garrow_tensor_is_column_major    (GArrowTensor *tensor);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/tensor.hpp b/c_glib/arrow-glib/tensor.hpp
new file mode 100644
index 0000000000000..392aeeebb6d2c
--- /dev/null
+++ b/c_glib/arrow-glib/tensor.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/tensor.h>
+
+GArrowTensor *garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor);
+std::shared_ptr<arrow::Tensor> garrow_tensor_get_raw(GArrowTensor *tensor);
diff --git a/c_glib/arrow-glib/uint8-tensor.cpp b/c_glib/arrow-glib/uint8-tensor.cpp
new file mode 100644
index 0000000000000..69f0f694530ad
--- /dev/null
+++ b/c_glib/arrow-glib/uint8-tensor.cpp
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/buffer.hpp>
+#include <arrow-glib/uint8-tensor.h>
+#include <arrow-glib/numeric-tensor.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: uint8-tensor
+ * @short_description: 8-bit unsigned integer tensor class
+ *
+ * #GArrowUint8Tensor is a class for 8-bit unsigned integer tensor. It
+ * can store zero or more 8-bit integer data.
+ */
+
+G_DEFINE_TYPE(GArrowUInt8Tensor,               \
+              garrow_uint8_tensor,             \
+              GARROW_TYPE_TENSOR)
+
+static void
+garrow_uint8_tensor_init(GArrowUInt8Tensor *object)
+{
+}
+
+static void
+garrow_uint8_tensor_class_init(GArrowUInt8TensorClass *klass)
+{
+}
+
+/**
+ * garrow_uint8_tensor_new:
+ * @data: A #GArrowBuffer that contains tensor data.
+ * @shape: (array length=n_dimensions): A list of dimension sizes.
+ * @n_dimensions: The number of dimensions.
+ * @strides: (array length=n_strides) (nullable): A list of the number of
+ *   bytes in each dimension.
+ * @n_strides: The number of strides.
+ * @dimention_names: (array length=n_dimention_names) (nullable): A list of
+ *   dimension names.
+ * @n_dimention_names: The number of dimension names
+ *
+ * Returns: The newly created #GArrowUInt8Tensor.
+ *
+ * Since: 0.3.0
+ */
+GArrowUInt8Tensor *
+garrow_uint8_tensor_new(GArrowBuffer *data,
+                       gint64 *shape,
+                       gsize n_dimensions,
+                       gint64 *strides,
+                       gsize n_strides,
+                       gchar **dimension_names,
+                       gsize n_dimension_names)
+{
+  auto tensor =
+    garrow::numeric_tensor_new<arrow::UInt8Tensor>(data,
+                                                  shape,
+                                                  n_dimensions,
+                                                  strides,
+                                                  n_strides,
+                                                  dimension_names,
+                                                  n_dimension_names);
+  return GARROW_UINT8_TENSOR(tensor);
+}
+
+/**
+ * garrow_uint8_tensor_get_raw_data:
+ * @tensor: A #GArrowUInt8Tensor.
+ * @n_data: (out): The number of data.
+ *
+ * Returns: (array length=n_data): The raw data in the tensor.
+ *
+ * Since: 0.3.0
+ */
+const guint8 *
+garrow_uint8_tensor_get_raw_data(GArrowUInt8Tensor *tensor,
+                                 gint64 *n_data)
+{
+  return garrow::numeric_tensor_get_raw_data<arrow::UInt8Tensor, uint8_t>(GARROW_TENSOR(tensor),
+                                                                          n_data);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-tensor.h b/c_glib/arrow-glib/uint8-tensor.h
new file mode 100644
index 0000000000000..248c507b4f646
--- /dev/null
+++ b/c_glib/arrow-glib/uint8-tensor.h
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/tensor.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_UINT8_TENSOR                \
+  (garrow_uint8_tensor_get_type())
+#define GARROW_UINT8_TENSOR(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT8_TENSOR, \
+                              GArrowUInt8Tensor))
+#define GARROW_UINT8_TENSOR_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT8_TENSOR,    \
+                           GArrowUInt8TensorClass))
+#define GARROW_IS_UINT8_TENSOR(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT8_TENSOR))
+#define GARROW_IS_UINT8_TENSOR_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT8_TENSOR))
+#define GARROW_UINT8_TENSOR_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT8_TENSOR,  \
+                             GArrowUInt8TensorClass))
+
+typedef struct _GArrowUInt8Tensor         GArrowUInt8Tensor;
+typedef struct _GArrowUInt8TensorClass    GArrowUInt8TensorClass;
+
+/**
+ * GArrowUInt8Tensor:
+ *
+ * It wraps `arrow::UInt8Tensor`.
+ */
+struct _GArrowUInt8Tensor
+{
+  /*< private >*/
+  GArrowTensor parent_instance;
+};
+
+struct _GArrowUInt8TensorClass
+{
+  GArrowTensorClass parent_class;
+};
+
+GType garrow_uint8_tensor_get_type(void) G_GNUC_CONST;
+
+GArrowUInt8Tensor *garrow_uint8_tensor_new(GArrowBuffer *data,
+                                           gint64 *shape,
+                                           gsize n_dimensions,
+                                           gint64 *strides,
+                                           gsize n_strides,
+                                           gchar **dimention_names,
+                                           gsize n_dimention_names);
+
+const guint8 *garrow_uint8_tensor_get_raw_data(GArrowUInt8Tensor *tensor,
+                                               gint64 *n_data);
+
+G_END_DECLS
diff --git a/c_glib/test/helper/omittable.rb b/c_glib/test/helper/omittable.rb
new file mode 100644
index 0000000000000..a16ad32485e15
--- /dev/null
+++ b/c_glib/test/helper/omittable.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module Omittable
+    def require_gi(major, minor, micro)
+      return if GLib.check_binding_version?(major, minor, micro)
+      message =
+        "Require gobject-introspection #{major}.#{minor}.#{micro} or later: " +
+        GLib::BINDING_VERSION.join(".")
+      omit(message)
+    end
+  end
+end
diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb
index 53805caef374f..50f548f3f5b3b 100755
--- a/c_glib/test/run-test.rb
+++ b/c_glib/test/run-test.rb
@@ -35,5 +35,6 @@
 
 require "tempfile"
 require_relative "helper/buildable"
+require_relative "helper/omittable"
 
 exit(Test::Unit::AutoRunner.run(true, test_dir.to_s))
diff --git a/c_glib/test/test-int8-tensor.rb b/c_glib/test/test-int8-tensor.rb
new file mode 100644
index 0000000000000..a96a4076b5a8e
--- /dev/null
+++ b/c_glib/test/test-int8-tensor.rb
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestInt8Tensor < Test::Unit::TestCase
+  include Helper::Omittable
+
+  def setup
+    @raw_data = [
+      1, 2,
+      3, 4,
+
+      5, 6,
+      7, 8,
+
+      9, 10,
+      11, 12,
+    ]
+    data = Arrow::Buffer.new(@raw_data.pack("c*"))
+    shape = [3, 2, 2]
+    strides = []
+    names = []
+    @tensor = Arrow::Int8Tensor.new(data, shape, strides, names)
+  end
+
+  def test_raw_data
+    require_gi(3, 1, 2)
+    assert_equal(@raw_data, @tensor.raw_data)
+  end
+end
diff --git a/c_glib/test/test-tensor.rb b/c_glib/test/test-tensor.rb
new file mode 100644
index 0000000000000..455b0d9d90acb
--- /dev/null
+++ b/c_glib/test/test-tensor.rb
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTensor < Test::Unit::TestCase
+  include Helper::Omittable
+
+  def setup
+    @raw_data = [
+      1, 2,
+      3, 4,
+
+      5, 6,
+      7, 8,
+
+      9, 10,
+      11, 12,
+    ]
+    data = Arrow::Buffer.new(@raw_data.pack("c*"))
+    @shape = [3, 2, 2]
+    strides = []
+    names = ["a", "b", "c"]
+    @tensor = Arrow::Int8Tensor.new(data, @shape, strides, names)
+  end
+
+  def test_value_data_type
+    assert_equal(Arrow::Int8DataType, @tensor.value_data_type.class)
+  end
+
+  def test_value_type
+    assert_equal(Arrow::Type::INT8, @tensor.value_type)
+  end
+
+  def test_buffer
+    assert_equal(@raw_data, @tensor.buffer.data)
+  end
+
+  def test_shape
+    require_gi(3, 1, 2)
+    assert_equal(@shape, @tensor.shape)
+  end
+
+  def test_strides
+    require_gi(3, 1, 2)
+    assert_equal([4, 2, 1], @tensor.strides)
+  end
+
+  def test_n_dimensions
+    assert_equal(@shape.size, @tensor.n_dimensions)
+  end
+
+  def test_dimension_name
+    dimension_names = @tensor.n_dimensions.times.collect do |i|
+      @tensor.get_dimension_name(i)
+    end
+    assert_equal(["a", "b", "c"],
+                 dimension_names)
+  end
+
+  def test_size
+    assert_equal(@raw_data.size, @tensor.size)
+  end
+
+  def test_mutable?
+    assert do
+      not @tensor.mutable?
+    end
+  end
+
+  def test_contiguous?
+    assert do
+      @tensor.contiguous?
+    end
+  end
+
+  def test_row_major?
+    assert do
+      @tensor.row_major?
+    end
+  end
+
+  def test_column_major?
+    assert do
+      not @tensor.column_major?
+    end
+  end
+end
diff --git a/c_glib/test/test-uint8-tensor.rb b/c_glib/test/test-uint8-tensor.rb
new file mode 100644
index 0000000000000..0fe758ba676cc
--- /dev/null
+++ b/c_glib/test/test-uint8-tensor.rb
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUInt8Tensor < Test::Unit::TestCase
+  include Helper::Omittable
+
+  def setup
+    @raw_data = [
+      1, 2,
+      3, 4,
+
+      5, 6,
+      7, 8,
+
+      9, 10,
+      11, 12,
+    ]
+    data = Arrow::Buffer.new(@raw_data.pack("c*"))
+    shape = [3, 2, 2]
+    strides = []
+    names = []
+    @tensor = Arrow::UInt8Tensor.new(data, shape, strides, names)
+  end
+
+  def test_raw_data
+    require_gi(3, 1, 2)
+    assert_equal(@raw_data, @tensor.raw_data)
+  end
+end

From a68f31b0f3f2c094c5d6660a2d936baa05da3103 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 20 Apr 2017 09:36:21 +0200
Subject: [PATCH 0542/1644] ARROW-860: [C++] Remove typed Tensor containers

cc @kou for opinions -- this patch breaks glib for the moment. Since tensors are all fixed width types, there's less reason to have strongly-typed containers for them (unlike the `arrow::Array` subclasses, where ListArray is quite different from Int8Array).

My view is that if the visitor pattern needs to be employed, we can do it using the `type()` member on the tensor (which also provides compile-time access to `TypeClass::c_type` if needed)

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Kouhei Sutou <kou@clear-code.com>

Closes #571 from wesm/ARROW-860 and squashes the following commits:

fe0b4d8 [Kouhei Sutou] Remove typed Tensors from glib
357f441 [Wes McKinney] Remove typed Tensor containers
---
 c_glib/arrow-glib/Makefile.am            |   5 --
 c_glib/arrow-glib/arrow-glib.h           |   2 -
 c_glib/arrow-glib/int8-tensor.cpp        | 105 -----------------------
 c_glib/arrow-glib/int8-tensor.h          |  79 -----------------
 c_glib/arrow-glib/numeric-tensor.hpp     |  64 --------------
 c_glib/arrow-glib/tensor.cpp             | 103 +++++++++++-----------
 c_glib/arrow-glib/tensor.h               |   8 ++
 c_glib/arrow-glib/uint8-tensor.cpp       | 105 -----------------------
 c_glib/arrow-glib/uint8-tensor.h         |  79 -----------------
 c_glib/test/test-int8-tensor.rb          |  43 ----------
 c_glib/test/test-tensor.rb               |   6 +-
 c_glib/test/test-uint8-tensor.rb         |  43 ----------
 cpp/src/arrow/compare.cc                 |  37 ++------
 cpp/src/arrow/ipc/ipc-read-write-test.cc |   6 +-
 cpp/src/arrow/ipc/reader.cc              |   3 +-
 cpp/src/arrow/python/numpy_convert.cc    |   3 +-
 cpp/src/arrow/tensor-test.cc             |  14 +--
 cpp/src/arrow/tensor.cc                  |  65 --------------
 cpp/src/arrow/tensor.h                   |  47 +---------
 cpp/src/arrow/visitor_inline.h           |  25 ------
 20 files changed, 94 insertions(+), 748 deletions(-)
 delete mode 100644 c_glib/arrow-glib/int8-tensor.cpp
 delete mode 100644 c_glib/arrow-glib/int8-tensor.h
 delete mode 100644 c_glib/arrow-glib/numeric-tensor.hpp
 delete mode 100644 c_glib/arrow-glib/uint8-tensor.cpp
 delete mode 100644 c_glib/arrow-glib/uint8-tensor.h
 delete mode 100644 c_glib/test/test-int8-tensor.rb
 delete mode 100644 c_glib/test/test-uint8-tensor.rb

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index fbfe3a4071000..11b6508df0745 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -65,7 +65,6 @@ libarrow_glib_la_headers =			\
 	int8-array.h				\
 	int8-array-builder.h			\
 	int8-data-type.h			\
-	int8-tensor.h				\
 	int16-array.h				\
 	int16-array-builder.h			\
 	int16-data-type.h			\
@@ -94,7 +93,6 @@ libarrow_glib_la_headers =			\
 	uint8-array.h				\
 	uint8-array-builder.h			\
 	uint8-data-type.h			\
-	uint8-tensor.h				\
 	uint16-array.h				\
 	uint16-array-builder.h			\
 	uint16-data-type.h			\
@@ -155,7 +153,6 @@ libarrow_glib_la_sources =			\
 	int8-array.cpp				\
 	int8-array-builder.cpp			\
 	int8-data-type.cpp			\
-	int8-tensor.cpp				\
 	int16-array.cpp				\
 	int16-array-builder.cpp			\
 	int16-data-type.cpp			\
@@ -184,7 +181,6 @@ libarrow_glib_la_sources =			\
 	uint8-array.cpp				\
 	uint8-array-builder.cpp			\
 	uint8-data-type.cpp			\
-	uint8-tensor.cpp			\
 	uint16-array.cpp			\
 	uint16-array-builder.cpp		\
 	uint16-data-type.cpp			\
@@ -226,7 +222,6 @@ libarrow_glib_la_cpp_headers =			\
 	data-type.hpp				\
 	error.hpp				\
 	field.hpp				\
-	numeric-tensor.hpp			\
 	record-batch.hpp			\
 	schema.hpp				\
 	table.hpp				\
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index eec9e25ebf690..8d9bfe2da9c38 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -42,7 +42,6 @@
 #include <arrow-glib/int8-array.h>
 #include <arrow-glib/int8-array-builder.h>
 #include <arrow-glib/int8-data-type.h>
-#include <arrow-glib/int8-tensor.h>
 #include <arrow-glib/int16-array.h>
 #include <arrow-glib/int16-array-builder.h>
 #include <arrow-glib/int16-data-type.h>
@@ -71,7 +70,6 @@
 #include <arrow-glib/uint8-array.h>
 #include <arrow-glib/uint8-array-builder.h>
 #include <arrow-glib/uint8-data-type.h>
-#include <arrow-glib/uint8-tensor.h>
 #include <arrow-glib/uint16-array.h>
 #include <arrow-glib/uint16-array-builder.h>
 #include <arrow-glib/uint16-data-type.h>
diff --git a/c_glib/arrow-glib/int8-tensor.cpp b/c_glib/arrow-glib/int8-tensor.cpp
deleted file mode 100644
index 06521a00997c0..0000000000000
--- a/c_glib/arrow-glib/int8-tensor.cpp
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/buffer.hpp>
-#include <arrow-glib/int8-tensor.h>
-#include <arrow-glib/numeric-tensor.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int8-tensor
- * @short_description: 8-bit integer tensor class
- *
- * #GArrowInt8Tensor is a class for 8-bit integer tensor. It can store
- * zero or more 8-bit integer data.
- */
-
-G_DEFINE_TYPE(GArrowInt8Tensor,               \
-              garrow_int8_tensor,             \
-              GARROW_TYPE_TENSOR)
-
-static void
-garrow_int8_tensor_init(GArrowInt8Tensor *object)
-{
-}
-
-static void
-garrow_int8_tensor_class_init(GArrowInt8TensorClass *klass)
-{
-}
-
-/**
- * garrow_int8_tensor_new:
- * @data: A #GArrowBuffer that contains tensor data.
- * @shape: (array length=n_dimensions): A list of dimension sizes.
- * @n_dimensions: The number of dimensions.
- * @strides: (array length=n_strides) (nullable): A list of the number of
- *   bytes in each dimension.
- * @n_strides: The number of strides.
- * @dimention_names: (array length=n_dimention_names) (nullable): A list of
- *   dimension names.
- * @n_dimention_names: The number of dimension names
- *
- * Returns: The newly created #GArrowInt8Tensor.
- *
- * Since: 0.3.0
- */
-GArrowInt8Tensor *
-garrow_int8_tensor_new(GArrowBuffer *data,
-                       gint64 *shape,
-                       gsize n_dimensions,
-                       gint64 *strides,
-                       gsize n_strides,
-                       gchar **dimension_names,
-                       gsize n_dimension_names)
-{
-  auto tensor =
-    garrow::numeric_tensor_new<arrow::Int8Tensor>(data,
-                                                  shape,
-                                                  n_dimensions,
-                                                  strides,
-                                                  n_strides,
-                                                  dimension_names,
-                                                  n_dimension_names);
-  return GARROW_INT8_TENSOR(tensor);
-}
-
-/**
- * garrow_int8_tensor_get_raw_data:
- * @tensor: A #GArrowInt8Tensor.
- * @n_data: (out): The number of data.
- *
- * Returns: (array length=n_data): The raw data in the tensor.
- *
- * Since: 0.3.0
- */
-const gint8 *
-garrow_int8_tensor_get_raw_data(GArrowInt8Tensor *tensor,
-                                gint64 *n_data)
-{
-  return garrow::numeric_tensor_get_raw_data<arrow::Int8Tensor, int8_t>(GARROW_TENSOR(tensor),
-                                                                        n_data);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-tensor.h b/c_glib/arrow-glib/int8-tensor.h
deleted file mode 100644
index 76ed3c8d7a7ee..0000000000000
--- a/c_glib/arrow-glib/int8-tensor.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/tensor.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT8_TENSOR                 \
-  (garrow_int8_tensor_get_type())
-#define GARROW_INT8_TENSOR(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT8_TENSOR,  \
-                              GArrowInt8Tensor))
-#define GARROW_INT8_TENSOR_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT8_TENSOR,     \
-                           GArrowInt8TensorClass))
-#define GARROW_IS_INT8_TENSOR(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT8_TENSOR))
-#define GARROW_IS_INT8_TENSOR_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT8_TENSOR))
-#define GARROW_INT8_TENSOR_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT8_TENSOR,   \
-                             GArrowInt8TensorClass))
-
-typedef struct _GArrowInt8Tensor         GArrowInt8Tensor;
-typedef struct _GArrowInt8TensorClass    GArrowInt8TensorClass;
-
-/**
- * GArrowInt8Tensor:
- *
- * It wraps `arrow::Int8Tensor`.
- */
-struct _GArrowInt8Tensor
-{
-  /*< private >*/
-  GArrowTensor parent_instance;
-};
-
-struct _GArrowInt8TensorClass
-{
-  GArrowTensorClass parent_class;
-};
-
-GType garrow_int8_tensor_get_type(void) G_GNUC_CONST;
-
-GArrowInt8Tensor *garrow_int8_tensor_new(GArrowBuffer *data,
-                                         gint64 *shape,
-                                         gsize n_dimensions,
-                                         gint64 *strides,
-                                         gsize n_strides,
-                                         gchar **dimention_names,
-                                         gsize n_dimention_names);
-
-const gint8 *garrow_int8_tensor_get_raw_data(GArrowInt8Tensor *tensor,
-                                             gint64 *n_data);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/numeric-tensor.hpp b/c_glib/arrow-glib/numeric-tensor.hpp
deleted file mode 100644
index 07cea62bd7b25..0000000000000
--- a/c_glib/arrow-glib/numeric-tensor.hpp
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/tensor.hpp>
-
-namespace garrow {
-  template <typename T>
-  GArrowTensor *numeric_tensor_new(GArrowBuffer *data,
-                                   gint64 *shape,
-                                   gsize n_dimensions,
-                                   gint64 *strides,
-                                   gsize n_strides,
-                                   gchar **dimention_names,
-                                   gsize n_dimention_names) {
-    auto arrow_data = garrow_buffer_get_raw(data);
-    std::vector<int64_t> arrow_shape;
-    for (gsize i = 0; i < n_dimensions; ++i) {
-      arrow_shape.push_back(shape[i]);
-    }
-    std::vector<int64_t> arrow_strides;
-    for (gsize i = 0; i < n_strides; ++i) {
-      arrow_strides.push_back(strides[i]);
-    }
-    std::vector<std::string> arrow_dimention_names;
-    for (gsize i = 0; i < n_dimention_names; ++i) {
-      arrow_dimention_names.push_back(dimention_names[i]);
-    }
-    auto arrow_numeric_tensor =
-      std::make_shared<T>(arrow_data,
-                          arrow_shape,
-                          arrow_strides,
-                          arrow_dimention_names);
-    std::shared_ptr<arrow::Tensor> arrow_tensor = arrow_numeric_tensor;
-    auto tensor = garrow_tensor_new_raw(&arrow_tensor);
-    return tensor;
-  }
-
-  template <typename T, typename value_type>
-  const value_type *numeric_tensor_get_raw_data(GArrowTensor *tensor,
-                                                gint64 *n_data) {
-    auto arrow_tensor = garrow_tensor_get_raw(tensor);
-    auto arrow_numeric_tensor = static_cast<const T *>(arrow_tensor.get());
-    *n_data = arrow_numeric_tensor->size();
-    return arrow_numeric_tensor->raw_data();
-  }
-}
diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp
index cbc9d8e31fe9d..468eb0729357b 100644
--- a/c_glib/arrow-glib/tensor.cpp
+++ b/c_glib/arrow-glib/tensor.cpp
@@ -23,10 +23,8 @@
 
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/data-type.hpp>
-#include <arrow-glib/int8-tensor.h>
 #include <arrow-glib/tensor.hpp>
 #include <arrow-glib/type.hpp>
-#include <arrow-glib/uint8-tensor.h>
 
 G_BEGIN_DECLS
 
@@ -121,6 +119,58 @@ garrow_tensor_class_init(GArrowTensorClass *klass)
   g_object_class_install_property(gobject_class, PROP_TENSOR, spec);
 }
 
+/**
+ * garrow_tensor_new:
+ * @data_type: A #GArrowDataType that indicates each element type
+ *   in the tensor.
+ * @data: A #GArrowBuffer that contains tensor data.
+ * @shape: (array length=n_dimensions): A list of dimension sizes.
+ * @n_dimensions: The number of dimensions.
+ * @strides: (array length=n_strides) (nullable): A list of the number of
+ *   bytes in each dimension.
+ * @n_strides: The number of strides.
+ * @dimention_names: (array length=n_dimention_names) (nullable): A list of
+ *   dimension names.
+ * @n_dimention_names: The number of dimension names
+ *
+ * Returns: The newly created #GArrowTensor.
+ *
+ * Since: 0.3.0
+ */
+GArrowTensor *
+garrow_tensor_new(GArrowDataType *data_type,
+                  GArrowBuffer *data,
+                  gint64 *shape,
+                  gsize n_dimensions,
+                  gint64 *strides,
+                  gsize n_strides,
+                  gchar **dimension_names,
+                  gsize n_dimension_names)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  auto arrow_data = garrow_buffer_get_raw(data);
+  std::vector<int64_t> arrow_shape;
+  for (gsize i = 0; i < n_dimensions; ++i) {
+    arrow_shape.push_back(shape[i]);
+  }
+  std::vector<int64_t> arrow_strides;
+  for (gsize i = 0; i < n_strides; ++i) {
+    arrow_strides.push_back(strides[i]);
+  }
+  std::vector<std::string> arrow_dimension_names;
+  for (gsize i = 0; i < n_dimension_names; ++i) {
+    arrow_dimension_names.push_back(dimension_names[i]);
+  }
+  auto arrow_tensor =
+    std::make_shared<arrow::Tensor>(arrow_data_type,
+                                    arrow_data,
+                                    arrow_shape,
+                                    arrow_strides,
+                                    arrow_dimension_names);
+  auto tensor = garrow_tensor_new_raw(&arrow_tensor);
+  return tensor;
+}
+
 /**
  * garrow_tensor_get_value_data_type:
  * @tensor: A #GArrowTensor.
@@ -333,52 +383,9 @@ G_END_DECLS
 GArrowTensor *
 garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor)
 {
-  GType type;
-  GArrowTensor *tensor;
-
-  switch ((*arrow_tensor)->type_id()) {
-  case arrow::Type::type::UINT8:
-    type = GARROW_TYPE_UINT8_TENSOR;
-    break;
-  case arrow::Type::type::INT8:
-    type = GARROW_TYPE_INT8_TENSOR;
-    break;
-/*
-  case arrow::Type::type::UINT16:
-    type = GARROW_TYPE_UINT16_TENSOR;
-    break;
-  case arrow::Type::type::INT16:
-    type = GARROW_TYPE_INT16_TENSOR;
-    break;
-  case arrow::Type::type::UINT32:
-    type = GARROW_TYPE_UINT32_TENSOR;
-    break;
-  case arrow::Type::type::INT32:
-    type = GARROW_TYPE_INT32_TENSOR;
-    break;
-  case arrow::Type::type::UINT64:
-    type = GARROW_TYPE_UINT64_TENSOR;
-    break;
-  case arrow::Type::type::INT64:
-    type = GARROW_TYPE_INT64_TENSOR;
-    break;
-  case arrow::Type::type::HALF_FLOAT:
-    type = GARROW_TYPE_HALF_FLOAT_TENSOR;
-    break;
-  case arrow::Type::type::FLOAT:
-    type = GARROW_TYPE_FLOAT_TENSOR;
-    break;
-  case arrow::Type::type::DOUBLE:
-    type = GARROW_TYPE_DOUBLE_TENSOR;
-    break;
-*/
-  default:
-    type = GARROW_TYPE_TENSOR;
-    break;
-  }
-  tensor = GARROW_TENSOR(g_object_new(type,
-                                    "tensor", arrow_tensor,
-                                    NULL));
+  auto tensor = GARROW_TENSOR(g_object_new(GARROW_TYPE_TENSOR,
+                                           "tensor", arrow_tensor,
+                                           NULL));
   return tensor;
 }
 
diff --git a/c_glib/arrow-glib/tensor.h b/c_glib/arrow-glib/tensor.h
index bedc80324f581..71c6b4e9031dd 100644
--- a/c_glib/arrow-glib/tensor.h
+++ b/c_glib/arrow-glib/tensor.h
@@ -58,6 +58,14 @@ struct _GArrowTensorClass
 
 GType           garrow_tensor_get_type           (void) G_GNUC_CONST;
 
+GArrowTensor   *garrow_tensor_new                (GArrowDataType *data_type,
+                                                  GArrowBuffer *data,
+                                                  gint64 *shape,
+                                                  gsize n_dimensions,
+                                                  gint64 *strides,
+                                                  gsize n_strides,
+                                                  gchar **dimention_names,
+                                                  gsize n_dimention_names);
 GArrowDataType *garrow_tensor_get_value_data_type(GArrowTensor *tensor);
 GArrowType      garrow_tensor_get_value_type     (GArrowTensor *tensor);
 GArrowBuffer   *garrow_tensor_get_buffer         (GArrowTensor *tensor);
diff --git a/c_glib/arrow-glib/uint8-tensor.cpp b/c_glib/arrow-glib/uint8-tensor.cpp
deleted file mode 100644
index 69f0f694530ad..0000000000000
--- a/c_glib/arrow-glib/uint8-tensor.cpp
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/buffer.hpp>
-#include <arrow-glib/uint8-tensor.h>
-#include <arrow-glib/numeric-tensor.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint8-tensor
- * @short_description: 8-bit unsigned integer tensor class
- *
- * #GArrowUint8Tensor is a class for 8-bit unsigned integer tensor. It
- * can store zero or more 8-bit integer data.
- */
-
-G_DEFINE_TYPE(GArrowUInt8Tensor,               \
-              garrow_uint8_tensor,             \
-              GARROW_TYPE_TENSOR)
-
-static void
-garrow_uint8_tensor_init(GArrowUInt8Tensor *object)
-{
-}
-
-static void
-garrow_uint8_tensor_class_init(GArrowUInt8TensorClass *klass)
-{
-}
-
-/**
- * garrow_uint8_tensor_new:
- * @data: A #GArrowBuffer that contains tensor data.
- * @shape: (array length=n_dimensions): A list of dimension sizes.
- * @n_dimensions: The number of dimensions.
- * @strides: (array length=n_strides) (nullable): A list of the number of
- *   bytes in each dimension.
- * @n_strides: The number of strides.
- * @dimention_names: (array length=n_dimention_names) (nullable): A list of
- *   dimension names.
- * @n_dimention_names: The number of dimension names
- *
- * Returns: The newly created #GArrowUInt8Tensor.
- *
- * Since: 0.3.0
- */
-GArrowUInt8Tensor *
-garrow_uint8_tensor_new(GArrowBuffer *data,
-                       gint64 *shape,
-                       gsize n_dimensions,
-                       gint64 *strides,
-                       gsize n_strides,
-                       gchar **dimension_names,
-                       gsize n_dimension_names)
-{
-  auto tensor =
-    garrow::numeric_tensor_new<arrow::UInt8Tensor>(data,
-                                                  shape,
-                                                  n_dimensions,
-                                                  strides,
-                                                  n_strides,
-                                                  dimension_names,
-                                                  n_dimension_names);
-  return GARROW_UINT8_TENSOR(tensor);
-}
-
-/**
- * garrow_uint8_tensor_get_raw_data:
- * @tensor: A #GArrowUInt8Tensor.
- * @n_data: (out): The number of data.
- *
- * Returns: (array length=n_data): The raw data in the tensor.
- *
- * Since: 0.3.0
- */
-const guint8 *
-garrow_uint8_tensor_get_raw_data(GArrowUInt8Tensor *tensor,
-                                 gint64 *n_data)
-{
-  return garrow::numeric_tensor_get_raw_data<arrow::UInt8Tensor, uint8_t>(GARROW_TENSOR(tensor),
-                                                                          n_data);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-tensor.h b/c_glib/arrow-glib/uint8-tensor.h
deleted file mode 100644
index 248c507b4f646..0000000000000
--- a/c_glib/arrow-glib/uint8-tensor.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/tensor.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT8_TENSOR                \
-  (garrow_uint8_tensor_get_type())
-#define GARROW_UINT8_TENSOR(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT8_TENSOR, \
-                              GArrowUInt8Tensor))
-#define GARROW_UINT8_TENSOR_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT8_TENSOR,    \
-                           GArrowUInt8TensorClass))
-#define GARROW_IS_UINT8_TENSOR(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT8_TENSOR))
-#define GARROW_IS_UINT8_TENSOR_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT8_TENSOR))
-#define GARROW_UINT8_TENSOR_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT8_TENSOR,  \
-                             GArrowUInt8TensorClass))
-
-typedef struct _GArrowUInt8Tensor         GArrowUInt8Tensor;
-typedef struct _GArrowUInt8TensorClass    GArrowUInt8TensorClass;
-
-/**
- * GArrowUInt8Tensor:
- *
- * It wraps `arrow::UInt8Tensor`.
- */
-struct _GArrowUInt8Tensor
-{
-  /*< private >*/
-  GArrowTensor parent_instance;
-};
-
-struct _GArrowUInt8TensorClass
-{
-  GArrowTensorClass parent_class;
-};
-
-GType garrow_uint8_tensor_get_type(void) G_GNUC_CONST;
-
-GArrowUInt8Tensor *garrow_uint8_tensor_new(GArrowBuffer *data,
-                                           gint64 *shape,
-                                           gsize n_dimensions,
-                                           gint64 *strides,
-                                           gsize n_strides,
-                                           gchar **dimention_names,
-                                           gsize n_dimention_names);
-
-const guint8 *garrow_uint8_tensor_get_raw_data(GArrowUInt8Tensor *tensor,
-                                               gint64 *n_data);
-
-G_END_DECLS
diff --git a/c_glib/test/test-int8-tensor.rb b/c_glib/test/test-int8-tensor.rb
deleted file mode 100644
index a96a4076b5a8e..0000000000000
--- a/c_glib/test/test-int8-tensor.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class TestInt8Tensor < Test::Unit::TestCase
-  include Helper::Omittable
-
-  def setup
-    @raw_data = [
-      1, 2,
-      3, 4,
-
-      5, 6,
-      7, 8,
-
-      9, 10,
-      11, 12,
-    ]
-    data = Arrow::Buffer.new(@raw_data.pack("c*"))
-    shape = [3, 2, 2]
-    strides = []
-    names = []
-    @tensor = Arrow::Int8Tensor.new(data, shape, strides, names)
-  end
-
-  def test_raw_data
-    require_gi(3, 1, 2)
-    assert_equal(@raw_data, @tensor.raw_data)
-  end
-end
diff --git a/c_glib/test/test-tensor.rb b/c_glib/test/test-tensor.rb
index 455b0d9d90acb..3e1f541cfd4b5 100644
--- a/c_glib/test/test-tensor.rb
+++ b/c_glib/test/test-tensor.rb
@@ -33,7 +33,11 @@ def setup
     @shape = [3, 2, 2]
     strides = []
     names = ["a", "b", "c"]
-    @tensor = Arrow::Int8Tensor.new(data, @shape, strides, names)
+    @tensor = Arrow::Tensor.new(Arrow::Int8DataType.new,
+                                data,
+                                @shape,
+                                strides,
+                                names)
   end
 
   def test_value_data_type
diff --git a/c_glib/test/test-uint8-tensor.rb b/c_glib/test/test-uint8-tensor.rb
deleted file mode 100644
index 0fe758ba676cc..0000000000000
--- a/c_glib/test/test-uint8-tensor.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class TestUInt8Tensor < Test::Unit::TestCase
-  include Helper::Omittable
-
-  def setup
-    @raw_data = [
-      1, 2,
-      3, 4,
-
-      5, 6,
-      7, 8,
-
-      9, 10,
-      11, 12,
-    ]
-    data = Arrow::Buffer.new(@raw_data.pack("c*"))
-    shape = [3, 2, 2]
-    strides = []
-    names = []
-    @tensor = Arrow::UInt8Tensor.new(data, shape, strides, names)
-  end
-
-  def test_raw_data
-    require_gi(3, 1, 2)
-    assert_equal(@raw_data, @tensor.raw_data)
-  end
-end
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index ccb299e53a11e..562d4e1b4ddff 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -580,31 +580,6 @@ Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_star
 // ----------------------------------------------------------------------
 // Implement TensorEquals
 
-class TensorEqualsVisitor {
- public:
-  explicit TensorEqualsVisitor(const Tensor& right) : right_(right) {}
-
-  template <typename TensorType>
-  Status Visit(const TensorType& left) {
-    const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
-    const int byte_width = size_meta.bit_width() / 8;
-    DCHECK_GT(byte_width, 0);
-
-    const uint8_t* left_data = left.data()->data();
-    const uint8_t* right_data = right_.data()->data();
-
-    result_ =
-        memcmp(left_data, right_data, static_cast<size_t>(byte_width * left.size())) == 0;
-    return Status::OK();
-  }
-
-  bool result() const { return result_; }
-
- protected:
-  const Tensor& right_;
-  bool result_;
-};
-
 Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
   // The arrays are the same object
   if (&left == &right) {
@@ -619,9 +594,15 @@ Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
           "Comparison not implemented for non-contiguous tensors");
     }
 
-    TensorEqualsVisitor visitor(right);
-    RETURN_NOT_OK(VisitTensorInline(left, &visitor));
-    *are_equal = visitor.result();
+    const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
+    const int byte_width = size_meta.bit_width() / 8;
+    DCHECK_GT(byte_width, 0);
+
+    const uint8_t* left_data = left.data()->data();
+    const uint8_t* right_data = right.data()->data();
+
+    *are_equal =
+        memcmp(left_data, right_data, static_cast<size_t>(byte_width * left.size())) == 0;
   }
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index cd793e08a26be..b4a88b5519b7e 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -704,8 +704,8 @@ TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
 
   auto data = test::GetBufferFromVector(values);
 
-  Int64Tensor t0(data, shape, strides, dim_names);
-  Int64Tensor tzero(data, {}, {}, {});
+  Tensor t0(int64(), data, shape, strides, dim_names);
+  Tensor tzero(int64(), data, {}, {}, {});
 
   CheckTensorRoundTrip(t0);
   CheckTensorRoundTrip(tzero);
@@ -724,7 +724,7 @@ TEST_F(TestTensorRoundTrip, NonContiguous) {
   test::randint<int64_t>(24, 0, 100, &values);
 
   auto data = test::GetBufferFromVector(values);
-  Int64Tensor tensor(data, {4, 3}, {48, 16});
+  Tensor tensor(int64(), data, {4, 3}, {48, 16});
 
   int32_t metadata_length;
   int64_t body_length;
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 69fde1783d7d3..aea4c9cd5ec1c 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -507,7 +507,8 @@ Status ReadTensor(
   std::vector<std::string> dim_names;
   RETURN_NOT_OK(
       GetTensorMetadata(message->header(), &type, &shape, &strides, &dim_names));
-  return MakeTensor(type, data, shape, strides, dim_names, out);
+  *out = std::make_shared<Tensor>(type, data, shape, strides, dim_names);
+  return Status::OK();
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 2c1a5910f06d5..c391b5d7a1018 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -223,7 +223,8 @@ Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>*
   std::shared_ptr<DataType> type;
   RETURN_NOT_OK(
       GetTensorType(reinterpret_cast<PyObject*>(PyArray_DESCR(ndarray)), &type));
-  return MakeTensor(type, data, shape, strides, {}, out);
+  *out = std::make_shared<Tensor>(type, data, shape, strides);
+  return Status::OK();
 }
 
 Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out) {
diff --git a/cpp/src/arrow/tensor-test.cc b/cpp/src/arrow/tensor-test.cc
index 336905c21ae81..c41683a3db5a2 100644
--- a/cpp/src/arrow/tensor-test.cc
+++ b/cpp/src/arrow/tensor-test.cc
@@ -39,7 +39,7 @@ TEST(TestTensor, ZeroDim) {
   std::shared_ptr<MutableBuffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer));
 
-  Int64Tensor t0(buffer, shape);
+  Tensor t0(int64(), buffer, shape);
 
   ASSERT_EQ(1, t0.size());
 }
@@ -55,9 +55,9 @@ TEST(TestTensor, BasicCtors) {
   std::shared_ptr<MutableBuffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer));
 
-  Int64Tensor t1(buffer, shape);
-  Int64Tensor t2(buffer, shape, strides);
-  Int64Tensor t3(buffer, shape, strides, dim_names);
+  Tensor t1(int64(), buffer, shape);
+  Tensor t2(int64(), buffer, shape, strides);
+  Tensor t3(int64(), buffer, shape, strides, dim_names);
 
   ASSERT_EQ(24, t1.size());
   ASSERT_TRUE(t1.is_mutable());
@@ -84,9 +84,9 @@ TEST(TestTensor, IsContiguous) {
   std::vector<int64_t> c_strides = {48, 8};
   std::vector<int64_t> f_strides = {8, 32};
   std::vector<int64_t> noncontig_strides = {8, 8};
-  Int64Tensor t1(buffer, shape, c_strides);
-  Int64Tensor t2(buffer, shape, f_strides);
-  Int64Tensor t3(buffer, shape, noncontig_strides);
+  Tensor t1(int64(), buffer, shape, c_strides);
+  Tensor t2(int64(), buffer, shape, f_strides);
+  Tensor t3(int64(), buffer, shape, noncontig_strides);
 
   ASSERT_TRUE(t1.is_contiguous());
   ASSERT_TRUE(t2.is_contiguous());
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index fa3e203c998ba..909b05ebe8f80 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -118,69 +118,4 @@ bool Tensor::Equals(const Tensor& other) const {
   return are_equal;
 }
 
-template <typename T>
-NumericTensor<T>::NumericTensor(const std::shared_ptr<Buffer>& data,
-    const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
-    const std::vector<std::string>& dim_names)
-    : Tensor(TypeTraits<T>::type_singleton(), data, shape, strides, dim_names),
-      raw_data_(nullptr),
-      mutable_raw_data_(nullptr) {
-  if (data_) {
-    raw_data_ = reinterpret_cast<const value_type*>(data_->data());
-    if (data_->is_mutable()) {
-      auto mut_buf = static_cast<MutableBuffer*>(data_.get());
-      mutable_raw_data_ = reinterpret_cast<value_type*>(mut_buf->mutable_data());
-    }
-  }
-}
-
-template <typename T>
-NumericTensor<T>::NumericTensor(
-    const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape)
-    : NumericTensor(data, shape, {}, {}) {}
-
-template <typename T>
-NumericTensor<T>::NumericTensor(const std::shared_ptr<Buffer>& data,
-    const std::vector<int64_t>& shape, const std::vector<int64_t>& strides)
-    : NumericTensor(data, shape, strides, {}) {}
-
-template class ARROW_TEMPLATE_EXPORT NumericTensor<Int8Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt8Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<Int16Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt16Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<Int32Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt32Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<Int64Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt64Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<HalfFloatType>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<FloatType>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<DoubleType>;
-
-#define TENSOR_CASE(TYPE, TENSOR_TYPE)                                        \
-  case Type::TYPE:                                                            \
-    *tensor = std::make_shared<TENSOR_TYPE>(data, shape, strides, dim_names); \
-    break;
-
-Status ARROW_EXPORT MakeTensor(const std::shared_ptr<DataType>& type,
-    const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
-    const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names,
-    std::shared_ptr<Tensor>* tensor) {
-  switch (type->id()) {
-    TENSOR_CASE(INT8, Int8Tensor);
-    TENSOR_CASE(INT16, Int16Tensor);
-    TENSOR_CASE(INT32, Int32Tensor);
-    TENSOR_CASE(INT64, Int64Tensor);
-    TENSOR_CASE(UINT8, UInt8Tensor);
-    TENSOR_CASE(UINT16, UInt16Tensor);
-    TENSOR_CASE(UINT32, UInt32Tensor);
-    TENSOR_CASE(UINT64, UInt64Tensor);
-    TENSOR_CASE(HALF_FLOAT, HalfFloatTensor);
-    TENSOR_CASE(FLOAT, FloatTensor);
-    TENSOR_CASE(DOUBLE, DoubleTensor);
-    default:
-      return Status::NotImplemented(type->ToString());
-  }
-  return Status::OK();
-}
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index 7741c305f870d..371f5911a4396 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -76,6 +76,9 @@ class ARROW_EXPORT Tensor {
   std::shared_ptr<DataType> type() const { return type_; }
   std::shared_ptr<Buffer> data() const { return data_; }
 
+  const uint8_t* raw_data() const { return data_->data(); }
+  uint8_t* raw_data() { return data_->mutable_data(); }
+
   const std::vector<int64_t>& shape() const { return shape_; }
   const std::vector<int64_t>& strides() const { return strides_; }
 
@@ -117,50 +120,6 @@ class ARROW_EXPORT Tensor {
   DISALLOW_COPY_AND_ASSIGN(Tensor);
 };
 
-template <typename T>
-class ARROW_EXPORT NumericTensor : public Tensor {
- public:
-  using value_type = typename T::c_type;
-
-  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape);
-
-  /// Constructor with non-negative strides
-  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
-      const std::vector<int64_t>& strides);
-
-  /// Constructor with strides and dimension names
-  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
-      const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names);
-
-  const value_type* raw_data() const { return raw_data_; }
-  value_type* raw_data() { return mutable_raw_data_; }
-
- private:
-  const value_type* raw_data_;
-  value_type* mutable_raw_data_;
-};
-
-Status ARROW_EXPORT MakeTensor(const std::shared_ptr<DataType>& type,
-    const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
-    const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names,
-    std::shared_ptr<Tensor>* tensor);
-
-// ----------------------------------------------------------------------
-// extern templates and other details
-
-// Only instantiate these templates once
-ARROW_EXTERN_TEMPLATE NumericTensor<Int8Type>;
-ARROW_EXTERN_TEMPLATE NumericTensor<UInt8Type>;
-ARROW_EXTERN_TEMPLATE NumericTensor<Int16Type>;
-ARROW_EXTERN_TEMPLATE NumericTensor<UInt16Type>;
-ARROW_EXTERN_TEMPLATE NumericTensor<Int32Type>;
-ARROW_EXTERN_TEMPLATE NumericTensor<UInt32Type>;
-ARROW_EXTERN_TEMPLATE NumericTensor<Int64Type>;
-ARROW_EXTERN_TEMPLATE NumericTensor<UInt64Type>;
-ARROW_EXTERN_TEMPLATE NumericTensor<HalfFloatType>;
-ARROW_EXTERN_TEMPLATE NumericTensor<FloatType>;
-ARROW_EXTERN_TEMPLATE NumericTensor<DoubleType>;
-
 }  // namespace arrow
 
 #endif  // ARROW_TENSOR_H
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index bc5f493fa1f9a..7478950b894c5 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -104,31 +104,6 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
   return Status::NotImplemented("Type not implemented");
 }
 
-#define TENSOR_VISIT_INLINE(TYPE_CLASS) \
-  case TYPE_CLASS::type_id:             \
-    return visitor->Visit(              \
-        static_cast<const typename TypeTraits<TYPE_CLASS>::TensorType&>(array));
-
-template <typename VISITOR>
-inline Status VisitTensorInline(const Tensor& array, VISITOR* visitor) {
-  switch (array.type_id()) {
-    TENSOR_VISIT_INLINE(Int8Type);
-    TENSOR_VISIT_INLINE(UInt8Type);
-    TENSOR_VISIT_INLINE(Int16Type);
-    TENSOR_VISIT_INLINE(UInt16Type);
-    TENSOR_VISIT_INLINE(Int32Type);
-    TENSOR_VISIT_INLINE(UInt32Type);
-    TENSOR_VISIT_INLINE(Int64Type);
-    TENSOR_VISIT_INLINE(UInt64Type);
-    TENSOR_VISIT_INLINE(HalfFloatType);
-    TENSOR_VISIT_INLINE(FloatType);
-    TENSOR_VISIT_INLINE(DoubleType);
-    default:
-      break;
-  }
-  return Status::NotImplemented("Type not implemented");
-}
-
 }  // namespace arrow
 
 #endif  // ARROW_VISITOR_INLINE_H

From 3f9b26c0edc84fb0d5c121937f966553bb12c0bf Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 20 Apr 2017 10:06:15 -0400
Subject: [PATCH 0543/1644] ARROW-863: [GLib] Use GBytes to implement zero-copy

Author: Kouhei Sutou <kou@clear-code.com>

Closes #572 from kou/glib-buffer-use-gbytes and squashes the following commits:

dc37de3 [Kouhei Sutou] [GLib] Use GBytes to implement zero-copy
---
 c_glib/arrow-glib/buffer.cpp | 13 +++++++------
 c_glib/arrow-glib/buffer.h   |  3 +--
 c_glib/test/test-buffer.rb   |  6 +++---
 c_glib/test/test-tensor.rb   |  2 +-
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/c_glib/arrow-glib/buffer.cpp b/c_glib/arrow-glib/buffer.cpp
index 0ec52df0aee67..9853e896b3dcc 100644
--- a/c_glib/arrow-glib/buffer.cpp
+++ b/c_glib/arrow-glib/buffer.cpp
@@ -167,18 +167,19 @@ garrow_buffer_get_capacity(GArrowBuffer *buffer)
 /**
  * garrow_buffer_get_data:
  * @buffer: A #GArrowBuffer.
- * @size: (out): The number of bytes of the data.
  *
- * Returns: (array length=size): The data of the buffer.
+ * Returns: (transfer full): The data of the buffer. The data is owned by
+ *   the buffer. You should not free or modify the data.
  *
  * Since: 0.3.0
  */
-const guint8 *
-garrow_buffer_get_data(GArrowBuffer *buffer, gint64 *size)
+GBytes *
+garrow_buffer_get_data(GArrowBuffer *buffer)
 {
   auto arrow_buffer = garrow_buffer_get_raw(buffer);
-  *size = arrow_buffer->size();
-  return arrow_buffer->data();
+  auto data = g_bytes_new_static(arrow_buffer->data(),
+                                 arrow_buffer->size());
+  return data;
 }
 
 /**
diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h
index 1e7d55182fd1d..83e1d0d66bf28 100644
--- a/c_glib/arrow-glib/buffer.h
+++ b/c_glib/arrow-glib/buffer.h
@@ -61,8 +61,7 @@ GArrowBuffer  *garrow_buffer_new          (const guint8 *data,
                                            gint64 size);
 gboolean       garrow_buffer_is_mutable   (GArrowBuffer *buffer);
 gint64         garrow_buffer_get_capacity (GArrowBuffer *buffer);
-const guint8  *garrow_buffer_get_data     (GArrowBuffer *buffer,
-                                           gint64 *size);
+GBytes        *garrow_buffer_get_data     (GArrowBuffer *buffer);
 gint64         garrow_buffer_get_size     (GArrowBuffer *buffer);
 GArrowBuffer  *garrow_buffer_get_parent   (GArrowBuffer *buffer);
 
diff --git a/c_glib/test/test-buffer.rb b/c_glib/test/test-buffer.rb
index 1ea26f24ce873..6bb96714c8283 100644
--- a/c_glib/test/test-buffer.rb
+++ b/c_glib/test/test-buffer.rb
@@ -32,7 +32,7 @@ def test_capacity
   end
 
   def test_data
-    assert_equal(@data, @buffer.data.pack("C*"))
+    assert_equal(@data, @buffer.data.to_s)
   end
 
   def test_size
@@ -45,11 +45,11 @@ def test_parent
 
   def test_copy
     copied_buffer = @buffer.copy(1, 3)
-    assert_equal(@data[1, 3], copied_buffer.data.pack("C*"))
+    assert_equal(@data[1, 3], copied_buffer.data.to_s)
   end
 
   def test_slice
     sliced_buffer = @buffer.slice(1, 3)
-    assert_equal(@data[1, 3], sliced_buffer.data.pack("C*"))
+    assert_equal(@data[1, 3], sliced_buffer.data.to_s)
   end
 end
diff --git a/c_glib/test/test-tensor.rb b/c_glib/test/test-tensor.rb
index 3e1f541cfd4b5..225857b52da98 100644
--- a/c_glib/test/test-tensor.rb
+++ b/c_glib/test/test-tensor.rb
@@ -49,7 +49,7 @@ def test_value_type
   end
 
   def test_buffer
-    assert_equal(@raw_data, @tensor.buffer.data)
+    assert_equal(@raw_data, @tensor.buffer.data.to_s.unpack("c*"))
   end
 
   def test_shape

From 7c1fef51ca1add0af53dcdb43590f367974607c2 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 20 Apr 2017 10:08:56 -0400
Subject: [PATCH 0544/1644] ARROW-864: [GLib] Unify Array files

Author: Kouhei Sutou <kou@clear-code.com>

Closes #573 from kou/glib-unify-array-file and squashes the following commits:

4c47afb [Kouhei Sutou] [GLib] Unify Array files
---
 c_glib/arrow-glib/Makefile.am             |  32 -
 c_glib/arrow-glib/array.cpp               | 624 +++++++++++++++++-
 c_glib/arrow-glib/array.h                 | 735 ++++++++++++++++++++++
 c_glib/arrow-glib/arrow-glib.h            |  16 -
 c_glib/arrow-glib/binary-array.cpp        |  73 ---
 c_glib/arrow-glib/binary-array.h          |  72 ---
 c_glib/arrow-glib/boolean-array.cpp       |  69 --
 c_glib/arrow-glib/boolean-array.h         |  70 ---
 c_glib/arrow-glib/double-array.cpp        |  69 --
 c_glib/arrow-glib/double-array.h          |  71 ---
 c_glib/arrow-glib/float-array.cpp         |  69 --
 c_glib/arrow-glib/float-array.h           |  71 ---
 c_glib/arrow-glib/int16-array.cpp         |  69 --
 c_glib/arrow-glib/int16-array.h           |  71 ---
 c_glib/arrow-glib/int32-array.cpp         |  69 --
 c_glib/arrow-glib/int32-array.h           |  71 ---
 c_glib/arrow-glib/int64-array.cpp         |  69 --
 c_glib/arrow-glib/int64-array.h           |  71 ---
 c_glib/arrow-glib/int8-array.cpp          |  69 --
 c_glib/arrow-glib/int8-array.h            |  71 ---
 c_glib/arrow-glib/list-array.cpp          |  92 ---
 c_glib/arrow-glib/list-array.h            |  73 ---
 c_glib/arrow-glib/null-array.cpp          |  69 --
 c_glib/arrow-glib/null-array.h            |  70 ---
 c_glib/arrow-glib/string-array.cpp        |  74 ---
 c_glib/arrow-glib/string-array.h          |  71 ---
 c_glib/arrow-glib/struct-array.cpp        |  97 ---
 c_glib/arrow-glib/struct-array.h          |  73 ---
 c_glib/arrow-glib/uint16-array.cpp        |  69 --
 c_glib/arrow-glib/uint16-array.h          |  71 ---
 c_glib/arrow-glib/uint32-array.cpp        |  69 --
 c_glib/arrow-glib/uint32-array.h          |  71 ---
 c_glib/arrow-glib/uint64-array.cpp        |  69 --
 c_glib/arrow-glib/uint64-array.h          |  71 ---
 c_glib/arrow-glib/uint8-array.cpp         |  69 --
 c_glib/arrow-glib/uint8-array.h           |  71 ---
 c_glib/doc/reference/arrow-glib-docs.sgml |  16 -
 37 files changed, 1340 insertions(+), 2386 deletions(-)
 delete mode 100644 c_glib/arrow-glib/binary-array.cpp
 delete mode 100644 c_glib/arrow-glib/binary-array.h
 delete mode 100644 c_glib/arrow-glib/boolean-array.cpp
 delete mode 100644 c_glib/arrow-glib/boolean-array.h
 delete mode 100644 c_glib/arrow-glib/double-array.cpp
 delete mode 100644 c_glib/arrow-glib/double-array.h
 delete mode 100644 c_glib/arrow-glib/float-array.cpp
 delete mode 100644 c_glib/arrow-glib/float-array.h
 delete mode 100644 c_glib/arrow-glib/int16-array.cpp
 delete mode 100644 c_glib/arrow-glib/int16-array.h
 delete mode 100644 c_glib/arrow-glib/int32-array.cpp
 delete mode 100644 c_glib/arrow-glib/int32-array.h
 delete mode 100644 c_glib/arrow-glib/int64-array.cpp
 delete mode 100644 c_glib/arrow-glib/int64-array.h
 delete mode 100644 c_glib/arrow-glib/int8-array.cpp
 delete mode 100644 c_glib/arrow-glib/int8-array.h
 delete mode 100644 c_glib/arrow-glib/list-array.cpp
 delete mode 100644 c_glib/arrow-glib/list-array.h
 delete mode 100644 c_glib/arrow-glib/null-array.cpp
 delete mode 100644 c_glib/arrow-glib/null-array.h
 delete mode 100644 c_glib/arrow-glib/string-array.cpp
 delete mode 100644 c_glib/arrow-glib/string-array.h
 delete mode 100644 c_glib/arrow-glib/struct-array.cpp
 delete mode 100644 c_glib/arrow-glib/struct-array.h
 delete mode 100644 c_glib/arrow-glib/uint16-array.cpp
 delete mode 100644 c_glib/arrow-glib/uint16-array.h
 delete mode 100644 c_glib/arrow-glib/uint32-array.cpp
 delete mode 100644 c_glib/arrow-glib/uint32-array.h
 delete mode 100644 c_glib/arrow-glib/uint64-array.cpp
 delete mode 100644 c_glib/arrow-glib/uint64-array.h
 delete mode 100644 c_glib/arrow-glib/uint8-array.cpp
 delete mode 100644 c_glib/arrow-glib/uint8-array.h

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 11b6508df0745..570a033f4512c 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -44,62 +44,46 @@ libarrow_glib_la_headers =			\
 	array.h					\
 	array-builder.h				\
 	arrow-glib.h				\
-	binary-array.h				\
 	binary-array-builder.h			\
 	binary-data-type.h			\
-	boolean-array.h				\
 	boolean-array-builder.h			\
 	boolean-data-type.h			\
 	buffer.h				\
 	chunked-array.h				\
 	column.h				\
 	data-type.h				\
-	double-array.h				\
 	double-array-builder.h			\
 	double-data-type.h			\
 	error.h					\
 	field.h					\
-	float-array.h				\
 	float-array-builder.h			\
 	float-data-type.h			\
-	int8-array.h				\
 	int8-array-builder.h			\
 	int8-data-type.h			\
-	int16-array.h				\
 	int16-array-builder.h			\
 	int16-data-type.h			\
-	int32-array.h				\
 	int32-array-builder.h			\
 	int32-data-type.h			\
-	int64-array.h				\
 	int64-array-builder.h			\
 	int64-data-type.h			\
-	list-array.h				\
 	list-array-builder.h			\
 	list-data-type.h			\
-	null-array.h				\
 	null-data-type.h			\
 	record-batch.h				\
 	schema.h				\
-	string-array.h				\
 	string-array-builder.h			\
 	string-data-type.h			\
-	struct-array.h				\
 	struct-array-builder.h			\
 	struct-data-type.h			\
 	table.h					\
 	tensor.h				\
 	type.h					\
-	uint8-array.h				\
 	uint8-array-builder.h			\
 	uint8-data-type.h			\
-	uint16-array.h				\
 	uint16-array-builder.h			\
 	uint16-data-type.h			\
-	uint32-array.h				\
 	uint32-array-builder.h			\
 	uint32-data-type.h			\
-	uint64-array.h				\
 	uint64-array-builder.h			\
 	uint64-data-type.h
 
@@ -132,62 +116,46 @@ libarrow_glib_la_generated_sources =		\
 libarrow_glib_la_sources =			\
 	array.cpp				\
 	array-builder.cpp			\
-	binary-array.cpp			\
 	binary-array-builder.cpp		\
 	binary-data-type.cpp			\
-	boolean-array.cpp			\
 	boolean-array-builder.cpp		\
 	boolean-data-type.cpp			\
 	buffer.cpp				\
 	chunked-array.cpp			\
 	column.cpp				\
 	data-type.cpp				\
-	double-array.cpp			\
 	double-array-builder.cpp		\
 	double-data-type.cpp			\
 	error.cpp				\
 	field.cpp				\
-	float-array.cpp				\
 	float-array-builder.cpp			\
 	float-data-type.cpp			\
-	int8-array.cpp				\
 	int8-array-builder.cpp			\
 	int8-data-type.cpp			\
-	int16-array.cpp				\
 	int16-array-builder.cpp			\
 	int16-data-type.cpp			\
-	int32-array.cpp				\
 	int32-array-builder.cpp			\
 	int32-data-type.cpp			\
-	int64-array.cpp				\
 	int64-array-builder.cpp			\
 	int64-data-type.cpp			\
-	list-array.cpp				\
 	list-array-builder.cpp			\
 	list-data-type.cpp			\
-	null-array.cpp				\
 	null-data-type.cpp			\
 	record-batch.cpp			\
 	schema.cpp				\
-	string-array.cpp			\
 	string-array-builder.cpp		\
 	string-data-type.cpp			\
-	struct-array.cpp			\
 	struct-array-builder.cpp		\
 	struct-data-type.cpp			\
 	table.cpp				\
 	tensor.cpp				\
 	type.cpp				\
-	uint8-array.cpp				\
 	uint8-array-builder.cpp			\
 	uint8-data-type.cpp			\
-	uint16-array.cpp			\
 	uint16-array-builder.cpp		\
 	uint16-data-type.cpp			\
-	uint32-array.cpp			\
 	uint32-array-builder.cpp		\
 	uint32-data-type.cpp			\
-	uint64-array.cpp			\
 	uint64-array-builder.cpp		\
 	uint64-data-type.cpp			\
 	$(libarrow_glib_la_headers)		\
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 3bd7b40ff9493..c86bff90d40d6 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -22,24 +22,8 @@
 #endif
 
 #include <arrow-glib/array.hpp>
-#include <arrow-glib/binary-array.h>
-#include <arrow-glib/boolean-array.h>
 #include <arrow-glib/data-type.hpp>
-#include <arrow-glib/double-array.h>
-#include <arrow-glib/float-array.h>
-#include <arrow-glib/int8-array.h>
-#include <arrow-glib/int16-array.h>
-#include <arrow-glib/int32-array.h>
-#include <arrow-glib/int64-array.h>
-#include <arrow-glib/list-array.h>
-#include <arrow-glib/null-array.h>
-#include <arrow-glib/string-array.h>
-#include <arrow-glib/struct-array.h>
 #include <arrow-glib/type.hpp>
-#include <arrow-glib/uint8-array.h>
-#include <arrow-glib/uint16-array.h>
-#include <arrow-glib/uint32-array.h>
-#include <arrow-glib/uint64-array.h>
 
 #include <iostream>
 
@@ -47,13 +31,80 @@ G_BEGIN_DECLS
 
 /**
  * SECTION: array
- * @short_description: Base class for all array classes
+ * @section_id: array-classes
+ * @title: Array classes
+ * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowArray is a base class for all array classes such as
  * #GArrowBooleanArray.
  *
- * Array is immutable. You need to use array builder class such as
- * #GArrowBooleanArrayBuilder to create a new array.
+ * All array classes are immutable. You need to use array builder
+ * class such as #GArrowBooleanArrayBuilder to create a new array
+ * except #GArrowNullArray.
+ *
+ * #GArrowNullArray is a class for null array. It can store zero or
+ * more null values. You need to specify an array length to create a
+ * new array.
+ *
+ * #GArrowBooleanArray is a class for binary array. It can store zero
+ * or more boolean data. You need to use #GArrowBooleanArrayBuilder to
+ * create a new array.
+ *
+ * #GArrowInt8Array is a class for 8-bit integer array. It can store
+ * zero or more 8-bit integer data. You need to use
+ * #GArrowInt8ArrayBuilder to create a new array.
+ *
+ * #GArrowUInt8Array is a class for 8-bit unsigned integer array. It
+ * can store zero or more 8-bit unsigned integer data. You need to use
+ * #GArrowUInt8ArrayBuilder to create a new array.
+ *
+ * #GArrowInt16Array is a class for 16-bit integer array. It can store
+ * zero or more 16-bit integer data. You need to use
+ * #GArrowInt16ArrayBuilder to create a new array.
+ *
+ * #GArrowUInt16Array is a class for 16-bit unsigned integer array. It
+ * can store zero or more 16-bit unsigned integer data. You need to use
+ * #GArrowUInt16ArrayBuilder to create a new array.
+ *
+ * #GArrowInt32Array is a class for 32-bit integer array. It can store
+ * zero or more 32-bit integer data. You need to use
+ * #GArrowInt32ArrayBuilder to create a new array.
+ *
+ * #GArrowUInt32Array is a class for 32-bit unsigned integer array. It
+ * can store zero or more 32-bit unsigned integer data. You need to use
+ * #GArrowUInt32ArrayBuilder to create a new array.
+ *
+ * #GArrowInt64Array is a class for 64-bit integer array. It can store
+ * zero or more 64-bit integer data. You need to use
+ * #GArrowInt64ArrayBuilder to create a new array.
+ *
+ * #GArrowUInt64Array is a class for 64-bit unsigned integer array. It
+ * can store zero or more 64-bit unsigned integer data. You need to
+ * use #GArrowUInt64ArrayBuilder to create a new array.
+ *
+ * #GArrowFloatArray is a class for 32-bit floating point array. It
+ * can store zero or more 32-bit floating data. You need to use
+ * #GArrowFloatArrayBuilder to create a new array.
+ *
+ * #GArrowDoubleArray is a class for 64-bit floating point array. It
+ * can store zero or more 64-bit floating data. You need to use
+ * #GArrowDoubleArrayBuilder to create a new array.
+ *
+ * #GArrowBinaryArray is a class for binary array. It can store zero
+ * or more binary data. You need to use #GArrowBinaryArrayBuilder to
+ * create a new array.
+ *
+ * #GArrowStringArray is a class for UTF-8 encoded string array. It
+ * can store zero or more UTF-8 encoded string data. You need to use
+ * #GArrowStringArrayBuilder to create a new array.
+ *
+ * #GArrowListArray is a class for list array. It can store zero or
+ * more list data. You need to use #GArrowListArrayBuilder to create a
+ * new array.
+ *
+ * #GArrowStructArray is a class for struct array. It can store zero
+ * or more structs. One struct has zero or more fields. You need to
+ * use #GArrowStructArrayBuilder to create a new array.
  */
 
 typedef struct GArrowArrayPrivate_ {
@@ -243,6 +294,541 @@ garrow_array_slice(GArrowArray *array,
   return garrow_array_new_raw(&arrow_sub_array);
 }
 
+
+G_DEFINE_TYPE(GArrowNullArray,               \
+              garrow_null_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_null_array_init(GArrowNullArray *object)
+{
+}
+
+static void
+garrow_null_array_class_init(GArrowNullArrayClass *klass)
+{
+}
+
+/**
+ * garrow_null_array_new:
+ * @length: An array length.
+ *
+ * Returns: A newly created #GArrowNullArray.
+ */
+GArrowNullArray *
+garrow_null_array_new(gint64 length)
+{
+  auto arrow_null_array = std::make_shared<arrow::NullArray>(length);
+  std::shared_ptr<arrow::Array> arrow_array = arrow_null_array;
+  auto array = garrow_array_new_raw(&arrow_array);
+  return GARROW_NULL_ARRAY(array);
+}
+
+
+G_DEFINE_TYPE(GArrowBooleanArray,               \
+              garrow_boolean_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_boolean_array_init(GArrowBooleanArray *object)
+{
+}
+
+static void
+garrow_boolean_array_class_init(GArrowBooleanArrayClass *klass)
+{
+}
+
+/**
+ * garrow_boolean_array_get_value:
+ * @array: A #GArrowBooleanArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gboolean
+garrow_boolean_array_get_value(GArrowBooleanArray *array,
+                               gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::BooleanArray *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowInt8Array,               \
+              garrow_int8_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_int8_array_init(GArrowInt8Array *object)
+{
+}
+
+static void
+garrow_int8_array_class_init(GArrowInt8ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_int8_array_get_value:
+ * @array: A #GArrowInt8Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gint8
+garrow_int8_array_get_value(GArrowInt8Array *array,
+                            gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Int8Array *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowUInt8Array,               \
+              garrow_uint8_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_uint8_array_init(GArrowUInt8Array *object)
+{
+}
+
+static void
+garrow_uint8_array_class_init(GArrowUInt8ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_uint8_array_get_value:
+ * @array: A #GArrowUInt8Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+guint8
+garrow_uint8_array_get_value(GArrowUInt8Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::UInt8Array *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowInt16Array,               \
+              garrow_int16_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_int16_array_init(GArrowInt16Array *object)
+{
+}
+
+static void
+garrow_int16_array_class_init(GArrowInt16ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_int16_array_get_value:
+ * @array: A #GArrowInt16Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gint16
+garrow_int16_array_get_value(GArrowInt16Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Int16Array *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowUInt16Array,               \
+              garrow_uint16_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_uint16_array_init(GArrowUInt16Array *object)
+{
+}
+
+static void
+garrow_uint16_array_class_init(GArrowUInt16ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_uint16_array_get_value:
+ * @array: A #GArrowUInt16Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+guint16
+garrow_uint16_array_get_value(GArrowUInt16Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::UInt16Array *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowInt32Array,               \
+              garrow_int32_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_int32_array_init(GArrowInt32Array *object)
+{
+}
+
+static void
+garrow_int32_array_class_init(GArrowInt32ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_int32_array_get_value:
+ * @array: A #GArrowInt32Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gint32
+garrow_int32_array_get_value(GArrowInt32Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Int32Array *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowUInt32Array,               \
+              garrow_uint32_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_uint32_array_init(GArrowUInt32Array *object)
+{
+}
+
+static void
+garrow_uint32_array_class_init(GArrowUInt32ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_uint32_array_get_value:
+ * @array: A #GArrowUInt32Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+guint32
+garrow_uint32_array_get_value(GArrowUInt32Array *array,
+                              gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::UInt32Array *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowInt64Array,               \
+              garrow_int64_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_int64_array_init(GArrowInt64Array *object)
+{
+}
+
+static void
+garrow_int64_array_class_init(GArrowInt64ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_int64_array_get_value:
+ * @array: A #GArrowInt64Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gint64
+garrow_int64_array_get_value(GArrowInt64Array *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Int64Array *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowUInt64Array,               \
+              garrow_uint64_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_uint64_array_init(GArrowUInt64Array *object)
+{
+}
+
+static void
+garrow_uint64_array_class_init(GArrowUInt64ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_uint64_array_get_value:
+ * @array: A #GArrowUInt64Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+guint64
+garrow_uint64_array_get_value(GArrowUInt64Array *array,
+                              gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::UInt64Array *>(arrow_array.get())->Value(i);
+}
+
+G_DEFINE_TYPE(GArrowFloatArray,               \
+              garrow_float_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_float_array_init(GArrowFloatArray *object)
+{
+}
+
+static void
+garrow_float_array_class_init(GArrowFloatArrayClass *klass)
+{
+}
+
+/**
+ * garrow_float_array_get_value:
+ * @array: A #GArrowFloatArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gfloat
+garrow_float_array_get_value(GArrowFloatArray *array,
+                             gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::FloatArray *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowDoubleArray,               \
+              garrow_double_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_double_array_init(GArrowDoubleArray *object)
+{
+}
+
+static void
+garrow_double_array_class_init(GArrowDoubleArrayClass *klass)
+{
+}
+
+/**
+ * garrow_double_array_get_value:
+ * @array: A #GArrowDoubleArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ */
+gdouble
+garrow_double_array_get_value(GArrowDoubleArray *array,
+                              gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::DoubleArray *>(arrow_array.get())->Value(i);
+}
+
+
+G_DEFINE_TYPE(GArrowBinaryArray,               \
+              garrow_binary_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_binary_array_init(GArrowBinaryArray *object)
+{
+}
+
+static void
+garrow_binary_array_class_init(GArrowBinaryArrayClass *klass)
+{
+}
+
+/**
+ * garrow_binary_array_get_value:
+ * @array: A #GArrowBinaryArray.
+ * @i: The index of the target value.
+ * @length: (out): The length of the value.
+ *
+ * Returns: (array length=length): The i-th value.
+ */
+const guint8 *
+garrow_binary_array_get_value(GArrowBinaryArray *array,
+                              gint64 i,
+                              gint32 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_binary_array =
+    static_cast<arrow::BinaryArray *>(arrow_array.get());
+  return arrow_binary_array->GetValue(i, length);
+}
+
+
+G_DEFINE_TYPE(GArrowStringArray,               \
+              garrow_string_array,             \
+              GARROW_TYPE_BINARY_ARRAY)
+
+static void
+garrow_string_array_init(GArrowStringArray *object)
+{
+}
+
+static void
+garrow_string_array_class_init(GArrowStringArrayClass *klass)
+{
+}
+
+/**
+ * garrow_string_array_get_string:
+ * @array: A #GArrowStringArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th UTF-8 encoded string.
+ */
+gchar *
+garrow_string_array_get_string(GArrowStringArray *array,
+                               gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_string_array =
+    static_cast<arrow::StringArray *>(arrow_array.get());
+  gint32 length;
+  auto value =
+    reinterpret_cast<const gchar *>(arrow_string_array->GetValue(i, &length));
+  return g_strndup(value, length);
+}
+
+
+G_DEFINE_TYPE(GArrowListArray,               \
+              garrow_list_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_list_array_init(GArrowListArray *object)
+{
+}
+
+static void
+garrow_list_array_class_init(GArrowListArrayClass *klass)
+{
+}
+
+/**
+ * garrow_list_array_get_value_type:
+ * @array: A #GArrowListArray.
+ *
+ * Returns: (transfer full): The data type of value in each list.
+ */
+GArrowDataType *
+garrow_list_array_get_value_type(GArrowListArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_list_array =
+    static_cast<arrow::ListArray *>(arrow_array.get());
+  auto arrow_value_type = arrow_list_array->value_type();
+  return garrow_data_type_new_raw(&arrow_value_type);
+}
+
+/**
+ * garrow_list_array_get_value:
+ * @array: A #GArrowListArray.
+ * @i: The index of the target value.
+ *
+ * Returns: (transfer full): The i-th list.
+ */
+GArrowArray *
+garrow_list_array_get_value(GArrowListArray *array,
+                            gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_list_array =
+    static_cast<arrow::ListArray *>(arrow_array.get());
+  auto arrow_list =
+    arrow_list_array->values()->Slice(arrow_list_array->value_offset(i),
+                                      arrow_list_array->value_length(i));
+  return garrow_array_new_raw(&arrow_list);
+}
+
+
+G_DEFINE_TYPE(GArrowStructArray,               \
+              garrow_struct_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_struct_array_init(GArrowStructArray *object)
+{
+}
+
+static void
+garrow_struct_array_class_init(GArrowStructArrayClass *klass)
+{
+}
+
+/**
+ * garrow_struct_array_get_field
+ * @array: A #GArrowStructArray.
+ * @i: The index of the field in the struct.
+ *
+ * Returns: (transfer full): The i-th field.
+ */
+GArrowArray *
+garrow_struct_array_get_field(GArrowStructArray *array,
+                              gint i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_struct_array =
+    static_cast<arrow::StructArray *>(arrow_array.get());
+  auto arrow_field = arrow_struct_array->field(i);
+  return garrow_array_new_raw(&arrow_field);
+}
+
+/**
+ * garrow_struct_array_get_fields
+ * @array: A #GArrowStructArray.
+ *
+ * Returns: (element-type GArrowArray) (transfer full):
+ *   The fields in the struct.
+ */
+GList *
+garrow_struct_array_get_fields(GArrowStructArray *array)
+{
+  const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  const auto arrow_struct_array =
+    static_cast<const arrow::StructArray *>(arrow_array.get());
+
+  GList *fields = NULL;
+  for (auto arrow_field : arrow_struct_array->fields()) {
+    GArrowArray *field = garrow_array_new_raw(&arrow_field);
+    fields = g_list_prepend(fields, field);
+  }
+
+  return g_list_reverse(fields);
+}
+
 G_END_DECLS
 
 GArrowArray *
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 06a37e9b43ad6..b417cdbab3631 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -68,4 +68,739 @@ GArrowArray   *garrow_array_slice       (GArrowArray *array,
                                          gint64 offset,
                                          gint64 length);
 
+#define GARROW_TYPE_NULL_ARRAY                  \
+  (garrow_null_array_get_type())
+#define GARROW_NULL_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_NULL_ARRAY,   \
+                              GArrowNullArray))
+#define GARROW_NULL_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_NULL_ARRAY,      \
+                           GArrowNullArrayClass))
+#define GARROW_IS_NULL_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_NULL_ARRAY))
+#define GARROW_IS_NULL_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_NULL_ARRAY))
+#define GARROW_NULL_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_NULL_ARRAY,    \
+                             GArrowNullArrayClass))
+
+typedef struct _GArrowNullArray         GArrowNullArray;
+typedef struct _GArrowNullArrayClass    GArrowNullArrayClass;
+
+/**
+ * GArrowNullArray:
+ *
+ * It wraps `arrow::NullArray`.
+ */
+struct _GArrowNullArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowNullArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_null_array_get_type(void) G_GNUC_CONST;
+
+GArrowNullArray *garrow_null_array_new(gint64 length);
+
+
+#define GARROW_TYPE_BOOLEAN_ARRAY               \
+  (garrow_boolean_array_get_type())
+#define GARROW_BOOLEAN_ARRAY(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BOOLEAN_ARRAY,        \
+                              GArrowBooleanArray))
+#define GARROW_BOOLEAN_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_BOOLEAN_ARRAY,   \
+                           GArrowBooleanArrayClass))
+#define GARROW_IS_BOOLEAN_ARRAY(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BOOLEAN_ARRAY))
+#define GARROW_IS_BOOLEAN_ARRAY_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_BOOLEAN_ARRAY))
+#define GARROW_BOOLEAN_ARRAY_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_BOOLEAN_ARRAY, \
+                             GArrowBooleanArrayClass))
+
+typedef struct _GArrowBooleanArray         GArrowBooleanArray;
+typedef struct _GArrowBooleanArrayClass    GArrowBooleanArrayClass;
+
+/**
+ * GArrowBooleanArray:
+ *
+ * It wraps `arrow::BooleanArray`.
+ */
+struct _GArrowBooleanArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowBooleanArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType          garrow_boolean_array_get_type  (void) G_GNUC_CONST;
+gboolean       garrow_boolean_array_get_value (GArrowBooleanArray *array,
+                                               gint64 i);
+
+
+#define GARROW_TYPE_INT8_ARRAY                  \
+  (garrow_int8_array_get_type())
+#define GARROW_INT8_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT8_ARRAY,   \
+                              GArrowInt8Array))
+#define GARROW_INT8_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT8_ARRAY,      \
+                           GArrowInt8ArrayClass))
+#define GARROW_IS_INT8_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT8_ARRAY))
+#define GARROW_IS_INT8_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT8_ARRAY))
+#define GARROW_INT8_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT8_ARRAY,    \
+                             GArrowInt8ArrayClass))
+
+typedef struct _GArrowInt8Array         GArrowInt8Array;
+typedef struct _GArrowInt8ArrayClass    GArrowInt8ArrayClass;
+
+/**
+ * GArrowInt8Array:
+ *
+ * It wraps `arrow::Int8Array`.
+ */
+struct _GArrowInt8Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowInt8ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_int8_array_get_type(void) G_GNUC_CONST;
+
+gint8 garrow_int8_array_get_value(GArrowInt8Array *array,
+                                  gint64 i);
+
+
+#define GARROW_TYPE_UINT8_ARRAY                 \
+  (garrow_uint8_array_get_type())
+#define GARROW_UINT8_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT8_ARRAY,  \
+                              GArrowUInt8Array))
+#define GARROW_UINT8_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT8_ARRAY,     \
+                           GArrowUInt8ArrayClass))
+#define GARROW_IS_UINT8_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_UINT8_ARRAY))
+#define GARROW_IS_UINT8_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT8_ARRAY))
+#define GARROW_UINT8_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT8_ARRAY,   \
+                             GArrowUInt8ArrayClass))
+
+typedef struct _GArrowUInt8Array         GArrowUInt8Array;
+typedef struct _GArrowUInt8ArrayClass    GArrowUInt8ArrayClass;
+
+/**
+ * GArrowUInt8Array:
+ *
+ * It wraps `arrow::UInt8Array`.
+ */
+struct _GArrowUInt8Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowUInt8ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_uint8_array_get_type(void) G_GNUC_CONST;
+
+guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array,
+                                    gint64 i);
+
+
+#define GARROW_TYPE_INT16_ARRAY                  \
+  (garrow_int16_array_get_type())
+#define GARROW_INT16_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT16_ARRAY,  \
+                              GArrowInt16Array))
+#define GARROW_INT16_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT16_ARRAY,     \
+                           GArrowInt16ArrayClass))
+#define GARROW_IS_INT16_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT16_ARRAY))
+#define GARROW_IS_INT16_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT16_ARRAY))
+#define GARROW_INT16_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT16_ARRAY,   \
+                             GArrowInt16ArrayClass))
+
+typedef struct _GArrowInt16Array         GArrowInt16Array;
+typedef struct _GArrowInt16ArrayClass    GArrowInt16ArrayClass;
+
+/**
+ * GArrowInt16Array:
+ *
+ * It wraps `arrow::Int16Array`.
+ */
+struct _GArrowInt16Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowInt16ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_int16_array_get_type(void) G_GNUC_CONST;
+
+gint16 garrow_int16_array_get_value(GArrowInt16Array *array,
+                                    gint64 i);
+
+
+#define GARROW_TYPE_UINT16_ARRAY                 \
+  (garrow_uint16_array_get_type())
+#define GARROW_UINT16_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT16_ARRAY, \
+                              GArrowUInt16Array))
+#define GARROW_UINT16_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT16_ARRAY,    \
+                           GArrowUInt16ArrayClass))
+#define GARROW_IS_UINT16_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT16_ARRAY))
+#define GARROW_IS_UINT16_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT16_ARRAY))
+#define GARROW_UINT16_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT16_ARRAY,  \
+                             GArrowUInt16ArrayClass))
+
+typedef struct _GArrowUInt16Array         GArrowUInt16Array;
+typedef struct _GArrowUInt16ArrayClass    GArrowUInt16ArrayClass;
+
+/**
+ * GArrowUInt16Array:
+ *
+ * It wraps `arrow::UInt16Array`.
+ */
+struct _GArrowUInt16Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowUInt16ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_uint16_array_get_type(void) G_GNUC_CONST;
+
+guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array,
+                                      gint64 i);
+
+
+#define GARROW_TYPE_INT32_ARRAY                 \
+  (garrow_int32_array_get_type())
+#define GARROW_INT32_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT32_ARRAY,  \
+                              GArrowInt32Array))
+#define GARROW_INT32_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT32_ARRAY,     \
+                           GArrowInt32ArrayClass))
+#define GARROW_IS_INT32_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT32_ARRAY))
+#define GARROW_IS_INT32_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT32_ARRAY))
+#define GARROW_INT32_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT32_ARRAY,   \
+                             GArrowInt32ArrayClass))
+
+typedef struct _GArrowInt32Array         GArrowInt32Array;
+typedef struct _GArrowInt32ArrayClass    GArrowInt32ArrayClass;
+
+/**
+ * GArrowInt32Array:
+ *
+ * It wraps `arrow::Int32Array`.
+ */
+struct _GArrowInt32Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowInt32ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_int32_array_get_type(void) G_GNUC_CONST;
+
+gint32 garrow_int32_array_get_value(GArrowInt32Array *array,
+                                    gint64 i);
+
+
+#define GARROW_TYPE_UINT32_ARRAY                \
+  (garrow_uint32_array_get_type())
+#define GARROW_UINT32_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT32_ARRAY, \
+                              GArrowUInt32Array))
+#define GARROW_UINT32_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT32_ARRAY,    \
+                           GArrowUInt32ArrayClass))
+#define GARROW_IS_UINT32_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT32_ARRAY))
+#define GARROW_IS_UINT32_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT32_ARRAY))
+#define GARROW_UINT32_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT32_ARRAY,  \
+                             GArrowUInt32ArrayClass))
+
+typedef struct _GArrowUInt32Array         GArrowUInt32Array;
+typedef struct _GArrowUInt32ArrayClass    GArrowUInt32ArrayClass;
+
+/**
+ * GArrowUInt32Array:
+ *
+ * It wraps `arrow::UInt32Array`.
+ */
+struct _GArrowUInt32Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowUInt32ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_uint32_array_get_type(void) G_GNUC_CONST;
+
+guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array,
+                                      gint64 i);
+
+
+#define GARROW_TYPE_INT64_ARRAY                 \
+  (garrow_int64_array_get_type())
+#define GARROW_INT64_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT64_ARRAY,  \
+                              GArrowInt64Array))
+#define GARROW_INT64_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT64_ARRAY,     \
+                           GArrowInt64ArrayClass))
+#define GARROW_IS_INT64_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT64_ARRAY))
+#define GARROW_IS_INT64_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT64_ARRAY))
+#define GARROW_INT64_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT64_ARRAY,   \
+                             GArrowInt64ArrayClass))
+
+typedef struct _GArrowInt64Array         GArrowInt64Array;
+typedef struct _GArrowInt64ArrayClass    GArrowInt64ArrayClass;
+
+/**
+ * GArrowInt64Array:
+ *
+ * It wraps `arrow::Int64Array`.
+ */
+struct _GArrowInt64Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowInt64ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_int64_array_get_type(void) G_GNUC_CONST;
+
+gint64 garrow_int64_array_get_value(GArrowInt64Array *array,
+                                    gint64 i);
+
+
+#define GARROW_TYPE_UINT64_ARRAY                \
+  (garrow_uint64_array_get_type())
+#define GARROW_UINT64_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT64_ARRAY, \
+                              GArrowUInt64Array))
+#define GARROW_UINT64_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT64_ARRAY,    \
+                           GArrowUInt64ArrayClass))
+#define GARROW_IS_UINT64_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT64_ARRAY))
+#define GARROW_IS_UINT64_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT64_ARRAY))
+#define GARROW_UINT64_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT64_ARRAY,  \
+                             GArrowUInt64ArrayClass))
+
+typedef struct _GArrowUInt64Array         GArrowUInt64Array;
+typedef struct _GArrowUInt64ArrayClass    GArrowUInt64ArrayClass;
+
+/**
+ * GArrowUInt64Array:
+ *
+ * It wraps `arrow::UInt64Array`.
+ */
+struct _GArrowUInt64Array
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowUInt64ArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_uint64_array_get_type(void) G_GNUC_CONST;
+
+guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array,
+                                      gint64 i);
+
+
+#define GARROW_TYPE_FLOAT_ARRAY                 \
+  (garrow_float_array_get_type())
+#define GARROW_FLOAT_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_FLOAT_ARRAY,  \
+                              GArrowFloatArray))
+#define GARROW_FLOAT_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_FLOAT_ARRAY,     \
+                           GArrowFloatArrayClass))
+#define GARROW_IS_FLOAT_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_FLOAT_ARRAY))
+#define GARROW_IS_FLOAT_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_FLOAT_ARRAY))
+#define GARROW_FLOAT_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_FLOAT_ARRAY,   \
+                             GArrowFloatArrayClass))
+
+typedef struct _GArrowFloatArray         GArrowFloatArray;
+typedef struct _GArrowFloatArrayClass    GArrowFloatArrayClass;
+
+/**
+ * GArrowFloatArray:
+ *
+ * It wraps `arrow::FloatArray`.
+ */
+struct _GArrowFloatArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowFloatArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_float_array_get_type(void) G_GNUC_CONST;
+
+gfloat garrow_float_array_get_value(GArrowFloatArray *array,
+                                    gint64 i);
+
+
+#define GARROW_TYPE_DOUBLE_ARRAY                \
+  (garrow_double_array_get_type())
+#define GARROW_DOUBLE_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DOUBLE_ARRAY, \
+                              GArrowDoubleArray))
+#define GARROW_DOUBLE_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DOUBLE_ARRAY,    \
+                           GArrowDoubleArrayClass))
+#define GARROW_IS_DOUBLE_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DOUBLE_ARRAY))
+#define GARROW_IS_DOUBLE_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DOUBLE_ARRAY))
+#define GARROW_DOUBLE_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DOUBLE_ARRAY,  \
+                             GArrowDoubleArrayClass))
+
+typedef struct _GArrowDoubleArray         GArrowDoubleArray;
+typedef struct _GArrowDoubleArrayClass    GArrowDoubleArrayClass;
+
+/**
+ * GArrowDoubleArray:
+ *
+ * It wraps `arrow::DoubleArray`.
+ */
+struct _GArrowDoubleArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowDoubleArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_double_array_get_type(void) G_GNUC_CONST;
+
+gdouble garrow_double_array_get_value(GArrowDoubleArray *array,
+                                      gint64 i);
+
+
+#define GARROW_TYPE_BINARY_ARRAY                \
+  (garrow_binary_array_get_type())
+#define GARROW_BINARY_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_BINARY_ARRAY, \
+                              GArrowBinaryArray))
+#define GARROW_BINARY_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_BINARY_ARRAY,    \
+                           GArrowBinaryArrayClass))
+#define GARROW_IS_BINARY_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BINARY_ARRAY))
+#define GARROW_IS_BINARY_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_BINARY_ARRAY))
+#define GARROW_BINARY_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_BINARY_ARRAY,  \
+                             GArrowBinaryArrayClass))
+
+typedef struct _GArrowBinaryArray         GArrowBinaryArray;
+typedef struct _GArrowBinaryArrayClass    GArrowBinaryArrayClass;
+
+/**
+ * GArrowBinaryArray:
+ *
+ * It wraps `arrow::BinaryArray`.
+ */
+struct _GArrowBinaryArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowBinaryArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_binary_array_get_type(void) G_GNUC_CONST;
+
+const guint8 *garrow_binary_array_get_value(GArrowBinaryArray *array,
+                                            gint64 i,
+                                            gint32 *length);
+
+#define GARROW_TYPE_STRING_ARRAY                \
+  (garrow_string_array_get_type())
+#define GARROW_STRING_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_STRING_ARRAY, \
+                              GArrowStringArray))
+#define GARROW_STRING_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_STRING_ARRAY,    \
+                           GArrowStringArrayClass))
+#define GARROW_IS_STRING_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRING_ARRAY))
+#define GARROW_IS_STRING_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_STRING_ARRAY))
+#define GARROW_STRING_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_STRING_ARRAY,  \
+                             GArrowStringArrayClass))
+
+typedef struct _GArrowStringArray         GArrowStringArray;
+typedef struct _GArrowStringArrayClass    GArrowStringArrayClass;
+
+/**
+ * GArrowStringArray:
+ *
+ * It wraps `arrow::StringArray`.
+ */
+struct _GArrowStringArray
+{
+  /*< private >*/
+  GArrowBinaryArray parent_instance;
+};
+
+struct _GArrowStringArrayClass
+{
+  GArrowBinaryArrayClass parent_class;
+};
+
+GType garrow_string_array_get_type(void) G_GNUC_CONST;
+
+gchar *garrow_string_array_get_string(GArrowStringArray *array,
+                                      gint64 i);
+
+
+#define GARROW_TYPE_LIST_ARRAY                  \
+  (garrow_list_array_get_type())
+#define GARROW_LIST_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_LIST_ARRAY,   \
+                              GArrowListArray))
+#define GARROW_LIST_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_LIST_ARRAY,      \
+                           GArrowListArrayClass))
+#define GARROW_IS_LIST_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_LIST_ARRAY))
+#define GARROW_IS_LIST_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_LIST_ARRAY))
+#define GARROW_LIST_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_LIST_ARRAY,    \
+                             GArrowListArrayClass))
+
+typedef struct _GArrowListArray         GArrowListArray;
+typedef struct _GArrowListArrayClass    GArrowListArrayClass;
+
+/**
+ * GArrowListArray:
+ *
+ * It wraps `arrow::ListArray`.
+ */
+struct _GArrowListArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowListArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_list_array_get_type(void) G_GNUC_CONST;
+
+GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array);
+GArrowArray *garrow_list_array_get_value(GArrowListArray *array,
+                                         gint64 i);
+
+
+#define GARROW_TYPE_STRUCT_ARRAY                \
+  (garrow_struct_array_get_type())
+#define GARROW_STRUCT_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_STRUCT_ARRAY, \
+                              GArrowStructArray))
+#define GARROW_STRUCT_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_STRUCT_ARRAY,    \
+                           GArrowStructArrayClass))
+#define GARROW_IS_STRUCT_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRUCT_ARRAY))
+#define GARROW_IS_STRUCT_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_STRUCT_ARRAY))
+#define GARROW_STRUCT_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_STRUCT_ARRAY,  \
+                             GArrowStructArrayClass))
+
+typedef struct _GArrowStructArray         GArrowStructArray;
+typedef struct _GArrowStructArrayClass    GArrowStructArrayClass;
+
+/**
+ * GArrowStructArray:
+ *
+ * It wraps `arrow::StructArray`.
+ */
+struct _GArrowStructArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowStructArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_struct_array_get_type(void) G_GNUC_CONST;
+
+GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
+                                           gint i);
+GList *garrow_struct_array_get_fields(GArrowStructArray *array);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index 8d9bfe2da9c38..ee408cde3615e 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -21,62 +21,46 @@
 
 #include <arrow-glib/array.h>
 #include <arrow-glib/array-builder.h>
-#include <arrow-glib/binary-array.h>
 #include <arrow-glib/binary-array-builder.h>
 #include <arrow-glib/binary-data-type.h>
-#include <arrow-glib/boolean-array.h>
 #include <arrow-glib/boolean-array-builder.h>
 #include <arrow-glib/boolean-data-type.h>
 #include <arrow-glib/chunked-array.h>
 #include <arrow-glib/column.h>
 #include <arrow-glib/data-type.h>
-#include <arrow-glib/double-array.h>
 #include <arrow-glib/double-array-builder.h>
 #include <arrow-glib/double-data-type.h>
 #include <arrow-glib/enums.h>
 #include <arrow-glib/error.h>
 #include <arrow-glib/field.h>
-#include <arrow-glib/float-array.h>
 #include <arrow-glib/float-array-builder.h>
 #include <arrow-glib/float-data-type.h>
-#include <arrow-glib/int8-array.h>
 #include <arrow-glib/int8-array-builder.h>
 #include <arrow-glib/int8-data-type.h>
-#include <arrow-glib/int16-array.h>
 #include <arrow-glib/int16-array-builder.h>
 #include <arrow-glib/int16-data-type.h>
-#include <arrow-glib/int32-array.h>
 #include <arrow-glib/int32-array-builder.h>
 #include <arrow-glib/int32-data-type.h>
-#include <arrow-glib/int64-array.h>
 #include <arrow-glib/int64-array-builder.h>
 #include <arrow-glib/int64-data-type.h>
-#include <arrow-glib/list-array.h>
 #include <arrow-glib/list-array-builder.h>
 #include <arrow-glib/list-data-type.h>
-#include <arrow-glib/null-array.h>
 #include <arrow-glib/null-data-type.h>
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
-#include <arrow-glib/string-array.h>
 #include <arrow-glib/string-array-builder.h>
 #include <arrow-glib/string-data-type.h>
-#include <arrow-glib/struct-array.h>
 #include <arrow-glib/struct-array-builder.h>
 #include <arrow-glib/struct-data-type.h>
 #include <arrow-glib/table.h>
 #include <arrow-glib/tensor.h>
 #include <arrow-glib/type.h>
-#include <arrow-glib/uint8-array.h>
 #include <arrow-glib/uint8-array-builder.h>
 #include <arrow-glib/uint8-data-type.h>
-#include <arrow-glib/uint16-array.h>
 #include <arrow-glib/uint16-array-builder.h>
 #include <arrow-glib/uint16-data-type.h>
-#include <arrow-glib/uint32-array.h>
 #include <arrow-glib/uint32-array-builder.h>
 #include <arrow-glib/uint32-data-type.h>
-#include <arrow-glib/uint64-array.h>
 #include <arrow-glib/uint64-array-builder.h>
 #include <arrow-glib/uint64-data-type.h>
 
diff --git a/c_glib/arrow-glib/binary-array.cpp b/c_glib/arrow-glib/binary-array.cpp
deleted file mode 100644
index c149d14025ae7..0000000000000
--- a/c_glib/arrow-glib/binary-array.cpp
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/binary-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: binary-array
- * @short_description: Binary array class
- *
- * #GArrowBinaryArray is a class for binary array. It can store zero
- * or more binary data.
- *
- * #GArrowBinaryArray is immutable. You need to use
- * #GArrowBinaryArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowBinaryArray,               \
-              garrow_binary_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_binary_array_init(GArrowBinaryArray *object)
-{
-}
-
-static void
-garrow_binary_array_class_init(GArrowBinaryArrayClass *klass)
-{
-}
-
-/**
- * garrow_binary_array_get_value:
- * @array: A #GArrowBinaryArray.
- * @i: The index of the target value.
- * @length: (out): The length of the value.
- *
- * Returns: (array length=length): The i-th value.
- */
-const guint8 *
-garrow_binary_array_get_value(GArrowBinaryArray *array,
-                              gint64 i,
-                              gint32 *length)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_binary_array =
-    static_cast<arrow::BinaryArray *>(arrow_array.get());
-  return arrow_binary_array->GetValue(i, length);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/binary-array.h b/c_glib/arrow-glib/binary-array.h
deleted file mode 100644
index ab63ece9844f8..0000000000000
--- a/c_glib/arrow-glib/binary-array.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_BINARY_ARRAY                \
-  (garrow_binary_array_get_type())
-#define GARROW_BINARY_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_BINARY_ARRAY, \
-                              GArrowBinaryArray))
-#define GARROW_BINARY_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_BINARY_ARRAY,    \
-                           GArrowBinaryArrayClass))
-#define GARROW_IS_BINARY_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BINARY_ARRAY))
-#define GARROW_IS_BINARY_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_BINARY_ARRAY))
-#define GARROW_BINARY_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_BINARY_ARRAY,  \
-                             GArrowBinaryArrayClass))
-
-typedef struct _GArrowBinaryArray         GArrowBinaryArray;
-typedef struct _GArrowBinaryArrayClass    GArrowBinaryArrayClass;
-
-/**
- * GArrowBinaryArray:
- *
- * It wraps `arrow::BinaryArray`.
- */
-struct _GArrowBinaryArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowBinaryArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_binary_array_get_type(void) G_GNUC_CONST;
-
-const guint8 *garrow_binary_array_get_value(GArrowBinaryArray *array,
-                                            gint64 i,
-                                            gint32 *length);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-array.cpp b/c_glib/arrow-glib/boolean-array.cpp
deleted file mode 100644
index 62fc40fd54112..0000000000000
--- a/c_glib/arrow-glib/boolean-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/boolean-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: boolean-array
- * @short_description: Boolean array class
- *
- * #GArrowBooleanArray is a class for binary array. It can store zero
- * or more boolean data.
- *
- * #GArrowBooleanArray is immutable. You need to use
- * #GArrowBooleanArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowBooleanArray,               \
-              garrow_boolean_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_boolean_array_init(GArrowBooleanArray *object)
-{
-}
-
-static void
-garrow_boolean_array_class_init(GArrowBooleanArrayClass *klass)
-{
-}
-
-/**
- * garrow_boolean_array_get_value:
- * @array: A #GArrowBooleanArray.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-gboolean
-garrow_boolean_array_get_value(GArrowBooleanArray *array,
-                               gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::BooleanArray *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-array.h b/c_glib/arrow-glib/boolean-array.h
deleted file mode 100644
index 9899fdf0ceca8..0000000000000
--- a/c_glib/arrow-glib/boolean-array.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_BOOLEAN_ARRAY               \
-  (garrow_boolean_array_get_type())
-#define GARROW_BOOLEAN_ARRAY(obj)                               \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BOOLEAN_ARRAY,        \
-                              GArrowBooleanArray))
-#define GARROW_BOOLEAN_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_BOOLEAN_ARRAY,   \
-                           GArrowBooleanArrayClass))
-#define GARROW_IS_BOOLEAN_ARRAY(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BOOLEAN_ARRAY))
-#define GARROW_IS_BOOLEAN_ARRAY_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_BOOLEAN_ARRAY))
-#define GARROW_BOOLEAN_ARRAY_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_BOOLEAN_ARRAY, \
-                             GArrowBooleanArrayClass))
-
-typedef struct _GArrowBooleanArray         GArrowBooleanArray;
-typedef struct _GArrowBooleanArrayClass    GArrowBooleanArrayClass;
-
-/**
- * GArrowBooleanArray:
- *
- * It wraps `arrow::BooleanArray`.
- */
-struct _GArrowBooleanArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowBooleanArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType          garrow_boolean_array_get_type  (void) G_GNUC_CONST;
-gboolean       garrow_boolean_array_get_value (GArrowBooleanArray *array,
-                                               gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/double-array.cpp b/c_glib/arrow-glib/double-array.cpp
deleted file mode 100644
index ecc55d7541689..0000000000000
--- a/c_glib/arrow-glib/double-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/double-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: double-array
- * @short_description: 64-bit floating point array class
- *
- * #GArrowDoubleArray is a class for 64-bit floating point array. It
- * can store zero or more 64-bit floating data.
- *
- * #GArrowDoubleArray is immutable. You need to use
- * #GArrowDoubleArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowDoubleArray,               \
-              garrow_double_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_double_array_init(GArrowDoubleArray *object)
-{
-}
-
-static void
-garrow_double_array_class_init(GArrowDoubleArrayClass *klass)
-{
-}
-
-/**
- * garrow_double_array_get_value:
- * @array: A #GArrowDoubleArray.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-gdouble
-garrow_double_array_get_value(GArrowDoubleArray *array,
-                              gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::DoubleArray *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/double-array.h b/c_glib/arrow-glib/double-array.h
deleted file mode 100644
index b9a236532e3bf..0000000000000
--- a/c_glib/arrow-glib/double-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_DOUBLE_ARRAY                \
-  (garrow_double_array_get_type())
-#define GARROW_DOUBLE_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DOUBLE_ARRAY, \
-                              GArrowDoubleArray))
-#define GARROW_DOUBLE_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DOUBLE_ARRAY,    \
-                           GArrowDoubleArrayClass))
-#define GARROW_IS_DOUBLE_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DOUBLE_ARRAY))
-#define GARROW_IS_DOUBLE_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DOUBLE_ARRAY))
-#define GARROW_DOUBLE_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DOUBLE_ARRAY,  \
-                             GArrowDoubleArrayClass))
-
-typedef struct _GArrowDoubleArray         GArrowDoubleArray;
-typedef struct _GArrowDoubleArrayClass    GArrowDoubleArrayClass;
-
-/**
- * GArrowDoubleArray:
- *
- * It wraps `arrow::DoubleArray`.
- */
-struct _GArrowDoubleArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowDoubleArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_double_array_get_type(void) G_GNUC_CONST;
-
-gdouble garrow_double_array_get_value(GArrowDoubleArray *array,
-                                      gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/float-array.cpp b/c_glib/arrow-glib/float-array.cpp
deleted file mode 100644
index 28e8047652f7e..0000000000000
--- a/c_glib/arrow-glib/float-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/float-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: float-array
- * @short_description: 32-bit floating point array class
- *
- * #GArrowFloatArray is a class for 32-bit floating point array. It
- * can store zero or more 32-bit floating data.
- *
- * #GArrowFloatArray is immutable. You need to use
- * #GArrowFloatArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowFloatArray,               \
-              garrow_float_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_float_array_init(GArrowFloatArray *object)
-{
-}
-
-static void
-garrow_float_array_class_init(GArrowFloatArrayClass *klass)
-{
-}
-
-/**
- * garrow_float_array_get_value:
- * @array: A #GArrowFloatArray.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-gfloat
-garrow_float_array_get_value(GArrowFloatArray *array,
-                             gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::FloatArray *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/float-array.h b/c_glib/arrow-glib/float-array.h
deleted file mode 100644
index d113f9757a511..0000000000000
--- a/c_glib/arrow-glib/float-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_FLOAT_ARRAY                 \
-  (garrow_float_array_get_type())
-#define GARROW_FLOAT_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_FLOAT_ARRAY,  \
-                              GArrowFloatArray))
-#define GARROW_FLOAT_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_FLOAT_ARRAY,     \
-                           GArrowFloatArrayClass))
-#define GARROW_IS_FLOAT_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_FLOAT_ARRAY))
-#define GARROW_IS_FLOAT_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_FLOAT_ARRAY))
-#define GARROW_FLOAT_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_FLOAT_ARRAY,   \
-                             GArrowFloatArrayClass))
-
-typedef struct _GArrowFloatArray         GArrowFloatArray;
-typedef struct _GArrowFloatArrayClass    GArrowFloatArrayClass;
-
-/**
- * GArrowFloatArray:
- *
- * It wraps `arrow::FloatArray`.
- */
-struct _GArrowFloatArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowFloatArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_float_array_get_type(void) G_GNUC_CONST;
-
-gfloat garrow_float_array_get_value(GArrowFloatArray *array,
-                                    gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-array.cpp b/c_glib/arrow-glib/int16-array.cpp
deleted file mode 100644
index 456d085a3449a..0000000000000
--- a/c_glib/arrow-glib/int16-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/int16-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int16-array
- * @short_description: 16-bit integer array class
- *
- * #GArrowInt16Array is a class for 16-bit integer array. It can store
- * zero or more 16-bit integer data.
- *
- * #GArrowInt16Array is immutable. You need to use
- * #GArrowInt16ArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowInt16Array,               \
-              garrow_int16_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_int16_array_init(GArrowInt16Array *object)
-{
-}
-
-static void
-garrow_int16_array_class_init(GArrowInt16ArrayClass *klass)
-{
-}
-
-/**
- * garrow_int16_array_get_value:
- * @array: A #GArrowInt16Array.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-gint16
-garrow_int16_array_get_value(GArrowInt16Array *array,
-                             gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::Int16Array *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-array.h b/c_glib/arrow-glib/int16-array.h
deleted file mode 100644
index d37144cef51f2..0000000000000
--- a/c_glib/arrow-glib/int16-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT16_ARRAY                  \
-  (garrow_int16_array_get_type())
-#define GARROW_INT16_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT16_ARRAY,   \
-                              GArrowInt16Array))
-#define GARROW_INT16_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT16_ARRAY,      \
-                           GArrowInt16ArrayClass))
-#define GARROW_IS_INT16_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT16_ARRAY))
-#define GARROW_IS_INT16_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT16_ARRAY))
-#define GARROW_INT16_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT16_ARRAY,    \
-                             GArrowInt16ArrayClass))
-
-typedef struct _GArrowInt16Array         GArrowInt16Array;
-typedef struct _GArrowInt16ArrayClass    GArrowInt16ArrayClass;
-
-/**
- * GArrowInt16Array:
- *
- * It wraps `arrow::Int16Array`.
- */
-struct _GArrowInt16Array
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowInt16ArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_int16_array_get_type(void) G_GNUC_CONST;
-
-gint16 garrow_int16_array_get_value(GArrowInt16Array *array,
-                                  gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-array.cpp b/c_glib/arrow-glib/int32-array.cpp
deleted file mode 100644
index 8bd6f35fd6431..0000000000000
--- a/c_glib/arrow-glib/int32-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/int32-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int32-array
- * @short_description: 32-bit integer array class
- *
- * #GArrowInt32Array is a class for 32-bit integer array. It can store
- * zero or more 32-bit integer data.
- *
- * #GArrowInt32Array is immutable. You need to use
- * #GArrowInt32ArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowInt32Array,               \
-              garrow_int32_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_int32_array_init(GArrowInt32Array *object)
-{
-}
-
-static void
-garrow_int32_array_class_init(GArrowInt32ArrayClass *klass)
-{
-}
-
-/**
- * garrow_int32_array_get_value:
- * @array: A #GArrowInt32Array.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-gint32
-garrow_int32_array_get_value(GArrowInt32Array *array,
-                             gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::Int32Array *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-array.h b/c_glib/arrow-glib/int32-array.h
deleted file mode 100644
index cce2b41aafe26..0000000000000
--- a/c_glib/arrow-glib/int32-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT32_ARRAY                  \
-  (garrow_int32_array_get_type())
-#define GARROW_INT32_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT32_ARRAY,   \
-                              GArrowInt32Array))
-#define GARROW_INT32_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT32_ARRAY,      \
-                           GArrowInt32ArrayClass))
-#define GARROW_IS_INT32_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT32_ARRAY))
-#define GARROW_IS_INT32_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT32_ARRAY))
-#define GARROW_INT32_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT32_ARRAY,    \
-                             GArrowInt32ArrayClass))
-
-typedef struct _GArrowInt32Array         GArrowInt32Array;
-typedef struct _GArrowInt32ArrayClass    GArrowInt32ArrayClass;
-
-/**
- * GArrowInt32Array:
- *
- * It wraps `arrow::Int32Array`.
- */
-struct _GArrowInt32Array
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowInt32ArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_int32_array_get_type(void) G_GNUC_CONST;
-
-gint32 garrow_int32_array_get_value(GArrowInt32Array *array,
-                                  gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-array.cpp b/c_glib/arrow-glib/int64-array.cpp
deleted file mode 100644
index be49d5bf35251..0000000000000
--- a/c_glib/arrow-glib/int64-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/int64-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int64-array
- * @short_description: 64-bit integer array class
- *
- * #GArrowInt64Array is a class for 64-bit integer array. It can store
- * zero or more 64-bit integer data.
- *
- * #GArrowInt64Array is immutable. You need to use
- * #GArrowInt64ArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowInt64Array,               \
-              garrow_int64_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_int64_array_init(GArrowInt64Array *object)
-{
-}
-
-static void
-garrow_int64_array_class_init(GArrowInt64ArrayClass *klass)
-{
-}
-
-/**
- * garrow_int64_array_get_value:
- * @array: A #GArrowInt64Array.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-gint64
-garrow_int64_array_get_value(GArrowInt64Array *array,
-                             gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::Int64Array *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-array.h b/c_glib/arrow-glib/int64-array.h
deleted file mode 100644
index 73d4c6453a6d5..0000000000000
--- a/c_glib/arrow-glib/int64-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT64_ARRAY                  \
-  (garrow_int64_array_get_type())
-#define GARROW_INT64_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT64_ARRAY,   \
-                              GArrowInt64Array))
-#define GARROW_INT64_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT64_ARRAY,      \
-                           GArrowInt64ArrayClass))
-#define GARROW_IS_INT64_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT64_ARRAY))
-#define GARROW_IS_INT64_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT64_ARRAY))
-#define GARROW_INT64_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT64_ARRAY,    \
-                             GArrowInt64ArrayClass))
-
-typedef struct _GArrowInt64Array         GArrowInt64Array;
-typedef struct _GArrowInt64ArrayClass    GArrowInt64ArrayClass;
-
-/**
- * GArrowInt64Array:
- *
- * It wraps `arrow::Int64Array`.
- */
-struct _GArrowInt64Array
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowInt64ArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_int64_array_get_type(void) G_GNUC_CONST;
-
-gint64 garrow_int64_array_get_value(GArrowInt64Array *array,
-                                    gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-array.cpp b/c_glib/arrow-glib/int8-array.cpp
deleted file mode 100644
index d3f12ece9bbf7..0000000000000
--- a/c_glib/arrow-glib/int8-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/int8-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int8-array
- * @short_description: 8-bit integer array class
- *
- * #GArrowInt8Array is a class for 8-bit integer array. It can store
- * zero or more 8-bit integer data.
- *
- * #GArrowInt8Array is immutable. You need to use
- * #GArrowInt8ArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowInt8Array,               \
-              garrow_int8_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_int8_array_init(GArrowInt8Array *object)
-{
-}
-
-static void
-garrow_int8_array_class_init(GArrowInt8ArrayClass *klass)
-{
-}
-
-/**
- * garrow_int8_array_get_value:
- * @array: A #GArrowInt8Array.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-gint8
-garrow_int8_array_get_value(GArrowInt8Array *array,
-                            gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::Int8Array *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-array.h b/c_glib/arrow-glib/int8-array.h
deleted file mode 100644
index 0e1e901f4fdb6..0000000000000
--- a/c_glib/arrow-glib/int8-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT8_ARRAY                  \
-  (garrow_int8_array_get_type())
-#define GARROW_INT8_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT8_ARRAY,   \
-                              GArrowInt8Array))
-#define GARROW_INT8_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT8_ARRAY,      \
-                           GArrowInt8ArrayClass))
-#define GARROW_IS_INT8_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT8_ARRAY))
-#define GARROW_IS_INT8_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT8_ARRAY))
-#define GARROW_INT8_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT8_ARRAY,    \
-                             GArrowInt8ArrayClass))
-
-typedef struct _GArrowInt8Array         GArrowInt8Array;
-typedef struct _GArrowInt8ArrayClass    GArrowInt8ArrayClass;
-
-/**
- * GArrowInt8Array:
- *
- * It wraps `arrow::Int8Array`.
- */
-struct _GArrowInt8Array
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowInt8ArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_int8_array_get_type(void) G_GNUC_CONST;
-
-gint8 garrow_int8_array_get_value(GArrowInt8Array *array,
-                                  gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/list-array.cpp b/c_glib/arrow-glib/list-array.cpp
deleted file mode 100644
index 2b3fb311280d0..0000000000000
--- a/c_glib/arrow-glib/list-array.cpp
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/list-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: list-array
- * @short_description: List array class
- * @include: arrow-glib/arrow-glib.h
- *
- * #GArrowListArray is a class for list array. It can store zero
- * or more list data.
- *
- * #GArrowListArray is immutable. You need to use
- * #GArrowListArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowListArray,               \
-              garrow_list_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_list_array_init(GArrowListArray *object)
-{
-}
-
-static void
-garrow_list_array_class_init(GArrowListArrayClass *klass)
-{
-}
-
-/**
- * garrow_list_array_get_value_type:
- * @array: A #GArrowListArray.
- *
- * Returns: (transfer full): The data type of value in each list.
- */
-GArrowDataType *
-garrow_list_array_get_value_type(GArrowListArray *array)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_list_array =
-    static_cast<arrow::ListArray *>(arrow_array.get());
-  auto arrow_value_type = arrow_list_array->value_type();
-  return garrow_data_type_new_raw(&arrow_value_type);
-}
-
-/**
- * garrow_list_array_get_value:
- * @array: A #GArrowListArray.
- * @i: The index of the target value.
- *
- * Returns: (transfer full): The i-th list.
- */
-GArrowArray *
-garrow_list_array_get_value(GArrowListArray *array,
-                            gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_list_array =
-    static_cast<arrow::ListArray *>(arrow_array.get());
-  auto arrow_list =
-    arrow_list_array->values()->Slice(arrow_list_array->value_offset(i),
-                                      arrow_list_array->value_length(i));
-  return garrow_array_new_raw(&arrow_list);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/list-array.h b/c_glib/arrow-glib/list-array.h
deleted file mode 100644
index c49aed1b9599e..0000000000000
--- a/c_glib/arrow-glib/list-array.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_LIST_ARRAY                  \
-  (garrow_list_array_get_type())
-#define GARROW_LIST_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_LIST_ARRAY,   \
-                              GArrowListArray))
-#define GARROW_LIST_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_LIST_ARRAY,      \
-                           GArrowListArrayClass))
-#define GARROW_IS_LIST_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_LIST_ARRAY))
-#define GARROW_IS_LIST_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_LIST_ARRAY))
-#define GARROW_LIST_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_LIST_ARRAY,    \
-                             GArrowListArrayClass))
-
-typedef struct _GArrowListArray         GArrowListArray;
-typedef struct _GArrowListArrayClass    GArrowListArrayClass;
-
-/**
- * GArrowListArray:
- *
- * It wraps `arrow::ListArray`.
- */
-struct _GArrowListArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowListArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_list_array_get_type(void) G_GNUC_CONST;
-
-GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array);
-GArrowArray *garrow_list_array_get_value(GArrowListArray *array,
-                                         gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/null-array.cpp b/c_glib/arrow-glib/null-array.cpp
deleted file mode 100644
index 0e0ea51e24c04..0000000000000
--- a/c_glib/arrow-glib/null-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/null-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: null-array
- * @short_description: Null array class
- *
- * #GArrowNullArray is a class for null array. It can store zero
- * or more null values.
- *
- * #GArrowNullArray is immutable. You need to specify an array length
- * to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowNullArray,               \
-              garrow_null_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_null_array_init(GArrowNullArray *object)
-{
-}
-
-static void
-garrow_null_array_class_init(GArrowNullArrayClass *klass)
-{
-}
-
-/**
- * garrow_null_array_new:
- * @length: An array length.
- *
- * Returns: A newly created #GArrowNullArray.
- */
-GArrowNullArray *
-garrow_null_array_new(gint64 length)
-{
-  auto arrow_null_array = std::make_shared<arrow::NullArray>(length);
-  std::shared_ptr<arrow::Array> arrow_array = arrow_null_array;
-  auto array = garrow_array_new_raw(&arrow_array);
-  return GARROW_NULL_ARRAY(array);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/null-array.h b/c_glib/arrow-glib/null-array.h
deleted file mode 100644
index e25f3054843e4..0000000000000
--- a/c_glib/arrow-glib/null-array.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_NULL_ARRAY                  \
-  (garrow_null_array_get_type())
-#define GARROW_NULL_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_NULL_ARRAY,   \
-                              GArrowNullArray))
-#define GARROW_NULL_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_NULL_ARRAY,      \
-                           GArrowNullArrayClass))
-#define GARROW_IS_NULL_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_NULL_ARRAY))
-#define GARROW_IS_NULL_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_NULL_ARRAY))
-#define GARROW_NULL_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_NULL_ARRAY,    \
-                             GArrowNullArrayClass))
-
-typedef struct _GArrowNullArray         GArrowNullArray;
-typedef struct _GArrowNullArrayClass    GArrowNullArrayClass;
-
-/**
- * GArrowNullArray:
- *
- * It wraps `arrow::NullArray`.
- */
-struct _GArrowNullArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowNullArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_null_array_get_type(void) G_GNUC_CONST;
-
-GArrowNullArray *garrow_null_array_new(gint64 length);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/string-array.cpp b/c_glib/arrow-glib/string-array.cpp
deleted file mode 100644
index 329c742ccafe1..0000000000000
--- a/c_glib/arrow-glib/string-array.cpp
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/string-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: string-array
- * @short_description: UTF-8 encoded string array class
- *
- * #GArrowStringArray is a class for UTF-8 encoded string array. It
- * can store zero or more UTF-8 encoded string data.
- *
- * #GArrowStringArray is immutable. You need to use
- * #GArrowStringArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowStringArray,               \
-              garrow_string_array,             \
-              GARROW_TYPE_BINARY_ARRAY)
-
-static void
-garrow_string_array_init(GArrowStringArray *object)
-{
-}
-
-static void
-garrow_string_array_class_init(GArrowStringArrayClass *klass)
-{
-}
-
-/**
- * garrow_string_array_get_string:
- * @array: A #GArrowStringArray.
- * @i: The index of the target value.
- *
- * Returns: The i-th UTF-8 encoded string.
- */
-gchar *
-garrow_string_array_get_string(GArrowStringArray *array,
-                               gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_string_array =
-    static_cast<arrow::StringArray *>(arrow_array.get());
-  gint32 length;
-  auto value =
-    reinterpret_cast<const gchar *>(arrow_string_array->GetValue(i, &length));
-  return g_strndup(value, length);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/string-array.h b/c_glib/arrow-glib/string-array.h
deleted file mode 100644
index 41a53cd5f1d4a..0000000000000
--- a/c_glib/arrow-glib/string-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/binary-array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_STRING_ARRAY                \
-  (garrow_string_array_get_type())
-#define GARROW_STRING_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_STRING_ARRAY, \
-                              GArrowStringArray))
-#define GARROW_STRING_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_STRING_ARRAY,    \
-                           GArrowStringArrayClass))
-#define GARROW_IS_STRING_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STRING_ARRAY))
-#define GARROW_IS_STRING_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_STRING_ARRAY))
-#define GARROW_STRING_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_STRING_ARRAY,  \
-                             GArrowStringArrayClass))
-
-typedef struct _GArrowStringArray         GArrowStringArray;
-typedef struct _GArrowStringArrayClass    GArrowStringArrayClass;
-
-/**
- * GArrowStringArray:
- *
- * It wraps `arrow::StringArray`.
- */
-struct _GArrowStringArray
-{
-  /*< private >*/
-  GArrowBinaryArray parent_instance;
-};
-
-struct _GArrowStringArrayClass
-{
-  GArrowBinaryArrayClass parent_class;
-};
-
-GType garrow_string_array_get_type(void) G_GNUC_CONST;
-
-gchar *garrow_string_array_get_string(GArrowStringArray *array,
-                                      gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array.cpp b/c_glib/arrow-glib/struct-array.cpp
deleted file mode 100644
index 14c2d17cdd737..0000000000000
--- a/c_glib/arrow-glib/struct-array.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/struct-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: struct-array
- * @short_description: Struct array class
- * @include: arrow-glib/arrow-glib.h
- *
- * #GArrowStructArray is a class for struct array. It can store zero
- * or more structs. One struct has zero or more fields.
- *
- * #GArrowStructArray is immutable. You need to use
- * #GArrowStructArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowStructArray,               \
-              garrow_struct_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_struct_array_init(GArrowStructArray *object)
-{
-}
-
-static void
-garrow_struct_array_class_init(GArrowStructArrayClass *klass)
-{
-}
-
-/**
- * garrow_struct_array_get_field
- * @array: A #GArrowStructArray.
- * @i: The index of the field in the struct.
- *
- * Returns: (transfer full): The i-th field.
- */
-GArrowArray *
-garrow_struct_array_get_field(GArrowStructArray *array,
-                              gint i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_struct_array =
-    static_cast<arrow::StructArray *>(arrow_array.get());
-  auto arrow_field = arrow_struct_array->field(i);
-  return garrow_array_new_raw(&arrow_field);
-}
-
-/**
- * garrow_struct_array_get_fields
- * @array: A #GArrowStructArray.
- *
- * Returns: (element-type GArrowArray) (transfer full):
- *   The fields in the struct.
- */
-GList *
-garrow_struct_array_get_fields(GArrowStructArray *array)
-{
-  const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  const auto arrow_struct_array =
-    static_cast<const arrow::StructArray *>(arrow_array.get());
-
-  GList *fields = NULL;
-  for (auto arrow_field : arrow_struct_array->fields()) {
-    GArrowArray *field = garrow_array_new_raw(&arrow_field);
-    fields = g_list_prepend(fields, field);
-  }
-
-  return g_list_reverse(fields);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array.h b/c_glib/arrow-glib/struct-array.h
deleted file mode 100644
index f96e9d468f350..0000000000000
--- a/c_glib/arrow-glib/struct-array.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_STRUCT_ARRAY                \
-  (garrow_struct_array_get_type())
-#define GARROW_STRUCT_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_STRUCT_ARRAY, \
-                              GArrowStructArray))
-#define GARROW_STRUCT_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_STRUCT_ARRAY,    \
-                           GArrowStructArrayClass))
-#define GARROW_IS_STRUCT_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STRUCT_ARRAY))
-#define GARROW_IS_STRUCT_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_STRUCT_ARRAY))
-#define GARROW_STRUCT_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_STRUCT_ARRAY,  \
-                             GArrowStructArrayClass))
-
-typedef struct _GArrowStructArray         GArrowStructArray;
-typedef struct _GArrowStructArrayClass    GArrowStructArrayClass;
-
-/**
- * GArrowStructArray:
- *
- * It wraps `arrow::StructArray`.
- */
-struct _GArrowStructArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowStructArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_struct_array_get_type(void) G_GNUC_CONST;
-
-GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
-                                           gint i);
-GList *garrow_struct_array_get_fields(GArrowStructArray *array);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-array.cpp b/c_glib/arrow-glib/uint16-array.cpp
deleted file mode 100644
index 6c416c6592935..0000000000000
--- a/c_glib/arrow-glib/uint16-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/uint16-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint16-array
- * @short_description: 16-bit unsigned integer array class
- *
- * #GArrowUInt16Array is a class for 16-bit unsigned integer array. It
- * can store zero or more 16-bit unsigned integer data.
- *
- * #GArrowUInt16Array is immutable. You need to use
- * #GArrowUInt16ArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowUInt16Array,               \
-              garrow_uint16_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_uint16_array_init(GArrowUInt16Array *object)
-{
-}
-
-static void
-garrow_uint16_array_class_init(GArrowUInt16ArrayClass *klass)
-{
-}
-
-/**
- * garrow_uint16_array_get_value:
- * @array: A #GArrowUInt16Array.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-guint16
-garrow_uint16_array_get_value(GArrowUInt16Array *array,
-                             gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::UInt16Array *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-array.h b/c_glib/arrow-glib/uint16-array.h
deleted file mode 100644
index 44725510062c8..0000000000000
--- a/c_glib/arrow-glib/uint16-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT16_ARRAY                 \
-  (garrow_uint16_array_get_type())
-#define GARROW_UINT16_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT16_ARRAY,  \
-                              GArrowUInt16Array))
-#define GARROW_UINT16_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT16_ARRAY,     \
-                           GArrowUInt16ArrayClass))
-#define GARROW_IS_UINT16_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_UINT16_ARRAY))
-#define GARROW_IS_UINT16_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT16_ARRAY))
-#define GARROW_UINT16_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT16_ARRAY,   \
-                             GArrowUInt16ArrayClass))
-
-typedef struct _GArrowUInt16Array         GArrowUInt16Array;
-typedef struct _GArrowUInt16ArrayClass    GArrowUInt16ArrayClass;
-
-/**
- * GArrowUInt16Array:
- *
- * It wraps `arrow::UInt16Array`.
- */
-struct _GArrowUInt16Array
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowUInt16ArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_uint16_array_get_type(void) G_GNUC_CONST;
-
-guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array,
-                                    gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-array.cpp b/c_glib/arrow-glib/uint32-array.cpp
deleted file mode 100644
index 18a9aedc0658f..0000000000000
--- a/c_glib/arrow-glib/uint32-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/uint32-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint32-array
- * @short_description: 32-bit unsigned integer array class
- *
- * #GArrowUInt32Array is a class for 32-bit unsigned integer array. It
- * can store zero or more 32-bit unsigned integer data.
- *
- * #GArrowUInt32Array is immutable. You need to use
- * #GArrowUInt32ArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowUInt32Array,               \
-              garrow_uint32_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_uint32_array_init(GArrowUInt32Array *object)
-{
-}
-
-static void
-garrow_uint32_array_class_init(GArrowUInt32ArrayClass *klass)
-{
-}
-
-/**
- * garrow_uint32_array_get_value:
- * @array: A #GArrowUInt32Array.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-guint32
-garrow_uint32_array_get_value(GArrowUInt32Array *array,
-                              gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::UInt32Array *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-array.h b/c_glib/arrow-glib/uint32-array.h
deleted file mode 100644
index 57d4beaee6186..0000000000000
--- a/c_glib/arrow-glib/uint32-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT32_ARRAY                 \
-  (garrow_uint32_array_get_type())
-#define GARROW_UINT32_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT32_ARRAY,  \
-                              GArrowUInt32Array))
-#define GARROW_UINT32_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT32_ARRAY,     \
-                           GArrowUInt32ArrayClass))
-#define GARROW_IS_UINT32_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_UINT32_ARRAY))
-#define GARROW_IS_UINT32_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT32_ARRAY))
-#define GARROW_UINT32_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT32_ARRAY,   \
-                             GArrowUInt32ArrayClass))
-
-typedef struct _GArrowUInt32Array         GArrowUInt32Array;
-typedef struct _GArrowUInt32ArrayClass    GArrowUInt32ArrayClass;
-
-/**
- * GArrowUInt32Array:
- *
- * It wraps `arrow::UInt32Array`.
- */
-struct _GArrowUInt32Array
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowUInt32ArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_uint32_array_get_type(void) G_GNUC_CONST;
-
-guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array,
-                                    gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-array.cpp b/c_glib/arrow-glib/uint64-array.cpp
deleted file mode 100644
index 1f900842674b8..0000000000000
--- a/c_glib/arrow-glib/uint64-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/uint64-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint64-array
- * @short_description: 64-bit unsigned integer array class
- *
- * #GArrowUInt64Array is a class for 64-bit unsigned integer array. It
- * can store zero or more 64-bit unsigned integer data.
- *
- * #GArrowUInt64Array is immutable. You need to use
- * #GArrowUInt64ArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowUInt64Array,               \
-              garrow_uint64_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_uint64_array_init(GArrowUInt64Array *object)
-{
-}
-
-static void
-garrow_uint64_array_class_init(GArrowUInt64ArrayClass *klass)
-{
-}
-
-/**
- * garrow_uint64_array_get_value:
- * @array: A #GArrowUInt64Array.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-guint64
-garrow_uint64_array_get_value(GArrowUInt64Array *array,
-                             gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::UInt64Array *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-array.h b/c_glib/arrow-glib/uint64-array.h
deleted file mode 100644
index b5abde52bd263..0000000000000
--- a/c_glib/arrow-glib/uint64-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT64_ARRAY                 \
-  (garrow_uint64_array_get_type())
-#define GARROW_UINT64_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT64_ARRAY,  \
-                              GArrowUInt64Array))
-#define GARROW_UINT64_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT64_ARRAY,     \
-                           GArrowUInt64ArrayClass))
-#define GARROW_IS_UINT64_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_UINT64_ARRAY))
-#define GARROW_IS_UINT64_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT64_ARRAY))
-#define GARROW_UINT64_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT64_ARRAY,   \
-                             GArrowUInt64ArrayClass))
-
-typedef struct _GArrowUInt64Array         GArrowUInt64Array;
-typedef struct _GArrowUInt64ArrayClass    GArrowUInt64ArrayClass;
-
-/**
- * GArrowUInt64Array:
- *
- * It wraps `arrow::UInt64Array`.
- */
-struct _GArrowUInt64Array
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowUInt64ArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_uint64_array_get_type(void) G_GNUC_CONST;
-
-guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array,
-                                    gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-array.cpp b/c_glib/arrow-glib/uint8-array.cpp
deleted file mode 100644
index b5a2595b1ef09..0000000000000
--- a/c_glib/arrow-glib/uint8-array.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/uint8-array.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint8-array
- * @short_description: 8-bit unsigned integer array class
- *
- * #GArrowUInt8Array is a class for 8-bit unsigned integer array. It
- * can store zero or more 8-bit unsigned integer data.
- *
- * #GArrowUInt8Array is immutable. You need to use
- * #GArrowUInt8ArrayBuilder to create a new array.
- */
-
-G_DEFINE_TYPE(GArrowUInt8Array,               \
-              garrow_uint8_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_uint8_array_init(GArrowUInt8Array *object)
-{
-}
-
-static void
-garrow_uint8_array_class_init(GArrowUInt8ArrayClass *klass)
-{
-}
-
-/**
- * garrow_uint8_array_get_value:
- * @array: A #GArrowUInt8Array.
- * @i: The index of the target value.
- *
- * Returns: The i-th value.
- */
-guint8
-garrow_uint8_array_get_value(GArrowUInt8Array *array,
-                             gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  return static_cast<arrow::UInt8Array *>(arrow_array.get())->Value(i);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-array.h b/c_glib/arrow-glib/uint8-array.h
deleted file mode 100644
index a572bc549670e..0000000000000
--- a/c_glib/arrow-glib/uint8-array.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT8_ARRAY                 \
-  (garrow_uint8_array_get_type())
-#define GARROW_UINT8_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT8_ARRAY,  \
-                              GArrowUInt8Array))
-#define GARROW_UINT8_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT8_ARRAY,     \
-                           GArrowUInt8ArrayClass))
-#define GARROW_IS_UINT8_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_UINT8_ARRAY))
-#define GARROW_IS_UINT8_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT8_ARRAY))
-#define GARROW_UINT8_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT8_ARRAY,   \
-                             GArrowUInt8ArrayClass))
-
-typedef struct _GArrowUInt8Array         GArrowUInt8Array;
-typedef struct _GArrowUInt8ArrayClass    GArrowUInt8ArrayClass;
-
-/**
- * GArrowUInt8Array:
- *
- * It wraps `arrow::UInt8Array`.
- */
-struct _GArrowUInt8Array
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowUInt8ArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_uint8_array_get_type(void) G_GNUC_CONST;
-
-guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array,
-                                    gint64 i);
-
-G_END_DECLS
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 3c1d8d161179c..11e6a4de244d4 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -36,22 +36,6 @@
     <chapter id="array">
       <title>Array</title>
       <xi:include href="xml/array.xml"/>
-      <xi:include href="xml/null-array.xml"/>
-      <xi:include href="xml/boolean-array.xml"/>
-      <xi:include href="xml/int8-array.xml"/>
-      <xi:include href="xml/uint8-array.xml"/>
-      <xi:include href="xml/int16-array.xml"/>
-      <xi:include href="xml/uint16-array.xml"/>
-      <xi:include href="xml/int32-array.xml"/>
-      <xi:include href="xml/uint32-array.xml"/>
-      <xi:include href="xml/int64-array.xml"/>
-      <xi:include href="xml/uint64-array.xml"/>
-      <xi:include href="xml/float-array.xml"/>
-      <xi:include href="xml/double-array.xml"/>
-      <xi:include href="xml/binary-array.xml"/>
-      <xi:include href="xml/string-array.xml"/>
-      <xi:include href="xml/list-array.xml"/>
-      <xi:include href="xml/struct-array.xml"/>
     </chapter>
     <chapter id="array-builder">
       <title>Array builder</title>

From 6c352e2057d5f9a442c1ebf0d35c716f475fd343 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 20 Apr 2017 21:42:20 -0400
Subject: [PATCH 0545/1644] ARROW-822: [Python] StreamWriter Wrapper for Socket
 and File-like Objects without tell()

Added a wrapper for StreamWriter to implement the required tell() method so that python sockets and file-like objects can be used as sinks.  The tell() method will report the position by starting at 0 when the StreamWriter is created and incrementing by number of bytes after each write.

Added unittests that use local socket as the source/sink for streaming.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #569 from BryanCutler/pyarrow-stream-writer-socket-ARROW-822 and squashes the following commits:

6cdec4f [Bryan Cutler] Removed StreamWriter wrapper and put position handling in PyStreamWriter instead
2bd669f [Bryan Cutler] Added StreamSinkWrapper to ensure stream sink has tell() method, added unittest for StreamWriter and StreamReader over local socket
---
 cpp/src/arrow/python/io.cc       |  7 +--
 cpp/src/arrow/python/io.h        |  1 +
 python/pyarrow/tests/test_ipc.py | 79 ++++++++++++++++++++++++++++++++
 3 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index ba82a45411c4c..327e8fe9ff781 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -189,7 +189,7 @@ bool PyReadableFile::supports_zero_copy() const {
 // ----------------------------------------------------------------------
 // Output stream
 
-PyOutputStream::PyOutputStream(PyObject* file) {
+PyOutputStream::PyOutputStream(PyObject* file) : position_(0) {
   file_.reset(new PythonFile(file));
 }
 
@@ -201,12 +201,13 @@ Status PyOutputStream::Close() {
 }
 
 Status PyOutputStream::Tell(int64_t* position) {
-  PyAcquireGIL lock;
-  return file_->Tell(position);
+  *position = position_;
+  return Status::OK();
 }
 
 Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
   PyAcquireGIL lock;
+  position_ += nbytes;
   return file_->Write(data, nbytes);
 }
 
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index bf14cd6f45dbd..ebd4c5a61e938 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -82,6 +82,7 @@ class ARROW_EXPORT PyOutputStream : public io::OutputStream {
 
  private:
   std::unique_ptr<PythonFile> file_;
+  int64_t position_;
 };
 
 // A zero-copy reader backed by a PyBuffer object
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 31d418d5150ac..81213ede3151e 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -17,6 +17,8 @@
 
 import io
 import pytest
+import socket
+import threading
 
 import numpy as np
 
@@ -126,6 +128,83 @@ def test_read_all(self):
         assert result.equals(expected)
 
 
+class TestSocket(MessagingTest, unittest.TestCase):
+
+    class StreamReaderServer(threading.Thread):
+
+        def init(self, do_read_all):
+            self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            self._sock.bind(('127.0.0.1', 0))
+            self._sock.listen(1)
+            host, port = self._sock.getsockname()
+            self._do_read_all = do_read_all
+            self._schema = None
+            self._batches = []
+            self._table = None
+            return port
+
+        def run(self):
+            connection, client_address = self._sock.accept()
+            try:
+                source = connection.makefile(mode='rb')
+                reader = pa.StreamReader(source)
+                self._schema = reader.schema
+                if self._do_read_all:
+                    self._table = reader.read_all()
+                else:
+                    for i, batch in enumerate(reader):
+                        self._batches.append(batch)
+            finally:
+                connection.close()
+
+        def get_result(self):
+            return(self._schema, self._table if self._do_read_all else self._batches)
+
+    def setUp(self):
+        # NOTE: must start and stop server in test
+        pass
+
+    def start_server(self, do_read_all):
+        self._server = TestSocket.StreamReaderServer()
+        port = self._server.init(do_read_all)
+        self._server.start()
+        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self._sock.connect(('127.0.0.1', port))
+        self.sink = self._get_sink()
+
+    def stop_and_get_result(self):
+        import struct
+        self.sink.write(struct.pack('i', 0))
+        self.sink.flush()
+        self._sock.close()
+        self._server.join()
+        return self._server.get_result()
+
+    def _get_sink(self):
+        return self._sock.makefile(mode='wb')
+
+    def _get_writer(self, sink, schema):
+        return pa.StreamWriter(sink, schema)
+
+    def test_simple_roundtrip(self):
+        self.start_server(do_read_all=False)
+        writer_batches = self.write_batches()
+        reader_schema, reader_batches = self.stop_and_get_result()
+
+        assert reader_schema.equals(writer_batches[0].schema)
+        assert len(reader_batches) == len(writer_batches)
+        for i, batch in enumerate(writer_batches):
+            assert reader_batches[i].equals(batch)
+
+    def test_read_all(self):
+        self.start_server(do_read_all=True)
+        writer_batches = self.write_batches()
+        _, result = self.stop_and_get_result()
+
+        expected = pa.Table.from_batches(writer_batches)
+        assert result.equals(expected)
+
+
 class TestInMemoryFile(TestFile):
 
     def _get_sink(self):

From 6867e93cc78bffdb42b38ad9581999b567de28d6 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Fri, 21 Apr 2017 17:42:54 -0400
Subject: [PATCH 0546/1644] ARROW-869 [JS] Rename directory to js/

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #578 from TheNeuralBit/js-rename and squashes the following commits:

62244d0 [Brian Hulette] moved javascript/ to js/
---
 {javascript => js}/.gitignore               | 0
 {javascript => js}/README.md                | 0
 {javascript => js}/bin/arrow2csv.js         | 0
 {javascript => js}/bin/arrow_schema.js      | 0
 {javascript => js}/examples/read_file.html  | 0
 {javascript => js}/lib/Arrow_generated.d.ts | 0
 {javascript => js}/lib/arrow.ts             | 0
 {javascript => js}/lib/bitarray.ts          | 0
 {javascript => js}/lib/types.ts             | 0
 {javascript => js}/package.json             | 0
 {javascript => js}/postinstall.sh           | 0
 {javascript => js}/tsconfig.json            | 0
 {javascript => js}/webpack.config.js        | 0
 13 files changed, 0 insertions(+), 0 deletions(-)
 rename {javascript => js}/.gitignore (100%)
 rename {javascript => js}/README.md (100%)
 rename {javascript => js}/bin/arrow2csv.js (100%)
 rename {javascript => js}/bin/arrow_schema.js (100%)
 rename {javascript => js}/examples/read_file.html (100%)
 rename {javascript => js}/lib/Arrow_generated.d.ts (100%)
 rename {javascript => js}/lib/arrow.ts (100%)
 rename {javascript => js}/lib/bitarray.ts (100%)
 rename {javascript => js}/lib/types.ts (100%)
 rename {javascript => js}/package.json (100%)
 rename {javascript => js}/postinstall.sh (100%)
 rename {javascript => js}/tsconfig.json (100%)
 rename {javascript => js}/webpack.config.js (100%)

diff --git a/javascript/.gitignore b/js/.gitignore
similarity index 100%
rename from javascript/.gitignore
rename to js/.gitignore
diff --git a/javascript/README.md b/js/README.md
similarity index 100%
rename from javascript/README.md
rename to js/README.md
diff --git a/javascript/bin/arrow2csv.js b/js/bin/arrow2csv.js
similarity index 100%
rename from javascript/bin/arrow2csv.js
rename to js/bin/arrow2csv.js
diff --git a/javascript/bin/arrow_schema.js b/js/bin/arrow_schema.js
similarity index 100%
rename from javascript/bin/arrow_schema.js
rename to js/bin/arrow_schema.js
diff --git a/javascript/examples/read_file.html b/js/examples/read_file.html
similarity index 100%
rename from javascript/examples/read_file.html
rename to js/examples/read_file.html
diff --git a/javascript/lib/Arrow_generated.d.ts b/js/lib/Arrow_generated.d.ts
similarity index 100%
rename from javascript/lib/Arrow_generated.d.ts
rename to js/lib/Arrow_generated.d.ts
diff --git a/javascript/lib/arrow.ts b/js/lib/arrow.ts
similarity index 100%
rename from javascript/lib/arrow.ts
rename to js/lib/arrow.ts
diff --git a/javascript/lib/bitarray.ts b/js/lib/bitarray.ts
similarity index 100%
rename from javascript/lib/bitarray.ts
rename to js/lib/bitarray.ts
diff --git a/javascript/lib/types.ts b/js/lib/types.ts
similarity index 100%
rename from javascript/lib/types.ts
rename to js/lib/types.ts
diff --git a/javascript/package.json b/js/package.json
similarity index 100%
rename from javascript/package.json
rename to js/package.json
diff --git a/javascript/postinstall.sh b/js/postinstall.sh
similarity index 100%
rename from javascript/postinstall.sh
rename to js/postinstall.sh
diff --git a/javascript/tsconfig.json b/js/tsconfig.json
similarity index 100%
rename from javascript/tsconfig.json
rename to js/tsconfig.json
diff --git a/javascript/webpack.config.js b/js/webpack.config.js
similarity index 100%
rename from javascript/webpack.config.js
rename to js/webpack.config.js

From 16ea3703022304843c1eaef4a75636dbdc49e8e5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 21 Apr 2017 17:44:26 -0400
Subject: [PATCH 0547/1644] ARROW-616: [C++] Do not include debug symbols in
 release builds by default

This reduces binary size on Linux by about 80-90%. If the user wants them, they can enable with `-DARROW_CXXFLAGS="-g"`.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #574 from wesm/ARROW-616 and squashes the following commits:

71fc105 [Wes McKinney] Do not include debug symbols in release builds by default
---
 cpp/cmake_modules/SetupCxxFlags.cmake | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 7e229ff90a3e7..e2106559ba028 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -71,11 +71,12 @@ endif()
 #   Same as DEBUG, except with some optimizations on.
 # For CMAKE_BUILD_TYPE=Release
 #   -O3: Enable all compiler optimizations
-#   -g: Enable symbols for profiler tools (TODO: remove for shipping)
+#   Debug symbols are stripped for reduced binary size. Add
+#   -DARROW_CXXFLAGS="-g" to add them
 if (NOT MSVC)
   set(CXX_FLAGS_DEBUG "-ggdb -O0")
   set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
-  set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
+  set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
 endif()
 
 set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate")

From b4a75b1e17ef0356892ec9d5d210a6e156517440 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 21 Apr 2017 17:50:28 -0400
Subject: [PATCH 0548/1644] ARROW-871: [GLib] Unify DataType files

Author: Kouhei Sutou <kou@clear-code.com>

Closes #577 from kou/glib-data-type-unify-file and squashes the following commits:

188a12c [Kouhei Sutou] [GLib] Unify DataType files
---
 c_glib/arrow-glib/Makefile.am             |  34 +-
 c_glib/arrow-glib/arrow-glib.h            |  16 -
 c_glib/arrow-glib/binary-data-type.cpp    |  67 --
 c_glib/arrow-glib/binary-data-type.h      |  69 ---
 c_glib/arrow-glib/boolean-data-type.cpp   |  67 --
 c_glib/arrow-glib/boolean-data-type.h     |  69 ---
 c_glib/arrow-glib/data-type.cpp           | 599 +++++++++++++++++-
 c_glib/arrow-glib/data-type.h             | 711 ++++++++++++++++++++++
 c_glib/arrow-glib/double-data-type.cpp    |  68 ---
 c_glib/arrow-glib/double-data-type.h      |  70 ---
 c_glib/arrow-glib/float-data-type.cpp     |  68 ---
 c_glib/arrow-glib/float-data-type.h       |  69 ---
 c_glib/arrow-glib/int16-data-type.cpp     |  67 --
 c_glib/arrow-glib/int16-data-type.h       |  69 ---
 c_glib/arrow-glib/int32-data-type.cpp     |  67 --
 c_glib/arrow-glib/int32-data-type.h       |  69 ---
 c_glib/arrow-glib/int64-data-type.cpp     |  67 --
 c_glib/arrow-glib/int64-data-type.h       |  69 ---
 c_glib/arrow-glib/int8-data-type.cpp      |  67 --
 c_glib/arrow-glib/int8-data-type.h        |  69 ---
 c_glib/arrow-glib/list-data-type.cpp      |  91 ---
 c_glib/arrow-glib/list-data-type.h        |  73 ---
 c_glib/arrow-glib/null-data-type.cpp      |  67 --
 c_glib/arrow-glib/null-data-type.h        |  69 ---
 c_glib/arrow-glib/string-data-type.cpp    |  68 ---
 c_glib/arrow-glib/string-data-type.h      |  69 ---
 c_glib/arrow-glib/struct-array-builder.h  |   2 +-
 c_glib/arrow-glib/struct-data-type.cpp    |  75 ---
 c_glib/arrow-glib/struct-data-type.h      |  71 ---
 c_glib/arrow-glib/uint16-data-type.cpp    |  67 --
 c_glib/arrow-glib/uint16-data-type.h      |  69 ---
 c_glib/arrow-glib/uint32-data-type.cpp    |  67 --
 c_glib/arrow-glib/uint32-data-type.h      |  69 ---
 c_glib/arrow-glib/uint64-data-type.cpp    |  67 --
 c_glib/arrow-glib/uint64-data-type.h      |  69 ---
 c_glib/arrow-glib/uint8-data-type.cpp     |  67 --
 c_glib/arrow-glib/uint8-data-type.h       |  69 ---
 c_glib/doc/reference/arrow-glib-docs.sgml |  16 -
 38 files changed, 1294 insertions(+), 2302 deletions(-)
 delete mode 100644 c_glib/arrow-glib/binary-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/binary-data-type.h
 delete mode 100644 c_glib/arrow-glib/boolean-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/boolean-data-type.h
 delete mode 100644 c_glib/arrow-glib/double-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/double-data-type.h
 delete mode 100644 c_glib/arrow-glib/float-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/float-data-type.h
 delete mode 100644 c_glib/arrow-glib/int16-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/int16-data-type.h
 delete mode 100644 c_glib/arrow-glib/int32-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/int32-data-type.h
 delete mode 100644 c_glib/arrow-glib/int64-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/int64-data-type.h
 delete mode 100644 c_glib/arrow-glib/int8-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/int8-data-type.h
 delete mode 100644 c_glib/arrow-glib/list-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/list-data-type.h
 delete mode 100644 c_glib/arrow-glib/null-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/null-data-type.h
 delete mode 100644 c_glib/arrow-glib/string-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/string-data-type.h
 delete mode 100644 c_glib/arrow-glib/struct-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/struct-data-type.h
 delete mode 100644 c_glib/arrow-glib/uint16-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/uint16-data-type.h
 delete mode 100644 c_glib/arrow-glib/uint32-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/uint32-data-type.h
 delete mode 100644 c_glib/arrow-glib/uint64-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/uint64-data-type.h
 delete mode 100644 c_glib/arrow-glib/uint8-data-type.cpp
 delete mode 100644 c_glib/arrow-glib/uint8-data-type.h

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 570a033f4512c..d0c8c799b71cf 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -45,47 +45,31 @@ libarrow_glib_la_headers =			\
 	array-builder.h				\
 	arrow-glib.h				\
 	binary-array-builder.h			\
-	binary-data-type.h			\
 	boolean-array-builder.h			\
-	boolean-data-type.h			\
 	buffer.h				\
 	chunked-array.h				\
 	column.h				\
 	data-type.h				\
 	double-array-builder.h			\
-	double-data-type.h			\
 	error.h					\
 	field.h					\
 	float-array-builder.h			\
-	float-data-type.h			\
 	int8-array-builder.h			\
-	int8-data-type.h			\
 	int16-array-builder.h			\
-	int16-data-type.h			\
 	int32-array-builder.h			\
-	int32-data-type.h			\
 	int64-array-builder.h			\
-	int64-data-type.h			\
 	list-array-builder.h			\
-	list-data-type.h			\
-	null-data-type.h			\
 	record-batch.h				\
 	schema.h				\
 	string-array-builder.h			\
-	string-data-type.h			\
 	struct-array-builder.h			\
-	struct-data-type.h			\
 	table.h					\
 	tensor.h				\
 	type.h					\
 	uint8-array-builder.h			\
-	uint8-data-type.h			\
 	uint16-array-builder.h			\
-	uint16-data-type.h			\
 	uint32-array-builder.h			\
-	uint32-data-type.h			\
-	uint64-array-builder.h			\
-	uint64-data-type.h
+	uint64-array-builder.h
 
 libarrow_glib_la_headers +=			\
 	file.h					\
@@ -117,47 +101,31 @@ libarrow_glib_la_sources =			\
 	array.cpp				\
 	array-builder.cpp			\
 	binary-array-builder.cpp		\
-	binary-data-type.cpp			\
 	boolean-array-builder.cpp		\
-	boolean-data-type.cpp			\
 	buffer.cpp				\
 	chunked-array.cpp			\
 	column.cpp				\
 	data-type.cpp				\
 	double-array-builder.cpp		\
-	double-data-type.cpp			\
 	error.cpp				\
 	field.cpp				\
 	float-array-builder.cpp			\
-	float-data-type.cpp			\
 	int8-array-builder.cpp			\
-	int8-data-type.cpp			\
 	int16-array-builder.cpp			\
-	int16-data-type.cpp			\
 	int32-array-builder.cpp			\
-	int32-data-type.cpp			\
 	int64-array-builder.cpp			\
-	int64-data-type.cpp			\
 	list-array-builder.cpp			\
-	list-data-type.cpp			\
-	null-data-type.cpp			\
 	record-batch.cpp			\
 	schema.cpp				\
 	string-array-builder.cpp		\
-	string-data-type.cpp			\
 	struct-array-builder.cpp		\
-	struct-data-type.cpp			\
 	table.cpp				\
 	tensor.cpp				\
 	type.cpp				\
 	uint8-array-builder.cpp			\
-	uint8-data-type.cpp			\
 	uint16-array-builder.cpp		\
-	uint16-data-type.cpp			\
 	uint32-array-builder.cpp		\
-	uint32-data-type.cpp			\
 	uint64-array-builder.cpp		\
-	uint64-data-type.cpp			\
 	$(libarrow_glib_la_headers)		\
 	$(libarrow_glib_la_generated_sources)
 
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index ee408cde3615e..46e98d2b8ed4c 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -22,47 +22,31 @@
 #include <arrow-glib/array.h>
 #include <arrow-glib/array-builder.h>
 #include <arrow-glib/binary-array-builder.h>
-#include <arrow-glib/binary-data-type.h>
 #include <arrow-glib/boolean-array-builder.h>
-#include <arrow-glib/boolean-data-type.h>
 #include <arrow-glib/chunked-array.h>
 #include <arrow-glib/column.h>
 #include <arrow-glib/data-type.h>
 #include <arrow-glib/double-array-builder.h>
-#include <arrow-glib/double-data-type.h>
 #include <arrow-glib/enums.h>
 #include <arrow-glib/error.h>
 #include <arrow-glib/field.h>
 #include <arrow-glib/float-array-builder.h>
-#include <arrow-glib/float-data-type.h>
 #include <arrow-glib/int8-array-builder.h>
-#include <arrow-glib/int8-data-type.h>
 #include <arrow-glib/int16-array-builder.h>
-#include <arrow-glib/int16-data-type.h>
 #include <arrow-glib/int32-array-builder.h>
-#include <arrow-glib/int32-data-type.h>
 #include <arrow-glib/int64-array-builder.h>
-#include <arrow-glib/int64-data-type.h>
 #include <arrow-glib/list-array-builder.h>
-#include <arrow-glib/list-data-type.h>
-#include <arrow-glib/null-data-type.h>
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
 #include <arrow-glib/string-array-builder.h>
-#include <arrow-glib/string-data-type.h>
 #include <arrow-glib/struct-array-builder.h>
-#include <arrow-glib/struct-data-type.h>
 #include <arrow-glib/table.h>
 #include <arrow-glib/tensor.h>
 #include <arrow-glib/type.h>
 #include <arrow-glib/uint8-array-builder.h>
-#include <arrow-glib/uint8-data-type.h>
 #include <arrow-glib/uint16-array-builder.h>
-#include <arrow-glib/uint16-data-type.h>
 #include <arrow-glib/uint32-array-builder.h>
-#include <arrow-glib/uint32-data-type.h>
 #include <arrow-glib/uint64-array-builder.h>
-#include <arrow-glib/uint64-data-type.h>
 
 #include <arrow-glib/file.h>
 #include <arrow-glib/file-mode.h>
diff --git a/c_glib/arrow-glib/binary-data-type.cpp b/c_glib/arrow-glib/binary-data-type.cpp
deleted file mode 100644
index e5187f7d94efe..0000000000000
--- a/c_glib/arrow-glib/binary-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/binary-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: binary-data-type
- * @short_description: Binary data type
- *
- * #GArrowBinaryDataType is a class for binary data type.
- */
-
-G_DEFINE_TYPE(GArrowBinaryDataType,                \
-              garrow_binary_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_binary_data_type_init(GArrowBinaryDataType *object)
-{
-}
-
-static void
-garrow_binary_data_type_class_init(GArrowBinaryDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_binary_data_type_new:
- *
- * Returns: The newly created binary data type.
- */
-GArrowBinaryDataType *
-garrow_binary_data_type_new(void)
-{
-  auto arrow_data_type = arrow::binary();
-
-  GArrowBinaryDataType *data_type =
-    GARROW_BINARY_DATA_TYPE(g_object_new(GARROW_TYPE_BINARY_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/binary-data-type.h b/c_glib/arrow-glib/binary-data-type.h
deleted file mode 100644
index 9654fe216376e..0000000000000
--- a/c_glib/arrow-glib/binary-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_BINARY_DATA_TYPE            \
-  (garrow_binary_data_type_get_type())
-#define GARROW_BINARY_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BINARY_DATA_TYPE,     \
-                              GArrowBinaryDataType))
-#define GARROW_BINARY_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BINARY_DATA_TYPE,        \
-                           GArrowBinaryDataTypeClass))
-#define GARROW_IS_BINARY_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BINARY_DATA_TYPE))
-#define GARROW_IS_BINARY_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BINARY_DATA_TYPE))
-#define GARROW_BINARY_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BINARY_DATA_TYPE,      \
-                             GArrowBinaryDataTypeClass))
-
-typedef struct _GArrowBinaryDataType         GArrowBinaryDataType;
-typedef struct _GArrowBinaryDataTypeClass    GArrowBinaryDataTypeClass;
-
-/**
- * GArrowBinaryDataType:
- *
- * It wraps `arrow::BinaryType`.
- */
-struct _GArrowBinaryDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowBinaryDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_binary_data_type_get_type (void) G_GNUC_CONST;
-GArrowBinaryDataType *garrow_binary_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-data-type.cpp b/c_glib/arrow-glib/boolean-data-type.cpp
deleted file mode 100644
index 99c73d9ff8873..0000000000000
--- a/c_glib/arrow-glib/boolean-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/boolean-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: boolean-data-type
- * @short_description: Boolean data type
- *
- * #GArrowBooleanDataType is a class for boolean data type.
- */
-
-G_DEFINE_TYPE(GArrowBooleanDataType,                \
-              garrow_boolean_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_boolean_data_type_init(GArrowBooleanDataType *object)
-{
-}
-
-static void
-garrow_boolean_data_type_class_init(GArrowBooleanDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_boolean_data_type_new:
- *
- * Returns: The newly created boolean data type.
- */
-GArrowBooleanDataType *
-garrow_boolean_data_type_new(void)
-{
-  auto arrow_data_type = arrow::boolean();
-
-  GArrowBooleanDataType *data_type =
-    GARROW_BOOLEAN_DATA_TYPE(g_object_new(GARROW_TYPE_BOOLEAN_DATA_TYPE,
-                                          "data-type", &arrow_data_type,
-                                          NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-data-type.h b/c_glib/arrow-glib/boolean-data-type.h
deleted file mode 100644
index ad30c99960a8e..0000000000000
--- a/c_glib/arrow-glib/boolean-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_BOOLEAN_DATA_TYPE           \
-  (garrow_boolean_data_type_get_type())
-#define GARROW_BOOLEAN_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BOOLEAN_DATA_TYPE,    \
-                              GArrowBooleanDataType))
-#define GARROW_BOOLEAN_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BOOLEAN_DATA_TYPE,       \
-                           GArrowBooleanDataTypeClass))
-#define GARROW_IS_BOOLEAN_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BOOLEAN_DATA_TYPE))
-#define GARROW_IS_BOOLEAN_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BOOLEAN_DATA_TYPE))
-#define GARROW_BOOLEAN_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BOOLEAN_DATA_TYPE,     \
-                             GArrowBooleanDataTypeClass))
-
-typedef struct _GArrowBooleanDataType         GArrowBooleanDataType;
-typedef struct _GArrowBooleanDataTypeClass    GArrowBooleanDataTypeClass;
-
-/**
- * GArrowBooleanDataType:
- *
- * It wraps `arrow::BooleanType`.
- */
-struct _GArrowBooleanDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowBooleanDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                  garrow_boolean_data_type_get_type (void) G_GNUC_CONST;
-GArrowBooleanDataType *garrow_boolean_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/data-type.cpp
index 12932a16269e8..2fd261dc91947 100644
--- a/c_glib/arrow-glib/data-type.cpp
+++ b/c_glib/arrow-glib/data-type.cpp
@@ -21,34 +21,55 @@
 #  include <config.h>
 #endif
 
-#include <arrow-glib/boolean-data-type.h>
-#include <arrow-glib/binary-data-type.h>
-#include <arrow-glib/boolean-data-type.h>
 #include <arrow-glib/data-type.hpp>
-#include <arrow-glib/double-data-type.h>
-#include <arrow-glib/float-data-type.h>
-#include <arrow-glib/int8-data-type.h>
-#include <arrow-glib/int16-data-type.h>
-#include <arrow-glib/int32-data-type.h>
-#include <arrow-glib/int64-data-type.h>
-#include <arrow-glib/list-data-type.h>
-#include <arrow-glib/null-data-type.h>
-#include <arrow-glib/string-data-type.h>
-#include <arrow-glib/struct-data-type.h>
+#include <arrow-glib/field.hpp>
 #include <arrow-glib/type.hpp>
-#include <arrow-glib/uint8-data-type.h>
-#include <arrow-glib/uint16-data-type.h>
-#include <arrow-glib/uint32-data-type.h>
-#include <arrow-glib/uint64-data-type.h>
 
 G_BEGIN_DECLS
 
 /**
  * SECTION: data-type
- * @short_description: Base class for all data type classes
+ * @section_id: data-type-classes
+ * @title: Data type classes
+ * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowDataType is a base class for all data type classes such as
  * #GArrowBooleanDataType.
+ *
+ * #GArrowNullDataType is a class for null data type.
+ *
+ * #GArrowBooleanDataType is a class for boolean data type.
+ *
+ * #GArrowInt8DataType is a class for 8-bit integer data type.
+ *
+ * #GArrowUInt8DataType is a class for 8-bit unsigned integer data type.
+ *
+ * #GArrowInt16DataType is a class for 16-bit integer data type.
+ *
+ * #GArrowUInt16DataType is a class for 16-bit unsigned integer data type.
+ *
+ * #GArrowInt32DataType is a class for 32-bit integer data type.
+ *
+ * #GArrowUInt32DataType is a class for 32-bit unsigned integer data type.
+ *
+ * #GArrowInt64DataType is a class for 64-bit integer data type.
+ *
+ * #GArrowUInt64DataType is a class for 64-bit unsigned integer data type.
+ *
+ * #GArrowFloatDataType is a class for 32-bit floating point data
+ * type.
+ *
+ * #GArrowDoubleDataType is a class for 64-bit floating point data
+ * type.
+ *
+ * #GArrowBinaryDataType is a class for binary data type.
+ *
+ * #GArrowStringDataType is a class for UTF-8 encoded string data
+ * type.
+ *
+ * #GArrowListDataType is a class for list data type.
+ *
+ * #GArrowStructDataType is a class for struct data type.
  */
 
 typedef struct GArrowDataTypePrivate_ {
@@ -183,6 +204,548 @@ garrow_data_type_type(GArrowDataType *data_type)
   return garrow_type_from_raw(arrow_data_type->id());
 }
 
+
+G_DEFINE_TYPE(GArrowNullDataType,                \
+              garrow_null_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_null_data_type_init(GArrowNullDataType *object)
+{
+}
+
+static void
+garrow_null_data_type_class_init(GArrowNullDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_null_data_type_new:
+ *
+ * Returns: The newly created null data type.
+ */
+GArrowNullDataType *
+garrow_null_data_type_new(void)
+{
+  auto arrow_data_type = arrow::null();
+
+  GArrowNullDataType *data_type =
+    GARROW_NULL_DATA_TYPE(g_object_new(GARROW_TYPE_NULL_DATA_TYPE,
+                                       "data-type", &arrow_data_type,
+                                       NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowBooleanDataType,                \
+              garrow_boolean_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_boolean_data_type_init(GArrowBooleanDataType *object)
+{
+}
+
+static void
+garrow_boolean_data_type_class_init(GArrowBooleanDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_boolean_data_type_new:
+ *
+ * Returns: The newly created boolean data type.
+ */
+GArrowBooleanDataType *
+garrow_boolean_data_type_new(void)
+{
+  auto arrow_data_type = arrow::boolean();
+
+  GArrowBooleanDataType *data_type =
+    GARROW_BOOLEAN_DATA_TYPE(g_object_new(GARROW_TYPE_BOOLEAN_DATA_TYPE,
+                                          "data-type", &arrow_data_type,
+                                          NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowInt8DataType,                \
+              garrow_int8_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_int8_data_type_init(GArrowInt8DataType *object)
+{
+}
+
+static void
+garrow_int8_data_type_class_init(GArrowInt8DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_int8_data_type_new:
+ *
+ * Returns: The newly created 8-bit integer data type.
+ */
+GArrowInt8DataType *
+garrow_int8_data_type_new(void)
+{
+  auto arrow_data_type = arrow::int8();
+
+  GArrowInt8DataType *data_type =
+    GARROW_INT8_DATA_TYPE(g_object_new(GARROW_TYPE_INT8_DATA_TYPE,
+                                       "data-type", &arrow_data_type,
+                                       NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowUInt8DataType,                \
+              garrow_uint8_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_uint8_data_type_init(GArrowUInt8DataType *object)
+{
+}
+
+static void
+garrow_uint8_data_type_class_init(GArrowUInt8DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_uint8_data_type_new:
+ *
+ * Returns: The newly created 8-bit unsigned integer data type.
+ */
+GArrowUInt8DataType *
+garrow_uint8_data_type_new(void)
+{
+  auto arrow_data_type = arrow::uint8();
+
+  GArrowUInt8DataType *data_type =
+    GARROW_UINT8_DATA_TYPE(g_object_new(GARROW_TYPE_UINT8_DATA_TYPE,
+                                        "data-type", &arrow_data_type,
+                                        NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowInt16DataType,                \
+              garrow_int16_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_int16_data_type_init(GArrowInt16DataType *object)
+{
+}
+
+static void
+garrow_int16_data_type_class_init(GArrowInt16DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_int16_data_type_new:
+ *
+ * Returns: The newly created 16-bit integer data type.
+ */
+GArrowInt16DataType *
+garrow_int16_data_type_new(void)
+{
+  auto arrow_data_type = arrow::int16();
+
+  GArrowInt16DataType *data_type =
+    GARROW_INT16_DATA_TYPE(g_object_new(GARROW_TYPE_INT16_DATA_TYPE,
+                                        "data-type", &arrow_data_type,
+                                        NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowUInt16DataType,                \
+              garrow_uint16_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_uint16_data_type_init(GArrowUInt16DataType *object)
+{
+}
+
+static void
+garrow_uint16_data_type_class_init(GArrowUInt16DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_uint16_data_type_new:
+ *
+ * Returns: The newly created 16-bit unsigned integer data type.
+ */
+GArrowUInt16DataType *
+garrow_uint16_data_type_new(void)
+{
+  auto arrow_data_type = arrow::uint16();
+
+  GArrowUInt16DataType *data_type =
+    GARROW_UINT16_DATA_TYPE(g_object_new(GARROW_TYPE_UINT16_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowInt32DataType,                \
+              garrow_int32_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_int32_data_type_init(GArrowInt32DataType *object)
+{
+}
+
+static void
+garrow_int32_data_type_class_init(GArrowInt32DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_int32_data_type_new:
+ *
+ * Returns: The newly created 32-bit integer data type.
+ */
+GArrowInt32DataType *
+garrow_int32_data_type_new(void)
+{
+  auto arrow_data_type = arrow::int32();
+
+  GArrowInt32DataType *data_type =
+    GARROW_INT32_DATA_TYPE(g_object_new(GARROW_TYPE_INT32_DATA_TYPE,
+                                        "data-type", &arrow_data_type,
+                                        NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowUInt32DataType,                \
+              garrow_uint32_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_uint32_data_type_init(GArrowUInt32DataType *object)
+{
+}
+
+static void
+garrow_uint32_data_type_class_init(GArrowUInt32DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_uint32_data_type_new:
+ *
+ * Returns: The newly created 32-bit unsigned integer data type.
+ */
+GArrowUInt32DataType *
+garrow_uint32_data_type_new(void)
+{
+  auto arrow_data_type = arrow::uint32();
+
+  GArrowUInt32DataType *data_type =
+    GARROW_UINT32_DATA_TYPE(g_object_new(GARROW_TYPE_UINT32_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowInt64DataType,                \
+              garrow_int64_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_int64_data_type_init(GArrowInt64DataType *object)
+{
+}
+
+static void
+garrow_int64_data_type_class_init(GArrowInt64DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_int64_data_type_new:
+ *
+ * Returns: The newly created 64-bit integer data type.
+ */
+GArrowInt64DataType *
+garrow_int64_data_type_new(void)
+{
+  auto arrow_data_type = arrow::int64();
+
+  GArrowInt64DataType *data_type =
+    GARROW_INT64_DATA_TYPE(g_object_new(GARROW_TYPE_INT64_DATA_TYPE,
+                                        "data-type", &arrow_data_type,
+                                        NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowUInt64DataType,                \
+              garrow_uint64_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_uint64_data_type_init(GArrowUInt64DataType *object)
+{
+}
+
+static void
+garrow_uint64_data_type_class_init(GArrowUInt64DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_uint64_data_type_new:
+ *
+ * Returns: The newly created 64-bit unsigned integer data type.
+ */
+GArrowUInt64DataType *
+garrow_uint64_data_type_new(void)
+{
+  auto arrow_data_type = arrow::uint64();
+
+  GArrowUInt64DataType *data_type =
+    GARROW_UINT64_DATA_TYPE(g_object_new(GARROW_TYPE_UINT64_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowFloatDataType,                \
+              garrow_float_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_float_data_type_init(GArrowFloatDataType *object)
+{
+}
+
+static void
+garrow_float_data_type_class_init(GArrowFloatDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_float_data_type_new:
+ *
+ * Returns: The newly created float data type.
+ */
+GArrowFloatDataType *
+garrow_float_data_type_new(void)
+{
+  auto arrow_data_type = arrow::float32();
+
+  GArrowFloatDataType *data_type =
+    GARROW_FLOAT_DATA_TYPE(g_object_new(GARROW_TYPE_FLOAT_DATA_TYPE,
+                                        "data-type", &arrow_data_type,
+                                        NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowDoubleDataType,                \
+              garrow_double_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_double_data_type_init(GArrowDoubleDataType *object)
+{
+}
+
+static void
+garrow_double_data_type_class_init(GArrowDoubleDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_double_data_type_new:
+ *
+ * Returns: The newly created 64-bit floating point data type.
+ */
+GArrowDoubleDataType *
+garrow_double_data_type_new(void)
+{
+  auto arrow_data_type = arrow::float64();
+
+  GArrowDoubleDataType *data_type =
+    GARROW_DOUBLE_DATA_TYPE(g_object_new(GARROW_TYPE_DOUBLE_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowBinaryDataType,                \
+              garrow_binary_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_binary_data_type_init(GArrowBinaryDataType *object)
+{
+}
+
+static void
+garrow_binary_data_type_class_init(GArrowBinaryDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_binary_data_type_new:
+ *
+ * Returns: The newly created binary data type.
+ */
+GArrowBinaryDataType *
+garrow_binary_data_type_new(void)
+{
+  auto arrow_data_type = arrow::binary();
+
+  GArrowBinaryDataType *data_type =
+    GARROW_BINARY_DATA_TYPE(g_object_new(GARROW_TYPE_BINARY_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowStringDataType,                \
+              garrow_string_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_string_data_type_init(GArrowStringDataType *object)
+{
+}
+
+static void
+garrow_string_data_type_class_init(GArrowStringDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_string_data_type_new:
+ *
+ * Returns: The newly created UTF-8 encoded string data type.
+ */
+GArrowStringDataType *
+garrow_string_data_type_new(void)
+{
+  auto arrow_data_type = arrow::utf8();
+
+  GArrowStringDataType *data_type =
+    GARROW_STRING_DATA_TYPE(g_object_new(GARROW_TYPE_STRING_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowListDataType,                \
+              garrow_list_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_list_data_type_init(GArrowListDataType *object)
+{
+}
+
+static void
+garrow_list_data_type_class_init(GArrowListDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_list_data_type_new:
+ * @field: The field of elements
+ *
+ * Returns: The newly created list data type.
+ */
+GArrowListDataType *
+garrow_list_data_type_new(GArrowField *field)
+{
+  auto arrow_field = garrow_field_get_raw(field);
+  auto arrow_data_type =
+    std::make_shared<arrow::ListType>(arrow_field);
+
+  GArrowListDataType *data_type =
+    GARROW_LIST_DATA_TYPE(g_object_new(GARROW_TYPE_LIST_DATA_TYPE,
+                                       "data-type", &arrow_data_type,
+                                       NULL));
+  return data_type;
+}
+
+/**
+ * garrow_list_data_type_get_value_field:
+ * @list_data_type: A #GArrowListDataType.
+ *
+ * Returns: (transfer full): The field of value.
+ */
+GArrowField *
+garrow_list_data_type_get_value_field(GArrowListDataType *list_data_type)
+{
+  auto arrow_data_type =
+    garrow_data_type_get_raw(GARROW_DATA_TYPE(list_data_type));
+  auto arrow_list_data_type =
+    static_cast<arrow::ListType *>(arrow_data_type.get());
+
+  auto arrow_field = arrow_list_data_type->value_field();
+  auto field = garrow_field_new_raw(&arrow_field);
+
+  return field;
+}
+
+
+G_DEFINE_TYPE(GArrowStructDataType,                \
+              garrow_struct_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_struct_data_type_init(GArrowStructDataType *object)
+{
+}
+
+static void
+garrow_struct_data_type_class_init(GArrowStructDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_struct_data_type_new:
+ * @fields: (element-type GArrowField): The fields of the struct.
+ *
+ * Returns: The newly created struct data type.
+ */
+GArrowStructDataType *
+garrow_struct_data_type_new(GList *fields)
+{
+  std::vector<std::shared_ptr<arrow::Field>> arrow_fields;
+  for (GList *node = fields; node; node = g_list_next(node)) {
+    auto field = GARROW_FIELD(node->data);
+    auto arrow_field = garrow_field_get_raw(field);
+    arrow_fields.push_back(arrow_field);
+  }
+
+  auto arrow_data_type = std::make_shared<arrow::StructType>(arrow_fields);
+  GArrowStructDataType *data_type =
+    GARROW_STRUCT_DATA_TYPE(g_object_new(GARROW_TYPE_STRUCT_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
 G_END_DECLS
 
 GArrowDataType *
diff --git a/c_glib/arrow-glib/data-type.h b/c_glib/arrow-glib/data-type.h
index 3203d09b5c651..babf0ee1712a0 100644
--- a/c_glib/arrow-glib/data-type.h
+++ b/c_glib/arrow-glib/data-type.h
@@ -19,10 +19,16 @@
 
 #pragma once
 
+#include <glib-object.h>
+
 #include <arrow-glib/type.h>
 
 G_BEGIN_DECLS
 
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowField         GArrowField;
+#endif
+
 #define GARROW_TYPE_DATA_TYPE                   \
   (garrow_data_type_get_type())
 #define GARROW_DATA_TYPE(obj)                           \
@@ -69,4 +75,709 @@ gboolean   garrow_data_type_equal     (GArrowDataType *data_type,
 gchar     *garrow_data_type_to_string (GArrowDataType *data_type);
 GArrowType garrow_data_type_type      (GArrowDataType *data_type);
 
+
+#define GARROW_TYPE_NULL_DATA_TYPE              \
+  (garrow_null_data_type_get_type())
+#define GARROW_NULL_DATA_TYPE(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_NULL_DATA_TYPE,       \
+                              GArrowNullDataType))
+#define GARROW_NULL_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_NULL_DATA_TYPE,  \
+                           GArrowNullDataTypeClass))
+#define GARROW_IS_NULL_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_NULL_DATA_TYPE))
+#define GARROW_IS_NULL_DATA_TYPE_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_NULL_DATA_TYPE))
+#define GARROW_NULL_DATA_TYPE_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_NULL_DATA_TYPE,        \
+                             GArrowNullDataTypeClass))
+
+typedef struct _GArrowNullDataType         GArrowNullDataType;
+typedef struct _GArrowNullDataTypeClass    GArrowNullDataTypeClass;
+
+/**
+ * GArrowNullDataType:
+ *
+ * It wraps `arrow::NullType`.
+ */
+struct _GArrowNullDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowNullDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType               garrow_null_data_type_get_type (void) G_GNUC_CONST;
+GArrowNullDataType *garrow_null_data_type_new      (void);
+
+
+#define GARROW_TYPE_BOOLEAN_DATA_TYPE           \
+  (garrow_boolean_data_type_get_type())
+#define GARROW_BOOLEAN_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BOOLEAN_DATA_TYPE,    \
+                              GArrowBooleanDataType))
+#define GARROW_BOOLEAN_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BOOLEAN_DATA_TYPE,       \
+                           GArrowBooleanDataTypeClass))
+#define GARROW_IS_BOOLEAN_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BOOLEAN_DATA_TYPE))
+#define GARROW_IS_BOOLEAN_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BOOLEAN_DATA_TYPE))
+#define GARROW_BOOLEAN_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BOOLEAN_DATA_TYPE,     \
+                             GArrowBooleanDataTypeClass))
+
+typedef struct _GArrowBooleanDataType         GArrowBooleanDataType;
+typedef struct _GArrowBooleanDataTypeClass    GArrowBooleanDataTypeClass;
+
+/**
+ * GArrowBooleanDataType:
+ *
+ * It wraps `arrow::BooleanType`.
+ */
+struct _GArrowBooleanDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowBooleanDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                  garrow_boolean_data_type_get_type (void) G_GNUC_CONST;
+GArrowBooleanDataType *garrow_boolean_data_type_new      (void);
+
+
+#define GARROW_TYPE_INT8_DATA_TYPE            \
+  (garrow_int8_data_type_get_type())
+#define GARROW_INT8_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
+                              GARROW_TYPE_INT8_DATA_TYPE,     \
+                              GArrowInt8DataType))
+#define GARROW_INT8_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
+                           GARROW_TYPE_INT8_DATA_TYPE,        \
+                           GArrowInt8DataTypeClass))
+#define GARROW_IS_INT8_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
+                              GARROW_TYPE_INT8_DATA_TYPE))
+#define GARROW_IS_INT8_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
+                           GARROW_TYPE_INT8_DATA_TYPE))
+#define GARROW_INT8_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
+                             GARROW_TYPE_INT8_DATA_TYPE,      \
+                             GArrowInt8DataTypeClass))
+
+typedef struct _GArrowInt8DataType         GArrowInt8DataType;
+typedef struct _GArrowInt8DataTypeClass    GArrowInt8DataTypeClass;
+
+/**
+ * GArrowInt8DataType:
+ *
+ * It wraps `arrow::Int8Type`.
+ */
+struct _GArrowInt8DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt8DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int8_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt8DataType   *garrow_int8_data_type_new      (void);
+
+
+#define GARROW_TYPE_UINT8_DATA_TYPE            \
+  (garrow_uint8_data_type_get_type())
+#define GARROW_UINT8_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
+                              GARROW_TYPE_UINT8_DATA_TYPE,     \
+                              GArrowUInt8DataType))
+#define GARROW_UINT8_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
+                           GARROW_TYPE_UINT8_DATA_TYPE,        \
+                           GArrowUInt8DataTypeClass))
+#define GARROW_IS_UINT8_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
+                              GARROW_TYPE_UINT8_DATA_TYPE))
+#define GARROW_IS_UINT8_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
+                           GARROW_TYPE_UINT8_DATA_TYPE))
+#define GARROW_UINT8_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
+                             GARROW_TYPE_UINT8_DATA_TYPE,      \
+                             GArrowUInt8DataTypeClass))
+
+typedef struct _GArrowUInt8DataType         GArrowUInt8DataType;
+typedef struct _GArrowUInt8DataTypeClass    GArrowUInt8DataTypeClass;
+
+/**
+ * GArrowUInt8DataType:
+ *
+ * It wraps `arrow::UInt8Type`.
+ */
+struct _GArrowUInt8DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt8DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint8_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt8DataType  *garrow_uint8_data_type_new      (void);
+
+
+#define GARROW_TYPE_INT16_DATA_TYPE            \
+  (garrow_int16_data_type_get_type())
+#define GARROW_INT16_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
+                              GARROW_TYPE_INT16_DATA_TYPE,     \
+                              GArrowInt16DataType))
+#define GARROW_INT16_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
+                           GARROW_TYPE_INT16_DATA_TYPE,        \
+                           GArrowInt16DataTypeClass))
+#define GARROW_IS_INT16_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
+                              GARROW_TYPE_INT16_DATA_TYPE))
+#define GARROW_IS_INT16_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
+                           GARROW_TYPE_INT16_DATA_TYPE))
+#define GARROW_INT16_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
+                             GARROW_TYPE_INT16_DATA_TYPE,      \
+                             GArrowInt16DataTypeClass))
+
+typedef struct _GArrowInt16DataType         GArrowInt16DataType;
+typedef struct _GArrowInt16DataTypeClass    GArrowInt16DataTypeClass;
+
+/**
+ * GArrowInt16DataType:
+ *
+ * It wraps `arrow::Int16Type`.
+ */
+struct _GArrowInt16DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt16DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int16_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt16DataType  *garrow_int16_data_type_new      (void);
+
+
+#define GARROW_TYPE_UINT16_DATA_TYPE            \
+  (garrow_uint16_data_type_get_type())
+#define GARROW_UINT16_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT16_DATA_TYPE,     \
+                              GArrowUInt16DataType))
+#define GARROW_UINT16_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT16_DATA_TYPE,        \
+                           GArrowUInt16DataTypeClass))
+#define GARROW_IS_UINT16_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT16_DATA_TYPE))
+#define GARROW_IS_UINT16_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT16_DATA_TYPE))
+#define GARROW_UINT16_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT16_DATA_TYPE,      \
+                             GArrowUInt16DataTypeClass))
+
+typedef struct _GArrowUInt16DataType         GArrowUInt16DataType;
+typedef struct _GArrowUInt16DataTypeClass    GArrowUInt16DataTypeClass;
+
+/**
+ * GArrowUInt16DataType:
+ *
+ * It wraps `arrow::UInt16Type`.
+ */
+struct _GArrowUInt16DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt16DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint16_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt16DataType *garrow_uint16_data_type_new      (void);
+
+
+#define GARROW_TYPE_INT32_DATA_TYPE            \
+  (garrow_int32_data_type_get_type())
+#define GARROW_INT32_DATA_TYPE(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT32_DATA_TYPE,      \
+                              GArrowInt32DataType))
+#define GARROW_INT32_DATA_TYPE_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT32_DATA_TYPE, \
+                           GArrowInt32DataTypeClass))
+#define GARROW_IS_INT32_DATA_TYPE(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT32_DATA_TYPE))
+#define GARROW_IS_INT32_DATA_TYPE_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT32_DATA_TYPE))
+#define GARROW_INT32_DATA_TYPE_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT32_DATA_TYPE,       \
+                             GArrowInt32DataTypeClass))
+
+typedef struct _GArrowInt32DataType         GArrowInt32DataType;
+typedef struct _GArrowInt32DataTypeClass    GArrowInt32DataTypeClass;
+
+/**
+ * GArrowInt32DataType:
+ *
+ * It wraps `arrow::Int32Type`.
+ */
+struct _GArrowInt32DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt32DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int32_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt32DataType  *garrow_int32_data_type_new      (void);
+
+
+#define GARROW_TYPE_UINT32_DATA_TYPE            \
+  (garrow_uint32_data_type_get_type())
+#define GARROW_UINT32_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT32_DATA_TYPE,     \
+                              GArrowUInt32DataType))
+#define GARROW_UINT32_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT32_DATA_TYPE,        \
+                           GArrowUInt32DataTypeClass))
+#define GARROW_IS_UINT32_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT32_DATA_TYPE))
+#define GARROW_IS_UINT32_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT32_DATA_TYPE))
+#define GARROW_UINT32_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT32_DATA_TYPE,      \
+                             GArrowUInt32DataTypeClass))
+
+typedef struct _GArrowUInt32DataType         GArrowUInt32DataType;
+typedef struct _GArrowUInt32DataTypeClass    GArrowUInt32DataTypeClass;
+
+/**
+ * GArrowUInt32DataType:
+ *
+ * It wraps `arrow::UInt32Type`.
+ */
+struct _GArrowUInt32DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt32DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint32_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt32DataType *garrow_uint32_data_type_new      (void);
+
+
+#define GARROW_TYPE_INT64_DATA_TYPE            \
+  (garrow_int64_data_type_get_type())
+#define GARROW_INT64_DATA_TYPE(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT64_DATA_TYPE,      \
+                              GArrowInt64DataType))
+#define GARROW_INT64_DATA_TYPE_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT64_DATA_TYPE, \
+                           GArrowInt64DataTypeClass))
+#define GARROW_IS_INT64_DATA_TYPE(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT64_DATA_TYPE))
+#define GARROW_IS_INT64_DATA_TYPE_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT64_DATA_TYPE))
+#define GARROW_INT64_DATA_TYPE_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT64_DATA_TYPE,       \
+                             GArrowInt64DataTypeClass))
+
+typedef struct _GArrowInt64DataType         GArrowInt64DataType;
+typedef struct _GArrowInt64DataTypeClass    GArrowInt64DataTypeClass;
+
+/**
+ * GArrowInt64DataType:
+ *
+ * It wraps `arrow::Int64Type`.
+ */
+struct _GArrowInt64DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt64DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int64_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt64DataType  *garrow_int64_data_type_new      (void);
+
+
+#define GARROW_TYPE_UINT64_DATA_TYPE            \
+  (garrow_uint64_data_type_get_type())
+#define GARROW_UINT64_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT64_DATA_TYPE,     \
+                              GArrowUInt64DataType))
+#define GARROW_UINT64_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT64_DATA_TYPE,        \
+                           GArrowUInt64DataTypeClass))
+#define GARROW_IS_UINT64_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT64_DATA_TYPE))
+#define GARROW_IS_UINT64_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT64_DATA_TYPE))
+#define GARROW_UINT64_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT64_DATA_TYPE,      \
+                             GArrowUInt64DataTypeClass))
+
+typedef struct _GArrowUInt64DataType         GArrowUInt64DataType;
+typedef struct _GArrowUInt64DataTypeClass    GArrowUInt64DataTypeClass;
+
+/**
+ * GArrowUInt64DataType:
+ *
+ * It wraps `arrow::UInt64Type`.
+ */
+struct _GArrowUInt64DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt64DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint64_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt64DataType *garrow_uint64_data_type_new      (void);
+
+
+#define GARROW_TYPE_FLOAT_DATA_TYPE           \
+  (garrow_float_data_type_get_type())
+#define GARROW_FLOAT_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
+                              GARROW_TYPE_FLOAT_DATA_TYPE,    \
+                              GArrowFloatDataType))
+#define GARROW_FLOAT_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
+                           GARROW_TYPE_FLOAT_DATA_TYPE,       \
+                           GArrowFloatDataTypeClass))
+#define GARROW_IS_FLOAT_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
+                              GARROW_TYPE_FLOAT_DATA_TYPE))
+#define GARROW_IS_FLOAT_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
+                           GARROW_TYPE_FLOAT_DATA_TYPE))
+#define GARROW_FLOAT_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
+                             GARROW_TYPE_FLOAT_DATA_TYPE,     \
+                             GArrowFloatDataTypeClass))
+
+typedef struct _GArrowFloatDataType         GArrowFloatDataType;
+typedef struct _GArrowFloatDataTypeClass    GArrowFloatDataTypeClass;
+
+/**
+ * GArrowFloatDataType:
+ *
+ * It wraps `arrow::FloatType`.
+ */
+struct _GArrowFloatDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowFloatDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                garrow_float_data_type_get_type (void) G_GNUC_CONST;
+GArrowFloatDataType *garrow_float_data_type_new      (void);
+
+
+#define GARROW_TYPE_DOUBLE_DATA_TYPE           \
+  (garrow_double_data_type_get_type())
+#define GARROW_DOUBLE_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DOUBLE_DATA_TYPE,     \
+                              GArrowDoubleDataType))
+#define GARROW_DOUBLE_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DOUBLE_DATA_TYPE,        \
+                           GArrowDoubleDataTypeClass))
+#define GARROW_IS_DOUBLE_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DOUBLE_DATA_TYPE))
+#define GARROW_IS_DOUBLE_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DOUBLE_DATA_TYPE))
+#define GARROW_DOUBLE_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DOUBLE_DATA_TYPE,      \
+                             GArrowDoubleDataTypeClass))
+
+typedef struct _GArrowDoubleDataType         GArrowDoubleDataType;
+typedef struct _GArrowDoubleDataTypeClass    GArrowDoubleDataTypeClass;
+
+/**
+ * GArrowDoubleDataType:
+ *
+ * It wraps `arrow::DoubleType`.
+ */
+struct _GArrowDoubleDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowDoubleDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_double_data_type_get_type (void) G_GNUC_CONST;
+GArrowDoubleDataType *garrow_double_data_type_new      (void);
+
+
+#define GARROW_TYPE_BINARY_DATA_TYPE            \
+  (garrow_binary_data_type_get_type())
+#define GARROW_BINARY_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BINARY_DATA_TYPE,     \
+                              GArrowBinaryDataType))
+#define GARROW_BINARY_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BINARY_DATA_TYPE,        \
+                           GArrowBinaryDataTypeClass))
+#define GARROW_IS_BINARY_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BINARY_DATA_TYPE))
+#define GARROW_IS_BINARY_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BINARY_DATA_TYPE))
+#define GARROW_BINARY_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BINARY_DATA_TYPE,      \
+                             GArrowBinaryDataTypeClass))
+
+typedef struct _GArrowBinaryDataType         GArrowBinaryDataType;
+typedef struct _GArrowBinaryDataTypeClass    GArrowBinaryDataTypeClass;
+
+/**
+ * GArrowBinaryDataType:
+ *
+ * It wraps `arrow::BinaryType`.
+ */
+struct _GArrowBinaryDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowBinaryDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_binary_data_type_get_type (void) G_GNUC_CONST;
+GArrowBinaryDataType *garrow_binary_data_type_new      (void);
+
+
+#define GARROW_TYPE_STRING_DATA_TYPE            \
+  (garrow_string_data_type_get_type())
+#define GARROW_STRING_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
+                              GARROW_TYPE_STRING_DATA_TYPE,    \
+                              GArrowStringDataType))
+#define GARROW_STRING_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
+                           GARROW_TYPE_STRING_DATA_TYPE,       \
+                           GArrowStringDataTypeClass))
+#define GARROW_IS_STRING_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
+                              GARROW_TYPE_STRING_DATA_TYPE))
+#define GARROW_IS_STRING_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
+                           GARROW_TYPE_STRING_DATA_TYPE))
+#define GARROW_STRING_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
+                             GARROW_TYPE_STRING_DATA_TYPE,     \
+                             GArrowStringDataTypeClass))
+
+typedef struct _GArrowStringDataType         GArrowStringDataType;
+typedef struct _GArrowStringDataTypeClass    GArrowStringDataTypeClass;
+
+/**
+ * GArrowStringDataType:
+ *
+ * It wraps `arrow::StringType`.
+ */
+struct _GArrowStringDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowStringDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_string_data_type_get_type (void) G_GNUC_CONST;
+GArrowStringDataType *garrow_string_data_type_new      (void);
+
+
+#define GARROW_TYPE_LIST_DATA_TYPE              \
+  (garrow_list_data_type_get_type())
+#define GARROW_LIST_DATA_TYPE(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_LIST_DATA_TYPE,       \
+                              GArrowListDataType))
+#define GARROW_LIST_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_LIST_DATA_TYPE,  \
+                           GArrowListDataTypeClass))
+#define GARROW_IS_LIST_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_LIST_DATA_TYPE))
+#define GARROW_IS_LIST_DATA_TYPE_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_LIST_DATA_TYPE))
+#define GARROW_LIST_DATA_TYPE_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_LIST_DATA_TYPE,        \
+                             GArrowListDataTypeClass))
+
+typedef struct _GArrowListDataType         GArrowListDataType;
+typedef struct _GArrowListDataTypeClass    GArrowListDataTypeClass;
+
+/**
+ * GArrowListDataType:
+ *
+ * It wraps `arrow::ListType`.
+ */
+struct _GArrowListDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowListDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType               garrow_list_data_type_get_type (void) G_GNUC_CONST;
+GArrowListDataType *garrow_list_data_type_new      (GArrowField *field);
+GArrowField *garrow_list_data_type_get_value_field (GArrowListDataType *list_data_type);
+
+
+#define GARROW_TYPE_STRUCT_DATA_TYPE            \
+  (garrow_struct_data_type_get_type())
+#define GARROW_STRUCT_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STRUCT_DATA_TYPE,     \
+                              GArrowStructDataType))
+#define GARROW_STRUCT_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_STRUCT_DATA_TYPE,        \
+                           GArrowStructDataTypeClass))
+#define GARROW_IS_STRUCT_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRUCT_DATA_TYPE))
+#define GARROW_IS_STRUCT_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_STRUCT_DATA_TYPE))
+#define GARROW_STRUCT_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_STRUCT_DATA_TYPE,      \
+                             GArrowStructDataTypeClass))
+
+typedef struct _GArrowStructDataType         GArrowStructDataType;
+typedef struct _GArrowStructDataTypeClass    GArrowStructDataTypeClass;
+
+/**
+ * GArrowStructDataType:
+ *
+ * It wraps `arrow::StructType`.
+ */
+struct _GArrowStructDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowStructDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_struct_data_type_get_type (void) G_GNUC_CONST;
+GArrowStructDataType *garrow_struct_data_type_new      (GList *fields);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/double-data-type.cpp b/c_glib/arrow-glib/double-data-type.cpp
deleted file mode 100644
index c132f97ebe58f..0000000000000
--- a/c_glib/arrow-glib/double-data-type.cpp
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/double-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: double-data-type
- * @short_description: 64-bit floating point data type
- *
- * #GArrowDoubleDataType is a class for 64-bit floating point data
- * type.
- */
-
-G_DEFINE_TYPE(GArrowDoubleDataType,                \
-              garrow_double_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_double_data_type_init(GArrowDoubleDataType *object)
-{
-}
-
-static void
-garrow_double_data_type_class_init(GArrowDoubleDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_double_data_type_new:
- *
- * Returns: The newly created 64-bit floating point data type.
- */
-GArrowDoubleDataType *
-garrow_double_data_type_new(void)
-{
-  auto arrow_data_type = arrow::float64();
-
-  GArrowDoubleDataType *data_type =
-    GARROW_DOUBLE_DATA_TYPE(g_object_new(GARROW_TYPE_DOUBLE_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/double-data-type.h b/c_glib/arrow-glib/double-data-type.h
deleted file mode 100644
index ec725cbed3ba2..0000000000000
--- a/c_glib/arrow-glib/double-data-type.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_DOUBLE_DATA_TYPE           \
-  (garrow_double_data_type_get_type())
-#define GARROW_DOUBLE_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_DOUBLE_DATA_TYPE,    \
-                              GArrowDoubleDataType))
-#define GARROW_DOUBLE_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_DOUBLE_DATA_TYPE,       \
-                           GArrowDoubleDataTypeClass))
-#define GARROW_IS_DOUBLE_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DOUBLE_DATA_TYPE))
-#define GARROW_IS_DOUBLE_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_DOUBLE_DATA_TYPE))
-#define GARROW_DOUBLE_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_DOUBLE_DATA_TYPE,     \
-                             GArrowDoubleDataTypeClass))
-
-typedef struct _GArrowDoubleDataType         GArrowDoubleDataType;
-typedef struct _GArrowDoubleDataTypeClass    GArrowDoubleDataTypeClass;
-
-/**
- * GArrowDoubleDataType:
- *
- * It wraps `arrow::DoubleType`.
- */
-struct _GArrowDoubleDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowDoubleDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType garrow_double_data_type_get_type(void) G_GNUC_CONST;
-
-GArrowDoubleDataType *garrow_double_data_type_new(void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/float-data-type.cpp b/c_glib/arrow-glib/float-data-type.cpp
deleted file mode 100644
index ce7f28acfcb45..0000000000000
--- a/c_glib/arrow-glib/float-data-type.cpp
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/float-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: float-data-type
- * @short_description: 32-bit floating point data type
- *
- * #GArrowFloatDataType is a class for 32-bit floating point data
- * type.
- */
-
-G_DEFINE_TYPE(GArrowFloatDataType,                \
-              garrow_float_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_float_data_type_init(GArrowFloatDataType *object)
-{
-}
-
-static void
-garrow_float_data_type_class_init(GArrowFloatDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_float_data_type_new:
- *
- * Returns: The newly created float data type.
- */
-GArrowFloatDataType *
-garrow_float_data_type_new(void)
-{
-  auto arrow_data_type = arrow::float32();
-
-  GArrowFloatDataType *data_type =
-    GARROW_FLOAT_DATA_TYPE(g_object_new(GARROW_TYPE_FLOAT_DATA_TYPE,
-                                        "data-type", &arrow_data_type,
-                                        NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/float-data-type.h b/c_glib/arrow-glib/float-data-type.h
deleted file mode 100644
index dcb6c2ab13d25..0000000000000
--- a/c_glib/arrow-glib/float-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_FLOAT_DATA_TYPE           \
-  (garrow_float_data_type_get_type())
-#define GARROW_FLOAT_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
-                              GARROW_TYPE_FLOAT_DATA_TYPE,    \
-                              GArrowFloatDataType))
-#define GARROW_FLOAT_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
-                           GARROW_TYPE_FLOAT_DATA_TYPE,       \
-                           GArrowFloatDataTypeClass))
-#define GARROW_IS_FLOAT_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
-                              GARROW_TYPE_FLOAT_DATA_TYPE))
-#define GARROW_IS_FLOAT_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
-                           GARROW_TYPE_FLOAT_DATA_TYPE))
-#define GARROW_FLOAT_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
-                             GARROW_TYPE_FLOAT_DATA_TYPE,     \
-                             GArrowFloatDataTypeClass))
-
-typedef struct _GArrowFloatDataType         GArrowFloatDataType;
-typedef struct _GArrowFloatDataTypeClass    GArrowFloatDataTypeClass;
-
-/**
- * GArrowFloatDataType:
- *
- * It wraps `arrow::FloatType`.
- */
-struct _GArrowFloatDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowFloatDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                garrow_float_data_type_get_type (void) G_GNUC_CONST;
-GArrowFloatDataType *garrow_float_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-data-type.cpp b/c_glib/arrow-glib/int16-data-type.cpp
deleted file mode 100644
index 45e109e1759dc..0000000000000
--- a/c_glib/arrow-glib/int16-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/int16-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int16-data-type
- * @short_description: 16-bit integer data type
- *
- * #GArrowInt16DataType is a class for 16-bit integer data type.
- */
-
-G_DEFINE_TYPE(GArrowInt16DataType,                \
-              garrow_int16_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_int16_data_type_init(GArrowInt16DataType *object)
-{
-}
-
-static void
-garrow_int16_data_type_class_init(GArrowInt16DataTypeClass *klass)
-{
-}
-
-/**
- * garrow_int16_data_type_new:
- *
- * Returns: The newly created 16-bit integer data type.
- */
-GArrowInt16DataType *
-garrow_int16_data_type_new(void)
-{
-  auto arrow_data_type = arrow::int16();
-
-  GArrowInt16DataType *data_type =
-    GARROW_INT16_DATA_TYPE(g_object_new(GARROW_TYPE_INT16_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-data-type.h b/c_glib/arrow-glib/int16-data-type.h
deleted file mode 100644
index eaa199c4fc7f8..0000000000000
--- a/c_glib/arrow-glib/int16-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT16_DATA_TYPE            \
-  (garrow_int16_data_type_get_type())
-#define GARROW_INT16_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT16_DATA_TYPE,     \
-                              GArrowInt16DataType))
-#define GARROW_INT16_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_INT16_DATA_TYPE,        \
-                           GArrowInt16DataTypeClass))
-#define GARROW_IS_INT16_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT16_DATA_TYPE))
-#define GARROW_IS_INT16_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT16_DATA_TYPE))
-#define GARROW_INT16_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT16_DATA_TYPE,      \
-                             GArrowInt16DataTypeClass))
-
-typedef struct _GArrowInt16DataType         GArrowInt16DataType;
-typedef struct _GArrowInt16DataTypeClass    GArrowInt16DataTypeClass;
-
-/**
- * GArrowInt16DataType:
- *
- * It wraps `arrow::Int16Type`.
- */
-struct _GArrowInt16DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowInt16DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_int16_data_type_get_type (void) G_GNUC_CONST;
-GArrowInt16DataType *garrow_int16_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-data-type.cpp b/c_glib/arrow-glib/int32-data-type.cpp
deleted file mode 100644
index add21135364f9..0000000000000
--- a/c_glib/arrow-glib/int32-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/int32-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int32-data-type
- * @short_description: 32-bit integer data type
- *
- * #GArrowInt32DataType is a class for 32-bit integer data type.
- */
-
-G_DEFINE_TYPE(GArrowInt32DataType,                \
-              garrow_int32_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_int32_data_type_init(GArrowInt32DataType *object)
-{
-}
-
-static void
-garrow_int32_data_type_class_init(GArrowInt32DataTypeClass *klass)
-{
-}
-
-/**
- * garrow_int32_data_type_new:
- *
- * Returns: The newly created 32-bit integer data type.
- */
-GArrowInt32DataType *
-garrow_int32_data_type_new(void)
-{
-  auto arrow_data_type = arrow::int32();
-
-  GArrowInt32DataType *data_type =
-    GARROW_INT32_DATA_TYPE(g_object_new(GARROW_TYPE_INT32_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-data-type.h b/c_glib/arrow-glib/int32-data-type.h
deleted file mode 100644
index 75cccbd40560d..0000000000000
--- a/c_glib/arrow-glib/int32-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT32_DATA_TYPE            \
-  (garrow_int32_data_type_get_type())
-#define GARROW_INT32_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT32_DATA_TYPE,     \
-                              GArrowInt32DataType))
-#define GARROW_INT32_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_INT32_DATA_TYPE,        \
-                           GArrowInt32DataTypeClass))
-#define GARROW_IS_INT32_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT32_DATA_TYPE))
-#define GARROW_IS_INT32_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT32_DATA_TYPE))
-#define GARROW_INT32_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT32_DATA_TYPE,      \
-                             GArrowInt32DataTypeClass))
-
-typedef struct _GArrowInt32DataType         GArrowInt32DataType;
-typedef struct _GArrowInt32DataTypeClass    GArrowInt32DataTypeClass;
-
-/**
- * GArrowInt32DataType:
- *
- * It wraps `arrow::Int32Type`.
- */
-struct _GArrowInt32DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowInt32DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_int32_data_type_get_type (void) G_GNUC_CONST;
-GArrowInt32DataType *garrow_int32_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-data-type.cpp b/c_glib/arrow-glib/int64-data-type.cpp
deleted file mode 100644
index 8e85b9d2ab922..0000000000000
--- a/c_glib/arrow-glib/int64-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/int64-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int64-data-type
- * @short_description: 64-bit integer data type
- *
- * #GArrowInt64DataType is a class for 64-bit integer data type.
- */
-
-G_DEFINE_TYPE(GArrowInt64DataType,                \
-              garrow_int64_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_int64_data_type_init(GArrowInt64DataType *object)
-{
-}
-
-static void
-garrow_int64_data_type_class_init(GArrowInt64DataTypeClass *klass)
-{
-}
-
-/**
- * garrow_int64_data_type_new:
- *
- * Returns: The newly created 64-bit integer data type.
- */
-GArrowInt64DataType *
-garrow_int64_data_type_new(void)
-{
-  auto arrow_data_type = arrow::int64();
-
-  GArrowInt64DataType *data_type =
-    GARROW_INT64_DATA_TYPE(g_object_new(GARROW_TYPE_INT64_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-data-type.h b/c_glib/arrow-glib/int64-data-type.h
deleted file mode 100644
index 499e79f7ab7a7..0000000000000
--- a/c_glib/arrow-glib/int64-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT64_DATA_TYPE            \
-  (garrow_int64_data_type_get_type())
-#define GARROW_INT64_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT64_DATA_TYPE,     \
-                              GArrowInt64DataType))
-#define GARROW_INT64_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_INT64_DATA_TYPE,        \
-                           GArrowInt64DataTypeClass))
-#define GARROW_IS_INT64_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT64_DATA_TYPE))
-#define GARROW_IS_INT64_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT64_DATA_TYPE))
-#define GARROW_INT64_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT64_DATA_TYPE,      \
-                             GArrowInt64DataTypeClass))
-
-typedef struct _GArrowInt64DataType         GArrowInt64DataType;
-typedef struct _GArrowInt64DataTypeClass    GArrowInt64DataTypeClass;
-
-/**
- * GArrowInt64DataType:
- *
- * It wraps `arrow::Int64Type`.
- */
-struct _GArrowInt64DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowInt64DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_int64_data_type_get_type (void) G_GNUC_CONST;
-GArrowInt64DataType *garrow_int64_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-data-type.cpp b/c_glib/arrow-glib/int8-data-type.cpp
deleted file mode 100644
index 55b1ebc852d10..0000000000000
--- a/c_glib/arrow-glib/int8-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/int8-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int8-data-type
- * @short_description: 8-bit integer data type
- *
- * #GArrowInt8DataType is a class for 8-bit integer data type.
- */
-
-G_DEFINE_TYPE(GArrowInt8DataType,                \
-              garrow_int8_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_int8_data_type_init(GArrowInt8DataType *object)
-{
-}
-
-static void
-garrow_int8_data_type_class_init(GArrowInt8DataTypeClass *klass)
-{
-}
-
-/**
- * garrow_int8_data_type_new:
- *
- * Returns: The newly created 8-bit integer data type.
- */
-GArrowInt8DataType *
-garrow_int8_data_type_new(void)
-{
-  auto arrow_data_type = arrow::int8();
-
-  GArrowInt8DataType *data_type =
-    GARROW_INT8_DATA_TYPE(g_object_new(GARROW_TYPE_INT8_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-data-type.h b/c_glib/arrow-glib/int8-data-type.h
deleted file mode 100644
index 4343bd17a725b..0000000000000
--- a/c_glib/arrow-glib/int8-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT8_DATA_TYPE            \
-  (garrow_int8_data_type_get_type())
-#define GARROW_INT8_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT8_DATA_TYPE,     \
-                              GArrowInt8DataType))
-#define GARROW_INT8_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_INT8_DATA_TYPE,        \
-                           GArrowInt8DataTypeClass))
-#define GARROW_IS_INT8_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT8_DATA_TYPE))
-#define GARROW_IS_INT8_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT8_DATA_TYPE))
-#define GARROW_INT8_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT8_DATA_TYPE,      \
-                             GArrowInt8DataTypeClass))
-
-typedef struct _GArrowInt8DataType         GArrowInt8DataType;
-typedef struct _GArrowInt8DataTypeClass    GArrowInt8DataTypeClass;
-
-/**
- * GArrowInt8DataType:
- *
- * It wraps `arrow::Int8Type`.
- */
-struct _GArrowInt8DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowInt8DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_int8_data_type_get_type (void) G_GNUC_CONST;
-GArrowInt8DataType *garrow_int8_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/list-data-type.cpp b/c_glib/arrow-glib/list-data-type.cpp
deleted file mode 100644
index e82e6fdee48ba..0000000000000
--- a/c_glib/arrow-glib/list-data-type.cpp
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/field.hpp>
-#include <arrow-glib/list-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: list-data-type
- * @short_description: List data type
- *
- * #GArrowListDataType is a class for list data type.
- */
-
-G_DEFINE_TYPE(GArrowListDataType,                \
-              garrow_list_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_list_data_type_init(GArrowListDataType *object)
-{
-}
-
-static void
-garrow_list_data_type_class_init(GArrowListDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_list_data_type_new:
- * @field: The field of elements
- *
- * Returns: The newly created list data type.
- */
-GArrowListDataType *
-garrow_list_data_type_new(GArrowField *field)
-{
-  auto arrow_field = garrow_field_get_raw(field);
-  auto arrow_data_type =
-    std::make_shared<arrow::ListType>(arrow_field);
-
-  GArrowListDataType *data_type =
-    GARROW_LIST_DATA_TYPE(g_object_new(GARROW_TYPE_LIST_DATA_TYPE,
-                                       "data-type", &arrow_data_type,
-                                       NULL));
-  return data_type;
-}
-
-/**
- * garrow_list_data_type_get_value_field:
- * @list_data_type: A #GArrowListDataType.
- *
- * Returns: (transfer full): The field of value.
- */
-GArrowField *
-garrow_list_data_type_get_value_field(GArrowListDataType *list_data_type)
-{
-  auto arrow_data_type =
-    garrow_data_type_get_raw(GARROW_DATA_TYPE(list_data_type));
-  auto arrow_list_data_type =
-    static_cast<arrow::ListType *>(arrow_data_type.get());
-
-  auto arrow_field = arrow_list_data_type->value_field();
-  auto field = garrow_field_new_raw(&arrow_field);
-
-  return field;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/list-data-type.h b/c_glib/arrow-glib/list-data-type.h
deleted file mode 100644
index bb406e2c62074..0000000000000
--- a/c_glib/arrow-glib/list-data-type.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-#include <arrow-glib/field.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_LIST_DATA_TYPE              \
-  (garrow_list_data_type_get_type())
-#define GARROW_LIST_DATA_TYPE(obj)                              \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_LIST_DATA_TYPE,       \
-                              GArrowListDataType))
-#define GARROW_LIST_DATA_TYPE_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_LIST_DATA_TYPE,  \
-                           GArrowListDataTypeClass))
-#define GARROW_IS_LIST_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_LIST_DATA_TYPE))
-#define GARROW_IS_LIST_DATA_TYPE_CLASS(klass)           \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_LIST_DATA_TYPE))
-#define GARROW_LIST_DATA_TYPE_GET_CLASS(obj)                    \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_LIST_DATA_TYPE,        \
-                             GArrowListDataTypeClass))
-
-typedef struct _GArrowListDataType         GArrowListDataType;
-typedef struct _GArrowListDataTypeClass    GArrowListDataTypeClass;
-
-/**
- * GArrowListDataType:
- *
- * It wraps `arrow::ListType`.
- */
-struct _GArrowListDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowListDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType               garrow_list_data_type_get_type (void) G_GNUC_CONST;
-
-GArrowListDataType *garrow_list_data_type_new      (GArrowField *field);
-
-GArrowField *garrow_list_data_type_get_value_field (GArrowListDataType *list_data_type);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/null-data-type.cpp b/c_glib/arrow-glib/null-data-type.cpp
deleted file mode 100644
index 1f75d3bb88c37..0000000000000
--- a/c_glib/arrow-glib/null-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/null-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: null-data-type
- * @short_description: Null data type
- *
- * #GArrowNullDataType is a class for null data type.
- */
-
-G_DEFINE_TYPE(GArrowNullDataType,                \
-              garrow_null_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_null_data_type_init(GArrowNullDataType *object)
-{
-}
-
-static void
-garrow_null_data_type_class_init(GArrowNullDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_null_data_type_new:
- *
- * Returns: The newly created null data type.
- */
-GArrowNullDataType *
-garrow_null_data_type_new(void)
-{
-  auto arrow_data_type = arrow::null();
-
-  GArrowNullDataType *data_type =
-    GARROW_NULL_DATA_TYPE(g_object_new(GARROW_TYPE_NULL_DATA_TYPE,
-                                       "data-type", &arrow_data_type,
-                                       NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/null-data-type.h b/c_glib/arrow-glib/null-data-type.h
deleted file mode 100644
index 006b76c961f3b..0000000000000
--- a/c_glib/arrow-glib/null-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_NULL_DATA_TYPE              \
-  (garrow_null_data_type_get_type())
-#define GARROW_NULL_DATA_TYPE(obj)                              \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_NULL_DATA_TYPE,       \
-                              GArrowNullDataType))
-#define GARROW_NULL_DATA_TYPE_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_NULL_DATA_TYPE,  \
-                           GArrowNullDataTypeClass))
-#define GARROW_IS_NULL_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_NULL_DATA_TYPE))
-#define GARROW_IS_NULL_DATA_TYPE_CLASS(klass)           \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_NULL_DATA_TYPE))
-#define GARROW_NULL_DATA_TYPE_GET_CLASS(obj)                    \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_NULL_DATA_TYPE,        \
-                             GArrowNullDataTypeClass))
-
-typedef struct _GArrowNullDataType         GArrowNullDataType;
-typedef struct _GArrowNullDataTypeClass    GArrowNullDataTypeClass;
-
-/**
- * GArrowNullDataType:
- *
- * It wraps `arrow::NullType`.
- */
-struct _GArrowNullDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowNullDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType               garrow_null_data_type_get_type (void) G_GNUC_CONST;
-GArrowNullDataType *garrow_null_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/string-data-type.cpp b/c_glib/arrow-glib/string-data-type.cpp
deleted file mode 100644
index 96a31bf2f906a..0000000000000
--- a/c_glib/arrow-glib/string-data-type.cpp
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/string-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: string-data-type
- * @short_description: UTF-8 encoded string data type
- *
- * #GArrowStringDataType is a class for UTF-8 encoded string data
- * type.
- */
-
-G_DEFINE_TYPE(GArrowStringDataType,                \
-              garrow_string_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_string_data_type_init(GArrowStringDataType *object)
-{
-}
-
-static void
-garrow_string_data_type_class_init(GArrowStringDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_string_data_type_new:
- *
- * Returns: The newly created UTF-8 encoded string data type.
- */
-GArrowStringDataType *
-garrow_string_data_type_new(void)
-{
-  auto arrow_data_type = arrow::utf8();
-
-  GArrowStringDataType *data_type =
-    GARROW_STRING_DATA_TYPE(g_object_new(GARROW_TYPE_STRING_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/string-data-type.h b/c_glib/arrow-glib/string-data-type.h
deleted file mode 100644
index d10a325e1bb6c..0000000000000
--- a/c_glib/arrow-glib/string-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_STRING_DATA_TYPE            \
-  (garrow_string_data_type_get_type())
-#define GARROW_STRING_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
-                              GARROW_TYPE_STRING_DATA_TYPE,    \
-                              GArrowStringDataType))
-#define GARROW_STRING_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
-                           GARROW_TYPE_STRING_DATA_TYPE,       \
-                           GArrowStringDataTypeClass))
-#define GARROW_IS_STRING_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
-                              GARROW_TYPE_STRING_DATA_TYPE))
-#define GARROW_IS_STRING_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
-                           GARROW_TYPE_STRING_DATA_TYPE))
-#define GARROW_STRING_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
-                             GARROW_TYPE_STRING_DATA_TYPE,     \
-                             GArrowStringDataTypeClass))
-
-typedef struct _GArrowStringDataType         GArrowStringDataType;
-typedef struct _GArrowStringDataTypeClass    GArrowStringDataTypeClass;
-
-/**
- * GArrowStringDataType:
- *
- * It wraps `arrow::StringType`.
- */
-struct _GArrowStringDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowStringDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_string_data_type_get_type (void) G_GNUC_CONST;
-GArrowStringDataType *garrow_string_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array-builder.h b/c_glib/arrow-glib/struct-array-builder.h
index 7dd86625616e3..237b2b3264f24 100644
--- a/c_glib/arrow-glib/struct-array-builder.h
+++ b/c_glib/arrow-glib/struct-array-builder.h
@@ -20,7 +20,7 @@
 #pragma once
 
 #include <arrow-glib/array-builder.h>
-#include <arrow-glib/struct-data-type.h>
+#include <arrow-glib/data-type.h>
 
 G_BEGIN_DECLS
 
diff --git a/c_glib/arrow-glib/struct-data-type.cpp b/c_glib/arrow-glib/struct-data-type.cpp
deleted file mode 100644
index 9a4f2a2deead0..0000000000000
--- a/c_glib/arrow-glib/struct-data-type.cpp
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/field.hpp>
-#include <arrow-glib/struct-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: struct-data-type
- * @short_description: Struct data type
- *
- * #GArrowStructDataType is a class for struct data type.
- */
-
-G_DEFINE_TYPE(GArrowStructDataType,                \
-              garrow_struct_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_struct_data_type_init(GArrowStructDataType *object)
-{
-}
-
-static void
-garrow_struct_data_type_class_init(GArrowStructDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_struct_data_type_new:
- * @fields: (element-type GArrowField): The fields of the struct.
- *
- * Returns: The newly created struct data type.
- */
-GArrowStructDataType *
-garrow_struct_data_type_new(GList *fields)
-{
-  std::vector<std::shared_ptr<arrow::Field>> arrow_fields;
-  for (GList *node = fields; node; node = g_list_next(node)) {
-    auto field = GARROW_FIELD(node->data);
-    auto arrow_field = garrow_field_get_raw(field);
-    arrow_fields.push_back(arrow_field);
-  }
-
-  auto arrow_data_type = std::make_shared<arrow::StructType>(arrow_fields);
-  GArrowStructDataType *data_type =
-    GARROW_STRUCT_DATA_TYPE(g_object_new(GARROW_TYPE_STRUCT_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-data-type.h b/c_glib/arrow-glib/struct-data-type.h
deleted file mode 100644
index 0a2c743e280b7..0000000000000
--- a/c_glib/arrow-glib/struct-data-type.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-#include <arrow-glib/field.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_STRUCT_DATA_TYPE            \
-  (garrow_struct_data_type_get_type())
-#define GARROW_STRUCT_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_STRUCT_DATA_TYPE,     \
-                              GArrowStructDataType))
-#define GARROW_STRUCT_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_STRUCT_DATA_TYPE,        \
-                           GArrowStructDataTypeClass))
-#define GARROW_IS_STRUCT_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STRUCT_DATA_TYPE))
-#define GARROW_IS_STRUCT_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_STRUCT_DATA_TYPE))
-#define GARROW_STRUCT_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_STRUCT_DATA_TYPE,      \
-                             GArrowStructDataTypeClass))
-
-typedef struct _GArrowStructDataType         GArrowStructDataType;
-typedef struct _GArrowStructDataTypeClass    GArrowStructDataTypeClass;
-
-/**
- * GArrowStructDataType:
- *
- * It wraps `arrow::StructType`.
- */
-struct _GArrowStructDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowStructDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType               garrow_struct_data_type_get_type (void) G_GNUC_CONST;
-
-GArrowStructDataType *garrow_struct_data_type_new(GList *fields);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-data-type.cpp b/c_glib/arrow-glib/uint16-data-type.cpp
deleted file mode 100644
index 918b75d61c3eb..0000000000000
--- a/c_glib/arrow-glib/uint16-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/uint16-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint16-data-type
- * @short_description: 16-bit unsigned integer data type
- *
- * #GArrowUInt16DataType is a class for 16-bit unsigned integer data type.
- */
-
-G_DEFINE_TYPE(GArrowUInt16DataType,                \
-              garrow_uint16_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_uint16_data_type_init(GArrowUInt16DataType *object)
-{
-}
-
-static void
-garrow_uint16_data_type_class_init(GArrowUInt16DataTypeClass *klass)
-{
-}
-
-/**
- * garrow_uint16_data_type_new:
- *
- * Returns: The newly created 16-bit unsigned integer data type.
- */
-GArrowUInt16DataType *
-garrow_uint16_data_type_new(void)
-{
-  auto arrow_data_type = arrow::uint16();
-
-  GArrowUInt16DataType *data_type =
-    GARROW_UINT16_DATA_TYPE(g_object_new(GARROW_TYPE_UINT16_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-data-type.h b/c_glib/arrow-glib/uint16-data-type.h
deleted file mode 100644
index b65189d888fcd..0000000000000
--- a/c_glib/arrow-glib/uint16-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT16_DATA_TYPE            \
-  (garrow_uint16_data_type_get_type())
-#define GARROW_UINT16_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT16_DATA_TYPE,     \
-                              GArrowUInt16DataType))
-#define GARROW_UINT16_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT16_DATA_TYPE,        \
-                           GArrowUInt16DataTypeClass))
-#define GARROW_IS_UINT16_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT16_DATA_TYPE))
-#define GARROW_IS_UINT16_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT16_DATA_TYPE))
-#define GARROW_UINT16_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT16_DATA_TYPE,      \
-                             GArrowUInt16DataTypeClass))
-
-typedef struct _GArrowUInt16DataType         GArrowUInt16DataType;
-typedef struct _GArrowUInt16DataTypeClass    GArrowUInt16DataTypeClass;
-
-/**
- * GArrowUInt16DataType:
- *
- * It wraps `arrow::UInt16Type`.
- */
-struct _GArrowUInt16DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowUInt16DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_uint16_data_type_get_type (void) G_GNUC_CONST;
-GArrowUInt16DataType *garrow_uint16_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-data-type.cpp b/c_glib/arrow-glib/uint32-data-type.cpp
deleted file mode 100644
index fde14f3274174..0000000000000
--- a/c_glib/arrow-glib/uint32-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/uint32-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint32-data-type
- * @short_description: 32-bit unsigned integer data type
- *
- * #GArrowUInt32DataType is a class for 32-bit unsigned integer data type.
- */
-
-G_DEFINE_TYPE(GArrowUInt32DataType,                \
-              garrow_uint32_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_uint32_data_type_init(GArrowUInt32DataType *object)
-{
-}
-
-static void
-garrow_uint32_data_type_class_init(GArrowUInt32DataTypeClass *klass)
-{
-}
-
-/**
- * garrow_uint32_data_type_new:
- *
- * Returns: The newly created 32-bit unsigned integer data type.
- */
-GArrowUInt32DataType *
-garrow_uint32_data_type_new(void)
-{
-  auto arrow_data_type = arrow::uint32();
-
-  GArrowUInt32DataType *data_type =
-    GARROW_UINT32_DATA_TYPE(g_object_new(GARROW_TYPE_UINT32_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-data-type.h b/c_glib/arrow-glib/uint32-data-type.h
deleted file mode 100644
index 4fe60cd850ba8..0000000000000
--- a/c_glib/arrow-glib/uint32-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT32_DATA_TYPE            \
-  (garrow_uint32_data_type_get_type())
-#define GARROW_UINT32_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT32_DATA_TYPE,     \
-                              GArrowUInt32DataType))
-#define GARROW_UINT32_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT32_DATA_TYPE,        \
-                           GArrowUInt32DataTypeClass))
-#define GARROW_IS_UINT32_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT32_DATA_TYPE))
-#define GARROW_IS_UINT32_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT32_DATA_TYPE))
-#define GARROW_UINT32_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT32_DATA_TYPE,      \
-                             GArrowUInt32DataTypeClass))
-
-typedef struct _GArrowUInt32DataType         GArrowUInt32DataType;
-typedef struct _GArrowUInt32DataTypeClass    GArrowUInt32DataTypeClass;
-
-/**
- * GArrowUInt32DataType:
- *
- * It wraps `arrow::UInt32Type`.
- */
-struct _GArrowUInt32DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowUInt32DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_uint32_data_type_get_type (void) G_GNUC_CONST;
-GArrowUInt32DataType *garrow_uint32_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-data-type.cpp b/c_glib/arrow-glib/uint64-data-type.cpp
deleted file mode 100644
index 7c18b36a01b3b..0000000000000
--- a/c_glib/arrow-glib/uint64-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/uint64-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint64-data-type
- * @short_description: 64-bit unsigned integer data type
- *
- * #GArrowUInt64DataType is a class for 64-bit unsigned integer data type.
- */
-
-G_DEFINE_TYPE(GArrowUInt64DataType,                \
-              garrow_uint64_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_uint64_data_type_init(GArrowUInt64DataType *object)
-{
-}
-
-static void
-garrow_uint64_data_type_class_init(GArrowUInt64DataTypeClass *klass)
-{
-}
-
-/**
- * garrow_uint64_data_type_new:
- *
- * Returns: The newly created 64-bit unsigned integer data type.
- */
-GArrowUInt64DataType *
-garrow_uint64_data_type_new(void)
-{
-  auto arrow_data_type = arrow::uint64();
-
-  GArrowUInt64DataType *data_type =
-    GARROW_UINT64_DATA_TYPE(g_object_new(GARROW_TYPE_UINT64_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-data-type.h b/c_glib/arrow-glib/uint64-data-type.h
deleted file mode 100644
index 221023c863818..0000000000000
--- a/c_glib/arrow-glib/uint64-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT64_DATA_TYPE            \
-  (garrow_uint64_data_type_get_type())
-#define GARROW_UINT64_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT64_DATA_TYPE,     \
-                              GArrowUInt64DataType))
-#define GARROW_UINT64_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT64_DATA_TYPE,        \
-                           GArrowUInt64DataTypeClass))
-#define GARROW_IS_UINT64_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT64_DATA_TYPE))
-#define GARROW_IS_UINT64_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT64_DATA_TYPE))
-#define GARROW_UINT64_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT64_DATA_TYPE,      \
-                             GArrowUInt64DataTypeClass))
-
-typedef struct _GArrowUInt64DataType         GArrowUInt64DataType;
-typedef struct _GArrowUInt64DataTypeClass    GArrowUInt64DataTypeClass;
-
-/**
- * GArrowUInt64DataType:
- *
- * It wraps `arrow::UInt64Type`.
- */
-struct _GArrowUInt64DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowUInt64DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_uint64_data_type_get_type (void) G_GNUC_CONST;
-GArrowUInt64DataType *garrow_uint64_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-data-type.cpp b/c_glib/arrow-glib/uint8-data-type.cpp
deleted file mode 100644
index 7c93e455a4e96..0000000000000
--- a/c_glib/arrow-glib/uint8-data-type.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/uint8-data-type.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint8-data-type
- * @short_description: 8-bit unsigned integer data type
- *
- * #GArrowUInt8DataType is a class for 8-bit unsigned integer data type.
- */
-
-G_DEFINE_TYPE(GArrowUInt8DataType,                \
-              garrow_uint8_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_uint8_data_type_init(GArrowUInt8DataType *object)
-{
-}
-
-static void
-garrow_uint8_data_type_class_init(GArrowUInt8DataTypeClass *klass)
-{
-}
-
-/**
- * garrow_uint8_data_type_new:
- *
- * Returns: The newly created 8-bit unsigned integer data type.
- */
-GArrowUInt8DataType *
-garrow_uint8_data_type_new(void)
-{
-  auto arrow_data_type = arrow::uint8();
-
-  GArrowUInt8DataType *data_type =
-    GARROW_UINT8_DATA_TYPE(g_object_new(GARROW_TYPE_UINT8_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-data-type.h b/c_glib/arrow-glib/uint8-data-type.h
deleted file mode 100644
index 6e058524f4b10..0000000000000
--- a/c_glib/arrow-glib/uint8-data-type.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT8_DATA_TYPE            \
-  (garrow_uint8_data_type_get_type())
-#define GARROW_UINT8_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT8_DATA_TYPE,     \
-                              GArrowUInt8DataType))
-#define GARROW_UINT8_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT8_DATA_TYPE,        \
-                           GArrowUInt8DataTypeClass))
-#define GARROW_IS_UINT8_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT8_DATA_TYPE))
-#define GARROW_IS_UINT8_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT8_DATA_TYPE))
-#define GARROW_UINT8_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT8_DATA_TYPE,      \
-                             GArrowUInt8DataTypeClass))
-
-typedef struct _GArrowUInt8DataType         GArrowUInt8DataType;
-typedef struct _GArrowUInt8DataTypeClass    GArrowUInt8DataTypeClass;
-
-/**
- * GArrowUInt8DataType:
- *
- * It wraps `arrow::UInt8Type`.
- */
-struct _GArrowUInt8DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowUInt8DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_uint8_data_type_get_type (void) G_GNUC_CONST;
-GArrowUInt8DataType *garrow_uint8_data_type_new      (void);
-
-G_END_DECLS
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 11e6a4de244d4..5df9f64a85c92 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -60,22 +60,6 @@
       <title>Type</title>
       <xi:include href="xml/type.xml"/>
       <xi:include href="xml/data-type.xml"/>
-      <xi:include href="xml/null-data-type.xml"/>
-      <xi:include href="xml/boolean-data-type.xml"/>
-      <xi:include href="xml/int8-data-type.xml"/>
-      <xi:include href="xml/uint8-data-type.xml"/>
-      <xi:include href="xml/int16-data-type.xml"/>
-      <xi:include href="xml/uint16-data-type.xml"/>
-      <xi:include href="xml/int32-data-type.xml"/>
-      <xi:include href="xml/uint32-data-type.xml"/>
-      <xi:include href="xml/int64-data-type.xml"/>
-      <xi:include href="xml/uint64-data-type.xml"/>
-      <xi:include href="xml/float-data-type.xml"/>
-      <xi:include href="xml/double-data-type.xml"/>
-      <xi:include href="xml/binary-data-type.xml"/>
-      <xi:include href="xml/string-data-type.xml"/>
-      <xi:include href="xml/list-data-type.xml"/>
-      <xi:include href="xml/struct-data-type.xml"/>
     </chapter>
     <chapter id="schema">
       <title>Schema</title>

From 423235ccb39737d66e1c47d119879787d9e10847 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 21 Apr 2017 17:51:31 -0400
Subject: [PATCH 0549/1644] ARROW-868: [GLib] Use GBytes to reduce copy

Author: Kouhei Sutou <kou@clear-code.com>

Closes #576 from kou/glib-binary-array-use-gbytes and squashes the following commits:

7aeb799 [Kouhei Sutou] [GLib] Use GBytes to reduce copy
---
 c_glib/arrow-glib/array.cpp      | 13 +++++++------
 c_glib/arrow-glib/array.h        |  5 ++---
 c_glib/test/test-binary-array.rb |  5 +++--
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index c86bff90d40d6..dc1386b0daab9 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -672,19 +672,20 @@ garrow_binary_array_class_init(GArrowBinaryArrayClass *klass)
  * garrow_binary_array_get_value:
  * @array: A #GArrowBinaryArray.
  * @i: The index of the target value.
- * @length: (out): The length of the value.
  *
- * Returns: (array length=length): The i-th value.
+ * Returns: (transfer full): The i-th value.
  */
-const guint8 *
+GBytes *
 garrow_binary_array_get_value(GArrowBinaryArray *array,
-                              gint64 i,
-                              gint32 *length)
+                              gint64 i)
 {
   auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
   auto arrow_binary_array =
     static_cast<arrow::BinaryArray *>(arrow_array.get());
-  return arrow_binary_array->GetValue(i, length);
+
+  int32_t length;
+  auto value = arrow_binary_array->GetValue(i, &length);
+  return g_bytes_new_static(value, length);
 }
 
 
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index b417cdbab3631..74064562d6f39 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -660,9 +660,8 @@ struct _GArrowBinaryArrayClass
 
 GType garrow_binary_array_get_type(void) G_GNUC_CONST;
 
-const guint8 *garrow_binary_array_get_value(GArrowBinaryArray *array,
-                                            gint64 i,
-                                            gint32 *length);
+GBytes *garrow_binary_array_get_value(GArrowBinaryArray *array,
+                                      gint64 i);
 
 #define GARROW_TYPE_STRING_ARRAY                \
   (garrow_string_array_get_type())
diff --git a/c_glib/test/test-binary-array.rb b/c_glib/test/test-binary-array.rb
index 82a537ef29e9e..6fe89247c8649 100644
--- a/c_glib/test/test-binary-array.rb
+++ b/c_glib/test/test-binary-array.rb
@@ -17,9 +17,10 @@
 
 class TestBinaryArray < Test::Unit::TestCase
   def test_value
+    data = "\x00\x01\x02"
     builder = Arrow::BinaryArrayBuilder.new
-    builder.append("\x00\x01\x02")
+    builder.append(data)
     array = builder.finish
-    assert_equal([0, 1, 2], array.get_value(0))
+    assert_equal(data, array.get_value(0).to_s)
   end
 end

From 76dfd9878529c010b43726058ef3e913a78501f0 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 22 Apr 2017 10:43:01 -0400
Subject: [PATCH 0550/1644] ARROW-876: [GLib] Unify ArrayBuilder files

Author: Kouhei Sutou <kou@clear-code.com>

Closes #581 from kou/glib-array-builder-unify-file and squashes the following commits:

5449d50 [Kouhei Sutou] [GLib] Unify ArrayBuilder files
---
 c_glib/arrow-glib/Makefile.am               |   32 +-
 c_glib/arrow-glib/array-builder.cpp         | 1381 ++++++++++++++++++-
 c_glib/arrow-glib/array-builder.h           |  769 +++++++++++
 c_glib/arrow-glib/arrow-glib.h              |   15 -
 c_glib/arrow-glib/binary-array-builder.cpp  |  122 --
 c_glib/arrow-glib/binary-array-builder.h    |   77 --
 c_glib/arrow-glib/boolean-array-builder.cpp |  120 --
 c_glib/arrow-glib/boolean-array-builder.h   |   76 -
 c_glib/arrow-glib/double-array-builder.cpp  |  120 --
 c_glib/arrow-glib/double-array-builder.h    |   76 -
 c_glib/arrow-glib/float-array-builder.cpp   |  120 --
 c_glib/arrow-glib/float-array-builder.h     |   76 -
 c_glib/arrow-glib/int16-array-builder.cpp   |  120 --
 c_glib/arrow-glib/int16-array-builder.h     |   76 -
 c_glib/arrow-glib/int32-array-builder.cpp   |  120 --
 c_glib/arrow-glib/int32-array-builder.h     |   76 -
 c_glib/arrow-glib/int64-array-builder.cpp   |  120 --
 c_glib/arrow-glib/int64-array-builder.h     |   76 -
 c_glib/arrow-glib/int8-array-builder.cpp    |  120 --
 c_glib/arrow-glib/int8-array-builder.h      |   76 -
 c_glib/arrow-glib/list-array-builder.cpp    |  173 ---
 c_glib/arrow-glib/list-array-builder.h      |   77 --
 c_glib/arrow-glib/string-array-builder.cpp  |   97 --
 c_glib/arrow-glib/string-array-builder.h    |   74 -
 c_glib/arrow-glib/struct-array-builder.cpp  |  187 ---
 c_glib/arrow-glib/struct-array-builder.h    |   81 --
 c_glib/arrow-glib/uint16-array-builder.cpp  |  120 --
 c_glib/arrow-glib/uint16-array-builder.h    |   76 -
 c_glib/arrow-glib/uint32-array-builder.cpp  |  120 --
 c_glib/arrow-glib/uint32-array-builder.h    |   76 -
 c_glib/arrow-glib/uint64-array-builder.cpp  |  120 --
 c_glib/arrow-glib/uint64-array-builder.h    |   76 -
 c_glib/arrow-glib/uint8-array-builder.cpp   |  120 --
 c_glib/arrow-glib/uint8-array-builder.h     |   76 -
 c_glib/doc/reference/arrow-glib-docs.sgml   |   15 -
 35 files changed, 2135 insertions(+), 3121 deletions(-)
 delete mode 100644 c_glib/arrow-glib/binary-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/binary-array-builder.h
 delete mode 100644 c_glib/arrow-glib/boolean-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/boolean-array-builder.h
 delete mode 100644 c_glib/arrow-glib/double-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/double-array-builder.h
 delete mode 100644 c_glib/arrow-glib/float-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/float-array-builder.h
 delete mode 100644 c_glib/arrow-glib/int16-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/int16-array-builder.h
 delete mode 100644 c_glib/arrow-glib/int32-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/int32-array-builder.h
 delete mode 100644 c_glib/arrow-glib/int64-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/int64-array-builder.h
 delete mode 100644 c_glib/arrow-glib/int8-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/int8-array-builder.h
 delete mode 100644 c_glib/arrow-glib/list-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/list-array-builder.h
 delete mode 100644 c_glib/arrow-glib/string-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/string-array-builder.h
 delete mode 100644 c_glib/arrow-glib/struct-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/struct-array-builder.h
 delete mode 100644 c_glib/arrow-glib/uint16-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/uint16-array-builder.h
 delete mode 100644 c_glib/arrow-glib/uint32-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/uint32-array-builder.h
 delete mode 100644 c_glib/arrow-glib/uint64-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/uint64-array-builder.h
 delete mode 100644 c_glib/arrow-glib/uint8-array-builder.cpp
 delete mode 100644 c_glib/arrow-glib/uint8-array-builder.h

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index d0c8c799b71cf..bbc11011474bc 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -44,32 +44,17 @@ libarrow_glib_la_headers =			\
 	array.h					\
 	array-builder.h				\
 	arrow-glib.h				\
-	binary-array-builder.h			\
-	boolean-array-builder.h			\
 	buffer.h				\
 	chunked-array.h				\
 	column.h				\
 	data-type.h				\
-	double-array-builder.h			\
 	error.h					\
 	field.h					\
-	float-array-builder.h			\
-	int8-array-builder.h			\
-	int16-array-builder.h			\
-	int32-array-builder.h			\
-	int64-array-builder.h			\
-	list-array-builder.h			\
 	record-batch.h				\
 	schema.h				\
-	string-array-builder.h			\
-	struct-array-builder.h			\
 	table.h					\
 	tensor.h				\
-	type.h					\
-	uint8-array-builder.h			\
-	uint16-array-builder.h			\
-	uint32-array-builder.h			\
-	uint64-array-builder.h
+	type.h
 
 libarrow_glib_la_headers +=			\
 	file.h					\
@@ -100,32 +85,17 @@ libarrow_glib_la_generated_sources =		\
 libarrow_glib_la_sources =			\
 	array.cpp				\
 	array-builder.cpp			\
-	binary-array-builder.cpp		\
-	boolean-array-builder.cpp		\
 	buffer.cpp				\
 	chunked-array.cpp			\
 	column.cpp				\
 	data-type.cpp				\
-	double-array-builder.cpp		\
 	error.cpp				\
 	field.cpp				\
-	float-array-builder.cpp			\
-	int8-array-builder.cpp			\
-	int16-array-builder.cpp			\
-	int32-array-builder.cpp			\
-	int64-array-builder.cpp			\
-	list-array-builder.cpp			\
 	record-batch.cpp			\
 	schema.cpp				\
-	string-array-builder.cpp		\
-	struct-array-builder.cpp		\
 	table.cpp				\
 	tensor.cpp				\
 	type.cpp				\
-	uint8-array-builder.cpp			\
-	uint16-array-builder.cpp		\
-	uint32-array-builder.cpp		\
-	uint64-array-builder.cpp		\
 	$(libarrow_glib_la_headers)		\
 	$(libarrow_glib_la_generated_sources)
 
diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index aea93d00bafe4..97d43e1f0c022 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -22,32 +22,66 @@
 #endif
 
 #include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/binary-array-builder.h>
-#include <arrow-glib/boolean-array-builder.h>
-#include <arrow-glib/double-array-builder.h>
-#include <arrow-glib/float-array-builder.h>
-#include <arrow-glib/int8-array-builder.h>
-#include <arrow-glib/int16-array-builder.h>
-#include <arrow-glib/int32-array-builder.h>
-#include <arrow-glib/int64-array-builder.h>
-#include <arrow-glib/list-array-builder.h>
-#include <arrow-glib/string-array-builder.h>
-#include <arrow-glib/struct-array-builder.h>
-#include <arrow-glib/uint8-array-builder.h>
-#include <arrow-glib/uint16-array-builder.h>
-#include <arrow-glib/uint32-array-builder.h>
-#include <arrow-glib/uint64-array-builder.h>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/error.hpp>
 
 G_BEGIN_DECLS
 
 /**
  * SECTION: array-builder
- * @short_description: Base class for all array builder classes.
+ * @section_id: array-builder-classes
+ * @title: Array builder classes
+ * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowArrayBuilder is a base class for all array builder classes
  * such as #GArrowBooleanArrayBuilder.
  *
  * You need to use array builder class to create a new array.
+ *
+ * #GArrowBooleanArrayBuilder is the class to create a new
+ * #GArrowBooleanArray.
+ *
+ * #GArrowInt8ArrayBuilder is the class to create a new
+ * #GArrowInt8Array.
+ *
+ * #GArrowUInt8ArrayBuilder is the class to create a new
+ * #GArrowUInt8Array.
+ *
+ * #GArrowInt16ArrayBuilder is the class to create a new
+ * #GArrowInt16Array.
+ *
+ * #GArrowUInt16ArrayBuilder is the class to create a new
+ * #GArrowUInt16Array.
+ *
+ * #GArrowInt32ArrayBuilder is the class to create a new
+ * #GArrowInt32Array.
+ *
+ * #GArrowUInt32ArrayBuilder is the class to create a new
+ * #GArrowUInt32Array.
+ *
+ * #GArrowInt64ArrayBuilder is the class to create a new
+ * #GArrowInt64Array.
+ *
+ * #GArrowUInt64ArrayBuilder is the class to create a new
+ * #GArrowUInt64Array.
+ *
+ * #GArrowFloatArrayBuilder is the class to creating a new
+ * #GArrowFloatArray.
+ *
+ * #GArrowDoubleArrayBuilder is the class to create a new
+ * #GArrowDoubleArray.
+ *
+ * #GArrowBinaryArrayBuilder is the class to create a new
+ * #GArrowBinaryArray.
+ *
+ * #GArrowStringArrayBuilder is the class to create a new
+ * #GArrowStringArray.
+ *
+ * #GArrowListArrayBuilder is the class to create a new
+ * #GArrowListArray.
+ *
+ * #GArrowStructArrayBuilder is the class to create a new
+ * #GArrowStructArray.
  */
 
 typedef struct GArrowArrayBuilderPrivate_ {
@@ -154,6 +188,1321 @@ garrow_array_builder_finish(GArrowArrayBuilder *builder)
   return garrow_array_new_raw(&arrow_array);
 }
 
+
+G_DEFINE_TYPE(GArrowBooleanArrayBuilder,
+              garrow_boolean_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_boolean_array_builder_init(GArrowBooleanArrayBuilder *builder)
+{
+}
+
+static void
+garrow_boolean_array_builder_class_init(GArrowBooleanArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_boolean_array_builder_new:
+ *
+ * Returns: A newly created #GArrowBooleanArrayBuilder.
+ */
+GArrowBooleanArrayBuilder *
+garrow_boolean_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_boolean_builder =
+    std::make_shared<arrow::BooleanBuilder>(memory_pool);
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_boolean_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_BOOLEAN_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_boolean_array_builder_append:
+ * @builder: A #GArrowBooleanArrayBuilder.
+ * @value: A boolean value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
+                                    gboolean value,
+                                    GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::BooleanBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(static_cast<bool>(value));
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[boolean-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_boolean_array_builder_append_null:
+ * @builder: A #GArrowBooleanArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
+                                         GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::BooleanBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[boolean-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowInt8ArrayBuilder,
+              garrow_int8_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int8_array_builder_init(GArrowInt8ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int8_array_builder_class_init(GArrowInt8ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int8_array_builder_new:
+ *
+ * Returns: A newly created #GArrowInt8ArrayBuilder.
+ */
+GArrowInt8ArrayBuilder *
+garrow_int8_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_int8_builder =
+    std::make_shared<arrow::Int8Builder>(memory_pool, arrow::int8());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_int8_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_INT8_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_int8_array_builder_append:
+ * @builder: A #GArrowInt8ArrayBuilder.
+ * @value: A int8 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
+                                 gint8 value,
+                                 GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int8Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int8-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_int8_array_builder_append_null:
+ * @builder: A #GArrowInt8ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
+                                      GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int8Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int8-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowUInt8ArrayBuilder,
+              garrow_uint8_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint8_array_builder_init(GArrowUInt8ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint8_array_builder_class_init(GArrowUInt8ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint8_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUInt8ArrayBuilder.
+ */
+GArrowUInt8ArrayBuilder *
+garrow_uint8_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_uint8_builder =
+    std::make_shared<arrow::UInt8Builder>(memory_pool, arrow::uint8());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_uint8_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_UINT8_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_uint8_array_builder_append:
+ * @builder: A #GArrowUInt8ArrayBuilder.
+ * @value: An uint8 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
+                                  guint8 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt8Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint8-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_uint8_array_builder_append_null:
+ * @builder: A #GArrowUInt8ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt8Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint8-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowInt16ArrayBuilder,
+              garrow_int16_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int16_array_builder_init(GArrowInt16ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int16_array_builder_class_init(GArrowInt16ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int16_array_builder_new:
+ *
+ * Returns: A newly created #GArrowInt16ArrayBuilder.
+ */
+GArrowInt16ArrayBuilder *
+garrow_int16_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_int16_builder =
+    std::make_shared<arrow::Int16Builder>(memory_pool, arrow::int16());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_int16_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_INT16_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_int16_array_builder_append:
+ * @builder: A #GArrowInt16ArrayBuilder.
+ * @value: A int16 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
+                                  gint16 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int16Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int16-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_int16_array_builder_append_null:
+ * @builder: A #GArrowInt16ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int16Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int16-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowUInt16ArrayBuilder,
+              garrow_uint16_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint16_array_builder_init(GArrowUInt16ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint16_array_builder_class_init(GArrowUInt16ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint16_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUInt16ArrayBuilder.
+ */
+GArrowUInt16ArrayBuilder *
+garrow_uint16_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_uint16_builder =
+    std::make_shared<arrow::UInt16Builder>(memory_pool, arrow::uint16());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_uint16_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_UINT16_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_uint16_array_builder_append:
+ * @builder: A #GArrowUInt16ArrayBuilder.
+ * @value: An uint16 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
+                                   guint16 value,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt16Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint16-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_uint16_array_builder_append_null:
+ * @builder: A #GArrowUInt16ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
+                                        GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt16Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint16-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowInt32ArrayBuilder,
+              garrow_int32_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int32_array_builder_init(GArrowInt32ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int32_array_builder_class_init(GArrowInt32ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int32_array_builder_new:
+ *
+ * Returns: A newly created #GArrowInt32ArrayBuilder.
+ */
+GArrowInt32ArrayBuilder *
+garrow_int32_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_int32_builder =
+    std::make_shared<arrow::Int32Builder>(memory_pool, arrow::int32());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_int32_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_INT32_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_int32_array_builder_append:
+ * @builder: A #GArrowInt32ArrayBuilder.
+ * @value: A int32 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
+                                  gint32 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int32Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int32-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_int32_array_builder_append_null:
+ * @builder: A #GArrowInt32ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
+                                      GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int32Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int32-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowUInt32ArrayBuilder,
+              garrow_uint32_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint32_array_builder_init(GArrowUInt32ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint32_array_builder_class_init(GArrowUInt32ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint32_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUInt32ArrayBuilder.
+ */
+GArrowUInt32ArrayBuilder *
+garrow_uint32_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_uint32_builder =
+    std::make_shared<arrow::UInt32Builder>(memory_pool, arrow::uint32());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_uint32_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_UINT32_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_uint32_array_builder_append:
+ * @builder: A #GArrowUInt32ArrayBuilder.
+ * @value: An uint32 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
+                                   guint32 value,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt32Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint32-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_uint32_array_builder_append_null:
+ * @builder: A #GArrowUInt32ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
+                                        GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt32Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint32-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowInt64ArrayBuilder,
+              garrow_int64_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int64_array_builder_init(GArrowInt64ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int64_array_builder_class_init(GArrowInt64ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int64_array_builder_new:
+ *
+ * Returns: A newly created #GArrowInt64ArrayBuilder.
+ */
+GArrowInt64ArrayBuilder *
+garrow_int64_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_int64_builder =
+    std::make_shared<arrow::Int64Builder>(memory_pool, arrow::int64());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_int64_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_INT64_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_int64_array_builder_append:
+ * @builder: A #GArrowInt64ArrayBuilder.
+ * @value: A int64 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
+                                  gint64 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int64Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int64-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_int64_array_builder_append_null:
+ * @builder: A #GArrowInt64ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::Int64Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[int64-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowUInt64ArrayBuilder,
+              garrow_uint64_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint64_array_builder_init(GArrowUInt64ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint64_array_builder_class_init(GArrowUInt64ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint64_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUInt64ArrayBuilder.
+ */
+GArrowUInt64ArrayBuilder *
+garrow_uint64_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_uint64_builder =
+    std::make_shared<arrow::UInt64Builder>(memory_pool, arrow::uint64());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_uint64_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_UINT64_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_uint64_array_builder_append:
+ * @builder: A #GArrowUInt64ArrayBuilder.
+ * @value: An uint64 value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
+                                  guint64 value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt64Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint64-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_uint64_array_builder_append_null:
+ * @builder: A #GArrowUInt64ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::UInt64Builder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[uint64-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowFloatArrayBuilder,
+              garrow_float_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_float_array_builder_init(GArrowFloatArrayBuilder *builder)
+{
+}
+
+static void
+garrow_float_array_builder_class_init(GArrowFloatArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_float_array_builder_new:
+ *
+ * Returns: A newly created #GArrowFloatArrayBuilder.
+ */
+GArrowFloatArrayBuilder *
+garrow_float_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_float_builder =
+    std::make_shared<arrow::FloatBuilder>(memory_pool, arrow::float32());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_float_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_FLOAT_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_float_array_builder_append:
+ * @builder: A #GArrowFloatArrayBuilder.
+ * @value: A float value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
+                                  gfloat value,
+                                  GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::FloatBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[float-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_float_array_builder_append_null:
+ * @builder: A #GArrowFloatArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
+                                       GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::FloatBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[float-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowDoubleArrayBuilder,
+              garrow_double_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_double_array_builder_init(GArrowDoubleArrayBuilder *builder)
+{
+}
+
+static void
+garrow_double_array_builder_class_init(GArrowDoubleArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_double_array_builder_new:
+ *
+ * Returns: A newly created #GArrowDoubleArrayBuilder.
+ */
+GArrowDoubleArrayBuilder *
+garrow_double_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_double_builder =
+    std::make_shared<arrow::DoubleBuilder>(memory_pool, arrow::float64());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_double_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_DOUBLE_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_double_array_builder_append:
+ * @builder: A #GArrowDoubleArrayBuilder.
+ * @value: A double value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
+                                   gdouble value,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::DoubleBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[double-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_double_array_builder_append_null:
+ * @builder: A #GArrowDoubleArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
+                                        GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::DoubleBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[double-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowBinaryArrayBuilder,
+              garrow_binary_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_binary_array_builder_init(GArrowBinaryArrayBuilder *builder)
+{
+}
+
+static void
+garrow_binary_array_builder_class_init(GArrowBinaryArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_binary_array_builder_new:
+ *
+ * Returns: A newly created #GArrowBinaryArrayBuilder.
+ */
+GArrowBinaryArrayBuilder *
+garrow_binary_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_binary_builder =
+    std::make_shared<arrow::BinaryBuilder>(memory_pool, arrow::binary());
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_binary_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_BINARY_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_binary_array_builder_append:
+ * @builder: A #GArrowBinaryArrayBuilder.
+ * @value: (array length=length): A binary value.
+ * @length: A value length.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_binary_array_builder_append(GArrowBinaryArrayBuilder *builder,
+                                   const guint8 *value,
+                                   gint32 length,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::BinaryBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value, length);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[binary-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_binary_array_builder_append_null:
+ * @builder: A #GArrowBinaryArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
+                                        GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::BinaryBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[binary-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowStringArrayBuilder,
+              garrow_string_array_builder,
+              GARROW_TYPE_BINARY_ARRAY_BUILDER)
+
+static void
+garrow_string_array_builder_init(GArrowStringArrayBuilder *builder)
+{
+}
+
+static void
+garrow_string_array_builder_class_init(GArrowStringArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_string_array_builder_new:
+ *
+ * Returns: A newly created #GArrowStringArrayBuilder.
+ */
+GArrowStringArrayBuilder *
+garrow_string_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_string_builder =
+    std::make_shared<arrow::StringBuilder>(memory_pool);
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_string_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_STRING_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_string_array_builder_append:
+ * @builder: A #GArrowStringArrayBuilder.
+ * @value: A string value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ */
+gboolean
+garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
+                                   const gchar *value,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::StringBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append(value,
+                                      static_cast<gint32>(strlen(value)));
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[string-array-builder][append]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowListArrayBuilder,
+              garrow_list_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_list_array_builder_init(GArrowListArrayBuilder *builder)
+{
+}
+
+static void
+garrow_list_array_builder_class_init(GArrowListArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_list_array_builder_new:
+ * @value_builder: A #GArrowArrayBuilder for value array.
+ *
+ * Returns: A newly created #GArrowListArrayBuilder.
+ */
+GArrowListArrayBuilder *
+garrow_list_array_builder_new(GArrowArrayBuilder *value_builder)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_value_builder = garrow_array_builder_get_raw(value_builder);
+  auto arrow_list_builder =
+    std::make_shared<arrow::ListBuilder>(memory_pool, arrow_value_builder);
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_list_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_LIST_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_list_array_builder_append:
+ * @builder: A #GArrowListArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * It appends a new list element. To append a new list element, you
+ * need to call this function then append list element values to
+ * `value_builder`. `value_builder` is the #GArrowArrayBuilder
+ * specified to constructor. You can get `value_builder` by
+ * garrow_list_array_builder_get_value_builder().
+ *
+ * |[<!-- language="C" -->
+ * GArrowInt8ArrayBuilder *value_builder;
+ * GArrowListArrayBuilder *builder;
+ *
+ * value_builder = garrow_int8_array_builder_new();
+ * builder = garrow_list_array_builder_new(value_builder, NULL);
+ *
+ * // Start 0th list element: [1, 0, -1]
+ * garrow_list_array_builder_append(builder, NULL);
+ * garrow_int8_array_builder_append(value_builder, 1);
+ * garrow_int8_array_builder_append(value_builder, 0);
+ * garrow_int8_array_builder_append(value_builder, -1);
+ *
+ * // Start 1st list element: [-29, 29]
+ * garrow_list_array_builder_append(builder, NULL);
+ * garrow_int8_array_builder_append(value_builder, -29);
+ * garrow_int8_array_builder_append(value_builder, 29);
+ *
+ * {
+ *   // [[1, 0, -1], [-29, 29]]
+ *   GArrowArray *array = garrow_array_builder_finish(builder);
+ *   // Now, builder is needless.
+ *   g_object_unref(builder);
+ *   g_object_unref(value_builder);
+ *
+ *   // Use array...
+ *   g_object_unref(array);
+ * }
+ * ]|
+ */
+gboolean
+garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
+                                 GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::ListBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[list-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_list_array_builder_append_null:
+ * @builder: A #GArrowListArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * It appends a new NULL element.
+ */
+gboolean
+garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
+                                      GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::ListBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[list-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_list_array_builder_get_value_builder:
+ * @builder: A #GArrowListArrayBuilder.
+ *
+ * Returns: (transfer full): The #GArrowArrayBuilder for values.
+ */
+GArrowArrayBuilder *
+garrow_list_array_builder_get_value_builder(GArrowListArrayBuilder *builder)
+{
+  auto arrow_builder =
+    static_cast<arrow::ListBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+  auto arrow_value_builder = arrow_builder->value_builder();
+  return garrow_array_builder_new_raw(&arrow_value_builder);
+}
+
+
+G_DEFINE_TYPE(GArrowStructArrayBuilder,
+              garrow_struct_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_struct_array_builder_init(GArrowStructArrayBuilder *builder)
+{
+}
+
+static void
+garrow_struct_array_builder_class_init(GArrowStructArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_struct_array_builder_new:
+ * @data_type: #GArrowStructDataType for the struct.
+ * @field_builders: (element-type GArrowArray): #GArrowArrayBuilders
+ *   for fields.
+ *
+ * Returns: A newly created #GArrowStructArrayBuilder.
+ */
+GArrowStructArrayBuilder *
+garrow_struct_array_builder_new(GArrowStructDataType *data_type,
+                                GList *field_builders)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  std::vector<std::shared_ptr<arrow::ArrayBuilder>> arrow_field_builders;
+  for (GList *node = field_builders; node; node = g_list_next(node)) {
+    auto field_builder = static_cast<GArrowArrayBuilder *>(node->data);
+    auto arrow_field_builder = garrow_array_builder_get_raw(field_builder);
+    arrow_field_builders.push_back(arrow_field_builder);
+  }
+
+  auto arrow_struct_builder =
+    std::make_shared<arrow::StructBuilder>(memory_pool,
+                                           arrow_data_type,
+                                           arrow_field_builders);
+  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_struct_builder;
+  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  return GARROW_STRUCT_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_struct_array_builder_append:
+ * @builder: A #GArrowStructArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * It appends a new struct element. To append a new struct element,
+ * you need to call this function then append struct element field
+ * values to all `field_builder`s. `field_value`s are the
+ * #GArrowArrayBuilder specified to constructor. You can get
+ * `field_builder` by garrow_struct_array_builder_get_field_builder()
+ * or garrow_struct_array_builder_get_field_builders().
+ *
+ * |[<!-- language="C" -->
+ * // TODO
+ * ]|
+ */
+gboolean
+garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
+                                   GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::StructBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->Append();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[struct-array-builder][append]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_struct_array_builder_append_null:
+ * @builder: A #GArrowStructArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * It appends a new NULL element.
+ */
+gboolean
+garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
+                                        GError **error)
+{
+  auto arrow_builder =
+    static_cast<arrow::StructBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  auto status = arrow_builder->AppendNull();
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[struct-array-builder][append-null]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_struct_array_builder_get_field_builder:
+ * @builder: A #GArrowStructArrayBuilder.
+ * @i: The index of the field in the struct.
+ *
+ * Returns: (transfer full): The #GArrowArrayBuilder for the i-th field.
+ */
+GArrowArrayBuilder *
+garrow_struct_array_builder_get_field_builder(GArrowStructArrayBuilder *builder,
+                                              gint i)
+{
+  auto arrow_builder =
+    static_cast<arrow::StructBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+  auto arrow_field_builder = arrow_builder->field_builder(i);
+  return garrow_array_builder_new_raw(&arrow_field_builder);
+}
+
+/**
+ * garrow_struct_array_builder_get_field_builders:
+ * @builder: A #GArrowStructArrayBuilder.
+ *
+ * Returns: (element-type GArrowArray) (transfer full):
+ *   The #GArrowArrayBuilder for all fields.
+ */
+GList *
+garrow_struct_array_builder_get_field_builders(GArrowStructArrayBuilder *builder)
+{
+  auto arrow_struct_builder =
+    static_cast<arrow::StructBuilder *>(
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+
+  GList *field_builders = NULL;
+  for (auto arrow_field_builder : arrow_struct_builder->field_builders()) {
+    auto field_builder = garrow_array_builder_new_raw(&arrow_field_builder);
+    field_builders = g_list_prepend(field_builders, field_builder);
+  }
+
+  return g_list_reverse(field_builders);
+}
+
+
 G_END_DECLS
 
 GArrowArrayBuilder *
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index 3717aef04a2f4..ad72f9ae8488b 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -67,4 +67,773 @@ GType               garrow_array_builder_get_type (void) G_GNUC_CONST;
 
 GArrowArray        *garrow_array_builder_finish   (GArrowArrayBuilder *builder);
 
+
+#define GARROW_TYPE_BOOLEAN_ARRAY_BUILDER       \
+  (garrow_boolean_array_builder_get_type())
+#define GARROW_BOOLEAN_ARRAY_BUILDER(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_BOOLEAN_ARRAY_BUILDER,        \
+                              GArrowBooleanArrayBuilder))
+#define GARROW_BOOLEAN_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BOOLEAN_ARRAY_BUILDER,   \
+                           GArrowBooleanArrayBuilderClass))
+#define GARROW_IS_BOOLEAN_ARRAY_BUILDER(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_BOOLEAN_ARRAY_BUILDER))
+#define GARROW_IS_BOOLEAN_ARRAY_BUILDER_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BOOLEAN_ARRAY_BUILDER))
+#define GARROW_BOOLEAN_ARRAY_BUILDER_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BOOLEAN_ARRAY_BUILDER, \
+                             GArrowBooleanArrayBuilderClass))
+
+typedef struct _GArrowBooleanArrayBuilder         GArrowBooleanArrayBuilder;
+typedef struct _GArrowBooleanArrayBuilderClass    GArrowBooleanArrayBuilderClass;
+
+/**
+ * GArrowBooleanArrayBuilder:
+ *
+ * It wraps `arrow::BooleanBuilder`.
+ */
+struct _GArrowBooleanArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowBooleanArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_boolean_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowBooleanArrayBuilder *garrow_boolean_array_builder_new(void);
+
+gboolean garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
+                                             gboolean value,
+                                             GError **error);
+gboolean garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
+                                                  GError **error);
+
+
+#define GARROW_TYPE_INT8_ARRAY_BUILDER          \
+  (garrow_int8_array_builder_get_type())
+#define GARROW_INT8_ARRAY_BUILDER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT8_ARRAY_BUILDER,   \
+                              GArrowInt8ArrayBuilder))
+#define GARROW_INT8_ARRAY_BUILDER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT8_ARRAY_BUILDER,      \
+                           GArrowInt8ArrayBuilderClass))
+#define GARROW_IS_INT8_ARRAY_BUILDER(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT8_ARRAY_BUILDER))
+#define GARROW_IS_INT8_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT8_ARRAY_BUILDER))
+#define GARROW_INT8_ARRAY_BUILDER_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT8_ARRAY_BUILDER,    \
+                             GArrowInt8ArrayBuilderClass))
+
+typedef struct _GArrowInt8ArrayBuilder         GArrowInt8ArrayBuilder;
+typedef struct _GArrowInt8ArrayBuilderClass    GArrowInt8ArrayBuilderClass;
+
+/**
+ * GArrowInt8ArrayBuilder:
+ *
+ * It wraps `arrow::Int8Builder`.
+ */
+struct _GArrowInt8ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowInt8ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int8_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowInt8ArrayBuilder *garrow_int8_array_builder_new(void);
+
+gboolean garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
+                                          gint8 value,
+                                          GError **error);
+gboolean garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
+                                               GError **error);
+
+
+#define GARROW_TYPE_UINT8_ARRAY_BUILDER         \
+  (garrow_uint8_array_builder_get_type())
+#define GARROW_UINT8_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT8_ARRAY_BUILDER,  \
+                              GArrowUInt8ArrayBuilder))
+#define GARROW_UINT8_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT8_ARRAY_BUILDER,     \
+                           GArrowUInt8ArrayBuilderClass))
+#define GARROW_IS_UINT8_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT8_ARRAY_BUILDER))
+#define GARROW_IS_UINT8_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT8_ARRAY_BUILDER))
+#define GARROW_UINT8_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT8_ARRAY_BUILDER,   \
+                             GArrowUInt8ArrayBuilderClass))
+
+typedef struct _GArrowUInt8ArrayBuilder         GArrowUInt8ArrayBuilder;
+typedef struct _GArrowUInt8ArrayBuilderClass    GArrowUInt8ArrayBuilderClass;
+
+/**
+ * GArrowUInt8ArrayBuilder:
+ *
+ * It wraps `arrow::UInt8Builder`.
+ */
+struct _GArrowUInt8ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowUInt8ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_uint8_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowUInt8ArrayBuilder *garrow_uint8_array_builder_new(void);
+
+gboolean garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
+                                           guint8 value,
+                                           GError **error);
+gboolean garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
+                                                GError **error);
+
+
+#define GARROW_TYPE_INT16_ARRAY_BUILDER         \
+  (garrow_int16_array_builder_get_type())
+#define GARROW_INT16_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT16_ARRAY_BUILDER,  \
+                              GArrowInt16ArrayBuilder))
+#define GARROW_INT16_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT16_ARRAY_BUILDER,     \
+                           GArrowInt16ArrayBuilderClass))
+#define GARROW_IS_INT16_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT16_ARRAY_BUILDER))
+#define GARROW_IS_INT16_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT16_ARRAY_BUILDER))
+#define GARROW_INT16_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT16_ARRAY_BUILDER,   \
+                             GArrowInt16ArrayBuilderClass))
+
+typedef struct _GArrowInt16ArrayBuilder         GArrowInt16ArrayBuilder;
+typedef struct _GArrowInt16ArrayBuilderClass    GArrowInt16ArrayBuilderClass;
+
+/**
+ * GArrowInt16ArrayBuilder:
+ *
+ * It wraps `arrow::Int16Builder`.
+ */
+struct _GArrowInt16ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowInt16ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int16_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowInt16ArrayBuilder *garrow_int16_array_builder_new(void);
+
+gboolean garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
+                                           gint16 value,
+                                           GError **error);
+gboolean garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
+                                                GError **error);
+
+
+#define GARROW_TYPE_UINT16_ARRAY_BUILDER        \
+  (garrow_uint16_array_builder_get_type())
+#define GARROW_UINT16_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT16_ARRAY_BUILDER, \
+                              GArrowUInt16ArrayBuilder))
+#define GARROW_UINT16_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT16_ARRAY_BUILDER,    \
+                           GArrowUInt16ArrayBuilderClass))
+#define GARROW_IS_UINT16_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_UINT16_ARRAY_BUILDER))
+#define GARROW_IS_UINT16_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT16_ARRAY_BUILDER))
+#define GARROW_UINT16_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT16_ARRAY_BUILDER,  \
+                             GArrowUInt16ArrayBuilderClass))
+
+typedef struct _GArrowUInt16ArrayBuilder         GArrowUInt16ArrayBuilder;
+typedef struct _GArrowUInt16ArrayBuilderClass    GArrowUInt16ArrayBuilderClass;
+
+/**
+ * GArrowUInt16ArrayBuilder:
+ *
+ * It wraps `arrow::UInt16Builder`.
+ */
+struct _GArrowUInt16ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowUInt16ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_uint16_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowUInt16ArrayBuilder *garrow_uint16_array_builder_new(void);
+
+gboolean garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
+                                            guint16 value,
+                                            GError **error);
+gboolean garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
+                                                 GError **error);
+
+
+#define GARROW_TYPE_INT32_ARRAY_BUILDER         \
+  (garrow_int32_array_builder_get_type())
+#define GARROW_INT32_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT32_ARRAY_BUILDER,  \
+                              GArrowInt32ArrayBuilder))
+#define GARROW_INT32_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT32_ARRAY_BUILDER,     \
+                           GArrowInt32ArrayBuilderClass))
+#define GARROW_IS_INT32_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT32_ARRAY_BUILDER))
+#define GARROW_IS_INT32_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT32_ARRAY_BUILDER))
+#define GARROW_INT32_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT32_ARRAY_BUILDER,   \
+                             GArrowInt32ArrayBuilderClass))
+
+typedef struct _GArrowInt32ArrayBuilder         GArrowInt32ArrayBuilder;
+typedef struct _GArrowInt32ArrayBuilderClass    GArrowInt32ArrayBuilderClass;
+
+/**
+ * GArrowInt32ArrayBuilder:
+ *
+ * It wraps `arrow::Int32Builder`.
+ */
+struct _GArrowInt32ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowInt32ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int32_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowInt32ArrayBuilder *garrow_int32_array_builder_new(void);
+
+gboolean garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
+                                           gint32 value,
+                                           GError **error);
+gboolean garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
+                                                GError **error);
+
+
+#define GARROW_TYPE_UINT32_ARRAY_BUILDER        \
+  (garrow_uint32_array_builder_get_type())
+#define GARROW_UINT32_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT32_ARRAY_BUILDER, \
+                              GArrowUInt32ArrayBuilder))
+#define GARROW_UINT32_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT32_ARRAY_BUILDER,    \
+                           GArrowUInt32ArrayBuilderClass))
+#define GARROW_IS_UINT32_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_UINT32_ARRAY_BUILDER))
+#define GARROW_IS_UINT32_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT32_ARRAY_BUILDER))
+#define GARROW_UINT32_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT32_ARRAY_BUILDER,  \
+                             GArrowUInt32ArrayBuilderClass))
+
+typedef struct _GArrowUInt32ArrayBuilder         GArrowUInt32ArrayBuilder;
+typedef struct _GArrowUInt32ArrayBuilderClass    GArrowUInt32ArrayBuilderClass;
+
+/**
+ * GArrowUInt32ArrayBuilder:
+ *
+ * It wraps `arrow::UInt32Builder`.
+ */
+struct _GArrowUInt32ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowUInt32ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_uint32_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowUInt32ArrayBuilder *garrow_uint32_array_builder_new(void);
+
+gboolean garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
+                                            guint32 value,
+                                            GError **error);
+gboolean garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
+                                                 GError **error);
+
+
+#define GARROW_TYPE_INT64_ARRAY_BUILDER         \
+  (garrow_int64_array_builder_get_type())
+#define GARROW_INT64_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT64_ARRAY_BUILDER,  \
+                              GArrowInt64ArrayBuilder))
+#define GARROW_INT64_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT64_ARRAY_BUILDER,     \
+                           GArrowInt64ArrayBuilderClass))
+#define GARROW_IS_INT64_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT64_ARRAY_BUILDER))
+#define GARROW_IS_INT64_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT64_ARRAY_BUILDER))
+#define GARROW_INT64_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT64_ARRAY_BUILDER,   \
+                             GArrowInt64ArrayBuilderClass))
+
+typedef struct _GArrowInt64ArrayBuilder         GArrowInt64ArrayBuilder;
+typedef struct _GArrowInt64ArrayBuilderClass    GArrowInt64ArrayBuilderClass;
+
+/**
+ * GArrowInt64ArrayBuilder:
+ *
+ * It wraps `arrow::Int64Builder`.
+ */
+struct _GArrowInt64ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowInt64ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int64_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowInt64ArrayBuilder *garrow_int64_array_builder_new(void);
+
+gboolean garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
+                                           gint64 value,
+                                           GError **error);
+gboolean garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
+                                                GError **error);
+
+
+#define GARROW_TYPE_UINT64_ARRAY_BUILDER        \
+  (garrow_uint64_array_builder_get_type())
+#define GARROW_UINT64_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT64_ARRAY_BUILDER, \
+                              GArrowUInt64ArrayBuilder))
+#define GARROW_UINT64_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT64_ARRAY_BUILDER,    \
+                           GArrowUInt64ArrayBuilderClass))
+#define GARROW_IS_UINT64_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_UINT64_ARRAY_BUILDER))
+#define GARROW_IS_UINT64_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT64_ARRAY_BUILDER))
+#define GARROW_UINT64_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT64_ARRAY_BUILDER,  \
+                             GArrowUInt64ArrayBuilderClass))
+
+typedef struct _GArrowUInt64ArrayBuilder         GArrowUInt64ArrayBuilder;
+typedef struct _GArrowUInt64ArrayBuilderClass    GArrowUInt64ArrayBuilderClass;
+
+/**
+ * GArrowUInt64ArrayBuilder:
+ *
+ * It wraps `arrow::UInt64Builder`.
+ */
+struct _GArrowUInt64ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowUInt64ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_uint64_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowUInt64ArrayBuilder *garrow_uint64_array_builder_new(void);
+
+gboolean garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
+                                            guint64 value,
+                                            GError **error);
+gboolean garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
+                                                 GError **error);
+
+
+#define GARROW_TYPE_FLOAT_ARRAY_BUILDER         \
+  (garrow_float_array_builder_get_type())
+#define GARROW_FLOAT_ARRAY_BUILDER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_FLOAT_ARRAY_BUILDER,  \
+                              GArrowFloatArrayBuilder))
+#define GARROW_FLOAT_ARRAY_BUILDER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_FLOAT_ARRAY_BUILDER,     \
+                           GArrowFloatArrayBuilderClass))
+#define GARROW_IS_FLOAT_ARRAY_BUILDER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_FLOAT_ARRAY_BUILDER))
+#define GARROW_IS_FLOAT_ARRAY_BUILDER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_FLOAT_ARRAY_BUILDER))
+#define GARROW_FLOAT_ARRAY_BUILDER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_FLOAT_ARRAY_BUILDER,   \
+                             GArrowFloatArrayBuilderClass))
+
+typedef struct _GArrowFloatArrayBuilder         GArrowFloatArrayBuilder;
+typedef struct _GArrowFloatArrayBuilderClass    GArrowFloatArrayBuilderClass;
+
+/**
+ * GArrowFloatArrayBuilder:
+ *
+ * It wraps `arrow::FloatBuilder`.
+ */
+struct _GArrowFloatArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowFloatArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_float_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowFloatArrayBuilder *garrow_float_array_builder_new(void);
+
+gboolean garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
+                                           gfloat value,
+                                           GError **error);
+gboolean garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
+                                                GError **error);
+
+
+#define GARROW_TYPE_DOUBLE_ARRAY_BUILDER        \
+  (garrow_double_array_builder_get_type())
+#define GARROW_DOUBLE_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DOUBLE_ARRAY_BUILDER, \
+                              GArrowDoubleArrayBuilder))
+#define GARROW_DOUBLE_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DOUBLE_ARRAY_BUILDER,    \
+                           GArrowDoubleArrayBuilderClass))
+#define GARROW_IS_DOUBLE_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_DOUBLE_ARRAY_BUILDER))
+#define GARROW_IS_DOUBLE_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DOUBLE_ARRAY_BUILDER))
+#define GARROW_DOUBLE_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DOUBLE_ARRAY_BUILDER,  \
+                             GArrowDoubleArrayBuilderClass))
+
+typedef struct _GArrowDoubleArrayBuilder         GArrowDoubleArrayBuilder;
+typedef struct _GArrowDoubleArrayBuilderClass    GArrowDoubleArrayBuilderClass;
+
+/**
+ * GArrowDoubleArrayBuilder:
+ *
+ * It wraps `arrow::DoubleBuilder`.
+ */
+struct _GArrowDoubleArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowDoubleArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_double_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowDoubleArrayBuilder *garrow_double_array_builder_new(void);
+
+gboolean garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
+                                            gdouble value,
+                                            GError **error);
+gboolean garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
+                                                 GError **error);
+
+
+#define GARROW_TYPE_BINARY_ARRAY_BUILDER        \
+  (garrow_binary_array_builder_get_type())
+#define GARROW_BINARY_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BINARY_ARRAY_BUILDER, \
+                              GArrowBinaryArrayBuilder))
+#define GARROW_BINARY_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BINARY_ARRAY_BUILDER,    \
+                           GArrowBinaryArrayBuilderClass))
+#define GARROW_IS_BINARY_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_BINARY_ARRAY_BUILDER))
+#define GARROW_IS_BINARY_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BINARY_ARRAY_BUILDER))
+#define GARROW_BINARY_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BINARY_ARRAY_BUILDER,  \
+                             GArrowBinaryArrayBuilderClass))
+
+typedef struct _GArrowBinaryArrayBuilder         GArrowBinaryArrayBuilder;
+typedef struct _GArrowBinaryArrayBuilderClass    GArrowBinaryArrayBuilderClass;
+
+/**
+ * GArrowBinaryArrayBuilder:
+ *
+ * It wraps `arrow::BinaryBuilder`.
+ */
+struct _GArrowBinaryArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowBinaryArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_binary_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowBinaryArrayBuilder *garrow_binary_array_builder_new(void);
+
+gboolean garrow_binary_array_builder_append(GArrowBinaryArrayBuilder *builder,
+                                            const guint8 *value,
+                                            gint32 length,
+                                            GError **error);
+gboolean garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
+                                                 GError **error);
+
+
+#define GARROW_TYPE_STRING_ARRAY_BUILDER        \
+  (garrow_string_array_builder_get_type())
+#define GARROW_STRING_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STRING_ARRAY_BUILDER, \
+                              GArrowStringArrayBuilder))
+#define GARROW_STRING_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_STRING_ARRAY_BUILDER,    \
+                           GArrowStringArrayBuilderClass))
+#define GARROW_IS_STRING_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_STRING_ARRAY_BUILDER))
+#define GARROW_IS_STRING_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_STRING_ARRAY_BUILDER))
+#define GARROW_STRING_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_STRING_ARRAY_BUILDER,  \
+                             GArrowStringArrayBuilderClass))
+
+typedef struct _GArrowStringArrayBuilder         GArrowStringArrayBuilder;
+typedef struct _GArrowStringArrayBuilderClass    GArrowStringArrayBuilderClass;
+
+/**
+ * GArrowStringArrayBuilder:
+ *
+ * It wraps `arrow::StringBuilder`.
+ */
+struct _GArrowStringArrayBuilder
+{
+  /*< private >*/
+  GArrowBinaryArrayBuilder parent_instance;
+};
+
+struct _GArrowStringArrayBuilderClass
+{
+  GArrowBinaryArrayBuilderClass parent_class;
+};
+
+GType garrow_string_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowStringArrayBuilder *garrow_string_array_builder_new(void);
+
+gboolean garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
+                                            const gchar *value,
+                                            GError **error);
+
+
+#define GARROW_TYPE_LIST_ARRAY_BUILDER          \
+  (garrow_list_array_builder_get_type())
+#define GARROW_LIST_ARRAY_BUILDER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_LIST_ARRAY_BUILDER,   \
+                              GArrowListArrayBuilder))
+#define GARROW_LIST_ARRAY_BUILDER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_LIST_ARRAY_BUILDER,      \
+                           GArrowListArrayBuilderClass))
+#define GARROW_IS_LIST_ARRAY_BUILDER(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_LIST_ARRAY_BUILDER))
+#define GARROW_IS_LIST_ARRAY_BUILDER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_LIST_ARRAY_BUILDER))
+#define GARROW_LIST_ARRAY_BUILDER_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_LIST_ARRAY_BUILDER,    \
+                             GArrowListArrayBuilderClass))
+
+typedef struct _GArrowListArrayBuilder         GArrowListArrayBuilder;
+typedef struct _GArrowListArrayBuilderClass    GArrowListArrayBuilderClass;
+
+/**
+ * GArrowListArrayBuilder:
+ *
+ * It wraps `arrow::ListBuilder`.
+ */
+struct _GArrowListArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowListArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_list_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowListArrayBuilder *garrow_list_array_builder_new(GArrowArrayBuilder *value_builder);
+
+gboolean garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
+                                          GError **error);
+gboolean garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
+                                               GError **error);
+
+GArrowArrayBuilder *garrow_list_array_builder_get_value_builder(GArrowListArrayBuilder *builder);
+
+
+#define GARROW_TYPE_STRUCT_ARRAY_BUILDER        \
+  (garrow_struct_array_builder_get_type())
+#define GARROW_STRUCT_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STRUCT_ARRAY_BUILDER, \
+                              GArrowStructArrayBuilder))
+#define GARROW_STRUCT_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_STRUCT_ARRAY_BUILDER,    \
+                           GArrowStructArrayBuilderClass))
+#define GARROW_IS_STRUCT_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_STRUCT_ARRAY_BUILDER))
+#define GARROW_IS_STRUCT_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_STRUCT_ARRAY_BUILDER))
+#define GARROW_STRUCT_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_STRUCT_ARRAY_BUILDER,  \
+                             GArrowStructArrayBuilderClass))
+
+typedef struct _GArrowStructArrayBuilder         GArrowStructArrayBuilder;
+typedef struct _GArrowStructArrayBuilderClass    GArrowStructArrayBuilderClass;
+
+/**
+ * GArrowStructArrayBuilder:
+ *
+ * It wraps `arrow::StructBuilder`.
+ */
+struct _GArrowStructArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowStructArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_struct_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowStructArrayBuilder *garrow_struct_array_builder_new(GArrowStructDataType *data_type,
+                                                          GList *field_builders);
+
+gboolean garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
+                                            GError **error);
+gboolean garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
+                                                 GError **error);
+
+GArrowArrayBuilder *garrow_struct_array_builder_get_field_builder(GArrowStructArrayBuilder *builder,
+                                                                  gint i);
+GList *garrow_struct_array_builder_get_field_builders(GArrowStructArrayBuilder *builder);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index 46e98d2b8ed4c..efff5710308a8 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -21,32 +21,17 @@
 
 #include <arrow-glib/array.h>
 #include <arrow-glib/array-builder.h>
-#include <arrow-glib/binary-array-builder.h>
-#include <arrow-glib/boolean-array-builder.h>
 #include <arrow-glib/chunked-array.h>
 #include <arrow-glib/column.h>
 #include <arrow-glib/data-type.h>
-#include <arrow-glib/double-array-builder.h>
 #include <arrow-glib/enums.h>
 #include <arrow-glib/error.h>
 #include <arrow-glib/field.h>
-#include <arrow-glib/float-array-builder.h>
-#include <arrow-glib/int8-array-builder.h>
-#include <arrow-glib/int16-array-builder.h>
-#include <arrow-glib/int32-array-builder.h>
-#include <arrow-glib/int64-array-builder.h>
-#include <arrow-glib/list-array-builder.h>
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
-#include <arrow-glib/string-array-builder.h>
-#include <arrow-glib/struct-array-builder.h>
 #include <arrow-glib/table.h>
 #include <arrow-glib/tensor.h>
 #include <arrow-glib/type.h>
-#include <arrow-glib/uint8-array-builder.h>
-#include <arrow-glib/uint16-array-builder.h>
-#include <arrow-glib/uint32-array-builder.h>
-#include <arrow-glib/uint64-array-builder.h>
 
 #include <arrow-glib/file.h>
 #include <arrow-glib/file-mode.h>
diff --git a/c_glib/arrow-glib/binary-array-builder.cpp b/c_glib/arrow-glib/binary-array-builder.cpp
deleted file mode 100644
index ab11535eb8595..0000000000000
--- a/c_glib/arrow-glib/binary-array-builder.cpp
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/binary-array-builder.h>
-#include <arrow-glib/error.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: binary-array-builder
- * @short_description: Binary array builder class
- *
- * #GArrowBinaryArrayBuilder is the class to create a new
- * #GArrowBinaryArray.
- */
-
-G_DEFINE_TYPE(GArrowBinaryArrayBuilder,
-              garrow_binary_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_binary_array_builder_init(GArrowBinaryArrayBuilder *builder)
-{
-}
-
-static void
-garrow_binary_array_builder_class_init(GArrowBinaryArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_binary_array_builder_new:
- *
- * Returns: A newly created #GArrowBinaryArrayBuilder.
- */
-GArrowBinaryArrayBuilder *
-garrow_binary_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::BinaryBuilder>(memory_pool, arrow::binary());
-  auto builder =
-    GARROW_BINARY_ARRAY_BUILDER(g_object_new(GARROW_TYPE_BINARY_ARRAY_BUILDER,
-                                             "array-builder", &arrow_builder,
-                                             NULL));
-  return builder;
-}
-
-/**
- * garrow_binary_array_builder_append:
- * @builder: A #GArrowBinaryArrayBuilder.
- * @value: (array length=length): A binary value.
- * @length: A value length.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_binary_array_builder_append(GArrowBinaryArrayBuilder *builder,
-                                   const guint8 *value,
-                                   gint32 length,
-                                   GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::BinaryBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value, length);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[binary-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_binary_array_builder_append_null:
- * @builder: A #GArrowBinaryArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
-                                        GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::BinaryBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[binary-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/binary-array-builder.h b/c_glib/arrow-glib/binary-array-builder.h
deleted file mode 100644
index 111a83a3a09b0..0000000000000
--- a/c_glib/arrow-glib/binary-array-builder.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_BINARY_ARRAY_BUILDER        \
-  (garrow_binary_array_builder_get_type())
-#define GARROW_BINARY_ARRAY_BUILDER(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BINARY_ARRAY_BUILDER, \
-                              GArrowBinaryArrayBuilder))
-#define GARROW_BINARY_ARRAY_BUILDER_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BINARY_ARRAY_BUILDER,    \
-                           GArrowBinaryArrayBuilderClass))
-#define GARROW_IS_BINARY_ARRAY_BUILDER(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_BINARY_ARRAY_BUILDER))
-#define GARROW_IS_BINARY_ARRAY_BUILDER_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BINARY_ARRAY_BUILDER))
-#define GARROW_BINARY_ARRAY_BUILDER_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BINARY_ARRAY_BUILDER,  \
-                             GArrowBinaryArrayBuilderClass))
-
-typedef struct _GArrowBinaryArrayBuilder         GArrowBinaryArrayBuilder;
-typedef struct _GArrowBinaryArrayBuilderClass    GArrowBinaryArrayBuilderClass;
-
-/**
- * GArrowBinaryArrayBuilder:
- *
- * It wraps `arrow::BinaryBuilder`.
- */
-struct _GArrowBinaryArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowBinaryArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_binary_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowBinaryArrayBuilder *garrow_binary_array_builder_new(void);
-
-gboolean garrow_binary_array_builder_append(GArrowBinaryArrayBuilder *builder,
-                                            const guint8 *value,
-                                            gint32 length,
-                                            GError **error);
-gboolean garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
-                                                 GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-array-builder.cpp b/c_glib/arrow-glib/boolean-array-builder.cpp
deleted file mode 100644
index 146eb31e8bdf8..0000000000000
--- a/c_glib/arrow-glib/boolean-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/boolean-array-builder.h>
-#include <arrow-glib/error.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: boolean-array-builder
- * @short_description: Boolean array builder class
- *
- * #GArrowBooleanArrayBuilder is the class to create a new
- * #GArrowBooleanArray.
- */
-
-G_DEFINE_TYPE(GArrowBooleanArrayBuilder,
-              garrow_boolean_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_boolean_array_builder_init(GArrowBooleanArrayBuilder *builder)
-{
-}
-
-static void
-garrow_boolean_array_builder_class_init(GArrowBooleanArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_boolean_array_builder_new:
- *
- * Returns: A newly created #GArrowBooleanArrayBuilder.
- */
-GArrowBooleanArrayBuilder *
-garrow_boolean_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::BooleanBuilder>(memory_pool);
-  auto builder =
-    GARROW_BOOLEAN_ARRAY_BUILDER(g_object_new(GARROW_TYPE_BOOLEAN_ARRAY_BUILDER,
-                                              "array-builder", &arrow_builder,
-                                              NULL));
-  return builder;
-}
-
-/**
- * garrow_boolean_array_builder_append:
- * @builder: A #GArrowBooleanArrayBuilder.
- * @value: A boolean value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
-                                    gboolean value,
-                                    GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::BooleanBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(static_cast<bool>(value));
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[boolean-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_boolean_array_builder_append_null:
- * @builder: A #GArrowBooleanArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
-                                         GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::BooleanBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[boolean-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/boolean-array-builder.h b/c_glib/arrow-glib/boolean-array-builder.h
deleted file mode 100644
index ca50e9797d41c..0000000000000
--- a/c_glib/arrow-glib/boolean-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_BOOLEAN_ARRAY_BUILDER \
-  (garrow_boolean_array_builder_get_type())
-#define GARROW_BOOLEAN_ARRAY_BUILDER(obj)                               \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
-                              GARROW_TYPE_BOOLEAN_ARRAY_BUILDER,        \
-                              GArrowBooleanArrayBuilder))
-#define GARROW_BOOLEAN_ARRAY_BUILDER_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BOOLEAN_ARRAY_BUILDER,   \
-                           GArrowBooleanArrayBuilderClass))
-#define GARROW_IS_BOOLEAN_ARRAY_BUILDER(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_BOOLEAN_ARRAY_BUILDER))
-#define GARROW_IS_BOOLEAN_ARRAY_BUILDER_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BOOLEAN_ARRAY_BUILDER))
-#define GARROW_BOOLEAN_ARRAY_BUILDER_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BOOLEAN_ARRAY_BUILDER, \
-                             GArrowBooleanArrayBuilderClass))
-
-typedef struct _GArrowBooleanArrayBuilder         GArrowBooleanArrayBuilder;
-typedef struct _GArrowBooleanArrayBuilderClass    GArrowBooleanArrayBuilderClass;
-
-/**
- * GArrowBooleanArrayBuilder:
- *
- * It wraps `arrow::BooleanBuilder`.
- */
-struct _GArrowBooleanArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowBooleanArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_boolean_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowBooleanArrayBuilder *garrow_boolean_array_builder_new(void);
-
-gboolean garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
-                                             gboolean value,
-                                             GError **error);
-gboolean garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
-                                                  GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/double-array-builder.cpp b/c_glib/arrow-glib/double-array-builder.cpp
deleted file mode 100644
index cc44eeabfb686..0000000000000
--- a/c_glib/arrow-glib/double-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/double-array-builder.h>
-#include <arrow-glib/error.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: double-array-builder
- * @short_description: 64-bit floating point array builder class
- *
- * #GArrowDoubleArrayBuilder is the class to create a new
- * #GArrowDoubleArray.
- */
-
-G_DEFINE_TYPE(GArrowDoubleArrayBuilder,
-              garrow_double_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_double_array_builder_init(GArrowDoubleArrayBuilder *builder)
-{
-}
-
-static void
-garrow_double_array_builder_class_init(GArrowDoubleArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_double_array_builder_new:
- *
- * Returns: A newly created #GArrowDoubleArrayBuilder.
- */
-GArrowDoubleArrayBuilder *
-garrow_double_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::DoubleBuilder>(memory_pool, arrow::float64());
-  auto builder =
-    GARROW_DOUBLE_ARRAY_BUILDER(g_object_new(GARROW_TYPE_DOUBLE_ARRAY_BUILDER,
-                                             "array-builder", &arrow_builder,
-                                             NULL));
-  return builder;
-}
-
-/**
- * garrow_double_array_builder_append:
- * @builder: A #GArrowDoubleArrayBuilder.
- * @value: A double value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
-                                   gdouble value,
-                                   GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::DoubleBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[double-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_double_array_builder_append_null:
- * @builder: A #GArrowDoubleArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
-                                        GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::DoubleBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[double-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/double-array-builder.h b/c_glib/arrow-glib/double-array-builder.h
deleted file mode 100644
index 5d95c898bc8a7..0000000000000
--- a/c_glib/arrow-glib/double-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_DOUBLE_ARRAY_BUILDER        \
-  (garrow_double_array_builder_get_type())
-#define GARROW_DOUBLE_ARRAY_BUILDER(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_DOUBLE_ARRAY_BUILDER, \
-                              GArrowDoubleArrayBuilder))
-#define GARROW_DOUBLE_ARRAY_BUILDER_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_DOUBLE_ARRAY_BUILDER,    \
-                           GArrowDoubleArrayBuilderClass))
-#define GARROW_IS_DOUBLE_ARRAY_BUILDER(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_DOUBLE_ARRAY_BUILDER))
-#define GARROW_IS_DOUBLE_ARRAY_BUILDER_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_DOUBLE_ARRAY_BUILDER))
-#define GARROW_DOUBLE_ARRAY_BUILDER_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_DOUBLE_ARRAY_BUILDER,  \
-                             GArrowDoubleArrayBuilderClass))
-
-typedef struct _GArrowDoubleArrayBuilder         GArrowDoubleArrayBuilder;
-typedef struct _GArrowDoubleArrayBuilderClass    GArrowDoubleArrayBuilderClass;
-
-/**
- * GArrowDoubleArrayBuilder:
- *
- * It wraps `arrow::DoubleBuilder`.
- */
-struct _GArrowDoubleArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowDoubleArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_double_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowDoubleArrayBuilder *garrow_double_array_builder_new(void);
-
-gboolean garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
-                                            gdouble value,
-                                            GError **error);
-gboolean garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
-                                                 GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/float-array-builder.cpp b/c_glib/arrow-glib/float-array-builder.cpp
deleted file mode 100644
index 77a9a0bb75a05..0000000000000
--- a/c_glib/arrow-glib/float-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/float-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: float-array-builder
- * @short_description: 32-bit floating point array builder class
- *
- * #GArrowFloatArrayBuilder is the class to creating a new
- * #GArrowFloatArray.
- */
-
-G_DEFINE_TYPE(GArrowFloatArrayBuilder,
-              garrow_float_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_float_array_builder_init(GArrowFloatArrayBuilder *builder)
-{
-}
-
-static void
-garrow_float_array_builder_class_init(GArrowFloatArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_float_array_builder_new:
- *
- * Returns: A newly created #GArrowFloatArrayBuilder.
- */
-GArrowFloatArrayBuilder *
-garrow_float_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::FloatBuilder>(memory_pool, arrow::float32());
-  auto builder =
-    GARROW_FLOAT_ARRAY_BUILDER(g_object_new(GARROW_TYPE_FLOAT_ARRAY_BUILDER,
-                                            "array-builder", &arrow_builder,
-                                            NULL));
-  return builder;
-}
-
-/**
- * garrow_float_array_builder_append:
- * @builder: A #GArrowFloatArrayBuilder.
- * @value: A float value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
-                                  gfloat value,
-                                  GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::FloatBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[float-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_float_array_builder_append_null:
- * @builder: A #GArrowFloatArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
-                                       GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::FloatBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[float-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/float-array-builder.h b/c_glib/arrow-glib/float-array-builder.h
deleted file mode 100644
index 003900313cca4..0000000000000
--- a/c_glib/arrow-glib/float-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_FLOAT_ARRAY_BUILDER         \
-  (garrow_float_array_builder_get_type())
-#define GARROW_FLOAT_ARRAY_BUILDER(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_FLOAT_ARRAY_BUILDER,  \
-                              GArrowFloatArrayBuilder))
-#define GARROW_FLOAT_ARRAY_BUILDER_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_FLOAT_ARRAY_BUILDER,     \
-                           GArrowFloatArrayBuilderClass))
-#define GARROW_IS_FLOAT_ARRAY_BUILDER(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_FLOAT_ARRAY_BUILDER))
-#define GARROW_IS_FLOAT_ARRAY_BUILDER_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_FLOAT_ARRAY_BUILDER))
-#define GARROW_FLOAT_ARRAY_BUILDER_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_FLOAT_ARRAY_BUILDER,   \
-                             GArrowFloatArrayBuilderClass))
-
-typedef struct _GArrowFloatArrayBuilder         GArrowFloatArrayBuilder;
-typedef struct _GArrowFloatArrayBuilderClass    GArrowFloatArrayBuilderClass;
-
-/**
- * GArrowFloatArrayBuilder:
- *
- * It wraps `arrow::FloatBuilder`.
- */
-struct _GArrowFloatArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowFloatArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_float_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowFloatArrayBuilder *garrow_float_array_builder_new(void);
-
-gboolean garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
-                                           gfloat value,
-                                           GError **error);
-gboolean garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
-                                                GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-array-builder.cpp b/c_glib/arrow-glib/int16-array-builder.cpp
deleted file mode 100644
index fbf18ef1e6ce7..0000000000000
--- a/c_glib/arrow-glib/int16-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/int16-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int16-array-builder
- * @short_description: 16-bit integer array builder class
- *
- * #GArrowInt16ArrayBuilder is the class to create a new
- * #GArrowInt16Array.
- */
-
-G_DEFINE_TYPE(GArrowInt16ArrayBuilder,
-              garrow_int16_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_int16_array_builder_init(GArrowInt16ArrayBuilder *builder)
-{
-}
-
-static void
-garrow_int16_array_builder_class_init(GArrowInt16ArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_int16_array_builder_new:
- *
- * Returns: A newly created #GArrowInt16ArrayBuilder.
- */
-GArrowInt16ArrayBuilder *
-garrow_int16_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::Int16Builder>(memory_pool, arrow::int16());
-  auto builder =
-    GARROW_INT16_ARRAY_BUILDER(g_object_new(GARROW_TYPE_INT16_ARRAY_BUILDER,
-                                           "array-builder", &arrow_builder,
-                                           NULL));
-  return builder;
-}
-
-/**
- * garrow_int16_array_builder_append:
- * @builder: A #GArrowInt16ArrayBuilder.
- * @value: A int16 value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
-                                 gint16 value,
-                                 GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::Int16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int16-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_int16_array_builder_append_null:
- * @builder: A #GArrowInt16ArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
-                                      GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::Int16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int16-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int16-array-builder.h b/c_glib/arrow-glib/int16-array-builder.h
deleted file mode 100644
index f222cfdccc9b7..0000000000000
--- a/c_glib/arrow-glib/int16-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT16_ARRAY_BUILDER          \
-  (garrow_int16_array_builder_get_type())
-#define GARROW_INT16_ARRAY_BUILDER(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT16_ARRAY_BUILDER,   \
-                              GArrowInt16ArrayBuilder))
-#define GARROW_INT16_ARRAY_BUILDER_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_INT16_ARRAY_BUILDER,      \
-                           GArrowInt16ArrayBuilderClass))
-#define GARROW_IS_INT16_ARRAY_BUILDER(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT16_ARRAY_BUILDER))
-#define GARROW_IS_INT16_ARRAY_BUILDER_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT16_ARRAY_BUILDER))
-#define GARROW_INT16_ARRAY_BUILDER_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT16_ARRAY_BUILDER,    \
-                             GArrowInt16ArrayBuilderClass))
-
-typedef struct _GArrowInt16ArrayBuilder         GArrowInt16ArrayBuilder;
-typedef struct _GArrowInt16ArrayBuilderClass    GArrowInt16ArrayBuilderClass;
-
-/**
- * GArrowInt16ArrayBuilder:
- *
- * It wraps `arrow::Int16Builder`.
- */
-struct _GArrowInt16ArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowInt16ArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_int16_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowInt16ArrayBuilder *garrow_int16_array_builder_new(void);
-
-gboolean garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
-                                          gint16 value,
-                                          GError **error);
-gboolean garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
-                                               GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-array-builder.cpp b/c_glib/arrow-glib/int32-array-builder.cpp
deleted file mode 100644
index 30cc4702f68fb..0000000000000
--- a/c_glib/arrow-glib/int32-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/int32-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int32-array-builder
- * @short_description: 32-bit integer array builder class
- *
- * #GArrowInt32ArrayBuilder is the class to create a new
- * #GArrowInt32Array.
- */
-
-G_DEFINE_TYPE(GArrowInt32ArrayBuilder,
-              garrow_int32_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_int32_array_builder_init(GArrowInt32ArrayBuilder *builder)
-{
-}
-
-static void
-garrow_int32_array_builder_class_init(GArrowInt32ArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_int32_array_builder_new:
- *
- * Returns: A newly created #GArrowInt32ArrayBuilder.
- */
-GArrowInt32ArrayBuilder *
-garrow_int32_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::Int32Builder>(memory_pool, arrow::int32());
-  auto builder =
-    GARROW_INT32_ARRAY_BUILDER(g_object_new(GARROW_TYPE_INT32_ARRAY_BUILDER,
-                                           "array-builder", &arrow_builder,
-                                           NULL));
-  return builder;
-}
-
-/**
- * garrow_int32_array_builder_append:
- * @builder: A #GArrowInt32ArrayBuilder.
- * @value: A int32 value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
-                                 gint32 value,
-                                 GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::Int32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int32-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_int32_array_builder_append_null:
- * @builder: A #GArrowInt32ArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
-                                      GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::Int32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int32-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int32-array-builder.h b/c_glib/arrow-glib/int32-array-builder.h
deleted file mode 100644
index bdb380d6070b0..0000000000000
--- a/c_glib/arrow-glib/int32-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT32_ARRAY_BUILDER         \
-  (garrow_int32_array_builder_get_type())
-#define GARROW_INT32_ARRAY_BUILDER(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT32_ARRAY_BUILDER,  \
-                              GArrowInt32ArrayBuilder))
-#define GARROW_INT32_ARRAY_BUILDER_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_INT32_ARRAY_BUILDER,     \
-                           GArrowInt32ArrayBuilderClass))
-#define GARROW_IS_INT32_ARRAY_BUILDER(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT32_ARRAY_BUILDER))
-#define GARROW_IS_INT32_ARRAY_BUILDER_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT32_ARRAY_BUILDER))
-#define GARROW_INT32_ARRAY_BUILDER_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT32_ARRAY_BUILDER,   \
-                             GArrowInt32ArrayBuilderClass))
-
-typedef struct _GArrowInt32ArrayBuilder         GArrowInt32ArrayBuilder;
-typedef struct _GArrowInt32ArrayBuilderClass    GArrowInt32ArrayBuilderClass;
-
-/**
- * GArrowInt32ArrayBuilder:
- *
- * It wraps `arrow::Int32Builder`.
- */
-struct _GArrowInt32ArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowInt32ArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_int32_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowInt32ArrayBuilder *garrow_int32_array_builder_new(void);
-
-gboolean garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
-                                           gint32 value,
-                                           GError **error);
-gboolean garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
-                                                GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-array-builder.cpp b/c_glib/arrow-glib/int64-array-builder.cpp
deleted file mode 100644
index b5eff114f92c9..0000000000000
--- a/c_glib/arrow-glib/int64-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/int64-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int64-array-builder
- * @short_description: 64-bit integer array builder class
- *
- * #GArrowInt64ArrayBuilder is the class to create a new
- * #GArrowInt64Array.
- */
-
-G_DEFINE_TYPE(GArrowInt64ArrayBuilder,
-              garrow_int64_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_int64_array_builder_init(GArrowInt64ArrayBuilder *builder)
-{
-}
-
-static void
-garrow_int64_array_builder_class_init(GArrowInt64ArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_int64_array_builder_new:
- *
- * Returns: A newly created #GArrowInt64ArrayBuilder.
- */
-GArrowInt64ArrayBuilder *
-garrow_int64_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::Int64Builder>(memory_pool, arrow::int64());
-  auto builder =
-    GARROW_INT64_ARRAY_BUILDER(g_object_new(GARROW_TYPE_INT64_ARRAY_BUILDER,
-                                           "array-builder", &arrow_builder,
-                                           NULL));
-  return builder;
-}
-
-/**
- * garrow_int64_array_builder_append:
- * @builder: A #GArrowInt64ArrayBuilder.
- * @value: A int64 value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
-                                  gint64 value,
-                                  GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::Int64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int64-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_int64_array_builder_append_null:
- * @builder: A #GArrowInt64ArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
-                                      GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::Int64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int64-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int64-array-builder.h b/c_glib/arrow-glib/int64-array-builder.h
deleted file mode 100644
index 8f4947eb7d9b1..0000000000000
--- a/c_glib/arrow-glib/int64-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT64_ARRAY_BUILDER          \
-  (garrow_int64_array_builder_get_type())
-#define GARROW_INT64_ARRAY_BUILDER(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT64_ARRAY_BUILDER,   \
-                              GArrowInt64ArrayBuilder))
-#define GARROW_INT64_ARRAY_BUILDER_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_INT64_ARRAY_BUILDER,      \
-                           GArrowInt64ArrayBuilderClass))
-#define GARROW_IS_INT64_ARRAY_BUILDER(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT64_ARRAY_BUILDER))
-#define GARROW_IS_INT64_ARRAY_BUILDER_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT64_ARRAY_BUILDER))
-#define GARROW_INT64_ARRAY_BUILDER_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT64_ARRAY_BUILDER,    \
-                             GArrowInt64ArrayBuilderClass))
-
-typedef struct _GArrowInt64ArrayBuilder         GArrowInt64ArrayBuilder;
-typedef struct _GArrowInt64ArrayBuilderClass    GArrowInt64ArrayBuilderClass;
-
-/**
- * GArrowInt64ArrayBuilder:
- *
- * It wraps `arrow::Int64Builder`.
- */
-struct _GArrowInt64ArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowInt64ArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_int64_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowInt64ArrayBuilder *garrow_int64_array_builder_new(void);
-
-gboolean garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
-                                          gint64 value,
-                                          GError **error);
-gboolean garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
-                                               GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-array-builder.cpp b/c_glib/arrow-glib/int8-array-builder.cpp
deleted file mode 100644
index 5107a6fae1f6a..0000000000000
--- a/c_glib/arrow-glib/int8-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/int8-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: int8-array-builder
- * @short_description: 8-bit integer array builder class
- *
- * #GArrowInt8ArrayBuilder is the class to create a new
- * #GArrowInt8Array.
- */
-
-G_DEFINE_TYPE(GArrowInt8ArrayBuilder,
-              garrow_int8_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_int8_array_builder_init(GArrowInt8ArrayBuilder *builder)
-{
-}
-
-static void
-garrow_int8_array_builder_class_init(GArrowInt8ArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_int8_array_builder_new:
- *
- * Returns: A newly created #GArrowInt8ArrayBuilder.
- */
-GArrowInt8ArrayBuilder *
-garrow_int8_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::Int8Builder>(memory_pool, arrow::int8());
-  auto builder =
-    GARROW_INT8_ARRAY_BUILDER(g_object_new(GARROW_TYPE_INT8_ARRAY_BUILDER,
-                                           "array-builder", &arrow_builder,
-                                           NULL));
-  return builder;
-}
-
-/**
- * garrow_int8_array_builder_append:
- * @builder: A #GArrowInt8ArrayBuilder.
- * @value: A int8 value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
-                                 gint8 value,
-                                 GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::Int8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int8-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_int8_array_builder_append_null:
- * @builder: A #GArrowInt8ArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
-                                      GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::Int8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int8-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/int8-array-builder.h b/c_glib/arrow-glib/int8-array-builder.h
deleted file mode 100644
index 321e9310a6447..0000000000000
--- a/c_glib/arrow-glib/int8-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_INT8_ARRAY_BUILDER          \
-  (garrow_int8_array_builder_get_type())
-#define GARROW_INT8_ARRAY_BUILDER(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT8_ARRAY_BUILDER,   \
-                              GArrowInt8ArrayBuilder))
-#define GARROW_INT8_ARRAY_BUILDER_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_INT8_ARRAY_BUILDER,      \
-                           GArrowInt8ArrayBuilderClass))
-#define GARROW_IS_INT8_ARRAY_BUILDER(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT8_ARRAY_BUILDER))
-#define GARROW_IS_INT8_ARRAY_BUILDER_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT8_ARRAY_BUILDER))
-#define GARROW_INT8_ARRAY_BUILDER_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT8_ARRAY_BUILDER,    \
-                             GArrowInt8ArrayBuilderClass))
-
-typedef struct _GArrowInt8ArrayBuilder         GArrowInt8ArrayBuilder;
-typedef struct _GArrowInt8ArrayBuilderClass    GArrowInt8ArrayBuilderClass;
-
-/**
- * GArrowInt8ArrayBuilder:
- *
- * It wraps `arrow::Int8Builder`.
- */
-struct _GArrowInt8ArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowInt8ArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_int8_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowInt8ArrayBuilder *garrow_int8_array_builder_new(void);
-
-gboolean garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
-                                          gint8 value,
-                                          GError **error);
-gboolean garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
-                                               GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/list-array-builder.cpp b/c_glib/arrow-glib/list-array-builder.cpp
deleted file mode 100644
index 6c8f53da1fc98..0000000000000
--- a/c_glib/arrow-glib/list-array-builder.cpp
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/list-array-builder.h>
-#include <arrow-glib/error.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: list-array-builder
- * @short_description: List array builder class
- * @include: arrow-glib/arrow-glib.h
- *
- * #GArrowListArrayBuilder is the class to create a new
- * #GArrowListArray.
- */
-
-G_DEFINE_TYPE(GArrowListArrayBuilder,
-              garrow_list_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_list_array_builder_init(GArrowListArrayBuilder *builder)
-{
-}
-
-static void
-garrow_list_array_builder_class_init(GArrowListArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_list_array_builder_new:
- * @value_builder: A #GArrowArrayBuilder for value array.
- *
- * Returns: A newly created #GArrowListArrayBuilder.
- */
-GArrowListArrayBuilder *
-garrow_list_array_builder_new(GArrowArrayBuilder *value_builder)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_value_builder = garrow_array_builder_get_raw(value_builder);
-  auto arrow_list_builder =
-    std::make_shared<arrow::ListBuilder>(memory_pool, arrow_value_builder);
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_list_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
-  return GARROW_LIST_ARRAY_BUILDER(builder);
-}
-
-/**
- * garrow_list_array_builder_append:
- * @builder: A #GArrowListArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- *
- * It appends a new list element. To append a new list element, you
- * need to call this function then append list element values to
- * `value_builder`. `value_builder` is the #GArrowArrayBuilder
- * specified to constructor. You can get `value_builder` by
- * garrow_list_array_builder_get_value_builder().
- *
- * |[<!-- language="C" -->
- * GArrowInt8ArrayBuilder *value_builder;
- * GArrowListArrayBuilder *builder;
- *
- * value_builder = garrow_int8_array_builder_new();
- * builder = garrow_list_array_builder_new(value_builder, NULL);
- *
- * // Start 0th list element: [1, 0, -1]
- * garrow_list_array_builder_append(builder, NULL);
- * garrow_int8_array_builder_append(value_builder, 1);
- * garrow_int8_array_builder_append(value_builder, 0);
- * garrow_int8_array_builder_append(value_builder, -1);
- *
- * // Start 1st list element: [-29, 29]
- * garrow_list_array_builder_append(builder, NULL);
- * garrow_int8_array_builder_append(value_builder, -29);
- * garrow_int8_array_builder_append(value_builder, 29);
- *
- * {
- *   // [[1, 0, -1], [-29, 29]]
- *   GArrowArray *array = garrow_array_builder_finish(builder);
- *   // Now, builder is needless.
- *   g_object_unref(builder);
- *   g_object_unref(value_builder);
- *
- *   // Use array...
- *   g_object_unref(array);
- * }
- * ]|
- */
-gboolean
-garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
-                                 GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::ListBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[list-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_list_array_builder_append_null:
- * @builder: A #GArrowListArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- *
- * It appends a new NULL element.
- */
-gboolean
-garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
-                                      GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::ListBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[list-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_list_array_builder_get_value_builder:
- * @builder: A #GArrowListArrayBuilder.
- *
- * Returns: (transfer full): The #GArrowArrayBuilder for values.
- */
-GArrowArrayBuilder *
-garrow_list_array_builder_get_value_builder(GArrowListArrayBuilder *builder)
-{
-  auto arrow_builder =
-    static_cast<arrow::ListBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-  auto arrow_value_builder = arrow_builder->value_builder();
-  return garrow_array_builder_new_raw(&arrow_value_builder);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/list-array-builder.h b/c_glib/arrow-glib/list-array-builder.h
deleted file mode 100644
index 2c2e58e54309b..0000000000000
--- a/c_glib/arrow-glib/list-array-builder.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_LIST_ARRAY_BUILDER          \
-  (garrow_list_array_builder_get_type())
-#define GARROW_LIST_ARRAY_BUILDER(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_LIST_ARRAY_BUILDER,   \
-                              GArrowListArrayBuilder))
-#define GARROW_LIST_ARRAY_BUILDER_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_LIST_ARRAY_BUILDER,      \
-                           GArrowListArrayBuilderClass))
-#define GARROW_IS_LIST_ARRAY_BUILDER(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_LIST_ARRAY_BUILDER))
-#define GARROW_IS_LIST_ARRAY_BUILDER_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_LIST_ARRAY_BUILDER))
-#define GARROW_LIST_ARRAY_BUILDER_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_LIST_ARRAY_BUILDER,    \
-                             GArrowListArrayBuilderClass))
-
-typedef struct _GArrowListArrayBuilder         GArrowListArrayBuilder;
-typedef struct _GArrowListArrayBuilderClass    GArrowListArrayBuilderClass;
-
-/**
- * GArrowListArrayBuilder:
- *
- * It wraps `arrow::ListBuilder`.
- */
-struct _GArrowListArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowListArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_list_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowListArrayBuilder *garrow_list_array_builder_new(GArrowArrayBuilder *value_builder);
-
-gboolean garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
-                                          GError **error);
-gboolean garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
-                                               GError **error);
-
-GArrowArrayBuilder *garrow_list_array_builder_get_value_builder(GArrowListArrayBuilder *builder);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/string-array-builder.cpp b/c_glib/arrow-glib/string-array-builder.cpp
deleted file mode 100644
index ebad53a18704a..0000000000000
--- a/c_glib/arrow-glib/string-array-builder.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/string-array-builder.h>
-#include <arrow-glib/error.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: string-array-builder
- * @short_description: UTF-8 encoded string array builder class
- *
- * #GArrowStringArrayBuilder is the class to create a new
- * #GArrowStringArray.
- */
-
-G_DEFINE_TYPE(GArrowStringArrayBuilder,
-              garrow_string_array_builder,
-              GARROW_TYPE_BINARY_ARRAY_BUILDER)
-
-static void
-garrow_string_array_builder_init(GArrowStringArrayBuilder *builder)
-{
-}
-
-static void
-garrow_string_array_builder_class_init(GArrowStringArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_string_array_builder_new:
- *
- * Returns: A newly created #GArrowStringArrayBuilder.
- */
-GArrowStringArrayBuilder *
-garrow_string_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::StringBuilder>(memory_pool);
-  auto builder =
-    GARROW_STRING_ARRAY_BUILDER(g_object_new(GARROW_TYPE_STRING_ARRAY_BUILDER,
-                                             "array-builder", &arrow_builder,
-                                             NULL));
-  return builder;
-}
-
-/**
- * garrow_string_array_builder_append:
- * @builder: A #GArrowStringArrayBuilder.
- * @value: A string value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
-                                   const gchar *value,
-                                   GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::StringBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value,
-                                      static_cast<gint32>(strlen(value)));
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[string-array-builder][append]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/string-array-builder.h b/c_glib/arrow-glib/string-array-builder.h
deleted file mode 100644
index f370ed9edec9d..0000000000000
--- a/c_glib/arrow-glib/string-array-builder.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/binary-array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_STRING_ARRAY_BUILDER        \
-  (garrow_string_array_builder_get_type())
-#define GARROW_STRING_ARRAY_BUILDER(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_STRING_ARRAY_BUILDER, \
-                              GArrowStringArrayBuilder))
-#define GARROW_STRING_ARRAY_BUILDER_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_STRING_ARRAY_BUILDER,    \
-                           GArrowStringArrayBuilderClass))
-#define GARROW_IS_STRING_ARRAY_BUILDER(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_STRING_ARRAY_BUILDER))
-#define GARROW_IS_STRING_ARRAY_BUILDER_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_STRING_ARRAY_BUILDER))
-#define GARROW_STRING_ARRAY_BUILDER_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_STRING_ARRAY_BUILDER,  \
-                             GArrowStringArrayBuilderClass))
-
-typedef struct _GArrowStringArrayBuilder         GArrowStringArrayBuilder;
-typedef struct _GArrowStringArrayBuilderClass    GArrowStringArrayBuilderClass;
-
-/**
- * GArrowStringArrayBuilder:
- *
- * It wraps `arrow::StringBuilder`.
- */
-struct _GArrowStringArrayBuilder
-{
-  /*< private >*/
-  GArrowBinaryArrayBuilder parent_instance;
-};
-
-struct _GArrowStringArrayBuilderClass
-{
-  GArrowBinaryArrayBuilderClass parent_class;
-};
-
-GType garrow_string_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowStringArrayBuilder *garrow_string_array_builder_new(void);
-
-gboolean garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
-                                            const gchar *value,
-                                            GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array-builder.cpp b/c_glib/arrow-glib/struct-array-builder.cpp
deleted file mode 100644
index 2453a5baf2ec8..0000000000000
--- a/c_glib/arrow-glib/struct-array-builder.cpp
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/data-type.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/field.hpp>
-#include <arrow-glib/struct-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: struct-array-builder
- * @short_description: Struct array builder class
- * @include: arrow-glib/arrow-glib.h
- *
- * #GArrowStructArrayBuilder is the class to create a new
- * #GArrowStructArray.
- */
-
-G_DEFINE_TYPE(GArrowStructArrayBuilder,
-              garrow_struct_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_struct_array_builder_init(GArrowStructArrayBuilder *builder)
-{
-}
-
-static void
-garrow_struct_array_builder_class_init(GArrowStructArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_struct_array_builder_new:
- * @data_type: #GArrowStructDataType for the struct.
- * @field_builders: (element-type GArrowArray): #GArrowArrayBuilders
- *   for fields.
- *
- * Returns: A newly created #GArrowStructArrayBuilder.
- */
-GArrowStructArrayBuilder *
-garrow_struct_array_builder_new(GArrowStructDataType *data_type,
-                                GList *field_builders)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
-  std::vector<std::shared_ptr<arrow::ArrayBuilder>> arrow_field_builders;
-  for (GList *node = field_builders; node; node = g_list_next(node)) {
-    auto field_builder = static_cast<GArrowArrayBuilder *>(node->data);
-    auto arrow_field_builder = garrow_array_builder_get_raw(field_builder);
-    arrow_field_builders.push_back(arrow_field_builder);
-  }
-
-  auto arrow_struct_builder =
-    std::make_shared<arrow::StructBuilder>(memory_pool,
-                                           arrow_data_type,
-                                           arrow_field_builders);
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_struct_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
-  return GARROW_STRUCT_ARRAY_BUILDER(builder);
-}
-
-/**
- * garrow_struct_array_builder_append:
- * @builder: A #GArrowStructArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- *
- * It appends a new struct element. To append a new struct element,
- * you need to call this function then append struct element field
- * values to all `field_builder`s. `field_value`s are the
- * #GArrowArrayBuilder specified to constructor. You can get
- * `field_builder` by garrow_struct_array_builder_get_field_builder()
- * or garrow_struct_array_builder_get_field_builders().
- *
- * |[<!-- language="C" -->
- * // TODO
- * ]|
- */
-gboolean
-garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
-                                   GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[struct-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_struct_array_builder_append_null:
- * @builder: A #GArrowStructArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- *
- * It appends a new NULL element.
- */
-gboolean
-garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
-                                        GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[struct-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_struct_array_builder_get_field_builder:
- * @builder: A #GArrowStructArrayBuilder.
- * @i: The index of the field in the struct.
- *
- * Returns: (transfer full): The #GArrowArrayBuilder for the i-th field.
- */
-GArrowArrayBuilder *
-garrow_struct_array_builder_get_field_builder(GArrowStructArrayBuilder *builder,
-                                              gint i)
-{
-  auto arrow_builder =
-    static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-  auto arrow_field_builder = arrow_builder->field_builder(i);
-  return garrow_array_builder_new_raw(&arrow_field_builder);
-}
-
-/**
- * garrow_struct_array_builder_get_field_builders:
- * @builder: A #GArrowStructArrayBuilder.
- *
- * Returns: (element-type GArrowArray) (transfer full):
- *   The #GArrowArrayBuilder for all fields.
- */
-GList *
-garrow_struct_array_builder_get_field_builders(GArrowStructArrayBuilder *builder)
-{
-  auto arrow_struct_builder =
-    static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  GList *field_builders = NULL;
-  for (auto arrow_field_builder : arrow_struct_builder->field_builders()) {
-    auto field_builder = garrow_array_builder_new_raw(&arrow_field_builder);
-    field_builders = g_list_prepend(field_builders, field_builder);
-  }
-
-  return g_list_reverse(field_builders);
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/struct-array-builder.h b/c_glib/arrow-glib/struct-array-builder.h
deleted file mode 100644
index 237b2b3264f24..0000000000000
--- a/c_glib/arrow-glib/struct-array-builder.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_STRUCT_ARRAY_BUILDER        \
-  (garrow_struct_array_builder_get_type())
-#define GARROW_STRUCT_ARRAY_BUILDER(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_STRUCT_ARRAY_BUILDER, \
-                              GArrowStructArrayBuilder))
-#define GARROW_STRUCT_ARRAY_BUILDER_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_STRUCT_ARRAY_BUILDER,    \
-                           GArrowStructArrayBuilderClass))
-#define GARROW_IS_STRUCT_ARRAY_BUILDER(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_STRUCT_ARRAY_BUILDER))
-#define GARROW_IS_STRUCT_ARRAY_BUILDER_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_STRUCT_ARRAY_BUILDER))
-#define GARROW_STRUCT_ARRAY_BUILDER_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_STRUCT_ARRAY_BUILDER,  \
-                             GArrowStructArrayBuilderClass))
-
-typedef struct _GArrowStructArrayBuilder         GArrowStructArrayBuilder;
-typedef struct _GArrowStructArrayBuilderClass    GArrowStructArrayBuilderClass;
-
-/**
- * GArrowStructArrayBuilder:
- *
- * It wraps `arrow::StructBuilder`.
- */
-struct _GArrowStructArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowStructArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_struct_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowStructArrayBuilder *garrow_struct_array_builder_new(GArrowStructDataType *data_type,
-                                                          GList *field_builders);
-
-gboolean garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
-                                            GError **error);
-gboolean garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
-                                                 GError **error);
-
-GArrowArrayBuilder *garrow_struct_array_builder_get_field_builder(GArrowStructArrayBuilder *builder,
-                                                                  gint i);
-GList *garrow_struct_array_builder_get_field_builders(GArrowStructArrayBuilder *builder);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-array-builder.cpp b/c_glib/arrow-glib/uint16-array-builder.cpp
deleted file mode 100644
index bfade2de7a84d..0000000000000
--- a/c_glib/arrow-glib/uint16-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/uint16-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint16-array-builder
- * @short_description: 16-bit unsigned integer array builder class
- *
- * #GArrowUInt16ArrayBuilder is the class to create a new
- * #GArrowUInt16Array.
- */
-
-G_DEFINE_TYPE(GArrowUInt16ArrayBuilder,
-              garrow_uint16_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_uint16_array_builder_init(GArrowUInt16ArrayBuilder *builder)
-{
-}
-
-static void
-garrow_uint16_array_builder_class_init(GArrowUInt16ArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_uint16_array_builder_new:
- *
- * Returns: A newly created #GArrowUInt16ArrayBuilder.
- */
-GArrowUInt16ArrayBuilder *
-garrow_uint16_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::UInt16Builder>(memory_pool, arrow::uint16());
-  auto builder =
-    GARROW_UINT16_ARRAY_BUILDER(g_object_new(GARROW_TYPE_UINT16_ARRAY_BUILDER,
-                                            "array-builder", &arrow_builder,
-                                            NULL));
-  return builder;
-}
-
-/**
- * garrow_uint16_array_builder_append:
- * @builder: A #GArrowUInt16ArrayBuilder.
- * @value: An uint16 value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
-                                  guint16 value,
-                                  GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::UInt16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint16-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_uint16_array_builder_append_null:
- * @builder: A #GArrowUInt16ArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
-                                       GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::UInt16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint16-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint16-array-builder.h b/c_glib/arrow-glib/uint16-array-builder.h
deleted file mode 100644
index c08966ecc1d91..0000000000000
--- a/c_glib/arrow-glib/uint16-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT16_ARRAY_BUILDER         \
-  (garrow_uint16_array_builder_get_type())
-#define GARROW_UINT16_ARRAY_BUILDER(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT16_ARRAY_BUILDER,  \
-                              GArrowUInt16ArrayBuilder))
-#define GARROW_UINT16_ARRAY_BUILDER_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT16_ARRAY_BUILDER,     \
-                           GArrowUInt16ArrayBuilderClass))
-#define GARROW_IS_UINT16_ARRAY_BUILDER(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT16_ARRAY_BUILDER))
-#define GARROW_IS_UINT16_ARRAY_BUILDER_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT16_ARRAY_BUILDER))
-#define GARROW_UINT16_ARRAY_BUILDER_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT16_ARRAY_BUILDER,   \
-                             GArrowUInt16ArrayBuilderClass))
-
-typedef struct _GArrowUInt16ArrayBuilder         GArrowUInt16ArrayBuilder;
-typedef struct _GArrowUInt16ArrayBuilderClass    GArrowUInt16ArrayBuilderClass;
-
-/**
- * GArrowUInt16ArrayBuilder:
- *
- * It wraps `arrow::UInt16Builder`.
- */
-struct _GArrowUInt16ArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowUInt16ArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_uint16_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowUInt16ArrayBuilder *garrow_uint16_array_builder_new(void);
-
-gboolean garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
-                                           guint16 value,
-                                           GError **error);
-gboolean garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
-                                                GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-array-builder.cpp b/c_glib/arrow-glib/uint32-array-builder.cpp
deleted file mode 100644
index 35b1893619fa5..0000000000000
--- a/c_glib/arrow-glib/uint32-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/uint32-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint32-array-builder
- * @short_description: 32-bit unsigned integer array builder class
- *
- * #GArrowUInt32ArrayBuilder is the class to create a new
- * #GArrowUInt32Array.
- */
-
-G_DEFINE_TYPE(GArrowUInt32ArrayBuilder,
-              garrow_uint32_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_uint32_array_builder_init(GArrowUInt32ArrayBuilder *builder)
-{
-}
-
-static void
-garrow_uint32_array_builder_class_init(GArrowUInt32ArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_uint32_array_builder_new:
- *
- * Returns: A newly created #GArrowUInt32ArrayBuilder.
- */
-GArrowUInt32ArrayBuilder *
-garrow_uint32_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::UInt32Builder>(memory_pool, arrow::uint32());
-  auto builder =
-    GARROW_UINT32_ARRAY_BUILDER(g_object_new(GARROW_TYPE_UINT32_ARRAY_BUILDER,
-                                            "array-builder", &arrow_builder,
-                                            NULL));
-  return builder;
-}
-
-/**
- * garrow_uint32_array_builder_append:
- * @builder: A #GArrowUInt32ArrayBuilder.
- * @value: An uint32 value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
-                                  guint32 value,
-                                  GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::UInt32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint32-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_uint32_array_builder_append_null:
- * @builder: A #GArrowUInt32ArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
-                                       GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::UInt32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint32-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint32-array-builder.h b/c_glib/arrow-glib/uint32-array-builder.h
deleted file mode 100644
index 4881d3b17ff0d..0000000000000
--- a/c_glib/arrow-glib/uint32-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT32_ARRAY_BUILDER         \
-  (garrow_uint32_array_builder_get_type())
-#define GARROW_UINT32_ARRAY_BUILDER(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT32_ARRAY_BUILDER,  \
-                              GArrowUInt32ArrayBuilder))
-#define GARROW_UINT32_ARRAY_BUILDER_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT32_ARRAY_BUILDER,     \
-                           GArrowUInt32ArrayBuilderClass))
-#define GARROW_IS_UINT32_ARRAY_BUILDER(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT32_ARRAY_BUILDER))
-#define GARROW_IS_UINT32_ARRAY_BUILDER_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT32_ARRAY_BUILDER))
-#define GARROW_UINT32_ARRAY_BUILDER_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT32_ARRAY_BUILDER,   \
-                             GArrowUInt32ArrayBuilderClass))
-
-typedef struct _GArrowUInt32ArrayBuilder         GArrowUInt32ArrayBuilder;
-typedef struct _GArrowUInt32ArrayBuilderClass    GArrowUInt32ArrayBuilderClass;
-
-/**
- * GArrowUInt32ArrayBuilder:
- *
- * It wraps `arrow::UInt32Builder`.
- */
-struct _GArrowUInt32ArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowUInt32ArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_uint32_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowUInt32ArrayBuilder *garrow_uint32_array_builder_new(void);
-
-gboolean garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
-                                           guint32 value,
-                                           GError **error);
-gboolean garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
-                                                GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-array-builder.cpp b/c_glib/arrow-glib/uint64-array-builder.cpp
deleted file mode 100644
index 85d24ca54ab8b..0000000000000
--- a/c_glib/arrow-glib/uint64-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/uint64-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint64-array-builder
- * @short_description: 64-bit unsigned integer array builder class
- *
- * #GArrowUInt64ArrayBuilder is the class to create a new
- * #GArrowUInt64Array.
- */
-
-G_DEFINE_TYPE(GArrowUInt64ArrayBuilder,
-              garrow_uint64_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_uint64_array_builder_init(GArrowUInt64ArrayBuilder *builder)
-{
-}
-
-static void
-garrow_uint64_array_builder_class_init(GArrowUInt64ArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_uint64_array_builder_new:
- *
- * Returns: A newly created #GArrowUInt64ArrayBuilder.
- */
-GArrowUInt64ArrayBuilder *
-garrow_uint64_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::UInt64Builder>(memory_pool, arrow::uint64());
-  auto builder =
-    GARROW_UINT64_ARRAY_BUILDER(g_object_new(GARROW_TYPE_UINT64_ARRAY_BUILDER,
-                                            "array-builder", &arrow_builder,
-                                            NULL));
-  return builder;
-}
-
-/**
- * garrow_uint64_array_builder_append:
- * @builder: A #GArrowUInt64ArrayBuilder.
- * @value: An uint64 value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
-                                  guint64 value,
-                                  GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::UInt64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint64-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_uint64_array_builder_append_null:
- * @builder: A #GArrowUInt64ArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
-                                       GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::UInt64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint64-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint64-array-builder.h b/c_glib/arrow-glib/uint64-array-builder.h
deleted file mode 100644
index c51d1e2485d6f..0000000000000
--- a/c_glib/arrow-glib/uint64-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT64_ARRAY_BUILDER         \
-  (garrow_uint64_array_builder_get_type())
-#define GARROW_UINT64_ARRAY_BUILDER(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT64_ARRAY_BUILDER,  \
-                              GArrowUInt64ArrayBuilder))
-#define GARROW_UINT64_ARRAY_BUILDER_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT64_ARRAY_BUILDER,     \
-                           GArrowUInt64ArrayBuilderClass))
-#define GARROW_IS_UINT64_ARRAY_BUILDER(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT64_ARRAY_BUILDER))
-#define GARROW_IS_UINT64_ARRAY_BUILDER_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT64_ARRAY_BUILDER))
-#define GARROW_UINT64_ARRAY_BUILDER_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT64_ARRAY_BUILDER,   \
-                             GArrowUInt64ArrayBuilderClass))
-
-typedef struct _GArrowUInt64ArrayBuilder         GArrowUInt64ArrayBuilder;
-typedef struct _GArrowUInt64ArrayBuilderClass    GArrowUInt64ArrayBuilderClass;
-
-/**
- * GArrowUInt64ArrayBuilder:
- *
- * It wraps `arrow::UInt64Builder`.
- */
-struct _GArrowUInt64ArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowUInt64ArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_uint64_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowUInt64ArrayBuilder *garrow_uint64_array_builder_new(void);
-
-gboolean garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
-                                           guint64 value,
-                                           GError **error);
-gboolean garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
-                                                GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-array-builder.cpp b/c_glib/arrow-glib/uint8-array-builder.cpp
deleted file mode 100644
index 2f49693236b24..0000000000000
--- a/c_glib/arrow-glib/uint8-array-builder.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/uint8-array-builder.h>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: uint8-array-builder
- * @short_description: 8-bit unsigned integer array builder class
- *
- * #GArrowUInt8ArrayBuilder is the class to create a new
- * #GArrowUInt8Array.
- */
-
-G_DEFINE_TYPE(GArrowUInt8ArrayBuilder,
-              garrow_uint8_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_uint8_array_builder_init(GArrowUInt8ArrayBuilder *builder)
-{
-}
-
-static void
-garrow_uint8_array_builder_class_init(GArrowUInt8ArrayBuilderClass *klass)
-{
-}
-
-/**
- * garrow_uint8_array_builder_new:
- *
- * Returns: A newly created #GArrowUInt8ArrayBuilder.
- */
-GArrowUInt8ArrayBuilder *
-garrow_uint8_array_builder_new(void)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_builder =
-    std::make_shared<arrow::UInt8Builder>(memory_pool, arrow::uint8());
-  auto builder =
-    GARROW_UINT8_ARRAY_BUILDER(g_object_new(GARROW_TYPE_UINT8_ARRAY_BUILDER,
-                                            "array-builder", &arrow_builder,
-                                            NULL));
-  return builder;
-}
-
-/**
- * garrow_uint8_array_builder_append:
- * @builder: A #GArrowUInt8ArrayBuilder.
- * @value: An uint8 value.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
-                                  guint8 value,
-                                  GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::UInt8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint8-array-builder][append]");
-    return FALSE;
-  }
-}
-
-/**
- * garrow_uint8_array_builder_append_null:
- * @builder: A #GArrowUInt8ArrayBuilder.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
-                                       GError **error)
-{
-  auto arrow_builder =
-    static_cast<arrow::UInt8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint8-array-builder][append-null]");
-    return FALSE;
-  }
-}
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/uint8-array-builder.h b/c_glib/arrow-glib/uint8-array-builder.h
deleted file mode 100644
index e7216931a511c..0000000000000
--- a/c_glib/arrow-glib/uint8-array-builder.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array-builder.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_UINT8_ARRAY_BUILDER         \
-  (garrow_uint8_array_builder_get_type())
-#define GARROW_UINT8_ARRAY_BUILDER(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT8_ARRAY_BUILDER,  \
-                              GArrowUInt8ArrayBuilder))
-#define GARROW_UINT8_ARRAY_BUILDER_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT8_ARRAY_BUILDER,     \
-                           GArrowUInt8ArrayBuilderClass))
-#define GARROW_IS_UINT8_ARRAY_BUILDER(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT8_ARRAY_BUILDER))
-#define GARROW_IS_UINT8_ARRAY_BUILDER_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT8_ARRAY_BUILDER))
-#define GARROW_UINT8_ARRAY_BUILDER_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT8_ARRAY_BUILDER,   \
-                             GArrowUInt8ArrayBuilderClass))
-
-typedef struct _GArrowUInt8ArrayBuilder         GArrowUInt8ArrayBuilder;
-typedef struct _GArrowUInt8ArrayBuilderClass    GArrowUInt8ArrayBuilderClass;
-
-/**
- * GArrowUInt8ArrayBuilder:
- *
- * It wraps `arrow::UInt8Builder`.
- */
-struct _GArrowUInt8ArrayBuilder
-{
-  /*< private >*/
-  GArrowArrayBuilder parent_instance;
-};
-
-struct _GArrowUInt8ArrayBuilderClass
-{
-  GArrowArrayBuilderClass parent_class;
-};
-
-GType garrow_uint8_array_builder_get_type(void) G_GNUC_CONST;
-
-GArrowUInt8ArrayBuilder *garrow_uint8_array_builder_new(void);
-
-gboolean garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
-                                           guint8 value,
-                                           GError **error);
-gboolean garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
-                                                GError **error);
-
-G_END_DECLS
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 5df9f64a85c92..bfb2776f621cc 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -40,21 +40,6 @@
     <chapter id="array-builder">
       <title>Array builder</title>
       <xi:include href="xml/array-builder.xml"/>
-      <xi:include href="xml/boolean-array-builder.xml"/>
-      <xi:include href="xml/int8-array-builder.xml"/>
-      <xi:include href="xml/uint8-array-builder.xml"/>
-      <xi:include href="xml/int16-array-builder.xml"/>
-      <xi:include href="xml/uint16-array-builder.xml"/>
-      <xi:include href="xml/int32-array-builder.xml"/>
-      <xi:include href="xml/uint32-array-builder.xml"/>
-      <xi:include href="xml/int64-array-builder.xml"/>
-      <xi:include href="xml/uint64-array-builder.xml"/>
-      <xi:include href="xml/float-array-builder.xml"/>
-      <xi:include href="xml/double-array-builder.xml"/>
-      <xi:include href="xml/binary-array-builder.xml"/>
-      <xi:include href="xml/string-array-builder.xml"/>
-      <xi:include href="xml/list-array-builder.xml"/>
-      <xi:include href="xml/struct-array-builder.xml"/>
     </chapter>
     <chapter id="type">
       <title>Type</title>

From 578b0ff15ebc2d3751c9b4ee87d9e31f1c7ae0b6 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 22 Apr 2017 10:49:17 -0400
Subject: [PATCH 0551/1644] ARROW-877: [GLib] Add
 garrow_array_get_null_bitmap()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #582 from kou/glib-array-null-bitmap and squashes the following commits:

7f679f6 [Kouhei Sutou] [GLib] Add garrow_array_get_null_bitmap()
---
 c_glib/arrow-glib/array.cpp | 19 +++++++++++++++++++
 c_glib/arrow-glib/array.h   |  2 ++
 c_glib/test/test-array.rb   | 11 +++++++++++
 3 files changed, 32 insertions(+)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index dc1386b0daab9..1229f27ff906f 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -22,6 +22,7 @@
 #endif
 
 #include <arrow-glib/array.hpp>
+#include <arrow-glib/buffer.hpp>
 #include <arrow-glib/data-type.hpp>
 #include <arrow-glib/type.hpp>
 
@@ -242,6 +243,24 @@ garrow_array_get_n_nulls(GArrowArray *array)
   return arrow_array->null_count();
 }
 
+/**
+ * garrow_array_get_null_bitmap:
+ * @array: A #GArrowArray.
+ *
+ * Returns: (transfer full) (nullable): The bitmap that indicates null
+ *   value indexes for the array as #GArrowBuffer or %NULL when
+ *   garrow_array_get_n_nulls() returns 0.
+ *
+ * Since: 0.3.0
+ */
+GArrowBuffer *
+garrow_array_get_null_bitmap(GArrowArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto arrow_null_bitmap = arrow_array->null_bitmap();
+  return garrow_buffer_new_raw(&arrow_null_bitmap);
+}
+
 /**
  * garrow_array_get_value_data_type:
  * @array: A #GArrowArray.
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 74064562d6f39..f08ab84ef9e15 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include <arrow-glib/buffer.h>
 #include <arrow-glib/data-type.h>
 
 G_BEGIN_DECLS
@@ -62,6 +63,7 @@ gboolean       garrow_array_is_null     (GArrowArray *array,
 gint64         garrow_array_get_length  (GArrowArray *array);
 gint64         garrow_array_get_offset  (GArrowArray *array);
 gint64         garrow_array_get_n_nulls (GArrowArray *array);
+GArrowBuffer  *garrow_array_get_null_bitmap(GArrowArray *array);
 GArrowDataType *garrow_array_get_value_data_type(GArrowArray *array);
 GArrowType     garrow_array_get_value_type(GArrowArray *array);
 GArrowArray   *garrow_array_slice       (GArrowArray *array,
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
index 06102eb36575b..a2a2a1e003862 100644
--- a/c_glib/test/test-array.rb
+++ b/c_glib/test/test-array.rb
@@ -40,6 +40,17 @@ def test_n_nulls
     assert_equal(2, array.n_nulls)
   end
 
+  def test_null_bitmap
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append_null
+    builder.append(true)
+    builder.append(false)
+    builder.append_null
+    builder.append(false)
+    array = builder.finish
+    assert_equal(0b10110, array.null_bitmap.data.to_s.unpack("c*")[0])
+  end
+
   def test_value_data_type
     builder = Arrow::BooleanArrayBuilder.new
     array = builder.finish

From 07c6ade9b8362ba30c5d784986aedcb3cfb6483a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 22 Apr 2017 10:52:08 -0400
Subject: [PATCH 0552/1644] ARROW-849: [C++] Support setting production build
 dependencies with ARROW_BUILD_TOOLCHAIN

Opening this for comment. If we like this, we can do the same thing in parquet-cpp. Will need to be documented in the README.

I did not use the environment variable for gflags/gtest/gbenchmark, since this are test/benchmark-only dependencies, and they build automatically when needed.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #565 from wesm/ARROW-849 and squashes the following commits:

4507712 [Wes McKinney] Fix use of RAPIDJSON_HOME
e9fa400 [Wes McKinney] Use ARROW_BUILD_TOOLCHAIN if it's defined, but override with environment variables
d056a83 [Wes McKinney] Pull environment variables by default, override if toolchain variable is present
ec003c6 [Wes McKinney] Support setting production build dependencies with ARROW_BUILD_TOOLCHAIN environment variable
---
 cpp/CMakeLists.txt                      | 50 ++++++++++++++++++-------
 cpp/cmake_modules/FindFlatbuffers.cmake |  6 +--
 cpp/cmake_modules/Findjemalloc.cmake    |  4 +-
 3 files changed, 42 insertions(+), 18 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 81e4c90c73147..978f70a361756 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -46,13 +46,6 @@ set(ARROW_SO_VERSION "0")
 set(ARROW_ABI_VERSION "${ARROW_SO_VERSION}.0.0")
 
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
-set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
-
-set(GFLAGS_VERSION "2.1.2")
-set(GTEST_VERSION "1.8.0")
-set(GBENCHMARK_VERSION "1.1.0")
-set(FLATBUFFERS_VERSION "1.6.0")
-set(JEMALLOC_VERSION "4.4.0")
 
 find_package(ClangTools)
 if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
@@ -387,6 +380,40 @@ enable_testing()
 # Dependencies
 ############################################################
 
+# ----------------------------------------------------------------------
+# Thirdparty toolchain
+
+set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
+set(GFLAGS_VERSION "2.1.2")
+set(GTEST_VERSION "1.8.0")
+set(GBENCHMARK_VERSION "1.1.0")
+set(FLATBUFFERS_VERSION "1.6.0")
+set(JEMALLOC_VERSION "4.4.0")
+
+if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
+  set(FLATBUFFERS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(RAPIDJSON_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+
+  if (NOT DEFINED ENV{BOOST_ROOT})
+    # Since we have to set this in the environment, we check whether
+    # $BOOST_ROOT is defined inside here
+    set(ENV{BOOST_ROOT} "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  endif()
+endif()
+
+if (DEFINED ENV{FLATBUFFERS_HOME})
+  set(FLATBUFFERS_HOME "$ENV{FLATBUFFERS_HOME}")
+endif()
+
+if (DEFINED ENV{RAPIDJSON_HOME})
+  set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}")
+endif()
+
+if (DEFINED ENV{JEMALLOC_HOME})
+  set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}")
+endif()
+
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
@@ -451,9 +478,6 @@ SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 
-# ----------------------------------------------------------------------
-# Enable / disable tests and benchmarks
-
 if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
   add_custom_target(unittest ctest -L unittest)
 
@@ -616,7 +640,7 @@ endif()
 
 if (ARROW_IPC)
   # RapidJSON, header only dependency
-  if("$ENV{RAPIDJSON_HOME}" STREQUAL "")
+  if("${RAPIDJSON_HOME}" STREQUAL "")
     ExternalProject_Add(rapidjson_ep
       PREFIX "${CMAKE_BINARY_DIR}"
       URL "https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz"
@@ -630,14 +654,14 @@ if (ARROW_IPC)
     set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include")
     set(RAPIDJSON_VENDORED 1)
   else()
-    set(RAPIDJSON_INCLUDE_DIR "$ENV{RAPIDJSON_HOME}/include")
+    set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_HOME}/include")
     set(RAPIDJSON_VENDORED 0)
   endif()
   message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
   include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
 
   ## Flatbuffers
-  if("$ENV{FLATBUFFERS_HOME}" STREQUAL "")
+  if("${FLATBUFFERS_HOME}" STREQUAL "")
     set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
     ExternalProject_Add(flatbuffers_ep
       URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
diff --git a/cpp/cmake_modules/FindFlatbuffers.cmake b/cpp/cmake_modules/FindFlatbuffers.cmake
index ee472d1c8995f..7fa640ac9542f 100644
--- a/cpp/cmake_modules/FindFlatbuffers.cmake
+++ b/cpp/cmake_modules/FindFlatbuffers.cmake
@@ -31,8 +31,8 @@
 #  FLATBUFFERS_STATIC_LIB, path to libflatbuffers.a
 #  FLATBUFFERS_FOUND, whether flatbuffers has been found
 
-if( NOT "$ENV{FLATBUFFERS_HOME}" STREQUAL "")
-    file( TO_CMAKE_PATH "$ENV{FLATBUFFERS_HOME}" _native_path )
+if( NOT "${FLATBUFFERS_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${FLATBUFFERS_HOME}" _native_path )
     list( APPEND _flatbuffers_roots ${_native_path} )
 elseif ( Flatbuffers_HOME )
     list( APPEND _flatbuffers_roots ${Flatbuffers_HOME} )
@@ -52,7 +52,7 @@ else ()
 endif ()
 
 find_program(FLATBUFFERS_COMPILER flatc
-  $ENV{FLATBUFFERS_HOME}/bin
+  ${FLATBUFFERS_HOME}/bin
   /usr/local/bin
   /usr/bin
   NO_DEFAULT_PATH
diff --git a/cpp/cmake_modules/Findjemalloc.cmake b/cpp/cmake_modules/Findjemalloc.cmake
index e511d4dde0f71..93458982b1dc6 100644
--- a/cpp/cmake_modules/Findjemalloc.cmake
+++ b/cpp/cmake_modules/Findjemalloc.cmake
@@ -30,8 +30,8 @@
 #  JEMALLOC_SHARED_LIB, path to libjemalloc.so/dylib
 #  JEMALLOC_FOUND, whether flatbuffers has been found
 
-if( NOT "$ENV{JEMALLOC_HOME}" STREQUAL "")
-    file( TO_CMAKE_PATH "$ENV{JEMALLOC_HOME}" _native_path )
+if( NOT "${JEMALLOC_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${JEMALLOC_HOME}" _native_path )
     list( APPEND _jemalloc_roots ${_native_path} )
 elseif ( JEMALLOC_HOME )
     list( APPEND _jemalloc_roots ${JEMALLOC_HOME} )

From 39a37f76fa2cbf1dd52d3bc51b277553b772c343 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 22 Apr 2017 11:59:59 -0400
Subject: [PATCH 0553/1644] ARROW-878: [GLib] Add
 garrow_binary_array_get_buffer()

This will be conflicted with #582 .

Author: Kouhei Sutou <kou@clear-code.com>

Closes #583 from kou/glib-binary-array-buffer and squashes the following commits:

a84b8e8 [Kouhei Sutou] [GLib] Add garrow_binary_array_get_buffer()
---
 c_glib/arrow-glib/array.cpp      | 16 ++++++++++++++++
 c_glib/arrow-glib/array.h        |  1 +
 c_glib/test/test-binary-array.rb | 10 ++++++++++
 c_glib/test/test-string-array.rb |  8 ++++++++
 4 files changed, 35 insertions(+)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 1229f27ff906f..2fd09015d39ec 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -707,6 +707,22 @@ garrow_binary_array_get_value(GArrowBinaryArray *array,
   return g_bytes_new_static(value, length);
 }
 
+/**
+ * garrow_binary_array_get_buffer:
+ * @array: A #GArrowBinaryArray.
+ *
+ * Returns: (transfer full): The data of the array as #GArrowBuffer.
+ */
+GArrowBuffer *
+garrow_binary_array_get_buffer(GArrowBinaryArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_binary_array =
+    static_cast<arrow::BinaryArray *>(arrow_array.get());
+  auto arrow_data = arrow_binary_array->data();
+  return garrow_buffer_new_raw(&arrow_data);
+}
+
 
 G_DEFINE_TYPE(GArrowStringArray,               \
               garrow_string_array,             \
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index f08ab84ef9e15..f8c6734a88308 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -664,6 +664,7 @@ GType garrow_binary_array_get_type(void) G_GNUC_CONST;
 
 GBytes *garrow_binary_array_get_value(GArrowBinaryArray *array,
                                       gint64 i);
+GArrowBuffer *garrow_binary_array_get_buffer(GArrowBinaryArray *array);
 
 #define GARROW_TYPE_STRING_ARRAY                \
   (garrow_string_array_get_type())
diff --git a/c_glib/test/test-binary-array.rb b/c_glib/test/test-binary-array.rb
index 6fe89247c8649..ccdf378ad41b9 100644
--- a/c_glib/test/test-binary-array.rb
+++ b/c_glib/test/test-binary-array.rb
@@ -23,4 +23,14 @@ def test_value
     array = builder.finish
     assert_equal(data, array.get_value(0).to_s)
   end
+
+  def test_buffer
+    data1 = "\x00\x01\x02"
+    data2 = "\x03\x04\x05"
+    builder = Arrow::BinaryArrayBuilder.new
+    builder.append(data1)
+    builder.append(data2)
+    array = builder.finish
+    assert_equal(data1 + data2, array.buffer.data.to_s)
+  end
 end
diff --git a/c_glib/test/test-string-array.rb b/c_glib/test/test-string-array.rb
index a0f5a7b6b0fda..a076c228e0a4f 100644
--- a/c_glib/test/test-string-array.rb
+++ b/c_glib/test/test-string-array.rb
@@ -22,4 +22,12 @@ def test_value
     array = builder.finish
     assert_equal("Hello", array.get_string(0))
   end
+
+  def test_buffer
+    builder = Arrow::StringArrayBuilder.new
+    builder.append("Hello")
+    builder.append("World")
+    array = builder.finish
+    assert_equal("HelloWorld", array.buffer.data.to_s)
+  end
 end

From a0a925b42541d1ed2711c547c9eccaaa91820711 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Sat, 22 Apr 2017 15:07:00 -0400
Subject: [PATCH 0554/1644] ARROW-875: Avoid setting an extra empty in
 fillEmpties()

Author: Steven Phillips <steven@dremio.com>

Closes #579 from StevenMPhillips/fillEmpties and squashes the following commits:

e454876 [Steven Phillips] ARROW-875: Avoid setting an extra empty in fillEmpties()
---
 .../main/codegen/templates/NullableValueVectors.java |  4 ++--
 .../org/apache/arrow/vector/TestValueVector.java     | 12 ++++++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index acee6cb738d76..178d5bd913910 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -526,8 +526,8 @@ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.widt
 
     private void fillEmpties(int index){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
-      for (int i = lastSet; i < index; i++) {
-        valuesMutator.setSafe(i + 1, emptyByteArray);
+      for (int i = lastSet + 1; i < index; i++) {
+        valuesMutator.setSafe(i, emptyByteArray);
       }
       while(index > bits.getValueCapacity()) {
         bits.reAlloc();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 78ca14dc406ea..e6e49ab8d9341 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -466,4 +466,16 @@ public void testReAllocNullableVariableWidthVector() {
     }
   }
 
+  @Test
+  public void testFillEmptiesNotOverfill() {
+    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+      vector.allocateNew();
+
+      vector.getMutator().setSafe(4094, "hello".getBytes(), 0, 5);
+      vector.getMutator().setValueCount(4095);
+      assertEquals(4096 * 4, vector.getFieldBuffers().get(1).capacity());
+    }
+  }
+
+
 }

From 26e5bb1627f3b9768afccf018946720a688cf6f6 Mon Sep 17 00:00:00 2001
From: Jeff Reback <jeff.reback@twosigma.com>
Date: Sun, 23 Apr 2017 18:37:00 -0400
Subject: [PATCH 0555/1644] ARROW-879: compat with pandas v0.20.0

Author: Jeff Reback <jeff.reback@twosigma.com>

Closes #585 from jreback/compat and squashes the following commits:

1f1f4ed [Jeff Reback] use permanent pandas.api.types import
28c6608 [Jeff Reback] compat with pandas v0.20.0
---
 python/pyarrow/compat.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 4dcc11677e7dd..8d15c4c1e3fb5 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -32,9 +32,18 @@
 
 try:
     import pandas as pd
-    if LooseVersion(pd.__version__) < '0.19.0':
-        pdapi = pd.core.common
+    pdver = LooseVersion(pd.__version__)
+    if pdver >= '0.20.0':
+        try:
+            from pandas.api.types import DatetimeTZDtype
+        except AttributeError:
+            # can be removed once 0.20.0 is released
+            from pandas.core.dtypes.dtypes import DatetimeTZDtype
+
+        pdapi = pd.api.types
+    elif pdver < '0.19.0':
         from pandas.core.dtypes import DatetimeTZDtype
+        pdapi = pd.core.common
     else:
         from pandas.types.dtypes import DatetimeTZDtype
         pdapi = pd.api.types

From 33ac8a29176df340faa204b6c2e61b2973db028e Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sun, 23 Apr 2017 21:56:19 -0400
Subject: [PATCH 0556/1644] =?UTF-8?q?ARROW-882:=20[C++]=20Rename=20statica?=
 =?UTF-8?q?lly=20build=20library=20on=20Windows=20to=20avoid=20=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…conflict with shared version

Currently, statically built arrow.lib file overwrites previously built arrow.lib file of shared build. To resolve this, statically built library renamed to arrow_static.lib

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #590 from MaxRis/ARROW-882 and squashes the following commits:

4f2f3f0 [Max Risuhin] ARROW-882: [C++] Rename statically build library on Windows to avoid conflict with shared version
---
 cpp/cmake_modules/BuildUtils.cmake | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 4e6532be9aa7a..db83efed35031 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -147,11 +147,16 @@ function(ADD_ARROW_LIB LIB_NAME)
   endif()
 
   if (ARROW_BUILD_STATIC)
+      if (MSVC)
+        set(LIB_NAME_STATIC ${LIB_NAME}_static)
+      else()
+        set(LIB_NAME_STATIC ${LIB_NAME})
+      endif()
       add_library(${LIB_NAME}_static STATIC $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
     set_target_properties(${LIB_NAME}_static
       PROPERTIES
       LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
-      OUTPUT_NAME ${LIB_NAME})
+      OUTPUT_NAME ${LIB_NAME_STATIC})
 
   target_link_libraries(${LIB_NAME}_static
       LINK_PUBLIC ${ARG_STATIC_LINK_LIBS}

From 95f489c4c62f964cc32374686e4917774aa8aef2 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 23 Apr 2017 22:15:46 -0400
Subject: [PATCH 0557/1644] ARROW-880: [GLib] Support getting raw data of
 primitive arrays

Author: Kouhei Sutou <kou@clear-code.com>

Closes #586 from kou/glib-primitive-array-buffer and squashes the following commits:

970b109 [Kouhei Sutou] [GLib] Support getting raw data of primitive arrays
---
 c_glib/arrow-glib/array.cpp       | 53 ++++++++++++++----
 c_glib/arrow-glib/array.h         | 89 +++++++++++++++++++++++--------
 c_glib/test/test-boolean-array.rb |  9 ++++
 c_glib/test/test-double-array.rb  |  9 ++++
 c_glib/test/test-float-array.rb   |  9 ++++
 c_glib/test/test-int16-array.rb   |  9 ++++
 c_glib/test/test-int32-array.rb   |  9 ++++
 c_glib/test/test-int64-array.rb   |  9 ++++
 c_glib/test/test-int8-array.rb    |  9 ++++
 c_glib/test/test-uint16-array.rb  |  9 ++++
 c_glib/test/test-uint32-array.rb  |  9 ++++
 c_glib/test/test-uint64-array.rb  |  9 ++++
 c_glib/test/test-uint8-array.rb   |  9 ++++
 13 files changed, 208 insertions(+), 33 deletions(-)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 2fd09015d39ec..3ca860d2ff6d3 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -344,9 +344,40 @@ garrow_null_array_new(gint64 length)
 }
 
 
+G_DEFINE_TYPE(GArrowPrimitiveArray,             \
+              garrow_primitive_array,           \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_primitive_array_init(GArrowPrimitiveArray *object)
+{
+}
+
+static void
+garrow_primitive_array_class_init(GArrowPrimitiveArrayClass *klass)
+{
+}
+
+/**
+ * garrow_primitive_array_get_buffer:
+ * @array: A #GArrowPrimitiveArray.
+ *
+ * Returns: (transfer full): The data of the array as #GArrowBuffer.
+ */
+GArrowBuffer *
+garrow_primitive_array_get_buffer(GArrowPrimitiveArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_primitive_array =
+    static_cast<arrow::PrimitiveArray *>(arrow_array.get());
+  auto arrow_data = arrow_primitive_array->data();
+  return garrow_buffer_new_raw(&arrow_data);
+}
+
+
 G_DEFINE_TYPE(GArrowBooleanArray,               \
               garrow_boolean_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_boolean_array_init(GArrowBooleanArray *object)
@@ -376,7 +407,7 @@ garrow_boolean_array_get_value(GArrowBooleanArray *array,
 
 G_DEFINE_TYPE(GArrowInt8Array,               \
               garrow_int8_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_int8_array_init(GArrowInt8Array *object)
@@ -406,7 +437,7 @@ garrow_int8_array_get_value(GArrowInt8Array *array,
 
 G_DEFINE_TYPE(GArrowUInt8Array,               \
               garrow_uint8_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_uint8_array_init(GArrowUInt8Array *object)
@@ -436,7 +467,7 @@ garrow_uint8_array_get_value(GArrowUInt8Array *array,
 
 G_DEFINE_TYPE(GArrowInt16Array,               \
               garrow_int16_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_int16_array_init(GArrowInt16Array *object)
@@ -466,7 +497,7 @@ garrow_int16_array_get_value(GArrowInt16Array *array,
 
 G_DEFINE_TYPE(GArrowUInt16Array,               \
               garrow_uint16_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_uint16_array_init(GArrowUInt16Array *object)
@@ -496,7 +527,7 @@ garrow_uint16_array_get_value(GArrowUInt16Array *array,
 
 G_DEFINE_TYPE(GArrowInt32Array,               \
               garrow_int32_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_int32_array_init(GArrowInt32Array *object)
@@ -526,7 +557,7 @@ garrow_int32_array_get_value(GArrowInt32Array *array,
 
 G_DEFINE_TYPE(GArrowUInt32Array,               \
               garrow_uint32_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_uint32_array_init(GArrowUInt32Array *object)
@@ -556,7 +587,7 @@ garrow_uint32_array_get_value(GArrowUInt32Array *array,
 
 G_DEFINE_TYPE(GArrowInt64Array,               \
               garrow_int64_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_int64_array_init(GArrowInt64Array *object)
@@ -586,7 +617,7 @@ garrow_int64_array_get_value(GArrowInt64Array *array,
 
 G_DEFINE_TYPE(GArrowUInt64Array,               \
               garrow_uint64_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_uint64_array_init(GArrowUInt64Array *object)
@@ -615,7 +646,7 @@ garrow_uint64_array_get_value(GArrowUInt64Array *array,
 
 G_DEFINE_TYPE(GArrowFloatArray,               \
               garrow_float_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_float_array_init(GArrowFloatArray *object)
@@ -645,7 +676,7 @@ garrow_float_array_get_value(GArrowFloatArray *array,
 
 G_DEFINE_TYPE(GArrowDoubleArray,               \
               garrow_double_array,             \
-              GARROW_TYPE_ARRAY)
+              GARROW_TYPE_PRIMITIVE_ARRAY)
 
 static void
 garrow_double_array_init(GArrowDoubleArray *object)
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index f8c6734a88308..9bb502e4044a9 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -115,6 +115,51 @@ GType garrow_null_array_get_type(void) G_GNUC_CONST;
 GArrowNullArray *garrow_null_array_new(gint64 length);
 
 
+#define GARROW_TYPE_PRIMITIVE_ARRAY             \
+  (garrow_primitive_array_get_type())
+#define GARROW_PRIMITIVE_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_PRIMITIVE_ARRAY,      \
+                              GArrowPrimitiveArray))
+#define GARROW_PRIMITIVE_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_PRIMITIVE_ARRAY, \
+                           GArrowPrimitiveArrayClass))
+#define GARROW_IS_PRIMITIVE_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_PRIMITIVE_ARRAY))
+#define GARROW_IS_PRIMITIVE_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_PRIMITIVE_ARRAY))
+#define GARROW_PRIMITIVE_ARRAY_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_PRIMITIVE_ARRAY,       \
+                             GArrowPrimitiveArrayClass))
+
+typedef struct _GArrowPrimitiveArray         GArrowPrimitiveArray;
+typedef struct _GArrowPrimitiveArrayClass    GArrowPrimitiveArrayClass;
+
+/**
+ * GArrowPrimitiveArray:
+ *
+ * It wraps `arrow::PrimitiveArray`.
+ */
+struct _GArrowPrimitiveArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowPrimitiveArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_primitive_array_get_type(void) G_GNUC_CONST;
+
+GArrowBuffer *garrow_primitive_array_get_buffer(GArrowPrimitiveArray *array);
+
+
 #define GARROW_TYPE_BOOLEAN_ARRAY               \
   (garrow_boolean_array_get_type())
 #define GARROW_BOOLEAN_ARRAY(obj)                               \
@@ -147,12 +192,12 @@ typedef struct _GArrowBooleanArrayClass    GArrowBooleanArrayClass;
 struct _GArrowBooleanArray
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowBooleanArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType          garrow_boolean_array_get_type  (void) G_GNUC_CONST;
@@ -192,12 +237,12 @@ typedef struct _GArrowInt8ArrayClass    GArrowInt8ArrayClass;
 struct _GArrowInt8Array
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowInt8ArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_int8_array_get_type(void) G_GNUC_CONST;
@@ -238,12 +283,12 @@ typedef struct _GArrowUInt8ArrayClass    GArrowUInt8ArrayClass;
 struct _GArrowUInt8Array
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowUInt8ArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_uint8_array_get_type(void) G_GNUC_CONST;
@@ -284,12 +329,12 @@ typedef struct _GArrowInt16ArrayClass    GArrowInt16ArrayClass;
 struct _GArrowInt16Array
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowInt16ArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_int16_array_get_type(void) G_GNUC_CONST;
@@ -330,12 +375,12 @@ typedef struct _GArrowUInt16ArrayClass    GArrowUInt16ArrayClass;
 struct _GArrowUInt16Array
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowUInt16ArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_uint16_array_get_type(void) G_GNUC_CONST;
@@ -376,12 +421,12 @@ typedef struct _GArrowInt32ArrayClass    GArrowInt32ArrayClass;
 struct _GArrowInt32Array
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowInt32ArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_int32_array_get_type(void) G_GNUC_CONST;
@@ -422,12 +467,12 @@ typedef struct _GArrowUInt32ArrayClass    GArrowUInt32ArrayClass;
 struct _GArrowUInt32Array
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowUInt32ArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_uint32_array_get_type(void) G_GNUC_CONST;
@@ -468,12 +513,12 @@ typedef struct _GArrowInt64ArrayClass    GArrowInt64ArrayClass;
 struct _GArrowInt64Array
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowInt64ArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_int64_array_get_type(void) G_GNUC_CONST;
@@ -514,12 +559,12 @@ typedef struct _GArrowUInt64ArrayClass    GArrowUInt64ArrayClass;
 struct _GArrowUInt64Array
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowUInt64ArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_uint64_array_get_type(void) G_GNUC_CONST;
@@ -560,12 +605,12 @@ typedef struct _GArrowFloatArrayClass    GArrowFloatArrayClass;
 struct _GArrowFloatArray
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowFloatArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_float_array_get_type(void) G_GNUC_CONST;
@@ -606,12 +651,12 @@ typedef struct _GArrowDoubleArrayClass    GArrowDoubleArrayClass;
 struct _GArrowDoubleArray
 {
   /*< private >*/
-  GArrowArray parent_instance;
+  GArrowPrimitiveArray parent_instance;
 };
 
 struct _GArrowDoubleArrayClass
 {
-  GArrowArrayClass parent_class;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
 GType garrow_double_array_get_type(void) G_GNUC_CONST;
diff --git a/c_glib/test/test-boolean-array.rb b/c_glib/test/test-boolean-array.rb
index 9cc3c94d554bf..15df1ed95b274 100644
--- a/c_glib/test/test-boolean-array.rb
+++ b/c_glib/test/test-boolean-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestBooleanArray < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append(true)
+    builder.append(false)
+    builder.append(true)
+    array = builder.finish
+    assert_equal([0b101].pack("C*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::BooleanArrayBuilder.new
     builder.append(true)
diff --git a/c_glib/test/test-double-array.rb b/c_glib/test/test-double-array.rb
index f9c000d23f173..c644ac6cc0c07 100644
--- a/c_glib/test/test-double-array.rb
+++ b/c_glib/test/test-double-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestDoubleArray < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::DoubleArrayBuilder.new
+    builder.append(-1.1)
+    builder.append(2.2)
+    builder.append(-4.4)
+    array = builder.finish
+    assert_equal([-1.1, 2.2, -4.4].pack("d*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::DoubleArrayBuilder.new
     builder.append(1.5)
diff --git a/c_glib/test/test-float-array.rb b/c_glib/test/test-float-array.rb
index 020c705aad241..84876f9754da7 100644
--- a/c_glib/test/test-float-array.rb
+++ b/c_glib/test/test-float-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestFloatArray < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::FloatArrayBuilder.new
+    builder.append(-1.1)
+    builder.append(2.2)
+    builder.append(-4.4)
+    array = builder.finish
+    assert_equal([-1.1, 2.2, -4.4].pack("f*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::FloatArrayBuilder.new
     builder.append(1.5)
diff --git a/c_glib/test/test-int16-array.rb b/c_glib/test/test-int16-array.rb
index 2aa5b0c054563..4b30ddd99ff9b 100644
--- a/c_glib/test/test-int16-array.rb
+++ b/c_glib/test/test-int16-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestInt16Array < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::Int16ArrayBuilder.new
+    builder.append(-1)
+    builder.append(2)
+    builder.append(-4)
+    array = builder.finish
+    assert_equal([-1, 2, -4].pack("s*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::Int16ArrayBuilder.new
     builder.append(-1)
diff --git a/c_glib/test/test-int32-array.rb b/c_glib/test/test-int32-array.rb
index 9dd6b3afc8676..90cf0224c1c30 100644
--- a/c_glib/test/test-int32-array.rb
+++ b/c_glib/test/test-int32-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestInt32Array < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::Int32ArrayBuilder.new
+    builder.append(-1)
+    builder.append(2)
+    builder.append(-4)
+    array = builder.finish
+    assert_equal([-1, 2, -4].pack("l*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::Int32ArrayBuilder.new
     builder.append(-1)
diff --git a/c_glib/test/test-int64-array.rb b/c_glib/test/test-int64-array.rb
index 612a8b4f69276..d3022017bb0ee 100644
--- a/c_glib/test/test-int64-array.rb
+++ b/c_glib/test/test-int64-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestInt64Array < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::Int64ArrayBuilder.new
+    builder.append(-1)
+    builder.append(2)
+    builder.append(-4)
+    array = builder.finish
+    assert_equal([-1, 2, -4].pack("q*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::Int64ArrayBuilder.new
     builder.append(-1)
diff --git a/c_glib/test/test-int8-array.rb b/c_glib/test/test-int8-array.rb
index ab009964ab16f..9f28fa7fcd3a3 100644
--- a/c_glib/test/test-int8-array.rb
+++ b/c_glib/test/test-int8-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestInt8Array < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::Int8ArrayBuilder.new
+    builder.append(-1)
+    builder.append(2)
+    builder.append(-4)
+    array = builder.finish
+    assert_equal([-1, 2, -4].pack("c*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::Int8ArrayBuilder.new
     builder.append(-1)
diff --git a/c_glib/test/test-uint16-array.rb b/c_glib/test/test-uint16-array.rb
index ad85f09326bd3..82e898e733625 100644
--- a/c_glib/test/test-uint16-array.rb
+++ b/c_glib/test/test-uint16-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestUInt16Array < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::UInt16ArrayBuilder.new
+    builder.append(1)
+    builder.append(2)
+    builder.append(4)
+    array = builder.finish
+    assert_equal([1, 2, 4].pack("S*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::UInt16ArrayBuilder.new
     builder.append(1)
diff --git a/c_glib/test/test-uint32-array.rb b/c_glib/test/test-uint32-array.rb
index 59e19f3ed796f..c8be06fead5b9 100644
--- a/c_glib/test/test-uint32-array.rb
+++ b/c_glib/test/test-uint32-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestUInt32Array < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::UInt32ArrayBuilder.new
+    builder.append(1)
+    builder.append(2)
+    builder.append(4)
+    array = builder.finish
+    assert_equal([1, 2, 4].pack("L*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::UInt32ArrayBuilder.new
     builder.append(1)
diff --git a/c_glib/test/test-uint64-array.rb b/c_glib/test/test-uint64-array.rb
index e0195c1d49817..03082f33014ce 100644
--- a/c_glib/test/test-uint64-array.rb
+++ b/c_glib/test/test-uint64-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestUInt64Array < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::UInt64ArrayBuilder.new
+    builder.append(1)
+    builder.append(2)
+    builder.append(4)
+    array = builder.finish
+    assert_equal([1, 2, 4].pack("Q*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::UInt64ArrayBuilder.new
     builder.append(1)
diff --git a/c_glib/test/test-uint8-array.rb b/c_glib/test/test-uint8-array.rb
index 02f3470774c10..d7464e336da79 100644
--- a/c_glib/test/test-uint8-array.rb
+++ b/c_glib/test/test-uint8-array.rb
@@ -16,6 +16,15 @@
 # under the License.
 
 class TestUInt8Array < Test::Unit::TestCase
+  def test_buffer
+    builder = Arrow::UInt8ArrayBuilder.new
+    builder.append(1)
+    builder.append(2)
+    builder.append(4)
+    array = builder.finish
+    assert_equal([1, 2, 4].pack("C*"), array.buffer.data.to_s)
+  end
+
   def test_value
     builder = Arrow::UInt8ArrayBuilder.new
     builder.append(1)

From de54eff19af024c1ca0e82f4b45c6021443a635b Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 24 Apr 2017 08:30:08 -0400
Subject: [PATCH 0558/1644] ARROW-659: [C++] Add multithreaded memcpy
 implementation

parallelize memcopy operations for large objects with a multi-threaded implementation.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #580 from atumanov/parallel-memcpy and squashes the following commits:

6ea9873 [Philipp Moritz] fix windows build (?)
66dfa74 [Philipp Moritz] linting
9dd6f3f [Philipp Moritz] cleanup
e81bad9 [Philipp Moritz] add license header
0beb870 [Philipp Moritz] add pthread library
1d73612 [Philipp Moritz] add test of parallel memcopy
1a27431 [Philipp Moritz] restructure code
70d767c [Philipp Moritz] add benchmarks
b320b47 [Philipp Moritz] make memcopy generic
f99606a [Philipp Moritz] add parallel memcpy, contributed by Alexey Tumanov <atumanov@gmail.com>
---
 cpp/CMakeLists.txt                      |  3 +-
 cpp/src/arrow/io/CMakeLists.txt         |  2 +
 cpp/src/arrow/io/io-memory-benchmark.cc | 75 +++++++++++++++++++++++++
 cpp/src/arrow/io/io-memory-test.cc      | 22 ++++++++
 cpp/src/arrow/io/memory.cc              | 31 +++++++++-
 cpp/src/arrow/io/memory.h               |  8 +++
 cpp/src/arrow/ipc/CMakeLists.txt        |  1 +
 cpp/src/arrow/util/memory.h             | 69 +++++++++++++++++++++++
 8 files changed, 207 insertions(+), 4 deletions(-)
 create mode 100644 cpp/src/arrow/io/io-memory-benchmark.cc
 create mode 100644 cpp/src/arrow/util/memory.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 978f70a361756..2d8c00fd80803 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -902,7 +902,8 @@ set(ARROW_STATIC_PRIVATE_LINK_LIBS
 if (NOT MSVC)
   set(ARROW_LINK_LIBS
     ${ARROW_LINK_LIBS}
-    ${CMAKE_DL_LIBS})
+    ${CMAKE_DL_LIBS}
+    pthread)
 endif()
 
 if(RAPIDJSON_VENDORED)
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index c0199d7ef2599..cd489746b48ea 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -22,6 +22,8 @@ ADD_ARROW_TEST(io-file-test)
 ADD_ARROW_TEST(io-hdfs-test)
 ADD_ARROW_TEST(io-memory-test)
 
+ADD_ARROW_BENCHMARK(io-memory-benchmark)
+
 # Headers: top level
 install(FILES
   file.h
diff --git a/cpp/src/arrow/io/io-memory-benchmark.cc b/cpp/src/arrow/io/io-memory-benchmark.cc
new file mode 100644
index 0000000000000..59b511a6cf8fe
--- /dev/null
+++ b/cpp/src/arrow/io/io-memory-benchmark.cc
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/api.h"
+#include "arrow/io/memory.h"
+#include "arrow/test-util.h"
+
+#include "benchmark/benchmark.h"
+
+#include <iostream>
+
+namespace arrow {
+
+static void BM_SerialMemcopy(benchmark::State& state) {  // NOLINT non-const reference
+  constexpr int64_t kTotalSize = 100 * 1024 * 1024; // 100MB
+
+  auto buffer1 = std::make_shared<PoolBuffer>(default_memory_pool());
+  buffer1->Resize(kTotalSize);
+
+  auto buffer2 = std::make_shared<PoolBuffer>(default_memory_pool());
+  buffer2->Resize(kTotalSize);
+  test::random_bytes(kTotalSize, 0, buffer2->mutable_data());
+
+  while (state.KeepRunning()) {
+    io::FixedSizeBufferWriter writer(buffer1);
+    writer.Write(buffer2->data(), buffer2->size());
+  }
+  state.SetBytesProcessed(int64_t(state.iterations()) * kTotalSize);
+}
+
+static void BM_ParallelMemcopy(benchmark::State& state) {  // NOLINT non-const reference
+  constexpr int64_t kTotalSize = 100 * 1024 * 1024; // 100MB
+
+  auto buffer1 = std::make_shared<PoolBuffer>(default_memory_pool());
+  buffer1->Resize(kTotalSize);
+
+  auto buffer2 = std::make_shared<PoolBuffer>(default_memory_pool());
+  buffer2->Resize(kTotalSize);
+  test::random_bytes(kTotalSize, 0, buffer2->mutable_data());
+
+  while (state.KeepRunning()) {
+    io::FixedSizeBufferWriter writer(buffer1);
+    writer.set_memcopy_threads(4);
+    writer.Write(buffer2->data(), buffer2->size());
+  }
+  state.SetBytesProcessed(int64_t(state.iterations()) * kTotalSize);
+}
+
+BENCHMARK(BM_SerialMemcopy)
+    ->RangeMultiplier(4)
+    ->Range(1, 1 << 13)
+    ->MinTime(1.0)
+    ->UseRealTime();
+
+BENCHMARK(BM_ParallelMemcopy)
+    ->RangeMultiplier(4)
+    ->Range(1, 1 << 13)
+    ->MinTime(1.0)
+    ->UseRealTime();
+
+} // namespace arrow
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 4704fe8f4d391..33249cb27f200 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -17,6 +17,7 @@
 
 #include <cstdint>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <memory>
 #include <string>
@@ -114,5 +115,26 @@ TEST(TestBufferReader, RetainParentReference) {
   ASSERT_EQ(0, std::memcmp(slice2->data(), data.c_str() + 4, 6));
 }
 
+TEST(TestMemcopy, ParallelMemcopy) {
+  for (int i = 0; i < 5; ++i) {
+    // randomize size so the memcopy alignment is tested
+    int64_t total_size = 3 * 1024 * 1024 + std::rand() % 100;
+
+    auto buffer1 = std::make_shared<PoolBuffer>(default_memory_pool());
+    buffer1->Resize(total_size);
+
+    auto buffer2 = std::make_shared<PoolBuffer>(default_memory_pool());
+    buffer2->Resize(total_size);
+    test::random_bytes(total_size, 0, buffer2->mutable_data());
+
+    io::FixedSizeBufferWriter writer(buffer1);
+    writer.set_memcopy_threads(4);
+    writer.set_memcopy_threshold(1024 * 1024);
+    writer.Write(buffer2->data(), buffer2->size());
+
+    ASSERT_EQ(0, memcmp(buffer1->data(), buffer2->data(), buffer1->size()));
+  }
+}
+
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 2e701e1104d1c..95c6206f0fab0 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -29,6 +29,7 @@
 #include "arrow/io/interfaces.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/memory.h"
 
 namespace arrow {
 namespace io {
@@ -80,7 +81,7 @@ Status BufferOutputStream::Tell(int64_t* position) {
 Status BufferOutputStream::Write(const uint8_t* data, int64_t nbytes) {
   DCHECK(buffer_);
   RETURN_NOT_OK(Reserve(nbytes));
-  std::memcpy(mutable_data_ + position_, data, nbytes);
+  memcpy(mutable_data_ + position_, data, nbytes);
   position_ += nbytes;
   return Status::OK();
 }
@@ -101,8 +102,15 @@ Status BufferOutputStream::Reserve(int64_t nbytes) {
 // ----------------------------------------------------------------------
 // In-memory buffer writer
 
+static constexpr int kMemcopyDefaultNumThreads = 1;
+static constexpr int64_t kMemcopyDefaultBlocksize = 64;
+static constexpr int64_t kMemcopyDefaultThreshold = 1024 * 1024;
+
 /// Input buffer must be mutable, will abort if not
-FixedSizeBufferWriter::FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer) {
+FixedSizeBufferWriter::FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer)
+    : memcopy_num_threads_(kMemcopyDefaultNumThreads),
+      memcopy_blocksize_(kMemcopyDefaultBlocksize),
+      memcopy_threshold_(kMemcopyDefaultThreshold) {
   buffer_ = buffer;
   DCHECK(buffer->is_mutable()) << "Must pass mutable buffer";
   mutable_data_ = buffer->mutable_data();
@@ -131,7 +139,12 @@ Status FixedSizeBufferWriter::Tell(int64_t* position) {
 }
 
 Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
-  std::memcpy(mutable_data_ + position_, data, nbytes);
+  if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) {
+    parallel_memcopy(mutable_data_ + position_, data, nbytes,
+                     memcopy_blocksize_, memcopy_num_threads_);
+  } else {
+    memcpy(mutable_data_ + position_, data, nbytes);
+  }
   position_ += nbytes;
   return Status::OK();
 }
@@ -143,6 +156,18 @@ Status FixedSizeBufferWriter::WriteAt(
   return Write(data, nbytes);
 }
 
+void FixedSizeBufferWriter::set_memcopy_threads(int num_threads) {
+  memcopy_num_threads_ = num_threads;
+}
+
+void FixedSizeBufferWriter::set_memcopy_blocksize(int64_t blocksize) {
+  memcopy_blocksize_ = blocksize;
+}
+
+void FixedSizeBufferWriter::set_memcopy_threshold(int64_t threshold) {
+  memcopy_threshold_ = threshold;
+}
+
 // ----------------------------------------------------------------------
 // In-memory buffer reader
 
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index fbb186b728022..f1b59905d8a3a 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -81,12 +81,20 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
   Status Write(const uint8_t* data, int64_t nbytes) override;
   Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
 
+  void set_memcopy_threads(int num_threads);
+  void set_memcopy_blocksize(int64_t blocksize);
+  void set_memcopy_threshold(int64_t threshold);
+
  private:
   std::mutex lock_;
   std::shared_ptr<Buffer> buffer_;
   uint8_t* mutable_data_;
   int64_t size_;
   int64_t position_;
+
+  int memcopy_num_threads_;
+  int64_t memcopy_blocksize_;
+  int64_t memcopy_threshold_;
 };
 
 class ARROW_EXPORT BufferReader : public RandomAccessFile {
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index fc1d53e18a3dc..41ab5d7a1f39a 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -95,6 +95,7 @@ if(MSVC)
 else()
   set(UTIL_LINK_LIBS
     arrow_static
+    pthread
     ${BOOST_FILESYSTEM_LIBRARY}
     ${BOOST_SYSTEM_LIBRARY}
     dl)
diff --git a/cpp/src/arrow/util/memory.h b/cpp/src/arrow/util/memory.h
new file mode 100644
index 0000000000000..7feeb291ef4a0
--- /dev/null
+++ b/cpp/src/arrow/util/memory.h
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_MEMORY_H
+#define ARROW_UTIL_MEMORY_H
+
+#include <thread>
+#include <vector>
+
+namespace arrow {
+
+uint8_t* pointer_logical_and(const uint8_t* address, uintptr_t bits) {
+  uintptr_t value = reinterpret_cast<uintptr_t>(address);
+  return reinterpret_cast<uint8_t*>(value & bits);
+}
+
+// A helper function for doing memcpy with multiple threads. This is required
+// to saturate the memory bandwidth of modern cpus.
+void parallel_memcopy(uint8_t* dst, const uint8_t* src, int64_t nbytes,
+                      uintptr_t block_size, int num_threads) {
+  std::vector<std::thread> threadpool(num_threads);
+  uint8_t* left = pointer_logical_and(src + block_size - 1, ~(block_size - 1));
+  uint8_t* right = pointer_logical_and(src + nbytes, ~(block_size - 1));
+  int64_t num_blocks = (right - left) / block_size;
+
+  // Update right address
+  right = right - (num_blocks % num_threads) * block_size;
+
+  // Now we divide these blocks between available threads. The remainder is
+  // handled on the main thread.
+  int64_t chunk_size = (right - left) / num_threads;
+  int64_t prefix = left - src;
+  int64_t suffix = src + nbytes - right;
+  // Now the data layout is | prefix | k * num_threads * block_size | suffix |.
+  // We have chunk_size = k * block_size, therefore the data layout is
+  // | prefix | num_threads * chunk_size | suffix |.
+  // Each thread gets a "chunk" of k blocks.
+
+  // Start all threads first and handle leftovers while threads run.
+  for (int i = 0; i < num_threads; i++) {
+    threadpool[i] = std::thread(memcpy, dst + prefix + i * chunk_size,
+        left + i * chunk_size, chunk_size);
+  }
+
+  memcpy(dst, src, prefix);
+  memcpy(dst + prefix + num_threads * chunk_size, right, suffix);
+
+  for (auto& t : threadpool) {
+    if (t.joinable()) { t.join(); }
+  }
+}
+
+} // namespace arrow
+
+#endif  // ARROW_UTIL_MEMORY_H

From 76d56d3aa9607976b162f6d924a23c12c8800236 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 24 Apr 2017 15:57:31 -0400
Subject: [PATCH 0559/1644] ARROW-95: Add Jekyll-based website publishing
 toolchain, migrate existing arrow-site

This also renders the format Markdown documents on the website. Used the Apache Calcite website for guidance about best practices with Jekyll.

See rendered website at https://wesm.github.io/arrow-site-test/

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #589 from wesm/ARROW-95 and squashes the following commits:

a6b65cb [Wes McKinney] Fix some incomplete instructions
2806d26 [Wes McKinney] Exclude flatbuffers from C++ API docs. Add C++ docs link to site
512ea71 [Wes McKinney] Migrate website to Jekyll with bootstrap-sass. Add navbar. Render specification Markdown documents with website. Instructions for publishing Java and Python docs
---
 cpp/apidoc/Doxyfile              |   3 +-
 dev/release/run-rat.sh           |   4 +-
 java/pom.xml                     |  25 ++++
 site/.gitignore                  |   6 +
 site/Gemfile                     |  25 ++++
 site/README.md                   |  85 +++++++++++++
 site/_config.yml                 |  43 +++++++
 site/_docs/.gitignore            |   1 +
 site/_docs/ipc.md                |  25 ++++
 site/_docs/memory_layout.md      |  25 ++++
 site/_docs/metadata.md           |  25 ++++
 site/_includes/footer.html       |   4 +
 site/_includes/header.html       |  53 ++++++++
 site/_includes/top.html          |  20 +++
 site/_layouts/default.html       |  12 ++
 site/_layouts/docs.html          |  14 +++
 site/_sass/_font-awesome.scss    |  25 ++++
 site/css/main.scss               |  10 ++
 site/img/asf_logo.svg            | 210 +++++++++++++++++++++++++++++++
 site/img/copy.png                | Bin 0 -> 23204 bytes
 site/img/copy2.png               | Bin 0 -> 37973 bytes
 site/img/shared.png              | Bin 0 -> 37973 bytes
 site/img/shared2.png             | Bin 0 -> 23204 bytes
 site/img/simd.png                | Bin 0 -> 101031 bytes
 site/index.html                  | 171 +++++++++++++++++++++++++
 site/scripts/sync_format_docs.sh |  23 ++++
 26 files changed, 807 insertions(+), 2 deletions(-)
 create mode 100644 site/.gitignore
 create mode 100644 site/Gemfile
 create mode 100644 site/README.md
 create mode 100644 site/_config.yml
 create mode 100644 site/_docs/.gitignore
 create mode 100644 site/_docs/ipc.md
 create mode 100644 site/_docs/memory_layout.md
 create mode 100644 site/_docs/metadata.md
 create mode 100644 site/_includes/footer.html
 create mode 100644 site/_includes/header.html
 create mode 100644 site/_includes/top.html
 create mode 100644 site/_layouts/default.html
 create mode 100644 site/_layouts/docs.html
 create mode 100644 site/_sass/_font-awesome.scss
 create mode 100644 site/css/main.scss
 create mode 100644 site/img/asf_logo.svg
 create mode 100644 site/img/copy.png
 create mode 100644 site/img/copy2.png
 create mode 100644 site/img/shared.png
 create mode 100644 site/img/shared2.png
 create mode 100644 site/img/simd.png
 create mode 100644 site/index.html
 create mode 100755 site/scripts/sync_format_docs.sh

diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile
index 51f5543b2de1b..3127662413328 100644
--- a/cpp/apidoc/Doxyfile
+++ b/cpp/apidoc/Doxyfile
@@ -891,7 +891,7 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                =
+EXCLUDE =
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -908,6 +908,7 @@ EXCLUDE_SYMLINKS       = NO
 # exclude all test directories for example use the pattern */test/*
 
 EXCLUDE_PATTERNS       = *-test.cc \
+                         *_generated.h \
                          *-benchmark.cc
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index a3c12a0ce8a92..9c34e073e628e 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -58,13 +58,15 @@ $RAT $1 \
   -e "*.html" \
   -e "*.css" \
   -e "*.png" \
+  -e "*.svg" \
   -e "*.devhelp2" \
+  -e "*.scss" \
   > rat.txt
 cat rat.txt
 UNAPPROVED=`cat rat.txt  | grep "Unknown Licenses" | head -n 1 | cut -d " " -f 1`
 
 if [ "0" -eq "${UNAPPROVED}" ]; then
-  echo "No unnaproved licenses"
+  echo "No unapproved licenses"
 else
   echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
   exit 1
diff --git a/java/pom.xml b/java/pom.xml
index 5d07186e3e714..e586005e395c0 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -532,6 +532,31 @@
 
   </dependencies>
 
+  <reporting>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>2.9</version>
+        <reportSets>
+          <reportSet><!-- by default, id = "default" -->
+            <reports><!-- select non-aggregate reports -->
+              <report>javadoc</report>
+              <report>test-javadoc</report>
+            </reports>
+          </reportSet>
+          <reportSet><!-- aggregate reportSet, to define in poms having modules -->
+            <id>aggregate</id>
+            <inherited>false</inherited><!-- don't run aggregate in child modules -->
+            <reports>
+              <report>aggregate</report>
+            </reports>
+          </reportSet>
+        </reportSets>
+      </plugin>
+    </plugins>
+  </reporting>
+
   <modules>
     <module>format</module>
     <module>memory</module>
diff --git a/site/.gitignore b/site/.gitignore
new file mode 100644
index 0000000000000..46bc466d3028e
--- /dev/null
+++ b/site/.gitignore
@@ -0,0 +1,6 @@
+_site
+.sass-cache
+.jekyll-metadata
+Gemfile.lock
+asf-site
+build/
diff --git a/site/Gemfile b/site/Gemfile
new file mode 100644
index 0000000000000..98decaf35dbe6
--- /dev/null
+++ b/site/Gemfile
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source "https://rubygems.org"
+ruby RUBY_VERSION
+gem "jekyll", "3.4.3"
+gem 'jekyll-bootstrap-sass'
+gem 'github-pages'
+group :jekyll_plugins do
+   gem "jekyll-feed", "~> 0.6"
+end
+gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
diff --git a/site/README.md b/site/README.md
new file mode 100644
index 0000000000000..3f8da2252f965
--- /dev/null
+++ b/site/README.md
@@ -0,0 +1,85 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+## Apache Arrow Website
+
+### Development instructions
+
+If you are planning to publish the website, you must first clone the arrow-site
+git repository:
+
+```shell
+git clone --branch=asf-site https://git-wip-us.apache.org/repos/asf/arrow-site.git asf-site
+```
+
+Now, with Ruby >= 2.1 installed, run:
+
+```shell
+gem install jekyll bundler
+bundle install
+
+# This imports the format Markdown documents so they will be rendered
+scripts/sync_format_docs.sh
+
+bundle exec jekyll serve
+```
+
+### Publishing
+
+After following the above instructions the base `site/` directory, run:
+
+```shell
+bundle exec jekyll build
+rsync -r build/ asf-site/
+cd asf-site
+git status
+```
+
+Now `git add` any new files, then commit everything, and push:
+
+```
+git push
+```
+
+### Updating Code Documentation
+
+#### Java
+
+```
+cd ../java
+mvn install
+mvn site
+rsync -r target/site/apidocs/ ../site/asf-site/docs/java/
+```
+
+#### C++
+
+```
+cd ../cpp/apidoc
+doxygen Doxyfile
+rsync -r html/ ../../site/asf-site/docs/cpp
+```
+
+#### Python
+
+First, build PyArrow with all optional extensions (Apache Parquet, jemalloc).
+
+```
+cd ../python
+python setup.py build_ext --inplace --with-parquet --with-jemalloc
+python setup.py build_sphinx -s doc/source
+rsync -r doc/_build/html/ ../site/asf-site/docs/python/
+```
+
+Then add/commit/push from the site/asf-site git checkout.
\ No newline at end of file
diff --git a/site/_config.yml b/site/_config.yml
new file mode 100644
index 0000000000000..922af4a08059c
--- /dev/null
+++ b/site/_config.yml
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+markdown: kramdown
+repository: https://github.com/apache/arrow
+destination: build
+
+exclude:
+  - Gemfile
+  - Gemfile.lock
+  - _docs/format/*
+  - asf-site
+  - scripts
+  - README.md
+
+collections:
+  docs:
+    output: true
+
+sass:
+  style: compressed
+
+# The base path where the website is deployed
+baseurl:
+
+gems:
+  - jekyll-feed
+  - jekyll-bootstrap-sass
+
+bootstrap:
+  assets: true
diff --git a/site/_docs/.gitignore b/site/_docs/.gitignore
new file mode 100644
index 0000000000000..1e942fc5eadfb
--- /dev/null
+++ b/site/_docs/.gitignore
@@ -0,0 +1 @@
+format/
\ No newline at end of file
diff --git a/site/_docs/ipc.md b/site/_docs/ipc.md
new file mode 100644
index 0000000000000..bc22dc3bfa7b7
--- /dev/null
+++ b/site/_docs/ipc.md
@@ -0,0 +1,25 @@
+---
+layout: docs
+title: Arrow Messaging and IPC
+permalink: /docs/ipc.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+{% include_relative format/IPC.md %}
\ No newline at end of file
diff --git a/site/_docs/memory_layout.md b/site/_docs/memory_layout.md
new file mode 100644
index 0000000000000..74cd7ed7f7de0
--- /dev/null
+++ b/site/_docs/memory_layout.md
@@ -0,0 +1,25 @@
+---
+layout: docs
+title: Physical Memory Layout
+permalink: /docs/memory_layout.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+{% include_relative format/Layout.md %}
\ No newline at end of file
diff --git a/site/_docs/metadata.md b/site/_docs/metadata.md
new file mode 100644
index 0000000000000..382ab0eaaf61b
--- /dev/null
+++ b/site/_docs/metadata.md
@@ -0,0 +1,25 @@
+---
+layout: docs
+title: Arrow Metadata
+permalink: /docs/metadata.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+{% include_relative format/Metadata.md %}
\ No newline at end of file
diff --git a/site/_includes/footer.html b/site/_includes/footer.html
new file mode 100644
index 0000000000000..c2a7d5e92bb20
--- /dev/null
+++ b/site/_includes/footer.html
@@ -0,0 +1,4 @@
+<footer class="footer">
+  <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+  <p>&copy; 2017 Apache Software Foundation</p>
+</footer>
diff --git a/site/_includes/header.html b/site/_includes/header.html
new file mode 100644
index 0000000000000..5963c22abea0d
--- /dev/null
+++ b/site/_includes/header.html
@@ -0,0 +1,53 @@
+<nav class="navbar navbar-default">
+  <div class="container-fluid">
+    <div class="navbar-header">
+      <a class="navbar-brand" href="{{ site.baseurl }}/">Apache Arrow&#8482;&nbsp;&nbsp;&nbsp;</a>
+    </div>
+
+    <!-- Collect the nav links, forms, and other content for toggling -->
+    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+      <ul class="nav navbar-nav">
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Specification<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="{{ site.baseurl }}/docs/memory_layout.html">Memory Layout</a></li>
+            <li><a href="{{ site.baseurl }}/docs/metadata.html">Metadata</a></li>
+            <li><a href="{{ site.baseurl }}/docs/ipc.html">Messaging / IPC</a></li>
+          </ul>
+        </li>
+
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Code Documentation<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="{{ site.baseurl }}/docs/cpp">C++</a></li>
+            <li><a href="{{ site.baseurl }}/docs/java">Java</a></li>
+            <li><a href="{{ site.baseurl }}/docs/python">Python</a></li>
+          </ul>
+        </li>
+        <!-- <li><a href="{{ site.baseurl }}/blog">Blog</a></li> -->
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">ASF Links<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="http://www.apache.org/">ASF Website</a></li>
+            <li><a href="http://www.apache.org/licenses/">License</a></li>
+            <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
+            <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+            <li><a href="http://www.apache.org/security/">Security</a></li>
+          </ul>
+        </li>
+      </ul>
+      <a href="http://www.apache.org/">
+        <img style="float:right;" src="{{ site.baseurl }}/img/asf_logo.svg" width="120px"/>
+      </a>
+      </div><!-- /.navbar-collapse -->
+    </div>
+  </nav>
diff --git a/site/_includes/top.html b/site/_includes/top.html
new file mode 100644
index 0000000000000..cc537bac07ba3
--- /dev/null
+++ b/site/_includes/top.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="UTF-8">
+    <title>{{ page.title }}</title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="generator" content="Jekyll v{{ jekyll.version }}">
+    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+    <link rel="icon" type="image/x-icon" href="{{ site.baseurl }}/favicon.ico">
+
+    <title>Apache Arrow Homepage</title>
+    <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+    <link href="{{ site.baseurl }}/css/main.css" rel="stylesheet">
+    <script src="https://code.jquery.com/jquery-3.2.1.min.js"
+            integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+            crossorigin="anonymous"></script>
+    <script src="{{ site.baseurl }}/assets/javascripts/bootstrap.min.js"></script>
+  </head>
diff --git a/site/_layouts/default.html b/site/_layouts/default.html
new file mode 100644
index 0000000000000..d0ff799b97ab3
--- /dev/null
+++ b/site/_layouts/default.html
@@ -0,0 +1,12 @@
+{% include top.html %}
+
+<body class="wrap">
+  <div class="container">
+    {% include header.html %}
+
+    {{ content }}
+
+    {% include footer.html %}
+  </div>
+</body>
+</html>
diff --git a/site/_layouts/docs.html b/site/_layouts/docs.html
new file mode 100644
index 0000000000000..2ef9cf485e47c
--- /dev/null
+++ b/site/_layouts/docs.html
@@ -0,0 +1,14 @@
+{% include top.html %}
+
+<body class="wrap">
+  <div class="container">
+    {% include header.html %}
+
+    {{ content }}
+
+    <hr></hr>
+
+    {% include footer.html %}
+  </div>
+</body>
+</html>
diff --git a/site/_sass/_font-awesome.scss b/site/_sass/_font-awesome.scss
new file mode 100644
index 0000000000000..d90676c2b9e59
--- /dev/null
+++ b/site/_sass/_font-awesome.scss
@@ -0,0 +1,25 @@
+/*!
+ *  Font Awesome 4.2.0 by @davegandy - http://fontawesome.io - @fontawesome
+ *  License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License)
+ */
+@font-face {
+  font-family: 'FontAwesome';
+  src: url('../fonts/fontawesome-webfont.eot?v=4.2.0');
+  src: url('../fonts/fontawesome-webfont.eot?#iefix&v=4.2.0') format('embedded-opentype'), url('../fonts/fontawesome-webfont.woff?v=4.2.0') format('woff'), url('../fonts/fontawesome-webfont.ttf?v=4.2.0') format('truetype'), url('../fonts/fontawesome-webfont.svg?v=4.2.0#fontawesomeregular') format('svg');
+  font-weight: normal;
+  font-style: normal;
+}
+.fa {
+  display: inline-block;
+  font: normal normal normal 14px/1 FontAwesome;
+  font-size: inherit;
+  text-rendering: auto;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+.fa-link:before {
+  content: "\f0c1";
+}
+.fa-pencil:before {
+  content: "\f040";
+}
diff --git a/site/css/main.scss b/site/css/main.scss
new file mode 100644
index 0000000000000..24b46ae24ccf2
--- /dev/null
+++ b/site/css/main.scss
@@ -0,0 +1,10 @@
+---
+---
+
+$container-desktop: 960px;
+$container-large-desktop: $container-desktop;
+$grid-gutter-width: 15px;
+
+@import "bootstrap-sprockets";
+@import "bootstrap";
+@import "font-awesome";
diff --git a/site/img/asf_logo.svg b/site/img/asf_logo.svg
new file mode 100644
index 0000000000000..620694c52418a
--- /dev/null
+++ b/site/img/asf_logo.svg
@@ -0,0 +1,210 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_2" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 7127.6 2890" enable-background="new 0 0 7127.6 2890" xml:space="preserve">
+<path fill="#6D6E71" d="M7104.7,847.8c15.3,15.3,22.9,33.7,22.9,55.2c0,21.5-7.6,39.9-22.9,55.4c-15.3,15.4-33.8,23.1-55.6,23.1
+	c-21.8,0-40.2-7.6-55.4-22.9c-15.1-15.3-22.7-33.7-22.7-55.2c0-21.5,7.6-39.9,22.9-55.4c15.3-15.4,33.7-23.1,55.4-23.1
+	C7070.9,824.9,7089.4,832.5,7104.7,847.8z M7098.1,951.9c13.3-13.6,20-29.8,20-48.7s-6.6-35-19.8-48.5
+	c-13.2-13.4-29.4-20.1-48.6-20.1c-19.2,0-35.4,6.7-48.7,20.2c-13.3,13.5-19.9,29.7-19.9,48.7c0,19,6.6,35.2,19.7,48.6
+	c13.1,13.4,29.3,20.1,48.5,20.1S7084.7,965.4,7098.1,951.9z M7087.1,888.1c0,14-6.1,22.8-18.4,26.4l22.5,30.5h-18.2l-20.3-28.3
+	h-18.6v28.3h-14.7v-84.6h31.8c12.8,0,22,2.2,27.6,6.6C7084.4,871.4,7087.1,878.4,7087.1,888.1z M7068.2,900c3-2.4,4.4-6.5,4.4-12
+	c0-5.5-1.5-9.4-4.5-11.6c-3-2.2-8.4-3.2-16-3.2h-18v30.5h17.5C7059.7,903.6,7065.3,902.4,7068.2,900z"/>
+<path fill="#6D6E71" d="M1803.6,499.8v155.4h-20V499.8h-56.8v-19.2h133.9v19.2H1803.6z"/>
+<path fill="#6D6E71" d="M2082.2,655.2v-76.9h-105.2v76.9h-20V480.5h20v78.9h105.2v-78.9h20v174.7H2082.2z"/>
+<path fill="#6D6E71" d="M2241.4,499.8v57.4h88.1v19.2h-88.1v59.8h101.8v19h-121.8V480.5H2340v19.2H2241.4z"/>
+<path fill="#D22128" d="M1574.5,1852.4l417.3-997.6h80.1l417.3,997.6h-105.4l-129.3-311.9h-448.2l-127.9,311.9H1574.5z M2032.6,970
+	l-205.1,493.2h404.7L2032.6,970z"/>
+<path fill="#D22128" d="M2596.9,1852.4V854.8H3010c171.4,0,295.1,158.8,295.1,313.3c0,163-115.2,316.1-286.6,316.1h-324.6v368.1
+	H2596.9z M2693.9,1397.1h318.9c118,0,193.9-108.2,193.9-229c0-125.1-92.7-226.2-202.3-226.2h-310.5V1397.1z"/>
+<path fill="#D22128" d="M3250.5,1852.4l417.3-997.6h80.1l417.3,997.6h-105.4l-129.3-311.9h-448.2l-127.9,311.9H3250.5z M3708.6,970
+	l-205.1,493.2h404.7L3708.6,970z"/>
+<path fill="#D22128" d="M4637.3,849.1c177,0,306.3,89.9,368.1,217.8l-78.7,47.8c-63.2-132.1-186.9-177-295.1-177
+	c-238.9,0-369.5,213.6-369.5,414.5c0,220.6,161.6,420.1,373.7,420.1c112.4,0,244.5-56.2,307.7-185.5l81.5,42.1
+	c-64.6,148.9-241.7,231.8-394.8,231.8c-274,0-466.5-261.3-466.5-514.2C4163.8,1106.3,4336.6,849.1,4637.3,849.1z"/>
+<path fill="#D22128" d="M5949.1,854.8v997.6h-98.4v-466.5h-591.5v466.5h-96.9V854.8h96.9v444h591.5v-444H5949.1z"/>
+<path fill="#D22128" d="M6844.6,1765.2v87.1h-670.2V854.8H6832v87.1h-560.6v359.7h489v82.9h-489v380.8H6844.6z"/>
+<path fill="#6D6E71" d="M1667.6,2063.6c11.8,3.5,22.2,8.3,31,14.2l-10.3,22.6c-9-6-18.6-10.4-28.9-13.4c-10.2-2.9-20-4.4-29.2-4.4
+	c-13.6,0-24.5,2.4-32.6,7.3c-8.1,4.9-12.2,11.8-12.2,20.7c0,7.6,2.3,14,6.8,19c4.5,5,10.2,8.9,17,11.7c6.8,2.8,16.1,6,28,9.6
+	c14.4,4.6,26,8.9,34.7,12.9c8.8,4,16.3,9.9,22.5,17.8c6.2,7.8,9.3,18.2,9.3,31c0,11.7-3.2,21.8-9.5,30.6
+	c-6.3,8.7-15.3,15.5-26.8,20.3c-11.6,4.8-24.9,7.2-40,7.2c-15.1,0-29.7-2.9-43.9-8.7c-14.2-5.8-26.4-13.6-36.6-23.4l10.7-21.6
+	c9.6,9.4,20.7,16.7,33.3,21.9c12.6,5.2,24.8,7.8,36.8,7.8c15.3,0,27.3-3,36.1-8.9c8.8-5.9,13.2-13.9,13.2-23.9
+	c0-7.8-2.3-14.3-6.9-19.4c-4.6-5.1-10.3-9-17.1-11.9c-6.8-2.8-16.1-6-28-9.6c-14.2-4.2-25.7-8.3-34.6-12.2
+	c-8.9-3.9-16.4-9.7-22.5-17.5c-6.1-7.7-9.2-17.9-9.2-30.6c0-10.9,3-20.4,9-28.6c6-8.2,14.6-14.6,25.6-19.1
+	c11.1-4.5,23.8-6.8,38.2-6.8C1643.8,2058.3,1655.7,2060.1,1667.6,2063.6z"/>
+<path fill="#6D6E71" d="M1980.1,2072.8c16.8,9.4,30.2,22.3,40,38.4c9.8,16.2,14.8,33.9,14.8,53.3c0,19.5-4.9,37.4-14.8,53.6
+	c-9.8,16.3-23.2,29.1-40,38.6c-16.8,9.5-35.3,14.3-55.2,14.3c-20.3,0-38.8-4.7-55.7-14.3c-16.8-9.5-30.2-22.4-40-38.6
+	c-9.8-16.3-14.8-34.1-14.8-53.6c0-19.5,4.9-37.3,14.8-53.5c9.8-16.2,23.2-29,40-38.3c16.8-9.4,35.4-14,55.7-14
+	C1944.8,2058.6,1963.2,2063.3,1980.1,2072.8z M1881.9,2092.7c-13.1,7.4-23.6,17.5-31.4,30.1c-7.8,12.6-11.8,26.5-11.8,41.7
+	c0,15.3,3.9,29.3,11.8,42c7.8,12.7,18.3,22.8,31.4,30.2c13.1,7.4,27.4,11.1,42.9,11.1c15.5,0,29.7-3.7,42.7-11.1
+	c13-7.4,23.3-17.4,31.1-30.2c7.7-12.7,11.6-26.7,11.6-42s-3.9-29.2-11.6-41.8c-7.7-12.6-18.1-22.6-31.1-30
+	c-13-7.4-27.2-11.2-42.6-11.2C1909.4,2081.5,1895.1,2085.2,1881.9,2092.7z"/>
+<path fill="#6D6E71" d="M2186.5,2082.4v74h98.4v23.2h-98.4v90.2h-24.1v-210.6h133.8v23.2H2186.5z"/>
+<path fill="#6D6E71" d="M2491.6,2082.4v187.4h-24.1v-187.4h-68.4v-23.2h161.4v23.2H2491.6z"/>
+<path fill="#6D6E71" d="M2871.8,2269.8l-56.8-177.4l-57.6,177.4h-24.5l-70.5-210.6h25.9l57.9,182.7l57.1-182.4l24.1-0.3l57.7,182.7
+	l57.1-182.7h25l-70.6,210.6H2871.8z"/>
+<path fill="#6D6E71" d="M3087.3,2216.6l-23.5,53.2h-25.6l94.4-210.6h25l94.1,210.6h-26.1l-23.5-53.2H3087.3z M3144.5,2086.6
+	l-46.9,106.8h94.4L3144.5,2086.6z"/>
+<path fill="#6D6E71" d="M3461.1,2202.7c-6,0.4-10.7,0.6-14.1,0.6h-56v66.5H3367v-210.6h80c26.2,0,46.6,6.2,61.2,18.5
+	c14.5,12.3,21.8,29.8,21.8,52.3c0,17.2-4.1,31.7-12.2,43.3c-8.1,11.6-19.8,20-35,25l49.2,71.5h-27.3L3461.1,2202.7z M3491.3,2167.6
+	c10.3-8.4,15.5-20.8,15.5-37c0-15.9-5.2-27.9-15.5-36c-10.3-8.1-25.1-12.2-44.3-12.2h-56v97.8h56
+	C3466.2,2180.2,3481,2176,3491.3,2167.6z"/>
+<path fill="#6D6E71" d="M3688.3,2082.4v69.2h106.2v23.2h-106.2v72.1h122.8v22.9h-146.9v-210.6h142.9v23.2H3688.3z"/>
+<path fill="#6D6E71" d="M4147,2082.4v74h98.4v23.2H4147v90.2h-24.1v-210.6h133.8v23.2H4147z"/>
+<path fill="#6D6E71" d="M4523.3,2072.8c16.8,9.4,30.2,22.3,40,38.4c9.8,16.2,14.8,33.9,14.8,53.3c0,19.5-4.9,37.4-14.8,53.6
+	c-9.8,16.3-23.2,29.1-40,38.6c-16.8,9.5-35.3,14.3-55.2,14.3c-20.3,0-38.8-4.7-55.7-14.3c-16.8-9.5-30.2-22.4-40-38.6
+	c-9.8-16.3-14.8-34.1-14.8-53.6c0-19.5,4.9-37.3,14.8-53.5c9.8-16.2,23.2-29,40-38.3c16.8-9.4,35.4-14,55.7-14
+	C4488.1,2058.6,4506.5,2063.3,4523.3,2072.8z M4425.2,2092.7c-13.1,7.4-23.6,17.5-31.4,30.1c-7.8,12.6-11.8,26.5-11.8,41.7
+	c0,15.3,3.9,29.3,11.8,42c7.8,12.7,18.3,22.8,31.4,30.2c13.1,7.4,27.4,11.1,42.9,11.1c15.5,0,29.7-3.7,42.7-11.1
+	c13-7.4,23.3-17.4,31.1-30.2c7.7-12.7,11.6-26.7,11.6-42s-3.9-29.2-11.6-41.8c-7.7-12.6-18.1-22.6-31.1-30
+	c-13-7.4-27.2-11.2-42.6-11.2C4452.6,2081.5,4438.3,2085.2,4425.2,2092.7z"/>
+<path fill="#6D6E71" d="M4854.7,2247.7c-15.7,15.5-37.3,23.3-64.8,23.3c-27.7,0-49.4-7.8-65.1-23.3c-15.7-15.5-23.6-37-23.6-64.6
+	v-124h24.1v124c0,20.3,5.8,36.1,17.3,47.5c11.6,11.4,27.3,17.1,47.3,17.1c20.1,0,35.8-5.7,47.1-17c11.4-11.3,17-27.2,17-47.7v-124
+	h24.1v124C4878.2,2210.7,4870.4,2232.2,4854.7,2247.7z"/>
+<path fill="#6D6E71" d="M5169.5,2269.8l-126.3-169.1v169.1h-24.1v-210.6h25l126.3,169.3v-169.3h23.8v210.6H5169.5z"/>
+<path fill="#6D6E71" d="M5478.4,2073.1c16.4,9.3,29.4,21.9,38.9,37.9c9.6,16,14.3,33.9,14.3,53.5s-4.8,37.6-14.3,53.6
+	c-9.5,16.1-22.6,28.7-39.3,37.9c-16.6,9.2-35.2,13.8-55.5,13.8h-84.3v-210.6h85.2C5443.7,2059.2,5462,2063.8,5478.4,2073.1z
+	 M5362.3,2246.9h61.4c15.5,0,29.6-3.5,42.3-10.6c12.7-7.1,22.8-16.9,30.2-29.5c7.4-12.5,11.1-26.5,11.1-42
+	c0-15.5-3.8-29.4-11.3-41.9c-7.5-12.5-17.7-22.3-30.6-29.6c-12.8-7.2-27-10.9-42.6-10.9h-60.5V2246.9z"/>
+<path fill="#6D6E71" d="M5668.6,2216.6l-23.5,53.2h-25.6l94.4-210.6h25l94.1,210.6H5807l-23.5-53.2H5668.6z M5725.8,2086.6
+	l-46.9,106.8h94.4L5725.8,2086.6z"/>
+<path fill="#6D6E71" d="M5991,2082.4v187.4H5967v-187.4h-68.4v-23.2h161.4v23.2H5991z"/>
+<path fill="#6D6E71" d="M6175.9,2269.8v-210.6h24.1v210.6H6175.9z"/>
+<path fill="#6D6E71" d="M6493.7,2072.8c16.8,9.4,30.2,22.3,40,38.4c9.8,16.2,14.8,33.9,14.8,53.3c0,19.5-4.9,37.4-14.8,53.6
+	c-9.8,16.3-23.2,29.1-40,38.6c-16.8,9.5-35.3,14.3-55.2,14.3c-20.3,0-38.8-4.7-55.7-14.3c-16.8-9.5-30.2-22.4-40-38.6
+	c-9.8-16.3-14.8-34.1-14.8-53.6c0-19.5,4.9-37.3,14.8-53.5c9.8-16.2,23.2-29,40-38.3c16.8-9.4,35.4-14,55.7-14
+	C6458.5,2058.6,6476.9,2063.3,6493.7,2072.8z M6395.6,2092.7c-13.1,7.4-23.6,17.5-31.4,30.1c-7.8,12.6-11.8,26.5-11.8,41.7
+	c0,15.3,3.9,29.3,11.8,42c7.8,12.7,18.3,22.8,31.4,30.2c13.1,7.4,27.4,11.1,42.9,11.1c15.5,0,29.7-3.7,42.7-11.1
+	c13-7.4,23.3-17.4,31.1-30.2c7.7-12.7,11.6-26.7,11.6-42s-3.9-29.2-11.6-41.8c-7.7-12.6-18.1-22.6-31.1-30
+	c-13-7.4-27.2-11.2-42.6-11.2C6423,2081.5,6408.8,2085.2,6395.6,2092.7z"/>
+<path fill="#6D6E71" d="M6826.5,2269.8l-126.3-169.1v169.1h-24.1v-210.6h25l126.3,169.3v-169.3h23.8v210.6H6826.5z"/>
+<linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="-4516.6152" y1="-2338.7222" x2="-4108.4111" y2="-1861.3982" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0" style="stop-color:#F69923"/>
+	<stop  offset="0.3123" style="stop-color:#F79A23"/>
+	<stop  offset="0.8383" style="stop-color:#E97826"/>
+</linearGradient>
+<path fill="url(#SVGID_1_)" d="M1230.1,13.7c-45.3,26.8-120.6,102.5-210.5,212.3l82.6,155.9c58-82.9,116.9-157.5,176.3-221.2
+	c4.6-5.1,7-7.5,7-7.5c-2.3,2.5-4.6,5-7,7.5c-19.2,21.2-77.5,89.2-165.5,224.4c84.7-4.2,214.9-21.6,321.1-39.7
+	c31.6-177-31-258-31-258S1323.4-41.4,1230.1,13.7z"/>
+<path fill="none" d="M1090.2,903.1c0.6-0.1,1.2-0.2,1.8-0.3l-11.9,1.3c-0.7,0.3-1.4,0.7-2.1,1
+	C1082.1,904.4,1086.2,903.7,1090.2,903.1z"/>
+<path fill="none" d="M1005.9,1182.3c-6.7,1.5-13.7,2.7-20.7,3.7C992.3,1185,999.2,1183.8,1005.9,1182.3z"/>
+<path fill="none" d="M432.9,1808.8c0.9-2.3,1.8-4.7,2.6-7c18.2-48,36.2-94.7,54-140.1c20-51,39.8-100.4,59.3-148.3
+	c20.6-50.4,40.9-99.2,60.9-146.3c21-49.4,41.7-97,62-142.8c16.5-37.3,32.8-73.4,48.9-108.3c5.4-11.7,10.7-23.2,16-34.6
+	c10.5-22.7,21-44.8,31.3-66.5c9.5-20,19-39.6,28.3-58.8c3.1-6.4,6.2-12.8,9.3-19.1c0.5-1,1-2,1.5-3.1l-10.2,1.1l-8-15.9
+	c-0.8,1.6-1.6,3.1-2.4,4.6c-14.5,28.8-28.9,57.9-43.1,87.2c-8.2,16.9-16.4,34-24.6,51c-22.6,47.4-44.8,95.2-66.6,143.3
+	c-22.1,48.6-43.7,97.5-64.9,146.5c-20.8,48.1-41.3,96.2-61.2,144.2c-20,48-39.5,95.7-58.5,143.2c-19.9,49.5-39.2,98.7-58,147.2
+	c-4.2,10.9-8.5,21.9-12.7,32.8c-15,39.2-29.7,77.8-44,116l12.7,25.1l11.4-1.2c0.4-1.1,0.8-2.3,1.3-3.4
+	C396.7,1905.4,414.9,1856.4,432.9,1808.8z"/>
+<path fill="none" d="M980,1186.8L980,1186.8c0.1,0,0.1,0,0.1-0.1C980.1,1186.8,980.1,1186.8,980,1186.8z"/>
+<path fill="#BE202E" d="M952.6,1323c-10.6,1.9-21.4,3.8-32.5,5.7c-0.1,0-0.1,0.1-0.2,0.1c5.6-0.8,11.2-1.7,16.6-2.6
+	C942,1325.2,947.3,1324.1,952.6,1323z"/>
+<path opacity="0.35" fill="#BE202E" d="M952.6,1323c-10.6,1.9-21.4,3.8-32.5,5.7c-0.1,0-0.1,0.1-0.2,0.1c5.6-0.8,11.2-1.7,16.6-2.6
+	C942,1325.2,947.3,1324.1,952.6,1323z"/>
+<path fill="#BE202E" d="M980.3,1186.7C980.2,1186.7,980.2,1186.7,980.3,1186.7c-0.1,0.1-0.2,0.1-0.2,0.1c1.8-0.2,3.5-0.5,5.2-0.8
+	c7-1,13.9-2.2,20.7-3.7C997.5,1183.8,989,1185.2,980.3,1186.7L980.3,1186.7L980.3,1186.7z"/>
+<path opacity="0.35" fill="#BE202E" d="M980.3,1186.7C980.2,1186.7,980.2,1186.7,980.3,1186.7c-0.1,0.1-0.2,0.1-0.2,0.1
+	c1.8-0.2,3.5-0.5,5.2-0.8c7-1,13.9-2.2,20.7-3.7C997.5,1183.8,989,1185.2,980.3,1186.7L980.3,1186.7L980.3,1186.7z"/>
+<linearGradient id="SVGID_2_" gradientUnits="userSpaceOnUse" x1="-7537.7339" y1="-2391.4075" x2="-4625.4141" y2="-2391.4075" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0.3233" style="stop-color:#9E2064"/>
+	<stop  offset="0.6302" style="stop-color:#C92037"/>
+	<stop  offset="0.7514" style="stop-color:#CD2335"/>
+	<stop  offset="1" style="stop-color:#E97826"/>
+</linearGradient>
+<path fill="url(#SVGID_2_)" d="M858.6,784.7c25.1-46.9,50.5-92.8,76.2-137.4c26.7-46.4,53.7-91.3,80.9-134.7
+	c1.6-2.6,3.2-5.2,4.8-7.7c27-42.7,54.2-83.7,81.6-122.9L1019.5,226c-6.2,7.6-12.5,15.3-18.8,23.2c-23.8,29.7-48.6,61.6-73.9,95.5
+	c-28.6,38.2-58,78.9-87.8,121.7c-27.6,39.5-55.5,80.9-83.5,123.7c-23.8,36.5-47.7,74-71.4,112.5c-0.9,1.4-1.8,2.9-2.6,4.3
+	l107.5,212.3C811.8,873.6,835.1,828.7,858.6,784.7z"/>
+<linearGradient id="SVGID_3_" gradientUnits="userSpaceOnUse" x1="-7186.1777" y1="-2099.3059" x2="-5450.7183" y2="-2099.3059" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0" style="stop-color:#282662"/>
+	<stop  offset="9.548390e-02" style="stop-color:#662E8D"/>
+	<stop  offset="0.7882" style="stop-color:#9F2064"/>
+	<stop  offset="0.9487" style="stop-color:#CD2032"/>
+</linearGradient>
+<path fill="url(#SVGID_3_)" d="M369,1981c-14.2,39.1-28.5,78.9-42.9,119.6c-0.2,0.6-0.4,1.2-0.6,1.8c-2,5.7-4.1,11.5-6.1,17.2
+	c-9.7,27.4-18,52.1-37.3,108.2c31.7,14.5,57.1,52.5,81.1,95.6c-2.6-44.7-21-86.6-56.2-119.1c156.1,7,290.6-32.4,360.1-146.6
+	c6.2-10.2,11.9-20.9,17-32.2c-31.6,40.1-70.8,57.1-144.5,53c-0.2,0.1-0.3,0.1-0.5,0.2c0.2-0.1,0.3-0.1,0.5-0.2
+	c108.6-48.6,163.1-95.3,211.2-172.6c11.4-18.3,22.5-38.4,33.8-60.6c-94.9,97.5-205,125.3-320.9,104.2l-86.9,9.5
+	C374.4,1966.3,371.7,1973.6,369,1981z"/>
+<linearGradient id="SVGID_4_" gradientUnits="userSpaceOnUse" x1="-7374.1626" y1="-2418.5454" x2="-4461.8428" y2="-2418.5454" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0.3233" style="stop-color:#9E2064"/>
+	<stop  offset="0.6302" style="stop-color:#C92037"/>
+	<stop  offset="0.7514" style="stop-color:#CD2335"/>
+	<stop  offset="1" style="stop-color:#E97826"/>
+</linearGradient>
+<path fill="url(#SVGID_4_)" d="M409.6,1786.3c18.8-48.5,38.1-97.7,58-147.2c19-47.4,38.5-95.2,58.5-143.2
+	c20-48,40.4-96.1,61.2-144.2c21.2-49,42.9-97.8,64.9-146.5c21.8-48.1,44-95.9,66.6-143.3c8.1-17.1,16.3-34.1,24.6-51
+	c14.2-29.3,28.6-58.4,43.1-87.2c0.8-1.6,1.6-3.1,2.4-4.6L681.4,706.8c-1.8,2.9-3.5,5.8-5.3,8.6c-25.1,40.9-50,82.7-74.4,125.4
+	c-24.7,43.1-49,87.1-72.7,131.7c-20,37.6-39.6,75.6-58.6,113.9c-3.8,7.8-7.6,15.5-11.3,23.2c-23.4,48.2-44.6,94.8-63.7,139.5
+	c-21.7,50.7-40.7,99.2-57.5,145.1c-11,30.2-21,59.4-30.1,87.4c-7.5,24-14.7,47.9-21.5,71.8c-16,56.3-29.9,112.4-41.2,168.3
+	L353,1935.1c14.3-38.1,28.9-76.8,44-116C401.1,1808.2,405.4,1797.3,409.6,1786.3z"/>
+<linearGradient id="SVGID_5_" gradientUnits="userSpaceOnUse" x1="-7161.7642" y1="-2379.1431" x2="-5631.2524" y2="-2379.1431" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0" style="stop-color:#282662"/>
+	<stop  offset="9.548390e-02" style="stop-color:#662E8D"/>
+	<stop  offset="0.7882" style="stop-color:#9F2064"/>
+	<stop  offset="0.9487" style="stop-color:#CD2032"/>
+</linearGradient>
+<path fill="url(#SVGID_5_)" d="M243.5,1729.4c-13.6,68.2-23.2,136.2-28,203.8c-0.2,2.4-0.4,4.7-0.5,7.1
+	c-33.7-54-124-106.8-123.8-106.2c64.6,93.7,113.7,186.7,120.9,278c-34.6,7.1-82-3.2-136.8-23.3c57.1,52.5,100,67,116.7,70.9
+	c-52.5,3.3-107.1,39.3-162.1,80.8c80.5-32.8,145.5-45.8,192.1-35.3C148.1,2414.2,74.1,2645,0,2890c22.7-6.7,36.2-21.9,43.9-42.6
+	c13.2-44.4,100.8-335.6,238-718.2c3.9-10.9,7.8-21.8,11.8-32.9c1.1-3,2.2-6.1,3.3-9.2c14.5-40.1,29.5-81.1,45.1-122.9
+	c3.5-9.5,7.1-19,10.7-28.6c0.1-0.2,0.1-0.4,0.2-0.6l-107.9-213.2C244.6,1724.4,244,1726.9,243.5,1729.4z"/>
+<linearGradient id="SVGID_6_" gradientUnits="userSpaceOnUse" x1="-7374.1626" y1="-2117.1309" x2="-4461.8428" y2="-2117.1309" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0.3233" style="stop-color:#9E2064"/>
+	<stop  offset="0.6302" style="stop-color:#C92037"/>
+	<stop  offset="0.7514" style="stop-color:#CD2335"/>
+	<stop  offset="1" style="stop-color:#E97826"/>
+</linearGradient>
+<path fill="url(#SVGID_6_)" d="M805.6,937c-3.1,6.3-6.2,12.7-9.3,19.1c-9.3,19.2-18.8,38.8-28.3,58.8
+	c-10.3,21.7-20.7,43.9-31.3,66.5c-5.3,11.4-10.6,22.9-16,34.6c-16.1,35-32.4,71.1-48.9,108.3c-20.3,45.8-41,93.4-62,142.8
+	c-20,47.1-40.3,95.9-60.9,146.3c-19.5,47.9-39.3,97.3-59.3,148.3c-17.8,45.4-35.9,92.1-54,140.1c-0.9,2.3-1.8,4.7-2.6,7
+	c-18,47.6-36.2,96.6-54.6,146.8c-0.4,1.1-0.8,2.3-1.3,3.4l86.9-9.5c-1.7-0.3-3.5-0.5-5.2-0.9c103.9-13,242.1-90.6,331.4-186.5
+	c41.1-44.2,78.5-96.3,113-157.3c25.7-45.4,49.8-95.8,72.8-151.5c20.1-48.7,39.4-101.4,58-158.6c-23.9,12.6-51.2,21.8-81.4,28.2
+	c-5.3,1.1-10.7,2.2-16.1,3.1c-5.5,1-11,1.8-16.6,2.6l0,0l0,0c0.1,0,0.1-0.1,0.2-0.1c96.9-37.3,158-109.2,202.4-197.4
+	c-25.5,17.4-66.9,40.1-116.6,51.1c-6.7,1.5-13.7,2.7-20.7,3.7c-1.7,0.3-3.5,0.6-5.2,0.8l0,0l0,0c0.1,0,0.1,0,0.1-0.1
+	c0,0,0.1,0,0.1,0l0,0c33.6-14.1,62-29.8,86.6-48.4c5.3-4,10.4-8.1,15.3-12.3c7.5-6.5,14.7-13.3,21.5-20.5c4.4-4.6,8.6-9.3,12.7-14.2
+	c9.6-11.5,18.7-23.9,27.1-37.3c2.6-4.1,5.1-8.3,7.6-12.6c3.2-6.2,6.3-12.3,9.3-18.3c13.5-27.2,24.4-51.5,33-72.8
+	c4.3-10.6,8.1-20.5,11.3-29.7c1.3-3.7,2.5-7.2,3.7-10.6c3.4-10.2,6.2-19.3,8.4-27.3c3.3-12,5.3-21.5,6.4-28.4l0,0l0,0
+	c-3.3,2.6-7.1,5.2-11.3,7.7c-29.3,17.5-79.5,33.4-119.9,40.8l79.8-8.8l-79.8,8.8c-0.6,0.1-1.2,0.2-1.8,0.3c-4,0.7-8.1,1.3-12.2,2
+	c0.7-0.3,1.4-0.7,2.1-1l-273,29.9C806.6,935,806.1,936,805.6,937z"/>
+<linearGradient id="SVGID_7_" gradientUnits="userSpaceOnUse" x1="-7554.8232" y1="-2132.0981" x2="-4642.5034" y2="-2132.0981" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0.3233" style="stop-color:#9E2064"/>
+	<stop  offset="0.6302" style="stop-color:#C92037"/>
+	<stop  offset="0.7514" style="stop-color:#CD2335"/>
+	<stop  offset="1" style="stop-color:#E97826"/>
+</linearGradient>
+<path fill="url(#SVGID_7_)" d="M1112.9,385.1c-24.3,37.3-50.8,79.6-79.4,127.5c-1.5,2.5-3,5.1-4.5,7.6
+	c-24.6,41.5-50.8,87.1-78.3,137c-23.8,43.1-48.5,89.3-74.3,139c-22.4,43.3-45.6,89.2-69.4,137.8l273-29.9
+	c79.5-36.6,115.1-69.7,149.6-117.6c9.2-13.2,18.4-27,27.5-41.3c28-43.8,55.6-92,80.1-139.9c23.7-46.3,44.7-92.2,60.7-133.5
+	c10.2-26.3,18.4-50.8,24.1-72.3c5-19,8.9-36.9,11.9-54.1C1327.9,363.5,1197.6,380.9,1112.9,385.1z"/>
+<path fill="#BE202E" d="M936.5,1326.1c-5.5,1-11,1.8-16.6,2.6l0,0C925.5,1328,931,1327.1,936.5,1326.1z"/>
+<path opacity="0.35" fill="#BE202E" d="M936.5,1326.1c-5.5,1-11,1.8-16.6,2.6l0,0C925.5,1328,931,1327.1,936.5,1326.1z"/>
+<linearGradient id="SVGID_8_" gradientUnits="userSpaceOnUse" x1="-7374.1626" y1="-2027.484" x2="-4461.8433" y2="-2027.484" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0.3233" style="stop-color:#9E2064"/>
+	<stop  offset="0.6302" style="stop-color:#C92037"/>
+	<stop  offset="0.7514" style="stop-color:#CD2335"/>
+	<stop  offset="1" style="stop-color:#E97826"/>
+</linearGradient>
+<path fill="url(#SVGID_8_)" d="M936.5,1326.1c-5.5,1-11,1.8-16.6,2.6l0,0C925.5,1328,931,1327.1,936.5,1326.1z"/>
+<path fill="#BE202E" d="M980,1186.8c1.8-0.2,3.5-0.5,5.2-0.8C983.5,1186.3,981.8,1186.6,980,1186.8L980,1186.8z"/>
+<path opacity="0.35" fill="#BE202E" d="M980,1186.8c1.8-0.2,3.5-0.5,5.2-0.8C983.5,1186.3,981.8,1186.6,980,1186.8L980,1186.8z"/>
+<linearGradient id="SVGID_9_" gradientUnits="userSpaceOnUse" x1="-7374.1626" y1="-2037.7417" x2="-4461.8433" y2="-2037.7417" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0.3233" style="stop-color:#9E2064"/>
+	<stop  offset="0.6302" style="stop-color:#C92037"/>
+	<stop  offset="0.7514" style="stop-color:#CD2335"/>
+	<stop  offset="1" style="stop-color:#E97826"/>
+</linearGradient>
+<path fill="url(#SVGID_9_)" d="M980,1186.8c1.8-0.2,3.5-0.5,5.2-0.8C983.5,1186.3,981.8,1186.6,980,1186.8L980,1186.8z"/>
+<path fill="#BE202E" d="M980.2,1186.7C980.2,1186.7,980.2,1186.7,980.2,1186.7L980.2,1186.7L980.2,1186.7L980.2,1186.7
+	C980.2,1186.7,980.2,1186.7,980.2,1186.7z"/>
+<path opacity="0.35" fill="#BE202E" d="M980.2,1186.7C980.2,1186.7,980.2,1186.7,980.2,1186.7L980.2,1186.7L980.2,1186.7
+	L980.2,1186.7C980.2,1186.7,980.2,1186.7,980.2,1186.7z"/>
+<linearGradient id="SVGID_10_" gradientUnits="userSpaceOnUse" x1="-5738.0635" y1="-2039.799" x2="-5094.3457" y2="-2039.799" gradientTransform="matrix(0.4226 -0.9063 0.9063 0.4226 5117.8774 -2859.9343)">
+	<stop  offset="0.3233" style="stop-color:#9E2064"/>
+	<stop  offset="0.6302" style="stop-color:#C92037"/>
+	<stop  offset="0.7514" style="stop-color:#CD2335"/>
+	<stop  offset="1" style="stop-color:#E97826"/>
+</linearGradient>
+<path fill="url(#SVGID_10_)" d="M980.2,1186.7C980.2,1186.7,980.2,1186.7,980.2,1186.7L980.2,1186.7L980.2,1186.7L980.2,1186.7
+	C980.2,1186.7,980.2,1186.7,980.2,1186.7z"/>
+</svg>
diff --git a/site/img/copy.png b/site/img/copy.png
new file mode 100644
index 0000000000000000000000000000000000000000..a1e04999eb3fd3bd7c350a0850659740d4a8cd03
GIT binary patch
literal 23204
zcmeFZWmuG5*9HvSV1Nq93@9N8(gGq~LrB-qLxXf9-6f(j2&f214&5*`(jg%rAYIa(
zGebAug>gUc`yR*l@B98e9)~b<x!1n-UVE>#&b`jH4OLN=xs6YbkA;PGTTWI|4GRk=
z9t#V*_{KHh$<4vGr@$|47d06PtfF4ZwX0_@WOZDyun0)5{$gXrC*A`Z3|nhxyJ{;b
z3Yt3Fvp+X;G%;uQw0{A##=;Wz6a+rno4Y=z@wB&da1rzrq5aiD5cqucn1hz)R})uT
z5n62}6`E&`&gL{c?A+{6Xd(DCG&I7_W)^~KlG1;~fo~$TR<5ou1UWc7JUrMvxY!+?
zEjc&^1OzyqJmq-$lnrRX=Hlhx`rMPv!G-R3Cx82qG<Pv|wtnGi?dU*r)$el?M>kgy
zTH31@{rl(lI9;tR{_`dWmp{`2CdhI1FC3igPdNVV8xR$~dMfzL(f)<Axr+<XKZHm4
zSIhtP>_6}M-Co7Y(bW;qg0r=$oP(>mGa&5x{AzTNr+=jX-&_3OzM#(5=D@6fNpt>@
z{-4kOv=`>Mn)v@v#BVMCdJ3o+f-lVR?~+0Aix-|wVPT13$w`W9cw%o&T~E{)9`C?d
z64ZO(Vq23+f>c2&rvIp#B0se5v=?!fNGcVSjNbD>jrnlA#ltHhfXkNji+elP?p`2j
zzdt3I^yo^8o5Aezmw9X_`fMNG5we8VT0Fuf62rp&?~niL;J^Fezh>}XSMdM4C%lm^
zrhJb;DvszILs0z#J(A9>-P^m>cSI|-a{L*0#&7gx)7v4!(ce_=;&zdJW4kmrl=_cv
zO7X_g;LcxZul<j)vJj=D>0lFa&;?@Q+@QJop#cqqTkQYzB>VH7CQu9y5M#xfH~)8b
zLF+eF`U{dNP!j)ko+Ac`;fd9~``ZtViU>AK{IJV{!S8;rItSkX#IWAR692nX@)*UW
z%Df%w2>!L66dXVdjD>mnpZD+M=2&U<g?l{w&md_4F&r90X{@05n~Tezx26O-CV296
zAaz1qxM#aRN}M`fFnt|aoRhp~)r-?-udr~|9=%5>hh!Gq00tKHwW_N`-Qydp|8ANl
zaYA+L=6-0%!FC@p>pM^*=#cUF#O0vHU}rN+6_3|c4u=qQ_%Z5vL6d;c%7p*aXq6I3
znzkm{LX_j}7%K@auZ8<k(Uy_-QhZ0_YqBviF8K5UB61SKQEVNia4AqpDJ+8(w0Acp
zEzWQ>Y=BmUE+}k<qqr^4%-!yzso&(vSoOKHeOVT9?Mi+N`BHa6*)!weiMQvc=V^WL
zDn)NrH-wV&W>!$OxcP1b2g?QJIohYSAz4RTKll3`f!@18n_!^p`q5;rK8c{4yR?<Y
znd5{mbO#4e;#%)MeIbp)q{s+Py%JsmasDNio1WbscHK*SW#6YigGw$EmdC(8i+F4!
z92eUD-C0#8(t1zxU0=8ZdMyV5-NwX*OVANf2lWrqR;r)s+p6~2#n*`YR6Z4QnAG1G
z5oTOfroK>U-9D}PF_M>>SFek#d+Id)2G6~q2I5oIQ^GTGF@GfC<9~M2)s{7Za`bbt
z&({*Dl#QMk9TK5V7$UNGV;5%{T(g{KK5W<fwCqg-TRCwfcxevlbN`xKPs!_wdfSoB
z51b(fTLt?aZ+jwFa>?#x_Vu4Of+mc7RwlYh@L%@z?)xlF=ta>r^9ZNO6!8?8y6^4c
zRo3f@^$n~nl=BVMGzjjSXXD<Wd9Mv@t<ZgX4_pbhA5b<`1(<EYlv-KMiTw+U8DY&k
z_*)}=_=PeXl)Fw|E-_2E;7;M&eH$6<hpd|8zWU(C>;{f<zuKj1rlO2J95-}p$a76R
zCmXzY{Yu8&5TFUiFxutlcQjtN)vOua#!(ZTqYuuqRJ>$1iwm8%W@;BFGQ9A3)tAPQ
zJ`q%3q#|mp2<+Br%j_o?wWs#KDug|qyG$jym(LmR<KD}2`OFkv6cBbI;Qm&(Tckck
zR9XwHjeZjv&^y8Xy(G~M@#aU;4Lk{~pfC6;{qI;A;~o+@1c~s}dc^5|ARpzSD7G%L
zA0OD9_z)saFeLAmgvis%vWm|N6$-AAEP&Rg!7W@*lMrMRv0P!gWGI;JO2d@1olu2?
zcWfSvh+?#}giEIAHUs>~XZxh|$HkXwmV^!RM>5dJvDLMW$OUjOpawi}SDXUfNNNep
zzsFxsdngH2Xr6&4_->%8(LQq9S)>1PjYZF%fY9+i&Tb#+O>Ep!b-+cq8+!B#D&QTc
zdbkCxV})r++s&W7of5A3k(D$2gx4!<kd)22z}&BF!plbK=WNxh<zNsaN)S=24~A8k
zd9P1AmMI$#v!ElhXFk0wj1I`olIp<tlIPkE4+@EbCz{y3c=z55Q*hFopw9KJ7W_H9
z!!O^!jH6ylneU3Kq^+@@i%ZaR<u%jcj58NE!(`oQUyR%^3v0qdfpMdP<b{lenYZ?+
zT<k5~vIDY5<cw;v^6C>RhT68keLEjvvc1&rEpOX`l^(}cnfSG}&K!p6KE2#=4%!-F
z?r?|w6y%?I9l>_1-8v?g1DUDS1mi@TmswaJ{R7ON=<U@?$)n$j#Sk3fCQS)0kKII)
z;u?(h_UHK*xoNSQn@1sgsyi$!46XNL)S(r8_l%Am3*-0=3JopuyCLxxLtG~Xt4_z{
zWmP;MS{D|#l-Lpl@~mQhzCpCmJ<<Nk=x!Hl%V$-naToBMM66;`Tw1{^H_S@JL@J(m
z4+#Dg-x+V7t`X2HYhg{8*5o@()mxhE^mE=CcM6H$d+uIm@t*9chJRx{-#^ic!h-Gg
zl(5al$En@gywZ*6Aysv{Wu(UsDpLkq)fZ~(c)nYTh{Pr4ng|P{@VTv-haH9&s$rM5
zosa*|%bRRT=xqAAeU{O}yQFPMo`!8FHyy&wv<XA{!Np^Xs(0+gQI(@7iF(VqFRczl
z&wU*l_j}?At#|tH{&C(4q1$;G_pL8S;v|<Px?~Mhe`)j&oP}$lyy@_E`bd?3`*z3f
zBK}ZkUrEAsutE2j&_79h1;wo=@ttg7nP*r*PY4QZO>ejmGgIKiu@#S++RBfVs`=I^
z@u+)-u*5e|Dk*)0(z4jnzV#^N>vtlxN$M$e2pAV*LqO!L+-`TPM!Veew<}aRSH3lA
zna&auSLrwgp@>`VJ~k`DVvCRSu@~UN&7+@<ZuHD3Wr<6!@(AxUs-jfZ>CnyUV!v8*
zF`9ZjmKnyF=@q%P{{+2{kAcPMoTa?;H|TvM1sv13bEW|FZ-}Mo1Y&N~t=_r6p*qNp
z81Tnzu=riczcKg!vu=p;CJX70e@1!pJJ82!Y&#RS7cO=!$+d@F_KR0)Q6|3&c6#@P
zKscY0k%c|Nr`Jr|8oNQAMJXChb*#*c_)OZf<L3#+#+~PQ5nlRJt0ds8jG+UUU^Dxy
z3Z@@DyYOBF=Z4u^EO5`Q>i3D)aI7B{%bSbnI8~+jlSd0phOLiwA;+2F(c=-#$8>>W
zOIN|ZH?$tmT>}m^Q!OEQI$T+v5W7)}4)k>}{mZAj*tjh;@2Ou)s9eJ(`X(Rb;x;nF
z`kJNPEK}?xv7FG2->^jU1Qhp?Hc;#yu2ui7ubL^hZkS~^p#--An77r@wM=yg>_*;0
zR(6%|w;ef|@8jHfr5b3o{k)Fl)6JkwVo&Yt^@Vfg?qqf4S}qX-dc#z-t<)9J$$KO=
z>J|NE>^?Bn$3&|Bkm^N}wlH`czM3cQL@pDAx2&-R3BFqPoR7fv(4051#3hmqY>MpK
zWpA^;`7PnGEyL{SqH^1-c=NBd#9}mhcsW)U8__Fu#5ZD_hFO#yLy&{A_-_J{E;UIl
z$lJeqiUIUyQ-Wat{6`s<t!?}L^4H@$KTiF6zDI%l?PURMC022zv9NHyhk;@^xYMv?
zI#4Vljbmv+d=q%$JlIXO!lRV$$|cY>nWfyjl?>>m3$Iu8W;-*Ww5&SraUCoayQo%)
zxqC9iG8_vB14@zNN{)Znm+wRcxZzH=#fUBE0?Yix2T|#n6~Gv>4t9<`TK<6tuq<a*
zGEq_mh2tG;Cz>YkPJ0UJwOl!#x={uP`pF8@h}$BM`V57i&KL>__H9pd^m$iA=p8Is
zxY{B0g^URx7E834`PVpv)?aPfaw!^``fif(tIKHTc|Mwxna8rGd(qEceenIcoI@tl
zvlRf4*_|pmcX>wlJO`z02)~2TN0{&6FJ;y)O(=*;Z=M?w6gc=t+}`|XDzo>kqQnJ%
z!Shk61dRhue+7>WQ%5YQpzuE5gfo{762_sv$Lhghcx6g?S5OjPd+o~x#K2{11y45v
z-EEcm)MTdDWNOK^E=EUd86!C31FB+)_+IJ)9bTI*`o~96Q4^b=9oW{%!18>r&7$wM
zYSS%KB4;Mg)A=UU$}79IKLG%%aVZqpnA7C)#GUnzOe!LK0LtXutKA2yQCqGdJ#3Rr
zLnegJV&6_3x}_Fso)7PSvFH;PI-SBfwMFSu{CL<QtsFWQj*w=p+8|tC`eIfk?1Ng7
z@KG%+c!b+UjNYecgl*<~<|m$e<}Tx0X9}|i$&3oKI)8HwT0LyPO&8|B;xT;KVmV0b
zUgp0-I4=D)lee$LH~cfX0FOykfT2}BKD>d^=EOif-qveIDRZl@a*^PPC;RcToXC7W
z37o~}5D0fG2i>>ZD_bKRItLk3)T1bCb-`_Y*)p4(9{8JaP9fr%&M&IcL^js@K5(5b
zk29WRVydd}Hw`tP5dNg?xTEE{BRs*^dz0n8e^gEFu=mM!9|eq7Z<w#v$v8P2L6Q0~
zZU`zq@!&fuXuB$VZ=Z61?IFFaSil9IhlQYOB!oBSl~^@SKPo(6^OGesM|^%glE6M;
zM8G}DztO|N!o7%`YWqCz$-OMM#gUE1tI3o-Ph~lAn0OK3uN)3oVXhN&LN)s_uMlP!
zN62JD|MXcl<z$s!k&+Q*357Ixj2|&m%RYrP!cW)i-{V$d3m2-29P^(z-0H-uR4Yu&
zZb%iC8_l#<pyka`Xg#r#KHgmzg)s#rBO47H$G(6EXGZFh@fu$NfP9b4ft%w2agLbr
z7sE?Qr?~l-_y<c%>o@gttLn3la=aZ^48~bz>vc^n$o#JRqHS9vRBf00IY47Smn_ux
zM9Qif7%OcPY$GELF3*W0QSmt*=8Xwyr@NoYv30rC7m|n4d;3LCgfy{*caCv{F6(i+
z`>8)qhp4hHuOsy@eQ1~_q2#0E$_Dc#otNG}E0rR)GMDIWR+hBWC!_*$r4Ns_*^)Ce
zw<ecyxu(eS{MKXoGhvbnKGc&$+KjWdw#g~zA_FyFyh@n+Gtf#qpPqB~wV<(?qp}#%
z^To!MbnBD5gNKrP*=Ze~zN4SM_j7kvoT0LAAFiw{;{QmC(?@-x?$HCG)HjxvYFF@V
zKOB30A5i0}5F;(fC&^Q`&PIK<6>^L>omoSU-alDVErsjo-r1;9G9)U2^7ya3qjI)q
zT;kO2le?iIZX9uNUN!F@()V`xpGU9wA6=>sj(!)0a*EzL{(y(0Nt)B?p`_H9kf*Ob
zt-=OG&S4V;1fScFmde2)zXzv1KQoH_BP6em6Qt(vd^ghN#6@#ouCBdjbFt$qZ1Z|q
zGaFk((L4;GuJ(D2qMB-L4|#t#iPrex@Ug~gi`N!}>lF~M#K5=@h8jAUeeIQ~%jEK^
zn{>KNQs+zJEv4=NN8}WS86KW<hdjejFPJ8ti6AupMC*XCHqe9dXM<z%R4Fw1IBjbx
zA)(y>(Dv7vc*+uU+Yv;wDtYl1R}00En&<?t0J(Y`Ff=J}kN3o~oZBGHu}SrGjn44i
z>Su{lX$4lVs(R`-SHn0PTp<ZAQYkNNIUTET4N|H&z8jM8kw`XPOt{f@et+syc~sxP
zKmBQO&1BQp<xd|wp$;HWu3ic&#ifB^Vjokr*17?*ggT+K%4V+-MeuilZd&)@4!pJv
z`vzXlG*YE9%R7B#OL>Xjho=jse$vl}FO{nu%u=VG%62nv#plyU<13Vxx5vC$x$%h>
z9b7q;kFMZ}WLa|K)ij({-Z~L&%jjwh5h2d0{xzQCDUDq&x;Le`dDD+IwUq5RV8+Of
zkpf%7#~xL$8deVl4;?nghDt)mpIu*`^!f^CiJ2a_H#u;>)=}N98mS^RI4Y^I!hJTT
z>$^R8h_kM4wE(hYnpU0}l(p_89O*}@@2M3hs>*Xn?GrwDjrx@L3G!ivC-=f(UMTYn
zio+JabZRz~PoecxL8uylWDSm+b}a+b-MqLTX=Yt?>uX0DOYAbWQ)x5}n?^r=G2oOG
zr!U)a*t~BU<mo?(o&v3ek*sXyO&{K0!Fg|BgIDKsKB*H@>8ja9p|4KR+&?HcD*|@U
zm@jo@ah2`zj_$$~nMlfZJ>x8u3`E60uKtmk$iNn#nPN7`fbjC4F8;W{Q?_;{Fj@}g
z*P}n2REMM<vD^32R^y@4zfu*VV~p?O$bn~T8FGK73f^OjPpzQ@UBkv@VFv8cFyjlf
z`ST>af2TZ*kdGhM93KbvDY=ewqg4(#MkUg80u4X1&w?2d@$Gj-z=M|twcpim6?(!B
zu97SBi~v?Sd)x_Nh>xtZ|Aaxcd^0?CI2FVvv{Qn44GPAb7GlR<0=b$;T7jfJttTIa
zu0A%6j_hr_7?FI`sA)2o@GYCq9x$?C1+c?=e!$E^%jmBFO+-_)prTP5q*Fj&mug8R
zx-wz@bvX-9h!j@PyDN;!8T*k$oW>R}&DM6BF34LazZetBH<C38Vq|vB$GbWoy!Qoq
z>*QN`n@e`2Eyb^yTH6@)RMPl!?}B<;=6z=fhCe(%x>FAho&Rv(>y<=GxV%%$e3{Cp
z*Al-nRG(PTGWLG*C9;^j7F2?g!e^oZ;p5n14xb^eo<6z<ZHJ+v5P>`$;iq(wpYXYd
z`Y+Qw^-9DrIVSI40(l?+2ssCh8O;)Cj;wOLFmieSDH#Vs;P;6(0hoLr>RuZszM41w
zkXp%cFuw?rDpMNJraaiHu&|$<+41I%&iBXH)mb!i_`LC4GOlEVp)07WxJQ|>fdTH<
znYOjQYd&d}xp75|(2(Hy#uUxS_{`U!R~o`UJ;2e+|J=x0$0{<Z#+X#h@qL<pQdu~>
zuj);gfA~BsWJYUWLP`ej_ql-pw@!0VkI#}70z0D8DlNfDA5_j?H2Mj-!#6oVAvhOc
zJ}H+yckGb&^imyDXc9l~EyeLiQ~}R7<aH2S1_VjorD`j|r(Ti6M+8E*Pjon$e5Dk$
zL+`+P7eJ7PNDw3_i)A1bP6(P_-{d1d3Z_lyAlcZH`<1Yxz#kfZO$SCYk+LRhQKrli
z7~Xg&Z`irlWx-p(G;3?otFi}s-$JJ%BPF1SNo#>sdn!8>r&az|aiCZTPJ{0|sPO$%
zH1-+feE=C=AxPqe_|EpOt%OR1B-xxY)$Ft+^{X>h`q_GFAC?CiN6wNcW@VKl>VN)_
zN_lLv{~Y<zTzbHbz=qA3Q09!Gn@66yh<l$pN1rn$%_xR=^n6rf{Gs>ez^Ci9;z-^p
z!i40v3V^f-!BLYx^{l8Srf}Xj1d2sNZ!`;{a?YS7(Tm%4Gc*Ouu*!Cltrk2pEVze6
zKPQJV@xtE!K5NOs17QxnO^qJ5y)fS#emU`tD58&_&dA@XB_?Jp(C<FExc+X2^Ytg;
zAT7b6`}miN$4NO3u+0LR-|5NsYS@U!uJ7k{LZGzXa&ufEe$K?qB{(9{^)P&3>D31;
z1c?##n}s$Rt@zw32OUh)Dvvd8X>R7DNzJc}1ITfvl2~Qjr9$4zLf`y{koiFEND@cQ
z1h+`D63W1^@oDiSCc;}>J|dCo`o4SyjG<m_Em{F-=51cq@K^a0H_A2T`m}s=(V@7I
zRAV1J<_4UG@BJL`AWRQ0ef9B<u0Pp*-J$tD)YR-W)i&Ggw8-|<A{+RR<_8RoU6O*J
zOh<BaB)5Fi#L4iOAvmW@6o~Q<y<frcIcFEwZ!X}35M`H#>{P4GA;MQ1g<d~uy%}rw
zOM@-UMD{vyh4X%S{OWTg#IigjXuJbtkq#-d-{F*C&GlRhoQA#k9O8=##62oMGtl(#
zt78mcnP$;}9+7vngp1H6CXOtxnS5X3<gVjDH0C^{BJ&z~mQZL>NFC3h9*>+R8Cq%(
zs&YiT4&j|3)0|On9<zG*XZ_HjHv-h^inKF4B}x!4OS~m=Bm^~2{nOBykpuL{JO@s3
z5J;4USJ1@7zHyri{w3dTT8krrj*R%9WOmR~TwSZvHTVa7$JE+q{ZR2d?IzaG5X!j>
zxX10D2V{AVrsI`>XGkQ{$Oq9fiaZx{N=wWAqkAkIP0*O)sZqv}_#6qjugTXKe6?|$
z$<<5&_=q{`Zahpd|B>PIbx{1>kTRNjjRFeLAZUi=BG(G}Xa59>CF1%@1VimU7okJ%
zK;}}DUY3_MxlJs7C2UEaR>iI#kOX{qI`6r{+*f>mb^XHC)PE&$@O)neLtW7~38%?<
z^)UhPc6_xDfylz+d=pd}^|Ry4bj>`#AEe8%+(kd9gY+Jlwh6Zxz_IcFY`hy8Voi(?
z$~;f;5^YS~JW&Yr)3*a|ZL9_d>nS3Mqt7jI@FWTCN;9&~OD2AKOdf!UnA@nq8ciW$
zzm-Sx7^i`|*9eSa^$m?Z1=q6r;z(6yS-Jw=4ZVocjN88Ty!xYNhA@@Y`+3z682gOY
znN+{ZznXtfa5UDl3z}i_Rm_HFjA?za&n;Bux3jf9fW`Sf#Mn`;7pM202EcXTeIQHb
zf#(A;aJj#$Hh^0>AdQg4<hL+t9pQ%A#V`5B_l|a)g`}ES#dL8hJAI+mQ{zioU4qNO
zL*d<^hVWm(0N8i;8D>dXeP6cehrk-@?k<x&c`D$A=Tbi3N|d<RsQG&I*_Fde4Znl-
zXj_4g!}UNyOn){k&3inYV?i0j3rt1}DsLyg#Zr7!-OL34>qNW(-Ue5$-*~d3Vz{eX
z2a9hf1GUHTTa%O)nB2e16@$<H!|PH96Vt+)6v=#`dh{w>QGRs8>~$9U)$6sfl`Qm8
zD1SL29={l-yhT(9$rPdS$|<W6{wo^+m|Bnqe)&N2Wfr=~0<}}8*$?}0Gq*u`9@b{a
z%pYg!`kfQ5m{WJE^2z|@!RmXX{YD!z+~VJ*rcq(WMy=Ota02^OfF0%v%W#>1g&+!K
zEy5CDO%}|)?QVkEg_=jUphlJkb^V-gL(j1UdH!ycM+!}h&PT8AAfFuL5+YXTNFW28
zBKZ<Vajh?Bkfu|~tqX0tDhVuQyWnYZA<en3gx<`5brvZ06VKP=4HWGu35ZQ=Qf8w=
zY1TgSs*3yB_Y{6qF-Dmr7s)arvRF`Za6<4d_>je2{nc|XCUEfI6&t6So$8ZeGSqsk
zCY;W%qN+fA;97&fYDz$&<K_GYgWo7+H5O&0U^KZ)A2QQ5q~}MdmhrVhgY$P}0{}=T
z4&)BTtW5^)1@{5fvfS1Apnfo{JpEKi7c*+JqJp8|7-9Fn+%9I|8&kl%HwkL7Y@<fX
zW{<$YvHy<Y(GPJ(pCFS06ZpKY6jW;LV<b;vIy&wEqdfH~74~I{*3Q}oylT;=*cJ;f
z1N-MI|J5DMdo3Kp<~rp$urfkro<yNV<G4nA$xELY#jSte8;|{QZWz9EXiM!*fSP{l
znEo&8$fGA3+0L^`y5$Ju{L9JEGjg?84)SpdN969Sd9M(UNd}fB7E=_yLn%A?Mn753
zVaXeRk4FLi`SaLqW+9G|(hX`<mjZlzRqlp{$)bI%k00w(g-U9jF4SnJZ<Wk(?OkWS
z$ke9VjTbda(*}{S?g^`Q>e)o9eer^|4Z<105VOFmnGeYV;4}QllM8^0Dow1;r_6OY
z<&pFMw2{C^L9F1+OpA%tsapFHDHjo5`yVOd`QH2M6<Q(1A^r|o_{LEcwxhhdWfn@o
z(X6nMVN<G1o(V^#bm?&XcwT$moby8WtqFk&o8hiu+Y4<4(b0he9FgUsFv=xsZ|ijJ
z3Y%!=nfV_WyZ!ZHV*9Ywk{8E2OI{OM+4<)IA^0=fNHTW)FHbx$^KI4LF>KwiAvLMt
z?qZ1!{UYrw{H^)K#Ke#Zh!H9-zixT8-_rl|Rse!ol$D`wzUKK3e770DY76;7y^1P}
zN#!b%1bqR>%69~GaX^OcC0<<!1{XkvC<|&>qyM=;-rg`Qv!tHIjZLdSyI&)^xS}V4
z&uBPHDWT;KmEdBf-FT^1zE<IS&iv=e3^@{3R&qWEySy*VlP?&M#d$l{VWey<T+4lk
zM(Glp^`Tr_%kQ6`yR7sUW*!cyOL>}4)wq%6Jr&jiPev=mJho25x3NrG^T2mx_Scv*
zRk_;5fDN^rCh%-$%fG$?wAiSI?Tpp!)R;EIJpKa>1p~5*^@^ds{K*R2sv4kUFQCW#
zQ9%1=%1MGcla=;i&kFq?incA1L8GAwB0la!d9NluF{{%?F+vSfwF_6O%Pe~GvgIqQ
zmX${ra5W`6zWwa5QdU#c@MLBFKY7WV)Ibk=$9(o#`{9W(D4X3Rn?a4MdIf)x8hxp8
zf_-J~WV%eG;8d+g6<gKM0YUf8)u}Na;YW@f3Qc+njEbL)qZpJ*LrEB=8&0=t%|*}9
z&zc<@&zGZrMCZLy*2u@K)JPjK5gO4OCz@#L9G6i24=M!<C-g4b0hOeoO8KMs`PX`$
z<QEhO?y~(6J>-OScGCLc_EfDF(y(4JS|MHPscLM>$#HL@z+n5k`=+KH5fvN_M<&NG
zBfknQw$yz$HcO^a)%D&0qezI;H&#T*FHflgr0UB)zhnfI%!rK$EURi4d5|(vJf%_c
zQxe<>_Sr0vImiCz+9IlMw^tnIkr*Mx%S?pkOt-<5tENkjza^&A@;$?J;JGY+?;Pc!
z&WEogkQ1Clz?TXUijU=+_vhZ<cy)E2`&@L*T%1M(@9gj!meO~DZ_gH(g>ogT*MB@C
zv>!qmiB7st+AAqsWUR!guFDe~Y5(XGz5F@twUWextUs9OvL4J@${%)Ftn=D6RAE3e
zS!fAzxl8`916eN4MQwUxyT*g;k$gHHImVT@bP+j?eUQ^dpQVA=(}(6&flx7WUc1lr
z$IA)bhW^J(zURbG#ZASje|Y{}d9={Y@c}r}5CFMmi)N{15b@jeK#qs5@OgbU3XQ{~
zg-FL2L#i@M4kc}*PQti4=JA0RC6l$N2HR(E?ii$w^iulyLo_pY$&*%m;@pc834bHq
z8yW^FNj2hcPSWONsQvcfjt$4s-LQ{DIW>W#-y(ko0GY5t_F@fF!R*y^&zCK+m9&3q
z0K{T$=J=1JS=@Zs8y7z+Xypo(uk#R02Z`Osq4q_Ko4uiO*G_wQufbv^<?(4%+Zye!
z(t-EGI0^Y51-9NlyQ6clfozOX)imb9;UV)}j<ck#Nr<X?F<lhkbZf0^-AE`=_-cq^
zyf<72?FnK<55`PGc#zT>kT-zJ9@GAOJ2F~umY`ObZJ~4ByaC-p(9IS)lKSo&cUtQ8
zrR~^44x9T0Z(m!0zI{VUw?4%iN9(<I`NiA-jD?oq#y16y`cHd;eNG)zWaDO0N-yD)
zu}1!Jdke6Yz<zj^#e__IB(;$Rsjdt+KL<;^`CX~<m*N?YGYzZ>JhtvIt&8r34B~HM
z*w>6LXQh@t5#@vi)^ww^Hh#tj6GFZ)jB6a%>pJ!ih1if2*wmb7Ou50lhk1OL&vh)5
zG}pZvf$?TZ^@NWfXYPO;_x80!Pgc_fz0b;cx3$l3pujrY1Jp#;R%ZI_jYraAY%W5!
zO5Za$ULDT{6YBc=H=G~$1yrgl3$!JBuQTfv8;gfVQF2?UsNAn#N%r}U3Z5=CR~Que
z671u8+UH*@$yRMv6Ht@%;w99ASDiJ^z@>+0q|6ekPCrpO=}?<;H{ZQtcHcBYpj{bq
zyg2K%7Z7Xcox8s_*ipW0>KawQ)5Y5CH_BsM<Egh9mEcg*`I?a0;mUS5Eq2fQgiyjl
zTjhg_T!jjC%jUH}W2LHva_MmLxEY?#-M3kKj`hRJ!Wc=P`B27}eu5s`c7(#{z+>|0
z>km9r)*dX9PLcXWiZ&ch`&4ce)n2=EQG+ll$Xv-EF&$jTt^{0C0)x-^AXL;|&%Pph
z3NXVGoj9Rfo{^&3?V7ujB>TZNz4o(BL2jQe-9rVh9sk~FfnspnA$tL(!!q{T04{7K
z2n3SUcnHsVd4I7;UR5$0@vVn%Mt|Na@4ZwxjW3bMw3p>#>YE4nhSYox`ajs}yV+|c
zHTrK=p1!ky=Dm<EPw|xmx4lJqTpX{+Xw1FKS>pCI0OuQTE)HZW1S9Gn+fP=8TNQT5
zauWuKdkDgFoXIvxP8+R8s~wu}eQH$CRwkx)yr%@ycaqAG3L_<+5@0O7yf}@$m%jE4
zuf~0g-0NW665wgWg^w4bn!Qdvm7&0H^iqjvyx2#60K7488h%=&>8<wSX<);CKsB7Q
z?*v>P1`tmPK|Hrn{WxpQeSjZzGi!Nuvsq2TMlO8SaEEwGaKPi{&eo6AXJ#E#9`AD0
zsHy=;+QDpSWR3)RVl4}*+5e)^%3t(+PaRV!DcX<5LG~a%r3K(nlKx>v@6xBpkL~7V
z6ezoz)AC-98U={tFtr!;t$`&gu%w6bwCnO=+_kQ2cXdk5<S@JecMGkOo!alyS3(o{
zInz!HWrhTGO=@+?g7Fi!yQwsaeW@!Xubp}>4RnOwx&QRh-JU+yi2aSxNa2g41^OKf
zwFzrH5G})V(hT)OlLXyt&=4fZ8(wJTO|LTgB3+meh)!>7W`ob`)7FmL7?7XymdZS%
zU@w5LU`MD;$!(UlHlfu}g!*QQu#u?c%!34f!1)`Wrbj|_>%7pj&BV9UnJxtSp#W+i
zR#n))8}0E%(yzUf49L=5CG2(z1n%NkKTL_K9~>I4`&kQ*iq??BZ|BIX=;cmjxPcl-
z680*c5vrBstULW#+oYgi<My~@y!N#w=hF(;n6Ygy&)8aOg~FsPpR(%)3k^|`B%0=Z
zt{v$q6hCYG&Wm_$)^640E0~ox6WUbYs%deo!$_YVuOO5Pv$NnON#5(Z6r0xKRTYkP
z2NSlzr_HA#b0QER9-nSi&A<;~^rc7DqL?D|NTD8?;~-W1O9zj}(J#7>w@y)wGpoL*
zhbU^-vRWB#9>;0nRo=*p%}B`Wy~u#G^eN|Vwq^^>p?@yR`XEB?Rj))mOf^4^u40&}
z7Q0_jy?qC8q1HIZazb~n1L)S3_{eX()ZA|v&LeL6uwIBib%WeHWD^*}opzFoIl=&@
z=Zjz{0+gXuad(pS^qK3srPt9FtMy~`CJS<=YQgwvGc5g#+g9!M)(@Scg2O2V23(O2
zgML4iHiWf2t&kddtP>F;2ig=oHX15&`*Q3Q4Af+6ovZ_f;^9jDJ8cg6-(5tFklK!(
z4Hspx57$)<4I#-x$Y7YKU!65-y57EO%AFO(=H#Ws76;*VS$cP$t3fr)ty8?rNdV^x
zEwbRsU*smj@4isPCAyIwh%_HlijG*bq1Wd$@A~vA8na!Cx}&=YU`31dYwX0+r3dX4
zj-N^2AxAJi`|IF3=!Q#-XJeJhNh=*<YP(bKiQmZS{-}|oVf8+#Sb*?qO7Ld&N-~$*
z*7fNAyK$c|!LU_diEx;TUrX<_56WOY*^t-GXmYhbJ-CzK5cBSJY@uLhcimEq-o)#b
zjSlTP&k`w-uI@?JXv59}EgiSiNzn^T<c@n}x2;1#%!E0)%Sj+h9DM>}!5R0dIFS6j
z!opARb7BR*;4Q%=f4&^(SjZ*dn>S@pP#_><ztzrf_@WuFxf>#_&i0yOi<~Mw_x>bB
zebz{qaaunk!uKvl&&uY&^~tvCR5Sg-eyF5P$-6|CUV*krBcH7b;{=u$#MFH{x^SxW
z`=>Q5@x+?jCVbI=W8yfHe1OQwAe}P)nTzO%vrN9PN&wg+wZa-ns7Vw{5rvZc02?J;
zeEoyXWs)$B@$eiRYQ-lzCD}I7w-~67{n|yMV;s>fHJ8@cyq{zZcI34y--1suKIrp%
zmow6{r>CMk@+Ls#-X*pQz^(tPN0hA-6Jl%J{lcQp%1!4fwV9C=ufhMo|7gxwYyUH+
zY1j95)>QI0_b~EmZY=TZ7k3!;tvb_Fb}~X4n$4)(qGI{8`c@C{bvNJnAkD0W7p1|q
zV|+21{qLq2PlVl{zIWV(9{>wS5gOBDIYI5U_<`7s`nn@?p@g=o)Zh%i>*|jb=fX&-
zC+X9X`C4l<&`TrhW75VPM*nGlvh@6-8WwjK{p3@(J)Q0N)upP$y)0*=H}L6-)=P@z
z8^7~suMAUiW6rC^3KLy;C+*7G@`?-g<GR*9`i^H5XUzm$ij+QIs>(9oC)I46RMs5a
z?GvSbT()Qqm)U`D<Rcr_qgR*W>l-Dw)@DW->-RYEQZE_Bu!+S|Sa5DqEgD6?Eat$u
zkr5quxxJdr-nXFfK|tQ1Kp;<YX!KQ_Z=s9zaMJgWS~&OhwT<|vJ6ujOvep@lH2lYk
zpJ%!8$67OqHIT+4QsOX&wh{xoT`pQ~yGwq*$4&Fz0LO5RGR0wrf@j(}t!5Oqru;O_
zZ%ek($jkKu+Oz!K2bp=i@(hjE+7-Xs19^{(UQCmDj4q}Q>ke3#=r+XK`mR0k<Mp-A
zRw~^<;97=Z;iv+fY5HAZpw^)dH=Z@Up8H~3j4i(`fmCQlvMNA7wae14XSw(A{#4Db
za$STJ1OV;(f&oW!Z;UO@uR!}MWE*eY0-0$Gk&eom)?huHWEmlY?;oNS#RUW&0ctP=
zK>Iopn_i4&5YHFW*<dqAo&x_itxMDp%aT&TQ({?RK{-13ZdG?$X=AQ?_k9cf>|x~S
z7Kc(*Tlp1ssb)|p9Hxp~w@uu3jduOS5(AlbE;mVRVblj;Hdc@cz>u_ogvGIf9^sew
zm(;aX7h32a4t}ZM(!kzIS;=<`YpbZ?@GvNJ(#iHY+-|OUPbK8et*H^EUAHOti9s%I
z2G43BL+)brP{}BZv>RtCH79xGL_b%4NwSCZQgQT~9&-naze+(DJAE$@Z1d=VV~(Ir
z-F0l-yANj-e!8l;9{zBauP<-y<;|OQ9l0yTw!dprC{V3iHnm&9tt_G`yDR9n4s!kK
z1FL&;0F|sB5QG}Non~A<GC68MH*mOQ;4gI{{7<Z-lULSxkcvh9W=U(_i79x49aaaY
zo=JO;LO6alJ}Jaz0$_HA2B1v&V*!4k`dBR`>|l5D?t{!6<h^uwzpv(FJ?j>EAI*2V
zHjk}HdfYjFyvoqDpD1VE1aL*s16yked8&#}%Qmm;qQQBs4z82ZRCL^(9g;Q}Z*@Mm
zxYYYowxIl>LW2}He*5j8)yLjYk+_0SgE%6+BV(O-d9u7e3i<d(JK5CgGQJpL(tl-f
zTMdBm{FLh?0(i7XKh%yXbMK|0wR$bV&%lre#RWqe97l?DsQ5#)aD2{2RRv7m%~qx{
z*ON<AIU#3YSQh-{n(jhLGD}_s4`fJhe<Wu>Ck1YudeDd2_LeF#>{gXxlZ`hpN<r-I
z1ve%yRD})g-kqELl)ZLuIl9%#O797QQDlfsZqM)nd7JLc_T-r=idh4X6J~3?oJY@R
zq^j_Ypzao{uxmjkxpT30VI)C?U&AyPEP7MGI}z!!JJKNlBuK|M9>6x7zAiSNdT4&x
znuDW2P$u8CRJ>oOS|T!FdpSLSL;_S5MFm!AJ;*@gzn|GXxg7!voYpMRe8h5$l0cLX
znHe0pa3~bI^(#hRs{btb+~A;xj%V?!@O4(_*1+Ct&Zy4AEm93}4}SoYc{VxP+Dhtu
zLDp6cU*eqfL(t<pI=ySh7lCT*?2bX_mQR^{o%?_^5t#Ka&+zM<v7_OPoE6SOlUDPA
z4cDwwZYX%G;9=kvb%W|%=y*>{-QY9wwfYX$*UWY1yARt8@g0bnasQ^$G9tx{!24Pu
z9vE+rqZv>k=()mlo3K*<6KuTQ?el8*gPY%Kvvf{~UX_83SS(oTcszXmFhoFHyOAlz
z7rpM>_NWcFEfZb@FJ?epqek-c2jK}zc|~fx@?wgDKB$qvM74;p11V*;zb~Z$C?3ZO
zAq7la3uErjv@!7ycRE=$kdMBk7&f12IH|piBtdR^Xh~PG*3o-^n!hD=PcKdLS7GEY
zk|Qn9gX>f>wh9axL8-RUghX~I#x52jrwgYZ-o)F^H9ae`N_u75wn22<Wo6W+5)xUs
zAF9{E)WTh<)%Ni3@t5I-XWK#~K4NyhEYnDnGBYPzwqq|Df6EjU6<stOl&m<?Jnu`1
zNJT{}BJI302U!t@H~0H+e{qXfF+HbJETDc@Ii%{suYg~(U;jgIr)7OWx~v630qk3V
z>Me~7mz@fQdZDS>;T5L3gsIxezq!*uu}mP-ZU|$pGvQXoIEKtXo|33>CB2HIS8C%Q
z@N@l%0K0+X-V1I#@=eT#((HsvLQ<^={-brKhd-1>kLTx&ShH9iH0-j!mAf`7uU3MR
zKM{(2VF>Af4yF28KQf}Vem$=cLZ~n%`7b;07>pKLOiV#z;JaW#rUR8lrl*{@f5>^!
zPmr-gHFtDQAh#aA14`t+wvq+RD9nX5pGuZIhcAFHbm;!7?KQ#B$nkN_hdyRfRznTS
zteo_3iy8`yhr#ndfMh>2%U4;cEUne_){8hneay!e#`hxm{IYWh@xM!N_BIe^<*xoS
z8y(Y42x4T3XDX#X<{uoDa4z*1foLXzKJGk9zo#^sgnHwhnEeGdhv)?KzXAHAJhQAO
zUctf*kUH2Dbg5AeGt*noBRSY!k9a7ENoMIr!CiIrxD0(0SkiaF7jb)iSpm}LW%7R+
z4uBoOStMg>X8~BB{4w1RJ!=L@bx>D)y&c<KvoZYW#hoL|L8`N|b%f64=Ib>=9Ppdp
z$!alo0`9>zAHGc)d{j)GOiBJr2e~rQ&St&d9l&7XN-Hgd_0NBcd2f^j#6Y!<EEE3b
zx#dXF75DfOt(;OSk2A;sIw_JMF?W^1<h`FpB?A0QIOm*e6R~6o3IlPlMyMD+`fHw{
zcbRP0K1+uR8pp3}kSK4<2D^C5o7XYi?uvuco>-pphY$v6=Xh$DRJB~P{bBFKXzt+D
zM|UW}@54F3maG#jvgQPPg9_WjUp5UScIC5>h4(qEJWk%s6SO^U3jq6p{&1E=IcZ{+
zTLB+hM*a7ShFQUSOtN4C&(V!Td}`hI>}w#fYsO2bis)LCsfx(}P!w2{g<bV%9p|q!
z)3cP2N<<MFzk@Qw<J7yGGS>!&%;u0U?;z$AY76`w3TDi#d$a}%Oww4K1eJjuJhRfG
z|I)1^1Vgb+{rE<TKN%sysbkO&NE%IpCcT$_%YNCvc+1t?yJi~`@tCH=XuNBz4`4^0
zy?o$jpxb{3Un<?yNOY(QctMWVDo}tHI)v#Jc~;fFztLxi!C(c;QPi655mEHyW#DdR
zp&<-?UH>inEvaANTsE;OqU+?!1BPpzDT$LIh~_^<s6iT5Jp`++l~r0KLT+)6x89M%
zkea^EXq`oat^hqym0Xj16wAy|6sEBMEd%ditH80}q+iT&uM;q^aW7jwTU!Qce=hUd
zxkt5ZBrZ(Ozi$!IaN;!F>b_#%fM1uO>3BK$#HOjPlcH90#9ks|fEX!vqFS~QSnpQT
zU^nL427|kRw*F31g7Jq6{Yh9$#|?HB;k<hpXv}AT0{3Lis<QCTx5#>QX=+}SCX<z@
z^0x0QV8H5_6qAN|k+z#~SM6ZUwnx8A*eowF67Q!GK({gO*X5y|UP5{!K2c&fUYlW#
zQ0$%i=>19EmD^~&D;7daFOw2+$K~_9)D;i^ze*F-a8EXvJXq@5)=1i{@ewb?<Qtlf
z1sC-?MeWogR<G8&pfC=!9e!s(Lr+=5Hw7KjdkP9rQNfVm{I1Ptz=1rUWohGpJHZ23
z1({`YNyZf49CZ1w41pic$!zD+9VxDj+!Vcgoe7#!Z+MnHuh+)iwhuS_%e@VgqC>GE
z;!TYBN3Vh?z?00fb3~r%ZC_|8rR1!bi!`?(!oi?B*wlPJS7jlDlV4Q&RsVRw6bWnu
z){B=djJTd2aZn<3C@_>j(1Rd4jZY}>@f@>Si~PK)e?7gQaZ7|)`!F$@eea6>t26g|
zH4l<<`CA@gf=#^|@V@2x1&3B$4*Djx8z=*P6DYml0*$*;ANjd90Hv}GeJ16S6?=<V
zZG-p-$vt=rV-6>$%J0e+3cNZV6TY^3fIlMyL(rm8G2i;v%*=K|wGYh}<Dvu9pljx+
zQu+|RlvKD!!8}z6hugv17QJ+xb;aKu0HE2q%?+Ll=fp=;)aAYeievmVzsKPth!gE|
z!<ENst#tIb{EF&~${MdKV0tJVR5w$86Q^eH#jYoX!I(N&vY3iA?Fwmt(woSRJ<&T6
z%1D!rit=oxDuBBGz?c~;#s@z96W;M>q<^_l*=Io(EW1~{;xx~;!D21T(MwW^_^e$G
z2UNuO<we8{6nrtxkckYe`aY8fV7`8MBeO2+Z|d?P8NdFLj~q~UR4`8?#fHaRkLTN?
zhT`4+IGs%#J8+6792t2>VBW=raY{I4dZ7MTqb~EP?By#l-5WogqcXaOwqBjm$_awr
z<cb;Zd}TZ>m4q@Gsyxm$_ApeY4jR=Q`hZV;Ajaa1?mFZ&HhUM?r*bIz?#-&uQsjLX
zEo*<;6XjHP&0d~8jN&8QQYB;K%A^;hiAWBfVI3z~(nMQxC<cG)D7Zxw^_At6;eeo`
z2|s#!%}snkDu)9c5{ADhjX=KzD7=i^KxEvS>A131f!v^rtYk^OnsZ)D$AeL&R_t#i
zSJ>c+<CBu8i@?HpPWD9?&sTkJYB}R04uo@^E2Phm;mQ)_$bbs$wMEJ+h|Dy*ar>l@
zp$cxA92YYz>MF4SdIp}3R=!gBM+UJli-8;gBA{+fr^#{wlEN=jLeuMIspekh_LPiy
z8}`KdVN}uMm6u}V0N<t<941K&oH#Xt8g27A=620;k)@nkvyk)IMilhj)Hjza=hctn
zn#8F4`3%S!5#vxruQh^&=xKLveIK-BLS!6?Vc;eazdZ9JeW0}M6^RShuPS`1F5m(R
z^yW7q^|CWV#^q<XFJf~Uzj*;g^G5)u@fsDcLW@hpWYjd;wORBf^=!(s0u;nrUWhO;
zW=57=+5OvEpw|0K)W?Tc)Mv_vodzYKByaecg?-O@HHIBfTh>*nehY>dcwK0dreGfO
z_OX+EMDqRX=Jq|`zeFVB0jWKp1i$=?A_%y{@aTt()2FoS<UFLt_C4ezAF+YMk2Emg
zE~XNI9Ldma8vVZKM$@PM9qRV{>Fg2UlCa4>sEg510e2b{-{fm3-=E~QHKv~NRm41C
zg&`|)8#zj#?Mh{}MwjiRj}0`>bjJ{`4wJ1hNPW|cetah^tb6)b{}(_R89SkmDzniJ
z-bL24leEoeQ30#(MADFjUUl=5A+UM58=qK;yGORR=3EZT8mD(+!jBv#q)*Rw(GBi9
z{*x10j3UOi1hk}6qBRTg6(J|}4c{B$xx2T$w<u1%M$%Rlge`&qMck_XMp(a_RKLH}
zH&XE2V6tK*%YC+YYLjr%wfI?)XJ-sjH-XgBYy3U-LHmm6xC=E@Hf%@jTgFYhJUWAd
z)!1@6i;f^|Juaalm#*HO^=wFmV?eU$?e(G3t=fu`uO3q^uMK}pA2co=m)qdV-8X(s
z`MGEpxL`84%wFOc5N}HTF3tod%-QAOx$Ux1DyL3S_bj$t;A_tWw{k+M+K&wUqxiTg
z76$tLoN=coBO~7<HWELUB?_C0qwBeM-mJNw<{M4!_xW2C#t&6t>z&R-WB?a%q`00T
z_fBUnNE3AuEcSytAXAfLGlbiwB7&9WFI(Q$^Ljg@BmI@cQ{8N)#vTa!6rZovjBX0M
z1>M@LkE=i3=Ww5UM)&g2dn3mq%++pq%s)}E@4<tE`XryR2=}H;j<NWcI6kO}^vx`?
zhT>6Cy+-!~9>z_J<7xr-i(qOzuf?_F+*3!()59&BEnV(%xk0kA0Hd=O2$ks9%FQ1?
zd-;uhCNsRA2|ZVVPI`3o@*c#_Yz1)mf$L%~%`2M31y6x%Q5F>vCeRCf^UL6t(Gkj+
zv1cEAcnQAi&F9<3^%!XEt@b7G<x%*e_9pX^Dy%f1Y$;`1^D?3(%r2y--IbHP2@?Wu
zwyg{%cq`&|V<*Ds7$?ph!z)^3Tf0sTZ%%)|lz0f-{{Y24ya*Yq^~k0fL<J<-?r$tk
zW<EQabfIGy%Zv=aRr91s&;gc^8c;sHWXLKyDp+A{Qk7I$b<S(~!Qgx^yO|%_XFNkC
zIH|<A^}y8<;c<Cg3+diqkbCI8#0`t?NPJ0YJ5&0BCr2;8Rj%Alzi(&P8D8bpRt7z+
zbl=@YOfqS**ZW*HG@?x37#x>+*%n;x+$%8kJ3Du6(@1DEGspONeXPK_?Y3OSdXc7`
zRDmBViW#fm@sk$4Ji6<c93!)u(tgeH-q@20D}9fzO3i(Mwo8lFb@G!CmQ#~_Wo1KS
zm?N|k?@7n~wo`=Pw>w%X*CM&L?^)d57%mOF9GY=}SQ84(ufq!522b2a<APsGhg!;<
zSyAcoRwija+?UVF-f4g_<Z)hwd6xLOFL!v@+uRNGCgGa41raPS>jkUu+pQXMF%i@O
zqHNzzQ@V{<AnnL9vhj!Kwb%cttWUeIG_89~?N-~7HAuK>FRB=4NhVV2Ki+bE`hhK9
z@)w^?hb732V)WRv>vj%?s(u^YMlb#9_g#J*B#eKkwYP>@Q}%sx8Vuf=9^UD!Sy{5^
z3LB?+k*T~RBBfg9$(!i3-plE3uhjTP_-0TzZm)prb-ns5<MgfD>m$=v364iAb3%Ee
z`9GXpoC`w8sT&gcyOWPsJMNwu?u;?g`}?nd#kjYX)Z&JdO`mFJYbARHcMgDU?dlrR
zg@rYjeKIB6&)z=;CeEB5i3;-gS!a)qP45#tWPekJ@|&49gyn`k=+xu&aTw|H&GJr~
z@Y3SQUR(-T$u~-`y%sr5yA}}Q-B!wwE7L*RJP93l6cTcJft)YU<+C?7f+lO8aY2dR
z60a7nG$i8KTC}G;I${Rk>^=UevO`q#jzyBAr+-60`CN)-l(%^B*9{n@1>?+#@)HD&
zd%$J#m`1(nMbbj9)A>Z6@kP;`=HOQU?g0V0()8K9rO&f)kfGO-54{%0&pw3_?iK?r
zd00eXqz;BXbl@ZG9*3GoYx1OfIR`^8z=@lvXM0%J#*D?pCggKIspj<&?}?$%t+ud#
zkGecRWb5I6JkDCB)%ZmAh8@jp6o^{oOkH)zx1{;b*=M34U^{yB6T?p!Nt~IU_V5PH
z@UZVr^L5~I^%2gP@6N-QFZ^n8O`l|+HH@{N^YU3WzneHxlk1p1viNaYiHb1Te0m#u
z*Xc|~e?JdXe^(R#k`CO1HcBUtMRWM<mpA+Imie)J_mXmOkgqQn#_QSDV6+p)f(zBi
zIw=dStB-3MpQIzbVA9;jMlsXy2}1+QlFnNBH0ze3W77W5<f!SH;%J^4!pIw~HxhGK
z!oZPjb_>?dxZ8uzBVdU+#`a1N-_JA7hCjGCpPQjb=Cz(~au6_w3T+)c=k=Y!9Wmgh
zn|h<s#^4CvM>`ynu?uL=81%eDMQf+N5Iv0fN9{EM_mE@~rRLd4IHD3;Wa4^ZLbQNA
zBEv<cahq3Vfw_KTAuy>Xo^U!s30QIa>sA&u9y3082aeYrsBeu$Y;*ZdtNC0+g_18$
zK?C9($Q6~>M~bYUh+8AK;0$bqhI+<a<wo3*g+dkY8j?CT?^+=T>ouI@-ILs{dPnp)
zp=$c+R#fq7s_)A|XonX@qJuCIOlaoerRj^}y8d+C>eeVI?d^ty5uW7J!w#eCj_STe
z)6xDr4U?o$-l@`O1ce0k46D)k-w)-Tq=n^jfqO{IOVHkC#L%n_qF-dE@tNIP*Kl)d
zf>3$Zc=I_QvKY&w;lQn!vcE8UJJb9Feow4Yje-Yd&En^aHq?3Fj}Pu0_y5`IBp@sf
zStk$J3C6giC}wm~4jYZx>;b+RH8SqSdeBLAQ^$4O!j$CEF{5SBipxyG%!Gh0ZzpRd
z<W^Iys;my<i`w2XQ<IS-FHdEDcl#dG7WK#SA02QmA>QXAq;(#4bqp~lPrVzK%d$JU
zNGC{8mnm^e|JYB@N0Vn&C@L)ZIS5&W+=Vuj$~8XN+H|~b@96K5O3$h_IfHs&_>&73
zhl-}5yNFc_+CNWVW|Ll~uA97hnCV*=g@i2!+%D3u#}qWfivl^Is`r?0IdHfaTNIoY
z>~1Lsw8pH(eCQv_5U0{Bx6|C~cuJkXQ$TXog2wPsWKHPWbOuywWWS?W)*R`JqK3rv
z`RVZIPWcDk2`Ed;HgU5F`?xQIWW+Yo+cm?pwSA3VAOdcNc$Ro}TNKvm>uI+izG+-O
z{j^^bn_h>C_KYqo@I6vyR>-mv`g2``M<-#RwvtsoI@iqm!O~QKU&VzDy?LhCKS}#Z
za8t0xQ}s(?0Hh5W1s5=j+XYKmGj5p*cITU&0Cnev%xus~dT1bLd}mqjXSyDP@}JVS
z{r^upSN;zL`o~v^a>P0+VM$+#+30Yk9Bt7mXYOm*$`q5vk$W7IP1`uy%@yNJhlnA^
zgfZ$RIYtaJXpnm_!_Z(1Gh?4A_O*Y)_lNyuUa$G#^UP;H&+~kq_vd;CGcFxDBzXh{
zN$uJ|OH8CH4SsT01{Tkvn(YPchiN;iZo;O#v4iFpZ~Q?1dH+a~NyXcch}9d-<Yu=p
zseJ-uvKTQfUjblsaCCek#*XUlkW(3Lj6=%hh{|;EM823NdAOxid>pGcO5KejiQ06F
zaaEth?!I32cGlob<Kef5<)YGOqXJnk*LWUdaQGvBO+jN1DHy7%$tQQv7pby?W<Z4A
zpRj(7ZZx^$_faBs1%h6IKWg4|o&-^X93QvGd)QbBqqL>oSbDEyPD}R`bqu<G0*<M&
zFO~@w@#V952pvR#BE>;bI#!{^!QE~myH(j`htrG+VF&B&b;Um?Ks|IFsAsVWZ<SO2
z^Y)GvhtYbOXiOX0Y0Vl}NgM*)Q>89?LDng!z`S-Ce?<O;8XuH<3-StcjIONJWj<Y&
z0>)pigQo67rHYojo3?8S>}m}`S=6Pu*Lf1lhc6{}nS~kI{<d-J_lxn5lx+yWW>XuB
z8e1(!Km#4n&EhDXjN3{Bf*>eW3luz%O8;U<`4@cOVkbd;1!C!prjo5H6$pF!04aux
zrT$1fSmImDI{(*Xx8|R}R}oF}v%u(f$(|O<)^`vEweJUF1PSa`a|jT&=W`+GjD^W>
zuM=y+24Y0kn8Q{z5^R_+Fvlx31;qL9Rag*%9Q6cZY$H6{<n!)NFVk*V%6~G)d`)*C
z4P`5wDe}~#4b%m=)R!Bc7c{8ym2s!xhqA>^AGX>lAgC+T`o({zF&MC3-#tm48c^Tx
z`-USubbYRd*VgL7-<{m>?~tiqVt4s1eeIhKI!U|a3b5I210RWv{_{$MsizZq%Q%_#
zqc^k+NFpLCt1oC!NPsJVzUB8j2m43ZbNlP*OMj_COWGx(>TuQa)<eT18`FAjHG!56
zBkcrm?A5?O?4A6@)&gjAbMOsx(VHRL@xqAHeLH3aj=_c1`i&53#Rn_z59ezw+94uC
zn{Z2@Jxgk)&fG3?ZHP-x3FxJNE$G8uCl^WL!5DUr88F9ZcMTD=iMaq`1x~@E=0n>=
zRb6>TdB=&l94~xdS##^-M^`!acA(igdKu59WAb1WwgmF#U%y%gO>%)&vp}m;T#=w?
zIrVa8`uwf8<ar_;s%bCoH8pp4Jc!ZTki_MV8j%aHU4|S@9Do-d`4$+!MMNymtHky1
zRV=U7DR!c$Szr*Gk+Tqa?nGjsC$ob{#HD<29tk2VpW!C|wu;+{!|@D1Q|797t+tjk
z<_kgz5KRiqFKfU%W&uG7_VE)<FB8*fYu1{3KxK{&#>FjAn0p!UoskF_BbuAwu}1nM
z@Sze$xC^dHdhWx`f5axfG7TYnG{tq<YWJ5UQ<~?<ojB&g=JC3=)!E8SF0ClX?@K}+
zPPU|9h$1mmi^;)%sd=ZPXdIlI#AUO_xm<H^ruvdvG&pHCd=IiRRp7rnRkJ+pLu>f@
z9Zq6T=pTcwjaCE?gQty#dROmytPlK3A-Ks$6)=2HO>2p3ATQ3>JQE)*yq~kyh?}Qo
zCcI+o!#Eo@wAh7X9a6CmtLfyj%;A#u>3J;DkA+_<ReX6Zd}?CDllqw|4Re*Yghp6R
z{1B7VpkMUVBDdl#&)|)dKV1PU00}qBv6pL@uWFdbolc<7y)gb3-lCJ0@R^%^&iMeG
z)%1tt(DN=XSJx&u4yfZAq&e>-F#)3D6Gcw-j&!xCcwd)|^rW9dhcDzsh0eUHz|~Vg
zj7h5IDHlCnrykdJP|F+OhJzHx7`U5qJotq@KMwqwAC(Y4SCc3x9itl&5GPJ_Dy@#w
zjCmPG+k)N{P2V3^(wjT|FhVKuYF*9;ELson<`K;s=`2@Mw%MuL@dws%OGqA5yesPV
zMS9J>6-u#(P4}|M?cVs{`Joc1`U0<U{iXH1bv;5h+Y@waacqxyZH8a9r#}zuDtVvG
zvGEhYPI~e(1jscmJLL_K+3aAfl>s+)WVI<ZJf!=u(*8y+3fCf(f<o&p3JM6Gw2D3F
zE*ENky5N@Nnq_50Lo@b#32=~;$6`lZXNNO_eW@-J>!z0vH7vFX!izMSEUfWoUZKc6
zT$mZyB~z9<<jV{V-e;&}ITCCH=e-{HevJ1HnPVg*Kr*|zmm-k>bzi0^RNYDsfj(R4
z_@E(*WDyxPv|8(89gg${6Y3`1q+Fc!0Gmu|h^ibn)-PU<UgZ-W$%1G(3ahBvbrq%b
zyhFS*ngvzVo({0BmEx(!bmy&8{sGFB8ZXzivec|Ry&^pptLE#kWhm#eS*Q>y+*5c$
zn;2R81X;+=U^%-Dj}sK^g0GxS_RN`p(IRu#{qfag{g0K}<WMwEKWwVEE$#WDH|{J<
z%NaV$DnAS+isFHH=I35a)At~Cd}@*u^oBbPiXKKSgE7@T2P@i+tZ2iXFUsCkUCn5S
z`0QobF+hDnMfdkD-zFW?^@DQKpSd=XS~|ns{U@zL{2Ld}nf8VrrB%02x*=mv=B_yt
zjTb<tR?izpnoh*mdsIlucmSte;eHfZbZGsn6WJ^;2W~a+5!T^WZ0ru(rKRanp?2V5
zeYEu{&oJXG$5Qq3)1pzcrLb<xcN6jbbx)9&@R>>=#dS<@#=ts(c;V2G1J0v8Ek-q_
zM5jSWhN3T3p)M|*Hl2*YRZ9!4ROywS?dUYT)iP=+P9uxd4HjPR79-ISh~Y%FJVksc
z!|L6`Y9><~R{4OP=w&_C;Al%YCnU>ZH2)?2@knfT%zLYIa2G~d>HKOhi<}6{6K^Cu
zx#Ft3q#y0WeqD-0UH*wXV8%T@ykHQN&qfR6f=mx2EV|<-Hy5T^a9mff@!nZGmBWo>
z;aQ2tC<=m~EvekOEc`+4AWjaWkz!5f@yV|i`QBwy=gIFNwKQ6qChWd`m%aZDPGn^B
z;XBu8SPat}jmI#RSOg_J!`v_r7j`d+!;q8@CQ6=hmPRd*#uvKLFBo?UTu(GM(84z6
zlCSAR>VM&#y2E{nc?~VGAUxZ57FKAzx7JxUqbX!?$f%Zm;8>9M+uJ>gNyw{LOx1sC
z!Ok4O6Dnf09!?SExvw0Tg{$bPxNKu=00HaU_KFxUMhaGtvY4*32@OFj=Fyr?#78kp
zh9l;nx}3T~KP_$mlU08|W8lR}_APgn8UQIC?`lV!+#S>gsAc3-NFcL2{+b@#6j6OE
zjoi6hed6-vk)A4KUzQXN{$YX2@T{yM;W0r?_9JlFsruib-ry}m7oRFT4Q*isT1Kx;
z0*;x){u_lqGs6(r5_1?&*up2TV5S1!hd+MB`K-N4o#AT5N%s4NyhJL%Isco;J}tCk
zNzyA3c2=fsCrWbn+FH%m9*!!67&+J;8N+wKh+5&K@Ct)8i<n4!vXY%luA`mus+xee
z_ZURNJ{9bZY51Z6pguVtTgLQv7runrhBVi+!JSF0f0_+)w%;+1<W`a*bqBwA#0Mj-
z{CQx$+c~t2R+JUMGujlOcy!mc)Sm(Thjrmm;4j<w$wZ%@X`L{z5&1`Ig5OU*q&oHc
zHdsRW15_#25}YryEw!!JcU`{qasT)6pK<;>7XLRF!sMnPPl<gz+Y)KZy2_`HHaE4p
KQ1*-0-M;}*C#gyR

literal 0
HcmV?d00001

diff --git a/site/img/copy2.png b/site/img/copy2.png
new file mode 100644
index 0000000000000000000000000000000000000000..7869daddefe9f5476f678395e3aae91442ffddb6
GIT binary patch
literal 37973
zcmeFZWmuG7*Ec-DfRfT364Kp6D-0nG5>kV7*HBV3AfPamG)SW$BGLvmh;)Z^41$C-
zNJ%~8|GMt`x$gHk-f!=R_ruMXnRCyMbFIDCZ>_z87y7#DB!mov004kQQ$y7d005-`
z06_FDd|XRp{5d7=1L$X{ejiXZ#I%Wfxb3ZB?gs!6;eH^Z1Ojqi(cwCLcQH0YnCWOk
z?YumMZ0)_CI0yxKc;k8l0P;al+*c0=ge_Z;hr6dAG)RH{pB_-$_rJ};>}>yZLAWWf
zo9XDYJ@E2%V3QP*5E5ZmBxGY_llQfEgc_>C{yiM`q`>ZsKzKuig#!Zvg#yKeynLO6
zMP+4Wg+;`K#l!@0Jp}!NJrTA+f}VaH|C;2#=23O<v-5TFM!0x+vi+Uc_KBB2LV=zA
z??V6m_pkRvxH$gbl|23aO%{$o;lIBKiwcPd|M%Ruq4IxQp%1(~ynP+~{BZLtO3MG!
z^MADc-*x`gU*Fja;f3RZuZx|gC&IxOHyC02_w5wL{yqBty~qD^E<Ilt2OQP^j28X(
z=>Kf{cYk@|zl8q>BmTwZe_C;DRwR@c{_ibQB&=Q$TL1u*0h+2R#zDaCJc1yqM~u;4
z6ge8|PrV4s;b7G}Dk>1_5L4Bf_!jsSs~im#QGh{pNn{*}ni@m}s6t+<nVM(}d1im@
z7o$Dx-ncig?djXP9$aVSnJb=~=hYgrhZYs%WrLI=sggm4%7hxg|9$vh9{le)_+MH0
z{~I3ka6)Ru-qyIjqw0}>BnEjjrMq|tRqb3==xx^jX&DdX2y1oOJXz3Sb(o;wy<?8r
z?rCZ;{hxS7^+;LQ-aw~b?LV8@;y}LaCAUf!{s;Q0Q72QH8&i6u{O3V<UW|Oz!vk{p
z_@BH{Hh`?E(tKC7_#Y5j%)`bl7N<Nb_@BtCZw!=8QIi~{`A>{R@euaBK(i+YJp51S
z{r~b+m#KS(#NLFY3K)HSawair<V%G;EwX+6wYz^${R-*Pz7=icrbDQK(l#tKz9(to
zkq$B}EC-d<_lae)q#0G)bx8FKEZGlq`yCF@5(%U_QE&T(6;XLU@Y;Q4$C91wmf>+&
z%|2yO1p-rl+Qqt$zw4~Zx}Wc`9Li7!&vSs?*k*6?+sS=gZ|s#?&G_IusMjB`wtQ;W
zvuas~u-iDs1UQ@HRv$3LZH{A}aDxfqvq#typU;jQOg%B_b=ntC&DZ`U&b;xV?w6P2
z5_V_VNd_f?<^s&tCVk}E_|ZIKUqaz|0!JE2?sF-jzsfe2N|sD59Df})gJMT{Eede^
zN~QOAkA{ybO88KOmo<}OTa64y7YzNOQcGf|%=sR=_tOr)Z+5xQ7=R$7ce`H$I3`i#
z-7frSdhsc-*ok2=aMRD}l(kbQ^+XM^KO3rSD;1V6UL&b`G;s4euDTeNyjDg)!Yy61
zNsnuFju+78aaFxxu5^CZbME8Kk24lJuf7*#1c$rk&!7NC3aWV<JoCpoRhv<}Jy&i{
z**nwM*Ol_(zLP@TMT%oqGm3Nd6)6r<7~88`R%@sjK!$9MLxyavxfFYGSLL7XtjvgB
z@fF$~fvAg_7>-<rY(j@=E7yVQ&EU06>ZV_}1oZmfn!-V?>MPd+;#VibQYF_t4Q+B{
z%qbp<c}_Fy`xBEhZ|<Ah4bf%0kt(kq@`Wl1F-Dc2MZEQDdi>WE_=r1^zfEk?g&>A+
zl+{&}rEX$tJaQ9wV{d+6@A!Ey4~y$*_lduEbEHp<vN+bu*x_7*%UE(CqL#qwzA~@A
zNJsRGE95%O*$J*+=eenu^L^c9924)pY#7ZVXd4!fmokeXedh6-)RuW9A>6sc{b;OS
zKFuRL)xsgyLe44ovzy<bZ+(eryuf(K2LVLxS`HT@+QovuKW$F(m7mkyKe+*f1DLFy
zj$C`jMOA^W^AFaF{jZ+$Nbe}jJG=Kwp6C#?IW_RvXIEbCJ(y*F&cKPI=^c(H#3Nnm
zdl17i%yJgwf~;66y^ntUZFr;0=U7p)%VgZ<HWtfW7^=oHS;T3Is7o~rp!@@&DMKe^
z-0Rr|nR{r$RwR$Z@S3@!k}B8Cpyv7q!x@3`n^@+PQlhoswRL&F?R;;}Bk&8QYlm#s
zo7qQXdA<3Ae(p{_=Z^gn<*MD2S!!E>0X5=%Tn{54lkPJ!T{%ro?Jwqohbn%A1RQjK
zj#(1S#$gMW5e_G~j+(d1<BLG*MMSc+Q>P>rg5v95{qle8^3BoCHie>w^`^%Hb?U)(
zeXDGQbiav^pQ#~TyW5gCUwPH=UQSlJ(m00gy=rLkdwA9+X%Jq*C%0uQOFg=vc@Y|(
zzd0(=jg}GZM%%b&3)8M<XTJ;F9bfwWs8BaL&tqjU*3QVg?k>TY!Gi+BLVLbDh}*o`
z>>K5;m3{Q#n9R>7-*WW3K@p>ti$ZfuXrWyj@<*#H7C&};1QI-*py?^-p)Q>~U+w5I
zIc}HAh1&j2c&idQN<PrIYIBUI9|6&0vg$);r8p(Uqm8RhUYh`iPS#6GOyC-k+JBK5
zi^I+_gZsFp>G(0dLRJip*+)tP4Na-R<&u3)x%Zw6b-9L~*njNm8oBH6E!E_nv;4~Y
zpnF)G=O$M1o`~+D`EK*s^Ygs`JH?JtV&4NGeoC)^OiJ{)vtSsN48sG4+3$l=QTqb=
ze|c@lrjKJb$`yw}R7Q}12lmM^P8xGv=JjZ;s&CtyW;;X(g@TESw9jW4_oJZ!dF?`x
zH^PpO*AHhz&Rco5e4Zq|7BxQn-9i+Y%}S`ALAvpbD`WW@KPa_VUVh|p##6|dyM`{J
zSa$e@dy;5ftVEAkiQ`K9agcl3KeXN{$Eg%$$$9}u+*=UqV9lKu5^Fm~zQi0TBhh>h
znJ+0J64BhWC4(1pS?x+|Iq{^JBW{Z|%(?iQPo>S0kEzU|gLWzX2=82TplhOn-KmF6
zF6FWa{w!Bd#KW?@#6E5LS5HTtSsC<W5RXqxQMIpY2TIg;<ug%Um7i_ht&7&x|IG1y
zzXvH*hVmz~Drv0WtMD*n^P`se(XbP(P)^0FMLkfy0OEd`WUtvJbC-L*lrSLB6sE9s
zz|`yg!E&CN{GslrbK%~AZ+^r!$GmwaZGku?{9L{NM~2iwD`TgQ4YS_CBjuEyA0eLp
z_J)PeL5S`7YHw1`e#gM$rSqg`w@s&%yzL$SrF@fT2)`|6rq6W0o%|o#@Dq+p8nm6p
z2LGX>d3bRWEx_yaxSRJsbSy1SZ*LZ7@oD{sisw@MD^5SPw`6dT{a?2nY%xylTTvOQ
zY5rfeTUd%ad9|glT$s!LyB{0G(pXuyH2z5i&wps`G%6hMXQYY0Ur7VG6k<+v+y-OL
zQ^yOrI!kT`&wb2JPhE;Xhf4{a_3ktE{9_n+@+6yl0Q#*U4Gm(fZ88YIKbU$ul^H$B
z)%$)crR_FmuC`<rqTvHNqp`$;MoD67YR$}u^YO?4tTv&~XXBC{eJ!%YuL80n`&Bqs
z9@cmPO;Px8##{9dYQ!iIctoH;RWn7pix}j|xl4Zcx`c>CghD=9>&o+`8tMakzA6`g
z9nAvqcWzp%enlGg_F?s=glP!K#_x238kyvYUxLcr2Oj%EXP^l{;%M*;D+00tWF|1*
zCY*qc5wsV;Sx3OCer${!Bn`WSC*XEc`!M$qf>+=t{%k1|68J<YJMo(8k98F5++NUW
zslQ@HU#c8{-|$EWXG>rrAg)Dy=|096!efa5+9){;8JxNtz+@2vfhwUxb?}{l=Jt@s
zaA}UPI^|$*fDG*=K0AHG6}`TjtL_f?J(CFxrQSY`VHOpThUG*9H^aMqw~)s{G-Yg}
z9JzlluVNaw4W>ubP8jnhgCKNjmMl&!!|Yelg;bv?<ROY6L^xAjwFn&(sVBajG6x%f
zC7>yr{(Bx^10Y4P{n8o_w_53&iAKi4$qD)CjjXX3p`1bL#9NgdBEH$$oQLXZT-;*#
zqv@)IrKz|>Grs9wE@;ShMO9d`Z~AKRiK;8hMqWWu;_Cf<-dhOH8rN_LLF8V$u{SgJ
ztmUP9(=vARnI3^K@&LKM>{(5tD(0$A@|H{Tb>oCg+CpRkn(5vT<y!dwtDE-d(2`UO
z24g>|ZwIpEE_pTSKK4e2IQk6V;y}W<;=`BM%76Hk*SQ`~@J|?woZAMzv!jLE3ek}S
z1WK(%cy_OY3z;_{(^rovy1OO2e3XbYKv1kS(gf>*u?3Bya}4BMH3Z-CHjp@Q!&j)W
zGS!mCk#vna(Gn<K&bk47gwRTZWKT6i{R`=mF`WD*Lve@vjc&+y0f?9iD#m&B2Ofi#
z)4?r$UZoGw!|{M7rEZk|DBtcDDGLPsSdqXr>W8QN8d-bVCD`G2vhJ#x@<K7aWU<Kv
zdbUR1m40aUNq0QbUw|x=r!OGi6s8^3ZRR_0k9kRTTh9K;X$uYyQN#u~BQWL_kE9V{
zsRjzaTslQ#6}Eao@rr816RMht{LoHT$a2epqdFzA2Or}Ta08}R_NmR0p@T8Z9N*9t
zQkt{Q-=c{$L+9u-x~je}r*DY2Z>Uc7Kakf>NE7K(gcesyPrvkG?=Gp<ZHKs8;g<aU
z_fqgDpRqxkurvx3Qbf+Ql=b|;N6G0~Pmp+{C`DH<yUv5{^*6`Zd(}$LptZ<;Wttvi
z{^%I-KhlH)K&GT1QD<;@H_e@R1F)-#PC3b6iUnSdZe9}0gBRn|L}J3+kI;8EajORV
zwez^%D_FVn(m9oir}Zy>wXU0}--En-grq%MLk?-NpNNY2<9&Z%@Az@c3L7$V_glEr
z_!EKOK!x@f$do1`74u|Xn0n|PGs2B+S%Yq(>VcJ{??D6d0j|n|<+mG(V54YC&qZKr
z`-4%6?&^Y+p3S>sH6HSds_wVNJOCG*FU^B-XtZyGlb36^Du&m}Z+J%TkXV7Qb*kG3
z)YT7HMb2Yw<?`6y$dahm+EgEDU~Vxnmq%l1^dY4sJBctK;tkb(!5*-cveUlUH1QKj
z7LFwUyVx3|zob8DC`(S3hP={ViP?$2e_u8wzPg2UTCiQK^}b@_>vSB@CIfLa4<J)Z
z?`ff^ct=3TwR4j6jL`$#>O5G=?UM2RTc0HFL!G9i&1CQq1n3|^0NT~pSF_^10E0Iu
zh*AKe9u=aZfDyV3YFHILm<~eAo3Mx{T`}HCvP$}fzBO*JM6TB*;_}c=jw5CK{`PQ(
z6s&4yQFZ&F{jUrS*iYPL4hKf7s((Iwc1;6i7X!LmqI2ZQ^)9;EeQ1>l_+R<BTZMR2
z1OZjG;ujigtipgCPhM4b91pjfvyJl5)pmIx$~UH4@J%A$VWbOX;dlYWJ&+oPxgk4j
z&3;ypyL?wGa_eALw#Qp?10vO#E35tIsydJKpw0GVxa%at_C#%NK!Y={$E+1R+BX)C
zlWchtoNnm`T|5fqx?IiZJ$ToccU=E2*Wu*tvR7Wkpef|5PX6;U@~xDth5T5x-g?l*
zrn?Db3H67K=7Q*jCqVl~tZ=+uKH@XKVoS9l&u<>g9}^&qUUt%3*jpxuv=VYacWfK%
zE|-Mrm%qotm?TPWmw9T3Ea<1;Wz%MXUdk_epAZD-$E&(CiygdSpFQPDsZHk|q~2KM
z?4S6Mww7RgM97QNiOEdaAAOxW1jK)QV>-Hac|D-+w3=1a;;WsTbIqh)AT_i0spU8I
zq54kR#E%%t-k1pd=?2C@x)~^>861v(fN=A5Jen({Ed(Di1~Cl~JOFl6nBc!)@EZLB
zf<8wC?cpN=PVi$R%K#iHIDUSWQUI@^?jk)L=YomlX;_Sk1MY~mk|q+MiNClAgcv>p
zrTVa&Z<iH1Jv(gZSA%$Nz4BO8oLRr6nsu@?FWb$e&&@wz?V+m<Sp}A^n7^peLD1}N
zzx22#P@{dL7nlXK`I!yb67+L*%bTR*GOpbx+^-lT9I4Tq{NYb1qj;#jfA5nNs?U_2
zjKh07Kmq8DUgScOx4ghc6pBATNit-18Nzob;HFa`-q>dbKta&AHLx=?+AaG9N{k^!
zYP9#Y_zX?;pnFJdSJm$%3<TM|)hMdfV|UA`%}zbuTh4BpW$%<Lh8?F-G=RcX@Q-ML
z2h=hZ=8fjnq($h0qr)VOSX*Va*l*{FZ3!df%KnCkC^V~5M-3Q)jDh>0qr(yq=%5w?
zx)gcVZ*yvBfs>Guh)!<idNE|;d<yn9#wP^vLuN8Lkxc0$m_@Dxb;(4ah>vijN1+`N
zG@Hr|_-I3@M28qhq+L5RHl&*5lZU?nmKH0etUl)PE(ydRX--bKb{aE#&-JoZ#BdJp
zGTZrXA2-g-I>pKFZ)>|@*XdgD@EB$|IFFD{qv5l&<Z-nZ{3Da!+$YPwcV53cUvB#@
z+l9+>52MfgGwXVN66Tk1_u$@+wGY?z14l+j1Egnf1Xiv){016qR&&BHoN{$ZY6r;T
z1&{VsyL}cO3D5bcXEGiAjFdmR-nRO=r&aWt*bq6CVPPN;PTz$r!4R>TSj0VZ?`}a_
zb+3b<{x4=DdU`IOt<VL6(SUqnx|(K)Oxp{$n&w-^1oMG1(*P+D3t1kMA+Iz2jZS0T
zrp#;kT@@thDh>8^&3pOAvi5$KM-z~riAjc1{y%Ca@oQuji{cm)66A%^TK%EMn9rWX
ze{9#i!o@9V?+G>xl$;GJh1d|=&WiV^eSJShwgC+B_06BjXgY$XY&YDSxC{Ft6|MEb
z<C-6P3IAhL=c@e}MRwC8t}fLtD$JB=*J|xNqix|iV=6<F9~JQ%mR>gy<_%IG=n?-A
z6>wZT@RPtHSJL8e!0AZ9h;#{wyLJzh9UTr(rXFf<<(RL1iK-i{h^QUtG0(7qwP=|)
z`h9XQafo66^_l%y$Kw+DKYyOCW@$#4`?94f*0~rcbJzb6r?vRHtCvBaMA`f6V-yfM
z+OUk3^k(b<zo5J3ckiy@SG-;F$_SH-Bv*%qAcQsn5V_}x<5|&R50NXzhvVjN##bBa
zp7X`gd7io_z<wN!vWG0pdGo~mk&5}T#!kXf`vRI$l6)G##!bO~<}&2Jl0z!HHp3hI
z3EymOmUL!~{Id1Kl4JH3xxTX%N55tK);BFn?wH}};5?0!$CRPy(JGzx&4zr2zcvRf
z(A3omoz)Y~x|U;fIwTb<*(Zi`DCD+bH%M}l-5jeHA9WI)-8~~8V)o!=e?~A>r!8E1
z<!)Uy6X(44+RmFl+v}mU`C8AL4B0#4A2wB?jaaXF|7)s0U#v(nEw3@!(cpeLBl8-4
z>>Z}gJRz3bc|D9TdG`2M9VWc#&VZHxN0c+ownTtR%ug(%2<SHu0RSiHVhd1=W1*j!
zX4MJK)RnvYA8sXnjqTD$t*D06KA6?l?6O^%L#8YsZR+$j8wQCW(~Oc~KZo1^QB+QM
z34^;rK0A`&e-h~Oe<N2#dGUQHhVt{)pXIKGF4+vT4?4O}!`{Gv*O)#rGt{HA*l>oS
z1s4v#K^>d59D#2d)9>}6*a6RlV<k`4%}f8T35R)hOwiO+qe+G=D9MZ`HPwMH!?xNX
zGc2|$UOzy2M<4`t_UC;U@wKS1F@;CJJmgnV?{h)!%P$@J>hhF~^b2;GZpDrG^huBp
zASgF#Vc4u8Vd?U!y#X#^Mr2KZjyv!Z#vH_DMg^)=gJ0;-JIFc<Cao}fL-xmV3cas=
zy`5DgF2J{Q)E`pWD#fyd5+)`k!<yuc+4gB8I@j{tF7pDnKj@r^rTI(-gn#jQ&udv+
z6VlIal^Z$kwqOInLVM!kL1;Y1r1F&oE5B_g51&gro_BFQww|qR1poFdSSs~j#x)H&
zoZ0ehE5`8IPBjK^35b)gI8-Uq1{0(maAY{pe*kpf4c|L7*pF{_w&&ired(YSf63%_
z#x6rm4E&C<EhO@0Pe3pTgWf6oNL+`Hqe?sclgJ2yIDcWQBhB1dDDuc3e!b<Qh-s|@
z4%Z2QsWsLfr*1d!f`9h)-0Oca&@<jxH%cP%#BH-HujDeXqa2+m*k3F1hWsvT*US6U
zs4TcR>k#YCw+!<)MER1zieEaVj|}nN87R(4yk*Gc|4e@LQ0cNm>`}k@T88e-TA!ZZ
z==YjBrmI0S2&qc{=Kg07N&~q1zHKDLuadn4O@In0ybDKc2k(mq5mGsok;j_C(VxaN
zg-CSq<d9P^?w5&H0#d9maynm%+-F$iX$DjvkIBbeOmp~>2O93pxEbYW8E)rLk8GN6
zC%!jhz0*`su$^s(&zf#`hB=^l-tpTt-(k4$nu^l#dZu-2UvEjSKfk?2PDt8mFV~@B
zC&OVe|MU2Z`Gib1x=Y=;vqDoC*GhwF*?P81Q1hLpG4mvW5^8w;J^@)GGLe^g!mN^*
zV$`6SxEaq8+15^3NPWieiZp;Ap<PSl4%4l;rUO<4T{)pHKJaG8?5Hc~{RUy{^O_(S
zdv14@lx6MUbo(D&!c674@rOX^50ai%xnmEDeoQf_v5WtH<vSu;622YRT4dKa!)!+<
zedClZ>L>UhR%m2eO<{F|$%uMrr6NMWZ!hpOWH|NGIWby;6INaD((bzR^>eEq9is6~
z>f>j7!Yr3}hlv#_lBl2(j7ije{8?Nq(=>u*Eg(eS<49hX`ZQh6Jo&^GKiIEB>E}cE
zTZqEcX?$->W)8Fx$_c>St4}&E=I$i1%Y$)V)%GSayA@Z`5)<VLcnbOrIeQXYL<6I>
zV`iu(cac?jOH&wCP~MV&rj5*`6jVOHG{OwJCFD-syP@I<mh5Ep<qKv|Q~1GQJ(thg
z9iKB%hZh<s)sz8#Z|iV*cp~KBN!K*)V*gA;<UV~?BjK0!k64@n2zRIr`(9SxI^LUy
z@$4Bo@Xt%-f1}~4C=@%ie1WB`CMX=ArV_xf0~b=B5#6HeN@|!!lMgUrj<E*&bQ+~h
z_7CM&0&@eOBiMz(StjebBru&<r3f!3N#8Tju$q{`&O&ko1U@ChZIXWLVXT{${QZ#N
z_nOBInp0^#OL#D#eWx-CP9QdMH`<RaTgrRVs=S%L-a}^g53L!)?_MH<<vW5>J#jpk
znc)K38HO-$NxZ*G7>0*_Il3T`y<ITjN`vLw$j-WOYoZ=B(GL@W5wtTu1&5PuC=Y=H
zBTZHAEwPWH>?JZ1XU;Kud%j*1z0O|pxFdeftkLztULw_wLk~C%GRNn^t~I=&v&t32
z4s<T5%BCE+Wr)g7|Gestldq#EdoHLVMbrJ5Oyt*h=8?C{P!1aeC-b^5c?SKA8OgL-
zx1LkBl@HhV#4gk+2{Z^Qg1ECuXQ&sBFXU(8QB9%0#C1OKm7Dv?i2172tW}`|T`x6R
z!f_tqY?KNzrPl%z%QlYwGme@iV7h%7z-CdRYt1k7ij5^V{@dV>-aGepa;L)ulPT$8
zg~nb&MN>J({rNP<N3kj<+SFL$9mcLdVJ(`Uv$lT`Ib2qall#8Xf@Uy%=#012(dS?l
zvh!BH+x>EAJtlMm4(-1rwq8PeZ2oE(r?*wDW+>0U>2+{wxtsBpfVl;8m0mqmr44mN
z>Ha81dZ4e@BAuluZx1DA^mJaO8km@l5Xs-dM{WV;@_A2~82WPAp~+_-P8R5*nevq1
z6Qz`efX*Z9LSslelsPcDO6EDd`u9Ibx&+V|jx))oydAY_`11inYTR2B$eA<xf*2vN
z2_G9R26ycU9%g>lC5SH|{aMpY?migH@#r(tCH<!as{quZVqS|fQNz&hpNBIf1t{+j
zEHv}8Ai+rQ72^o+Y8iNnANdw*e&<sJ^W@tC#uG+DTh&3Y3q0y@{0&Si$r>u8U7F1*
zVf~7Jgpi5Ea9O$dvERLo;b<8T;FxrRxL*0{T+IQAHwzsdAFA>14f5ai!S|d+!jWaW
ziG*)%Gd=pZWfB>g4WZuc1*o|`xKdLPGf|>_%;R+_Ys*da*liNdc_|iC&JZ%W0uZaZ
zR@!Ro=bm9_kz?E3D+%agM|&=oX<yz9yrDhTtIb`~aYKl$MwMXN*r*%1`QG1d)hj$y
zN-5I>o!@t$k4e}A$)*%dj7sq)v;Q?!G=d&kxKkIFd?ny;n(aslRL>1T5+O6Yg!YsG
z-7`7|wZ*p_#4Di0&G@=Nvd9Y{bD9BiT_nAWkr1}X7nzTghP<sE*B8zEh-J(+n?0hX
z@G-;l%6h4QA8xv#!AjT){_EfxQ$}q|I7j8MLFSE-c1v`fDV=42y!6}175SD%V}BO{
zR@%JIKxIa6IjmW)8XSP#k2a=b7J}9yI76cB^Mt8V(nepD!d}oZ4@YALo$YPXxKnYS
zZyl;+%ItgG_XoXofKV_$=FZ4sgJFDx3itAzrPEE32z3R)&<g%Z*Xigb`Vk+u+tbt+
zgww2N4os<kZo{cA>@nTcug4x29r9sgIT=J*gBiI~lti{}RqE4{M)OpekP!rU`Y6aS
zn;sb9Tr3B2Wn5R!Nl^f1K=+q};ttWEK;XyXyF3-s=JZVD9G}q0bK8donjA`eT7`)&
zJ${PLgpS}LEyrgWsSleG*3<r1kiu~_Zg$$z*+0d=4`BWmF#@o~F3nZ4fxg_1Jzy*w
z745rduq;@fU=6v6lok`?O(y^AZg@VP_Bp{)_FTd4=jR9oOQpcbnc6?Y-d_bUi72?9
zHw}Dhl@n8AOdF}C-1rc+YxW9gvNOB7ImVurU!_~>hjpvjzk?m@X7iRZBell2Qlatg
z-_QspI%96RRr*x>UJg9SLx!NaE1lE*?xB<}dxW<M)FP9>YoPEL-<iMdp**SLZk#k9
zf}K9E%x{2O<rN|O>>5{FP6APQToz33AD8rI1%TD5j-fuW2=(zUiEy&Nod%=c%dnIz
z+COE=gLGnt#XWMDr-1IO7NE@5cO54{r$cE=w293xyMD%!DmBH)b(2SklPYi<vJ-2K
zVNAELye&3N5XfotoFF))P+Pj03g&%&<=cN2(TrE`|4x~#-`3(3*zChhN&{LfPrx%p
zT>-|wxn919lA^)=%KX@vnt^oqk__RBJ-<?f_z&ltms&X4yG>{r1@mPxmslZVW$vX$
z>gMJ)5)g(E@5{zC7j`q)+S?xziqKy|b_oWRHS6PKHWCUek_c=F{jUgoLo3I3DuUX>
z@I;`!x7gcOYUYjanxh)pDcT}3soXUSAW#L<FC(uW=y$2o_T3hUBt0F$khs)ZGVaI#
z<pCQT9-s)MDNY@?D6@CnjJ^YUNuA}Yb-%iK;5nVb^DJ=(<wMoAWVQIb`%eg?`luEk
zE~24nP$)r)0B+E%09oS@&2YtAKR#b1zCOd(_$>xU{KnTO#gS}VCXQh>yxy<7kjuZ+
z$R7<*e0-xqm<Yf9%RMXd=|a`ZM3EbUd_a6VlgO=+;2$K$GAo=cu!!OSs~CPv$4*np
zb{jA++WtKYA$(#$r=+ft#n`)UM5ce1WHXrb1Hm33|Dpj-{E3nYQh~Iu6cr=Gm#6hY
zC1P~sh@%gr#TR|g86S3=;Q}y7Dh(~5HJ~g~gSH<-SYa1kr`JYcEpz9+{_X{wcY`1Z
zmGpgsssN-2cEbrOK2_iaAwiHYA?YE(L~p#ePaW7>8Qe~+%zZ$QxuFa{WtPJyY7Pnv
zlq#KBcW@?S=8#6Wk<2NZOA`|D*l}hElbD_Qrbwp~L>>V+3NHsQXthC?c`}R=U{884
zEf_dlodxe0H5VX4XbJqnsW0M{_QPMEvK(fPdrzL*g!{2+O2st{!hBQUYvp_zw>L85
z4eEbBv<qN%bAl2`cNdVO(Y;s-jL_>Jj{ZK%h^+onEM`$T?;zC6aO7L*JNGyWne7#h
zYVu2vA_Q$FDppV>MMB~?-u`-gbpB_Q!$o=7+$0|41pgx#I#$<iBEsB^Uj<@D&fSP3
zjD_*qBA41bk#o(}Vfq_<7-yGeSec(=b9S2b->A<WU0}eRAn*XjKv%Qe8NI;X1QxF3
zCd?yH#CxGiSevrvyf0)%Ha)3Z_-WcoCb`TAAwpKTXkla=$XDTDd}+QfopnnJ46`GM
zTv1_8CY>Zr$RZ1>ThBO%+4cU2<6dkV>kHU7zPrLVne_e+h?MmwlvuABZmfzoO<~6)
z!XO8<*kJ@p+OK)0&~N+osHQbOMGJ~GvAJ3F<v+0|OCva^mey))^w3H?GI0@wA+r01
zy@%N#6DFosW;X$6R}r8+U%IJ0N`4C~bL5?tRwKZ~pBnL2%<Z|pJwAf-m8aXtmT0RZ
zZ-7ui1bdL8jN&nkvQx3x{@mG(4Cbd6eY@Q7Pcid(YHZ7`pN1<fDqgry1j+b19C$`h
zP5Tq#%_T#8c78LkyZRC6#m7Q0h@3KEq~4zv)~qg$>-k7}_SPs;=aJOkz=|fS(Rte!
z!@8r2wj(!<VnpVWCu17-2@rHxt02~7Z3Q%(7>w3EEN)Wjk&q)Laa66u2$*Y8@yGz>
z3N(9fjdEZuFwR_FX0WJO%sFx>)^v!=t9oN%Kc-iwi7a|y*AaE@klxKb*S{y`Z)bnO
zrM$C|>;;+Fvr|@27<!_o94nZ86Jf8{6ZD{eDZilI(wn-_ZY_57;DUM|pUYq+0aLCj
z&Tn&I%tn2q(P^Z~{QV*sE{{#}XP*RQ!vIA4u>qp{UuR23xQ~d|)S~TJC>s=#57@ow
zv1(hqA{3VdV+%pH4i>SpB(Dj63T%xlL2BW)uKisnEz0~OVyr@`I`9BRghmD)hVhI1
zoex$C#M>;29E)g!v7?(P>6}cMCheyX>B@<r*wHzxgXbLO$n816*<1$;NkFGu!DWHA
zDuKP<_g{q)uHLt^@M8e5f~(rv@{NZg`0`n*uWi_bIX>>)Gl;3B5r=%e3Zky-m{>k5
z1PD_UpzgulB99L)g6tHx!x{47vZJ*%`F_9%{h1bjuo)4Br1f?Z_)3Kl!#e8(6n?Vl
zOfp(-qim(`U4^qr){ND>**PT36$7A&TpVuAxBcLpDT$oS{n5x#=-0g<k@=xLjFd9L
zw!l7z#-y)4W*0g7)ehiAeBL(b?-ZgYa3n|9r)fXosiw&P0GDH63*;dQ2Pa@)$-1!E
zOFnOjAm;tGV=`>n!N<+8YWjU?;Zep;XpbBv)t^Dkm&#}SIFIqO2%x3n<*EMSDct+z
z&qkH->y07&=8*O1A)<H4`+IoJ7%QShyk3S94pYE2Jc1-2e?=KO)8b5=bd`gF^}ZlB
z?r8gom)D~~A`qQ-49L@h<|6psp?=*hM|&WC^+j4%lU@=v0-!8%0A{k6wABOtyCp_+
z#Qi0d=VWkm?d~rTgEoPU;*lBzqG_o;VKuY2I9GK>IcgmDMVRxJ#xdUu`3hIg2sP59
z<f#0m30g1#$>j%c!o5a>M?q;Hg7|PG1m#Z|GgIfa4XYSo;`PP4;%CE<DJe(5{^P#c
zB5Wp*LmKh;JiGGXzLxqSAtwIMIQCKX?b5O7Dm7&Hr4#(Ysw#fP@=`5<u04AKYrM7>
ziVT?db>UEHlj;DfLigP_BX8Ja8m2CK2m0a$*u9}W2u5(WRD<<5oJZ(J$aSMfUD}8(
ziQgd0Fx<$YfQsz^<4_72V$O@E4!bYprE?^rIh;5vu26;-cQb7eV!0Cmpany15y)mS
z=+lpp4OlG<(Ho0N2&9FC?&q@4HpI|DzA^B@BVBI8wKh=EQ1tNJ;}XRH<~p}&{*pTL
zp?6;mc0VG?mD}En^Mw-ZM}*vh>#0J`Y$Tp2mM`D8dt8KmjTu~0rOkQG{#OYk5^l#w
zJ>eAY?u>f(tkxPt1kBSCPM}2{VAZQX_EUdX2@XW1BU31$nc+D13lXNhi0^nQ%rL>2
zZQ}EO@LXN_9%<5+<@3YbATxp{H>V)8aQG-9<^mW{bOV&3$?147*7>yX?q!0lbR%Qe
z9t=uM?Y&}V9P>A}T!=AsUYTL3ujB5!B=UYm-ZbAV7Vy2s!j<HVl?Oa8Vf}2{V~A+P
z-W%#Yz6tZWn_y)@d4IPZ_8IZkfNlPK-zCIjqJt;L)M{|ZR{Pu6yF+p61>^d=J}l{j
zPZ%Rr;DTqjm~Nk0r06!1G1M_2d}|C48T3AX16XX3Py;O8gji=uPDSCoDUic8_Nj5K
zkpM?5E7BaU<Wv%0&<r@nUy4-2GU$^BIV_c6Q#+{h8ECZ{V&qsAJtA%$%P&8$+AB${
zUQ)>XwXQ?q)i*D3og`M<oezdOp?f5Oo6eUlkBNhfeq|h((SJ8KZM2B1V8Sh2%!gsM
zhRrl@S1Hi^6b2_n)nXpXL88U^g2C9DU|WKsk&l3_<>)-{@j`uv0o}0Xbi4{@;O<`A
zO(RRlw&**rrPfJmQ%l>HtCKksYisLLfvW9X5!*=rb_lImruEae$K9+Esset$Cf-iw
z1unYe`A?cpdVE}P)X9|+eDYK&z;mf}VYbFj{aeMne6LSNr!9K~7&hH3l?d70^WnN&
zx?~>Nd^lM?n2ywYeg?6;H64{))hNf%rLjFIB)$JM?etZ@Lxl{El2T=3$h1{|Q=ssB
zK0;RD8t6l}*5UnrjYS!K+8~Dn{e3tl{=%(L(7U5m?e4lNAJ40@qj5bY^9Hx6MqSyT
zR9mkzt!u4A57gfItq)CJoi4Eo#q-b63O=^~{@!Bh&tbpt`xFlK#|aFQgj>lZ3=%yR
zW;MnB{emCOyqXVUj<Hc>kB@(SuM5Aq3N)R4>3mmuDYQ<~aUkpY3`ZVi(kY4Zd51z(
z2XDT3^q!xI{G~jRLy}(c<z&wVu^B;-QReN1PfUd3`0`})QrBzDI{`&}djFUmI;e$Y
z<bu**2^YdvscAS9v?wYbiA@)c3uH8dO#83Begs>V2C__<B@d7Vo30js*%O<He{PD*
z24R2%W?+0`o_aZH>8aUThlfQTo*&RG3bWAVXOx6-uf;!m%r!>(JE0Pv%+xx(*G#=9
zo4Y+6aeZDk@|4d+SLy0E!k?YK#fJ0*7;dIyd{|3HeITbu_Dy^u`ey1fVvU47L|W!N
z!A?!QQ-%qpdU>OZg}dGzeyR^eHTSXqV|qSDh3JEtYA!_*xtIvAK`jbZ)e3#<(ZAF%
zo}9u7O4xN!z}qXrA-v{|PP=BAd|JT=JltEj15?juTn<<B?QtB=_we6aT$pt#2&-&4
zneBgibv(6GoFQQT#H(#RKcw<xXU-ylQMwV9z~InwJZWLFx7b2dQBpEn^nk!^Z6F&m
z_o*7*4tf`U^=ERn!Hvj+%=+{0mn{LPTMF7^Hsa>${n}uzMJ#LhCrP(S0l9PBf$&fl
z)`M<xzQgguWU9*QJ?;lH+)`DNp=bM%>@$xRUq7S`3H$b({G=w}*F-OLr+&Qhcx!5I
zF63m}C2YT)U~(~J&x08;qC%6@<odm0vh`xuEt9bi=-B1F-dS=*TQZ-dz$a&1(^7pf
zt}iD)4U0WKqU(GALgHf-7go+2+9MJBVm-xbw|NsWH=X<Y%UpDY{6FWrl5iXKY@6m#
z3`#do17oA}4`JMAUrv9d+>WlS=FeN9zpCcYzbzb4StrZ`DK>ZFGz0h+bJZ&p6)FuY
z-8An2@I8dgeg?H=2wFVeYrUvy*_pD5Xkg}sS`pC-)oZ2mIfPy8d^}MMH~C<_@c5%=
z-lIB49k;$T-jkCgrR!<v?~w;YeF&Nj<FQS*sj9rZ*tzxL0<Wv{^#Wari#qirmdOq>
zB}-=uEaBYOs-2(xnHK0mQWp7X923%820Zt{Wbz{c#=!yzycwOF<yh8Pa<ej~#D06l
z#%!w#goMk37~-0ZZ)4NC(DCh<z*oD8%yMt+IBSVpi5@C9IWOI&J>5&H4dPghJ-B&t
z_f}wNUiBUchKDgk_%g1mn9p>_JFO8U;ny%>6&)7YNg4RwRg;kF5!>Q(*eC@qIs2I@
zoFpI&GIXop+p)q@(Kjh%;<y)<2^;MJE22eO05}9bf<D|^D0H4H=gM0p0w%e!z&Rsz
zRb-kN<#vIemnl?dt=}4Q)%mP+Rcm4=veWrZ8PJ*a<7^Brq;lILN1J7GQMx#yrJrMG
zF3w(jtjl`+Fq1)hNEFArPc2uy+yxR-mVP=#Q0(*Uk|f1hHdmKK*)=-w`T*QFsohLH
zYIp0tiztoQ6Go-;Rmvt}nTUIGa&wl5QMH&ISn0TsQ`^;v`1guO;<&0P{Gm$Qh04@p
zTlsR-xEt1(L7PuZApUp(F|PB*rY+oY{*w=WqJv6E5EBWbl!W}QL}R$~9Pxf?l860N
zFmu7LIn>j$ymdUu?rA&BP5LI5TI99_2;F!P$Ua^PVHgn2Ekm8NNVE#{t$Gd0koB3Q
zbZVf?*y%ery#0B2^|1~U!QA^#Mgl)|%)~JxcIgsA$$tYgfx_YL>c#UfkPC_gU})fS
zi_kWrT}CseTG91kP<tqFf125j_KDSgPTk!_J=H09r}#vRt;zTC_UfkfJ6Vuj<=-j8
zuZZXEg>NqQ+8oE<=)_zIw_W;1@syme<$4XwfA$+Z=w`LyQMflor+PqK{7j1O$T?Bw
z$E`a5tx3@OcrOOAQ6&|)6c(7$D5B%dxT!(mNham#G<P06<B*wM^dOc2q0Wd1%;kGj
zL086{2p+gZC^GX~L3ctP1b4JEvLph3kDC<Lx+15TpuyI2BF5g78$@)FpEo~)Ff&~?
zk=l)X&hjPCq`@qyw3GDFzGnmY!i1*Tf3pWsX4F{f6Jh-c@Qa3Lt7A3c)-sckb1J3f
z%mLi)Gd<^2S>VU_EVZ7{lPNcn0h$_vD$!r670}g81W&{QdShzeeQ2GH1iR?u%g=;Y
z4&!jiX?TQ1$~{ZL;){8mBgCqasrz|(zCv&G!pzUU^g5gbJJR-7`6SA%+IcyLy6fSz
z(zfT5kOvsn#JttBKHRQPThIzyyGwe^PR#lZE4=^y&Xl0EUO=XdW^SZRkCn*4AY`fH
z(K~_0kj3U;nr?q~sF_IP%N_XB0Gm2FM)J<cLz#CMKjMWOr)C^lx>-V|S6IvR3Ov0o
zerDQi`{eMOR8~*PviJ_Yl06Y?s#ChT^b46e?>NW|Wuj;r?J(Eo?ru?S&i<jjXgn=u
z{CGGv?p!GXbOyO3*oBZN`-ni7ZBXj+idiZ=H$q8U`Un5;FpKIF*;|$b-q(RIg2?Gf
z98n$3@4E6awG2OLN#6&X*(#4R&2?^UX<=?{p0YU-t*?#%i|f?X(W&k@>%vZt$LE!7
zqU`FYuZ<pav$}V=-;FSFQy9SJ?v_%@`26@5=_=EDXW=I#@kx8)Wi+Gbr*@Wz={Guz
zTR6?#*p<ZUbr<BUm+bQ^;Bdr6eg!RF*_X~grz?=B3Vrsf{z1rEw$t`Oy5QsbYBQEO
z?f6wyU!^bAJKgj?Gop%X&%3|&iX*C)B5uM$OnJBu(@m{wy6t>pn#6U-c)QQh=XhKN
z$<q1{+f{mL)Aes}pPtt4&hp%lV(eA}Z?r|IUlGT}E1<yiz-AWc1A=BdA99`UyEu!+
zk$;4E&?(Ra`ukY)ehpnSr~)jErq!Ahy%Th0$69gaQFFjr%`#6?5m~+_BblNo?X}1h
zAk7hp!iDs7*##$7kQ=ME#J2DZ_I~u!Vxp#=6Ai{6E_S89#XEk*rK4j9)67m%#<S(z
zT`hQT(f1E|6!E&h1j<0T{5!|w;LLP{Cy+$u7tu4w(O&J3M?QcEWFHHwhG3eWV0@G#
z%fr_{aDK-9hrq;TxzP`O#S=!r3sx|~{Q@Lhd5`d?Xx-Mx7V8!E#7~|_xI(8r9xe!%
zkSQK#>TB0@v?!#cw<Ki^94}M>Ux+HUUuexjhvHeg?vgU?j!Y)?eVsU<XgnpV6J8v=
zEcilE?S$%(5+a~h;km$P+(bImGNZfdV(s);Nq;a+9ORQ7m*WzOa10wD^!DxV7A-sB
zUJfr`<5qU^vLUMwh!&f<`lfzZe^Pj<uZF9iS^s2?8HzCrC#-~Q@0L6;_%<J~NEg;I
zu$$aj8!vBp>e)lw8Dy72xjcxUe>&B|3tclLxXZ=e#A$a~5XK^6cydpMO0|wuqZ-e<
z%je!>D~UhCBmHGvWa<jE7si<t|5W#UD+NSgZtx|aXzkb(yBl}1YA^mAUp``Y4e|^>
z4@VCcmte~6c-$frnDjbDQ%zA~g_yXSf{e*n6K_zn#WYy$2tRUNDP0PXu9T(>?z=l2
z3<qIFQY?TRR|nHPoUPD&Vwaa0ZMmsHrzaN<{rcoy^8cheqv{Q@%=K%L^{%5;UHb<d
z!@z*^gUEEqyu?tUqeN+2ZZv-ScgR+APX^g6OLJfOmO7jq{wym|g4KiLAqq6HXcxlt
zlkDJAqFLNttg0|BBCIr+f3{jp3B@qc_!3vQy*oosnN2bzdrH@gCayf<4B~&xPJ1eS
z-@M;SK<{69&z445?KIVOuDl&(iNwHb^wa9C0iBf<oi$|0f`CMuKtP_K`?RJ{TG*|z
za01836(g+DTP>bD-RD}Iym=s~?E!3;;w}H<e9B}bdAo)QgPa~@1;DX(3TBC7gvF*w
z$7Me;EiN(waMXm;N0clby%YP+@z>sbu8z{uXukc7DV0`Fg{91a>ZL*IBP>1Ekv7mj
z?UWMxERzYFVHF>#s8?NW-ShEC<+<-xs3}{!AB|h1twvB_<qHMf?t}Pdm)j28ia$k&
zPw+xXbioLo1&M3caZ!&g$0C;W@3(&*o|e_9|BU57(cJ!|IIruG`L~*@^f7991ZiZr
z@SJYzLnPu=@>ilcHMY?z341~NpEa=9y$976MG7z8PpC)*d~eVYXv&LE6fe1R1f1*I
z4g9d<V(c9~m0lS7ON_bNqMDtDH{cE_f*w15Ypnfo33(fV#IqZ{X4E-dB_yid8uPFT
zFFIny0V0%U#ptl#%>47d`2kIPk5hM$IB(0}By)@~<c#C`i%t4xV?D;SBf&zBvBB3k
zDUfH<20Tg?II>!0&9bN04QSjl#Kcd_!S%4|3bB#rVf$knmmlzHL(DaEGOob0{^1M_
z33{Y(&T7(6U;!YdiVe1FkSgwlE5gzVnOgEjTPJd?{3HbP(e2!$`STTR+CkHk?|(D8
zsGdIpT5C*#ugx5D4!+g_LIF8nMuK2B_$c$j@W`Jvb4jDjA0y6!4c6TsL*c@J#C61+
zOmA_)T=ZI|DM^96-Z;kkd`@eudsAlmEpn*_A<`IS&@GR-xhX0X)QDz|(V~s?eOoTC
zmG_|f=0S4Y(IoY;F5quat5o?hKZdn}WBg0>GNUO?(}rcuMFJ^Xckn$ors1A)T)I9{
zMdEMtc72k#jePIS>j))y;JFoM@cF2BS|E-09H}}g0u`%5|B+}IdAPwO2x5xLa(x$#
zzn>!SK0J(N)yGAOV8y_XufLG193Ms4vXnE`FMN2EU#5Opp?&;MkuSv~r0%%*3rF45
z&}JE5UmPZp!==r|CJC}X0x}riwV<AbUu_(Hy`d?r3xRrfz>1P7=XrSAh@oi2_NFpR
zO51zR@DbS`cVdm{OPnVPJm3=txCHh>p8*f?5Z;CenU!3FfPnX9Q9eBb4Ap^zj7Q|Z
z0*-D(qhpIJaCpo!M7W(4Wk>7oQ+?%qXDM>CkYA%~NX?OGoAJTjmXsI+qPC`0XpwF}
zs_#6_u$`C`O?C*Uk+Sk)hM8j%3<z_6STV}`B(2$p5U>0N)C+KFeR#7hrcaJHYhM{e
zFH2U*KYO^Ku^(Qqs<{2(8FUS#ILnjX<y^zqfiMgvuCTxvLllL8k^n{wOXr4}to2J;
z(2#`Ay&P|rL-8;4E=)Gm+cx;nC}X-|8zoGD7{kN$yX3h@?O4v2qC8|A?dTttO!Y0C
zRjqMRt;5~m;rmt#?KoA0t1SfIaw>SDKI|<Qwm8&t%rr8${xs@c3CD%~KCEDBbK9>R
zI9&Em^-J;z;r<iD`7gR!D_K?eOl3XsyiG_-Q5a`tq_+X;V~v+LJEZAVEO}ZikXZ&+
z7<73+SMtbXrT0!+#-GC!1GMK3ef<Z{DwdciYvR5O@BL8XK|v(t;2R)}`8@2oBS$j!
zmGg_x`6Y<IAh4a>s4d4^knhK7CaDMC4&3<Kh>x-G%RlkA10HXF3MB@;7-B68dXF$b
zCFC?xRY)IQRTJ_Sc&nbMS`6$RsrSW<NaESbTe=qtQi(7wam8JMi8{i0pzn|Jx=;6B
zgNN_3TQ6pbYWYh9kyV0+qoS1MPv|Ic^)L9WW3A7i`TIrrSiX%<2XX@k*>4+(Mv<n5
z3%9-t|4Rl@LmFoDd!bt^bmtudJ@yfIyh;;k=Yo+2tPFXQ&^PZty<{xKhX2?90*DEq
znw205#g~DJ=G4f|%QiZ@<Xtg=L@6%uPXeWWjnCtebOn`yIa)u07bo^3GCJ`2kgD6B
z_UxPXk4EXjY3I(Tc@pOw@P5;=ZlR68!?*3d{#AN~i<nkltl3nD!0M$w!*=7hw_BY`
zchh9zztC@PW{u9|fTlgNhD-<(N{*Ts<Vj`1#b0@RO^jO(h?!nwoXd!cRfeiP*?IHl
zx>fa&Ge11O;F1;A7y-Pze(HhV2q$IP|57Z|#~NRc7VF1B56+cWf(fdxSMREqI*(yN
zvA>DDjqI<PRnX{hUgNHD)QCc$=|~~&r?1%<@ffCVi01Fueil0+0dHR`;+4McEqj;%
z5(=l^m|oa;61*Ju#5l7pOm>-~6fWmAH0{TZbB-z#+M_|IMX$l)ZyI(b(hQ!Dw=?Zi
z%h0KcLtmk1uZSsjU}%$A(~G%6tPgUb&@!NshGqi}tK&$4-wt9p6Sd`h9D>W5Odxvw
zxFbLLEKov)hLqNYDPl6q5Ai??>K;mE^u<h4!v4V*2^g*{6}R5E3P6)y(5VeYg2D4}
zxG2_b2*fN<Nkd1l;MYtetSF4MWux2yN#t0~lSTT%y+kOVi6_#BkO<%*ipLjMhhD>g
znQe4n{e06lO-cO|G4D_P9=eU65_zz(_CyBPFW)K$^vb(4`{(drG&w@vXj)u)!i4x%
zD@WF_5BDKKlz6{t7$?=E2ryGvU(D|taJHjxNjrjRlgf`qlmm_6doYtdtwG>N;!HwU
zgL7yqQ3cVZ8-q+gTbk4KF_NA|V0uEK^i4JN%>VLsifI91D~%x78?7%j?@nlxyML*T
z+-Ziypp@+->_lZ`z3pNIPgO&LUl9R05)&mcpzg41!dY2H@t@+7J~H2bwV#&o_St_7
zcu3p7HsA!ixXk)Q=~eeZ4PQ4hRTK4z8oQ8!E4wfR!d}b3k`to_=ycgXlkdldaclrK
zB=Bd$&6eH!cHvL4iNufBh<AwqPI)$DGV!_Olj`kk)nNOy(Y*e0eA?75$98#{w1@g@
zYDl|`l5kr8T?HAiDAnfZ=cFP>E+49%0Gi*t5W`XoYkA-Ys_%J;-#wK}x}tQDc3kJv
z<gnYn0NuD`mbnrjM{2e+B4g&7A5#Zs!>%Ol5*P^2YH*~2toint2%YLL1BCevW(?WJ
zsWrMsp1yIbIO#6K)#iMCD%tVtmNz^6bSe*fPlVF@A_`6)MC&a!6oRDGYmCya&2e@&
zQg+HpnTkVp`?3~}4~Uj@cD>Qbb^PlO!+ExxawJ@y{B>^DErnkMKMj3n`avp5@lUz=
z7sOlH&C*o&ua7=?E<nZIr$xl=x`^v8cAIjAtiEI!=_z~<qFg1KiYuP`=(*t9m;P|Z
zIrC_=m>y1i_SSjppIUb{_G4y^23+Y<U>;DhHFmTc7c<ddV2$e7wN4zK#oy3Yg!}?R
zv$im)Y^jwT3?Zb29GdDuJaF0*6mo^@oTmh=TvLs6y-hO(OxrqMob0Sk+a*a-^PAKw
zISuCg+&mb)U1|Bb_VZef+rUELl1EpP6SArN>hfs#4erA4`1RTCwUGy8k~n8UTkv5o
z_h1y+VK`UAWNX-ku@oG_x2`j^g-o5d|9iE>Wf?dU?k|s~j#cWsbtdih4+G#T33|`&
z?8Rc1aCJ;YfHt7@_SnJ(=Qz-slx}&G^#07}=}#2iq4wwcx5s=e%6Wy_ZmydiHM*O-
zjg@Lf2izPDD@9MXnAx`6T%8tJ{T|2`e;-XmOY`=gu<$Sxiv(8!!T40OnYp=c9y7JC
zN2J<Jr}1Ul4c;9uH(^+i&!t@r7~kUM^I$tZn^ONZA3uY>k0>UB1vb}IS^K~7;N@w?
z#7%;%S7pNA@oSiFMaKvNg(LZpD^QN(^}{o3Fp-Lk6J^s__kAm_4^6w!t>5E%WDWg`
zF!lElH`gY)QXT>bKMPY!pXg(puYUI8>G3azmcu@N6Wn%I;*&5;Uu^vT#}^+j+-j!3
zHu2U-Q1uScWL|U>nd0;x+?Dv%_v8Yl>$8$aMfV~9hpx8_ild9Vg<)`af<u4+f#B{g
z2?-M1f?IGHTn7m@c(C9SB)A86cXt>dSa1k#U-P{8y><V5RsCaXs^(0e)4Naaz4ltG
zS>n}%y{~LbyswYv#O@DGkLu>X%pFv;+b7qV4dNe!(M4C<w?Dcq<Jrv=#a#jjf$qV4
zMZ?vKi?R&>SRuJ$QwU6u@mR(B+9n(NH5w2vUk}j5ELdmy-C27-1HhQ=QT?+0Tt)lS
zS4!6zRZTay`q?@gt%Er|2UaE)kIqmBxb_M~_|T}}S{R@8Dm9W+CZ^qYphwm7wvtMI
zg!6@g*A2O#)nI{#|3!zxn5WY1FY)32S;fEdEM<XGfEtXkaQiB0@eTE;7Y%%dvRvR)
zlOpGg`3q!8+*BidRNb)Zwrn>?YME}#`S<F2-Cs}C`>OVL)LglNaHJ9wWj_Eh>yG~5
zt?7f3n)+TXT53iTvmT0kbYJt_)d5h;4HJmnTFc^pP%r?h=&t)-XS7moVU$t(-Q>_<
zy~^}|^29Lc3#mscKU}El{t5)d<#F@LQriAX)kmRnsBIE((`A253MqFKUTxgXw8;@(
z`C`#p_jR_K80bH&IuEnX3GSwwR_RNjcy9KD*CR9nkc{?LUsMgB-ICdEx3*G=&+Yk@
zbH4awn*`g@u6r^bSnN}FnC#cAE6+lSID#CkHkNG8xHiRdbZMrJ@<p=O^~u4Xz;|Cj
zjBsuneKt^;HS`|=)&U6XYX}sEDIZuuhANQTgL;jqJ5UOQ`z}WJta-Nbpef0d+50rB
z>1I4{Tbv%~C^{<9#GY<Urz;G_gTm4vE*3B&Qh^AheY~g|#)istx)_nEY`qmn@}^%$
z;B*m$!l`tiy@2HT+i^3T@y82HV~vZ(yY6!l1>v^-;;|7TwGUu(gjK#5yW{*UTIIEj
zAWpj6xT~8nW%0)uHWXE9mx0$5ae>0Y_N$l_ir%Dn8W$@r<D>!|$k;{u3srg63za#m
zR7vh<y|0!#ocPoVK2}{&xQuYRk%mE+4y><2*H?33pJEd%?9^QHC58xLIDkGdUl(48
z0L)CB-}G8+zMcQ?J&9xd;Q0HwQ8d0o`z<0*00OiC9(@9^LRU$1Wdx;<F!~>YG!x9m
z<}5etj;6up*zPca!GunEm?sCWEsC`rICJ9BM%gwekHi*S4Qfbc));bY37mGy7}LrS
zay>KvQ1@0+5sykq_v#}LWT7v67WIsi6OycI1yE8^uUbK%J@c{E>vH7z>AGD?mTt+K
z_?@;9hud_fzM2)IX5e!Fqb;3@V(0s#EysHIy@^hB0Iv4|rVKz-nVfrL_0u(?{Hx#u
zQAq#WfaQrYyW|5~#p!YH$?~xP+BGU0ay8xyO1v*Y#yD5*uqB%cUJ?`B`oUrkFNuY|
zKPM9TOKp&$!MJg%Pb^2tpf?8a-@Zh@HKyh1I;&Q^1}Dd)MZe8UV3jbGkay;S(FIip
zB+KFKx^@iEQMNhGVG1v_W`xa3F$l#&o9dPF;VRC9|JH8Z$hk>h?@#v!5?C>AXm`G#
zon2dgr7s|y6yfCBm{T~=^l5S3isD~7I2@DX8UOMG?5|}&Ff16fuT<JC^I>5B)M$1)
zq9w3c@P!VTOY-wE&R6;g_!FlKHMew#+hSN)>LoBL*Ef%)W}32{4Nx}^FYe1(QVFDM
z0=Lm&xKx;sZkQP{VMx)>RXIz9ewVJ)+UD-XH_FXh(ZYwE8Ha6!PMdhmYl2_!@pQb?
zE`TQxFEPc(d+m&WV{IME6=Rku&Ym^HIgIj1$2hjiuyY_H&9dl3rUt$Ce<s@+2)iHK
zt^&9%WKiF5X9CjXdaz`wE+0){&D-UnflZBSK9jlr3Y*shnBt)^>hF!p@lQqn{Op<m
zX!~e`Jnk=s2#RhJu%-QBK1IiV6ru9Vr^Y<GP=e66dmT>xrke0<#gR!<U4^%1fS4VE
z%q)CM8?3_EQ{xFn$d3;vtpEI$ALA)NkSm04A?<5BJ*)$vU@hzfMr}NVqU95Wv)Ovv
z(o-r}LUYt`EY-I-#m2toC`V_=Tvv%2Tb{ID*Y}${Q<wr9N?Da@Kk)}el&EOLX_t7c
z47KP&8Lh^Q=v!=&QwfaKElmJvey0?eBkMvdXpK5KAhpH2uRfi#ouVYdINg;OQ`#H(
zDilnsgOcfB{Q^EW{ry$)n`{bS#9+`&m@t*poSkHv{Fkvvd-0Fk=)IbdJr~IufsJr}
zs?MaLLKMV?Hy#-TQ#Yc~W`pT6+b@<eH%zA#*xUG2O8>hcZ68)+-HC4qd19(4#%3&d
zICV^U+<nCKm5O_v-AG(9%ssy4%Yvem{_JM?+lv=4suPAs8=T6dc0B02QB1*(Hs-qP
zV)iq1wl>cDYBKuCY&<8Ye(vS3L%rHjS|I3x+;10CPtx)H$3?Ff2(tDbwv2(kqC>0s
z<?(8>#8;tT(5rJiB}iYG)8pM`8PGgNwr8uRINFCF=~%o;=a_BrbU8Tf#%Vb9QER4H
z`^e$-UOr2i81~LfE=I8U>&@u~lIs}*B2GoaygZb|czon!rZ~YjuPgO|M~P`G0nww|
zWkM2Vv40dzA4tY34D6XuQf@zkk2T7Gy2iSo1GjdUxjtMRlk1sC0d}uK?YV{=*>UEy
z(@wD;+Ly|x@9Z{dh$2va_0KW<Vd3~n$0dD0W!vv^R*RTAj!rR^uJdG-5yr{EKI5t2
zwj&%$?E`fD+$JB))L(6}l>Okf(M^GxIrRUYkBxE9ReN6;v;KM~$LV)x76~z))nq1>
z1WK&I@^CVHLl$y?({IfZ&9Xytxe8A@Y57Ec!T-0fmx;mlyTb(N>`l4}{ooN(4N!U)
zb(B9nV(+o*q(-7#s*q1DuK#{6j(An#JylS(WgBSO5uh|bb8y|EuE`uM_c|i_M8C=7
zNPjzemgEhYq@gmMi5)8>@3TT=|C6o`W`}*ewp!g+h=01eoHrzw@sr{sj7zU{$VN+*
zBwy7GSQl0bYeo%D$4xmLaTi$|@Oz{uADdS>C;ILa^2V|a$ZWP&Y}<9h9O%(`1Wb1J
z@?!YHbjij%LR~BbFftuzHW7bGL|*+1P!_xGVM7?p5_S5U8_C9%UNt_b&C}LIAkBlx
z*nVm!Uz+AxEe1v5O;<-OocedPJYV^4u0LGEhZde{SDk}9%lu40ASkP!b(jK)BqI4e
zL>*u;am`v9?fyw~pM<^%P$H%57QwX=t?bg72UFpI`~zhGI;WEfG>s1-<OpR!EXkJ{
z#@7zTk$0cH!9H;8lYbcaTn^VSXOCznU57ZyJ8#jJr?_rp6(|}+iDSbb#`VMDbfKWp
zWtZ$@AG12pD_e+Bm-;?|>!&3UnRW8oJzbemx<wfWmVk)bG?_t*F4jmXDeL`go@`+8
z4V&`o1f^Yj4W+3U#uDDbw~o|#j~y#X9o+#6B*O5KbQg_uy`dB1NyDF;K0W*Hwn3Bh
zV?U@s_8^ze#DGjpM%)$@0k=fsm^B(z6nvdebTCc&_-(-}9OApcXNXg_`03P8&gnE<
zzdM7_^2+d4vF!<AQB>}2QE@^iWzMh8U>a%5h!YM=^=NG3^{i-}rJY$VL*5j9vps!I
zDS2tF8p;gmS2eBCI_Jo%16N$s1R;xa^%r5G)#M*LxKbX>%3IDq2%}pu4n0ewxZM5Q
zM;d<?xj(EqG8gg#%OOOmeiAD58DnAySZuyIRFy*_b^25Deo;)0*qUV8N}L%zq5JUC
z(VxWn0SJbXVfJ=gd8O27^Rp^75)^CU@8GYVBOB0hnqHFCSZtf$B1>!l)HtXPL!-Ew
z6#+-b#NAd%gjE>G+j>)<{*p$9Z|E{?;hIDGLRR7IbF~wi{Uegd>D`JkOtuo+2Yw8x
z`QCp{osKBrs<?7o<*kFJErvToeCq9&a(%)3IAxz%dds4VQL$(zpV!nCwZ`4ZmTEc5
zVnPvHIJZKdqBAJP{aQj$vqdHQQe98X!v6h(E9i6)J332DR~<hWP>Lsh&}I-5g~N$!
zxIV*as$>-#W16^0p`*fJ96zo43={-yt->=mhGMi4TbzR3tE7yi+Qi9}kC&-JNH;-D
z37}$4*c%=rkjHy*%wj<z5(lCiy^ev?+W=EcMv&NtU73!(H8mp<g_*suYK&M_kqUOu
zA$2o#I@%b(QIaWp!!@Nt52gwYnun(}S^l@``u+5pd$0LGep@R${-Wcx5ef@*d!-wX
zYvUUe%t_%}9Dlz+JuY?{B*DsEx;qE#wwQ$@`Y%?e$|U5m<5o%-q)BpV_K!0&2j{Go
zGyB)?SGUBB(WFrpP|fPc5dTcK?GK%e>l%$own#h-pACb-Dgg!shY{a@eu2OGDT=(u
zM2S%Vx8$0QW0cIto(^V^5AYb*qx++S+2Tf{8uBQYMt*kt#)CivYfrsT*ZouePXsS)
zqhzs+KGTj02~92{*l3T$1pETFSt(aAAXXBefMBw*FF?%r`Kc;QIZTBGtaC19`bZd-
z`lN{ZEAb-{22BEEM8D-G_E;i;@^Hko%Kqj{KtR-{`#Psi`NDVVEn-mjbDwpNrXWlz
zJL8+CMn~}*tLDc{Ba!BDs_SkAP(QNMiW)O((tPMfv>$ct8QuiN#2Fb<*W$*)Bw(Tf
z9hekMLoneKDn^jF*2Zs6DIgx&{5^IVp!8Z?nzUQ8A3=^n7+#4Zx8{N9)MUjT<NL6c
zgVrfI6IvfRuF}}1C4-JNj~kZ<atrRlsY1IRWZ%SK#{sP$!I{sc@{B(mPiWMnkL7n)
z2vd;kbdx2znYU&&dJ?FL1(z597Rbw%Pp&cmiq|6(n#jYPUkeho-o{U-Z?7$dHvzc-
zjk{PT?e{bJ3~riR`^~;6a&i&w(M-X8%y^u{{t7#f+4N57l01!TUVI;P-5>?Pvd6gL
z>U}oW0%98i1^EKG%>5^0dS|@TWzF2K@_?~po=R~?DF5_o$FLFU9>yA+Yp18tyVEYn
zyCCOv_=cePO^HpFBUDbx>W?+v6R_UwScLsv$q9b+ZHa2OvA`sR{gA)RY5*qvQ3J-v
z(?Ur;E_`Ryo#8(8h1zXQaCgDM)+p`f3I|&S$J}RtZM_`#p)bY}p-Vb(W44~ekXrLi
zI6F>8P7OfG1LKlfuOI|-eF;0ktn2Q#E6gsR4H#QSFnXJ9I>_cd)^s^6kYk%~nD|pw
z6eo`r>Sh`5DruS=6uWLy@#L;yZ-&IRkC9&wC>~%LAPiL=I`!9pXTuk2;{92DppYzl
zj9_iEhEy!;R-3&aslf3*tm-eV853eTXD7Ylvzq9T>fy2#YLovBl=oMMa&5GjF>_2F
zM|u@sS`SHMePTG)<(w`?`SpA#7c#w%t(|uAO%YJAYb|BYc^y48(;x3R##(&-#K|Ah
zi?Fp_ZU9wx?I{T^M$n1c*<B@qBN^$Wsrwnn1S#T@W9dJ%71#w=b6o{I^x+YuAgH5=
zUemW~OJ!Ip7J6~4C@os_$M3sm3CgyYhM76x2<o574Q9tD`OEzZ(8gG3Jz(;p;!J2n
z82QSjnVe@vS*3P9RMwmK6}93<i*!;n1+}PTAsZ0f083${3l6|?fI8o52_SA*wvR6?
zcFafQJYKOhr`vUA0d;wrYaglQLIp-Q_xzz&uoHPyOalprQ9y4Pi-o>!N|e>c>#i$d
ztJ?)kvZmoO-MX)fuBX0r(AaYg(|o-K`>QQ+Vm5uN-PVKh=0kaZ4jfk53Y-%3PvuB8
zaQ*EDI3&HKIM@h&WTA>2pYPBd)0gQg_2FC4NJ4BeAFA()!d4GZ?lp{m%yQ!%y?5mi
zqI&PKbzzo|c_sei>84yF-zC@giOI<D_ha#gIt559I!~vj8)PSBB{FIfxUfgF`aUj*
zhNH5=0Ms@Dc;+7J_ygbfq4D}?vl*WCHb{3&)=;Oyqm22v1RKyiIr`wQx!8zDcP|n@
z42B*$$7A(#Yn18ERzf*V`v~c`4vqw{(W7%kuM%QzjP8M}EZ95-tlGRE29hC0pAS$l
z^LQ|%VNoMOcl#F2t0*05?M62}Yvm=SSM;<)bCH8Iv#A$eji5l)=MZ!{6t_~EO*_er
zNMY}6lwyk!?m$Z47cqZ7zB#4aLu;w0P?zXW<c8@KzH22Cz#Gl0q4wecdo>u8C4e$n
zRE93d)FFgnr4rnS1ul4Vo4&pzl)PO!(9XUFu8GN9S#MvG^>>|UyZ|w-fw4f~B*&FP
ztniS_e27Hp-%E}A_=DZj8^#EDatQYE1%IW!=BBjY{}hT%`4Qkyj=nPVF+S}u3_7+I
z7CzWQq;4yj<Fwan@u+&dir`7Fyg^ucPm-#S<RRJiss;56JcZ59w@FU(Yr(weLO40e
zeVH$4bAgUmgl>-d$0z~O?EQzH=I%K#R-;%gA~T6baDIcigs@F^lyoOr!0x-!4S%+&
zPpSxJ>2M@byE4DPHCR?EbRA3pGSKt0qr5~{fAPCE`tJe>bw;kGgYMkc9~h=ccA-%?
z|9pXB4$5sR1bln*m0fHpR1?6;T!37A`2f5Scr+<b&71yKso!O}K4V^fjA_?qa1WOX
z0JyGLx>I5*s7!@_v;N>D^SoYlCl^}_t7*4Rzl~}TBoRZ2o|KdO5M*dctZh%$Dptz1
z)LR7egYk4=yrOKK3{aI8dKFG$87_`U98}nu6@aJgwFm~$3dSzjEz>{Pp^U+GOA5m3
z&k(=B2{6n!H=klM$GAHQb7i-V=S8sxnbGwrsvzgBJlInqPmOvI^rOCbBmoGi6FSFv
zw=j6Q3+q?-)m^1R#m<sL`(vpzwqTDR)<RW3m5s8l)8Fuaca`SDNjhOyI*trSTHp=e
zI^wm<ubsCwFcl$LLLcW+dc)n2W4jUng1+Kvm=vf<yS@Dd=yajOD{>fq+w%|<Dyt|;
z;kmW`s<bdU-51i|{B0<K4Un*zp7ENA@F@*Q5>9QVD72xoE==uR8XGvqw=a4YU_2%D
z<5QM|!`L19t&np`r&o|I;IKn~Qy*f*^9sc{faDks81%Fyo(i-r2<z24>f@Vkn6O)5
z{$ihG1yGLehzMIvKelwP#wM=A^S2d-H{@E4RaEVt=BCyC=iD0mgbtM=STr!xt3)9D
zDtf~y1hL*TLehoZ8|=p-ku#R-yNvR7reT!Th$(r6&*Fx^<2hsCp#CCjfyWQsMx=K5
z?d*9N7u`9zzBTU0<gXnp<l(ff4J5{^>&XGk9h?C2gKs(Gh~$A!0cxp)XZo%>jm<{b
zCOnia?VO5XP83R;*&(T)(TCE-*<2#2hqL4%G-PI0-G*Y$s7vqgg9HLp55I|w8i1A>
z(Ap4!f84T`$gS8x_Y2W1t-y&?H!N}s{d?>-CAh_zacL%|qglF*MD5LW$q?7@=6ozV
z)}P_Z07by!oT<ZpUDAlnNG9yn8cxTN5yb$~mqlr&l<4(-nAa(p!pyhVbd-i}Wy8?-
z*T=SQ#?zA-$DM;-<mS%MKZV)c3vi$9TqXkMitU5{j(-HSsunw*ENo>K$@D0=Y?prn
z1fuZ_HeN>-BJcr-aqFXTp-}UNsiH#8%CZ9E`A?DvQ@vk-5x3SCr~3j_<W()m!D7X?
zr22O^h}#tCdkV+!R4BHJB66X>@r(qkGtmFQx??Vq!`7gFS_nAw@q&FKF*KA8VVNkt
zkojl#BHG;1uC)ZZf(fO!xN?v~j^#QX^qU+Bd3CPI;Rq5U%<FQSvB0^5ZuA-PW~0I8
z5QR7ex7$qYH53Z+!AP<^e~Qy}fX>q;Up5i$gTwK`gYMKz3XZN21(FA!m%S}nI){K(
zfy4+}dpGFD3|qm?Pow%Yk$vCM*kQwXO*T_S>Vr50Q76k7&@8qM_zS={66o}~9oz#w
z)YBjg{sQIYRt-2t*lu6~Y!uX-x37S7<#)R2iEs3Zy7^KJ$yd@Gcko7lNbc$j+(CzB
zfRZB;Eb;8yJrM`eOmq&u5Iqb0@>`bD4vx|*)|R*1HV;a^D0~0z%^#Fn(e@OQu1cnG
zFHfHxSq9Lml+*s}ds>%t@YI${8y&Q`CaQZ0AUR?aLl8jYA;ZJzJl#c9>`DC)JR{K?
ze0{`)5WtNL<bezeNr~QThMB=Mz`K0kTS-nmx=8a9xjTq9@V}oPiCpjIry|eih;G4W
zycdt+*T{o+$p5_GGb;b+4f-FOOFBYWh#LVD7=>zVaMI0zsKfI&X$xu#N9L9{JmqX1
zOwUoU2yLbI7gH%U3C3PFHcEeh)H-J=qYYSBE9lr5Jy*J8khFRNy(MIHhQpxr4Oy2{
z{in|}G;;g58Rq%R4vcexW^2KhZkj-+4Va^XJCc$VLc2nZT=gd~_GeFssaeu$6V%Nq
zwtX2JHv-cdytlwhQN|)@wu4K)c$^t9Mo8Ho67XthsRL*0j069M(&&*rfKCG2v0Eq)
z*G<b1SA79Xdv_wBT5`FrNf8pKEn+_Z^+WQF?+x0iKWkz=IrB4>NMf$sN|0Ch|7??Z
z=q=wgxyTZ+`<kC20ny7(Z@;R@&)k!0BeEg;3C0xt>L7Xj*RYcK8MN%f<47U@g|Gto
zvcqxWResmsyr}~~)=30=-yyH~hWyeR!nJ_qVdtkK4|5T;f-~D7GP*vh+Hk*b+I5?`
zYOLV>R#8gpc7`16P=B`UmD9!kSgwBgpI!uCHJgy@5$i8Q{NlFA!^RUX072-p2`6pt
zjhxb6G5$u)@A8zXt9R{OH#|=^eKhj+gg42L!E2`u*b*?Q>e%a`)UQlx3b81<<KsYh
zhX8$`z;*^yf|RKqBhd0P(2NSP4#a8U)2N!=eJ(tdp|JZTHm~<s8-`fxIfk4iCP-*T
zFmP2Yrtu%`N<kZ5&X(a#`f@9jRb}Q~ht4MW+e58Vk?p(e3y;^Q&3@`SJtV337`4pR
zg{R1Fv(67QHUS2(bx4VXDDD`aX{esT_Qu~ZcVR~S70@269G)FssD~ntI7UNi6AQ+#
zB9;k#!*gUt;eJm@E$m=c7})mKUcTt>urZy(4i$FYbhq6DtL=gDEwk>+a|{*%I#l}8
zE1cM8uO%1FcHW4Wp-SBxMI3+u$8NE27L(0+9c+c$Z|gL&M;?%%*@mnYAVl+HveAs+
zSx8Ts=a@_)G^M|PON6v%laPy*@~>8>j0)C37fS~}!4aij1G~4qG%JO7dbuVNRvL)v
zx(dRcSFY$=s;u!p7v^)o)>E!*Gd{CG-$#n>ddy0eMfSXl8zLA<(}iS{V2o9~9m>YZ
z%ER{RnJ2Rc3da;5u_Fk?Sk_}IPjkk9W~Sh@q+68Le~Yw$Dh9U;hwY16PhJJtTG%t2
zrUJDG-RZ}Ls0TbhM@b!@S<+E1%+jom<I!D1mz)P=iBFe2q#q;Z6N`NQBSLYZbAWEt
zxRB`MlHISts5jjsdnwd&7>PbOI{|Gp1{v}^qkPd(x@c|i6m*wpdnmtxWRB3yeE23u
z-j2zO-UUnsV85ZXc3b5~23Nicf>|B9-Y)BHs%fgjM`wK6xmU04S!nn}^VZ*=4JxYm
zLG8RRWb!Dd!$!Uu?XOV`Wd1T6{eSidPNX$?zV_zW#m~{5x~L1H>c}3}Eh-sk76JP(
za4go;EQt;#v}LqQC_`ROYNe7<b%58N@cSkgiJb_%<KdrmBViACFk<I!a-5>j7G*$b
zB}lJIxBQf0eGWOBVGY$RUPHpvlkP4TCn_^a2>UdLN(11|0u(eoAS^x3wqtU$<WRo7
z`nM%oUi)sSozSs*#AAXWzobey4v)9q1;GO$9djs16rOS6w|Q1xcHRo}JFKA`x8cq@
zm{>;|l3PqQ%PD=OaQ4|l1xVge)`F!&;USvPx^v)^tmCXA)dJX;F+32u)6^?$Indii
zOS&}5UnqDqq)0bge=Hj;G-mk8D({;(Plpz^!30KH6Y>FZ#o*f3$)e>>LW#oAxF}PJ
zqJcwwTL+944k3LkfD48-WCXr)ca&Ap+e`U}0#4zFD0gmW^8*d^4D&v92<m|v4I*8y
zIB4w;;I8xSTTN&{yY;q+P#B#mTVCe>VZpuapgs#R^3T$p6>D$nKh;^v6S|gYyaq*|
zc~KrBObrUu?;8OY1Suo3FegV6*ln2EM|YciM9$+92>8I{W7_c`iL9yC9)T+0_z|Jj
zQ3AM{flhG%DICU!>e=qjX436wjp2hhM+(@p3<J6U@EXVv!Ebzp&rjTwK*rMWf3)Z1
zNJ<il40kg>Y*caR?4Te|W9Tm|_=8TB3B>$J&R-sc9%S!O+9BQ&EnDecG~+llI;L93
z4THq-47XYG!eYpi6sNEqs$eYTKszH05`=O@zyhGA)v?3>#%F?6g}S+FdjFY+6m1Vx
z9H9WTmfvWXrvekSfA#D9zvtl~69;8#>+w`cgpe3qLcpu7Ox@fS<A69`x$CbGaODT9
z9?}3jbfJ3ilRe_7WUibgn64<v3jX$XZ(<rU1a5{$ImIZ7?SY)$U%Z0FVn^HMC5YpQ
zAVni>OFHqdjXeC>lAPMue3tg7KKNJi3sMv&ifNBSwBdjjYWhF+yw*7#PFB4<F?v%7
zFXFm@DYp4pq-{=6Z=_hit0r<WpBskrNCr1@%uzG$45B?L9lqroorU26f2Vl!e4%?Z
ztB%hq>mlEGWJxhPHEzx8I(I(>7?h>Kk25;JU7>&jBGM>B5H>mQ%(b4|TWbOmetlWL
z!Fbo)Gli<q5XcnuIpTXTx`{z=mp!5C8EqJ=Nu(DmqHGKL7uk3Jloe0|X=tbLr&bM~
zcLh1R4~1G2n#YHCjmOe2N-jieT-Gk0OjIW6nS`t9rTwCB7E3$%@wmiB4g)^O6^Rhh
zfoN$!I<y=Jjm_r8i0i3>3+Zv?8WXo|2_Zi8IL`X;RYyKc?MNnIwoJ`eiPu331!WVV
z6F*fXnaJGe&?z%}1N~+FqK7|f+Jt>`u|lM?isKqPUIQ5<7I%PK+DSHdJ>h$JvLwM>
z`#u&Z2hj;P0&<L98=w}qSyOexmS4v6+|=1=GL`UjDsz;00Pi?F05wcP8*zzlK1+p%
z7QUs5J=jBy8C|w(@4Iu$lCu@p3{r&LvZ}4ba3tXojSvybYd7CEL?;u>j1?<#hc2jd
zu#N537baaL``<_CeALZo^Ag}gT01a(jAZL0p+0ecdC*Oz1S2GC&^?DG$1DneHL|`P
zxwnfU8-FgDpp-z#nY-ttM1jORhCw!$Ch_%u%k!FpAeD47GJjI=mcNGy!J#OQW34hP
zM0<y6&z6a*&nW|{;)`w<>i6wVqAtgSBqN#GSks5>oHHT~s)-tviQ)9ovKBfiD@O7U
zy*f}=p&uoE61SOrS7KZmOtY|NR>?Rq@Wg4gXIErO_FWj+_00z&pQ4@hgH=2$TxuhJ
ziH)Zt`D`?1DMT5XLV`VbIB+1Xb<0`xh!bu<lEv)boPd~YyNMDNbEdgIv{w`;{Af!^
zzVNH4iGiDT>7_2dDN)(nB3vUaXCr-~-%qmnO(#(Oz9mbFN|#33{o#PL;XV@!WX9vz
zvP-;@lFgQ><)0{O&RyXU!?pjmc?l!hp1#oYc|&1-f+>#D_)vi9<46;aiJt!Pt&9Y4
zLg$E6s(*xvR1qWF;LXo=I>_xc+h|f7xO<M-+~%hsk=vj6kuK}LefdTbu8TBjypQ@*
zGTh8Pm-8fakw$Kigj0zxFOM58H!rC(dT>*Urdt;EVTXl2T(*|mQE@XNP}=)O;*Rut
z;z<JY!1&2>>37H>7alG=9%@EM%>WnAKie-Z-}~1bvKLh}b1}fSC*P7Jpi3N|<wDtr
z-O!mgZutDF976mcR3U$*!fjnA1kTa>{bU$~`UcHf7}_NtN*}s6HzkXwOrL+_wz)fc
z42y)VRtYYCb~-<l&F0pj#h_`EU$d!>HPn%PnXDNoG0v2`$Npz23Ej5_sdth5*N&8J
zLj(8tGP`#4GZYaaPSDTdRzf{zQF+<qG(z78&C(_ori^(!6jN@yikNA2yt2P<KP|oU
zXA?sVY-#F^v!^7iI2)1V<F**Qc^nM&#7-_EvM4i%b#PAcO#FlhEU^?r2YY$AiY6ny
zx-99|Yq`U&N04oJi=hPA0$GYE+lq{lUmv{{;mo41>OH&EVUyC%zyOYwKO+-iNm!4K
z^dRnTEv4$SS5ZN`TTg$;=d!2r@ct^}1;53uR=yYlK>3doNZAD!Pwx2?XmSaolp{!#
z1!gLY_Zxwc8*4LJS)k7cEU|xc87RfdD1N?z`G`FlY9ER&F<H&6avFr&ujTBFz-YFD
zpb=CZDH=*(OfNuaiWnq~8{bKm7uPa~3^f|eh{S==h{+tqSs)RPwqMfxh^u`6BiG!g
zvXvc5ycQv17#gRn^ItU4gP>wn2e<!Xoije_W28;L7BbUNKF0o9!u44cr^*kz)B#1d
zdrFmwl3poT2d(P<0;`By28B;}DN&$!Tw0{+N#WTn%yTm?eeB6=ugi!zCI+BpBIu_D
z3XHY)*27P2|LcD6%fzTEq*q4en2Jrpw#vXgXDUVw3qTG27_ge@D>ddS(rxXLgWy5l
z!*-?ujdbEZ!B9z5;X+Qm`k3#HN`}L^G=N6ulUq2QMgTQv8I}^6<WyK^?GdR`@<V6V
z2dxs?_B!uwO4D>)ZcOP@BP!%|_oGd0>`1#6iDEoFk^8lvKC*)t(Sp5d#8TDt>_M~M
zEiNOEa^%K%3k}A_+n?Bt-y%hZr2Ai=8<t6gaI?C9_`&)Jn^W*9Xp}+DBYP!5f$^K#
zgrC=vc`9xNW1F6CIs>==yK|rlN8cV*bGQ~#f&jKPdfwRJgu-62(Lo^wk*)5}ND7EP
z2lY=U)Gn(q9PqMF{`;oY$(F)Xx{;F+DUI9Rw-_=sQrF)lX6cX4zUSLDl4X{m{$V!|
zJ-m2dF~xM)!pK!^6S`gmCVMMK#V84;Sd<;J(oo_PQmV;mnoyPuU<&1-82(X&ZAP|q
zg~fx1z66oc)iJ?SkaZz}w2%++VVl+BO;aZ$klGCi7WxRr3GV71eB{P23T`;csmJYC
zp4uSAxC|o65kehQ!q2AjB_o6qybtx+87xJs>qYrCPV?o&k%9C^Bo$CPFHYo0+a0eo
zYE~La3SY0d907`M{&h*7ELkL9*xqE`)e9pF0BK7#%XF_g5OB1g9@;{8p01W1t^h{c
z($r7IGC(5h21p5Y0CG-XIXf8|4z<`J$529OG6VfV<V<9?>g(;#CV04zFg#S(9h)9z
zO-Wi@Q_z0ViHQ|Q5BWIeo=8{?wv9Bj+G%B60BC*Cm~nCC5s*STc^+&y(6(ZCL}vz3
z=PoCz6lR|RdY$gkQr!{2VyJ%+>;i0qBcRjW07#M9K5KZe?v^#IRjcO7Edqo+D?mnB
z6H2uyztH?5Odk_I5fKLO#To#ND$OF5*SoHrfk<oIJys6wevh_?+G6b?W)Eqtlug|E
z`XAnbXmM3R5i)Xwh3JYjAWR#_7Q#FzlrIx$(&^8iESA@7@PwR<AaiqfJt%<)|3E@M
z!L3}ZwP~s?4fEDQ<J?4!g2o7^#S2nz;Qx3@n%+kx>b1}rH0eu~D(2f%MI}dof=yus
z<?-SciU7}%RM}4DhYJsiUay!Up%Xf`$MpA|Zx2)U3rnm%cxNDu20aC2hAh}rV#foI
z^q%kl+Bhdr#n(JJ7}fWY#ZU@5S@~}@cv-<8B*9E5k}foYu>rV&li{VLW?o)z!3zX~
z$MU9ZfkIN=bc>aw+!hO)y^$}3NYaSW4B-YqojbnRU^PkmpM2SKC)Mq4zi>GWk1qWY
zV8*&#iC@mBxdHS(VFTcG7BTfNq_ZaQ{w7;Y^xqfy9l%O+KiQHCLdH5wa}a1DXrE5W
zJf0@?2KY0jz=)(23Oi1MvG>;}{6gP>u0|a|1zdU|t^(9nQ~ZOK#v0De#a|jMECx-r
zqUQs2V^2U+V(c3Z^|9@RxKJe&JoUn*+tt^rwKCTF;`#Fkpb9Ro7ppNA3b!g{2+jR=
zGTrig2Dl}2fPho@`QhSGaFstdpjgEmh_2ACLYgaz7SJl96G_NPn-9fzVEJT{02cAU
zxw`ckV~72IOwKPA-@Vsill=&>?|eZ1uWjBishr3X<u78{1>pDE%Sn4l`Gmo`Wo4pF
zX%-WYoCmmG`TZHd>xY=uo&zFg$EK|P{K@^zs&E2Ep)E%L+wE86fJF60bnFJWJ&@!<
zYb;&F5*8cQ{cMVhi<Lw(w*d+2iX1?K^1XNVfH|dcnMX{UA9;;y%`F$<A@r<8jjaGA
z<qST6Yc>2LX$2O%Ho*f>{?<Ehe|T|`;9k1`?mf3Yb>|kw1_M5iQ={fT@rvnu*T<{N
zg6#Dn(SHb(MK<V7DPJrYT*Ls&f&|gKy}W3(QG5TSahHF_P^mT?tl$q4U|PYH3r{QF
z9;RDi+zR&o6X!Wmmd95;wPF~3oT)(Sn1c96<{O&QB7PDa<u~P=QMBc*Kei<I=JOIn
zAfEQO(a-oapWfjZLWcG})<PDZ!t^$}LhN>x$^-taKHVL#FIO4|afgSzB?&R@0rrtL
z7p{rhKlBu2hJtvZctg#{O~YS2LnQfJ`n1-7j?Gy8fw)o;DVJH;*6Adk@6ASN6oFlN
z;|`hP`@y{fmxsJ|K0m<dAo^lx&$`N_H=Ro#Pibzx$<?ON04g(OT%b0Hx;<B>X9pM@
z^u3og^u0`6b?nB94jQ-<5KF*5yE)s6iY^}ZpaQtm|MtPnQ@0+z1owtP#8Nh|u?u|Q
zo2WLW>Ab1fzKI1bEIYpaht`8pi+Tsjb@$<wVdTMjx$%PE(`@m7N(o^~1J)**rlXE}
zSaSF=owYxQA%b8L&CZiD7hzE0t2%VG8oE+dl8UWqk!LEuJ2JiZ$5YSQ8E|2!x0z?_
zM{Btl6D|X|$4W8|gYj=jn;LZH0KWFg$D+4}Z7tlKAxZ|z^>(Y+S(d(lABbyH3JNC4
zX-qwb!46-w@5=6|2b-<1;mPkU?81_?)qkr^t~I%`k-ECNKkBTf`;OphFvnBpJuh}H
zHElWWn_jDa+H)2UkAW=&5}5EznU~fg_h87lLZ~3Wq``9gMxVjJzmYkhk#OOi$^+9&
zYyU390HTeZMk_@QIESa9k02PFSmOxWq7fve``X7D3@%t`9WJ7vd^Z+QkkEg;PsaOI
zV2r3a-#T5PYmcH|VM^fk8+VB{g`ze<3-9-Qr&FD8;5w_})=Q)%utg+F$yrdQS@wfl
z_1iK9&SH(F@@{Jbo3R}4e!4UM7|<9`?F9g~A{x4%hhpwg4*7J$oM|hs21u+axGkUr
zc;=L~fVl-ZUD|SkgFD5C=!QRn_grl6YrU`Svy=|PeL_y}n89>CMIJ?c;p!JTs<?F1
zT2hu$P<^-HwZJ(E3awlEf5JU`+n9TFd+WFvT5{~uuYo{^Lmdd8>3>N|wLvX|T=@IL
zpPd5ucXZnDd#wDWXD$SL%58<<RT}{7mcjj@?WV}XwQ}|1cy8n1Nl77piiUx3{&d+<
zG#-E#W%lho8(2zk<C`hMen(?do<rj{v9Kd7VAX@pw-9>Og~2XhN<v!pqNPr+O45IJ
z>Wv_ZcG*k)5xrDvJxvi>XEFLiGgk)96ns9TrtHcvC33Z3DrXt9Jt1iR%6lhVDs#KB
zUI#lF!vyiXv_{|`u8p~|^Vn95xIM66xG;>Lx~_?=`#k_$bQ|D->^wCExim(odM3B}
z5wo>%Z1LNblLy(P;bv4tv4~I|?JrtX(4K&^&vxhpa@dz=dMnhPu!7;vcL8>(sDDU6
zjm{&fa%VDmw*(rhwr=(Bbj4MPW`~zBerg3AU#lt-9Qu=QF4TJHu&RqtH6h&zO+`t3
zkJ~zYSjrUjPVen&eX|L*9PjNAtdAXgWywAr?lQXLw!K{~3J~GPIUv2RBbP&Wd;dbJ
ziyABVoHrbP42QeU;vKW}8&4Hun~Au)1>Fmw9PX+_Oy^KSh9Wwh*7%2|cxl{1z1uW<
zy}i1IeZnSvgv$LsT@$u@n0TpN6F&b|d!U|EeJN@j3A964-L&qxs3O7n16l@aR~Hsa
zDDm7Xw8s)&B@@eHPr6a;7hoFh!ofp{>a;ZORKdu_tlN@tEipq++HH}hzgT?`v!#*U
z<Dru?FZOzZPKa5;4NcWfFvBKqO=2guPy$;<<Po`2i#Srn;j$rGnZViK#?iA>_>7PF
zap4kDxPaj&Zn9()E+n6(lBn$6e&NqgPKPMs{dt~6T<C=egXO)-Ir$u8pLagmI>?3E
z1AFJjn8}r<8P~~0I(@?4ZfDWk%bMhG1;&O%MdxT$!!0%ae9vty{xpsf44`m1z~?j9
z059h1`&-QkhIWrS@A0RgP`lXz1HSF?*{iwv_Qm>Dl$Cyb-@Fg0wDs=Kua_%VcT);w
z#F6SwZpeIk9|Ej^Abp^SnR^{Qoen38fE5zox8OUx?bt$UDSof(J*DS$q=*iBnA*W~
z97=Q2capUVZwQ)PnBRmtUv(m692O+E*Sz<CnD+NQt!8kuDQwa_NV+~ejWgZ(aPoCH
zr(6E4wXNTRRmZ_0`gSy_O!<32RY{0f6a(-^l*xSZoa#{MV|UxVYbufAxFzes_lC&v
zV*A|;olZ8-O^ju4*bPYmss>S=3Dv>cN?Yn#a#VT?(^uCbSP@WYRsW9B6_A4_;1J&n
zP?fEw!%3nzbM%nbbKxmifKVoHehxZ<l>NCzcit^qAx)p_2L1hcRGn0~cp5sav+I`^
z&)gA*SxV3t<xER(`#0Tc`R{16?U-SI@^;GU{W_mRuLZfswC7xos>c2LjP`m>H-cs(
z7}j~YkyxtFN#9|Ja7Gkv`^|K%=_GeiM5y+#YX|=wyIc)zA}g?U#&t~9Ew5bvEWVgu
zFg&PVE^)7|+OhoJP~m^eVrtSlTn{`|88&`Wiid7hj(&J0-1bAJo9t(q<@`6zW|Q~A
z&i_)PQrl*b{jMS@g`8)Y6PayK*4I1Bz3icvat$)gcZo-#i^FB(xaCTaEH?@D1Op->
z<+rvNT6R@9e(9Dt`b!IJf1d@<sjHs7uReHhT%?k=5oYc?mH+%RW{%VPoS-I0WxKxC
znCWW&d2}dAq2Y~9{i5P?n)NhsY27pnVM8<9SEE8=<p!#w1MU7k4cp`MPgg^)&b}b^
zSvNglM8Vw`3lv1Ilc!3<s@!NJv$4dby0ax?KLWdFG-CbX((X)~XaAX2+uJWpSuX3=
zx+SHFOy8;FhOw^pD&JmR%dvHa8(Im~v|Rq3Tlkgq_Ul3Yv0EVRi68%bv<V)f|1}X~
z&66T3pr9hy<O-e(tG^T^CTC%yqt&5d?Y(n%u%KgPt$w0ba&02YY-;;i)=N#_+<8&(
zr~bpw$ORks+PXw?_l4=_j2&o+*;91XuLX>TTIZ$|?{?wN^TL9KgaO5VZhnexeR*e{
z#`us@qqdfh$Mx|c7kVEu^^dQ?%;a-SrtHx_jK#a1%#!bgMcFGAvmT~MWX)@95&r%V
zalon_sy|*j87qm(r|aM}NcT(kgb`#6gqnc)z%-3+)OoR-t~8wA&i<f?%Q2>fd|l8i
z_a8V`9n+3UcFLW$vXY6#@<7?Y5o~50`1gSr9z=wz0F(B&U{Mt?+%XutuYQBBQ>PZG
z`}G4=iE4SuYNSLT@NUW4eT@00>2?42bPr+=s?9d1F-e)?6{21m+}>wN7f0uX@_NO&
zNQa}%%fj%X;|#2|=x;EV1s(bJC8i!$wTS6dUMph0tmu}|jh$@k*PpiQ5a}dj#*<R!
zLV?L{2hKLf^OjkqHa3Jd$6doJ<4}XC(r?^1sY?07B<4O3ThP`h_b=+}JI}4Xci0(+
zA1qFM#pjV%SB4(NCgjWoidKKj^9bdvKQ^trA1jAmVKFZg?Pl658?TJah&PfUADitx
zr8t``FXlYQ$H8U6d1#%1X>Xq}nTRpGU5JNpCJ6T=6?W3#PWo1J7AM#z7*#}|Sy7gO
zr71Xc8L!K)-$uTMd|4%1?39*&QZ?=LDSlq)r^NX(x3GUS6t(*cANAcslSixXom$_m
z)T7v+to`@*W}6`I%_d?cqrg^8YtIXT&e+v^>_yN@9+NJ|bz;KLuPm9sUD%i47wgJj
zuG8naxpipkhOqK{<#GQM+`Ypbh&L#dsLVOGr=&P)dh24|Qfq2i=J>|-3+ekm({veq
zrP@akZ_*DNTz^^ey9*2(r~}JyclEj4`Sq{`qP|l8uC<fzfOw8oXo{ed<vSjN2um}p
zS;nZz#S`>T0XOP`I1xXq$n(g)Pd1;ffeOqCs~G`+ciLO%C<+9>ru8s7Hm3h(x;YS3
zNH}<cNYLiD`MLRWwyLk{KC|y$#`E9TvrK6geHy@)p8?ZT*R<Qq-5Er?KBjN6P6O9z
ztld=MF{OBT1kIE)Y2fXzk2aNA_yil>!@QDxC3(<9WVf^Fx>vnsy%%<s#M9j-qAeD@
zRLN?iczh@h_OV)tV)qRt^(6HT>%QTNH5Uf0r4JAb3`1Xs4rKAs#jZ>H{8i<*wDQ>(
znc}?1{|eVTC0P=Frzv%qiEVaG*NJYWo((9lwGsJ@DgG?4KOK)uw3W(!Y9Lr_e6L|S
zvbCP(wv@%fFB~VYHQcB_;TqKD>NP6FSN=7|Sev+i)bMFWtKdE&^7Cf<Q256On?DU9
z^Yz7m>%V3_#$9x6hBuCoHEEPcnU7QZEq3;@3EiA+spsb4L}JP+2HO_F>%6S(i6fZx
zVVmP(u7+MzJ7PcE^~j{E7FnwW3&`7%FBRo|T~C`8TSfONQt~S=@+Qjo5G9zYnKdR>
z5F47P_k!SG8cUAJ@gaq4jv3teH?QH5c|4wBzCmtZ&jPQ`?aGd#=lyG)7#V`<J4WoV
zUCw#rxZk>k_^(y492@k0ye3kX3UAF2u1YJML!yn1e$P&EaB?TRzEd|Ws5&Td>yb|I
zrHQA^_=rrM&rMz*?8dt#VWphb3Jx*|CS5uFT<GF8-iwvqIbP0VQ-4^mszz+u0ag=H
zO>$?rQ)kzkNj;n{`0|Evv9nv=mUXIeNFeX?do8zC21ZDNGMU8#_J%;cl`L%|V&`*q
zPAr9j#6X~ESc~~<xd#(-3LH`>!7cI>R-mRtH9bv7xpg_t;H;#;`-|0kA(+_bj&H^x
zIAn}wLe~ttTC}5${wp&ZAForZyzl3tT~`VCl`B+02UOI~Ueke%6~8g2<n+;)@JY2E
zbSiuhk(pg~E@qOOmX9A%GTy%nYX4zpkXH3-u)USjqP~WHyriL_a|*DTRYYkGH|*MX
z=k*QTy1F?n=DR6(<I5eFt*2a_rDJ$k+0dS9dRgKUwY$X!%PZciwhHF$(lsqr{6Wxa
zrsI<A0De>D-j-q7S_#I@&~zKEv5`(<MV;x@Dg7256Bcv$b`tVHya_EMr_%#RzdOPm
zyF(8%{ZJ^yoo%H@pe}(jndREu$&A}?KNKI_(f&L;v-<h!D{BY+C$=}cy3-!}OGj5{
zH@7K})v@}+m6#kg?@Ij@^{uUPI5>Rx_cD^|m^Q(QB3AnF`PN6(Zpz8XeTJ=zbtl=E
zJ~rvi@~kQM5zX*RVdRj6$9N|jsdFv;)_&o1BvP#{FMXegW&hKxngb9oRFQ{5)NSVW
zeMx!B;^^Yy<uS!q?^Es&HkQvEqi%<2BBjOJUOz6(JI>frS&4?R&LZj3`7GFm=kI?~
zulyRXWfzEBd?t^`k?z-8TE%UCM@RW)B!;U!C`poE^7%o+xbWRO&?~z616*pj`cMt2
z)6|>VfzxRrYMrSn|JPt5o<;c|6b-_7$8X~T$HL6kL>ldyWA0I6IHM}IY(v}cKjbg%
zmr$&#`NRpf8OA9*`^4{9Y0iByb9bS^0P<ndC%^G~noK_P#psS(fxK}#A5Ex)7}_8N
z%CSJu>l>o=r6EU!C(l1o1$al{y)%u|09B}8z#oZtIX<3oqu_?=Es_uZ*z^}S%@rOj
zYWF7c5~j4K{X%cQF146eeCiL}I9y8#55NIsQb%9J?l4mCcP)+%5Lz0)f)GM5Lya6g
zz352qiB8biJhh9-Ue8zfIXi!<JfMl5eL;5UU&G7NCY<KV)(n=XI!~5Ln=|slyKy)7
z@y_&JVOxC)9xLJ)GJbWHHBpAgtk&zn*l`?M@;W*)9S({67HLq5YpgbNn@*nX&>!V5
zg56o1Tr<$9z)uIzHZ(uTj0m0r2MA1ionfU$pBD=isn&DBrbjtPgfX={{+3|hYgcJw
zn%QQ@_vHR-D&ws{_*Wwi=mv7q0@4<_Y>g;XGXhYi68(H=Itk>0IfAQCayqA(RwXk6
zHM1jB;SQx|+m+TD$}<)Ihj-NWP1P3Z9^an!PLLNnkWq@fWQ^l=pA}_k@KEz}@yZp<
z$ZaiMMd&FbLSMTTub~W@C<lz`TNbGymWUCU+Xc6{YT)!d83PsW3ygzL0F)v<uarN%
zf{+jxfy1|m-K8mT8c}r7LTu6(IMFs8EWbL-{wA!+bYP5tAtqnv;OnrDz0DDP+SUD@
z8g<VefNUm~eGSZJP#&l{&JDQhv=KLD1Gb3w9V;=;fhw%qu}p-^StmIr+i!z7*9VV|
z1L+M`=o3K>e(PhV7y6JN#wPpDsie#cvhe3+GPg+T7qT?p5Cz<v?!)h@H)WEfNx?Mh
z0hywznO`=QVX8b3Jd3fVv0t7pv?sDT2ib8ttopYvmm>&3?|Ob?e+@yUM}U}E{oxB+
zT`J3Vq>H1b1^jVaL4)|AaLM+6v0Kq6CvoOU%y$q=-%ufa?{za&ASe{cwo*fk&wz74
z1{z-wCcFV9aYN#_GR<0JRsTL*(rV~WY|}4|C-2wb{ref|)g^OpEAtqv0YQ}s1A+h#
zRBQq?Rs!j8aZtE^C%sL4>2!oiFah)VCZ%Zo($xCjg91plE+v5~^#2~Hsla*1#=Q;t
z??=GzUXFAHU*!M$8{mT!0}w1`W<xYDjne-;7(ZwIKjYv+V$lY|F-cRo{u|-{j0V6L
zmjC<TF9QP2HD_9BxdP1p9pq&+;FL)J@A&{9{{K&j1`RcTd(>1Gqu`CC;v+w&d%6DH
zPixl_?O&D8!9qX&Au^|b9%!8Nbl&}N=Q4=~oQVfx?51zSRRn-er%3>rw3}X`*IeU!
z-V(2`c*6<84Q1=~{R~e}4OnEn-EPOCBqt2t^!?YquUvJp?k>|w`&GB|A7cu(bIC+^
zSo*Tp)5%9KYK%t>s#pPOmaRfXG@sj?v-e?MF==$jd4X{h7vJ8F=9PbYS*_?jPDs4Q
z%i_le11n75sPfffj8>)E(Qj8yY4<E49OLM+!eTio!@JC)adWQK)y|uWe|OpN{<Wp#
z<f2@L1dwbmq2Pe4-};x=!*_SP)8wtGXaK7pi~AC%J?1KOGBkAXatA|%>{c0fzL~F0
z_)RCh$T^K6KX+yozC(Ndv6Ub$Jc^XMGu|InU|mgk695=>X;RSxu;hywz&RcIg}C~?
zmxrZK59?lV58WP~i$*lxKLyFJ?9NuoSsyIc<;<3APeoIRT9_8F*Fmx#O+uq7crhix
z(XKXMrqQ>R>tstBmRJl&s(~`#PPDJ<F;NfAl<e8X!MoYzG{z0c7)o`HoZU`O@yPGC
zM98_WaNL)fli5sS45ml^YOtCwyPddG7+p)+R;FK{oWQ`vS8b)?btQ(2ekrXhcbKVv
z8vMq&YBO9xKO>19DF(I=+!tCMqFuz~%Rgr|wS{aeeyUnKQ34-TV*qKGe7Q)!keJZW
z6{X1GVf#(`-<QgNjMt#@Yviaw{a{8Wd1q3>>x4BRD(HEDQXkFNbLMxtLN;+<{$v5G
zHu>p4jQpS4&&Q*-fAVGLp1X_Zvc0TVO7)hO%9<%HkB3~88;*D9>(dNx*OJPXjXx;~
z&z{rajILKnY&+YUURL)W+6ly0#AMn%-&#Z)K$`ZnNA#)|E#B!`%5vaY(f6&EEmTfB
zNhFo8TJXCH3>OQ(YSkkdE=H8Us1sgV8j2D=+<#W8(MWUC$-c`NxnmzjM1Q4F$9KuR
z7?jLkH7b0yXjc*CN`)+f!3N18x>dXII_|P$sz_wf8F0=@;hC;dC4#!znbR?{chzOW
z1Tr2dymUFXJqqUQfz4p^WvjiJS7046MjT31kk{Jew8a0Et+wtBt8^tj4G~_9z!E-;
zx8N;BqD_UOYQ9yQNe!q0AGujNcmEb-S+#zo#859UiDeYa+<|6o-S5heRSu7fj{cZq
zJ`jile~B#EtwG5oJ35Z{ze})o18fz>ZX1VS2U0TSRL?h-;|6}?EYeb~XlZbiH}La(
z4P3qGOQ%l$WEI{5Zo9IVc2Dbir6&`E$A>Q!T&nigS`HSl5!%X2Ae2<EzL`aPBMeDQ
zsT)x+2vG|(L=KG;mLpyKnUeIQAN`d;8^y$~8nKR0Q&m6CL$WBgB_!@c@~!aBhpiI*
z`LpglE_c%Ant6xStuGN_c(cF)H&H5R2{}{%mD6eca=3z*FndLK9k&_C|J}M7D>Vs(
z>58zqV3c%hB3K1ugBip-MC2m=@&8-PtoWCp+&mpv;-0L2d*xG(SHi#hv*#LTmxi(S
z7~XmKU8j=yeRZgDcI@^=XJhv8#U3hf_;^0G`+Cl1GyebAfnAALFD}=|=j8XyeB0Cg
zrCcXq=ekdS_bywsdX~3PYx2Yu_H(L=v*!7$>1_y-5U<ofcI_YU@%k@YmO3tea4Gmo
zTW)ppxy3D@%nPb&-W1A~irwruutrrn=hgh|H}~{--!8WglT^2^kWxP(;_ZL<Y~A$O
z+spb&52&;QdnhIArBALqdHIxhdHT&>(_a>0y;VQDHm}XTEY&A`l4V~XYw)_=O!Cq)
zw<{jryxPO}Fma8$$KypW4@RiG;$J4W)Y9|8RsHnH%$hRLh|Su&o22<wU(apc{av=`
z*#3Dwna5lo$Cf>~Ab5Mn+|&M|#czw^jN@NBJWV;V#vJHkPe+HlH$4kyUO6$xZNdNc
z$+^exF`NC`n0IY;v~qIDroyvE*A~~x{(g4S`)#AQw6({j&reHt?TbCV(d4*JrkiT)
zvB)jCufvP(EnD{Fpu|a~V>36K)E{S>`dUQb!T$fgu{m>mK3;g5ZXLMp*Hy2ZugkX`
zJrjRU%Kl|bg|?c|*J|zX-+zQ(nCEQzYg6C5x3b*&@tLYyYk<ywyZZe4)T7)_i|zcs
z$6hjUcHDC<_Cvv?b&jdlOFim#bMW(*C(c>2oHe*Fy(+gGSZvq<i;d3u-YbV>JQ@Oz
zb^of*+hb`w>-)P}*$ph->5qeK-&E~+|Mnx_RKXkfHWknJtzEhM@1LUb*Ik~b%^MAN
zo?e-<$gacgT;*rB*``0N&TR7Cc)Wf7oCA+uM;JV1+odxn<D<r$N#4;{wrJ|d^)9WO
zb({OgQdz%GoGHM>5g=>z&z~tqsmuG1j`Nq_wmN%{bbQz7S^bymw43d<y_V~e%hQuW
z<o`UH@Vp{|Z{m-S+eN><dYhYbr}+OSvx<V3Kd<fJ$!yYmK415e958FSs6DWknQ%fk
z{I(0QRt<Uo(a&XCj^FwD*ZC51J=XnwaaKa{R`r8`wy(;zmks{|$4q-;`)7g+J5Zay
z+V28q`m_8cuhMr<t1HNMs=e>}YV*VH$|c{QA8ygLn^W<dXXmS=Ex%uET>S1K_qxCH
z)*qbmc&44$?apsEJC0he<`e8-VeIYfVQ~d6z6w-3@G<3I&g*r5|MAV<|L0#HtJd}W
zrE=_t3l(PC27RqIah;tg^daWKy>P)f3*}EVX{rZETIm66LJ5{DbLUE2cU+5TJv1%>
zwQE=EO_$P!H&mE99f7UGq!{+Cz{ORt*2ttTU?y6!D<*3VqT!>$28!Q8*(+R#_K}Ac
zuzfQ5Rnf|gz;p~nK#PG@@UhYWZMbnR3M+wgH9o6OFFgiqYe1Vo4oRS~c2ox<H<?D+
nqaiRF0;3^7b_g`C5d2ZU-*tVHTT{mu1|aZs^>bP0l+XkKpBSAK

literal 0
HcmV?d00001

diff --git a/site/img/shared.png b/site/img/shared.png
new file mode 100644
index 0000000000000000000000000000000000000000..7869daddefe9f5476f678395e3aae91442ffddb6
GIT binary patch
literal 37973
zcmeFZWmuG7*Ec-DfRfT364Kp6D-0nG5>kV7*HBV3AfPamG)SW$BGLvmh;)Z^41$C-
zNJ%~8|GMt`x$gHk-f!=R_ruMXnRCyMbFIDCZ>_z87y7#DB!mov004kQQ$y7d005-`
z06_FDd|XRp{5d7=1L$X{ejiXZ#I%Wfxb3ZB?gs!6;eH^Z1Ojqi(cwCLcQH0YnCWOk
z?YumMZ0)_CI0yxKc;k8l0P;al+*c0=ge_Z;hr6dAG)RH{pB_-$_rJ};>}>yZLAWWf
zo9XDYJ@E2%V3QP*5E5ZmBxGY_llQfEgc_>C{yiM`q`>ZsKzKuig#!Zvg#yKeynLO6
zMP+4Wg+;`K#l!@0Jp}!NJrTA+f}VaH|C;2#=23O<v-5TFM!0x+vi+Uc_KBB2LV=zA
z??V6m_pkRvxH$gbl|23aO%{$o;lIBKiwcPd|M%Ruq4IxQp%1(~ynP+~{BZLtO3MG!
z^MADc-*x`gU*Fja;f3RZuZx|gC&IxOHyC02_w5wL{yqBty~qD^E<Ilt2OQP^j28X(
z=>Kf{cYk@|zl8q>BmTwZe_C;DRwR@c{_ibQB&=Q$TL1u*0h+2R#zDaCJc1yqM~u;4
z6ge8|PrV4s;b7G}Dk>1_5L4Bf_!jsSs~im#QGh{pNn{*}ni@m}s6t+<nVM(}d1im@
z7o$Dx-ncig?djXP9$aVSnJb=~=hYgrhZYs%WrLI=sggm4%7hxg|9$vh9{le)_+MH0
z{~I3ka6)Ru-qyIjqw0}>BnEjjrMq|tRqb3==xx^jX&DdX2y1oOJXz3Sb(o;wy<?8r
z?rCZ;{hxS7^+;LQ-aw~b?LV8@;y}LaCAUf!{s;Q0Q72QH8&i6u{O3V<UW|Oz!vk{p
z_@BH{Hh`?E(tKC7_#Y5j%)`bl7N<Nb_@BtCZw!=8QIi~{`A>{R@euaBK(i+YJp51S
z{r~b+m#KS(#NLFY3K)HSawair<V%G;EwX+6wYz^${R-*Pz7=icrbDQK(l#tKz9(to
zkq$B}EC-d<_lae)q#0G)bx8FKEZGlq`yCF@5(%U_QE&T(6;XLU@Y;Q4$C91wmf>+&
z%|2yO1p-rl+Qqt$zw4~Zx}Wc`9Li7!&vSs?*k*6?+sS=gZ|s#?&G_IusMjB`wtQ;W
zvuas~u-iDs1UQ@HRv$3LZH{A}aDxfqvq#typU;jQOg%B_b=ntC&DZ`U&b;xV?w6P2
z5_V_VNd_f?<^s&tCVk}E_|ZIKUqaz|0!JE2?sF-jzsfe2N|sD59Df})gJMT{Eede^
zN~QOAkA{ybO88KOmo<}OTa64y7YzNOQcGf|%=sR=_tOr)Z+5xQ7=R$7ce`H$I3`i#
z-7frSdhsc-*ok2=aMRD}l(kbQ^+XM^KO3rSD;1V6UL&b`G;s4euDTeNyjDg)!Yy61
zNsnuFju+78aaFxxu5^CZbME8Kk24lJuf7*#1c$rk&!7NC3aWV<JoCpoRhv<}Jy&i{
z**nwM*Ol_(zLP@TMT%oqGm3Nd6)6r<7~88`R%@sjK!$9MLxyavxfFYGSLL7XtjvgB
z@fF$~fvAg_7>-<rY(j@=E7yVQ&EU06>ZV_}1oZmfn!-V?>MPd+;#VibQYF_t4Q+B{
z%qbp<c}_Fy`xBEhZ|<Ah4bf%0kt(kq@`Wl1F-Dc2MZEQDdi>WE_=r1^zfEk?g&>A+
zl+{&}rEX$tJaQ9wV{d+6@A!Ey4~y$*_lduEbEHp<vN+bu*x_7*%UE(CqL#qwzA~@A
zNJsRGE95%O*$J*+=eenu^L^c9924)pY#7ZVXd4!fmokeXedh6-)RuW9A>6sc{b;OS
zKFuRL)xsgyLe44ovzy<bZ+(eryuf(K2LVLxS`HT@+QovuKW$F(m7mkyKe+*f1DLFy
zj$C`jMOA^W^AFaF{jZ+$Nbe}jJG=Kwp6C#?IW_RvXIEbCJ(y*F&cKPI=^c(H#3Nnm
zdl17i%yJgwf~;66y^ntUZFr;0=U7p)%VgZ<HWtfW7^=oHS;T3Is7o~rp!@@&DMKe^
z-0Rr|nR{r$RwR$Z@S3@!k}B8Cpyv7q!x@3`n^@+PQlhoswRL&F?R;;}Bk&8QYlm#s
zo7qQXdA<3Ae(p{_=Z^gn<*MD2S!!E>0X5=%Tn{54lkPJ!T{%ro?Jwqohbn%A1RQjK
zj#(1S#$gMW5e_G~j+(d1<BLG*MMSc+Q>P>rg5v95{qle8^3BoCHie>w^`^%Hb?U)(
zeXDGQbiav^pQ#~TyW5gCUwPH=UQSlJ(m00gy=rLkdwA9+X%Jq*C%0uQOFg=vc@Y|(
zzd0(=jg}GZM%%b&3)8M<XTJ;F9bfwWs8BaL&tqjU*3QVg?k>TY!Gi+BLVLbDh}*o`
z>>K5;m3{Q#n9R>7-*WW3K@p>ti$ZfuXrWyj@<*#H7C&};1QI-*py?^-p)Q>~U+w5I
zIc}HAh1&j2c&idQN<PrIYIBUI9|6&0vg$);r8p(Uqm8RhUYh`iPS#6GOyC-k+JBK5
zi^I+_gZsFp>G(0dLRJip*+)tP4Na-R<&u3)x%Zw6b-9L~*njNm8oBH6E!E_nv;4~Y
zpnF)G=O$M1o`~+D`EK*s^Ygs`JH?JtV&4NGeoC)^OiJ{)vtSsN48sG4+3$l=QTqb=
ze|c@lrjKJb$`yw}R7Q}12lmM^P8xGv=JjZ;s&CtyW;;X(g@TESw9jW4_oJZ!dF?`x
zH^PpO*AHhz&Rco5e4Zq|7BxQn-9i+Y%}S`ALAvpbD`WW@KPa_VUVh|p##6|dyM`{J
zSa$e@dy;5ftVEAkiQ`K9agcl3KeXN{$Eg%$$$9}u+*=UqV9lKu5^Fm~zQi0TBhh>h
znJ+0J64BhWC4(1pS?x+|Iq{^JBW{Z|%(?iQPo>S0kEzU|gLWzX2=82TplhOn-KmF6
zF6FWa{w!Bd#KW?@#6E5LS5HTtSsC<W5RXqxQMIpY2TIg;<ug%Um7i_ht&7&x|IG1y
zzXvH*hVmz~Drv0WtMD*n^P`se(XbP(P)^0FMLkfy0OEd`WUtvJbC-L*lrSLB6sE9s
zz|`yg!E&CN{GslrbK%~AZ+^r!$GmwaZGku?{9L{NM~2iwD`TgQ4YS_CBjuEyA0eLp
z_J)PeL5S`7YHw1`e#gM$rSqg`w@s&%yzL$SrF@fT2)`|6rq6W0o%|o#@Dq+p8nm6p
z2LGX>d3bRWEx_yaxSRJsbSy1SZ*LZ7@oD{sisw@MD^5SPw`6dT{a?2nY%xylTTvOQ
zY5rfeTUd%ad9|glT$s!LyB{0G(pXuyH2z5i&wps`G%6hMXQYY0Ur7VG6k<+v+y-OL
zQ^yOrI!kT`&wb2JPhE;Xhf4{a_3ktE{9_n+@+6yl0Q#*U4Gm(fZ88YIKbU$ul^H$B
z)%$)crR_FmuC`<rqTvHNqp`$;MoD67YR$}u^YO?4tTv&~XXBC{eJ!%YuL80n`&Bqs
z9@cmPO;Px8##{9dYQ!iIctoH;RWn7pix}j|xl4Zcx`c>CghD=9>&o+`8tMakzA6`g
z9nAvqcWzp%enlGg_F?s=glP!K#_x238kyvYUxLcr2Oj%EXP^l{;%M*;D+00tWF|1*
zCY*qc5wsV;Sx3OCer${!Bn`WSC*XEc`!M$qf>+=t{%k1|68J<YJMo(8k98F5++NUW
zslQ@HU#c8{-|$EWXG>rrAg)Dy=|096!efa5+9){;8JxNtz+@2vfhwUxb?}{l=Jt@s
zaA}UPI^|$*fDG*=K0AHG6}`TjtL_f?J(CFxrQSY`VHOpThUG*9H^aMqw~)s{G-Yg}
z9JzlluVNaw4W>ubP8jnhgCKNjmMl&!!|Yelg;bv?<ROY6L^xAjwFn&(sVBajG6x%f
zC7>yr{(Bx^10Y4P{n8o_w_53&iAKi4$qD)CjjXX3p`1bL#9NgdBEH$$oQLXZT-;*#
zqv@)IrKz|>Grs9wE@;ShMO9d`Z~AKRiK;8hMqWWu;_Cf<-dhOH8rN_LLF8V$u{SgJ
ztmUP9(=vARnI3^K@&LKM>{(5tD(0$A@|H{Tb>oCg+CpRkn(5vT<y!dwtDE-d(2`UO
z24g>|ZwIpEE_pTSKK4e2IQk6V;y}W<;=`BM%76Hk*SQ`~@J|?woZAMzv!jLE3ek}S
z1WK(%cy_OY3z;_{(^rovy1OO2e3XbYKv1kS(gf>*u?3Bya}4BMH3Z-CHjp@Q!&j)W
zGS!mCk#vna(Gn<K&bk47gwRTZWKT6i{R`=mF`WD*Lve@vjc&+y0f?9iD#m&B2Ofi#
z)4?r$UZoGw!|{M7rEZk|DBtcDDGLPsSdqXr>W8QN8d-bVCD`G2vhJ#x@<K7aWU<Kv
zdbUR1m40aUNq0QbUw|x=r!OGi6s8^3ZRR_0k9kRTTh9K;X$uYyQN#u~BQWL_kE9V{
zsRjzaTslQ#6}Eao@rr816RMht{LoHT$a2epqdFzA2Or}Ta08}R_NmR0p@T8Z9N*9t
zQkt{Q-=c{$L+9u-x~je}r*DY2Z>Uc7Kakf>NE7K(gcesyPrvkG?=Gp<ZHKs8;g<aU
z_fqgDpRqxkurvx3Qbf+Ql=b|;N6G0~Pmp+{C`DH<yUv5{^*6`Zd(}$LptZ<;Wttvi
z{^%I-KhlH)K&GT1QD<;@H_e@R1F)-#PC3b6iUnSdZe9}0gBRn|L}J3+kI;8EajORV
zwez^%D_FVn(m9oir}Zy>wXU0}--En-grq%MLk?-NpNNY2<9&Z%@Az@c3L7$V_glEr
z_!EKOK!x@f$do1`74u|Xn0n|PGs2B+S%Yq(>VcJ{??D6d0j|n|<+mG(V54YC&qZKr
z`-4%6?&^Y+p3S>sH6HSds_wVNJOCG*FU^B-XtZyGlb36^Du&m}Z+J%TkXV7Qb*kG3
z)YT7HMb2Yw<?`6y$dahm+EgEDU~Vxnmq%l1^dY4sJBctK;tkb(!5*-cveUlUH1QKj
z7LFwUyVx3|zob8DC`(S3hP={ViP?$2e_u8wzPg2UTCiQK^}b@_>vSB@CIfLa4<J)Z
z?`ff^ct=3TwR4j6jL`$#>O5G=?UM2RTc0HFL!G9i&1CQq1n3|^0NT~pSF_^10E0Iu
zh*AKe9u=aZfDyV3YFHILm<~eAo3Mx{T`}HCvP$}fzBO*JM6TB*;_}c=jw5CK{`PQ(
z6s&4yQFZ&F{jUrS*iYPL4hKf7s((Iwc1;6i7X!LmqI2ZQ^)9;EeQ1>l_+R<BTZMR2
z1OZjG;ujigtipgCPhM4b91pjfvyJl5)pmIx$~UH4@J%A$VWbOX;dlYWJ&+oPxgk4j
z&3;ypyL?wGa_eALw#Qp?10vO#E35tIsydJKpw0GVxa%at_C#%NK!Y={$E+1R+BX)C
zlWchtoNnm`T|5fqx?IiZJ$ToccU=E2*Wu*tvR7Wkpef|5PX6;U@~xDth5T5x-g?l*
zrn?Db3H67K=7Q*jCqVl~tZ=+uKH@XKVoS9l&u<>g9}^&qUUt%3*jpxuv=VYacWfK%
zE|-Mrm%qotm?TPWmw9T3Ea<1;Wz%MXUdk_epAZD-$E&(CiygdSpFQPDsZHk|q~2KM
z?4S6Mww7RgM97QNiOEdaAAOxW1jK)QV>-Hac|D-+w3=1a;;WsTbIqh)AT_i0spU8I
zq54kR#E%%t-k1pd=?2C@x)~^>861v(fN=A5Jen({Ed(Di1~Cl~JOFl6nBc!)@EZLB
zf<8wC?cpN=PVi$R%K#iHIDUSWQUI@^?jk)L=YomlX;_Sk1MY~mk|q+MiNClAgcv>p
zrTVa&Z<iH1Jv(gZSA%$Nz4BO8oLRr6nsu@?FWb$e&&@wz?V+m<Sp}A^n7^peLD1}N
zzx22#P@{dL7nlXK`I!yb67+L*%bTR*GOpbx+^-lT9I4Tq{NYb1qj;#jfA5nNs?U_2
zjKh07Kmq8DUgScOx4ghc6pBATNit-18Nzob;HFa`-q>dbKta&AHLx=?+AaG9N{k^!
zYP9#Y_zX?;pnFJdSJm$%3<TM|)hMdfV|UA`%}zbuTh4BpW$%<Lh8?F-G=RcX@Q-ML
z2h=hZ=8fjnq($h0qr)VOSX*Va*l*{FZ3!df%KnCkC^V~5M-3Q)jDh>0qr(yq=%5w?
zx)gcVZ*yvBfs>Guh)!<idNE|;d<yn9#wP^vLuN8Lkxc0$m_@Dxb;(4ah>vijN1+`N
zG@Hr|_-I3@M28qhq+L5RHl&*5lZU?nmKH0etUl)PE(ydRX--bKb{aE#&-JoZ#BdJp
zGTZrXA2-g-I>pKFZ)>|@*XdgD@EB$|IFFD{qv5l&<Z-nZ{3Da!+$YPwcV53cUvB#@
z+l9+>52MfgGwXVN66Tk1_u$@+wGY?z14l+j1Egnf1Xiv){016qR&&BHoN{$ZY6r;T
z1&{VsyL}cO3D5bcXEGiAjFdmR-nRO=r&aWt*bq6CVPPN;PTz$r!4R>TSj0VZ?`}a_
zb+3b<{x4=DdU`IOt<VL6(SUqnx|(K)Oxp{$n&w-^1oMG1(*P+D3t1kMA+Iz2jZS0T
zrp#;kT@@thDh>8^&3pOAvi5$KM-z~riAjc1{y%Ca@oQuji{cm)66A%^TK%EMn9rWX
ze{9#i!o@9V?+G>xl$;GJh1d|=&WiV^eSJShwgC+B_06BjXgY$XY&YDSxC{Ft6|MEb
z<C-6P3IAhL=c@e}MRwC8t}fLtD$JB=*J|xNqix|iV=6<F9~JQ%mR>gy<_%IG=n?-A
z6>wZT@RPtHSJL8e!0AZ9h;#{wyLJzh9UTr(rXFf<<(RL1iK-i{h^QUtG0(7qwP=|)
z`h9XQafo66^_l%y$Kw+DKYyOCW@$#4`?94f*0~rcbJzb6r?vRHtCvBaMA`f6V-yfM
z+OUk3^k(b<zo5J3ckiy@SG-;F$_SH-Bv*%qAcQsn5V_}x<5|&R50NXzhvVjN##bBa
zp7X`gd7io_z<wN!vWG0pdGo~mk&5}T#!kXf`vRI$l6)G##!bO~<}&2Jl0z!HHp3hI
z3EymOmUL!~{Id1Kl4JH3xxTX%N55tK);BFn?wH}};5?0!$CRPy(JGzx&4zr2zcvRf
z(A3omoz)Y~x|U;fIwTb<*(Zi`DCD+bH%M}l-5jeHA9WI)-8~~8V)o!=e?~A>r!8E1
z<!)Uy6X(44+RmFl+v}mU`C8AL4B0#4A2wB?jaaXF|7)s0U#v(nEw3@!(cpeLBl8-4
z>>Z}gJRz3bc|D9TdG`2M9VWc#&VZHxN0c+ownTtR%ug(%2<SHu0RSiHVhd1=W1*j!
zX4MJK)RnvYA8sXnjqTD$t*D06KA6?l?6O^%L#8YsZR+$j8wQCW(~Oc~KZo1^QB+QM
z34^;rK0A`&e-h~Oe<N2#dGUQHhVt{)pXIKGF4+vT4?4O}!`{Gv*O)#rGt{HA*l>oS
z1s4v#K^>d59D#2d)9>}6*a6RlV<k`4%}f8T35R)hOwiO+qe+G=D9MZ`HPwMH!?xNX
zGc2|$UOzy2M<4`t_UC;U@wKS1F@;CJJmgnV?{h)!%P$@J>hhF~^b2;GZpDrG^huBp
zASgF#Vc4u8Vd?U!y#X#^Mr2KZjyv!Z#vH_DMg^)=gJ0;-JIFc<Cao}fL-xmV3cas=
zy`5DgF2J{Q)E`pWD#fyd5+)`k!<yuc+4gB8I@j{tF7pDnKj@r^rTI(-gn#jQ&udv+
z6VlIal^Z$kwqOInLVM!kL1;Y1r1F&oE5B_g51&gro_BFQww|qR1poFdSSs~j#x)H&
zoZ0ehE5`8IPBjK^35b)gI8-Uq1{0(maAY{pe*kpf4c|L7*pF{_w&&ired(YSf63%_
z#x6rm4E&C<EhO@0Pe3pTgWf6oNL+`Hqe?sclgJ2yIDcWQBhB1dDDuc3e!b<Qh-s|@
z4%Z2QsWsLfr*1d!f`9h)-0Oca&@<jxH%cP%#BH-HujDeXqa2+m*k3F1hWsvT*US6U
zs4TcR>k#YCw+!<)MER1zieEaVj|}nN87R(4yk*Gc|4e@LQ0cNm>`}k@T88e-TA!ZZ
z==YjBrmI0S2&qc{=Kg07N&~q1zHKDLuadn4O@In0ybDKc2k(mq5mGsok;j_C(VxaN
zg-CSq<d9P^?w5&H0#d9maynm%+-F$iX$DjvkIBbeOmp~>2O93pxEbYW8E)rLk8GN6
zC%!jhz0*`su$^s(&zf#`hB=^l-tpTt-(k4$nu^l#dZu-2UvEjSKfk?2PDt8mFV~@B
zC&OVe|MU2Z`Gib1x=Y=;vqDoC*GhwF*?P81Q1hLpG4mvW5^8w;J^@)GGLe^g!mN^*
zV$`6SxEaq8+15^3NPWieiZp;Ap<PSl4%4l;rUO<4T{)pHKJaG8?5Hc~{RUy{^O_(S
zdv14@lx6MUbo(D&!c674@rOX^50ai%xnmEDeoQf_v5WtH<vSu;622YRT4dKa!)!+<
zedClZ>L>UhR%m2eO<{F|$%uMrr6NMWZ!hpOWH|NGIWby;6INaD((bzR^>eEq9is6~
z>f>j7!Yr3}hlv#_lBl2(j7ije{8?Nq(=>u*Eg(eS<49hX`ZQh6Jo&^GKiIEB>E}cE
zTZqEcX?$->W)8Fx$_c>St4}&E=I$i1%Y$)V)%GSayA@Z`5)<VLcnbOrIeQXYL<6I>
zV`iu(cac?jOH&wCP~MV&rj5*`6jVOHG{OwJCFD-syP@I<mh5Ep<qKv|Q~1GQJ(thg
z9iKB%hZh<s)sz8#Z|iV*cp~KBN!K*)V*gA;<UV~?BjK0!k64@n2zRIr`(9SxI^LUy
z@$4Bo@Xt%-f1}~4C=@%ie1WB`CMX=ArV_xf0~b=B5#6HeN@|!!lMgUrj<E*&bQ+~h
z_7CM&0&@eOBiMz(StjebBru&<r3f!3N#8Tju$q{`&O&ko1U@ChZIXWLVXT{${QZ#N
z_nOBInp0^#OL#D#eWx-CP9QdMH`<RaTgrRVs=S%L-a}^g53L!)?_MH<<vW5>J#jpk
znc)K38HO-$NxZ*G7>0*_Il3T`y<ITjN`vLw$j-WOYoZ=B(GL@W5wtTu1&5PuC=Y=H
zBTZHAEwPWH>?JZ1XU;Kud%j*1z0O|pxFdeftkLztULw_wLk~C%GRNn^t~I=&v&t32
z4s<T5%BCE+Wr)g7|Gestldq#EdoHLVMbrJ5Oyt*h=8?C{P!1aeC-b^5c?SKA8OgL-
zx1LkBl@HhV#4gk+2{Z^Qg1ECuXQ&sBFXU(8QB9%0#C1OKm7Dv?i2172tW}`|T`x6R
z!f_tqY?KNzrPl%z%QlYwGme@iV7h%7z-CdRYt1k7ij5^V{@dV>-aGepa;L)ulPT$8
zg~nb&MN>J({rNP<N3kj<+SFL$9mcLdVJ(`Uv$lT`Ib2qall#8Xf@Uy%=#012(dS?l
zvh!BH+x>EAJtlMm4(-1rwq8PeZ2oE(r?*wDW+>0U>2+{wxtsBpfVl;8m0mqmr44mN
z>Ha81dZ4e@BAuluZx1DA^mJaO8km@l5Xs-dM{WV;@_A2~82WPAp~+_-P8R5*nevq1
z6Qz`efX*Z9LSslelsPcDO6EDd`u9Ibx&+V|jx))oydAY_`11inYTR2B$eA<xf*2vN
z2_G9R26ycU9%g>lC5SH|{aMpY?migH@#r(tCH<!as{quZVqS|fQNz&hpNBIf1t{+j
zEHv}8Ai+rQ72^o+Y8iNnANdw*e&<sJ^W@tC#uG+DTh&3Y3q0y@{0&Si$r>u8U7F1*
zVf~7Jgpi5Ea9O$dvERLo;b<8T;FxrRxL*0{T+IQAHwzsdAFA>14f5ai!S|d+!jWaW
ziG*)%Gd=pZWfB>g4WZuc1*o|`xKdLPGf|>_%;R+_Ys*da*liNdc_|iC&JZ%W0uZaZ
zR@!Ro=bm9_kz?E3D+%agM|&=oX<yz9yrDhTtIb`~aYKl$MwMXN*r*%1`QG1d)hj$y
zN-5I>o!@t$k4e}A$)*%dj7sq)v;Q?!G=d&kxKkIFd?ny;n(aslRL>1T5+O6Yg!YsG
z-7`7|wZ*p_#4Di0&G@=Nvd9Y{bD9BiT_nAWkr1}X7nzTghP<sE*B8zEh-J(+n?0hX
z@G-;l%6h4QA8xv#!AjT){_EfxQ$}q|I7j8MLFSE-c1v`fDV=42y!6}175SD%V}BO{
zR@%JIKxIa6IjmW)8XSP#k2a=b7J}9yI76cB^Mt8V(nepD!d}oZ4@YALo$YPXxKnYS
zZyl;+%ItgG_XoXofKV_$=FZ4sgJFDx3itAzrPEE32z3R)&<g%Z*Xigb`Vk+u+tbt+
zgww2N4os<kZo{cA>@nTcug4x29r9sgIT=J*gBiI~lti{}RqE4{M)OpekP!rU`Y6aS
zn;sb9Tr3B2Wn5R!Nl^f1K=+q};ttWEK;XyXyF3-s=JZVD9G}q0bK8donjA`eT7`)&
zJ${PLgpS}LEyrgWsSleG*3<r1kiu~_Zg$$z*+0d=4`BWmF#@o~F3nZ4fxg_1Jzy*w
z745rduq;@fU=6v6lok`?O(y^AZg@VP_Bp{)_FTd4=jR9oOQpcbnc6?Y-d_bUi72?9
zHw}Dhl@n8AOdF}C-1rc+YxW9gvNOB7ImVurU!_~>hjpvjzk?m@X7iRZBell2Qlatg
z-_QspI%96RRr*x>UJg9SLx!NaE1lE*?xB<}dxW<M)FP9>YoPEL-<iMdp**SLZk#k9
zf}K9E%x{2O<rN|O>>5{FP6APQToz33AD8rI1%TD5j-fuW2=(zUiEy&Nod%=c%dnIz
z+COE=gLGnt#XWMDr-1IO7NE@5cO54{r$cE=w293xyMD%!DmBH)b(2SklPYi<vJ-2K
zVNAELye&3N5XfotoFF))P+Pj03g&%&<=cN2(TrE`|4x~#-`3(3*zChhN&{LfPrx%p
zT>-|wxn919lA^)=%KX@vnt^oqk__RBJ-<?f_z&ltms&X4yG>{r1@mPxmslZVW$vX$
z>gMJ)5)g(E@5{zC7j`q)+S?xziqKy|b_oWRHS6PKHWCUek_c=F{jUgoLo3I3DuUX>
z@I;`!x7gcOYUYjanxh)pDcT}3soXUSAW#L<FC(uW=y$2o_T3hUBt0F$khs)ZGVaI#
z<pCQT9-s)MDNY@?D6@CnjJ^YUNuA}Yb-%iK;5nVb^DJ=(<wMoAWVQIb`%eg?`luEk
zE~24nP$)r)0B+E%09oS@&2YtAKR#b1zCOd(_$>xU{KnTO#gS}VCXQh>yxy<7kjuZ+
z$R7<*e0-xqm<Yf9%RMXd=|a`ZM3EbUd_a6VlgO=+;2$K$GAo=cu!!OSs~CPv$4*np
zb{jA++WtKYA$(#$r=+ft#n`)UM5ce1WHXrb1Hm33|Dpj-{E3nYQh~Iu6cr=Gm#6hY
zC1P~sh@%gr#TR|g86S3=;Q}y7Dh(~5HJ~g~gSH<-SYa1kr`JYcEpz9+{_X{wcY`1Z
zmGpgsssN-2cEbrOK2_iaAwiHYA?YE(L~p#ePaW7>8Qe~+%zZ$QxuFa{WtPJyY7Pnv
zlq#KBcW@?S=8#6Wk<2NZOA`|D*l}hElbD_Qrbwp~L>>V+3NHsQXthC?c`}R=U{884
zEf_dlodxe0H5VX4XbJqnsW0M{_QPMEvK(fPdrzL*g!{2+O2st{!hBQUYvp_zw>L85
z4eEbBv<qN%bAl2`cNdVO(Y;s-jL_>Jj{ZK%h^+onEM`$T?;zC6aO7L*JNGyWne7#h
zYVu2vA_Q$FDppV>MMB~?-u`-gbpB_Q!$o=7+$0|41pgx#I#$<iBEsB^Uj<@D&fSP3
zjD_*qBA41bk#o(}Vfq_<7-yGeSec(=b9S2b->A<WU0}eRAn*XjKv%Qe8NI;X1QxF3
zCd?yH#CxGiSevrvyf0)%Ha)3Z_-WcoCb`TAAwpKTXkla=$XDTDd}+QfopnnJ46`GM
zTv1_8CY>Zr$RZ1>ThBO%+4cU2<6dkV>kHU7zPrLVne_e+h?MmwlvuABZmfzoO<~6)
z!XO8<*kJ@p+OK)0&~N+osHQbOMGJ~GvAJ3F<v+0|OCva^mey))^w3H?GI0@wA+r01
zy@%N#6DFosW;X$6R}r8+U%IJ0N`4C~bL5?tRwKZ~pBnL2%<Z|pJwAf-m8aXtmT0RZ
zZ-7ui1bdL8jN&nkvQx3x{@mG(4Cbd6eY@Q7Pcid(YHZ7`pN1<fDqgry1j+b19C$`h
zP5Tq#%_T#8c78LkyZRC6#m7Q0h@3KEq~4zv)~qg$>-k7}_SPs;=aJOkz=|fS(Rte!
z!@8r2wj(!<VnpVWCu17-2@rHxt02~7Z3Q%(7>w3EEN)Wjk&q)Laa66u2$*Y8@yGz>
z3N(9fjdEZuFwR_FX0WJO%sFx>)^v!=t9oN%Kc-iwi7a|y*AaE@klxKb*S{y`Z)bnO
zrM$C|>;;+Fvr|@27<!_o94nZ86Jf8{6ZD{eDZilI(wn-_ZY_57;DUM|pUYq+0aLCj
z&Tn&I%tn2q(P^Z~{QV*sE{{#}XP*RQ!vIA4u>qp{UuR23xQ~d|)S~TJC>s=#57@ow
zv1(hqA{3VdV+%pH4i>SpB(Dj63T%xlL2BW)uKisnEz0~OVyr@`I`9BRghmD)hVhI1
zoex$C#M>;29E)g!v7?(P>6}cMCheyX>B@<r*wHzxgXbLO$n816*<1$;NkFGu!DWHA
zDuKP<_g{q)uHLt^@M8e5f~(rv@{NZg`0`n*uWi_bIX>>)Gl;3B5r=%e3Zky-m{>k5
z1PD_UpzgulB99L)g6tHx!x{47vZJ*%`F_9%{h1bjuo)4Br1f?Z_)3Kl!#e8(6n?Vl
zOfp(-qim(`U4^qr){ND>**PT36$7A&TpVuAxBcLpDT$oS{n5x#=-0g<k@=xLjFd9L
zw!l7z#-y)4W*0g7)ehiAeBL(b?-ZgYa3n|9r)fXosiw&P0GDH63*;dQ2Pa@)$-1!E
zOFnOjAm;tGV=`>n!N<+8YWjU?;Zep;XpbBv)t^Dkm&#}SIFIqO2%x3n<*EMSDct+z
z&qkH->y07&=8*O1A)<H4`+IoJ7%QShyk3S94pYE2Jc1-2e?=KO)8b5=bd`gF^}ZlB
z?r8gom)D~~A`qQ-49L@h<|6psp?=*hM|&WC^+j4%lU@=v0-!8%0A{k6wABOtyCp_+
z#Qi0d=VWkm?d~rTgEoPU;*lBzqG_o;VKuY2I9GK>IcgmDMVRxJ#xdUu`3hIg2sP59
z<f#0m30g1#$>j%c!o5a>M?q;Hg7|PG1m#Z|GgIfa4XYSo;`PP4;%CE<DJe(5{^P#c
zB5Wp*LmKh;JiGGXzLxqSAtwIMIQCKX?b5O7Dm7&Hr4#(Ysw#fP@=`5<u04AKYrM7>
ziVT?db>UEHlj;DfLigP_BX8Ja8m2CK2m0a$*u9}W2u5(WRD<<5oJZ(J$aSMfUD}8(
ziQgd0Fx<$YfQsz^<4_72V$O@E4!bYprE?^rIh;5vu26;-cQb7eV!0Cmpany15y)mS
z=+lpp4OlG<(Ho0N2&9FC?&q@4HpI|DzA^B@BVBI8wKh=EQ1tNJ;}XRH<~p}&{*pTL
zp?6;mc0VG?mD}En^Mw-ZM}*vh>#0J`Y$Tp2mM`D8dt8KmjTu~0rOkQG{#OYk5^l#w
zJ>eAY?u>f(tkxPt1kBSCPM}2{VAZQX_EUdX2@XW1BU31$nc+D13lXNhi0^nQ%rL>2
zZQ}EO@LXN_9%<5+<@3YbATxp{H>V)8aQG-9<^mW{bOV&3$?147*7>yX?q!0lbR%Qe
z9t=uM?Y&}V9P>A}T!=AsUYTL3ujB5!B=UYm-ZbAV7Vy2s!j<HVl?Oa8Vf}2{V~A+P
z-W%#Yz6tZWn_y)@d4IPZ_8IZkfNlPK-zCIjqJt;L)M{|ZR{Pu6yF+p61>^d=J}l{j
zPZ%Rr;DTqjm~Nk0r06!1G1M_2d}|C48T3AX16XX3Py;O8gji=uPDSCoDUic8_Nj5K
zkpM?5E7BaU<Wv%0&<r@nUy4-2GU$^BIV_c6Q#+{h8ECZ{V&qsAJtA%$%P&8$+AB${
zUQ)>XwXQ?q)i*D3og`M<oezdOp?f5Oo6eUlkBNhfeq|h((SJ8KZM2B1V8Sh2%!gsM
zhRrl@S1Hi^6b2_n)nXpXL88U^g2C9DU|WKsk&l3_<>)-{@j`uv0o}0Xbi4{@;O<`A
zO(RRlw&**rrPfJmQ%l>HtCKksYisLLfvW9X5!*=rb_lImruEae$K9+Esset$Cf-iw
z1unYe`A?cpdVE}P)X9|+eDYK&z;mf}VYbFj{aeMne6LSNr!9K~7&hH3l?d70^WnN&
zx?~>Nd^lM?n2ywYeg?6;H64{))hNf%rLjFIB)$JM?etZ@Lxl{El2T=3$h1{|Q=ssB
zK0;RD8t6l}*5UnrjYS!K+8~Dn{e3tl{=%(L(7U5m?e4lNAJ40@qj5bY^9Hx6MqSyT
zR9mkzt!u4A57gfItq)CJoi4Eo#q-b63O=^~{@!Bh&tbpt`xFlK#|aFQgj>lZ3=%yR
zW;MnB{emCOyqXVUj<Hc>kB@(SuM5Aq3N)R4>3mmuDYQ<~aUkpY3`ZVi(kY4Zd51z(
z2XDT3^q!xI{G~jRLy}(c<z&wVu^B;-QReN1PfUd3`0`})QrBzDI{`&}djFUmI;e$Y
z<bu**2^YdvscAS9v?wYbiA@)c3uH8dO#83Begs>V2C__<B@d7Vo30js*%O<He{PD*
z24R2%W?+0`o_aZH>8aUThlfQTo*&RG3bWAVXOx6-uf;!m%r!>(JE0Pv%+xx(*G#=9
zo4Y+6aeZDk@|4d+SLy0E!k?YK#fJ0*7;dIyd{|3HeITbu_Dy^u`ey1fVvU47L|W!N
z!A?!QQ-%qpdU>OZg}dGzeyR^eHTSXqV|qSDh3JEtYA!_*xtIvAK`jbZ)e3#<(ZAF%
zo}9u7O4xN!z}qXrA-v{|PP=BAd|JT=JltEj15?juTn<<B?QtB=_we6aT$pt#2&-&4
zneBgibv(6GoFQQT#H(#RKcw<xXU-ylQMwV9z~InwJZWLFx7b2dQBpEn^nk!^Z6F&m
z_o*7*4tf`U^=ERn!Hvj+%=+{0mn{LPTMF7^Hsa>${n}uzMJ#LhCrP(S0l9PBf$&fl
z)`M<xzQgguWU9*QJ?;lH+)`DNp=bM%>@$xRUq7S`3H$b({G=w}*F-OLr+&Qhcx!5I
zF63m}C2YT)U~(~J&x08;qC%6@<odm0vh`xuEt9bi=-B1F-dS=*TQZ-dz$a&1(^7pf
zt}iD)4U0WKqU(GALgHf-7go+2+9MJBVm-xbw|NsWH=X<Y%UpDY{6FWrl5iXKY@6m#
z3`#do17oA}4`JMAUrv9d+>WlS=FeN9zpCcYzbzb4StrZ`DK>ZFGz0h+bJZ&p6)FuY
z-8An2@I8dgeg?H=2wFVeYrUvy*_pD5Xkg}sS`pC-)oZ2mIfPy8d^}MMH~C<_@c5%=
z-lIB49k;$T-jkCgrR!<v?~w;YeF&Nj<FQS*sj9rZ*tzxL0<Wv{^#Wari#qirmdOq>
zB}-=uEaBYOs-2(xnHK0mQWp7X923%820Zt{Wbz{c#=!yzycwOF<yh8Pa<ej~#D06l
z#%!w#goMk37~-0ZZ)4NC(DCh<z*oD8%yMt+IBSVpi5@C9IWOI&J>5&H4dPghJ-B&t
z_f}wNUiBUchKDgk_%g1mn9p>_JFO8U;ny%>6&)7YNg4RwRg;kF5!>Q(*eC@qIs2I@
zoFpI&GIXop+p)q@(Kjh%;<y)<2^;MJE22eO05}9bf<D|^D0H4H=gM0p0w%e!z&Rsz
zRb-kN<#vIemnl?dt=}4Q)%mP+Rcm4=veWrZ8PJ*a<7^Brq;lILN1J7GQMx#yrJrMG
zF3w(jtjl`+Fq1)hNEFArPc2uy+yxR-mVP=#Q0(*Uk|f1hHdmKK*)=-w`T*QFsohLH
zYIp0tiztoQ6Go-;Rmvt}nTUIGa&wl5QMH&ISn0TsQ`^;v`1guO;<&0P{Gm$Qh04@p
zTlsR-xEt1(L7PuZApUp(F|PB*rY+oY{*w=WqJv6E5EBWbl!W}QL}R$~9Pxf?l860N
zFmu7LIn>j$ymdUu?rA&BP5LI5TI99_2;F!P$Ua^PVHgn2Ekm8NNVE#{t$Gd0koB3Q
zbZVf?*y%ery#0B2^|1~U!QA^#Mgl)|%)~JxcIgsA$$tYgfx_YL>c#UfkPC_gU})fS
zi_kWrT}CseTG91kP<tqFf125j_KDSgPTk!_J=H09r}#vRt;zTC_UfkfJ6Vuj<=-j8
zuZZXEg>NqQ+8oE<=)_zIw_W;1@syme<$4XwfA$+Z=w`LyQMflor+PqK{7j1O$T?Bw
z$E`a5tx3@OcrOOAQ6&|)6c(7$D5B%dxT!(mNham#G<P06<B*wM^dOc2q0Wd1%;kGj
zL086{2p+gZC^GX~L3ctP1b4JEvLph3kDC<Lx+15TpuyI2BF5g78$@)FpEo~)Ff&~?
zk=l)X&hjPCq`@qyw3GDFzGnmY!i1*Tf3pWsX4F{f6Jh-c@Qa3Lt7A3c)-sckb1J3f
z%mLi)Gd<^2S>VU_EVZ7{lPNcn0h$_vD$!r670}g81W&{QdShzeeQ2GH1iR?u%g=;Y
z4&!jiX?TQ1$~{ZL;){8mBgCqasrz|(zCv&G!pzUU^g5gbJJR-7`6SA%+IcyLy6fSz
z(zfT5kOvsn#JttBKHRQPThIzyyGwe^PR#lZE4=^y&Xl0EUO=XdW^SZRkCn*4AY`fH
z(K~_0kj3U;nr?q~sF_IP%N_XB0Gm2FM)J<cLz#CMKjMWOr)C^lx>-V|S6IvR3Ov0o
zerDQi`{eMOR8~*PviJ_Yl06Y?s#ChT^b46e?>NW|Wuj;r?J(Eo?ru?S&i<jjXgn=u
z{CGGv?p!GXbOyO3*oBZN`-ni7ZBXj+idiZ=H$q8U`Un5;FpKIF*;|$b-q(RIg2?Gf
z98n$3@4E6awG2OLN#6&X*(#4R&2?^UX<=?{p0YU-t*?#%i|f?X(W&k@>%vZt$LE!7
zqU`FYuZ<pav$}V=-;FSFQy9SJ?v_%@`26@5=_=EDXW=I#@kx8)Wi+Gbr*@Wz={Guz
zTR6?#*p<ZUbr<BUm+bQ^;Bdr6eg!RF*_X~grz?=B3Vrsf{z1rEw$t`Oy5QsbYBQEO
z?f6wyU!^bAJKgj?Gop%X&%3|&iX*C)B5uM$OnJBu(@m{wy6t>pn#6U-c)QQh=XhKN
z$<q1{+f{mL)Aes}pPtt4&hp%lV(eA}Z?r|IUlGT}E1<yiz-AWc1A=BdA99`UyEu!+
zk$;4E&?(Ra`ukY)ehpnSr~)jErq!Ahy%Th0$69gaQFFjr%`#6?5m~+_BblNo?X}1h
zAk7hp!iDs7*##$7kQ=ME#J2DZ_I~u!Vxp#=6Ai{6E_S89#XEk*rK4j9)67m%#<S(z
zT`hQT(f1E|6!E&h1j<0T{5!|w;LLP{Cy+$u7tu4w(O&J3M?QcEWFHHwhG3eWV0@G#
z%fr_{aDK-9hrq;TxzP`O#S=!r3sx|~{Q@Lhd5`d?Xx-Mx7V8!E#7~|_xI(8r9xe!%
zkSQK#>TB0@v?!#cw<Ki^94}M>Ux+HUUuexjhvHeg?vgU?j!Y)?eVsU<XgnpV6J8v=
zEcilE?S$%(5+a~h;km$P+(bImGNZfdV(s);Nq;a+9ORQ7m*WzOa10wD^!DxV7A-sB
zUJfr`<5qU^vLUMwh!&f<`lfzZe^Pj<uZF9iS^s2?8HzCrC#-~Q@0L6;_%<J~NEg;I
zu$$aj8!vBp>e)lw8Dy72xjcxUe>&B|3tclLxXZ=e#A$a~5XK^6cydpMO0|wuqZ-e<
z%je!>D~UhCBmHGvWa<jE7si<t|5W#UD+NSgZtx|aXzkb(yBl}1YA^mAUp``Y4e|^>
z4@VCcmte~6c-$frnDjbDQ%zA~g_yXSf{e*n6K_zn#WYy$2tRUNDP0PXu9T(>?z=l2
z3<qIFQY?TRR|nHPoUPD&Vwaa0ZMmsHrzaN<{rcoy^8cheqv{Q@%=K%L^{%5;UHb<d
z!@z*^gUEEqyu?tUqeN+2ZZv-ScgR+APX^g6OLJfOmO7jq{wym|g4KiLAqq6HXcxlt
zlkDJAqFLNttg0|BBCIr+f3{jp3B@qc_!3vQy*oosnN2bzdrH@gCayf<4B~&xPJ1eS
z-@M;SK<{69&z445?KIVOuDl&(iNwHb^wa9C0iBf<oi$|0f`CMuKtP_K`?RJ{TG*|z
za01836(g+DTP>bD-RD}Iym=s~?E!3;;w}H<e9B}bdAo)QgPa~@1;DX(3TBC7gvF*w
z$7Me;EiN(waMXm;N0clby%YP+@z>sbu8z{uXukc7DV0`Fg{91a>ZL*IBP>1Ekv7mj
z?UWMxERzYFVHF>#s8?NW-ShEC<+<-xs3}{!AB|h1twvB_<qHMf?t}Pdm)j28ia$k&
zPw+xXbioLo1&M3caZ!&g$0C;W@3(&*o|e_9|BU57(cJ!|IIruG`L~*@^f7991ZiZr
z@SJYzLnPu=@>ilcHMY?z341~NpEa=9y$976MG7z8PpC)*d~eVYXv&LE6fe1R1f1*I
z4g9d<V(c9~m0lS7ON_bNqMDtDH{cE_f*w15Ypnfo33(fV#IqZ{X4E-dB_yid8uPFT
zFFIny0V0%U#ptl#%>47d`2kIPk5hM$IB(0}By)@~<c#C`i%t4xV?D;SBf&zBvBB3k
zDUfH<20Tg?II>!0&9bN04QSjl#Kcd_!S%4|3bB#rVf$knmmlzHL(DaEGOob0{^1M_
z33{Y(&T7(6U;!YdiVe1FkSgwlE5gzVnOgEjTPJd?{3HbP(e2!$`STTR+CkHk?|(D8
zsGdIpT5C*#ugx5D4!+g_LIF8nMuK2B_$c$j@W`Jvb4jDjA0y6!4c6TsL*c@J#C61+
zOmA_)T=ZI|DM^96-Z;kkd`@eudsAlmEpn*_A<`IS&@GR-xhX0X)QDz|(V~s?eOoTC
zmG_|f=0S4Y(IoY;F5quat5o?hKZdn}WBg0>GNUO?(}rcuMFJ^Xckn$ors1A)T)I9{
zMdEMtc72k#jePIS>j))y;JFoM@cF2BS|E-09H}}g0u`%5|B+}IdAPwO2x5xLa(x$#
zzn>!SK0J(N)yGAOV8y_XufLG193Ms4vXnE`FMN2EU#5Opp?&;MkuSv~r0%%*3rF45
z&}JE5UmPZp!==r|CJC}X0x}riwV<AbUu_(Hy`d?r3xRrfz>1P7=XrSAh@oi2_NFpR
zO51zR@DbS`cVdm{OPnVPJm3=txCHh>p8*f?5Z;CenU!3FfPnX9Q9eBb4Ap^zj7Q|Z
z0*-D(qhpIJaCpo!M7W(4Wk>7oQ+?%qXDM>CkYA%~NX?OGoAJTjmXsI+qPC`0XpwF}
zs_#6_u$`C`O?C*Uk+Sk)hM8j%3<z_6STV}`B(2$p5U>0N)C+KFeR#7hrcaJHYhM{e
zFH2U*KYO^Ku^(Qqs<{2(8FUS#ILnjX<y^zqfiMgvuCTxvLllL8k^n{wOXr4}to2J;
z(2#`Ay&P|rL-8;4E=)Gm+cx;nC}X-|8zoGD7{kN$yX3h@?O4v2qC8|A?dTttO!Y0C
zRjqMRt;5~m;rmt#?KoA0t1SfIaw>SDKI|<Qwm8&t%rr8${xs@c3CD%~KCEDBbK9>R
zI9&Em^-J;z;r<iD`7gR!D_K?eOl3XsyiG_-Q5a`tq_+X;V~v+LJEZAVEO}ZikXZ&+
z7<73+SMtbXrT0!+#-GC!1GMK3ef<Z{DwdciYvR5O@BL8XK|v(t;2R)}`8@2oBS$j!
zmGg_x`6Y<IAh4a>s4d4^knhK7CaDMC4&3<Kh>x-G%RlkA10HXF3MB@;7-B68dXF$b
zCFC?xRY)IQRTJ_Sc&nbMS`6$RsrSW<NaESbTe=qtQi(7wam8JMi8{i0pzn|Jx=;6B
zgNN_3TQ6pbYWYh9kyV0+qoS1MPv|Ic^)L9WW3A7i`TIrrSiX%<2XX@k*>4+(Mv<n5
z3%9-t|4Rl@LmFoDd!bt^bmtudJ@yfIyh;;k=Yo+2tPFXQ&^PZty<{xKhX2?90*DEq
znw205#g~DJ=G4f|%QiZ@<Xtg=L@6%uPXeWWjnCtebOn`yIa)u07bo^3GCJ`2kgD6B
z_UxPXk4EXjY3I(Tc@pOw@P5;=ZlR68!?*3d{#AN~i<nkltl3nD!0M$w!*=7hw_BY`
zchh9zztC@PW{u9|fTlgNhD-<(N{*Ts<Vj`1#b0@RO^jO(h?!nwoXd!cRfeiP*?IHl
zx>fa&Ge11O;F1;A7y-Pze(HhV2q$IP|57Z|#~NRc7VF1B56+cWf(fdxSMREqI*(yN
zvA>DDjqI<PRnX{hUgNHD)QCc$=|~~&r?1%<@ffCVi01Fueil0+0dHR`;+4McEqj;%
z5(=l^m|oa;61*Ju#5l7pOm>-~6fWmAH0{TZbB-z#+M_|IMX$l)ZyI(b(hQ!Dw=?Zi
z%h0KcLtmk1uZSsjU}%$A(~G%6tPgUb&@!NshGqi}tK&$4-wt9p6Sd`h9D>W5Odxvw
zxFbLLEKov)hLqNYDPl6q5Ai??>K;mE^u<h4!v4V*2^g*{6}R5E3P6)y(5VeYg2D4}
zxG2_b2*fN<Nkd1l;MYtetSF4MWux2yN#t0~lSTT%y+kOVi6_#BkO<%*ipLjMhhD>g
znQe4n{e06lO-cO|G4D_P9=eU65_zz(_CyBPFW)K$^vb(4`{(drG&w@vXj)u)!i4x%
zD@WF_5BDKKlz6{t7$?=E2ryGvU(D|taJHjxNjrjRlgf`qlmm_6doYtdtwG>N;!HwU
zgL7yqQ3cVZ8-q+gTbk4KF_NA|V0uEK^i4JN%>VLsifI91D~%x78?7%j?@nlxyML*T
z+-Ziypp@+->_lZ`z3pNIPgO&LUl9R05)&mcpzg41!dY2H@t@+7J~H2bwV#&o_St_7
zcu3p7HsA!ixXk)Q=~eeZ4PQ4hRTK4z8oQ8!E4wfR!d}b3k`to_=ycgXlkdldaclrK
zB=Bd$&6eH!cHvL4iNufBh<AwqPI)$DGV!_Olj`kk)nNOy(Y*e0eA?75$98#{w1@g@
zYDl|`l5kr8T?HAiDAnfZ=cFP>E+49%0Gi*t5W`XoYkA-Ys_%J;-#wK}x}tQDc3kJv
z<gnYn0NuD`mbnrjM{2e+B4g&7A5#Zs!>%Ol5*P^2YH*~2toint2%YLL1BCevW(?WJ
zsWrMsp1yIbIO#6K)#iMCD%tVtmNz^6bSe*fPlVF@A_`6)MC&a!6oRDGYmCya&2e@&
zQg+HpnTkVp`?3~}4~Uj@cD>Qbb^PlO!+ExxawJ@y{B>^DErnkMKMj3n`avp5@lUz=
z7sOlH&C*o&ua7=?E<nZIr$xl=x`^v8cAIjAtiEI!=_z~<qFg1KiYuP`=(*t9m;P|Z
zIrC_=m>y1i_SSjppIUb{_G4y^23+Y<U>;DhHFmTc7c<ddV2$e7wN4zK#oy3Yg!}?R
zv$im)Y^jwT3?Zb29GdDuJaF0*6mo^@oTmh=TvLs6y-hO(OxrqMob0Sk+a*a-^PAKw
zISuCg+&mb)U1|Bb_VZef+rUELl1EpP6SArN>hfs#4erA4`1RTCwUGy8k~n8UTkv5o
z_h1y+VK`UAWNX-ku@oG_x2`j^g-o5d|9iE>Wf?dU?k|s~j#cWsbtdih4+G#T33|`&
z?8Rc1aCJ;YfHt7@_SnJ(=Qz-slx}&G^#07}=}#2iq4wwcx5s=e%6Wy_ZmydiHM*O-
zjg@Lf2izPDD@9MXnAx`6T%8tJ{T|2`e;-XmOY`=gu<$Sxiv(8!!T40OnYp=c9y7JC
zN2J<Jr}1Ul4c;9uH(^+i&!t@r7~kUM^I$tZn^ONZA3uY>k0>UB1vb}IS^K~7;N@w?
z#7%;%S7pNA@oSiFMaKvNg(LZpD^QN(^}{o3Fp-Lk6J^s__kAm_4^6w!t>5E%WDWg`
zF!lElH`gY)QXT>bKMPY!pXg(puYUI8>G3azmcu@N6Wn%I;*&5;Uu^vT#}^+j+-j!3
zHu2U-Q1uScWL|U>nd0;x+?Dv%_v8Yl>$8$aMfV~9hpx8_ild9Vg<)`af<u4+f#B{g
z2?-M1f?IGHTn7m@c(C9SB)A86cXt>dSa1k#U-P{8y><V5RsCaXs^(0e)4Naaz4ltG
zS>n}%y{~LbyswYv#O@DGkLu>X%pFv;+b7qV4dNe!(M4C<w?Dcq<Jrv=#a#jjf$qV4
zMZ?vKi?R&>SRuJ$QwU6u@mR(B+9n(NH5w2vUk}j5ELdmy-C27-1HhQ=QT?+0Tt)lS
zS4!6zRZTay`q?@gt%Er|2UaE)kIqmBxb_M~_|T}}S{R@8Dm9W+CZ^qYphwm7wvtMI
zg!6@g*A2O#)nI{#|3!zxn5WY1FY)32S;fEdEM<XGfEtXkaQiB0@eTE;7Y%%dvRvR)
zlOpGg`3q!8+*BidRNb)Zwrn>?YME}#`S<F2-Cs}C`>OVL)LglNaHJ9wWj_Eh>yG~5
zt?7f3n)+TXT53iTvmT0kbYJt_)d5h;4HJmnTFc^pP%r?h=&t)-XS7moVU$t(-Q>_<
zy~^}|^29Lc3#mscKU}El{t5)d<#F@LQriAX)kmRnsBIE((`A253MqFKUTxgXw8;@(
z`C`#p_jR_K80bH&IuEnX3GSwwR_RNjcy9KD*CR9nkc{?LUsMgB-ICdEx3*G=&+Yk@
zbH4awn*`g@u6r^bSnN}FnC#cAE6+lSID#CkHkNG8xHiRdbZMrJ@<p=O^~u4Xz;|Cj
zjBsuneKt^;HS`|=)&U6XYX}sEDIZuuhANQTgL;jqJ5UOQ`z}WJta-Nbpef0d+50rB
z>1I4{Tbv%~C^{<9#GY<Urz;G_gTm4vE*3B&Qh^AheY~g|#)istx)_nEY`qmn@}^%$
z;B*m$!l`tiy@2HT+i^3T@y82HV~vZ(yY6!l1>v^-;;|7TwGUu(gjK#5yW{*UTIIEj
zAWpj6xT~8nW%0)uHWXE9mx0$5ae>0Y_N$l_ir%Dn8W$@r<D>!|$k;{u3srg63za#m
zR7vh<y|0!#ocPoVK2}{&xQuYRk%mE+4y><2*H?33pJEd%?9^QHC58xLIDkGdUl(48
z0L)CB-}G8+zMcQ?J&9xd;Q0HwQ8d0o`z<0*00OiC9(@9^LRU$1Wdx;<F!~>YG!x9m
z<}5etj;6up*zPca!GunEm?sCWEsC`rICJ9BM%gwekHi*S4Qfbc));bY37mGy7}LrS
zay>KvQ1@0+5sykq_v#}LWT7v67WIsi6OycI1yE8^uUbK%J@c{E>vH7z>AGD?mTt+K
z_?@;9hud_fzM2)IX5e!Fqb;3@V(0s#EysHIy@^hB0Iv4|rVKz-nVfrL_0u(?{Hx#u
zQAq#WfaQrYyW|5~#p!YH$?~xP+BGU0ay8xyO1v*Y#yD5*uqB%cUJ?`B`oUrkFNuY|
zKPM9TOKp&$!MJg%Pb^2tpf?8a-@Zh@HKyh1I;&Q^1}Dd)MZe8UV3jbGkay;S(FIip
zB+KFKx^@iEQMNhGVG1v_W`xa3F$l#&o9dPF;VRC9|JH8Z$hk>h?@#v!5?C>AXm`G#
zon2dgr7s|y6yfCBm{T~=^l5S3isD~7I2@DX8UOMG?5|}&Ff16fuT<JC^I>5B)M$1)
zq9w3c@P!VTOY-wE&R6;g_!FlKHMew#+hSN)>LoBL*Ef%)W}32{4Nx}^FYe1(QVFDM
z0=Lm&xKx;sZkQP{VMx)>RXIz9ewVJ)+UD-XH_FXh(ZYwE8Ha6!PMdhmYl2_!@pQb?
zE`TQxFEPc(d+m&WV{IME6=Rku&Ym^HIgIj1$2hjiuyY_H&9dl3rUt$Ce<s@+2)iHK
zt^&9%WKiF5X9CjXdaz`wE+0){&D-UnflZBSK9jlr3Y*shnBt)^>hF!p@lQqn{Op<m
zX!~e`Jnk=s2#RhJu%-QBK1IiV6ru9Vr^Y<GP=e66dmT>xrke0<#gR!<U4^%1fS4VE
z%q)CM8?3_EQ{xFn$d3;vtpEI$ALA)NkSm04A?<5BJ*)$vU@hzfMr}NVqU95Wv)Ovv
z(o-r}LUYt`EY-I-#m2toC`V_=Tvv%2Tb{ID*Y}${Q<wr9N?Da@Kk)}el&EOLX_t7c
z47KP&8Lh^Q=v!=&QwfaKElmJvey0?eBkMvdXpK5KAhpH2uRfi#ouVYdINg;OQ`#H(
zDilnsgOcfB{Q^EW{ry$)n`{bS#9+`&m@t*poSkHv{Fkvvd-0Fk=)IbdJr~IufsJr}
zs?MaLLKMV?Hy#-TQ#Yc~W`pT6+b@<eH%zA#*xUG2O8>hcZ68)+-HC4qd19(4#%3&d
zICV^U+<nCKm5O_v-AG(9%ssy4%Yvem{_JM?+lv=4suPAs8=T6dc0B02QB1*(Hs-qP
zV)iq1wl>cDYBKuCY&<8Ye(vS3L%rHjS|I3x+;10CPtx)H$3?Ff2(tDbwv2(kqC>0s
z<?(8>#8;tT(5rJiB}iYG)8pM`8PGgNwr8uRINFCF=~%o;=a_BrbU8Tf#%Vb9QER4H
z`^e$-UOr2i81~LfE=I8U>&@u~lIs}*B2GoaygZb|czon!rZ~YjuPgO|M~P`G0nww|
zWkM2Vv40dzA4tY34D6XuQf@zkk2T7Gy2iSo1GjdUxjtMRlk1sC0d}uK?YV{=*>UEy
z(@wD;+Ly|x@9Z{dh$2va_0KW<Vd3~n$0dD0W!vv^R*RTAj!rR^uJdG-5yr{EKI5t2
zwj&%$?E`fD+$JB))L(6}l>Okf(M^GxIrRUYkBxE9ReN6;v;KM~$LV)x76~z))nq1>
z1WK&I@^CVHLl$y?({IfZ&9Xytxe8A@Y57Ec!T-0fmx;mlyTb(N>`l4}{ooN(4N!U)
zb(B9nV(+o*q(-7#s*q1DuK#{6j(An#JylS(WgBSO5uh|bb8y|EuE`uM_c|i_M8C=7
zNPjzemgEhYq@gmMi5)8>@3TT=|C6o`W`}*ewp!g+h=01eoHrzw@sr{sj7zU{$VN+*
zBwy7GSQl0bYeo%D$4xmLaTi$|@Oz{uADdS>C;ILa^2V|a$ZWP&Y}<9h9O%(`1Wb1J
z@?!YHbjij%LR~BbFftuzHW7bGL|*+1P!_xGVM7?p5_S5U8_C9%UNt_b&C}LIAkBlx
z*nVm!Uz+AxEe1v5O;<-OocedPJYV^4u0LGEhZde{SDk}9%lu40ASkP!b(jK)BqI4e
zL>*u;am`v9?fyw~pM<^%P$H%57QwX=t?bg72UFpI`~zhGI;WEfG>s1-<OpR!EXkJ{
z#@7zTk$0cH!9H;8lYbcaTn^VSXOCznU57ZyJ8#jJr?_rp6(|}+iDSbb#`VMDbfKWp
zWtZ$@AG12pD_e+Bm-;?|>!&3UnRW8oJzbemx<wfWmVk)bG?_t*F4jmXDeL`go@`+8
z4V&`o1f^Yj4W+3U#uDDbw~o|#j~y#X9o+#6B*O5KbQg_uy`dB1NyDF;K0W*Hwn3Bh
zV?U@s_8^ze#DGjpM%)$@0k=fsm^B(z6nvdebTCc&_-(-}9OApcXNXg_`03P8&gnE<
zzdM7_^2+d4vF!<AQB>}2QE@^iWzMh8U>a%5h!YM=^=NG3^{i-}rJY$VL*5j9vps!I
zDS2tF8p;gmS2eBCI_Jo%16N$s1R;xa^%r5G)#M*LxKbX>%3IDq2%}pu4n0ewxZM5Q
zM;d<?xj(EqG8gg#%OOOmeiAD58DnAySZuyIRFy*_b^25Deo;)0*qUV8N}L%zq5JUC
z(VxWn0SJbXVfJ=gd8O27^Rp^75)^CU@8GYVBOB0hnqHFCSZtf$B1>!l)HtXPL!-Ew
z6#+-b#NAd%gjE>G+j>)<{*p$9Z|E{?;hIDGLRR7IbF~wi{Uegd>D`JkOtuo+2Yw8x
z`QCp{osKBrs<?7o<*kFJErvToeCq9&a(%)3IAxz%dds4VQL$(zpV!nCwZ`4ZmTEc5
zVnPvHIJZKdqBAJP{aQj$vqdHQQe98X!v6h(E9i6)J332DR~<hWP>Lsh&}I-5g~N$!
zxIV*as$>-#W16^0p`*fJ96zo43={-yt->=mhGMi4TbzR3tE7yi+Qi9}kC&-JNH;-D
z37}$4*c%=rkjHy*%wj<z5(lCiy^ev?+W=EcMv&NtU73!(H8mp<g_*suYK&M_kqUOu
zA$2o#I@%b(QIaWp!!@Nt52gwYnun(}S^l@``u+5pd$0LGep@R${-Wcx5ef@*d!-wX
zYvUUe%t_%}9Dlz+JuY?{B*DsEx;qE#wwQ$@`Y%?e$|U5m<5o%-q)BpV_K!0&2j{Go
zGyB)?SGUBB(WFrpP|fPc5dTcK?GK%e>l%$own#h-pACb-Dgg!shY{a@eu2OGDT=(u
zM2S%Vx8$0QW0cIto(^V^5AYb*qx++S+2Tf{8uBQYMt*kt#)CivYfrsT*ZouePXsS)
zqhzs+KGTj02~92{*l3T$1pETFSt(aAAXXBefMBw*FF?%r`Kc;QIZTBGtaC19`bZd-
z`lN{ZEAb-{22BEEM8D-G_E;i;@^Hko%Kqj{KtR-{`#Psi`NDVVEn-mjbDwpNrXWlz
zJL8+CMn~}*tLDc{Ba!BDs_SkAP(QNMiW)O((tPMfv>$ct8QuiN#2Fb<*W$*)Bw(Tf
z9hekMLoneKDn^jF*2Zs6DIgx&{5^IVp!8Z?nzUQ8A3=^n7+#4Zx8{N9)MUjT<NL6c
zgVrfI6IvfRuF}}1C4-JNj~kZ<atrRlsY1IRWZ%SK#{sP$!I{sc@{B(mPiWMnkL7n)
z2vd;kbdx2znYU&&dJ?FL1(z597Rbw%Pp&cmiq|6(n#jYPUkeho-o{U-Z?7$dHvzc-
zjk{PT?e{bJ3~riR`^~;6a&i&w(M-X8%y^u{{t7#f+4N57l01!TUVI;P-5>?Pvd6gL
z>U}oW0%98i1^EKG%>5^0dS|@TWzF2K@_?~po=R~?DF5_o$FLFU9>yA+Yp18tyVEYn
zyCCOv_=cePO^HpFBUDbx>W?+v6R_UwScLsv$q9b+ZHa2OvA`sR{gA)RY5*qvQ3J-v
z(?Ur;E_`Ryo#8(8h1zXQaCgDM)+p`f3I|&S$J}RtZM_`#p)bY}p-Vb(W44~ekXrLi
zI6F>8P7OfG1LKlfuOI|-eF;0ktn2Q#E6gsR4H#QSFnXJ9I>_cd)^s^6kYk%~nD|pw
z6eo`r>Sh`5DruS=6uWLy@#L;yZ-&IRkC9&wC>~%LAPiL=I`!9pXTuk2;{92DppYzl
zj9_iEhEy!;R-3&aslf3*tm-eV853eTXD7Ylvzq9T>fy2#YLovBl=oMMa&5GjF>_2F
zM|u@sS`SHMePTG)<(w`?`SpA#7c#w%t(|uAO%YJAYb|BYc^y48(;x3R##(&-#K|Ah
zi?Fp_ZU9wx?I{T^M$n1c*<B@qBN^$Wsrwnn1S#T@W9dJ%71#w=b6o{I^x+YuAgH5=
zUemW~OJ!Ip7J6~4C@os_$M3sm3CgyYhM76x2<o574Q9tD`OEzZ(8gG3Jz(;p;!J2n
z82QSjnVe@vS*3P9RMwmK6}93<i*!;n1+}PTAsZ0f083${3l6|?fI8o52_SA*wvR6?
zcFafQJYKOhr`vUA0d;wrYaglQLIp-Q_xzz&uoHPyOalprQ9y4Pi-o>!N|e>c>#i$d
ztJ?)kvZmoO-MX)fuBX0r(AaYg(|o-K`>QQ+Vm5uN-PVKh=0kaZ4jfk53Y-%3PvuB8
zaQ*EDI3&HKIM@h&WTA>2pYPBd)0gQg_2FC4NJ4BeAFA()!d4GZ?lp{m%yQ!%y?5mi
zqI&PKbzzo|c_sei>84yF-zC@giOI<D_ha#gIt559I!~vj8)PSBB{FIfxUfgF`aUj*
zhNH5=0Ms@Dc;+7J_ygbfq4D}?vl*WCHb{3&)=;Oyqm22v1RKyiIr`wQx!8zDcP|n@
z42B*$$7A(#Yn18ERzf*V`v~c`4vqw{(W7%kuM%QzjP8M}EZ95-tlGRE29hC0pAS$l
z^LQ|%VNoMOcl#F2t0*05?M62}Yvm=SSM;<)bCH8Iv#A$eji5l)=MZ!{6t_~EO*_er
zNMY}6lwyk!?m$Z47cqZ7zB#4aLu;w0P?zXW<c8@KzH22Cz#Gl0q4wecdo>u8C4e$n
zRE93d)FFgnr4rnS1ul4Vo4&pzl)PO!(9XUFu8GN9S#MvG^>>|UyZ|w-fw4f~B*&FP
ztniS_e27Hp-%E}A_=DZj8^#EDatQYE1%IW!=BBjY{}hT%`4Qkyj=nPVF+S}u3_7+I
z7CzWQq;4yj<Fwan@u+&dir`7Fyg^ucPm-#S<RRJiss;56JcZ59w@FU(Yr(weLO40e
zeVH$4bAgUmgl>-d$0z~O?EQzH=I%K#R-;%gA~T6baDIcigs@F^lyoOr!0x-!4S%+&
zPpSxJ>2M@byE4DPHCR?EbRA3pGSKt0qr5~{fAPCE`tJe>bw;kGgYMkc9~h=ccA-%?
z|9pXB4$5sR1bln*m0fHpR1?6;T!37A`2f5Scr+<b&71yKso!O}K4V^fjA_?qa1WOX
z0JyGLx>I5*s7!@_v;N>D^SoYlCl^}_t7*4Rzl~}TBoRZ2o|KdO5M*dctZh%$Dptz1
z)LR7egYk4=yrOKK3{aI8dKFG$87_`U98}nu6@aJgwFm~$3dSzjEz>{Pp^U+GOA5m3
z&k(=B2{6n!H=klM$GAHQb7i-V=S8sxnbGwrsvzgBJlInqPmOvI^rOCbBmoGi6FSFv
zw=j6Q3+q?-)m^1R#m<sL`(vpzwqTDR)<RW3m5s8l)8Fuaca`SDNjhOyI*trSTHp=e
zI^wm<ubsCwFcl$LLLcW+dc)n2W4jUng1+Kvm=vf<yS@Dd=yajOD{>fq+w%|<Dyt|;
z;kmW`s<bdU-51i|{B0<K4Un*zp7ENA@F@*Q5>9QVD72xoE==uR8XGvqw=a4YU_2%D
z<5QM|!`L19t&np`r&o|I;IKn~Qy*f*^9sc{faDks81%Fyo(i-r2<z24>f@Vkn6O)5
z{$ihG1yGLehzMIvKelwP#wM=A^S2d-H{@E4RaEVt=BCyC=iD0mgbtM=STr!xt3)9D
zDtf~y1hL*TLehoZ8|=p-ku#R-yNvR7reT!Th$(r6&*Fx^<2hsCp#CCjfyWQsMx=K5
z?d*9N7u`9zzBTU0<gXnp<l(ff4J5{^>&XGk9h?C2gKs(Gh~$A!0cxp)XZo%>jm<{b
zCOnia?VO5XP83R;*&(T)(TCE-*<2#2hqL4%G-PI0-G*Y$s7vqgg9HLp55I|w8i1A>
z(Ap4!f84T`$gS8x_Y2W1t-y&?H!N}s{d?>-CAh_zacL%|qglF*MD5LW$q?7@=6ozV
z)}P_Z07by!oT<ZpUDAlnNG9yn8cxTN5yb$~mqlr&l<4(-nAa(p!pyhVbd-i}Wy8?-
z*T=SQ#?zA-$DM;-<mS%MKZV)c3vi$9TqXkMitU5{j(-HSsunw*ENo>K$@D0=Y?prn
z1fuZ_HeN>-BJcr-aqFXTp-}UNsiH#8%CZ9E`A?DvQ@vk-5x3SCr~3j_<W()m!D7X?
zr22O^h}#tCdkV+!R4BHJB66X>@r(qkGtmFQx??Vq!`7gFS_nAw@q&FKF*KA8VVNkt
zkojl#BHG;1uC)ZZf(fO!xN?v~j^#QX^qU+Bd3CPI;Rq5U%<FQSvB0^5ZuA-PW~0I8
z5QR7ex7$qYH53Z+!AP<^e~Qy}fX>q;Up5i$gTwK`gYMKz3XZN21(FA!m%S}nI){K(
zfy4+}dpGFD3|qm?Pow%Yk$vCM*kQwXO*T_S>Vr50Q76k7&@8qM_zS={66o}~9oz#w
z)YBjg{sQIYRt-2t*lu6~Y!uX-x37S7<#)R2iEs3Zy7^KJ$yd@Gcko7lNbc$j+(CzB
zfRZB;Eb;8yJrM`eOmq&u5Iqb0@>`bD4vx|*)|R*1HV;a^D0~0z%^#Fn(e@OQu1cnG
zFHfHxSq9Lml+*s}ds>%t@YI${8y&Q`CaQZ0AUR?aLl8jYA;ZJzJl#c9>`DC)JR{K?
ze0{`)5WtNL<bezeNr~QThMB=Mz`K0kTS-nmx=8a9xjTq9@V}oPiCpjIry|eih;G4W
zycdt+*T{o+$p5_GGb;b+4f-FOOFBYWh#LVD7=>zVaMI0zsKfI&X$xu#N9L9{JmqX1
zOwUoU2yLbI7gH%U3C3PFHcEeh)H-J=qYYSBE9lr5Jy*J8khFRNy(MIHhQpxr4Oy2{
z{in|}G;;g58Rq%R4vcexW^2KhZkj-+4Va^XJCc$VLc2nZT=gd~_GeFssaeu$6V%Nq
zwtX2JHv-cdytlwhQN|)@wu4K)c$^t9Mo8Ho67XthsRL*0j069M(&&*rfKCG2v0Eq)
z*G<b1SA79Xdv_wBT5`FrNf8pKEn+_Z^+WQF?+x0iKWkz=IrB4>NMf$sN|0Ch|7??Z
z=q=wgxyTZ+`<kC20ny7(Z@;R@&)k!0BeEg;3C0xt>L7Xj*RYcK8MN%f<47U@g|Gto
zvcqxWResmsyr}~~)=30=-yyH~hWyeR!nJ_qVdtkK4|5T;f-~D7GP*vh+Hk*b+I5?`
zYOLV>R#8gpc7`16P=B`UmD9!kSgwBgpI!uCHJgy@5$i8Q{NlFA!^RUX072-p2`6pt
zjhxb6G5$u)@A8zXt9R{OH#|=^eKhj+gg42L!E2`u*b*?Q>e%a`)UQlx3b81<<KsYh
zhX8$`z;*^yf|RKqBhd0P(2NSP4#a8U)2N!=eJ(tdp|JZTHm~<s8-`fxIfk4iCP-*T
zFmP2Yrtu%`N<kZ5&X(a#`f@9jRb}Q~ht4MW+e58Vk?p(e3y;^Q&3@`SJtV337`4pR
zg{R1Fv(67QHUS2(bx4VXDDD`aX{esT_Qu~ZcVR~S70@269G)FssD~ntI7UNi6AQ+#
zB9;k#!*gUt;eJm@E$m=c7})mKUcTt>urZy(4i$FYbhq6DtL=gDEwk>+a|{*%I#l}8
zE1cM8uO%1FcHW4Wp-SBxMI3+u$8NE27L(0+9c+c$Z|gL&M;?%%*@mnYAVl+HveAs+
zSx8Ts=a@_)G^M|PON6v%laPy*@~>8>j0)C37fS~}!4aij1G~4qG%JO7dbuVNRvL)v
zx(dRcSFY$=s;u!p7v^)o)>E!*Gd{CG-$#n>ddy0eMfSXl8zLA<(}iS{V2o9~9m>YZ
z%ER{RnJ2Rc3da;5u_Fk?Sk_}IPjkk9W~Sh@q+68Le~Yw$Dh9U;hwY16PhJJtTG%t2
zrUJDG-RZ}Ls0TbhM@b!@S<+E1%+jom<I!D1mz)P=iBFe2q#q;Z6N`NQBSLYZbAWEt
zxRB`MlHISts5jjsdnwd&7>PbOI{|Gp1{v}^qkPd(x@c|i6m*wpdnmtxWRB3yeE23u
z-j2zO-UUnsV85ZXc3b5~23Nicf>|B9-Y)BHs%fgjM`wK6xmU04S!nn}^VZ*=4JxYm
zLG8RRWb!Dd!$!Uu?XOV`Wd1T6{eSidPNX$?zV_zW#m~{5x~L1H>c}3}Eh-sk76JP(
za4go;EQt;#v}LqQC_`ROYNe7<b%58N@cSkgiJb_%<KdrmBViACFk<I!a-5>j7G*$b
zB}lJIxBQf0eGWOBVGY$RUPHpvlkP4TCn_^a2>UdLN(11|0u(eoAS^x3wqtU$<WRo7
z`nM%oUi)sSozSs*#AAXWzobey4v)9q1;GO$9djs16rOS6w|Q1xcHRo}JFKA`x8cq@
zm{>;|l3PqQ%PD=OaQ4|l1xVge)`F!&;USvPx^v)^tmCXA)dJX;F+32u)6^?$Indii
zOS&}5UnqDqq)0bge=Hj;G-mk8D({;(Plpz^!30KH6Y>FZ#o*f3$)e>>LW#oAxF}PJ
zqJcwwTL+944k3LkfD48-WCXr)ca&Ap+e`U}0#4zFD0gmW^8*d^4D&v92<m|v4I*8y
zIB4w;;I8xSTTN&{yY;q+P#B#mTVCe>VZpuapgs#R^3T$p6>D$nKh;^v6S|gYyaq*|
zc~KrBObrUu?;8OY1Suo3FegV6*ln2EM|YciM9$+92>8I{W7_c`iL9yC9)T+0_z|Jj
zQ3AM{flhG%DICU!>e=qjX436wjp2hhM+(@p3<J6U@EXVv!Ebzp&rjTwK*rMWf3)Z1
zNJ<il40kg>Y*caR?4Te|W9Tm|_=8TB3B>$J&R-sc9%S!O+9BQ&EnDecG~+llI;L93
z4THq-47XYG!eYpi6sNEqs$eYTKszH05`=O@zyhGA)v?3>#%F?6g}S+FdjFY+6m1Vx
z9H9WTmfvWXrvekSfA#D9zvtl~69;8#>+w`cgpe3qLcpu7Ox@fS<A69`x$CbGaODT9
z9?}3jbfJ3ilRe_7WUibgn64<v3jX$XZ(<rU1a5{$ImIZ7?SY)$U%Z0FVn^HMC5YpQ
zAVni>OFHqdjXeC>lAPMue3tg7KKNJi3sMv&ifNBSwBdjjYWhF+yw*7#PFB4<F?v%7
zFXFm@DYp4pq-{=6Z=_hit0r<WpBskrNCr1@%uzG$45B?L9lqroorU26f2Vl!e4%?Z
ztB%hq>mlEGWJxhPHEzx8I(I(>7?h>Kk25;JU7>&jBGM>B5H>mQ%(b4|TWbOmetlWL
z!Fbo)Gli<q5XcnuIpTXTx`{z=mp!5C8EqJ=Nu(DmqHGKL7uk3Jloe0|X=tbLr&bM~
zcLh1R4~1G2n#YHCjmOe2N-jieT-Gk0OjIW6nS`t9rTwCB7E3$%@wmiB4g)^O6^Rhh
zfoN$!I<y=Jjm_r8i0i3>3+Zv?8WXo|2_Zi8IL`X;RYyKc?MNnIwoJ`eiPu331!WVV
z6F*fXnaJGe&?z%}1N~+FqK7|f+Jt>`u|lM?isKqPUIQ5<7I%PK+DSHdJ>h$JvLwM>
z`#u&Z2hj;P0&<L98=w}qSyOexmS4v6+|=1=GL`UjDsz;00Pi?F05wcP8*zzlK1+p%
z7QUs5J=jBy8C|w(@4Iu$lCu@p3{r&LvZ}4ba3tXojSvybYd7CEL?;u>j1?<#hc2jd
zu#N537baaL``<_CeALZo^Ag}gT01a(jAZL0p+0ecdC*Oz1S2GC&^?DG$1DneHL|`P
zxwnfU8-FgDpp-z#nY-ttM1jORhCw!$Ch_%u%k!FpAeD47GJjI=mcNGy!J#OQW34hP
zM0<y6&z6a*&nW|{;)`w<>i6wVqAtgSBqN#GSks5>oHHT~s)-tviQ)9ovKBfiD@O7U
zy*f}=p&uoE61SOrS7KZmOtY|NR>?Rq@Wg4gXIErO_FWj+_00z&pQ4@hgH=2$TxuhJ
ziH)Zt`D`?1DMT5XLV`VbIB+1Xb<0`xh!bu<lEv)boPd~YyNMDNbEdgIv{w`;{Af!^
zzVNH4iGiDT>7_2dDN)(nB3vUaXCr-~-%qmnO(#(Oz9mbFN|#33{o#PL;XV@!WX9vz
zvP-;@lFgQ><)0{O&RyXU!?pjmc?l!hp1#oYc|&1-f+>#D_)vi9<46;aiJt!Pt&9Y4
zLg$E6s(*xvR1qWF;LXo=I>_xc+h|f7xO<M-+~%hsk=vj6kuK}LefdTbu8TBjypQ@*
zGTh8Pm-8fakw$Kigj0zxFOM58H!rC(dT>*Urdt;EVTXl2T(*|mQE@XNP}=)O;*Rut
z;z<JY!1&2>>37H>7alG=9%@EM%>WnAKie-Z-}~1bvKLh}b1}fSC*P7Jpi3N|<wDtr
z-O!mgZutDF976mcR3U$*!fjnA1kTa>{bU$~`UcHf7}_NtN*}s6HzkXwOrL+_wz)fc
z42y)VRtYYCb~-<l&F0pj#h_`EU$d!>HPn%PnXDNoG0v2`$Npz23Ej5_sdth5*N&8J
zLj(8tGP`#4GZYaaPSDTdRzf{zQF+<qG(z78&C(_ori^(!6jN@yikNA2yt2P<KP|oU
zXA?sVY-#F^v!^7iI2)1V<F**Qc^nM&#7-_EvM4i%b#PAcO#FlhEU^?r2YY$AiY6ny
zx-99|Yq`U&N04oJi=hPA0$GYE+lq{lUmv{{;mo41>OH&EVUyC%zyOYwKO+-iNm!4K
z^dRnTEv4$SS5ZN`TTg$;=d!2r@ct^}1;53uR=yYlK>3doNZAD!Pwx2?XmSaolp{!#
z1!gLY_Zxwc8*4LJS)k7cEU|xc87RfdD1N?z`G`FlY9ER&F<H&6avFr&ujTBFz-YFD
zpb=CZDH=*(OfNuaiWnq~8{bKm7uPa~3^f|eh{S==h{+tqSs)RPwqMfxh^u`6BiG!g
zvXvc5ycQv17#gRn^ItU4gP>wn2e<!Xoije_W28;L7BbUNKF0o9!u44cr^*kz)B#1d
zdrFmwl3poT2d(P<0;`By28B;}DN&$!Tw0{+N#WTn%yTm?eeB6=ugi!zCI+BpBIu_D
z3XHY)*27P2|LcD6%fzTEq*q4en2Jrpw#vXgXDUVw3qTG27_ge@D>ddS(rxXLgWy5l
z!*-?ujdbEZ!B9z5;X+Qm`k3#HN`}L^G=N6ulUq2QMgTQv8I}^6<WyK^?GdR`@<V6V
z2dxs?_B!uwO4D>)ZcOP@BP!%|_oGd0>`1#6iDEoFk^8lvKC*)t(Sp5d#8TDt>_M~M
zEiNOEa^%K%3k}A_+n?Bt-y%hZr2Ai=8<t6gaI?C9_`&)Jn^W*9Xp}+DBYP!5f$^K#
zgrC=vc`9xNW1F6CIs>==yK|rlN8cV*bGQ~#f&jKPdfwRJgu-62(Lo^wk*)5}ND7EP
z2lY=U)Gn(q9PqMF{`;oY$(F)Xx{;F+DUI9Rw-_=sQrF)lX6cX4zUSLDl4X{m{$V!|
zJ-m2dF~xM)!pK!^6S`gmCVMMK#V84;Sd<;J(oo_PQmV;mnoyPuU<&1-82(X&ZAP|q
zg~fx1z66oc)iJ?SkaZz}w2%++VVl+BO;aZ$klGCi7WxRr3GV71eB{P23T`;csmJYC
zp4uSAxC|o65kehQ!q2AjB_o6qybtx+87xJs>qYrCPV?o&k%9C^Bo$CPFHYo0+a0eo
zYE~La3SY0d907`M{&h*7ELkL9*xqE`)e9pF0BK7#%XF_g5OB1g9@;{8p01W1t^h{c
z($r7IGC(5h21p5Y0CG-XIXf8|4z<`J$529OG6VfV<V<9?>g(;#CV04zFg#S(9h)9z
zO-Wi@Q_z0ViHQ|Q5BWIeo=8{?wv9Bj+G%B60BC*Cm~nCC5s*STc^+&y(6(ZCL}vz3
z=PoCz6lR|RdY$gkQr!{2VyJ%+>;i0qBcRjW07#M9K5KZe?v^#IRjcO7Edqo+D?mnB
z6H2uyztH?5Odk_I5fKLO#To#ND$OF5*SoHrfk<oIJys6wevh_?+G6b?W)Eqtlug|E
z`XAnbXmM3R5i)Xwh3JYjAWR#_7Q#FzlrIx$(&^8iESA@7@PwR<AaiqfJt%<)|3E@M
z!L3}ZwP~s?4fEDQ<J?4!g2o7^#S2nz;Qx3@n%+kx>b1}rH0eu~D(2f%MI}dof=yus
z<?-SciU7}%RM}4DhYJsiUay!Up%Xf`$MpA|Zx2)U3rnm%cxNDu20aC2hAh}rV#foI
z^q%kl+Bhdr#n(JJ7}fWY#ZU@5S@~}@cv-<8B*9E5k}foYu>rV&li{VLW?o)z!3zX~
z$MU9ZfkIN=bc>aw+!hO)y^$}3NYaSW4B-YqojbnRU^PkmpM2SKC)Mq4zi>GWk1qWY
zV8*&#iC@mBxdHS(VFTcG7BTfNq_ZaQ{w7;Y^xqfy9l%O+KiQHCLdH5wa}a1DXrE5W
zJf0@?2KY0jz=)(23Oi1MvG>;}{6gP>u0|a|1zdU|t^(9nQ~ZOK#v0De#a|jMECx-r
zqUQs2V^2U+V(c3Z^|9@RxKJe&JoUn*+tt^rwKCTF;`#Fkpb9Ro7ppNA3b!g{2+jR=
zGTrig2Dl}2fPho@`QhSGaFstdpjgEmh_2ACLYgaz7SJl96G_NPn-9fzVEJT{02cAU
zxw`ckV~72IOwKPA-@Vsill=&>?|eZ1uWjBishr3X<u78{1>pDE%Sn4l`Gmo`Wo4pF
zX%-WYoCmmG`TZHd>xY=uo&zFg$EK|P{K@^zs&E2Ep)E%L+wE86fJF60bnFJWJ&@!<
zYb;&F5*8cQ{cMVhi<Lw(w*d+2iX1?K^1XNVfH|dcnMX{UA9;;y%`F$<A@r<8jjaGA
z<qST6Yc>2LX$2O%Ho*f>{?<Ehe|T|`;9k1`?mf3Yb>|kw1_M5iQ={fT@rvnu*T<{N
zg6#Dn(SHb(MK<V7DPJrYT*Ls&f&|gKy}W3(QG5TSahHF_P^mT?tl$q4U|PYH3r{QF
z9;RDi+zR&o6X!Wmmd95;wPF~3oT)(Sn1c96<{O&QB7PDa<u~P=QMBc*Kei<I=JOIn
zAfEQO(a-oapWfjZLWcG})<PDZ!t^$}LhN>x$^-taKHVL#FIO4|afgSzB?&R@0rrtL
z7p{rhKlBu2hJtvZctg#{O~YS2LnQfJ`n1-7j?Gy8fw)o;DVJH;*6Adk@6ASN6oFlN
z;|`hP`@y{fmxsJ|K0m<dAo^lx&$`N_H=Ro#Pibzx$<?ON04g(OT%b0Hx;<B>X9pM@
z^u3og^u0`6b?nB94jQ-<5KF*5yE)s6iY^}ZpaQtm|MtPnQ@0+z1owtP#8Nh|u?u|Q
zo2WLW>Ab1fzKI1bEIYpaht`8pi+Tsjb@$<wVdTMjx$%PE(`@m7N(o^~1J)**rlXE}
zSaSF=owYxQA%b8L&CZiD7hzE0t2%VG8oE+dl8UWqk!LEuJ2JiZ$5YSQ8E|2!x0z?_
zM{Btl6D|X|$4W8|gYj=jn;LZH0KWFg$D+4}Z7tlKAxZ|z^>(Y+S(d(lABbyH3JNC4
zX-qwb!46-w@5=6|2b-<1;mPkU?81_?)qkr^t~I%`k-ECNKkBTf`;OphFvnBpJuh}H
zHElWWn_jDa+H)2UkAW=&5}5EznU~fg_h87lLZ~3Wq``9gMxVjJzmYkhk#OOi$^+9&
zYyU390HTeZMk_@QIESa9k02PFSmOxWq7fve``X7D3@%t`9WJ7vd^Z+QkkEg;PsaOI
zV2r3a-#T5PYmcH|VM^fk8+VB{g`ze<3-9-Qr&FD8;5w_})=Q)%utg+F$yrdQS@wfl
z_1iK9&SH(F@@{Jbo3R}4e!4UM7|<9`?F9g~A{x4%hhpwg4*7J$oM|hs21u+axGkUr
zc;=L~fVl-ZUD|SkgFD5C=!QRn_grl6YrU`Svy=|PeL_y}n89>CMIJ?c;p!JTs<?F1
zT2hu$P<^-HwZJ(E3awlEf5JU`+n9TFd+WFvT5{~uuYo{^Lmdd8>3>N|wLvX|T=@IL
zpPd5ucXZnDd#wDWXD$SL%58<<RT}{7mcjj@?WV}XwQ}|1cy8n1Nl77piiUx3{&d+<
zG#-E#W%lho8(2zk<C`hMen(?do<rj{v9Kd7VAX@pw-9>Og~2XhN<v!pqNPr+O45IJ
z>Wv_ZcG*k)5xrDvJxvi>XEFLiGgk)96ns9TrtHcvC33Z3DrXt9Jt1iR%6lhVDs#KB
zUI#lF!vyiXv_{|`u8p~|^Vn95xIM66xG;>Lx~_?=`#k_$bQ|D->^wCExim(odM3B}
z5wo>%Z1LNblLy(P;bv4tv4~I|?JrtX(4K&^&vxhpa@dz=dMnhPu!7;vcL8>(sDDU6
zjm{&fa%VDmw*(rhwr=(Bbj4MPW`~zBerg3AU#lt-9Qu=QF4TJHu&RqtH6h&zO+`t3
zkJ~zYSjrUjPVen&eX|L*9PjNAtdAXgWywAr?lQXLw!K{~3J~GPIUv2RBbP&Wd;dbJ
ziyABVoHrbP42QeU;vKW}8&4Hun~Au)1>Fmw9PX+_Oy^KSh9Wwh*7%2|cxl{1z1uW<
zy}i1IeZnSvgv$LsT@$u@n0TpN6F&b|d!U|EeJN@j3A964-L&qxs3O7n16l@aR~Hsa
zDDm7Xw8s)&B@@eHPr6a;7hoFh!ofp{>a;ZORKdu_tlN@tEipq++HH}hzgT?`v!#*U
z<Dru?FZOzZPKa5;4NcWfFvBKqO=2guPy$;<<Po`2i#Srn;j$rGnZViK#?iA>_>7PF
zap4kDxPaj&Zn9()E+n6(lBn$6e&NqgPKPMs{dt~6T<C=egXO)-Ir$u8pLagmI>?3E
z1AFJjn8}r<8P~~0I(@?4ZfDWk%bMhG1;&O%MdxT$!!0%ae9vty{xpsf44`m1z~?j9
z059h1`&-QkhIWrS@A0RgP`lXz1HSF?*{iwv_Qm>Dl$Cyb-@Fg0wDs=Kua_%VcT);w
z#F6SwZpeIk9|Ej^Abp^SnR^{Qoen38fE5zox8OUx?bt$UDSof(J*DS$q=*iBnA*W~
z97=Q2capUVZwQ)PnBRmtUv(m692O+E*Sz<CnD+NQt!8kuDQwa_NV+~ejWgZ(aPoCH
zr(6E4wXNTRRmZ_0`gSy_O!<32RY{0f6a(-^l*xSZoa#{MV|UxVYbufAxFzes_lC&v
zV*A|;olZ8-O^ju4*bPYmss>S=3Dv>cN?Yn#a#VT?(^uCbSP@WYRsW9B6_A4_;1J&n
zP?fEw!%3nzbM%nbbKxmifKVoHehxZ<l>NCzcit^qAx)p_2L1hcRGn0~cp5sav+I`^
z&)gA*SxV3t<xER(`#0Tc`R{16?U-SI@^;GU{W_mRuLZfswC7xos>c2LjP`m>H-cs(
z7}j~YkyxtFN#9|Ja7Gkv`^|K%=_GeiM5y+#YX|=wyIc)zA}g?U#&t~9Ew5bvEWVgu
zFg&PVE^)7|+OhoJP~m^eVrtSlTn{`|88&`Wiid7hj(&J0-1bAJo9t(q<@`6zW|Q~A
z&i_)PQrl*b{jMS@g`8)Y6PayK*4I1Bz3icvat$)gcZo-#i^FB(xaCTaEH?@D1Op->
z<+rvNT6R@9e(9Dt`b!IJf1d@<sjHs7uReHhT%?k=5oYc?mH+%RW{%VPoS-I0WxKxC
znCWW&d2}dAq2Y~9{i5P?n)NhsY27pnVM8<9SEE8=<p!#w1MU7k4cp`MPgg^)&b}b^
zSvNglM8Vw`3lv1Ilc!3<s@!NJv$4dby0ax?KLWdFG-CbX((X)~XaAX2+uJWpSuX3=
zx+SHFOy8;FhOw^pD&JmR%dvHa8(Im~v|Rq3Tlkgq_Ul3Yv0EVRi68%bv<V)f|1}X~
z&66T3pr9hy<O-e(tG^T^CTC%yqt&5d?Y(n%u%KgPt$w0ba&02YY-;;i)=N#_+<8&(
zr~bpw$ORks+PXw?_l4=_j2&o+*;91XuLX>TTIZ$|?{?wN^TL9KgaO5VZhnexeR*e{
z#`us@qqdfh$Mx|c7kVEu^^dQ?%;a-SrtHx_jK#a1%#!bgMcFGAvmT~MWX)@95&r%V
zalon_sy|*j87qm(r|aM}NcT(kgb`#6gqnc)z%-3+)OoR-t~8wA&i<f?%Q2>fd|l8i
z_a8V`9n+3UcFLW$vXY6#@<7?Y5o~50`1gSr9z=wz0F(B&U{Mt?+%XutuYQBBQ>PZG
z`}G4=iE4SuYNSLT@NUW4eT@00>2?42bPr+=s?9d1F-e)?6{21m+}>wN7f0uX@_NO&
zNQa}%%fj%X;|#2|=x;EV1s(bJC8i!$wTS6dUMph0tmu}|jh$@k*PpiQ5a}dj#*<R!
zLV?L{2hKLf^OjkqHa3Jd$6doJ<4}XC(r?^1sY?07B<4O3ThP`h_b=+}JI}4Xci0(+
zA1qFM#pjV%SB4(NCgjWoidKKj^9bdvKQ^trA1jAmVKFZg?Pl658?TJah&PfUADitx
zr8t``FXlYQ$H8U6d1#%1X>Xq}nTRpGU5JNpCJ6T=6?W3#PWo1J7AM#z7*#}|Sy7gO
zr71Xc8L!K)-$uTMd|4%1?39*&QZ?=LDSlq)r^NX(x3GUS6t(*cANAcslSixXom$_m
z)T7v+to`@*W}6`I%_d?cqrg^8YtIXT&e+v^>_yN@9+NJ|bz;KLuPm9sUD%i47wgJj
zuG8naxpipkhOqK{<#GQM+`Ypbh&L#dsLVOGr=&P)dh24|Qfq2i=J>|-3+ekm({veq
zrP@akZ_*DNTz^^ey9*2(r~}JyclEj4`Sq{`qP|l8uC<fzfOw8oXo{ed<vSjN2um}p
zS;nZz#S`>T0XOP`I1xXq$n(g)Pd1;ffeOqCs~G`+ciLO%C<+9>ru8s7Hm3h(x;YS3
zNH}<cNYLiD`MLRWwyLk{KC|y$#`E9TvrK6geHy@)p8?ZT*R<Qq-5Er?KBjN6P6O9z
ztld=MF{OBT1kIE)Y2fXzk2aNA_yil>!@QDxC3(<9WVf^Fx>vnsy%%<s#M9j-qAeD@
zRLN?iczh@h_OV)tV)qRt^(6HT>%QTNH5Uf0r4JAb3`1Xs4rKAs#jZ>H{8i<*wDQ>(
znc}?1{|eVTC0P=Frzv%qiEVaG*NJYWo((9lwGsJ@DgG?4KOK)uw3W(!Y9Lr_e6L|S
zvbCP(wv@%fFB~VYHQcB_;TqKD>NP6FSN=7|Sev+i)bMFWtKdE&^7Cf<Q256On?DU9
z^Yz7m>%V3_#$9x6hBuCoHEEPcnU7QZEq3;@3EiA+spsb4L}JP+2HO_F>%6S(i6fZx
zVVmP(u7+MzJ7PcE^~j{E7FnwW3&`7%FBRo|T~C`8TSfONQt~S=@+Qjo5G9zYnKdR>
z5F47P_k!SG8cUAJ@gaq4jv3teH?QH5c|4wBzCmtZ&jPQ`?aGd#=lyG)7#V`<J4WoV
zUCw#rxZk>k_^(y492@k0ye3kX3UAF2u1YJML!yn1e$P&EaB?TRzEd|Ws5&Td>yb|I
zrHQA^_=rrM&rMz*?8dt#VWphb3Jx*|CS5uFT<GF8-iwvqIbP0VQ-4^mszz+u0ag=H
zO>$?rQ)kzkNj;n{`0|Evv9nv=mUXIeNFeX?do8zC21ZDNGMU8#_J%;cl`L%|V&`*q
zPAr9j#6X~ESc~~<xd#(-3LH`>!7cI>R-mRtH9bv7xpg_t;H;#;`-|0kA(+_bj&H^x
zIAn}wLe~ttTC}5${wp&ZAForZyzl3tT~`VCl`B+02UOI~Ueke%6~8g2<n+;)@JY2E
zbSiuhk(pg~E@qOOmX9A%GTy%nYX4zpkXH3-u)USjqP~WHyriL_a|*DTRYYkGH|*MX
z=k*QTy1F?n=DR6(<I5eFt*2a_rDJ$k+0dS9dRgKUwY$X!%PZciwhHF$(lsqr{6Wxa
zrsI<A0De>D-j-q7S_#I@&~zKEv5`(<MV;x@Dg7256Bcv$b`tVHya_EMr_%#RzdOPm
zyF(8%{ZJ^yoo%H@pe}(jndREu$&A}?KNKI_(f&L;v-<h!D{BY+C$=}cy3-!}OGj5{
zH@7K})v@}+m6#kg?@Ij@^{uUPI5>Rx_cD^|m^Q(QB3AnF`PN6(Zpz8XeTJ=zbtl=E
zJ~rvi@~kQM5zX*RVdRj6$9N|jsdFv;)_&o1BvP#{FMXegW&hKxngb9oRFQ{5)NSVW
zeMx!B;^^Yy<uS!q?^Es&HkQvEqi%<2BBjOJUOz6(JI>frS&4?R&LZj3`7GFm=kI?~
zulyRXWfzEBd?t^`k?z-8TE%UCM@RW)B!;U!C`poE^7%o+xbWRO&?~z616*pj`cMt2
z)6|>VfzxRrYMrSn|JPt5o<;c|6b-_7$8X~T$HL6kL>ldyWA0I6IHM}IY(v}cKjbg%
zmr$&#`NRpf8OA9*`^4{9Y0iByb9bS^0P<ndC%^G~noK_P#psS(fxK}#A5Ex)7}_8N
z%CSJu>l>o=r6EU!C(l1o1$al{y)%u|09B}8z#oZtIX<3oqu_?=Es_uZ*z^}S%@rOj
zYWF7c5~j4K{X%cQF146eeCiL}I9y8#55NIsQb%9J?l4mCcP)+%5Lz0)f)GM5Lya6g
zz352qiB8biJhh9-Ue8zfIXi!<JfMl5eL;5UU&G7NCY<KV)(n=XI!~5Ln=|slyKy)7
z@y_&JVOxC)9xLJ)GJbWHHBpAgtk&zn*l`?M@;W*)9S({67HLq5YpgbNn@*nX&>!V5
zg56o1Tr<$9z)uIzHZ(uTj0m0r2MA1ionfU$pBD=isn&DBrbjtPgfX={{+3|hYgcJw
zn%QQ@_vHR-D&ws{_*Wwi=mv7q0@4<_Y>g;XGXhYi68(H=Itk>0IfAQCayqA(RwXk6
zHM1jB;SQx|+m+TD$}<)Ihj-NWP1P3Z9^an!PLLNnkWq@fWQ^l=pA}_k@KEz}@yZp<
z$ZaiMMd&FbLSMTTub~W@C<lz`TNbGymWUCU+Xc6{YT)!d83PsW3ygzL0F)v<uarN%
zf{+jxfy1|m-K8mT8c}r7LTu6(IMFs8EWbL-{wA!+bYP5tAtqnv;OnrDz0DDP+SUD@
z8g<VefNUm~eGSZJP#&l{&JDQhv=KLD1Gb3w9V;=;fhw%qu}p-^StmIr+i!z7*9VV|
z1L+M`=o3K>e(PhV7y6JN#wPpDsie#cvhe3+GPg+T7qT?p5Cz<v?!)h@H)WEfNx?Mh
z0hywznO`=QVX8b3Jd3fVv0t7pv?sDT2ib8ttopYvmm>&3?|Ob?e+@yUM}U}E{oxB+
zT`J3Vq>H1b1^jVaL4)|AaLM+6v0Kq6CvoOU%y$q=-%ufa?{za&ASe{cwo*fk&wz74
z1{z-wCcFV9aYN#_GR<0JRsTL*(rV~WY|}4|C-2wb{ref|)g^OpEAtqv0YQ}s1A+h#
zRBQq?Rs!j8aZtE^C%sL4>2!oiFah)VCZ%Zo($xCjg91plE+v5~^#2~Hsla*1#=Q;t
z??=GzUXFAHU*!M$8{mT!0}w1`W<xYDjne-;7(ZwIKjYv+V$lY|F-cRo{u|-{j0V6L
zmjC<TF9QP2HD_9BxdP1p9pq&+;FL)J@A&{9{{K&j1`RcTd(>1Gqu`CC;v+w&d%6DH
zPixl_?O&D8!9qX&Au^|b9%!8Nbl&}N=Q4=~oQVfx?51zSRRn-er%3>rw3}X`*IeU!
z-V(2`c*6<84Q1=~{R~e}4OnEn-EPOCBqt2t^!?YquUvJp?k>|w`&GB|A7cu(bIC+^
zSo*Tp)5%9KYK%t>s#pPOmaRfXG@sj?v-e?MF==$jd4X{h7vJ8F=9PbYS*_?jPDs4Q
z%i_le11n75sPfffj8>)E(Qj8yY4<E49OLM+!eTio!@JC)adWQK)y|uWe|OpN{<Wp#
z<f2@L1dwbmq2Pe4-};x=!*_SP)8wtGXaK7pi~AC%J?1KOGBkAXatA|%>{c0fzL~F0
z_)RCh$T^K6KX+yozC(Ndv6Ub$Jc^XMGu|InU|mgk695=>X;RSxu;hywz&RcIg}C~?
zmxrZK59?lV58WP~i$*lxKLyFJ?9NuoSsyIc<;<3APeoIRT9_8F*Fmx#O+uq7crhix
z(XKXMrqQ>R>tstBmRJl&s(~`#PPDJ<F;NfAl<e8X!MoYzG{z0c7)o`HoZU`O@yPGC
zM98_WaNL)fli5sS45ml^YOtCwyPddG7+p)+R;FK{oWQ`vS8b)?btQ(2ekrXhcbKVv
z8vMq&YBO9xKO>19DF(I=+!tCMqFuz~%Rgr|wS{aeeyUnKQ34-TV*qKGe7Q)!keJZW
z6{X1GVf#(`-<QgNjMt#@Yviaw{a{8Wd1q3>>x4BRD(HEDQXkFNbLMxtLN;+<{$v5G
zHu>p4jQpS4&&Q*-fAVGLp1X_Zvc0TVO7)hO%9<%HkB3~88;*D9>(dNx*OJPXjXx;~
z&z{rajILKnY&+YUURL)W+6ly0#AMn%-&#Z)K$`ZnNA#)|E#B!`%5vaY(f6&EEmTfB
zNhFo8TJXCH3>OQ(YSkkdE=H8Us1sgV8j2D=+<#W8(MWUC$-c`NxnmzjM1Q4F$9KuR
z7?jLkH7b0yXjc*CN`)+f!3N18x>dXII_|P$sz_wf8F0=@;hC;dC4#!znbR?{chzOW
z1Tr2dymUFXJqqUQfz4p^WvjiJS7046MjT31kk{Jew8a0Et+wtBt8^tj4G~_9z!E-;
zx8N;BqD_UOYQ9yQNe!q0AGujNcmEb-S+#zo#859UiDeYa+<|6o-S5heRSu7fj{cZq
zJ`jile~B#EtwG5oJ35Z{ze})o18fz>ZX1VS2U0TSRL?h-;|6}?EYeb~XlZbiH}La(
z4P3qGOQ%l$WEI{5Zo9IVc2Dbir6&`E$A>Q!T&nigS`HSl5!%X2Ae2<EzL`aPBMeDQ
zsT)x+2vG|(L=KG;mLpyKnUeIQAN`d;8^y$~8nKR0Q&m6CL$WBgB_!@c@~!aBhpiI*
z`LpglE_c%Ant6xStuGN_c(cF)H&H5R2{}{%mD6eca=3z*FndLK9k&_C|J}M7D>Vs(
z>58zqV3c%hB3K1ugBip-MC2m=@&8-PtoWCp+&mpv;-0L2d*xG(SHi#hv*#LTmxi(S
z7~XmKU8j=yeRZgDcI@^=XJhv8#U3hf_;^0G`+Cl1GyebAfnAALFD}=|=j8XyeB0Cg
zrCcXq=ekdS_bywsdX~3PYx2Yu_H(L=v*!7$>1_y-5U<ofcI_YU@%k@YmO3tea4Gmo
zTW)ppxy3D@%nPb&-W1A~irwruutrrn=hgh|H}~{--!8WglT^2^kWxP(;_ZL<Y~A$O
z+spb&52&;QdnhIArBALqdHIxhdHT&>(_a>0y;VQDHm}XTEY&A`l4V~XYw)_=O!Cq)
zw<{jryxPO}Fma8$$KypW4@RiG;$J4W)Y9|8RsHnH%$hRLh|Su&o22<wU(apc{av=`
z*#3Dwna5lo$Cf>~Ab5Mn+|&M|#czw^jN@NBJWV;V#vJHkPe+HlH$4kyUO6$xZNdNc
z$+^exF`NC`n0IY;v~qIDroyvE*A~~x{(g4S`)#AQw6({j&reHt?TbCV(d4*JrkiT)
zvB)jCufvP(EnD{Fpu|a~V>36K)E{S>`dUQb!T$fgu{m>mK3;g5ZXLMp*Hy2ZugkX`
zJrjRU%Kl|bg|?c|*J|zX-+zQ(nCEQzYg6C5x3b*&@tLYyYk<ywyZZe4)T7)_i|zcs
z$6hjUcHDC<_Cvv?b&jdlOFim#bMW(*C(c>2oHe*Fy(+gGSZvq<i;d3u-YbV>JQ@Oz
zb^of*+hb`w>-)P}*$ph->5qeK-&E~+|Mnx_RKXkfHWknJtzEhM@1LUb*Ik~b%^MAN
zo?e-<$gacgT;*rB*``0N&TR7Cc)Wf7oCA+uM;JV1+odxn<D<r$N#4;{wrJ|d^)9WO
zb({OgQdz%GoGHM>5g=>z&z~tqsmuG1j`Nq_wmN%{bbQz7S^bymw43d<y_V~e%hQuW
z<o`UH@Vp{|Z{m-S+eN><dYhYbr}+OSvx<V3Kd<fJ$!yYmK415e958FSs6DWknQ%fk
z{I(0QRt<Uo(a&XCj^FwD*ZC51J=XnwaaKa{R`r8`wy(;zmks{|$4q-;`)7g+J5Zay
z+V28q`m_8cuhMr<t1HNMs=e>}YV*VH$|c{QA8ygLn^W<dXXmS=Ex%uET>S1K_qxCH
z)*qbmc&44$?apsEJC0he<`e8-VeIYfVQ~d6z6w-3@G<3I&g*r5|MAV<|L0#HtJd}W
zrE=_t3l(PC27RqIah;tg^daWKy>P)f3*}EVX{rZETIm66LJ5{DbLUE2cU+5TJv1%>
zwQE=EO_$P!H&mE99f7UGq!{+Cz{ORt*2ttTU?y6!D<*3VqT!>$28!Q8*(+R#_K}Ac
zuzfQ5Rnf|gz;p~nK#PG@@UhYWZMbnR3M+wgH9o6OFFgiqYe1Vo4oRS~c2ox<H<?D+
nqaiRF0;3^7b_g`C5d2ZU-*tVHTT{mu1|aZs^>bP0l+XkKpBSAK

literal 0
HcmV?d00001

diff --git a/site/img/shared2.png b/site/img/shared2.png
new file mode 100644
index 0000000000000000000000000000000000000000..a1e04999eb3fd3bd7c350a0850659740d4a8cd03
GIT binary patch
literal 23204
zcmeFZWmuG5*9HvSV1Nq93@9N8(gGq~LrB-qLxXf9-6f(j2&f214&5*`(jg%rAYIa(
zGebAug>gUc`yR*l@B98e9)~b<x!1n-UVE>#&b`jH4OLN=xs6YbkA;PGTTWI|4GRk=
z9t#V*_{KHh$<4vGr@$|47d06PtfF4ZwX0_@WOZDyun0)5{$gXrC*A`Z3|nhxyJ{;b
z3Yt3Fvp+X;G%;uQw0{A##=;Wz6a+rno4Y=z@wB&da1rzrq5aiD5cqucn1hz)R})uT
z5n62}6`E&`&gL{c?A+{6Xd(DCG&I7_W)^~KlG1;~fo~$TR<5ou1UWc7JUrMvxY!+?
zEjc&^1OzyqJmq-$lnrRX=Hlhx`rMPv!G-R3Cx82qG<Pv|wtnGi?dU*r)$el?M>kgy
zTH31@{rl(lI9;tR{_`dWmp{`2CdhI1FC3igPdNVV8xR$~dMfzL(f)<Axr+<XKZHm4
zSIhtP>_6}M-Co7Y(bW;qg0r=$oP(>mGa&5x{AzTNr+=jX-&_3OzM#(5=D@6fNpt>@
z{-4kOv=`>Mn)v@v#BVMCdJ3o+f-lVR?~+0Aix-|wVPT13$w`W9cw%o&T~E{)9`C?d
z64ZO(Vq23+f>c2&rvIp#B0se5v=?!fNGcVSjNbD>jrnlA#ltHhfXkNji+elP?p`2j
zzdt3I^yo^8o5Aezmw9X_`fMNG5we8VT0Fuf62rp&?~niL;J^Fezh>}XSMdM4C%lm^
zrhJb;DvszILs0z#J(A9>-P^m>cSI|-a{L*0#&7gx)7v4!(ce_=;&zdJW4kmrl=_cv
zO7X_g;LcxZul<j)vJj=D>0lFa&;?@Q+@QJop#cqqTkQYzB>VH7CQu9y5M#xfH~)8b
zLF+eF`U{dNP!j)ko+Ac`;fd9~``ZtViU>AK{IJV{!S8;rItSkX#IWAR692nX@)*UW
z%Df%w2>!L66dXVdjD>mnpZD+M=2&U<g?l{w&md_4F&r90X{@05n~Tezx26O-CV296
zAaz1qxM#aRN}M`fFnt|aoRhp~)r-?-udr~|9=%5>hh!Gq00tKHwW_N`-Qydp|8ANl
zaYA+L=6-0%!FC@p>pM^*=#cUF#O0vHU}rN+6_3|c4u=qQ_%Z5vL6d;c%7p*aXq6I3
znzkm{LX_j}7%K@auZ8<k(Uy_-QhZ0_YqBviF8K5UB61SKQEVNia4AqpDJ+8(w0Acp
zEzWQ>Y=BmUE+}k<qqr^4%-!yzso&(vSoOKHeOVT9?Mi+N`BHa6*)!weiMQvc=V^WL
zDn)NrH-wV&W>!$OxcP1b2g?QJIohYSAz4RTKll3`f!@18n_!^p`q5;rK8c{4yR?<Y
znd5{mbO#4e;#%)MeIbp)q{s+Py%JsmasDNio1WbscHK*SW#6YigGw$EmdC(8i+F4!
z92eUD-C0#8(t1zxU0=8ZdMyV5-NwX*OVANf2lWrqR;r)s+p6~2#n*`YR6Z4QnAG1G
z5oTOfroK>U-9D}PF_M>>SFek#d+Id)2G6~q2I5oIQ^GTGF@GfC<9~M2)s{7Za`bbt
z&({*Dl#QMk9TK5V7$UNGV;5%{T(g{KK5W<fwCqg-TRCwfcxevlbN`xKPs!_wdfSoB
z51b(fTLt?aZ+jwFa>?#x_Vu4Of+mc7RwlYh@L%@z?)xlF=ta>r^9ZNO6!8?8y6^4c
zRo3f@^$n~nl=BVMGzjjSXXD<Wd9Mv@t<ZgX4_pbhA5b<`1(<EYlv-KMiTw+U8DY&k
z_*)}=_=PeXl)Fw|E-_2E;7;M&eH$6<hpd|8zWU(C>;{f<zuKj1rlO2J95-}p$a76R
zCmXzY{Yu8&5TFUiFxutlcQjtN)vOua#!(ZTqYuuqRJ>$1iwm8%W@;BFGQ9A3)tAPQ
zJ`q%3q#|mp2<+Br%j_o?wWs#KDug|qyG$jym(LmR<KD}2`OFkv6cBbI;Qm&(Tckck
zR9XwHjeZjv&^y8Xy(G~M@#aU;4Lk{~pfC6;{qI;A;~o+@1c~s}dc^5|ARpzSD7G%L
zA0OD9_z)saFeLAmgvis%vWm|N6$-AAEP&Rg!7W@*lMrMRv0P!gWGI;JO2d@1olu2?
zcWfSvh+?#}giEIAHUs>~XZxh|$HkXwmV^!RM>5dJvDLMW$OUjOpawi}SDXUfNNNep
zzsFxsdngH2Xr6&4_->%8(LQq9S)>1PjYZF%fY9+i&Tb#+O>Ep!b-+cq8+!B#D&QTc
zdbkCxV})r++s&W7of5A3k(D$2gx4!<kd)22z}&BF!plbK=WNxh<zNsaN)S=24~A8k
zd9P1AmMI$#v!ElhXFk0wj1I`olIp<tlIPkE4+@EbCz{y3c=z55Q*hFopw9KJ7W_H9
z!!O^!jH6ylneU3Kq^+@@i%ZaR<u%jcj58NE!(`oQUyR%^3v0qdfpMdP<b{lenYZ?+
zT<k5~vIDY5<cw;v^6C>RhT68keLEjvvc1&rEpOX`l^(}cnfSG}&K!p6KE2#=4%!-F
z?r?|w6y%?I9l>_1-8v?g1DUDS1mi@TmswaJ{R7ON=<U@?$)n$j#Sk3fCQS)0kKII)
z;u?(h_UHK*xoNSQn@1sgsyi$!46XNL)S(r8_l%Am3*-0=3JopuyCLxxLtG~Xt4_z{
zWmP;MS{D|#l-Lpl@~mQhzCpCmJ<<Nk=x!Hl%V$-naToBMM66;`Tw1{^H_S@JL@J(m
z4+#Dg-x+V7t`X2HYhg{8*5o@()mxhE^mE=CcM6H$d+uIm@t*9chJRx{-#^ic!h-Gg
zl(5al$En@gywZ*6Aysv{Wu(UsDpLkq)fZ~(c)nYTh{Pr4ng|P{@VTv-haH9&s$rM5
zosa*|%bRRT=xqAAeU{O}yQFPMo`!8FHyy&wv<XA{!Np^Xs(0+gQI(@7iF(VqFRczl
z&wU*l_j}?At#|tH{&C(4q1$;G_pL8S;v|<Px?~Mhe`)j&oP}$lyy@_E`bd?3`*z3f
zBK}ZkUrEAsutE2j&_79h1;wo=@ttg7nP*r*PY4QZO>ejmGgIKiu@#S++RBfVs`=I^
z@u+)-u*5e|Dk*)0(z4jnzV#^N>vtlxN$M$e2pAV*LqO!L+-`TPM!Veew<}aRSH3lA
zna&auSLrwgp@>`VJ~k`DVvCRSu@~UN&7+@<ZuHD3Wr<6!@(AxUs-jfZ>CnyUV!v8*
zF`9ZjmKnyF=@q%P{{+2{kAcPMoTa?;H|TvM1sv13bEW|FZ-}Mo1Y&N~t=_r6p*qNp
z81Tnzu=riczcKg!vu=p;CJX70e@1!pJJ82!Y&#RS7cO=!$+d@F_KR0)Q6|3&c6#@P
zKscY0k%c|Nr`Jr|8oNQAMJXChb*#*c_)OZf<L3#+#+~PQ5nlRJt0ds8jG+UUU^Dxy
z3Z@@DyYOBF=Z4u^EO5`Q>i3D)aI7B{%bSbnI8~+jlSd0phOLiwA;+2F(c=-#$8>>W
zOIN|ZH?$tmT>}m^Q!OEQI$T+v5W7)}4)k>}{mZAj*tjh;@2Ou)s9eJ(`X(Rb;x;nF
z`kJNPEK}?xv7FG2->^jU1Qhp?Hc;#yu2ui7ubL^hZkS~^p#--An77r@wM=yg>_*;0
zR(6%|w;ef|@8jHfr5b3o{k)Fl)6JkwVo&Yt^@Vfg?qqf4S}qX-dc#z-t<)9J$$KO=
z>J|NE>^?Bn$3&|Bkm^N}wlH`czM3cQL@pDAx2&-R3BFqPoR7fv(4051#3hmqY>MpK
zWpA^;`7PnGEyL{SqH^1-c=NBd#9}mhcsW)U8__Fu#5ZD_hFO#yLy&{A_-_J{E;UIl
z$lJeqiUIUyQ-Wat{6`s<t!?}L^4H@$KTiF6zDI%l?PURMC022zv9NHyhk;@^xYMv?
zI#4Vljbmv+d=q%$JlIXO!lRV$$|cY>nWfyjl?>>m3$Iu8W;-*Ww5&SraUCoayQo%)
zxqC9iG8_vB14@zNN{)Znm+wRcxZzH=#fUBE0?Yix2T|#n6~Gv>4t9<`TK<6tuq<a*
zGEq_mh2tG;Cz>YkPJ0UJwOl!#x={uP`pF8@h}$BM`V57i&KL>__H9pd^m$iA=p8Is
zxY{B0g^URx7E834`PVpv)?aPfaw!^``fif(tIKHTc|Mwxna8rGd(qEceenIcoI@tl
zvlRf4*_|pmcX>wlJO`z02)~2TN0{&6FJ;y)O(=*;Z=M?w6gc=t+}`|XDzo>kqQnJ%
z!Shk61dRhue+7>WQ%5YQpzuE5gfo{762_sv$Lhghcx6g?S5OjPd+o~x#K2{11y45v
z-EEcm)MTdDWNOK^E=EUd86!C31FB+)_+IJ)9bTI*`o~96Q4^b=9oW{%!18>r&7$wM
zYSS%KB4;Mg)A=UU$}79IKLG%%aVZqpnA7C)#GUnzOe!LK0LtXutKA2yQCqGdJ#3Rr
zLnegJV&6_3x}_Fso)7PSvFH;PI-SBfwMFSu{CL<QtsFWQj*w=p+8|tC`eIfk?1Ng7
z@KG%+c!b+UjNYecgl*<~<|m$e<}Tx0X9}|i$&3oKI)8HwT0LyPO&8|B;xT;KVmV0b
zUgp0-I4=D)lee$LH~cfX0FOykfT2}BKD>d^=EOif-qveIDRZl@a*^PPC;RcToXC7W
z37o~}5D0fG2i>>ZD_bKRItLk3)T1bCb-`_Y*)p4(9{8JaP9fr%&M&IcL^js@K5(5b
zk29WRVydd}Hw`tP5dNg?xTEE{BRs*^dz0n8e^gEFu=mM!9|eq7Z<w#v$v8P2L6Q0~
zZU`zq@!&fuXuB$VZ=Z61?IFFaSil9IhlQYOB!oBSl~^@SKPo(6^OGesM|^%glE6M;
zM8G}DztO|N!o7%`YWqCz$-OMM#gUE1tI3o-Ph~lAn0OK3uN)3oVXhN&LN)s_uMlP!
zN62JD|MXcl<z$s!k&+Q*357Ixj2|&m%RYrP!cW)i-{V$d3m2-29P^(z-0H-uR4Yu&
zZb%iC8_l#<pyka`Xg#r#KHgmzg)s#rBO47H$G(6EXGZFh@fu$NfP9b4ft%w2agLbr
z7sE?Qr?~l-_y<c%>o@gttLn3la=aZ^48~bz>vc^n$o#JRqHS9vRBf00IY47Smn_ux
zM9Qif7%OcPY$GELF3*W0QSmt*=8Xwyr@NoYv30rC7m|n4d;3LCgfy{*caCv{F6(i+
z`>8)qhp4hHuOsy@eQ1~_q2#0E$_Dc#otNG}E0rR)GMDIWR+hBWC!_*$r4Ns_*^)Ce
zw<ecyxu(eS{MKXoGhvbnKGc&$+KjWdw#g~zA_FyFyh@n+Gtf#qpPqB~wV<(?qp}#%
z^To!MbnBD5gNKrP*=Ze~zN4SM_j7kvoT0LAAFiw{;{QmC(?@-x?$HCG)HjxvYFF@V
zKOB30A5i0}5F;(fC&^Q`&PIK<6>^L>omoSU-alDVErsjo-r1;9G9)U2^7ya3qjI)q
zT;kO2le?iIZX9uNUN!F@()V`xpGU9wA6=>sj(!)0a*EzL{(y(0Nt)B?p`_H9kf*Ob
zt-=OG&S4V;1fScFmde2)zXzv1KQoH_BP6em6Qt(vd^ghN#6@#ouCBdjbFt$qZ1Z|q
zGaFk((L4;GuJ(D2qMB-L4|#t#iPrex@Ug~gi`N!}>lF~M#K5=@h8jAUeeIQ~%jEK^
zn{>KNQs+zJEv4=NN8}WS86KW<hdjejFPJ8ti6AupMC*XCHqe9dXM<z%R4Fw1IBjbx
zA)(y>(Dv7vc*+uU+Yv;wDtYl1R}00En&<?t0J(Y`Ff=J}kN3o~oZBGHu}SrGjn44i
z>Su{lX$4lVs(R`-SHn0PTp<ZAQYkNNIUTET4N|H&z8jM8kw`XPOt{f@et+syc~sxP
zKmBQO&1BQp<xd|wp$;HWu3ic&#ifB^Vjokr*17?*ggT+K%4V+-MeuilZd&)@4!pJv
z`vzXlG*YE9%R7B#OL>Xjho=jse$vl}FO{nu%u=VG%62nv#plyU<13Vxx5vC$x$%h>
z9b7q;kFMZ}WLa|K)ij({-Z~L&%jjwh5h2d0{xzQCDUDq&x;Le`dDD+IwUq5RV8+Of
zkpf%7#~xL$8deVl4;?nghDt)mpIu*`^!f^CiJ2a_H#u;>)=}N98mS^RI4Y^I!hJTT
z>$^R8h_kM4wE(hYnpU0}l(p_89O*}@@2M3hs>*Xn?GrwDjrx@L3G!ivC-=f(UMTYn
zio+JabZRz~PoecxL8uylWDSm+b}a+b-MqLTX=Yt?>uX0DOYAbWQ)x5}n?^r=G2oOG
zr!U)a*t~BU<mo?(o&v3ek*sXyO&{K0!Fg|BgIDKsKB*H@>8ja9p|4KR+&?HcD*|@U
zm@jo@ah2`zj_$$~nMlfZJ>x8u3`E60uKtmk$iNn#nPN7`fbjC4F8;W{Q?_;{Fj@}g
z*P}n2REMM<vD^32R^y@4zfu*VV~p?O$bn~T8FGK73f^OjPpzQ@UBkv@VFv8cFyjlf
z`ST>af2TZ*kdGhM93KbvDY=ewqg4(#MkUg80u4X1&w?2d@$Gj-z=M|twcpim6?(!B
zu97SBi~v?Sd)x_Nh>xtZ|Aaxcd^0?CI2FVvv{Qn44GPAb7GlR<0=b$;T7jfJttTIa
zu0A%6j_hr_7?FI`sA)2o@GYCq9x$?C1+c?=e!$E^%jmBFO+-_)prTP5q*Fj&mug8R
zx-wz@bvX-9h!j@PyDN;!8T*k$oW>R}&DM6BF34LazZetBH<C38Vq|vB$GbWoy!Qoq
z>*QN`n@e`2Eyb^yTH6@)RMPl!?}B<;=6z=fhCe(%x>FAho&Rv(>y<=GxV%%$e3{Cp
z*Al-nRG(PTGWLG*C9;^j7F2?g!e^oZ;p5n14xb^eo<6z<ZHJ+v5P>`$;iq(wpYXYd
z`Y+Qw^-9DrIVSI40(l?+2ssCh8O;)Cj;wOLFmieSDH#Vs;P;6(0hoLr>RuZszM41w
zkXp%cFuw?rDpMNJraaiHu&|$<+41I%&iBXH)mb!i_`LC4GOlEVp)07WxJQ|>fdTH<
znYOjQYd&d}xp75|(2(Hy#uUxS_{`U!R~o`UJ;2e+|J=x0$0{<Z#+X#h@qL<pQdu~>
zuj);gfA~BsWJYUWLP`ej_ql-pw@!0VkI#}70z0D8DlNfDA5_j?H2Mj-!#6oVAvhOc
zJ}H+yckGb&^imyDXc9l~EyeLiQ~}R7<aH2S1_VjorD`j|r(Ti6M+8E*Pjon$e5Dk$
zL+`+P7eJ7PNDw3_i)A1bP6(P_-{d1d3Z_lyAlcZH`<1Yxz#kfZO$SCYk+LRhQKrli
z7~Xg&Z`irlWx-p(G;3?otFi}s-$JJ%BPF1SNo#>sdn!8>r&az|aiCZTPJ{0|sPO$%
zH1-+feE=C=AxPqe_|EpOt%OR1B-xxY)$Ft+^{X>h`q_GFAC?CiN6wNcW@VKl>VN)_
zN_lLv{~Y<zTzbHbz=qA3Q09!Gn@66yh<l$pN1rn$%_xR=^n6rf{Gs>ez^Ci9;z-^p
z!i40v3V^f-!BLYx^{l8Srf}Xj1d2sNZ!`;{a?YS7(Tm%4Gc*Ouu*!Cltrk2pEVze6
zKPQJV@xtE!K5NOs17QxnO^qJ5y)fS#emU`tD58&_&dA@XB_?Jp(C<FExc+X2^Ytg;
zAT7b6`}miN$4NO3u+0LR-|5NsYS@U!uJ7k{LZGzXa&ufEe$K?qB{(9{^)P&3>D31;
z1c?##n}s$Rt@zw32OUh)Dvvd8X>R7DNzJc}1ITfvl2~Qjr9$4zLf`y{koiFEND@cQ
z1h+`D63W1^@oDiSCc;}>J|dCo`o4SyjG<m_Em{F-=51cq@K^a0H_A2T`m}s=(V@7I
zRAV1J<_4UG@BJL`AWRQ0ef9B<u0Pp*-J$tD)YR-W)i&Ggw8-|<A{+RR<_8RoU6O*J
zOh<BaB)5Fi#L4iOAvmW@6o~Q<y<frcIcFEwZ!X}35M`H#>{P4GA;MQ1g<d~uy%}rw
zOM@-UMD{vyh4X%S{OWTg#IigjXuJbtkq#-d-{F*C&GlRhoQA#k9O8=##62oMGtl(#
zt78mcnP$;}9+7vngp1H6CXOtxnS5X3<gVjDH0C^{BJ&z~mQZL>NFC3h9*>+R8Cq%(
zs&YiT4&j|3)0|On9<zG*XZ_HjHv-h^inKF4B}x!4OS~m=Bm^~2{nOBykpuL{JO@s3
z5J;4USJ1@7zHyri{w3dTT8krrj*R%9WOmR~TwSZvHTVa7$JE+q{ZR2d?IzaG5X!j>
zxX10D2V{AVrsI`>XGkQ{$Oq9fiaZx{N=wWAqkAkIP0*O)sZqv}_#6qjugTXKe6?|$
z$<<5&_=q{`Zahpd|B>PIbx{1>kTRNjjRFeLAZUi=BG(G}Xa59>CF1%@1VimU7okJ%
zK;}}DUY3_MxlJs7C2UEaR>iI#kOX{qI`6r{+*f>mb^XHC)PE&$@O)neLtW7~38%?<
z^)UhPc6_xDfylz+d=pd}^|Ry4bj>`#AEe8%+(kd9gY+Jlwh6Zxz_IcFY`hy8Voi(?
z$~;f;5^YS~JW&Yr)3*a|ZL9_d>nS3Mqt7jI@FWTCN;9&~OD2AKOdf!UnA@nq8ciW$
zzm-Sx7^i`|*9eSa^$m?Z1=q6r;z(6yS-Jw=4ZVocjN88Ty!xYNhA@@Y`+3z682gOY
znN+{ZznXtfa5UDl3z}i_Rm_HFjA?za&n;Bux3jf9fW`Sf#Mn`;7pM202EcXTeIQHb
zf#(A;aJj#$Hh^0>AdQg4<hL+t9pQ%A#V`5B_l|a)g`}ES#dL8hJAI+mQ{zioU4qNO
zL*d<^hVWm(0N8i;8D>dXeP6cehrk-@?k<x&c`D$A=Tbi3N|d<RsQG&I*_Fde4Znl-
zXj_4g!}UNyOn){k&3inYV?i0j3rt1}DsLyg#Zr7!-OL34>qNW(-Ue5$-*~d3Vz{eX
z2a9hf1GUHTTa%O)nB2e16@$<H!|PH96Vt+)6v=#`dh{w>QGRs8>~$9U)$6sfl`Qm8
zD1SL29={l-yhT(9$rPdS$|<W6{wo^+m|Bnqe)&N2Wfr=~0<}}8*$?}0Gq*u`9@b{a
z%pYg!`kfQ5m{WJE^2z|@!RmXX{YD!z+~VJ*rcq(WMy=Ota02^OfF0%v%W#>1g&+!K
zEy5CDO%}|)?QVkEg_=jUphlJkb^V-gL(j1UdH!ycM+!}h&PT8AAfFuL5+YXTNFW28
zBKZ<Vajh?Bkfu|~tqX0tDhVuQyWnYZA<en3gx<`5brvZ06VKP=4HWGu35ZQ=Qf8w=
zY1TgSs*3yB_Y{6qF-Dmr7s)arvRF`Za6<4d_>je2{nc|XCUEfI6&t6So$8ZeGSqsk
zCY;W%qN+fA;97&fYDz$&<K_GYgWo7+H5O&0U^KZ)A2QQ5q~}MdmhrVhgY$P}0{}=T
z4&)BTtW5^)1@{5fvfS1Apnfo{JpEKi7c*+JqJp8|7-9Fn+%9I|8&kl%HwkL7Y@<fX
zW{<$YvHy<Y(GPJ(pCFS06ZpKY6jW;LV<b;vIy&wEqdfH~74~I{*3Q}oylT;=*cJ;f
z1N-MI|J5DMdo3Kp<~rp$urfkro<yNV<G4nA$xELY#jSte8;|{QZWz9EXiM!*fSP{l
znEo&8$fGA3+0L^`y5$Ju{L9JEGjg?84)SpdN969Sd9M(UNd}fB7E=_yLn%A?Mn753
zVaXeRk4FLi`SaLqW+9G|(hX`<mjZlzRqlp{$)bI%k00w(g-U9jF4SnJZ<Wk(?OkWS
z$ke9VjTbda(*}{S?g^`Q>e)o9eer^|4Z<105VOFmnGeYV;4}QllM8^0Dow1;r_6OY
z<&pFMw2{C^L9F1+OpA%tsapFHDHjo5`yVOd`QH2M6<Q(1A^r|o_{LEcwxhhdWfn@o
z(X6nMVN<G1o(V^#bm?&XcwT$moby8WtqFk&o8hiu+Y4<4(b0he9FgUsFv=xsZ|ijJ
z3Y%!=nfV_WyZ!ZHV*9Ywk{8E2OI{OM+4<)IA^0=fNHTW)FHbx$^KI4LF>KwiAvLMt
z?qZ1!{UYrw{H^)K#Ke#Zh!H9-zixT8-_rl|Rse!ol$D`wzUKK3e770DY76;7y^1P}
zN#!b%1bqR>%69~GaX^OcC0<<!1{XkvC<|&>qyM=;-rg`Qv!tHIjZLdSyI&)^xS}V4
z&uBPHDWT;KmEdBf-FT^1zE<IS&iv=e3^@{3R&qWEySy*VlP?&M#d$l{VWey<T+4lk
zM(Glp^`Tr_%kQ6`yR7sUW*!cyOL>}4)wq%6Jr&jiPev=mJho25x3NrG^T2mx_Scv*
zRk_;5fDN^rCh%-$%fG$?wAiSI?Tpp!)R;EIJpKa>1p~5*^@^ds{K*R2sv4kUFQCW#
zQ9%1=%1MGcla=;i&kFq?incA1L8GAwB0la!d9NluF{{%?F+vSfwF_6O%Pe~GvgIqQ
zmX${ra5W`6zWwa5QdU#c@MLBFKY7WV)Ibk=$9(o#`{9W(D4X3Rn?a4MdIf)x8hxp8
zf_-J~WV%eG;8d+g6<gKM0YUf8)u}Na;YW@f3Qc+njEbL)qZpJ*LrEB=8&0=t%|*}9
z&zc<@&zGZrMCZLy*2u@K)JPjK5gO4OCz@#L9G6i24=M!<C-g4b0hOeoO8KMs`PX`$
z<QEhO?y~(6J>-OScGCLc_EfDF(y(4JS|MHPscLM>$#HL@z+n5k`=+KH5fvN_M<&NG
zBfknQw$yz$HcO^a)%D&0qezI;H&#T*FHflgr0UB)zhnfI%!rK$EURi4d5|(vJf%_c
zQxe<>_Sr0vImiCz+9IlMw^tnIkr*Mx%S?pkOt-<5tENkjza^&A@;$?J;JGY+?;Pc!
z&WEogkQ1Clz?TXUijU=+_vhZ<cy)E2`&@L*T%1M(@9gj!meO~DZ_gH(g>ogT*MB@C
zv>!qmiB7st+AAqsWUR!guFDe~Y5(XGz5F@twUWextUs9OvL4J@${%)Ftn=D6RAE3e
zS!fAzxl8`916eN4MQwUxyT*g;k$gHHImVT@bP+j?eUQ^dpQVA=(}(6&flx7WUc1lr
z$IA)bhW^J(zURbG#ZASje|Y{}d9={Y@c}r}5CFMmi)N{15b@jeK#qs5@OgbU3XQ{~
zg-FL2L#i@M4kc}*PQti4=JA0RC6l$N2HR(E?ii$w^iulyLo_pY$&*%m;@pc834bHq
z8yW^FNj2hcPSWONsQvcfjt$4s-LQ{DIW>W#-y(ko0GY5t_F@fF!R*y^&zCK+m9&3q
z0K{T$=J=1JS=@Zs8y7z+Xypo(uk#R02Z`Osq4q_Ko4uiO*G_wQufbv^<?(4%+Zye!
z(t-EGI0^Y51-9NlyQ6clfozOX)imb9;UV)}j<ck#Nr<X?F<lhkbZf0^-AE`=_-cq^
zyf<72?FnK<55`PGc#zT>kT-zJ9@GAOJ2F~umY`ObZJ~4ByaC-p(9IS)lKSo&cUtQ8
zrR~^44x9T0Z(m!0zI{VUw?4%iN9(<I`NiA-jD?oq#y16y`cHd;eNG)zWaDO0N-yD)
zu}1!Jdke6Yz<zj^#e__IB(;$Rsjdt+KL<;^`CX~<m*N?YGYzZ>JhtvIt&8r34B~HM
z*w>6LXQh@t5#@vi)^ww^Hh#tj6GFZ)jB6a%>pJ!ih1if2*wmb7Ou50lhk1OL&vh)5
zG}pZvf$?TZ^@NWfXYPO;_x80!Pgc_fz0b;cx3$l3pujrY1Jp#;R%ZI_jYraAY%W5!
zO5Za$ULDT{6YBc=H=G~$1yrgl3$!JBuQTfv8;gfVQF2?UsNAn#N%r}U3Z5=CR~Que
z671u8+UH*@$yRMv6Ht@%;w99ASDiJ^z@>+0q|6ekPCrpO=}?<;H{ZQtcHcBYpj{bq
zyg2K%7Z7Xcox8s_*ipW0>KawQ)5Y5CH_BsM<Egh9mEcg*`I?a0;mUS5Eq2fQgiyjl
zTjhg_T!jjC%jUH}W2LHva_MmLxEY?#-M3kKj`hRJ!Wc=P`B27}eu5s`c7(#{z+>|0
z>km9r)*dX9PLcXWiZ&ch`&4ce)n2=EQG+ll$Xv-EF&$jTt^{0C0)x-^AXL;|&%Pph
z3NXVGoj9Rfo{^&3?V7ujB>TZNz4o(BL2jQe-9rVh9sk~FfnspnA$tL(!!q{T04{7K
z2n3SUcnHsVd4I7;UR5$0@vVn%Mt|Na@4ZwxjW3bMw3p>#>YE4nhSYox`ajs}yV+|c
zHTrK=p1!ky=Dm<EPw|xmx4lJqTpX{+Xw1FKS>pCI0OuQTE)HZW1S9Gn+fP=8TNQT5
zauWuKdkDgFoXIvxP8+R8s~wu}eQH$CRwkx)yr%@ycaqAG3L_<+5@0O7yf}@$m%jE4
zuf~0g-0NW665wgWg^w4bn!Qdvm7&0H^iqjvyx2#60K7488h%=&>8<wSX<);CKsB7Q
z?*v>P1`tmPK|Hrn{WxpQeSjZzGi!Nuvsq2TMlO8SaEEwGaKPi{&eo6AXJ#E#9`AD0
zsHy=;+QDpSWR3)RVl4}*+5e)^%3t(+PaRV!DcX<5LG~a%r3K(nlKx>v@6xBpkL~7V
z6ezoz)AC-98U={tFtr!;t$`&gu%w6bwCnO=+_kQ2cXdk5<S@JecMGkOo!alyS3(o{
zInz!HWrhTGO=@+?g7Fi!yQwsaeW@!Xubp}>4RnOwx&QRh-JU+yi2aSxNa2g41^OKf
zwFzrH5G})V(hT)OlLXyt&=4fZ8(wJTO|LTgB3+meh)!>7W`ob`)7FmL7?7XymdZS%
zU@w5LU`MD;$!(UlHlfu}g!*QQu#u?c%!34f!1)`Wrbj|_>%7pj&BV9UnJxtSp#W+i
zR#n))8}0E%(yzUf49L=5CG2(z1n%NkKTL_K9~>I4`&kQ*iq??BZ|BIX=;cmjxPcl-
z680*c5vrBstULW#+oYgi<My~@y!N#w=hF(;n6Ygy&)8aOg~FsPpR(%)3k^|`B%0=Z
zt{v$q6hCYG&Wm_$)^640E0~ox6WUbYs%deo!$_YVuOO5Pv$NnON#5(Z6r0xKRTYkP
z2NSlzr_HA#b0QER9-nSi&A<;~^rc7DqL?D|NTD8?;~-W1O9zj}(J#7>w@y)wGpoL*
zhbU^-vRWB#9>;0nRo=*p%}B`Wy~u#G^eN|Vwq^^>p?@yR`XEB?Rj))mOf^4^u40&}
z7Q0_jy?qC8q1HIZazb~n1L)S3_{eX()ZA|v&LeL6uwIBib%WeHWD^*}opzFoIl=&@
z=Zjz{0+gXuad(pS^qK3srPt9FtMy~`CJS<=YQgwvGc5g#+g9!M)(@Scg2O2V23(O2
zgML4iHiWf2t&kddtP>F;2ig=oHX15&`*Q3Q4Af+6ovZ_f;^9jDJ8cg6-(5tFklK!(
z4Hspx57$)<4I#-x$Y7YKU!65-y57EO%AFO(=H#Ws76;*VS$cP$t3fr)ty8?rNdV^x
zEwbRsU*smj@4isPCAyIwh%_HlijG*bq1Wd$@A~vA8na!Cx}&=YU`31dYwX0+r3dX4
zj-N^2AxAJi`|IF3=!Q#-XJeJhNh=*<YP(bKiQmZS{-}|oVf8+#Sb*?qO7Ld&N-~$*
z*7fNAyK$c|!LU_diEx;TUrX<_56WOY*^t-GXmYhbJ-CzK5cBSJY@uLhcimEq-o)#b
zjSlTP&k`w-uI@?JXv59}EgiSiNzn^T<c@n}x2;1#%!E0)%Sj+h9DM>}!5R0dIFS6j
z!opARb7BR*;4Q%=f4&^(SjZ*dn>S@pP#_><ztzrf_@WuFxf>#_&i0yOi<~Mw_x>bB
zebz{qaaunk!uKvl&&uY&^~tvCR5Sg-eyF5P$-6|CUV*krBcH7b;{=u$#MFH{x^SxW
z`=>Q5@x+?jCVbI=W8yfHe1OQwAe}P)nTzO%vrN9PN&wg+wZa-ns7Vw{5rvZc02?J;
zeEoyXWs)$B@$eiRYQ-lzCD}I7w-~67{n|yMV;s>fHJ8@cyq{zZcI34y--1suKIrp%
zmow6{r>CMk@+Ls#-X*pQz^(tPN0hA-6Jl%J{lcQp%1!4fwV9C=ufhMo|7gxwYyUH+
zY1j95)>QI0_b~EmZY=TZ7k3!;tvb_Fb}~X4n$4)(qGI{8`c@C{bvNJnAkD0W7p1|q
zV|+21{qLq2PlVl{zIWV(9{>wS5gOBDIYI5U_<`7s`nn@?p@g=o)Zh%i>*|jb=fX&-
zC+X9X`C4l<&`TrhW75VPM*nGlvh@6-8WwjK{p3@(J)Q0N)upP$y)0*=H}L6-)=P@z
z8^7~suMAUiW6rC^3KLy;C+*7G@`?-g<GR*9`i^H5XUzm$ij+QIs>(9oC)I46RMs5a
z?GvSbT()Qqm)U`D<Rcr_qgR*W>l-Dw)@DW->-RYEQZE_Bu!+S|Sa5DqEgD6?Eat$u
zkr5quxxJdr-nXFfK|tQ1Kp;<YX!KQ_Z=s9zaMJgWS~&OhwT<|vJ6ujOvep@lH2lYk
zpJ%!8$67OqHIT+4QsOX&wh{xoT`pQ~yGwq*$4&Fz0LO5RGR0wrf@j(}t!5Oqru;O_
zZ%ek($jkKu+Oz!K2bp=i@(hjE+7-Xs19^{(UQCmDj4q}Q>ke3#=r+XK`mR0k<Mp-A
zRw~^<;97=Z;iv+fY5HAZpw^)dH=Z@Up8H~3j4i(`fmCQlvMNA7wae14XSw(A{#4Db
za$STJ1OV;(f&oW!Z;UO@uR!}MWE*eY0-0$Gk&eom)?huHWEmlY?;oNS#RUW&0ctP=
zK>Iopn_i4&5YHFW*<dqAo&x_itxMDp%aT&TQ({?RK{-13ZdG?$X=AQ?_k9cf>|x~S
z7Kc(*Tlp1ssb)|p9Hxp~w@uu3jduOS5(AlbE;mVRVblj;Hdc@cz>u_ogvGIf9^sew
zm(;aX7h32a4t}ZM(!kzIS;=<`YpbZ?@GvNJ(#iHY+-|OUPbK8et*H^EUAHOti9s%I
z2G43BL+)brP{}BZv>RtCH79xGL_b%4NwSCZQgQT~9&-naze+(DJAE$@Z1d=VV~(Ir
z-F0l-yANj-e!8l;9{zBauP<-y<;|OQ9l0yTw!dprC{V3iHnm&9tt_G`yDR9n4s!kK
z1FL&;0F|sB5QG}Non~A<GC68MH*mOQ;4gI{{7<Z-lULSxkcvh9W=U(_i79x49aaaY
zo=JO;LO6alJ}Jaz0$_HA2B1v&V*!4k`dBR`>|l5D?t{!6<h^uwzpv(FJ?j>EAI*2V
zHjk}HdfYjFyvoqDpD1VE1aL*s16yked8&#}%Qmm;qQQBs4z82ZRCL^(9g;Q}Z*@Mm
zxYYYowxIl>LW2}He*5j8)yLjYk+_0SgE%6+BV(O-d9u7e3i<d(JK5CgGQJpL(tl-f
zTMdBm{FLh?0(i7XKh%yXbMK|0wR$bV&%lre#RWqe97l?DsQ5#)aD2{2RRv7m%~qx{
z*ON<AIU#3YSQh-{n(jhLGD}_s4`fJhe<Wu>Ck1YudeDd2_LeF#>{gXxlZ`hpN<r-I
z1ve%yRD})g-kqELl)ZLuIl9%#O797QQDlfsZqM)nd7JLc_T-r=idh4X6J~3?oJY@R
zq^j_Ypzao{uxmjkxpT30VI)C?U&AyPEP7MGI}z!!JJKNlBuK|M9>6x7zAiSNdT4&x
znuDW2P$u8CRJ>oOS|T!FdpSLSL;_S5MFm!AJ;*@gzn|GXxg7!voYpMRe8h5$l0cLX
znHe0pa3~bI^(#hRs{btb+~A;xj%V?!@O4(_*1+Ct&Zy4AEm93}4}SoYc{VxP+Dhtu
zLDp6cU*eqfL(t<pI=ySh7lCT*?2bX_mQR^{o%?_^5t#Ka&+zM<v7_OPoE6SOlUDPA
z4cDwwZYX%G;9=kvb%W|%=y*>{-QY9wwfYX$*UWY1yARt8@g0bnasQ^$G9tx{!24Pu
z9vE+rqZv>k=()mlo3K*<6KuTQ?el8*gPY%Kvvf{~UX_83SS(oTcszXmFhoFHyOAlz
z7rpM>_NWcFEfZb@FJ?epqek-c2jK}zc|~fx@?wgDKB$qvM74;p11V*;zb~Z$C?3ZO
zAq7la3uErjv@!7ycRE=$kdMBk7&f12IH|piBtdR^Xh~PG*3o-^n!hD=PcKdLS7GEY
zk|Qn9gX>f>wh9axL8-RUghX~I#x52jrwgYZ-o)F^H9ae`N_u75wn22<Wo6W+5)xUs
zAF9{E)WTh<)%Ni3@t5I-XWK#~K4NyhEYnDnGBYPzwqq|Df6EjU6<stOl&m<?Jnu`1
zNJT{}BJI302U!t@H~0H+e{qXfF+HbJETDc@Ii%{suYg~(U;jgIr)7OWx~v630qk3V
z>Me~7mz@fQdZDS>;T5L3gsIxezq!*uu}mP-ZU|$pGvQXoIEKtXo|33>CB2HIS8C%Q
z@N@l%0K0+X-V1I#@=eT#((HsvLQ<^={-brKhd-1>kLTx&ShH9iH0-j!mAf`7uU3MR
zKM{(2VF>Af4yF28KQf}Vem$=cLZ~n%`7b;07>pKLOiV#z;JaW#rUR8lrl*{@f5>^!
zPmr-gHFtDQAh#aA14`t+wvq+RD9nX5pGuZIhcAFHbm;!7?KQ#B$nkN_hdyRfRznTS
zteo_3iy8`yhr#ndfMh>2%U4;cEUne_){8hneay!e#`hxm{IYWh@xM!N_BIe^<*xoS
z8y(Y42x4T3XDX#X<{uoDa4z*1foLXzKJGk9zo#^sgnHwhnEeGdhv)?KzXAHAJhQAO
zUctf*kUH2Dbg5AeGt*noBRSY!k9a7ENoMIr!CiIrxD0(0SkiaF7jb)iSpm}LW%7R+
z4uBoOStMg>X8~BB{4w1RJ!=L@bx>D)y&c<KvoZYW#hoL|L8`N|b%f64=Ib>=9Ppdp
z$!alo0`9>zAHGc)d{j)GOiBJr2e~rQ&St&d9l&7XN-Hgd_0NBcd2f^j#6Y!<EEE3b
zx#dXF75DfOt(;OSk2A;sIw_JMF?W^1<h`FpB?A0QIOm*e6R~6o3IlPlMyMD+`fHw{
zcbRP0K1+uR8pp3}kSK4<2D^C5o7XYi?uvuco>-pphY$v6=Xh$DRJB~P{bBFKXzt+D
zM|UW}@54F3maG#jvgQPPg9_WjUp5UScIC5>h4(qEJWk%s6SO^U3jq6p{&1E=IcZ{+
zTLB+hM*a7ShFQUSOtN4C&(V!Td}`hI>}w#fYsO2bis)LCsfx(}P!w2{g<bV%9p|q!
z)3cP2N<<MFzk@Qw<J7yGGS>!&%;u0U?;z$AY76`w3TDi#d$a}%Oww4K1eJjuJhRfG
z|I)1^1Vgb+{rE<TKN%sysbkO&NE%IpCcT$_%YNCvc+1t?yJi~`@tCH=XuNBz4`4^0
zy?o$jpxb{3Un<?yNOY(QctMWVDo}tHI)v#Jc~;fFztLxi!C(c;QPi655mEHyW#DdR
zp&<-?UH>inEvaANTsE;OqU+?!1BPpzDT$LIh~_^<s6iT5Jp`++l~r0KLT+)6x89M%
zkea^EXq`oat^hqym0Xj16wAy|6sEBMEd%ditH80}q+iT&uM;q^aW7jwTU!Qce=hUd
zxkt5ZBrZ(Ozi$!IaN;!F>b_#%fM1uO>3BK$#HOjPlcH90#9ks|fEX!vqFS~QSnpQT
zU^nL427|kRw*F31g7Jq6{Yh9$#|?HB;k<hpXv}AT0{3Lis<QCTx5#>QX=+}SCX<z@
z^0x0QV8H5_6qAN|k+z#~SM6ZUwnx8A*eowF67Q!GK({gO*X5y|UP5{!K2c&fUYlW#
zQ0$%i=>19EmD^~&D;7daFOw2+$K~_9)D;i^ze*F-a8EXvJXq@5)=1i{@ewb?<Qtlf
z1sC-?MeWogR<G8&pfC=!9e!s(Lr+=5Hw7KjdkP9rQNfVm{I1Ptz=1rUWohGpJHZ23
z1({`YNyZf49CZ1w41pic$!zD+9VxDj+!Vcgoe7#!Z+MnHuh+)iwhuS_%e@VgqC>GE
z;!TYBN3Vh?z?00fb3~r%ZC_|8rR1!bi!`?(!oi?B*wlPJS7jlDlV4Q&RsVRw6bWnu
z){B=djJTd2aZn<3C@_>j(1Rd4jZY}>@f@>Si~PK)e?7gQaZ7|)`!F$@eea6>t26g|
zH4l<<`CA@gf=#^|@V@2x1&3B$4*Djx8z=*P6DYml0*$*;ANjd90Hv}GeJ16S6?=<V
zZG-p-$vt=rV-6>$%J0e+3cNZV6TY^3fIlMyL(rm8G2i;v%*=K|wGYh}<Dvu9pljx+
zQu+|RlvKD!!8}z6hugv17QJ+xb;aKu0HE2q%?+Ll=fp=;)aAYeievmVzsKPth!gE|
z!<ENst#tIb{EF&~${MdKV0tJVR5w$86Q^eH#jYoX!I(N&vY3iA?Fwmt(woSRJ<&T6
z%1D!rit=oxDuBBGz?c~;#s@z96W;M>q<^_l*=Io(EW1~{;xx~;!D21T(MwW^_^e$G
z2UNuO<we8{6nrtxkckYe`aY8fV7`8MBeO2+Z|d?P8NdFLj~q~UR4`8?#fHaRkLTN?
zhT`4+IGs%#J8+6792t2>VBW=raY{I4dZ7MTqb~EP?By#l-5WogqcXaOwqBjm$_awr
z<cb;Zd}TZ>m4q@Gsyxm$_ApeY4jR=Q`hZV;Ajaa1?mFZ&HhUM?r*bIz?#-&uQsjLX
zEo*<;6XjHP&0d~8jN&8QQYB;K%A^;hiAWBfVI3z~(nMQxC<cG)D7Zxw^_At6;eeo`
z2|s#!%}snkDu)9c5{ADhjX=KzD7=i^KxEvS>A131f!v^rtYk^OnsZ)D$AeL&R_t#i
zSJ>c+<CBu8i@?HpPWD9?&sTkJYB}R04uo@^E2Phm;mQ)_$bbs$wMEJ+h|Dy*ar>l@
zp$cxA92YYz>MF4SdIp}3R=!gBM+UJli-8;gBA{+fr^#{wlEN=jLeuMIspekh_LPiy
z8}`KdVN}uMm6u}V0N<t<941K&oH#Xt8g27A=620;k)@nkvyk)IMilhj)Hjza=hctn
zn#8F4`3%S!5#vxruQh^&=xKLveIK-BLS!6?Vc;eazdZ9JeW0}M6^RShuPS`1F5m(R
z^yW7q^|CWV#^q<XFJf~Uzj*;g^G5)u@fsDcLW@hpWYjd;wORBf^=!(s0u;nrUWhO;
zW=57=+5OvEpw|0K)W?Tc)Mv_vodzYKByaecg?-O@HHIBfTh>*nehY>dcwK0dreGfO
z_OX+EMDqRX=Jq|`zeFVB0jWKp1i$=?A_%y{@aTt()2FoS<UFLt_C4ezAF+YMk2Emg
zE~XNI9Ldma8vVZKM$@PM9qRV{>Fg2UlCa4>sEg510e2b{-{fm3-=E~QHKv~NRm41C
zg&`|)8#zj#?Mh{}MwjiRj}0`>bjJ{`4wJ1hNPW|cetah^tb6)b{}(_R89SkmDzniJ
z-bL24leEoeQ30#(MADFjUUl=5A+UM58=qK;yGORR=3EZT8mD(+!jBv#q)*Rw(GBi9
z{*x10j3UOi1hk}6qBRTg6(J|}4c{B$xx2T$w<u1%M$%Rlge`&qMck_XMp(a_RKLH}
zH&XE2V6tK*%YC+YYLjr%wfI?)XJ-sjH-XgBYy3U-LHmm6xC=E@Hf%@jTgFYhJUWAd
z)!1@6i;f^|Juaalm#*HO^=wFmV?eU$?e(G3t=fu`uO3q^uMK}pA2co=m)qdV-8X(s
z`MGEpxL`84%wFOc5N}HTF3tod%-QAOx$Ux1DyL3S_bj$t;A_tWw{k+M+K&wUqxiTg
z76$tLoN=coBO~7<HWELUB?_C0qwBeM-mJNw<{M4!_xW2C#t&6t>z&R-WB?a%q`00T
z_fBUnNE3AuEcSytAXAfLGlbiwB7&9WFI(Q$^Ljg@BmI@cQ{8N)#vTa!6rZovjBX0M
z1>M@LkE=i3=Ww5UM)&g2dn3mq%++pq%s)}E@4<tE`XryR2=}H;j<NWcI6kO}^vx`?
zhT>6Cy+-!~9>z_J<7xr-i(qOzuf?_F+*3!()59&BEnV(%xk0kA0Hd=O2$ks9%FQ1?
zd-;uhCNsRA2|ZVVPI`3o@*c#_Yz1)mf$L%~%`2M31y6x%Q5F>vCeRCf^UL6t(Gkj+
zv1cEAcnQAi&F9<3^%!XEt@b7G<x%*e_9pX^Dy%f1Y$;`1^D?3(%r2y--IbHP2@?Wu
zwyg{%cq`&|V<*Ds7$?ph!z)^3Tf0sTZ%%)|lz0f-{{Y24ya*Yq^~k0fL<J<-?r$tk
zW<EQabfIGy%Zv=aRr91s&;gc^8c;sHWXLKyDp+A{Qk7I$b<S(~!Qgx^yO|%_XFNkC
zIH|<A^}y8<;c<Cg3+diqkbCI8#0`t?NPJ0YJ5&0BCr2;8Rj%Alzi(&P8D8bpRt7z+
zbl=@YOfqS**ZW*HG@?x37#x>+*%n;x+$%8kJ3Du6(@1DEGspONeXPK_?Y3OSdXc7`
zRDmBViW#fm@sk$4Ji6<c93!)u(tgeH-q@20D}9fzO3i(Mwo8lFb@G!CmQ#~_Wo1KS
zm?N|k?@7n~wo`=Pw>w%X*CM&L?^)d57%mOF9GY=}SQ84(ufq!522b2a<APsGhg!;<
zSyAcoRwija+?UVF-f4g_<Z)hwd6xLOFL!v@+uRNGCgGa41raPS>jkUu+pQXMF%i@O
zqHNzzQ@V{<AnnL9vhj!Kwb%cttWUeIG_89~?N-~7HAuK>FRB=4NhVV2Ki+bE`hhK9
z@)w^?hb732V)WRv>vj%?s(u^YMlb#9_g#J*B#eKkwYP>@Q}%sx8Vuf=9^UD!Sy{5^
z3LB?+k*T~RBBfg9$(!i3-plE3uhjTP_-0TzZm)prb-ns5<MgfD>m$=v364iAb3%Ee
z`9GXpoC`w8sT&gcyOWPsJMNwu?u;?g`}?nd#kjYX)Z&JdO`mFJYbARHcMgDU?dlrR
zg@rYjeKIB6&)z=;CeEB5i3;-gS!a)qP45#tWPekJ@|&49gyn`k=+xu&aTw|H&GJr~
z@Y3SQUR(-T$u~-`y%sr5yA}}Q-B!wwE7L*RJP93l6cTcJft)YU<+C?7f+lO8aY2dR
z60a7nG$i8KTC}G;I${Rk>^=UevO`q#jzyBAr+-60`CN)-l(%^B*9{n@1>?+#@)HD&
zd%$J#m`1(nMbbj9)A>Z6@kP;`=HOQU?g0V0()8K9rO&f)kfGO-54{%0&pw3_?iK?r
zd00eXqz;BXbl@ZG9*3GoYx1OfIR`^8z=@lvXM0%J#*D?pCggKIspj<&?}?$%t+ud#
zkGecRWb5I6JkDCB)%ZmAh8@jp6o^{oOkH)zx1{;b*=M34U^{yB6T?p!Nt~IU_V5PH
z@UZVr^L5~I^%2gP@6N-QFZ^n8O`l|+HH@{N^YU3WzneHxlk1p1viNaYiHb1Te0m#u
z*Xc|~e?JdXe^(R#k`CO1HcBUtMRWM<mpA+Imie)J_mXmOkgqQn#_QSDV6+p)f(zBi
zIw=dStB-3MpQIzbVA9;jMlsXy2}1+QlFnNBH0ze3W77W5<f!SH;%J^4!pIw~HxhGK
z!oZPjb_>?dxZ8uzBVdU+#`a1N-_JA7hCjGCpPQjb=Cz(~au6_w3T+)c=k=Y!9Wmgh
zn|h<s#^4CvM>`ynu?uL=81%eDMQf+N5Iv0fN9{EM_mE@~rRLd4IHD3;Wa4^ZLbQNA
zBEv<cahq3Vfw_KTAuy>Xo^U!s30QIa>sA&u9y3082aeYrsBeu$Y;*ZdtNC0+g_18$
zK?C9($Q6~>M~bYUh+8AK;0$bqhI+<a<wo3*g+dkY8j?CT?^+=T>ouI@-ILs{dPnp)
zp=$c+R#fq7s_)A|XonX@qJuCIOlaoerRj^}y8d+C>eeVI?d^ty5uW7J!w#eCj_STe
z)6xDr4U?o$-l@`O1ce0k46D)k-w)-Tq=n^jfqO{IOVHkC#L%n_qF-dE@tNIP*Kl)d
zf>3$Zc=I_QvKY&w;lQn!vcE8UJJb9Feow4Yje-Yd&En^aHq?3Fj}Pu0_y5`IBp@sf
zStk$J3C6giC}wm~4jYZx>;b+RH8SqSdeBLAQ^$4O!j$CEF{5SBipxyG%!Gh0ZzpRd
z<W^Iys;my<i`w2XQ<IS-FHdEDcl#dG7WK#SA02QmA>QXAq;(#4bqp~lPrVzK%d$JU
zNGC{8mnm^e|JYB@N0Vn&C@L)ZIS5&W+=Vuj$~8XN+H|~b@96K5O3$h_IfHs&_>&73
zhl-}5yNFc_+CNWVW|Ll~uA97hnCV*=g@i2!+%D3u#}qWfivl^Is`r?0IdHfaTNIoY
z>~1Lsw8pH(eCQv_5U0{Bx6|C~cuJkXQ$TXog2wPsWKHPWbOuywWWS?W)*R`JqK3rv
z`RVZIPWcDk2`Ed;HgU5F`?xQIWW+Yo+cm?pwSA3VAOdcNc$Ro}TNKvm>uI+izG+-O
z{j^^bn_h>C_KYqo@I6vyR>-mv`g2``M<-#RwvtsoI@iqm!O~QKU&VzDy?LhCKS}#Z
za8t0xQ}s(?0Hh5W1s5=j+XYKmGj5p*cITU&0Cnev%xus~dT1bLd}mqjXSyDP@}JVS
z{r^upSN;zL`o~v^a>P0+VM$+#+30Yk9Bt7mXYOm*$`q5vk$W7IP1`uy%@yNJhlnA^
zgfZ$RIYtaJXpnm_!_Z(1Gh?4A_O*Y)_lNyuUa$G#^UP;H&+~kq_vd;CGcFxDBzXh{
zN$uJ|OH8CH4SsT01{Tkvn(YPchiN;iZo;O#v4iFpZ~Q?1dH+a~NyXcch}9d-<Yu=p
zseJ-uvKTQfUjblsaCCek#*XUlkW(3Lj6=%hh{|;EM823NdAOxid>pGcO5KejiQ06F
zaaEth?!I32cGlob<Kef5<)YGOqXJnk*LWUdaQGvBO+jN1DHy7%$tQQv7pby?W<Z4A
zpRj(7ZZx^$_faBs1%h6IKWg4|o&-^X93QvGd)QbBqqL>oSbDEyPD}R`bqu<G0*<M&
zFO~@w@#V952pvR#BE>;bI#!{^!QE~myH(j`htrG+VF&B&b;Um?Ks|IFsAsVWZ<SO2
z^Y)GvhtYbOXiOX0Y0Vl}NgM*)Q>89?LDng!z`S-Ce?<O;8XuH<3-StcjIONJWj<Y&
z0>)pigQo67rHYojo3?8S>}m}`S=6Pu*Lf1lhc6{}nS~kI{<d-J_lxn5lx+yWW>XuB
z8e1(!Km#4n&EhDXjN3{Bf*>eW3luz%O8;U<`4@cOVkbd;1!C!prjo5H6$pF!04aux
zrT$1fSmImDI{(*Xx8|R}R}oF}v%u(f$(|O<)^`vEweJUF1PSa`a|jT&=W`+GjD^W>
zuM=y+24Y0kn8Q{z5^R_+Fvlx31;qL9Rag*%9Q6cZY$H6{<n!)NFVk*V%6~G)d`)*C
z4P`5wDe}~#4b%m=)R!Bc7c{8ym2s!xhqA>^AGX>lAgC+T`o({zF&MC3-#tm48c^Tx
z`-USubbYRd*VgL7-<{m>?~tiqVt4s1eeIhKI!U|a3b5I210RWv{_{$MsizZq%Q%_#
zqc^k+NFpLCt1oC!NPsJVzUB8j2m43ZbNlP*OMj_COWGx(>TuQa)<eT18`FAjHG!56
zBkcrm?A5?O?4A6@)&gjAbMOsx(VHRL@xqAHeLH3aj=_c1`i&53#Rn_z59ezw+94uC
zn{Z2@Jxgk)&fG3?ZHP-x3FxJNE$G8uCl^WL!5DUr88F9ZcMTD=iMaq`1x~@E=0n>=
zRb6>TdB=&l94~xdS##^-M^`!acA(igdKu59WAb1WwgmF#U%y%gO>%)&vp}m;T#=w?
zIrVa8`uwf8<ar_;s%bCoH8pp4Jc!ZTki_MV8j%aHU4|S@9Do-d`4$+!MMNymtHky1
zRV=U7DR!c$Szr*Gk+Tqa?nGjsC$ob{#HD<29tk2VpW!C|wu;+{!|@D1Q|797t+tjk
z<_kgz5KRiqFKfU%W&uG7_VE)<FB8*fYu1{3KxK{&#>FjAn0p!UoskF_BbuAwu}1nM
z@Sze$xC^dHdhWx`f5axfG7TYnG{tq<YWJ5UQ<~?<ojB&g=JC3=)!E8SF0ClX?@K}+
zPPU|9h$1mmi^;)%sd=ZPXdIlI#AUO_xm<H^ruvdvG&pHCd=IiRRp7rnRkJ+pLu>f@
z9Zq6T=pTcwjaCE?gQty#dROmytPlK3A-Ks$6)=2HO>2p3ATQ3>JQE)*yq~kyh?}Qo
zCcI+o!#Eo@wAh7X9a6CmtLfyj%;A#u>3J;DkA+_<ReX6Zd}?CDllqw|4Re*Yghp6R
z{1B7VpkMUVBDdl#&)|)dKV1PU00}qBv6pL@uWFdbolc<7y)gb3-lCJ0@R^%^&iMeG
z)%1tt(DN=XSJx&u4yfZAq&e>-F#)3D6Gcw-j&!xCcwd)|^rW9dhcDzsh0eUHz|~Vg
zj7h5IDHlCnrykdJP|F+OhJzHx7`U5qJotq@KMwqwAC(Y4SCc3x9itl&5GPJ_Dy@#w
zjCmPG+k)N{P2V3^(wjT|FhVKuYF*9;ELson<`K;s=`2@Mw%MuL@dws%OGqA5yesPV
zMS9J>6-u#(P4}|M?cVs{`Joc1`U0<U{iXH1bv;5h+Y@waacqxyZH8a9r#}zuDtVvG
zvGEhYPI~e(1jscmJLL_K+3aAfl>s+)WVI<ZJf!=u(*8y+3fCf(f<o&p3JM6Gw2D3F
zE*ENky5N@Nnq_50Lo@b#32=~;$6`lZXNNO_eW@-J>!z0vH7vFX!izMSEUfWoUZKc6
zT$mZyB~z9<<jV{V-e;&}ITCCH=e-{HevJ1HnPVg*Kr*|zmm-k>bzi0^RNYDsfj(R4
z_@E(*WDyxPv|8(89gg${6Y3`1q+Fc!0Gmu|h^ibn)-PU<UgZ-W$%1G(3ahBvbrq%b
zyhFS*ngvzVo({0BmEx(!bmy&8{sGFB8ZXzivec|Ry&^pptLE#kWhm#eS*Q>y+*5c$
zn;2R81X;+=U^%-Dj}sK^g0GxS_RN`p(IRu#{qfag{g0K}<WMwEKWwVEE$#WDH|{J<
z%NaV$DnAS+isFHH=I35a)At~Cd}@*u^oBbPiXKKSgE7@T2P@i+tZ2iXFUsCkUCn5S
z`0QobF+hDnMfdkD-zFW?^@DQKpSd=XS~|ns{U@zL{2Ld}nf8VrrB%02x*=mv=B_yt
zjTb<tR?izpnoh*mdsIlucmSte;eHfZbZGsn6WJ^;2W~a+5!T^WZ0ru(rKRanp?2V5
zeYEu{&oJXG$5Qq3)1pzcrLb<xcN6jbbx)9&@R>>=#dS<@#=ts(c;V2G1J0v8Ek-q_
zM5jSWhN3T3p)M|*Hl2*YRZ9!4ROywS?dUYT)iP=+P9uxd4HjPR79-ISh~Y%FJVksc
z!|L6`Y9><~R{4OP=w&_C;Al%YCnU>ZH2)?2@knfT%zLYIa2G~d>HKOhi<}6{6K^Cu
zx#Ft3q#y0WeqD-0UH*wXV8%T@ykHQN&qfR6f=mx2EV|<-Hy5T^a9mff@!nZGmBWo>
z;aQ2tC<=m~EvekOEc`+4AWjaWkz!5f@yV|i`QBwy=gIFNwKQ6qChWd`m%aZDPGn^B
z;XBu8SPat}jmI#RSOg_J!`v_r7j`d+!;q8@CQ6=hmPRd*#uvKLFBo?UTu(GM(84z6
zlCSAR>VM&#y2E{nc?~VGAUxZ57FKAzx7JxUqbX!?$f%Zm;8>9M+uJ>gNyw{LOx1sC
z!Ok4O6Dnf09!?SExvw0Tg{$bPxNKu=00HaU_KFxUMhaGtvY4*32@OFj=Fyr?#78kp
zh9l;nx}3T~KP_$mlU08|W8lR}_APgn8UQIC?`lV!+#S>gsAc3-NFcL2{+b@#6j6OE
zjoi6hed6-vk)A4KUzQXN{$YX2@T{yM;W0r?_9JlFsruib-ry}m7oRFT4Q*isT1Kx;
z0*;x){u_lqGs6(r5_1?&*up2TV5S1!hd+MB`K-N4o#AT5N%s4NyhJL%Isco;J}tCk
zNzyA3c2=fsCrWbn+FH%m9*!!67&+J;8N+wKh+5&K@Ct)8i<n4!vXY%luA`mus+xee
z_ZURNJ{9bZY51Z6pguVtTgLQv7runrhBVi+!JSF0f0_+)w%;+1<W`a*bqBwA#0Mj-
z{CQx$+c~t2R+JUMGujlOcy!mc)Sm(Thjrmm;4j<w$wZ%@X`L{z5&1`Ig5OU*q&oHc
zHdsRW15_#25}YryEw!!JcU`{qasT)6pK<;>7XLRF!sMnPPl<gz+Y)KZy2_`HHaE4p
KQ1*-0-M;}*C#gyR

literal 0
HcmV?d00001

diff --git a/site/img/simd.png b/site/img/simd.png
new file mode 100644
index 0000000000000000000000000000000000000000..b532f387f17315325de57d20d1418a4b44727345
GIT binary patch
literal 101031
zcmeFZWmFtZ*EX6EA%tKdxO;*PHn_V62=2jka2Y%h+}+)MaJS&@?(Xi+ncUBv`+dLX
z`ObRJS?jFx>ti)TPjz*5)voT{dtcX8_{)41L3oe-{>6(I2x6jw-(I|URr%rtOyIk>
z(2}mC4jt&9mp0!-_+OL^;qF4ez*&l_*t~cFkNW%f<%@(QOlXBkV|is;WhqH^T?-Jc
zww{HKJ}nqz39bF&1t*vtx(m{`)g}ak%*<`r!Cb_D)nJG2|1Jg)6aH1j)|87_SxSab
zz`|Oe5J<~JOGnK8o{*4`(^}7f{hOfh-={;raS<EY+FG&$01gfgv<{547S@IUdNwvT
z038E>fq@2EgT}_u+*TV*V{Sw8uSWi~BdBkqYi(?4Yiwao_`6+g9Sb{KE@I-}7y9St
zU;VT-Hu$eAncMt5Ea(6MzmEXuY3Ts}v<*F#^LHt`fCb3XTHnS7+CDds^RJr!OWA*2
z=U??@j4W&|pjNOp))h0i)whP8Y^(jdciaqrpZ$Md<KNqowl>y>j{2{&>Hj|aznA@8
zpA+zV;Q!4K|FZI5rBE|-zvl$}GiBWGOV=3|U%cRbAtuNt4}N)&3=5W@obqY85T_&}
zeaTN~8)lw4FVkrKhT88tKwVb0%(F#9_5gf{QTF`}^^btO&+kOTmx!XmcnQ6j#`asL
z@C_Km$m3tY(JVWqMB5A<Z6|B9?iW>Ezx_<g3-f=r|6_swV}XCOK(|EeCK%`Yqmy)+
z;JXuN`Qw(i-5=7TmQ!^Nx&vOc@I0X7BE5Wks>PRnRV4A(BV>67H-gM|oBC^)+jFcW
z6u`N7?6J-x_F9o=8Y4Mp*(t)wuqKzo<5Efbmk-Ky%2Ca)(^OIzyq35ku5-2>(o@5l
z-(8SVdwpY2UM*UV_mS}B|GIggivM_f2-i6{B>Lm^zilRgFA(PDh<ok-tsMIG>*tr)
zl@y;?I7t4jy_X;JXkeI(Oan>hiT>jdYx--WkrbJQgP?!ElxSM@yOFeFnh3LPFVquj
z2ca;CbMh6=+V)F_ngR#AodxO_!LOS3Y(JPS@Dp}Wy*Q#Z@8E+mWO^Aovch=x1h)Cx
z(lYbVD@-xVRy(x4<K(IY8Z6Cc4&BN9bi8`Sbvlw3qi{w46rg+oLj9)C%@`8Q`@<Jd
z&tj%uc(A$WkqSuk6KPO4l;A+20Kp^@x5cI&`+pzg{_&%NBlB`2cV)Wyph24UpE;dG
z%~QKpymV!ZeH{|mid1-|3*z6hs*=^%e8~7HsS@@>p@Ff0s;iNmaD=1?f!4yFRx!tn
zrFc8`(^@+XkIv>Vkll#dq}0;b6~n-C1GX4jz5<30BHYABk0ek-7=t0V^*X9)z7Vs(
z-_F#K8)|dICqtL6rPXDtU72EuTO)0L4d&tMLPF$S9Mvz|?rLrYQHs-WAziZ5uH}z+
z5^0~gP+^-N(w|^fk|SThg+aGgdgL@d=MN*woopWS_blzvi|tasip6HZ+EeyfdT1G9
zr7ZZk6EKGv*o2T$%w@<zXg)m6l3xnF2PTE12RFWGKW)c`V?(K}2z!Az!7SB#yzFF=
zmRX%U|A^{tjVYMqL?ExW)faR#F(6298)Ln3hVaAg5@ujcuJ*>QtP$zCru%Z7=eE9j
zCz-<XFl>!&8xj@LN_SOn&PYpx_xe_|)Zl^!FvqotHMhdVsXN9-Zr=lQ=?vzk)_E<{
z4M}$~9S^MAE-ATtPuM))^v>pry7P7wkB%t^sXFWUR}~HH%6D^ie9fzuCFY|@?W9~S
zwaCZj3Eo%@=m~sGr||70$-bGcmP}^>@F#~QLUHAjI)tfsQ|L$z&FC*v+(i3_Bbp7q
z&w31k4;vds$n(BDjX%8~?|$wOXry*zG><yfm~F&pbM-8vr@;?}#Cf{1`AYIsyyf<N
zHh;jcslxLjly;X~I-y~BUw$eYyZY|(eTQdO6?iQ*|MTZBZMq-1n`lT-ieVC0BFMR9
zzNrQvF%SP@;dtg`wk*;oL1xw?^t?+oEjeGiCf?1oNbAZF$&_KjXM5dTtRtf;@6}$&
zywa*vArp}p8YXeq3K1|RZRGBp`g)=3K%{%hBdS%48*G957SvBgjQi`=kkA}~MDS@o
zNH_@led^5C=UxEs))`Rr$snXsE9foPF#k;St5y7&7edBZGtalzJ+tNQQ*NE8!+yd>
z+-@J%@Lf^JX6yW1^<l1DZ^+twG~cJjjDDVRM|S49fw^_QQYh*({o9-5)%<?$=0ls0
z%p0B0c|Y*5d{k<x8pF)d$b1z>CY)X4OSOdh>a-i!3pTq$udS<HImF>0W~*q;Y664Q
z8d(it-4??<{RjczK3v7<g0aWWxDOFRiSoA1`}ZskZGe5G0FMT&vQ@XPT36w%*^qnd
z3S(2=d!L!4DDOlb^+1Dk?wgaT3Q5*eTr6pYk89g-1k(MiLm;fz$&B|8MXV2vv{;A4
zkEc@zc!dRh7rLZ$azP{LFHyQ)xEM_4cCk+8@Hf5c=&7P^-Q3*rfX}R7g=E=ba$Z|4
zES`$fyAiZW!}ZgBA7cFgTEbax&<ypCNL7jZ`Rq^6={&sLwPR-!eyp8(AD)zGFh-v2
zOrfd#$hJlH^y;$5slGn*bvD@Tl1Z}V=8Fp!3xRt8QGt(foCC6F(}UH0n^Lg$!`f_(
z?t;qn_k}Ts!A0R)V8@Rh+FH;rsxHhFmnW{7sA?RNZE2?U=i29X9lz^NC5|neS5A_?
zRGoxR?;u5wZ(T`-+V*t>A#9%vKP>8YC><BxRtF08v^M)WgNXweD^})n-UQfKudc0X
z2XzLuC=s0&hptXCeG8hWLIFBu(d%e5HLWhSw&3zy|9ES8dB$XH?$&HX>Z9Kk5$)%`
z%k@cLPdxD`XK(;*KaB2Hw{`N(%S8?>UB&LvjS^Wu_gmKuvw)utfd+nVCy4ZlS`k)w
zw9}!NZJm>7_(Xh`;mg8k!=*C114oHxRrG!&=FM-Vj+;@eLL1-pz$3r^%PT=0(~pO1
ziFc$>2Yx}{KRu3MSo4m7>NO-}_>+YhoKqDdUnvAOrZYQTpM-}A=^X7R1M1_PYMX>a
zmJSQOkDdN%)`d$)?nXkoLbN&0(Ngix)#F1S2e;*5&x~(obBW&4PvTzKWeATn!WJQz
zlY0n?>rUg%)I|%?9@QwD1m6$mi@@BKBD#!*4pV!tx?VNXuF_rgvwo?PSh3WmwGKE7
z{sra@8oKbL_Qlh~*jn|yA?u+T>MmN87*R&INGP)*df$1^W;3m&6A_c4|DE)X>Bx(a
zo<A0U#!tB}1kQ-s#-=8{1v7;*JDaThg@Wl0W#69hVzOOp7QsM2GQ`;bA{7D8*UA2R
zJ7;auZhWu`yVeFZ1_uJkx7-*;YQVrDaq9e4EE>*?F45wI#KCaw@9PkVSubVQ`Mgl`
zR)Vx8uM+e8x)?pw2Ifx4wtDk?1M{6<`u5%?89Y^$PhcXBb@h>;F(1)GL#1yb1Sul-
z*=wwgc;j&4XVh#9Nm{pxCDQvmS&>`k>p&3)W^0FzOgb`zr{ruYdfP~{;>uFw?6q`-
zpGssK8B0A&jRIrugB=utC)y@#v6Re$@5<4|;~#O}M<T`36FfX&fQ5iQhF}3x=FmTH
z<zu)J0kz{6m?|V*3|XhkZM<3=UPehw+(4B=#Fbzzdm_%(am+}Q8C9WnY79^nbuHsn
zMR4tQ#MaI2uZps7b21i(DFStQl#6pd_#|WtH@w1pbtkI`{^$U|Ew+fF2V!uoQWKeU
zCp*g6Pz1;rEz;5TJ#r6iA7kd{Fd1DmAToTw<~f>0q0d{x!0?6KQx$UQ=cvBxuT3?S
zy6Dnfu6Vwy)BO~y1oZq`%8zf>LNTjyc)fYBGm799^I59X!NFCb1%e1R`*D?$F+uHe
zie@F^WAKPL(&h@ck`7;a1)Q#>?-7H2Gy;0mAkmhq*)z|bNGE2WJlg~qwwtreLimnC
zxN`VHee_$zCNE_Og^QZNK4OVFSuyAs^Aw3Tmz4#pMcUnpy>q2#Z(zR_#frF65B1U8
zP1qEzc7OUOU`yMhpdTahSSO^@?R3jCQ9i;U;rM><VMdy>#h2-~`~<22H&e2dXpTR-
zH1cibI;}iE;RT93O@7qnp8ejKA2N2RHj^42>Vv=uTpOOUSKCz+YHIo=!XJ6Kk9+8t
zF!1u~dNpY_bJM=OpF)qoE3q&!ez+jL15H12;Ahi1WEz>;Ml<!mEpq^b=P??n2~i$U
z-(OHkUcbKy$e#(ThQ|=Gi@hzYs)!Er;dgj%?9qA8!hVqG52OgYy4T5b>ShwEu&N;e
z5cPuP7k_r57OpmmwAs1pP%oXe2^e>MwqyIKsXm@E(R%p~F-n=@R=a{Rs;U{ggzU|m
zE@!pztEA}hx)w@y&HwTk-Y*0~Zl&_<V6=6Vt9f)J&6iVy2a@=A+9V|$vWRliUhC4D
zn17HyR2J|C4$L@}3@7Y<a}%*fEI$t@kYb4!d7?%iHcl@(vehN%e^oFPjz<<K{7ADp
z2#bU<gTb|RPF9{a(Hf#X(wV!^X^p=C#2rO&4XNW&EK1-tdD322vGWduds9-Mb)M8h
zmc=5%S$l1ZOKd~E0Z*Qx-Ggml#^n%x!2|~%(!UT`L@bflu(iu-OQ6c*U;%D$Okg+F
z3#SAcENU!Rrci&i;~b+W?V`rtBmpf?U0JUmK9X5=o-(N%@7D)<<ogtfFQqhn9vTW4
zN<9m!vag7rpF0VFmr5~Qr}IL@{!wJ7Ht;a<(D8U;j&8hFUoeVe`4uBKy=iv8CQY!h
zwEkUmKwtxoC*a@>FlhWq*)wg4YjwUlyLY^<mI&k8C$R-mSF%g+Trr15qK=XJ9&dtR
z7N=%Bt@sOP%)bKsJdD>n>BSfX=l)H-UFDu9s7log@QF9QZBo#{ZtYs?OV=0cV?&&d
z`?>e<6iF6}c8iX5umCZmvY2rNKY~c1SnJN6!DzHFf=4Hb=v}nq&$(%s*Dm!ap@D|i
zIowjKt6MNYUb~mMa}n{+r0KfjZ*T0)94clegV4m{0M|nKt}rd6MXk}JO{KOy^EaAX
zS$i2$-IVniCvifNr}Ec0r5k-U1J_@-E51gJ=m6z?dx@n&9dP{B;gE)CC|d>kmn!To
zqp9Cz2XgNgbcRyjBab`KwyHH9@ih>tSSen0P2QF_;W$)A^UR?h%3)LlT)+bZzb@8n
zfEanMjpns)8DQ|!`TQCm+0XAY;kocM8^jdxHBB4K>*M4$K+_qkXc$=Qa={C(<uK*q
zF;U~U5So-`f==Ixux2nBA|!uFO*0}klndhKm&Z{eu#^2bK-pct^GVNJH@TrOkPV1s
z^HX2@%W13OQw932JKiEZeCWvcO1FLETHpyIh`jmScy{pYpQLHUufda0bY%uE%HGu_
zy?w5pPM%K|E&#U!W51wdFL<&u<3NZjwZ6U#JMZe7LcJ^2eqFT&>8krAs9TrDnOa0T
z-k}g)S~euLb#MDFWBfv`(5$C>u%G%)Uf&~g;3VR<u%0!P_Cyb`2fDPiVF{F-&E2mn
zaZ;xa6v!_yGO~Pyv`mRn$lz^qVhD{~ut@IW6;k=QD(ld?<bRUyxrYCpjk0g7>>ZLV
z$;HbZuK4mQaWbE4QhC9d3hZf2)<Mt&{P65`V)O8o7puF&Ywy5nXSf&Jd5ys0M+!(U
zW$!T-q$seVK?U5G(7k?&x^vU12kQ7(ZS^o%_U12#=n*OS1Ai{^bu+iE+UV!1mTeb$
zTRea479q4C6&fXl`b_2mlX+Pcsc|k70$e>f_)5TeD@LLj70rC^cwAMkY|VDPrxi%N
z8LWWkPfM&ostFXyz}Iw^ZLcnfjy>$h8)OTf%}iIQDBt5M7Pl2zD-cwo<VMf$i<_)w
z+L+bU(ZaWIHy)C6SM!?C!fnBwIEmToQqQL+|D9t&H(oAc-k;U@t}~4a-LIcQB)Us`
zV~Uw^W4}|vO>lc0+tV-<(oehfo1My2AYYHb1F_ed^Op(TA^JG^*b3i?{)E-Ozr*Tw
zq~sb&7*Z$}K+RLTxq;+K!kP`2^;~xmg--x0@^^Na3-hwX+J(~PJ?^VN5!JtX0{v{2
zMTO>k6Ftfv|B-jE$P@DZypq59#QOH1_spLik4$ptvApmynEz}spZCW-B>8<R|6$0g
znDJ5m9H@M5%2q(pEwn(WW5DFy$-Nu`t^pfa>v5=@Ahiqv6xKNDbOZi~6Z+lX+*dEL
ziF4a?ZB0VUD)-mnr4Lc8ZF8wTCWkMDa(8P|%GcLhzk*=)At{~+Bu89Tu)U+n<*(V@
z=vBJAw$tRUP}f?2V-jfwzi|zNB;|nr9M>ljXx~5G^RnXp8O}d&ghv8O8lM{DSL(m|
z@=rehd&e>WddNAzl@J&4KgP^E_#2|YqKF~>r=>qTtewz9T%Fv=h`#^P6KE%)a~L!p
zFW)C+zUAe1R8s!MQvwZ##K`GKsB^SS@}?S9y@HL%WR*p+J45J^IqiuxWb5tMt(C9V
z%e3X0!asbqsxtkIk!ABGfB0Hd*)eJD&jvyoFcvf}QhjwtT@1fzxDK8>Tw6^EX1f?(
zYvseB_Muo_7N^uzJXi<sB7IZfG1z=RHRiqVf2(16dF2_XHo5F-cw1##W9IXBcSHPn
zCu2J3f!*R?-$_P3?1x76M(Z~2Y-a!h(EX>XF2WHF{6jEYYf3oAuNs75SAzFBR*JP8
zbh`pb)nZLHLK=$R_+0t@@Wr(4^@TrKJ7A?=NHFPR?XO{;Xr0H46tUxldZ?qM`lST3
zkm(duk5LRkfqp%VPwS7jr#EZIuA6;3-q0K|C-^7;bYDT|b~hz{tNJ|7!J)ewV%x}M
zC{nz6?^M6)Po<WwH7|1>^e|lVfs@{b>RSao)urt6NA=Dy*Szzp+5#aSaBV#i8w=hr
z<Ovr(Vk$-;U3d%Dm7sN{4ps7Q5wE1<BnE@9rS?U3#MxfG#feSZnYG=;n*R~Y_mZq*
z_9U;XU!&lH`(PuqnU1J?*JDb~+u`otk>vEDXToY-N=?Aa1<bn6+Z$qY?xoqApOp}<
zs9udjSlvpaR2AKD*)UN52>p{*jV$I&7KM5z=3IQd7lo2r&}DvunLcjvj%&~^9VSA@
z%~V-qgB5yC=i^qMQC%f%df83hvH;j<_I|`U|1-CF%FS6@cNvfb1^TXP)w>Se3F>V+
zqTeRVCiA&Y5U`4CT;(7&AYj@Ze;eeRo5<nwd_^rzxzK<<a-Re|aV(OYH87o)=fraF
zK>pCOW&K)f0*4m==(Bjg;1~sCSd~+zKGV<{UeIif6g(4-0|%lD7p!p$fR>5~RF!zg
zpuX7u#4)&-?3awHSsiU`eI5Lc6OwqSDqX$!`~e!}M@v*_*52I`-u>c`RaN#{1UyVb
zr*&w~WTb*IVbMXNjvcl+AvkI_x`ktjUJAu=-V)3b$h66*RMg&#)WR!f-dtxN6N<gy
zT<3A(`S%UD_k(x#xz%ojHPKddMM(^gc24wye{@>t(grMENvmJV#4RN+*^5f@IOnrc
zS2!o?8{N>DZ6TMmOo=(&*!1y!?2PX0fjkZ@rS`$a5v%+xV3k_yn6Su$VyUi-qz#6%
z446OgV}uiQe-xOgdtiuD<|Iu%6o}B32o0eFqtvQ61OjLD9?;4ro%{)&9jFoe<1`-~
zbBjYBd&{<X7>j_OkDPQnlbfGE{d~wzFFwt|+|FE5(_MCH;%q5CB)l+LKxub#{pegF
zXDLu9LARU4VL2L#v}T#!3}4=}IXKkm>1x_JLj5BJ$#@AzV4KThqtVvZV&?tvwd0G#
zu{D%byH?dV>6$NDsj}YSYg`dc)H*jzH{BR)!vx(tI(j+5t}HHHuP%iM&{++}>PX)C
zS>4d(m?s8?RzC#A{dm_n?o4g{{#-JFxnTJiP6N`NE#*9SSIXm9*>C4}g3w9E!xbb+
zZ`+-0hoH%0@qzaCl*rd7ed>_rn!9>kW})dusz}@)75(g+XjdJZYbZVD;d~7b(#1u4
zpy6`JKpXD;hV?dTeX!t<C>v6Y{<!xN0Z>RYSo_K21cx$}FXcqRn5yeBEN&^TZXY5(
zX<_dA1^f23f;bNr90H^k8KK^xE#x#ZXcQ9~l`=J5lXq}&qIko~tx~tF#o^fQ`{gwV
zvt0~1F(V3y*(2I>-p3TH$~xUHV?xM-!%wCtdIF`H^u^E4I(ulfx>RSmSdm9VpJ4H=
z2j-igHy#JOX}{W?xLWOpFBKt}rDWNkMZcC!)}+*e9f|MKIl}Jdbq<`?fVFR!c`ryZ
zM|Tq3wVx>%p|{6;es7c7{d)MHJ20Tt-`}vHa5OdLu;5;5F-%Qx8U2yZHq1z)uO-Xs
z@^vWkqFgMV%KXtd`Rie=$tf`BwV^Vn#uO*l6&hrtp{1K-RR%vmLeEm-s!guVTo+rF
zUnqIg6!Ua~gI!m_<!!U3>eg-5ReIOO0NOnn4{G+Bs9HG}LgKZ4dLlUJP~URevNGhP
zJywE>ht-5_@OYN^9g=$$mQd#=mvwg}0op<EQ`?Hr=}nHm2s2d+CSV{yAhGMIs}RJ~
zbq8Jolrt`&19no1^gac4pHv*S1UQUjev3PqT>F(w+m$CCI})Q@>Hd@6;L=II&;DSL
z%`1e6-pLph*t2zTizAfXxkKs$GM9b1G|x1zPS7LOe%*i(NWU-_MSwnz0o%#?^!*$+
zvPGjY;n*VK=e`c$M`GaS_W3S8OGYB*FcPin@RaYRVDREnFvn_@-FSFJtOpD|dail0
z>4z4Br@OEHWf!QS`Ae}&Kb<@;o0+o@nT6}uv12AMIwCpMj%fI5*&^IUuKDL%b8cMj
zk3-6X4cUOtbWjZK=k)Y2FX}U$Eh0#pszEqtpMLGlwG|bC+u{fZa;B=5XO%Cd2;B^2
zYHMVgCRf<mx=SJ~lg%vl#-X_2Puj!5&q<r}NRut5u!6yLFa0gt1KOgP?x}4`-s)-c
ze|#Si0W!I?On_k^XoEtg&~$0$tdPNA@qzTyQ}2CPg+46epqg?%U?PnMLXOh2*gQX_
z^y}qm>3-<<H>V)1;D`qp+=nBKD0Ga>&p^Cg_?TZ1kc8ZrN;A5((=poJ2HD?1#KfDI
z{irG$l>^MC;@?9*Ro3!Q121-EgVkM$C#P?*v5ORy<95tPt1}cGg}&yf0X8tHu_+2b
zGyT?eI{@}>?lRh87E=7KLxSRVEfIb{sw=zpds8|Kf%s`c;vRUivytd1b9%SrO~Wm8
ztXGz62Y5M}omareV68r1+dHTFgb`iSS@d8Rdi(Dm@b883nF1+!PJk$x=fC)9=(O+9
z4~9#<R#f))PQhmlKqNZtsrl<z#~9!QczvmT2gSw&Tz7DMDVVCo^Ze!^qPA_S*ohVR
z$vzb`PjMF$_h&|g4qUPnotWA<>L@>Fp5o|jfwqnj2n~{>Fwu%T*N9|rR0+n6C0Ov}
zvmsy<&a4Vb^<}XRpiXs=K9yJeV51pU`8L>9U}S)2K|UUeq?zB@L|`^YbD5h2Y0??u
zPG!e9URSxInTl4_<(-JM7iy_lJWHzyI5mOnY@d!hDl7#2lwPR~CaNMat@a*+##G)!
zXY2El;<N5k!U;X`elXqEKEw&}5}Y4ieY6_>hC49WN&}*$nfCBegNQD9M#I)bOYo>*
zumh^)gMiLozI^`SoVMv5Q{i&&RL8Mac3)T1*A{)qxT%iN(<ZQc5>cu#Y<U}BLf1Vd
z!CBkMR%dCt0(-pkdHuF%#OttJnX^eut_3A}Sd+65Et?#)3$DaNz_7=Fm?}gx8FN%G
zKbx`XDsA`0Y^c<-^-tR;FFGr@GA94+giIoTID?qGs!ar2rNnOo_9DByGim~tNDDFw
zWApYms76hNIQ*N*@BDX=W$j21QC&4B1Z=bN>cuB!xIR18<#GGRa&h&0`(bw%P)JVf
z=j}-*c;Bt;RgjD{Ii_KTsD(!;G=H_b-Losszc9fII^hL8lz*jH*&6&>UtK(Kxah3`
zYeObjXDxwBI>A~~`Wgwr#GeojLGz1^y4I~RETWz&{!Iu44?_<QEG}lR32WhXj1Cbp
zjqo~<<~~!Tof&L)`y3QI-q*FnQD|JThLNdZDd)nxLQk)QN@`YJHSqEg-6~%HJZBi`
z_<LisvrV67j`I9?rVl1k@b1qVy`wzwEG>Zt*YA;RHBVx_{Y*Z#QCfNNKTUy(?Jh9I
zLIqq)pMAEq#P)HqTNM=K0T#Nc$ni*xZJ+#+f7UC3MI=w};Pzw>sh4&WJ9-+At__1l
z8gw|8KB_+U)n%S;-B}FA0kdY~KZN2D+JwXRW3970*pJOYBxkCe!hn@(!TY~Px7Q1n
zQ@WzpUUH+6nl2i6X-*N$uI*Cyu?Dr;G_*k(m#nAejt|2s&5vB43=R0IC}lItJiu1K
zPctN>@3Mu2L?4bRdV&L46c)$^C&yn&9o50nG7L{u$i9m(eDD(Bd0H*)Eviby%Q4Lu
zukMJm$7rt$A0SW!lqm_4=vjm?u8-1f?+FG!ti0SNx=kd&L$`@KK3KwI?hyQw6YBrw
zSFwx`5^xCh?E{%VX^^pt+J@%^r0mO`KKrBl?~U)1%G%ZC?HkwcypshQ0X|MDgnbk8
z4QK`w2t#7nKw0vQ<!t5OBb^Kzx?k%&<wRMBz!}~G>qciRwK%!5fcUAh)hlC}&jCUU
z9=I+o<!0vDu4>zrAKGR=nmi0#&!4U*=-oIstbQ?GkKUitjRMDpvS_sz!UL!3J&h7_
z8zj6Z?ws?N;V)xi`po1IUOmxRWyd=dEKPHxB%iD$0L&~WUd_^e##ZYfDNOIvW=K6>
z5I!VxvCqbIjn7G<_jI5soh;)ywYBRu-Cp$0)@dCIsKMk;$gvSW6lyBLetSsdC_+V|
z3t;fM>bXTVRj*Ru{giTO#n=V8Nj>C_!JYAIt~h1{<-uNV+8lM97v>Os5phSP><7oM
z<HVL>EFV1;TIXo=?OQ)nN<FlVw2ot-DoT3zXwL`I-eo;5)0_vT-WXCI?K6E?KGgJD
z^u3<P;K<*;=Yc}JA_zkL82HRfqhp@@pALBZ+No&ZyV?4A5`Go3OoHXBA~2(ZMKs|w
zYK`w*hvN*&r!6|S)Cj79yYt-Wg9J_RY>5tp4u&chKZ<#Q6kq2f+yQFGP770>i7kMH
z=VfIt1T4Q-TI^Q~rrH{;h56K^(N)@1W6!GQg%(`-l*_YCYWwG+e<%EQVy#uQIEM3x
za1Z49?2bo@qgNSMRsr<Z$)8Q0UxQrhjc9in1sei|12jqQs&-g1l>7_SEYfz2^+(9F
z`u2($Vz0K4rq?{L5U9N8+52PfixpLYSCQVC?5mrp_By}N``eqb3w}0u_cu-w2Q`s)
z8EM6z&y%+;C;8qv_o&bFQb#HTyDQuAHd<wi4u5G&Jhx95h<s~6U$8zn?OkdsTNp6*
zxfR}-EL4XmEG(JY#<-<Bk4LjZNOp0gLKY|T6tZJ8_`bOwGfLs;*fx3qs1a(Y=;|X~
zGu%Ey_6(#tMzok+{xo``t1)^r>>7=+_zh{xU4JxNV9j+BEu^@d<vxr)sCe2*{?(Yo
z*7;FnV`jK)F>TjnF>PBhyk<34&d^>>OmysWtU{7j#G8UAOFHvnp6YSrijS2;Xo_zO
zurpkRD$Xr42TZwgW!2PY8)H=@`x78Gwsy3|n)}2mit%{QECvUvej>5%eiH6Z9Rt}_
zTeAdO(QLnBfm}Fup1kel%f-Y4v&}>PoPgdzXl~QoI`u5c+h+RwQ>6X-f$Osk+<H}`
zpmmJ<fG*n4Zx||PAK<;6Ww4a)udJCniQJc8-c*jI-mjw?g3t^%3D1uePKt8?7^nCw
z&L@cLy-U}>Vx!+~P%kT{zr8l6>PP+K0EB)!0F*6RW}re}A$`Hyyyx)xyLwHoGINoV
z%aOyLx~=@O26_4R5CFzxZBMG8bATpAfj)CU%Gk$^eME$71xT*Kd;MS8P^jF<udKqw
z6wk2xD*Oq2oD)JWc@ujH3+k*P*I%t8sMbASqK2{y*n?rslX6Lf({*piSgE_rJHMFK
zZw82`co7j;JUPYdE--cEs)Nf~VsY60e>HNimV$~dPf2^mm%X}8ZXgO15(R3jg7+-U
zV)e<PLmE}F_P+*6ld2nIWgwHLg$#oH{*&M3jtiQ0U_|PzPl-;~LSjn|?rDst`WKWC
z<f>!O+vgJ}0}hqGRu4Pame_o?Bg$7pmn;|yR4q%vUM9G}qv$$NiZf+Nt<|ZsaG?2^
z=^jJ<+HNF=UF1r}&Bv@Rk{EygHD|?{T*X{Pm8272%&E)%nruB<xWd-48c6e#ndCf3
zZFVNX(0!uVrFo8ix#);fNYQ<&bX!hV-L-M#;z`YFXPXB^)OxjU8)KnWowHrfZsl)Q
zBWGJ1K&OuYk7)FxZQ?BDIH6jrL>*zy8VND1d8}UnnkDavcO!)|cO5J_a8t3QD)vJW
zk!oEo_9h2YK76r%32xHR;cBOJ)XesF;DW*_?`%OD_?L!kO0kh$x%rn9suwRu9W$JT
z6mkMho_I3$i2KlzsqTvMuzPYF^J*CRx57-`L9ij%IvnLnpNRF3c9yM-yKq!B+@#E9
z5ZY_>B~pjckAXINYToyJ3QZ6!D5FvHe9$e)p+P0A*OsmxS>kh;nur)b=>sscm?cr%
z--VHN7Ztr&ZOm97S|$(VqnV3&zj!86b0n*4|FEKb5Kgary;j`on&PiNF%$3!X`S(-
zL19<1&*NS-mn=i7U9Sg%K|kK2;S}`2j{fx#4h<j0IUSUqr3}Z1|5<I9I_5-K<AZq3
zMf{f~5$DrL@0g`y8=m(8fhWDyS|bc}eFq}*>Q_k$@#KY6`}}hCD-jKWI7|=MOJ4=D
z4;oDnxbL}-%pBg7#LBE==%rJ*BWaB{A35%C?rCckE!3G)Wv$=I-_1I*_9}MoNr8x|
z@%I_{8#|k2)@^*eWakT=5~6*N1PTE#6Z<BqkV2GWl;fHlHH=nDmLJ&ac}iEomyG&v
zOdcKCfn<9_`?5*VAjLB~CVr9IdeGMnDJ6x5w1_Gi3<nk_Y&UO}A7bsnTqNye8{l_i
zPhnw~By#Ve>W#o3HVwTGDl9vk6Mi_py=@Y=J(@M0OsLEU3^#Trrcj(Tv!dEoyP%Y6
zv!Ni6uQ#9v!70T%t=X`DVYM?y!(C;wZZ$eK&IY)ZNp_)fUC%{I#lH_Dttq9iO$7z+
z*Qi^_ug$U72>y5jT^JD_xS|kP`Qe1&nENJS+vc6a41xen@;i?mk__y~g04Ly*+Qe4
zQ_~ES3Dr#-wmL=m;Kc$#fyB!<K9HULj7Y)bkI~a-b|g^d2KLg#$+;F=_k>$T77T^)
z9%`9q&3>L|9W>nSlWZppt;<<kJ#QzPtyb4dLE}i(UE_TBbgwVYwRNqNlWzj?d9>QR
z#_5jw2f@Z*j@NfiE1Ga$6s!8ql6Nt=?3ymi`*btB9=S;mIjw~xW+iHh2NI||A?1|k
z{pFvd`cAh@an|)wsDVcAxYOg9>dyyXyERhHMv9yO>oLuyALe*UMj-6SfD}apBE`^5
zY(Hosa=^z$k=|vpF^G5Z@l1~jUujlxJFs>+h@vc%x**Gj^H51_s{T2A(y@Ul2soGo
zaBiBD_zl#NcU?-kH+yY6Ef718BciPXkhHo6yM5LA(1);)XFH2ni-ars06#BM6~DY4
z!1tYL6`kGMS|)!?UW#FZZ1F#FQmxzB5+p$2S>FWEL(b(CipziNgS2ia=X%^qr>hS_
z3G@QfrFKuJ4p5!s1#Wa5;VcURj)42~^>6IC$#@(-BC5$2^j!y-1g!)*k?8K4tk+FA
zfp}cR+_*L!gxhTF%*}87+!H7L>^KN9SXh_{c-~H-d5*hj>7l!pU*&e0W){Mx_r}Q@
zY_1(h%_BdLH5l~c0zCz2k6tD37tG(~2`+sBP!1jGeE5!`q*R&A05{7}16;lQ(uG`7
zPLp5Qm2Z)V>4nfRQK=B-^3H$FaWLFFUEbDW>BkXX_}L?;7ebTdjj|ig<1nR#j78l!
zJ@g?Yo7~!u;TbwwwC&^hL+y9Jrk4J0w*Z&OLk@#&kuUKyH=N6r<Fi-$&3L-I-?=Fy
zhX*KZ=aFSL>pL0F+V3ErTRr~J;hbQKUShp)cDmKj|FfKPx{p{knxG!<8oZTN;#LA+
zmuji%!x{6Aoe(fxToVrND$`MJuGodG$SuqaP|fiG(=x(ex`qqcBJOTgaz!`RPdy~y
zy6UAAKPD|EOyu`7G#_bpA7~W!TP62YPit|#!4Y0i+t*MH8ZVbI%f;*A`MxNi*z`-m
zf2+`{(7Aaoa7b1`U?O}Jq&4)^SWsQ0ug=NYL1+)8!c!X`knPHc=iDV$sUio~UaRgG
z#^CJzW~0J~`Kkt@2g85!EB@rXNQeI9n&(eUQ8z&tz_s^OoGY`-mCzir+|Eujx(sx9
z3_iCT{Su%H!|pKU7Oj05)l#ONv)(51ttzOAzOZ*Wkm4M?DbHYVsTl&4zX7vN_}&dw
z*XDQda<)!lRj6uS``D3o>!>SE##B9yweizO6f4<_U~QLO7*6wVjzhuJO>xP;6_*}!
zB&}?fRoxLZzdQH8Zb`h*;6a+u%>F;Q4;km*L<PP?wcy_o=6^9GmN0Noe#4`57VY2L
z3*$ZfO{O?g&wl)mL#=S6P!8sGn}O1Qwe&|BUL^wMk0$Z^<+8IW^a-VtTYv4OFh1ZX
z^R5^wD?L0VX%p<gHRHLtf2BKNEaLc09z9`QrJwzlqV5qu$)n3-M#v!t?cv=`+njmi
z;Up?dVd{S1*Kvzt3WxsrqJOxf;zNU3O)jF5)4*P*To=Rwic}Vh$k(DP17xi%k%mWx
z{r{n0^<YeRfiTB-x4!w;-2Qj-wZ=<8#j^4DvuFRImR$+@&6Nc`fgS%yQU8pT)N<=1
zlvCPrt8QfccPffC^S50yc{G=B|J`~?b<o*JsY@-+<TwxO;1$8Li{GM5B(9Y&qxZ~#
zy{T22ZGuK_LNzAMy?-%BoFW!TN_qI9^4&Y&Ij+V1`koN%m-udzOTv;r{lxWQI9&>h
zGjCOgHC+~)R*u#TQss}-&y&UME`h_#i`^=y(PznvEw0`PC)QnE&0Hz5D3+&|zJalZ
zD*T|NI{fA1E?I>1dIB*}kIIv*U02)RGitf^P97wdR7*dRo08KzsOi4pW3!0~xU1U<
z%n-`3BSXkqy%vO^Oj6_xX4KzqGmNrFmyEsyuiqhp_oo-*%7}ZXVzDjPX-z%SxrkR*
zw>&r6-+-;`ana4Cfci<BYB(k<hatV0>}hFvuN}T>SV%E<n-2T8?!afH>zBg^oIX8(
zT~4o?t@SVXgGY@s8+7KMsDo0QtG+c)JzoqBDc6(W0S0Ha`z<~m1XK7{^2hG*R)nt>
z+<f5V)}@MIgqMtmpZ8tc+@G88+wX<ST2D^3{5Qu6bmU89xEd_cGLY<=(Uq!pV3Cwq
zNVjv=ZzofA`Nrm~Nhf+9(vDwYayod5wKTLx{}!ljU|^)lf78^sMsWM8tXUGG$vOJ?
zb|Csf!#8q-n-yEyuI0DxGtRo@Si_s*`4)3l$Z@M)Lxz!a`dpFz2M$RMD}h$WU^fSj
zd)QT#FGF47<M$x+Lq=0-sFW0h#oqP2&;0&X;)PdXWBTm>Oe*C)WZDlEvNJ!Du<Kew
zoF(V~pHoLZA~CMt_mbYb6Pwq+cT1l8zP%1fBV}1;Af+drmvY4_InsZAvvu^fV+anQ
zgkA0|^CII2_vLSCD9slkzHbay&ChJ_uL!)P&1@>ccTeJWOsgY}-ZRg=lD>{5Zro6j
z;MXX0bnR>SXFKJt=b~yOhAVW^#yg^N7EsOqo^5J*hhvY0jnMQ-?a1XrrwZGZ%GLrD
zV|`C>YVjMNNfc&lUKS5#_``00MfNCv9+Z~xJ$T#l{^x2&upr=*tA;#}Z?53J_)?Tl
zBHkzyTQl+9Vfeld+m!!>DURoMi3v&QT=Zf^%>^jnT<y(zLXDD+mMXq*%JOSeh;g6?
z03e{KNuRX2|Dzonyp{|VuTr<7szvsFcsDp+ecW8j*5y`>Qh2Sm);ROMk*)kywP+nw
z{$1%O)un}^RvdeC>fXIp8nio^>M*yr(@RA%r#-$zy}$v=#M~^2V`{|1e^jEzztM^O
zGMy1lU;yF%5sFS8w;wU~?LevuQ;eMNNxr*BHHRDD!j$7(6ShM~sa9&o(!r;Nk-d+S
z?AwUy7wbP+)$@)$sjVF=Y|6ebtI{*Ur)>))Tz8E)N)xgH@D_2Z+wNWQTXQPG9fo+N
zihOxWFNUBZEs<E)cLXfbjf?x5E)_N}g&&?cp;Fir^W%F<i*BO0+ARrsa@P~t7GHY1
zd3Be{O~a1Yf-A{Oy5VwhhGUhok8aXCPF%F0x$!CA$KFhh5&<uZ&$PF@+Ij)$97jc_
zCwGpzmc5}!Y)l`j%q{A!%kX^Hjyf%L6vUHC!(#avh6Z<?gMJw;H>Yr{E66eb59AWS
z2K2nWH5Ho?E*<7NCdPxrnhWoFm;5tXx>3_$pf8UT$2Xmn1C}GjwV$~w<KnCZek#aM
zt%R>EYVnM~(fFIXXBUYtZ?HHp%*LTZ_x|UwT3|wEVJ>WfbS;jr5{Nq1yvY&?QSc)3
z*~1t2-2S+VJeH&W%?*jkg=p5pi7A_|BD^EeILNDpn@`gvBVP_G>d9d0)}0=~1ny8_
zw^k1&Z4!LhS+05oGkk1b*!3r4XN`WB)C$g(+pU(~Q+}WAU?*UX;;ME5JIV)jk#|^H
z<ac3t^`es*38dIxu&Goj#&V_JrL+_jpMsBuKU;Jd`?}8itSv6Dhg*J@PDMGrjc7Wt
zbr6yi&|<_lyu}vgybKwVf$U66^mDdMCCUm|Z3R3=q{5Yvl<7HRGavW!vz%hi_pg+!
z8sV+_0%F<SN+%vtXgp6&loDm-)wjX}KOUQ{^8y-~9Vo2f#lzy&yev9jEz*uHPr399
zH!D+g*}90A$Tlfioy>G#k*NK00+0Rw!!~h|p!|QwHf3TDfhVJaonqw9k#sfJc)+fY
zADB*&1hz-et7)hGthj_5SGdhL+aWkkusZVl5~Wu%DzASQkBPBcA;Ix)6i*Wy@7Zop
zM1>BVpl8zkRxzqgt{0e`FDYU|EcP+80G|LYPcr^lG|KOfx1`dDdhXVBiY>VAsw})+
zjG=kQQ%5WY3lE%~`x*C!*{n8a?WRqC$plGJmvtYt>$5S%+1daV@j@l&5F)OtSRk~-
zjoS=TBi25XZxHVL3gr>Sppr1bCFTW7>cZYXfds4FZ{5jlY9~IHXXm3fRv<T5L3ko{
z0c2$vO@WG9(zeD(O|~#E*RKAOKjc@2RffKF8ZrfPTI8G9097G@Fa<<4#Hwcpfi>-g
zr&0C?J5sM&9VC{<mbd53Su+tNo{b$dR6^hCJ8$e_Aa6I|E7s_&t`Oyhr$)k2rQFwJ
zH8vw$xiqA9%7A6wm`S>@T`Of4)uKJ+$=QxsKYTv%u;#rXmt6LI#1i303*9(GgIGUf
z#9&*LS2g7!TO6>-Z*c`7Tnia!qRa(`<Ln)4(-=URQavhVB<YgWXBamzb?pY5OG}f_
z2i){B#bflWZ-vM@XA-nee^^LK3&NkKsS0fu`uL|;DKsW1%D%EOI_Hk78(-5_ce8|l
zJLo!q89c?{hQHMFrQBtSt5Z&ee!!7$44df+fsgUoQck~X$h&rTSWM?e@hT#p-sxA8
zm2#}=*5WjDT?5hJRz@2E0>ll}^vDB3*uAWt9KCHp9d1S-(u{HMgWgD9Qr<-0+rz_*
z?%5i7j#L7E)?w9V4`y!$nATnfY|tn>TB_AuMA{Rjbdx`0`44QG(sUVldw_xdysPQ9
zU|^t^VeLbu_-z3xzzxutfjwW|MBE;XQn=cNCbT`njO#*==*Cf_6H})sZ80`sLEV-9
zI8W0s1{G_1DCy~vHnR-UPFnAgrEIPxsY5k_KOJnN#@U*T&ttz|Ge&m)OjOCIk+O_I
zf*Ulq9u8)`Sa;btqP%z5D5WQl_*X#NDn?8QiTgqk_apHl*Id-~W0TB<{b-_~f{UO#
z#-sR!>gd-Dh`$>#yP*7{U%Ox`h623tfAUPP9A(DbKGGalTgsd$M~;=lr~eOxljUjN
z)Bj93@gV&c<}10)dAt(Y>QOaSC$!NAm@OkCkiEW(KdwpCbLB=nyKIevPb8sgc}K6^
zd?T_~c$!3V?x<kW*Yl|)*FB@ts4++1{laMGZ79ui>Z!G=?2NM^5Ty8^QweV#F#ML2
zT-eOhYp^<oEp3^Yw&K9l%?%8@ez}+^cg4^-mx2%M02@gXG?LkfC=fqluN}XNLAH)0
zbpLk75KM%(iD9y}q8dC&m&$7zZW78;jqleIF~!F8qp%=PbE%7)h=Bm?{pBOkTRX@}
zNZi_G<Y}JjzCvdqxmgiNIka6Dwnu%V&AO|0C^@<EiVD@f7+(+J5V|74^5V93uO`<@
z-xgqc8R_%^-!kP?vi=U7Ip_>Xqa#D>T^`_@Vh^02;3|EIEs9y}bL`|tQB#@s&j0!G
zb)D5|bXPI{61lE<7+d9FmU`Lg5L7v;l5L%U%w7t3kBHAdH8+TO4lJwZyrlF+YJqvf
zurw39AryEK`!sDG=s^v5VA2vuNL;W%Ro8oh7-k}q4&R7<7~;XbGHfD9hCL8X+Vs`B
z2b`*JuZ~E&i!KB2-8lAqjsGA4vA@$tGL3E!QrN~~27-{1KjKFXD08`eH*w;gad)Sc
zs!8EL^ig4Y+;$Wt!(>F0v|f`*!8UK_KEvd~Bz|v$yZ3;?Nz=~ztGs4DXI8#P?$YcL
zy+>Tj7onkkCx;j<#qvGI%P4NHtSXh#FPDDd#Wou2h#M>S(fT%2hia_3`p82uGP*z+
zclF2*?iMVZWS~akK+fQg$K_2%pRJwDiRJ`vGF_v1CY-5ieg<l)8c?BsCK{77=YX%d
zw(w!m*ZwR%_-^N#zbQNWqfwf6YYSNk)A`rQ`#FjXR894B@8a~Bm*GYjpMq)D!Y5@m
z>EeeS%`>k)ayNVn;turf!~E5NKC?HI3a+nTuR1?!`$qrWuoFP9^8b#wx=gip%nwkK
znlX}q^gk9tm#hiAlGc(`rhkL&0|M0VL|(bq3^2Mu7nIdP#_3#YoKfrvmMpXmHj?}2
zvxKJ~;N$zJFr|&kF!hKqF8Rk{&AUUoH5rx^fZr5?MQ^)EAWY{vHOtqM*;c0S%6u@k
zk0peS%P=bkGgg-C@XqYXqNw)w*1bpi0j*08xlm#}0bZjvf5#I2D&SX9iuQ3_j-T5O
z;tr?EWrP7_Ibg7c+st&V#+5pO#|gul`Iz@;XznuY&cQgeRg<e!xA6%>@DC^<7yDyB
zpQDn|UazG$d3U2uO=v!FuKP<@k6lLIBkF~9ouV-<yMVmX*2pbn+Ru&VE_g>F1G}45
zrjq^Vm0K?XRe^AouEbY{uUVu$#A59KVy|FMt*`vPwB!X<xESh!CTFD4L-xF4-RJz@
zs7+k?(#yj-z_X-`m&JDQRzU`C6@JPO137CGi_{w>6@gFMlP^!Z{C_3L>3tYQl){ed
z%saTBjE>iOvZ5e=tQA1_Hu*v|JGZSm#7HXWbaeM~XJOzYv1;9GIb0Ocp06==mCL#l
z1c0@SJoBB|yx=caP(mxc5M^ISMzOACT!CTQmMcZ&5{rA08%a0zVY;Z1qfy^oPOv@H
zPH3IyI@+BmJ$|X2b>e@=X-y!j1LB7pq|ZWxg{$f@r)%op)zE@x4dU1fmsJ%t-Lw>~
z+{DoH?%IFVQcXbuVi}pI><zk~gQ2S&*G4x9*2Yrt>1~XQ9k05T^uM0yOAY%p6ONP~
zII!$;)W``;?C))@tfG?s`CkCXfvyCuy686iVsPKKAH6+a!LGV1HuU|BiY3o)q46<P
z{!Hn_u=wzZY;`^<8YL9R6r_w{JjZnkV#2mIuA|cfF<q3uG{{~p0&a5MMROFfcS3cA
zxDReA^afQ?*n{~kwEVoG6}XChv!5t@uArn>fK{CPfWOX$NO1E~(Z+&I(8bCATatt2
z*tiihu4#qLFMrU5g5PUZ&$#{b$EvV1O@7=FVEHubC`lL1hnsTQa#)u*njFxgO_r|r
z3$>s=idE+B*k!01Wag1L3Vu9BIRsz>oW@9drtz?k3N@t*KSAxpn2U_9b3nbU+C3|~
zUDE6zCxS};5QrJ6IGYgVz3ny=GFZ-UrDIYu`e9d%s}s(;IFJ)BhECgAx@>BaMoIpR
z^n;2(T(|lsP)V^nBL0v_X{uo~6}d0)q~KGDS#iHEmIrkGXQTCz5$cnvlmZ~1R{7JR
zMSOqW{O!(yg0nHkf+YWserLE#)uw7n0n#XC1m??<xj<UWEGi6wkANdEoN4uPghA7Q
z>m;4NGaFB1lS=W7+os;Vi^w+nwme2ez5Xv!(oJQ}B34Sn)SRwsSl4cU`g^pkxj~9*
zK;|}Zbu%cqYVh`6F%jbp&ba-&B6BsAv$Ug1h1d_U6mm$1Po_fIGdZDv3LAa#q6f7}
zIyeri@Y}TPH~IkHgYtoZb}|ME#iq=wH5wZ;1=~RU!cPQ4%s%A~lBlL1ZObTtMv;Ya
z9Hvw|8-m;2iXX8Y#N6Dn^&$zcckxGFc;|06?S#M+2o;n{+=0m%Vv3?LUVagVm!o#w
zv4f(h=Z*xO8fBmVLb_V9#9J>vG(pBUtGjx5FL&>nNM1dugxqypmgcES7Yh*}CsWib
z$?MnWZ9S6|&=R}CjG>99peekX4d*2C-;1Aof)Tp3Ybglbfy2187k=%Sh^z_J<XY6A
z`-=xd$!S?13u39Pe@OUH8wvil97E&Et=2d_5Dr2+f>pR|E~vSfcfZojNiLCMrKY0Y
zPOt@<Jule4tLG^?A}bv3QkBN=yA$BgK<ek0_Fq$(D)uSao5egT4>cF)C2&xYs2~el
zN_FzF*k7KQu)&Be9;#@{FeMpKPLYpXo!)2IeWT(RC6tyDPNxQ(VmeT&IrNXXEs{Vq
zqf<ND;jWnKOe-pl5zRaCn}G+xlXZ%}JL8=dH!Sn&-qVq(&mNXM<(>;nrLMeiOTCws
zQTLLre)Np-kTu|$+bZ8K>SI#TP{By37u#y8Z>Z&KcRxO4lJ5(^7#!j-TQ=xC=a|><
z*e9E~d84w^yWBoq32|O>s@r1%cW2DM->v12cC+u(tW(ugsKNba{0~EB*(K4h)Gd4-
zDl|cL-*p#X4pa5SnautzDj_7Pm}BsYot9yFqH|CR-K1U={o9$yRv!Yx^#nq&UP^(_
z`5uqCw91B9&6?P~K-+*mneL*ZNKO5yV|6`CF?Nr(0O^g8`#V8?+9L!6z|icsYI(k$
z7Fs&|m7>Y$Wr{s~2oz<SdMDErp&%2QXIMKV8s3SnTEp8z<{H-gJ-!Zc&z%-#=+NNk
z*W@%T3JW6RYR|05)?3p!%XX^HNLBcVI@@R!CUK0AZ&F%~ZH=TO=MP->Ch4%ue18Mk
zRRXPNJhq3Gor+1QBfSd0GGgb^4nJ?73sLyFeJ=3Xg3Y*lz47HYQ8ub4i)IIZfYPLS
zAt4)0B?#P-o-eo&v<u0iL9-)$4i_w>Km6Q6Zk(+_0I(ocrlLuLhG@SjvxHKOWK(u1
za_Od?81zmhZMQeQco4c~TeGHmG;DE`V_eG4@K$xlI3E&7@94+3ODU||4E`E>DB4~N
zU|G+F9cG^2i|aagGP=*qEvb<xOVV_A`^4V!FnwB17p*W9+hj^mmqCW@Z>X>>^+Z|7
z&_W%^zsp@mSq~f$h^-L>|JVp*G#agzS8;S~P?@2uPY+d*6yIJe*lC>WWpJ=>#_YTL
zhDUO5ZMb*)*P<WW;AQ|cnh3|PtS2ZCZv?AqMU~EH_a+Xyd`s@mpWmMeq}|C9UJ{b3
zCy}mRvx)}UHT&Ho9Q&<Kil8fe)4k>iTq=cl!s<)U;OI4AiHsReDX*iM<4k-SzN{7c
zFfSTEoahT!-&#ME;+ZM<;CG;-h^v@6+pX~duY%wD52fbf=J7OMLOm>QRB6|r9#D7w
zR*He@i;|=k`L#QAWwcegmsSg<rr9EggPc?2fsGmv0Lo-%PFU8ehD%u3$4`1()KI|Q
zgKFTrk;+dL6?CWUkM>zVpp+|5k@R0s6dmb5PRf;PcQi$_kG%;ZwrD_Pe^wxKAJ#mG
zWUIh#D?Hb>3jvj4`j#w}*E?yx9Q)0!DUAumJ+N2P-*+6j9nn-Yg-DxKVumR7_O-8`
z9XEb5p`&oUN(QOnFEUn7QlSnkl}1<Zp^(%LHVl)jA4mTNIT>R_XCo+94rseRD?qHo
z_@KC^nNZulT+rykP66M$@XqWBWT;#O4RQ<+y$6No<5-&iU+leiP*mHt|GPy6Q8FT_
z$pT6gk({%Vm7FEVCg&!jG)Pi`CTEeHp-J5!IX56VHIh?9lXLjB`|N%1x#!&X>b+mR
zSM~ejR?S+~T@*d%8gtDs=4X7z(s|YB`<Vlq8N2+$XA7qpmW(mVuF#_Uj25Xm7#df3
z%w-fNCGI5@+qjLrT<MuWmgK=zc2Qb4e_ZeF|0r!ORX*nG?f23JzlW*vn*#~FGQWi~
zP9KJ(th>5<n=lza3cBx|$pmZ#oNbA5KCaDF1F^+G6+@vcp<3per<sF@I*JIgZNdI>
z-Da+`YuFrSkt4^zxyF_idx|PVZRs;(SEsGk(ybddNV(Ib7G;e#LF3+6Uz<{cxq=Zv
zOaM90yTBKqM*aS2x?1JC`-WCetY*KKJ0KsWj-F^9Y|T6z`_-YaHa#;mT`H9z%(-OG
zo6+A`*i+2W?Yb7O()luf)I$rq^q6*xy4YYS{?mx=n~te5^~I%<Ds)6`ROc`!-P-mN
z1Jc;X<$Kt#k#OIL!6y=A(pl-X*|bw63Jg8(-gX0{H1xg|rS*svw(jl)^pmZ8#4wkB
z^qt>pO&aCBxuK|8K_+xtt3?qnTHIsT!L$?nvS}6*%%M|*P)(=I)zhL_7S6)t?HCcJ
zo}pZ*K&022Sn7(cIyqE$Hx@PeaGE;}*fo4Y&J}HGCkC)lRtu|c8#kPom-Y0Es}0dq
z7V54kH)_WgcMyv-GxNNl!4Ny$=*~~?^mT1usbr(+Ms&q4dI>zm*P8)Q0=eBy-AlJA
zTGGEFKz_ZV$MCxRR&WQyU*7pYK_^WNSAxDI{r)c;@ArLS|0YD(bR^)vL&E+Ov#BRz
z7*?NhlJvi$JgVP&Dk{I%0LFhsPM;-zhZeH)n*N8v_CMP3!xp3W`d!)2w*>odp5KnW
zoe;`cYCLCaQRtSb2$)<w%?}8?=?0x%-2k{-FN{oDB>XB(T3YVHKqU0Ryg`59f8qTl
zBidW1K6AYPk=5q`>c*Y9;-aB><JI-VsK*)L^bJeJ&DQg&7fZ-E(TARajU>M%e>B3G
zp{kRFpO;xMN~>TdjN0p~$v@<Oe=Dzk=d%eO{ffEU(F614tiLS!kI0%ouQtO6BL9Kn
z{^NtQ{%<CyAK(CZ^>0@A17AM?=f6pLqx74c_Qytt{%*{FEf@mY(tP*A2b{KH;F+L8
zY*;i!AvJo}Z>|{d^}kTM|9ASXIaFL<x_vWmVOjIqdDEq!ea^e)>dU(FEPw9?-ziot
z;YprnI(@`g7fc%Cgqm`%$KyR*+(IK2j#2FVIFm)U^Gz3mx6N91<&7Wg0-hTAuYxwC
zr1*}Dw;?`%JMEMAil9%o8rLtTZ(cR*Y{$;|mI0RKB*^{W*1WDMlg<2TLJF4|({L}}
zpct9X@nFN`^VsV;rrDb)Z{jhR`&!D)=lVPAiwsEihYVih|2DmAn{6i>mG|m|Uh7dv
z_}l0CsniF)$CFQ^`<S7|Rkdm|CH|P(WD@CF${M%6)7BlxycDctj@jzPx%x~*8r;@B
zDPHDX+};dhEpzx70F)W5UXf^exz{mJ`g)M7=1FOtf(}nT?Uo;BncWB>ZJFb1gi7~e
zu|zi}HOp#J!IyKX1)LY&yZ7P(yuj={u+{rL++31{-(Zcq)K_U0Fhd%+cg`h#68mNo
z0c$oUc}&Ju0F(ZGeev>z)3Ua>TfZn&j;KcY{|?Kez!%|-+gs_2trYi2ERD(~g~#&H
zVCM!?EC@rq1Q2aC^$P&9k0<-?r>VmdRgpYeI)J%Pxa{K6b^+f@{cgh%-t~vhA1<+T
zE51>fpwI3)m1RbdG!PD^VwOh@=(n1VJ!7|L1aSg>;0{zMi~mf}{+_#A_ufJVpN9iJ
zozMu+0M(f&CM{7rQ@?mYl7Cy7zgcQ-=jf>j3jG(eGstjCePYAE>j^mXCBQ6^4oY6`
z@3i!EoVaC?0W?(Z+cr^cX@j$0&UV+IAw*FHlyPfY>!BMsXjxlRV$YVn1=m$+>=U`E
z&%w@zhjNRpnyAzRzA6Dnb{%={t0B8c7IP=5wqiO=6wZm_`3bIRJC{sucA{B-dTh2J
zdpBftvXGO}shNt)IK%n)NVe;1;OC>lz2{t&2_tbuTZ!x)kNz5qf^UA$hTeNEiiz!V
z)DgzdE*c902-G4b5=2?>P1t-P1>$lN^cM*>@ivV%>91R$4Bop%`~+~R|DT9oy;CZY
z;f$U{*}>0$B6?AO5j`oos|Fn2+i8<B>DaYk3V+jA{5yj$H;3@-qu)yq;`w?AJ?x|p
zw6n2?J?}a%IGu<u)}iBe{6CES9eb+&f5iSIxUsz|bnfECi~o)UbQnq<^fwZ?71DUc
zPfx;~DMA&?bxUMdA#iN})M2;#CQZ|?!MwS&A<|O$BeUEF@kG~)Tgfy0gduvhv!WWZ
z<$%}9P6KntiKQ{eD}TYl<AZNBe<8ob?FCF+{vhjIG$xzU%+K*Zq8shw-G;u66_h!Q
zZEP<-`4tN^&wPn(H<r#sbUjQnbsktgb5SMX@=suJyQ%Xfu4p+mRm6V7dhIQP^(@!D
zQwj1B!dUsacJ8cSlAlVmPRRLupS}w?tMxV4HqMfrb0@2r|FpD*j{JT)AL36J5HXNf
z9DQg0Dgrd(Jp9MEft&evu#^k3*=4@JR-`lzEUj*l^b+Fq^Xn&loJG{GksGr)65#|S
zrAiIbRDsf(Cd=dzMZ~514nVnW*^6-EvnwrNbhlezSZ7Z)x}e;fdEr`&78iUF#rx}!
z8MOG^w2lX#Gdl-yz!&AcMU!61CVtn1oAdBW|5DxD_b+J3``<%DNMJ<`#1^pevHN!Z
zA!#Sb@U1@oL8mYe1QQV$y#<d4xXcv3byTUNE#G(&qxOWE(IjfJ;JwokIX1@O3x2iV
zW3@`Y{87)vW^{^fYrG8oe>Oo(F!`#xef)2Nwc4oTsg806>~QOo`}^<n*XZQ6-*ROb
z@!<nQQc_%O8zNJWh}ykVF&aBz29arR5Vzw`BW_?$h9&S{f#VgQnPDt3cYUnEx${&Q
zGMHlgf}}*Ord|8wTY8VgS;)5)*R9EBvIAyL3^+u6@~Zx4{aTG3Sl>u5RzBNi7*6_C
zJcEIDnj?N?&2&LwbdiSU{_%$P<KxCIQ6D;2C@RP|l3j7b^7_lgHe+1uSCVgoCLyR&
z5u%qBCJi}uuZkBlUMelbYIA<6BLq9mLS83{*1lsX|NezVC0Z|G2?eky(|wewZFy^4
zRP4mIPx$wviG_*!v&jD;xwTJp(gQb`qkHn;;_)|m$10ngjiwyC#Sam?Pih6EO}=bz
z=4j(WNNZs=_#hz#u4brGIK(abd1ReZ@%Siy1lZ+LVS8`#3(?Z2=!Lx-s)1ScHJ9)Q
z2PEbdkKb%QS)SAjuyXvu^9L&eX}YE5U^2RJW~tL)VWOr(>g%B5z<|A}i_08<O64S}
zGPF~H(s=YG`9@0^r9Cqq?^_J5Tx{+cV3)pp9r)n_G_@rpyg4pRq0$kaUO8vx=Ved(
zzOAGVJp?72R9}w%sTP|QhhDA1-+G#E__xrl>Awiw7;)3*G4%2Cf6~WK|4-;+pFrZj
z(?_fSppTH#EBo(Mx#R<sV2?r(6}{`+onl>?ltDR60wUw<8Q;smna>qwP6jmTw<7zf
zSt3SbJ-%kXr=V1dS?v+^WqgvKbG!owZrP^&6QC+cFR6^U=9wn5zDZ{Q0Xh~0S$gk?
z?P%*}+h9lK2xs(pq)l4_pfV`?^`?H<c_u8ByQ|e{ea2VE-1*eaiNW3@xqAYlxA?8G
zD!`26NuDKrpxU%1Ha;I>z7oeyR_8GvRowRgCbrDd<DJPb(!#2ksGlV{q<Y!RVgmKI
zQe6Z6o=*Q?WYZK8XqD<bEsq@R{atI8eu{qSp;F63(yza^771e2ubtW%XB>_Fru*Sr
zTL#;Ef4eWNsX>>$w31?|1(jXB`Oo;6upPM35zAXLz2v)8+s6qTwK?gsCZyN0?%@UE
zws`+n;<xmI*-eR@YuR%0T3sF2ce6Rh3R>1bImU2wGHHT$Nn%8#*2=$IB$R6yHTxRb
zv)QSKMb;fBdVK1A>E!vd7HH{+=0XW4u`~1~7&KnyP*_B+yU#4aZJwGAp5$!j-#hnx
zKk#v|vq}F$(hHLffG%H*n?<vuT!T_yp=p{<0h&0dhu#U$Q$e9}VtJ~R;_LMM3D$_&
zE922B3Ex)^q1}+%A#b5l2Nb+Ig`YiuXXd^@r}bT0QYL<>HY<aEz_1f>qMnsQ8XdCL
z+y8Ve9<E#o%~!zZp`{wdaq69WoA7!+;O4&<$YHiKr5?xL=G(z%?6&DLcc+@ut{Z%P
zs4qYC`28Yd5t7E+GG3(N>D*@JmB*(kdv@ujLCn0Yqv!fQ6xU7M0MC$b$pCW_IcOH|
zSjl_BK0!w-BDuy$4&m+cXx{l&Ym!`He@t(0Q%5x!nmg33)gl(no(s})So>xI?n@gB
z2?hVoOa$cEQqd}n(A1g8HGS}qP%ESMt@ks0TGMQT)H*t_{~a}3CAsYxB~v?L(b$?*
zT9OZYi#l2K%w(0^y^%>dX5VgW=gdm%pDEt&jjo1OyAu`xjxk~Q(H&mjllRsg_9;N0
z>KdN1l7j<oMkf?Kj)5P&|MqcNfBU?ow|DS9b9qH>#jUZu#TPLClRs)+1?FHxwrb^Z
zvepqr|EY^J;x1CFl95{=*UHuD*JQR++0;m*9WS`nuBkd^j$l<(TxC$kmpn#z#^QEd
zsXSi=kRuN4U#cE|!!XGIUL^<b{DXEZk2tKQl<uwNh;{dG%Gm!Po@nEFUHZq_%UcQ1
z`W<7reb0RXG7@QY_9>T&x-2n~-2F$-_$7%@7ZKOxIgBSEaPkf$NJ>slO7qiWDaoQE
zBty-;gY@c#R0qdhcmwVm`d**Rbqi~2D{Je3rs*a$0_6Y%QIX2{lp0ij(k^u<@qfj<
ze0~S4!+NO#YXee{Q{y}7tr7k5Y0c==OshggT-lDhufsJ?nlsel31Xt#aQAoaKT{Hl
zmj!JBMTV4UH(G-{p|BxUwM;;-OUP~N=jLbuB5F#YntS66OMCFO-8r2q$?}<M;pEhI
zpXo5q>;6J~o2hn}8l0%WXDAiwghJC(8vlrSo}Jnw&#TISGLQ2M-~1@!J6k>XRIfTN
zMg=W=#un&i9w>)(_NGEY3BjbdPFw7%jhE|hYBIH}H7(g`sAD7Y<f*}?py)Vpm)prD
zvE`;)D)Y%Yj2zm8zZbG&xIXA9`$0&<xzl(acpQmdQSxUTZck46dVcjQX2a$sC+pgp
zn%L*_uc&ys!S$>p?>nR-d8j$dViWJ#%{QES<&eYX-B$Re!(rXBTb0PNg*I+!s{Tvg
zq|@?Qo5IUegF>t55{r2%C<wU(NXQ#tq{sSQoCR^;tLK#j;a+{X-d}CVR(G=zS<x4}
zr!9pP`BvE6(Wo1MRSPD`KFlJ_j?Hpo@Z3KBAT3k_Q_8L$aC|J>YLwk>$IcT$vXBQ~
z8E+{C9V+UD%~~a97_Cxkv44!n)t@#uPn`J>xOtiCQDTBXbX=C8^k_U4#T3%c3f~;x
zV>QHW2P>Qa&Di<Tm61FeJcoE<mHbu(e5`&Bp(QdQb><I_PTlzRw`Wk^LoZHNj6X(X
zAyaBFk|RiA+JneA{d0qam&^`)KP_zUot3R0y0ugUw}?83+0|4NXr&0^!nXn2Fv7mx
zEW@Vk@doF)wJA6+5?c2py4o~QbY0Mi{ME*m#p0eh!sco_MQ@j7v;S)nU`t53v4#>}
zp0+quJN_n>rhCe0{i}?#(l37`-|PMed<XZcW2dq9Fja4kd7eSX%dRA!bIXCHs|PFv
z7tE6I2LCMKml>q5T%<WC#SJftk1uWrM9yaoc=l#cr5~5maI5Fsxmt{G^8F7MWUUVh
zG`eqwd=e*8cf;tqMOyR6w`}|*f0#~lBbIsVn1<LgCI-I>WO;dNuNj}fik@3DtUXn`
zd192!Vf73#ecozpkrL>WCdx)BR`semv1vcjWwSlx^rr5o&pLEqL_ug%=)655XXzIa
zci_hTZc5|RlXd87{u_v3y@~3lx5mUocYYT^)V-Bq+Gd#-*iGMg6>(m*>O_<2R<ZK$
zxcKQ?h^SGe&-W66X--4L!ClfcS&tYT(YNm?Kp_o;<>XMpZJ3N59$9@tWGp|ysminD
zk%5)Fg-~;^Dwl8<&&@^cBs8@7RefGGRZ~YmQ1#|aNNp49hMRoaZ-5_W*H5c3Tj_N2
zxV+<H_Znws#3^y=G|=g_n%BaP<;|c}*gG9AqqYfjk8?$Ri-x{-*JbTSdU|8mJiIzB
z)GNwfE4=>|OfXwEzP3njX02o{`MHye!E^G|_qi{axWTuLUB>TJbv0SDQ%F09&iCfE
z%K6w$sAYn9Pf1+U;m{g!3PojbG`!e&`JO^rgEucy1b(+YMW|$=R>=D_p*<e%X*j-y
zaOgxP$H6y)+x*t!N#Ecn?-i8dzYrB1=67#4*>ojMn<hkohAum9eDoAlUv&0n8=4dH
zv%*}#GEMu2$hiwq_6Yu__9D3L^5lR!?c;&@Mh<@~bEih?wR}Hu-IvJZd7oESfornw
z6jEHUScGe+MbU^g51R7;pu@Og7v|?8qu6cty~#aKb1WS-b^Al#fE*SY@gYHKo8Hlx
z>n`9M+8>{PafCTd)6R7(`68m=O^AVBfa6lNzx{-PgV}gmpxPv-q|)DgjrY5+u{Sb|
zj6+|b%S;YB{lUy)IR*@#Im3%EVk(@{FdKK(<Qwu1g$r2wIc+0hPAJL=m9t25y-kGn
zR+1KwI?kfY!m0x{X_N>f8Q?hWL7Onw@SPw89|tiy=BB~B*fNn;_2VXmc&yq5Ec(>s
zJjNlnC=cLrz;g0EAWUs*uUfe5CXBR!M64<*-&_f2N@dxVBh3J@*!F_e)ocizXJjuF
zCncA=4HapA1CpO*+l&*hj6sdSt3bdp(}bPiI8^P&XXZcKIFqCrvN3r(gsZ*MriVAl
zJX<=q2Tju2W>3&nvdUN<++ou{8&)gy0-#`Y=z)=XfGc<**@HFIb^0neyN#Srv^V)#
zgv*ttikhF|x))yTs)hR1x1szE|0Dh9$~7HBlY1HhPFz>5RPm*Yez8!lW+5dIPBpt*
zPGPGznopRqPw?urQQQ9uIou)xY3h`Gv&f#3ZhFl#mQ+o|+R-7hdYD?r%y>@a_uiaZ
zkNYHttFz8EIdh2>Y~yb1<m5)|8@WtF&{CRi?o~&<Qj?BAVUjN`87&^`v0&!72_A$<
z+m#=owU78?^Kz8NS<KGt2EGJIsTq3Ld5S-bZ$GZ?2F0+N@0imMIg!V~bHidjxnUs?
zTR3+oOrP$^UT@_YoZ@a->_fxD%&8^ltP>D|y-QTf2Y3p#hKVtd;qP)64qdaiNu~)e
zKBTC=;%s*DFLv(3g#x8#wh3{8DF%7fYPi-qQ2UdXn8SgWHh_%5Dxte>cCKp;cx`Oz
z$+D97ewSY_h_gnYvI}DQeJK_l>#y#IJctE{$&^b2L{1)dh-B%8BeoKTvBR}V_3s>m
zkzgBZNx)%u)fLEwIAM1<qlautdP8`FwG5Dx2Zi%i$Nho`xOlS)dJt>BWFd*$_@@8R
z*Yx~6)oBr+-F;7m2{VjUvE`QYfm7_l9-I8uVomN=R1rIBZ7(p#pAY6P<)*}y>t9*+
zYBz0OpYxxr#dtf_e$arN685*a4gj3uNp}Mc@$KI-wVn^CLu6&0-xWAHZQ0PsE4zN(
z*8Eg$pz8F#`z(ykEoqO}+spF`R%04EFh-63Wjqa!IR-iUJ^@2E_bcDV$jp+Vr+zM8
z*iZ8}U5<V<Rr%hz6B#lHYl*DANXkX?QnQNX$n>MLe?oHek4^EK(qX2}hvLr{mRHt2
zU|qCE!aibq&#4f<s3V59V0CA#pgS>>xJjB3h+VC?i~~nAf#uW+@?|&PVAO~5Fd=xG
z%KmxkWh^aLH$O4@aJTl9JhpCq-9Nb*S0cHc66}Mwycb|M<Iq+(v9q08Vmd7gsuQ`b
zt)B6xpxOy=?UC@9+l&?FeSX=kCUwc3dRCwQVR14xmppX=kzNOE;H~kMy_7<%gLGqU
zlg>5~w&m7aLV3J5R&}$e{b@Q&%n;V`W7$YX+*;E}v<MgE-l*1zpp=whWF?4_?PJcN
z%B>`h?LZB0-2TASQKy<~H)3wf<md%k;48cOQoobIBei1-MDQb2ajc+P`dRdhCaekJ
zr6Y9Lr4O2pkbRSOKN&&8sf*l+yK%*UTG)}_`k|N~LGr7>%F-^p4#YQ&`Wrr<{X4Ta
z>+S}pYx$~r5oar^$uU3<EOe)Qg_lQQ74voV4{dRwnK${N`sb*OCt8j1JKfHkwYd@l
zXA>NK#jd-XXHjb8k>Q`(tV2$+a4v9WKV`SoY&n<o*ErceDqBlsPFlfRq|z~QAfDnP
zseX27WN^~W1-no^+S>K%EA)GU2vpuwC1_gXaJG8CHZZv8fVN=k2-I&r#j`KJ%#!xQ
z)AIN|$<p63>WQ5@3|i)l%Z?W<VH6X!Q=o-Vj%DLp)49tFNs(BS5?D;<;mmxeUslZ8
z9kG$AxBzRApX@CKkdjL8Ygc>gy#P<-{fJ1MlGzSe3Pa3PK>QcK1#J3_jt{OIy(E;-
zsAdv+E_Q)xOb6afkm>n)jPiaU4sQoRkzStdQ%_B*on2XP)%GjMlM#`(vfAhRU)puO
zUw}{4i#u@RHm+8Cem(vw#`@tv{~pbng<+SN3<$p2+7@&7VLoq?TvxYHHHo!i_f2|4
z(XXfL9gIrZC1N(*&He^y$^<|c;TgN)2jP-$S%{3{sdaj0pZu~c#7F?p!>-<mo;{a%
zdVi$XTeRqj!Kp10cLza&ZI|x|sMa9DPtDP_953z&TN$rM@#0AwtLdxh{NnuCe7)si
zl|0lUATR~ZUW$Ec4LLsYvZ7qaW185MeGIk`^0aCYI61(cvbQ5hSE_q(k`?wcz>GDr
z)fO>$b9Pgv{O1%TdG8^6a8H}Q%ecW`-~XDMFw;|(H|b{?16XW}KKGxS-hXU;|Ga*L
z1OaBGK^vMbe=Uyx*bZT?&ybi^@(Fsmd?GUFLHIux$!unNFm#w!Sj+HVtNJ8xN2sRr
z@~rsyul@CZRHbf%yCG8Z@nraZ>DhyCbEL2M(4=R157QNVhHC5M3l8(^0h?F0`H0M`
zMSVlX)$?4O%g$As;&jaK$YI*}g%M=)sKm4oh)&aeFESeQ=}B<%saK^o9U-TF^gH{9
zQ9sfGa(6N#%h)TfZ58AqCHqe^Oheu2G;0Wo^_j33)$wpQdM$WFH|(nZ>&*l`czhqr
z)h4p-ttRz<TI?UwdKLu<#%_3ojVT=eYB#xHjNNjaACF`Io87{G+pRscjOE|#_Ld%F
zw`TykTB(1t8^dqA?PZa>$1&}{{SZ!1C;#3rxqc%{tB-4b{smh{GN8!M>k<VSagcrR
zD1nxm#*{fnM)QSiL#NntER(Tkq#A5b!+i6Kt$Q_CSj1QOmDfFuBV)NtM~L4d3zSX2
zTU%#Ct@Wkd4L#+aUB71e@#x*>NIhB2c)5;_4<-JZg~RKXkZ7hydyT|Pj8w$b=9(`e
z!y};KB$cgS$^L$5&K2qFS-+GwaY8?~ghEbFYXb-&{n=jG;Oxf*^d-EW2kh?RN0jm9
z6Hrx3P*LC`&`G!DNpDT<=4DQx_o(f0DSBj_lZCB~A2sJh7zm$Y;hI`n>E)W|EO76&
z0a?9&G@I!<Tt>m$9qkEOr_-1(Lm77`vD)6=0r_0Hg3v>J$p_A>^<k?!Bac$1@zID{
z+gsq!*8t1Y8Bd=&;fbAA3E*W$3HZ6L@3AX0s;rtyuc^^8osb5h+dRK}?f0d0zL~pU
z+;MyUM+{IMdC~G}^y>*t&sXC)Mn)+VWMX}`;aIIEspr{jg>|afi$kN-iW9W*vki;p
zpR4jBhnI`0SGAl7=p!okI)9uK12!|-%<C`yb*^^bf%UFlmytW5j@vr7Qa)~z?p<4n
z9;$@W&6P`^V?65pNoq)Z)uf)2b{TYj!Sos6mmPHdhd9V--y3AQoA@jf)KgKMnrW*e
z&FaTD07a`Qs=cQ{+X)Q&PN3d@aSns*>e@4XZ!vX)UL|I*YG%2OLr%Q&*>@TjGOG)z
zZ}Ahx-`HzZX2S!PI6eF06QhCqb&{(drO26#&8tAwiO@nE8e}y;H`My(d`_Dg=)SbD
z=?U(Xmz%vWF>j4?J%hk=Sl1DwtJq9^GIRSKw&dU@c+<gGVt=fFByyfi?N+Jra0*Cy
zUV8D%tyHs^1$u;d>*=gEajPQW<`;LNefhoL;TpU!HIW(UmCIRWfvW^VNwd$F<t?Lj
zGg^AI;b&zuQ<c18Y9Hg&D*DzltlsXscsirSXZJR}psSa-o0~VmJ?~t!*VouN_=Gwk
zVrPcmS5hVks;bgwEI%1|x5FVYQ@8Jekhg&=(NKE?R?T^wvh59f!jRBl(UI8r4SdN{
zq{80I*Ht;;llJmKWE3!e=f(?n<^-DO@TN&sK9Sv=udKdjiE6BJEVK|BxPxtQeb59A
zG@22LGyeSd$o?FQ8MkXpbM$hz;b)Js*oNN>#}UQ0i>U_~$f1G=jwY^<#QhSLw)dNE
z=UALoe!(xMKXcWh>g88=VC(s_DWxFb4;Q|j?86lOq3-WfWpEshCif>g4ellI7X^7*
zT%`7oCaeUBl+C9nGYlug;9`?jWmBf<ziPzAvOZo%`td?61xr_zI;Y_4ub`8xWdlnC
zRZMG8Bz~!Nu_DLI$d8KFGX;-QbB&uy*a4<9G}f<bE-(2iv&#n&n%c2X5>Nx4TEOA{
z!2aapFAX)Dn-dNCN+Rps#86?|IHS3Z$DpI6{?hI`fmwuw_#?$?<J#C-T>G1_+=c~;
zU5!)kZINBa*$S`J+VZg;WAR1#)H;s_&V^d!d<PrknYn^ifm5A|&-hY~+0Hp)a#}%&
zbxKY`;ysxXq>?<g1zBc4-A$Wv+?)^-5$fh$9IbZ-hu@&Rw9Az}K#OOlP!XSU(r#&=
z079qP1=Z4V27fx_4<L}MW^12pC%&?7hCr}UfW{+3Y<iPQWOdd6OE?S(WRHJ_Z|xfn
zKS57bq)k^<=Ajm25~pKU@baIsuBJx`Gt((o=CTX8B(-1#_`Bepgm9NWPrDR9j;T|-
zy&N<sA#MJ)U#QJoW(YY8iJ7XlXy9x-4_0o9mPMP*FHX)NDBXBF6NSD<Cy4A*82FyX
zF{JAW7xR_TQdD?ByArNBi<Cn+0iLnTK|foI-TGIdM`hjV{>(LwR$2%&kX;^b)<n)x
z`yn?)e<zi@4&JXVL}2#)p(%5tpI))stk#pzRitI{bzG_JpdIvJS^$VX&{CzMS&1*D
zoP6aKit$ZmgXcjG*CLrsZ}k%E)km^Vy*|%J)jCnwfgoiyWU+7u;*k+;w<%_Ae{~6i
z^A&ql$d(ME`tF;iorU5($;}^9#r#>ZBa3V8car~F(jGq!#sUm!s;dYuoyJ8`RZ)L6
z;vAAOk<OPn2OhC4gdoG7te+xOZQixg3f2|Q@ALTLK=Y5?eNLzIY~?`v>#I=LLt`Y)
zbW>mnR2Q$mROdPHwXx?WKwfn1V4}--h>+F#=&JEDi}!l7#^d#ZY2n7}OY;%!)E2A6
zTAA3&N%<37hue~wO~D$ZcNkin8eZHJxQ!DG>Xgo#ECx6Yl8~Yz(1|n1mC%?B%G1o+
zQjLiX*(Pl=#96vEUk!Lz1a9MRl|MJA0Z3$JfkM|u9yOXxVv7XPw%3tgha+2)INCp4
zX5*Zi3+C4FlTlB7^SBj6a7^Yq>01<gIYA+0TP}R_8=w8Fm(-Nrh`lfUHMRR@+KGGc
zaz@Xj#k$zP66Gb>9-V%*KDKQ}y$EP|c}vaeKXRgVasYEewS~=dvN7%)2pw!lO%Nas
z2jff>7r@k6)gZrG?UN2vEN~RL1EoVOCEZ^Ti80x^CqDWR`1@G&IyA#jxMp!GbmPqp
z70>|JF_@$2V5%Bsxu8m2x5W)|MX+k~W>?jgB~6HQA6NnP9VWBLMY-4x_qG+8`)7W1
z<xef7Cf=I?wstMjWr%$=POwAvoG@_Ns&?JFx_@+)&v)8MiUlt8l`sj+DYDi8P5H_i
zrnJRwA<RyFR~(<klJaKjs%oA5LTIOroIjn;q)bXczk1kxJGJ{siM%+Zvqi9PLATN8
z(DjytCkr*fO_4{;;HQ(+F7UWSc&{E_dx(-|`BqIk(1hL^YM5wVqIuW_-P+`V_|l(+
z$9D8{Id4@SB=hKbc0xNfIh?kU*KGv5vuMFGgYH{Xmt(uU^=K!<A*f2ng9pYPrUtCa
znA+{lHuy3}`u^HcWxap!ZT<_{Kyyvvk9*qm!ljo?oatV%?yf!xeF}Tr(noD<GW}WK
zKim7^-BdP(So(w`*!uVCQi)r4<Gv83&-(ERM<l|<X9EuL8BP|aJGqCdTy7l{W7qcP
zLhGjunm7l~2f6%l!fwRR#?1snl+t#7K03HhaXUts40iA8mKT<ZoreLw8kQ@<=8rd}
z1=vOpcm+FT6fli$f`M6?lnUGl+{A+zyf1$ZG|G2Fw4XCf=v9%QiXXkd74@q27vrtC
zyJW!;7I+aGI2@qM#|d9WEv~3f@j0s-+3PiD<N-wAKZ2IK=R9#Rusj|K-J)91DaBK;
zZ{@#<%%h|kX!cO6P)q!N{KEACdUIhR=1z_G)?)QL9%i7I;>_?U!A9W`-I}QB4oC45
zlU6%cO%Ldq#ilIv)poA4U-ik?%DLTzIg9E#4v$Cyh?*PtRX*$4eL?pWv9|X7=o2NL
z?#jS|<-9Ih!%1v`-3Z!MZ^3nwhPTO(?7sf^kveIGlr?|w`7Bu+a5-a#|66Ziv!U%H
zg1QXoqDg8q&))63?*sQ?ZC)bm={{A!E7WC_*m7q1n$}&1?y;6pKvh&c2gAu3SJSQD
zEFZ5Mnu`|_qM-<OUHF6tPLhc}<<PqWuKJ?!gl6p+efTxQ^b4~y{Lu?oGCbYB+iyj1
zT5yq<fA6EBW-;ohQ9gldRd6sEX^_XH4}8Dy3dhgQ3TLiGQn;Xh*B=`=wV1hrXL@sb
zFuOzV>thrtH;r8E040_~N%)EydbJrViw^07W%4Zl@-5WW?;TOpX1%sXu|90|4)@Ao
zl}eNfWuUT-I;Xt&LzvF)zStc&?s2f-4GOSv=NUHDAC<))5L_nuTle=>MVEIe*(;W_
z46+E+gXe4=R22EyM2d}g5EmXLjywAqo6bjTjxz~9sp^|35By1C0WK%4en<tK6=@tL
zNWNSoFN1gPJZ^i&;H57`b8-XH_}o(N!hZ!IxLTt~F$3t4&T}i+htBsRL}1*D^G2!<
zJ?zV|p*ZM+vZ*xB(5X6qzWi=}asIxpg@6)`MDhTA$3PB(7Ng)Tt@x#_AGV_9QaU1y
z??{{cx9nB9^Jm4K&>YLn`ZrCO<J*4ZuBS-aR00M6ECo6!z$((qLXf&!3%Gc6QHD(4
zf6>Vqt3<%FYk<B7dV-GmNus1xB{4bV*0!6e0`d}!agTmSmhR<jWL@iOs3)M*vq~?R
zfjNPr8NTBwP(r1HEOYXRDQ$J$9#U(){Ng+}Eq?O!@Opg9x!{}&wLxN@PCGuakxq-p
zdf1kUcJr;7cb&8JTx~dq^*^HWY;Wr6O-^&!y&P$ga1Jx7woa325^9)8_K2x=3#78O
zp4>!nE`>kPK}~m_IL)Dsq8iEA7E0qrc_}G%ex|^V=zWdrrov*ID~+moeVylFD4#qm
zJ>|TI5SWhv<eP=ySDu9PFa>QTn|Xj(=BmSGCT|rBW0+WdYZSZ>AZ~Pci!CA(WL~UU
zfu9*wJGgN3wHX(bPPjX3J<h#r|2B`*;q<B#da^ULhqRyY-PME5O~;_y!m57K#8kbc
z(Kk?TsDfV9o{m=k)GLobZr5Yia`LSPjfymFeXJWDCHu=a4i5S4$I(BtMk)xDaD|?k
zv7l{K!7Ng*!hCf6$Ys0Zj5K4VQLd(wqus-DL^LzB!vLE{W8}V+Iivle#>GKo!aX`N
zwqD%Gv37u}Tsg{7&<B+cG$}8M&a8VrpKUQ-r9_y2fnVH34Suq4-itN6<W?orOG>l5
z7;aF{)th4Oy*HX<_{qb^o+OhRkWwiCL8Fm=zW$es6Xp)%5V&@e*_+Gv2UN3Q+!Ddf
zaX!|`+RQaa*Dz|<9i_?u8j_ueo57u{b?d1E6-qcGSUau_&NuYv1oFKtdilk#Ol=h}
zfbf5^#3O9}XK^-zedCpZBTW6-eF^*G>y=t5=4U8(L4R8@^%&Btg%3wSSnyPj4Jc=H
zoS%{arPF`w{FS2HOA+2>tV0kQBc-njlx(Ms$rXElCBq;9uR5oMlF0;ac2wh}km~Fk
znN)e3FA|<~Uo=dtZf~c*oC>{|;?bG9%Zse6ZSoQ7v71g<ciewEqgJd}RFx1D?d1y=
z!ke{EJBqk$d4i@~xSpiw-oNpierf=9OBa|WCXR_>Hzfmk)QxjD$GxRHLTC$>*X<;|
z?BB!{3o>mZ^L2;n3m97|IR<uYnhx{;ggdfp3!f(s5jsk(a}8I_lGGN7#62tR>QF?!
z$(2SkGx6b)iVo8G+WX4-9foP2LW>t`xLTXaHpm>OI}~ee9PEw#7oaLBR&QNg5I1%v
z68F%!5?`8)RPQ<vvPOPtcQzvs@?_02<9;>U;53MBZR6HE!xx3dLwA0*`=`xYPsq%d
zlQ`91tRKz}=F|&>R%r5d3Ac;s(KJ6bOGdY(GbMMjWj91OWHy*K3=5u)ksf&2(8(Hd
zj1l{e<<ICE<GD~AnARpZADwaSFqhu|=Tmj6a<^SH-NP&sGrsUT7rd=~mn2w}Aja=m
zsB$K-Y}JB~jP?h%J?u_ig&gh$>?mkV=+{~KAwCOqy#&9<D%d4dfzsVh+oXE3h=~2*
zfQyF-Cw78<5c<{j2>PX90+%uA5|Pi5%gQzh0({S|@=BCvg1l`1D}s#PGFsQQ#x_yK
zUY)@&MRbbPxS^1gPO}D)UDFj38E#fcc$Sn7>mxATD9@G0Act4nk>wj@f@1)nYjDGV
z6|cLjRCNX=*tIL1=uZFNBFI((sDIO^Xw>-m!gs>_wGM4Mj`Q(GR_I_=Z-ushT5{HT
z*$YZ_K#srsjw=5HbS;LOptn~8XIO7lYG%JT+vUexXC86^AvBCjE-+F*{nD5&oO=Ms
zwb^G=L@<=Ir~XsEhg({?QT19L8WP7Q$K0`OG#Q(AD!ubdYWC%>z-&tZ6N`f#B!fFm
z@}P?o@l^H3Czn0G*5c?Ps8n`GR{6xew$y){cUI_{&gorQd(%2+7;gh|iE2Opu1sLc
zMBTT*CsR;djkHsJLX={a2Rd+dKswEc+o3?wx&FAZPQ11@iJ|f0R{&x~_bqN-$p!PZ
z0~t%flJl;L$&F1vtuM0ZNQiIr#fthWo627&O9vsC^C?(XQq#+>_q@Wn?vkW_a(k8%
zCmr(V6Ad=&k9eaIyA~UO0`d-*FXjvc%*{k*84aH&&#;gWY$6mciLAUn-40vGuC`#H
zCRAa_$qtv9HznDRe%$kYI2V}0I+4~9e|r1k=PwAq9g5`A6w#%ex-*lyr2J$xdc~ST
z=&4|NbbI0Xd6tOy&E#gf<+;dWhx{J1e#_(v<A_I-mjrCq%wg4x)M2brN4P!E2oJ@)
z`a#Zv<R@}+dEM`9w`ejYp*))5M=}Vv0by*QPlw0+y}b*Gk>G?dAq-)Z@OrN-g|W*w
zDgU>a(q`CL6YZJSJAG-)J(}a(yjLxCb21$m_v<JS!_oP_+uycY)L4s%H^lMR2+tjK
zl_zhCK1*uy(7PDuy185ymRLP({ko4pA#!;xd0D>-|BNE#7|EXh<?r<*LG*J?_i0Pw
zc_ug3f}hHK=6=xAw3EH-nW>_LD0FFKm+8Ph{?^p4T>soN_6Xy_#yS)8?H5O0-7A!v
z`(VHiA!4KNHOx+Lw&f)1LEPY43pU734vjM6N?75$@*Qo#SB-r>SINB<7BFr=G-Y}(
z^eMnS<urZfu1H)O_#X52_`uX_hQH1pd#tf+TinlKq-1pQKlolR&L1l^0~SnsMT!g<
zyhSD`8Zk_>Vfnp;e9~s16bkFt)EGMZl3uvyh994p`_fiyLT}AlJ$M#6|6Chx_@?Pk
zoY^I>F!h^fR{FiUBt^$s{pOhl;~c=D<v4Mt+vb^@+4^oOEzZ>fGb|G^>=%Odp)uvQ
zwQTY36SHGn=IeBEHYN!A6c}%S{syri+AkI&lWJtpV%{>?hBqj%WB_9}XXK<>Sb~33
zQnAKf!M(NGopz~v&eadR^wMMWr9gM$wYWP9p%OE93WHDeZgp>C4|{0>tS8d(`c@Bc
z!XBE8?R`6n_)2B=!1dqJW#Puxvzj4%=&j@@ctkb~zi-01S;2sOAuSIcI-MXi@|e}|
zQ%rc3yrtcgN9hG4y8bgkrWc*gE0OjK^Md`*vlvHdWtfI|i;n^0%N@g`HIMd%(zGQQ
zsX?(O8EX+|Gp7<biTA~*T}bB+kW1Rh1vYYib+5euZl=cTQ@@ZsSSFf347f4HAWJv#
zPP{Q#Jitf0(#tD<ay;-QYqL_{(E1)JFR?crA0yktIQe5|q`-V|hDHECII}8(%RqQm
z-sRoXlbZv$RzM>d;jlD(=s?%pzO^e*W=gZpE1KGUV|J-sUPn?(LesE%NL`Vq&o}64
zGN4|zU8%h`<>?*B-^aMGGP@OuaA~$rT>+hXb(>WGW}EEMr$&plBESOviddc-dcgSy
z{=w3p)3)%F=^f|WFcrkz-~(0<Pl`%#op7IGTb<4HVUzVV+xnv9syGEj2iUFRAuAmr
zHuDRmLsK8^6r;F!u0M~@)MatMT|Xon>7of9T#7VYWk8#wA-Z&g_+aCYF(Jmm-r)A^
zHh1mv5B!p^#4n8hW}qfg9Q`feCvWvoY8|53s~=@jXHF0Usv%8v5$z(2Mk5cpO_f@s
z(w}d9TwS+qz6OP2UcRNmbiwb*5^5IE%zLp>r{u8pvnE2RN@<;+sq*>_#dpA6>(64T
zX>YyzzLt89ULFEbgEXa7CyP!Tw|zWrX<oY1_P(b1vXSzHU0ksCKHO~aj7ENc(trza
zChZbjAHd>hhUOHJPz`W@*@lyjMujQH%BLIC_{Q^o^PUpXBUB<&R`wtl#~*7I81N+q
z7#}REuP&MKqS0oh;%gh116^-oH--;D1vRT1w*C|gAAcjy{ZU@O*inz&IZmN$(sURt
z-|Vch-zQd*%yA#5eYF!TKso8&CP27JN5!{qaF@fWOTd)-1iybUJ}7~-nQHCmp#Y~&
z|E`4Z1Hh~-ti5p(X(Ecu$UL?5J#mZxM)4Ai+Qvk74Z?V3T#Gy|FIYTi?!Nmw7l6Gk
z{|Isi5x7(0Nf##HlnFRci<Z^XsT1sX`H;($>p<Sj-jA^XCP*jl_$gjX002I9<+%$3
z_FoHlv2-mpLCY}1*2k#04r(oYo9S_NwUrffAu_*P!Og>?RUuqWGFd0uU1VNWa;vh)
z%yA1;Z5cTklAE-gCQk2{&kYk<us;qc>K^Q2XpKlTP7){GZzd}S;<E@YHD&q{CM*Q6
z4s<mw!bVb>FT(e8T5*%#?FalJ@v(h6yiV<+pHT@SskbhYiC%Q;d8*u2bCKuull3bm
zXT2C=-D%Fhso%(reC7abT*1OE9)z3oeO4H6J&$=w1ZpV^bxG4cDWjdsMBy<RpRd>m
zRs%~FTShsd4~?UL@bTFqrPt%6E+_MkJKR74>yY-4%_O%6Jtw`qBfM7)yap~Ff=J7m
zx|;hs3!Nneye;LR%29Kgi0Q0z6@wG>Zy>s^A$GN*UK=^&!cw`^>9juJquQha&4b6O
zTEj20+|s9`g%<A6Jt^sI#^Ukv^!{*ddyeES`LY`tQ?Z=0^&6*2s^V@)XK0w+Pm~~N
zQVfHkN1x0g+ZQy{!Tf2A8&L<B)o(7b2#*K3#M1mN62UT=UOV`(TNHFK(jCh4NhVq>
z^Q<@0^^pu^D?q<)H_bajtY(+hapM(%{WQ7iRw6>cXx<OWe7JYWC-Rlow_~Vmc6!ap
z=u4i?WRsESq9DwA+uvHC>B=WTRUus_?3z=2-seOO7fxFO8#bUxE^GzNINSGUsEXpZ
ztD76!@kXh#r8fa6k$S)E@1;pTXYDdJTjv!BkohQ+(8_=bj-Xtku0}l<tM>|AW9TW!
z<o%Lh0+QK@v_<)d_bg5|qX#e6W5cD?FjLr-G0^m-P|e3po5Hf+o;*>Sr2F2}Jb)FJ
z9{N3@+s6a*Q$H_$p$-*2&rP4V`R$tnRX4W4fh){g7i)^2#kYD{!>jQQ;Xm_l-fZ_i
zoDg5z7p@*yUr42Pm{s2gw<lZrV+mD9d9@_;JF9Z&`UpMGEm%=;6R#G|!jM$l4`Ou<
ze=MVw!5a5kbW+prUm!R^dT!M%p-H&H@p((vb8|Ifn#qc!e@2B71v}$z(9Af$d7k;N
z)G+)9v798q4zY)zCgr~(z~BGYlZ~Gke-QD{-1N`SfVUXTK>HRflR*5BD*oyEAQ7`^
z``@VyLsl2RV%}{gpH+1F_`Sz@Eq1!X{Rgn{9mRDuVv(%-bAWV-`*@h|G3&9kDDukC
zqP71*bpNfD|MS=XvB3XW;D0QDvA|<GESll5T+zX3nffAYG0hjOJ~9b$sWlW1V>Cs{
zsnNA68N|=PU)k<I<J9#FD)~f$2_5-<FT{v7q?24j*$(<Qn<Bt!$xbPp-iY<H=~IqB
zugy1{V4}2JiC~X*el*lbjTb+u(vP+zLj}Rg@T-`FVal10<ZybI<>_Mon7wMIe%k>d
z%o0Po*-K0$O4CX0WX(~~`PQ#<^cVkPf2!-}c#;i#C=g8d)<46`|Dh?b`*n|?O=EyX
zh5GA1J!1OU2Nqs#8468rdG0I!GDQu|@(N>Z!wMTmHDd8IoCEIPO11K3m1fZ3NuG%o
zq_W1eO8Q(^0Zzu(3R>xH+{07mB5RiR0zID3G|;l$O@A;lB1mIL$Qk5n2J2mwiP0t+
zx;q)t*|e$ugb=HCSUDzZ>OOv$IQU`i=EWD6F-&XTXMg^C36Z3@H{~YG9pwj3>1+Y>
ztNXaW3&4$crdV#<$)1lC<_a|n9ZQHYVyhu!+Y+ns7?Cfp^hyP`^ixw>dGjB@Y3Q9Q
z&i>FN)B1pmr!Y$(9V>%sUB9(AeJQTIavN4gX9c^7lQZhEZw&CZefE#QmCsCAUxfNf
z2N<0<?mavN6*=Xc*Jfu+y_gF<(;Hrp`WfkZq|@9rJ}Ac|l=9v@XeXo-r>CfZQTaX5
zOqb46WY7lS<j-3nzAD(fAWn;Z_Yba9#Xt|{r<RlxJlCy>Mt?X+pHRs1AOdyR0ABVR
z6AN3TwB9!xf1cb(3cz&{+<zF}o_sx&|0O&Xdqj|@TWR>Yy3z)3CY9-%!!?yi9+A6u
znhhFg^dE>lVwNnc#OKZG8|EciShWzz_yB_!EY{$pTG+dNUS<k+P`p^({p4YH?{Sad
zQ!kR5yD5Z5gmRK)aPS|X=h!j;&;FAG`3!+QRPqcSFsLW%-byV>uo9J;6*J6D6uWso
z+&?TE?gxW}lhGc!!M{v@i1Xhp7CZ0R<{uIc70Z$%8Vcse^k)XOLnuhq{Q(IYDguB6
z&qSJ8evFc|9!s1I_oB~GgURpTwk?FDFT>>b47u>nP1bd>hs18^IKrS(q*-wcll2RP
zb7Tu&*LO$?@=dNrb=Qv>k}7W+1MYp#-Bx(d^=VKsi#^BABASbzCqFlj{Gwyq;Kef9
zYxI2t!_xUD1=WB{na6ogfA1}N46uU30AAf(@3JQukZEE#(LpxclDATmlJ0&RnywT;
z|8Q;W9L(e6Z7N1~L1gA*B2!kGrqWC%93&vkkth)BE?8hkG?<#WohZ$LW)4RjSq@We
zI6b4vX#l?Wakudy4j^6ftdlOYoXlYB=>cx2V^>kgHg2jAsRoCgTV9cEmmP|eEuM;P
zCNd3BLv^ys)B|nqy5p?O$ljwkrIB4T&k`H9*#L#B0_v>AV6z|FBP?(K7+Ji<VjP_y
zN`DRMu7Z0CXhUUik*`h8?r=1y7%d2s!y10Mm|!<_lS<9vN1OdHw`LgFnUxWZE&K2_
zt9N)%JfZugx@at@p5tyUd20Vd$~zf{uoKO_@zmQ%(ZCH&U(P8ZBu#RX#IOe!!TzIc
z%VDAdfg@~pqNQ8<==GI51uu2{PF_Gb4uJw#{kk?dujWYCu|2Mx-`t~hE!kGU&YTPL
zeczr>FaFr?t}vl%^rX*?xbJaR>W0CA@E}e^Yg7v^a))wvJ+Xmy;uZ(~2>uX1$cBdD
z7C3(#nFd(AUX*!0)2%M>jKp=BXkbEBhQO<3_dt9|o7MYguQk19TO8v3J$KwA?2ebJ
z!G_Dnp)Lt?4>-8@IunjNXzccc9`BLWQ2Fw{HdP@SvL92E`Y9uKSnv2KfnlH~yGMcL
z<)8h|i^co)r)bO1IcoT3c<h1Fa%Z9dsEki8h`gKHkJ!DS9@$%mxX*ob$RNyL1`qnd
z;?-G~_F<Jfz{T)*jq~GiG*KW0e1epHjys(uWj3t;r}pSAm$Z%IMT(y1PGJa)<5Yi$
zANfAYMA#{2Qfj|hdERgHs`Z->jXu<p_lSX1w2Q7Y{d0c+W1AbjF!0sEx6sskn8A4+
z`A2{L$NmV*YX6%U`=?6~>1Fy!iaNj&p&LEuk1rgN9tMwq$E8C*z5?J-YW7?h@q^6!
zE+FBWvVGl_r7~~G&AyN+WdS@Rhm7OiJ#YM_&;)KVkJ5-2G^$7#=pX&+&2_I4vA>pz
zU~{^G@l&$1Gu$i)1l7YqZZTQ=fzBKqxAv#E2?X~KYlE&-CS&m!e^2&3#lx>3*WC5R
zX}sS^x74sRBlDmnWaC@Il2iTUnTQMyR}!Z@Ryvaxh->J`V)x^VTb;mA@2K<3eF53!
z+MBnTXHBrKYSs1UfJpi84Z}ObHU)$ow?4O?^lSyuk`K`ebB)eE`K+gW532DUW(|lA
zRYT8iw`UR`?9E7icanTQd`z_SCVVo^z5y%p!<j+fwlveT@}0Yn&#XR6H=qr&x&G|s
zq?L~{B9B(*NbD38NrPcee?)#L@h1HxmqBh?zMr$phy1})2H~!vhPZcmS9(R0@4$Ea
z7`O~fLE@_`_h;?nD!$@*d^8R*%)Pr^yHM#`#md-p;$0O+wt_Dk7%mYMD|IRvD7_$c
z<N278P0~FG$A`q8G3}4>&?lEmKA{72uc>@)?vICgN|m`ti21V%Exw|e_1k^2Wbl*6
zo7Rk)O1h-GkSO->k@~YH(ui$_8)sVl<FKA<lJM^55*)0U!6+cH_3h<htR&rIBW&O9
z>PW2zYC&8f-qju3!S>lPK3Us_0uDW?W%nKLG_MZn9>^ynILiI~hTW+;d|!o9NsBJ5
zOLxn(NW<xk!4!$<(X!Y^5}5UIR}`03j4Jso@Q2&h7b5BsNqyC{7D+k($k)Qao34e*
zkzwlL7d;DSfp121yGa>@4TN}ZxDLG5<Zb?N2Xrhf?5nFfh`j{Q2+#;6j@mwm{U|IM
zD7h{PE4RVr{AhC(*g{Bk_siYDyXP&}fU&#!*jiyp5Ypn10i&#~VN6_q4dKIXZ%(#H
zr6I6Mg<GHjLk`p9yZdL#p;&tN7=1X`{2!M`m(@xJb+^J>&hbo<<Qce?@YQT62Jiuy
zj&eyAsTs5%E*26ARYs{XI8iq&lo17<CQ6AuL7sfZryLK~t)t7bB-*(b-d(`kajp6R
zTY8k9mRFA$)BD+&-ro~eGA1#qY@>^@Qs)d8fVdmmz=D*6&zR!qw1;#y^jq5EDsBih
zC%KV<3FPo+Ej?wrgcGzaCDUbG3Nbp_1`H>PhdF(8jgs2!<h_yT{QyeiR^h-uLMUfl
zesH@;Ow2=9KWJ1eUkUd$kvEbwQZ@oED3_xBo>7B4ESvG%<I!Mk>2?O>gB9@V2g{0+
zH3hh))eR8^iXF-g$p?A9B``%kJqc;v>i`R{1^+y|t?sFYQ?JxUYm+bC-Z^#Q>)2kH
z9)4=W^NLpZ(FNH-`J<eWlrlP7fsKB@5AN3we2Xkne=Oh#-x<t)@QjSjMubp=1Ajg}
zR%8vnrA1VJzdHMP7&z=`^D>^U>sBKLlqu7VGO&5h<bOg1A(RrMfB8#hM<gq3`2B`h
zIDxY&U`D)04JUc>`z^gOh+lHo1Gn|Fr|>&_nVqc6ED44{5HWEmgK@hz!TxCr{^K5F
z$*1LR^I{Q`3<H<g2fujUUpCMohbnHyvkr*1e=zURZ78tD^FMAPYx`fky;WFT+tMzK
zySqzpPjIJ!K!Uq_@Zj#!cp$ii;1DFZy9I|P(73xa1b2tiYps9n{r%s?xjZ+}&BHTi
z%~>_dM!oOoC{8jE+Gi;h@3-SSbrFyFE?Kc{2{uwCj(RNYL>-U>(VRf^qOQN8U_uvu
zPk>igPN=>#mvl$^?(5fzDBLK{PseI*7ZE)Q%*E7KDE;U?pKY*qmb4M%TiEVv3=i4r
zLrfM$O~@tPTU<XaHZn0;w)#*5Oi`}*`_Y+$1mQ9s3x|czdb85gh_k%e*TXr1cYOoo
zmu~r4OH}q}m^0K{E?Y`4PE~h>Njhn{?3ZP6tV~WQGg9xEB&YM~yWfz`BamSP!0yi7
zy>17^n(4V?*&I$0jSnNODV`5T3D`a5mIPouV{g4;Rx21{&qG(8qWpaPm}sxjHs6iD
z4e$HN9os178RKEQr=YfZe8~^CU#-QmHhvE82s4dM96oE7vb=}biKLhet|#N+RHM2p
z=S0d&xre!avP{f~`)OB>vvJyNcaD;Us1*?*xh`!_R|$Jngg<&$Pxvbq=YGm|6)6DC
z2rY4b7N?rheW|y$Y`A~>0%1qYSZ&k(2;>&)bT66Ufb=G(`!+i-qf7MHXi2g+(hQ+K
zO6y>ND8f0_<hm}u15v+U%uownk1y>Bf0U*3t7wq0v;-hDuQ1;Vr}|a&=2c2Z%0k+_
zvm=fr9{={3O65hu*>A9Z1wW4itOn-&sNIipHmJb}XAIX^>x$@3byX|y3x6+O$BCJ0
z={UwK$nXvxcT_!*1eX<lYG3pc`JTXrVn-ht;jZvJog8<NJL@J)O)-t#4)+qn9lf8r
z!jzvHRU}_r)mWs#XW)y3e!Ao-pqzHCbZ5q!V-p+f8YMv8fT?Q0lyV+9#MKPdo0|cX
zL%IZl_cS4@5xNpqO^j~85zP6Zn|bh+bAh);-^DX%KSJ<jqD{cL<6zt8w{T<RgB0)D
zN*9U|c4a8sEdC@cG~$ivLiIOB$7JVD=6$k?B8Dgig9+>kHH?uAhP2wb_q%@X@bm*k
zT@v;PzI5+2i|s7Ko(kQ)zAA8f>S8S?#M*V!Z3qLt7H<2%_rG4|=DX3(kJ8l)8J)BH
zR7-WOq+!h&fN#MNFn`p+=F7=7z&P!iW~UK;EmayLH}qHnXlR`_aGyTm*T9rtpNF5%
zkT4(EH_Fi!bz5t=sf5AvM0_!sQRMQfBc0g_cZm6nd7jhDH}|Vud9K6pOm{YzUvlj-
z`FaVPLD~txyOv2WNc+s<ir~#X{Juw_Ctzz9Ez#YEP3l(Kl4jmmDo^5yte-mLGq^HA
z09zZLjH}>ex)}*22=V^)<5>Ye+7_Z4%N2(vr7zi5EFI;DGY3>h25CWE%ee2z3PP#J
zJZU$@qa;+zX}Nc?iGnn}w^#7D21EtYbh7qLA^0>e&Z|h_8o9}d@mOp@civAv;tgjU
zwa#A!DB$zh@^fjMZjG7$r{ZGoh|%`w<0O{=Q4mU}EjRP{PlXD!FDe)M^|*iFJt9wn
zS0X)7gvEZPn+sv;51=gg?yK0KTrsAg?e@DrgRkkBAPS`&3JKnrycE7z4+Le%=M+Wf
zURSJ4DUTAlrW(RRFS6DVuRMsBPZDF5977ReQud0*M)^cwjHvF3j@P`(@G{~2uh0nt
z!Hc58{>NAoT4D1-+9hLVL#8sK3&+Tn$X=AaRu}5zIu<^-1HSxh@EnYf59fK88S@nm
zFA*8sc@B-?W%sEcsGe4+KF7;<%R7L<n(XivuYa2d{gWOh?JX^RNXA;Tl2urR4gQ8i
zp&~*Dg12Nh`qpo4JYMqeXr0)tgE6#RHYHMuC53XG=$6)niDroX<kH3{2Qx@@xKTUt
zJdGF+h)W0_A)bIueZgbYX7*SAjvmg7c#IlxG?zY_HD~sSJ;?ytwJ7RftbvQ@Y2s3Z
zAa5W4-ddf6FU$T|tC%hZFD)I&t%Gj+Ui&gp5`mrh)3kfPD|eYdD|2$oyBxr6RYUVA
zfxTZxI`m`VNQfXD*(cc@A*8L}+fLvhFec8cj!%wHEX0@MDoy~;Ea#Mg_Jw)}T8%?h
zPy$i(yl<w3590Bu;veC3qbhn0cl|6q1X1((AoZ@J3H)nI3@w{!iG7~{x^f}(2<G_*
zs<u(x4#@dfV8N(C!QyapP{L2)8P2S+)-malE*<{mI!t)l9F=vcBARLj^YB}Limqj%
z&#33OYj*D3eFr;dj@uDBZ+Z@SsI&Zbg<Fhpf`s0`SW6R2NdW#y_oad*qQ0n>MpAWv
zAvYmf??6<cA5x^%HP3cx)O&4JbP*{(6<;ifZ{mPZn(p(RzlTVzqiCPe4yF&>Fksbw
z1!O{)K!8UL!$}65*y3|hu_+FT2_GYgxU0h4#=umZ0pu_4AzS(S;7hRf@Uug)hHPYv
zZPnfKNh5s)Mko}M-T8{`k0Ub0l?{opqmZjb6DBW1;tSyneLBW{k3jCRnJZ8|e}CrX
z7$sFLNtP`(Pe$hqBMc&zEQ!VLrV1~E0;7w-b~p~0Gs~!!FMU`r;ua`F!~T@KR0VWH
zk(Ta`trBFXt=}Ra2z=LxJt_B&rV_=TA1~u+$M`Qq53pM2uz?O;ggGYuHm{Tf+cDcA
z4qidihi}Frfje7`A~kg|pJC6%lOW-CqD)L}Y_MrN!^nXGQYW5-IGz)&IbBzCx)*CF
zXg0)GnC~AEw$6){LODI9Y{iCG{mnT3X;?&)0P!cOUc4!c`j?9SYJu-az#2(h<hs;|
z@k+vFV;gt>3<aNpYZL6;saD**TGy1WIFjLZNOwo1Bj;A<jKm>wC57imJ9Cyiy(ZEx
z+TSSj=d$A_2~_BOX*)39z;pXi6->)5qfVk;H|OH~GRLW!1U};zT#h~4hkcjk753l^
zI(2PI{vOcAl{wu$f{f!x+TDh#dlL$k$S$mhabIlsc*8mVQoO%5Cs)#cVXu))_a0>l
z_PpZ>*ELoYN<1(ofUy;LuL>MzorN!`4c&H&+O1>yCY_TT+y%4|b)?HmpwOpA`ulP7
z_VK;(m7dD>1vEGapiKm1xVFF@_!$bK5nPocEsn&#;|n^G?5YAs`|bQVGEkfv1~*vF
zqS`Ky)X#I<O|Qjo+!D$JKPr*pR)E)3jnO*Z(A;)p#JL-*s{6m;OaB`<^$&dN|M5?n
z5`;UYRfW7G@ej$<u5L*ZOddAzAu3~Xo1<)SGKm0pHT#q$`hVdbP*_=34^0CX>O+5Z
zc5JOUjxXv1H80!}TC1l39Dcq@{&#4w=@tu(8c>D1%Hw*62qFZRfaOgnh}q+zXJIcn
z>E17pt9|T!jr3hZKNqQ~rYIJM{J;uT8}bE*!Bw$!!}?M7J5jIiHlKoO))1pZ(*Qfe
zMRwUZ8~1vYOE3Yj0m%0T=<f{?V?(<x6n5?hk+B_U)6OaSFK5^9Fb#8FmuYiKUbK;M
zJM)&nWKj^YSm=4txS!#U!;2-WrP^ud;j(YJ=v=BYVL_@{qKgEgns7jIP5)BM&wl@7
zI|}&302m4u*nXKKz<s7!yP6g%(D>6BQ#&<CtVXJ^_a)fV#~sFu%#Sr=ocxkGYJ3!~
zly8R&WQXO`ybfU`s@u&hJ$(j4B5T(^NKuCH<tqQ-0ITmSmtyZYi8k*^^5mc3WdC|U
z+-QYjAIb<26&R8djWjHegdrUZt_s_4@U0}!E|xaR5286a<tjzmaZ-5`0y)BYMW~1m
zBa*U~Vt13+A$#Szd*wkb=<`RuqTS(}Yvp_&IiLC%gpb8!&j@0J`;-5hEs6ERe@zZ8
zu1EVQy0i907Lsf$R2!M-;}L~v{C<g`l)eFt3}Y3h^@an^oyyhj$W(~5puX=dWx*+J
zb#3J;jWD)^)REeyK6(WOa;=Ud<rQfIzH98cSiU|^_USpe8z*+mSA@3yUU%@J_XS;8
z#Xq42`GM5cudMa?0F2yaX~f<dG)HHM-i%ctv8+c`M(nuhlTs}_+KSLdA&e@Fd(mts
zit9wv0VfRWO4&P)d!z4Ak%ic~8zYLWP-lawbT!~ZET>C**xL0m$Lo;R5pq+*eL%3U
zxbG6>aicF$VTkMzY~_dVZ<a%><GqzmJNDcU7)h`@eY4R7irwa>P+ogNqkaQx%xm#K
z(UsvbJw%$#71g8HwMYBt)1;>Qfh5^-!nPEFSvrF`Cc>pE((dvHybmHY(l`R7pRu#i
zCpV8VT4f<SUZrd^IAk!BKb<RC;9~NA;pk109%8l1#t}cfXBz0%Nw}wq2n=sIR*i~t
z;)>FS_oenl_r(#W)`z9U4!7H!fieOapsXTY2X|o9d_Qd>@Y{Izz>e$}4Oq*dz2FIm
zV9Lg_p&kxc*pQRQZX0K(yK^>U(~M>lH@~yoXS9js8gJjYzaSwP&bZ!l>#Lyi3r${)
zvQKN>Oi@ad$C3evHRSIz|2xnIbun>@a0aRz!jimLUo4#T=t?pA$^GYFB6~c&5JAy7
zHRK(n!0Z#Ww0MAA&NU~>eCR@q9-=Sm#nMk*meVhHNSi-%u2m(4Ke}_01KbgwD#_mD
zgpUUgKH}vLx)RKcGzVfrYt0HTn!?a_j6bBzxGrq8X(}QKO~Y5|aV(^bp@vNREMX>L
z!vl>Z2usrJ<`nKkyr^hHb>-3y1Q(+2e7u7?G_BsZr!iP9rmMvWigIkp*ciRFGjF+#
z;zbGbF7xbYg*S~wFxmW_f8{J=f_W0`Hd9Xbc9S|QETq?}R^wr)O*^oRc4Z&Z5_K;&
zC*bypONkmNw(c+Sck}}g#(~4lY%|ms&?7=zgc>k>$sY(^IZBS5o2R*n|3a?8edqu0
zjxw-MF<6-`TW1Y4Ny-L$zPGlXUu2Gp-b%EKBnpZ~Nx!PNEMB*BS7eo58L9=V58I*Y
znzh-vIMRG&0Da<GdITqf=T-4`^aT*Gq}=gTu4Ekh>nQtuxsL_*iJ}HwwFRNsr(((U
zD;Y1+1N`WhzF)Le?uypZRh`j2Z%doVfeh4CH8~9#s>n<!0ztlcBu%%4hD7Nh$BvkF
zmEC=$^acx)?r@-cR}<jw1P<lLdGD&X*b^n@G&jRth`Teo<9;~Q?25MzV;2zs=;Y)J
z`dkb&HGy%X?sq%*v~N(#@czR$BJ9~nR!OZqa+)-SxQ6n4bydnJ$i8>fD^u=2>C}Ia
z%s4=M6nB{i;eC2&9;vjCx-YbE93^n&zO(?p+c=<WhD}z<(J#seEh)X}CftQl-QgDc
zhMv7+ch9X}RVx_z1E~t!YVpZnTL;@tLIW{qgx~Yth)=ct=#xWE7wfNRpCA5m{zyC5
zDnc;4pKx=d;7xg$`!G-o`DQ;{lfgCS*>Xbz*WJP<WQbJOC&p|ZXCgH5Dcb>U=uN^G
z2P}H=tKpyL-p|RuWmh1&`;R#H5pQCL=v68CY1aHWy<>hxrRssxapEOnAL$C;fzQRo
z;ut`izLZhej_Ub233e-SK2;ucPibFt*HJOgW4v@)pVz_jj$ziMSWGIOh;Gu*Wz?Eq
z?C{?Zy>W)O5VD#c{veerc}aUiR##$!nldk2vdtMaK0*f;Wv;d-L~BDT(C~f7OI85?
zh}-~Iz!ShgP+>Ik(Qk=K_`==QG14*E@R1ZYI&K;rysV1JOkohG2)Vbk2YLukxW{$I
zRhTr18mxvTkx6xUsho2x&QYog4j$)VBqiFvVkCJoPr6eanC9RV;0TAfe{`EFH^II4
zVTZQMI7MF?m`mCFBZ&z(V7E%cZ}T8|AM&qN>GqMM27NDdFMi3<7=(Tlw>Mh^eqH7Y
z+g}*Pf``pn`8$99cRasW#+0oLv{o#x%4EaWPo=On{ED#KzW`5ouo&E_1#~<aqRr|2
z<OP7InIEN91sVNH>eO2nJ<Ja|I$%e9MxLb2D^zyNPNBxN3fPYlh4~%4Fg^-qjhch~
zj#)dtOe;)sasw^yO4-eM++jXFcO1BX>%<~Fl!jPx^>a&vorF1t<b;<=S4?E$fJV_l
zRam0MqHFWgr+~sdD=+~wg}I+WWb(WBR5^5&{s*Zw76H19KFJc#Xm#7~q9WU*(xHvG
z`8xm*f&}`EAqMvhcm7oLZdEK@mudq0#GI`GjurbnA9I3`+XlUr9}~EA67A$en6}a_
zkn>a(E*`mG@j^|292;&^l>dwqA3(TIR{G`&@%?s}o=T3&m>;e?;|zI$<q!);`b%^$
z3Ian<Of)@G_$I~1w5s0Z8&#5Vky*9*4>|B_hbqz)_4AV^>$E3?HY79f1^90Cq?E`_
z1Bc*_{h00FeAzI??c=Z^aU6pt!ZN(V@W2-)Wq0*y$Gwbk+M%lKcmUc=5Am;yCy=R%
zZp$1x>47MC^2fBy&~8w|e9NmIPW&=w)O}8bVPR;$Fc{WsRXlY6uH}Xm&QUo1GH=}R
zP1;fZl@t8@>BWu^v6-+Ddk{7+QUQ;&9n!lXrK-ClN%kNYonGAtjmS;rNC(A`9H3)(
z7Z7c9#VyX^Iyz3GlB(aEN`Z<khg=6>{D{t`Sm?bXNS{8QZx@PtmUbMmeqp&!M4x?+
z=kL>jF5FNK-K0ndiidcw9Y^d*Gf7pQKbia3BVT@v6o!5VMrV$T@<5M&tFjy#`CVd3
zg}Z@|l!3C72Kc-z!nEgU;Gi>xI@ZyRj=E2#IW`e0ZPQgH5eZY>;`GPs*%q6sV+3pH
zo6}offvmAne10Iwjs><ZZv#<5GS2YAp$^uk2@*<?Y8Ss^o>=AR?#Yq7WCe&_a?mHN
zIW}4@%6Gy|k$nqn@^0C~4UKdVF4F1Ab(G2TTY;Of3*C3ILw}sfC^d3;{G+xSnJdl&
z+aYw>PC<nlV{Y$sMV>G1#Hdlv03-ih{{VzKCTHF{#mJKgSxKOyJ1z%^7WSs43f-Js
zxD1%3<tPtfP6_QdR{EpLAu#TIVDCA~kLb4bEFh*%sn?w2j`d9WULaOMxb8))c<m|#
z<1#<$H5B(Gcj%KCJgq(Q6!Ri3AlSU)UzFD03=o^Rrq$zg;Q0jEjIg1K7Km$ht|@3c
zMueu(1OY2N*w+R-Jg159-%%*$BLt#6*l`h@K+Mgz3;e1~QKtslksY4G+Eu~E7qYDE
zaIh%*zdpb)N8<rI-az-Dt3Q@e{G0Uxdq0{IvpyRnM91F_T#B{l=9@oXtj1qY;zW-9
ze|SWw%fJ*8#{f6}%Ls)7l#&TP6}$=Adz}5AyqvW~T%z^&^?-2nP^b@#nVrsmGOqBn
z9-^72J($K*Vj5+&fBua49)RTl3!HQO&s#<l15!`Iy=Eva6F+GGg#GK^RHL9nVI2L)
z)is)8to?f0Z59nO5#LzzEfjyK<+BUFhlNt=5K$6-Le59y%xh3*xNGHsVnYCdIN)n0
zUM-^_sY}#^Q7=pDy8(g0wmtgu(1O2ZWM+oT7Yh$53%}8zM+NIqy6M^ZnW>huAB+5y
zVIcQ&`|!AXV}5+wF1(B>c4#@OQSUG3=kby4@pydO9`6dRS8KH2YP2tvt$n`fCtZAV
zolC|k!*?deg-4y{M$|r$#au71zX1Th99Q(`In6arh1S{xAmyrh$53qgQH<FK<kc4_
zDMPlfXQ>YV;b0VMTHvqRFE^JCLrc|O?sZ&Ff=j}4klWV3?7>G8FrUjC-S)CGu$WTx
zbusYWlS|p@e_u9KmrcO{;^tLNPD;CK6W^5Ca*Fh#1I)7i<!1<QEmN1lXUdZd3u)uW
z<p<AFs}f2rA+gianD2|D9DbD`_#oZeNT%gqn)S=R0*Wf}fRe*wYhGUA@RvcV)e_{U
z8WV_9lk)A0V<&ZPv~T}1pV*%VYY&P|<OjamY(IDv)kXNY+l=F~B@XJb<{tri-9E-$
znqOD_l+FDDw&XK!)^FH%krhrC%j%3~JL6U_UPMJacj)~to3(C^1~@9T!;|zzRl}ev
z8%NoHNFG#}&&|pf5E|}$16b=(c@J-)nez7p2w)A$Dw5eOszrv9BI$JS4P>zVb$9u=
zZ^W+ttx%Pk&!fX8(WC7Nqe6J0aL{kgN4Il+y9i?~eM~5fiOrxl=PvN)@nR6z^GL3C
zqPS8?#X6^b!qMXRPfD$U#9exMMupbzsfCl^&Wl4d1)+q`hh9g`T-H_1o*p2J(V^7G
z4c#CSyU@yo4gm#ytt*suA8l#B{lkGKn*jARciFE;(+e+pjnghR@RlfP;u|f{+36HM
z(}q4Q3f{+~Y7c(=O>|*P4={XcxgahMum{2Yd1RyDPz5PQ6sr<j)p!Y8HyA<YpRvas
zE-0zVJ*ZhXO&R&>S2a-|u3<=ymUl(q&CO}~W|WoxeQ;;xmtFZEBB|xgCyiCu%1uUo
zJihPk@*u7QdtwI+0sIcxn3Q76{KreVR1l;E4d1=wAffR(avikOsVSkGrGcUOnknZl
zlY7h7S}f!3hX4qFZ7#&IsoL7{jIk(kLZY~BR@&*(D3?<){t_vqs2;f$G3(?Dx}cd=
zgwmZ?y+>(@*%0Y_0z4q}B%y1j*o-O(Aqg+nnt@^%pItCgZ~^mQUVs?|F2h8WJxc5|
z^*~4ubxc#l>^NK7Y?C=zN<As4X_<H$I`ABE+L$xbk)+j&v**{Xo021r+*!aoXl;$k
zP*^cBIeXMfonB-uR;F!B?(VNdezwJ463Hv}i+F#z>-%3gc)_1D-W`y3B;^})365q7
zqLRj#&gS`S^@%U9J8d^<UmfS1!01n-Tx?mJllfEllkiFg-2u5yPPQ?ay4I%+P!6CC
zFx2YspB{c3PUL4BbXC@49lVQ~NKVp<`p4?xpx)>AJ8p-BbcCBCiuYW0b$r!oOJ#-W
zX{Wz^W2GaSkL8*eXYmu5el7TA;>X0s)u<8kH?t@-Eeeq6D{d$BM@~3dVHY6#ROV`o
zxGR76|2^W7O*={9nAN~F@Zj;Wf$2e@`7$@x3?y(|3u>ROt&|(#e@aGdayCkG!DH=e
zZYP({UWZawT0CgZgr|LN@`*jnx3a(4(}K2>Ho&ORCG$&lJ{N<?EUXR-llzh!f-lQ5
z`sVL`O#%}F?~b6#+p>feJa719T9>=?sn~W08ehfd*gjY4N6(*P565_lXtOjsOo30$
z9*mJ>;NB)v-HYJ~o(Mz}bZb7O+zVS&CSMX?J(jNe<#r`4@`cuj*`c?e#=k%H=)`2*
zdfmqzB73iUFhq!6KAY&1cXU54g%QT_fUbSKvF{35?4V?<@`jihUA}hu6ZV!@7rwFh
zixsPBt@Cr4RW;B>zlI03pwN;k2~zf66hv2>z6C|Sf5O2<D0Hm%DX)xM8t+a*FeRJJ
zDp1Frj?uA9<DG--&Uh@yTy{%1V|1a#EjN{m@bIe2N*<*SkC@zcqF~4!-8bm;Jr}FB
zoM)_brpbd(YvocMlhIN3#$(G7?AjNvV7fcJN^5dQ@-J1(on5!iUdKu!S8<%v_nivL
zvNGtYUHHpY0Iq6@P-W*dNX62oeYFJC5#LJ#=BbT>E;hz49zBLb@wA|B3pxZ0sQSxD
z_=Fw48Ne;S3-d;>Lf)$wyduA+)KxaU0CoIUPAq?W<oy@ViJ|`}6Pei0RJ>0|#nWVh
zr;Cou*-6Ka_nmrykDd})GtGe}mY8KJb+TFtdaLrbSeG5M?ME78tVgp}g7wMgfk)mm
z!1^~6f1!bLwAU9vAYsF)B1VcoB-{N;4W|%$_qAUh67|!Aw$cRwLUaxnpra`hnbVWJ
zZ95@{5;}h<C8DSQHrh81AFZZi4JZi{a0R-8hZN^?S8^qm)c@`lZ4A@pd5On$n5%#L
zoB~Cbe1Sa_sdF;7=4Qe=ww&1O+6ncJ_b6Yzdhs9n(n;0lKXI(P`l*mE`&N1V*Vs_?
z7LQ0(_CiC*>Qp&Ui$kOL05v;o;z=}$w#R=xZSV?t+~H3OA!N+dzh}hjqA>Yb-hbW0
z6jITb-fGm0{MvB554>&`ILpXyW~_9j&@a~d<-E~;W=rEWw<+VTl6E@nYV$7kbkS9U
z7<!}0#Kb0a=ohKd_>$s)V&-=DG?!oRzIfWs>ud9i4<3*PZh&N9yDdQ-GLu$o>*2qr
z_)3|nRzhr(t0bV&^g&Ui|4_y6=#Hy98@vBOB+y0wpJa*|2CglG9cEa@n^@9`xzB-z
z$!5!%KSU4C*Wk97RQWuP6e~@Ldg~zJgG=eVKR9E8F?F~2(l7T=K+T!Ws|wnEx*EYe
z_IX4d^Bc8HS6QghoQ<A5BpJ0q9i_;Df9o!I56vKdH7eJF48|c$JaUCR#3t-flH%o>
zYA3t#g1188YM=!3xEJj!WE>To4h~ANyT%nnf55yDXa5;>9g{Zz_DZmOqL&JaTIYE}
zadrCfb`SIbNTn4wmGeW34^~8o=7xRY8GXM|gNw2|R;S*6W9c7r<E)u<I;Wn`p27n<
zu2bNO8@Ff9fzHx|yvaMnUVrpD{f!>oc0&gj9@leRdQRD;f7|O?!R61H=5AqYna`A+
zpoh%f9Z`_y31+~lMC-r%gVTJB<g!3&9pz6fOtf{2z6`KE<;sYa@_yT&nXe4yjq=3o
z=bc~i&5Djv@lR~5p*XPutN$gM_(<aa8u7Pa0MFph_TW42pbFfE9nvUTr|tcgj%+H!
zx6>w8xT%|3=mR;}jP<Q)&95Bm<zz<Mt_kyb@|Yh~<_g3!#e}?3y+VPeiM~Uc9MDPH
z|MW11$>G-WKdsSXu6a8N>QZ6dx=wf8>nV6;AB;i1lTUbf8ANoRuQ}I<uE;X}O_;U>
zB}|)r``-)vb0y{_Xsd|mc)NV=e~BFhNc@Y`ZJ0`*?_WN-|KzgK{II`W4IAQ}0-HGr
zbB!lUaxXGnCpp$1b5tiCiuBV={z?M4>IMo8je??wT*hOxeJ4^)Ps@?~yk^@gA8jd{
zl~3B*ui_zXUXRj?%?3?rZ&c|~he=tIA{%?iQkVX0=OZ@p9GBYQew%jUCI4ZkIw@R%
zxcFWA1<OHFTl-PC4uxXNtLC0V?X+2Hu?2|J{m8;bhH!U~)t)y6R!a~OIfo0=0W>@$
zh?Jjw4uibI8|+w#Xd?WMHTg~{_|BH<<>WlGF-Xc?eX+G)<sv2cn-h2c=@2*dMB@dH
z1OHCocw{~6yeYjsYQqd_#_V9-7p}366)PrpYF_lDFMve)JgTKPKDx7iCj<XB84U4A
zz8sHZ)No|W%$r>?PPe*KDmsZ~Pg1R+n0(7UIv}FSQD)Cx2J8usyx6*z84WFE$__nP
zm>`_qPFyfTOBiU9w3g2y|3Smyzim4x53wfP6TiJ$UBO&VfxG=onkWtQ=Ayx+Z~mhV
z=m#`RSSVtb5N&38RMy*1I=BBJ_ba=_V9px3zlW8dIGTD8R$(B)7T6*Kg+LxkZil{W
zrcosGKel6B<$_N|*Gc!9@Dol!Z!f9j7H<l<tE54jL(fv0jvQ0QD2Y|{xPk?!HpAz*
z0p6NI|Jo5y3J`oMK8l77#1tz1Y>#z8C8y`6ewuYAn6y;d78U$gw@DcHzp_G;k`&>F
z-;Exy_;&=4hlsQ2b?dLv{xzG!lKesig+di~i!^*s*yrnravBXN1ZaN=J7A{H@E>90
zaNyRQ;@Y&fL^A4nrGs;aNXZM@tZ6Elg&IKM9Efy-j@%p7QmC9R`QrYqRsM=q&10Jp
zmY=pbxcf_FGmNX<uN8~Z;=Z7_V)8Usee;QU*pFgXh{6*8LOreU-fI1*G-~4&&+?P&
z3ygTXujS_!ZK&`USpdzmAX_#g?xi=<7z7-o__mtNF<5E<fBO`f58(@vr6GNapeoEw
z0{mz-!DZq;(I?r`?vea!rh>oeFPUkeyA~WY;I7rPOskWAxTkM7WHxX5zo4c`BAU_F
zCb#FygZG}6Hb?lDmN_aXnps_T+3z=CA5BNR{rqo+^gO-(qLX_})j00XwmWVq<FNN$
zejy?@>XnxrSXv;|INH=ZHIQdjvs_@|i*KUhTUvw?2<eJmtj#*+T~{o^3KZiuxew`+
zzk9~atPiKN8Iz-446N}`vfpl*o#yAP*_<|R=6Lk9Ac(@DJcJ0FSU+MHFBIW_u#|7l
zlk{pY9>~xp^GQ;_S&DJ%uJa;#O<{?DmG`{9m^d}+J}3?3&zl^XyO{=cP9AIm4yGTw
zEZ!yBzK)CBFbPo@o1A4ox7|2dw*#3^AG9G0h!X2gF-_I-Tp;q6l4lx`az{ws<Mm%j
zUz|If`&AC<LhnrpMFZR5*ofw8vc6_rrgFKHmi&<Z1yJ+RW|YkDBC^2_iD!=8WJ>nR
z)&d%HJ*vog(wv$_B=k(lJ)PrHG1{S|$n6~ll!rGPxlI8o^1f1+{Rr)cN3wG@pO>U-
zO!ge=5>43aegtGxTdw4J9CP3NNhmyOir-tl^U>oPX})$e9pYzpdN%o<Vf%QrbO!cO
zf^x2n9(a7(7-K44EUWaJ-j;eNKk)-V0|&GUO_e5(nn@%dq8_`<8&etL`Lq0%j)b?L
zTX0gn#tbAWFxH1)+~Vz)>H>>qD7fsVvq5hV;yoUmWkmWqvE~u1qvo)hP4y8ajEE)$
zY`3okKpZdzNUBMy=6(udR*@Nq{uFSxOn41kAm<0L=jDCmni7GP!bSy#AtWUPRkmhi
zwhA;CC1oBx2e9$DEoId+3T24-<p!*1&RbY+jPdkH$fnj==ixez2xNBkAeb`ZV1p5h
z8^WxR+A<9d^ab87JK)X&7_Xm*35KwD=KBe!nq<xIH?oi|%tswFI<3FxDSi%@ksJE{
z+u|wd_J^-cnBs|Bq-gDv^u3S^DZv;@hUZDBt9iNPMfY#FBM)!xibbJ|wLv}ErMvNY
zx#zORosdOrG{ROF%IZX&hZP3mF<*dk=(x4t%9}~E*&V5!+|iM=FJg{K0{Jb*0;i;Y
znI3(1!y?b8JSnG2A8r?>eWt37k?*}C%Xdz~t4BwFI7~Gg`}t5_3s9|cuq8DKrZgIR
zAEq}}Tt8(!WbY)&c2>w?L2@pbDjyK2VbkR6OCMZ)lGiT&=y`#DUz)s^XeM(|-F(JQ
z*?kw}iVXL`R;x1~BZAxDdv|c0*9@Cw-W5qHyv%i_8s!gV%~Jcd+!N=*J&*EH9-Zd2
zp{_-Cw|twXpVTZGj&bQqkJCFPe37wrTqf(<{6|X3eiFu9v~Nwi1Uo9fc2|V2ydEIY
zQLm4GJw)HWHv+5KOa>4}U(GArf`+|x6e^=|e6|O%T<7myF1Faa1ojlJS$|y|)y%ny
z@~}xvx_XKeoU*kI4wrhpx~N(OkivsRpLQ9|4ec)wQV#ltE(u*#DD>L$N~%az(JSqY
zpIA~)rKpMP`!ua2IIPGSw4QmmFx6awmZVABlKi)5P3lvCuD^q=P0CzXJ-SZ}xqZ8-
z@5sI(G$4P6DPlP-Jb!`d{+L#F6^_?p`bu`s;37ou!*J2rJ^d-bxw9e0lVU5O_}OCb
zL?!~UiGhF>Emd!{w!%H5-HX96=3ecc3E79a)NiblbysJMy2@~Q&VwKe-N0mS4a~zo
zn#|n|X!VADQA^w~X?m=_BTVrM`koPrhWE7%Rg)jv<{WK#<p+Eof+I0^uj(BxFPu`m
zWDzgQDK9aL63p)pdxRGgl4w>h{t%Z2BRD;~YDL;!m(8Rwnbrncr5?W4u3hldsgQM4
ziq|RS7~OyQHBuDox9rr6n5NhE`8M{zRE0<7B!>EP^XE)kUrnT^L?AQs7Y(OF`i8Q=
z+Hcbpg;}%Q`Y$U<?A?_rNrwKv7F)QKh#W*%9n7?f`g?^9s(rsH=G04bD`%&f*K~~Q
zHm&DOaL4yOEH%odh~C{Z8Vi5)@B3VuPN(s*z_D2KLKUWU`>2InWM`NnEo7`1_DRg{
znszaOe*N=jqG%wu_~t0l#Y+6L#i8VfqLpQmeWcYNo`hf7WIP!xNorW@*95sA^FB0V
zoB;)cUNz7w4cbO6BOdwWyS(Pl5eII48I0=aAJp*tQ?6PX#4LNPkwOAEe<CWx09So5
zGjqrLoS*4?sw4S=wR!|i`2fvHGCdZ-uAFDIVEDJvRhPbqXJn?SEG>7~8!e`@`UPR*
z5-S%m(h8bqpXff1BuSGWA1Jo7R&}qmZFI{nHO+M5Me|i@E@4@yu18{Hv&Uwdp*D_!
ziy^}TBE%$ens1TE>G|&QaPb@s!F=C1$0E7o&<KSz_CI%EoeYcpX#fs*l;^{U*Z-lD
zR_iQWCA50?c&^9XH?GbnxZ*74sVq?4c^zY(3%*9W4_N=a-BmHv*v@rLMK{g=iV^OU
zzL@g7&#`X&_B3qn;99k#(Da_40a0FTwf&&3V}JfS0x{F4^ROv({LcBTC&>%ruwKtN
zel%Eu8-VCOg}uCUTFJs~1=xm@zO&gh{(;M%-`FS_`&O^*xiqncn6nk4tZ#2cKD6+C
zQ&pJf$d)IKGC^nWC}pVzK?GEn{nK@T4MdqlSCf`DTC3%FM9emu=dW@fkbd=f{HM6d
zLdh3t1JKI#hn*p<y_uZcQwNOL&svEMWA;DQ2;Epy+-}5p*j``O1V_7*b6U-UN7*YI
zv>Eu9l=PH1b1}|xMY_2gU8g%i2~>xS6N<$p*(~nscheorp>&u8#}_B&$C2#@Zc~{D
zjBe;tBazYE8_gJ{QhrucMp`TU2pbkx!hrPD0GM6NJSJy;fJ6gw@H%j=N6M7fl(m7*
zMY+BgXTvBs`O<kqrbll~pg~>7rc$6d>Ep9qIN85k0O7Xxn;hKc)-Q%CmTq24w2OXk
zMwkPyG{1}1T?P2u9z9wj&b9J?Eu|Z(=iVu->|V+&1RXKN`i<0><TjKD<$nF_k^h8x
zcxI++Pu?TtxADHt4OiMMP9}$?q3waS0Y&5B#WwzT(vh8*T{lTqnUi7%eKEr#;ga9x
zTvo)n^7=!J8PL;l-k{6Pr!C;NVi{@fu$r>2c-tJ_sHyt%l1bP2<6CEwr?>Z}m-cew
z?v3nY`EAC`@PWTzQuR#2(6~}ZQc-_itVY8lmSNcID2&_bEUOyk4wC6=%rP##XdALL
zSdm_mkW$f0x6LOD1S-8v{$XDDnY+wHn)LkU^T^0N&+)yEM>&&GO<=lY(Vp0f!Hwid
z;GuwbThG=X_Nqa^p`-W51krephWuZ!Q!v_)X2=O%<{NvvUpjGFrzP{ANKKm+G0$kA
zd=7}O{tM{_G3%w#sFo9iW-?RCu5#$Mp}i8d$o~atNq|t_#BqQ)dpUrm)G`G&$O1sP
z#B(`w#}GftL^$^f<BnW>=Cguii2BWHAksw~DX&2OI1JyJqNmy+9Uec}42Ezi2KFV{
z2s;H`C?gqhe%^daTzFjfu#M?CVUmkg%j7uA>69*%c~a|MRW>?*->SekUbwOdb8ekG
z0Y4mA+~UZGI9km{I=SV4U|f>=_1kS$A|)E4Ai88uR=8x@HyvGDQbIQT_I3JrcE4_b
zDvOGP(Cy*Q0ekCyN#v2%gS1>1M%AyrK)=x(E3xx5r!>nACZw@4Dl*#^t)TR?^i58O
zz02V|ooc{HK<TA(MtGh9ym`B6tjiDohFu+7ECyW}(o#i?toJ>oKN{F=i4Bzg^vSm~
zElysibM}4Usr$h<x7ZXDOQ7$BPN&Ie(p8ns4!rhaG>l<!zYTa<YT`Z^I<AwrwgB#t
z@ZMKRx<n}OuEtKQSb=J&FfA@Hk9U>?bjVKn=H#meYe;SnMb4f4YEP8P&6Qj_@>{Bh
zmP}Le%uEUOYgrpSTqu$0VOU=#U|#Uh8N!2pGHYHUq7mN&hEu8l$v=m1tLBh0A5p*e
zp^A~X=D^CXgonB{NBQD*ly0O5I1yNl(yJ<j?{Hsw-fW-+N3*RaNFbE{nnkktS!Cea
ziYvQRBs%92_SG=}SR2u(?CoJLcHA=HwEXDfQke5GhNq(1J-rCS!M6$b+WU+vez__i
zp}TX_Z+qn)IkolBAj=v-lADg?R7v6ugqEzX-Z}R8JDo^YvY0WC4Icc5&frpzJK>F%
z;WbD<{bXq3sztAfPMD{%B?YC0*R7nc%Ff8uCUod&VA#YxVg17y2!Z&|uIh{Ju~h?v
zPB}3vR_g8@@?naTjZfFg(hVL$-XD(Y%1m0#I;|Ao=wMHSgiukmsH+S*DQXOhQEFwK
zsaqY6Smta3q$(fXIVr12NoDR6=L9LmA42B*5!;fTpH2sp*2(S21GHexVE14=CDnZ*
ze?16^`7`HyC)uxpzy-8sv*qrH3Ec4eKUCLYcGj(obF6EFJ$0Z>r0fKDDd9H-b*v5A
zH_;x-;d6TXMyVTFrSrRKK*xR3^~%yCp;ya!D3LCw{*Yi!zOvSDm&@$r(?ZtK;C1di
zwBrmv_jc#Q$)08yqb~>X3eD!<3CLmj>*>>Upu9Y1lz{wR`S3KSPbM%-Fl~c|=!S5L
z2;-$+kDVKF8@>SZ6opa}KkFL$h#{O*Nu21@3$@;Nd18<>B6AKGbAw!0MkD=ZrevDM
z=QifB;_D3TGMq+#S8^rgt9p`7dUX#BcspER`8k*NyJy<*+?K_Du-J`nezDTN^|e@E
z`y-&8V{rXb%qDeSo_V9nWUkH73FJv9UAB~5DCGe0o5CWWb5ObeVEW7-{;Mo-^>F$1
zNBYaXlsLs(D!Y_hnz!i(v5rkYK3L{$aXVuZjP8oWlL7O_)Tm@P81>(=1Q^IDSThGY
z^$yIZ^bM)VyT30rGZOO+(P*&#iATm<9os_|*Z{H_)n@os^*a+cYLs<Fm>D&ii=$_#
z7D+HT#wOdi8s4n!_n1NCp##QTlMl4MQhIM7zTf-i$WXgFz^Z39{kL)HbI}Bc%`cvj
z9+?)Q_43|v3#QStweyyAkx*nKnJR=p&c2uZ$`l};B~8tHgXuvuP>-UfH{l&d-awjF
z-9e8z%zD+`eIOj5Xzqzgu6cV6=C=<I3>wqVNLbd^dwqJ&@i4H|o#yT2RFy2g{|&}l
zI4n#j@t!FCu|ryy=ObYD*b9efjqyW&Lo%n2TC8mm#p<6oeSbb4(i)}wHdVEvq#A1I
zG;|Cue@92E!D7*15h8!KHzTI2(ymQ)DlGr`(;*dWeY^)bYWiL%w}exWww*!McxxrN
zy=Khy_yCJ}!Rm3vbvW7k5AmT3Tb^U10jU!B(*xg1wbEEfh}=NVJci9K$GV2Z%5EjF
z?v=jj<5rtxs{9BHo&HpLAuj_`?DTej_LYpRxmQL}Y7QFl7E?q6cL9-x(~fw}{7o5G
zH{=*L6Qy|@*P1USza|Q)FG*j78^LdmUtq-}K+TG-Hw5E$O+!_^giA4H&x7s<!%bnV
z^o$u=GyH?qJuET>kh`5EP47e+R61Oan|~j8l&rwU_2@jkh?%b%IsByb;VXYfteJB&
zi8;2uen@M??JtE)1>rYb(+}n9pD@@zDBPP6IQ&-6*J2@q_n*jDAGmvCNn7cXJX$d3
zyPYhWfK6!G;{>f7jx&PXHQ$TCQ^Oj<ixAyO1n<F$=OS#V2isY(XzIVhV!`b#kJx8*
zBm2X&;J*~!k-U}oR1>fS^9Md|500MT1Ica+QN>S<+IV{#)u=2$|K@>i;r-%`Pw#ae
z>JavD7k(2l&!za~O=XhK=7VtSL^d7;3A34Gegxa`9fQW#>ndWtBU+u;haY+?f`jw+
ztX_?fn*N$RGA17h#;T`bU5N+q6w3mdb;?-l4x)6cal~<;qt{Yo>ftA3z9U8qVZtbz
zz8K_g+8Audl-iiJlytv!#|k-0@}NeX;<umbN%I#KdE(a|`0(crHh&|BJQ<oR)#z1M
z#IK(gm<P)<SelXzis5WTHca7scDg!*Ah(IEtZX>^=tl22VC)_qQ8AhwMcI2F9+&pJ
zn=vcgqk@I|q&`;G^oM>;Q&*6}2Lwg56nxbyaoHX7wrlnMDUlaY>;+6pPh`z4d-Kgb
z`tp;gka>Nl1f=07%LJAKyh0g=?M;v`*)CFNi>{+|9t-P8ia1As1L8#>*0+X_>Sn-e
z#_igmBl88a{rC?LBr?rZAJf|<44Fm_nu=gCPIhcDR@`pJFSo!$b7x^Asg@$@u@7?}
zK{<jKa}vJ=NH#(~2by**_9D->oY9T5D16U(7=$6z*H^@b&xM(W={ihFJELmQZ-Sm9
zfAg_=lW~GNIlf@}1cNFw;r6Elmi%J{#y5<eXAyp`Yuz{!l=mig>femx9(GnzSMI7E
zU6kEg_Qb1KH0Dx%xdd238w7p6pNoa43r%fzn?#BgxGM|JCzUK^dvnKS0=lrUP`Fp_
z3=zLvv?k)Q>ms80$tv##Li9w&?p_MqR8e1y2@8IrtEF+dUg5BC_#j|%!MwP-h4Ovm
zij&G*wuH;%YI_U7Z?17+N~~#eZO^nXpQvf?ElN-h7mcbKOyls%a@w}V7%S`@-_T}_
z!>I7xvu7mmdLa$w-xF$L7ufXuuJPU2*E*OQPFa}HApFo4CY58kjW#PK@hSQ-E7z;q
z&iMyZR=snaA3s;3+v4#!qWt*-WrG-F_45apteq|6z62}Z+vkW&NzPH5ByLq&5|k>O
z!qRV{g0W`1b|HbnvgO<$TL1JK@ZrZeOBX%oF8lCh4mFCA@!u>pjFp1Ngw2kYI#w~D
z+&M0Oc!@&90^WgHbZy(UYOsT!y49jC8GE+1dP`H<8GX*+2Frv-=647W{H2;JI(Srl
z8#XHl#XHbB4bAsY8jy3C))=Yfy&c&ZzNwp5j~oly`Y+Zb&4zQ$AJ&EDEU-v6h?;6h
zDX^9RQf6w58A*XI<Qq5x;Fn+)oy{<!n1CMc4T_$WA(uKf><SYJE_0f=R~WzG)E-0_
zONz5_7puof4CFZxgtJ53Rp9UNt%kiRVHchnth__A--m1BQ7>`abg&xx<G<c9z~{mz
z21Bcn2@F*^l2hJb&z7!P7A3BjAplAlHmu+W7e{}@eFOrXKLxODMD1|q-!$nry$xfN
zZFeMPNYCt$Iyw{Xt~VMkM}xWZFz!g<Rr0$@0p>Iq`&g>uh~h|`h%YeTlw)K*<Z(D-
z_YmW={&)sz(!k_xc?$rV>ji@y)JrSo_{Z%P$oICgEUOV)s_ePXXnWF!&pNi{DIVcC
zfD*xoFrb!X5{vgcmg<Rt=q;3Y1RMH&xiC<Nw@!LT1i#@+^0Jz7L8h#|^Gk8seB^CU
ze4zPd4cU-w4->%5D9JBx93zivLq5ci;lV}U=_=d5H3|OXj$!wTR&g;4{7VlYPr_aO
zu@r+DeiXo97EF|@4O5AI)mbw2(af~y9%kAXF00+vLyS1jj;=Br9;h13HP8OR%+DHL
zlM_GqcW}L7NZ6kqa{yxvhBqz=#<#S!ZNVU{yn?ZGs+%J8jq;V~kRW&sU~<nJQ-Qn!
z3{3<i#~zxycOwLWf?LQ;FAhIbsNl0FuU=svVFGnq@K9j=6W!EGX#st4YzQ=+Z%~J}
zKW8b@VZ*adnR>9N#=~|I{DeyS*XvHfMq<n(kAc-!JjhXqVF~A=@R!@uamVl8^d|@G
zUWjkVZM^Lv7VVl*o9x7}!Bq8mLd>f>VpmE`K(R33PTR{K(wnL%8FH4~UDWv^Ey=pX
zItq}n2&PWIgJquO7Sw;Sf_-Y=S}*A3L4E>jX>R&|c<F|}fvbkehEb6^WQWj?$5!>k
z`w_|sQt514konM1|DDYBaPdHYL*-+J-wg#8?hj`7bOQ~5DzK~t%dq-JL-18Ql=OJ3
z!PLOU9%PuxpP?{NCJI<%WKM*NUScx+PocqC9T;s08sBv01=-fH^{YP<RHIf_Vx-t1
z5#@Vz23G^I(sC|N<%^C;oOhGPd{U?%D|&^l4QcQ;hLEi9VNoH%Ob_RL5-G-;1+NV&
z#jsa<YHCPNX|P&0&o`5)F>#PjhlI)z(1%qcalXb=OYvk?r>s(2#h`~p2Wr4{S9Bwf
z!HUD%B<?CKFmJfgP~=IqQRu?c!#CtzI3;t&8<vlzw_d|kHjelDuR$a?eDWBaB>*zM
zTwqG(s+v>$;8*w?7&;i8xli<u?5zvFG7#$AgQ*Z@jFiErzV21d9^DNYK#+uFFKNnn
zivT1`4n~3?5Jfj!o&qATleEh^89yzUvmLk+blk?hkqDJqY^pHBm;^=F>$%f7+NOzn
z<2w*5x2u@T5Ao+gEWIqXINW>qLLkP5iYdk!sw!dLEX2T67=U;cK!a0?q$>H$c=}`_
z*oF85vZ+sz9nK|04BxOJjt~Z?32;J|mtXMYGQ|h%K2YYha5};ctVR+AsQ&zck#aGV
zAe*NJY36xtZzMDNhSX9AuJhW<8ga8mbx}UMw2Lcg%Y1hh6s&Kvnov0M@++9N;pIH5
zAswoERUwi)t_Zmq1RLAN(V-TaJ9R0<RJ%*&7$6o*>Y`##13UQQEy6|&MrF%f^`*+j
z604o_BzY-ZP1hp7a%?4csiDEH@L_o7rDxhzvS5i;8XEtELZ5Ji6|}P98HIUu?1q2H
zoix@w$-E3*Wa{DFpzg_T01WcxeaVD#H>&zLg&e8}u>!Owh@<6^H$lByItId{Wp(59
z-)5<4kCc_FcW9$gNu1_Z-uqdOpFmS%q_MbNc)~e#pZ1t=$7m;|8-G4suqY#4vjuw3
z=wSdm(lt4F(AN3MyB<{+?JJU|ArD{zC-8_h+Bq;25nS}`gUMcdxmp#MsvbQCyu-d#
zjUp2@SKA1*X`i>klZWQe8uWfi66()Ykw!V-Z}Y9gqPbNwNb~9d#J<*+X$pvrNKLS;
z3==RMHPbNq7TKZVv?GOM(vELbCzMz?4@*BfZa>u|ZCh0!C%m+0iYjR%J%#*b&|B|p
znNt6X6RLlOi@Ypu07zS<q^mQ+F7Rlw@r!al!R3@r`X{Gmy$HBf7_X{<Q`&gPHbNO9
zbDHKhQVb!n9Y&SpRM>7)0HJ=jx8joAzn6kWo=6NZt6#cC_ulqHARtwb=~J|~Z5x4#
z^z&jc7w16NO0;~Ih|KsZ9+t@xN4%wA&h>7s`4W?59zBU_coV(u4FBH&>>t6nxz8w&
zlx{UOTClE=db0gW^pxY*NY99h7W~j<xYQC?Vn?K8$W<nDE9NQ^pq()TTIzBv`G213
zuZNS*09x4u)l7?t_g@>A@OEBh1Cv}?ZahD<kx5`TAhdMN=m`rm-Q9a&U*+b&k4wpT
zvGs5)x4gFgt^HD|!}R#yLBf=}42H_qTcI9R<*&pG2;&NYz(gdo<(4e&i!HR_$Gn5$
zQ^W`DuVHynWl70_y$<MW7pvMc7?ycSJR7#7r9@Qh2Tb+54TlGEz0xg83^!7vt`ZOg
zWTcNxnMh?3NFyv`I*US%j;dHWO7C@X1XayH;Zh@V)Q(@&xrG=%_QM3B-n-<{8%2w)
zdLwX?)|A(M#KgF=7YIF$ziRAxZT<gQ%gf@g&qg92I}@epTNwAiM-*}|$6P{p_wk+s
za|T`IYV)lRpMLty%GBA8^(7^zRX=GRPRfzRdpvdb`!PJ<Wlnx=_Sd;BTWZVTrSdAL
zc=}ZQW6bEA3H@8uCFKA)xX{a2cWyj9%b`N9#Zhn2gwu#%xln2M45@D|f>z}x=9kE>
zd$}aYPSE@7wDq$SHT6yvok;EGmD>gjN@3{U0r4on(mZ(dj~(4I=;zTiw-xL9?8R=!
zO{3+?!0#W9jjAIdb@#hT0T-haBFBhivXLNsX?=MJ<1D5`XKK5;W$%3EF1S|XhFi2X
zz|_3e0f0~X);95nmBsSq5?@rx5K?mr(K~1d=9D~X=jUltn<j+_UtH3y;|cS2_X_hR
zKfzPGNBZjupTkbC_mDTgj-oN+7nTfe2rx%oAR6dJx%+X6h4iWykIu#W6v_3lHI|`F
zzUD~eJIh31w^G`P;2&Md>W+6C72u2Xs5#EmULHXB&10_)n=7VcQNM(ZBeCXUxTt;A
z5&iz%b0+Yq+4Yqw@}`U89nM@$7jK7k1AP~tQ5=^6UH-eqqLDih`ooXW^sg5s-XN~G
z1iJEv@l^T%7amZ!Km`*EdfLY+Y<|SFy)=!;3o+}C-OS@}A6S=bz2f?#lSR=hGl1qc
zox-kUX*0+8Ce0;Q)Jl!HexPRJemRShCiUATki{)XW2b+Q5^u3KmiJ}CAqf@nzQ6d`
z_;OL3WUvczjm_YuQ!vR}xw*)4|NIW`I+9y;<bUsay?yw<?s{E$68PLdG}W0oeP4Mz
z_eBMMxgFSauza%Q*m+SAAi2*X^A0fcci3Kf2-RNI$^RJC>E8X-->Y#Z8YNM{@rcGN
zo0^Ghe94i6bv9p7T%@9rVq(noqX3uat{Vk*Q0vIP04O&&_4>>uvsVo57BkXgf%X4z
z_7*^Gb>H4FkOIYvJCst~io3KFcXubayGskDEmqv!-QB$qT!LG%;uHujZ|I}<fA5|5
z{l1yZbb3fmPWE1B@3nri&L#TS8ir-npkso$_eZy;ZFe<0iMsQ3>8*Yy-h?*`$n!C!
zI`BH4XAUcPo@HsLwgxDvSdeO6wi9fx8GMU_Y<HI;4=m`yIADF+uX@raZpwWc5BQyd
z7@ifJE*@DAfG7b<g8XmaF51!79IO_7lyZ8eK2VsR7JlRfawk*ys1qVH!!-gNvsOu~
z3A3jEPN;*uIsG8YMbax%)F(Hm(_T069olLCqMVrO*T`;kT&(d3+h`Gq`O$oaKg-gY
z3xY|&T(3^MR)hroD4p{oSFFYb_hCkObd_@?tGdA{Jj;R+#N@T_A+@)^YEm6r=}ue}
ztTsu(3RucQ?cVs;102r3s4BMnFX6Y7rW5r!lX&WC(u}1wkm8WRO<w-j8`GA&2%!Bi
z;O*fx;~7tBg<i`0!H*eJ-#2J~p_OFmN622AG^|_syii|0)*y{7cT|dicjUzU3Mril
zF}=i`mCkMPY3AiF(u2R3kJ6JOM34o>bFT}IxRC9hE+K9_7(S5YU{H>Jjbky4({I{O
ztomTme+sBZ-s!pJcykH~Q9$Ks?w>1WgcRlD(X*f`eo0w}TYVMoU84@I1$dY>Hq?Oi
zuLO6c`S8>%7<NChj2Iz`%`KUiUGn&z#y-=iFnjF@Y5;F|4z+ert?$kndJ1f(0(Di4
zwr-|Q^1}}<EAh_b?U_=ld=p*^oHgf`Y1_-yXVjFHQc6&jIs!D)*N=8g`ylc5_Rnc<
z3;d`Mg`LkUkx@}<7{0HZ_82ma@<vg-oq6@TRjTw^P|;3ooPl6PDykxE+IbI`=h48S
z12{9QPf19-5&!E?aqm};LB!L~YZ&6Z*Q>pca{Hp=r7$E@dcmj?exE#%;4cG~PP>St
zKA=?8FfJz1YqVA`-N?zkGGxkgvWX+U?UUxDT~+a0-4X~V=o8rDE>?rT2gMVqiRP*a
z-ret%LF(%51v_j-Os;{p(GE6@i)7N5A4Uejuh{&jFn1?#WZtX#05d~FKUihCzWtg(
zjhiYUfbrUYByh)xDo$ta7S~OHYDU3Uf{+#)VS1}k)}maOTS})!iYUBI?A1bD>^Yha
z6=Qw+bo-q^<KzOIFf@+M7)%zKDd`%;-=ZL{KKqz8KGU|XEQ*O*VxTd}Z5PwaEmRAi
za~gl_Qj=<8`K}S0erv|CFR#S@IT&%r%$)eL`f@_1y~QnwY`!QhXu6o9C{(?faXWPk
zG8mXe@yXjA27LYrTt*ktp3Wp;c7IgePE8Mp3l8jPz&kH@;Jjc}Jon<PpkTmO-fF>x
zM`Hsl&P1Or%)~D^82J%d8_H>=Ky5j|X6`Sj7||5V`-8+C)N`S3HmY*s5Nle>75ziD
z{<MZWf=1<#kY|Huni9lBBrnkn)%?7Vn6s*n7;85t`x1|OapLnxWJfL;*C7<TQ+#DV
z*a2J#Id?zKiQNS~cM>LQPJ3Yd=a1pEn|GBFxp$AE5npbQLXo_0D;?i#odk|<1_lr1
zI_pX#DeQCm;sV*9l2!8$5M)1v=25b%O@7OhighdkGI|;el26rY#9vokOg2|eQN%D9
z-=rL;8dhN8XsP`myKR^zbfPHD5GE)eAUiwc^ySZ*iGA~I$f*tUV{l+neT46VBy^yU
zn2o~sWDXMdfyayFZ?o##7<E9+Lkb2hT3joqnQfXgZcMN3?$Vr$H{1s2sut7BX`pE-
zG^dYwP7oIxPjzTqBIT=TnKm0asdi~09$(wfx9C<C^(lDDRi<2}$fn&#k6)|$zHm6m
znl+etV$5Jsox1Bo>#9`IIaLM_FQ=^?`*FL7VmU>9AbH(Gl+_@zu0uU*^5JZuN4%0Y
zKLlYuew>vcUd=;J!wffr@UuJp;CGYa7d%nQ5iAm!_AK}{{@~!f!*M)E>fQwm_sdAJ
zeggL-N%QxO7y_Ht@ob7vfkJS-t}xl-p)PVZ5w`Bm{P~3U=J$hDD(rnxxhFb_;UH=v
zGP0|C5JsjPx;~ch*-d52(md8HTIsozk9(#1P7hKo==kaPxER=-j+0jV-=Olg8|wuw
zdV;taQs*k=<d67i)wH?Pg2AuFpad`?LjHY{#ir0dWX8J&-E8w3qSGbLG9WW}xp_Wo
zKzoFz!1|bhyEewOkxCeHi0A#V{PD*N3wjYgkuTJ=x;5TVE~Dq2w7apwb$MweXni~N
zH)mm7t(FB`qZM}{yJ;Tb19>0HRcV<KzS;kYM1*VGEuwuJ$){~QA|Tjo`f`De^N~DN
z<&%EA48m!HG_BVq2~AjbgUV7e)kNeSCvM+iI9LN=6b4W{o(?<7FvYaL{H9lP(eo!r
zi#g<wX%$Gt@}+A+DLQUAGQ7M`r+r_$Ttzgy1hiXfGbnUqfy*l64ccj5rN&rcz7@XH
zj$qOUs@IUWZ7rgl1|7{P&Af42U1B2L+Y4rcf;EkZ$Xlmr65%3>+utxKKVy}p#<rJb
z=dt7TaQoLVxf(94Z_rTjy_&8pClibsc^#=t#i6KAJh3vj9WoJHCjCY)g`4t4oSFsc
zmX~h1qaN&~xxzKrYnvCXgcDP4em?vv;w+f}JTDlZsWMP?0d1Nf2C-BUttXS@PI>b|
zyiCsJhSq#+2Ac+<kw*EmiuT)Cq5?QCt#X1ZwhC@0-Se+UC^-S!%%`PNo=4h~s9z$O
zF~G`s?(zv=Ndx9F6x@wJLoi^$Rmis^0x=lY>FEprzaHpa>33FmSCKO_wnsey@B&>(
z7{o87rosUn63FM9(t-v!V{?!F=oANd1Hque@?X)Xx2w@<@pkXg%N^Ym<{>{?#Fgx?
zyA&n&b$KPx{cmeb3%vZuk)ES>r&lF>uc9gmFLQlYrQfjfnWDd~>BXpy5Pxr*i-OI(
z<$LP#$?Pv2euiNrNxTa3M}o9Q;Wn7!yH3OH9-s<GJNr^>A*z9dM@rbNEUX^a-nn`j
zQtL!DXob_s2`F-)@H3WVu+ggNnmf%m$)Rnk*YNdx#7GHb!dwKm3bdwWk@z*7W=;o$
zo-~H05FfR+)7D@bz3t}`q<OLHant?YQlGo~rzGC?=V!FhOX!~~hqm-gOj%#yrQD8t
zfykGb`k%9LVl1KucVYQlUOPrd+_LUf(b?`ve4j6~=G8PGX}#P&j=yEGqJBIOV&2c}
z)t@XXKetv>Dfcc}li;k0AR{HHOahZe_nryA7^wDg-6mX1C{hq~$zeHdrK`2sV8^hH
zEnw7kq=_#;w={?ijDDiDqfJ#7tJym_Ga&ZO;^>{>7dT6XX?-ewb#}k7X39S5WtDw8
zo8z|+f}OlDs+uWLAj){xV0dW5T$-b%!yTOM5#<=F5)xtfxENU(5$nt?l;kyN-dX(f
z>{g1tG$;Sv;&vPUd&5)1&O9&5is;1sir(gV7Bn%gm$Z^U$eK`~HqLN@A=9CyjxZ6~
z{*iHE`)f`_2!8q5mO6j<QJd5UO3K$}S6c{STql&a_jvQ1yK)uRs`~rrE$@j|IF&8n
zq*nvHSX!CFXv?6en;AlSb8#Hz@R#zIz$={x>uxR0f#g>kLW>s9<cs00_ir0q-izdk
zWc|2i{Xpj%VQWLXv~6~mK)e+wlL8W*vyKmpe(dHFRv4YLDL7B%;@q0gvOSpiV*XfA
zpky~58XAll%-_{5vpdy2!Qk4ND|XRlco>em9^^f`;;Bz^Gp_A4-US8sw~jDodEYCW
zJv1;}@V__bvhOQ{I{V;3{uR~?GnjNq@$26C6)|ZcGTkUstunzv#qV^o$Sct0p%w65
zIIQ_Bz<<Bv;&PbnTg%anw@f>tDyrxQs!P>}8)kcMq~}P1!5ri5Tj9>03fEj)wJ)p(
z#yg%$SwLu|%&%|cFjPs>^*!li%*4M0qkOZR_J@#<ycvK7)jb6jzw-GI;{+kNTM<oq
z)Qq%XO@hoXi+7DNlA1c~HS#j`Gv;%8>+Ih~A5<vJPl~gVmvhn@aOkac9#rjT*v)HN
z#M1nKD7lAEd64%&*ySy~wEPIyYIYJFno(M!y}!-zv1|~L_&V;p&Wt9>5`RibDet{|
zb=jOUI&SNUu6X{XSFR_y6g3tXV3ozhkPi`RXNL(DHHy{ZN==D_Z|n#z@S*=0jCDR%
z#s>N%u!--c+kPJEeNy}c*{&0w<PKNYacI5+)p+c8u=rMl4VALAdeCiHbx^zVUcA%h
z1+^31|B|4pE96k-_3shg%KCoWFWPQK2iQsu>~vb@KTQ3NR{jNAW<Y@z&?9^GMP&q}
z8`RWMzCkD3vG27mQWN(ixzn})K{h5v>HYv=pRic+H;M#XT^OHK)?2Pv#R*Vqo!FIc
zatAiI(mv)el(eQ4Z&5cz=Fy;<{jdw~$J>nhldKB{+molc>HY(9JRvyGQOC1|x<uOy
zzH!sCJPMH@`Qw#eZ)+Rk0yE#SWik)oo>uK;{sXK$VIxp@N*r@PZq|y(CZ|=1@KEE+
zn@qd~tSlwsKRJZo5R3?6;Y(D3K*u-F_%7FsRjsG)6DK~n17ghrnlujeQ|%331p)bq
z%wI`3>ZZGseKBDPz>H@P?1MS(a(8S-41o>fXnppEyW}Y1j0h%JT^-u1!HuT?!~x{*
zl<bS`4<JlS0?xt>z$|~ndtLZ6D35PC2NknOu6m9;Eg!FICe!0Z5w31GFgkV?22QV2
zzp_8NIR9$l&L(*Xw@8rw6v)CjcIze+v%$1$uQgdwoBL=aRggV+PryTS09OnX``i$t
z(`7panNEHSqcc@>uAHtlBk7?q=`-c6d4A0*6wS>u6k)2xJ!ZvsHE7Fzqjcc=GSo?>
zOof%8Q|KG;x<#8+a!N^)@Hy>~P%XIsyw!;{^GspOYo(8*4C1f$EstrxT5!LqF08Kn
zjK9ItmTV-yY!-Y2b}PHAN!I~s?kC&sD-jlcL)CnIPQMA`s`zKL{MUbQW1z9au<bmf
z^sn&vE4UQbpE9RzVPn$FzYaZt<bR&wfBPs-2JC*bs=;X4F&KQRJ7u-4H8n4KmV6Da
zt1#mOCAxar$u*?6F!b<2haec(c@t`2gjfVn`GY<BbLOEj(%C64lV%%#Ru=E<3?ieZ
zg5ebiw5cKu%LqA%(b-kRg0Wi`<%NBH@~oRJB%FN_qef^^rZU&QI@OthLZ$S+CZ6>S
zQi!T5iBK9f{XhMF5lli4GDzbXW84UDJXd=+Z&}fj-;Rn5-(^}Gi3O66v6Q)Q{v~2N
zVxt&Fu!P-hudV09Gub@+f>W4E<jicVBlA!dPwX|a211xyqoFY#LgZ;!AvWx%?{q;>
z%_{q;pgB<R&uBphO;HtLQt_89ew-!e{jeBcjNi~l=L_W5FGi~YNEJLZKW?6%G>5uq
zy9+x+jo!y~I7XFAK}X0;L@$Eje1nPnBg*NCjyjWoCR|yGSZXr~85V624bsK9$jtal
zB=l1UfCAcW;*Kq9JIBTsN`pOXLMSroFcMJE!DR2>vSswJevY>WL_Xu_|6cjB0;0!W
zQ1o{w_(ShM?MDxJzj>K<#&)jiw$vdZCX@VfSArhTLu(K;1&~^{%8HJQVK88xr>VGW
zozl-Ya%mJ`&CPSMQ(ivV(Q0B)qxNajLMP=1vh>6(B67tqB5*{(#^6a=iPsBO=zWy*
zv~iHJ#Kbnry>F+GO24ry$<5eARFl}9=8;}$Y03$EbjlfC?2OEcV0)<Eo^0`~T{;=1
z54l<CKluYQ^Sa*z;l?CL86yjqF7#qcm=Oxn|FN2$H*6i+H!$vyyPu&*EageX-mRh?
zZb1yNH}`?JDBspB!=UW9<!}|Nl2Np8vh#soZH<b<-b?ek&++&n4RN<ZS}Y*9e7?=y
z;J>1L4FlDK%lEqHd2QrYbReIi>f>ca#AGdcLOhT3S+m+!Qi%>|Gs8_XlIAfwW%W^_
zd{0S%`6#jCuFc0+XFevC&RqDm0oBK^>K7K*Lp9#TCyOeNt5ZC-R(F7Rvgf;!p%TU1
z$2pJpHv(;n%(utN=Q+hou70Uny?&+oRGR6eVC@L=WErgkt^UDl#ny_SM3fjl?Gta4
zAV5^cvy+4r)9)o9G~KHoScpI{wnRkRI0JO9X;A^`5A{CRi%ccg1Iu^aRmv1wh+Qwa
z$!J&lukJW)w;oTTDrZ$tfLAFDPq^o1a(cuW(B=JS0bf^!>(vi}#lv{p__84AsQxDF
z7@$$iq!F28>_Uw!{bPEO52>u!fRWikji&@yd)Gvz<)@SJx~US+W+{O>w4OAszOq=^
z*eI<`Lr|7_Y@UL!Kpw=Zsc+uNXS7{2yiwWi77Y2(HHL!lE7V>S4dP@vMBYM03|{?M
zx#!ohm+2B7YOIEs^t{gtyeC?HE7?9X&G-Tvmma&^hGOi#QLdg$mOCaqr-5}Gty$`j
zo<QU4f%MQDhbxbE*Lh+IbnYn>=n$MF!P@)ltUS1ZtR8-Gbe!Aksic!Gh?8Ag)sRRs
z@o;+*Q{rx^CHl)nV*8QYxl9O&7S&;0Pk&RdX8zTfHLg7c`FnFap@HHVa~bLJ8Rv1x
zKXb|J2Um4eOos#d;l1uB-XWm8ws~i5{gKG1+Jzr*L4o*(m=`pBOQ^$6B2Dx0dfv3Z
z8gM$4nx;$~Z)l#ixP)7w`4RT(dqXCsIFpj4ITrQex7X>Xx6HdUt)DnSn$BE~7S>5E
z51;QEzXl?a_AL?+%@n_~Oqy5DvHF~qT63JFb$xl+AGNW3{=_o)06P~~;~P4bs}ucs
zc07{}3QbpTIyn0W+Wie)Xc+!n>6g^l#J(ytj~gx}DZ)(>Hqc)zY3B&WAB5Pm1{6zE
z5RSVSyu-@x`_0AB$|1NYGt0R#I4-K1`C`(ys#0rE=MK3upyL#>Rq8SER_NJ^I{N;G
zM?&$1zA^-Z8BNG2dp?><UV5D(fU}Y&tCTniXDYJ7JiO~RH}{jw>UC$z$iNk9_y}>y
zmk##xh*Y;_t8Rm<m5qF%C9LB&SY%kQ$4p3VEgS+k0%oocn_7A=*lhojwn8ItEeY1(
zv0nH7P+Nz&?0R7FE;>4$Dv0J6hxn+<V_SLJxrO$!1C<>@pQ}rUc(tYQ$2usp)qMFO
zzrXbY(zg6!V6ApI2p<%?>W;5zvR{M|$HP*J6{#0e{+l;%GJ)0$PM4DO8X@wp3{{o2
z<7-39s3AsqdDf|FKiPaZC}p%~Bi|=U_IO#3T)N2vPl2s-On-?&XTAR|3Z-r|J^Y_U
zq1vTlIVe?Xy5UBwN3-rfRxgZ~e;Hqc3q3>~4LW8olr6`mv7tfBLy6jWvsmYr%dwr&
zJ<0-pxbl(*CdhD68(Ja=4^4;$=`%<A0ClpvM7&fJ$;C3;m=Gs}-<}h=2dFRa`fquU
zU3qm#Ycj7+rwLzuc$7UoJD2?<L(9Y=9OtaflPu>tObfL1r#2=%5TAFv3JMGAo7Td!
zk&jO>JN$-kZi_DtaK$F?Yf5&seHKDAp&vaZKd7>5SC!C2JY@B}{&W|H>Snc4`NbcI
z1mQfly2Z$w4`TO~GnySTC`UV=yIT=if_43O7@^wA(|>6zze3kjDCggLe;D6?r!se)
zS19XS=hC73B@Z9Jqjvf;&Li}Z?lvN=aMI?z<R1PHcA`mJoKuAGo+672t*XC912%|o
zbv*Btchk6;vVKon(Hkf;Su|c8?q$42MjPv^&C`xSBEFVSt?4ekyGT!j`6Y#i!}01$
z>Py6P&e9TeLObT-AcUd)N}ks}H=skmQXRGNx@*$LM`~*GxvsL0(V16H_e+kYu;^>A
zxC;)M;Iib*x667UTnIaMN|+U2K4&B9X$GZ%_UfK~3K;;pw8(fr`~v#X4Nk1OJF1vm
z_oO&uAB;Hp3Ik5Hg1zV264Spbp!PP><{_7%D04_X%|u`|l&oB+Hq<&^?-Zfe{hT4<
zA}7w2zw+JJkT7%evQ^L!d)-aoN9iXv9G&ZDe+cz%>7Imok?G%8t-)DJAhGwF-_}>W
zk)`*}gTG9++6C42`qesq=98U=Nm#XyCwJ(?q^$|9Ito*IONi^q`6_57($~py3rKFq
z*Fu4__adoduwJ38ffe-#&*b2(jLe(a6Cn>vXQ=_us~oZtXiC?aWCC4K>@3HG+lN`G
z`bNd(kZqs;fEBobP*eVBnLkRc3kfHH3t#&ryK6Jtg|>MchdA=e>wH+m7k|FK6P=mu
zkDm&y2V4jP__DH5agqX;n6dF1OYf%D%EAogRexH{3w(LrAyZ%Qy>X0-Ncupt@f!V5
ziySrvF5FeeCBz$J@gT8Vkbpb@ivSzfOcUz?TE51t!3{&|akiaxQ0RNr!k+i@iZ{Jw
zVUC9;7=il^Z!dvh;wKWW8*ZO*s&ZufyaVfSQFmHwJa(Y+p;SU8h02owoa882wp%@L
z>qXQ9Pih59+v$@MZGE%diBoMOQQZ2*Ii-ilNP&d+vf`C~1ysU?F0+&Q^~wQcChN!l
z=#gR|x%xrk*6@p1z%L~xVrFXHF%Yi>Sh!vjW@S4Pk5#P---Cvd=&nH?_VOodx94fI
z5ZY^=H;R)lXaZofe&*|H<tCnuHLO`yVr%FsWJ{M}(|HyB4rQ%3nJh8DE~mb>kVSlH
zgOFhuxup-?Y45uih7Rqn-nml;A$z{i3un?f;<@aZB1jw%VelbTb2QCNWQmOk-&F&e
zb{_AL8*C>``c<C4lmeCdl=t_g`n&yT>eWggAxq`1*WE8-TSEav(N1F|qIjTNGb@t<
zuey5s+EFFdlo0570eeomgZ{9i=O==6H3vEiQnVk<C6l)9o9e+aY#_Gu{!Rfkv%zR6
z@Bct>a^3)1iNCnp0Np7sl&XmX>l0J!CGL3&xWK%6`@lZKHC|K`TZ&KJg;AgOqat0C
zGnW(sS}dZ%Cb@78fBwqpWDOXyrwMR^@2uE*)*=0?XJ|*nxieOehtOVMGva_|zfXJe
zT$S)d2by$179|QuHU6##0@=*ggqe@HL1M|&YC;I3ecz6mvW&Rj@e>Ceb(X<Yc*8IR
zB*2!uiQI1<fQ3F?KtxRhs`lqV{V7x%x2?1<@-MXk0jfYmb57dc6x&4{RGX(RkB}U@
zR%sxjQ#*74C$G8SJMLtcPQ4R%jU^%Ob_@6k7=~Ak4V3PJme34+Cg8V!rbf{FMFf*z
z4Np)5&K^#osqm3*(k7}|*VIZm*m%t~Y+HVfGyrqj4L}Jq1b~WXNDPHe1VoSkg{eo+
zG}EQc^*Upw-)+0X2xn@>X(_l4x8`}$xcI9>rvwcOCzNqSY9ki^Q#ddae3n>{#owin
zbgmPN)QSaH-VUP(s(T<<83#qea04y@E-Zc+iD~XOp(cWcsNY}}sR5*cfiQ6k1%Vg=
zAXxYA)7TE%kazd3D0%FGRKdl0!Evy{=1}>Buz$dUFLVR?v*x1317sC0H-2L(eSb|1
zySTj>Tw2<99}ioeuuY^kMBMp0;OoJ=muQ({aa~EYcQ3mk6{;UB<ES)`i~>5GkAfy5
z>GG9rTAUu8#r4(MNvvV++Bj~%CFCIjGmW40Bz=DEd4kQlHoh+u7q}-b7#IOfxNfF#
z?uNh{*|U26qE?&z@1Z>+O8y#6Z9_Gl4jntXj$Zk$_rb9Am3M+p1*yt`dCD?kHC%>6
zGCW8eEV?AKJ9!hz`yjr#NX3XDmN-ptp0cepvV7xKWX+p!Uj^6Eacf{FIkbcYlV@mO
z?>dXzX*9%UtXL@AeGRPZ5gL!A@PqXF?Ad4uP3flj`*=}ne}@MIC;Cq+`BI9A1zsU(
zB0_<I6@oQB9s1-|n0DHHGjD#!cL_%2S!swo{yysh)^fLav+7iM32s`!HE&_cdcl17
zYhDZ@dAd-oq4!fL-%1n${;p|b1C8IDVC;b%l`@b;*bLR4>EcYi26h^*z45zv_5rG*
z!GO~>k!jduI4##%<9U$?2UR?`FQYTU_yMTlh+*mnAx(#YXRrpFgg8I+o1rTwtY0P}
zLZvT4x-v9)o&nNz$>@7wTwnZM*FiTe55s`+9x3KdNLBH$*Gu^t5`rr@jk`BaB$=*p
z-H>Vq7;VvO{tLvlP*zLMAy>G<fJr!+twitw!J1d6#(cyFb67bjrqYIH_v{nE0LoU2
z1RG{DF89zsGva%W2u+d40@2%`m50N%iDTMFWYOe!=uUYS8HXmwf&<`t#QDn1q|1Xq
zFg-XQmw=GFCnCL`T%BN6S&(y4lAY#I#<ITDkX@Hw2epRw2Cx`>8B%huWBB*wKiz6g
zz|OdC527}vVm449o<<UoxZt$Y`3YZQeZrUa9QO?WE57vme}w{1svOVsx(L<3BK%(y
zf!qhh?*FeJ<(;(RF;Wa*iz@t9JSVv1yF|XtU8cU8#WM4**rT+kJI#vIy%3w^#p|dp
zHIm6v{9z1~&4=xDD%#HbjQ6UNK|4dtPHhlSleC9o;fteQe7mKY85dw?H5AE--0`fc
zz8CuG@#peHPYoc#DYnUAxTLrhI*#Y%IRfCpU<W+Q<jCVE+VQja<;iVCaHSJq;yFv@
zMS!y?NA}aDY`RQYv|4`W`_AIdp#Q9o+8SA;dyCTcffhY~<Z(ic@d7AE&+Z$2m(eF>
zZtqJhPd|uxbec@@>DGr&%UgGI<)>p&hswA>@_jyKW}7)tH~^gKK-K+e?^nxeF3$p`
zz-t9pVx}FgpEiXvAawh9iG+KFXazp>6qa`T^EM6nw9$BfCJ)|6Galj<pUkD12jrqp
zY`gPGrx!Qmn;W)EM5Ap`XSDOkCrsDP-J_g5nZtN-VGWX%SCNdjBIssw#*!WzW_@bM
zl9HK+W7l&<E+0c4Z6Bl_=Ee?j#l-7LUYl%G#IHMNM3ohN8ExP$&p6<cE)Uyh&}(lR
z0Y1^X58VNy!s$C=3S*AC14e9cXTdF=ooemSV+1n-f2?eQlV+W5yv4Q0x=oArLF?F;
z0MdY@0B?3`wop0_rc*N2{1Q01!1~S_kuNf#g$Qg#I~|)_oCZJgEW*NW?mY<?&Pkeb
zl~_W&XOKgqBREqA$!{N`_OdY9n#M>^7n<#%7I#OdiGyF0xI@d~WeB{}>wT4GJKtyd
zuSi8qeRlS9{bsbCx(*QYb{gD~aludrI<VnHXmJCt2#|LJHJ{&Z+(#!lORog|u!?J5
z=QLet!jP(=m5^ZtotMnMeOO+nC%nB(>DMIFW)tgoZ5D5yu>;y>cQ!Tn@oHRk2)PfX
z*Y-Gp-7mk-3op?WR6*8z(rk4`u7^f))iNYIjvtxaS$y>B$z)<RpwBeCx3TQN#a($^
z1J1<N7OyBGh$cZZfx9%=H2>DI(4+KUybyle#DX%Q;MV?~2~c|M&)^)2enOa|5AVFG
zHe7@|<;rMa4+E)1ERi^+>_+c7Ix7#o1ByE5p+jgu8;EjI5oLgtAj$-aLz8eHTt%Ff
zX|WifK(|~_FgwyW)il(3mz8~&+BnQZ29i!_^493CYYcX@eXt<W(l9Q551Q;rIe~u(
zgKXX^`)P}tj(w)GTz*+#pz<FFce@2uh5)nWJRzIo*1%-5IHjnTS&9o~y3m@DR#dH+
z&|<$Jh%}hR!Qe14@~*63^MS8Yo-6k?5AD|o&MRQIb4MzuCIeCo?nMk>c}=;K>u{ht
zC)e(?MNPS=?hc8vJv|swK$T~<U0U>OVX7RSRBN$E?a1O8^8F=xsBSwz(x!}X7YZFS
zWbVgAg$Dh&<kM#YX6agxs-#Va^Z1IhNPwQ#?|KD_`F(m?g+7_KG|e2fy3i-bzFFfn
zL=@?BXC5ca95+u;*Hano8)VKbPYT!w3CKZG4oW7%PfJITUDwvu@XkxR78CMo+`=bp
zxU%*3rT3`9tTTl=WkYI9e%ye#`5!nY>+_D{6iXeqWMb&#j98a_rgI14rDcwv+|l@|
z@s=ZcF|x%rPyXmssF6^8T9i;C_-iVi3#UG!`zD#|0@OlHmZ1fS5<tmIP**ZHy4kGD
z6*MSD2q7*XxTcFRU%3$zBHQQdu6(FAGxmx;zg=H-5@6_Ad7=01X~g>V328_`UbHJp
zl|CuaaCr~+ay7xsAHl98f7zms`pHMKpIq<p&;b(|3mrYfAm=baY@PNFlU=-H7n=KJ
z@HNK+<jz=j7RgTTRrW{XquBg3Y|VGg)@gPfGl4>##GM75LIK_Iser&5*k<?&I6uU~
zTIOeV&twU|#7G)S>c7K@=z*p0O3y`LdxV$$rCRv<^vJ>;e#w(Oc1shR&!j-k6WF4Q
zEgfuo{N?oqmxMyzAf?P|g8Nvz0og$E9NqmfH)9&P?fuLOTJx~}WMjhJx043Jk!w@c
z5|k|k%@B+p<3*VGF#ye#T&EJc>1)xha$AMmnHh~Nu|(-h1#g>pQ-gi2TimaX=iy%x
z;+42cYU{$!D!HzB8?7(DSkfqzCz_dFcZb)ZrB5AJn=f-!pN~_Al<r>5iR3*N=ieXh
zUiM+cU~rAaZw_x<a0o&-bU5e+I+vc^aJ{3?SR&bjShuU=?|41ew#Bb8c|DMX*{1%d
z<3bMMgG`O1?III^_?ze!JIWQtiIB+pStD8_)|-*97#lvQB4QA0L$())2k^{@)G)dP
z*US#Am1vq6jo8U=I$-?(79&h*|KaRiHp|=Mo2zgeZCFit8~c;q<N4cfg?~AF<-+^>
zn>F!FGx3`d3Pi45sVKEmS(f)}qcP7c*2HPeX(P0rpKdauD)+T^OM?nEljiQJpILkT
zr?ppCeNcQ=8FRXbsec(gb;4^Oh1(`9FoWPtiI+w0y869dx51~DT8KER7i1{8Vk>L3
z2Z;B<4lV2u+PWj^bRym$k2m&-XnOUup?+?-E)W)))ccf&^yC!Y%DIfUSBDOi={u+*
z<2SMk91&Sx-2*j;;GX$85~!0J!oNV<go)LBi8D7CDv&kVf@OiMIH>nb7go_x4^5FS
z4?!Y^lBq7@p-QsyIa3cRs_To!zG!``g0NbdtkA=sV0;2=&HW$^Zg7b9ax{zG^pxV-
zr~8}SWr(}QR84r{z(;@2B?}YU*tG`k%3Jtg7M0ls9tFGf$85d%r4+yS7QbBZ#P#)^
zU$4r|^RCWtexz=jdKB`6RZ8$*m00k_*0?`abr#CT!frFv^UazixMp$;zUoF>+~ix`
zdj3@`lsQL`-0|zC&5AoZm~M6a)TgQLkm@<i6mejq+4DFX=)hz@aYkL96E|VIes{8w
zDM0>h6m3U3Oikr3DkNQvj?Hm|$m`Ikj-M|(#`f+^zGVvknMsX4hgOF^;uhR?%_zbW
z0BqQ)#~8Z-4mvJ4S)^_$8bfz|I9zr5+F2N=zcqljv#ygWfIm3Ce6R*KSyBQnA1iz+
zI<AVeoJ;p>SsB@oWE^!ABrKQ0!TsKG_SN8^pe;)`jHIYJiOOHS0K5&8>dH;F)P<IE
z;%^7~)k<jUdU6iH+TLQROG`Jen#%h`M3a{79a;55>*~xLXln$1yIz6&%{uI3%Tc~5
zP8U41H5n*zZw>nNI#Tzi!mraW?(m{mHcQMjIZe<?x$TCvQPm;)E4T;}N3mKp2QrU7
z6x?;~bBccENBTC@P_7J`vAaA}3BEJfCNIDAx0@;GW||}brp>wk_&fnKwyU>6_xSu&
zZsNa26f}(ljx=uEoBcRdTVoHv(@7r82Smc^n&_Y<!YCHqV`YvXHgb24?Ly6`ahlE4
zQy=z+mU)nOg#k2y34s?P5}9a<Ki;9Nt1L^%v=GUKxmKd`aI{{-P+GAv!HNF3nFsJE
zaT|CTwpH2H%IK?_Xd{U?G=Hz;oAi>z2eK2yYK082+Oi}NuUQRC_D9Xp#A5~`(RZb&
zTp9`<YL^a@0%=23&B*t;u$V^>(HIKrNZogq)PL$}nl)XxAmZ56_}cMqN(n$hA&t1-
znji|VaIT4U^bEsTd0syM>h0?J!u!gJMcZjNoLJ%BAweSeVQ`C_Xg_{AZo=iMwi>@4
zIBWfGZk6VL6?J6I`1pnug}m{~C~6k9aEGabwu2H{0t9oG<@~X#uSLMR^12Pk&gZFM
z9-{g@b3i-LTziDkcDVMz3Xp*<Ps&rYyhqGjbU+r#+JXqK0rVTc(R_1L*S~?6zjjfp
z_l<TRqTQp)2?KxNbfy3BHW(rJJJNfS^a#%64BHGE-N-nHyMSe6!xfX2cC}M_T>}~u
zVend~2oef1r|x{e$*M-Vdt0FqMibG|cl9q7^QuQyV_W^N>>-r4*U(f#qW@+hBwoHt
z9Vf?hl!}F52*PG_>3o|}v5DVHINOJAH77^lLjPhy2p_{w6pws~)&#$ej^(m_Su=)@
z>%odbR?EpE=c#YWc3#hfPKQ}HpJ3~?NGsbmk@W}cOhe(yE7F4^?>2$SErh|55zcx+
zFPm%C96g04o~6cb_^ZdpVXV+w*7%R&V8Zi(S4F{KCxgAObw%ShjQDjxM5K>cIhYW$
zY)38N+({t&$rJf&PVne#sLWB8kYrYaEB}D2w7@L+j1@Uv-LA!?`Slb%5&5#J+3UMq
z-RT``Z6N?0oG!w@9Fe0%a7>e5Q@@-a=+2U@k1pX@bVg!<(b?NOev)iTjFqOks_kUF
z5_;i&rhu7p2D0LNyxKCEl4p1#iPoZF>&BaN^K@T31zNUKYce;hUdcl-4e$eVSBu(-
zCSC&k4oJPR4YLnZ^zAT{JPMW0Y#bYiKNo23(Hh0hO@?Bf4^@6YMG=H0pyhhN)lyFF
zFWhH{8_@iz-ba}HFzxr^{U23f05P<@b`=ey{(U9?qNRVoFV_BvyjmTT@cyq9?4R#j
z`vlErmtW)z{^v=5f74dVQ?`3~HnH+QaN?P$jYc93!w%4IhWaBYS@(~>DX3af;G@1d
zs`XQKej7%8-3u*BgeG!>m)<~kC1PU^t>5*hzllg3roXrGR!K+GetAcO4@A;?gB#W5
z>YVdJZRcf&r}0*Kj{ap9le-`oJ&qW`g?z#3jJB5Y_a)+_z}7|Jq`b5J8?E`LmW2hb
z$wFcA!z4q!<KNKYADi7jKTXl*X)w5g7yk1Pzel4p=?R@??`mFA`R{cy6wjS|%EWAt
z>;Cra{QDCAQl`4L75q<=BvSPxAeh6k`b#tL&qY0bxA~JJXzba(*Vrr87n*g*ZcI#F
zwlW`wPG&RiFTj_bYzGQa=>h6b*>)hb0>}DkRATbG05?+n?*({b#?te%9yrmZ&21V)
z_{I!C_PbZ7C^sJ4Wd@@oL5RV~>+(M9{d#(Ge<O?1p@D0YR+@`y2;W6a2e>9rR=Gjx
zkL@(O#PthOmL8Y1MJoj*_sKLmN8Z}i=%kL48)EN$h%Fta-XJ|kiR&C!l)CqBI>>kk
zmmIbH(qvRQ-EJxp$R|5@eO}^Ze1@Lj=~~{dxu@?MWkYmUxtW{hpq2mdGpo_vdY*M1
z$eZRksg59RzF5Q}W$5sbAus+e`@rLzD#7+mmDRr4q{mX3(@AJjG@H9opi>~GTJE5K
zzoead{cBmT91$UGo{Mel0jceyw&<&Vmjob(5_IltP*J}gQe(I@tM927QD_x7Z=M~V
zZFX3<(~FF4EZfEMddClqZrpB_CSIIcBoOCHJ0xaJkF2!4PdB(vGTqtcnQqB6a_1GU
zFS&CzZBZ4QA83w<MIpZUCgM}M;Aqn!P-UE4Dz{=eZS<hBEDmZy<0H0$RQIH8O#mNb
zHe&`z7IKa6#aF$zrgv?wXFnv6o_^vcr**IEcvqYy>*bc;+V+5W?XhT&4oqcfg`y%p
zJ)fb^?tpe+$a9m)$zVXYkCKS@eTm4f9G1P(sO31rkA-ECf*qMA&Fyiux}C8ZdH+ZN
zk@$ar1^$e(BZBY#4DK55rs4u$vAw{-rK@U){FZy{@2Y4vudCVPRsGwDl5n`S?XoR3
zC@;n2P>}h(c+%ovr24pRY;*318WxTcR7Tw~t7shKm&y>*aj&JkeSfIPl5LAp#D8uK
zRYMO@EH`$_(BP_>0~eYMRVzl7H&XShH-2fH#%bXomuX1I7i4^Ua>-~63Taoanwiy=
zlfJ$fjj}7N>6)1NZNzytFdWooRtG0Re;!EC^2C{Wm}QEo0+OcB3=~O(zZqa_NG=kE
zZ+&zo`xZ{u0=!74x)`eK&dxO%F=O)^^i~$+ct?HNB$}@IkOeKPWi8Os{(p4DlPMQ~
zf)ZcQOQ`W*J*vvr%N{t!v?BFEnmxzn7E6isMOSwHGqQZ_`Sy6sA%7whP2TGz)ohk2
z=Vi0{`V#ly67Ha{xl9*asR~cid-n)wQUMnnM7KP6jVQ`KF7CK_Hl9J)+$K_loY^gB
zXxO|jl`Zkq44CaFb@T=I?V3CvLVp$<ic&)n3KtS;lLw!^>x;=^FHu5NAXkG(1ohw2
zqZLAj<JHZ!@_0oA0k#+X+g^in?K07RS2I;4y1qSo1Cloja!yu{6oB$rEp=q;>CQqK
zV0<(Hcsf_@pDPW@?}zMS6JBr)*IT77%S4-ve(Ylh(VE;(tbx~+>*Sp0&lXgzt9v{h
zMu;ZY;;n&?(-oVW>AvWm>x#YXmj~{R1<PLhidf*?e|v3AYg{x#6?yG%dNp~gEL%vf
zfq|T$_0$a{dn|@Q;W2l~*(Zep?RSxXK_qX7{((q-Qlkfla2(RGyR)z{Rx)KNDisbZ
zM*#CcOKlA6+DqMyT>cypr3|cU*}&a(xk^(6$quhyY=rTvxx=bG&3w2+EM?t{vv0EN
zQ3HVSU-eH1H=^UyD6lpB7|X0VqRT6TlnZ$j$WJP>QvK>}(>;lAe?7G;{O>*P7E!e2
zq0^^4%>xwPS1Y4qsknZ}<|I#K^5``H$8XT?6cc4z(sJ{i%hPR(27xAl>&Uia8Bfz}
zw|JR5|D)iDKog5!;c~HU{_fM7^t+rmiG!~5zCdXZ?$^NNk2f-!AHAZlMhD>cg1d&2
z0Ad`4+Cf=`6KUYI)Sy!W0xU*~y6i)-IlFD<S@7_8`NP+>c1%xRLT}G~q_QR?g$@Mk
z${bKe8!BTX+A#seK>P!x1U_ROO$v<>+P(}pDhCVP7#P$)?P%=@^rSyS8f#^?ECHv?
zkVdGEUMk#ADyDALS}(rWv=RK$^nl!xESF%43!R#i;m!~pKd1(HwH`nF13s~mj9USz
zljKt~I-i;i1xj0;J$<A;qFFCBs|658aGoPOW0v|8K6q_baJYeuI~>S9Fqph~2Smf9
zSQmF}MsP1#mPQAX;L<zgkI#6JE=74yTbjI9a`ZEskvL_H)z@}O3*L>H<!hGKKtt!&
zjZ6b2&;E(GzzsFZRmsXKMEeQf=|q?V6Xiq*D@=Ak8g>V9j@?zIAwNjrK@D_tihHZ8
z9K`B29x0g^`RC<}>U<kLT`k1^95Ls=`vW}f_&8q!*WSf}yN3cWP{+Fo+3%cq<6;?^
z^gT44YMeKJX2@)vpf(<DZ6lunL`8Gg1#_ritd&If!LgzSWAT(Vek|i&^4iU>+aXo!
z@hRvs)NyLsz{tCEiNI>vLjD6Rsbu*5gQ5>$t!;Cg#lDZvW<Rq<j0$HXp0hGG2C8-E
zLHJ_~wdCSO4Mt;YIa9*2yJkwHI5qWm*+@S)Zkm~!$nA$*xRo7lk;q_o$K>AEOxU`?
z@HSskKRHL8tXAK?&$isJFEx820^TA(A)R1FBkc&zEOxhrSdI7}$%^!i-U_b?<08_f
zV<h6+S&q|vJ4!qGH0WJaPlOhdkM-Uq%*zm6MjLeqSx!4bO#TzjiAKTU{}zP`-%>A_
zrmkTSjAul@EPPm{742JQeu)231p-BLdTU?z{IIs&5GY04zFFgQ_~_@<$3Y3gw=1<h
zNwr%Q+^-LW+*tns<|G`JBwr*{oC%>W;S0+O%?ap4vq$pB&OF`xzi)?5jVE3xj*bc1
zYXh}D)>lC8q8$=tB!y<bAek)Skm8VbfJvLOYt0Sn*CDHI(5TZx`9kbpkUG~$y01^l
z^eS-O2y4D5+|<bTum}*6RhS<Axt@tifLUGMX3ot1Bh5^+I|pOaQE3f&u%2EgpU!16
zXmc)l=F<!Z4W)`z`l<%evVVz<{}p`CNTS{MAPQOo`(8t%fRZwk47VBMs>!`bGHM#^
zOVETc9)1aFSt+WjI)ePRX%_^NSQ{C^4sJe@YS6MQQuVE4Uk%QHB16SK|3-%XzxR9#
zGL$>Ue@?$u#I)m=CXhyje}DcBL_hyihaL>)yzI#DAiiJ5vQ){6;Ah-fx|bF8NeAMN
z<WG1s2_|O!Im4jnM7}4WYew-C+&RarX+lCrZhNhkEQ$o>Q~$&=zIKbOKbGKaF>4nn
zvvwA?hjiTb9pR-mIY+*Jyh(Wfc0Cu`970uAp&M?{wQ3ZZ=_9vV;JeoQb$qma=6!Jt
z0U08Z`NfOO!@dcN$RS#|w}jyNn1WRM^(oG>a@2Qq$NmOH>W1pl4_gXfgeq9FIUC|k
zOD|ntXr{#Y>(JIQ^UYRXnUTA0^dBe=HiUk@D0@xxwkUf26{+(wu6Q239I=dpORS@>
z>BVt$S}`8l>fE$)TpTR4mD_lqo69~*7-y+#cM0*nXqnw0KKGs_twU{la(RbnO1kZP
zL9#p0hB&e4|Juf_)`JgacCMT^`Z)g}^>{D&Z43d=9{hW~2WYcP4KdyPro9Bv2hT5O
zm~n|e;rI7Y+WNoSKQG=w;VvCrQvhY%O=W{D#w1c-@upm+eRLh-t`#t_v-SR>J6YJ!
zT5tDBaqwTGJS1YM#9#&O2ucdjCD#$DTprnt@zh!6F@Hyh7Ew&CH3$!&nP~Rjd$|Oc
zC6N4Xi@Wx1*t4LZL@BJ9lF%&G{JZ31b&Ij{U$1Ed0Xlugwh3GMr9=VkQ)~y)1Zh_s
z;QGl|dePlK$8LqLppE1LHfXBX{p1+9#}x3DELjGHtaOG%JWUO4LQ2pQjICGEfw|H#
z-@U{qUo^HnG+IYNH}HCVqs~IX%HiZ8M)t-yjLX8jP&?f^FJ*p#A-l3Vjo!hh4r}|`
zs2s8L_jvdlb(%WX2qA1B=ZXc7_v-cKEu3ddr1K4Q=8+%SZkrSd&()ToS&q<h;<2T0
zFGZ5i%zGEJ@<2#`xK`9tSGAVzweM+c*73f(87!OmV0_}C<KgAKK;zSFi9eYsC^&~~
zJK-^<O-3Mj@0~Z@?(*^j^*s2Cn(d6NmHxOv#At(UlG4V~3f4>u19%KU%5a8W*?)TG
zQ+9lMgFI)oGk=lb72F28ZVS;DUyFGJs!Hj^2`);&Vh+KMmU$Dxtky(LJ&<E<UBc~D
z$z*lGi}KU+*$RYb@KT2iqY@{@lDzfCX%nl>n#8hH#K{#|sx>x4$8%iIy?;2Nkl7*i
zy4LsM9)*Ko)YVOGmXUxLl~Lk->Ur<j72$Jg?qSt_8pYqOSy1>C-&$B|<RHDnfG2<r
zfoA-M@0qqC-x~ccI_0;Q@yRq<e2o{R5?TxUF`u_`G|z~Anu!97Hq)kbWe)QRL|A$^
zEx~A*h_glX3RQ(YmsLbsNo_{lx11f?(32n>qKJc5A>lzXXK#Cut!xLDc^h!O@Z+`0
z1(#vr7i}_h&^@sf?u`ZQ;CnOGV&BuAa@RL^5|}u^Yzt^slJ}CeFNMbIdTsPu<bD$_
z92fq=6y0d2CH7ZdLd*Olfzd^%^Mgf*u7a0yP|fA=zJL{_6G)4q2x(7df?7+7fJI@f
z9x6R8mBI|!xB2#SuUhmy7R$rb_wiVimiZ*Hmf|B>3@8yht|sO})hm}VK1$^op)ogf
zv*w9+kbSBiyyXk6W7<{|ZK-l!v8d_&(fnjQsuF8h;_YgDOy|y*`slAmW1BBv&*AEH
zsh?xdI}p+Hio_<2<|wKq$l4qFG^Oa+CVxWMGR;KHFF9k!G(~g9-(3*x&Q?&yy8&T-
zv&5dbukZ{k-*d^|TQ^Pk0c>n>W@>6C{Ix$U`kZ9vRxC1l>tA<%pCym`5NHv`MppR%
zr_citfyt@`$?|Q;yTnwA$ura}#vg?LZ@tte$-&=)HV{o$zfhucr#0BewIa{X4$eh#
z68MKeTd7t5Tthfe;_uTu_EPF0bE;E$Q+Smuw^e?jL3G$UV_z!j4b4JLOUZpLc5ytj
z7ip862T?vbwV^KFFCEolF#JZ=M&=$l-@H`B`vY0CIDox%I$jLtEP^t?c-^s7AzVXc
zC=b^9d?alUYwFUeo)nDU)&%QPB>8|3_nkl7O$9E}d~-o)4vK^S9aq!V{7$6(x3Je?
z1m=zb+9Tx^`)^N@pn5)hywVpvq)Vebr<9bKjYPD4nmCCa$-Y2L3kE>^;M8^bhX`Cf
z-6l(rm{iDX?Z`d{PPeLh6x-_Ku+9TUE-{U=g3;E-=A)BcYgsU_WY9x8H;ky1WV-HW
z6XG>B_AJzhjCHbAScBNEHt8RDO}`A5e^inFzd&|4zPtdY4BgQ+ddb+cd(e+&WSonz
z)DIw93$fAnUT)=6sj0Jc=5|Bb6?v|abv<E`pZFeQ@wyhue?u|A?@z{`ZwEsj4DLgW
zO>07>%<y<{(AF(m@+RTdS$%RkD+}xDqCd0tjV-32s9g6R@<B429OSzQHhz;ZWjL*j
z<4k0AtN)i(Q1Ni}uVbuM9r#Ef@2HGURV1Sl2`f(T(vMojYF9nFnpRQm`W;?yQcCn;
zTz&pu(j}bmz|L&7DeXV2ajMQvfW^m$;vJYX*p!Nv?>wnGOnE7ef~cDC2e9IB32-{F
zpJpF$o+Hv4!T2l#X)1}YunJKW_h425-mu`b=g(kkhusj;VdgNjKhwE*=i`0%u52rT
zQ9+)fsUNs6F9(3(P$%wLU%>u0^ZbQIDVicoztb_@tIhs>q|a<kuy!2J@c9aBfaNQb
zX>x1D!u&=uRK>2`A5wWP@Z*S|Vew!d;Z|UvA}ZMUJggfIk;frWU;)fZ4y#CC1HXty
z5P#qd%>Lu^#86qnG+TknXN|;HFRsusKj<V1Da~(xP@%8653mS0tUMfmw*2}RSoXD7
z#4yUrlIqXZKiY7OmPiWlb4MD)a8c;Qpc2)SDc1o&49I|uH^3N%=Yyqys@rh&wSnHS
zwUzqN_D8w_qbK!@Aa7;N`~yJYoN^RUI8h?2`<kW`KH(TH3T8j$hfKzB^)64AQyTJX
z7;>1W&Z_@{?P%Ga+-hvD+P@34K=yu^xHVbOvL$O?Uz{{!+=7{iZD(wPLpMeuc7PNN
zt|5I-Ykf1<#^l={4z4I7eTj!E?`%WhgqI;jB3-S|Rx-fA|N8rep<IYqzC6Qkm7L-c
z!CWNSMBWu~fOJWXB2vEhuMSvkCt-L$Sc_}dcyUQ+QzAe*z@m=n{b$G63r63lwnb;m
zk-y<z=!JK(Lw(6!+XLiK15ovpr%pxe7867tHzbhlUXN*@mQ;Mz3l1u}Dn<;<M#hyd
z2j$^jAP3mi!63k?z|cpfz-mkEewEcQo|**tIoLAI9-9GviNKujeuLLdb`KYgdJR`M
z3cC$(2{iML1c=4y+O%~)Ks%L*8Hwy<WLM_9{vJALPa!D$iB3NBSmRE%rlzVuoN-Pj
z8SpF`2B|F={liP-uOHy^=xCqgy`K=yF4PoJeZlW@!TeS4tr(`*vplKew%~6;LFezh
z?#)I=R+ju?z6Hg)`+Qm!v~~AK*-uO3b!I+|wmL=Bm#SzYW;u15Xv@NyYjKd2d(bNz
zNu=6zr!O&m&>OmWx!?C)W$D@#98;RyMUPPa9Y&ouV>m9ZS5`%<FNfh2z2rRRp`qph
zL~oL6hzs!G+?NLrNt3zzd{%26z-qrE+Yc#?!$tJoA6IuXJTG~CcEXzbAZ)Tk@jO7J
zha!MH7HvmMV<4?Nvkbi$%~Y1lVoUxPI=%PGN-M*eVu`frs12pM%zE#GZo2XM`y*ZI
z6)h#X<dX=3MPm{Tb-V3|jGC{QOgk?k?J@WJB#}@TLc3-wZ>!%jz??i-&iSJ`T)u>X
zg9O0{)Iv7j{@@aUdA27ISj=Gt|0#%Ehs`W+G@6xROZ%$o!1~q7QZbIqPT|lmR=no{
zFuz|SfLAXKPAH(KczP+OJu?ko)SB^x3q$*T0D5fv4N_;OZQ>ZI`|m@XQfrJE2cgLV
z6r=F}yyZFcEd*nP?nr+f`r~}i3yXk3UGTb;bQJybEB^We^t}Jq4_U9G@y=iAY;PRN
zMsVTDJGeW(R?_)lV*;a~LF43B?Fgp>m{m+mNKrvcLn?p`{3O;n+ZRJ(TRQv-VkE>A
zmU-3aF?T>2?#*FoJv}hr^`$jUFl-1FBX?zFqObD%kM@|45cjl-(1bprdgMdB1WN-2
zb1kyD=0-lggngmX$N-`jFtAUrLN7SY`hvYA<6dPr<I;A{CWbA1ys7g%#PCMl<vO|%
zviVPNKDtGjg+YFEWHK_~)QJY9U(N;BIg2V|;PC0O2|@EI%cIkc*l$LyINSF<+7rH_
zI#c6fI&^{sHoL>jZ&x^u<1eO}6)!1IGAO0a?^tN)7{+{dL1fT}25+>qOvMP3<Uu5c
zHE65UA1BMwem0+&-B`z`DTTT#%UUttUwil?pKv}pCZ;AS^cc*S-^dcYct2HcCDu+h
zijxvA!%{9F#{X(EtiJ~q8k2oUs2E40k(Z6(Z~{asZyhmx-Ms|c9M>dj0@DwjY3>Qz
zK91^s&H({hCfVDLUSe?VEv4yNx=y=!d9F67Y8u~-Hge^%=wUCVT+n1tv%>5^(q(6Z
z&o?lf(Q%?MCC=Ra<q=g}j@gR*+s9O!=?e?IA`fK+<i`%~!jw<{4|8uF)aKUy{Z?ox
zZGjewOIqA1?p|DiyA%l?+?~?mQrs!7!CeBy-QC@-xCTzxyZd?j{Pz3KIe(lpb7nI4
zWV%Rt-z!VL*XNSO7+fU$de`xJf9CCqV4dUYu82MQ3`z!KeJByYB2p<%@(^$r-N<=X
zEr&Py)O8Xx(`dV@Tv4*fV?M{0la$VVy)owEa`;PvYWe=t7w*I%H^cE96GWS(o4I7C
zh(lY-cF?Y}=M2xrQyX1<6%t}TN$bnKOTek+oQD!?EzBwbgExi@q9H~;s;?No4H!I&
z7i9B8N!O-6U@B~3_&q76NIt`COZqKx5gyMXB$TFIPebP^A@Q}7>0mE%TKw5?%=@<j
zu#{OTN6)e!9j9h8#PzW_e#MIm@#zXSCS5`I!zHoRat!xHF2@}X*OAATR4zN&4kRLJ
zdH$jo(yW>HJPA@yb*2Q_8cThOTO$b@$i^xtS@iUe+m8sg1ueYQlHX0`$KE*Y_~|oW
z_iITMyV}>oC-Pzx>cw_%r3*to8^I*((L13_Z{5P}Bg?QMuT8d5tVF%7eXe?WNl(XZ
zl*jNfOXBU&C{U4m4?Vs1228Oh$SB-7$9H_w5}&MY=re93vuGTfZ$gIQ)&Ib)bs1jq
zMjg`d^T^gMFdX$C2g)C2?{5#Fw1!-FHD9u-j2!!i<llCupyxfqbDb>m_;P*E?bG&I
zi}9BL)bB!=6)mw;&YfUV=WI43N+Sw*Shj3R@%SKQ`=nq)7EebXSXzNI*MP?}2Y%gu
zeW+zmZ(HU0soZrg|FvW*gVY6l>gl>3iFrD#qgRj_=G6J+-8V{9m*{d-w%X^%KTTW-
zLL89o*j>^AWD8Im$qQOgLB~83mOoY)A5Jnmt+$>I#Z+$_h`fe6UkSM-@1KF&mfRsX
zPO#Et89$K^J4wEvDa60#@gwhJG%o=SZFyrAizh4Anl3u1XvI4hvnzeqS_=zdtjS_&
zO{nB@NO<|8`b;LfdCWU~+DhS{&j(m7*F=KugHw}k^?E)J?gF`1PtQg+)Xp2vE%qgW
zZjMqYAtc;wem5AyqW2<lA6z|Grwi(BEV<86&R9&z5noM}+_~#K5hmSC7@y9M{kS<H
z&2w3&ux&0MI@0zEw4l$CqfDn!8|v-0_`||qc{AS5`J?-w_BM|ZQ*LhDR*u%D3H>1u
z+t?yiqv7MbFOOcRl6>?J`59u3xK$jd)zj)kvQGoU@{FsytYkIi<23(R7&w#9U9vx&
ziuN``h9u*C&_wcVYg#)9Fr3*C?fB@K%jIxro4)x5M@ch|Owja6VpL*LLxE)2eO^P$
z2whtNB{FCbg6%@-j73rY(GPR)Dv*=aA}<((18nh+MRpH#AWu%UoTfIVQmNV!;1e-l
zx{ln@X8pnUNmQOV<hAf9?|D&g1#GJ1LsVR%2!KE<OVuCotd+RHSg&4}<in7j@vC~a
z%ct;@vD5Dzt-jSEda?mvRvh1)17{o86i!!CiZLSb)RBiKe=J#(D!R&kW^<3PIuW~M
zhpK2$w_+l|_c-{gNAl5gu(NoYRxOhM<GQU$h0_tsoayrXS87aqx9c&s=Kh4^SOL{6
zxuuB<x!j!}>Q!iGMB-~fX&%jcwZRet!xvX3o6dEcqfevSJ~U*Pok($psJHMLBFpTz
z+nThqvqo_tB1WR(EdpS+F|Av#&FpTfb~IRzmUgK#J5uPz(rRPpof|KZ!NQ&dQbe_E
zNADd$8Ls+y)yiBazWUu`^pH{qe_Hs0OZvjidWAj#y6^4|Wcc+3njoua_(g6yZiw1g
zxQ$H7feMCSdyGH1L3nq~s%zdw01O%d^oBaQs3Z+L8<FCxG}I!>6P<z^H0|DEqT$jk
z*CKJ4koYE%p_#h*w}#)a>}CeGS!vf?NV<fFEUb+D(k?7C&vH8wSd}#a8aZOPm`@E@
zEiTw2g*ijXDG!TJcPAmlU<zd9iH_+v)nN&)*X54^xq`_-5`oj-X@m5-X~&?Bi8X8^
zb=cr_ZItrm{oWKyY5D2g`?V?4gnn)HR_FewPdF8TcUHfahZifdr{E*<K~C_0gYW2t
zZ|Iom=)Lb{vSSLi?O0EFp*dXp0a7_S4nNp-nIMwrF4mf&xI+_R+DjZI3HZUasod2w
zrG<sn_2?l7vq{P&u_}=VwC=_|OK)5fgnZkDOsURyp1NyIf3#1Jbtc>=LQ&JcAA6L*
zO)+JMZBLV;iNtXCm8XM9V9y&f063_iu^)!Oi+RY5fWV(>`ht51^v>^uSD+pZI0Q7Z
z>nI4OIRUl*<cUqrl!=P@Ar3se{K|`eOu^G@tH)QjBDd}hJ6=<VeG}{2J1eBi&>_Ab
zjg3E$TK+gMy+<Hp5C(qZYCIS_z`$be$V>hWc`y5#=%_A6rHz<I<N`3v3AAy(!O@N-
zaln0tEwk@#uWeF0aB04JVJ~m-9OHOS|LV<q5qDg?3{;0&wf@NHH5K4_g4pz4cu3tF
z%Of<By?Rf6@|p0@_yXm@e60S%d}Mns-J3*PI!ALlRTLx+p1I>V{5|&hQ;86gs7J8D
z0;%~(Yt}8fnSD=o(DsCArzELQ43UKe?MH21=IXz)v{)_VPVzlFKi5ho*g6Z)GQQbn
zR&P&hCqhGm9fOli-nxEYl?v`T9+ON2b1$Sm1M!*P__Cy5WPhb@z5P`yWs;7?Ff)lS
zH8xmmPupKd(_QjfI1~&rXEhIUhfAXygyX3KiViiECRvtA7EZ|vl<&?Zd@0kZ0R9LH
z@mw20nS4gOc09_IUYR5s@s0LCv2;o#WEs%vWQ3HdBk~z|!ketnQ(P%%S@oxQij;7K
zqPUiD7S!@1?UpMR5A+1RJX5}^gArQPu<``=s1x5EOLCG@Tl=Ud@a>P*4;`o$&OsZe
zaSw;NB!AU4JD>A#H6`d)O%Pil7mpk78rS%ybdOf6_D$04tSmZHMG}Q3hl7WF=rGC7
zOx241*!Cow9tg>*GDi^lr&}2jsO~(+dy<~H^Z$J5J;;F*(KVVeGi-1kCy`s0CZ<|s
ze9s!Pou$Dw!kOYojg;O)j5brh7D^Y@qT4-q?<Tmop0=E})*=F8V@($pHW)Yv;dvVE
zr`Qqz9+&K@$lF`_xY?CyzO&F1`x;qY9G?^?xf2T2SX|f4A7GjdhBlZ?gZN&MhAV3(
zlujxyB5}c0GX~lFcn4vUHV4yAvvK?hLwqu7TS4U32+^v&K~NgA$G+|L(4sYWwPG3J
z;_+m;X|T09LX)RK)(jg=D7bcZR!iE^(n-nEJf&A6yMP0)EKh7>{b@Hyw<r57CY01<
z1W|=su-Iy1-Ba&E%~Y?nZ^S;HHEK7iP05$Y)^$EflOksKQ)p&N7Ug|o`gx9BO^k>i
zN;YaJw%?0h?nN#0+n{QVG9L0o0~_V!t1v$2lw*SZ;xv=w;>}N5Pubs20!4gFCr~tg
zOKA%F$6xVdW8cjv;dtMd3UHJpAED-cl1U#+Ko$=dAh=~Gwl>Q+A=U08=A&0x*J8;p
zu~}N)5wo<-UVV3d2d}W|D^l{6)wAS)rag0QOQ|7)5umbBGgDA)C!zLoa<R9aCiv33
zK!f_wyh30k?-ln2YaqICJmdFFIYj;7*pm`Fx+rh>{z?9^U{)1F-h7Ja19>tuuwQA|
z7{w8`zo)$JJvQU-jo1I7%lAYTL<vyKP@V`wUSBs)sJq#LR;u$&FU(til}IJ&fK&yC
zL;==nE7ff%I55Ml=B)-G=In>cD*7;BZB>|WpP~La7wEORrHQ8geqFG}c9DW$3G({!
zPdd}9<ns8A5AE*LW;{MZVVmanQ++R@4V26V_mzVoIOQ`O+OzjO^W)Z;j2<e&;L*U+
zXTRl(#9O$0p)}vJV?Y?;`08}Gg+gUkN}?ZOuWeZmc?MIcgU6}(5diDBzjhqS^A7BZ
zgLn2@%qFUomf-_y<AYoF<jVR+W{c;^w3t>Esqf5lCplN`FxIHevOxUh?@Jg-N>j2R
z_bG5vsp>>pr3$_N+z7dBe-H++sBhSP#kVS|JwBRrx0Ebd-D&uHsI=1ogs!|)nQk_W
z?b8wdLd?!W@}<tQH1vtF6{6?0dB%V~cnb3dK7`lF-xIcM-pFLpg=iEwmqb_@3(lrJ
z-1;Z9!}ORbaXkCpt5)XkAvM}rY9UbeqmeNbVnqjCq=M)-pi$ltN0-gSEff@%7^#kh
z1{sKfBqj#&rIQQuU=B~0txe8%gQuAK4%$Du@kj72W4?;lql^~PP@jHqH3S`)AmB~S
zT$ZhUr!BSD8Ue;UiO5tU(mCSR?KYt4J!R)&Kbo8QoPfjYOHvzSz&!CO2Fqo?Z2OeZ
zHE(@^PNXf>(_#|Kbm6!?*r)Ng0s9^y$@?)vzR%ASea@kAt%Up`El~#@ixgLmL4E10
z=|w_Pq-k)=!UT{VC#s;F7*zh@5+l~Vvv<nBi;)e!XGRek5nH1a^H5I*vYjyU4Z<=L
zd&$5>duxN;>73i;=*1q8CR4WCaWDzaw{|g1oT&jIxRTG8Zbo7|>D!eaDOa(-p(a5-
znli@Gvx8u<9KS|(gD7#e9nS><qo`yxu3{RDwbczNOQ}7n5O#$!C*S+(y*J6?eG>_v
zU%@SWq$hg*EV`CE@-olYdvwdEldPR^28VCYPhIN1L{bTBvja<~>txlzmuID}+tMiR
z(PzrZ1Oyy&6VX*2fHZRFd6js5PNGae$hwt*5Yh!I+;m%j1TNrl_m%_-B{nIZh;Hs(
z`fpss3wmnb2Qn5p9^3F;SgPyI#cZ~tpl)Yu6mT9ha?Dyd`&G3AveYydvnCPeiZ?O`
zUIHZ36U>3^5~Hk*X#}|7eXU8Va~Ys9SFllc%+@9XaNTO<oBLsQy!qBkdNskVRsE*j
zP7I+7WbGUkMyywh`QM|BsukNmVn6+@4=?Xa-@M7?_KwiDWm9fsyNE)NF6#?pIWn{~
zvk+h~nlmV<!VJ`G!d{DL$S)&W9?q70spJ5u8?!uY`CwD)ggBn%r(9oDNpjoiYOWpL
z_bfWgnz@$L?hS!-NZGdTDF|3VQ%@`&{;7=`_W(<TPdEl7KG8vasQg+G5k<5#iX^{5
zUlGKG$~r|l9hs*}%?4HhgrbJtD_T4~{Biy3qctjgKq05VxefAN@z1PhpL^q?se288
z#$Zg>Nu3lkOaBgY=bBXH7n(GWh4hs>u1QB4RX%3?WS3YWDJG_NVUSmCIV|n*JgJLQ
z(AYEUj*Ir4=%GdVBKQxqAVqLGCxpwOzr^yN*B!#g*cJAN@yme%|7weS^jiIv<!|iq
z=QTGH;q%Z$cc*XA^S{@kKdxQ^K}}D8&yN4#okvgXBOb|5Om=rFo6)IHRQ_B{Q$Jzf
znfoTTpyH-jU>jCAq4{xvuX00ftov2-O^ysNt0wb<jD!PH2E73U(sg(sJVDnn>bDr=
zkVg~V=ElH=o<pp|A5@|<R?pjRa^M-FbCht=2_t`WC<gQFs1f@8;QzVp+}MYE?(Gar
zeDcTLJ~Vq@1L3yA|4%+-dXmjut6U+{_5iQCuWdTz0UM(&Tk39_+~()u9XtsYEIX@v
zOP>}72t`1?i1LRUTY9Pn){EWE*L+AiSoA^e<`^^ebfb+a0~XoBBArE{1M79D88YmT
zyq`;?rZsuuZs~V|eg4wu%pac&An$q5pY4aPyi!$jc`Muy0-EpQjAe<a<P@g)2~<NY
z5Bx^CncRols=m(1U4zQAu*&-Km$B5dcHz_c?X%MkZ$SURiTMw%?ZGVqyA45nEyAhj
zlx=%N=MuTigEfhQjH=Pzo5rjbGyx^DN#D~fvoEfrcWa|0&zBk&BDd?ld#r6O!Rtd=
zGkaca+I?IP-g$pEco8q5^6X5J4a#)H&QpHEB+=bVf%pAl&F_%Q$57tG`onzvk$pxT
z64-CpM@uvuwYr<CFY5yJUP34NdNwy?+Z(#e2wtqUrdFRiBxF5ERFIJ)bhkC-fjT;o
zM*s385=fCYhM!Y8TJe<Qm-~SXclSkY)JmCxBJ6+x@ck8DJrT~8T41YD=Y6W@>Ulv~
z_rBjK<X0)P?z|vpyv0>G-=xvYd3;WPm%<w0Su^0htO^qI=|$MGt!u(!-qWXbznWmr
zc3>b?BNI=DizDWMMXZ>Gr^hL`rc<L8EO#%U6^D54jt!ot6eUmp;-gkb@beo)wY~(I
z@aLF!Ur=3Bn6XVLK_=$oG}WFRqlZ~1R5F+`P3TQHsupO@Ts<F7zq=UOMf6w#2#=cd
z@X?Rflu&P9$Hd|!7X&2I+h14;-FF=1h!km8I4wOrok;s(GL=8gtalbUR&U~3SKexs
z!E5ueu|K+kTX4@ho!nI>6VrT8tNt2k=l<k1nvZ?=PsB6A1*~Z?^A3?Sdri-A%P@Hh
zIbLY5Z$TPD4O{kLae@%oErh3i#L()*;?$~|fqA#daU9w=efPHiz`B5u2D$U&e6^#N
z?|wTmY&se^ln|G^LzsPcLp9&xyFP|8nP-S{tD=AO6Q{?C;C8o_*?3(_qDwj}_8Q29
z?~Az)OL$)F7DqU9%$nTipT<V->0ew9Z4AL~2s;W>k{n5s^XJUT;*9d<*CO(|08Woa
zgn1s*Ior+NtTP%Gn-90{@o2H)hP!z+$bOeD-oIZcQQcpK>H8wUQym{VoaGaIwj+D9
za)i{3uc*eB)9LN^%p_FSWg?@}JU=y`kW%Kj7lpXPJNtR*+Zu+0M=*?XNoVY^%;c>5
z6;uT*zEur;0@gTXVXzY6)VaBAfWI7hgxq)G-GTk(p%sT6)tij}$5iEZNTR!*KmHp>
z#pgklu~KVE5KV6!nn;8BmO=&BHiCPvwwyvZ!0G>csFLa?#1I3wEw`aF9W3lmyz7J?
z){2T8(!*{e4wrRd_+d^#Jwr1+RkeOiflt(GE~s1eV0Zn6h;sLa$;RgssD0^%nQs^P
zTok9^xDmdp=ITEAXl#aE^KDc(I%YlA0))0M33&_Ab1Kr`@1`PRs9zZ$U!z*!=*3ON
z_;vnDSsyFYg2s~$$Ye||av<)rN;L~xY13g&<av_d?~S!L{xzKuy@Ci79Et?}qWsBR
z9+WdO7*0KC9Su-Q_*(kGkz2YogBk;P7|4V-?)`N&knMs$Fm)s#HmH6keCRXMpmdiw
zF~b^jxa_JBe&}47XIy~)a)Z@j>?si*v^{GXs;)6Dakb*pkv5VtaKH*BI4_cP>f7ZJ
zclKPzhb7o&B4?#F+t8GKumG@JoaQJmcvFql=4QqrPe0Kjy^q((z1baUgN8TuS&1I3
zf5cm8+`F0}i4M62h8B_`Q8Fh@a=%YyI$J|U1Q5cc%>rp(jHx|AhQWwYWfmBM*f~M_
zweuCnd!8bLBd0Lp=;P(R%EzPIC#z<cKFPQA?W%$e3ByS#=v&<cfYB@btHlU#^Gzo!
zVC-+!qDcF=GoJt@BRbNH1cWAO@V?+Htj8czRrziFPmW;w7iLBiP{@5r;BeiyGZ|%R
zvXfA@-K~))ItwzNF^H!Uf|mn})%?mA%nId{qWS*CNNE4VNTUB2Bgs9ca<gmC*b$wM
zC7;}(7Zx-uwzS8A;hNk&m0t96%(jX<AUj7PFY9FW=c(9~+>FDm3I`Z8Yvu$vY4uXC
zyv5A82{TS6uWf+6E|x^&LVo!==e&`#xI)v&%^`nSR=gHRvCB4WUC0aj%)~sHcWT{`
z1*jaM!!vD01S%J0-4Se%<J;$Np}Rq}OD90;tRSk?>Hy%iZ2{07zBS&4|2jI|J{pgT
za23$;C3usIih#RwBbv_Z!nVuL$mY2!eJ{GG#iIg<*S)?A-{g+WQ8K_>8!C(RRCC)K
zJa1HR{=E6?9V(9Hkr$$3W^j9lrvF743nRtf+&CBI$-p=RMMs6?(u?Xhs|uPHVWl1|
zR78WTyc3CPEY&Lt4SdN;ugBLRL25#3gQW{?oUuw|A|73w5ohmqb(PJCHl+;r+h<}c
z`=H&3O0>><O7B)71v<G2yST|&CnG)FQ-DG9p&v32%;fVQ%;eTb>Mv%3^(Qk~2Utf=
z1o<RRt?u?A<-P1rkcgnlE6A<5K+b;Vewd!)=%E***fXJ;JsGW{$xnP2mvN}K&Dq{d
zt*yqeV$0rk*-UFWoJ5FjJda!)`O&;TW!4f{NQ@(iE`Cd5y8(9w6qM}f5U~|lnG|YQ
zMVy*;d$0G*9&?e;<6_PQ-Oe>^_oAES(c(43#WfHCcLm<d^o9<2(q_&*oq{PY-`f>v
zEx@)IgL$6cI||)HiI`SH7Wh<tiPqsco3>mw{iwd&LbMXh>%Cl!Bbn8(cffN#N%C|?
zrCWA}RTq28Eg~KQcwX-P?VcR_=pA^UU%dPZl9Tgl{U)MKs$LDI8l)=xMNLa>uuCA%
zzrD<lC0HqVt`L%;+HN)9dHt+F)k<2eUBL_=(EE&x$T}-srQD;p?u!x^-=T7$Shm(%
zq|)S>BXt(PN*urLo+R1UlC^N!8(TGskWGN5`qm-)IiuLbs>Yp*atK3c$kIg^MDZru
zvnwp`|3p$S%2)r3qC9iGYJ)SB^Z&_ENM2j@G!^H<C<%(PhSPkueTX|gv@i}jV&UDd
z?8lf7c9X>?DH+|Z0#kC_L~qQD)i-xJ9``+xX*-Jy*M(HVx%UpIx9{a$II2Q~fn{?F
zJ%_hKsLR(yn;7RSe+mFHF)&)#UhMLy#lvyb#6w5yU4TCpFz-!<>c+UGB)cFqu|j25
zhp0N&#kq-ww<bwhyV`3_i*EQMk<&fJ>&w5brX{pD{9Q++UfM1_<qgR_bzVwA_faMK
z)|6ES<81+sE+rz-nEr4l`F<f}vD~oyuG<A~_Y4yATAh;qCTakY#)gg|zdf}`K7Jew
zc}dBN&-{PHD+zF~bXimwM2>p5)A!hhReIk2Y(c&7hxy?FyCA77n!d6|he!|V@tis>
zM-+aLSd9g{kuqzNV?jabU5@6gsE}bf!%Z^d^TKN8qEZK~jzYUh$`4b%HqM{1DsKnW
z&a<&4bk5ZWmxfk3X8Bl|1WDDR`{L2Vs#x@u@g&Ql7XG9w-kiXPy^oRoRVmHBlDyt6
z1t<kV62<f`0KwVhkj*rpVyo|y2O_z=dDQ`N!XCM;9sIzDb}}E1qf4Td$CC($fDnYL
z-%>*wE;Xdl*?1SJc-<nExU=*$dy0KPkP}tB((94|5Jtv1qViiLF2bKwKSn+EFy0a?
z3r4p4m{HDMGRN}uxbPJ?d1;i;>CU(1C${x)DUlvOyT58w$dg>1xJ_OZAjMA@a>(sb
zr|DeL8bjkU*_^Yv`-p5nT%=;6tbc?iwf|Aj{VcXb{fI8^JiKv~Yw?q#Rc@&P<M{>)
z8mUHEE_QS0XofTIzCcPO*EbTp672GmYpIRz8&2gUQZx2(79h1W)M+MaTOje&&xwv$
zQa|&6!rcguvUlG}3em0tH<2XC^wqK-eCeAL&qw=yF_3hFUweA>919j}u_iC*(^t;T
zw)<nB4(?wI_G>i8Sr1_$nBsZ+-rWUkn&7!|s0&Rzjy-EwPqqPe?mC-#ql(7vsf6|Z
zf)T&P>vg{N%WgVBq`NAi=!}rHbl=n7Y47#yo>@G;i*Z~;=OG)pwsqymOK}4h)-1sk
ztPQsw|0SnCCqEJlEiBK=!)5s7TISKs8DWpPdtVxesFDIC*+qsh8YzDYR=!dqDymmh
z4j+dA$~~wMF+Y)|1tMP1!Zr5#EbI{CSgKs>|6$@QG*6NPCyLsy-_W*W8N@BB(6V6A
zsJ!;=#D(6=!RBiCl+3TO$E9E(G!)-rS3+YoY$&qfprDU|;O$6QhYfpK+Si*0BgIl#
z_z)S>97MG2P&PM^L;yIFf2XiDxT_<_*WvIN5<$~wlrrF*`t_aHv@xI9m7!|>7zG@}
zCsAJ2C|QhJqth>*40l$4Vwp~{ipmDN32`yc%+wbt?IouSIbidBMS~?R<>AXTlpM}e
zaRmK}ApOeAOBN<Zew=aoClRs#@KuQcv*VHXjNro%U*q6zKJ340Fx=aaJBCI@>AKu+
zkeYJ(A~tCtFY@Jkhw$l@p*iQJTx@S^R~J+kuR|-zT*#qy_gr|;J(Xpc{*Slne=p5Z
zVP13=xo+U$d_yGvi#Z>bf<WM`e`n<24yaTwnomIdGVEe%kJeTIWP&ekfkg00R3XFM
z1>1dkl&beYOrGTJQK0Y5w_#xTifmDmzW{}<Q%uAsC5yJ8T~a=({~{(Ti#oqh@=;Mw
zM&_O_@Hx~-%5zAjbws-lime4h*v!~j+6G9+W=tDAP4_dJzrpD)+!2NJQ%FPCfMr4e
zQp+e=vLlnIZ7(zp=pKrTo>13&tHC4PIwfN>mAKU|rYjykK=01rqo}JR*#UWNS=jIr
z7?Q@H+Wf|N>kx3|WwhN30xdEfr@ejhJGP}V<k7~^L>2j$-M@A3e|Q7@4Fwh)t*p0v
z75O&~fq(4v%*^>GE1rM1trI=+snU!adhst>lUntFd}mW$6e0e*O%tKSK&19#S~$j&
zhyWZ<KgdskQmrRtgp9oT)9s2mxLr)&y!f@=FTraZ{8I}6Za!%_Nz{lu2zI!-!6sh!
z`bwsZ6op^so;@z1=;cNGWWLc^GXDsr>=}}M)r%#Veh|uzIoi*gPBKEp2l2_DOauom
z$u62Ts{a$XJbDa&M)m)!g?~P$eIWelInQdWO8@-Vzu)7+`wSP!{NFHY=iLLS+Ob~N
z`*#~N+}L12`^WY87u@)*87Sc`yJM>@?Z4fUf3z9HfsiRl3Dgb&pZeG2P`8VQsAF~D
zyiZ6eeo|6bb9<KF>a^+p&)&j7^|zw*sMIXa+cB06hLG#u56L32yVT6yIU40G1Sf_#
z`l`(a(UzPH)J!cS-*7oQ;}Rz9&la%ldOi8wDY*hq23VZvlJdRZZ2dYvSzB40+Ov2y
z%CvPE?0jv*u%gb#nk@11C8f<U*R_oqSAR!nrWXV8*3@ZEE(MduVjmCl*Dr@nK6785
z<gwHLke@Km{EFfECOT?sHts+bPcnEV)dame$C2$!i2Y_uJka^xjH;*RGbci29A{`l
z3zvM`++-d*`5;7_`q$o+uX=xq$kAeP`k1{&g<d5xw&&9;RDe|b)w3P;yIma$_g%sa
zR*ZFnF&xDALywTUnie-AMI*}_{9Xo8tRMEjyPvnbcKiZ$k?ev%ncS|=QkBaOSa*mv
zC9d}^l?;m)a`m^p#~q)0j_ov@ve?+s-RL;V7`ztdK1nFbfx1LSUZFO9NeICcmq{U1
zOiIjINTl?%7^*H)0UQs#Gi5HRZiMb7IjudT5))&kdWcc_>nBqrhab*Ot-PuB(ZPr4
z4TC2VM_n!MyzSE0#Nc3Aqb8BD>u$uZZ`HNoo53>)1SL6w!J?x8nUR5TPycTNFaT+>
zda(z8U76hMb-=b_8{a^LN+A=TL__98M?{AnHrJKQXn@}j1z(3>r?Yuys~2)@4Q#<3
z$5044OXD$Ppp!#@w7#+jJK|uR`LN`ev8)bDM;-^gg03m2vx8}`l%ecEym1fTkK6&3
z4>BSu{!iZ+$05uKs>^oWXUq4w_HMNdJYB<`QiPhlj8LFlFP1haUI<P|cJy@>vz3^y
zNsjOs$0<?N6ud$bzt-Iw-^GhLm~t_yVYL)#SDUtTX`a2d_-3+w?e2PZ?c1<ApTYEO
z&Vg7MdiNFt2z!5ckeoZNe{pof?pqd?T%|#;UwZ@<g;S{Yln^<gAUT>t#uvYPag5~g
z?olF(GX}c~lt?x%LgdOC$%U4ZN}ewUP311@K%r%_Wn)mDFCpT7#`t2vCtF&!L;F-i
zvVZ0pnLzBD2~N~yFK)!insBH5%Ai?pccswO49op4*o-FDi)?L_JZXhjXMk{8idE5;
z^1K|-4YmIm(D@1d=o(?(7O$NaCYKDPW}A1p*pX;PT@YtDq|5k{sW@tSy!S>V{w8e4
z^HEP&*OxYfW8+!%%0deAa`e<k2)OSbFW&J?@pBahtJT78FG(Z}i!&_mQv!XVE(!)!
z3pgXnuh)_y!>=#2Jh(lB8U0a8RV<x&T@PO~gxY4T_6)Qis<m#taXs_hL~G@+pZDD1
zp_!yUB)W<t15A9{8%OuksjqyI3GwMXoqVc4@%`iZ-|LHMpw|`sT>>Z5!06`}%y{ku
ze!~ajerXt`ejhBhQG4FMv-IT=*SZG?H~zxJ^T08bhSpVNymCdsiS8|xjO-v;c&Sjz
zfqDNu5@6sgS9)!MW}0Nn8Q4%)F&SN2O}6GO9#yK(mj#|nvotax-#k?d)VyQ`9q^dv
zB|vdEyJ*>Eb;wU(NqFU*3BAHpJ>q<C9Qacoqg$=yy<VYYeit9OG}k4HF<rp6M}7nE
z1bO=B1-BaZsAhS5eo<z6#oA`%8t^W_DR|EjGN=BmbTfX@@|KtJ9lRUB>pq&d)zrvA
z>}4I&mhWc^99J*Q<Qz~quAK71JW$BwI7FVV{^JYC4~NUCg12<OWH#WpKhs*=x5m(e
zg01}7QX3*{cOa%ab5oO$Ajwg(DCb`_a?ZNf&(;g=!5hG<(jyW3s?K8-KFko#U*2D-
z>X^;a1KWvd2wyYsa(R^a^AuA?{c5G^;kN3lNIX6c%$ZEc1kVUC-m~r1s`dDv?aYOC
zUWXj(t(<C6wTM4BzlXfwYf*bl`GFiDtt>QYEli|8Z(CYrH-~kqfKAwFH~GW!EG9g=
z$JGm|uLamV7C{6&!VHIQX-xV1<B3U93s~@O(u=yWTJ!sQEs^piqD9c24#qN*$HtL7
zoaPutw<DCgjqgzyZUX(3&t6xT`^y3`Wpo#J35e9IqHuh8Z>&}FkLwlp1AL1X`N%dG
zi4aWCyf8iQa%ip;X=P`<0k9YDXtlZ26qjS@4ub^x``{J~L4tzm?T=SkmL~8h^>29B
zwdhv98{C^cm+KYH8Q#v(meX}KOa{N}LItC;8rx10Q|pE^>P!@9kV`B!F6yCC!PCxO
z!UI^Myxb5t2fN3=93=CH9u!4&o(yX}O(<KgmZ~gK(fRQGwupmQzo;=NvYPzcfNLBH
z73tlqj?z|Xzl;0e@xK40q9RgLSOT!_8wtZ`w;Fvek?YySz{*6~Gx0_*+e1&2qj~*2
z-XJ~_Lg*$E?0X76EqV{fqq4A+>?D5*caHzWn$#h^U#qD@Vp}>Qz3s2o0ZLqhz?E`c
zn;bb2sy(s>2YF5?ZLaEXDaEAZM6yf(M)xA776neiYlvohMtr$%gsAA0#SIMxk{mAv
zvgh{*5oPd&u`W9u=W-Smj(0pMT#m?h5Niy0eBtZ9VhTT%5ex6t))UUVrmMA|_uWm!
znkj(%0H{l$mT+~1JC9F!lxr1g`P8^Jv|VQ6<#e3PmFufon^XU+EsS_9v+_neO(dNd
z13T+pkMvjdAvu;}=&olo@N&%@!p`__S9RAHDjoij4g5;zaCJ5Qi6JG0{}Danhh;M=
z@xaCkQ%8e=csBZEeG2X6<f!KOP9y~?BE}elPtj|#31Y^j<m4tj8=3UNY8-o+>l|jc
zLc_B7>{p879*w$kwdbUBQ_Lis^sEBT;9izTxNt!_MKW|7I!t2Ah3;`VugB$jhRHQe
zX?K2-^+YDvNP2HvW7jB8lXnB-q|E3W77Y+wqC$V-A@sxL`+(;aI>?+!n@(!LsCZ#3
zfuvz(((iDd`*DgK7yjS7q!&NN!!xw=GbDw2VVgyeDwLtBC^on{q-9rYY7|*?d)`VG
z;n&Ts`RcAMt$jkn#p;rjBY`!hg!Dbh?^2BjrnFsr8{MoLbMRK5bt^?fsVkktAp%~e
zp+WdW{FXSL=ci3$c1F>1CdEQja|$vNP7JAc#TwJO1%H{(wGP`ST67X{d&aymKYh@N
z^?t}V7r6e>e-c7>DC8l8m`k%7@pxVlGOn_g{9&Qjuv#<chHX&I)BV~$r$gvkL1Now
z^n8c<c)-faA^nl|om8+?{M~!WsP1}-vmf{DK1ougC0g_s^k&;Vh<tCu916#6L{^{!
zTv&sL1MjnDZhE!dR|}qhKlEDIX()4352Dl`Z&;>zb%ij7bcFbij6i?9`YvGH4(+vu
z<=;BDJ<+V)thz^@<@!pStniCdV+>8k2fmiJR<*tkqlvWDchEHj(`Lz)%k%9H+tH0Y
zZQcd|_o~b~LG&4z)F<y!3<y6qqg$J*cc<&Rl#yI%$AB`mZQV1s5rd$2cs^uvVLm(N
za9JmWs=2oowQ+>pH2l|Usw??UIpL;oa)xTkDA{tb_B>}kz&~PIHD>AU_URFIgWGqX
zm)<eX(BoEhM(!NTg{+C8Xn=QFu$Z;>W80;nE*7_aYP(4~7}z@Ex{l^SqtgKnQ|oTu
z(C=4jxZijA6_ywo$=$Lwl{8n6?@@vO>hgea^*Ig^*lcX%8aau=tz*+p?CXe3r>ci8
z|6Y5;=(iY#Eba`%*xWcN<4RN_A_J#16}m8d;cC23?=i8u3&I#%F5F~Ciu!?**w;$F
z1wUtXf8yr2v)b!uvf!#aqL6pD3#oD~e(h=Qt1o|A7J$dMaWnw(iZZF6JE#)9{k>{E
zd<}|U$XIQtiO|$lTb>t%E85Vg>C3eQ^`pr|qtN-*Ky0bCKMM`RhM2V(+e#Kzcz<*T
z@Zgu7iVC6KrkaZZ>(vh`CmKBQHQDuA<N5m5Rqv`T3mRJvUpwnqrFf<H(e%oK$0QBJ
zk^CqUWfDucWv?XG)yG6dUTOb24zV&(UxwM5)T}}tOb%HB1ge<lRh&hS^uKsfK2FIn
zHKj;kEwwW@?32il2qpdC*1-e{8O&v}>u<%Do@`=10ImBMm!{#AD~qLKBhc}9OEfUS
z#ru?Og~v9$Gj9jJUwnguCAwx2chEw?#G!<Wk7F#4VD4{xZ1E&cNMrsh1Mlx4`FrL*
z7*myR;FFhDm4V{lP4*A-`v2We{Vd}d|8f?AVoJ?%NDBR!gpVNgqH9sfO_9m_OREh!
z#YW7fr)sTrS{l;h22Uxkr8f^Y=ML55TkPBdV3C8S<3lQqS!eVT)egQl^p#&1%PN85
zW{PHE?b42#U)97MAzdaX(r1F1oy?%p2The~i1d?7zP!Pj^<TE4H`RmD8_2JR{qLp(
zQh1Z0^Q!s3jYXvgUmk{c+sA)5u{;C>_aApm{~7G?_xpT*xcN!-BtZAyO{OvMriAs7
z@&|J8-!H|taQjmU0j*V)e92(dGCr)HiDO1JU~IlU{1Z5!JmfLAR&v*LcL$O=vZgwD
zE7ufG%LI>0J;(IEPG~PbIy}xmQtapOXzN{2D$%6TZkAlQb<e1>_UChLz0j(~@^ud6
zncx58&#d=LZj>^CwydxJM}~>T^-htr$$1y+yQNi$XTqeXt>>!hWgEeb(*8fV`T_GP
zE>#wv^dt{B5@l>-0XbiT-Q4k}Yduw|>nx4UaO;ezvo_!D?A9STk2Yw`DAe=QE1eI9
zlepZ)Mn*-eWLjqIb35y6JjCN3Tz)QAb1$A~JoR+6%c`Cf897e2R>y)(Oz>z=ck9gS
zrxxR3VD|EK9I+euJBF+sJ9wP#d>uILn)EY)N33UXK5R&eDPE2jGVhLm&A)?nbVk{)
zWqPE3_sf>e@P?jsmf5a^W97d8Fr6!f0yk0RhO}0@oBN~jwX`T<O<M1&&7aB1F+5wa
ze#HXy$W;SSvY)gh!fQCR=xLRg5>tNed%?*HqonHTtD4GgAG3Bl2z|$JVNW&CJzA-o
zQGPNq*)?3d=Q+<2&hYN2&kVEkyWn6bICp%{<lQ}tN<yo;9g>_8i&*Jl&Kdpgo%~w5
zW8Y2|{{eLW%XAc;eW>1onqevO%io}3N&YRxP18zN-IJthZ8YN!-yK(lIyojC6;8rf
zkX;z~DYwb;R4B7$yR~|!uVD`6!s+n*vK<Xw<Q@C>+?hrXS0hdvx=2G0`#8<1$PdpK
zBiX7lM4LF$ciN}lcTRp|W5LxxY0J1F!yD;!Y#Uxfy^uqv$8y;musB^}Nju#+%n6y%
zY4ttJUGQdJZ-p6lK(I4kb=n*Fs7smJmO6&fFV4iLsCfiW0K~In?ha1rPW*v`8HZC&
zlWk$e``=0lw#q|>KN_-7Onu?Ucw=>Zmc4XCE;rIsDFR1a`<Nq1>5F$?Yz8}Wncz#m
zoNoSACkeSF(3E4>A`o_|(KA&Qo7BKfc{&~6<1c*;CLq0Q@RgsuH?phLPM($fSrZ<G
z{uZoQgmT$V>YJa(0G=7TC`%c;?CGUgnI;8w(6HHn7lH%J;c;|ds?W4uj`kI66r66)
zhkP-Ts@h4mKR5eS_`}Uh(<u?pS!UO;3d3h1{E*;3crE;qUa;w^Lf-1++U~NH<l$~w
zL;j*A(}k(_pE(+?)L|WlBNGZHWAV^@qUOD1@IqGz<~@!8$5lIAs7(Ht;>sVFOf;Il
znMs7^{NLqhV0Cqz&n3<dOEx8^M;>*ETwoBuw%e_fx3&RcqgSVz86GyD4+-047kafp
z42Ht+JfH2v!csf?Niw+efAwmZU`w{|8ED89^q?#?xxJURs}%;e%cD+r)mNJ@pT=th
z@%L~#1s|`XMXAG6c_KA>-;b*VtA!J>k})|PkcXY??$oeziSqffDzAQsl^e-F+dHC>
z1;4(3k-+NB%~hGTRULziiMP*n!<L=uYLQWs#V4<AH{D5ZvNquBWMP*y%yG7$ks;^b
z;A)?>gm{=4=NKy|AfGQzOS#Hzen->8>+qL>uUw4L1*@q>$&O;dt04LZxuq&UK*j<>
zQCJN&13wnbB@nrN6kpp6O{cf|@^>c1BE(6<2e)2Mv24JKJ2pY`cQVCtbMxNve<xE!
znc#n+K_)SG9E;HEQJvs)I?_~U2xSSkrTcuiTi4osI=g^SsM>@<Y&&CwXgqrK&OLrP
z6zUo!yJpUsk4##f>Yw0LzY}Buk7)f?TJ5qBh<?rZ;1AT~etf^I@?AsLHt=eRy1aRA
zs#mSeH#agYmT`<EX(C~*^k{2qMT$=P_Eg$nZmv8A8o$6(_uY&iUE<2Myb-z_>SRl-
z;}isK#_??j-Q4QEbpDJ6+M5EJ;5kM`#~##vt4RH_#%yDT@u|qWAL62&Rf~NR{%SG1
z59-(75&nc>U^V70^|evC^P4Q5xi5(dOZyp7Y#M?Q|H+=jd$5yOd81vdr0<iNFJ7qD
z6>bY@41>UUqbXf>*PsjMa<bMs=Cp!(=B#Y|mfI(T-s5ZU@sfz-Er1R4$;Nhz8>c9P
zKH~;bKQ_jKHa8_ka?gOssj95$i#InkFh>c;`8>5}B&L$)^BKV>9c^^*9FuUylG&1F
zPJ1g!3}ssVKu4#IO5L4?UZeA6=eG8WKxh{Lh7Q`V&MK4?Ab{%)%Ew1m&t+-?jw9uB
z=c$2m;<|6M;5K*wA58o>T5^X#ON(O7Q=!{?GrjKlr9#W`c+~9;pW*8u#&U`cXtAQI
z1*R)wyDCPp`7z)=p)&dlTnH+R9PTCeSKl6Vao8)L*;<(6)21VDu-T;^FLAomD0HG?
zmKw2qL-SA-jz_c$rx*xv>+<6%Mq}J>xsa1r?Xp#GdJonF<4<FP45(OWnC1uPbk22$
zD+UCx`x}B*nr=dl^;ZL2>{!8H^#5V27!QGcft^EE^h9CkBBUTp+1Sqj(`IdTq<)rn
zNH{XRl&PWIgH@6xYylTlL@PUc{Ke(EwJ<x}!kL_tHi?N9L_fd$??y;$13lE4N3bu(
zM+911RgM=!)0vDwE8zjqlya~ZQhZ8p+OQsTkfT~dAMif+Mwco8()QWI7hNlv{fBTw
zo5ThzP{ncDuy+B_dQ^(~<o)M(FA@!^hVSB=3peu4^7F0g^1Gdf#PQ^beI14q&8v2&
zDdeZgJoObBL*z!X=fk7ihb$60bX=ha7Hjhv=or6lXKqgd%yTy)RW{Gp2yvyRMP+a*
z&$ef^oNWr7jY4oWI6ohn|14ya?>KkwF;kC<l)RgKogNb=lQ9Z}rndx>zE)F2s<!0a
z#*9EvAvc14Hfal@GGnmkUT<%cn`xS*GsU<29lz<lL+15;b%!kY2#j{=rV#v+FI22A
zyW2;oZ^mNn`{JZWm~3|v30`RkhD+ad{+euLo%~BbQoegzHPvD#NxUQOMJ!LAyp7E=
zTSZdSK*J1$jSJ=I+b;1ui;bbp@Zx=0ldQ%S*<E689o0<o!F!m2P!yx#x@r&UtK*@8
z4e_E(LimC&%ljl^ob4#K=~TD5)+n74v~PF}@)?cr*G=do6a#BRbn(O&)|#ImO~)=M
zbo{!a_f%eP6ENSy3Bb-sdLiL#bkdh2fL&Q>^Ils;ToB`C8kGt^E->f;w?+Yeh%L<1
zI*CrtCLLd3+JGm%LY&<SlX@&lRVGo^I>qxyn7OQ0D6)8=b=Lk4{xeuWhJS)(O<%!v
zU*)<ubL#FTz*Q8n%FNuZGa#uZ&mLWSGaMv~$TuK1H&b&bNO<L3n{OZ`f}xDUtx2#A
zK6lD4xloC@VbG-7d@^`szXV7%V1!PF7p`miI9DnA7Fit;6SuOTl%YfG>EsvLn5u`2
zLyHx=%Az9rUOx1P^Ao+V@ksQJ3P~z;ry(Y!O|O#N-E-f!yCr;eay(g$RLm`#kPg<F
zOy#$+vK|iXx+v4`$(7Qo7uQi9PELm;ja)<=1$BEDYjL;6d@4i7I+;^G5>v?%JKvlF
zHj!UpzrUbY?^=du0mQ&oByjhYoI|i*{q`&UcBo8Xd)uLWNEw`VCp9abisn(jiXaiY
z&S56D`X{sC%uHrL24!q)_S>W_xL{*C8#D`)g69>qNk0Fk!0Sg%KZ_;=e%g@BQ!I)%
zqiFO|a$?mz$@@}H8&Rd9#wN7h_iIV>o*-Ume6f%-!)FmYOOXuc#v+<U;^onVTj!7A
z*(IXtF(+@g^RN+ydw7o2(ai-?i_l-J;g#@pLjw;PVNE<^r@p#L%;H%4OVeq{S30iG
zvcOYC17nx@kJq?T*#>{u6Nv&*O&LHC#<4Pq%{-TjJwa*9TII^<+Jc1!l)~T-D_hIi
z9Wi}BuW3567S!T>%uKg%LbINi9Es{mB&b(cU1biFoTv~aCFO4Cnz1`avw`+_GmF@~
zHlGKD)6&Z@<Y$3*`0kIsPU}UGIvI$k=Ko6RnW>5R*&}N!lcKB7Di<A~`|7uQ5bk>Q
z>PLF<LNS5=!F-~1@bHjAjPC3H79xD)wt?~PgnL3U0k$s(Y8zPjDz<uE{a(dLl*Ll9
z#hPye<iH%x@)^*SMoSK~QAp|QZ|EB&LU|i=RYfFE-@D^4u$RV&jg{xnsxvQgfodil
zi5TxkFWHn9R3*>pYW$R)a+Et|afV)MlajX^p3lNth>1o6X5(f0E|f$NHLTG^r_CUi
z)qi+nZ)**n&p5Tj+7kaUKkE~7>+I{nD(*y@HfocMj`%#B$@Md%Ghs2w*NBhnzC^BA
zDB&9IT)7^+R#z@<MbhdW*ZeH=dfjQWMhU2eulshp@9RGFP|yr?r+xk51f3e`Yi)e$
zlD8&Z^w|8@anmk@n4IF^WWDEv!l!bH)Yk3BE@PExBd-*cWXM+T-TUEPgcMce=28#0
zGC1!?hstS=p}y$QHV1toPDQr<)vuanrh~N$%qA?3`Z8?a<M`dS;M)A+a!=xyb&f>N
z_9n#x$yjhff2FxXX}@IN%h(Ce;IWCH)ksPN=Iyc;d{vIP>?zO?jc$-&#0zSs9ob%0
znR>Ztdt@J5GOQu5*fx3hGz$)3c6^*nh~a*S=u(=LB|RC#X!|X8Vfrh@alE@>gEghK
z;VKOBv<Ol4i>z5T%Wi3SM8XAt^x!Xw{)@45(L0Ff^LQ%lVQyJU0gbo5bi?#)LT^V2
zcGUO>Ju5EpT!tU3@&N|AteWXt@>-(OKd>I+Mv9M4c>0Z^=(Q6Iw)<j`)qNrZ5S7c&
zG|<TDfyn~hA}Qe|rIZ~RCfRzT^n}!YJdxbhm2-L^8dq4mYoE@O{vv_1RfhE$(x>Hl
zTd%d8fYP<9oW25P0WW(V4Bu1>H2#>?x~sXJex+IXYD~s-BIRMOe3Xn;GO2VbFCoNi
zNKa~+RNwJ}Ccl(PjUyb+G!l9_A(-DHY2$IrM~dP$Ym`+X!5~Zgqncw3b&C%D2Q|we
z=uJy31dqjk02(l|CDkTNR7b=IZ@GQK&yS7QFvu{(Rn<O{D`3v)o^;~H#JdT52#*?z
zS}T{M8Dayt6Sp-#YnZqFX2gTGK-Dz1JTTaB9$syO@H0U#718GEr4nZ=Zu|99vquHA
zSANv+vY0iY)nyBPpC)`DW#3O#Ali^O|BE5yGQ$_|Xc52AzerCZf`P}PBZR)J%7CEN
zB}Qq@UYWrGL}X*6%plnAH5M!--nG`BAMj}vq1^QFR=J@m${uG|R+%9N1hG{8p@-3`
z5<JVS9NrGmJMH1fDyGT3r7WNgHye7b-0k7OR3L>y(1$y~>2)Tq{Ex%!!3s|BI$zBm
zd74W6cb`<grPv5>A9}a?G2)E906XWRbifFs*o?aXJ58R~yRzXJh3z)N9?y(!7Y11o
zhD9n3xF*;W6hMGXA@-yN*nP337P<t@J3Z>!;o?8&OR~#F$^r%4(5e5~ui(Eel{CM{
zBZ0r^hAF@stuYf%O-C<8Z~o`?8#BTJ><A-OmhjWX6%g|1%bwEl2B`X|?1EYb0k(O&
zKSo9sasg%;&3JLCcAw}l$pO7D=YM@Q9IL=1*8X}JMJ{hGH{WGQzbc=ed!kaXra1CK
zgc5fZQ4(gXOUWG(BFz;<y{4Foq7$IUTwOIp7usv1s!|fzS(R__29;#<>%$=V&+8}Y
zhxfVxE5cvC2!QTTx)uTc_D@hW?VlEGJk#pBK|`Y<TCQ6u1-I?Aa{f?4qwVHa3(taZ
zsV0`Vls9%#4T>e|Q$Igsf%w|+01V0{#7f1eC%R3;wxky<jK;b1_nmZFQo!ABi=sXn
zzK!Rj-fWOZQ6<sj2tRwwk(;>$^SKDUO)i40z!bVYowlAo3|M%O26si}e!y7auJw!i
zqm(-P_L>=Wp{1ouVEbnuRr2?~J5Y#TaL=v6@T{t^Vd)*9;45;kD|dua@S-h*=#{Fc
zeh4T;H=3i$v1}${T$6JD?sN2;jn@<8s=}GK4<%;w;(C&1g&`Q>+!nWQ3Y=W#Ro5GN
z4GJOb@CXb;&`r9|Q}Um?Lo2Y+pZ6$B4}QrVRQPy4+f-YTf&aQYSdSSfNJmThbnJ~V
zP$6Vx^g5B*r7k6brDhaw-G-u-xN9Q5ihqik8UD1fq>gsf78t0)HkvBHdN}wQ!$+}0
z3RFq1C50;%cj}~Ei^UKZr23EX0>`3Oa&TF#{f_r9iRf)g{=VZ!E@^m)xPtyfNwQ%d
zZcMb%;!qrmIsGORvtjY7{?WU{9$HPSxr&lg)rR^%ryv?U1fu|f`1!*Q=nb!}hrDRN
zX)|}lYr#~2zd!Wn*!sjT$ts^FWBmG|_u(&$_wQ}pn{xSGnSVRp9=^fm!91mtGDi6Q
zM}Pn6!&P?nA>|wlM40?Lv*NGs_}wf^2fvHnZ%843&x${<rg#qm5B@Wk?;hD7k%*Qx
z&ixqkyQ=tDX9mDRO?1z+4kH7HT*K1?!>h|7+Inj6k}-11e5JH9C=>VMmvv6l2f^y_
z8><ujIegvZT%KrMwlFcQ6*E!jHHF>V+#P_gxz|+O<Ialcff)1<O0s(>;~pDmHR9NU
z@_YCz@;x@-DDvp=HNtL3q?UYTfZU-v5dyJQBUUq$(B4EB?-(|9y=(Qk8CvDB$aoai
zp(v~QX)#$FXo-J0ym$A%t<++~X5(m+H7+;1z3#p!uN!b=6a7a9&RRG4cc~U)(Yt2M
z|CDMO$h8%Row#R@C<*XHtNpJOoJdmM-cNtjYROoljQrnb;M7CBlc70RDbJtVr6KKN
z=H59ZjP4z*n#kWQkD0Z*8%nQpHsPK#gG2ZP>Gy9$bedfI5Z6|6U6cMR>xMPbxj}7N
z{|Xm#r27=xov$@{^I9q`Q3z3uyI7ZN&xapOHnhCI+1=h$I!%h1Z8UU*b8@<_*Y*d{
zJ6)e}NC+Vif2=PuRJsk6lgp;*EkM5*dh@dJ9>wO>%gmJqHwm@!a5Uvx0Y&+A7KT~%
zKO$*BUWNc3c!Ywdv@njZ<eObzxZ#BF{LL}bzqVMxS|(+hqvFYb6GdTa<yo&Iw(s}9
zniZ3ZH880DXDrR%!%_ArLVka0p0`<XZ;o*{7QOP!du0Ygl~Uhw_4{XS(Mlw2N9@%q
z!>X8Ykc#&y=z6i&cQvPi)sW5JVlI#udTh;^fGb>(QN4S^Y<-e-a0lM%-UlcUtI(@+
z|9YIzwEIdOy*#oF@A^s!IFwA_I||QZ8oLNjJ^~kIhK1b~^`z`^gsVHTrkSbm3Fvk8
zA9zsH;#|Jhy17|Z>|@v2b40zWy&AID{Gp>L_Ud|G7+$@F(;6lSi+iZx;zLa+*Ktu=
zVeDSJ_u671gz$jX7oMj>wlB^p?33V_Rb6{OTtP9DY}b-onDvyQ(NdTADD*6-2h^uM
z4L{#^@t9FkD39^#kyYCJM0i#+R4q~1wP_>l5m#=A<*T~9vqvr`$9`J}$i>I@DT?6s
zCHLTv_3gx%xU>BsiceZb(znl;xj0__U+tY&R8w2K_Z1PWpi)$-H0dCM^r{r;NbkK!
z@4bl#BB5-0Pmtb`(7OtR9$M&-NDI9;NhoKrU)_7Z-x=T4ITz>R&B(~bTv;QTbImp9
zn$Pq5|EGB7t(%g4MlYir%B1Ol@I`mmjem<G!V5|#uLd#^EmRDBMOtb^fvJcEj|1bI
zY(yB&j`-?-F+lNYofhkTQ=lZol=iUp9WzEFN5S14*PcNFaDY&Cz^Q0>@^R>pc1zM>
z4ou5X`u+e%g>_%J35Bp3RBGEUTf%?zQ9im!A*671ak7w|J4HeOSh8ODWeFOE@IN-7
zA^0`IWTJ0qM?;G0ZQq7j$<3ujnHj(v!m3$qZO7IUHT~X4Yg?%=fBIW!1XApIAQ<bB
zcn%U%FgT5HbatCC)+MRd@HU>VlwhERmAsagAT1TD{rCzrU1+u$t%<x!yfP;v^dP#T
zb2R8K+)1|&jLMmDTQV@{ft7oO<Qjx&sgd=7b+`^x_c&B6O$gX0_DS6BW@3H#?8`%m
z+q@?S^W&W{n}vr{)(3iA&jKSRG9zBrC-pBA4#gl|uc=KLp$5N4=`UAqhy=t8SogEq
zOqAAt6zjJNcArC*-vJ$B`D&>o&(hqpCiTRg?d!^Cr+*jjus7Oc7F`|>@4j1#w}OQj
z50recA-rY29sNo8fvD2<C~x6+ni~jaoDgQ~c<Q9g@z^Zdy)ERYKOv;HZcx5J9|YJH
zK}l`T=)n^+z4w1{NM!#%4k?W7!^#v{FGJ?dlY^W2bpuI&N+j;?sDAj_hK!3NrNX?$
zXnRWYfW+~Pw6lDm`tnGRVlA?{L3JnCQ?;+Lx8Mo$XjQ7P(1$nZ^X#7F1i864e<UkV
z*}HGAhPyl$x_ayP<qW-?VrT|=m54NSlZ+Y?K527SNPh{RV}+dfLc>qHFa=X1XU-&!
za46mi!6?LN2f}?fs@h-|7TC$MTh&<geQhG(m;cMzps5$0?~Bg`eYn|BuFqP7;bDqJ
z8|oMVmnDHRQ-t*c7#Oy&aq~sIvx(L)pli~Id6e9y*`Iff`Mn&YA;*@D%*$Xb&!+gl
z8^BbUFwutYe{VJ#2H72i%Z#h%0Ol{>44(qBCXeP)mjTd(P}9+jgRb{cD-_<u=)Uix
ztco-$eTm*`;WZZY5AeH7-naKUlA*o$asR475O`*nD`Oz0m9$3c{}ncA()5S!r{d$f
z<$+J<ezaw`S}Rsu5Au1HIk~dVAK}4CLQh}p%hJ<hX|{uJaZk14q}6+bT=_3z3N}KV
zlcAO5;bt;Glg;6>F`UCy&+BNZ&VkR{X-Zh=CAXhtdU!^U;9qtlhR7UWRT6HZAymkU
zpu{rUFA(?($qNbO52!AFuVr@KHjy8V_dTMhzmC7QP9ZYWUC-rPQm>zrm40x|<Tom7
z1E`s&$nXq(1eKAzrj%VQakVWbqX8XZ-vlBO#CFyGT_nYy7}2C7lSKM+&jY78M4NBv
zW}=ZfNPc>(3mwFft-i8T)=qRV@;GCcg)#92YWf54G*KOo`KLh2CIApf_3U6TEo9u~
zep|>y-j(#fhD;fr6P#*C&Pg6k;8lD)@~f%eWyCXiHf~L8IKCI|^`T?soKAOHwr0%H
zZsOkBY)SB~TBO&=#gM1gz^JdZ!HFo|^-wimqYN!wSaP>Xf68@%s*M~t$Xsx%vUyvb
zYC?d*lzoAnYUkaadRom$6vHns87Bo+)oF{<3W<nwE)%VlUh13F$`C3eMdo_+{=q>c
zAV_up<DD(&3IWMhws1jc5+}HOPYzyK*;7}EK;~K#u^2~x`AnQ=q=M8Px3r*MuJV~S
zJ5iH>?rIO0KM_dKPkr&`RhW$S%U}zn0}$zblNJ)qyj{uLGN;+pR8Jvlr#Q^JnW@hU
zZUd)Dv2T$n+k*o?JP7y+P+s(2uNg_^at^=!_H(jHy-dVDEfYfBN48gnAMSA#Tjqp6
zJQN)}DVf)d=T)wdtAA9~tJ7E+3T^n(<ENbxECkjtr(b8*98uombDE^r6o~du!sRPs
z>1Jj5F*tr_ZS7$Zy1{1Qm%fTX!-esthEkWTM(9>ygf<E9%%^3P4eh5qbEFLmd26D%
zYNKrdCwa`ihksd7hZEZmdgVM?*`l1XtYIWyGU-7x2#h@Wq@&8*R?&_XsGNQa7Ha}+
zP1k)0E%S-jgx)_#FA=I1cs^Moc{ms*h8`Ry3M_Ka`V!+u_Yto*-Aw-ZJ0dv)7;JU=
zV_;rIV1ND_TXYewiaZ{fOqBdQV+!@@=wSZpbn)_#qg8E@lXV*x!|G`4wO)bl(f-A`
zMd=5z8_Q@#BQO)z$#{Ya`$Q$_=08w06et_Soj^?bn6S4yQ!YHb`&+%~_9Ofki6>Y&
zSd!o_9B;;l&cY5QV`bzMO{Cv%!MW*SbF#wopshJ;o^&Q%c;Xp6GAbK@EqfNcD*-%Z
zm%<PdP7reR>uK6a7O;VA=K1;Rq>569;YBWW@eznA`52I4eX&vfD&(u(rfsHO3ym_R
zgf~PV-X3cynRg*pE_L=}8(5r&4(KRDMPV2gzyV5?13M1YOaqiyH^rjr?#ZhOttpN6
zHOHhXV;xKMTa={#WozTVV2U|yC-ZT?@H{wM4OAa9EA8j!eV?@NrzuYho4NFu@#SsG
z<KrE$U{K=*a)JF8QJeK_8CDY|!*W6NiVwDTS<~H5sDsyQ5KZj9G7JDk2W-Ci2RbX(
z6q8sf43g1}@OTA~IU#3Td6oK%NQ;zS!P1etHj}iI)Q={l`p0JzojwG7n#n@L!})x7
zzbzs9#UeiBK8-z<ooe!adp%NHL?+~oymn3DY4?7-+0XNQ_ZIN^xI0g?`wF9ld0I^y
z*A0XPjsp#Q*KcHQR$6pGAfa6qFxF89GA&-nB5$`a6QPl__wB^7^a^?N8>rX)_U^Z>
zm6VWR;qoK=jLy2cWYzPjcGz25Uu&T>k4j$(erI9>V`cY$z;lZ$TSZW+>CFhd-zy&w
zFl9Bn5iYPnd|1|TG}+I&6OWgO{N*(j)4lTXZPl%V9+j95B6~0*w!Fo47zeXv@F*ga
z9Syyx2x#+ZP+yv%b)hIfD%y$GnS~YoVcJGR<T6}X?dht)liFs`;mZ^Xej59XB{fp8
z&Aa3;6&HplKX0przIQ9RXO>cCu-&6gT^hw7d~YFGxK>l1Aonn)+NMpQ5<1tUR|=of
zEZfYcshmtci)w#V#rQZqWFn}UIj`8GOL%gI*1o99ftZWmXbKyto5du^9rKtT5MMa}
zyA(54dSY=t(mqVPgKl2{h2iTH$!apVdW5`zc0(m~$?aKnVz<}$W83)+NvaiDVSs4v
zZqPety3q$braKx*UBViUhJu|c0Diz6%pFanrXgw0HvTn_zWXyzjCUzGJ%<@<P9L3;
zjbzQ^B*j+})rfUGH5|fBPsvTV1eENqi~Y&<*&n_Fl4*a2F%Z1KDwA`QNT(w|Fz!%K
zv-=6{SoiZW?~=VjL(^6?H2e+rT5;G1N<!o2s6$50`&_0BJUYU4d2=xRoLC9}E<MH8
zdH;&1OI~8O3E_`q2aYr349IFV_l+6m20zri(ik({eH}$WS)#m=li>xp<n)kF7V{KW
zJY+OXoih2*!@)+=@F1&%A$S<m<?&CO9O>f!W|JevIZFrIXcCQ^R3lR+mm9ZwLD4(h
za_N-Axk~r9<f}wh_@m;sg=1k!8|(V2pb-C%2xYleFa8-(aT~k*RoP*d`&_X}u0N~u
zS}TK?Lz^k&Uls#qF@+DUG!m;lLm=tcPgF$KXZ6h+R};4%lRV^Ffh;h@nj@d%;;xZj
z<r*{ZChHc}f3v$a8Ip?30zIu5=R1iG>`r*)nKtt*;!=oFYH|Bk)26f>#$SXy<+^%I
z1^r_+bPF9za0MGs1G_<y+9DNW*$w*1w9051m3`Ul?x0_oA`K(8W6y;DYi~~r*(b}=
zOSd1xfwcQ+IeAv%%~&jPv`6;UW~TkGT)wrDm!RV1yFdfi%>k_IE_;YEZLe&I9R+Dr
zN}6+QP?yZ7P<F5NV**RebiC(@rJ!)**{-5)Qe=$sx|!T<VK3jPLwn7XKwIUPbrST&
z5rJH)56%}vEYCVte5$bg4b}Y`i))K7e>v>Lvo$<7Iymys8o8FXL>ej8!!R?JB0}I`
zcf%o4EGx0;{<V|n7|xN2>H+5HtTEQ~R}QUIPlra(v~w-cnW+3qa?TKLDRvaycy(M7
zHl=Z&yE-CzOdkC&Pn}#r=))J$%>~o*(ZeOB+rviVt#bV${c(-I{dDe{)Fd$Yn%|_-
zv6PX#qoxxj^Zb$TPIGCiGuKC&@&zXpg#GAjAEQ+;9*#Y5ha%u3%?`^9pH3$b?4Mr@
z32A88csWXWM4!AfoM`T2%ZBp4`(RFJJv57c(Tg8eULF8jY0|w|o1mh&D7*vmRB``m
z|Hv%yQd86+WdASL5?U3>aC839z#n_O=v}5nPX#yS6|E$o%wJ)EsNObw&{?a@2hL_e
zk(?CMZsQNssd>v-_)nV}nrY&kDDO~=0ZaGllBg1DR#(+rr9>P#eU5Usq8)7@bxQ2E
zZEtfzb&?<Ben6ao{lN;fOa$Xxr^K-m!b$`V5|pX%i;g|*215STHD%_!i40zng8rws
zlsTMLXa=8`YbBD#-Wga;^NcC^JZ;*ejUEi%&VDDrud+whazxT#n{-~oFY9lR=zwZs
zNQwq1$AGXpkISCJ#PBZ3d-JUL3DT8I9S9#Y#7YH^?f6(^Xz7s2#Ob}n9~we1Es2cw
zDAcxWSSMI`R{dfATlJQvkd}zmZpwLpXU8n7UXNcW`Kr6mjv1jHF+~x#v#@uE6Kt#F
zIQEREQTWWpj2wto?G_>mZJ{=Uh6sZl-<J$;P3dZB=fA=nZj#qYt9Dm(eNyXnHmdxf
zmKzMB;aHyue>kDi{)9BU;pO*bXVV4E#8vCpIKeeEBK9+^K$za2(W!!}0RvxFMM6H{
zCh5FS+LPm$p9ux{AUavl#wY7Xo3cj^;f4{!iNvE$0aml4;aBZX<J7baADe0VgnuRk
zLVE?$II+1C?J8^olYJ37tLohqubT>am5R9CPBJWMNq8-D#f;2eaa$^WnfntY6wizz
z`bse+cz(tC<kw8~SG*3;R{<8%1<F*%e;H5zKIWbdJQx12o@6`;4#g}Mqgxn=xX%eR
z?)HZAhts=s$W({^1WLiX=Y?cZcioAb0?F3mI{g9X9E1?gGwHQiy3!%?BCz`t+XGT3
zsJzsbb#X(My+d@9*99U?MrWT7Kw$g1J0JfEJ~rF{z()gfE{<RG#NR=0@#;$x+EOnr
z{{Mc+{W8+|f9EW(1R^Ksxz^awY(IK62l=hL+mbw64&N|Orz)>~2-fWFJ<&ueQ}_|6
zoH{3fjI!-@U^{6^FD+_;BE-F`%VdItN+a)XxOri{U3MMqJUT;GAG`U%gtJ`$4HMJi
zTq83kGMah4OEz0@Bp68ZseVnq?`DTlQ;$z6E%zY2`}qfU`$=u4I19WtA7GZ4O**J0
z85oKpeYNF*Y#&a2?Jd>L$3lcE8=WW>AVwE(XN2<KV|2MCsgZEFyb+zuGBi-|7aV9|
zFL$h~Kq+H5N6Bouwaz=hAcm*+w;)e?;pfq|L_cHXQ$S{>uBt7`Y|96r)vSm1I53p>
zc^kf(b3hOa=mPUl4%HxI6is>W?OWWP^C1yupvz35{AM^WtIwm4W6|0}&T(zVn}jGV
zx%C6ZtbU~Q-E^n8nENaVN-apcFr}=V1Y3LZ$_<7DYO6@RBVIB!0tyikCbz8%Cht)n
z%*D}`aLoXe;L6VtciTx3569VLitk^UU#UF15iX%Zk9+NYJyC+~1MbN8l2X^?7OBI2
zNT|~n428X8!f{B)fr~FDi%*y@I6W*`hdUV5Qi-B!50b!U>y|px195KtsN-WW1eL~k
zwuPmz-STJBSj_Z*ktF57N)NrK+X~rGoe_@u>0_b?(Faqbd^vkVbR!Y0oQgzZo;E)F
zM~#9RoV*dIKT6bJET3|4XOYA)=H&J34G-Hoa!^ix33z2)?{0z^svJp<rpz#G#=t1?
zD(@G-f1)j{U!ON09+H%PeAvpGuPCdLlmXG4*_*D~T@7KzIX?DV-)lncRhfL_wmztk
zAsN?pSt3+xYq)DY(RXdX9-Y%Lo|tRzs97tNM1pfoO8hJ?7Gss!v<Z&8auo;nWhRz4
z%J0U3N~XysrpkVFZCKgW%q(2N?hvwED`Ymxu$<G7>zMN*(EH2mQ535hC&ur7347ye
zO%!#mzw*!5HrlOR8Yphg4)?XBQiajL)FfDjn_oa9ey#~VoC#V}!7du@t9NmnKfFV6
z>aHP8vD>14xO97X4{|DKZ&X<}!huytORwS3)9|VBf=8twzX=G59D=NJHEPc)>L5yt
z-oCF+c1)}<w&99;r0j|>_Nlk4ro-B-@}`!J<j5gDEnU2h{XuK!hore({4$R{cFqeu
z=6a=`k*_<0H%DJszYVYq`6^DLa#0Y7x7N5_fAijxXDZimtF9fUZFek#cP8UpxhIx~
zk49hQ7Dt^eI@f>NS*#MaqDB>#_?k=joTa9@{HUoIDQhDXVzT!+>hPF0^Y5zTVultD
zMVus*8|v8vPzXsIP34tNPf?CEJh*zaj%pPc6+Kp6&+}-dvu$X$^e^h`Phx$lxRTz{
zKe=d~l%Zv=J}!3d)@UYF#)DSaE}GtA!a*Sr!@>=lqA!j`QWMV#6z@R>M9ja}sUEqn
zle4N?jOGl}1cz?L;YD|+Ae#(m#4~{_IdYhmdA>T;p^1Zg<KBudPAbK5^LB~ytaC^F
zr~1O380&75@q{g*=gC93-qWeotC{QNESS);!1+~m|6|nDb#N&@g55h?n?l%9eLZh@
zn`WUicj_T&(wNz2leRosay>2+yQ!MJQV)m}(Nfo1S<6C;$F!an8Q)mm&(*R!aK%I!
zG@FA393d?>hYk&`I_<2@AM=fHl^H?Z;2^ADYDix)NGm@aSI2Hk8GxTZFA-9Ey4cc2
zXGs~$+uB55>bn8ypfMrs|Ks=qIYS#p8L%R8qI;Z&St|BG#~;xQZeK@;tw91VVtvIg
zYxo*Rm-E|&hVlGkNYCV?)s%&mnQ?xoVzqULa-bTn>CgvWOg*nu>o}HbG8g-p$K2`u
zy;>nALcGzLiTIwWJ2S+Pq$U6SfwPxI{?5+f#$Om2Qtx*vzj1-N?Rn?tTPGPM8>aXw
z#*{|3)DKDWA6b`7r^3KqcB$vppyA=-GR{=y<Hb~Gg*%T(f^F`K7`Li(iQ=z0mUd50
zc3ZzCc)RW*%{!JYwzZ?fJrbHviq7;?Q|4Ua$2pT0Kg&K}<>RcxrxAa5CCK4AkHA(U
z48M`}Ucal1|631TS7odyl<U3Ip~$|xqFJ`%$`n3CAv+55mBYwIZ;PAk8cQ|%5TCnB
z#~-vpQw_~*<=e-HAkfBU59Ff7eShl=9^`t5R|D8(MCOPeTC+qKnw!wtKquwef<VNN
z!2@aW(?;tFcRx3}mp<BINu?ey_u`Jk_A!RLDhgt&s+g_3v}Q3M)m;M}z<zC<uC8n0
zrig{vhUDOz2iLA1Mq~1+n_yHym!pNpOL6f=qSqd)DPP;U#S9c^h^`t;57*_EZQAs@
zP<<^dY{6Q)2~rP288A&zU1|$e>}@Eq${S4U`|lLrn6@uH*3#F-t&AaY$bjc>a)}s3
z88<LP^92@W>gqX^_0pD0{e2rU$6OPatZb2E?vU9r{|1Z*7hi9bm9?HJ&q7Z(5lC@p
zrp_#G4U<p|DzTq7V<UPLhO-7qbTl~QzK04JPyiY@h~Y^lr8dg5T*Mw%kLye^LUg9W
zPJ73W3{n|E5(j%oeOKt`JD#JQh47(|W^G}UA9Hu}A&A{C9OJfBUThkRd9^XsY^<Eh
z2_yMAabX3-SFYTkla-WEm$k}B9b3%n?e}&VlX%4nDzwY$cHps!Dp6cI5YANFnkHvR
zU4_Mq5Ahvk$W8v(BbKn_>Tx2_Hw({9rt_H1zOCA$m~Q4a$__GOuaT1G-?BQjX3X1*
zp&N!->Ee}z*SiVf70w#eI**{$-FGK**YEaclXV*Fl3!GR;is1trt;m}?KqIBpaI(-
zaLKq$8`U<hPbg)!-Yf`2h@EX_90>zA$&<lmaV17(TpzZ<(|G}-)x95QE|7`jqdWwv
zYSU2QQMwnJS?U?7gprzofv*2?Q`OdIMo@F!BD143r2s2Cct69`-`ub))m0+qEW*zA
zpgNK(=z@EBx+7i~+gH%DKMB@5U}ne6p_!eEw*4~sJ%;P^^LO*pRoFaFt%ex1mLw8f
zE@p)Z?IP?bMv#&zC!lp#-P)43H5C5NW(|->k^C5#H1(vvSb&0c&Z0Y^Sc)dfJFt*?
z`$t9g2!b;sBb53GIj?_sVGyH<KCc=o5fl>_T(lBn!VuTOgM?lv8}O{PRk{YE_J~vB
z^C&(z!u|b{X1#1H0(n@eeCK6A=dB%;OL-k!*WbiH`eD-(VzszHI6yGHziUxwX&7+S
zs0(hMu;x6s8ryaVZ&btI>_80}2H37qA1X&1y#YQ}DP9A8s9EVNb)4_4)^NT<!L%5Y
z=Mgh4lkM6Bt^j@j*E=<HZO`2JCF9!qfbq1W8ij?*+x;2)XHsIbu&Mr8q#<qr7okI^
zRal?BIY@|Ce(OlmF;GRH6SL%@F&_Xm8*;2I9ylJhKhK<zZ%uA-)mwu&w6Gp3^F6cY
zEe@PA)}|MKCr<tFS&uXI1r8Ro>3Pd?A;>g--r$ZUsLkhIAIv%CWF|Lu+fnA5U3zi-
zXRZr!F<@H-Ff05^oAHaS{6^#2v!&AePX>X3X9Uj-A0$*_gb)+@WB)#I3*R^heJ5Jg
z{CR`yva<l=(Pi1aCto^&Ing&AaTAyy|0)#E<io}NAAeV2y0TLibw~swF1;1h^5ouj
z(S`c@Z4?5I%EnkNmu{SF-6{w~8U$)F{{6$h|JHv>{&|u9(;EM2jep&)|LF<;r+Pw=
cr{0gi2L=kN&5efCSAdtSl#*oGOXK(d1zO?5jsO4v

literal 0
HcmV?d00001

diff --git a/site/index.html b/site/index.html
new file mode 100644
index 0000000000000..aecaea525166e
--- /dev/null
+++ b/site/index.html
@@ -0,0 +1,171 @@
+---
+layout: default
+---
+    <div class="container">
+      <div class="jumbotron">
+        <h1>Apache Arrow</h1>
+        <p class="lead">Powering Columnar In-Memory Analytics</p>
+        <p>
+          <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
+        </p>
+      </div>
+      <div class="row">
+        <div class="col-lg-4">
+          <h2>Fast</h2>
+          <p>Apache Arrow&#8482; enables execution engines to take advantage of the latest SIMD (Single input multiple data) operations included in modern processors, for native vectorized optimization of analytical data processing. Columnar layout of data also allows for a better use of CPU caches by placing all data relevant to a column operation in as compact of a format as possible.</p>
+        </div>
+        <div class="col-lg-4">
+          <h2>Flexible</h2>
+          <p>Arrow acts as a new high-performance interface between various systems. It is also focused on supporting a wide variety of industry-standard programming languages. Java, C, C++, Python are underway and more languages are expected soon.</p>
+        </div>
+        <div class="col-lg-4">
+          <h2>Standard</h2>
+          <p>Apache Arrow is backed by key developers of 13 major open source projects, including Calcite, Cassandra, Drill, Hadoop, HBase, Ibis, Impala, Kudu, Pandas, Parquet, Phoenix, Spark, and Storm making it the de-facto standard for columnar in-memory analytics.</p>
+        </div>
+     </div> <!-- close "row" div -->
+
+      <div class="row marketing">
+        <div class="col-lg-4">
+          <h4>Developer Mailing List</h4>
+            <ul>
+            <li><a href="mailto:dev-subscribe@arrow.apache.org">Subscribe</a></li>
+            <li><a href="mailto:dev-unsubscribe@arrow.apache.org">Unsubscribe</a></li>
+            <li><a href="mailto:dev@arrow.apache.org">Post</a></li>
+            <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Archive</a></li>
+            </ul>
+          </div>
+        <div class="col-lg-4">
+          <h4>Developer Resources</h4>
+            <p>Arrow is still early in development. </p>
+            <p>Source Code (<a href="https://git-wip-us.apache.org/repos/asf?p=arrow.git">http</a>) (<a href="git://git.apache.org/arrow.git">git</a>)</p>
+            <p><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker (JIRA)</a></p>
+            <p><a href="https://apachearrowslackin.herokuapp.com">Chat Room (Slack)</a></p>
+        </div>
+        <div class="col-lg-4">
+          <h4>Latest release</h4>
+            <p>Apache Arrow 0.2.0 is an early release and the APIs are still evolving. The metadata and physical data representation should be fairly stable as we have spent time finalizing the details. </p>
+            <p><a href="https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0">source release</a></p>
+            <p><a href="https://github.com/apache/arrow/releases/tag/apache-arrow-0.2.0">tag apache-arrow-0.2.0</a></p>
+            <p><a href="http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.2.0%22">java artifacts on maven central</a></p>
+        </div>
+    </div>
+<h2>Performance Advantage of Columnar In-Memory</h2>
+<div align="center">
+  <img src="img/simd.png" alt="SIMD" style="width:60%" />
+</div>
+<h2>Advantages of a Common Data Layer</h2>
+
+<div class="row">
+<div class="col-lg-4" style="width:50%">
+<img src="img/copy2.png" alt="common data layer" style="width:100%" />
+<ul>
+    <li>Each system has its own internal memory format</li>
+    <li>70-80% CPU wasted on serialization and deserialization</li>
+    <li>Similar functionality implemented in multiple projects</li>
+</ul>
+</div>
+<div class="col-lg-4" style="width:50%">
+<img src="img/shared2.png" alt="common data layer" style="width:100%" />
+<ul>
+    <li>All systems utilize the same memory format</li>
+    <li>No overhead for cross-system communication</li>
+    <li>Projects can share functionality (eg, Parquet-to-Arrow reader)</li>
+</ul>
+</div>
+</div>
+<h2>Committers</h2>
+<table class="table"><thead>
+<tr>
+<th>Name</th>
+<th>Alias (email is &lt;alias&gt;@apache.org)</th>
+</tr>
+</thead><tbody>
+<tr>
+<td>Jacques Nadeau</td>
+<td>jacques</td>
+</tr>
+<tr>
+<td>Todd Lipcon</td>
+<td>todd</td>
+</tr>
+<tr>
+<td>Ted Dunning</td>
+<td>tdunning</td>
+</tr>
+<tr>
+<td>Michael Stack</td>
+<td>stack</td>
+</tr>
+<tr>
+<td>P. Taylor Goetz</td>
+<td>ptgoetz</td>
+</tr>
+<tr>
+<td>Julian Hyde</td>
+<td>jhyde</td>
+</tr>
+<tr>
+<td>Reynold Xin</td>
+<td>rxin</td>
+</tr>
+<tr>
+<td>James Taylor</td>
+<td>jamestaylor</td>
+</tr>
+<tr>
+<td>Julien Le Dem</td>
+<td>julien</td>
+</tr>
+<tr>
+<td>Jake Luciani</td>
+<td>jake</td>
+</tr>
+<tr>
+<td>Jason Altekruse</td>
+<td>json</td>
+</tr>
+<tr>
+<td>Alex Levenson</td>
+<td>alexlevenson</td>
+</tr>
+<tr>
+<td>Parth Chandra</td>
+<td>parthc</td>
+</tr>
+<tr>
+<td>Marcel Kornacker</td>
+<td>marcel</td>
+</tr>
+<tr>
+<td>Steven Phillips</td>
+<td>smp</td>
+</tr>
+<tr>
+<td>Hanifi Gunes</td>
+<td>hg</td>
+</tr>
+<tr>
+<td>Abdelhakim Deneche</td>
+<td>adeneche</td>
+</tr>
+<tr>
+<td>Wes McKinney</td>
+<td>wesm</td>
+</tr>
+<tr>
+<td>David Alves</td>
+<td>dralves</td>
+</tr>
+<tr>
+<td>Ippokratis Pandis</td>
+<td>ippokratis</td>
+</tr>
+<tr>
+<td>Uwe L. Korn</td>
+<td>uwe</td>
+</tr>
+</tbody></table>
+
+    </div> <!-- /container -->
+  </body>
+</html>
diff --git a/site/scripts/sync_format_docs.sh b/site/scripts/sync_format_docs.sh
new file mode 100755
index 0000000000000..4b50f9d1707d0
--- /dev/null
+++ b/site/scripts/sync_format_docs.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -ex
+
+cp -r $(dirname "$BASH_SOURCE")/../../format _docs/format

From 6239abd1a61fc254818548a7b6ee3f8a88777a7f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 24 Apr 2017 15:58:19 -0400
Subject: [PATCH 0560/1644] ARROW-862: [Python] Simplify README landing
 documentation to direct users and developers toward the documentation

Also migrates DEVELOPMENT.md to the Sphinx docs

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #584 from wesm/ARROW-862 and squashes the following commits:

50049dd [Wes McKinney] Revise python/README.md. Move DEVELOPMENT.md to Sphinx docs. Other cleaning
2187c1c [Wes McKinney] Migrate DEVELOPMENT.md to sphinx docs
---
 python/DEVELOPMENT.md             | 207 ----------------------------
 python/README.md                  |  71 ++--------
 python/doc/source/development.rst | 215 ++++++++++++++++++++++++++++++
 python/doc/source/index.rst       |   1 +
 python/doc/source/install.rst     | 117 ++--------------
 5 files changed, 236 insertions(+), 375 deletions(-)
 delete mode 100644 python/DEVELOPMENT.md
 create mode 100644 python/doc/source/development.rst

diff --git a/python/DEVELOPMENT.md b/python/DEVELOPMENT.md
deleted file mode 100644
index 7f08169d613f0..0000000000000
--- a/python/DEVELOPMENT.md
+++ /dev/null
@@ -1,207 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-## Developer guide for conda users
-
-### Linux and macOS
-
-#### System Requirements
-
-On macOS, any modern XCode (6.4 or higher; the current version is 8.3.1) is
-sufficient.
-
-On Linux, for this guide, we recommend using gcc 4.8 or 4.9, or clang 3.7 or
-higher. You can check your version by running
-
-```shell
-$ gcc --version
-```
-
-On Ubuntu 16.04 and higher, you can obtain gcc 4.9 with:
-
-```shell
-$ sudo apt-get install g++-4.9
-```
-
-Finally, set gcc 4.9 as the active compiler using:
-
-```shell
-export CC=gcc-4.9
-export CXX=g++-4.9
-```
-
-#### Environment Setup and Build
-
-First, let's create a conda environment with all the C++ build and Python
-dependencies from conda-forge:
-
-```shell
-conda create -y -q -n pyarrow-dev \
-      python=3.6 numpy six setuptools cython pandas pytest \
-      cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
-      brotli jemalloc -c conda-forge
-source activate pyarrow-dev
-```
-
-Now, let's clone the Arrow and Parquet git repositories:
-
-```shell
-mkdir repos
-cd repos
-git clone https://github.com/apache/arrow.git
-git clone https://github.com/apache/parquet-cpp.git
-```
-
-You should now see
-
-```shell
-$ ls -l
-total 8
-drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 arrow/
-drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 parquet-cpp/
-```
-
-We need to set a number of environment variables to let Arrow's build system
-know about our build toolchain:
-
-```
-export ARROW_BUILD_TYPE=release
-
-export BOOST_ROOT=$CONDA_PREFIX
-export BOOST_LIBRARYDIR=$CONDA_PREFIX/lib
-
-export FLATBUFFERS_HOME=$CONDA_PREFIX
-export RAPIDJSON_HOME=$CONDA_PREFIX
-export THRIFT_HOME=$CONDA_PREFIX
-export ZLIB_HOME=$CONDA_PREFIX
-export SNAPPY_HOME=$CONDA_PREFIX
-export BROTLI_HOME=$CONDA_PREFIX
-export JEMALLOC_HOME=$CONDA_PREFIX
-export ARROW_HOME=$CONDA_PREFIX
-export PARQUET_HOME=$CONDA_PREFIX
-```
-
-Now build and install the Arrow C++ libraries:
-
-```shell
-mkdir arrow/cpp/build
-pushd arrow/cpp/build
-
-cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-      -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
-      -DARROW_PYTHON=on \
-      -DARROW_BUILD_TESTS=OFF \
-      ..
-make -j4
-make install
-popd
-```
-
-Now build and install the Apache Parquet libraries in your toolchain:
-
-```shell
-mkdir parquet-cpp/build
-pushd parquet-cpp/build
-
-cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-      -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
-      -DPARQUET_BUILD_BENCHMARKS=off \
-      -DPARQUET_BUILD_EXECUTABLES=off \
-      -DPARQUET_ZLIB_VENDORED=off \
-      -DPARQUET_BUILD_TESTS=off \
-      ..
-
-make -j4
-make install
-popd
-```
-
-Now, build pyarrow:
-
-```shell
-cd arrow/python
-python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --with-parquet --inplace
-```
-
-You should be able to run the unit tests with:
-
-```shell
-$ py.test pyarrow
-================================ test session starts ================================
-platform linux -- Python 3.6.1, pytest-3.0.7, py-1.4.33, pluggy-0.4.0
-rootdir: /home/wesm/arrow-clone/python, inifile:
-collected 198 items
-
-pyarrow/tests/test_array.py ...........
-pyarrow/tests/test_convert_builtin.py .....................
-pyarrow/tests/test_convert_pandas.py .............................
-pyarrow/tests/test_feather.py ..........................
-pyarrow/tests/test_hdfs.py sssssssssssssss
-pyarrow/tests/test_io.py ..................
-pyarrow/tests/test_ipc.py ........
-pyarrow/tests/test_jemalloc.py ss
-pyarrow/tests/test_parquet.py ....................
-pyarrow/tests/test_scalars.py ..........
-pyarrow/tests/test_schema.py .........
-pyarrow/tests/test_table.py .............
-pyarrow/tests/test_tensor.py ................
-
-====================== 181 passed, 17 skipped in 0.98 seconds =======================
-```
-
-### Windows
-
-First, make sure you can [build the C++ library][1].
-
-Now, we need to build and install the C++ libraries someplace.
-
-```shell
-mkdir cpp\build
-cd cpp\build
-set ARROW_HOME=C:\thirdparty
-cmake -G "Visual Studio 14 2015 Win64" ^
-      -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
-      -DCMAKE_BUILD_TYPE=Release ^
-      -DARROW_BUILD_TESTS=off ^
-      -DARROW_PYTHON=on ..
-cmake --build . --target INSTALL --config Release
-cd ..\..
-```
-
-After that, we must put the install directory's bin path in our `%PATH%`:
-
-```shell
-set PATH=%ARROW_HOME%\bin;%PATH%
-```
-
-Now, we can build pyarrow:
-
-```shell
-cd python
-python setup.py build_ext --inplace
-```
-
-#### Running C++ unit tests with Python
-
-Getting `python-test.exe` to run is a bit tricky because your `%PYTHONPATH%`
-must be configured given the active conda environment:
-
-```shell
-set CONDA_ENV=C:\Users\wesm\Miniconda\envs\arrow-test
-set PYTHONPATH=%CONDA_ENV%\Lib;%CONDA_ENV%\Lib\site-packages;%CONDA_ENV%\python35.zip;%CONDA_ENV%\DLLs;%CONDA_ENV%
-```
-
-Now `python-test.exe` or simply `ctest` (to run all tests) should work.
-
-[1]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
\ No newline at end of file
diff --git a/python/README.md b/python/README.md
index ed008ea975d21..816fbf0c85daf 100644
--- a/python/README.md
+++ b/python/README.md
@@ -18,78 +18,31 @@ This library provides a Pythonic API wrapper for the reference Arrow C++
 implementation, along with tools for interoperability with pandas, NumPy, and
 other traditional Python scientific computing packages.
 
-### Development details
-
-This project is layered in two pieces:
-
-* arrow_python, a library part of the main Arrow C++ project for Python,
-  pandas, and NumPy interoperability
-* Cython extensions and pure Python code under pyarrow/ which expose Arrow C++
-  and pyarrow to pure Python users
+## Installing
 
-#### PyArrow Dependencies:
-
-To build pyarrow, first build and install Arrow C++ with the Python component
-enabled using `-DARROW_PYTHON=on`, see
-(https://github.com/apache/arrow/blob/master/cpp/README.md) . These components
-must be installed either in the default system location (e.g. `/usr/local`) or
-in a custom `$ARROW_HOME` location.
+Across platforms, you can install a recent version of pyarrow with the conda
+package manager:
 
 ```shell
-mkdir cpp/build
-pushd cpp/build
-cmake -DARROW_PYTHON=on -DCMAKE_INSTALL_PREFIX=$ARROW_HOME ..
-make -j4
-make install
-```
-
-If you build with a custom `CMAKE_INSTALL_PREFIX`, during development, you must
-set `ARROW_HOME` as an environment variable and add it to your
-`LD_LIBRARY_PATH` on Linux and OS X:
-
-```bash
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_HOME/lib
-```
-
-5. **Python dependencies: numpy, pandas, cython, pytest**
-
-#### Build pyarrow and run the unit tests
-
-```bash
-python setup.py build_ext --inplace
-py.test pyarrow
-```
-
-To change the build type, use the `--build-type` option or set
-`$PYARROW_BUILD_TYPE`:
-
-```bash
-python setup.py build_ext --build-type=release --inplace
+conda install pyarrow -c conda-forge
 ```
 
-To pass through other build options to CMake, set the environment variable
-`$PYARROW_CMAKE_OPTIONS`.
-
-#### Build the pyarrow Parquet file extension
+On Linux, you can also install binary wheels from PyPI with pip:
 
-To build the integration with [parquet-cpp][1], pass `--with-parquet` to
-the `build_ext` option in setup.py:
-
-```
-python setup.py build_ext --with-parquet install
+```shell
+pip install pyarrow
 ```
 
-Alternately, add `-DPYARROW_BUILD_PARQUET=on` to the general CMake options.
+### Development details
 
-```
-export PYARROW_CMAKE_OPTIONS=-DPYARROW_BUILD_PARQUET=on
-```
+See the [Development][2] page in the documentation.
 
-#### Build the documentation
+### Building the documentation
 
 ```bash
 pip install -r doc/requirements.txt
 python setup.py build_sphinx -s doc/source
 ```
 
-[1]: https://github.com/apache/parquet-cpp
\ No newline at end of file
+[1]: https://github.com/apache/parquet-cpp
+[2]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst
\ No newline at end of file
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
new file mode 100644
index 0000000000000..01add1142642a
--- /dev/null
+++ b/python/doc/source/development.rst
@@ -0,0 +1,215 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+.. _development:
+
+***********
+Development
+***********
+
+Developing with conda
+=====================
+
+Linux and macOS
+---------------
+
+System Requirements
+~~~~~~~~~~~~~~~~~~~
+
+On macOS, any modern XCode (6.4 or higher; the current version is 8.3.1) is
+sufficient.
+
+On Linux, for this guide, we recommend using gcc 4.8 or 4.9, or clang 3.7 or
+higher. You can check your version by running
+
+.. code-block:: shell
+
+   $ gcc --version
+
+On Ubuntu 16.04 and higher, you can obtain gcc 4.9 with:
+
+.. code-block:: shell
+
+   $ sudo apt-get install g++-4.9
+
+Finally, set gcc 4.9 as the active compiler using:
+
+.. code-block:: shell
+
+   export CC=gcc-4.9
+   export CXX=g++-4.9
+
+Environment Setup and Build
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+First, let's create a conda environment with all the C++ build and Python
+dependencies from conda-forge:
+
+.. code-block:: shell
+
+   conda create -y -q -n pyarrow-dev \
+         python=3.6 numpy six setuptools cython pandas pytest \
+         cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
+         brotli jemalloc -c conda-forge
+   source activate pyarrow-dev
+
+Now, let's clone the Arrow and Parquet git repositories:
+
+.. code-block:: shell
+
+   mkdir repos
+   cd repos
+   git clone https://github.com/apache/arrow.git
+   git clone https://github.com/apache/parquet-cpp.git
+
+You should now see
+
+
+.. code-block:: shell
+
+   $ ls -l
+   total 8
+   drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 arrow/
+   drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 parquet-cpp/
+
+We need to set some environment variables to let Arrow's build system know
+about our build toolchain:
+
+.. code-block:: shell
+
+   export ARROW_BUILD_TYPE=release
+   export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
+   export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX
+
+Now build and install the Arrow C++ libraries:
+
+.. code-block:: shell
+
+   mkdir arrow/cpp/build
+   pushd arrow/cpp/build
+
+   cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+         -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+         -DARROW_PYTHON=on \
+         -DARROW_BUILD_TESTS=OFF \
+         ..
+   make -j4
+   make install
+   popd
+
+Now, optionally build and install the Apache Parquet libraries in your
+toolchain:
+
+.. code-block:: shell
+
+   mkdir parquet-cpp/build
+   pushd parquet-cpp/build
+
+   cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+         -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+         -DPARQUET_BUILD_BENCHMARKS=off \
+         -DPARQUET_BUILD_EXECUTABLES=off \
+         -DPARQUET_ZLIB_VENDORED=off \
+         -DPARQUET_BUILD_TESTS=off \
+         ..
+
+   make -j4
+   make install
+   popd
+
+Now, build pyarrow:
+
+.. code-block:: shell
+
+   cd arrow/python
+   python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
+          --with-parquet --with-jemalloc --inplace
+
+If you did not build parquet-cpp, you can omit ``--with-parquet``.
+
+You should be able to run the unit tests with:
+
+.. code-block:: shell
+
+   $ py.test pyarrow
+   ================================ test session starts ====================
+   platform linux -- Python 3.6.1, pytest-3.0.7, py-1.4.33, pluggy-0.4.0
+   rootdir: /home/wesm/arrow-clone/python, inifile:
+   collected 198 items
+
+   pyarrow/tests/test_array.py ...........
+   pyarrow/tests/test_convert_builtin.py .....................
+   pyarrow/tests/test_convert_pandas.py .............................
+   pyarrow/tests/test_feather.py ..........................
+   pyarrow/tests/test_hdfs.py sssssssssssssss
+   pyarrow/tests/test_io.py ..................
+   pyarrow/tests/test_ipc.py ........
+   pyarrow/tests/test_jemalloc.py ss
+   pyarrow/tests/test_parquet.py ....................
+   pyarrow/tests/test_scalars.py ..........
+   pyarrow/tests/test_schema.py .........
+   pyarrow/tests/test_table.py .............
+   pyarrow/tests/test_tensor.py ................
+
+   ====================== 181 passed, 17 skipped in 0.98 seconds ===========
+
+Windows
+=======
+
+First, make sure you can `build the C++ library <https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md>`_.
+
+Now, we need to build and install the C++ libraries someplace.
+
+.. code-block:: shell
+
+   mkdir cpp\build
+   cd cpp\build
+   set ARROW_HOME=C:\thirdparty
+   cmake -G "Visual Studio 14 2015 Win64" ^
+         -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
+         -DCMAKE_BUILD_TYPE=Release ^
+         -DARROW_BUILD_TESTS=off ^
+         -DARROW_PYTHON=on ..
+   cmake --build . --target INSTALL --config Release
+   cd ..\..
+
+After that, we must put the install directory's bin path in our ``%PATH%``:
+
+.. code-block:: shell
+
+   set PATH=%ARROW_HOME%\bin;%PATH%
+
+Now, we can build pyarrow:
+
+.. code-block:: shell
+
+   cd python
+   python setup.py build_ext --inplace
+
+Running C++ unit tests with Python
+----------------------------------
+
+Getting ``python-test.exe`` to run is a bit tricky because your
+``%PYTHONPATH%`` must be configured given the active conda environment:
+
+.. code-block:: shell
+
+   set CONDA_ENV=C:\Users\wesm\Miniconda\envs\arrow-test
+   set PYTHONPATH=%CONDA_ENV%\Lib;%CONDA_ENV%\Lib\site-packages;%CONDA_ENV%\python35.zip;%CONDA_ENV%\DLLs;%CONDA_ENV%
+
+Now ``python-test.exe`` or simply ``ctest`` (to run all tests) should work.
diff --git a/python/doc/source/index.rst b/python/doc/source/index.rst
index ecb8e8f4830f3..55b4efc79bc3f 100644
--- a/python/doc/source/index.rst
+++ b/python/doc/source/index.rst
@@ -35,6 +35,7 @@ structures.
    :caption: Getting Started
 
    install
+   development
    pandas
    filesystems
    parquet
diff --git a/python/doc/source/install.rst b/python/doc/source/install.rst
index 278b466941a6f..a2a6520be4884 100644
--- a/python/doc/source/install.rst
+++ b/python/doc/source/install.rst
@@ -37,115 +37,14 @@ Install the latest version from PyPI:
     pip install pyarrow
 
 .. note::
-    Currently there are only binary artifcats available for Linux and MacOS.
-    Otherwise this will only pull the python sources and assumes an existing
-    installation of the C++ part of Arrow.
-    To retrieve the binary artifacts, you'll need a recent ``pip`` version that
-    supports features like the ``manylinux1`` tag.
-
-Building from source
---------------------
-
-First, clone the master git repository:
-
-.. code-block:: bash
-
-    git clone https://github.com/apache/arrow.git arrow
-
-System requirements
-~~~~~~~~~~~~~~~~~~~
-
-Building pyarrow requires:
-
-* A C++11 compiler
-
-  * Linux: gcc >= 4.8 or clang >= 3.5
-  * OS X: XCode 6.4 or higher preferred
-
-* `CMake <https://cmake.org/>`_
-
-Python requirements
-~~~~~~~~~~~~~~~~~~~
-
-You will need Python (CPython) 2.7, 3.4, or 3.5 installed. Earlier releases and
-are not being targeted.
-
-.. note::
-    This library targets CPython only due to an emphasis on interoperability with
-    pandas and NumPy, which are only available for CPython.
-
-The build requires NumPy, Cython, and a few other Python dependencies:
-
-.. code-block:: bash
-
-    pip install cython
-    cd arrow/python
-    pip install -r requirements.txt
-
-Installing Arrow C++ library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-First, you should choose an installation location for Arrow C++. In the future
-using the default system install location will work, but for now we are being
-explicit:
-
-.. code-block:: bash
-
-    export ARROW_HOME=$HOME/local
-
-Now, we build Arrow:
-
-.. code-block:: bash
-
-    cd arrow/cpp
-
-    mkdir dev-build
-    cd dev-build
-
-    cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME ..
-
-    make
-
-    # Use sudo here if $ARROW_HOME requires it
-    make install
-
-To get the optional Parquet support, you should also build and install
-`parquet-cpp <https://github.com/apache/parquet-cpp/blob/master/README.md>`_.
 
-Install `pyarrow`
-~~~~~~~~~~~~~~~~~
-
-
-.. code-block:: bash
-
-    cd arrow/python
-
-    # --with-parquet enables the Apache Parquet support in PyArrow
-    # --with-jemalloc enables the jemalloc allocator support in PyArrow
-    # --build-type=release disables debugging information and turns on
-    #       compiler optimizations for native code
-    python setup.py build_ext --with-parquet --with-jemalloc --build-type=release install
-    python setup.py install
-
-.. warning::
-    On XCode 6 and prior there are some known OS X `@rpath` issues. If you are
-    unable to import pyarrow, upgrading XCode may be the solution.
-
-.. note::
-    In development installations, you will also need to set a correct
-    ``LD_LIBRARY_PATH``. This is most probably done with
-    ``export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH``.
-
-
-.. code-block:: python
+    Currently there are only binary artifacts available for Linux and MacOS.
+    Otherwise this will only pull the python sources and assumes an existing
+    installation of the C++ part of Arrow.  To retrieve the binary artifacts,
+    you'll need a recent ``pip`` version that supports features like the
+    ``manylinux1`` tag.
 
-    In [1]: import pyarrow
+Installing from source
+----------------------
 
-    In [2]: pyarrow.array([1,2,3])
-    Out[2]:
-    <pyarrow.array.Int64Array object at 0x7f899f3e60e8>
-    [
-      1,
-      2,
-      3
-    ]
+See :ref:`development`.

From eaf2118efc56823d93dbd57c7e9afdb1d904ac2f Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Mon, 24 Apr 2017 13:29:20 -0700
Subject: [PATCH 0561/1644] ARROW-887: add default value to units for backward
 compatibility

Author: Julien Le Dem <julien@apache.org>

Closes #592 from julienledem/ARROW-887 and squashes the following commits:

bc49f8a [Julien Le Dem] ARROW-887: add default value to units for backward compatibility
---
 format/Schema.fbs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/format/Schema.fbs b/format/Schema.fbs
index ff6119931dd34..b48859f50eea2 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -100,7 +100,7 @@ enum DateUnit: short {
 ///   leap seconds), where the values are evenly divisible by 86400000
 /// * Days (32 bits) since the UNIX epoch
 table Date {
-  unit: DateUnit;
+  unit: DateUnit = MILLISECOND;
 }
 
 enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }
@@ -109,8 +109,8 @@ enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }
 /// - SECOND and MILLISECOND: 32 bits
 /// - MICROSECOND and NANOSECOND: 64 bits
 table Time {
-  unit: TimeUnit;
-  bitWidth: int;
+  unit: TimeUnit = MILLISECOND;
+  bitWidth: int = 32;
 }
 
 /// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.

From f00e2ab590ad8f04409e7bc09f70622e73ebd741 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 25 Apr 2017 08:45:01 +0200
Subject: [PATCH 0562/1644] ARROW-890: [GLib] Add GArrowMutableBuffer

Author: Kouhei Sutou <kou@clear-code.com>

Closes #596 from kou/glib-mutable-buffer and squashes the following commits:

73c2663 [Kouhei Sutou] [GLib] Support running tests on Ubuntu 14.04
d211a22 [Kouhei Sutou] [GLib] Add GArrowMutableBuffer
---
 c_glib/arrow-glib/buffer.cpp       | 97 +++++++++++++++++++++++++++++-
 c_glib/arrow-glib/buffer.h         | 52 ++++++++++++++++
 c_glib/arrow-glib/buffer.hpp       |  2 +
 c_glib/test/test-buffer.rb         |  7 +++
 c_glib/test/test-mutable-buffer.rb | 38 ++++++++++++
 5 files changed, 195 insertions(+), 1 deletion(-)
 create mode 100644 c_glib/test/test-mutable-buffer.rb

diff --git a/c_glib/arrow-glib/buffer.cpp b/c_glib/arrow-glib/buffer.cpp
index 9853e896b3dcc..5fc3b077a1cdb 100644
--- a/c_glib/arrow-glib/buffer.cpp
+++ b/c_glib/arrow-glib/buffer.cpp
@@ -28,10 +28,16 @@ G_BEGIN_DECLS
 
 /**
  * SECTION: buffer
- * @short_description: Buffer class
+ * @section_id: buffer-classes
+ * @title: Buffer classes
+ * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowBuffer is a class for keeping data. Other classes such as
  * #GArrowArray and #GArrowTensor can use data in buffer.
+ *
+ * #GArrowBuffer is immutable.
+ *
+ * #GArrowMutableBuffer is mutable.
  */
 
 typedef struct GArrowBufferPrivate_ {
@@ -182,6 +188,27 @@ garrow_buffer_get_data(GArrowBuffer *buffer)
   return data;
 }
 
+/**
+ * garrow_buffer_get_mutable_data:
+ * @buffer: A #GArrowBuffer.
+ *
+ * Returns: (transfer full) (nullable): The data of the buffer. If the
+ *   buffer is imutable, it returns %NULL. The data is owned by the
+ *   buffer. You should not free the data.
+ *
+ * Since: 0.3.0
+ */
+GBytes *
+garrow_buffer_get_mutable_data(GArrowBuffer *buffer)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  if (!arrow_buffer->is_mutable()) {
+    return NULL;
+  }
+  return g_bytes_new_static(arrow_buffer->mutable_data(),
+                            arrow_buffer->size());
+}
+
 /**
  * garrow_buffer_get_size:
  * @buffer: A #GArrowBuffer.
@@ -271,6 +298,65 @@ garrow_buffer_slice(GArrowBuffer *buffer, gint64 offset, gint64 size)
   return garrow_buffer_new_raw(&arrow_buffer);
 }
 
+
+G_DEFINE_TYPE(GArrowMutableBuffer,              \
+              garrow_mutable_buffer,            \
+              GARROW_TYPE_BUFFER)
+
+static void
+garrow_mutable_buffer_init(GArrowMutableBuffer *object)
+{
+}
+
+static void
+garrow_mutable_buffer_class_init(GArrowMutableBufferClass *klass)
+{
+}
+
+/**
+ * garrow_mutable_buffer_new:
+ * @data: (array length=size): Data for the buffer.
+ *   They aren't owned by the new buffer.
+ *   You must not free the data while the new buffer is alive.
+ * @size: The number of bytes of the data.
+ *
+ * Returns: A newly created #GArrowMutableBuffer.
+ *
+ * Since: 0.3.0
+ */
+GArrowMutableBuffer *
+garrow_mutable_buffer_new(guint8 *data, gint64 size)
+{
+  auto arrow_buffer = std::make_shared<arrow::MutableBuffer>(data, size);
+  return garrow_mutable_buffer_new_raw(&arrow_buffer);
+}
+
+/**
+ * garrow_mutable_buffer_slice:
+ * @buffer: A #GArrowMutableBuffer.
+ * @offset: An offset in the buffer data in byte.
+ * @size: The number of bytes of the sliced data.
+ *
+ * Returns: (transfer full): A newly created #GArrowMutableBuffer that
+ *   shares data of the base #GArrowMutableBuffer. The created
+ *   #GArrowMutableBuffer has data start with offset from the base
+ *   buffer data and are the specified bytes size.
+ *
+ * Since: 0.3.0
+ */
+GArrowMutableBuffer *
+garrow_mutable_buffer_slice(GArrowMutableBuffer *buffer,
+                            gint64 offset,
+                            gint64 size)
+{
+  auto arrow_parent_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer));
+  auto arrow_buffer =
+    std::make_shared<arrow::MutableBuffer>(arrow_parent_buffer,
+                                           offset,
+                                           size);
+  return garrow_mutable_buffer_new_raw(&arrow_buffer);
+}
+
 G_END_DECLS
 
 GArrowBuffer *
@@ -288,3 +374,12 @@ garrow_buffer_get_raw(GArrowBuffer *buffer)
   auto priv = GARROW_BUFFER_GET_PRIVATE(buffer);
   return priv->buffer;
 }
+
+GArrowMutableBuffer *
+garrow_mutable_buffer_new_raw(std::shared_ptr<arrow::MutableBuffer> *arrow_buffer)
+{
+  auto buffer = GARROW_MUTABLE_BUFFER(g_object_new(GARROW_TYPE_MUTABLE_BUFFER,
+                                                   "buffer", arrow_buffer,
+                                                   NULL));
+  return buffer;
+}
diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h
index 83e1d0d66bf28..5334614c151c9 100644
--- a/c_glib/arrow-glib/buffer.h
+++ b/c_glib/arrow-glib/buffer.h
@@ -62,6 +62,7 @@ GArrowBuffer  *garrow_buffer_new          (const guint8 *data,
 gboolean       garrow_buffer_is_mutable   (GArrowBuffer *buffer);
 gint64         garrow_buffer_get_capacity (GArrowBuffer *buffer);
 GBytes        *garrow_buffer_get_data     (GArrowBuffer *buffer);
+GBytes        *garrow_buffer_get_mutable_data(GArrowBuffer *buffer);
 gint64         garrow_buffer_get_size     (GArrowBuffer *buffer);
 GArrowBuffer  *garrow_buffer_get_parent   (GArrowBuffer *buffer);
 
@@ -73,4 +74,55 @@ GArrowBuffer  *garrow_buffer_slice        (GArrowBuffer *buffer,
                                            gint64 offset,
                                            gint64 size);
 
+
+#define GARROW_TYPE_MUTABLE_BUFFER              \
+  (garrow_mutable_buffer_get_type())
+#define GARROW_MUTABLE_BUFFER(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_MUTABLE_BUFFER,       \
+                              GArrowMutableBuffer))
+#define GARROW_MUTABLE_BUFFER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_MUTABLE_BUFFER,  \
+                           GArrowMutableBufferClass))
+#define GARROW_IS_MUTABLE_BUFFER(obj)                                   \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_MUTABLE_BUFFER))
+#define GARROW_IS_MUTABLE_BUFFER_CLASS(klass)                           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_MUTABLE_BUFFER))
+#define GARROW_MUTABLE_BUFFER_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_MUTABLE_BUFFER,        \
+                             GArrowMutableBufferClass))
+
+typedef struct _GArrowMutableBuffer         GArrowMutableBuffer;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowMutableBufferClass    GArrowMutableBufferClass;
+#endif
+
+/**
+ * GArrowMutableBuffer:
+ *
+ * It wraps `arrow::MutableBuffer`.
+ */
+struct _GArrowMutableBuffer
+{
+  /*< private >*/
+  GArrowBuffer parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowMutableBufferClass
+{
+  GArrowBufferClass parent_class;
+};
+#endif
+
+GType garrow_mutable_buffer_get_type(void) G_GNUC_CONST;
+
+GArrowMutableBuffer *garrow_mutable_buffer_new  (guint8 *data,
+                                                 gint64 size);
+GArrowMutableBuffer *garrow_mutable_buffer_slice(GArrowMutableBuffer *buffer,
+                                                 gint64 offset,
+                                                 gint64 size);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/buffer.hpp b/c_glib/arrow-glib/buffer.hpp
index 00dd3de3bfd50..1337d9ed596f9 100644
--- a/c_glib/arrow-glib/buffer.hpp
+++ b/c_glib/arrow-glib/buffer.hpp
@@ -25,3 +25,5 @@
 
 GArrowBuffer *garrow_buffer_new_raw(std::shared_ptr<arrow::Buffer> *arrow_buffer);
 std::shared_ptr<arrow::Buffer> garrow_buffer_get_raw(GArrowBuffer *buffer);
+
+GArrowMutableBuffer *garrow_mutable_buffer_new_raw(std::shared_ptr<arrow::MutableBuffer> *arrow_buffer);
diff --git a/c_glib/test/test-buffer.rb b/c_glib/test/test-buffer.rb
index 6bb96714c8283..9f76a805f7577 100644
--- a/c_glib/test/test-buffer.rb
+++ b/c_glib/test/test-buffer.rb
@@ -16,6 +16,8 @@
 # under the License.
 
 class TestBuffer < Test::Unit::TestCase
+  include Helper::Omittable
+
   def setup
     @data = "Hello"
     @buffer = Arrow::Buffer.new(@data)
@@ -35,6 +37,11 @@ def test_data
     assert_equal(@data, @buffer.data.to_s)
   end
 
+  def test_mutable_data
+    require_gi(3, 1, 2)
+    assert_nil(@buffer.mutable_data)
+  end
+
   def test_size
     assert_equal(@data.bytesize, @buffer.size)
   end
diff --git a/c_glib/test/test-mutable-buffer.rb b/c_glib/test/test-mutable-buffer.rb
new file mode 100644
index 0000000000000..df62dcf1e8d15
--- /dev/null
+++ b/c_glib/test/test-mutable-buffer.rb
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestMutableBuffer < Test::Unit::TestCase
+  def setup
+    @data = "Hello"
+    @buffer = Arrow::MutableBuffer.new(@data)
+  end
+
+  def test_mutable?
+    assert do
+      @buffer.mutable?
+    end
+  end
+
+  def test_mutable_data
+    assert_equal(@data, @buffer.mutable_data.to_s)
+  end
+
+  def test_slice
+    sliced_buffer = @buffer.slice(1, 3)
+    assert_equal(@data[1, 3], sliced_buffer.data.to_s)
+  end
+end

From 1a73c352d023dfa0e8aca4c16f3e421745524ea8 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Tue, 25 Apr 2017 11:25:03 -0700
Subject: [PATCH 0563/1644] ARROW-895: Fix lastSet in fillEmpties() and
 copyFrom()

Author: Steven Phillips <steven@dremio.com>

Closes #601 from StevenMPhillips/fillEmpties4 and squashes the following commits:

4707673 [Steven Phillips] ARROW-895: Fix lastSet in fillEmpties() and copyFrom()
---
 .../templates/NullableValueVectors.java       |  5 ++-
 .../apache/arrow/vector/TestValueVector.java  | 38 +++++++++++++++++++
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 178d5bd913910..31adc2bdd0789 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -393,6 +393,7 @@ public void copyFrom(int fromIndex, int thisIndex, ${className} from){
     if (!fromAccessor.isNull(fromIndex)) {
       mutator.set(thisIndex, fromAccessor.get(fromIndex));
     }
+    <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
   }
 
   public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from){
@@ -401,6 +402,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from){
     </#if>
     values.copyFromSafe(fromIndex, thisIndex, from);
     bits.getMutator().setSafeToOne(thisIndex);
+    <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
   }
 
   public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
@@ -409,6 +411,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
     </#if>
     bits.copyFromSafe(fromIndex, thisIndex, from.bits);
     values.copyFromSafe(fromIndex, thisIndex, from.values);
+    <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
   }
 
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
@@ -532,7 +535,7 @@ private void fillEmpties(int index){
       while(index > bits.getValueCapacity()) {
         bits.reAlloc();
       }
-      lastSet = index;
+      lastSet = index - 1;
     }
 
     @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index e6e49ab8d9341..63543b0932908 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -21,6 +21,7 @@
 import static org.apache.arrow.vector.TestUtils.newVector;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.nio.charset.Charset;
@@ -473,9 +474,46 @@ public void testFillEmptiesNotOverfill() {
 
       vector.getMutator().setSafe(4094, "hello".getBytes(), 0, 5);
       vector.getMutator().setValueCount(4095);
+
       assertEquals(4096 * 4, vector.getFieldBuffers().get(1).capacity());
     }
   }
 
+  @Test
+  public void testCopyFromWithNulls() {
+    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
+          final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+      vector.allocateNew();
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          continue;
+        }
+        byte[] b = Integer.toString(i).getBytes();
+        vector.getMutator().setSafe(i, b, 0, b.length);
+      }
+
+      vector.getMutator().setValueCount(4095);
+
+      vector2.allocateNew();
+
+      for (int i = 0; i < 4095; i++) {
+        vector2.copyFromSafe(i, i, vector);
+      }
+
+      vector2.getMutator().setValueCount(4095);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          assertNull(vector2.getAccessor().getObject(i));
+        } else {
+          assertEquals(Integer.toString(i), vector2.getAccessor().getObject(i).toString());
+        }
+
+      }
+
+
+    }
+  }
 
 }

From 0bee8040e29ebbb4542bc267804f56dcf7feaf4e Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Tue, 25 Apr 2017 11:36:32 -0700
Subject: [PATCH 0564/1644] ARROW-888: Transfer ownership of buffer in
 BitVector transferTo()

Author: Steven Phillips <steven@dremio.com>

Closes #594 from StevenMPhillips/bitVectorOwnership and squashes the following commits:

117f6b2 [Steven Phillips] ARROW-888: Transfer ownership of buffer in BitVector transferTo()
---
 .../org/apache/arrow/vector/BitVector.java    |  6 +-
 .../vector/TestBufferOwnershipTransfer.java   | 65 +++++++++++++++++++
 2 files changed, 66 insertions(+), 5 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index ed574333beacd..82cbd47d75816 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -251,11 +251,7 @@ public TransferPair makeTransferPair(ValueVector to) {
 
   public void transferTo(BitVector target) {
     target.clear();
-    if (target.data != null) {
-      target.data.release();
-    }
-    target.data = data;
-    target.data.retain(1);
+    target.data = data.transferOwnership(target.allocator).buffer;
     target.valueCount = valueCount;
     clear();
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
new file mode 100644
index 0000000000000..fa657875d6d92
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.junit.Test;
+
+public class TestBufferOwnershipTransfer {
+
+  @Test
+  public void testTransferFixedWidth() {
+    BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+    BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 100000, 100000);
+    BufferAllocator childAllocator2 = allocator.newChildAllocator("child2", 100000, 100000);
+
+    NullableIntVector v1 = new NullableIntVector("v1", childAllocator1);
+    v1.allocateNew();
+    v1.getMutator().setValueCount(4095);
+
+    NullableIntVector v2 = new NullableIntVector("v2", childAllocator2);
+
+    v1.makeTransferPair(v2).transfer();
+
+    assertEquals(0, childAllocator1.getAllocatedMemory());
+    assertEquals(5 * 4096, childAllocator2.getAllocatedMemory());
+  }
+
+  @Test
+  public void testTransferVariableidth() {
+    BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+    BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 100000, 100000);
+    BufferAllocator childAllocator2 = allocator.newChildAllocator("child2", 100000, 100000);
+
+    NullableVarCharVector v1 = new NullableVarCharVector("v1", childAllocator1);
+    v1.allocateNew();
+    v1.getMutator().setSafe(4094, "hello world".getBytes(), 0, 11);
+    v1.getMutator().setValueCount(4001);
+
+    NullableVarCharVector v2 = new NullableVarCharVector("v2", childAllocator2);
+
+    v1.makeTransferPair(v2).transfer();
+
+    assertEquals(0, childAllocator1.getAllocatedMemory());
+    int expected = 8*4096 + 4*4096 + 4096;
+    assertEquals(expected, childAllocator2.getAllocatedMemory());
+  }
+}

From 68decb6f33cb1ed262006d4b237137e36f89057c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 25 Apr 2017 14:54:18 -0400
Subject: [PATCH 0565/1644] ARROW-865: [Python] Add unit tests validating
 Parquet date/time type roundtrips

Requires PARQUET-915 https://github.com/apache/parquet-cpp/pull/311

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #595 from wesm/ARROW-865 and squashes the following commits:

db16940 [Wes McKinney] Add tests for auto-casted types, and unsupported nanosecond time
475fa3f [Wes McKinney] Fix test case
fad3934 [Wes McKinney] Update test case
da96a38 [Wes McKinney] Add failing Parquet test case. Enable same-type-size cases in pandas_convert.cc
---
 cpp/src/arrow/python/pandas_convert.cc |  2 +-
 cpp/src/arrow/python/type_traits.h     | 48 +++++++++++++++++++++
 cpp/src/arrow/util/stl.h               |  2 +-
 python/pyarrow/tests/test_ipc.py       |  3 +-
 python/pyarrow/tests/test_parquet.py   | 60 ++++++++++++++++++++++++++
 5 files changed, 112 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 636a3fd15c044..9f65af41bb294 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -444,7 +444,7 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   // Handle LONGLONG->INT64 and other fun things
   int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
 
-  if (traits::npy_type != type_num_compat) {
+  if (numpy_type_size(traits::npy_type) != numpy_type_size(type_num_compat)) {
     return Status::NotImplemented("NumPy type casts not yet implemented");
   }
 
diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h
index 26b15bdc9f464..b6761ae0d2611 100644
--- a/cpp/src/arrow/python/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+// Internal header
+
 #include "arrow/python/platform.h"
 
 #include <cstdint>
@@ -24,6 +26,7 @@
 
 #include "arrow/builder.h"
 #include "arrow/type.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace py {
@@ -224,5 +227,50 @@ struct arrow_traits<Type::BINARY> {
   static constexpr bool supports_nulls = true;
 };
 
+static inline int numpy_type_size(int npy_type) {
+  switch (npy_type) {
+    case NPY_BOOL:
+      return 1;
+    case NPY_INT8:
+      return 1;
+    case NPY_INT16:
+      return 2;
+    case NPY_INT32:
+      return 4;
+    case NPY_INT64:
+      return 8;
+#if (NPY_INT64 != NPY_LONGLONG)
+    case NPY_LONGLONG:
+      return 8;
+#endif
+    case NPY_UINT8:
+      return 1;
+    case NPY_UINT16:
+      return 2;
+    case NPY_UINT32:
+      return 4;
+    case NPY_UINT64:
+      return 8;
+#if (NPY_UINT64 != NPY_ULONGLONG)
+    case NPY_ULONGLONG:
+      return 8;
+#endif
+    case NPY_FLOAT16:
+      return 2;
+    case NPY_FLOAT32:
+      return 4;
+    case NPY_FLOAT64:
+      return 8;
+    case NPY_DATETIME:
+      return 8;
+    case NPY_OBJECT:
+      return sizeof(void*);
+    default:
+      DCHECK(false) << "unhandled numpy type";
+      break;
+  }
+  return -1;
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/stl.h b/cpp/src/arrow/util/stl.h
index bfce111ff8a22..d58689b748896 100644
--- a/cpp/src/arrow/util/stl.h
+++ b/cpp/src/arrow/util/stl.h
@@ -20,7 +20,7 @@
 
 #include <vector>
 
-#include <arrow/util/logging.h>
+#include "arrow/util/logging.h"
 
 namespace arrow {
 
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 81213ede3151e..02040678958ed 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -158,7 +158,8 @@ def run(self):
                 connection.close()
 
         def get_result(self):
-            return(self._schema, self._table if self._do_read_all else self._batches)
+            return(self._schema, self._table if self._do_read_all
+                   else self._batches)
 
     def setUp(self):
         # NOTE: must start and stop server in test
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 268e87af7dda4..8c446af03fc16 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -348,6 +348,66 @@ def test_column_of_lists(tmpdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@parquet
+def test_date_time_types(tmpdir):
+    buf = io.BytesIO()
+
+    t1 = pa.date32()
+    data1 = np.array([17259, 17260, 17261], dtype='int32')
+    a1 = pa.Array.from_pandas(data1, type=t1)
+
+    t2 = pa.date64()
+    data2 = data1.astype('int64') * 86400000
+    a2 = pa.Array.from_pandas(data2, type=t2)
+
+    t3 = pa.timestamp('us')
+    start = pd.Timestamp('2000-01-01').value / 1000
+    data3 = np.array([start, start + 1, start + 2], dtype='int64')
+    a3 = pa.Array.from_pandas(data3, type=t3)
+
+    t4 = pa.time32('ms')
+    data4 = np.arange(3, dtype='i4')
+    a4 = pa.Array.from_pandas(data4, type=t4)
+
+    t5 = pa.time64('us')
+    a5 = pa.Array.from_pandas(data4.astype('int64'), type=t5)
+
+    t6 = pa.time32('s')
+    a6 = pa.Array.from_pandas(data4, type=t6)
+
+    ex_t6 = pa.time32('ms')
+    ex_a6 = pa.Array.from_pandas(data4 * 1000, type=ex_t6)
+
+    table = pa.Table.from_arrays([a1, a2, a3, a4, a5, a6],
+                                 ['date32', 'date64', 'timestamp[us]',
+                                  'time32[s]', 'time64[us]', 'time32[s]'])
+
+    # date64 as date32
+    # time32[s] to time32[ms]
+    expected = pa.Table.from_arrays([a1, a1, a3, a4, a5, ex_a6],
+                                    ['date32', 'date64', 'timestamp[us]',
+                                     'time32[s]', 'time64[us]', 'time32[s]'])
+
+    pq.write_table(table, buf, version="2.0")
+    buf.seek(0)
+
+    result = pq.read_table(buf)
+    assert result.equals(expected)
+
+    # Unsupported stuff
+    def _assert_unsupported(array):
+        table = pa.Table.from_arrays([array], ['unsupported'])
+        buf = io.BytesIO()
+
+        with pytest.raises(NotImplementedError):
+            pq.write_table(table, buf, version="2.0")
+
+    t7 = pa.time64('ns')
+    a7 = pa.Array.from_pandas(data4.astype('int64'), type=t7)
+
+    _assert_unsupported(a7)
+
+
 @parquet
 def test_multithreaded_read():
     df = alltypes_sample(size=10000)

From 6ae49a1dd6a3a8c4292987643cd11af4f35ab9b2 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 25 Apr 2017 15:09:25 -0400
Subject: [PATCH 0566/1644] ARROW-892: [GLib] Fix GArrowTensor document

Author: Kouhei Sutou <kou@clear-code.com>

Closes #598 from kou/glib-tensor-doc and squashes the following commits:

3982db1 [Kouhei Sutou] [GLib] Fix GArrowTensor document
---
 c_glib/arrow-glib/tensor.cpp              | 7 +++----
 c_glib/doc/reference/arrow-glib-docs.sgml | 4 ++++
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp
index 468eb0729357b..82f66352d66a0 100644
--- a/c_glib/arrow-glib/tensor.cpp
+++ b/c_glib/arrow-glib/tensor.cpp
@@ -30,11 +30,10 @@ G_BEGIN_DECLS
 
 /**
  * SECTION: tensor
- * @short_description: Base class for all tensor classes
+ * @short_description: Tensor class.
+ * @include: arrow-glib/arrow-glib.h
  *
- * #GArrowTensor is a base class for all tensor classes such as
- * #GArrowInt8Tensor.
- * #GArrowBooleanTensorBuilder to create a new tensor.
+ * #GArrowTensor is a tensor class.
  *
  * Since: 0.3.0
  */
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index bfb2776f621cc..75e4a0a37286f 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -41,6 +41,10 @@
       <title>Array builder</title>
       <xi:include href="xml/array-builder.xml"/>
     </chapter>
+    <chapter id="tensor">
+      <title>Tensor</title>
+      <xi:include href="xml/tensor.xml"/>
+    </chapter>
     <chapter id="type">
       <title>Type</title>
       <xi:include href="xml/type.xml"/>

From 015b2849299be4bee9b470e3965465e1b0278881 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 25 Apr 2017 15:47:53 -0400
Subject: [PATCH 0567/1644] ARROW-894: [GLib] Add GArrowResizableBuffer and
 GArrowPoolBuffer

Author: Kouhei Sutou <kou@clear-code.com>

Closes #600 from kou/glib-pool-buffer and squashes the following commits:

f8845aa [Kouhei Sutou] [GLib] Add GArrowResizableBuffer and GArrowPoolBuffer
---
 c_glib/arrow-glib/buffer.cpp    | 114 ++++++++++++++++++++++++++++++++
 c_glib/arrow-glib/buffer.h      |  99 +++++++++++++++++++++++++++
 c_glib/arrow-glib/buffer.hpp    |   1 +
 c_glib/test/test-pool-buffer.rb |  32 +++++++++
 4 files changed, 246 insertions(+)
 create mode 100644 c_glib/test/test-pool-buffer.rb

diff --git a/c_glib/arrow-glib/buffer.cpp b/c_glib/arrow-glib/buffer.cpp
index 5fc3b077a1cdb..5c28daf674e4e 100644
--- a/c_glib/arrow-glib/buffer.cpp
+++ b/c_glib/arrow-glib/buffer.cpp
@@ -38,6 +38,11 @@ G_BEGIN_DECLS
  * #GArrowBuffer is immutable.
  *
  * #GArrowMutableBuffer is mutable.
+ *
+ * #GArrowResizableBuffer is mutable and
+ * resizable. #GArrowResizableBuffer isn't instantiatable.
+ *
+ * #GArrowPoolBuffer is mutable, resizable and instantiatable.
  */
 
 typedef struct GArrowBufferPrivate_ {
@@ -357,6 +362,106 @@ garrow_mutable_buffer_slice(GArrowMutableBuffer *buffer,
   return garrow_mutable_buffer_new_raw(&arrow_buffer);
 }
 
+
+G_DEFINE_TYPE(GArrowResizableBuffer,              \
+              garrow_resizable_buffer,            \
+              GARROW_TYPE_MUTABLE_BUFFER)
+
+static void
+garrow_resizable_buffer_init(GArrowResizableBuffer *object)
+{
+}
+
+static void
+garrow_resizable_buffer_class_init(GArrowResizableBufferClass *klass)
+{
+}
+
+/**
+ * garrow_resizable_buffer_resize:
+ * @buffer: A #GArrowResizableBuffer.
+ * @new_size: The new buffer size in bytes.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.3.0
+ */
+gboolean
+garrow_resizable_buffer_resize(GArrowResizableBuffer *buffer,
+                               gint64 new_size,
+                               GError **error)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer));
+  auto arrow_resizable_buffer =
+    std::static_pointer_cast<arrow::ResizableBuffer>(arrow_buffer);
+  auto status = arrow_resizable_buffer->Resize(new_size);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[resizable-buffer][resize]");
+    return FALSE;
+  }
+}
+
+/**
+ * garrow_resizable_buffer_reserve:
+ * @buffer: A #GArrowResizableBuffer.
+ * @new_capacity: The new buffer capacity in bytes.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.3.0
+ */
+gboolean
+garrow_resizable_buffer_reserve(GArrowResizableBuffer *buffer,
+                                gint64 new_capacity,
+                                GError **error)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer));
+  auto arrow_resizable_buffer =
+    std::static_pointer_cast<arrow::ResizableBuffer>(arrow_buffer);
+  auto status = arrow_resizable_buffer->Reserve(new_capacity);
+  if (status.ok()) {
+    return TRUE;
+  } else {
+    garrow_error_set(error, status, "[resizable-buffer][capacity]");
+    return FALSE;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowPoolBuffer,              \
+              garrow_pool_buffer,            \
+              GARROW_TYPE_RESIZABLE_BUFFER)
+
+static void
+garrow_pool_buffer_init(GArrowPoolBuffer *object)
+{
+}
+
+static void
+garrow_pool_buffer_class_init(GArrowPoolBufferClass *klass)
+{
+}
+
+/**
+ * garrow_pool_buffer_new:
+ *
+ * Returns: A newly created #GArrowPoolBuffer.
+ *
+ * Since: 0.3.0
+ */
+GArrowPoolBuffer *
+garrow_pool_buffer_new(void)
+{
+  auto arrow_memory_pool = arrow::default_memory_pool();
+  auto arrow_buffer = std::make_shared<arrow::PoolBuffer>(arrow_memory_pool);
+  return garrow_pool_buffer_new_raw(&arrow_buffer);
+}
+
+
 G_END_DECLS
 
 GArrowBuffer *
@@ -383,3 +488,12 @@ garrow_mutable_buffer_new_raw(std::shared_ptr<arrow::MutableBuffer> *arrow_buffe
                                                    NULL));
   return buffer;
 }
+
+GArrowPoolBuffer *
+garrow_pool_buffer_new_raw(std::shared_ptr<arrow::PoolBuffer> *arrow_buffer)
+{
+  auto buffer = GARROW_POOL_BUFFER(g_object_new(GARROW_TYPE_POOL_BUFFER,
+                                                "buffer", arrow_buffer,
+                                                NULL));
+  return buffer;
+}
diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h
index 5334614c151c9..22a5e9bb2549a 100644
--- a/c_glib/arrow-glib/buffer.h
+++ b/c_glib/arrow-glib/buffer.h
@@ -125,4 +125,103 @@ GArrowMutableBuffer *garrow_mutable_buffer_slice(GArrowMutableBuffer *buffer,
                                                  gint64 offset,
                                                  gint64 size);
 
+
+#define GARROW_TYPE_RESIZABLE_BUFFER            \
+  (garrow_resizable_buffer_get_type())
+#define GARROW_RESIZABLE_BUFFER(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_RESIZABLE_BUFFER,     \
+                              GArrowResizableBuffer))
+#define GARROW_RESIZABLE_BUFFER_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_RESIZABLE_BUFFER,        \
+                           GArrowResizableBufferClass))
+#define GARROW_IS_RESIZABLE_BUFFER(obj)                                 \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_RESIZABLE_BUFFER))
+#define GARROW_IS_RESIZABLE_BUFFER_CLASS(klass)                         \
+  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_RESIZABLE_BUFFER))
+#define GARROW_RESIZABLE_BUFFER_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_RESIZABLE_BUFFER,      \
+                             GArrowResizableBufferClass))
+
+typedef struct _GArrowResizableBuffer         GArrowResizableBuffer;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowResizableBufferClass    GArrowResizableBufferClass;
+#endif
+
+/**
+ * GArrowResizableBuffer:
+ *
+ * It wraps `arrow::ResizableBuffer`.
+ */
+struct _GArrowResizableBuffer
+{
+  /*< private >*/
+  GArrowMutableBuffer parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowResizableBufferClass
+{
+  GArrowMutableBufferClass parent_class;
+};
+#endif
+
+GType garrow_resizable_buffer_get_type(void) G_GNUC_CONST;
+
+gboolean garrow_resizable_buffer_resize(GArrowResizableBuffer *buffer,
+                                        gint64 new_size,
+                                        GError **error);
+gboolean garrow_resizable_buffer_reserve(GArrowResizableBuffer *buffer,
+                                         gint64 new_capacity,
+                                         GError **error);
+
+
+#define GARROW_TYPE_POOL_BUFFER                 \
+  (garrow_pool_buffer_get_type())
+#define GARROW_POOL_BUFFER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_POOL_BUFFER,  \
+                              GArrowPoolBuffer))
+#define GARROW_POOL_BUFFER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_POOL_BUFFER,     \
+                           GArrowPoolBufferClass))
+#define GARROW_IS_POOL_BUFFER(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_POOL_BUFFER))
+#define GARROW_IS_POOL_BUFFER_CLASS(klass)                      \
+  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_POOL_BUFFER))
+#define GARROW_POOL_BUFFER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_POOL_BUFFER,   \
+                             GArrowPoolBufferClass))
+
+typedef struct _GArrowPoolBuffer         GArrowPoolBuffer;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowPoolBufferClass    GArrowPoolBufferClass;
+#endif
+
+/**
+ * GArrowPoolBuffer:
+ *
+ * It wraps `arrow::PoolBuffer`.
+ */
+struct _GArrowPoolBuffer
+{
+  /*< private >*/
+  GArrowResizableBuffer parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowPoolBufferClass
+{
+  GArrowResizableBufferClass parent_class;
+};
+#endif
+
+GType garrow_pool_buffer_get_type(void) G_GNUC_CONST;
+
+GArrowPoolBuffer *garrow_pool_buffer_new(void);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/buffer.hpp b/c_glib/arrow-glib/buffer.hpp
index 1337d9ed596f9..d1664b11b17c9 100644
--- a/c_glib/arrow-glib/buffer.hpp
+++ b/c_glib/arrow-glib/buffer.hpp
@@ -27,3 +27,4 @@ GArrowBuffer *garrow_buffer_new_raw(std::shared_ptr<arrow::Buffer> *arrow_buffer
 std::shared_ptr<arrow::Buffer> garrow_buffer_get_raw(GArrowBuffer *buffer);
 
 GArrowMutableBuffer *garrow_mutable_buffer_new_raw(std::shared_ptr<arrow::MutableBuffer> *arrow_buffer);
+GArrowPoolBuffer *garrow_pool_buffer_new_raw(std::shared_ptr<arrow::PoolBuffer> *arrow_buffer);
diff --git a/c_glib/test/test-pool-buffer.rb b/c_glib/test/test-pool-buffer.rb
new file mode 100644
index 0000000000000..57f3458ef1efb
--- /dev/null
+++ b/c_glib/test/test-pool-buffer.rb
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestPoolBuffer < Test::Unit::TestCase
+  def setup
+    @buffer = Arrow::PoolBuffer.new
+  end
+
+  def test_resize
+    @buffer.resize(1)
+    assert_equal(1, @buffer.size)
+  end
+
+  def test_reserve
+    @buffer.reserve(1)
+    assert_equal(64, @buffer.capacity)
+  end
+end

From 949249d9e85d2464a3f1c65b176b636d1cfbaf1a Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 25 Apr 2017 17:29:41 -0400
Subject: [PATCH 0568/1644] ARROW-893: Add GLib document to Web site

Author: Kouhei Sutou <kou@clear-code.com>

Closes #599 from kou/site-glib-doc and squashes the following commits:

f85ad44 [Kouhei Sutou] Add GLib document to Web site
---
 site/README.md             | 21 ++++++++++++++++++++-
 site/_includes/header.html |  1 +
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/site/README.md b/site/README.md
index 3f8da2252f965..aeebaa1790ef9 100644
--- a/site/README.md
+++ b/site/README.md
@@ -82,4 +82,23 @@ python setup.py build_sphinx -s doc/source
 rsync -r doc/_build/html/ ../site/asf-site/docs/python/
 ```
 
-Then add/commit/push from the site/asf-site git checkout.
\ No newline at end of file
+#### C (GLib)
+
+First, build Apache Arrow C++ and Apache Arrow GLib.
+
+```
+mkdir -p ../cpp/build
+cd ../cpp/build
+cmake .. -DCMAKE_BUILD_TYPE=debug
+make
+cd ../../c_glib
+./autogen.sh
+./configure \
+  --with-arrow-cpp-build-dir=$PWD/../cpp/build \
+  --with-arrow-cpp-build-type=debug \
+  --enable-gtk-doc
+LD_LIBRARY_PATH=$PWD/../cpp/build/debug make GTK_DOC_V_XREF=": "
+rsync -r doc/reference/html/ ../site/asf-site/docs/c_glib/
+```
+
+Then add/commit/push from the site/asf-site git checkout.
diff --git a/site/_includes/header.html b/site/_includes/header.html
index 5963c22abea0d..3d61494f2f109 100644
--- a/site/_includes/header.html
+++ b/site/_includes/header.html
@@ -28,6 +28,7 @@
             <li><a href="{{ site.baseurl }}/docs/cpp">C++</a></li>
             <li><a href="{{ site.baseurl }}/docs/java">Java</a></li>
             <li><a href="{{ site.baseurl }}/docs/python">Python</a></li>
+            <li><a href="{{ site.baseurl }}/docs/c_glib">C (GLib)</a></li>
           </ul>
         </li>
         <!-- <li><a href="{{ site.baseurl }}/blog">Blog</a></li> -->

From 7d433dc27bf70b5d80b8c88261a19cdc615defdb Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Tue, 25 Apr 2017 17:36:31 -0400
Subject: [PATCH 0569/1644] ARROW-483: [C++/Python] Provide access to
 "custom_metadata" Field attribute in IPC setting

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #588 from cpcloud/ARROW-483 and squashes the following commits:

f671ba4 [Phillip Cloud] ARROW-483: [C++/Python] Provide access to "custom_metadata" Field attribute in IPC setting
---
 cpp/CMakeLists.txt                            |  1 +
 cpp/src/arrow/array.cc                        |  2 +-
 cpp/src/arrow/builder.cc                      | 13 ++-
 cpp/src/arrow/ipc/metadata.cc                 | 30 +++++-
 cpp/src/arrow/type-test.cc                    | 34 +++++++
 cpp/src/arrow/type.cc                         | 20 +++-
 cpp/src/arrow/type.h                          | 10 +-
 cpp/src/arrow/util/CMakeLists.txt             |  2 +
 cpp/src/arrow/util/key-value-metadata-test.cc | 87 ++++++++++++++++
 cpp/src/arrow/util/key_value_metadata.cc      | 99 +++++++++++++++++++
 cpp/src/arrow/util/key_value_metadata.h       | 56 +++++++++++
 format/Schema.fbs                             |  2 +-
 python/.gitignore                             |  1 +
 python/pyarrow/_array.pxd                     |  2 +
 python/pyarrow/_array.pyx                     |  7 ++
 python/pyarrow/_table.pyx                     | 64 +++++++-----
 python/pyarrow/includes/common.pxd            |  3 +-
 python/pyarrow/includes/libarrow.pxd          | 11 ++-
 18 files changed, 401 insertions(+), 43 deletions(-)
 create mode 100644 cpp/src/arrow/util/key-value-metadata-test.cc
 create mode 100644 cpp/src/arrow/util/key_value_metadata.cc
 create mode 100644 cpp/src/arrow/util/key_value_metadata.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 2d8c00fd80803..5abe5f1436ea7 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -944,6 +944,7 @@ set(ARROW_SRCS
 
   src/arrow/util/bit-util.cc
   src/arrow/util/decimal.cc
+  src/arrow/util/key_value_metadata.cc
 )
 
 if (ARROW_IPC)
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index e640bbd4a206e..76dda2ca7b94f 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -113,7 +113,7 @@ Status Array::Validate() const {
 static inline void ConformSliceParams(
     int64_t array_offset, int64_t array_length, int64_t* offset, int64_t* length) {
   DCHECK_LE(*offset, array_length);
-  DCHECK_GE(offset, 0);
+  DCHECK_NE(offset, nullptr);
   *length = std::min(array_length - *offset, *length);
   *offset = array_offset + *offset;
 }
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index d85eb32652c47..4ecb8d3500981 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -363,8 +363,6 @@ ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal128& value) {
   return Status::OK();
 }
 
-template ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal128& val);
-
 Status DecimalBuilder::Init(int64_t capacity) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Init(capacity));
   if (byte_width_ == 16) {
@@ -408,16 +406,17 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
     const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(
-          pool, type ? type : std::static_pointer_cast<DataType>(
-                                  std::make_shared<ListType>(value_builder->type()))),
+    : ArrayBuilder(pool,
+          type ? type : std::static_pointer_cast<DataType>(
+                            std::make_shared<ListType>(value_builder->type()))),
       offset_builder_(pool),
       value_builder_(value_builder) {}
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<Array> values,
     const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
-                                           std::make_shared<ListType>(values->type()))),
+    : ArrayBuilder(pool,
+          type ? type : std::static_pointer_cast<DataType>(
+                            std::make_shared<ListType>(values->type()))),
       offset_builder_(pool),
       values_(values) {}
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 791948b50b0ac..c0b518a0d8e50 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -45,6 +45,7 @@ namespace ipc {
 using FBB = flatbuffers::FlatBufferBuilder;
 using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
 using FieldOffset = flatbuffers::Offset<flatbuf::Field>;
+using KeyValueOffset = flatbuffers::Offset<flatbuf::KeyValue>;
 using RecordBatchOffset = flatbuffers::Offset<flatbuf::RecordBatch>;
 using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
@@ -583,6 +584,7 @@ flatbuf::Endianness endianness() {
 
 static Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
     DictionaryMemo* dictionary_memo, flatbuffers::Offset<flatbuf::Schema>* out) {
+  /// Fields
   std::vector<FieldOffset> field_offsets;
   for (int i = 0; i < schema.num_fields(); ++i) {
     std::shared_ptr<Field> field = schema.field(i);
@@ -591,7 +593,20 @@ static Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
     field_offsets.push_back(offset);
   }
 
-  *out = flatbuf::CreateSchema(fbb, endianness(), fbb.CreateVector(field_offsets));
+  /// Custom metadata
+  const auto& custom_metadata_ = schema.custom_metadata();
+  std::vector<KeyValueOffset> key_value_offsets;
+  size_t metadata_size = custom_metadata_.size();
+  key_value_offsets.reserve(metadata_size);
+  for (size_t i = 0; i < metadata_size; ++i) {
+    const auto& key = custom_metadata_.key(i);
+    const auto& value = custom_metadata_.value(i);
+    key_value_offsets.push_back(
+        flatbuf::CreateKeyValue(fbb, fbb.CreateString(key), fbb.CreateString(value)));
+  }
+
+  *out = flatbuf::CreateSchema(fbb, endianness(), fbb.CreateVector(field_offsets),
+      fbb.CreateVector(key_value_offsets));
   return Status::OK();
 }
 
@@ -939,7 +954,18 @@ Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_mem
     const flatbuf::Field* field = schema->fields()->Get(i);
     RETURN_NOT_OK(FieldFromFlatbuffer(field, dictionary_memo, &fields[i]));
   }
-  *out = std::make_shared<Schema>(fields);
+
+  KeyValueMetadata custom_metadata;
+  auto fb_metadata = schema->custom_metadata();
+  if (fb_metadata != nullptr) {
+    custom_metadata.reserve(fb_metadata->size());
+
+    for (const auto& pair : *fb_metadata) {
+      custom_metadata.Append(pair->key()->str(), pair->value()->str());
+    }
+  }
+
+  *out = std::make_shared<Schema>(fields, custom_metadata);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index dec7268a5a8b5..8e2dfd50e431d 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -117,6 +117,40 @@ TEST_F(TestSchema, GetFieldByName) {
   ASSERT_TRUE(result == nullptr);
 }
 
+TEST_F(TestSchema, TestCustomMetadataConstruction) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8(), false);
+  auto f2 = field("f2", utf8());
+  vector<shared_ptr<Field>> fields = {f0, f1, f2};
+  KeyValueMetadata metadata({"foo", "bar"}, {"bizz", "buzz"});
+  auto schema = std::make_shared<Schema>(fields, metadata);
+  ASSERT_TRUE(metadata.Equals(schema->custom_metadata()));
+}
+
+TEST_F(TestSchema, TestAddCustomMetadata) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8(), false);
+  auto f2 = field("f2", utf8());
+  vector<shared_ptr<Field>> fields = {f0, f1, f2};
+  KeyValueMetadata metadata({"foo", "bar"}, {"bizz", "buzz"});
+  auto schema = std::make_shared<Schema>(fields);
+  std::shared_ptr<Schema> new_schema;
+  schema->AddCustomMetadata(metadata, &new_schema);
+  ASSERT_TRUE(metadata.Equals(new_schema->custom_metadata()));
+}
+
+TEST_F(TestSchema, TestRemoveCustomMetadata) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8(), false);
+  auto f2 = field("f2", utf8());
+  vector<shared_ptr<Field>> fields = {f0, f1, f2};
+  KeyValueMetadata metadata({"foo", "bar"}, {"bizz", "buzz"});
+  auto schema = std::make_shared<Schema>(fields);
+  std::shared_ptr<Schema> new_schema;
+  schema->RemoveCustomMetadata(&new_schema);
+  ASSERT_EQ(0, new_schema->custom_metadata().size());
+}
+
 #define PRIMITIVE_TEST(KLASS, ENUM, NAME)        \
   TEST(TypesTest, TestPrimitive_##ENUM) {        \
     KLASS tp;                                    \
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 2e454ae81886f..f59f8fb26c9ba 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -24,6 +24,7 @@
 #include "arrow/array.h"
 #include "arrow/compare.h"
 #include "arrow/status.h"
+#include "arrow/util/key_value_metadata.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/stl.h"
 #include "arrow/visitor.h"
@@ -231,7 +232,9 @@ std::string NullType::ToString() const {
 // ----------------------------------------------------------------------
 // Schema implementation
 
-Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields) : fields_(fields) {}
+Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields,
+    const KeyValueMetadata& custom_metadata)
+    : fields_(fields), custom_metadata_(custom_metadata) {}
 
 bool Schema::Equals(const Schema& other) const {
   if (this == &other) { return true; }
@@ -263,7 +266,18 @@ Status Schema::AddField(
   DCHECK_GE(i, 0);
   DCHECK_LE(i, this->num_fields());
 
-  *out = std::make_shared<Schema>(AddVectorElement(fields_, i, field));
+  *out = std::make_shared<Schema>(AddVectorElement(fields_, i, field), custom_metadata_);
+  return Status::OK();
+}
+
+Status Schema::AddCustomMetadata(
+    const KeyValueMetadata& custom_metadata, std::shared_ptr<Schema>* out) const {
+  *out = std::make_shared<Schema>(fields_, custom_metadata);
+  return Status::OK();
+}
+
+Status Schema::RemoveCustomMetadata(std::shared_ptr<Schema>* out) {
+  *out = std::make_shared<Schema>(fields_, KeyValueMetadata());
   return Status::OK();
 }
 
@@ -271,7 +285,7 @@ Status Schema::RemoveField(int i, std::shared_ptr<Schema>* out) const {
   DCHECK_GE(i, 0);
   DCHECK_LT(i, this->num_fields());
 
-  *out = std::make_shared<Schema>(DeleteVectorElement(fields_, i));
+  *out = std::make_shared<Schema>(DeleteVectorElement(fields_, i), custom_metadata_);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index ea4ea03ff569a..dc9456137030f 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -28,6 +28,7 @@
 
 #include "arrow/status.h"
 #include "arrow/type_fwd.h"
+#include "arrow/util/key_value_metadata.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 #include "arrow/visitor.h"
@@ -677,7 +678,8 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 
 class ARROW_EXPORT Schema {
  public:
-  explicit Schema(const std::vector<std::shared_ptr<Field>>& fields);
+  explicit Schema(const std::vector<std::shared_ptr<Field>>& fields,
+      const KeyValueMetadata& custom_metadata = KeyValueMetadata());
 
   // Returns true if all of the schema fields are equal
   bool Equals(const Schema& other) const;
@@ -689,6 +691,7 @@ class ARROW_EXPORT Schema {
   std::shared_ptr<Field> GetFieldByName(const std::string& name);
 
   const std::vector<std::shared_ptr<Field>>& fields() const { return fields_; }
+  const KeyValueMetadata& custom_metadata() const { return custom_metadata_; }
 
   // Render a string representation of the schema suitable for debugging
   std::string ToString() const;
@@ -697,11 +700,16 @@ class ARROW_EXPORT Schema {
       int i, const std::shared_ptr<Field>& field, std::shared_ptr<Schema>* out) const;
   Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
 
+  Status AddCustomMetadata(
+      const KeyValueMetadata& metadata, std::shared_ptr<Schema>* out) const;
+  Status RemoveCustomMetadata(std::shared_ptr<Schema>* out);
+
   int num_fields() const { return static_cast<int>(fields_.size()); }
 
  private:
   std::vector<std::shared_ptr<Field>> fields_;
   std::unordered_map<std::string, int> name_to_index_;
+  KeyValueMetadata custom_metadata_;
 };
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index b22c8aca11c5d..ac7e86615eb40 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -26,6 +26,7 @@ install(FILES
   macros.h
   random.h
   visibility.h
+  key_value_metadata.h
   DESTINATION include/arrow/util)
 
 #######################################
@@ -52,3 +53,4 @@ endif()
 ADD_ARROW_TEST(bit-util-test)
 ADD_ARROW_TEST(stl-util-test)
 ADD_ARROW_TEST(decimal-test)
+ADD_ARROW_TEST(key-value-metadata-test)
diff --git a/cpp/src/arrow/util/key-value-metadata-test.cc b/cpp/src/arrow/util/key-value-metadata-test.cc
new file mode 100644
index 0000000000000..aadc989cb403f
--- /dev/null
+++ b/cpp/src/arrow/util/key-value-metadata-test.cc
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/util/key_value_metadata.h"
+
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+TEST(KeyValueMetadataTest, SimpleConstruction) {
+  KeyValueMetadata metadata;
+  ASSERT_EQ(0, metadata.size());
+}
+
+TEST(KeyValueMetadataTest, StringVectorConstruction) {
+  std::vector<std::string> keys = {"foo", "bar"};
+  std::vector<std::string> values = {"bizz", "buzz"};
+
+  KeyValueMetadata metadata(keys, values);
+  ASSERT_EQ("foo", metadata.key(0));
+  ASSERT_EQ("bar", metadata.key(1));
+  ASSERT_EQ("bizz", metadata.value(0));
+  ASSERT_EQ("buzz", metadata.value(1));
+  ASSERT_EQ(2, metadata.size());
+}
+
+TEST(KeyValueMetadataTest, StringMapConstruction) {
+  std::unordered_map<std::string, std::string> pairs = {{"foo", "bizz"}, {"bar", "buzz"}};
+  std::unordered_map<std::string, std::string> result_map;
+  result_map.reserve(pairs.size());
+
+  KeyValueMetadata metadata(pairs);
+  metadata.ToUnorderedMap(&result_map);
+  ASSERT_EQ(pairs, result_map);
+  ASSERT_EQ(2, metadata.size());
+}
+
+TEST(KeyValueMetadataTest, StringAppend) {
+  std::vector<std::string> keys = {"foo", "bar"};
+  std::vector<std::string> values = {"bizz", "buzz"};
+
+  KeyValueMetadata metadata(keys, values);
+  ASSERT_EQ("foo", metadata.key(0));
+  ASSERT_EQ("bar", metadata.key(1));
+  ASSERT_EQ("bizz", metadata.value(0));
+  ASSERT_EQ("buzz", metadata.value(1));
+  ASSERT_EQ(2, metadata.size());
+
+  metadata.Append("purple", "orange");
+  metadata.Append("blue", "red");
+
+  ASSERT_EQ("purple", metadata.key(2));
+  ASSERT_EQ("blue", metadata.key(3));
+
+  ASSERT_EQ("orange", metadata.value(2));
+  ASSERT_EQ("red", metadata.value(3));
+}
+
+TEST(KeyValueMetadataTest, Equals) {
+  std::vector<std::string> keys = {"foo", "bar"};
+  std::vector<std::string> values = {"bizz", "buzz"};
+
+  KeyValueMetadata metadata(keys, values);
+  KeyValueMetadata metadata2(keys, values);
+  KeyValueMetadata metadata3(keys, {"buzz", "bizz"});
+
+  ASSERT_TRUE(metadata.Equals(metadata2));
+  ASSERT_FALSE(metadata.Equals(metadata3));
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/key_value_metadata.cc b/cpp/src/arrow/util/key_value_metadata.cc
new file mode 100644
index 0000000000000..c91478bd1acd6
--- /dev/null
+++ b/cpp/src/arrow/util/key_value_metadata.cc
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <algorithm>
+
+#include "arrow/util/key_value_metadata.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+static std::vector<std::string> UnorderedMapKeys(
+    const std::unordered_map<std::string, std::string>& map) {
+  std::vector<std::string> keys;
+  keys.reserve(map.size());
+  for (const auto& pair : map) {
+    keys.push_back(pair.first);
+  }
+  return keys;
+}
+
+static std::vector<std::string> UnorderedMapValues(
+    const std::unordered_map<std::string, std::string>& map) {
+  std::vector<std::string> values;
+  values.reserve(map.size());
+  for (const auto& pair : map) {
+    values.push_back(pair.second);
+  }
+  return values;
+}
+
+KeyValueMetadata::KeyValueMetadata() : keys_(), values_() {}
+
+KeyValueMetadata::KeyValueMetadata(
+    const std::unordered_map<std::string, std::string>& map)
+    : keys_(UnorderedMapKeys(map)), values_(UnorderedMapValues(map)) {}
+
+KeyValueMetadata::KeyValueMetadata(
+    const std::vector<std::string>& keys, const std::vector<std::string>& values)
+    : keys_(keys), values_(values) {
+  DCHECK_EQ(keys.size(), values.size());
+}
+
+void KeyValueMetadata::ToUnorderedMap(
+    std::unordered_map<std::string, std::string>* out) const {
+  DCHECK_NE(out, nullptr);
+  const int64_t n = size();
+  out->reserve(n);
+  for (int64_t i = 0; i < n; ++i) {
+    out->insert(std::make_pair(key(i), value(i)));
+  }
+}
+
+void KeyValueMetadata::Append(const std::string& key, const std::string& value) {
+  keys_.push_back(key);
+  values_.push_back(value);
+}
+
+void KeyValueMetadata::reserve(int64_t n) {
+  DCHECK_GE(n, 0);
+  const auto m = static_cast<size_t>(n);
+  keys_.reserve(m);
+  values_.reserve(m);
+}
+
+int64_t KeyValueMetadata::size() const {
+  DCHECK_EQ(keys_.size(), values_.size());
+  return static_cast<int64_t>(keys_.size());
+}
+
+std::string KeyValueMetadata::key(int64_t i) const {
+  DCHECK_GE(i, 0);
+  return keys_[static_cast<size_t>(i)];
+}
+
+std::string KeyValueMetadata::value(int64_t i) const {
+  DCHECK_GE(i, 0);
+  return values_[static_cast<size_t>(i)];
+}
+
+bool KeyValueMetadata::Equals(const KeyValueMetadata& other) const {
+  return size() == other.size() &&
+         std::equal(keys_.cbegin(), keys_.cend(), other.keys_.cbegin()) &&
+         std::equal(values_.cbegin(), values_.cend(), other.values_.cbegin());
+}
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/key_value_metadata.h b/cpp/src/arrow/util/key_value_metadata.h
new file mode 100644
index 0000000000000..713b2c0b0bcfb
--- /dev/null
+++ b/cpp/src/arrow/util/key_value_metadata.h
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_KEY_VALUE_METADATA_H
+#define ARROW_UTIL_KEY_VALUE_METADATA_H
+
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class ARROW_EXPORT KeyValueMetadata {
+ public:
+  KeyValueMetadata();
+  KeyValueMetadata(
+      const std::vector<std::string>& keys, const std::vector<std::string>& values);
+  explicit KeyValueMetadata(const std::unordered_map<std::string, std::string>& map);
+
+  void ToUnorderedMap(std::unordered_map<std::string, std::string>* out) const;
+
+  void Append(const std::string& key, const std::string& value);
+
+  void reserve(int64_t n);
+  int64_t size() const;
+
+  std::string key(int64_t i) const;
+  std::string value(int64_t i) const;
+
+  bool Equals(const KeyValueMetadata& other) const;
+
+ private:
+  std::vector<std::string> keys_;
+  std::vector<std::string> values_;
+};
+
+}  // namespace arrow
+
+#endif  //  ARROW_UTIL_KEY_VALUE_METADATA_H
diff --git a/format/Schema.fbs b/format/Schema.fbs
index b48859f50eea2..8de5c6d466c36 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -200,7 +200,7 @@ table VectorLayout {
 
 table KeyValue {
   key: string;
-  value: [ubyte];
+  value: string;
 }
 
 /// ----------------------------------------------------------------------
diff --git a/python/.gitignore b/python/.gitignore
index ba40c3ea88882..6c0d5a93cd35c 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -33,3 +33,4 @@ coverage.xml
 
 # benchmark working dir
 .asv
+pyarrow/_table_api.h
diff --git a/python/pyarrow/_array.pxd b/python/pyarrow/_array.pxd
index 464de316f0437..4d5db8618a377 100644
--- a/python/pyarrow/_array.pxd
+++ b/python/pyarrow/_array.pxd
@@ -81,6 +81,8 @@ cdef class Schema:
     cdef init(self, const vector[shared_ptr[CField]]& fields)
     cdef init_schema(self, const shared_ptr[CSchema]& schema)
 
+    cpdef dict custom_metadata(self)
+
 
 cdef class Scalar:
     cdef readonly:
diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/_array.pyx
index 1c571ba153dfa..2fb20b7553e93 100644
--- a/python/pyarrow/_array.pyx
+++ b/python/pyarrow/_array.pyx
@@ -244,6 +244,13 @@ cdef class Schema:
         self.schema = schema.get()
         self.sp_schema = schema
 
+    cpdef dict custom_metadata(self):
+        cdef:
+            CKeyValueMetadata metadata = self.schema.custom_metadata()
+            unordered_map[c_string, c_string] result
+        metadata.ToUnorderedMap(&result)
+        return result
+
     def equals(self, other):
         """
         Test if this schema is equal to the other
diff --git a/python/pyarrow/_table.pyx b/python/pyarrow/_table.pyx
index 78fec75cf3e7d..ed0782bbba0a3 100644
--- a/python/pyarrow/_table.pyx
+++ b/python/pyarrow/_table.pyx
@@ -34,7 +34,6 @@ from pyarrow._error import ArrowException
 from pyarrow._array import field
 from pyarrow.compat import frombytes, tobytes
 
-
 from collections import OrderedDict
 
 
@@ -273,15 +272,22 @@ cdef class Column:
         return chunked_array
 
 
-cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
+cdef CKeyValueMetadata key_value_metadata_from_dict(dict metadata):
+    cdef:
+        unordered_map[c_string, c_string] unordered_metadata = metadata
+        CKeyValueMetadata c_metadata = CKeyValueMetadata(unordered_metadata)
+    return c_metadata
+
+
+cdef int _schema_from_arrays(
+        arrays, names, dict metadata, shared_ptr[CSchema]* schema) except -1:
     cdef:
         Array arr
         Column col
         c_string c_name
         vector[shared_ptr[CField]] fields
-        cdef shared_ptr[CDataType] type_
-
-    cdef int K = len(arrays)
+        shared_ptr[CDataType] type_
+        int K = len(arrays)
 
     fields.resize(K)
 
@@ -306,15 +312,16 @@ cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
     else:
         raise TypeError(type(arrays[0]))
 
-    schema.reset(new CSchema(fields))
-
+    schema.reset(new CSchema(fields, key_value_metadata_from_dict(metadata)))
+    return 0
 
 
-cdef _dataframe_to_arrays(df, timestamps_to_ms, Schema schema):
+cdef tuple _dataframe_to_arrays(df, bint timestamps_to_ms, Schema schema):
     cdef:
         list names = []
         list arrays = []
         DataType type = None
+        dict metadata = {}
 
     for name in df.columns:
         col = df[name]
@@ -326,7 +333,7 @@ cdef _dataframe_to_arrays(df, timestamps_to_ms, Schema schema):
         names.append(name)
         arrays.append(arr)
 
-    return names, arrays
+    return names, arrays, metadata
 
 
 cdef class RecordBatch:
@@ -486,11 +493,11 @@ cdef class RecordBatch:
         -------
         pyarrow.table.RecordBatch
         """
-        names, arrays = _dataframe_to_arrays(df, False, schema)
-        return cls.from_arrays(arrays, names)
+        names, arrays, metadata = _dataframe_to_arrays(df, False, schema)
+        return cls.from_arrays(arrays, names, metadata)
 
     @staticmethod
-    def from_arrays(arrays, names):
+    def from_arrays(list arrays, list names, dict metadata=None):
         """
         Construct a RecordBatch from multiple pyarrow.Arrays
 
@@ -512,15 +519,17 @@ cdef class RecordBatch:
             shared_ptr[CRecordBatch] batch
             vector[shared_ptr[CArray]] c_arrays
             int64_t num_rows
+            int64_t i
+            int64_t number_of_arrays = len(arrays)
 
-        if len(arrays) == 0:
+        if not number_of_arrays:
             raise ValueError('Record batch cannot contain no arrays (for now)')
 
         num_rows = len(arrays[0])
-        _schema_from_arrays(arrays, names, &schema)
+        _schema_from_arrays(arrays, names, metadata or {}, &schema)
 
-        for i in range(len(arrays)):
-            arr = arrays[i]
+        c_arrays.reserve(len(arrays))
+        for arr in arrays:
             c_arrays.push_back(arr.sp_array)
 
         batch.reset(new CRecordBatch(schema, num_rows, c_arrays))
@@ -656,13 +665,13 @@ cdef class Table:
         >>> pa.Table.from_pandas(df)
         <pyarrow.table.Table object at 0x7f05d1fb1b40>
         """
-        names, arrays = _dataframe_to_arrays(df,
+        names, arrays, metadata = _dataframe_to_arrays(df,
                                              timestamps_to_ms=timestamps_to_ms,
                                              schema=schema)
-        return cls.from_arrays(arrays, names=names)
+        return cls.from_arrays(arrays, names=names, metadata=metadata)
 
     @staticmethod
-    def from_arrays(arrays, names=None):
+    def from_arrays(arrays, names=None, dict metadata=None):
         """
         Construct a Table from Arrow arrays or columns
 
@@ -680,22 +689,25 @@ cdef class Table:
 
         """
         cdef:
-            vector[shared_ptr[CField]] fields
             vector[shared_ptr[CColumn]] columns
             shared_ptr[CSchema] schema
             shared_ptr[CTable] table
+            size_t K = len(arrays)
 
-        _schema_from_arrays(arrays, names, &schema)
+        _schema_from_arrays(arrays, names, metadata or {}, &schema)
 
-        cdef int K = len(arrays)
-        columns.resize(K)
+        columns.reserve(K)
 
         for i in range(K):
             if isinstance(arrays[i], Array):
-                columns[i].reset(new CColumn(schema.get().field(i),
-                                             (<Array> arrays[i]).sp_array))
+                columns.push_back(
+                    make_shared[CColumn](
+                        schema.get().field(i),
+                        (<Array> arrays[i]).sp_array
+                    )
+                )
             elif isinstance(arrays[i], Column):
-                columns[i] = (<Column> arrays[i]).sp_column
+                columns.push_back((<Column> arrays[i]).sp_column)
             else:
                 raise ValueError(type(arrays[i]))
 
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 44723faa7400e..cc3b4b6fdaf92 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -19,9 +19,10 @@
 
 from libc.stdint cimport *
 from libcpp cimport bool as c_bool
-from libcpp.memory cimport shared_ptr, unique_ptr
+from libcpp.memory cimport shared_ptr, unique_ptr, make_shared
 from libcpp.string cimport string as c_string
 from libcpp.vector cimport vector
+from libcpp.unordered_map cimport unordered_map
 
 from cpython cimport PyObject
 cimport cpython
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 473a0b9cd9b6d..ef1a332bed52f 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -1,4 +1,4 @@
-#t Licensed to the Apache Software Foundation (ASF) under one
+# Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
@@ -19,6 +19,12 @@
 
 from pyarrow.includes.common cimport *
 
+cdef extern from "arrow/util/key_value_metadata.h" namespace "arrow" nogil:
+    cdef cppclass CKeyValueMetadata" arrow::KeyValueMetadata":
+        CKeyValueMetadata()
+        CKeyValueMetadata(const unordered_map[c_string, c_string]&)
+        void ToUnorderedMap(unordered_map[c_string, c_string]*) const
+
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     enum Type" arrow::Type::type":
@@ -170,10 +176,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CSchema" arrow::Schema":
         CSchema(const vector[shared_ptr[CField]]& fields)
+        CSchema(const vector[shared_ptr[CField]]& fields,
+                const CKeyValueMetadata& custom_metadata)
 
         c_bool Equals(const CSchema& other)
 
         shared_ptr[CField] field(int i)
+        const CKeyValueMetadata& custom_metadata() const
         shared_ptr[CField] GetFieldByName(c_string& name)
         int num_fields()
         c_string ToString()

From 3ad9d09f39ead51266299ec4bbb703724b8ac69d Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 26 Apr 2017 10:30:26 -0400
Subject: [PATCH 0570/1644] ARROW-904: [GLib] Simplify error check codes

Author: Kouhei Sutou <kou@clear-code.com>

Closes #604 from kou/glib-simplify-error-check and squashes the following commits:

1cf6b76 [Kouhei Sutou] [GLib] Simplify error check codes
---
 c_glib/arrow-glib/array-builder.cpp      | 210 +++++------------------
 c_glib/arrow-glib/buffer.cpp             |  17 +-
 c_glib/arrow-glib/error.cpp              |  25 +--
 c_glib/arrow-glib/error.hpp              |   6 +-
 c_glib/arrow-glib/file-output-stream.cpp |   2 +-
 c_glib/arrow-glib/file-reader.cpp        |   8 +-
 c_glib/arrow-glib/file-writer.cpp        |  19 +-
 c_glib/arrow-glib/file.cpp               |  10 +-
 c_glib/arrow-glib/memory-mapped-file.cpp |   2 +-
 c_glib/arrow-glib/random-access-file.cpp |  10 +-
 c_glib/arrow-glib/readable.cpp           |   7 +-
 c_glib/arrow-glib/stream-reader.cpp      |   8 +-
 c_glib/arrow-glib/stream-writer.cpp      |  19 +-
 c_glib/arrow-glib/table.cpp              |   6 +-
 c_glib/arrow-glib/writeable-file.cpp     |   7 +-
 c_glib/arrow-glib/writeable.cpp          |  14 +-
 16 files changed, 90 insertions(+), 280 deletions(-)

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 97d43e1f0c022..30158b07b11d3 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -237,12 +237,7 @@ garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(static_cast<bool>(value));
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[boolean-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[boolean-array-builder][append]");
 }
 
 /**
@@ -261,12 +256,9 @@ garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[boolean-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error,
+                            status,
+                            "[boolean-array-builder][append-null]");
 }
 
 
@@ -318,12 +310,7 @@ garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int8-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[int8-array-builder][append]");
 }
 
 /**
@@ -342,12 +329,7 @@ garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int8-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[int8-array-builder][append-null]");
 }
 
 
@@ -399,12 +381,7 @@ garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint8-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[uint8-array-builder][append]");
 }
 
 /**
@@ -423,12 +400,7 @@ garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint8-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[uint8-array-builder][append-null]");
 }
 
 
@@ -480,12 +452,7 @@ garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int16-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[int16-array-builder][append]");
 }
 
 /**
@@ -504,12 +471,7 @@ garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int16-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[int16-array-builder][append-null]");
 }
 
 
@@ -561,12 +523,7 @@ garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint16-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[uint16-array-builder][append]");
 }
 
 /**
@@ -585,12 +542,9 @@ garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint16-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error,
+                            status,
+                            "[uint16-array-builder][append-null]");
 }
 
 
@@ -642,12 +596,7 @@ garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int32-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[int32-array-builder][append]");
 }
 
 /**
@@ -666,12 +615,7 @@ garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int32-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[int32-array-builder][append-null]");
 }
 
 
@@ -723,12 +667,7 @@ garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint32-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[uint32-array-builder][append]");
 }
 
 /**
@@ -747,12 +686,9 @@ garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint32-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error,
+                            status,
+                            "[uint32-array-builder][append-null]");
 }
 
 
@@ -804,12 +740,7 @@ garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int64-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[int64-array-builder][append]");
 }
 
 /**
@@ -828,12 +759,7 @@ garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[int64-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[int64-array-builder][append-null]");
 }
 
 
@@ -885,12 +811,7 @@ garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[uint64-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[uint64-array-builder][append]");
 }
 
 /**
@@ -912,7 +833,7 @@ garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
   if (status.ok()) {
     return TRUE;
   } else {
-    garrow_error_set(error, status, "[uint64-array-builder][append-null]");
+    garrow_error_check(error, status, "[uint64-array-builder][append-null]");
     return FALSE;
   }
 }
@@ -966,12 +887,7 @@ garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[float-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[float-array-builder][append]");
 }
 
 /**
@@ -990,12 +906,7 @@ garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[float-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[float-array-builder][append-null]");
 }
 
 
@@ -1047,12 +958,7 @@ garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[double-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[double-array-builder][append]");
 }
 
 /**
@@ -1071,12 +977,9 @@ garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[double-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error,
+                            status,
+                            "[double-array-builder][append-null]");
 }
 
 
@@ -1130,12 +1033,7 @@ garrow_binary_array_builder_append(GArrowBinaryArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append(value, length);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[binary-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[binary-array-builder][append]");
 }
 
 /**
@@ -1154,12 +1052,9 @@ garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[binary-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error,
+                            status,
+                            "[binary-array-builder][append-null]");
 }
 
 
@@ -1212,12 +1107,7 @@ garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
 
   auto status = arrow_builder->Append(value,
                                       static_cast<gint32>(strlen(value)));
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[string-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[string-array-builder][append]");
 }
 
 
@@ -1305,12 +1195,7 @@ garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[list-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[list-array-builder][append]");
 }
 
 /**
@@ -1331,12 +1216,7 @@ garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[list-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[list-array-builder][append-null]");
 }
 
 /**
@@ -1427,12 +1307,7 @@ garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->Append();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[struct-array-builder][append]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[struct-array-builder][append]");
 }
 
 /**
@@ -1453,12 +1328,9 @@ garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
 
   auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[struct-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_error_check(error,
+                            status,
+                            "[struct-array-builder][append-null]");
 }
 
 /**
diff --git a/c_glib/arrow-glib/buffer.cpp b/c_glib/arrow-glib/buffer.cpp
index 5c28daf674e4e..4373ef1c83447 100644
--- a/c_glib/arrow-glib/buffer.cpp
+++ b/c_glib/arrow-glib/buffer.cpp
@@ -272,10 +272,9 @@ garrow_buffer_copy(GArrowBuffer *buffer,
   auto arrow_buffer = garrow_buffer_get_raw(buffer);
   std::shared_ptr<arrow::Buffer> arrow_copied_buffer;
   auto status = arrow_buffer->Copy(start, size, &arrow_copied_buffer);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[buffer][copy]")) {
     return garrow_buffer_new_raw(&arrow_copied_buffer);
   } else {
-    garrow_error_set(error, status, "[buffer][copy]");
     return NULL;
   }
 }
@@ -396,12 +395,7 @@ garrow_resizable_buffer_resize(GArrowResizableBuffer *buffer,
   auto arrow_resizable_buffer =
     std::static_pointer_cast<arrow::ResizableBuffer>(arrow_buffer);
   auto status = arrow_resizable_buffer->Resize(new_size);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[resizable-buffer][resize]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[resizable-buffer][resize]");
 }
 
 /**
@@ -423,12 +417,7 @@ garrow_resizable_buffer_reserve(GArrowResizableBuffer *buffer,
   auto arrow_resizable_buffer =
     std::static_pointer_cast<arrow::ResizableBuffer>(arrow_buffer);
   auto status = arrow_resizable_buffer->Reserve(new_capacity);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[resizable-buffer][capacity]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[resizable-buffer][capacity]");
 }
 
 
diff --git a/c_glib/arrow-glib/error.cpp b/c_glib/arrow-glib/error.cpp
index efbc6ae60452a..e5d2ad6f6eb40 100644
--- a/c_glib/arrow-glib/error.cpp
+++ b/c_glib/arrow-glib/error.cpp
@@ -63,19 +63,20 @@ garrow_error_code(const arrow::Status &status)
 
 G_END_DECLS
 
-void
-garrow_error_set(GError **error,
-                 const arrow::Status &status,
-                 const char *context)
+gboolean
+garrow_error_check(GError **error,
+                   const arrow::Status &status,
+                   const char *context)
 {
   if (status.ok()) {
-    return;
+    return TRUE;
+  } else {
+    g_set_error(error,
+                GARROW_ERROR,
+                garrow_error_code(status),
+                "%s: %s",
+                context,
+                status.ToString().c_str());
+    return FALSE;
   }
-
-  g_set_error(error,
-              GARROW_ERROR,
-              garrow_error_code(status),
-              "%s: %s",
-              context,
-              status.ToString().c_str());
 }
diff --git a/c_glib/arrow-glib/error.hpp b/c_glib/arrow-glib/error.hpp
index 357d293c4f127..dad27bd5c9b5a 100644
--- a/c_glib/arrow-glib/error.hpp
+++ b/c_glib/arrow-glib/error.hpp
@@ -23,6 +23,6 @@
 
 #include <arrow-glib/error.h>
 
-void garrow_error_set(GError **error,
-                      const arrow::Status &status,
-                      const char *context);
+gboolean garrow_error_check(GError **error,
+                            const arrow::Status &status,
+                            const char *context);
diff --git a/c_glib/arrow-glib/file-output-stream.cpp b/c_glib/arrow-glib/file-output-stream.cpp
index b6ca42a1d59da..e1e1e27a06193 100644
--- a/c_glib/arrow-glib/file-output-stream.cpp
+++ b/c_glib/arrow-glib/file-output-stream.cpp
@@ -204,7 +204,7 @@ garrow_file_output_stream_open(const gchar *path,
     std::string context("[io][file-output-stream][open]: <");
     context += path;
     context += ">";
-    garrow_error_set(error, status, context.c_str());
+    garrow_error_check(error, status, context.c_str());
     return NULL;
   }
 }
diff --git a/c_glib/arrow-glib/file-reader.cpp b/c_glib/arrow-glib/file-reader.cpp
index c2aeabe5eed21..b952b52ddbe6d 100644
--- a/c_glib/arrow-glib/file-reader.cpp
+++ b/c_glib/arrow-glib/file-reader.cpp
@@ -146,10 +146,9 @@ garrow_file_reader_open(GArrowRandomAccessFile *file,
   auto status =
     arrow::ipc::FileReader::Open(garrow_random_access_file_get_raw(file),
                                  &arrow_file_reader);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[ipc][file-reader][open]")) {
     return garrow_file_reader_new_raw(&arrow_file_reader);
   } else {
-    garrow_error_set(error, status, "[ipc][file-reader][open]");
     return NULL;
   }
 }
@@ -217,10 +216,11 @@ garrow_file_reader_get_record_batch(GArrowFileReader *file_reader,
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
   auto status = arrow_file_reader->GetRecordBatch(i, &arrow_record_batch);
 
-  if (status.ok()) {
+  if (garrow_error_check(error,
+                         status,
+                         "[ipc][file-reader][get-record-batch]")) {
     return garrow_record_batch_new_raw(&arrow_record_batch);
   } else {
-    garrow_error_set(error, status, "[ipc][file-reader][get-record-batch]");
     return NULL;
   }
 }
diff --git a/c_glib/arrow-glib/file-writer.cpp b/c_glib/arrow-glib/file-writer.cpp
index 68eca2edf77c1..e615cf554fd64 100644
--- a/c_glib/arrow-glib/file-writer.cpp
+++ b/c_glib/arrow-glib/file-writer.cpp
@@ -75,10 +75,9 @@ garrow_file_writer_open(GArrowOutputStream *sink,
     arrow::ipc::FileWriter::Open(garrow_output_stream_get_raw(sink).get(),
                                  garrow_schema_get_raw(schema),
                                  &arrow_file_writer);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[ipc][file-writer][open]")) {
     return garrow_file_writer_new_raw(&arrow_file_writer);
   } else {
-    garrow_error_set(error, status, "[ipc][file-writer][open]");
     return NULL;
   }
 }
@@ -104,12 +103,9 @@ garrow_file_writer_write_record_batch(GArrowFileWriter *file_writer,
     arrow_record_batch.get();
 
   auto status = arrow_file_writer->WriteRecordBatch(*arrow_record_batch_raw);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[ipc][file-writer][write-record-batch]");
-    return FALSE;
-  }
+  return garrow_error_check(error,
+                            status,
+                            "[ipc][file-writer][write-record-batch]");
 }
 
 /**
@@ -127,12 +123,7 @@ garrow_file_writer_close(GArrowFileWriter *file_writer,
     garrow_file_writer_get_raw(file_writer);
 
   auto status = arrow_file_writer->Close();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[ipc][file-writer][close]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[ipc][file-writer][close]");
 }
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/file.cpp b/c_glib/arrow-glib/file.cpp
index 0d0fe1d8b9c83..775339386c6b5 100644
--- a/c_glib/arrow-glib/file.cpp
+++ b/c_glib/arrow-glib/file.cpp
@@ -60,12 +60,7 @@ garrow_file_close(GArrowFile *file,
   auto arrow_file = garrow_file_get_raw(file);
 
   auto status = arrow_file->Close();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[io][file][close]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[io][file][close]");
 }
 
 /**
@@ -83,10 +78,9 @@ garrow_file_tell(GArrowFile *file,
 
   gint64 position;
   auto status = arrow_file->Tell(&position);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[io][file][tell]")) {
     return position;
   } else {
-    garrow_error_set(error, status, "[io][file][tell]");
     return -1;
   }
 }
diff --git a/c_glib/arrow-glib/memory-mapped-file.cpp b/c_glib/arrow-glib/memory-mapped-file.cpp
index a3e1d0c45f142..f9cbf079105c1 100644
--- a/c_glib/arrow-glib/memory-mapped-file.cpp
+++ b/c_glib/arrow-glib/memory-mapped-file.cpp
@@ -260,7 +260,7 @@ garrow_memory_mapped_file_open(const gchar *path,
     std::string context("[io][memory-mapped-file][open]: <");
     context += path;
     context += ">";
-    garrow_error_set(error, status, context.c_str());
+    garrow_error_check(error, status, context.c_str());
     return NULL;
   }
 }
diff --git a/c_glib/arrow-glib/random-access-file.cpp b/c_glib/arrow-glib/random-access-file.cpp
index 71f315ec7efaa..976a80dce0693 100644
--- a/c_glib/arrow-glib/random-access-file.cpp
+++ b/c_glib/arrow-glib/random-access-file.cpp
@@ -61,10 +61,9 @@ garrow_random_access_file_get_size(GArrowRandomAccessFile *file,
   int64_t size;
 
   auto status = arrow_random_access_file->GetSize(&size);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[io][random-access-file][get-size]")) {
     return size;
   } else {
-    garrow_error_set(error, status, "[io][random-access-file][get-size]");
     return 0;
   }
 }
@@ -110,12 +109,7 @@ garrow_random_access_file_read_at(GArrowRandomAccessFile *file,
                                                  n_bytes,
                                                  n_read_bytes,
                                                  buffer);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[io][random-access-file][read-at]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[io][random-access-file][read-at]");
 }
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/readable.cpp b/c_glib/arrow-glib/readable.cpp
index b8c0cd99df06a..d893853eea015 100644
--- a/c_glib/arrow-glib/readable.cpp
+++ b/c_glib/arrow-glib/readable.cpp
@@ -66,12 +66,7 @@ garrow_readable_read(GArrowReadable *readable,
   const auto arrow_readable = garrow_readable_get_raw(readable);
 
   auto status = arrow_readable->Read(n_bytes, n_read_bytes, buffer);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[io][readable][read]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[io][readable][read]");
 }
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/stream-reader.cpp b/c_glib/arrow-glib/stream-reader.cpp
index c4ccebe56f6ba..017d5e91a8a4d 100644
--- a/c_glib/arrow-glib/stream-reader.cpp
+++ b/c_glib/arrow-glib/stream-reader.cpp
@@ -147,10 +147,9 @@ garrow_stream_reader_open(GArrowInputStream *stream,
   auto status =
     arrow::ipc::StreamReader::Open(garrow_input_stream_get_raw(stream),
                                    &arrow_stream_reader);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[ipc][stream-reader][open]")) {
     return garrow_stream_reader_new_raw(&arrow_stream_reader);
   } else {
-    garrow_error_set(error, status, "[ipc][stream-reader][open]");
     return NULL;
   }
 }
@@ -187,14 +186,15 @@ garrow_stream_reader_get_next_record_batch(GArrowStreamReader *stream_reader,
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
   auto status = arrow_stream_reader->GetNextRecordBatch(&arrow_record_batch);
 
-  if (status.ok()) {
+  if (garrow_error_check(error,
+                       status,
+                       "[ipc][stream-reader][get-next-record-batch]")) {
     if (arrow_record_batch == nullptr) {
       return NULL;
     } else {
       return garrow_record_batch_new_raw(&arrow_record_batch);
     }
   } else {
-    garrow_error_set(error, status, "[ipc][stream-reader][get-next-record-batch]");
     return NULL;
   }
 }
diff --git a/c_glib/arrow-glib/stream-writer.cpp b/c_glib/arrow-glib/stream-writer.cpp
index 016ce93759c87..cc24f263bfca9 100644
--- a/c_glib/arrow-glib/stream-writer.cpp
+++ b/c_glib/arrow-glib/stream-writer.cpp
@@ -150,10 +150,9 @@ garrow_stream_writer_open(GArrowOutputStream *sink,
     arrow::ipc::StreamWriter::Open(garrow_output_stream_get_raw(sink).get(),
                                  garrow_schema_get_raw(schema),
                                  &arrow_stream_writer);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[ipc][stream-writer][open]")) {
     return garrow_stream_writer_new_raw(&arrow_stream_writer);
   } else {
-    garrow_error_set(error, status, "[ipc][stream-writer][open]");
     return NULL;
   }
 }
@@ -179,12 +178,9 @@ garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
     arrow_record_batch.get();
 
   auto status = arrow_stream_writer->WriteRecordBatch(*arrow_record_batch_raw);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[ipc][stream-writer][write-record-batch]");
-    return FALSE;
-  }
+  return garrow_error_check(error,
+                            status,
+                            "[ipc][stream-writer][write-record-batch]");
 }
 
 /**
@@ -202,12 +198,7 @@ garrow_stream_writer_close(GArrowStreamWriter *stream_writer,
     garrow_stream_writer_get_raw(stream_writer);
 
   auto status = arrow_stream_writer->Close();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[ipc][stream-writer][close]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[ipc][stream-writer][close]");
 }
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/table.cpp b/c_glib/arrow-glib/table.cpp
index 1d743b70731bb..2aba21b564243 100644
--- a/c_glib/arrow-glib/table.cpp
+++ b/c_glib/arrow-glib/table.cpp
@@ -226,10 +226,9 @@ garrow_table_add_column(GArrowTable *table,
   const auto arrow_column = garrow_column_get_raw(column);
   std::shared_ptr<arrow::Table> arrow_new_table;
   auto status = arrow_table->AddColumn(i, arrow_column, &arrow_new_table);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[table][add-column]")) {
     return garrow_table_new_raw(&arrow_new_table);
   } else {
-    garrow_error_set(error, status, "[table][add-column]");
     return NULL;
   }
 }
@@ -253,10 +252,9 @@ garrow_table_remove_column(GArrowTable *table,
   const auto arrow_table = garrow_table_get_raw(table);
   std::shared_ptr<arrow::Table> arrow_new_table;
   auto status = arrow_table->RemoveColumn(i, &arrow_new_table);
-  if (status.ok()) {
+  if (garrow_error_check(error, status, "[table][remove-column]")) {
     return garrow_table_new_raw(&arrow_new_table);
   } else {
-    garrow_error_set(error, status, "[table][remove-column]");
     return NULL;
   }
 }
diff --git a/c_glib/arrow-glib/writeable-file.cpp b/c_glib/arrow-glib/writeable-file.cpp
index d0937ea2612d2..b717c32932fc0 100644
--- a/c_glib/arrow-glib/writeable-file.cpp
+++ b/c_glib/arrow-glib/writeable-file.cpp
@@ -66,12 +66,7 @@ garrow_writeable_file_write_at(GArrowWriteableFile *writeable_file,
     garrow_writeable_file_get_raw(writeable_file);
 
   auto status = arrow_writeable_file->WriteAt(position, data, n_bytes);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[io][writeable-file][write-at]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[io][writeable-file][write-at]");
 }
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/writeable.cpp b/c_glib/arrow-glib/writeable.cpp
index 6f4c63008ae49..eb6adfee8c985 100644
--- a/c_glib/arrow-glib/writeable.cpp
+++ b/c_glib/arrow-glib/writeable.cpp
@@ -64,12 +64,7 @@ garrow_writeable_write(GArrowWriteable *writeable,
   const auto arrow_writeable = garrow_writeable_get_raw(writeable);
 
   auto status = arrow_writeable->Write(data, n_bytes);
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[io][writeable][write]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[io][writeable][write]");
 }
 
 /**
@@ -88,12 +83,7 @@ garrow_writeable_flush(GArrowWriteable *writeable,
   const auto arrow_writeable = garrow_writeable_get_raw(writeable);
 
   auto status = arrow_writeable->Flush();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_set(error, status, "[io][writeable][flush]");
-    return FALSE;
-  }
+  return garrow_error_check(error, status, "[io][writeable][flush]");
 }
 
 G_END_DECLS

From 02c32ff938de4507483fc69f39847291319f427f Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 26 Apr 2017 10:33:44 -0400
Subject: [PATCH 0571/1644] ARROW-903: [GLib] Remove a needless "."

Author: Kouhei Sutou <kou@clear-code.com>

Closes #603 from kou/glib-tensor-doc and squashes the following commits:

527c4db [Kouhei Sutou] [GLib] Remove a needless "."
---
 c_glib/arrow-glib/tensor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp
index 82f66352d66a0..27af7532f3451 100644
--- a/c_glib/arrow-glib/tensor.cpp
+++ b/c_glib/arrow-glib/tensor.cpp
@@ -30,7 +30,7 @@ G_BEGIN_DECLS
 
 /**
  * SECTION: tensor
- * @short_description: Tensor class.
+ * @short_description: Tensor class
  * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowTensor is a tensor class.

From 8bf61d1682b883a7a538678f7f3c68dc06bb758d Mon Sep 17 00:00:00 2001
From: Holden Karau <holden@us.ibm.com>
Date: Wed, 26 Apr 2017 15:14:49 -0400
Subject: [PATCH 0572/1644] ARROW-697: JAVA Throw exception for record batches
 > 2GB

Add a test to verify that we throw a clear error message for record batches over 2GB. This entry point is easist to test without adding some magic bytes to the tests suite since its explicit on the input, and the other public entry points for deserialization have the same checks (just extracted from the metadata).

Author: Holden Karau <holden@us.ibm.com>

Closes #597 from holdenk/ARROW-697-java-raise-exception-for-large-batch-size and squashes the following commits:

d2d6b3d [Holden Karau] Merge branch 'master' into ARROW-697-java-raise-exception-for-large-batch-size
d56daab [Holden Karau] Throw IOException if record batch length, node length, or null count are larger than Int.MAX_VALUE
0a96b74 [Holden Karau] Add a test to verify that we throw a clear error message for record batches over 2GB in size
---
 .../arrow/vector/stream/MessageSerializer.java | 10 +++++++++-
 .../vector/stream/MessageSerializerTest.java   | 18 ++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index ec7e0f2ffb115..228ab613466d2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -201,12 +201,17 @@ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock
 
   // Deserializes a record batch given the Flatbuffer metadata and in-memory body
   private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
-      ArrowBuf body) {
+      ArrowBuf body) throws IOException {
     // Now read the body
     int nodesLength = recordBatchFB.nodesLength();
     List<ArrowFieldNode> nodes = new ArrayList<>();
     for (int i = 0; i < nodesLength; ++i) {
       FieldNode node = recordBatchFB.nodes(i);
+      if ((int)node.length() != node.length() ||
+          (int)node.nullCount() != node.nullCount()) {
+        throw new IOException("Cannot currently deserialize record batches with " +
+                              "node length larger than Int.MAX_VALUE");
+      }
       nodes.add(new ArrowFieldNode((int)node.length(), (int)node.nullCount()));
     }
     List<ArrowBuf> buffers = new ArrayList<>();
@@ -215,6 +220,9 @@ private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB
       ArrowBuf vectorBuffer = body.slice((int)bufferFB.offset(), (int)bufferFB.length());
       buffers.add(vectorBuffer);
     }
+    if ((int)recordBatchFB.length() != recordBatchFB.length()) {
+      throw new IOException("Cannot currently deserialize record batches over 2GB");
+    }
     ArrowRecordBatch arrowRecordBatch =
         new ArrowRecordBatch((int)recordBatchFB.length(), nodes, buffers);
     body.release();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
index d3d49d5fb8096..27879efeaf117 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
@@ -31,6 +31,7 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.file.ArrowBlock;
 import org.apache.arrow.vector.file.ReadChannel;
 import org.apache.arrow.vector.file.WriteChannel;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
@@ -41,6 +42,8 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Test;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
 
 import io.netty.buffer.ArrowBuf;
 
@@ -87,6 +90,21 @@ public void testSchemaDictionaryMessageSerialization() throws IOException {
     assertEquals(schema, deserialized);
   }
 
+  @Rule
+  public ExpectedException expectedEx = ExpectedException.none();
+
+  @Test
+  public void testdeSerializeRecordBatchLongMetaData() throws IOException {
+    expectedEx.expect(IOException.class);
+    expectedEx.expectMessage("Cannot currently deserialize record batches over 2GB");
+    int offset = 0;
+    int metadataLength = 1;
+    long bodyLength = Integer.MAX_VALUE + 10L;
+    ArrowBlock block = new ArrowBlock(offset, metadataLength, bodyLength);
+    long totalLen = block.getMetadataLength() + block.getBodyLength();
+    MessageSerializer.deserializeRecordBatch(null, block, null);
+  }
+
   @Test
   public void testSerializeRecordBatch() throws IOException {
     byte[] validity = new byte[] { (byte)255, 0};

From 3fdeac74c80593ebde7a8eeb148cea9f6e0d1b38 Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Wed, 26 Apr 2017 15:16:53 -0400
Subject: [PATCH 0573/1644] ARROW-886 [Java] Fixing reallocation of
 VariableLengthVector offsets

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #591 from elahrvivaz/ARROW-886 and squashes the following commits:

5f6b4be [Emilio Lahr-Vivaz] ARROW-886 Fixing reallocation of VariableLengthVector offsets
---
 .../templates/VariableLengthVectors.java      |  1 +
 .../arrow/vector/TestVectorReAlloc.java       | 27 ++++++++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index 4a460c5475323..11f0cc894d004 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -355,6 +355,7 @@ public void reset() {
   }
 
   public void reAlloc() {
+    offsetVector.reAlloc();
     final long newAllocationSize = allocationSizeInBytes*2L;
     if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
       throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index a7c35b6363cf1..40c7bc5ac9add 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -72,6 +72,31 @@ public void testFixedType() {
     }
   }
 
+  @Test
+  public void testVariableLengthType() {
+    try (final VarCharVector vector = new VarCharVector("", allocator)) {
+      final VarCharVector.Mutator m = vector.getMutator();
+      // note: capacity ends up being - 1 due to offsets vector
+      vector.setInitialCapacity(511);
+      vector.allocateNew();
+
+      assertEquals(511, vector.getValueCapacity());
+
+      try {
+        m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(1023, vector.getValueCapacity());
+
+      m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+      assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+    }
+  }
+
   @Test
   public void testNullableType() {
     try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
@@ -89,7 +114,7 @@ public void testNullableType() {
       }
 
       vector.reAlloc();
-      assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+      assertEquals(1024, vector.getValueCapacity());
 
       m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
       assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));

From e876abbdf4f7bac53ae5d56f4680259f021ea8d9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 27 Apr 2017 09:28:54 -0400
Subject: [PATCH 0574/1644] ARROW-898: [C++/Python] Use shared_ptr to avoid
 copying KeyValueMetadata, add to Field type also

This also adds support for adding and removing schema-level metadata to the Python Schema wrapper object. Need to do the same for Field but putting this up for @cpcloud to review since he's working on using this in parquet-cpp

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #605 from wesm/ARROW-898 and squashes the following commits:

03873f7 [Wes McKinney] RemoveMetadata not return Status
c621c2c [Wes McKinney] Add metadata methods to Field, some code cleaning
581b9fa [Wes McKinney] Add unit tests for passing metadata to Field constructor
51fae29 [Wes McKinney] Add metadata to Field
2ce4003 [Wes McKinney] Test sharing metadata
48aa3ca [Wes McKinney] Use shared_ptr<const T> for KeyValueMetadata so metadata can be shared / not copied
---
 cpp/src/arrow/builder.cc                      |  11 +-
 cpp/src/arrow/io/io-memory-benchmark.cc       |   6 +-
 cpp/src/arrow/io/memory.cc                    |   4 +-
 cpp/src/arrow/ipc/metadata.cc                 |  38 +--
 cpp/src/arrow/type-test.cc                    |  61 ++++-
 cpp/src/arrow/type.cc                         |  60 +++--
 cpp/src/arrow/type.h                          |  32 ++-
 cpp/src/arrow/util/key-value-metadata-test.cc |   9 +
 cpp/src/arrow/util/key_value_metadata.cc      |   4 +
 cpp/src/arrow/util/key_value_metadata.h       |   6 +
 cpp/src/arrow/util/memory.h                   |   8 +-
 python/pyarrow/_array.pxd                     |   2 -
 python/pyarrow/_array.pyx                     | 220 ++++++++++++++----
 python/pyarrow/_table.pyx                     |   7 +-
 python/pyarrow/includes/libarrow.pxd          |  31 ++-
 python/pyarrow/tests/pandas_examples.py       |   4 +-
 python/pyarrow/tests/test_convert_pandas.py   |  64 ++---
 python/pyarrow/tests/test_schema.py           |  48 +++-
 18 files changed, 466 insertions(+), 149 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 4ecb8d3500981..ab43c2a51baf4 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -406,17 +406,16 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
     const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool,
-          type ? type : std::static_pointer_cast<DataType>(
-                            std::make_shared<ListType>(value_builder->type()))),
+    : ArrayBuilder(
+          pool, type ? type : std::static_pointer_cast<DataType>(
+                                  std::make_shared<ListType>(value_builder->type()))),
       offset_builder_(pool),
       value_builder_(value_builder) {}
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<Array> values,
     const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool,
-          type ? type : std::static_pointer_cast<DataType>(
-                            std::make_shared<ListType>(values->type()))),
+    : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
+                                           std::make_shared<ListType>(values->type()))),
       offset_builder_(pool),
       values_(values) {}
 
diff --git a/cpp/src/arrow/io/io-memory-benchmark.cc b/cpp/src/arrow/io/io-memory-benchmark.cc
index 59b511a6cf8fe..6aa9577f0a1d9 100644
--- a/cpp/src/arrow/io/io-memory-benchmark.cc
+++ b/cpp/src/arrow/io/io-memory-benchmark.cc
@@ -26,7 +26,7 @@
 namespace arrow {
 
 static void BM_SerialMemcopy(benchmark::State& state) {  // NOLINT non-const reference
-  constexpr int64_t kTotalSize = 100 * 1024 * 1024; // 100MB
+  constexpr int64_t kTotalSize = 100 * 1024 * 1024;      // 100MB
 
   auto buffer1 = std::make_shared<PoolBuffer>(default_memory_pool());
   buffer1->Resize(kTotalSize);
@@ -43,7 +43,7 @@ static void BM_SerialMemcopy(benchmark::State& state) {  // NOLINT non-const ref
 }
 
 static void BM_ParallelMemcopy(benchmark::State& state) {  // NOLINT non-const reference
-  constexpr int64_t kTotalSize = 100 * 1024 * 1024; // 100MB
+  constexpr int64_t kTotalSize = 100 * 1024 * 1024;        // 100MB
 
   auto buffer1 = std::make_shared<PoolBuffer>(default_memory_pool());
   buffer1->Resize(kTotalSize);
@@ -72,4 +72,4 @@ BENCHMARK(BM_ParallelMemcopy)
     ->MinTime(1.0)
     ->UseRealTime();
 
-} // namespace arrow
+}  // namespace arrow
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 95c6206f0fab0..faf02d2fa22f7 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -140,8 +140,8 @@ Status FixedSizeBufferWriter::Tell(int64_t* position) {
 
 Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
   if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) {
-    parallel_memcopy(mutable_data_ + position_, data, nbytes,
-                     memcopy_blocksize_, memcopy_num_threads_);
+    parallel_memcopy(mutable_data_ + position_, data, nbytes, memcopy_blocksize_,
+        memcopy_num_threads_);
   } else {
     memcpy(mutable_data_ + position_, data, nbytes);
   }
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index c0b518a0d8e50..706ab2e8aab0e 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -593,20 +593,27 @@ static Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
     field_offsets.push_back(offset);
   }
 
+  auto fb_offsets = fbb.CreateVector(field_offsets);
+
   /// Custom metadata
-  const auto& custom_metadata_ = schema.custom_metadata();
-  std::vector<KeyValueOffset> key_value_offsets;
-  size_t metadata_size = custom_metadata_.size();
-  key_value_offsets.reserve(metadata_size);
-  for (size_t i = 0; i < metadata_size; ++i) {
-    const auto& key = custom_metadata_.key(i);
-    const auto& value = custom_metadata_.value(i);
-    key_value_offsets.push_back(
-        flatbuf::CreateKeyValue(fbb, fbb.CreateString(key), fbb.CreateString(value)));
+  const KeyValueMetadata* metadata = schema.metadata().get();
+
+  if (metadata != nullptr) {
+    std::vector<KeyValueOffset> key_value_offsets;
+    size_t metadata_size = metadata->size();
+    key_value_offsets.reserve(metadata_size);
+    for (size_t i = 0; i < metadata_size; ++i) {
+      const auto& key = metadata->key(i);
+      const auto& value = metadata->value(i);
+      key_value_offsets.push_back(
+          flatbuf::CreateKeyValue(fbb, fbb.CreateString(key), fbb.CreateString(value)));
+    }
+    *out = flatbuf::CreateSchema(
+        fbb, endianness(), fb_offsets, fbb.CreateVector(key_value_offsets));
+  } else {
+    *out = flatbuf::CreateSchema(fbb, endianness(), fb_offsets);
   }
 
-  *out = flatbuf::CreateSchema(fbb, endianness(), fbb.CreateVector(field_offsets),
-      fbb.CreateVector(key_value_offsets));
   return Status::OK();
 }
 
@@ -955,17 +962,16 @@ Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_mem
     RETURN_NOT_OK(FieldFromFlatbuffer(field, dictionary_memo, &fields[i]));
   }
 
-  KeyValueMetadata custom_metadata;
+  auto metadata = std::make_shared<KeyValueMetadata>();
   auto fb_metadata = schema->custom_metadata();
   if (fb_metadata != nullptr) {
-    custom_metadata.reserve(fb_metadata->size());
-
+    metadata->reserve(fb_metadata->size());
     for (const auto& pair : *fb_metadata) {
-      custom_metadata.Append(pair->key()->str(), pair->value()->str());
+      metadata->Append(pair->key()->str(), pair->value()->str());
     }
   }
 
-  *out = std::make_shared<Schema>(fields, custom_metadata);
+  *out = std::make_shared<Schema>(fields, metadata);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 8e2dfd50e431d..e73adecdcb5aa 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -23,6 +23,7 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/test-util.h"
 #include "arrow/type.h"
 
 using std::shared_ptr;
@@ -50,6 +51,40 @@ TEST(TestField, Equals) {
   ASSERT_FALSE(f0.Equals(f0_nn));
 }
 
+TEST(TestField, TestMetadataConstruction) {
+  auto metadata = std::shared_ptr<KeyValueMetadata>(
+      new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
+  auto metadata2 = metadata->Copy();
+  auto f0 = field("f0", int32(), true, metadata);
+  auto f1 = field("f0", int32(), true, metadata2);
+  ASSERT_TRUE(metadata->Equals(*f0->metadata()));
+  ASSERT_TRUE(f0->Equals(*f1));
+}
+
+TEST(TestField, TestAddMetadata) {
+  auto metadata = std::shared_ptr<KeyValueMetadata>(
+      new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
+  auto f0 = field("f0", int32());
+  auto f1 = field("f0", int32(), true, metadata);
+  std::shared_ptr<Field> f2;
+  ASSERT_OK(f0->AddMetadata(metadata, &f2));
+
+  ASSERT_FALSE(f2->Equals(*f0));
+  ASSERT_TRUE(f2->Equals(*f1));
+
+  // Not copied
+  ASSERT_TRUE(metadata.get() == f1->metadata().get());
+}
+
+TEST(TestField, TestRemoveMetadata) {
+  auto metadata = std::shared_ptr<KeyValueMetadata>(
+      new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
+  auto f0 = field("f0", int32());
+  auto f1 = field("f0", int32(), true, metadata);
+  std::shared_ptr<Field> f2 = f1->RemoveMetadata();
+  ASSERT_TRUE(f2->metadata() == nullptr);
+}
+
 class TestSchema : public ::testing::Test {
  public:
   void SetUp() {}
@@ -117,38 +152,42 @@ TEST_F(TestSchema, GetFieldByName) {
   ASSERT_TRUE(result == nullptr);
 }
 
-TEST_F(TestSchema, TestCustomMetadataConstruction) {
+TEST_F(TestSchema, TestMetadataConstruction) {
   auto f0 = field("f0", int32());
   auto f1 = field("f1", uint8(), false);
   auto f2 = field("f2", utf8());
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
-  KeyValueMetadata metadata({"foo", "bar"}, {"bizz", "buzz"});
+  auto metadata = std::shared_ptr<KeyValueMetadata>(
+      new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
   auto schema = std::make_shared<Schema>(fields, metadata);
-  ASSERT_TRUE(metadata.Equals(schema->custom_metadata()));
+  ASSERT_TRUE(metadata->Equals(*schema->metadata()));
 }
 
-TEST_F(TestSchema, TestAddCustomMetadata) {
+TEST_F(TestSchema, TestAddMetadata) {
   auto f0 = field("f0", int32());
   auto f1 = field("f1", uint8(), false);
   auto f2 = field("f2", utf8());
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
-  KeyValueMetadata metadata({"foo", "bar"}, {"bizz", "buzz"});
+  auto metadata = std::shared_ptr<KeyValueMetadata>(
+      new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
   auto schema = std::make_shared<Schema>(fields);
   std::shared_ptr<Schema> new_schema;
-  schema->AddCustomMetadata(metadata, &new_schema);
-  ASSERT_TRUE(metadata.Equals(new_schema->custom_metadata()));
+  schema->AddMetadata(metadata, &new_schema);
+  ASSERT_TRUE(metadata->Equals(*new_schema->metadata()));
+
+  // Not copied
+  ASSERT_TRUE(metadata.get() == new_schema->metadata().get());
 }
 
-TEST_F(TestSchema, TestRemoveCustomMetadata) {
+TEST_F(TestSchema, TestRemoveMetadata) {
   auto f0 = field("f0", int32());
   auto f1 = field("f1", uint8(), false);
   auto f2 = field("f2", utf8());
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
   KeyValueMetadata metadata({"foo", "bar"}, {"bizz", "buzz"});
   auto schema = std::make_shared<Schema>(fields);
-  std::shared_ptr<Schema> new_schema;
-  schema->RemoveCustomMetadata(&new_schema);
-  ASSERT_EQ(0, new_schema->custom_metadata().size());
+  std::shared_ptr<Schema> new_schema = schema->RemoveMetadata();
+  ASSERT_TRUE(new_schema->metadata() == nullptr);
 }
 
 #define PRIMITIVE_TEST(KLASS, ENUM, NAME)        \
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index f59f8fb26c9ba..b1e322ce1b321 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -31,10 +31,31 @@
 
 namespace arrow {
 
+Status Field::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
+    std::shared_ptr<Field>* out) const {
+  *out = std::make_shared<Field>(name_, type_, nullable_, metadata);
+  return Status::OK();
+}
+
+std::shared_ptr<Field> Field::RemoveMetadata() const {
+  return std::make_shared<Field>(name_, type_, nullable_);
+}
+
 bool Field::Equals(const Field& other) const {
-  return (this == &other) ||
-         (this->name_ == other.name_ && this->nullable_ == other.nullable_ &&
-             this->type_->Equals(*other.type_.get()));
+  if (this == &other) {
+    return true;
+  }
+  if (this->name_ == other.name_ && this->nullable_ == other.nullable_ &&
+      this->type_->Equals(*other.type_.get())) {
+    if (metadata_ == nullptr && other.metadata_ == nullptr) {
+      return true;
+    } else if ((metadata_ == nullptr) ^ (other.metadata_ == nullptr)) {
+      return false;
+    } else {
+      return metadata_->Equals(*other.metadata_);
+    }
+  }
+  return false;
 }
 
 bool Field::Equals(const std::shared_ptr<Field>& other) const {
@@ -233,8 +254,8 @@ std::string NullType::ToString() const {
 // Schema implementation
 
 Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields,
-    const KeyValueMetadata& custom_metadata)
-    : fields_(fields), custom_metadata_(custom_metadata) {}
+    const std::shared_ptr<const KeyValueMetadata>& metadata)
+    : fields_(fields), metadata_(metadata) {}
 
 bool Schema::Equals(const Schema& other) const {
   if (this == &other) { return true; }
@@ -266,26 +287,25 @@ Status Schema::AddField(
   DCHECK_GE(i, 0);
   DCHECK_LE(i, this->num_fields());
 
-  *out = std::make_shared<Schema>(AddVectorElement(fields_, i, field), custom_metadata_);
+  *out = std::make_shared<Schema>(AddVectorElement(fields_, i, field), metadata_);
   return Status::OK();
 }
 
-Status Schema::AddCustomMetadata(
-    const KeyValueMetadata& custom_metadata, std::shared_ptr<Schema>* out) const {
-  *out = std::make_shared<Schema>(fields_, custom_metadata);
+Status Schema::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
+    std::shared_ptr<Schema>* out) const {
+  *out = std::make_shared<Schema>(fields_, metadata);
   return Status::OK();
 }
 
-Status Schema::RemoveCustomMetadata(std::shared_ptr<Schema>* out) {
-  *out = std::make_shared<Schema>(fields_, KeyValueMetadata());
-  return Status::OK();
+std::shared_ptr<Schema> Schema::RemoveMetadata() const {
+  return std::make_shared<Schema>(fields_);
 }
 
 Status Schema::RemoveField(int i, std::shared_ptr<Schema>* out) const {
   DCHECK_GE(i, 0);
   DCHECK_LT(i, this->num_fields());
 
-  *out = std::make_shared<Schema>(DeleteVectorElement(fields_, i), custom_metadata_);
+  *out = std::make_shared<Schema>(DeleteVectorElement(fields_, i), metadata_);
   return Status::OK();
 }
 
@@ -298,6 +318,15 @@ std::string Schema::ToString() const {
     buffer << field->ToString();
     ++i;
   }
+
+  if (metadata_) {
+    buffer << "\n-- metadata --";
+    for (int64_t i = 0; i < metadata_->size(); ++i) {
+      buffer << "\n" << metadata_->key(i) << ": "
+             << metadata_->value(i);
+    }
+  }
+
   return buffer.str();
 }
 
@@ -391,8 +420,9 @@ std::shared_ptr<DataType> dictionary(const std::shared_ptr<DataType>& index_type
 }
 
 std::shared_ptr<Field> field(
-    const std::string& name, const TypePtr& type, bool nullable) {
-  return std::make_shared<Field>(name, type, nullable);
+    const std::string& name, const std::shared_ptr<DataType>& type, bool nullable,
+    const std::shared_ptr<const KeyValueMetadata>& metadata) {
+  return std::make_shared<Field>(name, type, nullable, metadata);
 }
 
 std::shared_ptr<DataType> decimal(int precision, int scale) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index dc9456137030f..bb258578da327 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -203,8 +203,16 @@ class NoExtraMeta {};
 class ARROW_EXPORT Field {
  public:
   Field(const std::string& name, const std::shared_ptr<DataType>& type,
-      bool nullable = true)
-      : name_(name), type_(type), nullable_(nullable) {}
+      bool nullable = true,
+      const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr)
+    : name_(name), type_(type), nullable_(nullable), metadata_(metadata) {}
+
+  std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
+
+  Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
+      std::shared_ptr<Field>* out) const;
+
+  std::shared_ptr<Field> RemoveMetadata() const;
 
   bool Equals(const Field& other) const;
   bool Equals(const std::shared_ptr<Field>& other) const;
@@ -224,6 +232,9 @@ class ARROW_EXPORT Field {
 
   // Fields can be nullable
   bool nullable_;
+
+  // The field's metadata, if any
+  std::shared_ptr<const KeyValueMetadata> metadata_;
 };
 
 typedef std::shared_ptr<Field> FieldPtr;
@@ -679,7 +690,7 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 class ARROW_EXPORT Schema {
  public:
   explicit Schema(const std::vector<std::shared_ptr<Field>>& fields,
-      const KeyValueMetadata& custom_metadata = KeyValueMetadata());
+      const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
 
   // Returns true if all of the schema fields are equal
   bool Equals(const Schema& other) const;
@@ -691,7 +702,7 @@ class ARROW_EXPORT Schema {
   std::shared_ptr<Field> GetFieldByName(const std::string& name);
 
   const std::vector<std::shared_ptr<Field>>& fields() const { return fields_; }
-  const KeyValueMetadata& custom_metadata() const { return custom_metadata_; }
+  std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
 
   // Render a string representation of the schema suitable for debugging
   std::string ToString() const;
@@ -700,16 +711,18 @@ class ARROW_EXPORT Schema {
       int i, const std::shared_ptr<Field>& field, std::shared_ptr<Schema>* out) const;
   Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
 
-  Status AddCustomMetadata(
-      const KeyValueMetadata& metadata, std::shared_ptr<Schema>* out) const;
-  Status RemoveCustomMetadata(std::shared_ptr<Schema>* out);
+  Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
+      std::shared_ptr<Schema>* out) const;
+
+  std::shared_ptr<Schema> RemoveMetadata() const;
 
   int num_fields() const { return static_cast<int>(fields_.size()); }
 
  private:
   std::vector<std::shared_ptr<Field>> fields_;
   std::unordered_map<std::string, int> name_to_index_;
-  KeyValueMetadata custom_metadata_;
+
+  std::shared_ptr<const KeyValueMetadata> metadata_;
 };
 
 // ----------------------------------------------------------------------
@@ -741,7 +754,8 @@ std::shared_ptr<DataType> ARROW_EXPORT dictionary(
     const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& values);
 
 std::shared_ptr<Field> ARROW_EXPORT field(
-    const std::string& name, const std::shared_ptr<DataType>& type, bool nullable = true);
+    const std::string& name, const std::shared_ptr<DataType>& type, bool nullable = true,
+    const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
 
 // ----------------------------------------------------------------------
 //
diff --git a/cpp/src/arrow/util/key-value-metadata-test.cc b/cpp/src/arrow/util/key-value-metadata-test.cc
index aadc989cb403f..59cfdf597308c 100644
--- a/cpp/src/arrow/util/key-value-metadata-test.cc
+++ b/cpp/src/arrow/util/key-value-metadata-test.cc
@@ -72,6 +72,15 @@ TEST(KeyValueMetadataTest, StringAppend) {
   ASSERT_EQ("red", metadata.value(3));
 }
 
+TEST(KeyValueMetadataTest, Copy) {
+  std::vector<std::string> keys = {"foo", "bar"};
+  std::vector<std::string> values = {"bizz", "buzz"};
+
+  KeyValueMetadata metadata(keys, values);
+  auto metadata2 = metadata.Copy();
+  ASSERT_TRUE(metadata.Equals(*metadata2));
+}
+
 TEST(KeyValueMetadataTest, Equals) {
   std::vector<std::string> keys = {"foo", "bar"};
   std::vector<std::string> values = {"bizz", "buzz"};
diff --git a/cpp/src/arrow/util/key_value_metadata.cc b/cpp/src/arrow/util/key_value_metadata.cc
index c91478bd1acd6..8bddd5d0164c2 100644
--- a/cpp/src/arrow/util/key_value_metadata.cc
+++ b/cpp/src/arrow/util/key_value_metadata.cc
@@ -91,6 +91,10 @@ std::string KeyValueMetadata::value(int64_t i) const {
   return values_[static_cast<size_t>(i)];
 }
 
+std::shared_ptr<KeyValueMetadata> KeyValueMetadata::Copy() const {
+  return std::make_shared<KeyValueMetadata>(keys_, values_);
+}
+
 bool KeyValueMetadata::Equals(const KeyValueMetadata& other) const {
   return size() == other.size() &&
          std::equal(keys_.cbegin(), keys_.cend(), other.keys_.cbegin()) &&
diff --git a/cpp/src/arrow/util/key_value_metadata.h b/cpp/src/arrow/util/key_value_metadata.h
index 713b2c0b0bcfb..bae4ad806dd62 100644
--- a/cpp/src/arrow/util/key_value_metadata.h
+++ b/cpp/src/arrow/util/key_value_metadata.h
@@ -19,10 +19,12 @@
 #define ARROW_UTIL_KEY_VALUE_METADATA_H
 
 #include <cstdint>
+#include <memory>
 #include <string>
 #include <unordered_map>
 #include <vector>
 
+#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -44,11 +46,15 @@ class ARROW_EXPORT KeyValueMetadata {
   std::string key(int64_t i) const;
   std::string value(int64_t i) const;
 
+  std::shared_ptr<KeyValueMetadata> Copy() const;
+
   bool Equals(const KeyValueMetadata& other) const;
 
  private:
   std::vector<std::string> keys_;
   std::vector<std::string> values_;
+
+  DISALLOW_COPY_AND_ASSIGN(KeyValueMetadata);
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/memory.h b/cpp/src/arrow/util/memory.h
index 7feeb291ef4a0..c5c17ef907c22 100644
--- a/cpp/src/arrow/util/memory.h
+++ b/cpp/src/arrow/util/memory.h
@@ -31,7 +31,7 @@ uint8_t* pointer_logical_and(const uint8_t* address, uintptr_t bits) {
 // A helper function for doing memcpy with multiple threads. This is required
 // to saturate the memory bandwidth of modern cpus.
 void parallel_memcopy(uint8_t* dst, const uint8_t* src, int64_t nbytes,
-                      uintptr_t block_size, int num_threads) {
+    uintptr_t block_size, int num_threads) {
   std::vector<std::thread> threadpool(num_threads);
   uint8_t* left = pointer_logical_and(src + block_size - 1, ~(block_size - 1));
   uint8_t* right = pointer_logical_and(src + nbytes, ~(block_size - 1));
@@ -52,8 +52,8 @@ void parallel_memcopy(uint8_t* dst, const uint8_t* src, int64_t nbytes,
 
   // Start all threads first and handle leftovers while threads run.
   for (int i = 0; i < num_threads; i++) {
-    threadpool[i] = std::thread(memcpy, dst + prefix + i * chunk_size,
-        left + i * chunk_size, chunk_size);
+    threadpool[i] = std::thread(
+        memcpy, dst + prefix + i * chunk_size, left + i * chunk_size, chunk_size);
   }
 
   memcpy(dst, src, prefix);
@@ -64,6 +64,6 @@ void parallel_memcopy(uint8_t* dst, const uint8_t* src, int64_t nbytes,
   }
 }
 
-} // namespace arrow
+}  // namespace arrow
 
 #endif  // ARROW_UTIL_MEMORY_H
diff --git a/python/pyarrow/_array.pxd b/python/pyarrow/_array.pxd
index 4d5db8618a377..464de316f0437 100644
--- a/python/pyarrow/_array.pxd
+++ b/python/pyarrow/_array.pxd
@@ -81,8 +81,6 @@ cdef class Schema:
     cdef init(self, const vector[shared_ptr[CField]]& fields)
     cdef init_schema(self, const shared_ptr[CSchema]& schema)
 
-    cpdef dict custom_metadata(self)
-
 
 cdef class Scalar:
     cdef readonly:
diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/_array.pyx
index 2fb20b7553e93..658f4b314b3a2 100644
--- a/python/pyarrow/_array.pyx
+++ b/python/pyarrow/_array.pyx
@@ -172,7 +172,14 @@ cdef class DecimalType(FixedSizeBinaryType):
 
 
 cdef class Field:
+    """
+    Represents a named field, with a data type, nullability, and optional
+    metadata
 
+    Notes
+    -----
+    Do not use this class's constructor directly; use pyarrow.field
+    """
     def __cinit__(self):
         pass
 
@@ -181,32 +188,77 @@ cdef class Field:
         self.field = field.get()
         self.type = box_data_type(field.get().type())
 
-    @classmethod
-    def from_py(cls, object name, DataType type, bint nullable=True):
-        cdef Field result = Field()
-        result.type = type
-        result.sp_field.reset(new CField(tobytes(name), type.sp_type,
-                                         nullable))
-        result.field = result.sp_field.get()
+    def equals(self, Field other):
+        """
+        Test if this field is equal to the other
+        """
+        return self.field.Equals(deref(other.field))
 
-        return result
+    def __str__(self):
+        self._check_null()
+        return 'pyarrow.Field<{0}>'.format(frombytes(self.field.ToString()))
 
     def __repr__(self):
-        return 'Field({0!r}, type={1})'.format(self.name, str(self.type))
+        return self.__str__()
 
     property nullable:
 
         def __get__(self):
+            self._check_null()
             return self.field.nullable()
 
     property name:
 
         def __get__(self):
-            if box_field(self.sp_field) is None:
-                raise ReferenceError(
-                    'Field not initialized (references NULL pointer)')
+            self._check_null()
             return frombytes(self.field.name())
 
+    property metadata:
+
+        def __get__(self):
+            self._check_null()
+            return box_metadata(self.field.metadata().get())
+
+    def _check_null(self):
+        if self.field == NULL:
+            raise ReferenceError(
+                'Field not initialized (references NULL pointer)')
+
+    def add_metadata(self, dict metadata):
+        """
+        Add metadata as dict of string keys and values to Field
+
+        Parameters
+        ----------
+        metadata : dict
+            Keys and values must be string-like / coercible to bytes
+
+        Returns
+        -------
+        field : pyarrow.Field
+        """
+        cdef shared_ptr[CKeyValueMetadata] c_meta
+        convert_metadata(metadata, &c_meta)
+
+        cdef shared_ptr[CField] new_field
+        with nogil:
+            check_status(self.field.AddMetadata(c_meta, &new_field))
+
+        return box_field(new_field)
+
+    def remove_metadata(self):
+        """
+        Create new field without metadata, if any
+
+        Returns
+        -------
+        field : pyarrow.Field
+        """
+        cdef shared_ptr[CField] new_field
+        with nogil:
+            new_field = self.field.RemoveMetadata()
+        return box_field(new_field)
+
 
 cdef class Schema:
 
@@ -226,6 +278,14 @@ cdef class Schema:
 
         return result
 
+    cdef init(self, const vector[shared_ptr[CField]]& fields):
+        self.schema = new CSchema(fields)
+        self.sp_schema.reset(self.schema)
+
+    cdef init_schema(self, const shared_ptr[CSchema]& schema):
+        self.schema = schema.get()
+        self.sp_schema = schema
+
     property names:
 
         def __get__(self):
@@ -236,20 +296,10 @@ cdef class Schema:
                 result.append(name)
             return result
 
-    cdef init(self, const vector[shared_ptr[CField]]& fields):
-        self.schema = new CSchema(fields)
-        self.sp_schema.reset(self.schema)
-
-    cdef init_schema(self, const shared_ptr[CSchema]& schema):
-        self.schema = schema.get()
-        self.sp_schema = schema
+    property metadata:
 
-    cpdef dict custom_metadata(self):
-        cdef:
-            CKeyValueMetadata metadata = self.schema.custom_metadata()
-            unordered_map[c_string, c_string] result
-        metadata.ToUnorderedMap(&result)
-        return result
+        def __get__(self):
+            return box_metadata(self.schema.metadata().get())
 
     def equals(self, other):
         """
@@ -274,23 +324,40 @@ cdef class Schema:
         """
         return box_field(self.schema.GetFieldByName(tobytes(name)))
 
-    @classmethod
-    def from_fields(cls, fields):
-        cdef:
-            Schema result
-            Field field
-            vector[shared_ptr[CField]] c_fields
+    def add_metadata(self, dict metadata):
+        """
+        Add metadata as dict of string keys and values to Schema
 
-        c_fields.resize(len(fields))
+        Parameters
+        ----------
+        metadata : dict
+            Keys and values must be string-like / coercible to bytes
 
-        for i in range(len(fields)):
-            field = fields[i]
-            c_fields[i] = field.sp_field
+        Returns
+        -------
+        schema : pyarrow.Schema
+        """
+        cdef shared_ptr[CKeyValueMetadata] c_meta
+        convert_metadata(metadata, &c_meta)
 
-        result = Schema()
-        result.init(c_fields)
+        cdef shared_ptr[CSchema] new_schema
+        with nogil:
+            check_status(self.schema.AddMetadata(c_meta, &new_schema))
 
-        return result
+        return box_schema(new_schema)
+
+    def remove_metadata(self):
+        """
+        Create new schema without metadata, if any
+
+        Returns
+        -------
+        schema : pyarrow.Schema
+        """
+        cdef shared_ptr[CSchema] new_schema
+        with nogil:
+            new_schema = self.schema.RemoveMetadata()
+        return box_schema(new_schema)
 
     def __str__(self):
         return frombytes(self.schema.ToString())
@@ -299,6 +366,15 @@ cdef class Schema:
         return self.__str__()
 
 
+cdef box_metadata(const CKeyValueMetadata* metadata):
+    cdef unordered_map[c_string, c_string] result
+    if metadata != NULL:
+        metadata.ToUnorderedMap(&result)
+        return result
+    else:
+        return None
+
+
 cdef dict _type_cache = {}
 
 
@@ -315,8 +391,49 @@ cdef DataType primitive_type(Type type):
 #------------------------------------------------------------
 # Type factory functions
 
-def field(name, type, bint nullable=True):
-    return Field.from_py(name, type, nullable)
+cdef int convert_metadata(dict metadata,
+                          shared_ptr[CKeyValueMetadata]* out) except -1:
+    cdef:
+        shared_ptr[CKeyValueMetadata] meta = (
+            make_shared[CKeyValueMetadata]())
+        c_string key, value
+
+    for py_key, py_value in metadata.items():
+        key = tobytes(py_key)
+        value = tobytes(py_value)
+        meta.get().Append(key, value)
+    out[0] = meta
+    return 0
+
+
+def field(name, DataType type, bint nullable=True, dict metadata=None):
+    """
+    Create a pyarrow.Field instance
+
+    Parameters
+    ----------
+    name : string or bytes
+    type : pyarrow.DataType
+    nullable : boolean, default True
+    metadata : dict, default None
+        Keys and values must be coercible to bytes
+
+    Returns
+    -------
+    field : pyarrow.Field
+    """
+    cdef:
+        shared_ptr[CKeyValueMetadata] c_meta
+        Field result = Field()
+
+    if metadata is not None:
+        convert_metadata(metadata, &c_meta)
+
+    result.sp_field.reset(new CField(tobytes(name), type.sp_type,
+                                     nullable, c_meta))
+    result.field = result.sp_field.get()
+    result.type = type
+    return result
 
 
 cdef set PRIMITIVE_TYPES = set([
@@ -554,7 +671,28 @@ def struct(fields):
 
 
 def schema(fields):
-    return Schema.from_fields(fields)
+    """
+    Construct pyarrow.Schema from collection of fields
+
+    Parameters
+    ----------
+    field : list or iterable
+
+    Returns
+    -------
+    schema : pyarrow.Schema
+    """
+    cdef:
+        Schema result
+        Field field
+        vector[shared_ptr[CField]] c_fields
+
+    for i, field in enumerate(fields):
+        c_fields.push_back(field.sp_field)
+
+    result = Schema()
+    result.init(c_fields)
+    return result
 
 
 cdef DataType box_data_type(const shared_ptr[CDataType]& type):
diff --git a/python/pyarrow/_table.pyx b/python/pyarrow/_table.pyx
index ed0782bbba0a3..599e046e956b6 100644
--- a/python/pyarrow/_table.pyx
+++ b/python/pyarrow/_table.pyx
@@ -272,11 +272,12 @@ cdef class Column:
         return chunked_array
 
 
-cdef CKeyValueMetadata key_value_metadata_from_dict(dict metadata):
+cdef shared_ptr[const CKeyValueMetadata] key_value_metadata_from_dict(
+    dict metadata):
     cdef:
         unordered_map[c_string, c_string] unordered_metadata = metadata
-        CKeyValueMetadata c_metadata = CKeyValueMetadata(unordered_metadata)
-    return c_metadata
+    return (<shared_ptr[const CKeyValueMetadata]>
+            make_shared[CKeyValueMetadata](unordered_metadata))
 
 
 cdef int _schema_from_arrays(
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index ef1a332bed52f..8a730b3988441 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -23,6 +23,10 @@ cdef extern from "arrow/util/key_value_metadata.h" namespace "arrow" nogil:
     cdef cppclass CKeyValueMetadata" arrow::KeyValueMetadata":
         CKeyValueMetadata()
         CKeyValueMetadata(const unordered_map[c_string, c_string]&)
+
+        c_bool Equals(const CKeyValueMetadata& other)
+
+        void Append(const c_string& key, const c_string& value)
         void ToUnorderedMap(unordered_map[c_string, c_string]*) const
 
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
@@ -168,25 +172,48 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CDataType] type()
         c_bool nullable()
 
+        c_string ToString()
+        c_bool Equals(const CField& other)
+
+        shared_ptr[const CKeyValueMetadata] metadata()
+
         CField(const c_string& name, const shared_ptr[CDataType]& type,
                c_bool nullable)
 
+        CField(const c_string& name, const shared_ptr[CDataType]& type,
+               c_bool nullable, const shared_ptr[CKeyValueMetadata]& metadata)
+
+        # Removed const in Cython so don't have to cast to get code to generate
+        CStatus AddMetadata(const shared_ptr[CKeyValueMetadata]& metadata,
+                            shared_ptr[CField]* out)
+        shared_ptr[CField] RemoveMetadata()
+
+
     cdef cppclass CStructType" arrow::StructType"(CDataType):
         CStructType(const vector[shared_ptr[CField]]& fields)
 
     cdef cppclass CSchema" arrow::Schema":
         CSchema(const vector[shared_ptr[CField]]& fields)
         CSchema(const vector[shared_ptr[CField]]& fields,
-                const CKeyValueMetadata& custom_metadata)
+                const shared_ptr[const CKeyValueMetadata]& metadata)
+
+        # Does not actually exist, but gets Cython to not complain
+        CSchema(const vector[shared_ptr[CField]]& fields,
+                const shared_ptr[CKeyValueMetadata]& metadata)
 
         c_bool Equals(const CSchema& other)
 
         shared_ptr[CField] field(int i)
-        const CKeyValueMetadata& custom_metadata() const
+        shared_ptr[const CKeyValueMetadata] metadata()
         shared_ptr[CField] GetFieldByName(c_string& name)
         int num_fields()
         c_string ToString()
 
+        # Removed const in Cython so don't have to cast to get code to generate
+        CStatus AddMetadata(const shared_ptr[CKeyValueMetadata]& metadata,
+                            shared_ptr[CSchema]* out)
+        shared_ptr[CSchema] RemoveMetadata()
+
     cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray):
         c_bool Value(int i)
 
diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py
index e081c38713057..313a3ae9f1747 100644
--- a/python/pyarrow/tests/pandas_examples.py
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -73,7 +73,7 @@ def dataframe_with_arrays():
     ]
 
     df = pd.DataFrame(arrays)
-    schema = pa.Schema.from_fields(fields)
+    schema = pa.schema(fields)
 
     return df, schema
 
@@ -114,6 +114,6 @@ def dataframe_with_lists():
     ]
 
     df = pd.DataFrame(arrays)
-    schema = pa.Schema.from_fields(fields)
+    schema = pa.schema(fields)
 
     return df, schema
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index f3602347a78a6..2779da3320c6f 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -106,10 +106,10 @@ def test_float_no_nulls(self):
         for numpy_dtype, arrow_dtype in dtypes:
             values = np.random.randn(num_values)
             data[numpy_dtype] = values.astype(numpy_dtype)
-            fields.append(pa.Field.from_py(numpy_dtype, arrow_dtype))
+            fields.append(pa.field(numpy_dtype, arrow_dtype))
 
         df = pd.DataFrame(data)
-        schema = pa.Schema.from_fields(fields)
+        schema = pa.schema(fields)
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_float_nulls(self):
@@ -127,7 +127,7 @@ def test_float_nulls(self):
 
             arr = pa.Array.from_pandas(values, null_mask)
             arrays.append(arr)
-            fields.append(pa.Field.from_py(name, arrow_dtype))
+            fields.append(pa.field(name, arrow_dtype))
             values[null_mask] = np.nan
 
             expected_cols.append(values)
@@ -136,7 +136,7 @@ def test_float_nulls(self):
                                 columns=names)
 
         table = pa.Table.from_arrays(arrays, names)
-        assert table.schema.equals(pa.Schema.from_fields(fields))
+        assert table.schema.equals(pa.schema(fields))
         result = table.to_pandas()
         tm.assert_frame_equal(result, ex_frame)
 
@@ -159,10 +159,10 @@ def test_integer_no_nulls(self):
                                        min(info.max, np.iinfo('i8').max),
                                        size=num_values)
             data[dtype] = values.astype(dtype)
-            fields.append(pa.Field.from_py(dtype, arrow_dtype))
+            fields.append(pa.field(dtype, arrow_dtype))
 
         df = pd.DataFrame(data)
-        schema = pa.Schema.from_fields(fields)
+        schema = pa.schema(fields)
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_integer_with_nulls(self):
@@ -200,8 +200,8 @@ def test_boolean_no_nulls(self):
         np.random.seed(0)
 
         df = pd.DataFrame({'bools': np.random.randn(num_values) > 0})
-        field = pa.Field.from_py('bools', pa.bool_())
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('bools', pa.bool_())
+        schema = pa.schema([field])
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_boolean_nulls(self):
@@ -217,8 +217,8 @@ def test_boolean_nulls(self):
         expected = values.astype(object)
         expected[mask] = None
 
-        field = pa.Field.from_py('bools', pa.bool_())
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('bools', pa.bool_())
+        schema = pa.schema([field])
         ex_frame = pd.DataFrame({'bools': expected})
 
         table = pa.Table.from_arrays([arr], ['bools'])
@@ -230,16 +230,16 @@ def test_boolean_nulls(self):
     def test_boolean_object_nulls(self):
         arr = np.array([False, None, True] * 100, dtype=object)
         df = pd.DataFrame({'bools': arr})
-        field = pa.Field.from_py('bools', pa.bool_())
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('bools', pa.bool_())
+        schema = pa.schema([field])
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_unicode(self):
         repeats = 1000
         values = [u'foo', None, u'bar', u'mañana', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
-        field = pa.Field.from_py('strings', pa.string())
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
 
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
@@ -257,7 +257,7 @@ def test_bytes_to_binary(self):
     def test_fixed_size_bytes(self):
         values = [b'foo', None, b'bar', None, None, b'hey']
         df = pd.DataFrame({'strings': values})
-        schema = pa.Schema.from_fields([pa.field('strings', pa.binary(3))])
+        schema = pa.schema([pa.field('strings', pa.binary(3))])
         table = pa.Table.from_pandas(df, schema=schema)
         assert table.schema[0].type == schema[0].type
         assert table.schema[0].name == schema[0].name
@@ -267,7 +267,7 @@ def test_fixed_size_bytes(self):
     def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
         values = [b'foo', None, b'ba', None, None, b'hey']
         df = pd.DataFrame({'strings': values})
-        schema = pa.Schema.from_fields([pa.field('strings', pa.binary(3))])
+        schema = pa.schema([pa.field('strings', pa.binary(3))])
         with self.assertRaises(pa.ArrowInvalid):
             pa.Table.from_pandas(df, schema=schema)
 
@@ -279,8 +279,8 @@ def test_timestamps_notimezone_no_nulls(self):
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
             })
-        field = pa.Field.from_py('datetime64', pa.timestamp('ms'))
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('datetime64', pa.timestamp('ms'))
+        schema = pa.schema([field])
         self._check_pandas_roundtrip(df, timestamps_to_ms=True,
                                      expected_schema=schema)
 
@@ -291,8 +291,8 @@ def test_timestamps_notimezone_no_nulls(self):
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
             })
-        field = pa.Field.from_py('datetime64', pa.timestamp('ns'))
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('datetime64', pa.timestamp('ns'))
+        schema = pa.schema([field])
         self._check_pandas_roundtrip(df, timestamps_to_ms=False,
                                      expected_schema=schema)
 
@@ -304,8 +304,8 @@ def test_timestamps_notimezone_nulls(self):
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
             })
-        field = pa.Field.from_py('datetime64', pa.timestamp('ms'))
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('datetime64', pa.timestamp('ms'))
+        schema = pa.schema([field])
         self._check_pandas_roundtrip(df, timestamps_to_ms=True,
                                      expected_schema=schema)
 
@@ -316,8 +316,8 @@ def test_timestamps_notimezone_nulls(self):
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
             })
-        field = pa.Field.from_py('datetime64', pa.timestamp('ns'))
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('datetime64', pa.timestamp('ns'))
+        schema = pa.schema([field])
         self._check_pandas_roundtrip(df, timestamps_to_ms=False,
                                      expected_schema=schema)
 
@@ -353,8 +353,8 @@ def test_date_infer(self):
                      datetime.date(1970, 1, 1),
                      datetime.date(2040, 2, 26)]})
         table = pa.Table.from_pandas(df)
-        field = pa.Field.from_py('date', pa.date32())
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('date', pa.date32())
+        schema = pa.schema([field])
         assert table.schema.equals(schema)
         result = table.to_pandas()
         expected = df.copy()
@@ -526,8 +526,8 @@ def test_decimal_32_from_pandas(self):
             ]
         })
         converted = pa.Table.from_pandas(expected)
-        field = pa.Field.from_py('decimals', pa.decimal(7, 3))
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('decimals', pa.decimal(7, 3))
+        schema = pa.schema([field])
         assert converted.schema.equals(schema)
 
     def test_decimal_32_to_pandas(self):
@@ -549,8 +549,8 @@ def test_decimal_64_from_pandas(self):
             ]
         })
         converted = pa.Table.from_pandas(expected)
-        field = pa.Field.from_py('decimals', pa.decimal(12, 6))
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('decimals', pa.decimal(12, 6))
+        schema = pa.schema([field])
         assert converted.schema.equals(schema)
 
     def test_decimal_64_to_pandas(self):
@@ -572,8 +572,8 @@ def test_decimal_128_from_pandas(self):
             ]
         })
         converted = pa.Table.from_pandas(expected)
-        field = pa.Field.from_py('decimals', pa.decimal(26, 11))
-        schema = pa.Schema.from_fields([field])
+        field = pa.field('decimals', pa.decimal(26, 11))
+        schema = pa.schema([field])
         assert converted.schema.equals(schema)
 
     def test_decimal_128_to_pandas(self):
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index da704f378873b..b3abc0f04f418 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -118,7 +118,7 @@ def test_field():
     assert f.name == 'foo'
     assert f.nullable
     assert f.type is t
-    assert repr(f) == "Field('foo', type=string)"
+    assert repr(f) == "pyarrow.Field<foo: string>"
 
     f = pa.field('foo', t, False)
     assert not f.nullable
@@ -152,6 +152,52 @@ def test_field_empty():
         repr(f)
 
 
+def test_field_add_remove_metadata():
+    f0 = pa.field('foo', pa.int32())
+
+    assert f0.metadata is None
+
+    metadata = {b'foo': b'bar', b'pandas': b'badger'}
+
+    f1 = f0.add_metadata(metadata)
+    assert f1.metadata == metadata
+
+    f3 = f1.remove_metadata()
+    assert f3.metadata is None
+
+    # idempotent
+    f4 = f3.remove_metadata()
+    assert f4.metadata is None
+
+    f5 = pa.field('foo', pa.int32(), True, metadata)
+    f6 = f0.add_metadata(metadata)
+    assert f5.equals(f6)
+
+
+def test_schema_add_remove_metadata():
+    fields = [
+        pa.field('foo', pa.int32()),
+        pa.field('bar', pa.string()),
+        pa.field('baz', pa.list_(pa.int8()))
+    ]
+
+    s1 = pa.schema(fields)
+
+    assert s1.metadata is None
+
+    metadata = {b'foo': b'bar', b'pandas': b'badger'}
+
+    s2 = s1.add_metadata(metadata)
+    assert s2.metadata == metadata
+
+    s3 = s2.remove_metadata()
+    assert s3.metadata is None
+
+    # idempotent
+    s4 = s3.remove_metadata()
+    assert s4.metadata is None
+
+
 def test_schema_equals():
     fields = [
         pa.field('foo', pa.int32()),

From 909f826b55973e93f4656c43a84c8e740a86601f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 27 Apr 2017 11:19:02 -0400
Subject: [PATCH 0575/1644] ARROW-867: [Python] pyarrow MSVC fixes

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #575 from wesm/ARROW-867 and squashes the following commits:

0483cfb [Wes McKinney] Do not encode file paths to utf-16le on Windows. Fix date/time conversion, platform ints. Add release/acquire methods to PyAcquireGIL lock object. Remove a couple unneeded GIL acquisitions
---
 ci/msvc-build.bat                           |  5 ++
 cpp/src/arrow/python/builtin_convert.cc     | 16 +---
 cpp/src/arrow/python/common.h               | 22 +++++-
 cpp/src/arrow/python/helpers.cc             |  2 -
 cpp/src/arrow/python/init.cc                | 12 +--
 cpp/src/arrow/python/init.h                 | 12 +--
 cpp/src/arrow/python/numpy_interop.h        |  1 +
 cpp/src/arrow/python/pandas_convert.cc      | 17 +++--
 cpp/src/arrow/python/util/datetime.h        | 23 ++++++
 cpp/src/arrow/python/util/test_main.cc      |  2 +-
 python/pyarrow/_array.pyx                   |  6 +-
 python/pyarrow/_config.pyx                  |  6 +-
 python/pyarrow/_table.pyx                   |  2 +-
 python/pyarrow/compat.py                    | 11 +--
 python/pyarrow/includes/pyarrow.pxd         |  6 +-
 python/pyarrow/tests/test_convert_pandas.py |  4 +-
 python/pyarrow/tests/test_feather.py        |  5 ++
 python/pyarrow/tests/test_io.py             | 83 ++++++++++-----------
 python/pyarrow/tests/test_schema.py         |  1 -
 python/pyarrow/tests/test_tensor.py         | 32 ++++----
 20 files changed, 140 insertions(+), 128 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index de428b6e46e14..08c5033849539 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -43,6 +43,8 @@ set PYTHONPATH=%CONDA_ENV%\Lib;%CONDA_ENV%\Lib\site-packages;%CONDA_ENV%\python3
 
 ctest -VV  || exit /B
 
+set PYTHONPATH=
+
 @rem Build and import pyarrow
 
 set PATH=%ARROW_HOME%\bin;%PATH%
@@ -50,3 +52,6 @@ set PATH=%ARROW_HOME%\bin;%PATH%
 cd ..\..\python
 python setup.py build_ext --inplace  || exit /B
 python -c "import pyarrow"  || exit /B
+
+@rem TODO: re-enable when last tests are fixed
+@rem py.test pyarrow -v -s || exit /B
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 137937c0946df..3197c2ade4bae 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -358,22 +358,8 @@ class TimestampConverter : public TypedConverter<TimestampBuilder> {
       } else {
         PyDateTime_DateTime* pydatetime =
             reinterpret_cast<PyDateTime_DateTime*>(item.obj());
-        struct tm datetime = {0};
-        datetime.tm_year = PyDateTime_GET_YEAR(pydatetime) - 1900;
-        datetime.tm_mon = PyDateTime_GET_MONTH(pydatetime) - 1;
-        datetime.tm_mday = PyDateTime_GET_DAY(pydatetime);
-        datetime.tm_hour = PyDateTime_DATE_GET_HOUR(pydatetime);
-        datetime.tm_min = PyDateTime_DATE_GET_MINUTE(pydatetime);
-        datetime.tm_sec = PyDateTime_DATE_GET_SECOND(pydatetime);
-        int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
+        typed_builder_->Append(PyDateTime_to_us(pydatetime));
         RETURN_IF_PYERROR();
-        struct tm epoch = {0};
-        epoch.tm_year = 70;
-        epoch.tm_mday = 1;
-        // Microseconds since the epoch
-        int64_t val = static_cast<int64_t>(
-            lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000 + us);
-        typed_builder_->Append(val);
       }
     }
     return Status::OK();
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index 882bb156224c0..0211823c8c8fe 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -34,11 +34,29 @@ namespace py {
 
 class ARROW_EXPORT PyAcquireGIL {
  public:
-  PyAcquireGIL() { state_ = PyGILState_Ensure(); }
+  PyAcquireGIL() : acquired_gil_(false) {
+    acquire();
+  }
+
+  ~PyAcquireGIL() { release(); }
 
-  ~PyAcquireGIL() { PyGILState_Release(state_); }
+  void acquire() {
+    if (!acquired_gil_) {
+      state_ = PyGILState_Ensure();
+      acquired_gil_ = true;
+    }
+  }
+
+  // idempotent
+  void release() {
+    if (acquired_gil_) {
+      PyGILState_Release(state_);
+      acquired_gil_ = false;
+    }
+  }
 
  private:
+  bool acquired_gil_;
   PyGILState_STATE state_;
   DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL);
 };
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index f7c73a87fbf16..e5d1d388cb54c 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -55,7 +55,6 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
 }
 
 Status ImportModule(const std::string& module_name, OwnedRef* ref) {
-  PyAcquireGIL lock;
   PyObject* module = PyImport_ImportModule(module_name.c_str());
   RETURN_IF_PYERROR();
   ref->reset(module);
@@ -66,7 +65,6 @@ Status ImportFromModule(const OwnedRef& module, const std::string& name, OwnedRe
   /// Assumes that ImportModule was called first
   DCHECK_NE(module.obj(), nullptr) << "Cannot import from nullptr Python module";
 
-  PyAcquireGIL lock;
   PyObject* attr = PyObject_GetAttrString(module.obj(), name.c_str());
   RETURN_IF_PYERROR();
   ref->reset(attr);
diff --git a/cpp/src/arrow/python/init.cc b/cpp/src/arrow/python/init.cc
index fa70af7e44db3..db648915465a8 100644
--- a/cpp/src/arrow/python/init.cc
+++ b/cpp/src/arrow/python/init.cc
@@ -15,20 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/python/platform.h"
-
 // Trigger the array import (inversion of NO_IMPORT_ARRAY)
 #define NUMPY_IMPORT_ARRAY
 
 #include "arrow/python/init.h"
 #include "arrow/python/numpy_interop.h"
 
-namespace arrow {
-namespace py {
-
-void InitNumPy() {
-  import_numpy();
+int arrow_init_numpy() {
+  return arrow::py::import_numpy();
 }
-
-}  // namespace py
-}  // namespace arrow
diff --git a/cpp/src/arrow/python/init.h b/cpp/src/arrow/python/init.h
index a2533d8059273..1daa5a3d2624d 100644
--- a/cpp/src/arrow/python/init.h
+++ b/cpp/src/arrow/python/init.h
@@ -19,17 +19,11 @@
 #define ARROW_PYTHON_INIT_H
 
 #include "arrow/python/platform.h"
-
-#include "arrow/python/numpy_interop.h"
 #include "arrow/util/visibility.h"
 
-namespace arrow {
-namespace py {
-
+extern "C" {
 ARROW_EXPORT
-void InitNumPy();
-
-}  // namespace py
-}  // namespace arrow
+int arrow_init_numpy();
+}
 
 #endif  // ARROW_PYTHON_INIT_H
diff --git a/cpp/src/arrow/python/numpy_interop.h b/cpp/src/arrow/python/numpy_interop.h
index b93200cc8972d..023fdc8249c0c 100644
--- a/cpp/src/arrow/python/numpy_interop.h
+++ b/cpp/src/arrow/python/numpy_interop.h
@@ -47,6 +47,7 @@ namespace py {
 
 inline int import_numpy() {
 #ifdef NUMPY_IMPORT_ARRAY
+  std::cout << "Importing NumPy" << std::endl;
   import_array1(-1);
   import_umath1(-1);
 #endif
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 9f65af41bb294..b54197e5145b0 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -763,11 +763,10 @@ Status PandasConverter::ConvertObjects() {
 
   Ndarray1DIndexer<PyObject*> objects;
 
-  {
-    PyAcquireGIL lock;
-    objects.Init(arr_);
-    PyDateTime_IMPORT;
-  }
+  PyAcquireGIL lock;
+  objects.Init(arr_);
+  PyDateTime_IMPORT;
+  lock.release();
 
   // This means we received an explicit type from the user
   if (type_) {
@@ -792,6 +791,9 @@ Status PandasConverter::ConvertObjects() {
         return Status::TypeError("No known conversion to Arrow type");
     }
   } else {
+    // Re-acquire GIL
+    lock.acquire();
+
     OwnedRef decimal;
     OwnedRef Decimal;
     RETURN_NOT_OK(ImportModule("decimal", &decimal));
@@ -2196,7 +2198,12 @@ class ArrowDeserializer {
     RETURN_IF_PYERROR();
 
     PyObject* dictionary;
+
+    // Release GIL before calling ConvertArrayToPandas, will be reacquired
+    // there if needed
+    lock.release();
     RETURN_NOT_OK(ConvertArrayToPandas(dict_type->dictionary(), nullptr, &dictionary));
+    lock.acquire();
 
     PyDict_SetItemString(result_, "indices", block->block_arr());
     PyDict_SetItemString(result_, "dictionary", dictionary);
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index ad0ee0f5056da..7ebf46a92fd5c 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -42,6 +42,29 @@ static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
 #endif
 }
 
+static inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) {
+  struct tm datetime = {0};
+  datetime.tm_year = PyDateTime_GET_YEAR(pydatetime) - 1900;
+  datetime.tm_mon = PyDateTime_GET_MONTH(pydatetime) - 1;
+  datetime.tm_mday = PyDateTime_GET_DAY(pydatetime);
+  datetime.tm_hour = PyDateTime_DATE_GET_HOUR(pydatetime);
+  datetime.tm_min = PyDateTime_DATE_GET_MINUTE(pydatetime);
+  datetime.tm_sec = PyDateTime_DATE_GET_SECOND(pydatetime);
+  int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
+  struct tm epoch = {0};
+  epoch.tm_year = 70;
+  epoch.tm_mday = 1;
+#ifdef _MSC_VER
+  // Microseconds since the epoch
+  const int64_t current_timestamp = static_cast<int64_t>(_mkgmtime64(&datetime));
+  const int64_t epoch_timestamp = static_cast<int64_t>(_mkgmtime64(&epoch));
+  return (current_timestamp - epoch_timestamp) * 1000000L + us;
+#else
+  return static_cast<int64_t>(
+      lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000 + us);
+#endif
+}
+
 static inline int32_t PyDate_to_days(PyDateTime_Date* pydate) {
   return static_cast<int32_t>(PyDate_to_ms(pydate) / 86400000LL);
 }
diff --git a/cpp/src/arrow/python/util/test_main.cc b/cpp/src/arrow/python/util/test_main.cc
index c24da40aadcf6..efb44754b2b3a 100644
--- a/cpp/src/arrow/python/util/test_main.cc
+++ b/cpp/src/arrow/python/util/test_main.cc
@@ -25,7 +25,7 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
 
   Py_Initialize();
-  arrow::py::InitNumPy();
+  arrow_init_numpy();
 
   int ret = RUN_ALL_TESTS();
 
diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/_array.pyx
index 658f4b314b3a2..f01cff6cc99f3 100644
--- a/python/pyarrow/_array.pyx
+++ b/python/pyarrow/_array.pyx
@@ -1288,8 +1288,7 @@ cdef class Array:
 
         with nogil:
             check_status(
-                pyarrow.ConvertArrayToPandas(self.sp_array, <PyObject*> self,
-                                             &out))
+                pyarrow.ConvertArrayToPandas(self.sp_array, self, &out))
         return wrap_array_output(out)
 
     def to_pylist(self):
@@ -1326,8 +1325,7 @@ strides: {2}""".format(self.type, self.shape, self.strides)
         cdef:
             PyObject* out
 
-        check_status(pyarrow.TensorToNdarray(deref(self.tp), <PyObject*> self,
-                                             &out))
+        check_status(pyarrow.TensorToNdarray(deref(self.tp), self, &out))
         return PyObject_to_object(out)
 
     def equals(self, Tensor other):
diff --git a/python/pyarrow/_config.pyx b/python/pyarrow/_config.pyx
index 2c1e6bf3143aa..e5fdbef8af5f6 100644
--- a/python/pyarrow/_config.pyx
+++ b/python/pyarrow/_config.pyx
@@ -14,13 +14,13 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-cdef extern from 'arrow/python/init.h' namespace 'arrow::py':
-    void InitNumPy()
+cdef extern from 'arrow/python/init.h':
+    int arrow_init_numpy() except -1
 
 cdef extern from 'arrow/python/config.h' namespace 'arrow::py':
     void set_numpy_nan(object o)
 
-InitNumPy()
+arrow_init_numpy()
 
 import numpy as np
 set_numpy_nan(np.nan)
diff --git a/python/pyarrow/_table.pyx b/python/pyarrow/_table.pyx
index 599e046e956b6..223fe27ea9819 100644
--- a/python/pyarrow/_table.pyx
+++ b/python/pyarrow/_table.pyx
@@ -164,7 +164,7 @@ cdef class Column:
             PyObject* out
 
         check_status(pyarrow.ConvertColumnToPandas(self.sp_column,
-                                                   <PyObject*> self, &out))
+                                                   self, &out))
 
         return _pandas().Series(wrap_array_output(out), name=self.name)
 
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 8d15c4c1e3fb5..928a2c0724298 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -139,14 +139,9 @@ def encode_file_path(path):
     import os
     # Windows requires utf-16le encoding for unicode file names
     if isinstance(path, unicode_type):
-        if os.name == 'nt':
-            # try:
-            #     encoded_path = path.encode('ascii')
-            # except:
-            encoded_path = path.encode('utf-16le')
-        else:
-            # POSIX systems can handle utf-8
-            encoded_path = path.encode('utf-8')
+        # POSIX systems can handle utf-8. UTF8 is converted to utf16-le in
+        # libarrow
+        encoded_path = path.encode('utf-8')
     else:
         encoded_path = path
 
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
index c40df3db8a9c5..35c71107f8db1 100644
--- a/python/pyarrow/includes/pyarrow.pxd
+++ b/python/pyarrow/includes/pyarrow.pxd
@@ -47,14 +47,14 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
                             shared_ptr[CTensor]* out);
 
-    CStatus TensorToNdarray(const CTensor& tensor, PyObject* base,
+    CStatus TensorToNdarray(const CTensor& tensor, object base,
                             PyObject** out)
 
     CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
-                                 PyObject* py_ref, PyObject** out)
+                                 object py_ref, PyObject** out)
 
     CStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
-                                  PyObject* py_ref, PyObject** out)
+                                  object py_ref, PyObject** out)
 
     CStatus ConvertTableToPandas(const shared_ptr[CTable]& table,
                                  int nthreads, PyObject** out)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 2779da3320c6f..9b9b7519fd994 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -155,8 +155,8 @@ def test_integer_no_nulls(self):
 
         for dtype, arrow_dtype in numpy_dtypes:
             info = np.iinfo(dtype)
-            values = np.random.randint(info.min,
-                                       min(info.max, np.iinfo('i8').max),
+            values = np.random.randint(max(info.min, np.iinfo(np.int_).min),
+                                       min(info.max, np.iinfo(np.int_).max),
                                        size=num_values)
             data[dtype] = values.astype(dtype)
             fields.append(pa.field(dtype, arrow_dtype))
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index ef73a8feeb65c..7a8abf486f4b5 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import sys
 import unittest
 import pytest
 
@@ -251,6 +252,9 @@ def test_boolean_object_nulls(self):
         self._check_pandas_roundtrip(df, null_counts=[1 * repeats])
 
     def test_delete_partial_file_on_error(self):
+        if sys.platform == 'win32':
+            pytest.skip('Windows hangs on to file handle for some reason')
+
         # strings will fail
         df = pd.DataFrame(
             {
@@ -361,6 +365,7 @@ def test_read_columns(self):
 
     def test_overwritten_file(self):
         path = random_path()
+        self.test_files.append(path)
 
         num_values = 100
         np.random.seed(0)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index a14898ff2ffd0..610dedc6a7640 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -230,12 +230,13 @@ def test_nativefile_write_memoryview():
 
 
 @pytest.fixture
-def sample_disk_data(request):
+def sample_disk_data(request, tmpdir):
     SIZE = 4096
     arr = np.random.randint(0, 256, size=SIZE).astype('u1')
     data = arr.tobytes()[:SIZE]
 
-    path = guid()
+    path = os.path.join(str(tmpdir), guid())
+
     with open(path, 'wb') as f:
         f.write(data)
 
@@ -298,68 +299,62 @@ def _try_delete(path):
         pass
 
 
-def test_memory_map_writer():
+def test_memory_map_writer(tmpdir):
     SIZE = 4096
     arr = np.random.randint(0, 256, size=SIZE).astype('u1')
     data = arr.tobytes()[:SIZE]
 
-    path = guid()
-    try:
-        with open(path, 'wb') as f:
-            f.write(data)
+    path = os.path.join(str(tmpdir), guid())
+    with open(path, 'wb') as f:
+        f.write(data)
 
-        f = pa.memory_map(path, mode='r+w')
+    f = pa.memory_map(path, mode='r+w')
 
-        f.seek(10)
-        f.write('peekaboo')
-        assert f.tell() == 18
+    f.seek(10)
+    f.write('peekaboo')
+    assert f.tell() == 18
 
-        f.seek(10)
-        assert f.read(8) == b'peekaboo'
+    f.seek(10)
+    assert f.read(8) == b'peekaboo'
 
-        f2 = pa.memory_map(path, mode='r+w')
+    f2 = pa.memory_map(path, mode='r+w')
 
-        f2.seek(10)
-        f2.write(b'booapeak')
-        f2.seek(10)
+    f2.seek(10)
+    f2.write(b'booapeak')
+    f2.seek(10)
 
-        f.seek(10)
-        assert f.read(8) == b'booapeak'
+    f.seek(10)
+    assert f.read(8) == b'booapeak'
 
-        # Does not truncate file
-        f3 = pa.memory_map(path, mode='w')
-        f3.write('foo')
+    # Does not truncate file
+    f3 = pa.memory_map(path, mode='w')
+    f3.write('foo')
 
-        with pa.memory_map(path) as f4:
-            assert f4.size() == SIZE
+    with pa.memory_map(path) as f4:
+        assert f4.size() == SIZE
 
-        with pytest.raises(IOError):
-            f3.read(5)
+    with pytest.raises(IOError):
+        f3.read(5)
 
-        f.seek(0)
-        assert f.read(3) == b'foo'
-    finally:
-        _try_delete(path)
+    f.seek(0)
+    assert f.read(3) == b'foo'
 
 
-def test_os_file_writer():
+def test_os_file_writer(tmpdir):
     SIZE = 4096
     arr = np.random.randint(0, 256, size=SIZE).astype('u1')
     data = arr.tobytes()[:SIZE]
 
-    path = guid()
-    try:
-        with open(path, 'wb') as f:
-            f.write(data)
+    path = os.path.join(str(tmpdir), guid())
+    with open(path, 'wb') as f:
+        f.write(data)
 
-        # Truncates file
-        f2 = pa.OSFile(path, mode='w')
-        f2.write('foo')
+    # Truncates file
+    f2 = pa.OSFile(path, mode='w')
+    f2.write('foo')
 
-        with pa.OSFile(path) as f3:
-            assert f3.size() == 3
+    with pa.OSFile(path) as f3:
+        assert f3.size() == 3
 
-        with pytest.raises(IOError):
-            f2.read(5)
-    finally:
-        _try_delete(path)
+    with pytest.raises(IOError):
+        f2.read(5)
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index b3abc0f04f418..2d98865b56e73 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -206,7 +206,6 @@ def test_schema_equals():
     ]
 
     sch1 = pa.schema(fields)
-    print(dir(sch1))
     sch2 = pa.schema(fields)
     assert sch1.equals(sch2)
 
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index ec71735b2a540..b0924e3504ff7 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -77,41 +77,37 @@ def test_tensor_numpy_roundtrip(dtype_str, arrow_type):
 
 
 def _try_delete(path):
+    import gc
+    gc.collect()
     try:
         os.remove(path)
     except os.error:
         pass
 
 
-def test_tensor_ipc_roundtrip():
+def test_tensor_ipc_roundtrip(tmpdir):
     data = np.random.randn(10, 4)
     tensor = pa.Tensor.from_numpy(data)
 
-    path = 'pyarrow-tensor-ipc-roundtrip'
-    try:
-        mmap = pa.create_memory_map(path, 1024)
+    path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-roundtrip')
+    mmap = pa.create_memory_map(path, 1024)
 
-        pa.write_tensor(tensor, mmap)
+    pa.write_tensor(tensor, mmap)
 
-        mmap.seek(0)
-        result = pa.read_tensor(mmap)
+    mmap.seek(0)
+    result = pa.read_tensor(mmap)
 
-        assert result.equals(tensor)
-    finally:
-        _try_delete(path)
+    assert result.equals(tensor)
 
 
-def test_tensor_ipc_strided():
+def test_tensor_ipc_strided(tmpdir):
     data = np.random.randn(10, 4)
     tensor = pa.Tensor.from_numpy(data[::2])
 
-    path = 'pyarrow-tensor-ipc-strided'
-    try:
-        with pytest.raises(ValueError):
-            mmap = pa.create_memory_map(path, 1024)
-            pa.write_tensor(tensor, mmap)
-    finally:
-        _try_delete(path)
+    path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-strided')
+    with pytest.raises(ValueError):
+        mmap = pa.create_memory_map(path, 1024)
+        pa.write_tensor(tensor, mmap)
 
 
 def test_tensor_size():

From 81be9c6679466177d4b8e5dbca55f81185bb3ec6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 27 Apr 2017 18:10:24 +0200
Subject: [PATCH 0576/1644] ARROW-866: [Python] Be robust to PyErr_Fetch
 returning a null exc value

cc @BryanCutler. This was a tricky one. I am not sure how to reproduce with our current code -- I reverted the patch from ARROW-822 to get a reproduction so I could fix this. Now, the error raised is:

```
/home/wesm/code/arrow/python/pyarrow/_error.pyx in pyarrow._error.check_status (/home/wesm/code/arrow/python/build/temp.linux-x86_64-2.7/_error.cxx:1324)()
     58             raise ArrowInvalid(message)
     59         elif status.IsIOError():
---> 60             raise ArrowIOError(message)
     61         elif status.IsOutOfMemory():
     62             raise ArrowMemoryError(message)

ArrowIOError: IOError: Error message was null
```

I'm not sure why calling `tell` on the socket object results in a bad exception state, but in any case it seems that the result of `PyErr_Fetch` cannot be relied upon to be non-null even when `PyErr_Occurred()` returns non-null

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #606 from wesm/ARROW-866 and squashes the following commits:

fa395cd [Wes McKinney] Enable other kinds of Status errors to be returned
0bd11c2 [Wes McKinney] Consolidate error handling code a bit
9d59dd2 [Wes McKinney] Be robust to PyErr_Fetch returning a null exc value
---
 cpp/src/arrow/python/common.cc | 22 ++++++++++++++++++++++
 cpp/src/arrow/python/common.h  | 29 ++++++++++++++---------------
 cpp/src/arrow/python/io.cc     | 29 +++++++----------------------
 cpp/src/arrow/status.h         |  3 +++
 4 files changed, 46 insertions(+), 37 deletions(-)

diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
index 717cb5c5cc122..bedd458c783f4 100644
--- a/cpp/src/arrow/python/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -64,5 +64,27 @@ PyBuffer::~PyBuffer() {
   Py_XDECREF(obj_);
 }
 
+Status CheckPyError(StatusCode code) {
+  if (PyErr_Occurred()) {
+    PyObject *exc_type, *exc_value, *traceback;
+    PyErr_Fetch(&exc_type, &exc_value, &traceback);
+    PyObjectStringify stringified(exc_value);
+    Py_XDECREF(exc_type);
+    Py_XDECREF(exc_value);
+    Py_XDECREF(traceback);
+    PyErr_Clear();
+
+    // ARROW-866: in some esoteric cases, formatting exc_value can fail. This
+    // was encountered when calling tell() on a socket file
+    if (stringified.bytes != nullptr) {
+      std::string message(stringified.bytes);
+      return Status(code, message);
+    } else {
+      return Status(code, "Error message was null");
+    }
+  }
+  return Status::OK();
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index 0211823c8c8fe..c5745a53f70dc 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -98,27 +98,26 @@ struct ARROW_EXPORT PyObjectStringify {
     if (PyUnicode_Check(obj)) {
       bytes_obj = PyUnicode_AsUTF8String(obj);
       tmp_obj.reset(bytes_obj);
+      bytes = PyBytes_AsString(bytes_obj);
+      size = PyBytes_GET_SIZE(bytes_obj);
+    } else if (PyBytes_Check(obj)) {
+      bytes = PyBytes_AsString(obj);
+      size = PyBytes_GET_SIZE(obj);
     } else {
-      bytes_obj = obj;
+      bytes = nullptr;
+      size = -1;
     }
-    bytes = PyBytes_AsString(bytes_obj);
-    size = PyBytes_GET_SIZE(bytes_obj);
   }
 };
 
+Status CheckPyError(StatusCode code = StatusCode::UnknownError);
+
 // TODO(wesm): We can just let errors pass through. To be explored later
-#define RETURN_IF_PYERROR()                         \
-  if (PyErr_Occurred()) {                           \
-    PyObject *exc_type, *exc_value, *traceback;     \
-    PyErr_Fetch(&exc_type, &exc_value, &traceback); \
-    PyObjectStringify stringified(exc_value);       \
-    std::string message(stringified.bytes);         \
-    Py_DECREF(exc_type);                            \
-    Py_XDECREF(exc_value);                          \
-    Py_XDECREF(traceback);                          \
-    PyErr_Clear();                                  \
-    return Status::UnknownError(message);           \
-  }
+#define RETURN_IF_PYERROR()                     \
+  RETURN_NOT_OK(CheckPyError());
+
+#define PY_RETURN_IF_ERROR(CODE)                \
+  RETURN_NOT_OK(CheckPyError(CODE));
 
 // Return the common PyArrow memory pool
 ARROW_EXPORT void set_default_memory_pool(MemoryPool* pool);
diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index 327e8fe9ff781..a7193854c4d01 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -41,21 +41,6 @@ PythonFile::~PythonFile() {
   Py_DECREF(file_);
 }
 
-static Status CheckPyError() {
-  if (PyErr_Occurred()) {
-    PyObject *exc_type, *exc_value, *traceback;
-    PyErr_Fetch(&exc_type, &exc_value, &traceback);
-    PyObjectStringify stringified(exc_value);
-    std::string message(stringified.bytes);
-    Py_XDECREF(exc_type);
-    Py_XDECREF(exc_value);
-    Py_XDECREF(traceback);
-    PyErr_Clear();
-    return Status::IOError(message);
-  }
-  return Status::OK();
-}
-
 // This is annoying: because C++11 does not allow implicit conversion of string
 // literals to non-const char*, we need to go through some gymnastics to use
 // PyObject_CallMethod without a lot of pain (its arguments are non-const
@@ -71,7 +56,7 @@ Status PythonFile::Close() {
   // whence: 0 for relative to start of file, 2 for end of file
   PyObject* result = cpp_PyObject_CallMethod(file_, "close", "()");
   Py_XDECREF(result);
-  ARROW_RETURN_NOT_OK(CheckPyError());
+  PY_RETURN_IF_ERROR(StatusCode::IOError);
   return Status::OK();
 }
 
@@ -79,13 +64,13 @@ Status PythonFile::Seek(int64_t position, int whence) {
   // whence: 0 for relative to start of file, 2 for end of file
   PyObject* result = cpp_PyObject_CallMethod(file_, "seek", "(ii)", position, whence);
   Py_XDECREF(result);
-  ARROW_RETURN_NOT_OK(CheckPyError());
+  PY_RETURN_IF_ERROR(StatusCode::IOError);
   return Status::OK();
 }
 
 Status PythonFile::Read(int64_t nbytes, PyObject** out) {
   PyObject* result = cpp_PyObject_CallMethod(file_, "read", "(i)", nbytes);
-  ARROW_RETURN_NOT_OK(CheckPyError());
+  PY_RETURN_IF_ERROR(StatusCode::IOError);
   *out = result;
   return Status::OK();
 }
@@ -93,24 +78,24 @@ Status PythonFile::Read(int64_t nbytes, PyObject** out) {
 Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
   PyObject* py_data =
       PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), nbytes);
-  ARROW_RETURN_NOT_OK(CheckPyError());
+  PY_RETURN_IF_ERROR(StatusCode::IOError);
 
   PyObject* result = cpp_PyObject_CallMethod(file_, "write", "(O)", py_data);
   Py_XDECREF(py_data);
   Py_XDECREF(result);
-  ARROW_RETURN_NOT_OK(CheckPyError());
+  PY_RETURN_IF_ERROR(StatusCode::IOError);
   return Status::OK();
 }
 
 Status PythonFile::Tell(int64_t* position) {
   PyObject* result = cpp_PyObject_CallMethod(file_, "tell", "()");
-  ARROW_RETURN_NOT_OK(CheckPyError());
+  PY_RETURN_IF_ERROR(StatusCode::IOError);
 
   *position = PyLong_AsLongLong(result);
   Py_DECREF(result);
 
   // PyLong_AsLongLong can raise OverflowError
-  ARROW_RETURN_NOT_OK(CheckPyError());
+  PY_RETURN_IF_ERROR(StatusCode::IOError);
 
   return Status::OK();
 }
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index dd65b753fef31..6a8cee27cb756 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -91,6 +91,9 @@ class ARROW_EXPORT Status {
   Status() : state_(NULL) {}
   ~Status() { delete[] state_; }
 
+  Status(StatusCode code, const std::string& msg)
+    : Status(code, msg, -1) {}
+
   // Copy the specified status.
   Status(const Status& s);
   void operator=(const Status& s);

From 03dce9dcab1df587f2293decf49708f872aaad3d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 27 Apr 2017 18:11:44 +0200
Subject: [PATCH 0577/1644] ARROW-900: [Python] Fix UnboundLocalError in
 ParquetDatasetPiece.read

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #607 from wesm/ARROW-900 and squashes the following commits:

81f8394 [Wes McKinney] Fix UnboundLocalError in ParquetDatasetPiece.read
---
 python/pyarrow/parquet.py            |  3 +++
 python/pyarrow/tests/test_parquet.py | 14 ++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 94ad227fbefa9..21359f137f24e 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -208,6 +208,9 @@ def read(self, columns=None, nthreads=1, partitions=None,
             reader = self._open(open_file_func)
         elif file is not None:
             reader = ParquetFile(file)
+        else:
+            # try to read the local path
+            reader = ParquetFile(self.path)
 
         if self.row_group is not None:
             table = reader.read_row_group(self.row_group, columns=columns,
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 8c446af03fc16..bb3a9ed5f4a25 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -492,6 +492,20 @@ def test_read_single_row_group():
     tm.assert_frame_equal(df[cols], result.to_pandas())
 
 
+@parquet
+def test_parquet_piece_read(tmpdir):
+    df = _test_dataframe(1000)
+    table = pa.Table.from_pandas(df)
+
+    path = tmpdir.join('parquet_piece_read.parquet').strpath
+    pq.write_table(table, path, version='2.0')
+
+    piece1 = pq.ParquetDatasetPiece(path)
+
+    result = piece1.read()
+    assert result.equals(table)
+
+
 @parquet
 def test_parquet_piece_basics():
     path = '/baz.parq'

From 14bec24c584dc6fa05b84b6ed00d7474d62fd1d7 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 27 Apr 2017 18:13:47 +0200
Subject: [PATCH 0578/1644] ARROW-908: [GLib] Unify OutputStream files

Author: Kouhei Sutou <kou@clear-code.com>

Closes #609 from kou/glib-unify-output-stream and squashes the following commits:

f62f869 [Kouhei Sutou] [GLib] Unify OutputStream files
---
 c_glib/arrow-glib/Makefile.am             |   3 -
 c_glib/arrow-glib/arrow-glib.h            |   1 -
 c_glib/arrow-glib/arrow-glib.hpp          |   1 -
 c_glib/arrow-glib/file-output-stream.cpp  | 231 ----------------------
 c_glib/arrow-glib/file-output-stream.h    |  72 -------
 c_glib/arrow-glib/file-output-stream.hpp  |  28 ---
 c_glib/arrow-glib/output-stream.cpp       | 201 ++++++++++++++++++-
 c_glib/arrow-glib/output-stream.h         |  52 +++++
 c_glib/arrow-glib/output-stream.hpp       |   5 +-
 c_glib/doc/reference/arrow-glib-docs.sgml |   3 +-
 10 files changed, 256 insertions(+), 341 deletions(-)
 delete mode 100644 c_glib/arrow-glib/file-output-stream.cpp
 delete mode 100644 c_glib/arrow-glib/file-output-stream.h
 delete mode 100644 c_glib/arrow-glib/file-output-stream.hpp

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index bbc11011474bc..54fb7f8c7a799 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -59,7 +59,6 @@ libarrow_glib_la_headers =			\
 libarrow_glib_la_headers +=			\
 	file.h					\
 	file-mode.h				\
-	file-output-stream.h			\
 	input-stream.h				\
 	memory-mapped-file.h			\
 	output-stream.h				\
@@ -102,7 +101,6 @@ libarrow_glib_la_sources =			\
 libarrow_glib_la_sources +=			\
 	file.cpp				\
 	file-mode.cpp				\
-	file-output-stream.cpp			\
 	input-stream.cpp			\
 	memory-mapped-file.cpp			\
 	output-stream.cpp			\
@@ -137,7 +135,6 @@ libarrow_glib_la_cpp_headers =			\
 libarrow_glib_la_cpp_headers +=			\
 	file.hpp				\
 	file-mode.hpp				\
-	file-output-stream.hpp			\
 	input-stream.hpp			\
 	memory-mapped-file.hpp			\
 	output-stream.hpp			\
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index efff5710308a8..e88b66b6ae9b2 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -35,7 +35,6 @@
 
 #include <arrow-glib/file.h>
 #include <arrow-glib/file-mode.h>
-#include <arrow-glib/file-output-stream.h>
 #include <arrow-glib/input-stream.h>
 #include <arrow-glib/memory-mapped-file.h>
 #include <arrow-glib/output-stream.h>
diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
index d6ef370095bdf..339773f651de1 100644
--- a/c_glib/arrow-glib/arrow-glib.hpp
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -40,7 +40,6 @@
 
 #include <arrow-glib/file.hpp>
 #include <arrow-glib/file-mode.hpp>
-#include <arrow-glib/file-output-stream.hpp>
 #include <arrow-glib/input-stream.hpp>
 #include <arrow-glib/memory-mapped-file.hpp>
 #include <arrow-glib/output-stream.hpp>
diff --git a/c_glib/arrow-glib/file-output-stream.cpp b/c_glib/arrow-glib/file-output-stream.cpp
deleted file mode 100644
index e1e1e27a06193..0000000000000
--- a/c_glib/arrow-glib/file-output-stream.cpp
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow/io/file.h>
-
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/file.hpp>
-#include <arrow-glib/file-output-stream.hpp>
-#include <arrow-glib/output-stream.hpp>
-#include <arrow-glib/writeable.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: file-output-stream
- * @short_description: A file output stream.
- *
- * The #GArrowFileOutputStream is a class for file output stream.
- */
-
-typedef struct GArrowFileOutputStreamPrivate_ {
-  std::shared_ptr<arrow::io::FileOutputStream> file_output_stream;
-} GArrowFileOutputStreamPrivate;
-
-enum {
-  PROP_0,
-  PROP_FILE_OUTPUT_STREAM
-};
-
-static std::shared_ptr<arrow::io::FileInterface>
-garrow_file_output_stream_get_raw_file_interface(GArrowFile *file)
-{
-  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(file);
-  auto arrow_file_output_stream =
-    garrow_file_output_stream_get_raw(file_output_stream);
-  return arrow_file_output_stream;
-}
-
-static void
-garrow_file_interface_init(GArrowFileInterface *iface)
-{
-  iface->get_raw = garrow_file_output_stream_get_raw_file_interface;
-}
-
-static std::shared_ptr<arrow::io::Writeable>
-garrow_file_output_stream_get_raw_writeable_interface(GArrowWriteable *writeable)
-{
-  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(writeable);
-  auto arrow_file_output_stream =
-    garrow_file_output_stream_get_raw(file_output_stream);
-  return arrow_file_output_stream;
-}
-
-static void
-garrow_writeable_interface_init(GArrowWriteableInterface *iface)
-{
-  iface->get_raw = garrow_file_output_stream_get_raw_writeable_interface;
-}
-
-static std::shared_ptr<arrow::io::OutputStream>
-garrow_file_output_stream_get_raw_output_stream_interface(GArrowOutputStream *output_stream)
-{
-  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(output_stream);
-  auto arrow_file_output_stream =
-    garrow_file_output_stream_get_raw(file_output_stream);
-  return arrow_file_output_stream;
-}
-
-static void
-garrow_output_stream_interface_init(GArrowOutputStreamInterface *iface)
-{
-  iface->get_raw = garrow_file_output_stream_get_raw_output_stream_interface;
-}
-
-G_DEFINE_TYPE_WITH_CODE(GArrowFileOutputStream,
-                        garrow_file_output_stream,
-                        G_TYPE_OBJECT,
-                        G_ADD_PRIVATE(GArrowFileOutputStream)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_FILE,
-                                              garrow_file_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE,
-                                              garrow_writeable_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_OUTPUT_STREAM,
-                                              garrow_output_stream_interface_init));
-
-#define GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(obj)                   \
-  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
-                               GARROW_TYPE_FILE_OUTPUT_STREAM,       \
-                               GArrowFileOutputStreamPrivate))
-
-static void
-garrow_file_output_stream_finalize(GObject *object)
-{
-  GArrowFileOutputStreamPrivate *priv;
-
-  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
-
-  priv->file_output_stream = nullptr;
-
-  G_OBJECT_CLASS(garrow_file_output_stream_parent_class)->finalize(object);
-}
-
-static void
-garrow_file_output_stream_set_property(GObject *object,
-                                          guint prop_id,
-                                          const GValue *value,
-                                          GParamSpec *pspec)
-{
-  GArrowFileOutputStreamPrivate *priv;
-
-  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
-
-  switch (prop_id) {
-  case PROP_FILE_OUTPUT_STREAM:
-    priv->file_output_stream =
-      *static_cast<std::shared_ptr<arrow::io::FileOutputStream> *>(g_value_get_pointer(value));
-    break;
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_file_output_stream_get_property(GObject *object,
-                                          guint prop_id,
-                                          GValue *value,
-                                          GParamSpec *pspec)
-{
-  switch (prop_id) {
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_file_output_stream_init(GArrowFileOutputStream *object)
-{
-}
-
-static void
-garrow_file_output_stream_class_init(GArrowFileOutputStreamClass *klass)
-{
-  GObjectClass *gobject_class;
-  GParamSpec *spec;
-
-  gobject_class = G_OBJECT_CLASS(klass);
-
-  gobject_class->finalize     = garrow_file_output_stream_finalize;
-  gobject_class->set_property = garrow_file_output_stream_set_property;
-  gobject_class->get_property = garrow_file_output_stream_get_property;
-
-  spec = g_param_spec_pointer("file-output-stream",
-                              "io::FileOutputStream",
-                              "The raw std::shared<arrow::io::FileOutputStream> *",
-                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
-                                                       G_PARAM_CONSTRUCT_ONLY));
-  g_object_class_install_property(gobject_class, PROP_FILE_OUTPUT_STREAM, spec);
-}
-
-/**
- * garrow_file_output_stream_open:
- * @path: The path of the file output stream.
- * @append: Whether the path is opened as append mode or recreate mode.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowFileOutputStream or %NULL on error.
- */
-GArrowFileOutputStream *
-garrow_file_output_stream_open(const gchar *path,
-                                  gboolean append,
-                                  GError **error)
-{
-  std::shared_ptr<arrow::io::FileOutputStream> arrow_file_output_stream;
-  auto status =
-    arrow::io::FileOutputStream::Open(std::string(path),
-                                      append,
-                                      &arrow_file_output_stream);
-  if (status.ok()) {
-    return garrow_file_output_stream_new_raw(&arrow_file_output_stream);
-  } else {
-    std::string context("[io][file-output-stream][open]: <");
-    context += path;
-    context += ">";
-    garrow_error_check(error, status, context.c_str());
-    return NULL;
-  }
-}
-
-G_END_DECLS
-
-GArrowFileOutputStream *
-garrow_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream)
-{
-  auto file_output_stream =
-    GARROW_FILE_OUTPUT_STREAM(g_object_new(GARROW_TYPE_FILE_OUTPUT_STREAM,
-                                              "file-output-stream", arrow_file_output_stream,
-                                              NULL));
-  return file_output_stream;
-}
-
-std::shared_ptr<arrow::io::FileOutputStream>
-garrow_file_output_stream_get_raw(GArrowFileOutputStream *file_output_stream)
-{
-  GArrowFileOutputStreamPrivate *priv;
-
-  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(file_output_stream);
-  return priv->file_output_stream;
-}
diff --git a/c_glib/arrow-glib/file-output-stream.h b/c_glib/arrow-glib/file-output-stream.h
deleted file mode 100644
index bef3700039921..0000000000000
--- a/c_glib/arrow-glib/file-output-stream.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <glib-object.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_FILE_OUTPUT_STREAM       \
-  (garrow_file_output_stream_get_type())
-#define GARROW_FILE_OUTPUT_STREAM(obj)                               \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
-                              GARROW_TYPE_FILE_OUTPUT_STREAM,        \
-                              GArrowFileOutputStream))
-#define GARROW_FILE_OUTPUT_STREAM_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_FILE_OUTPUT_STREAM,   \
-                           GArrowFileOutputStreamClass))
-#define GARROW_IS_FILE_OUTPUT_STREAM(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_FILE_OUTPUT_STREAM))
-#define GARROW_IS_FILE_OUTPUT_STREAM_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_FILE_OUTPUT_STREAM))
-#define GARROW_FILE_OUTPUT_STREAM_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_FILE_OUTPUT_STREAM, \
-                             GArrowFileOutputStreamClass))
-
-typedef struct _GArrowFileOutputStream         GArrowFileOutputStream;
-typedef struct _GArrowFileOutputStreamClass    GArrowFileOutputStreamClass;
-
-/**
- * GArrowFileOutputStream:
- *
- * It wraps `arrow::io::FileOutputStream`.
- */
-struct _GArrowFileOutputStream
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowFileOutputStreamClass
-{
-  GObjectClass parent_class;
-};
-
-GType garrow_file_output_stream_get_type(void) G_GNUC_CONST;
-
-GArrowFileOutputStream *garrow_file_output_stream_open(const gchar *path,
-                                                            gboolean append,
-                                                            GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/file-output-stream.hpp b/c_glib/arrow-glib/file-output-stream.hpp
deleted file mode 100644
index 0b10418cdf176..0000000000000
--- a/c_glib/arrow-glib/file-output-stream.hpp
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow/api.h>
-#include <arrow/io/file.h>
-
-#include <arrow-glib/file-output-stream.h>
-
-GArrowFileOutputStream *garrow_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream);
-std::shared_ptr<arrow::io::FileOutputStream> garrow_file_output_stream_get_raw(GArrowFileOutputStream *file_output_stream);
diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index bbc29b794f7c6..037814c1ffeb4 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -24,17 +24,22 @@
 #include <arrow/api.h>
 
 #include <arrow-glib/error.hpp>
+#include <arrow-glib/file.hpp>
 #include <arrow-glib/output-stream.hpp>
+#include <arrow-glib/writeable.hpp>
 
 G_BEGIN_DECLS
 
 /**
  * SECTION: output-stream
- * @title: GArrowOutputStream
- * @short_description: Stream output interface
+ * @section_id: output-stream-classes
+ * @title: Output stream classes
+ * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowOutputStream is an interface for stream output. Stream
  * output is file based and writeable
+ *
+ * #GArrowFileOutputStream is a class for file output stream.
  */
 
 G_DEFINE_INTERFACE(GArrowOutputStream,
@@ -46,6 +51,178 @@ garrow_output_stream_default_init (GArrowOutputStreamInterface *iface)
 {
 }
 
+
+typedef struct GArrowFileOutputStreamPrivate_ {
+  std::shared_ptr<arrow::io::FileOutputStream> file_output_stream;
+} GArrowFileOutputStreamPrivate;
+
+enum {
+  PROP_0,
+  PROP_FILE_OUTPUT_STREAM
+};
+
+static std::shared_ptr<arrow::io::FileInterface>
+garrow_file_output_stream_get_raw_file_interface(GArrowFile *file)
+{
+  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(file);
+  auto arrow_file_output_stream =
+    garrow_file_output_stream_get_raw(file_output_stream);
+  return arrow_file_output_stream;
+}
+
+static void
+garrow_file_interface_init(GArrowFileInterface *iface)
+{
+  iface->get_raw = garrow_file_output_stream_get_raw_file_interface;
+}
+
+static std::shared_ptr<arrow::io::Writeable>
+garrow_file_output_stream_get_raw_writeable_interface(GArrowWriteable *writeable)
+{
+  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(writeable);
+  auto arrow_file_output_stream =
+    garrow_file_output_stream_get_raw(file_output_stream);
+  return arrow_file_output_stream;
+}
+
+static void
+garrow_writeable_interface_init(GArrowWriteableInterface *iface)
+{
+  iface->get_raw = garrow_file_output_stream_get_raw_writeable_interface;
+}
+
+static std::shared_ptr<arrow::io::OutputStream>
+garrow_file_output_stream_get_raw_output_stream_interface(GArrowOutputStream *output_stream)
+{
+  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(output_stream);
+  auto arrow_file_output_stream =
+    garrow_file_output_stream_get_raw(file_output_stream);
+  return arrow_file_output_stream;
+}
+
+static void
+garrow_output_stream_interface_init(GArrowOutputStreamInterface *iface)
+{
+  iface->get_raw = garrow_file_output_stream_get_raw_output_stream_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(GArrowFileOutputStream,
+                        garrow_file_output_stream,
+                        G_TYPE_OBJECT,
+                        G_ADD_PRIVATE(GArrowFileOutputStream)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_FILE,
+                                              garrow_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE,
+                                              garrow_writeable_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_OUTPUT_STREAM,
+                                              garrow_output_stream_interface_init));
+
+#define GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(obj)              \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_TYPE_FILE_OUTPUT_STREAM,  \
+                               GArrowFileOutputStreamPrivate))
+
+static void
+garrow_file_output_stream_finalize(GObject *object)
+{
+  GArrowFileOutputStreamPrivate *priv;
+
+  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
+
+  priv->file_output_stream = nullptr;
+
+  G_OBJECT_CLASS(garrow_file_output_stream_parent_class)->finalize(object);
+}
+
+static void
+garrow_file_output_stream_set_property(GObject *object,
+                                          guint prop_id,
+                                          const GValue *value,
+                                          GParamSpec *pspec)
+{
+  GArrowFileOutputStreamPrivate *priv;
+
+  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_FILE_OUTPUT_STREAM:
+    priv->file_output_stream =
+      *static_cast<std::shared_ptr<arrow::io::FileOutputStream> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_file_output_stream_get_property(GObject *object,
+                                          guint prop_id,
+                                          GValue *value,
+                                          GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_file_output_stream_init(GArrowFileOutputStream *object)
+{
+}
+
+static void
+garrow_file_output_stream_class_init(GArrowFileOutputStreamClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_file_output_stream_finalize;
+  gobject_class->set_property = garrow_file_output_stream_set_property;
+  gobject_class->get_property = garrow_file_output_stream_get_property;
+
+  spec = g_param_spec_pointer("file-output-stream",
+                              "io::FileOutputStream",
+                              "The raw std::shared<arrow::io::FileOutputStream> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_FILE_OUTPUT_STREAM, spec);
+}
+
+/**
+ * garrow_file_output_stream_open:
+ * @path: The path of the file output stream.
+ * @append: Whether the path is opened as append mode or recreate mode.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): A newly opened
+ *   #GArrowFileOutputStream or %NULL on error.
+ */
+GArrowFileOutputStream *
+garrow_file_output_stream_open(const gchar *path,
+                                  gboolean append,
+                                  GError **error)
+{
+  std::shared_ptr<arrow::io::FileOutputStream> arrow_file_output_stream;
+  auto status =
+    arrow::io::FileOutputStream::Open(std::string(path),
+                                      append,
+                                      &arrow_file_output_stream);
+  if (status.ok()) {
+    return garrow_file_output_stream_new_raw(&arrow_file_output_stream);
+  } else {
+    std::string context("[io][file-output-stream][open]: <");
+    context += path;
+    context += ">";
+    garrow_error_check(error, status, context.c_str());
+    return NULL;
+  }
+}
+
 G_END_DECLS
 
 std::shared_ptr<arrow::io::OutputStream>
@@ -54,3 +231,23 @@ garrow_output_stream_get_raw(GArrowOutputStream *output_stream)
   auto *iface = GARROW_OUTPUT_STREAM_GET_IFACE(output_stream);
   return iface->get_raw(output_stream);
 }
+
+
+GArrowFileOutputStream *
+garrow_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream)
+{
+  auto file_output_stream =
+    GARROW_FILE_OUTPUT_STREAM(g_object_new(GARROW_TYPE_FILE_OUTPUT_STREAM,
+                                           "file-output-stream", arrow_file_output_stream,
+                                           NULL));
+  return file_output_stream;
+}
+
+std::shared_ptr<arrow::io::FileOutputStream>
+garrow_file_output_stream_get_raw(GArrowFileOutputStream *file_output_stream)
+{
+  GArrowFileOutputStreamPrivate *priv;
+
+  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(file_output_stream);
+  return priv->file_output_stream;
+}
diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h
index 3481072c27d8b..043832efccd78 100644
--- a/c_glib/arrow-glib/output-stream.h
+++ b/c_glib/arrow-glib/output-stream.h
@@ -42,4 +42,56 @@ typedef struct _GArrowOutputStreamInterface GArrowOutputStreamInterface;
 
 GType garrow_output_stream_get_type(void) G_GNUC_CONST;
 
+
+#define GARROW_TYPE_FILE_OUTPUT_STREAM          \
+  (garrow_file_output_stream_get_type())
+#define GARROW_FILE_OUTPUT_STREAM(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_FILE_OUTPUT_STREAM,   \
+                              GArrowFileOutputStream))
+#define GARROW_FILE_OUTPUT_STREAM_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_FILE_OUTPUT_STREAM,      \
+                           GArrowFileOutputStreamClass))
+#define GARROW_IS_FILE_OUTPUT_STREAM(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_FILE_OUTPUT_STREAM))
+#define GARROW_IS_FILE_OUTPUT_STREAM_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_FILE_OUTPUT_STREAM))
+#define GARROW_FILE_OUTPUT_STREAM_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_FILE_OUTPUT_STREAM,    \
+                             GArrowFileOutputStreamClass))
+
+typedef struct _GArrowFileOutputStream         GArrowFileOutputStream;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowFileOutputStreamClass    GArrowFileOutputStreamClass;
+#endif
+
+/**
+ * GArrowFileOutputStream:
+ *
+ * It wraps `arrow::io::FileOutputStream`.
+ */
+struct _GArrowFileOutputStream
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowFileOutputStreamClass
+{
+  GObjectClass parent_class;
+};
+#endif
+
+GType garrow_file_output_stream_get_type(void) G_GNUC_CONST;
+
+GArrowFileOutputStream *garrow_file_output_stream_open(const gchar *path,
+                                                       gboolean append,
+                                                       GError **error);
+
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/output-stream.hpp b/c_glib/arrow-glib/output-stream.hpp
index 635da10e24766..e8e73216c499b 100644
--- a/c_glib/arrow-glib/output-stream.hpp
+++ b/c_glib/arrow-glib/output-stream.hpp
@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <arrow/io/interfaces.h>
+#include <arrow/io/file.h>
 
 #include <arrow-glib/output-stream.h>
 
@@ -36,3 +36,6 @@ struct _GArrowOutputStreamInterface
 };
 
 std::shared_ptr<arrow::io::OutputStream> garrow_output_stream_get_raw(GArrowOutputStream *output_stream);
+
+GArrowFileOutputStream *garrow_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream);
+std::shared_ptr<arrow::io::FileOutputStream> garrow_file_output_stream_get_raw(GArrowFileOutputStream *file_output_stream);
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 75e4a0a37286f..7ba37b45068e0 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -87,9 +87,8 @@
     <chapter id="output">
       <title>Output</title>
       <xi:include href="xml/writeable.xml"/>
-      <xi:include href="xml/output-stream.xml"/>
       <xi:include href="xml/writeable-file.xml"/>
-      <xi:include href="xml/file-output-stream.xml"/>
+      <xi:include href="xml/output-stream.xml"/>
     </chapter>
     <chapter id="input-output">
       <title>Input and output</title>

From f13a9286c1444391c04fc0d20909a672122d10c1 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 27 Apr 2017 21:37:12 -0400
Subject: [PATCH 0579/1644] ARROW-907: C++: Construct Table from schema and
 arrays

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #610 from xhochy/ARROW-907 and squashes the following commits:

b8ee8dc [Uwe L. Korn] Fix signed comparison
25518b3 [Uwe L. Korn] ARROW-907: C++: Construct Table from schema and arrays
---
 cpp/src/arrow/table-test.cc |  5 +++++
 cpp/src/arrow/table.cc      | 21 +++++++++++++++++++++
 cpp/src/arrow/table.h       |  3 +++
 3 files changed, 29 insertions(+)

diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 0da4c0f9641a3..e46fdc77cf761 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -233,6 +233,11 @@ TEST_F(TestTable, Ctors) {
   table_.reset(new Table(schema_, columns_, length));
   ASSERT_OK(table_->ValidateColumns());
   ASSERT_EQ(length, table_->num_rows());
+
+  ASSERT_OK(MakeTable(schema_, arrays_, &table_));
+  ASSERT_OK(table_->ValidateColumns());
+  ASSERT_EQ(length, table_->num_rows());
+  ASSERT_EQ(3, table_->num_columns());
 }
 
 TEST_F(TestTable, Metadata) {
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index db17da72a6a33..c110ec16a0494 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -366,4 +366,25 @@ Status Table::ValidateColumns() const {
   return Status::OK();
 }
 
+Status ARROW_EXPORT MakeTable(const std::shared_ptr<Schema>& schema,
+    const std::vector<std::shared_ptr<Array>>& arrays, std::shared_ptr<Table>* table) {
+  // Make sure the length of the schema corresponds to the length of the vector
+  if (schema->num_fields() != static_cast<int>(arrays.size())) {
+    std::stringstream ss;
+    ss << "Schema and Array vector have different lengths: " << schema->num_fields()
+       << " != " << arrays.size();
+    return Status::Invalid(ss.str());
+  }
+
+  std::vector<std::shared_ptr<Column>> columns;
+  columns.reserve(schema->num_fields());
+  for (int i = 0; i < schema->num_fields(); i++) {
+    columns.emplace_back(std::make_shared<Column>(schema->field(i), arrays[i]));
+  }
+
+  *table = std::make_shared<Table>(schema, columns);
+
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index efc2077bd009a..67710a8216010 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -208,6 +208,9 @@ class ARROW_EXPORT Table {
 Status ARROW_EXPORT ConcatenateTables(
     const std::vector<std::shared_ptr<Table>>& tables, std::shared_ptr<Table>* table);
 
+Status ARROW_EXPORT MakeTable(const std::shared_ptr<Schema>& schema,
+    const std::vector<std::shared_ptr<Array>>& arrays, std::shared_ptr<Table>* table);
+
 }  // namespace arrow
 
 #endif  // ARROW_TABLE_H

From f7ab7270bb07466dabf84c015a6db2a192eb3dad Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 27 Apr 2017 21:38:23 -0400
Subject: [PATCH 0580/1644] ARROW-896: Support Jupyter Notebook in Web site

We can embed a Jupyter Notebook (`getting_started.ipynb`) in the same directory by the following code:

```markdown
{::nomarkdown}
{% jupyter_notebook getting_started.ipynb %}
{:/nomarkdown}
```

Author: Kouhei Sutou <kou@clear-code.com>

Closes #608 from kou/site-support-jupyter-notebook and squashes the following commits:

12186b1 [Kouhei Sutou] Support Jupyter Notebook in Web site
---
 site/Gemfile     | 1 +
 site/_config.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/site/Gemfile b/site/Gemfile
index 98decaf35dbe6..e6691a0857140 100644
--- a/site/Gemfile
+++ b/site/Gemfile
@@ -21,5 +21,6 @@ gem 'jekyll-bootstrap-sass'
 gem 'github-pages'
 group :jekyll_plugins do
    gem "jekyll-feed", "~> 0.6"
+   gem "jekyll-jupyter-notebook"
 end
 gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
diff --git a/site/_config.yml b/site/_config.yml
index 922af4a08059c..346565e6d5cca 100644
--- a/site/_config.yml
+++ b/site/_config.yml
@@ -38,6 +38,7 @@ baseurl:
 gems:
   - jekyll-feed
   - jekyll-bootstrap-sass
+  - jekyll-jupyter-notebook
 
 bootstrap:
   assets: true

From 53c093b521d87794cba066032e827788c75d42fe Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sat, 29 Apr 2017 20:53:19 -0400
Subject: [PATCH 0581/1644] ARROW-914 [C++/Python] Fix Decimal ToBytes

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #613 from cpcloud/ARROW-914 and squashes the following commits:

b0f3c10 [Phillip Cloud] Use a more appropriate name
418fc9c [Phillip Cloud] ARROW-914 [C++/Python] Fix Decimal ToBytes
---
 ci/msvc-build.bat             | 3 +--
 cpp/src/arrow/util/decimal.cc | 8 ++++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 08c5033849539..aca1f8cc3c073 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -53,5 +53,4 @@ cd ..\..\python
 python setup.py build_ext --inplace  || exit /B
 python -c "import pyarrow"  || exit /B
 
-@rem TODO: re-enable when last tests are fixed
-@rem py.test pyarrow -v -s || exit /B
+py.test pyarrow -v -s || exit /B
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 2fe9da4aba9a2..3d9fbd31bf22a 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -147,7 +147,7 @@ void FromBytes(const uint8_t* bytes, Decimal64* decimal) {
 constexpr static const size_t BYTES_IN_128_BITS = 128 / CHAR_BIT;
 constexpr static const size_t LIMB_SIZE =
     sizeof(std::remove_pointer<int128_t::backend_type::limb_pointer>::type);
-constexpr static const size_t BYTES_PER_LIMB = BYTES_IN_128_BITS / LIMB_SIZE;
+constexpr static const size_t LIMBS_IN_INT128 = BYTES_IN_128_BITS / LIMB_SIZE;
 
 void FromBytes(const uint8_t* bytes, bool is_negative, Decimal128* decimal) {
   DCHECK_NE(bytes, nullptr);
@@ -155,7 +155,7 @@ void FromBytes(const uint8_t* bytes, bool is_negative, Decimal128* decimal) {
 
   auto& decimal_value(decimal->value);
   int128_t::backend_type& backend(decimal_value.backend());
-  backend.resize(BYTES_PER_LIMB, BYTES_PER_LIMB);
+  backend.resize(LIMBS_IN_INT128, LIMBS_IN_INT128);
   std::memcpy(backend.limbs(), bytes, BYTES_IN_128_BITS);
   if (is_negative) { decimal->value = -decimal->value; }
 }
@@ -177,8 +177,8 @@ void ToBytes(const Decimal128& decimal, uint8_t** bytes, bool* is_negative) {
   /// TODO(phillipc): boost multiprecision is unreliable here, int128_t can't be
   /// roundtripped
   const auto& backend(decimal.value.backend());
-  auto boost_bytes = reinterpret_cast<const uint8_t*>(backend.limbs());
-  std::memcpy(*bytes, boost_bytes, BYTES_IN_128_BITS);
+  const size_t bytes_in_use = LIMB_SIZE * backend.size();
+  std::memcpy(*bytes, backend.limbs(), bytes_in_use);
   *is_negative = backend.isneg();
 }
 

From ed5a1d4f9ae13c0474418d2b0534cbacdca57ef8 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 30 Apr 2017 13:08:13 -0400
Subject: [PATCH 0582/1644] ARROW-916: [GLib] Add GArrowBufferOutputStream

Author: Kouhei Sutou <kou@clear-code.com>

Closes #616 from kou/glib-buffer-output-stream and squashes the following commits:

75c89cb [Kouhei Sutou] [GLib] Add GArrowBufferOutputStream
---
 c_glib/arrow-glib/output-stream.cpp    | 204 ++++++++++++++-----------
 c_glib/arrow-glib/output-stream.h      |  99 ++++++++++--
 c_glib/arrow-glib/output-stream.hpp    |  17 +--
 c_glib/test/test-buffer-output-file.rb |  26 ++++
 4 files changed, 237 insertions(+), 109 deletions(-)
 create mode 100644 c_glib/test/test-buffer-output-file.rb

diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index 037814c1ffeb4..b757d44cef44e 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -22,7 +22,9 @@
 #endif
 
 #include <arrow/api.h>
+#include <arrow/io/memory.h>
 
+#include <arrow-glib/buffer.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/file.hpp>
 #include <arrow-glib/output-stream.hpp>
@@ -40,114 +42,87 @@ G_BEGIN_DECLS
  * output is file based and writeable
  *
  * #GArrowFileOutputStream is a class for file output stream.
+ *
+ * #GArrowBufferOutputStream is a class for buffer output stream.
  */
 
-G_DEFINE_INTERFACE(GArrowOutputStream,
-                   garrow_output_stream,
-                   G_TYPE_OBJECT)
-
-static void
-garrow_output_stream_default_init (GArrowOutputStreamInterface *iface)
-{
-}
-
-
-typedef struct GArrowFileOutputStreamPrivate_ {
-  std::shared_ptr<arrow::io::FileOutputStream> file_output_stream;
-} GArrowFileOutputStreamPrivate;
+typedef struct GArrowOutputStreamPrivate_ {
+  std::shared_ptr<arrow::io::OutputStream> output_stream;
+} GArrowOutputStreamPrivate;
 
 enum {
   PROP_0,
-  PROP_FILE_OUTPUT_STREAM
+  PROP_OUTPUT_STREAM
 };
 
 static std::shared_ptr<arrow::io::FileInterface>
-garrow_file_output_stream_get_raw_file_interface(GArrowFile *file)
+garrow_output_stream_get_raw_file_interface(GArrowFile *file)
 {
-  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(file);
-  auto arrow_file_output_stream =
-    garrow_file_output_stream_get_raw(file_output_stream);
-  return arrow_file_output_stream;
+  auto output_stream = GARROW_OUTPUT_STREAM(file);
+  auto arrow_output_stream = garrow_output_stream_get_raw(output_stream);
+  return arrow_output_stream;
 }
 
 static void
-garrow_file_interface_init(GArrowFileInterface *iface)
+garrow_output_stream_file_interface_init(GArrowFileInterface *iface)
 {
-  iface->get_raw = garrow_file_output_stream_get_raw_file_interface;
+  iface->get_raw = garrow_output_stream_get_raw_file_interface;
 }
 
 static std::shared_ptr<arrow::io::Writeable>
-garrow_file_output_stream_get_raw_writeable_interface(GArrowWriteable *writeable)
+garrow_output_stream_get_raw_writeable_interface(GArrowWriteable *writeable)
 {
-  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(writeable);
-  auto arrow_file_output_stream =
-    garrow_file_output_stream_get_raw(file_output_stream);
-  return arrow_file_output_stream;
+  auto output_stream = GARROW_OUTPUT_STREAM(writeable);
+  auto arrow_output_stream = garrow_output_stream_get_raw(output_stream);
+  return arrow_output_stream;
 }
 
 static void
-garrow_writeable_interface_init(GArrowWriteableInterface *iface)
-{
-  iface->get_raw = garrow_file_output_stream_get_raw_writeable_interface;
-}
-
-static std::shared_ptr<arrow::io::OutputStream>
-garrow_file_output_stream_get_raw_output_stream_interface(GArrowOutputStream *output_stream)
+garrow_output_stream_writeable_interface_init(GArrowWriteableInterface *iface)
 {
-  auto file_output_stream = GARROW_FILE_OUTPUT_STREAM(output_stream);
-  auto arrow_file_output_stream =
-    garrow_file_output_stream_get_raw(file_output_stream);
-  return arrow_file_output_stream;
+  iface->get_raw = garrow_output_stream_get_raw_writeable_interface;
 }
 
-static void
-garrow_output_stream_interface_init(GArrowOutputStreamInterface *iface)
-{
-  iface->get_raw = garrow_file_output_stream_get_raw_output_stream_interface;
-}
-
-G_DEFINE_TYPE_WITH_CODE(GArrowFileOutputStream,
-                        garrow_file_output_stream,
+G_DEFINE_TYPE_WITH_CODE(GArrowOutputStream,
+                        garrow_output_stream,
                         G_TYPE_OBJECT,
-                        G_ADD_PRIVATE(GArrowFileOutputStream)
+                        G_ADD_PRIVATE(GArrowOutputStream)
                         G_IMPLEMENT_INTERFACE(GARROW_TYPE_FILE,
-                                              garrow_file_interface_init)
+                                              garrow_output_stream_file_interface_init)
                         G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE,
-                                              garrow_writeable_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_OUTPUT_STREAM,
-                                              garrow_output_stream_interface_init));
+                                              garrow_output_stream_writeable_interface_init));
 
-#define GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(obj)              \
+#define GARROW_OUTPUT_STREAM_GET_PRIVATE(obj)                   \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
-                               GARROW_TYPE_FILE_OUTPUT_STREAM,  \
-                               GArrowFileOutputStreamPrivate))
+                               GARROW_TYPE_OUTPUT_STREAM,       \
+                               GArrowOutputStreamPrivate))
 
 static void
-garrow_file_output_stream_finalize(GObject *object)
+garrow_output_stream_finalize(GObject *object)
 {
-  GArrowFileOutputStreamPrivate *priv;
+  GArrowOutputStreamPrivate *priv;
 
-  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
+  priv = GARROW_OUTPUT_STREAM_GET_PRIVATE(object);
 
-  priv->file_output_stream = nullptr;
+  priv->output_stream = nullptr;
 
-  G_OBJECT_CLASS(garrow_file_output_stream_parent_class)->finalize(object);
+  G_OBJECT_CLASS(garrow_output_stream_parent_class)->finalize(object);
 }
 
 static void
-garrow_file_output_stream_set_property(GObject *object,
+garrow_output_stream_set_property(GObject *object,
                                           guint prop_id,
                                           const GValue *value,
                                           GParamSpec *pspec)
 {
-  GArrowFileOutputStreamPrivate *priv;
+  GArrowOutputStreamPrivate *priv;
 
-  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(object);
+  priv = GARROW_OUTPUT_STREAM_GET_PRIVATE(object);
 
   switch (prop_id) {
-  case PROP_FILE_OUTPUT_STREAM:
-    priv->file_output_stream =
-      *static_cast<std::shared_ptr<arrow::io::FileOutputStream> *>(g_value_get_pointer(value));
+  case PROP_OUTPUT_STREAM:
+    priv->output_stream =
+      *static_cast<std::shared_ptr<arrow::io::OutputStream> *>(g_value_get_pointer(value));
     break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
@@ -156,7 +131,7 @@ garrow_file_output_stream_set_property(GObject *object,
 }
 
 static void
-garrow_file_output_stream_get_property(GObject *object,
+garrow_output_stream_get_property(GObject *object,
                                           guint prop_id,
                                           GValue *value,
                                           GParamSpec *pspec)
@@ -169,28 +144,43 @@ garrow_file_output_stream_get_property(GObject *object,
 }
 
 static void
-garrow_file_output_stream_init(GArrowFileOutputStream *object)
+garrow_output_stream_init(GArrowOutputStream *object)
 {
 }
 
 static void
-garrow_file_output_stream_class_init(GArrowFileOutputStreamClass *klass)
+garrow_output_stream_class_init(GArrowOutputStreamClass *klass)
 {
   GObjectClass *gobject_class;
   GParamSpec *spec;
 
   gobject_class = G_OBJECT_CLASS(klass);
 
-  gobject_class->finalize     = garrow_file_output_stream_finalize;
-  gobject_class->set_property = garrow_file_output_stream_set_property;
-  gobject_class->get_property = garrow_file_output_stream_get_property;
+  gobject_class->finalize     = garrow_output_stream_finalize;
+  gobject_class->set_property = garrow_output_stream_set_property;
+  gobject_class->get_property = garrow_output_stream_get_property;
 
-  spec = g_param_spec_pointer("file-output-stream",
-                              "io::FileOutputStream",
-                              "The raw std::shared<arrow::io::FileOutputStream> *",
+  spec = g_param_spec_pointer("output-stream",
+                              "io::OutputStream",
+                              "The raw std::shared<arrow::io::OutputStream> *",
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
-  g_object_class_install_property(gobject_class, PROP_FILE_OUTPUT_STREAM, spec);
+  g_object_class_install_property(gobject_class, PROP_OUTPUT_STREAM, spec);
+}
+
+
+G_DEFINE_TYPE(GArrowFileOutputStream,
+              garrow_file_output_stream,
+              GARROW_TYPE_OUTPUT_STREAM);
+
+static void
+garrow_file_output_stream_init(GArrowFileOutputStream *file_output_stream)
+{
+}
+
+static void
+garrow_file_output_stream_class_init(GArrowFileOutputStreamClass *klass)
+{
 }
 
 /**
@@ -204,8 +194,8 @@ garrow_file_output_stream_class_init(GArrowFileOutputStreamClass *klass)
  */
 GArrowFileOutputStream *
 garrow_file_output_stream_open(const gchar *path,
-                                  gboolean append,
-                                  GError **error)
+                               gboolean append,
+                               GError **error)
 {
   std::shared_ptr<arrow::io::FileOutputStream> arrow_file_output_stream;
   auto status =
@@ -223,13 +213,56 @@ garrow_file_output_stream_open(const gchar *path,
   }
 }
 
+
+G_DEFINE_TYPE(GArrowBufferOutputStream,
+              garrow_buffer_output_stream,
+              GARROW_TYPE_OUTPUT_STREAM);
+
+static void
+garrow_buffer_output_stream_init(GArrowBufferOutputStream *buffer_output_stream)
+{
+}
+
+static void
+garrow_buffer_output_stream_class_init(GArrowBufferOutputStreamClass *klass)
+{
+}
+
+/**
+ * garrow_buffer_output_stream_new:
+ * @buffer: The resizable buffer to be output.
+ *
+ * Returns: (transfer full): A newly created #GArrowBufferOutputStream.
+ */
+GArrowBufferOutputStream *
+garrow_buffer_output_stream_new(GArrowResizableBuffer *buffer)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer));
+  auto arrow_resizable_buffer =
+    std::static_pointer_cast<arrow::ResizableBuffer>(arrow_buffer);
+  auto arrow_buffer_output_stream =
+    std::make_shared<arrow::io::BufferOutputStream>(arrow_resizable_buffer);
+  return garrow_buffer_output_stream_new_raw(&arrow_buffer_output_stream);
+}
 G_END_DECLS
 
+GArrowOutputStream *
+garrow_output_stream_new_raw(std::shared_ptr<arrow::io::OutputStream> *arrow_output_stream)
+{
+  auto output_stream =
+    GARROW_OUTPUT_STREAM(g_object_new(GARROW_TYPE_OUTPUT_STREAM,
+                                      "output-stream", arrow_output_stream,
+                                      NULL));
+  return output_stream;
+}
+
 std::shared_ptr<arrow::io::OutputStream>
 garrow_output_stream_get_raw(GArrowOutputStream *output_stream)
 {
-  auto *iface = GARROW_OUTPUT_STREAM_GET_IFACE(output_stream);
-  return iface->get_raw(output_stream);
+  GArrowOutputStreamPrivate *priv;
+
+  priv = GARROW_OUTPUT_STREAM_GET_PRIVATE(output_stream);
+  return priv->output_stream;
 }
 
 
@@ -238,16 +271,17 @@ garrow_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *
 {
   auto file_output_stream =
     GARROW_FILE_OUTPUT_STREAM(g_object_new(GARROW_TYPE_FILE_OUTPUT_STREAM,
-                                           "file-output-stream", arrow_file_output_stream,
+                                           "output-stream", arrow_file_output_stream,
                                            NULL));
   return file_output_stream;
 }
 
-std::shared_ptr<arrow::io::FileOutputStream>
-garrow_file_output_stream_get_raw(GArrowFileOutputStream *file_output_stream)
+GArrowBufferOutputStream *
+garrow_buffer_output_stream_new_raw(std::shared_ptr<arrow::io::BufferOutputStream> *arrow_buffer_output_stream)
 {
-  GArrowFileOutputStreamPrivate *priv;
-
-  priv = GARROW_FILE_OUTPUT_STREAM_GET_PRIVATE(file_output_stream);
-  return priv->file_output_stream;
+  auto buffer_output_stream =
+    GARROW_BUFFER_OUTPUT_STREAM(g_object_new(GARROW_TYPE_BUFFER_OUTPUT_STREAM,
+                                             "output-stream", arrow_buffer_output_stream,
+                                             NULL));
+  return buffer_output_stream;
 }
diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h
index 043832efccd78..2a14a24ea9051 100644
--- a/c_glib/arrow-glib/output-stream.h
+++ b/c_glib/arrow-glib/output-stream.h
@@ -21,24 +21,53 @@
 
 #include <glib-object.h>
 
+#include <arrow-glib/buffer.h>
+
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_OUTPUT_STREAM            \
+#define GARROW_TYPE_OUTPUT_STREAM               \
   (garrow_output_stream_get_type())
-#define GARROW_OUTPUT_STREAM(obj)                            \
+#define GARROW_OUTPUT_STREAM(obj)                               \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_OUTPUT_STREAM,     \
+                              GARROW_TYPE_OUTPUT_STREAM,        \
                               GArrowOutputStream))
-#define GARROW_IS_OUTPUT_STREAM(obj)                         \
+#define GARROW_OUTPUT_STREAM_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_OUTPUT_STREAM,   \
+                           GArrowOutputStreamClass))
+#define GARROW_IS_OUTPUT_STREAM(obj)                            \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
                               GARROW_TYPE_OUTPUT_STREAM))
-#define GARROW_OUTPUT_STREAM_GET_IFACE(obj)                          \
-  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
-                                 GARROW_TYPE_OUTPUT_STREAM,          \
-                                 GArrowOutputStreamInterface))
+#define GARROW_IS_OUTPUT_STREAM_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_OUTPUT_STREAM))
+#define GARROW_OUTPUT_STREAM_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_OUTPUT_STREAM, \
+                             GArrowOutputStreamClass))
 
 typedef struct _GArrowOutputStream          GArrowOutputStream;
-typedef struct _GArrowOutputStreamInterface GArrowOutputStreamInterface;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowOutputStreamClass     GArrowOutputStreamClass;
+#endif
+
+/**
+ * GArrowOutputStream:
+ *
+ * It wraps `arrow::io::OutputStream`.
+ */
+struct _GArrowOutputStream
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowOutputStreamClass
+{
+  GObjectClass parent_class;
+};
+#endif
 
 GType garrow_output_stream_get_type(void) G_GNUC_CONST;
 
@@ -77,13 +106,13 @@ typedef struct _GArrowFileOutputStreamClass    GArrowFileOutputStreamClass;
 struct _GArrowFileOutputStream
 {
   /*< private >*/
-  GObject parent_instance;
+  GArrowOutputStream parent_instance;
 };
 
 #ifndef __GTK_DOC_IGNORE__
 struct _GArrowFileOutputStreamClass
 {
-  GObjectClass parent_class;
+  GArrowOutputStreamClass parent_class;
 };
 #endif
 
@@ -94,4 +123,52 @@ GArrowFileOutputStream *garrow_file_output_stream_open(const gchar *path,
                                                        GError **error);
 
 
+#define GARROW_TYPE_BUFFER_OUTPUT_STREAM        \
+  (garrow_buffer_output_stream_get_type())
+#define GARROW_BUFFER_OUTPUT_STREAM(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BUFFER_OUTPUT_STREAM, \
+                              GArrowBufferOutputStream))
+#define GARROW_BUFFER_OUTPUT_STREAM_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BUFFER_OUTPUT_STREAM,    \
+                           GArrowBufferOutputStreamClass))
+#define GARROW_IS_BUFFER_OUTPUT_STREAM(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_BUFFER_OUTPUT_STREAM))
+#define GARROW_IS_BUFFER_OUTPUT_STREAM_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BUFFER_OUTPUT_STREAM))
+#define GARROW_BUFFER_OUTPUT_STREAM_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BUFFER_OUTPUT_STREAM,  \
+                             GArrowBufferOutputStreamClass))
+
+typedef struct _GArrowBufferOutputStream         GArrowBufferOutputStream;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowBufferOutputStreamClass    GArrowBufferOutputStreamClass;
+#endif
+
+/**
+ * GArrowBufferOutputStream:
+ *
+ * It wraps `arrow::io::BufferOutputStream`.
+ */
+struct _GArrowBufferOutputStream
+{
+  /*< private >*/
+  GArrowOutputStream parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowBufferOutputStreamClass
+{
+  GArrowOutputStreamClass parent_class;
+};
+#endif
+
+GType garrow_buffer_output_stream_get_type(void) G_GNUC_CONST;
+
+GArrowBufferOutputStream *garrow_buffer_output_stream_new(GArrowResizableBuffer *buffer);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/output-stream.hpp b/c_glib/arrow-glib/output-stream.hpp
index e8e73216c499b..5d22f1d2e7026 100644
--- a/c_glib/arrow-glib/output-stream.hpp
+++ b/c_glib/arrow-glib/output-stream.hpp
@@ -20,22 +20,13 @@
 #pragma once
 
 #include <arrow/io/file.h>
+#include <arrow/io/memory.h>
 
 #include <arrow-glib/output-stream.h>
 
-/**
- * GArrowOutputStreamInterface:
- *
- * It wraps `arrow::io::OutputStream`.
- */
-struct _GArrowOutputStreamInterface
-{
-  GTypeInterface parent_iface;
-
-  std::shared_ptr<arrow::io::OutputStream> (*get_raw)(GArrowOutputStream *file);
-};
-
+GArrowOutputStream *garrow_output_stream_new_raw(std::shared_ptr<arrow::io::OutputStream> *arrow_output_stream);
 std::shared_ptr<arrow::io::OutputStream> garrow_output_stream_get_raw(GArrowOutputStream *output_stream);
 
+
 GArrowFileOutputStream *garrow_file_output_stream_new_raw(std::shared_ptr<arrow::io::FileOutputStream> *arrow_file_output_stream);
-std::shared_ptr<arrow::io::FileOutputStream> garrow_file_output_stream_get_raw(GArrowFileOutputStream *file_output_stream);
+GArrowBufferOutputStream *garrow_buffer_output_stream_new_raw(std::shared_ptr<arrow::io::BufferOutputStream> *arrow_buffer_output_stream);
diff --git a/c_glib/test/test-buffer-output-file.rb b/c_glib/test/test-buffer-output-file.rb
new file mode 100644
index 0000000000000..1b7fae9e6f6d4
--- /dev/null
+++ b/c_glib/test/test-buffer-output-file.rb
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBufferOutputStream < Test::Unit::TestCase
+  def test_new
+    buffer = Arrow::PoolBuffer.new
+    output_stream = Arrow::BufferOutputStream.new(buffer)
+    output_stream.write("Hello")
+    output_stream.close
+    assert_equal("Hello", buffer.data.to_s)
+  end
+end

From ce0c96221f0db74b51af5484bd39f0619b71e58f Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 30 Apr 2017 13:19:22 -0400
Subject: [PATCH 0583/1644] ARROW-917: [GLib] Add GArrowBufferReader

Author: Kouhei Sutou <kou@clear-code.com>

Closes #617 from kou/glib-buffer-reader and squashes the following commits:

399acda [Kouhei Sutou] [GLib] Add GArrowBufferReader
---
 c_glib/arrow-glib/input-stream.cpp       | 232 +++++++++++++++++++++--
 c_glib/arrow-glib/input-stream.h         | 102 ++++++++--
 c_glib/arrow-glib/input-stream.hpp       |  16 +-
 c_glib/arrow-glib/memory-mapped-file.cpp |  17 --
 c_glib/test/test-buffer-reader.rb        |  26 +++
 5 files changed, 341 insertions(+), 52 deletions(-)
 create mode 100644 c_glib/test/test-buffer-reader.rb

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 36bef80422489..56b811ad1c368 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -21,36 +21,246 @@
 #  include <config.h>
 #endif
 
-#include <arrow/api.h>
+#include <arrow/io/interfaces.h>
+#include <arrow/io/memory.h>
 
+#include <arrow-glib/buffer.hpp>
 #include <arrow-glib/error.hpp>
+#include <arrow-glib/file.hpp>
 #include <arrow-glib/input-stream.hpp>
+#include <arrow-glib/random-access-file.hpp>
+#include <arrow-glib/readable.hpp>
 
 G_BEGIN_DECLS
 
 /**
  * SECTION: input-stream
- * @title: GArrowInputStream
- * @short_description: Stream input interface
+ * @section_id: input-stream-classes
+ * @title: Input stream classes
+ * @include: arrow-glib/arrow-glib.h
  *
- * #GArrowInputStream is an interface for stream input. Stream input
- * is file based and readable.
+ * #GArrowInputStream is a base class for input stream.
+ *
+ * #GArrowBufferReader is a class for buffer input stream.
  */
 
-G_DEFINE_INTERFACE(GArrowInputStream,
-                   garrow_input_stream,
-                   G_TYPE_OBJECT)
+typedef struct GArrowInputStreamPrivate_ {
+  std::shared_ptr<arrow::io::InputStream> input_stream;
+} GArrowInputStreamPrivate;
+
+enum {
+  PROP_0,
+  PROP_INPUT_STREAM
+};
+
+static std::shared_ptr<arrow::io::FileInterface>
+garrow_input_stream_get_raw_file_interface(GArrowFile *file)
+{
+  auto input_stream = GARROW_INPUT_STREAM(file);
+  auto arrow_input_stream =
+    garrow_input_stream_get_raw(input_stream);
+  return arrow_input_stream;
+}
+
+static void
+garrow_input_stream_file_interface_init(GArrowFileInterface *iface)
+{
+  iface->get_raw = garrow_input_stream_get_raw_file_interface;
+}
+
+static std::shared_ptr<arrow::io::Readable>
+garrow_input_stream_get_raw_readable_interface(GArrowReadable *readable)
+{
+  auto input_stream = GARROW_INPUT_STREAM(readable);
+  auto arrow_input_stream = garrow_input_stream_get_raw(input_stream);
+  return arrow_input_stream;
+}
+
+static void
+garrow_input_stream_readable_interface_init(GArrowReadableInterface *iface)
+{
+  iface->get_raw = garrow_input_stream_get_raw_readable_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(GArrowInputStream,
+                        garrow_input_stream,
+                        G_TYPE_OBJECT,
+                        G_ADD_PRIVATE(GArrowInputStream)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_FILE,
+                                              garrow_input_stream_file_interface_init)
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_READABLE,
+                                              garrow_input_stream_readable_interface_init));
+
+#define GARROW_INPUT_STREAM_GET_PRIVATE(obj)                    \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_TYPE_INPUT_STREAM,        \
+                               GArrowInputStreamPrivate))
+
+static void
+garrow_input_stream_finalize(GObject *object)
+{
+  GArrowInputStreamPrivate *priv;
+
+  priv = GARROW_INPUT_STREAM_GET_PRIVATE(object);
+
+  priv->input_stream = nullptr;
+
+  G_OBJECT_CLASS(garrow_input_stream_parent_class)->finalize(object);
+}
+
+static void
+garrow_input_stream_set_property(GObject *object,
+                                 guint prop_id,
+                                 const GValue *value,
+                                 GParamSpec *pspec)
+{
+  GArrowInputStreamPrivate *priv;
+
+  priv = GARROW_INPUT_STREAM_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_INPUT_STREAM:
+    priv->input_stream =
+      *static_cast<std::shared_ptr<arrow::io::InputStream> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_input_stream_get_property(GObject *object,
+                                 guint prop_id,
+                                 GValue *value,
+                                 GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_input_stream_init(GArrowInputStream *object)
+{
+}
+
+static void
+garrow_input_stream_class_init(GArrowInputStreamClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_input_stream_finalize;
+  gobject_class->set_property = garrow_input_stream_set_property;
+  gobject_class->get_property = garrow_input_stream_get_property;
+
+  spec = g_param_spec_pointer("input-stream",
+                              "Input stream",
+                              "The raw std::shared<arrow::io::InputStream> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_INPUT_STREAM, spec);
+}
+
+
+static std::shared_ptr<arrow::io::RandomAccessFile>
+garrow_buffer_reader_get_raw_random_access_file_interface(GArrowRandomAccessFile *random_access_file)
+{
+  auto input_stream = GARROW_INPUT_STREAM(random_access_file);
+  auto arrow_input_stream = garrow_input_stream_get_raw(input_stream);
+  auto arrow_buffer_reader =
+    std::static_pointer_cast<arrow::io::BufferReader>(arrow_input_stream);
+  return arrow_buffer_reader;
+}
 
 static void
-garrow_input_stream_default_init (GArrowInputStreamInterface *iface)
+garrow_buffer_reader_random_access_file_interface_init(GArrowRandomAccessFileInterface *iface)
+{
+  iface->get_raw = garrow_buffer_reader_get_raw_random_access_file_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(GArrowBufferReader,               \
+                        garrow_buffer_reader,             \
+                        GARROW_TYPE_INPUT_STREAM,
+                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_RANDOM_ACCESS_FILE,
+                                              garrow_buffer_reader_random_access_file_interface_init));
+
+static void
+garrow_buffer_reader_init(GArrowBufferReader *object)
+{
+}
+
+static void
+garrow_buffer_reader_class_init(GArrowBufferReaderClass *klass)
+{
+}
+
+/**
+ * garrow_buffer_reader_new:
+ * @buffer: The buffer to be read.
+ *
+ * Returns: A newly created #GArrowBufferReader.
+ */
+GArrowBufferReader *
+garrow_buffer_reader_new(GArrowBuffer *buffer)
+{
+  auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  auto arrow_buffer_reader =
+    std::make_shared<arrow::io::BufferReader>(arrow_buffer);
+  return garrow_buffer_reader_new_raw(&arrow_buffer_reader);
+}
+
+/**
+ * garrow_buffer_reader_get_buffer:
+ * @buffer_reader: A #GArrowBufferReader.
+ *
+ * Returns: (transfer full): The data of the array as #GArrowBuffer.
+ */
+GArrowBuffer *
+garrow_buffer_reader_get_buffer(GArrowBufferReader *buffer_reader)
 {
+  auto arrow_input_stream =
+    garrow_input_stream_get_raw(GARROW_INPUT_STREAM(buffer_reader));
+  auto arrow_buffer_reader =
+    std::static_pointer_cast<arrow::io::BufferReader>(arrow_input_stream);
+  auto arrow_buffer = arrow_buffer_reader->buffer();
+  return garrow_buffer_new_raw(&arrow_buffer);
 }
 
+
 G_END_DECLS
 
+GArrowInputStream *
+garrow_input_stream_new_raw(std::shared_ptr<arrow::io::InputStream> *arrow_input_stream)
+{
+  auto input_stream =
+    GARROW_INPUT_STREAM(g_object_new(GARROW_TYPE_INPUT_STREAM,
+                                     "input-stream", arrow_input_stream,
+                                     NULL));
+  return input_stream;
+}
+
 std::shared_ptr<arrow::io::InputStream>
 garrow_input_stream_get_raw(GArrowInputStream *input_stream)
 {
-  auto *iface = GARROW_INPUT_STREAM_GET_IFACE(input_stream);
-  return iface->get_raw(input_stream);
+  GArrowInputStreamPrivate *priv;
+
+  priv = GARROW_INPUT_STREAM_GET_PRIVATE(input_stream);
+  return priv->input_stream;
+}
+
+
+GArrowBufferReader *
+garrow_buffer_reader_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader)
+{
+  auto buffer_reader =
+    GARROW_BUFFER_READER(g_object_new(GARROW_TYPE_BUFFER_READER,
+                                      "input-stream", arrow_buffer_reader,
+                                      NULL));
+  return buffer_reader;
 }
diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h
index 4b331b93fb27f..caa11b575452b 100644
--- a/c_glib/arrow-glib/input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -19,27 +19,105 @@
 
 #pragma once
 
-#include <glib-object.h>
+#include <arrow-glib/buffer.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_INPUT_STREAM             \
+#define GARROW_TYPE_INPUT_STREAM                \
   (garrow_input_stream_get_type())
-#define GARROW_INPUT_STREAM(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INPUT_STREAM,      \
+#define GARROW_INPUT_STREAM(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INPUT_STREAM, \
                               GArrowInputStream))
-#define GARROW_IS_INPUT_STREAM(obj)                          \
+#define GARROW_INPUT_STREAM_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INPUT_STREAM,    \
+                           GArrowInputStreamClass))
+#define GARROW_IS_INPUT_STREAM(obj)                             \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
                               GARROW_TYPE_INPUT_STREAM))
-#define GARROW_INPUT_STREAM_GET_IFACE(obj)                   \
-  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                         \
-                                 GARROW_TYPE_INPUT_STREAM,   \
-                                 GArrowInputStreamInterface))
+#define GARROW_IS_INPUT_STREAM_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INPUT_STREAM))
+#define GARROW_INPUT_STREAM_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INPUT_STREAM,  \
+                             GArrowInputStreamClass))
+
+typedef struct _GArrowInputStream         GArrowInputStream;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowInputStreamClass    GArrowInputStreamClass;
+#endif
+
+/**
+ * GArrowInputStream:
+ *
+ * It wraps `arrow::io::InputStream`.
+ */
+struct _GArrowInputStream
+{
+  /*< private >*/
+  GObject parent_instance;
+};
 
-typedef struct _GArrowInputStream          GArrowInputStream;
-typedef struct _GArrowInputStreamInterface GArrowInputStreamInterface;
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowInputStreamClass
+{
+  GObjectClass parent_class;
+};
+#endif
 
 GType garrow_input_stream_get_type(void) G_GNUC_CONST;
 
+
+#define GARROW_TYPE_BUFFER_READER               \
+  (garrow_buffer_reader_get_type())
+#define GARROW_BUFFER_READER(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BUFFER_READER,        \
+                              GArrowBufferReader))
+#define GARROW_BUFFER_READER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_BUFFER_READER,   \
+                           GArrowBufferReaderClass))
+#define GARROW_IS_BUFFER_READER(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BUFFER_READER))
+#define GARROW_IS_BUFFER_READER_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_BUFFER_READER))
+#define GARROW_BUFFER_READER_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_BUFFER_READER, \
+                             GArrowBufferReaderClass))
+
+typedef struct _GArrowBufferReader         GArrowBufferReader;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowBufferReaderClass    GArrowBufferReaderClass;
+#endif
+
+/**
+ * GArrowBufferReader:
+ *
+ * It wraps `arrow::io::BufferReader`.
+ */
+struct _GArrowBufferReader
+{
+  /*< private >*/
+  GArrowInputStream parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowBufferReaderClass
+{
+  GArrowInputStreamClass parent_class;
+};
+#endif
+
+GType garrow_buffer_reader_get_type(void) G_GNUC_CONST;
+
+GArrowBufferReader *garrow_buffer_reader_new(GArrowBuffer *buffer);
+
+GArrowBuffer *garrow_buffer_reader_get_buffer(GArrowBufferReader *buffer_reader);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/input-stream.hpp b/c_glib/arrow-glib/input-stream.hpp
index 7958df1585887..008f5f2b4e157 100644
--- a/c_glib/arrow-glib/input-stream.hpp
+++ b/c_glib/arrow-glib/input-stream.hpp
@@ -20,19 +20,11 @@
 #pragma once
 
 #include <arrow/io/interfaces.h>
+#include <arrow/io/memory.h>
 
 #include <arrow-glib/input-stream.h>
 
-/**
- * GArrowInputStreamInterface:
- *
- * It wraps `arrow::io::InputStream`.
- */
-struct _GArrowInputStreamInterface
-{
-  GTypeInterface parent_iface;
-
-  std::shared_ptr<arrow::io::InputStream> (*get_raw)(GArrowInputStream *file);
-};
-
+GArrowInputStream *garrow_input_stream_new_raw(std::shared_ptr<arrow::io::InputStream> *arrow_input_stream);
 std::shared_ptr<arrow::io::InputStream> garrow_input_stream_get_raw(GArrowInputStream *input_stream);
+
+GArrowBufferReader *garrow_buffer_reader_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader);
diff --git a/c_glib/arrow-glib/memory-mapped-file.cpp b/c_glib/arrow-glib/memory-mapped-file.cpp
index f9cbf079105c1..71a9a6dad3134 100644
--- a/c_glib/arrow-glib/memory-mapped-file.cpp
+++ b/c_glib/arrow-glib/memory-mapped-file.cpp
@@ -82,21 +82,6 @@ garrow_readable_interface_init(GArrowReadableInterface *iface)
   iface->get_raw = garrow_memory_mapped_file_get_raw_readable_interface;
 }
 
-static std::shared_ptr<arrow::io::InputStream>
-garrow_memory_mapped_file_get_raw_input_stream_interface(GArrowInputStream *input_stream)
-{
-  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(input_stream);
-  auto arrow_memory_mapped_file =
-    garrow_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_input_stream_interface_init(GArrowInputStreamInterface *iface)
-{
-  iface->get_raw = garrow_memory_mapped_file_get_raw_input_stream_interface;
-}
-
 static std::shared_ptr<arrow::io::RandomAccessFile>
 garrow_memory_mapped_file_get_raw_random_access_file_interface(GArrowRandomAccessFile *file)
 {
@@ -150,8 +135,6 @@ G_DEFINE_TYPE_WITH_CODE(GArrowMemoryMappedFile,
                                               garrow_file_interface_init)
                         G_IMPLEMENT_INTERFACE(GARROW_TYPE_READABLE,
                                               garrow_readable_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_INPUT_STREAM,
-                                              garrow_input_stream_interface_init)
                         G_IMPLEMENT_INTERFACE(GARROW_TYPE_RANDOM_ACCESS_FILE,
                                               garrow_random_access_file_interface_init)
                         G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE,
diff --git a/c_glib/test/test-buffer-reader.rb b/c_glib/test/test-buffer-reader.rb
new file mode 100644
index 0000000000000..b3517b230e421
--- /dev/null
+++ b/c_glib/test/test-buffer-reader.rb
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBufferReader < Test::Unit::TestCase
+  def test_read
+    buffer = Arrow::Buffer.new("Hello World")
+    buffer_reader = Arrow::BufferReader.new(buffer)
+    read_buffer = " " * 5
+    _success, n_read_bytes = buffer_reader.read(read_buffer)
+    assert_equal("Hello", read_buffer.byteslice(0, n_read_bytes))
+  end
+end

From 2d5142cd3fc9a5f5150daf6ea6335029de8002ae Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 30 Apr 2017 13:31:49 -0400
Subject: [PATCH 0584/1644] ARROW-918: [GLib] Use GArrowBuffer for read buffer

It's efficient.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #618 from kou/glib-read-buffer and squashes the following commits:

e14bb40 [Kouhei Sutou] [GLib] Use GArrowBuffer for read buffer
---
 c_glib/arrow-glib/random-access-file.cpp | 20 +++++++++++---------
 c_glib/arrow-glib/random-access-file.h   | 10 ++++------
 c_glib/arrow-glib/readable.cpp           | 23 +++++++++++++----------
 c_glib/arrow-glib/readable.h             | 10 ++++------
 c_glib/test/test-memory-mapped-file.rb   | 18 +++++++-----------
 5 files changed, 39 insertions(+), 42 deletions(-)

diff --git a/c_glib/arrow-glib/random-access-file.cpp b/c_glib/arrow-glib/random-access-file.cpp
index 976a80dce0693..744131632cbc2 100644
--- a/c_glib/arrow-glib/random-access-file.cpp
+++ b/c_glib/arrow-glib/random-access-file.cpp
@@ -23,6 +23,7 @@
 
 #include <arrow/api.h>
 
+#include <arrow-glib/buffer.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/random-access-file.hpp>
 
@@ -88,28 +89,29 @@ garrow_random_access_file_get_support_zero_copy(GArrowRandomAccessFile *file)
  * @file: A #GArrowRandomAccessFile.
  * @position: The read start position.
  * @n_bytes: The number of bytes to be read.
- * @n_read_bytes: (out): The read number of bytes.
- * @buffer: (array length=n_bytes): The buffer to be read data.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: %TRUE on success, %FALSE if there was an error.
+ * Returns: (transfer full) (nullable): #GArrowBuffer that has read
+ *   data on success, %NULL if there was an error.
  */
-gboolean
+GArrowBuffer *
 garrow_random_access_file_read_at(GArrowRandomAccessFile *file,
                                      gint64 position,
                                      gint64 n_bytes,
-                                     gint64 *n_read_bytes,
-                                     guint8 *buffer,
                                      GError **error)
 {
   const auto arrow_random_access_file =
     garrow_random_access_file_get_raw(file);
 
+  std::shared_ptr<arrow::Buffer> arrow_buffer;
   auto status = arrow_random_access_file->ReadAt(position,
                                                  n_bytes,
-                                                 n_read_bytes,
-                                                 buffer);
-  return garrow_error_check(error, status, "[io][random-access-file][read-at]");
+                                                 &arrow_buffer);
+  if (garrow_error_check(error, status, "[io][random-access-file][read-at]")) {
+    return garrow_buffer_new_raw(&arrow_buffer);
+  } else {
+    return NULL;
+  }
 }
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/random-access-file.h b/c_glib/arrow-glib/random-access-file.h
index 8a7f6b4218a31..83a7d8cd14b95 100644
--- a/c_glib/arrow-glib/random-access-file.h
+++ b/c_glib/arrow-glib/random-access-file.h
@@ -45,11 +45,9 @@ GType garrow_random_access_file_get_type(void) G_GNUC_CONST;
 guint64 garrow_random_access_file_get_size(GArrowRandomAccessFile *file,
                                               GError **error);
 gboolean garrow_random_access_file_get_support_zero_copy(GArrowRandomAccessFile *file);
-gboolean garrow_random_access_file_read_at(GArrowRandomAccessFile *file,
-                                              gint64 position,
-                                              gint64 n_bytes,
-                                              gint64 *n_read_bytes,
-                                              guint8 *buffer,
-                                              GError **error);
+GArrowBuffer *garrow_random_access_file_read_at(GArrowRandomAccessFile *file,
+                                                gint64 position,
+                                                gint64 n_bytes,
+                                                GError **error);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/readable.cpp b/c_glib/arrow-glib/readable.cpp
index d893853eea015..6a9023e6cddf0 100644
--- a/c_glib/arrow-glib/readable.cpp
+++ b/c_glib/arrow-glib/readable.cpp
@@ -23,6 +23,7 @@
 
 #include <arrow/api.h>
 
+#include <arrow-glib/buffer.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/readable.hpp>
 
@@ -50,23 +51,25 @@ garrow_readable_default_init (GArrowReadableInterface *iface)
  * garrow_readable_read:
  * @readable: A #GArrowReadable.
  * @n_bytes: The number of bytes to be read.
- * @n_read_bytes: (out): The read number of bytes.
- * @buffer: (array length=n_bytes): The buffer to be read data.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: %TRUE on success, %FALSE if there was an error.
+ * Returns: (transfer full) (nullable): #GArrowBuffer that has read
+ *   data on success, %NULL if there was an error.
  */
-gboolean
+GArrowBuffer *
 garrow_readable_read(GArrowReadable *readable,
-                        gint64 n_bytes,
-                        gint64 *n_read_bytes,
-                        guint8 *buffer,
-                        GError **error)
+                     gint64 n_bytes,
+                     GError **error)
 {
   const auto arrow_readable = garrow_readable_get_raw(readable);
 
-  auto status = arrow_readable->Read(n_bytes, n_read_bytes, buffer);
-  return garrow_error_check(error, status, "[io][readable][read]");
+  std::shared_ptr<arrow::Buffer> arrow_buffer;
+  auto status = arrow_readable->Read(n_bytes, &arrow_buffer);
+  if (garrow_error_check(error, status, "[io][readable][read]")) {
+    return garrow_buffer_new_raw(&arrow_buffer);
+  } else {
+    return NULL;
+  }
 }
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/readable.h b/c_glib/arrow-glib/readable.h
index bde4b01ee1f15..216e7369c76f7 100644
--- a/c_glib/arrow-glib/readable.h
+++ b/c_glib/arrow-glib/readable.h
@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <glib-object.h>
+#include <arrow-glib/buffer.h>
 
 G_BEGIN_DECLS
 
@@ -42,10 +42,8 @@ typedef struct _GArrowReadableInterface GArrowReadableInterface;
 
 GType garrow_readable_get_type(void) G_GNUC_CONST;
 
-gboolean garrow_readable_read(GArrowReadable *readable,
-                                 gint64 n_bytes,
-                                 gint64 *n_read_bytes,
-                                 guint8 *buffer,
-                                 GError **error);
+GArrowBuffer *garrow_readable_read(GArrowReadable *readable,
+                                   gint64 n_bytes,
+                                   GError **error);
 
 G_END_DECLS
diff --git a/c_glib/test/test-memory-mapped-file.rb b/c_glib/test/test-memory-mapped-file.rb
index e78d07a72d3b8..e09e3697074c9 100644
--- a/c_glib/test/test-memory-mapped-file.rb
+++ b/c_glib/test/test-memory-mapped-file.rb
@@ -22,9 +22,8 @@ def test_open
     tempfile.close
     file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
-      buffer = " " * 5
-      file.read(buffer)
-      assert_equal("Hello", buffer)
+      buffer = file.read(5)
+      assert_equal("Hello", buffer.data.to_s)
     ensure
       file.close
     end
@@ -48,9 +47,8 @@ def test_read
     tempfile.close
     file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
-      buffer = " " * 5
-      _success, n_read_bytes = file.read(buffer)
-      assert_equal("Hello", buffer.byteslice(0, n_read_bytes))
+      buffer = file.read(5)
+      assert_equal("Hello", buffer.data.to_s)
     ensure
       file.close
     end
@@ -62,9 +60,8 @@ def test_read_at
     tempfile.close
     file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
-      buffer = " " * 5
-      _success, n_read_bytes = file.read_at(6, buffer)
-      assert_equal("World", buffer.byteslice(0, n_read_bytes))
+      buffer = file.read_at(6, 5)
+      assert_equal("World", buffer.data.to_s)
     ensure
       file.close
     end
@@ -116,8 +113,7 @@ def test_tell
     tempfile.close
     file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
     begin
-      buffer = " " * 5
-      file.read(buffer)
+      file.read(5)
       assert_equal(5, file.tell)
     ensure
       file.close

From b4886da0f19484fe829fdb23a231a864070bf58c Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 30 Apr 2017 13:33:06 -0400
Subject: [PATCH 0585/1644] ARROW-919: [GLib] Use "id" to get type enum value
 from GArrowDataType

Author: Kouhei Sutou <kou@clear-code.com>

Closes #619 from kou/glib-data-type-use-id and squashes the following commits:

e20d445 [Kouhei Sutou] [GLib] Use "id" to get type enum value from GArrowDataType
---
 c_glib/arrow-glib/data-type.cpp       | 6 +++---
 c_glib/arrow-glib/data-type.h         | 2 +-
 c_glib/test/test-binary-data-type.rb  | 2 +-
 c_glib/test/test-boolean-data-type.rb | 2 +-
 c_glib/test/test-double-data-type.rb  | 2 +-
 c_glib/test/test-float-data-type.rb   | 2 +-
 c_glib/test/test-int16-data-type.rb   | 2 +-
 c_glib/test/test-int32-data-type.rb   | 2 +-
 c_glib/test/test-int64-data-type.rb   | 2 +-
 c_glib/test/test-int8-data-type.rb    | 2 +-
 c_glib/test/test-list-data-type.rb    | 2 +-
 c_glib/test/test-null-data-type.rb    | 2 +-
 c_glib/test/test-string-data-type.rb  | 2 +-
 c_glib/test/test-uint16-data-type.rb  | 2 +-
 c_glib/test/test-uint32-data-type.rb  | 2 +-
 c_glib/test/test-uint64-data-type.rb  | 2 +-
 c_glib/test/test-uint8-data-type.rb   | 2 +-
 17 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/data-type.cpp
index 2fd261dc91947..c3c7fdb0f7c21 100644
--- a/c_glib/arrow-glib/data-type.cpp
+++ b/c_glib/arrow-glib/data-type.cpp
@@ -192,13 +192,13 @@ garrow_data_type_to_string(GArrowDataType *data_type)
 }
 
 /**
- * garrow_data_type_type:
+ * garrow_data_type_get_id:
  * @data_type: A #GArrowDataType.
  *
- * Returns: The type of the data type.
+ * Returns: The #GArrowType of the data type.
  */
 GArrowType
-garrow_data_type_type(GArrowDataType *data_type)
+garrow_data_type_get_id(GArrowDataType *data_type)
 {
   const auto arrow_data_type = garrow_data_type_get_raw(data_type);
   return garrow_type_from_raw(arrow_data_type->id());
diff --git a/c_glib/arrow-glib/data-type.h b/c_glib/arrow-glib/data-type.h
index babf0ee1712a0..933fcfc4b2ccb 100644
--- a/c_glib/arrow-glib/data-type.h
+++ b/c_glib/arrow-glib/data-type.h
@@ -73,7 +73,7 @@ GType      garrow_data_type_get_type  (void) G_GNUC_CONST;
 gboolean   garrow_data_type_equal     (GArrowDataType *data_type,
                                        GArrowDataType *other_data_type);
 gchar     *garrow_data_type_to_string (GArrowDataType *data_type);
-GArrowType garrow_data_type_type      (GArrowDataType *data_type);
+GArrowType garrow_data_type_get_id    (GArrowDataType *data_type);
 
 
 #define GARROW_TYPE_NULL_DATA_TYPE              \
diff --git a/c_glib/test/test-binary-data-type.rb b/c_glib/test/test-binary-data-type.rb
index 3d4095c1b0648..5a1cb89b30061 100644
--- a/c_glib/test/test-binary-data-type.rb
+++ b/c_glib/test/test-binary-data-type.rb
@@ -18,7 +18,7 @@
 class TestBinaryDataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::BinaryDataType.new
-    assert_equal(Arrow::Type::BINARY, data_type.type)
+    assert_equal(Arrow::Type::BINARY, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-boolean-data-type.rb b/c_glib/test/test-boolean-data-type.rb
index ac5667140fb8e..39b8128989de3 100644
--- a/c_glib/test/test-boolean-data-type.rb
+++ b/c_glib/test/test-boolean-data-type.rb
@@ -18,7 +18,7 @@
 class TestBooleanDataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::BooleanDataType.new
-    assert_equal(Arrow::Type::BOOL, data_type.type)
+    assert_equal(Arrow::Type::BOOL, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-double-data-type.rb b/c_glib/test/test-double-data-type.rb
index 18c870cb9e62b..0edd64eed300f 100644
--- a/c_glib/test/test-double-data-type.rb
+++ b/c_glib/test/test-double-data-type.rb
@@ -18,7 +18,7 @@
 class TestDoubleDataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::DoubleDataType.new
-    assert_equal(Arrow::Type::DOUBLE, data_type.type)
+    assert_equal(Arrow::Type::DOUBLE, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-float-data-type.rb b/c_glib/test/test-float-data-type.rb
index ab315fd336b84..8384b526e0203 100644
--- a/c_glib/test/test-float-data-type.rb
+++ b/c_glib/test/test-float-data-type.rb
@@ -18,7 +18,7 @@
 class TestFloatDataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::FloatDataType.new
-    assert_equal(Arrow::Type::FLOAT, data_type.type)
+    assert_equal(Arrow::Type::FLOAT, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-int16-data-type.rb b/c_glib/test/test-int16-data-type.rb
index 273ec809c198e..aad5f11fbf60d 100644
--- a/c_glib/test/test-int16-data-type.rb
+++ b/c_glib/test/test-int16-data-type.rb
@@ -18,7 +18,7 @@
 class TestInt16DataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::Int16DataType.new
-    assert_equal(Arrow::Type::INT16, data_type.type)
+    assert_equal(Arrow::Type::INT16, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-int32-data-type.rb b/c_glib/test/test-int32-data-type.rb
index f6b9b34e1d827..2d9d44d66236a 100644
--- a/c_glib/test/test-int32-data-type.rb
+++ b/c_glib/test/test-int32-data-type.rb
@@ -18,7 +18,7 @@
 class TestInt32DataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::Int32DataType.new
-    assert_equal(Arrow::Type::INT32, data_type.type)
+    assert_equal(Arrow::Type::INT32, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-int64-data-type.rb b/c_glib/test/test-int64-data-type.rb
index 032b24dac3ecc..3c5263e848ca2 100644
--- a/c_glib/test/test-int64-data-type.rb
+++ b/c_glib/test/test-int64-data-type.rb
@@ -18,7 +18,7 @@
 class TestInt64DataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::Int64DataType.new
-    assert_equal(Arrow::Type::INT64, data_type.type)
+    assert_equal(Arrow::Type::INT64, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-int8-data-type.rb b/c_glib/test/test-int8-data-type.rb
index d33945614db8e..40de1be95c652 100644
--- a/c_glib/test/test-int8-data-type.rb
+++ b/c_glib/test/test-int8-data-type.rb
@@ -18,7 +18,7 @@
 class TestInt8DataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::Int8DataType.new
-    assert_equal(Arrow::Type::INT8, data_type.type)
+    assert_equal(Arrow::Type::INT8, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-list-data-type.rb b/c_glib/test/test-list-data-type.rb
index 6fde203517684..aa6a8fa65fd8c 100644
--- a/c_glib/test/test-list-data-type.rb
+++ b/c_glib/test/test-list-data-type.rb
@@ -19,7 +19,7 @@ class TestListDataType < Test::Unit::TestCase
   def test_type
     field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
     data_type = Arrow::ListDataType.new(field)
-    assert_equal(Arrow::Type::LIST, data_type.type)
+    assert_equal(Arrow::Type::LIST, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-null-data-type.rb b/c_glib/test/test-null-data-type.rb
index 95e54833b0896..fd766675e40c3 100644
--- a/c_glib/test/test-null-data-type.rb
+++ b/c_glib/test/test-null-data-type.rb
@@ -18,7 +18,7 @@
 class TestNullDataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::NullDataType.new
-    assert_equal(Arrow::Type::NA, data_type.type)
+    assert_equal(Arrow::Type::NA, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-string-data-type.rb b/c_glib/test/test-string-data-type.rb
index daba7fd9ec768..550bf13f19f39 100644
--- a/c_glib/test/test-string-data-type.rb
+++ b/c_glib/test/test-string-data-type.rb
@@ -18,7 +18,7 @@
 class TestStringDataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::StringDataType.new
-    assert_equal(Arrow::Type::STRING, data_type.type)
+    assert_equal(Arrow::Type::STRING, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-uint16-data-type.rb b/c_glib/test/test-uint16-data-type.rb
index f5a6cc0be28bb..56f6cf4a2f0b4 100644
--- a/c_glib/test/test-uint16-data-type.rb
+++ b/c_glib/test/test-uint16-data-type.rb
@@ -18,7 +18,7 @@
 class TestUInt16DataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::UInt16DataType.new
-    assert_equal(Arrow::Type::UINT16, data_type.type)
+    assert_equal(Arrow::Type::UINT16, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-uint32-data-type.rb b/c_glib/test/test-uint32-data-type.rb
index 7a50257d6d3b9..7ad3f5697e510 100644
--- a/c_glib/test/test-uint32-data-type.rb
+++ b/c_glib/test/test-uint32-data-type.rb
@@ -18,7 +18,7 @@
 class TestUInt32DataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::UInt32DataType.new
-    assert_equal(Arrow::Type::UINT32, data_type.type)
+    assert_equal(Arrow::Type::UINT32, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-uint64-data-type.rb b/c_glib/test/test-uint64-data-type.rb
index 403fc9acdfcfa..f5bf3c9786b93 100644
--- a/c_glib/test/test-uint64-data-type.rb
+++ b/c_glib/test/test-uint64-data-type.rb
@@ -18,7 +18,7 @@
 class TestUInt64DataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::UInt64DataType.new
-    assert_equal(Arrow::Type::UINT64, data_type.type)
+    assert_equal(Arrow::Type::UINT64, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-uint8-data-type.rb b/c_glib/test/test-uint8-data-type.rb
index eb91da2761efe..d4bf797a95b7e 100644
--- a/c_glib/test/test-uint8-data-type.rb
+++ b/c_glib/test/test-uint8-data-type.rb
@@ -18,7 +18,7 @@
 class TestUInt8DataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::UInt8DataType.new
-    assert_equal(Arrow::Type::UINT8, data_type.type)
+    assert_equal(Arrow::Type::UINT8, data_type.id)
   end
 
   def test_to_s

From 00994b82015365fec8474605bf09bd11637859af Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 30 Apr 2017 13:35:29 -0400
Subject: [PATCH 0586/1644] ARROW-920: [GLib] Add Lua examples

Author: Kouhei Sutou <kou@clear-code.com>

Closes #620 from kou/glib-lua-examples and squashes the following commits:

491c0e4 [Kouhei Sutou] [GLib] Add Lua examples
---
 c_glib/README.md                    |  2 +
 c_glib/configure.ac                 |  1 +
 c_glib/example/Makefile.am          |  6 ++-
 c_glib/example/README.md            |  5 +-
 c_glib/example/lua/Makefile.am      | 24 ++++++++++
 c_glib/example/lua/README.md        | 45 ++++++++++++++++++
 c_glib/example/lua/read-batch.lua   | 44 +++++++++++++++++
 c_glib/example/lua/read-stream.lua  | 51 ++++++++++++++++++++
 c_glib/example/lua/write-batch.lua  | 74 +++++++++++++++++++++++++++++
 c_glib/example/lua/write-stream.lua | 74 +++++++++++++++++++++++++++++
 10 files changed, 323 insertions(+), 3 deletions(-)
 create mode 100644 c_glib/example/lua/Makefile.am
 create mode 100644 c_glib/example/lua/README.md
 create mode 100644 c_glib/example/lua/read-batch.lua
 create mode 100644 c_glib/example/lua/read-stream.lua
 create mode 100644 c_glib/example/lua/write-batch.lua
 create mode 100644 c_glib/example/lua/write-stream.lua

diff --git a/c_glib/README.md b/c_glib/README.md
index b253d32b266d4..6eadb797032bc 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -194,10 +194,12 @@ You can use Arrow GLib with non C languages with GObject Introspection
 based bindings. Here are languages that support GObject Introspection:
 
   * Ruby: [red-arrow gem](https://rubygems.org/gems/red-arrow) should be used.
+    * Examples: https://github.com/red-data-tools/red-arrow/tree/master/example
 
   * Python: [PyGObject](https://wiki.gnome.org/Projects/PyGObject) should be used. (Note that you should use PyArrow than Arrow GLib.)
 
   * Lua: [LGI](https://github.com/pavouk/lgi) should be used.
+    * Examples: `example/lua/` directory.
 
   * Go: [Go-gir-generator](https://github.com/linuxdeepin/go-gir-generator) should be used.
 
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index f36719284711b..e010d962f377a 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -97,6 +97,7 @@ AC_CONFIG_FILES([
   doc/Makefile
   doc/reference/Makefile
   example/Makefile
+  example/lua/Makefile
 ])
 
 AC_OUTPUT
diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am
index 8bf3c15526759..66d2cddcac5fb 100644
--- a/c_glib/example/Makefile.am
+++ b/c_glib/example/Makefile.am
@@ -15,6 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+SUBDIRS	=					\
+	lua
+
 AM_CPPFLAGS =					\
 	-I$(top_builddir)			\
 	-I$(top_srcdir)
@@ -41,7 +44,8 @@ read_batch_SOURCES =				\
 read_stream_SOURCES =				\
 	read-stream.c
 
-example_DATA =					\
+dist_example_DATA =				\
+	README.md				\
 	$(build_SOURCES)			\
 	$(read_batch_SOURCES)			\
 	$(read_stream_SOURCES)
diff --git a/c_glib/example/README.md b/c_glib/example/README.md
index b1ba259534cb1..99730d59ce1c2 100644
--- a/c_glib/example/README.md
+++ b/c_glib/example/README.md
@@ -16,7 +16,9 @@
 
 There are example codes in this directory.
 
-C example codes exist in this directory.
+C example codes exist in this directory. Language bindings example
+codes exists in sub directories. For example, Lua example codes exists
+in `lua/` sub directory.
 
 ## C example codes
 
@@ -39,4 +41,3 @@ Here are example codes in this directory:
 
   * `read-stream.c`: It shows how to read Arrow array from file in
     stream mode.
-
diff --git a/c_glib/example/lua/Makefile.am b/c_glib/example/lua/Makefile.am
new file mode 100644
index 0000000000000..9019d24741c1a
--- /dev/null
+++ b/c_glib/example/lua/Makefile.am
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+lua_exampledir = $(exampledir)/lua
+dist_lua_example_DATA =				\
+	README.md				\
+	read-batch.lua				\
+	read-stream.lua				\
+	write-batch.lua				\
+	write-stream.lua
diff --git a/c_glib/example/lua/README.md b/c_glib/example/lua/README.md
new file mode 100644
index 0000000000000..d127573bcc368
--- /dev/null
+++ b/c_glib/example/lua/README.md
@@ -0,0 +1,45 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Arrow Lua example
+
+There are Lua example codes in this directory.
+
+## How to run
+
+All example codes use [LGI](https://github.com/pavouk/lgi) to use
+Arrow GLib based bindings.
+
+Here are command lines to install LGI on Debian GNU/Linux and Ubuntu:
+
+```text
+% sudo apt install -y luarocks
+% sudo luarocks install lgi
+```
+
+## Lua example codes
+
+Here are example codes in this directory:
+
+  * `write-batch.lua`: It shows how to write Arrow array to file in
+    batch mode.
+
+  * `read-batch.lua`: It shows how to read Arrow array from file in
+    batch mode.
+
+  * `write-stream.lua`: It shows how to write Arrow array to file in
+    stream mode.
+
+  * `read-stream.lua`: It shows how to read Arrow array from file in
+    stream mode.
diff --git a/c_glib/example/lua/read-batch.lua b/c_glib/example/lua/read-batch.lua
new file mode 100644
index 0000000000000..b28d346863820
--- /dev/null
+++ b/c_glib/example/lua/read-batch.lua
@@ -0,0 +1,44 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local input_path = arg[1] or "/tmp/batch.arrow";
+
+local input = Arrow.MemoryMappedFile.open(input_path, Arrow.FileMode.READ)
+local reader = Arrow.FileReader.open(input)
+
+for i = 0, reader:get_n_record_batches() - 1 do
+   local record_batch = reader:get_record_batch(i)
+   print(string.rep("=", 40))
+   print("record-batch["..i.."]:")
+   for j = 0, record_batch:get_n_columns() - 1 do
+      local column = record_batch:get_column(j)
+      local column_name = record_batch:get_column_name(j)
+      io.write("  "..column_name..": [")
+      for k = 0, record_batch:get_n_rows() - 1 do
+	 if k > 0 then
+	    io.write(", ")
+	 end
+	 io.write(column:get_value(k))
+      end
+      print("]")
+   end
+end
+
+input:close()
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
new file mode 100644
index 0000000000000..3b0820627e6b2
--- /dev/null
+++ b/c_glib/example/lua/read-stream.lua
@@ -0,0 +1,51 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local input_path = arg[1] or "/tmp/stream.arrow";
+
+local input = Arrow.MemoryMappedFile.open(input_path, Arrow.FileMode.READ)
+local reader = Arrow.StreamReader.open(input)
+
+local i = 0
+while true do
+   local record_batch = reader:get_next_record_batch(i)
+   if not record_batch then
+      break
+   end
+
+   print(string.rep("=", 40))
+   print("record-batch["..i.."]:")
+   for j = 0, record_batch:get_n_columns() - 1 do
+      local column = record_batch:get_column(j)
+      local column_name = record_batch:get_column_name(j)
+      io.write("  "..column_name..": [")
+      for k = 0, record_batch:get_n_rows() - 1 do
+	 if k > 0 then
+	    io.write(", ")
+	 end
+	 io.write(column:get_value(k))
+      end
+      print("]")
+   end
+
+   i = i + 1
+end
+
+input:close()
diff --git a/c_glib/example/lua/write-batch.lua b/c_glib/example/lua/write-batch.lua
new file mode 100644
index 0000000000000..3a22cd57fd81e
--- /dev/null
+++ b/c_glib/example/lua/write-batch.lua
@@ -0,0 +1,74 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local output_path = arg[1] or "/tmp/batch.arrow";
+
+local fields = {
+  Arrow.Field.new("uint8",  Arrow.UInt8DataType.new()),
+  Arrow.Field.new("uint16", Arrow.UInt16DataType.new()),
+  Arrow.Field.new("uint32", Arrow.UInt32DataType.new()),
+  Arrow.Field.new("uint64", Arrow.UInt64DataType.new()),
+  Arrow.Field.new("int8",   Arrow.Int8DataType.new()),
+  Arrow.Field.new("int16",  Arrow.Int16DataType.new()),
+  Arrow.Field.new("int32",  Arrow.Int32DataType.new()),
+  Arrow.Field.new("int64",  Arrow.Int64DataType.new()),
+  Arrow.Field.new("float",  Arrow.FloatDataType.new()),
+  Arrow.Field.new("double", Arrow.DoubleDataType.new()),
+}
+local schema = Arrow.Schema.new(fields)
+
+local output = Arrow.FileOutputStream.open(output_path, false)
+local writer = Arrow.FileWriter.open(output, schema)
+
+function build_array(builder, values)
+   for _, value in pairs(values) do
+      builder:append(value)
+   end
+   return builder:finish()
+end
+
+local uints = {1, 2, 4, 8}
+local ints = {1, -2, 4, -8}
+local floats = {1.1, -2.2, 4.4, -8.8}
+local columns = {
+   build_array(Arrow.UInt8ArrayBuilder.new(), uints),
+   build_array(Arrow.UInt16ArrayBuilder.new(), uints),
+   build_array(Arrow.UInt32ArrayBuilder.new(), uints),
+   build_array(Arrow.UInt64ArrayBuilder.new(), uints),
+   build_array(Arrow.Int8ArrayBuilder.new(), ints),
+   build_array(Arrow.Int16ArrayBuilder.new(), ints),
+   build_array(Arrow.Int32ArrayBuilder.new(), ints),
+   build_array(Arrow.Int64ArrayBuilder.new(), ints),
+   build_array(Arrow.FloatArrayBuilder.new(), floats),
+   build_array(Arrow.DoubleArrayBuilder.new(), floats),
+}
+
+local record_batch = Arrow.RecordBatch.new(schema, 4, columns)
+writer:write_record_batch(record_batch)
+
+local sliced_columns = {}
+for i, column in pairs(columns) do
+   sliced_columns[i] = column:slice(1, 3)
+end
+record_batch = Arrow.RecordBatch.new(schema, 3, sliced_columns)
+writer:write_record_batch(record_batch)
+
+writer:close()
+output:close()
diff --git a/c_glib/example/lua/write-stream.lua b/c_glib/example/lua/write-stream.lua
new file mode 100644
index 0000000000000..37c6bb54cd8f4
--- /dev/null
+++ b/c_glib/example/lua/write-stream.lua
@@ -0,0 +1,74 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local output_path = arg[1] or "/tmp/stream.arrow";
+
+local fields = {
+  Arrow.Field.new("uint8",  Arrow.UInt8DataType.new()),
+  Arrow.Field.new("uint16", Arrow.UInt16DataType.new()),
+  Arrow.Field.new("uint32", Arrow.UInt32DataType.new()),
+  Arrow.Field.new("uint64", Arrow.UInt64DataType.new()),
+  Arrow.Field.new("int8",   Arrow.Int8DataType.new()),
+  Arrow.Field.new("int16",  Arrow.Int16DataType.new()),
+  Arrow.Field.new("int32",  Arrow.Int32DataType.new()),
+  Arrow.Field.new("int64",  Arrow.Int64DataType.new()),
+  Arrow.Field.new("float",  Arrow.FloatDataType.new()),
+  Arrow.Field.new("double", Arrow.DoubleDataType.new()),
+}
+local schema = Arrow.Schema.new(fields)
+
+local output = Arrow.FileOutputStream.open(output_path, false)
+local writer = Arrow.StreamWriter.open(output, schema)
+
+function build_array(builder, values)
+   for _, value in pairs(values) do
+      builder:append(value)
+   end
+   return builder:finish()
+end
+
+local uints = {1, 2, 4, 8}
+local ints = {1, -2, 4, -8}
+local floats = {1.1, -2.2, 4.4, -8.8}
+local columns = {
+   build_array(Arrow.UInt8ArrayBuilder.new(), uints),
+   build_array(Arrow.UInt16ArrayBuilder.new(), uints),
+   build_array(Arrow.UInt32ArrayBuilder.new(), uints),
+   build_array(Arrow.UInt64ArrayBuilder.new(), uints),
+   build_array(Arrow.Int8ArrayBuilder.new(), ints),
+   build_array(Arrow.Int16ArrayBuilder.new(), ints),
+   build_array(Arrow.Int32ArrayBuilder.new(), ints),
+   build_array(Arrow.Int64ArrayBuilder.new(), ints),
+   build_array(Arrow.FloatArrayBuilder.new(), floats),
+   build_array(Arrow.DoubleArrayBuilder.new(), floats),
+}
+
+local record_batch = Arrow.RecordBatch.new(schema, 4, columns)
+writer:write_record_batch(record_batch)
+
+local sliced_columns = {}
+for i, column in pairs(columns) do
+   sliced_columns[i] = column:slice(1, 3)
+end
+record_batch = Arrow.RecordBatch.new(schema, 3, sliced_columns)
+writer:write_record_batch(record_batch)
+
+writer:close()
+output:close()

From d4a2a75a50cb8ccaddb29bc5462ed3aa34af1d9f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 30 Apr 2017 13:39:42 -0400
Subject: [PATCH 0587/1644] ARROW-910: [C++] Write 0 length at EOS in
 StreamWriter

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #614 from wesm/ARROW-910 and squashes the following commits:

e1ef336 [Wes McKinney] Write 0 length at EOS
---
 cpp/src/arrow/ipc/writer.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 61caf6403c8dc..78d6b9eb92b4e 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -620,7 +620,11 @@ class StreamWriter::StreamWriterImpl {
   virtual Status Close() {
     // Write the schema if not already written
     // User is responsible for closing the OutputStream
-    return CheckStarted();
+    RETURN_NOT_OK(CheckStarted());
+
+    // Write 0 EOS message
+    const int32_t kEos = 0;
+    return Write(reinterpret_cast<const uint8_t*>(&kEos), sizeof(int32_t));
   }
 
   Status CheckStarted() {

From 6950e45db78934924a41c39b79bb2c99996d4d56 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 30 Apr 2017 23:01:12 -0400
Subject: [PATCH 0588/1644] ARROW-922: Allow Flatbuffers and RapidJSON to be
 used locally on Windows

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #621 from cpcloud/ARROW-922 and squashes the following commits:

0da7a56 [Phillip Cloud] Add parallel builds
8b377da [Phillip Cloud] ARROW-922: Allow Flatbuffers and RapidJSON to be used locally on Windows
---
 ci/msvc-build.bat                       | 33 +++++++++++--------------
 cpp/cmake_modules/FindFlatbuffers.cmake | 16 ++++++------
 2 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index aca1f8cc3c073..504da7638daa0 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -17,40 +17,37 @@
 
 @echo on
 
-set CONDA_ENV=C:\arrow-conda-env
-set ARROW_HOME=C:\arrow-install
-
-conda create -p %CONDA_ENV% -q -y python=%PYTHON% ^
+conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython
-call activate %CONDA_ENV%
+conda install -n arrow -q -y -c conda-forge flatbuffers rapidjson
+call activate arrow
+
+set ARROW_HOME=%CONDA_PREFIX%\Library
+set FLATBUFFERS_HOME=%CONDA_PREFIX%\Library
+set RAPIDJSON_HOME=%CONDA_PREFIX%\Library
 
 @rem Build and test Arrow C++ libraries
 
-cd cpp
-mkdir build
-cd build
+mkdir cpp\build
+cd cpp\build
+
 cmake -G "%GENERATOR%" ^
-      -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
+      -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
       -DARROW_BOOST_USE_SHARED=OFF ^
       -DCMAKE_BUILD_TYPE=Release ^
-      -DARROW_CXXFLAGS="/WX" ^
-      -DARROW_PYTHON=on ^
+      -DARROW_CXXFLAGS="/WX /MP" ^
+      -DARROW_PYTHON=ON ^
       ..  || exit /B
 cmake --build . --target INSTALL --config Release  || exit /B
 
 @rem Needed so python-test.exe works
-set PYTHONPATH=%CONDA_ENV%\Lib;%CONDA_ENV%\Lib\site-packages;%CONDA_ENV%\python35.zip;%CONDA_ENV%\DLLs;%CONDA_ENV%
+set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%
 
 ctest -VV  || exit /B
 
-set PYTHONPATH=
-
 @rem Build and import pyarrow
-
-set PATH=%ARROW_HOME%\bin;%PATH%
+set PYTHONPATH=
 
 cd ..\..\python
 python setup.py build_ext --inplace  || exit /B
-python -c "import pyarrow"  || exit /B
-
 py.test pyarrow -v -s || exit /B
diff --git a/cpp/cmake_modules/FindFlatbuffers.cmake b/cpp/cmake_modules/FindFlatbuffers.cmake
index 7fa640ac9542f..804f4797241da 100644
--- a/cpp/cmake_modules/FindFlatbuffers.cmake
+++ b/cpp/cmake_modules/FindFlatbuffers.cmake
@@ -33,18 +33,18 @@
 
 if( NOT "${FLATBUFFERS_HOME}" STREQUAL "")
     file( TO_CMAKE_PATH "${FLATBUFFERS_HOME}" _native_path )
-    list( APPEND _flatbuffers_roots ${_native_path} )
+    list( APPEND _flatbuffers_roots "${_native_path}" )
 elseif ( Flatbuffers_HOME )
-    list( APPEND _flatbuffers_roots ${Flatbuffers_HOME} )
+    list( APPEND _flatbuffers_roots "${Flatbuffers_HOME}" )
 endif()
 
 # Try the parameterized roots, if they exist
 if ( _flatbuffers_roots )
     find_path( FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h
-        PATHS ${_flatbuffers_roots} NO_DEFAULT_PATH
+        PATHS "${_flatbuffers_roots}" NO_DEFAULT_PATH
         PATH_SUFFIXES "include" )
     find_library( FLATBUFFERS_LIBRARIES NAMES flatbuffers
-        PATHS ${_flatbuffers_roots} NO_DEFAULT_PATH
+        PATHS "${_flatbuffers_roots}" NO_DEFAULT_PATH
         PATH_SUFFIXES "lib" )
 else ()
     find_path( FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h )
@@ -52,7 +52,7 @@ else ()
 endif ()
 
 find_program(FLATBUFFERS_COMPILER flatc
-  ${FLATBUFFERS_HOME}/bin
+  "${FLATBUFFERS_HOME}/bin"
   /usr/local/bin
   /usr/bin
   NO_DEFAULT_PATH
@@ -60,9 +60,9 @@ find_program(FLATBUFFERS_COMPILER flatc
 
 if (FLATBUFFERS_INCLUDE_DIR AND FLATBUFFERS_LIBRARIES)
   set(FLATBUFFERS_FOUND TRUE)
-  get_filename_component( FLATBUFFERS_LIBS ${FLATBUFFERS_LIBRARIES} PATH )
+  get_filename_component( FLATBUFFERS_LIBS "${FLATBUFFERS_LIBRARIES}" PATH )
   set(FLATBUFFERS_LIB_NAME libflatbuffers)
-  set(FLATBUFFERS_STATIC_LIB ${FLATBUFFERS_LIBS}/${FLATBUFFERS_LIB_NAME}.a)
+  set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_LIBS}/${FLATBUFFERS_LIB_NAME}.a")
 else ()
   set(FLATBUFFERS_FOUND FALSE)
 endif ()
@@ -75,7 +75,7 @@ else ()
   if (NOT Flatbuffers_FIND_QUIETLY)
     set(FLATBUFFERS_ERR_MSG "Could not find the Flatbuffers library. Looked in ")
     if ( _flatbuffers_roots )
-      set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} in ${_flatbuffers_roots}.")
+      set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} ${_flatbuffers_roots}.")
     else ()
       set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} system search paths.")
     endif ()

From 8013cf3189f3be4785f3f88ee3fbcaea94bd4960 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 30 Apr 2017 23:04:01 -0400
Subject: [PATCH 0589/1644] ARROW-925: [GLib] Fix GArrowBufferReader test

GArrowReadable#read API was changed.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #622 from kou/glib-fix-buffer-reader-test and squashes the following commits:

e41374b [Kouhei Sutou] [GLib] Fix GArrowBufferReader test
---
 c_glib/test/test-buffer-reader.rb | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/c_glib/test/test-buffer-reader.rb b/c_glib/test/test-buffer-reader.rb
index b3517b230e421..d05ed062ebdb7 100644
--- a/c_glib/test/test-buffer-reader.rb
+++ b/c_glib/test/test-buffer-reader.rb
@@ -19,8 +19,7 @@ class TestBufferReader < Test::Unit::TestCase
   def test_read
     buffer = Arrow::Buffer.new("Hello World")
     buffer_reader = Arrow::BufferReader.new(buffer)
-    read_buffer = " " * 5
-    _success, n_read_bytes = buffer_reader.read(read_buffer)
-    assert_equal("Hello", read_buffer.byteslice(0, n_read_bytes))
+    read_buffer = buffer_reader.read(5)
+    assert_equal("Hello", read_buffer.data.to_s)
   end
 end

From c9e61cd77be59b0709610e60484df64c3810b4ca Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 1 May 2017 10:38:43 -0400
Subject: [PATCH 0590/1644] ARROW-926: Add wesm to KEYS

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #623 from wesm/add-wesm-keys and squashes the following commits:

5892a67 [Wes McKinney] Add wesm to KEYS
---
 KEYS | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 60 insertions(+), 1 deletion(-)

diff --git a/KEYS b/KEYS
index ad0f5cf6f4340..05862c8f643ca 100644
--- a/KEYS
+++ b/KEYS
@@ -2,7 +2,7 @@ This file contains the PGP keys of various developers.
 
 Users: pgp < KEYS
   gpg --import KEYS
-Developers: 
+Developers:
   pgp -kxa <your name> and append it to this file.
   (pgpk -ll <your name> && pgpk -xa <your name>) >> this file.
   (gpg --list-sigs <your name>
@@ -178,3 +178,62 @@ wfxeCKBeSpMuy3pvOnNy8uNYjNqizVlpNBx01I2R1MD8P14Pxteg6APi0jcusXrD
 s8g7c7dzdXM0lxreeXge8JSmxuwcCqVUswac6zbX4li03m/lov2YYxCwuw==
 =ESbR
 -----END PGP PUBLIC KEY BLOCK-----
+
+pub   4096R/1735623D 2017-05-01
+uid                  William Wesley McKinney (CODE SIGNING KEY) <wesm@apache.org>
+sig 3        1735623D 2017-05-01  William Wesley McKinney (CODE SIGNING KEY) <wesm@apache.org>
+sub   4096R/E83E9940 2017-05-01
+sig          1735623D 2017-05-01  William Wesley McKinney (CODE SIGNING KEY) <wesm@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBFkGqIQBEACuKfRxQ2zjpWtuEpKTr0qhpucl5h57cnbPG8M2t2eAbl7fD6mD
+ZyLePZEHSoNgUTqFTh8b850qD2b1loyuk6fx5mesweeWlSxt24Y5pXneH7WL/a8K
+H81jl+Qy5J8DfG8oEnlQp8bPjb3n8xFgNkpt09kxj9lRhDCK0+M0lN/JRGK2BfTx
+TCJWH2vC8Xh+apXmlSR5vohx7dj5RoFlIwNXsi+5JRkZCLoER8Fvozdq7qYNNmgL
+a8l38VnW5fQkx1Pl0mMBi0d4XwFCY6W5BfzfAU3t+ujb0a/6ZzFHiW6q53Fct4BM
+dMX91Xi73Myb3AF3x8dnv7E09dwXaShwUQu76WD/v7js1COS9o3SaCZfOdrJ9+KN
+bYc2zuzXCWtDQ1GU07ocq2Z8VnhGC/qAUwOY9K0JagFOx7xV3gc8bkWqFII0XeCK
+QBhKZHx7oFGz6bH2W/THLolbezwC7+0iuiWeDjY6y6Hk1/S25120wqdUfpa2QDlz
+5V+ayyF8Lt77CnowYeMuDSFZzBjg67SpbbkyZJwKUtTJBUOLKiJF37QCAYENHthB
+lmRgvOcCIic5cnJivgIs6Q7hCpFahWgr2g/6clu04YKFSaup+LU6F3UGvbKW6nnF
+HRSsVFkof0+Ni+yT/oiQUAYyCbrfptpgUZXrVuee8d4frbPfKeiWd4MTrwARAQAB
+tDxXaWxsaWFtIFdlc2xleSBNY0tpbm5leSAoQ09ERSBTSUdOSU5HIEtFWSkgPHdl
+c21AYXBhY2hlLm9yZz6JAjcEEwEKACEFAlkGqIQCGwMFCwkIBwMFFQoJCAsFFgID
+AQACHgECF4AACgkQ8QWIOhc1Yj1IQRAAm71yO273ulTxYlpFTN+CnTqTdxAQIGmc
+gfS55/XmjKfQySQTKOfQPafJe7MazbVG/jG5CZeKHEgHvM0qi8vnAezzeTKEDHPP
+Q1ziHyTt7ND+GbKChrLKA/lbgJkoBxKohyi6eQfz33cvh0fPsv8zej5M6+FAVJaA
+GCMUS/yIC0Oiq0JgYH38sPOhNtw3z8pODg6WjJFWKHXw5qGng11/3BtTVu5KXzqf
+85IJHqMgyOnU0r4mdKgqmSdaCpU/CMJlT3iflF5wN79c46FwAceCiYT8eJiWl1cB
+wAV/mRhTzWGQkWVhE+6EK6+PyuzkjJgGhMtv3zuzKKN8iOv3eb7xptzZydEPqRFf
+50f1cERfsf8um8W9IXQb60vrALyWwQFjF9B2oxsk28ZgzZ5ibA1xU9TJAS+iFo3e
+eITPZnxxT3jZ2WQVWIQB8/yn0sAg5mLQ+Clcghik60KQsjAVS27QrlMTimK6eXey
+tKTS4cw7LPo7GkuiBy3FuERX/ABXg8Wxd+EXOvLuZXNV/p9uBhU0w5tfaasnXFy0
+0NoKAVQ9ffW9MTV3CjrPakjHGLIzHgfFYuHnBdo67E3LR16kLcTusH8e8A3wYgbM
+/gbXNS1C+i31ATNWfHaZtAFrUdzUvotDVo2UTw4nRqy27XBM9NVS+EwfwiZLWoLH
+9gZEMGFQ0MW5Ag0EWQaohAEQAJnHTGcy0ol//23alysOuwYFHsS7PFizcCHuy4jv
+iB8YR5Y4Ts8nAgo8gz2O2m9bgNfbFHStDoqOUWTV7ILYv/CDZiNhvR+fAeWl3Gmt
+o72YYu85r+KZj22YfiXtfOb70IT7jYsTpjlgaqFUFHEHXzoa7EMscra6r+i1qDa9
+QfDjMIBaBCu/Q9CFfhHtIhBHV2Wt8IEJfgYSMHb24Db0RY+OSQixX4QRcVeSnJ5D
+I4ZjA3//o9DwthrJf+GxW/f8TGZy6vtertnxLJXGoHFPVuI895m7wSfzt1+/2nlc
+0obAMX4Q1yRYTOKQGPDeDZ7k5pxnhYkOHDf2gtY5wORw6vN9KR51YFJYXVmK+2zr
+P0fKr0AUG3C7CwQp6bDeYaTndon8S9VNyPypvJ7lpxKy/DIujdvbaJHF3i4rI+w0
+veScfkGtLDc37OeVQEBV4vnHcMvDIC2SEtli4BZjwOcihOv3DgtmQnAjkkAZLtys
+x/W4/MPoZiIWl0DnQev/ujwLkwHCYg/Oo7E70OKpdxDk/2cZyM1US2Uz2NQ4lo5O
+8M+F9sMWj2EPX/kJxZpb6N/+xJnKf4oIdJkaammVllX0TGtoxGOadPST9D8gtSCr
+yRdLMp0bB0+Ghbc+STGo78atg+J+HRvgzXG/gwaEiCIezuLB4W6rFjbldYfbeKTs
+OoAlABEBAAGJAh8EGAEKAAkFAlkGqIQCGwwACgkQ8QWIOhc1Yj23pw/+JNWYULOd
+uM4Khfyx3NgCLiX9VqmwZ7PQQsPKtxviQXdEgs+NJUrCePmjSV9Sf+exTZ4wqSTC
+BilGUppAJbO9avR2wRkYbdiYW+g0jDwAD9cyfAiDBSUiRTimKsKqYN0PbIKJ2Ric
+xvtBw4jW/f1lHkrySqOHetmFTe2ocXkFm8BjqDpt5XCoZa4ADcofNpRJYwVu0Uck
+8MQ/wYjoNRZiz0Sjx9vOBVW9ZKMWS6RgnPStsK3UJiG3c7c83kpDx8nk4bUp8seY
+cBjiViXh6QMXRPdlqsGEMiBVtyXF7Sy3cK3gUcH7808VmKMHEgWvq9MRrZoE0rLK
+74pZrEuWnwD6o77w4DCBtKJyDNlR23kLObS+1Ur7fIXe2yXmbqwEmjpSX4H2Teth
+77PU7nKMAkFsPJDNI7K/kEy3x7KM3G1gIcWaz3pL5gthLV+H3RfIojrK1hS7ZSSI
+gCzYEkQCMsigT5YTgK5+n0I4U7zoDBd1sttwK2FahvuCKUDwc+ZiX/ciYiAjUMb9
+6yTNHlNr/H31EWVZMEd7+fhFZWXJjFsQD11GkXvy6vMBn3Kq+Vd7Yr4CJUGTV3rW
+bWo1vt2ED7h5rbZTrS1UssxLUpy5iXrjyGwn2h/Ei9MzXpNvH8p2raf0eQ0Qn65Q
+UoUryip3RD0yaMCyL/IK3KoPt74f2eJsFwM=
+=feO2
+-----END PGP PUBLIC KEY BLOCK-----
\ No newline at end of file

From da523ce72524de6243b8ea3c40cf50f92d60ac3e Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Mon, 1 May 2017 16:38:21 -0400
Subject: [PATCH 0591/1644] ARROW-928: [C++] Detect supported MSVC versions

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #625 from MaxRis/ARROW-928 and squashes the following commits:

db81a27 [Max Risuhin] ARROW-928: [C++] Detect supported MSVC versions
---
 cpp/cmake_modules/CompilerInfo.cmake | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index 3c603918a82ec..21e2dafba2e24 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -17,7 +17,11 @@
 #
 # Sets COMPILER_FAMILY to 'clang' or 'gcc'
 # Sets COMPILER_VERSION to the version
-execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
+if (NOT MSVC)
+  set(COMPILER_GET_VERSION_SWITCH "-v")
+endif()
+
+execute_process(COMMAND "${CMAKE_CXX_COMPILER}" ${COMPILER_GET_VERSION_SWITCH}
                 ERROR_VARIABLE COMPILER_VERSION_FULL)
 message(INFO "Compiler version: ${COMPILER_VERSION_FULL}")
 message(INFO "Compiler id: ${CMAKE_CXX_COMPILER_ID}")
@@ -25,6 +29,13 @@ string(TOLOWER "${COMPILER_VERSION_FULL}" COMPILER_VERSION_FULL_LOWER)
 
 if(MSVC)
   set(COMPILER_FAMILY "msvc")
+  if ("${COMPILER_VERSION_FULL}" MATCHES ".*Microsoft \\(R\\) C/C\\+\\+ Optimizing Compiler Version 19.*x64")
+    string(REGEX REPLACE ".*Optimizing Compiler Version ([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+).*" "\\1"
+      COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+  elseif(NOT "${COMPILER_VERSION_FULL}" STREQUAL "")
+    message(FATAL_ERROR "Not supported MSVC compiler:\n${COMPILER_VERSION_FULL}\n"
+      "Supported MSVC versions: Visual Studio 2015 2017 x64")
+  endif()
 
 # clang on Linux and Mac OS X before 10.9
 elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")

From 569426b917007d7eb8f238d657184d5789527646 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Tue, 2 May 2017 11:27:19 -0400
Subject: [PATCH 0592/1644] ARROW-930: javadoc generation fails with java 8

Author: Julien Le Dem <julien@apache.org>

Closes #627 from julienledem/fix_javadoc and squashes the following commits:

4160d5b [Julien Le Dem] ARROW-930: javadoc generation fails with java 8
---
 .../arrow/memory/AllocationManager.java       | 22 ++++---
 .../arrow/memory/AllocationReservation.java   | 13 ++--
 .../apache/arrow/memory/BufferAllocator.java  | 10 ++-
 .../apache/arrow/memory/BufferManager.java    |  8 ---
 .../arrow/memory/util/HistoricalLog.java      | 27 +++-----
 .../templates/AbstractFieldReader.java        | 63 +++++++++----------
 .../AbstractPromotableFieldWriter.java        |  7 +--
 .../src/main/codegen/templates/ArrowType.java |  2 +-
 .../main/codegen/templates/BaseWriter.java    |  6 +-
 .../main/codegen/templates/ComplexCopier.java |  4 +-
 .../codegen/templates/FixedValueVectors.java  |  2 +-
 .../templates/NullableValueVectors.java       |  7 +--
 .../templates/VariableLengthVectors.java      |  2 +-
 .../arrow/vector/SchemaChangeCallBack.java    |  1 +
 .../org/apache/arrow/vector/ValueVector.java  | 29 ++++++---
 .../arrow/vector/VariableWidthVector.java     |  4 +-
 .../org/apache/arrow/vector/VectorLoader.java |  2 +-
 .../complex/AbstractContainerVector.java      |  2 +
 .../vector/complex/AbstractMapVector.java     | 18 ++++--
 .../complex/BaseRepeatedValueVector.java      | 11 ++--
 .../vector/complex/FixedSizeListVector.java   | 13 ++--
 .../vector/complex/RepeatedValueVector.java   | 15 ++---
 .../RepeatedVariableWidthVectorLike.java      |  2 +-
 .../apache/arrow/vector/file/ArrowWriter.java | 10 +--
 .../apache/arrow/vector/file/ReadChannel.java |  7 +++
 .../vector/stream/ArrowStreamReader.java      | 14 +++--
 .../vector/stream/MessageSerializer.java      | 47 +++++++++++++-
 .../arrow/vector/types/pojo/Schema.java       |  2 +-
 .../arrow/vector/util/MapWithOrdinal.java     | 14 +++--
 .../util/OversizedAllocationException.java    |  3 +-
 .../org/apache/arrow/vector/util/Text.java    | 50 +++++++++++----
 .../apache/arrow/vector/util/Validator.java   |  4 ++
 32 files changed, 253 insertions(+), 168 deletions(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index 683752e6a4980..70ca1dc32a1b3 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -18,15 +18,7 @@
 
 package org.apache.arrow.memory;
 
-import com.google.common.base.Preconditions;
-
-import io.netty.buffer.ArrowBuf;
-import io.netty.buffer.PooledByteBufAllocatorL;
-import io.netty.buffer.UnsafeDirectLittleEndian;
-
-import org.apache.arrow.memory.BaseAllocator.Verbosity;
-import org.apache.arrow.memory.util.AutoCloseableLock;
-import org.apache.arrow.memory.util.HistoricalLog;
+import static org.apache.arrow.memory.BaseAllocator.indent;
 
 import java.util.IdentityHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -34,7 +26,15 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import static org.apache.arrow.memory.BaseAllocator.indent;
+import org.apache.arrow.memory.BaseAllocator.Verbosity;
+import org.apache.arrow.memory.util.AutoCloseableLock;
+import org.apache.arrow.memory.util.HistoricalLog;
+
+import com.google.common.base.Preconditions;
+
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.PooledByteBufAllocatorL;
+import io.netty.buffer.UnsafeDirectLittleEndian;
 
 /**
  * Manages the relationship between one or more allocators and a particular UDLE. Ensures that
@@ -328,6 +328,8 @@ private void inc() {
      * Decrement the ledger's reference count. If the ledger is decremented to zero, this ledger
      * should release its
      * ownership back to the AllocationManager
+     * @param decrement amout to decrease the reference count by
+     * @return the new reference count
      */
     public int decrement(int decrement) {
       allocator.assertOpen();
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
index 7f5aa313779a7..b0ce574deefcb 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java
@@ -30,13 +30,14 @@
  * For the purposes of airtight memory accounting, the reservation must be close()d whether it is
  * used or not.
  * This is not threadsafe.
+ * </p>
  */
 public interface AllocationReservation extends AutoCloseable {
 
   /**
    * Add to the current reservation.
-   * <p>
-   * <p>Adding may fail if the allocator is not allowed to consume any more space.
+   *
+   * <p>Adding may fail if the allocator is not allowed to consume any more space.</p>
    *
    * @param nBytes the number of bytes to add
    * @return true if the addition is possible, false otherwise
@@ -46,8 +47,8 @@ public interface AllocationReservation extends AutoCloseable {
 
   /**
    * Requests a reservation of additional space.
-   * <p>
-   * <p>The implementation of the allocator's inner class provides this.
+   *
+   * <p>The implementation of the allocator's inner class provides this.</p>
    *
    * @param nBytes the amount to reserve
    * @return true if the reservation can be satisfied, false otherwise
@@ -56,9 +57,9 @@ public interface AllocationReservation extends AutoCloseable {
 
   /**
    * Allocate a buffer whose size is the total of all the add()s made.
-   * <p>
+   *
    * <p>The allocation request can still fail, even if the amount of space
-   * requested is available, if the allocation cannot be made contiguously.
+   * requested is available, if the allocation cannot be made contiguously.</p>
    *
    * @return the buffer, or null, if the request cannot be satisfied
    * @throws IllegalStateException if called called more than once
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
index c05e9acb0aa96..8a40441863889 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -70,9 +70,9 @@ public interface BufferAllocator extends AutoCloseable {
 
   /**
    * Close and release all buffers generated from this buffer pool.
-   * <p>
+   *
    * <p>When assertions are on, complains if there are any outstanding buffers; to avoid
-   * that, release all buffers before the allocator is closed.
+   * that, release all buffers before the allocator is closed.</p>
    */
   @Override
   public void close();
@@ -116,7 +116,8 @@ public interface BufferAllocator extends AutoCloseable {
   /**
    * Create an allocation reservation. A reservation is a way of building up
    * a request for a buffer whose size is not known in advance. See
-   * {@see AllocationReservation}.
+   *
+   * @see AllocationReservation
    *
    * @return the newly created reservation
    */
@@ -127,6 +128,7 @@ public interface BufferAllocator extends AutoCloseable {
    * special because we don't
    * worry about them leaking or managing reference counts on them since they don't actually
    * point to any memory.
+   * @return the empty buffer
    */
   public ArrowBuf getEmpty();
 
@@ -134,6 +136,7 @@ public interface BufferAllocator extends AutoCloseable {
    * Return the name of this allocator. This is a human readable name that can help debugging.
    * Typically provides
    * coordinates about where this allocator was created
+   * @return the name of the allocator
    */
   public String getName();
 
@@ -142,6 +145,7 @@ public interface BufferAllocator extends AutoCloseable {
    * that an allocator is
    * over its limit, all consumers of that allocator should aggressively try to addrss the
    * overlimit situation.
+   * @return whether or not this allocator (or one if its parents) is over its limits
    */
   public boolean isOverLimit();
 
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
index 2fe763e10aff9..3075ebeef996f 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java
@@ -25,14 +25,6 @@
  * re-allocation the old buffer will be freed. Managing a list of these buffers
  * prevents some parts of the system from needing to define a correct location
  * to place the final call to free them.
- * <p>
- * The current uses of these types of buffers are within the pluggable components of Drill.
- * In UDFs, memory management should not be a concern. We provide access to re-allocatable
- * ArrowBufs to give UDF writers general purpose buffers we can account for. To prevent the need
- * for UDFs to contain boilerplate to close all of the buffers they request, this list
- * is tracked at a higher level and all of the buffers are freed once we are sure that
- * the code depending on them is done executing (currently {@link FragmentContext}
- * and {@link QueryContext}.
  */
 public interface BufferManager extends AutoCloseable {
 
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
index c464598bfb856..0452dc9adf256 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
@@ -18,11 +18,11 @@
 
 package org.apache.arrow.memory.util;
 
-import org.slf4j.Logger;
-
 import java.util.Arrays;
 import java.util.LinkedList;
 
+import org.slf4j.Logger;
+
 /**
  * Utility class that can be used to log activity within a class
  * for later logging and debugging. Supports recording events and
@@ -98,19 +98,20 @@ public synchronized void recordEvent(final String noteFormat, Object... args) {
    * events with their stack traces.
    *
    * @param sb {@link StringBuilder} to write to
+   * @param includeStackTrace whether to include the stacktrace of each event in the history
    */
   public void buildHistory(final StringBuilder sb, boolean includeStackTrace) {
     buildHistory(sb, 0, includeStackTrace);
   }
 
   /**
-   *
-   * @param sb
-   * @param indent
-   * @param includeStackTrace
+   * build the history and write it to sb
+   * @param sb output
+   * @param indent starting indent (usually "")
+   * @param includeStackTrace whether to include the stacktrace of each event.
    */
-  public synchronized void buildHistory(final StringBuilder sb, int indent, boolean
-      includeStackTrace) {
+  public synchronized void buildHistory(
+      final StringBuilder sb, int indent, boolean includeStackTrace) {
     final char[] indentation = new char[indent];
     final char[] innerIndentation = new char[indent + 2];
     Arrays.fill(indentation, ' ');
@@ -150,16 +151,6 @@ public synchronized void buildHistory(final StringBuilder sb, int indent, boolea
     }
   }
 
-  /**
-   * Write the history of this object to the given {@link StringBuilder}. The history
-   * includes the identifying string provided at construction time, and all the recorded
-   * events with their stack traces.
-   *
-   * @param sb {@link StringBuilder} to write to
-   * @param additional an extra string that will be written between the identifying
-   *     information and the history; often used for a current piece of state
-   */
-
   /**
    * Write the history of this object to the given {@link Logger}. The history
    * includes the identifying string provided at construction time, and all the recorded
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
index e0d0fc9715ba2..79d4c122f0e4e 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -26,16 +26,19 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
 @SuppressWarnings("unused")
 abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
-  
+
   AbstractFieldReader(){
     super();
   }
 
   /**
    * Returns true if the current value of the reader is not null
-   * @return
+   * @return whether the current value is set
    */
   public boolean isSet() {
     return true;
@@ -52,78 +55,74 @@ public Field getField() {
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-  
-  public ${friendlyType} read${safeType}(int arrayIndex){
+  public ${friendlyType} read${safeType}(int arrayIndex) {
     fail("read${safeType}(int arrayIndex)");
     return null;
   }
-  
-  public ${friendlyType} read${safeType}(){
+
+  public ${friendlyType} read${safeType}() {
     fail("read${safeType}()");
     return null;
   }
-  
+
   </#list>
-  
-  public void copyAsValue(MapWriter writer){
+  public void copyAsValue(MapWriter writer) {
     fail("CopyAsValue MapWriter");
   }
-  public void copyAsField(String name, MapWriter writer){
+
+  public void copyAsField(String name, MapWriter writer) {
     fail("CopyAsField MapWriter");
   }
 
-  public void copyAsField(String name, ListWriter writer){
+  public void copyAsField(String name, ListWriter writer) {
     fail("CopyAsFieldList");
   }
-  
+
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign boxedType = (minor.boxedType!type.boxedType) />
-
-  public void read(${name}Holder holder){
+  public void read(${name}Holder holder) {
     fail("${name}");
   }
 
-  public void read(Nullable${name}Holder holder){
+  public void read(Nullable${name}Holder holder) {
     fail("${name}");
   }
-  
-  public void read(int arrayIndex, ${name}Holder holder){
+
+  public void read(int arrayIndex, ${name}Holder holder) {
     fail("Repeated${name}");
   }
-  
-  public void read(int arrayIndex, Nullable${name}Holder holder){
+
+  public void read(int arrayIndex, Nullable${name}Holder holder) {
     fail("Repeated${name}");
   }
-  
-  public void copyAsValue(${name}Writer writer){
+
+  public void copyAsValue(${name}Writer writer) {
     fail("CopyAsValue${name}");
   }
-  public void copyAsField(String name, ${name}Writer writer){
+
+  public void copyAsField(String name, ${name}Writer writer) {
     fail("CopyAsField${name}");
   }
+
   </#list></#list>
-  
-  public FieldReader reader(String name){
+  public FieldReader reader(String name) {
     fail("reader(String name)");
     return null;
   }
 
-  public FieldReader reader(){
+  public FieldReader reader() {
     fail("reader()");
     return null;
-    
   }
-  
-  public int size(){
+
+  public int size() {
     fail("size()");
     return -1;
   }
-  
-  private void fail(String name){
+
+  private void fail(String name) {
     throw new IllegalArgumentException(String.format("You tried to read a [%s] type when you are using a field reader of type [%s].", name, this.getClass().getSimpleName()));
   }
-  
-  
 }
 
 
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 60dd0c7b7adf8..ada0b1d5c7816 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -39,14 +39,13 @@
 abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
   /**
    * Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type
-   * @param type
-   * @return
+   * @param type the type of the values we want to write
+   * @return the corresponding field writer
    */
   abstract protected FieldWriter getWriter(MinorType type);
 
   /**
-   * Return the current FieldWriter
-   * @return
+   * @return the current FieldWriter
    */
   abstract protected FieldWriter getWriter();
 
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index a9e875a2095f7..dc99aad0bb3a2 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -100,7 +100,7 @@ private ArrowTypeID(byte flatbufType) {
   /**
    * to visit the ArrowTypes
    * <code>
-   *   type.accept(new ArrowTypeVisitor<Type>() {
+   *   type.accept(new ArrowTypeVisitor&lt;Type&gt;() {
    *   ...
    *   });
    * </code>
diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java
index 08bd39eae2358..3da02b00a0dbf 100644
--- a/java/vector/src/main/codegen/templates/BaseWriter.java
+++ b/java/vector/src/main/codegen/templates/BaseWriter.java
@@ -30,7 +30,7 @@
  * File generated from ${.template_name} using FreeMarker.
  */
 @SuppressWarnings("unused")
-  public interface BaseWriter extends AutoCloseable, Positionable {
+public interface BaseWriter extends AutoCloseable, Positionable {
   int getValueCapacity();
 
   public interface MapWriter extends BaseWriter {
@@ -39,12 +39,12 @@ public interface MapWriter extends BaseWriter {
 
     /**
      * Whether this writer is a map writer and is empty (has no children).
-     * 
+     *
      * <p>
      *   Intended only for use in determining whether to add dummy vector to
      *   avoid empty (zero-column) schema, as in JsonReader.
      * </p>
-     * 
+     * @return whether the map is empty
      */
     boolean isEmptyMap();
 
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index 89368ce6e0b96..fb7ae0f2ef57e 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -34,8 +34,8 @@ public class ComplexCopier {
 
   /**
    * Do a deep copy of the value in input into output
-   * @param in
-   * @param out
+   * @param input field to read from
+   * @param output field to write to
    */
   public static void copy(FieldReader input, FieldWriter output) {
     writeValue(input, output);
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 5c09e30c71487..05faaae1e9e2f 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -150,7 +150,7 @@ public boolean allocateNewSafe() {
    *
    * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
    *
-   * @param valueCount
+   * @param valueCount the number of values to allocate for
    * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
    */
   @Override
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 31adc2bdd0789..76d2bad36bc18 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -421,9 +421,8 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m
     /**
      * Get the element at the specified position.
      *
-     * @param   index   position of the value
-     * @return  value of the element, if not null
-     * @throws  NullValueException if the value is null
+     * @param  index   position of the value
+     * @return value of the element, if not null
      */
     public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
       if (isNull(index)) {
@@ -509,7 +508,7 @@ public void setIndexDefined(int index){
      * Set the variable length element at the specified index to the supplied byte array.
      *
      * @param index   position of the bit to set
-     * @param bytes   array of bytes to write
+     * @param value   array of bytes (or int if smaller than 4 bytes) to write
      */
     public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
       setCount++;
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index 11f0cc894d004..3d933addb6208 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -139,7 +139,7 @@ public int getCurrentSizeInBytes() {
 
   /**
    * Return the number of bytes contained in the current var len byte vector.
-   * @return
+   * @return the number of bytes contained in the current var len byte vector
    */
   public int getVarByteLength(){
     final int valueCount = getAccessor().getValueCount();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java b/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
index fc0a066749a91..6fdcda20480f8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
@@ -42,6 +42,7 @@ public void doWork() {
   /**
    * Returns the value of schema-changed state, <strong>resetting</strong> the
    * schema-changed state to {@code false}.
+   * @return the previous schema-changed state
    */
   public boolean getSchemaChangedAndReset() {
     final boolean current = schemaChanged;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 685b0be010a08..2e83836b64626 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -52,7 +52,7 @@
  *
  * This interface "should" strive to guarantee this order of operation:
  * <blockquote>
- * allocate > mutate > setvaluecount > access > clear (or allocate to start the process over).
+ * allocate &gt; mutate &gt; setvaluecount &gt; access &gt; clear (or allocate to start the process over).
  * </blockquote>
  */
 public interface ValueVector extends Closeable, Iterable<ValueVector> {
@@ -84,6 +84,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * Returns the maximum number of values that can be stored in this vector instance.
+   * @return the maximum number of values that can be stored in this vector instance.
    */
   int getValueCapacity();
 
@@ -100,13 +101,16 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * Get information about how this field is materialized.
+   * @return the field corresponding to this vector
    */
   Field getField();
 
   MinorType getMinorType();
 
   /**
-   * Returns a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
+   * to transfer quota responsibility
+   * @param allocator the target allocator
+   * @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
    * the same type.
    */
   TransferPair getTransferPair(BufferAllocator allocator);
@@ -116,31 +120,33 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
   TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack);
 
   /**
-   * Returns a new {@link org.apache.arrow.vector.util.TransferPair transfer pair} that is used to transfer underlying
+   * makes a new transfer pair used to transfer underlying buffers
+   * @param target the target for the transfer
+   * @return a new {@link org.apache.arrow.vector.util.TransferPair transfer pair} that is used to transfer underlying
    * buffers into the target vector.
    */
   TransferPair makeTransferPair(ValueVector target);
 
   /**
-   * Returns an {@link org.apache.arrow.vector.ValueVector.Accessor accessor} that is used to read from this vector
+   * @return an {@link org.apache.arrow.vector.ValueVector.Accessor accessor} that is used to read from this vector
    * instance.
    */
   Accessor getAccessor();
 
   /**
-   * Returns an {@link org.apache.arrow.vector.ValueVector.Mutator mutator} that is used to write to this vector
+   * @return an {@link org.apache.arrow.vector.ValueVector.Mutator mutator} that is used to write to this vector
    * instance.
    */
   Mutator getMutator();
 
   /**
-   * Returns a {@link org.apache.arrow.vector.complex.reader.FieldReader field reader} that supports reading values
+   * @return a {@link org.apache.arrow.vector.complex.reader.FieldReader field reader} that supports reading values
    * from this vector.
    */
   FieldReader getReader();
 
   /**
-   * Returns the number of bytes that is used by this vector instance.
+   * @return the number of bytes that is used by this vector instance.
    */
   int getBufferSize();
 
@@ -177,21 +183,23 @@ interface Accessor {
      *
      * @param index
      *          Index of the value to get
+     * @return the friendly java type
      */
     Object getObject(int index);
 
     /**
-     * Returns the number of values that is stored in this vector.
+     * @return the number of values that is stored in this vector.
      */
     int getValueCount();
 
     /**
-     * Returns true if the value at the given index is null, false otherwise.
+     * @param index the index to check for nullity
+     * @return true if the value at the given index is null, false otherwise.
      */
     boolean isNull(int index);
 
     /**
-     * Returns the number of null values
+     * @return the number of null values
      */
     int getNullCount();
   }
@@ -214,6 +222,7 @@ interface Mutator {
 
     /**
      * @deprecated  this has nothing to do with value vector abstraction and should be removed.
+     * @param values the number of values to generate
      */
     @Deprecated
     void generateTestData(int values);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
index 971a241adafc2..ed164b548b5bd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
@@ -17,8 +17,6 @@
  */
 package org.apache.arrow.vector;
 
-import io.netty.buffer.ArrowBuf;
-
 public interface VariableWidthVector extends ValueVector{
 
   /**
@@ -31,7 +29,7 @@ public interface VariableWidthVector extends ValueVector{
 
   /**
    * Provide the maximum amount of variable width bytes that can be stored in this vector.
-   * @return
+   * @return the byte capacity of this vector
    */
   int getByteCapacity();
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 76de250e0e972..33a608cd92922 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -50,7 +50,7 @@ public VectorLoader(VectorSchemaRoot root) {
   /**
    * Loads the record batch in the vectors
    * will not close the record batch
-   * @param recordBatch
+   * @param recordBatch the batch to load
    */
   public void load(ArrowRecordBatch recordBatch) {
     Iterator<ArrowBuf> buffers = recordBatch.getBuffers().iterator();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 71f2bea5b8fe1..7f8e6796285fd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -58,6 +58,8 @@ public BufferAllocator getAllocator() {
 
   /**
    * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
+   * @param name the name of the child to return
+   * @return the corresponding FieldVector
    */
   public FieldVector getChild(String name) {
     return getChild(name, FieldVector.class);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index 15e8a5bc624ac..4b6d82cc8b291 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -151,14 +151,19 @@ private boolean nullFilled(ValueVector vector) {
 
   /**
    * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given ordinal identifier.
+   * @param id the ordinal of the child to return
+   * @return the corresponding child
    */
   public ValueVector getChildByOrdinal(int id) {
     return vectors.getByOrdinal(id);
   }
 
   /**
-   * Returns a {@link org.apache.arrow.vector.ValueVector} instance of subtype of <T> corresponding to the given
+   * Returns a {@link org.apache.arrow.vector.ValueVector} instance of subtype of T corresponding to the given
    * field name if exists or null.
+   * @param name the name of the child to return
+   * @param clazz the expected type of the child
+   * @return the child corresponding to this name
    */
   @Override
   public <T extends FieldVector> T getChild(String name, Class<T> clazz) {
@@ -186,6 +191,8 @@ protected ValueVector add(String childName, FieldType fieldType) {
    * Inserts the vector with the given name if it does not exist else replaces it with the new value.
    *
    * Note that this method does not enforce any vector type check nor throws a schema change exception.
+   * @param name the name of the child to add
+   * @param vector the vector to add as a child
    */
   protected void putChild(String name, FieldVector vector) {
     putVector(name, vector);
@@ -208,7 +215,7 @@ protected void putVector(String name, FieldVector vector) {
   }
 
   /**
-   * Returns a sequence of underlying child vectors.
+   * @return a sequence of underlying child vectors.
    */
   protected List<FieldVector> getChildren() {
     int size = vectors.size();
@@ -228,7 +235,7 @@ protected List<String> getChildFieldNames() {
   }
 
   /**
-   * Returns the number of underlying child vectors.
+   * @return the number of underlying child vectors.
    */
   @Override
   public int size() {
@@ -241,7 +248,7 @@ public Iterator<ValueVector> iterator() {
   }
 
   /**
-   * Returns a list of scalar child vectors recursing the entire vector hierarchy.
+   * @return a list of scalar child vectors recursing the entire vector hierarchy.
    */
   public List<ValueVector> getPrimitiveVectors() {
     final List<ValueVector> primitiveVectors = Lists.newArrayList();
@@ -257,7 +264,8 @@ public List<ValueVector> getPrimitiveVectors() {
   }
 
   /**
-   * Returns a vector with its corresponding ordinal mapping if field exists or null.
+   * @param name the name of the child to return
+   * @return a vector with its corresponding ordinal mapping if field exists or null.
    */
   @Override
   public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index c9a9319c69154..5ff4c2c8172c3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -20,10 +20,6 @@
 import java.util.Collections;
 import java.util.Iterator;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -35,6 +31,11 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
   public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -151,7 +152,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
   }
 
   /**
-   * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
+   * @return 1 if inner vector is explicitly set via #addOrGetVector else 0
    */
   public int size() {
     return vector == DEFAULT_DATA_VECTOR ? 0:1;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 7ac9f3bd5137f..0dceeed50d484 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -26,11 +26,6 @@
 import java.util.List;
 import java.util.Objects;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
@@ -53,6 +48,12 @@
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 import org.apache.arrow.vector.util.TransferPair;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+
 public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {
 
   private FieldVector vector;
@@ -236,7 +237,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
   }
 
   /**
-   * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
+   * @return 1 if inner vector is explicitly set via #addOrGetVector else 0
    */
   public int size() {
     return vector == ZeroVector.INSTANCE ? 0 : 1;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index b01a4e7cf49d4..de58eda0b11a2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -33,14 +33,12 @@ public interface RepeatedValueVector extends ValueVector {
   final static int DEFAULT_REPEAT_PER_RECORD = 5;
 
   /**
-   * Returns the underlying offset vector or null if none exists.
-   *
-   * TODO(DRILL-2995): eliminate exposing low-level interfaces.
+   * @return the underlying offset vector or null if none exists.
    */
   UInt4Vector getOffsetVector();
 
   /**
-   * Returns the underlying data vector or null if none exists.
+   * @return the underlying data vector or null if none exists.
    */
   ValueVector getDataVector();
 
@@ -52,22 +50,21 @@ public interface RepeatedValueVector extends ValueVector {
 
   interface RepeatedAccessor extends ValueVector.Accessor {
     /**
-     * Returns total number of cells that vector contains.
-     *
      * The result includes empty, null valued cells.
+     * @return total number of cells that vector contains.
      */
     int getInnerValueCount();
 
 
     /**
-     * Returns number of cells that the value at the given index contains.
+     * @param index the index of the value for which we want the size
+     * @return number of cells that the value at the given index contains.
      */
     int getInnerValueCountAt(int index);
 
     /**
-     * Returns true if the value at the given index is empty, false otherwise.
-     *
      * @param index  value index
+     * @return true if the value at the given index is empty, false otherwise.
      */
     boolean isEmpty(int index);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
index 93b744e108719..29f9d75c74671 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
@@ -29,7 +29,7 @@ public interface RepeatedVariableWidthVectorLike {
 
   /**
    * Provide the maximum amount of variable width bytes that can be stored int his vector.
-   * @return
+   * @return the byte capacity
    */
   int getByteCapacity();
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 60a6afb565318..1716287f722ff 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -25,8 +25,6 @@
 import java.util.List;
 import java.util.Map;
 
-import com.google.common.collect.ImmutableList;
-
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
@@ -42,6 +40,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.ImmutableList;
+
 public abstract class ArrowWriter implements AutoCloseable {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowWriter.class);
@@ -62,9 +62,9 @@ public abstract class ArrowWriter implements AutoCloseable {
   /**
    * Note: fields are not closed when the writer is closed
    *
-   * @param root
-   * @param provider
-   * @param out
+   * @param root the vectors to write to the output
+   * @param provider where to find the dictionaries
+   * @param out the output where to write
    */
   protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
     this.unloader = new VectorUnloader(root);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
index b062f3826eab3..87450e38f6852 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
@@ -42,6 +42,9 @@ public ReadChannel(ReadableByteChannel in) {
   /**
    * Reads bytes into buffer until it is full (buffer.remaining() == 0). Returns the
    * number of bytes read which can be less than full if there are no more.
+   * @param buffer The buffer to read to
+   * @return the number of byte read
+   * @throws IOException if nit enough bytes left to read
    */
   public int readFully(ByteBuffer buffer) throws IOException {
     LOGGER.debug("Reading buffer with size: " + buffer.remaining());
@@ -58,6 +61,10 @@ public int readFully(ByteBuffer buffer) throws IOException {
 
   /**
    * Reads up to len into buffer. Returns bytes read.
+   * @param buffer the buffer to read to
+   * @param l the amount of bytes to read
+   * @return the number of bytes read
+   * @throws IOException if nit enough bytes left to read
    */
   public int readFully(ArrowBuf buffer, int l) throws IOException {
     int n = readFully(buffer.nioBuffer(buffer.writerIndex(), l));
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
index 2deef37cd4e56..641978a516ae4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
@@ -17,17 +17,17 @@
  */
 package org.apache.arrow.vector.stream;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.file.ArrowReader;
 import org.apache.arrow.vector.file.ReadChannel;
 import org.apache.arrow.vector.schema.ArrowMessage;
 import org.apache.arrow.vector.types.pojo.Schema;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-
 /**
  * This classes reads from an input stream and produces ArrowRecordBatches.
  */
@@ -35,6 +35,8 @@ public class ArrowStreamReader extends ArrowReader<ReadChannel> {
 
     /**
     * Constructs a streaming read, reading bytes from 'in'. Non-blocking.
+    * @param in the stream to read from
+    * @param allocator to allocate new buffers
     */
     public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) {
         super(new ReadChannel(in), allocator);
@@ -46,6 +48,8 @@ public ArrowStreamReader(InputStream in, BufferAllocator allocator) {
 
     /**
      * Reads the schema message from the beginning of the stream.
+     * @param in to allocate new buffers
+     * @return the deserialized arrow schema
      */
     @Override
     protected Schema readSchema(ReadChannel in) throws IOException {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 228ab613466d2..2fd93749976c6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -22,9 +22,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import com.google.flatbuffers.FlatBufferBuilder;
-
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.flatbuf.Buffer;
 import org.apache.arrow.flatbuf.DictionaryBatch;
 import org.apache.arrow.flatbuf.FieldNode;
@@ -43,6 +40,10 @@
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.Schema;
 
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import io.netty.buffer.ArrowBuf;
+
 /**
  * Utility class for serializing Messages. Messages are all serialized a similar way.
  * 1. 4 byte little endian message header prefix
@@ -69,6 +70,10 @@ public static int bytesToInt(byte[] bytes) {
 
   /**
    * Serialize a schema object.
+   * @param out where to write the schema
+   * @param schema the object to serialize to out
+   * @return the resulting size of the serialized schema
+   * @throws IOException if something went wrong
    */
   public static long serialize(WriteChannel out, Schema schema) throws IOException {
     FlatBufferBuilder builder = new FlatBufferBuilder();
@@ -81,6 +86,9 @@ public static long serialize(WriteChannel out, Schema schema) throws IOException
 
   /**
    * Deserializes a schema object. Format is from serialize().
+   * @param in the channel to deserialize from
+   * @return the deserialized object
+   * @throws IOException if something went wrong
    */
   public static Schema deserializeSchema(ReadChannel in) throws IOException {
     Message message = deserializeMessage(in);
@@ -98,6 +106,10 @@ public static Schema deserializeSchema(ReadChannel in) throws IOException {
 
   /**
    * Serializes an ArrowRecordBatch. Returns the offset and length of the written batch.
+   * @param out where to write the batch
+   * @param batch the object to serialize to out
+   * @return the serialized block metadata
+   * @throws IOException if something went wrong
    */
   public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
           throws IOException {
@@ -153,6 +165,11 @@ private static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch)
 
   /**
    * Deserializes a RecordBatch
+   * @param in the channel to deserialize from
+   * @param message the object to derialize to
+   * @param alloc to allocate buffers
+   * @return the deserialized object
+   * @throws IOException if something went wrong
    */
   private static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message message, BufferAllocator alloc)
       throws IOException {
@@ -171,6 +188,11 @@ private static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message m
   /**
    * Deserializes a RecordBatch knowing the size of the entire message up front. This
    * minimizes the number of reads to the underlying stream.
+   * @param in the channel to deserialize from
+   * @param block the object to derialize to
+   * @param alloc to allocate buffers
+   * @return the deserialized object
+   * @throws IOException if something went wrong
    */
   public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock block,
       BufferAllocator alloc) throws IOException {
@@ -231,6 +253,10 @@ private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB
 
   /**
    * Serializes a dictionary ArrowRecordBatch. Returns the offset and length of the written batch.
+   * @param out where to serialize
+   * @param batch the batch to serialize
+   * @return the metadata of the serialized block
+   * @throws IOException if something went wrong
    */
   public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch) throws IOException {
     long start = out.getCurrentPosition();
@@ -264,6 +290,11 @@ public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch)
 
   /**
    * Deserializes a DictionaryBatch
+   * @param in where to read from
+   * @param message the message message metadata to deserialize
+   * @param alloc the allocator for new buffers
+   * @return the corresponding dictionary batch
+   * @throws IOException if something went wrong
    */
   private static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
                                                                  Message message,
@@ -284,6 +315,11 @@ private static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
   /**
    * Deserializes a DictionaryBatch knowing the size of the entire message up front. This
    * minimizes the number of reads to the underlying stream.
+   * @param in where to read from
+   * @param block block metadata for deserializing
+   * @param alloc to allocate new buffers
+   * @return the corresponding dictionary
+   * @throws IOException if something went wrong
    */
   public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
                                                                 ArrowBlock block,
@@ -331,6 +367,11 @@ public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocat
 
   /**
    * Serializes a message header.
+   * @param builder to write the flatbuf to
+   * @param headerType headerType field
+   * @param headerOffset header offset field
+   * @param bodyLength body length field
+   * @return the corresponding ByteBuffer
    */
   private static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte headerType,
       int headerOffset, int bodyLength) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
index c33bd6e6e61b0..cede3e801c1e4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -44,7 +44,7 @@
 public class Schema {
 
   /**
-   * @param the list of the fields
+   * @param fields the list of the fields
    * @param name the name of the field to return
    * @return the corresponding field
    * @throws IllegalArgumentException if the field was not found
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
index d7f9d382e4865..b35aaa401bae4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -24,16 +24,18 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+
 import io.netty.util.collection.IntObjectHashMap;
 import io.netty.util.collection.IntObjectMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * An implementation of map that supports constant time look-up by a generic key or an ordinal.
@@ -194,7 +196,7 @@ public V get(Object key) {
    * assignment. A new ordinal is assigned if key does not exists. Otherwise the same ordinal is re-used but the value
    * is replaced.
    *
-   * {@see java.util.Map#put}
+   * @see java.util.Map#put
    */
   @Override
   public V put(K key, V value) {
@@ -217,11 +219,11 @@ public boolean containsValue(Object value) {
   }
 
   /**
-   * Removes the element corresponding to the key if exists extending the semantics of {@link Map#remove} with ordinal
+   * Removes the element corresponding to the key if exists extending the semantics of {@link java.util.Map#remove} with ordinal
    * re-cycling. The ordinal corresponding to the given key may be re-assigned to another tuple. It is important that
-   * consumer checks the ordinal value via {@link #getOrdinal(Object)} before attempting to look-up by ordinal.
+   * consumer checks the ordinal value via {@link org.apache.arrow.vector.util.MapWithOrdinal#getOrdinal(Object)} before attempting to look-up by ordinal.
    *
-   * {@see java.util.Map#remove}
+   * @see java.util.Map#remove
    */
   @Override
   public V remove(Object key) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java b/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
index ec628b22c2d90..bd7396249a72c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
@@ -22,8 +22,7 @@
  * An exception that is used to signal that allocation request in bytes is greater than the maximum allowed by
  * {@link org.apache.arrow.memory.BufferAllocator#buffer(int) allocator}.
  *
- * <p>Operators should handle this exception to split the batch and later resume the execution on the next
- * {@link RecordBatch#next() iteration}.</p>
+ * <p>Operators should handle this exception to split the batch and later resume the execution on the next iteration.</p>
  *
  */
 public class OversizedAllocationException extends RuntimeException {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
index 3db4358ea9155..ce82f445ad883 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
@@ -74,18 +74,22 @@ public Text() {
 
   /**
    * Construct from a string.
+   * @param string initialize from that string
    */
   public Text(String string) {
     set(string);
   }
 
-  /** Construct from another text. */
+  /** Construct from another text.
+   * @param utf8 initialize from that Text
+   */
   public Text(Text utf8) {
     set(utf8);
   }
 
   /**
    * Construct from a byte array.
+   * @param utf8 initialize from that byte array
    */
   public Text(byte[] utf8) {
     set(utf8);
@@ -94,6 +98,7 @@ public Text(byte[] utf8) {
   /**
    * Get a copy of the bytes that is exactly the length of the data. See {@link #getBytes()} for faster access to the
    * underlying array.
+   * @return a copy of the underlying array
    */
   public byte[] copyBytes() {
     byte[] result = new byte[length];
@@ -104,12 +109,13 @@ public byte[] copyBytes() {
   /**
    * Returns the raw bytes; however, only data up to {@link #getLength()} is valid. Please use {@link #copyBytes()} if
    * you need the returned array to be precisely the length of the data.
+   * @return the underlying array
    */
   public byte[] getBytes() {
     return bytes;
   }
 
-  /** Returns the number of bytes in the byte array */
+  /** @return the number of bytes in the byte array */
   public int getLength() {
     return length;
   }
@@ -118,6 +124,7 @@ public int getLength() {
    * Returns the Unicode Scalar Value (32-bit integer value) for the character at <code>position</code>. Note that this
    * method avoids using the converter or doing String instantiation
    *
+   * @param position the index of the char we want to retrieve
    * @return the Unicode scalar value at position or -1 if the position is invalid or points to a trailing byte
    */
   public int charAt(int position) {
@@ -143,6 +150,8 @@ public int find(String what) {
    * starting position is measured in bytes and the return value is in terms of byte position in the buffer. The backing
    * buffer is not converted to a string for this operation.
    *
+   * @param what the string to search for
+   * @param start where to start from
    * @return byte position of the first occurence of the search string in the UTF-8 buffer or -1 if not found
    */
   public int find(String what, int start) {
@@ -187,6 +196,7 @@ public int find(String what, int start) {
 
   /**
    * Set to contain the contents of a string.
+   * @param string the string to initialize from
    */
   public void set(String string) {
     try {
@@ -200,12 +210,14 @@ public void set(String string) {
 
   /**
    * Set to a utf8 byte array
+   * @param utf8 the byte array to initialize from
    */
   public void set(byte[] utf8) {
     set(utf8, 0, utf8.length);
   }
 
-  /** copy a text. */
+  /** copy a text.
+   * @param other the text to initialize from */
   public void set(Text other) {
     set(other.getBytes(), 0, other.getLength());
   }
@@ -253,13 +265,12 @@ public void clear() {
     length = 0;
   }
 
-  /*
+  /**
    * Sets the capacity of this Text object to <em>at least</em> <code>len</code> bytes. If the current buffer is longer,
    * then the capacity and existing content of the buffer are unchanged. If <code>len</code> is larger than the current
    * capacity, the Text object's capacity is increased to match.
    *
    * @param len the number of bytes we need
-   *
    * @param keepData should the old data be kept
    */
   private void setCapacity(int len, boolean keepData) {
@@ -272,11 +283,6 @@ private void setCapacity(int len, boolean keepData) {
     }
   }
 
-  /**
-   * Convert text back to string
-   *
-   * @see java.lang.Object#toString()
-   */
   @Override
   public String toString() {
     try {
@@ -289,6 +295,9 @@ public String toString() {
   /**
    * Read a Text object whose length is already known. This allows creating Text from a stream which uses a different
    * serialization format.
+   * @param in the input to initialize from
+   * @param len how many bytes to read from in
+   * @throws IOException if something bad happens
    */
   public void readWithKnownLength(DataInput in, int len) throws IOException {
     setCapacity(len, false);
@@ -296,7 +305,6 @@ public void readWithKnownLength(DataInput in, int len) throws IOException {
     length = len;
   }
 
-  /** Returns true iff <code>o</code> is a Text with the same contents. */
   @Override
   public boolean equals(Object o) {
     if (o == this) {
@@ -326,7 +334,7 @@ public boolean equals(Object o) {
   /**
    * Copied from Arrays.hashCode so we don't have to copy the byte array
    *
-   * @return
+   * @return hashCode
    */
   @Override
   public int hashCode() {
@@ -346,6 +354,9 @@ public int hashCode() {
   /**
    * Converts the provided byte array to a String using the UTF-8 encoding. If the input is malformed, replace by a
    * default value.
+   * @param utf8 bytes to decode
+   * @return the decoded string
+   * @throws CharacterCodingException if this is not valid UTF-8
    */
   public static String decode(byte[] utf8) throws CharacterCodingException {
     return decode(ByteBuffer.wrap(utf8), true);
@@ -360,6 +371,12 @@ public static String decode(byte[] utf8, int start, int length)
    * Converts the provided byte array to a String using the UTF-8 encoding. If <code>replace</code> is true, then
    * malformed input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a
    * MalformedInputException.
+   * @param utf8 the bytes to decode
+   * @param start where to start from
+   * @param length length of the bytes to decode
+   * @param replace whether to replace malformed characters with U+FFFD
+   * @return the decoded string
+   * @throws CharacterCodingException if the input could not be decoded
    */
   public static String decode(byte[] utf8, int start, int length, boolean replace)
       throws CharacterCodingException {
@@ -387,9 +404,10 @@ private static String decode(ByteBuffer utf8, boolean replace)
    * Converts the provided String to bytes using the UTF-8 encoding. If the input is malformed, invalid chars are
    * replaced by a default value.
    *
+   * @param string the string to encode
    * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit()
+   * @throws CharacterCodingException if the string could not be encoded
    */
-
   public static ByteBuffer encode(String string)
       throws CharacterCodingException {
     return encode(string, true);
@@ -400,7 +418,11 @@ public static ByteBuffer encode(String string)
    * input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a
    * MalformedInputException.
    *
+
+   * @param string the string to encode
+   * @param replace whether to replace malformed characters with U+FFFD
    * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit()
+   * @throws CharacterCodingException if the string could not be encoded
    */
   public static ByteBuffer encode(String string, boolean replace)
       throws CharacterCodingException {
@@ -553,6 +575,8 @@ public static void validateUTF8(byte[] utf8, int start, int len)
   /**
    * Returns the next code point at the current position in the buffer. The buffer's position will be incremented. Any
    * mark set on this buffer will be changed by this method!
+   * @param bytes the incoming bytes
+   * @return the corresponding unicode codepoint
    */
   public static int bytesToCodePoint(ByteBuffer bytes) {
     bytes.mark();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
index f294e20b029c5..3035576da3327 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
@@ -36,6 +36,8 @@ public class Validator {
     /**
      * Validate two arrow schemas are equal.
      *
+     * @param schema1 the 1st shema to compare
+     * @param schema2 the 2nd shema to compare
      * @throws IllegalArgumentException if they are different.
      */
     public static void compareSchemas(Schema schema1, Schema schema2) {
@@ -47,6 +49,8 @@ public static void compareSchemas(Schema schema1, Schema schema2) {
     /**
      * Validate two arrow vectorSchemaRoot are equal.
      *
+     * @param root1 the 1st shema to compare
+     * @param root2 the 2nd shema to compare
      * @throws IllegalArgumentException if they are different.
      */
     public static void compareVectorSchemaRoot(VectorSchemaRoot root1, VectorSchemaRoot root2) {

From 02a121f18b6b5a34b63dd8d2bf7b1955ac7e11b2 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 2 May 2017 13:40:47 -0400
Subject: [PATCH 0593/1644] ARROW-927: C++/Python: Add manylinux1 builds to
 Travis matrix

Also pre-builds flatbuffers and gtest so that they are only build once.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #624 from xhochy/ARROW-927 and squashes the following commits:

c901b7a [Uwe L. Korn] Separate scripts folder
0dd4e08 [Uwe L. Korn] Add scripts folder
e705813 [Uwe L. Korn] Move boost and openssl to scripts
1b44878 [Uwe L. Korn] Add base image for thirdparties
f4ff321 [Uwe L. Korn] ARROW-927: C++/Python: Add manylinux1 builds to Travis matrix
---
 .travis.yml                                 | 11 +++++
 python/manylinux1/Dockerfile-x86_64         | 41 +---------------
 python/manylinux1/Dockerfile-x86_64_base    | 52 +++++++++++++++++++++
 python/manylinux1/build_arrow.sh            |  3 +-
 python/manylinux1/scripts/build_boost.sh    | 21 +++++++++
 python/manylinux1/scripts/build_jemalloc.sh | 21 +++++++++
 python/manylinux1/scripts/build_openssl.sh  | 21 +++++++++
 7 files changed, 128 insertions(+), 42 deletions(-)
 create mode 100644 python/manylinux1/Dockerfile-x86_64_base
 create mode 100755 python/manylinux1/scripts/build_boost.sh
 create mode 100755 python/manylinux1/scripts/build_jemalloc.sh
 create mode 100755 python/manylinux1/scripts/build_openssl.sh

diff --git a/.travis.yml b/.travis.yml
index 6ebebd4513fc7..19e71ae1e68f0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,6 +19,8 @@ addons:
     - gtk-doc-tools
     - autoconf-archive
     - libgirepository1.0-dev
+services:
+  - docker
 
 cache:
   ccache: true
@@ -54,6 +56,15 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
+  - language: cpp
+    before_script:
+    - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-927
+    script: |
+        pushd python/manylinux1
+        git clone ../../ arrow
+        docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
+        docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
+        ls -l dist/
   - language: java
     os: linux
     jdk: oraclejdk7
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 56b27ad2ae808..8f55ba7e1deed 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -9,46 +9,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License. See accompanying LICENSE file.
-
-FROM quay.io/pypa/manylinux1_x86_64:latest
-
-# Install dependencies
-RUN yum install -y flex zlib-devel
-
-# Build a newer OpenSSL version to support Thrift 0.10.0, note that we don't trigger the SSL code in Arrow.
-WORKDIR /
-RUN wget --no-check-certificate https://www.openssl.org/source/openssl-1.0.2k.tar.gz -O openssl-1.0.2k.tar.gz
-RUN tar xf openssl-1.0.2k.tar.gz
-WORKDIR openssl-1.0.2k
-RUN ./config -fpic shared --prefix=/usr
-RUN make -j5
-RUN make install
-
-WORKDIR /
-RUN wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
-RUN tar xf boost_1_60_0.tar.gz
-WORKDIR /boost_1_60_0
-RUN ./bootstrap.sh
-RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install
-
-WORKDIR /
-RUN wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2
-RUN tar xf jemalloc-4.4.0.tar.bz2
-WORKDIR /jemalloc-4.4.0
-RUN ./configure
-RUN make -j5
-RUN make install
-
-WORKDIR /
-# Install cmake manylinux1 package
-RUN /opt/python/cp35-cp35m/bin/pip install cmake
-RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
-
-WORKDIR /
-RUN git clone https://github.com/matthew-brett/multibuild.git
-WORKDIR /multibuild
-RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
-
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-927
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
new file mode 100644
index 0000000000000..e38296d78de1c
--- /dev/null
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -0,0 +1,52 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+FROM quay.io/pypa/manylinux1_x86_64:latest
+
+# Install dependencies
+RUN yum install -y flex zlib-devel
+
+ADD scripts/build_openssl.sh /
+RUN /build_openssl.sh
+
+ADD scripts/build_boost.sh /
+RUN /build_boost.sh
+
+ADD scripts/build_jemalloc.sh /
+RUN /build_jemalloc.sh
+
+WORKDIR /
+# Install cmake manylinux1 package
+RUN /opt/python/cp35-cp35m/bin/pip install cmake
+RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
+
+WORKDIR /
+RUN git clone https://github.com/matthew-brett/multibuild.git
+WORKDIR /multibuild
+RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
+
+WORKDIR /
+RUN wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -O googletest-release-1.7.0.tar.gz
+RUN tar xf googletest-release-1.7.0.tar.gz
+WORKDIR /googletest-release-1.7.0
+RUN cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON .
+RUN make -j5
+ENV GTEST_HOME /googletest-release-1.7.0
+
+WORKDIR /
+RUN wget https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -O flatbuffers-1.6.0.tar.gz
+RUN tar xf flatbuffers-1.6.0.tar.gz
+WORKDIR /flatbuffers-1.6.0
+RUN cmake "-DCMAKE_CXX_FLAGS=-fPIC" "-DCMAKE_INSTALL_PREFIX:PATH=/usr" "-DFLATBUFFERS_BUILD_TESTS=OFF"
+RUN make -j5
+RUN make install
+ENV FLATBUFFERS_HOME /usr
+
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 8ef087c7d262f..a11d3d41f49f7 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -31,7 +31,6 @@ cd /arrow/python
 
 # PyArrow build configuration
 export PYARROW_BUILD_TYPE='release'
-export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_TESTS=ON'
 export PYARROW_WITH_PARQUET=1
 export PYARROW_WITH_JEMALLOC=1
 export PYARROW_BUNDLE_ARROW_CPP=1
@@ -51,7 +50,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
 
     echo "=== (${PYTHON}) Installing build dependencies ==="
     $PIPI_IO "numpy==1.9.0"
-    $PIPI_IO "cython==0.24"
+    $PIPI_IO "cython==0.25.2"
     $PIPI_IO "pandas==0.19.2"
 
     echo "=== (${PYTHON}) Building Arrow C++ libraries ==="
diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh
new file mode 100755
index 0000000000000..6a313366494c6
--- /dev/null
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+
+wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
+tar xf boost_1_60_0.tar.gz
+pushd /boost_1_60_0
+./bootstrap.sh
+./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install
+popd
+rm -rf boost_1_60_0.tar.gz boost_1_60_0
diff --git a/python/manylinux1/scripts/build_jemalloc.sh b/python/manylinux1/scripts/build_jemalloc.sh
new file mode 100755
index 0000000000000..8153baa097e52
--- /dev/null
+++ b/python/manylinux1/scripts/build_jemalloc.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2
+tar xf jemalloc-4.4.0.tar.bz2
+pushd /jemalloc-4.4.0
+./configure
+make -j5
+make install
+popd
+rm -rf jemalloc-4.4.0.tar.bz2 jemalloc-4.4.0
diff --git a/python/manylinux1/scripts/build_openssl.sh b/python/manylinux1/scripts/build_openssl.sh
new file mode 100755
index 0000000000000..3bcb2b9a053a9
--- /dev/null
+++ b/python/manylinux1/scripts/build_openssl.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+wget --no-check-certificate https://www.openssl.org/source/openssl-1.0.2k.tar.gz -O openssl-1.0.2k.tar.gz
+tar xf openssl-1.0.2k.tar.gz
+pushd openssl-1.0.2k
+./config -fpic shared --prefix=/usr
+make -j5
+make install
+popd
+rm -rf openssl-1.0.2k.tar.gz openssl-1.0.2k

From f1bd49d5bb32636f5a8bbe8d26b4269b678dec55 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 2 May 2017 13:49:37 -0400
Subject: [PATCH 0594/1644] ARROW-931: [GLib] Reconstruct input stream

* GArrowRandomAccessFile -> GArrowSeekableInputStream
* GArrowBufferReader -> GArrowBufferInputStream
* GArrowMemoryMappedFile -> GArrowMemoryMappedInputStream

Author: Kouhei Sutou <kou@clear-code.com>

Closes #628 from kou/glib-reconstruct-input-stream and squashes the following commits:

3488cad [Kouhei Sutou] [GLib] Reconstruct input stream
---
 c_glib/arrow-glib/Makefile.am                 |   6 -
 c_glib/arrow-glib/arrow-glib.h                |   2 -
 c_glib/arrow-glib/file-reader.cpp             |  12 +-
 c_glib/arrow-glib/file-reader.h               |   6 +-
 c_glib/arrow-glib/input-stream.cpp            | 211 +++++++++++---
 c_glib/arrow-glib/input-stream.h              | 163 +++++++++--
 c_glib/arrow-glib/input-stream.hpp            |   8 +-
 c_glib/arrow-glib/memory-mapped-file.cpp      | 270 ------------------
 c_glib/arrow-glib/memory-mapped-file.h        |  72 -----
 c_glib/arrow-glib/memory-mapped-file.hpp      |  28 --
 c_glib/arrow-glib/random-access-file.cpp      | 124 --------
 c_glib/arrow-glib/random-access-file.h        |  53 ----
 c_glib/arrow-glib/random-access-file.hpp      |  38 ---
 c_glib/doc/reference/arrow-glib-docs.sgml     |   2 -
 c_glib/example/lua/read-batch.lua             |   2 +-
 c_glib/example/lua/read-stream.lua            |   2 +-
 c_glib/example/read-batch.c                   |   9 +-
 c_glib/example/read-stream.c                  |   6 +-
 ...-reader.rb => test-buffer-input-stream.rb} |   6 +-
 c_glib/test/test-file-writer.rb               |   2 +-
 c_glib/test/test-memory-mapped-file.rb        | 134 ---------
 .../test/test-memory-mapped-input-stream.rb   |  82 ++++++
 c_glib/test/test-stream-writer.rb             |   2 +-
 23 files changed, 420 insertions(+), 820 deletions(-)
 delete mode 100644 c_glib/arrow-glib/memory-mapped-file.cpp
 delete mode 100644 c_glib/arrow-glib/memory-mapped-file.h
 delete mode 100644 c_glib/arrow-glib/memory-mapped-file.hpp
 delete mode 100644 c_glib/arrow-glib/random-access-file.cpp
 delete mode 100644 c_glib/arrow-glib/random-access-file.h
 delete mode 100644 c_glib/arrow-glib/random-access-file.hpp
 rename c_glib/test/{test-buffer-reader.rb => test-buffer-input-stream.rb} (85%)
 delete mode 100644 c_glib/test/test-memory-mapped-file.rb
 create mode 100644 c_glib/test/test-memory-mapped-input-stream.rb

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 54fb7f8c7a799..242507273f451 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -60,9 +60,7 @@ libarrow_glib_la_headers +=			\
 	file.h					\
 	file-mode.h				\
 	input-stream.h				\
-	memory-mapped-file.h			\
 	output-stream.h				\
-	random-access-file.h			\
 	readable.h				\
 	writeable.h				\
 	writeable-file.h
@@ -102,9 +100,7 @@ libarrow_glib_la_sources +=			\
 	file.cpp				\
 	file-mode.cpp				\
 	input-stream.cpp			\
-	memory-mapped-file.cpp			\
 	output-stream.cpp			\
-	random-access-file.cpp			\
 	readable.cpp				\
 	writeable.cpp				\
 	writeable-file.cpp
@@ -136,9 +132,7 @@ libarrow_glib_la_cpp_headers +=			\
 	file.hpp				\
 	file-mode.hpp				\
 	input-stream.hpp			\
-	memory-mapped-file.hpp			\
 	output-stream.hpp			\
-	random-access-file.hpp			\
 	readable.hpp				\
 	writeable.hpp				\
 	writeable-file.hpp
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index e88b66b6ae9b2..0a06cb824dc85 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -36,9 +36,7 @@
 #include <arrow-glib/file.h>
 #include <arrow-glib/file-mode.h>
 #include <arrow-glib/input-stream.h>
-#include <arrow-glib/memory-mapped-file.h>
 #include <arrow-glib/output-stream.h>
-#include <arrow-glib/random-access-file.h>
 #include <arrow-glib/readable.h>
 #include <arrow-glib/writeable.h>
 #include <arrow-glib/writeable-file.h>
diff --git a/c_glib/arrow-glib/file-reader.cpp b/c_glib/arrow-glib/file-reader.cpp
index b952b52ddbe6d..bbba5a1ede7b2 100644
--- a/c_glib/arrow-glib/file-reader.cpp
+++ b/c_glib/arrow-glib/file-reader.cpp
@@ -27,7 +27,7 @@
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 
-#include <arrow-glib/random-access-file.hpp>
+#include <arrow-glib/input-stream.hpp>
 
 #include <arrow-glib/file-reader.hpp>
 #include <arrow-glib/metadata-version.hpp>
@@ -132,19 +132,21 @@ garrow_file_reader_class_init(GArrowFileReaderClass *klass)
 
 /**
  * garrow_file_reader_open:
- * @file: The file to be read.
+ * @input_stream: The seekable input stream to read data.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: (nullable) (transfer full): A newly opened
  *   #GArrowFileReader or %NULL on error.
  */
 GArrowFileReader *
-garrow_file_reader_open(GArrowRandomAccessFile *file,
-                            GError **error)
+garrow_file_reader_open(GArrowSeekableInputStream *input_stream,
+                        GError **error)
 {
+  auto arrow_random_access_file =
+    garrow_seekable_input_stream_get_raw(input_stream);
   std::shared_ptr<arrow::ipc::FileReader> arrow_file_reader;
   auto status =
-    arrow::ipc::FileReader::Open(garrow_random_access_file_get_raw(file),
+    arrow::ipc::FileReader::Open(arrow_random_access_file,
                                  &arrow_file_reader);
   if (garrow_error_check(error, status, "[ipc][file-reader][open]")) {
     return garrow_file_reader_new_raw(&arrow_file_reader);
diff --git a/c_glib/arrow-glib/file-reader.h b/c_glib/arrow-glib/file-reader.h
index 084f7148ed903..b737269a2945b 100644
--- a/c_glib/arrow-glib/file-reader.h
+++ b/c_glib/arrow-glib/file-reader.h
@@ -22,7 +22,7 @@
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
 
-#include <arrow-glib/random-access-file.h>
+#include <arrow-glib/input-stream.h>
 
 #include <arrow-glib/metadata-version.h>
 
@@ -70,8 +70,8 @@ struct _GArrowFileReaderClass
 
 GType garrow_file_reader_get_type(void) G_GNUC_CONST;
 
-GArrowFileReader *garrow_file_reader_open(GArrowRandomAccessFile *file,
-                                                 GError **error);
+GArrowFileReader *garrow_file_reader_open(GArrowSeekableInputStream *input_stream,
+                                          GError **error);
 
 GArrowSchema *garrow_file_reader_get_schema(GArrowFileReader *file_reader);
 guint garrow_file_reader_get_n_record_batches(GArrowFileReader *file_reader);
diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 56b811ad1c368..b931cf8250607 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -28,7 +28,6 @@
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/file.hpp>
 #include <arrow-glib/input-stream.hpp>
-#include <arrow-glib/random-access-file.hpp>
 #include <arrow-glib/readable.hpp>
 
 G_BEGIN_DECLS
@@ -41,7 +40,13 @@ G_BEGIN_DECLS
  *
  * #GArrowInputStream is a base class for input stream.
  *
- * #GArrowBufferReader is a class for buffer input stream.
+ * #GArrowSeekableInputStream is a base class for input stream that
+ * supports random access.
+ *
+ * #GArrowBufferInputStream is a class to read data on buffer.
+ *
+ * #GArrowMemoryMappedFile is a class to read data in file by mapping
+ * the file on memory. It supports zero copy.
  */
 
 typedef struct GArrowInputStreamPrivate_ {
@@ -168,71 +173,174 @@ garrow_input_stream_class_init(GArrowInputStreamClass *klass)
 }
 
 
-static std::shared_ptr<arrow::io::RandomAccessFile>
-garrow_buffer_reader_get_raw_random_access_file_interface(GArrowRandomAccessFile *random_access_file)
+G_DEFINE_TYPE(GArrowSeekableInputStream,                \
+              garrow_seekable_input_stream,             \
+              GARROW_TYPE_INPUT_STREAM);
+
+static void
+garrow_seekable_input_stream_init(GArrowSeekableInputStream *object)
 {
-  auto input_stream = GARROW_INPUT_STREAM(random_access_file);
-  auto arrow_input_stream = garrow_input_stream_get_raw(input_stream);
-  auto arrow_buffer_reader =
-    std::static_pointer_cast<arrow::io::BufferReader>(arrow_input_stream);
-  return arrow_buffer_reader;
 }
 
 static void
-garrow_buffer_reader_random_access_file_interface_init(GArrowRandomAccessFileInterface *iface)
+garrow_seekable_input_stream_class_init(GArrowSeekableInputStreamClass *klass)
+{
+}
+
+/**
+ * garrow_seekable_input_stream_get_size:
+ * @input_stream: A #GArrowSeekableInputStream.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: The size of the file.
+ */
+guint64
+garrow_seekable_input_stream_get_size(GArrowSeekableInputStream *input_stream,
+                                      GError **error)
+{
+  auto arrow_random_access_file =
+    garrow_seekable_input_stream_get_raw(input_stream);
+  int64_t size;
+  auto status = arrow_random_access_file->GetSize(&size);
+  if (garrow_error_check(error, status, "[seekable-input-stream][get-size]")) {
+    return size;
+  } else {
+    return 0;
+  }
+}
+
+/**
+ * garrow_seekable_input_stream_get_support_zero_copy:
+ * @input_stream: A #GArrowSeekableInputStream.
+ *
+ * Returns: Whether zero copy read is supported or not.
+ */
+gboolean
+garrow_seekable_input_stream_get_support_zero_copy(GArrowSeekableInputStream *input_stream)
+{
+  auto arrow_random_access_file =
+    garrow_seekable_input_stream_get_raw(input_stream);
+  return arrow_random_access_file->supports_zero_copy();
+}
+
+/**
+ * garrow_seekable_input_stream_read_at:
+ * @input_stream: A #GArrowSeekableInputStream.
+ * @position: The read start position.
+ * @n_bytes: The number of bytes to be read.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full) (nullable): #GArrowBuffer that has read
+ *   data on success, %NULL if there was an error.
+ */
+GArrowBuffer *
+garrow_seekable_input_stream_read_at(GArrowSeekableInputStream *input_stream,
+                                     gint64 position,
+                                     gint64 n_bytes,
+                                     GError **error)
 {
-  iface->get_raw = garrow_buffer_reader_get_raw_random_access_file_interface;
+  auto arrow_random_access_file =
+    garrow_seekable_input_stream_get_raw(input_stream);
+
+  std::shared_ptr<arrow::Buffer> arrow_buffer;
+  auto status = arrow_random_access_file->ReadAt(position,
+                                                 n_bytes,
+                                                 &arrow_buffer);
+  if (garrow_error_check(error, status, "[seekable-input-stream][read-at]")) {
+    return garrow_buffer_new_raw(&arrow_buffer);
+  } else {
+    return NULL;
+  }
 }
 
-G_DEFINE_TYPE_WITH_CODE(GArrowBufferReader,               \
-                        garrow_buffer_reader,             \
-                        GARROW_TYPE_INPUT_STREAM,
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_RANDOM_ACCESS_FILE,
-                                              garrow_buffer_reader_random_access_file_interface_init));
+
+G_DEFINE_TYPE(GArrowBufferInputStream,                       \
+              garrow_buffer_input_stream,                     \
+              GARROW_TYPE_SEEKABLE_INPUT_STREAM);
 
 static void
-garrow_buffer_reader_init(GArrowBufferReader *object)
+garrow_buffer_input_stream_init(GArrowBufferInputStream *object)
 {
 }
 
 static void
-garrow_buffer_reader_class_init(GArrowBufferReaderClass *klass)
+garrow_buffer_input_stream_class_init(GArrowBufferInputStreamClass *klass)
 {
 }
 
 /**
- * garrow_buffer_reader_new:
+ * garrow_buffer_input_stream_new:
  * @buffer: The buffer to be read.
  *
- * Returns: A newly created #GArrowBufferReader.
+ * Returns: A newly created #GArrowBufferInputStream.
  */
-GArrowBufferReader *
-garrow_buffer_reader_new(GArrowBuffer *buffer)
+GArrowBufferInputStream *
+garrow_buffer_input_stream_new(GArrowBuffer *buffer)
 {
   auto arrow_buffer = garrow_buffer_get_raw(buffer);
   auto arrow_buffer_reader =
     std::make_shared<arrow::io::BufferReader>(arrow_buffer);
-  return garrow_buffer_reader_new_raw(&arrow_buffer_reader);
+  return garrow_buffer_input_stream_new_raw(&arrow_buffer_reader);
 }
 
 /**
- * garrow_buffer_reader_get_buffer:
- * @buffer_reader: A #GArrowBufferReader.
+ * garrow_buffer_input_stream_get_buffer:
+ * @input_stream: A #GArrowBufferInputStream.
  *
  * Returns: (transfer full): The data of the array as #GArrowBuffer.
  */
 GArrowBuffer *
-garrow_buffer_reader_get_buffer(GArrowBufferReader *buffer_reader)
+garrow_buffer_input_stream_get_buffer(GArrowBufferInputStream *input_stream)
 {
-  auto arrow_input_stream =
-    garrow_input_stream_get_raw(GARROW_INPUT_STREAM(buffer_reader));
-  auto arrow_buffer_reader =
-    std::static_pointer_cast<arrow::io::BufferReader>(arrow_input_stream);
+  auto arrow_buffer_reader = garrow_buffer_input_stream_get_raw(input_stream);
   auto arrow_buffer = arrow_buffer_reader->buffer();
   return garrow_buffer_new_raw(&arrow_buffer);
 }
 
 
+G_DEFINE_TYPE(GArrowMemoryMappedInputStream,            \
+              garrow_memory_mapped_input_stream,        \
+              GARROW_TYPE_SEEKABLE_INPUT_STREAM);
+
+static void
+garrow_memory_mapped_input_stream_init(GArrowMemoryMappedInputStream *object)
+{
+}
+
+static void
+garrow_memory_mapped_input_stream_class_init(GArrowMemoryMappedInputStreamClass *klass)
+{
+}
+
+/**
+ * garrow_memory_mapped_input_stream_new:
+ * @path: The path of the file to be mapped on memory.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowMemoryMappedInputStream
+ *   or %NULL on error.
+ */
+GArrowMemoryMappedInputStream *
+garrow_memory_mapped_input_stream_new(const gchar *path,
+                                      GError **error)
+{
+  std::shared_ptr<arrow::io::MemoryMappedFile> arrow_memory_mapped_file;
+  auto status =
+    arrow::io::MemoryMappedFile::Open(std::string(path),
+                                      arrow::io::FileMode::READ,
+                                      &arrow_memory_mapped_file);
+  if (status.ok()) {
+    return garrow_memory_mapped_input_stream_new_raw(&arrow_memory_mapped_file);
+  } else {
+    std::string context("[memory-mapped-input-stream][open]: <");
+    context += path;
+    context += ">";
+    garrow_error_check(error, status, context.c_str());
+    return NULL;
+  }
+}
+
+
 G_END_DECLS
 
 GArrowInputStream *
@@ -254,13 +362,42 @@ garrow_input_stream_get_raw(GArrowInputStream *input_stream)
   return priv->input_stream;
 }
 
+std::shared_ptr<arrow::io::RandomAccessFile>
+garrow_seekable_input_stream_get_raw(GArrowSeekableInputStream *seekable_input_stream)
+{
+  auto arrow_input_stream =
+    garrow_input_stream_get_raw(GARROW_INPUT_STREAM(seekable_input_stream));
+  auto arrow_random_access_file =
+    std::static_pointer_cast<arrow::io::RandomAccessFile>(arrow_input_stream);
+  return arrow_random_access_file;
+}
+
+GArrowBufferInputStream *
+garrow_buffer_input_stream_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader)
+{
+  auto buffer_input_stream =
+    GARROW_BUFFER_INPUT_STREAM(g_object_new(GARROW_TYPE_BUFFER_INPUT_STREAM,
+                                            "input-stream", arrow_buffer_reader,
+                                            NULL));
+  return buffer_input_stream;
+}
+
+std::shared_ptr<arrow::io::BufferReader>
+garrow_buffer_input_stream_get_raw(GArrowBufferInputStream *buffer_input_stream)
+{
+  auto arrow_input_stream =
+    garrow_input_stream_get_raw(GARROW_INPUT_STREAM(buffer_input_stream));
+  auto arrow_buffer_reader =
+    std::static_pointer_cast<arrow::io::BufferReader>(arrow_input_stream);
+  return arrow_buffer_reader;
+}
 
-GArrowBufferReader *
-garrow_buffer_reader_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader)
+GArrowMemoryMappedInputStream *
+garrow_memory_mapped_input_stream_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file)
 {
-  auto buffer_reader =
-    GARROW_BUFFER_READER(g_object_new(GARROW_TYPE_BUFFER_READER,
-                                      "input-stream", arrow_buffer_reader,
-                                      NULL));
-  return buffer_reader;
+  auto object = g_object_new(GARROW_TYPE_MEMORY_MAPPED_INPUT_STREAM,
+                             "input-stream", arrow_memory_mapped_file,
+                             NULL);
+  auto memory_mapped_input_stream = GARROW_MEMORY_MAPPED_INPUT_STREAM(object);
+  return memory_mapped_input_stream;
 }
diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h
index caa11b575452b..511882863760d 100644
--- a/c_glib/arrow-glib/input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -70,54 +70,159 @@ struct _GArrowInputStreamClass
 GType garrow_input_stream_get_type(void) G_GNUC_CONST;
 
 
-#define GARROW_TYPE_BUFFER_READER               \
-  (garrow_buffer_reader_get_type())
-#define GARROW_BUFFER_READER(obj)                               \
+#define GARROW_TYPE_SEEKABLE_INPUT_STREAM       \
+  (garrow_seekable_input_stream_get_type())
+#define GARROW_SEEKABLE_INPUT_STREAM(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_SEEKABLE_INPUT_STREAM,        \
+                              GArrowSeekableInputStream))
+#define GARROW_SEEKABLE_INPUT_STREAM_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_SEEKABLE_INPUT_STREAM,   \
+                           GArrowSeekableInputStreamClass))
+#define GARROW_IS_SEEKABLE_INPUT_STREAM(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_SEEKABLE_INPUT_STREAM))
+#define GARROW_IS_SEEKABLE_INPUT_STREAM_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_SEEKABLE_INPUT_STREAM))
+#define GARROW_SEEKABLE_INPUT_STREAM_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_SEEKABLE_INPUT_STREAM, \
+                             GArrowSeekableInputStreamClass))
+
+typedef struct _GArrowSeekableInputStream         GArrowSeekableInputStream;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowSeekableInputStreamClass    GArrowSeekableInputStreamClass;
+#endif
+
+/**
+ * GArrowSeekableInputStream:
+ *
+ * It wraps `arrow::io::RandomAccessFile`.
+ */
+struct _GArrowSeekableInputStream
+{
+  /*< private >*/
+  GArrowInputStream parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowSeekableInputStreamClass
+{
+  GArrowInputStreamClass parent_class;
+};
+#endif
+
+GType garrow_seekable_input_stream_get_type(void) G_GNUC_CONST;
+
+guint64 garrow_seekable_input_stream_get_size(GArrowSeekableInputStream *input_stream,
+                                              GError **error);
+gboolean garrow_seekable_input_stream_get_support_zero_copy(GArrowSeekableInputStream *input_stream);
+GArrowBuffer *garrow_seekable_input_stream_read_at(GArrowSeekableInputStream *input_stream,
+                                                   gint64 position,
+                                                   gint64 n_bytes,
+                                                   GError **error);
+
+
+#define GARROW_TYPE_BUFFER_INPUT_STREAM         \
+  (garrow_buffer_input_stream_get_type())
+#define GARROW_BUFFER_INPUT_STREAM(obj)                         \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BUFFER_READER,        \
-                              GArrowBufferReader))
-#define GARROW_BUFFER_READER_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_BUFFER_READER,   \
-                           GArrowBufferReaderClass))
-#define GARROW_IS_BUFFER_READER(obj)                            \
+                              GARROW_TYPE_BUFFER_INPUT_STREAM,  \
+                              GArrowBufferInputStream))
+#define GARROW_BUFFER_INPUT_STREAM_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BUFFER_INPUT_STREAM,     \
+                           GArrowBufferInputStreamClass))
+#define GARROW_IS_BUFFER_INPUT_STREAM(obj)                      \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BUFFER_READER))
-#define GARROW_IS_BUFFER_READER_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_BUFFER_READER))
-#define GARROW_BUFFER_READER_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_BUFFER_READER, \
-                             GArrowBufferReaderClass))
-
-typedef struct _GArrowBufferReader         GArrowBufferReader;
+                              GARROW_TYPE_BUFFER_INPUT_STREAM))
+#define GARROW_IS_BUFFER_INPUT_STREAM_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BUFFER_INPUT_STREAM))
+#define GARROW_BUFFER_INPUT_STREAM_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BUFFER_INPUT_STREAM,   \
+                             GArrowBufferInputStreamClass))
+
+typedef struct _GArrowBufferInputStream         GArrowBufferInputStream;
 #ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowBufferReaderClass    GArrowBufferReaderClass;
+typedef struct _GArrowBufferInputStreamClass    GArrowBufferInputStreamClass;
 #endif
 
 /**
- * GArrowBufferReader:
+ * GArrowBufferInputStream:
  *
  * It wraps `arrow::io::BufferReader`.
  */
-struct _GArrowBufferReader
+struct _GArrowBufferInputStream
 {
   /*< private >*/
-  GArrowInputStream parent_instance;
+  GArrowSeekableInputStream parent_instance;
 };
 
 #ifndef __GTK_DOC_IGNORE__
-struct _GArrowBufferReaderClass
+struct _GArrowBufferInputStreamClass
 {
-  GArrowInputStreamClass parent_class;
+  GArrowSeekableInputStreamClass parent_class;
 };
 #endif
 
-GType garrow_buffer_reader_get_type(void) G_GNUC_CONST;
+GType garrow_buffer_input_stream_get_type(void) G_GNUC_CONST;
+
+GArrowBufferInputStream *garrow_buffer_input_stream_new(GArrowBuffer *buffer);
+
+GArrowBuffer *garrow_buffer_input_stream_get_buffer(GArrowBufferInputStream *input_stream);
+
+
+#define GARROW_TYPE_MEMORY_MAPPED_INPUT_STREAM          \
+  (garrow_memory_mapped_input_stream_get_type())
+#define GARROW_MEMORY_MAPPED_INPUT_STREAM(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_MEMORY_MAPPED_INPUT_STREAM,   \
+                              GArrowMemoryMappedInputStream))
+#define GARROW_MEMORY_MAPPED_INPUT_STREAM_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                                     \
+                           GARROW_TYPE_MEMORY_MAPPED_INPUT_STREAM,      \
+                           GArrowMemoryMappedInputStreamClass))
+#define GARROW_IS_MEMORY_MAPPED_INPUT_STREAM(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_MEMORY_MAPPED_INPUT_STREAM))
+#define GARROW_IS_MEMORY_MAPPED_INPUT_STREAM_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                                     \
+                           GARROW_TYPE_MEMORY_MAPPED_INPUT_STREAM))
+#define GARROW_MEMORY_MAPPED_INPUT_STREAM_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                                     \
+                             GARROW_TYPE_MEMORY_MAPPED_INPUT_STREAM,    \
+                             GArrowMemoryMappedInputStreamClass))
+
+typedef struct _GArrowMemoryMappedInputStream         GArrowMemoryMappedInputStream;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowMemoryMappedInputStreamClass    GArrowMemoryMappedInputStreamClass;
+#endif
+
+/**
+ * GArrowMemoryMappedInputStream:
+ *
+ * It wraps `arrow::io::MemoryMappedFile`.
+ */
+struct _GArrowMemoryMappedInputStream
+{
+  /*< private >*/
+  GArrowSeekableInputStream parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowMemoryMappedInputStreamClass
+{
+  GArrowSeekableInputStreamClass parent_class;
+};
+#endif
 
-GArrowBufferReader *garrow_buffer_reader_new(GArrowBuffer *buffer);
+GType garrow_memory_mapped_input_stream_get_type(void) G_GNUC_CONST;
 
-GArrowBuffer *garrow_buffer_reader_get_buffer(GArrowBufferReader *buffer_reader);
+GArrowMemoryMappedInputStream *garrow_memory_mapped_input_stream_new(const gchar *path,
+                                                                     GError **error);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/input-stream.hpp b/c_glib/arrow-glib/input-stream.hpp
index 008f5f2b4e157..17d2bd92422d6 100644
--- a/c_glib/arrow-glib/input-stream.hpp
+++ b/c_glib/arrow-glib/input-stream.hpp
@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include <arrow/io/file.h>
 #include <arrow/io/interfaces.h>
 #include <arrow/io/memory.h>
 
@@ -27,4 +28,9 @@
 GArrowInputStream *garrow_input_stream_new_raw(std::shared_ptr<arrow::io::InputStream> *arrow_input_stream);
 std::shared_ptr<arrow::io::InputStream> garrow_input_stream_get_raw(GArrowInputStream *input_stream);
 
-GArrowBufferReader *garrow_buffer_reader_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader);
+std::shared_ptr<arrow::io::RandomAccessFile> garrow_seekable_input_stream_get_raw(GArrowSeekableInputStream *input_stream);
+
+GArrowBufferInputStream *garrow_buffer_input_stream_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader);
+std::shared_ptr<arrow::io::BufferReader> garrow_buffer_input_stream_get_raw(GArrowBufferInputStream *input_stream);
+
+GArrowMemoryMappedInputStream *garrow_memory_mapped_input_stream_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file);
diff --git a/c_glib/arrow-glib/memory-mapped-file.cpp b/c_glib/arrow-glib/memory-mapped-file.cpp
deleted file mode 100644
index 71a9a6dad3134..0000000000000
--- a/c_glib/arrow-glib/memory-mapped-file.cpp
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow/io/file.h>
-
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/file.hpp>
-#include <arrow-glib/file-mode.hpp>
-#include <arrow-glib/input-stream.hpp>
-#include <arrow-glib/memory-mapped-file.hpp>
-#include <arrow-glib/readable.hpp>
-#include <arrow-glib/random-access-file.hpp>
-#include <arrow-glib/writeable.hpp>
-#include <arrow-glib/writeable-file.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: memory-mapped-file
- * @short_description: Memory mapped file class
- *
- * #GArrowMemoryMappedFile is a class for memory mapped file. It's
- * readable and writeable. It supports zero copy.
- */
-
-typedef struct GArrowMemoryMappedFilePrivate_ {
-  std::shared_ptr<arrow::io::MemoryMappedFile> memory_mapped_file;
-} GArrowMemoryMappedFilePrivate;
-
-enum {
-  PROP_0,
-  PROP_MEMORY_MAPPED_FILE
-};
-
-static std::shared_ptr<arrow::io::FileInterface>
-garrow_memory_mapped_file_get_raw_file_interface(GArrowFile *file)
-{
-  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(file);
-  auto arrow_memory_mapped_file =
-    garrow_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_file_interface_init(GArrowFileInterface *iface)
-{
-  iface->get_raw = garrow_memory_mapped_file_get_raw_file_interface;
-}
-
-static std::shared_ptr<arrow::io::Readable>
-garrow_memory_mapped_file_get_raw_readable_interface(GArrowReadable *readable)
-{
-  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(readable);
-  auto arrow_memory_mapped_file =
-    garrow_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_readable_interface_init(GArrowReadableInterface *iface)
-{
-  iface->get_raw = garrow_memory_mapped_file_get_raw_readable_interface;
-}
-
-static std::shared_ptr<arrow::io::RandomAccessFile>
-garrow_memory_mapped_file_get_raw_random_access_file_interface(GArrowRandomAccessFile *file)
-{
-  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(file);
-  auto arrow_memory_mapped_file =
-    garrow_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_random_access_file_interface_init(GArrowRandomAccessFileInterface *iface)
-{
-  iface->get_raw = garrow_memory_mapped_file_get_raw_random_access_file_interface;
-}
-
-static std::shared_ptr<arrow::io::Writeable>
-garrow_memory_mapped_file_get_raw_writeable_interface(GArrowWriteable *writeable)
-{
-  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(writeable);
-  auto arrow_memory_mapped_file =
-    garrow_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_writeable_interface_init(GArrowWriteableInterface *iface)
-{
-  iface->get_raw = garrow_memory_mapped_file_get_raw_writeable_interface;
-}
-
-static std::shared_ptr<arrow::io::WriteableFile>
-garrow_memory_mapped_file_get_raw_writeable_file_interface(GArrowWriteableFile *file)
-{
-  auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(file);
-  auto arrow_memory_mapped_file =
-    garrow_memory_mapped_file_get_raw(memory_mapped_file);
-  return arrow_memory_mapped_file;
-}
-
-static void
-garrow_writeable_file_interface_init(GArrowWriteableFileInterface *iface)
-{
-  iface->get_raw = garrow_memory_mapped_file_get_raw_writeable_file_interface;
-}
-
-G_DEFINE_TYPE_WITH_CODE(GArrowMemoryMappedFile,
-                        garrow_memory_mapped_file,
-                        G_TYPE_OBJECT,
-                        G_ADD_PRIVATE(GArrowMemoryMappedFile)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_FILE,
-                                              garrow_file_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_READABLE,
-                                              garrow_readable_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_RANDOM_ACCESS_FILE,
-                                              garrow_random_access_file_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE,
-                                              garrow_writeable_interface_init)
-                        G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE_FILE,
-                                              garrow_writeable_file_interface_init));
-
-#define GARROW_MEMORY_MAPPED_FILE_GET_PRIVATE(obj)                   \
-  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
-                               GARROW_TYPE_MEMORY_MAPPED_FILE,       \
-                               GArrowMemoryMappedFilePrivate))
-
-static void
-garrow_memory_mapped_file_finalize(GObject *object)
-{
-  GArrowMemoryMappedFilePrivate *priv;
-
-  priv = GARROW_MEMORY_MAPPED_FILE_GET_PRIVATE(object);
-
-  priv->memory_mapped_file = nullptr;
-
-  G_OBJECT_CLASS(garrow_memory_mapped_file_parent_class)->finalize(object);
-}
-
-static void
-garrow_memory_mapped_file_set_property(GObject *object,
-                                          guint prop_id,
-                                          const GValue *value,
-                                          GParamSpec *pspec)
-{
-  GArrowMemoryMappedFilePrivate *priv;
-
-  priv = GARROW_MEMORY_MAPPED_FILE_GET_PRIVATE(object);
-
-  switch (prop_id) {
-  case PROP_MEMORY_MAPPED_FILE:
-    priv->memory_mapped_file =
-      *static_cast<std::shared_ptr<arrow::io::MemoryMappedFile> *>(g_value_get_pointer(value));
-    break;
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_memory_mapped_file_get_property(GObject *object,
-                                          guint prop_id,
-                                          GValue *value,
-                                          GParamSpec *pspec)
-{
-  switch (prop_id) {
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_memory_mapped_file_init(GArrowMemoryMappedFile *object)
-{
-}
-
-static void
-garrow_memory_mapped_file_class_init(GArrowMemoryMappedFileClass *klass)
-{
-  GObjectClass *gobject_class;
-  GParamSpec *spec;
-
-  gobject_class = G_OBJECT_CLASS(klass);
-
-  gobject_class->finalize     = garrow_memory_mapped_file_finalize;
-  gobject_class->set_property = garrow_memory_mapped_file_set_property;
-  gobject_class->get_property = garrow_memory_mapped_file_get_property;
-
-  spec = g_param_spec_pointer("memory-mapped-file",
-                              "io::MemoryMappedFile",
-                              "The raw std::shared<arrow::io::MemoryMappedFile> *",
-                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
-                                                       G_PARAM_CONSTRUCT_ONLY));
-  g_object_class_install_property(gobject_class, PROP_MEMORY_MAPPED_FILE, spec);
-}
-
-/**
- * garrow_memory_mapped_file_open:
- * @path: The path of the memory mapped file.
- * @mode: The mode of the memory mapped file.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowMemoryMappedFile or %NULL on error.
- */
-GArrowMemoryMappedFile *
-garrow_memory_mapped_file_open(const gchar *path,
-                                  GArrowFileMode mode,
-                                  GError **error)
-{
-  std::shared_ptr<arrow::io::MemoryMappedFile> arrow_memory_mapped_file;
-  auto status =
-    arrow::io::MemoryMappedFile::Open(std::string(path),
-                                      garrow_file_mode_to_raw(mode),
-                                      &arrow_memory_mapped_file);
-  if (status.ok()) {
-    return garrow_memory_mapped_file_new_raw(&arrow_memory_mapped_file);
-  } else {
-    std::string context("[io][memory-mapped-file][open]: <");
-    context += path;
-    context += ">";
-    garrow_error_check(error, status, context.c_str());
-    return NULL;
-  }
-}
-
-G_END_DECLS
-
-GArrowMemoryMappedFile *
-garrow_memory_mapped_file_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file)
-{
-  auto memory_mapped_file =
-    GARROW_MEMORY_MAPPED_FILE(g_object_new(GARROW_TYPE_MEMORY_MAPPED_FILE,
-                                              "memory-mapped-file", arrow_memory_mapped_file,
-                                              NULL));
-  return memory_mapped_file;
-}
-
-std::shared_ptr<arrow::io::MemoryMappedFile>
-garrow_memory_mapped_file_get_raw(GArrowMemoryMappedFile *memory_mapped_file)
-{
-  GArrowMemoryMappedFilePrivate *priv;
-
-  priv = GARROW_MEMORY_MAPPED_FILE_GET_PRIVATE(memory_mapped_file);
-  return priv->memory_mapped_file;
-}
diff --git a/c_glib/arrow-glib/memory-mapped-file.h b/c_glib/arrow-glib/memory-mapped-file.h
deleted file mode 100644
index 40b8de04a5a75..0000000000000
--- a/c_glib/arrow-glib/memory-mapped-file.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/file-mode.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_MEMORY_MAPPED_FILE       \
-  (garrow_memory_mapped_file_get_type())
-#define GARROW_MEMORY_MAPPED_FILE(obj)                               \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
-                              GARROW_TYPE_MEMORY_MAPPED_FILE,        \
-                              GArrowMemoryMappedFile))
-#define GARROW_MEMORY_MAPPED_FILE_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_MEMORY_MAPPED_FILE,   \
-                           GArrowMemoryMappedFileClass))
-#define GARROW_IS_MEMORY_MAPPED_FILE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_MEMORY_MAPPED_FILE))
-#define GARROW_IS_MEMORY_MAPPED_FILE_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_MEMORY_MAPPED_FILE))
-#define GARROW_MEMORY_MAPPED_FILE_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_MEMORY_MAPPED_FILE, \
-                             GArrowMemoryMappedFileClass))
-
-typedef struct _GArrowMemoryMappedFile         GArrowMemoryMappedFile;
-typedef struct _GArrowMemoryMappedFileClass    GArrowMemoryMappedFileClass;
-
-/**
- * GArrowMemoryMappedFile:
- *
- * It wraps `arrow::io::MemoryMappedFile`.
- */
-struct _GArrowMemoryMappedFile
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowMemoryMappedFileClass
-{
-  GObjectClass parent_class;
-};
-
-GType garrow_memory_mapped_file_get_type(void) G_GNUC_CONST;
-
-GArrowMemoryMappedFile *garrow_memory_mapped_file_open(const gchar *path,
-                                                            GArrowFileMode mode,
-                                                            GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/memory-mapped-file.hpp b/c_glib/arrow-glib/memory-mapped-file.hpp
deleted file mode 100644
index 522e43d117f39..0000000000000
--- a/c_glib/arrow-glib/memory-mapped-file.hpp
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow/api.h>
-#include <arrow/io/file.h>
-
-#include <arrow-glib/memory-mapped-file.h>
-
-GArrowMemoryMappedFile *garrow_memory_mapped_file_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file);
-std::shared_ptr<arrow::io::MemoryMappedFile> garrow_memory_mapped_file_get_raw(GArrowMemoryMappedFile *memory_mapped_file);
diff --git a/c_glib/arrow-glib/random-access-file.cpp b/c_glib/arrow-glib/random-access-file.cpp
deleted file mode 100644
index 744131632cbc2..0000000000000
--- a/c_glib/arrow-glib/random-access-file.cpp
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow/api.h>
-
-#include <arrow-glib/buffer.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/random-access-file.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: random-access-file
- * @title: GArrowRandomAccessFile
- * @short_description: File input interface
- *
- * #GArrowRandomAccessFile is an interface for file input.
- */
-
-G_DEFINE_INTERFACE(GArrowRandomAccessFile,
-                   garrow_random_access_file,
-                   G_TYPE_OBJECT)
-
-static void
-garrow_random_access_file_default_init (GArrowRandomAccessFileInterface *iface)
-{
-}
-
-/**
- * garrow_random_access_file_get_size:
- * @file: A #GArrowRandomAccessFile.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: The size of the file.
- */
-guint64
-garrow_random_access_file_get_size(GArrowRandomAccessFile *file,
-                                 GError **error)
-{
-  auto *iface = GARROW_RANDOM_ACCESS_FILE_GET_IFACE(file);
-  auto arrow_random_access_file = iface->get_raw(file);
-  int64_t size;
-
-  auto status = arrow_random_access_file->GetSize(&size);
-  if (garrow_error_check(error, status, "[io][random-access-file][get-size]")) {
-    return size;
-  } else {
-    return 0;
-  }
-}
-
-/**
- * garrow_random_access_file_get_support_zero_copy:
- * @file: A #GArrowRandomAccessFile.
- *
- * Returns: Whether zero copy read is supported or not.
- */
-gboolean
-garrow_random_access_file_get_support_zero_copy(GArrowRandomAccessFile *file)
-{
-  auto *iface = GARROW_RANDOM_ACCESS_FILE_GET_IFACE(file);
-  auto arrow_random_access_file = iface->get_raw(file);
-
-  return arrow_random_access_file->supports_zero_copy();
-}
-
-/**
- * garrow_random_access_file_read_at:
- * @file: A #GArrowRandomAccessFile.
- * @position: The read start position.
- * @n_bytes: The number of bytes to be read.
- * @error: (nullable): Return location for a #GError or %NULL.
- *
- * Returns: (transfer full) (nullable): #GArrowBuffer that has read
- *   data on success, %NULL if there was an error.
- */
-GArrowBuffer *
-garrow_random_access_file_read_at(GArrowRandomAccessFile *file,
-                                     gint64 position,
-                                     gint64 n_bytes,
-                                     GError **error)
-{
-  const auto arrow_random_access_file =
-    garrow_random_access_file_get_raw(file);
-
-  std::shared_ptr<arrow::Buffer> arrow_buffer;
-  auto status = arrow_random_access_file->ReadAt(position,
-                                                 n_bytes,
-                                                 &arrow_buffer);
-  if (garrow_error_check(error, status, "[io][random-access-file][read-at]")) {
-    return garrow_buffer_new_raw(&arrow_buffer);
-  } else {
-    return NULL;
-  }
-}
-
-G_END_DECLS
-
-std::shared_ptr<arrow::io::RandomAccessFile>
-garrow_random_access_file_get_raw(GArrowRandomAccessFile *random_access_file)
-{
-  auto *iface = GARROW_RANDOM_ACCESS_FILE_GET_IFACE(random_access_file);
-  return iface->get_raw(random_access_file);
-}
diff --git a/c_glib/arrow-glib/random-access-file.h b/c_glib/arrow-glib/random-access-file.h
deleted file mode 100644
index 83a7d8cd14b95..0000000000000
--- a/c_glib/arrow-glib/random-access-file.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <glib-object.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_RANDOM_ACCESS_FILE       \
-  (garrow_random_access_file_get_type())
-#define GARROW_RANDOM_ACCESS_FILE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                 \
-                              GARROW_TYPE_RANDOM_ACCESS_FILE,     \
-                              GArrowRandomAccessFile))
-#define GARROW_IS_RANDOM_ACCESS_FILE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_RANDOM_ACCESS_FILE))
-#define GARROW_RANDOM_ACCESS_FILE_GET_IFACE(obj)                     \
-  (G_TYPE_INSTANCE_GET_INTERFACE((obj),                                 \
-                                 GARROW_TYPE_RANDOM_ACCESS_FILE,     \
-                                 GArrowRandomAccessFileInterface))
-
-typedef struct _GArrowRandomAccessFile          GArrowRandomAccessFile;
-typedef struct _GArrowRandomAccessFileInterface GArrowRandomAccessFileInterface;
-
-GType garrow_random_access_file_get_type(void) G_GNUC_CONST;
-
-guint64 garrow_random_access_file_get_size(GArrowRandomAccessFile *file,
-                                              GError **error);
-gboolean garrow_random_access_file_get_support_zero_copy(GArrowRandomAccessFile *file);
-GArrowBuffer *garrow_random_access_file_read_at(GArrowRandomAccessFile *file,
-                                                gint64 position,
-                                                gint64 n_bytes,
-                                                GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/random-access-file.hpp b/c_glib/arrow-glib/random-access-file.hpp
deleted file mode 100644
index 6d6fed70b4b62..0000000000000
--- a/c_glib/arrow-glib/random-access-file.hpp
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow/io/interfaces.h>
-
-#include <arrow-glib/random-access-file.h>
-
-/**
- * GArrowRandomAccessFileInterface:
- *
- * It wraps `arrow::io::RandomAccessFile`.
- */
-struct _GArrowRandomAccessFileInterface
-{
-  GTypeInterface parent_iface;
-
-  std::shared_ptr<arrow::io::RandomAccessFile> (*get_raw)(GArrowRandomAccessFile *file);
-};
-
-std::shared_ptr<arrow::io::RandomAccessFile> garrow_random_access_file_get_raw(GArrowRandomAccessFile *random_access_file);
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 7ba37b45068e0..8c691de93bce4 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -82,7 +82,6 @@
       <title>Input</title>
       <xi:include href="xml/readable.xml"/>
       <xi:include href="xml/input-stream.xml"/>
-      <xi:include href="xml/random-access-file.xml"/>
     </chapter>
     <chapter id="output">
       <title>Output</title>
@@ -93,7 +92,6 @@
     <chapter id="input-output">
       <title>Input and output</title>
       <xi:include href="xml/file.xml"/>
-      <xi:include href="xml/memory-mapped-file.xml"/>
     </chapter>
   </part>
 
diff --git a/c_glib/example/lua/read-batch.lua b/c_glib/example/lua/read-batch.lua
index b28d346863820..8b129c9e4e7a3 100644
--- a/c_glib/example/lua/read-batch.lua
+++ b/c_glib/example/lua/read-batch.lua
@@ -20,7 +20,7 @@ local Arrow = lgi.Arrow
 
 local input_path = arg[1] or "/tmp/batch.arrow";
 
-local input = Arrow.MemoryMappedFile.open(input_path, Arrow.FileMode.READ)
+local input = Arrow.MemoryMappedInputStream.new(input_path)
 local reader = Arrow.FileReader.open(input)
 
 for i = 0, reader:get_n_record_batches() - 1 do
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index 3b0820627e6b2..e744bed22ad4b 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -20,7 +20,7 @@ local Arrow = lgi.Arrow
 
 local input_path = arg[1] or "/tmp/stream.arrow";
 
-local input = Arrow.MemoryMappedFile.open(input_path, Arrow.FileMode.READ)
+local input = Arrow.MemoryMappedInputStream.new(input_path)
 local reader = Arrow.StreamReader.open(input)
 
 local i = 0
diff --git a/c_glib/example/read-batch.c b/c_glib/example/read-batch.c
index dce96b8713362..25f19b24393b2 100644
--- a/c_glib/example/read-batch.c
+++ b/c_glib/example/read-batch.c
@@ -87,14 +87,13 @@ int
 main(int argc, char **argv)
 {
   const char *input_path = "/tmp/batch.arrow";
-  GArrowMemoryMappedFile *input;
+  GArrowMemoryMappedInputStream *input;
   GError *error = NULL;
 
   if (argc > 1)
     input_path = argv[1];
-  input = garrow_memory_mapped_file_open(input_path,
-                                         GARROW_FILE_MODE_READ,
-                                         &error);
+  input = garrow_memory_mapped_input_stream_new(input_path,
+                                                &error);
   if (!input) {
     g_print("failed to open file: %s\n", error->message);
     g_error_free(error);
@@ -104,7 +103,7 @@ main(int argc, char **argv)
   {
     GArrowFileReader *reader;
 
-    reader = garrow_file_reader_open(GARROW_RANDOM_ACCESS_FILE(input),
+    reader = garrow_file_reader_open(GARROW_SEEKABLE_INPUT_STREAM(input),
                                      &error);
     if (!reader) {
       g_print("failed to open file reader: %s\n", error->message);
diff --git a/c_glib/example/read-stream.c b/c_glib/example/read-stream.c
index ba461e3ad6aed..ca5b9d97cc9df 100644
--- a/c_glib/example/read-stream.c
+++ b/c_glib/example/read-stream.c
@@ -87,14 +87,12 @@ int
 main(int argc, char **argv)
 {
   const char *input_path = "/tmp/stream.arrow";
-  GArrowMemoryMappedFile *input;
+  GArrowMemoryMappedInputStream *input;
   GError *error = NULL;
 
   if (argc > 1)
     input_path = argv[1];
-  input = garrow_memory_mapped_file_open(input_path,
-                                         GARROW_FILE_MODE_READ,
-                                         &error);
+  input = garrow_memory_mapped_input_stream_new(input_path, &error);
   if (!input) {
     g_print("failed to open file: %s\n", error->message);
     g_error_free(error);
diff --git a/c_glib/test/test-buffer-reader.rb b/c_glib/test/test-buffer-input-stream.rb
similarity index 85%
rename from c_glib/test/test-buffer-reader.rb
rename to c_glib/test/test-buffer-input-stream.rb
index d05ed062ebdb7..51ed8b3961a75 100644
--- a/c_glib/test/test-buffer-reader.rb
+++ b/c_glib/test/test-buffer-input-stream.rb
@@ -15,11 +15,11 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class TestBufferReader < Test::Unit::TestCase
+class TestBufferInputStream < Test::Unit::TestCase
   def test_read
     buffer = Arrow::Buffer.new("Hello World")
-    buffer_reader = Arrow::BufferReader.new(buffer)
-    read_buffer = buffer_reader.read(5)
+    buffer_input_stream = Arrow::BufferInputStream.new(buffer)
+    read_buffer = buffer_input_stream.read(5)
     assert_equal("Hello", read_buffer.data.to_s)
   end
 end
diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb
index 31c094dd3bd29..6d4100a8cd38a 100644
--- a/c_glib/test/test-file-writer.rb
+++ b/c_glib/test/test-file-writer.rb
@@ -33,7 +33,7 @@ def test_write_record_batch
       output.close
     end
 
-    input = Arrow::MemoryMappedFile.open(tempfile.path, :read)
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
       file_reader = Arrow::FileReader.open(input)
       assert_equal(["enabled"],
diff --git a/c_glib/test/test-memory-mapped-file.rb b/c_glib/test/test-memory-mapped-file.rb
deleted file mode 100644
index e09e3697074c9..0000000000000
--- a/c_glib/test/test-memory-mapped-file.rb
+++ /dev/null
@@ -1,134 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class TestMemoryMappedFile < Test::Unit::TestCase
-  def test_open
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
-    begin
-      buffer = file.read(5)
-      assert_equal("Hello", buffer.data.to_s)
-    ensure
-      file.close
-    end
-  end
-
-  def test_size
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
-    begin
-      assert_equal(5, file.size)
-    ensure
-      file.close
-    end
-  end
-
-  def test_read
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello World")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
-    begin
-      buffer = file.read(5)
-      assert_equal("Hello", buffer.data.to_s)
-    ensure
-      file.close
-    end
-  end
-
-  def test_read_at
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello World")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
-    begin
-      buffer = file.read_at(6, 5)
-      assert_equal("World", buffer.data.to_s)
-    ensure
-      file.close
-    end
-  end
-
-  def test_write
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :readwrite)
-    begin
-      file.write("World")
-    ensure
-      file.close
-    end
-    assert_equal("World", File.read(tempfile.path))
-  end
-
-  def test_write_at
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :readwrite)
-    begin
-      file.write_at(2, "rld")
-    ensure
-      file.close
-    end
-    assert_equal("Herld", File.read(tempfile.path))
-  end
-
-  def test_flush
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :readwrite)
-    begin
-      file.write("World")
-      file.flush
-      assert_equal("World", File.read(tempfile.path))
-    ensure
-      file.close
-    end
-  end
-
-  def test_tell
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello World")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :read)
-    begin
-      file.read(5)
-      assert_equal(5, file.tell)
-    ensure
-      file.close
-    end
-  end
-
-  def test_mode
-    tempfile = Tempfile.open("arrow-io-memory-mapped-file")
-    tempfile.write("Hello World")
-    tempfile.close
-    file = Arrow::MemoryMappedFile.open(tempfile.path, :readwrite)
-    begin
-      assert_equal(Arrow::FileMode::READWRITE, file.mode)
-    ensure
-      file.close
-    end
-  end
-end
diff --git a/c_glib/test/test-memory-mapped-input-stream.rb b/c_glib/test/test-memory-mapped-input-stream.rb
new file mode 100644
index 0000000000000..c3a5f62fbeeb5
--- /dev/null
+++ b/c_glib/test/test-memory-mapped-input-stream.rb
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestMemoryMappedInputStream < Test::Unit::TestCase
+  def test_new
+    tempfile = Tempfile.open("arrow-memory-mapped-input-stream")
+    tempfile.write("Hello")
+    tempfile.close
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      buffer = input.read(5)
+      assert_equal("Hello", buffer.data.to_s)
+    ensure
+      input.close
+    end
+  end
+
+  def test_size
+    tempfile = Tempfile.open("arrow-memory-mapped-input-stream")
+    tempfile.write("Hello")
+    tempfile.close
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      assert_equal(5, input.size)
+    ensure
+      input.close
+    end
+  end
+
+  def test_read
+    tempfile = Tempfile.open("arrow-memory-mapped-input-stream")
+    tempfile.write("Hello World")
+    tempfile.close
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      buffer = input.read(5)
+      assert_equal("Hello", buffer.data.to_s)
+    ensure
+      input.close
+    end
+  end
+
+  def test_read_at
+    tempfile = Tempfile.open("arrow-memory-mapped-input-stream")
+    tempfile.write("Hello World")
+    tempfile.close
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      buffer = input.read_at(6, 5)
+      assert_equal("World", buffer.data.to_s)
+    ensure
+      input.close
+    end
+  end
+
+
+  def test_mode
+    tempfile = Tempfile.open("arrow-memory-mapped-input-stream")
+    tempfile.write("Hello World")
+    tempfile.close
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      assert_equal(Arrow::FileMode::READWRITE, input.mode)
+    ensure
+      input.close
+    end
+  end
+end
diff --git a/c_glib/test/test-stream-writer.rb b/c_glib/test/test-stream-writer.rb
index 306115ee78925..4280c1b32e0f7 100644
--- a/c_glib/test/test-stream-writer.rb
+++ b/c_glib/test/test-stream-writer.rb
@@ -38,7 +38,7 @@ def test_write_record_batch
       output.close
     end
 
-    input = Arrow::MemoryMappedFile.open(tempfile.path, :read)
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
       stream_reader = Arrow::StreamReader.open(input)
       assert_equal(["enabled"],

From 0eff2174f9454e54c24acd988706b2f10a2a380d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 2 May 2017 15:10:23 -0400
Subject: [PATCH 0595/1644] ARROW-933: [Python] Remove debug print statement

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #629 from wesm/ARROW-933 and squashes the following commits:

a6fc22e [Wes McKinney] Remove debug print statement
---
 cpp/src/arrow/python/numpy_interop.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cpp/src/arrow/python/numpy_interop.h b/cpp/src/arrow/python/numpy_interop.h
index 023fdc8249c0c..b93200cc8972d 100644
--- a/cpp/src/arrow/python/numpy_interop.h
+++ b/cpp/src/arrow/python/numpy_interop.h
@@ -47,7 +47,6 @@ namespace py {
 
 inline int import_numpy() {
 #ifdef NUMPY_IMPORT_ARRAY
-  std::cout << "Importing NumPy" << std::endl;
   import_array1(-1);
   import_umath1(-1);
 #endif

From e794a598b89427d5be0442b5009d61086a4af789 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Tue, 2 May 2017 15:13:41 -0400
Subject: [PATCH 0596/1644] ARROW-936: fix release README

Author: Julien Le Dem <julien@apache.org>

Closes #626 from julienledem/fix_README and squashes the following commits:

0489913 [Julien Le Dem] update README
e9f2aec [Julien Le Dem] move tag to rc; add set -e
38f5017 [Julien Le Dem] fix release README
---
 dev/release/00-prepare.sh | 34 ++++++++++----------
 dev/release/01-perform.sh |  1 +
 dev/release/02-source.sh  | 66 ++++++++++++++++++---------------------
 dev/release/README        | 15 ++++++---
 4 files changed, 60 insertions(+), 56 deletions(-)

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
index 00af5e7768161..398f15d8270f1 100644
--- a/dev/release/00-prepare.sh
+++ b/dev/release/00-prepare.sh
@@ -17,30 +17,30 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+set -e
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-if [ -z "$1" ]; then
-  echo "Usage: $0 <version> <nextVersion>"
-  exit
-fi
+if [ "$#" -eq 3 ]; then
+  version=$1
+  nextVersion=$2
+  nextVersionSNAPSHOT=${nextVersion}-SNAPSHOT
+  rcnum=$3
+  tag=apache-arrow-${version}-rc${rcnum}
 
-if [ -z "$2" ]; then
-  echo "Usage: $0 <version> <nextVersion>"
-  exit
-fi
+  echo "prepare release ${version} rc ${rcnum} on tag ${tag} then reset to version ${nextVersionSNAPSHOT}" 
 
-version=$1
+  cd "${SOURCE_DIR}/../../java"
 
-tag=apache-arrow-${version}
+  mvn release:clean
+  mvn release:prepare -Dtag=${tag} -DreleaseVersion=${version} -DautoVersionSubmodules -DdevelopmentVersion=${nextVersionSNAPSHOT}
 
-nextVersion=$2
+  cd -
 
-cd "${SOURCE_DIR}/../../java"
+  echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
 
-mvn release:clean
-mvn release:prepare -Dtag=${tag} -DreleaseVersion=${version} -DautoVersionSubmodules -DdevelopmentVersion=${nextVersion}-SNAPSHOT
-
-cd -
+else  
+  echo "Usage: $0 <version> <nextVersion> <rc-num>"
+  exit
+fi
 
-echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
diff --git a/dev/release/01-perform.sh b/dev/release/01-perform.sh
index d7140f6cba1e7..876dae8442dfb 100644
--- a/dev/release/01-perform.sh
+++ b/dev/release/01-perform.sh
@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+set -e
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index 924b94fd6caa0..3bb72045988cd 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -17,15 +17,11 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+set -e
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-if [ -z "$1" ]; then
-  echo "Usage: $0 <version> <rc-num>"
-  exit
-fi
-
-if [ -z "$2" ]; then
+if [ "$#" -ne 2 ]; then
   echo "Usage: $0 <version> <rc-num>"
   exit
 fi
@@ -38,42 +34,42 @@ if [ -d tmp/ ]; then
   exit
 fi
 
-tag=apache-arrow-$version
+tag=apache-arrow-${version}
 tagrc=${tag}-rc${rc}
 
-echo "Preparing source for $tagrc"
+echo "Preparing source for ${tagrc}"
 
-release_hash=`git rev-list $tag 2> /dev/null | head -n 1 `
+release_hash=`git rev-list $tagrc 2> /dev/null | head -n 1 `
 
 if [ -z "$release_hash" ]; then
-  echo "Cannot continue: unknown git tag: $tag"
+  echo "Cannot continue: unknown git tag: $tagrc"
   exit
 fi
 
 echo "Using commit $release_hash"
 
-tarball=$tag.tar.gz
+tarball=${tag}.tar.gz
 
 extract_dir=tmp-apache-arrow
-rm -rf $extract_dir
+rm -rf ${extract_dir}
 # be conservative and use the release hash, even though git produces the same
 # archive (identical hashes) using the scm tag
-git archive $release_hash --prefix $extract_dir/ | tar xf -
+git archive ${release_hash} --prefix ${extract_dir}/ | tar xf -
 
 # build Apache Arrow C++ before building Apache Arrow GLib because
 # Apache Arrow GLib requires Apache Arrow C++.
-mkdir -p $extract_dir/cpp/build
-cpp_install_dir=$PWD/$extract_dir/cpp/install
-cd $extract_dir/cpp/build
+mkdir -p ${extract_dir}/cpp/build
+cpp_install_dir=${PWD}/${extract_dir}/cpp/install
+cd ${extract_dir}/cpp/build
 cmake .. \
-  -DCMAKE_INSTALL_PREFIX=$cpp_install_dir \
+  -DCMAKE_INSTALL_PREFIX=${cpp_install_dir} \
   -DARROW_BUILD_TESTS=no
 make -j8
 make install
 cd -
 
 # build source archive for Apache Arrow GLib by "make dist".
-cd $extract_dir/c_glib
+cd ${extract_dir}/c_glib
 ./autogen.sh
 ./configure \
   PKG_CONFIG_PATH=$cpp_install_dir/lib/pkgconfig \
@@ -84,38 +80,38 @@ tar xzf *.tar.gz
 rm *.tar.gz
 cd -
 rm -rf tmp-c_glib/
-mv $extract_dir/c_glib/apache-arrow-glib-* tmp-c_glib/
-rm -rf $extract_dir
+mv ${extract_dir}/c_glib/apache-arrow-glib-* tmp-c_glib/
+rm -rf ${extract_dir}
 
 # replace c_glib/ by tar.gz generated by "make dist"
-rm -rf $tag
-git archive $release_hash --prefix $tag/ | tar xf -
-rm -rf $tag/c_glib
-mv tmp-c_glib $tag/c_glib
-tar czf $tarball $tag
-rm -rf $tag
+rm -rf ${tag}
+git archive $release_hash --prefix ${tag}/ | tar xf -
+rm -rf ${tag}/c_glib
+mv tmp-c_glib ${tag}/c_glib
+tar czf ${tarball} ${tag}
+rm -rf ${tag}
 
-${SOURCE_DIR}/run-rat.sh $tarball
+${SOURCE_DIR}/run-rat.sh ${tarball}
 
 # sign the archive
-gpg --armor --output ${tarball}.asc --detach-sig $tarball
-gpg --print-md MD5 $tarball > ${tarball}.md5
+gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
+gpg --print-md MD5 ${tarball} > ${tarball}.md5
 shasum $tarball > ${tarball}.sha
 
 # check out the arrow RC folder
 svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp
 
 # add the release candidate for the tag
-mkdir -p tmp/$tagrc
-cp ${tarball}* tmp/$tagrc
-svn add tmp/$tagrc
-svn ci -m 'Apache Arrow $version RC${rc}' tmp/$tagrc
+mkdir -p tmp/${tagrc}
+cp ${tarball}* tmp/${tagrc}
+svn add tmp/${tagrc}
+svn ci -m 'Apache Arrow ${version} RC${rc}' tmp/${tagrc}
 
 # clean up
 rm -rf tmp
 
 echo "Success! The release candidate is available here:"
-echo "  https://dist.apache.org/repos/dist/dev/arrow/$tagrc"
+echo "  https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
 echo ""
-echo "Commit SHA1: $release_hash"
+echo "Commit SHA1: ${release_hash}"
 
diff --git a/dev/release/README b/dev/release/README
index 07402030bf699..265a23494e43e 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -1,17 +1,24 @@
 requirements:
 - being a committer to be able to push to dist and maven repository
 - a gpg key to sign the artifacts
+- use java 7. check your JAVA_HOME environment variable (at least for now. See ARROW-930) 
+- have the build requirements for cpp and c_glibg installed (see their README)
 
 to release, run the following (replace 0.1.0 with version to release):
 
 #create a release branch
 git co -b release-0_1_0
-# prepare release v 0.1.0 (run tests, sign artifacts). Next version will be 0.1.1-SNAPSHOT
-dev/release/00-prepare.sh 0.1.0 0.1.1
+#setup gpg agent for signing artifacts
+source dev/release/setup-gpg-agent.sh
+
+# prepare release v 0.1.0 (run tests, sign artifacts). Next version will be 0.1.1-SNAPSHOT, RC 0
+sh dev/release/00-prepare.sh 0.1.0 0.1.1 0
+# push the tag
+git push apache apache-arrow-0.1.0-rc0
 # tag and push to maven repo (repo will have to be finalized separately)
-dev/release/01-perform.sh
+sh dev/release/01-perform.sh
 # create the source release
-dev/release/02-source.sh 0.1.0 0
+sh dev/release/02-source.sh 0.1.0 0
 
 useful commands:
 - to set the mvn version in the poms

From 32a4d70db7faf196543b5701c3a5d4b527b7f947 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Tue, 2 May 2017 15:36:55 -0400
Subject: [PATCH 0597/1644] ARROW-936: add missing file; revert tag change

Change-Id: I22603129e9b442ded0def4cd36b564de4a712fc8

Author: Julien Le Dem <julien@apache.org>

Closes #631 from julienledem/fix_README and squashes the following commits:

13bc318 [Julien Le Dem] revert tag change
2f6b14c [Julien Le Dem] add missing file; revert tag change
---
 dev/release/00-prepare.sh      | 7 +++----
 dev/release/02-source.sh       | 6 +++---
 dev/release/README             | 6 +++---
 dev/release/setup-gpg-agent.sh | 3 +++
 4 files changed, 12 insertions(+), 10 deletions(-)
 create mode 100644 dev/release/setup-gpg-agent.sh

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
index 398f15d8270f1..c8d7909ba06f9 100644
--- a/dev/release/00-prepare.sh
+++ b/dev/release/00-prepare.sh
@@ -21,12 +21,11 @@ set -e
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-if [ "$#" -eq 3 ]; then
+if [ "$#" -eq 2 ]; then
   version=$1
   nextVersion=$2
   nextVersionSNAPSHOT=${nextVersion}-SNAPSHOT
-  rcnum=$3
-  tag=apache-arrow-${version}-rc${rcnum}
+  tag=apache-arrow-${version}
 
   echo "prepare release ${version} rc ${rcnum} on tag ${tag} then reset to version ${nextVersionSNAPSHOT}" 
 
@@ -40,7 +39,7 @@ if [ "$#" -eq 3 ]; then
   echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
 
 else  
-  echo "Usage: $0 <version> <nextVersion> <rc-num>"
+  echo "Usage: $0 <version> <nextVersion>"
   exit
 fi
 
diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index 3bb72045988cd..d3d94af0468cb 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -37,12 +37,12 @@ fi
 tag=apache-arrow-${version}
 tagrc=${tag}-rc${rc}
 
-echo "Preparing source for ${tagrc}"
+echo "Preparing source for tag ${tag}"
 
-release_hash=`git rev-list $tagrc 2> /dev/null | head -n 1 `
+release_hash=`git rev-list $tag 2> /dev/null | head -n 1 `
 
 if [ -z "$release_hash" ]; then
-  echo "Cannot continue: unknown git tag: $tagrc"
+  echo "Cannot continue: unknown git tag: $tag"
   exit
 fi
 
diff --git a/dev/release/README b/dev/release/README
index 265a23494e43e..cf68028005e14 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -11,10 +11,10 @@ git co -b release-0_1_0
 #setup gpg agent for signing artifacts
 source dev/release/setup-gpg-agent.sh
 
-# prepare release v 0.1.0 (run tests, sign artifacts). Next version will be 0.1.1-SNAPSHOT, RC 0
-sh dev/release/00-prepare.sh 0.1.0 0.1.1 0
+# prepare release v 0.1.0 (run tests, sign artifacts). Next version will be 0.1.1-SNAPSHOT
+sh dev/release/00-prepare.sh 0.1.0 0.1.1
 # push the tag
-git push apache apache-arrow-0.1.0-rc0
+git push apache apache-arrow-0.1.0
 # tag and push to maven repo (repo will have to be finalized separately)
 sh dev/release/01-perform.sh
 # create the source release
diff --git a/dev/release/setup-gpg-agent.sh b/dev/release/setup-gpg-agent.sh
new file mode 100644
index 0000000000000..acc08094bc5c1
--- /dev/null
+++ b/dev/release/setup-gpg-agent.sh
@@ -0,0 +1,3 @@
+# source me
+eval $(gpg-agent --daemon --allow-preset-passphrase)
+gpg --use-agent -s LICENSE.txt 

From 928b63f40d4d8234644acca36d41bed9390f5f3a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 2 May 2017 15:59:30 -0400
Subject: [PATCH 0598/1644] ARROW-938: Fix Rat license warnings

The .sgml files are generated by the GTK-Doc in c_glib.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #632 from wesm/ARROW-938 and squashes the following commits:

65ca79a [Wes McKinney] Fix Rat warnings
---
 dev/release/run-rat.sh         |  1 +
 dev/release/setup-gpg-agent.sh | 23 ++++++++++++++++++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index 9c34e073e628e..3ff9ef083e523 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -56,6 +56,7 @@ $RAT $1 \
   -e arrow-glib-overrides.txt \
   -e gtk-doc.make \
   -e "*.html" \
+  -e "*.sgml" \
   -e "*.css" \
   -e "*.png" \
   -e "*.svg" \
diff --git a/dev/release/setup-gpg-agent.sh b/dev/release/setup-gpg-agent.sh
index acc08094bc5c1..3e31d0e4e3c55 100644
--- a/dev/release/setup-gpg-agent.sh
+++ b/dev/release/setup-gpg-agent.sh
@@ -1,3 +1,24 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 # source me
 eval $(gpg-agent --daemon --allow-preset-passphrase)
-gpg --use-agent -s LICENSE.txt 
+gpg --use-agent -s LICENSE.txt
+rm -rf LICENSE.txt.gpg

From 2c3e111d45c056d429cef312533c9f3f96b08ae8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 5 May 2017 08:18:53 +0200
Subject: [PATCH 0599/1644] ARROW-923: Changelog generation Python script, add
 0.1.0 and 0.2.0 changelog

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #640 from wesm/ARROW-923 and squashes the following commits:

289d3cd [Wes McKinney] Add license header
96f55f8 [Wes McKinney] Add option to write Markdown JIRA links (for website)
6c808da [Wes McKinney] Changelog Python script, add 0.1.0 and 0.2.0 changelog
---
 CHANGELOG.md          | 403 ++++++++++++++++++++++++++++++++++++++++++
 dev/make_changelog.py |  85 +++++++++
 2 files changed, 488 insertions(+)
 create mode 100644 CHANGELOG.md
 create mode 100644 dev/make_changelog.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000000000..3d54838e1a7f0
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,403 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Apache Arrow 0.2.0 (15 February 2017)
+
+## Bug
+
+* ARROW-112 - [C++]  Style fix for constants/enums
+* ARROW-202 - [C++] Integrate with appveyor ci for windows support and get arrow building on windows
+* ARROW-220 - [C++] Build conda artifacts in a build environment with better cross-linux ABI compatibility
+* ARROW-224 - [C++] Address static linking of boost dependencies
+* ARROW-230 - Python: Do not name modules like native ones (i.e. rename pyarrow.io)
+* ARROW-239 - [Python] HdfsFile.read called with no arguments should read remainder of file
+* ARROW-261 - [C++] Refactor BinaryArray/StringArray classes to not inherit from ListArray
+* ARROW-275 - Add tests for UnionVector in Arrow File
+* ARROW-294 - [C++] Do not use fopen / fclose / etc. methods for memory mapped file implementation
+* ARROW-322 - [C++] Do not build HDFS IO interface optionally
+* ARROW-323 - [Python] Opt-in to PyArrow parquet build rather than skipping silently on failure
+* ARROW-334 - [Python] OS X rpath issues on some configurations
+* ARROW-337 - UnionListWriter.list() is doing more than it should, this can cause data corruption
+* ARROW-339 - Make merge_arrow_pr script work with Python 3
+* ARROW-340 - [C++] Opening a writeable file on disk that already exists does not truncate to zero
+* ARROW-342 - Set Python version on release
+* ARROW-345 - libhdfs integration doesn't work for Mac
+* ARROW-346 - Python API Documentation
+* ARROW-348 - [Python] CMake build type should be configurable on the command line
+* ARROW-349 - Six is missing as a requirement in the python setup.py
+* ARROW-351 - Time type has no unit
+* ARROW-354 - Connot compare an array of empty strings to another
+* ARROW-357 - Default Parquet chunk_size of 64k is too small
+* ARROW-358 - [C++] libhdfs can be in non-standard locations in some Hadoop distributions
+* ARROW-362 - Python: Calling to_pandas on a table read from Parquet leaks memory
+* ARROW-371 - Python: Table with null timestamp becomes float in pandas
+* ARROW-375 - columns parameter in parquet.read_table() raises KeyError for valid column
+* ARROW-384 - Align Java and C++ RecordBatch data and metadata layout
+* ARROW-386 - [Java] Respect case of struct / map field names
+* ARROW-387 - [C++] arrow::io::BufferReader does not permit shared memory ownership in zero-copy reads
+* ARROW-390 - C++: CMake fails on json-integration-test with ARROW_BUILD_TESTS=OFF
+* ARROW-392 - Fix string/binary integration tests
+* ARROW-393 - [JAVA] JSON file reader fails to set the buffer size on String data vector
+* ARROW-395 - Arrow file format writes record batches in reverse order.
+* ARROW-398 - [Java] Java file format requires bitmaps of all 1's to be written when there are no nulls
+* ARROW-399 - [Java] ListVector.loadFieldBuffers ignores the ArrowFieldNode length metadata
+* ARROW-400 - [Java] ArrowWriter writes length 0 for Struct types
+* ARROW-401 - [Java] Floating point vectors should do an approximate comparison in integration tests
+* ARROW-402 - [Java] "refCnt gone negative" error in integration tests
+* ARROW-403 - [JAVA] UnionVector: Creating a transfer pair doesn't transfer the schema to destination vector
+* ARROW-404 - [Python] Closing an HdfsClient while there are still open file handles results in a crash
+* ARROW-405 - [C++] Be less stringent about finding include/hdfs.h in HADOOP_HOME
+* ARROW-406 - [C++] Large HDFS reads must utilize the set file buffer size when making RPCs
+* ARROW-408 - [C++/Python] Remove defunct conda recipes
+* ARROW-414 - [Java] "Buffer too large to resize to ..." error
+* ARROW-420 - Align Date implementation between Java and C++
+* ARROW-421 - [Python] Zero-copy buffers read by pyarrow::PyBytesReader must retain a reference to the parent PyBytes to avoid premature garbage collection issues
+* ARROW-422 - C++: IPC should depend on rapidjson_ep if RapidJSON is vendored
+* ARROW-429 - git-archive SHA-256 checksums are changing
+* ARROW-433 - [Python] Date conversion is locale-dependent
+* ARROW-434 - Segfaults and encoding issues in Python Parquet reads
+* ARROW-435 - C++: Spelling mistake in if(RAPIDJSON_VENDORED)
+* ARROW-437 - [C++] clang compiler warnings from overridden virtual functions
+* ARROW-445 - C++: arrow_ipc is built before arrow/ipc/Message_generated.h was generated
+* ARROW-447 - Python: Align scalar/pylist string encoding with pandas' one.
+* ARROW-455 - [C++] BufferOutputStream dtor does not call Close()
+* ARROW-469 - C++: Add option so that resize doesn't decrease the capacity
+* ARROW-481 - [Python] Fix Python 2.7 regression in patch for PARQUET-472
+* ARROW-486 - [C++] arrow::io::MemoryMappedFile can't be casted to arrow::io::FileInterface
+* ARROW-487 - Python: ConvertTableToPandas segfaults if ObjectBlock::Write fails
+* ARROW-494 - [C++] When MemoryMappedFile is destructed, memory is unmapped even if buffer referecnes still exist
+* ARROW-499 - Update file serialization to use streaming serialization format
+* ARROW-505 - [C++] Fix compiler warnings in release mode
+* ARROW-511 - [Python] List[T] conversions not implemented for single arrays
+* ARROW-513 - [C++] Fix Appveyor build
+* ARROW-519 - [C++] Missing vtable in libarrow.dylib on Xcode 6.4
+* ARROW-523 - Python: Account for changes in PARQUET-834
+* ARROW-533 - [C++] arrow::TimestampArray / TimeArray has a broken constructor
+* ARROW-535 - [Python] Add type mapping for NPY_LONGLONG
+* ARROW-537 - [C++] StringArray/BinaryArray comparisons may be incorrect when values with non-zero length are null
+* ARROW-540 - [C++] Fix build in aftermath of ARROW-33
+* ARROW-543 - C++: Lazily computed null_counts counts number of non-null entries
+* ARROW-544 - [C++] ArrayLoader::LoadBinary fails for length-0 arrays
+* ARROW-545 - [Python] Ignore files without .parq or .parquet prefix when reading directory of files
+* ARROW-548 - [Python] Add nthreads option to pyarrow.Filesystem.read_parquet
+* ARROW-551 - C++: Construction of Column with nullptr Array segfaults
+* ARROW-556 - [Integration] Can not run Integration tests if different cpp build path
+* ARROW-561 - Update java & python dependencies to improve downstream packaging experience
+
+## Improvement
+
+* ARROW-189 - C++: Use ExternalProject to build thirdparty dependencies
+* ARROW-191 - Python: Provide infrastructure for manylinux1 wheels
+* ARROW-328 - [C++] Return shared_ptr by value instead of const-ref?
+* ARROW-330 - [C++] CMake functions to simplify shared / static library configuration
+* ARROW-333 - Make writers update their internal schema even when no data is written.
+* ARROW-335 - Improve Type apis and toString() by encapsulating flatbuffers better
+* ARROW-336 - Run Apache Rat in Travis builds
+* ARROW-338 - [C++] Refactor IPC vector "loading" and "unloading" to be based on cleaner visitor pattern
+* ARROW-350 - Add Kerberos support to HDFS shim
+* ARROW-355 - Add tests for serialising arrays of empty strings to Parquet
+* ARROW-356 - Add documentation about reading Parquet
+* ARROW-360 - C++: Add method to shrink PoolBuffer using realloc
+* ARROW-361 - Python: Support reading a column-selection from Parquet files
+* ARROW-365 - Python: Provide Array.to_pandas()
+* ARROW-366 - [java] implement Dictionary vector
+* ARROW-374 - Python: clarify unicode vs. binary in API
+* ARROW-379 - Python: Use setuptools_scm/setuptools_scm_git_archive to provide the version number
+* ARROW-380 - [Java] optimize null count when serializing vectors.
+* ARROW-382 - Python: Extend API documentation
+* ARROW-396 - Python: Add pyarrow.schema.Schema.equals
+* ARROW-409 - Python: Change pyarrow.Table.dataframe_from_batches API to create Table instead
+* ARROW-411 - [Java] Move Intergration.compare and Intergration.compareSchemas to a public utils class
+* ARROW-423 - C++: Define BUILD_BYPRODUCTS in external project to support non-make CMake generators
+* ARROW-425 - Python: Expose a C function to convert arrow::Table to pyarrow.Table
+* ARROW-426 - Python: Conversion from pyarrow.Array to a Python list
+* ARROW-430 - Python: Better version handling
+* ARROW-432 - [Python] Avoid unnecessary memory copy in to_pandas conversion by using low-level pandas internals APIs
+* ARROW-450 - Python: Fixes for PARQUET-818
+* ARROW-457 - Python: Better control over memory pool
+* ARROW-458 - Python: Expose jemalloc MemoryPool
+* ARROW-463 - C++: Support jemalloc 4.x
+* ARROW-466 - C++: ExternalProject for jemalloc
+* ARROW-468 - Python: Conversion of nested data in pd.DataFrames to/from Arrow structures
+* ARROW-474 - Create an Arrow streaming file fomat
+* ARROW-479 - Python: Test for expected schema in Pandas conversion
+* ARROW-485 - [Java] Users are required to initialize VariableLengthVectors.offsetVector before calling VariableLengthVectors.mutator.getSafe
+* ARROW-490 - Python: Update manylinux1 build scripts
+* ARROW-524 - [java] provide apis to access nested vectors and buffers
+* ARROW-525 - Python: Add more documentation to the package
+* ARROW-529 - Python: Add jemalloc and Python 3.6 to manylinux1 build
+* ARROW-546 - Python: Account for changes in PARQUET-867
+* ARROW-553 - C++: Faster valid bitmap building
+
+## New Feature
+
+* ARROW-108 - [C++] Add IPC round trip for union types
+* ARROW-221 - Add switch for writing Parquet 1.0 compatible logical types
+* ARROW-227 - [C++/Python] Hook arrow_io generic reader / writer interface into arrow_parquet
+* ARROW-228 - [Python] Create an Arrow-cpp-compatible interface for reading bytes from Python file-like objects
+* ARROW-243 - [C++] Add "driver" option to HdfsClient to choose between libhdfs and libhdfs3 at runtime
+* ARROW-303 - [C++] Also build static libraries for leaf libraries
+* ARROW-312 - [Python] Provide Python API to read/write the Arrow IPC file format
+* ARROW-317 - [C++] Implement zero-copy Slice method on arrow::Buffer that retains reference to parent
+* ARROW-33 - C++: Implement zero-copy array slicing
+* ARROW-332 - [Python] Add helper function to convert RecordBatch to pandas.DataFrame
+* ARROW-363 - Set up Java/C++ integration test harness
+* ARROW-369 - [Python] Add ability to convert multiple record batches at once to pandas
+* ARROW-373 - [C++] Implement C++ version of JSON file format for testing
+* ARROW-377 - Python: Add support for conversion of Pandas.Categorical
+* ARROW-381 - [C++] Simplify primitive array type builders to use a default type singleton
+* ARROW-383 - [C++] Implement C++ version of ARROW-367 integration test validator
+* ARROW-389 - Python: Write Parquet files to pyarrow.io.NativeFile objects
+* ARROW-394 - Add integration tests for boolean, list, struct, and other basic types
+* ARROW-410 - [C++] Add Flush method to arrow::io::OutputStream
+* ARROW-415 - C++: Add Equals implementation to compare Tables
+* ARROW-416 - C++: Add Equals implementation to compare Columns
+* ARROW-417 - C++: Add Equals implementation to compare ChunkedArrays
+* ARROW-418 - [C++] Consolidate array container and builder code, remove arrow/types
+* ARROW-419 - [C++] Promote util/{status.h, buffer.h, memory-pool.h} to top level of arrow/ source directory
+* ARROW-427 - [C++] Implement dictionary-encoded array container
+* ARROW-428 - [Python] Deserialize from Arrow record batches to pandas in parallel using a thread pool
+* ARROW-438 - [Python] Concatenate Table instances with equal schemas
+* ARROW-440 - [C++] Support pkg-config
+* ARROW-441 - [Python] Expose Arrow's file and memory map classes as NativeFile subclasses
+* ARROW-442 - [Python] Add public Python API to inspect Parquet file metadata
+* ARROW-444 - [Python] Avoid unnecessary memory copies from use of PyBytes_* C APIs
+* ARROW-449 - Python: Conversion from pyarrow.{Table,RecordBatch} to a Python dict
+* ARROW-456 - C++: Add jemalloc based MemoryPool
+* ARROW-461 - [Python] Implement conversion between arrow::DictionaryArray and pandas.Categorical
+* ARROW-467 - [Python] Run parquet-cpp unit tests in Travis CI
+* ARROW-470 - [Python] Add "FileSystem" abstraction to access directories of files in a uniform way
+* ARROW-471 - [Python] Enable ParquetFile to pass down separately-obtained file metadata
+* ARROW-472 - [Python] Expose parquet::{SchemaDescriptor, ColumnDescriptor}::Equals
+* ARROW-475 - [Python] High level support for reading directories of Parquet files (as a single Arrow table) from supported file system interfaces
+* ARROW-476 - [Integration] Add integration tests for Binary / Varbytes type
+* ARROW-477 - [Java] Add support for second/microsecond/nanosecond timestamps in-memory and in IPC/JSON layer
+* ARROW-478 - [Python] Accept a PyBytes object in the pyarrow.io.BufferReader ctor
+* ARROW-484 - Add more detail about what of technology can be found in the Arrow implementations to README
+* ARROW-495 - [C++] Add C++ implementation of streaming serialized format
+* ARROW-497 - [Java] Integration test harness for streaming format
+* ARROW-498 - [C++] Integration test harness for streaming format
+* ARROW-503 - [Python] Interface to streaming binary format
+* ARROW-508 - [C++] Make file/memory-mapped file interfaces threadsafe
+* ARROW-509 - [Python] Add support for PARQUET-835 (parallel column reads)
+* ARROW-512 - C++: Add method to check for primitive types
+* ARROW-514 - [Python] Accept pyarrow.io.Buffer as input to StreamReader, FileReader classes
+* ARROW-515 - [Python] Add StreamReader/FileReader methods that read all record batches as a Table
+* ARROW-521 - [C++/Python] Track peak memory use in default MemoryPool
+* ARROW-531 - Python: Document jemalloc, extend Pandas section, add Getting Involved
+* ARROW-538 - [C++] Set up AddressSanitizer (ASAN) builds
+* ARROW-547 - [Python] Expose Array::Slice and RecordBatch::Slice
+* ARROW-81 - [Format] Add a Category logical type (distinct from dictionary-encoding)
+
+## Task
+
+* ARROW-268 - [C++] Flesh out union implementation to have all required methods for IPC
+* ARROW-327 - [Python] Remove conda builds from Travis CI processes
+* ARROW-353 - Arrow release 0.2
+* ARROW-359 - Need to document ARROW_LIBHDFS_DIR
+* ARROW-367 - [java] converter csv/json <=> Arrow file format for Integration tests
+* ARROW-368 - Document use of LD_LIBRARY_PATH when using Python
+* ARROW-372 - Create JSON arrow file format for integration tests
+* ARROW-506 - Implement Arrow Echo server for integration testing
+* ARROW-527 - clean drill-module.conf file
+* ARROW-558 - Add KEYS files
+* ARROW-96 - C++: API documentation using Doxygen
+* ARROW-97 - Python: API documentation via sphinx-apidoc
+
+# Apache Arrow 0.1.0 (7 October 2016)
+
+## Bug
+
+* ARROW-103 - Missing patterns from .gitignore
+* ARROW-104 - Update Layout.md based on discussion on the mailing list
+* ARROW-105 - Unit tests fail if assertions are disabled
+* ARROW-113 - TestValueVector test fails if cannot allocate 2GB of memory
+* ARROW-16 - Building cpp issues on XCode 7.2.1
+* ARROW-17 - Set some vector fields to default access level for Drill compatibility
+* ARROW-18 - Fix bug with decimal precision and scale
+* ARROW-185 - [C++] Make sure alignment and memory padding conform to spec
+* ARROW-188 - Python: Add numpy as install requirement
+* ARROW-193 - For the instruction, typos "int his" should be "in this"
+* ARROW-194 - C++: Allow read-only memory mapped source
+* ARROW-200 - [Python] Convert Values String looks like it has incorrect error handling
+* ARROW-209 - [C++] Broken builds: llvm.org apt repos are unavailable
+* ARROW-210 - [C++] Tidy up the type system a little bit
+* ARROW-211 - Several typos/errors in Layout.md examples
+* ARROW-217 - Fix Travis w.r.t conda 4.1.0 changes
+* ARROW-219 - [C++] Passed CMAKE_CXX_FLAGS are being dropped, fix compiler warnings
+* ARROW-223 - Do not link against libpython
+* ARROW-225 - [C++/Python] master Travis CI build is broken
+* ARROW-244 - [C++] Some global APIs of IPC module should be visible to the outside
+* ARROW-246 - [Java] UnionVector doesn't call allocateNew() when creating it's vectorType
+* ARROW-247 - [C++] Missing explicit destructor in RowBatchReader causes an incomplete type error
+* ARROW-250 - Fix for ARROW-246 may cause memory leaks
+* ARROW-259 - Use flatbuffer fields in java implementation
+* ARROW-265 - Negative decimal values have wrong padding
+* ARROW-266 - [C++] Fix the broken build
+* ARROW-274 - Make the MapVector nullable
+* ARROW-278 - [Format] Struct type name consistency in implementations and metadata
+* ARROW-283 - [C++] Update arrow_parquet to account for API changes in PARQUET-573
+* ARROW-284 - [C++] Triage builds by disabling Arrow-Parquet module
+* ARROW-287 - [java] Make nullable vectors use a BitVecor instead of UInt1Vector for bits
+* ARROW-297 - Fix Arrow pom for release
+* ARROW-304 - NullableMapReaderImpl.isSet() always returns true
+* ARROW-308 - UnionListWriter.setPosition() should not call startList()
+* ARROW-309 - Types.getMinorTypeForArrowType() does not work for Union type
+* ARROW-313 - XCode 8.0 breaks builds
+* ARROW-314 - JSONScalar is unnecessary and unused.
+* ARROW-320 - ComplexCopier.copy(FieldReader, FieldWriter) should not start a list if reader is not set
+* ARROW-321 - Fix Arrow licences
+* ARROW-36 - Remove fixVersions from patch tool (until we have them)
+* ARROW-46 - Port DRILL-4410 to Arrow
+* ARROW-5 - Error when run maven install
+* ARROW-51 - Move ValueVector test from Drill project
+* ARROW-55 - Python: fix legacy Python (2.7) tests and add to Travis CI
+* ARROW-62 - Format: Are the nulls bits 0 or 1 for null values?
+* ARROW-63 - C++: ctest fails if Python 3 is the active Python interpreter
+* ARROW-65 - Python: FindPythonLibsNew does not work in a virtualenv
+* ARROW-69 - Change permissions for assignable users
+* ARROW-72 - FindParquet searches for non-existent header
+* ARROW-75 - C++: Fix handling of empty strings
+* ARROW-77 - C++: conform null bit interpretation to match ARROW-62
+* ARROW-80 - Segmentation fault on len(Array) for empty arrays
+* ARROW-88 - C++: Refactor given PARQUET-572
+* ARROW-93 - XCode 7.3 breaks builds
+* ARROW-94 - Expand list example to clarify null vs empty list
+
+## Improvement
+
+* ARROW-10 - Fix mismatch of javadoc names and method parameters
+* ARROW-15 - Fix a naming typo for memory.AllocationManager.AllocationOutcome
+* ARROW-190 - Python: Provide installable sdist builds
+* ARROW-199 - [C++] Refine third party dependency
+* ARROW-206 - [C++] Expose an equality API for arrays that compares a range of slots on two arrays
+* ARROW-212 - [C++] Clarify the fact that PrimitiveArray is now abstract class
+* ARROW-213 - Exposing static arrow build
+* ARROW-218 - Add option to use GitHub API token via environment variable when merging PRs
+* ARROW-234 - [C++] Build with libhdfs support in arrow_io in conda builds
+* ARROW-238 - C++: InternalMemoryPool::Free() should throw an error when there is insufficient allocated memory
+* ARROW-245 - [Format] Clarify Arrow's relationship with big endian platforms
+* ARROW-252 - Add implementation guidelines to the documentation
+* ARROW-253 - Int types should only have width of 8*2^n (8, 16, 32, 64)
+* ARROW-254 - Remove Bit type as it is redundant with boolean
+* ARROW-255 - Finalize Dictionary representation
+* ARROW-256 - Add versioning to the arrow spec.
+* ARROW-257 - Add a typeids Vector to Union type
+* ARROW-264 - Create an Arrow File format
+* ARROW-270 - [Format] Define more generic Interval logical type
+* ARROW-271 - Update Field structure to be more explicit
+* ARROW-279 - rename vector module to arrow-vector for consistency
+* ARROW-280 - [C++] Consolidate file and shared memory IO interfaces
+* ARROW-285 - Allow for custom flatc compiler
+* ARROW-286 - Build thirdparty dependencies in parallel
+* ARROW-289 - Install test-util.h
+* ARROW-290 - Specialize alloc() in ArrowBuf
+* ARROW-292 - [Java] Upgrade Netty to 4.041
+* ARROW-299 - Use absolute namespace in macros
+* ARROW-305 - Add compression and use_dictionary options to Parquet interface
+* ARROW-306 - Add option to pass cmake arguments via environment variable
+* ARROW-315 - Finalize timestamp type
+* ARROW-319 - Add canonical Arrow Schema json representation
+* ARROW-324 - Update arrow metadata diagram
+* ARROW-325 - make TestArrowFile not dependent on timezone
+* ARROW-50 - C++: Enable library builds for 3rd-party users without having to build thirdparty googletest
+* ARROW-54 - Python: rename package to "pyarrow"
+* ARROW-64 - Add zsh support to C++ build scripts
+* ARROW-66 - Maybe some missing steps in installation guide
+* ARROW-68 - Update setup_build_env and third-party script to be more userfriendly
+* ARROW-71 - C++: Add script to run clang-tidy on codebase
+* ARROW-73 - Support CMake 2.8
+* ARROW-78 - C++: Add constructor for DecimalType
+* ARROW-79 - Python: Add benchmarks
+* ARROW-8 - Set up Travis CI
+* ARROW-85 - C++: memcmp can be avoided in Equal when comparing with the same Buffer
+* ARROW-86 - Python: Implement zero-copy Arrow-to-Pandas conversion
+* ARROW-87 - Implement Decimal schema conversion for all ways supported in Parquet
+* ARROW-89 - Python: Add benchmarks for Arrow<->Pandas conversion
+* ARROW-9 - Rename some unchanged "Drill" to "Arrow"
+* ARROW-91 - C++: First draft of an adapter class for parquet-cpp's ParquetFileReader that produces Arrow table/row batch objects
+
+## New Feature
+
+* ARROW-100 - [C++] Computing RowBatch size
+* ARROW-106 - Add IPC round trip for string types (string, char, varchar, binary)
+* ARROW-107 - [C++] add ipc round trip for struct types
+* ARROW-13 - Add PR merge tool similar to that used in Parquet
+* ARROW-19 - C++: Externalize memory allocations and add a MemoryPool abstract interface to builder classes
+* ARROW-197 - [Python] Add conda dev recipe for pyarrow
+* ARROW-2 - Post Simple Website
+* ARROW-20 - C++: Add null count member to Array containers, remove nullable member
+* ARROW-201 - C++: Initial ParquetWriter implementation
+* ARROW-203 - Python: Basic filename based Parquet read/write
+* ARROW-204 - [Python] Automate uploading conda build artifacts for libarrow and pyarrow
+* ARROW-21 - C++: Add in-memory schema metadata container
+* ARROW-214 - C++: Add String support to Parquet I/O
+* ARROW-215 - C++: Support other integer types in Parquet I/O
+* ARROW-22 - C++: Add schema adapter routines for converting flat Parquet schemas to in-memory Arrow schemas
+* ARROW-222 - [C++] Create prototype file-like interface to HDFS (via libhdfs) and begin defining more general IO interface for Arrow data adapters
+* ARROW-23 - C++: Add logical "Column" container for chunked data
+* ARROW-233 - [C++] Add visibility defines for limiting shared library symbol visibility
+* ARROW-236 - [Python] Enable Parquet read/write to work with HDFS file objects
+* ARROW-237 - [C++] Create Arrow specializations of Parquet allocator and read interfaces
+* ARROW-24 - C++: Add logical "Table" container
+* ARROW-242 - C++/Python: Support Timestamp Data Type
+* ARROW-26 - C++: Add developer instructions for building parquet-cpp integration
+* ARROW-262 - [Format] Add a new format document for metadata and logical types for messaging and IPC / on-wire/file representations
+* ARROW-267 - [C++] C++ implementation of file-like layout for RPC / IPC
+* ARROW-28 - C++: Add google/benchmark to the 3rd-party build toolchain
+* ARROW-293 - [C++] Implementations of IO interfaces for operating system files
+* ARROW-296 - [C++] Remove arrow_parquet C++ module and related parts of build system
+* ARROW-3 - Post Initial Arrow Format Spec
+* ARROW-30 - Python: pandas/NumPy to/from Arrow conversion routines
+* ARROW-301 - [Format] Add some form of user field metadata to IPC schemas
+* ARROW-302 - [Python] Add support to use the Arrow file format with file-like objects
+* ARROW-31 - Python: basic PyList <-> Arrow marshaling code
+* ARROW-318 - [Python] Revise README to reflect current state of project
+* ARROW-37 - C++: Represent boolean array data in bit-packed form
+* ARROW-4 - Initial Arrow CPP Implementation
+* ARROW-42 - Python: Add to Travis CI build
+* ARROW-43 - Python: Add rudimentary console __repr__ for array types
+* ARROW-44 - Python: Implement basic object model for scalar values (i.e. results of arrow_arr[i])
+* ARROW-48 - Python: Add Schema object wrapper
+* ARROW-49 - Python: Add Column and Table wrapper interface
+* ARROW-53 - Python: Fix RPATH and add source installation instructions
+* ARROW-56 - Format: Specify LSB bit ordering in bit arrays
+* ARROW-57 - Format: Draft data headers IDL for data interchange
+* ARROW-58 - Format: Draft type metadata ("schemas") IDL
+* ARROW-59 - Python: Boolean data support for builtin data structures
+* ARROW-60 - C++: Struct type builder API
+* ARROW-67 - C++: Draft type metadata conversion to/from IPC representation
+* ARROW-7 - Add Python library build toolchain
+* ARROW-70 - C++: Add "lite" DCHECK macros used in parquet-cpp
+* ARROW-76 - Revise format document to include null count, defer non-nullable arrays to the domain of metadata
+* ARROW-82 - C++: Implement IPC exchange for List types
+* ARROW-90 - Apache Arrow cpp code does not support power architecture
+* ARROW-92 - C++: Arrow to Parquet Schema conversion
+
+## Task
+
+* ARROW-1 - Import Initial Codebase
+* ARROW-101 - Fix java warnings emitted by java compiler
+* ARROW-102 - travis-ci support for java project
+* ARROW-11 - Mirror JIRA activity to dev@arrow.apache.org
+* ARROW-14 - Add JIRA components
+* ARROW-251 - [C++] Expose APIs for getting code and message of the status
+* ARROW-272 - Arrow release 0.1
+* ARROW-298 - create release scripts
+* ARROW-35 - Add a short call-to-action / how-to-get-involved to the main README.md
+
+## Test
+
+* ARROW-260 - TestValueVector.testFixedVectorReallocation and testVariableVectorReallocation are flaky
+* ARROW-83 - Add basic test infrastructure for DecimalType
diff --git a/dev/make_changelog.py b/dev/make_changelog.py
new file mode 100644
index 0000000000000..0ad1607c79326
--- /dev/null
+++ b/dev/make_changelog.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Utility for generating changelogs for fix versions
+# requirements: pip install jira
+# Set $JIRA_USERNAME, $JIRA_PASSWORD environment variables
+
+from collections import defaultdict
+from io import StringIO
+import os
+import sys
+
+import jira.client
+
+# ASF JIRA username
+JIRA_USERNAME = os.environ.get("JIRA_USERNAME")
+# ASF JIRA password
+JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD")
+
+JIRA_API_BASE = "https://issues.apache.org/jira"
+
+asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
+                            basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
+
+
+def get_issues_for_version(version):
+    jql = ("project=ARROW "
+           "AND fixVersion='{0}' "
+           "AND status = Resolved "
+           "AND resolution in (Fixed, Done) "
+           "ORDER BY issuetype DESC").format(version)
+
+    return asf_jira.search_issues(jql, maxResults=9999)
+
+
+LINK_TEMPLATE = '[{0}](https://issues.apache.org/jira/browse/{0})'
+
+
+def format_changelog_markdown(issues, out, links=False):
+    issues_by_type = defaultdict(list)
+    for issue in issues:
+        issues_by_type[issue.fields.issuetype.name].append(issue)
+
+
+    for issue_type, issue_group in sorted(issues_by_type.items()):
+        issue_group.sort(key=lambda x: x.key)
+
+        out.write('## {0}\n\n'.format(issue_type))
+        for issue in issue_group:
+            if links:
+                name = LINK_TEMPLATE.format(issue.key)
+            else:
+                name = issue.key
+            out.write('* {0} - {1}\n'.format(name,
+                                             issue.fields.summary))
+        out.write('\n')
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('Usage: make_changelog.py $FIX_VERSION [$LINKS]')
+
+    buf = StringIO()
+
+    links = len(sys.argv) > 2 and sys.argv[2] == '1'
+
+    issues_for_version = get_issues_for_version(sys.argv[1])
+    format_changelog_markdown(issues_for_version, buf, links=links)
+    print(buf.getvalue())

From 80b72d43e093f90e4c207e61a0f408aef7057c94 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 2 May 2017 16:07:00 -0400
Subject: [PATCH 0600/1644] [maven-release-plugin] prepare for next development
 iteration

Change-Id: I2c3a4705c6e7c39c75cd1feb195ad167aeb12084
---
 java/format/pom.xml |  2 +-
 java/memory/pom.xml | 20 ++++++++++----------
 java/pom.xml        |  2 +-
 java/tools/pom.xml  |  2 +-
 java/vector/pom.xml |  2 +-
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 98a113a30cf78..4ef748235152d 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.2.1-SNAPSHOT</version>
+  <version>0.4.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index f20228b1bee62..e6d9900e24e80 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -1,20 +1,20 @@
 <?xml version="1.0"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
-  license agreements. See the NOTICE file distributed with this work for additional 
-  information regarding copyright ownership. The ASF licenses this file to 
-  You under the Apache License, Version 2.0 (the "License"); you may not use 
-  this file except in compliance with the License. You may obtain a copy of 
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
-  by applicable law or agreed to in writing, software distributed under the 
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
-  OF ANY KIND, either express or implied. See the License for the specific 
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
   language governing permissions and limitations under the License. -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.2.1-SNAPSHOT</version>
+    <version>0.4.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index e586005e395c0..1fa8ef9b457be 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.2.1-SNAPSHOT</version>
+  <version>0.4.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 35e5599b3b64c..6124b85379fe4 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.2.1-SNAPSHOT</version>
+        <version>0.4.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index fc3ce66ac1f80..e09193692a94d 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.2.1-SNAPSHOT</version>
+    <version>0.4.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From bcf073c3aeca872e41f86cee14d2c43598ce3149 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 5 May 2017 10:38:33 -0400
Subject: [PATCH 0601/1644] ARROW-945: [GLib] Add a Lua example to show Torch
 integration

Author: Kouhei Sutou <kou@clear-code.com>

Closes #637 from kou/glib-lua-to-torch-tensor and squashes the following commits:

4aba395 [Kouhei Sutou] [GLib] Add a Lua example to show Torch integration
---
 c_glib/example/lua/Makefile.am                |   1 +
 c_glib/example/lua/README.md                  |   5 +
 c_glib/example/lua/read-stream.lua            |   2 +-
 c_glib/example/lua/stream-to-torch-tensor.lua | 101 ++++++++++++++++++
 4 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 c_glib/example/lua/stream-to-torch-tensor.lua

diff --git a/c_glib/example/lua/Makefile.am b/c_glib/example/lua/Makefile.am
index 9019d24741c1a..86bdbed8a0228 100644
--- a/c_glib/example/lua/Makefile.am
+++ b/c_glib/example/lua/Makefile.am
@@ -20,5 +20,6 @@ dist_lua_example_DATA =				\
 	README.md				\
 	read-batch.lua				\
 	read-stream.lua				\
+	stream-to-torch-tensor.lua		\
 	write-batch.lua				\
 	write-stream.lua
diff --git a/c_glib/example/lua/README.md b/c_glib/example/lua/README.md
index d127573bcc368..6145bc74ddd5a 100644
--- a/c_glib/example/lua/README.md
+++ b/c_glib/example/lua/README.md
@@ -43,3 +43,8 @@ Here are example codes in this directory:
 
   * `read-stream.lua`: It shows how to read Arrow array from file in
     stream mode.
+
+  * `stream-to-torch-tensor.lua`: It shows how to read Arrow array
+    from file in stream mode and convert it to
+    [Torch](http://torch.ch/)'s
+    [`Tensor` object](http://torch7.readthedocs.io/en/rtd/tensor/index.html).
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index e744bed22ad4b..987d463b981cf 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -25,7 +25,7 @@ local reader = Arrow.StreamReader.open(input)
 
 local i = 0
 while true do
-   local record_batch = reader:get_next_record_batch(i)
+   local record_batch = reader:get_next_record_batch()
    if not record_batch then
       break
    end
diff --git a/c_glib/example/lua/stream-to-torch-tensor.lua b/c_glib/example/lua/stream-to-torch-tensor.lua
new file mode 100644
index 0000000000000..237d759d93e20
--- /dev/null
+++ b/c_glib/example/lua/stream-to-torch-tensor.lua
@@ -0,0 +1,101 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local torch = require 'torch'
+
+Arrow.Array.torch_types = function(self)
+   return nil
+end
+
+Arrow.Array.to_torch = function(self)
+   local types = self:torch_types()
+   if not types then
+      return nil
+   end
+
+   local storage_type = types[1]
+   local tensor_type = types[2]
+
+   local size = self:get_length()
+   local storage = storage_type(size)
+   if not storage then
+      return nil
+   end
+
+   for i = 1, size do
+      storage[i] = self:get_value(i - 1)
+   end
+   return tensor_type(storage)
+end
+
+Arrow.UInt8Array.torch_types = function(self)
+   return {torch.ByteStorage, torch.ByteTensor}
+end
+
+Arrow.Int8Array.torch_types = function(self)
+   return {torch.CharStorage, torch.CharTensor}
+end
+
+Arrow.Int16Array.torch_types = function(self)
+   return {torch.ShortStorage, torch.ShortTensor}
+end
+
+Arrow.Int32Array.torch_types = function(self)
+   return {torch.IntStorage, torch.IntTensor}
+end
+
+Arrow.Int64Array.torch_types = function(self)
+   return {torch.LongStorage, torch.LongTensor}
+end
+
+Arrow.FloatArray.torch_types = function(self)
+   return {torch.FloatStorage, torch.FloatTensor}
+end
+
+Arrow.DoubleArray.torch_types = function(self)
+   return {torch.DoubleStorage, torch.DoubleTensor}
+end
+
+
+local input_path = arg[1] or "/tmp/stream.arrow";
+
+local input = Arrow.MemoryMappedInputStream.new(input_path)
+local reader = Arrow.StreamReader.open(input)
+
+local i = 0
+while true do
+   local record_batch = reader:get_next_record_batch()
+   if not record_batch then
+      break
+   end
+
+   print(string.rep("=", 40))
+   print("record-batch["..i.."]:")
+   for j = 0, record_batch:get_n_columns() - 1 do
+      local column = record_batch:get_column(j)
+      local column_name = record_batch:get_column_name(j)
+      print("  "..column_name..":")
+      print(column:to_torch())
+   end
+
+   i = i + 1
+end
+
+input:close()

From 9a48773afa369cbbfd4c3354134125e82e0691b7 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 5 May 2017 10:44:20 -0400
Subject: [PATCH 0602/1644] ARROW-943: [GLib] Support running unit tests with
 source archive

Author: Kouhei Sutou <kou@clear-code.com>

Closes #635 from kou/glib-dist-test and squashes the following commits:

2c30729 [Kouhei Sutou] [GLib] Support running unit tests with source archive
---
 c_glib/Makefile.am      |  1 +
 c_glib/README.md        | 50 ++++++++++++++++++++++++++++++++++++++++-
 c_glib/test/run-test.sh |  2 +-
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am
index bb52ce503e04e..2e23f125683ba 100644
--- a/c_glib/Makefile.am
+++ b/c_glib/Makefile.am
@@ -24,6 +24,7 @@ SUBDIRS =					\
 
 EXTRA_DIST =					\
 	README.md				\
+	test					\
 	version
 
 arrow_glib_docdir = ${datarootdir}/doc/arrow-glib
diff --git a/c_glib/README.md b/c_glib/README.md
index 6eadb797032bc..b6e08e358d244 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -143,7 +143,7 @@ You need to install Arrow C++ before you install Arrow GLib. See Arrow
 C++ document about how to install Arrow C++.
 
 You need [GTK-Doc](https://www.gtk.org/gtk-doc/) and
-[GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection)
+[GObject Introspection](https://wiki.gnome.org/Projects/GObjectIntrospection)
 to build Arrow GLib. You can install them by the followings:
 
 On Debian GNU/Linux or Ubuntu:
@@ -206,3 +206,51 @@ based bindings. Here are languages that support GObject Introspection:
 See also
 [Projects/GObjectIntrospection/Users - GNOME Wiki!](https://wiki.gnome.org/Projects/GObjectIntrospection/Users)
 for other languages.
+
+## How to run test
+
+Arrow GLib has unit tests. You can confirm that you install Apache
+GLib correctly by running unit tests.
+
+You need to install the followings to run unit tests:
+
+  * [Ruby](https://www.ruby-lang.org/)
+  * [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection)
+  * [test-unit gem](https://rubygems.org/gems/test-unit)
+
+You can install them by the followings:
+
+On Debian GNU/Linux or Ubuntu:
+
+```text
+% sudo apt install -y -V ruby-dev
+% sudo gem install gobject-introspection test-unit
+```
+
+On CentOS 7 or later:
+
+```text
+% sudo yum install -y git
+% git clone https://github.com/sstephenson/rbenv.git ~/.rbenv
+% git clone https://github.com/sstephenson/ruby-build.git ~/.rbenv/plugins/ruby-build
+% echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile
+% echo 'eval "$(rbenv init -)"' >> ~/.bash_profile
+% exec ${SHELL} --login
+% sudo yum install -y gcc make patch openssl-devel readline-devel zlib-devel
+% rbenv install 2.4.1
+% rbenv global 2.4.1
+% gem install gobject-introspection test-unit
+```
+
+On macOS with [Homebrew](https://brew.sh/):
+
+```text
+% gem install gobject-introspection test-unit
+```
+
+Now, you can run unit tests by the followings:
+
+```text
+% cd c_glib
+% test/run-test.sh
+```
diff --git a/c_glib/test/run-test.sh b/c_glib/test/run-test.sh
index 9b0ec8e45f52f..efa2829d74a29 100755
--- a/c_glib/test/run-test.sh
+++ b/c_glib/test/run-test.sh
@@ -22,7 +22,7 @@ lib_dir="${base_dir}/arrow-glib/.libs"
 
 LD_LIBRARY_PATH="${lib_dir}:${LD_LIBRARY_PATH}"
 
-if [ "${NO_MAKE}" != "yes" ]; then
+if [ -f "Makefile" -a "${NO_MAKE}" != "yes" ]; then
   make -j8 > /dev/null || exit $?
 fi
 

From ba2880c77c5e0ebb4baf83322899223f7c5e1068 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 5 May 2017 14:51:59 -0400
Subject: [PATCH 0603/1644] ARROW-946: [GLib] Use "new" instead of "open" for
 constructor name

Because "new" is the standard constructor name.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #638 from kou/glib-use-new and squashes the following commits:

6b16b5d [Kouhei Sutou] [GLib] Use "new" instead of "open" for constructor name
---
 c_glib/arrow-glib/file-reader.cpp      | 10 +++++-----
 c_glib/arrow-glib/file-reader.h        |  4 ++--
 c_glib/arrow-glib/file-writer.cpp      | 12 ++++++------
 c_glib/arrow-glib/file-writer.h        |  6 +++---
 c_glib/arrow-glib/output-stream.cpp    | 12 ++++++------
 c_glib/arrow-glib/output-stream.h      |  6 +++---
 c_glib/arrow-glib/stream-reader.cpp    | 10 +++++-----
 c_glib/arrow-glib/stream-reader.h      |  4 ++--
 c_glib/arrow-glib/stream-writer.cpp    | 12 ++++++------
 c_glib/arrow-glib/stream-writer.h      |  6 +++---
 c_glib/example/lua/read-batch.lua      |  2 +-
 c_glib/example/lua/read-stream.lua     |  2 +-
 c_glib/example/lua/write-batch.lua     |  4 ++--
 c_glib/example/lua/write-stream.lua    |  4 ++--
 c_glib/example/read-batch.c            |  4 ++--
 c_glib/example/read-stream.c           |  4 ++--
 c_glib/test/test-file-output-stream.rb |  6 +++---
 c_glib/test/test-file-writer.rb        |  6 +++---
 c_glib/test/test-stream-writer.rb      |  6 +++---
 19 files changed, 60 insertions(+), 60 deletions(-)

diff --git a/c_glib/arrow-glib/file-reader.cpp b/c_glib/arrow-glib/file-reader.cpp
index bbba5a1ede7b2..c16bf194821cd 100644
--- a/c_glib/arrow-glib/file-reader.cpp
+++ b/c_glib/arrow-glib/file-reader.cpp
@@ -131,16 +131,16 @@ garrow_file_reader_class_init(GArrowFileReaderClass *klass)
 }
 
 /**
- * garrow_file_reader_open:
+ * garrow_file_reader_new:
  * @input_stream: The seekable input stream to read data.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
- * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowFileReader or %NULL on error.
+ * Returns: (nullable): A newly created #GArrowFileReader or %NULL on
+ *   error.
  */
 GArrowFileReader *
-garrow_file_reader_open(GArrowSeekableInputStream *input_stream,
-                        GError **error)
+garrow_file_reader_new(GArrowSeekableInputStream *input_stream,
+                       GError **error)
 {
   auto arrow_random_access_file =
     garrow_seekable_input_stream_get_raw(input_stream);
diff --git a/c_glib/arrow-glib/file-reader.h b/c_glib/arrow-glib/file-reader.h
index b737269a2945b..551e05a3d1413 100644
--- a/c_glib/arrow-glib/file-reader.h
+++ b/c_glib/arrow-glib/file-reader.h
@@ -70,8 +70,8 @@ struct _GArrowFileReaderClass
 
 GType garrow_file_reader_get_type(void) G_GNUC_CONST;
 
-GArrowFileReader *garrow_file_reader_open(GArrowSeekableInputStream *input_stream,
-                                          GError **error);
+GArrowFileReader *garrow_file_reader_new(GArrowSeekableInputStream *input_stream,
+                                         GError **error);
 
 GArrowSchema *garrow_file_reader_get_schema(GArrowFileReader *file_reader);
 guint garrow_file_reader_get_n_record_batches(GArrowFileReader *file_reader);
diff --git a/c_glib/arrow-glib/file-writer.cpp b/c_glib/arrow-glib/file-writer.cpp
index e615cf554fd64..e3c721c49b16f 100644
--- a/c_glib/arrow-glib/file-writer.cpp
+++ b/c_glib/arrow-glib/file-writer.cpp
@@ -57,18 +57,18 @@ garrow_file_writer_class_init(GArrowFileWriterClass *klass)
 }
 
 /**
- * garrow_file_writer_open:
+ * garrow_file_writer_new:
  * @sink: The output of the writer.
  * @schema: The schema of the writer.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
- * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowFileWriter or %NULL on error.
+ * Returns: (nullable): A newly created #GArrowFileWriter or %NULL on
+ *   error.
  */
 GArrowFileWriter *
-garrow_file_writer_open(GArrowOutputStream *sink,
-                            GArrowSchema *schema,
-                            GError **error)
+garrow_file_writer_new(GArrowOutputStream *sink,
+                       GArrowSchema *schema,
+                       GError **error)
 {
   std::shared_ptr<arrow::ipc::FileWriter> arrow_file_writer;
   auto status =
diff --git a/c_glib/arrow-glib/file-writer.h b/c_glib/arrow-glib/file-writer.h
index 7f9a4f0399454..346dc6f242ae3 100644
--- a/c_glib/arrow-glib/file-writer.h
+++ b/c_glib/arrow-glib/file-writer.h
@@ -65,9 +65,9 @@ struct _GArrowFileWriterClass
 
 GType garrow_file_writer_get_type(void) G_GNUC_CONST;
 
-GArrowFileWriter *garrow_file_writer_open(GArrowOutputStream *sink,
-                                                 GArrowSchema *schema,
-                                                 GError **error);
+GArrowFileWriter *garrow_file_writer_new(GArrowOutputStream *sink,
+                                         GArrowSchema *schema,
+                                         GError **error);
 
 gboolean garrow_file_writer_write_record_batch(GArrowFileWriter *file_writer,
                                                    GArrowRecordBatch *record_batch,
diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index b757d44cef44e..48c48b8fdc327 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -184,18 +184,18 @@ garrow_file_output_stream_class_init(GArrowFileOutputStreamClass *klass)
 }
 
 /**
- * garrow_file_output_stream_open:
+ * garrow_file_output_stream_new:
  * @path: The path of the file output stream.
  * @append: Whether the path is opened as append mode or recreate mode.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowFileOutputStream or %NULL on error.
+ * Returns: (nullable): A newly opened #GArrowFileOutputStream or
+ *   %NULL on error.
  */
 GArrowFileOutputStream *
-garrow_file_output_stream_open(const gchar *path,
-                               gboolean append,
-                               GError **error)
+garrow_file_output_stream_new(const gchar *path,
+                              gboolean append,
+                              GError **error)
 {
   std::shared_ptr<arrow::io::FileOutputStream> arrow_file_output_stream;
   auto status =
diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h
index 2a14a24ea9051..48b891c19733d 100644
--- a/c_glib/arrow-glib/output-stream.h
+++ b/c_glib/arrow-glib/output-stream.h
@@ -118,9 +118,9 @@ struct _GArrowFileOutputStreamClass
 
 GType garrow_file_output_stream_get_type(void) G_GNUC_CONST;
 
-GArrowFileOutputStream *garrow_file_output_stream_open(const gchar *path,
-                                                       gboolean append,
-                                                       GError **error);
+GArrowFileOutputStream *garrow_file_output_stream_new(const gchar *path,
+                                                      gboolean append,
+                                                      GError **error);
 
 
 #define GARROW_TYPE_BUFFER_OUTPUT_STREAM        \
diff --git a/c_glib/arrow-glib/stream-reader.cpp b/c_glib/arrow-glib/stream-reader.cpp
index 017d5e91a8a4d..cc18cd84d3142 100644
--- a/c_glib/arrow-glib/stream-reader.cpp
+++ b/c_glib/arrow-glib/stream-reader.cpp
@@ -132,16 +132,16 @@ garrow_stream_reader_class_init(GArrowStreamReaderClass *klass)
 }
 
 /**
- * garrow_stream_reader_open:
+ * garrow_stream_reader_new:
  * @stream: The stream to be read.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
- * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowStreamReader or %NULL on error.
+ * Returns: (nullable): A newly created #GArrowStreamReader or %NULL
+ *   on error.
  */
 GArrowStreamReader *
-garrow_stream_reader_open(GArrowInputStream *stream,
-                              GError **error)
+garrow_stream_reader_new(GArrowInputStream *stream,
+                         GError **error)
 {
   std::shared_ptr<arrow::ipc::StreamReader> arrow_stream_reader;
   auto status =
diff --git a/c_glib/arrow-glib/stream-reader.h b/c_glib/arrow-glib/stream-reader.h
index 16a7f57bf801b..2ea2c26a9e541 100644
--- a/c_glib/arrow-glib/stream-reader.h
+++ b/c_glib/arrow-glib/stream-reader.h
@@ -70,8 +70,8 @@ struct _GArrowStreamReaderClass
 
 GType garrow_stream_reader_get_type(void) G_GNUC_CONST;
 
-GArrowStreamReader *garrow_stream_reader_open(GArrowInputStream *stream,
-                                                     GError **error);
+GArrowStreamReader *garrow_stream_reader_new(GArrowInputStream *stream,
+                                             GError **error);
 
 GArrowSchema *garrow_stream_reader_get_schema(GArrowStreamReader *stream_reader);
 GArrowRecordBatch *garrow_stream_reader_get_next_record_batch(GArrowStreamReader *stream_reader,
diff --git a/c_glib/arrow-glib/stream-writer.cpp b/c_glib/arrow-glib/stream-writer.cpp
index cc24f263bfca9..45e2fb099535b 100644
--- a/c_glib/arrow-glib/stream-writer.cpp
+++ b/c_glib/arrow-glib/stream-writer.cpp
@@ -132,18 +132,18 @@ garrow_stream_writer_class_init(GArrowStreamWriterClass *klass)
 }
 
 /**
- * garrow_stream_writer_open:
+ * garrow_stream_writer_new:
  * @sink: The output of the writer.
  * @schema: The schema of the writer.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
- * Returns: (nullable) (transfer full): A newly opened
- *   #GArrowStreamWriter or %NULL on error.
+ * Returns: (nullable): A newly created #GArrowStreamWriter or %NULL on
+ *   error.
  */
 GArrowStreamWriter *
-garrow_stream_writer_open(GArrowOutputStream *sink,
-                              GArrowSchema *schema,
-                              GError **error)
+garrow_stream_writer_new(GArrowOutputStream *sink,
+                         GArrowSchema *schema,
+                         GError **error)
 {
   std::shared_ptr<arrow::ipc::StreamWriter> arrow_stream_writer;
   auto status =
diff --git a/c_glib/arrow-glib/stream-writer.h b/c_glib/arrow-glib/stream-writer.h
index 6e773f1fc316e..d718b188d8fff 100644
--- a/c_glib/arrow-glib/stream-writer.h
+++ b/c_glib/arrow-glib/stream-writer.h
@@ -69,9 +69,9 @@ struct _GArrowStreamWriterClass
 
 GType garrow_stream_writer_get_type(void) G_GNUC_CONST;
 
-GArrowStreamWriter *garrow_stream_writer_open(GArrowOutputStream *sink,
-                                                     GArrowSchema *schema,
-                                                     GError **error);
+GArrowStreamWriter *garrow_stream_writer_new(GArrowOutputStream *sink,
+                                             GArrowSchema *schema,
+                                             GError **error);
 
 gboolean garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
                                                      GArrowRecordBatch *record_batch,
diff --git a/c_glib/example/lua/read-batch.lua b/c_glib/example/lua/read-batch.lua
index 8b129c9e4e7a3..090a857ee555d 100644
--- a/c_glib/example/lua/read-batch.lua
+++ b/c_glib/example/lua/read-batch.lua
@@ -21,7 +21,7 @@ local Arrow = lgi.Arrow
 local input_path = arg[1] or "/tmp/batch.arrow";
 
 local input = Arrow.MemoryMappedInputStream.new(input_path)
-local reader = Arrow.FileReader.open(input)
+local reader = Arrow.FileReader.new(input)
 
 for i = 0, reader:get_n_record_batches() - 1 do
    local record_batch = reader:get_record_batch(i)
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index 987d463b981cf..d7ac5ebbd2d97 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -21,7 +21,7 @@ local Arrow = lgi.Arrow
 local input_path = arg[1] or "/tmp/stream.arrow";
 
 local input = Arrow.MemoryMappedInputStream.new(input_path)
-local reader = Arrow.StreamReader.open(input)
+local reader = Arrow.StreamReader.new(input)
 
 local i = 0
 while true do
diff --git a/c_glib/example/lua/write-batch.lua b/c_glib/example/lua/write-batch.lua
index 3a22cd57fd81e..663f8ef995551 100644
--- a/c_glib/example/lua/write-batch.lua
+++ b/c_glib/example/lua/write-batch.lua
@@ -34,8 +34,8 @@ local fields = {
 }
 local schema = Arrow.Schema.new(fields)
 
-local output = Arrow.FileOutputStream.open(output_path, false)
-local writer = Arrow.FileWriter.open(output, schema)
+local output = Arrow.FileOutputStream.new(output_path, false)
+local writer = Arrow.FileWriter.new(output, schema)
 
 function build_array(builder, values)
    for _, value in pairs(values) do
diff --git a/c_glib/example/lua/write-stream.lua b/c_glib/example/lua/write-stream.lua
index 37c6bb54cd8f4..fb6cc557a98c2 100644
--- a/c_glib/example/lua/write-stream.lua
+++ b/c_glib/example/lua/write-stream.lua
@@ -34,8 +34,8 @@ local fields = {
 }
 local schema = Arrow.Schema.new(fields)
 
-local output = Arrow.FileOutputStream.open(output_path, false)
-local writer = Arrow.StreamWriter.open(output, schema)
+local output = Arrow.FileOutputStream.new(output_path, false)
+local writer = Arrow.StreamWriter.new(output, schema)
 
 function build_array(builder, values)
    for _, value in pairs(values) do
diff --git a/c_glib/example/read-batch.c b/c_glib/example/read-batch.c
index 25f19b24393b2..212b2a7a342f0 100644
--- a/c_glib/example/read-batch.c
+++ b/c_glib/example/read-batch.c
@@ -103,8 +103,8 @@ main(int argc, char **argv)
   {
     GArrowFileReader *reader;
 
-    reader = garrow_file_reader_open(GARROW_SEEKABLE_INPUT_STREAM(input),
-                                     &error);
+    reader = garrow_file_reader_new(GARROW_SEEKABLE_INPUT_STREAM(input),
+                                    &error);
     if (!reader) {
       g_print("failed to open file reader: %s\n", error->message);
       g_error_free(error);
diff --git a/c_glib/example/read-stream.c b/c_glib/example/read-stream.c
index ca5b9d97cc9df..28a3f5e2b9e1c 100644
--- a/c_glib/example/read-stream.c
+++ b/c_glib/example/read-stream.c
@@ -102,8 +102,8 @@ main(int argc, char **argv)
   {
     GArrowStreamReader *reader;
 
-    reader = garrow_stream_reader_open(GARROW_INPUT_STREAM(input),
-                                       &error);
+    reader = garrow_stream_reader_new(GARROW_INPUT_STREAM(input),
+                                      &error);
     if (!reader) {
       g_print("failed to open stream reader: %s\n", error->message);
       g_error_free(error);
diff --git a/c_glib/test/test-file-output-stream.rb b/c_glib/test/test-file-output-stream.rb
index 26737c0c87b38..237781ac00e2b 100644
--- a/c_glib/test/test-file-output-stream.rb
+++ b/c_glib/test/test-file-output-stream.rb
@@ -16,12 +16,12 @@
 # under the License.
 
 class TestFileOutputStream < Test::Unit::TestCase
-  sub_test_case(".open") do
+  sub_test_case(".new") do
     def test_create
       tempfile = Tempfile.open("arrow-io-file-output-stream")
       tempfile.write("Hello")
       tempfile.close
-      file = Arrow::FileOutputStream.open(tempfile.path, false)
+      file = Arrow::FileOutputStream.new(tempfile.path, false)
       file.close
       assert_equal("", File.read(tempfile.path))
     end
@@ -30,7 +30,7 @@ def test_append
       tempfile = Tempfile.open("arrow-io-file-output-stream")
       tempfile.write("Hello")
       tempfile.close
-      file = Arrow::FileOutputStream.open(tempfile.path, true)
+      file = Arrow::FileOutputStream.new(tempfile.path, true)
       file.close
       assert_equal("Hello", File.read(tempfile.path))
     end
diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb
index 6d4100a8cd38a..1d9102b6b0085 100644
--- a/c_glib/test/test-file-writer.rb
+++ b/c_glib/test/test-file-writer.rb
@@ -18,11 +18,11 @@
 class TestFileWriter < Test::Unit::TestCase
   def test_write_record_batch
     tempfile = Tempfile.open("arrow-ipc-file-writer")
-    output = Arrow::FileOutputStream.open(tempfile.path, false)
+    output = Arrow::FileOutputStream.new(tempfile.path, false)
     begin
       field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
       schema = Arrow::Schema.new([field])
-      file_writer = Arrow::FileWriter.open(output, schema)
+      file_writer = Arrow::FileWriter.new(output, schema)
       begin
         record_batch = Arrow::RecordBatch.new(schema, 0, [])
         file_writer.write_record_batch(record_batch)
@@ -35,7 +35,7 @@ def test_write_record_batch
 
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
-      file_reader = Arrow::FileReader.open(input)
+      file_reader = Arrow::FileReader.new(input)
       assert_equal(["enabled"],
                    file_reader.schema.fields.collect(&:name))
     ensure
diff --git a/c_glib/test/test-stream-writer.rb b/c_glib/test/test-stream-writer.rb
index 4280c1b32e0f7..d27eaa54fc53c 100644
--- a/c_glib/test/test-stream-writer.rb
+++ b/c_glib/test/test-stream-writer.rb
@@ -20,11 +20,11 @@ class TestStreamWriter < Test::Unit::TestCase
 
   def test_write_record_batch
     tempfile = Tempfile.open("arrow-ipc-stream-writer")
-    output = Arrow::FileOutputStream.open(tempfile.path, false)
+    output = Arrow::FileOutputStream.new(tempfile.path, false)
     begin
       field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
       schema = Arrow::Schema.new([field])
-      stream_writer = Arrow::StreamWriter.open(output, schema)
+      stream_writer = Arrow::StreamWriter.new(output, schema)
       begin
         columns = [
           build_boolean_array([true]),
@@ -40,7 +40,7 @@ def test_write_record_batch
 
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
-      stream_reader = Arrow::StreamReader.open(input)
+      stream_reader = Arrow::StreamReader.new(input)
       assert_equal(["enabled"],
                    stream_reader.schema.fields.collect(&:name))
       assert_equal(true,

From cc06197bc2825e4602a72730611d523dbc3b80e8 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 5 May 2017 15:06:42 -0400
Subject: [PATCH 0604/1644] ARROW-948: [GLib] Update C++ header file list

Author: Kouhei Sutou <kou@clear-code.com>

Closes #641 from kou/glib-fix-cpp-header-list and squashes the following commits:

f6b63ab [Kouhei Sutou] [GLib] Update C++ header file list
---
 c_glib/arrow-glib/arrow-glib.hpp | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
index 339773f651de1..3184761d4e148 100644
--- a/c_glib/arrow-glib/arrow-glib.hpp
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -20,18 +20,15 @@
 #pragma once
 
 #include <arrow-glib/arrow-glib.h>
+
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/array-builder.hpp>
-#include <arrow-glib/boolean-array.hpp>
-#include <arrow-glib/boolean-data-type.hpp>
+#include <arrow-glib/buffer.hpp>
 #include <arrow-glib/chunked-array.hpp>
 #include <arrow-glib/column.hpp>
 #include <arrow-glib/data-type.hpp>
-#include <arrow-glib/double-array.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/field.hpp>
-#include <arrow-glib/float-array.hpp>
-#include <arrow-glib/numeric-tensor.hpp>
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.hpp>
 #include <arrow-glib/table.hpp>
@@ -41,11 +38,10 @@
 #include <arrow-glib/file.hpp>
 #include <arrow-glib/file-mode.hpp>
 #include <arrow-glib/input-stream.hpp>
-#include <arrow-glib/memory-mapped-file.hpp>
 #include <arrow-glib/output-stream.hpp>
-#include <arrow-glib/random-access-file.hpp>
 #include <arrow-glib/readable.hpp>
 #include <arrow-glib/writeable.hpp>
+#include <arrow-glib/writeable-file.hpp>
 
 #include <arrow-glib/file-reader.hpp>
 #include <arrow-glib/file-writer.hpp>

From f63ff08643a79a7f9902fb01157e88902c85c9fc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 5 May 2017 19:28:38 -0400
Subject: [PATCH 0605/1644] ARROW-52: Set up project blog, draft 0.3 release
 posting

This will need additional updates after the 0.3 release to fix dates and URLs, but wanted to get this up for review and comment.

Some things I did here:

* Fixed top navbar on mobile devices
* Set up blogroll and simple blog templates
* Added "Project Links" menu
* Moved committers to a separate page

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #639 from wesm/ARROW-52 and squashes the following commits:

a104fbd [Wes McKinney] Code review comments and fix typos
5262583 [Wes McKinney] Typo
2236a7c [Wes McKinney] tweak
3f57b51 [Wes McKinney] Typo
4b22731 [Wes McKinney] Finish 0.3 release blog draft
5e34079 [Wes McKinney] Drafting 0.3 release blog post
058c625 [Wes McKinney] Rename post
d22490d [Wes McKinney] Fix mobile navbar. Move committers to separate page. Add project links nav, install page. Blog page placeholder
---
 site/_config.yml                      |   2 +
 site/_includes/blog_contents.html     |  12 ++
 site/_includes/blog_entry.html        |  39 ++++
 site/_includes/footer.html            |   1 +
 site/_includes/header.html            |  31 ++-
 site/_layouts/blog.html               |  15 ++
 site/_layouts/post.html               |  34 ++++
 site/_posts/2017-05-08-0.3-release.md | 263 ++++++++++++++++++++++++++
 site/blog.html                        |  28 +++
 site/committers.html                  | 100 ++++++++++
 site/css/main.scss                    |   5 +-
 site/index.html                       | 120 +-----------
 site/install.html                     |  11 ++
 13 files changed, 537 insertions(+), 124 deletions(-)
 create mode 100644 site/_includes/blog_contents.html
 create mode 100644 site/_includes/blog_entry.html
 create mode 100644 site/_layouts/blog.html
 create mode 100644 site/_layouts/post.html
 create mode 100644 site/_posts/2017-05-08-0.3-release.md
 create mode 100644 site/blog.html
 create mode 100644 site/committers.html
 create mode 100644 site/install.html

diff --git a/site/_config.yml b/site/_config.yml
index 346565e6d5cca..d7e0bb37e3eb0 100644
--- a/site/_config.yml
+++ b/site/_config.yml
@@ -14,8 +14,10 @@
 # limitations under the License.
 #
 markdown: kramdown
+permalink: /blog/:year/:month/:day/:title/
 repository: https://github.com/apache/arrow
 destination: build
+excerpt_separator: ""
 
 exclude:
   - Gemfile
diff --git a/site/_includes/blog_contents.html b/site/_includes/blog_contents.html
new file mode 100644
index 0000000000000..b3b785d4318cd
--- /dev/null
+++ b/site/_includes/blog_contents.html
@@ -0,0 +1,12 @@
+<div>
+  <aside>
+    <h4>Posts</h4>
+    <ul>
+        {% for post in site.posts %}
+        <li class="{% if page.title == post.title %}current{% endif %}">
+          <a href="{{ site.baseurl }}{{ post.url }}">{{ post.title }}</a>
+        </li>
+        {% endfor %}
+    </ul>
+  </aside>
+</div>
diff --git a/site/_includes/blog_entry.html b/site/_includes/blog_entry.html
new file mode 100644
index 0000000000000..cdc0060669c2f
--- /dev/null
+++ b/site/_includes/blog_entry.html
@@ -0,0 +1,39 @@
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+  <div class="container">
+    <h2>
+      {{ post.title }}
+      <a href="{{ post.url }}" class="permalink" title="Permalink">∞</a>
+    </h2>
+
+    <div class="panel">
+      <div class="panel-body">
+        <div>
+          <span class="label label-default">Published</span>
+          <span class="published">
+            <i class="fa fa-calendar"></i>
+            {{ post.date | date_to_string }}
+          </span>
+        </div>
+        <div>
+          <span class="label label-default">By</span>
+          <a href="{{ homepage }}"><i class="fa fa-user"></i> {{ post.author }}</a>
+        </div>
+      </div>
+    </div>
+    {{ post.content }}
+  </div>
diff --git a/site/_includes/footer.html b/site/_includes/footer.html
index c2a7d5e92bb20..c3ce968b8fb8e 100644
--- a/site/_includes/footer.html
+++ b/site/_includes/footer.html
@@ -1,3 +1,4 @@
+<hr/>
 <footer class="footer">
   <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
   <p>&copy; 2017 Apache Software Foundation</p>
diff --git a/site/_includes/header.html b/site/_includes/header.html
index 3d61494f2f109..d1526f69faf16 100644
--- a/site/_includes/header.html
+++ b/site/_includes/header.html
@@ -1,12 +1,33 @@
 <nav class="navbar navbar-default">
   <div class="container-fluid">
     <div class="navbar-header">
+      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#arrow-navbar">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
       <a class="navbar-brand" href="{{ site.baseurl }}/">Apache Arrow&#8482;&nbsp;&nbsp;&nbsp;</a>
     </div>
 
     <!-- Collect the nav links, forms, and other content for toggling -->
-    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+    <div class="collapse navbar-collapse" id="arrow-navbar">
       <ul class="nav navbar-nav">
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Project Links<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="{{ site.baseurl }}/install/">Install</a></li>
+            <li><a href="{{ site.baseurl }}/blog/">Blog</a></li>
+            <li><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a></li>
+            <li><a href="https://github.com/apache/arrow">Source Code</a></li>
+            <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Mailing List</a></li>
+            <li><a href="https://apachearrowslackin.herokuapp.com">Slack Channel</a></li>
+            <li><a href="{{ site.baseurl }}/committers/">Committers</a></li>
+          </ul>
+        </li>
         <li class="dropdown">
           <a href="#" class="dropdown-toggle" data-toggle="dropdown"
              role="button" aria-haspopup="true"
@@ -22,13 +43,13 @@
         <li class="dropdown">
           <a href="#" class="dropdown-toggle" data-toggle="dropdown"
              role="button" aria-haspopup="true"
-             aria-expanded="false">Code Documentation<span class="caret"></span>
+             aria-expanded="false">Documentation<span class="caret"></span>
           </a>
           <ul class="dropdown-menu">
-            <li><a href="{{ site.baseurl }}/docs/cpp">C++</a></li>
-            <li><a href="{{ site.baseurl }}/docs/java">Java</a></li>
             <li><a href="{{ site.baseurl }}/docs/python">Python</a></li>
-            <li><a href="{{ site.baseurl }}/docs/c_glib">C (GLib)</a></li>
+            <li><a href="{{ site.baseurl }}/docs/cpp">C++ API</a></li>
+            <li><a href="{{ site.baseurl }}/docs/java">Java API</a></li>
+            <li><a href="{{ site.baseurl }}/docs/c_glib">C GLib API</a></li>
           </ul>
         </li>
         <!-- <li><a href="{{ site.baseurl }}/blog">Blog</a></li> -->
diff --git a/site/_layouts/blog.html b/site/_layouts/blog.html
new file mode 100644
index 0000000000000..31fd7195ef8a1
--- /dev/null
+++ b/site/_layouts/blog.html
@@ -0,0 +1,15 @@
+---
+layout: default
+---
+
+<section class="blog">
+  <div class="grid">
+
+    <div class="">
+      {{ content }}
+    </div>
+
+    <!-- {% include blog_contents.html %} -->
+    <div class="clear"></div>
+  </div>
+</section>
diff --git a/site/_layouts/post.html b/site/_layouts/post.html
new file mode 100644
index 0000000000000..53de1fcfcedd5
--- /dev/null
+++ b/site/_layouts/post.html
@@ -0,0 +1,34 @@
+{% include top.html %}
+{% capture homepage %}http://people.apache.org/~{{ page.author }}{% endcapture %}
+
+<body class="wrap">
+  <div class="container">
+    {% include header.html %}
+
+    <h2>
+      {{ page.title }}
+      <a href="{{ page.url }}" class="permalink" title="Permalink">∞</a>
+    </h2>
+
+    <div class="panel">
+      <div class="panel-body">
+        <div>
+          <span class="label label-default">Published</span>
+          <span class="published">
+            <i class="fa fa-calendar"></i>
+            {{ page.date | date_to_string }}
+          </span>
+        </div>
+        <div>
+          <span class="label label-default">By</span>
+          <a href="{{ homepage }}"><i class="fa fa-user"></i> {{ page.author }}</a>
+        </div>
+      </div>
+    </div>
+
+    {{ content }}
+
+    {% include footer.html %}
+  </div>
+</body>
+</html>
diff --git a/site/_posts/2017-05-08-0.3-release.md b/site/_posts/2017-05-08-0.3-release.md
new file mode 100644
index 0000000000000..2920565be7bb9
--- /dev/null
+++ b/site/_posts/2017-05-08-0.3-release.md
@@ -0,0 +1,263 @@
+---
+layout: post
+title: "Apache Arrow 0.3 Release"
+date: "2017-05-01     07:00:00 -0500"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.3.0 release of the
+project. It is the product of an intense 10 weeks of development since the
+0.2.0 release from this past February. It includes [**306 resolved JIRAs**][1]
+from [**23 contributors**][14].
+
+While we have added many new features to the different Arrow implementations,
+one of the major development focuses in 2017 has been hardening the in-memory
+format, type metadata, and messaging protocol to provide a **stable,
+production-ready foundation** for big data applications. We are excited to be
+collaborating with the [Apache Spark][3] and [GeoMesa][2] communities on
+utilizing Arrow for high performance IO and in-memory data processing.
+
+See the [Install Page][6] to learn how to get the libraries for your platform.
+
+We will be publishing more information about the Apache Arrow roadmap as we
+forge ahead with using Arrow to accelerate big data systems.
+
+We are looking for more contributors from within our existing communities and
+from other communities (such as Go, R, or Julia) to get involved in Arrow
+development.
+
+### File and Streaming Format Hardening
+
+The 0.2.0 release brought with it the first iterations of the **random access**
+and **streaming** Arrow wire formats. See the [IPC specification][15] for
+implementation details and [example blog post][4] with some use cases. These
+provide low-overhead, zero-copy access to Arrow record batch payloads.
+
+In 0.3.0 we have solidified a number of small details with the binary format
+and improved our integration and unit testing particularly in the Java, C++,
+and Python libraries. Using the [Google Flatbuffers][5] project has helped with
+adding new features to our metadata without breaking forward compatibility.
+
+We are not yet ready to make a firm commitment to strong forward compatibility
+(in case we find something needs to change) in the binary format, but we will
+make efforts between major releases to not make unnecessary
+breakages. Contributions to the website and component user and API
+documentation would also be most welcome.
+
+### Dictionary Encoding Support
+
+[Emilio Lahr-Vivaz][13] from the [GeoMesa][2] project contributed Java support
+for dictionary-encoded Arrow vectors. We followed up with C++ and Python
+support (and `pandas.Categorical` integration). We have not yet implemented
+full integration tests for dictionaries (for sending this data between C++ and
+Java), but hope to achieve this in the 0.4.0 Arrow release.
+
+This common data representation technique for categorical data allows multiple
+record batches to share a common "dictionary", with the values in the batches
+being represented as integers referencing the dictionary. This data is called
+"categorical" or "factor" in statistical languages, while in file formats like
+Apache Parquet it is strictly used for data compression.
+
+### Expanded Date, Time, and Fixed Size Types
+
+A notable omission from the 0.2.0 release was complete and integration-tested
+support for the gamut of date and time types that occur in the wild. These are
+needed for [Apache Parquet][21] and Apache Spark integration.
+
+* **Date**: 32-bit (days unit) and 64-bit (milliseconds unit)
+* **Time**: 64-bit integer with unit (second, millisecond, microsecond, nanosecond)
+* **Timestamp**: 64-bit integer with unit, with or without timezone
+* **Fixed Size Binary**: Primitive values occupying certain number of bytes
+* **Fixed Size List**: List values with constant size (no separate offsets vector)
+
+We have additionally added experimental support for exact decimals in C++ using
+[Boost.Multiprecision][12], though we have not yet hardened the Decimal memory
+format between the Java and C++ implementations.
+
+### C++ and Python Support on Windows
+
+We have made many general improvements to development and packaging for general
+C++ and Python development. 0.3.0 is the first release to bring full C++ and
+Python support for Windows on Visual Studio (MSVC) 2015 and 2017. In addition
+to adding Appveyor continuous integration for MSVC, we have also written guides
+for building from source on Windows: [C++][7] and [Python][8].
+
+For the first time, you can install the Arrow Python library on Windows from
+[conda-forge][31]:
+
+```shell
+conda install pyarrow -c conda-forge
+```
+
+### C (GLib) Bindings, with support for Ruby, Lua, and more
+
+[Kouhei Sutou][9] is a new Apache Arrow contributor and has contributed GLib C
+bindings (to the C++ libraries) for Linux. Using a C middleware framework
+called [GObject Introspection][10], it is possible to use these bindings
+seamlessly in Ruby, Lua, Go, and [other programming languages][11]. We will
+probably need to publish some follow up blogs explaining how these bindings
+work and how to use them.
+
+### Apache Spark Integration for PySpark
+
+We have been collaborating with the Apache Spark community on [SPARK-13534][16]
+to add support for using Arrow to accelerate `DataFrame.toPandas` in
+PySpark. We have observed over [**40x speedup**][17] from the more efficient
+data serialization.
+
+Using Arrow in PySpark opens the door to many other performance optimizations,
+particularly around UDF evaluation (e.g. `map` and `filter` operations with
+Python lambda functions).
+
+### New Python Feature: Memory Views, Feather, Apache Parquet support
+
+Arrow's Python library `pyarrow` is a Cython binding for the `libarrow` and
+`libarrow_python` C++ libraries, which handle inteoperability with NumPy,
+[pandas][29], and the Python standard library.
+
+At the heart of Arrow's C++ libraries is the `arrow::Buffer` object, which is a
+managed memory view supporting zero-copy reads and slices. [Jeff Knupp][25]
+contributed integration between Arrow buffers and the Python buffer protocol
+and memoryviews, so now code like this is possible:
+
+```python
+In [6]: import pyarrow as pa
+
+In [7]: buf = pa.frombuffer(b'foobarbaz')
+
+In [8]: buf
+Out[8]: <pyarrow._io.Buffer at 0x7f6c0a84b538>
+
+In [9]: memoryview(buf)
+Out[9]: <memory at 0x7f6c0a8c5e88>
+
+In [10]: buf.to_pybytes()
+Out[10]: b'foobarbaz'
+```
+
+We have significantly expanded [**Apache Parquet**][21] support via the C++
+Parquet implementation [parquet-cpp][23]. This includes support for partitioned
+datasets on disk or in HDFS. We added initial Arrow-powered Parquet support [in
+the Dask project][24], and look forward to more collaborations with the Dask
+developers on distributed processing of pandas data.
+
+With Arrow's support for pandas maturing, we were able to merge in the
+[**Feather format**][22] implementation, which is essentially a special case of
+the Arrow random access format. We'll be continuing Feather development within
+the Arrow codebase. For example, Feather can now read and write with Python
+file objects using Arrow's Python binding layer.
+
+We also implemented more robust support for pandas-specific data types, like
+`DatetimeTZ` and `Categorical`.
+
+### Support for Tensors and beyond in C++ Library
+
+There has been increased interest in using Apache Arrow as a tool for zero-copy
+shared memory management for machine learning applications. A flagship example
+is the [Ray project][26] from the UC Berkeley [RISELab][27].
+
+Machine learning deals in additional kinds of data structures beyond what the
+Arrow columnar format supports, like multidimensional arrays aka "tensors". As
+such, we implemented the [`arrow::Tensor`][28] C++ type which can utilize the
+rest of Arrow's zero-copy shared memory machinery (using `arrow::Buffer` for
+managing memory lifetime). In C++ in particular, we will want to provide for
+additional data structures utilizing common IO and memory management tools.
+
+### Start of JavaScript (TypeScript) Implementation
+
+[Brian Hulette][20] started developing an Arrow implementation in
+[TypeScript][30] for use in NodeJS and browser-side applications. We are
+benefitting from Flatbuffers' first class support for JavaScript.
+
+### Improved Website and Developer Documentation
+
+Since 0.2.0 we have implemented a new website stack for publishing
+documentation and blogs based on [Jekyll][18]. Kouhei Sutou developed a [Jekyll
+Jupyter Notebook plugin][19] so that we can use Jupyter to author content for
+the Arrow website.
+
+On the website, we have now published API documentation for the C, C++, Java,
+and Python subcomponents. Within these you will find easier-to-follow developer
+instructions for getting started.
+
+### Contributors
+
+Thanks to all who contributed patches to this release.
+
+```
+$ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
+    119 Wes McKinney
+     55 Kouhei Sutou
+     18 Uwe L. Korn
+     17 Julien Le Dem
+      9 Phillip Cloud
+      6 Bryan Cutler
+      5 Philipp Moritz
+      5 Emilio Lahr-Vivaz
+      4 Max Risuhin
+      4 Johan Mabille
+      4 Jeff Knupp
+      3 Steven Phillips
+      3 Miki Tebeka
+      2 Leif Walsh
+      2 Jeff Reback
+      2 Brian Hulette
+      1 Tsuyoshi Ozawa
+      1 rvernica
+      1 Nong Li
+      1 Julien Lafaye
+      1 Itai Incze
+      1 Holden Karau
+      1 Deepak Majeti
+```
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.3.0
+[2]: http://www.geomesa.org/
+[3]: http://spark.apache.org
+[4]: http://wesmckinney.com/blog/arrow-streaming-columnar/
+[5]: http://github.com/google/flatbuffers
+[6]: http://arrow.apache.org/install
+[7]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
+[8]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst
+[9]: http://github.com/kou
+[10]: https://wiki.gnome.org/Projects/GObjectIntrospection
+[11]: https://wiki.gnome.org/Projects/GObjectIntrospection/Users
+[12]: https://github.com/boostorg/multiprecision
+[13]: https://github.com/elahrvivaz
+[14]: https://github.com/apache/arrow/graphs/contributors
+[15]: http://arrow.apache.org/docs/ipc.html
+[16]: https://issues.apache.org/jira/browse/SPARK-13534
+[17]: https://github.com/apache/spark/pull/15821#issuecomment-282175163
+[18]: https://jekyllrb.com
+[19]: https://github.com/red-data-tools/jekyll-jupyter-notebook
+[20]: https://github.com/TheNeuralBit
+[21]: http://parquet.apache.org
+[22]: https://github.com/wesm/feather
+[23]: https://github.com/apache/parquet-cpp
+[24]: https://github.com/dask/dask/commit/68f9e417924a985c1f2e2a587126833c70a2e9f4
+[25]: https://github.com/JeffKnupp
+[26]: https://github.com/ray-project/ray
+[27]: https://rise.cs.berkeley.edu/
+[28]: http://arrow.apache.org/docs/cpp/classarrow_1_1_tensor.html
+[29]: http://pandas.pydata.org
+[30]: https://github.com/apache/arrow/tree/master/js
+[30]: https://conda-forge.github.io
diff --git a/site/blog.html b/site/blog.html
new file mode 100644
index 0000000000000..ddfb47fb364d0
--- /dev/null
+++ b/site/blog.html
@@ -0,0 +1,28 @@
+---
+layout: default
+title: Blog / News
+permalink: /blog/
+---
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+
+<h2>Project News and Blog</h2>
+<hr>
+
+{% for post in site.posts %}
+  {% include blog_entry.html %}
+{% endfor %}
diff --git a/site/committers.html b/site/committers.html
new file mode 100644
index 0000000000000..91afbabe0d4e6
--- /dev/null
+++ b/site/committers.html
@@ -0,0 +1,100 @@
+---
+layout: default
+---
+    <div class="container">
+<h2>Committers</h2>
+<table class="table"><thead>
+<tr>
+<th>Name</th>
+<th>Alias (email is &lt;alias&gt;@apache.org)</th>
+</tr>
+</thead><tbody>
+<tr>
+<td>Jacques Nadeau</td>
+<td>jacques</td>
+</tr>
+<tr>
+<td>Todd Lipcon</td>
+<td>todd</td>
+</tr>
+<tr>
+<td>Ted Dunning</td>
+<td>tdunning</td>
+</tr>
+<tr>
+<td>Michael Stack</td>
+<td>stack</td>
+</tr>
+<tr>
+<td>P. Taylor Goetz</td>
+<td>ptgoetz</td>
+</tr>
+<tr>
+<td>Julian Hyde</td>
+<td>jhyde</td>
+</tr>
+<tr>
+<td>Reynold Xin</td>
+<td>rxin</td>
+</tr>
+<tr>
+<td>James Taylor</td>
+<td>jamestaylor</td>
+</tr>
+<tr>
+<td>Julien Le Dem</td>
+<td>julien</td>
+</tr>
+<tr>
+<td>Jake Luciani</td>
+<td>jake</td>
+</tr>
+<tr>
+<td>Jason Altekruse</td>
+<td>json</td>
+</tr>
+<tr>
+<td>Alex Levenson</td>
+<td>alexlevenson</td>
+</tr>
+<tr>
+<td>Parth Chandra</td>
+<td>parthc</td>
+</tr>
+<tr>
+<td>Marcel Kornacker</td>
+<td>marcel</td>
+</tr>
+<tr>
+<td>Steven Phillips</td>
+<td>smp</td>
+</tr>
+<tr>
+<td>Hanifi Gunes</td>
+<td>hg</td>
+</tr>
+<tr>
+<td>Abdelhakim Deneche</td>
+<td>adeneche</td>
+</tr>
+<tr>
+<td>Wes McKinney</td>
+<td>wesm</td>
+</tr>
+<tr>
+<td>David Alves</td>
+<td>dralves</td>
+</tr>
+<tr>
+<td>Ippokratis Pandis</td>
+<td>ippokratis</td>
+</tr>
+<tr>
+<td>Uwe L. Korn</td>
+<td>uwe</td>
+</tr>
+</tbody></table>
+
+    </div> <!-- /container -->
+  </body>
+</html>
diff --git a/site/css/main.scss b/site/css/main.scss
index 24b46ae24ccf2..e8b2165bbcda7 100644
--- a/site/css/main.scss
+++ b/site/css/main.scss
@@ -5,6 +5,9 @@ $container-desktop: 960px;
 $container-large-desktop: $container-desktop;
 $grid-gutter-width: 15px;
 
+$font-family-base: "Droid Serif",Georgia,Helvetica,sans-serif;
+$font-size-base: 14px;
+
 @import "bootstrap-sprockets";
 @import "bootstrap";
-@import "font-awesome";
+@import "font-awesome";
\ No newline at end of file
diff --git a/site/index.html b/site/index.html
index aecaea525166e..d80925ce24bcd 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,8 +7,10 @@ <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
         <p>
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
+          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.3.0 Release - May 5, 2017)</a>
         </p>
       </div>
+      <h4>Latest News: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.3.0 release</a></h4>
       <div class="row">
         <div class="col-lg-4">
           <h2>Fast</h2>
@@ -24,31 +26,6 @@ <h2>Standard</h2>
         </div>
      </div> <!-- close "row" div -->
 
-      <div class="row marketing">
-        <div class="col-lg-4">
-          <h4>Developer Mailing List</h4>
-            <ul>
-            <li><a href="mailto:dev-subscribe@arrow.apache.org">Subscribe</a></li>
-            <li><a href="mailto:dev-unsubscribe@arrow.apache.org">Unsubscribe</a></li>
-            <li><a href="mailto:dev@arrow.apache.org">Post</a></li>
-            <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Archive</a></li>
-            </ul>
-          </div>
-        <div class="col-lg-4">
-          <h4>Developer Resources</h4>
-            <p>Arrow is still early in development. </p>
-            <p>Source Code (<a href="https://git-wip-us.apache.org/repos/asf?p=arrow.git">http</a>) (<a href="git://git.apache.org/arrow.git">git</a>)</p>
-            <p><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker (JIRA)</a></p>
-            <p><a href="https://apachearrowslackin.herokuapp.com">Chat Room (Slack)</a></p>
-        </div>
-        <div class="col-lg-4">
-          <h4>Latest release</h4>
-            <p>Apache Arrow 0.2.0 is an early release and the APIs are still evolving. The metadata and physical data representation should be fairly stable as we have spent time finalizing the details. </p>
-            <p><a href="https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0">source release</a></p>
-            <p><a href="https://github.com/apache/arrow/releases/tag/apache-arrow-0.2.0">tag apache-arrow-0.2.0</a></p>
-            <p><a href="http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.2.0%22">java artifacts on maven central</a></p>
-        </div>
-    </div>
 <h2>Performance Advantage of Columnar In-Memory</h2>
 <div align="center">
   <img src="img/simd.png" alt="SIMD" style="width:60%" />
@@ -73,99 +50,6 @@ <h2>Advantages of a Common Data Layer</h2>
 </ul>
 </div>
 </div>
-<h2>Committers</h2>
-<table class="table"><thead>
-<tr>
-<th>Name</th>
-<th>Alias (email is &lt;alias&gt;@apache.org)</th>
-</tr>
-</thead><tbody>
-<tr>
-<td>Jacques Nadeau</td>
-<td>jacques</td>
-</tr>
-<tr>
-<td>Todd Lipcon</td>
-<td>todd</td>
-</tr>
-<tr>
-<td>Ted Dunning</td>
-<td>tdunning</td>
-</tr>
-<tr>
-<td>Michael Stack</td>
-<td>stack</td>
-</tr>
-<tr>
-<td>P. Taylor Goetz</td>
-<td>ptgoetz</td>
-</tr>
-<tr>
-<td>Julian Hyde</td>
-<td>jhyde</td>
-</tr>
-<tr>
-<td>Reynold Xin</td>
-<td>rxin</td>
-</tr>
-<tr>
-<td>James Taylor</td>
-<td>jamestaylor</td>
-</tr>
-<tr>
-<td>Julien Le Dem</td>
-<td>julien</td>
-</tr>
-<tr>
-<td>Jake Luciani</td>
-<td>jake</td>
-</tr>
-<tr>
-<td>Jason Altekruse</td>
-<td>json</td>
-</tr>
-<tr>
-<td>Alex Levenson</td>
-<td>alexlevenson</td>
-</tr>
-<tr>
-<td>Parth Chandra</td>
-<td>parthc</td>
-</tr>
-<tr>
-<td>Marcel Kornacker</td>
-<td>marcel</td>
-</tr>
-<tr>
-<td>Steven Phillips</td>
-<td>smp</td>
-</tr>
-<tr>
-<td>Hanifi Gunes</td>
-<td>hg</td>
-</tr>
-<tr>
-<td>Abdelhakim Deneche</td>
-<td>adeneche</td>
-</tr>
-<tr>
-<td>Wes McKinney</td>
-<td>wesm</td>
-</tr>
-<tr>
-<td>David Alves</td>
-<td>dralves</td>
-</tr>
-<tr>
-<td>Ippokratis Pandis</td>
-<td>ippokratis</td>
-</tr>
-<tr>
-<td>Uwe L. Korn</td>
-<td>uwe</td>
-</tr>
-</tbody></table>
-
     </div> <!-- /container -->
   </body>
 </html>
diff --git a/site/install.html b/site/install.html
new file mode 100644
index 0000000000000..7734eeb303315
--- /dev/null
+++ b/site/install.html
@@ -0,0 +1,11 @@
+---
+layout: default
+---
+<h2>Current Version: 0.2.0</h2>
+<h3>Released: May 5, 2017</h2>
+g<p>Apache Arrow 0.2.0 is an early release and the APIs are still evolving. The
+metadata and physical data representation should be fairly stable as we have
+spent time finalizing the details. </p>
+<p><a href="https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0">source release</a></p>
+<p><a href="https://github.com/apache/arrow/releases/tag/apache-arrow-0.2.0">tag apache-arrow-0.2.0</a></p>
+<p><a href="http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.2.0%22">java artifacts on maven central</a></p>

From 1a6d135bed84919c166e5a08d894811d26eb3ea7 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 5 May 2017 19:42:03 -0400
Subject: [PATCH 0606/1644] ARROW-952: fix regex include from C++ standard
 library

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #643 from pcmoritz/fix-regex-include and squashes the following commits:

d72e2c2 [Philipp Moritz] fix regex include from C++ standard library
---
 cpp/src/arrow/util/decimal.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index c73bae1b4c995..f113c3359eaeb 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -21,7 +21,6 @@
 #include <cmath>
 #include <cstdlib>
 #include <iterator>
-#include <regex>
 #include <string>
 
 #include "arrow/status.h"

From 316c63dbaec8e5df33d0cf0fa78d38ac8cc375b8 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Fri, 5 May 2017 22:05:46 -0400
Subject: [PATCH 0607/1644] ARROW-824: Date and Time Vectors should reflect
 timezone-less semantics

The current java vector support the timezone less version of time related types but in an incomplete way.
This change fixes it and clarifies what the vectors implementation do.
We'll need separate vectors or adapt those to deal will timezone aware time types.

Author: Julien Le Dem <julien@apache.org>

Closes #568 from julienledem/ARROW-824 and squashes the following commits:

3528ad8 [Julien Le Dem] add license
e37385c [Julien Le Dem] centralize LocalDateTime.toMillis
bdac7ff [Julien Le Dem] make integration tests output more readable
b0da88c [Julien Le Dem] fix failing integration test
61518ec [Julien Le Dem] improve travis integration
ec19e7d [Julien Le Dem] ARROW-824: Date and Time Vectors should reflect timezone-less semantics
---
 ci/travis_script_integration.sh               |  3 +-
 integration/integration_test.py               | 27 ++++++++++-------
 .../main/codegen/data/ValueVectorTypes.tdd    | 14 ++++-----
 .../src/main/codegen/includes/vv_imports.ftl  |  1 +
 .../templates/AbstractFieldReader.java        |  6 ++--
 .../src/main/codegen/templates/ArrowType.java |  1 +
 .../main/codegen/templates/BaseReader.java    |  5 ++--
 .../codegen/templates/ComplexReaders.java     |  7 +++++
 .../codegen/templates/ComplexWriters.java     |  3 ++
 .../codegen/templates/FixedValueVectors.java  | 18 ++++-------
 .../codegen/templates/HolderReaderImpl.java   | 12 ++++----
 .../main/codegen/templates/NullReader.java    |  6 ++--
 .../main/codegen/templates/UnionReader.java   |  6 ++--
 .../main/codegen/templates/UnionVector.java   |  1 +
 .../main/codegen/templates/ValueHolders.java  |  3 ++
 .../org/apache/arrow/vector/types/Types.java  | 11 ++++---
 .../apache/arrow/vector/util/DateUtility.java | 10 +++++++
 .../java/org/joda/time/LocalDateTimes.java    | 30 +++++++++++++++++++
 .../complex/writer/TestComplexWriter.java     | 19 ++++++------
 .../arrow/vector/file/BaseFileTest.java       | 20 ++++++-------
 .../apache/arrow/vector/pojo/TestConvert.java |  2 +-
 21 files changed, 134 insertions(+), 71 deletions(-)
 create mode 100644 java/vector/src/main/java/org/joda/time/LocalDateTimes.java

diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 56f5ab7d2d35e..6e93ed79a2266 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 pushd $ARROW_JAVA_DIR
 
-mvn package
+echo "mvn package"
+mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)
 
 popd
 
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 661f5c97770d9..646646997f72c 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -544,7 +544,8 @@ def get_json(self):
 
 class JSONFile(object):
 
-    def __init__(self, schema, batches):
+    def __init__(self, name, schema, batches):
+        self.name = name
         self.schema = schema
         self.batches = batches
 
@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
         raise TypeError(dtype)
 
 
-def _generate_file(fields, batch_sizes):
+def _generate_file(name, fields, batch_sizes):
     schema = JSONSchema(fields)
     batches = []
     for size in batch_sizes:
@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):
 
         batches.append(JSONRecordBatch(size, columns))
 
-    return JSONFile(schema, batches)
+    return JSONFile(name, schema, batches)
 
 
 def generate_primitive_case(batch_sizes):
@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
         fields.append(get_field(type_ + "_nullable", type_, True))
         fields.append(get_field(type_ + "_nonnullable", type_, False))
 
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("primitive", fields, batch_sizes)
 
 
 def generate_datetime_case():
@@ -619,11 +620,11 @@ def generate_datetime_case():
         TimestampType('f7', 'ms'),
         TimestampType('f8', 'us'),
         TimestampType('f9', 'ns'),
-        TimestampType('f10', 'ms', tz='America/New_York')
+        TimestampType('f10', 'ms', tz=None)
     ]
 
     batch_sizes = [7, 10]
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("datetime", fields, batch_sizes)
 
 
 def generate_nested_case():
@@ -637,7 +638,7 @@ def generate_nested_case():
     ]
 
     batch_sizes = [7, 10]
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("nested", fields, batch_sizes)
 
 
 def get_generated_json_files():
@@ -655,7 +656,7 @@ def _temp_path():
 
     generated_paths = []
     for file_obj in file_objs:
-        out_path = os.path.join(temp_dir, guid() + '.json')
+        out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
         file_obj.write(out_path)
         generated_paths.append(out_path)
 
@@ -684,11 +685,15 @@ def _compare_implementations(self, producer, consumer):
                                                        consumer.name))
 
         for json_path in self.json_files:
+            print('=====================================================================================')
             print('Testing file {0}'.format(json_path))
+            print('=====================================================================================')
+
+            name = os.path.splitext(os.path.basename(json_path))[0]
 
             # Make the random access file
             print('-- Creating binary inputs')
-            producer_file_path = os.path.join(self.temp_dir, guid())
+            producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
             producer.json_to_file(json_path, producer_file_path)
 
             # Validate the file
@@ -696,8 +701,8 @@ def _compare_implementations(self, producer, consumer):
             consumer.validate(json_path, producer_file_path)
 
             print('-- Validating stream')
-            producer_stream_path = os.path.join(self.temp_dir, guid())
-            consumer_file_path = os.path.join(self.temp_dir, guid())
+            producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
+            consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
             producer.file_to_stream(producer_file_path,
                                     producer_stream_path)
             consumer.stream_to_file(producer_stream_path,
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index d472b559347f0..ca6d9ecbe85e6 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -59,7 +59,7 @@
         { class: "DateDay" },
         { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
         { class: "TimeSec" },
-        { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
+        { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
       ]
     },
     {
@@ -71,12 +71,12 @@
       minor: [
         { class: "BigInt"},
         { class: "UInt8" },
-        { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
-        { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
-        { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
-        { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
-        { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
-        { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
+        { class: "Float8",         javaType: "double", boxedType: "Double",                              fields: [{name: "value", type: "double"}], },
+        { class: "DateMilli",      javaType: "long",                      friendlyType: "LocalDateTime" },
+        { class: "TimeStampSec",   javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampMilli", javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampMicro", javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampNano",  javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
         { class: "TimeMicro" },
         { class: "TimeNano" }
       ]
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
index 9b4b79bfd7b90..e723e7d7ea3d0 100644
--- a/java/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -57,6 +57,7 @@ import java.math.BigDecimal;
 import java.math.BigInteger;
 
 import org.joda.time.DateTime;
+import org.joda.time.LocalDateTime;
 import org.joda.time.Period;
 
 
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
index 79d4c122f0e4e..b16ee162fde9e 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -26,8 +26,8 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
-/*
- * This class is generated using freemarker and the ${.template_name} template.
+/**
+ * Source code generated using FreeMarker template ${.template_name}
  */
 @SuppressWarnings("unused")
 abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
@@ -51,7 +51,7 @@ public Field getField() {
   }
 
   <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
-          "Character", "DateTime", "Period", "Double", "Float",
+          "Character", "LocalDateTime", "Period", "Double", "Float",
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index dc99aad0bb3a2..93746303d9311 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -38,6 +38,7 @@
 
 /**
  * Arrow types
+ * Source code generated using FreeMarker template ${.template_name}
  **/
 @JsonTypeInfo(
   use = JsonTypeInfo.Id.NAME,
diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java
index 72fea58d0bc9e..ea3286e86817a 100644
--- a/java/vector/src/main/codegen/templates/BaseReader.java
+++ b/java/vector/src/main/codegen/templates/BaseReader.java
@@ -26,8 +26,9 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
-
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public interface BaseReader extends Positionable{
   Field getField();
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index d53744539aae8..38cd1bfdeb3c5 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -47,6 +47,9 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class ${name}ReaderImpl extends AbstractFieldReader {
   
@@ -123,12 +126,16 @@ public Object readObject(){
 package org.apache.arrow.vector.complex.reader;
 
 <#include "/@includes/vv_imports.ftl" />
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public interface ${name}Reader extends BaseReader{
   
   public void read(${minor.class?cap_first}Holder h);
   public void read(Nullable${minor.class?cap_first}Holder h);
   public Object readObject();
+  // read friendly type
   public ${friendlyType} read${safeType}();
   public boolean isSet();
   public void copyAsValue(${minor.class}Writer writer);
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 3457545cea5d7..c23b89d1bbcfb 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -139,6 +139,9 @@ public void writeNull() {
 package org.apache.arrow.vector.complex.writer;
 
 <#include "/@includes/vv_imports.ftl" />
+/*
+ * This class is generated using FreeMarker on the ${.template_name} template.
+ */
 @SuppressWarnings("unused")
 public interface ${eName}Writer extends BaseWriter {
   public void write(${minor.class}Holder h);
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 05faaae1e9e2f..f403ecfac1f93 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -495,16 +495,14 @@ public long getTwoAsLong(int index) {
     <#elseif minor.class == "DateMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-      org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
     <#elseif minor.class == "TimeMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-      org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
       return time;
     }
 
@@ -512,16 +510,14 @@ public long getTwoAsLong(int index) {
     @Override
     public ${friendlyType} getObject(int index) {
       long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
     <#elseif minor.class == "TimeStampMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-        org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+        org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
         return date;
     }
 
@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
     public ${friendlyType} getObject(int index) {
       // value is truncated when converting microseconds to milliseconds in order to use DateTime type
       long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
     public ${friendlyType} getObject(int index) {
       // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
       long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index d66577bc1e444..e990fcc933479 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -84,7 +84,7 @@ public boolean isSet() {
     
   }
 
-@Override
+  @Override
   public void read(${name}Holder h) {
   <#list fields as field>
     h.${field.name} = holder.${field.name};
@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
     h.isSet = isSet() ? 1 : 0;
   }
 
-
+  // read friendly type
   @Override
   public ${friendlyType} read${safeType}(){
 <#if nullMode == "Nullable">
@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
       byte[] value = new byte [length];
       holder.buffer.getBytes(holder.start, value, 0, length);
 
-<#if minor.class == "VarBinary">
+  <#if minor.class == "VarBinary">
       return value;
-<#elseif minor.class == "Var16Char">
+  <#elseif minor.class == "Var16Char">
       return new String(value);
-<#elseif minor.class == "VarChar">
+  <#elseif minor.class == "VarChar">
       Text text = new Text();
       text.set(value);
       return text;
-</#if>
+  </#if>
 
 <#elseif minor.class == "Interval">
       Period p = new Period();
diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java
index ba0c088add7c9..7c75b3ae1df9d 100644
--- a/java/vector/src/main/codegen/templates/NullReader.java
+++ b/java/vector/src/main/codegen/templates/NullReader.java
@@ -29,7 +29,9 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class NullReader extends AbstractBaseReader implements FieldReader{
   
@@ -127,7 +129,7 @@ private void fail(String name){
   }
   
   <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
-          "Character", "DateTime", "Period", "Double", "Float",
+          "Character", "LocalDateTime", "Period", "Double", "Float",
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index c56e95c89dc81..0b5a209d40ac4 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -28,7 +28,9 @@
 package org.apache.arrow.vector.complex.impl;
 
 <#include "/@includes/vv_imports.ftl" />
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class UnionReader extends AbstractFieldReader {
 
@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
   }
 
   <#list ["Object", "Integer", "Long", "Boolean",
-          "Character", "DateTime", "Double", "Float",
+          "Character", "LocalDateTime", "Double", "Float",
           "Text", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index d70cbae02bf33..9d5dee5d237e6 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -50,6 +50,7 @@
  *
  * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
  * each time the vector is accessed.
+ * Source code generated using FreeMarker template ${.template_name}
  */
 public class UnionVector implements FieldVector {
 
diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java
index d744c523265f7..a474b691080c8 100644
--- a/java/vector/src/main/codegen/templates/ValueHolders.java
+++ b/java/vector/src/main/codegen/templates/ValueHolders.java
@@ -29,6 +29,9 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 public final class ${className} implements ValueHolder{
   
     <#if mode.name == "Repeated">
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 6023f1c9500e7..d5076d82c2a4d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -241,7 +241,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
-    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, "UTC")) {
+    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
         return new NullableTimeStampSecVector(name, fieldType, allocator);
@@ -253,7 +253,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
-    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, "UTC")) {
+    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
         return new NullableTimeStampMilliVector(name, fieldType, allocator);
@@ -265,7 +265,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
-    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC")) {
+    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
         return new NullableTimeStampMicroVector(name, fieldType, allocator);
@@ -277,7 +277,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
-    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC")) {
+    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
         return new NullableTimeStampNanoVector(name, fieldType, allocator);
@@ -580,6 +580,9 @@ public MinorType visit(FloatingPoint type) {
       }
 
       @Override public MinorType visit(Timestamp type) {
+        if (type.getTimezone() != null) {
+          throw new IllegalArgumentException("only timezone-less timestamps are supported for now: " + type);
+        }
         switch (type.getUnit()) {
           case SECOND:
             return MinorType.TIMESTAMPSEC;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
index 1f8ce069cf9cf..8aad41744f673 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
@@ -18,6 +18,9 @@
 
 package org.apache.arrow.vector.util;
 
+import org.joda.time.DateTimeZone;
+import org.joda.time.LocalDateTime;
+import org.joda.time.LocalDateTimes;
 import org.joda.time.Period;
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
@@ -679,4 +682,11 @@ public static int millisFromPeriod(final Period period){
       (period.getMillis());
     }
 
+    public static long toMillis(LocalDateTime localDateTime) {
+      return LocalDateTimes.getLocalMillis(localDateTime);
+    }
+
+    public static int toMillisOfDay(final LocalDateTime localDateTime) {
+      return localDateTime.toDateTime(DateTimeZone.UTC).millisOfDay().get();
+    }
 }
diff --git a/java/vector/src/main/java/org/joda/time/LocalDateTimes.java b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java
new file mode 100644
index 0000000000000..e4f999e1d828e
--- /dev/null
+++ b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.joda.time;
+
+/**
+ * Workaround to access package protected fields in JODA
+ *
+ */
+public class LocalDateTimes {
+
+  public static long getLocalMillis(LocalDateTime localDateTime) {
+    return localDateTime.getLocalMillis();
+  }
+
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 99ba19bec80e7..aba65dbf374d4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -56,8 +56,7 @@
 import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.Text;
 import org.apache.arrow.vector.util.TransferPair;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
+import org.joda.time.LocalDateTime;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -602,10 +601,10 @@ public void timeStampWriters() throws Exception {
     final long expectedMicros = 981173106123456L;
     final long expectedMillis = 981173106123L;
     final long expectedSecs = 981173106L;
-    final DateTime expectedSecDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 0).withZoneRetainFields(DateTimeZone.getDefault());
-    final DateTime expectedMilliDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 123).withZoneRetainFields(DateTimeZone.getDefault());
-    final DateTime expectedMicroDateTime = expectedMilliDateTime;
-    final DateTime expectedNanoDateTime = expectedMilliDateTime;
+    final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0);
+    final LocalDateTime expectedMilliDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
+    final LocalDateTime expectedMicroDateTime = expectedMilliDateTime;
+    final LocalDateTime expectedNanoDateTime = expectedMilliDateTime;
 
     // write
     MapVector parent = new MapVector("parent", allocator, null);
@@ -650,28 +649,28 @@ public void timeStampWriters() throws Exception {
 
     FieldReader secReader = rootReader.reader("sec");
     secReader.setPosition(0);
-    DateTime secDateTime = secReader.readDateTime();
+    LocalDateTime secDateTime = secReader.readLocalDateTime();
     Assert.assertEquals(expectedSecDateTime, secDateTime);
     long secLong = secReader.readLong();
     Assert.assertEquals(expectedSecs, secLong);
 
     FieldReader milliReader = rootReader.reader("milli");
     milliReader.setPosition(1);
-    DateTime milliDateTime = milliReader.readDateTime();
+    LocalDateTime milliDateTime = milliReader.readLocalDateTime();
     Assert.assertEquals(expectedMilliDateTime, milliDateTime);
     long milliLong = milliReader.readLong();
     Assert.assertEquals(expectedMillis, milliLong);
 
     FieldReader microReader = rootReader.reader("micro");
     microReader.setPosition(2);
-    DateTime microDateTime = microReader.readDateTime();
+    LocalDateTime microDateTime = microReader.readLocalDateTime();
     Assert.assertEquals(expectedMicroDateTime, microDateTime);
     long microLong = microReader.readLong();
     Assert.assertEquals(expectedMicros, microLong);
 
     FieldReader nanoReader = rootReader.reader("nano");
     nanoReader.setPosition(3);
-    DateTime nanoDateTime = nanoReader.readDateTime();
+    LocalDateTime nanoDateTime = nanoReader.readLocalDateTime();
     Assert.assertEquals(expectedNanoDateTime, nanoDateTime);
     long nanoLong = nanoReader.readLong();
     Assert.assertEquals(expectedNanos, nanoLong);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 5ca083aa2dfab..5cc36a3b82000 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -39,8 +39,8 @@
 import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
 import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter;
 import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
-import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.LocalDateTime;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -144,8 +144,8 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) {
     }
   }
 
-  private DateTime makeDateTimeFromCount(int i) {
-    return new DateTime(2000 + i, 1 + i, 1 + i, i, i, i, i, DateTimeZone.UTC);
+  private LocalDateTime makeDateTimeFromCount(int i) {
+    return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i);
   }
 
   protected void writeDateTimeData(int count, NullableMapVector parent) {
@@ -156,17 +156,17 @@ protected void writeDateTimeData(int count, NullableMapVector parent) {
     TimeMilliWriter timeWriter = rootWriter.timeMilli("time");
     TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli");
     for (int i = 0; i < count; i++) {
-      DateTime dt = makeDateTimeFromCount(i);
+      LocalDateTime dt = makeDateTimeFromCount(i);
       // Number of days in milliseconds since epoch, stored as 64-bit integer, only date part is used
       dateWriter.setPosition(i);
-      long dateLong = dt.minusMillis(dt.getMillisOfDay()).getMillis();
+      long dateLong = org.apache.arrow.vector.util.DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay()));
       dateWriter.writeDateMilli(dateLong);
       // Time is a value in milliseconds since midnight, stored as 32-bit integer
       timeWriter.setPosition(i);
       timeWriter.writeTimeMilli(dt.getMillisOfDay());
       // Timestamp is milliseconds since the epoch, stored as 64-bit integer
       timeStampMilliWriter.setPosition(i);
-      timeStampMilliWriter.writeTimeStampMilli(dt.getMillis());
+      timeStampMilliWriter.writeTimeStampMilli(org.apache.arrow.vector.util.DateUtility.toMillis(dt));
     }
     writer.setValueCount(count);
   }
@@ -176,13 +176,13 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
     printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
       long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i);
-      DateTime dt = makeDateTimeFromCount(i);
-      DateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
-      Assert.assertEquals(dateExpected.getMillis(), dateVal);
+      LocalDateTime dt = makeDateTimeFromCount(i);
+      LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
+      Assert.assertEquals(org.apache.arrow.vector.util.DateUtility.toMillis(dateExpected), dateVal);
       long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i);
       Assert.assertEquals(dt.getMillisOfDay(), timeVal);
       Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
-      Assert.assertTrue(dt.withZoneRetainFields(DateTimeZone.getDefault()).equals(timestampMilliVal));
+      Assert.assertEquals(dt, timestampMilliVal);
     }
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 824c62aa5fbf3..f9c8f726ab6c6 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -81,7 +81,7 @@ public void nestedSchema() {
         new Field("child4.1", true, Utf8.INSTANCE, null)
         )));
     childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
-        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null),
+        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, null), null),
         new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));
     Schema initialSchema = new Schema(childrenBuilder.build());

From 5af8069d234a7b16ab324085ecc802e6f915ae88 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Fri, 5 May 2017 22:07:42 -0400
Subject: [PATCH 0608/1644] ARROW-866: [Python] Normalize PyErr exc_value to be
 more predictable

It is possible when using `PyErr_Fetch(&exc_type, &exc_value, &traceback)` for the `exc_value` to be a string, tuple or NULL.  Calling `PyErr_Normalize` after this will cause `exc_value` to always be a valid object of the same type as `exc_type` which can then be converted to a string predictably.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #630 from BryanCutler/python-pyerr_normalize-ARROW866 and squashes the following commits:

fb93356 [Bryan Cutler] use PyObjectStringify to be Unicode safe
d56c6bf [Bryan Cutler] Added PyErr_NormalizeException to CheckPyError to make more predictable exc_value
---
 cpp/src/arrow/python/common.cc | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
index bedd458c783f4..5702c71b4d8d5 100644
--- a/cpp/src/arrow/python/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -68,20 +68,16 @@ Status CheckPyError(StatusCode code) {
   if (PyErr_Occurred()) {
     PyObject *exc_type, *exc_value, *traceback;
     PyErr_Fetch(&exc_type, &exc_value, &traceback);
-    PyObjectStringify stringified(exc_value);
+    PyErr_NormalizeException(&exc_type, &exc_value, &traceback);
+    PyObject *exc_value_str = PyObject_Str(exc_value);
+    PyObjectStringify stringified(exc_value_str);
+    std::string message(stringified.bytes);
     Py_XDECREF(exc_type);
     Py_XDECREF(exc_value);
+    Py_XDECREF(exc_value_str);
     Py_XDECREF(traceback);
     PyErr_Clear();
-
-    // ARROW-866: in some esoteric cases, formatting exc_value can fail. This
-    // was encountered when calling tell() on a socket file
-    if (stringified.bytes != nullptr) {
-      std::string message(stringified.bytes);
-      return Status(code, message);
-    } else {
-      return Status(code, "Error message was null");
-    }
+    return Status(code, message);
   }
   return Status::OK();
 }

From 995317ae9ecb54bc1aec02f7c7e133ab61ac387f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 6 May 2017 12:56:46 +0200
Subject: [PATCH 0609/1644] ARROW-929: Remove KEYS file from git

I have updated the SVN KEYS file at https://dist.apache.org/repos/dist/release/arrow/KEYS

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #646 from wesm/ARROW-929 and squashes the following commits:

8ad3c0a [Wes McKinney] Remove KEYS file from git
---
 KEYS | 239 -----------------------------------------------------------
 1 file changed, 239 deletions(-)
 delete mode 100644 KEYS

diff --git a/KEYS b/KEYS
deleted file mode 100644
index 05862c8f643ca..0000000000000
--- a/KEYS
+++ /dev/null
@@ -1,239 +0,0 @@
-This file contains the PGP keys of various developers.
-
-Users: pgp < KEYS
-  gpg --import KEYS
-Developers:
-  pgp -kxa <your name> and append it to this file.
-  (pgpk -ll <your name> && pgpk -xa <your name>) >> this file.
-  (gpg --list-sigs <your name>
-    && gpg --armor --export <your name>) >> this file.
-
-pub   2048R/7AE7E47B 2013-04-10 [expires: 2017-04-10]
-uid                  Julien Le Dem <julien@ledem.net>
-sig 3        7AE7E47B 2013-04-10  Julien Le Dem <julien@ledem.net>
-sig          D3924CCD 2014-09-08  Ryan Blue (CODE SIGNING KEY) <blue@apache.org>
-sig          71F0F13B 2014-09-08  Tianshuo Deng <tdeng@twitter.com>
-sub   2048R/03C4E111 2013-04-10 [expires: 2017-04-10]
-sig          7AE7E47B 2013-04-10  Julien Le Dem <julien@ledem.net>
-
-pub   4096R/1679D194 2016-09-19 [expires: 2020-09-19]
-uid                  Julien Le Dem <julien@ledem.net>
-sig 3        1679D194 2016-09-19  Julien Le Dem <julien@ledem.net>
-sub   4096R/61C65CFD 2016-09-19 [expires: 2020-09-19]
-sig          1679D194 2016-09-19  Julien Le Dem <julien@ledem.net>
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-
-mQENBFFll5kBCACk/tTfHSxUT2W9phkLQzJs6AV4GElqcFo7ZNE1DwAB/gk8uJwR
-Po7WYaO2/91hNu4y1SooDRGnqz0FvZzOA8sW/KujK13MMqmGYb1jJdwPjNq6KOK/
-3EygCxq9DxSS+TILvq3NsFgYGdopdJxRl9zh15Po/3c/jNMPtnGZzP39EsfMhgIS
-YwwiEHPVPB00Q0IGRQMhtJqh1AQ5KrxqK4+uEwwu3Sb52DpBjfgffl8GMGKfH/tk
-VvJ6L+7rPXtNqho5b7i8379//Bn9xwgO2YCtjPoZMVg37M6f6hVWMr3fFmX/OXgU
-UWwLGOTAeuLKWkikFJr5y0rzDaF2qcD9t7wfABEBAAG0IEp1bGllbiBMZSBEZW0g
-PGp1bGllbkBsZWRlbS5uZXQ+iQE9BBMBCgAnBQJRZZeZAhsvBQkHhh+ABQsJCAcD
-BRUKCQgLBRYCAwEAAh4BAheAAAoJEJfX6GR65+R7au4IAIfZVA9eWBZn9NuaWX7L
-Xi+xDtzrfUrsWZxMIP6zkQsIspiX9AThGv3zDn+Tpfw7svV1QfUQX0LHbwMMYqq+
-mRJB/kqYutpLxw7h63zrWR2k2Sdzvole2c3Rfk1vblIdWZk7ArLSivqTk/oGwr7d
-MejvOMmKSzqW0vQF6dNbYerLOiqPr4mKqONWm4nOLZEBzjE3IfbK3gNBSFq+92jV
-iWY6ozqAxydYafNUSZRrcniYskxd9JCSSLZiIZW3X9lToA/74LjpPbmzvQtkH68D
-0EnC1mkPTKCA4r+CLb3a9GJ9Surg2T0OptyPHsXipgViVryXgopD2odA3fh9SY5l
-Ee+JAhwEEAECAAYFAlQN+kQACgkQ/LPL2dOSTM3+OA//dYj9kiZhZNVb6hMfrubn
-OjTmY8Hcax8G+aJWxRrGE8HrCUjEJ4NThK523+fmol1PxNWsguljlsZvJ189YPOh
-weDJzNmKwhLntq/uBgtJyWBN1v9bUzkR9Ud+UdD1tPbNj7sNiIQE1ZqWMxra3sq/
-gcodVgqSADGgjKO9tenQhWvQXxBR55MOqZbxnyazRPEYS0mkN0A0DwtG82tHNRL7
-Z3vs/kG5hoW3kYifCZn5pW3wKtfIY5JH7usYOzA86p7GH4hOfO+dzhDANH+C+u9O
-ZRbCdUE8oEp3fAWY9+3VzlO5ixpFOeHGfbSJp44Jv6wUOxNwRmD/gk+DxVrsS/Yn
-rLFCZgDHgkFHGJ1D7PnxTy4qtwGasYxWYJOUiaAJbOvRa8nbhan2/wsrgnJTbXAH
-+7v5tFfCV77Po//V0fojYZNvbkEO8/yRpQL+uKiVRaRD5dMfHRb31OR0A59ssYX9
-63QpBEof/OeELC0VowG+KCc+4CfSMmAGnQMdEhMAUPz+79nJw7ijeF5C82Z5mQof
-v+nf+kdqr80UbG+RoODKtlHFETxJ5STQe6uiPOfvb+EADPA0cZ34u5tD3Z+SMV1k
-Gf7Jxi45jmkn9Z9AkVj6KgdDeSjV7EkRiY0pm43Vvd6WvV5t54cgJcwXrjG+h03f
-65w7F+KBrh7YAcUvrf4JeXKJARwEEAECAAYFAlQN/XwACgkQfNgniXHw8TtU9Af/
-b9CYFtsG9q1ZbnV9SChxjLLUipGsmKTUjCnz7oiZvJJ04e+0np1NQJKJbthGfEDM
-eLt1WiYpTDu66zAuLDA7ACcbv3UUXXsUTEfN76J+9DJHrtK1soHGLkKLW2hZeWKp
-PKya/HRF4Rv3/aAwWtRjEuQr9pLt/wAOedV6mrpyTngOKQn97tzo/yUeDNG7be8A
-xtUStQY/2zJmHkaLeULKOspgUchBQ1S+M4q46dE+tyel47BLyHIECqk/geLOlZmh
-lo6TtVgnBSXC5SqMwh5pz/P5ntQ8FVLedGQI9dwVhxbjoo5DNB/6ntfbwkheiak1
-CFBm0ZVPJjX7F2XFcq7VCrkBDQRRZZeZAQgA4eixR7xHvnTyF12CYLsnFE8x1tI+
-78FCjKm0n1YPCzEYa70bnnZmpW4KCwO0flN4RhhP+g2KRCCov2ZH7bxvhTxe4n/j
-T6I/+61Fpba4I7qExYqX+tylyjUKhynLcWCbvRQnyjOMTaLbMVrftV+ATVmj7fi0
-PdzRW/7QvCSrDsMFtTSaNBdeMbzptpoXAxTgVZOIoHbWOIfovN1uPnFItrmNnKXX
-KGyDPX2s2KCz10G1lrw0l9tqDg+BtqE9/xCtqWoZJMnT8jAJZeJ0V37R1jDBDEHK
-AfPOUKNYf5GWxJeCWYzL77ve8VdItKwPhtjW7zFKuyrqiBHE40fgTLKvNQARAQAB
-iQJEBBgBCgAPBQJRZZeZAhsuBQkHhh+AASkJEJfX6GR65+R7wF0gBBkBCgAGBQJR
-ZZeZAAoJECrRWHEDxOERzmEIAOCrfYGPdLyzBn/xAdymx2FaNTS48ybMIGjcu6Od
-nKzvgBJObLPQf0+WKhkbQf2HEHYinBVpX8K4dNY9RhzIRbQNhCWY5E5/leI/nQ9O
-ZBUMpT8Gw5saj0YtF3By4E9ywxNWiAyX2SAHjPv/lub0PEaUiWWe6s9MaX5fp71C
-TupkdElpxucEpVefUaUOSMQ2ecOniCh/9ltPLYcjwnC1ti+Et8/cAK2N554GNE+x
-fO3qtGXGUleWhpt3fblTcCyO+odAPKxm70jnABLk8m+KpffcdBYSJ5ai5hPkrnyq
-3NBRDPGlLdtDkzn0/xKYnVbLW1d+d2NFwJzEKncQphHoo0T19wf8DSfym7dIsstj
-jwFI8+N/1yCdMD886x8bgmsSsNiD9tro+1083yr+IL5+gUs8Q4ETpsow+IS6sfp2
-fzA0TaLBLEOFYy/XFxnzO+YtVNIDAnrDEgTOMahFUrJ/HVZF9xT+kKwhyHaRNIQL
-CYc4VoSWldqoDVOGI30NjtVo5EGzf3qVWkTm4yplBhJvJanxrMHuJAWRgFX8D48B
-cs/senr8s+O0oXQQYIjz/FkZh/mQFtrgsvnzyUR52SnwEzNMmXjZNkydPZwcY6mu
-cqCIvQIvmBpPdlyaoglwJ8wWb76uIE6VFcN71FF3EfV51/yUeQGJaoExWLY6IH8x
-Xtn3IWkBWJkCDQRX4AxuARAAzzTxE1FGdmJYPZyTys51oDi8+CJ8VXF6wlTkjuOW
-abkGUu0FjnN++D7G9LRDvN7QnVUHU+h6QWPZ0LanmjYh0ABO5SeWCjOX6ajcACkz
-pEzMv2DbOPfJuPJmtuFfiLOQAUVBB1dSSPFMPPaGTco2iE7uLr8edtQBvgpx/PGd
-52lma3qAAZFzonKWyTRonUjV4SU3C96Xhbs+DExTL8H6MX8NzZCz4UZj5u4NsEH+
-oQD0U4tSOe3xgroJpOR6ZPvlhgbWVqlYvkEWt0AaPJsXJwnWe17GgDmxME2cwsuI
-fgv/9shB7VYmLglY6dV/6HYoHh+2qKXndTMjlqXXvUHW0J3uRryoCR+C2gin38/f
-sPFICpt5yJVnR517O/jsviDz4TwjhqFsUUM7Ud8IydriJX02Oj5UitUF7l5MSqkS
-/Z7jwPEErCRWwmfj4ZjjWWV60I9SYgPZhBp0//s2o/gbIBBtIdHI2+xaMt0lWOsA
-Xi7dsY1NLGoSGUlhdSiP032tVHpGiOV3AWwf399Qus4iuwf6N8KEVSTRdaA7b1Um
-b7PepfEHIrOS5oUYjgZJK+JFGey5SOsPvG3Yv9cbEKWqmoEzEDb6y3HI/iRbk/qC
-SWGKvEiqYSo6wlvZFDv1qoApylfBaI8Lf32vawlMCSI37KCWfua1RqbCYMi/4wux
-bfsAEQEAAbQgSnVsaWVuIExlIERlbSA8anVsaWVuQGxlZGVtLm5ldD6JAj8EEwEI
-ACkFAlfgDG4CGwMFCQeGH4AHCwkIBwMCAQYVCAIJCgsEFgIDAQIeAQIXgAAKCRAC
-2r/fFnnRlHqND/oCaPPGn8u5oyVml9J3+lpYWwT69qHwYV5IX+72zqq02uvYEqlY
-CseEwOvkfDClh81KWO4A9kzVwWcu611d/UFsA94EnZuJ46m8DflPeidhJoTRnNpr
-IRzH2lL72QyQFeT9viWHdxu3cKlJkChQuD3zR9yyVH/QVFOlBvdx/ZT0dOFpbgJd
-2n4fy8ExGSXLP6wGf5RQRKEYiZX4VB4Bkz6sK0Con6GPsqqtaUgj9fCxA1ebhGA2
-k1m79mR9gh4oJWeefSuXyf3x8oBoQ46Lury8HiuxLh6cy9SqHZ8uXu3hfQEZ6rhd
-C9yBjK2+8z6GLhjgSasMkJK0OAR8yLgARZJwt9+wV/Ww15Sq76B6IrKJnSR6P4FM
-jA+ItCDtiooGz6rJGFidsH99fU2IydcsSqbTN3h3/2cBXFgxspesHWsEeTvtXSgR
-I82kUyA0g1v9ESY0leiLVzKyL0zmCjaPg0nHoH8tFqFkqaSXSZu75TefnsokkoXN
-ewkDf+yD2J/BMtUHAgFOlvYkywGzS9cbAxxzc607Jvww5LjtPI0wYRIwzOlvZfws
-xoYPrqJe1R7hRy0QS6pnSL7TgdbvwbGtiUAZ9w/Y5FEugV8bgyZMvF7Z9gt3ThMg
-XOSWlZrsDym7jg/yd/h/4aPZuPC73oNvgV4g6OT510fkkMNWbZR8C2uHX7kCDQRX
-4AxuARAAphEmWY5Z3Q+gQ1X9+b55VE17ORMKjXtE2gQnYk9Fxpt31F0kZcoK/25Y
-BItkjcmIaC4LTLjbdwe6IW4zlOjULxaTstTJsfCcrJONlSmEJ0OWaXx9i/tAXt8d
-0IZn2hkQ9aevJuoWqta+wFNhpLdPuPQq6vO6hIl7j0w1tAGFHV+IQ7Q7VFuUVo/h
-gZbtJOjufZWqulz6pMVu4p3TW9OM98CWioO1eidcwKYEsgk/fJ1uKc599SSCz2Cg
-+lEho6rHtvojk34TLD9QQHaEcCZToq7WSqwqLi4OCuhcfAVuwydj0RMByE1TOpsg
-RwOh2egBLNplK+0k2jVaQPX38laolOAMNLg+VVRy73T1MpyelY/m+cRr8292X/f4
-GgHNHbmQU+LDzsezC+ryPXdP3FjVo66xNlYHzw1x0hRdnwExkqOYmdTz1YN83Z+6
-p0d2RkTZTpLnE449KiNsgTPttplBGE8QKNqYxoKIk40DlDuya7q/acgTcqe8vW0v
-34E6RRIX8dbCJeTBB2vUDp6bD3ICXGI09EuUAh9yy/FlNv1OdggDfTnF/NztWmmT
-CpNwmdx+GTT2Sv0i6H9RelOl0uGj351+7PSFSFrHV9T3TUaMB0QkkZDxklIvPVZv
-dhx7UGXFJPDjQyJxcN7UW6Pc7m+m2W3/u2MZaL7xPbWRPVkqs48AEQEAAYkCJQQY
-AQgADwUCV+AMbgIbDAUJB4YfgAAKCRAC2r/fFnnRlGwCEACXcfMAz79G2sLs6z1N
-6tMbO0qGGQJ9vAXRKjb7JN/yd+z+zaejs/+cmRM+wHKZtANYtnSzGiWJO4TIP5A+
-DeRE93GJaVr0ly0C+du/uSm6wVg+w1wgy6JE6q/IsMW5qHd0qWi/npq4esfH1Uho
-T7Kl/AxUyT0N23n5oK0GrVUFhFcU9dUx6auhHxEOM5tIgNqV6lAn72lykPYUvV5f
-aAiz2OAlVYxgBb6wxjXTUVlrhaxbgNQ7PPzkjzMVZaE/TZrcyl4Ck3grYDBFZEGi
-jhjsl/HX+/lhJvr5gcFkisG5A2pnrkAe1wnXm4HoKGN2xUWCCipN5oPc3Lw6ge76
-YDX1t5CXqd94cDBlwFDtd4kykI3rJDvTI3P/fevMNqVS3tzW9AwkHkPil1DE+4rI
-/qCib+G6BAgloUGYLuNxSa1ySOd0yckFTrNBB5yk+yWvrLpKGFVdQS7BwUcgdeCJ
-3XU3fyhfXcIn3tMHabZ6laB3Xzi3Gi8iL6SJywSXIqTGw3MmLJlxr1IKWTMNxjjs
-d/XBF7ZpCCisH7s9hyMCAet72YFAxVcB3bwbd3mzcGfTg/Y+sSum82vaSvAJ0QBc
-pp4X8HzEsSsJ88N6ON7IU92r+1mxWhglKZx2NORHIvNFwIrvAzKWhqGdHd5/xq3f
-EwCykGi6RtdCStNFh6h16kCkgA==
-=YkSF
------END PGP PUBLIC KEY BLOCK-----
-pub   4096R/8CAAD602 2016-11-06
-uid                  Uwe L. Korn <uwe@apache.org>
-sig 3        8CAAD602 2016-11-06  Uwe L. Korn <uwe@apache.org>
-sub   4096R/7BD1BC86 2016-11-06
-sig          8CAAD602 2016-11-06  Uwe L. Korn <uwe@apache.org>
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-
-mQINBFgfd4wBEACylQqqVH/aK00fgU/v1ZggNwtgJhzH7yswAzQz9eUU5t4Q9kzI
-zdkR1yJvaEDHtZy2D0mCM1CuGVPXzf+0kSFDaRPcm6LNAD15KC7eUzyad1Y4MwNn
-UYE3pZlnvSwUBAigQSN1quw+u1eHc+IJc32iCRcK8DihQgrDivg8yZckoGGZj/6w
-Epfp8SLrI+OmqBgwYYjRqy9uC0aWypKb9waZmc2NIZZu1y3bL6hx54+Dk+4hF01E
-OtT79HQV1e4MyqiuGUKa34QAHb1CGrju+1Z9sDNdI7hBDqfQKjisR2WaJM4kXHjj
-m7Tv3M1LUB4eh1+Yd514d/wpSChkLvMCJ9tYGSpQ8c+qrLAFvgRD7YCYp4ypslcx
-Sg30gU0bcTu8aiIm7qfl9CUjtBYwirUGC/t2SUxnhOpxWuzZdAiUJHi0QFa+LnZa
-ecA5fIoMfqTWAqfQr3noxB6qLLNCgZd7IIH5KXIIhJZHpO3eMCCTJuDXiMS1Z/uo
-D1FvUL8c19nmMjPJSfQo95Uynw6gZKFy0d3xg7NKUvnJBsVI24/PTVabzRrDh/qb
-RCHvQOFjXOSYsPm2sz1BPs+ucV4AoxPZFgsCfUN2t4FRbcb39vr6oYFb+Nd3sIKX
-7wknSwAid6pATvfZuLC9NI8ykjcEDGeLL0sET3kdUeuGYjpj2kuhnrV4cwARAQAB
-tBxVd2UgTC4gS29ybiA8dXdlQGFwYWNoZS5vcmc+iQI4BBMBAgAiBQJYH3eMAhsD
-BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAp2U4ijKrWAos1D/98UBoLbt6L
-c7mnXTww069nkt0vOOHSz/QWJxo5rQsqFSKcSRuBhwLuaVMGTjBqCOLdEmA+XKJ8
-O+OgCZz0QZXuwL3PklX3DFvsYO0wIEIssovEJMu5e3XxDcCf5ZZtfszW5dnbWTjc
-JXP0TlEbjOR5Z0/O/24iysGtoEMiktRTLOz9R5oRXFQLN4jQSykvMfKhanCVFljX
-qEdMszjtvZhLwOiCaWkIOEo3jCrCDhdThI5nTiu/pH3vi7mkFYTNKpiva3XYKH7V
-ITEdn5WO/QNFu/VBRjtOxT+F068vuuNpvAddn5rOtZOyGMCHnEqnlRnqIIZGtJeo
-EJ87N2ytn8CtKpQKhyJIJhQIfW5jS3YW8qj1HeKN2s5wqQKnBYYsJOh+/QC9g3oE
-nllgoSHAKSzys2Y1VoOQbRxYipCqRx7uS2aAqFr6r3hQpzySWeKQuxVZSZD7ar/0
-AFB0Hg4EgUGDl6Lw5icJ7scXTgoQKZWH1UmNc/FwFbG/F1GVbU88R6DlF84D1X/P
-ArtP20eT+B3u5nfO2pCaBVi6GYyMsL2WKHO7AQAgURMgEPk0AQZZpv/OSJFa/TzI
-UQ8xTLgmwZRL/XjjNFYWs+eYecGQsHKLbKNm1BpZMEfbVSFw54PiyJgoOhdMKdyA
-Cmb+aUBkbPXf5S0ScZOoq8e8k1dYseDGOLkCDQRYH3eMARAAx/joL6ScsKMmPGRn
-n79gQ3zbcKxWSfEDMYeeFfSssRgRd2iIrgvjzr9phka2yknzPnQPi7C8GLkUTj5e
-V1dBxIGkGmP28n0DoowMqGb1xqn0WeoxDL0VQycGjkv5SOkxcbCCKS/MHOn6zenh
-patSJsEHkCqk3f4GtPngYN5oMRTXUfUj1s7AooNti1ONSQSvZNbOMKAg8MgAjAHm
-z3A+INLVTa59vqUNr5ptG/n+cB65ggeNhJf3gMaDyUy7oRZtOhrmA4D9CLpy2OBA
-gezgOCZk/mPNP5jW0sbRiL6nYqC9VTp0E+f3hYSdgXNTWGIcxOwK7xe09SRqUQ7u
-WnoKBTjkkYdCaCN4rv8IhJdrufgYdfqMGuldQZ9R/gcN3Iel7JMdon2onk94KZPs
-W58/1DCD2eRuz8CsIgleUHVXJ+mCpkdtAt45ZGyv5pFC/+6s8mS/pBQEdVl7wjEX
-kf2lrtFZCfK1uUiUTDnJJdtXNhwdtvnxJYeRg51jlD9Qg/mPV6m8KFyINtLKedLv
-hChFkAIfFsdC/r1Xt4fMiCv2eZ8Dop2dM6xV/6Ueicti0lywoTpVtugSUWPO1j8a
-N48jUfkZUV0jdELNHAloZaIDeLc7mU0uZJ3JykC4laD+YDwHT8tYUvamtU2uNgh1
-V7I3jrEu8YO4T2fiXe+0EzBwzjEAEQEAAYkCHwQYAQIACQUCWB93jAIbDAAKCRAp
-2U4ijKrWAs3bD/wOE8NLnzKqebz0v+lxQf7fRL+RMaJ8mFda/t7UFtxj6XdePGZy
-HWdqlvBFSDo/K6aEiicmpEIPbMi+V7d1Dg3tGhwtkHzgbpxNVoolR+2cF4jtrkoV
-NC7uAMaDPt0X+wqinGg4E7IFuJoT4WiS+i4lzCUbD8n7lxe6Kj9bDt8tb6gOCgld
-oweGN2k3bc4hIzeRt0jqGu1xm91Zbf8YbI3vyi8WQqmxX3zugY46NWwj8a+4Mhxz
-Ysd7SI1pPs5k7vdHif3MD3Wwx68CCuZSm2KzNsm0iGxrCXSA6dXVflK9rlq6O1Us
-UTxfX60o6S8PdFr4oOPFHYXmvDU5PY575xscWB2VVAyuSCyZWtq8d1BBU9JxcozS
-6PTefVUqgr0XXRwVldAIabSA5q13j+b5+vU6LnAuoeMlFFprRlcJN03XTWKXF/gP
-SpCDscCEMbz7aHpox8wmFckeiT+TgwDLMKO5PKRSMEBErUk+SsOyBnFpuGaPsCem
-Pi6NwQyPCt3eep4Ti0dPo3u/dCUEtdKWMpOhsPIoCvGpgqS7o5PuBC2MDHQCc7q8
-wfxeCKBeSpMuy3pvOnNy8uNYjNqizVlpNBx01I2R1MD8P14Pxteg6APi0jcusXrD
-s8g7c7dzdXM0lxreeXge8JSmxuwcCqVUswac6zbX4li03m/lov2YYxCwuw==
-=ESbR
------END PGP PUBLIC KEY BLOCK-----
-
-pub   4096R/1735623D 2017-05-01
-uid                  William Wesley McKinney (CODE SIGNING KEY) <wesm@apache.org>
-sig 3        1735623D 2017-05-01  William Wesley McKinney (CODE SIGNING KEY) <wesm@apache.org>
-sub   4096R/E83E9940 2017-05-01
-sig          1735623D 2017-05-01  William Wesley McKinney (CODE SIGNING KEY) <wesm@apache.org>
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-
-mQINBFkGqIQBEACuKfRxQ2zjpWtuEpKTr0qhpucl5h57cnbPG8M2t2eAbl7fD6mD
-ZyLePZEHSoNgUTqFTh8b850qD2b1loyuk6fx5mesweeWlSxt24Y5pXneH7WL/a8K
-H81jl+Qy5J8DfG8oEnlQp8bPjb3n8xFgNkpt09kxj9lRhDCK0+M0lN/JRGK2BfTx
-TCJWH2vC8Xh+apXmlSR5vohx7dj5RoFlIwNXsi+5JRkZCLoER8Fvozdq7qYNNmgL
-a8l38VnW5fQkx1Pl0mMBi0d4XwFCY6W5BfzfAU3t+ujb0a/6ZzFHiW6q53Fct4BM
-dMX91Xi73Myb3AF3x8dnv7E09dwXaShwUQu76WD/v7js1COS9o3SaCZfOdrJ9+KN
-bYc2zuzXCWtDQ1GU07ocq2Z8VnhGC/qAUwOY9K0JagFOx7xV3gc8bkWqFII0XeCK
-QBhKZHx7oFGz6bH2W/THLolbezwC7+0iuiWeDjY6y6Hk1/S25120wqdUfpa2QDlz
-5V+ayyF8Lt77CnowYeMuDSFZzBjg67SpbbkyZJwKUtTJBUOLKiJF37QCAYENHthB
-lmRgvOcCIic5cnJivgIs6Q7hCpFahWgr2g/6clu04YKFSaup+LU6F3UGvbKW6nnF
-HRSsVFkof0+Ni+yT/oiQUAYyCbrfptpgUZXrVuee8d4frbPfKeiWd4MTrwARAQAB
-tDxXaWxsaWFtIFdlc2xleSBNY0tpbm5leSAoQ09ERSBTSUdOSU5HIEtFWSkgPHdl
-c21AYXBhY2hlLm9yZz6JAjcEEwEKACEFAlkGqIQCGwMFCwkIBwMFFQoJCAsFFgID
-AQACHgECF4AACgkQ8QWIOhc1Yj1IQRAAm71yO273ulTxYlpFTN+CnTqTdxAQIGmc
-gfS55/XmjKfQySQTKOfQPafJe7MazbVG/jG5CZeKHEgHvM0qi8vnAezzeTKEDHPP
-Q1ziHyTt7ND+GbKChrLKA/lbgJkoBxKohyi6eQfz33cvh0fPsv8zej5M6+FAVJaA
-GCMUS/yIC0Oiq0JgYH38sPOhNtw3z8pODg6WjJFWKHXw5qGng11/3BtTVu5KXzqf
-85IJHqMgyOnU0r4mdKgqmSdaCpU/CMJlT3iflF5wN79c46FwAceCiYT8eJiWl1cB
-wAV/mRhTzWGQkWVhE+6EK6+PyuzkjJgGhMtv3zuzKKN8iOv3eb7xptzZydEPqRFf
-50f1cERfsf8um8W9IXQb60vrALyWwQFjF9B2oxsk28ZgzZ5ibA1xU9TJAS+iFo3e
-eITPZnxxT3jZ2WQVWIQB8/yn0sAg5mLQ+Clcghik60KQsjAVS27QrlMTimK6eXey
-tKTS4cw7LPo7GkuiBy3FuERX/ABXg8Wxd+EXOvLuZXNV/p9uBhU0w5tfaasnXFy0
-0NoKAVQ9ffW9MTV3CjrPakjHGLIzHgfFYuHnBdo67E3LR16kLcTusH8e8A3wYgbM
-/gbXNS1C+i31ATNWfHaZtAFrUdzUvotDVo2UTw4nRqy27XBM9NVS+EwfwiZLWoLH
-9gZEMGFQ0MW5Ag0EWQaohAEQAJnHTGcy0ol//23alysOuwYFHsS7PFizcCHuy4jv
-iB8YR5Y4Ts8nAgo8gz2O2m9bgNfbFHStDoqOUWTV7ILYv/CDZiNhvR+fAeWl3Gmt
-o72YYu85r+KZj22YfiXtfOb70IT7jYsTpjlgaqFUFHEHXzoa7EMscra6r+i1qDa9
-QfDjMIBaBCu/Q9CFfhHtIhBHV2Wt8IEJfgYSMHb24Db0RY+OSQixX4QRcVeSnJ5D
-I4ZjA3//o9DwthrJf+GxW/f8TGZy6vtertnxLJXGoHFPVuI895m7wSfzt1+/2nlc
-0obAMX4Q1yRYTOKQGPDeDZ7k5pxnhYkOHDf2gtY5wORw6vN9KR51YFJYXVmK+2zr
-P0fKr0AUG3C7CwQp6bDeYaTndon8S9VNyPypvJ7lpxKy/DIujdvbaJHF3i4rI+w0
-veScfkGtLDc37OeVQEBV4vnHcMvDIC2SEtli4BZjwOcihOv3DgtmQnAjkkAZLtys
-x/W4/MPoZiIWl0DnQev/ujwLkwHCYg/Oo7E70OKpdxDk/2cZyM1US2Uz2NQ4lo5O
-8M+F9sMWj2EPX/kJxZpb6N/+xJnKf4oIdJkaammVllX0TGtoxGOadPST9D8gtSCr
-yRdLMp0bB0+Ghbc+STGo78atg+J+HRvgzXG/gwaEiCIezuLB4W6rFjbldYfbeKTs
-OoAlABEBAAGJAh8EGAEKAAkFAlkGqIQCGwwACgkQ8QWIOhc1Yj23pw/+JNWYULOd
-uM4Khfyx3NgCLiX9VqmwZ7PQQsPKtxviQXdEgs+NJUrCePmjSV9Sf+exTZ4wqSTC
-BilGUppAJbO9avR2wRkYbdiYW+g0jDwAD9cyfAiDBSUiRTimKsKqYN0PbIKJ2Ric
-xvtBw4jW/f1lHkrySqOHetmFTe2ocXkFm8BjqDpt5XCoZa4ADcofNpRJYwVu0Uck
-8MQ/wYjoNRZiz0Sjx9vOBVW9ZKMWS6RgnPStsK3UJiG3c7c83kpDx8nk4bUp8seY
-cBjiViXh6QMXRPdlqsGEMiBVtyXF7Sy3cK3gUcH7808VmKMHEgWvq9MRrZoE0rLK
-74pZrEuWnwD6o77w4DCBtKJyDNlR23kLObS+1Ur7fIXe2yXmbqwEmjpSX4H2Teth
-77PU7nKMAkFsPJDNI7K/kEy3x7KM3G1gIcWaz3pL5gthLV+H3RfIojrK1hS7ZSSI
-gCzYEkQCMsigT5YTgK5+n0I4U7zoDBd1sttwK2FahvuCKUDwc+ZiX/ciYiAjUMb9
-6yTNHlNr/H31EWVZMEd7+fhFZWXJjFsQD11GkXvy6vMBn3Kq+Vd7Yr4CJUGTV3rW
-bWo1vt2ED7h5rbZTrS1UssxLUpy5iXrjyGwn2h/Ei9MzXpNvH8p2raf0eQ0Qn65Q
-UoUryip3RD0yaMCyL/IK3KoPt74f2eJsFwM=
-=feO2
------END PGP PUBLIC KEY BLOCK-----
\ No newline at end of file

From 8febd03f862eab0ca83871e9ff8c5062550b646d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 6 May 2017 12:57:14 +0200
Subject: [PATCH 0610/1644] ARROW-953: Use conda-forge cmake, curl in CI
 toolchain

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #645 from wesm/ARROW-953 and squashes the following commits:

4f719c1 [Wes McKinney] Use conda-forge cmake, curl in CI toolchain
---
 .travis.yml                | 2 --
 ci/travis_install_conda.sh | 4 ++--
 ci/travis_script_python.sh | 4 ++--
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 19e71ae1e68f0..d821b5accb973 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,13 +4,11 @@ addons:
   apt:
     sources:
     - ubuntu-toolchain-r-test
-    - kalakris-cmake
     packages:
     - gcc-4.9   # Needed for C++11
     - g++-4.9   # Needed for C++11
     - gdb
     - ccache
-    - cmake
     - valgrind
     - libboost-dev
     - libboost-filesystem-dev
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index 7d185ee82275b..369820b37f5c1 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -40,7 +40,7 @@ conda config --add channels https://repo.continuum.io/pkgs/free
 conda config --add channels conda-forge
 conda info -a
 
-conda install --yes conda-build jinja2 anaconda-client
-
 # faster builds, please
 conda install -y nomkl
+
+conda install --y conda-build jinja2 anaconda-client cmake curl
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index c1426da7247b2..20b0f2aadb900 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -23,7 +23,7 @@ export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 
 build_parquet_cpp() {
   export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
-  conda create -y -q -p $PARQUET_HOME python=3.6
+  conda create -y -q -p $PARQUET_HOME python=3.6 cmake curl
   source activate $PARQUET_HOME
 
   # In case some package wants to download the MKL
@@ -89,7 +89,7 @@ python_version_tests() {
   export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION
   export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib
 
-  conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION
+  conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
   source activate $CONDA_ENV_DIR
 
   python --version

From c3a122e1cbe83028531bfd73f9a4e1401031c824 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 6 May 2017 10:56:00 -0400
Subject: [PATCH 0611/1644] ARROW-939: fix division by zero if one of the
 tensor dimensions is zero

This was reported and fixed by @stephanie-wang, see https://github.com/ray-project/ray/issues/500

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #634 from pcmoritz/master and squashes the following commits:

399681b [Philipp Moritz] fix linting
44ee13a [Philipp Moritz] fix strides if one of the tensor dimensions is zero
4d831ed [Philipp Moritz] fix division by zero if one of the tensor dimensions is zero
---
 cpp/src/arrow/tensor-test.cc | 10 ++++++++++
 cpp/src/arrow/tensor.cc      | 11 +++++++++++
 2 files changed, 21 insertions(+)

diff --git a/cpp/src/arrow/tensor-test.cc b/cpp/src/arrow/tensor-test.cc
index c41683a3db5a2..0a11422b75d13 100644
--- a/cpp/src/arrow/tensor-test.cc
+++ b/cpp/src/arrow/tensor-test.cc
@@ -93,4 +93,14 @@ TEST(TestTensor, IsContiguous) {
   ASSERT_FALSE(t3.is_contiguous());
 }
 
+TEST(TestTensor, ZeroDimensionalTensor) {
+  std::vector<int64_t> shape = {0};
+
+  std::shared_ptr<MutableBuffer> buffer;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), 0, &buffer));
+
+  Tensor t(int64(), buffer, shape);
+  ASSERT_EQ(t.strides().size(), 1);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index 909b05ebe8f80..bcd9d8d94c6b4 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -41,6 +41,11 @@ static void ComputeRowMajorStrides(const FixedWidthType& type,
     remaining *= dimsize;
   }
 
+  if (remaining == 0) {
+    strides->assign(shape.size(), type.bit_width() / 8);
+    return;
+  }
+
   for (int64_t dimsize : shape) {
     remaining /= dimsize;
     strides->push_back(remaining);
@@ -50,6 +55,12 @@ static void ComputeRowMajorStrides(const FixedWidthType& type,
 static void ComputeColumnMajorStrides(const FixedWidthType& type,
     const std::vector<int64_t>& shape, std::vector<int64_t>* strides) {
   int64_t total = type.bit_width() / 8;
+  for (int64_t dimsize : shape) {
+    if (dimsize == 0) {
+      strides->assign(shape.size(), type.bit_width() / 8);
+      return;
+    }
+  }
   for (int64_t dimsize : shape) {
     strides->push_back(total);
     total *= dimsize;

From 75ebf5ca809ee648a3f64aa9b967246167c509f1 Mon Sep 17 00:00:00 2001
From: Jeff Reback <jeff.reback@twosigma.com>
Date: Sat, 6 May 2017 15:40:43 -0400
Subject: [PATCH 0612/1644] ARROW-956: [Python] compat with pandas >= 0.20.0

ARROW-944: [Python] compat with pandas < 0.19.0

Author: Jeff Reback <jeff.reback@twosigma.com>

Closes #649 from jreback/pandas_compat2 and squashes the following commits:

91fd9fb [Jeff Reback] ARROW-956: [Python] compat with pandas >= 0.20.0
025639c [Jeff Reback] ARROW-944: [Python] compat with pandas < 0.19.0
---
 python/pyarrow/compat.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 928a2c0724298..7be35dfc2c81f 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -34,19 +34,14 @@
     import pandas as pd
     pdver = LooseVersion(pd.__version__)
     if pdver >= '0.20.0':
-        try:
-            from pandas.api.types import DatetimeTZDtype
-        except AttributeError:
-            # can be removed once 0.20.0 is released
-            from pandas.core.dtypes.dtypes import DatetimeTZDtype
-
+        from pandas.api.types import DatetimeTZDtype
         pdapi = pd.api.types
-    elif pdver < '0.19.0':
-        from pandas.core.dtypes import DatetimeTZDtype
-        pdapi = pd.core.common
-    else:
+    elif pdver >= '0.19.0':
         from pandas.types.dtypes import DatetimeTZDtype
         pdapi = pd.api.types
+    else:
+        from pandas.types.dtypes import DatetimeTZDtype
+        pdapi = pd.core.common
 
     PandasSeries = pd.Series
     Categorical = pd.Categorical

From 959ec47b3e2828a44088d054bbdc5eabac9113e5 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 6 May 2017 17:54:31 -0400
Subject: [PATCH 0613/1644] ARROW-856: Also read compiler info from stdout

As I cannot reproduce the problem, this may not the be the correct solution. Still due to the output of `gcc -v` in the ticket, it is very likely that some systems print the info on stdout. Also the additional message should improve debugging.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #650 from xhochy/ARROW-856 and squashes the following commits:

d87c7bd [Uwe L. Korn] ARROW-856: Also read compiler info from stdout
---
 cpp/cmake_modules/CompilerInfo.cmake | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index 21e2dafba2e24..4a18376df59f4 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -21,7 +21,11 @@ if (NOT MSVC)
   set(COMPILER_GET_VERSION_SWITCH "-v")
 endif()
 
+message(INFO "Compiler command: ${CMAKE_CXX_COMPILER}")
+# Some gcc seem to output their version on stdout, most do it on stderr, simply
+# merge both pipes into a single variable
 execute_process(COMMAND "${CMAKE_CXX_COMPILER}" ${COMPILER_GET_VERSION_SWITCH}
+                OUTPUT_VARIABLE COMPILER_VERSION_FULL
                 ERROR_VARIABLE COMPILER_VERSION_FULL)
 message(INFO "Compiler version: ${COMPILER_VERSION_FULL}")
 message(INFO "Compiler id: ${CMAKE_CXX_COMPILER_ID}")

From bd36f6f590e3f5ebe3ad8ed2cc81b988272c9215 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 6 May 2017 17:57:29 -0400
Subject: [PATCH 0614/1644] ARROW-899: [Doc] Add 0.3.0 changelog

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #652 from wesm/ARROW-899 and squashes the following commits:

c3af6b5 [Wes McKinney] Remove asterisks causing weird Markdown formatting
b1e707c [Wes McKinney] Add 0.3.0 changelog
---
 CHANGELOG.md | 307 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 307 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d54838e1a7f0..85a43ef7952d9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,313 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
+# Apache Arrow 0.3.0 (5 May 2017)
+
+## Bug
+
+* ARROW-109 - [C++] Investigate recursive data types limit in flatbuffers
+* ARROW-208 - Add checkstyle policy to java project
+* ARROW-347 - Add method to pass CallBack when creating a transfer pair
+* ARROW-413 - DATE type is not specified clearly
+* ARROW-431 - [Python] Review GIL release and acquisition in to_pandas conversion
+* ARROW-443 - [Python] Support for converting from strided pandas data in Table.from_pandas
+* ARROW-451 - [C++] Override DataType::Equals for other types with additional metadata
+* ARROW-454 - pojo.Field doesn't implement hashCode()
+* ARROW-526 - [Format] Update IPC.md to account for File format changes and Streaming format
+* ARROW-565 - [C++] Examine "Field::dictionary" member
+* ARROW-570 - Determine Java tools JAR location from project metadata
+* ARROW-584 - [C++] Fix compiler warnings exposed with -Wconversion
+* ARROW-588 - [C++] Fix compiler warnings on 32-bit platforms
+* ARROW-595 - [Python] StreamReader.schema returns None
+* ARROW-604 - Python: boxed Field instances are missing the reference to DataType
+* ARROW-613 - [JS] Implement random-access file format
+* ARROW-617 - Time type is not specified clearly
+* ARROW-619 - Python: Fix typos in setup.py args and LD_LIBRARY_PATH
+* ARROW-623 - segfault with __repr__ of empty Field
+* ARROW-624 - [C++] Restore MakePrimitiveArray function
+* ARROW-627 - [C++] Compatibility macros for exported extern template class declarations
+* ARROW-628 - [Python] Install nomkl metapackage when building parquet-cpp for faster Travis builds
+* ARROW-630 - [C++] IPC unloading for BooleanArray does not account for offset
+* ARROW-636 - [C++] Add Boost / other system requirements to C++ README
+* ARROW-639 - [C++] Invalid offset in slices
+* ARROW-642 - [Java] Remove temporary file in java/tools
+* ARROW-644 - Python: Cython should be a setup-only requirement
+* ARROW-652 - Remove trailing f in merge script output
+* ARROW-654 - [C++] Support timezone metadata in file/stream formats
+* ARROW-668 - [Python] Convert nanosecond timestamps to pandas.Timestamp when converting from TimestampValue
+* ARROW-671 - [GLib] License file isn't installed
+* ARROW-673 - [Java] Support additional Time metadata
+* ARROW-677 - [java] Fix checkstyle jcl-over-slf4j conflict issue
+* ARROW-678 - [GLib] Fix dependenciesfff
+* ARROW-680 - [C++] Multiarch support impacts user-supplied install prefix
+* ARROW-682 - Add self-validation checks in integration tests
+* ARROW-683 - [C++] Support date32 (DateUnit::DAY) in IPC metadata, rename date to date64
+* ARROW-686 - [C++] Account for time metadata changes, add time32 and time64 types
+* ARROW-689 - [GLib] Install header files and documents to wrong directories
+* ARROW-691 - [Java] Encode dictionary Int type in message format
+* ARROW-697 - [Java] Raise appropriate exceptions when encountering large (> INT32_MAX) record batches
+* ARROW-699 - [C++] Arrow dynamic libraries are missed on run of unit tests on Windows
+* ARROW-702 - Fix BitVector.copyFromSafe to reAllocate instead of returning false
+* ARROW-703 - Fix issue where setValueCount(0) doesn’t work in the case that we’ve shipped vectors across the wire
+* ARROW-704 - Fix bad import caused by conflicting changes
+* ARROW-709 - [C++] Restore type comparator for DecimalType
+* ARROW-713 - [C++] Fix linking issue with ipc benchmark
+* ARROW-715 - Python: Explicit pandas import makes it a hard requirement
+* ARROW-716 - error building arrow/python
+* ARROW-720 - [java] arrow should not have a dependency on slf4j bridges in compile
+* ARROW-723 - Arrow freezes on write if chunk_size=0
+* ARROW-726 - [C++] PyBuffer dtor may segfault if constructor passed an object not exporting buffer protocol
+* ARROW-732 - Schema comparison bugs in struct and union types
+* ARROW-736 - [Python] Mixed-type object DataFrame columns should not silently coerce to an Arrow type by default
+* ARROW-738 - [Python] Fix manylinux1 packaging
+* ARROW-739 - Parallel build fails non-deterministically.
+* ARROW-740 - FileReader fails for large objects
+* ARROW-747 - [C++] Fix spurious warning caused by passing dl to add_dependencies
+* ARROW-749 - [Python] Delete incomplete binary files when writing fails
+* ARROW-753 - [Python] Unit tests in arrow/python fail to link on some OS X platforms
+* ARROW-756 - [C++] Do not pass -fPIC when compiling with MSVC
+* ARROW-757 - [C++] MSVC build fails on googletest when using NMake
+* ARROW-762 - Kerberos Problem with PyArrow
+* ARROW-776 - [GLib] Cast type is wrong
+* ARROW-777 - [Java] Resolve getObject behavior per changes / discussion in ARROW-729
+* ARROW-778 - Modify merge tool to work on Windows
+* ARROW-781 - [Python/C++] Increase reference count for base object?
+* ARROW-783 - Integration tests fail for length-0 record batch
+* ARROW-787 - [GLib] Fix compilation errors caused by ARROW-758
+* ARROW-793 - [GLib] Wrong indent
+* ARROW-794 - [C++] Check whether data is contiguous in ipc::WriteTensor
+* ARROW-797 - [Python] Add updated pyarrow. public API listing in Sphinx docs
+* ARROW-800 - [C++] Boost headers being transitively included in pyarrow
+* ARROW-805 - listing empty HDFS directory returns an error instead of returning empty list
+* ARROW-809 - C++: Writing sliced record batch to IPC writes the entire array
+* ARROW-812 - Pip install pyarrow on mac failed.
+* ARROW-817 - [C++] Fix incorrect code comment from ARROW-722
+* ARROW-821 - [Python] Extra file _table_api.h generated during Python build process
+* ARROW-822 - [Python] StreamWriter fails to open with socket as sink
+* ARROW-826 - Compilation error on Mac with -DARROW_PYTHON=on
+* ARROW-829 - Python: Parquet: Dictionary encoding is deactivated if column-wise compression was selected
+* ARROW-830 - Python: jemalloc is not anymore publicly exposed
+* ARROW-839 - [C++] Portable alternative to PyDate_to_ms function
+* ARROW-847 - C++: BUILD_BYPRODUCTS not specified anymore for gtest
+* ARROW-852 - Python: Also set Arrow Library PATHS when detection was done through pkg-config
+* ARROW-853 - [Python] It is no longer necessary to modify the RPATH of the Cython extensions on many environments
+* ARROW-858 - Remove dependency on boost regex
+* ARROW-866 - [Python] Error from file object destructor
+* ARROW-867 - [Python] Miscellaneous pyarrow MSVC fixes
+* ARROW-875 - Nullable variable length vector fillEmpties() fills an extra value
+* ARROW-879 - compat with pandas 0.20.0
+* ARROW-882 - [C++] On Windows statically built lib file overwrites lib file of shared build
+* ARROW-886 - VariableLengthVectors don't reAlloc offsets
+* ARROW-887 - [format] For backward compatibility, new unit fields must have default values matching previous implied unit
+* ARROW-888 - BitVector transfer() does not transfer ownership
+* ARROW-895 - Nullable variable length vector lastSet not set correctly
+* ARROW-900 - [Python] UnboundLocalError in ParquetDatasetPiece
+* ARROW-903 - [GLib] Remove a needless "."
+* ARROW-914 - [C++/Python] Fix Decimal ToBytes
+* ARROW-922 - Allow Flatbuffers and RapidJSON to be used locally on Windows
+* ARROW-928 - Update CMAKE script to detect unsupported msvc compilers versions
+* ARROW-933 - [Python] arrow_python bindings have debug print statement
+* ARROW-934 - [GLib] Glib sources missing from result of 02-source.sh
+* ARROW-936 - Fix release README
+* ARROW-938 - Fix Apache Rat errors from source release build
+
+## Improvement
+
+* ARROW-316 - Finalize Date type
+* ARROW-542 - [Java] Implement dictionaries in stream/file encoding
+* ARROW-563 - C++: Support non-standard gcc version strings
+* ARROW-566 - Python: Deterministic position of libarrow in manylinux1 wheels
+* ARROW-569 - [C++] Set version for .pc
+* ARROW-577 - [C++] Refactor StreamWriter and FileWriter to have private implementations
+* ARROW-580 - C++: Also provide jemalloc_X targets if only a static or shared version is found
+* ARROW-582 - [Java] Add Date/Time Support to JSON File
+* ARROW-589 - C++: Use system provided shared jemalloc if static is unavailable
+* ARROW-593 - [C++] Rename ReadableFileInterface to RandomAccessFile
+* ARROW-612 - [Java] Field toString should show nullable flag status
+* ARROW-615 - Move ByteArrayReadableSeekableByteChannel to vector.util package
+* ARROW-631 - [GLib] Import C API (C++ API wrapper) based on GLib from https://github.com/kou/arrow-glib
+* ARROW-646 - Cache miniconda packages
+* ARROW-647 - [C++] Don't require Boost static libraries to support CentOS 7
+* ARROW-648 - [C++] Support multiarch on Debian
+* ARROW-650 - [GLib] Follow eadableFileInterface -> RnadomAccessFile change
+* ARROW-651 - [C++] Set shared library version for .deb packages
+* ARROW-655 - Implement DecimalArray
+* ARROW-662 - [Format] Factor Flatbuffer schema metadata into a Schema.fbs
+* ARROW-664 - Make C++ Arrow serialization deterministic
+* ARROW-674 - [Java] Support additional Timestamp timezone metadata
+* ARROW-675 - [GLib] Update package metadata
+* ARROW-676 - [java] move from MinorType to FieldType in ValueVectors to carry all the relevant type bits
+* ARROW-679 - [Format] Change RecordBatch and Field length members from int to long
+* ARROW-681 - [C++] Build Arrow on Windows with dynamically linked boost
+* ARROW-684 - Python: More informative message when parquet-cpp but not parquet-arrow is available
+* ARROW-688 - [C++] Use CMAKE_INSTALL_INCLUDEDIR for consistency
+* ARROW-690 - Only send JIRA updates to issues@arrow.apache.org
+* ARROW-700 - Add headroom interface for allocator.
+* ARROW-706 - [GLib] Add package install document
+* ARROW-707 - Python: All none-Pandas column should be converted to NullArray
+* ARROW-708 - [C++] Some IPC code simplification, perf analysis
+* ARROW-712 - [C++] Implement Array::Accept as inline visitor
+* ARROW-719 - [GLib] Support prepared source archive release
+* ARROW-724 - Add "How to Contribute" section to README
+* ARROW-725 - [Format] Constant length list type
+* ARROW-727 - [Python] Write memoryview-compatible objects in NativeFile.write with zero copy
+* ARROW-728 - [C++/Python] Add arrow::Table function for removing a column
+* ARROW-731 - [C++] Add shared library related versions to .pc
+* ARROW-741 - [Python] Add Python 3.6 to Travis CI
+* ARROW-743 - [C++] Consolidate unit tests for code in array.h
+* ARROW-744 - [GLib] Re-add an assertion to garrow_table_new() test
+* ARROW-745 - [C++] Allow use of system cpplint
+* ARROW-746 - [GLib] Add garrow_array_get_data_type()
+* ARROW-751 - [Python] Rename all Cython extensions to "private" status with leading underscore
+* ARROW-752 - [Python] Construct pyarrow.DictionaryArray from boxed pyarrow array objects
+* ARROW-754 - [GLib] Add garrow_array_is_null()
+* ARROW-755 - [GLib] Add garrow_array_get_value_type()
+* ARROW-758 - [C++] Fix compiler warnings on MSVC x64
+* ARROW-761 - [Python] Add function to compute the total size of tensor payloads, including metadata and padding
+* ARROW-763 - C++: Use `python-config` to find libpythonX.X.dylib
+* ARROW-765 - [Python] Make generic ArrowException subclass value error
+* ARROW-769 - [GLib] Support building without installed Arrow C++
+* ARROW-770 - [C++] Move clang-tidy/format config files back to C++ source tree
+* ARROW-774 - [GLib] Remove needless LICENSE.txt copy
+* ARROW-775 - [Java] add simple constructors to value vectors
+* ARROW-779 - [C++/Python] Raise exception if old metadata encountered
+* ARROW-782 - [C++] Change struct to class for objects that meet the criteria in the Google style guide
+* ARROW-788 - Possible nondeterminism in Tensor serialization code
+* ARROW-795 - [C++] Combine libarrow/libarrow_io/libarrow_ipc
+* ARROW-802 - [GLib] Add read examples
+* ARROW-803 - [GLib] Update package repository URL
+* ARROW-804 - [GLib] Update build document
+* ARROW-806 - [GLib] Support add/remove a column from table
+* ARROW-807 - [GLib] Update "Since" tag
+* ARROW-808 - [GLib] Remove needless ignore entries
+* ARROW-810 - [GLib] Remove io/ipc prefix
+* ARROW-811 - [GLib] Add GArrowBuffer
+* ARROW-815 - [Java] Allow for expanding underlying buffer size after allocation
+* ARROW-816 - [C++] Use conda packages for RapidJSON, Flatbuffers to speed up builds
+* ARROW-818 - [Python] Review public pyarrow. API completeness and update docs
+* ARROW-820 - [C++] Build dependencies for Parquet library without arrow support
+* ARROW-825 - [Python] Generalize pyarrow.from_pylist to accept any object implementing the PySequence protocol
+* ARROW-827 - [Python] Variety of Parquet improvements to support Dask integration
+* ARROW-828 - [CPP] Document new requirement (libboost-regex-dev) in README.md
+* ARROW-832 - [C++] Upgrade thirdparty gtest to 1.8.0
+* ARROW-833 - [Python] "Quickstart" build / environment setup guide for Python developers
+* ARROW-841 - [Python] Add pyarrow build to Appveyor
+* ARROW-844 - [Format] Revise format/README.md to reflect progress reaching a more complete specification
+* ARROW-845 - [Python] Sync FindArrow.cmake changes from parquet-cpp
+* ARROW-846 - [GLib] Add GArrowTensor, GArrowInt8Tensor and GArrowUInt8Tensor
+* ARROW-848 - [Python] Improvements / fixes to conda quickstart guide
+* ARROW-849 - [C++] Add optional $ARROW_BUILD_TOOLCHAIN environment variable option for configuring build environment
+* ARROW-857 - [Python] Automate publishing Python documentation to arrow-site
+* ARROW-860 - [C++] Decide if typed Tensor subclasses are worthwhile
+* ARROW-861 - [Python] Move DEVELOPMENT.md to Sphinx docs
+* ARROW-862 - [Python] Improve source build instructions in README
+* ARROW-863 - [GLib] Use GBytes to implement zero-copy
+* ARROW-864 - [GLib] Unify Array files
+* ARROW-868 - [GLib] Use GBytes to reduce copy
+* ARROW-871 - [GLib] Unify DataType files
+* ARROW-876 - [GLib] Unify ArrayBuffer files
+* ARROW-877 - [GLib] Add garrow_array_get_null_bitmap()
+* ARROW-878 - [GLib] Add garrow_binary_array_get_buffer()
+* ARROW-892 - [GLib] Fix GArrowTensor document
+* ARROW-893 - Add GLib document to Web site
+* ARROW-894 - [GLib] Add GArrowPoolBuffer
+* ARROW-896 - [Docs] Add Jekyll plugin for including rendered Jupyter notebooks on website
+* ARROW-898 - [C++] Expand metadata support to field level, provide for sharing instances of KeyValueMetadata
+* ARROW-904 - [GLib] Simplify error check codes
+* ARROW-907 - C++: Convenience construct Table from schema and arrays
+* ARROW-908 - [GLib] Unify OutputStream files
+* ARROW-910 - [C++] Write 0-length EOS indicator at end of stream
+* ARROW-916 - [GLib] Add GArrowBufferOutputStream
+* ARROW-917 - [GLib] Add GArrowBufferReader
+* ARROW-918 - [GLib] Use GArrowBuffer for read
+* ARROW-919 - [GLib] Use "id" to get type enum value from GArrowDataType
+* ARROW-920 - [GLib] Add Lua examples
+* ARROW-925 - [GLib] Fix GArrowBufferReader test
+* ARROW-930 - javadoc generation fails with java 8
+* ARROW-931 - [GLib] Reconstruct input stream
+
+## New Feature
+
+* ARROW-231 - C++: Add typed Resize to PoolBuffer
+* ARROW-281 - [C++] IPC/RPC support on Win32 platforms
+* ARROW-341 - [Python] Making libpyarrow available to third parties
+* ARROW-452 - [C++/Python] Merge "Feather" file format implementation
+* ARROW-459 - [C++] Implement IPC round trip for DictionaryArray, dictionaries shared across record batches
+* ARROW-483 - [C++/Python] Provide access to "custom_metadata" Field attribute in IPC setting
+* ARROW-491 - [C++] Add FixedWidthBinary type
+* ARROW-493 - [C++] Allow in-memory array over 2^31 -1 elements but require splitting at IPC / RPC boundaries
+* ARROW-502 - [C++/Python] Add MemoryPool implementation that logs allocation activity to std::cout
+* ARROW-510 - Add integration tests for date and time types
+* ARROW-520 - [C++] Add STL-compliant allocator that hooks into an arrow::MemoryPool
+* ARROW-528 - [Python] Support _metadata or _common_metadata files when reading Parquet directories
+* ARROW-534 - [C++] Add IPC tests for date/time types
+* ARROW-539 - [Python] Support reading Parquet datasets with standard partition directory schemes
+* ARROW-550 - [Format] Add a TensorMessage type
+* ARROW-552 - [Python] Add scalar value support for Dictionary type
+* ARROW-557 - [Python] Explicitly opt in to HDFS unit tests
+* ARROW-568 - [C++] Add default implementations for TypeVisitor, ArrayVisitor methods that return NotImplemented
+* ARROW-574 - Python: Add support for nested Python lists in Pandas conversion
+* ARROW-576 - [C++] Complete round trip Union file/stream IPC tests
+* ARROW-578 - [C++] Add CMake option to add custom $CXXFLAGS
+* ARROW-598 - [Python] Add support for converting pyarrow.Buffer to a memoryview with zero copy
+* ARROW-603 - [C++] Add RecordBatch::Validate method that at least checks that schema matches the array metadata
+* ARROW-605 - [C++] Refactor generic ArrayLoader class, support work for Feather merge
+* ARROW-606 - [C++] Upgrade to flatbuffers 1.6.0
+* ARROW-608 - [Format] Days since epoch date type
+* ARROW-610 - [C++] Win32 compatibility in file.cc
+* ARROW-616 - [C++] Remove -g flag in release builds
+* ARROW-618 - [Python] Implement support for DatetimeTZ custom type from pandas
+* ARROW-620 - [C++] Add date/time support to JSON reader/writer for integration testing
+* ARROW-621 - [C++] Implement an "inline visitor" template that enables visitor-pattern-like code without virtual function dispatch
+* ARROW-625 - [C++] Add time unit to TimeType::ToString
+* ARROW-626 - [Python] Enable pyarrow.BufferReader to read from any Python object implementing the buffer/memoryview protocol
+* ARROW-632 - [Python] Add support for FixedWidthBinary type
+* ARROW-635 - [C++] Add JSON read/write support for FixedWidthBinary
+* ARROW-637 - [Format] Add time zone metadata to Timestamp type
+* ARROW-656 - [C++] Implement IO interface that can read and write to a fixed-size mutable buffer
+* ARROW-657 - [Python] Write and read tensors (with zero copy) into shared memory
+* ARROW-658 - [C++] Implement in-memory arrow::Tensor objects
+* ARROW-659 - [C++] Add multithreaded memcpy implementation (for hardware where it helps)
+* ARROW-660 - [C++] Restore function that can read a complete encapsulated record batch message
+* ARROW-661 - [C++] Add a Flatbuffer metadata type that supports array data over 2^31 - 1 elements
+* ARROW-663 - [Java] Support additional Time metadata + vector value accessors
+* ARROW-669 - [Python] Attach proper tzinfo when computing boxed scalars for TimestampArray
+* ARROW-687 - [C++] Build and run full test suite in Appveyor
+* ARROW-698 - [C++] Add options to StreamWriter/FileWriter to permit large record batches
+* ARROW-701 - [Java] Support additional Date metadata
+* ARROW-710 - [Python] Enable Feather APIs to read and write using Python file-like objects
+* ARROW-717 - [C++] IPC zero-copy round trips for arrow::Tensor
+* ARROW-718 - [Python] Expose arrow::Tensor with conversions to/from NumPy arrays
+* ARROW-722 - [Python] pandas conversions for new date and time types/metadata
+* ARROW-729 - [Java] Add vector type for 32-bit date as days since UNIX epoch
+* ARROW-733 - [C++/Format] Change name of Fixed Width Binary to Fixed Size Binary for consistency
+* ARROW-734 - [Python] Support for pyarrow on Windows / MSVC
+* ARROW-735 - [C++] Developer instruction document for MSVC on Windows
+* ARROW-737 - [C++] Support obtaining mutable slices of mutable buffers
+* ARROW-768 - [Java] Change the "boxed" object representation of date and time types
+* ARROW-771 - [Python] Add APIs for reading individual Parquet row groups
+* ARROW-773 - [C++] Add function to create arrow::Table with column appended to existing table
+* ARROW-865 - [Python] Verify Parquet roundtrips for new date/time types
+* ARROW-880 - [GLib] Add garrow_primitive_array_get_buffer()
+* ARROW-890 - [GLib] Add GArrowMutableBuffer
+* ARROW-926 - Update KEYS to include wesm
+
+## Task
+
+* ARROW-52 - Set up project blog
+* ARROW-670 - Arrow 0.3 release
+* ARROW-672 - [Format] Bump metadata version for 0.3 release
+* ARROW-748 - [Python] Pin runtime library versions in conda-forge packages to force upgrades
+* ARROW-798 - [Docs] Publish Format Markdown documents somehow on arrow.apache.org
+* ARROW-869 - [JS] Rename directory to js/
+* ARROW-95 - Scaffold Main Documentation using asciidoc
+* ARROW-98 - Java: API documentation
+
+## Test
+
+* ARROW-836 - Test for timedelta compat with pandas
+* ARROW-927 - C++/Python: Add manylinux1 builds to Travis matrix
+
 # Apache Arrow 0.2.0 (15 February 2017)
 
 ## Bug

From 03c242c7c614a4ca2089ea378796c38ae1d9fa3a Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 6 May 2017 17:58:10 -0400
Subject: [PATCH 0615/1644] ARROW-947: [Python] Improve execution time of
 manylinux1 build

Got it down to 14min: https://travis-ci.org/xhochy/arrow/builds/229461571

We can probably squeeze out 1-2 minutes more but that won't be easy. Build times are probably longer here as we build in release mode.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #648 from xhochy/ARROW-947 and squashes the following commits:

ebe7507 [Uwe L. Korn] Use this branch as docker tag
3203b22 [Uwe L. Korn] Move manylinux commands to script
a6397da [Uwe L. Korn] Reset docker tag to latest
f2b5909 [Uwe L. Korn] Explicit set e
352215f [Uwe L. Korn] Pre-build virtualenvs
c1a1c89 [Uwe L. Korn] Pre-install python packages
0e83522 [Uwe L. Korn] Fix image tags
60a0c8c [Uwe L. Korn] Add ccache to the image
3426195 [Uwe L. Korn] Add brotli
dc0802b [Uwe L. Korn] Add more dependencies
9cadeb5 [Uwe L. Korn] Move gtest and flatbuffers to scripts
---
 .travis.yml                                   | 10 ++---
 ci/travis_script_manylinux.sh                 | 21 ++++++++++
 python/manylinux1/Dockerfile-x86_64           |  2 +-
 python/manylinux1/Dockerfile-x86_64_base      | 41 +++++++++++--------
 python/manylinux1/build_arrow.sh              | 17 +++-----
 python/manylinux1/scripts/build_brotli.sh     | 30 ++++++++++++++
 python/manylinux1/scripts/build_ccache.sh     | 21 ++++++++++
 .../manylinux1/scripts/build_flatbuffers.sh   | 21 ++++++++++
 python/manylinux1/scripts/build_gtest.sh      | 21 ++++++++++
 python/manylinux1/scripts/build_snappy.sh     | 22 ++++++++++
 python/manylinux1/scripts/build_thrift.sh     | 37 +++++++++++++++++
 .../manylinux1/scripts/build_virtualenvs.sh   | 41 +++++++++++++++++++
 12 files changed, 248 insertions(+), 36 deletions(-)
 create mode 100755 ci/travis_script_manylinux.sh
 create mode 100755 python/manylinux1/scripts/build_brotli.sh
 create mode 100755 python/manylinux1/scripts/build_ccache.sh
 create mode 100755 python/manylinux1/scripts/build_flatbuffers.sh
 create mode 100755 python/manylinux1/scripts/build_gtest.sh
 create mode 100755 python/manylinux1/scripts/build_snappy.sh
 create mode 100755 python/manylinux1/scripts/build_thrift.sh
 create mode 100755 python/manylinux1/scripts/build_virtualenvs.sh

diff --git a/.travis.yml b/.travis.yml
index d821b5accb973..e6941620c2c91 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -56,13 +56,9 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
   - language: cpp
     before_script:
-    - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-927
-    script: |
-        pushd python/manylinux1
-        git clone ../../ arrow
-        docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
-        docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
-        ls -l dist/
+    - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
   - language: java
     os: linux
     jdk: oraclejdk7
diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh
new file mode 100755
index 0000000000000..69feb685b5136
--- /dev/null
+++ b/ci/travis_script_manylinux.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+
+set -ex
+
+pushd python/manylinux1
+git clone ../../ arrow
+docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
+docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 8f55ba7e1deed..08fecb0da9276 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -9,7 +9,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License. See accompanying LICENSE file.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-927
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-947
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index e38296d78de1c..2ae7e0ff4f98f 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -28,25 +28,34 @@ WORKDIR /
 RUN /opt/python/cp35-cp35m/bin/pip install cmake
 RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
 
+ADD scripts/build_gtest.sh /
+RUN /build_gtest.sh
+ENV GTEST_HOME /googletest-release-1.7.0
+
+ADD scripts/build_flatbuffers.sh /
+RUN /build_flatbuffers.sh
+ENV FLATBUFFERS_HOME /usr
+
+ADD scripts/build_thrift.sh /
+RUN /build_thrift.sh
+ENV THRIFT_HOME /usr
+
+ADD scripts/build_brotli.sh /
+RUN /build_brotli.sh
+ENV BROTLI_HOME /usr
+
+ADD scripts/build_snappy.sh /
+RUN /build_snappy.sh
+ENV SNAPPY_HOME /usr
+
+ADD scripts/build_ccache.sh /
+RUN /build_ccache.sh
+
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git
 WORKDIR /multibuild
 RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
-
-WORKDIR /
-RUN wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -O googletest-release-1.7.0.tar.gz
-RUN tar xf googletest-release-1.7.0.tar.gz
-WORKDIR /googletest-release-1.7.0
-RUN cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON .
-RUN make -j5
-ENV GTEST_HOME /googletest-release-1.7.0
-
 WORKDIR /
-RUN wget https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -O flatbuffers-1.6.0.tar.gz
-RUN tar xf flatbuffers-1.6.0.tar.gz
-WORKDIR /flatbuffers-1.6.0
-RUN cmake "-DCMAKE_CXX_FLAGS=-fPIC" "-DCMAKE_INSTALL_PREFIX:PATH=/usr" "-DFLATBUFFERS_BUILD_TESTS=OFF"
-RUN make -j5
-RUN make install
-ENV FLATBUFFERS_HOME /usr
 
+ADD scripts/build_virtualenvs.sh /
+RUN /build_virtualenvs.sh
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index a11d3d41f49f7..e0727495cff4a 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -27,6 +27,9 @@ MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
 
 source /multibuild/manylinux_utils.sh
 
+# Quit on failure
+set -e
+
 cd /arrow/python
 
 # PyArrow build configuration
@@ -48,11 +51,6 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     PIPI_IO="$PIP install -f $MANYLINUX_URL"
     PATH="$PATH:$(cpython_path $PYTHON)"
 
-    echo "=== (${PYTHON}) Installing build dependencies ==="
-    $PIPI_IO "numpy==1.9.0"
-    $PIPI_IO "cython==0.25.2"
-    $PIPI_IO "pandas==0.19.2"
-
     echo "=== (${PYTHON}) Building Arrow C++ libraries ==="
     ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
     mkdir -p "${ARROW_BUILD_DIR}"
@@ -77,14 +75,9 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/
 
     echo "=== (${PYTHON}) Testing manylinux1 wheel ==="
-    # Fix version to keep build reproducible"
-    $PIPI_IO "virtualenv==15.1.0"
-    rm -rf venv
-    "$(cpython_path $PYTHON)/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download venv
-    source ./venv/bin/activate
+    source /venv-test-${PYTHON}/bin/activate
     pip install repaired_wheels/*.whl
-    pip install pytest pandas
-    py.test venv/lib/*/site-packages/pyarrow
+    py.test /venv-test-${PYTHON}/lib/*/site-packages/pyarrow
     deactivate
 
     mv repaired_wheels/*.whl /io/dist
diff --git a/python/manylinux1/scripts/build_brotli.sh b/python/manylinux1/scripts/build_brotli.sh
new file mode 100755
index 0000000000000..4b4cbf17ca9bf
--- /dev/null
+++ b/python/manylinux1/scripts/build_brotli.sh
@@ -0,0 +1,30 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+export BROTLI_VERSION="0.6.0"
+wget "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -O brotli-${BROTLI_VERSION}.tar.gz
+tar xf brotli-${BROTLI_VERSION}.tar.gz
+pushd brotli-${BROTLI_VERSION}
+mkdir build
+pushd build
+cmake -DCMAKE_BUILD_TYPE=release \
+    "-DCMAKE_CXX_FLAGS=-fPIC" \
+    "-DCMAKE_C_FLAGS=-fPIC" \
+    -DCMAKE_INSTALL_PREFIX=/usr \
+    -DBUILD_SHARED_LIBS=OFF \
+    ..
+make -j5
+make install
+popd
+popd
+rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION}
diff --git a/python/manylinux1/scripts/build_ccache.sh b/python/manylinux1/scripts/build_ccache.sh
new file mode 100755
index 0000000000000..6ad5d29f83292
--- /dev/null
+++ b/python/manylinux1/scripts/build_ccache.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+wget https://www.samba.org/ftp/ccache/ccache-3.3.4.tar.bz2 -O ccache-3.3.4.tar.bz2
+tar xf ccache-3.3.4.tar.bz2
+pushd ccache-3.3.4
+./configure --prefix=/usr
+make -j5
+make install
+popd
+rm -rf ccache-3.3.4.tar.bz2 ccache-3.3.4
diff --git a/python/manylinux1/scripts/build_flatbuffers.sh b/python/manylinux1/scripts/build_flatbuffers.sh
new file mode 100755
index 0000000000000..7703855b6efbf
--- /dev/null
+++ b/python/manylinux1/scripts/build_flatbuffers.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+wget https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -O flatbuffers-1.6.0.tar.gz
+tar xf flatbuffers-1.6.0.tar.gz
+pushd flatbuffers-1.6.0
+cmake "-DCMAKE_CXX_FLAGS=-fPIC" "-DCMAKE_INSTALL_PREFIX:PATH=/usr" "-DFLATBUFFERS_BUILD_TESTS=OFF"
+make -j5
+make install
+popd
+rm -rf flatbuffers-1.6.0.tar.gz flatbuffers-1.6.0
diff --git a/python/manylinux1/scripts/build_gtest.sh b/python/manylinux1/scripts/build_gtest.sh
new file mode 100755
index 0000000000000..3427bed091ed3
--- /dev/null
+++ b/python/manylinux1/scripts/build_gtest.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -O googletest-release-1.7.0.tar.gz
+tar xf googletest-release-1.7.0.tar.gz
+ls -l
+pushd googletest-release-1.7.0
+cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON .
+make -j5
+popd
+rm -rf googletest-release-1.7.0.tar.gz
diff --git a/python/manylinux1/scripts/build_snappy.sh b/python/manylinux1/scripts/build_snappy.sh
new file mode 100755
index 0000000000000..973b4ff7d8089
--- /dev/null
+++ b/python/manylinux1/scripts/build_snappy.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+export SNAPPY_VERSION="1.1.3"
+wget "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz" -O snappy-${SNAPPY_VERSION}.tar.gz
+tar xf snappy-${SNAPPY_VERSION}.tar.gz
+pushd snappy-${SNAPPY_VERSION}
+./configure --with-pic "--prefix=/usr" CXXFLAGS='-DNDEBUG -O2'
+make -j5
+make install
+popd
+rm -rf snappy-${SNAPPY_VERSION}.tar.gz snappy-${SNAPPY_VERSION}
diff --git a/python/manylinux1/scripts/build_thrift.sh b/python/manylinux1/scripts/build_thrift.sh
new file mode 100755
index 0000000000000..1db745855489f
--- /dev/null
+++ b/python/manylinux1/scripts/build_thrift.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+export THRIFT_VERSION=0.10.0
+wget http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz
+tar xf thrift-${THRIFT_VERSION}.tar.gz
+pushd thrift-${THRIFT_VERSION}
+mkdir build-tmp
+pushd build-tmp
+cmake -DCMAKE_BUILD_TYPE=release \
+    "-DCMAKE_CXX_FLAGS=-fPIC" \
+    "-DCMAKE_C_FLAGS=-fPIC" \
+    "-DCMAKE_INSTALL_PREFIX=/usr" \
+    "-DCMAKE_INSTALL_RPATH=/usr/lib" \
+    "-DBUILD_SHARED_LIBS=OFF" \
+    "-DBUILD_TESTING=OFF" \
+    "-DWITH_QT4=OFF" \
+    "-DWITH_C_GLIB=OFF" \
+    "-DWITH_JAVA=OFF" \
+    "-DWITH_PYTHON=OFF" \
+    "-DWITH_CPP=ON" \
+    "-DWITH_STATIC_LIB=ON" ..
+make -j5
+make install
+popd
+popd
+rm -rf thrift-${THRIFT_VERSION}.tar.gz thrift-${THRIFT_VERSION}
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
new file mode 100755
index 0000000000000..ee8a82730281f
--- /dev/null
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -0,0 +1,41 @@
+#!/bin/bash -e
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
+# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
+
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5 3.6}"
+
+# Package index with only manylinux1 builds
+MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
+
+source /multibuild/manylinux_utils.sh
+
+for PYTHON in ${PYTHON_VERSIONS}; do
+    PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
+    PIP="$(cpython_path $PYTHON)/bin/pip"
+    PIPI_IO="$PIP install -f $MANYLINUX_URL"
+    PATH="$PATH:$(cpython_path $PYTHON)"
+
+    echo "=== (${PYTHON}) Installing build dependencies ==="
+    $PIPI_IO "numpy==1.9.0"
+    $PIPI_IO "cython==0.25.2"
+    $PIPI_IO "pandas==0.20.1"
+    $PIPI_IO "virtualenv==15.1.0"
+
+    echo "=== (${PYTHON}) Preparing virtualenv for tests ==="
+    "$(cpython_path $PYTHON)/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}
+    source /venv-test-${PYTHON}/bin/activate
+    pip install pytest 'numpy==1.12.1' 'pandas==0.20.1'
+    deactivate
+done

From 20228a2becc22fbbf72b8e5e9b3c875ac835c0af Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 6 May 2017 17:59:18 -0400
Subject: [PATCH 0616/1644] ARROW-909: Link jemalloc statically if build as
 external project

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #651 from xhochy/ARROW-909 and squashes the following commits:

a3e7a44 [Uwe L. Korn] ARROW-909: Link jemalloc statically if build as external project
---
 cpp/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5abe5f1436ea7..72e5ea90948b9 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -692,6 +692,7 @@ if (ARROW_JEMALLOC)
   find_package(jemalloc)
 
   if(NOT JEMALLOC_FOUND)
+    set(ARROW_JEMALLOC_USE_SHARED OFF)
     set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
     set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
     set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")

From c48f6493fa7301260fce709eb16ce5382bc4673e Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 7 May 2017 10:34:41 -0400
Subject: [PATCH 0617/1644] ARROW-963: [GLib] Add equal

Author: Kouhei Sutou <kou@clear-code.com>

Closes #654 from kou/glib-equal and squashes the following commits:

63f071d [Kouhei Sutou] [GLib] Add equal
---
 c_glib/arrow-glib/array.cpp         | 66 +++++++++++++++++++++++++++++
 c_glib/arrow-glib/array.h           | 10 +++++
 c_glib/arrow-glib/buffer.cpp        | 39 +++++++++++++++++
 c_glib/arrow-glib/buffer.h          |  5 +++
 c_glib/arrow-glib/chunked-array.cpp | 20 +++++++++
 c_glib/arrow-glib/chunked-array.h   |  3 ++
 c_glib/arrow-glib/column.cpp        | 18 ++++++++
 c_glib/arrow-glib/column.h          |  3 ++
 c_glib/arrow-glib/data-type.cpp     |  5 ++-
 c_glib/arrow-glib/field.cpp         |  5 ++-
 c_glib/arrow-glib/record-batch.cpp  | 20 +++++++++
 c_glib/arrow-glib/record-batch.h    |  3 ++
 c_glib/arrow-glib/schema.cpp        | 18 ++++++++
 c_glib/arrow-glib/schema.h          |  2 +
 c_glib/arrow-glib/table.cpp         | 18 ++++++++
 c_glib/arrow-glib/table.h           |  3 ++
 c_glib/arrow-glib/tensor.cpp        | 18 ++++++++
 c_glib/arrow-glib/tensor.h          |  2 +
 c_glib/test/test-array.rb           | 23 ++++++++++
 c_glib/test/test-buffer.rb          | 13 ++++++
 c_glib/test/test-chunked-array.rb   | 13 ++++++
 c_glib/test/test-column.rb          | 13 ++++++
 c_glib/test/test-field.rb           |  5 +++
 c_glib/test/test-record-batch.rb    | 15 +++++++
 c_glib/test/test-schema.rb          | 11 +++++
 c_glib/test/test-table.rb           | 14 ++++++
 c_glib/test/test-tensor.rb          | 13 ++++++
 27 files changed, 374 insertions(+), 4 deletions(-)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 3ca860d2ff6d3..8a78984349c62 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -188,6 +188,72 @@ garrow_array_class_init(GArrowArrayClass *klass)
   g_object_class_install_property(gobject_class, PROP_ARRAY, spec);
 }
 
+/**
+ * garrow_array_equal:
+ * @array: A #GArrowArray.
+ * @other_array: A #GArrowArray to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_array_equal(GArrowArray *array, GArrowArray *other_array)
+{
+  const auto arrow_array = garrow_array_get_raw(array);
+  const auto arrow_other_array = garrow_array_get_raw(other_array);
+  return arrow_array->Equals(arrow_other_array);
+}
+
+/**
+ * garrow_array_equal_approx:
+ * @array: A #GArrowArray.
+ * @other_array: A #GArrowArray to be compared.
+ *
+ * Returns: %TRUE if both of them have the approx same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_array_equal_approx(GArrowArray *array, GArrowArray *other_array)
+{
+  const auto arrow_array = garrow_array_get_raw(array);
+  const auto arrow_other_array = garrow_array_get_raw(other_array);
+  return arrow_array->ApproxEquals(arrow_other_array);
+}
+
+/**
+ * garrow_array_equal_range:
+ * @array: A #GArrowArray.
+ * @start_index: The start index of @array to be used.
+ * @other_array: A #GArrowArray to be compared.
+ * @other_start_index: The start index of @other_array to be used.
+ * @end_index: The end index of @array to be used. The end index of
+ *   @other_array is "@other_start_index + (@end_index -
+ *   @start_index)".
+ *
+ * Returns: %TRUE if both of them have the same data in the range,
+ *   %FALSE otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_array_equal_range(GArrowArray *array,
+                         gint64 start_index,
+                         GArrowArray *other_array,
+                         gint64 other_start_index,
+                         gint64 end_index)
+{
+  const auto arrow_array = garrow_array_get_raw(array);
+  const auto arrow_other_array = garrow_array_get_raw(other_array);
+  return arrow_array->RangeEquals(*arrow_other_array,
+                                  start_index,
+                                  end_index,
+                                  other_start_index);
+}
+
 /**
  * garrow_array_is_null:
  * @array: A #GArrowArray.
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 9bb502e4044a9..f750ee10f8cbe 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -58,6 +58,16 @@ struct _GArrowArrayClass
 
 GType          garrow_array_get_type    (void) G_GNUC_CONST;
 
+gboolean       garrow_array_equal       (GArrowArray *array,
+                                         GArrowArray *other_array);
+gboolean       garrow_array_equal_approx(GArrowArray *array,
+                                         GArrowArray *other_array);
+gboolean       garrow_array_equal_range (GArrowArray *array,
+                                         gint64 start_index,
+                                         GArrowArray *other_array,
+                                         gint64 other_start_index,
+                                         gint64 end_index);
+
 gboolean       garrow_array_is_null     (GArrowArray *array,
                                          gint64 i);
 gint64         garrow_array_get_length  (GArrowArray *array);
diff --git a/c_glib/arrow-glib/buffer.cpp b/c_glib/arrow-glib/buffer.cpp
index 4373ef1c83447..0970128ae3862 100644
--- a/c_glib/arrow-glib/buffer.cpp
+++ b/c_glib/arrow-glib/buffer.cpp
@@ -144,6 +144,45 @@ garrow_buffer_new(const guint8 *data, gint64 size)
 
 }
 
+/**
+ * garrow_buffer_equal:
+ * @buffer: A #GArrowBuffer.
+ * @other_buffer: A #GArrowBuffer to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_buffer_equal(GArrowBuffer *buffer, GArrowBuffer *other_buffer)
+{
+  const auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  const auto arrow_other_buffer = garrow_buffer_get_raw(other_buffer);
+  return arrow_buffer->Equals(*arrow_other_buffer);
+}
+
+/**
+ * garrow_buffer_equal_n_bytes:
+ * @buffer: A #GArrowBuffer.
+ * @other_buffer: A #GArrowBuffer to be compared.
+ * @n_bytes: The number of first bytes to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data in the first
+ *   `n_bytes`, %FALSE otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_buffer_equal_n_bytes(GArrowBuffer *buffer,
+                            GArrowBuffer *other_buffer,
+                            gint64 n_bytes)
+{
+  const auto arrow_buffer = garrow_buffer_get_raw(buffer);
+  const auto arrow_other_buffer = garrow_buffer_get_raw(other_buffer);
+  return arrow_buffer->Equals(*arrow_other_buffer, n_bytes);
+}
+
 /**
  * garrow_buffer_is_mutable:
  * @buffer: A #GArrowBuffer.
diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h
index 22a5e9bb2549a..b3f3a2cdc5e9b 100644
--- a/c_glib/arrow-glib/buffer.h
+++ b/c_glib/arrow-glib/buffer.h
@@ -59,6 +59,11 @@ GType          garrow_buffer_get_type     (void) G_GNUC_CONST;
 
 GArrowBuffer  *garrow_buffer_new          (const guint8 *data,
                                            gint64 size);
+gboolean       garrow_buffer_equal        (GArrowBuffer *buffer,
+                                           GArrowBuffer *other_buffer);
+gboolean       garrow_buffer_equal_n_bytes(GArrowBuffer *buffer,
+                                           GArrowBuffer *other_buffer,
+                                           gint64 n_bytes);
 gboolean       garrow_buffer_is_mutable   (GArrowBuffer *buffer);
 gint64         garrow_buffer_get_capacity (GArrowBuffer *buffer);
 GBytes        *garrow_buffer_get_data     (GArrowBuffer *buffer);
diff --git a/c_glib/arrow-glib/chunked-array.cpp b/c_glib/arrow-glib/chunked-array.cpp
index e732ece73c7f9..62d666fbcaaba 100644
--- a/c_glib/arrow-glib/chunked-array.cpp
+++ b/c_glib/arrow-glib/chunked-array.cpp
@@ -143,6 +143,26 @@ garrow_chunked_array_new(GList *chunks)
   return garrow_chunked_array_new_raw(&arrow_chunked_array);
 }
 
+/**
+ * garrow_chunked_array_equal:
+ * @chunked_array: A #GArrowChunkedArray.
+ * @other_chunked_array: A #GArrowChunkedArray to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_chunked_array_equal(GArrowChunkedArray *chunked_array,
+                           GArrowChunkedArray *other_chunked_array)
+{
+  const auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+  const auto arrow_other_chunked_array =
+    garrow_chunked_array_get_raw(other_chunked_array);
+  return arrow_chunked_array->Equals(arrow_other_chunked_array);
+}
+
 /**
  * garrow_chunked_array_get_length:
  * @chunked_array: A #GArrowChunkedArray.
diff --git a/c_glib/arrow-glib/chunked-array.h b/c_glib/arrow-glib/chunked-array.h
index 338930b9bd84a..c5f986a631835 100644
--- a/c_glib/arrow-glib/chunked-array.h
+++ b/c_glib/arrow-glib/chunked-array.h
@@ -67,6 +67,9 @@ GType garrow_chunked_array_get_type(void) G_GNUC_CONST;
 
 GArrowChunkedArray *garrow_chunked_array_new(GList *chunks);
 
+gboolean garrow_chunked_array_equal(GArrowChunkedArray *chunked_array,
+                                    GArrowChunkedArray *other_chunked_array);
+
 guint64 garrow_chunked_array_get_length (GArrowChunkedArray *chunked_array);
 guint64 garrow_chunked_array_get_n_nulls(GArrowChunkedArray *chunked_array);
 guint   garrow_chunked_array_get_n_chunks (GArrowChunkedArray *chunked_array);
diff --git a/c_glib/arrow-glib/column.cpp b/c_glib/arrow-glib/column.cpp
index 94df640d6b2b5..a7222b17650bb 100644
--- a/c_glib/arrow-glib/column.cpp
+++ b/c_glib/arrow-glib/column.cpp
@@ -160,6 +160,24 @@ garrow_column_new_chunked_array(GArrowField *field,
   return garrow_column_new_raw(&arrow_column);
 }
 
+/**
+ * garrow_column_equal:
+ * @column: A #GArrowColumn.
+ * @other_column: A #GArrowColumn to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_column_equal(GArrowColumn *column, GArrowColumn *other_column)
+{
+  const auto arrow_column = garrow_column_get_raw(column);
+  const auto arrow_other_column = garrow_column_get_raw(other_column);
+  return arrow_column->Equals(arrow_other_column);
+}
+
 /**
  * garrow_column_get_length:
  * @column: A #GArrowColumn.
diff --git a/c_glib/arrow-glib/column.h b/c_glib/arrow-glib/column.h
index fba3c26b2f08f..b649c5f1e50be 100644
--- a/c_glib/arrow-glib/column.h
+++ b/c_glib/arrow-glib/column.h
@@ -72,6 +72,9 @@ GArrowColumn *garrow_column_new_array(GArrowField *field,
 GArrowColumn *garrow_column_new_chunked_array(GArrowField *field,
                                               GArrowChunkedArray *chunked_array);
 
+gboolean            garrow_column_equal         (GArrowColumn *column,
+                                                 GArrowColumn *other_column);
+
 guint64             garrow_column_get_length    (GArrowColumn *column);
 guint64             garrow_column_get_n_nulls   (GArrowColumn *column);
 GArrowField        *garrow_column_get_field     (GArrowColumn *column);
diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/data-type.cpp
index c3c7fdb0f7c21..9ce8c16e914e3 100644
--- a/c_glib/arrow-glib/data-type.cpp
+++ b/c_glib/arrow-glib/data-type.cpp
@@ -164,9 +164,10 @@ garrow_data_type_class_init(GArrowDataTypeClass *klass)
 /**
  * garrow_data_type_equal:
  * @data_type: A #GArrowDataType.
- * @other_data_type: A #GArrowDataType.
+ * @other_data_type: A #GArrowDataType to be compared.
  *
- * Returns: Whether they are equal or not.
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
  */
 gboolean
 garrow_data_type_equal(GArrowDataType *data_type,
diff --git a/c_glib/arrow-glib/field.cpp b/c_glib/arrow-glib/field.cpp
index 5fd0c4d221bba..09c7ca33e6a13 100644
--- a/c_glib/arrow-glib/field.cpp
+++ b/c_glib/arrow-glib/field.cpp
@@ -204,9 +204,10 @@ garrow_field_is_nullable(GArrowField *field)
 /**
  * garrow_field_equal:
  * @field: A #GArrowField.
- * @other_field: A #GArrowField.
+ * @other_field: A #GArrowField to be compared.
  *
- * Returns: Whether they are equal or not.
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
  */
 gboolean
 garrow_field_equal(GArrowField *field,
diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp
index 8ac1791feef8c..3eed1a097c9e7 100644
--- a/c_glib/arrow-glib/record-batch.cpp
+++ b/c_glib/arrow-glib/record-batch.cpp
@@ -153,6 +153,26 @@ garrow_record_batch_new(GArrowSchema *schema,
   return garrow_record_batch_new_raw(&arrow_record_batch);
 }
 
+/**
+ * garrow_record_batch_equal:
+ * @record_batch: A #GArrowRecordBatch.
+ * @other_record_batch: A #GArrowRecordBatch to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_record_batch_equal(GArrowRecordBatch *record_batch,
+                          GArrowRecordBatch *other_record_batch)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  const auto arrow_other_record_batch =
+    garrow_record_batch_get_raw(other_record_batch);
+  return arrow_record_batch->Equals(*arrow_other_record_batch);
+}
+
 /**
  * garrow_record_batch_get_schema:
  * @record_batch: A #GArrowRecordBatch.
diff --git a/c_glib/arrow-glib/record-batch.h b/c_glib/arrow-glib/record-batch.h
index 92eee4d9af973..61e8f3d42b1c8 100644
--- a/c_glib/arrow-glib/record-batch.h
+++ b/c_glib/arrow-glib/record-batch.h
@@ -70,6 +70,9 @@ GArrowRecordBatch *garrow_record_batch_new(GArrowSchema *schema,
                                            guint32 n_rows,
                                            GList *columns);
 
+gboolean garrow_record_batch_equal(GArrowRecordBatch *record_batch,
+                                   GArrowRecordBatch *other_record_batch);
+
 GArrowSchema *garrow_record_batch_get_schema     (GArrowRecordBatch *record_batch);
 GArrowArray  *garrow_record_batch_get_column     (GArrowRecordBatch *record_batch,
                                                   guint i);
diff --git a/c_glib/arrow-glib/schema.cpp b/c_glib/arrow-glib/schema.cpp
index 4d5ae5af4fb4a..be3ea4bbb8c3e 100644
--- a/c_glib/arrow-glib/schema.cpp
+++ b/c_glib/arrow-glib/schema.cpp
@@ -142,6 +142,24 @@ garrow_schema_new(GList *fields)
   return garrow_schema_new_raw(&arrow_schema);
 }
 
+/**
+ * garrow_schema_equal:
+ * @schema: A #GArrowSchema.
+ * @other_schema: A #GArrowSchema to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_schema_equal(GArrowSchema *schema, GArrowSchema *other_schema)
+{
+  const auto arrow_schema = garrow_schema_get_raw(schema);
+  const auto arrow_other_schema = garrow_schema_get_raw(other_schema);
+  return arrow_schema->Equals(*arrow_other_schema);
+}
+
 /**
  * garrow_schema_get_field:
  * @schema: A #GArrowSchema.
diff --git a/c_glib/arrow-glib/schema.h b/c_glib/arrow-glib/schema.h
index 7615634021bc3..483d55e562d31 100644
--- a/c_glib/arrow-glib/schema.h
+++ b/c_glib/arrow-glib/schema.h
@@ -67,6 +67,8 @@ GType            garrow_schema_get_type         (void) G_GNUC_CONST;
 
 GArrowSchema    *garrow_schema_new              (GList *fields);
 
+gboolean         garrow_schema_equal            (GArrowSchema *schema,
+                                                 GArrowSchema *other_schema);
 GArrowField     *garrow_schema_get_field        (GArrowSchema *schema,
                                                  guint i);
 GArrowField     *garrow_schema_get_field_by_name(GArrowSchema *schema,
diff --git a/c_glib/arrow-glib/table.cpp b/c_glib/arrow-glib/table.cpp
index 2aba21b564243..779f2ef62b8f5 100644
--- a/c_glib/arrow-glib/table.cpp
+++ b/c_glib/arrow-glib/table.cpp
@@ -148,6 +148,24 @@ garrow_table_new(GArrowSchema *schema,
   return garrow_table_new_raw(&arrow_table);
 }
 
+/**
+ * garrow_table_equal:
+ * @table: A #GArrowTable.
+ * @other_table: A #GArrowTable to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_table_equal(GArrowTable *table, GArrowTable *other_table)
+{
+  const auto arrow_table = garrow_table_get_raw(table);
+  const auto arrow_other_table = garrow_table_get_raw(other_table);
+  return arrow_table->Equals(*arrow_other_table);
+}
+
 /**
  * garrow_table_get_schema:
  * @table: A #GArrowTable.
diff --git a/c_glib/arrow-glib/table.h b/c_glib/arrow-glib/table.h
index 9ae0cce1b7d9d..9e21669cd11da 100644
--- a/c_glib/arrow-glib/table.h
+++ b/c_glib/arrow-glib/table.h
@@ -69,6 +69,9 @@ GType           garrow_table_get_type      (void) G_GNUC_CONST;
 GArrowTable    *garrow_table_new           (GArrowSchema *schema,
                                             GList *columns);
 
+gboolean        garrow_table_equal         (GArrowTable *table,
+                                            GArrowTable *other_table);
+
 GArrowSchema   *garrow_table_get_schema    (GArrowTable *table);
 GArrowColumn   *garrow_table_get_column    (GArrowTable *table,
                                             guint i);
diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp
index 27af7532f3451..89e971c726ecb 100644
--- a/c_glib/arrow-glib/tensor.cpp
+++ b/c_glib/arrow-glib/tensor.cpp
@@ -170,6 +170,24 @@ garrow_tensor_new(GArrowDataType *data_type,
   return tensor;
 }
 
+/**
+ * garrow_tensor_equal:
+ * @tensor: A #GArrowTensor.
+ * @other_tensor: A #GArrowTensor to be compared.
+ *
+ * Returns: %TRUE if both of them have the same data, %FALSE
+ *   otherwise.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_tensor_equal(GArrowTensor *tensor, GArrowTensor *other_tensor)
+{
+  const auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  const auto arrow_other_tensor = garrow_tensor_get_raw(other_tensor);
+  return arrow_tensor->Equals(*arrow_other_tensor);
+}
+
 /**
  * garrow_tensor_get_value_data_type:
  * @tensor: A #GArrowTensor.
diff --git a/c_glib/arrow-glib/tensor.h b/c_glib/arrow-glib/tensor.h
index 71c6b4e9031dd..6529282f5f34b 100644
--- a/c_glib/arrow-glib/tensor.h
+++ b/c_glib/arrow-glib/tensor.h
@@ -66,6 +66,8 @@ GArrowTensor   *garrow_tensor_new                (GArrowDataType *data_type,
                                                   gsize n_strides,
                                                   gchar **dimention_names,
                                                   gsize n_dimention_names);
+gboolean        garrow_tensor_equal              (GArrowTensor *tensor,
+                                                  GArrowTensor *other_tensor);
 GArrowDataType *garrow_tensor_get_value_data_type(GArrowTensor *tensor);
 GArrowType      garrow_tensor_get_value_type     (GArrowTensor *tensor);
 GArrowBuffer   *garrow_tensor_get_buffer         (GArrowTensor *tensor);
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
index a2a2a1e003862..ca02fa283b014 100644
--- a/c_glib/test/test-array.rb
+++ b/c_glib/test/test-array.rb
@@ -16,6 +16,29 @@
 # under the License.
 
 class TestArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_equal
+    assert_equal(build_boolean_array([true, false]),
+                 build_boolean_array([true, false]))
+  end
+
+  def test_equal_approx
+    array1 = build_double_array([1.1, 2.2 + Float::EPSILON * 10])
+    array2 = build_double_array([1.1, 2.2])
+    assert do
+      array1.equal_approx(array2)
+    end
+  end
+
+  def test_equal_range
+    array1 = build_int32_array([1, 2, 3, 4, 5])
+    array2 = build_int32_array([-2, -1, 0, 1, 2, 3, 4, 999])
+    assert do
+      array1.equal_range(1, array2, 4, 3)
+    end
+  end
+
   def test_is_null
     builder = Arrow::BooleanArrayBuilder.new
     builder.append_null
diff --git a/c_glib/test/test-buffer.rb b/c_glib/test/test-buffer.rb
index 9f76a805f7577..39ae631a0f68d 100644
--- a/c_glib/test/test-buffer.rb
+++ b/c_glib/test/test-buffer.rb
@@ -23,6 +23,19 @@ def setup
     @buffer = Arrow::Buffer.new(@data)
   end
 
+  def test_equal
+    assert_equal(@buffer,
+                 Arrow::Buffer.new(@data.dup))
+  end
+
+  def test_equal_n_bytes
+    buffer1 = Arrow::Buffer.new("Hello!")
+    buffer2 = Arrow::Buffer.new("Hello World!")
+    assert do
+      buffer1.equal_n_bytes(buffer2, 5)
+    end
+  end
+
   def test_mutable?
     assert do
       not @buffer.mutable?
diff --git a/c_glib/test/test-chunked-array.rb b/c_glib/test/test-chunked-array.rb
index 167d5d1033e42..cde7a8b0c61f1 100644
--- a/c_glib/test/test-chunked-array.rb
+++ b/c_glib/test/test-chunked-array.rb
@@ -18,6 +18,19 @@
 class TestChunkedArray < Test::Unit::TestCase
   include Helper::Buildable
 
+  def test_equal
+    chunks1 = [
+      build_boolean_array([true, false]),
+      build_boolean_array([true]),
+    ]
+    chunks2 = [
+      build_boolean_array([true]),
+      build_boolean_array([false, true]),
+    ]
+    assert_equal(Arrow::ChunkedArray.new(chunks1),
+                 Arrow::ChunkedArray.new(chunks2))
+  end
+
   def test_length
     chunks = [
       build_boolean_array([true, false]),
diff --git a/c_glib/test/test-column.rb b/c_glib/test/test-column.rb
index ec75194edb830..96e02b60319fd 100644
--- a/c_glib/test/test-column.rb
+++ b/c_glib/test/test-column.rb
@@ -38,6 +38,19 @@ def test_chunked_array
     end
   end
 
+  def test_equal
+    field1 = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    array1 = build_boolean_array([true, false])
+    field2 = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    chunks = [
+      build_boolean_array([true]),
+      build_boolean_array([false]),
+    ]
+    array2 = Arrow::ChunkedArray.new(chunks)
+    assert_equal(Arrow::Column.new(field1, array1),
+                 Arrow::Column.new(field2, array2))
+  end
+
   def test_length
     field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
     array = build_boolean_array([true, false])
diff --git a/c_glib/test/test-field.rb b/c_glib/test/test-field.rb
index a20802c2ac653..1b9c46e8cd037 100644
--- a/c_glib/test/test-field.rb
+++ b/c_glib/test/test-field.rb
@@ -16,6 +16,11 @@
 # under the License.
 
 class TestField < Test::Unit::TestCase
+  def test_equal
+    assert_equal(Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
+                 Arrow::Field.new("enabled", Arrow::BooleanDataType.new))
+  end
+
   def test_name
     field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
     assert_equal("enabled", field.name)
diff --git a/c_glib/test/test-record-batch.rb b/c_glib/test/test-record-batch.rb
index 941ff35060154..048f6deb7245c 100644
--- a/c_glib/test/test-record-batch.rb
+++ b/c_glib/test/test-record-batch.rb
@@ -46,6 +46,21 @@ def setup
       @record_batch = Arrow::RecordBatch.new(schema, 5, columns)
     end
 
+    def test_equal
+      fields = [
+        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
+      ]
+      schema = Arrow::Schema.new(fields)
+      columns = [
+        build_boolean_array([true, false, true, false, true, false]),
+        build_boolean_array([false, true, false, true, false]),
+      ]
+      other_record_batch = Arrow::RecordBatch.new(schema, 5, columns)
+      assert_equal(@record_batch,
+                   other_record_batch)
+    end
+
     def test_schema
       assert_equal(["visible", "valid"],
                    @record_batch.schema.fields.collect(&:name))
diff --git a/c_glib/test/test-schema.rb b/c_glib/test/test-schema.rb
index c9cbb756944bb..4c09ecb40f51e 100644
--- a/c_glib/test/test-schema.rb
+++ b/c_glib/test/test-schema.rb
@@ -16,6 +16,17 @@
 # under the License.
 
 class TestSchema < Test::Unit::TestCase
+  def test_equal
+    fields1 = [
+      Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
+    ]
+    fields2 = [
+      Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
+    ]
+    assert_equal(Arrow::Schema.new(fields1),
+                 Arrow::Schema.new(fields2))
+  end
+
   def test_field
     field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
     schema = Arrow::Schema.new([field])
diff --git a/c_glib/test/test-table.rb b/c_glib/test/test-table.rb
index da6871ec1d090..08dd34861e51a 100644
--- a/c_glib/test/test-table.rb
+++ b/c_glib/test/test-table.rb
@@ -66,6 +66,20 @@ def setup
       @table = Arrow::Table.new(schema, columns)
     end
 
+    def test_equal
+      fields = [
+        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
+      ]
+      schema = Arrow::Schema.new(fields)
+      columns = [
+        Arrow::Column.new(fields[0], build_boolean_array([true])),
+        Arrow::Column.new(fields[1], build_boolean_array([false])),
+      ]
+      other_table = Arrow::Table.new(schema, columns)
+      assert_equal(@table, other_table)
+    end
+
     def test_schema
       assert_equal(["visible", "valid"],
                    @table.schema.fields.collect(&:name))
diff --git a/c_glib/test/test-tensor.rb b/c_glib/test/test-tensor.rb
index 225857b52da98..780c9f179e18d 100644
--- a/c_glib/test/test-tensor.rb
+++ b/c_glib/test/test-tensor.rb
@@ -40,6 +40,19 @@ def setup
                                 names)
   end
 
+  def test_equal
+    data = Arrow::Buffer.new(@raw_data.pack("c*"))
+    strides = []
+    names = ["a", "b", "c"]
+    other_tensor = Arrow::Tensor.new(Arrow::Int8DataType.new,
+                                     data,
+                                     @shape,
+                                     strides,
+                                     names)
+    assert_equal(@tensor,
+                 other_tensor)
+  end
+
   def test_value_data_type
     assert_equal(Arrow::Int8DataType, @tensor.value_data_type.class)
   end

From d7a2a1e18457acb8a18cfcb7fbb3c3ba41543d4a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 7 May 2017 17:48:18 +0200
Subject: [PATCH 0618/1644] ARROW-958: [Python] Fix conda source build
 instructions

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #653 from wesm/ARROW-958 and squashes the following commits:

88c3c1d [Wes McKinney] Fix conda build instructions
---
 python/doc/source/development.rst | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 01add1142642a..440c1c459eed5 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -93,8 +93,11 @@ about our build toolchain:
 .. code-block:: shell
 
    export ARROW_BUILD_TYPE=release
+
    export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
    export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX
+   export ARROW_HOME=$CONDA_PREFIX
+   export PARQUET_HOME=$CONDA_PREFIX
 
 Now build and install the Arrow C++ libraries:
 
@@ -104,7 +107,7 @@ Now build and install the Arrow C++ libraries:
    pushd arrow/cpp/build
 
    cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-         -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+         -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
          -DARROW_PYTHON=on \
          -DARROW_BUILD_TESTS=OFF \
          ..
@@ -121,7 +124,7 @@ toolchain:
    pushd parquet-cpp/build
 
    cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-         -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+         -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
          -DPARQUET_BUILD_BENCHMARKS=off \
          -DPARQUET_BUILD_EXECUTABLES=off \
          -DPARQUET_ZLIB_VENDORED=off \

From cb5e7b6fa7d75e14e163ce43cb333b02e9fe1c03 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 8 May 2017 00:49:37 -0400
Subject: [PATCH 0619/1644] ARROW-446: [Python] Expand Sphinx documentation for
 0.3

I am going to finish the data model section and revamp the Parquet section, so we can get this pushed out with the release announcement tomorrow. We should continue to add a lot of new documentation over the coming weeks

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #656 from wesm/ARROW-446 and squashes the following commits:

b92c6d2 [Wes McKinney] Make pass over Parquet docs
a46f846 [Wes McKinney] Make a pass over Parquet documentation
066d0b9 [Wes McKinney] Finish first cut at data model section
4f510fb [Wes McKinney] Install IPython before building docs
4885222 [Wes McKinney] Start on a data model section
1d512e9 [Wes McKinney] Add barebones IPC section
0f800d8 [Wes McKinney] Add section on OSFile, MemoryMappedFile
aabf5b2 [Wes McKinney] Add draft about memory/io
5968847 [Wes McKinney] More on Memory/IO section
---
 ci/travis_script_python.sh        |   2 +-
 python/doc/requirements.txt       |   2 +
 python/doc/source/api.rst         |  19 +-
 python/doc/source/conf.py         |  14 +-
 python/doc/source/data.rst        | 316 ++++++++++++++++++++++++++++++
 python/doc/source/filesystems.rst |   8 +-
 python/doc/source/index.rst       |   5 +-
 python/doc/source/ipc.rst         | 136 +++++++++++++
 python/doc/source/jemalloc.rst    |   9 +-
 python/doc/source/memory.rst      | 235 ++++++++++++++++++++++
 python/doc/source/pandas.rst      |  36 ++--
 python/doc/source/parquet.rst     | 243 ++++++++++++++++++-----
 python/pyarrow/_io.pyx            |   1 +
 13 files changed, 936 insertions(+), 90 deletions(-)
 create mode 100644 python/doc/source/data.rst
 create mode 100644 python/doc/source/ipc.rst
 create mode 100644 python/doc/source/memory.rst

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 20b0f2aadb900..ce5f7ec506b73 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -117,7 +117,7 @@ python_version_tests() {
   # Build documentation once
   if [[ "$PYTHON_VERSION" == "3.6" ]]
   then
-      pip install -r doc/requirements.txt
+      conda install -y -q --file=doc/requirements.txt
       python setup.py build_sphinx -s doc/source
   fi
 }
diff --git a/python/doc/requirements.txt b/python/doc/requirements.txt
index ce0793c31de26..f3c3414a4be9a 100644
--- a/python/doc/requirements.txt
+++ b/python/doc/requirements.txt
@@ -1,3 +1,5 @@
+ipython
+matplotlib
 numpydoc
 sphinx
 sphinx_rtd_theme
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 08a06948a3fba..a8dd8c5e110ac 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -22,7 +22,7 @@
 API Reference
 *************
 
-.. _api.functions:
+.. _api.types:
 
 Type and Schema Factory Functions
 ---------------------------------
@@ -58,6 +58,8 @@ Type and Schema Factory Functions
    schema
    from_numpy_dtype
 
+.. _api.value:
+
 Scalar Value Types
 ------------------
 
@@ -88,6 +90,7 @@ Scalar Value Types
    TimestampValue
    DecimalValue
 
+.. _api.array:
 
 Array Types and Constructors
 ----------------------------
@@ -122,6 +125,8 @@ Array Types and Constructors
    DecimalArray
    ListArray
 
+.. _api.table:
+
 Tables and Record Batches
 -------------------------
 
@@ -134,6 +139,8 @@ Tables and Record Batches
    Table
    get_record_batch_size
 
+.. _api.tensor:
+
 Tensor type and Functions
 -------------------------
 
@@ -145,6 +152,8 @@ Tensor type and Functions
    get_tensor_size
    read_tensor
 
+.. _api.io:
+
 Input / Output and Shared Memory
 --------------------------------
 
@@ -160,6 +169,8 @@ Input / Output and Shared Memory
    create_memory_map
    PythonFile
 
+.. _api.ipc:
+
 Interprocess Communication and Messaging
 ----------------------------------------
 
@@ -171,6 +182,8 @@ Interprocess Communication and Messaging
    StreamReader
    StreamWriter
 
+.. _api.memory_pool:
+
 Memory Pools
 ------------
 
@@ -183,6 +196,8 @@ Memory Pools
    total_allocated_bytes
    set_memory_pool
 
+.. _api.type_classes:
+
 Type Classes
 ------------
 
@@ -201,6 +216,8 @@ Type Classes
 
 .. currentmodule:: pyarrow.parquet
 
+.. _api.parquet:
+
 Apache Parquet
 --------------
 
diff --git a/python/doc/source/conf.py b/python/doc/source/conf.py
index a9262bf7db3dd..7f98979e88ff8 100644
--- a/python/doc/source/conf.py
+++ b/python/doc/source/conf.py
@@ -25,19 +25,11 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-import inspect
 import os
 import sys
 
 import sphinx_rtd_theme
 
-on_rtd = os.environ.get('READTHEDOCS') == 'True'
-
-if not on_rtd:
-    # Hack: On RTD we use the pyarrow package from conda-forge as we cannot
-    # build pyarrow there.
-    sys.path.insert(0, os.path.abspath('..'))
-
 sys.path.extend([
     os.path.join(os.path.dirname(__file__),
                  '..', '../..')
@@ -60,6 +52,8 @@
     'sphinx.ext.mathjax',
     'sphinx.ext.viewcode',
     'sphinx.ext.napoleon',
+    'IPython.sphinxext.ipython_directive',
+    'IPython.sphinxext.ipython_console_highlighting'
 ]
 
 # numpydoc configuration
@@ -86,7 +80,7 @@
 
 # General information about the project.
 project = u'pyarrow'
-copyright = u'2016 Apache Software Foundation'
+copyright = u'2016-2017 Apache Software Foundation'
 author = u'Apache Software Foundation'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -156,7 +150,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = 'sphinxdoc'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
diff --git a/python/doc/source/data.rst b/python/doc/source/data.rst
new file mode 100644
index 0000000000000..04e74ae64d437
--- /dev/null
+++ b/python/doc/source/data.rst
@@ -0,0 +1,316 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+.. _data:
+
+In-Memory Data Model
+====================
+
+Apache Arrow defines columnar array data structures by composing type metadata
+with memory buffers, like the ones explained in the documentation on
+:ref:`Memory and IO <io>`. These data structures are exposed in Python through
+a series of interrelated classes:
+
+* **Type Metadata**: Instances of ``pyarrow.DataType``, which describe a logical
+  array type
+* **Schemas**: Instances of ``pyarrow.Schema``, which describe a named
+  collection of types. These can be thought of as the column types in a
+  table-like object.
+* **Arrays**: Instances of ``pyarrow.Array``, which are atomic, contiguous
+  columnar data structures composed from Arrow Buffer objects
+* **Record Batches**: Instances of ``pyarrow.RecordBatch``, which are a
+  collection of Array objects with a particular Schema
+* **Tables**: Instances of ``pyarrow.Table``, a logical table data structure in
+  which each column consists of one or more ``pyarrow.Array`` objects of the
+  same type.
+
+We will examine these in the sections below in a series of examples.
+
+.. _data.types:
+
+Type Metadata
+-------------
+
+Apache Arrow defines language agnostic column-oriented data structures for
+array data. These include:
+
+* **Fixed-length primitive types**: numbers, booleans, date and times, fixed
+  size binary, decimals, and other values that fit into a given number
+* **Variable-length primitive types**: binary, string
+* **Nested types**: list, struct, and union
+* **Dictionary type**: An encoded categorical type (more on this later)
+
+Each logical data type in Arrow has a corresponding factory function for
+creating an instance of that type object in Python:
+
+.. ipython:: python
+
+   import pyarrow as pa
+   t1 = pa.int32()
+   t2 = pa.string()
+   t3 = pa.binary()
+   t4 = pa.binary(10)
+   t5 = pa.timestamp('ms')
+
+   t1
+   print(t1)
+   print(t4)
+   print(t5)
+
+We use the name **logical type** because the **physical** storage may be the
+same for one or more types. For example, ``int64``, ``float64``, and
+``timestamp[ms]`` all occupy 64 bits per value.
+
+These objects are `metadata`; they are used for describing the data in arrays,
+schemas, and record batches. In Python, they can be used in functions where the
+input data (e.g. Python objects) may be coerced to more than one Arrow type.
+
+The :class:`~pyarrow.Field` type is a type plus a name and optional
+user-defined metadata:
+
+.. ipython:: python
+
+   f0 = pa.field('int32_field', t1)
+   f0
+   f0.name
+   f0.type
+
+Arrow supports **nested value types** like list, struct, and union. When
+creating these, you must pass types or fields to indicate the data types of the
+types' children. For example, we can define a list of int32 values with:
+
+.. ipython:: python
+
+   t6 = pa.list_(t1)
+   t6
+
+A `struct` is a collection of named fields:
+
+.. ipython:: python
+
+   fields = [
+       pa.field('s0', t1),
+       pa.field('s1', t2),
+       pa.field('s2', t4),
+       pa.field('s3', t6)
+   ]
+
+   t7 = pa.struct(fields)
+   print(t7)
+
+See :ref:`Data Types API <api.types>` for a full listing of data type
+functions.
+
+.. _data.schema:
+
+Schemas
+-------
+
+The :class:`~pyarrow.Schema` type is similar to the ``struct`` array type; it
+defines the column names and types in a record batch or table data
+structure. The ``pyarrow.schema`` factory function makes new Schema objects in
+Python:
+
+.. ipython:: python
+
+   fields = [
+       pa.field('s0', t1),
+       pa.field('s1', t2),
+       pa.field('s2', t4),
+       pa.field('s3', t6)
+   ]
+
+   my_schema = pa.schema(fields)
+   my_schema
+
+In some applications, you may not create schemas directly, only using the ones
+that are embedded in :ref:`IPC messages <ipc>`.
+
+.. _data.array:
+
+Arrays
+------
+
+For each data type, there is an accompanying array data structure for holding
+memory buffers that define a single contiguous chunk of columnar array
+data. When you are using PyArrow, this data may come from IPC tools, though it
+can also be created from various types of Python sequences (lists, NumPy
+arrays, pandas data).
+
+A simple way to create arrays is with ``pyarrow.array``, which is similar to
+the ``numpy.array`` function:
+
+.. ipython:: python
+
+   arr = pa.array([1, 2, None, 3])
+   arr
+
+The array's ``type`` attribute is the corresponding piece of type metadata:
+
+.. ipython:: python
+
+   arr.type
+
+Each in-memory array has a known length and null count (which will be 0 if
+there are no null values):
+
+.. ipython:: python
+
+   len(arr)
+   arr.null_count
+
+Scalar values can be selected with normal indexing.  ``pyarrow.array`` converts
+``None`` values to Arrow nulls; we return the special ``pyarrow.NA`` value for
+nulls:
+
+.. ipython:: python
+
+   arr[0]
+   arr[2]
+
+Arrow data is immutable, so values can be selected but not assigned.
+
+Arrays can be sliced without copying:
+
+.. ipython:: python
+
+   arr[3]
+
+``pyarrow.array`` can create simple nested data structures like lists:
+
+.. ipython:: python
+
+   nested_arr = pa.array([[], None, [1, 2], [None, 1]])
+   print(nested_arr.type)
+
+Dictionary Arrays
+~~~~~~~~~~~~~~~~~
+
+The **Dictionary** type in PyArrow is a special array type that is similar to a
+factor in R or a ``pandas.Categorical``. It enables one or more record batches
+in a file or stream to transmit integer *indices* referencing a shared
+**dictionary** containing the distinct values in the logical array. This is
+particularly often used with strings to save memory and improve performance.
+
+The way that dictionaries are handled in the Apache Arrow format and the way
+they appear in C++ and Python is slightly different. We define a special
+:class:`~.DictionaryArray` type with a corresponding dictionary type. Let's
+consider an example:
+
+.. ipython:: python
+
+   indices = pa.array([0, 1, 0, 1, 2, 0, None, 2])
+   dictionary = pa.array(['foo', 'bar', 'baz'])
+
+   dict_array = pa.DictionaryArray.from_arrays(indices, dictionary)
+   dict_array
+
+Here we have:
+
+.. ipython:: python
+
+   print(dict_array.type)
+   dict_array.indices
+   dict_array.dictionary
+
+When using :class:`~.DictionaryArray` with pandas, the analogue is
+``pandas.Categorical`` (more on this later):
+
+.. ipython:: python
+
+   dict_array.to_pandas()
+
+.. _data.record_batch:
+
+Record Batches
+--------------
+
+A **Record Batch** in Apache Arrow is a collection of equal-length array
+instances. Let's consider a collection of arrays:
+
+.. ipython:: python
+
+   data = [
+       pa.array([1, 2, 3, 4]),
+       pa.array(['foo', 'bar', 'baz', None]),
+       pa.array([True, None, False, True])
+   ]
+
+A record batch can be created from this list of arrays using
+``RecordBatch.from_arrays``:
+
+.. ipython:: python
+
+   batch = pa.RecordBatch.from_arrays(data, ['f0', 'f1', 'f2'])
+   batch.num_columns
+   batch.num_rows
+   batch.schema
+
+   batch[1]
+
+A record batch can be sliced without copying memory like an array:
+
+.. ipython:: python
+
+   batch2 = batch.slice(1, 3)
+   batch2[1]
+
+.. _data.table:
+
+Tables
+------
+
+The PyArrow :class:`~.Table` type is not part of the Apache Arrow
+specification, but is rather a tool to help with wrangling multiple record
+batches and array pieces as a single logical dataset. As a relevant example, we
+may receive multiple small record batches in a socket stream, then need to
+concatenate them into contiguous memory for use in NumPy or pandas. The Table
+object makes this efficient without requiring additional memory copying.
+
+Considering the record batch we created above, we can create a Table containing
+one or more copies of the batch using ``Table.from_batches``:
+
+.. ipython:: python
+
+   batches = [batch] * 5
+   table = pa.Table.from_batches(batches)
+   table
+   table.num_rows
+
+The table's columns are instances of :class:`~.Column`, which is a container
+for one or more arrays of the same type.
+
+.. ipython:: python
+
+   c = table[0]
+   c
+   c.data
+   c.data.num_chunks
+   c.data.chunk(0)
+
+As you'll see in the :ref:`pandas section <pandas>`, we can convert thee
+objects to contiguous NumPy arrays for use in pandas:
+
+.. ipython:: python
+
+   c.to_pandas()
+
+Custom Schema and Field Metadata
+--------------------------------
+
+TODO
diff --git a/python/doc/source/filesystems.rst b/python/doc/source/filesystems.rst
index 9e00ddd558127..61c03c57dfad9 100644
--- a/python/doc/source/filesystems.rst
+++ b/python/doc/source/filesystems.rst
@@ -15,10 +15,12 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
-File interfaces and Memory Maps
-===============================
+Filesystem Interfaces
+=====================
 
-PyArrow features a number of file-like interfaces
+In this section, we discuss filesystem-like interfaces in PyArrow.
+
+.. _hdfs:
 
 Hadoop File System (HDFS)
 -------------------------
diff --git a/python/doc/source/index.rst b/python/doc/source/index.rst
index 55b4efc79bc3f..4bfbe44605767 100644
--- a/python/doc/source/index.rst
+++ b/python/doc/source/index.rst
@@ -36,8 +36,11 @@ structures.
 
    install
    development
-   pandas
+   memory
+   data
+   ipc
    filesystems
+   pandas
    parquet
    api
    getting_involved
diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
new file mode 100644
index 0000000000000..e63e7455bb815
--- /dev/null
+++ b/python/doc/source/ipc.rst
@@ -0,0 +1,136 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+
+.. _ipc:
+
+IPC: Fast Streaming and Serialization
+=====================================
+
+Arrow defines two types of binary formats for serializing record batches:
+
+* **Streaming format**: for sending an arbitrary length sequence of record
+  batches. The format must be processed from start to end, and does not support
+  random access
+
+* **File or Random Access format**: for serializing a fixed number of record
+  batches. Supports random access, and thus is very useful when used with
+  memory maps
+
+To follow this section, make sure to first read the section on :ref:`Memory and
+IO <io>`.
+
+Writing and Reading Streams
+---------------------------
+
+First, let's create a small record batch:
+
+.. ipython:: python
+
+   import pyarrow as pa
+
+   data = [
+       pa.array([1, 2, 3, 4]),
+       pa.array(['foo', 'bar', 'baz', None]),
+       pa.array([True, None, False, True])
+   ]
+
+   batch = pa.RecordBatch.from_arrays(data, ['f0', 'f1', 'f2'])
+   batch.num_rows
+   batch.num_columns
+
+Now, we can begin writing a stream containing some number of these batches. For
+this we use :class:`~pyarrow.StreamWriter`, which can write to a writeable
+``NativeFile`` object or a writeable Python object:
+
+.. ipython:: python
+
+   sink = pa.InMemoryOutputStream()
+   writer = pa.StreamWriter(sink, batch.schema)
+
+Here we used an in-memory Arrow buffer stream, but this could have been a
+socket or some other IO sink.
+
+When creating the ``StreamWriter``, we pass the schema, since the schema
+(column names and types) must be the same for all of the batches sent in this
+particular stream. Now we can do:
+
+.. ipython:: python
+
+   for i in range(5):
+      writer.write_batch(batch)
+   writer.close()
+
+   buf = sink.get_result()
+   buf.size
+
+Now ``buf`` contains the complete stream as an in-memory byte buffer. We can
+read such a stream with :class:`~pyarrow.StreamReader`:
+
+.. ipython:: python
+
+   reader = pa.StreamReader(buf)
+   reader.schema
+
+   batches = [b for b in reader]
+   len(batches)
+
+We can check the returned batches are the same as the original input:
+
+.. ipython:: python
+
+   batches[0].equals(batch)
+
+An important point is that if the input source supports zero-copy reads
+(e.g. like a memory map, or ``pyarrow.BufferReader``), then the returned
+batches are also zero-copy and do not allocate any new memory on read.
+
+Writing and Reading Random Access Files
+---------------------------------------
+
+The :class:`~pyarrow.FileWriter` has the same API as
+:class:`~pyarrow.StreamWriter`:
+
+.. ipython:: python
+
+   sink = pa.InMemoryOutputStream()
+   writer = pa.FileWriter(sink, batch.schema)
+
+   for i in range(10):
+      writer.write_batch(batch)
+   writer.close()
+
+   buf = sink.get_result()
+   buf.size
+
+The difference between :class:`~pyarrow.FileReader` and
+:class:`~pyarrow.StreamReader` is that the input source must have a ``seek``
+method for random access. The stream reader only requires read operations:
+
+.. ipython:: python
+
+   reader = pa.FileReader(buf)
+
+Because we have access to the entire payload, we know the number of record
+batches in the file, and can read any at random:
+
+.. ipython:: python
+
+   reader.num_record_batches
+   b = reader.get_batch(3)
+   b.equals(batch)
diff --git a/python/doc/source/jemalloc.rst b/python/doc/source/jemalloc.rst
index 8d7a5dc4a82ec..9389dcbd25cfe 100644
--- a/python/doc/source/jemalloc.rst
+++ b/python/doc/source/jemalloc.rst
@@ -18,7 +18,7 @@
 jemalloc MemoryPool
 ===================
 
-Arrow's default :class:`~pyarrow.memory.MemoryPool` uses the system's allocator
+Arrow's default :class:`~pyarrow.MemoryPool` uses the system's allocator
 through the POSIX APIs. Although this already provides aligned allocation, the
 POSIX interface doesn't support aligned reallocation. The default reallocation
 strategy is to allocate a new region, copy over the old data and free the
@@ -27,10 +27,9 @@ the existing memory allocation to the requested size. While this may still be
 linear in the size of allocated memory, it is magnitudes faster as only the page
 mapping in the kernel is touched, not the actual data.
 
-The :mod:`~pyarrow.jemalloc` allocator is not enabled by default to allow the
-use of the system allocator and/or other allocators like ``tcmalloc``. You can
-either explicitly make it the default allocator or pass it only to single
-operations.
+The jemalloc-based allocator is not enabled by default to allow the use of the
+system allocator and/or other allocators like ``tcmalloc``. You can either
+explicitly make it the default allocator or pass it only to single operations.
 
 .. code:: python
 
diff --git a/python/doc/source/memory.rst b/python/doc/source/memory.rst
new file mode 100644
index 0000000000000..d1020da246407
--- /dev/null
+++ b/python/doc/source/memory.rst
@@ -0,0 +1,235 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+.. _io:
+
+Memory and IO Interfaces
+========================
+
+This section will introduce you to the major concepts in PyArrow's memory
+management and IO systems:
+
+* Buffers
+* File-like and stream-like objects
+* Memory pools
+
+pyarrow.Buffer
+--------------
+
+The :class:`~pyarrow.Buffer` object wraps the C++ ``arrow::Buffer`` type and is
+the primary tool for memory management in Apache Arrow in C++. It permits
+higher-level array classes to safely interact with memory which they may or may
+not own. ``arrow::Buffer`` can be zero-copy sliced to permit Buffers to cheaply
+reference other Buffers, while preserving memory lifetime and clean
+parent-child relationships.
+
+There are many implementations of ``arrow::Buffer``, but they all provide a
+standard interface: a data pointer and length. This is similar to Python's
+built-in `buffer protocol` and ``memoryview`` objects.
+
+A :class:`~pyarrow.Buffer` can be created from any Python object which
+implements the buffer protocol. Let's consider a bytes object:
+
+.. ipython:: python
+
+   import pyarrow as pa
+
+   data = b'abcdefghijklmnopqrstuvwxyz'
+   buf = pa.frombuffer(data)
+   buf
+   buf.size
+
+Creating a Buffer in this way does not allocate any memory; it is a zero-copy
+view on the memory exported from the ``data`` bytes object.
+
+The Buffer's ``to_pybytes`` method can convert to a Python byte string:
+
+.. ipython:: python
+
+   buf.to_pybytes()
+
+Buffers can be used in circumstances where a Python buffer or memoryview is
+required, and such conversions are also zero-copy:
+
+.. ipython:: python
+
+   memoryview(buf)
+
+.. _io.native_file:
+
+Native Files
+------------
+
+The Arrow C++ libraries have several abstract interfaces for different kinds of
+IO objects:
+
+* Read-only streams
+* Read-only files supporting random access
+* Write-only streams
+* Write-only files supporting random access
+* File supporting reads, writes, and random access
+
+In the the interest of making these objects behave more like Python's built-in
+``file`` objects, we have defined a :class:`~pyarrow.NativeFile` base class
+which is intended to mimic Python files and able to be used in functions where
+a Python file (such as ``file`` or ``BytesIO``) is expected.
+
+:class:`~pyarrow.NativeFile` has some important features which make it
+preferable to using Python files with PyArrow where possible:
+
+* Other Arrow classes can access the internal C++ IO objects natively, and do
+  not need to acquire the Python GIL
+* Native C++ IO may be able to do zero-copy IO, such as with memory maps
+
+There are several kinds of :class:`~pyarrow.NativeFile` options available:
+
+* :class:`~pyarrow.OSFile`, a native file that uses your operating system's
+  file descriptors
+* :class:`~pyarrow.MemoryMappedFile`, for reading (zero-copy) and writing with
+  memory maps
+* :class:`~pyarrow.BufferReader`, for reading :class:`~pyarrow.Buffer` objects
+  as a file
+* :class:`~pyarrow.InMemoryOutputStream`, for writing data in-memory, producing
+  a Buffer at the end
+* :class:`~pyarrow.HdfsFile`, for reading and writing data to the Hadoop Filesystem
+* :class:`~pyarrow.PythonFile`, for interfacing with Python file objects in C++
+
+We will discuss these in the following sections after explaining memory pools.
+
+Memory Pools
+------------
+
+All memory allocations and deallocations (like ``malloc`` and ``free`` in C)
+are tracked in an instance of ``arrow::MemoryPool``. This means that we can
+then precisely track amount of memory that has been allocated:
+
+.. ipython:: python
+
+   pa.total_allocated_bytes()
+
+PyArrow uses a default built-in memory pool, but in the future there may be
+additional memory pools (and subpools) to choose from. Let's consider an
+``InMemoryOutputStream``, which is like a ``BytesIO``:
+
+.. ipython:: python
+
+   stream = pa.InMemoryOutputStream()
+   stream.write(b'foo')
+   pa.total_allocated_bytes()
+   for i in range(1024): stream.write(b'foo')
+   pa.total_allocated_bytes()
+
+The default allocator requests memory in a minimum increment of 64 bytes. If
+the stream is garbaged-collected, all of the memory is freed:
+
+.. ipython:: python
+
+   stream = None
+   pa.total_allocated_bytes()
+
+Classes and functions that may allocate memory will often have an option to
+pass in a custom memory pool:
+
+.. ipython:: python
+
+   my_pool = pa.jemalloc_memory_pool()
+   my_pool
+   my_pool.bytes_allocated()
+   stream = pa.InMemoryOutputStream(my_pool)
+   stream.write(b'foo')
+   my_pool.bytes_allocated()
+
+On-Disk and Memory Mapped Files
+-------------------------------
+
+PyArrow includes two ways to interact with data on disk: standard operating
+system-level file APIs, and memory-mapped files. In regular Python we can
+write:
+
+.. ipython:: python
+
+   with open('example.dat', 'wb') as f:
+       f.write(b'some example data')
+
+Using pyarrow's :class:`~pyarrow.OSFile` class, you can write:
+
+.. ipython:: python
+
+   with pa.OSFile('example2.dat', 'wb') as f:
+       f.write(b'some example data')
+
+For reading files, you can use ``OSFile`` or
+:class:`~pyarrow.MemoryMappedFile`. The difference between these is that
+:class:`~pyarrow.OSFile` allocates new memory on each read, like Python file
+objects. In reads from memory maps, the library constructs a buffer referencing
+the mapped memory without any memory allocation or copying:
+
+.. ipython:: python
+
+   file_obj = pa.OSFile('example.dat')
+   mmap = pa.memory_map('example.dat')
+   file_obj.read(4)
+   mmap.read(4)
+
+The ``read`` method implements the standard Python file ``read`` API. To read
+into Arrow Buffer objects, use ``read_buffer``:
+
+.. ipython:: python
+
+   mmap.seek(0)
+   buf = mmap.read_buffer(4)
+   print(buf)
+   buf.to_pybytes()
+
+Many tools in PyArrow, particular the Apache Parquet interface and the file and
+stream messaging tools, are more efficient when used with these ``NativeFile``
+types than with normal Python file objects.
+
+.. ipython:: python
+   :suppress:
+
+   buf = mmap = file_obj = None
+   !rm example.dat
+   !rm example2.dat
+
+In-Memory Reading and Writing
+-----------------------------
+
+To assist with serialization and deserialization of in-memory data, we have
+file interfaces that can read and write to Arrow Buffers.
+
+.. ipython:: python
+
+   writer = pa.InMemoryOutputStream()
+   writer.write(b'hello, friends')
+
+   buf = writer.get_result()
+   buf
+   buf.size
+   reader = pa.BufferReader(buf)
+   reader.seek(7)
+   reader.read(7)
+
+These have similar semantics to Python's built-in ``io.BytesIO``.
+
+Hadoop Filesystem
+-----------------
+
+:class:`~pyarrow.HdfsFile` is an implementation of :class:`~pyarrow.NativeFile`
+that can read and write to the Hadoop filesytem. Read more in the
+:ref:`Filesystems Section <hdfs>`.
diff --git a/python/doc/source/pandas.rst b/python/doc/source/pandas.rst
index 34445aed517d3..cb7a56d020d19 100644
--- a/python/doc/source/pandas.rst
+++ b/python/doc/source/pandas.rst
@@ -15,17 +15,17 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
-Pandas Interface
-================
+Using PyArrow with pandas
+=========================
 
-To interface with Pandas, PyArrow provides various conversion routines to
-consume Pandas structures and convert back to them.
+To interface with pandas, PyArrow provides various conversion routines to
+consume pandas structures and convert back to them.
 
 DataFrames
 ----------
 
-The equivalent to a Pandas DataFrame in Arrow is a :class:`pyarrow.table.Table`.
-Both consist of a set of named columns of equal length. While Pandas only
+The equivalent to a pandas DataFrame in Arrow is a :class:`pyarrow.table.Table`.
+Both consist of a set of named columns of equal length. While pandas only
 supports flat columns, the Table also provides nested columns, thus it can
 represent more data than a DataFrame, so a full conversion is not always possible.
 
@@ -33,9 +33,9 @@ Conversion from a Table to a DataFrame is done by calling
 :meth:`pyarrow.table.Table.to_pandas`. The inverse is then achieved by using
 :meth:`pyarrow.Table.from_pandas`. This conversion routine provides the
 convience parameter ``timestamps_to_ms``. Although Arrow supports timestamps of
-different resolutions, Pandas only supports nanosecond timestamps and most
+different resolutions, pandas only supports nanosecond timestamps and most
 other systems (e.g. Parquet) only work on millisecond timestamps. This parameter
-can be used to already do the time conversion during the Pandas to Arrow
+can be used to already do the time conversion during the pandas to Arrow
 conversion.
 
 .. code-block:: python
@@ -44,35 +44,35 @@ conversion.
     import pandas as pd
 
     df = pd.DataFrame({"a": [1, 2, 3]})
-    # Convert from Pandas to Arrow
+    # Convert from pandas to Arrow
     table = pa.Table.from_pandas(df)
-    # Convert back to Pandas
+    # Convert back to pandas
     df_new = table.to_pandas()
 
 
 Series
 ------
 
-In Arrow, the most similar structure to a Pandas Series is an Array.
+In Arrow, the most similar structure to a pandas Series is an Array.
 It is a vector that contains data of the same type as linear memory. You can
-convert a Pandas Series to an Arrow Array using :meth:`pyarrow.array.from_pandas_series`.
+convert a pandas Series to an Arrow Array using :meth:`pyarrow.array.from_pandas_series`.
 As Arrow Arrays are always nullable, you can supply an optional mask using
 the ``mask`` parameter to mark all null-entries.
 
 Type differences
 ----------------
 
-With the current design of Pandas and Arrow, it is not possible to convert all
-column types unmodified. One of the main issues here is that Pandas has no
+With the current design of pandas and Arrow, it is not possible to convert all
+column types unmodified. One of the main issues here is that pandas has no
 support for nullable columns of arbitrary type. Also ``datetime64`` is currently
 fixed to nanosecond resolution. On the other side, Arrow might be still missing
 support for some types.
 
-Pandas -> Arrow Conversion
+pandas -> Arrow Conversion
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 +------------------------+--------------------------+
-| Source Type (Pandas)   | Destination Type (Arrow) |
+| Source Type (pandas)   | Destination Type (Arrow) |
 +========================+==========================+
 | ``bool``               | ``BOOL``                 |
 +------------------------+--------------------------+
@@ -91,11 +91,11 @@ Pandas -> Arrow Conversion
 | ``datetime.date``      | ``DATE``                 |
 +------------------------+--------------------------+
 
-Arrow -> Pandas Conversion
+Arrow -> pandas Conversion
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 +-------------------------------------+--------------------------------------------------------+
-| Source Type (Arrow)                 | Destination Type (Pandas)                              |
+| Source Type (Arrow)                 | Destination Type (pandas)                              |
 +=====================================+========================================================+
 | ``BOOL``                            | ``bool``                                               |
 +-------------------------------------+--------------------------------------------------------+
diff --git a/python/doc/source/parquet.rst b/python/doc/source/parquet.rst
index 8e011e4f19857..3317b99c0b685 100644
--- a/python/doc/source/parquet.rst
+++ b/python/doc/source/parquet.rst
@@ -15,77 +15,218 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
-Reading/Writing Parquet files
-=============================
+.. currentmodule:: pyarrow
+.. _parquet:
 
-If you have built ``pyarrow`` with Parquet support, i.e. ``parquet-cpp`` was
-found during the build, you can read files in the Parquet format to/from Arrow
-memory structures. The Parquet support code is located in the
-:mod:`pyarrow.parquet` module and your package needs to be built with the
-``--with-parquet`` flag for ``build_ext``.
+Reading and Writing the Apache Parquet Format
+=============================================
 
-Reading Parquet
----------------
+The `Apache Parquet <http://parquet.apache.org/>`_ project provides a
+standardized open-source columnar storage format for use in data analysis
+systems. It was created originally for use in `Apache Hadoop
+<http://hadoop.apache.org/>`_ with systems like `Apache Drill
+<http://drill.apache.org>`_, `Apache Hive <http://hive.apache.org>`_, `Apache
+Impala (incubating) <http://impala.apache.org>`_, and `Apache Spark
+<http://spark.apache.org>`_ adopting it as a shared standard for high
+performance data IO.
 
-To read a Parquet file into Arrow memory, you can use the following code
-snippet. It will read the whole Parquet file into memory as an
-:class:`~pyarrow.table.Table`.
+Apache Arrow is an ideal in-memory transport layer for data that is being read
+or written with Parquet files. We have been concurrently developing the `C++
+implementation of Apache Parquet <http://github.com/apache/parquet-cpp>`_,
+which includes a native, multithreaded C++ adapter to and from in-memory Arrow
+data. PyArrow includes Python bindings to this code, which thus enables reading
+and writing Parquet files with pandas as well.
 
-.. code-block:: python
+Obtaining PyArrow with Parquet Support
+--------------------------------------
+
+If you installed ``pyarrow`` with pip or conda, it should be built with Parquet
+support bundled:
+
+.. ipython:: python
+
+   import pyarrow.parquet as pq
+
+If you are building ``pyarrow`` from source, you must also build `parquet-cpp
+<http://github.com/apache/parquet-cpp>`_ and enable the Parquet extensions when
+building ``pyarrow``. See the :ref:`Development <development>` page for more
+details.
+
+Reading and Writing Single Files
+--------------------------------
+
+The functions :func:`~.parquet.read_table` and :func:`~.parquet.write_table`
+read and write the :ref:`pyarrow.Table <data.table>` objects, respectively.
+
+Let's look at a simple table:
+
+.. ipython:: python
+
+   import numpy as np
+   import pandas as pd
+   import pyarrow as pa
+
+   df = pd.DataFrame({'one': [-1, np.nan, 2.5],
+                      'two': ['foo', 'bar', 'baz'],
+                      'three': [True, False, True]})
+   table = pa.Table.from_pandas(df)
+
+We write this to Parquet format with ``write_table``:
+
+.. ipython:: python
+
+   import pyarrow.parquet as pq
+   pq.write_table(table, 'example.parquet')
+
+This creates a single Parquet file. In practice, a Parquet dataset may consist
+of many files in many directories. We can read a single file back with
+``read_table``:
+
+.. ipython:: python
+
+   table2 = pq.read_table('example.parquet')
+   table2.to_pandas()
+
+You can pass a subset of columns to read, which can be much faster than reading
+the whole file (due to the columnar layout):
+
+.. ipython:: python
+
+   pq.read_table('example.parquet', columns=['one', 'three'])
+
+We need not use a string to specify the origin of the file. It can be any of:
+
+* A file path as a string
+* A :ref:`NativeFile <io.native_file>` from PyArrow
+* A Python file object
+
+In general, a Python file object will have the worst read performance, while a
+string file path or an instance of :class:`~.NativeFIle` (especially memory
+maps) will perform the best.
 
-    import pyarrow.parquet as pq
+Finer-grained Reading and Writing
+---------------------------------
 
-    table = pq.read_table('<filename>')
+``read_table`` uses the :class:`~.ParquetFile` class, which has other features:
 
-As DataFrames stored as Parquet are often stored in multiple files, a
-convenience method :meth:`~pyarrow.parquet.read_multiple_files` is provided.
+.. ipython:: python
 
-If you already have the Parquet available in memory or get it via non-file
-source, you can utilize :class:`pyarrow.io.BufferReader` to read it from
-memory. As input to the :class:`~pyarrow.io.BufferReader` you can either supply
-a Python ``bytes`` object or a :class:`pyarrow.io.Buffer`.
+   parquet_file = pq.ParquetFile('example.parquet')
+   parquet_file.metadata
+   parquet_file.schema
 
-.. code:: python
+As you can learn more in the `Apache Parquet format
+<https://github.com/apache/parquet-format>`_, a Parquet file consists of
+multiple row groups. ``read_table`` will read all of the row groups and
+concatenate them into a single table. You can read individual row groups with
+``read_row_group``:
 
-    import pyarrow.io as paio
-    import pyarrow.parquet as pq
+.. ipython:: python
 
-    buf = ... # either bytes or paio.Buffer
-    reader = paio.BufferReader(buf)
-    table = pq.read_table(reader)
+   parquet_file.num_row_groups
+   parquet_file.read_row_group(0)
 
-Writing Parquet
----------------
+We can similarly write a Parquet file with multiple row groups by using
+``ParquetWriter``:
 
-Given an instance of :class:`pyarrow.table.Table`, the most simple way to
-persist it to Parquet is by using the :meth:`pyarrow.parquet.write_table`
-method.
+.. ipython:: python
+
+   writer = pq.ParquetWriter('example2.parquet', table.schema)
+   for i in range(3):
+       writer.write_table(table)
+   writer.close()
+
+   pf2 = pq.ParquetFile('example2.parquet')
+   pf2.num_row_groups
+
+.. ipython:: python
+   :suppress:
+
+   !rm example.parquet
+   !rm example2.parquet
+
+Compression, Encoding, and File Compatibility
+---------------------------------------------
+
+The most commonly used Parquet implementations use dictionary encoding when
+writing files; if the dictionaries grow too large, then they "fall back" to
+plain encoding. Whether dictionary encoding is used can be toggled using the
+``use_dictionary`` option:
 
 .. code-block:: python
 
-    import pyarrow as pa
-    import pyarrow.parquet as pq
+   pq.write_table(table, where, use_dictionary=False)
 
-    table = pa.Table(..)
-    pq.write_table(table, '<filename>')
+The data pages within a column in a row group can be compressed after the
+encoding passes (dictionary, RLE encoding). In PyArrow we use Snappy
+compression by default, but Brotli, Gzip, and uncompressed are also supported:
 
-By default this will write the Table as a single RowGroup using ``DICTIONARY``
-encoding. To increase the potential of parallelism a query engine can process
-a Parquet file, set the ``chunk_size`` to a fraction of the total number of rows.
+.. code-block:: python
+
+   pq.write_table(table, where, compression='snappy')
+   pq.write_table(table, where, compression='gzip')
+   pq.write_table(table, where, compression='brotli')
+   pq.write_table(table, where, compression='none')
+
+Snappy generally results in better performance, while Gzip may yield smaller
+files.
+
+These settings can also be set on a per-column basis:
+
+.. code-block:: python
 
-If you also want to compress the columns, you can select a compression
-method using the ``compression`` argument. Typically, ``GZIP`` is the choice if
-you want to minimize size and ``SNAPPY`` for performance.
+   pa.write_table(table, where, compression={'foo': 'snappy', 'bar': 'gzip'},
+                  use_dictionary=['foo', 'bar'])
 
-Instead of writing to a file, you can also write to Python ``bytes`` by
-utilizing an :class:`pyarrow.io.InMemoryOutputStream()`:
+Reading Multiples Files and Partitioned Datasets
+------------------------------------------------
 
-.. code:: python
+Multiple Parquet files constitute a Parquet *dataset*. These may present in a
+number of ways:
 
-    import pyarrow.io as paio
-    import pyarrow.parquet as pq
+* A list of Parquet absolute file paths
+* A directory name containing nested directories defining a partitioned dataset
+
+A dataset partitioned by year and month may look like on disk:
+
+.. code-block:: text
+
+   dataset_name/
+     year=2007/
+       month=01/
+          0.parq
+          1.parq
+          ...
+       month=02/
+          0.parq
+          1.parq
+          ...
+       month=03/
+       ...
+     year=2008/
+       month=01/
+       ...
+     ...
+
+The :class:`~.ParquetDataset` class accepts either a directory name or a list
+or file paths, and can discover and infer some common partition structures,
+such as those produced by Hive:
+
+.. code-block:: python
+
+   dataset = pq.ParquetDataset('dataset_name/')
+   table = dataset.read()
+
+Multithreaded Reads
+-------------------
+
+Each of the reading functions have an ``nthreads`` argument which will read
+columns with the indicated level of parallelism. Depending on the speed of IO
+and how expensive it is to decode the columns in a particular file
+(particularly with GZIP compression), this can yield significantly higher data
+throughput:
+
+.. code-block:: python
 
-    table = ...
-    output = paio.InMemoryOutputStream()
-    pq.write_table(table, output)
-    pybytes = output.get_result().to_pybytes()
+   pq.read_table(where, nthreads=4)
+   pq.ParquetDataset(where).read(nthreads=4)
diff --git a/python/pyarrow/_io.pyx b/python/pyarrow/_io.pyx
index 40c76f8363cd2..e9e2ba01c0678 100644
--- a/python/pyarrow/_io.pyx
+++ b/python/pyarrow/_io.pyx
@@ -522,6 +522,7 @@ cdef class Buffer:
         buffer.strides = self.strides
         buffer.suboffsets = NULL
 
+
 cdef shared_ptr[PoolBuffer] allocate_buffer(CMemoryPool* pool):
     cdef shared_ptr[PoolBuffer] result
     result.reset(new PoolBuffer(pool))

From 3d19831717297e91a74e008d44c71695088b39fd Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 8 May 2017 20:10:12 +0200
Subject: [PATCH 0620/1644] ARROW-967: [GLib] Support initializing array with
 buffer

It's for zero-copy data conversion.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #657 from kou/glib-array-new-with-buffer and squashes the following commits:

57f4266 [Kouhei Sutou] [GLib] Support initializing array with buffer
---
 c_glib/arrow-glib/array.cpp       | 600 ++++++++++++++++++++++++++++--
 c_glib/arrow-glib/array.h         |  80 ++++
 c_glib/arrow-glib/buffer.cpp      |   3 +
 c_glib/test/helper/buildable.rb   |  55 +++
 c_glib/test/test-binary-array.rb  |  13 +
 c_glib/test/test-boolean-array.rb |  10 +
 c_glib/test/test-double-array.rb  |  10 +
 c_glib/test/test-float-array.rb   |  10 +
 c_glib/test/test-int16-array.rb   |  10 +
 c_glib/test/test-int32-array.rb   |  10 +
 c_glib/test/test-int64-array.rb   |  10 +
 c_glib/test/test-int8-array.rb    |  10 +
 c_glib/test/test-list-array.rb    |  15 +
 c_glib/test/test-string-array.rb  |  13 +
 c_glib/test/test-struct-array.rb  |  33 ++
 c_glib/test/test-uint16-array.rb  |  10 +
 c_glib/test/test-uint32-array.rb  |  10 +
 c_glib/test/test-uint64-array.rb  |  10 +
 c_glib/test/test-uint8-array.rb   |  10 +
 19 files changed, 889 insertions(+), 33 deletions(-)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 8a78984349c62..8bc6ea95d6a9d 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -39,73 +39,89 @@ G_BEGIN_DECLS
  * #GArrowArray is a base class for all array classes such as
  * #GArrowBooleanArray.
  *
- * All array classes are immutable. You need to use array builder
- * class such as #GArrowBooleanArrayBuilder to create a new array
- * except #GArrowNullArray.
+ * All array classes are immutable. You need to use binary data or
+ * array builder to create a new array except #GArrowNullArray. If you
+ * have binary data that uses Arrow format data, you can create a new
+ * array with the binary data as #GArrowBuffer object. If you don't
+ * have binary data, you can use array builder class such as
+ * #GArrowBooleanArrayBuilder that creates Arrow format data
+ * internally and a new array from the data.
  *
  * #GArrowNullArray is a class for null array. It can store zero or
  * more null values. You need to specify an array length to create a
  * new array.
  *
  * #GArrowBooleanArray is a class for binary array. It can store zero
- * or more boolean data. You need to use #GArrowBooleanArrayBuilder to
- * create a new array.
+ * or more boolean data. If you don't have Arrow format data, you need
+ * to use #GArrowBooleanArrayBuilder to create a new array.
  *
  * #GArrowInt8Array is a class for 8-bit integer array. It can store
- * zero or more 8-bit integer data. You need to use
- * #GArrowInt8ArrayBuilder to create a new array.
+ * zero or more 8-bit integer data. If you don't have Arrow format
+ * data, you need to use #GArrowInt8ArrayBuilder to create a new
+ * array.
  *
  * #GArrowUInt8Array is a class for 8-bit unsigned integer array. It
- * can store zero or more 8-bit unsigned integer data. You need to use
- * #GArrowUInt8ArrayBuilder to create a new array.
+ * can store zero or more 8-bit unsigned integer data. If you don't
+ * have Arrow format data, you need to use #GArrowUInt8ArrayBuilder to
+ * create a new array.
  *
  * #GArrowInt16Array is a class for 16-bit integer array. It can store
- * zero or more 16-bit integer data. You need to use
- * #GArrowInt16ArrayBuilder to create a new array.
+ * zero or more 16-bit integer data. If you don't have Arrow format
+ * data, you need to use #GArrowInt16ArrayBuilder to create a new
+ * array.
  *
  * #GArrowUInt16Array is a class for 16-bit unsigned integer array. It
- * can store zero or more 16-bit unsigned integer data. You need to use
- * #GArrowUInt16ArrayBuilder to create a new array.
+ * can store zero or more 16-bit unsigned integer data. If you don't
+ * have Arrow format data, you need to use #GArrowUInt16ArrayBuilder
+ * to create a new array.
  *
  * #GArrowInt32Array is a class for 32-bit integer array. It can store
- * zero or more 32-bit integer data. You need to use
- * #GArrowInt32ArrayBuilder to create a new array.
+ * zero or more 32-bit integer data. If you don't have Arrow format
+ * data, you need to use #GArrowInt32ArrayBuilder to create a new
+ * array.
  *
  * #GArrowUInt32Array is a class for 32-bit unsigned integer array. It
- * can store zero or more 32-bit unsigned integer data. You need to use
- * #GArrowUInt32ArrayBuilder to create a new array.
+ * can store zero or more 32-bit unsigned integer data. If you don't
+ * have Arrow format data, you need to use #GArrowUInt32ArrayBuilder
+ * to create a new array.
  *
  * #GArrowInt64Array is a class for 64-bit integer array. It can store
- * zero or more 64-bit integer data. You need to use
- * #GArrowInt64ArrayBuilder to create a new array.
+ * zero or more 64-bit integer data. If you don't have Arrow format
+ * data, you need to use #GArrowInt64ArrayBuilder to create a new
+ * array.
  *
  * #GArrowUInt64Array is a class for 64-bit unsigned integer array. It
- * can store zero or more 64-bit unsigned integer data. You need to
- * use #GArrowUInt64ArrayBuilder to create a new array.
+ * can store zero or more 64-bit unsigned integer data. If you don't
+ * have Arrow format data, you need to use #GArrowUInt64ArrayBuilder
+ * to create a new array.
  *
  * #GArrowFloatArray is a class for 32-bit floating point array. It
- * can store zero or more 32-bit floating data. You need to use
- * #GArrowFloatArrayBuilder to create a new array.
+ * can store zero or more 32-bit floating data. If you don't have
+ * Arrow format data, you need to use #GArrowFloatArrayBuilder to
+ * create a new array.
  *
  * #GArrowDoubleArray is a class for 64-bit floating point array. It
- * can store zero or more 64-bit floating data. You need to use
- * #GArrowDoubleArrayBuilder to create a new array.
+ * can store zero or more 64-bit floating data. If you don't have
+ * Arrow format data, you need to use #GArrowDoubleArrayBuilder to
+ * create a new array.
  *
  * #GArrowBinaryArray is a class for binary array. It can store zero
- * or more binary data. You need to use #GArrowBinaryArrayBuilder to
- * create a new array.
+ * or more binary data. If you don't have Arrow format data, you need
+ * to use #GArrowBinaryArrayBuilder to create a new array.
  *
  * #GArrowStringArray is a class for UTF-8 encoded string array. It
- * can store zero or more UTF-8 encoded string data. You need to use
- * #GArrowStringArrayBuilder to create a new array.
+ * can store zero or more UTF-8 encoded string data. If you don't have
+ * Arrow format data, you need to use #GArrowStringArrayBuilder to
+ * create a new array.
  *
  * #GArrowListArray is a class for list array. It can store zero or
- * more list data. You need to use #GArrowListArrayBuilder to create a
- * new array.
+ * more list data. If you don't have Arrow format data, you need to
+ * use #GArrowListArrayBuilder to create a new array.
  *
  * #GArrowStructArray is a class for struct array. It can store zero
- * or more structs. One struct has zero or more fields. You need to
- * use #GArrowStructArrayBuilder to create a new array.
+ * or more structs. One struct has zero or more fields. If you don't
+ * have Arrow format data, you need to use #GArrowStructArrayBuilder
+ * to create a new array.
  */
 
 typedef struct GArrowArrayPrivate_ {
@@ -455,6 +471,39 @@ garrow_boolean_array_class_init(GArrowBooleanArrayClass *klass)
 {
 }
 
+/**
+ * garrow_boolean_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowBooleanArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowBooleanArray *
+garrow_boolean_array_new(gint64 length,
+                         GArrowBuffer *data,
+                         GArrowBuffer *null_bitmap,
+                         gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_boolean_array =
+    std::make_shared<arrow::BooleanArray>(length,
+                                          arrow_data,
+                                          arrow_bitmap,
+                                          n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_boolean_array);
+  return GARROW_BOOLEAN_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_boolean_array_get_value:
  * @array: A #GArrowBooleanArray.
@@ -485,6 +534,39 @@ garrow_int8_array_class_init(GArrowInt8ArrayClass *klass)
 {
 }
 
+/**
+ * garrow_int8_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowInt8Array.
+ *
+ * Since: 0.4.0
+ */
+GArrowInt8Array *
+garrow_int8_array_new(gint64 length,
+                      GArrowBuffer *data,
+                      GArrowBuffer *null_bitmap,
+                      gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_int8_array =
+    std::make_shared<arrow::Int8Array>(length,
+                                       arrow_data,
+                                       arrow_bitmap,
+                                       n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_int8_array);
+  return GARROW_INT8_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_int8_array_get_value:
  * @array: A #GArrowInt8Array.
@@ -515,6 +597,39 @@ garrow_uint8_array_class_init(GArrowUInt8ArrayClass *klass)
 {
 }
 
+/**
+ * garrow_uint8_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowUInt8Array.
+ *
+ * Since: 0.4.0
+ */
+GArrowUInt8Array *
+garrow_uint8_array_new(gint64 length,
+                       GArrowBuffer *data,
+                       GArrowBuffer *null_bitmap,
+                       gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_uint8_array =
+    std::make_shared<arrow::UInt8Array>(length,
+                                        arrow_data,
+                                        arrow_bitmap,
+                                        n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_uint8_array);
+  return GARROW_UINT8_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_uint8_array_get_value:
  * @array: A #GArrowUInt8Array.
@@ -545,6 +660,39 @@ garrow_int16_array_class_init(GArrowInt16ArrayClass *klass)
 {
 }
 
+/**
+ * garrow_int16_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowInt16Array.
+ *
+ * Since: 0.4.0
+ */
+GArrowInt16Array *
+garrow_int16_array_new(gint64 length,
+                       GArrowBuffer *data,
+                       GArrowBuffer *null_bitmap,
+                       gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_int16_array =
+    std::make_shared<arrow::Int16Array>(length,
+                                        arrow_data,
+                                        arrow_bitmap,
+                                        n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_int16_array);
+  return GARROW_INT16_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_int16_array_get_value:
  * @array: A #GArrowInt16Array.
@@ -575,6 +723,39 @@ garrow_uint16_array_class_init(GArrowUInt16ArrayClass *klass)
 {
 }
 
+/**
+ * garrow_uint16_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowUInt16Array.
+ *
+ * Since: 0.4.0
+ */
+GArrowUInt16Array *
+garrow_uint16_array_new(gint64 length,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_uint16_array =
+    std::make_shared<arrow::UInt16Array>(length,
+                                         arrow_data,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_uint16_array);
+  return GARROW_UINT16_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_uint16_array_get_value:
  * @array: A #GArrowUInt16Array.
@@ -605,6 +786,39 @@ garrow_int32_array_class_init(GArrowInt32ArrayClass *klass)
 {
 }
 
+/**
+ * garrow_int32_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowInt32Array.
+ *
+ * Since: 0.4.0
+ */
+GArrowInt32Array *
+garrow_int32_array_new(gint64 length,
+                       GArrowBuffer *data,
+                       GArrowBuffer *null_bitmap,
+                       gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_int32_array =
+    std::make_shared<arrow::Int32Array>(length,
+                                        arrow_data,
+                                        arrow_bitmap,
+                                        n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_int32_array);
+  return GARROW_INT32_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_int32_array_get_value:
  * @array: A #GArrowInt32Array.
@@ -635,6 +849,39 @@ garrow_uint32_array_class_init(GArrowUInt32ArrayClass *klass)
 {
 }
 
+/**
+ * garrow_uint32_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowUInt32Array.
+ *
+ * Since: 0.4.0
+ */
+GArrowUInt32Array *
+garrow_uint32_array_new(gint64 length,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_uint32_array =
+    std::make_shared<arrow::UInt32Array>(length,
+                                         arrow_data,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_uint32_array);
+  return GARROW_UINT32_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_uint32_array_get_value:
  * @array: A #GArrowUInt32Array.
@@ -665,6 +912,39 @@ garrow_int64_array_class_init(GArrowInt64ArrayClass *klass)
 {
 }
 
+/**
+ * garrow_int64_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowInt64Array.
+ *
+ * Since: 0.4.0
+ */
+GArrowInt64Array *
+garrow_int64_array_new(gint64 length,
+                       GArrowBuffer *data,
+                       GArrowBuffer *null_bitmap,
+                       gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_int64_array =
+    std::make_shared<arrow::Int64Array>(length,
+                                        arrow_data,
+                                        arrow_bitmap,
+                                        n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_int64_array);
+  return GARROW_INT64_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_int64_array_get_value:
  * @array: A #GArrowInt64Array.
@@ -695,6 +975,39 @@ garrow_uint64_array_class_init(GArrowUInt64ArrayClass *klass)
 {
 }
 
+/**
+ * garrow_uint64_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowUInt64Array.
+ *
+ * Since: 0.4.0
+ */
+GArrowUInt64Array *
+garrow_uint64_array_new(gint64 length,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_uint64_array =
+    std::make_shared<arrow::UInt64Array>(length,
+                                         arrow_data,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_uint64_array);
+  return GARROW_UINT64_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_uint64_array_get_value:
  * @array: A #GArrowUInt64Array.
@@ -724,6 +1037,39 @@ garrow_float_array_class_init(GArrowFloatArrayClass *klass)
 {
 }
 
+/**
+ * garrow_float_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowFloatArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowFloatArray *
+garrow_float_array_new(gint64 length,
+                       GArrowBuffer *data,
+                       GArrowBuffer *null_bitmap,
+                       gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_float_array =
+    std::make_shared<arrow::FloatArray>(length,
+                                        arrow_data,
+                                        arrow_bitmap,
+                                        n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_float_array);
+  return GARROW_FLOAT_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_float_array_get_value:
  * @array: A #GArrowFloatArray.
@@ -754,6 +1100,39 @@ garrow_double_array_class_init(GArrowDoubleArrayClass *klass)
 {
 }
 
+/**
+ * garrow_double_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowDoubleArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowDoubleArray *
+garrow_double_array_new(gint64 length,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_double_array =
+    std::make_shared<arrow::DoubleArray>(length,
+                                         arrow_data,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_double_array);
+  return GARROW_DOUBLE_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_double_array_get_value:
  * @array: A #GArrowDoubleArray.
@@ -784,6 +1163,43 @@ garrow_binary_array_class_init(GArrowBinaryArrayClass *klass)
 {
 }
 
+/**
+ * garrow_binary_array_new:
+ * @length: The number of elements.
+ * @value_offsets: The value offsets of @data in Arrow format.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowBinaryArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowBinaryArray *
+garrow_binary_array_new(gint64 length,
+                        GArrowBuffer *value_offsets,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets);
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_binary_array =
+    std::make_shared<arrow::BinaryArray>(length,
+                                         arrow_value_offsets,
+                                         arrow_data,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_binary_array);
+  return GARROW_BINARY_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_binary_array_get_value:
  * @array: A #GArrowBinaryArray.
@@ -835,6 +1251,43 @@ garrow_string_array_class_init(GArrowStringArrayClass *klass)
 {
 }
 
+/**
+ * garrow_string_array_new:
+ * @length: The number of elements.
+ * @value_offsets: The value offsets of @data in Arrow format.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowStringArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowStringArray *
+garrow_string_array_new(gint64 length,
+                        GArrowBuffer *value_offsets,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets);
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_string_array =
+    std::make_shared<arrow::StringArray>(length,
+                                         arrow_value_offsets,
+                                         arrow_data,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_string_array);
+  return GARROW_STRING_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_string_array_get_string:
  * @array: A #GArrowStringArray.
@@ -870,6 +1323,45 @@ garrow_list_array_class_init(GArrowListArrayClass *klass)
 {
 }
 
+/**
+ * garrow_list_array_new:
+ * @length: The number of elements.
+ * @value_offsets: The offsets of @values in Arrow format.
+ * @values: The values as #GArrowArray.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowListArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowListArray *
+garrow_list_array_new(gint64 length,
+                      GArrowBuffer *value_offsets,
+                      GArrowArray *values,
+                      GArrowBuffer *null_bitmap,
+                      gint64 n_nulls)
+{
+  const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets);
+  const auto arrow_values = garrow_array_get_raw(values);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_data_type = arrow::list(arrow_values->type());
+  auto arrow_list_array =
+    std::make_shared<arrow::ListArray>(arrow_data_type,
+                                       length,
+                                       arrow_value_offsets,
+                                       arrow_values,
+                                       arrow_bitmap,
+                                       n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_list_array);
+  return GARROW_LIST_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_list_array_get_value_type:
  * @array: A #GArrowListArray.
@@ -921,6 +1413,48 @@ garrow_struct_array_class_init(GArrowStructArrayClass *klass)
 {
 }
 
+/**
+ * garrow_struct_array_new:
+ * @data_type: The data type of the struct.
+ * @length: The number of elements.
+ * @children: (element-type GArrowArray): The arrays for each field
+ *   as #GList of #GArrowArray.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowStructArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowStructArray *
+garrow_struct_array_new(GArrowDataType *data_type,
+                        gint64 length,
+                        GList *children,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  std::vector<std::shared_ptr<arrow::Array>> arrow_children;
+  for (GList *node = children; node; node = node->next) {
+    GArrowArray *child = GARROW_ARRAY(node->data);
+    arrow_children.push_back(garrow_array_get_raw(child));
+  }
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_struct_array =
+    std::make_shared<arrow::StructArray>(arrow_data_type,
+                                         length,
+                                         arrow_children,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_struct_array);
+  return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
 /**
  * garrow_struct_array_get_field
  * @array: A #GArrowStructArray.
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index f750ee10f8cbe..c4efeafd6404a 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -211,6 +211,12 @@ struct _GArrowBooleanArrayClass
 };
 
 GType          garrow_boolean_array_get_type  (void) G_GNUC_CONST;
+
+GArrowBooleanArray *garrow_boolean_array_new(gint64 length,
+                                             GArrowBuffer *data,
+                                             GArrowBuffer *null_bitmap,
+                                             gint64 n_nulls);
+
 gboolean       garrow_boolean_array_get_value (GArrowBooleanArray *array,
                                                gint64 i);
 
@@ -257,6 +263,11 @@ struct _GArrowInt8ArrayClass
 
 GType garrow_int8_array_get_type(void) G_GNUC_CONST;
 
+GArrowInt8Array *garrow_int8_array_new(gint64 length,
+                                       GArrowBuffer *data,
+                                       GArrowBuffer *null_bitmap,
+                                       gint64 n_nulls);
+
 gint8 garrow_int8_array_get_value(GArrowInt8Array *array,
                                   gint64 i);
 
@@ -303,6 +314,11 @@ struct _GArrowUInt8ArrayClass
 
 GType garrow_uint8_array_get_type(void) G_GNUC_CONST;
 
+GArrowUInt8Array *garrow_uint8_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
 guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array,
                                     gint64 i);
 
@@ -349,6 +365,11 @@ struct _GArrowInt16ArrayClass
 
 GType garrow_int16_array_get_type(void) G_GNUC_CONST;
 
+GArrowInt16Array *garrow_int16_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
 gint16 garrow_int16_array_get_value(GArrowInt16Array *array,
                                     gint64 i);
 
@@ -395,6 +416,11 @@ struct _GArrowUInt16ArrayClass
 
 GType garrow_uint16_array_get_type(void) G_GNUC_CONST;
 
+GArrowUInt16Array *garrow_uint16_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
 guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array,
                                       gint64 i);
 
@@ -441,6 +467,11 @@ struct _GArrowInt32ArrayClass
 
 GType garrow_int32_array_get_type(void) G_GNUC_CONST;
 
+GArrowInt32Array *garrow_int32_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
 gint32 garrow_int32_array_get_value(GArrowInt32Array *array,
                                     gint64 i);
 
@@ -487,6 +518,11 @@ struct _GArrowUInt32ArrayClass
 
 GType garrow_uint32_array_get_type(void) G_GNUC_CONST;
 
+GArrowUInt32Array *garrow_uint32_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
 guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array,
                                       gint64 i);
 
@@ -533,6 +569,11 @@ struct _GArrowInt64ArrayClass
 
 GType garrow_int64_array_get_type(void) G_GNUC_CONST;
 
+GArrowInt64Array *garrow_int64_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
 gint64 garrow_int64_array_get_value(GArrowInt64Array *array,
                                     gint64 i);
 
@@ -579,6 +620,11 @@ struct _GArrowUInt64ArrayClass
 
 GType garrow_uint64_array_get_type(void) G_GNUC_CONST;
 
+GArrowUInt64Array *garrow_uint64_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
 guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array,
                                       gint64 i);
 
@@ -625,6 +671,11 @@ struct _GArrowFloatArrayClass
 
 GType garrow_float_array_get_type(void) G_GNUC_CONST;
 
+GArrowFloatArray *garrow_float_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
 gfloat garrow_float_array_get_value(GArrowFloatArray *array,
                                     gint64 i);
 
@@ -671,6 +722,11 @@ struct _GArrowDoubleArrayClass
 
 GType garrow_double_array_get_type(void) G_GNUC_CONST;
 
+GArrowDoubleArray *garrow_double_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
 gdouble garrow_double_array_get_value(GArrowDoubleArray *array,
                                       gint64 i);
 
@@ -717,6 +773,12 @@ struct _GArrowBinaryArrayClass
 
 GType garrow_binary_array_get_type(void) G_GNUC_CONST;
 
+GArrowBinaryArray *garrow_binary_array_new(gint64 length,
+                                           GArrowBuffer *value_offsets,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
 GBytes *garrow_binary_array_get_value(GArrowBinaryArray *array,
                                       gint64 i);
 GArrowBuffer *garrow_binary_array_get_buffer(GArrowBinaryArray *array);
@@ -763,6 +825,12 @@ struct _GArrowStringArrayClass
 
 GType garrow_string_array_get_type(void) G_GNUC_CONST;
 
+GArrowStringArray *garrow_string_array_new(gint64 length,
+                                           GArrowBuffer *value_offsets,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
 gchar *garrow_string_array_get_string(GArrowStringArray *array,
                                       gint64 i);
 
@@ -809,6 +877,12 @@ struct _GArrowListArrayClass
 
 GType garrow_list_array_get_type(void) G_GNUC_CONST;
 
+GArrowListArray *garrow_list_array_new(gint64 length,
+                                       GArrowBuffer *value_offsets,
+                                       GArrowArray *values,
+                                       GArrowBuffer *null_bitmap,
+                                       gint64 n_nulls);
+
 GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array);
 GArrowArray *garrow_list_array_get_value(GArrowListArray *array,
                                          gint64 i);
@@ -856,6 +930,12 @@ struct _GArrowStructArrayClass
 
 GType garrow_struct_array_get_type(void) G_GNUC_CONST;
 
+GArrowStructArray *garrow_struct_array_new(GArrowDataType *data_type,
+                                           gint64 length,
+                                           GList *children,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
 GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
                                            gint i);
 GList *garrow_struct_array_get_fields(GArrowStructArray *array);
diff --git a/c_glib/arrow-glib/buffer.cpp b/c_glib/arrow-glib/buffer.cpp
index 0970128ae3862..4be8fed18ea01 100644
--- a/c_glib/arrow-glib/buffer.cpp
+++ b/c_glib/arrow-glib/buffer.cpp
@@ -504,6 +504,9 @@ garrow_buffer_new_raw(std::shared_ptr<arrow::Buffer> *arrow_buffer)
 std::shared_ptr<arrow::Buffer>
 garrow_buffer_get_raw(GArrowBuffer *buffer)
 {
+  if (!buffer)
+    return nullptr;
+
   auto priv = GARROW_BUFFER_GET_PRIVATE(buffer);
   return priv->buffer;
 }
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index 900e180675b45..4120eed6fe2d0 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -61,6 +61,61 @@ def build_double_array(values)
       build_array(Arrow::DoubleArrayBuilder, values)
     end
 
+    def build_binary_array(values)
+      build_array(Arrow::BinaryArrayBuilder, values)
+    end
+
+    def build_string_array(values)
+      build_array(Arrow::StringArrayBuilder, values)
+    end
+
+    def build_list_array(value_builder_class, values_list)
+      value_builder = value_builder_class.new
+      builder = Arrow::ListArrayBuilder.new(value_builder)
+      values_list.each do |values|
+        if values.nil?
+          builder.append_null
+        else
+          builder.append
+          values.each do |value|
+            if value.nil?
+              value_builder.append_null
+            else
+              value_builder.append(value)
+            end
+          end
+        end
+      end
+      builder.finish
+    end
+
+    def build_struct_array(fields, structs)
+      field_builders = fields.collect do |field|
+        data_type_name = field.data_type.class.name
+        builder_name = data_type_name.gsub(/DataType/, "ArrayBuilder")
+        Arrow.const_get(builder_name).new
+      end
+      data_type = Arrow::StructDataType.new(fields)
+      builder = Arrow::StructArrayBuilder.new(data_type, field_builders)
+      structs.each do |struct|
+        if struct.nil?
+          builder.append_null
+        else
+          builder.append
+          struct.each do |name, value|
+            field_builder_index = fields.index {|field| field.name == name}
+            field_builder = builder.get_field_builder(field_builder_index)
+            if value.nil?
+              field_builder.append_null
+            else
+              field_builder.append(value)
+            end
+          end
+        end
+      end
+      builder.finish
+    end
+
     private
     def build_array(builder_class, values)
       builder = builder_class.new
diff --git a/c_glib/test/test-binary-array.rb b/c_glib/test/test-binary-array.rb
index ccdf378ad41b9..9ae122a9a742b 100644
--- a/c_glib/test/test-binary-array.rb
+++ b/c_glib/test/test-binary-array.rb
@@ -16,6 +16,19 @@
 # under the License.
 
 class TestBinaryArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    value_offsets = Arrow::Buffer.new([0, 2, 5, 5].pack("l*"))
+    data = Arrow::Buffer.new("\x00\x01\x02\x03\x04")
+    assert_equal(build_binary_array(["\x00\x01", "\x02\x03\x04", nil]),
+                 Arrow::BinaryArray.new(3,
+                                        value_offsets,
+                                        data,
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
   def test_value
     data = "\x00\x01\x02"
     builder = Arrow::BinaryArrayBuilder.new
diff --git a/c_glib/test/test-boolean-array.rb b/c_glib/test/test-boolean-array.rb
index 15df1ed95b274..43b83655638e3 100644
--- a/c_glib/test/test-boolean-array.rb
+++ b/c_glib/test/test-boolean-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestBooleanArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_boolean_array([true, false, nil]),
+                 Arrow::BooleanArray.new(3,
+                                         Arrow::Buffer.new([0b001].pack("C*")),
+                                         Arrow::Buffer.new([0b011].pack("C*")),
+                                         -1))
+  end
+
   def test_buffer
     builder = Arrow::BooleanArrayBuilder.new
     builder.append(true)
diff --git a/c_glib/test/test-double-array.rb b/c_glib/test/test-double-array.rb
index c644ac6cc0c07..935fbe5b93dd9 100644
--- a/c_glib/test/test-double-array.rb
+++ b/c_glib/test/test-double-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestDoubleArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_double_array([-1.1, 2.2, nil]),
+                 Arrow::DoubleArray.new(3,
+                                        Arrow::Buffer.new([-1.1, 2.2].pack("d*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
   def test_buffer
     builder = Arrow::DoubleArrayBuilder.new
     builder.append(-1.1)
diff --git a/c_glib/test/test-float-array.rb b/c_glib/test/test-float-array.rb
index 84876f9754da7..fcac9021e56d2 100644
--- a/c_glib/test/test-float-array.rb
+++ b/c_glib/test/test-float-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestFloatArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_float_array([-1.1, 2.2, nil]),
+                 Arrow::FloatArray.new(3,
+                                       Arrow::Buffer.new([-1.1, 2.2].pack("f*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
   def test_buffer
     builder = Arrow::FloatArrayBuilder.new
     builder.append(-1.1)
diff --git a/c_glib/test/test-int16-array.rb b/c_glib/test/test-int16-array.rb
index 4b30ddd99ff9b..6bc7f8815c26e 100644
--- a/c_glib/test/test-int16-array.rb
+++ b/c_glib/test/test-int16-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestInt16Array < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_int16_array([-1, 2, nil]),
+                 Arrow::Int16Array.new(3,
+                                       Arrow::Buffer.new([-1, 2].pack("s*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
   def test_buffer
     builder = Arrow::Int16ArrayBuilder.new
     builder.append(-1)
diff --git a/c_glib/test/test-int32-array.rb b/c_glib/test/test-int32-array.rb
index 90cf0224c1c30..0b68273aca7dd 100644
--- a/c_glib/test/test-int32-array.rb
+++ b/c_glib/test/test-int32-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestInt32Array < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_int32_array([-1, 2, nil]),
+                 Arrow::Int32Array.new(3,
+                                       Arrow::Buffer.new([-1, 2].pack("l*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
   def test_buffer
     builder = Arrow::Int32ArrayBuilder.new
     builder.append(-1)
diff --git a/c_glib/test/test-int64-array.rb b/c_glib/test/test-int64-array.rb
index d3022017bb0ee..c2174345746c2 100644
--- a/c_glib/test/test-int64-array.rb
+++ b/c_glib/test/test-int64-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestInt64Array < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_int64_array([-1, 2, nil]),
+                 Arrow::Int64Array.new(3,
+                                       Arrow::Buffer.new([-1, 2].pack("q*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
   def test_buffer
     builder = Arrow::Int64ArrayBuilder.new
     builder.append(-1)
diff --git a/c_glib/test/test-int8-array.rb b/c_glib/test/test-int8-array.rb
index 9f28fa7fcd3a3..c7ff2165056cb 100644
--- a/c_glib/test/test-int8-array.rb
+++ b/c_glib/test/test-int8-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestInt8Array < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_int8_array([-1, 2, nil]),
+                 Arrow::Int8Array.new(3,
+                                      Arrow::Buffer.new([-1, 2].pack("c*")),
+                                      Arrow::Buffer.new([0b011].pack("C*")),
+                                      -1))
+  end
+
   def test_buffer
     builder = Arrow::Int8ArrayBuilder.new
     builder.append(-1)
diff --git a/c_glib/test/test-list-array.rb b/c_glib/test/test-list-array.rb
index 34177de9dcdeb..8e481e8367053 100644
--- a/c_glib/test/test-list-array.rb
+++ b/c_glib/test/test-list-array.rb
@@ -16,6 +16,21 @@
 # under the License.
 
 class TestListArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    value_offsets = Arrow::Buffer.new([0, 2, 5, 5].pack("l*"))
+    data = Arrow::Buffer.new([1, 2, 3, 4, 5].pack("c*"))
+    values = Arrow::Int8Array.new(5, data, nil, 0)
+    assert_equal(build_list_array(Arrow::Int8ArrayBuilder,
+                                  [[1, 2], [3, 4, 5], nil]),
+                 Arrow::ListArray.new(3,
+                                      value_offsets,
+                                      values,
+                                      Arrow::Buffer.new([0b011].pack("C*")),
+                                      -1))
+  end
+
   def test_value
     builder = Arrow::ListArrayBuilder.new(Arrow::Int8ArrayBuilder.new)
     value_builder = builder.value_builder
diff --git a/c_glib/test/test-string-array.rb b/c_glib/test/test-string-array.rb
index a076c228e0a4f..a9edb0ae49152 100644
--- a/c_glib/test/test-string-array.rb
+++ b/c_glib/test/test-string-array.rb
@@ -16,6 +16,19 @@
 # under the License.
 
 class TestStringArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    value_offsets = Arrow::Buffer.new([0, 5, 11, 11].pack("l*"))
+    data = Arrow::Buffer.new("HelloWorld!")
+    assert_equal(build_string_array(["Hello", "World!", nil]),
+                 Arrow::StringArray.new(3,
+                                        value_offsets,
+                                        data,
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
   def test_value
     builder = Arrow::StringArrayBuilder.new
     builder.append("Hello")
diff --git a/c_glib/test/test-struct-array.rb b/c_glib/test/test-struct-array.rb
index cf450f52d299a..ef0bc7179f290 100644
--- a/c_glib/test/test-struct-array.rb
+++ b/c_glib/test/test-struct-array.rb
@@ -16,6 +16,39 @@
 # under the License.
 
 class TestStructArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    fields = [
+      Arrow::Field.new("score", Arrow::Int8DataType.new),
+      Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
+    ]
+    structs = [
+      {
+        "score" => -29,
+        "enabled" => true,
+      },
+      {
+        "score" => 2,
+        "enabled" => false,
+      },
+      nil,
+    ]
+    struct_array1 = build_struct_array(fields, structs)
+
+    data_type = Arrow::StructDataType.new(fields)
+    children = [
+      Arrow::Int8Array.new(2, Arrow::Buffer.new([-29, 2].pack("C*")), nil, 0),
+      Arrow::BooleanArray.new(2, Arrow::Buffer.new([0b01].pack("C*")), nil, 0),
+    ]
+    assert_equal(struct_array1,
+                 Arrow::StructArray.new(data_type,
+                                        3,
+                                        children,
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
   def test_fields
     fields = [
       Arrow::Field.new("score", Arrow::Int8DataType.new),
diff --git a/c_glib/test/test-uint16-array.rb b/c_glib/test/test-uint16-array.rb
index 82e898e733625..e3ffa5d28b6fa 100644
--- a/c_glib/test/test-uint16-array.rb
+++ b/c_glib/test/test-uint16-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestUInt16Array < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_uint16_array([1, 2, nil]),
+                 Arrow::UInt16Array.new(3,
+                                        Arrow::Buffer.new([1, 2].pack("S*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
   def test_buffer
     builder = Arrow::UInt16ArrayBuilder.new
     builder.append(1)
diff --git a/c_glib/test/test-uint32-array.rb b/c_glib/test/test-uint32-array.rb
index c8be06fead5b9..95aee79921929 100644
--- a/c_glib/test/test-uint32-array.rb
+++ b/c_glib/test/test-uint32-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestUInt32Array < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_uint32_array([1, 2, nil]),
+                 Arrow::UInt32Array.new(3,
+                                        Arrow::Buffer.new([1, 2].pack("L*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
   def test_buffer
     builder = Arrow::UInt32ArrayBuilder.new
     builder.append(1)
diff --git a/c_glib/test/test-uint64-array.rb b/c_glib/test/test-uint64-array.rb
index 03082f33014ce..7d9185459b295 100644
--- a/c_glib/test/test-uint64-array.rb
+++ b/c_glib/test/test-uint64-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestUInt64Array < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_uint64_array([1, 2, nil]),
+                 Arrow::UInt64Array.new(3,
+                                        Arrow::Buffer.new([1, 2].pack("Q*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
   def test_buffer
     builder = Arrow::UInt64ArrayBuilder.new
     builder.append(1)
diff --git a/c_glib/test/test-uint8-array.rb b/c_glib/test/test-uint8-array.rb
index d7464e336da79..9c93abe7c349e 100644
--- a/c_glib/test/test-uint8-array.rb
+++ b/c_glib/test/test-uint8-array.rb
@@ -16,6 +16,16 @@
 # under the License.
 
 class TestUInt8Array < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_new
+    assert_equal(build_uint8_array([1, 2, nil]),
+                 Arrow::UInt8Array.new(3,
+                                       Arrow::Buffer.new([1, 2].pack("C*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
   def test_buffer
     builder = Arrow::UInt8ArrayBuilder.new
     builder.append(1)

From fe945a276206d597039f027004c14e141fffa0f5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 8 May 2017 15:22:41 -0400
Subject: [PATCH 0621/1644] ARROW-965: Website updates for 0.3.0

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #658 from wesm/ARROW-965 and squashes the following commits:

61c69d1 [Wes McKinney] Add license header
9e2c969 [Wes McKinney] Fix post template
5a55645 [Wes McKinney] Add Release page with changelog
7e674e1 [Wes McKinney] Website updates for 0.3.0
---
 NOTICE.txt                            |   2 +
 site/_config.yml                      |   2 +
 site/_data/contributors.yml           |  32 +++
 site/_includes/blog_entry.html        |  20 +-
 site/_includes/header.html            |   1 +
 site/_layouts/post.html               |  20 +-
 site/_posts/2017-05-08-0.3-release.md |   4 +-
 site/_release/0.1.0.md                | 222 +++++++++++++++++
 site/_release/0.2.0.md                | 238 ++++++++++++++++++
 site/_release/0.3.0.md                | 343 ++++++++++++++++++++++++++
 site/_release/index.md                |  35 +++
 site/install.html                     |  11 -
 site/install.md                       |  75 ++++++
 13 files changed, 990 insertions(+), 15 deletions(-)
 create mode 100644 site/_data/contributors.yml
 create mode 100644 site/_release/0.1.0.md
 create mode 100644 site/_release/0.2.0.md
 create mode 100644 site/_release/0.3.0.md
 create mode 100644 site/_release/index.md
 delete mode 100644 site/install.html
 create mode 100644 site/install.md

diff --git a/NOTICE.txt b/NOTICE.txt
index e71835c233de6..c02e75f91d966 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -46,6 +46,8 @@ This product includes software from the Ibis project (Apache 2.0)
  * Copyright (c) 2015 Cloudera, Inc.
  * https://github.com/cloudera/ibis
 
+The web site includes files generated by Jekyll.
+
 --------------------------------------------------------------------------------
 
 This product includes code from Apache Kudu, which includes the following in
diff --git a/site/_config.yml b/site/_config.yml
index d7e0bb37e3eb0..8bb969abe848c 100644
--- a/site/_config.yml
+++ b/site/_config.yml
@@ -30,6 +30,8 @@ exclude:
 collections:
   docs:
     output: true
+  release:
+    output: true
 
 sass:
   style: compressed
diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml
new file mode 100644
index 0000000000000..7bed83d21823d
--- /dev/null
+++ b/site/_data/contributors.yml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Database of contributors to Apache Arrow (WIP)
+# Blogs and other pages use this data
+#
+- name: Wes McKinney
+  apacheId: wesm
+  githubId: wesm
+  homepage: http://wesmckinney.com
+  role: PMC
+- name: Uwe Korn
+  apacheId: uwe
+  githubId: xhochy
+  role: PMC
+- name: Julien Le Dem
+  apacheId: julienledem
+  githubId: julienledem
+  role: PMC
+# End contributors.yml
diff --git a/site/_includes/blog_entry.html b/site/_includes/blog_entry.html
index cdc0060669c2f..ffcbd7a7ae208 100644
--- a/site/_includes/blog_entry.html
+++ b/site/_includes/blog_entry.html
@@ -20,6 +20,24 @@ <h2>
       <a href="{{ post.url }}" class="permalink" title="Permalink">∞</a>
     </h2>
 
+    {% capture discard %}
+      {% for c in site.data.contributors %}
+        {% if c.apacheId == post.author %}
+          {% assign author_name = c.name %}
+          {% if c.homepage %}
+            {% assign homepage = c.homepage %}
+          {% else %}
+            {% capture homepage %}http://github.com/{{ c.githubId }}{% endcapture %}
+          {% endif %}
+          {% if c.avatar %}
+            {% assign avatar = c.avatar %}
+          {% else %}
+            {% capture avatar %}http://github.com/{{ c.githubId }}.png{% endcapture %}
+          {% endif %}
+        {% endif %}
+      {% endfor %}
+    {% endcapture %}{% assign discard = nil %}
+
     <div class="panel">
       <div class="panel-body">
         <div>
@@ -31,7 +49,7 @@ <h2>
         </div>
         <div>
           <span class="label label-default">By</span>
-          <a href="{{ homepage }}"><i class="fa fa-user"></i> {{ post.author }}</a>
+          <a href="{{ homepage }}"><i class="fa fa-user"></i> {{ author_name }} ({{ post.author }})</a>
         </div>
       </div>
     </div>
diff --git a/site/_includes/header.html b/site/_includes/header.html
index d1526f69faf16..6c0ec30f39ca7 100644
--- a/site/_includes/header.html
+++ b/site/_includes/header.html
@@ -21,6 +21,7 @@
           <ul class="dropdown-menu">
             <li><a href="{{ site.baseurl }}/install/">Install</a></li>
             <li><a href="{{ site.baseurl }}/blog/">Blog</a></li>
+            <li><a href="{{ site.baseurl }}/release/">Releases</a></li>
             <li><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a></li>
             <li><a href="https://github.com/apache/arrow">Source Code</a></li>
             <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Mailing List</a></li>
diff --git a/site/_layouts/post.html b/site/_layouts/post.html
index 53de1fcfcedd5..8b663cdeae22a 100644
--- a/site/_layouts/post.html
+++ b/site/_layouts/post.html
@@ -10,6 +10,24 @@ <h2>
       <a href="{{ page.url }}" class="permalink" title="Permalink">∞</a>
     </h2>
 
+    {% capture discard %}
+      {% for c in site.data.contributors %}
+        {% if c.apacheId == page.author %}
+          {% assign author_name = c.name %}
+          {% if c.homepage %}
+            {% assign homepage = c.homepage %}
+          {% else %}
+            {% capture homepage %}http://github.com/{{ c.githubId }}{% endcapture %}
+          {% endif %}
+          {% if c.avatar %}
+            {% assign avatar = c.avatar %}
+          {% else %}
+            {% capture avatar %}http://github.com/{{ c.githubId }}.png{% endcapture %}
+          {% endif %}
+        {% endif %}
+      {% endfor %}
+    {% endcapture %}{% assign discard = nil %}
+
     <div class="panel">
       <div class="panel-body">
         <div>
@@ -21,7 +39,7 @@ <h2>
         </div>
         <div>
           <span class="label label-default">By</span>
-          <a href="{{ homepage }}"><i class="fa fa-user"></i> {{ page.author }}</a>
+          <a href="{{ homepage }}"><i class="fa fa-user"></i> {{ author_name }} ({{ page.author }})</a>
         </div>
       </div>
     </div>
diff --git a/site/_posts/2017-05-08-0.3-release.md b/site/_posts/2017-05-08-0.3-release.md
index 2920565be7bb9..0779784ff12cb 100644
--- a/site/_posts/2017-05-08-0.3-release.md
+++ b/site/_posts/2017-05-08-0.3-release.md
@@ -1,7 +1,7 @@
 ---
 layout: post
-title: "Apache Arrow 0.3 Release"
-date: "2017-05-01     07:00:00 -0500"
+title: "Apache Arrow 0.3.0 Release"
+date: "2017-05-08 00:00:00 -0400"
 author: wesm
 categories: [release]
 ---
diff --git a/site/_release/0.1.0.md b/site/_release/0.1.0.md
new file mode 100644
index 0000000000000..e668e5698652d
--- /dev/null
+++ b/site/_release/0.1.0.md
@@ -0,0 +1,222 @@
+---
+layout: default
+title: 0.1.0 Release
+permalink: /release/0.1.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.1.0 (10 October 2016)
+
+## Download
+
+* **Source Release**: [apache-arrow-0.1.0.tar.gz][6]
+* **Verification**: [md5][3], [asc][7]
+
+# Changelog
+
+## Bug
+
+* [ARROW-103](https://issues.apache.org/jira/browse/ARROW-103) - Missing patterns from .gitignore
+* [ARROW-104](https://issues.apache.org/jira/browse/ARROW-104) - Update Layout.md based on discussion on the mailing list
+* [ARROW-105](https://issues.apache.org/jira/browse/ARROW-105) - Unit tests fail if assertions are disabled
+* [ARROW-113](https://issues.apache.org/jira/browse/ARROW-113) - TestValueVector test fails if cannot allocate 2GB of memory
+* [ARROW-16](https://issues.apache.org/jira/browse/ARROW-16) - Building cpp issues on XCode 7.2.1
+* [ARROW-17](https://issues.apache.org/jira/browse/ARROW-17) - Set some vector fields to default access level for Drill compatibility
+* [ARROW-18](https://issues.apache.org/jira/browse/ARROW-18) - Fix bug with decimal precision and scale
+* [ARROW-185](https://issues.apache.org/jira/browse/ARROW-185) - [C++] Make sure alignment and memory padding conform to spec
+* [ARROW-188](https://issues.apache.org/jira/browse/ARROW-188) - Python: Add numpy as install requirement
+* [ARROW-193](https://issues.apache.org/jira/browse/ARROW-193) - For the instruction, typos "int his" should be "in this"
+* [ARROW-194](https://issues.apache.org/jira/browse/ARROW-194) - C++: Allow read-only memory mapped source
+* [ARROW-200](https://issues.apache.org/jira/browse/ARROW-200) - [Python] Convert Values String looks like it has incorrect error handling
+* [ARROW-209](https://issues.apache.org/jira/browse/ARROW-209) - [C++] Broken builds: llvm.org apt repos are unavailable
+* [ARROW-210](https://issues.apache.org/jira/browse/ARROW-210) - [C++] Tidy up the type system a little bit
+* [ARROW-211](https://issues.apache.org/jira/browse/ARROW-211) - Several typos/errors in Layout.md examples
+* [ARROW-217](https://issues.apache.org/jira/browse/ARROW-217) - Fix Travis w.r.t conda 4.1.0 changes
+* [ARROW-219](https://issues.apache.org/jira/browse/ARROW-219) - [C++] Passed CMAKE_CXX_FLAGS are being dropped, fix compiler warnings
+* [ARROW-223](https://issues.apache.org/jira/browse/ARROW-223) - Do not link against libpython
+* [ARROW-225](https://issues.apache.org/jira/browse/ARROW-225) - [C++/Python] master Travis CI build is broken
+* [ARROW-244](https://issues.apache.org/jira/browse/ARROW-244) - [C++] Some global APIs of IPC module should be visible to the outside
+* [ARROW-246](https://issues.apache.org/jira/browse/ARROW-246) - [Java] UnionVector doesn't call allocateNew() when creating it's vectorType
+* [ARROW-247](https://issues.apache.org/jira/browse/ARROW-247) - [C++] Missing explicit destructor in RowBatchReader causes an incomplete type error
+* [ARROW-250](https://issues.apache.org/jira/browse/ARROW-250) - Fix for ARROW-246 may cause memory leaks
+* [ARROW-259](https://issues.apache.org/jira/browse/ARROW-259) - Use flatbuffer fields in java implementation
+* [ARROW-265](https://issues.apache.org/jira/browse/ARROW-265) - Negative decimal values have wrong padding
+* [ARROW-266](https://issues.apache.org/jira/browse/ARROW-266) - [C++] Fix the broken build
+* [ARROW-274](https://issues.apache.org/jira/browse/ARROW-274) - Make the MapVector nullable
+* [ARROW-278](https://issues.apache.org/jira/browse/ARROW-278) - [Format] Struct type name consistency in implementations and metadata
+* [ARROW-283](https://issues.apache.org/jira/browse/ARROW-283) - [C++] Update arrow_parquet to account for API changes in PARQUET-573
+* [ARROW-284](https://issues.apache.org/jira/browse/ARROW-284) - [C++] Triage builds by disabling Arrow-Parquet module
+* [ARROW-287](https://issues.apache.org/jira/browse/ARROW-287) - [java] Make nullable vectors use a BitVecor instead of UInt1Vector for bits
+* [ARROW-297](https://issues.apache.org/jira/browse/ARROW-297) - Fix Arrow pom for release
+* [ARROW-304](https://issues.apache.org/jira/browse/ARROW-304) - NullableMapReaderImpl.isSet() always returns true
+* [ARROW-308](https://issues.apache.org/jira/browse/ARROW-308) - UnionListWriter.setPosition() should not call startList()
+* [ARROW-309](https://issues.apache.org/jira/browse/ARROW-309) - Types.getMinorTypeForArrowType() does not work for Union type
+* [ARROW-313](https://issues.apache.org/jira/browse/ARROW-313) - XCode 8.0 breaks builds
+* [ARROW-314](https://issues.apache.org/jira/browse/ARROW-314) - JSONScalar is unnecessary and unused.
+* [ARROW-320](https://issues.apache.org/jira/browse/ARROW-320) - ComplexCopier.copy(FieldReader, FieldWriter) should not start a list if reader is not set
+* [ARROW-321](https://issues.apache.org/jira/browse/ARROW-321) - Fix Arrow licences
+* [ARROW-36](https://issues.apache.org/jira/browse/ARROW-36) - Remove fixVersions from patch tool (until we have them)
+* [ARROW-46](https://issues.apache.org/jira/browse/ARROW-46) - Port DRILL-4410 to Arrow
+* [ARROW-5](https://issues.apache.org/jira/browse/ARROW-5) - Error when run maven install
+* [ARROW-51](https://issues.apache.org/jira/browse/ARROW-51) - Move ValueVector test from Drill project
+* [ARROW-55](https://issues.apache.org/jira/browse/ARROW-55) - Python: fix legacy Python (2.7) tests and add to Travis CI
+* [ARROW-62](https://issues.apache.org/jira/browse/ARROW-62) - Format: Are the nulls bits 0 or 1 for null values?
+* [ARROW-63](https://issues.apache.org/jira/browse/ARROW-63) - C++: ctest fails if Python 3 is the active Python interpreter
+* [ARROW-65](https://issues.apache.org/jira/browse/ARROW-65) - Python: FindPythonLibsNew does not work in a virtualenv
+* [ARROW-69](https://issues.apache.org/jira/browse/ARROW-69) - Change permissions for assignable users
+* [ARROW-72](https://issues.apache.org/jira/browse/ARROW-72) - FindParquet searches for non-existent header
+* [ARROW-75](https://issues.apache.org/jira/browse/ARROW-75) - C++: Fix handling of empty strings
+* [ARROW-77](https://issues.apache.org/jira/browse/ARROW-77) - C++: conform null bit interpretation to match ARROW-62
+* [ARROW-80](https://issues.apache.org/jira/browse/ARROW-80) - Segmentation fault on len(Array) for empty arrays
+* [ARROW-88](https://issues.apache.org/jira/browse/ARROW-88) - C++: Refactor given PARQUET-572
+* [ARROW-93](https://issues.apache.org/jira/browse/ARROW-93) - XCode 7.3 breaks builds
+* [ARROW-94](https://issues.apache.org/jira/browse/ARROW-94) - Expand list example to clarify null vs empty list
+
+## Improvement
+
+* [ARROW-10](https://issues.apache.org/jira/browse/ARROW-10) - Fix mismatch of javadoc names and method parameters
+* [ARROW-15](https://issues.apache.org/jira/browse/ARROW-15) - Fix a naming typo for memory.AllocationManager.AllocationOutcome
+* [ARROW-190](https://issues.apache.org/jira/browse/ARROW-190) - Python: Provide installable sdist builds
+* [ARROW-199](https://issues.apache.org/jira/browse/ARROW-199) - [C++] Refine third party dependency
+* [ARROW-206](https://issues.apache.org/jira/browse/ARROW-206) - [C++] Expose an equality API for arrays that compares a range of slots on two arrays
+* [ARROW-212](https://issues.apache.org/jira/browse/ARROW-212) - [C++] Clarify the fact that PrimitiveArray is now abstract class
+* [ARROW-213](https://issues.apache.org/jira/browse/ARROW-213) - Exposing static arrow build
+* [ARROW-218](https://issues.apache.org/jira/browse/ARROW-218) - Add option to use GitHub API token via environment variable when merging PRs
+* [ARROW-234](https://issues.apache.org/jira/browse/ARROW-234) - [C++] Build with libhdfs support in arrow_io in conda builds
+* [ARROW-238](https://issues.apache.org/jira/browse/ARROW-238) - C++: InternalMemoryPool::Free() should throw an error when there is insufficient allocated memory
+* [ARROW-245](https://issues.apache.org/jira/browse/ARROW-245) - [Format] Clarify Arrow's relationship with big endian platforms
+* [ARROW-252](https://issues.apache.org/jira/browse/ARROW-252) - Add implementation guidelines to the documentation
+* [ARROW-253](https://issues.apache.org/jira/browse/ARROW-253) - Int types should only have width of 8*2^n (8, 16, 32, 64)
+* [ARROW-254](https://issues.apache.org/jira/browse/ARROW-254) - Remove Bit type as it is redundant with boolean
+* [ARROW-255](https://issues.apache.org/jira/browse/ARROW-255) - Finalize Dictionary representation
+* [ARROW-256](https://issues.apache.org/jira/browse/ARROW-256) - Add versioning to the arrow spec.
+* [ARROW-257](https://issues.apache.org/jira/browse/ARROW-257) - Add a typeids Vector to Union type
+* [ARROW-264](https://issues.apache.org/jira/browse/ARROW-264) - Create an Arrow File format
+* [ARROW-270](https://issues.apache.org/jira/browse/ARROW-270) - [Format] Define more generic Interval logical type
+* [ARROW-271](https://issues.apache.org/jira/browse/ARROW-271) - Update Field structure to be more explicit
+* [ARROW-279](https://issues.apache.org/jira/browse/ARROW-279) - rename vector module to arrow-vector for consistency
+* [ARROW-280](https://issues.apache.org/jira/browse/ARROW-280) - [C++] Consolidate file and shared memory IO interfaces
+* [ARROW-285](https://issues.apache.org/jira/browse/ARROW-285) - Allow for custom flatc compiler
+* [ARROW-286](https://issues.apache.org/jira/browse/ARROW-286) - Build thirdparty dependencies in parallel
+* [ARROW-289](https://issues.apache.org/jira/browse/ARROW-289) - Install test-util.h
+* [ARROW-290](https://issues.apache.org/jira/browse/ARROW-290) - Specialize alloc() in ArrowBuf
+* [ARROW-292](https://issues.apache.org/jira/browse/ARROW-292) - [Java] Upgrade Netty to 4.041
+* [ARROW-299](https://issues.apache.org/jira/browse/ARROW-299) - Use absolute namespace in macros
+* [ARROW-305](https://issues.apache.org/jira/browse/ARROW-305) - Add compression and use_dictionary options to Parquet interface
+* [ARROW-306](https://issues.apache.org/jira/browse/ARROW-306) - Add option to pass cmake arguments via environment variable
+* [ARROW-315](https://issues.apache.org/jira/browse/ARROW-315) - Finalize timestamp type
+* [ARROW-319](https://issues.apache.org/jira/browse/ARROW-319) - Add canonical Arrow Schema json representation
+* [ARROW-324](https://issues.apache.org/jira/browse/ARROW-324) - Update arrow metadata diagram
+* [ARROW-325](https://issues.apache.org/jira/browse/ARROW-325) - make TestArrowFile not dependent on timezone
+* [ARROW-50](https://issues.apache.org/jira/browse/ARROW-50) - C++: Enable library builds for 3rd-party users without having to build thirdparty googletest
+* [ARROW-54](https://issues.apache.org/jira/browse/ARROW-54) - Python: rename package to "pyarrow"
+* [ARROW-64](https://issues.apache.org/jira/browse/ARROW-64) - Add zsh support to C++ build scripts
+* [ARROW-66](https://issues.apache.org/jira/browse/ARROW-66) - Maybe some missing steps in installation guide
+* [ARROW-68](https://issues.apache.org/jira/browse/ARROW-68) - Update setup_build_env and third-party script to be more userfriendly
+* [ARROW-71](https://issues.apache.org/jira/browse/ARROW-71) - C++: Add script to run clang-tidy on codebase
+* [ARROW-73](https://issues.apache.org/jira/browse/ARROW-73) - Support CMake 2.8
+* [ARROW-78](https://issues.apache.org/jira/browse/ARROW-78) - C++: Add constructor for DecimalType
+* [ARROW-79](https://issues.apache.org/jira/browse/ARROW-79) - Python: Add benchmarks
+* [ARROW-8](https://issues.apache.org/jira/browse/ARROW-8) - Set up Travis CI
+* [ARROW-85](https://issues.apache.org/jira/browse/ARROW-85) - C++: memcmp can be avoided in Equal when comparing with the same Buffer
+* [ARROW-86](https://issues.apache.org/jira/browse/ARROW-86) - Python: Implement zero-copy Arrow-to-Pandas conversion
+* [ARROW-87](https://issues.apache.org/jira/browse/ARROW-87) - Implement Decimal schema conversion for all ways supported in Parquet
+* [ARROW-89](https://issues.apache.org/jira/browse/ARROW-89) - Python: Add benchmarks for Arrow<->Pandas conversion
+* [ARROW-9](https://issues.apache.org/jira/browse/ARROW-9) - Rename some unchanged "Drill" to "Arrow"
+* [ARROW-91](https://issues.apache.org/jira/browse/ARROW-91) - C++: First draft of an adapter class for parquet-cpp's ParquetFileReader that produces Arrow table/row batch objects
+
+## New Feature
+
+* [ARROW-100](https://issues.apache.org/jira/browse/ARROW-100) - [C++] Computing RowBatch size
+* [ARROW-106](https://issues.apache.org/jira/browse/ARROW-106) - Add IPC round trip for string types (string, char, varchar, binary)
+* [ARROW-107](https://issues.apache.org/jira/browse/ARROW-107) - [C++] add ipc round trip for struct types
+* [ARROW-13](https://issues.apache.org/jira/browse/ARROW-13) - Add PR merge tool similar to that used in Parquet
+* [ARROW-19](https://issues.apache.org/jira/browse/ARROW-19) - C++: Externalize memory allocations and add a MemoryPool abstract interface to builder classes
+* [ARROW-197](https://issues.apache.org/jira/browse/ARROW-197) - [Python] Add conda dev recipe for pyarrow
+* [ARROW-2](https://issues.apache.org/jira/browse/ARROW-2) - Post Simple Website
+* [ARROW-20](https://issues.apache.org/jira/browse/ARROW-20) - C++: Add null count member to Array containers, remove nullable member
+* [ARROW-201](https://issues.apache.org/jira/browse/ARROW-201) - C++: Initial ParquetWriter implementation
+* [ARROW-203](https://issues.apache.org/jira/browse/ARROW-203) - Python: Basic filename based Parquet read/write
+* [ARROW-204](https://issues.apache.org/jira/browse/ARROW-204) - [Python] Automate uploading conda build artifacts for libarrow and pyarrow
+* [ARROW-21](https://issues.apache.org/jira/browse/ARROW-21) - C++: Add in-memory schema metadata container
+* [ARROW-214](https://issues.apache.org/jira/browse/ARROW-214) - C++: Add String support to Parquet I/O
+* [ARROW-215](https://issues.apache.org/jira/browse/ARROW-215) - C++: Support other integer types in Parquet I/O
+* [ARROW-22](https://issues.apache.org/jira/browse/ARROW-22) - C++: Add schema adapter routines for converting flat Parquet schemas to in-memory Arrow schemas
+* [ARROW-222](https://issues.apache.org/jira/browse/ARROW-222) - [C++] Create prototype file-like interface to HDFS (via libhdfs) and begin defining more general IO interface for Arrow data adapters
+* [ARROW-23](https://issues.apache.org/jira/browse/ARROW-23) - C++: Add logical "Column" container for chunked data
+* [ARROW-233](https://issues.apache.org/jira/browse/ARROW-233) - [C++] Add visibility defines for limiting shared library symbol visibility
+* [ARROW-236](https://issues.apache.org/jira/browse/ARROW-236) - [Python] Enable Parquet read/write to work with HDFS file objects
+* [ARROW-237](https://issues.apache.org/jira/browse/ARROW-237) - [C++] Create Arrow specializations of Parquet allocator and read interfaces
+* [ARROW-24](https://issues.apache.org/jira/browse/ARROW-24) - C++: Add logical "Table" container
+* [ARROW-242](https://issues.apache.org/jira/browse/ARROW-242) - C++/Python: Support Timestamp Data Type
+* [ARROW-26](https://issues.apache.org/jira/browse/ARROW-26) - C++: Add developer instructions for building parquet-cpp integration
+* [ARROW-262](https://issues.apache.org/jira/browse/ARROW-262) - [Format] Add a new format document for metadata and logical types for messaging and IPC / on-wire/file representations
+* [ARROW-267](https://issues.apache.org/jira/browse/ARROW-267) - [C++] C++ implementation of file-like layout for RPC / IPC
+* [ARROW-28](https://issues.apache.org/jira/browse/ARROW-28) - C++: Add google/benchmark to the 3rd-party build toolchain
+* [ARROW-293](https://issues.apache.org/jira/browse/ARROW-293) - [C++] Implementations of IO interfaces for operating system files
+* [ARROW-296](https://issues.apache.org/jira/browse/ARROW-296) - [C++] Remove arrow_parquet C++ module and related parts of build system
+* [ARROW-3](https://issues.apache.org/jira/browse/ARROW-3) - Post Initial Arrow Format Spec
+* [ARROW-30](https://issues.apache.org/jira/browse/ARROW-30) - Python: pandas/NumPy to/from Arrow conversion routines
+* [ARROW-301](https://issues.apache.org/jira/browse/ARROW-301) - [Format] Add some form of user field metadata to IPC schemas
+* [ARROW-302](https://issues.apache.org/jira/browse/ARROW-302) - [Python] Add support to use the Arrow file format with file-like objects
+* [ARROW-31](https://issues.apache.org/jira/browse/ARROW-31) - Python: basic PyList <-> Arrow marshaling code
+* [ARROW-318](https://issues.apache.org/jira/browse/ARROW-318) - [Python] Revise README to reflect current state of project
+* [ARROW-37](https://issues.apache.org/jira/browse/ARROW-37) - C++: Represent boolean array data in bit-packed form
+* [ARROW-4](https://issues.apache.org/jira/browse/ARROW-4) - Initial Arrow CPP Implementation
+* [ARROW-42](https://issues.apache.org/jira/browse/ARROW-42) - Python: Add to Travis CI build
+* [ARROW-43](https://issues.apache.org/jira/browse/ARROW-43) - Python: Add rudimentary console __repr__ for array types
+* [ARROW-44](https://issues.apache.org/jira/browse/ARROW-44) - Python: Implement basic object model for scalar values (i.e. results of arrow_arr[i])
+* [ARROW-48](https://issues.apache.org/jira/browse/ARROW-48) - Python: Add Schema object wrapper
+* [ARROW-49](https://issues.apache.org/jira/browse/ARROW-49) - Python: Add Column and Table wrapper interface
+* [ARROW-53](https://issues.apache.org/jira/browse/ARROW-53) - Python: Fix RPATH and add source installation instructions
+* [ARROW-56](https://issues.apache.org/jira/browse/ARROW-56) - Format: Specify LSB bit ordering in bit arrays
+* [ARROW-57](https://issues.apache.org/jira/browse/ARROW-57) - Format: Draft data headers IDL for data interchange
+* [ARROW-58](https://issues.apache.org/jira/browse/ARROW-58) - Format: Draft type metadata ("schemas") IDL
+* [ARROW-59](https://issues.apache.org/jira/browse/ARROW-59) - Python: Boolean data support for builtin data structures
+* [ARROW-60](https://issues.apache.org/jira/browse/ARROW-60) - C++: Struct type builder API
+* [ARROW-67](https://issues.apache.org/jira/browse/ARROW-67) - C++: Draft type metadata conversion to/from IPC representation
+* [ARROW-7](https://issues.apache.org/jira/browse/ARROW-7) - Add Python library build toolchain
+* [ARROW-70](https://issues.apache.org/jira/browse/ARROW-70) - C++: Add "lite" DCHECK macros used in parquet-cpp
+* [ARROW-76](https://issues.apache.org/jira/browse/ARROW-76) - Revise format document to include null count, defer non-nullable arrays to the domain of metadata
+* [ARROW-82](https://issues.apache.org/jira/browse/ARROW-82) - C++: Implement IPC exchange for List types
+* [ARROW-90](https://issues.apache.org/jira/browse/ARROW-90) - Apache Arrow cpp code does not support power architecture
+* [ARROW-92](https://issues.apache.org/jira/browse/ARROW-92) - C++: Arrow to Parquet Schema conversion
+
+## Task
+
+* [ARROW-1](https://issues.apache.org/jira/browse/ARROW-1) - Import Initial Codebase
+* [ARROW-101](https://issues.apache.org/jira/browse/ARROW-101) - Fix java warnings emitted by java compiler
+* [ARROW-102](https://issues.apache.org/jira/browse/ARROW-102) - travis-ci support for java project
+* [ARROW-11](https://issues.apache.org/jira/browse/ARROW-11) - Mirror JIRA activity to dev@arrow.apache.org
+* [ARROW-14](https://issues.apache.org/jira/browse/ARROW-14) - Add JIRA components
+* [ARROW-251](https://issues.apache.org/jira/browse/ARROW-251) - [C++] Expose APIs for getting code and message of the status
+* [ARROW-272](https://issues.apache.org/jira/browse/ARROW-272) - Arrow release 0.1
+* [ARROW-298](https://issues.apache.org/jira/browse/ARROW-298) - create release scripts
+* [ARROW-35](https://issues.apache.org/jira/browse/ARROW-35) - Add a short call-to-action / how-to-get-involved to the main README.md
+
+## Test
+
+* [ARROW-260](https://issues.apache.org/jira/browse/ARROW-260) - TestValueVector.testFixedVectorReallocation and testVariableVectorReallocation are flaky
+* [ARROW-83](https://issues.apache.org/jira/browse/ARROW-83) - Add basic test infrastructure for DecimalType
+
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.1.0
+[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.1.0/apache-arrow-0.1.0.tar.gz.md5
+[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.1.0/apache-arrow-0.1.0.tar.gz
+[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.1.0/apache-arrow-0.1.0.tar.gz.asc
diff --git a/site/_release/0.2.0.md b/site/_release/0.2.0.md
new file mode 100644
index 0000000000000..d66afd066d762
--- /dev/null
+++ b/site/_release/0.2.0.md
@@ -0,0 +1,238 @@
+---
+layout: default
+title: 0.2.0 Release
+permalink: /release/0.2.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.2.0 (18 February 2017)
+
+## Download
+
+* **Source Release**: [apache-arrow-0.2.0.tar.gz][6]
+* **Verification**: [md5][3], [asc][7]
+
+# Changelog
+
+## Bug
+
+* [ARROW-112](https://issues.apache.org/jira/browse/ARROW-112) - [C++]  Style fix for constants/enums
+* [ARROW-202](https://issues.apache.org/jira/browse/ARROW-202) - [C++] Integrate with appveyor ci for windows support and get arrow building on windows
+* [ARROW-220](https://issues.apache.org/jira/browse/ARROW-220) - [C++] Build conda artifacts in a build environment with better cross-linux ABI compatibility
+* [ARROW-224](https://issues.apache.org/jira/browse/ARROW-224) - [C++] Address static linking of boost dependencies
+* [ARROW-230](https://issues.apache.org/jira/browse/ARROW-230) - Python: Do not name modules like native ones (i.e. rename pyarrow.io)
+* [ARROW-239](https://issues.apache.org/jira/browse/ARROW-239) - [Python] HdfsFile.read called with no arguments should read remainder of file
+* [ARROW-261](https://issues.apache.org/jira/browse/ARROW-261) - [C++] Refactor BinaryArray/StringArray classes to not inherit from ListArray
+* [ARROW-275](https://issues.apache.org/jira/browse/ARROW-275) - Add tests for UnionVector in Arrow File
+* [ARROW-294](https://issues.apache.org/jira/browse/ARROW-294) - [C++] Do not use fopen / fclose / etc. methods for memory mapped file implementation
+* [ARROW-322](https://issues.apache.org/jira/browse/ARROW-322) - [C++] Do not build HDFS IO interface optionally
+* [ARROW-323](https://issues.apache.org/jira/browse/ARROW-323) - [Python] Opt-in to PyArrow parquet build rather than skipping silently on failure
+* [ARROW-334](https://issues.apache.org/jira/browse/ARROW-334) - [Python] OS X rpath issues on some configurations
+* [ARROW-337](https://issues.apache.org/jira/browse/ARROW-337) - UnionListWriter.list() is doing more than it should, this can cause data corruption
+* [ARROW-339](https://issues.apache.org/jira/browse/ARROW-339) - Make merge_arrow_pr script work with Python 3
+* [ARROW-340](https://issues.apache.org/jira/browse/ARROW-340) - [C++] Opening a writeable file on disk that already exists does not truncate to zero
+* [ARROW-342](https://issues.apache.org/jira/browse/ARROW-342) - Set Python version on release
+* [ARROW-345](https://issues.apache.org/jira/browse/ARROW-345) - libhdfs integration doesn't work for Mac
+* [ARROW-346](https://issues.apache.org/jira/browse/ARROW-346) - Python API Documentation
+* [ARROW-348](https://issues.apache.org/jira/browse/ARROW-348) - [Python] CMake build type should be configurable on the command line
+* [ARROW-349](https://issues.apache.org/jira/browse/ARROW-349) - Six is missing as a requirement in the python setup.py
+* [ARROW-351](https://issues.apache.org/jira/browse/ARROW-351) - Time type has no unit
+* [ARROW-354](https://issues.apache.org/jira/browse/ARROW-354) - Connot compare an array of empty strings to another
+* [ARROW-357](https://issues.apache.org/jira/browse/ARROW-357) - Default Parquet chunk_size of 64k is too small
+* [ARROW-358](https://issues.apache.org/jira/browse/ARROW-358) - [C++] libhdfs can be in non-standard locations in some Hadoop distributions
+* [ARROW-362](https://issues.apache.org/jira/browse/ARROW-362) - Python: Calling to_pandas on a table read from Parquet leaks memory
+* [ARROW-371](https://issues.apache.org/jira/browse/ARROW-371) - Python: Table with null timestamp becomes float in pandas
+* [ARROW-375](https://issues.apache.org/jira/browse/ARROW-375) - columns parameter in parquet.read_table() raises KeyError for valid column
+* [ARROW-384](https://issues.apache.org/jira/browse/ARROW-384) - Align Java and C++ RecordBatch data and metadata layout
+* [ARROW-386](https://issues.apache.org/jira/browse/ARROW-386) - [Java] Respect case of struct / map field names
+* [ARROW-387](https://issues.apache.org/jira/browse/ARROW-387) - [C++] arrow::io::BufferReader does not permit shared memory ownership in zero-copy reads
+* [ARROW-390](https://issues.apache.org/jira/browse/ARROW-390) - C++: CMake fails on json-integration-test with ARROW_BUILD_TESTS=OFF
+* [ARROW-392](https://issues.apache.org/jira/browse/ARROW-392) - Fix string/binary integration tests
+* [ARROW-393](https://issues.apache.org/jira/browse/ARROW-393) - [JAVA] JSON file reader fails to set the buffer size on String data vector
+* [ARROW-395](https://issues.apache.org/jira/browse/ARROW-395) - Arrow file format writes record batches in reverse order.
+* [ARROW-398](https://issues.apache.org/jira/browse/ARROW-398) - [Java] Java file format requires bitmaps of all 1's to be written when there are no nulls
+* [ARROW-399](https://issues.apache.org/jira/browse/ARROW-399) - [Java] ListVector.loadFieldBuffers ignores the ArrowFieldNode length metadata
+* [ARROW-400](https://issues.apache.org/jira/browse/ARROW-400) - [Java] ArrowWriter writes length 0 for Struct types
+* [ARROW-401](https://issues.apache.org/jira/browse/ARROW-401) - [Java] Floating point vectors should do an approximate comparison in integration tests
+* [ARROW-402](https://issues.apache.org/jira/browse/ARROW-402) - [Java] "refCnt gone negative" error in integration tests
+* [ARROW-403](https://issues.apache.org/jira/browse/ARROW-403) - [JAVA] UnionVector: Creating a transfer pair doesn't transfer the schema to destination vector
+* [ARROW-404](https://issues.apache.org/jira/browse/ARROW-404) - [Python] Closing an HdfsClient while there are still open file handles results in a crash
+* [ARROW-405](https://issues.apache.org/jira/browse/ARROW-405) - [C++] Be less stringent about finding include/hdfs.h in HADOOP_HOME
+* [ARROW-406](https://issues.apache.org/jira/browse/ARROW-406) - [C++] Large HDFS reads must utilize the set file buffer size when making RPCs
+* [ARROW-408](https://issues.apache.org/jira/browse/ARROW-408) - [C++/Python] Remove defunct conda recipes
+* [ARROW-414](https://issues.apache.org/jira/browse/ARROW-414) - [Java] "Buffer too large to resize to ..." error
+* [ARROW-420](https://issues.apache.org/jira/browse/ARROW-420) - Align Date implementation between Java and C++
+* [ARROW-421](https://issues.apache.org/jira/browse/ARROW-421) - [Python] Zero-copy buffers read by pyarrow::PyBytesReader must retain a reference to the parent PyBytes to avoid premature garbage collection issues
+* [ARROW-422](https://issues.apache.org/jira/browse/ARROW-422) - C++: IPC should depend on rapidjson_ep if RapidJSON is vendored
+* [ARROW-429](https://issues.apache.org/jira/browse/ARROW-429) - git-archive SHA-256 checksums are changing
+* [ARROW-433](https://issues.apache.org/jira/browse/ARROW-433) - [Python] Date conversion is locale-dependent
+* [ARROW-434](https://issues.apache.org/jira/browse/ARROW-434) - Segfaults and encoding issues in Python Parquet reads
+* [ARROW-435](https://issues.apache.org/jira/browse/ARROW-435) - C++: Spelling mistake in if(RAPIDJSON_VENDORED)
+* [ARROW-437](https://issues.apache.org/jira/browse/ARROW-437) - [C++] clang compiler warnings from overridden virtual functions
+* [ARROW-445](https://issues.apache.org/jira/browse/ARROW-445) - C++: arrow_ipc is built before arrow/ipc/Message_generated.h was generated
+* [ARROW-447](https://issues.apache.org/jira/browse/ARROW-447) - Python: Align scalar/pylist string encoding with pandas' one.
+* [ARROW-455](https://issues.apache.org/jira/browse/ARROW-455) - [C++] BufferOutputStream dtor does not call Close()
+* [ARROW-469](https://issues.apache.org/jira/browse/ARROW-469) - C++: Add option so that resize doesn't decrease the capacity
+* [ARROW-481](https://issues.apache.org/jira/browse/ARROW-481) - [Python] Fix Python 2.7 regression in patch for PARQUET-472
+* [ARROW-486](https://issues.apache.org/jira/browse/ARROW-486) - [C++] arrow::io::MemoryMappedFile can't be casted to arrow::io::FileInterface
+* [ARROW-487](https://issues.apache.org/jira/browse/ARROW-487) - Python: ConvertTableToPandas segfaults if ObjectBlock::Write fails
+* [ARROW-494](https://issues.apache.org/jira/browse/ARROW-494) - [C++] When MemoryMappedFile is destructed, memory is unmapped even if buffer referecnes still exist
+* [ARROW-499](https://issues.apache.org/jira/browse/ARROW-499) - Update file serialization to use streaming serialization format
+* [ARROW-505](https://issues.apache.org/jira/browse/ARROW-505) - [C++] Fix compiler warnings in release mode
+* [ARROW-511](https://issues.apache.org/jira/browse/ARROW-511) - [Python] List[T] conversions not implemented for single arrays
+* [ARROW-513](https://issues.apache.org/jira/browse/ARROW-513) - [C++] Fix Appveyor build
+* [ARROW-519](https://issues.apache.org/jira/browse/ARROW-519) - [C++] Missing vtable in libarrow.dylib on Xcode 6.4
+* [ARROW-523](https://issues.apache.org/jira/browse/ARROW-523) - Python: Account for changes in PARQUET-834
+* [ARROW-533](https://issues.apache.org/jira/browse/ARROW-533) - [C++] arrow::TimestampArray / TimeArray has a broken constructor
+* [ARROW-535](https://issues.apache.org/jira/browse/ARROW-535) - [Python] Add type mapping for NPY_LONGLONG
+* [ARROW-537](https://issues.apache.org/jira/browse/ARROW-537) - [C++] StringArray/BinaryArray comparisons may be incorrect when values with non-zero length are null
+* [ARROW-540](https://issues.apache.org/jira/browse/ARROW-540) - [C++] Fix build in aftermath of ARROW-33
+* [ARROW-543](https://issues.apache.org/jira/browse/ARROW-543) - C++: Lazily computed null_counts counts number of non-null entries
+* [ARROW-544](https://issues.apache.org/jira/browse/ARROW-544) - [C++] ArrayLoader::LoadBinary fails for length-0 arrays
+* [ARROW-545](https://issues.apache.org/jira/browse/ARROW-545) - [Python] Ignore files without .parq or .parquet prefix when reading directory of files
+* [ARROW-548](https://issues.apache.org/jira/browse/ARROW-548) - [Python] Add nthreads option to pyarrow.Filesystem.read_parquet
+* [ARROW-551](https://issues.apache.org/jira/browse/ARROW-551) - C++: Construction of Column with nullptr Array segfaults
+* [ARROW-556](https://issues.apache.org/jira/browse/ARROW-556) - [Integration] Can not run Integration tests if different cpp build path
+* [ARROW-561](https://issues.apache.org/jira/browse/ARROW-561) - Update java & python dependencies to improve downstream packaging experience
+
+## Improvement
+
+* [ARROW-189](https://issues.apache.org/jira/browse/ARROW-189) - C++: Use ExternalProject to build thirdparty dependencies
+* [ARROW-191](https://issues.apache.org/jira/browse/ARROW-191) - Python: Provide infrastructure for manylinux1 wheels
+* [ARROW-328](https://issues.apache.org/jira/browse/ARROW-328) - [C++] Return shared_ptr by value instead of const-ref?
+* [ARROW-330](https://issues.apache.org/jira/browse/ARROW-330) - [C++] CMake functions to simplify shared / static library configuration
+* [ARROW-333](https://issues.apache.org/jira/browse/ARROW-333) - Make writers update their internal schema even when no data is written.
+* [ARROW-335](https://issues.apache.org/jira/browse/ARROW-335) - Improve Type apis and toString() by encapsulating flatbuffers better
+* [ARROW-336](https://issues.apache.org/jira/browse/ARROW-336) - Run Apache Rat in Travis builds
+* [ARROW-338](https://issues.apache.org/jira/browse/ARROW-338) - [C++] Refactor IPC vector "loading" and "unloading" to be based on cleaner visitor pattern
+* [ARROW-350](https://issues.apache.org/jira/browse/ARROW-350) - Add Kerberos support to HDFS shim
+* [ARROW-355](https://issues.apache.org/jira/browse/ARROW-355) - Add tests for serialising arrays of empty strings to Parquet
+* [ARROW-356](https://issues.apache.org/jira/browse/ARROW-356) - Add documentation about reading Parquet
+* [ARROW-360](https://issues.apache.org/jira/browse/ARROW-360) - C++: Add method to shrink PoolBuffer using realloc
+* [ARROW-361](https://issues.apache.org/jira/browse/ARROW-361) - Python: Support reading a column-selection from Parquet files
+* [ARROW-365](https://issues.apache.org/jira/browse/ARROW-365) - Python: Provide Array.to_pandas()
+* [ARROW-366](https://issues.apache.org/jira/browse/ARROW-366) - [java] implement Dictionary vector
+* [ARROW-374](https://issues.apache.org/jira/browse/ARROW-374) - Python: clarify unicode vs. binary in API
+* [ARROW-379](https://issues.apache.org/jira/browse/ARROW-379) - Python: Use setuptools_scm/setuptools_scm_git_archive to provide the version number
+* [ARROW-380](https://issues.apache.org/jira/browse/ARROW-380) - [Java] optimize null count when serializing vectors.
+* [ARROW-382](https://issues.apache.org/jira/browse/ARROW-382) - Python: Extend API documentation
+* [ARROW-396](https://issues.apache.org/jira/browse/ARROW-396) - Python: Add pyarrow.schema.Schema.equals
+* [ARROW-409](https://issues.apache.org/jira/browse/ARROW-409) - Python: Change pyarrow.Table.dataframe_from_batches API to create Table instead
+* [ARROW-411](https://issues.apache.org/jira/browse/ARROW-411) - [Java] Move Intergration.compare and Intergration.compareSchemas to a public utils class
+* [ARROW-423](https://issues.apache.org/jira/browse/ARROW-423) - C++: Define BUILD_BYPRODUCTS in external project to support non-make CMake generators
+* [ARROW-425](https://issues.apache.org/jira/browse/ARROW-425) - Python: Expose a C function to convert arrow::Table to pyarrow.Table
+* [ARROW-426](https://issues.apache.org/jira/browse/ARROW-426) - Python: Conversion from pyarrow.Array to a Python list
+* [ARROW-430](https://issues.apache.org/jira/browse/ARROW-430) - Python: Better version handling
+* [ARROW-432](https://issues.apache.org/jira/browse/ARROW-432) - [Python] Avoid unnecessary memory copy in to_pandas conversion by using low-level pandas internals APIs
+* [ARROW-450](https://issues.apache.org/jira/browse/ARROW-450) - Python: Fixes for PARQUET-818
+* [ARROW-457](https://issues.apache.org/jira/browse/ARROW-457) - Python: Better control over memory pool
+* [ARROW-458](https://issues.apache.org/jira/browse/ARROW-458) - Python: Expose jemalloc MemoryPool
+* [ARROW-463](https://issues.apache.org/jira/browse/ARROW-463) - C++: Support jemalloc 4.x
+* [ARROW-466](https://issues.apache.org/jira/browse/ARROW-466) - C++: ExternalProject for jemalloc
+* [ARROW-468](https://issues.apache.org/jira/browse/ARROW-468) - Python: Conversion of nested data in pd.DataFrames to/from Arrow structures
+* [ARROW-474](https://issues.apache.org/jira/browse/ARROW-474) - Create an Arrow streaming file fomat
+* [ARROW-479](https://issues.apache.org/jira/browse/ARROW-479) - Python: Test for expected schema in Pandas conversion
+* [ARROW-485](https://issues.apache.org/jira/browse/ARROW-485) - [Java] Users are required to initialize VariableLengthVectors.offsetVector before calling VariableLengthVectors.mutator.getSafe
+* [ARROW-490](https://issues.apache.org/jira/browse/ARROW-490) - Python: Update manylinux1 build scripts
+* [ARROW-524](https://issues.apache.org/jira/browse/ARROW-524) - [java] provide apis to access nested vectors and buffers
+* [ARROW-525](https://issues.apache.org/jira/browse/ARROW-525) - Python: Add more documentation to the package
+* [ARROW-529](https://issues.apache.org/jira/browse/ARROW-529) - Python: Add jemalloc and Python 3.6 to manylinux1 build
+* [ARROW-546](https://issues.apache.org/jira/browse/ARROW-546) - Python: Account for changes in PARQUET-867
+* [ARROW-553](https://issues.apache.org/jira/browse/ARROW-553) - C++: Faster valid bitmap building
+
+## New Feature
+
+* [ARROW-108](https://issues.apache.org/jira/browse/ARROW-108) - [C++] Add IPC round trip for union types
+* [ARROW-221](https://issues.apache.org/jira/browse/ARROW-221) - Add switch for writing Parquet 1.0 compatible logical types
+* [ARROW-227](https://issues.apache.org/jira/browse/ARROW-227) - [C++/Python] Hook arrow_io generic reader / writer interface into arrow_parquet
+* [ARROW-228](https://issues.apache.org/jira/browse/ARROW-228) - [Python] Create an Arrow-cpp-compatible interface for reading bytes from Python file-like objects
+* [ARROW-243](https://issues.apache.org/jira/browse/ARROW-243) - [C++] Add "driver" option to HdfsClient to choose between libhdfs and libhdfs3 at runtime
+* [ARROW-303](https://issues.apache.org/jira/browse/ARROW-303) - [C++] Also build static libraries for leaf libraries
+* [ARROW-312](https://issues.apache.org/jira/browse/ARROW-312) - [Python] Provide Python API to read/write the Arrow IPC file format
+* [ARROW-317](https://issues.apache.org/jira/browse/ARROW-317) - [C++] Implement zero-copy Slice method on arrow::Buffer that retains reference to parent
+* [ARROW-33](https://issues.apache.org/jira/browse/ARROW-33) - C++: Implement zero-copy array slicing
+* [ARROW-332](https://issues.apache.org/jira/browse/ARROW-332) - [Python] Add helper function to convert RecordBatch to pandas.DataFrame
+* [ARROW-363](https://issues.apache.org/jira/browse/ARROW-363) - Set up Java/C++ integration test harness
+* [ARROW-369](https://issues.apache.org/jira/browse/ARROW-369) - [Python] Add ability to convert multiple record batches at once to pandas
+* [ARROW-373](https://issues.apache.org/jira/browse/ARROW-373) - [C++] Implement C++ version of JSON file format for testing
+* [ARROW-377](https://issues.apache.org/jira/browse/ARROW-377) - Python: Add support for conversion of Pandas.Categorical
+* [ARROW-381](https://issues.apache.org/jira/browse/ARROW-381) - [C++] Simplify primitive array type builders to use a default type singleton
+* [ARROW-383](https://issues.apache.org/jira/browse/ARROW-383) - [C++] Implement C++ version of ARROW-367 integration test validator
+* [ARROW-389](https://issues.apache.org/jira/browse/ARROW-389) - Python: Write Parquet files to pyarrow.io.NativeFile objects
+* [ARROW-394](https://issues.apache.org/jira/browse/ARROW-394) - Add integration tests for boolean, list, struct, and other basic types
+* [ARROW-410](https://issues.apache.org/jira/browse/ARROW-410) - [C++] Add Flush method to arrow::io::OutputStream
+* [ARROW-415](https://issues.apache.org/jira/browse/ARROW-415) - C++: Add Equals implementation to compare Tables
+* [ARROW-416](https://issues.apache.org/jira/browse/ARROW-416) - C++: Add Equals implementation to compare Columns
+* [ARROW-417](https://issues.apache.org/jira/browse/ARROW-417) - C++: Add Equals implementation to compare ChunkedArrays
+* [ARROW-418](https://issues.apache.org/jira/browse/ARROW-418) - [C++] Consolidate array container and builder code, remove arrow/types
+* [ARROW-419](https://issues.apache.org/jira/browse/ARROW-419) - [C++] Promote util/{status.h, buffer.h, memory-pool.h} to top level of arrow/ source directory
+* [ARROW-427](https://issues.apache.org/jira/browse/ARROW-427) - [C++] Implement dictionary-encoded array container
+* [ARROW-428](https://issues.apache.org/jira/browse/ARROW-428) - [Python] Deserialize from Arrow record batches to pandas in parallel using a thread pool
+* [ARROW-438](https://issues.apache.org/jira/browse/ARROW-438) - [Python] Concatenate Table instances with equal schemas
+* [ARROW-440](https://issues.apache.org/jira/browse/ARROW-440) - [C++] Support pkg-config
+* [ARROW-441](https://issues.apache.org/jira/browse/ARROW-441) - [Python] Expose Arrow's file and memory map classes as NativeFile subclasses
+* [ARROW-442](https://issues.apache.org/jira/browse/ARROW-442) - [Python] Add public Python API to inspect Parquet file metadata
+* [ARROW-444](https://issues.apache.org/jira/browse/ARROW-444) - [Python] Avoid unnecessary memory copies from use of PyBytes_* C APIs
+* [ARROW-449](https://issues.apache.org/jira/browse/ARROW-449) - Python: Conversion from pyarrow.{Table,RecordBatch} to a Python dict
+* [ARROW-456](https://issues.apache.org/jira/browse/ARROW-456) - C++: Add jemalloc based MemoryPool
+* [ARROW-461](https://issues.apache.org/jira/browse/ARROW-461) - [Python] Implement conversion between arrow::DictionaryArray and pandas.Categorical
+* [ARROW-467](https://issues.apache.org/jira/browse/ARROW-467) - [Python] Run parquet-cpp unit tests in Travis CI
+* [ARROW-470](https://issues.apache.org/jira/browse/ARROW-470) - [Python] Add "FileSystem" abstraction to access directories of files in a uniform way
+* [ARROW-471](https://issues.apache.org/jira/browse/ARROW-471) - [Python] Enable ParquetFile to pass down separately-obtained file metadata
+* [ARROW-472](https://issues.apache.org/jira/browse/ARROW-472) - [Python] Expose parquet::{SchemaDescriptor, ColumnDescriptor}::Equals
+* [ARROW-475](https://issues.apache.org/jira/browse/ARROW-475) - [Python] High level support for reading directories of Parquet files (as a single Arrow table) from supported file system interfaces
+* [ARROW-476](https://issues.apache.org/jira/browse/ARROW-476) - [Integration] Add integration tests for Binary / Varbytes type
+* [ARROW-477](https://issues.apache.org/jira/browse/ARROW-477) - [Java] Add support for second/microsecond/nanosecond timestamps in-memory and in IPC/JSON layer
+* [ARROW-478](https://issues.apache.org/jira/browse/ARROW-478) - [Python] Accept a PyBytes object in the pyarrow.io.BufferReader ctor
+* [ARROW-484](https://issues.apache.org/jira/browse/ARROW-484) - Add more detail about what of technology can be found in the Arrow implementations to README
+* [ARROW-495](https://issues.apache.org/jira/browse/ARROW-495) - [C++] Add C++ implementation of streaming serialized format
+* [ARROW-497](https://issues.apache.org/jira/browse/ARROW-497) - [Java] Integration test harness for streaming format
+* [ARROW-498](https://issues.apache.org/jira/browse/ARROW-498) - [C++] Integration test harness for streaming format
+* [ARROW-503](https://issues.apache.org/jira/browse/ARROW-503) - [Python] Interface to streaming binary format
+* [ARROW-508](https://issues.apache.org/jira/browse/ARROW-508) - [C++] Make file/memory-mapped file interfaces threadsafe
+* [ARROW-509](https://issues.apache.org/jira/browse/ARROW-509) - [Python] Add support for PARQUET-835 (parallel column reads)
+* [ARROW-512](https://issues.apache.org/jira/browse/ARROW-512) - C++: Add method to check for primitive types
+* [ARROW-514](https://issues.apache.org/jira/browse/ARROW-514) - [Python] Accept pyarrow.io.Buffer as input to StreamReader, FileReader classes
+* [ARROW-515](https://issues.apache.org/jira/browse/ARROW-515) - [Python] Add StreamReader/FileReader methods that read all record batches as a Table
+* [ARROW-521](https://issues.apache.org/jira/browse/ARROW-521) - [C++/Python] Track peak memory use in default MemoryPool
+* [ARROW-531](https://issues.apache.org/jira/browse/ARROW-531) - Python: Document jemalloc, extend Pandas section, add Getting Involved
+* [ARROW-538](https://issues.apache.org/jira/browse/ARROW-538) - [C++] Set up AddressSanitizer (ASAN) builds
+* [ARROW-547](https://issues.apache.org/jira/browse/ARROW-547) - [Python] Expose Array::Slice and RecordBatch::Slice
+* [ARROW-81](https://issues.apache.org/jira/browse/ARROW-81) - [Format] Add a Category logical type (distinct from dictionary-encoding)
+
+## Task
+
+* [ARROW-268](https://issues.apache.org/jira/browse/ARROW-268) - [C++] Flesh out union implementation to have all required methods for IPC
+* [ARROW-327](https://issues.apache.org/jira/browse/ARROW-327) - [Python] Remove conda builds from Travis CI processes
+* [ARROW-353](https://issues.apache.org/jira/browse/ARROW-353) - Arrow release 0.2
+* [ARROW-359](https://issues.apache.org/jira/browse/ARROW-359) - Need to document ARROW_LIBHDFS_DIR
+* [ARROW-367](https://issues.apache.org/jira/browse/ARROW-367) - [java] converter csv/json <=> Arrow file format for Integration tests
+* [ARROW-368](https://issues.apache.org/jira/browse/ARROW-368) - Document use of LD_LIBRARY_PATH when using Python
+* [ARROW-372](https://issues.apache.org/jira/browse/ARROW-372) - Create JSON arrow file format for integration tests
+* [ARROW-506](https://issues.apache.org/jira/browse/ARROW-506) - Implement Arrow Echo server for integration testing
+* [ARROW-527](https://issues.apache.org/jira/browse/ARROW-527) - clean drill-module.conf file
+* [ARROW-558](https://issues.apache.org/jira/browse/ARROW-558) - Add KEYS files
+* [ARROW-96](https://issues.apache.org/jira/browse/ARROW-96) - C++: API documentation using Doxygen
+* [ARROW-97](https://issues.apache.org/jira/browse/ARROW-97) - Python: API documentation via sphinx-apidoc
+
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.2.0
+[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz.md5
+[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz
+[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz.asc
diff --git a/site/_release/0.3.0.md b/site/_release/0.3.0.md
new file mode 100644
index 0000000000000..d76804250efe1
--- /dev/null
+++ b/site/_release/0.3.0.md
@@ -0,0 +1,343 @@
+---
+layout: default
+title: 0.3.0 Release
+permalink: /release/0.3.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.3.0 (5 May 2017)
+
+## Download
+
+* **Source Release**: [apache-arrow-0.3.0.tar.gz][6]
+* **Verification**: [md5][3], [asc][7]
+* [Git tag d8db8f8][2]
+
+# Changelog
+
+## Bug
+
+* [ARROW-109](https://issues.apache.org/jira/browse/ARROW-109) - [C++] Investigate recursive data types limit in flatbuffers
+* [ARROW-208](https://issues.apache.org/jira/browse/ARROW-208) - Add checkstyle policy to java project
+* [ARROW-347](https://issues.apache.org/jira/browse/ARROW-347) - Add method to pass CallBack when creating a transfer pair
+* [ARROW-413](https://issues.apache.org/jira/browse/ARROW-413) - DATE type is not specified clearly
+* [ARROW-431](https://issues.apache.org/jira/browse/ARROW-431) - [Python] Review GIL release and acquisition in to_pandas conversion
+* [ARROW-443](https://issues.apache.org/jira/browse/ARROW-443) - [Python] Support for converting from strided pandas data in Table.from_pandas
+* [ARROW-451](https://issues.apache.org/jira/browse/ARROW-451) - [C++] Override DataType::Equals for other types with additional metadata
+* [ARROW-454](https://issues.apache.org/jira/browse/ARROW-454) - pojo.Field doesn't implement hashCode()
+* [ARROW-526](https://issues.apache.org/jira/browse/ARROW-526) - [Format] Update IPC.md to account for File format changes and Streaming format
+* [ARROW-565](https://issues.apache.org/jira/browse/ARROW-565) - [C++] Examine "Field::dictionary" member
+* [ARROW-570](https://issues.apache.org/jira/browse/ARROW-570) - Determine Java tools JAR location from project metadata
+* [ARROW-584](https://issues.apache.org/jira/browse/ARROW-584) - [C++] Fix compiler warnings exposed with -Wconversion
+* [ARROW-588](https://issues.apache.org/jira/browse/ARROW-588) - [C++] Fix compiler warnings on 32-bit platforms
+* [ARROW-595](https://issues.apache.org/jira/browse/ARROW-595) - [Python] StreamReader.schema returns None
+* [ARROW-604](https://issues.apache.org/jira/browse/ARROW-604) - Python: boxed Field instances are missing the reference to DataType
+* [ARROW-613](https://issues.apache.org/jira/browse/ARROW-613) - [JS] Implement random-access file format
+* [ARROW-617](https://issues.apache.org/jira/browse/ARROW-617) - Time type is not specified clearly
+* [ARROW-619](https://issues.apache.org/jira/browse/ARROW-619) - Python: Fix typos in setup.py args and LD_LIBRARY_PATH
+* [ARROW-623](https://issues.apache.org/jira/browse/ARROW-623) - segfault with __repr__ of empty Field
+* [ARROW-624](https://issues.apache.org/jira/browse/ARROW-624) - [C++] Restore MakePrimitiveArray function
+* [ARROW-627](https://issues.apache.org/jira/browse/ARROW-627) - [C++] Compatibility macros for exported extern template class declarations
+* [ARROW-628](https://issues.apache.org/jira/browse/ARROW-628) - [Python] Install nomkl metapackage when building parquet-cpp for faster Travis builds
+* [ARROW-630](https://issues.apache.org/jira/browse/ARROW-630) - [C++] IPC unloading for BooleanArray does not account for offset
+* [ARROW-636](https://issues.apache.org/jira/browse/ARROW-636) - [C++] Add Boost / other system requirements to C++ README
+* [ARROW-639](https://issues.apache.org/jira/browse/ARROW-639) - [C++] Invalid offset in slices
+* [ARROW-642](https://issues.apache.org/jira/browse/ARROW-642) - [Java] Remove temporary file in java/tools
+* [ARROW-644](https://issues.apache.org/jira/browse/ARROW-644) - Python: Cython should be a setup-only requirement
+* [ARROW-652](https://issues.apache.org/jira/browse/ARROW-652) - Remove trailing f in merge script output
+* [ARROW-654](https://issues.apache.org/jira/browse/ARROW-654) - [C++] Support timezone metadata in file/stream formats
+* [ARROW-668](https://issues.apache.org/jira/browse/ARROW-668) - [Python] Convert nanosecond timestamps to pandas.Timestamp when converting from TimestampValue
+* [ARROW-671](https://issues.apache.org/jira/browse/ARROW-671) - [GLib] License file isn't installed
+* [ARROW-673](https://issues.apache.org/jira/browse/ARROW-673) - [Java] Support additional Time metadata
+* [ARROW-677](https://issues.apache.org/jira/browse/ARROW-677) - [java] Fix checkstyle jcl-over-slf4j conflict issue
+* [ARROW-678](https://issues.apache.org/jira/browse/ARROW-678) - [GLib] Fix dependenciesfff
+* [ARROW-680](https://issues.apache.org/jira/browse/ARROW-680) - [C++] Multiarch support impacts user-supplied install prefix
+* [ARROW-682](https://issues.apache.org/jira/browse/ARROW-682) - Add self-validation checks in integration tests
+* [ARROW-683](https://issues.apache.org/jira/browse/ARROW-683) - [C++] Support date32 (DateUnit::DAY) in IPC metadata, rename date to date64
+* [ARROW-686](https://issues.apache.org/jira/browse/ARROW-686) - [C++] Account for time metadata changes, add time32 and time64 types
+* [ARROW-689](https://issues.apache.org/jira/browse/ARROW-689) - [GLib] Install header files and documents to wrong directories
+* [ARROW-691](https://issues.apache.org/jira/browse/ARROW-691) - [Java] Encode dictionary Int type in message format
+* [ARROW-697](https://issues.apache.org/jira/browse/ARROW-697) - [Java] Raise appropriate exceptions when encountering large (> INT32_MAX) record batches
+* [ARROW-699](https://issues.apache.org/jira/browse/ARROW-699) - [C++] Arrow dynamic libraries are missed on run of unit tests on Windows
+* [ARROW-702](https://issues.apache.org/jira/browse/ARROW-702) - Fix BitVector.copyFromSafe to reAllocate instead of returning false
+* [ARROW-703](https://issues.apache.org/jira/browse/ARROW-703) - Fix issue where setValueCount(0) doesn’t work in the case that we’ve shipped vectors across the wire
+* [ARROW-704](https://issues.apache.org/jira/browse/ARROW-704) - Fix bad import caused by conflicting changes
+* [ARROW-709](https://issues.apache.org/jira/browse/ARROW-709) - [C++] Restore type comparator for DecimalType
+* [ARROW-713](https://issues.apache.org/jira/browse/ARROW-713) - [C++] Fix linking issue with ipc benchmark
+* [ARROW-715](https://issues.apache.org/jira/browse/ARROW-715) - Python: Explicit pandas import makes it a hard requirement
+* [ARROW-716](https://issues.apache.org/jira/browse/ARROW-716) - error building arrow/python
+* [ARROW-720](https://issues.apache.org/jira/browse/ARROW-720) - [java] arrow should not have a dependency on slf4j bridges in compile
+* [ARROW-723](https://issues.apache.org/jira/browse/ARROW-723) - Arrow freezes on write if chunk_size=0
+* [ARROW-726](https://issues.apache.org/jira/browse/ARROW-726) - [C++] PyBuffer dtor may segfault if constructor passed an object not exporting buffer protocol
+* [ARROW-732](https://issues.apache.org/jira/browse/ARROW-732) - Schema comparison bugs in struct and union types
+* [ARROW-736](https://issues.apache.org/jira/browse/ARROW-736) - [Python] Mixed-type object DataFrame columns should not silently coerce to an Arrow type by default
+* [ARROW-738](https://issues.apache.org/jira/browse/ARROW-738) - [Python] Fix manylinux1 packaging
+* [ARROW-739](https://issues.apache.org/jira/browse/ARROW-739) - Parallel build fails non-deterministically.
+* [ARROW-740](https://issues.apache.org/jira/browse/ARROW-740) - FileReader fails for large objects
+* [ARROW-747](https://issues.apache.org/jira/browse/ARROW-747) - [C++] Fix spurious warning caused by passing dl to add_dependencies
+* [ARROW-749](https://issues.apache.org/jira/browse/ARROW-749) - [Python] Delete incomplete binary files when writing fails
+* [ARROW-753](https://issues.apache.org/jira/browse/ARROW-753) - [Python] Unit tests in arrow/python fail to link on some OS X platforms
+* [ARROW-756](https://issues.apache.org/jira/browse/ARROW-756) - [C++] Do not pass -fPIC when compiling with MSVC
+* [ARROW-757](https://issues.apache.org/jira/browse/ARROW-757) - [C++] MSVC build fails on googletest when using NMake
+* [ARROW-762](https://issues.apache.org/jira/browse/ARROW-762) - Kerberos Problem with PyArrow
+* [ARROW-776](https://issues.apache.org/jira/browse/ARROW-776) - [GLib] Cast type is wrong
+* [ARROW-777](https://issues.apache.org/jira/browse/ARROW-777) - [Java] Resolve getObject behavior per changes / discussion in ARROW-729
+* [ARROW-778](https://issues.apache.org/jira/browse/ARROW-778) - Modify merge tool to work on Windows
+* [ARROW-781](https://issues.apache.org/jira/browse/ARROW-781) - [Python/C++] Increase reference count for base object?
+* [ARROW-783](https://issues.apache.org/jira/browse/ARROW-783) - Integration tests fail for length-0 record batch
+* [ARROW-787](https://issues.apache.org/jira/browse/ARROW-787) - [GLib] Fix compilation errors caused by ARROW-758
+* [ARROW-793](https://issues.apache.org/jira/browse/ARROW-793) - [GLib] Wrong indent
+* [ARROW-794](https://issues.apache.org/jira/browse/ARROW-794) - [C++] Check whether data is contiguous in ipc::WriteTensor
+* [ARROW-797](https://issues.apache.org/jira/browse/ARROW-797) - [Python] Add updated pyarrow.* public API listing in Sphinx docs
+* [ARROW-800](https://issues.apache.org/jira/browse/ARROW-800) - [C++] Boost headers being transitively included in pyarrow
+* [ARROW-805](https://issues.apache.org/jira/browse/ARROW-805) - listing empty HDFS directory returns an error instead of returning empty list
+* [ARROW-809](https://issues.apache.org/jira/browse/ARROW-809) - C++: Writing sliced record batch to IPC writes the entire array
+* [ARROW-812](https://issues.apache.org/jira/browse/ARROW-812) - Pip install pyarrow on mac failed.
+* [ARROW-817](https://issues.apache.org/jira/browse/ARROW-817) - [C++] Fix incorrect code comment from ARROW-722
+* [ARROW-821](https://issues.apache.org/jira/browse/ARROW-821) - [Python] Extra file _table_api.h generated during Python build process
+* [ARROW-822](https://issues.apache.org/jira/browse/ARROW-822) - [Python] StreamWriter fails to open with socket as sink
+* [ARROW-826](https://issues.apache.org/jira/browse/ARROW-826) - Compilation error on Mac with -DARROW_PYTHON=on
+* [ARROW-829](https://issues.apache.org/jira/browse/ARROW-829) - Python: Parquet: Dictionary encoding is deactivated if column-wise compression was selected
+* [ARROW-830](https://issues.apache.org/jira/browse/ARROW-830) - Python: jemalloc is not anymore publicly exposed
+* [ARROW-839](https://issues.apache.org/jira/browse/ARROW-839) - [C++] Portable alternative to PyDate_to_ms function
+* [ARROW-847](https://issues.apache.org/jira/browse/ARROW-847) - C++: BUILD_BYPRODUCTS not specified anymore for gtest
+* [ARROW-852](https://issues.apache.org/jira/browse/ARROW-852) - Python: Also set Arrow Library PATHS when detection was done through pkg-config
+* [ARROW-853](https://issues.apache.org/jira/browse/ARROW-853) - [Python] It is no longer necessary to modify the RPATH of the Cython extensions on many environments
+* [ARROW-858](https://issues.apache.org/jira/browse/ARROW-858) - Remove dependency on boost regex
+* [ARROW-866](https://issues.apache.org/jira/browse/ARROW-866) - [Python] Error from file object destructor
+* [ARROW-867](https://issues.apache.org/jira/browse/ARROW-867) - [Python] Miscellaneous pyarrow MSVC fixes
+* [ARROW-875](https://issues.apache.org/jira/browse/ARROW-875) - Nullable variable length vector fillEmpties() fills an extra value
+* [ARROW-879](https://issues.apache.org/jira/browse/ARROW-879) - compat with pandas 0.20.0
+* [ARROW-882](https://issues.apache.org/jira/browse/ARROW-882) - [C++] On Windows statically built lib file overwrites lib file of shared build
+* [ARROW-886](https://issues.apache.org/jira/browse/ARROW-886) - VariableLengthVectors don't reAlloc offsets
+* [ARROW-887](https://issues.apache.org/jira/browse/ARROW-887) - [format] For backward compatibility, new unit fields must have default values matching previous implied unit
+* [ARROW-888](https://issues.apache.org/jira/browse/ARROW-888) - BitVector transfer() does not transfer ownership
+* [ARROW-895](https://issues.apache.org/jira/browse/ARROW-895) - Nullable variable length vector lastSet not set correctly
+* [ARROW-900](https://issues.apache.org/jira/browse/ARROW-900) - [Python] UnboundLocalError in ParquetDatasetPiece
+* [ARROW-903](https://issues.apache.org/jira/browse/ARROW-903) - [GLib] Remove a needless "."
+* [ARROW-914](https://issues.apache.org/jira/browse/ARROW-914) - [C++/Python] Fix Decimal ToBytes
+* [ARROW-922](https://issues.apache.org/jira/browse/ARROW-922) - Allow Flatbuffers and RapidJSON to be used locally on Windows
+* [ARROW-928](https://issues.apache.org/jira/browse/ARROW-928) - Update CMAKE script to detect unsupported msvc compilers versions
+* [ARROW-933](https://issues.apache.org/jira/browse/ARROW-933) - [Python] arrow_python bindings have debug print statement
+* [ARROW-934](https://issues.apache.org/jira/browse/ARROW-934) - [GLib] Glib sources missing from result of 02-source.sh
+* [ARROW-936](https://issues.apache.org/jira/browse/ARROW-936) - Fix release README
+* [ARROW-938](https://issues.apache.org/jira/browse/ARROW-938) - Fix Apache Rat errors from source release build
+
+## Improvement
+
+* [ARROW-316](https://issues.apache.org/jira/browse/ARROW-316) - Finalize Date type
+* [ARROW-542](https://issues.apache.org/jira/browse/ARROW-542) - [Java] Implement dictionaries in stream/file encoding
+* [ARROW-563](https://issues.apache.org/jira/browse/ARROW-563) - C++: Support non-standard gcc version strings
+* [ARROW-566](https://issues.apache.org/jira/browse/ARROW-566) - Python: Deterministic position of libarrow in manylinux1 wheels
+* [ARROW-569](https://issues.apache.org/jira/browse/ARROW-569) - [C++] Set version for *.pc
+* [ARROW-577](https://issues.apache.org/jira/browse/ARROW-577) - [C++] Refactor StreamWriter and FileWriter to have private implementations
+* [ARROW-580](https://issues.apache.org/jira/browse/ARROW-580) - C++: Also provide jemalloc_X targets if only a static or shared version is found
+* [ARROW-582](https://issues.apache.org/jira/browse/ARROW-582) - [Java] Add Date/Time Support to JSON File
+* [ARROW-589](https://issues.apache.org/jira/browse/ARROW-589) - C++: Use system provided shared jemalloc if static is unavailable
+* [ARROW-593](https://issues.apache.org/jira/browse/ARROW-593) - [C++] Rename ReadableFileInterface to RandomAccessFile
+* [ARROW-612](https://issues.apache.org/jira/browse/ARROW-612) - [Java] Field toString should show nullable flag status
+* [ARROW-615](https://issues.apache.org/jira/browse/ARROW-615) - Move ByteArrayReadableSeekableByteChannel to vector.util package
+* [ARROW-631](https://issues.apache.org/jira/browse/ARROW-631) - [GLib] Import C API (C++ API wrapper) based on GLib from https://github.com/kou/arrow-glib
+* [ARROW-646](https://issues.apache.org/jira/browse/ARROW-646) - Cache miniconda packages
+* [ARROW-647](https://issues.apache.org/jira/browse/ARROW-647) - [C++] Don't require Boost static libraries to support CentOS 7
+* [ARROW-648](https://issues.apache.org/jira/browse/ARROW-648) - [C++] Support multiarch on Debian
+* [ARROW-650](https://issues.apache.org/jira/browse/ARROW-650) - [GLib] Follow eadableFileInterface -> RnadomAccessFile change
+* [ARROW-651](https://issues.apache.org/jira/browse/ARROW-651) - [C++] Set shared library version for .deb packages
+* [ARROW-655](https://issues.apache.org/jira/browse/ARROW-655) - Implement DecimalArray
+* [ARROW-662](https://issues.apache.org/jira/browse/ARROW-662) - [Format] Factor Flatbuffer schema metadata into a Schema.fbs
+* [ARROW-664](https://issues.apache.org/jira/browse/ARROW-664) - Make C++ Arrow serialization deterministic
+* [ARROW-674](https://issues.apache.org/jira/browse/ARROW-674) - [Java] Support additional Timestamp timezone metadata
+* [ARROW-675](https://issues.apache.org/jira/browse/ARROW-675) - [GLib] Update package metadata
+* [ARROW-676](https://issues.apache.org/jira/browse/ARROW-676) - [java] move from MinorType to FieldType in ValueVectors to carry all the relevant type bits
+* [ARROW-679](https://issues.apache.org/jira/browse/ARROW-679) - [Format] Change RecordBatch and Field length members from int to long
+* [ARROW-681](https://issues.apache.org/jira/browse/ARROW-681) - [C++] Build Arrow on Windows with dynamically linked boost
+* [ARROW-684](https://issues.apache.org/jira/browse/ARROW-684) - Python: More informative message when parquet-cpp but not parquet-arrow is available
+* [ARROW-688](https://issues.apache.org/jira/browse/ARROW-688) - [C++] Use CMAKE_INSTALL_INCLUDEDIR for consistency
+* [ARROW-690](https://issues.apache.org/jira/browse/ARROW-690) - Only send JIRA updates to issues@arrow.apache.org
+* [ARROW-700](https://issues.apache.org/jira/browse/ARROW-700) - Add headroom interface for allocator.
+* [ARROW-706](https://issues.apache.org/jira/browse/ARROW-706) - [GLib] Add package install document
+* [ARROW-707](https://issues.apache.org/jira/browse/ARROW-707) - Python: All none-Pandas column should be converted to NullArray
+* [ARROW-708](https://issues.apache.org/jira/browse/ARROW-708) - [C++] Some IPC code simplification, perf analysis
+* [ARROW-712](https://issues.apache.org/jira/browse/ARROW-712) - [C++] Implement Array::Accept as inline visitor
+* [ARROW-719](https://issues.apache.org/jira/browse/ARROW-719) - [GLib] Support prepared source archive release
+* [ARROW-724](https://issues.apache.org/jira/browse/ARROW-724) - Add "How to Contribute" section to README
+* [ARROW-725](https://issues.apache.org/jira/browse/ARROW-725) - [Format] Constant length list type
+* [ARROW-727](https://issues.apache.org/jira/browse/ARROW-727) - [Python] Write memoryview-compatible objects in NativeFile.write with zero copy
+* [ARROW-728](https://issues.apache.org/jira/browse/ARROW-728) - [C++/Python] Add arrow::Table function for removing a column
+* [ARROW-731](https://issues.apache.org/jira/browse/ARROW-731) - [C++] Add shared library related versions to .pc
+* [ARROW-741](https://issues.apache.org/jira/browse/ARROW-741) - [Python] Add Python 3.6 to Travis CI
+* [ARROW-743](https://issues.apache.org/jira/browse/ARROW-743) - [C++] Consolidate unit tests for code in array.h
+* [ARROW-744](https://issues.apache.org/jira/browse/ARROW-744) - [GLib] Re-add an assertion to garrow_table_new() test
+* [ARROW-745](https://issues.apache.org/jira/browse/ARROW-745) - [C++] Allow use of system cpplint
+* [ARROW-746](https://issues.apache.org/jira/browse/ARROW-746) - [GLib] Add garrow_array_get_data_type()
+* [ARROW-751](https://issues.apache.org/jira/browse/ARROW-751) - [Python] Rename all Cython extensions to "private" status with leading underscore
+* [ARROW-752](https://issues.apache.org/jira/browse/ARROW-752) - [Python] Construct pyarrow.DictionaryArray from boxed pyarrow array objects
+* [ARROW-754](https://issues.apache.org/jira/browse/ARROW-754) - [GLib] Add garrow_array_is_null()
+* [ARROW-755](https://issues.apache.org/jira/browse/ARROW-755) - [GLib] Add garrow_array_get_value_type()
+* [ARROW-758](https://issues.apache.org/jira/browse/ARROW-758) - [C++] Fix compiler warnings on MSVC x64
+* [ARROW-761](https://issues.apache.org/jira/browse/ARROW-761) - [Python] Add function to compute the total size of tensor payloads, including metadata and padding
+* [ARROW-763](https://issues.apache.org/jira/browse/ARROW-763) - C++: Use `python-config` to find libpythonX.X.dylib
+* [ARROW-765](https://issues.apache.org/jira/browse/ARROW-765) - [Python] Make generic ArrowException subclass value error
+* [ARROW-769](https://issues.apache.org/jira/browse/ARROW-769) - [GLib] Support building without installed Arrow C++
+* [ARROW-770](https://issues.apache.org/jira/browse/ARROW-770) - [C++] Move clang-tidy/format config files back to C++ source tree
+* [ARROW-774](https://issues.apache.org/jira/browse/ARROW-774) - [GLib] Remove needless LICENSE.txt copy
+* [ARROW-775](https://issues.apache.org/jira/browse/ARROW-775) - [Java] add simple constructors to value vectors
+* [ARROW-779](https://issues.apache.org/jira/browse/ARROW-779) - [C++/Python] Raise exception if old metadata encountered
+* [ARROW-782](https://issues.apache.org/jira/browse/ARROW-782) - [C++] Change struct to class for objects that meet the criteria in the Google style guide
+* [ARROW-788](https://issues.apache.org/jira/browse/ARROW-788) - Possible nondeterminism in Tensor serialization code
+* [ARROW-795](https://issues.apache.org/jira/browse/ARROW-795) - [C++] Combine libarrow/libarrow_io/libarrow_ipc
+* [ARROW-802](https://issues.apache.org/jira/browse/ARROW-802) - [GLib] Add read examples
+* [ARROW-803](https://issues.apache.org/jira/browse/ARROW-803) - [GLib] Update package repository URL
+* [ARROW-804](https://issues.apache.org/jira/browse/ARROW-804) - [GLib] Update build document
+* [ARROW-806](https://issues.apache.org/jira/browse/ARROW-806) - [GLib] Support add/remove a column from table
+* [ARROW-807](https://issues.apache.org/jira/browse/ARROW-807) - [GLib] Update "Since" tag
+* [ARROW-808](https://issues.apache.org/jira/browse/ARROW-808) - [GLib] Remove needless ignore entries
+* [ARROW-810](https://issues.apache.org/jira/browse/ARROW-810) - [GLib] Remove io/ipc prefix
+* [ARROW-811](https://issues.apache.org/jira/browse/ARROW-811) - [GLib] Add GArrowBuffer
+* [ARROW-815](https://issues.apache.org/jira/browse/ARROW-815) - [Java] Allow for expanding underlying buffer size after allocation
+* [ARROW-816](https://issues.apache.org/jira/browse/ARROW-816) - [C++] Use conda packages for RapidJSON, Flatbuffers to speed up builds
+* [ARROW-818](https://issues.apache.org/jira/browse/ARROW-818) - [Python] Review public pyarrow.* API completeness and update docs
+* [ARROW-820](https://issues.apache.org/jira/browse/ARROW-820) - [C++] Build dependencies for Parquet library without arrow support
+* [ARROW-825](https://issues.apache.org/jira/browse/ARROW-825) - [Python] Generalize pyarrow.from_pylist to accept any object implementing the PySequence protocol
+* [ARROW-827](https://issues.apache.org/jira/browse/ARROW-827) - [Python] Variety of Parquet improvements to support Dask integration
+* [ARROW-828](https://issues.apache.org/jira/browse/ARROW-828) - [CPP] Document new requirement (libboost-regex-dev) in README.md
+* [ARROW-832](https://issues.apache.org/jira/browse/ARROW-832) - [C++] Upgrade thirdparty gtest to 1.8.0
+* [ARROW-833](https://issues.apache.org/jira/browse/ARROW-833) - [Python] "Quickstart" build / environment setup guide for Python developers
+* [ARROW-841](https://issues.apache.org/jira/browse/ARROW-841) - [Python] Add pyarrow build to Appveyor
+* [ARROW-844](https://issues.apache.org/jira/browse/ARROW-844) - [Format] Revise format/README.md to reflect progress reaching a more complete specification
+* [ARROW-845](https://issues.apache.org/jira/browse/ARROW-845) - [Python] Sync FindArrow.cmake changes from parquet-cpp
+* [ARROW-846](https://issues.apache.org/jira/browse/ARROW-846) - [GLib] Add GArrowTensor, GArrowInt8Tensor and GArrowUInt8Tensor
+* [ARROW-848](https://issues.apache.org/jira/browse/ARROW-848) - [Python] Improvements / fixes to conda quickstart guide
+* [ARROW-849](https://issues.apache.org/jira/browse/ARROW-849) - [C++] Add optional $ARROW_BUILD_TOOLCHAIN environment variable option for configuring build environment
+* [ARROW-857](https://issues.apache.org/jira/browse/ARROW-857) - [Python] Automate publishing Python documentation to arrow-site
+* [ARROW-860](https://issues.apache.org/jira/browse/ARROW-860) - [C++] Decide if typed Tensor subclasses are worthwhile
+* [ARROW-861](https://issues.apache.org/jira/browse/ARROW-861) - [Python] Move DEVELOPMENT.md to Sphinx docs
+* [ARROW-862](https://issues.apache.org/jira/browse/ARROW-862) - [Python] Improve source build instructions in README
+* [ARROW-863](https://issues.apache.org/jira/browse/ARROW-863) - [GLib] Use GBytes to implement zero-copy
+* [ARROW-864](https://issues.apache.org/jira/browse/ARROW-864) - [GLib] Unify Array files
+* [ARROW-868](https://issues.apache.org/jira/browse/ARROW-868) - [GLib] Use GBytes to reduce copy
+* [ARROW-871](https://issues.apache.org/jira/browse/ARROW-871) - [GLib] Unify DataType files
+* [ARROW-876](https://issues.apache.org/jira/browse/ARROW-876) - [GLib] Unify ArrayBuffer files
+* [ARROW-877](https://issues.apache.org/jira/browse/ARROW-877) - [GLib] Add garrow_array_get_null_bitmap()
+* [ARROW-878](https://issues.apache.org/jira/browse/ARROW-878) - [GLib] Add garrow_binary_array_get_buffer()
+* [ARROW-892](https://issues.apache.org/jira/browse/ARROW-892) - [GLib] Fix GArrowTensor document
+* [ARROW-893](https://issues.apache.org/jira/browse/ARROW-893) - Add GLib document to Web site
+* [ARROW-894](https://issues.apache.org/jira/browse/ARROW-894) - [GLib] Add GArrowPoolBuffer
+* [ARROW-896](https://issues.apache.org/jira/browse/ARROW-896) - [Docs] Add Jekyll plugin for including rendered Jupyter notebooks on website
+* [ARROW-898](https://issues.apache.org/jira/browse/ARROW-898) - [C++] Expand metadata support to field level, provide for sharing instances of KeyValueMetadata
+* [ARROW-904](https://issues.apache.org/jira/browse/ARROW-904) - [GLib] Simplify error check codes
+* [ARROW-907](https://issues.apache.org/jira/browse/ARROW-907) - C++: Convenience construct Table from schema and arrays
+* [ARROW-908](https://issues.apache.org/jira/browse/ARROW-908) - [GLib] Unify OutputStream files
+* [ARROW-910](https://issues.apache.org/jira/browse/ARROW-910) - [C++] Write 0-length EOS indicator at end of stream
+* [ARROW-916](https://issues.apache.org/jira/browse/ARROW-916) - [GLib] Add GArrowBufferOutputStream
+* [ARROW-917](https://issues.apache.org/jira/browse/ARROW-917) - [GLib] Add GArrowBufferReader
+* [ARROW-918](https://issues.apache.org/jira/browse/ARROW-918) - [GLib] Use GArrowBuffer for read
+* [ARROW-919](https://issues.apache.org/jira/browse/ARROW-919) - [GLib] Use "id" to get type enum value from GArrowDataType
+* [ARROW-920](https://issues.apache.org/jira/browse/ARROW-920) - [GLib] Add Lua examples
+* [ARROW-925](https://issues.apache.org/jira/browse/ARROW-925) - [GLib] Fix GArrowBufferReader test
+* [ARROW-930](https://issues.apache.org/jira/browse/ARROW-930) - javadoc generation fails with java 8
+* [ARROW-931](https://issues.apache.org/jira/browse/ARROW-931) - [GLib] Reconstruct input stream
+
+## New Feature
+
+* [ARROW-231](https://issues.apache.org/jira/browse/ARROW-231) - C++: Add typed Resize to PoolBuffer
+* [ARROW-281](https://issues.apache.org/jira/browse/ARROW-281) - [C++] IPC/RPC support on Win32 platforms
+* [ARROW-341](https://issues.apache.org/jira/browse/ARROW-341) - [Python] Making libpyarrow available to third parties
+* [ARROW-452](https://issues.apache.org/jira/browse/ARROW-452) - [C++/Python] Merge "Feather" file format implementation
+* [ARROW-459](https://issues.apache.org/jira/browse/ARROW-459) - [C++] Implement IPC round trip for DictionaryArray, dictionaries shared across record batches
+* [ARROW-483](https://issues.apache.org/jira/browse/ARROW-483) - [C++/Python] Provide access to "custom_metadata" Field attribute in IPC setting
+* [ARROW-491](https://issues.apache.org/jira/browse/ARROW-491) - [C++] Add FixedWidthBinary type
+* [ARROW-493](https://issues.apache.org/jira/browse/ARROW-493) - [C++] Allow in-memory array over 2^31 -1 elements but require splitting at IPC / RPC boundaries
+* [ARROW-502](https://issues.apache.org/jira/browse/ARROW-502) - [C++/Python] Add MemoryPool implementation that logs allocation activity to std::cout
+* [ARROW-510](https://issues.apache.org/jira/browse/ARROW-510) - Add integration tests for date and time types
+* [ARROW-520](https://issues.apache.org/jira/browse/ARROW-520) - [C++] Add STL-compliant allocator that hooks into an arrow::MemoryPool
+* [ARROW-528](https://issues.apache.org/jira/browse/ARROW-528) - [Python] Support _metadata or _common_metadata files when reading Parquet directories
+* [ARROW-534](https://issues.apache.org/jira/browse/ARROW-534) - [C++] Add IPC tests for date/time types
+* [ARROW-539](https://issues.apache.org/jira/browse/ARROW-539) - [Python] Support reading Parquet datasets with standard partition directory schemes
+* [ARROW-550](https://issues.apache.org/jira/browse/ARROW-550) - [Format] Add a TensorMessage type
+* [ARROW-552](https://issues.apache.org/jira/browse/ARROW-552) - [Python] Add scalar value support for Dictionary type
+* [ARROW-557](https://issues.apache.org/jira/browse/ARROW-557) - [Python] Explicitly opt in to HDFS unit tests
+* [ARROW-568](https://issues.apache.org/jira/browse/ARROW-568) - [C++] Add default implementations for TypeVisitor, ArrayVisitor methods that return NotImplemented
+* [ARROW-574](https://issues.apache.org/jira/browse/ARROW-574) - Python: Add support for nested Python lists in Pandas conversion
+* [ARROW-576](https://issues.apache.org/jira/browse/ARROW-576) - [C++] Complete round trip Union file/stream IPC tests
+* [ARROW-578](https://issues.apache.org/jira/browse/ARROW-578) - [C++] Add CMake option to add custom $CXXFLAGS
+* [ARROW-598](https://issues.apache.org/jira/browse/ARROW-598) - [Python] Add support for converting pyarrow.Buffer to a memoryview with zero copy
+* [ARROW-603](https://issues.apache.org/jira/browse/ARROW-603) - [C++] Add RecordBatch::Validate method that at least checks that schema matches the array metadata
+* [ARROW-605](https://issues.apache.org/jira/browse/ARROW-605) - [C++] Refactor generic ArrayLoader class, support work for Feather merge
+* [ARROW-606](https://issues.apache.org/jira/browse/ARROW-606) - [C++] Upgrade to flatbuffers 1.6.0
+* [ARROW-608](https://issues.apache.org/jira/browse/ARROW-608) - [Format] Days since epoch date type
+* [ARROW-610](https://issues.apache.org/jira/browse/ARROW-610) - [C++] Win32 compatibility in file.cc
+* [ARROW-616](https://issues.apache.org/jira/browse/ARROW-616) - [C++] Remove -g flag in release builds
+* [ARROW-618](https://issues.apache.org/jira/browse/ARROW-618) - [Python] Implement support for DatetimeTZ custom type from pandas
+* [ARROW-620](https://issues.apache.org/jira/browse/ARROW-620) - [C++] Add date/time support to JSON reader/writer for integration testing
+* [ARROW-621](https://issues.apache.org/jira/browse/ARROW-621) - [C++] Implement an "inline visitor" template that enables visitor-pattern-like code without virtual function dispatch
+* [ARROW-625](https://issues.apache.org/jira/browse/ARROW-625) - [C++] Add time unit to TimeType::ToString
+* [ARROW-626](https://issues.apache.org/jira/browse/ARROW-626) - [Python] Enable pyarrow.BufferReader to read from any Python object implementing the buffer/memoryview protocol
+* [ARROW-632](https://issues.apache.org/jira/browse/ARROW-632) - [Python] Add support for FixedWidthBinary type
+* [ARROW-635](https://issues.apache.org/jira/browse/ARROW-635) - [C++] Add JSON read/write support for FixedWidthBinary
+* [ARROW-637](https://issues.apache.org/jira/browse/ARROW-637) - [Format] Add time zone metadata to Timestamp type
+* [ARROW-656](https://issues.apache.org/jira/browse/ARROW-656) - [C++] Implement IO interface that can read and write to a fixed-size mutable buffer
+* [ARROW-657](https://issues.apache.org/jira/browse/ARROW-657) - [Python] Write and read tensors (with zero copy) into shared memory
+* [ARROW-658](https://issues.apache.org/jira/browse/ARROW-658) - [C++] Implement in-memory arrow::Tensor objects
+* [ARROW-659](https://issues.apache.org/jira/browse/ARROW-659) - [C++] Add multithreaded memcpy implementation (for hardware where it helps)
+* [ARROW-660](https://issues.apache.org/jira/browse/ARROW-660) - [C++] Restore function that can read a complete encapsulated record batch message
+* [ARROW-661](https://issues.apache.org/jira/browse/ARROW-661) - [C++] Add a Flatbuffer metadata type that supports array data over 2^31 - 1 elements
+* [ARROW-663](https://issues.apache.org/jira/browse/ARROW-663) - [Java] Support additional Time metadata + vector value accessors
+* [ARROW-669](https://issues.apache.org/jira/browse/ARROW-669) - [Python] Attach proper tzinfo when computing boxed scalars for TimestampArray
+* [ARROW-687](https://issues.apache.org/jira/browse/ARROW-687) - [C++] Build and run full test suite in Appveyor
+* [ARROW-698](https://issues.apache.org/jira/browse/ARROW-698) - [C++] Add options to StreamWriter/FileWriter to permit large record batches
+* [ARROW-701](https://issues.apache.org/jira/browse/ARROW-701) - [Java] Support additional Date metadata
+* [ARROW-710](https://issues.apache.org/jira/browse/ARROW-710) - [Python] Enable Feather APIs to read and write using Python file-like objects
+* [ARROW-717](https://issues.apache.org/jira/browse/ARROW-717) - [C++] IPC zero-copy round trips for arrow::Tensor
+* [ARROW-718](https://issues.apache.org/jira/browse/ARROW-718) - [Python] Expose arrow::Tensor with conversions to/from NumPy arrays
+* [ARROW-722](https://issues.apache.org/jira/browse/ARROW-722) - [Python] pandas conversions for new date and time types/metadata
+* [ARROW-729](https://issues.apache.org/jira/browse/ARROW-729) - [Java] Add vector type for 32-bit date as days since UNIX epoch
+* [ARROW-733](https://issues.apache.org/jira/browse/ARROW-733) - [C++/Format] Change name of Fixed Width Binary to Fixed *Size* Binary for consistency
+* [ARROW-734](https://issues.apache.org/jira/browse/ARROW-734) - [Python] Support for pyarrow on Windows / MSVC
+* [ARROW-735](https://issues.apache.org/jira/browse/ARROW-735) - [C++] Developer instruction document for MSVC on Windows
+* [ARROW-737](https://issues.apache.org/jira/browse/ARROW-737) - [C++] Support obtaining mutable slices of mutable buffers
+* [ARROW-768](https://issues.apache.org/jira/browse/ARROW-768) - [Java] Change the "boxed" object representation of date and time types
+* [ARROW-771](https://issues.apache.org/jira/browse/ARROW-771) - [Python] Add APIs for reading individual Parquet row groups
+* [ARROW-773](https://issues.apache.org/jira/browse/ARROW-773) - [C++] Add function to create arrow::Table with column appended to existing table
+* [ARROW-865](https://issues.apache.org/jira/browse/ARROW-865) - [Python] Verify Parquet roundtrips for new date/time types
+* [ARROW-880](https://issues.apache.org/jira/browse/ARROW-880) - [GLib] Add garrow_primitive_array_get_buffer()
+* [ARROW-890](https://issues.apache.org/jira/browse/ARROW-890) - [GLib] Add GArrowMutableBuffer
+* [ARROW-926](https://issues.apache.org/jira/browse/ARROW-926) - Update KEYS to include wesm
+
+## Task
+
+* [ARROW-52](https://issues.apache.org/jira/browse/ARROW-52) - Set up project blog
+* [ARROW-670](https://issues.apache.org/jira/browse/ARROW-670) - Arrow 0.3 release
+* [ARROW-672](https://issues.apache.org/jira/browse/ARROW-672) - [Format] Bump metadata version for 0.3 release
+* [ARROW-748](https://issues.apache.org/jira/browse/ARROW-748) - [Python] Pin runtime library versions in conda-forge packages to force upgrades
+* [ARROW-798](https://issues.apache.org/jira/browse/ARROW-798) - [Docs] Publish Format Markdown documents somehow on arrow.apache.org
+* [ARROW-869](https://issues.apache.org/jira/browse/ARROW-869) - [JS] Rename directory to js/
+* [ARROW-95](https://issues.apache.org/jira/browse/ARROW-95) - Scaffold Main Documentation using asciidoc
+* [ARROW-98](https://issues.apache.org/jira/browse/ARROW-98) - Java: API documentation
+
+## Test
+
+* [ARROW-836](https://issues.apache.org/jira/browse/ARROW-836) - Test for timedelta compat with pandas
+* [ARROW-927](https://issues.apache.org/jira/browse/ARROW-927) - C++/Python: Add manylinux1 builds to Travis matrix
+
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.3.0
+[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.md5
+[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
+[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.asc
diff --git a/site/_release/index.md b/site/_release/index.md
new file mode 100644
index 0000000000000..d42227529fde6
--- /dev/null
+++ b/site/_release/index.md
@@ -0,0 +1,35 @@
+---
+layout: default
+title: 0.3.0 Release
+permalink: /release/index.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow Releases
+
+Navigate to the release page for downloads and the changelog.
+
+* [0.3.0 (5 May 2017)][1]
+* [0.2.0 (18 February 2017)][2]
+* [0.1.0 (10 October 2016)][3]
+
+[1]: {{ site.baseurl }}/release/0.3.0.html
+[2]: {{ site.baseurl }}/release/0.2.0.html
+[3]: {{ site.baseurl }}/release/0.1.0.html
diff --git a/site/install.html b/site/install.html
deleted file mode 100644
index 7734eeb303315..0000000000000
--- a/site/install.html
+++ /dev/null
@@ -1,11 +0,0 @@
----
-layout: default
----
-<h2>Current Version: 0.2.0</h2>
-<h3>Released: May 5, 2017</h2>
-g<p>Apache Arrow 0.2.0 is an early release and the APIs are still evolving. The
-metadata and physical data representation should be fairly stable as we have
-spent time finalizing the details. </p>
-<p><a href="https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0">source release</a></p>
-<p><a href="https://github.com/apache/arrow/releases/tag/apache-arrow-0.2.0">tag apache-arrow-0.2.0</a></p>
-<p><a href="http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.2.0%22">java artifacts on maven central</a></p>
diff --git a/site/install.md b/site/install.md
new file mode 100644
index 0000000000000..b1e7990c1b8f3
--- /dev/null
+++ b/site/install.md
@@ -0,0 +1,75 @@
+---
+layout: default
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+## Current Version: 0.3.0
+
+### Released: 5 May 2017
+
+Apache Arrow 0.3.0 is the third major release of the project and has seen
+significant iteration and hardening of logical types and the binary formats. It
+is safe for production use, though there may be API changes and binary format
+breaks in the future.
+
+### Source release
+
+* **Source Release**: [apache-arrow-0.3.0.tar.gz][6]
+* **Verification**: [md5][3], [asc][7]
+* [Git tag d8db8f8][2]
+
+### Java Packages
+
+[Java Artifacts on Maven Central][4]
+
+### C++ and Python Conda Packages (Unofficial)
+
+We have provided binary conda packages on [conda-forge][5] for the following
+platforms:
+
+* Linux and OS X (Python 2.7, 3.5, and 3.6)
+* Windows (Python 3.5 and 3.6)
+
+Install them with:
+
+
+```shell
+conda install arrow-cpp -c conda-forge
+conda install pyarrow -c conda-forge
+```
+
+### Python Wheels on PyPI (Unofficial)
+
+We have provided Linux binary wheels on PyPI, which can be installed with pip.
+
+```shell
+pip install pyarrow
+```
+
+These include the Apache Arrow and Apache Parquet C++ binary libraries bundled
+with the wheel.
+
+[1]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.3.0
+[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.md5
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.3.0%22
+[5]: http://conda-forge.github.io
+[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
+[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.asc

From af0c21e9a793c4cc9ec011d09cb18648678b5b43 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 8 May 2017 22:50:31 -0400
Subject: [PATCH 0622/1644] ARROW-957: [Doc] Add HDFS and Windows documents to
 doxygen output

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #659 from xhochy/ARROW-957 and squashes the following commits:

8ff87ad [Uwe L. Korn] ARROW-957: [Doc] Add HDFS and Windows documents to doxygen output
---
 cpp/{doc => apidoc}/HDFS.md    | 0
 cpp/{doc => apidoc}/Windows.md | 6 ++++--
 cpp/apidoc/index.md            | 6 ++++++
 3 files changed, 10 insertions(+), 2 deletions(-)
 rename cpp/{doc => apidoc}/HDFS.md (100%)
 rename cpp/{doc => apidoc}/Windows.md (94%)

diff --git a/cpp/doc/HDFS.md b/cpp/apidoc/HDFS.md
similarity index 100%
rename from cpp/doc/HDFS.md
rename to cpp/apidoc/HDFS.md
diff --git a/cpp/doc/Windows.md b/cpp/apidoc/Windows.md
similarity index 94%
rename from cpp/doc/Windows.md
rename to cpp/apidoc/Windows.md
index 64f6a1b98f62e..b9a6143a744a4 100644
--- a/cpp/doc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -26,7 +26,9 @@ other development instructions for Windows here.
 Microsoft provides the free Visual Studio 2017 Community edition. When doing
 development, you must launch the developer command prompt using
 
-```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64```
+```
+"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64
+```
 
 It's easiest to configure a console emulator like [cmder][3] to automatically
 launch this when starting a new development console.
@@ -80,4 +82,4 @@ cmake --build . --config Release
 
 [1]: https://conda.io/miniconda.html
 [2]: https://conda-forge.github.io/
-[3]: http://cmder.net/
\ No newline at end of file
+[3]: http://cmder.net/
diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
index fdac4969beb9b..fc10e0fddeb80 100644
--- a/cpp/apidoc/index.md
+++ b/cpp/apidoc/index.md
@@ -27,6 +27,12 @@ the usage of the C++ API for Arrow and the leaf libraries that add additional
 functionality such as using [jemalloc](http://jemalloc.net/) as an allocator
 for Arrow structures.
 
+Table of Contents
+-----------------
+
+ * Instructions on how to build Arrow C++ on [Windows](Windows.md)
+ * How to access [HDFS](HDFS.md)
+
 Getting Started
 ---------------
 

From 74ad4a82b1c6c376809dd0e2a2be9fe2fa54a9ac Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 9 May 2017 08:58:59 -0400
Subject: [PATCH 0623/1644] ARROW-984: [GLib] Add Go examples

Author: Kouhei Sutou <kou@clear-code.com>

Closes #661 from kou/glib-example-go and squashes the following commits:

22e35cb [Kouhei Sutou] [GLib] Add Go examples
---
 c_glib/.gitignore                       |   5 +
 c_glib/example/go/Makefile              |  35 +++++
 c_glib/example/go/README.md             |  78 ++++++++++++
 c_glib/example/go/arrow-1.0/arrow.go.in |  38 ++++++
 c_glib/example/go/arrow-1.0/config.json |  21 +++
 c_glib/example/go/read-batch.go         | 100 +++++++++++++++
 c_glib/example/go/read-stream.go        | 101 +++++++++++++++
 c_glib/example/go/write-batch.go        | 162 ++++++++++++++++++++++++
 c_glib/example/go/write-stream.go       | 162 ++++++++++++++++++++++++
 9 files changed, 702 insertions(+)
 create mode 100644 c_glib/example/go/Makefile
 create mode 100644 c_glib/example/go/README.md
 create mode 100644 c_glib/example/go/arrow-1.0/arrow.go.in
 create mode 100644 c_glib/example/go/arrow-1.0/config.json
 create mode 100644 c_glib/example/go/read-batch.go
 create mode 100644 c_glib/example/go/read-stream.go
 create mode 100644 c_glib/example/go/write-batch.go
 create mode 100644 c_glib/example/go/write-stream.go

diff --git a/c_glib/.gitignore b/c_glib/.gitignore
index 6f2de80d4f79e..796b842083f9e 100644
--- a/c_glib/.gitignore
+++ b/c_glib/.gitignore
@@ -41,3 +41,8 @@ Makefile.in
 /example/build
 /example/read-batch
 /example/read-stream
+!/example/go/Makefile
+/example/go/read-batch
+/example/go/read-stream
+/example/go/write-batch
+/example/go/write-stream
diff --git a/c_glib/example/go/Makefile b/c_glib/example/go/Makefile
new file mode 100644
index 0000000000000..d8831122d4dc6
--- /dev/null
+++ b/c_glib/example/go/Makefile
@@ -0,0 +1,35 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+PROGRAMS =					\
+	read-batch				\
+	read-stream				\
+	write-batch				\
+	write-stream
+
+all: build
+
+generate:
+	$$GOPATH/bin/gir-generator		\
+	  -o $$GOPATH/src/gir/arrow-1.0		\
+	  -config arrow-1.0/config.json		\
+	  arrow-1.0/arrow.go.in
+
+build: $(PROGRAMS)
+
+clean:
+	rm -f $(PROGRAMS)
+
+.SUFFIXES: .go
+
+.go:
+	go build -o $@ $<
diff --git a/c_glib/example/go/README.md b/c_glib/example/go/README.md
new file mode 100644
index 0000000000000..2054055e6554e
--- /dev/null
+++ b/c_glib/example/go/README.md
@@ -0,0 +1,78 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Arrow Go example
+
+There are Go example codes in this directory.
+
+## How to run
+
+All example codes use
+[go-gir-generator](https://github.com/linuxdeepin/go-gir-generator) to
+use Arrow GLib based bindings.
+
+See [../../README.md](../../README.md) how to install Arrow GLib. You
+can use packages to install Arrow GLib. The following instructions
+assumes that you've installed Arrow GLib by package. Package name is
+`libarrow-glib-dev` on Debian GNU/Linux and Ubuntu, `arrow-glib-devel`
+on CentOS.
+
+Here are command lines to install go-gir-generator on Debian GNU/Linux
+and Ubuntu:
+
+```text
+% sudo apt install -V -y libarrow-glib-dev golang git libgirepository1.0-dev libgudev-1.0-dev
+% export GOPATH=$HOME
+% go get github.com/linuxdeepin/go-gir-generator
+% cd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
+% make build copyfile
+% mkdir -p $GOPATH/bin/
+% cp -a out/gir-generator $GOPATH/bin/
+% cp -a out/src/gir/ $GOPATH/src/
+```
+
+Now, you can generate Arrow bindings for Go:
+
+```text
+% git clone https://github.com/apache/arrow.git ~/arrow
+% cd ~/arrow/c_glib/example/go
+% make generate
+```
+
+Then you can build all example codes:
+
+```text
+% cd ~/arrow/c_glib/example/go
+% make
+% ./write-batch  # Write data in batch mode
+% ./read-batch   # Read the written batch mode data
+% ./write-stream # Write data in stream mode
+% ./read-stream  # Read the written stream mode data
+```
+
+## Go example codes
+
+Here are example codes in this directory:
+
+  * `write-batch.go`: It shows how to write Arrow array to file in
+    batch mode.
+
+  * `read-batch.go`: It shows how to read Arrow array from file in
+    batch mode.
+
+  * `write-stream.go`: It shows how to write Arrow array to file in
+    stream mode.
+
+  * `read-stream.go`: It shows how to read Arrow array from file in
+    stream mode.
diff --git a/c_glib/example/go/arrow-1.0/arrow.go.in b/c_glib/example/go/arrow-1.0/arrow.go.in
new file mode 100644
index 0000000000000..bd124afd596b2
--- /dev/null
+++ b/c_glib/example/go/arrow-1.0/arrow.go.in
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package arrow
+
+/*
+#include "arrow.gen.h"
+#cgo pkg-config: arrow-glib
+
+[<.g_list_funcs>]
+[<.g_error_free>]
+[<.g_free>]
+*/
+import "C"
+import "unsafe"
+
+import (
+	"gir/glib-2.0"
+	"gir/gobject-2.0"
+)
+
+[<.go_utils>]
+
+[<.go_bindings>]
diff --git a/c_glib/example/go/arrow-1.0/config.json b/c_glib/example/go/arrow-1.0/config.json
new file mode 100644
index 0000000000000..eea9a898cdcdf
--- /dev/null
+++ b/c_glib/example/go/arrow-1.0/config.json
@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+{
+    "namespace": "Arrow",
+    "version": "1.0"
+}
diff --git a/c_glib/example/go/read-batch.go b/c_glib/example/go/read-batch.go
new file mode 100644
index 0000000000000..3ad842376b02b
--- /dev/null
+++ b/c_glib/example/go/read-batch.go
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"os"
+	"log"
+	"fmt"
+	"strings"
+)
+
+import "gir/arrow-1.0"
+
+func PrintColumnValue(column *arrow.Array, i int64) {
+	valueType := column.GetValueType()
+	switch valueType {
+	case arrow.TypeUint8:
+		fmt.Print(arrow.ToUInt8Array(column).GetValue(i))
+	case arrow.TypeUint16:
+		fmt.Print(arrow.ToUInt16Array(column).GetValue(i))
+	case arrow.TypeUint32:
+		fmt.Print(arrow.ToUInt32Array(column).GetValue(i))
+	case arrow.TypeUint64:
+		fmt.Print(arrow.ToUInt64Array(column).GetValue(i))
+	case arrow.TypeInt8:
+		fmt.Print(arrow.ToInt8Array(column).GetValue(i))
+	case arrow.TypeInt16:
+		fmt.Print(arrow.ToInt16Array(column).GetValue(i))
+	case arrow.TypeInt32:
+		fmt.Print(arrow.ToInt32Array(column).GetValue(i))
+	case arrow.TypeInt64:
+		fmt.Print(arrow.ToInt64Array(column).GetValue(i))
+	case arrow.TypeFloat:
+		fmt.Print(arrow.ToFloatArray(column).GetValue(i))
+	case arrow.TypeDouble:
+		fmt.Print(arrow.ToDoubleArray(column).GetValue(i))
+	default:
+		fmt.Printf("unknown(%s)", valueType)
+	}
+}
+
+func PrintRecordBatch(recordBatch *arrow.RecordBatch) {
+	nColumns := recordBatch.GetNColumns()
+	for i := uint32(0); i < nColumns; i++ {
+		column := recordBatch.GetColumn(i)
+		columnName := recordBatch.GetColumnName(i)
+		fmt.Printf("  %s: [", columnName)
+		nRows := recordBatch.GetNRows()
+		for j := int64(0); j < nRows; j++ {
+			if j > 0 {
+				fmt.Print(", ")
+			}
+			PrintColumnValue(column, j)
+		}
+		fmt.Println("]")
+	}
+}
+
+func main() {
+	var path string
+	if len(os.Args) < 2 {
+		path = "/tmp/batch.arrow"
+	} else {
+		path = os.Args[1]
+	}
+	input, err := arrow.NewMemoryMappedInputStream(path);
+	if err != nil {
+		log.Fatalf("Failed to open path: <%s>: %v", path, err)
+	}
+	reader, err := arrow.NewFileReader(input)
+	if err != nil {
+		log.Fatalf("Failed to parse data: %v", err)
+	}
+	var i uint32
+	nRecordBatches := reader.GetNRecordBatches()
+	for i = 0; i < nRecordBatches; i++ {
+		recordBatch, err := reader.GetRecordBatch(i)
+		if err != nil {
+			log.Fatalf("Failed to get record batch[%d]: %v", i, err)
+		}
+		fmt.Println(strings.Repeat("=", 40))
+		fmt.Printf("record-batch[%d]:\n", i)
+		PrintRecordBatch(recordBatch)
+	}
+}
diff --git a/c_glib/example/go/read-stream.go b/c_glib/example/go/read-stream.go
new file mode 100644
index 0000000000000..f3367988223e4
--- /dev/null
+++ b/c_glib/example/go/read-stream.go
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"os"
+	"log"
+	"fmt"
+	"strings"
+)
+
+import "gir/arrow-1.0"
+
+func PrintColumnValue(column *arrow.Array, i int64) {
+	valueType := column.GetValueType()
+	switch valueType {
+	case arrow.TypeUint8:
+		fmt.Print(arrow.ToUInt8Array(column).GetValue(i))
+	case arrow.TypeUint16:
+		fmt.Print(arrow.ToUInt16Array(column).GetValue(i))
+	case arrow.TypeUint32:
+		fmt.Print(arrow.ToUInt32Array(column).GetValue(i))
+	case arrow.TypeUint64:
+		fmt.Print(arrow.ToUInt64Array(column).GetValue(i))
+	case arrow.TypeInt8:
+		fmt.Print(arrow.ToInt8Array(column).GetValue(i))
+	case arrow.TypeInt16:
+		fmt.Print(arrow.ToInt16Array(column).GetValue(i))
+	case arrow.TypeInt32:
+		fmt.Print(arrow.ToInt32Array(column).GetValue(i))
+	case arrow.TypeInt64:
+		fmt.Print(arrow.ToInt64Array(column).GetValue(i))
+	case arrow.TypeFloat:
+		fmt.Print(arrow.ToFloatArray(column).GetValue(i))
+	case arrow.TypeDouble:
+		fmt.Print(arrow.ToDoubleArray(column).GetValue(i))
+	default:
+		fmt.Printf("unknown(%s)", valueType)
+	}
+}
+
+func PrintRecordBatch(recordBatch *arrow.RecordBatch) {
+	nColumns := recordBatch.GetNColumns()
+	for i := uint32(0); i < nColumns; i++ {
+		column := recordBatch.GetColumn(i)
+		columnName := recordBatch.GetColumnName(i)
+		fmt.Printf("  %s: [", columnName)
+		nRows := recordBatch.GetNRows()
+		for j := int64(0); j < nRows; j++ {
+			if j > 0 {
+				fmt.Print(", ")
+			}
+			PrintColumnValue(column, j)
+		}
+		fmt.Println("]")
+	}
+}
+
+func main() {
+	var path string
+	if len(os.Args) < 2 {
+		path = "/tmp/stream.arrow"
+	} else {
+		path = os.Args[1]
+	}
+	input, err := arrow.NewMemoryMappedInputStream(path);
+	if err != nil {
+		log.Fatalf("Failed to open path: <%s>: %v", path, err)
+	}
+	reader, err := arrow.NewStreamReader(input)
+	if err != nil {
+		log.Fatalf("Failed to parse data: %v", err)
+	}
+	for i := 0; true; i++ {
+		recordBatch, err := reader.GetNextRecordBatch()
+		if err != nil {
+			log.Fatalf("Failed to get next record batch: %v", err)
+		}
+		if recordBatch == nil {
+			break
+		}
+		fmt.Println(strings.Repeat("=", 40))
+		fmt.Printf("record-batch[%d]:\n", i)
+		PrintRecordBatch(recordBatch)
+	}
+}
diff --git a/c_glib/example/go/write-batch.go b/c_glib/example/go/write-batch.go
new file mode 100644
index 0000000000000..f5e946a0c5c23
--- /dev/null
+++ b/c_glib/example/go/write-batch.go
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"os"
+	"log"
+)
+
+import "gir/arrow-1.0"
+
+func BuildUInt8Array() *arrow.Array {
+	builder := arrow.NewUInt8ArrayBuilder()
+	for _, value := range []uint8{1, 2, 4, 8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildUInt16Array() *arrow.Array {
+	builder := arrow.NewUInt16ArrayBuilder()
+	for _, value := range []uint16{1, 2, 4, 8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildUInt32Array() *arrow.Array {
+	builder := arrow.NewUInt32ArrayBuilder()
+	for _, value := range []uint32{1, 2, 4, 8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildUInt64Array() *arrow.Array {
+	builder := arrow.NewUInt64ArrayBuilder()
+	for _, value := range []uint64{1, 2, 4, 8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildInt8Array() *arrow.Array {
+	builder := arrow.NewInt8ArrayBuilder()
+	for _, value := range []int8{1, -2, 4, -8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildInt16Array() *arrow.Array {
+	builder := arrow.NewInt16ArrayBuilder()
+	for _, value := range []int16{1, -2, 4, -8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildInt32Array() *arrow.Array {
+	builder := arrow.NewInt32ArrayBuilder()
+	for _, value := range []int32{1, -2, 4, -8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildInt64Array() *arrow.Array {
+	builder := arrow.NewInt64ArrayBuilder()
+	for _, value := range []int64{1, -2, 4, -8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildFloatArray() *arrow.Array {
+	builder := arrow.NewFloatArrayBuilder()
+	for _, value := range []float32{1.1, -2.2, 4.4, -8.8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildDoubleArray() *arrow.Array {
+	builder := arrow.NewDoubleArrayBuilder()
+	for _, value := range []float64{1.1, -2.2, 4.4, -8.8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func main() {
+	var output_path string
+	if len(os.Args) < 2 {
+		output_path = "/tmp/batch.arrow"
+	} else {
+		output_path = os.Args[1]
+	}
+
+	fields := []*arrow.Field{
+		arrow.NewField("uint8",  arrow.NewUInt8DataType()),
+		arrow.NewField("uint16", arrow.NewUInt16DataType()),
+		arrow.NewField("uint32", arrow.NewUInt32DataType()),
+		arrow.NewField("uint64", arrow.NewUInt64DataType()),
+		arrow.NewField("int8",   arrow.NewInt8DataType()),
+		arrow.NewField("int16",  arrow.NewInt16DataType()),
+		arrow.NewField("int32",  arrow.NewInt32DataType()),
+		arrow.NewField("int64",  arrow.NewInt64DataType()),
+		arrow.NewField("float",  arrow.NewFloatDataType()),
+		arrow.NewField("double", arrow.NewDoubleDataType()),
+	}
+	schema := arrow.NewSchema(fields)
+
+	output, err := arrow.NewFileOutputStream(output_path, false)
+	if err != nil {
+		log.Fatalf("Failed to open path: <%s>: %v", output_path, err)
+	}
+	writer, err := arrow.NewFileWriter(output, schema)
+	if err != nil {
+		log.Fatalf("Failed to create writer: %v", err)
+	}
+
+	columns := []*arrow.Array{
+		BuildUInt8Array(),
+		BuildUInt16Array(),
+		BuildUInt32Array(),
+		BuildUInt64Array(),
+		BuildInt8Array(),
+		BuildInt16Array(),
+		BuildInt32Array(),
+		BuildInt64Array(),
+		BuildFloatArray(),
+		BuildDoubleArray(),
+	}
+
+	recordBatch := arrow.NewRecordBatch(schema, 4, columns)
+	writer.WriteRecordBatch(recordBatch)
+
+	slicedColumns := make([]*arrow.Array, len(columns))
+	for i, column := range columns {
+		slicedColumns[i] = column.Slice(1, 3)
+	}
+	recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	writer.WriteRecordBatch(recordBatch)
+
+	writer.Close()
+}
diff --git a/c_glib/example/go/write-stream.go b/c_glib/example/go/write-stream.go
new file mode 100644
index 0000000000000..80ebd2ba1b01f
--- /dev/null
+++ b/c_glib/example/go/write-stream.go
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"os"
+	"log"
+)
+
+import "gir/arrow-1.0"
+
+func BuildUInt8Array() *arrow.Array {
+	builder := arrow.NewUInt8ArrayBuilder()
+	for _, value := range []uint8{1, 2, 4, 8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildUInt16Array() *arrow.Array {
+	builder := arrow.NewUInt16ArrayBuilder()
+	for _, value := range []uint16{1, 2, 4, 8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildUInt32Array() *arrow.Array {
+	builder := arrow.NewUInt32ArrayBuilder()
+	for _, value := range []uint32{1, 2, 4, 8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildUInt64Array() *arrow.Array {
+	builder := arrow.NewUInt64ArrayBuilder()
+	for _, value := range []uint64{1, 2, 4, 8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildInt8Array() *arrow.Array {
+	builder := arrow.NewInt8ArrayBuilder()
+	for _, value := range []int8{1, -2, 4, -8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildInt16Array() *arrow.Array {
+	builder := arrow.NewInt16ArrayBuilder()
+	for _, value := range []int16{1, -2, 4, -8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildInt32Array() *arrow.Array {
+	builder := arrow.NewInt32ArrayBuilder()
+	for _, value := range []int32{1, -2, 4, -8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildInt64Array() *arrow.Array {
+	builder := arrow.NewInt64ArrayBuilder()
+	for _, value := range []int64{1, -2, 4, -8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildFloatArray() *arrow.Array {
+	builder := arrow.NewFloatArrayBuilder()
+	for _, value := range []float32{1.1, -2.2, 4.4, -8.8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func BuildDoubleArray() *arrow.Array {
+	builder := arrow.NewDoubleArrayBuilder()
+	for _, value := range []float64{1.1, -2.2, 4.4, -8.8} {
+		builder.Append(value)
+	}
+	return builder.Finish()
+}
+
+func main() {
+	var output_path string
+	if len(os.Args) < 2 {
+		output_path = "/tmp/stream.arrow"
+	} else {
+		output_path = os.Args[1]
+	}
+
+	fields := []*arrow.Field{
+		arrow.NewField("uint8",  arrow.NewUInt8DataType()),
+		arrow.NewField("uint16", arrow.NewUInt16DataType()),
+		arrow.NewField("uint32", arrow.NewUInt32DataType()),
+		arrow.NewField("uint64", arrow.NewUInt64DataType()),
+		arrow.NewField("int8",   arrow.NewInt8DataType()),
+		arrow.NewField("int16",  arrow.NewInt16DataType()),
+		arrow.NewField("int32",  arrow.NewInt32DataType()),
+		arrow.NewField("int64",  arrow.NewInt64DataType()),
+		arrow.NewField("float",  arrow.NewFloatDataType()),
+		arrow.NewField("double", arrow.NewDoubleDataType()),
+	}
+	schema := arrow.NewSchema(fields)
+
+	output, err := arrow.NewFileOutputStream(output_path, false)
+	if err != nil {
+		log.Fatalf("Failed to open path: <%s>: %v", output_path, err)
+	}
+	writer, err := arrow.NewStreamWriter(output, schema)
+	if err != nil {
+		log.Fatalf("Failed to create writer: %v", err)
+	}
+
+	columns := []*arrow.Array{
+		BuildUInt8Array(),
+		BuildUInt16Array(),
+		BuildUInt32Array(),
+		BuildUInt64Array(),
+		BuildInt8Array(),
+		BuildInt16Array(),
+		BuildInt32Array(),
+		BuildInt64Array(),
+		BuildFloatArray(),
+		BuildDoubleArray(),
+	}
+
+	recordBatch := arrow.NewRecordBatch(schema, 4, columns)
+	writer.WriteRecordBatch(recordBatch)
+
+	slicedColumns := make([]*arrow.Array, len(columns))
+	for i, column := range columns {
+		slicedColumns[i] = column.Slice(1, 3)
+	}
+	recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	writer.WriteRecordBatch(recordBatch)
+
+	writer.Close()
+}

From ccf83f48516386e75b844bbc9ad46c1a9a259a92 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 9 May 2017 09:00:35 -0400
Subject: [PATCH 0624/1644] ARROW-985: [GLib] Update package information

Author: Kouhei Sutou <kou@clear-code.com>

Closes #662 from kou/glib-update-package-info and squashes the following commits:

9005ddd [Kouhei Sutou] [GLib] Update package information
---
 c_glib/README.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index b6e08e358d244..7d0ea3a9d030b 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -55,6 +55,7 @@ There are supported platforms:
   * Debian GNU/Linux Jessie
   * Ubuntu 16.04 LTS
   * Ubuntu 16.10
+  * Ubuntu 17.04
   * CentOS 7
 
 You can feedback to https://github.com/kou/arrow-packages about
@@ -62,6 +63,13 @@ packages things.
 
 #### Debian GNU/Linux jessie
 
+You need to install `apt-transport-https` to use HTTPS for APT
+repository.
+
+```text
+% sudo apt install -y apt-transport-https
+```
+
 You need to add the following apt-lines to
 `/etc/apt/sources.list.d/groonga.list`:
 
@@ -84,7 +92,7 @@ Now you can install Arrow GLib packages:
 % sudo apt install -y libarrow-glib-dev
 ```
 
-#### Ubuntu 16.04 LTS and Ubuntu 16.10
+#### Ubuntu
 
 You need to add an APT repository:
 

From 670612e6fdf699486641ed0d39d22257eb8acdb2 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Tue, 9 May 2017 16:01:05 +0200
Subject: [PATCH 0625/1644] ARROW-954: Flag for compiling Arrow with
 header-only boost

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #647 from pcmoritz/boost-header-only and squashes the following commits:

3605341 [Philipp Moritz] run find_package(Boost) in header-only mode
445de50 [Philipp Moritz] bring back tests
3f1ef1f [Philipp Moritz] reintroduce ARROW_HDFS
a047ad4 [Philipp Moritz] deactivate json-integration-test for header-only boost
b2b2015 [Philipp Moritz] make it possible to compile arrow with header-only boost
---
 cpp/CMakeLists.txt               | 75 +++++++++++++++++++++-----------
 cpp/src/arrow/io/CMakeLists.txt  |  4 +-
 cpp/src/arrow/ipc/CMakeLists.txt | 29 ++++++------
 3 files changed, 69 insertions(+), 39 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 72e5ea90948b9..21463794879fc 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -101,6 +101,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Rely on jemalloc shared libraries where relevant"
     ON)
 
+  option(ARROW_HDFS
+    "Build the Arrow HDFS bridge"
+    ON)
+
   option(ARROW_BOOST_USE_SHARED
     "Rely on boost shared libraries where relevant"
     ON)
@@ -136,6 +140,12 @@ if(NOT ARROW_BUILD_BENCHMARKS)
   set(NO_BENCHMARKS 1)
 endif()
 
+if(ARROW_HDFS)
+  set(ARROW_BOOST_HEADER_ONLY 0)
+else()
+  set(ARROW_BOOST_HEADER_ONLY 1)
+endif()
+
 include(BuildUtils)
 
 ############################################################
@@ -437,44 +447,54 @@ if (ARROW_BOOST_USE_SHARED)
     add_definitions(-DBOOST_ALL_DYN_LINK)
   endif()
 
-  find_package(Boost COMPONENTS system filesystem REQUIRED)
-  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-    set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-    set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+  if (ARROW_BOOST_HEADER_ONLY)
+    find_package(Boost)
   else()
-    set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-    set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+    find_package(Boost COMPONENTS system filesystem REQUIRED)
+    if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+      set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+      set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+    else()
+      set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+      set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+    endif()
+    set(BOOST_SYSTEM_LIBRARY boost_system_shared)
+    set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
   endif()
-  set(BOOST_SYSTEM_LIBRARY boost_system_shared)
-  set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
 else()
   # Find static boost headers and libs
   # TODO Differentiate here between release and debug builds
   set(Boost_USE_STATIC_LIBS ON)
-  find_package(Boost COMPONENTS system filesystem regex REQUIRED)
-  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-    set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-    set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+  if (ARROW_BOOST_HEADER_ONLY)
+    find_package(Boost)
   else()
-    set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-    set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+    find_package(Boost COMPONENTS system filesystem regex REQUIRED)
+    if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+      set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+      set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+    else()
+      set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+      set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+    endif()
+    set(BOOST_SYSTEM_LIBRARY boost_system_static)
+    set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
   endif()
-  set(BOOST_SYSTEM_LIBRARY boost_system_static)
-  set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
 endif()
 
 message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
 message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
 
-ADD_THIRDPARTY_LIB(boost_system
-    STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}"
-    SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
+if (NOT ARROW_BOOST_HEADER_ONLY)
+  ADD_THIRDPARTY_LIB(boost_system
+      STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}"
+      SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
 
-ADD_THIRDPARTY_LIB(boost_filesystem
-    STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
-    SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
+  ADD_THIRDPARTY_LIB(boost_filesystem
+      STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
+      SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
 
-SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
+  SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
+endif()
 
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 
@@ -938,8 +958,6 @@ set(ARROW_SRCS
   src/arrow/visitor.cc
 
   src/arrow/io/file.cc
-  src/arrow/io/hdfs.cc
-  src/arrow/io/hdfs-internal.cc
   src/arrow/io/interfaces.cc
   src/arrow/io/memory.cc
 
@@ -948,6 +966,13 @@ set(ARROW_SRCS
   src/arrow/util/key_value_metadata.cc
 )
 
+if (NOT ARROW_BOOST_HEADER_ONLY)
+  set(ARROW_SRCS ${ARROW_SRCS}
+    src/arrow/io/hdfs.cc
+    src/arrow/io/hdfs-internal.cc
+  )
+endif()
+
 if (ARROW_IPC)
   set(ARROW_SRCS ${ARROW_SRCS}
     src/arrow/ipc/feather.cc
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index cd489746b48ea..544b4b79430c2 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -19,7 +19,9 @@
 # arrow_io : Arrow IO interfaces
 
 ADD_ARROW_TEST(io-file-test)
-ADD_ARROW_TEST(io-hdfs-test)
+if (NOT ARROW_BOOST_HEADER_ONLY)
+  ADD_ARROW_TEST(io-hdfs-test)
+endif()
 ADD_ARROW_TEST(io-memory-test)
 
 ADD_ARROW_BENCHMARK(io-memory-benchmark)
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 41ab5d7a1f39a..9cc61bced0619 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -21,19 +21,22 @@
 ADD_ARROW_TEST(feather-test)
 ADD_ARROW_TEST(ipc-read-write-test)
 ADD_ARROW_TEST(ipc-json-test)
-ADD_ARROW_TEST(json-integration-test)
-
-if (ARROW_BUILD_TESTS)
-  target_link_libraries(json-integration-test
-    gflags)
-
-  if (UNIX)
-    if (APPLE)
-      set_target_properties(json-integration-test
-        PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-    else()
-      target_link_libraries(json-integration-test
-        pthread)
+
+if (NOT ARROW_BOOST_HEADER_ONLY)
+  ADD_ARROW_TEST(json-integration-test)
+
+  if (ARROW_BUILD_TESTS)
+    target_link_libraries(json-integration-test
+      gflags)
+
+    if (UNIX)
+      if (APPLE)
+        set_target_properties(json-integration-test
+          PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+      else()
+        target_link_libraries(json-integration-test
+          pthread)
+      endif()
     endif()
   endif()
 endif()

From 22c738cc7b26bff9e7319d438dd3fef1238d46ad Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Tue, 9 May 2017 15:55:27 -0400
Subject: [PATCH 0626/1644] ARROW-874: [JS] Read dictionary-encoded vectors

Author: Brian Hulette <brian.hulette@ccri.com>
Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Author: Brian Hulette <hulettbh@gmail.com>

Closes #655 from TheNeuralBit/js_dictionary and squashes the following commits:

4fbaf9d [Brian Hulette] add unit tests, fix errors in file format dictionary reading
d89c84b [Brian Hulette] Add dictionary support for file format
5bdf8a1 [Emilio Lahr-Vivaz] dictionary encoding
c6eff38 [Brian Hulette] Updated API documenation
4c84362 [Brian Hulette] added struct_example tests (Struct type, and multiple record batches)
5a2efe6 [Brian Hulette] add tests for streaming format
9aed94b [Brian Hulette] Fix file format, unit tests
e2e0d4d [Emilio Lahr-Vivaz] renaming reader method
844b5e9 [Emilio Lahr-Vivaz] fix for file format
bfb7754 [Emilio Lahr-Vivaz] cleanup
162c9be [Emilio Lahr-Vivaz] working for streaming
290497f [Emilio Lahr-Vivaz] js support multiple record batches
4b3b412 [Emilio Lahr-Vivaz] initial support for streaming file format, added FixedSizeList
c9d705d [Brian Hulette] Created npm build script
53db587 [Brian Hulette] Fixes to make tests pass
304c669 [Brian Hulette] Added basic js unit tests
---
 js/.gitignore                         |   2 +
 js/README.md                          |  17 +-
 js/bin/arrow2csv.js                   |   9 +-
 js/bin/arrow_schema.js                |   8 +-
 js/examples/read_file.html            |   6 +-
 js/{postinstall.sh => flatbuffers.sh} |   1 +
 js/lib/arrow.ts                       | 440 +++++++++++++++----
 js/lib/bitarray.ts                    |  17 +-
 js/lib/types.ts                       | 581 +++++++++++++++++++-------
 js/package.json                       |  12 +-
 js/spec/arrow.js                      | 179 ++++++++
 js/spec/dictionary-stream.arrow       | Bin 0 -> 1776 bytes
 js/spec/dictionary.arrow              | Bin 0 -> 2522 bytes
 js/spec/simple-stream.arrow           | Bin 0 -> 1188 bytes
 js/spec/simple.arrow                  | Bin 0 -> 1642 bytes
 js/spec/struct_example-stream.arrow   | Bin 0 -> 1884 bytes
 js/spec/struct_example.arrow          | Bin 0 -> 2354 bytes
 17 files changed, 1008 insertions(+), 264 deletions(-)
 rename js/{postinstall.sh => flatbuffers.sh} (96%)
 create mode 100644 js/spec/arrow.js
 create mode 100644 js/spec/dictionary-stream.arrow
 create mode 100644 js/spec/dictionary.arrow
 create mode 100644 js/spec/simple-stream.arrow
 create mode 100644 js/spec/simple.arrow
 create mode 100644 js/spec/struct_example-stream.arrow
 create mode 100644 js/spec/struct_example.arrow

diff --git a/js/.gitignore b/js/.gitignore
index 3b97e3ab95707..f67c1cc9fc764 100644
--- a/js/.gitignore
+++ b/js/.gitignore
@@ -2,3 +2,5 @@ lib/*_generated.js
 dist
 node_modules
 typings
+.idea
+*.iml
diff --git a/js/README.md b/js/README.md
index 98ef75674ede0..cdabf5479ae11 100644
--- a/js/README.md
+++ b/js/README.md
@@ -20,6 +20,7 @@ From this directory, run:
 $ npm install   # pull dependencies
 $ tsc           # build typescript
 $ webpack       # bundle for the browser
+$ npm test      # run unit tests
 ```
 
 ### Usage
@@ -42,9 +43,19 @@ Include `dist/arrow-bundle.js` in a `<script />` tag:
 See [examples/read_file.html](examples/read_file.html) for a usage example - or try it out now at [theneuralbit.github.io/arrow](http://theneuralbit.github.io/arrow)
 
 ### API
-##### `arrow.loadSchema(buffer)`
+##### `arrow.getReader(buffer)`
+Returns an `ArrowReader` object representing the Arrow file or stream contained in
+the `buffer`.
+
+##### `ArrowReader.loadNextBatch()`
+Loads the next record batch and returns it's length.
+
+##### `ArrowReader.getSchema()`
 Returns a JSON representation of the file's Arrow schema.
 
-##### `arrow.loadVectors(buffer)`
-Returns a dictionary of `Vector` objects, one for each column, indexed by the column's name.
+##### `ArrowReader.getVectors()`
+Returns a list of `Vector` objects, one for each column.
 Vector objects have, at minimum, a `get(i)` method and a `length` attribute.
+
+##### `ArrowReader.getVector(name: String)`
+Return a Vector object for column `name`
diff --git a/js/bin/arrow2csv.js b/js/bin/arrow2csv.js
index 48df2f9648319..8122e95a883fe 100755
--- a/js/bin/arrow2csv.js
+++ b/js/bin/arrow2csv.js
@@ -19,7 +19,7 @@
 
 var fs = require('fs')
 var process = require('process');
-var loadVectors = require('../dist/arrow.js').loadVectors;
+var arrow = require('../dist/arrow.js');
 var program = require('commander');
 
 function list (val) {
@@ -38,10 +38,11 @@ if (!program.schema) {
 }
 
 var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
-var vectors = loadVectors(buf);
+var reader = arrow.getReader(buf);
+reader.loadNextBatch();
 
-for (var i = 0; i < vectors[program.schema[0]].length; i += 1|0) {
+for (var i = 0; i < reader.getVector(program.schema[0]).length; i += 1|0) {
     console.log(program.schema.map(function (field) {
-        return '' + vectors[field].get(i);
+        return '' + reader.getVector(field).get(i);
     }).join(','));
 }
diff --git a/js/bin/arrow_schema.js b/js/bin/arrow_schema.js
index 704477863bf95..44dabb413dbe8 100755
--- a/js/bin/arrow_schema.js
+++ b/js/bin/arrow_schema.js
@@ -19,7 +19,11 @@
 
 var fs = require('fs');
 var process = require('process');
-var loadSchema = require('../dist/arrow.js').loadSchema;
+var arrow = require('../dist/arrow.js');
 
 var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
-console.log(JSON.stringify(loadSchema(buf), null, '\t'));
+var reader = arrow.getReader(buf);
+console.log(JSON.stringify(reader.getSchema(), null, '\t'));
+//console.log(JSON.stringify(reader.getVectors(), null, '\t'));
+console.log('block count: ' + reader.getBatchCount());
+
diff --git a/js/examples/read_file.html b/js/examples/read_file.html
index 02b6f0805c770..933b142b527ce 100644
--- a/js/examples/read_file.html
+++ b/js/examples/read_file.html
@@ -40,9 +40,11 @@
 }
 reader.onload = function (evt) {
   var buf = new Uint8Array(evt.target.result);
-  var schema = arrow.loadSchema(buf);
-  var vectors = arrow.loadVectors(buf);
+  var schema = arrow.loadSchemaFromStream(buf);
+  var vectors = arrow.loadVectorsFromStream(buf);
   var length = vectors[schema[0].name].length;
+console.log(JSON.stringify(schema, null, '\t'));
+console.log(JSON.stringify(vectors, null, '\t'));
 
   var thead = document.getElementById("thead");
   var tbody = document.getElementById("tbody");
diff --git a/js/postinstall.sh b/js/flatbuffers.sh
similarity index 96%
rename from js/postinstall.sh
rename to js/flatbuffers.sh
index 1e6622fa4f2ee..99d2815fd2071 100755
--- a/js/postinstall.sh
+++ b/js/flatbuffers.sh
@@ -15,4 +15,5 @@
 echo "Compiling flatbuffer schemas..."
 #flatc -o lib --js ../format/Message.fbs ../format/File.fbs
 flatc -o lib --js ../format/*.fbs
+rm -f lib/Arrow_generated.js
 cat lib/*_generated.js > lib/Arrow_generated.js
diff --git a/js/lib/arrow.ts b/js/lib/arrow.ts
index 0762885aef8cc..74def4dc94dc3 100644
--- a/js/lib/arrow.ts
+++ b/js/lib/arrow.ts
@@ -17,48 +17,186 @@
 
 import { flatbuffers } from 'flatbuffers';
 import { org } from './Arrow_generated';
-var arrow = org.apache.arrow;
 import { vectorFromField, Vector } from './types';
 
-export function loadVectors(buf) {
-    var fileLength = buf.length, bb, footerLengthOffset, footerLength,
-        footerOffset, footer, schema, field, type, type_str, i,
-        len, rb_metas, rb_meta, rtrn, recordBatchBlock, recordBatchBlocks = [];
-    var vectors : Vector[] = [];
+import ByteBuffer = flatbuffers.ByteBuffer;
+var Footer = org.apache.arrow.flatbuf.Footer;
+var Message = org.apache.arrow.flatbuf.Message;
+var MessageHeader = org.apache.arrow.flatbuf.MessageHeader;
+var RecordBatch = org.apache.arrow.flatbuf.RecordBatch;
+var DictionaryBatch = org.apache.arrow.flatbuf.DictionaryBatch;
+var Schema = org.apache.arrow.flatbuf.Schema;
+var Type = org.apache.arrow.flatbuf.Type;
+var VectorType = org.apache.arrow.flatbuf.VectorType;
+
+export class ArrowReader {
+
+    private bb;
+    private schema: any = [];
+    private vectors: Vector[];
+    private vectorMap: any = {};
+    private dictionaries: any = {};
+    private batches: any = [];
+    private batchIndex: number = 0;
+
+    constructor(bb, schema, vectors: Vector[], batches, dictionaries) {
+        this.bb = bb;
+        this.schema = schema;
+        this.vectors = vectors;
+        for (var i = 0; i < vectors.length; i += 1|0) {
+            this.vectorMap[vectors[i].name] = vectors[i]
+        }
+        this.batches = batches;
+        this.dictionaries = dictionaries;
+    }
+
+    loadNextBatch() {
+        if (this.batchIndex < this.batches.length) {
+            var batch = this.batches[this.batchIndex];
+            this.batchIndex += 1;
+            loadVectors(this.bb, this.vectors, batch);
+            return batch.length;
+        } else {
+            return 0;
+        }
+    }
+
+    getSchema() {
+        return this.schema;
+    }
+
+    getVectors() {
+        return this.vectors;
+    }
+
+    getVector(name) {
+        return this.vectorMap[name];
+    }
 
-    bb = new flatbuffers.ByteBuffer(buf);
+    getBatchCount() {
+        return this.batches.length;
+    }
+
+    // the index of the next batch to be loaded
+    getBatchIndex() {
+        return this.batchIndex;
+    }
 
-    footer = _loadFooter(bb);
+    // set the index of the next batch to be loaded
+    setBatchIndex(i: number) {
+        this.batchIndex = i;
+    }
+}
 
-    schema = footer.schema();
+export function getSchema(buf) { return getReader(buf).getSchema(); }
+
+export function getReader(buf) : ArrowReader {
+    if (_checkMagic(buf, 0)) {
+        return getFileReader(buf);
+    } else {
+        return getStreamReader(buf);
+    }
+}
+
+export function getStreamReader(buf) : ArrowReader {
+    var bb = new ByteBuffer(buf);
+
+    var schema = _loadSchema(bb),
+        field,
+        vectors: Vector[] = [],
+        i,j,
+        iLen,jLen,
+        batch,
+        recordBatches = [],
+        dictionaryBatches = [],
+        dictionaries = {};
+
+    for (i = 0, iLen = schema.fieldsLength(); i < iLen; i += 1|0) {
+        field = schema.fields(i);
+        _createDictionaryVectors(field, dictionaries);
+        vectors.push(vectorFromField(field, dictionaries));
+    }
+
+    while (bb.position() < bb.capacity()) {
+      batch = _loadBatch(bb);
+      if (batch == null) {
+          break;
+      } else if (batch.type == MessageHeader.DictionaryBatch) {
+          dictionaryBatches.push(batch);
+      } else if (batch.type == MessageHeader.RecordBatch) {
+          recordBatches.push(batch)
+      } else {
+          console.error("Expected batch type" + MessageHeader.RecordBatch + " or " +
+              MessageHeader.DictionaryBatch + " but got " + batch.type);
+      }
+    }
+
+    // load dictionary vectors
+    for (i = 0; i < dictionaryBatches.length; i += 1|0) {
+      batch = dictionaryBatches[i];
+      loadVectors(bb, [dictionaries[batch.id]], batch);
+    }
+
+    return new ArrowReader(bb, parseSchema(schema), vectors, recordBatches, dictionaries);
+}
+
+export function getFileReader (buf) : ArrowReader {
+    var bb = new ByteBuffer(buf);
+
+    var footer = _loadFooter(bb);
+
+    var schema = footer.schema();
+    var i, len, field,
+        vectors: Vector[] = [],
+        block,
+        batch,
+        recordBatchBlocks = [],
+        dictionaryBatchBlocks = [],
+        dictionaries = {};
 
     for (i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
         field = schema.fields(i);
-        vectors.push(vectorFromField(field));
+        _createDictionaryVectors(field, dictionaries);
+        vectors.push(vectorFromField(field, dictionaries));
+    }
+
+    for (i = 0; i < footer.dictionariesLength(); i += 1|0) {
+        block = footer.dictionaries(i);
+        dictionaryBatchBlocks.push({
+            offset: block.offset().low,
+            metaDataLength: block.metaDataLength(),
+            bodyLength: block.bodyLength().low,
+        })
     }
 
     for (i = 0; i < footer.recordBatchesLength(); i += 1|0) {
-        recordBatchBlock = footer.recordBatches(i);
+        block = footer.recordBatches(i);
         recordBatchBlocks.push({
-            offset: recordBatchBlock.offset(),
-            metaDataLength: recordBatchBlock.metaDataLength(),
-            bodyLength: recordBatchBlock.bodyLength(),
+            offset: block.offset().low,
+            metaDataLength: block.metaDataLength(),
+            bodyLength: block.bodyLength().low,
         })
     }
 
-    loadBuffersIntoVectors(recordBatchBlocks, bb, vectors);
-    var rtrn : any = {};
-    for (var i : any = 0; i < vectors.length; i += 1|0) {
-      rtrn[vectors[i].name] = vectors[i]
+    var dictionaryBatches = dictionaryBatchBlocks.map(function (block) {
+        bb.setPosition(block.offset);
+        // TODO: Make sure this is a dictionary batch
+        return _loadBatch(bb);
+    });
+
+    var recordBatches = recordBatchBlocks.map(function (block) {
+        bb.setPosition(block.offset);
+        // TODO: Make sure this is a record batch
+        return _loadBatch(bb);
+    });
+
+    // load dictionary vectors
+    for (i = 0; i < dictionaryBatches.length; i += 1|0) {
+        batch = dictionaryBatches[i];
+        loadVectors(bb, [dictionaries[batch.id]], batch);
     }
-    return rtrn;
-}
 
-export function loadSchema(buf) {
-    var footer = _loadFooter(new flatbuffers.ByteBuffer(buf));
-    var schema = footer.schema();
-
-    return parseSchema(schema);
+    return new ArrowReader(bb, parseSchema(schema), vectors, recordBatches, dictionaries);
 }
 
 function _loadFooter(bb) {
@@ -81,7 +219,7 @@ function _loadFooter(bb) {
 
     var footerLengthOffset: number = fileLength - MAGIC.length - 4;
     bb.setPosition(footerLengthOffset);
-    var footerLength: number = Int64FromByteBuffer(bb, footerLengthOffset)
+    var footerLength: number = Int32FromByteBuffer(bb, footerLengthOffset)
 
     if (footerLength <= 0 || footerLength + MAGIC.length*2 + 4 > fileLength)  {
       console.log("Invalid footer length: " + footerLength)
@@ -89,19 +227,166 @@ function _loadFooter(bb) {
 
     var footerOffset: number = footerLengthOffset - footerLength;
     bb.setPosition(footerOffset);
-    var footer = arrow.flatbuf.Footer.getRootAsFooter(bb);
+    var footer = Footer.getRootAsFooter(bb);
 
     return footer;
 }
 
-function Int64FromByteBuffer(bb, offset) {
+function _loadSchema(bb) {
+    var message =_loadMessage(bb);
+    if (message.headerType() != MessageHeader.Schema) {
+        console.error("Expected header type " + MessageHeader.Schema + " but got " + message.headerType());
+        return;
+    }
+    return message.header(new Schema());
+}
+
+function _loadBatch(bb) {
+    var message = _loadMessage(bb);
+    if (message == null) {
+        return;
+    } else if (message.headerType() == MessageHeader.RecordBatch) {
+        var batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low }
+        return _loadRecordBatch(bb, batch);
+    } else if (message.headerType() == MessageHeader.DictionaryBatch) {
+        var batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low }
+        return _loadDictionaryBatch(bb, batch);
+    } else {
+        console.error("Expected header type " + MessageHeader.RecordBatch + " or " + MessageHeader.DictionaryBatch +
+            " but got " + message.headerType());
+        return;
+    }
+}
+
+function _loadRecordBatch(bb, batch) {
+    var data = batch.header;
+    var i, nodes_ = [], nodesLength = data.nodesLength();
+    var buffer, buffers_ = [], buffersLength = data.buffersLength();
+
+    for (i = 0; i < nodesLength; i += 1) {
+        nodes_.push(data.nodes(i));
+    }
+    for (i = 0; i < buffersLength; i += 1) {
+        buffer = data.buffers(i);
+        buffers_.push({ offset: bb.position() + buffer.offset().low, length: buffer.length().low });
+    }
+    // position the buffer after the body to read the next message
+    bb.setPosition(bb.position() + batch.length);
+
+    return { nodes: nodes_, buffers: buffers_, length: data.length().low, type: MessageHeader.RecordBatch };
+}
+
+function _loadDictionaryBatch(bb, batch) {
+    var id_ = batch.header.id().toFloat64().toString(), data = batch.header.data();
+    var i, nodes_ = [], nodesLength = data.nodesLength();
+    var buffer, buffers_ = [], buffersLength = data.buffersLength();
+
+    for (i = 0; i < nodesLength; i += 1) {
+        nodes_.push(data.nodes(i));
+    }
+    for (i = 0; i < buffersLength; i += 1) {
+        buffer = data.buffers(i);
+        buffers_.push({ offset: bb.position() + buffer.offset().low, length: buffer.length().low });
+    }
+    // position the buffer after the body to read the next message
+    bb.setPosition(bb.position() + batch.length);
+
+    return { id: id_, nodes: nodes_, buffers: buffers_, length: data.length().low, type: MessageHeader.DictionaryBatch };
+}
+
+function _loadMessage(bb) {
+    var messageLength: number = Int32FromByteBuffer(bb, bb.position());
+    if (messageLength == 0) {
+      return;
+    }
+    bb.setPosition(bb.position() + 4);
+    var message = Message.getRootAsMessage(bb);
+    // position the buffer at the end of the message so it's ready to read further
+    bb.setPosition(bb.position() + messageLength);
+
+    return message;
+}
+
+function _createDictionaryVectors(field, dictionaries) {
+    var encoding = field.dictionary();
+    if (encoding != null) {
+        var id = encoding.id().toFloat64().toString();
+        if (dictionaries[id] == null) {
+            // create a field for the dictionary
+            var dictionaryField = _createDictionaryField(id, field);
+            dictionaries[id] = vectorFromField(dictionaryField, null);
+        }
+    }
+
+    // recursively examine child fields
+    for (var i = 0, len = field.childrenLength(); i < len; i += 1|0) {
+        _createDictionaryVectors(field.children(i), dictionaries);
+    }
+}
+
+function _createDictionaryField(id, field) {
+    var builder = new flatbuffers.Builder();
+    var nameOffset = builder.createString("dict-" + id);
+
+    var typeType = field.typeType();
+    var typeOffset;
+    if (typeType === Type.Int) {
+        var type = field.type(new org.apache.arrow.flatbuf.Int());
+        org.apache.arrow.flatbuf.Int.startInt(builder);
+        org.apache.arrow.flatbuf.Int.addBitWidth(builder, type.bitWidth());
+        org.apache.arrow.flatbuf.Int.addIsSigned(builder, type.isSigned());
+        typeOffset = org.apache.arrow.flatbuf.Int.endInt(builder);
+    } else if (typeType === Type.FloatingPoint) {
+        var type = field.type(new org.apache.arrow.flatbuf.FloatingPoint());
+        org.apache.arrow.flatbuf.FloatingPoint.startFloatingPoint(builder);
+        org.apache.arrow.flatbuf.FloatingPoint.addPrecision(builder, type.precision());
+        typeOffset = org.apache.arrow.flatbuf.FloatingPoint.endFloatingPoint(builder);
+    } else if (typeType === Type.Utf8) {
+        org.apache.arrow.flatbuf.Utf8.startUtf8(builder);
+        typeOffset = org.apache.arrow.flatbuf.Utf8.endUtf8(builder);
+    } else if (typeType === Type.Date) {
+        var type = field.type(new org.apache.arrow.flatbuf.Date());
+        org.apache.arrow.flatbuf.Date.startDate(builder);
+        org.apache.arrow.flatbuf.Date.addUnit(builder, type.unit());
+        typeOffset = org.apache.arrow.flatbuf.Date.endDate(builder);
+    } else {
+        throw "Unimplemented dictionary type " + typeType;
+    }
+    if (field.childrenLength() > 0) {
+      throw "Dictionary encoded fields can't have children"
+    }
+    var childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, []);
+
+    var layout, layoutOffsets = [];
+    for (var i = 0, len = field.layoutLength(); i < len; i += 1|0) {
+        layout = field.layout(i);
+        org.apache.arrow.flatbuf.VectorLayout.startVectorLayout(builder);
+        org.apache.arrow.flatbuf.VectorLayout.addBitWidth(builder, layout.bitWidth());
+        org.apache.arrow.flatbuf.VectorLayout.addType(builder, layout.type());
+        layoutOffsets.push(org.apache.arrow.flatbuf.VectorLayout.endVectorLayout(builder));
+    }
+    var layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, layoutOffsets);
+
+    org.apache.arrow.flatbuf.Field.startField(builder);
+    org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
+    org.apache.arrow.flatbuf.Field.addNullable(builder, field.nullable());
+    org.apache.arrow.flatbuf.Field.addTypeType(builder, typeType);
+    org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
+    org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
+    org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
+    var offset = org.apache.arrow.flatbuf.Field.endField(builder);
+    builder.finish(offset);
+
+    return org.apache.arrow.flatbuf.Field.getRootAsField(builder.bb);
+}
+
+function Int32FromByteBuffer(bb, offset) {
     return ((bb.bytes_[offset + 3] & 255) << 24) |
            ((bb.bytes_[offset + 2] & 255) << 16) |
            ((bb.bytes_[offset + 1] & 255) << 8) |
            ((bb.bytes_[offset] & 255));
 }
 
-
 var MAGIC_STR = "ARROW1";
 var MAGIC = new Uint8Array(MAGIC_STR.length);
 for (var i = 0; i < MAGIC_STR.length; i += 1|0) {
@@ -118,27 +403,28 @@ function _checkMagic(buf, index) {
 }
 
 var TYPEMAP = {}
-TYPEMAP[arrow.flatbuf.Type.NONE]          = "NONE";
-TYPEMAP[arrow.flatbuf.Type.Null]          = "Null";
-TYPEMAP[arrow.flatbuf.Type.Int]           = "Int";
-TYPEMAP[arrow.flatbuf.Type.FloatingPoint] = "FloatingPoint";
-TYPEMAP[arrow.flatbuf.Type.Binary]        = "Binary";
-TYPEMAP[arrow.flatbuf.Type.Utf8]          = "Utf8";
-TYPEMAP[arrow.flatbuf.Type.Bool]          = "Bool";
-TYPEMAP[arrow.flatbuf.Type.Decimal]       = "Decimal";
-TYPEMAP[arrow.flatbuf.Type.Date]          = "Date";
-TYPEMAP[arrow.flatbuf.Type.Time]          = "Time";
-TYPEMAP[arrow.flatbuf.Type.Timestamp]     = "Timestamp";
-TYPEMAP[arrow.flatbuf.Type.Interval]      = "Interval";
-TYPEMAP[arrow.flatbuf.Type.List]          = "List";
-TYPEMAP[arrow.flatbuf.Type.Struct_]       = "Struct";
-TYPEMAP[arrow.flatbuf.Type.Union]         = "Union";
+TYPEMAP[Type.NONE]          = "NONE";
+TYPEMAP[Type.Null]          = "Null";
+TYPEMAP[Type.Int]           = "Int";
+TYPEMAP[Type.FloatingPoint] = "FloatingPoint";
+TYPEMAP[Type.Binary]        = "Binary";
+TYPEMAP[Type.Utf8]          = "Utf8";
+TYPEMAP[Type.Bool]          = "Bool";
+TYPEMAP[Type.Decimal]       = "Decimal";
+TYPEMAP[Type.Date]          = "Date";
+TYPEMAP[Type.Time]          = "Time";
+TYPEMAP[Type.Timestamp]     = "Timestamp";
+TYPEMAP[Type.Interval]      = "Interval";
+TYPEMAP[Type.List]          = "List";
+TYPEMAP[Type.FixedSizeList] = "FixedSizeList";
+TYPEMAP[Type.Struct_]       = "Struct";
+TYPEMAP[Type.Union]         = "Union";
 
 var VECTORTYPEMAP = {};
-VECTORTYPEMAP[arrow.flatbuf.VectorType.OFFSET]   = 'OFFSET';
-VECTORTYPEMAP[arrow.flatbuf.VectorType.DATA]     = 'DATA';
-VECTORTYPEMAP[arrow.flatbuf.VectorType.VALIDITY] = 'VALIDITY';
-VECTORTYPEMAP[arrow.flatbuf.VectorType.TYPE]     = 'TYPE';
+VECTORTYPEMAP[VectorType.OFFSET]   = 'OFFSET';
+VECTORTYPEMAP[VectorType.DATA]     = 'DATA';
+VECTORTYPEMAP[VectorType.VALIDITY] = 'VALIDITY';
+VECTORTYPEMAP[VectorType.TYPE]     = 'TYPE';
 
 function parseField(field) {
     var children = [];
@@ -149,7 +435,6 @@ function parseField(field) {
     var layouts = [];
     for (var i = 0; i < field.layoutLength(); i += 1|0) {
         layouts.push(VECTORTYPEMAP[field.layout(i).type()]);
-
     }
 
     return {
@@ -170,32 +455,39 @@ function parseSchema(schema) {
     return result;
 }
 
-function parseBuffer(buffer) {
-    return {
-        offset: buffer.offset(),
-        length: buffer.length()
-    };
+function loadVectors(bb, vectors: Vector[], recordBatch) {
+    var indices = { bufferIndex: 0, nodeIndex: 0 }, i;
+    for (i = 0; i < vectors.length; i += 1) {
+        loadVector(bb, vectors[i], recordBatch, indices);
+    }
 }
 
-function loadBuffersIntoVectors(recordBatchBlocks, bb, vectors : Vector[]) {
-    var fieldNode, recordBatchBlock, recordBatch, numBuffers, bufReader = {index: 0, node_index: 1}, field_ctr = 0;
-    var buffer = bb.bytes_.buffer;
-    var baseOffset = bb.bytes_.byteOffset;
-    for (var i = recordBatchBlocks.length - 1; i >= 0; i -= 1|0) {
-        recordBatchBlock = recordBatchBlocks[i];
-        bb.setPosition(recordBatchBlock.offset.low);
-        recordBatch = arrow.flatbuf.RecordBatch.getRootAsRecordBatch(bb);
-        bufReader.index = 0;
-        bufReader.node_index = 0;
-        numBuffers = recordBatch.buffersLength();
-
-        //console.log('num buffers: ' + recordBatch.buffersLength());
-        //console.log('num nodes: ' + recordBatch.nodesLength());
-
-        while (bufReader.index < numBuffers) {
-            //console.log('Allocating buffers starting at ' + bufReader.index + '/' + numBuffers + ' to field ' + field_ctr);
-            vectors[field_ctr].loadData(recordBatch, buffer, bufReader, baseOffset + recordBatchBlock.offset.low + recordBatchBlock.metaDataLength)
-            field_ctr += 1;
-        }
+/**
+ * Loads a vector with data from a batch
+ *   recordBatch: { nodes: org.apache.arrow.flatbuf.FieldNode[], buffers: { offset: number, length: number }[] }
+ */
+function loadVector(bb, vector: Vector, recordBatch, indices) {
+    var node = recordBatch.nodes[indices.nodeIndex], ownBuffersLength, ownBuffers = [], i;
+    indices.nodeIndex += 1;
+
+    // dictionary vectors are always ints, so will have a data vector plus optional null vector
+    if (vector.field.dictionary() == null) {
+        ownBuffersLength = vector.field.layoutLength();
+    } else if (vector.field.nullable()) {
+        ownBuffersLength = 2;
+    } else {
+        ownBuffersLength = 1;
+    }
+
+    for (i = 0; i < ownBuffersLength; i += 1) {
+        ownBuffers.push(recordBatch.buffers[indices.bufferIndex + i]);
+    }
+    indices.bufferIndex += ownBuffersLength;
+
+    vector.loadData(bb, node, ownBuffers);
+
+    var children = vector.getChildVectors();
+    for (i = 0; i < children.length; i++) {
+        loadVector(bb, children[i], recordBatch, indices);
     }
 }
diff --git a/js/lib/bitarray.ts b/js/lib/bitarray.ts
index 82fff32c194fa..fc3c0918eaf25 100644
--- a/js/lib/bitarray.ts
+++ b/js/lib/bitarray.ts
@@ -17,22 +17,9 @@
 
 export class BitArray {
     private view: Uint8Array;
-    constructor(buffer: ArrayBuffer, offset: number, length: number) {
-        //if (ArrayBuffer.isView(buffer)) {
-        //    var og_view = buffer;
-        //    buffer = buffer.buffer;
-        //    offset = og_view.offset;
-        //    length = og_view.length/og_view.BYTES_PER_ELEMENT*8;
-        //} else if (buffer instanceof ArrayBuffer) {
-        var offset = offset || 0;
-        var length = length;// || buffer.length*8;
-        //} else if (buffer instanceof Number) {
-        //    length = buffer;
-        //    buffer = new ArrayBuffer(Math.ceil(length/8));
-        //    offset = 0;
-        //}
 
-        this.view = new Uint8Array(buffer, offset, Math.ceil(length/8));
+    constructor(buffer: ArrayBuffer, offset: number, length: number) {
+        this.view = new Uint8Array(buffer, offset || 0, Math.ceil(length / 8));
     }
 
     get(i) {
diff --git a/js/lib/types.ts b/js/lib/types.ts
index bbc755810056f..d656c6aa98cf1 100644
--- a/js/lib/types.ts
+++ b/js/lib/types.ts
@@ -18,7 +18,8 @@
 import { BitArray } from './bitarray';
 import { TextDecoder } from 'text-encoding';
 import { org } from './Arrow_generated';
-var arrow = org.apache.arrow;
+
+var Type = org.apache.arrow.flatbuf.Type;
 
 interface ArrayView {
     slice(start: number, end: number) : ArrayView
@@ -26,72 +27,90 @@ interface ArrayView {
 }
 
 export abstract class Vector {
+    field: any;
     name: string;
     length: number;
     null_count: number;
-    constructor(name: string) {
-        this.name = name;
+
+    constructor(field) {
+        this.field = field;
+        this.name = field.name();
     }
+
     /* Access datum at index i */
     abstract get(i);
     /* Return array representing data in the range [start, end) */
     abstract slice(start: number, end: number);
-
-    /* Use recordBatch fieldNodes and Buffers to construct this Vector */
-    public loadData(recordBatch: any, buffer: any, bufReader: any, baseOffset: any) {
-        var fieldNode = recordBatch.nodes(bufReader.node_index);
-        this.length = fieldNode.length();
-        this.null_count = fieldNode.length();
-        bufReader.node_index += 1|0;
-
-        this.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
-    }
-
-    protected abstract loadBuffers(recordBatch: any, buffer: any, bufReader: any, baseOffset: any);
-
-    /* Helper function for loading a VALIDITY buffer (for Nullable types) */
-    static loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset) : BitArray {
-        var buf_meta = recordBatch.buffers(bufReader.index);
-        var offset = baseOffset + buf_meta.offset().low;
-        var length = buf_meta.length().low;
-        bufReader.index += 1|0;
-        return new BitArray(buffer, offset, length*8);
-    }
-
-    /* Helper function for loading an OFFSET buffer */
-    static loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset) : Int32Array {
-        var buf_meta = recordBatch.buffers(bufReader.index);
-        var offset = baseOffset + buf_meta.offset().low;
-        var length = buf_meta.length().low/Int32Array.BYTES_PER_ELEMENT;
-        bufReader.index += 1|0;
-        return new Int32Array(buffer, offset, length);
+    /* Return array of child vectors, for container types */
+    abstract getChildVectors();
+
+    /**
+     * Use recordBatch fieldNodes and Buffers to construct this Vector
+     *   bb: flatbuffers.ByteBuffer
+     *   node: org.apache.arrow.flatbuf.FieldNode
+     *   buffers: { offset: number, length: number }[]
+     */
+    public loadData(bb, node, buffers) {
+        this.length = node.length().low;
+        this.null_count = node.nullCount().low;
+        this.loadBuffers(bb, node, buffers);
+    }
+
+    protected abstract loadBuffers(bb, node, buffers);
+
+    /**
+     * Helper function for loading a VALIDITY buffer (for Nullable types)
+     *   bb: flatbuffers.ByteBuffer
+     *   buffer: org.apache.arrow.flatbuf.Buffer
+     */
+    static loadValidityBuffer(bb, buffer) : BitArray {
+        var arrayBuffer = bb.bytes_.buffer;
+        var offset = bb.bytes_.byteOffset + buffer.offset;
+        return new BitArray(arrayBuffer, offset, buffer.length * 8);
+    }
+
+    /**
+     * Helper function for loading an OFFSET buffer
+     *   buffer: org.apache.arrow.flatbuf.Buffer
+     */
+    static loadOffsetBuffer(bb, buffer) : Int32Array {
+        var arrayBuffer = bb.bytes_.buffer;
+        var offset  = bb.bytes_.byteOffset + buffer.offset;
+        var length = buffer.length / Int32Array.BYTES_PER_ELEMENT;
+        return new Int32Array(arrayBuffer, offset, length);
     }
 
 }
 
 class SimpleVector<T extends ArrayView> extends Vector {
     protected dataView: T;
-    private TypedArray: {new(buffer: any, offset: number, length: number) : T, BYTES_PER_ELEMENT: number};
+    private TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number };
 
-    constructor (TypedArray: {new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number}, name: string) {
-        super(name);
+    constructor (field, TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number }) {
+        super(field);
         this.TypedArray = TypedArray;
     }
 
+    getChildVectors() {
+        return [];
+    }
+
     get(i) {
         return this.dataView[i];
     }
 
-    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
-        this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+    loadBuffers(bb, node, buffers) {
+        this.loadDataBuffer(bb, buffers[0]);
     }
 
-    loadDataBuffer(recordBatch, buffer, bufReader, baseOffset) : T {
-        var buf_meta = recordBatch.buffers(bufReader.index);
-        var offset = baseOffset + buf_meta.offset().low;
-        var length = buf_meta.length().low/this.TypedArray.BYTES_PER_ELEMENT;
-        bufReader.index += 1|0;
-        return new this.TypedArray(buffer, offset, length);
+    /**
+      * buffer: org.apache.arrow.flatbuf.Buffer
+      */
+    protected loadDataBuffer(bb, buffer) {
+        var arrayBuffer = bb.bytes_.buffer;
+        var offset  = bb.bytes_.byteOffset + buffer.offset;
+        var length = buffer.length / this.TypedArray.BYTES_PER_ELEMENT;
+        this.dataView = new this.TypedArray(arrayBuffer, offset, length);
     }
 
     getDataView() {
@@ -108,77 +127,173 @@ class SimpleVector<T extends ArrayView> extends Vector {
 }
 
 class NullableSimpleVector<T extends ArrayView> extends SimpleVector<T> {
-    private validityView: BitArray;
+
+    protected validityView: BitArray;
+
+    get(i: number) {
+        if (this.validityView.get(i)) {
+            return this.dataView[i];
+        } else {
+          return null;
+        }
+    }
+
+    loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+        this.loadDataBuffer(bb, buffers[1]);
+    }
+
+    getValidityVector() {
+        return this.validityView;
+    }
+}
+
+class Uint8Vector   extends SimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   }; }
+class Uint16Vector  extends SimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  }; }
+class Uint32Vector  extends SimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  }; }
+class Int8Vector    extends SimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   }; }
+class Int16Vector   extends SimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  }; }
+class Int32Vector   extends SimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  }; }
+class Float32Vector extends SimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); }; }
+class Float64Vector extends SimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); }; }
+
+class NullableUint8Vector   extends NullableSimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   }; }
+class NullableUint16Vector  extends NullableSimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  }; }
+class NullableUint32Vector  extends NullableSimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  }; }
+class NullableInt8Vector    extends NullableSimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   }; }
+class NullableInt16Vector   extends NullableSimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  }; }
+class NullableInt32Vector   extends NullableSimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  }; }
+class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); }; }
+class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); }; }
+
+class Uint64Vector extends SimpleVector<Uint32Array>  {
+    constructor(field) {
+        super(field, Uint32Array);
+    }
 
     get(i: number) {
-        if (this.validityView.get(i)) return this.dataView[i];
-        else                          return null
+        return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
     }
+}
 
-    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
-        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
-        this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+class NullableUint64Vector extends NullableSimpleVector<Uint32Array>  {
+    constructor(field) {
+        super(field, Uint32Array);
     }
 
+    get(i: number) {
+        if (this.validityView.get(i)) {
+            return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
+        } else {
+          return null;
+        }
+    }
 }
 
-class Uint8Vector   extends SimpleVector<Uint8Array>   { constructor(name: string) { super(Uint8Array,  name);  }; }
-class Uint16Vector  extends SimpleVector<Uint16Array>  { constructor(name: string) { super(Uint16Array, name);  }; }
-class Uint32Vector  extends SimpleVector<Uint32Array>  { constructor(name: string) { super(Uint32Array, name);  }; }
-class Int8Vector    extends SimpleVector<Uint8Array>   { constructor(name: string) { super(Uint8Array,  name);  }; }
-class Int16Vector   extends SimpleVector<Uint16Array>  { constructor(name: string) { super(Uint16Array, name);  }; }
-class Int32Vector   extends SimpleVector<Uint32Array>  { constructor(name: string) { super(Uint32Array, name);  }; }
-class Float32Vector extends SimpleVector<Float32Array> { constructor(name: string) { super(Float32Array, name); }; }
-class Float64Vector extends SimpleVector<Float64Array> { constructor(name: string) { super(Float64Array, name); }; }
-
-class NullableUint8Vector   extends NullableSimpleVector<Uint8Array>   { constructor(name: string) { super(Uint8Array,  name);  }; }
-class NullableUint16Vector  extends NullableSimpleVector<Uint16Array>  { constructor(name: string) { super(Uint16Array, name);  }; }
-class NullableUint32Vector  extends NullableSimpleVector<Uint32Array>  { constructor(name: string) { super(Uint32Array, name);  }; }
-class NullableInt8Vector    extends NullableSimpleVector<Uint8Array>   { constructor(name: string) { super(Uint8Array,  name);  }; }
-class NullableInt16Vector   extends NullableSimpleVector<Uint16Array>  { constructor(name: string) { super(Uint16Array, name);  }; }
-class NullableInt32Vector   extends NullableSimpleVector<Uint32Array>  { constructor(name: string) { super(Uint32Array, name);  }; }
-class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(name: string) { super(Float32Array, name); }; }
-class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(name: string) { super(Float64Array, name); }; }
+class Int64Vector extends NullableSimpleVector<Uint32Array>  {
+    constructor(field) {
+        super(field, Uint32Array);
+    }
+
+    get(i: number) {
+        return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
+    }
+}
+
+class NullableInt64Vector extends NullableSimpleVector<Uint32Array>  {
+    constructor(field) {
+        super(field, Uint32Array);
+    }
+
+    get(i: number) {
+        if (this.validityView.get(i)) {
+            return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
+        } else {
+          return null;
+        }
+    }
+}
+
+class DateVector extends SimpleVector<Uint32Array> {
+    constructor(field) {
+        super(field, Uint32Array);
+    }
+
+    get (i) {
+        return new Date(super.get(2*i+1)*Math.pow(2,32) + super.get(2*i));
+    }
+}
+
+class NullableDateVector extends DateVector {
+    private validityView: BitArray;
+
+    loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+        this.loadDataBuffer(bb, buffers[1]);
+    }
+
+    get (i) {
+        if (this.validityView.get(i)) {
+            return super.get(i);
+        } else {
+            return null;
+        }
+    }
+
+    getValidityVector() {
+        return this.validityView;
+    }
+}
 
 class Utf8Vector extends SimpleVector<Uint8Array> {
     protected offsetView: Int32Array;
     static decoder: TextDecoder = new TextDecoder('utf8');
 
-    constructor(name: string) {
-        super(Uint8Array, name);
+    constructor(field) {
+        super(field, Uint8Array);
     }
 
-    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
-        this.offsetView = Vector.loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset);
-        this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+    loadBuffers(bb, node, buffers) {
+        this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]);
+        this.loadDataBuffer(bb, buffers[1]);
     }
 
     get(i) {
-        return Utf8Vector.decoder.decode
-            (this.dataView.slice(this.offsetView[i], this.offsetView[i + 1]));
+        return Utf8Vector.decoder.decode(this.dataView.slice(this.offsetView[i], this.offsetView[i + 1]));
     }
 
     slice(start: number, end: number) {
-        var rtrn: string[] = [];
+        var result: string[] = [];
         for (var i: number = start; i < end; i += 1|0) {
-            rtrn.push(this.get(i));
+            result.push(this.get(i));
         }
-        return rtrn;
+        return result;
+    }
+
+    getOffsetView() {
+        return this.offsetView;
     }
 }
 
 class NullableUtf8Vector extends Utf8Vector {
     private validityView: BitArray;
 
-    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
-        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
-        this.offsetView = Vector.loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset);
-        this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+    loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+        this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]);
+        this.loadDataBuffer(bb, buffers[2]);
     }
 
     get(i) {
-        if (!this.validityView.get(i)) return null;
-        return super.get(i);
+        if (this.validityView.get(i)) {
+            return super.get(i);
+        } else {
+            return null;
+        }
+    }
+
+    getValidityVector() {
+        return this.validityView;
     }
 }
 
@@ -186,14 +301,17 @@ class NullableUtf8Vector extends Utf8Vector {
 class ListVector extends Uint32Vector {
     private dataVector: Vector;
 
-    constructor(name, dataVector : Vector) {
-        super(name);
+    constructor(field, dataVector: Vector) {
+        super(field);
         this.dataVector = dataVector;
     }
 
-    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
-        super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
-        this.dataVector.loadData(recordBatch, buffer, bufReader, baseOffset);
+    getChildVectors() {
+        return [this.dataVector];
+    }
+
+    loadBuffers(bb, node, buffers) {
+        super.loadBuffers(bb, node, buffers);
         this.length -= 1;
     }
 
@@ -210,119 +328,262 @@ class ListVector extends Uint32Vector {
         return "length: " + (this.length);
     }
 
-    slice(start : number, end : number) { return []; };
+    slice(start: number, end: number) {
+        var result = [];
+        for (var i = start; i < end; i += 1|0) {
+            result.push(this.get(i));
+        }
+        return result;
+    }
 }
 
 class NullableListVector extends ListVector {
     private validityView: BitArray;
 
-    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
-        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
-        super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+    loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+        this.loadDataBuffer(bb, buffers[1]);
+        this.length -= 1;
     }
 
     get(i) {
-        if (!this.validityView.get(i)) return null;
-        return super.get(i);
+        if (this.validityView.get(i)) {
+            return super.get(i);
+        } else {
+            return null;
+        }
+    }
+
+    getValidityVector() {
+        return this.validityView;
+    }
+}
+
+class FixedSizeListVector extends Vector {
+    private size: number
+    private dataVector: Vector;
+
+    constructor(field, size: number, dataVector: Vector) {
+        super(field);
+        this.size = size;
+        this.dataVector = dataVector;
+    }
+
+    getChildVectors() {
+        return [this.dataVector];
+    }
+
+    loadBuffers(bb, node, buffers) {
+        // no buffers to load
+    }
+
+    get(i: number) {
+        return this.dataVector.slice(i * this.size, (i + 1) * this.size);
+    }
+
+    slice(start : number, end : number) {
+        var result = [];
+        for (var i = start; i < end; i += 1|0) {
+            result.push(this.get(i));
+        }
+        return result;
+    }
+
+    getListSize() {
+        return this.size;
+    }
+}
+
+class NullableFixedSizeListVector extends FixedSizeListVector {
+    private validityView: BitArray;
+
+    loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+    }
+
+    get(i: number) {
+        if (this.validityView.get(i)) {
+            return super.get(i);
+        } else {
+            return null;
+        }
+    }
+
+    getValidityVector() {
+        return this.validityView;
     }
 }
 
 class StructVector extends Vector {
     private validityView: BitArray;
-    private vectors : Vector[];
-    constructor(name: string, vectors: Vector[]) {
-        super(name);
+    private vectors: Vector[];
+
+    constructor(field, vectors: Vector[]) {
+        super(field);
         this.vectors = vectors;
     }
 
-    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
-        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
-        this.vectors.forEach((v: Vector) => v.loadData(recordBatch, buffer, bufReader, baseOffset));
+    getChildVectors() {
+        return this.vectors;
+    }
+
+    loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
     }
 
     get(i : number) {
-        if (!this.validityView.get(i)) return null;
-        return this.vectors.map((v: Vector) => v.get(i));
+        if (this.validityView.get(i)) {
+          return this.vectors.map((v: Vector) => v.get(i));
+        } else {
+            return null;
+        }
     }
 
     slice(start : number, end : number) {
-        var rtrn = [];
-        for (var i: number = start; i < end; i += 1|0) {
-            rtrn.push(this.get(i));
+        var result = [];
+        for (var i = start; i < end; i += 1|0) {
+            result.push(this.get(i));
         }
-        return rtrn;
+        return result;
+    }
+
+    getValidityVector() {
+        return this.validityView;
     }
 }
 
-class DateVector extends SimpleVector<Uint32Array> {
-    constructor (name: string) {
-        super(Uint32Array, name);
+class DictionaryVector extends Vector {
+
+    private indices: Vector;
+    private dictionary: Vector;
+
+    constructor (field, indices: Vector, dictionary: Vector) {
+        super(field);
+        this.indices = indices;
+        this.dictionary = dictionary;
     }
 
-    get (i) {
-        return new Date(super.get(2*i+1)*Math.pow(2,32) + super.get(2*i));
+    get(i) {
+        var encoded = this.indices.get(i);
+        if (encoded == null) {
+            return null;
+        } else {
+            return this.dictionary.get(encoded);
+        }
     }
-}
 
-class NullableDateVector extends DateVector {
-    private validityView: BitArray;
+    /** Get the dictionary encoded value */
+    public getEncoded(i) {
+        return this.indices.get(i);
+    }
 
-    loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
-        this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
-        super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+    slice(start, end) {
+        return this.indices.slice(start, end); // TODO decode
     }
 
-    get (i) {
-        if (!this.validityView.get(i)) return null;
-        return super.get(i);
+    getChildVectors() {
+        return this.indices.getChildVectors();
+    }
+
+    loadBuffers(bb, node, buffers) {
+        this.indices.loadData(bb, node, buffers);
+    }
+
+    /** Get the index (encoded) vector */
+    public getIndexVector() {
+        return this.indices;
+    }
+
+    /** Get the dictionary vector */
+    public getDictionaryVector() {
+        return this.dictionary;
+    }
+
+    toString() {
+        return this.indices.toString();
     }
 }
 
-var BASIC_TYPES = [arrow.flatbuf.Type.Int, arrow.flatbuf.Type.FloatingPoint, arrow.flatbuf.Type.Utf8, arrow.flatbuf.Type.Date];
-
-export function vectorFromField(field) : Vector {
-    var typeType = field.typeType();
-    if (BASIC_TYPES.indexOf(typeType) >= 0) {
-        var type = field.typeType();
-        if (type === arrow.flatbuf.Type.Int) {
-            type = field.type(new arrow.flatbuf.Int());
-            var VectorConstructor : {new(string) : Vector};
-            if (type.isSigned()) {
-                if (type.bitWidth() == 32)
-                    VectorConstructor = field.nullable() ? NullableInt32Vector : Int32Vector;
-                else if (type.bitWidth() == 16)
-                    VectorConstructor = field.nullable() ? NullableInt16Vector : Int16Vector;
-                else if (type.bitWidth() == 8)
-                    VectorConstructor = field.nullable() ? NullableInt8Vector : Int8Vector;
+export function vectorFromField(field, dictionaries) : Vector {
+    var dictionary = field.dictionary(), nullable = field.nullable();
+    if (dictionary == null) {
+        var typeType = field.typeType();
+        if (typeType === Type.List) {
+            var dataVector = vectorFromField(field.children(0), dictionaries);
+            return nullable ? new NullableListVector(field, dataVector) : new ListVector(field, dataVector);
+        } else if (typeType === Type.FixedSizeList) {
+            var dataVector = vectorFromField(field.children(0), dictionaries);
+            var size = field.type(new org.apache.arrow.flatbuf.FixedSizeList()).listSize();
+            if (nullable) {
+              return new NullableFixedSizeListVector(field, size, dataVector);
             } else {
-                if (type.bitWidth() == 32)
-                    VectorConstructor = field.nullable() ? NullableUint32Vector : Uint32Vector;
-                else if (type.bitWidth() == 16)
-                    VectorConstructor = field.nullable() ? NullableUint16Vector : Uint16Vector;
-                else if (type.bitWidth() == 8)
-                    VectorConstructor = field.nullable() ? NullableUint8Vector : Uint8Vector;
+              return new FixedSizeListVector(field, size, dataVector);
+            }
+         } else if (typeType === Type.Struct_) {
+            var vectors : Vector[] = [];
+            for (var i : number = 0; i < field.childrenLength(); i += 1|0) {
+                vectors.push(vectorFromField(field.children(i), dictionaries));
+            }
+            return new StructVector(field, vectors);
+        } else {
+            if (typeType === Type.Int) {
+                var type = field.type(new org.apache.arrow.flatbuf.Int());
+                return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable)
+            } else if (typeType === Type.FloatingPoint) {
+                var precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision();
+                if (precision == org.apache.arrow.flatbuf.Precision.SINGLE) {
+                    return nullable ? new NullableFloat32Vector(field) : new Float32Vector(field);
+                } else if (precision == org.apache.arrow.flatbuf.Precision.DOUBLE) {
+                    return nullable ? new NullableFloat64Vector(field) : new Float64Vector(field);
+                } else {
+                    throw "Unimplemented FloatingPoint precision " + precision;
+                }
+            } else if (typeType === Type.Utf8) {
+                return nullable ? new NullableUtf8Vector(field) : new Utf8Vector(field);
+            } else if (typeType === Type.Date) {
+                return nullable ? new NullableDateVector(field) : new DateVector(field);
+            } else {
+                throw "Unimplemented type " + typeType;
             }
-        } else if (type === arrow.flatbuf.Type.FloatingPoint) {
-            type = field.type(new arrow.flatbuf.FloatingPoint());
-            if (type.precision() == arrow.flatbuf.Precision.SINGLE)
-                VectorConstructor = field.nullable() ? NullableFloat32Vector : Float32Vector;
-            else if (type.precision() == arrow.flatbuf.Precision.DOUBLE)
-                VectorConstructor = field.nullable() ? NullableFloat64Vector : Float64Vector;
-        } else if (type === arrow.flatbuf.Type.Utf8) {
-            VectorConstructor = field.nullable() ? NullableUtf8Vector : Utf8Vector;
-        } else if (type === arrow.flatbuf.Type.Date) {
-            VectorConstructor = field.nullable() ? NullableDateVector : DateVector;
         }
+    } else {
+        // determine arrow type - default is signed 32 bit int
+        var type = dictionary.indexType(), bitWidth = 32, signed = true;
+        if (type != null) {
+            bitWidth = type.bitWidth();
+            signed = type.isSigned();
+        }
+        var indices = _createIntVector(field, bitWidth, signed, nullable);
+        return new DictionaryVector(field, indices, dictionaries[dictionary.id().toFloat64().toString()]);
+    }
+}
 
-        return new VectorConstructor(field.name());
-    } else if (typeType === arrow.flatbuf.Type.List) {
-        var dataVector = vectorFromField(field.children(0));
-        return field.nullable() ? new NullableListVector(field.name(), dataVector) : new ListVector(field.name(), dataVector);
-    } else if (typeType === arrow.flatbuf.Type.Struct_) {
-        var vectors : Vector[] = [];
-        for (var i : number = 0; i < field.childrenLength(); i += 1|0) {
-            vectors.push(vectorFromField(field.children(i)));
+function _createIntVector(field, bitWidth, signed, nullable) {
+    if (bitWidth == 64) {
+        if (signed) {
+            return nullable ? new NullableInt64Vector(field) : new Int64Vector(field);
+        } else {
+            return nullable ? new NullableUint64Vector(field) : new Uint64Vector(field);
+        }
+    } else if (bitWidth == 32) {
+        if (signed) {
+            return nullable ? new NullableInt32Vector(field) : new Int32Vector(field);
+        } else {
+            return nullable ? new NullableUint32Vector(field) : new Uint32Vector(field);
+        }
+    } else if (bitWidth == 16) {
+        if (signed) {
+            return nullable ? new NullableInt16Vector(field) : new Int16Vector(field);
+        } else {
+            return nullable ? new NullableUint16Vector(field) : new Uint16Vector(field);
+        }
+    } else if (bitWidth == 8) {
+        if (signed) {
+            return nullable ? new NullableInt8Vector(field) : new Int8Vector(field);
+        } else {
+            return nullable ? new NullableUint8Vector(field) : new Uint8Vector(field);
         }
-        return new StructVector(field.name(), vectors);
+    } else {
+         throw "Unimplemented Int bit width " + bitWidth;
     }
 }
diff --git a/js/package.json b/js/package.json
index b1e583b7d9da6..8687f50a83c83 100644
--- a/js/package.json
+++ b/js/package.json
@@ -4,16 +4,20 @@
   "description": "",
   "main": "dist/arrow.js",
   "scripts": {
-    "postinstall": "./postinstall.sh",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "postinstall": "./flatbuffers.sh",
+    "build": "./flatbuffers.sh && tsc && webpack",
+    "test": "./node_modules/mocha/bin/mocha ./spec/arrow.js"
   },
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
-    "flatbuffers": "^1.5.0",
-    "text-encoding": "^0.6.4"
+    "chai": "^3.5.0",
+    "mocha": "^3.3.0",
+    "webpack": "^2.3.3"
   },
   "dependencies": {
+    "flatbuffers": "^1.5.0",
+    "text-encoding": "^0.6.4",
     "commander": "^2.9.0"
   }
 }
diff --git a/js/spec/arrow.js b/js/spec/arrow.js
new file mode 100644
index 0000000000000..61a6f818e265a
--- /dev/null
+++ b/js/spec/arrow.js
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+var fs = require('fs');
+var chai = require('chai');
+var assert = chai.assert;
+var path= require('path');
+var arrow = require('../dist/arrow.js');
+
+test_files = [
+  {
+    name: 'simple',
+    batches: 1,
+    fields: [
+      {
+        "name": "foo",
+        "type": "Int",
+        "data": [[1, null, 3, 4, 5]]
+      },
+      {
+        "name": "bar",
+        "type": "FloatingPoint",
+        "data": [[1.0, null, null, 4.0, 5.0]]
+      },
+      {
+        "name": "baz",
+        "type": "Utf8",
+        "data": [["aa", null, null, "bbb", "cccc"]]
+      }
+    ]
+  },
+  {
+    name: 'struct_example',
+    batches: 2,
+    fields: [
+      {
+        "name": "struct_nullable",
+        "type": "Struct",
+        "data": [
+          [
+            null,
+            [null, 'MhRNxD4'],
+            [137773603, '3F9HBxK'],
+            [410361374, 'aVd88fp'],
+            null,
+            [null, '3loZrRf'],
+            null
+          ], [
+            null,
+            [null,null],
+            [null,null],
+            null,
+            [null, '78SLiRw'],
+            null,
+            null,
+            [null, '0ilsf82'],
+            [null, 'LjS9MbU'],
+            [null, null],
+          ]
+        ]
+      }
+    ]
+  },
+  {
+    name: 'dictionary',
+    batches: 2,
+    fields: [
+      {
+        "name": "example-csv",
+        "type": "Struct",
+        "data": [
+          [
+            ["Hermione", 25, new Float32Array([-53.235599517822266, 40.231998443603516])],
+            ["Severus", 30, new Float32Array([-62.22999954223633, 3])],
+          ], [
+            ["Harry", 20, new Float32Array([23, -100.23652648925781])]
+          ]
+        ]
+      }
+    ]
+  },
+];
+
+var buf;
+
+function makeSchemaChecks(fields) {
+  describe('schema', function () {
+    var schema;
+    beforeEach(function () {
+      schema = arrow.getSchema(buf);
+    });
+
+    it('should read the number of fields', function () {
+        assert.lengthOf(schema, fields.length);
+    });
+
+    it("should understand fields", function () {
+      for (i = 0; i < fields.length; i += 1|0) {
+          assert.equal(schema[i].name, fields[i].name);
+          assert.equal(schema[i].type, fields[i].type,
+                       'bad type for field ' + schema[i].name);
+      }
+    });
+  });
+}
+
+function makeDataChecks (batches, fields) {
+  describe('data', function() {
+    var reader;
+    beforeEach(function () {
+        reader = arrow.getReader(buf)
+    });
+    it('should read the correct number of record batches', function () {
+        assert.equal(reader.getBatchCount(), batches);
+    });
+    fields.forEach(function (field, i) {
+      it('should read ' + field.type + ' vector ' + field.name, function () {
+        for (var batch_idx = 0; batch_idx < batches; batch_idx += 1|0) {
+          reader.loadNextBatch();
+          var batch = field.data[batch_idx];
+          var vector = reader.getVector(field.name)
+          assert.isDefined(vector, "vector " + field.name);
+          assert.lengthOf(vector, batch.length, "vector " + field.name)
+          for (i = 0; i < vector.length; i += 1|0) {
+            if (field.type == "Date") {
+              assert.equal(vector.get(i).getTime(), batch[i].getTime(),
+                           "vector " + field.name + " index " + i);
+            } else {
+              assert.deepEqual(vector.get(i), batch[i],
+                               "vector " + field.name + " index " + i);
+            }
+          }
+        }
+      });
+    });
+  });
+}
+
+describe('arrow random-access file', function () {
+  test_files.forEach(function (test_file) {
+    describe(test_file.name, function () {
+      var fields = test_file.fields
+      beforeEach(function () {
+        buf = fs.readFileSync(path.resolve(__dirname, test_file.name + '.arrow'));
+      });
+
+      makeSchemaChecks(fields);
+      makeDataChecks(test_file.batches, fields);
+    })
+  });
+});
+
+describe('arrow streaming file format', function () {
+  test_files.forEach(function (test_file) {
+    describe(test_file.name, function () {
+      var fields = test_file.fields
+      beforeEach(function () {
+        buf = fs.readFileSync(path.resolve(__dirname, test_file.name + '-stream.arrow'));
+      });
+
+      makeSchemaChecks(fields);
+      makeDataChecks(test_file.batches, fields);
+    })
+  });
+});
diff --git a/js/spec/dictionary-stream.arrow b/js/spec/dictionary-stream.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..17ca48b3a97f51ad016faa004cc53a2a5e809eec
GIT binary patch
literal 1776
zcmbVLL2DCH5T0$)G{mHqHnv(t_Rxcff@l#56jJdJ!9$>4yt$ZX2qYWXY)dL0a`6ZF
zAN&!Xds6V=u_x(K5D$W9*YA7#rmU}Osk6+RH}l@id^7WQHYc(qB6V4miZrBZ_gG}k
z+9jx2TN#!bm@)ug1UIoZAy?onxWzN-<d71vL5(Fy`tW@CGs^<RE@A%w#*fb4&9Y4W
zX0QQOK&gIjouBG6cmD?NY=T8f0VPwi9_BiPPlyOd&KD+71xPFSNP(3by(o<~sPn9B
zZH#jPe7$eRqgHo3h||_28T7U`@7%r%dtUpBEdQZ)Vj=@gaEmKgF%5y6m{Lggf%FJ_
zh#<tW29Els^)OCN-z2R;+%way=h1Ltkn;*Uz9y@PRz_aViamSkWC!vUSY$q!Vr7qs
zgR^2kz93H(xvRDVoERyzI;*Cxm^kewEcD~t8RN(252m~R@(H#$Vg3c{12v%U5#p~x
z#?i>McG6@J_Z|K;*)_z#PvjifOrz0@p}F~NvL~OQvkp=nH7?6#=&VrtXpgYCH;GLZ
zw}_5;f5%pXPHbh1Tl>RaC>8f=<GU(S-kW@GMLTnm6fgJm^<ysZz|lNp15BSzlqCBd
zlZ*!A*zA}+lk8502%z!mnC>E8UOr*mcQKA@FP*IK5O?yIlwVft$FA7?tv=t*cS<V8
zqpo%P_d_0n{G8Wv^NGCtA4q<jd%4;ECvGe3n~&qHI^Ua5IxCfd3#$4hI9vs*e%=Mx
z{{Hyj&A0IW>#x%Tk#_rEb89=F)8<y}gWp4jD4tW-JHB1#=G#x3o8O=Rotxq!M$XO0
faaP@*GvjIum9&vm*MCTLouA0N?e+cln3>2g$Le<8

literal 0
HcmV?d00001

diff --git a/js/spec/dictionary.arrow b/js/spec/dictionary.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..34d41db1f200159406c459b8c076d5ce0c3d30ec
GIT binary patch
literal 2522
zcmeHJ&1(};5T9+*G{mHqHnv(ta_B)s!H*ykC?w(`f?#L`@#bQlX&@hEla^FG<l;rV
z_&@kZc+N?|lLzTZaumda;Mw^5y?s;GjipqtoiK0S%)I%2yPL|!#)C)KhchAzB9fJP
zNl8x9c8^77tX#Z|l_ep`07*UISzsP(9&`!Tf>YFyCl*NI8<1N7r49A1Uz3Rk-wEvR
zLio|iYm>=D?S_ycE00v|-a0$whwl6poGAiD5*{f-vL43ThfM(<#?BTZKnj_r@FRGZ
zuGRg%zeb+Aq?OUn8Q|rH>9$L?ZnM)b^@3)7y?FD+Ey%;#mt^rTwKFFovhl8Q1!_($
z<jtJ=pzH(TKK3A90m}-|Z<x}(PTw?4P-=GSX0ZOuZ}kjfUO>lJWEtL)h|5{AXHOih
zg1!KX%z6W?>@oA;tk{pQh?7R_wCw;VMkrXGRg>48IPE+nwBy|A<L8$TrndV0F}Bb!
z{+zV|>(O=$|I45qzimpJ{h--txcvLUj$sb`iJStOf#2S4nX6BGyYdM<>mX!N<D#4g
z&kD7VR)xjgNo>+M#q1dO&)hQLnOoB4t^Hvy6ry{T@m@tC>`gegsGP9~nlJaXo%T6L
zm4jK(9FSI3KM3}!CTKUi9kXe6O|aA3f(!MS#eC;+_re+Dyz_BTePX)8eVhsJQg~Ri
zKQP7SunOaH-c&-=A9<}je!f5@ARhBlD4vSTCxQ_7b2}8<C&g`reT(ODT%I>B3OXsJ
zo^z=DIUK$SRQ|LxuKaj7dh@+-=jFG-NTghDp@73?R(3W=%`Do3c>m}kswb~!;&Po?
zTz=Hd;=TFTnQ30kku!_uaa=w=WBOGeN+}~Kud`5I=O^;6vby(vr24GAVc{EsZx~`3
zxCLa~*UpghB~(KqWflh5cJi|2!iAz&F6qxsUX1AZ4rPz@ZAwZT&gbMSdh!)@&(D8-
MMaQ^*75ks=H#cC^v;Y7A

literal 0
HcmV?d00001

diff --git a/js/spec/simple-stream.arrow b/js/spec/simple-stream.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..2c68c0e44b0af2ed78a47edbd038ae8fa8ab574b
GIT binary patch
literal 1188
zcmcIiJ5Iw;5M2La6IsYbgd7PZ7aV|so>x*(asiu&HWDROj*yaLaD)_;9EOt4%&d3$
zIDjGrHd;TwnSC>FH(o>{m57vbC=<z~(04qdSaFdlIFS_0)VLHYBSAhR7f52)ec$6?
zc7^v-WNkR>&J3d$Bh?>poGB!NdI5h?f00HmfkzUnN7vrV<Di`c)KGhdq-Sh<dJQpA
zC2Emf<6MCL!xz0foUgSm*X}CSqb3{o`+xS}{g0LB9ZO`6)QB1^uh4I-j})Dd^ZVZ}
zmvW9d5Bi+-k<GePcS&NMl{SNOoOun#XZ{3)bIV9s<XB;ed4EK(SI2$ttK%!H8{?ZF
za5SP3jCY-2_-cf2M)jom4fy?ye&6#yWH|48Vkdm;5$;8QelPB^tK@vw!e8rNF21L$
fFyJ#=x2yibpY!x&{|DPP|Aw?}*L63DJsjQuh?Gj^

literal 0
HcmV?d00001

diff --git a/js/spec/simple.arrow b/js/spec/simple.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..838db6dc8eda5376aaaf977c5f96c50397582a5f
GIT binary patch
literal 1642
zcmeHIJxc>Y5S{x-E`~!=1UW$=g@vVpg@xdyvk-(J*x1}DwgF44l$Ms3{sezQ3QK>C
zrQP}7+>FLVB_!H8VfN$A?7W@X-AjKkI62#uh2=aD>0os#tJ0K)v~-Ln2o)D;D;5K4
zL)k~rXXvHPPphiJg2plCThPpK`t2A-4oa*);8@d=0I^%(73C1=$r12CLggrn8+oi_
zM>%{Ly9?zWLYtGTmZnBxEL!idE=T<PPw}p?K2yI088dUnqcUON|0D<ZzoB;SF@+{j
zm7s_274dtPM~qBp^Zj29hq8zI*6~^6HLG!|>=K1qD@_LLSm6wo7*Sa3Cn)RIfVw0Y
zfD-loklcoj``mlSN9H$;kH5i@h)QmDkCPidn!?9Z{xtIr`2NlOKIa)y-0pK?n{1X4
z=ORDvi*w8ya(!OMAN^hqzczRB8t+;^>-8J_tfwpUUzw%(Go&cWvb=yS$iRD_;^pu~
prFaE=t<2^C?KYHW$}@hjgWmT8pS(n{|9`)h`+nV`u6_NFc>}#ma&iCw

literal 0
HcmV?d00001

diff --git a/js/spec/struct_example-stream.arrow b/js/spec/struct_example-stream.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..4e97b7084f6b7c62df23acfe9e200661946314e3
GIT binary patch
literal 1884
zcmd5+O;1`u6us|>_(~{cQ4I?<B)Di?G*n8(9U7~~qNY$AbYnn$)#jys0P4b|F1zif
zOIPmnAGqNs@DsZ1vfCz2@SJ%w)HE8vrqN5z%-l2gW9D+-6fKdch=e5~LFpAotua`(
z;zdH(6A54^r1t`tDR2cZQf)Sy&>sKBIu0-%dG3D)Y~UDR>;`PleEZ%U^bsHi(7p#q
z0^}Hj5>P*pUHCYN=-?Lw&sfa%1-yj}%;AoSJVi8G^r=%vp_8Ln_B2Murp|oyc?Jvu
z6bC*IJpI^W#lihR{5bOdyq@O;Z5_l2DqH4%0^3S;zqVdo`&9G1QrX*-7z*I$W_<^A
zeZ%Tx(qpT8*I6VbA?RVm^y_>9jy34>H`T~{;~1#*AIa*a;f-!)!=GrozxnztJg%(d
zMDMraXg;fXPW<1@Ik~2^)1UihcOHlC?CCpxe*3k+RbuLuGkjW$*_VxKHzEJ^{3aLc
z6ao4**o&qgvHWkcu$5mtntqv@nV6e8dQ*D8k;%BbDQ{<WKks&ubg+ZoRv(xOc5`p@
zJuz}0y<OMIO>diTF>4Nmnf+|e$v1;y^hdth(oOG^{&ro_cHK<bZj8IIUD-S5bK6#a
zwce$+rneffw+&2>v$qq#D1c{F0n>w%$wGQ(GXHvhcBAxRy*QpJ<hJvN2jy9Jd}$?J
Z+e*9dD#>lH;%3Hj9}5!;<>ec0a0N6*h?f8W

literal 0
HcmV?d00001

diff --git a/js/spec/struct_example.arrow b/js/spec/struct_example.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..3d2c018e6c27c95057f2a18395a9dc6031e6b201
GIT binary patch
literal 2354
zcmeHJJ5L)y5MCPt7wZBk2y#J?B~nBw5RVv4cSZyvVqWKjR}<Ng3!qal*iI-Sg_1T>
z6s4k~Oa4I`{{(&lB_(a7NY3}|?Kp^s{6L5%V|}|jbMx4po7-{9b?24_Rns&dQtAP$
zZPlULRI9S|8bc_o<w`}hECNv!;0zNz7b}$txch&=#{v40=O<W)flmN^9z*vi&~HtH
ze+<L`>bC#`0A=(+is+wGdDvKpXyMleIejtO0puY>U<_|asV;b<MmtVIFI^Z39%KR|
zeUoQA+B^Yz0Fnh83zmUz6^cW43jcA$Jv$!<dFDPMh(le*@5Edv?v~byZ{C-DKb!O4
zsu&U!#AaT*O<rxiGu0B-=Pomenxf!s__TC>e?rC_v<DCM$a7;G$n`&R%u?jS%PjJJ
zqjynWzDABiY1%i>tA32!F>+3PZ?aFWNwxU?K&|QVQPY-z;pc@f8Ls-qN37n%QY`%I
zxY&rxo9<s?F;5-9Lj%^L)FYO+Ml&1k?B2w);mNTV<9jc&i|fgxmml`GS9e{no}fK-
z)VBV>w6m*goA-#&UDS5j1ee;DZ_$xWVP-wcK6z)5gufTCRbFbJ;&0~}qu0pfG@{&q
z?#j*&Ubi*rclBPXsd}vfYg-^S&f4w+o&s!FH)vU~zCB8165A>F`SjF!_TAd@Xfl)D
zbU%K`O?jhpD~ZxZ!h2O1*z^ltawxr(8Jo$yzOWVE-8OE2b_g3cIlB=%&N4;^kpA=4
j<eA?SNop`}iY+ehfz9d?{qM=u-d|@={*HX6^nazl^uV^z

literal 0
HcmV?d00001


From 2d6453b25318b81af967f0cfdddacf183a60098c Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Tue, 9 May 2017 18:31:28 -0400
Subject: [PATCH 0627/1644] ARROW-940: [JS] Generate multiple artifacts

Running `npm run build` now produces three sets of artifacts:
* `lib/`: CommonJS modules with typescript declarations
* `lib-esm/`: ES6 modules with typescript declarations
* `_bundles/`: minified and un-minified bundles with source maps for use in the browser

This PR also adds `.npmigore` and `bower.json` to get ready for packaging releases for both npm and bower

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #663 from TheNeuralBit/multiple-artifacts and squashes the following commits:

a056cd9 [Brian Hulette] update README
7779797 [Brian Hulette] add typescript dev dependency
895c95b [Brian Hulette] update npm main file
71aefb9 [Brian Hulette] Add bower config, add repo to npm config
0d47146 [Brian Hulette] updated read_file example
b01bd75 [Brian Hulette] JS lib now creates multiple artifacts: ES5/6 with .d.ts files, and bundles
---
 js/.gitignore                        |  7 ++---
 js/.npmignore                        |  7 +++++
 js/README.md                         | 13 +++++-----
 js/bin/arrow2csv.js                  | 13 +++++++---
 js/bin/arrow_schema.js               |  2 +-
 js/bower.json                        | 17 +++++++++++++
 js/examples/read_file.html           | 12 ++++-----
 js/flatbuffers.sh                    | 14 +++++++---
 js/package.json                      | 10 +++++---
 js/spec/arrow.js                     |  2 +-
 js/{lib => src}/Arrow_generated.d.ts |  0
 js/{lib => src}/arrow.ts             |  0
 js/{lib => src}/bitarray.ts          |  0
 js/{lib => src}/types.ts             |  0
 js/tsconfig.json                     | 14 +++++-----
 js/webpack.config.js                 | 38 ++++++++++++++++++++++++----
 16 files changed, 107 insertions(+), 42 deletions(-)
 create mode 100644 js/.npmignore
 create mode 100644 js/bower.json
 rename js/{lib => src}/Arrow_generated.d.ts (100%)
 rename js/{lib => src}/arrow.ts (100%)
 rename js/{lib => src}/bitarray.ts (100%)
 rename js/{lib => src}/types.ts (100%)

diff --git a/js/.gitignore b/js/.gitignore
index f67c1cc9fc764..ea5514f634d37 100644
--- a/js/.gitignore
+++ b/js/.gitignore
@@ -1,6 +1,7 @@
-lib/*_generated.js
-dist
+src/Arrow_generated.js
+lib
+lib-esm
+_bundles
 node_modules
-typings
 .idea
 *.iml
diff --git a/js/.npmignore b/js/.npmignore
new file mode 100644
index 0000000000000..333aeec571a21
--- /dev/null
+++ b/js/.npmignore
@@ -0,0 +1,7 @@
+.gitignore
+.npmignore
+src/
+spec/
+tsconfig.json
+webpack.config.js
+flatbuffers.sh
diff --git a/js/README.md b/js/README.md
index cdabf5479ae11..167bafcd724de 100644
--- a/js/README.md
+++ b/js/README.md
@@ -18,9 +18,8 @@ From this directory, run:
 
 ``` bash
 $ npm install   # pull dependencies
-$ tsc           # build typescript
-$ webpack       # bundle for the browser
-$ npm test      # run unit tests
+$ npm run build # build typescript (run tsc and webpack)
+$ npm run test  # run the unit tests (node.js only)
 ```
 
 ### Usage
@@ -30,17 +29,17 @@ The library is designed to be used with node.js or in the browser, this reposito
 Import the arrow module:
 
 ``` js
-var arrow = require("arrow.js");
+var arrow = require("arrow");
 ```
 
 See [bin/arrow_schema.js](bin/arrow_schema.js) and [bin/arrow2csv.js](bin/arrow2csv.js) for usage examples.
 
 #### Browser
-Include `dist/arrow-bundle.js` in a `<script />` tag:
+Include `_bundles/arrow.js` in a `<script />` tag:
 ``` html
-<script src="arrow-bundle.js"/>
+<script src="_bundles/arrow.js"/>
 ```
-See [examples/read_file.html](examples/read_file.html) for a usage example - or try it out now at [theneuralbit.github.io/arrow](http://theneuralbit.github.io/arrow)
+See [examples/read_file.html](examples/read_file.html) for a usage example.
 
 ### API
 ##### `arrow.getReader(buffer)`
diff --git a/js/bin/arrow2csv.js b/js/bin/arrow2csv.js
index 8122e95a883fe..c1801f7127bc0 100755
--- a/js/bin/arrow2csv.js
+++ b/js/bin/arrow2csv.js
@@ -19,7 +19,7 @@
 
 var fs = require('fs')
 var process = require('process');
-var arrow = require('../dist/arrow.js');
+var arrow = require('../lib/arrow.js');
 var program = require('commander');
 
 function list (val) {
@@ -39,10 +39,15 @@ if (!program.schema) {
 
 var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
 var reader = arrow.getReader(buf);
-reader.loadNextBatch();
+var nrecords
 
-for (var i = 0; i < reader.getVector(program.schema[0]).length; i += 1|0) {
+nrecords = reader.loadNextBatch();
+while (nrecords > 0) {
+  for (var i = 0; i < nrecords; i += 1|0) {
     console.log(program.schema.map(function (field) {
-        return '' + reader.getVector(field).get(i);
+      return '' + reader.getVector(field).get(i);
     }).join(','));
+  }
+  nrecords = reader.loadNextBatch();
+  if (nrecords > 0) console.log('---');
 }
diff --git a/js/bin/arrow_schema.js b/js/bin/arrow_schema.js
index 44dabb413dbe8..4917628641907 100755
--- a/js/bin/arrow_schema.js
+++ b/js/bin/arrow_schema.js
@@ -19,7 +19,7 @@
 
 var fs = require('fs');
 var process = require('process');
-var arrow = require('../dist/arrow.js');
+var arrow = require('../lib/arrow.js');
 
 var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
 var reader = arrow.getReader(buf);
diff --git a/js/bower.json b/js/bower.json
new file mode 100644
index 0000000000000..c2099f859c364
--- /dev/null
+++ b/js/bower.json
@@ -0,0 +1,17 @@
+{
+  "name": "arrow",
+  "description": "",
+  "main": "_bundles/arrow.js",
+  "authors": [],
+  "license": "Apache-2.0",
+  "homepage": "http://arrow.apache.org",
+  "ignore": [
+    ".gitignore",
+    ".npmignore",
+    "src/",
+    "spec/",
+    "tsconfig.json",
+    "webpack.config.js",
+    "flatbuffers.sh"
+  ]
+}
diff --git a/js/examples/read_file.html b/js/examples/read_file.html
index 933b142b527ce..5a650a09c8010 100644
--- a/js/examples/read_file.html
+++ b/js/examples/read_file.html
@@ -39,12 +39,10 @@
   tr.appendChild(td);
 }
 reader.onload = function (evt) {
-  var buf = new Uint8Array(evt.target.result);
-  var schema = arrow.loadSchemaFromStream(buf);
-  var vectors = arrow.loadVectorsFromStream(buf);
-  var length = vectors[schema[0].name].length;
+  var reader = new arrow.getReader(new Uint8Array(evt.target.result));
+  var schema = reader.getSchema();
+  var length = reader.loadNextBatch();
 console.log(JSON.stringify(schema, null, '\t'));
-console.log(JSON.stringify(vectors, null, '\t'));
 
   var thead = document.getElementById("thead");
   var tbody = document.getElementById("tbody");
@@ -58,7 +56,7 @@
 
   for (var i = 0; i < length; i += 1|0) {
     var tr = document.createElement("tr");
-    schema.forEach(function (d) { addCell(tr, "td", vectors[d.name].get(i)); });
+    schema.forEach(function (d) { addCell(tr, "td", reader.getVector(d.name).get(i)); });
     tbody.appendChild(tr);
   }
 }
@@ -76,6 +74,6 @@
       <tbody id="tbody">
       </tbody>
     </table>
-    <script type="text/javascript" src="../dist/arrow-bundle.js"></script>
+    <script type="text/javascript" src="../_bundles/arrow.js"></script>
   </body>
 </html>
diff --git a/js/flatbuffers.sh b/js/flatbuffers.sh
index 99d2815fd2071..55967f84a9fa9 100755
--- a/js/flatbuffers.sh
+++ b/js/flatbuffers.sh
@@ -13,7 +13,13 @@
 # limitations under the License. See accompanying LICENSE file.
 
 echo "Compiling flatbuffer schemas..."
-#flatc -o lib --js ../format/Message.fbs ../format/File.fbs
-flatc -o lib --js ../format/*.fbs
-rm -f lib/Arrow_generated.js
-cat lib/*_generated.js > lib/Arrow_generated.js
+mkdir -p lib lib-esm
+DIR=`mktemp -d`
+flatc -o $DIR --js ../format/*.fbs
+cat $DIR/*_generated.js > src/Arrow_generated.js
+
+# Duplicate in the tsc-generated outputs - we can't make tsc pull in .js files
+# and still prooduce declaration files
+cat $DIR/*_generated.js > lib/Arrow_generated.js
+cat $DIR/*_generated.js > lib-esm/Arrow_generated.js
+rm -rf $DIR
diff --git a/js/package.json b/js/package.json
index 8687f50a83c83..1739e38926545 100644
--- a/js/package.json
+++ b/js/package.json
@@ -2,17 +2,21 @@
   "name": "arrow",
   "version": "0.0.0",
   "description": "",
-  "main": "dist/arrow.js",
+  "main": "lib/arrow.js",
   "scripts": {
-    "postinstall": "./flatbuffers.sh",
-    "build": "./flatbuffers.sh && tsc && webpack",
+    "build": "./flatbuffers.sh && tsc && tsc -m es6 --outDir lib-esm && webpack",
+    "clean": "rm -rf lib lib-esm _bundles",
     "test": "./node_modules/mocha/bin/mocha ./spec/arrow.js"
   },
   "author": "",
+  "repository": "https://github.com/apache/arrow/",
   "license": "Apache-2.0",
   "devDependencies": {
+    "awesome-typescript-loader": "^3.1.3",
     "chai": "^3.5.0",
     "mocha": "^3.3.0",
+    "typescript": "^2.3.2",
+    "uglifyjs-webpack-plugin": "^0.4.3",
     "webpack": "^2.3.3"
   },
   "dependencies": {
diff --git a/js/spec/arrow.js b/js/spec/arrow.js
index 61a6f818e265a..52c586bfc7467 100644
--- a/js/spec/arrow.js
+++ b/js/spec/arrow.js
@@ -19,7 +19,7 @@ var fs = require('fs');
 var chai = require('chai');
 var assert = chai.assert;
 var path= require('path');
-var arrow = require('../dist/arrow.js');
+var arrow = require('../lib/arrow.js');
 
 test_files = [
   {
diff --git a/js/lib/Arrow_generated.d.ts b/js/src/Arrow_generated.d.ts
similarity index 100%
rename from js/lib/Arrow_generated.d.ts
rename to js/src/Arrow_generated.d.ts
diff --git a/js/lib/arrow.ts b/js/src/arrow.ts
similarity index 100%
rename from js/lib/arrow.ts
rename to js/src/arrow.ts
diff --git a/js/lib/bitarray.ts b/js/src/bitarray.ts
similarity index 100%
rename from js/lib/bitarray.ts
rename to js/src/bitarray.ts
diff --git a/js/lib/types.ts b/js/src/types.ts
similarity index 100%
rename from js/lib/types.ts
rename to js/src/types.ts
diff --git a/js/tsconfig.json b/js/tsconfig.json
index 89c31ef85a143..f2ad0e85fe842 100644
--- a/js/tsconfig.json
+++ b/js/tsconfig.json
@@ -1,14 +1,14 @@
 {
   "compilerOptions": {
-    "outDir": "./dist/",
-    "allowJs": true,
-    "target": "es5",
     "module": "commonjs",
-    "moduleResolution": "node"
+    "target": "es5",
+    "lib": ["es2015", "dom"],
+    "outDir": "lib",
+    "moduleResolution": "node",
+    "sourceMap": true,
+    "declaration": true
   },
   "include": [
-    "typings/index.d.ts",
-    "lib/*.js",
-    "lib/*.ts"
+    "src/*.ts"
   ]
 }
diff --git a/js/webpack.config.js b/js/webpack.config.js
index a0ed56370f6b1..b9c3e83a890a2 100644
--- a/js/webpack.config.js
+++ b/js/webpack.config.js
@@ -10,12 +10,40 @@
 // See the License for the specific language governing permissions and
 // limitations under the License. See accompanying LICENSE file.
 
+var path = require('path');
+var UglifyJSPlugin = require('uglifyjs-webpack-plugin');
+
 module.exports = {
-  entry: './dist/arrow.js',
+  entry: {
+    'arrow': './src/arrow.ts',
+    'arrow.min': './src/arrow.ts'
+  },
   output: {
-    path: __dirname + '/dist',
-    filename: 'arrow-bundle.js',
-    libraryTarget: 'var',
-    library: 'arrow'
+    path: path.resolve(__dirname, '_bundles'),
+    filename: '[name].js',
+    libraryTarget: 'umd',
+    library: 'arrow',
+    umdNamedDefine: true
+  },
+  resolve: {
+    extensions: ['.ts', '.js']
+  },
+  devtool: 'source-map',
+  plugins: [
+    new UglifyJSPlugin({
+      minimize: true,
+      sourceMap: true,
+      include: /\.min\.js$/
+    })
+  ],
+  module: {
+    loaders: [{
+      test: /\.ts$/,
+      loader: 'awesome-typescript-loader',
+      exclude: /node_modules/,
+      query: {
+        declaration: false
+      }
+    }]
   }
 };

From 02161456c3de5199ca0304484aff14fb7349bca4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 9 May 2017 20:59:42 -0400
Subject: [PATCH 0628/1644] ARROW-991: [Python] Create new dtype when
 deserializing from Arrow to NumPy datetime64

I am not sure how to reproduce the problem, but this will prevent us from mutating a cached dtype in the NumPy internals

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #664 from wesm/ARROW-991 and squashes the following commits:

088aa25 [Wes McKinney] Also new PyArray_NewFromDescr for blocks. Fixes pandas test suite
ee1904e [Wes McKinney] Remove unit test
2e70b08 [Wes McKinney] Always create new NPY_DATETIME dtype when converting to pandas
---
 cpp/src/arrow/python/common.cc         |   2 +-
 cpp/src/arrow/python/common.h          |  10 +--
 cpp/src/arrow/python/pandas_convert.cc | 103 ++++++++++++++-----------
 cpp/src/arrow/status.h                 |   3 +-
 cpp/src/arrow/type.cc                  |  11 +--
 cpp/src/arrow/type.h                   |   6 +-
 6 files changed, 70 insertions(+), 65 deletions(-)

diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
index 5702c71b4d8d5..ba7b6cf202e32 100644
--- a/cpp/src/arrow/python/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -69,7 +69,7 @@ Status CheckPyError(StatusCode code) {
     PyObject *exc_type, *exc_value, *traceback;
     PyErr_Fetch(&exc_type, &exc_value, &traceback);
     PyErr_NormalizeException(&exc_type, &exc_value, &traceback);
-    PyObject *exc_value_str = PyObject_Str(exc_value);
+    PyObject* exc_value_str = PyObject_Str(exc_value);
     PyObjectStringify stringified(exc_value_str);
     std::string message(stringified.bytes);
     Py_XDECREF(exc_type);
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index c5745a53f70dc..f6e706b6948d9 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -34,9 +34,7 @@ namespace py {
 
 class ARROW_EXPORT PyAcquireGIL {
  public:
-  PyAcquireGIL() : acquired_gil_(false) {
-    acquire();
-  }
+  PyAcquireGIL() : acquired_gil_(false) { acquire(); }
 
   ~PyAcquireGIL() { release(); }
 
@@ -113,11 +111,9 @@ struct ARROW_EXPORT PyObjectStringify {
 Status CheckPyError(StatusCode code = StatusCode::UnknownError);
 
 // TODO(wesm): We can just let errors pass through. To be explored later
-#define RETURN_IF_PYERROR()                     \
-  RETURN_NOT_OK(CheckPyError());
+#define RETURN_IF_PYERROR() RETURN_NOT_OK(CheckPyError());
 
-#define PY_RETURN_IF_ERROR(CODE)                \
-  RETURN_NOT_OK(CheckPyError(CODE));
+#define PY_RETURN_IF_ERROR(CODE) RETURN_NOT_OK(CheckPyError(CODE));
 
 // Return the common PyArrow memory pool
 ARROW_EXPORT void set_default_memory_pool(MemoryPool* pool);
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index b54197e5145b0..264bed11b04ec 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -977,6 +977,55 @@ Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
 // ----------------------------------------------------------------------
 // pandas 0.x DataFrame conversion internals
 
+inline void set_numpy_metadata(int type, DataType* datatype, PyArray_Descr* out) {
+  if (type == NPY_DATETIME) {
+    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(out->c_metadata);
+    if (datatype->id() == Type::TIMESTAMP) {
+      auto timestamp_type = static_cast<TimestampType*>(datatype);
+
+      switch (timestamp_type->unit()) {
+        case TimestampType::Unit::SECOND:
+          date_dtype->meta.base = NPY_FR_s;
+          break;
+        case TimestampType::Unit::MILLI:
+          date_dtype->meta.base = NPY_FR_ms;
+          break;
+        case TimestampType::Unit::MICRO:
+          date_dtype->meta.base = NPY_FR_us;
+          break;
+        case TimestampType::Unit::NANO:
+          date_dtype->meta.base = NPY_FR_ns;
+          break;
+      }
+    } else {
+      // datatype->type == Type::DATE64
+      date_dtype->meta.base = NPY_FR_D;
+    }
+  }
+}
+
+static inline PyArray_Descr* GetSafeNumPyDtype(int type) {
+  if (type == NPY_DATETIME) {
+    // It is not safe to mutate the result of DescrFromType
+    return PyArray_DescrNewFromType(type);
+  } else {
+    return PyArray_DescrFromType(type);
+  }
+}
+static inline PyObject* NewArray1DFromType(
+    DataType* arrow_type, int type, int64_t length, void* data) {
+  npy_intp dims[1] = {length};
+
+  PyArray_Descr* descr = GetSafeNumPyDtype(type);
+  if (descr == nullptr) {
+    // Error occurred, trust error state is set
+    return nullptr;
+  }
+
+  set_numpy_metadata(type, arrow_type, descr);
+  return PyArray_NewFromDescr(&PyArray_Type, descr, 1, dims, nullptr, data, 0, nullptr);
+}
+
 class PandasBlock {
  public:
   enum type {
@@ -1024,13 +1073,17 @@ class PandasBlock {
   Status AllocateNDArray(int npy_type, int ndim = 2) {
     PyAcquireGIL lock;
 
+    PyArray_Descr* descr = GetSafeNumPyDtype(npy_type);
+
     PyObject* block_arr;
     if (ndim == 2) {
       npy_intp block_dims[2] = {num_columns_, num_rows_};
-      block_arr = PyArray_SimpleNew(2, block_dims, npy_type);
+      block_arr = PyArray_NewFromDescr(
+          &PyArray_Type, descr, 2, block_dims, nullptr, nullptr, 0, nullptr);
     } else {
       npy_intp block_dims[1] = {num_rows_};
-      block_arr = PyArray_SimpleNew(1, block_dims, npy_type);
+      block_arr = PyArray_NewFromDescr(
+          &PyArray_Type, descr, 1, block_dims, nullptr, nullptr, 0, nullptr);
     }
 
     if (block_arr == NULL) {
@@ -1947,34 +2000,6 @@ class DataFrameBlockCreator {
   BlockMap datetimetz_blocks_;
 };
 
-inline void set_numpy_metadata(int type, DataType* datatype, PyArrayObject* out) {
-  if (type == NPY_DATETIME) {
-    PyArray_Descr* descr = PyArray_DESCR(out);
-    auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-    if (datatype->id() == Type::TIMESTAMP) {
-      auto timestamp_type = static_cast<TimestampType*>(datatype);
-
-      switch (timestamp_type->unit()) {
-        case TimestampType::Unit::SECOND:
-          date_dtype->meta.base = NPY_FR_s;
-          break;
-        case TimestampType::Unit::MILLI:
-          date_dtype->meta.base = NPY_FR_ms;
-          break;
-        case TimestampType::Unit::MICRO:
-          date_dtype->meta.base = NPY_FR_us;
-          break;
-        case TimestampType::Unit::NANO:
-          date_dtype->meta.base = NPY_FR_ns;
-          break;
-      }
-    } else {
-      // datatype->type == Type::DATE64
-      date_dtype->meta.base = NPY_FR_D;
-    }
-  }
-}
-
 class ArrowDeserializer {
  public:
   ArrowDeserializer(const std::shared_ptr<Column>& col, PyObject* py_ref)
@@ -1983,17 +2008,8 @@ class ArrowDeserializer {
   Status AllocateOutput(int type) {
     PyAcquireGIL lock;
 
-    npy_intp dims[1] = {col_->length()};
-    result_ = PyArray_SimpleNew(1, dims, type);
+    result_ = NewArray1DFromType(col_->type().get(), type, col_->length(), nullptr);
     arr_ = reinterpret_cast<PyArrayObject*>(result_);
-
-    if (arr_ == NULL) {
-      // Error occurred, trust that SimpleNew set the error state
-      return Status::OK();
-    }
-
-    set_numpy_metadata(type, col_->type().get(), arr_);
-
     return Status::OK();
   }
 
@@ -2010,17 +2026,14 @@ class ArrowDeserializer {
     PyAcquireGIL lock;
 
     // Zero-Copy. We can pass the data pointer directly to NumPy.
-    npy_intp dims[1] = {col_->length()};
-    result_ = PyArray_SimpleNewFromData(1, dims, npy_type, data);
+    result_ = NewArray1DFromType(col_->type().get(), npy_type, col_->length(), data);
     arr_ = reinterpret_cast<PyArrayObject*>(result_);
 
     if (arr_ == NULL) {
-      // Error occurred, trust that SimpleNew set the error state
+      // Error occurred, trust that error set
       return Status::OK();
     }
 
-    set_numpy_metadata(npy_type, col_->type().get(), arr_);
-
     if (PyArray_SetBaseObject(arr_, py_ref_) == -1) {
       // Error occurred, trust that SetBaseObject set the error state
       return Status::OK();
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 6a8cee27cb756..1688b96785367 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -91,8 +91,7 @@ class ARROW_EXPORT Status {
   Status() : state_(NULL) {}
   ~Status() { delete[] state_; }
 
-  Status(StatusCode code, const std::string& msg)
-    : Status(code, msg, -1) {}
+  Status(StatusCode code, const std::string& msg) : Status(code, msg, -1) {}
 
   // Copy the specified status.
   Status(const Status& s);
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index b1e322ce1b321..afb30272c0110 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -42,9 +42,7 @@ std::shared_ptr<Field> Field::RemoveMetadata() const {
 }
 
 bool Field::Equals(const Field& other) const {
-  if (this == &other) {
-    return true;
-  }
+  if (this == &other) { return true; }
   if (this->name_ == other.name_ && this->nullable_ == other.nullable_ &&
       this->type_->Equals(*other.type_.get())) {
     if (metadata_ == nullptr && other.metadata_ == nullptr) {
@@ -322,8 +320,7 @@ std::string Schema::ToString() const {
   if (metadata_) {
     buffer << "\n-- metadata --";
     for (int64_t i = 0; i < metadata_->size(); ++i) {
-      buffer << "\n" << metadata_->key(i) << ": "
-             << metadata_->value(i);
+      buffer << "\n" << metadata_->key(i) << ": " << metadata_->value(i);
     }
   }
 
@@ -419,8 +416,8 @@ std::shared_ptr<DataType> dictionary(const std::shared_ptr<DataType>& index_type
   return std::make_shared<DictionaryType>(index_type, dict_values);
 }
 
-std::shared_ptr<Field> field(
-    const std::string& name, const std::shared_ptr<DataType>& type, bool nullable,
+std::shared_ptr<Field> field(const std::string& name,
+    const std::shared_ptr<DataType>& type, bool nullable,
     const std::shared_ptr<const KeyValueMetadata>& metadata) {
   return std::make_shared<Field>(name, type, nullable, metadata);
 }
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index bb258578da327..40615f790fb0e 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -205,7 +205,7 @@ class ARROW_EXPORT Field {
   Field(const std::string& name, const std::shared_ptr<DataType>& type,
       bool nullable = true,
       const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr)
-    : name_(name), type_(type), nullable_(nullable), metadata_(metadata) {}
+      : name_(name), type_(type), nullable_(nullable), metadata_(metadata) {}
 
   std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
 
@@ -753,8 +753,8 @@ std::shared_ptr<DataType> ARROW_EXPORT union_(
 std::shared_ptr<DataType> ARROW_EXPORT dictionary(
     const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& values);
 
-std::shared_ptr<Field> ARROW_EXPORT field(
-    const std::string& name, const std::shared_ptr<DataType>& type, bool nullable = true,
+std::shared_ptr<Field> ARROW_EXPORT field(const std::string& name,
+    const std::shared_ptr<DataType>& type, bool nullable = true,
     const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
 
 // ----------------------------------------------------------------------

From 885654107f367cc92a3ccb9a7b5930de8707f7f5 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 11 May 2017 08:26:14 -0400
Subject: [PATCH 0629/1644] ARROW-1001: [GLib] Unify writer files

Author: Kouhei Sutou <kou@clear-code.com>

Closes #671 from kou/glib-unify-writer-files and squashes the following commits:

f1e58e0 [Kouhei Sutou] [GLib] Unify writer files
---
 c_glib/arrow-glib/Makefile.am                 |  11 +-
 c_glib/arrow-glib/arrow-glib.h                |   3 +-
 c_glib/arrow-glib/file-writer.cpp             | 149 ------------------
 c_glib/arrow-glib/file-writer.h               |  78 ---------
 c_glib/arrow-glib/file-writer.hpp             |  28 ----
 c_glib/arrow-glib/stream-writer.h             |  82 ----------
 .../{stream-writer.cpp => writer.cpp}         | 109 ++++++++++---
 c_glib/arrow-glib/writer.h                    | 137 ++++++++++++++++
 .../{stream-writer.hpp => writer.hpp}         |   5 +-
 c_glib/doc/reference/arrow-glib-docs.sgml     |   3 +-
 10 files changed, 232 insertions(+), 373 deletions(-)
 delete mode 100644 c_glib/arrow-glib/file-writer.cpp
 delete mode 100644 c_glib/arrow-glib/file-writer.h
 delete mode 100644 c_glib/arrow-glib/file-writer.hpp
 delete mode 100644 c_glib/arrow-glib/stream-writer.h
 rename c_glib/arrow-glib/{stream-writer.cpp => writer.cpp} (66%)
 create mode 100644 c_glib/arrow-glib/writer.h
 rename c_glib/arrow-glib/{stream-writer.hpp => writer.hpp} (83%)

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 242507273f451..0f83f569230c9 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -67,10 +67,9 @@ libarrow_glib_la_headers +=			\
 
 libarrow_glib_la_headers +=			\
 	file-reader.h				\
-	file-writer.h				\
 	stream-reader.h				\
-	stream-writer.h				\
-	metadata-version.h
+	metadata-version.h			\
+	writer.h
 
 libarrow_glib_la_generated_headers =		\
 	enums.h
@@ -107,10 +106,9 @@ libarrow_glib_la_sources +=			\
 
 libarrow_glib_la_sources +=			\
 	file-reader.cpp				\
-	file-writer.cpp				\
 	metadata-version.cpp			\
 	stream-reader.cpp			\
-	stream-writer.cpp
+	writer.cpp
 
 libarrow_glib_la_cpp_headers =			\
 	array.hpp				\
@@ -139,10 +137,9 @@ libarrow_glib_la_cpp_headers +=			\
 
 libarrow_glib_la_cpp_headers +=			\
 	file-reader.hpp				\
-	file-writer.hpp				\
 	metadata-version.hpp			\
 	stream-reader.hpp			\
-	stream-writer.hpp
+	writer.hpp
 
 libarrow_glib_la_SOURCES =			\
 	$(libarrow_glib_la_sources)		\
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index 0a06cb824dc85..f98899c5666cc 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -42,7 +42,6 @@
 #include <arrow-glib/writeable-file.h>
 
 #include <arrow-glib/file-reader.h>
-#include <arrow-glib/file-writer.h>
 #include <arrow-glib/metadata-version.h>
 #include <arrow-glib/stream-reader.h>
-#include <arrow-glib/stream-writer.h>
+#include <arrow-glib/writer.h>
diff --git a/c_glib/arrow-glib/file-writer.cpp b/c_glib/arrow-glib/file-writer.cpp
deleted file mode 100644
index e3c721c49b16f..0000000000000
--- a/c_glib/arrow-glib/file-writer.cpp
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow/ipc/api.h>
-
-#include <arrow-glib/array.hpp>
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/record-batch.hpp>
-#include <arrow-glib/schema.hpp>
-
-#include <arrow-glib/output-stream.hpp>
-
-#include <arrow-glib/stream-writer.hpp>
-#include <arrow-glib/file-writer.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: file-writer
- * @short_description: File writer class
- *
- * #GArrowFileWriter is a class for sending data by file based IPC.
- */
-
-G_DEFINE_TYPE(GArrowFileWriter,
-              garrow_file_writer,
-              GARROW_TYPE_STREAM_WRITER);
-
-static void
-garrow_file_writer_init(GArrowFileWriter *object)
-{
-}
-
-static void
-garrow_file_writer_class_init(GArrowFileWriterClass *klass)
-{
-}
-
-/**
- * garrow_file_writer_new:
- * @sink: The output of the writer.
- * @schema: The schema of the writer.
- * @error: (nullable): Return locatipcn for a #GError or %NULL.
- *
- * Returns: (nullable): A newly created #GArrowFileWriter or %NULL on
- *   error.
- */
-GArrowFileWriter *
-garrow_file_writer_new(GArrowOutputStream *sink,
-                       GArrowSchema *schema,
-                       GError **error)
-{
-  std::shared_ptr<arrow::ipc::FileWriter> arrow_file_writer;
-  auto status =
-    arrow::ipc::FileWriter::Open(garrow_output_stream_get_raw(sink).get(),
-                                 garrow_schema_get_raw(schema),
-                                 &arrow_file_writer);
-  if (garrow_error_check(error, status, "[ipc][file-writer][open]")) {
-    return garrow_file_writer_new_raw(&arrow_file_writer);
-  } else {
-    return NULL;
-  }
-}
-
-/**
- * garrow_file_writer_write_record_batch:
- * @file_writer: A #GArrowFileWriter.
- * @record_batch: The record batch to be written.
- * @error: (nullable): Return locatipcn for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_file_writer_write_record_batch(GArrowFileWriter *file_writer,
-                                          GArrowRecordBatch *record_batch,
-                                          GError **error)
-{
-  auto arrow_file_writer =
-    garrow_file_writer_get_raw(file_writer);
-  auto arrow_record_batch =
-    garrow_record_batch_get_raw(record_batch);
-  auto arrow_record_batch_raw =
-    arrow_record_batch.get();
-
-  auto status = arrow_file_writer->WriteRecordBatch(*arrow_record_batch_raw);
-  return garrow_error_check(error,
-                            status,
-                            "[ipc][file-writer][write-record-batch]");
-}
-
-/**
- * garrow_file_writer_close:
- * @file_writer: A #GArrowFileWriter.
- * @error: (nullable): Return locatipcn for a #GError or %NULL.
- *
- * Returns: %TRUE on success, %FALSE if there was an error.
- */
-gboolean
-garrow_file_writer_close(GArrowFileWriter *file_writer,
-                             GError **error)
-{
-  auto arrow_file_writer =
-    garrow_file_writer_get_raw(file_writer);
-
-  auto status = arrow_file_writer->Close();
-  return garrow_error_check(error, status, "[ipc][file-writer][close]");
-}
-
-G_END_DECLS
-
-GArrowFileWriter *
-garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer)
-{
-  auto file_writer =
-    GARROW_FILE_WRITER(g_object_new(GARROW_TYPE_FILE_WRITER,
-                                        "stream-writer", arrow_file_writer,
-                                        NULL));
-  return file_writer;
-}
-
-arrow::ipc::FileWriter *
-garrow_file_writer_get_raw(GArrowFileWriter *file_writer)
-{
-  auto arrow_stream_writer =
-    garrow_stream_writer_get_raw(GARROW_STREAM_WRITER(file_writer));
-  auto arrow_file_writer_raw =
-    dynamic_cast<arrow::ipc::FileWriter *>(arrow_stream_writer.get());
-  return arrow_file_writer_raw;
-}
diff --git a/c_glib/arrow-glib/file-writer.h b/c_glib/arrow-glib/file-writer.h
deleted file mode 100644
index 346dc6f242ae3..0000000000000
--- a/c_glib/arrow-glib/file-writer.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/stream-writer.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_FILE_WRITER             \
-  (garrow_file_writer_get_type())
-#define GARROW_FILE_WRITER(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_FILE_WRITER,      \
-                              GArrowFileWriter))
-#define GARROW_FILE_WRITER_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_FILE_WRITER, \
-                           GArrowFileWriterClass))
-#define GARROW_IS_FILE_WRITER(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_FILE_WRITER))
-#define GARROW_IS_FILE_WRITER_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_FILE_WRITER))
-#define GARROW_FILE_WRITER_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_FILE_WRITER,       \
-                             GArrowFileWriterClass))
-
-typedef struct _GArrowFileWriter         GArrowFileWriter;
-typedef struct _GArrowFileWriterClass    GArrowFileWriterClass;
-
-/**
- * GArrowFileWriter:
- *
- * It wraps `arrow::ipc::FileWriter`.
- */
-struct _GArrowFileWriter
-{
-  /*< private >*/
-  GArrowStreamWriter parent_instance;
-};
-
-struct _GArrowFileWriterClass
-{
-  GObjectClass parent_class;
-};
-
-GType garrow_file_writer_get_type(void) G_GNUC_CONST;
-
-GArrowFileWriter *garrow_file_writer_new(GArrowOutputStream *sink,
-                                         GArrowSchema *schema,
-                                         GError **error);
-
-gboolean garrow_file_writer_write_record_batch(GArrowFileWriter *file_writer,
-                                                   GArrowRecordBatch *record_batch,
-                                                   GError **error);
-gboolean garrow_file_writer_close(GArrowFileWriter *file_writer,
-                                      GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/file-writer.hpp b/c_glib/arrow-glib/file-writer.hpp
deleted file mode 100644
index f6a720a6cde7e..0000000000000
--- a/c_glib/arrow-glib/file-writer.hpp
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow/api.h>
-#include <arrow/ipc/api.h>
-
-#include <arrow-glib/file-writer.h>
-
-GArrowFileWriter *garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer);
-arrow::ipc::FileWriter *garrow_file_writer_get_raw(GArrowFileWriter *file_writer);
diff --git a/c_glib/arrow-glib/stream-writer.h b/c_glib/arrow-glib/stream-writer.h
deleted file mode 100644
index d718b188d8fff..0000000000000
--- a/c_glib/arrow-glib/stream-writer.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/array.h>
-#include <arrow-glib/record-batch.h>
-#include <arrow-glib/schema.h>
-
-#include <arrow-glib/output-stream.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_STREAM_WRITER           \
-  (garrow_stream_writer_get_type())
-#define GARROW_STREAM_WRITER(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_STREAM_WRITER,    \
-                              GArrowStreamWriter))
-#define GARROW_STREAM_WRITER_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_STREAM_WRITER,       \
-                           GArrowStreamWriterClass))
-#define GARROW_IS_STREAM_WRITER(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STREAM_WRITER))
-#define GARROW_IS_STREAM_WRITER_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_STREAM_WRITER))
-#define GARROW_STREAM_WRITER_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_STREAM_WRITER,     \
-                             GArrowStreamWriterClass))
-
-typedef struct _GArrowStreamWriter         GArrowStreamWriter;
-typedef struct _GArrowStreamWriterClass    GArrowStreamWriterClass;
-
-/**
- * GArrowStreamWriter:
- *
- * It wraps `arrow::ipc::StreamWriter`.
- */
-struct _GArrowStreamWriter
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowStreamWriterClass
-{
-  GObjectClass parent_class;
-};
-
-GType garrow_stream_writer_get_type(void) G_GNUC_CONST;
-
-GArrowStreamWriter *garrow_stream_writer_new(GArrowOutputStream *sink,
-                                             GArrowSchema *schema,
-                                             GError **error);
-
-gboolean garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
-                                                     GArrowRecordBatch *record_batch,
-                                                     GError **error);
-gboolean garrow_stream_writer_close(GArrowStreamWriter *stream_writer,
-                                        GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/stream-writer.cpp b/c_glib/arrow-glib/writer.cpp
similarity index 66%
rename from c_glib/arrow-glib/stream-writer.cpp
rename to c_glib/arrow-glib/writer.cpp
index 45e2fb099535b..625a19e3b0b07 100644
--- a/c_glib/arrow-glib/stream-writer.cpp
+++ b/c_glib/arrow-glib/writer.cpp
@@ -30,16 +30,20 @@
 
 #include <arrow-glib/output-stream.hpp>
 
-#include <arrow-glib/stream-writer.hpp>
+#include <arrow-glib/writer.hpp>
 
 G_BEGIN_DECLS
 
 /**
- * SECTION: stream-writer
- * @short_description: Stream writer class
+ * SECTION: writer
+ * @section_id: writer-classes
+ * @title: Writer classes
+ * @include: arrow-glib/arrow-glib.h
  *
- * #GArrowStreamWriter is a class for sending data by stream based
- * IPC.
+ * #GArrowStreamWriter is a class for writing data in stream mode.
+ *
+ * #GArrowFileWriter is a class for writing data in random access
+ * mode.
  */
 
 typedef struct GArrowStreamWriterPrivate_ {
@@ -55,9 +59,9 @@ G_DEFINE_TYPE_WITH_PRIVATE(GArrowStreamWriter,
                            garrow_stream_writer,
                            G_TYPE_OBJECT);
 
-#define GARROW_STREAM_WRITER_GET_PRIVATE(obj)               \
+#define GARROW_STREAM_WRITER_GET_PRIVATE(obj)                   \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
-                               GARROW_TYPE_STREAM_WRITER,   \
+                               GARROW_TYPE_STREAM_WRITER,       \
                                GArrowStreamWriterPrivate))
 
 static void
@@ -74,9 +78,9 @@ garrow_stream_writer_finalize(GObject *object)
 
 static void
 garrow_stream_writer_set_property(GObject *object,
-                                    guint prop_id,
-                                    const GValue *value,
-                                    GParamSpec *pspec)
+                                  guint prop_id,
+                                  const GValue *value,
+                                  GParamSpec *pspec)
 {
   GArrowStreamWriterPrivate *priv;
 
@@ -95,9 +99,9 @@ garrow_stream_writer_set_property(GObject *object,
 
 static void
 garrow_stream_writer_get_property(GObject *object,
-                                          guint prop_id,
-                                          GValue *value,
-                                          GParamSpec *pspec)
+                                  guint prop_id,
+                                  GValue *value,
+                                  GParamSpec *pspec)
 {
   switch (prop_id) {
   default:
@@ -167,15 +171,12 @@ garrow_stream_writer_new(GArrowOutputStream *sink,
  */
 gboolean
 garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
-                                            GArrowRecordBatch *record_batch,
-                                            GError **error)
+                                        GArrowRecordBatch *record_batch,
+                                        GError **error)
 {
-  auto arrow_stream_writer =
-    garrow_stream_writer_get_raw(stream_writer);
-  auto arrow_record_batch =
-    garrow_record_batch_get_raw(record_batch);
-  auto arrow_record_batch_raw =
-    arrow_record_batch.get();
+  auto arrow_stream_writer = garrow_stream_writer_get_raw(stream_writer);
+  auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  auto arrow_record_batch_raw = arrow_record_batch.get();
 
   auto status = arrow_stream_writer->WriteRecordBatch(*arrow_record_batch_raw);
   return garrow_error_check(error,
@@ -192,15 +193,55 @@ garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
  */
 gboolean
 garrow_stream_writer_close(GArrowStreamWriter *stream_writer,
-                               GError **error)
+                           GError **error)
 {
-  auto arrow_stream_writer =
-    garrow_stream_writer_get_raw(stream_writer);
+  auto arrow_stream_writer = garrow_stream_writer_get_raw(stream_writer);
 
   auto status = arrow_stream_writer->Close();
   return garrow_error_check(error, status, "[ipc][stream-writer][close]");
 }
 
+
+G_DEFINE_TYPE(GArrowFileWriter,
+              garrow_file_writer,
+              GARROW_TYPE_STREAM_WRITER);
+
+static void
+garrow_file_writer_init(GArrowFileWriter *object)
+{
+}
+
+static void
+garrow_file_writer_class_init(GArrowFileWriterClass *klass)
+{
+}
+
+/**
+ * garrow_file_writer_new:
+ * @sink: The output of the writer.
+ * @schema: The schema of the writer.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowFileWriter or %NULL on
+ *   error.
+ */
+GArrowFileWriter *
+garrow_file_writer_new(GArrowOutputStream *sink,
+                       GArrowSchema *schema,
+                       GError **error)
+{
+  std::shared_ptr<arrow::ipc::FileWriter> arrow_file_writer;
+  auto status =
+    arrow::ipc::FileWriter::Open(garrow_output_stream_get_raw(sink).get(),
+                                 garrow_schema_get_raw(schema),
+                                 &arrow_file_writer);
+  if (garrow_error_check(error, status, "[ipc][file-writer][open]")) {
+    return garrow_file_writer_new_raw(&arrow_file_writer);
+  } else {
+    return NULL;
+  }
+}
+
 G_END_DECLS
 
 GArrowStreamWriter *
@@ -221,3 +262,23 @@ garrow_stream_writer_get_raw(GArrowStreamWriter *stream_writer)
   priv = GARROW_STREAM_WRITER_GET_PRIVATE(stream_writer);
   return priv->stream_writer;
 }
+
+GArrowFileWriter *
+garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer)
+{
+  auto file_writer =
+    GARROW_FILE_WRITER(g_object_new(GARROW_TYPE_FILE_WRITER,
+                                    "stream-writer", arrow_file_writer,
+                                    NULL));
+  return file_writer;
+}
+
+arrow::ipc::FileWriter *
+garrow_file_writer_get_raw(GArrowFileWriter *file_writer)
+{
+  auto arrow_stream_writer =
+    garrow_stream_writer_get_raw(GARROW_STREAM_WRITER(file_writer));
+  auto arrow_file_writer_raw =
+    dynamic_cast<arrow::ipc::FileWriter *>(arrow_stream_writer.get());
+  return arrow_file_writer_raw;
+}
diff --git a/c_glib/arrow-glib/writer.h b/c_glib/arrow-glib/writer.h
new file mode 100644
index 0000000000000..2aaa776f8176f
--- /dev/null
+++ b/c_glib/arrow-glib/writer.h
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array.h>
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+
+#include <arrow-glib/output-stream.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_STREAM_WRITER               \
+  (garrow_stream_writer_get_type())
+#define GARROW_STREAM_WRITER(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STREAM_WRITER,        \
+                              GArrowStreamWriter))
+#define GARROW_STREAM_WRITER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_STREAM_WRITER,   \
+                           GArrowStreamWriterClass))
+#define GARROW_IS_STREAM_WRITER(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STREAM_WRITER))
+#define GARROW_IS_STREAM_WRITER_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_STREAM_WRITER))
+#define GARROW_STREAM_WRITER_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_STREAM_WRITER, \
+                             GArrowStreamWriterClass))
+
+typedef struct _GArrowStreamWriter         GArrowStreamWriter;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowStreamWriterClass    GArrowStreamWriterClass;
+#endif
+
+/**
+ * GArrowStreamWriter:
+ *
+ * It wraps `arrow::ipc::StreamWriter`.
+ */
+struct _GArrowStreamWriter
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowStreamWriterClass
+{
+  GObjectClass parent_class;
+};
+#endif
+
+GType garrow_stream_writer_get_type(void) G_GNUC_CONST;
+
+GArrowStreamWriter *garrow_stream_writer_new(GArrowOutputStream *sink,
+                                             GArrowSchema *schema,
+                                             GError **error);
+
+gboolean garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
+                                                 GArrowRecordBatch *record_batch,
+                                                 GError **error);
+gboolean garrow_stream_writer_close(GArrowStreamWriter *stream_writer,
+                                    GError **error);
+
+
+#define GARROW_TYPE_FILE_WRITER                 \
+  (garrow_file_writer_get_type())
+#define GARROW_FILE_WRITER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_FILE_WRITER,  \
+                              GArrowFileWriter))
+#define GARROW_FILE_WRITER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_FILE_WRITER,     \
+                           GArrowFileWriterClass))
+#define GARROW_IS_FILE_WRITER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_FILE_WRITER))
+#define GARROW_IS_FILE_WRITER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_FILE_WRITER))
+#define GARROW_FILE_WRITER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_FILE_WRITER,   \
+                             GArrowFileWriterClass))
+
+typedef struct _GArrowFileWriter         GArrowFileWriter;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowFileWriterClass    GArrowFileWriterClass;
+#endif
+
+/**
+ * GArrowFileWriter:
+ *
+ * It wraps `arrow::ipc::FileWriter`.
+ */
+struct _GArrowFileWriter
+{
+  /*< private >*/
+  GArrowStreamWriter parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowFileWriterClass
+{
+  GArrowStreamWriterClass parent_class;
+};
+#endif
+
+GType garrow_file_writer_get_type(void) G_GNUC_CONST;
+
+GArrowFileWriter *garrow_file_writer_new(GArrowOutputStream *sink,
+                                         GArrowSchema *schema,
+                                         GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/stream-writer.hpp b/c_glib/arrow-glib/writer.hpp
similarity index 83%
rename from c_glib/arrow-glib/stream-writer.hpp
rename to c_glib/arrow-glib/writer.hpp
index 994c83b8f4ad5..199f205e28fe0 100644
--- a/c_glib/arrow-glib/stream-writer.hpp
+++ b/c_glib/arrow-glib/writer.hpp
@@ -22,7 +22,10 @@
 #include <arrow/api.h>
 #include <arrow/ipc/api.h>
 
-#include <arrow-glib/stream-writer.h>
+#include <arrow-glib/writer.h>
 
 GArrowStreamWriter *garrow_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer);
 std::shared_ptr<arrow::ipc::StreamWriter> garrow_stream_writer_get_raw(GArrowStreamWriter *stream_writer);
+
+GArrowFileWriter *garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer);
+arrow::ipc::FileWriter *garrow_file_writer_get_raw(GArrowFileWriter *file_writer);
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 8c691de93bce4..dae89e2138a80 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -108,8 +108,7 @@
     </chapter>
     <chapter id="writer">
       <title>Writer</title>
-      <xi:include href="xml/file-writer.xml"/>
-      <xi:include href="xml/stream-writer.xml"/>
+      <xi:include href="xml/writer.xml"/>
     </chapter>
   </part>
 

From 3f22b5f768f389ab811dbd1c7688ea55fbb04174 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 11 May 2017 15:56:17 -0400
Subject: [PATCH 0630/1644] ARROW-1000: [GLib] Move install document to Website

Author: Kouhei Sutou <kou@clear-code.com>

Closes #670 from kou/glib-move-install-docuemnt and squashes the following commits:

50d86b3 [Kouhei Sutou] [GLib] Move install document to Website
---
 c_glib/README.md | 72 +-----------------------------------------------
 site/install.md  | 68 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 71 deletions(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index 7d0ea3a9d030b..f8269df5eac91 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -50,77 +50,7 @@ released in the future.
 
 ### Package
 
-There are supported platforms:
-
-  * Debian GNU/Linux Jessie
-  * Ubuntu 16.04 LTS
-  * Ubuntu 16.10
-  * Ubuntu 17.04
-  * CentOS 7
-
-You can feedback to https://github.com/kou/arrow-packages about
-packages things.
-
-#### Debian GNU/Linux jessie
-
-You need to install `apt-transport-https` to use HTTPS for APT
-repository.
-
-```text
-% sudo apt install -y apt-transport-https
-```
-
-You need to add the following apt-lines to
-`/etc/apt/sources.list.d/groonga.list`:
-
-```text
-deb https://packages.groonga.org/debian/ jessie main
-deb-src https://packages.groonga.org/debian/ jessie main
-```
-
-Then you need to run the following command lines:
-
-```text
-% sudo apt update
-% sudo apt install -y --allow-unauthenticated groonga-keyring
-% sudo apt update
-```
-
-Now you can install Arrow GLib packages:
-
-```text
-% sudo apt install -y libarrow-glib-dev
-```
-
-#### Ubuntu
-
-You need to add an APT repository:
-
-```text
-% sudo apt install -y software-properties-common
-% sudo add-apt-repository -y ppa:groonga/ppa
-% sudo apt update
-```
-
-Now you can install Arrow GLib packages:
-
-```text
-% sudo apt install -y libarrow-glib-dev
-```
-
-#### CentOS 7
-
-You need to add a Yum repository:
-
-```text
-% sudo yum install -y https://packages.groonga.org/centos/groonga-release-1.3.0-1.noarch.rpm
-```
-
-Now you can install Arrow GLib packages:
-
-```text
-% sudo yum install -y --enablerepo=epel arrow-glib-devel
-```
+See [install document][../site/install.md] for details.
 
 ### How to build by users
 
diff --git a/site/install.md b/site/install.md
index b1e7990c1b8f3..a575a1e0587f6 100644
--- a/site/install.md
+++ b/site/install.md
@@ -66,6 +66,72 @@ pip install pyarrow
 These include the Apache Arrow and Apache Parquet C++ binary libraries bundled
 with the wheel.
 
+### C++ and GLib (C) Packages for Debian GNU/Linux, Ubuntu and CentOS (Unofficial)
+
+We have provided APT and Yum repositories for Apache Arrow C++ and
+Apache Arrow GLib (C). Here are supported platforms:
+
+* Debian GNU/Linux Jessie
+* Ubuntu 16.04 LTS
+* Ubuntu 16.10
+* Ubuntu 17.04
+* CentOS 7
+
+Debian GNU/Linux Jessie:
+
+```shell
+sudo apt update
+sudo apt install -y -V apt-transport-https
+cat <<APT_LINE | sudo tee /etc/apt/sources.list.d/groonga.list
+deb https://packages.groonga.org/debian/ jessie main
+deb-src https://packages.groonga.org/debian/ jessie main
+APT_LINE
+sudo apt update
+sudo apt install -y -V --allow-unauthenticated groonga-keyring
+sudo apt update
+sudo apt install -y -V libarrow-dev # For C++
+sudo apt install -y -V libarrow-glib-dev # For GLib (C)
+```
+
+Ubuntu:
+
+```shell
+sudo apt install -y software-properties-common
+sudo add-apt-repository -y ppa:groonga/ppa
+sudo apt update
+sudo apt install -y -V libarrow-dev # For C++
+sudo apt install -y -V libarrow-glib-dev # For GLib (C)
+```
+
+CentOS:
+
+```shell
+sudo yum install -y https://packages.groonga.org/centos/groonga-release-1.3.0-1.noarch.rpm
+sudo yum install -y --enablerepo=epel arrow-devel # For C++
+sudo yum install -y --enablerepo=epel arrow-glib-devel # For GLib (C)
+```
+
+These repositories also provide Apache Parquet C++ and
+[Parquet GLib][8]. You can install them by the followings:
+
+Debian GNU/Linux and Ubuntu:
+
+```shell
+sudo apt install -y -V libparquet-dev # For Apache Parquet C++
+sudo apt install -y -V libparquet-glib-dev # For Parquet GLib (C)
+```
+
+CentOS:
+
+```shell
+sudo yum install -y --enablerepo=epel parquet-devel # For Apache Parquet C++
+sudo yum install -y --enablerepo=epel parquet-glib-devel # For Parquet GLib (C)
+```
+
+These repositories are managed at
+[red-data-tools/arrow-packages][9]. If you have any feedback, please
+send it to the project instead of Apache Arrow project.
+
 [1]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0
 [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.3.0
 [3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.md5
@@ -73,3 +139,5 @@ with the wheel.
 [5]: http://conda-forge.github.io
 [6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
 [7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.asc
+[8]: https://github.com/red-data-tools/parquet-glib
+[9]: https://github.com/red-data-tools/arrow-packages

From c10b7d798bfcd1e47d8f3dc6a5bdff2047f34293 Mon Sep 17 00:00:00 2001
From: bgosztonyi <b.alazsgosztonyi@gmail.com>
Date: Thu, 11 May 2017 15:58:31 -0400
Subject: [PATCH 0631/1644] ARROW-1003: [C++] Check flag _WIN32 instead of
 __WIN32

__WIN32 is not standard for all windows compilers.

Author: bgosztonyi <b.alazsgosztonyi@gmail.com>

Closes #669 from bgosztonyi/patch-1 and squashes the following commits:

618ab3e [bgosztonyi] _WIN32 instead of __WIN32
---
 cpp/src/arrow/io/hdfs-internal.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index e67419b5fa501..8b4a92b396789 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -69,7 +69,7 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
   std::string file_name;
 
 // OS-specific file name
-#ifdef __WIN32
+#ifdef _WIN32
   file_name = "hdfs.dll";
 #elif __APPLE__
   file_name = "libhdfs.dylib";
@@ -103,7 +103,7 @@ static std::vector<fs::path> get_potential_libhdfs3_paths() {
   std::string file_name;
 
 // OS-specific file name
-#ifdef __WIN32
+#ifdef _WIN32
   file_name = "hdfs3.dll";
 #elif __APPLE__
   file_name = "libhdfs3.dylib";

From 84413b0eceeeacc985e7fc6bb95e0bfe76506480 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 11 May 2017 16:00:36 -0400
Subject: [PATCH 0632/1644] ARROW-901: [Python] Add Parquet unit test for fixed
 size binary

This verifies that all is working properly after PARQUET-965.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #665 from wesm/ARROW-901 and squashes the following commits:

6433c14 [Wes McKinney] Add Parquet unit test for fixed size binary
---
 python/pyarrow/tests/test_parquet.py | 33 +++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index bb3a9ed5f4a25..7144de2a6852a 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -349,9 +349,7 @@ def test_column_of_lists(tmpdir):
 
 
 @parquet
-def test_date_time_types(tmpdir):
-    buf = io.BytesIO()
-
+def test_date_time_types():
     t1 = pa.date32()
     data1 = np.array([17259, 17260, 17261], dtype='int32')
     a1 = pa.Array.from_pandas(data1, type=t1)
@@ -388,11 +386,7 @@ def test_date_time_types(tmpdir):
                                     ['date32', 'date64', 'timestamp[us]',
                                      'time32[s]', 'time64[us]', 'time32[s]'])
 
-    pq.write_table(table, buf, version="2.0")
-    buf.seek(0)
-
-    result = pq.read_table(buf)
-    assert result.equals(expected)
+    _check_roundtrip(table, expected=expected, version='2.0')
 
     # Unsupported stuff
     def _assert_unsupported(array):
@@ -408,6 +402,29 @@ def _assert_unsupported(array):
     _assert_unsupported(a7)
 
 
+@parquet
+def test_fixed_size_binary():
+    t0 = pa.binary(10)
+    data = [b'fooooooooo', None, b'barooooooo', b'quxooooooo']
+    a0 = pa.array(data, type=t0)
+
+    table = pa.Table.from_arrays([a0],
+                                 ['binary[10]'])
+    _check_roundtrip(table)
+
+
+def _check_roundtrip(table, expected=None, **params):
+    buf = io.BytesIO()
+    pq.write_table(table, buf, **params)
+    buf.seek(0)
+
+    if expected is None:
+        expected = table
+
+    result = pq.read_table(buf)
+    assert result.equals(expected)
+
+
 @parquet
 def test_multithreaded_read():
     df = alltypes_sample(size=10000)

From 1c6f3ef82a1c76919d707a5ef28d148b9d88c3bc Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 11 May 2017 16:01:13 -0400
Subject: [PATCH 0633/1644] =?UTF-8?q?ARROW-813:=20[Python]=20setup.py=20sd?=
 =?UTF-8?q?ist=20must=20also=20bundle=20dependent=20cmake=20m=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…odules

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #660 from xhochy/ARROW-813 and squashes the following commits:

47459ba [Uwe L. Korn] Call git later
5b3b4a4 [Uwe L. Korn] Symlink fixes
70ca0d8 [Uwe L. Korn] ARROW-813: [Python] setup.py sdist must also bundle dependent cmake modules
---
 appveyor.yml                                 | 4 +++-
 dev/release/run-rat.sh                       | 4 ++++
 python/CMakeLists.txt                        | 5 -----
 python/cmake_modules/BuildUtils.cmake        | 1 +
 python/cmake_modules/CompilerInfo.cmake      | 1 +
 python/cmake_modules/FindNumPy.cmake         | 1 +
 python/cmake_modules/FindPythonLibsNew.cmake | 1 +
 python/cmake_modules/SetupCxxFlags.cmake     | 1 +
 8 files changed, 12 insertions(+), 6 deletions(-)
 create mode 120000 python/cmake_modules/BuildUtils.cmake
 create mode 120000 python/cmake_modules/CompilerInfo.cmake
 create mode 120000 python/cmake_modules/FindNumPy.cmake
 create mode 120000 python/cmake_modules/FindPythonLibsNew.cmake
 create mode 120000 python/cmake_modules/SetupCxxFlags.cmake

diff --git a/appveyor.yml b/appveyor.yml
index f2954a92e9e19..802d43bad4ee8 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -32,4 +32,6 @@ init:
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
 
 build_script:
- - call ci\msvc-build.bat
+  - git config core.symlinks true
+  - git reset --hard
+  - call ci\msvc-build.bat
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index 3ff9ef083e523..f225c66268a61 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -34,7 +34,11 @@ $RAT $1 \
   -e status.h \
   -e asan_symbolize.py \
   -e cpplint.py \
+  -e BuildUtils.cmake \
   -e FindPythonLibsNew.cmake \
+  -e FindNumPy.cmake \
+  -e SetupCxxFlags.cmake \
+  -e CompilerInfo.cmake \
   -e pax_global_header \
   -e MANIFEST.in \
   -e __init__.pxd \
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 0d34bcdfa6e49..a9db4d877fb0c 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -23,9 +23,6 @@ project(pyarrow)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
-# Use common cmake modules from Arrow C++ if available
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../cpp/cmake_modules")
-
 include(CMakeParseArguments)
 
 # Compatibility with CMake 3.1
@@ -34,8 +31,6 @@ if(POLICY CMP0054)
   cmake_policy(SET CMP0054 NEW)
 endif()
 
-set(BUILD_SUPPORT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../cpp/build-support")
-
 # Allow "make install" to not depend on all targets.
 #
 # Must be declared in the top-level CMakeLists.txt.
diff --git a/python/cmake_modules/BuildUtils.cmake b/python/cmake_modules/BuildUtils.cmake
new file mode 120000
index 0000000000000..e3c98c8fc53e5
--- /dev/null
+++ b/python/cmake_modules/BuildUtils.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/BuildUtils.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/CompilerInfo.cmake b/python/cmake_modules/CompilerInfo.cmake
new file mode 120000
index 0000000000000..559ac3240c393
--- /dev/null
+++ b/python/cmake_modules/CompilerInfo.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/CompilerInfo.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/FindNumPy.cmake b/python/cmake_modules/FindNumPy.cmake
new file mode 120000
index 0000000000000..767b320ec7e45
--- /dev/null
+++ b/python/cmake_modules/FindNumPy.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/FindNumPy.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/FindPythonLibsNew.cmake b/python/cmake_modules/FindPythonLibsNew.cmake
new file mode 120000
index 0000000000000..7087123cb2376
--- /dev/null
+++ b/python/cmake_modules/FindPythonLibsNew.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/FindPythonLibsNew.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/SetupCxxFlags.cmake b/python/cmake_modules/SetupCxxFlags.cmake
new file mode 120000
index 0000000000000..3121a89926d2d
--- /dev/null
+++ b/python/cmake_modules/SetupCxxFlags.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/SetupCxxFlags.cmake
\ No newline at end of file

From a7722dc14527e7424179637cddc4d2d656562143 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 11 May 2017 16:02:03 -0400
Subject: [PATCH 0634/1644] ARROW-993: [GLib] Add missing error checks in Go
 examples

Author: Kouhei Sutou <kou@clear-code.com>

Closes #666 from kou/glib-example-go-add-missing-error-handling and squashes the following commits:

7baa8ba [Kouhei Sutou] [GLib] Add missing error checks in Go examples
---
 c_glib/example/go/write-batch.go  | 10 ++++++++--
 c_glib/example/go/write-stream.go |  9 ++++++++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/c_glib/example/go/write-batch.go b/c_glib/example/go/write-batch.go
index f5e946a0c5c23..78ef1beca4fc5 100644
--- a/c_glib/example/go/write-batch.go
+++ b/c_glib/example/go/write-batch.go
@@ -149,14 +149,20 @@ func main() {
 	}
 
 	recordBatch := arrow.NewRecordBatch(schema, 4, columns)
-	writer.WriteRecordBatch(recordBatch)
+	_, err = writer.WriteRecordBatch(recordBatch)
+	if err != nil {
+		log.Fatalf("Failed to write record batch #1: %v", err)
+	}
 
 	slicedColumns := make([]*arrow.Array, len(columns))
 	for i, column := range columns {
 		slicedColumns[i] = column.Slice(1, 3)
 	}
 	recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns)
-	writer.WriteRecordBatch(recordBatch)
+	_, err = writer.WriteRecordBatch(recordBatch)
+	if err != nil {
+		log.Fatalf("Failed to write record batch #2: %v", err)
+	}
 
 	writer.Close()
 }
diff --git a/c_glib/example/go/write-stream.go b/c_glib/example/go/write-stream.go
index 80ebd2ba1b01f..3f08270275763 100644
--- a/c_glib/example/go/write-stream.go
+++ b/c_glib/example/go/write-stream.go
@@ -149,7 +149,10 @@ func main() {
 	}
 
 	recordBatch := arrow.NewRecordBatch(schema, 4, columns)
-	writer.WriteRecordBatch(recordBatch)
+	_, err = writer.WriteRecordBatch(recordBatch)
+	if err != nil {
+		log.Fatalf("Failed to write record batch #1: %v", err)
+	}
 
 	slicedColumns := make([]*arrow.Array, len(columns))
 	for i, column := range columns {
@@ -157,6 +160,10 @@ func main() {
 	}
 	recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns)
 	writer.WriteRecordBatch(recordBatch)
+	_, err = writer.WriteRecordBatch(recordBatch)
+	if err != nil {
+		log.Fatalf("Failed to write record batch #2: %v", err)
+	}
 
 	writer.Close()
 }

From a8338f1f83b7ab53a87c316a8a74ba3dbf5f0b7f Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 11 May 2017 16:45:57 -0400
Subject: [PATCH 0635/1644] ARROW-995: [Website] Fix a typo

Author: Kouhei Sutou <kou@clear-code.com>

Closes #667 from kou/site-fix-a-typo and squashes the following commits:

dc66a18 [Kouhei Sutou] [Website] Fix a typo
---
 site/_posts/2017-05-08-0.3-release.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/_posts/2017-05-08-0.3-release.md b/site/_posts/2017-05-08-0.3-release.md
index 0779784ff12cb..5f80e62f73040 100644
--- a/site/_posts/2017-05-08-0.3-release.md
+++ b/site/_posts/2017-05-08-0.3-release.md
@@ -260,4 +260,4 @@ $ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
 [28]: http://arrow.apache.org/docs/cpp/classarrow_1_1_tensor.html
 [29]: http://pandas.pydata.org
 [30]: https://github.com/apache/arrow/tree/master/js
-[30]: https://conda-forge.github.io
+[31]: https://conda-forge.github.io

From 010bd224515b798e2e2429d3dfaa51ad8223bdbc Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Thu, 11 May 2017 16:49:33 -0400
Subject: [PATCH 0636/1644] ARROW-482 [Java] Exposing custom field metadata

I added the metadata to the `FieldType` object. It doesn't necessarily seem the right place for it, but it minimizes having to change method signatures to pass around the metadata. I also tried to standardize vector constructors on `FieldType` to ensure it's passed correctly.

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #611 from elahrvivaz/ARROW-482 and squashes the following commits:

63a76e0 [Emilio Lahr-Vivaz] ARROW-482 [Java] Exposing custom field metadata
---
 .../arrow/tools/ArrowFileTestFixtures.java    |  2 +-
 .../apache/arrow/tools/EchoServerTest.java    | 10 +-
 .../apache/arrow/tools/TestIntegration.java   |  4 +-
 .../main/codegen/templates/NullReader.java    |  3 +-
 .../templates/NullableValueVectors.java       |  4 +-
 .../main/codegen/templates/UnionVector.java   |  6 +-
 .../org/apache/arrow/vector/ZeroVector.java   |  3 +-
 .../vector/complex/FixedSizeListVector.java   | 44 +++++----
 .../arrow/vector/complex/ListVector.java      | 34 ++++---
 .../arrow/vector/complex/MapVector.java       | 36 +++++--
 .../vector/complex/NullableMapVector.java     | 36 ++++---
 .../vector/dictionary/DictionaryEncoder.java  |  6 +-
 .../apache/arrow/vector/file/ArrowReader.java |  5 +-
 .../apache/arrow/vector/file/ArrowWriter.java |  3 +-
 .../org/apache/arrow/vector/types/Types.java  |  7 +-
 .../apache/arrow/vector/types/pojo/Field.java | 83 +++++++++++-----
 .../arrow/vector/types/pojo/FieldType.java    | 15 ++-
 .../arrow/vector/TestFixedSizeListVector.java |  6 +-
 .../apache/arrow/vector/TestListVector.java   |  4 +-
 .../arrow/vector/TestVectorReAlloc.java       |  4 +-
 .../arrow/vector/TestVectorUnloadLoad.java    |  9 +-
 .../complex/impl/TestPromotableWriter.java    |  4 +-
 .../complex/writer/TestComplexWriter.java     | 43 ++++----
 .../arrow/vector/file/TestArrowFile.java      | 99 ++++++++++++++++---
 .../arrow/vector/file/TestArrowFooter.java    |  3 +-
 .../vector/file/TestArrowReaderWriter.java    |  3 +-
 .../arrow/vector/file/json/TestJSONFile.java  |  8 +-
 .../apache/arrow/vector/pojo/TestConvert.java | 33 ++++---
 .../vector/stream/MessageSerializerTest.java  | 10 +-
 .../arrow/vector/types/pojo/TestSchema.java   |  2 +-
 30 files changed, 355 insertions(+), 174 deletions(-)

diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
index 34c93ed232c80..6d9a6c1323e76 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), testInFile);
     }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 7cca33955d93a..d7f6388721ebb 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
 
     Field field = new Field(
-        "testField", true,
-        new ArrowType.Int(8, true),
+        "testField",
+        new FieldType(true, new ArrowType.Int(8, true), null, null),
         Collections.<Field>emptyList());
     NullableTinyIntVector vector =
         new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
         NullableIntVector writeVector =
             new NullableIntVector(
                 "varchar",
-                new FieldType(true, MinorType.INT.getType(), writeEncoding),
+                new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
                 allocator);
         NullableVarCharVector writeDictionaryVector =
             new NullableVarCharVector(
@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
          NullableVarCharVector writeDictionaryVector =
              new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
-         ListVector writeVector = new ListVector("list", allocator, null, null)) {
+         ListVector writeVector = ListVector.empty("list", allocator)) {
 
       // data being written:
       // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
       writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
       writeDictionaryVector.getMutator().setValueCount(2);
 
-      writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
+      writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
       writeVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(writeVector);
       listWriter.startList();
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
index 7d9a41985bbe3..4b840df670fdc 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
       MapWriter rootWriter = writer.rootAsMap();
       Float8Writer floatWriter = rootWriter.float8("float");
@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
       MapWriter rootWriter = writer.rootAsMap();
diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java
index 7c75b3ae1df9d..284a8d38f23e0 100644
--- a/java/vector/src/main/codegen/templates/NullReader.java
+++ b/java/vector/src/main/codegen/templates/NullReader.java
@@ -55,10 +55,9 @@ public MinorType getMinorType() {
     return type;
   }
 
-
   @Override
   public Field getField() {
-    return new Field("", true, new Null(), null);
+    return new Field("", FieldType.nullable(new Null()), null);
   }
 
   public void copyAsValue(MapWriter writer) {}
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 76d2bad36bc18..ed2418e4a2a97 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   private final int scale;
 
   public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
-    this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
+    this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
   }
   <#else>
   public ${className}(String name, BufferAllocator allocator) {
-    this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
+    this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
   }
   </#if>
 
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 9d5dee5d237e6..ec42b12e809ec 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
   public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
     this.name = name;
     this.allocator = allocator;
-    this.internalMap = new MapVector("internal", allocator, callBack);
+    this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
     this.typeVector = new UInt1Vector("types", allocator);
     this.callBack = callBack;
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
   }
 
   private FieldType fieldType(MinorType type) {
-    return new FieldType(true, type.getType(), null);
+    return FieldType.nullable(type.getType());
   }
 
   private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
@@ -250,7 +250,7 @@ public Field getField() {
       typeIds[childFields.size()] = v.getMinorType().ordinal();
       childFields.add(v.getField());
     }
-    return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
+    return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index e48788c6ae7c0..01e22f2574346 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -29,6 +29,7 @@
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.TransferPair;
 
@@ -98,7 +99,7 @@ public void clear() { }
 
   @Override
   public Field getField() {
-    return new Field(name, true, new Null(), null);
+    return new Field(name, FieldType.nullable(new Null()), null);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 0dceeed50d484..ee22fbec03597 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -26,6 +26,11 @@
 import java.util.List;
 import java.util.Objects;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
@@ -48,18 +53,17 @@
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {
 
+  public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
+    FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
+    return new FixedSizeListVector(name, allocator, fieldType, null);
+  }
+
   private FieldVector vector;
   private final BitVector bits;
   private final int listSize;
-  private final DictionaryEncoding dictionary;
+  private final FieldType fieldType;
   private final List<BufferBacked> innerVectors;
 
   private UnionFixedSizeListReader reader;
@@ -67,17 +71,26 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector,
   private Mutator mutator = new Mutator();
   private Accessor accessor = new Accessor();
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public FixedSizeListVector(String name,
                              BufferAllocator allocator,
                              int listSize,
                              DictionaryEncoding dictionary,
                              CallBack schemaChangeCallback) {
+    this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
+  }
+
+  public FixedSizeListVector(String name,
+                             BufferAllocator allocator,
+                             FieldType fieldType,
+                             CallBack schemaChangeCallback) {
     super(name, allocator);
-    Preconditions.checkArgument(listSize > 0, "list size must be positive");
     this.bits = new BitVector("$bits$", allocator);
     this.vector = ZeroVector.INSTANCE;
-    this.listSize = listSize;
-    this.dictionary = dictionary;
+    this.fieldType = fieldType;
+    this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
+    Preconditions.checkArgument(listSize > 0, "list size must be positive");
     this.innerVectors = Collections.singletonList((BufferBacked) bits);
     this.reader = new UnionFixedSizeListReader(this);
   }
@@ -85,7 +98,7 @@ public FixedSizeListVector(String name,
   @Override
   public Field getField() {
     List<Field> children = ImmutableList.of(getDataVector().getField());
-    return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
+    return new Field(name, fieldType, children);
   }
 
   @Override
@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
       throw new IllegalArgumentException("Lists have only one child. Found: " + children);
     }
     Field field = children.get(0);
-    FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
-    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
+    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
     if (!addOrGetVector.isCreated()) {
       throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
     }
@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
     TransferPair pairs[] = new TransferPair[2];
 
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
-      this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
+      this(new FixedSizeListVector(name, allocator, fieldType, callBack));
     }
 
     public TransferImpl(FixedSizeListVector to) {
       this.to = to;
-      Field field = vector.getField();
-      FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
-      to.addOrGetVector(type);
+      to.addOrGetVector(vector.getField().getFieldType());
       pairs[0] = bits.makeTransferPair(to.bits);
       pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 9392afbccdaa8..c2f55767ba6f3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -18,12 +18,17 @@
  ******************************************************************************/
 package org.apache.arrow.vector.complex;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Collections.singletonList;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
@@ -41,6 +46,7 @@
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -48,13 +54,12 @@
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {
 
+  public static ListVector empty(String name, BufferAllocator allocator) {
+    return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
+  }
+
   final UInt4Vector offsets;
   final BitVector bits;
   private final List<BufferBacked> innerVectors;
@@ -62,19 +67,27 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
   private Accessor accessor = new Accessor();
   private UnionListReader reader;
   private CallBack callBack;
-  private final DictionaryEncoding dictionary;
+  private final FieldType fieldType;
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
-    this(name, allocator, null, callBack);
+    this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
   }
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
+    this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
+  }
+
+  public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
     super(name, allocator, callBack);
     this.bits = new BitVector("$bits$", allocator);
     this.offsets = getOffsetVector();
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
     this.reader = new UnionListReader(this);
-    this.dictionary = dictionary;
+    this.fieldType = checkNotNull(fieldType);
     this.callBack = callBack;
   }
 
@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
     TransferPair pairs[] = new TransferPair[3];
 
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
-      this(new ListVector(name, allocator, dictionary, callBack));
+      this(new ListVector(name, allocator, fieldType, callBack));
     }
 
     public TransferImpl(ListVector to) {
@@ -266,8 +279,7 @@ public int getBufferSize() {
 
   @Override
   public Field getField() {
-    return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
-            ImmutableList.of(getDataVector().getField()));
+    return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 997a6a38a080a..bdd30f88f2cc2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -17,6 +17,8 @@
  */
 package org.apache.arrow.vector.complex;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
@@ -25,6 +27,10 @@
 
 import javax.annotation.Nullable;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.FieldVector;
@@ -33,26 +39,36 @@
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
-
 public class MapVector extends AbstractMapVector {
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
+  public static MapVector empty(String name, BufferAllocator allocator) {
+    FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
+    return new MapVector(name, allocator, fieldType, null);
+  }
+
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
+  protected final FieldType fieldType;
   public int valueCount;
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
+    this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
+  }
+
+  public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
     super(name, allocator, callBack);
+    this.fieldType = checkNotNull(fieldType);
   }
 
   @Override
@@ -119,7 +135,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return new MapTransferPair(this, new MapVector(name, allocator, callBack), false);
+    return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false);
   }
 
   @Override
@@ -129,7 +145,7 @@ public TransferPair makeTransferPair(ValueVector to) {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new MapTransferPair(this, new MapVector(ref, allocator, callBack), false);
+    return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false);
   }
 
   protected static class MapTransferPair implements TransferPair{
@@ -212,8 +228,8 @@ public int getValueCapacity() {
       @Override
       public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
         return Ints.compare(
-            Preconditions.checkNotNull(left).getValueCapacity(),
-            Preconditions.checkNotNull(right).getValueCapacity()
+            checkNotNull(left).getValueCapacity(),
+            checkNotNull(right).getValueCapacity()
         );
       }
     };
@@ -294,7 +310,7 @@ public Field getField() {
     for (ValueVector child : getChildren()) {
       children.add(child.getField());
     }
-    return new Field(name, false, Struct.INSTANCE, children);
+    return new Field(name, fieldType, children);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 6456efba0dcb4..f1f86404d20e8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -23,6 +23,9 @@
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.BaseDataValueVector;
 import org.apache.arrow.vector.BitVector;
@@ -32,39 +35,48 @@
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
 import org.apache.arrow.vector.complex.impl.NullableMapWriter;
-import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public class NullableMapVector extends MapVector implements FieldVector {
 
+  public static NullableMapVector empty(String name, BufferAllocator allocator) {
+    FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
+    return new NullableMapVector(name, allocator, fieldType, null);
+  }
+
   private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
   private final NullableMapWriter writer = new NullableMapWriter(this);
 
   protected final BitVector bits;
 
   private final List<BufferBacked> innerVectors;
-  private final DictionaryEncoding dictionary;
 
   private final Accessor accessor;
   private final Mutator mutator;
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
-    this(name, allocator, null, callBack);
+    this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack);
   }
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
-    super(name, checkNotNull(allocator), callBack);
+    this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack);
+  }
+
+  public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+    super(name, checkNotNull(allocator), fieldType, callBack);
     this.bits = new BitVector("$bits$", allocator);
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits));
-    this.dictionary = dictionary;
     this.accessor = new Accessor();
     this.mutator = new Mutator();
   }
@@ -96,7 +108,7 @@ public NullableMapWriter getWriter() {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, dictionary, null), false);
+    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false);
   }
 
   @Override
@@ -106,12 +118,12 @@ public TransferPair makeTransferPair(ValueVector to) {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, null), false);
+    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, callBack), false);
+    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false);
   }
 
   protected class NullableMapTransferPair extends MapTransferPair {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
index 0666bc4137a9d..0f49ce61f1cdf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -29,6 +29,7 @@
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.TransferPair;
 
 public class DictionaryEncoder {
@@ -53,8 +54,9 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
     }
 
     Field valueField = vector.getField();
-    Field indexField = new Field(valueField.getName(), valueField.isNullable(),
-      dictionary.getEncoding().getIndexType(), dictionary.getEncoding(), null);
+    FieldType indexFieldType = new FieldType(valueField.isNullable(), dictionary.getEncoding().getIndexType(),
+      dictionary.getEncoding(), valueField.getMetadata());
+    Field indexField = new Field(valueField.getName(), indexFieldType, null);
 
     // vector to hold our indices (dictionary encoded values)
     FieldVector indices = indexField.createVector(vector.getAllocator());
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index 1d33913f71a95..f00088577ffa9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -40,6 +40,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 
 public abstract class ArrowReader<T extends ReadChannel> implements DictionaryProvider, AutoCloseable {
@@ -192,13 +193,13 @@ private Field toMemoryFormat(Field field, Map<Long, Dictionary> dictionaries) {
       // get existing or create dictionary vector
       if (!dictionaries.containsKey(encoding.getId())) {
         // create a new dictionary vector for the values
-        Field dictionaryField = new Field(field.getName(), field.isNullable(), field.getType(), null, children);
+        Field dictionaryField = new Field(field.getName(), new FieldType(field.isNullable(), field.getType(), null, null), children);
         FieldVector dictionaryVector = dictionaryField.createVector(allocator);
         dictionaries.put(encoding.getId(), new Dictionary(dictionaryVector, encoding));
       }
     }
 
-    return new Field(field.getName(), field.isNullable(), type, encoding, updatedChildren);
+    return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren);
   }
 
   private void load(ArrowDictionaryBatch dictionaryBatch) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 1716287f722ff..35b44fb682fb7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -36,6 +36,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -117,7 +118,7 @@ private Field toMessageFormat(Field field, DictionaryProvider provider, Map<Long
       }
     }
 
-    return new Field(field.getName(), field.isNullable(), type, encoding, updatedChildren);
+    return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren);
   }
 
   public void start() throws IOException {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index d5076d82c2a4d..2a0e47b7bf7c2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -122,7 +122,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     MAP(Struct.INSTANCE) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableMapVector(name, allocator, fieldType.getDictionary(), schemaChangeCallback);
+        return new NullableMapVector(name, allocator, fieldType, schemaChangeCallback);
       }
 
       @Override
@@ -430,7 +430,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     LIST(List.INSTANCE) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new ListVector(name, allocator, fieldType.getDictionary(), schemaChangeCallback);
+        return new ListVector(name, allocator, fieldType, schemaChangeCallback);
       }
 
       @Override
@@ -446,8 +446,7 @@ public ArrowType getType() {
 
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        int size = ((FixedSizeList)fieldType.getType()).getListSize();
-        return new FixedSizeListVector(name, allocator, size, fieldType.getDictionary(), schemaChangeCallback);
+        return new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback);
       }
 
       @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 05eb9cdceac23..a8f2ae5fbab8c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -21,15 +21,12 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
 
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.schema.TypeLayout;
-import org.apache.arrow.vector.schema.VectorLayout;
-import org.apache.arrow.vector.types.pojo.ArrowType.Int;
-
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -37,8 +34,16 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.flatbuffers.FlatBufferBuilder;
 
+import org.apache.arrow.flatbuf.KeyValue;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.schema.TypeLayout;
+import org.apache.arrow.vector.schema.VectorLayout;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+
 public class Field {
 
   public static Field nullablePrimitive(String name, ArrowType.PrimitiveType type) {
@@ -46,7 +51,7 @@ public static Field nullablePrimitive(String name, ArrowType.PrimitiveType type)
   }
 
   public static Field nullable(String name, ArrowType type) {
-    return new Field(name, true, type, null, null);
+    return new Field(name, FieldType.nullable(type), null);
   }
 
   private final String name;
@@ -61,28 +66,32 @@ private Field(
       @JsonProperty("type") ArrowType type,
       @JsonProperty("dictionary") DictionaryEncoding dictionary,
       @JsonProperty("children") List<Field> children,
-      @JsonProperty("typeLayout") TypeLayout typeLayout) {
-    this(name, new FieldType(nullable, type, dictionary), children, typeLayout);
+      @JsonProperty("typeLayout") TypeLayout typeLayout,
+      @JsonProperty("metadata") Map<String, String> metadata) {
+    this(name, new FieldType(nullable, type, dictionary, metadata), children, typeLayout);
   }
 
   private Field(String name, FieldType fieldType, List<Field> children, TypeLayout typeLayout) {
-    super();
     this.name = name;
     this.fieldType = checkNotNull(fieldType);
-    this.children = children == null ? ImmutableList.<Field>of() : children;
+    this.children = children == null ? ImmutableList.<Field>of() : ImmutableList.copyOf(children);
     this.typeLayout = checkNotNull(typeLayout);
   }
 
-  public Field(String name, FieldType fieldType, List<Field> children) {
-    this(name, fieldType, children, TypeLayout.getTypeLayout(fieldType.getType()));
-  }
-
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
-    this(name, nullable, type, null, children);
+    this(name, new FieldType(nullable, type, null, null), children);
   }
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public Field(String name, boolean nullable, ArrowType type, DictionaryEncoding dictionary, List<Field> children) {
-    this(name, new FieldType(nullable, type, dictionary), children);
+    this(name, new FieldType(nullable, type, dictionary, null), children);
+  }
+
+  public Field(String name, FieldType fieldType, List<Field> children) {
+    this(name, fieldType, children, fieldType == null ? null : TypeLayout.getTypeLayout(fieldType.getType()));
   }
 
   public FieldVector createVector(BufferAllocator allocator) {
@@ -114,7 +123,14 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
       childrenBuilder.add(convertField(field.children(i)));
     }
     List<Field> children = childrenBuilder.build();
-    return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()));
+    ImmutableMap.Builder<String, String> metadataBuilder = ImmutableMap.builder();
+    for (int i = 0; i < field.customMetadataLength(); i++) {
+      KeyValue kv = field.customMetadata(i);
+      String key = kv.key(), value = kv.value();
+      metadataBuilder.put(key == null ? "" : key, value == null ? "" : value);
+    }
+    Map<String, String> metadata = metadataBuilder.build();
+    return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()), metadata);
   }
 
   public void validate() {
@@ -147,7 +163,19 @@ public int getField(FlatBufferBuilder builder) {
       VectorLayout vectorLayout = typeLayout.getVectors().get(i);
       buffersData[i] = vectorLayout.writeTo(builder);
     }
-    int layoutOffset =  org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData);
+    int layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData);
+    int[] metadataOffsets = new int[getMetadata().size()];
+    Iterator<Entry<String, String>> metadataIterator = getMetadata().entrySet().iterator();
+    for (int i = 0; i < metadataOffsets.length; i ++) {
+      Entry<String, String> kv = metadataIterator.next();
+      int keyOffset = builder.createString(kv.getKey());
+      int valueOffset = builder.createString(kv.getValue());
+      KeyValue.startKeyValue(builder);
+      KeyValue.addKey(builder, keyOffset);
+      KeyValue.addValue(builder, valueOffset);
+      metadataOffsets[i] = KeyValue.endKeyValue(builder);
+    }
+    int metadataOffset = org.apache.arrow.flatbuf.Field.createCustomMetadataVector(builder, metadataOffsets);
     org.apache.arrow.flatbuf.Field.startField(builder);
     if (name != null) {
       org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
@@ -157,6 +185,7 @@ public int getField(FlatBufferBuilder builder) {
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
     org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
     org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
+    org.apache.arrow.flatbuf.Field.addCustomMetadata(builder, metadataOffset);
     if (dictionary != null) {
       org.apache.arrow.flatbuf.Field.addDictionary(builder, dictionaryOffset);
     }
@@ -193,9 +222,14 @@ public TypeLayout getTypeLayout() {
     return typeLayout;
   }
 
+  @JsonInclude(Include.NON_EMPTY)
+  public Map<String, String> getMetadata() {
+    return fieldType.getMetadata();
+  }
+
   @Override
   public int hashCode() {
-    return Objects.hash(name, isNullable(), getType(), getDictionary(), children);
+    return Objects.hash(name, isNullable(), getType(), getDictionary(), getMetadata(), children);
   }
 
   @Override
@@ -205,10 +239,11 @@ public boolean equals(Object obj) {
     }
     Field that = (Field) obj;
     return Objects.equals(this.name, that.name) &&
-            Objects.equals(this.isNullable(), that.isNullable()) &&
-            Objects.equals(this.getType(), that.getType()) &&
-           Objects.equals(this.getDictionary(), that.getDictionary()) &&
-            Objects.equals(this.children, that.children);
+             Objects.equals(this.isNullable(), that.isNullable()) &&
+             Objects.equals(this.getType(), that.getType()) &&
+             Objects.equals(this.getDictionary(), that.getDictionary()) &&
+             Objects.equals(this.getMetadata(), that.getMetadata()) &&
+             Objects.equals(this.children, that.children);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
index fe99e631360cc..c8fc689cd2c9c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
@@ -19,6 +19,10 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.types.Types;
@@ -28,18 +32,24 @@
 public class FieldType {
 
   public static FieldType nullable(ArrowType type) {
-    return new FieldType(true, type, null);
+    return new FieldType(true, type, null, null);
   }
 
   private final boolean nullable;
   private final ArrowType type;
   private final DictionaryEncoding dictionary;
+  private final Map<String, String> metadata;
 
   public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary) {
+    this(nullable, type, dictionary, null);
+  }
+
+  public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary, Map<String, String> metadata) {
     super();
     this.nullable = nullable;
     this.type = checkNotNull(type);
     this.dictionary = dictionary;
+    this.metadata = metadata == null ? ImmutableMap.<String, String>of() : ImmutableMap.copyOf(metadata);
   }
 
   public boolean isNullable() {
@@ -51,6 +61,9 @@ public ArrowType getType() {
   public DictionaryEncoding getDictionary() {
     return dictionary;
   }
+  public Map<String, String> getMetadata() {
+    return metadata;
+  }
 
   public FieldVector createNewSingleVector(String name, BufferAllocator allocator, CallBack schemaCallBack) {
     MinorType minorType = Types.getMinorTypeForArrowType(type);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index cfb7b3d2a26ac..f35ab54eaa88d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -49,7 +49,7 @@ public void terminate() throws Exception {
 
   @Test
   public void testIntType() {
-    try (FixedSizeListVector vector = new FixedSizeListVector("list", allocator, 2, null, null)) {
+    try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) {
       NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
       NullableIntVector.Mutator mutator = nested.getMutator();
       vector.allocateNew();
@@ -77,7 +77,7 @@ public void testIntType() {
 
   @Test
   public void testFloatTypeNullable() {
-    try (FixedSizeListVector vector = new FixedSizeListVector("list", allocator, 2, null, null)) {
+    try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) {
       NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
       NullableFloat4Vector.Mutator mutator = nested.getMutator();
       vector.allocateNew();
@@ -112,7 +112,7 @@ public void testFloatTypeNullable() {
 
   @Test
   public void testNestedInList() {
-    try (ListVector vector = new ListVector("list", allocator, null, null)) {
+    try (ListVector vector = ListVector.empty("list", allocator)) {
       ListVector.Mutator mutator = vector.getMutator();
       FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector();
       FixedSizeListVector.Mutator tupleMutator = tuples.getMutator();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 18d93b6401e39..11be3298f7533 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -42,8 +42,8 @@ public void terminate() throws Exception {
 
   @Test
   public void testCopyFrom() throws Exception {
-    try (ListVector inVector = new ListVector("input", allocator, null, null);
-         ListVector outVector = new ListVector("output", allocator, null, null)) {
+    try (ListVector inVector = ListVector.empty("input", allocator);
+         ListVector outVector = ListVector.empty("output", allocator)) {
       UnionListWriter writer = inVector.getWriter();
       writer.allocate();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index 40c7bc5ac9add..da9cb00361c0b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -123,7 +123,7 @@ public void testNullableType() {
 
   @Test
   public void testListType() {
-    try (final ListVector vector = new ListVector("", allocator, null)) {
+    try (final ListVector vector = ListVector.empty("", allocator)) {
       vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
 
       vector.setInitialCapacity(512);
@@ -146,7 +146,7 @@ public void testListType() {
 
   @Test
   public void testMapType() {
-    try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+    try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) {
       vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
 
       vector.setInitialCapacity(512);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 372bcf0da6e9a..6fb559ca0d728 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -42,6 +42,7 @@
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -58,7 +59,7 @@ public void testUnloadLoad() throws IOException {
 
     try (
         BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", originalVectorsAllocator)) {
 
       // write some data
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
@@ -106,7 +107,7 @@ public void testUnloadLoadAddPadding() throws IOException {
     Schema schema;
     try (
         BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", originalVectorsAllocator)) {
 
       // write some data
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
@@ -182,8 +183,8 @@ public void testUnloadLoadAddPadding() throws IOException {
   @Test
   public void testLoadEmptyValidityBuffer() throws IOException {
     Schema schema = new Schema(asList(
-        new Field("intDefined", true, new ArrowType.Int(32, true), Collections.<Field>emptyList()),
-        new Field("intNull", true, new ArrowType.Int(32, true), Collections.<Field>emptyList())
+        new Field("intDefined", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList()),
+        new Field("intNull", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList())
                                      ));
     int count = 10;
     ArrowBuf validity = allocator.buffer(10).slice(0, 0);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 65b193c0aee4c..e826fa53b0977 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -27,8 +27,10 @@
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -51,7 +53,7 @@ public void terminate() throws Exception {
   @Test
   public void testPromoteToUnion() throws Exception {
 
-    try (final MapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
+    try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator);
          final NullableMapVector v = container.addOrGetMap("test");
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index aba65dbf374d4..16139361756b8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -17,16 +17,13 @@
  */
 package org.apache.arrow.vector.complex.writer;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.SchemaChangeCallBack;
@@ -48,9 +45,11 @@
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.JsonStringHashMap;
@@ -60,8 +59,6 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import io.netty.buffer.ArrowBuf;
-
 public class TestComplexWriter {
 
   private static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
@@ -101,7 +98,7 @@ public void transferPairSchemaChange() {
   }
 
   private MapVector populateMapVector(CallBack callBack) {
-    MapVector parent = new MapVector("parent", allocator, callBack);
+    MapVector parent = new MapVector("parent", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
     IntWriter intWriter = rootWriter.integer("int");
@@ -118,7 +115,7 @@ private MapVector populateMapVector(CallBack callBack) {
 
   @Test
   public void nullableMap() {
-    try (MapVector mapVector = new MapVector("parent", allocator, null)) {
+    try (MapVector mapVector = MapVector.empty("parent", allocator)) {
       ComplexWriter writer = new ComplexWriterImpl("root", mapVector);
       MapWriter rootWriter = writer.rootAsMap();
       for (int i = 0; i < COUNT; i++) {
@@ -142,7 +139,7 @@ public void nullableMap() {
    */
   @Test
   public void nullableMap2() {
-    try (MapVector mapVector = new MapVector("parent", allocator, null)) {
+    try (MapVector mapVector = MapVector.empty("parent", allocator)) {
       ComplexWriter writer = new ComplexWriterImpl("root", mapVector);
       MapWriter rootWriter = writer.rootAsMap();
       MapWriter mapWriter = rootWriter.map("map");
@@ -181,7 +178,7 @@ private void checkNullableMap(MapVector mapVector) {
 
   @Test
   public void testList() {
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
 
@@ -210,7 +207,7 @@ public void testList() {
 
   @Test
   public void listScalarType() {
-    ListVector listVector = new ListVector("list", allocator, null, null);
+    ListVector listVector = ListVector.empty("list", allocator);
     listVector.allocateNew();
     UnionListWriter listWriter = new UnionListWriter(listVector);
     for (int i = 0; i < COUNT; i++) {
@@ -233,7 +230,7 @@ public void listScalarType() {
 
   @Test
   public void listScalarTypeNullable() {
-    ListVector listVector = new ListVector("list", allocator, null, null);
+    ListVector listVector = ListVector.empty("list", allocator);
     listVector.allocateNew();
     UnionListWriter listWriter = new UnionListWriter(listVector);
     for (int i = 0; i < COUNT; i++) {
@@ -262,7 +259,7 @@ public void listScalarTypeNullable() {
 
   @Test
   public void listMapType() {
-    ListVector listVector = new ListVector("list", allocator, null, null);
+    ListVector listVector = ListVector.empty("list", allocator);
     listVector.allocateNew();
     UnionListWriter listWriter = new UnionListWriter(listVector);
     MapWriter mapWriter = listWriter.map();
@@ -290,7 +287,7 @@ public void listMapType() {
 
   @Test
   public void listListType() {
-    try (ListVector listVector = new ListVector("list", allocator, null, null)) {
+    try (ListVector listVector = ListVector.empty("list", allocator)) {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       for (int i = 0; i < COUNT; i++) {
@@ -315,7 +312,7 @@ public void listListType() {
    */
   @Test
   public void listListType2() {
-    try (ListVector listVector = new ListVector("list", allocator, null, null)) {
+    try (ListVector listVector = ListVector.empty("list", allocator)) {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       ListWriter innerListWriter = listWriter.list();
@@ -353,7 +350,7 @@ private void checkListOfLists(final ListVector listVector) {
 
   @Test
   public void unionListListType() {
-    try (ListVector listVector = new ListVector("list", allocator, null, null)) {
+    try (ListVector listVector = ListVector.empty("list", allocator)) {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       for (int i = 0; i < COUNT; i++) {
@@ -382,7 +379,7 @@ public void unionListListType() {
    */
   @Test
   public void unionListListType2() {
-    try (ListVector listVector = new ListVector("list", allocator, null, null)) {
+    try (ListVector listVector = ListVector.empty("list", allocator)) {
       listVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(listVector);
       ListWriter innerListWriter = listWriter.list();
@@ -454,7 +451,7 @@ public void simpleUnion() {
 
   @Test
   public void promotableWriter() {
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
     for (int i = 0; i < 100; i++) {
@@ -503,7 +500,7 @@ public void promotableWriter() {
    */
   @Test
   public void promotableWriterSchema() {
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
     rootWriter.bigInt("a");
@@ -536,7 +533,7 @@ private Set<String> getFieldNames(List<Field> fields) {
   @Test
   public void mapWriterMixedCaseFieldNames() {
     // test case-sensitive MapWriter
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("rootCaseSensitive", parent, false, true);
     MapWriter rootWriterCaseSensitive = writer.rootAsMap();
     rootWriterCaseSensitive.bigInt("int_field");
@@ -607,7 +604,7 @@ public void timeStampWriters() throws Exception {
     final LocalDateTime expectedNanoDateTime = expectedMilliDateTime;
 
     // write
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
 
@@ -678,7 +675,7 @@ public void timeStampWriters() throws Exception {
 
   @Test
   public void complexCopierWithList() {
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
     ListWriter listWriter = rootWriter.list("list");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 3bed45361fc20..9156110a863d8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -27,10 +27,13 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -55,7 +58,9 @@
 import org.apache.arrow.vector.stream.ArrowStreamReader;
 import org.apache.arrow.vector.stream.ArrowStreamWriter;
 import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
@@ -77,7 +82,7 @@ public void testWrite() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), file, new ByteArrayOutputStream());
     }
@@ -89,7 +94,7 @@ public void testWriteComplex() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeComplexData(count, parent);
       FieldVector root = parent.getChild("root");
       validateComplexContent(count, new VectorSchemaRoot(root));
@@ -105,7 +110,7 @@ public void testWriteRead() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), file, stream);
     }
@@ -172,7 +177,7 @@ public void testWriteReadComplex() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
       writeComplexData(count, parent);
       write(parent.getChild("root"), file, stream);
     }
@@ -213,7 +218,7 @@ public void testWriteReadMultipleRBs() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = new MapVector("parent", originalVectorAllocator, null);
+         MapVector parent = MapVector.empty("parent", originalVectorAllocator);
          FileOutputStream fileOutputStream = new FileOutputStream(file)){
       writeData(counts[0], parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
@@ -286,7 +291,7 @@ public void testWriteReadUnion() throws IOException {
 
     // write
     try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+         NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeUnionData(count, parent);
       validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
       write(parent.getChild("root"), file, stream);
@@ -380,6 +385,77 @@ private void validateTinyData(VectorSchemaRoot root) {
     }
   }
 
+  @Test
+  public void testWriteReadFieldMetadata() throws IOException {
+    File file = new File("target/mytest_metadata.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+
+    List<Field> childFields = new ArrayList<Field>();
+    childFields.add(new Field("varchar-child", new FieldType(true, ArrowType.Utf8.INSTANCE, null, metadata(1)), null));
+    childFields.add(new Field("float-child", new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null, metadata(2)), null));
+    childFields.add(new Field("int-child", new FieldType(false, new ArrowType.Int(32, true), null, metadata(3)), null));
+    childFields.add(new Field("list-child", new FieldType(true, ArrowType.List.INSTANCE, null, metadata(4)),
+                              ImmutableList.of(new Field("l1", FieldType.nullable(new ArrowType.Int(16 ,true)), null))));
+    Field field = new Field("meta", new FieldType(true, ArrowType.Struct.INSTANCE, null, metadata(0)), childFields);
+    List<Field> fields = ImmutableList.of(field);
+
+    // write
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) {
+      vector.allocateNewSafe();
+      vector.getMutator().setValueCount(0);
+
+      List<FieldVector> vectors = ImmutableList.<FieldVector>of(vector);
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 0);
+
+      try (FileOutputStream fileOutputStream = new FileOutputStream(file);
+           ArrowFileWriter fileWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel());
+           ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, null, stream)) {
+        LOGGER.debug("writing schema: " + root.getSchema());
+        fileWriter.start();
+        streamWriter.start();
+        fileWriter.writeBatch();
+        streamWriter.writeBatch();
+        fileWriter.end();
+        streamWriter.end();
+      }
+    }
+
+    // read from file
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      Assert.assertEquals(fields, schema.getFields());
+      Field top = schema.getFields().get(0);
+      Assert.assertEquals(metadata(0), top.getMetadata());
+      for (int i = 0; i < 4; i ++) {
+        Assert.assertEquals(metadata(i + 1), top.getChildren().get(i).getMetadata());
+      }
+    }
+
+    // Read from stream
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      Assert.assertEquals(fields, schema.getFields());
+      Field top = schema.getFields().get(0);
+      Assert.assertEquals(metadata(0), top.getMetadata());
+      for (int i = 0; i < 4; i ++) {
+        Assert.assertEquals(metadata(i + 1), top.getChildren().get(i).getMetadata());
+      }
+    }
+  }
+
+  private Map<String, String> metadata(int i) {
+    return ImmutableMap.of("k_" + i, "v_" + i, "k2_" + i, "v2_" + i);
+  }
+
   @Test
   public void testWriteReadDictionary() throws IOException {
     File file = new File("target/mytest_dict.arrow");
@@ -491,7 +567,7 @@ public void testWriteReadNestedDictionary() throws IOException {
 
     // write
     try (NullableVarCharVector dictionaryVector = newNullableVarCharVector("dictionary", allocator);
-         ListVector listVector = new ListVector("list", allocator, null, null)) {
+         ListVector listVector = ListVector.empty("list", allocator)) {
 
       Dictionary dictionary = new Dictionary(dictionaryVector, encoding);
       MapDictionaryProvider provider = new MapDictionaryProvider();
@@ -590,10 +666,10 @@ public void testWriteReadFixedSizeList() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = new NullableMapVector("parent", originalVectorAllocator, null, null)) {
-      FixedSizeListVector tuples = parent.addOrGet("float-pairs", new FieldType(true, new FixedSizeList(2), null), FixedSizeListVector.class);
-      NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(new FieldType(true, MinorType.FLOAT4.getType(), null)).getVector();
-      NullableIntVector ints = parent.addOrGet("ints", new FieldType(true, new Int(32, true), null), NullableIntVector.class);
+         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+      FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class);
+      NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
+      NullableIntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), NullableIntVector.class);
       parent.allocateNew();
 
       for (int i = 0; i < 10; i++) {
@@ -641,6 +717,7 @@ public void testWriteReadFixedSizeList() throws IOException {
     }
   }
 
+
   /**
    * Writes the contents of parents to file. If outStream is non-null, also writes it
    * to outStream in the streaming serialized format.
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
index 1e514585e502f..3014e64b4eea4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
@@ -28,6 +28,7 @@
 import org.apache.arrow.flatbuf.Footer;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Test;
 
@@ -38,7 +39,7 @@ public class TestArrowFooter {
   @Test
   public void test() {
     Schema schema = new Schema(asList(
-        new Field("a", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())
+        new Field("a", FieldType.nullable(new ArrowType.Int(8, true)), Collections.<Field>emptyList())
         ));
     ArrowFooter footer = new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), Collections.<ArrowBlock>emptyList());
     ArrowFooter newFooter = roundTrip(footer);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index d00cb0f8c0065..55629d5107c86 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -41,6 +41,7 @@
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
 import org.junit.Before;
@@ -71,7 +72,7 @@ byte[] array(ArrowBuf buf) {
 
   @Test
   public void test() throws IOException {
-    Schema schema = new Schema(asList(new Field("testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
+    Schema schema = new Schema(asList(new Field("testField", FieldType.nullable(new ArrowType.Int(8, true)), Collections.<Field>emptyList())));
     ArrowType type = schema.getFields().get(0).getType();
     FieldVector vector = TestUtils.newVector(FieldVector.class, "testField", type, allocator);
     vector.initializeChildrenFromFields(schema.getFields().get(0).getChildren());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 6369c07c3205c..6c29cab1a0723 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -43,7 +43,7 @@ public void testWriteReadComplexJSON() throws IOException {
     // write
     try (
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
       writeComplexData(count, parent);
       writeJSON(file, new VectorSchemaRoot(parent.getChild("root")));
     }
@@ -70,7 +70,7 @@ public void testWriteComplexJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeComplexData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateComplexContent(root.getRowCount(), root);
@@ -92,7 +92,7 @@ public void testWriteReadUnionJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
 
       writeUnionData(count, parent);
 
@@ -127,7 +127,7 @@ public void testWriteReadDateTimeJSON() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
 
       writeDateTimeData(count, parent);
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index f9c8f726ab6c6..64f797030ffb4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -32,6 +32,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Test;
 
@@ -45,25 +46,25 @@ public class TestConvert {
 
   @Test
   public void simple() {
-    Field initialField = new Field("a", true, new Int(32, true), null);
+    Field initialField = new Field("a", FieldType.nullable(new Int(32, true)), null);
     run(initialField);
   }
 
   @Test
   public void complex() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
-    childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
-    childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
+    childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+    childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
 
-    Field initialField = new Field("a", true, Struct.INSTANCE, childrenBuilder.build());
+    Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), childrenBuilder.build());
     run(initialField);
   }
 
   @Test
   public void schema() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
-    childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
-    childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
+    childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+    childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
     Schema initialSchema = new Schema(childrenBuilder.build());
     run(initialSchema);
   }
@@ -71,18 +72,18 @@ public void schema() {
   @Test
   public void nestedSchema() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
-    childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
-    childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
-    childrenBuilder.add(new Field("child3", true, new Struct(), ImmutableList.<Field>of(
-        new Field("child3.1", true, Utf8.INSTANCE, null),
-        new Field("child3.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
+    childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+    childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
+    childrenBuilder.add(new Field("child3", FieldType.nullable(new Struct()), ImmutableList.<Field>of(
+        new Field("child3.1", FieldType.nullable(Utf8.INSTANCE), null),
+        new Field("child3.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of())
         )));
-    childrenBuilder.add(new Field("child4", true, new List(), ImmutableList.<Field>of(
-        new Field("child4.1", true, Utf8.INSTANCE, null)
+    childrenBuilder.add(new Field("child4", FieldType.nullable(new List()), ImmutableList.<Field>of(
+        new Field("child4.1", FieldType.nullable(Utf8.INSTANCE), null)
         )));
-    childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
-        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, null), null),
-        new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
+    childrenBuilder.add(new Field("child5", FieldType.nullable(new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } )), ImmutableList.<Field>of(
+        new Field("child5.1", FieldType.nullable(new Timestamp(TimeUnit.MILLISECOND, null)), null),
+        new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of())
         )));
     Schema initialSchema = new Schema(childrenBuilder.build());
     run(initialSchema);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
index 27879efeaf117..9678423c0fbbe 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
@@ -29,6 +29,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.file.ArrowBlock;
@@ -40,13 +41,12 @@
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
-import org.junit.Test;
 import org.junit.Rule;
+import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
-import io.netty.buffer.ArrowBuf;
-
 public class MessageSerializerTest {
 
   public static ArrowBuf buf(BufferAllocator alloc, byte[] bytes) {
@@ -79,7 +79,7 @@ public void testSchemaMessageSerialization() throws IOException {
   @Test
   public void testSchemaDictionaryMessageSerialization() throws IOException {
     DictionaryEncoding dictionary = new DictionaryEncoding(9L, false, new ArrowType.Int(8, true));
-    Field field = new Field("test", true, ArrowType.Utf8.INSTANCE, dictionary, null);
+    Field field = new Field("test", new FieldType(true, ArrowType.Utf8.INSTANCE, dictionary, null), null);
     Schema schema = new Schema(Collections.singletonList(field));
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     long size = MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), schema);
@@ -130,7 +130,7 @@ public void testSerializeRecordBatch() throws IOException {
 
   public static Schema testSchema() {
     return new Schema(asList(new Field(
-        "testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
+        "testField", FieldType.nullable(new ArrowType.Int(8, true)), Collections.<Field>emptyList())));
   }
 
   // Verifies batch contents matching test schema.
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 56fa73eccebf0..84cc10787f7b0 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -47,7 +47,7 @@
 public class TestSchema {
 
   private static Field field(String name, boolean nullable, ArrowType type, Field... children) {
-    return new Field(name, nullable, type, asList(children));
+    return new Field(name, new FieldType(nullable, type, null, null), asList(children));
   }
 
   private static Field field(String name, ArrowType type, Field... children) {

From b0666609a2fc2e6f7bb3598972478469df0911d6 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 11 May 2017 16:51:13 -0400
Subject: [PATCH 0637/1644] ARROW-996: [Website] Add 0.3.0 release announce in
 Japanese

Author: Kouhei Sutou <kou@clear-code.com>

Closes #668 from kou/website-0.3-release-announce-japanese and squashes the following commits:

cc7871f [Kouhei Sutou] [Website] Add 0.3.0 release announce in Japanese
---
 .../_posts/2017-05-08-0.3-release-japanese.md | 189 ++++++++++++++++++
 site/_posts/2017-05-08-0.3-release.md         |   2 +
 2 files changed, 191 insertions(+)
 create mode 100644 site/_posts/2017-05-08-0.3-release-japanese.md

diff --git a/site/_posts/2017-05-08-0.3-release-japanese.md b/site/_posts/2017-05-08-0.3-release-japanese.md
new file mode 100644
index 0000000000000..27e0166b5fc7d
--- /dev/null
+++ b/site/_posts/2017-05-08-0.3-release-japanese.md
@@ -0,0 +1,189 @@
+---
+layout: post
+title: "Apache Arrow 0.3.0リリース"
+date: "2017-05-08 00:00:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+[原文（English）]({% post_url 2017-05-08-0.3-release %})
+
+Apache Arrowチームは0.3.0のリリースをアナウンスできてうれしいです。2月にリリースした0.2.0から10週間の活発な開発の結果が今回のリリースです。[**23人のコントリビューター**][14]が[**306個のJIRAのissueを解決**][1]しました。
+
+複数のArrowの実装にたくさんの新しい機能を追加しています。2017年、特に注力して開発するのは、インメモリー用のフォーマット、型のメタデータ、メッセージング用のプロトコルです。これは、ビッグデータアプリケーションに**安定していてプロダクションで使える基盤**を提供するためです。高性能IOとインメモリーデータ処理にArrowを活用するために、[Apache Spark][3]・[GeoMesa][2]コミュニティーと協力していてとてもエキサイティングです。
+
+それぞれのプラットフォームでArrowを使う方法は[インストールページ][6]を見てください。
+
+Arrowでビッグデータシステムを高速化するケースを増やすために、近いうちにApache Arrowのロードマップを公開する予定です。
+
+Arrowの開発に参加するコントリビューターを募集しています。すでにArrowの開発に参加しているコミュニティーからのコントリビューターもそうですし、まだ参加していないGo、R、Juliaといったコミュニティーからのコントリビューターも募集しています。
+
+### ファイルフォーマットとストリーミングフォーマットの強化
+
+0.2.0では**ランダムアクセス**用と**ストリーミング**用のArrowのワイヤーフォーマットを導入しました。実装の詳細は[IPC仕様][15]を見てください。ユースケースは[使用例を紹介したブログ][4]を見てください。これらのフォーマットを使うと低オーバーヘッド・コピーなしでArrowのレコードバッチのペイロードにアクセスできます。
+
+0.3.0ではこのバイナリーフォマットの細かい詳細をたくさん固めました。Java、C++、Python間の連携のテストおよびそれぞれ言語での単体テストの整備も進めました。[Google Flatbuffers][5]は、前方互換性を壊さずにメタデータに新しい機能を追加するのに非常に助かりました。
+
+まだバイナリーフォーマットの前方互換性を必ず壊さないと約束できる状態ではありませんが（もしかしたら変更する必要があるなにかが見つかるかもしれない）、メジャーリリース間では不必要に互換性を壊さないように努力するつもりです。Apache ArrowのWebサイト、各コンポーネントのユーザー向けのドキュメントおよびAPIドキュメントへのコントリビューションを非常に歓迎します。
+
+### 辞書エンコーディングのサポート
+
+[GeoMesa][2]プロジェクトの[Emilio Lahr-Vivaz][13]はJavaのArrow実装に辞書エンコード対応ベクターをコントリビュートしました。これを受けて、C++とPythonでもサポートしました。（`pandas.Categorical`とも連携できます。）辞書エンコーディング用のインテグレーションテスト（C++とJava間でこのデータを送受信するテスト）はまだ完成していませんが、0.4.0までには完成させたいです。
+
+これはカテゴリーデータ用の一般的なデータ表現テクニックです。これを使うと、複数のレコードバッチで共通の「辞書」を共有し、各レコードバッチの値はこの辞書を参照する整数になります。このデータは統計的言語（statistical language）の分野では「カテゴリー（categorical）」や「因子（factor）」と呼ばれています。Apache Parquetのようなファイルフォーマットの分野ではデータ圧縮のためだけに使われています。
+
+### 日付、時刻、固定長型の拡張
+
+0.2.0では現実に使われている日付・時刻型をインテグレーションテスト付きで完全にサポートすることを諦めました。これらは[Apache Parquet][21]とApache Sparkとの連携に必要な機能です。
+
+* **日付**: 32-bit（日単位）と64-bit（ミリ秒単位）
+* **時刻**: 単位付き64-bit整数（単位：秒、ミリ秒、マイクロ秒、ナノ秒）
+* **タイムスタンプ（UNIXエポックからの経過時間）**: 単位付き64-bit整数のタイムゾーン付きとタイムゾーンなし
+* **固定長バイナリー**: 決まったバイト数のプリミティブな値
+* **固定長リスト**: 各要素が同じサイズのリスト（要素のベクターとは別にオフセットのベクターを持つ必要がない）
+
+C++のArrow実装では、[Boost.Multiprecision][12]を使ったexactな小数のサポートを実験的に追加しました。ただし、Java実装とC++実装間での小数のメモリーフォーマットはまだ固まっていません。
+
+### C++とPythonのWindowsサポート
+
+一般的なC++とPythonでの開発用に、パッケージ周りの改良も多数入っています。0.3.0はVisual Studio（MSVC）2015と2017を使ってWindowsを完全にサポートした最初のバージョンです。AppveyorでMSVC用のCIを実行しています。Windows上でソースからビルドするためのガイドも書きました。[C++][7]用と[Python][8]用。
+
+[conda-forge][31]からWindows用のArrowのPythonライブラリーをインストールできます。
+
+```shell
+conda install pyarrow -c conda-forge
+```
+
+### C（GLib）バインディングとRuby・Lua・他のサポート
+
+[Kouhei Sutou][9]は新しいApache Arrowのコントリビューターです。Linux用の（ArrowのC++実装の）GLibを使ったCバインディングをコントリビュートしました。[GObject Introspection][10]というCのミドルウェアを使うことでRuby、Lua、Goや[他にも様々なプログラミング言語][11]でシームレスにバインディングを使うことができます。これらのバインディングがどのように動いているか、これらのバインディングをどのように使うかを説明するブログ記事が別途必要な気がします。
+
+### PySparkを使ったApache Sparkとの連携
+
+[SPARK-13534][16]でApache Sparkコミュニティーと協力しています。PySparkでの`DataFrame.toPandas`をArrowを使って高速化しようとしています。効率的なデータのシリアライズにより[**40倍以上高速化**][17]できるケースがあります。
+
+PySparkでArrowを使うことでこれまでできなかったパフォーマンス最適化の道が開けました。特に、UDFの評価まわりでいろいろやれることがあるでしょう。（たとえば、Pythonのラムダ関数を使って`map`・`filter`を実行するケース。）
+
+### Python実装での新しい機能：メモリービュー、Feather、Apache Parquetのサポート
+
+ArrowのPythonライブラリーである`pyarrow`は`libarrow`と`libarrow_python`というC++ライブラリーのCythonバインディングです。`pyarrow`はNumPyと[pandas][29]とPythonの標準ライブラリー間のシームレスな連携を実現します。
+
+ArrowのC++ライブラリーで最も重要なものは`arrow::Buffer`オブジェクトです。これはメモリービューを管理します。コピーなしの読み込みとスライスをサポートしている点が重要です。[Jeff Knupp][25]はArrowのバッファーとPythonのバッファープロトコルとmemoryviewの連携処理をコントリビュートしました。これにより次のようなことができるようになりました。
+
+```python
+In [6]: import pyarrow as pa
+
+In [7]: buf = pa.frombuffer(b'foobarbaz')
+
+In [8]: buf
+Out[8]: <pyarrow._io.Buffer at 0x7f6c0a84b538>
+
+In [9]: memoryview(buf)
+Out[9]: <memory at 0x7f6c0a8c5e88>
+
+In [10]: buf.to_pybytes()
+Out[10]: b'foobarbaz'
+```
+
+C++でのParquet実装である[parquet-cpp][23]を使うことで大幅に[**Apache Parquet**][21]サポートを改良しました。たとえば、ディスク上にあるかHDFS上にあるか関係なく、パーティションされたデータセットをサポートしました。[Daskプロジェクト][24]はArrowを使ったParquetサポートを実装した最初のプロジェクトです。Dask開発者とはpandsデータを分散処理する文脈でさらに協力できることを楽しみにしています。
+
+pandasを成熟させるためにArrowを改良することもあり、[**Featherフォーマット**][22]の実装をマージしたのもその1つです。Featherフォーマットは本質的にはArrowのランダムアクセスフォーマットの特別なケースの1つです。ArrowのコードベースでFeatherの開発を続けます。たとえば、今のFeatherはArrowのPythonバインディングのレイヤーを使うことでPythonのファイルオブジェクトを読み書きできるようになっています。
+
+`DatetimeTZ`や`Categorical`といったpandas固有のデータ型のちゃんとした（robust）サポートも実装しました。
+
+### C++ライブラリーでのテンソルサポート
+
+Apache Arrowはコピーなしで共有メモリーを管理するツールという側面があります。機械学習アプリケーションの文脈でこの機能への関心が増えています。UCバークレー校の[RISELab][27]の[Rayプロジェクト][26]が最初の例です。
+
+機械学習ではは「テンソル」とも呼ばれる多次元配列というデータ構造を扱います。このようなデータ構造はArrowのカラムフォーマットがサポートしているデータ構造の範囲を超えています。今回のケースでは、[`arrow::Tensor`][28]というC++の型を追加で実装しました。これはArrowのコピーなしの共有メモリー機能を活用して実装しました。（メモリーの生存期間の管理に`arrow::Buffer`を使いました。）C++実装では、これからも、共通のIO・メモリー管理ツールとしてArrowを活用できるようにするため、追加のデータ構造を提供するつもりです。
+
+### JavaScript（TypeScript）実装の開始
+
+[Brian Hulette][20]はNodeJSとWebブラウザー上で動くアプリケーションで使うために[TypeScript][30]でのArrowの実装を始めました。FlatBuffersがJavaScriptをファーストクラスでサポートしているので実装が捗ります。
+
+### Webサイトと開発者用ドキュメントの改良
+
+0.2.0をリリースしてからドキュメントとブログを公開するためにWebサイトのシステムを[Jekyll][18]ベースで作りました。Kouhei Sutouは[Jekyll Jupyter Notebookプラグイン][19]を作りました。これによりArrowのWebサイトのコンテンツを作るためにJupyterを使うことができます。
+
+WebサイトにはC、C++、Java、PythonのAPIドキュメントを公開しました。これらの中にArrowを使い始めるための有益な情報を見つけられるでしょう。
+
+### コントリビューター
+
+このリリースにパッチをコントリビュートしたみなさんに感謝します。
+
+```
+$ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
+    119 Wes McKinney
+     55 Kouhei Sutou
+     18 Uwe L. Korn
+     17 Julien Le Dem
+      9 Phillip Cloud
+      6 Bryan Cutler
+      5 Philipp Moritz
+      5 Emilio Lahr-Vivaz
+      4 Max Risuhin
+      4 Johan Mabille
+      4 Jeff Knupp
+      3 Steven Phillips
+      3 Miki Tebeka
+      2 Leif Walsh
+      2 Jeff Reback
+      2 Brian Hulette
+      1 Tsuyoshi Ozawa
+      1 rvernica
+      1 Nong Li
+      1 Julien Lafaye
+      1 Itai Incze
+      1 Holden Karau
+      1 Deepak Majeti
+```
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.3.0
+[2]: http://www.geomesa.org/
+[3]: http://spark.apache.org
+[4]: http://wesmckinney.com/blog/arrow-streaming-columnar/
+[5]: http://github.com/google/flatbuffers
+[6]: http://arrow.apache.org/install
+[7]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
+[8]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst
+[9]: http://github.com/kou
+[10]: https://wiki.gnome.org/Projects/GObjectIntrospection
+[11]: https://wiki.gnome.org/Projects/GObjectIntrospection/Users
+[12]: https://github.com/boostorg/multiprecision
+[13]: https://github.com/elahrvivaz
+[14]: https://github.com/apache/arrow/graphs/contributors
+[15]: http://arrow.apache.org/docs/ipc.html
+[16]: https://issues.apache.org/jira/browse/SPARK-13534
+[17]: https://github.com/apache/spark/pull/15821#issuecomment-282175163
+[18]: https://jekyllrb.com
+[19]: https://github.com/red-data-tools/jekyll-jupyter-notebook
+[20]: https://github.com/TheNeuralBit
+[21]: http://parquet.apache.org
+[22]: https://github.com/wesm/feather
+[23]: https://github.com/apache/parquet-cpp
+[24]: https://github.com/dask/dask/commit/68f9e417924a985c1f2e2a587126833c70a2e9f4
+[25]: https://github.com/JeffKnupp
+[26]: https://github.com/ray-project/ray
+[27]: https://rise.cs.berkeley.edu/
+[28]: http://arrow.apache.org/docs/cpp/classarrow_1_1_tensor.html
+[29]: http://pandas.pydata.org
+[30]: https://github.com/apache/arrow/tree/master/js
+[31]: https://conda-forge.github.io
diff --git a/site/_posts/2017-05-08-0.3-release.md b/site/_posts/2017-05-08-0.3-release.md
index 5f80e62f73040..39a907d7ed645 100644
--- a/site/_posts/2017-05-08-0.3-release.md
+++ b/site/_posts/2017-05-08-0.3-release.md
@@ -24,6 +24,8 @@ limitations under the License.
 {% endcomment %}
 -->
 
+Translations: [日本語]({% post_url 2017-05-08-0.3-release-japanese %})
+
 The Apache Arrow team is pleased to announce the 0.3.0 release of the
 project. It is the product of an intense 10 weeks of development since the
 0.2.0 release from this past February. It includes [**306 resolved JIRAs**][1]

From a4f29f3a3ff1c64a6f547bfb0d5e4500142ea5ec Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Thu, 11 May 2017 20:13:10 -0400
Subject: [PATCH 0638/1644] ARROW-29: [C++] FindRe2 cmake module

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #674 from MaxRis/ARROW-29 and squashes the following commits:

e2d5d72 [Max Risuhin] ARROW-29: [C++] FindRe2 cmake module
---
 cpp/cmake_modules/FindRe2.cmake | 69 +++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 cpp/cmake_modules/FindRe2.cmake

diff --git a/cpp/cmake_modules/FindRe2.cmake b/cpp/cmake_modules/FindRe2.cmake
new file mode 100644
index 0000000000000..6450b22e3222d
--- /dev/null
+++ b/cpp/cmake_modules/FindRe2.cmake
@@ -0,0 +1,69 @@
+##############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##############################################################################
+
+# - Find re2 headers and lib.
+# RE2_HOME hints the location
+# This module defines
+#  RE2_INCLUDE_DIR, directory containing headers
+#  RE2_STATIC_LIB, path to libre2.a
+#  re2 imported static library
+#  RE2_FOUND, whether re2 has been found
+
+set(RE2_SEARCH_HEADER_PATHS ${RE2_HOME}/include)
+
+set(RE2_SEARCH_LIB_PATHS ${RE2_HOME}/lib)
+
+find_path(RE2_INCLUDE_DIR re2/re2.h
+  PATHS ${RE2_SEARCH_HEADER_PATHS}
+        NO_DEFAULT_PATH
+  DOC  "Google's re2 regex header path"
+)
+
+find_library(RE2_LIBS NAMES re2
+  PATHS ${RE2_SEARCH_LIB_PATHS}
+        NO_DEFAULT_PATH
+  DOC   "Google's re2 regex library"
+)
+
+find_library(RE2_STATIC_LIB NAMES libre2${CMAKE_STATIC_LIBRARY_SUFFIX}
+  PATHS ${RE2_SEARCH_LIB_PATHS}
+        NO_DEFAULT_PATH
+  DOC   "Google's re2 regex static library"
+)
+
+message(STATUS ${RE2_INCLUDE_DIR})
+
+if (NOT RE2_INCLUDE_DIR OR NOT RE2_LIBS OR
+    NOT RE2_STATIC_LIB)
+  set(RE2_FOUND FALSE)
+  message(FATAL_ERROR "Re2 includes and libraries NOT found. "
+    "Looked for headers in ${RE2_SEARCH_HEADER_PATHS}, "
+    "and for libs in ${RE2_SEARCH_LIB_PATHS}")
+else()
+    set(RE2_FOUND TRUE)
+    add_library(re2 STATIC IMPORTED)
+    set_target_properties(re2 PROPERTIES IMPORTED_LOCATION "${RE2_STATIC_LIB}")
+endif()
+
+mark_as_advanced(
+  RE2_INCLUDE_DIR
+  RE2_LIBS
+  RE2_STATIC_LIB
+  re2
+)

From 05e8f6894d6515626048a3701235b53e9ce26721 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 12 May 2017 15:03:23 -0400
Subject: [PATCH 0639/1644] ARROW-1010: [Website] Provide for translations
 without repeating blog post in blogroll

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #675 from wesm/ARROW-1010 and squashes the following commits:

35893e8 [Wes McKinney] Mark JA version of 0.3.0 release post
---
 site/_posts/2017-05-08-0.3-release-japanese.md | 2 +-
 site/blog.html                                 | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/site/_posts/2017-05-08-0.3-release-japanese.md b/site/_posts/2017-05-08-0.3-release-japanese.md
index 27e0166b5fc7d..e49bde1725957 100644
--- a/site/_posts/2017-05-08-0.3-release-japanese.md
+++ b/site/_posts/2017-05-08-0.3-release-japanese.md
@@ -3,7 +3,7 @@ layout: post
 title: "Apache Arrow 0.3.0リリース"
 date: "2017-05-08 00:00:00 -0400"
 author: wesm
-categories: [release]
+categories: [release,translation]
 ---
 <!--
 {% comment %}
diff --git a/site/blog.html b/site/blog.html
index ddfb47fb364d0..8549859078b05 100644
--- a/site/blog.html
+++ b/site/blog.html
@@ -24,5 +24,7 @@ <h2>Project News and Blog</h2>
 <hr>
 
 {% for post in site.posts %}
-  {% include blog_entry.html %}
+  {% unless post.categories contains 'translation' %}
+    {% include blog_entry.html %}
+  {% endunless %}
 {% endfor %}

From 95ee96b922496120a96a7bad3d681bb3e94689bf Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 12 May 2017 16:18:03 -0400
Subject: [PATCH 0640/1644] ARROW-1016: Python: Include C++ headers
 (optionally) in wheels

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #678 from xhochy/ARROW-1016 and squashes the following commits:

bc9b5ee [Uwe L. Korn] Build pyarrow using multiple cores
c982c97 [Uwe L. Korn] Remove existing include folder
0b3209a [Uwe L. Korn] More cmake debug output
590f629 [Uwe L. Korn] Add output to wrap cmake command
3de6a8a [Uwe L. Korn] ARROW-1016: Python: Include C++ headers (optionally) in wheels
---
 ci/travis_script_manylinux.sh |  2 +-
 python/CMakeLists.txt         |  1 +
 python/manylinux1/README.md   |  2 +-
 python/setup.py               | 12 ++++++++++++
 4 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh
index 69feb685b5136..4e6be62bd3e9d 100755
--- a/ci/travis_script_manylinux.sh
+++ b/ci/travis_script_manylinux.sh
@@ -18,4 +18,4 @@ set -ex
 pushd python/manylinux1
 git clone ../../ arrow
 docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
-docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
+docker run --rm -e PYARROW_PARALLEL=3 -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index a9db4d877fb0c..d8287108d4ca0 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -246,6 +246,7 @@ endfunction(bundle_arrow_lib)
 
 if (PYARROW_BUNDLE_ARROW_CPP)
   # arrow
+  file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
   bundle_arrow_lib(ARROW_SHARED_LIB)
   bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
 endif()
diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md
index 32af6f31da287..2e7f56bd6203f 100644
--- a/python/manylinux1/README.md
+++ b/python/manylinux1/README.md
@@ -32,7 +32,7 @@ git clone ../../ arrow
 # Build the native baseimage
 docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
 # Build the python packages
-docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
+docker run --rm -t -i -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
 # Now the new packages are located in the dist/ folder
 ls -l dist/
 ```
diff --git a/python/setup.py b/python/setup.py
index 1c46617066925..148224afb0574 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -161,14 +161,18 @@ def _run_cmake(self):
             cmake_command = (['cmake', self.extra_cmake_args] +
                              cmake_options + [source])
 
+            print("-- Runnning cmake for pyarrow")
             self.spawn(cmake_command)
+            print("-- Finished cmake for pyarrow")
             args = ['make']
             if os.environ.get('PYARROW_BUILD_VERBOSE', '0') == '1':
                 args.append('VERBOSE=1')
 
             if 'PYARROW_PARALLEL' in os.environ:
                 args.append('-j{0}'.format(os.environ['PYARROW_PARALLEL']))
+            print("-- Running cmake --build for pyarrow")
             self.spawn(args)
+            print("-- Finished cmake --build for pyarrow")
         else:
             import shlex
             cmake_generator = 'Visual Studio 14 2015 Win64'
@@ -183,9 +187,13 @@ def _run_cmake(self):
             if "-G" in self.extra_cmake_args:
                 cmake_command = cmake_command[:-2]
 
+            print("-- Runnning cmake for pyarrow")
             self.spawn(cmake_command)
+            print("-- Finished cmake for pyarrow")
             # Do the build
+            print("-- Running cmake --build for pyarrow")
             self.spawn(['cmake', '--build', '.', '--config', self.build_type])
+            print("-- Finished cmake --build for pyarrow")
 
         if self.inplace:
             # a bit hacky
@@ -225,6 +233,10 @@ def move_lib(lib_name):
                     os.symlink(lib_filename, link_name)
 
         if self.bundle_arrow_cpp:
+            print(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow'))
+            if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
+                shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
+            shutil.move(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow'))
             move_lib("arrow")
             move_lib("arrow_python")
             if self.with_jemalloc:

From 9e875a6843b7bd155f7e10d011f5e8d25a47494c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 13 May 2017 15:44:43 -0400
Subject: [PATCH 0641/1644] ARROW-819: Public Cython and C++ API in the style
 of lxml, arrow::py::import_pyarrow method

I have been looking at LXML's approach to creating both a public Cython API and C++ API

https://github.com/lxml/lxml

While this may seem like a somewhat radical reorganization of the code, putting all of the main symbols in a single Cython extension makes generating a C++ API for them significantly simpler. By using `.pxi` files we can break the codebase into as small pieces as we like (as long as there are no circular dependencies). As a convenient side effect, the build times are shorter.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #680 from wesm/ARROW-819 and squashes the following commits:

9e6ee246 [Wes McKinney] Fix up optional extensions
cff757de [Wes McKinney] Expose pyarrow C API in arrow/python/pyarrow.h
b39d19cd [Wes McKinney] Fix test suite. Move _config into lib
ff1b5e51 [Wes McKinney] Rename things a bit
d4a83912 [Wes McKinney] Reorganize Cython code in the style of lxml so make declaring a public C API easier
---
 cpp/CMakeLists.txt                         |   4 +-
 cpp/src/arrow/python/CMakeLists.txt        |   2 +
 cpp/src/arrow/python/pyarrow.cc            |  75 +++++++++++
 cpp/src/arrow/python/pyarrow.h             |  55 ++++++++
 cpp/src/arrow/python/pyarrow_api.h         | 143 +++++++++++++++++++++
 python/CMakeLists.txt                      |   7 +-
 python/pyarrow/__init__.pxd                |  34 +++++
 python/pyarrow/__init__.py                 | 112 ++++++++--------
 python/pyarrow/_error.pxd                  |  20 ---
 python/pyarrow/_io.pxd                     |  50 -------
 python/pyarrow/_jemalloc.pyx               |   2 +-
 python/pyarrow/_memory.pxd                 |  30 -----
 python/pyarrow/_parquet.pyx                |  22 ++--
 python/pyarrow/_table.pxd                  |  62 ---------
 python/pyarrow/{_array.pyx => array.pxi}   | 140 +++++---------------
 python/pyarrow/{_error.pyx => error.pxi}   |   0
 python/pyarrow/feather.py                  |   6 +-
 python/pyarrow/filesystem.py               |  14 +-
 python/pyarrow/formatting.py               |   4 +-
 python/pyarrow/includes/libarrow.pxd       |  61 ++++++++-
 python/pyarrow/includes/pyarrow.pxd        |  75 -----------
 python/pyarrow/{_io.pyx => io.pxi}         |  49 ++-----
 python/pyarrow/ipc.py                      |  10 +-
 python/pyarrow/{_array.pxd => lib.pxd}     | 102 ++++++++++++++-
 python/pyarrow/lib.pyx                     |  88 +++++++++++++
 python/pyarrow/{_memory.pyx => memory.pxi} |  11 +-
 python/pyarrow/parquet.py                  |  13 +-
 python/pyarrow/public-api.pxi              | 107 +++++++++++++++
 python/pyarrow/{_table.pyx => table.pxi}   |  98 ++++----------
 python/pyarrow/tests/test_feather.py       |   2 +-
 python/setup.py                            |   9 +-
 31 files changed, 832 insertions(+), 575 deletions(-)
 create mode 100644 cpp/src/arrow/python/pyarrow.cc
 create mode 100644 cpp/src/arrow/python/pyarrow.h
 create mode 100644 cpp/src/arrow/python/pyarrow_api.h
 create mode 100644 python/pyarrow/__init__.pxd
 delete mode 100644 python/pyarrow/_error.pxd
 delete mode 100644 python/pyarrow/_io.pxd
 delete mode 100644 python/pyarrow/_memory.pxd
 delete mode 100644 python/pyarrow/_table.pxd
 rename python/pyarrow/{_array.pyx => array.pxi} (91%)
 rename python/pyarrow/{_error.pyx => error.pxi} (100%)
 delete mode 100644 python/pyarrow/includes/pyarrow.pxd
 rename python/pyarrow/{_io.pyx => io.pxi} (96%)
 rename python/pyarrow/{_array.pxd => lib.pxd} (61%)
 create mode 100644 python/pyarrow/lib.pyx
 rename python/pyarrow/{_memory.pyx => memory.pxi} (82%)
 create mode 100644 python/pyarrow/public-api.pxi
 rename python/pyarrow/{_table.pyx => table.pxi} (90%)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 21463794879fc..6b2ceec32777e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -852,7 +852,8 @@ if (UNIX)
     )
 
   FOREACH(item ${LINT_FILES})
-    IF(NOT (item MATCHES "_generated.h"))
+    IF(NOT ((item MATCHES "_generated.h") OR
+            (item MATCHES "pyarrow_api.h")))
       LIST(APPEND FILTERED_LINT_FILES ${item})
     ENDIF()
   ENDFOREACH(item ${LINT_FILES})
@@ -878,6 +879,7 @@ if (${CLANG_FORMAT_FOUND})
     `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h |
     sed -e '/_generated/g' |
     sed -e '/windows_compatibility.h/g' |
+    sed -e '/pyarrow_api.h/g' |
     sed -e '/config.h/g' |   # python/config.h
     sed -e '/platform.h/g'`  # python/platform.h
     )
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index c5cbc50845de0..30852291d1a8d 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -50,6 +50,7 @@ set(ARROW_PYTHON_SRCS
   io.cc
   numpy_convert.cc
   pandas_convert.cc
+  pyarrow.cc
 )
 
 set(ARROW_PYTHON_SHARED_LINK_LIBS
@@ -90,6 +91,7 @@ install(FILES
   numpy_interop.h
   pandas_convert.h
   platform.h
+  pyarrow.h
   type_traits.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python")
 
diff --git a/cpp/src/arrow/python/pyarrow.cc b/cpp/src/arrow/python/pyarrow.cc
new file mode 100644
index 0000000000000..56c0381957f01
--- /dev/null
+++ b/cpp/src/arrow/python/pyarrow.cc
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/pyarrow.h"
+
+#include <memory>
+
+#include "arrow/array.h"
+#include "arrow/table.h"
+#include "arrow/tensor.h"
+#include "arrow/type.h"
+
+namespace {
+#include "arrow/python/pyarrow_api.h"
+}
+
+namespace arrow {
+namespace py {
+
+int import_pyarrow() {
+  return ::import_pyarrow__lib();
+}
+
+PyObject* wrap_buffer(const std::shared_ptr<Buffer>& buffer) {
+  return ::pyarrow_wrap_buffer(buffer);
+}
+
+PyObject* wrap_data_type(const std::shared_ptr<DataType>& type) {
+  return ::pyarrow_wrap_data_type(type);
+}
+
+PyObject* wrap_field(const std::shared_ptr<Field>& field) {
+  return ::pyarrow_wrap_field(field);
+}
+
+PyObject* wrap_schema(const std::shared_ptr<Schema>& schema) {
+  return ::pyarrow_wrap_schema(schema);
+}
+
+PyObject* wrap_array(const std::shared_ptr<Array>& array) {
+  return ::pyarrow_wrap_array(array);
+}
+
+PyObject* wrap_tensor(const std::shared_ptr<Tensor>& tensor) {
+  return ::pyarrow_wrap_tensor(tensor);
+}
+
+PyObject* wrap_column(const std::shared_ptr<Column>& column) {
+  return ::pyarrow_wrap_column(column);
+}
+
+PyObject* wrap_table(const std::shared_ptr<Table>& table) {
+  return ::pyarrow_wrap_table(table);
+}
+
+PyObject* wrap_record_batch(const std::shared_ptr<RecordBatch>& batch) {
+  return ::pyarrow_wrap_batch(batch);
+}
+
+}  // namespace py
+}  // namespace arrow
diff --git a/cpp/src/arrow/python/pyarrow.h b/cpp/src/arrow/python/pyarrow.h
new file mode 100644
index 0000000000000..7c618ce192575
--- /dev/null
+++ b/cpp/src/arrow/python/pyarrow.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PYTHON_PYARROW_H
+#define ARROW_PYTHON_PYARROW_H
+
+#include "arrow/python/platform.h"
+
+#include <memory>
+
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+class Buffer;
+class Column;
+class DataType;
+class Field;
+class RecordBatch;
+class Schema;
+class Table;
+class Tensor;
+
+namespace py {
+
+ARROW_EXPORT int import_pyarrow();
+ARROW_EXPORT PyObject* wrap_buffer(const std::shared_ptr<Buffer>& buffer);
+ARROW_EXPORT PyObject* wrap_data_type(const std::shared_ptr<DataType>& type);
+ARROW_EXPORT PyObject* wrap_field(const std::shared_ptr<Field>& field);
+ARROW_EXPORT PyObject* wrap_schema(const std::shared_ptr<Schema>& schema);
+ARROW_EXPORT PyObject* wrap_array(const std::shared_ptr<Array>& array);
+ARROW_EXPORT PyObject* wrap_tensor(const std::shared_ptr<Tensor>& tensor);
+ARROW_EXPORT PyObject* wrap_column(const std::shared_ptr<Column>& column);
+ARROW_EXPORT PyObject* wrap_table(const std::shared_ptr<Table>& table);
+ARROW_EXPORT PyObject* wrap_record_batch(const std::shared_ptr<RecordBatch>& batch);
+
+}  // namespace py
+}  // namespace arrow
+
+#endif  // ARROW_PYTHON_PYARROW_H
diff --git a/cpp/src/arrow/python/pyarrow_api.h b/cpp/src/arrow/python/pyarrow_api.h
new file mode 100644
index 0000000000000..7b7084466109b
--- /dev/null
+++ b/cpp/src/arrow/python/pyarrow_api.h
@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// DO NOT EDIT THIS FILE. Update from pyarrow/lib_api.h after pyarrow build
+
+/* Generated by Cython 0.25.2 */
+
+#ifndef __PYX_HAVE_API__pyarrow__lib
+#define __PYX_HAVE_API__pyarrow__lib
+#include "Python.h"
+
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_buffer)(std::shared_ptr< arrow::Buffer>  const &) = 0;
+#define pyarrow_wrap_buffer __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_buffer
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_data_type)(std::shared_ptr< arrow::DataType>  const &) = 0;
+#define pyarrow_wrap_data_type __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_data_type
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_field)(std::shared_ptr< arrow::Field>  const &) = 0;
+#define pyarrow_wrap_field __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_field
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_schema)(std::shared_ptr< arrow::Schema>  const &) = 0;
+#define pyarrow_wrap_schema __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_schema
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_array)(std::shared_ptr< arrow::Array>  const &) = 0;
+#define pyarrow_wrap_array __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_array
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_tensor)(std::shared_ptr< arrow::Tensor>  const &) = 0;
+#define pyarrow_wrap_tensor __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_tensor
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_column)(std::shared_ptr< arrow::Column>  const &) = 0;
+#define pyarrow_wrap_column __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_column
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_table)(std::shared_ptr< arrow::Table>  const &) = 0;
+#define pyarrow_wrap_table __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_table
+static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_batch)(std::shared_ptr< arrow::RecordBatch>  const &) = 0;
+#define pyarrow_wrap_batch __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_batch
+#if !defined(__Pyx_PyIdentifier_FromString)
+#if PY_MAJOR_VERSION < 3
+  #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s)
+#else
+  #define __Pyx_PyIdentifier_FromString(s) PyUnicode_FromString(s)
+#endif
+#endif
+
+#ifndef __PYX_HAVE_RT_ImportModule
+#define __PYX_HAVE_RT_ImportModule
+static PyObject *__Pyx_ImportModule(const char *name) {
+    PyObject *py_name = 0;
+    PyObject *py_module = 0;
+    py_name = __Pyx_PyIdentifier_FromString(name);
+    if (!py_name)
+        goto bad;
+    py_module = PyImport_Import(py_name);
+    Py_DECREF(py_name);
+    return py_module;
+bad:
+    Py_XDECREF(py_name);
+    return 0;
+}
+#endif
+
+#ifndef __PYX_HAVE_RT_ImportFunction
+#define __PYX_HAVE_RT_ImportFunction
+static int __Pyx_ImportFunction(PyObject *module, const char *funcname, void (**f)(void), const char *sig) {
+    PyObject *d = 0;
+    PyObject *cobj = 0;
+    union {
+        void (*fp)(void);
+        void *p;
+    } tmp;
+    d = PyObject_GetAttrString(module, (char *)"__pyx_capi__");
+    if (!d)
+        goto bad;
+    cobj = PyDict_GetItemString(d, funcname);
+    if (!cobj) {
+        PyErr_Format(PyExc_ImportError,
+            "%.200s does not export expected C function %.200s",
+                PyModule_GetName(module), funcname);
+        goto bad;
+    }
+#if PY_VERSION_HEX >= 0x02070000
+    if (!PyCapsule_IsValid(cobj, sig)) {
+        PyErr_Format(PyExc_TypeError,
+            "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)",
+             PyModule_GetName(module), funcname, sig, PyCapsule_GetName(cobj));
+        goto bad;
+    }
+    tmp.p = PyCapsule_GetPointer(cobj, sig);
+#else
+    {const char *desc, *s1, *s2;
+    desc = (const char *)PyCObject_GetDesc(cobj);
+    if (!desc)
+        goto bad;
+    s1 = desc; s2 = sig;
+    while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; }
+    if (*s1 != *s2) {
+        PyErr_Format(PyExc_TypeError,
+            "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)",
+             PyModule_GetName(module), funcname, sig, desc);
+        goto bad;
+    }
+    tmp.p = PyCObject_AsVoidPtr(cobj);}
+#endif
+    *f = tmp.fp;
+    if (!(*f))
+        goto bad;
+    Py_DECREF(d);
+    return 0;
+bad:
+    Py_XDECREF(d);
+    return -1;
+}
+#endif
+
+
+static int import_pyarrow__lib(void) {
+  PyObject *module = 0;
+  module = __Pyx_ImportModule("pyarrow.lib");
+  if (!module) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_buffer", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_buffer, "PyObject *(std::shared_ptr< arrow::Buffer>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_data_type", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_data_type, "PyObject *(std::shared_ptr< arrow::DataType>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_field", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_field, "PyObject *(std::shared_ptr< arrow::Field>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_schema", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_schema, "PyObject *(std::shared_ptr< arrow::Schema>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_array", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_array, "PyObject *(std::shared_ptr< arrow::Array>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_tensor", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_tensor, "PyObject *(std::shared_ptr< arrow::Tensor>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_column", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_column, "PyObject *(std::shared_ptr< arrow::Column>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_table", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_table, "PyObject *(std::shared_ptr< arrow::Table>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_wrap_batch", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_batch, "PyObject *(std::shared_ptr< arrow::RecordBatch>  const &)") < 0) goto bad;
+  Py_DECREF(module); module = 0;
+  return 0;
+  bad:
+  Py_XDECREF(module);
+  return -1;
+}
+
+#endif /* !__PYX_HAVE_API__pyarrow__lib */
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index d8287108d4ca0..123dd5d8d7a44 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -265,12 +265,7 @@ if (UNIX)
 endif()
 
 set(CYTHON_EXTENSIONS
-  _array
-  _config
-  _error
-  _io
-  _memory
-  _table
+  lib
 )
 
 set(LINK_LIBS
diff --git a/python/pyarrow/__init__.pxd b/python/pyarrow/__init__.pxd
new file mode 100644
index 0000000000000..4f43455791568
--- /dev/null
+++ b/python/pyarrow/__init__.pxd
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from libcpp.memory cimport shared_ptr
+from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CDataType,
+                                        CField, CRecordBatch, CSchema,
+                                        CTable, CTensor)
+
+
+cdef extern from "arrow/python/pyarrow.h" namespace "arrow::py":
+    cdef int import_pyarrow() except -1
+    cdef object wrap_buffer(const shared_ptr[CBuffer]& buffer)
+    cdef object wrap_data_type(const shared_ptr[CDataType]& type)
+    cdef object wrap_field(const shared_ptr[CField]& field)
+    cdef object wrap_schema(const shared_ptr[CSchema]& schema)
+    cdef object wrap_array(const shared_ptr[CArray]& sp_array)
+    cdef object wrap_tensor(const shared_ptr[CTensor]& sp_tensor)
+    cdef object wrap_column(const shared_ptr[CColumn]& ccolumn)
+    cdef object wrap_table(const shared_ptr[CTable]& ctable)
+    cdef object wrap_batch(const shared_ptr[CRecordBatch]& cbatch)
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 4d8da9f5a10ed..7d79811d9883e 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -25,53 +25,51 @@
    pass
 
 
-import pyarrow._config
-from pyarrow._config import cpu_count, set_cpu_count
+from pyarrow.lib import cpu_count, set_cpu_count
+from pyarrow.lib import (null, bool_,
+                         int8, int16, int32, int64,
+                         uint8, uint16, uint32, uint64,
+                         time32, time64, timestamp, date32, date64,
+                         float16, float32, float64,
+                         binary, string, decimal,
+                         list_, struct, dictionary, field,
+                         DataType,
+                         DecimalType,
+                         DictionaryType,
+                         FixedSizeBinaryType,
+                         TimestampType,
+                         Time32Type,
+                         Time64Type,
+                         Field,
+                         Schema,
+                         schema,
+                         Array, Tensor,
+                         array,
+                         from_numpy_dtype,
+                         NullArray,
+                         NumericArray, IntegerArray, FloatingPointArray,
+                         BooleanArray,
+                         Int8Array, UInt8Array,
+                         Int16Array, UInt16Array,
+                         Int32Array, UInt32Array,
+                         Int64Array, UInt64Array,
+                         ListArray,
+                         BinaryArray, StringArray,
+                         FixedSizeBinaryArray,
+                         DictionaryArray,
+                         Date32Array, Date64Array,
+                         TimestampArray, Time32Array, Time64Array,
+                         DecimalArray,
+                         ArrayValue, Scalar, NA, NAType,
+                         BooleanValue,
+                         Int8Value, Int16Value, Int32Value, Int64Value,
+                         UInt8Value, UInt16Value, UInt32Value, UInt64Value,
+                         FloatValue, DoubleValue, ListValue,
+                         BinaryValue, StringValue, FixedSizeBinaryValue,
+                         DecimalValue,
+                         Date32Value, Date64Value, TimestampValue)
 
-from pyarrow._array import (null, bool_,
-                            int8, int16, int32, int64,
-                            uint8, uint16, uint32, uint64,
-                            time32, time64, timestamp, date32, date64,
-                            float16, float32, float64,
-                            binary, string, decimal,
-                            list_, struct, dictionary, field,
-                            DataType,
-                            DecimalType,
-                            DictionaryType,
-                            FixedSizeBinaryType,
-                            TimestampType,
-                            Time32Type,
-                            Time64Type,
-                            Field,
-                            Schema,
-                            schema,
-                            Array, Tensor,
-                            array,
-                            from_numpy_dtype,
-                            NullArray,
-                            NumericArray, IntegerArray, FloatingPointArray,
-                            BooleanArray,
-                            Int8Array, UInt8Array,
-                            Int16Array, UInt16Array,
-                            Int32Array, UInt32Array,
-                            Int64Array, UInt64Array,
-                            ListArray,
-                            BinaryArray, StringArray,
-                            FixedSizeBinaryArray,
-                            DictionaryArray,
-                            Date32Array, Date64Array,
-                            TimestampArray, Time32Array, Time64Array,
-                            DecimalArray,
-                            ArrayValue, Scalar, NA, NAType,
-                            BooleanValue,
-                            Int8Value, Int16Value, Int32Value, Int64Value,
-                            UInt8Value, UInt16Value, UInt32Value, UInt64Value,
-                            FloatValue, DoubleValue, ListValue,
-                            BinaryValue, StringValue, FixedSizeBinaryValue,
-                            DecimalValue,
-                            Date32Value, Date64Value, TimestampValue)
-
-from pyarrow._io import (HdfsFile, NativeFile, PythonFile,
+from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
                          Buffer, BufferReader, InMemoryOutputStream,
                          OSFile, MemoryMappedFile, memory_map,
                          frombuffer, read_tensor, write_tensor,
@@ -79,17 +77,17 @@
                          get_record_batch_size, get_tensor_size,
                          have_libhdfs, have_libhdfs3)
 
-from pyarrow._memory import (MemoryPool, total_allocated_bytes,
-                             set_memory_pool, default_memory_pool)
-from pyarrow._table import (ChunkedArray, Column, RecordBatch, Table,
-                            concat_tables)
-from pyarrow._error import (ArrowException,
-                            ArrowKeyError,
-                            ArrowInvalid,
-                            ArrowIOError,
-                            ArrowMemoryError,
-                            ArrowNotImplementedError,
-                            ArrowTypeError)
+from pyarrow.lib import (MemoryPool, total_allocated_bytes,
+                         set_memory_pool, default_memory_pool)
+from pyarrow.lib import (ChunkedArray, Column, RecordBatch, Table,
+                         concat_tables)
+from pyarrow.lib import (ArrowException,
+                         ArrowKeyError,
+                         ArrowInvalid,
+                         ArrowIOError,
+                         ArrowMemoryError,
+                         ArrowNotImplementedError,
+                         ArrowTypeError)
 
 
 def jemalloc_memory_pool():
diff --git a/python/pyarrow/_error.pxd b/python/pyarrow/_error.pxd
deleted file mode 100644
index 4fb46c25fafe4..0000000000000
--- a/python/pyarrow/_error.pxd
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyarrow.includes.libarrow cimport CStatus
-
-cdef int check_status(const CStatus& status) nogil except -1
diff --git a/python/pyarrow/_io.pxd b/python/pyarrow/_io.pxd
deleted file mode 100644
index 0c37a09add574..0000000000000
--- a/python/pyarrow/_io.pxd
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# distutils: language = c++
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport *
-
-
-cdef class Buffer:
-    cdef:
-        shared_ptr[CBuffer] buffer
-        Py_ssize_t shape[1]
-        Py_ssize_t strides[1]
-
-    cdef init(self, const shared_ptr[CBuffer]& buffer)
-
-
-cdef class NativeFile:
-    cdef:
-        shared_ptr[RandomAccessFile] rd_file
-        shared_ptr[OutputStream] wr_file
-        bint is_readable
-        bint is_writeable
-        bint is_open
-        bint own_file
-
-    # By implementing these "virtual" functions (all functions in Cython
-    # extension classes are technically virtual in the C++ sense) we can expose
-    # the arrow::io abstract file interfaces to other components throughout the
-    # suite of Arrow C++ libraries
-    cdef read_handle(self, shared_ptr[RandomAccessFile]* file)
-    cdef write_handle(self, shared_ptr[OutputStream]* file)
-
-cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader)
-cdef get_writer(object source, shared_ptr[OutputStream]* writer)
diff --git a/python/pyarrow/_jemalloc.pyx b/python/pyarrow/_jemalloc.pyx
index 3b41964a39cb6..6f00c9d2dded8 100644
--- a/python/pyarrow/_jemalloc.pyx
+++ b/python/pyarrow/_jemalloc.pyx
@@ -20,7 +20,7 @@
 # cython: embedsignature = True
 
 from pyarrow.includes.libarrow_jemalloc cimport CJemallocMemoryPool
-from pyarrow._memory cimport MemoryPool
+from pyarrow.lib cimport MemoryPool
 
 def default_pool():
     cdef MemoryPool pool = MemoryPool()
diff --git a/python/pyarrow/_memory.pxd b/python/pyarrow/_memory.pxd
deleted file mode 100644
index bb1af85c8ea65..0000000000000
--- a/python/pyarrow/_memory.pxd
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
-
-
-cdef class MemoryPool:
-    cdef:
-        CMemoryPool* pool
-
-    cdef init(self, CMemoryPool* pool)
-
-cdef class LoggingMemoryPool(MemoryPool):
-    pass
-
-cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool)
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index c06eab2630210..51bd938c79a5e 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -22,16 +22,16 @@
 from cython.operator cimport dereference as deref
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
-cimport pyarrow.includes.pyarrow as pyarrow
-from pyarrow._array cimport Array, Schema, box_schema
-from pyarrow._error cimport check_status
-from pyarrow._memory cimport MemoryPool, maybe_unbox_memory_pool
-from pyarrow._table cimport Table, table_from_ctable
-from pyarrow._io cimport NativeFile, get_reader, get_writer
+from pyarrow.lib cimport (Array, Schema,
+                          check_status,
+                          MemoryPool, maybe_unbox_memory_pool,
+                          Table,
+                          pyarrow_wrap_schema,
+                          pyarrow_wrap_table,
+                          NativeFile, get_reader, get_writer)
 
 from pyarrow.compat import tobytes, frombytes
-from pyarrow._error import ArrowException
-from pyarrow._io import NativeFile
+from pyarrow.lib import ArrowException, NativeFile
 
 import six
 
@@ -213,7 +213,7 @@ cdef class ParquetSchema:
         with nogil:
             check_status(FromParquetSchema(self.schema, &sp_arrow_schema))
 
-        return box_schema(sp_arrow_schema)
+        return pyarrow_wrap_schema(sp_arrow_schema)
 
     def equals(self, ParquetSchema other):
         """
@@ -426,7 +426,7 @@ cdef class ParquetReader:
             with nogil:
                 check_status(self.reader.get()
                              .ReadRowGroup(i, &ctable))
-        return table_from_ctable(ctable)
+        return pyarrow_wrap_table(ctable)
 
     def read_all(self, column_indices=None):
         cdef:
@@ -445,7 +445,7 @@ cdef class ParquetReader:
             with nogil:
                 check_status(self.reader.get()
                              .ReadTable(&ctable))
-        return table_from_ctable(ctable)
+        return pyarrow_wrap_table(ctable)
 
     def column_name_idx(self, column_name):
         """
diff --git a/python/pyarrow/_table.pxd b/python/pyarrow/_table.pxd
deleted file mode 100644
index e61e90d7462f9..0000000000000
--- a/python/pyarrow/_table.pxd
+++ /dev/null
@@ -1,62 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyarrow.includes.common cimport shared_ptr
-from pyarrow.includes.libarrow cimport (CChunkedArray, CColumn, CTable,
-                                        CRecordBatch)
-from pyarrow._array cimport Schema
-
-
-cdef class ChunkedArray:
-    cdef:
-        shared_ptr[CChunkedArray] sp_chunked_array
-        CChunkedArray* chunked_array
-
-    cdef init(self, const shared_ptr[CChunkedArray]& chunked_array)
-    cdef _check_nullptr(self)
-
-
-cdef class Column:
-    cdef:
-        shared_ptr[CColumn] sp_column
-        CColumn* column
-
-    cdef init(self, const shared_ptr[CColumn]& column)
-    cdef _check_nullptr(self)
-
-
-cdef class Table:
-    cdef:
-        shared_ptr[CTable] sp_table
-        CTable* table
-
-    cdef init(self, const shared_ptr[CTable]& table)
-    cdef _check_nullptr(self)
-
-
-cdef class RecordBatch:
-    cdef:
-        shared_ptr[CRecordBatch] sp_batch
-        CRecordBatch* batch
-        Schema _schema
-
-    cdef init(self, const shared_ptr[CRecordBatch]& table)
-    cdef _check_nullptr(self)
-
-cdef object box_column(const shared_ptr[CColumn]& ccolumn)
-cdef api object table_from_ctable(const shared_ptr[CTable]& ctable)
-cdef api object batch_from_cbatch(const shared_ptr[CRecordBatch]& cbatch)
diff --git a/python/pyarrow/_array.pyx b/python/pyarrow/array.pxi
similarity index 91%
rename from python/pyarrow/_array.pyx
rename to python/pyarrow/array.pxi
index f01cff6cc99f3..46e94b4f4b3e6 100644
--- a/python/pyarrow/_array.pyx
+++ b/python/pyarrow/array.pxi
@@ -15,31 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from cython.operator cimport dereference as deref
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.common cimport PyObject_to_object
-cimport pyarrow.includes.pyarrow as pyarrow
-from pyarrow._error cimport check_status
-from pyarrow._memory cimport MemoryPool, maybe_unbox_memory_pool
-cimport cpython as cp
-
-
-import datetime
-import decimal as _pydecimal
-import numpy as np
-import six
-import pyarrow._config
-from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
-
-
-cdef _pandas():
-    import pandas as pd
-    return pd
-
 
 # These are imprecise because the type (in pandas 0.x) depends on the presence
 # of nulls
@@ -186,7 +162,7 @@ cdef class Field:
     cdef init(self, const shared_ptr[CField]& field):
         self.sp_field = field
         self.field = field.get()
-        self.type = box_data_type(field.get().type())
+        self.type = pyarrow_wrap_data_type(field.get().type())
 
     def equals(self, Field other):
         """
@@ -244,7 +220,7 @@ cdef class Field:
         with nogil:
             check_status(self.field.AddMetadata(c_meta, &new_field))
 
-        return box_field(new_field)
+        return pyarrow_wrap_field(new_field)
 
     def remove_metadata(self):
         """
@@ -257,7 +233,7 @@ cdef class Field:
         cdef shared_ptr[CField] new_field
         with nogil:
             new_field = self.field.RemoveMetadata()
-        return box_field(new_field)
+        return pyarrow_wrap_field(new_field)
 
 
 cdef class Schema:
@@ -274,7 +250,7 @@ cdef class Schema:
 
         cdef Field result = Field()
         result.init(self.schema.field(i))
-        result.type = box_data_type(result.field.type())
+        result.type = pyarrow_wrap_data_type(result.field.type())
 
         return result
 
@@ -322,7 +298,7 @@ cdef class Schema:
         -------
         field: pyarrow.Field
         """
-        return box_field(self.schema.GetFieldByName(tobytes(name)))
+        return pyarrow_wrap_field(self.schema.GetFieldByName(tobytes(name)))
 
     def add_metadata(self, dict metadata):
         """
@@ -344,7 +320,7 @@ cdef class Schema:
         with nogil:
             check_status(self.schema.AddMetadata(c_meta, &new_schema))
 
-        return box_schema(new_schema)
+        return pyarrow_wrap_schema(new_schema)
 
     def remove_metadata(self):
         """
@@ -357,7 +333,7 @@ cdef class Schema:
         cdef shared_ptr[CSchema] new_schema
         with nogil:
             new_schema = self.schema.RemoveMetadata()
-        return box_schema(new_schema)
+        return pyarrow_wrap_schema(new_schema)
 
     def __str__(self):
         return frombytes(self.schema.ToString())
@@ -383,7 +359,7 @@ cdef DataType primitive_type(Type type):
         return _type_cache[type]
 
     cdef DataType out = DataType()
-    out.init(pyarrow.GetPrimitiveType(type))
+    out.init(GetPrimitiveType(type))
 
     _type_cache[type] = out
     return out
@@ -604,7 +580,7 @@ def float64():
 cpdef DataType decimal(int precision, int scale=0):
     cdef shared_ptr[CDataType] decimal_type
     decimal_type.reset(new CDecimalType(precision, scale))
-    return box_data_type(decimal_type)
+    return pyarrow_wrap_data_type(decimal_type)
 
 
 def string():
@@ -629,7 +605,7 @@ def binary(int length=-1):
 
     cdef shared_ptr[CDataType] fixed_size_binary_type
     fixed_size_binary_type.reset(new CFixedSizeBinaryType(length))
-    return box_data_type(fixed_size_binary_type)
+    return pyarrow_wrap_data_type(fixed_size_binary_type)
 
 
 def list_(DataType value_type):
@@ -695,49 +671,15 @@ def schema(fields):
     return result
 
 
-cdef DataType box_data_type(const shared_ptr[CDataType]& type):
-    cdef:
-        DataType out
-
-    if type.get() == NULL:
-        return None
-
-    if type.get().id() == _Type_DICTIONARY:
-        out = DictionaryType()
-    elif type.get().id() == _Type_TIMESTAMP:
-        out = TimestampType()
-    elif type.get().id() == _Type_FIXED_SIZE_BINARY:
-        out = FixedSizeBinaryType()
-    elif type.get().id() == _Type_DECIMAL:
-        out = DecimalType()
-    else:
-        out = DataType()
-
-    out.init(type)
-    return out
-
-cdef Field box_field(const shared_ptr[CField]& field):
-    if field.get() == NULL:
-        return None
-    cdef Field out = Field()
-    out.init(field)
-    return out
-
-cdef Schema box_schema(const shared_ptr[CSchema]& type):
-    cdef Schema out = Schema()
-    out.init_schema(type)
-    return out
-
-
 def from_numpy_dtype(object dtype):
     """
     Convert NumPy dtype to pyarrow.DataType
     """
     cdef shared_ptr[CDataType] c_type
     with nogil:
-        check_status(pyarrow.NumPyDtypeToArrow(dtype, &c_type))
+        check_status(NumPyDtypeToArrow(dtype, &c_type))
 
-    return box_data_type(c_type)
+    return pyarrow_wrap_data_type(c_type)
 
 
 NA = None
@@ -960,7 +902,7 @@ cdef class ListValue(ArrayValue):
     cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
         self.sp_array = sp_array
         self.ap = <CListArray*> sp_array.get()
-        self.value_type = box_data_type(self.ap.value_type())
+        self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
 
     cdef getitem(self, int64_t i):
         cdef int64_t j = self.ap.value_offset(self.index) + i
@@ -1076,15 +1018,15 @@ def array(object sequence, DataType type=None, MemoryPool memory_pool=None):
 
     pool = maybe_unbox_memory_pool(memory_pool)
     if type is None:
-        check_status(pyarrow.ConvertPySequence(sequence, pool, &sp_array))
+        check_status(ConvertPySequence(sequence, pool, &sp_array))
     else:
         check_status(
-            pyarrow.ConvertPySequence(
+            ConvertPySequence(
                 sequence, pool, &sp_array, type.sp_type
             )
         )
 
-    return box_array(sp_array)
+    return pyarrow_wrap_array(sp_array)
 
 
 
@@ -1093,7 +1035,7 @@ cdef class Array:
     cdef init(self, const shared_ptr[CArray]& sp_array):
         self.sp_array = sp_array
         self.ap = sp_array.get()
-        self.type = box_data_type(self.sp_array.get().type())
+        self.type = pyarrow_wrap_data_type(self.sp_array.get().type())
 
     @staticmethod
     def from_pandas(obj, mask=None, DataType type=None,
@@ -1172,22 +1114,22 @@ cdef class Array:
             if type is not None:
                 c_type = type.sp_type
             with nogil:
-                check_status(pyarrow.PandasObjectsToArrow(
+                check_status(PandasObjectsToArrow(
                     pool, values, mask, c_type, &out))
         else:
             values, type = maybe_coerce_datetime64(
                 values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
 
             if type is None:
-                check_status(pyarrow.NumPyDtypeToArrow(values.dtype, &c_type))
+                check_status(NumPyDtypeToArrow(values.dtype, &c_type))
             else:
                 c_type = type.sp_type
 
             with nogil:
-                check_status(pyarrow.PandasToArrow(
+                check_status(PandasToArrow(
                     pool, values, mask, c_type, &out))
 
-        return box_array(out)
+        return pyarrow_wrap_array(out)
 
     property null_count:
 
@@ -1271,7 +1213,7 @@ cdef class Array:
         else:
             result = self.ap.Slice(offset, length)
 
-        return box_array(result)
+        return pyarrow_wrap_array(result)
 
     def to_pandas(self):
         """
@@ -1287,8 +1229,7 @@ cdef class Array:
             PyObject* out
 
         with nogil:
-            check_status(
-                pyarrow.ConvertArrayToPandas(self.sp_array, self, &out))
+            check_status(ConvertArrayToPandas(self.sp_array, self, &out))
         return wrap_array_output(out)
 
     def to_pylist(self):
@@ -1303,7 +1244,7 @@ cdef class Tensor:
     cdef init(self, const shared_ptr[CTensor]& sp_tensor):
         self.sp_tensor = sp_tensor
         self.tp = sp_tensor.get()
-        self.type = box_data_type(self.tp.type())
+        self.type = pyarrow_wrap_data_type(self.tp.type())
 
     def __repr__(self):
         return """<pyarrow.Tensor>
@@ -1314,9 +1255,8 @@ strides: {2}""".format(self.type, self.shape, self.strides)
     @staticmethod
     def from_numpy(obj):
         cdef shared_ptr[CTensor] ctensor
-        check_status(pyarrow.NdarrayToTensor(default_memory_pool(),
-                                             obj, &ctensor))
-        return box_tensor(ctensor)
+        check_status(NdarrayToTensor(c_default_memory_pool(), obj, &ctensor))
+        return pyarrow_wrap_tensor(ctensor)
 
     def to_numpy(self):
         """
@@ -1325,7 +1265,7 @@ strides: {2}""".format(self.type, self.shape, self.strides)
         cdef:
             PyObject* out
 
-        check_status(pyarrow.TensorToNdarray(deref(self.tp), self, &out))
+        check_status(TensorToNdarray(deref(self.tp), self, &out))
         return PyObject_to_object(out)
 
     def equals(self, Tensor other):
@@ -1502,7 +1442,7 @@ cdef class DictionaryArray(Array):
             cdef CDictionaryArray* darr = <CDictionaryArray*>(self.ap)
 
             if self._dictionary is None:
-                self._dictionary = box_array(darr.dictionary())
+                self._dictionary = pyarrow_wrap_array(darr.dictionary())
 
             return self._dictionary
 
@@ -1512,7 +1452,7 @@ cdef class DictionaryArray(Array):
             cdef CDictionaryArray* darr = <CDictionaryArray*>(self.ap)
 
             if self._indices is None:
-                self._indices = box_array(darr.indices())
+                self._indices = pyarrow_wrap_array(darr.indices())
 
             return self._indices
 
@@ -1597,28 +1537,6 @@ cdef dict _array_classes = {
     _Type_DECIMAL: DecimalArray,
 }
 
-cdef object box_array(const shared_ptr[CArray]& sp_array):
-    if sp_array.get() == NULL:
-        raise ValueError('Array was NULL')
-
-    cdef CDataType* data_type = sp_array.get().type().get()
-
-    if data_type == NULL:
-        raise ValueError('Array data type was NULL')
-
-    cdef Array arr = _array_classes[data_type.id()]()
-    arr.init(sp_array)
-    return arr
-
-
-cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor):
-    if sp_tensor.get() == NULL:
-        raise ValueError('Tensor was NULL')
-
-    cdef Tensor tensor = Tensor()
-    tensor.init(sp_tensor)
-    return tensor
-
 
 cdef object get_series_values(object obj):
     if isinstance(obj, PandasSeries):
diff --git a/python/pyarrow/_error.pyx b/python/pyarrow/error.pxi
similarity index 100%
rename from python/pyarrow/_error.pyx
rename to python/pyarrow/error.pxi
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index c7b118e60a46d..3754aec737299 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -22,9 +22,9 @@
 import pandas as pd
 
 from pyarrow.compat import pdapi
-from pyarrow._io import FeatherError  # noqa
-from pyarrow._table import Table
-import pyarrow._io as ext
+from pyarrow.lib import FeatherError  # noqa
+from pyarrow.lib import Table
+import pyarrow.lib as ext
 
 
 if LooseVersion(pd.__version__) < '0.17.0':
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 92dd91ce9de07..ac37fd87294c5 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -19,7 +19,7 @@
 import os
 
 from pyarrow.util import implements
-import pyarrow._io as io
+import pyarrow.lib as lib
 
 
 class Filesystem(object):
@@ -133,7 +133,7 @@ def open(self, path, mode='rb'):
         return open(path, mode=mode)
 
 
-class HdfsClient(io._HdfsClient, Filesystem):
+class HdfsClient(lib._HdfsClient, Filesystem):
     """
     Connect to an HDFS cluster. All parameters are optional and should
     only be set if the defaults need to be overridden.
@@ -168,19 +168,19 @@ def __init__(self, host="default", port=0, user=None, kerb_ticket=None,
 
     @implements(Filesystem.isdir)
     def isdir(self, path):
-        return io._HdfsClient.isdir(self, path)
+        return lib._HdfsClient.isdir(self, path)
 
     @implements(Filesystem.isfile)
     def isfile(self, path):
-        return io._HdfsClient.isfile(self, path)
+        return lib._HdfsClient.isfile(self, path)
 
     @implements(Filesystem.delete)
     def delete(self, path, recursive=False):
-        return io._HdfsClient.delete(self, path, recursive)
+        return lib._HdfsClient.delete(self, path, recursive)
 
     @implements(Filesystem.mkdir)
     def mkdir(self, path, create_parents=True):
-        return io._HdfsClient.mkdir(self, path)
+        return lib._HdfsClient.mkdir(self, path)
 
     def ls(self, path, full_info=False):
         """
@@ -196,4 +196,4 @@ def ls(self, path, full_info=False):
         -------
         result : list of dicts (full_info=True) or strings (full_info=False)
         """
-        return io._HdfsClient.ls(self, path, full_info)
+        return lib._HdfsClient.ls(self, path, full_info)
diff --git a/python/pyarrow/formatting.py b/python/pyarrow/formatting.py
index c3583448d6e17..0af2873b29393 100644
--- a/python/pyarrow/formatting.py
+++ b/python/pyarrow/formatting.py
@@ -17,7 +17,7 @@
 
 # Pretty-printing and other formatting utilities for Arrow data structures
 
-import pyarrow._array as _array
+import pyarrow.lib as lib
 
 
 def array_format(arr, window=None):
@@ -42,7 +42,7 @@ def array_format(arr, window=None):
 
 
 def value_format(x, indent_level=0):
-    if isinstance(x, _array.ListValue):
+    if isinstance(x, lib.ListValue):
         contents = ',\n'.join(value_format(item) for item in x)
         return '[{0}]'.format(_indent(contents, 1).strip())
     else:
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 8a730b3988441..3d56c14bae219 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -149,7 +149,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         PoolBuffer()
         PoolBuffer(CMemoryPool*)
 
-    cdef CMemoryPool* default_memory_pool()
+    cdef CMemoryPool* c_default_memory_pool" arrow::default_memory_pool"()
 
     cdef cppclass CListType" arrow::ListType"(CDataType):
         CListType(const shared_ptr[CDataType]& value_type)
@@ -625,3 +625,62 @@ cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
 
         CStatus GetColumn(int i, shared_ptr[CColumn]* out)
         c_string GetColumnName(int i)
+
+
+cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
+    shared_ptr[CDataType] GetPrimitiveType(Type type)
+    shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
+    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
+                              shared_ptr[CArray]* out)
+    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
+                              shared_ptr[CArray]* out,
+                              const shared_ptr[CDataType]& type)
+
+    CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
+
+    CStatus PandasToArrow(CMemoryPool* pool, object ao, object mo,
+                          const shared_ptr[CDataType]& type,
+                          shared_ptr[CArray]* out)
+
+    CStatus PandasObjectsToArrow(CMemoryPool* pool, object ao, object mo,
+                                 const shared_ptr[CDataType]& type,
+                                 shared_ptr[CArray]* out)
+
+    CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
+                            shared_ptr[CTensor]* out);
+
+    CStatus TensorToNdarray(const CTensor& tensor, object base,
+                            PyObject** out)
+
+    CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
+                                 object py_ref, PyObject** out)
+
+    CStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
+                                  object py_ref, PyObject** out)
+
+    CStatus ConvertTableToPandas(const shared_ptr[CTable]& table,
+                                 int nthreads, PyObject** out)
+
+    void c_set_default_memory_pool \
+        " arrow::py::set_default_memory_pool"(CMemoryPool* pool)\
+
+    CMemoryPool* c_get_memory_pool \
+        " arrow::py::get_memory_pool"()
+
+    cdef cppclass PyBuffer(CBuffer):
+        PyBuffer(object o)
+
+    cdef cppclass PyReadableFile(RandomAccessFile):
+        PyReadableFile(object fo)
+
+    cdef cppclass PyOutputStream(OutputStream):
+        PyOutputStream(object fo)
+
+    cdef cppclass PyBytesReader(CBufferReader):
+        PyBytesReader(object fo)
+
+cdef extern from 'arrow/python/init.h':
+    int arrow_init_numpy() except -1
+
+cdef extern from 'arrow/python/config.h' namespace 'arrow::py':
+    void set_numpy_nan(object o)
diff --git a/python/pyarrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd
deleted file mode 100644
index 35c71107f8db1..0000000000000
--- a/python/pyarrow/includes/pyarrow.pxd
+++ /dev/null
@@ -1,75 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# distutils: language = c++
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn, CDataType,
-                                        CTable, CTensor, CStatus, Type,
-                                        CMemoryPool, TimeUnit,
-                                        RandomAccessFile, OutputStream,
-                                        CBufferReader)
-
-
-cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
-    shared_ptr[CDataType] GetPrimitiveType(Type type)
-    shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
-    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
-                              shared_ptr[CArray]* out)
-    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
-                              shared_ptr[CArray]* out,
-                              const shared_ptr[CDataType]& type)
-
-    CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
-
-    CStatus PandasToArrow(CMemoryPool* pool, object ao, object mo,
-                          const shared_ptr[CDataType]& type,
-                          shared_ptr[CArray]* out)
-
-    CStatus PandasObjectsToArrow(CMemoryPool* pool, object ao, object mo,
-                                 const shared_ptr[CDataType]& type,
-                                 shared_ptr[CArray]* out)
-
-    CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
-                            shared_ptr[CTensor]* out);
-
-    CStatus TensorToNdarray(const CTensor& tensor, object base,
-                            PyObject** out)
-
-    CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
-                                 object py_ref, PyObject** out)
-
-    CStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
-                                  object py_ref, PyObject** out)
-
-    CStatus ConvertTableToPandas(const shared_ptr[CTable]& table,
-                                 int nthreads, PyObject** out)
-
-    void set_default_memory_pool(CMemoryPool* pool)
-    CMemoryPool* get_memory_pool()
-
-    cdef cppclass PyBuffer(CBuffer):
-        PyBuffer(object o)
-
-    cdef cppclass PyReadableFile(RandomAccessFile):
-        PyReadableFile(object fo)
-
-    cdef cppclass PyOutputStream(OutputStream):
-        PyOutputStream(object fo)
-
-    cdef cppclass PyBytesReader(CBufferReader):
-        PyBytesReader(object fo)
diff --git a/python/pyarrow/_io.pyx b/python/pyarrow/io.pxi
similarity index 96%
rename from python/pyarrow/_io.pyx
rename to python/pyarrow/io.pxi
index e9e2ba01c0678..a0a96e72864d4 100644
--- a/python/pyarrow/_io.pyx
+++ b/python/pyarrow/io.pxi
@@ -18,22 +18,7 @@
 # Cython wrappers for IO interfaces defined in arrow::io and messaging in
 # arrow::ipc
 
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from cython.operator cimport dereference as deref
 from libc.stdlib cimport malloc, free
-from pyarrow.includes.libarrow cimport *
-cimport pyarrow.includes.pyarrow as pyarrow
-from pyarrow._array cimport Array, Tensor, box_tensor, Schema
-from pyarrow._error cimport check_status
-from pyarrow._memory cimport MemoryPool, maybe_unbox_memory_pool
-from pyarrow._table cimport (Column, RecordBatch, batch_from_cbatch,
-                             table_from_ctable)
-cimport cpython as cp
-
-import pyarrow._config
 from pyarrow.compat import frombytes, tobytes, encode_file_path
 
 import re
@@ -52,6 +37,7 @@ cdef extern from "Python.h":
     PyObject* PyBytes_FromStringAndSizeNative" PyBytes_FromStringAndSize"(
         char *v, Py_ssize_t len) except NULL
 
+
 cdef class NativeFile:
 
     def __cinit__(self):
@@ -177,7 +163,7 @@ cdef class NativeFile:
         with nogil:
             check_status(self.rd_file.get().ReadB(c_nbytes, &output))
 
-        return wrap_buffer(output)
+        return pyarrow_wrap_buffer(output)
 
     def download(self, stream_or_path, buffer_size=None):
         """
@@ -315,11 +301,11 @@ cdef class PythonFile(NativeFile):
         self.handle = handle
 
         if mode.startswith('w'):
-            self.wr_file.reset(new pyarrow.PyOutputStream(handle))
+            self.wr_file.reset(new PyOutputStream(handle))
             self.is_readable = 0
             self.is_writeable = 1
         elif mode.startswith('r'):
-            self.rd_file.reset(new pyarrow.PyReadableFile(handle))
+            self.rd_file.reset(new PyReadableFile(handle))
             self.is_readable = 1
             self.is_writeable = 0
         else:
@@ -497,7 +483,7 @@ cdef class Buffer:
             if parent_buf.get() == NULL:
                 return None
             else:
-                return wrap_buffer(parent_buf)
+                return pyarrow_wrap_buffer(parent_buf)
 
     def __getitem__(self, key):
         # TODO(wesm): buffer slicing
@@ -545,7 +531,7 @@ cdef class InMemoryOutputStream(NativeFile):
     def get_result(self):
         check_status(self.wr_file.get().Close())
         self.is_open = False
-        return wrap_buffer(<shared_ptr[CBuffer]> self.buffer)
+        return pyarrow_wrap_buffer(<shared_ptr[CBuffer]> self.buffer)
 
 
 cdef class BufferReader(NativeFile):
@@ -554,7 +540,7 @@ cdef class BufferReader(NativeFile):
 
     Parameters
     ----------
-    obj : Python bytes or pyarrow.io.Buffer
+    obj : Python bytes or pyarrow.Buffer
     """
     cdef:
         Buffer buffer
@@ -579,19 +565,12 @@ def frombuffer(object obj):
     cdef shared_ptr[CBuffer] buf
     try:
         memoryview(obj)
-        buf.reset(new pyarrow.PyBuffer(obj))
-        return wrap_buffer(buf)
+        buf.reset(new PyBuffer(obj))
+        return pyarrow_wrap_buffer(buf)
     except TypeError:
         raise ValueError('Must pass object that implements buffer protocol')
 
 
-
-cdef Buffer wrap_buffer(const shared_ptr[CBuffer]& buf):
-    cdef Buffer result = Buffer()
-    result.init(buf)
-    return result
-
-
 cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
     cdef NativeFile nf
 
@@ -1007,7 +986,7 @@ cdef class _StreamReader:
         if batch.get() == NULL:
             raise StopIteration
 
-        return batch_from_cbatch(batch)
+        return pyarrow_wrap_batch(batch)
 
     def read_all(self):
         """
@@ -1027,7 +1006,7 @@ cdef class _StreamReader:
 
             check_status(CTable.FromRecordBatches(batches, &table))
 
-        return table_from_ctable(table)
+        return pyarrow_wrap_table(table)
 
 
 cdef class _FileWriter(_StreamWriter):
@@ -1080,7 +1059,7 @@ cdef class _FileReader:
         with nogil:
             check_status(self.reader.get().GetRecordBatch(i, &batch))
 
-        return batch_from_cbatch(batch)
+        return pyarrow_wrap_batch(batch)
 
     # TODO(wesm): ARROW-503: Function was renamed. Remove after a period of
     # time has passed
@@ -1103,7 +1082,7 @@ cdef class _FileReader:
                 check_status(self.reader.get().GetRecordBatch(i, &batches[i]))
             check_status(CTable.FromRecordBatches(batches, &table))
 
-        return table_from_ctable(table)
+        return pyarrow_wrap_table(table)
 
 
 #----------------------------------------------------------------------
@@ -1271,4 +1250,4 @@ def read_tensor(NativeFile source):
     with nogil:
         check_status(ReadTensor(offset, source.rd_file.get(), &sp_tensor))
 
-    return box_tensor(sp_tensor)
+    return pyarrow_wrap_tensor(sp_tensor)
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index f96ead3b92346..c37a1ce7df1e6 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -17,10 +17,10 @@
 
 # Arrow file and stream reader/writer classes, and other messaging tools
 
-import pyarrow._io as _io
+import pyarrow.lib as lib
 
 
-class StreamReader(_io._StreamReader):
+class StreamReader(lib._StreamReader):
     """
     Reader for the Arrow streaming binary format
 
@@ -37,7 +37,7 @@ def __iter__(self):
             yield self.get_next_batch()
 
 
-class StreamWriter(_io._StreamWriter):
+class StreamWriter(lib._StreamWriter):
     """
     Writer for the Arrow streaming binary format
 
@@ -52,7 +52,7 @@ def __init__(self, sink, schema):
         self._open(sink, schema)
 
 
-class FileReader(_io._FileReader):
+class FileReader(lib._FileReader):
     """
     Class for reading Arrow record batch data from the Arrow binary file format
 
@@ -68,7 +68,7 @@ def __init__(self, source, footer_offset=None):
         self._open(source, footer_offset=footer_offset)
 
 
-class FileWriter(_io._FileWriter):
+class FileWriter(lib._FileWriter):
     """
     Writer to create the Arrow binary file format
 
diff --git a/python/pyarrow/_array.pxd b/python/pyarrow/lib.pxd
similarity index 61%
rename from python/pyarrow/_array.pxd
rename to python/pyarrow/lib.pxd
index 464de316f0437..d3d03aaaefaa1 100644
--- a/python/pyarrow/_array.pxd
+++ b/python/pyarrow/lib.pxd
@@ -17,13 +17,32 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
-
 from cpython cimport PyObject
 
 cdef extern from "Python.h":
     int PySlice_Check(object)
 
 
+from pyarrow.includes.libarrow cimport CStatus
+
+
+cdef int check_status(const CStatus& status) nogil except -1
+
+
+cdef class MemoryPool:
+    cdef:
+        CMemoryPool* pool
+
+    cdef init(self, CMemoryPool* pool)
+
+
+cdef class LoggingMemoryPool(MemoryPool):
+    pass
+
+
+cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool)
+
+
 cdef class DataType:
     cdef:
         shared_ptr[CDataType] sp_type
@@ -237,11 +256,82 @@ cdef class DictionaryArray(Array):
 
 
 cdef wrap_array_output(PyObject* output)
-cdef DataType box_data_type(const shared_ptr[CDataType]& type)
-cdef Field box_field(const shared_ptr[CField]& field)
-cdef Schema box_schema(const shared_ptr[CSchema]& schema)
-cdef object box_array(const shared_ptr[CArray]& sp_array)
-cdef object box_tensor(const shared_ptr[CTensor]& sp_tensor)
 cdef object box_scalar(DataType type,
                        const shared_ptr[CArray]& sp_array,
                        int64_t index)
+
+
+cdef class ChunkedArray:
+    cdef:
+        shared_ptr[CChunkedArray] sp_chunked_array
+        CChunkedArray* chunked_array
+
+    cdef init(self, const shared_ptr[CChunkedArray]& chunked_array)
+    cdef _check_nullptr(self)
+
+
+cdef class Column:
+    cdef:
+        shared_ptr[CColumn] sp_column
+        CColumn* column
+
+    cdef init(self, const shared_ptr[CColumn]& column)
+    cdef _check_nullptr(self)
+
+
+cdef class Table:
+    cdef:
+        shared_ptr[CTable] sp_table
+        CTable* table
+
+    cdef init(self, const shared_ptr[CTable]& table)
+    cdef _check_nullptr(self)
+
+
+cdef class RecordBatch:
+    cdef:
+        shared_ptr[CRecordBatch] sp_batch
+        CRecordBatch* batch
+        Schema _schema
+
+    cdef init(self, const shared_ptr[CRecordBatch]& table)
+    cdef _check_nullptr(self)
+
+
+cdef class Buffer:
+    cdef:
+        shared_ptr[CBuffer] buffer
+        Py_ssize_t shape[1]
+        Py_ssize_t strides[1]
+
+    cdef init(self, const shared_ptr[CBuffer]& buffer)
+
+
+cdef class NativeFile:
+    cdef:
+        shared_ptr[RandomAccessFile] rd_file
+        shared_ptr[OutputStream] wr_file
+        bint is_readable
+        bint is_writeable
+        bint is_open
+        bint own_file
+
+    # By implementing these "virtual" functions (all functions in Cython
+    # extension classes are technically virtual in the C++ sense) we can expose
+    # the arrow::io abstract file interfaces to other components throughout the
+    # suite of Arrow C++ libraries
+    cdef read_handle(self, shared_ptr[RandomAccessFile]* file)
+    cdef write_handle(self, shared_ptr[OutputStream]* file)
+
+cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader)
+cdef get_writer(object source, shared_ptr[OutputStream]* writer)
+
+cdef public object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf)
+cdef public object pyarrow_wrap_data_type(const shared_ptr[CDataType]& type)
+cdef public object pyarrow_wrap_field(const shared_ptr[CField]& field)
+cdef public object pyarrow_wrap_schema(const shared_ptr[CSchema]& type)
+cdef public object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array)
+cdef public object pyarrow_wrap_tensor(const shared_ptr[CTensor]& sp_tensor)
+cdef public object pyarrow_wrap_column(const shared_ptr[CColumn]& ccolumn)
+cdef public object pyarrow_wrap_table(const shared_ptr[CTable]& ctable)
+cdef public object pyarrow_wrap_batch(const shared_ptr[CRecordBatch]& cbatch)
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
new file mode 100644
index 0000000000000..ae311aca8d0cc
--- /dev/null
+++ b/python/pyarrow/lib.pyx
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from cython.operator cimport dereference as deref
+from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.common cimport PyObject_to_object
+cimport pyarrow.includes.libarrow as libarrow
+cimport cpython as cp
+
+
+import datetime
+import decimal as _pydecimal
+import numpy as np
+import six
+from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
+
+cdef _pandas():
+    import pandas as pd
+    return pd
+
+
+arrow_init_numpy()
+
+import numpy as np
+set_numpy_nan(np.nan)
+
+import multiprocessing
+import os
+cdef int CPU_COUNT = int(
+    os.environ.get('OMP_NUM_THREADS',
+                   max(multiprocessing.cpu_count() // 2, 1)))
+
+
+def cpu_count():
+    """
+    Returns
+    -------
+    count : Number of CPUs to use by default in parallel operations. Default is
+      max(1, multiprocessing.cpu_count() / 2), but can be overridden by the
+      OMP_NUM_THREADS environment variable. For the default, we divide the CPU
+      count by 2 because most modern computers have hyperthreading turned on,
+      so doubling the CPU count beyond the number of physical cores does not
+      help.
+    """
+    return CPU_COUNT
+
+def set_cpu_count(count):
+    global CPU_COUNT
+    CPU_COUNT = max(int(count), 1)
+
+
+# Exception types
+include "error.pxi"
+
+# Memory pools and allocation
+include "memory.pxi"
+
+# Array types
+include "array.pxi"
+
+# Column, Table, Record Batch
+include "table.pxi"
+
+# File IO, IPC
+include "io.pxi"
+
+#----------------------------------------------------------------------
+# Public API
+
+include "public-api.pxi"
diff --git a/python/pyarrow/_memory.pyx b/python/pyarrow/memory.pxi
similarity index 82%
rename from python/pyarrow/_memory.pyx
rename to python/pyarrow/memory.pxi
index 8b73a17553edf..15d59d237ad6d 100644
--- a/python/pyarrow/_memory.pyx
+++ b/python/pyarrow/memory.pxi
@@ -19,9 +19,6 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
-from pyarrow.includes.pyarrow cimport set_default_memory_pool, get_memory_pool
-
 
 cdef class MemoryPool:
     cdef init(self, CMemoryPool* pool):
@@ -33,7 +30,7 @@ cdef class MemoryPool:
 
 cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool):
     if memory_pool is None:
-        return get_memory_pool()
+        return c_get_memory_pool()
     else:
         return memory_pool.pool
 
@@ -45,14 +42,14 @@ cdef class LoggingMemoryPool(MemoryPool):
 def default_memory_pool():
     cdef:
         MemoryPool pool = MemoryPool()
-    pool.init(get_memory_pool())
+    pool.init(c_get_memory_pool())
     return pool
 
 
 def set_memory_pool(MemoryPool pool):
-    set_default_memory_pool(pool.pool)
+    c_set_default_memory_pool(pool.pool)
 
 
 def total_allocated_bytes():
-    cdef CMemoryPool* pool = get_memory_pool()
+    cdef CMemoryPool* pool = c_get_memory_pool()
     return pool.bytes_allocated()
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 21359f137f24e..050ec3176d799 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -24,8 +24,7 @@
                               RowGroupMetaData, ParquetSchema,
                               ParquetWriter)
 import pyarrow._parquet as _parquet  # noqa
-import pyarrow._array as _array
-import pyarrow._table as _table
+import pyarrow.lib as lib
 
 
 # ----------------------------------------------------------------------
@@ -241,8 +240,8 @@ def read(self, columns=None, nthreads=1, partitions=None,
                 # manifest, so ['a', 'b', 'c'] as in our example above.
                 dictionary = partitions.levels[i].dictionary
 
-                arr = _array.DictionaryArray.from_arrays(indices, dictionary)
-                col = _table.Column.from_array(name, arr)
+                arr = lib.DictionaryArray.from_arrays(indices, dictionary)
+                col = lib.Column.from_array(name, arr)
                 table = table.append_column(col)
 
         return table
@@ -298,9 +297,9 @@ def dictionary(self):
         # Only integer and string partition types are supported right now
         try:
             integer_keys = [int(x) for x in self.keys]
-            dictionary = _array.array(integer_keys)
+            dictionary = lib.array(integer_keys)
         except ValueError:
-            dictionary = _array.array(self.keys)
+            dictionary = lib.array(self.keys)
 
         self._dictionary = dictionary
         return dictionary
@@ -539,7 +538,7 @@ def read(self, columns=None, nthreads=1):
                                open_file_func=open_file)
             tables.append(table)
 
-        all_data = _table.concat_tables(tables)
+        all_data = lib.concat_tables(tables)
         return all_data
 
     def _get_open_file_func(self):
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
new file mode 100644
index 0000000000000..7b5565124339b
--- /dev/null
+++ b/python/pyarrow/public-api.pxi
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from libcpp.memory cimport shared_ptr
+from pyarrow.includes.libarrow cimport (CArray, CColumn, CDataType, CField,
+                                        CRecordBatch, CSchema,
+                                        CTable, CTensor)
+
+
+cdef public api object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf):
+    cdef Buffer result = Buffer()
+    result.init(buf)
+    return result
+
+
+cdef public api object pyarrow_wrap_data_type(
+    const shared_ptr[CDataType]& type):
+    cdef:
+        DataType out
+
+    if type.get() == NULL:
+        return None
+
+    if type.get().id() == _Type_DICTIONARY:
+        out = DictionaryType()
+    elif type.get().id() == _Type_TIMESTAMP:
+        out = TimestampType()
+    elif type.get().id() == _Type_FIXED_SIZE_BINARY:
+        out = FixedSizeBinaryType()
+    elif type.get().id() == _Type_DECIMAL:
+        out = DecimalType()
+    else:
+        out = DataType()
+
+    out.init(type)
+    return out
+
+
+cdef public api object pyarrow_wrap_field(const shared_ptr[CField]& field):
+    if field.get() == NULL:
+        return None
+    cdef Field out = Field()
+    out.init(field)
+    return out
+
+
+cdef public api object pyarrow_wrap_schema(const shared_ptr[CSchema]& type):
+    cdef Schema out = Schema()
+    out.init_schema(type)
+    return out
+
+
+cdef public api object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array):
+    if sp_array.get() == NULL:
+        raise ValueError('Array was NULL')
+
+    cdef CDataType* data_type = sp_array.get().type().get()
+
+    if data_type == NULL:
+        raise ValueError('Array data type was NULL')
+
+    cdef Array arr = _array_classes[data_type.id()]()
+    arr.init(sp_array)
+    return arr
+
+
+cdef public api object pyarrow_wrap_tensor(
+    const shared_ptr[CTensor]& sp_tensor):
+    if sp_tensor.get() == NULL:
+        raise ValueError('Tensor was NULL')
+
+    cdef Tensor tensor = Tensor()
+    tensor.init(sp_tensor)
+    return tensor
+
+
+cdef public api object pyarrow_wrap_column(const shared_ptr[CColumn]& ccolumn):
+    cdef Column column = Column()
+    column.init(ccolumn)
+    return column
+
+
+cdef public api object pyarrow_wrap_table(const shared_ptr[CTable]& ctable):
+    cdef Table table = Table()
+    table.init(ctable)
+    return table
+
+
+cdef public api object pyarrow_wrap_batch(
+    const shared_ptr[CRecordBatch]& cbatch):
+    cdef RecordBatch batch = RecordBatch()
+    batch.init(cbatch)
+    return batch
diff --git a/python/pyarrow/_table.pyx b/python/pyarrow/table.pxi
similarity index 90%
rename from python/pyarrow/_table.pyx
rename to python/pyarrow/table.pxi
index 223fe27ea9819..8dd18cf4136f8 100644
--- a/python/pyarrow/_table.pyx
+++ b/python/pyarrow/table.pxi
@@ -15,33 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from cython.operator cimport dereference as deref
-
-from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.common cimport *
-cimport pyarrow.includes.pyarrow as pyarrow
-from pyarrow._array cimport (Array, box_array, wrap_array_output,
-                             box_data_type, box_schema, DataType, Field)
-from pyarrow._error cimport check_status
-cimport cpython
-
-import pyarrow._config
-from pyarrow._error import ArrowException
-from pyarrow._array import field
-from pyarrow.compat import frombytes, tobytes
-
 from collections import OrderedDict
 
 
-cdef _pandas():
-    import pandas as pd
-    return pd
-
-
 cdef class ChunkedArray:
     """
     Array backed via one or more memory chunks.
@@ -104,10 +80,10 @@ cdef class ChunkedArray:
 
         Returns
         -------
-        pyarrow.array.Array
+        pyarrow.Array
         """
         self._check_nullptr()
-        return box_array(self.chunked_array.chunk(i))
+        return pyarrow_wrap_array(self.chunked_array.chunk(i))
 
     def iterchunks(self):
         for i in range(self.num_chunks):
@@ -150,7 +126,7 @@ cdef class Column:
 
         cdef shared_ptr[CColumn] sp_column
         sp_column.reset(new CColumn(boxed_field.sp_field, arr.sp_array))
-        return box_column(sp_column)
+        return pyarrow_wrap_column(sp_column)
 
     def to_pandas(self):
         """
@@ -163,8 +139,8 @@ cdef class Column:
         cdef:
             PyObject* out
 
-        check_status(pyarrow.ConvertColumnToPandas(self.sp_column,
-                                                   self, &out))
+        check_status(libarrow.ConvertColumnToPandas(self.sp_column,
+                                                    self, &out))
 
         return _pandas().Series(wrap_array_output(out), name=self.name)
 
@@ -254,9 +230,9 @@ cdef class Column:
 
         Returns
         -------
-        pyarrow.schema.DataType
+        pyarrow.DataType
         """
-        return box_data_type(self.column.type())
+        return pyarrow_wrap_data_type(self.column.type())
 
     @property
     def data(self):
@@ -265,7 +241,7 @@ cdef class Column:
 
         Returns
         -------
-        pyarrow.table.ChunkedArray
+        pyarrow.ChunkedArray
         """
         cdef ChunkedArray chunked_array = ChunkedArray()
         chunked_array.init(self.column.data())
@@ -396,7 +372,7 @@ cdef class RecordBatch:
 
         Returns
         -------
-        pyarrow.schema.Schema
+        pyarrow.Schema
         """
         cdef Schema schema
         self._check_nullptr()
@@ -408,7 +384,7 @@ cdef class RecordBatch:
         return self._schema
 
     def __getitem__(self, i):
-        return box_array(self.batch.column(i))
+        return pyarrow_wrap_array(self.batch.column(i))
 
     def slice(self, offset=0, length=None):
         """
@@ -436,7 +412,7 @@ cdef class RecordBatch:
         else:
             result = self.batch.Slice(offset, length)
 
-        return batch_from_cbatch(result)
+        return pyarrow_wrap_batch(result)
 
     def equals(self, RecordBatch other):
         cdef:
@@ -492,7 +468,7 @@ cdef class RecordBatch:
 
         Returns
         -------
-        pyarrow.table.RecordBatch
+        pyarrow.RecordBatch
         """
         names, arrays, metadata = _dataframe_to_arrays(df, False, schema)
         return cls.from_arrays(arrays, names, metadata)
@@ -511,7 +487,7 @@ cdef class RecordBatch:
 
         Returns
         -------
-        pyarrow.table.RecordBatch
+        pyarrow.RecordBatch
         """
         cdef:
             Array arr
@@ -534,7 +510,7 @@ cdef class RecordBatch:
             c_arrays.push_back(arr.sp_array)
 
         batch.reset(new CRecordBatch(schema, num_rows, c_arrays))
-        return batch_from_cbatch(batch)
+        return pyarrow_wrap_batch(batch)
 
 
 cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
@@ -548,8 +524,8 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
     from pyarrow.compat import DatetimeTZDtype
 
     with nogil:
-        check_status(pyarrow.ConvertTableToPandas(table, nthreads,
-                                                  &result_obj))
+        check_status(libarrow.ConvertTableToPandas(table, nthreads,
+                                                   &result_obj))
 
     result = PyObject_to_object(result_obj)
 
@@ -652,7 +628,7 @@ cdef class Table:
 
         Returns
         -------
-        pyarrow.table.Table
+        pyarrow.Table
 
         Examples
         --------
@@ -664,7 +640,7 @@ cdef class Table:
             ...     'str': ['a', 'b']
             ... })
         >>> pa.Table.from_pandas(df)
-        <pyarrow.table.Table object at 0x7f05d1fb1b40>
+        <pyarrow.lib.Table object at 0x7f05d1fb1b40>
         """
         names, arrays, metadata = _dataframe_to_arrays(df,
                                              timestamps_to_ms=timestamps_to_ms,
@@ -686,7 +662,7 @@ cdef class Table:
 
         Returns
         -------
-        pyarrow.table.Table
+        pyarrow.Table
 
         """
         cdef:
@@ -713,7 +689,7 @@ cdef class Table:
                 raise ValueError(type(arrays[i]))
 
         table.reset(new CTable(schema, columns))
-        return table_from_ctable(table)
+        return pyarrow_wrap_table(table)
 
     @staticmethod
     def from_batches(batches):
@@ -737,7 +713,7 @@ cdef class Table:
         with nogil:
             check_status(CTable.FromRecordBatches(c_batches, &c_table))
 
-        return table_from_ctable(c_table)
+        return pyarrow_wrap_table(c_table)
 
     def to_pandas(self, nthreads=None):
         """
@@ -755,7 +731,7 @@ cdef class Table:
         pandas.DataFrame
         """
         if nthreads is None:
-            nthreads = pyarrow._config.cpu_count()
+            nthreads = cpu_count()
 
         mgr = table_to_blockmanager(self.sp_table, nthreads)
         return _pandas().DataFrame(mgr)
@@ -782,9 +758,9 @@ cdef class Table:
 
         Returns
         -------
-        pyarrow.schema.Schema
+        pyarrow.Schema
         """
-        return box_schema(self.table.schema())
+        return pyarrow_wrap_schema(self.table.schema())
 
     def column(self, index):
         """
@@ -796,7 +772,7 @@ cdef class Table:
 
         Returns
         -------
-        pyarrow.table.Column
+        pyarrow.Column
         """
         self._check_nullptr()
         cdef Column column = Column()
@@ -863,7 +839,7 @@ cdef class Table:
         with nogil:
             check_status(self.table.AddColumn(i, column.sp_column, &c_table))
 
-        return table_from_ctable(c_table)
+        return pyarrow_wrap_table(c_table)
 
     def append_column(self, Column column):
         """
@@ -880,7 +856,7 @@ cdef class Table:
         with nogil:
             check_status(self.table.RemoveColumn(i, &c_table))
 
-        return table_from_ctable(c_table)
+        return pyarrow_wrap_table(c_table)
 
 
 def concat_tables(tables):
@@ -905,22 +881,4 @@ def concat_tables(tables):
     with nogil:
         check_status(ConcatenateTables(c_tables, &c_result))
 
-    return table_from_ctable(c_result)
-
-
-cdef object box_column(const shared_ptr[CColumn]& ccolumn):
-    cdef Column column = Column()
-    column.init(ccolumn)
-    return column
-
-
-cdef api object table_from_ctable(const shared_ptr[CTable]& ctable):
-    cdef Table table = Table()
-    table.init(ctable)
-    return table
-
-
-cdef api object batch_from_cbatch(const shared_ptr[CRecordBatch]& cbatch):
-    cdef RecordBatch batch = RecordBatch()
-    batch.init(cbatch)
-    return batch
+    return pyarrow_wrap_table(c_result)
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 7a8abf486f4b5..69c32be5f3df7 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -27,7 +27,7 @@
 from pyarrow.compat import guid
 from pyarrow.feather import (read_feather, write_feather,
                              FeatherReader)
-from pyarrow._io import FeatherWriter
+from pyarrow.lib import FeatherWriter
 
 
 def random_path():
diff --git a/python/setup.py b/python/setup.py
index 148224afb0574..b38fca47b2399 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -106,14 +106,9 @@ def initialize_options(self):
             os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
 
     CYTHON_MODULE_NAMES = [
-        '_array',
-        '_config',
-        '_error',
-        '_io',
+        'lib',
         '_jemalloc',
-        '_memory',
-        '_parquet',
-        '_table']
+        '_parquet']
 
     def _run_cmake(self):
         # The directory containing this setup.py

From dbbbc6655149465c787b590833cae2b6f4e5a9dc Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Sat, 13 May 2017 15:53:31 -0400
Subject: [PATCH 0642/1644] ARROW-988 [JS] Add entry to Travis CI matrix

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #677 from TheNeuralBit/js_travis and squashes the following commits:

3d5b54b5 [Brian Hulette] Revert "Testing a failure"
a84b3641 [Brian Hulette] Testing a failure
70310b68 [Brian Hulette] add flatbuffers download
e0c1100d [Brian Hulette] Add travis config and scripts for JS
---
 .travis.yml                   |  7 +++++++
 ci/travis_before_script_js.sh | 34 ++++++++++++++++++++++++++++++++++
 ci/travis_env_common.sh       |  1 +
 ci/travis_script_js.sh        | 23 +++++++++++++++++++++++
 4 files changed, 65 insertions(+)
 create mode 100755 ci/travis_before_script_js.sh
 create mode 100755 ci/travis_script_js.sh

diff --git a/.travis.yml b/.travis.yml
index e6941620c2c91..022ffb3562cb4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -74,6 +74,13 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
+  - language: node_js
+    os: linux
+    node_js: node
+    before_script:
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
 
 before_install:
 - ulimit -c unlimited -S
diff --git a/ci/travis_before_script_js.sh b/ci/travis_before_script_js.sh
new file mode 100755
index 0000000000000..304c48137aa55
--- /dev/null
+++ b/ci/travis_before_script_js.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+
+set -ex
+
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
+# Download flatbuffers
+export FLATBUFFERS_HOME=$TRAVIS_BUILD_DIR/flatbuffers
+conda create -y -q -p $FLATBUFFERS_HOME python=2.7 flatbuffers
+export PATH="$FLATBUFFERS_HOME/bin:$PATH"
+
+npm install -g typescript
+npm install -g webpack
+
+pushd $ARROW_JS_DIR
+
+npm install
+npm run build
+
+popd
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index 5593f0079f411..3515314722b71 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -25,6 +25,7 @@ export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp
 export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
 export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+export ARROW_JS_DIR=${TRAVIS_BUILD_DIR}/js
 export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
 
 export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
diff --git a/ci/travis_script_js.sh b/ci/travis_script_js.sh
new file mode 100755
index 0000000000000..52ac3b9bdf8a3
--- /dev/null
+++ b/ci/travis_script_js.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+set -e
+
+JS_DIR=${TRAVIS_BUILD_DIR}/js
+
+pushd $JS_DIR
+
+npm test
+
+popd

From 99ff2408932061ae6a5f5620669d79ae559b0f4c Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Sat, 13 May 2017 21:28:23 -0400
Subject: [PATCH 0643/1644] ARROW-1011: [FORMAT] fix typo and mistakes in
 Layout.md

according to the specification in the `Null bitmaps` section:
> Bitmaps are to be initialized to be all unset at allocation time (this includes padding).

null bitmaps should always be padded with 0

Author: Wenchen Fan <wenchen@databricks.com>

Closes #673 from cloud-fan/first and squashes the following commits:

8566f7c9 [Wenchen Fan] fix typo and mistakes in Layout.md
---
 format/Layout.md | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index 251af9dd8a128..b073854c5cd61 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -51,7 +51,7 @@ Base requirements
 
 * A physical memory layout enabling zero-deserialization data interchange
   amongst a variety of systems handling flat and nested columnar data, including
-  such systems as Spark, Drill, Impala, Kudu, Ibis, Spark, ODBC protocols, and
+  such systems as Spark, Drill, Impala, Kudu, Ibis, ODBC protocols, and
   proprietary systems that utilize the open source components.
 * All array slots are accessible in constant time, with complexity growing
   linearly in the nesting level
@@ -114,7 +114,7 @@ data-structures over 64 bytes (which will be a common case for Arrow Arrays).
 
 Requiring padding to a multiple of 64 bytes allows for using [SIMD][4] instructions
 consistently in loops without additional conditional checks.
-This should allow for simpler and more efficient code.  
+This should allow for simpler and more efficient code.
 The specific padding length was chosen because it matches the largest known
 SIMD instruction registers available as of April 2016 (Intel AVX-512).
 Guaranteed padding can also allow certain compilers
@@ -146,7 +146,7 @@ signed integer, as it may be as large as the array length.
 Any relative type can have null value slots, whether primitive or nested type.
 
 An array with nulls must have a contiguous memory buffer, known as the null (or
-validity) bitmap, whose length is a multiple of 64 bytes (as discussed above)  
+validity) bitmap, whose length is a multiple of 64 bytes (as discussed above)
 and large enough to have at least 1 bit for each array
 slot.
 
@@ -205,7 +205,7 @@ Would look like:
 
   |Byte 0 (validity bitmap) | Bytes 1-63            |
   |-------------------------|-----------------------|
-  |00011011                 | 0 (padding)           |
+  | 00011011                | 0 (padding)           |
 
 * Value Buffer:
 
@@ -378,16 +378,16 @@ The layout for [{'joe', 1}, {null, 2}, null, {'mark', 4}] would be:
 * Length: 4, Null count: 1
 * Null bitmap buffer:
 
-  | Byte 0 (validity bitmap) | Bytes 1-7   | Bytes 8-63  |
-  |--------------------------|-------------|-------------|
-  | 00001011                 | 0 (padding) | unspecified |
+  |Byte 0 (validity bitmap) | Bytes 1-63            |
+  |-------------------------|-----------------------|
+  | 00001011                | 0 (padding)           |
 
 * Children arrays:
   * field-0 array (`List<char>`):
     * Length: 4, Null count: 1
     * Null bitmap buffer:
 
-      | Byte 0 (validity bitmap) | Bytes 1-7             |
+      | Byte 0 (validity bitmap) | Bytes 1-63            |
       |--------------------------|-----------------------|
       | 00001101                 | 0 (padding)           |
 
@@ -447,7 +447,7 @@ of overhead for each value. Its physical layout is as follows:
 * One child array for each relative type
 * Types buffer: A buffer of 8-bit signed integers, enumerated from 0 corresponding
   to each type.  A union with more then 127 possible types can be modeled as a
-  union of unions. 
+  union of unions.
 * Offsets buffer: A buffer of signed int32 values indicating the relative offset
   into the respective child array for the type in a given slot. The respective
   offsets for each child value array must be in order / increasing.
@@ -555,7 +555,7 @@ will have the following layout:
 
       |Byte 0 (validity bitmap) | Bytes 1-63            |
       |-------------------------|-----------------------|
-      |00001010                 | 0 (padding)           |
+      | 00001010                | 0 (padding)           |
 
     * Value buffer:
 
@@ -636,7 +636,7 @@ type: List<String>
 
 ## References
 
-Apache Drill Documentation - [Value Vectors][6] 
+Apache Drill Documentation - [Value Vectors][6]
 
 [1]: https://en.wikipedia.org/wiki/Bit_numbering
 [2]: https://software.intel.com/en-us/articles/practical-intel-avx-optimization-on-2nd-generation-intel-core-processors

From 5739e04b35aeb5be9df7e9aace866ba48ecbac8a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 14 May 2017 08:55:26 -0400
Subject: [PATCH 0644/1644] ARROW-1008: [C++] Add abstract stream writer and
 reader C++ APIs. Give clearer names to IPC reader/writer classes

The main motivation for this patch was to make `StreamReader` and `StreamWriter` abstract, so that other implementations can be created. I would also like to add the option for asynchronous reading and writing.

I also added a CMake option `ARROW_NO_DEPRECATED_API` for more graceful name deprecations.

@kou do you think these names for the IPC classes are more clear?

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #679 from wesm/ARROW-1008 and squashes the following commits:

d7b7c9ce [Wes McKinney] Add missing dtors for pimpl pattern
a797ee3e [Wes McKinney] Fix glib
04fa2854 [Wes McKinney] Feedback on ipc reader/writer names. Add open_stream/open_file Python APIs
22346d47 [Wes McKinney] Fix unit tests
10837a65 [Wes McKinney] Add abstract stream writer and reader C++ APIs. Rename record batch stream reader and writer classes for better clarity
---
 c_glib/arrow-glib/stream-reader.cpp        |  20 ++--
 c_glib/arrow-glib/stream-reader.h          |   2 +-
 c_glib/arrow-glib/stream-reader.hpp        |   4 +-
 c_glib/arrow-glib/writer.cpp               |  16 +--
 c_glib/arrow-glib/writer.h                 |   2 +-
 c_glib/arrow-glib/writer.hpp               |   8 +-
 ci/travis_before_script_cpp.sh             |   2 +
 cpp/CMakeLists.txt                         |   8 ++
 cpp/src/arrow/ipc/file-to-stream.cc        |  12 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc   |  25 ++--
 cpp/src/arrow/ipc/json-integration-test.cc |  13 +-
 cpp/src/arrow/ipc/reader.cc                |  54 +++++----
 cpp/src/arrow/ipc/reader.h                 | 107 +++++++++++++----
 cpp/src/arrow/ipc/stream-to-file.cc        |  12 +-
 cpp/src/arrow/ipc/writer.cc                |  70 ++++++-----
 cpp/src/arrow/ipc/writer.h                 | 132 ++++++++++++++-------
 python/doc/source/api.rst                  |  10 +-
 python/doc/source/ipc.rst                  |  23 ++--
 python/pyarrow/__init__.py                 |   5 +-
 python/pyarrow/includes/libarrow.pxd       |  38 +++---
 python/pyarrow/io.pxi                      |  48 +++++---
 python/pyarrow/ipc.py                      |  46 ++++++-
 python/pyarrow/tests/test_ipc.py           |  20 ++--
 23 files changed, 433 insertions(+), 244 deletions(-)

diff --git a/c_glib/arrow-glib/stream-reader.cpp b/c_glib/arrow-glib/stream-reader.cpp
index cc18cd84d3142..19c36c20fbb5d 100644
--- a/c_glib/arrow-glib/stream-reader.cpp
+++ b/c_glib/arrow-glib/stream-reader.cpp
@@ -43,7 +43,7 @@ G_BEGIN_DECLS
  */
 
 typedef struct GArrowStreamReaderPrivate_ {
-  std::shared_ptr<arrow::ipc::StreamReader> stream_reader;
+  std::shared_ptr<arrow::ipc::RecordBatchStreamReader> stream_reader;
 } GArrowStreamReaderPrivate;
 
 enum {
@@ -85,7 +85,7 @@ garrow_stream_reader_set_property(GObject *object,
   switch (prop_id) {
   case PROP_STREAM_READER:
     priv->stream_reader =
-      *static_cast<std::shared_ptr<arrow::ipc::StreamReader> *>(g_value_get_pointer(value));
+      *static_cast<std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *>(g_value_get_pointer(value));
     break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
@@ -124,8 +124,8 @@ garrow_stream_reader_class_init(GArrowStreamReaderClass *klass)
   gobject_class->get_property = garrow_stream_reader_get_property;
 
   spec = g_param_spec_pointer("stream-reader",
-                              "ipc::StreamReader",
-                              "The raw std::shared<arrow::ipc::StreamReader> *",
+                              "ipc::RecordBatchStreamReader",
+                              "The raw std::shared<arrow::ipc::RecordBatchStreamReader> *",
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property(gobject_class, PROP_STREAM_READER, spec);
@@ -143,10 +143,10 @@ GArrowStreamReader *
 garrow_stream_reader_new(GArrowInputStream *stream,
                          GError **error)
 {
-  std::shared_ptr<arrow::ipc::StreamReader> arrow_stream_reader;
+  std::shared_ptr<arrow::ipc::RecordBatchStreamReader> arrow_stream_reader;
   auto status =
-    arrow::ipc::StreamReader::Open(garrow_input_stream_get_raw(stream),
-                                   &arrow_stream_reader);
+    arrow::ipc::RecordBatchStreamReader::Open(garrow_input_stream_get_raw(stream),
+                                              &arrow_stream_reader);
   if (garrow_error_check(error, status, "[ipc][stream-reader][open]")) {
     return garrow_stream_reader_new_raw(&arrow_stream_reader);
   } else {
@@ -179,7 +179,7 @@ garrow_stream_reader_get_schema(GArrowStreamReader *stream_reader)
  */
 GArrowRecordBatch *
 garrow_stream_reader_get_next_record_batch(GArrowStreamReader *stream_reader,
-                                               GError **error)
+                                           GError **error)
 {
   auto arrow_stream_reader =
     garrow_stream_reader_get_raw(stream_reader);
@@ -202,7 +202,7 @@ garrow_stream_reader_get_next_record_batch(GArrowStreamReader *stream_reader,
 G_END_DECLS
 
 GArrowStreamReader *
-garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_stream_reader)
+garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *arrow_stream_reader)
 {
   auto stream_reader =
     GARROW_STREAM_READER(g_object_new(GARROW_TYPE_STREAM_READER,
@@ -211,7 +211,7 @@ garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_st
   return stream_reader;
 }
 
-std::shared_ptr<arrow::ipc::StreamReader>
+std::shared_ptr<arrow::ipc::RecordBatchStreamReader>
 garrow_stream_reader_get_raw(GArrowStreamReader *stream_reader)
 {
   GArrowStreamReaderPrivate *priv;
diff --git a/c_glib/arrow-glib/stream-reader.h b/c_glib/arrow-glib/stream-reader.h
index 2ea2c26a9e541..f6cdaea5f7a95 100644
--- a/c_glib/arrow-glib/stream-reader.h
+++ b/c_glib/arrow-glib/stream-reader.h
@@ -55,7 +55,7 @@ typedef struct _GArrowStreamReaderClass    GArrowStreamReaderClass;
 /**
  * GArrowStreamReader:
  *
- * It wraps `arrow::ipc::StreamReader`.
+ * It wraps `arrow::ipc::InputStreamReader`.
  */
 struct _GArrowStreamReader
 {
diff --git a/c_glib/arrow-glib/stream-reader.hpp b/c_glib/arrow-glib/stream-reader.hpp
index ca8e6895a4fd6..5191b4edb1cdf 100644
--- a/c_glib/arrow-glib/stream-reader.hpp
+++ b/c_glib/arrow-glib/stream-reader.hpp
@@ -24,5 +24,5 @@
 
 #include <arrow-glib/stream-reader.h>
 
-GArrowStreamReader *garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::StreamReader> *arrow_stream_reader);
-std::shared_ptr<arrow::ipc::StreamReader> garrow_stream_reader_get_raw(GArrowStreamReader *stream_reader);
+GArrowStreamReader *garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *arrow_stream_reader);
+std::shared_ptr<arrow::ipc::RecordBatchStreamReader> garrow_stream_reader_get_raw(GArrowStreamReader *stream_reader);
diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp
index 625a19e3b0b07..092993b5c2240 100644
--- a/c_glib/arrow-glib/writer.cpp
+++ b/c_glib/arrow-glib/writer.cpp
@@ -47,7 +47,7 @@ G_BEGIN_DECLS
  */
 
 typedef struct GArrowStreamWriterPrivate_ {
-  std::shared_ptr<arrow::ipc::StreamWriter> stream_writer;
+  std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> stream_writer;
 } GArrowStreamWriterPrivate;
 
 enum {
@@ -89,7 +89,7 @@ garrow_stream_writer_set_property(GObject *object,
   switch (prop_id) {
   case PROP_STREAM_WRITER:
     priv->stream_writer =
-      *static_cast<std::shared_ptr<arrow::ipc::StreamWriter> *>(g_value_get_pointer(value));
+      *static_cast<std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> *>(g_value_get_pointer(value));
     break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
@@ -128,8 +128,8 @@ garrow_stream_writer_class_init(GArrowStreamWriterClass *klass)
   gobject_class->get_property = garrow_stream_writer_get_property;
 
   spec = g_param_spec_pointer("stream-writer",
-                              "ipc::StreamWriter",
-                              "The raw std::shared<arrow::ipc::StreamWriter> *",
+                              "ipc::RecordBatchStreamWriter",
+                              "The raw std::shared<arrow::ipc::RecordBatchStreamWriter> *",
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property(gobject_class, PROP_STREAM_WRITER, spec);
@@ -149,11 +149,11 @@ garrow_stream_writer_new(GArrowOutputStream *sink,
                          GArrowSchema *schema,
                          GError **error)
 {
-  std::shared_ptr<arrow::ipc::StreamWriter> arrow_stream_writer;
+  std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> arrow_stream_writer;
   auto status =
-    arrow::ipc::StreamWriter::Open(garrow_output_stream_get_raw(sink).get(),
-                                 garrow_schema_get_raw(schema),
-                                 &arrow_stream_writer);
+    arrow::ipc::RecordBatchStreamWriter::Open(garrow_output_stream_get_raw(sink).get(),
+                                              garrow_schema_get_raw(schema),
+                                              &arrow_stream_writer);
   if (garrow_error_check(error, status, "[ipc][stream-writer][open]")) {
     return garrow_stream_writer_new_raw(&arrow_stream_writer);
   } else {
diff --git a/c_glib/arrow-glib/writer.h b/c_glib/arrow-glib/writer.h
index 2aaa776f8176f..2f8e90cd28599 100644
--- a/c_glib/arrow-glib/writer.h
+++ b/c_glib/arrow-glib/writer.h
@@ -56,7 +56,7 @@ typedef struct _GArrowStreamWriterClass    GArrowStreamWriterClass;
 /**
  * GArrowStreamWriter:
  *
- * It wraps `arrow::ipc::StreamWriter`.
+ * It wraps `arrow::ipc::RecordBatchStreamWriter`.
  */
 struct _GArrowStreamWriter
 {
diff --git a/c_glib/arrow-glib/writer.hpp b/c_glib/arrow-glib/writer.hpp
index 199f205e28fe0..47f5e6839654f 100644
--- a/c_glib/arrow-glib/writer.hpp
+++ b/c_glib/arrow-glib/writer.hpp
@@ -24,8 +24,8 @@
 
 #include <arrow-glib/writer.h>
 
-GArrowStreamWriter *garrow_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer);
-std::shared_ptr<arrow::ipc::StreamWriter> garrow_stream_writer_get_raw(GArrowStreamWriter *stream_writer);
+GArrowStreamWriter *garrow_stream_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> *arrow_stream_writer);
+std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> garrow_stream_writer_get_raw(GArrowStreamWriter *stream_writer);
 
-GArrowFileWriter *garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer);
-arrow::ipc::FileWriter *garrow_file_writer_get_raw(GArrowFileWriter *file_writer);
+GArrowFileWriter *garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileWriter> *arrow_file_writer);
+arrow::ipc::RecordBatchFileWriter *garrow_file_writer_get_raw(GArrowFileWriter *file_writer);
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 3f9f67c359289..7d4ecb709ed83 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -38,10 +38,12 @@ if [ $TRAVIS_OS_NAME == "linux" ]; then
     cmake -DARROW_TEST_MEMCHECK=on \
           $CMAKE_COMMON_FLAGS \
           -DARROW_CXXFLAGS="-Wconversion -Werror" \
+          -DARROW_NO_DEPRECATED_API=on \
           $ARROW_CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
           -DARROW_CXXFLAGS=-Werror \
+          -DARROW_NO_DEPRECATED_API=on \
           $ARROW_CPP_DIR
 fi
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6b2ceec32777e..0ad7ef560fb3c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -89,6 +89,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow micro benchmarks"
     OFF)
 
+  option(ARROW_NO_DEPRECATED_API
+    "Exclude deprecated APIs from build"
+    OFF)
+
   option(ARROW_IPC
     "Build the Arrow IPC extensions"
     ON)
@@ -154,6 +158,10 @@ include(BuildUtils)
 
 include(SetupCxxFlags)
 
+if (ARROW_NO_DEPRECATED_API)
+  add_definitions(-DARROW_NO_DEPRECATED_API)
+endif()
+
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMMON_FLAGS}")
 set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
diff --git a/cpp/src/arrow/ipc/file-to-stream.cc b/cpp/src/arrow/ipc/file-to-stream.cc
index 8161b191380dc..39c720cb35aa5 100644
--- a/cpp/src/arrow/ipc/file-to-stream.cc
+++ b/cpp/src/arrow/ipc/file-to-stream.cc
@@ -24,18 +24,19 @@
 #include "arrow/util/io-util.h"
 
 namespace arrow {
+namespace ipc {
 
 // Reads a file on the file system and prints to stdout the stream version of it.
 Status ConvertToStream(const char* path) {
   std::shared_ptr<io::ReadableFile> in_file;
-  std::shared_ptr<ipc::FileReader> reader;
+  std::shared_ptr<RecordBatchFileReader> reader;
 
   RETURN_NOT_OK(io::ReadableFile::Open(path, &in_file));
-  RETURN_NOT_OK(ipc::FileReader::Open(in_file, &reader));
+  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file, &reader));
 
   io::StdoutStream sink;
-  std::shared_ptr<ipc::StreamWriter> writer;
-  RETURN_NOT_OK(ipc::StreamWriter::Open(&sink, reader->schema(), &writer));
+  std::shared_ptr<RecordBatchStreamWriter> writer;
+  RETURN_NOT_OK(RecordBatchStreamWriter::Open(&sink, reader->schema(), &writer));
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> chunk;
     RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
@@ -44,6 +45,7 @@ Status ConvertToStream(const char* path) {
   return writer->Close();
 }
 
+}  // namespace ipc
 }  // namespace arrow
 
 int main(int argc, char** argv) {
@@ -51,7 +53,7 @@ int main(int argc, char** argv) {
     std::cerr << "Usage: file-to-stream <input arrow file>" << std::endl;
     return 1;
   }
-  arrow::Status status = arrow::ConvertToStream(argv[1]);
+  arrow::Status status = arrow::ipc::ConvertToStream(argv[1]);
   if (!status.ok()) {
     std::cerr << "Could not convert to stream: " << status.ToString() << std::endl;
     return 1;
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index b4a88b5519b7e..c99816c760e6f 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -140,16 +140,16 @@ class IpcTestFixture : public io::MemoryMapFixture {
     if (zero_data) { RETURN_NOT_OK(ZeroMemoryMap(mmap_.get())); }
     RETURN_NOT_OK(mmap_->Seek(0));
 
-    std::shared_ptr<FileWriter> file_writer;
-    RETURN_NOT_OK(FileWriter::Open(mmap_.get(), batch.schema(), &file_writer));
+    std::shared_ptr<RecordBatchFileWriter> file_writer;
+    RETURN_NOT_OK(RecordBatchFileWriter::Open(mmap_.get(), batch.schema(), &file_writer));
     RETURN_NOT_OK(file_writer->WriteRecordBatch(batch, true));
     RETURN_NOT_OK(file_writer->Close());
 
     int64_t offset;
     RETURN_NOT_OK(mmap_->Tell(&offset));
 
-    std::shared_ptr<FileReader> file_reader;
-    RETURN_NOT_OK(FileReader::Open(mmap_, offset, &file_reader));
+    std::shared_ptr<RecordBatchFileReader> file_reader;
+    RETURN_NOT_OK(RecordBatchFileReader::Open(mmap_, offset, &file_reader));
 
     return file_reader->GetRecordBatch(0, result);
   }
@@ -487,8 +487,9 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
 
   Status RoundTripHelper(const BatchVector& in_batches, BatchVector* out_batches) {
     // Write the file
-    std::shared_ptr<FileWriter> writer;
-    RETURN_NOT_OK(FileWriter::Open(sink_.get(), in_batches[0]->schema(), &writer));
+    std::shared_ptr<RecordBatchFileWriter> writer;
+    RETURN_NOT_OK(
+        RecordBatchFileWriter::Open(sink_.get(), in_batches[0]->schema(), &writer));
 
     const int num_batches = static_cast<int>(in_batches.size());
 
@@ -504,8 +505,8 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
 
     // Open the file
     auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
-    std::shared_ptr<FileReader> reader;
-    RETURN_NOT_OK(FileReader::Open(buf_reader, footer_offset, &reader));
+    std::shared_ptr<RecordBatchFileReader> reader;
+    RETURN_NOT_OK(RecordBatchFileReader::Open(buf_reader, footer_offset, &reader));
 
     EXPECT_EQ(num_batches, reader->num_record_batches());
     for (int i = 0; i < num_batches; ++i) {
@@ -553,8 +554,8 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
   Status RoundTripHelper(
       const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
     // Write the file
-    std::shared_ptr<StreamWriter> writer;
-    RETURN_NOT_OK(StreamWriter::Open(sink_.get(), batch.schema(), &writer));
+    std::shared_ptr<RecordBatchStreamWriter> writer;
+    RETURN_NOT_OK(RecordBatchStreamWriter::Open(sink_.get(), batch.schema(), &writer));
     int num_batches = 5;
     for (int i = 0; i < num_batches; ++i) {
       RETURN_NOT_OK(writer->WriteRecordBatch(batch));
@@ -565,8 +566,8 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
     // Open the file
     auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
 
-    std::shared_ptr<StreamReader> reader;
-    RETURN_NOT_OK(StreamReader::Open(buf_reader, &reader));
+    std::shared_ptr<RecordBatchStreamReader> reader;
+    RETURN_NOT_OK(RecordBatchStreamReader::Open(buf_reader, &reader));
 
     std::shared_ptr<RecordBatch> chunk;
     while (true) {
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index aa95500003ec0..424755a94068b 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -76,8 +76,9 @@ static Status ConvertJsonToArrow(
     std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
   }
 
-  std::shared_ptr<ipc::FileWriter> writer;
-  RETURN_NOT_OK(ipc::FileWriter::Open(out_file.get(), reader->schema(), &writer));
+  std::shared_ptr<ipc::RecordBatchFileWriter> writer;
+  RETURN_NOT_OK(
+      ipc::RecordBatchFileWriter::Open(out_file.get(), reader->schema(), &writer));
 
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
@@ -96,8 +97,8 @@ static Status ConvertArrowToJson(
   RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &in_file));
   RETURN_NOT_OK(io::FileOutputStream::Open(json_path, &out_file));
 
-  std::shared_ptr<ipc::FileReader> reader;
-  RETURN_NOT_OK(ipc::FileReader::Open(in_file, &reader));
+  std::shared_ptr<ipc::RecordBatchFileReader> reader;
+  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file, &reader));
 
   if (FLAGS_verbose) {
     std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
@@ -137,8 +138,8 @@ static Status ValidateArrowVsJson(
   std::shared_ptr<io::ReadableFile> arrow_file;
   RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &arrow_file));
 
-  std::shared_ptr<ipc::FileReader> arrow_reader;
-  RETURN_NOT_OK(ipc::FileReader::Open(arrow_file, &arrow_reader));
+  std::shared_ptr<ipc::RecordBatchFileReader> arrow_reader;
+  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(arrow_file, &arrow_reader));
 
   auto json_schema = json_reader->schema();
   auto arrow_schema = arrow_reader->schema();
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index aea4c9cd5ec1c..2b7b90f2f2ea7 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -156,7 +156,7 @@ Status ReadDictionary(const Message& metadata, const DictionaryTypeMap& dictiona
 }
 
 // ----------------------------------------------------------------------
-// StreamReader implementation
+// RecordBatchStreamReader implementation
 
 static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
   return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
@@ -176,10 +176,12 @@ static inline std::string message_type_name(Message::Type type) {
   return "unknown";
 }
 
-class StreamReader::StreamReaderImpl {
+RecordBatchReader::~RecordBatchReader() {}
+
+class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
  public:
-  StreamReaderImpl() {}
-  ~StreamReaderImpl() {}
+  RecordBatchStreamReaderImpl() {}
+  ~RecordBatchStreamReaderImpl() {}
 
   Status Open(const std::shared_ptr<io::InputStream>& stream) {
     stream_ = stream;
@@ -267,33 +269,33 @@ class StreamReader::StreamReaderImpl {
   std::shared_ptr<Schema> schema_;
 };
 
-StreamReader::StreamReader() {
-  impl_.reset(new StreamReaderImpl());
+RecordBatchStreamReader::RecordBatchStreamReader() {
+  impl_.reset(new RecordBatchStreamReaderImpl());
 }
 
-StreamReader::~StreamReader() {}
+RecordBatchStreamReader::~RecordBatchStreamReader() {}
 
-Status StreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
-    std::shared_ptr<StreamReader>* reader) {
+Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
+    std::shared_ptr<RecordBatchStreamReader>* reader) {
   // Private ctor
-  *reader = std::shared_ptr<StreamReader>(new StreamReader());
+  *reader = std::shared_ptr<RecordBatchStreamReader>(new RecordBatchStreamReader());
   return (*reader)->impl_->Open(stream);
 }
 
-std::shared_ptr<Schema> StreamReader::schema() const {
+std::shared_ptr<Schema> RecordBatchStreamReader::schema() const {
   return impl_->schema();
 }
 
-Status StreamReader::GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+Status RecordBatchStreamReader::GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
   return impl_->GetNextRecordBatch(batch);
 }
 
 // ----------------------------------------------------------------------
 // Reader implementation
 
-class FileReader::FileReaderImpl {
+class RecordBatchFileReader::RecordBatchFileReaderImpl {
  public:
-  FileReaderImpl() { dictionary_memo_ = std::make_shared<DictionaryMemo>(); }
+  RecordBatchFileReaderImpl() { dictionary_memo_ = std::make_shared<DictionaryMemo>(); }
 
   Status ReadFooter() {
     int magic_size = static_cast<int>(strlen(kArrowMagicBytes));
@@ -432,38 +434,38 @@ class FileReader::FileReaderImpl {
   std::shared_ptr<Schema> schema_;
 };
 
-FileReader::FileReader() {
-  impl_.reset(new FileReaderImpl());
+RecordBatchFileReader::RecordBatchFileReader() {
+  impl_.reset(new RecordBatchFileReaderImpl());
 }
 
-FileReader::~FileReader() {}
+RecordBatchFileReader::~RecordBatchFileReader() {}
 
-Status FileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
-    std::shared_ptr<FileReader>* reader) {
+Status RecordBatchFileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
+    std::shared_ptr<RecordBatchFileReader>* reader) {
   int64_t footer_offset;
   RETURN_NOT_OK(file->GetSize(&footer_offset));
   return Open(file, footer_offset, reader);
 }
 
-Status FileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
-    int64_t footer_offset, std::shared_ptr<FileReader>* reader) {
-  *reader = std::shared_ptr<FileReader>(new FileReader());
+Status RecordBatchFileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
+    int64_t footer_offset, std::shared_ptr<RecordBatchFileReader>* reader) {
+  *reader = std::shared_ptr<RecordBatchFileReader>(new RecordBatchFileReader());
   return (*reader)->impl_->Open(file, footer_offset);
 }
 
-std::shared_ptr<Schema> FileReader::schema() const {
+std::shared_ptr<Schema> RecordBatchFileReader::schema() const {
   return impl_->schema();
 }
 
-int FileReader::num_record_batches() const {
+int RecordBatchFileReader::num_record_batches() const {
   return impl_->num_record_batches();
 }
 
-MetadataVersion FileReader::version() const {
+MetadataVersion RecordBatchFileReader::version() const {
   return impl_->version();
 }
 
-Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
+Status RecordBatchFileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
   return impl_->GetRecordBatch(i, batch);
 }
 
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 1972446743bc1..dd29a36d40593 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -44,29 +44,50 @@ class RandomAccessFile;
 
 namespace ipc {
 
-class ARROW_EXPORT StreamReader {
+/// \brief Abstract interface for reading stream of record batches
+class ARROW_EXPORT RecordBatchReader {
  public:
-  ~StreamReader();
+  virtual ~RecordBatchReader();
 
-  // Open an stream.
-  static Status Open(const std::shared_ptr<io::InputStream>& stream,
-      std::shared_ptr<StreamReader>* reader);
+  /// \return the shared schema of the record batches in the stream
+  virtual std::shared_ptr<Schema> schema() const = 0;
 
-  std::shared_ptr<Schema> schema() const;
+  /// Read the next record batch in the stream. Return nullptr for batch when
+  /// reaching end of stream
+  ///
+  /// \param(out) batch the next loaded batch, nullptr at end of stream
+  /// \return Status
+  virtual Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) = 0;
+};
+
+/// \class RecordBatchStreamReader
+/// \brief Synchronous batch stream reader that reads from io::InputStream
+class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
+ public:
+  virtual ~RecordBatchStreamReader();
 
-  // Returned batch is nullptr when end of stream reached
-  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch);
+  /// Create batch reader from InputStream
+  ///
+  /// \param(in) stream an input stream instance
+  /// \param(out) reader the created reader object
+  /// \return Status
+  static Status Open(const std::shared_ptr<io::InputStream>& stream,
+      std::shared_ptr<RecordBatchStreamReader>* reader);
+
+  std::shared_ptr<Schema> schema() const override;
+  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) override;
 
  private:
-  StreamReader();
+  RecordBatchStreamReader();
 
-  class ARROW_NO_EXPORT StreamReaderImpl;
-  std::unique_ptr<StreamReaderImpl> impl_;
+  class ARROW_NO_EXPORT RecordBatchStreamReaderImpl;
+  std::unique_ptr<RecordBatchStreamReaderImpl> impl_;
 };
 
-class ARROW_EXPORT FileReader {
+/// \brief Reads the record batch file format
+class ARROW_EXPORT RecordBatchFileReader {
  public:
-  ~FileReader();
+  ~RecordBatchFileReader();
 
   // Open a file-like object that is assumed to be self-contained; i.e., the
   // end of the file interface is the end of the Arrow file. Note that there
@@ -74,7 +95,7 @@ class ARROW_EXPORT FileReader {
   // need only locate the end of the Arrow file stream to discover the metadata
   // and then proceed to read the data into memory.
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
-      std::shared_ptr<FileReader>* reader);
+      std::shared_ptr<RecordBatchFileReader>* reader);
 
   // If the file is embedded within some larger file or memory region, you can
   // pass the absolute memory offset to the end of the file (which contains the
@@ -84,46 +105,80 @@ class ARROW_EXPORT FileReader {
   // @param file: the data source
   // @param footer_offset: the position of the end of the Arrow "file"
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
-      int64_t footer_offset, std::shared_ptr<FileReader>* reader);
+      int64_t footer_offset, std::shared_ptr<RecordBatchFileReader>* reader);
 
   /// The schema includes any dictionaries
   std::shared_ptr<Schema> schema() const;
 
+  /// Returns number of record batches in the file
   int num_record_batches() const;
 
+  /// Returns MetadataVersion in the file metadata
   MetadataVersion version() const;
 
-  // Read a record batch from the file. Does not copy memory if the input
-  // source supports zero-copy.
-  //
-  // TODO(wesm): Make the copy/zero-copy behavior configurable (e.g. provide an
-  // "always copy" option)
+  /// Read a record batch from the file. Does not copy memory if the input
+  /// source supports zero-copy.
+  ///
+  /// \param(in) i the index of the record batch to return
+  /// \param(out) batch the read batch
+  /// \return Status
   Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch);
 
  private:
-  FileReader();
+  RecordBatchFileReader();
 
-  class ARROW_NO_EXPORT FileReaderImpl;
-  std::unique_ptr<FileReaderImpl> impl_;
+  class ARROW_NO_EXPORT RecordBatchFileReaderImpl;
+  std::unique_ptr<RecordBatchFileReaderImpl> impl_;
 };
 
-// Generic read functionsh; does not copy data if the input supports zero copy reads
+// Generic read functions; does not copy data if the input supports zero copy reads
+
+/// Read record batch from file given metadata and schema
+///
+/// \param(in) metadata a Message containing the record batch metadata
+/// \param(in) schema the record batch schema
+/// \param(in) file a random access file
+/// \param(out) out the read record batch
 Status ARROW_EXPORT ReadRecordBatch(const Message& metadata,
     const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
     std::shared_ptr<RecordBatch>* out);
 
+/// Read record batch from file given metadata and schema
+///
+/// \param(in) metadata a Message containing the record batch metadata
+/// \param(in) schema the record batch schema
+/// \param(in) file a random access file
+/// \param(in) max_recursion_depth the maximum permitted nesting depth
+/// \param(out) out the read record batch
 Status ARROW_EXPORT ReadRecordBatch(const Message& metadata,
     const std::shared_ptr<Schema>& schema, int max_recursion_depth,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
-/// Read encapsulated message and RecordBatch
+/// Read record batch as encapsulated IPC message with metadata size prefix and
+/// header
+///
+/// \param(in) schema the record batch schema
+/// \param(in) offset the file location of the start of the message
+/// \param(in) file the file where the batch is located
+/// \param(out) out the read record batch
 Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
-/// EXPERIMENTAL: Read arrow::Tensor from a contiguous message
+/// EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
+///
+/// \param(in) offset the file location of the start of the message
+/// \param(in) file the file where the batch is located
+/// \param(out) out the read tensor
 Status ARROW_EXPORT ReadTensor(
     int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out);
 
+/// Backwards-compatibility for Arrow < 0.4.0
+///
+#ifndef ARROW_NO_DEPRECATED_API
+using StreamReader = RecordBatchReader;
+using FileReader = RecordBatchFileReader;
+#endif
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/stream-to-file.cc b/cpp/src/arrow/ipc/stream-to-file.cc
index ec0ac435a9d0d..b94205427dfe0 100644
--- a/cpp/src/arrow/ipc/stream-to-file.cc
+++ b/cpp/src/arrow/ipc/stream-to-file.cc
@@ -24,18 +24,19 @@
 #include "arrow/util/io-util.h"
 
 namespace arrow {
+namespace ipc {
 
 // Converts a stream from stdin to a file written to standard out.
 // A typical usage would be:
 // $ <program that produces streaming output> | stream-to-file > file.arrow
 Status ConvertToFile() {
   std::shared_ptr<io::InputStream> input(new io::StdinStream);
-  std::shared_ptr<ipc::StreamReader> reader;
-  RETURN_NOT_OK(ipc::StreamReader::Open(input, &reader));
+  std::shared_ptr<RecordBatchStreamReader> reader;
+  RETURN_NOT_OK(RecordBatchStreamReader::Open(input, &reader));
 
   io::StdoutStream sink;
-  std::shared_ptr<ipc::FileWriter> writer;
-  RETURN_NOT_OK(ipc::FileWriter::Open(&sink, reader->schema(), &writer));
+  std::shared_ptr<RecordBatchFileWriter> writer;
+  RETURN_NOT_OK(RecordBatchFileWriter::Open(&sink, reader->schema(), &writer));
 
   std::shared_ptr<RecordBatch> batch;
   while (true) {
@@ -46,10 +47,11 @@ Status ConvertToFile() {
   return writer->Close();
 }
 
+}  // namespace ipc
 }  // namespace arrow
 
 int main(int argc, char** argv) {
-  arrow::Status status = arrow::ConvertToFile();
+  arrow::Status status = arrow::ipc::ConvertToFile();
   if (!status.ok()) {
     std::cerr << "Could not convert to file: " << status.ToString() << std::endl;
     return 1;
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 78d6b9eb92b4e..ced071020212a 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -88,9 +88,9 @@ static inline bool NeedTruncate(
   return offset != 0 || min_length < buffer->size();
 }
 
-class RecordBatchWriter : public ArrayVisitor {
+class RecordBatchSerializer : public ArrayVisitor {
  public:
-  RecordBatchWriter(MemoryPool* pool, int64_t buffer_start_offset,
+  RecordBatchSerializer(MemoryPool* pool, int64_t buffer_start_offset,
       int max_recursion_depth, bool allow_64bit)
       : pool_(pool),
         max_recursion_depth_(max_recursion_depth),
@@ -99,7 +99,7 @@ class RecordBatchWriter : public ArrayVisitor {
     DCHECK_GT(max_recursion_depth, 0);
   }
 
-  virtual ~RecordBatchWriter() = default;
+  virtual ~RecordBatchSerializer() = default;
 
   Status VisitArray(const Array& arr) {
     if (max_recursion_depth_ <= 0) {
@@ -480,9 +480,9 @@ class RecordBatchWriter : public ArrayVisitor {
   bool allow_64bit_;
 };
 
-class DictionaryWriter : public RecordBatchWriter {
+class DictionaryWriter : public RecordBatchSerializer {
  public:
-  using RecordBatchWriter::RecordBatchWriter;
+  using RecordBatchSerializer::RecordBatchSerializer;
 
   Status WriteMetadataMessage(
       int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
@@ -500,7 +500,7 @@ class DictionaryWriter : public RecordBatchWriter {
     auto schema = std::make_shared<Schema>(fields);
     RecordBatch batch(schema, dictionary->length(), {dictionary});
 
-    return RecordBatchWriter::Write(batch, dst, metadata_length, body_length);
+    return RecordBatchSerializer::Write(batch, dst, metadata_length, body_length);
   }
 
  private:
@@ -521,7 +521,8 @@ Status AlignStreamPosition(io::OutputStream* stream) {
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
     MemoryPool* pool, int max_recursion_depth, bool allow_64bit) {
-  RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth, allow_64bit);
+  RecordBatchSerializer writer(
+      pool, buffer_start_offset, max_recursion_depth, allow_64bit);
   return writer.Write(batch, dst, metadata_length, body_length);
 }
 
@@ -580,18 +581,22 @@ Status GetTensorSize(const Tensor& tensor, int64_t* size) {
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+
+RecordBatchWriter::~RecordBatchWriter() {}
+
 // ----------------------------------------------------------------------
 // Stream writer implementation
 
-class StreamWriter::StreamWriterImpl {
+class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
  public:
-  StreamWriterImpl()
+  RecordBatchStreamWriterImpl()
       : dictionary_memo_(std::make_shared<DictionaryMemo>()),
         pool_(default_memory_pool()),
         position_(-1),
         started_(false) {}
 
-  virtual ~StreamWriterImpl() = default;
+  virtual ~RecordBatchStreamWriterImpl() = default;
 
   Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema) {
     sink_ = sink;
@@ -721,37 +726,40 @@ class StreamWriter::StreamWriterImpl {
   std::vector<FileBlock> record_batches_;
 };
 
-StreamWriter::StreamWriter() {
-  impl_.reset(new StreamWriterImpl());
+RecordBatchStreamWriter::RecordBatchStreamWriter() {
+  impl_.reset(new RecordBatchStreamWriterImpl());
 }
 
-Status StreamWriter::WriteRecordBatch(const RecordBatch& batch, bool allow_64bit) {
+RecordBatchStreamWriter::~RecordBatchStreamWriter() {}
+
+Status RecordBatchStreamWriter::WriteRecordBatch(
+    const RecordBatch& batch, bool allow_64bit) {
   return impl_->WriteRecordBatch(batch, allow_64bit);
 }
 
-StreamWriter::~StreamWriter() {}
-
-void StreamWriter::set_memory_pool(MemoryPool* pool) {
+void RecordBatchStreamWriter::set_memory_pool(MemoryPool* pool) {
   impl_->set_memory_pool(pool);
 }
 
-Status StreamWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-    std::shared_ptr<StreamWriter>* out) {
+Status RecordBatchStreamWriter::Open(io::OutputStream* sink,
+    const std::shared_ptr<Schema>& schema,
+    std::shared_ptr<RecordBatchStreamWriter>* out) {
   // ctor is private
-  *out = std::shared_ptr<StreamWriter>(new StreamWriter());
+  *out = std::shared_ptr<RecordBatchStreamWriter>(new RecordBatchStreamWriter());
   return (*out)->impl_->Open(sink, schema);
 }
 
-Status StreamWriter::Close() {
+Status RecordBatchStreamWriter::Close() {
   return impl_->Close();
 }
 
 // ----------------------------------------------------------------------
 // File writer implementation
 
-class FileWriter::FileWriterImpl : public StreamWriter::StreamWriterImpl {
+class RecordBatchFileWriter::RecordBatchFileWriterImpl
+    : public RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
  public:
-  using BASE = StreamWriter::StreamWriterImpl;
+  using BASE = RecordBatchStreamWriter::RecordBatchStreamWriterImpl;
 
   Status Start() override {
     RETURN_NOT_OK(WriteAligned(
@@ -783,23 +791,25 @@ class FileWriter::FileWriterImpl : public StreamWriter::StreamWriterImpl {
   }
 };
 
-FileWriter::FileWriter() {
-  impl_.reset(new FileWriterImpl());
+RecordBatchFileWriter::RecordBatchFileWriter() {
+  impl_.reset(new RecordBatchFileWriterImpl());
 }
 
-FileWriter::~FileWriter() {}
+RecordBatchFileWriter::~RecordBatchFileWriter() {}
 
-Status FileWriter::Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-    std::shared_ptr<FileWriter>* out) {
-  *out = std::shared_ptr<FileWriter>(new FileWriter());  // ctor is private
+Status RecordBatchFileWriter::Open(io::OutputStream* sink,
+    const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatchFileWriter>* out) {
+  *out = std::shared_ptr<RecordBatchFileWriter>(
+      new RecordBatchFileWriter());  // ctor is private
   return (*out)->impl_->Open(sink, schema);
 }
 
-Status FileWriter::WriteRecordBatch(const RecordBatch& batch, bool allow_64bit) {
+Status RecordBatchFileWriter::WriteRecordBatch(
+    const RecordBatch& batch, bool allow_64bit) {
   return impl_->WriteRecordBatch(batch, allow_64bit);
 }
 
-Status FileWriter::Close() {
+Status RecordBatchFileWriter::Close() {
   return impl_->Close();
 }
 
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index b71becb8c73b8..899a1b2cc1e30 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -46,6 +46,84 @@ class OutputStream;
 
 namespace ipc {
 
+/// \class RecordBatchWriter
+/// \brief Abstract interface for writing a stream of record batches
+class ARROW_EXPORT RecordBatchWriter {
+ public:
+  virtual ~RecordBatchWriter();
+
+  /// Write a record batch to the stream
+  ///
+  /// \param allow_64bit boolean permitting field lengths exceeding INT32_MAX
+  /// \return Status indicate success or failure
+  virtual Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) = 0;
+
+  /// Perform any logic necessary to finish the stream
+  ///
+  /// \return Status indicate success or failure
+  virtual Status Close() = 0;
+
+  /// In some cases, writing may require memory allocation. We use the default
+  /// memory pool, but provide the option to override
+  ///
+  /// \param pool the memory pool to use for required allocations
+  virtual void set_memory_pool(MemoryPool* pool) = 0;
+};
+
+/// \class RecordBatchStreamWriter
+/// \brief Synchronous batch stream writer that writes the Arrow streaming
+/// format
+class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
+ public:
+  virtual ~RecordBatchStreamWriter();
+
+  /// Create a new writer from stream sink and schema. User is responsible for
+  /// closing the actual OutputStream.
+  ///
+  /// \param(in) sink output stream to write to
+  /// \param(in) schema the schema of the record batches to be written
+  /// \param(out) out the created stream writer
+  /// \return Status indicating success or failure
+  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+      std::shared_ptr<RecordBatchStreamWriter>* out);
+
+  Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
+  Status Close() override;
+  void set_memory_pool(MemoryPool* pool) override;
+
+ protected:
+  RecordBatchStreamWriter();
+  class ARROW_NO_EXPORT RecordBatchStreamWriterImpl;
+  std::unique_ptr<RecordBatchStreamWriterImpl> impl_;
+};
+
+/// \brief Creates the Arrow record batch file format
+///
+/// Implements the random access file format, which structurally is a record
+/// batch stream followed by a metadata footer at the end of the file. Magic
+/// numbers are written at the start and end of the file
+class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
+ public:
+  virtual ~RecordBatchFileWriter();
+
+  /// Create a new writer from stream sink and schema
+  ///
+  /// \param(in) sink output stream to write to
+  /// \param(in) schema the schema of the record batches to be written
+  /// \param(out) out the created stream writer
+  /// \return Status indicating success or failure
+  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+      std::shared_ptr<RecordBatchFileWriter>* out);
+
+  Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
+  Status Close() override;
+
+ private:
+  RecordBatchFileWriter();
+  class ARROW_NO_EXPORT RecordBatchFileWriterImpl;
+  std::unique_ptr<RecordBatchFileWriterImpl> impl_;
+};
+
 /// Write the RecordBatch (collection of equal-length Arrow arrays) to the
 /// output stream in a contiguous block. The record batch metadata is written as
 /// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
@@ -58,13 +136,13 @@ namespace ipc {
 /// to the end of the body and end of the metadata / data header (suffixed by
 /// the header size) is returned in out-variables
 ///
-/// @param(in) buffer_start_offset the start offset to use in the buffer metadata,
+/// \param(in) buffer_start_offset the start offset to use in the buffer metadata,
 /// default should be 0
-/// @param(in) allow_64bit permit field lengths exceeding INT32_MAX. May not be
+/// \param(in) allow_64bit permit field lengths exceeding INT32_MAX. May not be
 /// readable by other Arrow implementations
-/// @param(out) metadata_length: the size of the length-prefixed flatbuffer
+/// \param(out) metadata_length: the size of the length-prefixed flatbuffer
 /// including padding to a 64-byte boundary
-/// @param(out) body_length: the size of the contiguous buffer block plus
+/// \param(out) body_length: the size of the contiguous buffer block plus
 /// padding bytes
 Status ARROW_EXPORT WriteRecordBatch(const RecordBatch& batch,
     int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
@@ -85,45 +163,6 @@ Status ARROW_EXPORT GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 // write the tensor including metadata, padding, and data
 Status ARROW_EXPORT GetTensorSize(const Tensor& tensor, int64_t* size);
 
-class ARROW_EXPORT StreamWriter {
- public:
-  virtual ~StreamWriter();
-
-  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-      std::shared_ptr<StreamWriter>* out);
-
-  virtual Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false);
-
-  /// Perform any logic necessary to finish the stream. User is responsible for
-  /// closing the actual OutputStream
-  virtual Status Close();
-
-  // In some cases, writing may require memory allocation. We use the default
-  // memory pool, but provide the option to override
-  void set_memory_pool(MemoryPool* pool);
-
- protected:
-  StreamWriter();
-  class ARROW_NO_EXPORT StreamWriterImpl;
-  std::unique_ptr<StreamWriterImpl> impl_;
-};
-
-class ARROW_EXPORT FileWriter : public StreamWriter {
- public:
-  virtual ~FileWriter();
-
-  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-      std::shared_ptr<FileWriter>* out);
-
-  Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
-  Status Close() override;
-
- private:
-  FileWriter();
-  class ARROW_NO_EXPORT FileWriterImpl;
-  std::unique_ptr<FileWriterImpl> impl_;
-};
-
 /// EXPERIMENTAL: Write RecordBatch allowing lengths over INT32_MAX. This data
 /// may not be readable by all Arrow implementations
 Status ARROW_EXPORT WriteLargeRecordBatch(const RecordBatch& batch,
@@ -135,6 +174,13 @@ Status ARROW_EXPORT WriteLargeRecordBatch(const RecordBatch& batch,
 Status ARROW_EXPORT WriteTensor(const Tensor& tensor, io::OutputStream* dst,
     int32_t* metadata_length, int64_t* body_length);
 
+/// Backwards-compatibility for Arrow < 0.4.0
+///
+#ifndef ARROW_NO_DEPRECATED_API
+using FileWriter = RecordBatchFileWriter;
+using StreamWriter = RecordBatchStreamWriter;
+#endif
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index a8dd8c5e110ac..e7bea7013b9a2 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -177,10 +177,12 @@ Interprocess Communication and Messaging
 .. autosummary::
    :toctree: generated/
 
-   FileReader
-   FileWriter
-   StreamReader
-   StreamWriter
+   RecordBatchFileReader
+   RecordBatchFileWriter
+   RecordBatchStreamReader
+   RecordBatchStreamWriter
+   open_file
+   open_stream
 
 .. _api.memory_pool:
 
diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index e63e7455bb815..cce2ae8d81538 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -55,13 +55,13 @@ First, let's create a small record batch:
    batch.num_columns
 
 Now, we can begin writing a stream containing some number of these batches. For
-this we use :class:`~pyarrow.StreamWriter`, which can write to a writeable
+this we use :class:`~pyarrow.BatchStreamWriter`, which can write to a writeable
 ``NativeFile`` object or a writeable Python object:
 
 .. ipython:: python
 
    sink = pa.InMemoryOutputStream()
-   writer = pa.StreamWriter(sink, batch.schema)
+   writer = pa.BatchStreamWriter(sink, batch.schema)
 
 Here we used an in-memory Arrow buffer stream, but this could have been a
 socket or some other IO sink.
@@ -80,11 +80,11 @@ particular stream. Now we can do:
    buf.size
 
 Now ``buf`` contains the complete stream as an in-memory byte buffer. We can
-read such a stream with :class:`~pyarrow.StreamReader`:
+read such a stream with :class:`~pyarrow.BatchStreamReader`:
 
 .. ipython:: python
 
-   reader = pa.StreamReader(buf)
+   reader = pa.BatchStreamReader(buf)
    reader.schema
 
    batches = [b for b in reader]
@@ -103,13 +103,13 @@ batches are also zero-copy and do not allocate any new memory on read.
 Writing and Reading Random Access Files
 ---------------------------------------
 
-The :class:`~pyarrow.FileWriter` has the same API as
-:class:`~pyarrow.StreamWriter`:
+The :class:`~pyarrow.BatchFileWriter` has the same API as
+:class:`~pyarrow.BatchStreamWriter`:
 
 .. ipython:: python
 
    sink = pa.InMemoryOutputStream()
-   writer = pa.FileWriter(sink, batch.schema)
+   writer = pa.BatchFileWriter(sink, batch.schema)
 
    for i in range(10):
       writer.write_batch(batch)
@@ -118,13 +118,14 @@ The :class:`~pyarrow.FileWriter` has the same API as
    buf = sink.get_result()
    buf.size
 
-The difference between :class:`~pyarrow.FileReader` and
-:class:`~pyarrow.StreamReader` is that the input source must have a ``seek``
-method for random access. The stream reader only requires read operations:
+The difference between :class:`~pyarrow.BatchFileReader` and
+:class:`~pyarrow.BatchStreamReader` is that the input source must have a
+``seek`` method for random access. The stream reader only requires read
+operations:
 
 .. ipython:: python
 
-   reader = pa.FileReader(buf)
+   reader = pa.BatchFileReader(buf)
 
 Because we have access to the entire payload, we know the number of record
 batches in the file, and can read any at random:
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 7d79811d9883e..d6d2aa4a671e7 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -101,7 +101,10 @@ def jemalloc_memory_pool():
 
 from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
 
-from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
+from pyarrow.ipc import (RecordBatchFileReader, RecordBatchFileWriter,
+                         RecordBatchStreamReader, RecordBatchStreamWriter,
+                         open_stream,
+                         open_file)
 
 
 localfs = LocalFilesystem.get_instance()
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 3d56c14bae219..b03dd59dffcaf 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -547,38 +547,44 @@ cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
 
 cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
 
-    cdef cppclass CStreamWriter " arrow::ipc::StreamWriter":
-        @staticmethod
-        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
-                     shared_ptr[CStreamWriter]* out)
-
+    cdef cppclass CRecordBatchWriter \
+        " arrow::ipc::RecordBatchWriter":
         CStatus Close()
         CStatus WriteRecordBatch(const CRecordBatch& batch)
 
-    cdef cppclass CStreamReader " arrow::ipc::StreamReader":
+    cdef cppclass CRecordBatchReader \
+        " arrow::ipc::RecordBatchReader":
+        shared_ptr[CSchema] schema()
+        CStatus GetNextRecordBatch(shared_ptr[CRecordBatch]* batch)
 
+    cdef cppclass CRecordBatchStreamReader \
+        " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
         @staticmethod
         CStatus Open(const shared_ptr[InputStream]& stream,
-                     shared_ptr[CStreamReader]* out)
+                     shared_ptr[CRecordBatchStreamReader]* out)
 
-        shared_ptr[CSchema] schema()
-
-        CStatus GetNextRecordBatch(shared_ptr[CRecordBatch]* batch)
-
-    cdef cppclass CFileWriter " arrow::ipc::FileWriter"(CStreamWriter):
+    cdef cppclass CRecordBatchStreamWriter \
+        " arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter):
         @staticmethod
         CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
-                     shared_ptr[CFileWriter]* out)
+                     shared_ptr[CRecordBatchStreamWriter]* out)
 
-    cdef cppclass CFileReader " arrow::ipc::FileReader":
+    cdef cppclass CRecordBatchFileWriter \
+        " arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter):
+        @staticmethod
+        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
+                     shared_ptr[CRecordBatchFileWriter]* out)
 
+    cdef cppclass CRecordBatchFileReader \
+        " arrow::ipc::RecordBatchFileReader":
         @staticmethod
         CStatus Open(const shared_ptr[RandomAccessFile]& file,
-                     shared_ptr[CFileReader]* out)
+                     shared_ptr[CRecordBatchFileReader]* out)
 
         @staticmethod
         CStatus Open2" Open"(const shared_ptr[RandomAccessFile]& file,
-                     int64_t footer_offset, shared_ptr[CFileReader]* out)
+                             int64_t footer_offset,
+                             shared_ptr[CRecordBatchFileReader]* out)
 
         shared_ptr[CSchema] schema()
 
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index a0a96e72864d4..4cbf603c2a5df 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -916,9 +916,9 @@ cdef class HdfsFile(NativeFile):
 # ----------------------------------------------------------------------
 # File and stream readers and writers
 
-cdef class _StreamWriter:
+cdef class _RecordBatchWriter:
     cdef:
-        shared_ptr[CStreamWriter] writer
+        shared_ptr[CRecordBatchWriter] writer
         shared_ptr[OutputStream] sink
         bint closed
 
@@ -930,12 +930,18 @@ cdef class _StreamWriter:
             self.close()
 
     def _open(self, sink, Schema schema):
+        cdef:
+            shared_ptr[CRecordBatchStreamWriter] writer
+
         get_writer(sink, &self.sink)
 
         with nogil:
-            check_status(CStreamWriter.Open(self.sink.get(), schema.sp_schema,
-                                            &self.writer))
+            check_status(
+                CRecordBatchStreamWriter.Open(self.sink.get(),
+                                              schema.sp_schema,
+                                              &writer))
 
+        self.writer = <shared_ptr[CRecordBatchWriter]> writer
         self.closed = False
 
     def write_batch(self, RecordBatch batch):
@@ -949,9 +955,9 @@ cdef class _StreamWriter:
         self.closed = True
 
 
-cdef class _StreamReader:
+cdef class _RecordBatchReader:
     cdef:
-        shared_ptr[CStreamReader] reader
+        shared_ptr[CRecordBatchReader] reader
 
     cdef readonly:
         Schema schema
@@ -961,15 +967,17 @@ cdef class _StreamReader:
 
     def _open(self, source):
         cdef:
-            shared_ptr[RandomAccessFile] reader
+            shared_ptr[RandomAccessFile] file_handle
             shared_ptr[InputStream] in_stream
+            shared_ptr[CRecordBatchStreamReader] reader
 
-        get_reader(source, &reader)
-        in_stream = <shared_ptr[InputStream]> reader
+        get_reader(source, &file_handle)
+        in_stream = <shared_ptr[InputStream]> file_handle
 
         with nogil:
-            check_status(CStreamReader.Open(in_stream, &self.reader))
+            check_status(CRecordBatchStreamReader.Open(in_stream, &reader))
 
+        self.reader = <shared_ptr[CRecordBatchReader]> reader
         self.schema = Schema()
         self.schema.init_schema(self.reader.get().schema())
 
@@ -1009,24 +1017,25 @@ cdef class _StreamReader:
         return pyarrow_wrap_table(table)
 
 
-cdef class _FileWriter(_StreamWriter):
+cdef class _RecordBatchFileWriter(_RecordBatchWriter):
 
     def _open(self, sink, Schema schema):
-        cdef shared_ptr[CFileWriter] writer
+        cdef shared_ptr[CRecordBatchFileWriter] writer
         get_writer(sink, &self.sink)
 
         with nogil:
-            check_status(CFileWriter.Open(self.sink.get(), schema.sp_schema,
-                                          &writer))
+            check_status(
+                CRecordBatchFileWriter.Open(self.sink.get(), schema.sp_schema,
+                                      &writer))
 
         # Cast to base class, because has same interface
-        self.writer = <shared_ptr[CStreamWriter]> writer
+        self.writer = <shared_ptr[CRecordBatchWriter]> writer
         self.closed = False
 
 
-cdef class _FileReader:
+cdef class _RecordBatchFileReader:
     cdef:
-        shared_ptr[CFileReader] reader
+        shared_ptr[CRecordBatchFileReader] reader
 
     def __cinit__(self):
         pass
@@ -1041,9 +1050,10 @@ cdef class _FileReader:
 
         with nogil:
             if offset != 0:
-                check_status(CFileReader.Open2(reader, offset, &self.reader))
+                check_status(CRecordBatchFileReader.Open2(
+                    reader, offset, &self.reader))
             else:
-                check_status(CFileReader.Open(reader, &self.reader))
+                check_status(CRecordBatchFileReader.Open(reader, &self.reader))
 
     property num_record_batches:
 
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index c37a1ce7df1e6..8338de33a0d2c 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -20,7 +20,7 @@
 import pyarrow.lib as lib
 
 
-class StreamReader(lib._StreamReader):
+class RecordBatchStreamReader(lib._RecordBatchReader):
     """
     Reader for the Arrow streaming binary format
 
@@ -37,7 +37,7 @@ def __iter__(self):
             yield self.get_next_batch()
 
 
-class StreamWriter(lib._StreamWriter):
+class RecordBatchStreamWriter(lib._RecordBatchWriter):
     """
     Writer for the Arrow streaming binary format
 
@@ -52,7 +52,7 @@ def __init__(self, sink, schema):
         self._open(sink, schema)
 
 
-class FileReader(lib._FileReader):
+class RecordBatchFileReader(lib._RecordBatchFileReader):
     """
     Class for reading Arrow record batch data from the Arrow binary file format
 
@@ -68,7 +68,7 @@ def __init__(self, source, footer_offset=None):
         self._open(source, footer_offset=footer_offset)
 
 
-class FileWriter(lib._FileWriter):
+class RecordBatchFileWriter(lib._RecordBatchFileWriter):
     """
     Writer to create the Arrow binary file format
 
@@ -81,3 +81,41 @@ class FileWriter(lib._FileWriter):
     """
     def __init__(self, sink, schema):
         self._open(sink, schema)
+
+
+def open_stream(source):
+    """
+    Create reader for Arrow streaming format
+
+    Parameters
+    ----------
+    source : str, pyarrow.NativeFile, or file-like Python object
+        Either a file path, or a readable file object
+    footer_offset : int, default None
+        If the file is embedded in some larger file, this is the byte offset to
+        the very end of the file data
+
+    Returns
+    -------
+    reader : RecordBatchStreamReader
+    """
+    return RecordBatchStreamReader(source)
+
+
+def open_file(source, footer_offset=None):
+    """
+    Create reader for Arrow file format
+
+    Parameters
+    ----------
+    source : str, pyarrow.NativeFile, or file-like Python object
+        Either a file path, or a readable file object
+    footer_offset : int, default None
+        If the file is embedded in some larger file, this is the byte offset to
+        the very end of the file data
+
+    Returns
+    -------
+    reader : RecordBatchFileReader
+    """
+    return RecordBatchFileReader(source, footer_offset=footer_offset)
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 02040678958ed..4d19804dac2a4 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -70,13 +70,13 @@ class TestFile(MessagingTest, unittest.TestCase):
     # Also tests writing zero-copy NumPy array with additional padding
 
     def _get_writer(self, sink, schema):
-        return pa.FileWriter(sink, schema)
+        return pa.RecordBatchFileWriter(sink, schema)
 
     def test_simple_roundtrip(self):
         batches = self.write_batches()
         file_contents = self._get_source()
 
-        reader = pa.FileReader(file_contents)
+        reader = pa.open_file(file_contents)
 
         assert reader.num_record_batches == len(batches)
 
@@ -89,7 +89,7 @@ def test_read_all(self):
         batches = self.write_batches()
         file_contents = self._get_source()
 
-        reader = pa.FileReader(file_contents)
+        reader = pa.open_file(file_contents)
 
         result = reader.read_all()
         expected = pa.Table.from_batches(batches)
@@ -99,12 +99,12 @@ def test_read_all(self):
 class TestStream(MessagingTest, unittest.TestCase):
 
     def _get_writer(self, sink, schema):
-        return pa.StreamWriter(sink, schema)
+        return pa.RecordBatchStreamWriter(sink, schema)
 
     def test_simple_roundtrip(self):
         batches = self.write_batches()
         file_contents = self._get_source()
-        reader = pa.StreamReader(file_contents)
+        reader = pa.open_stream(file_contents)
 
         assert reader.schema.equals(batches[0].schema)
 
@@ -121,7 +121,7 @@ def test_simple_roundtrip(self):
     def test_read_all(self):
         batches = self.write_batches()
         file_contents = self._get_source()
-        reader = pa.StreamReader(file_contents)
+        reader = pa.open_stream(file_contents)
 
         result = reader.read_all()
         expected = pa.Table.from_batches(batches)
@@ -147,7 +147,7 @@ def run(self):
             connection, client_address = self._sock.accept()
             try:
                 source = connection.makefile(mode='rb')
-                reader = pa.StreamReader(source)
+                reader = pa.open_stream(source)
                 self._schema = reader.schema
                 if self._do_read_all:
                     self._table = reader.read_all()
@@ -185,7 +185,7 @@ def _get_sink(self):
         return self._sock.makefile(mode='wb')
 
     def _get_writer(self, sink, schema):
-        return pa.StreamWriter(sink, schema)
+        return pa.RecordBatchStreamWriter(sink, schema)
 
     def test_simple_roundtrip(self):
         self.start_server(do_read_all=False)
@@ -241,12 +241,12 @@ def test_get_record_batch_size():
 
 
 def write_file(batch, sink):
-    writer = pa.FileWriter(sink, batch.schema)
+    writer = pa.RecordBatchFileWriter(sink, batch.schema)
     writer.write_batch(batch)
     writer.close()
 
 
 def read_file(source):
-    reader = pa.FileReader(source)
+    reader = pa.open_file(source)
     return [reader.get_batch(i)
             for i in range(reader.num_record_batches)]

From d8d3d84354d827e45c8267cd05aecd2aa36cf60b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 14 May 2017 17:23:26 +0200
Subject: [PATCH 0645/1644] ARROW-1022: [Python] Add multithreaded read option
 to read_feather

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #682 from wesm/ARROW-1022 and squashes the following commits:

8fd241e [Wes McKinney] Add multithreaded read option to read_feather
---
 python/pyarrow/feather.py            | 10 ++++++----
 python/pyarrow/tests/test_feather.py | 11 +++++++++--
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 3754aec737299..34783a71e3dec 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -37,7 +37,7 @@ def __init__(self, source):
         self.source = source
         self.open(source)
 
-    def read(self, columns=None):
+    def read(self, columns=None, nthreads=1):
         if columns is not None:
             column_set = set(columns)
         else:
@@ -53,7 +53,7 @@ def read(self, columns=None):
                 names.append(name)
 
         table = Table.from_arrays(columns, names=names)
-        return table.to_pandas()
+        return table.to_pandas(nthreads=nthreads)
 
 
 class FeatherWriter(object):
@@ -118,7 +118,7 @@ def write_feather(df, dest):
         raise
 
 
-def read_feather(source, columns=None):
+def read_feather(source, columns=None, nthreads=1):
     """
     Read a pandas.DataFrame from Feather format
 
@@ -128,10 +128,12 @@ def read_feather(source, columns=None):
     columns : sequence, optional
         Only read a specific set of columns. If not provided, all columns are
         read
+    nthreads : int, default 1
+        Number of CPU threads to use when reading to pandas.DataFrame
 
     Returns
     -------
     df : pandas.DataFrame
     """
     reader = FeatherReader(source)
-    return reader.read(columns=columns)
+    return reader.read(columns=columns, nthreads=nthreads)
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 69c32be5f3df7..287e0da2f5594 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -61,7 +61,8 @@ def _get_null_counts(self, path, columns=None):
         return counts
 
     def _check_pandas_roundtrip(self, df, expected=None, path=None,
-                                columns=None, null_counts=None):
+                                columns=None, null_counts=None,
+                                nthreads=1):
         if path is None:
             path = random_path()
 
@@ -70,7 +71,7 @@ def _check_pandas_roundtrip(self, df, expected=None, path=None,
         if not os.path.exists(path):
             raise Exception('file not written')
 
-        result = read_feather(path, columns)
+        result = read_feather(path, columns, nthreads=nthreads)
         if expected is None:
             expected = df
 
@@ -293,6 +294,12 @@ def test_empty_strings(self):
         df = pd.DataFrame({'strings': [''] * 10})
         self._check_pandas_roundtrip(df)
 
+    def test_multithreaded_read(self):
+        data = {'c{0}'.format(i): [''] * 10
+                for i in range(100)}
+        df = pd.DataFrame(data)
+        self._check_pandas_roundtrip(df, nthreads=4)
+
     def test_nan_as_null(self):
         # Create a nan that is not numpy.nan
         values = np.array(['foo', np.nan, np.nan * 2, 'bar'] * 10)

From 852ee4fbf79bb76a2199237cfe7c6a6a29deba69 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 14 May 2017 12:04:25 -0400
Subject: [PATCH 0646/1644] ARROW-1024: Python: Update build time numpy version
 to 1.10.1

Change-Id: I5cfe95272d43fd0cb08cac6646ffde30adc94bdf

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #684 from xhochy/ARROW-1024 and squashes the following commits:

0d1f2f3d [Uwe L. Korn] ARROW-1024: Python: Update build time numpy version to 1.10.1
---
 python/manylinux1/Dockerfile-x86_64            | 2 +-
 python/manylinux1/scripts/build_virtualenvs.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 08fecb0da9276..ad40e79a9f20f 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -9,7 +9,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License. See accompanying LICENSE file.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-947
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-1024
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
index ee8a82730281f..60d6580de0c21 100755
--- a/python/manylinux1/scripts/build_virtualenvs.sh
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -28,7 +28,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     PATH="$PATH:$(cpython_path $PYTHON)"
 
     echo "=== (${PYTHON}) Installing build dependencies ==="
-    $PIPI_IO "numpy==1.9.0"
+    $PIPI_IO "numpy==1.10.1"
     $PIPI_IO "cython==0.25.2"
     $PIPI_IO "pandas==0.20.1"
     $PIPI_IO "virtualenv==15.1.0"

From c7839e9fab91e62cced9367f23e561afb6728652 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 14 May 2017 14:35:28 -0400
Subject: [PATCH 0647/1644] ARROW-1017: [Python] Fix memory leaks in conversion
 to pandas.DataFrame

Notes:

* `PyList_Append` increments ref count, so new objects must be DECREF'd after being inserted
* `PyArray_SimpleNewFromDescr` does not set `NPY_ARRAY_OWNDATA`, neither does `NewFromDescr`

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #685 from wesm/ARROW-1017 and squashes the following commits:

8459123 [Wes McKinney] Fix memory leak caused by list append ref count, lack of setting NPY_ARRAY_OWNDATA
---
 cpp/src/arrow/python/common.h          |  7 +++++-
 cpp/src/arrow/python/pandas_convert.cc | 25 +++++++++---------
 python/scripts/test_leak.py            | 35 ++++++++++++++++++++++++++
 3 files changed, 53 insertions(+), 14 deletions(-)
 create mode 100644 python/scripts/test_leak.py

diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index f6e706b6948d9..ec40d0eafa3b5 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -69,7 +69,7 @@ class ARROW_EXPORT OwnedRef {
 
   ~OwnedRef() {
     PyAcquireGIL lock;
-    Py_XDECREF(obj_);
+    release();
   }
 
   void reset(PyObject* obj) {
@@ -80,6 +80,11 @@ class ARROW_EXPORT OwnedRef {
     obj_ = obj;
   }
 
+  void release() {
+    Py_XDECREF(obj_);
+    obj_ = nullptr;
+  }
+
   PyObject* obj() const { return obj_; }
 
  private:
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 264bed11b04ec..b6fb05e1e5ebb 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -1023,7 +1023,8 @@ static inline PyObject* NewArray1DFromType(
   }
 
   set_numpy_metadata(type, arrow_type, descr);
-  return PyArray_NewFromDescr(&PyArray_Type, descr, 1, dims, nullptr, data, 0, nullptr);
+  return PyArray_NewFromDescr(&PyArray_Type, descr, 1, dims, nullptr, data,
+      NPY_ARRAY_OWNDATA | NPY_ARRAY_CARRAY, nullptr);
 }
 
 class PandasBlock {
@@ -1078,12 +1079,10 @@ class PandasBlock {
     PyObject* block_arr;
     if (ndim == 2) {
       npy_intp block_dims[2] = {num_columns_, num_rows_};
-      block_arr = PyArray_NewFromDescr(
-          &PyArray_Type, descr, 2, block_dims, nullptr, nullptr, 0, nullptr);
+      block_arr = PyArray_SimpleNewFromDescr(2, block_dims, descr);
     } else {
       npy_intp block_dims[1] = {num_rows_};
-      block_arr = PyArray_NewFromDescr(
-          &PyArray_Type, descr, 1, block_dims, nullptr, nullptr, 0, nullptr);
+      block_arr = PyArray_SimpleNewFromDescr(1, block_dims, descr);
     }
 
     if (block_arr == NULL) {
@@ -1091,6 +1090,8 @@ class PandasBlock {
       return Status::OK();
     }
 
+    PyArray_ENABLEFLAGS(reinterpret_cast<PyArrayObject*>(block_arr), NPY_ARRAY_OWNDATA);
+
     npy_intp placement_dims[1] = {num_columns_};
     PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
     if (placement_arr == NULL) {
@@ -1357,8 +1358,6 @@ inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values)
 class ObjectBlock : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
-  virtual ~ObjectBlock() {}
-
   Status Allocate() override { return AllocateNDArray(NPY_OBJECT); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
@@ -1416,7 +1415,6 @@ template <int ARROW_TYPE, typename C_TYPE>
 class IntBlock : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
-
   Status Allocate() override {
     return AllocateNDArray(arrow_traits<ARROW_TYPE>::npy_type);
   }
@@ -1450,7 +1448,6 @@ using Int64Block = IntBlock<Type::INT64, int64_t>;
 class Float32Block : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
-
   Status Allocate() override { return AllocateNDArray(NPY_FLOAT32); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
@@ -1470,7 +1467,6 @@ class Float32Block : public PandasBlock {
 class Float64Block : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
-
   Status Allocate() override { return AllocateNDArray(NPY_FLOAT64); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
@@ -1523,7 +1519,6 @@ class Float64Block : public PandasBlock {
 class BoolBlock : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
-
   Status Allocate() override { return AllocateNDArray(NPY_BOOL); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
@@ -1544,7 +1539,6 @@ class BoolBlock : public PandasBlock {
 class DatetimeBlock : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
-
   Status AllocateDatetime(int ndim) {
     RETURN_NOT_OK(AllocateNDArray(NPY_DATETIME, ndim));
 
@@ -1629,7 +1623,6 @@ template <int ARROW_INDEX_TYPE>
 class CategoricalBlock : public PandasBlock {
  public:
   explicit CategoricalBlock(int64_t num_rows) : PandasBlock(num_rows, 1) {}
-
   Status Allocate() override {
     constexpr int npy_type = arrow_traits<ARROW_INDEX_TYPE>::npy_type;
 
@@ -1960,6 +1953,9 @@ class DataFrameBlockCreator {
       PyObject* item;
       RETURN_NOT_OK(it.second->GetPyResult(&item));
       if (PyList_Append(list, item) < 0) { RETURN_IF_PYERROR(); }
+
+      // ARROW-1017; PyList_Append increments object refcount
+      Py_DECREF(item);
     }
     return Status::OK();
   }
@@ -2045,6 +2041,9 @@ class ArrowDeserializer {
     // Arrow data is immutable.
     PyArray_CLEARFLAGS(arr_, NPY_ARRAY_WRITEABLE);
 
+    // Arrow data is owned by another
+    PyArray_CLEARFLAGS(arr_, NPY_ARRAY_OWNDATA);
+
     return Status::OK();
   }
 
diff --git a/python/scripts/test_leak.py b/python/scripts/test_leak.py
new file mode 100644
index 0000000000000..2b197b6c13067
--- /dev/null
+++ b/python/scripts/test_leak.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pyarrow as pa
+import numpy as np
+import memory_profiler
+import gc
+
+
+def leak():
+    data = [pa.array(np.concatenate([np.random.randn(100000)] * 1000))]
+    table = pa.Table.from_arrays(data, ['foo'])
+    while True:
+        print('calling to_pandas')
+        print('memory_usage: {0}'.format(memory_profiler.memory_usage()))
+        table.to_pandas()
+        gc.collect()
+
+leak()

From 393f46abd0fb087ce7d53443b1fad374b82a3692 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 14 May 2017 14:37:43 -0400
Subject: [PATCH 0648/1644] ARROW-1023: Python: Fix bundling of arrow-cpp for
 macOS

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #683 from xhochy/macos-wheels and squashes the following commits:

bdc5651 [Uwe L. Korn] Correct the amount of dots
e8d643f [Uwe L. Korn] Correct the amount of dots
4fa6d33 [Uwe L. Korn] Correct the amount of dots
5853b7c [Uwe L. Korn] Prepend LIBARRY_DIR
51a2b13 [Uwe L. Korn] Fix typo
7074eda [Uwe L. Korn] Version numbers are at a different place on macOS
---
 python/CMakeLists.txt | 24 ++++++++++++++++++------
 python/setup.py       |  7 ++++++-
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 123dd5d8d7a44..ddd174ca8db31 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -230,16 +230,28 @@ find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 
 function(bundle_arrow_lib library_path)
+  get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY)
   get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
   configure_file(${${library_path}}
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
       COPYONLY)
-  configure_file(${${library_path}}.${ARROW_ABI_VERSION}
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
-      COPYONLY)
-  configure_file(${${library_path}}.${ARROW_SO_VERSION}
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}
-      COPYONLY)
+
+  if (APPLE)
+    configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        COPYONLY)
+    configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        COPYONLY)
+  else()
+    configure_file(${${library_path}}.${ARROW_ABI_VERSION}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
+        COPYONLY)
+    configure_file(${${library_path}}.${ARROW_SO_VERSION}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}
+        COPYONLY)
+  endif()
+
   SET(ARROW_SHARED_LIB
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 endfunction(bundle_arrow_lib)
diff --git a/python/setup.py b/python/setup.py
index b38fca47b2399..9374af36cc6ab 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -214,7 +214,12 @@ def move_lib(lib_name):
             lib_filename = (shared_library_prefix + lib_name +
                             shared_library_suffix)
             # Also copy libraries with ABI/SO version suffix
-            libs = glob.glob(pjoin(self.build_type, lib_filename) + '*')
+            if sys.platform == 'darwin':
+                lib_pattern = (shared_library_prefix + lib_name +
+                               ".*" + shared_library_suffix[1:])
+                libs = glob.glob(pjoin(self.build_type, lib_pattern))
+            else:
+                libs = glob.glob(pjoin(self.build_type, lib_filename) + '*')
             # Longest suffix library should be copied, all others symlinked
             libs.sort(key=lambda s: -len(s))
             print(libs, libs[0])

From 37dbddf0dc6582586a2bea98a436cb20726799a4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 14 May 2017 16:30:19 -0400
Subject: [PATCH 0649/1644] ARROW-1004: [Python] Add conversions for numpy
 object arrays with integers and floats

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #681 from wesm/ARROW-1004 and squashes the following commits:

9e0b2eae [Wes McKinney] Code review comments
45f1ecb9 [Wes McKinney] Fixes for manylinux1
4e4c7529 [Wes McKinney] Add conversions for numpy object arrays with integers and floats
---
 cpp/src/arrow/python/pandas_convert.cc      | 82 ++++++++++++++++++++-
 python/pyarrow/tests/test_convert_pandas.py | 18 +++++
 2 files changed, 98 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index b6fb05e1e5ebb..96dd09aa817a3 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -80,6 +80,14 @@ static inline bool PyObject_is_string(const PyObject* obj) {
 #endif
 }
 
+static inline bool PyObject_is_float(const PyObject* obj) {
+  return PyFloat_Check(obj);
+}
+
+static inline bool PyObject_is_integer(const PyObject* obj) {
+  return (!PyBool_Check(obj)) && PyArray_IsIntegerScalar(obj);
+}
+
 template <int TYPE>
 static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
   typedef npy_traits<TYPE> traits;
@@ -394,9 +402,11 @@ class PandasConverter {
   template <typename ArrowType>
   Status ConvertDates();
 
+  Status ConvertBooleans();
   Status ConvertObjectStrings();
+  Status ConvertObjectFloats();
   Status ConvertObjectFixedWidthBytes(const std::shared_ptr<DataType>& type);
-  Status ConvertBooleans();
+  Status ConvertObjectIntegers();
   Status ConvertLists(const std::shared_ptr<DataType>& type);
   Status ConvertObjects();
   Status ConvertDecimals();
@@ -610,6 +620,70 @@ Status PandasConverter::ConvertObjectStrings() {
   return Status::OK();
 }
 
+Status PandasConverter::ConvertObjectFloats() {
+  PyAcquireGIL lock;
+
+  DoubleBuilder builder(pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask_ != nullptr) {
+    mask_values.Init(mask_);
+    have_mask = true;
+  }
+
+  PyObject* obj;
+  for (int64_t i = 0; i < objects.size(); ++i) {
+    obj = objects[i];
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else if (PyFloat_Check(obj)) {
+      double val = PyFloat_AsDouble(obj);
+      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(builder.Append(val));
+    } else {
+      return InvalidConversion(obj, "float");
+    }
+  }
+
+  return builder.Finish(&out_);
+}
+
+Status PandasConverter::ConvertObjectIntegers() {
+  PyAcquireGIL lock;
+
+  Int64Builder builder(pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask_ != nullptr) {
+    mask_values.Init(mask_);
+    have_mask = true;
+  }
+
+  PyObject* obj;
+  for (int64_t i = 0; i < objects.size(); ++i) {
+    obj = objects[i];
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else if (PyObject_is_integer(obj)) {
+      const int64_t val = static_cast<int64_t>(PyLong_AsLong(obj));
+      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(builder.Append(val));
+    } else {
+      return InvalidConversion(obj, "integer");
+    }
+  }
+
+  return builder.Finish(&out_);
+}
+
 Status PandasConverter::ConvertObjectFixedWidthBytes(
     const std::shared_ptr<DataType>& type) {
   PyAcquireGIL lock;
@@ -804,8 +878,12 @@ Status PandasConverter::ConvertObjects() {
         continue;
       } else if (PyObject_is_string(objects[i])) {
         return ConvertObjectStrings();
+      } else if (PyObject_is_float(objects[i])) {
+        return ConvertObjectFloats();
       } else if (PyBool_Check(objects[i])) {
         return ConvertBooleans();
+      } else if (PyObject_is_integer(objects[i])) {
+        return ConvertObjectIntegers();
       } else if (PyDate_CheckExact(objects[i])) {
         // We could choose Date32 or Date64
         return ConvertDates<Date32Type>();
@@ -813,7 +891,7 @@ Status PandasConverter::ConvertObjects() {
         return ConvertDecimals();
       } else {
         return InvalidConversion(
-            const_cast<PyObject*>(objects[i]), "string, bool, or date");
+            const_cast<PyObject*>(objects[i]), "string, bool, float, int, date, decimal");
       }
     }
   }
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 9b9b7519fd994..be35905fc751d 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -140,6 +140,24 @@ def test_float_nulls(self):
         result = table.to_pandas()
         tm.assert_frame_equal(result, ex_frame)
 
+    def test_float_object_nulls(self):
+        arr = np.array([None, 1.5, np.float64(3.5)] * 5, dtype=object)
+        df = pd.DataFrame({'floats': arr})
+        expected = pd.DataFrame({'floats': pd.to_numeric(arr)})
+        field = pa.field('floats', pa.float64())
+        schema = pa.schema([field])
+        self._check_pandas_roundtrip(df, expected=expected,
+                                     expected_schema=schema)
+
+    def test_int_object_nulls(self):
+        arr = np.array([None, 1, np.int64(3)] * 5, dtype=object)
+        df = pd.DataFrame({'ints': arr})
+        expected = pd.DataFrame({'ints': pd.to_numeric(arr)})
+        field = pa.field('ints', pa.int64())
+        schema = pa.schema([field])
+        self._check_pandas_roundtrip(df, expected=expected,
+                                     expected_schema=schema)
+
     def test_integer_no_nulls(self):
         data = OrderedDict()
         fields = []

From 05433794783398041ebde6972c278cf5698164b1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 15 May 2017 09:05:07 -0400
Subject: [PATCH 0650/1644] ARROW-1028: [Python] Fix IPC docs per API changes

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #687 from wesm/python-update-ipc-docs and squashes the following commits:

a502abe [Wes McKinney] Fix IPC docs per API changes
---
 python/doc/source/ipc.rst | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index cce2ae8d81538..c7de9c466386d 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -55,13 +55,13 @@ First, let's create a small record batch:
    batch.num_columns
 
 Now, we can begin writing a stream containing some number of these batches. For
-this we use :class:`~pyarrow.BatchStreamWriter`, which can write to a writeable
+this we use :class:`~pyarrow.RecordBatchStreamWriter`, which can write to a writeable
 ``NativeFile`` object or a writeable Python object:
 
 .. ipython:: python
 
    sink = pa.InMemoryOutputStream()
-   writer = pa.BatchStreamWriter(sink, batch.schema)
+   writer = pa.RecordBatchStreamWriter(sink, batch.schema)
 
 Here we used an in-memory Arrow buffer stream, but this could have been a
 socket or some other IO sink.
@@ -80,11 +80,12 @@ particular stream. Now we can do:
    buf.size
 
 Now ``buf`` contains the complete stream as an in-memory byte buffer. We can
-read such a stream with :class:`~pyarrow.BatchStreamReader`:
+read such a stream with :class:`~pyarrow.RecordBatchStreamReader` or the
+convenience function ``pyarrow.open_stream``:
 
 .. ipython:: python
 
-   reader = pa.BatchStreamReader(buf)
+   reader = pa.open_stream(buf)
    reader.schema
 
    batches = [b for b in reader]
@@ -103,13 +104,13 @@ batches are also zero-copy and do not allocate any new memory on read.
 Writing and Reading Random Access Files
 ---------------------------------------
 
-The :class:`~pyarrow.BatchFileWriter` has the same API as
-:class:`~pyarrow.BatchStreamWriter`:
+The :class:`~pyarrow.RecordBatchFileWriter` has the same API as
+:class:`~pyarrow.RecordBatchStreamWriter`:
 
 .. ipython:: python
 
    sink = pa.InMemoryOutputStream()
-   writer = pa.BatchFileWriter(sink, batch.schema)
+   writer = pa.RecordBatchFileWriter(sink, batch.schema)
 
    for i in range(10):
       writer.write_batch(batch)
@@ -118,14 +119,14 @@ The :class:`~pyarrow.BatchFileWriter` has the same API as
    buf = sink.get_result()
    buf.size
 
-The difference between :class:`~pyarrow.BatchFileReader` and
-:class:`~pyarrow.BatchStreamReader` is that the input source must have a
+The difference between :class:`~pyarrow.RecordBatchFileReader` and
+:class:`~pyarrow.RecordBatchStreamReader` is that the input source must have a
 ``seek`` method for random access. The stream reader only requires read
-operations:
+operations. We can also use the ``pyarrow.open_file`` method to open a file:
 
 .. ipython:: python
 
-   reader = pa.BatchFileReader(buf)
+   reader = pa.open_file(buf)
 
 Because we have access to the entire payload, we know the number of record
 batches in the file, and can read any at random:

From edfb2dca81db724dd2b771fa8a50da98b5fc4159 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 15 May 2017 09:17:27 -0400
Subject: [PATCH 0651/1644] ARROW-1027: [Python] Allow negative indexing in
 fields/columns on pyarrow Table and Schema objects

Negative indexing causes an interpreter segfault right now. After this PR, no segfaults, and reasonable error messages for out of bounds indexes.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #686 from cpcloud/ARROW-1027 and squashes the following commits:

1bfcef4 [Phillip Cloud] ARROW-1027: [Python] Allow negative indexing in fields/columns on pyarrow Table and Schema objects
---
 python/pyarrow/array.pxi            | 20 +++++++++++++++-----
 python/pyarrow/table.pxi            | 21 +++++++++++++++++----
 python/pyarrow/tests/test_schema.py | 20 ++++++++++++++++++++
 python/pyarrow/tests/test_table.py  | 21 +++++++++++++++++++++
 4 files changed, 73 insertions(+), 9 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 46e94b4f4b3e6..c132269c56311 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -244,12 +244,22 @@ cdef class Schema:
     def __len__(self):
         return self.schema.num_fields()
 
-    def __getitem__(self, i):
-        if i < 0 or i >= len(self):
-            raise IndexError("{0} is out of bounds".format(i))
+    def __getitem__(self, int64_t i):
+
+        cdef:
+            Field result = Field()
+            int64_t num_fields = self.schema.num_fields()
+            int64_t index
+
+        if not -num_fields <= i < num_fields:
+            raise IndexError(
+                'Schema field index {:d} is out of range'.format(i)
+            )
+
+        index = i if i >= 0 else num_fields + i
+        assert index >= 0
 
-        cdef Field result = Field()
-        result.init(self.schema.field(i))
+        result.init(self.schema.field(index))
         result.type = pyarrow_wrap_data_type(result.field.type())
 
         return result
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 8dd18cf4136f8..bd8cce41400a9 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -762,24 +762,37 @@ cdef class Table:
         """
         return pyarrow_wrap_schema(self.table.schema())
 
-    def column(self, index):
+    def column(self, int64_t i):
         """
         Select a column by its numeric index.
 
         Parameters
         ----------
-        index: int
+        i : int
 
         Returns
         -------
         pyarrow.Column
         """
         self._check_nullptr()
-        cdef Column column = Column()
+
+        cdef:
+            Column column = Column()
+            int64_t num_columns = self.num_columns
+            int64_t index
+
+        if not -num_columns <= i < num_columns:
+            raise IndexError(
+                'Table column index {:d} is out of range'.format(i)
+            )
+
+        index = i if i >= 0 else num_columns + i
+        assert index >= 0
+
         column.init(self.table.column(index))
         return column
 
-    def __getitem__(self, i):
+    def __getitem__(self, int64_t i):
         return self.column(i)
 
     def itercolumns(self):
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 2d98865b56e73..a6fe1a5df0e2c 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -212,3 +212,23 @@ def test_schema_equals():
     del fields[-1]
     sch3 = pa.schema(fields)
     assert not sch1.equals(sch3)
+
+
+def test_schema_negative_indexing():
+    fields = [
+        pa.field('foo', pa.int32()),
+        pa.field('bar', pa.string()),
+        pa.field('baz', pa.list_(pa.int8()))
+    ]
+
+    schema = pa.schema(fields)
+
+    assert schema[-1].equals(schema[2])
+    assert schema[-2].equals(schema[1])
+    assert schema[-3].equals(schema[0])
+
+    with pytest.raises(IndexError):
+        schema[-4]
+
+    with pytest.raises(IndexError):
+        schema[3]
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 0567e8aba685a..72ce6967edf19 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -254,3 +254,24 @@ def test_table_pandas():
     assert set(df.columns) == set(('a', 'b'))
     assert df.shape == (5, 2)
     assert df.loc[0, 'b'] == -10
+
+
+def test_table_negative_indexing():
+    data = [
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10]),
+        pa.array([1.0, 2.0, 3.0]),
+        pa.array(['ab', 'bc', 'cd']),
+    ]
+    table = pa.Table.from_arrays(data, names=tuple('abcd'))
+
+    assert table[-1].equals(table[3])
+    assert table[-2].equals(table[2])
+    assert table[-3].equals(table[1])
+    assert table[-4].equals(table[0])
+
+    with pytest.raises(IndexError):
+        table[-5]
+
+    with pytest.raises(IndexError):
+        table[4]

From ba9348fb3aec4d39a66faa8ab9d063223dd7e7a9 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 15 May 2017 16:13:25 -0400
Subject: [PATCH 0652/1644] ARROW-1031: [GLib] Support pretty print

Author: Kouhei Sutou <kou@clear-code.com>

Closes #688 from kou/glib-support-pretty-print and squashes the following commits:

815f87f [Kouhei Sutou] [GLib] Support pretty print
---
 c_glib/arrow-glib/array.cpp        | 27 +++++++++++++++++++++++++++
 c_glib/arrow-glib/array.h          |  2 ++
 c_glib/arrow-glib/record-batch.cpp | 29 +++++++++++++++++++++++++++++
 c_glib/arrow-glib/record-batch.h   |  3 +++
 c_glib/arrow-glib/table.h          |  2 ++
 c_glib/test/test-array.rb          |  5 +++++
 c_glib/test/test-record-batch.rb   |  7 +++++++
 7 files changed, 75 insertions(+)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 8bc6ea95d6a9d..e9a6a494c0faf 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -24,9 +24,11 @@
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/data-type.hpp>
+#include <arrow-glib/error.hpp>
 #include <arrow-glib/type.hpp>
 
 #include <iostream>
+#include <sstream>
 
 G_BEGIN_DECLS
 
@@ -395,6 +397,31 @@ garrow_array_slice(GArrowArray *array,
   return garrow_array_new_raw(&arrow_sub_array);
 }
 
+/**
+ * garrow_array_to_string:
+ * @array: A #GArrowArray.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable): The formatted array content or %NULL on error.
+ *
+ *   The returned string should be freed when with g_free() when no
+ *   longer needed.
+ *
+ * Since: 0.4.0
+ */
+gchar *
+garrow_array_to_string(GArrowArray *array, GError **error)
+{
+  const auto arrow_array = garrow_array_get_raw(array);
+  std::stringstream sink;
+  auto status = arrow::PrettyPrint(*arrow_array, 0, &sink);
+  if (garrow_error_check(error, status, "[array][to-string]")) {
+    return g_strdup(sink.str().c_str());
+  } else {
+    return NULL;
+  }
+}
+
 
 G_DEFINE_TYPE(GArrowNullArray,               \
               garrow_null_array,             \
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index c4efeafd6404a..d32157b74cbc9 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -79,6 +79,8 @@ GArrowType     garrow_array_get_value_type(GArrowArray *array);
 GArrowArray   *garrow_array_slice       (GArrowArray *array,
                                          gint64 offset,
                                          gint64 length);
+gchar         *garrow_array_to_string   (GArrowArray *array,
+                                         GError **error);
 
 #define GARROW_TYPE_NULL_ARRAY                  \
   (garrow_null_array_get_type())
diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp
index 3eed1a097c9e7..cd030de526f4c 100644
--- a/c_glib/arrow-glib/record-batch.cpp
+++ b/c_glib/arrow-glib/record-batch.cpp
@@ -22,9 +22,12 @@
 #endif
 
 #include <arrow-glib/array.hpp>
+#include <arrow-glib/error.hpp>
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 
+#include <sstream>
+
 G_BEGIN_DECLS
 
 /**
@@ -286,6 +289,32 @@ garrow_record_batch_slice(GArrowRecordBatch *record_batch,
   return garrow_record_batch_new_raw(&arrow_sub_record_batch);
 }
 
+/**
+ * garrow_record_batch_to_string:
+ * @record_batch: A #GArrowRecordBatch.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable): The formatted record batch content or %NULL on error.
+ *
+ *   The returned string should be freed when with g_free() when no
+ *   longer needed.
+ *
+ * Since: 0.4.0
+ */
+gchar *
+garrow_record_batch_to_string(GArrowRecordBatch *record_batch, GError **error)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  std::stringstream sink;
+  auto status = arrow::PrettyPrint(*arrow_record_batch, 0, &sink);
+  if (garrow_error_check(error, status, "[record-batch][to-string]")) {
+    return g_strdup(sink.str().c_str());
+  } else {
+    return NULL;
+  }
+}
+
+
 G_END_DECLS
 
 GArrowRecordBatch *
diff --git a/c_glib/arrow-glib/record-batch.h b/c_glib/arrow-glib/record-batch.h
index 61e8f3d42b1c8..021f894f3f5a3 100644
--- a/c_glib/arrow-glib/record-batch.h
+++ b/c_glib/arrow-glib/record-batch.h
@@ -85,4 +85,7 @@ GArrowRecordBatch *garrow_record_batch_slice     (GArrowRecordBatch *record_batc
                                                   gint64 offset,
                                                   gint64 length);
 
+gchar        *garrow_record_batch_to_string      (GArrowRecordBatch *record_batch,
+                                                  GError **error);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/table.h b/c_glib/arrow-glib/table.h
index 9e21669cd11da..7f83872163ffd 100644
--- a/c_glib/arrow-glib/table.h
+++ b/c_glib/arrow-glib/table.h
@@ -86,4 +86,6 @@ GArrowTable    *garrow_table_remove_column (GArrowTable *table,
                                             guint i,
                                             GError **error);
 
+gchar          *garrow_table_to_string     (GArrowTable *table);
+
 G_END_DECLS
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
index ca02fa283b014..bd0c7d6ecda09 100644
--- a/c_glib/test/test-array.rb
+++ b/c_glib/test/test-array.rb
@@ -96,4 +96,9 @@ def test_slice
     assert_equal([false, true],
                  sub_array.length.times.collect {|i| sub_array.get_value(i)})
   end
+
+  def test_to_s
+    assert_equal("[true, false, true]",
+                 build_boolean_array([true, false, true]).to_s)
+  end
 end
diff --git a/c_glib/test/test-record-batch.rb b/c_glib/test/test-record-batch.rb
index 048f6deb7245c..9fd34b7d45cff 100644
--- a/c_glib/test/test-record-batch.rb
+++ b/c_glib/test/test-record-batch.rb
@@ -91,5 +91,12 @@ def test_slice
       assert_equal([false, true],
                    sub_visible_values)
     end
+
+    def test_to_s
+      assert_equal(<<-PRETTY_PRINT, @record_batch.to_s)
+visible: [true, false, true, false, true, false]
+valid: [false, true, false, true, false]
+      PRETTY_PRINT
+    end
   end
 end

From 4381845b5edf8f41bbebe853e11b18e17531ae4a Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 15 May 2017 16:16:01 -0400
Subject: [PATCH 0653/1644] ARROW-1033: [Python] pytest discovers
 scripts/test_leak.py

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #689 from cpcloud/ARROW-1033 and squashes the following commits:

60a3bc3 [Phillip Cloud] ARROW-1033: [Python] pytest discovers scripts/test_leak.py
---
 python/setup.cfg | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/python/setup.cfg b/python/setup.cfg
index caae3e081b6ca..9b8c7cec25c2a 100644
--- a/python/setup.cfg
+++ b/python/setup.cfg
@@ -18,3 +18,6 @@
 [build_sphinx]
 source-dir = doc/
 build-dir  = doc/_build
+
+[tool:pytest]
+addopts = --ignore=scripts

From 681afabb4fa138e7dab694fa63f60220810eb17e Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Mon, 15 May 2017 16:16:53 -0400
Subject: [PATCH 0654/1644] ARROW-977: [java] Add Timezone aware timestamp
 vectors

Author: Julien Le Dem <julien@apache.org>

Closes #644 from julienledem/TZ and squashes the following commits:

37987b9 [Julien Le Dem] add integration tests
a58fcae [Julien Le Dem] add integration test
39966aa [Julien Le Dem] add other vectors and tests
bf245ce [Julien Le Dem] add TZ vectors
---
 integration/integration_test.py               |   6 +-
 .../main/codegen/data/ValueVectorTypes.tdd    |  36 ++-
 .../templates/AbstractFieldWriter.java        |  10 +-
 .../AbstractPromotableFieldWriter.java        |  31 +--
 .../main/codegen/templates/BaseWriter.java    |   4 +-
 .../main/codegen/templates/ComplexCopier.java |   7 +-
 .../codegen/templates/FixedValueVectors.java  |  50 ++--
 .../main/codegen/templates/MapWriters.java    |  38 ++-
 .../templates/NullableValueVectors.java       |  36 ++-
 .../codegen/templates/UnionListWriter.java    |   2 +-
 .../main/codegen/templates/UnionWriter.java   |   6 +-
 .../templates/VariableLengthVectors.java      |  47 ++--
 .../vector/file/json/JsonFileReader.java      |  20 +-
 .../org/apache/arrow/vector/types/Types.java  |  79 ++++--
 .../complex/writer/TestComplexWriter.java     | 232 +++++++++++++-----
 .../arrow/vector/file/BaseFileTest.java       |  13 +-
 .../apache/arrow/vector/pojo/TestConvert.java |   3 +-
 17 files changed, 429 insertions(+), 191 deletions(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 646646997f72c..cc59593dbdc1b 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -620,7 +620,11 @@ def generate_datetime_case():
         TimestampType('f7', 'ms'),
         TimestampType('f8', 'us'),
         TimestampType('f9', 'ns'),
-        TimestampType('f10', 'ms', tz=None)
+        TimestampType('f10', 'ms', tz=None),
+        TimestampType('f11', 's', tz='UTC'),
+        TimestampType('f12', 'ms', tz='US/Eastern'),
+        TimestampType('f13', 'us', tz='Europe/Paris'),
+        TimestampType('f14', 'ns', tz='US/Pacific')
     ]
 
     batch_sizes = [7, 10]
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index ca6d9ecbe85e6..4d8c99e652ffd 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -71,12 +71,28 @@
       minor: [
         { class: "BigInt"},
         { class: "UInt8" },
-        { class: "Float8",         javaType: "double", boxedType: "Double",                              fields: [{name: "value", type: "double"}], },
-        { class: "DateMilli",      javaType: "long",                      friendlyType: "LocalDateTime" },
-        { class: "TimeStampSec",   javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
-        { class: "TimeStampMilli", javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
-        { class: "TimeStampMicro", javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
-        { class: "TimeStampNano",  javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "Float8",           javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}] },
+        { class: "DateMilli",        javaType: "long",                      friendlyType: "LocalDateTime" },
+        { class: "TimeStampSec",     javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampMilli",   javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampMicro",   javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampNano",    javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampSecTZ", javaType: "long",   boxedType: "Long",
+                                     typeParams: [ {name: "timezone", type: "String"} ],
+                                     arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp",
+                                     arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.SECOND", "timezone"] },
+        { class: "TimeStampMilliTZ", javaType: "long",   boxedType: "Long",
+                                     typeParams: [ {name: "timezone", type: "String"} ],
+                                     arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp",
+                                     arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.MILLISECOND", "timezone"] },
+        { class: "TimeStampMicroTZ", javaType: "long",   boxedType: "Long",
+                                     typeParams: [ {name: "timezone", type: "String"} ],
+                                     arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp",
+                                     arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.MICROSECOND", "timezone"] },
+        { class: "TimeStampNanoTZ", javaType: "long",   boxedType: "Long",
+                                     typeParams: [ {name: "timezone", type: "String"} ],
+                                     arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp",
+                                     arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.NANOSECOND", "timezone"] },
         { class: "TimeMicro" },
         { class: "TimeNano" }
       ]
@@ -97,7 +113,13 @@
       boxedType: "ArrowBuf",
 
       minor: [
-        { class: "Decimal", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+        {
+          class: "Decimal",
+          maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal",
+          typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}],
+          arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal",
+          fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}]
+        }
       ]
     },
     {
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index de076fc46ffb2..65bcc052ed42d 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -59,7 +59,7 @@ public void write(${name}Holder holder) {
   }
 
   <#if minor.class == "Decimal">
-  public void writeDecimal(int start, ArrowBuf buffer) {
+  public void write${minor.class}(int start, ArrowBuf buffer) {
     fail("${name}");
   }
   <#else>
@@ -114,9 +114,11 @@ public ListWriter list(String name) {
   <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
   <#assign upperName = minor.class?upper_case />
   <#assign capName = minor.class?cap_first />
-  <#if minor.class?starts_with("Decimal") >
-  public ${capName}Writer ${lowerName}(String name, int scale, int precision) {
-    fail("${capName}");
+  <#if minor.typeParams?? >
+
+  @Override
+  public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
+    fail("${capName}(" + <#list minor.typeParams as typeParam>"${typeParam.name}: " + ${typeParam.name} + ", " + </#list>")");
     return null;
   }
   </#if>
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index ada0b1d5c7816..636b305d6db35 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -72,31 +72,23 @@ public void endList() {
   }
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-  <#assign fields = minor.fields!type.fields />
-  <#if !minor.class?starts_with("Decimal") >
+    <#assign fields = minor.fields!type.fields />
   @Override
   public void write(${name}Holder holder) {
     getWriter(MinorType.${name?upper_case}).write(holder);
   }
 
+    <#if minor.class == "Decimal">
+  public void write${minor.class}(int start, ArrowBuf buffer) {
+    getWriter(MinorType.${name?upper_case}).write${minor.class}(start, buffer);
+  }
+    <#else>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
     getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
   }
-
-  <#else>
-  @Override
-  public void write(DecimalHolder holder) {
-    getWriter(MinorType.DECIMAL).write(holder);
-  }
-
-  public void writeDecimal(int start, ArrowBuf buffer) {
-    getWriter(MinorType.DECIMAL).writeDecimal(start, buffer);
-  }
-
-  </#if>
+    </#if>
 
   </#list></#list>
-
   public void writeNull() {
   }
 
@@ -125,10 +117,13 @@ public ListWriter list(String name) {
   <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
   <#assign upperName = minor.class?upper_case />
   <#assign capName = minor.class?cap_first />
-  <#if minor.class?starts_with("Decimal") >
-  public ${capName}Writer ${lowerName}(String name, int scale, int precision) {
-    return getWriter(MinorType.MAP).${lowerName}(name, scale, precision);
+
+  <#if minor.typeParams?? >
+  @Override
+  public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
+    return getWriter(MinorType.MAP).${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name}</#list>);
   }
+
   </#if>
   @Override
   public ${capName}Writer ${lowerName}(String name) {
diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java
index 3da02b00a0dbf..405f466cbc708 100644
--- a/java/vector/src/main/codegen/templates/BaseWriter.java
+++ b/java/vector/src/main/codegen/templates/BaseWriter.java
@@ -53,8 +53,8 @@ public interface MapWriter extends BaseWriter {
     <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
     <#assign upperName = minor.class?upper_case />
     <#assign capName = minor.class?cap_first />
-    <#if minor.class?starts_with("Decimal") >
-    ${capName}Writer ${lowerName}(String name, int scale, int precision);
+    <#if minor.typeParams?? >
+    ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>);
     </#if>
     ${capName}Writer ${lowerName}(String name);
     </#list></#list>
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index fb7ae0f2ef57e..518ad5d398ddb 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -72,7 +72,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
   <#assign uncappedName = name?uncap_first/>
-  <#if !minor.class?starts_with("Decimal")>
+
+  <#if !minor.typeParams?? >
 
       case ${name?upper_case}:
         if (reader.isSet()) {
@@ -94,7 +95,7 @@ private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter w
     <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
     <#assign fields = minor.fields!type.fields />
     <#assign uncappedName = name?uncap_first/>
-    <#if !minor.class?starts_with("Decimal")>
+    <#if !minor.typeParams?? >
     case ${name?upper_case}:
       return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name);
     </#if>
@@ -113,7 +114,7 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter
     <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
     <#assign fields = minor.fields!type.fields />
     <#assign uncappedName = name?uncap_first/>
-    <#if !minor.class?starts_with("Decimal")>
+    <#if !minor.typeParams?? >
     case ${name?upper_case}:
     return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>();
     </#if>
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index f403ecfac1f93..5d80b66b5332c 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -25,9 +25,10 @@
 <#list vv.types as type>
 <#list type.minor as minor>
 <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+<#assign className = "${minor.class}Vector" />
 
 <#if type.major == "Fixed">
-<@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" />
 <#include "/@includes/license.ftl" />
 
 package org.apache.arrow.vector;
@@ -43,8 +44,8 @@
  *
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
-public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+public final class ${className} extends BaseDataValueVector implements FixedWidthVector{
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
   public static final int TYPE_WIDTH = ${type.width};
 
@@ -53,24 +54,25 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
   private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
   private int allocationMonitor = 0;
+  <#if minor.typeParams??>
 
-  <#if minor.class == "Decimal">
+     <#list minor.typeParams as typeParam>
+  private final ${typeParam.type} ${typeParam.name};
+    </#list>
 
-  private int precision;
-  private int scale;
-
-  public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) {
+  public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
     super(name, allocator);
-    this.precision = precision;
-    this.scale = scale;
+    <#list minor.typeParams as typeParam>
+    this.${typeParam.name} = ${typeParam.name};
+    </#list>
   }
   <#else>
-  public ${minor.class}Vector(String name, BufferAllocator allocator) {
+
+  public ${className}(String name, BufferAllocator allocator) {
     super(name, allocator);
   }
   </#if>
 
-
   @Override
   public MinorType getMinorType() {
     return MinorType.${minor.class?upper_case};
@@ -219,17 +221,17 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator){
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((${minor.class}Vector) to);
+    return new TransferImpl((${className}) to);
   }
 
-  public void transferTo(${minor.class}Vector target){
+  public void transferTo(${className} target){
     target.clear();
     target.data = data.transferOwnership(target.allocator).buffer;
     target.data.writerIndex(data.writerIndex());
     clear();
   }
 
-  public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
+  public void splitAndTransferTo(int startIndex, int length, ${className} target) {
     final int startPoint = startIndex * ${type.width};
     final int sliceLength = length * ${type.width};
     target.clear();
@@ -238,22 +240,18 @@ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector
   }
 
   private class TransferImpl implements TransferPair{
-    private ${minor.class}Vector to;
+    private ${className} to;
 
     public TransferImpl(String name, BufferAllocator allocator){
-      <#if minor.class == "Decimal">
-      to = new ${minor.class}Vector(name, allocator, precision, scale);
-      <#else>
-      to = new ${minor.class}Vector(name, allocator);
-      </#if>
+      to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>,  ${className}.this.${typeParam.name}</#list></#if>);
     }
 
-    public TransferImpl(${minor.class}Vector to) {
+    public TransferImpl(${className} to) {
       this.to = to;
     }
 
     @Override
-    public ${minor.class}Vector getTo(){
+    public ${className} getTo(){
       return to;
     }
 
@@ -269,11 +267,11 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, ${className}.this);
     }
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+  public void copyFrom(int fromIndex, int thisIndex, ${className} from){
     <#if (type.width > 8 || minor.class == "IntervalDay")>
     from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
     <#else> <#-- type.width <= 8 -->
@@ -283,7 +281,7 @@ public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
     </#if> <#-- type.width -->
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+  public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
     while(thisIndex >= getValueCapacity()) {
         reAlloc();
     }
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index d3e6de9527123..05048c5c341c0 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -56,7 +56,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
     </#if>
     this.container = container;
     for (Field child : container.getField().getChildren()) {
-      switch (Types.getMinorTypeForArrowType(child.getType())) {
+      MinorType minorType = Types.getMinorTypeForArrowType(child.getType());
+      switch (minorType) {
       case MAP:
         map(child.getName());
         break;
@@ -71,15 +72,18 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
 <#assign lowerName = minor.class?uncap_first />
 <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
 <#assign upperName = minor.class?upper_case />
-      case ${upperName}:
-        <#if lowerName == "decimal" >
-        Decimal decimal = (Decimal)child.getType();
-        decimal(child.getName(), decimal.getScale(), decimal.getPrecision());
+      case ${upperName}: {
+        <#if minor.typeParams?? >
+        ${minor.arrowType} arrowType = (${minor.arrowType})child.getType();
+        ${lowerName}(child.getName()<#list minor.typeParams as typeParam>, arrowType.get${typeParam.name?cap_first}()</#list>);
         <#else>
         ${lowerName}(child.getName());
-       </#if>
+        </#if>
         break;
+      }
 </#list></#list>
+        default:
+          throw new UnsupportedOperationException("Unknown type: " + minorType);
       }
     }
   }
@@ -205,7 +209,8 @@ public void end() {
   <#assign vectName = capName />
   <#assign vectName = "Nullable${capName}" />
 
-  <#if minor.class?starts_with("Decimal") >
+  <#if minor.typeParams?? >
+  @Override
   public ${minor.class}Writer ${lowerName}(String name) {
     // returns existing writer
     final FieldWriter writer = fields.get(handleCase(name));
@@ -213,7 +218,8 @@ public void end() {
     return writer;
   }
 
-  public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) {
+  @Override
+  public ${minor.class}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
   <#else>
   @Override
   public ${minor.class}Writer ${lowerName}(String name) {
@@ -223,7 +229,21 @@ public void end() {
       ValueVector vector;
       ValueVector currentVector = container.getChild(name);
       ${vectName}Vector v = container.addOrGet(name, 
-          FieldType.nullable(<#if minor.class == "Decimal">new Decimal(precision, scale)<#else>MinorType.${upperName}.getType()</#if>),
+          FieldType.nullable(
+          <#if minor.typeParams??>
+            <#if minor.arrowTypeConstructorParams??>
+              <#assign constructorParams = minor.arrowTypeConstructorParams />
+            <#else>
+              <#assign constructorParams = [] />
+              <#list minor.typeParams as typeParam>
+                <#assign constructorParams = constructorParams + [ typeParam.name ] />
+              </#list>
+            </#if>    
+            new ${minor.arrowType}(${constructorParams?join(", ")})
+          <#else>
+            MinorType.${upperName}.getType()
+          </#if>
+          ),
           ${vectName}Vector.class);
       writer = new PromotableWriter(v, container, getNullableMapWriterFactory());
       vector = v;
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index ed2418e4a2a97..3231c4c6e0ae6 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -61,12 +61,27 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 
   private final List<BufferBacked> innerVectors;
 
-  <#if minor.class == "Decimal">
-  private final int precision;
-  private final int scale;
+  <#if minor.typeParams??>
+     <#list minor.typeParams as typeParam>
+  private final ${typeParam.type} ${typeParam.name};
+    </#list>
 
-  public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
-    this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
+  /**
+   * Assumes the type is nullable and not dictionary encoded
+   * @param name name of the field
+   * @param allocator allocator to use to resize the vector<#list minor.typeParams as typeParam>
+   * @param ${typeParam.name} type parameter ${typeParam.name}</#list>
+   */
+  public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
+    <#if minor.arrowTypeConstructorParams??>
+       <#assign constructorParams = minor.arrowTypeConstructorParams />
+    <#else>
+       <#assign constructorParams = [] />
+       <#list minor.typeParams as typeParam>
+         <#assign constructorParams = constructorParams + [ typeParam.name ] />
+      </#list>
+    </#if>
+    this(name, FieldType.nullable(new ${minor.arrowType}(${constructorParams?join(", ")})), allocator);
   }
   <#else>
   public ${className}(String name, BufferAllocator allocator) {
@@ -76,11 +91,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 
   public ${className}(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator);
-    <#if minor.class == "Decimal">
-    Decimal decimal = (Decimal)fieldType.getType();
-    this.precision = decimal.getPrecision();
-    this.scale = decimal.getScale();
-    this.values = new ${valuesName}(valuesField, allocator, precision, scale);
+    <#if minor.typeParams??>
+    ${minor.arrowType} arrowType = (${minor.arrowType})fieldType.getType();
+    <#list minor.typeParams as typeParam>
+    this.${typeParam.name} = arrowType.get${typeParam.name?cap_first}();
+    </#list>
+    this.values = new ${valuesName}(valuesField, allocator<#list minor.typeParams as typeParam>, ${typeParam.name}</#list>);
     <#else>
     this.values = new ${valuesName}(valuesField, allocator);
     </#if>
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index d980830923b31..d019a1edbb4a6 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -94,7 +94,7 @@ public void setPosition(int index) {
   <#assign fields = minor.fields!type.fields />
   <#assign uncappedName = name?uncap_first/>
 
-  <#if !minor.class?starts_with("Decimal")>
+  <#if !minor.typeParams?? >
 
   @Override
   public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>() {
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 880f537c0296f..4a7c47245cfb0 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -121,7 +121,7 @@ BaseWriter getWriter(MinorType minorType) {
       <#assign name = minor.class?cap_first />
       <#assign fields = minor.fields!type.fields />
       <#assign uncappedName = name?uncap_first/>
-      <#if !minor.class?starts_with("Decimal")>
+      <#if !minor.typeParams??>
     case ${name?upper_case}:
       return get${name}Writer();
       </#if>
@@ -136,7 +136,7 @@ BaseWriter getWriter(MinorType minorType) {
   <#assign fields = minor.fields!type.fields />
   <#assign uncappedName = name?uncap_first/>
 
-          <#if !minor.class?starts_with("Decimal")>
+          <#if !minor.typeParams?? >
 
   private ${name}Writer ${name?uncap_first}Writer;
 
@@ -206,7 +206,7 @@ public MapWriter map(String name) {
   <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
   <#assign upperName = minor.class?upper_case />
   <#assign capName = minor.class?cap_first />
-  <#if !minor.class?starts_with("Decimal")>
+  <#if !minor.typeParams?? >
   @Override
   public ${capName}Writer ${lowerName}(String name) {
     data.getMutator().setType(idx(), MinorType.MAP);
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index 3d933addb6208..f13291b79ade0 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -28,6 +28,7 @@
 <#list type.minor as minor>
 
 <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+<#assign className = "${minor.class}Vector" />
 
 <#if type.major == "VarLen">
 <@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" />
@@ -48,8 +49,8 @@
  *
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
-public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+public final class ${className} extends BaseDataValueVector implements VariableWidthVector{
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
   private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
   private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
@@ -66,22 +67,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
   private int allocationSizeInBytes = INITIAL_BYTE_COUNT;
   private int allocationMonitor = 0;
 
-  <#if minor.class == "Decimal">
+  <#if minor.typeParams??>
+     <#list minor.typeParams as typeParam>
+  private final ${typeParam.type} ${typeParam.name};
+    </#list>
 
-  private final int precision;
-  private final int scale;
-
-  public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) {
+  public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
     super(name, allocator);
     this.oAccessor = offsetVector.getAccessor();
     this.accessor = new Accessor();
     this.mutator = new Mutator();
-    this.precision = precision;
-    this.scale = scale;
+    <#list minor.typeParams as typeParam>
+    this.${typeParam.name} = ${typeParam.name};
+    </#list>
   }
   <#else>
-
-  public ${minor.class}Vector(String name, BufferAllocator allocator) {
+  public ${className}(String name, BufferAllocator allocator) {
     super(name, allocator);
     this.oAccessor = offsetVector.getAccessor();
     this.accessor = new Accessor();
@@ -188,10 +189,10 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator){
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((${minor.class}Vector) to);
+    return new TransferImpl((${className}) to);
   }
 
-  public void transferTo(${minor.class}Vector target){
+  public void transferTo(${className} target){
     target.clear();
     this.offsetVector.transferTo(target.offsetVector);
     target.data = data.transferOwnership(target.allocator).buffer;
@@ -199,7 +200,7 @@ public void transferTo(${minor.class}Vector target){
     clear();
   }
 
-  public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
+  public void splitAndTransferTo(int startIndex, int length, ${className} target) {
     UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor();
     final int startPoint = offsetVectorAccessor.get(startIndex);
     final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint;
@@ -214,7 +215,7 @@ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector
     target.getMutator().setValueCount(length);
 }
 
-  protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+  protected void copyFrom(int fromIndex, int thisIndex, ${className} from){
     final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
     final int start = fromOffsetVectorAccessor.get(fromIndex);
     final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
@@ -225,7 +226,7 @@ protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from)
     offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
   }
 
-  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+  public boolean copyFromSafe(int fromIndex, int thisIndex, ${className} from){
     final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
     final int start = fromOffsetVectorAccessor.get(fromIndex);
     final int end =   fromOffsetVectorAccessor.get(fromIndex + 1);
@@ -242,22 +243,18 @@ public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector f
   }
 
   private class TransferImpl implements TransferPair{
-    ${minor.class}Vector to;
+    ${className} to;
 
     public TransferImpl(String name, BufferAllocator allocator){
-      <#if minor.class == "Decimal">
-      to = new ${minor.class}Vector(name, allocator, precision, scale);
-      <#else>
-      to = new ${minor.class}Vector(name, allocator);
-      </#if>
+      to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>,  ${className}.this.${typeParam.name}</#list></#if>);
     }
 
-    public TransferImpl(${minor.class}Vector to){
+    public TransferImpl(${className} to){
       this.to = to;
     }
 
     @Override
-    public ${minor.class}Vector getTo(){
+    public ${className} getTo(){
       return to;
     }
 
@@ -273,7 +270,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, ${className}.this);
     }
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 21aa0372c6b38..3ef1484e7eaba 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -43,9 +43,13 @@
 import org.apache.arrow.vector.TimeMilliVector;
 import org.apache.arrow.vector.TimeNanoVector;
 import org.apache.arrow.vector.TimeSecVector;
+import org.apache.arrow.vector.TimeStampMicroTZVector;
 import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampMilliTZVector;
 import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampNanoTZVector;
 import org.apache.arrow.vector.TimeStampNanoVector;
+import org.apache.arrow.vector.TimeStampSecTZVector;
 import org.apache.arrow.vector.TimeStampSecVector;
 import org.apache.arrow.vector.TinyIntVector;
 import org.apache.arrow.vector.UInt1Vector;
@@ -61,14 +65,14 @@
 import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.databind.MappingJsonFactory;
 import com.google.common.base.Objects;
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Hex;
 
 public class JsonFileReader implements AutoCloseable {
   private final File inputFile;
@@ -278,6 +282,18 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
     case TIMESTAMPNANO:
       ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
       break;
+    case TIMESTAMPSECTZ:
+      ((TimeStampSecTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case TIMESTAMPMILLITZ:
+      ((TimeStampMilliTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case TIMESTAMPMICROTZ:
+      ((TimeStampMicroTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
+    case TIMESTAMPNANOTZ:
+      ((TimeStampNanoTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+      break;
     default:
       throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 2a0e47b7bf7c2..6591a4b16da16 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -38,9 +38,13 @@
 import org.apache.arrow.vector.NullableTimeMilliVector;
 import org.apache.arrow.vector.NullableTimeNanoVector;
 import org.apache.arrow.vector.NullableTimeSecVector;
+import org.apache.arrow.vector.NullableTimeStampMicroTZVector;
 import org.apache.arrow.vector.NullableTimeStampMicroVector;
+import org.apache.arrow.vector.NullableTimeStampMilliTZVector;
 import org.apache.arrow.vector.NullableTimeStampMilliVector;
+import org.apache.arrow.vector.NullableTimeStampNanoTZVector;
 import org.apache.arrow.vector.NullableTimeStampNanoVector;
+import org.apache.arrow.vector.NullableTimeStampSecTZVector;
 import org.apache.arrow.vector.NullableTimeStampSecVector;
 import org.apache.arrow.vector.NullableTinyIntVector;
 import org.apache.arrow.vector.NullableUInt1Vector;
@@ -71,9 +75,13 @@
 import org.apache.arrow.vector.complex.impl.TimeMilliWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeNanoWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeSecWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampMicroTZWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampMilliTZWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampNanoTZWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampSecTZWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl;
 import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.UInt1WriterImpl;
@@ -368,11 +376,6 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     DECIMAL(null) {
-      @Override
-      public ArrowType getType() {
-        throw new UnsupportedOperationException("Cannot get simple type for Decimal type");
-      }
-
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
         return new NullableDecimalVector(name, fieldType, allocator);
@@ -439,11 +442,6 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       }
     },
     FIXED_SIZE_LIST(null) {
-      @Override
-      public ArrowType getType() {
-        throw new UnsupportedOperationException("Cannot get simple type for FixedSizeList type");
-      }
-
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
         return new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback);
@@ -467,6 +465,50 @@ public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocato
       public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new UnionWriter((UnionVector) vector);
       }
+    },
+    TIMESTAMPSECTZ(null) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeStampSecTZVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeStampSecTZWriterImpl((NullableTimeStampSecTZVector) vector);
+      }
+    },
+    TIMESTAMPMILLITZ(null) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeStampMilliTZVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeStampMilliTZWriterImpl((NullableTimeStampMilliTZVector) vector);
+      }
+    },
+    TIMESTAMPMICROTZ(null) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeStampMicroTZVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeStampMicroTZWriterImpl((NullableTimeStampMicroTZVector) vector);
+      }
+    },
+    TIMESTAMPNANOTZ(null) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new NullableTimeStampNanoTZVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new TimeStampNanoTZWriterImpl((NullableTimeStampNanoTZVector) vector);
+      }
     };
 
     private final ArrowType type;
@@ -475,7 +517,10 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
       this.type = type;
     }
 
-    public ArrowType getType() {
+    public final ArrowType getType() {
+      if (type == null) {
+        throw new UnsupportedOperationException("Cannot get simple type for type " + name());
+      }
       return type;
     }
 
@@ -579,18 +624,16 @@ public MinorType visit(FloatingPoint type) {
       }
 
       @Override public MinorType visit(Timestamp type) {
-        if (type.getTimezone() != null) {
-          throw new IllegalArgumentException("only timezone-less timestamps are supported for now: " + type);
-        }
+        String tz = type.getTimezone();
         switch (type.getUnit()) {
           case SECOND:
-            return MinorType.TIMESTAMPSEC;
+            return tz == null ? MinorType.TIMESTAMPSEC : MinorType.TIMESTAMPSECTZ;
           case MILLISECOND:
-            return MinorType.TIMESTAMPMILLI;
+            return tz == null ? MinorType.TIMESTAMPMILLI : MinorType.TIMESTAMPMILLITZ;
           case MICROSECOND:
-            return MinorType.TIMESTAMPMICRO;
+            return tz == null ? MinorType.TIMESTAMPMICRO : MinorType.TIMESTAMPMICROTZ;
           case NANOSECOND:
-            return MinorType.TIMESTAMPNANO;
+            return tz == null ? MinorType.TIMESTAMPNANO : MinorType.TIMESTAMPNANOTZ;
           default:
             throw new IllegalArgumentException("unknown unit: " + type);
         }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 16139361756b8..ede8d65f07507 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -42,15 +42,18 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.DateUtility;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.Text;
@@ -592,85 +595,198 @@ public void mapWriterMixedCaseFieldNames() {
   }
 
   @Test
-  public void timeStampWriters() throws Exception {
+  public void timeStampSecWriter() throws Exception {
     // test values
-    final long expectedNanos = 981173106123456789L;
-    final long expectedMicros = 981173106123456L;
-    final long expectedMillis = 981173106123L;
     final long expectedSecs = 981173106L;
     final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0);
+
+    // write
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+
+    {
+      TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec");
+      timeStampSecWriter.setPosition(0);
+      timeStampSecWriter.writeTimeStampSec(expectedSecs);
+    }
+    {
+      TimeStampSecTZWriter timeStampSecTZWriter = rootWriter.timeStampSecTZ("secTZ", "UTC");
+      timeStampSecTZWriter.setPosition(1);
+      timeStampSecTZWriter.writeTimeStampSecTZ(expectedSecs);
+    }
+    // schema
+    List<Field> children = parent.getField().getChildren().get(0).getChildren();
+    checkTimestampField(children.get(0), "sec");
+    checkTimestampTZField(children.get(1), "secTZ", "UTC");
+
+    // read
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    {
+      FieldReader secReader = rootReader.reader("sec");
+      secReader.setPosition(0);
+      LocalDateTime secDateTime = secReader.readLocalDateTime();
+      Assert.assertEquals(expectedSecDateTime, secDateTime);
+      long secLong = secReader.readLong();
+      Assert.assertEquals(expectedSecs, secLong);
+    }
+    {
+      FieldReader secTZReader = rootReader.reader("secTZ");
+      secTZReader.setPosition(1);
+      long secTZLong = secTZReader.readLong();
+      Assert.assertEquals(expectedSecs, secTZLong);
+    }
+  }
+
+  @Test
+  public void timeStampMilliWriters() throws Exception {
+    // test values
+    final long expectedMillis = 981173106123L;
     final LocalDateTime expectedMilliDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
-    final LocalDateTime expectedMicroDateTime = expectedMilliDateTime;
-    final LocalDateTime expectedNanoDateTime = expectedMilliDateTime;
 
     // write
     MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
+    {
+      TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli");
+      timeStampWriter.setPosition(0);
+      timeStampWriter.writeTimeStampMilli(expectedMillis);
+    }
+    String tz = DateUtility.getTimeZone(10);
+    {
+      TimeStampMilliTZWriter timeStampTZWriter = rootWriter.timeStampMilliTZ("milliTZ", tz);
+      timeStampTZWriter.setPosition(0);
+      timeStampTZWriter.writeTimeStampMilliTZ(expectedMillis);
+    }
+    // schema
+    List<Field> children = parent.getField().getChildren().get(0).getChildren();
+    checkTimestampField(children.get(0), "milli");
+    checkTimestampTZField(children.get(1), "milliTZ", tz);
+
+    // read
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+
+    {
+      FieldReader milliReader = rootReader.reader("milli");
+      milliReader.setPosition(0);
+      LocalDateTime milliDateTime = milliReader.readLocalDateTime();
+      Assert.assertEquals(expectedMilliDateTime, milliDateTime);
+      long milliLong = milliReader.readLong();
+      Assert.assertEquals(expectedMillis, milliLong);
+    }
+    {
+      FieldReader milliTZReader = rootReader.reader("milliTZ");
+      milliTZReader.setPosition(0);
+      long milliTZLong = milliTZReader.readLong();
+      Assert.assertEquals(expectedMillis, milliTZLong);
+    }
 
-    TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec");
-    timeStampSecWriter.setPosition(0);
-    timeStampSecWriter.writeTimeStampSec(expectedSecs);
+  }
 
-    TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli");
-    timeStampWriter.setPosition(1);
-    timeStampWriter.writeTimeStampMilli(expectedMillis);
+  private void checkTimestampField(Field field, String name) {
+    Assert.assertEquals(name, field.getName());
+    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, field.getType().getTypeID());
+  }
 
-    TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro");
-    timeStampMicroWriter.setPosition(2);
-    timeStampMicroWriter.writeTimeStampMicro(expectedMicros);
+  private void checkTimestampTZField(Field field, String name, String tz) {
+    checkTimestampField(field, name);
+    Assert.assertEquals(tz, ((Timestamp)field.getType()).getTimezone());
+  }
 
-    TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano");
-    timeStampNanoWriter.setPosition(3);
-    timeStampNanoWriter.writeTimeStampNano(expectedNanos);
+  @Test
+  public void timeStampMicroWriters() throws Exception {
+    // test values
+    final long expectedMicros = 981173106123456L;
+    final LocalDateTime expectedMicroDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
+
+    // write
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+
+    {
+      TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro");
+      timeStampMicroWriter.setPosition(0);
+      timeStampMicroWriter.writeTimeStampMicro(expectedMicros);
+    }
+    String tz = DateUtility.getTimeZone(5);
+    {
+      TimeStampMicroTZWriter timeStampMicroWriter = rootWriter.timeStampMicroTZ("microTZ", tz);
+      timeStampMicroWriter.setPosition(1);
+      timeStampMicroWriter.writeTimeStampMicroTZ(expectedMicros);
+    }
 
     // schema
-    Field secField = parent.getField().getChildren().get(0).getChildren().get(0);
-    Assert.assertEquals("sec", secField.getName());
-    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, secField.getType().getTypeID());
+    List<Field> children = parent.getField().getChildren().get(0).getChildren();
+    checkTimestampField(children.get(0), "micro");
+    checkTimestampTZField(children.get(1), "microTZ", tz);
+
+    // read
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    {
+      FieldReader microReader = rootReader.reader("micro");
+      microReader.setPosition(0);
+      LocalDateTime microDateTime = microReader.readLocalDateTime();
+      Assert.assertEquals(expectedMicroDateTime, microDateTime);
+      long microLong = microReader.readLong();
+      Assert.assertEquals(expectedMicros, microLong);
+    }
+    {
+      FieldReader microReader = rootReader.reader("microTZ");
+      microReader.setPosition(1);
+      long microLong = microReader.readLong();
+      Assert.assertEquals(expectedMicros, microLong);
+    }
 
-    Field milliField = parent.getField().getChildren().get(0).getChildren().get(1);
-    Assert.assertEquals("milli", milliField.getName());
-    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, milliField.getType().getTypeID());
+  }
 
-    Field microField = parent.getField().getChildren().get(0).getChildren().get(2);
-    Assert.assertEquals("micro", microField.getName());
-    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, microField.getType().getTypeID());
+  @Test
+  public void timeStampNanoWriters() throws Exception {
+    // test values
+    final long expectedNanos = 981173106123456789L;
+    final LocalDateTime expectedNanoDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
 
-    Field nanoField = parent.getField().getChildren().get(0).getChildren().get(3);
-    Assert.assertEquals("nano", nanoField.getName());
-    Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, nanoField.getType().getTypeID());
+    // write
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
 
+    {
+      TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano");
+      timeStampNanoWriter.setPosition(0);
+      timeStampNanoWriter.writeTimeStampNano(expectedNanos);
+    }
+    String tz = DateUtility.getTimeZone(3);
+    {
+      TimeStampNanoTZWriter timeStampNanoWriter = rootWriter.timeStampNanoTZ("nanoTZ", tz);
+      timeStampNanoWriter.setPosition(0);
+      timeStampNanoWriter.writeTimeStampNanoTZ(expectedNanos);
+    }
+    // schema
+    List<Field> children = parent.getField().getChildren().get(0).getChildren();
+    checkTimestampField(children.get(0), "nano");
+    checkTimestampTZField(children.get(1), "nanoTZ", tz);
     // read
     MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
 
-    FieldReader secReader = rootReader.reader("sec");
-    secReader.setPosition(0);
-    LocalDateTime secDateTime = secReader.readLocalDateTime();
-    Assert.assertEquals(expectedSecDateTime, secDateTime);
-    long secLong = secReader.readLong();
-    Assert.assertEquals(expectedSecs, secLong);
-
-    FieldReader milliReader = rootReader.reader("milli");
-    milliReader.setPosition(1);
-    LocalDateTime milliDateTime = milliReader.readLocalDateTime();
-    Assert.assertEquals(expectedMilliDateTime, milliDateTime);
-    long milliLong = milliReader.readLong();
-    Assert.assertEquals(expectedMillis, milliLong);
-
-    FieldReader microReader = rootReader.reader("micro");
-    microReader.setPosition(2);
-    LocalDateTime microDateTime = microReader.readLocalDateTime();
-    Assert.assertEquals(expectedMicroDateTime, microDateTime);
-    long microLong = microReader.readLong();
-    Assert.assertEquals(expectedMicros, microLong);
-
-    FieldReader nanoReader = rootReader.reader("nano");
-    nanoReader.setPosition(3);
-    LocalDateTime nanoDateTime = nanoReader.readLocalDateTime();
-    Assert.assertEquals(expectedNanoDateTime, nanoDateTime);
-    long nanoLong = nanoReader.readLong();
-    Assert.assertEquals(expectedNanos, nanoLong);
+    {
+      FieldReader nanoReader = rootReader.reader("nano");
+      nanoReader.setPosition(0);
+      LocalDateTime nanoDateTime = nanoReader.readLocalDateTime();
+      Assert.assertEquals(expectedNanoDateTime, nanoDateTime);
+      long nanoLong = nanoReader.readLong();
+      Assert.assertEquals(expectedNanos, nanoLong);
+    }
+    {
+      FieldReader nanoReader = rootReader.reader("nanoTZ");
+      nanoReader.setPosition(0);
+      long nanoLong = nanoReader.readLong();
+      Assert.assertEquals(expectedNanos, nanoLong);
+      NullableTimeStampNanoTZHolder h = new NullableTimeStampNanoTZHolder();
+      nanoReader.read(h);
+      Assert.assertEquals(expectedNanos, h.value);
+    }
   }
 
   @Test
@@ -710,4 +826,4 @@ public void complexCopierWithList() {
     innerMap = (JsonStringHashMap<?,?>) object.get(3);
     assertEquals(2, innerMap.get("a"));
   }
-}
\ No newline at end of file
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 5cc36a3b82000..63027e6c4633b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -37,8 +37,10 @@
 import org.apache.arrow.vector.complex.writer.DateMilliWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
 import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
+import org.apache.arrow.vector.complex.writer.TimeStampMilliTZWriter;
 import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter;
 import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
+import org.apache.arrow.vector.util.DateUtility;
 import org.joda.time.DateTimeZone;
 import org.joda.time.LocalDateTime;
 import org.junit.After;
@@ -155,18 +157,21 @@ protected void writeDateTimeData(int count, NullableMapVector parent) {
     DateMilliWriter dateWriter = rootWriter.dateMilli("date");
     TimeMilliWriter timeWriter = rootWriter.timeMilli("time");
     TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli");
+    TimeStampMilliTZWriter timeStampMilliTZWriter = rootWriter.timeStampMilliTZ("timestamp-milliTZ", "Europe/Paris");
     for (int i = 0; i < count; i++) {
       LocalDateTime dt = makeDateTimeFromCount(i);
       // Number of days in milliseconds since epoch, stored as 64-bit integer, only date part is used
       dateWriter.setPosition(i);
-      long dateLong = org.apache.arrow.vector.util.DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay()));
+      long dateLong = DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay()));
       dateWriter.writeDateMilli(dateLong);
       // Time is a value in milliseconds since midnight, stored as 32-bit integer
       timeWriter.setPosition(i);
       timeWriter.writeTimeMilli(dt.getMillisOfDay());
       // Timestamp is milliseconds since the epoch, stored as 64-bit integer
       timeStampMilliWriter.setPosition(i);
-      timeStampMilliWriter.writeTimeStampMilli(org.apache.arrow.vector.util.DateUtility.toMillis(dt));
+      timeStampMilliWriter.writeTimeStampMilli(DateUtility.toMillis(dt));
+      timeStampMilliTZWriter.setPosition(i);
+      timeStampMilliTZWriter.writeTimeStampMilliTZ(DateUtility.toMillis(dt));
     }
     writer.setValueCount(count);
   }
@@ -178,11 +183,13 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
       long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i);
       LocalDateTime dt = makeDateTimeFromCount(i);
       LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
-      Assert.assertEquals(org.apache.arrow.vector.util.DateUtility.toMillis(dateExpected), dateVal);
+      Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal);
       long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i);
       Assert.assertEquals(dt.getMillisOfDay(), timeVal);
       Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
       Assert.assertEquals(dt, timestampMilliVal);
+      Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getAccessor().getObject(i);
+      Assert.assertEquals(DateUtility.toMillis(dt), timestampMilliTZVal);
     }
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 64f797030ffb4..e2dae2933217e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -83,7 +83,8 @@ public void nestedSchema() {
         )));
     childrenBuilder.add(new Field("child5", FieldType.nullable(new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } )), ImmutableList.<Field>of(
         new Field("child5.1", FieldType.nullable(new Timestamp(TimeUnit.MILLISECOND, null)), null),
-        new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of())
+        new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of()),
+        new Field("child5.3", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null)
         )));
     Schema initialSchema = new Schema(childrenBuilder.build());
     run(initialSchema);

From b23b8643aa91bb00610dbe6fa566d11ec2f50402 Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Mon, 15 May 2017 16:19:35 -0400
Subject: [PATCH 0655/1644] ARROW-1015 [Java] Schema-level metadata

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #676 from elahrvivaz/ARROW-1015 and squashes the following commits:

9e94e1a [Emilio Lahr-Vivaz] Changing accessor to getCustomMetadata
9ffdb5a [Emilio Lahr-Vivaz] ARROW-1015 [Java] Schema-level metadata
---
 .../apache/arrow/vector/VectorSchemaRoot.java | 12 +++--
 .../apache/arrow/vector/file/ArrowReader.java |  7 +--
 .../apache/arrow/vector/file/ArrowWriter.java |  2 +-
 .../arrow/vector/types/pojo/Schema.java       | 52 +++++++++++++++++--
 .../arrow/vector/file/TestArrowFile.java      | 17 ++++--
 .../apache/arrow/vector/pojo/TestConvert.java | 21 ++++++--
 6 files changed, 90 insertions(+), 21 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
index 29b96736001ce..73deb0b3a426e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -41,11 +41,15 @@ public VectorSchemaRoot(FieldVector parent) {
   }
 
   public VectorSchemaRoot(List<Field> fields, List<FieldVector> fieldVectors, int rowCount) {
-    if (fields.size() != fieldVectors.size()) {
+    this(new Schema(fields), fieldVectors, rowCount);
+  }
+
+  public VectorSchemaRoot(Schema schema, List<FieldVector> fieldVectors, int rowCount) {
+    if (schema.getFields().size() != fieldVectors.size()) {
       throw new IllegalArgumentException("Fields must match field vectors. Found " +
-          fieldVectors.size() + " vectors and " + fields.size() + " fields");
+          fieldVectors.size() + " vectors and " + schema.getFields().size() + " fields");
     }
-    this.schema = new Schema(fields);
+    this.schema = schema;
     this.rowCount = rowCount;
     this.fieldVectors = fieldVectors;
     for (int i = 0; i < schema.getFields().size(); ++i) {
@@ -65,7 +69,7 @@ public static VectorSchemaRoot create(Schema schema, BufferAllocator allocator)
       throw new IllegalArgumentException("The root vector did not create the right number of children. found " +
         fieldVectors.size() + " expected " + schema.getFields().size());
     }
-    return new VectorSchemaRoot(schema.getFields(), fieldVectors, 0);
+    return new VectorSchemaRoot(schema, fieldVectors, 0);
   }
 
   public List<FieldVector> getFieldVectors() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index f00088577ffa9..b331ea776cafc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -150,18 +150,19 @@ protected void ensureInitialized() throws IOException {
    * Reads the schema and initializes the vectors
    */
   private void initialize() throws IOException {
-    Schema schema = readSchema(in);
+    Schema originalSchema = readSchema(in);
     List<Field> fields = new ArrayList<>();
     List<FieldVector> vectors = new ArrayList<>();
     Map<Long, Dictionary> dictionaries = new HashMap<>();
 
-    for (Field field: schema.getFields()) {
+    for (Field field: originalSchema.getFields()) {
       Field updated = toMemoryFormat(field, dictionaries);
       fields.add(updated);
       vectors.add(updated.createVector(allocator));
     }
+    Schema schema = new Schema(fields, originalSchema.getCustomMetadata());
 
-    this.root = new VectorSchemaRoot(fields, vectors, 0);
+    this.root = new VectorSchemaRoot(schema, vectors, 0);
     this.loader = new VectorLoader(root);
     this.dictionaries = Collections.unmodifiableMap(dictionaries);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 35b44fb682fb7..4abaed0ee7608 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -78,7 +78,7 @@ protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, Writab
       fields.add(toMessageFormat(field, provider, dictionaryBatches));
     }
 
-    this.schema = new Schema(fields);
+    this.schema = new Schema(fields, root.getSchema().getCustomMetadata());
     this.dictionaries = Collections.unmodifiableList(new ArrayList<>(dictionaryBatches.values()));
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
index cede3e801c1e4..82e2ef55c20c6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -25,10 +25,15 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -36,8 +41,11 @@
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.flatbuffers.FlatBufferBuilder;
 
+import org.apache.arrow.flatbuf.KeyValue;
+
 /**
  * An Arrow Schema
  */
@@ -76,24 +84,43 @@ public static Schema convertSchema(org.apache.arrow.flatbuf.Schema schema) {
       childrenBuilder.add(convertField(schema.fields(i)));
     }
     List<Field> fields = childrenBuilder.build();
-    return new Schema(fields);
+    ImmutableMap.Builder<String, String> metadataBuilder = ImmutableMap.builder();
+    for (int i = 0; i < schema.customMetadataLength(); i++) {
+      KeyValue kv = schema.customMetadata(i);
+      String key = kv.key(), value = kv.value();
+      metadataBuilder.put(key == null ? "" : key, value == null ? "" : value);
+    }
+    Map<String, String> metadata = metadataBuilder.build();
+    return new Schema(fields, metadata);
   }
 
   private final List<Field> fields;
+  private final Map<String, String> metadata;
+
+  public Schema(Iterable<Field> fields) {
+    this(fields, null);
+  }
 
   @JsonCreator
-  public Schema(@JsonProperty("fields") Iterable<Field> fields) {
+  public Schema(@JsonProperty("fields") Iterable<Field> fields,
+                @JsonProperty("metadata") Map<String, String> metadata) {
     List<Field> fieldList = new ArrayList<>();
     for (Field field : fields) {
       fieldList.add(field);
     }
     this.fields = Collections.unmodifiableList(fieldList);
+    this.metadata = metadata == null ? ImmutableMap.<String, String>of() : ImmutableMap.copyOf(metadata);
   }
 
   public List<Field> getFields() {
     return fields;
   }
 
+  @JsonInclude(Include.NON_EMPTY)
+  public Map<String, String> getCustomMetadata() {
+    return metadata;
+  }
+
   /**
    * @param name the name of the field to return
    * @return the corresponding field
@@ -117,15 +144,28 @@ public int getSchema(FlatBufferBuilder builder) {
       fieldOffsets[i] = fields.get(i).getField(builder);
     }
     int fieldsOffset = org.apache.arrow.flatbuf.Schema.createFieldsVector(builder, fieldOffsets);
+    int[] metadataOffsets = new int[metadata.size()];
+    Iterator<Entry<String, String>> metadataIterator = metadata.entrySet().iterator();
+    for (int i = 0; i < metadataOffsets.length; i ++) {
+      Entry<String, String> kv = metadataIterator.next();
+      int keyOffset = builder.createString(kv.getKey());
+      int valueOffset = builder.createString(kv.getValue());
+      KeyValue.startKeyValue(builder);
+      KeyValue.addKey(builder, keyOffset);
+      KeyValue.addValue(builder, valueOffset);
+      metadataOffsets[i] = KeyValue.endKeyValue(builder);
+    }
+    int metadataOffset = org.apache.arrow.flatbuf.Field.createCustomMetadataVector(builder, metadataOffsets);
     org.apache.arrow.flatbuf.Schema.startSchema(builder);
     org.apache.arrow.flatbuf.Schema.addFields(builder, fieldsOffset);
+    org.apache.arrow.flatbuf.Schema.addCustomMetadata(builder, metadataOffset);
     return org.apache.arrow.flatbuf.Schema.endSchema(builder);
   }
 
 
   @Override
   public int hashCode() {
-    return Objects.hashCode(fields);
+    return Objects.hash(fields, metadata);
   }
 
   @Override
@@ -133,11 +173,13 @@ public boolean equals(Object obj) {
     if (!(obj instanceof Schema)) {
       return false;
     }
-    return Objects.equals(this.fields, ((Schema) obj).fields);
+    return Objects.equals(this.fields, ((Schema) obj).fields) &&
+           Objects.equals(this.metadata, ((Schema) obj).metadata);
   }
 
   @Override
   public String toString() {
-    return "Schema<" + Joiner.on(", ").join(fields) + ">";
+    String meta = metadata.isEmpty() ? "" : "(metadata: " + metadata.toString() + ")";
+    return "Schema<" + Joiner.on(", ").join(fields) + ">" + meta;
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 9156110a863d8..3353112d2b793 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -29,6 +29,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -386,7 +387,7 @@ private void validateTinyData(VectorSchemaRoot root) {
   }
 
   @Test
-  public void testWriteReadFieldMetadata() throws IOException {
+  public void testWriteReadMetadata() throws IOException {
     File file = new File("target/mytest_metadata.arrow");
     ByteArrayOutputStream stream = new ByteArrayOutputStream();
 
@@ -397,7 +398,11 @@ public void testWriteReadFieldMetadata() throws IOException {
     childFields.add(new Field("list-child", new FieldType(true, ArrowType.List.INSTANCE, null, metadata(4)),
                               ImmutableList.of(new Field("l1", FieldType.nullable(new ArrowType.Int(16 ,true)), null))));
     Field field = new Field("meta", new FieldType(true, ArrowType.Struct.INSTANCE, null, metadata(0)), childFields);
-    List<Field> fields = ImmutableList.of(field);
+    Map<String, String> metadata = new HashMap<>();
+    metadata.put("s1", "v1");
+    metadata.put("s2", "v2");
+    Schema originalSchema = new Schema(ImmutableList.of(field), metadata);
+    Assert.assertEquals(metadata, originalSchema.getCustomMetadata());
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
@@ -406,7 +411,7 @@ public void testWriteReadFieldMetadata() throws IOException {
       vector.getMutator().setValueCount(0);
 
       List<FieldVector> vectors = ImmutableList.<FieldVector>of(vector);
-      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 0);
+      VectorSchemaRoot root = new VectorSchemaRoot(originalSchema, vectors, 0);
 
       try (FileOutputStream fileOutputStream = new FileOutputStream(file);
            ArrowFileWriter fileWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel());
@@ -428,7 +433,8 @@ public void testWriteReadFieldMetadata() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      Assert.assertEquals(fields, schema.getFields());
+      Assert.assertEquals(originalSchema, schema);
+      Assert.assertEquals(originalSchema.getCustomMetadata(), schema.getCustomMetadata());
       Field top = schema.getFields().get(0);
       Assert.assertEquals(metadata(0), top.getMetadata());
       for (int i = 0; i < 4; i ++) {
@@ -443,7 +449,8 @@ public void testWriteReadFieldMetadata() throws IOException {
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
-      Assert.assertEquals(fields, schema.getFields());
+      Assert.assertEquals(originalSchema, schema);
+      Assert.assertEquals(originalSchema.getCustomMetadata(), schema.getCustomMetadata());
       Field top = schema.getFields().get(0);
       Assert.assertEquals(metadata(0), top.getMetadata());
       for (int i = 0; i < 4; i ++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index e2dae2933217e..62c21f7da0db6 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -21,6 +21,12 @@
 import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE;
 import static org.junit.Assert.assertEquals;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.UnionMode;
@@ -36,9 +42,6 @@
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Test;
 
-import com.google.common.collect.ImmutableList;
-import com.google.flatbuffers.FlatBufferBuilder;
-
 /**
  * Test conversion between Flatbuf and Pojo field representations
  */
@@ -69,6 +72,18 @@ public void schema() {
     run(initialSchema);
   }
 
+  @Test
+  public void schemaMetadata() {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+    childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
+    Map<String, String> metadata = new HashMap<>();
+    metadata.put("key1", "value1");
+    metadata.put("key2", "value2");
+    Schema initialSchema = new Schema(childrenBuilder.build(), metadata);
+    run(initialSchema);
+  }
+
   @Test
   public void nestedSchema() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();

From abbd815e9a6c8ef5a10565d81f7b91f889c6bc20 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 15 May 2017 21:33:14 -0400
Subject: [PATCH 0656/1644] ARROW-1025: [Website] Improved changelog for
 website, include git shortlog

This patch combines new features and improvements in the changelog into a single category and lists these first on the website (since these may be more interesting for readers). I also added the `git shortlog` to show who contributed to each release

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #691 from wesm/ARROW-1025 and squashes the following commits:

416c876 [Wes McKinney] Consolidated changelog for the website with new features/improvements shown first. Include git shortlog
---
 dev/make_changelog.py  |  51 ++++--
 site/_release/0.1.0.md | 276 +++++++++++++++--------------
 site/_release/0.2.0.md | 261 ++++++++++++++-------------
 site/_release/0.3.0.md | 392 ++++++++++++++++++++++-------------------
 4 files changed, 533 insertions(+), 447 deletions(-)

diff --git a/dev/make_changelog.py b/dev/make_changelog.py
index 0ad1607c79326..47127903b7b1f 100644
--- a/dev/make_changelog.py
+++ b/dev/make_changelog.py
@@ -52,34 +52,65 @@ def get_issues_for_version(version):
 LINK_TEMPLATE = '[{0}](https://issues.apache.org/jira/browse/{0})'
 
 
-def format_changelog_markdown(issues, out, links=False):
+def format_changelog_markdown(issues, out):
     issues_by_type = defaultdict(list)
     for issue in issues:
         issues_by_type[issue.fields.issuetype.name].append(issue)
 
-
     for issue_type, issue_group in sorted(issues_by_type.items()):
         issue_group.sort(key=lambda x: x.key)
 
         out.write('## {0}\n\n'.format(issue_type))
         for issue in issue_group:
-            if links:
-                name = LINK_TEMPLATE.format(issue.key)
-            else:
-                name = issue.key
-            out.write('* {0} - {1}\n'.format(name,
+            out.write('* {0} - {1}\n'.format(issue.key,
                                              issue.fields.summary))
         out.write('\n')
 
 
+def format_changelog_website(issues, out):
+    NEW_FEATURE = 'New Features and Improvements'
+    BUGFIX = 'Bug Fixes'
+
+    CATEGORIES = {
+        'New Feature': NEW_FEATURE,
+        'Improvement': NEW_FEATURE,
+        'Task': NEW_FEATURE,
+        'Test': NEW_FEATURE,
+        'Bug': BUGFIX
+    }
+
+    issues_by_category = defaultdict(list)
+    for issue in issues:
+        issue_type = issue.fields.issuetype.name
+        website_category = CATEGORIES[issue_type]
+        issues_by_category[website_category].append(issue)
+
+    WEBSITE_ORDER = [NEW_FEATURE, BUGFIX]
+
+    for issue_category in WEBSITE_ORDER:
+        issue_group = issues_by_category[issue_category]
+        issue_group.sort(key=lambda x: x.key)
+
+        out.write('## {0}\n\n'.format(issue_category))
+        for issue in issue_group:
+            name = LINK_TEMPLATE.format(issue.key)
+            out.write('* {0} - {1}\n'.format(name, issue.fields.summary))
+        out.write('\n')
+
+
 if __name__ == '__main__':
     if len(sys.argv) < 2:
-        print('Usage: make_changelog.py $FIX_VERSION [$LINKS]')
+        print('Usage: make_changelog.py $FIX_VERSION [$IS_WEBSITE]')
 
     buf = StringIO()
 
-    links = len(sys.argv) > 2 and sys.argv[2] == '1'
+    for_website = len(sys.argv) > 2 and sys.argv[2] == '1'
 
     issues_for_version = get_issues_for_version(sys.argv[1])
-    format_changelog_markdown(issues_for_version, buf, links=links)
+
+    if for_website:
+        format_changelog_website(issues_for_version, buf)
+    else:
+        format_changelog_markdown(issues_for_version, buf)
+
     print(buf.getvalue())
diff --git a/site/_release/0.1.0.md b/site/_release/0.1.0.md
index e668e5698652d..cf30fde9e62c6 100644
--- a/site/_release/0.1.0.md
+++ b/site/_release/0.1.0.md
@@ -31,153 +31,114 @@ limitations under the License.
 
 # Changelog
 
-## Bug
-
-* [ARROW-103](https://issues.apache.org/jira/browse/ARROW-103) - Missing patterns from .gitignore
-* [ARROW-104](https://issues.apache.org/jira/browse/ARROW-104) - Update Layout.md based on discussion on the mailing list
-* [ARROW-105](https://issues.apache.org/jira/browse/ARROW-105) - Unit tests fail if assertions are disabled
-* [ARROW-113](https://issues.apache.org/jira/browse/ARROW-113) - TestValueVector test fails if cannot allocate 2GB of memory
-* [ARROW-16](https://issues.apache.org/jira/browse/ARROW-16) - Building cpp issues on XCode 7.2.1
-* [ARROW-17](https://issues.apache.org/jira/browse/ARROW-17) - Set some vector fields to default access level for Drill compatibility
-* [ARROW-18](https://issues.apache.org/jira/browse/ARROW-18) - Fix bug with decimal precision and scale
-* [ARROW-185](https://issues.apache.org/jira/browse/ARROW-185) - [C++] Make sure alignment and memory padding conform to spec
-* [ARROW-188](https://issues.apache.org/jira/browse/ARROW-188) - Python: Add numpy as install requirement
-* [ARROW-193](https://issues.apache.org/jira/browse/ARROW-193) - For the instruction, typos "int his" should be "in this"
-* [ARROW-194](https://issues.apache.org/jira/browse/ARROW-194) - C++: Allow read-only memory mapped source
-* [ARROW-200](https://issues.apache.org/jira/browse/ARROW-200) - [Python] Convert Values String looks like it has incorrect error handling
-* [ARROW-209](https://issues.apache.org/jira/browse/ARROW-209) - [C++] Broken builds: llvm.org apt repos are unavailable
-* [ARROW-210](https://issues.apache.org/jira/browse/ARROW-210) - [C++] Tidy up the type system a little bit
-* [ARROW-211](https://issues.apache.org/jira/browse/ARROW-211) - Several typos/errors in Layout.md examples
-* [ARROW-217](https://issues.apache.org/jira/browse/ARROW-217) - Fix Travis w.r.t conda 4.1.0 changes
-* [ARROW-219](https://issues.apache.org/jira/browse/ARROW-219) - [C++] Passed CMAKE_CXX_FLAGS are being dropped, fix compiler warnings
-* [ARROW-223](https://issues.apache.org/jira/browse/ARROW-223) - Do not link against libpython
-* [ARROW-225](https://issues.apache.org/jira/browse/ARROW-225) - [C++/Python] master Travis CI build is broken
-* [ARROW-244](https://issues.apache.org/jira/browse/ARROW-244) - [C++] Some global APIs of IPC module should be visible to the outside
-* [ARROW-246](https://issues.apache.org/jira/browse/ARROW-246) - [Java] UnionVector doesn't call allocateNew() when creating it's vectorType
-* [ARROW-247](https://issues.apache.org/jira/browse/ARROW-247) - [C++] Missing explicit destructor in RowBatchReader causes an incomplete type error
-* [ARROW-250](https://issues.apache.org/jira/browse/ARROW-250) - Fix for ARROW-246 may cause memory leaks
-* [ARROW-259](https://issues.apache.org/jira/browse/ARROW-259) - Use flatbuffer fields in java implementation
-* [ARROW-265](https://issues.apache.org/jira/browse/ARROW-265) - Negative decimal values have wrong padding
-* [ARROW-266](https://issues.apache.org/jira/browse/ARROW-266) - [C++] Fix the broken build
-* [ARROW-274](https://issues.apache.org/jira/browse/ARROW-274) - Make the MapVector nullable
-* [ARROW-278](https://issues.apache.org/jira/browse/ARROW-278) - [Format] Struct type name consistency in implementations and metadata
-* [ARROW-283](https://issues.apache.org/jira/browse/ARROW-283) - [C++] Update arrow_parquet to account for API changes in PARQUET-573
-* [ARROW-284](https://issues.apache.org/jira/browse/ARROW-284) - [C++] Triage builds by disabling Arrow-Parquet module
-* [ARROW-287](https://issues.apache.org/jira/browse/ARROW-287) - [java] Make nullable vectors use a BitVecor instead of UInt1Vector for bits
-* [ARROW-297](https://issues.apache.org/jira/browse/ARROW-297) - Fix Arrow pom for release
-* [ARROW-304](https://issues.apache.org/jira/browse/ARROW-304) - NullableMapReaderImpl.isSet() always returns true
-* [ARROW-308](https://issues.apache.org/jira/browse/ARROW-308) - UnionListWriter.setPosition() should not call startList()
-* [ARROW-309](https://issues.apache.org/jira/browse/ARROW-309) - Types.getMinorTypeForArrowType() does not work for Union type
-* [ARROW-313](https://issues.apache.org/jira/browse/ARROW-313) - XCode 8.0 breaks builds
-* [ARROW-314](https://issues.apache.org/jira/browse/ARROW-314) - JSONScalar is unnecessary and unused.
-* [ARROW-320](https://issues.apache.org/jira/browse/ARROW-320) - ComplexCopier.copy(FieldReader, FieldWriter) should not start a list if reader is not set
-* [ARROW-321](https://issues.apache.org/jira/browse/ARROW-321) - Fix Arrow licences
-* [ARROW-36](https://issues.apache.org/jira/browse/ARROW-36) - Remove fixVersions from patch tool (until we have them)
-* [ARROW-46](https://issues.apache.org/jira/browse/ARROW-46) - Port DRILL-4410 to Arrow
-* [ARROW-5](https://issues.apache.org/jira/browse/ARROW-5) - Error when run maven install
-* [ARROW-51](https://issues.apache.org/jira/browse/ARROW-51) - Move ValueVector test from Drill project
-* [ARROW-55](https://issues.apache.org/jira/browse/ARROW-55) - Python: fix legacy Python (2.7) tests and add to Travis CI
-* [ARROW-62](https://issues.apache.org/jira/browse/ARROW-62) - Format: Are the nulls bits 0 or 1 for null values?
-* [ARROW-63](https://issues.apache.org/jira/browse/ARROW-63) - C++: ctest fails if Python 3 is the active Python interpreter
-* [ARROW-65](https://issues.apache.org/jira/browse/ARROW-65) - Python: FindPythonLibsNew does not work in a virtualenv
-* [ARROW-69](https://issues.apache.org/jira/browse/ARROW-69) - Change permissions for assignable users
-* [ARROW-72](https://issues.apache.org/jira/browse/ARROW-72) - FindParquet searches for non-existent header
-* [ARROW-75](https://issues.apache.org/jira/browse/ARROW-75) - C++: Fix handling of empty strings
-* [ARROW-77](https://issues.apache.org/jira/browse/ARROW-77) - C++: conform null bit interpretation to match ARROW-62
-* [ARROW-80](https://issues.apache.org/jira/browse/ARROW-80) - Segmentation fault on len(Array) for empty arrays
-* [ARROW-88](https://issues.apache.org/jira/browse/ARROW-88) - C++: Refactor given PARQUET-572
-* [ARROW-93](https://issues.apache.org/jira/browse/ARROW-93) - XCode 7.3 breaks builds
-* [ARROW-94](https://issues.apache.org/jira/browse/ARROW-94) - Expand list example to clarify null vs empty list
-
-## Improvement
+## Contributors
+
+```shell
+$ git shortlog -sn d5aa7c46..apache-arrow-0.1.0
+    49  Wes McKinney
+    27  Uwe L. Korn
+    25  Julien Le Dem
+    13  Micah Kornfield
+    11  Steven Phillips
+     6  Jihoon Son
+     5  Laurent Goujon
+     5  adeneche
+     4  Dan Robinson
+     4  proflin
+     2  Jacques Nadeau
+     1  Christopher C. Aycock
+     1  Edmon Begoli
+     1  Kai Zheng
+     1  MechCoder
+     1  Minji Kim
+     1  Philipp Moritz
+     1  Smyatkin Maxim
+     1  fengguangyuan
+     1  hyukjinkwon
+     1  hzhang2
+     1  lfzCarlosC
+```
+
+## New Features and Improvements
 
+* [ARROW-1](https://issues.apache.org/jira/browse/ARROW-1) - Import Initial Codebase
 * [ARROW-10](https://issues.apache.org/jira/browse/ARROW-10) - Fix mismatch of javadoc names and method parameters
-* [ARROW-15](https://issues.apache.org/jira/browse/ARROW-15) - Fix a naming typo for memory.AllocationManager.AllocationOutcome
-* [ARROW-190](https://issues.apache.org/jira/browse/ARROW-190) - Python: Provide installable sdist builds
-* [ARROW-199](https://issues.apache.org/jira/browse/ARROW-199) - [C++] Refine third party dependency
-* [ARROW-206](https://issues.apache.org/jira/browse/ARROW-206) - [C++] Expose an equality API for arrays that compares a range of slots on two arrays
-* [ARROW-212](https://issues.apache.org/jira/browse/ARROW-212) - [C++] Clarify the fact that PrimitiveArray is now abstract class
-* [ARROW-213](https://issues.apache.org/jira/browse/ARROW-213) - Exposing static arrow build
-* [ARROW-218](https://issues.apache.org/jira/browse/ARROW-218) - Add option to use GitHub API token via environment variable when merging PRs
-* [ARROW-234](https://issues.apache.org/jira/browse/ARROW-234) - [C++] Build with libhdfs support in arrow_io in conda builds
-* [ARROW-238](https://issues.apache.org/jira/browse/ARROW-238) - C++: InternalMemoryPool::Free() should throw an error when there is insufficient allocated memory
-* [ARROW-245](https://issues.apache.org/jira/browse/ARROW-245) - [Format] Clarify Arrow's relationship with big endian platforms
-* [ARROW-252](https://issues.apache.org/jira/browse/ARROW-252) - Add implementation guidelines to the documentation
-* [ARROW-253](https://issues.apache.org/jira/browse/ARROW-253) - Int types should only have width of 8*2^n (8, 16, 32, 64)
-* [ARROW-254](https://issues.apache.org/jira/browse/ARROW-254) - Remove Bit type as it is redundant with boolean
-* [ARROW-255](https://issues.apache.org/jira/browse/ARROW-255) - Finalize Dictionary representation
-* [ARROW-256](https://issues.apache.org/jira/browse/ARROW-256) - Add versioning to the arrow spec.
-* [ARROW-257](https://issues.apache.org/jira/browse/ARROW-257) - Add a typeids Vector to Union type
-* [ARROW-264](https://issues.apache.org/jira/browse/ARROW-264) - Create an Arrow File format
-* [ARROW-270](https://issues.apache.org/jira/browse/ARROW-270) - [Format] Define more generic Interval logical type
-* [ARROW-271](https://issues.apache.org/jira/browse/ARROW-271) - Update Field structure to be more explicit
-* [ARROW-279](https://issues.apache.org/jira/browse/ARROW-279) - rename vector module to arrow-vector for consistency
-* [ARROW-280](https://issues.apache.org/jira/browse/ARROW-280) - [C++] Consolidate file and shared memory IO interfaces
-* [ARROW-285](https://issues.apache.org/jira/browse/ARROW-285) - Allow for custom flatc compiler
-* [ARROW-286](https://issues.apache.org/jira/browse/ARROW-286) - Build thirdparty dependencies in parallel
-* [ARROW-289](https://issues.apache.org/jira/browse/ARROW-289) - Install test-util.h
-* [ARROW-290](https://issues.apache.org/jira/browse/ARROW-290) - Specialize alloc() in ArrowBuf
-* [ARROW-292](https://issues.apache.org/jira/browse/ARROW-292) - [Java] Upgrade Netty to 4.041
-* [ARROW-299](https://issues.apache.org/jira/browse/ARROW-299) - Use absolute namespace in macros
-* [ARROW-305](https://issues.apache.org/jira/browse/ARROW-305) - Add compression and use_dictionary options to Parquet interface
-* [ARROW-306](https://issues.apache.org/jira/browse/ARROW-306) - Add option to pass cmake arguments via environment variable
-* [ARROW-315](https://issues.apache.org/jira/browse/ARROW-315) - Finalize timestamp type
-* [ARROW-319](https://issues.apache.org/jira/browse/ARROW-319) - Add canonical Arrow Schema json representation
-* [ARROW-324](https://issues.apache.org/jira/browse/ARROW-324) - Update arrow metadata diagram
-* [ARROW-325](https://issues.apache.org/jira/browse/ARROW-325) - make TestArrowFile not dependent on timezone
-* [ARROW-50](https://issues.apache.org/jira/browse/ARROW-50) - C++: Enable library builds for 3rd-party users without having to build thirdparty googletest
-* [ARROW-54](https://issues.apache.org/jira/browse/ARROW-54) - Python: rename package to "pyarrow"
-* [ARROW-64](https://issues.apache.org/jira/browse/ARROW-64) - Add zsh support to C++ build scripts
-* [ARROW-66](https://issues.apache.org/jira/browse/ARROW-66) - Maybe some missing steps in installation guide
-* [ARROW-68](https://issues.apache.org/jira/browse/ARROW-68) - Update setup_build_env and third-party script to be more userfriendly
-* [ARROW-71](https://issues.apache.org/jira/browse/ARROW-71) - C++: Add script to run clang-tidy on codebase
-* [ARROW-73](https://issues.apache.org/jira/browse/ARROW-73) - Support CMake 2.8
-* [ARROW-78](https://issues.apache.org/jira/browse/ARROW-78) - C++: Add constructor for DecimalType
-* [ARROW-79](https://issues.apache.org/jira/browse/ARROW-79) - Python: Add benchmarks
-* [ARROW-8](https://issues.apache.org/jira/browse/ARROW-8) - Set up Travis CI
-* [ARROW-85](https://issues.apache.org/jira/browse/ARROW-85) - C++: memcmp can be avoided in Equal when comparing with the same Buffer
-* [ARROW-86](https://issues.apache.org/jira/browse/ARROW-86) - Python: Implement zero-copy Arrow-to-Pandas conversion
-* [ARROW-87](https://issues.apache.org/jira/browse/ARROW-87) - Implement Decimal schema conversion for all ways supported in Parquet
-* [ARROW-89](https://issues.apache.org/jira/browse/ARROW-89) - Python: Add benchmarks for Arrow<->Pandas conversion
-* [ARROW-9](https://issues.apache.org/jira/browse/ARROW-9) - Rename some unchanged "Drill" to "Arrow"
-* [ARROW-91](https://issues.apache.org/jira/browse/ARROW-91) - C++: First draft of an adapter class for parquet-cpp's ParquetFileReader that produces Arrow table/row batch objects
-
-## New Feature
-
 * [ARROW-100](https://issues.apache.org/jira/browse/ARROW-100) - [C++] Computing RowBatch size
+* [ARROW-101](https://issues.apache.org/jira/browse/ARROW-101) - Fix java warnings emitted by java compiler
+* [ARROW-102](https://issues.apache.org/jira/browse/ARROW-102) - travis-ci support for java project
 * [ARROW-106](https://issues.apache.org/jira/browse/ARROW-106) - Add IPC round trip for string types (string, char, varchar, binary)
 * [ARROW-107](https://issues.apache.org/jira/browse/ARROW-107) - [C++] add ipc round trip for struct types
+* [ARROW-11](https://issues.apache.org/jira/browse/ARROW-11) - Mirror JIRA activity to dev@arrow.apache.org
 * [ARROW-13](https://issues.apache.org/jira/browse/ARROW-13) - Add PR merge tool similar to that used in Parquet
+* [ARROW-14](https://issues.apache.org/jira/browse/ARROW-14) - Add JIRA components
+* [ARROW-15](https://issues.apache.org/jira/browse/ARROW-15) - Fix a naming typo for memory.AllocationManager.AllocationOutcome
 * [ARROW-19](https://issues.apache.org/jira/browse/ARROW-19) - C++: Externalize memory allocations and add a MemoryPool abstract interface to builder classes
+* [ARROW-190](https://issues.apache.org/jira/browse/ARROW-190) - Python: Provide installable sdist builds
 * [ARROW-197](https://issues.apache.org/jira/browse/ARROW-197) - [Python] Add conda dev recipe for pyarrow
+* [ARROW-199](https://issues.apache.org/jira/browse/ARROW-199) - [C++] Refine third party dependency
 * [ARROW-2](https://issues.apache.org/jira/browse/ARROW-2) - Post Simple Website
 * [ARROW-20](https://issues.apache.org/jira/browse/ARROW-20) - C++: Add null count member to Array containers, remove nullable member
 * [ARROW-201](https://issues.apache.org/jira/browse/ARROW-201) - C++: Initial ParquetWriter implementation
 * [ARROW-203](https://issues.apache.org/jira/browse/ARROW-203) - Python: Basic filename based Parquet read/write
 * [ARROW-204](https://issues.apache.org/jira/browse/ARROW-204) - [Python] Automate uploading conda build artifacts for libarrow and pyarrow
+* [ARROW-206](https://issues.apache.org/jira/browse/ARROW-206) - [C++] Expose an equality API for arrays that compares a range of slots on two arrays
 * [ARROW-21](https://issues.apache.org/jira/browse/ARROW-21) - C++: Add in-memory schema metadata container
+* [ARROW-212](https://issues.apache.org/jira/browse/ARROW-212) - [C++] Clarify the fact that PrimitiveArray is now abstract class
+* [ARROW-213](https://issues.apache.org/jira/browse/ARROW-213) - Exposing static arrow build
 * [ARROW-214](https://issues.apache.org/jira/browse/ARROW-214) - C++: Add String support to Parquet I/O
 * [ARROW-215](https://issues.apache.org/jira/browse/ARROW-215) - C++: Support other integer types in Parquet I/O
+* [ARROW-218](https://issues.apache.org/jira/browse/ARROW-218) - Add option to use GitHub API token via environment variable when merging PRs
 * [ARROW-22](https://issues.apache.org/jira/browse/ARROW-22) - C++: Add schema adapter routines for converting flat Parquet schemas to in-memory Arrow schemas
 * [ARROW-222](https://issues.apache.org/jira/browse/ARROW-222) - [C++] Create prototype file-like interface to HDFS (via libhdfs) and begin defining more general IO interface for Arrow data adapters
 * [ARROW-23](https://issues.apache.org/jira/browse/ARROW-23) - C++: Add logical "Column" container for chunked data
 * [ARROW-233](https://issues.apache.org/jira/browse/ARROW-233) - [C++] Add visibility defines for limiting shared library symbol visibility
+* [ARROW-234](https://issues.apache.org/jira/browse/ARROW-234) - [C++] Build with libhdfs support in arrow_io in conda builds
 * [ARROW-236](https://issues.apache.org/jira/browse/ARROW-236) - [Python] Enable Parquet read/write to work with HDFS file objects
 * [ARROW-237](https://issues.apache.org/jira/browse/ARROW-237) - [C++] Create Arrow specializations of Parquet allocator and read interfaces
+* [ARROW-238](https://issues.apache.org/jira/browse/ARROW-238) - C++: InternalMemoryPool::Free() should throw an error when there is insufficient allocated memory
 * [ARROW-24](https://issues.apache.org/jira/browse/ARROW-24) - C++: Add logical "Table" container
 * [ARROW-242](https://issues.apache.org/jira/browse/ARROW-242) - C++/Python: Support Timestamp Data Type
+* [ARROW-245](https://issues.apache.org/jira/browse/ARROW-245) - [Format] Clarify Arrow's relationship with big endian platforms
+* [ARROW-251](https://issues.apache.org/jira/browse/ARROW-251) - [C++] Expose APIs for getting code and message of the status
+* [ARROW-252](https://issues.apache.org/jira/browse/ARROW-252) - Add implementation guidelines to the documentation
+* [ARROW-253](https://issues.apache.org/jira/browse/ARROW-253) - Int types should only have width of 8*2^n (8, 16, 32, 64)
+* [ARROW-254](https://issues.apache.org/jira/browse/ARROW-254) - Remove Bit type as it is redundant with boolean
+* [ARROW-255](https://issues.apache.org/jira/browse/ARROW-255) - Finalize Dictionary representation
+* [ARROW-256](https://issues.apache.org/jira/browse/ARROW-256) - Add versioning to the arrow spec.
+* [ARROW-257](https://issues.apache.org/jira/browse/ARROW-257) - Add a typeids Vector to Union type
 * [ARROW-26](https://issues.apache.org/jira/browse/ARROW-26) - C++: Add developer instructions for building parquet-cpp integration
+* [ARROW-260](https://issues.apache.org/jira/browse/ARROW-260) - TestValueVector.testFixedVectorReallocation and testVariableVectorReallocation are flaky
 * [ARROW-262](https://issues.apache.org/jira/browse/ARROW-262) - [Format] Add a new format document for metadata and logical types for messaging and IPC / on-wire/file representations
+* [ARROW-264](https://issues.apache.org/jira/browse/ARROW-264) - Create an Arrow File format
 * [ARROW-267](https://issues.apache.org/jira/browse/ARROW-267) - [C++] C++ implementation of file-like layout for RPC / IPC
+* [ARROW-270](https://issues.apache.org/jira/browse/ARROW-270) - [Format] Define more generic Interval logical type
+* [ARROW-271](https://issues.apache.org/jira/browse/ARROW-271) - Update Field structure to be more explicit
+* [ARROW-272](https://issues.apache.org/jira/browse/ARROW-272) - Arrow release 0.1
+* [ARROW-279](https://issues.apache.org/jira/browse/ARROW-279) - rename vector module to arrow-vector for consistency
 * [ARROW-28](https://issues.apache.org/jira/browse/ARROW-28) - C++: Add google/benchmark to the 3rd-party build toolchain
+* [ARROW-280](https://issues.apache.org/jira/browse/ARROW-280) - [C++] Consolidate file and shared memory IO interfaces
+* [ARROW-285](https://issues.apache.org/jira/browse/ARROW-285) - Allow for custom flatc compiler
+* [ARROW-286](https://issues.apache.org/jira/browse/ARROW-286) - Build thirdparty dependencies in parallel
+* [ARROW-289](https://issues.apache.org/jira/browse/ARROW-289) - Install test-util.h
+* [ARROW-290](https://issues.apache.org/jira/browse/ARROW-290) - Specialize alloc() in ArrowBuf
+* [ARROW-292](https://issues.apache.org/jira/browse/ARROW-292) - [Java] Upgrade Netty to 4.041
 * [ARROW-293](https://issues.apache.org/jira/browse/ARROW-293) - [C++] Implementations of IO interfaces for operating system files
 * [ARROW-296](https://issues.apache.org/jira/browse/ARROW-296) - [C++] Remove arrow_parquet C++ module and related parts of build system
+* [ARROW-298](https://issues.apache.org/jira/browse/ARROW-298) - create release scripts
+* [ARROW-299](https://issues.apache.org/jira/browse/ARROW-299) - Use absolute namespace in macros
 * [ARROW-3](https://issues.apache.org/jira/browse/ARROW-3) - Post Initial Arrow Format Spec
 * [ARROW-30](https://issues.apache.org/jira/browse/ARROW-30) - Python: pandas/NumPy to/from Arrow conversion routines
 * [ARROW-301](https://issues.apache.org/jira/browse/ARROW-301) - [Format] Add some form of user field metadata to IPC schemas
 * [ARROW-302](https://issues.apache.org/jira/browse/ARROW-302) - [Python] Add support to use the Arrow file format with file-like objects
+* [ARROW-305](https://issues.apache.org/jira/browse/ARROW-305) - Add compression and use_dictionary options to Parquet interface
+* [ARROW-306](https://issues.apache.org/jira/browse/ARROW-306) - Add option to pass cmake arguments via environment variable
 * [ARROW-31](https://issues.apache.org/jira/browse/ARROW-31) - Python: basic PyList <-> Arrow marshaling code
+* [ARROW-315](https://issues.apache.org/jira/browse/ARROW-315) - Finalize timestamp type
 * [ARROW-318](https://issues.apache.org/jira/browse/ARROW-318) - [Python] Revise README to reflect current state of project
+* [ARROW-319](https://issues.apache.org/jira/browse/ARROW-319) - Add canonical Arrow Schema json representation
+* [ARROW-324](https://issues.apache.org/jira/browse/ARROW-324) - Update arrow metadata diagram
+* [ARROW-325](https://issues.apache.org/jira/browse/ARROW-325) - make TestArrowFile not dependent on timezone
+* [ARROW-35](https://issues.apache.org/jira/browse/ARROW-35) - Add a short call-to-action / how-to-get-involved to the main README.md
 * [ARROW-37](https://issues.apache.org/jira/browse/ARROW-37) - C++: Represent boolean array data in bit-packed form
 * [ARROW-4](https://issues.apache.org/jira/browse/ARROW-4) - Initial Arrow CPP Implementation
 * [ARROW-42](https://issues.apache.org/jira/browse/ARROW-42) - Python: Add to Travis CI build
@@ -185,38 +146,91 @@ limitations under the License.
 * [ARROW-44](https://issues.apache.org/jira/browse/ARROW-44) - Python: Implement basic object model for scalar values (i.e. results of arrow_arr[i])
 * [ARROW-48](https://issues.apache.org/jira/browse/ARROW-48) - Python: Add Schema object wrapper
 * [ARROW-49](https://issues.apache.org/jira/browse/ARROW-49) - Python: Add Column and Table wrapper interface
+* [ARROW-50](https://issues.apache.org/jira/browse/ARROW-50) - C++: Enable library builds for 3rd-party users without having to build thirdparty googletest
 * [ARROW-53](https://issues.apache.org/jira/browse/ARROW-53) - Python: Fix RPATH and add source installation instructions
+* [ARROW-54](https://issues.apache.org/jira/browse/ARROW-54) - Python: rename package to "pyarrow"
 * [ARROW-56](https://issues.apache.org/jira/browse/ARROW-56) - Format: Specify LSB bit ordering in bit arrays
 * [ARROW-57](https://issues.apache.org/jira/browse/ARROW-57) - Format: Draft data headers IDL for data interchange
 * [ARROW-58](https://issues.apache.org/jira/browse/ARROW-58) - Format: Draft type metadata ("schemas") IDL
 * [ARROW-59](https://issues.apache.org/jira/browse/ARROW-59) - Python: Boolean data support for builtin data structures
 * [ARROW-60](https://issues.apache.org/jira/browse/ARROW-60) - C++: Struct type builder API
+* [ARROW-64](https://issues.apache.org/jira/browse/ARROW-64) - Add zsh support to C++ build scripts
+* [ARROW-66](https://issues.apache.org/jira/browse/ARROW-66) - Maybe some missing steps in installation guide
 * [ARROW-67](https://issues.apache.org/jira/browse/ARROW-67) - C++: Draft type metadata conversion to/from IPC representation
+* [ARROW-68](https://issues.apache.org/jira/browse/ARROW-68) - Update setup_build_env and third-party script to be more userfriendly
 * [ARROW-7](https://issues.apache.org/jira/browse/ARROW-7) - Add Python library build toolchain
 * [ARROW-70](https://issues.apache.org/jira/browse/ARROW-70) - C++: Add "lite" DCHECK macros used in parquet-cpp
+* [ARROW-71](https://issues.apache.org/jira/browse/ARROW-71) - C++: Add script to run clang-tidy on codebase
+* [ARROW-73](https://issues.apache.org/jira/browse/ARROW-73) - Support CMake 2.8
 * [ARROW-76](https://issues.apache.org/jira/browse/ARROW-76) - Revise format document to include null count, defer non-nullable arrays to the domain of metadata
+* [ARROW-78](https://issues.apache.org/jira/browse/ARROW-78) - C++: Add constructor for DecimalType
+* [ARROW-79](https://issues.apache.org/jira/browse/ARROW-79) - Python: Add benchmarks
+* [ARROW-8](https://issues.apache.org/jira/browse/ARROW-8) - Set up Travis CI
 * [ARROW-82](https://issues.apache.org/jira/browse/ARROW-82) - C++: Implement IPC exchange for List types
+* [ARROW-83](https://issues.apache.org/jira/browse/ARROW-83) - Add basic test infrastructure for DecimalType
+* [ARROW-85](https://issues.apache.org/jira/browse/ARROW-85) - C++: memcmp can be avoided in Equal when comparing with the same Buffer
+* [ARROW-86](https://issues.apache.org/jira/browse/ARROW-86) - Python: Implement zero-copy Arrow-to-Pandas conversion
+* [ARROW-87](https://issues.apache.org/jira/browse/ARROW-87) - Implement Decimal schema conversion for all ways supported in Parquet
+* [ARROW-89](https://issues.apache.org/jira/browse/ARROW-89) - Python: Add benchmarks for Arrow<->Pandas conversion
+* [ARROW-9](https://issues.apache.org/jira/browse/ARROW-9) - Rename some unchanged "Drill" to "Arrow"
 * [ARROW-90](https://issues.apache.org/jira/browse/ARROW-90) - Apache Arrow cpp code does not support power architecture
+* [ARROW-91](https://issues.apache.org/jira/browse/ARROW-91) - C++: First draft of an adapter class for parquet-cpp's ParquetFileReader that produces Arrow table/row batch objects
 * [ARROW-92](https://issues.apache.org/jira/browse/ARROW-92) - C++: Arrow to Parquet Schema conversion
 
-## Task
-
-* [ARROW-1](https://issues.apache.org/jira/browse/ARROW-1) - Import Initial Codebase
-* [ARROW-101](https://issues.apache.org/jira/browse/ARROW-101) - Fix java warnings emitted by java compiler
-* [ARROW-102](https://issues.apache.org/jira/browse/ARROW-102) - travis-ci support for java project
-* [ARROW-11](https://issues.apache.org/jira/browse/ARROW-11) - Mirror JIRA activity to dev@arrow.apache.org
-* [ARROW-14](https://issues.apache.org/jira/browse/ARROW-14) - Add JIRA components
-* [ARROW-251](https://issues.apache.org/jira/browse/ARROW-251) - [C++] Expose APIs for getting code and message of the status
-* [ARROW-272](https://issues.apache.org/jira/browse/ARROW-272) - Arrow release 0.1
-* [ARROW-298](https://issues.apache.org/jira/browse/ARROW-298) - create release scripts
-* [ARROW-35](https://issues.apache.org/jira/browse/ARROW-35) - Add a short call-to-action / how-to-get-involved to the main README.md
+## Bug Fixes
 
-## Test
-
-* [ARROW-260](https://issues.apache.org/jira/browse/ARROW-260) - TestValueVector.testFixedVectorReallocation and testVariableVectorReallocation are flaky
-* [ARROW-83](https://issues.apache.org/jira/browse/ARROW-83) - Add basic test infrastructure for DecimalType
-
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.1.0
-[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.1.0/apache-arrow-0.1.0.tar.gz.md5
-[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.1.0/apache-arrow-0.1.0.tar.gz
-[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.1.0/apache-arrow-0.1.0.tar.gz.asc
+* [ARROW-103](https://issues.apache.org/jira/browse/ARROW-103) - Missing patterns from .gitignore
+* [ARROW-104](https://issues.apache.org/jira/browse/ARROW-104) - Update Layout.md based on discussion on the mailing list
+* [ARROW-105](https://issues.apache.org/jira/browse/ARROW-105) - Unit tests fail if assertions are disabled
+* [ARROW-113](https://issues.apache.org/jira/browse/ARROW-113) - TestValueVector test fails if cannot allocate 2GB of memory
+* [ARROW-16](https://issues.apache.org/jira/browse/ARROW-16) - Building cpp issues on XCode 7.2.1
+* [ARROW-17](https://issues.apache.org/jira/browse/ARROW-17) - Set some vector fields to default access level for Drill compatibility
+* [ARROW-18](https://issues.apache.org/jira/browse/ARROW-18) - Fix bug with decimal precision and scale
+* [ARROW-185](https://issues.apache.org/jira/browse/ARROW-185) - [C++] Make sure alignment and memory padding conform to spec
+* [ARROW-188](https://issues.apache.org/jira/browse/ARROW-188) - Python: Add numpy as install requirement
+* [ARROW-193](https://issues.apache.org/jira/browse/ARROW-193) - For the instruction, typos "int his" should be "in this"
+* [ARROW-194](https://issues.apache.org/jira/browse/ARROW-194) - C++: Allow read-only memory mapped source
+* [ARROW-200](https://issues.apache.org/jira/browse/ARROW-200) - [Python] Convert Values String looks like it has incorrect error handling
+* [ARROW-209](https://issues.apache.org/jira/browse/ARROW-209) - [C++] Broken builds: llvm.org apt repos are unavailable
+* [ARROW-210](https://issues.apache.org/jira/browse/ARROW-210) - [C++] Tidy up the type system a little bit
+* [ARROW-211](https://issues.apache.org/jira/browse/ARROW-211) - Several typos/errors in Layout.md examples
+* [ARROW-217](https://issues.apache.org/jira/browse/ARROW-217) - Fix Travis w.r.t conda 4.1.0 changes
+* [ARROW-219](https://issues.apache.org/jira/browse/ARROW-219) - [C++] Passed CMAKE_CXX_FLAGS are being dropped, fix compiler warnings
+* [ARROW-223](https://issues.apache.org/jira/browse/ARROW-223) - Do not link against libpython
+* [ARROW-225](https://issues.apache.org/jira/browse/ARROW-225) - [C++/Python] master Travis CI build is broken
+* [ARROW-244](https://issues.apache.org/jira/browse/ARROW-244) - [C++] Some global APIs of IPC module should be visible to the outside
+* [ARROW-246](https://issues.apache.org/jira/browse/ARROW-246) - [Java] UnionVector doesn't call allocateNew() when creating it's vectorType
+* [ARROW-247](https://issues.apache.org/jira/browse/ARROW-247) - [C++] Missing explicit destructor in RowBatchReader causes an incomplete type error
+* [ARROW-250](https://issues.apache.org/jira/browse/ARROW-250) - Fix for ARROW-246 may cause memory leaks
+* [ARROW-259](https://issues.apache.org/jira/browse/ARROW-259) - Use flatbuffer fields in java implementation
+* [ARROW-265](https://issues.apache.org/jira/browse/ARROW-265) - Negative decimal values have wrong padding
+* [ARROW-266](https://issues.apache.org/jira/browse/ARROW-266) - [C++] Fix the broken build
+* [ARROW-274](https://issues.apache.org/jira/browse/ARROW-274) - Make the MapVector nullable
+* [ARROW-278](https://issues.apache.org/jira/browse/ARROW-278) - [Format] Struct type name consistency in implementations and metadata
+* [ARROW-283](https://issues.apache.org/jira/browse/ARROW-283) - [C++] Update arrow_parquet to account for API changes in PARQUET-573
+* [ARROW-284](https://issues.apache.org/jira/browse/ARROW-284) - [C++] Triage builds by disabling Arrow-Parquet module
+* [ARROW-287](https://issues.apache.org/jira/browse/ARROW-287) - [java] Make nullable vectors use a BitVecor instead of UInt1Vector for bits
+* [ARROW-297](https://issues.apache.org/jira/browse/ARROW-297) - Fix Arrow pom for release
+* [ARROW-304](https://issues.apache.org/jira/browse/ARROW-304) - NullableMapReaderImpl.isSet() always returns true
+* [ARROW-308](https://issues.apache.org/jira/browse/ARROW-308) - UnionListWriter.setPosition() should not call startList()
+* [ARROW-309](https://issues.apache.org/jira/browse/ARROW-309) - Types.getMinorTypeForArrowType() does not work for Union type
+* [ARROW-313](https://issues.apache.org/jira/browse/ARROW-313) - XCode 8.0 breaks builds
+* [ARROW-314](https://issues.apache.org/jira/browse/ARROW-314) - JSONScalar is unnecessary and unused.
+* [ARROW-320](https://issues.apache.org/jira/browse/ARROW-320) - ComplexCopier.copy(FieldReader, FieldWriter) should not start a list if reader is not set
+* [ARROW-321](https://issues.apache.org/jira/browse/ARROW-321) - Fix Arrow licences
+* [ARROW-36](https://issues.apache.org/jira/browse/ARROW-36) - Remove fixVersions from patch tool (until we have them)
+* [ARROW-46](https://issues.apache.org/jira/browse/ARROW-46) - Port DRILL-4410 to Arrow
+* [ARROW-5](https://issues.apache.org/jira/browse/ARROW-5) - Error when run maven install
+* [ARROW-51](https://issues.apache.org/jira/browse/ARROW-51) - Move ValueVector test from Drill project
+* [ARROW-55](https://issues.apache.org/jira/browse/ARROW-55) - Python: fix legacy Python (2.7) tests and add to Travis CI
+* [ARROW-62](https://issues.apache.org/jira/browse/ARROW-62) - Format: Are the nulls bits 0 or 1 for null values?
+* [ARROW-63](https://issues.apache.org/jira/browse/ARROW-63) - C++: ctest fails if Python 3 is the active Python interpreter
+* [ARROW-65](https://issues.apache.org/jira/browse/ARROW-65) - Python: FindPythonLibsNew does not work in a virtualenv
+* [ARROW-69](https://issues.apache.org/jira/browse/ARROW-69) - Change permissions for assignable users
+* [ARROW-72](https://issues.apache.org/jira/browse/ARROW-72) - FindParquet searches for non-existent header
+* [ARROW-75](https://issues.apache.org/jira/browse/ARROW-75) - C++: Fix handling of empty strings
+* [ARROW-77](https://issues.apache.org/jira/browse/ARROW-77) - C++: conform null bit interpretation to match ARROW-62
+* [ARROW-80](https://issues.apache.org/jira/browse/ARROW-80) - Segmentation fault on len(Array) for empty arrays
+* [ARROW-88](https://issues.apache.org/jira/browse/ARROW-88) - C++: Refactor given PARQUET-572
+* [ARROW-93](https://issues.apache.org/jira/browse/ARROW-93) - XCode 7.3 breaks builds
+* [ARROW-94](https://issues.apache.org/jira/browse/ARROW-94) - Expand list example to clarify null vs empty list
\ No newline at end of file
diff --git a/site/_release/0.2.0.md b/site/_release/0.2.0.md
index d66afd066d762..ddac64e33f5e4 100644
--- a/site/_release/0.2.0.md
+++ b/site/_release/0.2.0.md
@@ -31,7 +31,146 @@ limitations under the License.
 
 # Changelog
 
-## Bug
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.1.0..apache-arrow-0.2.0
+    73  Wes McKinney
+    55  Uwe L. Korn
+    16  Julien Le Dem
+     4  Bryan Cutler
+     4  Nong Li
+     2  Christopher C. Aycock
+     2  Jingyuan Wang
+     2  Kouhei Sutou
+     2  Laurent Goujon
+     2  Leif Walsh
+     1  Emilio Lahr-Vivaz
+     1  Holden Karau
+     1  Li Jin
+     1  Mohamed Zenadi
+     1  Peter Hoffmann
+     1  Steven Phillips
+     1  adeneche
+     1  ahnj
+     1  vkorukanti
+```
+
+## New Features and Improvements
+
+* [ARROW-108](https://issues.apache.org/jira/browse/ARROW-108) - [C++] Add IPC round trip for union types
+* [ARROW-189](https://issues.apache.org/jira/browse/ARROW-189) - C++: Use ExternalProject to build thirdparty dependencies
+* [ARROW-191](https://issues.apache.org/jira/browse/ARROW-191) - Python: Provide infrastructure for manylinux1 wheels
+* [ARROW-221](https://issues.apache.org/jira/browse/ARROW-221) - Add switch for writing Parquet 1.0 compatible logical types
+* [ARROW-227](https://issues.apache.org/jira/browse/ARROW-227) - [C++/Python] Hook arrow_io generic reader / writer interface into arrow_parquet
+* [ARROW-228](https://issues.apache.org/jira/browse/ARROW-228) - [Python] Create an Arrow-cpp-compatible interface for reading bytes from Python file-like objects
+* [ARROW-243](https://issues.apache.org/jira/browse/ARROW-243) - [C++] Add "driver" option to HdfsClient to choose between libhdfs and libhdfs3 at runtime
+* [ARROW-268](https://issues.apache.org/jira/browse/ARROW-268) - [C++] Flesh out union implementation to have all required methods for IPC
+* [ARROW-303](https://issues.apache.org/jira/browse/ARROW-303) - [C++] Also build static libraries for leaf libraries
+* [ARROW-312](https://issues.apache.org/jira/browse/ARROW-312) - [Python] Provide Python API to read/write the Arrow IPC file format
+* [ARROW-317](https://issues.apache.org/jira/browse/ARROW-317) - [C++] Implement zero-copy Slice method on arrow::Buffer that retains reference to parent
+* [ARROW-327](https://issues.apache.org/jira/browse/ARROW-327) - [Python] Remove conda builds from Travis CI processes
+* [ARROW-328](https://issues.apache.org/jira/browse/ARROW-328) - [C++] Return shared_ptr by value instead of const-ref?
+* [ARROW-33](https://issues.apache.org/jira/browse/ARROW-33) - C++: Implement zero-copy array slicing
+* [ARROW-330](https://issues.apache.org/jira/browse/ARROW-330) - [C++] CMake functions to simplify shared / static library configuration
+* [ARROW-332](https://issues.apache.org/jira/browse/ARROW-332) - [Python] Add helper function to convert RecordBatch to pandas.DataFrame
+* [ARROW-333](https://issues.apache.org/jira/browse/ARROW-333) - Make writers update their internal schema even when no data is written.
+* [ARROW-335](https://issues.apache.org/jira/browse/ARROW-335) - Improve Type apis and toString() by encapsulating flatbuffers better
+* [ARROW-336](https://issues.apache.org/jira/browse/ARROW-336) - Run Apache Rat in Travis builds
+* [ARROW-338](https://issues.apache.org/jira/browse/ARROW-338) - [C++] Refactor IPC vector "loading" and "unloading" to be based on cleaner visitor pattern
+* [ARROW-350](https://issues.apache.org/jira/browse/ARROW-350) - Add Kerberos support to HDFS shim
+* [ARROW-353](https://issues.apache.org/jira/browse/ARROW-353) - Arrow release 0.2
+* [ARROW-355](https://issues.apache.org/jira/browse/ARROW-355) - Add tests for serialising arrays of empty strings to Parquet
+* [ARROW-356](https://issues.apache.org/jira/browse/ARROW-356) - Add documentation about reading Parquet
+* [ARROW-359](https://issues.apache.org/jira/browse/ARROW-359) - Need to document ARROW_LIBHDFS_DIR
+* [ARROW-360](https://issues.apache.org/jira/browse/ARROW-360) - C++: Add method to shrink PoolBuffer using realloc
+* [ARROW-361](https://issues.apache.org/jira/browse/ARROW-361) - Python: Support reading a column-selection from Parquet files
+* [ARROW-363](https://issues.apache.org/jira/browse/ARROW-363) - Set up Java/C++ integration test harness
+* [ARROW-365](https://issues.apache.org/jira/browse/ARROW-365) - Python: Provide Array.to_pandas()
+* [ARROW-366](https://issues.apache.org/jira/browse/ARROW-366) - [java] implement Dictionary vector
+* [ARROW-367](https://issues.apache.org/jira/browse/ARROW-367) - [java] converter csv/json <=> Arrow file format for Integration tests
+* [ARROW-368](https://issues.apache.org/jira/browse/ARROW-368) - Document use of LD_LIBRARY_PATH when using Python
+* [ARROW-369](https://issues.apache.org/jira/browse/ARROW-369) - [Python] Add ability to convert multiple record batches at once to pandas
+* [ARROW-372](https://issues.apache.org/jira/browse/ARROW-372) - Create JSON arrow file format for integration tests
+* [ARROW-373](https://issues.apache.org/jira/browse/ARROW-373) - [C++] Implement C++ version of JSON file format for testing
+* [ARROW-374](https://issues.apache.org/jira/browse/ARROW-374) - Python: clarify unicode vs. binary in API
+* [ARROW-377](https://issues.apache.org/jira/browse/ARROW-377) - Python: Add support for conversion of Pandas.Categorical
+* [ARROW-379](https://issues.apache.org/jira/browse/ARROW-379) - Python: Use setuptools_scm/setuptools_scm_git_archive to provide the version number
+* [ARROW-380](https://issues.apache.org/jira/browse/ARROW-380) - [Java] optimize null count when serializing vectors.
+* [ARROW-381](https://issues.apache.org/jira/browse/ARROW-381) - [C++] Simplify primitive array type builders to use a default type singleton
+* [ARROW-382](https://issues.apache.org/jira/browse/ARROW-382) - Python: Extend API documentation
+* [ARROW-383](https://issues.apache.org/jira/browse/ARROW-383) - [C++] Implement C++ version of ARROW-367 integration test validator
+* [ARROW-389](https://issues.apache.org/jira/browse/ARROW-389) - Python: Write Parquet files to pyarrow.io.NativeFile objects
+* [ARROW-394](https://issues.apache.org/jira/browse/ARROW-394) - Add integration tests for boolean, list, struct, and other basic types
+* [ARROW-396](https://issues.apache.org/jira/browse/ARROW-396) - Python: Add pyarrow.schema.Schema.equals
+* [ARROW-409](https://issues.apache.org/jira/browse/ARROW-409) - Python: Change pyarrow.Table.dataframe_from_batches API to create Table instead
+* [ARROW-410](https://issues.apache.org/jira/browse/ARROW-410) - [C++] Add Flush method to arrow::io::OutputStream
+* [ARROW-411](https://issues.apache.org/jira/browse/ARROW-411) - [Java] Move Intergration.compare and Intergration.compareSchemas to a public utils class
+* [ARROW-415](https://issues.apache.org/jira/browse/ARROW-415) - C++: Add Equals implementation to compare Tables
+* [ARROW-416](https://issues.apache.org/jira/browse/ARROW-416) - C++: Add Equals implementation to compare Columns
+* [ARROW-417](https://issues.apache.org/jira/browse/ARROW-417) - C++: Add Equals implementation to compare ChunkedArrays
+* [ARROW-418](https://issues.apache.org/jira/browse/ARROW-418) - [C++] Consolidate array container and builder code, remove arrow/types
+* [ARROW-419](https://issues.apache.org/jira/browse/ARROW-419) - [C++] Promote util/{status.h, buffer.h, memory-pool.h} to top level of arrow/ source directory
+* [ARROW-423](https://issues.apache.org/jira/browse/ARROW-423) - C++: Define BUILD_BYPRODUCTS in external project to support non-make CMake generators
+* [ARROW-425](https://issues.apache.org/jira/browse/ARROW-425) - Python: Expose a C function to convert arrow::Table to pyarrow.Table
+* [ARROW-426](https://issues.apache.org/jira/browse/ARROW-426) - Python: Conversion from pyarrow.Array to a Python list
+* [ARROW-427](https://issues.apache.org/jira/browse/ARROW-427) - [C++] Implement dictionary-encoded array container
+* [ARROW-428](https://issues.apache.org/jira/browse/ARROW-428) - [Python] Deserialize from Arrow record batches to pandas in parallel using a thread pool
+* [ARROW-430](https://issues.apache.org/jira/browse/ARROW-430) - Python: Better version handling
+* [ARROW-432](https://issues.apache.org/jira/browse/ARROW-432) - [Python] Avoid unnecessary memory copy in to_pandas conversion by using low-level pandas internals APIs
+* [ARROW-438](https://issues.apache.org/jira/browse/ARROW-438) - [Python] Concatenate Table instances with equal schemas
+* [ARROW-440](https://issues.apache.org/jira/browse/ARROW-440) - [C++] Support pkg-config
+* [ARROW-441](https://issues.apache.org/jira/browse/ARROW-441) - [Python] Expose Arrow's file and memory map classes as NativeFile subclasses
+* [ARROW-442](https://issues.apache.org/jira/browse/ARROW-442) - [Python] Add public Python API to inspect Parquet file metadata
+* [ARROW-444](https://issues.apache.org/jira/browse/ARROW-444) - [Python] Avoid unnecessary memory copies from use of PyBytes_* C APIs
+* [ARROW-449](https://issues.apache.org/jira/browse/ARROW-449) - Python: Conversion from pyarrow.{Table,RecordBatch} to a Python dict
+* [ARROW-450](https://issues.apache.org/jira/browse/ARROW-450) - Python: Fixes for PARQUET-818
+* [ARROW-456](https://issues.apache.org/jira/browse/ARROW-456) - C++: Add jemalloc based MemoryPool
+* [ARROW-457](https://issues.apache.org/jira/browse/ARROW-457) - Python: Better control over memory pool
+* [ARROW-458](https://issues.apache.org/jira/browse/ARROW-458) - Python: Expose jemalloc MemoryPool
+* [ARROW-461](https://issues.apache.org/jira/browse/ARROW-461) - [Python] Implement conversion between arrow::DictionaryArray and pandas.Categorical
+* [ARROW-463](https://issues.apache.org/jira/browse/ARROW-463) - C++: Support jemalloc 4.x
+* [ARROW-466](https://issues.apache.org/jira/browse/ARROW-466) - C++: ExternalProject for jemalloc
+* [ARROW-467](https://issues.apache.org/jira/browse/ARROW-467) - [Python] Run parquet-cpp unit tests in Travis CI
+* [ARROW-468](https://issues.apache.org/jira/browse/ARROW-468) - Python: Conversion of nested data in pd.DataFrames to/from Arrow structures
+* [ARROW-470](https://issues.apache.org/jira/browse/ARROW-470) - [Python] Add "FileSystem" abstraction to access directories of files in a uniform way
+* [ARROW-471](https://issues.apache.org/jira/browse/ARROW-471) - [Python] Enable ParquetFile to pass down separately-obtained file metadata
+* [ARROW-472](https://issues.apache.org/jira/browse/ARROW-472) - [Python] Expose parquet::{SchemaDescriptor, ColumnDescriptor}::Equals
+* [ARROW-474](https://issues.apache.org/jira/browse/ARROW-474) - Create an Arrow streaming file fomat
+* [ARROW-475](https://issues.apache.org/jira/browse/ARROW-475) - [Python] High level support for reading directories of Parquet files (as a single Arrow table) from supported file system interfaces
+* [ARROW-476](https://issues.apache.org/jira/browse/ARROW-476) - [Integration] Add integration tests for Binary / Varbytes type
+* [ARROW-477](https://issues.apache.org/jira/browse/ARROW-477) - [Java] Add support for second/microsecond/nanosecond timestamps in-memory and in IPC/JSON layer
+* [ARROW-478](https://issues.apache.org/jira/browse/ARROW-478) - [Python] Accept a PyBytes object in the pyarrow.io.BufferReader ctor
+* [ARROW-479](https://issues.apache.org/jira/browse/ARROW-479) - Python: Test for expected schema in Pandas conversion
+* [ARROW-484](https://issues.apache.org/jira/browse/ARROW-484) - Add more detail about what of technology can be found in the Arrow implementations to README
+* [ARROW-485](https://issues.apache.org/jira/browse/ARROW-485) - [Java] Users are required to initialize VariableLengthVectors.offsetVector before calling VariableLengthVectors.mutator.getSafe
+* [ARROW-490](https://issues.apache.org/jira/browse/ARROW-490) - Python: Update manylinux1 build scripts
+* [ARROW-495](https://issues.apache.org/jira/browse/ARROW-495) - [C++] Add C++ implementation of streaming serialized format
+* [ARROW-497](https://issues.apache.org/jira/browse/ARROW-497) - [Java] Integration test harness for streaming format
+* [ARROW-498](https://issues.apache.org/jira/browse/ARROW-498) - [C++] Integration test harness for streaming format
+* [ARROW-503](https://issues.apache.org/jira/browse/ARROW-503) - [Python] Interface to streaming binary format
+* [ARROW-506](https://issues.apache.org/jira/browse/ARROW-506) - Implement Arrow Echo server for integration testing
+* [ARROW-508](https://issues.apache.org/jira/browse/ARROW-508) - [C++] Make file/memory-mapped file interfaces threadsafe
+* [ARROW-509](https://issues.apache.org/jira/browse/ARROW-509) - [Python] Add support for PARQUET-835 (parallel column reads)
+* [ARROW-512](https://issues.apache.org/jira/browse/ARROW-512) - C++: Add method to check for primitive types
+* [ARROW-514](https://issues.apache.org/jira/browse/ARROW-514) - [Python] Accept pyarrow.io.Buffer as input to StreamReader, FileReader classes
+* [ARROW-515](https://issues.apache.org/jira/browse/ARROW-515) - [Python] Add StreamReader/FileReader methods that read all record batches as a Table
+* [ARROW-521](https://issues.apache.org/jira/browse/ARROW-521) - [C++/Python] Track peak memory use in default MemoryPool
+* [ARROW-524](https://issues.apache.org/jira/browse/ARROW-524) - [java] provide apis to access nested vectors and buffers
+* [ARROW-525](https://issues.apache.org/jira/browse/ARROW-525) - Python: Add more documentation to the package
+* [ARROW-527](https://issues.apache.org/jira/browse/ARROW-527) - clean drill-module.conf file
+* [ARROW-529](https://issues.apache.org/jira/browse/ARROW-529) - Python: Add jemalloc and Python 3.6 to manylinux1 build
+* [ARROW-531](https://issues.apache.org/jira/browse/ARROW-531) - Python: Document jemalloc, extend Pandas section, add Getting Involved
+* [ARROW-538](https://issues.apache.org/jira/browse/ARROW-538) - [C++] Set up AddressSanitizer (ASAN) builds
+* [ARROW-546](https://issues.apache.org/jira/browse/ARROW-546) - Python: Account for changes in PARQUET-867
+* [ARROW-547](https://issues.apache.org/jira/browse/ARROW-547) - [Python] Expose Array::Slice and RecordBatch::Slice
+* [ARROW-553](https://issues.apache.org/jira/browse/ARROW-553) - C++: Faster valid bitmap building
+* [ARROW-558](https://issues.apache.org/jira/browse/ARROW-558) - Add KEYS files
+* [ARROW-81](https://issues.apache.org/jira/browse/ARROW-81) - [Format] Add a Category logical type (distinct from dictionary-encoding)
+* [ARROW-96](https://issues.apache.org/jira/browse/ARROW-96) - C++: API documentation using Doxygen
+* [ARROW-97](https://issues.apache.org/jira/browse/ARROW-97) - Python: API documentation via sphinx-apidoc
+
+## Bug Fixes
 
 * [ARROW-112](https://issues.apache.org/jira/browse/ARROW-112) - [C++]  Style fix for constants/enums
 * [ARROW-202](https://issues.apache.org/jira/browse/ARROW-202) - [C++] Integrate with appveyor ci for windows support and get arrow building on windows
@@ -112,126 +251,6 @@ limitations under the License.
 * [ARROW-556](https://issues.apache.org/jira/browse/ARROW-556) - [Integration] Can not run Integration tests if different cpp build path
 * [ARROW-561](https://issues.apache.org/jira/browse/ARROW-561) - Update java & python dependencies to improve downstream packaging experience
 
-## Improvement
-
-* [ARROW-189](https://issues.apache.org/jira/browse/ARROW-189) - C++: Use ExternalProject to build thirdparty dependencies
-* [ARROW-191](https://issues.apache.org/jira/browse/ARROW-191) - Python: Provide infrastructure for manylinux1 wheels
-* [ARROW-328](https://issues.apache.org/jira/browse/ARROW-328) - [C++] Return shared_ptr by value instead of const-ref?
-* [ARROW-330](https://issues.apache.org/jira/browse/ARROW-330) - [C++] CMake functions to simplify shared / static library configuration
-* [ARROW-333](https://issues.apache.org/jira/browse/ARROW-333) - Make writers update their internal schema even when no data is written.
-* [ARROW-335](https://issues.apache.org/jira/browse/ARROW-335) - Improve Type apis and toString() by encapsulating flatbuffers better
-* [ARROW-336](https://issues.apache.org/jira/browse/ARROW-336) - Run Apache Rat in Travis builds
-* [ARROW-338](https://issues.apache.org/jira/browse/ARROW-338) - [C++] Refactor IPC vector "loading" and "unloading" to be based on cleaner visitor pattern
-* [ARROW-350](https://issues.apache.org/jira/browse/ARROW-350) - Add Kerberos support to HDFS shim
-* [ARROW-355](https://issues.apache.org/jira/browse/ARROW-355) - Add tests for serialising arrays of empty strings to Parquet
-* [ARROW-356](https://issues.apache.org/jira/browse/ARROW-356) - Add documentation about reading Parquet
-* [ARROW-360](https://issues.apache.org/jira/browse/ARROW-360) - C++: Add method to shrink PoolBuffer using realloc
-* [ARROW-361](https://issues.apache.org/jira/browse/ARROW-361) - Python: Support reading a column-selection from Parquet files
-* [ARROW-365](https://issues.apache.org/jira/browse/ARROW-365) - Python: Provide Array.to_pandas()
-* [ARROW-366](https://issues.apache.org/jira/browse/ARROW-366) - [java] implement Dictionary vector
-* [ARROW-374](https://issues.apache.org/jira/browse/ARROW-374) - Python: clarify unicode vs. binary in API
-* [ARROW-379](https://issues.apache.org/jira/browse/ARROW-379) - Python: Use setuptools_scm/setuptools_scm_git_archive to provide the version number
-* [ARROW-380](https://issues.apache.org/jira/browse/ARROW-380) - [Java] optimize null count when serializing vectors.
-* [ARROW-382](https://issues.apache.org/jira/browse/ARROW-382) - Python: Extend API documentation
-* [ARROW-396](https://issues.apache.org/jira/browse/ARROW-396) - Python: Add pyarrow.schema.Schema.equals
-* [ARROW-409](https://issues.apache.org/jira/browse/ARROW-409) - Python: Change pyarrow.Table.dataframe_from_batches API to create Table instead
-* [ARROW-411](https://issues.apache.org/jira/browse/ARROW-411) - [Java] Move Intergration.compare and Intergration.compareSchemas to a public utils class
-* [ARROW-423](https://issues.apache.org/jira/browse/ARROW-423) - C++: Define BUILD_BYPRODUCTS in external project to support non-make CMake generators
-* [ARROW-425](https://issues.apache.org/jira/browse/ARROW-425) - Python: Expose a C function to convert arrow::Table to pyarrow.Table
-* [ARROW-426](https://issues.apache.org/jira/browse/ARROW-426) - Python: Conversion from pyarrow.Array to a Python list
-* [ARROW-430](https://issues.apache.org/jira/browse/ARROW-430) - Python: Better version handling
-* [ARROW-432](https://issues.apache.org/jira/browse/ARROW-432) - [Python] Avoid unnecessary memory copy in to_pandas conversion by using low-level pandas internals APIs
-* [ARROW-450](https://issues.apache.org/jira/browse/ARROW-450) - Python: Fixes for PARQUET-818
-* [ARROW-457](https://issues.apache.org/jira/browse/ARROW-457) - Python: Better control over memory pool
-* [ARROW-458](https://issues.apache.org/jira/browse/ARROW-458) - Python: Expose jemalloc MemoryPool
-* [ARROW-463](https://issues.apache.org/jira/browse/ARROW-463) - C++: Support jemalloc 4.x
-* [ARROW-466](https://issues.apache.org/jira/browse/ARROW-466) - C++: ExternalProject for jemalloc
-* [ARROW-468](https://issues.apache.org/jira/browse/ARROW-468) - Python: Conversion of nested data in pd.DataFrames to/from Arrow structures
-* [ARROW-474](https://issues.apache.org/jira/browse/ARROW-474) - Create an Arrow streaming file fomat
-* [ARROW-479](https://issues.apache.org/jira/browse/ARROW-479) - Python: Test for expected schema in Pandas conversion
-* [ARROW-485](https://issues.apache.org/jira/browse/ARROW-485) - [Java] Users are required to initialize VariableLengthVectors.offsetVector before calling VariableLengthVectors.mutator.getSafe
-* [ARROW-490](https://issues.apache.org/jira/browse/ARROW-490) - Python: Update manylinux1 build scripts
-* [ARROW-524](https://issues.apache.org/jira/browse/ARROW-524) - [java] provide apis to access nested vectors and buffers
-* [ARROW-525](https://issues.apache.org/jira/browse/ARROW-525) - Python: Add more documentation to the package
-* [ARROW-529](https://issues.apache.org/jira/browse/ARROW-529) - Python: Add jemalloc and Python 3.6 to manylinux1 build
-* [ARROW-546](https://issues.apache.org/jira/browse/ARROW-546) - Python: Account for changes in PARQUET-867
-* [ARROW-553](https://issues.apache.org/jira/browse/ARROW-553) - C++: Faster valid bitmap building
-
-## New Feature
-
-* [ARROW-108](https://issues.apache.org/jira/browse/ARROW-108) - [C++] Add IPC round trip for union types
-* [ARROW-221](https://issues.apache.org/jira/browse/ARROW-221) - Add switch for writing Parquet 1.0 compatible logical types
-* [ARROW-227](https://issues.apache.org/jira/browse/ARROW-227) - [C++/Python] Hook arrow_io generic reader / writer interface into arrow_parquet
-* [ARROW-228](https://issues.apache.org/jira/browse/ARROW-228) - [Python] Create an Arrow-cpp-compatible interface for reading bytes from Python file-like objects
-* [ARROW-243](https://issues.apache.org/jira/browse/ARROW-243) - [C++] Add "driver" option to HdfsClient to choose between libhdfs and libhdfs3 at runtime
-* [ARROW-303](https://issues.apache.org/jira/browse/ARROW-303) - [C++] Also build static libraries for leaf libraries
-* [ARROW-312](https://issues.apache.org/jira/browse/ARROW-312) - [Python] Provide Python API to read/write the Arrow IPC file format
-* [ARROW-317](https://issues.apache.org/jira/browse/ARROW-317) - [C++] Implement zero-copy Slice method on arrow::Buffer that retains reference to parent
-* [ARROW-33](https://issues.apache.org/jira/browse/ARROW-33) - C++: Implement zero-copy array slicing
-* [ARROW-332](https://issues.apache.org/jira/browse/ARROW-332) - [Python] Add helper function to convert RecordBatch to pandas.DataFrame
-* [ARROW-363](https://issues.apache.org/jira/browse/ARROW-363) - Set up Java/C++ integration test harness
-* [ARROW-369](https://issues.apache.org/jira/browse/ARROW-369) - [Python] Add ability to convert multiple record batches at once to pandas
-* [ARROW-373](https://issues.apache.org/jira/browse/ARROW-373) - [C++] Implement C++ version of JSON file format for testing
-* [ARROW-377](https://issues.apache.org/jira/browse/ARROW-377) - Python: Add support for conversion of Pandas.Categorical
-* [ARROW-381](https://issues.apache.org/jira/browse/ARROW-381) - [C++] Simplify primitive array type builders to use a default type singleton
-* [ARROW-383](https://issues.apache.org/jira/browse/ARROW-383) - [C++] Implement C++ version of ARROW-367 integration test validator
-* [ARROW-389](https://issues.apache.org/jira/browse/ARROW-389) - Python: Write Parquet files to pyarrow.io.NativeFile objects
-* [ARROW-394](https://issues.apache.org/jira/browse/ARROW-394) - Add integration tests for boolean, list, struct, and other basic types
-* [ARROW-410](https://issues.apache.org/jira/browse/ARROW-410) - [C++] Add Flush method to arrow::io::OutputStream
-* [ARROW-415](https://issues.apache.org/jira/browse/ARROW-415) - C++: Add Equals implementation to compare Tables
-* [ARROW-416](https://issues.apache.org/jira/browse/ARROW-416) - C++: Add Equals implementation to compare Columns
-* [ARROW-417](https://issues.apache.org/jira/browse/ARROW-417) - C++: Add Equals implementation to compare ChunkedArrays
-* [ARROW-418](https://issues.apache.org/jira/browse/ARROW-418) - [C++] Consolidate array container and builder code, remove arrow/types
-* [ARROW-419](https://issues.apache.org/jira/browse/ARROW-419) - [C++] Promote util/{status.h, buffer.h, memory-pool.h} to top level of arrow/ source directory
-* [ARROW-427](https://issues.apache.org/jira/browse/ARROW-427) - [C++] Implement dictionary-encoded array container
-* [ARROW-428](https://issues.apache.org/jira/browse/ARROW-428) - [Python] Deserialize from Arrow record batches to pandas in parallel using a thread pool
-* [ARROW-438](https://issues.apache.org/jira/browse/ARROW-438) - [Python] Concatenate Table instances with equal schemas
-* [ARROW-440](https://issues.apache.org/jira/browse/ARROW-440) - [C++] Support pkg-config
-* [ARROW-441](https://issues.apache.org/jira/browse/ARROW-441) - [Python] Expose Arrow's file and memory map classes as NativeFile subclasses
-* [ARROW-442](https://issues.apache.org/jira/browse/ARROW-442) - [Python] Add public Python API to inspect Parquet file metadata
-* [ARROW-444](https://issues.apache.org/jira/browse/ARROW-444) - [Python] Avoid unnecessary memory copies from use of PyBytes_* C APIs
-* [ARROW-449](https://issues.apache.org/jira/browse/ARROW-449) - Python: Conversion from pyarrow.{Table,RecordBatch} to a Python dict
-* [ARROW-456](https://issues.apache.org/jira/browse/ARROW-456) - C++: Add jemalloc based MemoryPool
-* [ARROW-461](https://issues.apache.org/jira/browse/ARROW-461) - [Python] Implement conversion between arrow::DictionaryArray and pandas.Categorical
-* [ARROW-467](https://issues.apache.org/jira/browse/ARROW-467) - [Python] Run parquet-cpp unit tests in Travis CI
-* [ARROW-470](https://issues.apache.org/jira/browse/ARROW-470) - [Python] Add "FileSystem" abstraction to access directories of files in a uniform way
-* [ARROW-471](https://issues.apache.org/jira/browse/ARROW-471) - [Python] Enable ParquetFile to pass down separately-obtained file metadata
-* [ARROW-472](https://issues.apache.org/jira/browse/ARROW-472) - [Python] Expose parquet::{SchemaDescriptor, ColumnDescriptor}::Equals
-* [ARROW-475](https://issues.apache.org/jira/browse/ARROW-475) - [Python] High level support for reading directories of Parquet files (as a single Arrow table) from supported file system interfaces
-* [ARROW-476](https://issues.apache.org/jira/browse/ARROW-476) - [Integration] Add integration tests for Binary / Varbytes type
-* [ARROW-477](https://issues.apache.org/jira/browse/ARROW-477) - [Java] Add support for second/microsecond/nanosecond timestamps in-memory and in IPC/JSON layer
-* [ARROW-478](https://issues.apache.org/jira/browse/ARROW-478) - [Python] Accept a PyBytes object in the pyarrow.io.BufferReader ctor
-* [ARROW-484](https://issues.apache.org/jira/browse/ARROW-484) - Add more detail about what of technology can be found in the Arrow implementations to README
-* [ARROW-495](https://issues.apache.org/jira/browse/ARROW-495) - [C++] Add C++ implementation of streaming serialized format
-* [ARROW-497](https://issues.apache.org/jira/browse/ARROW-497) - [Java] Integration test harness for streaming format
-* [ARROW-498](https://issues.apache.org/jira/browse/ARROW-498) - [C++] Integration test harness for streaming format
-* [ARROW-503](https://issues.apache.org/jira/browse/ARROW-503) - [Python] Interface to streaming binary format
-* [ARROW-508](https://issues.apache.org/jira/browse/ARROW-508) - [C++] Make file/memory-mapped file interfaces threadsafe
-* [ARROW-509](https://issues.apache.org/jira/browse/ARROW-509) - [Python] Add support for PARQUET-835 (parallel column reads)
-* [ARROW-512](https://issues.apache.org/jira/browse/ARROW-512) - C++: Add method to check for primitive types
-* [ARROW-514](https://issues.apache.org/jira/browse/ARROW-514) - [Python] Accept pyarrow.io.Buffer as input to StreamReader, FileReader classes
-* [ARROW-515](https://issues.apache.org/jira/browse/ARROW-515) - [Python] Add StreamReader/FileReader methods that read all record batches as a Table
-* [ARROW-521](https://issues.apache.org/jira/browse/ARROW-521) - [C++/Python] Track peak memory use in default MemoryPool
-* [ARROW-531](https://issues.apache.org/jira/browse/ARROW-531) - Python: Document jemalloc, extend Pandas section, add Getting Involved
-* [ARROW-538](https://issues.apache.org/jira/browse/ARROW-538) - [C++] Set up AddressSanitizer (ASAN) builds
-* [ARROW-547](https://issues.apache.org/jira/browse/ARROW-547) - [Python] Expose Array::Slice and RecordBatch::Slice
-* [ARROW-81](https://issues.apache.org/jira/browse/ARROW-81) - [Format] Add a Category logical type (distinct from dictionary-encoding)
-
-## Task
-
-* [ARROW-268](https://issues.apache.org/jira/browse/ARROW-268) - [C++] Flesh out union implementation to have all required methods for IPC
-* [ARROW-327](https://issues.apache.org/jira/browse/ARROW-327) - [Python] Remove conda builds from Travis CI processes
-* [ARROW-353](https://issues.apache.org/jira/browse/ARROW-353) - Arrow release 0.2
-* [ARROW-359](https://issues.apache.org/jira/browse/ARROW-359) - Need to document ARROW_LIBHDFS_DIR
-* [ARROW-367](https://issues.apache.org/jira/browse/ARROW-367) - [java] converter csv/json <=> Arrow file format for Integration tests
-* [ARROW-368](https://issues.apache.org/jira/browse/ARROW-368) - Document use of LD_LIBRARY_PATH when using Python
-* [ARROW-372](https://issues.apache.org/jira/browse/ARROW-372) - Create JSON arrow file format for integration tests
-* [ARROW-506](https://issues.apache.org/jira/browse/ARROW-506) - Implement Arrow Echo server for integration testing
-* [ARROW-527](https://issues.apache.org/jira/browse/ARROW-527) - clean drill-module.conf file
-* [ARROW-558](https://issues.apache.org/jira/browse/ARROW-558) - Add KEYS files
-* [ARROW-96](https://issues.apache.org/jira/browse/ARROW-96) - C++: API documentation using Doxygen
-* [ARROW-97](https://issues.apache.org/jira/browse/ARROW-97) - Python: API documentation via sphinx-apidoc
-
 [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.2.0
 [3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz.md5
 [6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz
diff --git a/site/_release/0.3.0.md b/site/_release/0.3.0.md
index d76804250efe1..443bcadce78a6 100644
--- a/site/_release/0.3.0.md
+++ b/site/_release/0.3.0.md
@@ -32,161 +32,143 @@ limitations under the License.
 
 # Changelog
 
-## Bug
+## Contributors
 
-* [ARROW-109](https://issues.apache.org/jira/browse/ARROW-109) - [C++] Investigate recursive data types limit in flatbuffers
-* [ARROW-208](https://issues.apache.org/jira/browse/ARROW-208) - Add checkstyle policy to java project
-* [ARROW-347](https://issues.apache.org/jira/browse/ARROW-347) - Add method to pass CallBack when creating a transfer pair
-* [ARROW-413](https://issues.apache.org/jira/browse/ARROW-413) - DATE type is not specified clearly
-* [ARROW-431](https://issues.apache.org/jira/browse/ARROW-431) - [Python] Review GIL release and acquisition in to_pandas conversion
-* [ARROW-443](https://issues.apache.org/jira/browse/ARROW-443) - [Python] Support for converting from strided pandas data in Table.from_pandas
-* [ARROW-451](https://issues.apache.org/jira/browse/ARROW-451) - [C++] Override DataType::Equals for other types with additional metadata
-* [ARROW-454](https://issues.apache.org/jira/browse/ARROW-454) - pojo.Field doesn't implement hashCode()
-* [ARROW-526](https://issues.apache.org/jira/browse/ARROW-526) - [Format] Update IPC.md to account for File format changes and Streaming format
-* [ARROW-565](https://issues.apache.org/jira/browse/ARROW-565) - [C++] Examine "Field::dictionary" member
-* [ARROW-570](https://issues.apache.org/jira/browse/ARROW-570) - Determine Java tools JAR location from project metadata
-* [ARROW-584](https://issues.apache.org/jira/browse/ARROW-584) - [C++] Fix compiler warnings exposed with -Wconversion
-* [ARROW-588](https://issues.apache.org/jira/browse/ARROW-588) - [C++] Fix compiler warnings on 32-bit platforms
-* [ARROW-595](https://issues.apache.org/jira/browse/ARROW-595) - [Python] StreamReader.schema returns None
-* [ARROW-604](https://issues.apache.org/jira/browse/ARROW-604) - Python: boxed Field instances are missing the reference to DataType
-* [ARROW-613](https://issues.apache.org/jira/browse/ARROW-613) - [JS] Implement random-access file format
-* [ARROW-617](https://issues.apache.org/jira/browse/ARROW-617) - Time type is not specified clearly
-* [ARROW-619](https://issues.apache.org/jira/browse/ARROW-619) - Python: Fix typos in setup.py args and LD_LIBRARY_PATH
-* [ARROW-623](https://issues.apache.org/jira/browse/ARROW-623) - segfault with __repr__ of empty Field
-* [ARROW-624](https://issues.apache.org/jira/browse/ARROW-624) - [C++] Restore MakePrimitiveArray function
-* [ARROW-627](https://issues.apache.org/jira/browse/ARROW-627) - [C++] Compatibility macros for exported extern template class declarations
-* [ARROW-628](https://issues.apache.org/jira/browse/ARROW-628) - [Python] Install nomkl metapackage when building parquet-cpp for faster Travis builds
-* [ARROW-630](https://issues.apache.org/jira/browse/ARROW-630) - [C++] IPC unloading for BooleanArray does not account for offset
-* [ARROW-636](https://issues.apache.org/jira/browse/ARROW-636) - [C++] Add Boost / other system requirements to C++ README
-* [ARROW-639](https://issues.apache.org/jira/browse/ARROW-639) - [C++] Invalid offset in slices
-* [ARROW-642](https://issues.apache.org/jira/browse/ARROW-642) - [Java] Remove temporary file in java/tools
-* [ARROW-644](https://issues.apache.org/jira/browse/ARROW-644) - Python: Cython should be a setup-only requirement
-* [ARROW-652](https://issues.apache.org/jira/browse/ARROW-652) - Remove trailing f in merge script output
-* [ARROW-654](https://issues.apache.org/jira/browse/ARROW-654) - [C++] Support timezone metadata in file/stream formats
-* [ARROW-668](https://issues.apache.org/jira/browse/ARROW-668) - [Python] Convert nanosecond timestamps to pandas.Timestamp when converting from TimestampValue
-* [ARROW-671](https://issues.apache.org/jira/browse/ARROW-671) - [GLib] License file isn't installed
-* [ARROW-673](https://issues.apache.org/jira/browse/ARROW-673) - [Java] Support additional Time metadata
-* [ARROW-677](https://issues.apache.org/jira/browse/ARROW-677) - [java] Fix checkstyle jcl-over-slf4j conflict issue
-* [ARROW-678](https://issues.apache.org/jira/browse/ARROW-678) - [GLib] Fix dependenciesfff
-* [ARROW-680](https://issues.apache.org/jira/browse/ARROW-680) - [C++] Multiarch support impacts user-supplied install prefix
-* [ARROW-682](https://issues.apache.org/jira/browse/ARROW-682) - Add self-validation checks in integration tests
-* [ARROW-683](https://issues.apache.org/jira/browse/ARROW-683) - [C++] Support date32 (DateUnit::DAY) in IPC metadata, rename date to date64
-* [ARROW-686](https://issues.apache.org/jira/browse/ARROW-686) - [C++] Account for time metadata changes, add time32 and time64 types
-* [ARROW-689](https://issues.apache.org/jira/browse/ARROW-689) - [GLib] Install header files and documents to wrong directories
-* [ARROW-691](https://issues.apache.org/jira/browse/ARROW-691) - [Java] Encode dictionary Int type in message format
-* [ARROW-697](https://issues.apache.org/jira/browse/ARROW-697) - [Java] Raise appropriate exceptions when encountering large (> INT32_MAX) record batches
-* [ARROW-699](https://issues.apache.org/jira/browse/ARROW-699) - [C++] Arrow dynamic libraries are missed on run of unit tests on Windows
-* [ARROW-702](https://issues.apache.org/jira/browse/ARROW-702) - Fix BitVector.copyFromSafe to reAllocate instead of returning false
-* [ARROW-703](https://issues.apache.org/jira/browse/ARROW-703) - Fix issue where setValueCount(0) doesn’t work in the case that we’ve shipped vectors across the wire
-* [ARROW-704](https://issues.apache.org/jira/browse/ARROW-704) - Fix bad import caused by conflicting changes
-* [ARROW-709](https://issues.apache.org/jira/browse/ARROW-709) - [C++] Restore type comparator for DecimalType
-* [ARROW-713](https://issues.apache.org/jira/browse/ARROW-713) - [C++] Fix linking issue with ipc benchmark
-* [ARROW-715](https://issues.apache.org/jira/browse/ARROW-715) - Python: Explicit pandas import makes it a hard requirement
-* [ARROW-716](https://issues.apache.org/jira/browse/ARROW-716) - error building arrow/python
-* [ARROW-720](https://issues.apache.org/jira/browse/ARROW-720) - [java] arrow should not have a dependency on slf4j bridges in compile
-* [ARROW-723](https://issues.apache.org/jira/browse/ARROW-723) - Arrow freezes on write if chunk_size=0
-* [ARROW-726](https://issues.apache.org/jira/browse/ARROW-726) - [C++] PyBuffer dtor may segfault if constructor passed an object not exporting buffer protocol
-* [ARROW-732](https://issues.apache.org/jira/browse/ARROW-732) - Schema comparison bugs in struct and union types
-* [ARROW-736](https://issues.apache.org/jira/browse/ARROW-736) - [Python] Mixed-type object DataFrame columns should not silently coerce to an Arrow type by default
-* [ARROW-738](https://issues.apache.org/jira/browse/ARROW-738) - [Python] Fix manylinux1 packaging
-* [ARROW-739](https://issues.apache.org/jira/browse/ARROW-739) - Parallel build fails non-deterministically.
-* [ARROW-740](https://issues.apache.org/jira/browse/ARROW-740) - FileReader fails for large objects
-* [ARROW-747](https://issues.apache.org/jira/browse/ARROW-747) - [C++] Fix spurious warning caused by passing dl to add_dependencies
-* [ARROW-749](https://issues.apache.org/jira/browse/ARROW-749) - [Python] Delete incomplete binary files when writing fails
-* [ARROW-753](https://issues.apache.org/jira/browse/ARROW-753) - [Python] Unit tests in arrow/python fail to link on some OS X platforms
-* [ARROW-756](https://issues.apache.org/jira/browse/ARROW-756) - [C++] Do not pass -fPIC when compiling with MSVC
-* [ARROW-757](https://issues.apache.org/jira/browse/ARROW-757) - [C++] MSVC build fails on googletest when using NMake
-* [ARROW-762](https://issues.apache.org/jira/browse/ARROW-762) - Kerberos Problem with PyArrow
-* [ARROW-776](https://issues.apache.org/jira/browse/ARROW-776) - [GLib] Cast type is wrong
-* [ARROW-777](https://issues.apache.org/jira/browse/ARROW-777) - [Java] Resolve getObject behavior per changes / discussion in ARROW-729
-* [ARROW-778](https://issues.apache.org/jira/browse/ARROW-778) - Modify merge tool to work on Windows
-* [ARROW-781](https://issues.apache.org/jira/browse/ARROW-781) - [Python/C++] Increase reference count for base object?
-* [ARROW-783](https://issues.apache.org/jira/browse/ARROW-783) - Integration tests fail for length-0 record batch
-* [ARROW-787](https://issues.apache.org/jira/browse/ARROW-787) - [GLib] Fix compilation errors caused by ARROW-758
-* [ARROW-793](https://issues.apache.org/jira/browse/ARROW-793) - [GLib] Wrong indent
-* [ARROW-794](https://issues.apache.org/jira/browse/ARROW-794) - [C++] Check whether data is contiguous in ipc::WriteTensor
-* [ARROW-797](https://issues.apache.org/jira/browse/ARROW-797) - [Python] Add updated pyarrow.* public API listing in Sphinx docs
-* [ARROW-800](https://issues.apache.org/jira/browse/ARROW-800) - [C++] Boost headers being transitively included in pyarrow
-* [ARROW-805](https://issues.apache.org/jira/browse/ARROW-805) - listing empty HDFS directory returns an error instead of returning empty list
-* [ARROW-809](https://issues.apache.org/jira/browse/ARROW-809) - C++: Writing sliced record batch to IPC writes the entire array
-* [ARROW-812](https://issues.apache.org/jira/browse/ARROW-812) - Pip install pyarrow on mac failed.
-* [ARROW-817](https://issues.apache.org/jira/browse/ARROW-817) - [C++] Fix incorrect code comment from ARROW-722
-* [ARROW-821](https://issues.apache.org/jira/browse/ARROW-821) - [Python] Extra file _table_api.h generated during Python build process
-* [ARROW-822](https://issues.apache.org/jira/browse/ARROW-822) - [Python] StreamWriter fails to open with socket as sink
-* [ARROW-826](https://issues.apache.org/jira/browse/ARROW-826) - Compilation error on Mac with -DARROW_PYTHON=on
-* [ARROW-829](https://issues.apache.org/jira/browse/ARROW-829) - Python: Parquet: Dictionary encoding is deactivated if column-wise compression was selected
-* [ARROW-830](https://issues.apache.org/jira/browse/ARROW-830) - Python: jemalloc is not anymore publicly exposed
-* [ARROW-839](https://issues.apache.org/jira/browse/ARROW-839) - [C++] Portable alternative to PyDate_to_ms function
-* [ARROW-847](https://issues.apache.org/jira/browse/ARROW-847) - C++: BUILD_BYPRODUCTS not specified anymore for gtest
-* [ARROW-852](https://issues.apache.org/jira/browse/ARROW-852) - Python: Also set Arrow Library PATHS when detection was done through pkg-config
-* [ARROW-853](https://issues.apache.org/jira/browse/ARROW-853) - [Python] It is no longer necessary to modify the RPATH of the Cython extensions on many environments
-* [ARROW-858](https://issues.apache.org/jira/browse/ARROW-858) - Remove dependency on boost regex
-* [ARROW-866](https://issues.apache.org/jira/browse/ARROW-866) - [Python] Error from file object destructor
-* [ARROW-867](https://issues.apache.org/jira/browse/ARROW-867) - [Python] Miscellaneous pyarrow MSVC fixes
-* [ARROW-875](https://issues.apache.org/jira/browse/ARROW-875) - Nullable variable length vector fillEmpties() fills an extra value
-* [ARROW-879](https://issues.apache.org/jira/browse/ARROW-879) - compat with pandas 0.20.0
-* [ARROW-882](https://issues.apache.org/jira/browse/ARROW-882) - [C++] On Windows statically built lib file overwrites lib file of shared build
-* [ARROW-886](https://issues.apache.org/jira/browse/ARROW-886) - VariableLengthVectors don't reAlloc offsets
-* [ARROW-887](https://issues.apache.org/jira/browse/ARROW-887) - [format] For backward compatibility, new unit fields must have default values matching previous implied unit
-* [ARROW-888](https://issues.apache.org/jira/browse/ARROW-888) - BitVector transfer() does not transfer ownership
-* [ARROW-895](https://issues.apache.org/jira/browse/ARROW-895) - Nullable variable length vector lastSet not set correctly
-* [ARROW-900](https://issues.apache.org/jira/browse/ARROW-900) - [Python] UnboundLocalError in ParquetDatasetPiece
-* [ARROW-903](https://issues.apache.org/jira/browse/ARROW-903) - [GLib] Remove a needless "."
-* [ARROW-914](https://issues.apache.org/jira/browse/ARROW-914) - [C++/Python] Fix Decimal ToBytes
-* [ARROW-922](https://issues.apache.org/jira/browse/ARROW-922) - Allow Flatbuffers and RapidJSON to be used locally on Windows
-* [ARROW-928](https://issues.apache.org/jira/browse/ARROW-928) - Update CMAKE script to detect unsupported msvc compilers versions
-* [ARROW-933](https://issues.apache.org/jira/browse/ARROW-933) - [Python] arrow_python bindings have debug print statement
-* [ARROW-934](https://issues.apache.org/jira/browse/ARROW-934) - [GLib] Glib sources missing from result of 02-source.sh
-* [ARROW-936](https://issues.apache.org/jira/browse/ARROW-936) - Fix release README
-* [ARROW-938](https://issues.apache.org/jira/browse/ARROW-938) - Fix Apache Rat errors from source release build
+```shell
+$ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
+   119  Wes McKinney
+    55  Kouhei Sutou
+    18  Uwe L. Korn
+    17  Julien Le Dem
+     9  Phillip Cloud
+     6  Bryan Cutler
+     5  Emilio Lahr-Vivaz
+     5  Philipp Moritz
+     4  Jeff Knupp
+     4  Johan Mabille
+     4  Max Risuhin
+     3  Miki Tebeka
+     3  Steven Phillips
+     2  Brian Hulette
+     2  Jeff Reback
+     2  Leif Walsh
+     1  Deepak Majeti
+     1  Holden Karau
+     1  Itai Incze
+     1  Julien Lafaye
+     1  Nong Li
+     1  Tsuyoshi Ozawa
+     1  rvernica
+```
 
-## Improvement
+## New Features and Improvements
 
+* [ARROW-183](https://issues.apache.org/jira/browse/ARROW-183) - C++: Add storage type to DecimalType
+* [ARROW-231](https://issues.apache.org/jira/browse/ARROW-231) - C++: Add typed Resize to PoolBuffer
+* [ARROW-281](https://issues.apache.org/jira/browse/ARROW-281) - [C++] IPC/RPC support on Win32 platforms
 * [ARROW-316](https://issues.apache.org/jira/browse/ARROW-316) - Finalize Date type
+* [ARROW-341](https://issues.apache.org/jira/browse/ARROW-341) - [Python] Making libpyarrow available to third parties
+* [ARROW-452](https://issues.apache.org/jira/browse/ARROW-452) - [C++/Python] Merge "Feather" file format implementation
+* [ARROW-459](https://issues.apache.org/jira/browse/ARROW-459) - [C++] Implement IPC round trip for DictionaryArray, dictionaries shared across record batches
+* [ARROW-483](https://issues.apache.org/jira/browse/ARROW-483) - [C++/Python] Provide access to "custom_metadata" Field attribute in IPC setting
+* [ARROW-491](https://issues.apache.org/jira/browse/ARROW-491) - [C++] Add FixedWidthBinary type
+* [ARROW-493](https://issues.apache.org/jira/browse/ARROW-493) - [C++] Allow in-memory array over 2^31 -1 elements but require splitting at IPC / RPC boundaries
+* [ARROW-502](https://issues.apache.org/jira/browse/ARROW-502) - [C++/Python] Add MemoryPool implementation that logs allocation activity to std::cout
+* [ARROW-510](https://issues.apache.org/jira/browse/ARROW-510) - Add integration tests for date and time types
+* [ARROW-52](https://issues.apache.org/jira/browse/ARROW-52) - Set up project blog
+* [ARROW-520](https://issues.apache.org/jira/browse/ARROW-520) - [C++] Add STL-compliant allocator that hooks into an arrow::MemoryPool
+* [ARROW-528](https://issues.apache.org/jira/browse/ARROW-528) - [Python] Support _metadata or _common_metadata files when reading Parquet directories
+* [ARROW-534](https://issues.apache.org/jira/browse/ARROW-534) - [C++] Add IPC tests for date/time types
+* [ARROW-539](https://issues.apache.org/jira/browse/ARROW-539) - [Python] Support reading Parquet datasets with standard partition directory schemes
 * [ARROW-542](https://issues.apache.org/jira/browse/ARROW-542) - [Java] Implement dictionaries in stream/file encoding
+* [ARROW-550](https://issues.apache.org/jira/browse/ARROW-550) - [Format] Add a TensorMessage type
+* [ARROW-552](https://issues.apache.org/jira/browse/ARROW-552) - [Python] Add scalar value support for Dictionary type
+* [ARROW-557](https://issues.apache.org/jira/browse/ARROW-557) - [Python] Explicitly opt in to HDFS unit tests
 * [ARROW-563](https://issues.apache.org/jira/browse/ARROW-563) - C++: Support non-standard gcc version strings
 * [ARROW-566](https://issues.apache.org/jira/browse/ARROW-566) - Python: Deterministic position of libarrow in manylinux1 wheels
+* [ARROW-568](https://issues.apache.org/jira/browse/ARROW-568) - [C++] Add default implementations for TypeVisitor, ArrayVisitor methods that return NotImplemented
 * [ARROW-569](https://issues.apache.org/jira/browse/ARROW-569) - [C++] Set version for *.pc
+* [ARROW-574](https://issues.apache.org/jira/browse/ARROW-574) - Python: Add support for nested Python lists in Pandas conversion
+* [ARROW-576](https://issues.apache.org/jira/browse/ARROW-576) - [C++] Complete round trip Union file/stream IPC tests
 * [ARROW-577](https://issues.apache.org/jira/browse/ARROW-577) - [C++] Refactor StreamWriter and FileWriter to have private implementations
+* [ARROW-578](https://issues.apache.org/jira/browse/ARROW-578) - [C++] Add CMake option to add custom $CXXFLAGS
 * [ARROW-580](https://issues.apache.org/jira/browse/ARROW-580) - C++: Also provide jemalloc_X targets if only a static or shared version is found
 * [ARROW-582](https://issues.apache.org/jira/browse/ARROW-582) - [Java] Add Date/Time Support to JSON File
 * [ARROW-589](https://issues.apache.org/jira/browse/ARROW-589) - C++: Use system provided shared jemalloc if static is unavailable
 * [ARROW-593](https://issues.apache.org/jira/browse/ARROW-593) - [C++] Rename ReadableFileInterface to RandomAccessFile
+* [ARROW-598](https://issues.apache.org/jira/browse/ARROW-598) - [Python] Add support for converting pyarrow.Buffer to a memoryview with zero copy
+* [ARROW-603](https://issues.apache.org/jira/browse/ARROW-603) - [C++] Add RecordBatch::Validate method that at least checks that schema matches the array metadata
+* [ARROW-605](https://issues.apache.org/jira/browse/ARROW-605) - [C++] Refactor generic ArrayLoader class, support work for Feather merge
+* [ARROW-606](https://issues.apache.org/jira/browse/ARROW-606) - [C++] Upgrade to flatbuffers 1.6.0
+* [ARROW-608](https://issues.apache.org/jira/browse/ARROW-608) - [Format] Days since epoch date type
+* [ARROW-610](https://issues.apache.org/jira/browse/ARROW-610) - [C++] Win32 compatibility in file.cc
 * [ARROW-612](https://issues.apache.org/jira/browse/ARROW-612) - [Java] Field toString should show nullable flag status
 * [ARROW-615](https://issues.apache.org/jira/browse/ARROW-615) - Move ByteArrayReadableSeekableByteChannel to vector.util package
+* [ARROW-616](https://issues.apache.org/jira/browse/ARROW-616) - [C++] Remove -g flag in release builds
+* [ARROW-618](https://issues.apache.org/jira/browse/ARROW-618) - [Python] Implement support for DatetimeTZ custom type from pandas
+* [ARROW-620](https://issues.apache.org/jira/browse/ARROW-620) - [C++] Add date/time support to JSON reader/writer for integration testing
+* [ARROW-621](https://issues.apache.org/jira/browse/ARROW-621) - [C++] Implement an "inline visitor" template that enables visitor-pattern-like code without virtual function dispatch
+* [ARROW-625](https://issues.apache.org/jira/browse/ARROW-625) - [C++] Add time unit to TimeType::ToString
+* [ARROW-626](https://issues.apache.org/jira/browse/ARROW-626) - [Python] Enable pyarrow.BufferReader to read from any Python object implementing the buffer/memoryview protocol
 * [ARROW-631](https://issues.apache.org/jira/browse/ARROW-631) - [GLib] Import C API (C++ API wrapper) based on GLib from https://github.com/kou/arrow-glib
+* [ARROW-632](https://issues.apache.org/jira/browse/ARROW-632) - [Python] Add support for FixedWidthBinary type
+* [ARROW-635](https://issues.apache.org/jira/browse/ARROW-635) - [C++] Add JSON read/write support for FixedWidthBinary
+* [ARROW-637](https://issues.apache.org/jira/browse/ARROW-637) - [Format] Add time zone metadata to Timestamp type
 * [ARROW-646](https://issues.apache.org/jira/browse/ARROW-646) - Cache miniconda packages
 * [ARROW-647](https://issues.apache.org/jira/browse/ARROW-647) - [C++] Don't require Boost static libraries to support CentOS 7
 * [ARROW-648](https://issues.apache.org/jira/browse/ARROW-648) - [C++] Support multiarch on Debian
 * [ARROW-650](https://issues.apache.org/jira/browse/ARROW-650) - [GLib] Follow eadableFileInterface -> RnadomAccessFile change
 * [ARROW-651](https://issues.apache.org/jira/browse/ARROW-651) - [C++] Set shared library version for .deb packages
 * [ARROW-655](https://issues.apache.org/jira/browse/ARROW-655) - Implement DecimalArray
+* [ARROW-656](https://issues.apache.org/jira/browse/ARROW-656) - [C++] Implement IO interface that can read and write to a fixed-size mutable buffer
+* [ARROW-657](https://issues.apache.org/jira/browse/ARROW-657) - [Python] Write and read tensors (with zero copy) into shared memory
+* [ARROW-658](https://issues.apache.org/jira/browse/ARROW-658) - [C++] Implement in-memory arrow::Tensor objects
+* [ARROW-659](https://issues.apache.org/jira/browse/ARROW-659) - [C++] Add multithreaded memcpy implementation (for hardware where it helps)
+* [ARROW-660](https://issues.apache.org/jira/browse/ARROW-660) - [C++] Restore function that can read a complete encapsulated record batch message
+* [ARROW-661](https://issues.apache.org/jira/browse/ARROW-661) - [C++] Add a Flatbuffer metadata type that supports array data over 2^31 - 1 elements
 * [ARROW-662](https://issues.apache.org/jira/browse/ARROW-662) - [Format] Factor Flatbuffer schema metadata into a Schema.fbs
+* [ARROW-663](https://issues.apache.org/jira/browse/ARROW-663) - [Java] Support additional Time metadata + vector value accessors
 * [ARROW-664](https://issues.apache.org/jira/browse/ARROW-664) - Make C++ Arrow serialization deterministic
+* [ARROW-669](https://issues.apache.org/jira/browse/ARROW-669) - [Python] Attach proper tzinfo when computing boxed scalars for TimestampArray
+* [ARROW-670](https://issues.apache.org/jira/browse/ARROW-670) - Arrow 0.3 release
+* [ARROW-672](https://issues.apache.org/jira/browse/ARROW-672) - [Format] Bump metadata version for 0.3 release
 * [ARROW-674](https://issues.apache.org/jira/browse/ARROW-674) - [Java] Support additional Timestamp timezone metadata
 * [ARROW-675](https://issues.apache.org/jira/browse/ARROW-675) - [GLib] Update package metadata
 * [ARROW-676](https://issues.apache.org/jira/browse/ARROW-676) - [java] move from MinorType to FieldType in ValueVectors to carry all the relevant type bits
 * [ARROW-679](https://issues.apache.org/jira/browse/ARROW-679) - [Format] Change RecordBatch and Field length members from int to long
 * [ARROW-681](https://issues.apache.org/jira/browse/ARROW-681) - [C++] Build Arrow on Windows with dynamically linked boost
 * [ARROW-684](https://issues.apache.org/jira/browse/ARROW-684) - Python: More informative message when parquet-cpp but not parquet-arrow is available
+* [ARROW-687](https://issues.apache.org/jira/browse/ARROW-687) - [C++] Build and run full test suite in Appveyor
 * [ARROW-688](https://issues.apache.org/jira/browse/ARROW-688) - [C++] Use CMAKE_INSTALL_INCLUDEDIR for consistency
 * [ARROW-690](https://issues.apache.org/jira/browse/ARROW-690) - Only send JIRA updates to issues@arrow.apache.org
+* [ARROW-698](https://issues.apache.org/jira/browse/ARROW-698) - [C++] Add options to StreamWriter/FileWriter to permit large record batches
 * [ARROW-700](https://issues.apache.org/jira/browse/ARROW-700) - Add headroom interface for allocator.
+* [ARROW-701](https://issues.apache.org/jira/browse/ARROW-701) - [Java] Support additional Date metadata
 * [ARROW-706](https://issues.apache.org/jira/browse/ARROW-706) - [GLib] Add package install document
 * [ARROW-707](https://issues.apache.org/jira/browse/ARROW-707) - Python: All none-Pandas column should be converted to NullArray
 * [ARROW-708](https://issues.apache.org/jira/browse/ARROW-708) - [C++] Some IPC code simplification, perf analysis
+* [ARROW-710](https://issues.apache.org/jira/browse/ARROW-710) - [Python] Enable Feather APIs to read and write using Python file-like objects
 * [ARROW-712](https://issues.apache.org/jira/browse/ARROW-712) - [C++] Implement Array::Accept as inline visitor
+* [ARROW-717](https://issues.apache.org/jira/browse/ARROW-717) - [C++] IPC zero-copy round trips for arrow::Tensor
+* [ARROW-718](https://issues.apache.org/jira/browse/ARROW-718) - [Python] Expose arrow::Tensor with conversions to/from NumPy arrays
 * [ARROW-719](https://issues.apache.org/jira/browse/ARROW-719) - [GLib] Support prepared source archive release
+* [ARROW-722](https://issues.apache.org/jira/browse/ARROW-722) - [Python] pandas conversions for new date and time types/metadata
 * [ARROW-724](https://issues.apache.org/jira/browse/ARROW-724) - Add "How to Contribute" section to README
 * [ARROW-725](https://issues.apache.org/jira/browse/ARROW-725) - [Format] Constant length list type
 * [ARROW-727](https://issues.apache.org/jira/browse/ARROW-727) - [Python] Write memoryview-compatible objects in NativeFile.write with zero copy
 * [ARROW-728](https://issues.apache.org/jira/browse/ARROW-728) - [C++/Python] Add arrow::Table function for removing a column
+* [ARROW-729](https://issues.apache.org/jira/browse/ARROW-729) - [Java] Add vector type for 32-bit date as days since UNIX epoch
 * [ARROW-731](https://issues.apache.org/jira/browse/ARROW-731) - [C++] Add shared library related versions to .pc
+* [ARROW-733](https://issues.apache.org/jira/browse/ARROW-733) - [C++/Format] Change name of Fixed Width Binary to Fixed *Size* Binary for consistency
+* [ARROW-734](https://issues.apache.org/jira/browse/ARROW-734) - [Python] Support for pyarrow on Windows / MSVC
+* [ARROW-735](https://issues.apache.org/jira/browse/ARROW-735) - [C++] Developer instruction document for MSVC on Windows
+* [ARROW-737](https://issues.apache.org/jira/browse/ARROW-737) - [C++] Support obtaining mutable slices of mutable buffers
 * [ARROW-741](https://issues.apache.org/jira/browse/ARROW-741) - [Python] Add Python 3.6 to Travis CI
 * [ARROW-743](https://issues.apache.org/jira/browse/ARROW-743) - [C++] Consolidate unit tests for code in array.h
 * [ARROW-744](https://issues.apache.org/jira/browse/ARROW-744) - [GLib] Re-add an assertion to garrow_table_new() test
 * [ARROW-745](https://issues.apache.org/jira/browse/ARROW-745) - [C++] Allow use of system cpplint
 * [ARROW-746](https://issues.apache.org/jira/browse/ARROW-746) - [GLib] Add garrow_array_get_data_type()
+* [ARROW-748](https://issues.apache.org/jira/browse/ARROW-748) - [Python] Pin runtime library versions in conda-forge packages to force upgrades
 * [ARROW-751](https://issues.apache.org/jira/browse/ARROW-751) - [Python] Rename all Cython extensions to "private" status with leading underscore
 * [ARROW-752](https://issues.apache.org/jira/browse/ARROW-752) - [Python] Construct pyarrow.DictionaryArray from boxed pyarrow array objects
 * [ARROW-754](https://issues.apache.org/jira/browse/ARROW-754) - [GLib] Add garrow_array_is_null()
@@ -195,14 +177,18 @@ limitations under the License.
 * [ARROW-761](https://issues.apache.org/jira/browse/ARROW-761) - [Python] Add function to compute the total size of tensor payloads, including metadata and padding
 * [ARROW-763](https://issues.apache.org/jira/browse/ARROW-763) - C++: Use `python-config` to find libpythonX.X.dylib
 * [ARROW-765](https://issues.apache.org/jira/browse/ARROW-765) - [Python] Make generic ArrowException subclass value error
+* [ARROW-768](https://issues.apache.org/jira/browse/ARROW-768) - [Java] Change the "boxed" object representation of date and time types
 * [ARROW-769](https://issues.apache.org/jira/browse/ARROW-769) - [GLib] Support building without installed Arrow C++
 * [ARROW-770](https://issues.apache.org/jira/browse/ARROW-770) - [C++] Move clang-tidy/format config files back to C++ source tree
+* [ARROW-771](https://issues.apache.org/jira/browse/ARROW-771) - [Python] Add APIs for reading individual Parquet row groups
+* [ARROW-773](https://issues.apache.org/jira/browse/ARROW-773) - [C++] Add function to create arrow::Table with column appended to existing table
 * [ARROW-774](https://issues.apache.org/jira/browse/ARROW-774) - [GLib] Remove needless LICENSE.txt copy
 * [ARROW-775](https://issues.apache.org/jira/browse/ARROW-775) - [Java] add simple constructors to value vectors
 * [ARROW-779](https://issues.apache.org/jira/browse/ARROW-779) - [C++/Python] Raise exception if old metadata encountered
 * [ARROW-782](https://issues.apache.org/jira/browse/ARROW-782) - [C++] Change struct to class for objects that meet the criteria in the Google style guide
 * [ARROW-788](https://issues.apache.org/jira/browse/ARROW-788) - Possible nondeterminism in Tensor serialization code
 * [ARROW-795](https://issues.apache.org/jira/browse/ARROW-795) - [C++] Combine libarrow/libarrow_io/libarrow_ipc
+* [ARROW-798](https://issues.apache.org/jira/browse/ARROW-798) - [Docs] Publish Format Markdown documents somehow on arrow.apache.org
 * [ARROW-802](https://issues.apache.org/jira/browse/ARROW-802) - [GLib] Add read examples
 * [ARROW-803](https://issues.apache.org/jira/browse/ARROW-803) - [GLib] Update package repository URL
 * [ARROW-804](https://issues.apache.org/jira/browse/ARROW-804) - [GLib] Update build document
@@ -220,6 +206,7 @@ limitations under the License.
 * [ARROW-828](https://issues.apache.org/jira/browse/ARROW-828) - [CPP] Document new requirement (libboost-regex-dev) in README.md
 * [ARROW-832](https://issues.apache.org/jira/browse/ARROW-832) - [C++] Upgrade thirdparty gtest to 1.8.0
 * [ARROW-833](https://issues.apache.org/jira/browse/ARROW-833) - [Python] "Quickstart" build / environment setup guide for Python developers
+* [ARROW-836](https://issues.apache.org/jira/browse/ARROW-836) - Test for timedelta compat with pandas
 * [ARROW-841](https://issues.apache.org/jira/browse/ARROW-841) - [Python] Add pyarrow build to Appveyor
 * [ARROW-844](https://issues.apache.org/jira/browse/ARROW-844) - [Format] Revise format/README.md to reflect progress reaching a more complete specification
 * [ARROW-845](https://issues.apache.org/jira/browse/ARROW-845) - [Python] Sync FindArrow.cmake changes from parquet-cpp
@@ -232,11 +219,15 @@ limitations under the License.
 * [ARROW-862](https://issues.apache.org/jira/browse/ARROW-862) - [Python] Improve source build instructions in README
 * [ARROW-863](https://issues.apache.org/jira/browse/ARROW-863) - [GLib] Use GBytes to implement zero-copy
 * [ARROW-864](https://issues.apache.org/jira/browse/ARROW-864) - [GLib] Unify Array files
+* [ARROW-865](https://issues.apache.org/jira/browse/ARROW-865) - [Python] Verify Parquet roundtrips for new date/time types
 * [ARROW-868](https://issues.apache.org/jira/browse/ARROW-868) - [GLib] Use GBytes to reduce copy
+* [ARROW-869](https://issues.apache.org/jira/browse/ARROW-869) - [JS] Rename directory to js/
 * [ARROW-871](https://issues.apache.org/jira/browse/ARROW-871) - [GLib] Unify DataType files
 * [ARROW-876](https://issues.apache.org/jira/browse/ARROW-876) - [GLib] Unify ArrayBuffer files
 * [ARROW-877](https://issues.apache.org/jira/browse/ARROW-877) - [GLib] Add garrow_array_get_null_bitmap()
 * [ARROW-878](https://issues.apache.org/jira/browse/ARROW-878) - [GLib] Add garrow_binary_array_get_buffer()
+* [ARROW-880](https://issues.apache.org/jira/browse/ARROW-880) - [GLib] Add garrow_primitive_array_get_buffer()
+* [ARROW-890](https://issues.apache.org/jira/browse/ARROW-890) - [GLib] Add GArrowMutableBuffer
 * [ARROW-892](https://issues.apache.org/jira/browse/ARROW-892) - [GLib] Fix GArrowTensor document
 * [ARROW-893](https://issues.apache.org/jira/browse/ARROW-893) - Add GLib document to Web site
 * [ARROW-894](https://issues.apache.org/jira/browse/ARROW-894) - [GLib] Add GArrowPoolBuffer
@@ -252,90 +243,121 @@ limitations under the License.
 * [ARROW-919](https://issues.apache.org/jira/browse/ARROW-919) - [GLib] Use "id" to get type enum value from GArrowDataType
 * [ARROW-920](https://issues.apache.org/jira/browse/ARROW-920) - [GLib] Add Lua examples
 * [ARROW-925](https://issues.apache.org/jira/browse/ARROW-925) - [GLib] Fix GArrowBufferReader test
+* [ARROW-926](https://issues.apache.org/jira/browse/ARROW-926) - Update KEYS to include wesm
+* [ARROW-927](https://issues.apache.org/jira/browse/ARROW-927) - C++/Python: Add manylinux1 builds to Travis matrix
 * [ARROW-930](https://issues.apache.org/jira/browse/ARROW-930) - javadoc generation fails with java 8
 * [ARROW-931](https://issues.apache.org/jira/browse/ARROW-931) - [GLib] Reconstruct input stream
-
-## New Feature
-
-* [ARROW-231](https://issues.apache.org/jira/browse/ARROW-231) - C++: Add typed Resize to PoolBuffer
-* [ARROW-281](https://issues.apache.org/jira/browse/ARROW-281) - [C++] IPC/RPC support on Win32 platforms
-* [ARROW-341](https://issues.apache.org/jira/browse/ARROW-341) - [Python] Making libpyarrow available to third parties
-* [ARROW-452](https://issues.apache.org/jira/browse/ARROW-452) - [C++/Python] Merge "Feather" file format implementation
-* [ARROW-459](https://issues.apache.org/jira/browse/ARROW-459) - [C++] Implement IPC round trip for DictionaryArray, dictionaries shared across record batches
-* [ARROW-483](https://issues.apache.org/jira/browse/ARROW-483) - [C++/Python] Provide access to "custom_metadata" Field attribute in IPC setting
-* [ARROW-491](https://issues.apache.org/jira/browse/ARROW-491) - [C++] Add FixedWidthBinary type
-* [ARROW-493](https://issues.apache.org/jira/browse/ARROW-493) - [C++] Allow in-memory array over 2^31 -1 elements but require splitting at IPC / RPC boundaries
-* [ARROW-502](https://issues.apache.org/jira/browse/ARROW-502) - [C++/Python] Add MemoryPool implementation that logs allocation activity to std::cout
-* [ARROW-510](https://issues.apache.org/jira/browse/ARROW-510) - Add integration tests for date and time types
-* [ARROW-520](https://issues.apache.org/jira/browse/ARROW-520) - [C++] Add STL-compliant allocator that hooks into an arrow::MemoryPool
-* [ARROW-528](https://issues.apache.org/jira/browse/ARROW-528) - [Python] Support _metadata or _common_metadata files when reading Parquet directories
-* [ARROW-534](https://issues.apache.org/jira/browse/ARROW-534) - [C++] Add IPC tests for date/time types
-* [ARROW-539](https://issues.apache.org/jira/browse/ARROW-539) - [Python] Support reading Parquet datasets with standard partition directory schemes
-* [ARROW-550](https://issues.apache.org/jira/browse/ARROW-550) - [Format] Add a TensorMessage type
-* [ARROW-552](https://issues.apache.org/jira/browse/ARROW-552) - [Python] Add scalar value support for Dictionary type
-* [ARROW-557](https://issues.apache.org/jira/browse/ARROW-557) - [Python] Explicitly opt in to HDFS unit tests
-* [ARROW-568](https://issues.apache.org/jira/browse/ARROW-568) - [C++] Add default implementations for TypeVisitor, ArrayVisitor methods that return NotImplemented
-* [ARROW-574](https://issues.apache.org/jira/browse/ARROW-574) - Python: Add support for nested Python lists in Pandas conversion
-* [ARROW-576](https://issues.apache.org/jira/browse/ARROW-576) - [C++] Complete round trip Union file/stream IPC tests
-* [ARROW-578](https://issues.apache.org/jira/browse/ARROW-578) - [C++] Add CMake option to add custom $CXXFLAGS
-* [ARROW-598](https://issues.apache.org/jira/browse/ARROW-598) - [Python] Add support for converting pyarrow.Buffer to a memoryview with zero copy
-* [ARROW-603](https://issues.apache.org/jira/browse/ARROW-603) - [C++] Add RecordBatch::Validate method that at least checks that schema matches the array metadata
-* [ARROW-605](https://issues.apache.org/jira/browse/ARROW-605) - [C++] Refactor generic ArrayLoader class, support work for Feather merge
-* [ARROW-606](https://issues.apache.org/jira/browse/ARROW-606) - [C++] Upgrade to flatbuffers 1.6.0
-* [ARROW-608](https://issues.apache.org/jira/browse/ARROW-608) - [Format] Days since epoch date type
-* [ARROW-610](https://issues.apache.org/jira/browse/ARROW-610) - [C++] Win32 compatibility in file.cc
-* [ARROW-616](https://issues.apache.org/jira/browse/ARROW-616) - [C++] Remove -g flag in release builds
-* [ARROW-618](https://issues.apache.org/jira/browse/ARROW-618) - [Python] Implement support for DatetimeTZ custom type from pandas
-* [ARROW-620](https://issues.apache.org/jira/browse/ARROW-620) - [C++] Add date/time support to JSON reader/writer for integration testing
-* [ARROW-621](https://issues.apache.org/jira/browse/ARROW-621) - [C++] Implement an "inline visitor" template that enables visitor-pattern-like code without virtual function dispatch
-* [ARROW-625](https://issues.apache.org/jira/browse/ARROW-625) - [C++] Add time unit to TimeType::ToString
-* [ARROW-626](https://issues.apache.org/jira/browse/ARROW-626) - [Python] Enable pyarrow.BufferReader to read from any Python object implementing the buffer/memoryview protocol
-* [ARROW-632](https://issues.apache.org/jira/browse/ARROW-632) - [Python] Add support for FixedWidthBinary type
-* [ARROW-635](https://issues.apache.org/jira/browse/ARROW-635) - [C++] Add JSON read/write support for FixedWidthBinary
-* [ARROW-637](https://issues.apache.org/jira/browse/ARROW-637) - [Format] Add time zone metadata to Timestamp type
-* [ARROW-656](https://issues.apache.org/jira/browse/ARROW-656) - [C++] Implement IO interface that can read and write to a fixed-size mutable buffer
-* [ARROW-657](https://issues.apache.org/jira/browse/ARROW-657) - [Python] Write and read tensors (with zero copy) into shared memory
-* [ARROW-658](https://issues.apache.org/jira/browse/ARROW-658) - [C++] Implement in-memory arrow::Tensor objects
-* [ARROW-659](https://issues.apache.org/jira/browse/ARROW-659) - [C++] Add multithreaded memcpy implementation (for hardware where it helps)
-* [ARROW-660](https://issues.apache.org/jira/browse/ARROW-660) - [C++] Restore function that can read a complete encapsulated record batch message
-* [ARROW-661](https://issues.apache.org/jira/browse/ARROW-661) - [C++] Add a Flatbuffer metadata type that supports array data over 2^31 - 1 elements
-* [ARROW-663](https://issues.apache.org/jira/browse/ARROW-663) - [Java] Support additional Time metadata + vector value accessors
-* [ARROW-669](https://issues.apache.org/jira/browse/ARROW-669) - [Python] Attach proper tzinfo when computing boxed scalars for TimestampArray
-* [ARROW-687](https://issues.apache.org/jira/browse/ARROW-687) - [C++] Build and run full test suite in Appveyor
-* [ARROW-698](https://issues.apache.org/jira/browse/ARROW-698) - [C++] Add options to StreamWriter/FileWriter to permit large record batches
-* [ARROW-701](https://issues.apache.org/jira/browse/ARROW-701) - [Java] Support additional Date metadata
-* [ARROW-710](https://issues.apache.org/jira/browse/ARROW-710) - [Python] Enable Feather APIs to read and write using Python file-like objects
-* [ARROW-717](https://issues.apache.org/jira/browse/ARROW-717) - [C++] IPC zero-copy round trips for arrow::Tensor
-* [ARROW-718](https://issues.apache.org/jira/browse/ARROW-718) - [Python] Expose arrow::Tensor with conversions to/from NumPy arrays
-* [ARROW-722](https://issues.apache.org/jira/browse/ARROW-722) - [Python] pandas conversions for new date and time types/metadata
-* [ARROW-729](https://issues.apache.org/jira/browse/ARROW-729) - [Java] Add vector type for 32-bit date as days since UNIX epoch
-* [ARROW-733](https://issues.apache.org/jira/browse/ARROW-733) - [C++/Format] Change name of Fixed Width Binary to Fixed *Size* Binary for consistency
-* [ARROW-734](https://issues.apache.org/jira/browse/ARROW-734) - [Python] Support for pyarrow on Windows / MSVC
-* [ARROW-735](https://issues.apache.org/jira/browse/ARROW-735) - [C++] Developer instruction document for MSVC on Windows
-* [ARROW-737](https://issues.apache.org/jira/browse/ARROW-737) - [C++] Support obtaining mutable slices of mutable buffers
-* [ARROW-768](https://issues.apache.org/jira/browse/ARROW-768) - [Java] Change the "boxed" object representation of date and time types
-* [ARROW-771](https://issues.apache.org/jira/browse/ARROW-771) - [Python] Add APIs for reading individual Parquet row groups
-* [ARROW-773](https://issues.apache.org/jira/browse/ARROW-773) - [C++] Add function to create arrow::Table with column appended to existing table
-* [ARROW-865](https://issues.apache.org/jira/browse/ARROW-865) - [Python] Verify Parquet roundtrips for new date/time types
-* [ARROW-880](https://issues.apache.org/jira/browse/ARROW-880) - [GLib] Add garrow_primitive_array_get_buffer()
-* [ARROW-890](https://issues.apache.org/jira/browse/ARROW-890) - [GLib] Add GArrowMutableBuffer
-* [ARROW-926](https://issues.apache.org/jira/browse/ARROW-926) - Update KEYS to include wesm
-
-## Task
-
-* [ARROW-52](https://issues.apache.org/jira/browse/ARROW-52) - Set up project blog
-* [ARROW-670](https://issues.apache.org/jira/browse/ARROW-670) - Arrow 0.3 release
-* [ARROW-672](https://issues.apache.org/jira/browse/ARROW-672) - [Format] Bump metadata version for 0.3 release
-* [ARROW-748](https://issues.apache.org/jira/browse/ARROW-748) - [Python] Pin runtime library versions in conda-forge packages to force upgrades
-* [ARROW-798](https://issues.apache.org/jira/browse/ARROW-798) - [Docs] Publish Format Markdown documents somehow on arrow.apache.org
-* [ARROW-869](https://issues.apache.org/jira/browse/ARROW-869) - [JS] Rename directory to js/
 * [ARROW-95](https://issues.apache.org/jira/browse/ARROW-95) - Scaffold Main Documentation using asciidoc
+* [ARROW-965](https://issues.apache.org/jira/browse/ARROW-965) - Website updates for 0.3.0 release
 * [ARROW-98](https://issues.apache.org/jira/browse/ARROW-98) - Java: API documentation
 
-## Test
+## Bug Fixes
 
-* [ARROW-836](https://issues.apache.org/jira/browse/ARROW-836) - Test for timedelta compat with pandas
-* [ARROW-927](https://issues.apache.org/jira/browse/ARROW-927) - C++/Python: Add manylinux1 builds to Travis matrix
+* [ARROW-109](https://issues.apache.org/jira/browse/ARROW-109) - [C++] Investigate recursive data types limit in flatbuffers
+* [ARROW-208](https://issues.apache.org/jira/browse/ARROW-208) - Add checkstyle policy to java project
+* [ARROW-347](https://issues.apache.org/jira/browse/ARROW-347) - Add method to pass CallBack when creating a transfer pair
+* [ARROW-413](https://issues.apache.org/jira/browse/ARROW-413) - DATE type is not specified clearly
+* [ARROW-431](https://issues.apache.org/jira/browse/ARROW-431) - [Python] Review GIL release and acquisition in to_pandas conversion
+* [ARROW-443](https://issues.apache.org/jira/browse/ARROW-443) - [Python] Support for converting from strided pandas data in Table.from_pandas
+* [ARROW-451](https://issues.apache.org/jira/browse/ARROW-451) - [C++] Override DataType::Equals for other types with additional metadata
+* [ARROW-454](https://issues.apache.org/jira/browse/ARROW-454) - pojo.Field doesn't implement hashCode()
+* [ARROW-526](https://issues.apache.org/jira/browse/ARROW-526) - [Format] Update IPC.md to account for File format changes and Streaming format
+* [ARROW-565](https://issues.apache.org/jira/browse/ARROW-565) - [C++] Examine "Field::dictionary" member
+* [ARROW-570](https://issues.apache.org/jira/browse/ARROW-570) - Determine Java tools JAR location from project metadata
+* [ARROW-584](https://issues.apache.org/jira/browse/ARROW-584) - [C++] Fix compiler warnings exposed with -Wconversion
+* [ARROW-588](https://issues.apache.org/jira/browse/ARROW-588) - [C++] Fix compiler warnings on 32-bit platforms
+* [ARROW-595](https://issues.apache.org/jira/browse/ARROW-595) - [Python] StreamReader.schema returns None
+* [ARROW-604](https://issues.apache.org/jira/browse/ARROW-604) - Python: boxed Field instances are missing the reference to DataType
+* [ARROW-613](https://issues.apache.org/jira/browse/ARROW-613) - [JS] Implement random-access file format
+* [ARROW-617](https://issues.apache.org/jira/browse/ARROW-617) - Time type is not specified clearly
+* [ARROW-619](https://issues.apache.org/jira/browse/ARROW-619) - Python: Fix typos in setup.py args and LD_LIBRARY_PATH
+* [ARROW-623](https://issues.apache.org/jira/browse/ARROW-623) - segfault with __repr__ of empty Field
+* [ARROW-624](https://issues.apache.org/jira/browse/ARROW-624) - [C++] Restore MakePrimitiveArray function
+* [ARROW-627](https://issues.apache.org/jira/browse/ARROW-627) - [C++] Compatibility macros for exported extern template class declarations
+* [ARROW-628](https://issues.apache.org/jira/browse/ARROW-628) - [Python] Install nomkl metapackage when building parquet-cpp for faster Travis builds
+* [ARROW-630](https://issues.apache.org/jira/browse/ARROW-630) - [C++] IPC unloading for BooleanArray does not account for offset
+* [ARROW-636](https://issues.apache.org/jira/browse/ARROW-636) - [C++] Add Boost / other system requirements to C++ README
+* [ARROW-639](https://issues.apache.org/jira/browse/ARROW-639) - [C++] Invalid offset in slices
+* [ARROW-642](https://issues.apache.org/jira/browse/ARROW-642) - [Java] Remove temporary file in java/tools
+* [ARROW-644](https://issues.apache.org/jira/browse/ARROW-644) - Python: Cython should be a setup-only requirement
+* [ARROW-652](https://issues.apache.org/jira/browse/ARROW-652) - Remove trailing f in merge script output
+* [ARROW-654](https://issues.apache.org/jira/browse/ARROW-654) - [C++] Support timezone metadata in file/stream formats
+* [ARROW-668](https://issues.apache.org/jira/browse/ARROW-668) - [Python] Convert nanosecond timestamps to pandas.Timestamp when converting from TimestampValue
+* [ARROW-671](https://issues.apache.org/jira/browse/ARROW-671) - [GLib] License file isn't installed
+* [ARROW-673](https://issues.apache.org/jira/browse/ARROW-673) - [Java] Support additional Time metadata
+* [ARROW-677](https://issues.apache.org/jira/browse/ARROW-677) - [java] Fix checkstyle jcl-over-slf4j conflict issue
+* [ARROW-678](https://issues.apache.org/jira/browse/ARROW-678) - [GLib] Fix dependenciesfff
+* [ARROW-680](https://issues.apache.org/jira/browse/ARROW-680) - [C++] Multiarch support impacts user-supplied install prefix
+* [ARROW-682](https://issues.apache.org/jira/browse/ARROW-682) - Add self-validation checks in integration tests
+* [ARROW-683](https://issues.apache.org/jira/browse/ARROW-683) - [C++] Support date32 (DateUnit::DAY) in IPC metadata, rename date to date64
+* [ARROW-686](https://issues.apache.org/jira/browse/ARROW-686) - [C++] Account for time metadata changes, add time32 and time64 types
+* [ARROW-689](https://issues.apache.org/jira/browse/ARROW-689) - [GLib] Install header files and documents to wrong directories
+* [ARROW-691](https://issues.apache.org/jira/browse/ARROW-691) - [Java] Encode dictionary Int type in message format
+* [ARROW-697](https://issues.apache.org/jira/browse/ARROW-697) - [Java] Raise appropriate exceptions when encountering large (> INT32_MAX) record batches
+* [ARROW-699](https://issues.apache.org/jira/browse/ARROW-699) - [C++] Arrow dynamic libraries are missed on run of unit tests on Windows
+* [ARROW-702](https://issues.apache.org/jira/browse/ARROW-702) - Fix BitVector.copyFromSafe to reAllocate instead of returning false
+* [ARROW-703](https://issues.apache.org/jira/browse/ARROW-703) - Fix issue where setValueCount(0) doesn’t work in the case that we’ve shipped vectors across the wire
+* [ARROW-704](https://issues.apache.org/jira/browse/ARROW-704) - Fix bad import caused by conflicting changes
+* [ARROW-709](https://issues.apache.org/jira/browse/ARROW-709) - [C++] Restore type comparator for DecimalType
+* [ARROW-713](https://issues.apache.org/jira/browse/ARROW-713) - [C++] Fix linking issue with ipc benchmark
+* [ARROW-715](https://issues.apache.org/jira/browse/ARROW-715) - Python: Explicit pandas import makes it a hard requirement
+* [ARROW-716](https://issues.apache.org/jira/browse/ARROW-716) - error building arrow/python
+* [ARROW-720](https://issues.apache.org/jira/browse/ARROW-720) - [java] arrow should not have a dependency on slf4j bridges in compile
+* [ARROW-723](https://issues.apache.org/jira/browse/ARROW-723) - Arrow freezes on write if chunk_size=0
+* [ARROW-726](https://issues.apache.org/jira/browse/ARROW-726) - [C++] PyBuffer dtor may segfault if constructor passed an object not exporting buffer protocol
+* [ARROW-732](https://issues.apache.org/jira/browse/ARROW-732) - Schema comparison bugs in struct and union types
+* [ARROW-736](https://issues.apache.org/jira/browse/ARROW-736) - [Python] Mixed-type object DataFrame columns should not silently coerce to an Arrow type by default
+* [ARROW-738](https://issues.apache.org/jira/browse/ARROW-738) - [Python] Fix manylinux1 packaging
+* [ARROW-739](https://issues.apache.org/jira/browse/ARROW-739) - Parallel build fails non-deterministically.
+* [ARROW-740](https://issues.apache.org/jira/browse/ARROW-740) - FileReader fails for large objects
+* [ARROW-747](https://issues.apache.org/jira/browse/ARROW-747) - [C++] Fix spurious warning caused by passing dl to add_dependencies
+* [ARROW-749](https://issues.apache.org/jira/browse/ARROW-749) - [Python] Delete incomplete binary files when writing fails
+* [ARROW-753](https://issues.apache.org/jira/browse/ARROW-753) - [Python] Unit tests in arrow/python fail to link on some OS X platforms
+* [ARROW-756](https://issues.apache.org/jira/browse/ARROW-756) - [C++] Do not pass -fPIC when compiling with MSVC
+* [ARROW-757](https://issues.apache.org/jira/browse/ARROW-757) - [C++] MSVC build fails on googletest when using NMake
+* [ARROW-762](https://issues.apache.org/jira/browse/ARROW-762) - Kerberos Problem with PyArrow
+* [ARROW-776](https://issues.apache.org/jira/browse/ARROW-776) - [GLib] Cast type is wrong
+* [ARROW-777](https://issues.apache.org/jira/browse/ARROW-777) - [Java] Resolve getObject behavior per changes / discussion in ARROW-729
+* [ARROW-778](https://issues.apache.org/jira/browse/ARROW-778) - Modify merge tool to work on Windows
+* [ARROW-781](https://issues.apache.org/jira/browse/ARROW-781) - [Python/C++] Increase reference count for base object?
+* [ARROW-783](https://issues.apache.org/jira/browse/ARROW-783) - Integration tests fail for length-0 record batch
+* [ARROW-787](https://issues.apache.org/jira/browse/ARROW-787) - [GLib] Fix compilation errors caused by ARROW-758
+* [ARROW-793](https://issues.apache.org/jira/browse/ARROW-793) - [GLib] Wrong indent
+* [ARROW-794](https://issues.apache.org/jira/browse/ARROW-794) - [C++] Check whether data is contiguous in ipc::WriteTensor
+* [ARROW-797](https://issues.apache.org/jira/browse/ARROW-797) - [Python] Add updated pyarrow.* public API listing in Sphinx docs
+* [ARROW-800](https://issues.apache.org/jira/browse/ARROW-800) - [C++] Boost headers being transitively included in pyarrow
+* [ARROW-805](https://issues.apache.org/jira/browse/ARROW-805) - listing empty HDFS directory returns an error instead of returning empty list
+* [ARROW-809](https://issues.apache.org/jira/browse/ARROW-809) - C++: Writing sliced record batch to IPC writes the entire array
+* [ARROW-812](https://issues.apache.org/jira/browse/ARROW-812) - Pip install pyarrow on mac failed.
+* [ARROW-817](https://issues.apache.org/jira/browse/ARROW-817) - [C++] Fix incorrect code comment from ARROW-722
+* [ARROW-821](https://issues.apache.org/jira/browse/ARROW-821) - [Python] Extra file _table_api.h generated during Python build process
+* [ARROW-822](https://issues.apache.org/jira/browse/ARROW-822) - [Python] StreamWriter fails to open with socket as sink
+* [ARROW-826](https://issues.apache.org/jira/browse/ARROW-826) - Compilation error on Mac with -DARROW_PYTHON=on
+* [ARROW-829](https://issues.apache.org/jira/browse/ARROW-829) - Python: Parquet: Dictionary encoding is deactivated if column-wise compression was selected
+* [ARROW-830](https://issues.apache.org/jira/browse/ARROW-830) - Python: jemalloc is not anymore publicly exposed
+* [ARROW-839](https://issues.apache.org/jira/browse/ARROW-839) - [C++] Portable alternative to PyDate_to_ms function
+* [ARROW-847](https://issues.apache.org/jira/browse/ARROW-847) - C++: BUILD_BYPRODUCTS not specified anymore for gtest
+* [ARROW-852](https://issues.apache.org/jira/browse/ARROW-852) - Python: Also set Arrow Library PATHS when detection was done through pkg-config
+* [ARROW-853](https://issues.apache.org/jira/browse/ARROW-853) - [Python] It is no longer necessary to modify the RPATH of the Cython extensions on many environments
+* [ARROW-858](https://issues.apache.org/jira/browse/ARROW-858) - Remove dependency on boost regex
+* [ARROW-866](https://issues.apache.org/jira/browse/ARROW-866) - [Python] Error from file object destructor
+* [ARROW-867](https://issues.apache.org/jira/browse/ARROW-867) - [Python] Miscellaneous pyarrow MSVC fixes
+* [ARROW-875](https://issues.apache.org/jira/browse/ARROW-875) - Nullable variable length vector fillEmpties() fills an extra value
+* [ARROW-879](https://issues.apache.org/jira/browse/ARROW-879) - compat with pandas 0.20.0
+* [ARROW-882](https://issues.apache.org/jira/browse/ARROW-882) - [C++] On Windows statically built lib file overwrites lib file of shared build
+* [ARROW-886](https://issues.apache.org/jira/browse/ARROW-886) - VariableLengthVectors don't reAlloc offsets
+* [ARROW-887](https://issues.apache.org/jira/browse/ARROW-887) - [format] For backward compatibility, new unit fields must have default values matching previous implied unit
+* [ARROW-888](https://issues.apache.org/jira/browse/ARROW-888) - BitVector transfer() does not transfer ownership
+* [ARROW-895](https://issues.apache.org/jira/browse/ARROW-895) - Nullable variable length vector lastSet not set correctly
+* [ARROW-900](https://issues.apache.org/jira/browse/ARROW-900) - [Python] UnboundLocalError in ParquetDatasetPiece
+* [ARROW-903](https://issues.apache.org/jira/browse/ARROW-903) - [GLib] Remove a needless "."
+* [ARROW-914](https://issues.apache.org/jira/browse/ARROW-914) - [C++/Python] Fix Decimal ToBytes
+* [ARROW-922](https://issues.apache.org/jira/browse/ARROW-922) - Allow Flatbuffers and RapidJSON to be used locally on Windows
+* [ARROW-928](https://issues.apache.org/jira/browse/ARROW-928) - Update CMAKE script to detect unsupported msvc compilers versions
+* [ARROW-933](https://issues.apache.org/jira/browse/ARROW-933) - [Python] arrow_python bindings have debug print statement
+* [ARROW-934](https://issues.apache.org/jira/browse/ARROW-934) - [GLib] Glib sources missing from result of 02-source.sh
+* [ARROW-936](https://issues.apache.org/jira/browse/ARROW-936) - Fix release README
+* [ARROW-938](https://issues.apache.org/jira/browse/ARROW-938) - Fix Apache Rat errors from source release build
 
 [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.3.0
 [3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.md5

From 222cbfeea112ca18c36458418c773f1c8a8aa7ea Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 15 May 2017 21:34:38 -0400
Subject: [PATCH 0657/1644] ARROW-998: [Format] Clarify that the IPC file
 footer contains an additional copy of the schema

cc @TheNeuralBit

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #694 from wesm/ARROW-998 and squashes the following commits:

eec4fad [Wes McKinney] Clarify that the IPC file footer contains an additional copy of the schema
---
 format/IPC.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/format/IPC.md b/format/IPC.md
index f0a67e292186c..bf2aaa74b3b46 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -83,9 +83,11 @@ as an `int32` or simply closing the stream interface.
 We define a "file format" supporting random access in a very similar format to
 the streaming format. The file starts and ends with a magic string `ARROW1`
 (plus padding). What follows in the file is identical to the stream format. At
-the end of the file, we write a *footer* including offsets and sizes for each
-of the data blocks in the file, so that random access is possible. See
-[format/File.fbs][1] for the precise details of the file footer.
+the end of the file, we write a *footer* containing a redundant copy of the
+schema (which is a part of the streaming format) plus memory offsets and sizes
+for each of the data blocks in the file. This enables random access any record
+batch in the file. See [format/File.fbs][1] for the precise details of the file
+footer.
 
 Schematically we have:
 

From 86a905562873c7d93c75c906dae0f17bd3c58aa1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 15 May 2017 21:37:03 -0400
Subject: [PATCH 0658/1644] ARROW-182: [C++] Factor out Array::Validate into a
 separate function

This makes it easier to use templates for writing more validation logic, and also we don't need the additional array methods

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #692 from wesm/ARROW-182 and squashes the following commits:

9753a59 [Wes McKinney] Factor out Array::Validate into a separate function
---
 cpp/src/arrow/array-test.cc     |  34 ++---
 cpp/src/arrow/array.cc          | 255 ++++++++++++++++----------------
 cpp/src/arrow/array.h           |  27 ++--
 cpp/src/arrow/ipc/test-common.h |   2 +-
 4 files changed, 153 insertions(+), 165 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 99279f3a8bb65..636d97f9d059f 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -685,7 +685,7 @@ class TestStringArray : public ::testing::Test {
 TEST_F(TestStringArray, TestArrayBasics) {
   ASSERT_EQ(length_, strings_->length());
   ASSERT_EQ(1, strings_->null_count());
-  ASSERT_OK(strings_->Validate());
+  ASSERT_OK(ValidateArray(*strings_));
 }
 
 TEST_F(TestStringArray, TestType) {
@@ -801,7 +801,7 @@ class TestStringBuilder : public TestBuilder {
     EXPECT_OK(builder_->Finish(&out));
 
     result_ = std::dynamic_pointer_cast<StringArray>(out);
-    result_->Validate();
+    ASSERT_OK(ValidateArray(*result_));
   }
 
  protected:
@@ -899,7 +899,7 @@ class TestBinaryArray : public ::testing::Test {
 TEST_F(TestBinaryArray, TestArrayBasics) {
   ASSERT_EQ(length_, strings_->length());
   ASSERT_EQ(1, strings_->null_count());
-  ASSERT_OK(strings_->Validate());
+  ASSERT_OK(ValidateArray(*strings_));
 }
 
 TEST_F(TestBinaryArray, TestType) {
@@ -969,7 +969,7 @@ class TestBinaryBuilder : public TestBuilder {
     EXPECT_OK(builder_->Finish(&out));
 
     result_ = std::dynamic_pointer_cast<BinaryArray>(out);
-    result_->Validate();
+    ASSERT_OK(ValidateArray(*result_));
   }
 
  protected:
@@ -994,7 +994,7 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
     }
   }
   Done();
-  ASSERT_OK(result_->Validate());
+  ASSERT_OK(ValidateArray(*result_));
   ASSERT_EQ(reps * N, result_->length());
   ASSERT_EQ(reps, result_->null_count());
   ASSERT_EQ(reps * 6, result_->data()->size());
@@ -1354,7 +1354,7 @@ TEST_F(TestListBuilder, TestAppendNull) {
 
   Done();
 
-  ASSERT_OK(result_->Validate());
+  ASSERT_OK(ValidateArray(*result_));
   ASSERT_TRUE(result_->IsNull(0));
   ASSERT_TRUE(result_->IsNull(1));
 
@@ -1368,7 +1368,7 @@ TEST_F(TestListBuilder, TestAppendNull) {
 
 void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& values,
     const vector<uint8_t>& is_valid) {
-  ASSERT_OK(result->Validate());
+  ASSERT_OK(ValidateArray(*result));
   ASSERT_EQ(1, result->null_count());
   ASSERT_EQ(0, result->values()->null_count());
 
@@ -1446,13 +1446,13 @@ TEST_F(TestListBuilder, BulkAppendInvalid) {
   }
 
   Done();
-  ASSERT_RAISES(Invalid, result_->Validate());
+  ASSERT_RAISES(Invalid, ValidateArray(*result_));
 }
 
 TEST_F(TestListBuilder, TestZeroLength) {
   // All buffers are null
   Done();
-  ASSERT_OK(result_->Validate());
+  ASSERT_OK(ValidateArray(*result_));
 }
 
 // ----------------------------------------------------------------------
@@ -1569,9 +1569,9 @@ TEST(TestDictionary, Validate) {
   std::shared_ptr<Array> arr3 = std::make_shared<DictionaryArray>(dict_type, indices3);
 
   // Only checking index type for now
-  ASSERT_OK(arr->Validate());
-  ASSERT_RAISES(Invalid, arr2->Validate());
-  ASSERT_OK(arr3->Validate());
+  ASSERT_OK(ValidateArray(*arr));
+  ASSERT_RAISES(Invalid, ValidateArray(*arr2));
+  ASSERT_OK(ValidateArray(*arr3));
 }
 
 // ----------------------------------------------------------------------
@@ -1582,7 +1582,7 @@ void ValidateBasicStructArray(const StructArray* result,
     const vector<uint8_t>& list_is_valid, const vector<int>& list_lengths,
     const vector<int>& list_offsets, const vector<int32_t>& int_values) {
   ASSERT_EQ(4, result->length());
-  ASSERT_OK(result->Validate());
+  ASSERT_OK(ValidateArray(*result));
 
   auto list_char_arr = static_cast<ListArray*>(result->field(0).get());
   auto char_arr = static_cast<Int8Array*>(list_char_arr->values().get());
@@ -1666,7 +1666,7 @@ TEST_F(TestStructBuilder, TestAppendNull) {
 
   Done();
 
-  ASSERT_OK(result_->Validate());
+  ASSERT_OK(ValidateArray(*result_));
 
   ASSERT_EQ(2, static_cast<int>(result_->fields().size()));
   ASSERT_EQ(2, result_->length());
@@ -1777,8 +1777,8 @@ TEST_F(TestStructBuilder, BulkAppendInvalid) {
   }
 
   Done();
-  // Even null bitmap of the parent Struct is not valid, Validate() will ignore it.
-  ASSERT_OK(result_->Validate());
+  // Even null bitmap of the parent Struct is not valid, validate will ignore it.
+  ASSERT_OK(ValidateArray(*result_));
 }
 
 TEST_F(TestStructBuilder, TestEquality) {
@@ -1924,7 +1924,7 @@ TEST_F(TestStructBuilder, TestEquality) {
 TEST_F(TestStructBuilder, TestZeroLength) {
   // All buffers are null
   Done();
-  ASSERT_OK(result_->Validate());
+  ASSERT_OK(ValidateArray(*result_));
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 76dda2ca7b94f..c5acf3ee6aa4b 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -105,10 +105,6 @@ bool Array::RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
   return are_equal;
 }
 
-Status Array::Validate() const {
-  return Status::OK();
-}
-
 // Last two parameters are in-out parameters
 static inline void ConformSliceParams(
     int64_t array_offset, int64_t array_length, int64_t* offset, int64_t* length) {
@@ -166,58 +162,6 @@ std::shared_ptr<Array> BooleanArray::Slice(int64_t offset, int64_t length) const
 // ----------------------------------------------------------------------
 // ListArray
 
-Status ListArray::Validate() const {
-  if (length_ < 0) { return Status::Invalid("Length was negative"); }
-  if (length_ && !value_offsets_) { return Status::Invalid("value_offsets_ was null"); }
-  if (value_offsets_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
-    std::stringstream ss;
-    ss << "offset buffer size (bytes): " << value_offsets_->size()
-       << " isn't large enough for length: " << length_;
-    return Status::Invalid(ss.str());
-  }
-  const int32_t last_offset = this->value_offset(length_);
-  if (last_offset > 0) {
-    if (!values_) {
-      return Status::Invalid("last offset was non-zero and values was null");
-    }
-    if (values_->length() != last_offset) {
-      std::stringstream ss;
-      ss << "Final offset invariant not equal to values length: " << last_offset
-         << "!=" << values_->length();
-      return Status::Invalid(ss.str());
-    }
-
-    const Status child_valid = values_->Validate();
-    if (!child_valid.ok()) {
-      std::stringstream ss;
-      ss << "Child array invalid: " << child_valid.ToString();
-      return Status::Invalid(ss.str());
-    }
-  }
-
-  int32_t prev_offset = this->value_offset(0);
-  if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
-  for (int64_t i = 1; i <= length_; ++i) {
-    int32_t current_offset = this->value_offset(i);
-    if (IsNull(i - 1) && current_offset != prev_offset) {
-      std::stringstream ss;
-      ss << "Offset invariant failure at: " << i
-         << " inconsistent value_offsets for null slot" << current_offset
-         << "!=" << prev_offset;
-      return Status::Invalid(ss.str());
-    }
-    if (current_offset < prev_offset) {
-      std::stringstream ss;
-      ss << "Offset invariant failure: " << i
-         << " inconsistent offset for non-null slot: " << current_offset << "<"
-         << prev_offset;
-      return Status::Invalid(ss.str());
-    }
-    prev_offset = current_offset;
-  }
-  return Status::OK();
-}
-
 std::shared_ptr<Array> ListArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<ListArray>(
@@ -250,11 +194,6 @@ BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
   if (data_ != nullptr) { raw_data_ = data_->data(); }
 }
 
-Status BinaryArray::Validate() const {
-  // TODO(wesm): what to do here?
-  return Status::OK();
-}
-
 std::shared_ptr<Array> BinaryArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<BinaryArray>(
@@ -267,11 +206,6 @@ StringArray::StringArray(int64_t length, const std::shared_ptr<Buffer>& value_of
     : BinaryArray(kString, length, value_offsets, data, null_bitmap, null_count, offset) {
 }
 
-Status StringArray::Validate() const {
-  // TODO(emkornfield) Validate proper UTF8 code points?
-  return BinaryArray::Validate();
-}
-
 std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<StringArray>(
@@ -361,42 +295,6 @@ std::shared_ptr<Array> StructArray::field(int pos) const {
   return children_[pos];
 }
 
-Status StructArray::Validate() const {
-  if (length_ < 0) { return Status::Invalid("Length was negative"); }
-
-  if (null_count() > length_) {
-    return Status::Invalid("Null count exceeds the length of this struct");
-  }
-
-  if (children_.size() > 0) {
-    // Validate fields
-    int64_t array_length = children_[0]->length();
-    size_t idx = 0;
-    for (auto it : children_) {
-      if (it->length() != array_length) {
-        std::stringstream ss;
-        ss << "Length is not equal from field " << it->type()->ToString()
-           << " at position {" << idx << "}";
-        return Status::Invalid(ss.str());
-      }
-
-      const Status child_valid = it->Validate();
-      if (!child_valid.ok()) {
-        std::stringstream ss;
-        ss << "Child array invalid: " << child_valid.ToString() << " at position {" << idx
-           << "}";
-        return Status::Invalid(ss.str());
-      }
-      ++idx;
-    }
-
-    if (array_length > 0 && array_length != length_) {
-      return Status::Invalid("Struct's length is not equal to its child arrays");
-    }
-  }
-  return Status::OK();
-}
-
 std::shared_ptr<Array> StructArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<StructArray>(
@@ -427,17 +325,6 @@ std::shared_ptr<Array> UnionArray::child(int pos) const {
   return children_[pos];
 }
 
-Status UnionArray::Validate() const {
-  if (length_ < 0) { return Status::Invalid("Length was negative"); }
-
-  if (null_count() > length_) {
-    return Status::Invalid("Null count exceeds the length of this struct");
-  }
-
-  DCHECK(false) << "Validate not yet implemented";
-  return Status::OK();
-}
-
 std::shared_ptr<Array> UnionArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(offset_, length_, &offset, &length);
   return std::make_shared<UnionArray>(type_, length, children_, type_ids_, value_offsets_,
@@ -456,14 +343,6 @@ DictionaryArray::DictionaryArray(
   DCHECK_EQ(type->id(), Type::DICTIONARY);
 }
 
-Status DictionaryArray::Validate() const {
-  Type::type index_type_id = indices_->type()->id();
-  if (!is_integer(index_type_id)) {
-    return Status::Invalid("Dictionary indices must be integer type");
-  }
-  return Status::OK();
-}
-
 std::shared_ptr<Array> DictionaryArray::dictionary() const {
   return dict_type_->dictionary();
 }
@@ -476,20 +355,136 @@ std::shared_ptr<Array> DictionaryArray::Slice(int64_t offset, int64_t length) co
 // ----------------------------------------------------------------------
 // Implement Array::Accept as inline visitor
 
-struct AcceptVirtualVisitor {
-  explicit AcceptVirtualVisitor(ArrayVisitor* visitor) : visitor(visitor) {}
+Status Array::Accept(ArrayVisitor* visitor) const {
+  return VisitArrayInline(*this, visitor);
+}
+
+// ----------------------------------------------------------------------
+// Implement Array::Validate as inline visitor
+
+struct ValidateVisitor {
+  Status Visit(const NullArray& array) { return Status::OK(); }
+
+  Status Visit(const PrimitiveArray& array) { return Status::OK(); }
+
+  Status Visit(const BinaryArray& array) {
+    // TODO(wesm): what to do here?
+    return Status::OK();
+  }
+
+  Status Visit(const ListArray& array) {
+    if (array.length() < 0) { return Status::Invalid("Length was negative"); }
+
+    auto value_offsets = array.value_offsets();
+    if (array.length() && !value_offsets) {
+      return Status::Invalid("value_offsets_ was null");
+    }
+    if (value_offsets->size() / static_cast<int>(sizeof(int32_t)) < array.length()) {
+      std::stringstream ss;
+      ss << "offset buffer size (bytes): " << value_offsets->size()
+         << " isn't large enough for length: " << array.length();
+      return Status::Invalid(ss.str());
+    }
+    const int32_t last_offset = array.value_offset(array.length());
+    if (last_offset > 0) {
+      if (!array.values()) {
+        return Status::Invalid("last offset was non-zero and values was null");
+      }
+      if (array.values()->length() != last_offset) {
+        std::stringstream ss;
+        ss << "Final offset invariant not equal to values length: " << last_offset
+           << "!=" << array.values()->length();
+        return Status::Invalid(ss.str());
+      }
+
+      const Status child_valid = ValidateArray(*array.values());
+      if (!child_valid.ok()) {
+        std::stringstream ss;
+        ss << "Child array invalid: " << child_valid.ToString();
+        return Status::Invalid(ss.str());
+      }
+    }
+
+    int32_t prev_offset = array.value_offset(0);
+    if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
+    for (int64_t i = 1; i <= array.length(); ++i) {
+      int32_t current_offset = array.value_offset(i);
+      if (array.IsNull(i - 1) && current_offset != prev_offset) {
+        std::stringstream ss;
+        ss << "Offset invariant failure at: " << i
+           << " inconsistent value_offsets for null slot" << current_offset
+           << "!=" << prev_offset;
+        return Status::Invalid(ss.str());
+      }
+      if (current_offset < prev_offset) {
+        std::stringstream ss;
+        ss << "Offset invariant failure: " << i
+           << " inconsistent offset for non-null slot: " << current_offset << "<"
+           << prev_offset;
+        return Status::Invalid(ss.str());
+      }
+      prev_offset = current_offset;
+    }
+    return Status::OK();
+  }
 
-  ArrayVisitor* visitor;
+  Status Visit(const StructArray& array) {
+    if (array.length() < 0) { return Status::Invalid("Length was negative"); }
 
-  template <typename T>
-  Status Visit(const T& array) {
-    return visitor->Visit(array);
+    if (array.null_count() > array.length()) {
+      return Status::Invalid("Null count exceeds the length of this struct");
+    }
+
+    if (array.fields().size() > 0) {
+      // Validate fields
+      int64_t array_length = array.fields()[0]->length();
+      size_t idx = 0;
+      for (auto it : array.fields()) {
+        if (it->length() != array_length) {
+          std::stringstream ss;
+          ss << "Length is not equal from field " << it->type()->ToString()
+             << " at position {" << idx << "}";
+          return Status::Invalid(ss.str());
+        }
+
+        const Status child_valid = ValidateArray(*it);
+        if (!child_valid.ok()) {
+          std::stringstream ss;
+          ss << "Child array invalid: " << child_valid.ToString() << " at position {"
+             << idx << "}";
+          return Status::Invalid(ss.str());
+        }
+        ++idx;
+      }
+
+      if (array_length > 0 && array_length != array.length()) {
+        return Status::Invalid("Struct's length is not equal to its child arrays");
+      }
+    }
+    return Status::OK();
+  }
+
+  Status Visit(const UnionArray& array) {
+    if (array.length() < 0) { return Status::Invalid("Length was negative"); }
+
+    if (array.null_count() > array.length()) {
+      return Status::Invalid("Null count exceeds the length of this struct");
+    }
+    return Status::OK();
+  }
+
+  Status Visit(const DictionaryArray& array) {
+    Type::type index_type_id = array.indices()->type()->id();
+    if (!is_integer(index_type_id)) {
+      return Status::Invalid("Dictionary indices must be integer type");
+    }
+    return Status::OK();
   }
 };
 
-Status Array::Accept(ArrayVisitor* visitor) const {
-  AcceptVirtualVisitor inline_visitor(visitor);
-  return VisitArrayInline(*this, visitor);
+Status ValidateArray(const Array& array) {
+  ValidateVisitor validate_visitor;
+  return VisitArrayInline(array, &validate_visitor);
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 071d4e30f52dd..331c6c3e7fab8 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -108,11 +108,6 @@ class ARROW_EXPORT Array {
   bool RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
       int64_t other_start_idx) const;
 
-  /// Determines if the array is internally consistent.
-  ///
-  /// Defaults to always returning Status::OK. This can be an expensive check.
-  virtual Status Validate() const;
-
   Status Accept(ArrayVisitor* visitor) const;
 
   /// Construct a zero-copy slice of the array with the indicated offset and
@@ -238,8 +233,6 @@ class ARROW_EXPORT ListArray : public Array {
     values_ = values;
   }
 
-  Status Validate() const override;
-
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   std::shared_ptr<Array> values() const { return values_; }
@@ -306,8 +299,6 @@ class ARROW_EXPORT BinaryArray : public Array {
     return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
   }
 
-  Status Validate() const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
@@ -342,8 +333,6 @@ class ARROW_EXPORT StringArray : public BinaryArray {
     return std::string(reinterpret_cast<const char*>(str), nchars);
   }
 
-  Status Validate() const override;
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
@@ -406,8 +395,6 @@ class ARROW_EXPORT StructArray : public Array {
       std::shared_ptr<Buffer> null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0);
 
-  Status Validate() const override;
-
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   std::shared_ptr<Array> field(int pos) const;
@@ -436,8 +423,6 @@ class ARROW_EXPORT UnionArray : public Array {
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0);
 
-  Status Validate() const override;
-
   /// Note that this buffer does not account for any slice offset
   std::shared_ptr<Buffer> type_ids() const { return type_ids_; }
 
@@ -490,8 +475,6 @@ class ARROW_EXPORT DictionaryArray : public Array {
   DictionaryArray(
       const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices);
 
-  Status Validate() const override;
-
   std::shared_ptr<Array> indices() const { return indices_; }
   std::shared_ptr<Array> dictionary() const;
 
@@ -525,6 +508,16 @@ ARROW_EXTERN_TEMPLATE NumericArray<Time32Type>;
 ARROW_EXTERN_TEMPLATE NumericArray<Time64Type>;
 ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
 
+
+/// \brief Perform any validation checks to determine obvious inconsistencies
+/// with the array's internal data
+///
+/// This can be an expensive check.
+///
+/// \param array an Array instance
+/// \return Status
+Status ValidateArray(const Array& array);
+
 }  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index c8ca21cb8f17d..5caa3a9808c61 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -133,7 +133,7 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
   ListBuilder builder(pool, child_array);
   RETURN_NOT_OK(builder.Append(offsets.data(), num_lists, valid_lists.data()));
   RETURN_NOT_OK(builder.Finish(out));
-  return (*out)->Validate();
+  return ValidateArray(**out);
 }
 
 typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);

From 47e289a9a6d38fe9b0581dbc73a464cae084ef75 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 15 May 2017 21:38:15 -0400
Subject: [PATCH 0659/1644] ARROW-961: [Python] Rename InMemoryOutputStream to
 BufferOutputStream

Better API naming consistency with C++

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #690 from wesm/ARROW-961 and squashes the following commits:

85b352c [Wes McKinney] Add more graceful deprecation warnings for renamed classes, test suite
69a99cd [Wes McKinney] Fix Cython compilation
a15910a [Wes McKinney] Rename InMemoryOutputStream to BufferOutputStream
---
 python/doc/source/api.rst                 |  2 +-
 python/doc/source/ipc.rst                 |  4 +-
 python/doc/source/memory.rst              | 12 ++---
 python/pyarrow/__init__.py                | 37 ++++++++++++-
 python/pyarrow/includes/libarrow.pxd      |  5 +-
 python/pyarrow/io.pxi                     |  4 +-
 python/pyarrow/tests/test_deprecations.py | 64 +++++++++++++++++++++++
 python/pyarrow/tests/test_io.py           |  6 +--
 python/pyarrow/tests/test_ipc.py          |  4 +-
 python/pyarrow/tests/test_parquet.py      |  2 +-
 10 files changed, 120 insertions(+), 20 deletions(-)
 create mode 100644 python/pyarrow/tests/test_deprecations.py

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index e7bea7013b9a2..c1459358b80d3 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -162,7 +162,7 @@ Input / Output and Shared Memory
 
    Buffer
    BufferReader
-   InMemoryOutputStream
+   BufferOutputStream
    NativeFile
    MemoryMappedFile
    memory_map
diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index c7de9c466386d..f0844cd2a9cb5 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -60,7 +60,7 @@ this we use :class:`~pyarrow.RecordBatchStreamWriter`, which can write to a writ
 
 .. ipython:: python
 
-   sink = pa.InMemoryOutputStream()
+   sink = pa.BufferOutputStream()
    writer = pa.RecordBatchStreamWriter(sink, batch.schema)
 
 Here we used an in-memory Arrow buffer stream, but this could have been a
@@ -109,7 +109,7 @@ The :class:`~pyarrow.RecordBatchFileWriter` has the same API as
 
 .. ipython:: python
 
-   sink = pa.InMemoryOutputStream()
+   sink = pa.BufferOutputStream()
    writer = pa.RecordBatchFileWriter(sink, batch.schema)
 
    for i in range(10):
diff --git a/python/doc/source/memory.rst b/python/doc/source/memory.rst
index d1020da246407..ccc6298b66154 100644
--- a/python/doc/source/memory.rst
+++ b/python/doc/source/memory.rst
@@ -104,8 +104,8 @@ There are several kinds of :class:`~pyarrow.NativeFile` options available:
   memory maps
 * :class:`~pyarrow.BufferReader`, for reading :class:`~pyarrow.Buffer` objects
   as a file
-* :class:`~pyarrow.InMemoryOutputStream`, for writing data in-memory, producing
-  a Buffer at the end
+* :class:`~pyarrow.BufferOutputStream`, for writing data in-memory, producing a
+  Buffer at the end
 * :class:`~pyarrow.HdfsFile`, for reading and writing data to the Hadoop Filesystem
 * :class:`~pyarrow.PythonFile`, for interfacing with Python file objects in C++
 
@@ -124,11 +124,11 @@ then precisely track amount of memory that has been allocated:
 
 PyArrow uses a default built-in memory pool, but in the future there may be
 additional memory pools (and subpools) to choose from. Let's consider an
-``InMemoryOutputStream``, which is like a ``BytesIO``:
+``BufferOutputStream``, which is like a ``BytesIO``:
 
 .. ipython:: python
 
-   stream = pa.InMemoryOutputStream()
+   stream = pa.BufferOutputStream()
    stream.write(b'foo')
    pa.total_allocated_bytes()
    for i in range(1024): stream.write(b'foo')
@@ -150,7 +150,7 @@ pass in a custom memory pool:
    my_pool = pa.jemalloc_memory_pool()
    my_pool
    my_pool.bytes_allocated()
-   stream = pa.InMemoryOutputStream(my_pool)
+   stream = pa.BufferOutputStream(my_pool)
    stream.write(b'foo')
    my_pool.bytes_allocated()
 
@@ -215,7 +215,7 @@ file interfaces that can read and write to Arrow Buffers.
 
 .. ipython:: python
 
-   writer = pa.InMemoryOutputStream()
+   writer = pa.BufferOutputStream()
    writer.write(b'hello, friends')
 
    buf = writer.get_result()
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index d6d2aa4a671e7..632a443ed0deb 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -70,7 +70,7 @@
                          Date32Value, Date64Value, TimestampValue)
 
 from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
-                         Buffer, BufferReader, InMemoryOutputStream,
+                         Buffer, BufferReader, BufferOutputStream,
                          OSFile, MemoryMappedFile, memory_map,
                          frombuffer, read_tensor, write_tensor,
                          memory_map, create_memory_map,
@@ -108,3 +108,38 @@ def jemalloc_memory_pool():
 
 
 localfs = LocalFilesystem.get_instance()
+
+
+# ----------------------------------------------------------------------
+# 0.4.0 deprecations
+
+import warnings
+
+def _deprecate_class(old_name, new_name, klass, next_version='0.5.0'):
+    msg = ('pyarrow.{0} has been renamed to '
+           '{1}, will be removed in {2}'
+           .format(old_name, new_name, next_version))
+    def deprecated_factory(*args, **kwargs):
+        warnings.warn(msg, FutureWarning)
+        return klass(*args)
+    return deprecated_factory
+
+FileReader = _deprecate_class('FileReader',
+                              'RecordBatchFileReader',
+                              RecordBatchFileReader, '0.5.0')
+
+FileWriter = _deprecate_class('FileWriter',
+                              'RecordBatchFileWriter',
+                              RecordBatchFileWriter, '0.5.0')
+
+StreamReader = _deprecate_class('StreamReader',
+                                'RecordBatchStreamReader',
+                                RecordBatchStreamReader, '0.5.0')
+
+StreamWriter = _deprecate_class('StreamWriter',
+                                'RecordBatchStreamWriter',
+                                RecordBatchStreamWriter, '0.5.0')
+
+InMemoryOutputStream = _deprecate_class('InMemoryOutputStream',
+                                        'BufferOutputStream',
+                                        BufferOutputStream, '0.5.0')
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index b03dd59dffcaf..a7e2733ca815c 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -507,8 +507,9 @@ cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
         CBufferReader(const shared_ptr[CBuffer]& buffer)
         CBufferReader(const uint8_t* data, int64_t nbytes)
 
-    cdef cppclass BufferOutputStream(OutputStream):
-        BufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
+    cdef cppclass CBufferOutputStream" arrow::io::BufferOutputStream"\
+        (OutputStream):
+        CBufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
 
 
 cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 4cbf603c2a5df..a153f22270087 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -515,14 +515,14 @@ cdef shared_ptr[PoolBuffer] allocate_buffer(CMemoryPool* pool):
     return result
 
 
-cdef class InMemoryOutputStream(NativeFile):
+cdef class BufferOutputStream(NativeFile):
 
     cdef:
         shared_ptr[PoolBuffer] buffer
 
     def __cinit__(self, MemoryPool memory_pool=None):
         self.buffer = allocate_buffer(maybe_unbox_memory_pool(memory_pool))
-        self.wr_file.reset(new BufferOutputStream(
+        self.wr_file.reset(new CBufferOutputStream(
             <shared_ptr[ResizableBuffer]> self.buffer))
         self.is_readable = 0
         self.is_writeable = 1
diff --git a/python/pyarrow/tests/test_deprecations.py b/python/pyarrow/tests/test_deprecations.py
new file mode 100644
index 0000000000000..62b96663833a7
--- /dev/null
+++ b/python/pyarrow/tests/test_deprecations.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Check that various deprecation warnings are raised
+
+import pyarrow as pa
+import pytest
+
+
+def test_inmemory_output_stream():
+    with pytest.warns(FutureWarning):
+        stream = pa.InMemoryOutputStream()
+        assert isinstance(stream, pa.BufferOutputStream)
+
+
+def test_file_reader_writer():
+    data = [
+        pa.array([1, 2, 3, 4]),
+        pa.array(['foo', 'bar', 'baz', None]),
+        pa.array([True, None, False, True])
+    ]
+    batch = pa.RecordBatch.from_arrays(data, ['f0', 'f1', 'f2'])
+
+    sink = pa.BufferOutputStream()
+
+    with pytest.warns(FutureWarning):
+        stream_writer = pa.StreamWriter(sink, batch.schema)
+        assert isinstance(stream_writer, pa.RecordBatchStreamWriter)
+
+    sink2 = pa.BufferOutputStream()
+    with pytest.warns(FutureWarning):
+        file_writer = pa.FileWriter(sink2, batch.schema)
+        assert isinstance(file_writer, pa.RecordBatchFileWriter)
+
+    file_writer.write_batch(batch)
+    stream_writer.write_batch(batch)
+
+    file_writer.close()
+    stream_writer.close()
+
+    buf = sink.get_result()
+    buf2 = sink2.get_result()
+
+    with pytest.warns(FutureWarning):
+        stream_reader = pa.StreamReader(buf)
+        assert isinstance(stream_reader, pa.RecordBatchStreamReader)
+
+    with pytest.warns(FutureWarning):
+        file_reader = pa.FileReader(buf2)
+        assert isinstance(file_reader, pa.RecordBatchFileReader)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 610dedc6a7640..f91dc6fa05d57 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -179,7 +179,7 @@ def test_memory_output_stream():
     # 10 bytes
     val = b'dataabcdef'
 
-    f = pa.InMemoryOutputStream()
+    f = pa.BufferOutputStream()
 
     K = 1000
     for i in range(K):
@@ -192,7 +192,7 @@ def test_memory_output_stream():
 
 
 def test_inmemory_write_after_closed():
-    f = pa.InMemoryOutputStream()
+    f = pa.BufferOutputStream()
     f.write(b'ok')
     f.get_result()
 
@@ -212,7 +212,7 @@ def make_buffer(bytes_obj):
 
 
 def test_nativefile_write_memoryview():
-    f = pa.InMemoryOutputStream()
+    f = pa.BufferOutputStream()
     data = b'ok'
 
     arr = np.frombuffer(data, dtype='S1')
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 4d19804dac2a4..994876de3c14a 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -209,7 +209,7 @@ def test_read_all(self):
 class TestInMemoryFile(TestFile):
 
     def _get_sink(self):
-        return pa.InMemoryOutputStream()
+        return pa.BufferOutputStream()
 
     def _get_source(self):
         return self.sink.get_result()
@@ -219,7 +219,7 @@ def test_ipc_zero_copy_numpy():
     df = pd.DataFrame({'foo': [1.5]})
 
     batch = pa.RecordBatch.from_pandas(df)
-    sink = pa.InMemoryOutputStream()
+    sink = pa.BufferOutputStream()
     write_file(batch, sink)
     buffer = sink.get_result()
     reader = pa.BufferReader(buffer)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 7144de2a6852a..5dbe657475623 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -179,7 +179,7 @@ def _test_dataframe(size=10000, seed=0):
 def test_pandas_parquet_native_file_roundtrip(tmpdir):
     df = _test_dataframe(10000)
     arrow_table = pa.Table.from_pandas(df)
-    imos = pa.InMemoryOutputStream()
+    imos = pa.BufferOutputStream()
     pq.write_table(arrow_table, imos, version="2.0")
     buf = imos.get_result()
     reader = pa.BufferReader(buf)

From ce0bb5338496785e6c46d9832b75105883bef5de Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 15 May 2017 21:40:36 -0400
Subject: [PATCH 0660/1644] ARROW-1002: [C++] Fix inconsistency with padding at
 start of IPC file format

cc @TheNeuralBit -- the 64-byte padding in the C++ file writer was incorrect (http://arrow.apache.org/docs/ipc.html indicate padding to an 8-byte boundary), so this fixes that.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #693 from wesm/ARROW-1002 and squashes the following commits:

35c023f [Wes McKinney] Fix C++ inconsistency with padding at start of IPC file format
---
 cpp/src/arrow/ipc/writer.cc | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index ced071020212a..4f5edf2433edc 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -509,7 +509,7 @@ class DictionaryWriter : public RecordBatchSerializer {
 };
 
 // Adds padding bytes if necessary to ensure all memory blocks are written on
-// 8-byte boundaries.
+// 64-byte boundaries.
 Status AlignStreamPosition(io::OutputStream* stream) {
   int64_t position;
   RETURN_NOT_OK(stream->Tell(&position));
@@ -687,9 +687,9 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
   }
 
   // Adds padding bytes if necessary to ensure all memory blocks are written on
-  // 8-byte boundaries.
-  Status Align() {
-    int64_t remainder = PaddedLength(position_) - position_;
+  // 64-byte (or other alignment) boundaries.
+  Status Align(int64_t alignment = kArrowAlignment) {
+    int64_t remainder = PaddedLength(position_, alignment) - position_;
     if (remainder > 0) { return Write(kPaddingBytes, remainder); }
     return Status::OK();
   }
@@ -701,12 +701,6 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
     return Status::OK();
   }
 
-  // Write and align
-  Status WriteAligned(const uint8_t* data, int64_t nbytes) {
-    RETURN_NOT_OK(Write(data, nbytes));
-    return Align();
-  }
-
   void set_memory_pool(MemoryPool* pool) { pool_ = pool; }
 
  protected:
@@ -762,8 +756,10 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
   using BASE = RecordBatchStreamWriter::RecordBatchStreamWriterImpl;
 
   Status Start() override {
-    RETURN_NOT_OK(WriteAligned(
-        reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
+    // It is only necessary to align to 8-byte boundary at the start of the file
+    RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
+            strlen(kArrowMagicBytes)));
+    RETURN_NOT_OK(Align(8));
 
     // We write the schema at the start of the file (and the end). This also
     // writes all the dictionaries at the beginning of the file

From 8a8e7bb0da3e92fea17cf174f7193b6a44046168 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 16 May 2017 10:00:26 -0400
Subject: [PATCH 0661/1644] ARROW-1037: [GLib] Follow reader name change

Author: Kouhei Sutou <kou@clear-code.com>

Closes #695 from kou/glib-follow-reader-name-change and squashes the following commits:

2b4e0bc [Kouhei Sutou] [GLib] Follow reader name change
---
 c_glib/arrow-glib/Makefile.am                 |   9 +-
 c_glib/arrow-glib/arrow-glib.h                |   3 +-
 c_glib/arrow-glib/arrow-glib.hpp              |   6 +-
 c_glib/arrow-glib/file-reader.cpp             | 249 ---------
 c_glib/arrow-glib/file-reader.h               |  83 ---
 c_glib/arrow-glib/file-reader.hpp             |  28 --
 c_glib/arrow-glib/reader.cpp                  | 472 ++++++++++++++++++
 c_glib/arrow-glib/reader.h                    | 196 ++++++++
 .../{stream-reader.hpp => reader.hpp}         |  11 +-
 c_glib/arrow-glib/stream-reader.cpp           | 221 --------
 c_glib/arrow-glib/stream-reader.h             |  80 ---
 c_glib/doc/reference/arrow-glib-docs.sgml     |   3 +-
 c_glib/example/read-batch.c                   |  11 +-
 c_glib/example/read-stream.c                  |  13 +-
 c_glib/test/test-file-writer.rb               |   2 +-
 c_glib/test/test-stream-writer.rb             |   2 +-
 16 files changed, 699 insertions(+), 690 deletions(-)
 delete mode 100644 c_glib/arrow-glib/file-reader.cpp
 delete mode 100644 c_glib/arrow-glib/file-reader.h
 delete mode 100644 c_glib/arrow-glib/file-reader.hpp
 create mode 100644 c_glib/arrow-glib/reader.cpp
 create mode 100644 c_glib/arrow-glib/reader.h
 rename c_glib/arrow-glib/{stream-reader.hpp => reader.hpp} (56%)
 delete mode 100644 c_glib/arrow-glib/stream-reader.cpp
 delete mode 100644 c_glib/arrow-glib/stream-reader.h

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 0f83f569230c9..f0bb6011b65c7 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -66,9 +66,8 @@ libarrow_glib_la_headers +=			\
 	writeable-file.h
 
 libarrow_glib_la_headers +=			\
-	file-reader.h				\
-	stream-reader.h				\
 	metadata-version.h			\
+	reader.h				\
 	writer.h
 
 libarrow_glib_la_generated_headers =		\
@@ -105,9 +104,8 @@ libarrow_glib_la_sources +=			\
 	writeable-file.cpp
 
 libarrow_glib_la_sources +=			\
-	file-reader.cpp				\
 	metadata-version.cpp			\
-	stream-reader.cpp			\
+	reader.cpp				\
 	writer.cpp
 
 libarrow_glib_la_cpp_headers =			\
@@ -136,9 +134,8 @@ libarrow_glib_la_cpp_headers +=			\
 	writeable-file.hpp
 
 libarrow_glib_la_cpp_headers +=			\
-	file-reader.hpp				\
 	metadata-version.hpp			\
-	stream-reader.hpp			\
+	reader.hpp				\
 	writer.hpp
 
 libarrow_glib_la_SOURCES =			\
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index f98899c5666cc..7ab95317210e7 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -41,7 +41,6 @@
 #include <arrow-glib/writeable.h>
 #include <arrow-glib/writeable-file.h>
 
-#include <arrow-glib/file-reader.h>
 #include <arrow-glib/metadata-version.h>
-#include <arrow-glib/stream-reader.h>
+#include <arrow-glib/reader.h>
 #include <arrow-glib/writer.h>
diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
index 3184761d4e148..ae550702bed92 100644
--- a/c_glib/arrow-glib/arrow-glib.hpp
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -43,8 +43,6 @@
 #include <arrow-glib/writeable.hpp>
 #include <arrow-glib/writeable-file.hpp>
 
-#include <arrow-glib/file-reader.hpp>
-#include <arrow-glib/file-writer.hpp>
 #include <arrow-glib/metadata-version.hpp>
-#include <arrow-glib/stream-reader.hpp>
-#include <arrow-glib/stream-writer.hpp>
+#include <arrow-glib/reader.hpp>
+#include <arrow-glib/writer.hpp>
diff --git a/c_glib/arrow-glib/file-reader.cpp b/c_glib/arrow-glib/file-reader.cpp
deleted file mode 100644
index c16bf194821cd..0000000000000
--- a/c_glib/arrow-glib/file-reader.cpp
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow/ipc/api.h>
-
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/record-batch.hpp>
-#include <arrow-glib/schema.hpp>
-
-#include <arrow-glib/input-stream.hpp>
-
-#include <arrow-glib/file-reader.hpp>
-#include <arrow-glib/metadata-version.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: file-reader
- * @short_description: File reader class
- *
- * #GArrowFileReader is a class for receiving data by file based IPC.
- */
-
-typedef struct GArrowFileReaderPrivate_ {
-  std::shared_ptr<arrow::ipc::FileReader> file_reader;
-} GArrowFileReaderPrivate;
-
-enum {
-  PROP_0,
-  PROP_FILE_READER
-};
-
-G_DEFINE_TYPE_WITH_PRIVATE(GArrowFileReader,
-                           garrow_file_reader,
-                           G_TYPE_OBJECT);
-
-#define GARROW_FILE_READER_GET_PRIVATE(obj)                         \
-  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
-                               GARROW_TYPE_FILE_READER,             \
-                               GArrowFileReaderPrivate))
-
-static void
-garrow_file_reader_finalize(GObject *object)
-{
-  GArrowFileReaderPrivate *priv;
-
-  priv = GARROW_FILE_READER_GET_PRIVATE(object);
-
-  priv->file_reader = nullptr;
-
-  G_OBJECT_CLASS(garrow_file_reader_parent_class)->finalize(object);
-}
-
-static void
-garrow_file_reader_set_property(GObject *object,
-                                    guint prop_id,
-                                    const GValue *value,
-                                    GParamSpec *pspec)
-{
-  GArrowFileReaderPrivate *priv;
-
-  priv = GARROW_FILE_READER_GET_PRIVATE(object);
-
-  switch (prop_id) {
-  case PROP_FILE_READER:
-    priv->file_reader =
-      *static_cast<std::shared_ptr<arrow::ipc::FileReader> *>(g_value_get_pointer(value));
-    break;
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_file_reader_get_property(GObject *object,
-                                          guint prop_id,
-                                          GValue *value,
-                                          GParamSpec *pspec)
-{
-  switch (prop_id) {
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_file_reader_init(GArrowFileReader *object)
-{
-}
-
-static void
-garrow_file_reader_class_init(GArrowFileReaderClass *klass)
-{
-  GObjectClass *gobject_class;
-  GParamSpec *spec;
-
-  gobject_class = G_OBJECT_CLASS(klass);
-
-  gobject_class->finalize     = garrow_file_reader_finalize;
-  gobject_class->set_property = garrow_file_reader_set_property;
-  gobject_class->get_property = garrow_file_reader_get_property;
-
-  spec = g_param_spec_pointer("file-reader",
-                              "ipc::FileReader",
-                              "The raw std::shared<arrow::ipc::FileReader> *",
-                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
-                                                       G_PARAM_CONSTRUCT_ONLY));
-  g_object_class_install_property(gobject_class, PROP_FILE_READER, spec);
-}
-
-/**
- * garrow_file_reader_new:
- * @input_stream: The seekable input stream to read data.
- * @error: (nullable): Return locatipcn for a #GError or %NULL.
- *
- * Returns: (nullable): A newly created #GArrowFileReader or %NULL on
- *   error.
- */
-GArrowFileReader *
-garrow_file_reader_new(GArrowSeekableInputStream *input_stream,
-                       GError **error)
-{
-  auto arrow_random_access_file =
-    garrow_seekable_input_stream_get_raw(input_stream);
-  std::shared_ptr<arrow::ipc::FileReader> arrow_file_reader;
-  auto status =
-    arrow::ipc::FileReader::Open(arrow_random_access_file,
-                                 &arrow_file_reader);
-  if (garrow_error_check(error, status, "[ipc][file-reader][open]")) {
-    return garrow_file_reader_new_raw(&arrow_file_reader);
-  } else {
-    return NULL;
-  }
-}
-
-/**
- * garrow_file_reader_get_schema:
- * @file_reader: A #GArrowFileReader.
- *
- * Returns: (transfer full): The schema in the file.
- */
-GArrowSchema *
-garrow_file_reader_get_schema(GArrowFileReader *file_reader)
-{
-  auto arrow_file_reader =
-    garrow_file_reader_get_raw(file_reader);
-  auto arrow_schema = arrow_file_reader->schema();
-  return garrow_schema_new_raw(&arrow_schema);
-}
-
-/**
- * garrow_file_reader_get_n_record_batches:
- * @file_reader: A #GArrowFileReader.
- *
- * Returns: The number of record batches in the file.
- */
-guint
-garrow_file_reader_get_n_record_batches(GArrowFileReader *file_reader)
-{
-  auto arrow_file_reader =
-    garrow_file_reader_get_raw(file_reader);
-  return arrow_file_reader->num_record_batches();
-}
-
-/**
- * garrow_file_reader_get_version:
- * @file_reader: A #GArrowFileReader.
- *
- * Returns: The format version in the file.
- */
-GArrowMetadataVersion
-garrow_file_reader_get_version(GArrowFileReader *file_reader)
-{
-  auto arrow_file_reader =
-    garrow_file_reader_get_raw(file_reader);
-  auto arrow_version = arrow_file_reader->version();
-  return garrow_metadata_version_from_raw(arrow_version);
-}
-
-/**
- * garrow_file_reader_get_record_batch:
- * @file_reader: A #GArrowFileReader.
- * @i: The index of the target record batch.
- * @error: (nullable): Return locatipcn for a #GError or %NULL.
- *
- * Returns: (nullable) (transfer full):
- *   The i-th record batch in the file or %NULL on error.
- */
-GArrowRecordBatch *
-garrow_file_reader_get_record_batch(GArrowFileReader *file_reader,
-                                        guint i,
-                                        GError **error)
-{
-  auto arrow_file_reader =
-    garrow_file_reader_get_raw(file_reader);
-  std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
-  auto status = arrow_file_reader->GetRecordBatch(i, &arrow_record_batch);
-
-  if (garrow_error_check(error,
-                         status,
-                         "[ipc][file-reader][get-record-batch]")) {
-    return garrow_record_batch_new_raw(&arrow_record_batch);
-  } else {
-    return NULL;
-  }
-}
-
-G_END_DECLS
-
-GArrowFileReader *
-garrow_file_reader_new_raw(std::shared_ptr<arrow::ipc::FileReader> *arrow_file_reader)
-{
-  auto file_reader =
-    GARROW_FILE_READER(g_object_new(GARROW_TYPE_FILE_READER,
-                                        "file-reader", arrow_file_reader,
-                                        NULL));
-  return file_reader;
-}
-
-std::shared_ptr<arrow::ipc::FileReader>
-garrow_file_reader_get_raw(GArrowFileReader *file_reader)
-{
-  GArrowFileReaderPrivate *priv;
-
-  priv = GARROW_FILE_READER_GET_PRIVATE(file_reader);
-  return priv->file_reader;
-}
diff --git a/c_glib/arrow-glib/file-reader.h b/c_glib/arrow-glib/file-reader.h
deleted file mode 100644
index 551e05a3d1413..0000000000000
--- a/c_glib/arrow-glib/file-reader.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/record-batch.h>
-#include <arrow-glib/schema.h>
-
-#include <arrow-glib/input-stream.h>
-
-#include <arrow-glib/metadata-version.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_FILE_READER      \
-  (garrow_file_reader_get_type())
-#define GARROW_FILE_READER(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_FILE_READER,      \
-                              GArrowFileReader))
-#define GARROW_FILE_READER_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_FILE_READER, \
-                           GArrowFileReaderClass))
-#define GARROW_IS_FILE_READER(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_FILE_READER))
-#define GARROW_IS_FILE_READER_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_FILE_READER))
-#define GARROW_FILE_READER_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_FILE_READER,       \
-                             GArrowFileReaderClass))
-
-typedef struct _GArrowFileReader         GArrowFileReader;
-typedef struct _GArrowFileReaderClass    GArrowFileReaderClass;
-
-/**
- * GArrowFileReader:
- *
- * It wraps `arrow::ipc::FileReader`.
- */
-struct _GArrowFileReader
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowFileReaderClass
-{
-  GObjectClass parent_class;
-};
-
-GType garrow_file_reader_get_type(void) G_GNUC_CONST;
-
-GArrowFileReader *garrow_file_reader_new(GArrowSeekableInputStream *input_stream,
-                                         GError **error);
-
-GArrowSchema *garrow_file_reader_get_schema(GArrowFileReader *file_reader);
-guint garrow_file_reader_get_n_record_batches(GArrowFileReader *file_reader);
-GArrowMetadataVersion garrow_file_reader_get_version(GArrowFileReader *file_reader);
-GArrowRecordBatch *garrow_file_reader_get_record_batch(GArrowFileReader *file_reader,
-                                                           guint i,
-                                                           GError **error);
-
-G_END_DECLS
diff --git a/c_glib/arrow-glib/file-reader.hpp b/c_glib/arrow-glib/file-reader.hpp
deleted file mode 100644
index 152379bbda4ff..0000000000000
--- a/c_glib/arrow-glib/file-reader.hpp
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow/api.h>
-#include <arrow/ipc/api.h>
-
-#include <arrow-glib/file-reader.h>
-
-GArrowFileReader *garrow_file_reader_new_raw(std::shared_ptr<arrow::ipc::FileReader> *arrow_file_reader);
-std::shared_ptr<arrow::ipc::FileReader> garrow_file_reader_get_raw(GArrowFileReader *file_reader);
diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
new file mode 100644
index 0000000000000..c0923242ccb75
--- /dev/null
+++ b/c_glib/arrow-glib/reader.cpp
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow/ipc/api.h>
+
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/record-batch.hpp>
+#include <arrow-glib/schema.hpp>
+
+#include <arrow-glib/input-stream.hpp>
+
+#include <arrow-glib/metadata-version.hpp>
+#include <arrow-glib/reader.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: reader
+ * @section_id: reader-classes
+ * @title: Reader classes
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowRecordBatchReader is a base class for reading record batches
+ * in stream format from input.
+ *
+ * #GArrowRecordBatchStreamReader is a class for reading record
+ * batches in stream format from input synchronously.
+ *
+ * #GArrowRecordBatchFileReader is a class for reading record
+ * batches in file format from input.
+ */
+
+typedef struct GArrowRecordBatchReaderPrivate_ {
+  std::shared_ptr<arrow::ipc::RecordBatchReader> record_batch_reader;
+} GArrowRecordBatchReaderPrivate;
+
+enum {
+  PROP_0,
+  PROP_RECORD_BATCH_READER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowRecordBatchReader,
+                           garrow_record_batch_reader,
+                           G_TYPE_OBJECT);
+
+#define GARROW_RECORD_BATCH_READER_GET_PRIVATE(obj)             \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_TYPE_RECORD_BATCH_READER, \
+                               GArrowRecordBatchReaderPrivate))
+
+static void
+garrow_record_batch_reader_finalize(GObject *object)
+{
+  GArrowRecordBatchReaderPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_READER_GET_PRIVATE(object);
+
+  priv->record_batch_reader = nullptr;
+
+  G_OBJECT_CLASS(garrow_record_batch_reader_parent_class)->finalize(object);
+}
+
+static void
+garrow_record_batch_reader_set_property(GObject *object,
+                                        guint prop_id,
+                                        const GValue *value,
+                                        GParamSpec *pspec)
+{
+  GArrowRecordBatchReaderPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_READER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_RECORD_BATCH_READER:
+    priv->record_batch_reader =
+      *static_cast<std::shared_ptr<arrow::ipc::RecordBatchReader> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_record_batch_reader_get_property(GObject *object,
+                                        guint prop_id,
+                                        GValue *value,
+                                        GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_record_batch_reader_init(GArrowRecordBatchReader *object)
+{
+}
+
+static void
+garrow_record_batch_reader_class_init(GArrowRecordBatchReaderClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_record_batch_reader_finalize;
+  gobject_class->set_property = garrow_record_batch_reader_set_property;
+  gobject_class->get_property = garrow_record_batch_reader_get_property;
+
+  spec = g_param_spec_pointer("record-batch-reader",
+                              "arrow::ipc::RecordBatchReader",
+                              "The raw std::shared<arrow::ipc::RecordBatchRecordBatchReader> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_RECORD_BATCH_READER, spec);
+}
+
+/**
+ * garrow_record_batch_reader_get_schema:
+ * @reader: A #GArrowRecordBatchReader.
+ *
+ * Returns: (transfer full): The schema in the stream.
+ *
+ * Since: 0.4.0
+ */
+GArrowSchema *
+garrow_record_batch_reader_get_schema(GArrowRecordBatchReader *reader)
+{
+  auto arrow_reader = garrow_record_batch_reader_get_raw(reader);
+  auto arrow_schema = arrow_reader->schema();
+  return garrow_schema_new_raw(&arrow_schema);
+}
+
+/**
+ * garrow_record_batch_reader_get_next_record_batch:
+ * @reader: A #GArrowRecordBatchReader.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The next record batch in the stream or %NULL on end of stream.
+ *
+ * Since: 0.4.0
+ */
+GArrowRecordBatch *
+garrow_record_batch_reader_get_next_record_batch(GArrowRecordBatchReader *reader,
+                                                 GError **error)
+{
+  auto arrow_reader = garrow_record_batch_reader_get_raw(reader);
+  std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
+  auto status = arrow_reader->GetNextRecordBatch(&arrow_record_batch);
+
+  if (garrow_error_check(error,
+                         status,
+                         "[record-batch-reader][get-next-record-batch]")) {
+    if (arrow_record_batch == nullptr) {
+      return NULL;
+    } else {
+      return garrow_record_batch_new_raw(&arrow_record_batch);
+    }
+  } else {
+    return NULL;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowRecordBatchStreamReader,
+              garrow_record_batch_stream_reader,
+              GARROW_TYPE_RECORD_BATCH_READER);
+
+static void
+garrow_record_batch_stream_reader_init(GArrowRecordBatchStreamReader *object)
+{
+}
+
+static void
+garrow_record_batch_stream_reader_class_init(GArrowRecordBatchStreamReaderClass *klass)
+{
+}
+
+/**
+ * garrow_record_batch_stream_reader_new:
+ * @stream: The stream to be read.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowRecordBatchStreamReader
+ *   or %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowRecordBatchStreamReader *
+garrow_record_batch_stream_reader_new(GArrowInputStream *stream,
+                                      GError **error)
+{
+  auto arrow_input_stream = garrow_input_stream_get_raw(stream);
+  std::shared_ptr<arrow::ipc::RecordBatchStreamReader> arrow_reader;
+  auto status =
+    arrow::ipc::RecordBatchStreamReader::Open(arrow_input_stream, &arrow_reader);
+  if (garrow_error_check(error, status, "[record-batch-stream-reader][open]")) {
+    return garrow_record_batch_stream_reader_new_raw(&arrow_reader);
+  } else {
+    return NULL;
+  }
+}
+
+
+typedef struct GArrowRecordBatchFileReaderPrivate_ {
+  std::shared_ptr<arrow::ipc::RecordBatchFileReader> record_batch_file_reader;
+} GArrowRecordBatchFileReaderPrivate;
+
+enum {
+  PROP_0_,
+  PROP_RECORD_BATCH_FILE_READER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowRecordBatchFileReader,
+                           garrow_record_batch_file_reader,
+                           G_TYPE_OBJECT);
+
+#define GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(obj)                \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
+                               GARROW_TYPE_RECORD_BATCH_FILE_READER,    \
+                               GArrowRecordBatchFileReaderPrivate))
+
+static void
+garrow_record_batch_file_reader_finalize(GObject *object)
+{
+  GArrowRecordBatchFileReaderPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(object);
+
+  priv->record_batch_file_reader = nullptr;
+
+  G_OBJECT_CLASS(garrow_record_batch_file_reader_parent_class)->finalize(object);
+}
+
+static void
+garrow_record_batch_file_reader_set_property(GObject *object,
+                                             guint prop_id,
+                                             const GValue *value,
+                                             GParamSpec *pspec)
+{
+  GArrowRecordBatchFileReaderPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_RECORD_BATCH_FILE_READER:
+    priv->record_batch_file_reader =
+      *static_cast<std::shared_ptr<arrow::ipc::RecordBatchFileReader> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_record_batch_file_reader_get_property(GObject *object,
+                                             guint prop_id,
+                                             GValue *value,
+                                             GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_record_batch_file_reader_init(GArrowRecordBatchFileReader *object)
+{
+}
+
+static void
+garrow_record_batch_file_reader_class_init(GArrowRecordBatchFileReaderClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_record_batch_file_reader_finalize;
+  gobject_class->set_property = garrow_record_batch_file_reader_set_property;
+  gobject_class->get_property = garrow_record_batch_file_reader_get_property;
+
+  spec = g_param_spec_pointer("record-batch-file-reader",
+                              "arrow::ipc::RecordBatchFileReader",
+                              "The raw std::shared<arrow::ipc::RecordBatchFileReader> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_RECORD_BATCH_FILE_READER, spec);
+}
+
+
+/**
+ * garrow_record_batch_file_reader_new:
+ * @file: The file to be read.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowRecordBatchFileReader
+ *   or %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowRecordBatchFileReader *
+garrow_record_batch_file_reader_new(GArrowSeekableInputStream *file,
+                                    GError **error)
+{
+  auto arrow_random_access_file = garrow_seekable_input_stream_get_raw(file);
+  std::shared_ptr<arrow::ipc::RecordBatchFileReader> arrow_reader;
+  auto status =
+    arrow::ipc::RecordBatchFileReader::Open(arrow_random_access_file,
+                                            &arrow_reader);
+  if (garrow_error_check(error, status, "[record-batch-file-reader][open]")) {
+    return garrow_record_batch_file_reader_new_raw(&arrow_reader);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_record_batch_file_reader_get_schema:
+ * @reader: A #GArrowRecordBatchFileReader.
+ *
+ * Returns: (transfer full): The schema in the file.
+ *
+ * Since: 0.4.0
+ */
+GArrowSchema *
+garrow_record_batch_file_reader_get_schema(GArrowRecordBatchFileReader *reader)
+{
+  auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader);
+  auto arrow_schema = arrow_reader->schema();
+  return garrow_schema_new_raw(&arrow_schema);
+}
+
+/**
+ * garrow_record_batch_file_reader_get_n_record_batches:
+ * @reader: A #GArrowRecordBatchFileReader.
+ *
+ * Returns: The number of record batches in the file.
+ *
+ * Since: 0.4.0
+ */
+guint
+garrow_record_batch_file_reader_get_n_record_batches(GArrowRecordBatchFileReader *reader)
+{
+  auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader);
+  return arrow_reader->num_record_batches();
+}
+
+/**
+ * garrow_record_batch_file_reader_get_version:
+ * @reader: A #GArrowRecordBatchFileReader.
+ *
+ * Returns: The format version in the file.
+ *
+ * Since: 0.4.0
+ */
+GArrowMetadataVersion
+garrow_record_batch_file_reader_get_version(GArrowRecordBatchFileReader *reader)
+{
+  auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader);
+  auto arrow_version = arrow_reader->version();
+  return garrow_metadata_version_from_raw(arrow_version);
+}
+
+/**
+ * garrow_record_batch_file_reader_get_record_batch:
+ * @reader: A #GArrowRecordBatchFileReader.
+ * @i: The index of the target record batch.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The i-th record batch in the file or %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowRecordBatch *
+garrow_record_batch_file_reader_get_record_batch(GArrowRecordBatchFileReader *reader,
+                                                 guint i,
+                                                 GError **error)
+{
+  auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader);
+  std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
+  auto status = arrow_reader->GetRecordBatch(i, &arrow_record_batch);
+
+  if (garrow_error_check(error,
+                         status,
+                         "[record-batch-file-reader][get-record-batch]")) {
+    return garrow_record_batch_new_raw(&arrow_record_batch);
+  } else {
+    return NULL;
+  }
+}
+
+G_END_DECLS
+
+GArrowRecordBatchReader *
+garrow_record_batch_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchReader> *arrow_reader)
+{
+  auto reader =
+    GARROW_RECORD_BATCH_READER(g_object_new(GARROW_TYPE_RECORD_BATCH_READER,
+                                            "record-batch-reader", arrow_reader,
+                                            NULL));
+  return reader;
+}
+
+std::shared_ptr<arrow::ipc::RecordBatchReader>
+garrow_record_batch_reader_get_raw(GArrowRecordBatchReader *reader)
+{
+  GArrowRecordBatchReaderPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_READER_GET_PRIVATE(reader);
+  return priv->record_batch_reader;
+}
+
+GArrowRecordBatchStreamReader *
+garrow_record_batch_stream_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *arrow_reader)
+{
+  auto reader =
+    GARROW_RECORD_BATCH_STREAM_READER(
+      g_object_new(GARROW_TYPE_RECORD_BATCH_STREAM_READER,
+                   "record-batch-reader", arrow_reader,
+                   NULL));
+  return reader;
+}
+
+GArrowRecordBatchFileReader *
+garrow_record_batch_file_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileReader> *arrow_reader)
+{
+  auto reader =
+    GARROW_RECORD_BATCH_FILE_READER(
+      g_object_new(GARROW_TYPE_RECORD_BATCH_FILE_READER,
+                   "record-batch-file-reader", arrow_reader,
+                   NULL));
+  return reader;
+}
+
+std::shared_ptr<arrow::ipc::RecordBatchFileReader>
+garrow_record_batch_file_reader_get_raw(GArrowRecordBatchFileReader *reader)
+{
+  GArrowRecordBatchFileReaderPrivate *priv;
+
+  priv = GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(reader);
+  return priv->record_batch_file_reader;
+}
diff --git a/c_glib/arrow-glib/reader.h b/c_glib/arrow-glib/reader.h
new file mode 100644
index 0000000000000..477204b7ae9d3
--- /dev/null
+++ b/c_glib/arrow-glib/reader.h
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+
+#include <arrow-glib/input-stream.h>
+
+#include <arrow-glib/metadata-version.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_RECORD_BATCH_READER         \
+  (garrow_record_batch_reader_get_type())
+#define GARROW_RECORD_BATCH_READER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_RECORD_BATCH_READER,  \
+                              GArrowRecordBatchReader))
+#define GARROW_RECORD_BATCH_READER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_RECORD_BATCH_READER,     \
+                           GArrowRecordBatchReaderClass))
+#define GARROW_IS_RECORD_BATCH_READER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_RECORD_BATCH_READER))
+#define GARROW_IS_RECORD_BATCH_READER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_RECORD_BATCH_READER))
+#define GARROW_RECORD_BATCH_READER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_RECORD_BATCH_READER,   \
+                             GArrowRecordBatchReaderClass))
+
+typedef struct _GArrowRecordBatchReader      GArrowRecordBatchReader;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowRecordBatchReaderClass GArrowRecordBatchReaderClass;
+#endif
+
+/**
+ * GArrowRecordBatchReader:
+ *
+ * It wraps `arrow::ipc::RecordBatchReader`.
+ */
+struct _GArrowRecordBatchReader
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowRecordBatchReaderClass
+{
+  GObjectClass parent_class;
+};
+#endif
+
+GType garrow_record_batch_reader_get_type(void) G_GNUC_CONST;
+
+GArrowSchema *garrow_record_batch_reader_get_schema(
+  GArrowRecordBatchReader *reader);
+GArrowRecordBatch *garrow_record_batch_reader_get_next_record_batch(
+  GArrowRecordBatchReader *reader,
+  GError **error);
+
+
+#define GARROW_TYPE_RECORD_BATCH_STREAM_READER          \
+  (garrow_record_batch_stream_reader_get_type())
+#define GARROW_RECORD_BATCH_STREAM_READER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_RECORD_BATCH_STREAM_READER,   \
+                              GArrowRecordBatchStreamReader))
+#define GARROW_RECORD_BATCH_STREAM_READER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                                     \
+                           GARROW_TYPE_RECORD_BATCH_STREAM_READER,      \
+                           GArrowRecordBatchStreamReaderClass))
+#define GARROW_IS_RECORD_BATCH_STREAM_READER(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_RECORD_BATCH_STREAM_READER))
+#define GARROW_IS_RECORD_BATCH_STREAM_READER_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                                     \
+                           GARROW_TYPE_RECORD_BATCH_STREAM_READER))
+#define GARROW_RECORD_BATCH_STREAM_READER_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                                     \
+                             GARROW_TYPE_RECORD_BATCH_STREAM_READER,    \
+                             GArrowRecordBatchStreamReaderClass))
+
+typedef struct _GArrowRecordBatchStreamReader      GArrowRecordBatchStreamReader;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowRecordBatchStreamReaderClass GArrowRecordBatchStreamReaderClass;
+#endif
+
+/**
+ * GArrowRecordBatchStreamReader:
+ *
+ * It wraps `arrow::ipc::RecordBatchStreamReader`.
+ */
+struct _GArrowRecordBatchStreamReader
+{
+  /*< private >*/
+  GArrowRecordBatchReader parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowRecordBatchStreamReaderClass
+{
+  GArrowRecordBatchReaderClass parent_class;
+};
+#endif
+
+GType garrow_record_batch_stream_reader_get_type(void) G_GNUC_CONST;
+
+GArrowRecordBatchStreamReader *garrow_record_batch_stream_reader_new(
+  GArrowInputStream *stream,
+  GError **error);
+
+
+#define GARROW_TYPE_RECORD_BATCH_FILE_READER    \
+  (garrow_record_batch_file_reader_get_type())
+#define GARROW_RECORD_BATCH_FILE_READER(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_RECORD_BATCH_FILE_READER,     \
+                              GArrowRecordBatchFileReader))
+#define GARROW_RECORD_BATCH_FILE_READER_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                                     \
+                           GARROW_TYPE_RECORD_BATCH_FILE_READER,        \
+                           GArrowRecordBatchFileReaderClass))
+#define GARROW_IS_RECORD_BATCH_FILE_READER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_RECORD_BATCH_FILE_READER))
+#define GARROW_IS_RECORD_BATCH_FILE_READER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                                     \
+                           GARROW_TYPE_RECORD_BATCH_FILE_READER))
+#define GARROW_RECORD_BATCH_FILE_READER_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                                     \
+                             GARROW_TYPE_RECORD_BATCH_FILE_READER,      \
+                             GArrowRecordBatchFileReaderClass))
+
+typedef struct _GArrowRecordBatchFileReader      GArrowRecordBatchFileReader;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowRecordBatchFileReaderClass GArrowRecordBatchFileReaderClass;
+#endif
+
+/**
+ * GArrowRecordBatchFileReader:
+ *
+ * It wraps `arrow::ipc::RecordBatchFileReader`.
+ */
+struct _GArrowRecordBatchFileReader
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowRecordBatchFileReaderClass
+{
+  GObjectClass parent_class;
+};
+#endif
+
+GType garrow_record_batch_file_reader_get_type(void) G_GNUC_CONST;
+
+GArrowRecordBatchFileReader *garrow_record_batch_file_reader_new(
+  GArrowSeekableInputStream *file,
+  GError **error);
+
+GArrowSchema *garrow_record_batch_file_reader_get_schema(
+  GArrowRecordBatchFileReader *reader);
+guint garrow_record_batch_file_reader_get_n_record_batches(
+  GArrowRecordBatchFileReader *reader);
+GArrowMetadataVersion garrow_record_batch_file_reader_get_version(
+  GArrowRecordBatchFileReader *reader);
+GArrowRecordBatch *garrow_record_batch_file_reader_get_record_batch(
+  GArrowRecordBatchFileReader *reader,
+  guint i,
+  GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/stream-reader.hpp b/c_glib/arrow-glib/reader.hpp
similarity index 56%
rename from c_glib/arrow-glib/stream-reader.hpp
rename to c_glib/arrow-glib/reader.hpp
index 5191b4edb1cdf..7d0b24a65f608 100644
--- a/c_glib/arrow-glib/stream-reader.hpp
+++ b/c_glib/arrow-glib/reader.hpp
@@ -22,7 +22,12 @@
 #include <arrow/api.h>
 #include <arrow/ipc/api.h>
 
-#include <arrow-glib/stream-reader.h>
+#include <arrow-glib/reader.h>
 
-GArrowStreamReader *garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *arrow_stream_reader);
-std::shared_ptr<arrow::ipc::RecordBatchStreamReader> garrow_stream_reader_get_raw(GArrowStreamReader *stream_reader);
+GArrowRecordBatchReader *garrow_record_batch_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchReader> *arrow_reader);
+std::shared_ptr<arrow::ipc::RecordBatchReader> garrow_record_batch_reader_get_raw(GArrowRecordBatchReader *reader);
+
+GArrowRecordBatchStreamReader *garrow_record_batch_stream_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *arrow_reader);
+
+GArrowRecordBatchFileReader *garrow_record_batch_file_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileReader> *arrow_reader);
+std::shared_ptr<arrow::ipc::RecordBatchFileReader> garrow_record_batch_file_reader_get_raw(GArrowRecordBatchFileReader *reader);
diff --git a/c_glib/arrow-glib/stream-reader.cpp b/c_glib/arrow-glib/stream-reader.cpp
deleted file mode 100644
index 19c36c20fbb5d..0000000000000
--- a/c_glib/arrow-glib/stream-reader.cpp
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifdef HAVE_CONFIG_H
-#  include <config.h>
-#endif
-
-#include <arrow/ipc/api.h>
-
-#include <arrow-glib/error.hpp>
-#include <arrow-glib/record-batch.hpp>
-#include <arrow-glib/schema.hpp>
-
-#include <arrow-glib/input-stream.hpp>
-
-#include <arrow-glib/metadata-version.hpp>
-#include <arrow-glib/stream-reader.hpp>
-
-G_BEGIN_DECLS
-
-/**
- * SECTION: stream-reader
- * @short_description: Stream reader class
- *
- * #GArrowStreamReader is a class for receiving data by stream
- * based IPC.
- */
-
-typedef struct GArrowStreamReaderPrivate_ {
-  std::shared_ptr<arrow::ipc::RecordBatchStreamReader> stream_reader;
-} GArrowStreamReaderPrivate;
-
-enum {
-  PROP_0,
-  PROP_STREAM_READER
-};
-
-G_DEFINE_TYPE_WITH_PRIVATE(GArrowStreamReader,
-                           garrow_stream_reader,
-                           G_TYPE_OBJECT);
-
-#define GARROW_STREAM_READER_GET_PRIVATE(obj)               \
-  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
-                               GARROW_TYPE_STREAM_READER,   \
-                               GArrowStreamReaderPrivate))
-
-static void
-garrow_stream_reader_finalize(GObject *object)
-{
-  GArrowStreamReaderPrivate *priv;
-
-  priv = GARROW_STREAM_READER_GET_PRIVATE(object);
-
-  priv->stream_reader = nullptr;
-
-  G_OBJECT_CLASS(garrow_stream_reader_parent_class)->finalize(object);
-}
-
-static void
-garrow_stream_reader_set_property(GObject *object,
-                                    guint prop_id,
-                                    const GValue *value,
-                                    GParamSpec *pspec)
-{
-  GArrowStreamReaderPrivate *priv;
-
-  priv = GARROW_STREAM_READER_GET_PRIVATE(object);
-
-  switch (prop_id) {
-  case PROP_STREAM_READER:
-    priv->stream_reader =
-      *static_cast<std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *>(g_value_get_pointer(value));
-    break;
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_stream_reader_get_property(GObject *object,
-                                          guint prop_id,
-                                          GValue *value,
-                                          GParamSpec *pspec)
-{
-  switch (prop_id) {
-  default:
-    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
-    break;
-  }
-}
-
-static void
-garrow_stream_reader_init(GArrowStreamReader *object)
-{
-}
-
-static void
-garrow_stream_reader_class_init(GArrowStreamReaderClass *klass)
-{
-  GObjectClass *gobject_class;
-  GParamSpec *spec;
-
-  gobject_class = G_OBJECT_CLASS(klass);
-
-  gobject_class->finalize     = garrow_stream_reader_finalize;
-  gobject_class->set_property = garrow_stream_reader_set_property;
-  gobject_class->get_property = garrow_stream_reader_get_property;
-
-  spec = g_param_spec_pointer("stream-reader",
-                              "ipc::RecordBatchStreamReader",
-                              "The raw std::shared<arrow::ipc::RecordBatchStreamReader> *",
-                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
-                                                       G_PARAM_CONSTRUCT_ONLY));
-  g_object_class_install_property(gobject_class, PROP_STREAM_READER, spec);
-}
-
-/**
- * garrow_stream_reader_new:
- * @stream: The stream to be read.
- * @error: (nullable): Return locatipcn for a #GError or %NULL.
- *
- * Returns: (nullable): A newly created #GArrowStreamReader or %NULL
- *   on error.
- */
-GArrowStreamReader *
-garrow_stream_reader_new(GArrowInputStream *stream,
-                         GError **error)
-{
-  std::shared_ptr<arrow::ipc::RecordBatchStreamReader> arrow_stream_reader;
-  auto status =
-    arrow::ipc::RecordBatchStreamReader::Open(garrow_input_stream_get_raw(stream),
-                                              &arrow_stream_reader);
-  if (garrow_error_check(error, status, "[ipc][stream-reader][open]")) {
-    return garrow_stream_reader_new_raw(&arrow_stream_reader);
-  } else {
-    return NULL;
-  }
-}
-
-/**
- * garrow_stream_reader_get_schema:
- * @stream_reader: A #GArrowStreamReader.
- *
- * Returns: (transfer full): The schema in the stream.
- */
-GArrowSchema *
-garrow_stream_reader_get_schema(GArrowStreamReader *stream_reader)
-{
-  auto arrow_stream_reader =
-    garrow_stream_reader_get_raw(stream_reader);
-  auto arrow_schema = arrow_stream_reader->schema();
-  return garrow_schema_new_raw(&arrow_schema);
-}
-
-/**
- * garrow_stream_reader_get_next_record_batch:
- * @stream_reader: A #GArrowStreamReader.
- * @error: (nullable): Return locatipcn for a #GError or %NULL.
- *
- * Returns: (nullable) (transfer full):
- *   The next record batch in the stream or %NULL on end of stream.
- */
-GArrowRecordBatch *
-garrow_stream_reader_get_next_record_batch(GArrowStreamReader *stream_reader,
-                                           GError **error)
-{
-  auto arrow_stream_reader =
-    garrow_stream_reader_get_raw(stream_reader);
-  std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
-  auto status = arrow_stream_reader->GetNextRecordBatch(&arrow_record_batch);
-
-  if (garrow_error_check(error,
-                       status,
-                       "[ipc][stream-reader][get-next-record-batch]")) {
-    if (arrow_record_batch == nullptr) {
-      return NULL;
-    } else {
-      return garrow_record_batch_new_raw(&arrow_record_batch);
-    }
-  } else {
-    return NULL;
-  }
-}
-
-G_END_DECLS
-
-GArrowStreamReader *
-garrow_stream_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *arrow_stream_reader)
-{
-  auto stream_reader =
-    GARROW_STREAM_READER(g_object_new(GARROW_TYPE_STREAM_READER,
-                                          "stream-reader", arrow_stream_reader,
-                                          NULL));
-  return stream_reader;
-}
-
-std::shared_ptr<arrow::ipc::RecordBatchStreamReader>
-garrow_stream_reader_get_raw(GArrowStreamReader *stream_reader)
-{
-  GArrowStreamReaderPrivate *priv;
-
-  priv = GARROW_STREAM_READER_GET_PRIVATE(stream_reader);
-  return priv->stream_reader;
-}
diff --git a/c_glib/arrow-glib/stream-reader.h b/c_glib/arrow-glib/stream-reader.h
deleted file mode 100644
index f6cdaea5f7a95..0000000000000
--- a/c_glib/arrow-glib/stream-reader.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <arrow-glib/record-batch.h>
-#include <arrow-glib/schema.h>
-
-#include <arrow-glib/input-stream.h>
-
-#include <arrow-glib/metadata-version.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_STREAM_READER           \
-  (garrow_stream_reader_get_type())
-#define GARROW_STREAM_READER(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_STREAM_READER,    \
-                              GArrowStreamReader))
-#define GARROW_STREAM_READER_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_STREAM_READER,       \
-                           GArrowStreamReaderClass))
-#define GARROW_IS_STREAM_READER(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STREAM_READER))
-#define GARROW_IS_STREAM_READER_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_STREAM_READER))
-#define GARROW_STREAM_READER_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_STREAM_READER,     \
-                             GArrowStreamReaderClass))
-
-typedef struct _GArrowStreamReader         GArrowStreamReader;
-typedef struct _GArrowStreamReaderClass    GArrowStreamReaderClass;
-
-/**
- * GArrowStreamReader:
- *
- * It wraps `arrow::ipc::InputStreamReader`.
- */
-struct _GArrowStreamReader
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowStreamReaderClass
-{
-  GObjectClass parent_class;
-};
-
-GType garrow_stream_reader_get_type(void) G_GNUC_CONST;
-
-GArrowStreamReader *garrow_stream_reader_new(GArrowInputStream *stream,
-                                             GError **error);
-
-GArrowSchema *garrow_stream_reader_get_schema(GArrowStreamReader *stream_reader);
-GArrowRecordBatch *garrow_stream_reader_get_next_record_batch(GArrowStreamReader *stream_reader,
-                                                                  GError **error);
-
-G_END_DECLS
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index dae89e2138a80..4fa1b7c42def5 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -103,8 +103,7 @@
     </chapter>
     <chapter id="reader">
       <title>Reader</title>
-      <xi:include href="xml/file-reader.xml"/>
-      <xi:include href="xml/stream-reader.xml"/>
+      <xi:include href="xml/reader.xml"/>
     </chapter>
     <chapter id="writer">
       <title>Writer</title>
diff --git a/c_glib/example/read-batch.c b/c_glib/example/read-batch.c
index 212b2a7a342f0..4461c43c84777 100644
--- a/c_glib/example/read-batch.c
+++ b/c_glib/example/read-batch.c
@@ -101,10 +101,11 @@ main(int argc, char **argv)
   }
 
   {
-    GArrowFileReader *reader;
+    GArrowRecordBatchFileReader *reader;
 
-    reader = garrow_file_reader_new(GARROW_SEEKABLE_INPUT_STREAM(input),
-                                    &error);
+    reader =
+      garrow_record_batch_file_reader_new(GARROW_SEEKABLE_INPUT_STREAM(input),
+                                          &error);
     if (!reader) {
       g_print("failed to open file reader: %s\n", error->message);
       g_error_free(error);
@@ -115,12 +116,12 @@ main(int argc, char **argv)
     {
       guint i, n;
 
-      n = garrow_file_reader_get_n_record_batches(reader);
+      n = garrow_record_batch_file_reader_get_n_record_batches(reader);
       for (i = 0; i < n; i++) {
         GArrowRecordBatch *record_batch;
 
         record_batch =
-          garrow_file_reader_get_record_batch(reader, i, &error);
+          garrow_record_batch_file_reader_get_record_batch(reader, i, &error);
         if (!record_batch) {
           g_print("failed to open file reader: %s\n", error->message);
           g_error_free(error);
diff --git a/c_glib/example/read-stream.c b/c_glib/example/read-stream.c
index 28a3f5e2b9e1c..412be35135350 100644
--- a/c_glib/example/read-stream.c
+++ b/c_glib/example/read-stream.c
@@ -100,22 +100,25 @@ main(int argc, char **argv)
   }
 
   {
-    GArrowStreamReader *reader;
+    GArrowRecordBatchReader *reader;
+    GArrowRecordBatchStreamReader *stream_reader;
 
-    reader = garrow_stream_reader_new(GARROW_INPUT_STREAM(input),
-                                      &error);
-    if (!reader) {
+    stream_reader =
+      garrow_record_batch_stream_reader_new(GARROW_INPUT_STREAM(input),
+                                            &error);
+    if (!stream_reader) {
       g_print("failed to open stream reader: %s\n", error->message);
       g_error_free(error);
       g_object_unref(input);
       return EXIT_FAILURE;
     }
 
+    reader = GARROW_RECORD_BATCH_READER(stream_reader);
     while (TRUE) {
       GArrowRecordBatch *record_batch;
 
       record_batch =
-        garrow_stream_reader_get_next_record_batch(reader, &error);
+        garrow_record_batch_reader_get_next_record_batch(reader, &error);
       if (error) {
         g_print("failed to get record batch: %s\n", error->message);
         g_error_free(error);
diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb
index 1d9102b6b0085..02b8bd55fa017 100644
--- a/c_glib/test/test-file-writer.rb
+++ b/c_glib/test/test-file-writer.rb
@@ -35,7 +35,7 @@ def test_write_record_batch
 
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
-      file_reader = Arrow::FileReader.new(input)
+      file_reader = Arrow::RecordBatchFileReader.new(input)
       assert_equal(["enabled"],
                    file_reader.schema.fields.collect(&:name))
     ensure
diff --git a/c_glib/test/test-stream-writer.rb b/c_glib/test/test-stream-writer.rb
index d27eaa54fc53c..da6c2afa874b3 100644
--- a/c_glib/test/test-stream-writer.rb
+++ b/c_glib/test/test-stream-writer.rb
@@ -40,7 +40,7 @@ def test_write_record_batch
 
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
-      stream_reader = Arrow::StreamReader.new(input)
+      stream_reader = Arrow::RecordBatchStreamReader.new(input)
       assert_equal(["enabled"],
                    stream_reader.schema.fields.collect(&:name))
       assert_equal(true,

From 49c5398104d0f1749a411e013cb66fd561511401 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 16 May 2017 10:01:40 -0400
Subject: [PATCH 0662/1644] ARROW-1038: [GLib] Follow writer name change

Author: Kouhei Sutou <kou@clear-code.com>

Closes #696 from kou/glib-follow-writer-name-change and squashes the following commits:

85f7ce8 [Kouhei Sutou] [GLib] Follow writer name change
---
 c_glib/arrow-glib/writer.cpp      | 276 +++++++++++++++++-------------
 c_glib/arrow-glib/writer.h        | 187 ++++++++++++--------
 c_glib/arrow-glib/writer.hpp      |   9 +-
 c_glib/test/test-file-writer.rb   |   2 +-
 c_glib/test/test-stream-writer.rb |   2 +-
 5 files changed, 280 insertions(+), 196 deletions(-)

diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp
index 092993b5c2240..20dd281cfdf68 100644
--- a/c_glib/arrow-glib/writer.cpp
+++ b/c_glib/arrow-glib/writer.cpp
@@ -40,56 +40,60 @@ G_BEGIN_DECLS
  * @title: Writer classes
  * @include: arrow-glib/arrow-glib.h
  *
- * #GArrowStreamWriter is a class for writing data in stream mode.
+ * #GArrowRecordBatchWriter is a base class for writing record batches
+ * in stream format into output.
  *
- * #GArrowFileWriter is a class for writing data in random access
- * mode.
+ * #GArrowRecordBatchStreamWriter is a base class for writing record
+ * batches in stream format into output synchronously.
+ *
+ * #GArrowRecordBatchFileWriter is a class for writing record
+ * batches in file format into output.
  */
 
-typedef struct GArrowStreamWriterPrivate_ {
-  std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> stream_writer;
-} GArrowStreamWriterPrivate;
+typedef struct GArrowRecordBatchWriterPrivate_ {
+  std::shared_ptr<arrow::ipc::RecordBatchWriter> record_batch_writer;
+} GArrowRecordBatchWriterPrivate;
 
 enum {
   PROP_0,
-  PROP_STREAM_WRITER
+  PROP_RECORD_BATCH_WRITER
 };
 
-G_DEFINE_TYPE_WITH_PRIVATE(GArrowStreamWriter,
-                           garrow_stream_writer,
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowRecordBatchWriter,
+                           garrow_record_batch_writer,
                            G_TYPE_OBJECT);
 
-#define GARROW_STREAM_WRITER_GET_PRIVATE(obj)                   \
+#define GARROW_RECORD_BATCH_WRITER_GET_PRIVATE(obj)             \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
-                               GARROW_TYPE_STREAM_WRITER,       \
-                               GArrowStreamWriterPrivate))
+                               GARROW_TYPE_RECORD_BATCH_WRITER, \
+                               GArrowRecordBatchWriterPrivate))
 
 static void
-garrow_stream_writer_finalize(GObject *object)
+garrow_record_batch_writer_finalize(GObject *object)
 {
-  GArrowStreamWriterPrivate *priv;
+  GArrowRecordBatchWriterPrivate *priv;
 
-  priv = GARROW_STREAM_WRITER_GET_PRIVATE(object);
+  priv = GARROW_RECORD_BATCH_WRITER_GET_PRIVATE(object);
 
-  priv->stream_writer = nullptr;
+  priv->record_batch_writer = nullptr;
 
-  G_OBJECT_CLASS(garrow_stream_writer_parent_class)->finalize(object);
+  G_OBJECT_CLASS(garrow_record_batch_writer_parent_class)->finalize(object);
 }
 
 static void
-garrow_stream_writer_set_property(GObject *object,
-                                  guint prop_id,
-                                  const GValue *value,
-                                  GParamSpec *pspec)
+garrow_record_batch_writer_set_property(GObject *object,
+                                        guint prop_id,
+                                        const GValue *value,
+                                        GParamSpec *pspec)
 {
-  GArrowStreamWriterPrivate *priv;
+  GArrowRecordBatchWriterPrivate *priv;
 
-  priv = GARROW_STREAM_WRITER_GET_PRIVATE(object);
+  priv = GARROW_RECORD_BATCH_WRITER_GET_PRIVATE(object);
 
   switch (prop_id) {
-  case PROP_STREAM_WRITER:
-    priv->stream_writer =
-      *static_cast<std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> *>(g_value_get_pointer(value));
+  case PROP_RECORD_BATCH_WRITER:
+    priv->record_batch_writer =
+      *static_cast<std::shared_ptr<arrow::ipc::RecordBatchWriter> *>(g_value_get_pointer(value));
     break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
@@ -98,10 +102,10 @@ garrow_stream_writer_set_property(GObject *object,
 }
 
 static void
-garrow_stream_writer_get_property(GObject *object,
-                                  guint prop_id,
-                                  GValue *value,
-                                  GParamSpec *pspec)
+garrow_record_batch_writer_get_property(GObject *object,
+                                        guint prop_id,
+                                        GValue *value,
+                                        GParamSpec *pspec)
 {
   switch (prop_id) {
   default:
@@ -111,132 +115,157 @@ garrow_stream_writer_get_property(GObject *object,
 }
 
 static void
-garrow_stream_writer_init(GArrowStreamWriter *object)
+garrow_record_batch_writer_init(GArrowRecordBatchWriter *object)
 {
 }
 
 static void
-garrow_stream_writer_class_init(GArrowStreamWriterClass *klass)
+garrow_record_batch_writer_class_init(GArrowRecordBatchWriterClass *klass)
 {
   GObjectClass *gobject_class;
   GParamSpec *spec;
 
   gobject_class = G_OBJECT_CLASS(klass);
 
-  gobject_class->finalize     = garrow_stream_writer_finalize;
-  gobject_class->set_property = garrow_stream_writer_set_property;
-  gobject_class->get_property = garrow_stream_writer_get_property;
+  gobject_class->finalize     = garrow_record_batch_writer_finalize;
+  gobject_class->set_property = garrow_record_batch_writer_set_property;
+  gobject_class->get_property = garrow_record_batch_writer_get_property;
 
-  spec = g_param_spec_pointer("stream-writer",
-                              "ipc::RecordBatchStreamWriter",
-                              "The raw std::shared<arrow::ipc::RecordBatchStreamWriter> *",
+  spec = g_param_spec_pointer("record-batch-writer",
+                              "arrow::ipc::RecordBatchWriter",
+                              "The raw std::shared<arrow::ipc::RecordBatchWriter> *",
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
-  g_object_class_install_property(gobject_class, PROP_STREAM_WRITER, spec);
-}
-
-/**
- * garrow_stream_writer_new:
- * @sink: The output of the writer.
- * @schema: The schema of the writer.
- * @error: (nullable): Return locatipcn for a #GError or %NULL.
- *
- * Returns: (nullable): A newly created #GArrowStreamWriter or %NULL on
- *   error.
- */
-GArrowStreamWriter *
-garrow_stream_writer_new(GArrowOutputStream *sink,
-                         GArrowSchema *schema,
-                         GError **error)
-{
-  std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> arrow_stream_writer;
-  auto status =
-    arrow::ipc::RecordBatchStreamWriter::Open(garrow_output_stream_get_raw(sink).get(),
-                                              garrow_schema_get_raw(schema),
-                                              &arrow_stream_writer);
-  if (garrow_error_check(error, status, "[ipc][stream-writer][open]")) {
-    return garrow_stream_writer_new_raw(&arrow_stream_writer);
-  } else {
-    return NULL;
-  }
+  g_object_class_install_property(gobject_class, PROP_RECORD_BATCH_WRITER, spec);
 }
 
 /**
- * garrow_stream_writer_write_record_batch:
- * @stream_writer: A #GArrowStreamWriter.
+ * garrow_record_batch_writer_write_record_batch:
+ * @writer: A #GArrowRecordBatchWriter.
  * @record_batch: The record batch to be written.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.4.0
  */
 gboolean
-garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
-                                        GArrowRecordBatch *record_batch,
-                                        GError **error)
+garrow_record_batch_writer_write_record_batch(GArrowRecordBatchWriter *writer,
+                                              GArrowRecordBatch *record_batch,
+                                              GError **error)
 {
-  auto arrow_stream_writer = garrow_stream_writer_get_raw(stream_writer);
+  auto arrow_writer = garrow_record_batch_writer_get_raw(writer);
   auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
   auto arrow_record_batch_raw = arrow_record_batch.get();
 
-  auto status = arrow_stream_writer->WriteRecordBatch(*arrow_record_batch_raw);
+  auto status = arrow_writer->WriteRecordBatch(*arrow_record_batch_raw);
   return garrow_error_check(error,
                             status,
-                            "[ipc][stream-writer][write-record-batch]");
+                            "[record-batch-writer][write-record-batch]");
 }
 
 /**
- * garrow_stream_writer_close:
- * @stream_writer: A #GArrowStreamWriter.
+ * garrow_record_batch_writer_close:
+ * @writer: A #GArrowRecordBatchWriter.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
  * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.4.0
  */
 gboolean
-garrow_stream_writer_close(GArrowStreamWriter *stream_writer,
-                           GError **error)
+garrow_record_batch_writer_close(GArrowRecordBatchWriter *writer,
+                                 GError **error)
 {
-  auto arrow_stream_writer = garrow_stream_writer_get_raw(stream_writer);
+  auto arrow_writer = garrow_record_batch_writer_get_raw(writer);
 
-  auto status = arrow_stream_writer->Close();
-  return garrow_error_check(error, status, "[ipc][stream-writer][close]");
+  auto status = arrow_writer->Close();
+  return garrow_error_check(error, status, "[record-batch-writer][close]");
 }
 
 
-G_DEFINE_TYPE(GArrowFileWriter,
-              garrow_file_writer,
-              GARROW_TYPE_STREAM_WRITER);
+G_DEFINE_TYPE(GArrowRecordBatchStreamWriter,
+              garrow_record_batch_stream_writer,
+              GARROW_TYPE_RECORD_BATCH_WRITER);
 
 static void
-garrow_file_writer_init(GArrowFileWriter *object)
+garrow_record_batch_stream_writer_init(GArrowRecordBatchStreamWriter *object)
 {
 }
 
 static void
-garrow_file_writer_class_init(GArrowFileWriterClass *klass)
+garrow_record_batch_stream_writer_class_init(GArrowRecordBatchStreamWriterClass *klass)
 {
 }
 
 /**
- * garrow_file_writer_new:
+ * garrow_record_batch_stream_writer_new:
  * @sink: The output of the writer.
  * @schema: The schema of the writer.
  * @error: (nullable): Return locatipcn for a #GError or %NULL.
  *
- * Returns: (nullable): A newly created #GArrowFileWriter or %NULL on
- *   error.
+ * Returns: (nullable): A newly created #GArrowRecordBatchStreamWriter
+ *   or %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowRecordBatchStreamWriter *
+garrow_record_batch_stream_writer_new(GArrowOutputStream *sink,
+                                      GArrowSchema *schema,
+                                      GError **error)
+{
+  auto arrow_sink = garrow_output_stream_get_raw(sink).get();
+  std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> arrow_writer;
+  auto status =
+    arrow::ipc::RecordBatchStreamWriter::Open(arrow_sink,
+                                              garrow_schema_get_raw(schema),
+                                              &arrow_writer);
+  if (garrow_error_check(error, status, "[record-batch-stream-writer][open]")) {
+    return garrow_record_batch_stream_writer_new_raw(&arrow_writer);
+  } else {
+    return NULL;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowRecordBatchFileWriter,
+              garrow_record_batch_file_writer,
+              GARROW_TYPE_RECORD_BATCH_STREAM_WRITER);
+
+static void
+garrow_record_batch_file_writer_init(GArrowRecordBatchFileWriter *object)
+{
+}
+
+static void
+garrow_record_batch_file_writer_class_init(GArrowRecordBatchFileWriterClass *klass)
+{
+}
+
+/**
+ * garrow_record_batch_file_writer_new:
+ * @sink: The output of the writer.
+ * @schema: The schema of the writer.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowRecordBatchFileWriter
+ *   or %NULL on error.
+ *
+ * Since: 0.4.0
  */
-GArrowFileWriter *
-garrow_file_writer_new(GArrowOutputStream *sink,
+GArrowRecordBatchFileWriter *
+garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
                        GArrowSchema *schema,
                        GError **error)
 {
-  std::shared_ptr<arrow::ipc::FileWriter> arrow_file_writer;
+  auto arrow_sink = garrow_output_stream_get_raw(sink);
+  std::shared_ptr<arrow::ipc::RecordBatchFileWriter> arrow_writer;
   auto status =
-    arrow::ipc::FileWriter::Open(garrow_output_stream_get_raw(sink).get(),
-                                 garrow_schema_get_raw(schema),
-                                 &arrow_file_writer);
-  if (garrow_error_check(error, status, "[ipc][file-writer][open]")) {
-    return garrow_file_writer_new_raw(&arrow_file_writer);
+    arrow::ipc::RecordBatchFileWriter::Open(arrow_sink.get(),
+                                            garrow_schema_get_raw(schema),
+                                            &arrow_writer);
+  if (garrow_error_check(error, status, "[record-batch-file-writer][open]")) {
+    return garrow_record_batch_file_writer_new_raw(&arrow_writer);
   } else {
     return NULL;
   }
@@ -244,41 +273,44 @@ garrow_file_writer_new(GArrowOutputStream *sink,
 
 G_END_DECLS
 
-GArrowStreamWriter *
-garrow_stream_writer_new_raw(std::shared_ptr<arrow::ipc::StreamWriter> *arrow_stream_writer)
+GArrowRecordBatchWriter *
+garrow_record_batch_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchWriter> *arrow_writer)
 {
-  auto stream_writer =
-    GARROW_STREAM_WRITER(g_object_new(GARROW_TYPE_STREAM_WRITER,
-                                        "stream-writer", arrow_stream_writer,
-                                        NULL));
-  return stream_writer;
+  auto writer =
+    GARROW_RECORD_BATCH_WRITER(
+      g_object_new(GARROW_TYPE_RECORD_BATCH_WRITER,
+                   "record-batch-writer", arrow_writer,
+                   NULL));
+  return writer;
 }
 
-std::shared_ptr<arrow::ipc::StreamWriter>
-garrow_stream_writer_get_raw(GArrowStreamWriter *stream_writer)
+std::shared_ptr<arrow::ipc::RecordBatchWriter>
+garrow_record_batch_writer_get_raw(GArrowRecordBatchWriter *writer)
 {
-  GArrowStreamWriterPrivate *priv;
+  GArrowRecordBatchWriterPrivate *priv;
 
-  priv = GARROW_STREAM_WRITER_GET_PRIVATE(stream_writer);
-  return priv->stream_writer;
+  priv = GARROW_RECORD_BATCH_WRITER_GET_PRIVATE(writer);
+  return priv->record_batch_writer;
 }
 
-GArrowFileWriter *
-garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::FileWriter> *arrow_file_writer)
+GArrowRecordBatchStreamWriter *
+garrow_record_batch_stream_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> *arrow_writer)
 {
-  auto file_writer =
-    GARROW_FILE_WRITER(g_object_new(GARROW_TYPE_FILE_WRITER,
-                                    "stream-writer", arrow_file_writer,
-                                    NULL));
-  return file_writer;
+  auto writer =
+    GARROW_RECORD_BATCH_STREAM_WRITER(
+      g_object_new(GARROW_TYPE_RECORD_BATCH_STREAM_WRITER,
+                   "record-batch-writer", arrow_writer,
+                   NULL));
+  return writer;
 }
 
-arrow::ipc::FileWriter *
-garrow_file_writer_get_raw(GArrowFileWriter *file_writer)
+GArrowRecordBatchFileWriter *
+garrow_record_batch_file_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileWriter> *arrow_writer)
 {
-  auto arrow_stream_writer =
-    garrow_stream_writer_get_raw(GARROW_STREAM_WRITER(file_writer));
-  auto arrow_file_writer_raw =
-    dynamic_cast<arrow::ipc::FileWriter *>(arrow_stream_writer.get());
-  return arrow_file_writer_raw;
+  auto writer =
+    GARROW_RECORD_BATCH_FILE_WRITER(
+      g_object_new(GARROW_TYPE_RECORD_BATCH_FILE_WRITER,
+                   "record-batch-writer", arrow_writer,
+                   NULL));
+  return writer;
 }
diff --git a/c_glib/arrow-glib/writer.h b/c_glib/arrow-glib/writer.h
index 2f8e90cd28599..9a9846179e5df 100644
--- a/c_glib/arrow-glib/writer.h
+++ b/c_glib/arrow-glib/writer.h
@@ -27,111 +27,162 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_STREAM_WRITER               \
-  (garrow_stream_writer_get_type())
-#define GARROW_STREAM_WRITER(obj)                               \
+#define GARROW_TYPE_RECORD_BATCH_WRITER         \
+  (garrow_record_batch_writer_get_type())
+#define GARROW_RECORD_BATCH_WRITER(obj)                         \
   (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_STREAM_WRITER,        \
-                              GArrowStreamWriter))
-#define GARROW_STREAM_WRITER_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_STREAM_WRITER,   \
-                           GArrowStreamWriterClass))
-#define GARROW_IS_STREAM_WRITER(obj)                            \
+                              GARROW_TYPE_RECORD_BATCH_WRITER,  \
+                              GArrowRecordBatchWriter))
+#define GARROW_RECORD_BATCH_WRITER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_RECORD_BATCH_WRITER,     \
+                           GArrowRecordBatchWriterClass))
+#define GARROW_IS_RECORD_BATCH_WRITER(obj)                      \
   (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STREAM_WRITER))
-#define GARROW_IS_STREAM_WRITER_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_STREAM_WRITER))
-#define GARROW_STREAM_WRITER_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_STREAM_WRITER, \
-                             GArrowStreamWriterClass))
-
-typedef struct _GArrowStreamWriter         GArrowStreamWriter;
+                              GARROW_TYPE_RECORD_BATCH_WRITER))
+#define GARROW_IS_RECORD_BATCH_WRITER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_RECORD_BATCH_WRITER))
+#define GARROW_RECORD_BATCH_WRITER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_RECORD_BATCH_WRITER,   \
+                             GArrowRecordBatchWriterClass))
+
+typedef struct _GArrowRecordBatchWriter         GArrowRecordBatchWriter;
 #ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowStreamWriterClass    GArrowStreamWriterClass;
+typedef struct _GArrowRecordBatchWriterClass    GArrowRecordBatchWriterClass;
 #endif
 
 /**
- * GArrowStreamWriter:
+ * GArrowRecordBatchWriter:
  *
- * It wraps `arrow::ipc::RecordBatchStreamWriter`.
+ * It wraps `arrow::ipc::RecordBatchWriter`.
  */
-struct _GArrowStreamWriter
+struct _GArrowRecordBatchWriter
 {
   /*< private >*/
   GObject parent_instance;
 };
 
 #ifndef __GTK_DOC_IGNORE__
-struct _GArrowStreamWriterClass
+struct _GArrowRecordBatchWriterClass
 {
   GObjectClass parent_class;
 };
 #endif
 
-GType garrow_stream_writer_get_type(void) G_GNUC_CONST;
-
-GArrowStreamWriter *garrow_stream_writer_new(GArrowOutputStream *sink,
-                                             GArrowSchema *schema,
-                                             GError **error);
-
-gboolean garrow_stream_writer_write_record_batch(GArrowStreamWriter *stream_writer,
-                                                 GArrowRecordBatch *record_batch,
-                                                 GError **error);
-gboolean garrow_stream_writer_close(GArrowStreamWriter *stream_writer,
-                                    GError **error);
-
-
-#define GARROW_TYPE_FILE_WRITER                 \
-  (garrow_file_writer_get_type())
-#define GARROW_FILE_WRITER(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_FILE_WRITER,  \
-                              GArrowFileWriter))
-#define GARROW_FILE_WRITER_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_FILE_WRITER,     \
-                           GArrowFileWriterClass))
-#define GARROW_IS_FILE_WRITER(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_FILE_WRITER))
-#define GARROW_IS_FILE_WRITER_CLASS(klass)              \
+GType garrow_record_batch_writer_get_type(void) G_GNUC_CONST;
+
+gboolean garrow_record_batch_writer_write_record_batch(
+  GArrowRecordBatchWriter *writer,
+  GArrowRecordBatch *record_batch,
+  GError **error);
+gboolean garrow_record_batch_writer_close(
+  GArrowRecordBatchWriter *writer,
+  GError **error);
+
+
+#define GARROW_TYPE_RECORD_BATCH_STREAM_WRITER          \
+  (garrow_record_batch_stream_writer_get_type())
+#define GARROW_RECORD_BATCH_STREAM_WRITER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_RECORD_BATCH_STREAM_WRITER,   \
+                              GArrowRecordBatchStreamWriter))
+#define GARROW_RECORD_BATCH_STREAM_WRITER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                                     \
+                           GARROW_TYPE_RECORD_BATCH_STREAM_WRITER,      \
+                           GArrowRecordBatchStreamWriterClass))
+#define GARROW_IS_RECORD_BATCH_STREAM_WRITER(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_RECORD_BATCH_STREAM_WRITER))
+#define GARROW_IS_RECORD_BATCH_STREAM_WRITER_CLASS(klass)            \
   (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_FILE_WRITER))
-#define GARROW_FILE_WRITER_GET_CLASS(obj)               \
+                           GARROW_TYPE_RECORD_BATCH_STREAM_WRITER))
+#define GARROW_RECORD_BATCH_STREAM_WRITER_GET_CLASS(obj)             \
   (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_FILE_WRITER,   \
-                             GArrowFileWriterClass))
+                             GARROW_TYPE_RECORD_BATCH_STREAM_WRITER, \
+                             GArrowRecordBatchStreamWriterClass))
+
+typedef struct _GArrowRecordBatchStreamWriter      GArrowRecordBatchStreamWriter;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowRecordBatchStreamWriterClass GArrowRecordBatchStreamWriterClass;
+#endif
+
+/**
+ * GArrowRecordBatchStreamWriter:
+ *
+ * It wraps `arrow::ipc::RecordBatchStreamWriter`.
+ */
+struct _GArrowRecordBatchStreamWriter
+{
+  /*< private >*/
+  GArrowRecordBatchWriter parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowRecordBatchStreamWriterClass
+{
+  GArrowRecordBatchWriterClass parent_class;
+};
+#endif
 
-typedef struct _GArrowFileWriter         GArrowFileWriter;
+GType garrow_record_batch_stream_writer_get_type(void) G_GNUC_CONST;
+
+GArrowRecordBatchStreamWriter *garrow_record_batch_stream_writer_new(
+  GArrowOutputStream *sink,
+  GArrowSchema *schema,
+  GError **error);
+
+
+#define GARROW_TYPE_RECORD_BATCH_FILE_WRITER    \
+  (garrow_record_batch_file_writer_get_type())
+#define GARROW_RECORD_BATCH_FILE_WRITER(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_RECORD_BATCH_FILE_WRITER,     \
+                              GArrowRecordBatchFileWriter))
+#define GARROW_RECORD_BATCH_FILE_WRITER_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                                     \
+                           GARROW_TYPE_RECORD_BATCH_FILE_WRITER,        \
+                           GArrowRecordBatchFileWriterClass))
+#define GARROW_IS_RECORD_BATCH_FILE_WRITER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_RECORD_BATCH_FILE_WRITER))
+#define GARROW_IS_RECORD_BATCH_FILE_WRITER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                                     \
+                           GARROW_TYPE_RECORD_BATCH_FILE_WRITER))
+#define GARROW_RECORD_BATCH_FILE_WRITER_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                                     \
+                             GARROW_TYPE_RECORD_BATCH_FILE_WRITER,      \
+                             GArrowRecordBatchFileWriterClass))
+
+typedef struct _GArrowRecordBatchFileWriter      GArrowRecordBatchFileWriter;
 #ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowFileWriterClass    GArrowFileWriterClass;
+typedef struct _GArrowRecordBatchFileWriterClass GArrowRecordBatchFileWriterClass;
 #endif
 
 /**
- * GArrowFileWriter:
+ * GArrowRecordBatchFileWriter:
  *
- * It wraps `arrow::ipc::FileWriter`.
+ * It wraps `arrow::ipc::RecordBatchFileWriter`.
  */
-struct _GArrowFileWriter
+struct _GArrowRecordBatchFileWriter
 {
   /*< private >*/
-  GArrowStreamWriter parent_instance;
+  GArrowRecordBatchStreamWriter parent_instance;
 };
 
 #ifndef __GTK_DOC_IGNORE__
-struct _GArrowFileWriterClass
+struct _GArrowRecordBatchFileWriterClass
 {
-  GArrowStreamWriterClass parent_class;
+  GArrowRecordBatchStreamWriterClass parent_class;
 };
 #endif
 
-GType garrow_file_writer_get_type(void) G_GNUC_CONST;
+GType garrow_record_batch_file_writer_get_type(void) G_GNUC_CONST;
 
-GArrowFileWriter *garrow_file_writer_new(GArrowOutputStream *sink,
-                                         GArrowSchema *schema,
-                                         GError **error);
+GArrowRecordBatchFileWriter *garrow_record_batch_file_writer_new(
+  GArrowOutputStream *sink,
+  GArrowSchema *schema,
+  GError **error);
 
 G_END_DECLS
diff --git a/c_glib/arrow-glib/writer.hpp b/c_glib/arrow-glib/writer.hpp
index 47f5e6839654f..c9ea6603c9ea4 100644
--- a/c_glib/arrow-glib/writer.hpp
+++ b/c_glib/arrow-glib/writer.hpp
@@ -24,8 +24,9 @@
 
 #include <arrow-glib/writer.h>
 
-GArrowStreamWriter *garrow_stream_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> *arrow_stream_writer);
-std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> garrow_stream_writer_get_raw(GArrowStreamWriter *stream_writer);
+GArrowRecordBatchWriter *garrow_record_batch_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchWriter> *arrow_writer);
+std::shared_ptr<arrow::ipc::RecordBatchWriter> garrow_record_batch_writer_get_raw(GArrowRecordBatchWriter *writer);
 
-GArrowFileWriter *garrow_file_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileWriter> *arrow_file_writer);
-arrow::ipc::RecordBatchFileWriter *garrow_file_writer_get_raw(GArrowFileWriter *file_writer);
+GArrowRecordBatchStreamWriter *garrow_record_batch_stream_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> *arrow_writer);
+
+GArrowRecordBatchFileWriter *garrow_record_batch_file_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileWriter> *arrow_writer);
diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb
index 02b8bd55fa017..6ba5c7aebca44 100644
--- a/c_glib/test/test-file-writer.rb
+++ b/c_glib/test/test-file-writer.rb
@@ -22,7 +22,7 @@ def test_write_record_batch
     begin
       field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
       schema = Arrow::Schema.new([field])
-      file_writer = Arrow::FileWriter.new(output, schema)
+      file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
       begin
         record_batch = Arrow::RecordBatch.new(schema, 0, [])
         file_writer.write_record_batch(record_batch)
diff --git a/c_glib/test/test-stream-writer.rb b/c_glib/test/test-stream-writer.rb
index da6c2afa874b3..41fd14718e66e 100644
--- a/c_glib/test/test-stream-writer.rb
+++ b/c_glib/test/test-stream-writer.rb
@@ -24,7 +24,7 @@ def test_write_record_batch
     begin
       field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
       schema = Arrow::Schema.new([field])
-      stream_writer = Arrow::StreamWriter.new(output, schema)
+      stream_writer = Arrow::RecordBatchStreamWriter.new(output, schema)
       begin
         columns = [
           build_boolean_array([true]),

From e7e8d611cc415d8667673dc57a5c3c5732b25696 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 16 May 2017 10:03:29 -0400
Subject: [PATCH 0663/1644] ARROW-1040: [GLib] Support tensor IO

Author: Kouhei Sutou <kou@clear-code.com>

Closes #697 from kou/glib-support-tensor-io and squashes the following commits:

eca2c3f [Kouhei Sutou] [GLib] Support tensor IO
---
 c_glib/arrow-glib/input-stream.cpp  | 32 ++++++++++++++++++++++++++++
 c_glib/arrow-glib/input-stream.h    |  4 ++++
 c_glib/arrow-glib/output-stream.cpp | 33 ++++++++++++++++++++++++++++-
 c_glib/arrow-glib/output-stream.h   |  5 +++++
 c_glib/test/test-tensor.rb          |  9 ++++++++
 5 files changed, 82 insertions(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index b931cf8250607..d81e4a32d5db4 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -23,12 +23,14 @@
 
 #include <arrow/io/interfaces.h>
 #include <arrow/io/memory.h>
+#include <arrow/ipc/reader.h>
 
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/file.hpp>
 #include <arrow-glib/input-stream.hpp>
 #include <arrow-glib/readable.hpp>
+#include <arrow-glib/tensor.hpp>
 
 G_BEGIN_DECLS
 
@@ -253,6 +255,36 @@ garrow_seekable_input_stream_read_at(GArrowSeekableInputStream *input_stream,
   }
 }
 
+/**
+ * garrow_seekable_input_stream_read_tensor:
+ * @input_stream: A #GArrowSeekableInputStream.
+ * @position: The read start position.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full) (nullable):
+ *   #GArrowTensor on success, %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowTensor *
+garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream *input_stream,
+                                         gint64 position,
+                                         GError **error)
+{
+  auto arrow_random_access_file =
+    garrow_seekable_input_stream_get_raw(input_stream);
+
+  std::shared_ptr<arrow::Tensor> arrow_tensor;
+  auto status = arrow::ipc::ReadTensor(position,
+                                       arrow_random_access_file.get(),
+                                       &arrow_tensor);
+  if (garrow_error_check(error, status, "[seekable-input-stream][read-tensor]")) {
+    return garrow_tensor_new_raw(&arrow_tensor);
+  } else {
+    return NULL;
+  }
+}
+
 
 G_DEFINE_TYPE(GArrowBufferInputStream,                       \
               garrow_buffer_input_stream,                     \
diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h
index 511882863760d..8a4d362189f7d 100644
--- a/c_glib/arrow-glib/input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include <arrow-glib/buffer.h>
+#include <arrow-glib/tensor.h>
 
 G_BEGIN_DECLS
 
@@ -123,6 +124,9 @@ GArrowBuffer *garrow_seekable_input_stream_read_at(GArrowSeekableInputStream *in
                                                    gint64 position,
                                                    gint64 n_bytes,
                                                    GError **error);
+GArrowTensor *garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream *input_stream,
+                                                       gint64 position,
+                                                       GError **error);
 
 
 #define GARROW_TYPE_BUFFER_INPUT_STREAM         \
diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index 48c48b8fdc327..ffb6fecb5c16b 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -21,13 +21,14 @@
 #  include <config.h>
 #endif
 
-#include <arrow/api.h>
 #include <arrow/io/memory.h>
+#include <arrow/ipc/writer.h>
 
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/file.hpp>
 #include <arrow-glib/output-stream.hpp>
+#include <arrow-glib/tensor.hpp>
 #include <arrow-glib/writeable.hpp>
 
 G_BEGIN_DECLS
@@ -168,6 +169,36 @@ garrow_output_stream_class_init(GArrowOutputStreamClass *klass)
   g_object_class_install_property(gobject_class, PROP_OUTPUT_STREAM, spec);
 }
 
+/**
+ * garrow_output_stream_write_tensor:
+ * @stream: A #GArrowWriteable.
+ * @tensor: A #GArrowTensor to be written.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: The number of written bytes on success, -1 on error.
+ *
+ * Since: 0.4.0
+ */
+gint64
+garrow_output_stream_write_tensor(GArrowOutputStream *stream,
+                                  GArrowTensor *tensor,
+                                  GError **error)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  auto arrow_stream = garrow_output_stream_get_raw(stream);
+  int32_t metadata_length;
+  int64_t body_length;
+  auto status = arrow::ipc::WriteTensor(*arrow_tensor,
+                                        arrow_stream.get(),
+                                        &metadata_length,
+                                        &body_length);
+  if (garrow_error_check(error, status, "[output-stream][write-tensor]")) {
+    return metadata_length + body_length;
+  } else {
+    return -1;
+  }
+}
+
 
 G_DEFINE_TYPE(GArrowFileOutputStream,
               garrow_file_output_stream,
diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h
index 48b891c19733d..c86597bdd26c2 100644
--- a/c_glib/arrow-glib/output-stream.h
+++ b/c_glib/arrow-glib/output-stream.h
@@ -22,6 +22,7 @@
 #include <glib-object.h>
 
 #include <arrow-glib/buffer.h>
+#include <arrow-glib/tensor.h>
 
 G_BEGIN_DECLS
 
@@ -71,6 +72,10 @@ struct _GArrowOutputStreamClass
 
 GType garrow_output_stream_get_type(void) G_GNUC_CONST;
 
+gint64 garrow_output_stream_write_tensor(GArrowOutputStream *stream,
+                                         GArrowTensor *tensor,
+                                         GError **error);
+
 
 #define GARROW_TYPE_FILE_OUTPUT_STREAM          \
   (garrow_file_output_stream_get_type())
diff --git a/c_glib/test/test-tensor.rb b/c_glib/test/test-tensor.rb
index 780c9f179e18d..bdfc6576a3f1e 100644
--- a/c_glib/test/test-tensor.rb
+++ b/c_glib/test/test-tensor.rb
@@ -114,4 +114,13 @@ def test_column_major?
       not @tensor.column_major?
     end
   end
+
+  def test_io
+    buffer = Arrow::PoolBuffer.new
+    output = Arrow::BufferOutputStream.new(buffer)
+    output.write_tensor(@tensor)
+    input = Arrow::BufferInputStream.new(buffer)
+    assert_equal(@tensor,
+                 input.read_tensor(0))
+  end
 end

From bed01974321d9d1edeae9e474bd9df020b42ea10 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Tue, 16 May 2017 13:14:18 -0400
Subject: [PATCH 0664/1644] ARROW-881: [Python] Reconstruct Pandas DataFrame
 indexes using metadata

cc @mrocklin

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #612 from cpcloud/ARROW-881 and squashes the following commits:

4fa679d [Phillip Cloud] Add metadata test
60f71aa [Phillip Cloud] More doc
de616e8 [Phillip Cloud] Add doc
a42a084 [Phillip Cloud] Decode metadata to utf8 because JSON
2198dc5 [Phillip Cloud] Call column_name_idx on index_columns
32c5e64 [Phillip Cloud] Add test for read_pandas subset
2fa1f16 [Phillip Cloud] Do not write index_column metadata if not requested
21a8829 [Phillip Cloud] Add docs to pq.read_pandas
c35970c [Phillip Cloud] Add test for no index written and pq.read_pandas
59477b5 [Phillip Cloud] ARROW-881: [Python] Reconstruct Pandas DataFrame indexes using custom_metadata
---
 cpp/src/arrow/type-test.cc                  |  16 ++
 cpp/src/arrow/type.cc                       |  11 +-
 cpp/src/arrow/type.h                        |   7 +-
 python/pyarrow/__init__.py                  |   3 +-
 python/pyarrow/_parquet.pxd                 |   5 +
 python/pyarrow/_parquet.pyx                 |  61 ++++--
 python/pyarrow/array.pxi                    |  92 ++++----
 python/pyarrow/includes/common.pxd          |   2 +-
 python/pyarrow/includes/libarrow.pxd        |   6 +-
 python/pyarrow/io.pxi                       |   2 +-
 python/pyarrow/ipc.py                       |  40 ++++
 python/pyarrow/lib.pxd                      |  32 +--
 python/pyarrow/memory.pxi                   |   2 +-
 python/pyarrow/pandas_compat.py             | 104 +++++++++
 python/pyarrow/parquet.py                   |  63 +++++-
 python/pyarrow/table.pxi                    | 225 ++++++++++++++------
 python/pyarrow/tests/pandas_examples.py     |   8 +-
 python/pyarrow/tests/test_convert_pandas.py |  45 ++--
 python/pyarrow/tests/test_ipc.py            |  51 +++++
 python/pyarrow/tests/test_parquet.py        | 108 ++++++++--
 python/pyarrow/tests/test_table.py          |   2 +-
 21 files changed, 697 insertions(+), 188 deletions(-)
 create mode 100644 python/pyarrow/pandas_compat.py

diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index e73adecdcb5aa..1fbb683da6cf4 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -152,6 +152,22 @@ TEST_F(TestSchema, GetFieldByName) {
   ASSERT_TRUE(result == nullptr);
 }
 
+TEST_F(TestSchema, GetFieldIndex) {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", uint8(), false);
+  auto f2 = field("f2", utf8());
+  auto f3 = field("f3", list(int16()));
+
+  vector<shared_ptr<Field>> fields = {f0, f1, f2, f3};
+  auto schema = std::make_shared<Schema>(fields);
+
+  ASSERT_EQ(0, schema->GetFieldIndex(fields[0]->name()));
+  ASSERT_EQ(1, schema->GetFieldIndex(fields[1]->name()));
+  ASSERT_EQ(2, schema->GetFieldIndex(fields[2]->name()));
+  ASSERT_EQ(3, schema->GetFieldIndex(fields[3]->name()));
+  ASSERT_EQ(-1, schema->GetFieldIndex("not-found"));
+}
+
 TEST_F(TestSchema, TestMetadataConstruction) {
   auto f0 = field("f0", int32());
   auto f1 = field("f1", uint8(), false);
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index afb30272c0110..891045e689b51 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -265,7 +265,12 @@ bool Schema::Equals(const Schema& other) const {
   return true;
 }
 
-std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
+std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) const {
+  int64_t i = GetFieldIndex(name);
+  return i == -1 ? nullptr : fields_[i];
+}
+
+int64_t Schema::GetFieldIndex(const std::string& name) const {
   if (fields_.size() > 0 && name_to_index_.size() == 0) {
     for (size_t i = 0; i < fields_.size(); ++i) {
       name_to_index_[fields_[i]->name()] = static_cast<int>(i);
@@ -274,9 +279,9 @@ std::shared_ptr<Field> Schema::GetFieldByName(const std::string& name) {
 
   auto it = name_to_index_.find(name);
   if (it == name_to_index_.end()) {
-    return nullptr;
+    return -1;
   } else {
-    return fields_[it->second];
+    return it->second;
   }
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 40615f790fb0e..3e85291f043e4 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -699,7 +699,10 @@ class ARROW_EXPORT Schema {
   std::shared_ptr<Field> field(int i) const { return fields_[i]; }
 
   // Returns nullptr if name not found
-  std::shared_ptr<Field> GetFieldByName(const std::string& name);
+  std::shared_ptr<Field> GetFieldByName(const std::string& name) const;
+
+  // Returns -1 if name not found
+  int64_t GetFieldIndex(const std::string& name) const;
 
   const std::vector<std::shared_ptr<Field>>& fields() const { return fields_; }
   std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
@@ -720,7 +723,7 @@ class ARROW_EXPORT Schema {
 
  private:
   std::vector<std::shared_ptr<Field>> fields_;
-  std::unordered_map<std::string, int> name_to_index_;
+  mutable std::unordered_map<std::string, int> name_to_index_;
 
   std::shared_ptr<const KeyValueMetadata> metadata_;
 };
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 632a443ed0deb..0f34121f653c2 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -104,7 +104,8 @@ def jemalloc_memory_pool():
 from pyarrow.ipc import (RecordBatchFileReader, RecordBatchFileWriter,
                          RecordBatchStreamReader, RecordBatchStreamWriter,
                          open_stream,
-                         open_file)
+                         open_file,
+                         serialize_pandas, deserialize_pandas)
 
 
 localfs = LocalFilesystem.get_instance()
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 9f6edc0b31dc6..2f6b9a9d64898 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -20,6 +20,7 @@
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
                                         CTable, CMemoryPool,
+                                        CKeyValueMetadata,
                                         RandomAccessFile, OutputStream)
 
 
@@ -164,6 +165,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 
         unique_ptr[CRowGroupMetaData] RowGroup(int i)
         const SchemaDescriptor* schema()
+        shared_ptr[const CKeyValueMetadata] key_value_metadata() const
 
     cdef cppclass ReaderProperties:
         pass
@@ -229,8 +231,11 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 
 cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
     CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema,
+                              const shared_ptr[const CKeyValueMetadata]& key_value_metadata,
                               shared_ptr[CSchema]* out)
+
     CStatus ToParquetSchema(const CSchema* arrow_schema,
+                            const shared_ptr[const CKeyValueMetadata]& key_value_metadata,
                             shared_ptr[SchemaDescriptor]* out)
 
 
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 51bd938c79a5e..77ef7adadef7e 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -40,15 +40,15 @@ cdef class RowGroupMetaData:
     cdef:
         unique_ptr[CRowGroupMetaData] up_metadata
         CRowGroupMetaData* metadata
-        object parent
+        FileMetaData parent
 
     def __cinit__(self):
         pass
 
-    cdef init_from_file(self, FileMetaData parent, int i):
+    cdef void init_from_file(self, FileMetaData parent, int i):
         if i < 0 or i >= parent.num_row_groups:
             raise IndexError('{0} out of bounds'.format(i))
-        self.up_metadata = parent.metadata.RowGroup(i)
+        self.up_metadata = parent._metadata.RowGroup(i)
         self.metadata = self.up_metadata.get()
         self.parent = parent
 
@@ -80,15 +80,15 @@ cdef class RowGroupMetaData:
 cdef class FileMetaData:
     cdef:
         shared_ptr[CFileMetaData] sp_metadata
-        CFileMetaData* metadata
-        object _schema
+        CFileMetaData* _metadata
+        ParquetSchema _schema
 
     def __cinit__(self):
         pass
 
     cdef init(self, const shared_ptr[CFileMetaData]& metadata):
         self.sp_metadata = metadata
-        self.metadata = metadata.get()
+        self._metadata = metadata.get()
 
     def __repr__(self):
         return """{0}
@@ -116,27 +116,27 @@ cdef class FileMetaData:
     property serialized_size:
 
         def __get__(self):
-            return self.metadata.size()
+            return self._metadata.size()
 
     property num_columns:
 
         def __get__(self):
-            return self.metadata.num_columns()
+            return self._metadata.num_columns()
 
     property num_rows:
 
         def __get__(self):
-            return self.metadata.num_rows()
+            return self._metadata.num_rows()
 
     property num_row_groups:
 
         def __get__(self):
-            return self.metadata.num_row_groups()
+            return self._metadata.num_row_groups()
 
     property format_version:
 
         def __get__(self):
-            cdef ParquetVersion version = self.metadata.version()
+            cdef ParquetVersion version = self._metadata.version()
             if version == ParquetVersion_V1:
                 return '1.0'
             if version == ParquetVersion_V2:
@@ -149,7 +149,7 @@ cdef class FileMetaData:
     property created_by:
 
         def __get__(self):
-            return frombytes(self.metadata.created_by())
+            return frombytes(self._metadata.created_by())
 
     def row_group(self, int i):
         """
@@ -159,14 +159,26 @@ cdef class FileMetaData:
         result.init_from_file(self, i)
         return result
 
+    property metadata:
+
+        def __get__(self):
+            cdef:
+                unordered_map[c_string, c_string] metadata
+                const CKeyValueMetadata* underlying_metadata
+            underlying_metadata = self._metadata.key_value_metadata().get()
+            if underlying_metadata != NULL:
+                underlying_metadata.ToUnorderedMap(&metadata)
+                return metadata
+            else:
+                return None
+
 
 cdef class ParquetSchema:
     cdef:
-        object parent  # the FileMetaData owning the SchemaDescriptor
+        FileMetaData parent  # the FileMetaData owning the SchemaDescriptor
         const SchemaDescriptor* schema
 
     def __cinit__(self):
-        self.parent = None
         self.schema = NULL
 
     def __repr__(self):
@@ -186,7 +198,7 @@ cdef class ParquetSchema:
 
     cdef init_from_filemeta(self, FileMetaData container):
         self.parent = container
-        self.schema = container.metadata.schema()
+        self.schema = container._metadata.schema()
 
     def __len__(self):
         return self.schema.num_columns()
@@ -211,7 +223,9 @@ cdef class ParquetSchema:
             shared_ptr[CSchema] sp_arrow_schema
 
         with nogil:
-            check_status(FromParquetSchema(self.schema, &sp_arrow_schema))
+            check_status(FromParquetSchema(
+                self.schema, self.parent._metadata.key_value_metadata(),
+                &sp_arrow_schema))
 
         return pyarrow_wrap_schema(sp_arrow_schema)
 
@@ -232,7 +246,7 @@ cdef class ParquetSchema:
 
 cdef class ColumnSchema:
     cdef:
-        object parent
+        ParquetSchema parent
         const ColumnDescriptor* descr
 
     def __cinit__(self):
@@ -463,7 +477,7 @@ cdef class ParquetReader:
         """
         cdef:
             FileMetaData container = self.metadata
-            const CFileMetaData* metadata = container.metadata
+            const CFileMetaData* metadata = container._metadata
             int i = 0
 
         if self.column_idx_map is None:
@@ -488,12 +502,13 @@ cdef class ParquetReader:
         return array
 
 
-cdef check_compression_name(name):
+cdef int check_compression_name(name) except -1:
     if name.upper() not in ['NONE', 'SNAPPY', 'GZIP', 'LZO', 'BROTLI']:
         raise ArrowException("Unsupported compression: " + name)
+    return 0
 
 
-cdef ParquetCompression compression_from_name(object name):
+cdef ParquetCompression compression_from_name(str name):
     name = name.upper()
     if name == "SNAPPY":
         return ParquetCompression_SNAPPY
@@ -546,7 +561,7 @@ cdef class ParquetWriter:
                             maybe_unbox_memory_pool(memory_pool),
                             sink, properties, &self.writer))
 
-    cdef _set_version(self, WriterProperties.Builder* props):
+    cdef void _set_version(self, WriterProperties.Builder* props):
         if self.version is not None:
             if self.version == "1.0":
                 props.version(ParquetVersion_V1)
@@ -555,7 +570,7 @@ cdef class ParquetWriter:
             else:
                 raise ArrowException("Unsupported Parquet format version")
 
-    cdef _set_compression_props(self, WriterProperties.Builder* props):
+    cdef void _set_compression_props(self, WriterProperties.Builder* props):
         if isinstance(self.compression, basestring):
             check_compression_name(self.compression)
             props.compression(compression_from_name(self.compression))
@@ -564,7 +579,7 @@ cdef class ParquetWriter:
                 check_compression_name(codec)
                 props.compression(column, compression_from_name(codec))
 
-    cdef _set_dictionary_props(self, WriterProperties.Builder* props):
+    cdef void _set_dictionary_props(self, WriterProperties.Builder* props):
         if isinstance(self.use_dictionary, bool):
             if self.use_dictionary:
                 props.enable_dictionary()
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index c132269c56311..5930de392715b 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -159,7 +159,7 @@ cdef class Field:
     def __cinit__(self):
         pass
 
-    cdef init(self, const shared_ptr[CField]& field):
+    cdef void init(self, const shared_ptr[CField]& field):
         self.sp_field = field
         self.field = field.get()
         self.type = pyarrow_wrap_data_type(field.get().type())
@@ -264,11 +264,11 @@ cdef class Schema:
 
         return result
 
-    cdef init(self, const vector[shared_ptr[CField]]& fields):
+    cdef void init(self, const vector[shared_ptr[CField]]& fields):
         self.schema = new CSchema(fields)
         self.sp_schema.reset(self.schema)
 
-    cdef init_schema(self, const shared_ptr[CSchema]& schema):
+    cdef void init_schema(self, const shared_ptr[CSchema]& schema):
         self.schema = schema.get()
         self.sp_schema = schema
 
@@ -310,6 +310,9 @@ cdef class Schema:
         """
         return pyarrow_wrap_field(self.schema.GetFieldByName(tobytes(name)))
 
+    def get_field_index(self, name):
+        return self.schema.GetFieldIndex(tobytes(name))
+
     def add_metadata(self, dict metadata):
         """
         Add metadata as dict of string keys and values to Schema
@@ -352,9 +355,9 @@ cdef class Schema:
         return self.__str__()
 
 
-cdef box_metadata(const CKeyValueMetadata* metadata):
+cdef dict box_metadata(const CKeyValueMetadata* metadata):
     cdef unordered_map[c_string, c_string] result
-    if metadata != NULL:
+    if metadata != nullptr:
         metadata.ToUnorderedMap(&result)
         return result
     else:
@@ -813,45 +816,60 @@ cdef class Date64Value(ArrayValue):
             ap.Value(self.index) / 1000).date()
 
 
+cdef dict DATETIME_CONVERSION_FUNCTIONS
+
+try:
+    import pandas as pd
+except ImportError:
+    DATETIME_CONVERSION_FUNCTIONS = {
+        TimeUnit_SECOND: lambda x, tzinfo: (
+            datetime.datetime.utcfromtimestamp(x).replace(tzinfo=tzinfo)
+        ),
+        TimeUnit_MILLI: lambda x, tzinfo: (
+            datetime.datetime.utcfromtimestamp(x / 1e3).replace(tzinfo=tzinfo)
+        ),
+        TimeUnit_MICRO: lambda x, tzinfo: (
+            datetime.datetime.utcfromtimestamp(x / 1e6).replace(tzinfo=tzinfo)
+        ),
+    }
+else:
+    DATETIME_CONVERSION_FUNCTIONS = {
+        TimeUnit_SECOND: lambda x, tzinfo: pd.Timestamp(
+            x * 1000000000, tz=tzinfo, unit='ns',
+        ),
+        TimeUnit_MILLI: lambda x, tzinfo: pd.Timestamp(
+            x * 1000000, tz=tzinfo, unit='ns',
+        ),
+        TimeUnit_MICRO: lambda x, tzinfo: pd.Timestamp(
+            x * 1000, tz=tzinfo, unit='ns',
+        ),
+        TimeUnit_NANO: lambda x, tzinfo: pd.Timestamp(
+            x, tz=tzinfo, unit='ns',
+        )
+    }
+
+
 cdef class TimestampValue(ArrayValue):
 
     def as_py(self):
         cdef:
             CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
-            CTimestampType* dtype = <CTimestampType*>ap.type().get()
-            int64_t val = ap.Value(self.index)
+            CTimestampType* dtype = <CTimestampType*> ap.type().get()
+            int64_t value = ap.Value(self.index)
 
-        timezone = None
-        tzinfo = None
-        if dtype.timezone().size() > 0:
-            timezone = frombytes(dtype.timezone())
+        if not dtype.timezone().empty():
             import pytz
-            tzinfo = pytz.timezone(timezone)
+            tzinfo = pytz.timezone(frombytes(dtype.timezone()))
+        else:
+            tzinfo = None
 
         try:
-            pd = _pandas()
-            if dtype.unit() == TimeUnit_SECOND:
-                val = val * 1000000000
-            elif dtype.unit() == TimeUnit_MILLI:
-                val = val * 1000000
-            elif dtype.unit() == TimeUnit_MICRO:
-                val = val * 1000
-            return pd.Timestamp(val, tz=tzinfo)
-        except ImportError:
-            if dtype.unit() == TimeUnit_SECOND:
-                result = datetime.datetime.utcfromtimestamp(val)
-            elif dtype.unit() == TimeUnit_MILLI:
-                result = datetime.datetime.utcfromtimestamp(float(val) / 1000)
-            elif dtype.unit() == TimeUnit_MICRO:
-                result = datetime.datetime.utcfromtimestamp(
-                    float(val) / 1000000)
-            else:
-                # TimeUnit_NANO
-                raise NotImplementedError("Cannot convert nanosecond "
-                                          "timestamps without pandas")
-            if timezone is not None:
-                result = result.replace(tzinfo=tzinfo)
-            return result
+            converter = DATETIME_CONVERSION_FUNCTIONS[dtype.unit()]
+        except KeyError:
+            raise ValueError(
+                'Cannot convert nanosecond timestamps without pandas'
+            )
+        return converter(value, tzinfo=tzinfo)
 
 
 cdef class FloatValue(ArrayValue):
@@ -1042,7 +1060,7 @@ def array(object sequence, DataType type=None, MemoryPool memory_pool=None):
 
 cdef class Array:
 
-    cdef init(self, const shared_ptr[CArray]& sp_array):
+    cdef void init(self, const shared_ptr[CArray]& sp_array):
         self.sp_array = sp_array
         self.ap = sp_array.get()
         self.type = pyarrow_wrap_data_type(self.sp_array.get().type())
@@ -1251,7 +1269,7 @@ cdef class Array:
 
 cdef class Tensor:
 
-    cdef init(self, const shared_ptr[CTensor]& sp_tensor):
+    cdef void init(self, const shared_ptr[CTensor]& sp_tensor):
         self.sp_tensor = sp_tensor
         self.tp = sp_tensor.get()
         self.type = pyarrow_wrap_data_type(self.tp.type())
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index cc3b4b6fdaf92..73bfb4ff50896 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -18,7 +18,7 @@
 # distutils: language = c++
 
 from libc.stdint cimport *
-from libcpp cimport bool as c_bool
+from libcpp cimport bool as c_bool, nullptr
 from libcpp.memory cimport shared_ptr, unique_ptr, make_shared
 from libcpp.string cimport string as c_string
 from libcpp.vector cimport vector
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index a7e2733ca815c..9df31c80ccf62 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -29,6 +29,7 @@ cdef extern from "arrow/util/key_value_metadata.h" namespace "arrow" nogil:
         void Append(const c_string& key, const c_string& value)
         void ToUnorderedMap(unordered_map[c_string, c_string]*) const
 
+
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     enum Type" arrow::Type::type":
@@ -205,7 +206,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         shared_ptr[CField] field(int i)
         shared_ptr[const CKeyValueMetadata] metadata()
-        shared_ptr[CField] GetFieldByName(c_string& name)
+        shared_ptr[CField] GetFieldByName(const c_string& name)
+        int64_t GetFieldIndex(const c_string& name)
         int num_fields()
         c_string ToString()
 
@@ -686,8 +688,10 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     cdef cppclass PyBytesReader(CBufferReader):
         PyBytesReader(object fo)
 
+
 cdef extern from 'arrow/python/init.h':
     int arrow_init_numpy() except -1
 
+
 cdef extern from 'arrow/python/config.h' namespace 'arrow::py':
     void set_numpy_nan(object o)
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index a153f22270087..23eb6ef10823e 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -462,7 +462,7 @@ cdef class Buffer:
     def __cinit__(self):
         pass
 
-    cdef init(self, const shared_ptr[CBuffer]& buffer):
+    cdef void init(self, const shared_ptr[CBuffer]& buffer):
         self.buffer = buffer
         self.shape[0] = self.size
         self.strides[0] = <Py_ssize_t>(1)
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index 8338de33a0d2c..a61d746d65274 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -17,6 +17,7 @@
 
 # Arrow file and stream reader/writer classes, and other messaging tools
 
+import pyarrow as pa
 import pyarrow.lib as lib
 
 
@@ -119,3 +120,42 @@ def open_file(source, footer_offset=None):
     reader : RecordBatchFileReader
     """
     return RecordBatchFileReader(source, footer_offset=footer_offset)
+
+
+def serialize_pandas(df):
+    """Serialize a pandas DataFrame into a buffer protocol compatible object.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+
+    Returns
+    -------
+    buf : buffer
+        An object compatible with the buffer protocol
+    """
+    batch = pa.RecordBatch.from_pandas(df)
+    sink = pa.InMemoryOutputStream()
+    writer = pa.RecordBatchFileWriter(sink, batch.schema)
+    writer.write_batch(batch)
+    writer.close()
+    return sink.get_result()
+
+
+def deserialize_pandas(buf, nthreads=1):
+    """Deserialize a buffer protocol compatible object into a pandas DataFrame.
+
+    Parameters
+    ----------
+    buf : buffer
+        An object compatible with the buffer protocol
+    nthreads : int, optional
+        The number of threads to use to convert the buffer to a DataFrame.
+
+    Returns
+    -------
+    df : pandas.DataFrame
+    """
+    buffer_reader = pa.BufferReader(buf)
+    reader = pa.RecordBatchFileReader(buffer_reader)
+    return reader.read_all().to_pandas(nthreads=nthreads)
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index d3d03aaaefaa1..4a2ab8698f693 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -33,7 +33,7 @@ cdef class MemoryPool:
     cdef:
         CMemoryPool* pool
 
-    cdef init(self, CMemoryPool* pool)
+    cdef void init(self, CMemoryPool* pool)
 
 
 cdef class LoggingMemoryPool(MemoryPool):
@@ -89,7 +89,7 @@ cdef class Field:
     cdef readonly:
         DataType type
 
-    cdef init(self, const shared_ptr[CField]& field)
+    cdef void init(self, const shared_ptr[CField]& field)
 
 
 cdef class Schema:
@@ -97,8 +97,8 @@ cdef class Schema:
         shared_ptr[CSchema] sp_schema
         CSchema* schema
 
-    cdef init(self, const vector[shared_ptr[CField]]& fields)
-    cdef init_schema(self, const shared_ptr[CSchema]& schema)
+    cdef void init(self, const vector[shared_ptr[CField]]& fields)
+    cdef void init_schema(self, const shared_ptr[CSchema]& schema)
 
 
 cdef class Scalar:
@@ -155,7 +155,7 @@ cdef class Array:
     cdef readonly:
         DataType type
 
-    cdef init(self, const shared_ptr[CArray]& sp_array)
+    cdef void init(self, const shared_ptr[CArray]& sp_array)
     cdef getitem(self, int64_t i)
 
 
@@ -167,7 +167,7 @@ cdef class Tensor:
     cdef readonly:
         DataType type
 
-    cdef init(self, const shared_ptr[CTensor]& sp_tensor)
+    cdef void init(self, const shared_ptr[CTensor]& sp_tensor)
 
 
 cdef class NullArray(Array):
@@ -266,8 +266,8 @@ cdef class ChunkedArray:
         shared_ptr[CChunkedArray] sp_chunked_array
         CChunkedArray* chunked_array
 
-    cdef init(self, const shared_ptr[CChunkedArray]& chunked_array)
-    cdef _check_nullptr(self)
+    cdef void init(self, const shared_ptr[CChunkedArray]& chunked_array)
+    cdef int _check_nullptr(self) except -1
 
 
 cdef class Column:
@@ -275,8 +275,8 @@ cdef class Column:
         shared_ptr[CColumn] sp_column
         CColumn* column
 
-    cdef init(self, const shared_ptr[CColumn]& column)
-    cdef _check_nullptr(self)
+    cdef void init(self, const shared_ptr[CColumn]& column)
+    cdef int _check_nullptr(self) except -1
 
 
 cdef class Table:
@@ -284,8 +284,8 @@ cdef class Table:
         shared_ptr[CTable] sp_table
         CTable* table
 
-    cdef init(self, const shared_ptr[CTable]& table)
-    cdef _check_nullptr(self)
+    cdef void init(self, const shared_ptr[CTable]& table)
+    cdef int _check_nullptr(self) except -1
 
 
 cdef class RecordBatch:
@@ -294,8 +294,8 @@ cdef class RecordBatch:
         CRecordBatch* batch
         Schema _schema
 
-    cdef init(self, const shared_ptr[CRecordBatch]& table)
-    cdef _check_nullptr(self)
+    cdef void init(self, const shared_ptr[CRecordBatch]& table)
+    cdef int _check_nullptr(self) except -1
 
 
 cdef class Buffer:
@@ -304,7 +304,7 @@ cdef class Buffer:
         Py_ssize_t shape[1]
         Py_ssize_t strides[1]
 
-    cdef init(self, const shared_ptr[CBuffer]& buffer)
+    cdef void init(self, const shared_ptr[CBuffer]& buffer)
 
 
 cdef class NativeFile:
@@ -335,3 +335,5 @@ cdef public object pyarrow_wrap_tensor(const shared_ptr[CTensor]& sp_tensor)
 cdef public object pyarrow_wrap_column(const shared_ptr[CColumn]& ccolumn)
 cdef public object pyarrow_wrap_table(const shared_ptr[CTable]& ctable)
 cdef public object pyarrow_wrap_batch(const shared_ptr[CRecordBatch]& cbatch)
+
+cdef dict box_metadata(const CKeyValueMetadata* sp_metadata)
diff --git a/python/pyarrow/memory.pxi b/python/pyarrow/memory.pxi
index 15d59d237ad6d..6671a0125502d 100644
--- a/python/pyarrow/memory.pxi
+++ b/python/pyarrow/memory.pxi
@@ -21,7 +21,7 @@
 
 
 cdef class MemoryPool:
-    cdef init(self, CMemoryPool* pool):
+    cdef void init(self, CMemoryPool* pool):
         self.pool = pool
 
     def bytes_allocated(self):
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
new file mode 100644
index 0000000000000..2f72d6a6382f9
--- /dev/null
+++ b/python/pyarrow/pandas_compat.py
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import re
+import json
+import pandas as pd
+
+import six
+
+from pyarrow.compat import PY2
+
+
+INDEX_LEVEL_NAME_REGEX = re.compile(r'^__index_level_\d+__$')
+
+
+def is_unnamed_index_level(name):
+    return INDEX_LEVEL_NAME_REGEX.match(name) is not None
+
+
+def infer_dtype(column):
+    try:
+        return pd.api.types.infer_dtype(column)
+    except AttributeError:
+        return pd.lib.infer_dtype(column)
+
+
+def get_column_metadata(column, name):
+    inferred_dtype = infer_dtype(column)
+    dtype = column.dtype
+
+    if hasattr(dtype, 'categories'):
+        extra_metadata = {
+            'num_categories': len(column.cat.categories),
+            'ordered': column.cat.ordered,
+        }
+    elif hasattr(dtype, 'tz'):
+        extra_metadata = {'timezone': str(dtype.tz)}
+    else:
+        extra_metadata = None
+
+    if not isinstance(name, six.string_types):
+        raise TypeError(
+            'Column name must be a string. Got column {} of type {}'.format(
+                name, type(name).__name__
+            )
+        )
+
+    return {
+        'name': name,
+        'pandas_type': {
+            'string': 'bytes' if PY2 else 'unicode',
+            'datetime64': (
+                'datetimetz' if hasattr(dtype, 'tz')
+                else 'datetime'
+            ),
+            'integer': str(dtype),
+            'floating': str(dtype),
+        }.get(inferred_dtype, inferred_dtype),
+        'numpy_dtype': str(dtype),
+        'metadata': extra_metadata,
+    }
+
+
+def index_level_name(index, i):
+    return index.name or '__index_level_{:d}__'.format(i)
+
+
+def construct_metadata(df, index_levels, preserve_index):
+    return {
+        b'pandas': json.dumps(
+            {
+                'index_columns': [
+                    index_level_name(level, i)
+                    for i, level in enumerate(index_levels)
+                ] if preserve_index else [],
+                'columns': [
+                    get_column_metadata(df[name], name=name)
+                    for name in df.columns
+                ] + (
+                    [
+                        get_column_metadata(
+                            level, name=index_level_name(level, i)
+                        )
+                        for i, level in enumerate(index_levels)
+                    ] if preserve_index else []
+                ),
+                'pandas_version': pd.__version__,
+            }
+        ).encode('utf8')
+    }
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 050ec3176d799..e69d85eac4e64 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -15,6 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import itertools
+import json
+
 import six
 
 import numpy as np
@@ -99,6 +102,25 @@ def read(self, columns=None, nthreads=1):
             Content of the file as a table (of columns)
         """
         column_indices = self._get_column_indices(columns)
+        if nthreads is not None:
+            self.reader.set_num_threads(nthreads)
+
+        return self.reader.read_all(column_indices=column_indices)
+
+    def read_pandas(self, columns=None, nthreads=1):
+        column_indices = self._get_column_indices(columns)
+        custom_metadata = self.metadata.metadata
+
+        if custom_metadata and b'pandas' in custom_metadata:
+            index_columns = json.loads(
+                custom_metadata[b'pandas'].decode('utf8')
+            )['index_columns']
+        else:
+            index_columns = []
+
+        if column_indices is not None and index_columns:
+            column_indices += map(self.reader.column_name_idx, index_columns)
+
         if nthreads is not None:
             self.reader.set_num_threads(nthreads)
         return self.reader.read_all(column_indices=column_indices)
@@ -106,9 +128,7 @@ def read(self, columns=None, nthreads=1):
     def _get_column_indices(self, column_names):
         if column_names is None:
             return None
-        else:
-            return [self.reader.column_name_idx(column)
-                    for column in column_names]
+        return list(map(self.reader.column_name_idx, column_names))
 
 
 # ----------------------------------------------------------------------
@@ -618,6 +638,43 @@ def read_table(source, columns=None, nthreads=1, metadata=None):
     return pf.read(columns=columns, nthreads=nthreads)
 
 
+def read_pandas(source, columns=None, nthreads=1, metadata=None):
+    """
+    Read a Table from Parquet format, reconstructing the index values if
+    available.
+
+    Parameters
+    ----------
+    source: str or pyarrow.io.NativeFile
+        Location of Parquet dataset. If a string passed, can be a single file
+        name. For passing Python file objects or byte buffers,
+        see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
+    columns: list
+        If not None, only these columns will be read from the file.
+    nthreads : int, default 1
+        Number of columns to read in parallel. Requires that the underlying
+        file source is threadsafe
+    metadata : FileMetaData
+        If separately computed
+
+    Returns
+    -------
+    pyarrow.Table
+        Content of the file as a Table of Columns, including DataFrame indexes
+        as Columns.
+    """
+    if is_string(source):
+        fs = LocalFilesystem.get_instance()
+        if fs.isdir(source):
+            raise NotImplementedError(
+                'Reading a directory of Parquet files with DataFrame index '
+                'metadata is not yet supported'
+            )
+
+    pf = ParquetFile(source, metadata=metadata)
+    return pf.read_pandas(columns=columns, nthreads=nthreads)
+
+
 def write_table(table, where, row_group_size=None, version='1.0',
                 use_dictionary=True, compression='snappy', **kwargs):
     """
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index bd8cce41400a9..c9915c1be3c5c 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -15,8 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import json
+
 from collections import OrderedDict
 
+try:
+    import pandas as pd
+except ImportError:
+    # The pure-Python based API works without a pandas installation
+    pass
+else:
+    import pyarrow.pandas_compat as pdcompat
+
 
 cdef class ChunkedArray:
     """
@@ -30,14 +40,18 @@ cdef class ChunkedArray:
     def __cinit__(self):
         self.chunked_array = NULL
 
-    cdef init(self, const shared_ptr[CChunkedArray]& chunked_array):
+    cdef void init(self, const shared_ptr[CChunkedArray]& chunked_array):
         self.sp_chunked_array = chunked_array
         self.chunked_array = chunked_array.get()
 
-    cdef _check_nullptr(self):
+    cdef int _check_nullptr(self) except -1:
         if self.chunked_array == NULL:
-            raise ReferenceError("ChunkedArray object references a NULL "
-                                 "pointer. Not initialized.")
+            raise ReferenceError(
+                "{} object references a NULL pointer. Not initialized.".format(
+                    type(self).__name__
+                )
+            )
+        return 0
 
     def length(self):
         self._check_nullptr()
@@ -111,7 +125,7 @@ cdef class Column:
     def __cinit__(self):
         self.column = NULL
 
-    cdef init(self, const shared_ptr[CColumn]& column):
+    cdef void init(self, const shared_ptr[CColumn]& column):
         self.sp_column = column
         self.column = column.get()
 
@@ -142,7 +156,7 @@ cdef class Column:
         check_status(libarrow.ConvertColumnToPandas(self.sp_column,
                                                     self, &out))
 
-        return _pandas().Series(wrap_array_output(out), name=self.name)
+        return pd.Series(wrap_array_output(out), name=self.name)
 
     def equals(self, Column other):
         """
@@ -175,14 +189,17 @@ cdef class Column:
         """
         return self.data.to_pylist()
 
-    cdef _check_nullptr(self):
+    cdef int _check_nullptr(self) except -1:
         if self.column == NULL:
-            raise ReferenceError("Column object references a NULL pointer."
-                    "Not initialized.")
+            raise ReferenceError(
+                "{} object references a NULL pointer. Not initialized.".format(
+                    type(self).__name__
+                )
+            )
+        return 0
 
     def __len__(self):
-        self._check_nullptr()
-        return self.column.length()
+        return self.length()
 
     def length(self):
         self._check_nullptr()
@@ -248,8 +265,9 @@ cdef class Column:
         return chunked_array
 
 
-cdef shared_ptr[const CKeyValueMetadata] key_value_metadata_from_dict(
-    dict metadata):
+cdef shared_ptr[const CKeyValueMetadata] unbox_metadata(dict metadata):
+    if metadata is None:
+        return <shared_ptr[const CKeyValueMetadata]> nullptr
     cdef:
         unordered_map[c_string, c_string] unordered_metadata = metadata
     return (<shared_ptr[const CKeyValueMetadata]>
@@ -289,27 +307,45 @@ cdef int _schema_from_arrays(
     else:
         raise TypeError(type(arrays[0]))
 
-    schema.reset(new CSchema(fields, key_value_metadata_from_dict(metadata)))
+    schema.reset(new CSchema(fields, unbox_metadata(metadata)))
     return 0
 
 
-cdef tuple _dataframe_to_arrays(df, bint timestamps_to_ms, Schema schema):
+cdef tuple _dataframe_to_arrays(
+    df,
+    bint timestamps_to_ms,
+    Schema schema,
+    bint preserve_index
+):
     cdef:
         list names = []
         list arrays = []
+        list index_levels = []
         DataType type = None
-        dict metadata = {}
+        dict metadata
+
+    if preserve_index:
+        index_levels.extend(getattr(df.index, 'levels', [df.index]))
 
     for name in df.columns:
         col = df[name]
         if schema is not None:
             type = schema.field_by_name(name).type
 
-        arr = Array.from_pandas(col, type=type,
-                                timestamps_to_ms=timestamps_to_ms)
+        arr = arrays.append(
+            Array.from_pandas(
+                col, type=type, timestamps_to_ms=timestamps_to_ms
+            )
+        )
         names.append(name)
-        arrays.append(arr)
 
+    for i, level in enumerate(index_levels):
+        arrays.append(
+            Array.from_pandas(level, timestamps_to_ms=timestamps_to_ms)
+        )
+        names.append(pdcompat.index_level_name(level, i))
+
+    metadata = pdcompat.construct_metadata(df, index_levels, preserve_index)
     return names, arrays, metadata
 
 
@@ -327,13 +363,18 @@ cdef class RecordBatch:
         self.batch = NULL
         self._schema = None
 
-    cdef init(self, const shared_ptr[CRecordBatch]& batch):
+    cdef void init(self, const shared_ptr[CRecordBatch]& batch):
         self.sp_batch = batch
         self.batch = batch.get()
 
-    cdef _check_nullptr(self):
+    cdef int _check_nullptr(self) except -1:
         if self.batch == NULL:
-            raise ReferenceError("Object not initialized")
+            raise ReferenceError(
+                "{} object references a NULL pointer. Not initialized.".format(
+                    type(self).__name__
+                )
+            )
+        return 0
 
     def __len__(self):
         self._check_nullptr()
@@ -455,22 +496,27 @@ cdef class RecordBatch:
         return Table.from_batches([self]).to_pandas(nthreads=nthreads)
 
     @classmethod
-    def from_pandas(cls, df, schema=None):
+    def from_pandas(cls, df, Schema schema=None, bint preserve_index=True):
         """
         Convert pandas.DataFrame to an Arrow RecordBatch
 
         Parameters
         ----------
         df: pandas.DataFrame
-        schema: pyarrow.Schema (optional)
+        schema: pyarrow.Schema, optional
             The expected schema of the RecordBatch. This can be used to
             indicate the type of columns if we cannot infer it automatically.
+        preserve_index : bool, optional
+            Whether to store the index as an additional column in the resulting
+            ``RecordBatch``.
 
         Returns
         -------
         pyarrow.RecordBatch
         """
-        names, arrays, metadata = _dataframe_to_arrays(df, False, schema)
+        names, arrays, metadata = _dataframe_to_arrays(
+            df, False, schema, preserve_index
+        )
         return cls.from_arrays(arrays, names, metadata)
 
     @staticmethod
@@ -503,7 +549,7 @@ cdef class RecordBatch:
             raise ValueError('Record batch cannot contain no arrays (for now)')
 
         num_rows = len(arrays[0])
-        _schema_from_arrays(arrays, names, metadata or {}, &schema)
+        _schema_from_arrays(arrays, names, metadata, &schema)
 
         c_arrays.reserve(len(arrays))
         for arr in arrays:
@@ -513,19 +559,55 @@ cdef class RecordBatch:
         return pyarrow_wrap_batch(batch)
 
 
-cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
-    cdef:
-        PyObject* result_obj
-        CColumn* col
-        int i
-
+cdef table_to_blockmanager(const shared_ptr[CTable]& ctable, int nthreads):
     import pandas.core.internals as _int
     from pandas import RangeIndex, Categorical
     from pyarrow.compat import DatetimeTZDtype
 
+    cdef:
+        Table table = pyarrow_wrap_table(ctable)
+        Table block_table = pyarrow_wrap_table(ctable)
+        Schema schema = table.schema
+
+        size_t row_count = table.num_rows
+        size_t total_columns = table.num_columns
+
+        dict metadata = schema.metadata
+        dict pandas_metadata = None
+
+        list index_columns = []
+        list index_arrays = []
+
+    if metadata is not None and b'pandas' in metadata:
+        pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
+        index_columns = pandas_metadata['index_columns']
+
+    cdef:
+        Column col
+        int64_t i
+
+    for name in index_columns:
+        i = schema.get_field_index(name)
+        if i != -1:
+            col = table.column(i)
+            index_name = None if pdcompat.is_unnamed_index_level(name) else name
+            index_arrays.append(
+                pd.Index(col.to_pandas().values, name=index_name)
+            )
+            block_table = block_table.remove_column(
+                block_table.schema.get_field_index(name)
+            )
+
+    cdef:
+        PyObject* result_obj
+        shared_ptr[CTable] c_block_table = block_table.sp_table
+
     with nogil:
-        check_status(libarrow.ConvertTableToPandas(table, nthreads,
-                                                   &result_obj))
+        check_status(
+            libarrow.ConvertTableToPandas(
+                c_block_table, nthreads, &result_obj
+            )
+        )
 
     result = PyObject_to_object(result_obj)
 
@@ -549,12 +631,13 @@ cdef table_to_blockmanager(const shared_ptr[CTable]& table, int nthreads):
             block = _int.make_block(block_arr, placement=placement)
         blocks.append(block)
 
-    names = []
-    for i in range(table.get().num_columns()):
-        col = table.get().column(i).get()
-        names.append(frombytes(col.name()))
+    cdef list axes = [
+        [column.name for column in block_table.itercolumns()],
+        pd.MultiIndex.from_arrays(
+            index_arrays
+        ) if index_arrays else pd.RangeIndex(row_count),
+    ]
 
-    axes = [names, RangeIndex(table.get().num_rows())]
     return _int.BlockManager(blocks, axes)
 
 
@@ -572,16 +655,18 @@ cdef class Table:
         self.table = NULL
 
     def __repr__(self):
-        return 'pyarrow.Table\n{0}'.format(str(self.schema))
+        return 'pyarrow.{}\n{}'.format(type(self).__name__, str(self.schema))
 
-    cdef init(self, const shared_ptr[CTable]& table):
+    cdef void init(self, const shared_ptr[CTable]& table):
         self.sp_table = table
         self.table = table.get()
 
-    cdef _check_nullptr(self):
-        if self.table == NULL:
-            raise ReferenceError("Table object references a NULL pointer."
-                    "Not initialized.")
+    cdef int _check_nullptr(self) except -1:
+        if self.table == nullptr:
+            raise ReferenceError(
+                "Table object references a NULL pointer. Not initialized."
+            )
+        return 0
 
     def equals(self, Table other):
         """
@@ -609,22 +694,29 @@ cdef class Table:
         return result
 
     @classmethod
-    def from_pandas(cls, df, timestamps_to_ms=False, schema=None):
+    def from_pandas(
+        cls,
+        df,
+        bint timestamps_to_ms=False,
+        Schema schema=None,
+        bint preserve_index=True
+    ):
         """
         Convert pandas.DataFrame to an Arrow Table
 
         Parameters
         ----------
-        df: pandas.DataFrame
-
-        timestamps_to_ms: bool
+        df : pandas.DataFrame
+        timestamps_to_ms : bool
             Convert datetime columns to ms resolution. This is needed for
             compability with other functionality like Parquet I/O which
             only supports milliseconds.
-
-        schema: pyarrow.Schema (optional)
+        schema : pyarrow.Schema, optional
             The expected schema of the Arrow Table. This can be used to
             indicate the type of columns if we cannot infer it automatically.
+        preserve_index : bool, optional
+            Whether to store the index as an additional column in the resulting
+            ``Table``.
 
         Returns
         -------
@@ -642,9 +734,12 @@ cdef class Table:
         >>> pa.Table.from_pandas(df)
         <pyarrow.lib.Table object at 0x7f05d1fb1b40>
         """
-        names, arrays, metadata = _dataframe_to_arrays(df,
-                                             timestamps_to_ms=timestamps_to_ms,
-                                             schema=schema)
+        names, arrays, metadata = _dataframe_to_arrays(
+            df,
+            timestamps_to_ms=timestamps_to_ms,
+            schema=schema,
+            preserve_index=preserve_index
+        )
         return cls.from_arrays(arrays, names=names, metadata=metadata)
 
     @staticmethod
@@ -671,7 +766,7 @@ cdef class Table:
             shared_ptr[CTable] table
             size_t K = len(arrays)
 
-        _schema_from_arrays(arrays, names, metadata or {}, &schema)
+        _schema_from_arrays(arrays, names, metadata, &schema)
 
         columns.reserve(K)
 
@@ -734,7 +829,7 @@ cdef class Table:
             nthreads = cpu_count()
 
         mgr = table_to_blockmanager(self.sp_table, nthreads)
-        return _pandas().DataFrame(mgr)
+        return pd.DataFrame(mgr)
 
     def to_pydict(self):
         """
@@ -744,11 +839,16 @@ cdef class Table:
         -------
         OrderedDict
         """
-        entries = []
-        for i in range(self.table.num_columns()):
-            name = self.column(i).name
-            column = self.column(i).to_pylist()
-            entries.append((name, column))
+        cdef:
+            size_t i
+            size_t num_columns = self.table.num_columns()
+            list entries = []
+            Column column
+
+        for i in range(num_columns):
+            column = self.column(i)
+            entries.append((column.name, column.to_pylist()))
+
         return OrderedDict(entries)
 
     @property
@@ -846,8 +946,7 @@ cdef class Table:
         """
         Add column to Table at position. Returns new table
         """
-        cdef:
-            shared_ptr[CTable] c_table
+        cdef shared_ptr[CTable] c_table
 
         with nogil:
             check_status(self.table.AddColumn(i, column.sp_column, &c_table))
diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py
index 313a3ae9f1747..17ad4b22b9cb2 100644
--- a/python/pyarrow/tests/pandas_examples.py
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -23,7 +23,7 @@
 import pyarrow as pa
 
 
-def dataframe_with_arrays():
+def dataframe_with_arrays(include_index=False):
     """
     Dataframe with numpy arrays columns of every possible primtive type.
 
@@ -72,13 +72,15 @@ def dataframe_with_arrays():
                  dtype='datetime64[ms]'),
     ]
 
+    if include_index:
+        fields.append(pa.field('__index_level_0__', pa.int64()))
     df = pd.DataFrame(arrays)
     schema = pa.schema(fields)
 
     return df, schema
 
 
-def dataframe_with_lists():
+def dataframe_with_lists(include_index=False):
     """
     Dataframe with list columns of every possible primtive type.
 
@@ -113,6 +115,8 @@ def dataframe_with_lists():
         [u"1", u"2", u"3"]
     ]
 
+    if include_index:
+        fields.append(pa.field('__index_level_0__', pa.int64()))
     df = pd.DataFrame(arrays)
     schema = pa.schema(fields)
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index be35905fc751d..ca304558c5fd3 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -67,9 +67,10 @@ def tearDown(self):
 
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
                                 timestamps_to_ms=False, expected_schema=None,
-                                check_dtype=True, schema=None):
+                                check_dtype=True, schema=None,
+                                check_index=False):
         table = pa.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms,
-                                     schema=schema)
+                                     schema=schema, preserve_index=check_index)
         result = table.to_pandas(nthreads=nthreads)
         if expected_schema:
             assert table.schema.equals(expected_schema)
@@ -299,8 +300,11 @@ def test_timestamps_notimezone_no_nulls(self):
             })
         field = pa.field('datetime64', pa.timestamp('ms'))
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(df, timestamps_to_ms=True,
-                                     expected_schema=schema)
+        self._check_pandas_roundtrip(
+            df,
+            timestamps_to_ms=True,
+            expected_schema=schema,
+        )
 
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -311,8 +315,11 @@ def test_timestamps_notimezone_no_nulls(self):
             })
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(df, timestamps_to_ms=False,
-                                     expected_schema=schema)
+        self._check_pandas_roundtrip(
+            df,
+            timestamps_to_ms=False,
+            expected_schema=schema,
+        )
 
     def test_timestamps_notimezone_nulls(self):
         df = pd.DataFrame({
@@ -324,8 +331,11 @@ def test_timestamps_notimezone_nulls(self):
             })
         field = pa.field('datetime64', pa.timestamp('ms'))
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(df, timestamps_to_ms=True,
-                                     expected_schema=schema)
+        self._check_pandas_roundtrip(
+            df,
+            timestamps_to_ms=True,
+            expected_schema=schema,
+        )
 
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -336,8 +346,11 @@ def test_timestamps_notimezone_nulls(self):
             })
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(df, timestamps_to_ms=False,
-                                     expected_schema=schema)
+        self._check_pandas_roundtrip(
+            df,
+            timestamps_to_ms=False,
+            expected_schema=schema,
+        )
 
     def test_timestamps_with_timezone(self):
         df = pd.DataFrame({
@@ -370,7 +383,7 @@ def test_date_infer(self):
                      None,
                      datetime.date(1970, 1, 1),
                      datetime.date(2040, 2, 26)]})
-        table = pa.Table.from_pandas(df)
+        table = pa.Table.from_pandas(df, preserve_index=False)
         field = pa.field('date', pa.date32())
         schema = pa.schema([field])
         assert table.schema.equals(schema)
@@ -446,7 +459,7 @@ def test_timedelta(self):
     def test_column_of_arrays(self):
         df, schema = dataframe_with_arrays()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
-        table = pa.Table.from_pandas(df, schema=schema)
+        table = pa.Table.from_pandas(df, schema=schema, preserve_index=False)
         assert table.schema.equals(schema)
 
         for column in df.columns:
@@ -456,7 +469,7 @@ def test_column_of_arrays(self):
     def test_column_of_lists(self):
         df, schema = dataframe_with_lists()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
-        table = pa.Table.from_pandas(df, schema=schema)
+        table = pa.Table.from_pandas(df, schema=schema, preserve_index=False)
         assert table.schema.equals(schema)
 
         for column in df.columns:
@@ -543,7 +556,7 @@ def test_decimal_32_from_pandas(self):
                 decimal.Decimal('1234.439'),
             ]
         })
-        converted = pa.Table.from_pandas(expected)
+        converted = pa.Table.from_pandas(expected, preserve_index=False)
         field = pa.field('decimals', pa.decimal(7, 3))
         schema = pa.schema([field])
         assert converted.schema.equals(schema)
@@ -566,7 +579,7 @@ def test_decimal_64_from_pandas(self):
                 decimal.Decimal('129534.123731'),
             ]
         })
-        converted = pa.Table.from_pandas(expected)
+        converted = pa.Table.from_pandas(expected, preserve_index=False)
         field = pa.field('decimals', pa.decimal(12, 6))
         schema = pa.schema([field])
         assert converted.schema.equals(schema)
@@ -589,7 +602,7 @@ def test_decimal_128_from_pandas(self):
                 -decimal.Decimal('314292388910493.12343437128'),
             ]
         })
-        converted = pa.Table.from_pandas(expected)
+        converted = pa.Table.from_pandas(expected, preserve_index=False)
         field = pa.field('decimals', pa.decimal(26, 11))
         schema = pa.schema([field])
         assert converted.schema.equals(schema)
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 994876de3c14a..eeea39ab19416 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -240,6 +240,57 @@ def test_get_record_batch_size():
     assert pa.get_record_batch_size(batch) > (N * itemsize)
 
 
+def test_pandas_serialize_round_trip():
+    index = pd.Index([1, 2, 3], name='my_index')
+    columns = ['foo', 'bar']
+    df = pd.DataFrame(
+        {'foo': [1.5, 1.6, 1.7], 'bar': list('abc')},
+        index=index, columns=columns
+    )
+    buf = pa.serialize_pandas(df)
+    result = pa.deserialize_pandas(buf)
+    assert_frame_equal(result, df)
+
+
+def test_pandas_serialize_round_trip_nthreads():
+    index = pd.Index([1, 2, 3], name='my_index')
+    columns = ['foo', 'bar']
+    df = pd.DataFrame(
+        {'foo': [1.5, 1.6, 1.7], 'bar': list('abc')},
+        index=index, columns=columns
+    )
+    buf = pa.serialize_pandas(df)
+    result = pa.deserialize_pandas(buf, nthreads=2)
+    assert_frame_equal(result, df)
+
+
+def test_pandas_serialize_round_trip_multi_index():
+    index1 = pd.Index([1, 2, 3], name='level_1')
+    index2 = pd.Index(list('def'), name=None)
+    index = pd.MultiIndex.from_arrays([index1, index2])
+
+    columns = ['foo', 'bar']
+    df = pd.DataFrame(
+        {'foo': [1.5, 1.6, 1.7], 'bar': list('abc')},
+        index=index,
+        columns=columns,
+    )
+    buf = pa.serialize_pandas(df)
+    result = pa.deserialize_pandas(buf)
+    assert_frame_equal(result, df)
+
+
+@pytest.mark.xfail(
+    raises=TypeError,
+    reason='Non string columns are not supported',
+)
+def test_pandas_serialize_round_trip_not_string_columns():
+    df = pd.DataFrame(list(zip([1.5, 1.6, 1.7], 'abc')))
+    buf = pa.serialize_pandas(df)
+    result = pa.deserialize_pandas(buf)
+    assert_frame_equal(result, df)
+
+
 def write_file(batch, sink):
     writer = pa.RecordBatchFileWriter(sink, batch.schema)
     writer.write_batch(batch)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 5dbe657475623..db446d3c4e781 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -19,6 +19,7 @@
 import datetime
 import io
 import os
+import json
 import pytest
 
 from pyarrow.compat import guid, u
@@ -31,15 +32,11 @@
 
 import pandas.util.testing as tm
 
-try:
-    import pyarrow.parquet as pq
-    HAVE_PARQUET = True
-except ImportError:
-    HAVE_PARQUET = False
+# Skip all parquet tests if we can't import pyarrow.parquet
+pq = pytest.importorskip('pyarrow.parquet')
 
-# XXX: Make Parquet tests opt-in rather than skip-if-not-build
-parquet = pytest.mark.skipif(not HAVE_PARQUET,
-                             reason='Parquet support not built')
+# Ignore these with pytest ... -m 'not parquet'
+parquet = pytest.mark.parquet
 
 
 @parquet
@@ -91,8 +88,55 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
 
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    assert b'pandas' in arrow_table.schema.metadata
+
     pq.write_table(arrow_table, filename.strpath, version="2.0")
-    table_read = pq.read_table(filename.strpath)
+    table_read = pq.read_pandas(filename.strpath)
+    assert b'pandas' in table_read.schema.metadata
+
+    assert arrow_table.schema.metadata == table_read.schema.metadata
+
+    df_read = table_read.to_pandas()
+    tm.assert_frame_equal(df, df_read)
+
+
+@parquet
+def test_pandas_parquet_custom_metadata(tmpdir):
+    df = alltypes_sample(size=10000)
+
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    assert b'pandas' in arrow_table.schema.metadata
+
+    pq.write_table(arrow_table, filename.strpath, version="2.0")
+    pf = pq.ParquetFile(filename.strpath)
+
+    md = pf.metadata.metadata
+    assert b'pandas' in md
+
+    js = json.loads(md[b'pandas'].decode('utf8'))
+    assert js['index_columns'] == ['__index_level_0__']
+
+
+@parquet
+def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tmpdir):
+    df = alltypes_sample(size=10000)
+
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = pa.Table.from_pandas(
+        df, timestamps_to_ms=True, preserve_index=False
+    )
+    js = json.loads(arrow_table.schema.metadata[b'pandas'].decode('utf8'))
+    assert not js['index_columns']
+
+    pq.write_table(arrow_table, filename.strpath, version="2.0")
+    table_read = pq.read_pandas(filename.strpath)
+
+    js = json.loads(table_read.schema.metadata[b'pandas'].decode('utf8'))
+    assert not js['index_columns']
+
+    assert arrow_table.schema.metadata == table_read.schema.metadata
+
     df_read = table_read.to_pandas()
     tm.assert_frame_equal(df, df_read)
 
@@ -167,7 +211,6 @@ def _test_dataframe(size=10000, seed=0):
         'int32': _random_integers(size, np.int32),
         'int64': _random_integers(size, np.int64),
         'float32': np.random.randn(size).astype(np.float32),
-        'float64': np.random.randn(size),
         'float64': np.arange(size, dtype=np.float64),
         'bool': np.random.randn(size) > 0,
         'strings': [tm.rands(10) for i in range(size)]
@@ -187,6 +230,18 @@ def test_pandas_parquet_native_file_roundtrip(tmpdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@parquet
+def test_read_pandas_column_subset(tmpdir):
+    df = _test_dataframe(10000)
+    arrow_table = pa.Table.from_pandas(df)
+    imos = pa.BufferOutputStream()
+    pq.write_table(arrow_table, imos, version="2.0")
+    buf = imos.get_result()
+    reader = pa.BufferReader(buf)
+    df_read = pq.read_pandas(reader, columns=['strings', 'uint8']).to_pandas()
+    tm.assert_frame_equal(df[['strings', 'uint8']], df_read)
+
+
 @parquet
 def test_pandas_parquet_pyfile_roundtrip(tmpdir):
     filename = tmpdir.join('pandas_pyfile_roundtrip.parquet').strpath
@@ -270,7 +325,7 @@ def test_parquet_metadata_api():
     meta = fileh.metadata
     repr(meta)
     assert meta.num_rows == len(df)
-    assert meta.num_columns == ncols
+    assert meta.num_columns == ncols + 1  # +1 for index
     assert meta.num_row_groups == 1
     assert meta.format_version == '2.0'
     assert 'parquet-cpp' in meta.created_by
@@ -278,7 +333,7 @@ def test_parquet_metadata_api():
     # Schema
     schema = fileh.schema
     assert meta.schema is schema
-    assert len(schema) == ncols
+    assert len(schema) == ncols + 1  # +1 for index
     repr(schema)
 
     col = schema[0]
@@ -292,7 +347,7 @@ def test_parquet_metadata_api():
     assert col.logical_type == 'NONE'
 
     with pytest.raises(IndexError):
-        schema[ncols]
+        schema[ncols + 1]  # +1 for index
 
     with pytest.raises(IndexError):
         schema[-1]
@@ -302,7 +357,7 @@ def test_parquet_metadata_api():
     repr(rg_meta)
 
     assert rg_meta.num_rows == len(df)
-    assert rg_meta.num_columns == ncols
+    assert rg_meta.num_columns == ncols + 1  # +1 for index
 
 
 @parquet
@@ -502,9 +557,22 @@ def test_read_single_row_group():
     result = pa.concat_tables(row_groups)
     tm.assert_frame_equal(df, result.to_pandas())
 
+
+@parquet
+def test_read_single_row_group_with_column_subset():
+    N, K = 10000, 4
+    df = alltypes_sample(size=N)
+    a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+
+    buf = io.BytesIO()
+    pq.write_table(a_table, buf, row_group_size=N / K,
+                   compression='snappy', version='2.0')
+
+    buf.seek(0)
+    pf = pq.ParquetFile(buf)
+
     cols = df.columns[:2]
-    row_groups = [pf.read_row_group(i, columns=cols)
-                  for i in range(K)]
+    row_groups = [pf.read_row_group(i, columns=cols) for i in range(K)]
     result = pa.concat_tables(row_groups)
     tm.assert_frame_equal(df[cols], result.to_pandas())
 
@@ -696,6 +764,9 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
 
     assert result.equals(expected)
 
+    with pytest.raises(NotImplementedError):
+        pq.read_pandas(dirpath)
+
     # Read with provided metadata
     metadata = pq.ParquetFile(paths[0]).metadata
 
@@ -706,10 +777,11 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
     assert result3.equals(expected)
 
     # Read column subset
-    to_read = [result[0], result[3], result[6]]
+    to_read = [result[0], result[2], result[6], result[result.num_columns - 1]]
+
     result = pa.localfs.read_parquet(
         dirpath, columns=[c.name for c in to_read])
-    expected = pa.Table.from_arrays(to_read)
+    expected = pa.Table.from_arrays(to_read, metadata=result.schema.metadata)
     assert result.equals(expected)
 
     # Read with multiple threads
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 72ce6967edf19..ed2201171a9ea 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -142,7 +142,7 @@ def test_recordbatchlist_to_pandas():
 
     table = pa.Table.from_batches([batch1, batch2])
     result = table.to_pandas()
-    data = pd.concat([data1, data2], ignore_index=True)
+    data = pd.concat([data1, data2])
     assert_frame_equal(data, result)
 
 

From c4086febd2d2ce976100d6e65029a7b213583b39 Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Wed, 17 May 2017 11:51:57 -0400
Subject: [PATCH 0665/1644] ARROW-997: [Java] Implementing transferPair for
 FixedSizeListVector

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #672 from elahrvivaz/ARROW-997 and squashes the following commits:

68be6a56 [Emilio Lahr-Vivaz] ARROW-997: [Java] Implementing transferPair for FixedSizeListVector
---
 .../vector/complex/FixedSizeListVector.java   | 16 +++--
 .../arrow/vector/TestFixedSizeListVector.java | 66 +++++++++++++++++++
 2 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index ee22fbec03597..b6d938f3fd863 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -259,7 +259,7 @@ public int size() {
   @SuppressWarnings("unchecked")
   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType type) {
     boolean created = false;
-    if (vector instanceof ZeroVector) {
+    if (vector == ZeroVector.INSTANCE) {
       vector = type.createNewSingleVector(DATA_VECTOR_NAME, allocator, null);
       this.reader = new UnionFixedSizeListReader(this);
       created = true;
@@ -278,8 +278,9 @@ public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) {
     copyFrom(inIndex, outIndex, from);
   }
 
-  public void copyFrom(int inIndex, int outIndex, FixedSizeListVector from) {
-    throw new UnsupportedOperationException("FixedSizeListVector.copyFrom");
+  public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) {
+    TransferPair pair = from.makeTransferPair(this);
+    pair.copyValueSafe(fromIndex, thisIndex);
   }
 
   @Override
@@ -367,7 +368,7 @@ public TransferImpl(FixedSizeListVector to) {
       this.to = to;
       to.addOrGetVector(vector.getField().getFieldType());
       pairs[0] = bits.makeTransferPair(to.bits);
-      pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
+      pairs[1] = vector.makeTransferPair(to.vector);
     }
 
     @Override
@@ -392,7 +393,12 @@ public ValueVector getTo() {
 
     @Override
     public void copyValueSafe(int from, int to) {
-      this.to.copyFrom(from, to, FixedSizeListVector.this);
+      pairs[0].copyValueSafe(from, to);
+      int fromOffset = from * listSize;
+      int toOffset = to * listSize;
+      for (int i = 0; i < listSize; i++) {
+        pairs[1].copyValueSafe(fromOffset + i, toOffset + i);
+      }
     }
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index f35ab54eaa88d..304db9dcc00bd 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -28,6 +28,7 @@
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -153,4 +154,69 @@ public void testNestedInList() {
       }
     }
   }
+
+  @Test
+  public void testTransferPair() {
+    try (FixedSizeListVector from = new FixedSizeListVector("from", allocator, 2, null, null);
+         FixedSizeListVector to = new FixedSizeListVector("to", allocator, 2, null, null)) {
+      NullableFloat4Vector nested = (NullableFloat4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
+      NullableFloat4Vector.Mutator mutator = nested.getMutator();
+      from.allocateNew();
+
+      for (int i = 0; i < 10; i++) {
+        if (i % 2 == 0) {
+          from.getMutator().setNotNull(i);
+          mutator.set(i * 2, i + 0.1f);
+          mutator.set(i * 2 + 1, i + 10.1f);
+        }
+      }
+      from.getMutator().setValueCount(10);
+
+      TransferPair pair = from.makeTransferPair(to);
+
+      pair.copyValueSafe(0, 1);
+      pair.copyValueSafe(2, 2);
+      to.copyFromSafe(4, 3, from);
+      to.getMutator().setValueCount(10);
+
+      UnionFixedSizeListReader reader = to.getReader();
+
+      reader.setPosition(0);
+      Assert.assertFalse(reader.isSet());
+      Assert.assertNull(reader.readObject());
+
+      reader.setPosition(1);
+      Assert.assertTrue(reader.isSet());
+      Assert.assertTrue(reader.next());
+      Assert.assertEquals(0.1f, reader.reader().readFloat(), 0.00001);
+      Assert.assertTrue(reader.next());
+      Assert.assertEquals(10.1f, reader.reader().readFloat(), 0.00001);
+      Assert.assertFalse(reader.next());
+      Assert.assertEquals(Lists.newArrayList(0.1f, 10.1f), reader.readObject());
+
+      reader.setPosition(2);
+      Assert.assertTrue(reader.isSet());
+      Assert.assertTrue(reader.next());
+      Assert.assertEquals(2.1f, reader.reader().readFloat(), 0.00001);
+      Assert.assertTrue(reader.next());
+      Assert.assertEquals(12.1f, reader.reader().readFloat(), 0.00001);
+      Assert.assertFalse(reader.next());
+      Assert.assertEquals(Lists.newArrayList(2.1f, 12.1f), reader.readObject());
+
+      reader.setPosition(3);
+      Assert.assertTrue(reader.isSet());
+      Assert.assertTrue(reader.next());
+      Assert.assertEquals(4.1f, reader.reader().readFloat(), 0.00001);
+      Assert.assertTrue(reader.next());
+      Assert.assertEquals(14.1f, reader.reader().readFloat(), 0.00001);
+      Assert.assertFalse(reader.next());
+      Assert.assertEquals(Lists.newArrayList(4.1f, 14.1f), reader.readObject());
+
+      for (int i = 4; i < 10; i++) {
+        reader.setPosition(i);
+        Assert.assertFalse(reader.isSet());
+        Assert.assertNull(reader.readObject());
+      }
+    }
+  }
 }

From a4f3259b07eeaa5300abf09cc46b867690c4d80f Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 17 May 2017 11:53:51 -0400
Subject: [PATCH 0666/1644] ARROW-1030: Python: Account for library versioning
 in parquet-cpp

This mainly uses the same logic we already use for arrow-cpp

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #698 from xhochy/parquet-abi-version-bundling and squashes the following commits:

4aa17f82 [Uwe L. Korn] ARROW-1030: Python: Account for library versioning in parquet-cpp
---
 python/CMakeLists.txt                  | 44 +++++++++++++---
 python/cmake_modules/FindParquet.cmake | 70 +++++++++++++++++---------
 python/manylinux1/Dockerfile-x86_64    |  2 +-
 python/manylinux1/build_arrow.sh       |  2 -
 4 files changed, 85 insertions(+), 33 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index ddd174ca8db31..ded85e8b84c43 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -290,16 +290,48 @@ if (PYARROW_BUILD_PARQUET)
     message(FATAL_ERROR "Unable to locate Parquet libraries")
   endif()
   if (PYARROW_BUNDLE_ARROW_CPP)
+      get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY)
+      get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE)
+      get_filename_component(PARQUET_ARROW_LIBRARY_NAME ${PARQUET_ARROW_SHARED_LIB} NAME_WE)
+      message(STATUS "Configure start")
       configure_file(${PARQUET_SHARED_LIB}
-          ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}
+          ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
           COPYONLY)
-      SET(PARQUET_SHARED_LIB
-          ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX})
       configure_file(${PARQUET_ARROW_SHARED_LIB}
-          ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}
+          ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_ARROW_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
           COPYONLY)
-      SET(PARQUET_ARROW_SHARED_LIB
-          ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+      if (APPLE)
+          configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+              COPYONLY)
+          configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+              COPYONLY)
+          configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+              COPYONLY)
+          configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+              COPYONLY)
+      else()
+          configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
+              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
+              COPYONLY)
+          configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
+              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
+              COPYONLY)
+          configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
+              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
+              COPYONLY)
+          configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
+              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
+              COPYONLY)
+      endif()
+      message(STATUS "Configure end")
+      #SET(PARQUET_SHARED_LIB
+      #    ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX})
+      #SET(PARQUET_ARROW_SHARED_LIB
+      #    ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
   ADD_THIRDPARTY_LIB(parquet_arrow
     SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index a20b651e2b3c6..ffd043dad0d63 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -20,35 +20,57 @@
 #  PARQUET_SHARED_LIB, path to libparquet's shared library
 #  PARQUET_FOUND, whether parquet has been found
 
-if( NOT "$ENV{PARQUET_HOME}" STREQUAL "")
-    file( TO_CMAKE_PATH "$ENV{PARQUET_HOME}" _native_path )
-    list( APPEND _parquet_roots ${_native_path} )
-elseif ( Parquet_HOME )
-    list( APPEND _parquet_roots ${Parquet_HOME} )
-endif()
-
-# Try the parameterized roots, if they exist
-if ( _parquet_roots )
-  find_path( PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h
-    PATHS ${_parquet_roots} NO_DEFAULT_PATH
-    PATH_SUFFIXES "include" )
-  find_library( PARQUET_LIBRARIES NAMES parquet
-    PATHS ${_parquet_roots} NO_DEFAULT_PATH
-    PATH_SUFFIXES "lib" )
+include(FindPkgConfig)
 
-  find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
-    PATHS ${_parquet_roots} NO_DEFAULT_PATH
-    PATH_SUFFIXES "lib")
-else ()
-    find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
-    find_library(PARQUET_LIBRARIES NAMES parquet)
-    find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow)
-endif ()
+if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
+    set(PARQUET_HOME "$ENV{PARQUET_HOME}")
+endif()
 
+if(PARQUET_HOME)
+    set(PARQUET_SEARCH_HEADER_PATHS
+        ${PARQUET_HOME}/include
+        )
+    set(PARQUET_SEARCH_LIB_PATH
+        ${PARQUET_HOME}/lib
+        )
+    find_path(PARQUET_INCLUDE_DIR parquet/api/reader.h PATHS
+        ${PARQUET_SEARCH_HEADER_PATHS}
+        # make sure we don't accidentally pick up a different version
+        NO_DEFAULT_PATH
+        )
+    find_library(PARQUET_LIBRARIES NAMES parquet
+        PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib")
+    find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
+        PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib")
+    get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
+else()
+    pkg_check_modules(PARQUET parquet)
+    if (PARQUET_FOUND)
+        pkg_get_variable(PARQUET_ABI_VERSION parquet abi_version)
+        message(STATUS "Parquet C++ ABI version: ${PARQUET_ABI_VERSION}")
+        pkg_get_variable(PARQUET_SO_VERSION parquet so_version)
+        message(STATUS "Parquet C++ SO version: ${PARQUET_SO_VERSION}")
+        set(PARQUET_INCLUDE_DIR ${PARQUET_INCLUDE_DIRS})
+        set(PARQUET_LIBS ${PARQUET_LIBRARY_DIRS})
+        set(PARQUET_SEARCH_LIB_PATH ${PARQUET_LIBRARY_DIRS})
+        message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}")
+        find_library(PARQUET_LIBRARIES NAMES parquet
+            PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH)
+        find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
+            PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH)
+        message(STATUS "${PARQUET_ARROW_LIBRARIES}")
+    else()
+        find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
+        find_library(PARQUET_LIBRARIES NAMES parquet)
+        find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow)
+        get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
+    endif()
+endif()
 
 if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
   set(PARQUET_FOUND TRUE)
-  get_filename_component( PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
   set(PARQUET_LIB_NAME libparquet)
   set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
   set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index ad40e79a9f20f..363308b616e73 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -21,5 +21,5 @@ RUN make -j5 install
 WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .
 RUN make -j5 install
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index e0727495cff4a..4565e59bea222 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -39,9 +39,7 @@ export PYARROW_WITH_JEMALLOC=1
 export PYARROW_BUNDLE_ARROW_CPP=1
 # Need as otherwise arrow_io is sometimes not linked
 export LDFLAGS="-Wl,--no-as-needed"
-export PARQUET_HOME="/usr"
 export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
-
 # Ensure the target directory exists
 mkdir -p /io/dist
 

From fee44751048847ce1e08039ab72c09918b11b31b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 17 May 2017 13:32:08 -0400
Subject: [PATCH 0667/1644] ARROW-1029: [Python] Fixes for building pyarrow
 with Parquet support on MSVC. Add to appveyor build

Miscellaneous fixes to build with ``--with-parquet`` and pass the test suite. We still have a bunch of compiler warnings; not blocking for 0.4.0, but after PARQUET-991 is resolved we should fix the MSVC compiler warnings in pyarrow

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #700 from wesm/ARROW-1029 and squashes the following commits:

b651169c [Wes McKinney] Fix post rebase issue
0f438488 [Wes McKinney] Fixes for building pyarrow with Parquet support on MSVC. Add to appveyor build
---
 ci/msvc-build.bat                      | 32 +++++++++++++---
 python/CMakeLists.txt                  | 16 ++++----
 python/cmake_modules/FindArrow.cmake   |  2 -
 python/cmake_modules/FindParquet.cmake | 51 ++++++++++++++++----------
 python/doc/source/development.rst      | 51 ++++++++++++++++++++++++--
 python/pyarrow/filesystem.py           |  8 ++++
 python/pyarrow/parquet.py              |  2 +-
 python/pyarrow/tests/test_parquet.py   |  6 +--
 8 files changed, 127 insertions(+), 41 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 504da7638daa0..f756fc5195d2d 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -19,17 +19,19 @@
 
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython
-conda install -n arrow -q -y -c conda-forge flatbuffers rapidjson
+conda install -n arrow -q -y -c conda-forge ^
+      flatbuffers rapidjson ^
+      cmake git boost-cpp thrift-cpp snappy zlib brotli
+
 call activate arrow
 
 set ARROW_HOME=%CONDA_PREFIX%\Library
-set FLATBUFFERS_HOME=%CONDA_PREFIX%\Library
-set RAPIDJSON_HOME=%CONDA_PREFIX%\Library
+set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 
 @rem Build and test Arrow C++ libraries
 
 mkdir cpp\build
-cd cpp\build
+pushd cpp\build
 
 cmake -G "%GENERATOR%" ^
       -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
@@ -44,10 +46,28 @@ cmake --build . --target INSTALL --config Release  || exit /B
 set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%
 
 ctest -VV  || exit /B
+popd
+
+@rem Build parquet-cpp
+
+git clone https://github.com/apache/parquet-cpp.git || exit /B
+mkdir parquet-cpp\build
+pushd parquet-cpp\build
+
+set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+set PARQUET_HOME=%CONDA_PREFIX%\Library
+cmake -G "%GENERATOR%" ^
+     -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
+     -DCMAKE_BUILD_TYPE=Release ^
+     -DPARQUET_ZLIB_VENDORED=off ^
+     -DPARQUET_BUILD_TESTS=off .. || exit /B
+cmake --build . --target INSTALL --config Release || exit /B
+popd
 
 @rem Build and import pyarrow
 set PYTHONPATH=
 
-cd ..\..\python
-python setup.py build_ext --inplace  || exit /B
+pushd python
+python setup.py build_ext --inplace --with-parquet  || exit /B
 py.test pyarrow -v -s || exit /B
+popd
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index ded85e8b84c43..6f48f7f06eb43 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -219,12 +219,6 @@ include_directories(SYSTEM
 # Dependencies
 ############################################################
 
-## Parquet
-find_package(Parquet)
-if(PARQUET_FOUND)
-  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
-endif()
-
 ## Arrow
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
@@ -286,9 +280,14 @@ set(LINK_LIBS
 )
 
 if (PYARROW_BUILD_PARQUET)
+  ## Parquet
+  find_package(Parquet)
+
   if(NOT (PARQUET_FOUND AND PARQUET_ARROW_FOUND))
     message(FATAL_ERROR "Unable to locate Parquet libraries")
   endif()
+  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
+
   if (PYARROW_BUNDLE_ARROW_CPP)
       get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY)
       get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE)
@@ -333,11 +332,14 @@ if (PYARROW_BUILD_PARQUET)
       #SET(PARQUET_ARROW_SHARED_LIB
       #    ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
+  ADD_THIRDPARTY_LIB(parquet
+    SHARED_LIB ${PARQUET_SHARED_LIB})
   ADD_THIRDPARTY_LIB(parquet_arrow
     SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
   set(LINK_LIBS
     ${LINK_LIBS}
-    parquet_arrow)
+    parquet_shared
+    parquet_arrow_shared)
   set(CYTHON_EXTENSIONS
     ${CYTHON_EXTENSIONS}
     _parquet)
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 9fb1355fe1d52..c16a4bf05aa8b 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -76,10 +76,8 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   if (MSVC)
     set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
     set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
-    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
     set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
     set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
-    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
   else()
     set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
     set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index ffd043dad0d63..de53a29879e0d 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -1,16 +1,19 @@
-# Copyright 2012 Cloudera Inc.
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 # - Find PARQUET (parquet/parquet.h, libparquet.a, libparquet.so)
 # This module defines
@@ -71,9 +74,14 @@ endif()
 
 if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
   set(PARQUET_FOUND TRUE)
-  set(PARQUET_LIB_NAME libparquet)
-  set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
-  set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  if (MSVC)
+    set(PARQUET_STATIC_LIB "${PARQUET_LIBRARIES}_static")
+    set(PARQUET_SHARED_LIB "${PARQUET_LIBRARIES}")
+  else()
+    set(PARQUET_LIB_NAME libparquet)
+    set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
+    set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
 else ()
   set(PARQUET_FOUND FALSE)
 endif ()
@@ -81,11 +89,16 @@ endif ()
 if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES)
   set(PARQUET_ARROW_FOUND TRUE)
   get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH)
-  set(PARQUET_ARROW_LIB_NAME libparquet_arrow)
-  set(PARQUET_ARROW_STATIC_LIB
-    ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.a)
-  set(PARQUET_ARROW_SHARED_LIB
-    ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  if (MSVC)
+    set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBRARIES}_static")
+    set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_LIBRARIES}")
+  else()
+    set(PARQUET_ARROW_LIB_NAME libparquet_arrow)
+    set(PARQUET_ARROW_STATIC_LIB
+      ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.a)
+    set(PARQUET_ARROW_SHARED_LIB
+      ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
 else ()
   set(PARQUET_ARROW_FOUND FALSE)
 endif ()
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 440c1c459eed5..6ec563ed3ce36 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -174,14 +174,37 @@ You should be able to run the unit tests with:
 Windows
 =======
 
-First, make sure you can `build the C++ library <https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md>`_.
+First, we bootstrap a conda environment similar to the `C++ build instructions
+<https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md>`_. This
+includes all the dependencies for Arrow and the Apache Parquet C++ libraries.
 
-Now, we need to build and install the C++ libraries someplace.
+First, starting from fresh clones of Apache Arrow and parquet-cpp:
+
+.. code-block:: shell
+
+   git clone https://github.com/apache/arrow.git
+   git clone https://github.com/apache/parquet-cpp.git
+
+.. code-block:: shell
+
+   conda create -n arrow-dev cmake git boost-cpp ^
+         flatbuffers snappy zlib brotli thrift-cpp rapidjson
+   activate arrow-dev
+
+As one git housekeeping item, we must run this command in our Arrow clone:
+
+.. code-block:: shell
+
+   cd arrow
+   git config core.symlinks true
+
+Now, we build and install Arrow C++ libraries
 
 .. code-block:: shell
 
    mkdir cpp\build
    cd cpp\build
+   set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
    set ARROW_HOME=C:\thirdparty
    cmake -G "Visual Studio 14 2015 Win64" ^
          -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
@@ -191,6 +214,22 @@ Now, we need to build and install the C++ libraries someplace.
    cmake --build . --target INSTALL --config Release
    cd ..\..
 
+Now, we build parquet-cpp and install the result in the same place:
+
+.. code-block:: shell
+
+   mkdir ..\parquet-cpp\build
+   pushd ..\parquet-cpp\build
+   set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+   set PARQUET_HOME=C:\thirdparty
+   cmake -G "Visual Studio 14 2015 Win64" ^
+         -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
+         -DCMAKE_BUILD_TYPE=Release ^
+         -DPARQUET_ZLIB_VENDORED=off ^
+         -DPARQUET_BUILD_TESTS=off ..
+   cmake --build . --target INSTALL --config Release
+   popd
+
 After that, we must put the install directory's bin path in our ``%PATH%``:
 
 .. code-block:: shell
@@ -202,7 +241,13 @@ Now, we can build pyarrow:
 .. code-block:: shell
 
    cd python
-   python setup.py build_ext --inplace
+   python setup.py build_ext --inplace --with-parquet
+
+Then run the unit tests with:
+
+.. code-block:: shell
+
+   py.test pyarrow -v
 
 Running C++ unit tests with Python
 ----------------------------------
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index ac37fd87294c5..798d96b2cac8d 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -92,6 +92,10 @@ def read_parquet(self, path, columns=None, metadata=None, schema=None,
                                  filesystem=self)
         return dataset.read(columns=columns, nthreads=nthreads)
 
+    @property
+    def pathsep(self):
+        return '/'
+
 
 class LocalFilesystem(Filesystem):
 
@@ -132,6 +136,10 @@ def open(self, path, mode='rb'):
         """
         return open(path, mode=mode)
 
+    @property
+    def pathsep(self):
+        return os.path.sep
+
 
 class HdfsClient(lib._HdfsClient, Filesystem):
     """
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index e69d85eac4e64..f59a719b29efc 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -582,7 +582,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
 
     if is_string(path_or_paths) and fs.isdir(path_or_paths):
         manifest = ParquetManifest(path_or_paths, filesystem=fs,
-                                   pathsep=pathsep)
+                                   pathsep=fs.pathsep)
         metadata_path = manifest.metadata_path
         pieces = manifest.pieces
         partitions = manifest.partitions
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index db446d3c4e781..5f65f2800d170 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -192,10 +192,10 @@ def test_pandas_column_selection(tmpdir):
 
 def _random_integers(size, dtype):
     # We do not generate integers outside the int64 range
-    i64_info = np.iinfo('int64')
+    platform_int_info = np.iinfo('int_')
     iinfo = np.iinfo(dtype)
-    return np.random.randint(max(iinfo.min, i64_info.min),
-                             min(iinfo.max, i64_info.max),
+    return np.random.randint(max(iinfo.min, platform_int_info.min),
+                             min(iinfo.max, platform_int_info.max),
                              size=size).astype(dtype)
 
 

From 62a17b7e823162f7a07ddb9747e4ed0df3e1c9ca Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 17 May 2017 15:13:12 -0400
Subject: [PATCH 0668/1644] ARROW-1044: [GLib] Support Feather

Author: Kouhei Sutou <kou@clear-code.com>

Closes #699 from kou/glib-support-feather and squashes the following commits:

75ce6119 [Kouhei Sutou] [GLib] Support Feather
---
 c_glib/arrow-glib/reader.cpp            | 308 +++++++++++++++++++++++-
 c_glib/arrow-glib/reader.h              |  72 ++++++
 c_glib/arrow-glib/reader.hpp            |   4 +
 c_glib/arrow-glib/writer.cpp            | 216 ++++++++++++++++-
 c_glib/arrow-glib/writer.h              |  60 +++++
 c_glib/arrow-glib/writer.hpp            |   4 +
 c_glib/test/test-feather-file-reader.rb | 155 ++++++++++++
 c_glib/test/test-feather-file-writer.rb |  68 ++++++
 8 files changed, 883 insertions(+), 4 deletions(-)
 create mode 100644 c_glib/test/test-feather-file-reader.rb
 create mode 100644 c_glib/test/test-feather-file-writer.rb

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index c0923242ccb75..3ff6ba1bb5294 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -21,8 +21,7 @@
 #  include <config.h>
 #endif
 
-#include <arrow/ipc/api.h>
-
+#include <arrow-glib/column.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
@@ -48,6 +47,9 @@ G_BEGIN_DECLS
  *
  * #GArrowRecordBatchFileReader is a class for reading record
  * batches in file format from input.
+ *
+ * #GArrowFeatherFileReader is a class for reading columns in Feather
+ * file format from input.
  */
 
 typedef struct GArrowRecordBatchReaderPrivate_ {
@@ -419,6 +421,288 @@ garrow_record_batch_file_reader_get_record_batch(GArrowRecordBatchFileReader *re
   }
 }
 
+
+typedef struct GArrowFeatherFileReaderPrivate_ {
+  arrow::ipc::feather::TableReader *feather_table_reader;
+} GArrowFeatherFileReaderPrivate;
+
+enum {
+  PROP_0__,
+  PROP_FEATHER_TABLE_READER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowFeatherFileReader,
+                           garrow_feather_file_reader,
+                           G_TYPE_OBJECT);
+
+#define GARROW_FEATHER_FILE_READER_GET_PRIVATE(obj)             \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_TYPE_FEATHER_FILE_READER, \
+                               GArrowFeatherFileReaderPrivate))
+
+static void
+garrow_feather_file_reader_finalize(GObject *object)
+{
+  GArrowFeatherFileReaderPrivate *priv;
+
+  priv = GARROW_FEATHER_FILE_READER_GET_PRIVATE(object);
+
+  delete priv->feather_table_reader;
+
+  G_OBJECT_CLASS(garrow_feather_file_reader_parent_class)->finalize(object);
+}
+
+static void
+garrow_feather_file_reader_set_property(GObject *object,
+                                        guint prop_id,
+                                        const GValue *value,
+                                        GParamSpec *pspec)
+{
+  GArrowFeatherFileReaderPrivate *priv;
+
+  priv = GARROW_FEATHER_FILE_READER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_FEATHER_TABLE_READER:
+    priv->feather_table_reader =
+      static_cast<arrow::ipc::feather::TableReader *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_feather_file_reader_get_property(GObject *object,
+                                        guint prop_id,
+                                        GValue *value,
+                                        GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_feather_file_reader_init(GArrowFeatherFileReader *object)
+{
+}
+
+static void
+garrow_feather_file_reader_class_init(GArrowFeatherFileReaderClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_feather_file_reader_finalize;
+  gobject_class->set_property = garrow_feather_file_reader_set_property;
+  gobject_class->get_property = garrow_feather_file_reader_get_property;
+
+  spec = g_param_spec_pointer("feather-table-reader",
+                              "arrow::ipc::feather::TableReader",
+                              "The raw std::shared<arrow::ipc::feather::TableReader> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_FEATHER_TABLE_READER, spec);
+}
+
+
+/**
+ * garrow_feather_file_reader_new:
+ * @file: The file to be read.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowFeatherFileReader
+ *   or %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowFeatherFileReader *
+garrow_feather_file_reader_new(GArrowSeekableInputStream *file,
+                               GError **error)
+{
+  auto arrow_random_access_file = garrow_seekable_input_stream_get_raw(file);
+  std::unique_ptr<arrow::ipc::feather::TableReader> arrow_reader;
+  auto status =
+    arrow::ipc::feather::TableReader::Open(arrow_random_access_file,
+                                           &arrow_reader);
+  if (garrow_error_check(error, status, "[feather-file-reader][new]")) {
+    return garrow_feather_file_reader_new_raw(arrow_reader.release());
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_feather_file_reader_get_description:
+ * @reader: A #GArrowFeatherFileReader.
+ *
+ * Returns: (nullable): The description of the file if it exists,
+ *   %NULL otherwise. You can confirm whether description exists or not by
+ *   garrow_feather_file_reader_has_description().
+ *   It should be freed with g_free() when no longer needed.
+ *
+ * Since: 0.4.0
+ */
+gchar *
+garrow_feather_file_reader_get_description(GArrowFeatherFileReader *reader)
+{
+  auto arrow_reader = garrow_feather_file_reader_get_raw(reader);
+  if (arrow_reader->HasDescription()) {
+    auto description = arrow_reader->GetDescription();
+    return g_strndup(description.data(),
+                     description.size());
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_feather_file_reader_has_description:
+ * @reader: A #GArrowFeatherFileReader.
+ *
+ * Returns: Whether the file has description or not.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_feather_file_reader_has_description(GArrowFeatherFileReader *reader)
+{
+  auto arrow_reader = garrow_feather_file_reader_get_raw(reader);
+  return arrow_reader->HasDescription();
+}
+
+/**
+ * garrow_feather_file_reader_get_version:
+ * @reader: A #GArrowFeatherFileReader.
+ *
+ * Returns: The format version of the file.
+ *
+ * Since: 0.4.0
+ */
+gint
+garrow_feather_file_reader_get_version(GArrowFeatherFileReader *reader)
+{
+  auto arrow_reader = garrow_feather_file_reader_get_raw(reader);
+  return arrow_reader->version();
+}
+
+/**
+ * garrow_feather_file_reader_get_n_rows:
+ * @reader: A #GArrowFeatherFileReader.
+ *
+ * Returns: The number of rows in the file.
+ *
+ * Since: 0.4.0
+ */
+gint64
+garrow_feather_file_reader_get_n_rows(GArrowFeatherFileReader *reader)
+{
+  auto arrow_reader = garrow_feather_file_reader_get_raw(reader);
+  return arrow_reader->num_rows();
+}
+
+/**
+ * garrow_feather_file_reader_get_n_columns:
+ * @reader: A #GArrowFeatherFileReader.
+ *
+ * Returns: The number of columns in the file.
+ *
+ * Since: 0.4.0
+ */
+gint64
+garrow_feather_file_reader_get_n_columns(GArrowFeatherFileReader *reader)
+{
+  auto arrow_reader = garrow_feather_file_reader_get_raw(reader);
+  return arrow_reader->num_columns();
+}
+
+/**
+ * garrow_feather_file_reader_get_column_name:
+ * @reader: A #GArrowFeatherFileReader.
+ * @i: The index of the target column.
+ *
+ * Returns: The i-th column name in the file.
+ *   It should be freed with g_free() when no longer needed.
+ *
+ * Since: 0.4.0
+ */
+gchar *
+garrow_feather_file_reader_get_column_name(GArrowFeatherFileReader *reader,
+                                           gint i)
+{
+  auto arrow_reader = garrow_feather_file_reader_get_raw(reader);
+  auto column_name = arrow_reader->GetColumnName(i);
+  return g_strndup(column_name.data(),
+                   column_name.size());
+}
+
+/**
+ * garrow_feather_file_reader_get_column:
+ * @reader: A #GArrowFeatherFileReader.
+ * @i: The index of the target column.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The i-th column in the file or %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowColumn *
+garrow_feather_file_reader_get_column(GArrowFeatherFileReader *reader,
+                                      gint i,
+                                      GError **error)
+{
+  auto arrow_reader = garrow_feather_file_reader_get_raw(reader);
+  std::shared_ptr<arrow::Column> arrow_column;
+  auto status = arrow_reader->GetColumn(i, &arrow_column);
+
+  if (garrow_error_check(error, status, "[feather-file-reader][get-column]")) {
+    return garrow_column_new_raw(&arrow_column);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_feather_file_reader_get_columns:
+ * @reader: A #GArrowFeatherFileReader.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (element-type GArrowColumn) (transfer full):
+ *   The columns in the file.
+ *
+ * Since: 0.4.0
+ */
+GList *
+garrow_feather_file_reader_get_columns(GArrowFeatherFileReader *reader,
+                                       GError **error)
+{
+  GList *columns = NULL;
+  auto arrow_reader = garrow_feather_file_reader_get_raw(reader);
+  auto n_columns = arrow_reader->num_columns();
+  for (gint i = 0; i < n_columns; ++i) {
+    std::shared_ptr<arrow::Column> arrow_column;
+    auto status = arrow_reader->GetColumn(i, &arrow_column);
+    if (!garrow_error_check(error,
+                            status,
+                            "[feather-file-reader][get-columns]")) {
+      g_list_foreach(columns, (GFunc)g_object_unref, NULL);
+      g_list_free(columns);
+      return NULL;
+    }
+    columns = g_list_prepend(columns,
+                             garrow_column_new_raw(&arrow_column));
+  }
+  return g_list_reverse(columns);
+}
+
+
 G_END_DECLS
 
 GArrowRecordBatchReader *
@@ -470,3 +754,23 @@ garrow_record_batch_file_reader_get_raw(GArrowRecordBatchFileReader *reader)
   priv = GARROW_RECORD_BATCH_FILE_READER_GET_PRIVATE(reader);
   return priv->record_batch_file_reader;
 }
+
+GArrowFeatherFileReader *
+garrow_feather_file_reader_new_raw(arrow::ipc::feather::TableReader *arrow_reader)
+{
+  auto reader =
+    GARROW_FEATHER_FILE_READER(
+      g_object_new(GARROW_TYPE_FEATHER_FILE_READER,
+                   "feather-table-reader", arrow_reader,
+                   NULL));
+  return reader;
+}
+
+arrow::ipc::feather::TableReader *
+garrow_feather_file_reader_get_raw(GArrowFeatherFileReader *reader)
+{
+  GArrowFeatherFileReaderPrivate *priv;
+
+  priv = GARROW_FEATHER_FILE_READER_GET_PRIVATE(reader);
+  return priv->feather_table_reader;
+}
diff --git a/c_glib/arrow-glib/reader.h b/c_glib/arrow-glib/reader.h
index 477204b7ae9d3..b8bdbe221d646 100644
--- a/c_glib/arrow-glib/reader.h
+++ b/c_glib/arrow-glib/reader.h
@@ -193,4 +193,76 @@ GArrowRecordBatch *garrow_record_batch_file_reader_get_record_batch(
   guint i,
   GError **error);
 
+
+#define GARROW_TYPE_FEATHER_FILE_READER         \
+  (garrow_feather_file_reader_get_type())
+#define GARROW_FEATHER_FILE_READER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_FEATHER_FILE_READER,  \
+                              GArrowFeatherFileReader))
+#define GARROW_FEATHER_FILE_READER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_FEATHER_FILE_READER,     \
+                           GArrowFeatherFileReaderClass))
+#define GARROW_IS_FEATHER_FILE_READER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_FEATHER_FILE_READER))
+#define GARROW_IS_FEATHER_FILE_READER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_FEATHER_FILE_READER))
+#define GARROW_FEATHER_FILE_READER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_FEATHER_FILE_READER,   \
+                             GArrowFeatherFileReaderClass))
+
+typedef struct _GArrowFeatherFileReader      GArrowFeatherFileReader;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowFeatherFileReaderClass GArrowFeatherFileReaderClass;
+#endif
+
+/**
+ * GArrowFeatherFileReader:
+ *
+ * It wraps `arrow::ipc::feather::TableReader`.
+ */
+struct _GArrowFeatherFileReader
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowFeatherFileReaderClass
+{
+  GObjectClass parent_class;
+};
+#endif
+
+GType garrow_feather_file_reader_get_type(void) G_GNUC_CONST;
+
+GArrowFeatherFileReader *garrow_feather_file_reader_new(
+  GArrowSeekableInputStream *file,
+  GError **error);
+
+gchar *garrow_feather_file_reader_get_description(
+  GArrowFeatherFileReader *reader);
+gboolean garrow_feather_file_reader_has_description(
+  GArrowFeatherFileReader *reader);
+gint garrow_feather_file_reader_get_version(
+  GArrowFeatherFileReader *reader);
+gint64 garrow_feather_file_reader_get_n_rows(
+  GArrowFeatherFileReader *reader);
+gint64 garrow_feather_file_reader_get_n_columns(
+  GArrowFeatherFileReader *reader);
+gchar *garrow_feather_file_reader_get_column_name(
+  GArrowFeatherFileReader *reader,
+  gint i);
+GArrowColumn *garrow_feather_file_reader_get_column(
+  GArrowFeatherFileReader *reader,
+  gint i,
+  GError **error);
+GList *garrow_feather_file_reader_get_columns(
+  GArrowFeatherFileReader *reader,
+  GError **error);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/reader.hpp b/c_glib/arrow-glib/reader.hpp
index 7d0b24a65f608..7970e731deb01 100644
--- a/c_glib/arrow-glib/reader.hpp
+++ b/c_glib/arrow-glib/reader.hpp
@@ -21,6 +21,7 @@
 
 #include <arrow/api.h>
 #include <arrow/ipc/api.h>
+#include <arrow/ipc/feather.h>
 
 #include <arrow-glib/reader.h>
 
@@ -31,3 +32,6 @@ GArrowRecordBatchStreamReader *garrow_record_batch_stream_reader_new_raw(std::sh
 
 GArrowRecordBatchFileReader *garrow_record_batch_file_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileReader> *arrow_reader);
 std::shared_ptr<arrow::ipc::RecordBatchFileReader> garrow_record_batch_file_reader_get_raw(GArrowRecordBatchFileReader *reader);
+
+GArrowFeatherFileReader *garrow_feather_file_reader_new_raw(arrow::ipc::feather::TableReader *arrow_reader);
+arrow::ipc::feather::TableReader *garrow_feather_file_reader_get_raw(GArrowFeatherFileReader *reader);
diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp
index 20dd281cfdf68..d4c36c2a8ddea 100644
--- a/c_glib/arrow-glib/writer.cpp
+++ b/c_glib/arrow-glib/writer.cpp
@@ -21,8 +21,6 @@
 #  include <config.h>
 #endif
 
-#include <arrow/ipc/api.h>
-
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/record-batch.hpp>
@@ -48,6 +46,9 @@ G_BEGIN_DECLS
  *
  * #GArrowRecordBatchFileWriter is a class for writing record
  * batches in file format into output.
+ *
+ * #GArrowFeatherFileWriter is a class for writing arrays
+ * in Feather file format into output.
  */
 
 typedef struct GArrowRecordBatchWriterPrivate_ {
@@ -271,8 +272,199 @@ garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
   }
 }
 
+
+typedef struct GArrowFeatherFileWriterPrivate_ {
+  arrow::ipc::feather::TableWriter *feather_table_writer;
+} GArrowFeatherFileWriterPrivate;
+
+enum {
+  PROP_0_,
+  PROP_FEATHER_TABLE_WRITER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowFeatherFileWriter,
+                           garrow_feather_file_writer,
+                           G_TYPE_OBJECT);
+
+#define GARROW_FEATHER_FILE_WRITER_GET_PRIVATE(obj)             \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_TYPE_FEATHER_FILE_WRITER, \
+                               GArrowFeatherFileWriterPrivate))
+
+static void
+garrow_feather_file_writer_finalize(GObject *object)
+{
+  GArrowFeatherFileWriterPrivate *priv;
+
+  priv = GARROW_FEATHER_FILE_WRITER_GET_PRIVATE(object);
+
+  delete priv->feather_table_writer;
+
+  G_OBJECT_CLASS(garrow_feather_file_writer_parent_class)->finalize(object);
+}
+
+static void
+garrow_feather_file_writer_set_property(GObject *object,
+                                        guint prop_id,
+                                        const GValue *value,
+                                        GParamSpec *pspec)
+{
+  GArrowFeatherFileWriterPrivate *priv;
+
+  priv = GARROW_FEATHER_FILE_WRITER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_FEATHER_TABLE_WRITER:
+    priv->feather_table_writer =
+      static_cast<arrow::ipc::feather::TableWriter *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_feather_file_writer_get_property(GObject *object,
+                                        guint prop_id,
+                                        GValue *value,
+                                        GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_feather_file_writer_init(GArrowFeatherFileWriter *object)
+{
+}
+
+static void
+garrow_feather_file_writer_class_init(GArrowFeatherFileWriterClass *klass)
+{
+  GObjectClass *gobject_class;
+  GParamSpec *spec;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_feather_file_writer_finalize;
+  gobject_class->set_property = garrow_feather_file_writer_set_property;
+  gobject_class->get_property = garrow_feather_file_writer_get_property;
+
+  spec = g_param_spec_pointer("feather-table-writer",
+                              "arrow::ipc::feather::TableWriter",
+                              "The raw std::shared<arrow::ipc::feather::TableWriter> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_FEATHER_TABLE_WRITER, spec);
+}
+
+/**
+ * garrow_feather_file_writer_new:
+ * @sink: The output of the writer.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowFeatherFileWriter
+ *   or %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowFeatherFileWriter *
+garrow_feather_file_writer_new(GArrowOutputStream *sink,
+                               GError **error)
+{
+  auto arrow_sink = garrow_output_stream_get_raw(sink);
+  std::unique_ptr<arrow::ipc::feather::TableWriter> arrow_writer;
+  auto status = arrow::ipc::feather::TableWriter::Open(arrow_sink,
+                                                       &arrow_writer);
+  if (garrow_error_check(error, status, "[feature-file-writer][new]")) {
+    return garrow_feather_file_writer_new_raw(arrow_writer.release());
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_feather_file_writer_set_description:
+ * @writer: A #GArrowFeatherFileWriter.
+ * @description: The description of the file.
+ *
+ * Since: 0.4.0
+ */
+void
+garrow_feather_file_writer_set_description(GArrowFeatherFileWriter *writer,
+                                           const gchar *description)
+{
+  auto arrow_writer = garrow_feather_file_writer_get_raw(writer);
+  arrow_writer->SetDescription(std::string(description));
+}
+
+/**
+ * garrow_feather_file_writer_set_n_rows:
+ * @writer: A #GArrowFeatherFileWriter.
+ * @n_rows: The number of rows in the file.
+ *
+ * Since: 0.4.0
+ */
+void
+garrow_feather_file_writer_set_n_rows(GArrowFeatherFileWriter *writer,
+                                      gint64 n_rows)
+{
+  auto arrow_writer = garrow_feather_file_writer_get_raw(writer);
+  arrow_writer->SetNumRows(n_rows);
+}
+
+/**
+ * garrow_feather_file_writer_append:
+ * @writer: A #GArrowFeatherFileWriter.
+ * @name: The name of the array to be appended.
+ * @array: The array to be appended.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_feather_file_writer_append(GArrowFeatherFileWriter *writer,
+                                  const gchar *name,
+                                  GArrowArray *array,
+                                  GError **error)
+{
+  auto arrow_writer = garrow_feather_file_writer_get_raw(writer);
+  auto arrow_array = garrow_array_get_raw(array);
+
+  auto status = arrow_writer->Append(std::string(name), *arrow_array);
+  return garrow_error_check(error,
+                            status,
+                            "[feather-file-writer][append]");
+}
+
+/**
+ * garrow_feather_file_writer_close:
+ * @writer: A #GArrowFeatherFileWriter.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.4.0
+ */
+gboolean
+garrow_feather_file_writer_close(GArrowFeatherFileWriter *writer,
+                                 GError **error)
+{
+  auto arrow_writer = garrow_feather_file_writer_get_raw(writer);
+
+  auto status = arrow_writer->Finalize();
+  return garrow_error_check(error, status, "[feather-file-writer][close]");
+}
+
 G_END_DECLS
 
+
 GArrowRecordBatchWriter *
 garrow_record_batch_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchWriter> *arrow_writer)
 {
@@ -314,3 +506,23 @@ garrow_record_batch_file_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchF
                    NULL));
   return writer;
 }
+
+GArrowFeatherFileWriter *
+garrow_feather_file_writer_new_raw(arrow::ipc::feather::TableWriter *arrow_writer)
+{
+  auto writer =
+    GARROW_FEATHER_FILE_WRITER(
+      g_object_new(GARROW_TYPE_FEATHER_FILE_WRITER,
+                   "feather-table-writer", arrow_writer,
+                   NULL));
+  return writer;
+}
+
+arrow::ipc::feather::TableWriter *
+garrow_feather_file_writer_get_raw(GArrowFeatherFileWriter *writer)
+{
+  GArrowFeatherFileWriterPrivate *priv;
+
+  priv = GARROW_FEATHER_FILE_WRITER_GET_PRIVATE(writer);
+  return priv->feather_table_writer;
+}
diff --git a/c_glib/arrow-glib/writer.h b/c_glib/arrow-glib/writer.h
index 9a9846179e5df..3853c2bd72c50 100644
--- a/c_glib/arrow-glib/writer.h
+++ b/c_glib/arrow-glib/writer.h
@@ -185,4 +185,64 @@ GArrowRecordBatchFileWriter *garrow_record_batch_file_writer_new(
   GArrowSchema *schema,
   GError **error);
 
+
+#define GARROW_TYPE_FEATHER_FILE_WRITER         \
+  (garrow_feather_file_writer_get_type())
+#define GARROW_FEATHER_FILE_WRITER(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_FEATHER_FILE_WRITER,  \
+                              GArrowFeatherFileWriter))
+#define GARROW_FEATHER_FILE_WRITER_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_FEATHER_FILE_WRITER,     \
+                           GArrowFeatherFileWriterClass))
+#define GARROW_IS_FEATHER_FILE_WRITER(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_FEATHER_FILE_WRITER))
+#define GARROW_IS_FEATHER_FILE_WRITER_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_FEATHER_FILE_WRITER))
+#define GARROW_FEATHER_FILE_WRITER_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_FEATHER_FILE_WRITER,   \
+                             GArrowFeatherFileWriterClass))
+
+typedef struct _GArrowFeatherFileWriter      GArrowFeatherFileWriter;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowFeatherFileWriterClass GArrowFeatherFileWriterClass;
+#endif
+
+/**
+ * GArrowFeatherFileWriter:
+ *
+ * It wraps `arrow::ipc::feather::TableWriter`.
+ */
+struct _GArrowFeatherFileWriter
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowFeatherFileWriterClass
+{
+  GObjectClass parent_class;
+};
+#endif
+
+GType garrow_feather_file_writer_get_type(void) G_GNUC_CONST;
+
+GArrowFeatherFileWriter *garrow_feather_file_writer_new(GArrowOutputStream *sink,
+                                                        GError **error);
+void garrow_feather_file_writer_set_description(GArrowFeatherFileWriter *writer,
+                                                const gchar *description);
+void garrow_feather_file_writer_set_n_rows(GArrowFeatherFileWriter *writer,
+                                           gint64 n_rows);
+gboolean garrow_feather_file_writer_append(GArrowFeatherFileWriter *writer,
+                                           const gchar *name,
+                                           GArrowArray *array,
+                                           GError **error);
+gboolean garrow_feather_file_writer_close(GArrowFeatherFileWriter *writer,
+                                          GError **error);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/writer.hpp b/c_glib/arrow-glib/writer.hpp
index c9ea6603c9ea4..d57f69b657d9d 100644
--- a/c_glib/arrow-glib/writer.hpp
+++ b/c_glib/arrow-glib/writer.hpp
@@ -21,6 +21,7 @@
 
 #include <arrow/api.h>
 #include <arrow/ipc/api.h>
+#include <arrow/ipc/feather.h>
 
 #include <arrow-glib/writer.h>
 
@@ -30,3 +31,6 @@ std::shared_ptr<arrow::ipc::RecordBatchWriter> garrow_record_batch_writer_get_ra
 GArrowRecordBatchStreamWriter *garrow_record_batch_stream_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> *arrow_writer);
 
 GArrowRecordBatchFileWriter *garrow_record_batch_file_writer_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileWriter> *arrow_writer);
+
+GArrowFeatherFileWriter *garrow_feather_file_writer_new_raw(arrow::ipc::feather::TableWriter *arrow_writer);
+arrow::ipc::feather::TableWriter *garrow_feather_file_writer_get_raw(GArrowFeatherFileWriter *writer);
diff --git a/c_glib/test/test-feather-file-reader.rb b/c_glib/test/test-feather-file-reader.rb
new file mode 100644
index 0000000000000..91f2f7467f59b
--- /dev/null
+++ b/c_glib/test/test-feather-file-reader.rb
@@ -0,0 +1,155 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestFeatherFileReader < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def setup_file(data)
+    tempfile = Tempfile.open("arrow-feather-file-reader")
+    output = Arrow::FileOutputStream.new(tempfile.path, false)
+    begin
+      writer = Arrow::FeatherFileWriter.new(output)
+      begin
+        if data[:description]
+          writer.description = data[:description]
+        end
+        writer.n_rows = data[:n_rows] || 0
+        if data[:columns]
+          data[:columns].each do |name, array|
+            writer.append(name, array)
+          end
+        end
+      ensure
+        writer.close
+      end
+    ensure
+      output.close
+    end
+
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      reader = Arrow::FeatherFileReader.new(input)
+      yield(reader)
+    ensure
+      input.close
+    end
+  end
+
+  sub_test_case("#description") do
+    test("exist") do
+      setup_file(:description => "Log") do |reader|
+        assert_equal("Log", reader.description)
+      end
+    end
+
+    test("not exist") do
+      setup_file(:description => nil) do |reader|
+        assert_nil(reader.description)
+      end
+    end
+  end
+
+  sub_test_case("#has_description?") do
+    test("exist") do
+      setup_file(:description => "Log") do |reader|
+        assert do
+          reader.has_description?
+        end
+      end
+    end
+
+    test("not exist") do
+      setup_file(:description => nil) do |reader|
+        assert do
+          not reader.has_description?
+        end
+      end
+    end
+  end
+
+  test("#version") do
+    setup_file({}) do |reader|
+      assert do
+        reader.version >= 2
+      end
+    end
+  end
+
+  test("#n_rows") do
+    setup_file(:n_rows => 3) do |reader|
+      assert_equal(3, reader.n_rows)
+    end
+  end
+
+  test("#n_columns") do
+    columns = {
+      "messages" => build_string_array([]),
+      "is_critical" => build_boolean_array([]),
+    }
+    setup_file(:columns => columns) do |reader|
+      assert_equal(2, reader.n_columns)
+    end
+  end
+
+  test("#get_column_name") do
+    columns = {
+      "messages" => build_string_array([]),
+      "is_critical" => build_boolean_array([]),
+    }
+    setup_file(:columns => columns) do |reader|
+      assert_equal([
+                     "messages",
+                     "is_critical",
+                   ],
+                   [
+                     reader.get_column_name(0),
+                     reader.get_column_name(1),
+                   ])
+    end
+  end
+
+  test("#get_column") do
+    columns = {
+      "messages" => build_string_array([]),
+      "is_critical" => build_boolean_array([]),
+    }
+    setup_file(:columns => columns) do |reader|
+      assert_equal([
+                     "messages",
+                     "is_critical",
+                   ],
+                   [
+                     reader.get_column(0).name,
+                     reader.get_column(1).name,
+                   ])
+    end
+  end
+
+  test("#columns") do
+    columns = {
+      "messages" => build_string_array([]),
+      "is_critical" => build_boolean_array([]),
+    }
+    setup_file(:columns => columns) do |reader|
+      assert_equal([
+                     "messages",
+                     "is_critical",
+                   ],
+                   reader.columns.collect(&:name))
+    end
+  end
+end
diff --git a/c_glib/test/test-feather-file-writer.rb b/c_glib/test/test-feather-file-writer.rb
new file mode 100644
index 0000000000000..9939215912f52
--- /dev/null
+++ b/c_glib/test/test-feather-file-writer.rb
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestFeatherFileWriter < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_append
+    tempfile = Tempfile.open("arrow-feather-file-writer")
+    output = Arrow::FileOutputStream.new(tempfile.path, false)
+    begin
+      writer = Arrow::FeatherFileWriter.new(output)
+      begin
+        writer.description = "Log"
+        writer.n_rows = 3
+        writer.append("message",
+                      build_string_array(["Crash", "Error", "Shutdown"]))
+        writer.append("is_critical",
+                      build_boolean_array([true, true, false]))
+      ensure
+        writer.close
+      end
+    ensure
+      output.close
+    end
+
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      reader = Arrow::FeatherFileReader.new(input)
+      assert_equal([true, "Log"],
+                   [reader.has_description?, reader.description])
+      column_values = {}
+      reader.columns.each do |column|
+        values = []
+        column.data.chunks.each do |array|
+          array.length.times do |j|
+            if array.respond_to?(:get_string)
+              values << array.get_string(j)
+            else
+              values << array.get_value(j)
+            end
+          end
+        end
+        column_values[column.name] = values
+      end
+      assert_equal({
+                     "message" => ["Crash", "Error", "Shutdown"],
+                     "is_critical" => [true, true, false],
+                   },
+                   column_values)
+    ensure
+      input.close
+    end
+  end
+end

From 0eec40aa5d4ea955b45bd934899e178530fd9dc9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 17 May 2017 15:36:04 -0400
Subject: [PATCH 0669/1644] ARROW-1046: [Python] Reconcile pandas metadata spec

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #701 from wesm/reconcile-pandas-metadata and squashes the following commits:

f08d3ca3 [Wes McKinney] The pandas spec is numpy_type instead of numpy_dtype
---
 python/pyarrow/pandas_compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 2f72d6a6382f9..255b31a6751ad 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -70,7 +70,7 @@ def get_column_metadata(column, name):
             'integer': str(dtype),
             'floating': str(dtype),
         }.get(inferred_dtype, inferred_dtype),
-        'numpy_dtype': str(dtype),
+        'numpy_type': str(dtype),
         'metadata': extra_metadata,
     }
 

From 37cdc6e99bd935c3ce23e840d6d5ba4e070bc3b6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 17 May 2017 15:37:15 -0400
Subject: [PATCH 0670/1644] ARROW-970: [Python] Nicer experience if user
 accidentally calls pyarrow.Table ctor directly

This segfaulted before

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #702 from wesm/ARROW-970 and squashes the following commits:

508d9899 [Wes McKinney] Add some null checks into Table methods
---
 python/pyarrow/table.pxi           | 8 ++++++--
 python/pyarrow/tests/test_table.py | 5 +++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index c9915c1be3c5c..25a4f84cfc880 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -825,6 +825,7 @@ cdef class Table:
         -------
         pandas.DataFrame
         """
+        self._check_nullptr()
         if nthreads is None:
             nthreads = cpu_count()
 
@@ -845,6 +846,7 @@ cdef class Table:
             list entries = []
             Column column
 
+        self._check_nullptr()
         for i in range(num_columns):
             column = self.column(i)
             entries.append((column.name, column.to_pylist()))
@@ -860,6 +862,7 @@ cdef class Table:
         -------
         pyarrow.Schema
         """
+        self._check_nullptr()
         return pyarrow_wrap_schema(self.table.schema())
 
     def column(self, int64_t i):
@@ -874,13 +877,12 @@ cdef class Table:
         -------
         pyarrow.Column
         """
-        self._check_nullptr()
-
         cdef:
             Column column = Column()
             int64_t num_columns = self.num_columns
             int64_t index
 
+        self._check_nullptr()
         if not -num_columns <= i < num_columns:
             raise IndexError(
                 'Table column index {:d} is out of range'.format(i)
@@ -947,6 +949,7 @@ cdef class Table:
         Add column to Table at position. Returns new table
         """
         cdef shared_ptr[CTable] c_table
+        self._check_nullptr()
 
         with nogil:
             check_status(self.table.AddColumn(i, column.sp_column, &c_table))
@@ -964,6 +967,7 @@ cdef class Table:
         Create new Table with the indicated column removed
         """
         cdef shared_ptr[CTable] c_table
+        self._check_nullptr()
 
         with nogil:
             check_status(self.table.RemoveColumn(i, &c_table))
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index ed2201171a9ea..afc95205927be 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -275,3 +275,8 @@ def test_table_negative_indexing():
 
     with pytest.raises(IndexError):
         table[4]
+
+
+def test_table_ctor_errors():
+    with pytest.raises(ReferenceError):
+        repr(pa.Table())

From ff72951f0262ea230f7cfbb2f4c97ea7cf70845b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 19 May 2017 09:06:14 -0400
Subject: [PATCH 0671/1644] ARROW-1053: [Python] Remove unnecessary Py_INCREF
 in PyBuffer causing memory leak

cc @BryanCutler

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #704 from wesm/ARROW-1053 and squashes the following commits:

2f90337c [Wes McKinney] Remove unnecessary Py_INCREF in PyBuffer causing memory leak
---
 cpp/src/arrow/python/common.cc |  1 -
 python/scripts/test_leak.py    | 27 ++++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
index ba7b6cf202e32..a248db3abca48 100644
--- a/cpp/src/arrow/python/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -55,7 +55,6 @@ PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0), obj_(nullptr) {
     size_ = buffer->len;
     capacity_ = buffer->len;
     is_mutable_ = false;
-    Py_INCREF(obj_);
   }
 }
 
diff --git a/python/scripts/test_leak.py b/python/scripts/test_leak.py
index 2b197b6c13067..0b12fb5cd2566 100644
--- a/python/scripts/test_leak.py
+++ b/python/scripts/test_leak.py
@@ -21,6 +21,7 @@
 import numpy as np
 import memory_profiler
 import gc
+import io
 
 
 def leak():
@@ -32,4 +33,28 @@ def leak():
         table.to_pandas()
         gc.collect()
 
-leak()
+# leak()
+
+
+def leak2():
+    data = [pa.array(np.concatenate([np.random.randn(100000)] * 10))]
+    table = pa.Table.from_arrays(data, ['foo'])
+    while True:
+        print('calling to_pandas')
+        print('memory_usage: {0}'.format(memory_profiler.memory_usage()))
+        df = table.to_pandas()
+
+        batch = pa.RecordBatch.from_pandas(df)
+
+        sink = io.BytesIO()
+        writer = pa.RecordBatchFileWriter(sink, batch.schema)
+        writer.write_batch(batch)
+        writer.close()
+
+        buf_reader = pa.BufferReader(sink.getvalue())
+        reader = pa.open_file(buf_reader)
+        reader.read_all()
+
+        gc.collect()
+
+leak2()

From a8f8ba0cbcf5f596f042e90b7a208e7a0c3925b7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 19 May 2017 11:16:28 -0400
Subject: [PATCH 0672/1644] [maven-release-plugin] prepare release
 apache-arrow-0.4.0

Change-Id: I976a1bb24dde7f6b7b9fc2208b4f4995b1023756
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 4ef748235152d..6efdf93bc7c28 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.4.0-SNAPSHOT</version>
+  <version>0.4.0</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index e6d9900e24e80..bba043cdc001b 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.4.0-SNAPSHOT</version>
+    <version>0.4.0</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 1fa8ef9b457be..8237a30d19c25 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.4.0-SNAPSHOT</version>
+  <version>0.4.0</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>HEAD</tag>
+    <tag>apache-arrow-0.4.0</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 6124b85379fe4..31f931b2eae63 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.4.0-SNAPSHOT</version>
+        <version>0.4.0</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index e09193692a94d..260552348a706 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.4.0-SNAPSHOT</version>
+    <version>0.4.0</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From cf4ef5efb012a05ed7ce0086cb8136a1e5db412b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 22 May 2017 20:31:50 -0400
Subject: [PATCH 0673/1644] Increment version to 0.5.0-SNAPSHOT

Change-Id: I9b4a13830c41426ab728cdb2e2107c3537cd9ccf

Conflicts:
	java/format/pom.xml
	java/memory/pom.xml
	java/pom.xml
	java/tools/pom.xml
	java/vector/pom.xml
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 6efdf93bc7c28..e09275476d175 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.4.0</version>
+  <version>0.5.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index bba043cdc001b..dc4d0daf93d92 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.4.0</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 8237a30d19c25..1ec3d561121c3 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.4.0</version>
+  <version>0.5.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.4.0</tag>
+    <tag>HEAD</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 31f931b2eae63..53331f2f24541 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.4.0</version>
+        <version>0.5.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 260552348a706..a117a2fb3b7cb 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.4.0</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From a6e77f4b2f4e3d5affc2aef3c5f18fa534b90472 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 19 May 2017 11:12:33 -0400
Subject: [PATCH 0674/1644] ARROW-1054: [Python] Fix test failure on pandas
 0.19.2, some refactoring

For esoteric reasons, `MultiIndex.from_arrays` rejects non-writeable NumPy arrays. This problem isn't present in pandas 0.20.1

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #705 from wesm/ARROW-1054 and squashes the following commits:

f7dfbb24 [Wes McKinney] Factor out table to blocks into pandas_compat, fix pandas 0.19.2 test failure
---
 cpp/src/arrow/python/pandas_convert.cc |  2 +-
 python/pyarrow/ipc.py                  |  3 +-
 python/pyarrow/pandas_compat.py        | 73 +++++++++++++++++++++++++
 python/pyarrow/parquet.py              |  1 -
 python/pyarrow/table.pxi               | 76 ++------------------------
 5 files changed, 81 insertions(+), 74 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 96dd09aa817a3..ac61cbc13c6ef 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -1102,7 +1102,7 @@ static inline PyObject* NewArray1DFromType(
 
   set_numpy_metadata(type, arrow_type, descr);
   return PyArray_NewFromDescr(&PyArray_Type, descr, 1, dims, nullptr, data,
-      NPY_ARRAY_OWNDATA | NPY_ARRAY_CARRAY, nullptr);
+      NPY_ARRAY_OWNDATA | NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEABLE, nullptr);
 }
 
 class PandasBlock {
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index a61d746d65274..6173299bca6f5 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -158,4 +158,5 @@ def deserialize_pandas(buf, nthreads=1):
     """
     buffer_reader = pa.BufferReader(buf)
     reader = pa.RecordBatchFileReader(buffer_reader)
-    return reader.read_all().to_pandas(nthreads=nthreads)
+    table = reader.read_all()
+    return table.to_pandas(nthreads=nthreads)
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 255b31a6751ad..9711b722c2f8e 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -102,3 +102,76 @@ def construct_metadata(df, index_levels, preserve_index):
             }
         ).encode('utf8')
     }
+
+
+def table_to_blockmanager(table, nthreads=1):
+    import pandas.core.internals as _int
+    from pyarrow.compat import DatetimeTZDtype
+    import pyarrow.lib as lib
+
+    block_table = table
+
+    index_columns = []
+    index_arrays = []
+    index_names = []
+    schema = table.schema
+    row_count = table.num_rows
+    metadata = schema.metadata
+
+    if metadata is not None and b'pandas' in metadata:
+        pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
+        index_columns = pandas_metadata['index_columns']
+
+    for name in index_columns:
+        i = schema.get_field_index(name)
+        if i != -1:
+            col = table.column(i)
+            index_name = (None if is_unnamed_index_level(name)
+                          else name)
+            values = col.to_pandas().values
+            if not values.flags.writeable:
+                # ARROW-1054: in pandas 0.19.2, factorize will reject
+                # non-writeable arrays when calling MultiIndex.from_arrays
+                values = values.copy()
+
+            index_arrays.append(values)
+            index_names.append(index_name)
+            block_table = block_table.remove_column(
+                block_table.schema.get_field_index(name)
+            )
+
+    result = lib.table_to_blocks(block_table, nthreads)
+
+    blocks = []
+    for item in result:
+        block_arr = item['block']
+        placement = item['placement']
+        if 'dictionary' in item:
+            cat = pd.Categorical(block_arr,
+                                 categories=item['dictionary'],
+                                 ordered=False, fastpath=True)
+            block = _int.make_block(cat, placement=placement,
+                                    klass=_int.CategoricalBlock,
+                                    fastpath=True)
+        elif 'timezone' in item:
+            dtype = DatetimeTZDtype('ns', tz=item['timezone'])
+            block = _int.make_block(block_arr, placement=placement,
+                                    klass=_int.DatetimeTZBlock,
+                                    dtype=dtype, fastpath=True)
+        else:
+            block = _int.make_block(block_arr, placement=placement)
+        blocks.append(block)
+
+    if len(index_arrays) > 1:
+        index = pd.MultiIndex.from_arrays(index_arrays, names=index_names)
+    elif len(index_arrays) == 1:
+        index = pd.Index(index_arrays[0], name=index_names[0])
+    else:
+        index = pd.RangeIndex(row_count)
+
+    axes = [
+        [column.name for column in block_table.itercolumns()],
+        index
+    ]
+
+    return _int.BlockManager(blocks, axes)
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index f59a719b29efc..dc26daba90597 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import itertools
 import json
 
 import six
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 25a4f84cfc880..3f67ba40c7321 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -559,86 +559,20 @@ cdef class RecordBatch:
         return pyarrow_wrap_batch(batch)
 
 
-cdef table_to_blockmanager(const shared_ptr[CTable]& ctable, int nthreads):
-    import pandas.core.internals as _int
-    from pandas import RangeIndex, Categorical
-    from pyarrow.compat import DatetimeTZDtype
-
-    cdef:
-        Table table = pyarrow_wrap_table(ctable)
-        Table block_table = pyarrow_wrap_table(ctable)
-        Schema schema = table.schema
-
-        size_t row_count = table.num_rows
-        size_t total_columns = table.num_columns
-
-        dict metadata = schema.metadata
-        dict pandas_metadata = None
-
-        list index_columns = []
-        list index_arrays = []
-
-    if metadata is not None and b'pandas' in metadata:
-        pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
-        index_columns = pandas_metadata['index_columns']
-
-    cdef:
-        Column col
-        int64_t i
-
-    for name in index_columns:
-        i = schema.get_field_index(name)
-        if i != -1:
-            col = table.column(i)
-            index_name = None if pdcompat.is_unnamed_index_level(name) else name
-            index_arrays.append(
-                pd.Index(col.to_pandas().values, name=index_name)
-            )
-            block_table = block_table.remove_column(
-                block_table.schema.get_field_index(name)
-            )
-
+def table_to_blocks(Table table, int nthreads):
     cdef:
         PyObject* result_obj
-        shared_ptr[CTable] c_block_table = block_table.sp_table
+        shared_ptr[CTable] c_table = table.sp_table
 
     with nogil:
         check_status(
             libarrow.ConvertTableToPandas(
-                c_block_table, nthreads, &result_obj
+                c_table, nthreads, &result_obj
             )
         )
 
-    result = PyObject_to_object(result_obj)
-
-    blocks = []
-    for item in result:
-        block_arr = item['block']
-        placement = item['placement']
-        if 'dictionary' in item:
-            cat = Categorical(block_arr,
-                              categories=item['dictionary'],
-                              ordered=False, fastpath=True)
-            block = _int.make_block(cat, placement=placement,
-                                    klass=_int.CategoricalBlock,
-                                    fastpath=True)
-        elif 'timezone' in item:
-            dtype = DatetimeTZDtype('ns', tz=item['timezone'])
-            block = _int.make_block(block_arr, placement=placement,
-                                    klass=_int.DatetimeTZBlock,
-                                    dtype=dtype, fastpath=True)
-        else:
-            block = _int.make_block(block_arr, placement=placement)
-        blocks.append(block)
-
-    cdef list axes = [
-        [column.name for column in block_table.itercolumns()],
-        pd.MultiIndex.from_arrays(
-            index_arrays
-        ) if index_arrays else pd.RangeIndex(row_count),
-    ]
+    return PyObject_to_object(result_obj)
 
-    return _int.BlockManager(blocks, axes)
 
 
 cdef class Table:
@@ -829,7 +763,7 @@ cdef class Table:
         if nthreads is None:
             nthreads = cpu_count()
 
-        mgr = table_to_blockmanager(self.sp_table, nthreads)
+        mgr = pdcompat.table_to_blockmanager(self, nthreads)
         return pd.DataFrame(mgr)
 
     def to_pydict(self):

From b06602d08f5e5b741559449525edaa404bf1675c Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Mon, 22 May 2017 16:06:33 -0700
Subject: [PATCH 0675/1644] ARROW-1049: [java] vector template cleanup

Author: Julien Le Dem <julien@apache.org>

Closes #703 from julienledem/template_cleanup and squashes the following commits:

d67cb7e [Julien Le Dem] remove duplicated code
f17e8cd [Julien Le Dem] template cleanup
---
 .../main/codegen/data/ValueVectorTypes.tdd    |   2 +-
 .../templates/AbstractFieldWriter.java        |   6 -
 .../AbstractPromotableFieldWriter.java        |   6 -
 .../codegen/templates/ComplexWriters.java     |  11 -
 .../codegen/templates/FixedValueVectors.java  | 224 ++++--------------
 .../codegen/templates/HolderReaderImpl.java   | 134 +++--------
 .../templates/NullableValueVectors.java       |  31 +--
 .../codegen/templates/UnionListWriter.java    |  35 +--
 .../main/codegen/templates/UnionReader.java   |  41 ++--
 .../main/codegen/templates/UnionVector.java   |  68 +++---
 .../main/codegen/templates/UnionWriter.java   |  32 +--
 .../main/codegen/templates/ValueHolders.java  |   2 +-
 .../templates/VariableLengthVectors.java      |  11 +-
 13 files changed, 185 insertions(+), 418 deletions(-)

diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index 4d8c99e652ffd..970d887c76003 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -118,7 +118,7 @@
           maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal",
           typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}],
           arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal",
-          fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}]
+          fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}]
         }
       ]
     },
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 65bcc052ed42d..da8e4f54ec228 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -58,15 +58,9 @@ public void write(${name}Holder holder) {
     fail("${name}");
   }
 
-  <#if minor.class == "Decimal">
-  public void write${minor.class}(int start, ArrowBuf buffer) {
-    fail("${name}");
-  }
-  <#else>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
     fail("${name}");
   }
-  </#if>
 
   </#list></#list>
 
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 636b305d6db35..228c2c531f98f 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -78,15 +78,9 @@ public void write(${name}Holder holder) {
     getWriter(MinorType.${name?upper_case}).write(holder);
   }
 
-    <#if minor.class == "Decimal">
-  public void write${minor.class}(int start, ArrowBuf buffer) {
-    getWriter(MinorType.${name?upper_case}).write${minor.class}(start, buffer);
-  }
-    <#else>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
     getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
   }
-    </#if>
 
   </#list></#list>
   public void writeNull() {
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index c23b89d1bbcfb..8ebecf3e1de69 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -111,17 +111,10 @@ public void write(Nullable${minor.class}Holder h) {
     vector.getMutator().setValueCount(idx()+1);
   }
 
-  <#if minor.class == "Decimal">
-  public void writeDecimal(int start, ArrowBuf buffer) {
-    mutator.setSafe(idx(), 1, start, buffer);
-    vector.getMutator().setValueCount(idx()+1);
-  }
-  <#else>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
     mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
     vector.getMutator().setValueCount(idx()+1);
   }
-  </#if>
 
   <#if mode == "Nullable">
 
@@ -146,11 +139,7 @@ public void writeNull() {
 public interface ${eName}Writer extends BaseWriter {
   public void write(${minor.class}Holder h);
 
-  <#if minor.class == "Decimal">
-  public void writeDecimal(int start, ArrowBuf buffer);
-  <#else>
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
-  </#if>
 }
 
 </#list>
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 5d80b66b5332c..395417b1f6bc7 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -56,7 +56,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
   private int allocationMonitor = 0;
   <#if minor.typeParams??>
 
-     <#list minor.typeParams as typeParam>
+    <#list minor.typeParams as typeParam>
   private final ${typeParam.type} ${typeParam.name};
     </#list>
 
@@ -67,7 +67,6 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
     </#list>
   }
   <#else>
-
   public ${className}(String name, BufferAllocator allocator) {
     super(name, allocator);
   }
@@ -166,7 +165,7 @@ public void reset() {
     allocationMonitor = 0;
     zeroVector();
     super.reset();
-    }
+  }
 
   private void allocateBytes(final long size) {
     if (size > MAX_ALLOCATION_SIZE) {
@@ -180,11 +179,11 @@ private void allocateBytes(final long size) {
     allocationSizeInBytes = curSize;
   }
 
-/**
- * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
- *
- * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
- */
+  /**
+   * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+   *
+   * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
+   */
   public void reAlloc() {
     final long newAllocationSize = allocationSizeInBytes * 2L;
     if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
@@ -311,76 +310,12 @@ public boolean isNull(int index){
     }
 
     <#if (type.width > 8 || minor.class == "IntervalDay")>
-
     public ${minor.javaType!type.javaType} get(int index) {
       return data.slice(index * ${type.width}, ${type.width});
     }
 
-    <#if (minor.class == "Interval")>
-    public void get(int index, ${minor.class}Holder holder){
-
-      final int offsetIndex = index * ${type.width};
-      holder.months = data.getInt(offsetIndex);
-      holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
-      holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-    }
-
-    public void get(int index, Nullable${minor.class}Holder holder){
-      final int offsetIndex = index * ${type.width};
-      holder.isSet = 1;
-      holder.months = data.getInt(offsetIndex);
-      holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
-      holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-    }
-
-    @Override
-    public ${friendlyType} getObject(int index) {
-      final int offsetIndex = index * ${type.width};
-      final int months  = data.getInt(offsetIndex);
-      final int days    = data.getInt(offsetIndex + ${minor.daysOffset});
-      final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-      final Period p = new Period();
-      return p.plusMonths(months).plusDays(days).plusMillis(millis);
-    }
-
-    public StringBuilder getAsStringBuilder(int index) {
-
-      final int offsetIndex = index * ${type.width};
-
-      int months  = data.getInt(offsetIndex);
-      final int days    = data.getInt(offsetIndex + ${minor.daysOffset});
-      int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-
-      final int years  = (months / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-      months = (months % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-
-      final int hours  = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
-      millis     = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
-
-      final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
-      millis      = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
-
-      final long seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
-      millis      = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
-
-      final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
-      final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
-      final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
-
-
-      return(new StringBuilder().
-             append(years).append(yearString).
-             append(months).append(monthString).
-             append(days).append(dayString).
-             append(hours).append(":").
-             append(minutes).append(":").
-             append(seconds).append(".").
-             append(millis));
-    }
-
-    <#elseif (minor.class == "IntervalDay")>
+      <#if (minor.class == "IntervalDay")>
     public void get(int index, ${minor.class}Holder holder){
-
       final int offsetIndex = index * ${type.width};
       holder.days = data.getInt(offsetIndex);
       holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
@@ -402,7 +337,6 @@ public void get(int index, Nullable${minor.class}Holder holder){
       return p.plusDays(days).plusMillis(millis);
     }
 
-
     public StringBuilder getAsStringBuilder(int index) {
       final int offsetIndex = index * ${type.width};
 
@@ -428,8 +362,7 @@ public StringBuilder getAsStringBuilder(int index) {
               append(millis));
     }
 
-    <#elseif minor.class == "Decimal">
-
+      <#elseif minor.class == "Decimal">
     public void get(int index, ${minor.class}Holder holder) {
         holder.start = index * ${type.width};
         holder.buffer = data;
@@ -450,7 +383,7 @@ public void get(int index, Nullable${minor.class}Holder holder) {
       return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(data, ${type.width} * index, scale);
     }
 
-    <#else>
+      <#else>
     public void get(int index, ${minor.class}Holder holder){
       holder.buffer = data;
       holder.start = index * ${type.width};
@@ -467,44 +400,35 @@ public void get(int index, Nullable${minor.class}Holder holder){
       return data.slice(index * ${type.width}, ${type.width})
     }
 
-    </#if>
+      </#if>
     <#else> <#-- type.width <= 8 -->
 
     public ${minor.javaType!type.javaType} get(int index) {
       return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
     }
-
-    <#if type.width == 4>
+      <#if type.width == 4>
     public long getTwoAsLong(int index) {
       return data.getLong(index * ${type.width});
     }
 
-    </#if>
-
-    <#if minor.class == "DateDay" ||
-         minor.class == "TimeSec" ||
-         minor.class == "TimeMicro" ||
-         minor.class == "TimeNano">
+      </#if>
+      <#if minor.class == "DateDay" ||
+           minor.class == "TimeSec" ||
+           minor.class == "TimeMicro" ||
+           minor.class == "TimeNano">
     @Override
     public ${friendlyType} getObject(int index) {
       return get(index);
     }
 
-    <#elseif minor.class == "DateMilli">
+      <#elseif minor.class == "DateMilli" || minor.class == "TimeMilli" || minor.class == "TimeStampMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      return date;
+      org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
+      return ldt;
     }
 
-    <#elseif minor.class == "TimeMilli">
-    @Override
-    public ${friendlyType} getObject(int index) {
-      org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      return time;
-    }
-
-    <#elseif minor.class == "TimeStampSec">
+      <#elseif minor.class == "TimeStampSec">
     @Override
     public ${friendlyType} getObject(int index) {
       long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
@@ -512,14 +436,7 @@ public long getTwoAsLong(int index) {
       return date;
     }
 
-    <#elseif minor.class == "TimeStampMilli">
-    @Override
-    public ${friendlyType} getObject(int index) {
-        org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        return date;
-    }
-
-    <#elseif minor.class == "TimeStampMicro">
+      <#elseif minor.class == "TimeStampMicro">
     @Override
     public ${friendlyType} getObject(int index) {
       // value is truncated when converting microseconds to milliseconds in order to use DateTime type
@@ -528,7 +445,7 @@ public long getTwoAsLong(int index) {
       return date;
     }
 
-    <#elseif minor.class == "TimeStampNano">
+      <#elseif minor.class == "TimeStampNano">
     @Override
     public ${friendlyType} getObject(int index) {
       // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
@@ -537,7 +454,7 @@ public long getTwoAsLong(int index) {
       return date;
     }
 
-    <#elseif minor.class == "IntervalYear">
+      <#elseif minor.class == "IntervalYear">
     @Override
     public ${friendlyType} getObject(int index) {
 
@@ -550,7 +467,6 @@ public long getTwoAsLong(int index) {
     }
 
     public StringBuilder getAsStringBuilder(int index) {
-
       int months  = data.getInt(index);
 
       final int years  = (months / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
@@ -564,15 +480,7 @@ public StringBuilder getAsStringBuilder(int index) {
              append(months).append(monthString));
     }
 
-    <#elseif minor.class == "Decimal9" || minor.class == "Decimal18">
-    @Override
-    public ${friendlyType} getObject(int index) {
-
-        final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value());
-        return new BigDecimal(value, getField().getScale());
-    }
-
-    <#else>
+      <#else>
     @Override
     public ${friendlyType} getObject(int index) {
       return get(index);
@@ -580,14 +488,9 @@ public StringBuilder getAsStringBuilder(int index) {
     public ${minor.javaType!type.javaType} getPrimitiveObject(int index) {
       return get(index);
     }
-    </#if>
 
-    public void get(int index, ${minor.class}Holder holder){
-      <#if minor.class.startsWith("Decimal")>
-      holder.scale = getField().getScale();
-      holder.precision = getField().getPrecision();
       </#if>
-
+    public void get(int index, ${minor.class}Holder holder){
       holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
     }
 
@@ -596,22 +499,22 @@ public void get(int index, Nullable${minor.class}Holder holder){
       holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
     }
 
+    </#if> <#-- type.width -->
+  }
 
-   </#if> <#-- type.width -->
- }
-
- /**
-  * ${minor.class}.Mutator implements a mutable vector of fixed width values.  Elements in the
-  * vector are accessed by position from the logical start of the vector.  Values should be pushed
-  * onto the vector sequentially, but may be randomly accessed.
-  *   The width of each element is ${type.width} byte(s)
-  *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
-  *
-  * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
-  */
+  /**
+   * ${minor.class}.Mutator implements a mutable vector of fixed width values.  Elements in the
+   * vector are accessed by position from the logical start of the vector.  Values should be pushed
+   * onto the vector sequentially, but may be randomly accessed.
+   *   The width of each element is ${type.width} byte(s)
+   *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+   *
+   * NB: this class is automatically generated from FixedValueVectorTypes.tdd using FreeMarker.
+   */
   public final class Mutator extends BaseDataValueVector.BaseMutator {
 
     private Mutator(){};
+
    /**
     * Set the element at the given index to the given value.  Note that widths smaller than
     * 32 bits are handled by the ArrowBuf interface.
@@ -619,7 +522,7 @@ public final class Mutator extends BaseDataValueVector.BaseMutator {
     * @param index   position of the bit to set
     * @param value   value to set
     */
-  <#if (type.width > 8) || minor.class == "IntervalDay">
+    <#if (type.width > 8) || minor.class == "IntervalDay">
    public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
      data.setBytes(index * ${type.width}, value, 0, ${type.width});
    }
@@ -631,38 +534,7 @@ public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaT
      data.setBytes(index * ${type.width}, value, 0, ${type.width});
    }
 
-  <#if (minor.class == "Interval")>
-   public void set(int index, int months, int days, int milliseconds){
-     final int offsetIndex = index * ${type.width};
-     data.setInt(offsetIndex, months);
-     data.setInt((offsetIndex + ${minor.daysOffset}), days);
-     data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
-   }
-
-   protected void set(int index, ${minor.class}Holder holder){
-     set(index, holder.months, holder.days, holder.milliseconds);
-   }
-
-   protected void set(int index, Nullable${minor.class}Holder holder){
-     set(index, holder.months, holder.days, holder.milliseconds);
-   }
-
-   public void setSafe(int index, int months, int days, int milliseconds){
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     set(index, months, days, milliseconds);
-   }
-
-   public void setSafe(int index, Nullable${minor.class}Holder holder){
-     setSafe(index, holder.months, holder.days, holder.milliseconds);
-   }
-
-   public void setSafe(int index, ${minor.class}Holder holder){
-     setSafe(index, holder.months, holder.days, holder.milliseconds);
-   }
-
-   <#elseif (minor.class == "IntervalDay")>
+      <#if (minor.class == "IntervalDay")>
    public void set(int index, int days, int milliseconds){
      final int offsetIndex = index * ${type.width};
      data.setInt(offsetIndex, days);
@@ -672,6 +544,7 @@ public void set(int index, int days, int milliseconds){
    protected void set(int index, ${minor.class}Holder holder){
      set(index, holder.days, holder.milliseconds);
    }
+
    protected void set(int index, Nullable${minor.class}Holder holder){
      set(index, holder.days, holder.milliseconds);
    }
@@ -691,8 +564,7 @@ public void setSafe(int index, Nullable${minor.class}Holder holder){
      setSafe(index, holder.days, holder.milliseconds);
    }
 
-   <#elseif minor.class == "Decimal">
-
+       <#elseif minor.class == "Decimal">
    public void set(int index, ${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
    }
@@ -719,8 +591,7 @@ public void set(int index, int start, ArrowBuf buffer){
      data.setBytes(index * ${type.width}, buffer, start, ${type.width});
    }
 
-   <#else>
-
+       <#else>
    protected void set(int index, ${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
    }
@@ -736,6 +607,7 @@ public void set(int index, int start, ArrowBuf buffer){
    public void setSafe(int index, ${minor.class}Holder holder){
      setSafe(index, holder.start, holder.buffer);
    }
+
    public void setSafe(int index, Nullable${minor.class}Holder holder){
      setSafe(index, holder.start, holder.buffer);
    }
@@ -750,7 +622,7 @@ public void setSafe(int index, int start, ArrowBuf buffer){
    public void set(int index, Nullable${minor.class}Holder holder){
      data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
    }
-   </#if>
+       </#if>
 
    @Override
    public void generateTestData(int count) {
@@ -765,7 +637,7 @@ public void generateTestData(int count) {
      }
    }
 
-   <#else> <#-- type.width <= 8 -->
+     <#else> <#-- type.width <= 8 -->
    public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
      data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
    }
@@ -826,7 +698,7 @@ public void generateTestDataAlt(int size) {
      }
    }
 
-  </#if> <#-- type.width -->
+    </#if> <#-- type.width -->
 
    @Override
    public void setValueCount(int valueCount) {
@@ -846,6 +718,6 @@ public void setValueCount(int valueCount) {
  }
 }
 
-</#if> <#-- type.major -->
+  </#if> <#-- type.major -->
 </#list>
 </#list>
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index e990fcc933479..471fe4125725b 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -29,7 +29,7 @@
 <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-<#assign fields = minor.fields!type.fields />
+<#assign fields = minor.fields!type.fields + minor.typeParams![]/>
 
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${holderMode}${name}HolderReaderImpl.java" />
 <#include "/@includes/license.ftl" />
@@ -81,7 +81,6 @@ public boolean isSet() {
     <#else>
     return true;
     </#if>
-    
   }
 
   @Override
@@ -101,121 +100,50 @@ public void read(Nullable${name}Holder h) {
 
   // read friendly type
   @Override
-  public ${friendlyType} read${safeType}(){
-<#if nullMode == "Nullable">
+  public ${friendlyType} read${safeType}() {
+  <#if nullMode == "Nullable">
     if (!isSet()) {
       return null;
     }
-</#if>
-
-<#if type.major == "VarLen">
-
-      int length = holder.end - holder.start;
-      byte[] value = new byte [length];
-      holder.buffer.getBytes(holder.start, value, 0, length);
-
-  <#if minor.class == "VarBinary">
-      return value;
-  <#elseif minor.class == "Var16Char">
-      return new String(value);
-  <#elseif minor.class == "VarChar">
-      Text text = new Text();
-      text.set(value);
-      return text;
   </#if>
 
-<#elseif minor.class == "Interval">
-      Period p = new Period();
-      return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "IntervalDay">
-      Period p = new Period();
-      return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "Bit" >
-      return new Boolean(holder.value != 0);
-<#elseif minor.class == "Decimal" >
-        return (BigDecimal) readSingleObject();
-<#else>
-      ${friendlyType} value = new ${friendlyType}(this.holder.value);
-      return value;
-</#if>
-
+  <#if type.major == "VarLen">
+    int length = holder.end - holder.start;
+    byte[] value = new byte [length];
+    holder.buffer.getBytes(holder.start, value, 0, length);
+    <#if minor.class == "VarBinary">
+    return value;
+    <#elseif minor.class == "VarChar">
+    Text text = new Text();
+    text.set(value);
+    return text;
+    </#if>
+  <#elseif minor.class == "IntervalDay">
+    Period p = new Period();
+    return p.plusDays(holder.days).plusMillis(holder.milliseconds);
+  <#elseif minor.class == "Bit" >
+    return new Boolean(holder.value != 0);
+  <#elseif minor.class == "Decimal">
+    byte[] bytes = new byte[${type.width}];
+    holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
+    ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
+    return value;
+  <#else>
+    ${friendlyType} value = new ${friendlyType}(this.holder.value);
+    return value;
+  </#if>
   }
 
   @Override
   public Object readObject() {
-    return readSingleObject();
+    return read${safeType}();
   }
 
-  private Object readSingleObject() {
-<#if nullMode == "Nullable">
-    if (!isSet()) {
-      return null;
-    }
-</#if>
-
-<#if type.major == "VarLen">
-      <#if minor.class != "Decimal">
-      int length = holder.end - holder.start;
-      byte[] value = new byte [length];
-      holder.buffer.getBytes(holder.start, value, 0, length);
-      </#if>
-
-<#if minor.class == "VarBinary">
-      return value;
-<#elseif minor.class == "Var16Char">
-      return new String(value);
-<#elseif minor.class == "VarChar">
-      Text text = new Text();
-      text.set(value);
-      return text;
-<#elseif minor.class == "Decimal" >
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(holder.buffer, holder.start, holder.scale);
-</#if>
-
-<#elseif minor.class == "Interval">
-      Period p = new Period();
-      return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "IntervalDay">
-      Period p = new Period();
-      return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "Decimal28Dense" ||
-         minor.class == "Decimal38Dense">
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
-                                                                                holder.start,
-                                                                                holder.nDecimalDigits,
-                                                                                holder.scale,
-                                                                                holder.maxPrecision,
-                                                                                holder.WIDTH);
-
-<#elseif minor.class == "Decimal28Sparse" ||
-         minor.class == "Decimal38Sparse">
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
-                                                                                 holder.start,
-                                                                                 holder.nDecimalDigits,
-                                                                                 holder.scale);
-
-<#elseif minor.class == "Bit" >
-      return new Boolean(holder.value != 0);
-<#elseif minor.class == "Decimal">
-        byte[] bytes = new byte[${type.width}];
-        holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
-        ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
-        return value;
-<#else>
-      ${friendlyType} value = new ${friendlyType}(this.holder.value);
-      return value;
-</#if>
-  }
-
-<#if nullMode != "Nullable">
+  <#if nullMode != "Nullable">
   public void copyAsValue(${minor.class?cap_first}Writer writer){
     writer.write(holder);
   }
-</#if>
+  </#if>
 }
 
 </#list>
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 3231c4c6e0ae6..dfd57553acd09 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -470,11 +470,6 @@ public int getValueLength(int index) {
     public void get(int index, Nullable${minor.class}Holder holder){
       vAccessor.get(index, holder);
       holder.isSet = bAccessor.get(index);
-
-      <#if minor.class.startsWith("Decimal")>
-      holder.scale = scale;
-      holder.precision = precision;
-      </#if>
     }
 
     @Override
@@ -486,7 +481,7 @@ public void get(int index, Nullable${minor.class}Holder holder){
       }
     }
 
-    <#if minor.class == "Interval" || minor.class == "IntervalDay" || minor.class == "IntervalYear">
+    <#if minor.class == "Interval" || minor.class == "IntervalDay">
     public StringBuilder getAsStringBuilder(int index) {
       if (isNull(index)) {
           return null;
@@ -628,7 +623,7 @@ public boolean isSafe(int outIndex) {
     }
 
     <#assign fields = minor.fields!type.fields />
-    public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){
+    public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field</#list> ){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -644,7 +639,6 @@ public void setSafe(int index, int isSet<#list fields as field><#if field.includ
       <#if type.major == "VarLen">
       fillEmpties(index);
       </#if>
-
       bits.getMutator().setSafe(index, isSet);
       values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
       setCount++;
@@ -653,7 +647,6 @@ public void setSafe(int index, int isSet<#list fields as field><#if field.includ
 
 
     public void setSafe(int index, Nullable${minor.class}Holder value) {
-
       <#if type.major == "VarLen">
       fillEmpties(index);
       </#if>
@@ -664,7 +657,6 @@ public void setSafe(int index, Nullable${minor.class}Holder value) {
     }
 
     public void setSafe(int index, ${minor.class}Holder value) {
-
       <#if type.major == "VarLen">
       fillEmpties(index);
       </#if>
@@ -674,18 +666,17 @@ public void setSafe(int index, ${minor.class}Holder value) {
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
 
-    <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")>
-      public void setSafe(int index, ${minor.javaType!type.javaType} value) {
-        <#if type.major == "VarLen">
-        fillEmpties(index);
-        </#if>
-        bits.getMutator().setSafeToOne(index);
-        values.getMutator().setSafe(index, value);
-        setCount++;
-      }
+    <#if !(type.major == "VarLen" || minor.class == "IntervalYear" || minor.class == "IntervalDay")>
+    public void setSafe(int index, ${minor.javaType!type.javaType} value) {
+      <#if type.major == "VarLen">
+      fillEmpties(index);
+      </#if>
+      bits.getMutator().setSafeToOne(index);
+      values.getMutator().setSafe(index, value);
+      setCount++;
+    }
 
     </#if>
-
     @Override
     public void setValueCount(int valueCount) {
       assert valueCount >= 0;
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index d019a1edbb4a6..d6e2f9fd38d2c 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -89,27 +89,23 @@ public void close() throws Exception {
   public void setPosition(int index) {
     super.setPosition(index);
   }
-
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
   <#assign uncappedName = name?uncap_first/>
-
+  <#if uncappedName == "int" ><#assign uncappedName = "integer" /></#if>
   <#if !minor.typeParams?? >
 
   @Override
-  public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>() {
+  public ${name}Writer ${uncappedName}() {
     return this;
   }
 
   @Override
-  public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
-//    assert inMap;
+  public ${name}Writer ${uncappedName}(String name) {
     mapName = name;
-    return writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name);
+    return writer.${uncappedName}(name);
   }
-
   </#if>
-
   </#list></#list>
 
   @Override
@@ -149,33 +145,28 @@ public void endList() {
 
   @Override
   public void start() {
-//    assert inMap;
     writer.start();
   }
 
   @Override
   public void end() {
-//    if (inMap) {
     writer.end();
     inMap = false;
-//    }
   }
 
-  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-  <#assign fields = minor.fields!type.fields />
-  <#assign uncappedName = name?uncap_first/>
-
-  <#if !minor.class?starts_with("Decimal")>
-
+  <#list vv.types as type>
+    <#list type.minor as minor>
+      <#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#if !minor.typeParams?? >
   @Override
   public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-//    assert !inMap;
     writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
     writer.setPosition(writer.idx()+1);
   }
 
-  </#if>
-
-  </#list></#list>
-
+      </#if>
+    </#list>
+  </#list>
 }
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index 0b5a209d40ac4..2246fb36642b8 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -79,13 +79,16 @@ private FieldReader getReaderForIndex(int index) {
       return (FieldReader) getMap();
     case LIST:
       return (FieldReader) getList();
-    <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-    <#assign uncappedName = name?uncap_first/>
-    <#if !minor.class?starts_with("Decimal")>
+    <#list vv.types as type>
+      <#list type.minor as minor>
+        <#assign name = minor.class?cap_first />
+        <#assign uncappedName = name?uncap_first/>
+        <#if !minor.typeParams?? >
     case ${name?upper_case}:
       return (FieldReader) get${name}();
-    </#if>
-    </#list></#list>
+        </#if>
+      </#list>
+    </#list>
     default:
       throw new UnsupportedOperationException("Unsupported type: " + MinorType.values()[typeValue]);
     }
@@ -140,14 +143,16 @@ public int size() {
     return getReaderForIndex(idx()).size();
   }
 
-  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-          <#assign uncappedName = name?uncap_first/>
-  <#assign boxedType = (minor.boxedType!type.boxedType) />
-  <#assign javaType = (minor.javaType!type.javaType) />
-  <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-  <#assign safeType=friendlyType />
-  <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-  <#if !minor.class?starts_with("Decimal")>
+  <#list vv.types as type>
+    <#list type.minor as minor>
+      <#assign name = minor.class?cap_first />
+      <#assign uncappedName = name?uncap_first/>
+      <#assign boxedType = (minor.boxedType!type.boxedType) />
+      <#assign javaType = (minor.javaType!type.javaType) />
+      <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+      <#assign safeType=friendlyType />
+      <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+      <#if !minor.typeParams?? >
 
   private ${name}ReaderImpl ${uncappedName}Reader;
 
@@ -167,8 +172,9 @@ public void read(Nullable${name}Holder holder){
   public void copyAsValue(${name}Writer writer){
     getReaderForIndex(idx()).copyAsValue(writer);
   }
-  </#if>
-  </#list></#list>
+      </#if>
+    </#list>
+  </#list>
 
   @Override
   public void copyAsValue(ListWriter writer) {
@@ -184,7 +190,7 @@ public void setPosition(int index) {
       }
     }
   }
-  
+
   public FieldReader reader(String name){
     return getMap().reader(name);
   }
@@ -197,6 +203,3 @@ public boolean next() {
     return getReaderForIndex(idx()).next();
   }
 }
-
-
-
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index ec42b12e809ec..aa9d34d6e2666 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -145,12 +145,13 @@ public NullableMapVector getMap() {
     }
     return mapVector;
   }
-
-  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-  <#assign fields = minor.fields!type.fields />
-  <#assign uncappedName = name?uncap_first/>
-  <#assign lowerCaseName = name?lower_case/>
-  <#if !minor.class?starts_with("Decimal")>
+  <#list vv.types as type>
+    <#list type.minor as minor>
+      <#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#assign lowerCaseName = name?lower_case/>
+      <#if !minor.typeParams?? >
 
   private Nullable${name}Vector ${uncappedName}Vector;
 
@@ -167,10 +168,9 @@ public NullableMapVector getMap() {
     }
     return ${uncappedName}Vector;
   }
-
-  </#if>
-
-  </#list></#list>
+      </#if>
+    </#list>
+  </#list>
 
   public ListVector getList() {
     if (listVector == null) {
@@ -401,22 +401,23 @@ public Iterator<ValueVector> iterator() {
 
   public class Accessor extends BaseValueVector.BaseAccessor {
 
-
     @Override
     public Object getObject(int index) {
       int type = typeVector.getAccessor().get(index);
       switch (MinorType.values()[type]) {
       case NULL:
         return null;
-      <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-      <#assign fields = minor.fields!type.fields />
-      <#assign uncappedName = name?uncap_first/>
-      <#if !minor.class?starts_with("Decimal")>
+      <#list vv.types as type>
+        <#list type.minor as minor>
+          <#assign name = minor.class?cap_first />
+          <#assign fields = minor.fields!type.fields />
+          <#assign uncappedName = name?uncap_first/>
+          <#if !minor.typeParams?? >
       case ${name?upper_case}:
         return get${name}Vector().getAccessor().getObject(index);
-      </#if>
-
-      </#list></#list>
+          </#if>
+        </#list>
+      </#list>
       case MAP:
         return getMap().getAccessor().getObject(index);
       case LIST:
@@ -473,17 +474,20 @@ public void setSafe(int index, UnionHolder holder) {
       writer.setPosition(index);
       MinorType type = reader.getMinorType();
       switch (type) {
-      <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-      <#assign fields = minor.fields!type.fields />
-      <#assign uncappedName = name?uncap_first/>
-      <#if !minor.class?starts_with("Decimal")>
+      <#list vv.types as type>
+        <#list type.minor as minor>
+          <#assign name = minor.class?cap_first />
+          <#assign fields = minor.fields!type.fields />
+          <#assign uncappedName = name?uncap_first/>
+          <#if !minor.typeParams?? >
       case ${name?upper_case}:
         Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
         reader.read(${uncappedName}Holder);
         setSafe(index, ${uncappedName}Holder);
         break;
-      </#if>
-      </#list></#list>
+          </#if>
+        </#list>
+      </#list>
       case MAP: {
         ComplexCopier.copy(reader, writer);
         break;
@@ -496,18 +500,20 @@ public void setSafe(int index, UnionHolder holder) {
         throw new UnsupportedOperationException();
       }
     }
-
-    <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-    <#assign fields = minor.fields!type.fields />
-    <#assign uncappedName = name?uncap_first/>
-    <#if !minor.class?starts_with("Decimal")>
+    <#list vv.types as type>
+      <#list type.minor as minor>
+        <#assign name = minor.class?cap_first />
+        <#assign fields = minor.fields!type.fields />
+        <#assign uncappedName = name?uncap_first/>
+        <#if !minor.typeParams?? >
     public void setSafe(int index, Nullable${name}Holder holder) {
       setType(index, MinorType.${name?upper_case});
       get${name}Vector().getMutator().setSafe(index, holder);
     }
 
-    </#if>
-    </#list></#list>
+        </#if>
+      </#list>
+    </#list>
 
     public void setType(int index, MinorType type) {
       typeVector.getMutator().setSafe(index, (byte) type.ordinal());
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 4a7c47245cfb0..f892bac913392 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -117,26 +117,26 @@ BaseWriter getWriter(MinorType minorType) {
     case LIST:
       return getListWriter();
     <#list vv.types as type>
-    <#list type.minor as minor>
-      <#assign name = minor.class?cap_first />
-      <#assign fields = minor.fields!type.fields />
-      <#assign uncappedName = name?uncap_first/>
-      <#if !minor.typeParams??>
+      <#list type.minor as minor>
+        <#assign name = minor.class?cap_first />
+        <#assign fields = minor.fields!type.fields />
+        <#assign uncappedName = name?uncap_first/>
+        <#if !minor.typeParams??>
     case ${name?upper_case}:
       return get${name}Writer();
-      </#if>
-    </#list>
+        </#if>
+      </#list>
     </#list>
     default:
       throw new UnsupportedOperationException("Unknown type: " + minorType);
     }
   }
-
-  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
-  <#assign fields = minor.fields!type.fields />
-  <#assign uncappedName = name?uncap_first/>
-
-          <#if !minor.typeParams?? >
+  <#list vv.types as type>
+    <#list type.minor as minor>
+      <#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#if !minor.typeParams?? >
 
   private ${name}Writer ${name?uncap_first}Writer;
 
@@ -166,9 +166,9 @@ public void write(${name}Holder holder) {
     get${name}Writer().setPosition(idx());
     get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
   }
-  </#if>
-
-  </#list></#list>
+      </#if>
+    </#list>
+  </#list>
 
   public void writeNull() {
   }
diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java
index a474b691080c8..b74d91987fafb 100644
--- a/java/vector/src/main/codegen/templates/ValueHolders.java
+++ b/java/vector/src/main/codegen/templates/ValueHolders.java
@@ -50,7 +50,7 @@ public final class ${className} implements ValueHolder{
     
     <#if mode.name == "Optional">public int isSet;
     <#else>public final int isSet = 1;</#if>
-    <#assign fields = minor.fields!type.fields />
+    <#assign fields = (minor.fields!type.fields) + (minor.typeParams![]) />
     <#list fields as field>
     public ${field.type} ${field.name};
     </#list>
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index f13291b79ade0..2ad7d20de2651 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -421,23 +421,22 @@ public void get(int index, Nullable${minor.class}Holder holder){
       holder.buffer = data;
     }
 
-
     <#switch minor.class>
-    <#case "VarChar">
+      <#case "VarChar">
     @Override
     public ${friendlyType} getObject(int index) {
       Text text = new Text();
       text.set(get(index));
       return text;
     }
-    <#break>
-    <#case "Decimal">
+      <#break>
+      <#case "Decimal">
     @Override
     public ${friendlyType} getObject(int index) {
       return new BigDecimal(new BigInteger(get(index)), scale);
     }
-    <#break>
-    <#default>
+      <#break>
+      <#default>
     @Override
     public ${friendlyType} getObject(int index) {
       return get(index);

From d2cc199cc480f206110b34dc6abb0e3eb51eacac Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 23 May 2017 14:05:27 -0400
Subject: [PATCH 0676/1644] ARROW-1062: [GLib] Follow API changes in examples

Author: Kouhei Sutou <kou@clear-code.com>

Closes #709 from kou/glib-fix-examples and squashes the following commits:

ed5c465d [Kouhei Sutou] [GLib] Follow API changes in examples
---
 .travis.yml                                   |  1 +
 c_glib/arrow-glib/table.h                     |  2 --
 c_glib/example/go/read-batch.go               |  2 +-
 c_glib/example/go/read-stream.go              |  2 +-
 c_glib/example/go/write-batch.go              |  2 +-
 c_glib/example/go/write-stream.go             |  2 +-
 c_glib/example/lua/read-batch.lua             |  2 +-
 c_glib/example/lua/read-stream.lua            |  2 +-
 c_glib/example/lua/stream-to-torch-tensor.lua |  2 +-
 c_glib/example/lua/write-batch.lua            |  2 +-
 c_glib/example/lua/write-stream.lua           |  2 +-
 ci/travis_before_script_c_glib.sh             | 24 +++++++++++++++---
 ci/travis_env_common.sh                       |  1 +
 ci/travis_script_c_glib.sh                    | 25 ++++++++++++++++++-
 14 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 022ffb3562cb4..b212fcbcb9970 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,6 +17,7 @@ addons:
     - gtk-doc-tools
     - autoconf-archive
     - libgirepository1.0-dev
+    - gir1.2-gudev-1.0
 services:
   - docker
 
diff --git a/c_glib/arrow-glib/table.h b/c_glib/arrow-glib/table.h
index 7f83872163ffd..9e21669cd11da 100644
--- a/c_glib/arrow-glib/table.h
+++ b/c_glib/arrow-glib/table.h
@@ -86,6 +86,4 @@ GArrowTable    *garrow_table_remove_column (GArrowTable *table,
                                             guint i,
                                             GError **error);
 
-gchar          *garrow_table_to_string     (GArrowTable *table);
-
 G_END_DECLS
diff --git a/c_glib/example/go/read-batch.go b/c_glib/example/go/read-batch.go
index 3ad842376b02b..f47d3e20bc42e 100644
--- a/c_glib/example/go/read-batch.go
+++ b/c_glib/example/go/read-batch.go
@@ -82,7 +82,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Failed to open path: <%s>: %v", path, err)
 	}
-	reader, err := arrow.NewFileReader(input)
+	reader, err := arrow.NewRecordBatchFileReader(input)
 	if err != nil {
 		log.Fatalf("Failed to parse data: %v", err)
 	}
diff --git a/c_glib/example/go/read-stream.go b/c_glib/example/go/read-stream.go
index f3367988223e4..789b87bad6b1d 100644
--- a/c_glib/example/go/read-stream.go
+++ b/c_glib/example/go/read-stream.go
@@ -82,7 +82,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Failed to open path: <%s>: %v", path, err)
 	}
-	reader, err := arrow.NewStreamReader(input)
+	reader, err := arrow.NewRecordBatchStreamReader(input)
 	if err != nil {
 		log.Fatalf("Failed to parse data: %v", err)
 	}
diff --git a/c_glib/example/go/write-batch.go b/c_glib/example/go/write-batch.go
index 78ef1beca4fc5..cda09a9b4e8f7 100644
--- a/c_glib/example/go/write-batch.go
+++ b/c_glib/example/go/write-batch.go
@@ -130,7 +130,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Failed to open path: <%s>: %v", output_path, err)
 	}
-	writer, err := arrow.NewFileWriter(output, schema)
+	writer, err := arrow.NewRecordBatchFileWriter(output, schema)
 	if err != nil {
 		log.Fatalf("Failed to create writer: %v", err)
 	}
diff --git a/c_glib/example/go/write-stream.go b/c_glib/example/go/write-stream.go
index 3f08270275763..20cb03ef2e324 100644
--- a/c_glib/example/go/write-stream.go
+++ b/c_glib/example/go/write-stream.go
@@ -130,7 +130,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Failed to open path: <%s>: %v", output_path, err)
 	}
-	writer, err := arrow.NewStreamWriter(output, schema)
+	writer, err := arrow.NewRecordBatchStreamWriter(output, schema)
 	if err != nil {
 		log.Fatalf("Failed to create writer: %v", err)
 	}
diff --git a/c_glib/example/lua/read-batch.lua b/c_glib/example/lua/read-batch.lua
index 090a857ee555d..5e28d7efd5033 100644
--- a/c_glib/example/lua/read-batch.lua
+++ b/c_glib/example/lua/read-batch.lua
@@ -21,7 +21,7 @@ local Arrow = lgi.Arrow
 local input_path = arg[1] or "/tmp/batch.arrow";
 
 local input = Arrow.MemoryMappedInputStream.new(input_path)
-local reader = Arrow.FileReader.new(input)
+local reader = Arrow.RecordBatchFileReader.new(input)
 
 for i = 0, reader:get_n_record_batches() - 1 do
    local record_batch = reader:get_record_batch(i)
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index d7ac5ebbd2d97..95cf2e7a8a9f3 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -21,7 +21,7 @@ local Arrow = lgi.Arrow
 local input_path = arg[1] or "/tmp/stream.arrow";
 
 local input = Arrow.MemoryMappedInputStream.new(input_path)
-local reader = Arrow.StreamReader.new(input)
+local reader = Arrow.RecordBatchStreamReader.new(input)
 
 local i = 0
 while true do
diff --git a/c_glib/example/lua/stream-to-torch-tensor.lua b/c_glib/example/lua/stream-to-torch-tensor.lua
index 237d759d93e20..9825f105bd7cb 100644
--- a/c_glib/example/lua/stream-to-torch-tensor.lua
+++ b/c_glib/example/lua/stream-to-torch-tensor.lua
@@ -77,7 +77,7 @@ end
 local input_path = arg[1] or "/tmp/stream.arrow";
 
 local input = Arrow.MemoryMappedInputStream.new(input_path)
-local reader = Arrow.StreamReader.open(input)
+local reader = Arrow.RecordBatchStreamReader.new(input)
 
 local i = 0
 while true do
diff --git a/c_glib/example/lua/write-batch.lua b/c_glib/example/lua/write-batch.lua
index 663f8ef995551..26acc1dff764b 100644
--- a/c_glib/example/lua/write-batch.lua
+++ b/c_glib/example/lua/write-batch.lua
@@ -35,7 +35,7 @@ local fields = {
 local schema = Arrow.Schema.new(fields)
 
 local output = Arrow.FileOutputStream.new(output_path, false)
-local writer = Arrow.FileWriter.new(output, schema)
+local writer = Arrow.RecordBatchFileWriter.new(output, schema)
 
 function build_array(builder, values)
    for _, value in pairs(values) do
diff --git a/c_glib/example/lua/write-stream.lua b/c_glib/example/lua/write-stream.lua
index fb6cc557a98c2..07bbd79af43b1 100644
--- a/c_glib/example/lua/write-stream.lua
+++ b/c_glib/example/lua/write-stream.lua
@@ -35,7 +35,7 @@ local fields = {
 local schema = Arrow.Schema.new(fields)
 
 local output = Arrow.FileOutputStream.new(output_path, false)
-local writer = Arrow.StreamWriter.new(output, schema)
+local writer = Arrow.RecordBatchStreamWriter.new(output, schema)
 
 function build_array(builder, values)
    for _, value in pairs(values) do
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 74bdd94b96a8b..e397b0ecaf837 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -18,14 +18,32 @@ set -ex
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
-    brew install gtk-doc autoconf-archive gobject-introspection
+  brew install gtk-doc autoconf-archive gobject-introspection
 fi
 
 gem install gobject-introspection
 
-pushd $ARROW_C_GLIB_DIR
+git clone \
+  --quiet \
+  --depth 1 \
+  --recursive \
+  https://github.com/torch/distro.git ~/torch
+pushd ~/torch
+./install-deps > /dev/null
+echo "yes" | ./install.sh > /dev/null
+. ~/torch/install/bin/torch-activate
+popd
+luarocks install lgi
+
+go get github.com/linuxdeepin/go-gir-generator || :
+pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
+make build copyfile
+mkdir -p $GOPATH/bin/
+cp -a out/gir-generator $GOPATH/bin/
+cp -a out/src/gir/ $GOPATH/src/
+popd
 
-: ${ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install}
+pushd $ARROW_C_GLIB_DIR
 
 ./autogen.sh
 
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index 3515314722b71..b19fafab1d955 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -30,3 +30,4 @@ export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
 
 export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
 export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
+export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
diff --git a/ci/travis_script_c_glib.sh b/ci/travis_script_c_glib.sh
index 1492354405810..c4df713818c37 100755
--- a/ci/travis_script_c_glib.sh
+++ b/ci/travis_script_c_glib.sh
@@ -14,11 +14,34 @@
 
 set -e
 
-ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 pushd $ARROW_C_GLIB_DIR
 
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
 NO_MAKE=yes test/run-test.sh
 
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_C_GLIB_INSTALL/lib
+export GI_TYPELIB_PATH=$ARROW_C_GLIB_INSTALL/lib/girepository-1.0
+export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
+export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_C_GLIB_INSTALL/lib/pkgconfig
+
+pushd example/lua
+. ~/torch/install/bin/torch-activate
+luajit write-batch.lua
+luajit read-batch.lua
+luajit write-stream.lua
+luajit read-stream.lua
+luajit stream-to-torch-tensor.lua
+popd
+
+pushd example/go
+make generate
+make
+./write-batch
+./read-batch
+./write-stream
+./read-stream
+popd
+
 popd

From 84b7ee18e0cef288f74fa8e9d8825306daea850a Mon Sep 17 00:00:00 2001
From: "rip.nsk" <rip.nsk@gmail.com>
Date: Tue, 23 May 2017 14:06:32 -0400
Subject: [PATCH 0677/1644] ARROW-1057: Fix cmake warning and msvc debug
 asserts

Author: rip.nsk <rip.nsk@gmail.com>

Closes #706 from rip-nsk/ARROW-1057 and squashes the following commits:

9bfe3199 [rip.nsk] Fix dependency of "arrow_objlib".
0c1e75d3 [rip.nsk] Fix "string iterator not dereferencable" assert
0b842c09 [rip.nsk] Fix "vector subscript out of range" assert
---
 cpp/CMakeLists.txt                  | 3 +--
 cpp/src/arrow/array-decimal-test.cc | 3 +--
 cpp/src/arrow/util/decimal.cc       | 4 ++--
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0ad7ef560fb3c..d5483f228029c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -949,10 +949,9 @@ add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
 if (ARROW_IPC)
   add_subdirectory(src/arrow/ipc)
+  set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} metadata_fbs)
 endif()
 
-set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} metadata_fbs)
-
 set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/buffer.cc
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 4bde7abd9221a..fb4b8d983d345 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -75,8 +75,7 @@ class DecimalTest : public DecimalTestBase<T> {
  public:
   std::vector<uint8_t> data(
       const std::vector<T>& input, size_t byte_width) const override {
-    std::vector<uint8_t> result;
-    result.reserve(input.size() * byte_width);
+    std::vector<uint8_t> result(input.size() * byte_width);
     // TODO(phillipc): There's probably a better way to do this
     constexpr static const size_t bytes_per_element = sizeof(T);
     for (size_t i = 0, j = 0; i < input.size(); ++i, j += bytes_per_element) {
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 3d9fbd31bf22a..734df13e59d8f 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -29,8 +29,8 @@ ARROW_EXPORT Status FromString(
   }
 
   int8_t sign = 1;
-  auto charp = s.cbegin();
-  auto end = s.cend();
+  auto charp = s.c_str();
+  auto end = charp + s.length();
 
   if (*charp == '+' || *charp == '-') {
     if (*charp == '-') { sign = -1; }

From aa652cb3f5ea4b1f845ea2ef3e10344b050f45b3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 23 May 2017 14:08:39 -0400
Subject: [PATCH 0678/1644] ARROW-1060: [Python] Add unit tests for reference
 counts in memoryview interface

Add missing test case for ARROW-1053.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #710 from wesm/ARROW-1060 and squashes the following commits:

013fc248 [Wes McKinney] Add unit tests to check that reference counts stay constant via memoryview interface
---
 python/pyarrow/tests/test_io.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index f91dc6fa05d57..397b7a895c1e7 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -19,6 +19,7 @@
 import gc
 import os
 import pytest
+import sys
 
 import numpy as np
 
@@ -201,8 +202,6 @@ def test_inmemory_write_after_closed():
 
 
 def test_buffer_protocol_ref_counting():
-    import gc
-
     def make_buffer(bytes_obj):
         return bytearray(pa.frombuffer(bytes_obj))
 
@@ -210,6 +209,14 @@ def make_buffer(bytes_obj):
     gc.collect()
     assert buf == b'foo'
 
+    # ARROW-1053
+    val = b'foo'
+    refcount_before = sys.getrefcount(val)
+    for i in range(10):
+        make_buffer(val)
+    gc.collect()
+    assert refcount_before == sys.getrefcount(val)
+
 
 def test_nativefile_write_memoryview():
     f = pa.BufferOutputStream()

From 33117d9a7df9a38fb1100d428852c9f5c9d65e4d Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Tue, 23 May 2017 14:09:36 -0400
Subject: [PATCH 0679/1644] ARROW-1034: [PYTHON] Resolve wheel build issues on
 Windows

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #707 from MaxRis/ARROW-1034 and squashes the following commits:

796d1554 [Max Risuhin] ARROW-1034: [PYTHON] Resolve wheel build issues on Windows
---
 ci/msvc-build.bat                      |  2 +-
 python/CMakeLists.txt                  | 34 ++++++++++++++-------
 python/cmake_modules/FindArrow.cmake   | 41 ++++++++++++++++++++------
 python/cmake_modules/FindParquet.cmake | 41 +++++++++++++++++++-------
 python/setup.py                        | 23 +++++++++++----
 5 files changed, 105 insertions(+), 36 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index f756fc5195d2d..d13c11fabcf0e 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -68,6 +68,6 @@ popd
 set PYTHONPATH=
 
 pushd python
-python setup.py build_ext --inplace --with-parquet  || exit /B
+python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel  || exit /B
 py.test pyarrow -v -s || exit /B
 popd
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6f48f7f06eb43..b02f23bfff9ef 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -237,7 +237,7 @@ function(bundle_arrow_lib library_path)
     configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
         ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
         COPYONLY)
-  else()
+  else(NOT MSVC)
     configure_file(${${library_path}}.${ARROW_ABI_VERSION}
         ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
         COPYONLY)
@@ -257,10 +257,17 @@ if (PYARROW_BUNDLE_ARROW_CPP)
   bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
 endif()
 
-ADD_THIRDPARTY_LIB(arrow
-  SHARED_LIB ${ARROW_SHARED_LIB})
-ADD_THIRDPARTY_LIB(arrow_python
-  SHARED_LIB ${ARROW_PYTHON_SHARED_LIB})
+if (MSVC)
+  ADD_THIRDPARTY_LIB(arrow
+    SHARED_LIB ${ARROW_SHARED_IMP_LIB})
+  ADD_THIRDPARTY_LIB(arrow_python
+    SHARED_LIB ${ARROW_PYTHON_SHARED_IMP_LIB})
+else()
+  ADD_THIRDPARTY_LIB(arrow
+    SHARED_LIB ${ARROW_SHARED_LIB})
+  ADD_THIRDPARTY_LIB(arrow_python
+    SHARED_LIB ${ARROW_PYTHON_SHARED_LIB})
+endif()
 
 ############################################################
 # Subdirectories
@@ -312,7 +319,7 @@ if (PYARROW_BUILD_PARQUET)
           configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
               ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
               COPYONLY)
-      else()
+      elseif (NOT MSVC)
           configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
               ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
               COPYONLY)
@@ -332,10 +339,17 @@ if (PYARROW_BUILD_PARQUET)
       #SET(PARQUET_ARROW_SHARED_LIB
       #    ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
-  ADD_THIRDPARTY_LIB(parquet
-    SHARED_LIB ${PARQUET_SHARED_LIB})
-  ADD_THIRDPARTY_LIB(parquet_arrow
-    SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
+  if (MSVC)
+    ADD_THIRDPARTY_LIB(parquet
+      SHARED_LIB ${PARQUET_SHARED_IMP_LIB})
+    ADD_THIRDPARTY_LIB(parquet_arrow
+      SHARED_LIB ${PARQUET_ARROW_SHARED_IMP_LIB})
+  else()
+    ADD_THIRDPARTY_LIB(parquet
+      SHARED_LIB ${PARQUET_SHARED_LIB})
+    ADD_THIRDPARTY_LIB(parquet_arrow
+      SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
+  endif()
   set(LINK_LIBS
     ${LINK_LIBS}
     parquet_shared
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index c16a4bf05aa8b..4c8ed3dad2cdb 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -21,6 +21,7 @@
 #  ARROW_LIBS, directory containing arrow libraries
 #  ARROW_STATIC_LIB, path to libarrow.a
 #  ARROW_SHARED_LIB, path to libarrow's shared library
+#  ARROW_SHARED_IMP_LIB, path to libarrow's import library (MSVC only)
 #  ARROW_FOUND, whether arrow has been found
 
 include(FindPkgConfig)
@@ -69,23 +70,45 @@ find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
   PATHS
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
+get_filename_component(ARROW_PYTHON_LIBS ${ARROW_PYTHON_LIB_PATH} DIRECTORY)
+
+if (MSVC)
+  SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
+
+  if (MSVC AND NOT ARROW_MSVC_STATIC_LIB_SUFFIX)
+    set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
+  endif()
+
+  find_library(ARROW_SHARED_LIBRARIES NAMES arrow
+    PATHS ${ARROW_HOME} NO_DEFAULT_PATH
+    PATH_SUFFIXES "bin" )
+
+  find_library(ARROW_PYTHON_SHARED_LIBRARIES NAMES arrow_python
+    PATHS ${ARROW_HOME} NO_DEFAULT_PATH
+    PATH_SUFFIXES "bin" )
+  get_filename_component(ARROW_SHARED_LIBS ${ARROW_SHARED_LIBRARIES} PATH )
+  get_filename_component(ARROW_PYTHON_SHARED_LIBS ${ARROW_PYTHON_SHARED_LIBRARIES} PATH )
+endif ()
 
 if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   set(ARROW_FOUND TRUE)
-
+  set(ARROW_LIB_NAME arrow)
+  set(ARROW_PYTHON_LIB_NAME arrow_python)
   if (MSVC)
-    set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
-    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
-    set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
-    set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
+    set(ARROW_STATIC_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(ARROW_SHARED_LIB ${ARROW_SHARED_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_SHARED_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib)
+    set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib)
   else()
-    set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
-    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
+    set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_LIB_NAME}.a)
+    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_PYTHON_LIB_NAME}.a)
     set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
 
-    set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
     set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
-    set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
 endif()
 
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index de53a29879e0d..ef3c645f7682b 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -21,6 +21,7 @@
 #  PARQUET_LIBS, directory containing parquet libraries
 #  PARQUET_STATIC_LIB, path to libparquet.a
 #  PARQUET_SHARED_LIB, path to libparquet's shared library
+#  PARQUET_SHARED_IMP_LIB, path to libparquet's import library (MSVC only)
 #  PARQUET_FOUND, whether parquet has been found
 
 include(FindPkgConfig)
@@ -29,6 +30,24 @@ if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
     set(PARQUET_HOME "$ENV{PARQUET_HOME}")
 endif()
 
+if (MSVC)
+  SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
+
+  if (MSVC AND NOT PARQUET_MSVC_STATIC_LIB_SUFFIX)
+    set(PARQUET_MSVC_STATIC_LIB_SUFFIX "_static")
+  endif()
+
+  find_library(PARQUET_SHARED_LIBRARIES NAMES parquet
+    PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+    PATH_SUFFIXES "bin" )
+
+  find_library(PARQUET_ARROW_SHARED_LIBRARIES NAMES parquet_arrow
+    PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+    PATH_SUFFIXES "bin" )
+  get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )
+  get_filename_component(PARQUET_ARROW_SHARED_LIBS ${PARQUET_ARROW_SHARED_LIBRARIES} PATH )
+endif ()
+
 if(PARQUET_HOME)
     set(PARQUET_SEARCH_HEADER_PATHS
         ${PARQUET_HOME}/include
@@ -74,13 +93,14 @@ endif()
 
 if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
   set(PARQUET_FOUND TRUE)
+  set(PARQUET_LIB_NAME parquet)
   if (MSVC)
-    set(PARQUET_STATIC_LIB "${PARQUET_LIBRARIES}_static")
-    set(PARQUET_SHARED_LIB "${PARQUET_LIBRARIES}")
+    set(PARQUET_STATIC_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(PARQUET_SHARED_LIB "${PARQUET_SHARED_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(PARQUET_SHARED_IMP_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}.lib")
   else()
-    set(PARQUET_LIB_NAME libparquet)
-    set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
-    set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_LIB_NAME}.a)
+    set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
 else ()
   set(PARQUET_FOUND FALSE)
@@ -89,15 +109,16 @@ endif ()
 if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES)
   set(PARQUET_ARROW_FOUND TRUE)
   get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH)
+  set(PARQUET_ARROW_LIB_NAME parquet_arrow)
   if (MSVC)
-    set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBRARIES}_static")
-    set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_LIBRARIES}")
+    set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_SHARED_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(PARQUET_ARROW_SHARED_IMP_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.lib")
   else()
-    set(PARQUET_ARROW_LIB_NAME libparquet_arrow)
     set(PARQUET_ARROW_STATIC_LIB
-      ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.a)
+      ${PARQUET_ARROW_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}.a)
     set(PARQUET_ARROW_SHARED_LIB
-      ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+      ${PARQUET_ARROW_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
 else ()
   set(PARQUET_ARROW_FOUND FALSE)
diff --git a/python/setup.py b/python/setup.py
index 9374af36cc6ab..7e612c9e274f8 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -29,7 +29,7 @@
 
 
 import pkg_resources
-from setuptools import setup, Extension
+from setuptools import setup, Extension, Distribution
 
 from os.path import join as pjoin
 
@@ -210,6 +210,11 @@ def _run_cmake(self):
         except OSError:
             pass
 
+        if sys.platform == 'win32':
+            build_prefix = ''
+        else:
+            build_prefix = self.build_type
+
         def move_lib(lib_name):
             lib_filename = (shared_library_prefix + lib_name +
                             shared_library_suffix)
@@ -217,14 +222,14 @@ def move_lib(lib_name):
             if sys.platform == 'darwin':
                 lib_pattern = (shared_library_prefix + lib_name +
                                ".*" + shared_library_suffix[1:])
-                libs = glob.glob(pjoin(self.build_type, lib_pattern))
+                libs = glob.glob(pjoin(build_prefix, lib_pattern))
             else:
-                libs = glob.glob(pjoin(self.build_type, lib_filename) + '*')
+                libs = glob.glob(pjoin(build_prefix, lib_filename) + '*')
             # Longest suffix library should be copied, all others symlinked
             libs.sort(key=lambda s: -len(s))
             print(libs, libs[0])
             lib_filename = os.path.basename(libs[0])
-            shutil.move(pjoin(self.build_type, lib_filename),
+            shutil.move(pjoin(build_prefix, lib_filename),
                         pjoin(build_lib, 'pyarrow', lib_filename))
             for lib in libs[1:]:
                 filename = os.path.basename(lib)
@@ -233,10 +238,10 @@ def move_lib(lib_name):
                     os.symlink(lib_filename, link_name)
 
         if self.bundle_arrow_cpp:
-            print(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow'))
+            print(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
             if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
                 shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
-            shutil.move(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow'))
+            shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
             move_lib("arrow")
             move_lib("arrow_python")
             if self.with_jemalloc:
@@ -337,11 +342,17 @@ def get_outputs(self):
 language-bindings for structure manipulation. It also provides IPC
 and common algorithm implementations."""
 
+class BinaryDistribution(Distribution):
+    def has_ext_modules(foo):
+        return True
+
 setup(
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
     zip_safe=False,
     package_data={'pyarrow': ['*.pxd', '*.pyx']},
+    include_package_data=True,
+    distclass=BinaryDistribution,
     # Dummy extension to trigger build_ext
     ext_modules=[Extension('__dummy__', sources=[])],
 

From 1cb18d528bbb61943273eb8ca8d9dd1a8c631485 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Tue, 23 May 2017 20:29:44 -0400
Subject: [PATCH 0680/1644] ARROW-1061: [C++] Harden decimal parsing against
 invalid strings

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #708 from cpcloud/ARROW-1061 and squashes the following commits:

c93d1db8 [Phillip Cloud] ARROW-1061: [C++] Harden decimal parsing against invalid strings
---
 cpp/src/arrow/util/decimal-test.cc | 80 ++++++++++++++++++++++++++++++
 cpp/src/arrow/util/decimal.cc      | 75 ++++++++++++++++++++++------
 2 files changed, 139 insertions(+), 16 deletions(-)

diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 5d95c2cadc107..72107a21448d1 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -54,6 +54,29 @@ TYPED_TEST(DecimalTest, TestFromString) {
   ASSERT_EQ(scale, 5);
 }
 
+TEST(DecimalTest, TestStringStartingWithPlus) {
+  std::string plus_value("+234.234");
+  Decimal32 out;
+  int scale;
+  int precision;
+  ASSERT_OK(FromString(plus_value, &out, &precision, &scale));
+  ASSERT_EQ(234234, out.value);
+  ASSERT_EQ(6, precision);
+  ASSERT_EQ(3, scale);
+}
+
+TEST(DecimalTest, TestStringStartingWithPlus128) {
+  std::string plus_value("+2342394230592.232349023094");
+  decimal::int128_t expected_value("2342394230592232349023094");
+  Decimal128 out;
+  int scale;
+  int precision;
+  ASSERT_OK(FromString(plus_value, &out, &precision, &scale));
+  ASSERT_EQ(expected_value, out.value);
+  ASSERT_EQ(25, precision);
+  ASSERT_EQ(12, scale);
+}
+
 TEST(DecimalTest, TestStringToInt32) {
   int32_t value = 0;
   StringToInteger("123", "456", 1, &value);
@@ -160,6 +183,63 @@ TEST(DecimalTest, TestDecimal128StringAndBytesRoundTrip) {
   ASSERT_EQ(expected.value, result.value);
 }
 
+TEST(DecimalTest, TestInvalidInputMinus) {
+  std::string invalid_value("-");
+  Decimal32 out;
+  Status status = decimal::FromString(invalid_value, &out);
+  ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputDot) {
+  std::string invalid_value("0.0.0");
+  Decimal32 out;
+  Status status = decimal::FromString(invalid_value, &out);
+  ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputEmbeddedMinus) {
+  std::string invalid_value("0-13-32");
+  Decimal32 out;
+  Status status = decimal::FromString(invalid_value, &out);
+  ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputSingleChar) {
+  std::string invalid_value("a");
+  Decimal32 out;
+  Status status = decimal::FromString(invalid_value, &out);
+  ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
+  std::string invalid_value("-23092.235-");
+  Decimal32 out;
+  Status status = decimal::FromString(invalid_value, &out);
+  auto msg = status.message();
+  ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputWithMinusPlus) {
+  std::string invalid_value("-+23092.235");
+  Decimal32 out;
+  Status status = decimal::FromString(invalid_value, &out);
+  ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputWithPlusMinus) {
+  std::string invalid_value("+-23092.235");
+  Decimal32 out;
+  Status status = decimal::FromString(invalid_value, &out);
+  ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
+  std::string invalid_value("00a");
+  Decimal32 out;
+  Status status = decimal::FromString(invalid_value, &out);
+  ASSERT_RAISES(Invalid, status);
+}
+
 template <typename T>
 class DecimalZerosTest : public ::testing::Test {};
 TYPED_TEST_CASE(DecimalZerosTest, DecimalTypes);
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 734df13e59d8f..72ede35bef9b5 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -29,18 +29,27 @@ ARROW_EXPORT Status FromString(
   }
 
   int8_t sign = 1;
-  auto charp = s.c_str();
-  auto end = charp + s.length();
+  std::string::const_iterator charp = s.cbegin();
+  std::string::const_iterator end = s.cend();
 
-  if (*charp == '+' || *charp == '-') {
-    if (*charp == '-') { sign = -1; }
+  char first_char = *charp;
+  if (first_char == '+' || first_char == '-') {
+    if (first_char == '-') { sign = -1; }
     ++charp;
   }
 
-  auto numeric_string_start = charp;
+  if (charp == end) {
+    std::stringstream ss;
+    ss << "Single character: '" << first_char << "' is not a valid decimal value";
+    return Status::Invalid(ss.str());
+  }
+
+  std::string::const_iterator numeric_string_start = charp;
+
+  DCHECK_LT(charp, end);
 
   // skip leading zeros
-  while (*charp == '0') {
+  while (charp != end && *charp == '0') {
     ++charp;
   }
 
@@ -59,25 +68,59 @@ ARROW_EXPORT Status FromString(
     return Status::OK();
   }
 
-  auto whole_part_start = charp;
-  while (isdigit(*charp)) {
+  std::string::const_iterator whole_part_start = charp;
+
+  while (charp != end && isdigit(*charp)) {
     ++charp;
   }
-  auto whole_part_end = charp;
+
+  std::string::const_iterator whole_part_end = charp;
   std::string whole_part(whole_part_start, whole_part_end);
 
-  if (*charp == '.') {
+  if (charp != end && *charp == '.') {
     ++charp;
+
+    if (charp == end) {
+      return Status::Invalid(
+          "Decimal point must be followed by at least one base ten digit. Reached the "
+          "end of the string.");
+    }
+
+    if (!isdigit(*charp)) {
+      std::stringstream ss;
+      ss << "Decimal point must be followed by a base ten digit. Found '" << *charp
+         << "'";
+      return Status::Invalid(ss.str());
+    }
   } else {
-    // no decimal point
-    DCHECK_EQ(charp, end);
+    if (charp != end) {
+      std::stringstream ss;
+      ss << "Expected base ten digit or decimal point but found '" << *charp
+         << "' instead.";
+      return Status::Invalid(ss.str());
+    }
   }
 
-  auto fractional_part_start = charp;
-  while (isdigit(*charp)) {
-    ++charp;
+  std::string::const_iterator fractional_part_start = charp;
+
+  // The rest must be digits, because if we have a decimal point it must be followed by
+  // digits
+  if (charp != end) {
+    while (charp != end && isdigit(*charp)) {
+      ++charp;
+    }
+
+    // The while loop has ended before the end of the string which means we've hit a
+    // character that isn't a base ten digit
+    if (charp != end) {
+      std::stringstream ss;
+      ss << "Found non base ten digit character '" << *charp
+         << "' before the end of the string";
+      return Status::Invalid(ss.str());
+    }
   }
-  auto fractional_part_end = charp;
+
+  std::string::const_iterator fractional_part_end = charp;
   std::string fractional_part(fractional_part_start, fractional_part_end);
 
   if (precision != nullptr) {

From 078357af32514502200a2fb83b0376c6c36f623c Mon Sep 17 00:00:00 2001
From: Jeff Reback <jeff.reback@twosigma.com>
Date: Wed, 24 May 2017 17:18:45 -0400
Subject: [PATCH 0681/1644] ARROW-1066: [Python] pandas 0.20.1 deprecation of
 pd.lib causes a warning on import

Author: Jeff Reback <jeff.reback@twosigma.com>

Closes #715 from jreback/pandas_compat and squashes the following commits:

842b19f [Jeff Reback] COMPAT: pandas 0.20.1 deprecation of pd.lib causes a warning on import
---
 python/pyarrow/feather.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 34783a71e3dec..9f40ab419ad5a 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -26,6 +26,11 @@
 from pyarrow.lib import Table
 import pyarrow.lib as ext
 
+try:
+    infer_dtype = pdapi.infer_dtype
+except AttributeError:
+    infer_dtype = pd.lib.infer_dtype
+
 
 if LooseVersion(pd.__version__) < '0.17.0':
     raise ImportError("feather requires pandas >= 0.17.0")
@@ -75,7 +80,7 @@ def write(self, df):
             col = df.iloc[:, i]
 
             if pdapi.is_object_dtype(col):
-                inferred_type = pd.lib.infer_dtype(col)
+                inferred_type = infer_dtype(col)
                 msg = ("cannot serialize column {n} "
                        "named {name} with dtype {dtype}".format(
                            n=i, name=name, dtype=inferred_type))
@@ -83,7 +88,7 @@ def write(self, df):
                 if inferred_type in ['mixed']:
 
                     # allow columns with nulls + an inferable type
-                    inferred_type = pd.lib.infer_dtype(col[col.notnull()])
+                    inferred_type = infer_dtype(col[col.notnull()])
                     if inferred_type in ['mixed']:
                         raise ValueError(msg)
 

From 4e4435ecc4e4a02a9029e65c312bdfb00af7e291 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 24 May 2017 17:20:40 -0400
Subject: [PATCH 0682/1644] ARROW-424: [C++] Make ReadAt, Write HDFS functions
 threadsafe

This also fixes the HDFS test suite to actually use libhdfs3 (it was not by accident)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #712 from wesm/ARROW-424 and squashes the following commits:

0894719 [Wes McKinney] Make ReadAt, Write HDFS functions threadsafe
---
 cpp/src/arrow/array.h            |  1 -
 cpp/src/arrow/io/hdfs.cc         |  5 +++
 cpp/src/arrow/io/io-hdfs-test.cc | 68 +++++++++++++++++++++++++++-----
 3 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 331c6c3e7fab8..2c96ce051474e 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -508,7 +508,6 @@ ARROW_EXTERN_TEMPLATE NumericArray<Time32Type>;
 ARROW_EXTERN_TEMPLATE NumericArray<Time64Type>;
 ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
 
-
 /// \brief Perform any validation checks to determine obvious inconsistencies
 /// with the array's internal data
 ///
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index a27e132d155b1..ba9c2c207a013 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -89,6 +89,9 @@ class HdfsAnyFileImpl {
 
   LibHdfsShim* driver_;
 
+  // For threadsafety
+  std::mutex lock_;
+
   // These are pointers in libhdfs, so OK to copy
   hdfsFS fs_;
   hdfsFile file_;
@@ -116,6 +119,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
       ret = driver_->Pread(fs_, file_, static_cast<tOffset>(position),
           reinterpret_cast<void*>(buffer), static_cast<tSize>(nbytes));
     } else {
+      std::lock_guard<std::mutex> guard(lock_);
       RETURN_NOT_OK(Seek(position));
       return Read(nbytes, bytes_read, buffer);
     }
@@ -253,6 +257,7 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
   }
 
   Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
+    std::lock_guard<std::mutex> guard(lock_);
     tSize ret = driver_->Write(
         fs_, file_, reinterpret_cast<const void*>(buffer), static_cast<tSize>(nbytes));
     CHECK_FAILURE(ret, "Write");
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 0fdb897d94410..74f80428c4561 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -19,6 +19,7 @@
 #include <iostream>
 #include <sstream>
 #include <string>
+#include <thread>
 
 #include "gtest/gtest.h"
 
@@ -38,6 +39,14 @@ std::vector<uint8_t> RandomData(int64_t size) {
   return buffer;
 }
 
+struct JNIDriver {
+  static HdfsDriver type;
+};
+
+struct PivotalDriver {
+  static HdfsDriver type;
+};
+
 template <typename DRIVER>
 class TestHdfsClient : public ::testing::Test {
  public:
@@ -112,6 +121,7 @@ class TestHdfsClient : public ::testing::Test {
     conf_.host = host == nullptr ? "localhost" : host;
     conf_.user = user;
     conf_.port = port == nullptr ? 20500 : atoi(port);
+    conf_.driver = DRIVER::type;
 
     ASSERT_OK(HdfsClient::Connect(&conf_, &client_));
   }
@@ -133,20 +143,19 @@ class TestHdfsClient : public ::testing::Test {
   std::shared_ptr<HdfsClient> client_;
 };
 
+template <>
+std::string TestHdfsClient<PivotalDriver>::HdfsAbsPath(const std::string& relpath) {
+  std::stringstream ss;
+  ss << relpath;
+  return ss.str();
+}
+
 #define SKIP_IF_NO_DRIVER()                                  \
   if (!this->loaded_driver_) {                               \
     std::cout << "Driver not loaded, skipping" << std::endl; \
     return;                                                  \
   }
 
-struct JNIDriver {
-  static HdfsDriver type;
-};
-
-struct PivotalDriver {
-  static HdfsDriver type;
-};
-
 HdfsDriver JNIDriver::type = HdfsDriver::LIBHDFS;
 HdfsDriver PivotalDriver::type = HdfsDriver::LIBHDFS3;
 
@@ -364,7 +373,6 @@ TYPED_TEST(TestHdfsClient, LargeFile) {
 
 TYPED_TEST(TestHdfsClient, RenameFile) {
   SKIP_IF_NO_DRIVER();
-
   ASSERT_OK(this->MakeScratchDir());
 
   auto src_path = this->ScratchPath("src-file");
@@ -380,5 +388,47 @@ TYPED_TEST(TestHdfsClient, RenameFile) {
   ASSERT_TRUE(this->client_->Exists(dst_path));
 }
 
+TYPED_TEST(TestHdfsClient, ThreadSafety) {
+  SKIP_IF_NO_DRIVER();
+  ASSERT_OK(this->MakeScratchDir());
+
+  auto src_path = this->ScratchPath("threadsafety");
+
+  std::string data = "foobar";
+  ASSERT_OK(this->WriteDummyFile(src_path, reinterpret_cast<const uint8_t*>(data.c_str()),
+      static_cast<int64_t>(data.size())));
+
+  std::shared_ptr<HdfsReadableFile> file;
+  ASSERT_OK(this->client_->OpenReadable(src_path, &file));
+
+  std::atomic<int> correct_count(0);
+  const int niter = 1000;
+
+  auto ReadData = [&file, &correct_count, &data, niter]() {
+    for (int i = 0; i < niter; ++i) {
+      std::shared_ptr<Buffer> buffer;
+      if (i % 2 == 0) {
+        ASSERT_OK(file->ReadAt(3, 3, &buffer));
+        if (0 == memcmp(data.c_str() + 3, buffer->data(), 3)) { correct_count += 1; }
+      } else {
+        ASSERT_OK(file->ReadAt(0, 4, &buffer));
+        if (0 == memcmp(data.c_str() + 0, buffer->data(), 4)) { correct_count += 1; }
+      }
+    }
+  };
+
+  std::thread thread1(ReadData);
+  std::thread thread2(ReadData);
+  std::thread thread3(ReadData);
+  std::thread thread4(ReadData);
+
+  thread1.join();
+  thread2.join();
+  thread3.join();
+  thread4.join();
+
+  ASSERT_EQ(niter * 4, correct_count);
+}
+
 }  // namespace io
 }  // namespace arrow

From 8a700ccdad745c250fe5d91a9104e7c2d6364c1b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 24 May 2017 17:25:42 -0400
Subject: [PATCH 0683/1644] ARROW-1063: [Website] Updates for 0.4.0 release,
 release posting

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #711 from wesm/ARROW-1063 and squashes the following commits:

90f33ac [Wes McKinney] Update to use dynamic mirror links
62a04e8 [Wes McKinney] Use Apache mirrors for download links
ae88978 [Wes McKinney] Finish blog draft, website updates
c7adc0f [Wes McKinney] Start 0.4.0 items
---
 site/_config.yml                        |   4 +
 site/_includes/top.html                 |   1 +
 site/_posts/2017-05-23-0.4.0-release.md | 111 +++++++++++++
 site/_release/0.2.0.md                  |   8 +-
 site/_release/0.3.0.md                  |  10 +-
 site/_release/0.4.0.md                  | 138 ++++++++++++++++
 site/_release/index.md                  |   4 +-
 site/css/syntax.css                     | 209 ++++++++++++++++++++++++
 site/index.html                         |  10 +-
 site/install.md                         |  27 ++-
 10 files changed, 495 insertions(+), 27 deletions(-)
 create mode 100644 site/_posts/2017-05-23-0.4.0-release.md
 create mode 100644 site/_release/0.4.0.md
 create mode 100644 site/css/syntax.css

diff --git a/site/_config.yml b/site/_config.yml
index 8bb969abe848c..fcb76a374fb51 100644
--- a/site/_config.yml
+++ b/site/_config.yml
@@ -19,6 +19,10 @@ repository: https://github.com/apache/arrow
 destination: build
 excerpt_separator: ""
 
+kramdown:
+  input: GFM
+  syntax_highlighter: rouge
+
 exclude:
   - Gemfile
   - Gemfile.lock
diff --git a/site/_includes/top.html b/site/_includes/top.html
index cc537bac07ba3..cfc4cde137ae7 100644
--- a/site/_includes/top.html
+++ b/site/_includes/top.html
@@ -13,6 +13,7 @@
     <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
 
     <link href="{{ site.baseurl }}/css/main.css" rel="stylesheet">
+    <link href="{{ site.baseurl }}/css/syntax.css" rel="stylesheet">
     <script src="https://code.jquery.com/jquery-3.2.1.min.js"
             integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
             crossorigin="anonymous"></script>
diff --git a/site/_posts/2017-05-23-0.4.0-release.md b/site/_posts/2017-05-23-0.4.0-release.md
new file mode 100644
index 0000000000000..d94a68a586b04
--- /dev/null
+++ b/site/_posts/2017-05-23-0.4.0-release.md
@@ -0,0 +1,111 @@
+---
+layout: post
+title: "Apache Arrow 0.4.0 Release"
+date: "2017-05-23 00:00:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.4.0 release of the
+project. While only 17 days since the release, it includes [**77 resolved
+JIRAs**][1] with some important new features and bug fixes.
+
+See the [Install Page][6] to learn how to get the libraries for your platform.
+
+### Expanded JavaScript Implementation
+
+The TypeScript Arrow implementation has undergone some work since 0.3.0 and can
+now read a substantial portion of the Arrow streaming binary format. As this
+implementation develops, we will eventually want to include JS in the
+integration test suite along with Java and C++ to ensure wire
+cross-compatibility.
+
+### Python Support for Apache Parquet on Windows
+
+With the [1.1.0 C++ release][7] of [Apache Parquet][8], we have enabled the
+`pyarrow.parquet` extension on Windows for Python 3.5 and 3.6. This should
+appear in conda-forge packages and PyPI in the near future. Developers can
+follow the [source build instructions][9].
+
+### Generalizing Arrow Streams
+
+In the 0.2.0 release, we defined the first version of the Arrow streaming
+binary format for low-cost messaging with columnar data. These streams presume
+that the message components are written as a continuous byte stream over a
+socket or file.
+
+We would like to be able to support other other transport protocols, like
+[gRPC][3], for the message components of Arrow streams. To that end, in C++ we
+defined an abstract stream reader interface, for which the current contiguous
+streaming format is one implementation:
+
+{% highlight cpp %}
+class RecordBatchReader {
+ public:
+  virtual std::shared_ptr<Schema> schema() const = 0;
+  virtual Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) = 0;
+};
+{% endhighlight %}
+
+It would also be good to define abstract stream reader and writer interfaces in
+the Java implementation.
+
+In an upcoming blog post, we will explain in more depth how Arrow streams work,
+but you can learn more about them by reading the [IPC specification][4].
+
+### C++ and Cython API for Python Extensions
+
+As other Python libraries with C or C++ extensions use Apache Arrow, they will
+need to be able to return Python objects wrapping the underlying C++
+objects. In this release, we have implemented a prototype C++ API which enables
+Python wrapper objects to be constructed from C++ extension code:
+
+{% highlight cpp %}
+#include "arrow/python/pyarrow.h"
+
+if (!arrow::py::import_pyarrow()) {
+  // Error
+}
+
+std::shared_ptr<arrow::RecordBatch> cpp_batch = GetData(...);
+PyObject* py_batch = arrow::py::wrap_batch(cpp_batch);
+{% endhighlight %}
+
+This API is intended to be usable from Cython code as well:
+
+{% highlight cython %}
+cimport pyarrow
+pyarrow.import_pyarrow()
+{% endhighlight %}
+
+### Python Wheel Installers on macOS
+
+With this release, `pip install pyarrow` works on macOS (OS X) as well as
+Linux. We are working on providing binary wheel installers for Windows as well.
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.4.0
+[2]: http://arrow.apache.org/install
+[3]: http://grpc.io/
+[4]: http://arrow.apache.org/docs/ipc.html
+[6]: http://arrow.apache.org/install
+[7]: https://github.com/apache/parquet-cpp/releases/tag/apache-parquet-cpp-1.1.0
+[8]: http://parquet.apache.org
+[9]: http://arrow.apache.org/docs/python/development.html
\ No newline at end of file
diff --git a/site/_release/0.2.0.md b/site/_release/0.2.0.md
index ddac64e33f5e4..72bce7d5ddb31 100644
--- a/site/_release/0.2.0.md
+++ b/site/_release/0.2.0.md
@@ -26,8 +26,8 @@ limitations under the License.
 
 ## Download
 
-* **Source Release**: [apache-arrow-0.2.0.tar.gz][6]
-* **Verification**: [md5][3], [asc][7]
+* [**Source Artifacts**][6]
+* [Git tag][2]
 
 # Changelog
 
@@ -252,6 +252,4 @@ $ git shortlog -sn apache-arrow-0.1.0..apache-arrow-0.2.0
 * [ARROW-561](https://issues.apache.org/jira/browse/ARROW-561) - Update java & python dependencies to improve downstream packaging experience
 
 [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.2.0
-[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz.md5
-[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz
-[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.2.0/
diff --git a/site/_release/0.3.0.md b/site/_release/0.3.0.md
index 443bcadce78a6..a88ce4dd30beb 100644
--- a/site/_release/0.3.0.md
+++ b/site/_release/0.3.0.md
@@ -24,10 +24,11 @@ limitations under the License.
 
 # Apache Arrow 0.3.0 (5 May 2017)
 
+Read more in the [release blog post][8]
+
 ## Download
 
-* **Source Release**: [apache-arrow-0.3.0.tar.gz][6]
-* **Verification**: [md5][3], [asc][7]
+* [**Source Artifacts**][6]
 * [Git tag d8db8f8][2]
 
 # Changelog
@@ -360,6 +361,5 @@ $ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
 * [ARROW-938](https://issues.apache.org/jira/browse/ARROW-938) - Fix Apache Rat errors from source release build
 
 [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.3.0
-[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.md5
-[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
-[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.3.0/
+[8]: http://arrow.apache.org/blog/2017/05/08/0.3-release/
\ No newline at end of file
diff --git a/site/_release/0.4.0.md b/site/_release/0.4.0.md
new file mode 100644
index 0000000000000..d0060a3dffa2f
--- /dev/null
+++ b/site/_release/0.4.0.md
@@ -0,0 +1,138 @@
+---
+layout: default
+title: 0.4.0 Release
+permalink: /release/0.4.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.4.0 (22 May 2017)
+
+Read more in the [release blog post][8]
+
+## Download
+
+* [**Source Artifacts**][6]
+* [Git tag a8f8ba0][2]
+
+# Changelog
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.3.0..apache-arrow-0.4.0
+    28  Wes McKinney
+    18  Kouhei Sutou
+     9  Uwe L. Korn
+     3  Brian Hulette
+     3  Emilio Lahr-Vivaz
+     3  Philipp Moritz
+     3  Phillip Cloud
+     2  Julien Le Dem
+     1  Bryan Cutler
+     1  Jeff Reback
+     1  Max Risuhin
+     1  Wenchen Fan
+     1  bgosztonyi
+```
+## New Features and Improvements
+
+* [ARROW-1000](https://issues.apache.org/jira/browse/ARROW-1000) - [GLib] Move install document to Website
+* [ARROW-1001](https://issues.apache.org/jira/browse/ARROW-1001) - [GLib] Unify writer files
+* [ARROW-1002](https://issues.apache.org/jira/browse/ARROW-1002) - [C++] It is not necessary to add padding after the magic header in the FileWriter implementation
+* [ARROW-1008](https://issues.apache.org/jira/browse/ARROW-1008) - [C++] Define abstract interface for stream iteration
+* [ARROW-1010](https://issues.apache.org/jira/browse/ARROW-1010) - [Website] Only show English posts in /blog/
+* [ARROW-1011](https://issues.apache.org/jira/browse/ARROW-1011) - [Format] Clarify requirements around buffer padding in validity bitmaps
+* [ARROW-1015](https://issues.apache.org/jira/browse/ARROW-1015) - [Java] Implement schema-level metadata
+* [ARROW-1016](https://issues.apache.org/jira/browse/ARROW-1016) - Python: Include C++ headers (optionally) in wheels
+* [ARROW-1022](https://issues.apache.org/jira/browse/ARROW-1022) - [Python] Add nthreads option to Feather read method
+* [ARROW-1024](https://issues.apache.org/jira/browse/ARROW-1024) - Python: Update build time numpy version to 1.10.1
+* [ARROW-1025](https://issues.apache.org/jira/browse/ARROW-1025) - [Website] Improve changelog on website
+* [ARROW-1027](https://issues.apache.org/jira/browse/ARROW-1027) - [Python] Allow negative indexing in fields/columns on pyarrow Table and Schema objects
+* [ARROW-1028](https://issues.apache.org/jira/browse/ARROW-1028) - [Python] Documentation updates after ARROW-1008
+* [ARROW-1029](https://issues.apache.org/jira/browse/ARROW-1029) - [Python] Fix --with-parquet build on Windows, add unit tests to Appveyor
+* [ARROW-1030](https://issues.apache.org/jira/browse/ARROW-1030) - Python: Account for library versioning in parquet-cpp
+* [ARROW-1031](https://issues.apache.org/jira/browse/ARROW-1031) - [GLib] Support pretty print
+* [ARROW-1037](https://issues.apache.org/jira/browse/ARROW-1037) - [GLib] Follow reader name change
+* [ARROW-1038](https://issues.apache.org/jira/browse/ARROW-1038) - [GLib] Follow writer name change
+* [ARROW-1040](https://issues.apache.org/jira/browse/ARROW-1040) - [GLib] Follow tensor IO
+* [ARROW-1044](https://issues.apache.org/jira/browse/ARROW-1044) - [GLib] Support Feather
+* [ARROW-182](https://issues.apache.org/jira/browse/ARROW-182) - [C++] Remove Array::Validate virtual function and make a separate method
+* [ARROW-29](https://issues.apache.org/jira/browse/ARROW-29) - C++: Add re2 as optional 3rd-party toolchain dependency
+* [ARROW-446](https://issues.apache.org/jira/browse/ARROW-446) - [Python] Document NativeFile interfaces, HDFS client in Sphinx
+* [ARROW-482](https://issues.apache.org/jira/browse/ARROW-482) - [Java] Provide API access to "custom_metadata" Field attribute in IPC setting
+* [ARROW-532](https://issues.apache.org/jira/browse/ARROW-532) - [Python] Expand pyarrow.parquet documentation for 0.3 release
+* [ARROW-629](https://issues.apache.org/jira/browse/ARROW-629) - [JS] Add unit test suite
+* [ARROW-714](https://issues.apache.org/jira/browse/ARROW-714) - [C++] Add import_pyarrow C API in the style of NumPy for thirdparty C++ users
+* [ARROW-819](https://issues.apache.org/jira/browse/ARROW-819) - [Python] Define public Cython API
+* [ARROW-872](https://issues.apache.org/jira/browse/ARROW-872) - [JS] Read streaming format
+* [ARROW-873](https://issues.apache.org/jira/browse/ARROW-873) - [JS] Implement fixed width list type
+* [ARROW-874](https://issues.apache.org/jira/browse/ARROW-874) - [JS] Read dictionary-encoded vectors
+* [ARROW-899](https://issues.apache.org/jira/browse/ARROW-899) - [Docs] Add CHANGELOG for 0.3.0
+* [ARROW-901](https://issues.apache.org/jira/browse/ARROW-901) - [Python] Write FixedSizeBinary to Parquet
+* [ARROW-923](https://issues.apache.org/jira/browse/ARROW-923) - [Docs] Generate Changelog for website with JIRA links
+* [ARROW-929](https://issues.apache.org/jira/browse/ARROW-929) - Move KEYS file to SVN, remove from git
+* [ARROW-943](https://issues.apache.org/jira/browse/ARROW-943) - [GLib] Support running unit tests with source archive
+* [ARROW-945](https://issues.apache.org/jira/browse/ARROW-945) - [GLib] Add a Lua example to show Torch integration
+* [ARROW-946](https://issues.apache.org/jira/browse/ARROW-946) - [GLib] Use "new" instead of "open" for constructor name
+* [ARROW-947](https://issues.apache.org/jira/browse/ARROW-947) - [Python] Improve execution time of manylinux1 build
+* [ARROW-953](https://issues.apache.org/jira/browse/ARROW-953) - Use cmake / curl from conda-forge in CI builds
+* [ARROW-954](https://issues.apache.org/jira/browse/ARROW-954) - Make it possible to compile Arrow with header-only boost
+* [ARROW-956](https://issues.apache.org/jira/browse/ARROW-956) - remove pandas pre-0.20.0 compat
+* [ARROW-957](https://issues.apache.org/jira/browse/ARROW-957) - [Doc] Add HDFS and Windows documents to doxygen output
+* [ARROW-961](https://issues.apache.org/jira/browse/ARROW-961) - [Python] Rename InMemoryOutputStream to BufferOutputStream
+* [ARROW-963](https://issues.apache.org/jira/browse/ARROW-963) - [GLib] Add equal
+* [ARROW-967](https://issues.apache.org/jira/browse/ARROW-967) - [GLib] Support initializing array with buffer
+* [ARROW-970](https://issues.apache.org/jira/browse/ARROW-970) - [Python] Accidentally calling pyarrow.Table() should not segfault process
+* [ARROW-977](https://issues.apache.org/jira/browse/ARROW-977) - [java] Add Timezone aware timestamp vectors
+* [ARROW-984](https://issues.apache.org/jira/browse/ARROW-984) - [GLib] Add Go examples
+* [ARROW-985](https://issues.apache.org/jira/browse/ARROW-985) - [GLib] Update package information
+* [ARROW-988](https://issues.apache.org/jira/browse/ARROW-988) - [JS] Add entry to Travis CI matrix
+* [ARROW-993](https://issues.apache.org/jira/browse/ARROW-993) - [GLib] Add missing error checks in Go examples
+* [ARROW-996](https://issues.apache.org/jira/browse/ARROW-996) - [Website] Add 0.3 release announce in Japanese
+* [ARROW-997](https://issues.apache.org/jira/browse/ARROW-997) - [Java] Implement transfer in FixedSizeListVector
+
+## Bug Fixes
+
+* [ARROW-1003](https://issues.apache.org/jira/browse/ARROW-1003) - [C++] Hdfs and java dlls fail to load when built for Windows with MSVC
+* [ARROW-1004](https://issues.apache.org/jira/browse/ARROW-1004) - ArrowInvalid: Invalid: Python object of type float is not None and is not a string, bool, or date object
+* [ARROW-1017](https://issues.apache.org/jira/browse/ARROW-1017) - Python: Table.to_pandas leaks memory
+* [ARROW-1023](https://issues.apache.org/jira/browse/ARROW-1023) - Python: Fix bundling of arrow-cpp for macOS
+* [ARROW-1033](https://issues.apache.org/jira/browse/ARROW-1033) - [Python] pytest discovers scripts/test_leak.py
+* [ARROW-1046](https://issues.apache.org/jira/browse/ARROW-1046) - [Python] Conform DataFrame metadata to pandas spec
+* [ARROW-1053](https://issues.apache.org/jira/browse/ARROW-1053) - [Python] Memory leak with RecordBatchFileReader
+* [ARROW-1054](https://issues.apache.org/jira/browse/ARROW-1054) - [Python] Test suite fails on pandas 0.19.2
+* [ARROW-813](https://issues.apache.org/jira/browse/ARROW-813) - [Python] setup.py sdist must also bundle dependent cmake modules
+* [ARROW-824](https://issues.apache.org/jira/browse/ARROW-824) - Date and Time Vectors should reflect timezone-less semantics
+* [ARROW-856](https://issues.apache.org/jira/browse/ARROW-856) - CmakeError by Unknown compiler.
+* [ARROW-881](https://issues.apache.org/jira/browse/ARROW-881) - [Python] Reconstruct Pandas DataFrame indexes using custom_metadata
+* [ARROW-909](https://issues.apache.org/jira/browse/ARROW-909) - libjemalloc.so.2: cannot open shared object file:
+* [ARROW-939](https://issues.apache.org/jira/browse/ARROW-939) - Fix division by zero for zero-dimensional Tensors
+* [ARROW-940](https://issues.apache.org/jira/browse/ARROW-940) - [JS] Generate multiple sets of artifacts
+* [ARROW-944](https://issues.apache.org/jira/browse/ARROW-944) - Python: Compat broken for pandas==0.18.1
+* [ARROW-948](https://issues.apache.org/jira/browse/ARROW-948) - [GLib] Update C++ header file list
+* [ARROW-952](https://issues.apache.org/jira/browse/ARROW-952) - Compilation error on macOS with clang-802.0.42
+* [ARROW-958](https://issues.apache.org/jira/browse/ARROW-958) - [Python] Conda build guide still needs ARROW_HOME, PARQUET_HOME
+* [ARROW-991](https://issues.apache.org/jira/browse/ARROW-991) - [Python] PyArray_SimpleNew should not be used with NPY_DATETIME
+* [ARROW-995](https://issues.apache.org/jira/browse/ARROW-995) - [Website] 0.3 release announce has a typo in reference
+* [ARROW-998](https://issues.apache.org/jira/browse/ARROW-998) - [Doc] File format documents incorrect schema location
+
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.4.0
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.0/
+[8]: http://arrow.apache.org/blog/2017/05/23/0.4.0-release/
\ No newline at end of file
diff --git a/site/_release/index.md b/site/_release/index.md
index d42227529fde6..a7f9ab6377183 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -1,6 +1,6 @@
 ---
 layout: default
-title: 0.3.0 Release
+title: Releases
 permalink: /release/index.html
 ---
 <!--
@@ -26,6 +26,7 @@ limitations under the License.
 
 Navigate to the release page for downloads and the changelog.
 
+* [0.4.0 (22 May 2017)][4]
 * [0.3.0 (5 May 2017)][1]
 * [0.2.0 (18 February 2017)][2]
 * [0.1.0 (10 October 2016)][3]
@@ -33,3 +34,4 @@ Navigate to the release page for downloads and the changelog.
 [1]: {{ site.baseurl }}/release/0.3.0.html
 [2]: {{ site.baseurl }}/release/0.2.0.html
 [3]: {{ site.baseurl }}/release/0.1.0.html
+[4]: {{ site.baseurl }}/release/0.4.0.html
diff --git a/site/css/syntax.css b/site/css/syntax.css
new file mode 100644
index 0000000000000..daf76adee9895
--- /dev/null
+++ b/site/css/syntax.css
@@ -0,0 +1,209 @@
+.highlight table td { padding: 5px; }
+.highlight table pre { margin: 0; }
+.highlight .cm {
+  color: #999988;
+  font-style: italic;
+}
+.highlight .cp {
+  color: #999999;
+  font-weight: bold;
+}
+.highlight .c1 {
+  color: #999988;
+  font-style: italic;
+}
+.highlight .cs {
+  color: #999999;
+  font-weight: bold;
+  font-style: italic;
+}
+.highlight .c, .highlight .cd {
+  color: #999988;
+  font-style: italic;
+}
+.highlight .err {
+  color: #a61717;
+  background-color: #e3d2d2;
+}
+.highlight .gd {
+  color: #000000;
+  background-color: #ffdddd;
+}
+.highlight .ge {
+  color: #000000;
+  font-style: italic;
+}
+.highlight .gr {
+  color: #aa0000;
+}
+.highlight .gh {
+  color: #999999;
+}
+.highlight .gi {
+  color: #000000;
+  background-color: #ddffdd;
+}
+.highlight .go {
+  color: #888888;
+}
+.highlight .gp {
+  color: #555555;
+}
+.highlight .gs {
+  font-weight: bold;
+}
+.highlight .gu {
+  color: #aaaaaa;
+}
+.highlight .gt {
+  color: #aa0000;
+}
+.highlight .kc {
+  color: #000000;
+  font-weight: bold;
+}
+.highlight .kd {
+  color: #000000;
+  font-weight: bold;
+}
+.highlight .kn {
+  color: #000000;
+  font-weight: bold;
+}
+.highlight .kp {
+  color: #000000;
+  font-weight: bold;
+}
+.highlight .kr {
+  color: #000000;
+  font-weight: bold;
+}
+.highlight .kt {
+  color: #445588;
+  font-weight: bold;
+}
+.highlight .k, .highlight .kv {
+  color: #000000;
+  font-weight: bold;
+}
+.highlight .mf {
+  color: #009999;
+}
+.highlight .mh {
+  color: #009999;
+}
+.highlight .il {
+  color: #009999;
+}
+.highlight .mi {
+  color: #009999;
+}
+.highlight .mo {
+  color: #009999;
+}
+.highlight .m, .highlight .mb, .highlight .mx {
+  color: #009999;
+}
+.highlight .sb {
+  color: #d14;
+}
+.highlight .sc {
+  color: #d14;
+}
+.highlight .sd {
+  color: #d14;
+}
+.highlight .s2 {
+  color: #d14;
+}
+.highlight .se {
+  color: #d14;
+}
+.highlight .sh {
+  color: #d14;
+}
+.highlight .si {
+  color: #d14;
+}
+.highlight .sx {
+  color: #d14;
+}
+.highlight .sr {
+  color: #009926;
+}
+.highlight .s1 {
+  color: #d14;
+}
+.highlight .ss {
+  color: #990073;
+}
+.highlight .s {
+  color: #d14;
+}
+.highlight .na {
+  color: #008080;
+}
+.highlight .bp {
+  color: #999999;
+}
+.highlight .nb {
+  color: #0086B3;
+}
+.highlight .nc {
+  color: #445588;
+  font-weight: bold;
+}
+.highlight .no {
+  color: #008080;
+}
+.highlight .nd {
+  color: #3c5d5d;
+  font-weight: bold;
+}
+.highlight .ni {
+  color: #800080;
+}
+.highlight .ne {
+  color: #990000;
+  font-weight: bold;
+}
+.highlight .nf {
+  color: #990000;
+  font-weight: bold;
+}
+.highlight .nl {
+  color: #990000;
+  font-weight: bold;
+}
+.highlight .nn {
+  color: #555555;
+}
+.highlight .nt {
+  color: #000080;
+}
+.highlight .vc {
+  color: #008080;
+}
+.highlight .vg {
+  color: #008080;
+}
+.highlight .vi {
+  color: #008080;
+}
+.highlight .nv {
+  color: #008080;
+}
+.highlight .ow {
+  color: #000000;
+  font-weight: bold;
+}
+.highlight .o {
+  color: #000000;
+  font-weight: bold;
+}
+.highlight .w {
+  color: #bbbbbb;
+}
+.highlight {
+  background-color: #f8f8f8;
+}
diff --git a/site/index.html b/site/index.html
index d80925ce24bcd..620b75151c07f 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,10 +7,10 @@ <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
         <p>
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
-          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.3.0 Release - May 5, 2017)</a>
+          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.4.0 Release - May 22, 2017)</a>
         </p>
       </div>
-      <h4>Latest News: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.3.0 release</a></h4>
+      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.4.0 release</a></h4>
       <div class="row">
         <div class="col-lg-4">
           <h2>Fast</h2>
@@ -26,6 +26,12 @@ <h2>Standard</h2>
         </div>
      </div> <!-- close "row" div -->
 
+<h2>Zero-Copy IPC and Streaming Messaging</h2>
+<div align="left">
+  <h4>Apache Arrow supports zero-copy shared memory IPC and a streaming wire
+  format that fully avoids traditional data serialization costs</h4>
+</div>
+
 <h2>Performance Advantage of Columnar In-Memory</h2>
 <div align="center">
   <img src="img/simd.png" alt="SIMD" style="width:60%" />
diff --git a/site/install.md b/site/install.md
index a575a1e0587f6..9018c6c045408 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,20 +20,17 @@ limitations under the License.
 {% endcomment %}
 -->
 
-## Current Version: 0.3.0
+## Current Version: 0.4.0
 
-### Released: 5 May 2017
+### Released: 22 May 2017
 
-Apache Arrow 0.3.0 is the third major release of the project and has seen
-significant iteration and hardening of logical types and the binary formats. It
-is safe for production use, though there may be API changes and binary format
-breaks in the future.
+See the [release notes][10] and [blog post][11] for more about what's new.
 
 ### Source release
 
-* **Source Release**: [apache-arrow-0.3.0.tar.gz][6]
+* **Source Release**: [apache-arrow-0.4.0.tar.gz][6]
 * **Verification**: [md5][3], [asc][7]
-* [Git tag d8db8f8][2]
+* [Git tag a8f8ba0][2]
 
 ### Java Packages
 
@@ -132,12 +129,14 @@ These repositories are managed at
 [red-data-tools/arrow-packages][9]. If you have any feedback, please
 send it to the project instead of Apache Arrow project.
 
-[1]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.3.0
-[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.md5
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.3.0%22
+[1]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.4.0
+[3]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz.md5
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.4.0%22
 [5]: http://conda-forge.github.io
-[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
-[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.asc
+[6]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz
+[7]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz.asc
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
+[10]: http://arrow.apache.org/release/0.4.0.html
+[11]: http://arrow.apache.org/blog/2017/05/23/0.4.0-release/
\ No newline at end of file

From 03e8b54d8242daf0358e39cd2ccb1f0599223c9e Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Fri, 26 May 2017 12:47:14 -0400
Subject: [PATCH 0684/1644] ARROW-1069: Add instructions for publishing maven
 artifacts

Author: Julien Le Dem <julien@apache.org>

Closes #716 from julienledem/ARROW-1069 and squashes the following commits:

8896189 [Julien Le Dem] ARROW-1069: Add instructions for publishing maven artifacts
---
 dev/release/README | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/dev/release/README b/dev/release/README
index cf68028005e14..c53d1d8598135 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -15,10 +15,11 @@ source dev/release/setup-gpg-agent.sh
 sh dev/release/00-prepare.sh 0.1.0 0.1.1
 # push the tag
 git push apache apache-arrow-0.1.0
-# tag and push to maven repo (repo will have to be finalized separately)
+# tag and stage artifacts to maven repo (repo will have to be finalized separately)
 sh dev/release/01-perform.sh
 # create the source release
 sh dev/release/02-source.sh 0.1.0 0
+# once the vote has passed, publish the staged maven artifacts (see bellow)
 
 useful commands:
 - to set the mvn version in the poms
@@ -30,3 +31,21 @@ eval $(gpg-agent --daemon --allow-preset-passphrase)
 gpg --use-agent -s LICENSE.txt 
 - delete tag localy
 git tag -d apache-arrow-0.1.0
+
+How to stage maven artifacts:
+artifacts get staged during the perform phase of the scripts above.
+If you need to stage the artifacts again follow the instructions bellow:
+# checkout the relese tag
+git checkout apache-arrow-0.1.0
+# setup the gpg agent for signing artifacts
+source dev/release/setup-gpg-agent.sh
+# go in the java subfolder
+cd java
+# stage the artifacts
+mvn -Papache-release deploy
+
+How to publish the staged artifacts:
+Logon to the apache repository: https://repository.apache.org/#stagingRepositories
+Select the arrow staging repository you just just created: orgapachearrow-100x
+Click the "close" button
+Once validation has passed, click the "release" button

From 51b6bf2532646ce1f916613e22272782c918b2ee Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 26 May 2017 13:12:44 -0400
Subject: [PATCH 0685/1644] ARROW-897: [GLib] Extract CI configuration for GLib

Author: Kouhei Sutou <kou@clear-code.com>

Closes #713 from kou/glib-travis-split and squashes the following commits:

682f0bd [Kouhei Sutou] [GLib] Build only C++ libraries for GLib CI
7d02eb7 [Kouhei Sutou] [GLib] Extract CI configuration for GLib
---
 .travis.yml                    | 13 +++++++++--
 ci/travis_before_script_cpp.sh | 42 +++++++++++++++++++++++++++-------
 ci/travis_env_common.sh        |  5 ----
 3 files changed, 45 insertions(+), 15 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b212fcbcb9970..36e3c18d8a98a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,11 +40,9 @@ matrix:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
   - compiler: clang
     osx_image: xcode6.4
     os: osx
@@ -82,6 +80,17 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
+  - compiler: gcc
+    language: cpp
+    os: linux
+    group: deprecated
+    before_script:
+    - export CC="gcc-4.9"
+    - export CXX="g++-4.9"
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
 
 before_install:
 - ulimit -c unlimited -S
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 7d4ecb709ed83..99087350689db 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -15,11 +15,24 @@
 
 set -ex
 
+if [ "$1" == "--only-library" ]; then
+  only_library_mode=yes
+else
+  only_library_mode=no
+fi
+
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 
-# Set up C++ toolchain from conda-forge packages for faster builds
-conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson
+if [ $only_library_mode == "no" ]; then
+  # C++ toolchain
+  export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+  export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
+  export RAPIDJSON_HOME=$CPP_TOOLCHAIN
+
+  # Set up C++ toolchain from conda-forge packages for faster builds
+  source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+  conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson
+fi
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
   brew update > /dev/null
@@ -32,18 +45,31 @@ pushd $ARROW_CPP_BUILD_DIR
 
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
--DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
+-DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL
+-DARROW_NO_DEPRECATED_API=ON"
+CMAKE_LINUX_FLAGS=""
+CMAKE_OSX_FLAGS=""
+
+if [ $only_library_mode == "yes" ]; then
+  CMAKE_COMMON_FLAGS="\
+$CMAKE_COMMON_FLAGS \
+-DARROW_BUILD_TESTS=OFF \
+-DARROW_BUILD_UTILITIES=OFF"
+else
+  CMAKE_LINUX_FLAGS="\
+$CMAKE_LINUX_FLAGS \
+-DARROW_TEST_MEMCHECK=ON"
+fi
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
-    cmake -DARROW_TEST_MEMCHECK=on \
-          $CMAKE_COMMON_FLAGS \
+    cmake $CMAKE_COMMON_FLAGS \
+          $CMAKE_LINUX_FLAGS \
           -DARROW_CXXFLAGS="-Wconversion -Werror" \
-          -DARROW_NO_DEPRECATED_API=on \
           $ARROW_CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
+          $CMAKE_OSX_FLAGS \
           -DARROW_CXXFLAGS=-Werror \
-          -DARROW_NO_DEPRECATED_API=on \
           $ARROW_CPP_DIR
 fi
 
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index b19fafab1d955..cef2d655cffe0 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -16,11 +16,6 @@ export MINICONDA=$HOME/miniconda
 export PATH="$MINICONDA/bin:$PATH"
 export CONDA_PKGS_DIRS=$HOME/.conda_packages
 
-# C++ toolchain
-export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
-export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
-export RAPIDJSON_HOME=$CPP_TOOLCHAIN
-
 export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp
 export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib

From 530f0da3439dc3da7b2e3cf6a2175bcc5efe1eea Mon Sep 17 00:00:00 2001
From: kimanjun <kumar.m.kiran@gmail.com>
Date: Mon, 29 May 2017 14:52:17 +0530
Subject: [PATCH 0686/1644] [Doc] Change cpp api doc, std:shared_pointer_cast
 to std::static_pointer_cast

Closes #717

Change-Id: Ib5b61398cb06a5f76d701e403f61d17953a75bb8
---
 cpp/apidoc/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
index fc10e0fddeb80..923fc5b35bd24 100644
--- a/cpp/apidoc/index.md
+++ b/cpp/apidoc/index.md
@@ -70,7 +70,7 @@ As the fourth entry is null, the value at that position in the buffer is
 undefined.
 
     // Cast the Array to its actual type to access its data
-    std::shared_ptr<Int64Array> int64_array = std::shared_pointer_cast<Int64Array>(array);
+    std::shared_ptr<Int64Array> int64_array = std::static_pointer_cast<Int64Array>(array);
 
     // Get the pointer to the null bitmap.
     const uint8_t* null_bitmap = int64_array->null_bitmap_data();

From 8229688a6f6df668f1767f74f277c3454f0c6e6a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 31 May 2017 18:42:46 -0400
Subject: [PATCH 0687/1644] ARROW-1078: [Python] Account for Apache Parquet
 shared library consolidation

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #720 from wesm/ARROW-1078 and squashes the following commits:

4bcf580e [Wes McKinney] Remove move_lib for parquet_arrow
9bd6c365 [Wes McKinney] Remove PARQUET_ARROW CMake flag in build fileS
47b38cd6 [Wes McKinney] Clean up after libparquet/libparquet_arrow consolidation
---
 ci/travis_script_python.sh             |  1 -
 cpp/doc/Parquet.md                     |  2 +-
 python/CMakeLists.txt                  | 27 ++---------------
 python/cmake_modules/FindParquet.cmake | 40 +-------------------------
 python/manylinux1/Dockerfile-x86_64    |  2 +-
 python/setup.py                        |  1 -
 6 files changed, 5 insertions(+), 68 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index ce5f7ec506b73..c3735ccadac3e 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -49,7 +49,6 @@ build_parquet_cpp() {
   cmake \
       -DCMAKE_BUILD_TYPE=debug \
       -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
-      -DPARQUET_ARROW=on \
       -DPARQUET_BUILD_BENCHMARKS=off \
       -DPARQUET_BUILD_EXECUTABLES=off \
       -DPARQUET_ZLIB_VENDORED=off \
diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md
index 4985dd3b0bc2d..ce2961ab26a0d 100644
--- a/cpp/doc/Parquet.md
+++ b/cpp/doc/Parquet.md
@@ -24,7 +24,7 @@ export ARROW_HOME=$HOME/local
 
 git clone https://github.com/apache/parquet-cpp.git
 cd parquet-cpp
-cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_ARROW=on
+cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME
 make -j4
 make install
 ```
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index b02f23bfff9ef..f35326a642f58 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -290,7 +290,7 @@ if (PYARROW_BUILD_PARQUET)
   ## Parquet
   find_package(Parquet)
 
-  if(NOT (PARQUET_FOUND AND PARQUET_ARROW_FOUND))
+  if(NOT PARQUET_FOUND)
     message(FATAL_ERROR "Unable to locate Parquet libraries")
   endif()
   include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
@@ -298,14 +298,10 @@ if (PYARROW_BUILD_PARQUET)
   if (PYARROW_BUNDLE_ARROW_CPP)
       get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY)
       get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE)
-      get_filename_component(PARQUET_ARROW_LIBRARY_NAME ${PARQUET_ARROW_SHARED_LIB} NAME_WE)
       message(STATUS "Configure start")
       configure_file(${PARQUET_SHARED_LIB}
           ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
           COPYONLY)
-      configure_file(${PARQUET_ARROW_SHARED_LIB}
-          ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_ARROW_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
-          COPYONLY)
       if (APPLE)
           configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
               ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
@@ -313,12 +309,6 @@ if (PYARROW_BUILD_PARQUET)
           configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
               ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
               COPYONLY)
-          configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-              COPYONLY)
-          configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-              COPYONLY)
       elseif (NOT MSVC)
           configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
               ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
@@ -326,34 +316,21 @@ if (PYARROW_BUILD_PARQUET)
           configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
               ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
               COPYONLY)
-          configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
-              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
-              COPYONLY)
-          configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
-              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
-              COPYONLY)
       endif()
       message(STATUS "Configure end")
       #SET(PARQUET_SHARED_LIB
       #    ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX})
-      #SET(PARQUET_ARROW_SHARED_LIB
-      #    ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
   if (MSVC)
     ADD_THIRDPARTY_LIB(parquet
       SHARED_LIB ${PARQUET_SHARED_IMP_LIB})
-    ADD_THIRDPARTY_LIB(parquet_arrow
-      SHARED_LIB ${PARQUET_ARROW_SHARED_IMP_LIB})
   else()
     ADD_THIRDPARTY_LIB(parquet
       SHARED_LIB ${PARQUET_SHARED_LIB})
-    ADD_THIRDPARTY_LIB(parquet_arrow
-      SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
   endif()
   set(LINK_LIBS
     ${LINK_LIBS}
-    parquet_shared
-    parquet_arrow_shared)
+    parquet_shared)
   set(CYTHON_EXTENSIONS
     ${CYTHON_EXTENSIONS}
     _parquet)
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index ef3c645f7682b..88dca2ed6464c 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -41,11 +41,7 @@ if (MSVC)
     PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
     PATH_SUFFIXES "bin" )
 
-  find_library(PARQUET_ARROW_SHARED_LIBRARIES NAMES parquet_arrow
-    PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
-    PATH_SUFFIXES "bin" )
   get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )
-  get_filename_component(PARQUET_ARROW_SHARED_LIBS ${PARQUET_ARROW_SHARED_LIBRARIES} PATH )
 endif ()
 
 if(PARQUET_HOME)
@@ -63,9 +59,6 @@ if(PARQUET_HOME)
     find_library(PARQUET_LIBRARIES NAMES parquet
         PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
         PATH_SUFFIXES "lib")
-    find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
-        PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
-        PATH_SUFFIXES "lib")
     get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
 else()
     pkg_check_modules(PARQUET parquet)
@@ -80,13 +73,9 @@ else()
         message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}")
         find_library(PARQUET_LIBRARIES NAMES parquet
             PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH)
-        find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
-            PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH)
-        message(STATUS "${PARQUET_ARROW_LIBRARIES}")
     else()
         find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
         find_library(PARQUET_LIBRARIES NAMES parquet)
-        find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow)
         get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
     endif()
 endif()
@@ -106,28 +95,9 @@ else ()
   set(PARQUET_FOUND FALSE)
 endif ()
 
-if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES)
-  set(PARQUET_ARROW_FOUND TRUE)
-  get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH)
-  set(PARQUET_ARROW_LIB_NAME parquet_arrow)
-  if (MSVC)
-    set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_SHARED_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
-    set(PARQUET_ARROW_SHARED_IMP_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.lib")
-  else()
-    set(PARQUET_ARROW_STATIC_LIB
-      ${PARQUET_ARROW_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}.a)
-    set(PARQUET_ARROW_SHARED_LIB
-      ${PARQUET_ARROW_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-  endif()
-else ()
-  set(PARQUET_ARROW_FOUND FALSE)
-endif ()
-
-if (PARQUET_FOUND AND PARQUET_ARROW_FOUND)
+if (PARQUET_FOUND)
   if (NOT Parquet_FIND_QUIETLY)
     message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}")
-    message(STATUS "Found the Parquet Arrow library: ${PARQUET_ARROW_LIBS}")
   endif ()
 else ()
   if (NOT Parquet_FIND_QUIETLY)
@@ -135,9 +105,6 @@ else ()
       set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet library.")
     endif()
 
-    if (NOT PARQUET_ARROW_FOUND)
-      set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet_arrow library. Did you build with -DPARQUET_ARROW=on?")
-    endif()
     set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Looked in ")
     if ( _parquet_roots )
       set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} in ${_parquet_roots}.")
@@ -159,9 +126,4 @@ mark_as_advanced(
   PARQUET_LIBRARIES
   PARQUET_STATIC_LIB
   PARQUET_SHARED_LIB
-
-  PARQUET_ARROW_FOUND
-  PARQUET_ARROW_LIBS
-  PARQUET_ARROW_STATIC_LIB
-  PARQUET_ARROW_SHARED_LIB
 )
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 363308b616e73..11e66f7d0cdb1 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -21,5 +21,5 @@ RUN make -j5 install
 WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BOOST_USE_SHARED=OFF .
 RUN make -j5 install
diff --git a/python/setup.py b/python/setup.py
index 7e612c9e274f8..b64e2591298a5 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -248,7 +248,6 @@ def move_lib(lib_name):
                 move_lib("arrow_jemalloc")
             if self.with_parquet:
                 move_lib("parquet")
-                move_lib("parquet_arrow")
 
         # Move the built C-extension to the place expected by the Python build
         self._found_names = []

From 5c155c3538b0e04b99201a204907e548757a68e1 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 31 May 2017 22:54:22 -0400
Subject: [PATCH 0688/1644] ARROW-1075: [GLib] Fix build error on macOS

int64_t is "long long" but gint64 is "long".

Author: Kouhei Sutou <kou@clear-code.com>

Closes #719 from kou/glib-fix-build-error-on-macos and squashes the following commits:

ee51ffb [Kouhei Sutou] [GLib] Fix build error on macOS
---
 c_glib/arrow-glib/file.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/file.cpp b/c_glib/arrow-glib/file.cpp
index 775339386c6b5..e05743b6a0c98 100644
--- a/c_glib/arrow-glib/file.cpp
+++ b/c_glib/arrow-glib/file.cpp
@@ -76,7 +76,7 @@ garrow_file_tell(GArrowFile *file,
 {
   auto arrow_file = garrow_file_get_raw(file);
 
-  gint64 position;
+  int64_t position;
   auto status = arrow_file->Tell(&position);
   if (garrow_error_check(error, status, "[io][file][tell]")) {
     return position;

From 092afb6dc561161019990f3e8f0c88b56e6e5120 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Fri, 2 Jun 2017 14:06:03 -0400
Subject: [PATCH 0689/1644] ARROW-990: [JS] Add tslint support for linting
 TypeScript

Added tslint config and `npm run tslint` script. Modified current code to pass lint tests.

Currently the config disables the bitwise operation and max classes per file checks. I also ignored the long line test for all of the nullable primitive vectors, since that should be replaced anyway in the near future.

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #718 from TheNeuralBit/tslint and squashes the following commits:

727d80f2 [Brian Hulette] added npm lint script
68b7c8fc [Brian Hulette] misc tslint fixes
6f1583e5 [Brian Hulette] sort object literals
67d82cac [Brian Hulette] variable names, object shorthand
0a1b872f [Brian Hulette] fix public, private, protected ordering issues
08d60e67 [Brian Hulette] quotes, equality checks
c5d85f77 [Brian Hulette] whitespace, semicolons, Errors
2b4ff287 [Brian Hulette] added public, private, protected to all members
81c9868a [Brian Hulette] Replace vars with let/const, one def per line
d591b3c7 [Brian Hulette] add tslint config
---
 js/README.md       |   1 +
 js/package.json    |   4 +-
 js/src/arrow.ts    | 316 ++++++++++++++++++----------------
 js/src/bitarray.ts |  18 +-
 js/src/types.ts    | 410 +++++++++++++++++++++++----------------------
 js/tslint.json     |  12 ++
 6 files changed, 403 insertions(+), 358 deletions(-)
 create mode 100644 js/tslint.json

diff --git a/js/README.md b/js/README.md
index 167bafcd724de..de9070c59aaa6 100644
--- a/js/README.md
+++ b/js/README.md
@@ -18,6 +18,7 @@ From this directory, run:
 
 ``` bash
 $ npm install   # pull dependencies
+$ npm run lint -- <filename>  # run tslint
 $ npm run build # build typescript (run tsc and webpack)
 $ npm run test  # run the unit tests (node.js only)
 ```
diff --git a/js/package.json b/js/package.json
index 1739e38926545..e46b5bdc23c56 100644
--- a/js/package.json
+++ b/js/package.json
@@ -6,7 +6,8 @@
   "scripts": {
     "build": "./flatbuffers.sh && tsc && tsc -m es6 --outDir lib-esm && webpack",
     "clean": "rm -rf lib lib-esm _bundles",
-    "test": "./node_modules/mocha/bin/mocha ./spec/arrow.js"
+    "test": "./node_modules/mocha/bin/mocha ./spec/arrow.js",
+    "lint": "./node_modules/tslint/bin/tslint"
   },
   "author": "",
   "repository": "https://github.com/apache/arrow/",
@@ -15,6 +16,7 @@
     "awesome-typescript-loader": "^3.1.3",
     "chai": "^3.5.0",
     "mocha": "^3.3.0",
+    "tslint": "^5.3.2",
     "typescript": "^2.3.2",
     "uglifyjs-webpack-plugin": "^0.4.3",
     "webpack": "^2.3.3"
diff --git a/js/src/arrow.ts b/js/src/arrow.ts
index 74def4dc94dc3..ac87a8c0ed766 100644
--- a/js/src/arrow.ts
+++ b/js/src/arrow.ts
@@ -15,19 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { flatbuffers } from 'flatbuffers';
-import { org } from './Arrow_generated';
-import { vectorFromField, Vector } from './types';
+import { flatbuffers } from "flatbuffers";
+import { org } from "./Arrow_generated";
+import { Vector, vectorFromField } from "./types";
 
 import ByteBuffer = flatbuffers.ByteBuffer;
-var Footer = org.apache.arrow.flatbuf.Footer;
-var Message = org.apache.arrow.flatbuf.Message;
-var MessageHeader = org.apache.arrow.flatbuf.MessageHeader;
-var RecordBatch = org.apache.arrow.flatbuf.RecordBatch;
-var DictionaryBatch = org.apache.arrow.flatbuf.DictionaryBatch;
-var Schema = org.apache.arrow.flatbuf.Schema;
-var Type = org.apache.arrow.flatbuf.Type;
-var VectorType = org.apache.arrow.flatbuf.VectorType;
+const Footer = org.apache.arrow.flatbuf.Footer;
+const Message = org.apache.arrow.flatbuf.Message;
+const MessageHeader = org.apache.arrow.flatbuf.MessageHeader;
+const RecordBatch = org.apache.arrow.flatbuf.RecordBatch;
+const DictionaryBatch = org.apache.arrow.flatbuf.DictionaryBatch;
+const Schema = org.apache.arrow.flatbuf.Schema;
+const Type = org.apache.arrow.flatbuf.Type;
+const VectorType = org.apache.arrow.flatbuf.VectorType;
 
 export class ArrowReader {
 
@@ -43,16 +43,16 @@ export class ArrowReader {
         this.bb = bb;
         this.schema = schema;
         this.vectors = vectors;
-        for (var i = 0; i < vectors.length; i += 1|0) {
-            this.vectorMap[vectors[i].name] = vectors[i]
+        for (let i = 0; i < vectors.length; i++) {
+            this.vectorMap[vectors[i].name] = vectors[i];
         }
         this.batches = batches;
         this.dictionaries = dictionaries;
     }
 
-    loadNextBatch() {
+    public loadNextBatch() {
         if (this.batchIndex < this.batches.length) {
-            var batch = this.batches[this.batchIndex];
+            const batch = this.batches[this.batchIndex];
             this.batchIndex += 1;
             loadVectors(this.bb, this.vectors, batch);
             return batch.length;
@@ -61,36 +61,36 @@ export class ArrowReader {
         }
     }
 
-    getSchema() {
+    public getSchema() {
         return this.schema;
     }
 
-    getVectors() {
+    public getVectors() {
         return this.vectors;
     }
 
-    getVector(name) {
+    public getVector(name) {
         return this.vectorMap[name];
     }
 
-    getBatchCount() {
+    public getBatchCount() {
         return this.batches.length;
     }
 
     // the index of the next batch to be loaded
-    getBatchIndex() {
+    public getBatchIndex() {
         return this.batchIndex;
     }
 
     // set the index of the next batch to be loaded
-    setBatchIndex(i: number) {
+    public setBatchIndex(i: number) {
         this.batchIndex = i;
     }
 }
 
 export function getSchema(buf) { return getReader(buf).getSchema(); }
 
-export function getReader(buf) : ArrowReader {
+export function getReader(buf): ArrowReader {
     if (_checkMagic(buf, 0)) {
         return getFileReader(buf);
     } else {
@@ -98,20 +98,20 @@ export function getReader(buf) : ArrowReader {
     }
 }
 
-export function getStreamReader(buf) : ArrowReader {
-    var bb = new ByteBuffer(buf);
+export function getStreamReader(buf): ArrowReader {
+    const bb = new ByteBuffer(buf);
 
-    var schema = _loadSchema(bb),
-        field,
-        vectors: Vector[] = [],
-        i,j,
-        iLen,jLen,
-        batch,
-        recordBatches = [],
-        dictionaryBatches = [],
-        dictionaries = {};
+    const schema = _loadSchema(bb);
+    let field;
+    const vectors: Vector[] = [];
+    let i;
+    let iLen;
+    let batch;
+    const recordBatches = [];
+    const dictionaryBatches = [];
+    const dictionaries = {};
 
-    for (i = 0, iLen = schema.fieldsLength(); i < iLen; i += 1|0) {
+    for (i = 0, iLen = schema.fieldsLength(); i < iLen; i++) {
         field = schema.fields(i);
         _createDictionaryVectors(field, dictionaries);
         vectors.push(vectorFromField(field, dictionaries));
@@ -121,18 +121,18 @@ export function getStreamReader(buf) : ArrowReader {
       batch = _loadBatch(bb);
       if (batch == null) {
           break;
-      } else if (batch.type == MessageHeader.DictionaryBatch) {
+      } else if (batch.type === MessageHeader.DictionaryBatch) {
           dictionaryBatches.push(batch);
-      } else if (batch.type == MessageHeader.RecordBatch) {
-          recordBatches.push(batch)
+      } else if (batch.type === MessageHeader.RecordBatch) {
+          recordBatches.push(batch);
       } else {
-          console.error("Expected batch type" + MessageHeader.RecordBatch + " or " +
+          throw new Error("Expected batch type" + MessageHeader.RecordBatch + " or " +
               MessageHeader.DictionaryBatch + " but got " + batch.type);
       }
     }
 
     // load dictionary vectors
-    for (i = 0; i < dictionaryBatches.length; i += 1|0) {
+    for (i = 0; i < dictionaryBatches.length; i++) {
       batch = dictionaryBatches[i];
       loadVectors(bb, [dictionaries[batch.id]], batch);
     }
@@ -140,58 +140,60 @@ export function getStreamReader(buf) : ArrowReader {
     return new ArrowReader(bb, parseSchema(schema), vectors, recordBatches, dictionaries);
 }
 
-export function getFileReader (buf) : ArrowReader {
-    var bb = new ByteBuffer(buf);
+export function getFileReader(buf): ArrowReader {
+    const bb = new ByteBuffer(buf);
 
-    var footer = _loadFooter(bb);
+    const footer = _loadFooter(bb);
 
-    var schema = footer.schema();
-    var i, len, field,
-        vectors: Vector[] = [],
-        block,
-        batch,
-        recordBatchBlocks = [],
-        dictionaryBatchBlocks = [],
-        dictionaries = {};
+    const schema = footer.schema();
+    let i;
+    let len;
+    let field;
+    const vectors: Vector[] = [];
+    let block;
+    let batch;
+    const recordBatchBlocks = [];
+    const dictionaryBatchBlocks = [];
+    const dictionaries = {};
 
-    for (i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
+    for (i = 0, len = schema.fieldsLength(); i < len; i++) {
         field = schema.fields(i);
         _createDictionaryVectors(field, dictionaries);
         vectors.push(vectorFromField(field, dictionaries));
     }
 
-    for (i = 0; i < footer.dictionariesLength(); i += 1|0) {
+    for (i = 0; i < footer.dictionariesLength(); i++) {
         block = footer.dictionaries(i);
         dictionaryBatchBlocks.push({
-            offset: block.offset().low,
-            metaDataLength: block.metaDataLength(),
             bodyLength: block.bodyLength().low,
-        })
+            metaDataLength: block.metaDataLength(),
+            offset: block.offset().low,
+        });
     }
 
-    for (i = 0; i < footer.recordBatchesLength(); i += 1|0) {
+    for (i = 0; i < footer.recordBatchesLength(); i++) {
         block = footer.recordBatches(i);
         recordBatchBlocks.push({
-            offset: block.offset().low,
-            metaDataLength: block.metaDataLength(),
             bodyLength: block.bodyLength().low,
-        })
+            metaDataLength: block.metaDataLength(),
+            offset: block.offset().low,
+        });
     }
 
-    var dictionaryBatches = dictionaryBatchBlocks.map(function (block) {
-        bb.setPosition(block.offset);
+    const dictionaryBatches = dictionaryBatchBlocks.map((batchBlock) => {
+        bb.setPosition(batchBlock.offset);
         // TODO: Make sure this is a dictionary batch
         return _loadBatch(bb);
     });
 
-    var recordBatches = recordBatchBlocks.map(function (block) {
-        bb.setPosition(block.offset);
+    const recordBatches = recordBatchBlocks.map((batchBlock) => {
+        bb.setPosition(batchBlock.offset);
         // TODO: Make sure this is a record batch
         return _loadBatch(bb);
     });
 
     // load dictionary vectors
-    for (i = 0; i < dictionaryBatches.length; i += 1|0) {
+    for (i = 0; i < dictionaryBatches.length; i++) {
         batch = dictionaryBatches[i];
         loadVectors(bb, [dictionaries[batch.id]], batch);
     }
@@ -200,107 +202,124 @@ export function getFileReader (buf) : ArrowReader {
 }
 
 function _loadFooter(bb) {
-    var fileLength: number = bb.bytes_.length;
+    const fileLength: number = bb.bytes_.length;
 
-    if (fileLength < MAGIC.length*2 + 4) {
-      console.error("file too small " + fileLength);
-      return;
+    if (fileLength < MAGIC.length * 2 + 4) {
+      throw new Error("file too small " + fileLength);
     }
 
     if (!_checkMagic(bb.bytes_, 0)) {
-      console.error("missing magic bytes at beginning of file")
-      return;
+      throw new Error("missing magic bytes at beginning of file");
     }
 
     if (!_checkMagic(bb.bytes_, fileLength - MAGIC.length)) {
-      console.error("missing magic bytes at end of file")
-      return;
+      throw new Error("missing magic bytes at end of file");
     }
 
-    var footerLengthOffset: number = fileLength - MAGIC.length - 4;
+    const footerLengthOffset: number = fileLength - MAGIC.length - 4;
     bb.setPosition(footerLengthOffset);
-    var footerLength: number = Int32FromByteBuffer(bb, footerLengthOffset)
+    const footerLength: number = Int32FromByteBuffer(bb, footerLengthOffset);
 
-    if (footerLength <= 0 || footerLength + MAGIC.length*2 + 4 > fileLength)  {
-      console.log("Invalid footer length: " + footerLength)
+    if (footerLength <= 0 || footerLength + MAGIC.length * 2 + 4 > fileLength)  {
+      throw new Error("Invalid footer length: " + footerLength);
     }
 
-    var footerOffset: number = footerLengthOffset - footerLength;
+    const footerOffset: number = footerLengthOffset - footerLength;
     bb.setPosition(footerOffset);
-    var footer = Footer.getRootAsFooter(bb);
+    const footer = Footer.getRootAsFooter(bb);
 
     return footer;
 }
 
 function _loadSchema(bb) {
-    var message =_loadMessage(bb);
-    if (message.headerType() != MessageHeader.Schema) {
-        console.error("Expected header type " + MessageHeader.Schema + " but got " + message.headerType());
-        return;
+    const message = _loadMessage(bb);
+    if (message.headerType() !== MessageHeader.Schema) {
+        throw new Error("Expected header type " + MessageHeader.Schema + " but got " + message.headerType());
     }
     return message.header(new Schema());
 }
 
 function _loadBatch(bb) {
-    var message = _loadMessage(bb);
+    const message = _loadMessage(bb);
     if (message == null) {
         return;
-    } else if (message.headerType() == MessageHeader.RecordBatch) {
-        var batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low }
+    } else if (message.headerType() === MessageHeader.RecordBatch) {
+        const batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low };
         return _loadRecordBatch(bb, batch);
-    } else if (message.headerType() == MessageHeader.DictionaryBatch) {
-        var batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low }
+    } else if (message.headerType() === MessageHeader.DictionaryBatch) {
+        const batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low };
         return _loadDictionaryBatch(bb, batch);
     } else {
-        console.error("Expected header type " + MessageHeader.RecordBatch + " or " + MessageHeader.DictionaryBatch +
+        throw new Error("Expected header type " + MessageHeader.RecordBatch + " or " + MessageHeader.DictionaryBatch +
             " but got " + message.headerType());
-        return;
     }
 }
 
 function _loadRecordBatch(bb, batch) {
-    var data = batch.header;
-    var i, nodes_ = [], nodesLength = data.nodesLength();
-    var buffer, buffers_ = [], buffersLength = data.buffersLength();
+    const data = batch.header;
+    let i;
+    const nodesLength = data.nodesLength();
+    const nodes = new Array(nodesLength);
+    let buffer;
+    const buffersLength = data.buffersLength();
+    const buffers = new Array(buffersLength);
 
     for (i = 0; i < nodesLength; i += 1) {
-        nodes_.push(data.nodes(i));
+        nodes[i] = data.nodes(i);
     }
+
     for (i = 0; i < buffersLength; i += 1) {
         buffer = data.buffers(i);
-        buffers_.push({ offset: bb.position() + buffer.offset().low, length: buffer.length().low });
+        buffers[i] = {
+            length: buffer.length().low,
+            offset: bb.position() + buffer.offset().low,
+        };
     }
     // position the buffer after the body to read the next message
     bb.setPosition(bb.position() + batch.length);
 
-    return { nodes: nodes_, buffers: buffers_, length: data.length().low, type: MessageHeader.RecordBatch };
+    return { nodes, buffers, length: data.length().low, type: MessageHeader.RecordBatch };
 }
 
 function _loadDictionaryBatch(bb, batch) {
-    var id_ = batch.header.id().toFloat64().toString(), data = batch.header.data();
-    var i, nodes_ = [], nodesLength = data.nodesLength();
-    var buffer, buffers_ = [], buffersLength = data.buffersLength();
+    const id = batch.header.id().toFloat64().toString();
+    const data = batch.header.data();
+    let i;
+    const nodesLength = data.nodesLength();
+    const nodes = new Array(nodesLength);
+    let buffer;
+    const buffersLength = data.buffersLength();
+    const buffers = new Array(buffersLength);
 
     for (i = 0; i < nodesLength; i += 1) {
-        nodes_.push(data.nodes(i));
+        nodes[i] = data.nodes(i);
     }
     for (i = 0; i < buffersLength; i += 1) {
         buffer = data.buffers(i);
-        buffers_.push({ offset: bb.position() + buffer.offset().low, length: buffer.length().low });
+        buffers[i] = {
+            length: buffer.length().low,
+            offset: bb.position() + buffer.offset().low,
+        };
     }
     // position the buffer after the body to read the next message
     bb.setPosition(bb.position() + batch.length);
 
-    return { id: id_, nodes: nodes_, buffers: buffers_, length: data.length().low, type: MessageHeader.DictionaryBatch };
+    return {
+        buffers,
+        id,
+        length: data.length().low,
+        nodes,
+        type: MessageHeader.DictionaryBatch,
+    };
 }
 
 function _loadMessage(bb) {
-    var messageLength: number = Int32FromByteBuffer(bb, bb.position());
-    if (messageLength == 0) {
+    const messageLength: number = Int32FromByteBuffer(bb, bb.position());
+    if (messageLength === 0) {
       return;
     }
     bb.setPosition(bb.position() + 4);
-    var message = Message.getRootAsMessage(bb);
+    const message = Message.getRootAsMessage(bb);
     // position the buffer at the end of the message so it's ready to read further
     bb.setPosition(bb.position() + messageLength);
 
@@ -308,36 +327,36 @@ function _loadMessage(bb) {
 }
 
 function _createDictionaryVectors(field, dictionaries) {
-    var encoding = field.dictionary();
+    const encoding = field.dictionary();
     if (encoding != null) {
-        var id = encoding.id().toFloat64().toString();
+        const id = encoding.id().toFloat64().toString();
         if (dictionaries[id] == null) {
             // create a field for the dictionary
-            var dictionaryField = _createDictionaryField(id, field);
+            const dictionaryField = _createDictionaryField(id, field);
             dictionaries[id] = vectorFromField(dictionaryField, null);
         }
     }
 
     // recursively examine child fields
-    for (var i = 0, len = field.childrenLength(); i < len; i += 1|0) {
+    for (let i = 0, len = field.childrenLength(); i < len; i++) {
         _createDictionaryVectors(field.children(i), dictionaries);
     }
 }
 
 function _createDictionaryField(id, field) {
-    var builder = new flatbuffers.Builder();
-    var nameOffset = builder.createString("dict-" + id);
+    const builder = new flatbuffers.Builder();
+    const nameOffset = builder.createString("dict-" + id);
 
-    var typeType = field.typeType();
-    var typeOffset;
+    const typeType = field.typeType();
+    let typeOffset;
     if (typeType === Type.Int) {
-        var type = field.type(new org.apache.arrow.flatbuf.Int());
+        const type = field.type(new org.apache.arrow.flatbuf.Int());
         org.apache.arrow.flatbuf.Int.startInt(builder);
         org.apache.arrow.flatbuf.Int.addBitWidth(builder, type.bitWidth());
         org.apache.arrow.flatbuf.Int.addIsSigned(builder, type.isSigned());
         typeOffset = org.apache.arrow.flatbuf.Int.endInt(builder);
     } else if (typeType === Type.FloatingPoint) {
-        var type = field.type(new org.apache.arrow.flatbuf.FloatingPoint());
+        const type = field.type(new org.apache.arrow.flatbuf.FloatingPoint());
         org.apache.arrow.flatbuf.FloatingPoint.startFloatingPoint(builder);
         org.apache.arrow.flatbuf.FloatingPoint.addPrecision(builder, type.precision());
         typeOffset = org.apache.arrow.flatbuf.FloatingPoint.endFloatingPoint(builder);
@@ -345,27 +364,28 @@ function _createDictionaryField(id, field) {
         org.apache.arrow.flatbuf.Utf8.startUtf8(builder);
         typeOffset = org.apache.arrow.flatbuf.Utf8.endUtf8(builder);
     } else if (typeType === Type.Date) {
-        var type = field.type(new org.apache.arrow.flatbuf.Date());
+        const type = field.type(new org.apache.arrow.flatbuf.Date());
         org.apache.arrow.flatbuf.Date.startDate(builder);
         org.apache.arrow.flatbuf.Date.addUnit(builder, type.unit());
         typeOffset = org.apache.arrow.flatbuf.Date.endDate(builder);
     } else {
-        throw "Unimplemented dictionary type " + typeType;
+        throw new Error("Unimplemented dictionary type " + typeType);
     }
     if (field.childrenLength() > 0) {
-      throw "Dictionary encoded fields can't have children"
+      throw new Error("Dictionary encoded fields can't have children");
     }
-    var childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, []);
+    const childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, []);
 
-    var layout, layoutOffsets = [];
-    for (var i = 0, len = field.layoutLength(); i < len; i += 1|0) {
+    let layout;
+    const layoutOffsets = [];
+    for (let i = 0, len = field.layoutLength(); i < len; i++) {
         layout = field.layout(i);
         org.apache.arrow.flatbuf.VectorLayout.startVectorLayout(builder);
         org.apache.arrow.flatbuf.VectorLayout.addBitWidth(builder, layout.bitWidth());
         org.apache.arrow.flatbuf.VectorLayout.addType(builder, layout.type());
         layoutOffsets.push(org.apache.arrow.flatbuf.VectorLayout.endVectorLayout(builder));
     }
-    var layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, layoutOffsets);
+    const layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, layoutOffsets);
 
     org.apache.arrow.flatbuf.Field.startField(builder);
     org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
@@ -374,7 +394,7 @@ function _createDictionaryField(id, field) {
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
     org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
     org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
-    var offset = org.apache.arrow.flatbuf.Field.endField(builder);
+    const offset = org.apache.arrow.flatbuf.Field.endField(builder);
     builder.finish(offset);
 
     return org.apache.arrow.flatbuf.Field.getRootAsField(builder.bb);
@@ -387,22 +407,22 @@ function Int32FromByteBuffer(bb, offset) {
            ((bb.bytes_[offset] & 255));
 }
 
-var MAGIC_STR = "ARROW1";
-var MAGIC = new Uint8Array(MAGIC_STR.length);
-for (var i = 0; i < MAGIC_STR.length; i += 1|0) {
+const MAGIC_STR = "ARROW1";
+const MAGIC = new Uint8Array(MAGIC_STR.length);
+for (let i = 0; i < MAGIC_STR.length; i++) {
     MAGIC[i] = MAGIC_STR.charCodeAt(i);
 }
 
 function _checkMagic(buf, index) {
-    for (var i = 0; i < MAGIC.length; i += 1|0) {
-        if (MAGIC[i] != buf[index + i]) {
+    for (let i = 0; i < MAGIC.length; i++) {
+        if (MAGIC[i] !== buf[index + i]) {
             return false;
         }
     }
     return true;
 }
 
-var TYPEMAP = {}
+const TYPEMAP = {};
 TYPEMAP[Type.NONE]          = "NONE";
 TYPEMAP[Type.Null]          = "Null";
 TYPEMAP[Type.Int]           = "Int";
@@ -420,45 +440,44 @@ TYPEMAP[Type.FixedSizeList] = "FixedSizeList";
 TYPEMAP[Type.Struct_]       = "Struct";
 TYPEMAP[Type.Union]         = "Union";
 
-var VECTORTYPEMAP = {};
-VECTORTYPEMAP[VectorType.OFFSET]   = 'OFFSET';
-VECTORTYPEMAP[VectorType.DATA]     = 'DATA';
-VECTORTYPEMAP[VectorType.VALIDITY] = 'VALIDITY';
-VECTORTYPEMAP[VectorType.TYPE]     = 'TYPE';
+const VECTORTYPEMAP = {};
+VECTORTYPEMAP[VectorType.OFFSET]   = "OFFSET";
+VECTORTYPEMAP[VectorType.DATA]     = "DATA";
+VECTORTYPEMAP[VectorType.VALIDITY] = "VALIDITY";
+VECTORTYPEMAP[VectorType.TYPE]     = "TYPE";
 
 function parseField(field) {
-    var children = [];
-    for (var i = 0; i < field.childrenLength(); i += 1|0) {
+    const children = [];
+    for (let i = 0; i < field.childrenLength(); i++) {
         children.push(parseField(field.children(i)));
     }
 
-    var layouts = [];
-    for (var i = 0; i < field.layoutLength(); i += 1|0) {
+    const layouts = [];
+    for (let i = 0; i < field.layoutLength(); i++) {
         layouts.push(VECTORTYPEMAP[field.layout(i).type()]);
     }
 
     return {
+      children,
+      layout: layouts,
       name: field.name(),
       nullable: field.nullable(),
       type: TYPEMAP[field.typeType()],
-      children: children,
-      layout: layouts
     };
 }
 
 function parseSchema(schema) {
-    var result = [];
-    var this_result, type;
-    for (var i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
+    const result = [];
+    for (let i = 0, len = schema.fieldsLength(); i < len; i++) {
         result.push(parseField(schema.fields(i)));
     }
     return result;
 }
 
 function loadVectors(bb, vectors: Vector[], recordBatch) {
-    var indices = { bufferIndex: 0, nodeIndex: 0 }, i;
-    for (i = 0; i < vectors.length; i += 1) {
-        loadVector(bb, vectors[i], recordBatch, indices);
+    const indices = { bufferIndex: 0, nodeIndex: 0 };
+    for (const vector of vectors) {
+        loadVector(bb, vector, recordBatch, indices);
     }
 }
 
@@ -467,7 +486,10 @@ function loadVectors(bb, vectors: Vector[], recordBatch) {
  *   recordBatch: { nodes: org.apache.arrow.flatbuf.FieldNode[], buffers: { offset: number, length: number }[] }
  */
 function loadVector(bb, vector: Vector, recordBatch, indices) {
-    var node = recordBatch.nodes[indices.nodeIndex], ownBuffersLength, ownBuffers = [], i;
+    const node = recordBatch.nodes[indices.nodeIndex];
+    let ownBuffersLength;
+    const ownBuffers = [];
+    let i;
     indices.nodeIndex += 1;
 
     // dictionary vectors are always ints, so will have a data vector plus optional null vector
@@ -486,7 +508,7 @@ function loadVector(bb, vector: Vector, recordBatch, indices) {
 
     vector.loadData(bb, node, ownBuffers);
 
-    var children = vector.getChildVectors();
+    const children = vector.getChildVectors();
     for (i = 0; i < children.length; i++) {
         loadVector(bb, children[i], recordBatch, indices);
     }
diff --git a/js/src/bitarray.ts b/js/src/bitarray.ts
index fc3c0918eaf25..6b0a91aed606a 100644
--- a/js/src/bitarray.ts
+++ b/js/src/bitarray.ts
@@ -22,21 +22,21 @@ export class BitArray {
         this.view = new Uint8Array(buffer, offset || 0, Math.ceil(length / 8));
     }
 
-    get(i) {
-        var index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too.
-        var bit = i % 8;  // i % 8 is just as fast as i & 7
+    public get(i) {
+        const index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too.
+        const bit = i % 8;  // i % 8 is just as fast as i & 7
         return (this.view[index] & (1 << bit)) !== 0;
     }
 
-    set(i) {
-        var index = (i >> 3) | 0;
-        var bit = i % 8;
+    public set(i) {
+        const index = (i >> 3) | 0;
+        const bit = i % 8;
         this.view[index] |= 1 << bit;
     }
 
-    unset(i) {
-        var index = (i >> 3) | 0;
-        var bit = i % 8;
+    public unset(i) {
+        const index = (i >> 3) | 0;
+        const bit = i % 8;
         this.view[index] &= ~(1 << bit);
     }
 }
diff --git a/js/src/types.ts b/js/src/types.ts
index d656c6aa98cf1..c541098d53426 100644
--- a/js/src/types.ts
+++ b/js/src/types.ts
@@ -15,22 +15,50 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { BitArray } from './bitarray';
-import { TextDecoder } from 'text-encoding';
-import { org } from './Arrow_generated';
+import { org } from "./Arrow_generated";
+import { BitArray } from "./bitarray";
 
-var Type = org.apache.arrow.flatbuf.Type;
+import { TextDecoder } from "text-encoding";
 
-interface ArrayView {
-    slice(start: number, end: number) : ArrayView
-    toString() : string
+const Type = org.apache.arrow.flatbuf.Type;
+
+interface IArrayView {
+    slice(start: number, end: number): IArrayView;
+    toString(): string;
+}
+
+interface IViewConstructor<T extends IArrayView> {
+    BYTES_PER_ELEMENT: number;
+    new(buffer: any, offset: number, length: number): T;
 }
 
 export abstract class Vector {
-    field: any;
-    name: string;
-    length: number;
-    null_count: number;
+    /**
+     * Helper function for loading a VALIDITY buffer (for Nullable types)
+     *   bb: flatbuffers.ByteBuffer
+     *   buffer: org.apache.arrow.flatbuf.Buffer
+     */
+    public static loadValidityBuffer(bb, buffer): BitArray {
+        const arrayBuffer = bb.bytes_.buffer;
+        const offset = bb.bytes_.byteOffset + buffer.offset;
+        return new BitArray(arrayBuffer, offset, buffer.length * 8);
+    }
+
+    /**
+     * Helper function for loading an OFFSET buffer
+     *   buffer: org.apache.arrow.flatbuf.Buffer
+     */
+    public static loadOffsetBuffer(bb, buffer): Int32Array {
+        const arrayBuffer = bb.bytes_.buffer;
+        const offset  = bb.bytes_.byteOffset + buffer.offset;
+        const length = buffer.length / Int32Array.BYTES_PER_ELEMENT;
+        return new Int32Array(arrayBuffer, offset, length);
+    }
+
+    public field: any;
+    public name: string;
+    public length: number;
+    public nullCount: number;
 
     constructor(field) {
         this.field = field;
@@ -38,11 +66,13 @@ export abstract class Vector {
     }
 
     /* Access datum at index i */
-    abstract get(i);
+    public abstract get(i);
     /* Return array representing data in the range [start, end) */
-    abstract slice(start: number, end: number);
+    public abstract slice(start: number, end: number);
     /* Return array of child vectors, for container types */
-    abstract getChildVectors();
+    public getChildVectors() {
+        return [];
+    }
 
     /**
      * Use recordBatch fieldNodes and Buffers to construct this Vector
@@ -52,85 +82,58 @@ export abstract class Vector {
      */
     public loadData(bb, node, buffers) {
         this.length = node.length().low;
-        this.null_count = node.nullCount().low;
+        this.nullCount = node.nullCount().low;
         this.loadBuffers(bb, node, buffers);
     }
 
     protected abstract loadBuffers(bb, node, buffers);
-
-    /**
-     * Helper function for loading a VALIDITY buffer (for Nullable types)
-     *   bb: flatbuffers.ByteBuffer
-     *   buffer: org.apache.arrow.flatbuf.Buffer
-     */
-    static loadValidityBuffer(bb, buffer) : BitArray {
-        var arrayBuffer = bb.bytes_.buffer;
-        var offset = bb.bytes_.byteOffset + buffer.offset;
-        return new BitArray(arrayBuffer, offset, buffer.length * 8);
-    }
-
-    /**
-     * Helper function for loading an OFFSET buffer
-     *   buffer: org.apache.arrow.flatbuf.Buffer
-     */
-    static loadOffsetBuffer(bb, buffer) : Int32Array {
-        var arrayBuffer = bb.bytes_.buffer;
-        var offset  = bb.bytes_.byteOffset + buffer.offset;
-        var length = buffer.length / Int32Array.BYTES_PER_ELEMENT;
-        return new Int32Array(arrayBuffer, offset, length);
-    }
-
 }
 
-class SimpleVector<T extends ArrayView> extends Vector {
+class SimpleVector<T extends IArrayView> extends Vector {
     protected dataView: T;
-    private TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number };
+    private TypedArray: IViewConstructor<T>;
 
-    constructor (field, TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number }) {
+    constructor(field, TypedArray: IViewConstructor<T>) {
         super(field);
         this.TypedArray = TypedArray;
     }
 
-    getChildVectors() {
-        return [];
-    }
-
-    get(i) {
+    public get(i) {
         return this.dataView[i];
     }
 
-    loadBuffers(bb, node, buffers) {
-        this.loadDataBuffer(bb, buffers[0]);
+    public getDataView() {
+        return this.dataView;
     }
 
-    /**
-      * buffer: org.apache.arrow.flatbuf.Buffer
-      */
-    protected loadDataBuffer(bb, buffer) {
-        var arrayBuffer = bb.bytes_.buffer;
-        var offset  = bb.bytes_.byteOffset + buffer.offset;
-        var length = buffer.length / this.TypedArray.BYTES_PER_ELEMENT;
-        this.dataView = new this.TypedArray(arrayBuffer, offset, length);
+    public toString() {
+        return this.dataView.toString();
     }
 
-    getDataView() {
-        return this.dataView;
+    public slice(start, end) {
+        return this.dataView.slice(start, end);
     }
 
-    toString() {
-        return this.dataView.toString();
+    protected loadBuffers(bb, node, buffers) {
+        this.loadDataBuffer(bb, buffers[0]);
     }
 
-    slice(start, end) {
-        return this.dataView.slice(start, end);
+    /*
+     * buffer: org.apache.arrow.flatbuf.Buffer
+     */
+    protected loadDataBuffer(bb, buffer) {
+        const arrayBuffer = bb.bytes_.buffer;
+        const offset  = bb.bytes_.byteOffset + buffer.offset;
+        const length = buffer.length / this.TypedArray.BYTES_PER_ELEMENT;
+        this.dataView = new this.TypedArray(arrayBuffer, offset, length);
     }
-}
 
-class NullableSimpleVector<T extends ArrayView> extends SimpleVector<T> {
+}
 
+class NullableSimpleVector<T extends IArrayView> extends SimpleVector<T> {
     protected validityView: BitArray;
 
-    get(i: number) {
+    public get(i: number) {
         if (this.validityView.get(i)) {
             return this.dataView[i];
         } else {
@@ -138,40 +141,42 @@ class NullableSimpleVector<T extends ArrayView> extends SimpleVector<T> {
         }
     }
 
-    loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-        this.loadDataBuffer(bb, buffers[1]);
+    public getValidityVector() {
+        return this.validityView;
     }
 
-    getValidityVector() {
-        return this.validityView;
+    protected loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+        this.loadDataBuffer(bb, buffers[1]);
     }
 }
 
-class Uint8Vector   extends SimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   }; }
-class Uint16Vector  extends SimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  }; }
-class Uint32Vector  extends SimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  }; }
-class Int8Vector    extends SimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   }; }
-class Int16Vector   extends SimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  }; }
-class Int32Vector   extends SimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  }; }
-class Float32Vector extends SimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); }; }
-class Float64Vector extends SimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); }; }
-
-class NullableUint8Vector   extends NullableSimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   }; }
-class NullableUint16Vector  extends NullableSimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  }; }
-class NullableUint32Vector  extends NullableSimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  }; }
-class NullableInt8Vector    extends NullableSimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   }; }
-class NullableInt16Vector   extends NullableSimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  }; }
-class NullableInt32Vector   extends NullableSimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  }; }
-class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); }; }
-class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); }; }
+/* tslint:disable max-line-length */
+class Uint8Vector   extends SimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   } }
+class Uint16Vector  extends SimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  } }
+class Uint32Vector  extends SimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  } }
+class Int8Vector    extends SimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   } }
+class Int16Vector   extends SimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  } }
+class Int32Vector   extends SimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  } }
+class Float32Vector extends SimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); } }
+class Float64Vector extends SimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); } }
+
+class NullableUint8Vector   extends NullableSimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   } }
+class NullableUint16Vector  extends NullableSimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  } }
+class NullableUint32Vector  extends NullableSimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  } }
+class NullableInt8Vector    extends NullableSimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   } }
+class NullableInt16Vector   extends NullableSimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  } }
+class NullableInt32Vector   extends NullableSimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  } }
+class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); } }
+class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); } }
+/* tslint:enable max-line-length */
 
 class Uint64Vector extends SimpleVector<Uint32Array>  {
     constructor(field) {
         super(field, Uint32Array);
     }
 
-    get(i: number) {
+    public get(i: number) {
         return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
     }
 }
@@ -181,7 +186,7 @@ class NullableUint64Vector extends NullableSimpleVector<Uint32Array>  {
         super(field, Uint32Array);
     }
 
-    get(i: number) {
+    public get(i: number) {
         if (this.validityView.get(i)) {
             return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
         } else {
@@ -195,7 +200,7 @@ class Int64Vector extends NullableSimpleVector<Uint32Array>  {
         super(field, Uint32Array);
     }
 
-    get(i: number) {
+    public get(i: number) {
         return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
     }
 }
@@ -205,7 +210,7 @@ class NullableInt64Vector extends NullableSimpleVector<Uint32Array>  {
         super(field, Uint32Array);
     }
 
-    get(i: number) {
+    public get(i: number) {
         if (this.validityView.get(i)) {
             return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
         } else {
@@ -219,20 +224,15 @@ class DateVector extends SimpleVector<Uint32Array> {
         super(field, Uint32Array);
     }
 
-    get (i) {
-        return new Date(super.get(2*i+1)*Math.pow(2,32) + super.get(2*i));
+    public get(i) {
+        return new Date(super.get(2 * i + 1) * Math.pow(2, 32) + super.get(2 * i));
     }
 }
 
 class NullableDateVector extends DateVector {
     private validityView: BitArray;
 
-    loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-        this.loadDataBuffer(bb, buffers[1]);
-    }
-
-    get (i) {
+    public get(i) {
         if (this.validityView.get(i)) {
             return super.get(i);
         } else {
@@ -240,51 +240,51 @@ class NullableDateVector extends DateVector {
         }
     }
 
-    getValidityVector() {
+    public getValidityVector() {
         return this.validityView;
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+        this.loadDataBuffer(bb, buffers[1]);
+    }
 }
 
 class Utf8Vector extends SimpleVector<Uint8Array> {
+    private static decoder: TextDecoder = new TextDecoder("utf8");
+
     protected offsetView: Int32Array;
-    static decoder: TextDecoder = new TextDecoder('utf8');
 
     constructor(field) {
         super(field, Uint8Array);
     }
 
-    loadBuffers(bb, node, buffers) {
-        this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]);
-        this.loadDataBuffer(bb, buffers[1]);
-    }
-
-    get(i) {
+    public get(i) {
         return Utf8Vector.decoder.decode(this.dataView.slice(this.offsetView[i], this.offsetView[i + 1]));
     }
 
-    slice(start: number, end: number) {
-        var result: string[] = [];
-        for (var i: number = start; i < end; i += 1|0) {
+    public slice(start: number, end: number) {
+        const result: string[] = [];
+        for (let i: number = start; i < end; i++) {
             result.push(this.get(i));
         }
         return result;
     }
 
-    getOffsetView() {
+    public getOffsetView() {
         return this.offsetView;
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]);
+        this.loadDataBuffer(bb, buffers[1]);
+    }
 }
 
 class NullableUtf8Vector extends Utf8Vector {
     private validityView: BitArray;
 
-    loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-        this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]);
-        this.loadDataBuffer(bb, buffers[2]);
-    }
-
-    get(i) {
+    public get(i) {
         if (this.validityView.get(i)) {
             return super.get(i);
         } else {
@@ -292,9 +292,15 @@ class NullableUtf8Vector extends Utf8Vector {
         }
     }
 
-    getValidityVector() {
+    public getValidityVector() {
         return this.validityView;
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+        this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]);
+        this.loadDataBuffer(bb, buffers[2]);
+    }
 }
 
 // Nested Types
@@ -306,47 +312,41 @@ class ListVector extends Uint32Vector {
         this.dataVector = dataVector;
     }
 
-    getChildVectors() {
+    public getChildVectors() {
         return [this.dataVector];
     }
 
-    loadBuffers(bb, node, buffers) {
-        super.loadBuffers(bb, node, buffers);
-        this.length -= 1;
-    }
-
-    get(i) {
-        var offset = super.get(i)
+    public get(i) {
+        const offset = super.get(i);
         if (offset === null) {
             return null;
         }
-        var next_offset = super.get(i + 1)
-        return this.dataVector.slice(offset, next_offset)
+        const nextOffset = super.get(i + 1);
+        return this.dataVector.slice(offset, nextOffset);
     }
 
-    toString() {
+    public toString() {
         return "length: " + (this.length);
     }
 
-    slice(start: number, end: number) {
-        var result = [];
-        for (var i = start; i < end; i += 1|0) {
+    public slice(start: number, end: number) {
+        const result = [];
+        for (let i = start; i < end; i++) {
             result.push(this.get(i));
         }
         return result;
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        super.loadBuffers(bb, node, buffers);
+        this.length -= 1;
+    }
 }
 
 class NullableListVector extends ListVector {
     private validityView: BitArray;
 
-    loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-        this.loadDataBuffer(bb, buffers[1]);
-        this.length -= 1;
-    }
-
-    get(i) {
+    public get(i) {
         if (this.validityView.get(i)) {
             return super.get(i);
         } else {
@@ -354,13 +354,19 @@ class NullableListVector extends ListVector {
         }
     }
 
-    getValidityVector() {
+    public getValidityVector() {
         return this.validityView;
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+        this.loadDataBuffer(bb, buffers[1]);
+        this.length -= 1;
+    }
 }
 
 class FixedSizeListVector extends Vector {
-    private size: number
+    public size: number;
     private dataVector: Vector;
 
     constructor(field, size: number, dataVector: Vector) {
@@ -369,39 +375,35 @@ class FixedSizeListVector extends Vector {
         this.dataVector = dataVector;
     }
 
-    getChildVectors() {
+    public getChildVectors() {
         return [this.dataVector];
     }
 
-    loadBuffers(bb, node, buffers) {
-        // no buffers to load
-    }
-
-    get(i: number) {
+    public get(i: number) {
         return this.dataVector.slice(i * this.size, (i + 1) * this.size);
     }
 
-    slice(start : number, end : number) {
-        var result = [];
-        for (var i = start; i < end; i += 1|0) {
+    public slice(start: number, end: number) {
+        const result = [];
+        for (let i = start; i < end; i++) {
             result.push(this.get(i));
         }
         return result;
     }
 
-    getListSize() {
+    public getListSize() {
         return this.size;
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        // no buffers to load
+    }
 }
 
 class NullableFixedSizeListVector extends FixedSizeListVector {
     private validityView: BitArray;
 
-    loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-    }
-
-    get(i: number) {
+    public get(i: number) {
         if (this.validityView.get(i)) {
             return super.get(i);
         } else {
@@ -409,9 +411,13 @@ class NullableFixedSizeListVector extends FixedSizeListVector {
         }
     }
 
-    getValidityVector() {
+    public getValidityVector() {
         return this.validityView;
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+    }
 }
 
 class StructVector extends Vector {
@@ -423,15 +429,11 @@ class StructVector extends Vector {
         this.vectors = vectors;
     }
 
-    getChildVectors() {
+    public getChildVectors() {
         return this.vectors;
     }
 
-    loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-    }
-
-    get(i : number) {
+    public get(i: number) {
         if (this.validityView.get(i)) {
           return this.vectors.map((v: Vector) => v.get(i));
         } else {
@@ -439,32 +441,35 @@ class StructVector extends Vector {
         }
     }
 
-    slice(start : number, end : number) {
-        var result = [];
-        for (var i = start; i < end; i += 1|0) {
+    public slice(start: number, end: number) {
+        const result = [];
+        for (let i = start; i < end; i++) {
             result.push(this.get(i));
         }
         return result;
     }
 
-    getValidityVector() {
+    public getValidityVector() {
         return this.validityView;
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
+    }
 }
 
 class DictionaryVector extends Vector {
-
     private indices: Vector;
     private dictionary: Vector;
 
-    constructor (field, indices: Vector, dictionary: Vector) {
+    constructor(field, indices: Vector, dictionary: Vector) {
         super(field);
         this.indices = indices;
         this.dictionary = dictionary;
     }
 
-    get(i) {
-        var encoded = this.indices.get(i);
+    public get(i) {
+        const encoded = this.indices.get(i);
         if (encoded == null) {
             return null;
         } else {
@@ -477,18 +482,14 @@ class DictionaryVector extends Vector {
         return this.indices.get(i);
     }
 
-    slice(start, end) {
+    public slice(start, end) {
         return this.indices.slice(start, end); // TODO decode
     }
 
-    getChildVectors() {
+    public getChildVectors() {
         return this.indices.getChildVectors();
     }
 
-    loadBuffers(bb, node, buffers) {
-        this.indices.loadData(bb, node, buffers);
-    }
-
     /** Get the index (encoded) vector */
     public getIndexVector() {
         return this.indices;
@@ -499,91 +500,98 @@ class DictionaryVector extends Vector {
         return this.dictionary;
     }
 
-    toString() {
+    public toString() {
         return this.indices.toString();
     }
+
+    protected loadBuffers(bb, node, buffers) {
+        this.indices.loadData(bb, node, buffers);
+    }
 }
 
-export function vectorFromField(field, dictionaries) : Vector {
-    var dictionary = field.dictionary(), nullable = field.nullable();
+export function vectorFromField(field, dictionaries): Vector {
+    const dictionary = field.dictionary();
+    const nullable = field.nullable();
     if (dictionary == null) {
-        var typeType = field.typeType();
+        const typeType = field.typeType();
         if (typeType === Type.List) {
-            var dataVector = vectorFromField(field.children(0), dictionaries);
+            const dataVector = vectorFromField(field.children(0), dictionaries);
             return nullable ? new NullableListVector(field, dataVector) : new ListVector(field, dataVector);
         } else if (typeType === Type.FixedSizeList) {
-            var dataVector = vectorFromField(field.children(0), dictionaries);
-            var size = field.type(new org.apache.arrow.flatbuf.FixedSizeList()).listSize();
+            const dataVector = vectorFromField(field.children(0), dictionaries);
+            const size = field.type(new org.apache.arrow.flatbuf.FixedSizeList()).listSize();
             if (nullable) {
               return new NullableFixedSizeListVector(field, size, dataVector);
             } else {
               return new FixedSizeListVector(field, size, dataVector);
             }
          } else if (typeType === Type.Struct_) {
-            var vectors : Vector[] = [];
-            for (var i : number = 0; i < field.childrenLength(); i += 1|0) {
+            const vectors: Vector[] = [];
+            for (let i: number = 0; i < field.childrenLength(); i++) {
                 vectors.push(vectorFromField(field.children(i), dictionaries));
             }
             return new StructVector(field, vectors);
         } else {
             if (typeType === Type.Int) {
-                var type = field.type(new org.apache.arrow.flatbuf.Int());
-                return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable)
+                const type = field.type(new org.apache.arrow.flatbuf.Int());
+                return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable);
             } else if (typeType === Type.FloatingPoint) {
-                var precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision();
-                if (precision == org.apache.arrow.flatbuf.Precision.SINGLE) {
+                const precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision();
+                if (precision === org.apache.arrow.flatbuf.Precision.SINGLE) {
                     return nullable ? new NullableFloat32Vector(field) : new Float32Vector(field);
-                } else if (precision == org.apache.arrow.flatbuf.Precision.DOUBLE) {
+                } else if (precision === org.apache.arrow.flatbuf.Precision.DOUBLE) {
                     return nullable ? new NullableFloat64Vector(field) : new Float64Vector(field);
                 } else {
-                    throw "Unimplemented FloatingPoint precision " + precision;
+                    throw new Error("Unimplemented FloatingPoint precision " + precision);
                 }
             } else if (typeType === Type.Utf8) {
                 return nullable ? new NullableUtf8Vector(field) : new Utf8Vector(field);
             } else if (typeType === Type.Date) {
                 return nullable ? new NullableDateVector(field) : new DateVector(field);
             } else {
-                throw "Unimplemented type " + typeType;
+                throw new Error("Unimplemented type " + typeType);
             }
         }
     } else {
         // determine arrow type - default is signed 32 bit int
-        var type = dictionary.indexType(), bitWidth = 32, signed = true;
+        const type = dictionary.indexType();
+        let bitWidth = 32;
+        let signed = true;
         if (type != null) {
             bitWidth = type.bitWidth();
             signed = type.isSigned();
         }
-        var indices = _createIntVector(field, bitWidth, signed, nullable);
+        const indices = _createIntVector(field, bitWidth, signed, nullable);
         return new DictionaryVector(field, indices, dictionaries[dictionary.id().toFloat64().toString()]);
     }
 }
 
 function _createIntVector(field, bitWidth, signed, nullable) {
-    if (bitWidth == 64) {
+    if (bitWidth === 64) {
         if (signed) {
             return nullable ? new NullableInt64Vector(field) : new Int64Vector(field);
         } else {
             return nullable ? new NullableUint64Vector(field) : new Uint64Vector(field);
         }
-    } else if (bitWidth == 32) {
+    } else if (bitWidth === 32) {
         if (signed) {
             return nullable ? new NullableInt32Vector(field) : new Int32Vector(field);
         } else {
             return nullable ? new NullableUint32Vector(field) : new Uint32Vector(field);
         }
-    } else if (bitWidth == 16) {
+    } else if (bitWidth === 16) {
         if (signed) {
             return nullable ? new NullableInt16Vector(field) : new Int16Vector(field);
         } else {
             return nullable ? new NullableUint16Vector(field) : new Uint16Vector(field);
         }
-    } else if (bitWidth == 8) {
+    } else if (bitWidth === 8) {
         if (signed) {
             return nullable ? new NullableInt8Vector(field) : new Int8Vector(field);
         } else {
             return nullable ? new NullableUint8Vector(field) : new Uint8Vector(field);
         }
     } else {
-         throw "Unimplemented Int bit width " + bitWidth;
+         throw new Error("Unimplemented Int bit width " + bitWidth);
     }
 }
diff --git a/js/tslint.json b/js/tslint.json
new file mode 100644
index 0000000000000..5f2f6cdcf36e0
--- /dev/null
+++ b/js/tslint.json
@@ -0,0 +1,12 @@
+{
+    "defaultSeverity": "error",
+    "extends": [
+        "tslint:recommended"
+    ],
+    "jsRules": {},
+    "rules": {
+      "no-bitwise": false,
+      "max-classes-per-file": false
+    },
+    "rulesDirectory": []
+}

From ba97f343f631138a51a1c337eebba83e03add454 Mon Sep 17 00:00:00 2001
From: adeneche <adeneche@dremio.com>
Date: Fri, 2 Jun 2017 10:22:52 -0700
Subject: [PATCH 0690/1644] ARROW-1084: Implementations of BufferAllocator
 should handle Netty's OutOfDirectMemoryError

---
 .../org/apache/arrow/memory/BaseAllocator.java     | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index aaa7ce804c3e5..ddc78f03f0f72 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -22,6 +22,7 @@
 
 import io.netty.buffer.ArrowBuf;
 import io.netty.buffer.UnsafeDirectLittleEndian;
+import io.netty.util.internal.OutOfDirectMemoryError;
 
 import org.apache.arrow.memory.AllocationManager.BufferLedger;
 import org.apache.arrow.memory.util.AssertionUtil;
@@ -266,7 +267,18 @@ public ArrowBuf buffer(final int initialRequestSize, BufferManager manager) {
       success = true;
       listener.onAllocation(actualRequestSize);
       return buffer;
-    } finally {
+    } catch (OutOfMemoryError e) {
+      /*
+       * OutOfDirectMemoryError is thrown by Netty when we exceed the direct memory limit defined by -XX:MaxDirectMemorySize.
+       * OutOfMemoryError with "Direct buffer memory" message is thrown by java.nio.Bits when we exceed the direct memory limit.
+       *   This should never be hit in practice as Netty is expected to throw an OutOfDirectMemoryError first.
+       */
+      if (e instanceof OutOfDirectMemoryError || "Direct buffer memory".equals(e.getMessage())) {
+        throw new OutOfMemoryException(e);
+      }
+      throw e;
+    }
+    finally {
       if (!success) {
         releaseBytes(actualRequestSize);
       }

From 0576ff55ca9376dcb3c169046e22c403624cc57c Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Fri, 2 Jun 2017 16:09:05 -0700
Subject: [PATCH 0691/1644] =?UTF-8?q?ARROW-1085:=20[java]=20Follow=20up=20?=
 =?UTF-8?q?on=20template=20cleanup.=20Missing=20method=20for=20=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…IntervalYear

Author: Julien Le Dem <julien@apache.org>

Closes #726 from julienledem/followup_ARROW-1049 and squashes the following commits:

babe1a5 [Julien Le Dem] ARROW-1085: [java] Follow up on template cleanup. Missing method for IntervalYear
---
 .../vector/src/main/codegen/templates/NullableValueVectors.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index dfd57553acd09..a577d200111bc 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -481,7 +481,7 @@ public void get(int index, Nullable${minor.class}Holder holder){
       }
     }
 
-    <#if minor.class == "Interval" || minor.class == "IntervalDay">
+    <#if minor.class == "IntervalYear" || minor.class == "IntervalDay">
     public StringBuilder getAsStringBuilder(int index) {
       if (isNull(index)) {
           return null;

From 931a877570928486f5471f9850c38d68c1e18617 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 3 Jun 2017 17:14:36 +0200
Subject: [PATCH 0692/1644] ARROW-1070: [C++] Use physical types for Feather
 date/time types

This fixes a regression from Feather 0.3.1. Closes https://github.com/wesm/feather/issues/304

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #725 from wesm/ARROW-1070 and squashes the following commits:

fe1a5ba [Wes McKinney] Use physical types for date/time types
---
 cpp/src/arrow/ipc/feather.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 7d0abdda1aadc..023e7ec6e98b7 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -37,6 +37,7 @@
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace ipc {
@@ -467,16 +468,15 @@ fbs::Type ToFlatbufferType(Type::type type) {
     case Type::BINARY:
       return fbs::Type_BINARY;
     case Type::DATE32:
-      return fbs::Type_DATE;
+      return fbs::Type_INT32;
     case Type::TIMESTAMP:
-      return fbs::Type_TIMESTAMP;
+      return fbs::Type_INT64;
     case Type::TIME32:
-      return fbs::Type_TIME;
+      return fbs::Type_INT32;
     case Type::TIME64:
-      return fbs::Type_TIME;
-    case Type::DICTIONARY:
-      return fbs::Type_CATEGORY;
+      return fbs::Type_INT64;
     default:
+      DCHECK(false) << "Cannot reach this code";
       break;
   }
   // prevent compiler warning

From a81aefbd8cdcc081dbe8669cdaaf459a1c8127b4 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 3 Jun 2017 17:17:41 +0200
Subject: [PATCH 0693/1644] ARROW-1082: [GLib] Add CI on macOS

Author: Kouhei Sutou <kou@clear-code.com>

Closes #722 from kou/glib-travis-macos and squashes the following commits:

1a08aa0 [Kouhei Sutou] [GLib] Don't run Torch example on macOS
651f9ff [Kouhei Sutou] [GLib] Add CI on macOS
---
 .travis.yml                       | 12 ++++++++-
 ci/travis_before_script_c_glib.sh | 42 ++++++++++++++++++++-----------
 ci/travis_env_common.sh           |  4 +++
 ci/travis_script_c_glib.sh        | 19 +++++++++-----
 4 files changed, 56 insertions(+), 21 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 36e3c18d8a98a..7a8b16c2f198b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,6 @@ addons:
     - gtk-doc-tools
     - autoconf-archive
     - libgirepository1.0-dev
-    - gir1.2-gudev-1.0
 services:
   - docker
 
@@ -91,6 +90,17 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+  - compiler: clang
+    osx_image: xcode6.4
+    os: osx
+    cache:
+    addons:
+    rvm: 2.2
+    before_script:
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
 
 before_install:
 - ulimit -c unlimited -S
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index e397b0ecaf837..6387f4ddcc902 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -19,28 +19,38 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
   brew install gtk-doc autoconf-archive gobject-introspection
+  brew upgrade git cmake wget libtool
+
+  export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
 fi
 
-gem install gobject-introspection
-
-git clone \
-  --quiet \
-  --depth 1 \
-  --recursive \
-  https://github.com/torch/distro.git ~/torch
-pushd ~/torch
-./install-deps > /dev/null
-echo "yes" | ./install.sh > /dev/null
-. ~/torch/install/bin/torch-activate
-popd
+gem install test-unit gobject-introspection
+
+if [ $TRAVIS_OS_NAME == "osx" ]; then
+  brew install lua
+else
+  git clone \
+    --quiet \
+    --depth 1 \
+    --recursive \
+    https://github.com/torch/distro.git ~/torch
+  pushd ~/torch
+  ./install-deps > /dev/null
+  echo "yes" | ./install.sh > /dev/null
+  . ~/torch/install/bin/torch-activate
+  popd
+fi
 luarocks install lgi
 
 go get github.com/linuxdeepin/go-gir-generator || :
 pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
+mv Makefile{,.orig}
+sed -e 's/ gudev-1.0//' Makefile.orig > Makefile
+mkdir -p out/src/gir/gudev-1.0
 make build copyfile
 mkdir -p $GOPATH/bin/
 cp -a out/gir-generator $GOPATH/bin/
-cp -a out/src/gir/ $GOPATH/src/
+cp -a out/src/gir/ $GOPATH/src/gir/
 popd
 
 pushd $ARROW_C_GLIB_DIR
@@ -50,7 +60,11 @@ pushd $ARROW_C_GLIB_DIR
 export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
 
-./configure --prefix=${ARROW_C_GLIB_INSTALL} --enable-gtk-doc
+CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"
+if [ $TRAVIS_OS_NAME != "osx" ]; then
+  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"
+fi
+./configure $CONFIGURE_OPTIONS
 
 make -j4
 make install
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index cef2d655cffe0..9ded3e07e7ef7 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -26,3 +26,7 @@ export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
 export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
 export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
 export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
+
+if [ $TRAVIS_OS_NAME == "osx" ]; then
+  export GOPATH=$TRAVIS_BUILD_DIR/gopath
+fi
diff --git a/ci/travis_script_c_glib.sh b/ci/travis_script_c_glib.sh
index c4df713818c37..4bfa0c0af49ca 100755
--- a/ci/travis_script_c_glib.sh
+++ b/ci/travis_script_c_glib.sh
@@ -27,12 +27,19 @@ export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
 export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_C_GLIB_INSTALL/lib/pkgconfig
 
 pushd example/lua
-. ~/torch/install/bin/torch-activate
-luajit write-batch.lua
-luajit read-batch.lua
-luajit write-stream.lua
-luajit read-stream.lua
-luajit stream-to-torch-tensor.lua
+if [ $TRAVIS_OS_NAME == "osx" ]; then
+  lua write-batch.lua
+  lua read-batch.lua
+  lua write-stream.lua
+  lua read-stream.lua
+else
+  . ~/torch/install/bin/torch-activate
+  luajit write-batch.lua
+  luajit read-batch.lua
+  luajit write-stream.lua
+  luajit read-stream.lua
+  luajit stream-to-torch-tensor.lua
+fi
 popd
 
 pushd example/go

From 8f2b44b897b7083ee2a296c70397dc2d7d21d95e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 5 Jun 2017 12:18:32 +0200
Subject: [PATCH 0694/1644] ARROW-1051: [Python] Opt in to Parquet unit tests
 to avoid accidental suppression of dynamic linking errors

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #729 from wesm/ARROW-1051 and squashes the following commits:

019b9ec [Wes McKinney] Statically link boost in parquet-cpp
5103077 [Wes McKinney] See if updating conda helps
7eac948 [Wes McKinney] See if setting PATH solves problem
e246e19 [Wes McKinney] Red herring, issue was runtime library loading
6bc0492 [Wes McKinney] Set PARQUET_ARROW_VERSION in Windows build
a1f2d2b [Wes McKinney] Opt in to Parquet unit tests so that import errors from pyarrow.parquet bubble up
---
 ci/msvc-build.bat                    |  10 +-
 ci/travis_script_python.sh           |   2 +-
 python/pyarrow/tests/conftest.py     |   2 +-
 python/pyarrow/tests/test_parquet.py | 132 +++++++++++++++++----------
 4 files changed, 92 insertions(+), 54 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index d13c11fabcf0e..263d4bc580508 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -17,6 +17,8 @@
 
 @echo on
 
+conda update --yes --quiet conda
+
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython
 conda install -n arrow -q -y -c conda-forge ^
@@ -43,7 +45,7 @@ cmake -G "%GENERATOR%" ^
 cmake --build . --target INSTALL --config Release  || exit /B
 
 @rem Needed so python-test.exe works
-set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%
+set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
 
 ctest -VV  || exit /B
 popd
@@ -59,15 +61,17 @@ set PARQUET_HOME=%CONDA_PREFIX%\Library
 cmake -G "%GENERATOR%" ^
      -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
      -DCMAKE_BUILD_TYPE=Release ^
+     -DPARQUET_BOOST_USE_SHARED=OFF ^
      -DPARQUET_ZLIB_VENDORED=off ^
      -DPARQUET_BUILD_TESTS=off .. || exit /B
 cmake --build . --target INSTALL --config Release || exit /B
 popd
 
 @rem Build and import pyarrow
-set PYTHONPATH=
+@rem parquet-cpp has some additional runtime dependencies that we need to figure out
+@rem see PARQUET-1018
 
 pushd python
 python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel  || exit /B
-py.test pyarrow -v -s || exit /B
+py.test pyarrow -v -s --parquet || exit /B
 popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index c3735ccadac3e..904db52a69a1d 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -111,7 +111,7 @@ python_version_tests() {
   python -c "import pyarrow.parquet"
   python -c "import pyarrow._jemalloc"
 
-  python -m pytest -vv -r sxX pyarrow
+  python -m pytest -vv -r sxX pyarrow --parquet
 
   # Build documentation once
   if [[ "$PYTHON_VERSION" == "3.6" ]]
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index d5b4b69d97392..9b767fc7d52a7 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -18,7 +18,7 @@
 from pytest import skip
 
 
-groups = ['hdfs']
+groups = ['hdfs', 'parquet']
 
 
 def pytest_configure(config):
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 5f65f2800d170..052d395b9811c 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -32,13 +32,20 @@
 
 import pandas.util.testing as tm
 
-# Skip all parquet tests if we can't import pyarrow.parquet
-pq = pytest.importorskip('pyarrow.parquet')
-
 # Ignore these with pytest ... -m 'not parquet'
 parquet = pytest.mark.parquet
 
 
+def _write_table(*args, **kwargs):
+    import pyarrow.parquet as pq
+    return pq.write_table(*args, **kwargs)
+
+
+def _read_table(*args, **kwargs):
+    import pyarrow.parquet as pq
+    return pq.read_table(*args, **kwargs)
+
+
 @parquet
 def test_single_pylist_column_roundtrip(tmpdir):
     for dtype in [int, float]:
@@ -46,8 +53,8 @@ def test_single_pylist_column_roundtrip(tmpdir):
                                .format(dtype.__name__))
         data = [pa.array(list(map(dtype, range(5))))]
         table = pa.Table.from_arrays(data, names=('a', 'b'))
-        pq.write_table(table, filename.strpath)
-        table_read = pq.read_table(filename.strpath)
+        _write_table(table, filename.strpath)
+        table_read = _read_table(filename.strpath)
         for col_written, col_read in zip(table.itercolumns(),
                                          table_read.itercolumns()):
             assert col_written.name == col_read.name
@@ -84,13 +91,14 @@ def alltypes_sample(size=10000, seed=0):
 
 @parquet
 def test_pandas_parquet_2_0_rountrip(tmpdir):
+    import pyarrow.parquet as pq
     df = alltypes_sample(size=10000)
 
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
     assert b'pandas' in arrow_table.schema.metadata
 
-    pq.write_table(arrow_table, filename.strpath, version="2.0")
+    _write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pq.read_pandas(filename.strpath)
     assert b'pandas' in table_read.schema.metadata
 
@@ -102,13 +110,15 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
 
 @parquet
 def test_pandas_parquet_custom_metadata(tmpdir):
+    import pyarrow.parquet as pq
+
     df = alltypes_sample(size=10000)
 
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
     assert b'pandas' in arrow_table.schema.metadata
 
-    pq.write_table(arrow_table, filename.strpath, version="2.0")
+    _write_table(arrow_table, filename.strpath, version="2.0")
     pf = pq.ParquetFile(filename.strpath)
 
     md = pf.metadata.metadata
@@ -120,6 +130,8 @@ def test_pandas_parquet_custom_metadata(tmpdir):
 
 @parquet
 def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tmpdir):
+    import pyarrow.parquet as pq
+
     df = alltypes_sample(size=10000)
 
     filename = tmpdir.join('pandas_rountrip.parquet')
@@ -129,7 +141,7 @@ def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tmpdir):
     js = json.loads(arrow_table.schema.metadata[b'pandas'].decode('utf8'))
     assert not js['index_columns']
 
-    pq.write_table(arrow_table, filename.strpath, version="2.0")
+    _write_table(arrow_table, filename.strpath, version="2.0")
     table_read = pq.read_pandas(filename.strpath)
 
     js = json.loads(table_read.schema.metadata[b'pandas'].decode('utf8'))
@@ -163,8 +175,8 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = pa.Table.from_pandas(df)
-    pq.write_table(arrow_table, filename.strpath, version="1.0")
-    table_read = pq.read_table(filename.strpath)
+    _write_table(arrow_table, filename.strpath, version="1.0")
+    table_read = _read_table(filename.strpath)
     df_read = table_read.to_pandas()
 
     # We pass uint32_t as int64_t if we write Parquet version 1.0
@@ -183,8 +195,8 @@ def test_pandas_column_selection(tmpdir):
     })
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = pa.Table.from_pandas(df)
-    pq.write_table(arrow_table, filename.strpath)
-    table_read = pq.read_table(filename.strpath, columns=['uint8'])
+    _write_table(arrow_table, filename.strpath)
+    table_read = _read_table(filename.strpath, columns=['uint8'])
     df_read = table_read.to_pandas()
 
     tm.assert_frame_equal(df[['uint8']], df_read)
@@ -223,19 +235,21 @@ def test_pandas_parquet_native_file_roundtrip(tmpdir):
     df = _test_dataframe(10000)
     arrow_table = pa.Table.from_pandas(df)
     imos = pa.BufferOutputStream()
-    pq.write_table(arrow_table, imos, version="2.0")
+    _write_table(arrow_table, imos, version="2.0")
     buf = imos.get_result()
     reader = pa.BufferReader(buf)
-    df_read = pq.read_table(reader).to_pandas()
+    df_read = _read_table(reader).to_pandas()
     tm.assert_frame_equal(df, df_read)
 
 
 @parquet
 def test_read_pandas_column_subset(tmpdir):
+    import pyarrow.parquet as pq
+
     df = _test_dataframe(10000)
     arrow_table = pa.Table.from_pandas(df)
     imos = pa.BufferOutputStream()
-    pq.write_table(arrow_table, imos, version="2.0")
+    _write_table(arrow_table, imos, version="2.0")
     buf = imos.get_result()
     reader = pa.BufferReader(buf)
     df_read = pq.read_pandas(reader, columns=['strings', 'uint8']).to_pandas()
@@ -257,11 +271,11 @@ def test_pandas_parquet_pyfile_roundtrip(tmpdir):
     arrow_table = pa.Table.from_pandas(df)
 
     with open(filename, 'wb') as f:
-        pq.write_table(arrow_table, f, version="1.0")
+        _write_table(arrow_table, f, version="1.0")
 
     data = io.BytesIO(open(filename, 'rb').read())
 
-    table_read = pq.read_table(data)
+    table_read = _read_table(data)
     df_read = table_read.to_pandas()
     tm.assert_frame_equal(df, df_read)
 
@@ -287,27 +301,29 @@ def test_pandas_parquet_configuration_options(tmpdir):
     arrow_table = pa.Table.from_pandas(df)
 
     for use_dictionary in [True, False]:
-        pq.write_table(arrow_table, filename.strpath,
-                       version="2.0",
-                       use_dictionary=use_dictionary)
-        table_read = pq.read_table(filename.strpath)
+        _write_table(arrow_table, filename.strpath,
+                     version="2.0",
+                     use_dictionary=use_dictionary)
+        table_read = _read_table(filename.strpath)
         df_read = table_read.to_pandas()
         tm.assert_frame_equal(df, df_read)
 
     for compression in ['NONE', 'SNAPPY', 'GZIP']:
-        pq.write_table(arrow_table, filename.strpath,
-                       version="2.0",
-                       compression=compression)
-        table_read = pq.read_table(filename.strpath)
+        _write_table(arrow_table, filename.strpath,
+                     version="2.0",
+                     compression=compression)
+        table_read = _read_table(filename.strpath)
         df_read = table_read.to_pandas()
         tm.assert_frame_equal(df, df_read)
 
 
 def make_sample_file(df):
+    import pyarrow.parquet as pq
+
     a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
-    pq.write_table(a_table, buf, compression='SNAPPY', version='2.0')
+    _write_table(a_table, buf, compression='SNAPPY', version='2.0')
 
     buf.seek(0)
     return pq.ParquetFile(buf)
@@ -384,8 +400,8 @@ def test_column_of_arrays(tmpdir):
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True,
                                        schema=schema)
-    pq.write_table(arrow_table, filename.strpath, version="2.0")
-    table_read = pq.read_table(filename.strpath)
+    _write_table(arrow_table, filename.strpath, version="2.0")
+    table_read = _read_table(filename.strpath)
     df_read = table_read.to_pandas()
     tm.assert_frame_equal(df, df_read)
 
@@ -397,8 +413,8 @@ def test_column_of_lists(tmpdir):
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True,
                                        schema=schema)
-    pq.write_table(arrow_table, filename.strpath, version="2.0")
-    table_read = pq.read_table(filename.strpath)
+    _write_table(arrow_table, filename.strpath, version="2.0")
+    table_read = _read_table(filename.strpath)
     df_read = table_read.to_pandas()
     tm.assert_frame_equal(df, df_read)
 
@@ -449,7 +465,7 @@ def _assert_unsupported(array):
         buf = io.BytesIO()
 
         with pytest.raises(NotImplementedError):
-            pq.write_table(table, buf, version="2.0")
+            _write_table(table, buf, version="2.0")
 
     t7 = pa.time64('ns')
     a7 = pa.Array.from_pandas(data4.astype('int64'), type=t7)
@@ -470,13 +486,13 @@ def test_fixed_size_binary():
 
 def _check_roundtrip(table, expected=None, **params):
     buf = io.BytesIO()
-    pq.write_table(table, buf, **params)
+    _write_table(table, buf, **params)
     buf.seek(0)
 
     if expected is None:
         expected = table
 
-    result = pq.read_table(buf)
+    result = _read_table(buf)
     assert result.equals(expected)
 
 
@@ -487,13 +503,13 @@ def test_multithreaded_read():
     table = pa.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
-    pq.write_table(table, buf, compression='SNAPPY', version='2.0')
+    _write_table(table, buf, compression='SNAPPY', version='2.0')
 
     buf.seek(0)
-    table1 = pq.read_table(buf, nthreads=4)
+    table1 = _read_table(buf, nthreads=4)
 
     buf.seek(0)
-    table2 = pq.read_table(buf, nthreads=1)
+    table2 = _read_table(buf, nthreads=1)
 
     assert table1.equals(table2)
 
@@ -504,26 +520,28 @@ def test_min_chunksize():
     table = pa.Table.from_pandas(data.reset_index())
 
     buf = io.BytesIO()
-    pq.write_table(table, buf, chunk_size=-1)
+    _write_table(table, buf, chunk_size=-1)
 
     buf.seek(0)
-    result = pq.read_table(buf)
+    result = _read_table(buf)
 
     assert result.equals(table)
 
     with pytest.raises(ValueError):
-        pq.write_table(table, buf, chunk_size=0)
+        _write_table(table, buf, chunk_size=0)
 
 
 @parquet
 def test_pass_separate_metadata():
+    import pyarrow.parquet as pq
+
     # ARROW-471
     df = alltypes_sample(size=10000)
 
     a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
-    pq.write_table(a_table, buf, compression='snappy', version='2.0')
+    _write_table(a_table, buf, compression='snappy', version='2.0')
 
     buf.seek(0)
     metadata = pq.ParquetFile(buf).metadata
@@ -537,6 +555,8 @@ def test_pass_separate_metadata():
 
 @parquet
 def test_read_single_row_group():
+    import pyarrow.parquet as pq
+
     # ARROW-471
     N, K = 10000, 4
     df = alltypes_sample(size=N)
@@ -544,8 +564,8 @@ def test_read_single_row_group():
     a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
-    pq.write_table(a_table, buf, row_group_size=N / K,
-                   compression='snappy', version='2.0')
+    _write_table(a_table, buf, row_group_size=N / K,
+                 compression='snappy', version='2.0')
 
     buf.seek(0)
 
@@ -560,13 +580,15 @@ def test_read_single_row_group():
 
 @parquet
 def test_read_single_row_group_with_column_subset():
+    import pyarrow.parquet as pq
+
     N, K = 10000, 4
     df = alltypes_sample(size=N)
     a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
 
     buf = io.BytesIO()
-    pq.write_table(a_table, buf, row_group_size=N / K,
-                   compression='snappy', version='2.0')
+    _write_table(a_table, buf, row_group_size=N / K,
+                 compression='snappy', version='2.0')
 
     buf.seek(0)
     pf = pq.ParquetFile(buf)
@@ -579,11 +601,13 @@ def test_read_single_row_group_with_column_subset():
 
 @parquet
 def test_parquet_piece_read(tmpdir):
+    import pyarrow.parquet as pq
+
     df = _test_dataframe(1000)
     table = pa.Table.from_pandas(df)
 
     path = tmpdir.join('parquet_piece_read.parquet').strpath
-    pq.write_table(table, path, version='2.0')
+    _write_table(table, path, version='2.0')
 
     piece1 = pq.ParquetDatasetPiece(path)
 
@@ -593,6 +617,8 @@ def test_parquet_piece_read(tmpdir):
 
 @parquet
 def test_parquet_piece_basics():
+    import pyarrow.parquet as pq
+
     path = '/baz.parq'
 
     piece1 = pq.ParquetDatasetPiece(path)
@@ -612,6 +638,8 @@ def test_parquet_piece_basics():
 
 @parquet
 def test_partition_set_dictionary_type():
+    import pyarrow.parquet as pq
+
     set1 = pq.PartitionSet('key1', [u('foo'), u('bar'), u('baz')])
     set2 = pq.PartitionSet('key2', [2007, 2008, 2009])
 
@@ -625,6 +653,8 @@ def test_partition_set_dictionary_type():
 
 @parquet
 def test_read_partitioned_directory(tmpdir):
+    import pyarrow.parquet as pq
+
     foo_keys = [0, 1]
     bar_keys = ['a', 'b', 'c']
     partition_spec = [
@@ -681,7 +711,7 @@ def _visit_level(base_dir, level, part_keys):
 
                 filtered_df = _filter_partition(df, this_part_keys)
                 part_table = pa.Table.from_pandas(filtered_df)
-                pq.write_table(part_table, file_path)
+                _write_table(part_table, file_path)
             else:
                 _visit_level(level_dir, level + 1, this_part_keys)
 
@@ -690,6 +720,8 @@ def _visit_level(base_dir, level, part_keys):
 
 @parquet
 def test_read_common_metadata_files(tmpdir):
+    import pyarrow.parquet as pq
+
     N = 100
     df = pd.DataFrame({
         'index': np.arange(N),
@@ -700,7 +732,7 @@ def test_read_common_metadata_files(tmpdir):
     data_path = pjoin(base_path, 'data.parquet')
 
     table = pa.Table.from_pandas(df)
-    pq.write_table(table, data_path)
+    _write_table(table, data_path)
 
     metadata_path = pjoin(base_path, '_metadata')
     pq.write_metadata(table.schema, metadata_path)
@@ -729,6 +761,8 @@ def _filter_partition(df, part_keys):
 
 @parquet
 def test_read_multiple_files(tmpdir):
+    import pyarrow.parquet as pq
+
     nfiles = 10
     size = 5
 
@@ -746,7 +780,7 @@ def test_read_multiple_files(tmpdir):
         path = pjoin(dirpath, '{0}.parquet'.format(i))
 
         table = pa.Table.from_pandas(df)
-        pq.write_table(table, path)
+        _write_table(table, path)
 
         test_data.append(table)
         paths.append(path)
@@ -792,7 +826,7 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
     bad_apple_path = tmpdir.join('{0}.parquet'.format(guid())).strpath
 
     t = pa.Table.from_pandas(bad_apple)
-    pq.write_table(t, bad_apple_path)
+    _write_table(t, bad_apple_path)
 
     bad_meta = pq.ParquetFile(bad_apple_path).metadata
 

From a44155d6ec5d0c6c255d3305a494f51a6b1d2316 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 5 Jun 2017 12:20:35 +0200
Subject: [PATCH 0695/1644] ARROW-986: [Format] Add brief explanation of
 dictionary batches in IPC.md

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #732 from wesm/ARROW-986 and squashes the following commits:

4321106 [Wes McKinney] Add brief explanation of dictionary batches in IPC.md
---
 format/IPC.md | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/format/IPC.md b/format/IPC.md
index bf2aaa74b3b46..7d689216d5595 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -157,9 +157,24 @@ Some notes about this
 
 ### Dictionary Batches
 
-Dictionary batches have not yet been implemented, while they are provided for
-in the metadata. For the time being, the `DICTIONARY` segments shown above in
-the file do not appear in any of the file implementations.
+Dictionaries are written in the stream and file formats as a sequence of record
+batches, each having a single field. The complete semantic schema for a
+sequence of record batches, therefore, consists of the schema along with all of
+the dictionaries. The dictionary types are found in the schema, so it is
+necessary to read the schema to first determine the dictionary types so that
+the dictionaries can be properly interpreted.
+
+```
+table DictionaryBatch {
+  id: long;
+  data: RecordBatch;
+}
+```
+
+The dictionary `id` in the message metadata can be referenced one or more times
+in the schema, so that dictionaries can even be used for multiple fields. See
+the [Physical Layout][4] document for more about the semantics of
+dictionary-encoded data.
 
 ### Tensor (Multi-dimensional Array) Message Format
 
@@ -182,3 +197,4 @@ shared memory region) to be a multiple of 8:
 [1]: https://github.com/apache/arrow/blob/master/format/File.fbs
 [2]: https://github.com/apache/arrow/blob/master/format/Message.fbs
 [3]: https://github.com/google]/flatbuffers
+[4]: https://github.com/apache/arrow/blob/master/format/Layout.md

From cfaddabecd104595acf043e92ba8f3cb0841b278 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 5 Jun 2017 12:22:38 +0200
Subject: [PATCH 0696/1644] ARROW-1050: [C++] Export arrow::ValidateArray

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #730 from wesm/ARROW-1050 and squashes the following commits:

c50dd5a [Wes McKinney] Export arrow::ValidateArray
---
 cpp/src/arrow/array.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 2c96ce051474e..1c9769fff4615 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -515,7 +515,7 @@ ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
 ///
 /// \param array an Array instance
 /// \return Status
-Status ValidateArray(const Array& array);
+Status ARROW_EXPORT ValidateArray(const Array& array);
 
 }  // namespace arrow
 

From 316930cf6ce654aae042ee5ef269e6fef050a791 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 5 Jun 2017 12:26:13 +0200
Subject: [PATCH 0697/1644] ARROW-1056: [Python] Ignore pandas index in
 parquet+hdfs test

I have all the tests passing again:

```shell
$ py.test pyarrow --hdfs
===================================== test session starts =====================================
platform linux -- Python 3.5.1, pytest-3.0.6, py-1.4.31, pluggy-0.4.0
rootdir: /home/wesm/code/arrow/python, inifile: setup.cfg
collected 227 items

pyarrow/tests/test_array.py ...........
pyarrow/tests/test_convert_builtin.py ......................
pyarrow/tests/test_convert_pandas.py ............................x....
pyarrow/tests/test_deprecations.py ..
pyarrow/tests/test_feather.py .......................x....
pyarrow/tests/test_hdfs.py ...............
pyarrow/tests/test_io.py ..................
pyarrow/tests/test_ipc.py .............x
pyarrow/tests/test_jemalloc.py ..
pyarrow/tests/test_parquet.py ...........................
pyarrow/tests/test_scalars.py ..........
pyarrow/tests/test_schema.py ..............
pyarrow/tests/test_table.py ...............
pyarrow/tests/test_tensor.py ................
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #728 from wesm/ARROW-1056 and squashes the following commits:

aa3d468 [Wes McKinney] Ignore pandas index in parquet+hdfs test
---
 python/pyarrow/tests/test_hdfs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index d2a54790668d5..cea02fbecc75b 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -167,7 +167,7 @@ def test_hdfs_read_multiple_parquet_files(self):
 
             path = pjoin(tmpdir, '{0}.parquet'.format(i))
 
-            table = pa.Table.from_pandas(df)
+            table = pa.Table.from_pandas(df, preserve_index=False)
             with self.hdfs.open(path, 'wb') as f:
                 pq.write_table(table, f)
 

From 4e134e5b9e7f093d7483f027a3de260c3c38c961 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Mon, 5 Jun 2017 16:46:30 -0700
Subject: [PATCH 0698/1644] ARROW-1091: Decimal scale and precision are flipped

Author: Julien Le Dem <julien@apache.org>

Closes #735 from julienledem/template_cleanup_followup and squashes the following commits:

e968a68 [Julien Le Dem] add test
2911ea2 [Julien Le Dem] ARROW-1091: Decimal scale and precision are flipped
---
 java/vector/src/main/codegen/data/ValueVectorTypes.tdd      | 2 +-
 .../src/main/codegen/templates/NullableValueVectors.java    | 6 +-----
 .../java/org/apache/arrow/vector/TestDecimalVector.java     | 3 +++
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index 970d887c76003..b147c9e001f46 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -116,7 +116,7 @@
         {
           class: "Decimal",
           maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal",
-          typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}],
+          typeParams: [ {name: "precision", type: "int"}, { name: "scale", type: "int"}],
           arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal",
           fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}]
         }
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index a577d200111bc..36f8338605c4c 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -593,7 +593,6 @@ public void setSkipNull(int index, Nullable${minor.class}Holder holder){
       values.getMutator().set(index, holder);
     }
 
-
     public void set(int index, Nullable${minor.class}Holder holder){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
@@ -666,11 +665,8 @@ public void setSafe(int index, ${minor.class}Holder value) {
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
 
-    <#if !(type.major == "VarLen" || minor.class == "IntervalYear" || minor.class == "IntervalDay")>
+    <#if !(type.major == "VarLen" || minor.class == "IntervalDay")>
     public void setSafe(int index, ${minor.javaType!type.javaType} value) {
-      <#if type.major == "VarLen">
-      fillEmpties(index);
-      </#if>
       bits.getMutator().setSafeToOne(index);
       values.getMutator().setSafe(index, value);
       setCount++;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index ee7530c8d1085..b98c24d189528 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -45,6 +45,9 @@ public class TestDecimalVector {
   public void test() {
     BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
     NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(10, scale), allocator);
+    try (NullableDecimalVector oldConstructor = new NullableDecimalVector("decimal", allocator, 10, scale);) {
+      assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType());
+    }
     decimalVector.allocateNew();
     BigDecimal[] values = new BigDecimal[intValues.length];
     for (int i = 0; i < intValues.length; i++) {

From a367fd437b1fe7797a27be578435abfacf94d052 Mon Sep 17 00:00:00 2001
From: Jacob Scott <jhscott@infer.com>
Date: Tue, 6 Jun 2017 14:18:05 -0400
Subject: [PATCH 0699/1644] ARROW-1086: include additional pxd files during
 package build

Since this is purely a packaging change, I believe I would need to change`travis_script_python.sh` in order test it (build a wheel, set up a new conda environment and install the wheel, etc). Not sure if that's worth doing? Happy to take a stab if it is.

Author: Jacob Scott <jhscott@infer.com>

Closes #733 from snakescott/ARROW-1086 and squashes the following commits:

6c3e8598 [Jacob Scott] [ARROW-1086] Add more pxd files to package_data
---
 python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index b64e2591298a5..817c59f87a967 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -349,7 +349,7 @@ def has_ext_modules(foo):
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
     zip_safe=False,
-    package_data={'pyarrow': ['*.pxd', '*.pyx']},
+    package_data={'pyarrow': ['*.pxd', '*.pyx', 'includes/*.pxd']},
     include_package_data=True,
     distclass=BinaryDistribution,
     # Dummy extension to trigger build_ext

From 44dba7176f078a297b327183593704f939df52a7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 6 Jun 2017 14:20:06 -0400
Subject: [PATCH 0700/1644] ARROW-1020: [Format] Revise language for Timestamp
 type in Schema.fbs to avoid possible confusion about tz-naive timestamps

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #731 from wesm/ARROW-1020 and squashes the following commits:

5466f417 [Wes McKinney] Revise language for Timestamp type in Schema.fbs to avoid possible confusion about time zone naive timestamps (they are not necessarily normalized to UTC)
---
 format/Schema.fbs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/format/Schema.fbs b/format/Schema.fbs
index 8de5c6d466c36..a7e802b9dcba6 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -113,7 +113,12 @@ table Time {
   bitWidth: int = 32;
 }
 
-/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
+/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding
+/// leap seconds, as a 64-bit integer. Note that UNIX time does not include
+/// leap seconds.
+///
+/// The Timestamp metadata supports both "time zone naive" and "time zone
+/// aware" timestamps. Read about the timezone attribute for more detail
 table Timestamp {
   unit: TimeUnit;
 

From 1a72acdf4daa68fe6e7639acde08a0abf795a3fc Mon Sep 17 00:00:00 2001
From: Tony Kelman <tony@kelman.net>
Date: Wed, 24 May 2017 02:16:58 -0400
Subject: [PATCH 0701/1644] [Doc] Fix a few links for files moved in ARROW-957

Closes #714

Change-Id: Ic58e7ad6b0b2cf5a7d4d36859bdbffda65e13f79
---
 cpp/README.md                                  | 2 +-
 python/doc/source/development.rst              | 2 +-
 site/_posts/2017-05-08-0.3-release-japanese.md | 2 +-
 site/_posts/2017-05-08-0.3-release.md          | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 69c695020add5..5bb516fc99b5d 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -126,4 +126,4 @@ both of these options would be used rarely.  Current known uses-cases whent hey
 *  Parameterized tests in google test.
 
 [1]: https://brew.sh/
-[2]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
+[2]: https://github.com/apache/arrow/blob/master/cpp/apidoc/Windows.md
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 6ec563ed3ce36..410f838900ed1 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -175,7 +175,7 @@ Windows
 =======
 
 First, we bootstrap a conda environment similar to the `C++ build instructions
-<https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md>`_. This
+<https://github.com/apache/arrow/blob/master/cpp/apidoc/Windows.md>`_. This
 includes all the dependencies for Arrow and the Apache Parquet C++ libraries.
 
 First, starting from fresh clones of Apache Arrow and parquet-cpp:
diff --git a/site/_posts/2017-05-08-0.3-release-japanese.md b/site/_posts/2017-05-08-0.3-release-japanese.md
index e49bde1725957..6bad5adea9286 100644
--- a/site/_posts/2017-05-08-0.3-release-japanese.md
+++ b/site/_posts/2017-05-08-0.3-release-japanese.md
@@ -162,7 +162,7 @@ $ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
 [4]: http://wesmckinney.com/blog/arrow-streaming-columnar/
 [5]: http://github.com/google/flatbuffers
 [6]: http://arrow.apache.org/install
-[7]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
+[7]: https://github.com/apache/arrow/blob/master/cpp/apidoc/Windows.md
 [8]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst
 [9]: http://github.com/kou
 [10]: https://wiki.gnome.org/Projects/GObjectIntrospection
diff --git a/site/_posts/2017-05-08-0.3-release.md b/site/_posts/2017-05-08-0.3-release.md
index 39a907d7ed645..cf38a8ba844e1 100644
--- a/site/_posts/2017-05-08-0.3-release.md
+++ b/site/_posts/2017-05-08-0.3-release.md
@@ -238,7 +238,7 @@ $ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
 [4]: http://wesmckinney.com/blog/arrow-streaming-columnar/
 [5]: http://github.com/google/flatbuffers
 [6]: http://arrow.apache.org/install
-[7]: https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md
+[7]: https://github.com/apache/arrow/blob/master/cpp/apidoc/Windows.md
 [8]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst
 [9]: http://github.com/kou
 [10]: https://wiki.gnome.org/Projects/GObjectIntrospection

From 5589dda50de923abb0ab22b8ca30edb912b992ab Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 6 Jun 2017 14:23:31 -0400
Subject: [PATCH 0702/1644] ARROW-1080: C++: Add tutorial about converting
 to/from row-wise representation

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #721 from xhochy/ARROW-1080 and squashes the following commits:

556a37b2 [Uwe L. Korn] Fix chunk indices
e7ace72b [Uwe L. Korn] ARROW-1080: C++: Add tutorial about converting to/from row-wise representation
---
 cpp/apidoc/index.md                         |   2 +
 cpp/apidoc/tutorials/row_wise_conversion.md | 187 ++++++++++++++++++++
 2 files changed, 189 insertions(+)
 create mode 100644 cpp/apidoc/tutorials/row_wise_conversion.md

diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
index 923fc5b35bd24..4004e1ef42ea9 100644
--- a/cpp/apidoc/index.md
+++ b/cpp/apidoc/index.md
@@ -32,6 +32,8 @@ Table of Contents
 
  * Instructions on how to build Arrow C++ on [Windows](Windows.md)
  * How to access [HDFS](HDFS.md)
+ * Tutorials
+   * [Convert a vector of row-wise data into an Arrow table](tutorials/row_wise_conversion.md)
 
 Getting Started
 ---------------
diff --git a/cpp/apidoc/tutorials/row_wise_conversion.md b/cpp/apidoc/tutorials/row_wise_conversion.md
new file mode 100644
index 0000000000000..1486fc2a4e041
--- /dev/null
+++ b/cpp/apidoc/tutorials/row_wise_conversion.md
@@ -0,0 +1,187 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+Convert a vector of row-wise data into an Arrow table
+=====================================================
+
+While we want to use columnar data structures to build efficient operations, we
+often receive data in a row-wise fashion from other systems. In the following,
+we want give a brief introduction into the classes provided by Apache Arrow by
+showing how to transform row-wise data into a columnar table.
+
+The data in this example is stored in the following struct:
+
+```
+struct data_row {
+    int64_t id;
+    double cost;
+    std::vector<double> cost_components;
+};
+
+std::vector<data_row> rows;
+```
+
+The final representation should be an `arrow::Table` which in turn is made up of
+an `arrow::Schema` and a list of `arrow::Column`. An `arrow::Column` is again a
+named collection of one or more `arrow::Array` instances. As the first step, we
+will iterate over the data and build up the arrays incrementally. For this task,
+we provide `arrow::ArrayBuilder` classes that help in the construction of the
+final `arrow::Array` instances.
+
+For each type, Arrow has a specially typed builder class. For the primitive
+values `id` and `cost` we can use the respective `arrow::Int64Builder` and
+`arrow::DoubleBuilder`. For the `cost_components` vector, we need to have two
+builders, a top-level `arrow::ListBuilder` that builds the array of offsets and
+a nested `arrow::DoubleBuilder` that constructs the underlying values array that
+is referenced by the offsets in the former array.
+
+```
+// The builders are more efficient using
+// arrow::jemalloc::MemoryPool::default_pool() as this can increase the size of
+// the underlying memory regions in-place. At the moment, arrow::jemalloc is only
+// supported on Unix systems, not Windows.
+
+arrow::Int64Builder id_builder(arrow::default_memory_pool());
+arrow::DoubleBuilder cost_builder(arrow::default_memory_pool());
+std::shared_ptr<DoubleBuilder> components_values_builder =
+    std::make_shared<DoubleBuilder>(arrow::default_memory_pool());
+arrow::ListBuilder components_builder(arrow::default_memory_pool(),
+    components_values_builder);
+```
+
+Now we can loop over our existing data and insert it into the builders. The
+`Append` calls here may fail (e.g. we cannot allocate enough additional memory).
+Thus we need to check their return values. For more information on these values,
+check the documentation about `arrow::Status`.
+
+```
+for (const data_row& row : rows) {
+    ARROW_RETURN_NOT_OK(id_builder.Append(row.id));
+    ARROW_RETURN_NOT_OK(cost_builder.Append(row.cost));
+
+    // Indicate the start of a new list row. This will memorise the current
+    // offset in the values builder.
+    ARROW_RETURN_NOT_OK(components_builder.Append());
+    // Store the actual values. The final nullptr argument tells the underyling
+    // builder that all added values are valid, i.e. non-null.
+    ARROW_RETURN_NOT_OK(components_values_builder->Append(
+        row.cost_components.data(), row.cost_components.size(),
+        nullptr);
+}
+```
+
+At the end, we finalise the arrays, declare the (type) schema and combine them
+ into a single `arrow::Table`:
+
+```
+std::shared_ptr<arrow::Array> id_array;
+ARROW_RETURN_NOT_OK(id_builder.Finish(&id_array));
+std::shared_ptr<arrow::Array> cost_array;
+ARROW_RETURN_NOT_OK(cost_builder.Finish(&cost_array));
+std::shared_ptr<arrow::Array> cost_components_array;
+ARROW_RETURN_NOT_OK(components_builder.Finish(&cost_components_array));
+
+std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
+    arrow::field("id", arrow::int64()),
+    arrow::field("cost", arrow::float64()),
+    arrow::field("cost_components", arrow::list(arrow::float64()))
+};
+auto schema = std::make_shared<arrow::Schema>(schema_vector);
+
+std::shared_ptr<arrow::Table> table;
+ARROW_RETURN_NOT_OK(MakeTable(schema,
+    {id_array, cost_array, cost_components_array}, &table));
+```
+
+The final `table` variable is the one we then can pass on to other functions
+that can consume Apache Arrow memory structures. This object has ownership of
+all referenced data, thus we don't have to care about undefined references once
+we leave the scope of the function building the table and its underlying arrays.
+
+<!-- TODO: Add an example with nullable entries -->
+
+Converting an Arrow Table back into row-wise representation
+===========================================================
+
+To convert an Arrow table back into the same row-wise representation as in the
+above section, we first will check that the table conforms to our expected
+schema and then will build up the vector of rows incrementally.
+
+For the check if the table is as expected, we can utilise solely its schema. 
+
+```
+// This is our input that was passed in from the outside.
+std::shared_ptr<arrow::Table> table;
+
+std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
+    arrow::field("id", arrow::int64()),
+    arrow::field("cost", arrow::float64()),
+    arrow::field("cost_components", arrow::list(arrow::float64()))
+};
+auto expected_schema = std::make_shared<arrow::Schema>(schema_vector);
+
+if (!expected_schema->Equals(*table->schema())) {
+    // The table doesn't have the expected schema thus we cannot directly
+    // convert it to our target representation.
+    // TODO: Implement your custom error handling logic here.
+}
+```
+
+As we have ensured that the table has the expected structure, we can unpack the
+underlying arrays. For the primitive columns `id` and `cost` we can use the high
+level functions to get the values whereas for the nested column
+`cost_components` we need to access the C-pointer to the data to copy its
+contents into the resulting `std::vector<double>`. Here we need to be care to
+also add the offset to the pointer. This offset is needed to enable zero-copy
+slicing operations. While this could be adjusted automatically for double
+arrays, this cannot be done for the accompanying bitmap as often the slicing
+border would be inside a byte.
+
+```
+// For simplicity, we assume that all arrays consist of a single chunk here.
+// In a productive implementation this should either be explicitly check or code
+// added that can treat chunked arrays.
+
+auto ids = std::static_pointer_cast<arrow::Int64Array>(
+    table->column(0)->data()->chunk(0));
+auto costs = std::static_pointer_cast<arrow::DoubleArray(
+    table->column(1)->data()->chunk(0));
+auto cost_components = std::static_pointer_cast<arrow::ListArray(
+    table->column(2)->data()->chunk(0));
+auto cost_components_values = std::static_pointer_cast<arrow::DoubleArray>(
+    cost_components->values());
+// To enable zero-copy slices, the native values pointer might need to account
+// for this slicing offset. This is not needed for the higher level functions
+// like Value(…) that already account for this offset internally.
+const double* cost_components_values_ptr = cost_components_values->data()
+    + cost_components_values->offset();
+```
+
+After we have unpacked the arrays from the table, we can iterate over them in a
+row-wise fashion and fill our target, row-wise representation.
+
+```
+std::vector<data_row> rows;
+
+for (int64_t i = 0; i < table->num_rows(); i++) {
+    // Another simplification in this example is that we assume that there are
+    // no null entries, e.g. each row is fill with valid values.
+    int64_t id = ids->Value(i);
+    double cost = costs->Value(i);
+    const double* first = cost_components_values_ptr + cost_components->value_offset(i);
+    const double* last = cost_components_values_ptr + cost_components->value_offset(i + 1);
+    std::vector<double> components_vec(first, last);
+    rows.push_back({id, cost, components_vec});
+}
+```

From c3e865d5f67fe9059e80e6b59b2b20bd42e7bcc4 Mon Sep 17 00:00:00 2001
From: Jacob Scott <jhscott@infer.com>
Date: Tue, 6 Jun 2017 14:26:36 -0400
Subject: [PATCH 0703/1644] ARROW-1090: Improve build_ext usability with
 --bundle-arrow-cpp

This is based on a blackbox understanding of the build toolchain. It might be cleaner to update `FindArrow.cmake`, but I'm not sure what the consequences of that would be. This shouldn't be able to do much harm, since this codepath would previously crash.

Author: Jacob Scott <jhscott@infer.com>

Closes #734 from snakescott/ARROW-1090 and squashes the following commits:

642f9404 [Jacob Scott] Do not attempt to fix cmake on windows
39c8df62 [Jacob Scott] [ARROW-1090] Add doc with instructions for building a wheel
9c1d9c2e [Jacob Scott] [ARROW-1090] Improve build_ext usability with --bundle-arrow-cpp
---
 python/doc/source/development.rst | 8 ++++++++
 python/setup.py                   | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 410f838900ed1..cd2440610eb71 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -171,6 +171,14 @@ You should be able to run the unit tests with:
 
    ====================== 181 passed, 17 skipped in 0.98 seconds ===========
 
+You can build a wheel by running:
+
+.. code-block:: shell
+   python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
+          --with-parquet --with-jemalloc --bundle-arrow-cpp bdist_wheel
+
+Again, if you did not build parquet-cpp, you should omit ``--with-parquet``.
+
 Windows
 =======
 
diff --git a/python/setup.py b/python/setup.py
index 817c59f87a967..e10b4b84b01f5 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -148,6 +148,11 @@ def _run_cmake(self):
 
         if self.bundle_arrow_cpp:
             cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
+            # ARROW-1090: work around CMake rough edges
+            if 'ARROW_HOME' in os.environ and sys.platform != 'win32':
+                os.environ['PKG_CONFIG_PATH'] = pjoin(os.environ['ARROW_HOME'], 'lib', 'pkgconfig')
+                del os.environ['ARROW_HOME']
+
 
         cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                              .format(self.build_type.lower()))

From 402baa4ec391b61dd37c770ae7978d51b9b550fa Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Tue, 6 Jun 2017 11:32:39 -0700
Subject: [PATCH 0704/1644] ARROW-1092: More Decimal and scale flipped
 follow-up

Author: Julien Le Dem <julien@apache.org>

Closes #736 from julienledem/ARROW-1092 and squashes the following commits:

1bc1ade [Julien Le Dem] ARROW-1092: More Decimal and scale flipped follow-up
---
 .../src/main/codegen/data/ValueVectorTypes.tdd     |  2 +-
 .../main/codegen/templates/FixedValueVectors.java  |  9 +++++----
 .../main/codegen/templates/HolderReaderImpl.java   |  2 +-
 .../src/main/codegen/templates/MapWriters.java     |  2 +-
 .../codegen/templates/NullableValueVectors.java    | 14 ++++++++------
 5 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index b147c9e001f46..970d887c76003 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -116,7 +116,7 @@
         {
           class: "Decimal",
           maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal",
-          typeParams: [ {name: "precision", type: "int"}, { name: "scale", type: "int"}],
+          typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}],
           arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal",
           fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}]
         }
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 395417b1f6bc7..5d92cd232efb3 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -56,13 +56,14 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
   private int allocationMonitor = 0;
   <#if minor.typeParams??>
 
-    <#list minor.typeParams as typeParam>
+    <#assign typeParams = minor.typeParams?reverse />
+    <#list typeParams as typeParam>
   private final ${typeParam.type} ${typeParam.name};
     </#list>
 
-  public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
+  public ${className}(String name, BufferAllocator allocator<#list typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
     super(name, allocator);
-    <#list minor.typeParams as typeParam>
+    <#list typeParams as typeParam>
     this.${typeParam.name} = ${typeParam.name};
     </#list>
   }
@@ -242,7 +243,7 @@ private class TransferImpl implements TransferPair{
     private ${className} to;
 
     public TransferImpl(String name, BufferAllocator allocator){
-      to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>,  ${className}.this.${typeParam.name}</#list></#if>);
+      to = new ${className}(name, allocator<#if minor.typeParams??><#list typeParams as typeParam>,  ${className}.this.${typeParam.name}</#list></#if>);
     }
 
     public TransferImpl(${className} to) {
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index 471fe4125725b..c2aa83757b99c 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -29,7 +29,7 @@
 <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
 <#assign safeType=friendlyType />
 <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-<#assign fields = minor.fields!type.fields + minor.typeParams![]/>
+<#assign fields = (minor.fields!type.fields) + minor.typeParams![]/>
 
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${holderMode}${name}HolderReaderImpl.java" />
 <#include "/@includes/license.ftl" />
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 05048c5c341c0..14cc08d7db0e9 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -235,7 +235,7 @@ public void end() {
               <#assign constructorParams = minor.arrowTypeConstructorParams />
             <#else>
               <#assign constructorParams = [] />
-              <#list minor.typeParams as typeParam>
+              <#list minor.typeParams?reverse as typeParam>
                 <#assign constructorParams = constructorParams + [ typeParam.name ] />
               </#list>
             </#if>    
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 36f8338605c4c..4aa0076dc8bca 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -62,22 +62,23 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   private final List<BufferBacked> innerVectors;
 
   <#if minor.typeParams??>
-     <#list minor.typeParams as typeParam>
+    <#assign typeParams = minor.typeParams?reverse>
+    <#list typeParams as typeParam>
   private final ${typeParam.type} ${typeParam.name};
     </#list>
 
   /**
    * Assumes the type is nullable and not dictionary encoded
    * @param name name of the field
-   * @param allocator allocator to use to resize the vector<#list minor.typeParams as typeParam>
+   * @param allocator allocator to use to resize the vector<#list typeParams as typeParam>
    * @param ${typeParam.name} type parameter ${typeParam.name}</#list>
    */
-  public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
+  public ${className}(String name, BufferAllocator allocator<#list typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
     <#if minor.arrowTypeConstructorParams??>
        <#assign constructorParams = minor.arrowTypeConstructorParams />
     <#else>
        <#assign constructorParams = [] />
-       <#list minor.typeParams as typeParam>
+       <#list typeParams as typeParam>
          <#assign constructorParams = constructorParams + [ typeParam.name ] />
       </#list>
     </#if>
@@ -92,11 +93,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   public ${className}(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator);
     <#if minor.typeParams??>
+    <#assign typeParams = minor.typeParams?reverse>
     ${minor.arrowType} arrowType = (${minor.arrowType})fieldType.getType();
-    <#list minor.typeParams as typeParam>
+    <#list typeParams as typeParam>
     this.${typeParam.name} = arrowType.get${typeParam.name?cap_first}();
     </#list>
-    this.values = new ${valuesName}(valuesField, allocator<#list minor.typeParams as typeParam>, ${typeParam.name}</#list>);
+    this.values = new ${valuesName}(valuesField, allocator<#list typeParams as typeParam>, ${typeParam.name}</#list>);
     <#else>
     this.values = new ${valuesName}(valuesField, allocator);
     </#if>

From ac540758c7c013b2ffc7eb3d1e80ce6fb429de03 Mon Sep 17 00:00:00 2001
From: Jeff Knupp <jeff@jeffknupp.com>
Date: Tue, 6 Jun 2017 16:51:49 -0400
Subject: [PATCH 0705/1644] ARROW-1088: [Python] Only test unicode filenames if
 system supports them

Author: Jeff Knupp <jeff@jeffknupp.com>

Closes #727 from jeffknupp/fmaster and squashes the following commits:

c19b3dec [Jeff Knupp] ARROW-1088: [Python] Only test unicode filenames if system supports them
---
 python/pyarrow/tests/test_feather.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 287e0da2f5594..71e4fee83dc01 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -355,6 +355,7 @@ def test_non_string_columns(self):
         expected = df.rename(columns=str)
         self._check_pandas_roundtrip(df, expected)
 
+    @pytest.mark.skipif(not os.path.supports_unicode_filenames, reason='unicode filenames not supported')
     def test_unicode_filename(self):
         # GH #209
         name = (b'Besa_Kavaj\xc3\xab.feather').decode('utf-8')

From 2a12482b21df0f28bfc53fca114795f4f74d1617 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 6 Jun 2017 16:52:45 -0400
Subject: [PATCH 0706/1644] ARROW-1094: [C++] Always truncate buffer read in
 ReadableFile::Read if actual number of bytes less than request

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #737 from wesm/ARROW-1094 and squashes the following commits:

fcb613fb [Wes McKinney] Always truncate buffer read in ReadableFile::Read if less than requested
---
 cpp/src/arrow/io/file.cc         | 5 +----
 cpp/src/arrow/io/io-file-test.cc | 9 +++++++++
 cpp/src/arrow/ipc/writer.cc      | 4 ++--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index eb4b9fc43884f..c4c797c2f41f1 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -412,10 +412,7 @@ class ReadableFile::ReadableFileImpl : public OSFile {
 
     int64_t bytes_read = 0;
     RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
-
-    // XXX: heuristic
-    if (bytes_read < nbytes / 2) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
-
+    if (bytes_read < nbytes) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
     *out = buffer;
     return Status::OK();
   }
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index a5784de3752d9..3450bae13bc21 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -251,6 +251,15 @@ TEST_F(TestReadableFile, Read) {
   ASSERT_OK(file_->Read(10, &bytes_read, buffer));
   ASSERT_EQ(4, bytes_read);
   ASSERT_EQ(0, std::memcmp(buffer, "data", 4));
+
+  // Test incomplete read, ARROW-1094
+  std::shared_ptr<Buffer> buf;
+  int64_t size;
+  ASSERT_OK(file_->GetSize(&size));
+
+  ASSERT_OK(file_->Seek(1));
+  ASSERT_OK(file_->Read(size, &buf));
+  ASSERT_EQ(size - 1, buf->size());
 }
 
 TEST_F(TestReadableFile, ReadAt) {
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 4f5edf2433edc..5d4b94a805a64 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -757,8 +757,8 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
 
   Status Start() override {
     // It is only necessary to align to 8-byte boundary at the start of the file
-    RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
-            strlen(kArrowMagicBytes)));
+    RETURN_NOT_OK(Write(
+        reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
     RETURN_NOT_OK(Align(8));
 
     // We write the schema at the start of the file (and the end). This also

From 46315431aeda3b6968b3ac4c1087f6d41052b99d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 6 Jun 2017 22:13:26 -0400
Subject: [PATCH 0707/1644] [maven-release-plugin] prepare release
 apache-arrow-0.4.1

Change-Id: I36c4db377ed8cfc10de14604114a79454db10aa1
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index e09275476d175..9d509bcf1f2a3 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.5.0-SNAPSHOT</version>
+  <version>0.4.1</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index dc4d0daf93d92..6a8fab0a66bff 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.5.0-SNAPSHOT</version>
+    <version>0.4.1</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 1ec3d561121c3..094914bf31f7d 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.5.0-SNAPSHOT</version>
+  <version>0.4.1</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>HEAD</tag>
+    <tag>apache-arrow-0.4.1</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 53331f2f24541..f93cef87b7b49 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.5.0-SNAPSHOT</version>
+        <version>0.4.1</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index a117a2fb3b7cb..60e342de5f048 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.5.0-SNAPSHOT</version>
+    <version>0.4.1</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 41b58e4f7c6ad2f1aa0d0364349b6da30480349e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 6 Jun 2017 22:13:36 -0400
Subject: [PATCH 0708/1644] [maven-release-plugin] prepare for next development
 iteration

Change-Id: Iec07f0f0117c363e39666a76c5c99e7f94c73ab0
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 9d509bcf1f2a3..e09275476d175 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.4.1</version>
+  <version>0.5.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 6a8fab0a66bff..dc4d0daf93d92 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.4.1</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 094914bf31f7d..1ec3d561121c3 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.4.1</version>
+  <version>0.5.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.4.1</tag>
+    <tag>HEAD</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index f93cef87b7b49..53331f2f24541 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.4.1</version>
+        <version>0.5.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 60e342de5f048..a117a2fb3b7cb 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.4.1</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From e344066c46716b9144cc385a9d6546a09e2d4ae4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 7 Jun 2017 21:26:20 +0900
Subject: [PATCH 0709/1644] ARROW-1095: Add Arrow logo PNG to website img
 folder

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #739 from wesm/ARROW-1095 and squashes the following commits:

4a89d2d [Wes McKinney] Add Arrow logo PNG
---
 site/img/arrow.png | Bin 0 -> 21475 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 site/img/arrow.png

diff --git a/site/img/arrow.png b/site/img/arrow.png
new file mode 100644
index 0000000000000000000000000000000000000000..20b00bda778b72e66451e6ea75683dcf6ad3500b
GIT binary patch
literal 21475
zcmZ^~2UJsC*DZ{INL88$2!d4UCG-}0jdYPBoe=3FAk7ev-bHF?QKX2VND-wY0+AZ3
z^rAFDdM_dQPoDSr-uvA<#?Kgx5l;5mW$m@*oNMh(ys@DU4J9ij5fKrMp01`T5z&<q
z;P(wV8SrHMr?eFj5%o<s4Gm*G4Gm7?r#`N3o=_qp-T2JxYv$=aOz1Tsfv6~^xbTdj
zyu<4wp*jka<a*)8>g1$1L#@o8h`ft?{q7HCoR>(ru#B*!CF$l-*NfiXM$1oc_W3AY
zW8k6l!G43!&sGLo0&yVZnIuVg5s3lk54B99A1k*gonJgh>h%afUNLhLU6We8zHGTk
z9AV_|FL#RG-zGbZS$SW=GXcK1oG9x5#q{ZlYSG;?U&_gbm#IYI9yM1x)QQOFhP=fq
zj5&XOdHEpPn3G7eCJN+I$@@!+WkwhN+X>5gL84k(W7$DX#K?1o)Y_!Z(NQCsioyKa
zyXzCX*)!<k%b9rBsCQr>vQjtNGxylGFbV$yDN~r<)yoW<vMi3?#0b$Ye_F+&M)%qb
zyi*=m-hf+{BRdwEdvbm?kHglY^_bmtrcd3{OkO_Vv|scs#?iN?3s9zT9C0|@I{hY3
zLQQ1c?syif7ULH$sX4W$^41egukZW@%-SPF9e;a8%F(X>aadiP4K1x=H*)sn?r^@)
zseGFAqx3;4>ImwGtRcm6AW@T224$O0gSe<lqJ}0;MH$aAyqa&R{c=vKF(db|C;6FI
zqqw;CYGnBuQ&KWjty66=>)74UTdv(#`<$wIywyxkb7ZbhfK^McEIm2fdz&lqweh*L
zwzC1L{P0_xC5<fqJ#87x<hK_^lhu>+>R;Pk4#Jgtouak-4xM*MqlbMZz2vTUL~!h0
zgK)l(rdf^<ZvV^r_)hqOi%3NH)hJGqClM>}pu$`x(;cxGOJizWROi*JOTMim7NYlF
z9E&YkOpVeP8`vZk(@iZ3EI)e0aMC%q_vG|2X8d~KH(T`YoCN0<dXhL?r-Kvcu;-&|
zx8Dy&JRlE3e1RwODDfWB-l6-*&>1zuNnQW+R}L@b*%G$_kx{Xgb{l3{gbw#QpMoo#
zx%+BLiYSftN$S~RxAAB0?Zi;kOJl{scIgP-Z6clDk)hwJ<<<Y5P--m+whpYMUg{mf
z!QCcJFB)Dfy-TR`y$oBj1H0~Bc&Etif;M%GJg+`u{@ZToe5+cfl102a-ID5OJbU);
zseTZbEroMSm+oiicco?tjy&zWusjUnZ-MlC)q9sMYIg&ly>ZbvU7=Vd4`x9|LpV;g
zS0y!0JuCDeinnd}iFkGbr~WR{Xq?Y$GiSF>?HgTLw5CjES@0AlAxcrc8jjn}8(5T{
zwez?lH$T8ck?11xXrGga(l|;{gX9kpi326t<y!tCK2DaL=nW@jXM~G5$zTZIkLdTX
zh+qeadsm<iB9r9r+hsm6j77p1X&;k$ElSDV5RLSH!af`A@%BbDQRE^!vo>v1)DW#g
zQN$lE?YOH>v0ph0b&2RxIyrlE1z$z%($PuYuu90dyP|uuhQv)ZOe^JXSz4m*ji<MR
z;|{s2(v0rKn3_B=zb&o(!Q|H1H5!ejq4mdPxv5NV`-a9J^I9ivYvm1%{(&AU2x{BE
z)?jX5mLMyQwQ>^8;b4j`aahlxwF%F3x@jp=&*2f>cJu&}MB~8W%Vmg0e&v0H`WB=#
z*nO~f^;EE#vI?EITYLHddsih6!8=55O-l9R_T8wLnm;)ExiV-4$(ty2DApEJHn^L)
z>Eo%3D27?K2V9((d_{e4`U>q4?J-uzp61ADa-~V>GV$CnWsKJn<jH&~lYacl@KuN&
zy!eJoaE<6x3Y)Is-Gd&3A2*kkml>Ar_Cfof@?Yhr<>x)hVAJtVnPqLTefr_)!{zhk
z7=d0!hJ<UpDX+AwAmzoIW%+Ms-!`XZXHR4YW}>(->GnMK0tjQ=3#`Ujs`<_L^5dxr
z$$~{6OMjXenfRD33_X@|&XpC3w$Qb?@A10ejq+=_@oOPVfv9^=7?}jkg`@-y1)I$?
z%%n^YP4eGZR$9IZvfNAf9sVl+RrM<ry@6YV>E{uNl01ElUVo^~%Dt6GD-U`@GsFbJ
z5AT*USCGDYVxCpLX)gbEKPP3F=1-Wv3|s(y>yoX-DD$0r5o=LyiFIkC(Px7I<NI0z
zY9eZ0_12%KeWfKxoS8xD%<@UnNg7GFlZ+VZ_)j5=hL-Y<_^5e`opJv6p%s4P2IDv0
z*+0Y=e;Xqo`tj4UY`D_bo?$|2BELx9pjP`Jf3;n6Jbs&Diot;YPHvD6)wtwD!f<ZI
zXR``34_zA_MO~ZMEmjvv$KlsC&^G)Au*gA*f-u|0k7c4pfsxyKbKo3z2#4$S+E>9@
z^6$z@ZdLM+%)FT>k*)M8;;3rV#E0YHBnc6TYK`B>BA-R7Y1n=z{~jEZ921zdob)J3
zGFgTtfVo9hM#50a&U@AqJF7Xz(^))OJiJ^eFfi69wNUr-FqdVxze{+=v%>0&y!-a!
zLN`0FurA`yPihveJU{<IYUV5}#DmM9jodeQUy=$<gT7tS5Q=&5b)3G!wIcgx%uj)z
ze=8L$+ot8FmnS!C8ayhTZo4;i){h^5E@)t%`8d6ut304G6!r$5cqw#^>e@$*<nJOn
zjm}X^PtniNVupi;>SdRPmxD2b<%2f)K8^1hHJd7*HZ1BtISR1-Grf8~JkaxDXzS&S
z{dU&&)OP4B%18#;U(oEa?|cJ(BMW`~Mm7H^Xxy)<Y|;Rexbood(T|z-yrq_z3uG$N
zC<|kU%5IfuRbP19x^p>;UA-vSM`I(g2A565=ET2A-V@&;-MHpTDnoXiOrC6jM&Txe
z{5qB0t>++3#VGa>T1n;&HU;pZC$qG$Or_wa$=Beo;?2^;$>hnF9KK>B_g-;Kv0W`v
zFZ!u7mnsuKn2PhF^wmGi*bp#s_H<tD^6wmRhIMbp!*8wK{=&EI-R#zUuT@!1!qHk?
zqvJvo-?^{p`zl}rTin-^+ta+9#b~Op-n*Ugru>STtl?JPs@w;c>Ew03QJ7JioQ9wF
zVA2xkCalKwjv;R`U+K2qWq*RxcumZR`bd2-P0i(G-lUk^d_GNn*vD&M@-KwvxLcGi
zgJX2M&y#SFs{VxjD}@|Y2`7R=avnB6FE-z8Hg2Zncjl8cEYFV3IzPV|CWH4ZG@RdT
zz7rAsT9b1yux}v;e2<!8TU3>@^j00?`;NDUWk1Xh)BiZ8!At^%W34D^uVX9<2T*hJ
z=;wpI@|XmYL<BMs1I2iu%4B`^nqv>(t2>tA!n!s?twRpKYNFwZFPO%~N`(jQuov`s
zyekItqW2$7H$p<EilC_o%JcGS%{Q8sPyHG$_PvC5Z#Lhw720~}ZFlxL!z`q{zqGPw
z^FZ+Uy#aUz_vq~4$Hw8ftw;XPu}a>LD^)GkgC4f<hNr>4XG=$PfAeI-Wiz~kntWHg
z6K`I>IU{2t6E71l^j*kC=C-$8)0y7`*YP9wg$)v$y86o5{V*Yzhfi|g0M2WArneqG
zJvbBQp)-_HYP~d_P|#VRrHI<rpB&#y>=UDDk`Bz?!kMD07$(qTD6c2pk0;-LhTZ3$
zzURKVpy=guqJoEuT!@_P$hFHEKci)1WXCQ|jM@9JQ7VpwO|I<XxV3fa*-AJ>ywr17
zrD$YeGKn^?Y(i3^?|AqG_$Z&}jr#<6g?Npl$5X20)#R%;JAd^2PS+?SEh968<%i9x
z)Lw2Qi$N2EhVnB3qMNhP2N7!+|5N|`=0)E(TSYTn&e@MdrR|?J&W^251|J5Z7gC?e
z2T0oj>SJVUdTtuju%=n{aA&;&_Yh-G)$r^rWcjG@_MWWwT>dk&=ZSuQ&$jYss<s*D
zSX!I@-1%j<4llX5d86*m7_M*A{#vKu@1+};9M4n|;Ql8P7ki2PSB+FoL_+mLt@j`&
zR^R=a*1RqwF6R)C%eOnxi(R{_J*ZxUcd*;}{tx?kM8I`En&TGwR@UH_jpXjU#xK`>
zW@3wC?=<(pWwEV?k=?Q9siGW;D!gGKGD?3z>wYV@YQ;X-e{Y??-NVj8_bEzv-GnIe
z=6$1UivjFJbuK~A!cXK9UTorcMdCIG@A2*$2{*mfqf%-Z3nrm$BM*NPq+K2{PA+*f
zOh0A8cb@3;7mlj8C7g5fhFOV}J`zzkxVe4mu5f-P-qn(mlxz-gS?&|PcPv9xwhui~
z(YxCL!aeh#2smjW1!o~h(n>s)4x&LFK(;XMW^U<kc^{(S?BgZs=;Gr96%F!w0wfSb
zL`p#lz(+5rzawXmm#4R%LXa}oKP?o1&xFTfT%7+j@%K>Xvb=B1sp0b!$|)@>B`VIP
zLdnU=sr1yvRl!tK``_1rzm&Ni`};pp5EBaw3=|CniTXT+iAl)I%ZrIiib+a}04+rP
zg1!A6gG9Xjxc}41|Ma5?^>cpe_Qc=K$D5PTucMPsfWI;q7h$0P{`_a2{%)@SXC!aG
zf0qR;P>k@6n1raf*nj&5UR5GIRnYM9dh!(N=Lht!BBAt8%m3@y{~6~$?TwzgL4lbP
z=JVh7|NGhh+ur!GkG~IKwWn^*dfxufr@&zUzAgFhX#f8${_RUijIi+kWr+V+`JbnN
znN=v2#QwWwDwM>N;a7=>REhL7)y#vgY!&$YD0$d0kA7DIk@K3SwGrO$b?cP0G}WZN
zAxTfJu1S0?F)42JMai=9@7U9>gKuHR@3LNn{Wzlq|B0w(;|uzIjf7u-Kka*zj2eSK
zBicb{gngq}PC#3azms}k{U{F!+wVWvKA8&z{|!5~JE{G6dpi&xdNO^|`0;4R?&REW
z`9~V}PDYrnyBoumg9W}F+m8c?*eH(Vc3sl>k2`S7^PS?h>{gx@;m>uE;b%Uvf1&cy
z_&(#kP;Cbd(p_!*aqfKWufWTlGdgknnY?gM#F>~HW<i>Gf>y2+{gE54g#klda@E_h
zvRWst{keGf%TpTePx8cpe}rsxdY7!qolEnUa*l4Vf`$Afp5@`0*$VWR$f<?s3)X|z
zd%AtycrRAlZkA@Rjbg>nT|2B1$+Umay61)K4Eigl+5%@N5SIfERKo<TJxl#QA|RdA
zu68rt@aV-D)yq{|xvXHGx$p!e#f{Q_d^6pX?Icwq;@E&<sdsC|QuSHETu&@@DDg%H
z3(A;E#fQKoo0yp+h)690iVI*H<IPtb9_Dnuk72}#=?@^Hm#^ly;7C;c>Akq0jOI?E
zxzCz2^kHr{B4#pHLQ6Q@#QqLe(~9Fo$+R7=68}xo-P0eEEk1@=5an(z1-SAk;PxOp
zhFRdeP&wjN{FCR{aym<@i2H){G=dk9sZKBCHR8N|d}U3GY-*+At^dwNW=~fU6E)rp
zIkMj87SBTLfCe$j^5Xb|#o>m@yTqNkV#%j)r&6in6l5}y51D-gp24)4#UPYC2Y)OV
znDUVG)B2d<TIT*Ac&SkP{as%Hd<v3}^Nf}O9aL<4TlG>F9~=HRby|(6s-}Y3`A!Q)
zD$q(`T~|P?v)>0pqc-|8+rN0VZM|rz|Ag+VfmpZy>&qQ+<$iM%F9&f6yTA$&jvX|B
zR%Xh6R=f)4->Fxiz2a2t=_Zi*R)5i`SjF%oV^97<EP@Fo?^>ANW?C_T5O{$L3nAsw
zE-nbv@9(^T_}z@x#oby3-#V-Li7k8EPc{*!+Bs%7k+%Qm0^;Fv=8)Hq|2A2-tLLJ#
zalW7)?|Ubki+1j%De^gL-Sl0>^HJ<Wy}qAGl6afQX|U8o(x1v6fuq>kgabcQG(=X*
z`vQ-jCI!3U_=%)VUaf-ZtYoCgBL0*Y;H1O~%TJJ9yHXDXOJ-2#&u^gGxZc81Tju>;
z#fs57)0rgo-NWV9N$s0-IW^oR>U5wLS>XV#_DZ#h-TQlga)Jw*11;`PrI{8Q>dhga
zJ-+a*Cx^_XnId-~ExBI?xui;TrUUwKWd`hkKBnW#6-|4Iu%WI+hB&VWM8sxYy0N7a
zTt_P3?+R`;x8Ao}X(!tOdBq-t8U8|@BP#KwsM`_pcR$gj)*V3X=$;%y{t7D3?CL7u
z;*ooySs}EOeE2a$74ltds11EwyNseKtGR4x#PU_1&jWzR8X3$X$!Kq7bjp+-H$^a}
zwqj)B<oL~Q5zlz*mP7Kx{e5i3r91{iPx)O}k?{%%v!bUb6psU*_z1nAowP{ue=K^!
z5~jTYFV%3F4!YmJNd?$45OczK5%+K<;oPV{e@Z-&F6Sz76LZ=#Gi4(xwXlU6MbdiZ
z&kMeoCH0Vr?V4?SCgP?_eD>i*3bK)TkT}BDf`@ffr+6C@_%P$yR9XjE0<;5~WyjB?
zq#a)~Rf3`R=C^Bh>}7~0$Tu6gOC(ze#$i^Gy<b8Ggib%cw7(0B|EOrX(RP;lYovUW
zJi->R;Z%#3&#B5N!H6pExS6}5dWHjddSk4GD5?9dE^}rPRry0d$|#ZEk7dskw+HY{
zdOMdHe-SsR!P3OebnnW!hI7t6e^x_MF6{2Jm!EN<9ech!!W#@71SYpjle&B8x<TNm
z_2Y!HvxPf9rE#WS?B#zE$t9gK><OwO5+iTYDt{jWRx+Os_QK8l3b6u4mo&%Pjgyc4
zNfQ^;lE2dRMH^!C-2D*PIV5uT4f`s>nl!(Wj$HNiYDxy#Q9k2skx@=#KLCf=roZXQ
zWw`+uGP<~7ep@}fd@j(U(Eg)c2m}D5QQCmuT$k^rP!u=SP(OmXSeC~;6bN70p_N<G
zT*e+r{&6?&P@J$#gV?!7yZ_E)+!vL9ZSuJ;7C`OP1AqCCaQCvr?7h>Nu|hphx(c{^
z8wUKP`_%KNB@Q%>IN<1OF{4uFOw^f%X1l!;GeB+wuM!pZJS+~~g2HRqI0}SUPNN18
zN~MMpL=m3s0#@CTA?m&6RVB)#yRR+7JEb3GJtcKdO}pXu7#ACNdNfG)V=|4{IuP~7
z*;P?YT~UGvC%;^%J#HG)zSG;<n(mW_FKFN9P^ON6NW2PbPFkCXg)rCtg=ZBY(2$9x
zv)6boh-BqMD?2m%0A8*C{PF|W%Z9rp9jNpW>77%q^a4bE_o<560r<`p#3(r<SNoWE
zJ#&wEU(&mH<e5s}UCF;KAK@Io-aitLJC7fe7=55Cf}j1DLq5~}VC80aeVG~Cit&%r
zSFHMl>Vl;peFlS%Bc$YI@OcN@KBU|)-GJ98X(SkSKy#JhLuZv8(tT@G2W>28=R2Eb
z7ne0|IN8z#`#5z6Lw148-V>y^g`^x~>jE<r;f<4BLKp(Bf_I(0Xp5xH=^OLrxK2h^
zR+54|GI3jLI9o{iD-YQ^_VdJDJU%_Mpv`gCp?uPcd2;zX-4_^k7KwjAK5*f80QL#1
z=%34Xa{2D2=TzE?Iox&9c?;_F60O1r`9N%Ku4_AZzbOctlEz;<*ft`j%c;<rR_mm=
zXP-V|X_>|S<Y4~`Y)>>Ys>(p%TGlb(TU)-Z>1>!5?yKg<$a9s^$i#=o@ThBRn;ZQ7
zUx7e+aBtuQ7}egpqQ{DgUBF=}4S49DF!z&Y51wTc+x(tG3jbhO!pM2G9^1mf&7y+P
zU<^U6C$9tzO4QuN189dDaL4&*wpS&o-X#ty&Zitn5KmsefcUtCZksh-&Yr9EsjzM-
z%MvD%$u^oc=*2N5xVYP1_!5}S1KoiGKUW6;*_9e>`5&$#*fX%;+jmuU6wvUf3RO<m
z-Uu3D&BwOZw{^eCVlPxipBc2C`D9=8JXKkPM}23W+WmZjw3}m<>MY5e*j)(euPb3b
z`}8Oe-_w{`Pv6a5aG=uhz*AE4=sycRNI{UL|7#T$y_9ub8*9%RYR!KP1N~9Acjpcg
zz~sok&<3f0G!J1W?hTDkr%sUdHuRPa+ZT|5wh2g`4OxQU{L>^CtN!B8@X}TMhuWB1
z@9(eTMU-NWoVTlNmfOCP_!8@+1i7Ddo2vFl!AphHzSPCG7uI;<W|ms|s?T;_?k;JQ
z2e}7=9|b)IoDwYhq1hQ{;}{aGGJt>j2)likd?>B{Yw9ko?#Jhyf`b@6_|*x);?=W;
zGZcT--_>4hRbmEyM}ascjh&4_fj?ulmFOEb58r@RL9bRtpCekhchmQP0ZyK5PO^T7
zXXOvOx!nL1-<`EfwmiHFYgia3@wV=bsMYgpkR3@$FE8n}03ud|6K+P7Xtm7?H__Xo
zIP%${@psyM1>(qk{%ALM1|h-ih+FI4I6srX|8*W;Ml^EUqK=ymt5WrBz<B}i&ebE-
zD#Gr$ckb<jHUTG^%QnT4fgi3e=Qjl|286fnIsl7rMmZlW=zPaiaWAGEDlmG09Yapm
zOiR!I->*3rQ~V#PEqeaM=&rR&yRs=9;Iqz9V07%t5SAmxFcQ_k&2FSnaTbSkwI7%{
z-vT=ugr9LF&MLLa-omYY>rV0el~R`!ZgY5)ZhBq%hv2p@8U(|kon>Qv1|#tDyT)2~
z<gR;c_Hf|lA8g*Ci}9Qvyf^7}r%zlQ4?Ts7A&_U7N!JIk14WnNfpM!$3}<Oz1KT;x
zy4V5ICy@#eztKc|_{K9MLx=J+pNGT<&P#PSbsR2DG5Q;ao_$Mc)%xCjx2^pX=<72v
zSPwSaIY&Oa5)12YW}R8!Gsu}%woleN5MPG(YBOQRu;^_01CCU;Qby1oiekdutBd<G
z)>KJA^mBbDXSnQnTZtH&N7hXc@vz)%ZvnhsRhRko%qMZj(Hpb}uA8gU;Z)T*zrDUw
z0=Yl*&bpWG^xdl<wS<TICGV|r5^!MC6qQa3bpKWLH#vB9qI85Z1HR|YN6d4>%^-_h
z_*L8c$0+*3&?6K0YpqEehOjBGH!4ttyeTX?3vmj&JEB;U6+;_-NHwxaV|o90fnTW>
zlDC7z@75SO=R#4{XQ<MxrCl-uXL8j><<UvXE?scnRC7`x$E%-G%st*zk!ED&ES<m-
zFAcbGh1AP4-M-|s-0Z!HA(3C#3r6x{InIynk?!iwbHlaI|C$JR?ihGZoxM369qBX8
zz7k=4&r{+8Jz}pHUoauO3NB<P<F5n<>7R!@-#4swC6#-5>E`y#=QZEO9%Rz{m6lv}
z2_JUV!GH;qA?c2D?!!lRWK>KFgESnF0M(@pgUskrFC~j=GPuKG`zq`98T_9!8N-V0
z&vF=jdg&wEuVwc8nQlDXa>6;YoPT=dT))%S#rJpYS#rB0yFmRV<XqNt&>8AJVwyt4
zZTiu{*#ojQ`rQ27M-Q2MtYQ{H%egs&D`4FI_Qh2Kzqg4s*9$W}w-Gp{SE-hIK{RR-
zROXjsUBP<;MxV)E`qsXo^z&l&+;t`t6uAn*kJiCYc-%=m-D!#EVXXGZI5k+V3|J(z
zHqb$O%BVjnBvY#&;d$0}?>A@*G-zu0#e402`CO}-*!op7I*#mf(qa|KB^1bZkWow)
z?ecOB$}?&@`~9(iPI%6gSaD?r;<HjikC`~Wnzoh;M>4`+QuhJa*-tQ5MM1a*6n3}8
zK>u2Rxpl3>0{2L+QqyMrB2p5btJaTrK4l+w3aMxSWsEY)sGnbTJ0`1?xu^{akvfMB
zAReEskqRFyc<1d$ox;Su^y1q20N~%0=sL(hMCqTLbDRd#5m%;MMr63D$aGH#LNt_x
zyX@)@EPLrVExIWUuWtr5C^hH-6WiOxy?KSrIYj8W*^AKJ4Ttr*4^9IUId+lH%T76L
zKD!};zd>$97^?$fRC@KX<OF?0yD#s4rBvwJgY-*(!^K?3P|}JNc2Kei-kzb<iq#BI
zI?7wLNBH8gSELcugoy>f4nm}TI5KP2Jg4y-BW<Dvu0jj0l)3AFF#2$m8};%mtcP(2
zpKjXb&UC<mxqgH)8(M6AhFkJm2uiRUxUDD&*Iilf8<~E%cbs@B(DJUQ7UNAtb)JL#
z!L<sm#|DDTtWLiasF4Rr`*>;<aeoEJ<$3YcmT|n<!^}sFyMSY#xx^!L@uxMYc)VAg
z!~m9z`!O1_BseA;%}-@4*=ZoK+}Xcd>amejGt==x<$uB_kxPXN&(iz&Ycd@mL{0kF
zYn5<@0{nrNy|kyVTL&|r2`AzmtxRpsWF$bFfJpUy;`fSKqV`sCTY|x}R~h{XrOLXu
zo%iLhe)xAStr)3iE(zIa_d&?-@#bo1xSQW`o+}+-F>s+9AFeiI)Jd^yS6A^Zv%If!
zKYuag1Q^Oe;#T4OMU`?^NqvD2UKe~IQN9C8{&u^H5r=O&T*=|#zu$#LcO=3n@s+D<
zf+_|Y<ddLe_RJc|eKi4e50v?}!_rCd)KZ4@R4Qyg*o_Q`6Da;g92gn#e1<uZDB@}J
zXfEqp`N>yM>9wujg;zl;SA{-a&!;P-HU-6<r7pF2e7VE0XLWt+hq3nPB#`IsWyZr@
ztrW~CH{9GNXiuxzsL1$`POhtM`3GSw-#`PMZT+7$Im2P@RXp{Q)vvh|+0x%oxwT2`
z9!M(5<pmfKJe*~sZ-WxMv}ePr(Ow9g(1&QOTpJk4xs&<9Zh*yxEDl-t(v_CrN(DlW
zARsleJTH6nIn=n`Zsgh~Xhq4y;;wZsZMa4T+!Z0xBK*PX6Q)AmnZ-$Zt*Lku1my@E
zaKbH3IjT*37_rx#5^;1Rk67vzqT<WC|M0ZM_iOkXyQ%i&XR3-6D{;q<OX8faSu3P$
z2OQ|ohp|lYr`hFM_ej}5$yE^Z_rhq3k@Z@K_a5X}%ap4UwZawh0|@zr?+WShh?@@#
z7>R>QG5WDrJF3vm>d@&#uE)RqbF!ZH`zrwpq=-xxmH-h}v2vFI&mb!{qZf8dV#IMU
zZk-l%M9BTR+g5q*9tx~`ETvEN$}WsmVO7tq1gcdG<cT}`tPrYm>K<7sc!UVIefHCj
zpGp8NkIh~gTbPd}%In)gcBdF(C`N@1mbQ?cPZxsu`cGZb%q0Dhv0EqIzh1y{<MF~P
zbs>uJ$Qp-^oBY=!K^^Iaf|0Z!8{Gi}8yRxDc9!8JinpKe`SjKiu;ylr)JOUNni2J7
zwtrUg=MYY8=*i-Maw{EgR>^Cy#qoRFmFl&|Ft3Zl=2S)%k0P91HT=6<TD9z(9-Pnl
z+AnN;i9=p$o{N>t!`MiakK(>s#asx-r>lsq{899ld$S2_+mB9!mEq+UoE(5Vk*VMZ
zVZz^k-*oJ$v&yOsc2jM^)FmA_765+aFH`Z*@5?}2LIp4J6~N~|=t@;02Ui37r3w*D
zhg(|agCfOhaxVrEa-6$=_i|i*BdD}03sAf8(j}n*HJa-c!GMXDLTKkdw_skhrt9U>
zUcVKrZhbojxz*E&NN}=tPVy&bgX!Qb{>j6;+e&@U`A5(t*=YXSOdopCMfG92sVDEe
zvw%kUB<Mm~J}*<6LYnI0H9`=opSzZlM-`<GbmVApd32X}GuFwP>p=ID%*?G0P@jq!
zawDf)JWg06h%W`%?Zn!H8FPa!p=8?W!^53{2vQkUF;jq}`<?oTsD^5&O95|NDNIpd
zz+bVAOaDXL4vNN>>v)Ki81?&Ck$l}pzpWuScpskEF_+i%-0*;7{uuhZt#?&G16ads
zV#pE{DD)BhiO;oH<-70+vX#>Y-*BjG3V4#g;U;MC32G&;<^VPV&Qh=wq$TD7Mw$ws
zhm-C?P+9fM@`eQucU}4@?CiHXRGbeuY;d<aGIsUI-XsAe_v{Qqi2R^b;g#1B<%&5!
zN6;DroDiMD?D}p@#u=|?2PZG!RRKBt^;p<%um;P0=eI_x3BY!Zn3uNj@B#s@gr_!S
zE_R^}E%iFE*J2r9=4K~8c{LxTcJAoLWaOR`J8kEeEs*<5#q5;ddvG&qr-qUI%Byu1
zgJUKz|7B7JymTHWOiRhjfRDC1nurBFCkW-X`R6cSd|(+sc1zq8ENJ3W(F5#xhG*W>
z>88(hVw4!5P!|TE;V0u4y$jY6D9JV>awCf)C;%(=$1M3S@`g{1>Kk$B><D3r18OgG
z*4tJC9vA?!=s<lnZVOoSQw%2!hNsWnjtqW7uZ-<ceb%vh5WPY*TX;L0Q}OpoI<br7
zL&Nof?>8<(7>PR}^Sqba?|ght3|7H+d^YHyoZsQ4KD~n+`d|^>unieiPR3r5a}~J`
zum`;`&D}*CXD6K%Pmq(ePnMesfdlL&dEJ;W=B4MP!j81-%CKByIQ~NqoASu@X8U!D
zn|Ov1VTsnkV^$6;x<38+6a6;s54HfwXpFr*F+io^6jaW384<;dV7`)BK$Er$U4}EJ
zE)=@S7nYlTg3&7qLZcF4xzLKVv(z;X%(BZXi}1k(e$K9}E3ReR_dD<w{rR;z(^ukP
z`!)h6R@0y@V}VeYD12%$f|r-?L2V8`=-DnrZcWCZAJ`6mY?VRFge4~<CG))CWkbd}
z)Qy}FqwJZ#ku|x2VtOo+_&w|r(#3ip*!tMi`-23tIB4aA1o4}zspeD7TmYHoo-CQB
z>Q||nVq-J&J0W+ner^L>>C||_tE#NPH$V@9+to{g0usjn=?5~KZ9fFL#pDDBrxbJq
zkZvM7le911p8Kg!0M7Lhp5zgRi#csO38AUD=rKW_#MQYf)lPNycg8omp?)@IDUDq3
z83diZo8N`UunR&{g_SxK2cDqH&%~BOroOo8;0h0~3S0zvFdb(BQbky!AMy#KM1rrD
z4|LZ_p^5~Jc)o4R-GW;O&EJKN**E0?S_06A4@Ov`*5}a0(i3H#^&(NQQg}B(AQ2UV
zMAAOA|Md3B`XI3iZrucV<(7b;L+BR`(|^tQE}Dt-uD;j$1`4*wE&ue>TPV{_OwSQ8
z7Zf!pkO=F@l0!qB`-re8h$&v9-%~<cf%Kzd-qF%|ctaQv&6TSfpjntZuJ7zlscLin
zc)rxT&uGX(Ml$$in8kT+hH_1oB<UjRi2sSbA|5J#^yTF4A|N*2nl`&CT?&sj1Ck!0
zV5)=@eyShyQ)MQ-r_~diDq41|yYOPLLVlrxYlG$5EJzY6Eyj({1(Zc?By1m3oN8d7
zY^0*p#Ua#XWYCfeCY$!eqL-)3AE|}Y)j5||b00*D1HjBQ3Mr(39k?bCN*|Loxd2Cw
z4|LsG59`kvT_Y`4EJ?}UM8o<OJ*6Eo@PBf|e9jbsV0p4ZH?IePKjNOv9z@Jwj*R?&
z0c)!?&olF0nYUD?rJ)2pvchqL6|=ley=-4QToLTtrz(S+5Pg5Fdn2az#OhAcK#q|m
zUGl?oyY;~0&xvQLFmg=P1tcDc7bS!ToY+*jc<MBD0KQZ{1&Swl>!(~XAjARP2?jCn
zr`HK>5}3If%E^HDxM>zzT;aG6&#I7^J_N^QpQ()AH_5}`L7;EcU4X#P`e<e8h0D)K
z!3UI|<o<6qn1?Yd3U+~3NQ;Ff07Z-e7wVPf5OkXe$<!&wF-4d%{T0*zt4-P;h`gLq
zA`n|R7RX#-F%esO0hPeYPu55^KAct)FXfd`K8c&}JmM?|4E|LY{Rm-vMx+6N1*30k
z#{aPZA1+%)EG11edSyr!PCWQ#*AshJf`4#@*Spu<@OXRtpEQo?a8{7lwKq2^H+p{^
za0qJXZ?Mdo;lK&<J+;3bbZm*Gq-WeB>D(iGNs>17?f$m^N5ec<`77&8zDAZE&0UV4
zXbY|WM;awoXkL;D^!;QetZftvLBVLr;GQcR*|uP@@7xRo)y#T(4^kKarSyVGc0mRZ
zR^J;cm)`jG(u1JZBNC&Nk}u;C!-(ZL=i-MB_s&NP2F`s;ek;3b8QbA&c0euHwbG5^
zvTG+^Hb(yx_UV;dfU))&^WmKRw#Pm~bfy5HK^B@dOdFWiH|NS`g(^KzV)&0siynNs
zoR@o4vxh3lxA5VY0Lid4{qA(!joNiN7mm!q%gr|_9M;8Qo<2qwdInskijW{rF+<uV
z)x#3ClL?xUy+5S-O5wn=@LGEqfY-$T91z^$bqJ1~JGm;?w*SzOcfji!P>YR2HXEGJ
z?HBVIeZ!(PAj(D5KqW886I7fNkEaeQtr`9j)0v{*#PG5Pa)1OV3W;n7_4vzFJv`{n
zitxmiZG3W<IgUzNI;{|$UqU|>ew<$ucP6$LY6-I(`jTBtM4RFuG$DQ?O!-ba1j{c_
zD&JoRlWZMf8(SQbgUZ(dgC1aB6sF6+$7e=A1yYN8YIAg(bMfee_Tcr)4}f%G;sP>0
z2Nd9TB4`N@N<i;DH{%5Jmn<P{H(FMS-xy2o+2=e!z~jPCSh`d6_bjXinq)~EM<xgm
zp_C4hFaxY^LsqkA?>3r&5&qX-Yqvnq;W0I{pJ50;&zx0Ejr2y2r)>^?Xde}iF!703
zZ!hdwO<s9_y&K>b=SIqdToss_eOAu8$rMm!uU-jQ!zyaCjw^perR>x`;;0C~r$_5-
zf`I(#FPBUe-g)aAL9<(>8`nO{@&%Ta^#+1Pug;dAshDL(Jg$ma>U@p@Ab(ite&)P7
z7DjN;sf`N&G2MZPX#*k#Ku;ml3Du=HgaiTuoPu=lvvdLl#Q{W&WNvr&QccGFZd?9E
zg2f>xuQG@Z><3Ri4Svd11c=Q|kAS`HKhFY_{7*i31<L$ILlaJNx^};*m=R>Ugq~qm
z!dYsp%qTp|NAO|}Aox)~1q%ZfV&th<UwK>0jVo2F1QIOCPj}se{K6;s|FJzg3WDrH
zc28@6$U@(prF`eSjkoG`J9<VBYFg}NoX%0Yt5ln`FaN0h0<y$vs!#&l!`SYziw&M$
zu|KpaBGe0P(jG$^q+dj{vyk4PG@35dV<X!|0SJ8oF+u`{2g~DA(%yGk@DU_AbYdWL
zDr=nW$@2MUosw)JOlM-v4qpG)g2VxH%_54R#~}i!%~;&0!y6@l8v{;IXjArtggA%~
zqYoq!jxPmVdp}52s>tAdnHQ0SO2CsS<x#oqCS<wZ=@*^%Bi7}M$Y#OIC@F)aD${#a
zd0n`X6Wuny%YqP;{}(bOUM^)9GB4J(bYc6bCcU`ReIBKU*X{L8C}AU{FKa&qX2ZjQ
z!j*{t_h@uRI#iMzi%OdYC4UdU3g-vC6uZr$dGRwxZo6FP7j}mGLz~PmB#v-W=KE+k
zz9@(<3G2ahidKZ97=5NqQ&dYYgauHuuyZ$oXEwcbKxz@}UC=>8P7A69B#$I7kl0a*
zTCnE<@wW%zi$@B~J+20z1iprJ{RM7Rk}jC1SSjpY7=7-`JR}dL6TDP+$1RCFs6EX3
zHYPEZ(6S1?9tpuFh5(6PXTO|Fd!_(cq-@km@Oq(C<RUUeI*|Irmc5Y&Qrihh?X^G+
z*suUw;rdv?r!FmrrB?-H@dikEUSI!&rsZ|D2I}edecDc|2;qRso8(tI>efGuc%C3T
zz(OV>P;!^wZNg=^V-)~co?@+J-yb{6GeHi1?@4S_+2*W%w>!8K`%OIvKxDnrY0yow
zv=zjgL&-fcKM^`$wNPux8gcm#N`*M(--YFFeoV1fBtR+a{hZXLOr{4Mu>hTbGKQsO
z&OPV9>W@k5{zsUD81}FrIY4Rw1!v0Bz$9KhkKj0D{(!-3(#-;uyd&DzGN<=O;iYpF
zy7RE+E8m20nL+ZQG)+h3LO6ICM=j=%MSvX~xTD}I_<xIYY1rl;|GNO~N&=Gy80)0p
zu$ikQvbP<)GD@H<BCMjTevdnZa}Ge=Ud_n!ZJw1MShU{_JkqK37F28|_|uGa%D0m!
z?#OkGF0-cCj`Z#CYtX&7rh;Kv-C&X*o}lwLhvU7;()dwYO~szNb*fPwj;xJ7{dKV3
zf|%{pY3$`jGXt@~GD81bVEI>WTS|is*(P{Q{(xJ&TDbQ5j&8hG%boOPmvVr6AA{fV
ztn0?>MEY#)JF}enlmwO9$(jr@rBF13j7!)oS=HMs<on73V$7UxnTMqq_~;p@0KyyZ
z4?$(SBDZ||rCO<Hg>WbIV~9NFIHC&(P!6ktuqc=&&|wwrm@tT}m<&)<JhGLG{W|4k
zh!Wq$pzHD2NBzm96<F^~x~Yu!1?5M%qIF%oplCyn=?i>Y8j#b(A?NWWb;o_*<n>?W
zthc=kQ+1D5TlCV8d3H1)Dj!(PPJaO3JZ7O^b3|m6uw8K~`J#s~EgV}G09F(Se$Rur
zTLSu^>}t3zP=!jXu^bG{rHoSlm4C2SEUeNrk7O!cui{q=zY2nt6pqAyTU2vAQ>EXQ
zV8EfWa`AH1gA_(it5(Z-l0dQCkqFOUVs*>{6DS*>p;-T@3j~XJbs#9wq2esHD?E`T
zr{D*t)afdii9}&k(Yv?3)r(p!TyAPY-SjCJJF(b01^Ne+mi0mmCN7nh4B4ucJZ`^?
zzPS>e*==Ouf_Vfvi;;YFw-a=$IWYHzc2TOHGHdep#f)7SAx);eIy<%#Tjh{@RA)7#
zHjYkbFlniP-qFT*LAChupdtv^KXV~<#nL!pvJ9za>A0y~mw({c(jZ21RzrtVj!Rhs
zk8&#LnS7Y(=XaU^L`NtTeBu`yWfUl^1)w+_J|!&qH<asg?fjjf1m1Tx@Q56eYSC4b
zjHJF|;r6tOiw5zK2x}R0wZg8AXMi(^3s$W1@@>*VSb?KL>eWz|;wAj6q^-4Y81&LN
z)^uTz37~u^Z?~;cGERpN#kpEnMo%njgP2H44{Hh9wJ_*Z+3Lra0w}q7mnP(}^6<qN
zjT;+Tel=1p=cyB|7TO0upC9xL(s<wez@s&x_XfwBSJj0&{0((lIh;Q^xF-$Yo4F<#
z_i(LycYd7X`n&^PBtQs|HI}&Y3!$=ug|s&=lii<b+ZitVKI2C5b)v+ZH=I$-)=fi0
z-|?XyvQlYqgrJQ)<#rFL%u>9H=z!#+0nvKq<u|e3A)k|^t1$bLpCmkeqX8DAroqjf
z%FlRuxkO6+Ysd@D*0j!mQlC>2f6YrVMW6<@b{V2^iD6_gGB^J#b<^n3%B_v|G??Yi
zVlbk>f2Y>nXh%G5f1_V=@FF1Q%Ge%Ucj7FDP)-yg*?eXxUtk0C{q(?@_UnCnr$h~c
zjZ?a`e(62pNel{jM(8^dmhBc$3fK&Fj%gp0Bq9BY9Go*#!RTEihoiEW(x5#0EN|$J
zRED1pHidjwyB%mCn-;LK!MFb5qOG%2-gD(eT`j>gDyM&ICo#_5+&OnM;31ypx3F_N
zr@1jIFI60x#&bW1>1L}>8!G$Px7R%B{2MusSr+;>fNRc{tZLzD_cH%f`dc~fhUP7)
zmv!nHvykH08Rqhr9vpG?K}fNR`3x9D=@Y}4XYWI9@8w!ZQ{tY(lME>^kixhVijSBr
zg`@Cs<d}fMJ8Ix`<E)GX`b?}(RY8O0!?ZRNQ$TUmsOl-g;-Q=qB=Et&xlQ-*5Gpfw
zLXJA8x9@4<wIkGSBoJMkeN$6K^FUHLBk?Hyx41(Z8TYVo2WKx3l`oSKp+&y4W~uKk
zPV}tgT-=DRDA57;wzC@VCVzJr-)Oc@U@9aZ#jL7aC)qH|`p&et5c{Fi*GSnZnF6h1
z%?ezAxF%cV%Pg*}&z}pWXi`oikZur{6%$jB`Hk1nYW^2GI__?ySG0?0w*^`z6Eaqq
zvPJn;;YGe!GnC96e`Cd}s}Z<=;}MgFRv^!ZvbFVD+sqAi?DpkH)i*Rh)Pp3o^si;A
zH7h9AlSG6%Iazg%o>gN(K@Kt-Ia1BsM_Xb?4}@4lnoEUg!r<iFU;g|J`Fk<Ke}XVy
zQKT+XF)xA2C!fdkyC#1#qv$V7Lt88$Sag`Meu<bbzh5;=@CFvk{x<nuf$1S<(y(-*
z4YKTUX`oDGt0jM-2JqX5N!yB{pbkWgAtN<1Oy-1>jvG<xGAScv#h-xO)>V6R{g<GW
z!cbp84v0#q_5C9y!IHZJoc3918Ii)iv(!0~FMGuqBKXh0GU7yh$-@N{02(P^od3+w
zW{dA}S5MWPi1Azj;~wF^eH*b-FmZC)Ne|IC;w1fHltD)GIP1WS0?c;f$Cvc43GIm?
za@+&ragwmxT&mwy7zHu%ZhwW=iw-dLZ&;z0WRv{;D&MF^9#dNzrpozwNiN^HSVWN=
zi0Rp^3MuS(-mWDaWyzTr{ZSu#8bbAJYKvif)V;*R+)8g+(tbD?rn>)9s*h2CCQOP<
zq%6=R=!F0YqX7vyo7tVM=uR0f4CN@qhV9+1%4fch)SrykMBT{Vyn*p0#I&uXh$-wu
z=?W)vw!ep5hop*Xij<+pm=gnPeo(OV%EV^LC-z@hIZ~8bMHzx6@gK^%!lQ!Zt|ydS
zA<cQZ?{DbIT|--Hl>-0H@XaGkb2?<{781|uTvQZ<VjEW!ei7f_G}_qc^_XKaQnv9)
z@1j>*e+-eP=`Y*eCGkaaY3&oMD32yW`FWF$hcCAzt&X1+4S$%L!-b(|0BZ2}>Ha+Q
zF7>k_9ZlN-)H?GNEFr$;)nqN*N^@X9EPMiaivCG8G8hKVa#kzo==;)79TB!Rg~?SO
z-q#=X35tt%+1HF;s9}DOx(4P*y_Tq716k`E6Jp1atb!}CfmBpYh8fRJUf18kiuBO6
zGn5bq^C+|qKl=GgoPL1Q+?&h<0v73uUAP@fRjbGnC7|PK*sKuBhPgJLDtrV~IX}#6
zvzjQSMN&pCEU@K8@Afk?I4fY3(2nil8SB1e@1!XMTE*-NIM6^9T#aJ>opaHpsm#(^
z;5{D*xRDN!yV1PEYeUy>x4z?|z<c(H19IDcG`LSLHvtW8`#c!RcBC6Djjv`FFImW*
z4%(SnU5h3cx89G}s=gUF-T+C1(Nzs5tpF!E7|8+ymXdFQ{H(E83@PT7XVAhRo$&S(
ziE`z%uauh3QT!eCY1Ha04zyZbff}rlS4M+Z!DM}vP{~nF{jPD-7eu~!l~+GoG_Ajs
zoWpEviBt!KkGfk{5>7vb@%@^<LGjAN!qTsY43$i(`jb9rN);iM=#==;UB7#Y08bxB
z_G<!(^>$MPy!1&*LU;-36~VOZFK7!c?IPrHyaFF<A9(Z(PzR$Q{d#wGFUzQh>aAH-
z!^eY<HJyc&Bvyq-gF|aF)y8<;V#ov|lNXU)00E9>goYNqCl#T1TS}d5#&p}Nphsm`
z`1kl!HNXfNZjSK)H6ZTdkX&E*1!=8mx0QMBtmarg(kh}Gc)EEzk;5{@2#tcieD!Vc
z54@uYMlS*Mcmaugr$=ew5nn^gr|~_bt2?m+Gzu?DiSN!rkeTEYXLd12;H$9=td0&F
zBuu30R4@4T-|i54wI;_1SNQE|=7jW0s`fbWOAI%a0Eg^GX2&2A;XY%-iZ8=diB3YM
z_2Uim{LFoLtMOA|s=mkh1PhKKe)p>=W{-@AFUG4kDi426h=QU0?Z_JLYb)HjgH|q-
zEPMwbz0`MV%?0hyA{Og5)!QucCsY8<{rYwoRI2Y{RGhTq2nTKnSX=(XBsc~J==<3b
zBH}rb>p^e7ceewlPp_aT&cb_z=?y@Fna2-gF<*B!OiI<vN#<q`tqxU&C-v)Sp~!6p
zWs%UPUP3(8YynnY$v^o^bF@(52dIpZ;36vMS9@CQbEP|-%(a^gv*{l?pC~g(6z)I+
zcnu(c65OB9jJ{@GMgR=|!Y^{27CZ{Ykjy_Bn_Gfy9DUV`i<G4Yz>EL(n<$0k!gRDj
zJ|}4ry-L+<`E;+IUZNlyin<4FKZ)<&HzEOU3`lGRnz9rs^1as|N?43{y_bRXY`zEF
z34GkNlAu<85?|j32AQ-PnRC{01r@CoBNfW0y^i2TqDEFuMg5wzgL`(rCMEL(^107N
zc*%|90nJiREI~g|XT)gZ!I9%qld2{FqO%$_1VRaL93$P>`f<Q;twvJy#KV)Y+`*aj
z7V08P>a_B=@24zw`+vvh;@x^%luaJI$m+ZR{FA-{1WZP_R5~zMV<u-1GZuFa7_+94
z77t0AVYo>y61T&U{6SUf9ku>*)MKTO-v{5XP5>0@E5&||M0~e`1>;#C9Pp9S+~7CR
z-R+l&f=a;PK%q4GUgSN6Vd26!=b~z8=1`oGFg^K)e>*$w)CA1f$>6vlV8;KT=>sGc
z7_`2Rk%R#_&<IL2@QjA>Y`}}EbAHbI9&i0s8Wz4E=i?hMQc*bw^+oP5449HoZXE&*
zZpL+IXiN1?HGJu@*r7`jA@QsOz*QDsJsTJ)@Spe^O=Uj|6Gp!<6fonh;Wr%&Zkzwq
z&1nXi1-2WLOYT$@r7ujwIW+gXgHf~X?(V@}5G_L!x8Bq@hCRpIOAU4wMvLG+6%*hD
z1OD(%vx2N12{V-Me>&YFMhXtQ9dKCV%8igpCI|L*YZqIq>e$eLkE)Ng6pZ}eP@hn}
z031Dym#>k3-m2g)GETDf;)vIb;BI^KVkF4xGsZ_>G6085`N!d$05<@(Q<>N1f`$+w
zY7nJ9|Bn!5CtTm@ouAnT9T~KGS?5M9pOQ4g2Nu!Ac9x7b@i~KiHAWeds?@0|h)L$t
zBRPoWJHJQ3K{QaTcsNc21f4$OWpYqa1!WX?ftp)^XNO^qx>i0@;<uv}nTZs)UBPX8
z3*_8^ii!S1lBbYhCH?|4?)tyYr-WT$7=3Hy^gF0X>J*~3(>UM1egdK?(rPJTyyD{<
z=cQ^{obHCYS3g*OWhTIGR}=JM@E=zkKa28bBlr(_;XMhHQqI7+)Bcr{ZwFy+WK-#B
z1ecV!g$*pY0qiYRVn1GLMaAIqp$TxX?f|$hmjpc!1Jya(jn1>nC$+&v!oo8=2;9Q=
zy^gz%3A2UoR3EEZT56UcG&XwaBCnXq0~T9(EO}bCBn!}tOFZD@a)V9h8vMEIpKc;O
z1hj~%%7ussUZB66V=13v)yEY{U+R~t5BkSDZ^n`ujg62KL*#=LWb5fgqTI3o;Ndbq
ztz@@Iel{BKni~&XCRtgy(0o8)O`i%xjmf`22&U?j901q>$WQQceDnO%x?+uG&kM7E
z$`xVU8MrTmD?st_dHBhFgno`IVOo~kO)^yI&JkK-^YfJF1Zsy!{nszW3p@y+qtf~r
zT!u3vBhR!1Xhh%|c~y&pYweDKUhj1vntoTWfC&HyDDDsY8?!t#4jD)S(G071e)=un
zry{Yy&(epGF(Sa35kN*R<#<T~2^Nymw8}n0KPW!v02x3$jp_^2fS{AZA@5*n6ebAH
zh!Q#2ctpU33O@iB*iB3ImyvOnIC)R*K7W<1e1MSfuf(AVN_{Ghz2PN9SK`7nDfceZ
zGEk{#Zh&UyR7{vOat@fObM7buO4AlHdjyWT_;_8OFZa#iwdp@;k3Fvz0sjVj^A(KC
z>1<7~S$OnJiu)r#NFau2=ilQE86V@9*%?X{`K*+N(IxZDN@Oi73&Gkxh`+g__|%a3
zXnd9ap02P2`E#2dW3pDd^x5L_p&@+9*b$g%75wn2ebW7r^6{i_%wRG{yz4Z-yAe+o
z-m}6_nSl<tfpYIIJK4J|GTKSAJ66cQHcGz*N34DYBVN^z;Dui}P&Ov6omNv*k?~p?
zRyNw!kE|>5S$Cg(8g#L?ymu~ALLL-q;Ca<a^+h1+AK5^yFZp4RH%xYu6blqQsD1@&
zyAv>+vs05fewu;B-YtgaI^dSa6wk1sd|WArQk`01;TMSG&*x)BayYje%?b}W>v&S)
zJ1scLc#!|VHx_hn0FXM0Fkow8?)j3742kz_X~?-vO7FzXXV|BrUG6HuQGbH4HQCCg
z=-Pfw@s-D7QmvJe;?+`7;XZqDA2cfm`^@7BoDA#2U&c@0s2WW9C%^2RiJ^b+g5GS3
zBgifnerTV58FXw_dO)~|1s1{81{D!X;?qTt$j5|WX)o0E)Z*gHEvF~c^;tBb3e~-Z
zj|oH%fMqWLmhHM@5^0nAow`zeOxJ)Sv~q!IDUy5xe|sd?)vsm-*h)cT5sFcxDJrk~
z$6`_Z7n-$|gCKyQIK+5HH#`Z4!T)k{O<9uUrKWguY0AA?k<J`|6r_MayJIBPM`(rt
zdu0=7LnPjrtNLa<P#M&B0gSWuDH%eL6o5OP!V>bgG=N<Ve|-FzjvBLhIqn+DMDYFW
z(3Igm0^0@Q+!Fr*qWFifZVMjZZVl`|k%-&V1oq@@0WdMF86J+h$6G<?$P=Qi&sHVl
zBU+*Ss5ifdpqHSi^<g7DP~2aj&S#sKDUhH31~_`*4rNu`zXa~ga1TQ+H2+gC_#Ur?
zBv_STss(~<OH+nsaLRq#4bxa$TiR=9Xt5_?a!}!o3T`V`E&%ngGYkVm=U1oyT<`!P
zH<19{2f7l*Q|%dX_nQB6mkG<j)dM7XlGdM-+3t6BJFw!uYyZS~!k<4!{W-TBsD30-
z5UQT5_6T1lPyq}MT&dge@`v!$bJ;R3A&)5t>;DfV1G4WRUTZus)R#<>9=hR}a_5`q
zfxkVQR(h!mu1w(K#HiQ$(eG6RD~=_HP!hc8bP^h|%4BzPk}@Vp!SCSM3mD<f5};Rt
ziU5w+SikWpEn!=^s5+Be?2!A)tS2`G@U?O+2XN*&YC9;HeCCDL{4yMJAA4KY>!nK6
zLhCpDh5ZekMMpR`_YnCV{~^~1SnW+fI%AT`l?8z_=;0?b&GN)ST$>E+py~a<>)j#-
zkWMT|zoXun4DfFa)*U_dZFwt`n1N6lgfj>6|JT8l$3xkDaSdfq7=w^~WU_?8>$Q#X
zGL~csFCns%Y-P_jsl-%Bmh2>3+4p7A%qvpHl6^O!$iDAmp5Np5|MT2?Kj)r%?>XP|
z{hs1!{Bu@Oo@5tQHSh*gI)mu?yn`bPiF)eEs`bd88X2u3o3EmD{bjwsOFj;?&2QLu
zyk^~PQUoETLq8CVLiG$aRF%k)$G`JVlyEBfIRq7H*$~RvnEJxZ<Z#3KXH*ezi8=M7
z2k#;_aelV|biEp#*a|sV8kP0_Lt5y!nv8%O+#kbNc&$6;o=kLrcKYFO&@i8SVRu^*
zW1jW#NcQ;y56EiB;r`Dg8#0MxNaTO|_yFG#pG8Q#_A*HR3^vRC0X1*2o^xElqyjhR
za^7P$UY{ipFN}q=5(gjHfaEL3HjV$JQKmbX`1vo(x<w~0J)rL>#MUT2qa$yzd2&+6
z-44TzLfl?}4&QTVXCbtSO^-XH52ga^?fKHw$zD2!l1;;Z%I@=1k3xC+@iPJ>a>E-r
zU}dZ2#t{*B=ht~^JV>0#mGHZXfL4vgMzeKOYQK><p3)(?!3y2bFAzw!inTZM%d**x
z8*@S4?4fpQu>jJdX9-)AW$pyZy6L<ABtYPk<tvtUqTSwU_DLrS4`6bm4?Tmx=a@E*
z%m9y0S;>g!x{-WO&b^BmjH&!9Eg))lAx*6D4On)HO1F;PFd%XzO^ukL0MxD8`sk>k
z_o(kGDTdGhb3(dTMgNF>R|!tG#|kh1sP8X7%6NO>6mFzdfA=1Z{5^l&pLxps_YIZk
zS5RrS*CNguK^_WQ>~rsU)%xw`l-^QkE%I%SER@`v&$ANZU?HZI2gnwb*Ur%s^nkn!
zuA)*Q<sg^*W_+n}E%8+h@bY-sq|l^7AM%R#!SJ;O;w!GA9igSG|3yww_V1o}&_l;S
z#`re*Sn}?@1>=D5^;4swhg(?7z<gl~Pvi^l)Q3<KdPP(j^Y$<VxsFv613SYYPVKah
z;<_XUm0xKod~JWulLm;0mfF{rykZSt1+!M%UlcO!Y1$v6&3+WclkHqsL8gDNesbCv
zziNMv|Awy|{eR~f{QqEk78anl0H!5sW)E@Mx0~yns+ozJ_g5LI-OsJ#w?EQ+cZeEj
z|7|P*R4%aM_XL?1!`~fLdI>F?tnF)T4_dNU>9uGKdOdtcZ-*3bi2O+FlZO6y%mJ@O
zW<*z8so^6AE>w}qWY@7L_Y{w_;)hE2EtC2JW;`U!Tm0cWM8WqL^Q_*f{0*}9QC-yu
zxSAQ_G0{ewB)D|{>j%v#<k$w*BV7Gz10O+!%806FEDR|CMG_8SS90#9*49osmptJo
zAkbM1+$nO|YGv4UEL*3yOs;$!3osskn#n>V^fIo9C_kH_Ze-xQa6nPl+GOFQc5E#E
z61b5QzbG?;5XC%Dp%&}Iwv!QwpA2iV8K91~8x%H-A<$`Qm{Gki(dQX~Moc;!g8SH=
z^|lu!u8Pbl>SDPwIR-{f#bB9fsRc1@at-tjN3XytP^t8Z0QXKfaEzbEv%<Dk=wT1^
zT7>vmMGgXbBW-O$VtAo*b0vj#3{BrVoft-RNvMIwJf}c$sQ@%b1WQtCPIU)<92%T|
zy_#lu*BbsGckw~%UTQ9H)E;Q+P%dQA&OOc6s(Y<|3XOmh_Z(4Q<+x`+5K`7%w65^F
zfhQ^soG&ymj6MuIjrXw)ciTG8ok}9A^;`dFekh)X|0#|P&cC9)C>sE+n@OC*Yg~0T
z=o}D-T+CCCy`L23*Tem{y4gz@Pn;$hWv0$<d~I3PR47HMg{SaTbOwxYJ(i@DvWAcB
zPGya|nd)9@k_9DDC@1Ia!@7J{ypmvgTpnUir`vn8Rr>7|XrH$t^9M>=c^&bbCdnzy
zS-JN?rH8R}8(c(uLAMS~o28zZH8ef&OYDGhB7QjObqSiP!$8I8gCI%s=A2&7gcF(0
zi}{<_>$<`hqf%APbn^r9$_DB{BeT4)$|jV-_q;WBWZBBPqX9q}C{vA@)XhtYPiDq4
zPL%f&bo$WcjcWr9lN9GbrEHJJk}@Iq28L3eLfiYm?jZg`AsHN(KZ^ToT*W*)By+5t
zbC*B)6YQ|Ch7kaR&g_LxM}i9#SJcPh0;2Pyj?nH_#8tts`kZSl*`=;DXqEii_&Q8%
z2CgFe3+p+{-(PS<S{NTuyEDNNI>ni47QuuHBq#e2IfmT}uV*lyvvecCvze!l2VDbc
zo@S%KakGK%tSx$Ay^f=@*T^2L=bk4;{~o@1*&IRYm?Cp?hu+8hRUXKP>5NZ~cw)HJ
z0xz%aHuB~g+~;rDA`M}F?UrKm)INX_RmyJnU`cae!I$CY9haVFRp`0HPWVGaD7)59
zB=HIs_i$!BVEw+fgE6?NFr_<TjyJpuz0~%9#A-+cB3lGLd7f|`;tZCJ>y+I1su#7p
z7AfMR(W}6xp_>2eqsNah#p+p8>MOM8>}m8sGNm%z2vH@)upn1YUba;?4HG3u>JU!t
zia^{44X?f<7__%C1OSp?&A|@!lY)NJig1-&ge}z!KPn2QLFGc0=Rioe+`sAKIquX5
z$8pBUKByN<RJVWx%Cteb2|jJ<@nI!+Qw#sHy6GF}-9x})p)J+{1yR%aDSjF{5@F%x
zyQLpc+~Mwy>8<vqylsI^^PX2bb!44zP?~c#%S~@Jm!!@MKi11wn;O6Ds&MJU0Tc7H
z3(?llI-Eq<Q#r*<S2vMcLYfpWSc$@Xy5(B-<!1*YOwKl?@>_k&hTvO0RI>%#^4ibN
zfL+t0OHd?V-<n>w?QHLQ)RzWvFP6_<9B$?vmuhUj1D{=@`blTEF+jty9wVO<Hm%41
zMC8L?SBfdDBHUaX4)DfbaNG$)C)nay%7+WWuSImbz>E0WOf0BZR8|{4!Jg@^Pzt8f
z(mNXctT2ovUb9DVX;7;>=TBWnkz2wplk4eck6KFja(!6A-G9fzVdX-PGY%5L(5to;
z-G5B(=j7&qN8>i&IS|md3TU-GzsY-<_gUgJv)#ROxKIa~S>?jF-s^3ne^2H7vHVlH
zUuGAUKRHVway-_ILz(Tz!pz0K3DD7<&pPV`aCs*p{~SYQBG&sPb{4DxmH#tk2t-HB
zm7x36In+nJhB*<=ffcvieHsgf7ZNAn-kG?{-f0<tRxiwOe@P$gM<f`1A+hBj`NkET
z=NPTs;-2B6&@?Fs@Kk}+Uo05w>z%8aaMSOC1u+{5-_iyFLl5wOqlcxA!a9c3cApvw
zwQ_`np>Jl)bZ?_8(8+slK`8457e$cLBJ9&yr5{BseIYz05kF}&lmP7=5kcFmH#SY)
zBC5!+CEj!pgj9*SDzWM(wB;;6<-V9&b(=PAHv$%bep6MVnc2rcEp&GFQp6K3&SwH?
zq%K=^SZ?+w`c0|B?-jI@)hXt0RZPF3tCG6Mg?@wCT67S?rv8~%!!++|au7ZOAi$xS
zlT<<<WfxNL*`$tjJ=pbpF&A2P&#=9Mr)ev=Q2<d)^B2fR_g{ImS^5j=<&RxH!LCS2
zPwwll;YTbkG*|<8yvD8D``&S|*nb@Ea4m7AuNZKCv38N1T-GNz3pK8t({cK%v#y~&
zAuS@`a49TzlmBMoPF;PiQmblhZ(BooRPs?}65xHQfFE9-guBD_n55)XP}Vy?HRa1V
zwj#DIqG`BGFx#<52AlV>SD7S7w1F!<Q+u;R%-0dlxh3dC;e2reHv?LF*ACtS4!fMZ
znoLy_=fl29`fw#m0&QkoHtXcG0WTw5G;|f9*52N|V}AO0u$=Flx+w)|^Oug23`dlN
zZb8B~5&;nxV3Q;wcOEzk<%8tAwBR{UKx@C_RH>eTrL`eWo8{v)?uIRRQ^cU`aGi+U
zRk2OW+-oOTQ_8P@(H2T_98rHPGUM^OH3|_n{opE8(e8;v_O-;&Kp!zRW2s9J(@RTQ
zDsG&rdy*tYk~m871;ElJ+o_QOh_%}*D11b$?SrwZEp>82m_d|pAlgd>=8}*!E(3v4
zapvMel#1~)*UVQnqagXPu}R-!G_kLehaXKLj~{$qot&d%$oZFVYHs^qO&eoTAxPzS
z7t2K)I(MmI4B9SoOJ3A9x5U}$q3x<@AqSY}^#C$fFHd>~lH$Tw-T5yYOGrSj3VH<R
z<{dDJUnTzVI!g*O0WIOA$n2EnZrw+LK4uW_AZduRvkRE$q{i44Dt$$Cz*~K9UpNWY
zhVgvCoM7z$ZI1^|=`f!8vqPOwQXC+!fSC!Jn<XZLxXKpK$Zn63uk$;a0(|z^9{#ey
zbrI?{mza}MKZ9)X@6x(7_6#sT{+;xWxZWNVM^{?DB{yo}hLF}JK`*jfHHt*&P)@pj
z5+c)uq5_XF@5_$Ih}3X4|0)^4_s`2d17BY8pQGTn{3<oZubwgg8cyGlzMh5u)OAIr
z9N5yBeYA}`;iH;sggf=C9|c!DVYeDiygRfp2EPO$w7IOmF#_9ItipDMKWfcAF|_%e
zC=c(r%DInyN*64@gZ)|c;+JkfPS$~c;F#u%v=ZmJh`NP?@L>%zS;-8c#a>f6NKE;J
zYXVPWNJy@O+@Bk7oL!Yl{fUX~cQaL3qsWZmamQ@H3^J5XGjbBT3$t2C%o)anINlH{
zJkr1vm0c<hvQGCf3HS`o&`?G*7hJ{fn{N9fsvuF0I&<6!F_z(-LVb);vqS67T^WB*
z_Qg;vRCq^#=+7kPHKT*K6~_WN9|pHLO$x&LTJa20*|X)qwF2bAnaWB}F!q;Gcq%6L
zmZz)-*Sd|OHAi3PjIyaK4<{&3--m@XN;^v?q@>5kSlk?RdA_G+#(f#wuJ3t4ZT`BC
zKcM%SCg9l$WX8p~cuv`0=8V8gOgaiOJtX5w8f;b&pRmqkjcT+nRaXxUTn|tl&Ds|F
z)^w*3XIh)s;0^MUX$J#<a718@{lhk0hp$j7Zj#&PhppZ``mFWN4IN`_g_gsU{{g_Z
BdA0xm

literal 0
HcmV?d00001


From 7a7b0c22ad28e44a2a83f63ef521c53f489dd8dc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 7 Jun 2017 22:31:39 +0900
Subject: [PATCH 0710/1644] ARROW-1048: Use existing LD_LIBRARY_PATH in source
 release script to accommodate non-system toolchain libs

Tripped on this again when cutting the 0.4.1 rc0. cc @kou

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #738 from wesm/ARROW-1048 and squashes the following commits:

f335c09 [Wes McKinney] Use existing LD_LIBRARY_PATH in source release script to accommodate non-system toolchain libraries
---
 dev/release/02-source.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index d3d94af0468cb..508141e7198b2 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -74,7 +74,7 @@ cd ${extract_dir}/c_glib
 ./configure \
   PKG_CONFIG_PATH=$cpp_install_dir/lib/pkgconfig \
   --enable-gtk-doc
-LD_LIBRARY_PATH=$cpp_install_dir/lib make -j8
+LD_LIBRARY_PATH=$cpp_install_dir/lib:$LD_LIBRARY_PATH make -j8
 make dist
 tar xzf *.tar.gz
 rm *.tar.gz
@@ -114,4 +114,3 @@ echo "Success! The release candidate is available here:"
 echo "  https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
 echo ""
 echo "Commit SHA1: ${release_hash}"
-

From a38203469530fb0ce56be71d94a6848085ab2c1c Mon Sep 17 00:00:00 2001
From: vkorukanti <venki@dremio.com>
Date: Wed, 7 Jun 2017 17:01:30 -0700
Subject: [PATCH 0711/1644] ARROW-1101: Implement write(TypeHolder) methods in
 UnionListWriter

Author: vkorukanti <venki@dremio.com>

Closes #740 from vkorukanti/ARROW-1101 and squashes the following commits:

e2d097d [vkorukanti] ARROW-1101: Implement write(TypeHolder) methods in UnionListWriter
---
 .../src/main/codegen/templates/UnionListWriter.java      | 5 +++++
 .../arrow/vector/complex/writer/TestComplexWriter.java   | 9 ++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index d6e2f9fd38d2c..9fe41d0d96335 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -166,6 +166,11 @@ public void end() {
     writer.setPosition(writer.idx()+1);
   }
 
+  public void write(${name}Holder holder) {
+    writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
+    writer.setPosition(writer.idx()+1);
+  }
+
       </#if>
     </#list>
   </#list>
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index ede8d65f07507..5a9c80dc124a2 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -42,6 +42,7 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.holders.IntHolder;
 import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
@@ -216,7 +217,13 @@ public void listScalarType() {
     for (int i = 0; i < COUNT; i++) {
       listWriter.startList();
       for (int j = 0; j < i % 7; j++) {
-        listWriter.writeInt(j);
+        if (j%2 == 0) {
+          listWriter.writeInt(j);
+        } else {
+          IntHolder holder = new IntHolder();
+          holder.value = j;
+          listWriter.write(holder);
+        }
       }
       listWriter.endList();
     }

From a44d5842ce372d787050dc7aca535ccaa9c5fe3b Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Thu, 8 Jun 2017 16:40:41 -0400
Subject: [PATCH 0712/1644] ARROW-1102: Make
 MessageSerializer.serializeMessage() public

Author: Julien Le Dem <julien@apache.org>

Closes #741 from julienledem/ARROW-1102 and squashes the following commits:

284eaaf1 [Julien Le Dem] ARROW-1102: Make MessageSerializer.serializeMessage() public
---
 .../apache/arrow/vector/stream/MessageSerializer.java  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 2fd93749976c6..05ad92ded1d52 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -142,7 +142,7 @@ public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
     return new ArrowBlock(start, metadataLength + 4, bufferLength);
   }
 
-  private static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) throws IOException {
+  public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) throws IOException {
     long bufferStart = out.getCurrentPosition();
     List<ArrowBuf> buffers = batch.getBuffers();
     List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
@@ -171,7 +171,7 @@ private static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch)
    * @return the deserialized object
    * @throws IOException if something went wrong
    */
-  private static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message message, BufferAllocator alloc)
+  public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message message, BufferAllocator alloc)
       throws IOException {
     RecordBatch recordBatchFB = (RecordBatch) message.header(new RecordBatch());
 
@@ -222,7 +222,7 @@ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock
   }
 
   // Deserializes a record batch given the Flatbuffer metadata and in-memory body
-  private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
+  public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
       ArrowBuf body) throws IOException {
     // Now read the body
     int nodesLength = recordBatchFB.nodesLength();
@@ -296,7 +296,7 @@ public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch)
    * @return the corresponding dictionary batch
    * @throws IOException if something went wrong
    */
-  private static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
+  public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
                                                                  Message message,
                                                                  BufferAllocator alloc) throws IOException {
     DictionaryBatch dictionaryBatchFB = (DictionaryBatch) message.header(new DictionaryBatch());
@@ -373,7 +373,7 @@ public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocat
    * @param bodyLength body length field
    * @return the corresponding ByteBuffer
    */
-  private static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte headerType,
+  public static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte headerType,
       int headerOffset, int bodyLength) {
     Message.startMessage(builder);
     Message.addHeaderType(builder, headerType);

From ae6142da6171c9d61c2e2704af32b4da42b68f11 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Thu, 2 Feb 2017 18:53:54 -0800
Subject: [PATCH 0713/1644] ARROW-1107: [JAVA] Fix getField() for
 NullableMapVector

Closes #743
---
 .../org/apache/arrow/vector/complex/NullableMapVector.java | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index f1f86404d20e8..ee95fdef59401 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -41,6 +41,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.TransferPair;
 
@@ -81,6 +82,12 @@ public NullableMapVector(String name, BufferAllocator allocator, FieldType field
     this.mutator = new Mutator();
   }
 
+  @Override
+  public Field getField() {
+    Field f = super.getField();
+    return new Field(f.getName(), true, f.getType(), f.getChildren());
+  }
+
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
     BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);

From 06c26a2cd6d361a2df84e5ab4146c83513d3ee46 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Mon, 27 Mar 2017 14:24:01 -0700
Subject: [PATCH 0714/1644] ARROW-1108: [JAVA] Check if ArrowBuf is empty
 buffer in getActualConsumedMemory() and getPossibleConsumedMemory()

Closes #744
---
 java/memory/src/main/java/io/netty/buffer/ArrowBuf.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index e777b5a6a5d58..06272a7bae960 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -796,6 +796,9 @@ public void close() {
    * @return Size in bytes.
    */
   public int getPossibleMemoryConsumed() {
+    if (isEmpty) {
+      return 0;
+    }
     return ledger.getSize();
   }
 
@@ -807,6 +810,9 @@ public int getPossibleMemoryConsumed() {
    * @return Size in bytes.
    */
   public int getActualMemoryConsumed() {
+    if (isEmpty) {
+      return 0;
+    }
     return ledger.getAccountedSize();
   }
 

From 0e680f01c8035ded17496a369cafebfec9968392 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Tue, 11 Apr 2017 09:40:09 -0700
Subject: [PATCH 0715/1644] ARROW-1109: [JAVA] transferOwnership fails when
 readerIndex is not 0

Closes #745
---
 .../main/java/io/netty/buffer/ArrowBuf.java   | 29 +++++++++----------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index 06272a7bae960..0328a16719058 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -18,19 +18,6 @@
 
 package io.netty.buffer;
 
-import com.google.common.base.Preconditions;
-
-import io.netty.util.internal.PlatformDependent;
-
-import org.apache.arrow.memory.AllocationManager.BufferLedger;
-import org.apache.arrow.memory.ArrowByteBufAllocator;
-import org.apache.arrow.memory.BaseAllocator;
-import org.apache.arrow.memory.BaseAllocator.Verbosity;
-import org.apache.arrow.memory.BoundsChecking;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.BufferManager;
-import org.apache.arrow.memory.util.HistoricalLog;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -42,6 +29,19 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.arrow.memory.AllocationManager.BufferLedger;
+import org.apache.arrow.memory.ArrowByteBufAllocator;
+import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.memory.BaseAllocator.Verbosity;
+import org.apache.arrow.memory.BoundsChecking;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.BufferManager;
+import org.apache.arrow.memory.util.HistoricalLog;
+
+import com.google.common.base.Preconditions;
+
+import io.netty.util.internal.PlatformDependent;
+
 public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
 
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ArrowBuf.class);
@@ -245,8 +245,7 @@ public TransferResult transferOwnership(BufferAllocator target) {
 
     final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
     final ArrowBuf newBuf = otherLedger.newArrowBuf(offset, length, null);
-    newBuf.readerIndex(this.readerIndex);
-    newBuf.writerIndex(this.writerIndex);
+    newBuf.setIndex(this.readerIndex, this.writerIndex);
     final boolean allocationFit = this.ledger.transferBalance(otherLedger);
     return new TransferResult(allocationFit, newBuf);
   }

From c6cf1244227f9043561463116e24abd1d5713a85 Mon Sep 17 00:00:00 2001
From: Julien Le Dem <julien@apache.org>
Date: Fri, 14 Apr 2017 10:23:21 -0700
Subject: [PATCH 0716/1644] ARROW-1110: [JAVA] make union vector naming
 consistent

Closes #746
---
 .../main/java/org/apache/arrow/vector/complex/ListVector.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index c2f55767ba6f3..b5a27b51f2bc2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -311,7 +311,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
 
   @Override
   public UnionVector promoteToUnion() {
-    UnionVector vector = new UnionVector(name, allocator, callBack);
+    UnionVector vector = new UnionVector("$data$", allocator, callBack);
     replaceDataVector(vector);
     reader = new UnionListReader(this);
     if (callBack != null) {

From 11deee61aa2fd6782df1f8eefa15938a7676a546 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Tue, 6 Jun 2017 15:19:00 -0700
Subject: [PATCH 0717/1644] ARROW-1111: [JAVA] Make aligning buffers optional,
 and allow -1 for unknown null count

Closes #747
---
 .../java/org/apache/arrow/vector/VectorUnloader.java | 12 ++++++++++--
 .../apache/arrow/vector/schema/ArrowRecordBatch.java |  8 ++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 8e9ff6d462c5c..996524521cb68 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -29,9 +29,17 @@
 public class VectorUnloader {
 
   private final VectorSchemaRoot root;
+  private final boolean includeNullCount;
+  private final boolean alignBuffers;
 
   public VectorUnloader(VectorSchemaRoot root) {
+    this(root, true, true);
+  }
+
+  public VectorUnloader(VectorSchemaRoot root, boolean includeNullCount, boolean alignBuffers) {
     this.root = root;
+    this.includeNullCount = includeNullCount;
+    this.alignBuffers = alignBuffers;
   }
 
   public ArrowRecordBatch getRecordBatch() {
@@ -40,12 +48,12 @@ public ArrowRecordBatch getRecordBatch() {
     for (FieldVector vector : root.getFieldVectors()) {
       appendNodes(vector, nodes, buffers);
     }
-    return new ArrowRecordBatch(root.getRowCount(), nodes, buffers);
+    return new ArrowRecordBatch(root.getRowCount(), nodes, buffers, alignBuffers);
   }
 
   private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
     Accessor accessor = vector.getAccessor();
-    nodes.add(new ArrowFieldNode(accessor.getValueCount(), accessor.getNullCount()));
+    nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1));
     List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
     List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
     if (fieldBuffers.size() != expectedBuffers.size()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index 6ef514e568d2d..6a716fa138a7a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -48,12 +48,16 @@ public class ArrowRecordBatch implements ArrowMessage {
 
   private boolean closed = false;
 
+  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+    this(length, nodes, buffers, true);
+  }
+
   /**
    * @param length how many rows in this batch
    * @param nodes field level info
    * @param buffers will be retained until this recordBatch is closed
    */
-  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers, boolean alignBuffers) {
     super();
     this.length = length;
     this.nodes = nodes;
@@ -66,7 +70,7 @@ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> b
       arrowBuffers.add(new ArrowBuffer(0, offset, size));
       LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", offset, size));
       offset += size;
-      if (offset % 8 != 0) { // align on 8 byte boundaries
+      if (alignBuffers && offset % 8 != 0) { // align on 8 byte boundaries
         offset += 8 - (offset % 8);
       }
     }

From ac6485390b04786959a42d55cd28d64bfbb40589 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Tue, 6 Jun 2017 15:19:42 -0700
Subject: [PATCH 0718/1644] ARROW-1112: [JAVA] Set lastSet for VarLength and
 List vectors when loading

Closes #748
---
 java/vector/src/main/codegen/templates/NullableValueVectors.java | 1 +
 .../main/java/org/apache/arrow/vector/complex/ListVector.java    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 4aa0076dc8bca..092097bb2bd6d 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -143,6 +143,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
     org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1,
         values.offsetVector.getBufferSizeFor(
         fieldNode.getLength() == 0? 0 : fieldNode.getLength() + 1));
+    mutator.lastSet = fieldNode.getLength() - 1;
     <#else>
     // fixed width values truncate value vector to size (#1)
     org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.getBufferSizeFor(fieldNode.getLength()));
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index b5a27b51f2bc2..4ab624f3694cb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -115,6 +115,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
     // variable width values: truncate offset vector buffer to size (#1)
     org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, offsets.getBufferSizeFor(fieldNode.getLength() + 1));
     BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
+    lastSet = fieldNode.getLength();
   }
 
   @Override

From 2a2b1094e5370956917351ed49cf914ee2d5301f Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Mon, 12 Jun 2017 21:13:22 -0400
Subject: [PATCH 0719/1644] ARROW-742: [C++] std::wstring_convert exceptions
 handling

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #749 from MaxRis/ARROW-742 and squashes the following commits:

5479b3f4 [Max Risuhin] ARROW-742: [C++] std::wstring_convert exceptions handling
---
 cpp/src/arrow/io/file.cc         | 14 +++++++++++---
 cpp/src/arrow/io/io-file-test.cc | 12 ++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index c4c797c2f41f1..b2b880159fb6d 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -117,6 +117,10 @@ namespace io {
 
 // ----------------------------------------------------------------------
 // Cross-platform file compatability layer
+#if defined(_MSC_VER)
+constexpr const char* kRangeExceptionError =
+    "Range exception during wide-char string conversion";
+#endif
 
 static inline Status CheckOpenResult(
     int ret, int errno_actual, const char* filename, size_t filename_length) {
@@ -131,8 +135,10 @@ static inline Status CheckOpenResult(
     std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
     std::wstring wide_string(
         reinterpret_cast<const wchar_t*>(filename), filename_length / sizeof(wchar_t));
-    std::string byte_string = converter.to_bytes(wide_string);
-    ss << byte_string;
+    try {
+      std::string byte_string = converter.to_bytes(wide_string);
+      ss << byte_string;
+    } catch (const std::range_error&) { ss << kRangeExceptionError; }
 #else
     ss << filename;
 #endif
@@ -162,7 +168,9 @@ static inline Status ConvertToUtf16(const std::string& input, std::wstring* resu
   }
 
   std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> utf16_converter;
-  *result = utf16_converter.from_bytes(input);
+  try {
+    *result = utf16_converter.from_bytes(input);
+  } catch (const std::range_error&) { return Status::Invalid(kRangeExceptionError); }
   return Status::OK();
 }
 #endif
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 3450bae13bc21..1b7e052d651e6 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -96,6 +96,18 @@ class TestFileOutputStream : public FileTestFixture {
   std::shared_ptr<FileOutputStream> file_;
 };
 
+#if defined(_MSC_VER)
+TEST_F(TestFileOutputStream, FileNameWideCharConversionRangeException) {
+  std::shared_ptr<FileOutputStream> file;
+  // Form literal string with non-ASCII symbol(127 + 1)
+  std::string file_name = "\x80";
+  ASSERT_RAISES(Invalid, FileOutputStream::Open(file_name, &file));
+
+  std::shared_ptr<ReadableFile> rd_file;
+  ASSERT_RAISES(Invalid, ReadableFile::Open(file_name, &rd_file));
+}
+#endif
+
 TEST_F(TestFileOutputStream, DestructorClosesFile) {
   int fd;
   {

From 25ba44c3287de968ba22fc21577fe4639d81c4dc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 13 Jun 2017 11:54:18 -0400
Subject: [PATCH 0720/1644] ARROW-460: [C++] JSON read/write for dictionaries

cc @BryanCutler; this ended up being rather tedious.

As one uncertainty: I wasn't sure what to write for the `typeLayout` field for dictionary fields. We don't use this at all in C++, so let me know what you need in Java (if anything)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #750 from wesm/ARROW-460 and squashes the following commits:

36d6019e [Wes McKinney] Put dictionaries in top level of json object
92e2a95f [Wes McKinney] Some debugging help, get test suite passing
15b8fc6c [Wes McKinney] Schema read correct, but array reads incorrect
6581bd6d [Wes McKinney] Cleaning up JSON ArrayReader to use inline visitors. Progress
44115663 [Wes McKinney] Misc fixes, dictionary schema roundtrip not complete yet
59580222 [Wes McKinney] Draft JSON roundtrip with dictionaries, not yet tested
---
 cpp/src/arrow/ipc/ipc-json-test.cc |  23 +-
 cpp/src/arrow/ipc/json-internal.cc | 723 +++++++++++++++++++----------
 cpp/src/arrow/ipc/json-internal.h  |  30 +-
 cpp/src/arrow/ipc/json.cc          |  55 +--
 cpp/src/arrow/ipc/json.h           |   3 -
 cpp/src/arrow/ipc/metadata.h       |   3 +
 cpp/src/arrow/ipc/test-common.h    |  13 +-
 cpp/src/arrow/ipc/writer.cc        |  13 +-
 8 files changed, 535 insertions(+), 328 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 9cf6a88a6f3f1..e06af9f965e6f 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -39,18 +39,25 @@
 
 namespace arrow {
 namespace ipc {
+namespace json {
 
 void TestSchemaRoundTrip(const Schema& schema) {
   rj::StringBuffer sb;
   rj::Writer<rj::StringBuffer> writer(sb);
 
-  ASSERT_OK(WriteJsonSchema(schema, &writer));
+  writer.StartObject();
+  ASSERT_OK(internal::WriteSchema(schema, &writer));
+  writer.EndObject();
+
+  std::string json_schema = sb.GetString();
 
   rj::Document d;
-  d.Parse(sb.GetString());
+  d.Parse(json_schema);
 
   std::shared_ptr<Schema> out;
-  ASSERT_OK(ReadJsonSchema(d, &out));
+  if (!internal::ReadSchema(d, default_memory_pool(), &out).ok()) {
+    FAIL() << "Unable to read JSON schema: " << json_schema;
+  }
 
   if (!schema.Equals(*out)) {
     FAIL() << "In schema: " << schema.ToString() << "\nOut schema: " << out->ToString();
@@ -63,7 +70,7 @@ void TestArrayRoundTrip(const Array& array) {
   rj::StringBuffer sb;
   rj::Writer<rj::StringBuffer> writer(sb);
 
-  ASSERT_OK(WriteJsonArray(name, array, &writer));
+  ASSERT_OK(internal::WriteArray(name, array, &writer));
 
   std::string array_as_json = sb.GetString();
 
@@ -73,7 +80,7 @@ void TestArrayRoundTrip(const Array& array) {
   if (d.HasParseError()) { FAIL() << "JSON parsing failed"; }
 
   std::shared_ptr<Array> out;
-  ASSERT_OK(ReadJsonArray(default_memory_pool(), d, array.type(), &out));
+  ASSERT_OK(internal::ReadArray(default_memory_pool(), d, array.type(), &out));
 
   // std::cout << array_as_json << std::endl;
   CompareArraysDetailed(0, *out, array);
@@ -355,7 +362,8 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
 #define BATCH_CASES()                                                                   \
   ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
       &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
-      &MakeStruct, &MakeUnion, &MakeDates, &MakeTimestamps, &MakeTimes, &MakeFWBinary);
+      &MakeStruct, &MakeUnion, &MakeDates, &MakeTimestamps, &MakeTimes, &MakeFWBinary,  \
+      &MakeDictionary);
 
 class TestJsonRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*> {
  public:
@@ -364,6 +372,8 @@ class TestJsonRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*> {
 };
 
 void CheckRoundtrip(const RecordBatch& batch) {
+  TestSchemaRoundTrip(*batch.schema());
+
   std::unique_ptr<JsonWriter> writer;
   ASSERT_OK(JsonWriter::Open(batch.schema(), &writer));
   ASSERT_OK(writer->WriteRecordBatch(batch));
@@ -392,5 +402,6 @@ TEST_P(TestJsonRoundTrip, RoundTrip) {
 
 INSTANTIATE_TEST_CASE_P(TestJsonRoundTrip, TestJsonRoundTrip, BATCH_CASES());
 
+}  // namespace json
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 2ab3acba37f8d..bf2c194e4fb5a 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -32,8 +32,10 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/ipc/metadata.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
+#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -43,9 +45,8 @@
 
 namespace arrow {
 namespace ipc {
-
-using RjArray = rj::Value::ConstArray;
-using RjObject = rj::Value::ConstObject;
+namespace json {
+namespace internal {
 
 static std::string GetBufferTypeName(BufferType type) {
   switch (type) {
@@ -93,20 +94,67 @@ static std::string GetTimeUnitName(TimeUnit::type unit) {
   return "UNKNOWN";
 }
 
-class JsonSchemaWriter {
+class SchemaWriter {
  public:
-  explicit JsonSchemaWriter(const Schema& schema, RjWriter* writer)
+  explicit SchemaWriter(const Schema& schema, RjWriter* writer)
       : schema_(schema), writer_(writer) {}
 
   Status Write() {
+    writer_->Key("schema");
     writer_->StartObject();
     writer_->Key("fields");
     writer_->StartArray();
     for (const std::shared_ptr<Field>& field : schema_.fields()) {
-      RETURN_NOT_OK(VisitField(*field.get()));
+      RETURN_NOT_OK(VisitField(*field));
     }
     writer_->EndArray();
     writer_->EndObject();
+
+    // Write dictionaries, if any
+    if (dictionary_memo_.size() > 0) {
+      writer_->Key("dictionaries");
+      writer_->StartArray();
+      for (const auto& entry : dictionary_memo_.id_to_dictionary()) {
+        RETURN_NOT_OK(WriteDictionary(entry.first, entry.second));
+      }
+      writer_->EndArray();
+    }
+    return Status::OK();
+  }
+
+  Status WriteDictionary(int64_t id, const std::shared_ptr<Array>& dictionary) {
+    writer_->StartObject();
+    writer_->Key("id");
+    writer_->Int(static_cast<int32_t>(id));
+    writer_->Key("data");
+
+    // Make a dummy record batch. A bit tedious as we have to make a schema
+    auto schema = std::shared_ptr<Schema>(
+        new Schema({arrow::field("dictionary", dictionary->type())}));
+    RecordBatch batch(schema, dictionary->length(), {dictionary});
+    RETURN_NOT_OK(WriteRecordBatch(batch, writer_));
+    writer_->EndObject();
+    return Status::OK();
+  }
+
+  Status WriteDictionaryMetadata(const DictionaryType& type) {
+    int64_t dictionary_id = dictionary_memo_.GetId(type.dictionary());
+    writer_->Key("dictionary");
+
+    // Emulate DictionaryEncoding from Schema.fbs
+    writer_->StartObject();
+    writer_->Key("id");
+    writer_->Int(static_cast<int32_t>(dictionary_id));
+    writer_->Key("indexType");
+
+    writer_->StartObject();
+    RETURN_NOT_OK(VisitType(*type.index_type()));
+    writer_->EndObject();
+
+    writer_->Key("isOrdered");
+    writer_->Bool(type.ordered());
+    writer_->EndObject();
+
     return Status::OK();
   }
 
@@ -119,18 +167,33 @@ class JsonSchemaWriter {
     writer_->Key("nullable");
     writer_->Bool(field.nullable());
 
+    const DataType& type = *field.type();
+
     // Visit the type
-    RETURN_NOT_OK(VisitTypeInline(*field.type(), this));
+    writer_->Key("type");
+    writer_->StartObject();
+    RETURN_NOT_OK(VisitType(type));
+    writer_->EndObject();
+
+    if (type.id() == Type::DICTIONARY) {
+      const auto& dict_type = static_cast<const DictionaryType&>(type);
+      RETURN_NOT_OK(WriteDictionaryMetadata(dict_type));
+
+      const DataType& dictionary_type = *dict_type.dictionary()->type();
+      const DataType& index_type = *dict_type.index_type();
+      RETURN_NOT_OK(WriteChildren(dictionary_type.children()));
+      WriteBufferLayout(index_type.GetBufferLayout());
+    } else {
+      RETURN_NOT_OK(WriteChildren(type.children()));
+      WriteBufferLayout(type.GetBufferLayout());
+    }
+
     writer_->EndObject();
 
     return Status::OK();
   }
 
-  void SetNoChildren() {
-    writer_->Key("children");
-    writer_->StartArray();
-    writer_->EndArray();
-  }
+  Status VisitType(const DataType& type);
 
   template <typename T>
   typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
@@ -227,27 +290,20 @@ class JsonSchemaWriter {
 
   template <typename T>
   void WriteName(const std::string& typeclass, const T& type) {
-    writer_->Key("type");
-    writer_->StartObject();
     writer_->Key("name");
     writer_->String(typeclass);
     WriteTypeMetadata(type);
-    writer_->EndObject();
   }
 
   template <typename T>
   Status WritePrimitive(const std::string& typeclass, const T& type) {
     WriteName(typeclass, type);
-    SetNoChildren();
-    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
   template <typename T>
   Status WriteVarBytes(const std::string& typeclass, const T& type) {
     WriteName(typeclass, type);
-    SetNoChildren();
-    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
@@ -275,16 +331,14 @@ class JsonSchemaWriter {
     writer_->Key("children");
     writer_->StartArray();
     for (const std::shared_ptr<Field>& field : children) {
-      RETURN_NOT_OK(VisitField(*field.get()));
+      RETURN_NOT_OK(VisitField(*field));
     }
     writer_->EndArray();
     return Status::OK();
   }
 
   Status Visit(const NullType& type) { return WritePrimitive("null", type); }
-
   Status Visit(const BooleanType& type) { return WritePrimitive("bool", type); }
-
   Status Visit(const Integer& type) { return WritePrimitive("int", type); }
 
   Status Visit(const FloatingPoint& type) {
@@ -292,60 +346,57 @@ class JsonSchemaWriter {
   }
 
   Status Visit(const DateType& type) { return WritePrimitive("date", type); }
-
   Status Visit(const TimeType& type) { return WritePrimitive("time", type); }
-
   Status Visit(const StringType& type) { return WriteVarBytes("utf8", type); }
-
   Status Visit(const BinaryType& type) { return WriteVarBytes("binary", type); }
-
   Status Visit(const FixedSizeBinaryType& type) {
     return WritePrimitive("fixedsizebinary", type);
   }
 
   Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); }
-
   Status Visit(const IntervalType& type) { return WritePrimitive("interval", type); }
 
   Status Visit(const ListType& type) {
     WriteName("list", type);
-    RETURN_NOT_OK(WriteChildren(type.children()));
-    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
   Status Visit(const StructType& type) {
     WriteName("struct", type);
-    WriteChildren(type.children());
-    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
   Status Visit(const UnionType& type) {
     WriteName("union", type);
-    WriteChildren(type.children());
-    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
   Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
 
   Status Visit(const DictionaryType& type) {
-    return Status::NotImplemented("dictionary");
+    return VisitType(*type.dictionary()->type());
   }
 
  private:
+  DictionaryMemo dictionary_memo_;
+
   const Schema& schema_;
   RjWriter* writer_;
 };
 
-class JsonArrayWriter {
+Status SchemaWriter::VisitType(const DataType& type) {
+  return VisitTypeInline(type, this);
+}
+
+class ArrayWriter {
  public:
-  JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
+  ArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
       : name_(name), array_(array), writer_(writer) {}
 
   Status Write() { return VisitArray(name_, array_); }
 
+  Status VisitArrayValues(const Array& arr) { return VisitArrayInline(arr, this); }
+
   Status VisitArray(const std::string& name, const Array& arr) {
     writer_->StartObject();
     writer_->Key("name");
@@ -354,7 +405,7 @@ class JsonArrayWriter {
     writer_->Key("count");
     writer_->Int(static_cast<int32_t>(arr.length()));
 
-    RETURN_NOT_OK(VisitArrayInline(arr, this));
+    RETURN_NOT_OK(VisitArrayValues(arr));
 
     writer_->EndObject();
     return Status::OK();
@@ -461,7 +512,7 @@ class JsonArrayWriter {
     writer_->Key("children");
     writer_->StartArray();
     for (size_t i = 0; i < fields.size(); ++i) {
-      RETURN_NOT_OK(VisitArray(fields[i]->name(), *arrays[i].get()));
+      RETURN_NOT_OK(VisitArray(fields[i]->name(), *arrays[i]));
     }
     writer_->EndArray();
     return Status::OK();
@@ -493,30 +544,32 @@ class JsonArrayWriter {
 
   Status Visit(const DecimalArray& array) { return Status::NotImplemented("decimal"); }
 
-  Status Visit(const DictionaryArray& array) { return Status::NotImplemented("decimal"); }
+  Status Visit(const DictionaryArray& array) {
+    return VisitArrayValues(*array.indices());
+  }
 
   Status Visit(const ListArray& array) {
     WriteValidityField(array);
     WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length() + 1);
-    auto type = static_cast<const ListType*>(array.type().get());
-    return WriteChildren(type->children(), {array.values()});
+    const auto& type = static_cast<const ListType&>(*array.type());
+    return WriteChildren(type.children(), {array.values()});
   }
 
   Status Visit(const StructArray& array) {
     WriteValidityField(array);
-    auto type = static_cast<const StructType*>(array.type().get());
-    return WriteChildren(type->children(), array.fields());
+    const auto& type = static_cast<const StructType&>(*array.type());
+    return WriteChildren(type.children(), array.fields());
   }
 
   Status Visit(const UnionArray& array) {
     WriteValidityField(array);
-    auto type = static_cast<const UnionType*>(array.type().get());
+    const auto& type = static_cast<const UnionType&>(*array.type());
 
     WriteIntegerField("TYPE_ID", array.raw_type_ids(), array.length());
-    if (type->mode() == UnionMode::DENSE) {
+    if (type.mode() == UnionMode::DENSE) {
       WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length());
     }
-    return WriteChildren(type->children(), array.children());
+    return WriteChildren(type.children(), array.children());
   }
 
  private:
@@ -525,16 +578,38 @@ class JsonArrayWriter {
   RjWriter* writer_;
 };
 
+static Status GetObjectInt(const RjObject& obj, const std::string& key, int* out) {
+  const auto& it = obj.FindMember(key);
+  RETURN_NOT_INT(key, it, obj);
+  *out = it->value.GetInt();
+  return Status::OK();
+}
+
+static Status GetObjectBool(const RjObject& obj, const std::string& key, bool* out) {
+  const auto& it = obj.FindMember(key);
+  RETURN_NOT_BOOL(key, it, obj);
+  *out = it->value.GetBool();
+  return Status::OK();
+}
+
+static Status GetObjectString(
+    const RjObject& obj, const std::string& key, std::string* out) {
+  const auto& it = obj.FindMember(key);
+  RETURN_NOT_STRING(key, it, obj);
+  *out = it->value.GetString();
+  return Status::OK();
+}
+
 static Status GetInteger(
     const rj::Value::ConstObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& json_bit_width = json_type.FindMember("bitWidth");
-  RETURN_NOT_INT("bitWidth", json_bit_width, json_type);
+  const auto& it_bit_width = json_type.FindMember("bitWidth");
+  RETURN_NOT_INT("bitWidth", it_bit_width, json_type);
 
-  const auto& json_is_signed = json_type.FindMember("isSigned");
-  RETURN_NOT_BOOL("isSigned", json_is_signed, json_type);
+  const auto& it_is_signed = json_type.FindMember("isSigned");
+  RETURN_NOT_BOOL("isSigned", it_is_signed, json_type);
 
-  bool is_signed = json_is_signed->value.GetBool();
-  int bit_width = json_bit_width->value.GetInt();
+  bool is_signed = it_is_signed->value.GetBool();
+  int bit_width = it_bit_width->value.GetInt();
 
   switch (bit_width) {
     case 8:
@@ -559,10 +634,10 @@ static Status GetInteger(
 
 static Status GetFloatingPoint(
     const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& json_precision = json_type.FindMember("precision");
-  RETURN_NOT_STRING("precision", json_precision, json_type);
+  const auto& it_precision = json_type.FindMember("precision");
+  RETURN_NOT_STRING("precision", it_precision, json_type);
 
-  std::string precision = json_precision->value.GetString();
+  std::string precision = it_precision->value.GetString();
 
   if (precision == "DOUBLE") {
     *type = float64();
@@ -580,19 +655,19 @@ static Status GetFloatingPoint(
 
 static Status GetFixedSizeBinary(
     const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& json_byte_width = json_type.FindMember("byteWidth");
-  RETURN_NOT_INT("byteWidth", json_byte_width, json_type);
+  const auto& it_byte_width = json_type.FindMember("byteWidth");
+  RETURN_NOT_INT("byteWidth", it_byte_width, json_type);
 
-  int32_t byte_width = json_byte_width->value.GetInt();
+  int32_t byte_width = it_byte_width->value.GetInt();
   *type = fixed_size_binary(byte_width);
   return Status::OK();
 }
 
 static Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& json_unit = json_type.FindMember("unit");
-  RETURN_NOT_STRING("unit", json_unit, json_type);
+  const auto& it_unit = json_type.FindMember("unit");
+  RETURN_NOT_STRING("unit", it_unit, json_type);
 
-  std::string unit_str = json_unit->value.GetString();
+  std::string unit_str = it_unit->value.GetString();
 
   if (unit_str == "DAY") {
     *type = date32();
@@ -607,13 +682,13 @@ static Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type
 }
 
 static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& json_unit = json_type.FindMember("unit");
-  RETURN_NOT_STRING("unit", json_unit, json_type);
+  const auto& it_unit = json_type.FindMember("unit");
+  RETURN_NOT_STRING("unit", it_unit, json_type);
 
-  const auto& json_bit_width = json_type.FindMember("bitWidth");
-  RETURN_NOT_INT("bitWidth", json_bit_width, json_type);
+  const auto& it_bit_width = json_type.FindMember("bitWidth");
+  RETURN_NOT_INT("bitWidth", it_bit_width, json_type);
 
-  std::string unit_str = json_unit->value.GetString();
+  std::string unit_str = it_unit->value.GetString();
 
   if (unit_str == "SECOND") {
     *type = time32(TimeUnit::SECOND);
@@ -631,7 +706,7 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 
   const auto& fw_type = static_cast<const FixedWidthType&>(**type);
 
-  int bit_width = json_bit_width->value.GetInt();
+  int bit_width = it_bit_width->value.GetInt();
   if (bit_width != fw_type.bit_width()) {
     return Status::Invalid("Indicated bit width does not match unit");
   }
@@ -640,10 +715,10 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr<DataType>* type
 }
 
 static Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>* type) {
-  const auto& json_unit = json_type.FindMember("unit");
-  RETURN_NOT_STRING("unit", json_unit, json_type);
+  const auto& it_unit = json_type.FindMember("unit");
+  RETURN_NOT_STRING("unit", it_unit, json_type);
 
-  std::string unit_str = json_unit->value.GetString();
+  std::string unit_str = it_unit->value.GetString();
 
   TimeUnit::type unit;
   if (unit_str == "SECOND") {
@@ -660,11 +735,11 @@ static Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>*
     return Status::Invalid(ss.str());
   }
 
-  const auto& json_tz = json_type.FindMember("timezone");
-  if (json_tz == json_type.MemberEnd()) {
+  const auto& it_tz = json_type.FindMember("timezone");
+  if (it_tz == json_type.MemberEnd()) {
     *type = timestamp(unit);
   } else {
-    *type = timestamp(unit, json_tz->value.GetString());
+    *type = timestamp(unit, it_tz->value.GetString());
   }
 
   return Status::OK();
@@ -673,10 +748,10 @@ static Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>*
 static Status GetUnion(const RjObject& json_type,
     const std::vector<std::shared_ptr<Field>>& children,
     std::shared_ptr<DataType>* type) {
-  const auto& json_mode = json_type.FindMember("mode");
-  RETURN_NOT_STRING("mode", json_mode, json_type);
+  const auto& it_mode = json_type.FindMember("mode");
+  RETURN_NOT_STRING("mode", it_mode, json_type);
 
-  std::string mode_str = json_mode->value.GetString();
+  std::string mode_str = it_mode->value.GetString();
   UnionMode mode;
 
   if (mode_str == "SPARSE") {
@@ -689,11 +764,11 @@ static Status GetUnion(const RjObject& json_type,
     return Status::Invalid(ss.str());
   }
 
-  const auto& json_type_codes = json_type.FindMember("typeIds");
-  RETURN_NOT_ARRAY("typeIds", json_type_codes, json_type);
+  const auto& it_type_codes = json_type.FindMember("typeIds");
+  RETURN_NOT_ARRAY("typeIds", it_type_codes, json_type);
 
   std::vector<uint8_t> type_codes;
-  const auto& id_array = json_type_codes->value.GetArray();
+  const auto& id_array = it_type_codes->value.GetArray();
   for (const rj::Value& val : id_array) {
     DCHECK(val.IsUint());
     type_codes.push_back(static_cast<uint8_t>(val.GetUint()));
@@ -707,10 +782,10 @@ static Status GetUnion(const RjObject& json_type,
 static Status GetType(const RjObject& json_type,
     const std::vector<std::shared_ptr<Field>>& children,
     std::shared_ptr<DataType>* type) {
-  const auto& json_type_name = json_type.FindMember("name");
-  RETURN_NOT_STRING("name", json_type_name, json_type);
+  const auto& it_type_name = json_type.FindMember("name");
+  RETURN_NOT_STRING("name", it_type_name, json_type);
 
-  std::string type_name = json_type_name->value.GetString();
+  std::string type_name = it_type_name->value.GetString();
 
   if (type_name == "int") {
     return GetInteger(json_type, type);
@@ -733,6 +808,9 @@ static Status GetType(const RjObject& json_type,
   } else if (type_name == "timestamp") {
     return GetTimestamp(json_type, type);
   } else if (type_name == "list") {
+    if (children.size() != 1) {
+      return Status::Invalid("List must have exactly one child");
+    }
     *type = list(children[0]);
   } else if (type_name == "struct") {
     *type = struct_(children);
@@ -742,43 +820,83 @@ static Status GetType(const RjObject& json_type,
   return Status::OK();
 }
 
-static Status GetField(const rj::Value& obj, std::shared_ptr<Field>* field);
+static Status GetField(const rj::Value& obj, const DictionaryMemo* dictionary_memo,
+    std::shared_ptr<Field>* field);
 
-static Status GetFieldsFromArray(
-    const rj::Value& obj, std::vector<std::shared_ptr<Field>>* fields) {
+static Status GetFieldsFromArray(const rj::Value& obj,
+    const DictionaryMemo* dictionary_memo, std::vector<std::shared_ptr<Field>>* fields) {
   const auto& values = obj.GetArray();
 
   fields->resize(values.Size());
   for (rj::SizeType i = 0; i < fields->size(); ++i) {
-    RETURN_NOT_OK(GetField(values[i], &(*fields)[i]));
+    RETURN_NOT_OK(GetField(values[i], dictionary_memo, &(*fields)[i]));
   }
   return Status::OK();
 }
 
-static Status GetField(const rj::Value& obj, std::shared_ptr<Field>* field) {
+static Status ParseDictionary(const RjObject& obj, int64_t* id, bool* is_ordered,
+    std::shared_ptr<DataType>* index_type) {
+  int32_t int32_id;
+  RETURN_NOT_OK(GetObjectInt(obj, "id", &int32_id));
+  *id = int32_id;
+
+  RETURN_NOT_OK(GetObjectBool(obj, "isOrdered", is_ordered));
+
+  const auto& it_index_type = obj.FindMember("indexType");
+  RETURN_NOT_OBJECT("indexType", it_index_type, obj);
+
+  const auto& json_index_type = it_index_type->value.GetObject();
+
+  std::string type_name;
+  RETURN_NOT_OK(GetObjectString(json_index_type, "name", &type_name));
+  if (type_name != "int") {
+    return Status::Invalid("Dictionary indices can only be integers");
+  }
+  return GetInteger(json_index_type, index_type);
+}
+
+static Status GetField(const rj::Value& obj, const DictionaryMemo* dictionary_memo,
+    std::shared_ptr<Field>* field) {
   if (!obj.IsObject()) { return Status::Invalid("Field was not a JSON object"); }
   const auto& json_field = obj.GetObject();
 
-  const auto& json_name = json_field.FindMember("name");
-  RETURN_NOT_STRING("name", json_name, json_field);
+  std::string name;
+  bool nullable;
+  RETURN_NOT_OK(GetObjectString(json_field, "name", &name));
+  RETURN_NOT_OK(GetObjectBool(json_field, "nullable", &nullable));
 
-  const auto& json_nullable = json_field.FindMember("nullable");
-  RETURN_NOT_BOOL("nullable", json_nullable, json_field);
+  std::shared_ptr<DataType> type;
 
-  const auto& json_type = json_field.FindMember("type");
-  RETURN_NOT_OBJECT("type", json_type, json_field);
+  const auto& it_dictionary = json_field.FindMember("dictionary");
+  if (dictionary_memo != nullptr && it_dictionary != json_field.MemberEnd()) {
+    // Field is dictionary encoded. We must have already
+    RETURN_NOT_OBJECT("dictionary", it_dictionary, json_field);
+    int64_t dictionary_id;
+    bool is_ordered;
+    std::shared_ptr<DataType> index_type;
+    RETURN_NOT_OK(ParseDictionary(
+        it_dictionary->value.GetObject(), &dictionary_id, &is_ordered, &index_type));
 
-  const auto& json_children = json_field.FindMember("children");
-  RETURN_NOT_ARRAY("children", json_children, json_field);
+    std::shared_ptr<Array> dictionary;
+    RETURN_NOT_OK(dictionary_memo->GetDictionary(dictionary_id, &dictionary));
 
-  std::vector<std::shared_ptr<Field>> children;
-  RETURN_NOT_OK(GetFieldsFromArray(json_children->value, &children));
+    type = std::make_shared<DictionaryType>(index_type, dictionary, is_ordered);
+  } else {
+    // If the dictionary_memo was not passed, or if the field is not dictionary
+    // encoded, we are interested in the complete type including all children
 
-  std::shared_ptr<DataType> type;
-  RETURN_NOT_OK(GetType(json_type->value.GetObject(), children, &type));
+    const auto& it_type = json_field.FindMember("type");
+    RETURN_NOT_OBJECT("type", it_type, json_field);
+
+    const auto& it_children = json_field.FindMember("children");
+    RETURN_NOT_ARRAY("children", it_children, json_field);
 
-  *field = std::make_shared<Field>(
-      json_name->value.GetString(), type, json_nullable->value.GetBool());
+    std::vector<std::shared_ptr<Field>> children;
+    RETURN_NOT_OK(GetFieldsFromArray(it_children->value, dictionary_memo, &children));
+    RETURN_NOT_OK(GetType(it_type->value.GetObject(), children, &type));
+  }
+
+  *field = std::make_shared<Field>(name, type, nullable);
   return Status::OK();
 }
 
@@ -810,9 +928,13 @@ UnboxValue(const rj::Value& val) {
   return val.GetBool();
 }
 
-class JsonArrayReader {
+class ArrayReader {
  public:
-  explicit JsonArrayReader(MemoryPool* pool) : pool_(pool) {}
+  explicit ArrayReader(const rj::Value& json_array, const std::shared_ptr<DataType>& type,
+      MemoryPool* pool)
+      : json_array_(json_array), type_(type), pool_(pool) {}
+
+  Status ParseTypeValues(const DataType& type);
 
   Status GetValidityBuffer(const std::vector<bool>& is_valid, int32_t* null_count,
       std::shared_ptr<Buffer>* validity_buffer) {
@@ -841,18 +963,17 @@ class JsonArrayReader {
           std::is_base_of<TimestampType, T>::value ||
           std::is_base_of<TimeType, T>::value || std::is_base_of<BooleanType, T>::value,
       Status>::type
-  ReadArray(const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-    typename TypeTraits<T>::BuilderType builder(pool_, type);
+  Visit(const T& type) {
+    typename TypeTraits<T>::BuilderType builder(pool_, type_);
 
-    const auto& json_data = json_array.FindMember("DATA");
-    RETURN_NOT_ARRAY("DATA", json_data, json_array);
+    const auto& json_data = obj_->FindMember("DATA");
+    RETURN_NOT_ARRAY("DATA", json_data, *obj_);
 
     const auto& json_data_arr = json_data->value.GetArray();
 
-    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length);
-    for (int i = 0; i < length; ++i) {
-      if (!is_valid[i]) {
+    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
+    for (int i = 0; i < length_; ++i) {
+      if (!is_valid_[i]) {
         builder.AppendNull();
         continue;
       }
@@ -861,25 +982,24 @@ class JsonArrayReader {
       builder.Append(UnboxValue<T>(val));
     }
 
-    return builder.Finish(array);
+    return builder.Finish(&result_);
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<BinaryType, T>::value, Status>::type ReadArray(
-      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+  typename std::enable_if<std::is_base_of<BinaryType, T>::value, Status>::type Visit(
+      const T& type) {
     typename TypeTraits<T>::BuilderType builder(pool_);
 
-    const auto& json_data = json_array.FindMember("DATA");
-    RETURN_NOT_ARRAY("DATA", json_data, json_array);
+    const auto& json_data = obj_->FindMember("DATA");
+    RETURN_NOT_ARRAY("DATA", json_data, *obj_);
 
     const auto& json_data_arr = json_data->value.GetArray();
 
-    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length);
+    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
 
     auto byte_buffer = std::make_shared<PoolBuffer>(pool_);
-    for (int i = 0; i < length; ++i) {
-      if (!is_valid[i]) {
+    for (int i = 0; i < length_; ++i) {
+      if (!is_valid_[i]) {
         builder.AppendNull();
         continue;
       }
@@ -905,31 +1025,29 @@ class JsonArrayReader {
       }
     }
 
-    return builder.Finish(array);
+    return builder.Finish(&result_);
   }
 
   template <typename T>
   typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value, Status>::type
-  ReadArray(const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-    FixedSizeBinaryBuilder builder(pool_, type);
+  Visit(const T& type) {
+    FixedSizeBinaryBuilder builder(pool_, type_);
 
-    const auto& json_data = json_array.FindMember("DATA");
-    RETURN_NOT_ARRAY("DATA", json_data, json_array);
+    const auto& json_data = obj_->FindMember("DATA");
+    RETURN_NOT_ARRAY("DATA", json_data, *obj_);
 
     const auto& json_data_arr = json_data->value.GetArray();
 
-    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length);
-
-    int32_t byte_width = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
+    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
+    int32_t byte_width = type.byte_width();
 
     // Allocate space for parsed values
     std::shared_ptr<MutableBuffer> byte_buffer;
     RETURN_NOT_OK(AllocateBuffer(pool_, byte_width, &byte_buffer));
     uint8_t* byte_buffer_data = byte_buffer->mutable_data();
 
-    for (int i = 0; i < length; ++i) {
-      if (!is_valid[i]) {
+    for (int i = 0; i < length_; ++i) {
+      if (!is_valid_[i]) {
         builder.AppendNull();
         continue;
       }
@@ -946,7 +1064,7 @@ class JsonArrayReader {
       }
       RETURN_NOT_OK(builder.Append(byte_buffer_data));
     }
-    return builder.Finish(array);
+    return builder.Finish(&result_);
   }
 
   template <typename T>
@@ -966,99 +1084,97 @@ class JsonArrayReader {
     return Status::OK();
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<ListType, T>::value, Status>::type ReadArray(
-      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+  Status Visit(const ListType& type) {
     int32_t null_count = 0;
     std::shared_ptr<Buffer> validity_buffer;
-    RETURN_NOT_OK(GetValidityBuffer(is_valid, &null_count, &validity_buffer));
+    RETURN_NOT_OK(GetValidityBuffer(is_valid_, &null_count, &validity_buffer));
 
-    const auto& json_offsets = json_array.FindMember("OFFSET");
-    RETURN_NOT_ARRAY("OFFSET", json_offsets, json_array);
+    const auto& json_offsets = obj_->FindMember("OFFSET");
+    RETURN_NOT_ARRAY("OFFSET", json_offsets, *obj_);
     std::shared_ptr<Buffer> offsets_buffer;
     RETURN_NOT_OK(GetIntArray<int32_t>(
-        json_offsets->value.GetArray(), length + 1, &offsets_buffer));
+        json_offsets->value.GetArray(), length_ + 1, &offsets_buffer));
 
     std::vector<std::shared_ptr<Array>> children;
-    RETURN_NOT_OK(GetChildren(json_array, type, &children));
+    RETURN_NOT_OK(GetChildren(*obj_, type, &children));
     DCHECK_EQ(children.size(), 1);
 
-    *array = std::make_shared<ListArray>(
-        type, length, offsets_buffer, children[0], validity_buffer, null_count);
+    result_ = std::make_shared<ListArray>(
+        type_, length_, offsets_buffer, children[0], validity_buffer, null_count);
 
     return Status::OK();
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<StructType, T>::value, Status>::type ReadArray(
-      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+  Status Visit(const StructType& type) {
     int32_t null_count = 0;
     std::shared_ptr<Buffer> validity_buffer;
-    RETURN_NOT_OK(GetValidityBuffer(is_valid, &null_count, &validity_buffer));
+    RETURN_NOT_OK(GetValidityBuffer(is_valid_, &null_count, &validity_buffer));
 
     std::vector<std::shared_ptr<Array>> fields;
-    RETURN_NOT_OK(GetChildren(json_array, type, &fields));
+    RETURN_NOT_OK(GetChildren(*obj_, type, &fields));
 
-    *array =
-        std::make_shared<StructArray>(type, length, fields, validity_buffer, null_count);
+    result_ = std::make_shared<StructArray>(
+        type_, length_, fields, validity_buffer, null_count);
 
     return Status::OK();
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<UnionType, T>::value, Status>::type ReadArray(
-      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+  Status Visit(const UnionType& type) {
     int32_t null_count = 0;
 
-    const auto& union_type = static_cast<const UnionType&>(*type.get());
-
     std::shared_ptr<Buffer> validity_buffer;
     std::shared_ptr<Buffer> type_id_buffer;
     std::shared_ptr<Buffer> offsets_buffer;
 
-    RETURN_NOT_OK(GetValidityBuffer(is_valid, &null_count, &validity_buffer));
+    RETURN_NOT_OK(GetValidityBuffer(is_valid_, &null_count, &validity_buffer));
 
-    const auto& json_type_ids = json_array.FindMember("TYPE_ID");
-    RETURN_NOT_ARRAY("TYPE_ID", json_type_ids, json_array);
+    const auto& json_type_ids = obj_->FindMember("TYPE_ID");
+    RETURN_NOT_ARRAY("TYPE_ID", json_type_ids, *obj_);
     RETURN_NOT_OK(
-        GetIntArray<uint8_t>(json_type_ids->value.GetArray(), length, &type_id_buffer));
+        GetIntArray<uint8_t>(json_type_ids->value.GetArray(), length_, &type_id_buffer));
 
-    if (union_type.mode() == UnionMode::DENSE) {
-      const auto& json_offsets = json_array.FindMember("OFFSET");
-      RETURN_NOT_ARRAY("OFFSET", json_offsets, json_array);
+    if (type.mode() == UnionMode::DENSE) {
+      const auto& json_offsets = obj_->FindMember("OFFSET");
+      RETURN_NOT_ARRAY("OFFSET", json_offsets, *obj_);
       RETURN_NOT_OK(
-          GetIntArray<int32_t>(json_offsets->value.GetArray(), length, &offsets_buffer));
+          GetIntArray<int32_t>(json_offsets->value.GetArray(), length_, &offsets_buffer));
     }
 
     std::vector<std::shared_ptr<Array>> children;
-    RETURN_NOT_OK(GetChildren(json_array, type, &children));
+    RETURN_NOT_OK(GetChildren(*obj_, type, &children));
 
-    *array = std::make_shared<UnionArray>(type, length, children, type_id_buffer,
+    result_ = std::make_shared<UnionArray>(type_, length_, children, type_id_buffer,
         offsets_buffer, validity_buffer, null_count);
 
     return Status::OK();
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<NullType, T>::value, Status>::type ReadArray(
-      const RjObject& json_array, int32_t length, const std::vector<bool>& is_valid,
-      const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-    *array = std::make_shared<NullArray>(length);
+  Status Visit(const NullType& type) {
+    result_ = std::make_shared<NullArray>(length_);
+    return Status::OK();
+  }
+
+  Status Visit(const DictionaryType& type) {
+    // This stores the indices in result_
+    //
+    // XXX(wesm): slight hack
+    auto dict_type = type_;
+    type_ = type.index_type();
+    RETURN_NOT_OK(ParseTypeValues(*type_));
+    type_ = dict_type;
+    result_ = std::make_shared<DictionaryArray>(type_, result_);
     return Status::OK();
   }
 
-  Status GetChildren(const RjObject& json_array, const std::shared_ptr<DataType>& type,
+  Status GetChildren(const RjObject& obj, const DataType& type,
       std::vector<std::shared_ptr<Array>>* array) {
-    const auto& json_children = json_array.FindMember("children");
-    RETURN_NOT_ARRAY("children", json_children, json_array);
+    const auto& json_children = obj.FindMember("children");
+    RETURN_NOT_ARRAY("children", json_children, obj);
     const auto& json_children_arr = json_children->value.GetArray();
 
-    if (type->num_children() != static_cast<int>(json_children_arr.Size())) {
+    if (type.num_children() != static_cast<int>(json_children_arr.Size())) {
       std::stringstream ss;
-      ss << "Expected " << type->num_children() << " children, but got "
+      ss << "Expected " << type.num_children() << " children, but got "
          << json_children_arr.Size();
       return Status::Invalid(ss.str());
     }
@@ -1067,128 +1183,237 @@ class JsonArrayReader {
       const rj::Value& json_child = json_children_arr[i];
       DCHECK(json_child.IsObject());
 
-      std::shared_ptr<Field> child_field = type->child(i);
+      std::shared_ptr<Field> child_field = type.child(i);
 
       auto it = json_child.FindMember("name");
       RETURN_NOT_STRING("name", it, json_child);
 
       DCHECK_EQ(it->value.GetString(), child_field->name());
       std::shared_ptr<Array> child;
-      RETURN_NOT_OK(GetArray(json_children_arr[i], child_field->type(), &child));
+      RETURN_NOT_OK(ReadArray(pool_, json_children_arr[i], child_field->type(), &child));
       array->emplace_back(child);
     }
 
     return Status::OK();
   }
 
-  Status GetArray(const rj::Value& obj, const std::shared_ptr<DataType>& type,
-      std::shared_ptr<Array>* array) {
-    if (!obj.IsObject()) {
+  Status GetArray(std::shared_ptr<Array>* out) {
+    if (!json_array_.IsObject()) {
       return Status::Invalid("Array element was not a JSON object");
     }
-    const auto& json_array = obj.GetObject();
 
-    const auto& json_length = json_array.FindMember("count");
-    RETURN_NOT_INT("count", json_length, json_array);
-    int32_t length = json_length->value.GetInt();
+    auto obj = json_array_.GetObject();
+    obj_ = &obj;
 
-    const auto& json_valid_iter = json_array.FindMember("VALIDITY");
-    RETURN_NOT_ARRAY("VALIDITY", json_valid_iter, json_array);
+    RETURN_NOT_OK(GetObjectInt(obj, "count", &length_));
 
-    const auto& json_validity = json_valid_iter->value.GetArray();
-
-    DCHECK_EQ(static_cast<int>(json_validity.Size()), length);
+    const auto& json_valid_iter = obj.FindMember("VALIDITY");
+    RETURN_NOT_ARRAY("VALIDITY", json_valid_iter, obj);
 
-    std::vector<bool> is_valid;
+    const auto& json_validity = json_valid_iter->value.GetArray();
+    DCHECK_EQ(static_cast<int>(json_validity.Size()), length_);
     for (const rj::Value& val : json_validity) {
       DCHECK(val.IsInt());
-      is_valid.push_back(val.GetInt() != 0);
-    }
-
-#define TYPE_CASE(TYPE) \
-  case TYPE::type_id:   \
-    return ReadArray<TYPE>(json_array, length, is_valid, type, array);
-
-    switch (type->id()) {
-      TYPE_CASE(NullType);
-      TYPE_CASE(BooleanType);
-      TYPE_CASE(UInt8Type);
-      TYPE_CASE(Int8Type);
-      TYPE_CASE(UInt16Type);
-      TYPE_CASE(Int16Type);
-      TYPE_CASE(UInt32Type);
-      TYPE_CASE(Int32Type);
-      TYPE_CASE(UInt64Type);
-      TYPE_CASE(Int64Type);
-      TYPE_CASE(HalfFloatType);
-      TYPE_CASE(FloatType);
-      TYPE_CASE(DoubleType);
-      TYPE_CASE(StringType);
-      TYPE_CASE(BinaryType);
-      TYPE_CASE(FixedSizeBinaryType);
-      TYPE_CASE(Date32Type);
-      TYPE_CASE(Date64Type);
-      TYPE_CASE(TimestampType);
-      TYPE_CASE(Time32Type);
-      TYPE_CASE(Time64Type);
-      TYPE_CASE(ListType);
-      TYPE_CASE(StructType);
-      TYPE_CASE(UnionType);
-      default:
-        std::stringstream ss;
-        ss << type->ToString();
-        return Status::NotImplemented(ss.str());
+      is_valid_.push_back(val.GetInt() != 0);
     }
 
-#undef TYPE_CASE
-
+    RETURN_NOT_OK(ParseTypeValues(*type_));
+    *out = result_;
     return Status::OK();
   }
 
  private:
+  const rj::Value& json_array_;
+  const RjObject* obj_;
+  std::shared_ptr<DataType> type_;
   MemoryPool* pool_;
+
+  // Parsed common attributes
+  std::vector<bool> is_valid_;
+  int32_t length_;
+  std::shared_ptr<Array> result_;
 };
 
-Status WriteJsonSchema(const Schema& schema, RjWriter* json_writer) {
-  JsonSchemaWriter converter(schema, json_writer);
+Status ArrayReader::ParseTypeValues(const DataType& type) {
+  return VisitTypeInline(type, this);
+}
+
+Status WriteSchema(const Schema& schema, RjWriter* json_writer) {
+  SchemaWriter converter(schema, json_writer);
   return converter.Write();
 }
 
-Status ReadJsonSchema(const rj::Value& json_schema, std::shared_ptr<Schema>* schema) {
-  const auto& obj_schema = json_schema.GetObject();
+static Status LookForDictionaries(const rj::Value& obj, DictionaryTypeMap* id_to_field) {
+  const auto& json_field = obj.GetObject();
+
+  const auto& it_dictionary = json_field.FindMember("dictionary");
+  if (it_dictionary == json_field.MemberEnd()) {
+    // Not dictionary-encoded
+    return Status::OK();
+  }
+
+  // Dictionary encoded. Construct the field and set in the type map
+  std::shared_ptr<Field> dictionary_field;
+  RETURN_NOT_OK(GetField(obj, nullptr, &dictionary_field));
+
+  int id;
+  RETURN_NOT_OK(GetObjectInt(it_dictionary->value.GetObject(), "id", &id));
+  (*id_to_field)[id] = dictionary_field;
+  return Status::OK();
+}
+
+static Status GetDictionaryTypes(const RjArray& fields, DictionaryTypeMap* id_to_field) {
+  for (rj::SizeType i = 0; i < fields.Size(); ++i) {
+    RETURN_NOT_OK(LookForDictionaries(fields[i], id_to_field));
+  }
+  return Status::OK();
+}
+
+static Status ReadDictionary(const RjObject& obj, const DictionaryTypeMap& id_to_field,
+    MemoryPool* pool, int64_t* dictionary_id, std::shared_ptr<Array>* out) {
+  int id;
+  RETURN_NOT_OK(GetObjectInt(obj, "id", &id));
 
-  const auto& json_fields = obj_schema.FindMember("fields");
-  RETURN_NOT_ARRAY("fields", json_fields, obj_schema);
+  const auto& it_data = obj.FindMember("data");
+  RETURN_NOT_OBJECT("data", it_data, obj);
+
+  auto it = id_to_field.find(id);
+  if (it == id_to_field.end()) {
+    std::stringstream ss;
+    ss << "No dictionary with id " << id;
+    return Status::Invalid(ss.str());
+  }
+  std::vector<std::shared_ptr<Field>> fields = {it->second};
+
+  // We need a schema for the record batch
+  auto dummy_schema = std::make_shared<Schema>(fields);
+
+  // The dictionary is embedded in a record batch with a single column
+  std::shared_ptr<RecordBatch> batch;
+  RETURN_NOT_OK(ReadRecordBatch(it_data->value, dummy_schema, pool, &batch));
+
+  if (batch->num_columns() != 1) {
+    return Status::Invalid("Dictionary record batch must only contain one field");
+  }
+
+  *dictionary_id = id;
+  *out = batch->column(0);
+  return Status::OK();
+}
+
+static Status ReadDictionaries(const rj::Value& doc, const DictionaryTypeMap& id_to_field,
+    MemoryPool* pool, DictionaryMemo* dictionary_memo) {
+  auto it = doc.FindMember("dictionaries");
+  if (it == doc.MemberEnd()) {
+    // No dictionaries
+    return Status::OK();
+  }
+
+  RETURN_NOT_ARRAY("dictionaries", it, doc);
+  const auto& dictionary_array = it->value.GetArray();
+
+  for (const rj::Value& val : dictionary_array) {
+    DCHECK(val.IsObject());
+    int64_t dictionary_id;
+    std::shared_ptr<Array> dictionary;
+    RETURN_NOT_OK(
+        ReadDictionary(val.GetObject(), id_to_field, pool, &dictionary_id, &dictionary));
+
+    RETURN_NOT_OK(dictionary_memo->AddDictionary(dictionary_id, dictionary));
+  }
+  return Status::OK();
+}
+
+Status ReadSchema(
+    const rj::Value& json_schema, MemoryPool* pool, std::shared_ptr<Schema>* schema) {
+  auto it = json_schema.FindMember("schema");
+  RETURN_NOT_OBJECT("schema", it, json_schema);
+  const auto& obj_schema = it->value.GetObject();
+
+  const auto& it_fields = obj_schema.FindMember("fields");
+  RETURN_NOT_ARRAY("fields", it_fields, obj_schema);
+
+  // Determine the dictionary types
+  DictionaryTypeMap dictionary_types;
+  RETURN_NOT_OK(GetDictionaryTypes(it_fields->value.GetArray(), &dictionary_types));
+
+  // Read the dictionaries (if any) and cache in the memo
+  DictionaryMemo dictionary_memo;
+  RETURN_NOT_OK(ReadDictionaries(json_schema, dictionary_types, pool, &dictionary_memo));
 
   std::vector<std::shared_ptr<Field>> fields;
-  RETURN_NOT_OK(GetFieldsFromArray(json_fields->value, &fields));
+  RETURN_NOT_OK(GetFieldsFromArray(it_fields->value, &dictionary_memo, &fields));
 
   *schema = std::make_shared<Schema>(fields);
   return Status::OK();
 }
 
-Status WriteJsonArray(
-    const std::string& name, const Array& array, RjWriter* json_writer) {
-  JsonArrayWriter converter(name, array, json_writer);
+Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr<Schema>& schema,
+    MemoryPool* pool, std::shared_ptr<RecordBatch>* batch) {
+  DCHECK(json_obj.IsObject());
+  const auto& batch_obj = json_obj.GetObject();
+
+  auto it = batch_obj.FindMember("count");
+  RETURN_NOT_INT("count", it, batch_obj);
+  int32_t num_rows = static_cast<int32_t>(it->value.GetInt());
+
+  it = batch_obj.FindMember("columns");
+  RETURN_NOT_ARRAY("columns", it, batch_obj);
+  const auto& json_columns = it->value.GetArray();
+
+  std::vector<std::shared_ptr<Array>> columns(json_columns.Size());
+  for (int i = 0; i < static_cast<int>(columns.size()); ++i) {
+    const std::shared_ptr<DataType>& type = schema->field(i)->type();
+    RETURN_NOT_OK(ReadArray(pool, json_columns[i], type, &columns[i]));
+  }
+
+  *batch = std::make_shared<RecordBatch>(schema, num_rows, columns);
+  return Status::OK();
+}
+
+Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer) {
+  writer->StartObject();
+  writer->Key("count");
+  writer->Int(static_cast<int32_t>(batch.num_rows()));
+
+  writer->Key("columns");
+  writer->StartArray();
+
+  for (int i = 0; i < batch.num_columns(); ++i) {
+    const std::shared_ptr<Array>& column = batch.column(i);
+
+    DCHECK_EQ(batch.num_rows(), column->length())
+        << "Array length did not match record batch length";
+
+    RETURN_NOT_OK(WriteArray(batch.column_name(i), *column, writer));
+  }
+
+  writer->EndArray();
+  writer->EndObject();
+  return Status::OK();
+}
+
+Status WriteArray(const std::string& name, const Array& array, RjWriter* json_writer) {
+  ArrayWriter converter(name, array, json_writer);
   return converter.Write();
 }
 
-Status ReadJsonArray(MemoryPool* pool, const rj::Value& json_array,
+Status ReadArray(MemoryPool* pool, const rj::Value& json_array,
     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
-  JsonArrayReader converter(pool);
-  return converter.GetArray(json_array, type, array);
+  ArrayReader converter(json_array, type, pool);
+  return converter.GetArray(array);
 }
 
-Status ReadJsonArray(MemoryPool* pool, const rj::Value& json_array, const Schema& schema,
+Status ReadArray(MemoryPool* pool, const rj::Value& json_array, const Schema& schema,
     std::shared_ptr<Array>* array) {
   if (!json_array.IsObject()) { return Status::Invalid("Element was not a JSON object"); }
 
   const auto& json_obj = json_array.GetObject();
 
-  const auto& json_name = json_obj.FindMember("name");
-  RETURN_NOT_STRING("name", json_name, json_obj);
+  const auto& it_name = json_obj.FindMember("name");
+  RETURN_NOT_STRING("name", it_name, json_obj);
 
-  std::string name = json_name->value.GetString();
+  std::string name = it_name->value.GetString();
 
   std::shared_ptr<Field> result = nullptr;
   for (const std::shared_ptr<Field>& field : schema.fields()) {
@@ -1204,8 +1429,10 @@ Status ReadJsonArray(MemoryPool* pool, const rj::Value& json_array, const Schema
     return Status::KeyError(ss.str());
   }
 
-  return ReadJsonArray(pool, json_array, result->type(), array);
+  return ReadArray(pool, json_array, result->type(), array);
 }
 
+}  // namespace internal
+}  // namespace json
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/json-internal.h b/cpp/src/arrow/ipc/json-internal.h
index 0c167a4ec53a2..5571d9233969c 100644
--- a/cpp/src/arrow/ipc/json-internal.h
+++ b/cpp/src/arrow/ipc/json-internal.h
@@ -35,9 +35,11 @@
 
 namespace rj = rapidjson;
 using RjWriter = rj::Writer<rj::StringBuffer>;
+using RjArray = rj::Value::ConstArray;
+using RjObject = rj::Value::ConstObject;
 
 #define RETURN_NOT_FOUND(TOK, NAME, PARENT) \
-  if (NAME == PARENT.MemberEnd()) {         \
+  if (NAME == (PARENT).MemberEnd()) {       \
     std::stringstream ss;                   \
     ss << "field " << TOK << " not found";  \
     return Status::Invalid(ss.str());       \
@@ -90,21 +92,27 @@ using RjWriter = rj::Writer<rj::StringBuffer>;
 
 namespace arrow {
 namespace ipc {
+namespace json {
+namespace internal {
 
-// TODO(wesm): Only exporting these because arrow_ipc does not have a static
-// library at the moment. Better to not export
-Status ARROW_EXPORT WriteJsonSchema(const Schema& schema, RjWriter* json_writer);
-Status ARROW_EXPORT WriteJsonArray(
-    const std::string& name, const Array& array, RjWriter* json_writer);
+Status WriteSchema(const Schema& schema, RjWriter* writer);
+Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer);
+Status WriteArray(const std::string& name, const Array& array, RjWriter* writer);
 
-Status ARROW_EXPORT ReadJsonSchema(
-    const rj::Value& json_obj, std::shared_ptr<Schema>* schema);
-Status ARROW_EXPORT ReadJsonArray(MemoryPool* pool, const rj::Value& json_obj,
+Status ReadSchema(
+    const rj::Value& json_obj, MemoryPool* pool, std::shared_ptr<Schema>* schema);
+
+Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr<Schema>& schema,
+    MemoryPool* pool, std::shared_ptr<RecordBatch>* batch);
+
+Status ReadArray(MemoryPool* pool, const rj::Value& json_obj,
     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array);
 
-Status ARROW_EXPORT ReadJsonArray(MemoryPool* pool, const rj::Value& json_obj,
-    const Schema& schema, std::shared_ptr<Array>* array);
+Status ReadArray(MemoryPool* pool, const rj::Value& json_obj, const Schema& schema,
+    std::shared_ptr<Array>* array);
 
+}  // namespace internal
+}  // namespace json
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index 0abd6d7ffe3df..f8c0b62ca9a86 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -45,9 +45,7 @@ class JsonWriter::JsonWriterImpl {
 
   Status Start() {
     writer_->StartObject();
-
-    writer_->Key("schema");
-    RETURN_NOT_OK(WriteJsonSchema(*schema_.get(), writer_.get()));
+    RETURN_NOT_OK(json::internal::WriteSchema(*schema_, writer_.get()));
 
     // Record batches
     writer_->Key("batches");
@@ -65,26 +63,7 @@ class JsonWriter::JsonWriterImpl {
 
   Status WriteRecordBatch(const RecordBatch& batch) {
     DCHECK_EQ(batch.num_columns(), schema_->num_fields());
-
-    writer_->StartObject();
-    writer_->Key("count");
-    writer_->Int(static_cast<int32_t>(batch.num_rows()));
-
-    writer_->Key("columns");
-    writer_->StartArray();
-
-    for (int i = 0; i < schema_->num_fields(); ++i) {
-      const std::shared_ptr<Array>& column = batch.column(i);
-
-      DCHECK_EQ(batch.num_rows(), column->length())
-          << "Array length did not match record batch length";
-
-      RETURN_NOT_OK(WriteJsonArray(schema_->field(i)->name(), *column, writer_.get()));
-    }
-
-    writer_->EndArray();
-    writer_->EndObject();
-    return Status::OK();
+    return json::internal::WriteRecordBatch(batch, writer_.get());
   }
 
  private:
@@ -127,11 +106,9 @@ class JsonReader::JsonReaderImpl {
         static_cast<size_t>(data_->size()));
     if (doc_.HasParseError()) { return Status::IOError("JSON parsing failed"); }
 
-    auto it = doc_.FindMember("schema");
-    RETURN_NOT_OBJECT("schema", it, doc_);
-    RETURN_NOT_OK(ReadJsonSchema(it->value, &schema_));
+    RETURN_NOT_OK(json::internal::ReadSchema(doc_, pool_, &schema_));
 
-    it = doc_.FindMember("batches");
+    auto it = doc_.FindMember("batches");
     RETURN_NOT_ARRAY("batches", it, doc_);
     record_batches_ = &it->value;
 
@@ -143,27 +120,8 @@ class JsonReader::JsonReaderImpl {
     DCHECK_LT(i, static_cast<int>(record_batches_->GetArray().Size()))
         << "i out of bounds";
 
-    const auto& batch_val = record_batches_->GetArray()[i];
-    DCHECK(batch_val.IsObject());
-
-    const auto& batch_obj = batch_val.GetObject();
-
-    auto it = batch_obj.FindMember("count");
-    RETURN_NOT_INT("count", it, batch_obj);
-    int32_t num_rows = static_cast<int32_t>(it->value.GetInt());
-
-    it = batch_obj.FindMember("columns");
-    RETURN_NOT_ARRAY("columns", it, batch_obj);
-    const auto& json_columns = it->value.GetArray();
-
-    std::vector<std::shared_ptr<Array>> columns(json_columns.Size());
-    for (int i = 0; i < static_cast<int>(columns.size()); ++i) {
-      const std::shared_ptr<DataType>& type = schema_->field(i)->type();
-      RETURN_NOT_OK(ReadJsonArray(pool_, json_columns[i], type, &columns[i]));
-    }
-
-    *batch = std::make_shared<RecordBatch>(schema_, num_rows, columns);
-    return Status::OK();
+    return json::internal::ReadRecordBatch(
+        record_batches_->GetArray()[i], schema_, pool_, batch);
   }
 
   std::shared_ptr<Schema> schema() const { return schema_; }
@@ -178,7 +136,6 @@ class JsonReader::JsonReaderImpl {
   rj::Document doc_;
 
   const rj::Value* record_batches_;
-
   std::shared_ptr<Schema> schema_;
 };
 
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
index 0d88cef9a4d7b..ad94defdeabf3 100644
--- a/cpp/src/arrow/ipc/json.h
+++ b/cpp/src/arrow/ipc/json.h
@@ -44,10 +44,7 @@ class ARROW_EXPORT JsonWriter {
   static Status Open(
       const std::shared_ptr<Schema>& schema, std::unique_ptr<JsonWriter>* out);
 
-  // TODO(wesm): Write dictionaries
-
   Status WriteRecordBatch(const RecordBatch& batch);
-
   Status Finish(std::string* result);
 
  private:
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 84026c452ad27..ec7bc392f6fb8 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -77,6 +77,7 @@ class ARROW_EXPORT DictionaryMemo {
   // Returns KeyError if dictionary not found
   Status GetDictionary(int64_t id, std::shared_ptr<Array>* dictionary) const;
 
+  /// Return id for dictionary, computing new id if necessary
   int64_t GetId(const std::shared_ptr<Array>& dictionary);
 
   bool HasDictionary(const std::shared_ptr<Array>& dictionary) const;
@@ -88,6 +89,8 @@ class ARROW_EXPORT DictionaryMemo {
 
   const DictionaryMap& id_to_dictionary() const { return id_to_dictionary_; }
 
+  int size() const { return static_cast<int>(id_to_dictionary_.size()); }
+
  private:
   // Dictionary memory addresses, to track whether a dictionary has been seen
   // before
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 5caa3a9808c61..deaeb597f9095 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -55,10 +55,17 @@ static inline void CompareBatch(const RecordBatch& left, const RecordBatch& righ
   }
   ASSERT_EQ(left.num_columns(), right.num_columns())
       << left.schema()->ToString() << " result: " << right.schema()->ToString();
-  EXPECT_EQ(left.num_rows(), right.num_rows());
+  ASSERT_EQ(left.num_rows(), right.num_rows());
   for (int i = 0; i < left.num_columns(); ++i) {
-    EXPECT_TRUE(left.column(i)->Equals(right.column(i)))
-        << "Idx: " << i << " Name: " << left.column_name(i);
+    if (!left.column(i)->Equals(right.column(i))) {
+      std::stringstream ss;
+      ss << "Idx: " << i << " Name: " << left.column_name(i);
+      ss << std::endl << "Left: ";
+      ASSERT_OK(PrettyPrint(*left.column(i), 0, &ss));
+      ss << std::endl << "Right: ";
+      ASSERT_OK(PrettyPrint(*right.column(i), 0, &ss));
+      FAIL() << ss.str();
+    }
   }
 }
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 5d4b94a805a64..60b1f474a39c6 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -591,10 +591,7 @@ RecordBatchWriter::~RecordBatchWriter() {}
 class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
  public:
   RecordBatchStreamWriterImpl()
-      : dictionary_memo_(std::make_shared<DictionaryMemo>()),
-        pool_(default_memory_pool()),
-        position_(-1),
-        started_(false) {}
+      : pool_(default_memory_pool()), position_(-1), started_(false) {}
 
   virtual ~RecordBatchStreamWriterImpl() = default;
 
@@ -606,7 +603,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
 
   virtual Status Start() {
     std::shared_ptr<Buffer> schema_fb;
-    RETURN_NOT_OK(WriteSchemaMessage(*schema_, dictionary_memo_.get(), &schema_fb));
+    RETURN_NOT_OK(WriteSchemaMessage(*schema_, &dictionary_memo_, &schema_fb));
 
     int32_t flatbuffer_size = static_cast<int32_t>(schema_fb->size());
     RETURN_NOT_OK(
@@ -640,7 +637,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
   Status UpdatePosition() { return sink_->Tell(&position_); }
 
   Status WriteDictionaries() {
-    const DictionaryMap& id_to_dictionary = dictionary_memo_->id_to_dictionary();
+    const DictionaryMap& id_to_dictionary = dictionary_memo_.id_to_dictionary();
 
     dictionaries_.resize(id_to_dictionary.size());
 
@@ -709,7 +706,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
 
   // When writing out the schema, we keep track of all the dictionaries we
   // encounter, as they must be written out first in the stream
-  std::shared_ptr<DictionaryMemo> dictionary_memo_;
+  DictionaryMemo dictionary_memo_;
 
   MemoryPool* pool_;
 
@@ -770,7 +767,7 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
     // Write metadata
     int64_t initial_position = position_;
     RETURN_NOT_OK(WriteFileFooter(
-        *schema_, dictionaries_, record_batches_, dictionary_memo_.get(), sink_));
+        *schema_, dictionaries_, record_batches_, &dictionary_memo_, sink_));
     RETURN_NOT_OK(UpdatePosition());
 
     // Write footer length

From d25ea6314bce574bfc6f93e97c542c69442314b4 Mon Sep 17 00:00:00 2001
From: Hideo Hattori <hattori-h@klab.com>
Date: Tue, 13 Jun 2017 11:06:17 +0900
Subject: [PATCH 0721/1644] ARROW-1115: [C++] use CCACHE_FOUND value for ccache
 path

Closes #751

Change-Id: I7afa8a65d4ad1180ff7f8b85da635a68f04e62e3
---
 cpp/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d5483f228029c..c295fa0ba8005 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -57,8 +57,8 @@ endif()
 
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
-  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
-  set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
 endif(CCACHE_FOUND)
 
 # ----------------------------------------------------------------------

From 697df1b5083473eb4888ec20802b973556d135ab Mon Sep 17 00:00:00 2001
From: Kengo Seki <sekikn@apache.org>
Date: Wed, 14 Jun 2017 12:05:51 +0900
Subject: [PATCH 0722/1644] ARROW-1117: [Docs] Minor issues in GLib README

Author: Kengo Seki <sekikn@apache.org>

Closes #753 from sekikn/ARROW-1117 and squashes the following commits:

5411349 [Kengo Seki] ARROW-1117: [Docs] Minor issues in GLib README
---
 c_glib/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index f8269df5eac91..622938550d830 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -50,15 +50,15 @@ released in the future.
 
 ### Package
 
-See [install document][../site/install.md] for details.
+See [install document](../site/install.md) for details.
 
 ### How to build by users
 
 Arrow GLib users should use released source archive to build Arrow
-GLib:
+GLib (replace the version number in the following commands with the one you use):
 
 ```text
-% wget https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
+% wget https://archive.apache.org/dist/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
 % tar xf apache-arrow-0.3.0.tar.gz
 % cd apache-arrow-0.3.0
 ```

From d1de66bdaffedd07616d37fde6fc0265bf51868f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 15 Jun 2017 09:02:11 +0200
Subject: [PATCH 0723/1644] ARROW-1118: [Site] Website updates for 0.4.1

This has been deployed preliminarily for the release. I have not yet updated the subproject documentations yet; help would be appreciated

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #755 from wesm/ARROW-1118 and squashes the following commits:

0bdac0e [Wes McKinney] Website updates for 0.4.1
---
 CHANGELOG.md                            | 145 ++++++++++++++++++++++++
 site/_posts/2017-06-14-0.4.1-release.md |  67 +++++++++++
 site/_release/0.4.1.md                  |  98 ++++++++++++++++
 site/_release/index.md                  |   2 +
 site/index.html                         |   4 +-
 site/install.md                         |  26 ++---
 6 files changed, 327 insertions(+), 15 deletions(-)
 create mode 100644 site/_posts/2017-06-14-0.4.1-release.md
 create mode 100644 site/_release/0.4.1.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 85a43ef7952d9..55b02e0f9a1a8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,151 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
+# Apache Arrow 0.4.1 (9 June 2017)
+
+## Bug
+
+* ARROW-1039 - Python: pyarrow.Filesystem.read_parquet causing error if nthreads>1
+* ARROW-1050 - [C++] Export arrow::ValidateArray
+* ARROW-1051 - [Python] If pyarrow.parquet fails to import due to a shared library ABI conflict, the test_parquet.py tests silently do not run
+* ARROW-1056 - [Python] Parquet+HDFS test failure due to writing pandas index
+* ARROW-1057 - Fix cmake warning and msvc debug asserts
+* ARROW-1062 - [GLib] Examples use old API
+* ARROW-1066 - remove warning on feather for pandas >= 0.20.1
+* ARROW-1070 - [C++] Feather files for date/time types should be written with the physical types
+* ARROW-1075 - [GLib] Build error on macOS
+* ARROW-1085 - [java] Follow up on template cleanup. Missing method for IntervalYear
+* ARROW-1086 - [Python] pyarrow 0.4.0 on pypi is missing pxd files
+* ARROW-1088 - [Python] test_unicode_filename test fails when unicode filenames aren't supported by system
+* ARROW-1090 - [Python] build_ext usability
+* ARROW-1091 - Decimal scale and precision are flipped
+* ARROW-1092 - More Decimal and scale flipped follow-up
+* ARROW-1094 - [C++] Incomplete buffer reads in arrow::io::ReadableFile should exactly truncate returned buffer
+* ARROW-424 - [C++] Threadsafety in arrow/io/hdfs.h
+
+## Improvement
+
+* ARROW-1020 - [Format] Add additional language to Schema.fbs to clarify naive vs. localized Timestamp values
+* ARROW-1034 - [Python] Enable creation of binary wheels on Windows / MSVC
+* ARROW-1049 - [java] vector template cleanup
+* ARROW-1063 - [Website] Blog post and website updates for 0.4.0 release
+* ARROW-1078 - [Python] Account for PARQUET-967
+* ARROW-1080 - C++: Add tutorial about converting to/from row-wise representation
+* ARROW-897 - [GLib] Build arrow-glib as a separate build in the Travis CI build matrix
+* ARROW-986 - [Format] Update IPC.md to account for dictionary batches
+* ARROW-990 - [JS] Add tslint support for linting TypeScript
+
+## Task
+
+* ARROW-1068 - [Python] Create external repo with appveyor.yml configured for building Python wheel installers
+* ARROW-1069 - Add instructions for publishing maven artifacts
+* ARROW-1084 - Implementations of BufferAllocator should handle Netty's OutOfDirectMemoryError
+
+## Test
+
+* ARROW-1060 - [Python] Add unit test for ARROW-1053
+* ARROW-1082 - [GLib] Add CI on macOS
+
+# Apache Arrow 0.4.0 (22 May 2017)
+
+## Bug
+
+* ARROW-1003 - [C++] Hdfs and java dlls fail to load when built for Windows with MSVC
+* ARROW-1004 - ArrowInvalid: Invalid: Python object of type float is not None and is not a string, bool, or date object
+* ARROW-1017 - Python: Table.to_pandas leaks memory
+* ARROW-1023 - Python: Fix bundling of arrow-cpp for macOS
+* ARROW-1033 - [Python] pytest discovers scripts/test_leak.py
+* ARROW-1046 - [Python] Conform DataFrame metadata to pandas spec
+* ARROW-1053 - [Python] Memory leak with RecordBatchFileReader
+* ARROW-1054 - [Python] Test suite fails on pandas 0.19.2
+* ARROW-1061 - [C++] Harden decimal parsing against invalid strings
+* ARROW-1064 - ModuleNotFoundError: No module named 'pyarrow._parquet'
+* ARROW-813 - [Python] setup.py sdist must also bundle dependent cmake modules
+* ARROW-824 - Date and Time Vectors should reflect timezone-less semantics
+* ARROW-856 - CmakeError by Unknown compiler.
+* ARROW-881 - [Python] Reconstruct Pandas DataFrame indexes using custom_metadata
+* ARROW-909 - libjemalloc.so.2: cannot open shared object file:
+* ARROW-939 - Fix division by zero for zero-dimensional Tensors
+* ARROW-940 - [JS] Generate multiple sets of artifacts
+* ARROW-944 - Python: Compat broken for pandas==0.18.1
+* ARROW-948 - [GLib] Update C++ header file list
+* ARROW-952 - Compilation error on macOS with clang-802.0.42
+* ARROW-958 - [Python] Conda build guide still needs ARROW_HOME, PARQUET_HOME
+* ARROW-979 - [Python] Fix setuptools_scm version when release tag is not in the master timeline
+* ARROW-991 - [Python] PyArray_SimpleNew should not be used with NPY_DATETIME
+* ARROW-995 - [Website] 0.3 release announce has a typo in reference
+* ARROW-998 - [Doc] File format documents incorrect schema location
+
+## Improvement
+
+* ARROW-1000 - [GLib] Move install document to Website
+* ARROW-1001 - [GLib] Unify writer files
+* ARROW-1002 - [C++] It is not necessary to add padding after the magic header in the FileWriter implementation
+* ARROW-1010 - [Website] Only show English posts in /blog/
+* ARROW-1016 - Python: Include C++ headers (optionally) in wheels
+* ARROW-1022 - [Python] Add nthreads option to Feather read method
+* ARROW-1024 - Python: Update build time numpy version to 1.10.1
+* ARROW-1025 - [Website] Improve changelog on website
+* ARROW-1027 - [Python] Allow negative indexing in fields/columns on pyarrow Table and Schema objects
+* ARROW-1028 - [Python] Documentation updates after ARROW-1008
+* ARROW-1029 - [Python] Fix --with-parquet build on Windows, add unit tests to Appveyor
+* ARROW-1030 - Python: Account for library versioning in parquet-cpp
+* ARROW-1037 - [GLib] Follow reader name change
+* ARROW-1038 - [GLib] Follow writer name change
+* ARROW-1040 - [GLib] Follow tensor IO
+* ARROW-182 - [C++] Remove Array::Validate virtual function and make a separate method
+* ARROW-376 - Python: Convert non-range Pandas indices (optionally) to Arrow
+* ARROW-532 - [Python] Expand pyarrow.parquet documentation for 0.3 release
+* ARROW-579 - Python: Provide redistributable pyarrow wheels on OSX
+* ARROW-891 - [Python] Expand Windows build instructions to not require looking at separate C++ docs
+* ARROW-899 - [Docs] Add CHANGELOG for 0.3.0
+* ARROW-901 - [Python] Write FixedSizeBinary to Parquet
+* ARROW-913 - [Python] Only link jemalloc to the Cython extension where it's needed
+* ARROW-923 - [Docs] Generate Changelog for website with JIRA links
+* ARROW-929 - Move KEYS file to SVN, remove from git
+* ARROW-943 - [GLib] Support running unit tests with source archive
+* ARROW-945 - [GLib] Add a Lua example to show Torch integration
+* ARROW-946 - [GLib] Use "new" instead of "open" for constructor name
+* ARROW-947 - [Python] Improve execution time of manylinux1 build
+* ARROW-953 - Use cmake / curl from conda-forge in CI builds
+* ARROW-954 - Make it possible to compile Arrow with header-only boost
+* ARROW-961 - [Python] Rename InMemoryOutputStream to BufferOutputStream
+* ARROW-970 - [Python] Accidentally calling pyarrow.Table() should not segfault process
+* ARROW-982 - [Website] Improve website front copy to highlight serialization efficiency benefits
+* ARROW-984 - [GLib] Add Go examples
+* ARROW-985 - [GLib] Update package information
+* ARROW-988 - [JS] Add entry to Travis CI matrix
+* ARROW-993 - [GLib] Add missing error checks in Go examples
+* ARROW-996 - [Website] Add 0.3 release announce in Japanese
+
+## New Feature
+
+* ARROW-1008 - [C++] Define abstract interface for stream iteration
+* ARROW-1011 - [Format] Clarify requirements around buffer padding in validity bitmaps
+* ARROW-1014 - 0.4.0 release
+* ARROW-1031 - [GLib] Support pretty print
+* ARROW-1044 - [GLib] Support Feather
+* ARROW-29 - C++: Add re2 as optional 3rd-party toolchain dependency
+* ARROW-446 - [Python] Document NativeFile interfaces, HDFS client in Sphinx
+* ARROW-482 - [Java] Provide API access to "custom_metadata" Field attribute in IPC setting
+* ARROW-596 - [Python] Add convenience function to convert pandas.DataFrame to pyarrow.Buffer containing a file or stream representation
+* ARROW-714 - [C++] Add import_pyarrow C API in the style of NumPy for thirdparty C++ users
+* ARROW-819 - [Python] Define public Cython API
+* ARROW-872 - [JS] Read streaming format
+* ARROW-873 - [JS] Implement fixed width list type
+* ARROW-874 - [JS] Read dictionary-encoded vectors
+* ARROW-963 - [GLib] Add equal
+* ARROW-967 - [GLib] Support initializing array with buffer
+* ARROW-977 - [java] Add Timezone aware timestamp vectors
+
+## Task
+
+* ARROW-1015 - [Java] Implement schema-level metadata
+* ARROW-629 - [JS] Add unit test suite
+* ARROW-956 - remove pandas pre-0.20.0 compat
+* ARROW-957 - [Doc] Add HDFS and Windows documents to doxygen output
+* ARROW-997 - [Java] Implement transfer in FixedSizeListVector
+
 # Apache Arrow 0.3.0 (5 May 2017)
 
 ## Bug
diff --git a/site/_posts/2017-06-14-0.4.1-release.md b/site/_posts/2017-06-14-0.4.1-release.md
new file mode 100644
index 0000000000000..4dd8b4e1f0892
--- /dev/null
+++ b/site/_posts/2017-06-14-0.4.1-release.md
@@ -0,0 +1,67 @@
+---
+layout: post
+title: "Apache Arrow 0.4.1 Release"
+date: "2017-06-14 10:00:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.4.1 release of the
+project. This is a bug fix release that addresses a regression with Decimal
+types in the Java implementation introduced in 0.4.0 (see
+[ARROW-1091][10]). There were a total of [31 resolved JIRAs][1].
+
+See the [Install Page][2] to learn how to get the libraries for your platform.
+
+### Python Wheel Installers for Windows
+
+Max Risuhin contributed fixes to enable binary wheel installers to be generated
+for Python 3.5 and 3.6. Thus, 0.4.1 is the first Arrow release for which
+PyArrow including bundled [Apache Parquet][8] support that can be installed
+with either conda or pip across the 3 major platforms: Linux, macOS, and
+Windows. Use one of:
+
+```
+pip install pyarrow
+conda install pyarrow -c conda-forge
+```
+
+### Turbodbc 2.0.0 with Apache Arrow Support
+
+[Turbodbc][12], a fast C++ ODBC interface with Python bindings, released
+version 2.0.0 including reading SQL result sets as Arrow record batches. The
+team used the PyArrow C++ API introduced in version 0.4.0 to construct
+`pyarrow.Table` objects inside the `turbodbc` library. Learn more in their
+[documentation][13] and install with one of:
+
+```
+pip install turbodbc
+conda install turbodbc -c conda-forge
+```
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.4.1
+[2]: http://arrow.apache.org/install
+[8]: http://parquet.apache.org
+[9]: http://arrow.apache.org/docs/python/development.html
+[10]: https://issues.apache.org/jira/browse/ARROW-1091
+[11]: https://github.com/maxris
+[12]: http://turbodbc.readthedocs.io/
+[13]: http://turbodbc.readthedocs.io/en/latest/pages/advanced_usage.html#apache-arrow-support
\ No newline at end of file
diff --git a/site/_release/0.4.1.md b/site/_release/0.4.1.md
new file mode 100644
index 0000000000000..7d48a99a4cd25
--- /dev/null
+++ b/site/_release/0.4.1.md
@@ -0,0 +1,98 @@
+---
+layout: default
+title: 0.4.1 Release
+permalink: /release/0.4.1.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.4.1 (9 June 2017)
+
+This is primarily a bug fix release, but also includes some packaging and
+documentation improvements.
+
+Read more in the [release blog post][8].
+
+## Download
+
+* [**Source Artifacts**][6]
+* [Git tag 46315431][2]
+
+# Changelog
+
+## New Features and Improvements
+
+* [ARROW-1020](https://issues.apache.org/jira/browse/ARROW-1020) - [Format] Add additional language to Schema.fbs to clarify naive vs. localized Timestamp values
+* [ARROW-1034](https://issues.apache.org/jira/browse/ARROW-1034) - [Python] Enable creation of binary wheels on Windows / MSVC
+* [ARROW-1049](https://issues.apache.org/jira/browse/ARROW-1049) - [java] vector template cleanup
+* [ARROW-1060](https://issues.apache.org/jira/browse/ARROW-1060) - [Python] Add unit test for ARROW-1053
+* [ARROW-1063](https://issues.apache.org/jira/browse/ARROW-1063) - [Website] Blog post and website updates for 0.4.0 release
+* [ARROW-1068](https://issues.apache.org/jira/browse/ARROW-1068) - [Python] Create external repo with appveyor.yml configured for building Python wheel installers
+* [ARROW-1069](https://issues.apache.org/jira/browse/ARROW-1069) - Add instructions for publishing maven artifacts
+* [ARROW-1078](https://issues.apache.org/jira/browse/ARROW-1078) - [Python] Account for PARQUET-967
+* [ARROW-1080](https://issues.apache.org/jira/browse/ARROW-1080) - C++: Add tutorial about converting to/from row-wise representation
+* [ARROW-1082](https://issues.apache.org/jira/browse/ARROW-1082) - [GLib] Add CI on macOS
+* [ARROW-1084](https://issues.apache.org/jira/browse/ARROW-1084) - Implementations of BufferAllocator should handle Netty's OutOfDirectMemoryError
+* [ARROW-897](https://issues.apache.org/jira/browse/ARROW-897) - [GLib] Build arrow-glib as a separate build in the Travis CI build matrix
+* [ARROW-986](https://issues.apache.org/jira/browse/ARROW-986) - [Format] Update IPC.md to account for dictionary batches
+* [ARROW-990](https://issues.apache.org/jira/browse/ARROW-990) - [JS] Add tslint support for linting TypeScript
+
+## Bug Fixes
+
+* [ARROW-1039](https://issues.apache.org/jira/browse/ARROW-1039) - Python: pyarrow.Filesystem.read_parquet causing error if nthreads>1
+* [ARROW-1050](https://issues.apache.org/jira/browse/ARROW-1050) - [C++] Export arrow::ValidateArray
+* [ARROW-1051](https://issues.apache.org/jira/browse/ARROW-1051) - [Python] If pyarrow.parquet fails to import due to a shared library ABI conflict, the test_parquet.py tests silently do not run
+* [ARROW-1056](https://issues.apache.org/jira/browse/ARROW-1056) - [Python] Parquet+HDFS test failure due to writing pandas index
+* [ARROW-1057](https://issues.apache.org/jira/browse/ARROW-1057) - Fix cmake warning and msvc debug asserts
+* [ARROW-1062](https://issues.apache.org/jira/browse/ARROW-1062) - [GLib] Examples use old API
+* [ARROW-1066](https://issues.apache.org/jira/browse/ARROW-1066) - remove warning on feather for pandas >= 0.20.1
+* [ARROW-1070](https://issues.apache.org/jira/browse/ARROW-1070) - [C++] Feather files for date/time types should be written with the physical types
+* [ARROW-1075](https://issues.apache.org/jira/browse/ARROW-1075) - [GLib] Build error on macOS
+* [ARROW-1085](https://issues.apache.org/jira/browse/ARROW-1085) - [java] Follow up on template cleanup. Missing method for IntervalYear
+* [ARROW-1086](https://issues.apache.org/jira/browse/ARROW-1086) - [Python] pyarrow 0.4.0 on pypi is missing pxd files
+* [ARROW-1088](https://issues.apache.org/jira/browse/ARROW-1088) - [Python] test_unicode_filename test fails when unicode filenames aren't supported by system
+* [ARROW-1090](https://issues.apache.org/jira/browse/ARROW-1090) - [Python] build_ext usability
+* [ARROW-1091](https://issues.apache.org/jira/browse/ARROW-1091) - Decimal scale and precision are flipped
+* [ARROW-1092](https://issues.apache.org/jira/browse/ARROW-1092) - More Decimal and scale flipped follow-up
+* [ARROW-1094](https://issues.apache.org/jira/browse/ARROW-1094) - [C++] Incomplete buffer reads in arrow::io::ReadableFile should exactly truncate returned buffer
+* [ARROW-424](https://issues.apache.org/jira/browse/ARROW-424) - [C++] Threadsafety in arrow/io/hdfs.h
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.4.0..apache-arrow-0.4.1
+    16  Wes McKinney
+     5  Julien Le Dem
+     4  Kouhei Sutou
+     2  Jacob Scott
+     1  Brian Hulette
+     1  Jeff Knupp
+     1  Jeff Reback
+     1  Max Risuhin
+     1  Phillip Cloud
+     1  Tony Kelman
+     1  Uwe L. Korn
+     1  adeneche
+     1  kimanjun
+     1  rip.nsk
+```
+
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.4.1
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/
+[8]: http://arrow.apache.org/blog/2017/06/14/0.4.1-release/
\ No newline at end of file
diff --git a/site/_release/index.md b/site/_release/index.md
index a7f9ab6377183..2dd657976229d 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -26,6 +26,7 @@ limitations under the License.
 
 Navigate to the release page for downloads and the changelog.
 
+* [0.4.1 (9 June 2017)][5]
 * [0.4.0 (22 May 2017)][4]
 * [0.3.0 (5 May 2017)][1]
 * [0.2.0 (18 February 2017)][2]
@@ -35,3 +36,4 @@ Navigate to the release page for downloads and the changelog.
 [2]: {{ site.baseurl }}/release/0.2.0.html
 [3]: {{ site.baseurl }}/release/0.1.0.html
 [4]: {{ site.baseurl }}/release/0.4.0.html
+[5]: {{ site.baseurl }}/release/0.4.1.html
diff --git a/site/index.html b/site/index.html
index 620b75151c07f..5b60a5fc3e2ba 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,10 +7,10 @@ <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
         <p>
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
-          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.4.0 Release - May 22, 2017)</a>
+          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.4.1 Release - June 9, 2017)</a>
         </p>
       </div>
-      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.4.0 release</a></h4>
+      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.4.1 release</a></h4>
       <div class="row">
         <div class="col-lg-4">
           <h2>Fast</h2>
diff --git a/site/install.md b/site/install.md
index 9018c6c045408..4252e7f4bf905 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,17 +20,17 @@ limitations under the License.
 {% endcomment %}
 -->
 
-## Current Version: 0.4.0
+## Current Version: 0.4.1
 
-### Released: 22 May 2017
+### Released: 9 June 2017
 
 See the [release notes][10] and [blog post][11] for more about what's new.
 
 ### Source release
 
-* **Source Release**: [apache-arrow-0.4.0.tar.gz][6]
+* **Source Release**: [apache-arrow-0.4.1.tar.gz][6]
 * **Verification**: [md5][3], [asc][7]
-* [Git tag a8f8ba0][2]
+* [Git tag 46315431][2]
 
 ### Java Packages
 
@@ -54,7 +54,7 @@ conda install pyarrow -c conda-forge
 
 ### Python Wheels on PyPI (Unofficial)
 
-We have provided Linux binary wheels on PyPI, which can be installed with pip.
+We have provided binary wheels on PyPI for Linux, macOS, and Windows:
 
 ```shell
 pip install pyarrow
@@ -129,14 +129,14 @@ These repositories are managed at
 [red-data-tools/arrow-packages][9]. If you have any feedback, please
 send it to the project instead of Apache Arrow project.
 
-[1]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.4.0
-[3]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz.md5
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.4.0%22
+[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.4.1
+[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/apache-arrow-0.4.1.tar.gz.md5
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.4.1%22
 [5]: http://conda-forge.github.io
-[6]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz
-[7]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/apache-arrow-0.4.1.tar.gz
+[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/apache-arrow-0.4.1.tar.gz.asc
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
-[10]: http://arrow.apache.org/release/0.4.0.html
-[11]: http://arrow.apache.org/blog/2017/05/23/0.4.0-release/
\ No newline at end of file
+[10]: http://arrow.apache.org/release/0.4.1.html
+[11]: http://arrow.apache.org/blog/2017/06/14/0.4.1-release/
\ No newline at end of file

From 5b66c25c9f7ee5aeb5ce2646af4fcf3ccde702fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20K=C3=B6nig?= <michael.koenig@blue-yonder.com>
Date: Thu, 15 Jun 2017 13:21:49 +0200
Subject: [PATCH 0724/1644] ARROW-1122: [Website] Add turbodbc + arrow blog
 post

Closes #757

Change-Id: Idb147f5681e725f8852108ae3a1b870afc14973c
---
 site/_data/contributors.yml              |   3 +
 site/_posts/2017-06-16-turbodbc-arrow.md | 117 +++++++++++++++++++++++
 site/img/turbodbc_arrow.png              | Bin 0 -> 75697 bytes
 3 files changed, 120 insertions(+)
 create mode 100644 site/_posts/2017-06-16-turbodbc-arrow.md
 create mode 100644 site/img/turbodbc_arrow.png

diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml
index 7bed83d21823d..2a8d95b77b239 100644
--- a/site/_data/contributors.yml
+++ b/site/_data/contributors.yml
@@ -29,4 +29,7 @@
   apacheId: julienledem
   githubId: julienledem
   role: PMC
+- name: Michael König
+  apacheId: MathMagique
+  githubId: MathMagique
 # End contributors.yml
diff --git a/site/_posts/2017-06-16-turbodbc-arrow.md b/site/_posts/2017-06-16-turbodbc-arrow.md
new file mode 100644
index 0000000000000..438e56a0ecd84
--- /dev/null
+++ b/site/_posts/2017-06-16-turbodbc-arrow.md
@@ -0,0 +1,117 @@
+---
+layout: post
+title: "Connecting Relational Databases to the Apache Arrow World with turbodbc"
+date: "2017-06-16 00:00:00 +0200"
+author: MathMagique
+categories: [application]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+*[Michael König][15] is the lead developer of the [turbodbc project][2]*
+
+The [Apache Arrow][1] project set out to become the universal data layer for
+column-oriented data processing systems without incurring serialization costs
+or compromising on performance on a more general level. While relational
+databases still lag behind in Apache Arrow adoption, the Python database module
+[turbodbc][2] brings Apache Arrow support to these databases using a much
+older, more specialized data exchange layer: [ODBC][3].
+
+ODBC is a database interface that offers developers the option to transfer data
+either in row-wise or column-wise fashion. Previous Python ODBC modules typically
+use the row-wise approach, and often trade repeated database roundtrips for simplified
+buffer handling. This makes them less suited for data-intensive applications,
+particularly when interfacing with modern columnar analytical databases.
+
+In contrast, turbodbc was designed to leverage columnar data processing from day
+one. Naturally, this implies using the columnar portion of the ODBC API. Equally
+important, however, is to find new ways of providing columnar data to Python users
+that exceed the capabilities of the row-wise API mandated by Python’s [PEP 249][4].
+Turbodbc has adopted Apache Arrow for this very task with the recently released
+version 2.0.0:
+
+```python
+>>> from turbodbc import connect
+>>> connection = connect(dsn="My columnar database")
+>>> cursor = connection.cursor()
+>>> cursor.execute("SELECT some_integers, some_strings FROM my_table")
+>>> cursor.fetchallarrow()
+pyarrow.Table
+some_integers: int64
+some_strings: string
+```
+
+With this new addition, the data flow for a result set of a typical SELECT query
+is like this:
+*   The database prepares the result set and exposes it to the ODBC driver using
+    either row-wise or column-wise storage.
+*   Turbodbc has the ODBC driver write chunks of the result set into columnar buffers.
+*   These buffers are exposed to turbodbc's Apache Arrow frontend. This frontend
+    will create an Arrow table and fill in the buffered values.
+*   The previous steps are repeated until the entire result set is retrieved.
+
+![Data flow from relational databases to Python with turbodbc and the Apache Arrow frontend]({{ site.url }}/img/turbodbc_arrow.png){:class="img-responsive" width="75%"}
+
+In practice, it is possible to achieve the following ideal situation: A 64-bit integer
+column is stored as one contiguous block of memory in a columnar database. A huge chunk
+of 64-bit integers is transferred over the network and the ODBC driver directly writes
+it to a turbodbc buffer of 64-bit integers. The Arrow frontend accumulates these values
+by copying the entire 64-bit buffer into a free portion of an Arrow table's 64-bit
+integer column.
+
+Moving data from the database to an Arrow table and, thus, providing it to the Python
+user can be as simple as copying memory blocks around, megabytes equivalent to hundred
+thousands of rows at a time. The absence of serialization and conversion logic renders
+the process extremely efficient.
+
+Once the data is stored in an Arrow table, Python users can continue to do some
+actual work. They can convert it into a [Pandas DataFrame][5] for data analysis
+(using a quick `table.to_pandas()`), pass it on to other data processing
+systems such as [Apache Spark][6] or [Apache Impala (incubating)][7], or store
+it in the [Apache Parquet][8] file format. This way, non-Python systems are
+efficiently connected with relational databases.
+
+In the future, turbodbc’s Arrow support will be extended to use more
+sophisticated features such as [dictionary-encoded][9] string fields. We also
+plan to pick smaller than 64-bit [data types][10] where possible. Last but not
+least, Arrow support will be extended to cover the reverse direction of data
+flow, so that Python users can quickly insert Arrow tables into relational
+databases.
+
+If you would like to learn more about turbodbc, check out the [GitHub project][2] and the
+[project documentation][11]. If you want to learn more about how turbodbc implements the
+nitty-gritty details, check out parts [one][12] and [two][13] of the
+["Making of turbodbc"][12] series at [Blue Yonder's technology blog][14].
+
+
+[1]: https://arrow.apache.org/
+[2]: https://github.com/blue-yonder/turbodbc
+[3]: https://en.wikipedia.org/wiki/Open_Database_Connectivity
+[4]: https://www.python.org/dev/peps/pep-0249/
+[5]: https://arrow.apache.org/docs/python/pandas.html
+[6]: http://spark.apache.org/
+[7]: http://impala.apache.org/
+[8]: http://parquet.apache.org/
+[9]: https://arrow.apache.org/docs/memory_layout.html#dictionary-encoding
+[10]: https://arrow.apache.org/docs/metadata.html#integers
+[11]: http://turbodbc.readthedocs.io/
+[12]: https://tech.blue-yonder.com/making-of-turbodbc-part-1-wrestling-with-the-side-effects-of-a-c-api/
+[13]: https://tech.blue-yonder.com/making-of-turbodbc-part-2-c-to-python/
+[14]: https://tech.blue-yonder.com/
+[15]: https://github.com/mathmagique
\ No newline at end of file
diff --git a/site/img/turbodbc_arrow.png b/site/img/turbodbc_arrow.png
new file mode 100644
index 0000000000000000000000000000000000000000..b534bf99a0e048de49ab2db9568c176de2d3acb1
GIT binary patch
literal 75697
zcmZs?by$>N)HN)PG>CMgpomC!NeC(e(nvSb-HnvANJ)!=2-4jl3^>4u#Lyuzz|ceI
zcgElIyx;rBJ6>}U!hPRopS{;wd+kGvx~dW}0Ug1uTepaxD=TQ+x`mB@>lS7iJ}&r<
z+0VpD@B!ESrINy}8}z^Yw&E1<o!c(T`tG-G87iUQ7=5D!U%@x=Jf6Q&#9P88!4oEq
ztD@Jxb&KWJbA_i`KGR!szS%jRfhRZcljh;}A!+@7CW`8$cZw_o$+DepSX2#M+Pf2W
zMsIfOxz>OD*siFb(6H~?`P|^yphx&dOIROEj)jGlP+wV@WyX@DCRYqGnC)JUJil@s
z_H?!8PkFb+=_|vW7%L!<`J7|{m4h_QJ2^5UD&`>MEhM3Sz`O=i3UN8&)4CERvJBzK
zr?thVi{qnZF$z?{d}yKa`P{Y4PoXriF4zVipVorAvouKc$O4Zy7a=yD7%N+HzR%06
zg~~XQd_ubN9g>gm#geT*pTs-VNo2}bziP2d{A*SpPkSV(#LR|%xmojDFlZs;dP?rf
z33=)@7LmNvLRJ)Qj);E8eW`6<VOdA=GVXiV=MW>!*r`66mh<eb*RWBaTm(G+dI@e`
zCp1ZxR9e#W3~qWgE?!prCW=_=rhNJw8oa_g(??f$?WWeN>he7TW8RFiW#kjdof5Ba
zF`btA<20i>Qg2EbE1RYtTJ+IqUWq*JJdlLB8Nq#WZG3YPGI&Uq4whx9gkJjP5&C6!
z_{}8E@|(0(ndV%CZ-@ageXCvmZ!8Zisa%BTnbMIxW2ZEW$#&)SlcyV=*m&h~SmfZ9
zJSt68X{<y>R$kI``KF991G|>iVmhAhM_)$$CzgC1Wvrq%V+MUR%gsB{;tp{Li$46v
z2}QW!RnAGx<8%%Rj5=4%20wZ(I}2t|t@E49L7f#n$5=U?^L9y#aMK`0uWyT;*)Ix0
z#c;&UH(0u!wtiw!45c!~<sGUn8|ce;-dVe8QOL)kan)YAEJ_ZSI`^vMCWKb=_F*Sl
zFm~N!^J#6K*^wT$@{z(lU5!LePi?2_x8I~`W4_Nt+zmB@p~OSmauM4{lDpb4lov)z
zsP4CW=JMlYX6s~}7<QOgx(JPb%WSrY|K3C5g^)0oGS}MKQNt^ux8R_+80PO&!Qjvh
z5BbK&L7-p=CzeRCV2q_3tmsJaB-SyAWh{JCWgA!WqsLDYhTM~MCas+DoHObi-mI#4
z<A3*ils`>D*X<5hdsTItLy6BXtun6kB5bSaaBDH$L-f<EhWPsiF|BDV=;HbKSaf*O
z+3Ru^$$A9})T3QQRUS|hNS}11sc<~!NV!LGn<55#-zQs-`x!SCJ6$|Qj6#^w4dSu`
zQt!wOr5j1UO2AtbB`HU0aHYC1M7cGoROx-u;^LE(1fN*?_+-cxbpHNL79kfJAgu;&
zlnzaXAs(hfyI>PcUo~RjBe=O%`aPXV2;ZJ@l9oACYq}A9j--{S6`M#$K73Y`Q71Gs
z8oc^TD!j%7;{r7&zL#|U>7451wix7wR$F5V#Pg+G(hYjfeOnfd(w3pK2s^hwG@B8Q
zQ7~6vT!WkL`vfQuQ;|f4ix1X2b}#X!S((jv?d|HN3H|LWMK+5jxXTi+!i}$&fB*g+
zO;=V=@`x(73wDwCRpa65rOONpCBa>LyGkb0D1`5f6wH4LCE0<C{Emy;<Z8^B1ux=k
zgtXuaK$-*+M2uLprQU?N$MX@!@gZm|*61vXv?R<vYjI6Qy>PrFS@F&VLA1Aj^!V}P
z+S=ORlatDtns?&k;{^o;lT?cbEcq?@X+e0^*Dre8Z}j{b9I&4{;(dGd<C#oCs-bT#
z^*j9QC2#8vcIz5_TwL5RNPBzx;`;XX*M^4logF7XKR+5eIs#1mj%8-+;>SL@B6{23
z3=H9O_zsxD8nhe&{gdp(-JIVG-~S8eQ+*+b9y|6WIR@<_gK4OhKB`n`lG{RsAewpD
zc56gP?;b@#Nl8pv8rAglw5@{!gR3QfW<JTw;!$>|#)R??hZ&~V!SUricJ>ujc9kPX
za36MjJ7mR(ryfr_{p+ZslM}onT@w>#DJiML<Kyb!n`;*JZ*AlO-;N0*<P!Kec72Vz
zP_1*h2*-U%=M;o*RtWL<Z_`RYhP~xhFVtLztN$+??7D&wfoD}2<wcq`A*jdDH<oew
z8r#vgVJMB&gHtHUsrMm!GX8+&hvz9Ad~g5U<50=y{26oSt-XC=TU*ZYu@`@)3a@yl
zOyA3LhZ*U7Q|7tQXWo@n;tO2_d;3ff3A<qO5UdZS)8yFKh0ORK-rl3*<BB#moJK}Q
z9v&W*-n+));^OO@n*<vCGGg><q6|D(VsKBq5aM!VIZD#`@m@e(4M`k1%_-yfYdw~K
zh}<oQ5!}^&Kocta>6bVLg5H8kHxVbz$|}UNM2mp&3kUu)ZHPJF^vq1A;9H72oT?13
z!|UqmM*jSv7&E?RyczM#kohpvaUo;OD183yM^oQ-NMC7JUunlBObL&)&T~?aoy-nd
zIsL}L)HGyxc$i#hvOVx3Oq@F%io;G785IQy+$m7+FO9_Yg`w*GavLT*=R7+rLK(*^
zStOn@GKr^|g^E36bn|#kQ&j!3Q(=+9V(@;Nm62&&rdizccoD_U-FKkCJ;a9&H%7yk
z%W8(r|9p9_k}H{MD=5D9J4N|kn3X;|MgGX`*W0d*@`q}h&X?bki7f6rTtlvZXHFmf
zBE=$zz-;l_HiwrRZOyl3>6!!zi5{Fu0sxRJ;I*FCz7jbYe~n9T2oD@awc3Q(_&TOX
zjF@5ATK-q&`}jDQC0K>>p?#aKp?ubF>u@FDG$~$2B775jKG5ngI__j;mkE6v9HGgv
z63t9eS2oRJ)%%$$2?pU2Jxk=?f0Y&rm;({lTDmQ`-Fiyf>+W#RaJsVNJ^QM_i{mQ)
z6DO~o*@1GS7N4VW-Ki=IysN9Lr^Gs3seJ(?1?u1Qesmdo8lC9FP`Nm{pQ~BSnc0i)
z#pz}-pM-SoO?y$)sXdUhs{3`;xteD6SfG!)n){I^Pr5~6tg8i4iPn1swx{my0;shi
zi-|J*!3@6e8kls}Knf=wJKp8Wuid1&7*$8IMVZ_ZI|(~SIYe-qaMOX(ofS7RudT_h
z1V)L;dPjPL1}7`z#1G3FeYYlt2syJd4?>H<d}Q;56wF||dB>u&B1E^;EL7yX$?1&L
zf5l90yrY{qA|Gn-ZEMY|iEm8<T0L*8uLJcdtX<ys;k^kS&K4WY6o}dwEp|VivJtT8
zBch1;nE{eb%CA*?EAEy3GaTGPWTi%lhb*zMq~zevTuV5e!%V%S!&G%dc{vZ*(<Z~6
zkmh<i8lsl+nO)y`akyvREgg-_55JYH;MjRGcW0M86S#OQQ2z3T6t`o49fu+?b@kJZ
zsG!wrfM;HO{XjeC0wM%bYip_|QzlrNkcxXT+kJQmBpDl1BAyakG7AGqB2C}x8S>SB
zjml{pI{T>e(^&U0o)4!7&?hyn%g^c^rn;Lwk&`txYL&JSy#bxNLTL~Z`Oa`Hoz1O^
zp}ThCC1`YuW(eXn<vUn&z~a5DXQn?f-j@-}4Ygyi|5DG4i_YdV6r#puNp5W@3j*R2
zPG5hGAEw9c)%n}+2IWT+82F;yM@UF-8}t0D##;sAwZzl;YAeDl5ohbG^CJ+J_^z&}
z)l*v-p-)8_jwNCAE=Jz&b+{MzE1A}rv@2#9{0O1&yJV+bvfCz~u!}I@Z_+$-_|`C?
z)pXBD<CQs5zGyrgmcDRE<JRx~tQf1;>ca;(fnBGZD5|Hc@^o5OR+T6?NPYVg@%1`p
zut<bai&vG`_8ViLC7j&gYhM6L{nj;jWmaS)Q7_Ap$BpF|(Fk7|Jf?!`C${=e|EnV;
z0j|oBrw5HKRyl8<p^g5;-88GZk1xKuY?CUok^k8c1mLrBiGtaTd{*RsQyUFZBk^08
zdDUcw`{wFT96XQ>ApOQnnhX<ibD1V8Oqj<^K|Zi&4Sj$iNWUR~?C>@^!GU`Q<6egS
zTHl#&X<6#IO+a#&Y>da}R_Yly0&BO58|gPw&lzVt$(Azkdu;4y1hsMh{dB`_9BM<d
zLH*}nH9#Oh`1o{<%{ohH=}wgd>Ao8B;B9Dwk{T^~Rc74w$tKq~lG-F7IW#nse3blq
zX(AP%w9V$7#88RQZLRvfJ&A8rf|O_}MLW=eMCu}4O>y@O(yZEgs>_;k5xw^I-vH~%
z%VU9}Tz=*3cL$LdER90INH;6AxbDt=JhJDFz$kXIRc~p}H3=ssA*r^3G3ito4{eT@
z`tCF#);BiHOifKa7!jfgH5GJ}m(;0>Bo}t)X}yyY#7u%LdrHKoZABQ3l&49KGa%tp
zOSaVkBTY=`%%Ap&R69)Z^R%Cuf6B@V-^PyT>oWK4HGkcsxmjCt&L~6`%O@!Kv$OLS
z78cglbX}~;)uwKlVbg~?d-$K38HvB|?vO-DL0r)rywKV3&7nZl@DAnr);%=S@E=Tz
zzOa($UZ~+lHaN`W8cM`VFp!LbAfq#?-hmdH_SoXr$A?`yVO4q#3dBWO)khGTJ7cCJ
zwv-{)*Fiu*pqF;EIgY-gcB#(D__&a{xX%O1Zd5A<#FYuN{kCM6y=Etb@fEL*tSM@m
z>h|kJ^Tj*$jyI8f92-wO%;E~ZevKgHX$&bXZ+^cDO&^_@KnFTgg+njyY0kc&l)JVP
z)hT0LMMWaGVZHOhuso^MPb^Z&PUw>kNw$&iBv~7KlDqz$@R0PYYbZH)KI=aYZ5|72
z71m|on&}B1qd`d&sN0%v4>hQ8^pxq>G@fskHf??{%@A|vASNaj6cWO~j$M5Fu5$xY
zU1mBn0OYH1)>m*x09rcbh6CukRRmwzH@CEeXB~{^vmiu=u{s?)?YImk=(!4SXNTip
z7kp9`!)Y9#L4~S67+d$QbP*i{Dx&Fl+P-P08JPi~>eZ`Pg2KWY+*G7Z9fjGs=$#`^
zgkY@i>=c1pp|@=|@MKQ%Y$XMVh>nQ?Io4mR2Re(pp*LTu^NO{J@yA>hz-vW?5RWJ|
ztM=idRObzGgq*vbH><6a)7qbxPgc*76^LpmPvJk!mBvCz;|z5ZdnFOi_f(H8MqwR^
zECe!dC`$xYlXn$&vD1<$<$tVh07-|gO2bg?ShFvz;=@L0JEKGLk4b~DPOBYfVy{|m
z(yV+z9SL+C8Fci{%pVIEy~;u>&@?L`!fEN~G#;=_)-!G?@QI2ldwYu@;T4DkiJiqz
zJSO}w%$fRq$%?J!9koyxhJpS07x52hWX*oK6cW7cFT8gjaEw#j?K4lkmoEv?EWaCc
z(xP8&c^g347mX6rRsZ-2Dhrj$I}ym{t%Z&}0Al+od6(~)Z%!etU>9?jMf)mx7$Ew1
z0l1CM9#o5X*i;pdsZMS$R>&8qS30k~yM4z8=0D%#zi?q(q?V@&p1Y@~X9eZkH*KJa
z%|aLdL50KBD1CTXb)wt|y4|oaY>_C{x46h6#ONWgN6vNO!^J{|?j!{sv-k+?dCt_s
z_6e`GxO!L9O@@`E56A)h%K7_CfM7sZX_%OFpaY;;0l^Nl75sI5burcC&M9-+McTNW
zATinCY_cb{>gG#!O2{=z5~gB$NE!YFBd_%O7U-ECRkVJ{a%zrIjHMa@HBg|=k`8@a
z8`4^?`su}2-oD=>BVAV~3$Pyz^S{y_8%TC)$%i>D24kIQWCnkIpoPv1Rhw&}-5bny
zn;EZ{LJWIr!#VcSX9H?(Lqc#~vu^A-KA;BXvQvAk<5)9QC=wwXo15ly&7PM%GM8AS
zGM6suNZpIk1<c?#GY_wt7J@M@ee3ktQwCg6Ln5d%Sa2?QGIElr4J1;+biqGa9)zu+
z0OTmYqod<so=o1O^2RWrjxs@hE&DR1S>2a}UA$W4<3gFmy;1%5eDL%5U-&yOL-XFG
zjjvZQyYvu?WCUMAf>!`D7dJJDO`aGd3Hy=S+*A)K3BHWJwI2F}j<YIPs`=0-AxZ+A
z;OUcUJElDv)2VSake@|-xNy%l&|RRO+#Eb<-3{DDOZN75>Bz{)v2M4qr%$ExAl2Mp
z--oTzjUtiFl2?bL*U=vz>`^T)i1p8BqmhEGIFu&!VZGz^`j*V~vdmh=&3Q$pfDH*q
z6Cxra+m(8Tl>B@>ZhjEKVToNyG~0s|dB&5DisQp=#MHxas4JC0i6-H;x`;ZoqjGll
zCK}rBLaRYjrCt0HSR3KXqTta%1EkmN@ep*Y04B}!sd5oQw_bHh**!+Hj#zx)-Nx^J
z#DknNXV0Z?&S0Z|{>Xt^T6N2OvCZLjC<#Niyvq0VDorua6a;=O#+3n5$p3lu#i9<q
zkI0_C)-#F>1E;2lg|nOm)^zCW2b5zER~rWqC=f@UmMwmrA8&erHoTN+ra6@md@uWN
zL)7>dgD<-s&kj~EXF9IvL5B`#x-Nl%x{v04FIz66%65e9!@#|c%he}W5&Qz3&#*2j
z8VO9#4>ug9Yhw&~itBJ!-t9>y78P*_J4|*eQX47MxzjCPb&`(Bp+oCN=O?o#bu)c3
zWs_&s6@~+vxJ}$2*<{Je?=1NTT76MItEsw|Ph@N{sL%4+PTF>He%I5wb<<dY;H?rn
z`(V{r4~4<0I4_Co68_;v(WSkSfc*qFPwSps1eTlJnNUX_hL({f2Jy8`EY;)B6%}fX
znyNm^Nzl~{0UOor^x|(;=#{PrdZyiU%{I9Yth#;gT6qUI4Gav-_TOo$GjXc4lbHd+
ztV&NMmg3j#RZ)^;<u6{ggtl4o|F$L&$8!<{E?R+2G@(!@`A#27(!7f1N!#I{Nya;i
zQ8JgTU}H`(lJAg{+)H_G9I&q_>^QA;p@j3lj~r${_bN-c3MkJH17GAY#mts&X-%*K
zObFr)lC89YQnerba2}bQ#Me6hK`mws3Toen0fsu=vXPNEPaHq{_xv70Y`6C+B9`|>
zSF42&J}g5j_UY;A&p@hq?vZ2P5?KfkNR&Q*kf{>sB3eem1<6MEetdDCj*br4DHTCS
zrL*1(fuFqZqc_Fo{bDzqrl;*>9;(}MuFU``C@2WkDEUG@=QfxMwF2%2xQY^!@4GDL
z_Op=Ps?d(I?QD>Vc+3GwK=Pt>zeWk=S%CDZdvf9S$22R@c%qZl<YF3r`j@&xbfUot
zC+Hj19W-o?{=Ntyf8-%^JZbUMoR5lIPZofsyu7?<Lb#Q_-D5GXa^zo865wAnmP6SS
zCA&d9&CV|_NWbkXVPm%&@<0IQsd1#3P1nAyZD;_v0?Lxcf|9Zyxh6`Ii%1@@1RLWp
z*PQw!Xg~H6*>R2ZkvaH)iTQ8qG%+zlQXlF586WQjTc62mhFPT-2QW(#rm4=NL`ify
z+6;oYvA({jBy&zKGUst;Zf?%vdN4X<@xx0Vvy+`USu2yD=uiM^N`0tVgtl7%gCs0t
z`*VCx`PJo<8-;B4$yMUdyqBXoIyE&=m?-`0mB`%Ba8hZS0B68yps)O~{fz+s-zL_Q
z;sa17lr2!#Z*T#~i{^T;s?ph5m46fh{H{2JZ?oB>s1;F?gk`eWUO?>DE-pz&NI0H|
z?G-I%p+E>0W?1kwBG|ewj#l#K5(ReI5N@^H7k_z>-tp6w;mw;jpf{+kug7I4FobJ!
zrjlQ>aszgH@m2Y&#^K>%rS%Y<Q_Hq+`$1|&r9HfS@uFBIv)jAl!pLiTy0<T${w^81
z()wxdkQuu6-n|@JxFk-LI;H{Y@|K5}*Fh&1X(i}?fffbEUTs|+IYGpO)E8Y&c;CBz
zA{MeejnYB;3M{F!fZXkw22|T_V4}<oN(PZ>lY5)$VH{+6)?@TlD?(~_3B-J45~B+u
zbL}d9SU}3>I4y{7PW=4+F+gV`c{HXedC<@P5xQ!c;bUN6P?=-~+z;^*@y$$ng2wWf
zf^@`GO<Vs_*lf_ncEH9<6Z7%!+LW=1w*UhJ<+?1fa7L1D<VEW}@5@Oq?4U0D&t9=n
zk*x%{CVepls}4AG6Ly{-1f9H#QM!3m|M?NJ>E`-ksv}qiw3)VoFDxwBOSHI7nK~9V
ziM`e$F~P17M1V*FR0U}|)gCAzbG@H<)=k~s2@5(FI6prR%gX|?(&X%yZsSH#7BGJP
z&MaIXgN34{)36wV0o||(!42hgb*X{HLGX+K$UW3birT?%dD0O$T#1{ir`{2uF3^iK
z7A_1*W?i&Y1fmJ@_b6CXUx0+~N1(RmTGBc50zS+*cjQzAtUi+5|9OXqgalVc!uP;Z
z`lv*0f4m|nl~%y|)?N?ubrOJ|hySBuI9LEKl8DUvu%~dm`n&BMECEX5u|a*(kTnLl
zFc8O){=04IITGIHajgH0T3biQ`wS$Y@_NnrCeLJrCNs$iChK3ZCFfUFkwaX;0?k48
z1?>7SEIZ#EJD&lipsJ*#M8+m0D2QhGy&>nIl)H~~ZlN=82k4;1T@l&J;J!KUg&~85
z^NB^$sHmtWM6Q41cKwXc^%+8m(r3vf15gT(IeqW%K5AU~@Dg~3Uw}Z+Q?c;z5!IXg
zUCoVyTm0PSP))K*k6ToXn26~2%nW@Q-yVoz6}S)B8>LIeA3uH=x(^A>2B3zxfvEhM
z*LJn-j3}15D!JK^>h!ep{D%_6SD1|x9k}ht@|I_pON38aJJA|T>SVSD=r^THeSkju
ze|rr*$F<P~p_0KH3Z4xhYuI_IR~u#rNy_<;E?}oA-;?8}BDC893<|Qq2RJ0YdtG??
zO>Xyrd)f8MKLm)z5BFc5u5ii>9KlW0K$j|v)wmQ%9|n`h^XDJ2{n*fPwOyuH6?SvI
zd$WetB6=M`QovaPB`z)@(S#b7K<8$rxF=V-vEM^<g3r3cz5?)$jKl?I;KS4xAoqcN
z2R?bKo<Hno2il%s1vc`(`=X#<vmgSLhqfn2M|%Wdl0CmtANihd6mM*A6I1E$*EKY_
z11>(ON(_wq`Z7IX{Ue$x*bYA3f0?!y?;H@0PB+j4Ateug%L5$1<mKh<yrU-B1-vOJ
zyyM!Fr)h-(tuxYij9viu3f)$Th+tuxb*+=nzj0^}!UTPh0;pN1LEt=tMv9h(#uWG<
zlGmrp=<EfZ>mwS^zxKAarc2!s0ej)hD@MD1)Zj8vmnZU3er|m<{Fy||my3A52#JLO
zG<h+4&vt#+E(HnL?P-LIhd0&e$`0I>H2^4lAi2#q{(Lp`7*XUlZp~O*vj%3xhQ}u#
zN!2PnqJ-(r{NtpQQSz%{TYV~!Y+&y|bx{(aTNq$Bhlhs)>vnHY09NUDWS0wM!XmN`
zOfZ1ZH4MHwLJQ61R@5*%Q(|eAA{yCASSeyE0hWNp|N10ke|mk=0YfJcuvI}(5fKwt
z0Y4z<ASG`Pz$YM){{PFv=v)~{38x!m4l_0gKZ9R>?^2DY8<go@av{S7dk2&(07DS)
zKX5n%otvX-GD9Hd^zSgQDKWoo2~JK-^r795c#--3I=Ha}8_k~mB|8EBvl~;;MxzVe
zIS^@}-{8dPJ?BY%GUv@K;l1;vq5@2uHg<Q5J38_JI=CWoygv+B>ebqkzf9vc18&Y>
zjzqc^mm(mx)E6VOvkcsB>oVsBp*IX;Zxfw41^a+&q{IOD7rGkO-2pB_OfS6GP51=;
zMoXH5v&-nm{I?utia>V{M1wm30+*{JI9fb(FICc+{NqE<_r2!aM$I3=x{}laH#O1y
zFZ%5;Ss4n35eyI@WWgOK+x$gHrB7KvQ-yXyK^-K5R>T$fDu2et1V7^n^kLWOHW`vk
z@ctf}SvCkC50Y-i1p|UE`zb)lpqE3Np@44D;O2>(;N#^b+wyu{T2+-?qFH>9*3cT^
z4Rkv?Pb~R;`{`~xaj=&Gxpm=(0tJ*A#T=A@mH_CFR~ILEkM4m4uAbs<@!nlaR$^iW
zu4C^qkQe%mF0Af|;rl5ANy_)4A<JOs1xV5rJ<XE*OCu#{?Geqw@O#5fMz{}9NZ2qU
zc&ph-ed|IA@IaQZBN{}}q8fBhcsb(>I``KV3djHj3}%a^pj$)x@}@`>OeRl-qmUXM
zJcSh+{HUJ6PeQS30e|oOE0{Ax9s+dTi(PokmKVp{Kx}L9q=zHVz*D0@E@O+>Ack|G
z=tl)@1TaqXfBc9CDK=&Xnr*$byV$sON{%kP&DR=?eT_%yH)rc~+>``h2=?FDEjU;P
zaC&GchOytq*L+<6{a)&gjg12J2fSePWc}dx^z^TTRcO<}uSfb-<~ZPzll6f=Z<q@T
z-(t_yzjby-^wB%7Fz))`<>ci2t9mrSpRX5g%`_x}A(ZJ==+crIKu)xKz2qXgo+MyB
zXu31oq>?9{gWh(~lYMycHM-aQ=dvq6G+>YEaf6l)tq+baL5$5~+eR2Y%MuI{a~kVw
z;^DUo_`m$D-zH~--OQpSK^=j?mbr@y@477zd1(6;$dejRqz+ilePD)y|7A;u#?w^)
zMeT;G2mkL`0B`OL32z>>h=8U)L4fSOmoi|hn(|K~xqck5Y`i>x>I0)J@A|OV7mfLs
z`w%qd8`L`h$1em}ykP9QBf1L+8z@m=HpD=dRXz#EpjR|i^Up+I8_bk{`SNAuEn`V0
zQfcA7ID_z1=qlD`d6iv-UwIdp-q<n6^DR%9rdgo}qQDr)8sZleR47n41=0|`3JXX=
zKJHAFg%TOvsX-4M3=n|P#Q*ls7t^Z_C{IgVeSN(}eV@U)o{;f93Nzs6T^>8%2=DsA
zTWA0ZgVwt2P2KzPQr#Nh52G!;-CfdWSf``p$2)USO*JW@&v#Ta`6Gb-<pmlIxIcXW
zvc!;OHDOmW?8(ZNdh(J?oD?ypGhSqraX}1$ad)ZWc50VB|FUn)*5x2gO_)pXoD;^J
z_IBJ<-89_@&C~;iA|fT#<fa<1u6fD}J_0C0C+F5|QwrLrc1Ng!S;kXcU3m0eH3+aO
zz+4kdJ6%Q;sa*%|r|<QNv?rtUVZibu!@uE0HV6)Aq58FTje3D(Kraha0D4)V+ulc-
z+<XEK4_F$QO8ewO03@Ix-qRBp;?t(umq6zk7|;Wc3+;8H2QFaPB4G8464bBKm`R=c
z$`$!Qok4!0cCc82z?yJeu^sVR!|r-v{qX^_5uDiRJpy$`E~F=&X9LTXLfcYLiI0Ch
zssK89#}v^qGFl&QE=Bnn5H*-v{OyfxTK^o2_5#o*GT<cTdv1dlTI)U@W59O=EyE+7
zGE=Y!py&Pos&<D-{lk4QvCsn`4&(q@=n-o?pbQ(|1sw_$Sj@L0<bE#in|8Iv+~LH9
z0Lv=_Ote4+{?}UsbV`y0KY*;WGRpqex7Ur49CNKC5j5sxLb{d}dL@(HSU{_RgBnso
zRp%aY35)Gz0|kPvN)R$o7-5h${}b>eceqvfjX4j2s!LMgc<bo68b-)t3Yfysr3(k$
zNk83u@J})n<5~d1BhCh*8~qMg#9{_cR+ZD7c+eHn1f60NKMWxAZT{hA+p_A+x4<^+
z|BZcunTi9sx|I*;0zHRy{i}c;Z-Qk24F{Hi=Us6%4K-Puy^*Ag#}&%pTMIcys(b1=
z4hiW<c~!D3sUN}h>N1$IZ{exbp(ogO71MOwSwq#%3+?1Ujp8vfGJ+*N!6yYk5JpaI
z@%$bI@E2Lg&|?!oc}t+OzW~?NLp*Ty;`O?M#{?ihU^{`fvvu3i8MrS%8uUjJ)B6C}
z^9AoI;26v_IFqh5A>5i46-(yI3knO_Pbbx?E=ho#0HXP8Q&T3;9iKjZ0^*e@0cbQJ
zV8C|U3ewF6Rxn=kos+Z$$MGye=*0zeLU{e6LZ(e3M+zDH#4pbtOgW?hOe9xQQ&V#R
znk^BZ2s}55-V@U@Ev`(J56buYE&0)&n=fh&C$Yu`MpERBhNNiaz&$1FCnoLLz)8mw
zv1o9bqZ@QRfiuw4(6Hpe1Vlw?#z^Z3KU^SFo?3nKMeEf{ko*9B)Wd~DM8NEZj-LK`
z@n~1GdmWfu@ikMGKAwQgI4&KHnbsY_*OQ40NpGGx2xk#L>w-_1wzaiAC9V&?2|73n
zaAAiti@7`ig(_-2oIN0h$nLp3L;YhKY91oH^*fI}KHogGP}!4_0Gq2*rbj+IGHhMr
zFyESqc1J+PqrH6qE@7;EL7Yd$_+v#z?Tiv$ai_CtHP7%ct_gC3T(=hPo&+af7J99#
zY(IE6t-5vd`7~!FIH-<sFt0|lrPspo!wO7s|AzT0>1Q5?ddsog_JgpVyD<vN_b3z!
zUjqr3!l`Br02eK9KzqdNf9!yUUvMZTQ^k%E1KkfKk4AUY&-7B841zZUx0dk(N}r9d
z>0v?xXr=%U$rL9DGK`LobACanU;+#|l>&?nB*rJeP~@YBI3s0W5TlMk@VKhUUSieS
z3D#4WMVTizRbGME9bN~{U#Mbx^REvQr+cTbz(E`dd${o)R!EPPzE0(iF*j8q5GC%z
zVo?Cx_yN|U%|^i2=pBiUPkHRY<)L#EanfMj2!<Cf44e->`TP5a_tJyz0Nsg#69*0p
z?NaNtz+?pHWJ<K?T;Z3ds7=x$;Q_jwH)l^FdK=?Dg!-q<iK%+=b@+?OAXfPTQ>jRx
z%t1Q`@JYYM8hwhuLoS%_!?YsHKW)1*Xs)m0vRn_uf=JuOt4{mTcESkY>pK#z4ebKF
zPDmgz4mz^}9TzZdL8}6$yuhPjj~wop&&sTWy)7f3D@0cGpihXITg20eQaP?KL?0<>
zp0Z;Ce9aWHr$UQ$FzHsw5{{pkm{=@29N7|dna0CjzPlb&JqC0|wwOD5WD~B6BrSkh
z*PvMf$U|e_)w|$KKt=mG7eGZI*yHp<U?z^E%*RWdFD%4#a@K6cvm9r-By+)m13nHt
zg`dng;CPNKFelK{{*(b;AVxu306azl4bhI8e?$_^5GV8J)*=`u6SVIvgg&7p06i7p
zBJ$b(A#i6v<6tI)Rw-b5?84rl$NksWm^W;?qd;Qf)t?c<UwI!fo5VG`@`jy2&;*33
zVu6?62M`%}GH4<5KW8Gc=76&6df*j|7wt#!9if^vU}@{@Y;(Ld#eO|Yigif>&c6V4
zK#E3tQc_?n(Q;3MAq^N6S(}=&te1gKu8WHD8kQ*JY}Fb;sFcW0WB)3A+Vjq3>chuS
zHgH$4PGD349fhw9q8y&8I5R!X+ZU&|x2#2z@DHciiaBqJtoSprO%GNE*soi?SdV}J
zlY|wPmOg4UL&D&&RX4Dm$GZz^K>7j84mfmxIMKKM=F)*nAIR{WcrADbZVC(p5Aj#P
zD5-b611jyS1~8w{LI<c25s}gPc~#Kqg4{wMV365W^A}}=za`8`$bm2T_3XA<c$(%R
zeAzX)gCQZ)u&V?_;+s7smazo>J?1MZUMU#i`;+Ny*~@}^L69TFD3~(^4DI^-_xwkl
zCf@lv{{9z@3%gObg4$LqXcwKM*5RfJ%W;z*=9qDu;%KWp&9|prf<|^1j!yD)9eozz
zW_5W{Jxc;X7stOd=aa9@F8;fi+W{ty*jcSs+dmkMj(+gZFG1r04)%XP!4~$Ps|ddJ
z6yx7`9Q<g0$Uh?;$npO%m%z{d|Gzq%AJR#Wus~l0;7U))eKqd__N%9m(uT+bd|J)G
z9VXMwwM_a>;YGhi4HMWLCU^M1+cSK9A*F62lo8RzzbNWdHzw?PFeoKgZv;PDzD<bV
zT6l`5<|LqfQ~c^myUeNXcfESuWArn+aeN=cI?@gI&5d$|?=Bwsl`$s@{=sM5fVK)7
zd|sb`(S%jwQCoMLE;e0L6o~u9_oSIU7ayEAi!zc|om0SOa?u&Hb{m|SGVBY?!}@CM
z_QGup+XlPs=|G9!?bYwJmfZf43|^#gT`AuC)&dXhYL>n-e*Cq1-bChNm^PRdZ!;qa
zHz_MU=QF3JD(+D_tWlEeUA(eCuCf1SC~{lJWXcMWs#!^FcsyoICx5g*Xk_X(W_^{#
zi3*<+ySB5JNfzJ8RwR`cX$gf54X-=!;35CAv2kvntG!}jJ6e%walhQIKmU$*{dWu3
zDK~YgF!-2sYi1#~B&B5^Yl;E6dqw(ULuGC}QPZ4tf+x$v94qW`6GhLN+tuvFqo$8i
zdy+2I_nUh>DI9&09_!kDEC2jb*q|)$`Bk}+&1<-^oBGlwtK0X7`9h9r-nh}Fv@2Sz
zc0ZWd@YcD-J#|Sv?mns;tk*pTb2uEGoH4U2{a*?4O(aN2tNThiuMz`Ut)hdMnDG@h
zcABqaSL-)o&qK@xrAO>6DXf~lJDwhL$}@CQJb0c`mD!s{lF&cHrEU~z`cAz0_itbF
z@s{a`{ZI^<3CSLoaU;kHc36D9$Z2<%_T_YWhaZbRW7fi-j|F;M7UKi%yM$4d--RsO
z#uFWr+4G)K|Eb7~&Q{RHys?QRyqBD^ZH<F%qaeFSo)f%6X$f4v<P={_3b6#iFj<lp
zql7DCVg~WqcS=&=w1>D>amG#~{)N%_w%DQ3*@j(=(_lOf)boQ>!EFuI0AAn;`W}ub
z*>{v3;i(39*>CwWfhLGEM|IiPZ(|j^7-NZ-RO*lnIKz0YBJV-TB#j=Ozg)>LQqB2g
z<UOY-oTY8if7@zC7<jLXK;r3GWOucLQ4;#l41-cE@>fi%YO&bKtou}@SvZ}%0wr*s
zZviG+lzYCSki~EHp6+R2hyq(dWKzmG(*riQ>bJz0V#9(Jnj~sXhC1y1&qrrwN}s7S
ztFuSXV=1o075`5$e&JbP%VIDgz+Vv^rTedXo9om3K&>VSeEo0c_y7Fbh-gJr^VdU2
z;&S73jbWeV3{~IyB|BuFVh-iPpi9h(XM9Ez?ucZ{Mxz5QlH$cH7RdHrLuyktH~!d8
z^?n`s{>I(c1P>Jz06z5JZWZ0#hG}r8jtFJMwevbV^^=hxAFC$hX_Tm_-GscCrS-fH
zRTfpylz$V?$_6F`eGit$u-#@auS9lCuhtm#(#|tmi+BxJCS<PUsfKe3`jVEdp~pr6
zn=fykNz%QxQ10}U8kVxbHEBNUu_GF(t3v~iSh9}P4cntU{n8R#emU1lSr}xYA~I?D
zdvS0G9#g;vWv=qRq>$xCqBWEqK1ho|T@ZCub2f|j&)9xiyvD)imz#2};y$`9*3J#>
zey<b*`D{l-OHQW1PNxk<N2ao-^t`y6R+bf-f$g_4exWMxcov_8T4K;>iooXhp1BLG
zp5}Hs#JW8A5+eI7`kkDf_(}gh34R<a1pzt9hH>oJF=2Yre9DR_f8Usq)Z)#v$@&OK
z15W{=v3kc&ExUfdQs%_JO-%bO)A`FwsZ!Z~O}LNz8;ynTJ@6MTQr&UzPp!wL@d~Yu
z)qGMChveI`1E~vM{Z+hMex7=yTGE57Q8a^hH#t~nhFAAnUl8k80MI1Hr+b`Owmm+C
zAI1vr`RqqMe-J%o+;*utVyc~MKkK7cuk=MVY~%BOH2wd;Jw$F+IHOYWi|YHz&q(qL
zlBnS?A65u^JnI^3tyI3w6B{V>9KC+mBrCt(pNAyJ!)`DZOl>ugLE2JD*}m7w-Stj~
zacFR=&0<lE!%rgL#t%9;e=)`_Z|ZE~%jy58s}dr-RaIP6`QK@CYG=ek{(4v$C3IX3
zM$yu7O}xU;=Sm&;lG4c9bMnN2-B4+i`i>GGWHA~mOoYU9S(N1HfWSZZ<-O5JjB+jy
zGnKZFQ^r=;FPkGazA`+HU^u!1_4%@OPa#D-D;6CchOZlZSw-q)3EwgL-*Lf?FLB&m
zj2fJ#r;jANm&kJY4Cfhoz84D1JhUgf+&2kG9p1@p*2m|{dbEeyoqumdJAZwV$DZ?I
z>=nep(P>4=<bpoP?dC=MSx?LDsI6w7ov+nBc=PU3%swZc|L2aoy921*3d@p|k)4l*
zr-ig{r6q-WZ_c2>mceSljN^&EJ6976OZP?%_Yd1y)<utyN*OeDO(&m^@6eM;R(T1u
z9D7W_gmJIJDPP{_hW_0shRex+^7D=0RCB;Fh2-)u8MHBVdsdsfl?-M*Y0<NB`^T{s
zruOAOo69ke1Q9+4{tO`|QY{Hmj2FmnL4z9HX%Bj(>b=%Vvb6fbeg5_%d)ZCC@TPn(
z)qZdh7r}=M<=2a+r4uL-AHb8O`;&^Gi&az>`VuqtTvnYsH3m{kUHM<8ku?x2lzqCI
zQ9Cn_^r<s;P%iE#C5D64lJ4-r#-BO5L$)~L!B6$DHm<9B(}wval%IwxOHU`aL0hCl
zD~O(66EJ-}S3w;5YS9egpZad{jd`ku?DZq9Xt_1jjtYhT-V_E=1@S7~?S6M?&%vf)
z62~_#VjysU8>XwDfeY(7s_p$2&5&*%$Xn#bL}&EHXK`uCcX5N^9-Brw8brsQvY4Jx
zuk@+&q(4k!WPtQ~N`0@A<^A=*(%@YWYaCx=xvuUb+}QpSNHtNPo?hQ#Gtbv*a7;n8
z+~5wjR^;F5dPPyoJ0uY<ZNJ1`n1KRAD<3i3F4_KiaOi#|Dwc@)<;P!Ef2$ts+rCra
zegTeq@#7}GAXDcOBf1;waYAw{DO&0IPm9KKW)85#HO0?HQ$laQ>51KdUz|)+<Y!{1
ziY2=p!DYawsV`7s==C7&#mYesWh7p-BKw!WRlj@(UB~j-(4eTWqMZ86s@Weex|dDR
zO2bm)Io+Sz)+#waYoBk|HV7v_zaR5X4k**#F3Iy-EdIxys6WR!TN_(NE#4u+JI4Ga
z*vZP+O4J){ZZvUxha{E{B}iUn3O?SZ{WE}_GHs9{=1Gw<{pqcfN=kHZC(Vmh{_t>L
z@s^9ps+RB{6<bBcGB1^ENE48MK0beVkBs~^dV;9L#T3sQcz_KsxvMu8;XWTWHJBxp
zqs68wNWno_O?pZTLp@0Q1(w=c8^}p2b0m3#TCA9CJi3eP?yC_anoz_L-Onmo4vnkf
zaGUt4!(&_>LI}(0`RiMgF*zXzE~1gsC-`Q{m4+v6eFJtd=1$*uQD9QFc}DGPbt3(S
zPC)$TKY^Oe-Xz-LGH-ajN*LU|a{I?U^O2r&F$<y*<qX?<gAuXAOWlfyz3T>}A(@~H
z)%HNd6=h^3(z;ZOT+G+)3{&R3xMQ)~9q~c+e%J<XRBL6eqnvA&rDWe2GR?hb-1+Zy
zD(v8E@C}B&aI1m^$JZoTg%wKclH1^<kyI{{&;Q(VI5%kGX%7YrB|7#C6NIyIRe*Wv
z$<2y)7pnFOr05@)s-N2F#>$%VezPu}D3Z2YW`Yz;A{BmjWE!_&qBpZmL~De0hX0A|
z1HVi@;aN|k#Oh^ru5?$120D($R2ZqN?vBOo)hp@q_JuK==4b|T7|O*k-L1)u<e(<h
zZK_~Pl!5zEhLj`!LhI_NLLoWj>-W*w&4R;Psw2Pp`F(gU&7;oOH#VDTt+;2p$3JT?
zmpner+2p}qN{a98Z}Gi8j#SUQk(ij1t`K)yOoIiy+r3#y2cQ#wPc#F^8NbS`Zb+&k
zqxtGeoP6_$A#mFxF@ie6XJ7y5Xn#J^{+ACNh=cSbM>B&7jE3~fBXvV~Z^r(v)MHV9
zaptS-h6C6_Q>>og=VwfbH}l`@^G7e}`a34>f=qIwk!<jhd*Zgb;*!HE)Ln3<*kRPR
zv$Fx?&w!4OkHSpLWx46h%nbxUg2pIx`!L0oU&Zr*BL`MQyX{_L3++hhZq_!OI3rqG
zpZzK0O)=TCUXmV9fz~^F&SNOC<^wwa>1+4wccwqjAm1ZzopqC4O)*@BiyfaFE;lnd
z4iVSZ3I3TLBYT&;d^V`$yz*$_G#N9j$JcpG==Vm@4+9N$Q1{8eJ-JWI^Q94y`h{*;
z&~y9H><@7~5y`j5eJ#(*5?Ju3gd+;fyu7y%BgOE&l?r%`7s~7WyhgM<){5vA@mr_H
zO^xeaNKlrtsHEzgL6dDTye=JeBoR~;VmccuOe%eLQsX<bYeTzL0r}YJG8dPyd|hEE
z(z8r0!GU?CYgSAr8FYU0XZHB*(TTj#10AJ%o?a)_*2}_7Ms<;h2}d(XdD}+@JW2d0
zhhY=ZD7C=#Im1@V`O}U!vKHdKW6vEAZQSX41)XjN1wU*%ujJ}}wu>7*rhA|ynEU11
z#l6oVvMYOo#*Ob%17$EBg!<lYF7c#`7*-`I-6%Qokp#PT6F=ErlnFj2^OiZimopb=
ziHBMtjQBNPe>Bbb`|*r-@*_%g(o3x`w5(_AQ$npiwln_xQ|io_E=BIWOT>VBIK{U(
z#QE*(HKFK)ZsWWwDWjXdo?l&gD9JIK7VOc5|3y>qs$G!JiG112A^((NZdX94TNQSi
z_C{L*^pXGZ{$*sk-qq;Lz?VDAUS1)+C~MT@=o8Z!0{p(i$Vc<tJw`SKb=|5ak1iOj
zpBN7IA;zfCi>kEzH2omGTw6Z7j{%QvW{hN6JHCiNbuq8QpW5W2Yi?@_{>mgc&E=j7
zH$_oX2?jJSn>`zPxA7?JCa^ywpEc*Fr%>5>KF)VW<JXBIeQIG=N9Pn4QCGZj+M6_N
zOJqP{LKlLUNpg)^GOyaMJ~Br6aKGT9<^1?O{`A_s>StQI*WR)}^Q!!=^y!SZzx^_+
zw~T0^Q2nbO*P;?uRf|;$%THq@|H5A}=dt8s-8%dx+eWh95!L@4(FBrdYGG7KlB)1)
zli4h5QyQP;pXbe9JH4)RzvRr9!J&w|BV8rFhi~%s0giW^WE%K*s&RsG<O>e`s6OT~
z>X>(L9h@vj>vc(u=u;W%TIcB%;mzP~3eE}qo7INS33%f(GRjq2MPEzyE8$T4>@lz3
zF1Ji|AwdoXHQoxA0rgh<2g7ZGUs9s#jAI;U4iAZ4_I**LXKhb*XG@RpNVG-2`+EF%
zq~VMaT1Aw0G(GX=Xzv*!&*3rjdlrJ&Y0vPHLFTwxmO<KA!duCOWU<dk?7uq8Tj`dS
z=LWU!#VkmyoXG5NiL;Tf_OQ7nm6E~=>q?gPcsLxhLc177VW0XXf-EleIm->2ihRwM
z>&Z30D6Dvnnf8V7>3JSJTKNPQe)VI_^hg^zX6!KjD$Csib`)Jf7W<XIr<Xz}r*_Px
zN-t-3?KJ}rzajSogmBA`GB(vnodIIz)2JnaI)49^Usd0KHkM(R5C=*F=(TZ+IQH}s
zvLLf_az2w9cv5}=ncDz(?`fnQ>3~};_ILk5L*VuDa)RlBBq_AoC*bH!b+|8`(hoBh
zwy5GQ-$lQ%Aq~0QD9ym~Uh|UjO~_xD{kqj?22%L<yyS`=rIDuzqAZp2tXnRPPR6E>
zC`z1&oGRJOw#1u6g>$JTbuFgefv;QwXU_=Dc<K7M-3)s$zHwXojQsTtxPab>G?Fn(
z`9Vq|vFt@0my@;aZI$gwvB<^FUGf`bUTSg-<y|jbG%anugb?Fhy8ofR-k117;VP8d
zpFAO`&M2-Y(@=9Y`MdrxFi}K~3K^?SvUV)SN60Wq<&^kdAfq1*$u#Z1sh;X;rj?VH
zG`yD*Z9x>ULF#nRkEX{<Ed~xpDSzdf!D&0Y^u1i&?I47fCz^?sBM<rrsn%6^+!%U7
z!LFs7RKJ_)IDKfXnj`oZX4iuO=3MEF*XvJiHu4GMW1<<HM1PN_TbbM6$-60BSnAnr
za+*cb$grI57?JL+kJ{Th4cby;C9`{I+h<2xZCaD6;5Am?*C}BR-ZcU==~?r$PrXWk
zE4h=HR$8VbQ;*S6^3!d`vlOYnA&VjlS6N*y$;}UHLw#xwIxae7j$1tIj$=Ds?RDQr
z@H?6X-kse@s^7aS#)E<JTP6b`G9SR#5WLU9Q6JxCq|vlR_8k{9@2sU_|7Q*I?b|=S
zP0<X!o-vLiGkI1A@2IYq+=Kfi%Jr$<Vbo!KMfmz?t?iflT`NilLgCSn&p?kbL=f_m
zuW=yl84Em)*48BEtBD{#@_K2dVC+<-adJ1|8zs$U%xbo>dpFw**1-d%S%-GNk51>s
z9&#`C7!ZEhI`@e&%D->9Unj=8gM5U_%Tbi`{38KnYAt@;-7>=#-OnevVL;V}6>jL=
zt8u-&EB(rXBk(s66_QS8)1+q&!2Q|KGNeYZx&4geTRD)gZ4lhAFgd+bZq)8E=+b#4
zS+@Dt64ud}N*Z_!^*A{ioy`x?Sh1a*(O&F0Ui+5r?_xzA<nQ(Y-IPJMWxhD)2R+M2
z5XxU(m;{EoNe(E%JyXh3*anaCh0IaTgjs41vM!KgrgMVT^D{@TrWG5`O5)j?raRq7
z?luy42*EmTPaLi^*}r@;2E!%Fb`%0$j96cFlVp0^l3}}VrAueYJ1qGg#rA-U;DeKe
zQ}Q$`^4n3hEnlJ;_+Y3?$E+FYn|*jUhN$UENXN@AO_=kdMX=W+`i8a_u|-tT40VX}
z*x9p9Q=@S#l!n>77++V;$q!3d`;}gibnfME*))TjL(&I|6qlE&4o2gN@%z7HNb%Xm
z9}90<mqi>+Q_&yQ_bYsvrgpU8;tcom-SL^6?p0pNuvg|h3vLLAEOPC}fTIArHwA>k
zg3jmzE`~3Fs0D!7EJ~eR_~8-JesS<=(7f+ZR(M5xQ0Yle*aee+q|AA@Ox1R(l%Pn_
zxDOj7#PM5?U;cq1djVOp_?wlW2Mm4cKUk4Jf+T5UDW6?F`(7Br;&Z38C8TpKn{0Ev
zNq$ZximaqK7G)rfGP73A^hp}cDcI7Yx_1f7lL1E~3GftF)}K6=Oe{&64&T;l<LkzN
zmCnkXPxYwnM+FDbw}>}`DjFL~GEZ)9tS_(e>+CS2A%W}4lD&V2^r8-r_N4gN%7|T`
zdN26)@1D)3cMPB0AkyE-n|*lxn1F*#sX~tn)wOVL@>GvkXT+K4S{mE<f6oFq%pMak
zU(^^(jW*>?6yXYGM=LV|9S6zWp^0XARE~5V9xS=_@fV53TvAP3FI3{h1sQyq;UoA=
zl4|4=7MoifW*7yWrvn=ZSO)L7C9KqKQs?VG9W^#4)IWye?F9eH;KFh$g_TKc>|E$&
zO)(Z(xs#D-8br6E*PWGISCoYfqFSP5Jt9`aMV*8vhBxPavs|m{pcx+X&ysq@j<1j~
z=bTbt$Rog;!EQpsgz^VmP2bS|#>idPk%JAAV5Pq`*_P-d$iG&5-$KR8smigmN&)S>
z#VOu>-(S@tD$-i{mY(_aaAB!&XL7-UXvTZHxBjtEMl^$Fi9)sv_`Ck@`oYuPom?e(
z&1doV+0Yc8alrd*qkH3qpI&fqrIOZSeQU=8Qm!4>z|x3#3i=tk+ofFz3I3mrb*@O_
zT(4zZH>y9>!dV$e_Mx1)KX}wduf>#GiWrV6I%6&wlTuKT<){u|X8F$9sKQ2{qXvD#
zKf-;d)|+=&+m+y{TnxyI!Ti`Y&2(h5IE3Nm<k=NT_HE}izZgY~(Cj9-FPob~Ij1B?
zosfH1cavzjfS0yCcYS+xtfRz<`NI(MWQ%WlT&_LZ=@iwDt@}<c^gV%z(yv6zUaI`t
z0K7CorDKaWBZ-)wLBs0C%U{%Vy(SCXVOE`TLVZi%j~;*o+rOH*jtele9o&t1CqH{o
z$9zD#aQUPxNp)Yz_%?BHlx4F|wi3Tsr-o?^UBEl)5be%=gN<RD#J?`sr$$02OZ0=J
z`xU`Qw>4r;MPn&-oPvsn+tSwhYU#F=J$I%;7=th)>!z1CxF{Zt%y>%iYR9vJR+t0=
zrJtzbMnzqkD=Eagerm3+i)K8h#=(w)VS!e>PnMsY{^X)Iyo=9err7Ckfzr9}W$LoX
z_+m4yVo=pG6vE2#Ag42p3j-qF9~gqP4p?i<y?K(eS^d|Wj*_fZx*Yju{*RB|0;kea
zW)#!vuZf#N(5_5YtB^xKm4wKG9AUWD?ssv)cwGFXI9r#!Rl4_2fqlp2?@Epn24V}-
zqry2$Rpm!(2@u*th1?&B23ZU@s9=%4e8KHK{k|k9DWhOMHgDRnEM%qaHC)!Fudckm
z#H(EQPXEIfC1;te-#L;rRlk}_zFsc0fGq7wCXZ*9qzKc#e6tj?-u7CR?N#C3|1*nm
zv-?AdMV>7FcVU;oIibQze>zM>A%VWQd1xsEL2FU#4~BXFD*d$slprj`Rc*@)UlE(l
zt<0C{rv-T&Uq(0%hB!t{bha->XvuZ4v732fGOW%i*ZUPc+$f8lD>iW81~#7w8F-zF
zyfGM`tFh!r2~;zk#5B|)$FPFn@Zz&&MR($0w3T;i8{X==`!ml^o3eu7Z`fxR0{8=6
z@(4nLem4K;ci769teN)&QU<u^jE*jrTZJ?=OL8BlWR|R)1xe?8^3c`J6ZhOa^vy-(
zy1%0l2L6M(Z*{kvw3=?!x90>AWIYLpuE=_?O(Yk$d$i43=)sHV$LB15wB!7_QHT=@
z-h`5}W8v`*S}2NK2p>~(&MYzryPF_5@kjwLZIX9TFE{GJn0~a`&y#?(_Co@bi=`(U
zTxl971835tlE*|rs+xoNtdZ%{pX}PNmS?_w=h70y-<u7Ri+()TO438LiE$x-KSwqY
zM}s}El%4q}vCxdn<Kr)NE*r#KF5Gt`@sHDLJYwHIQA{UmuC=i`%V+U;OC?8?_;k^~
zb?@Mjq30azVrmr0^q%D%S&!l?&$|of`;J)r6z|9kl>fYw6&p<Wcw6rM8d<Hk=g9oX
z6oPl4N};JRh4<H{8-CC;eyMlI2fs(fEyEWF8H<09-L-sHTAwPvc_1zAa^N&t0`=AR
zEy^%-c{>;U;<3$^cuv5xgI3CJ49(g%1!O(vhN;k6Z;$5O=EDW-@1+M?cWeDVkeU1J
zOC}Q2&s<P)u>Mw;Q6+yT=fFymtv=-6;LMVj92BB)%Q<6FrDt@(l@e(nY-t_c)pxAg
zH?F1m%QySaq^yy&#JTH>@!y2~NyK4Lo3oGMTX9eDdy;O8ot${rL#>KTG3Nv@JL<X+
zug05Hw9P%WH+w0~MF$XWaTWB%ab$9FFATm18E%Ih-;r~mofrNQ8O~|*=mqCyM%(GE
z=Iw44fen(Lq~_)~dy?&KkS{75kf9Ai=IW;|s8<bOGM|<iV=FG+d<sj<2}k%Eo@_I}
z*3?W*4!eD{j6B`DfGt)j<*nrUGqa^9SM{#Cbr#_5!oa&?+7O@h9hqG8&`Ri0&g}Tu
zQ=MiaXy;7Pn9IsXI;Z9<o;NeX)=Dy7UakW;yG4dZJ-X@j>M6;;&&V?L_RfBi5VPBp
z8Tuoc^{+QS4VL|Xl)YtGlx@_oOLvS8-G~y>-5{+30!qrzE#2MSqJngTpwb9~)X++I
zJ9LAfbblAm^Ss}Ff9xN7a~yCC19RV3u4~;Z&gD5bUmc7;D4hix+~}o8_J&1Vs1%Ch
z$XTl9v_~!kVx3Cg&=@uSsn;Z}t`f7*sp>3pKjy;$_Xmxj!20d5y|}$GY4wJ!;qxzM
zXV8H1>n%sGwQt3sx5Qb_Xk9QtBVP|%l4>ZWBrr<<#gZw773)5>SjT$h!hao}ci9Mj
z>1JTnd)+7Vq)M|lZbzFVSt#xL)jxg2;T7m*ba?9>!OTf)H?8qgKsV*c?DXNONd$8Q
zfm}=mV_SqBWL#T^zq}Yrnkl^Ii|)2zj<At_2Yfc&PU4l!v?R4t-dih}y{hE{9K7@U
z;c<mp{|;W&i{waZ<80DUL}*CSw>sv~i;)`R_{>&s^h>32h69RvN}CA+m%C2>)6=&y
zVAcQk8zsj7eftSk@c+EaI30Jl$4zVhNDvQszE8v(G2rq`#^!agP(4mmPlL$*U;uN*
z`$QCn^<&?Ro`+(bN&_o)SRP*Zbaj}?(*_!KE?ZHfd$$IUUrpX0lEt~Z3-4bHnUd~?
zr@T15f8#n+gmJ!pO%AQ*lk42h31Uzud>HmmvzP|;iGSsEH}Lb<WS3J=iiyScr}90c
zWUOQ5FIHLJeV>q5{5`iH@(V-e2bXximLd+LIRuBX^XXwy4N`&PambU0IC<*BLmAW6
zyS3ML<(F6C41q5n9!LzJa{YIMFQ^@FWu>rJKTyg>X+VPMw6tRAAjyFR0(yCn2eChv
zNo39IoVmaNZ3;gJB8d2ZjnMy>fy>k)kl_1Po4Dbf28;ReXl^HYZ{}GO9S^TlI}ZlP
z)_cs~8TGDj4P>VDM%ib+ER}~~eCV^Vh@jBG3zt20%!TN?qR(~69eO5QKj1SDW!Sh}
z^`~DLQiRBIO}!uV<5H6P7+*9J!h@)<z06kP+OT5TlZmSM7S#2Gy}+-fG4EmL)4Kpc
zi;!NFfQVT^)150ql6<4RjfTL#l=o&I4eb!=ow`ajWad!%FA98gpYSnf9twrHWF$>8
zPAPtws{cDXYbha3ElMg>e;)(YYj~tKL5p&W6EF0w9}>5??&QU!kl>1z#fs9y-N#jV
z&2oYb6p+2WJ&>t9GCTXZAHo!~e<=Yxr+2-(OKx{**qz0bjbm8pN&M8@Y(U9~)fw>;
zCClP@fXDcDM0M<2y_&kTsmZS$eNi$Ha8E2$3n%aFw4BJd6B$bo0zDhL<A`S6U$u17
zOd#aWeos?JXUUf?k~RQkWvAO>H4QIKtQ0jUYhz&9KC&fSRuQ&tTH1fLXI08f?hgVk
z{wi}fcaA90mmL`LE_-Evw9DN1z&=M}5tJBkR4_g{`FLDEP6R*z?(!B4Z%ZICF}sFF
zi$Q5Ctmd8ctPz#ASv$KD+wm0zHvX~H7g?x6wclQ886Zw2PtvuN<hZD}h=M35<Q#iJ
zLj!j3$?svzl(x1*m=owDv9H!ID$L(pDVw)vdhIUJxQ<ivYei3J>fE*L9_d>cNUTbf
z_#5RSANN@U0*$14)Id1@x8w#K10aT+O{r7&(w@W=I+$wE(`cE{r<rc6s%v!Fka#c5
z#4^A*l*e3FPST4mP}E7Wm<lTcTEGiVqrkqEX^8nJ%n8!=v^?ti!!Ith6O_-qaK_J_
zs19Jdi%Twy{K{$=h#j<PHTQ-R73yJ3YfD+)i(~slQT0flgSu#ax(T@$Du`KB$h8n&
zQg43UgIl|;_Ms>|cIe*=T&-RHmY7zlGqu?{aegL-#+XMhBu{WcWP^=bzb9?S9q~kl
zjj#E=ND~xbviX74)<bXD)<;{^Pk~{=yGX0#7`fjTIDdKHkdevEC}>7d^o95vF3+a<
zek<~_zQHD9lOJn`EL96!P|b|k=NuS@UFXka<4E5Gy|C!eR%1D!eDZ!Rsw#|_HS>k$
zs_>WD^Y&%;G~6-b(6>YTLec)+I?gjw<r~RU>3sZ@yB8^hG_IRBBt2@Y*NNqW9|AIU
z-qU-B<`$7W;brKLky9t|A$giIN-G<ynl__Blt%)ww0`+6q1wknKIdw}5QwW$H&h)q
z>M)32egTab=X)8+;qN9)17-GUyf`tO)Hof(;TaY$q=}mM?Lkk^n$%p8LxU8d!;dDg
zS<l{{^1uI$rEKdy=(;ti9<IJ(OHuK!Z0(SV&ykc=w|9k%Pfd)f8%5?M)m9LrNerck
z_NNBWv;s+O+G^VOQT;zzJv(XI&lLLPe?&Lr?zGb9L<*(9ps!CjF-{!W4~BdW)x^eS
z>=b%osK+oabzlFUsI55`oT9189(`2jdCB-&LbD#Nm^F4yFKLY4BU<!S6{|tI6hy)=
znvm%Za-zSf92!wY&0q7mHL>kU-G_}6DC4nI1ENlW_yfXq<gk&gGit)BNPb1=>AA!V
zD<PH_(p;`e1f+()5FM;PWiR5dPa`}E@mJblNHjs25=fEP)le`jA$mzG&`y|cDh)t2
zLLxw*g7d{RgVBm`^R+QirKCFcS1dye?b(f=5&QDSp9L)ooIc(k#`;==<-k@HOow*3
z2MgB?t;G3?IjLivswC-qjIcA#E^?)<MDv;%tP+jy=MVlBtVXb$ny3#<Mog!T1Ol`E
zzNFSV1+ch0G3&bIq2O^DXpoIr3DN5W{TpZ3?c7r|U$k4%C^Igs&q3UAOTI8dJFCE+
zNI5s%?D}oq|Bkh)5gu;zMwA@1glNI)FJ*atoK8%Yj3WpB<W6aVQA|sEwcY1^06Nr)
zV*8Tn<W#SK2WiYb(;|%)@H#KpALYPi5i#w#9z?b%v#5+%Mm6;Hn|Bz8dJ8$}wlhR^
zDy3!9b#C68e09F*`f$zeU!k;ye8!zrT<c2~zPf(D-8%P*n1}KV)v1UMomX)?S2?Jj
zWjl9Wmv%h8B=iM_)SMEn8@1CZWILBSI7hQ26bbnf+9XeV^mamKh%k>Rra!VKi0MA?
zh;HXzq{{G+?c^E+FU;-ZR9$LeWn;Gx&zJVd;4t2;D8-L_mDrUKa?u0wMMK*Uj#lx<
zju|}C?9RJ1#@Hat$Hu2Sbr6jR>jnC6w3`m9Am`$l9V{p+BM+Sj&KWNc`I$@ppBt`4
za%PkpVy{kuxSt4Q$|P;D9JX%Y$Cp&C-`Fm(a2em*Y2H3O@m=gXIej&_uVwA@DO@LX
z=NCuue9al}A6MeSslVzl;ztM;3rG~NI`(x&bkAOJE86M9?Oew+hCs*sxS=|+P2R;)
zU0MNtZR~5xGlmSR$mH!@^Z|xcTT^L6>}Du@CcE)6C=%*E%!?RAWGKRkE=mnLbjw4#
z=N!8q^P}aTKe9yWnc1<vB)tEhE)sJ!O11tG)##bPSvgujB!2TPd9$3ZxZNc!tRfIe
zRVc_jE{)xUPX4S?t@o4*8yUIdpGei}J359>E{wB|DqZidhW-0#Va;sVz9>2z$(lJ+
zAK{Nm-V2K{cvT&o*%Q8uu2StLFvI(CY%G*)SvDbb`o?B_#YeP5*C+q9M?cgr-u}I0
z&bO0WF6AlnImcfp`=@Jl&F#NOOf2zmd1u{+t%AQmLyjK%B8Mu)HeIWR@|`cHmhw&T
zqQy3!FM2I%e+hlJ>~8U#_Q;X>sx_4dXMK^N_l<Jj9-I3xBf>khBviuqc|Oq_`X{sk
zBQ1_)YZIinOlBODG9%~v7&pKE&eyo+SSk*X4@}>ajq1;TWB~--#xFQdNMjjqCc!=b
z_-qsGoKL55OB!a4;n}z)V$PC*iFzGZjIg2CuL~O3>-MU<ccrHVW>FTuOfuZyaM_8Z
z4ZeWYwqIg94OqZFbZi9ctLXaSryAzTTyrf_U0%xa%LU-mIP7O-@h~GZc!Zy>&U9JU
zhE*Mn>ld6`!I6DAGoB7=YP!6W6s=fZvAAhFK^Z|tga-i|!S%~NHqws!2E{#3;vRfj
zj%#b3cGXZR4NdsgT2bh^p)w9d+Rd5Mx}iC2ju;}&*%l1IAL$tVQ%7|Myx<TkpW+Fb
z!14ph>j@d@PK7rECF<DCW7)pMXQ>nSTDu4pP^=cSspoDbEx&q^Zs*269+$BKd#<8*
zTt-y6Pv80Qo4orl=QYgr$iTizVjn~3brznaT+hD}O*Za6eU<(Am(0mrm@T2H$`K3Y
zRL?ng?Vi#rDdo%ZH?Sy&cVl+G_R>=|!8cRboYTj!H-YtNmrD_P<DYf|(GE;tZe0pb
zLMLvayNB$irsBPc_*<09SxxH|s7jCiE?zR<r`n_Xi@;<SDej`00u93cD$~*$U090V
zv_N0EH&J)pD0oDAk43SO=)#Wvxcz%7iloU~cXiEEKRbx~@YRP?wfa5bOA(J;cXc}Q
z38%CP8BWB`SE4tYpNPIspZqa(A6`4ke(0WQu<uOTa=UXf-&3*D>fC&$LHmKfW@Sj^
zP$2G`zjPe67+FPVvAyD)Xs@FJA1)(bB9*3W)K_SH@^ywidpx}XPNsSLiEyaM?Sg%y
zPOlXq+?z1J``XKP7*IzAOtbBC;svcQscaV(M6TF_h0$&;@*fJAEwWT8y1wKIiX{-i
z7v7NdORJromRj^#Xi3QUv3nZrnw15iWIUjR(l4}yt2x#l(nsRk+;BNP&*!}*TJ+&(
znSDxMpV|G%?y{A)V9^7%+U0p95~Z;SnL|zu9bSt25+PN9Lj*!sz1LdgCnTkf9j?v7
zSt-Wq@Mf@X&$UAn+kbK!>6rEmG7xycG%s>>vz@DS-+dS!_ROVV!ZD{^^(ocH*AHTw
zuiz!2J6~T!Z|7pdMaf$tV+&(y5Cq{vMAbyI_~d&K!ktuysblWZcyq>opEJzfuGM7;
z#%*L8$B&dKmG)9C2lB{0c<`EPCmdC%<#L4|-hjmw(_D`#JfF{IacgZDV(Zjw^Gs-1
zoWqBfE9@2r_S8iw=1D8#3G$;8)YD(Y8JXdn1ZDYzlgQ;mJg2L65oYe}(H&oq5O_*k
zcU?<>mXsuBz0Q;E>i3lhPc>?&o6@>fximC|=oPJ7gJcIics<(vr@m)9w-GFG=XB?e
z?gYLiWcEw!Ylah~7CeuJUL|wmBb<?8|M~gg1BaW4wsD!oh)e^;#2L{+m5?vaP*ps}
zFak8j$1803ow{H4V>-MBkKyW9-*s2~)L`6~+)gIn{!li$b{u4lKtdL}=&DPqTFL6y
zFHT4ku5TDxHu@|fekfD#BLzb|zbSfl(#)0f@EdFIwmHTZdFuFZXPxwX*x#)%xuKj3
z;QDiO%8@1BtwO=rO%=|mjxCt+v?6gr#&>ZY{T$Qbsa6vaRf5L?TxvV__=MoM?$R_p
z!Lo0QIaV9Cb4NOYP+^u<xQWqJTiRVc>$+K7QUit(NF_^);ehtTb}O2pq(hbGGN5lk
zt)vA1PRY?oQrBas6R@wsGIlB67HORJm_Zq$F8f5A#l6>5W=C~{=b*js>TL2!x;s2u
zk#M|aH0^3oARpQDTr#-tf(K_7JE=lr<s8Zzy11O`4H>bX`w2JfVsMsxv^b$_mac0S
zP$>^(qYKfGMimdF;D-#3F)=EpYQxk50*Ch1w{y>SuWY1Cs5fcZp6q%{5L;;@pA2=D
zHWJCv3Yej|4=4DKPob|ld{L94<HzAjNW;FT;V&GWy+rg=Y%|&9b0`Yg5_5XIYcnMi
zeGg&j>(4#lq5hvEK2jQuWx4%nVE9fE{@0|<u1<dv(~HXwu4mC=yj0Zs-SM{Z`0M6o
zV(~iz3?gj=A<W7{CzDV2Vv6{)BR0h^spx6K_g^Fm9?6NfbRt-0sc>%NcSz^jsG8{8
zj&-H(&9P989uUJ)Zwsg0z2Qr^_LEQ4`VNEdtL~mqaTR`NHu3x5wA9Xr(`h{}!?M)A
z6Uf^s#;TJ?^mv57Z7b)Zq;YgR_i(qNp=~>!&q3;g^|U%)7J6CeBuiX~Ijw*{#o>uB
zIFho09>IBUI1$ojmtVi_F(Fu9ZRft=e`#3yb|bV%ed#)*jK~i^H1-29NtAUk{P2By
z*f#VO$9v@TAivOu4wD-^q&2;5DiXwA1P{v!^nPk`1f66X)RqufbViaGr=O?e{JVgi
z;M>?{dcmNAKNUYMT8oFrT#qMem1<?vJ!<TAcALMH{qZ6_$YI-x^Vy}^YK-K)zZ&@{
zA$IhgP?BC?o`gxbe~T$rTi2vIz0&@1Z`*OP!c>zv<kY?rtF5R`lDs}b3^#NUho2{b
zsaZ)Yic=3eD=cI8_d1^Ya5f@IBy=zP!;t=ENob2npwW95SW)&3>+4c}4Qzuq?!yWh
z8rTZVYkA;klDrhDd-q4Zl<0dmLC9^VYLu?hKB-Qw)w7<qNf|l&TsJMRK!@kq_`JJk
z*&jtEhxdZkT}l3mg6DY~?$txD)QF>+VtS=E2CuRjm4*U&QFgQbSc~2fH8rIqQ_+uU
zF~-Rj&bDk}C-=MJBs9Yq%ZgAK>v(P>GfIAY&2!x3plR!`cxK|Rm-PIWek=g7TIsGM
z`)WuNseiHk9QO2f0PZwpzEanF5}&f&yVSj^WZ6DsdTAIO>jI~k!oPvShID?mcs?$#
z-#38k4PGw$z#PxF;O8i4$~}T|b~u-$Cweg%x_karwpWV10UCvEkKlTp`ev=X{6YRt
z_XTNLp+Uln-Nf8qD~BY(<Z*=>%Lk^ep><rtlLcO&1jOugB}~aQ_VnIjvu84u5&1|u
zw8ZP3WZpK&(rPLv+N9K97l>^ZPq*kS=ilS172E6va@}kGIlj_~>%c+}r`ZMF7kcm?
zp9ff#Y%I1E79VBuQVjSX9DlejuerE>1^@Yc#eLY#c;SPXb%}-NyEV_+Z5h=1&Xu&2
zTk+$kQJl|g*@6=Iy1sH~(?9tVT{)q;qxa2{Gdm}~GjgEpLny-LN5cIs**oj;l#m=Z
z(G0`1qp4uU_&Wa>pEVn%rZ-&+_E_;aALG8&+6`a9Y#ODjt*f1M5wm!L2?bw;c7bkZ
zlvA<b@DK`7J172R$j*o;M*Y#0#pGMnFX+;y?ejI=NP_v)<;U}9mulT~KSf*zGNWW)
z*gBgz?l|@Ncd`6=yzE067{7yh>ACm}mbi=o+n@3%96HEag~E$E1BjYvy=GcY8Rq@4
z8y}ujY;X7p`tQ0DmT&x4nO}vKv0cKf30J=aneBZSQ+3*XbN=U2zM$z1<n)x^A15@5
z879zyYPsF?olu&1N&CXUEUm_=?~|hYa4677t@hI5p%b+9LZNMHrJ*e|O0)vh!o<(3
zi9QRXFP==h>cmXQ=)6t`!&?qFQTHTX^x>|ux@MjOAE$Pfyf}<h$DYtU+hu3I80zL6
znT6#M%@PX>w~a)LS+}g+?qjDFKhzi+9f6DDxTI99IJ1APo!#%Z^`YrZF*mf&-eb{v
zPg?exz}O+)*YD()zm?@sYI+*ZN(v!08_8cUI+YwvIXOlkCFD}I_}vJFR+$`;f!)l`
zPG)HgVWr+W$<kiauq%RNZ~4}zR)f?-&NV$N7A!$Q(V}6~qh6B`BV*ayEI88gtIU-r
z6z<u5l1Mc(l!_SAM;EVkZ4SzCzOA1eR-uc;J-2qLW-Y88NNCJL?qn+UND|S+#Xb;e
zmv@iCA2A4y6*)tqlzY-T#ifxub{-3G)nnb79DVJhbC%}PY`EWA{1-mE6YQDlbgmeF
z<fiMxRlgsYVKAi#+c)ytb(_KXfgTF)f$q^r?^w06J2yu)I<8;^Q)G#4ihu;QP6Y;U
zZR9PHA9ekeO*NNyoh)rFiIo>&5z%Wq_u(=yNl<NRSXk`+r;{?R2-WJTg=3*l<1*|j
zovqQTOpqJJg~NiYV&c$6tS?jk{K})FW29d~iNI_IWdDe5+-o6-5cD8cPMIvBoHvaa
zRx2wfHH~!{oELkM$;+khg9tk&Zgs(3bFo|=!VP)*4B8F#ks_v(h$X;ut`YTLk`uzw
zmbzliqRsN6WXl3sf})W$K^xvpZu_CD_~&u+i*L8LAFsFkIj(JL%3R}a`jAzb$$(fe
zp@ULylnRmS*Ott9FqMDWYGjVEb#^5Z7~!`Rm_gh~I)qe7NE*UsFdR$UPvvPG=RSv>
z{d&5OTm7!5to<141<#WBAl3QKRvLa@1#Wx(X^xN#-_&Spclog+o9V4M+yo{0j3>Fn
zUikHjL}&c5S~2H^Zx$u}D`)<a%Sk5(p^^KvI<;=Z-DWo*e!Wi!kniI*Gi7JD!=?WA
z24l=9IZ1*tTC9YY+}YItDBN{cH7{w8urtrUiMp%1@74rQ%H&W*h;344dZ&I3y-8FZ
z=-hFFo@$A0C|#6<iec$um-P&24I9yF{-e1BgnG{S$4v1)@~Idc)<NP{%?P+-HVqs0
z_DetsYYF}*7QhLWQ9QAapK<?iG)BwKos^OPqN=~&<~^>9)X+ClerOjhnxIsFDc$VQ
z)>~=~`j(5!udr{x;h_E8RhfF|*ERoD=FlH<|FmdA_ZhE}khf`L%!i5MEkpL6(uj$v
zU-DXt{qg0d(TO>?4mXEaW58CIRG56kb5@JxVS^LjS%NxE2gM{fxb)FdE{yCw@q}f*
z5X$<%{+)S2URIzfZ@TB3+sRnvFOjI)vML>;TH<_0S_Kt?4<~k{I^5ixNhvI!L-}CX
z$(*YC%CzGJt`|aV*cqO(1koAVb$`wS1%XqE5z7G8FW^RAs`(W<eamu2(|1dIOR>}p
zeE{kCo%51LI@pZPE7_Tna_U}~18eVchgITNnS(cDVkxbYVL2cg8^N506P%)DJ3X8f
z|I9L!fuT0C<Xc&mj*j4x4rz7LiD^<wUHs>tU6W7AsDj#1#;6j8-NWVjdz{$6vm|#W
zsb`lEQ8l{1g_tSVrl38iee_(9(hL$-J6*U|@zue!>$4zmNXpDGv)OT1ReeZCsehY{
zWL0@7;_N3$&Fp5*9AezOvE(b&vj0A38qpbX0i2j7p091+e6~)Vew&69Vp=6{ulntc
z%fBYTh+yXyY|NZ~f+%9yH?CY|Q0i@){mtGF0Y`pAWqq*ZP=Yd39vlAA?HyhyQ2P}3
zbYQk^kJZ8;rV!R2&+YW?{3BtI0wrFijrrGO++h$(IVQ8t6or4hJ-j}4(a67laf(|r
zlO*nek9VT{>*R=|(|@4t_AqenI&L-k&UD$W%LIP@=QlDV`Tyq?`QMI{f7=7DC9gRA
zywbGq!T+^x^2wBNK=l8gODExy$S~A#{EveU@&L*j^5lOy^lxMSm&=MldK}$VG{<kS
z=d@;tclEI0H>3DDEWwZKW3(G~z>~qzZC|?c%$%%RY{nKAVqn_@eTiB^oJt%>{H08b
z$zApSB{>Q_0e;A#IH?~SE?!<<bc;!862uZe>=j=}+CjcL+MC(C+FBBXl&5HiFmxWu
zilyu8Qd^o`y`aDXemnHxk(sX9b?wi1;VOmAx*Vg(8PQecDYc+~Ex(d>9FgiAum0BL
zYs<~{mFq14SEc(vnU`V2kr@Rf?1|13Hums`VaE-|UIZadkbLhR@0yab^4@+zhnZnT
zsho~e$@M)rE#a{{us|29WnmyzqY3+g2n_d=g4%E=KUYbc&Q1T%of@-u9c)`^qb2of
z2cM+Ljsl~L$NWulYyi@vQExo{%g+x1{cA2^baB++?2-Z}U$CfP1LMb5!{=kv@-o0_
zwlO*~p`h5y-1RaIcLfw+NTfHs#>G2_DLy7BDRsGr-b3So(kL+4H56q=fx##~m?5}$
z4IN>9-J$T$xY`{PFh98LOk0_D^$RAzgz&Tu+N5zCTuF(2j{$oF^Z~;=(Wu3NDm*SO
zlSr<Fwv!^%{JbyY=OW>6WjW;0g(Tl4q-F2zHUGzT9s-+8@%O+ZSU=larMXQChyLrm
z?3{e&S#RF}qdSlR@~-9(?svi^MBqKPbK1W<Am-p>&y1fcjPQj{8A}b5T<`zsp;w!;
zfl8=lH8KrqXKrW|osGXz>skm<6*l;{%|%}mZlY!pp}cs77V@onQcuhlb8fG!uJ)Me
zy+ke+2!-c|?NP$=)}#7H2?wi10mxAo;9$9%4$zL^rGeuH<o_&5eKrk!;kenh2AIPa
zL0og<IL>yS-QP#IBZ&bdx0@#PX90y1cQ7Wvwfp7po&kqsjkAuX=w+!dZV!Oyv^F-)
zo=o*J{<G1ll3kRW_Nc#wyEedFrpq85&mkDU))9cE)K_iMhw{K`yVZYD7@_=xh&6Ci
zj(%A!@B+HX%%P4-+M!Aj0kXqE4!R8FAZ^Gm;*OK)m92y3x4098m;KLzoPN!Qx`5(0
zbod+q7=`aRGBS|2pmqgWY|JKEhtcNqqTkPOZ;w7n|4B;pd&r0Ff`~=zjeH2-Kml$v
z^j<*S;`5w$#zq0?9hZhC_r&OXOs`obhYHkLhet=-i#{6m0^|v%Iza~yF$bztUM4WT
zf?ny1=AfkLQp1mmxc#H=2rvLd0rK**H%e=?3Ge~S5qLzU_L<>LfJ!R@AcUyB>!>K1
z_~rFviw=u?uB9#j9@6#dg=qL4bqLVT0K^;YS<|W#6KvBT;ngrMPx9WbSaMgS_l7ed
z;730BdDw*JP@OX7%uZol^6tN$|8bJ6-^ZE+9l?ICW8kjk1^)dHK$f94i`$ZFFkmS|
zFu?GEYlF^#o3va^RJd%Hei|JeeTm_sUCt$_7Ic;$X;iI4t_~nykNp8Zi{OrUPUp3V
z3Xlf?H*nUXR;8;cAH4aIhX62TenIc8yNCkf2K;<I;|@JhKkuSUc_1;=7ij`V^CCV-
z+;m#Y{jYTyy><Q+UTPE(?EMc99GZ_O|At`VpLKBujC`pPdZXXR3c1z*`BP-E;3j$M
zxivNK?*N#0+#Y4FhKHNGyBw&PV>d>1<j$cC+hxrJXAd2s8zdIFVem??-Kw65oXhzQ
zuk|!5R<c+Uyzt+8Ldr$&^6})kAAIUPlmKf63@DhD-_56nT<}G-L6McXMF4Q(MitfP
zU2$F{S>#&#Zv}OqL7m=EL?q6wIyV3s+Xv(&@)V04=7b9Tzr=vH+;i#=E|<N*R;^$7
zrqXM}4zyO6qZ%L%%bWf_PY_*C1=fLZg+r9T@XbACrxEmD*-s?m<_da#`tUm?Z+JUK
ziE0)B_8ylfpc8trWVziJ{|#qycX$7Vtwa>QfBCECL5%&ECyL3M0&ysJw4ggQ<JDP@
z^m>ZE11{j&_KMWw<&vIBI$TZk^(p`tSx>u#4r&h7dW}a_Zk12i=mM@L;h|oIrOe7F
zE#RV>V>k@54Z=j6J++*em{@xux<+yf_*CZm&8L>z<E6i_<y}eS#Z>I{!%U{jI_@0y
zmw>#+p<frL@7SD7?Eq)=13->mOFU!RT4Mdnm7!<c45h0p%xwbVo4*jd(^5r%V{!f|
z%OgE-Ihv(<&Y_Xl|KiAyCAuGN=ulI=ez~23^hY6^ItQ6IK;C(6=0)DEmvcZ$gxJl)
ztL7*UBtT3L@G$8k2;rjG|5%_2(7;(7Wlu{l9DLOiB)J)V#sydv>&Tmv`pdgb>gY3;
zkqWqvT3TAAmcbj|+qJ)*<A&twM|P>A1_1_9mDeBBJN}JpA7?!VyG}X512cp!?iHSJ
zp|N`mHcxK!A`%0JLqpV=6=Z5)kQF$tp>*Cu=lj*O^PUoq>$(cFYWnz&G3j_&4bQqu
zgW)<@w|8_S;82g2zFH!lckZEG0yw^V0M5^&!D@skyS(6J&jrFD6a<_AYhu{u0L@|t
zq7*^I`%Alb;}F?$`Fn?3w-O<~o@xdFHRgcn+dR820|7aM6E=;c!#X`2h&$%cw>PAx
zNMCLsNB7$s_gloRZEf*4qi3;nG3T-U2+w?t7KGKmWuqe_^e+}wj}lmM{1%8UhqCut
z*Ek*jEL^P%*8pZA{~b3<C9V2rbbi&AhD@>vPaVGL(R22=F4H7I+sR6owda85Mh8IY
z<p(#H`@{fZ`VO-i>V<nod*E-Zkx%fZnDovlN{Ml&rkIY^{rQc!3|a}^3-Y=)w6nH8
zAQoMP-2PxnJW7!G`}7VQU28KYw569IX3s=hoG@2a){&nu!$Tn4@DT-&H|{`icR07k
z-```wH@j|wiv<KDSZzNXM?0O-N%BBxzL)Zv4AJ%$`_3_kG*y%_Xxd#n^af;`Eqae>
z6C^;a?b13SQ19q$cd#gjdARC}l6J9^{vdFdXh_r@xOH##+a2BRa=$ePd`tTQ9AxPF
z`Z{9LDSn}e*_iTxdhsLTu-E<{TlSrPhBc4?Cv8o@_CRCq$vAvs(;kL<1AuOu^gf$;
zNbusv3-P_q4{Wbe0nVG5CH4WqJ1P}1J2!U<yID5;VL;jdXp@TO6Dnef1C-N+z(B7B
z|I_fh#RdvzMD7eHPFr^)42s)+qd!Mi9aC2hio|ypMHP<jc7QKoJNq&n__xB0jQ02I
z7Xpyf$OEiE@NtW~O?g+nB~zqo>S5%wF$MxT(rFDLbQDm{VF=(-^_qPIL4XAgV8rU4
zErt?D#FBFp&CSidPeMbXcKTv0fpbOd#8UY}6zIf!ICpAkrzQVAFL5Uhu4-Q@Nhp;f
ziS3&=O<A7EnZOn5h0Q%6VL`s}{!knxKj<eKs4$^2p_QzH{zS|U-@4Met6l(zobT?_
z08G|(hua0^07#?&URcJ~))s;xlYmS@do<4zvf=a|u|ULOf|E4c;rz)q2;GUDzKN9l
zf-f1l#Ent}3Q83|d*9T2GTS$2ThV+4fRY)KC)1&T2sk=Cj5zzVQd(X<U;K=pXc<!*
z^P|HC+izKgz{?g;ed>$XF2Lm4I5-r4`<DFCC?M@qPC)vd4DbW&w+;|h5>74bK+wBZ
zt>zesH~$91Ht3Vni;3IU!5pPx;OzNt4Nw0>7hlQ$zJRxjse(%N?)$>&wF}tztw36N
z1De*S`YA(S@5|j9V!#yKn>0=qa~O*Yed_rvw8*xyka76qJ(OWkX9(cJz)^j-o~?5H
zQ&_USLhc&kb=MHU42}dupb^RaYu|llD!+Sh>iP$5=udjqVSP>u13t!q^E_c4TeyK~
zsnw36IYZc>gwPED_iye2(SQUf&*cFCn7RW#1E%cu_I61{#V5d|S_mO0oqvug6yQgO
z@BBJXdG3Ui5Te0B2I?i5bwq%i!f#ckU3hmDM##7g`#~9r5HLJquy1d)rUAyL82)Op
zyIf#qCkI}c>>CuQCqw$5i~^yN1X>6#gb5-Ku@m$R{$kF&$8gizu(Y_iJA>COq8>;?
zFHmyh)N{sy+~b$K4&XL*z@1mV3+m3EofayB#N=PHBC3HFGf$%gw2hcih8>mc3UvVX
z2CS9YZ?<~>g1+w9mP-JC3T*+XN)tc^6R=|#(6SaHtm_n*Zbo##S$~_DC?kug_;HBZ
z6)2jBQcWyn<igFK0|?^w_8UiJleb;#^$8N7;CJ^bJBW~gO>`$d!4?5=PJ9mYB5!PM
zOUue=HBDkCqvISbjk;w5H)wyHcH?UW=4f36)|PJ>m)B@M9s%1G#3q5IQV5v!ckn$>
z<>QVX4yd<eca2*Du-JAW9K*mQnuKo#{eb8?R*(k=PVelmzb4h<#{X9!f*6`@5+}LV
z1(c={f$4<eHfHTNJAg@+`}uQkhM4>3dZ6O@0KPU7$c|O}`d=RX8N!L+e6-JRyVxdG
z8<R{TqK^8#t$b=%j<ur}+2w9p*n#Dm^^9<Ge!f4JyJfnp?XE>q05!b?_>Nn<y99s`
zN7|1PL^fvpC0HpqQYDnrgr5mhv4bJ%J$g5)FWTj@tnmq&WttvOULuy_-5jd3dVzxx
zdAFs-J<Ely25B7ciV39)_JOIz8vCSX1!&ge<KyZE2EUriK7Y>t{Q0wy5{oiW55Ir^
z4s@;E<5@s=y_0P~oUKmi7+I|KCNBY2iEfpt!d+pX$bo?Y6M%M&?aH50V#KiK_xwz%
z8xwoqYZPy-np2AcgjrGx;Iv@)S*8xn4u*6)Gkv&;f3nAubKJ*a-J}^V2Z9j8sdDd4
z6Hm__O%E2gQEf^tT`^Y!MO&SfqzM<w>;=I4hj)t(%y<)kGx^@}Deq8XfERdi;q&>+
z7vE;k?2Gs(Kkfq!8(^aDnhw5-I_@gL08;G(S~4zxW&<3^2*6Px)+u{UN*1dCG}(f+
z1$LW*7yKBnom{_55=of6-nALCUgPx7I#23VgJuNQe{>{_QCwVb>hs)<z>iN0%djp$
zs$g;xbjZ5#-;wvhTFF1e%HBu@njiq5ekm>ebH`}}r(l!KSTV2rwx;91CAelXB)s_m
zaO;jn2s)*1WTYmQvmZc;hxw06T0X^&M^5>iJ0QJ)gfKj+?vGBX=IknI{4*f#ejCad
z-%(Wc>$zZ!9-LovDu{GI#{loP9cb_UC^+`+Q6(ta&)tIXm1c}jH?-zE$%bZkC|$pp
z;{d6jk{6IVL~e)Lh9v)Xl66Bd2xSbLPT4|m^0wfdi8^G%w-9v2tsdAOUGDRfJ%&>1
zcZzeYf{?@f55PYi<+KelEo}RH-Xsh<0|+y-WnC)1?zWYn6L-Ze0FbTZ^XKiNj(nZM
zR)C|cKZ$XhcCxO#anr5R-7V?>gEU;$rV@BZe7r-igQ`Gcuo_F%da|9(O&sN-m%1Ee
zxE(xH`M-4Xg}Vi&#pc!btoEcN<<vgec^BfV1bo1IV@&j8>;#RR2r3E_Uy(KO=qyX&
znd%SuAy86vea%B~?bQa@m7D<l3;WyH+W_P0-#9eX{P6~yZUEkA?H3O!*{K;A&;mJY
zNFoJwMKl4aYRD|B`L-?oGLd}tj3)v-mVpXcTuvH8fjT-mKw!?$hKwSSbaem{4{Cw{
zq8C8v;1tLLI*2U$55PlzfD>ZPhMyR$5bUPMXjfy8&&9VGKJCHlu-;cb!Wt!uQx$*z
z+Bo?c@g<EW^$)6t+;PSi9~UE_5q1HZrXR3}N_D}^kQG3^W(+-FH|vK}<4H;9wRs@G
z6wG{?B)}qOm6xT;1?L=KLNf=x?`_$~F3yoyI0bbj49E5KfwrIkeK=bg?m%m)v1mvV
z4RPO86yxB$l1?W^-6T6tI!hGG5EmEk@H%oY?8x^<eLy$~iUk0%Ti6C-bMvx%8J4AQ
z8tZtAJ7$B{c(0E`DABpoDS4P9>=TdE9{90WuJk{&##l$?-@83@QuZBBw}Ft(zKO6>
zF{cOw#SDqe*9TWAQaWc-40~-wGTUCvDOM1|kGYM$?I~CF8i(&y;9`QEFL0h?F>xjl
zvC~Fbo_%JfJABs4^zJ6S4=Ak-mafs|*Lnx1W^EoZFsN+QND<W(qd5LFUH!{ncE$<K
zmKiqc&@s~qUUU;(L0wqjv0x;Op8dN$E}^!{h{+!O#T03An>IK+lr8VO&FmvnANJY4
zTphqDX%GL|&$nXyW>*liKBnCqb|k}i^10fT#4f|?(AO~{^j&jImHi0b%cD~)V@^qS
z+^^m7uc8ermQ!fsPITmfIV(<(j4Ko~zS3!?1;LbILnPn4Z)72}@`wYrUX{x`-@fI!
zP~|kE6Qn$S#To^fPQ;AA-5sO`Pyc4|FcoRl1e?k&-fO`Dsf@4;3f}ToysDKIF}qr}
zPn(tHVPrczgmr_plT>4dUj|5`4EL&B&0m^ZC2wLmbPnvoMgBC<L$5{WAJRabp$`-*
ze0<i$qawKDIvR}sSch$=QZ9`_wU@`BqiJ@lVO6c@bBdT{vDtK29=`Uq_SBMxXm$uf
zZ0m&P$q|0KN4=ry?#U2Y^SF|AkC3wmiy3kxGr_cnDEL&|&oce{jM_rF)vBFlOyF9v
z5%)3g85}BbSYLheRknSD&)7*(q@uR5f(5JadMB$AI+3mcnKC9k%HGfA%4>V`Qb1R#
zxHX?Xy7p-h0*P-b#_`m;a`PK+Ue~f~jXijMd=dZ)#nfLRoa8<zE)Mx<iq<Ko@nD6~
zr)WTt@}WvF{4FuGSf={m`7XFsoov%5zoqI0lzp1QXOQ@7NAwd@nU&Fd(R7>70!b8x
zaU?9_o7(!7AO<^o9M*mt%-#>cvxt7~yPN0Rm2j+Bdg!ex$Bb1`*xQQ2j^#l7YFe4f
zM>wX0WME26`<5dp<U?-k<~y5Ru^Foiu95rw!8*YwN&M+@+^a*eiv#g}y6|1J#bTLa
z$w=xPZx8Mc5JA{kC;xQ+2L^A<Suc2o11mX0!O*%Ljw#ZIq}|;8A?1XQPvz+1=#B`X
zRpi?1u_vY#{JoDEFO6MZFF;tBL~gK>#$jz|xx_~-n}!ZFtxlmg5HGtl%Io@vBVOrc
z@vB~R;Id3?)xO<_x0q*s{U{~!0{e&>#AR^FssHL2y@Cwf=;!Imq~qX-C*0D_KBwN+
z!<*;bdIUt)uCXwDKhd4DPH?M|H|w;(1`<YUIMScmkzBU~%VKza4>8yKIO<H72Hc^5
zsk)%G0_EmStHGL7Kr8y98<{}L*)?Bc-TdanlHrJgOjB7EK~rQ43wzu;Eqtn6>yQYv
z_Qy^ogF>Wt2S;b5Z8%yW`<RIgCanas_5^jT96_%(N{9)o`UnPTheoR3yx7E8Js%@&
z@hV81h#KL=%@;dshEu+GWjl5=r-F5cboZ1o6N|bdH;b<K8iS^2G5?4-nLH@_vye&S
zptyOOD!TZS<}TfW3C6^jy`W+M$zdI)j&R$<>Nnf@Jsjt@Q<9ERz=k!{&S}D%Mo0by
zQy1?FUpHDmAisj{!zR_mV^;Cylrk^(Aa9K=*b1lpLO#nCu=6s~;$Lbn`@`4dZes9t
zfVRs^btHce(QdJi7NENS>^485!%=V8%k-4jOu_inFnLL6u~XiR<$KC|z2P%kvX(`u
z*Okq`n}Il>!FPf#3k}!U`BFRlY*zpx6Dybh;?l1?>$b>bSb!ab%32X<91<z&^CGSz
z^E=)I9jaa?B+@S6N#IEkf2CxO|0o)|NNQYcAv^AL;B0H`f!d1Eg3po1@`3NUYXWjd
zU(t-Vk4<=2ba4oQz2)5gyh`!8Tlwa6tt+Yu#w4mR+QwqLWB!k@&9hb2{w})jUT}-q
z7DEpD{A2L~Gv(`D>a2Q!uuF&mIbsj~s1BG-;s}%cIS&|pm>(Vmts&^ZN~C;W10s(6
zr-Sq_oD6q8d9eLHV#d=`x&E`B#OHBo?Ax1Z#>l_bU85fmF8&E4Vk-W$EXMIQ*D(NF
z6Wwu7De|~zD+>37CCFbT5|j5@_j7JXQ<!0lfc`9v=sAh12nDS+U2bj;R+A-G|Cg#2
ze1zSi&?5Siu{vEIaZV^PRsuTGBw_etkkW3ARZyE7`|4wo;>(D)Evp9+xE|&ETGI>k
zh3CtJrhyfS`&;`Lmn+X3%aiLSdda}Vk;!`uEhJVs=~4L7BAbWyKgKuL-u`~13m>l$
zpv!)GX;wPL3ijy3^TuzNL+uJs8thCg7}30KPMJ`akou^AJipuy;%nj7jnI?kRYJZp
z^a*u;Rc4XlXe_JgZycxz)lJew_4E7wCo^hFHBNEsiHEi#*pD~~-4XT9lE=5$G%06(
z;6O@$8#u$~%_+&HL&$^gh3ZA%CCnwpnCP1GeYX){4lHLHKF)qgLnh2%%zF(xqL8CI
zR+{Pe|IlqKCYtM<KJQt{%-!H@{&@*mZg=v)Dozjn`?&nmX7UG#S-NS=RE)3f;pojN
z?;>ujtM<h($?FbL7hAO)9bQJSejt>Mx@U2DPRGsld%1^rx2v=CwzK1t<^|vW?A}WT
zrIy}L<AOqC+1yV`H+_N~^pAjPTp1VF#;}IliuMR&Zm43pt1F#C2dAotjkwyNsq8=r
zRuMq!!0J&!MRO+L@buUPMxvX!Rjg;Z^nge^ok%}<nDH<eTjl+p$?L~fuS;f5((Ehr
z;7fyjdobIV0;<;QT}ucg++<Z@(Z*O0Q?j8s%lDm4_J%4o^W1qb?rE0YZMDJB^Tn<Y
zu0z#3840nS0h6_hPof^+++Z}7h1}zt74z3547s7gwzbR2GTWy$|J~cVi^W}Y{kxW$
zv<m7#qhuX^5Rj>VQcH=ym``>?#?CR);1Inh*Ux|xZ0p?oTdfE0;5{ObZR@rs?OxyR
zRmJP?YA<F^tzi^HF3Lmq`5*sQ<#iiGCF`Nf*V&yVqF9xjUMSy+(G7-u76bWe-4uNt
z@6VLBPWp>)>5CSsB_#BOUxK&eWFX7yAz$Q4EFkm&5_7dbYl`ht(u;j@L>@X{y@C_X
z+%NAq-3Oss={AmIZbMyz->Bf!(_ccIZrBg@i_;zu9`QN7j48P(wP`EzxJfR~lKSBC
z+kf}_gBr8wt$GDqCRA8w*NP<$N7ddKVh-B`5>{+b>XcRRCU>L9Yw7ylfirvREw?t0
zFj{M6#%>yq6ghkXk6Ps8CJTsZt-QW3j8z;#_sPzeecUtMYiKE-6dAvB9cYWmXoMgX
z)J@3HIDENC#Lu_9qOgwES|2ilC6Eqp?Mm?F@}LihjSV_$A^q{#7@J>y8FL*S;SM*U
zkZy0#7e}XWizlDOCEwzKaDiZ2d*1>{w>jR*pYnC2%(~en^~yNB)n0UACP@uQY0;%j
zE@Q5StB1WscH>G7O-c>53EY{1lAc$al*T#|6;DdGHF2ueY}KKTw#lTVY??pnA>;%8
zZ@2OnX04#A1y_@`viGK;XT<X`OV?nJxziqm#R;wO%7X^{BYDN4%9TKiQ#O<nbfn|P
z0^1||c*F#PSValsxA79fD{xPvN*hJB|0fn8%lRI`P=-*t&6G=-O<8!AzhhNxApR6C
zK9|d~wGTJ$E!lh%-VplKoVVo?Gc7DC_JfyYkhk=p;qg&_s}tPc<IozgaTJW?2p%AD
zuexb+Yj&GU;pz#itEBXTdA^VXV_D4w!Y0wRD}EJYmUAqJ=;DT>uvnES-2Hkq&#n{c
zvHmDWRQlL-g_OzFb84r1gu;v3#3&-ecn3;*CUN|t+ErRh5$GIwo|>4JVG?*SOh3z@
zQ-)4av&OmGQ$&ORLJhWo2-0HxnfMc_9GwK<-j0JDJTD@A!rQ;W;yt5sE0zmcKRizN
zdV*8+h%NbhWRs4*?~USGFSEQHfu;7aY!Zdhn|$Ae)C9Jwt;(=a=Mfb0RIfx`hX*2@
zFHQIy)^ALguTRrY#Ain-Z$%juxSLVQ+nnf|nZ9qlr8}W#E-wv=bvwIP-XS{pa4c}d
zv!aubJr&39DNGr9(2C%_UiBB~G%hO=cihN9zoO=Px5Vnh>_Fq({e3hV&q%d@SQpA(
zd+4;HJZmPr6%{6fvSpTdHH}{NJ`MF7Jo{#9_gZ*~rbuLnr62VxuoBC=n?vNybeLDV
zf2dTseZ>^plw2?Ds*6lxtQPXvv)z+Q-o*WL-clq@)%U7%phms3!2z8YCm(weT6LOT
z7qf`YhvUP0PdKvktdlqq9C9RMWouuKfC7m~`^rJVr}6NKy-K7)x`hzeeczC+#HXvR
zk4zw%?FFCE-NeV-R#chXXD^i=5rjmUTWw1_bu<${%M8RU_YAP|5i6Qn_BlI^-lHzM
zJ_gPjK}%TB&WFLY@;O3s&VyHDSzzHN?=>Z+m-4FTPe5F%+-mg#_QR<Ycpt~PhZ>v8
zK8ajk^}{3&N$L@m`ja94XidTf+K;Kt&}xeLfLvT>A4}Eoaaq&>PR1bW0qVTSN9U9l
zo)g;Rs_X17LJg5_l@JAwmW8FvB0}Dd2YmODPxdKAgkR@TSu0Sa;RMDSSjAG3kn6wx
zt@#T_Tkn`8W@`)=ZNQW1E?j{(<-k7&m@Laosb3JpJgc)$*s4T3(3@Xf4^M5eQMImz
z<GjG9!B@ripOJTmjVYXRsE$c!FqQ@KYe<rle*@y~(OIEQ$9>5@mcgf@H?2+`2>Lcc
zUM~V+x66Ha-gc)O@&!C^$C=iEug085B08g<VKmGFrg^#t-#pzjJO8dqg<%+z@v>7?
zDU}K8Ea~*%KrYjaMAcm69(kQ~s(PH!Qu}3VJiefwRnNX@x6EvVkfw;v-)pFoE9Si8
z)@zfgae^zcwKGrjjWnJGR1s1*&PKa|@nu}i9#v)|*~7nk5va&9(WAJXhNNs<iZ-90
z&SnwjkV2=GcwykM%m=C6!0}2U7e_Chy^<&moV7IM`T2o_>qVQD9k1liEl$1@3R7O%
zGTUYB-O9w=Q{SS4#p=)QvoIe_P5;4!F3OFVXw-Jl!M$<GGE#h!vNyxnXKLlnA1s>8
z%l{Q$Vt#Oiav%*~qn|UQvf*@?mU$WZ%fwVVMr1S5gP&~KwMP=;ia;29VGCaJ$9|IG
zDOo;>F!ILD_g1*d|3zo~fQYp7ZtILtnQeV09C?3GQfwG+O$mD0T9EkMLT!%BH0NWG
z0rb+qZm#1M-cNacrg!11bCy9WwYC0}<%gD;?@3hC|M>C?ce4!gs(;*iRo4vN80=yX
z7ZlmpGv<FN^!fMhKvfUj7nOg9A@r<;UKpoy*o}huEpHo@b3+;`%dvApA?iJ5H{BVe
zIQ3L)fX8xbn!ok;^JrIZOH27Lbn?pg6~+FD%;UK@x4dNB6J8Q?rgl1n`b`Tgox{D?
z^M7LJbSju8s=+p}ln<YLnq_;~pT?S#zdvI&o}H(`5w`XLjnZP&uJIY6$SF(uUiEC9
z^2@=~a4Ed8&eXTukC47n7Pmu3&gD!T23^zV9&N|Hw~TcQ32alilFG)NEUP8a*Uk5Y
zQM(;C^(+`3Gt|$V?4~k80+U#+lh1;RE_K<@dS+~cDS48D4`e2i?`9i30%^%pS^39Q
z=7i|885_Y$u8p8nE8$XZx-U=NmeylqcPh_3i?%AiTnwAM;xrg(h=CIg)Gy^zf(#?^
zX(1b;-6v{Rj<6D?juG))iNMZ*9V|(?rv0kDEjF>Qk5Z7qie+xv!)+s(u8DElOB4Vx
ziZ$0V(D^-*dZ)V4qTd64aCQ2KubNa?akU4Y{re4Uvq#@9XSVZH0&6!BpVFpXf%!^M
zn}NH1Kx773)1M5zjGkc27w5?fz#Wvyh{6v=g+Hjfb#ziDHjKj;{o}mEb!(B+(^E=g
z(cQ6^lJJgn{BZZsdbEq&32K%Zok8+PT${mPxd&uO;4bVjEvQvr-pF0ve*fU_(B2Sk
zlyBwz>~2ti{vwc<qCtz3WG#(P&X?119JL>^!!wUg-r(o63=dpfml3}we3XkqLpf&a
z9({dlmt$aT+8jhJ)`C^vc@<r}l#;F}-ezc%&b9mr{&U=@%e^|_7cx<;sRbf}6c`cS
zZQiq6c#F&`6<=I`49A^fqjg_a)W8XhvqAI)anHIN$P8U!93!ZmX}fcNv+NfTU7W)c
zW;exp^bKhe^=1E(7iftnx_-;Ea(=(^Z3G%{e>w9LNNPb$si5?9yn0ceVNe?)KG=Dp
z1?RoZsi7wSqwQlkMqt0t?Bt3D!|<aczyFu}@+*}v^LndF?g9nB0Lbz{F?6(4gmS$!
z<{K2fbm1uDcj`d%=nRjkU11)CW7cEzkoe+Mg&fGamKlEAQ&ApLJtELS&}HYOl~#`6
zRYzuI@mqPH3`qUG)}F8S_Z>YI%c@T;t!9^$qps6Jnw`o>OZ>Cq%u+#VLnG(y&|>1w
zVxPr<;tp9KV(v|}8}r#W3bK*3K|8}VR$}cv1G|B24qUbR(Bk%Tgrn+{kGpVwC-!#Q
z$ejaBVHw|VTLs)GxAw0&xGox1&d8nn8e2p3uNhwJEnexqc4b~BZdb?_PV)-v7nt!i
z6Z0a6S?wy$<GGC#?(=hI?Y~(4H5D9KfB02kPi#V9R`$XO*Wq)KKWF)&xKt;D<5u%q
zjW`Ah)zHPPlqCK#9xO{Nj;{E<O<OX!oU;bHmPdypHeF@LF{|Wz8~Jc^U+6fY_E~uW
z%fs8fb>tcR?K>mXp)}T}#O$Y`A*7qw1Drmsq8fRmSK`ofsxqtdfcdx4gei47)1H&V
z{N}k>!l4@h+SY$lTBJvPy%W#SFNiYDgpqWq6Yyxx-^Kfs`a{)+SMZ$68h(dwVXZTA
z>~|jVf5a3)$gV3!IlpU+P3~}QZy!8*F5~fLH)QIk7g0HrtK<PWw0k2R-po>UD=;Sd
zURCXvr4R9ZrmfYY&&bsBx?W??;O#?)WduCJ8@0X~p`<#kGjqD|AwQWp)uysB`gS8|
zKdqsXg^P;1@kTh~oOf)nVMJh|Vv|w~B7Mqe>7*hQYMhlu99U>I5&2wmg}DHm9F;Ds
zhIY#2nnALIxQ;E9A-gTZ6;|7XRp3^>%9~uyuBPTDA9((@Cge4ZaRBM2UY(iiq43^W
z@L(s~fAM`k7^W4g(^WS}reBJ2sLnlXZGCt16BwmXGsiwzkwyj=GD)G{IBivGu;f%7
zG4nTy^lm&paeWb?Zr#fkB0(chbuk;5iz%d1sPvD3!(kj1_+hN@XMMABvk~ia2i90L
zmBKe-fnP;3o0a`MufI^(^%+zyv95=HZwdcrP$7Dw6dgr5imxIqu#EKnUlAA-TA{$8
ziKU=D_DhHI0PEo2I2VZNj`>5ugha8=JzvXd2A<&+91GPbYL6L28wc&<{wrAOUOb4e
zDY9BREff^`=^Dtf_S0IFOmuECN%E96QNPv2HZL8is6j;W5w7;HfQ~AA|1>e)%AA?z
zYd7aBco{#ku1%MNfJv27ga)3uEBcfLRmbsBcr_jzPlVaou^fd+QJDthUxBVv2||xI
zb5Q+^!&(mU7HKRbFVo4WRSl$=2|}#xF1dq#y^=aK3*2yOo-;Jff$gs8oDsFQ9}}%#
z3%o$(`uaAd<*(>RFYnOlg?%P&>}n%<zB0_k2gWj{t=B<0kA8prcd$%%9&{d?-x5rD
zi>MDQ*+&SS3pm{{9O=$l{W-moYN?UFUi-8;lxN-8a&r7K0o{xAjKoQ7?hFzN5g|_v
zk7aloK&Ja$7Dnt5ovr?nwRvrt-FU2lB>(VXe0Wf=*>7y0S8whwK-H9Ax5ajU_yFbR
zwpb`p$6}k(Ev+p2Kcu|{RF!S_K1d1D-6<j<(%nc2N{JvL-5}lFN=Sp!A&ny4Asx~P
zD%~Z*p(W3OxgXy5`+hTPX3hL({m**ewGhrz_r2rV``P>28*jBcxg+<L*q0JNbXq*V
zf8bF2q~xF5(&L+&KBaz`v!TJS#vOq;LLJxi->$74Tiqumntd2yQ?1V^Ve(ED%e0?q
zavxu2?j^4ZikPFMb0!8H+l%CUREG`cnw6}pw}Ut`njNXG3WXap)duBPl!X&5H+8C|
zP|f?s<X66LF?n*-NS>I=e02!2SxP;>aZ^&_$Ov2SXIZhXL|DF}3L!b9B^`V{=ZMGM
z)a4(A<!Zm%>OF+bM!|gOhiYA8ufRBhkMby};&U6x?RJ}%76wtjIv}1Dakfv6vWZe0
zQjSYQHXIBfR_<1=zDZHsH(5#@XA>w;;C@r&`T50Z#hrJji^|4q5?Oib3a@F)&3sIE
zbzYoU6|5D_HvQ4SiE6CRAec1%Nm`p_wH3?%Dh9ShFn1=Ktlb~?0fmRElwucK8Cf#J
zB`Fp)7VVn25y^|18}%Hea-8?VYnff2H5x-46OQC1$ebC69E-%M8gV+b+h`kHShX)j
zK8;Q8AVndGOJjUzopf1`xg4Q$Z>D!bYza%2P6u}$X}-6J?=2m2rj<9^cqFz|6S`OD
zVpJYG8In~W7M!=Gp2gHAqCOm%o~PSH;vldC|B?6`q@~*ZOh?OD5R_ncaWmnN&NKDV
zWpAx`Q>C%I`9@+jbGn1cL4diN)_Ct%e`+cxN(D&jolUn{5H}U0u=L=(X>pPzH_W~H
zNhoQ1<uA5TV;dA6l*8<{@NC&q%tL$9`vx0MKK|N9a<V*ksCKM^cq^b`yD2W!LwvsW
zC)RSi*%ZabNO`iASO)qqa%s>`a641EYuI*gPP)heGM`#<`Zbytg9pOXt`xnqeeP8I
zVGak<qE+6xCUH-$g=FJB%pBBg)a?4RG1NuW)}O;DPNTmRU}Af5q(_f-_qBs=$gdV}
zP+p<i5cr{bVH{ADvs_!xqlGD;1jfNUV@X@dRZ`<n@p@wfL>92zdsaH=%N|Q2*Xkil
zke%BOGels0C$)@bugCBR_Mb}VxX;74V!e#7<wak6#?^y+zocj6lsg+&|8q0Q%Mda{
z;Lcr+m{>B~ORC3RXAz6WeAfJ7Im->hB;8q$+b?`&Yzfn=5djY8R?NTk0TYKdlb$x{
zo#DYtEI1)bD#UHHQ%N_xoUfLTJ7O)-{igVJs1JNvl7&z``$t{VZAyMxGQ)?I<2`3>
z>3z9Y{g&mHA540<H{vbw?Or-Y!KiH}20HGY(vH|s-3ws($`Z4*i4$UG-@?36!|Jwf
z=ZhG*HtJHzJQHz!w-~$g@@%)r5FbuMeh2A5Y%ZII?`~{t8md(fPRP?u!3Cm6%K?L^
zi8wqWlUS1)r!$0FPL9<wl!pulQkG0|x$u}xvIWrEE@_277p&!#_uz4Tf0JQP`3BY_
zWpX5y)9Zl7Z$UY0^JNhkyUnUq;lIG-yP-lK5zASKo+z3U%qs`O)p#*`QW<e%!CP?f
zYH$$eo?P-Yy;ob8l~hle-KoGln**%qeo?<0d1GvUQqY`p)>l{&Uj>S!bw9rO+c)Ww
zl1h4(0or_|sz|?PJg;e|?)k_2?~f@T?AtW&?leZ6u6wHq#GHSz8Ji5EUlA=Cdk4D_
z(q}m#=CN-e=ImPO#|&kdc)x$A?aj{ny=P6yEuhabB0D0)evl#6sFQ{vd0?sIuUCIl
zPyK#@r`jd1hiqJ5*}DVwa*Tj{%vDyl<%JIP7LlpGSgFo>sfaXWo1v=*-KOQa#X@vS
z`?!rx2?!@*Rj<>JawJYBtV6;3dckwJIwV6G1myLooQWZ+$OMN!-^>s>(~tVM3}XEu
z(l_rVSsps|Ze<7{KbNCFdmSpF3}R$UFEHNtV$`!u*4@31S2mgwR0s$|^G`tW4n&mn
z<#{EH?_rJ#@4{c4gTo%HvLZgl@>*ES?}bm7_lS4)=|4)3YrmKFUA>W4y3vhXYqKuu
z5cN^JJ|j4&q-3<&`Z|9a*+|$pdc;tU;i&Z8gDXAF1+x-WLUty<)eBjODKD`;nvuk?
zy^IQNx$Ds-qrIIU3GzYQ`)EX@e?S2xS}M0xRyNN$%jzaiV=qn!qAWhs@=8G{A5ZvQ
zcBG)VvHjr4rNc8SN8Zro5laF&Aq1_Z)(<_OW+!6fpp1ND_2Gi^F9vHj)1KEuSO##D
zxt*ID&n0KAA7|J7MtrV=rv@I<$!zZ;_^ae%)bCT}6JgH>322do#PksDs|+LLiOr%V
zo|xtl@$`kH+eZ5Qw8iP8BG!8@&He0m<!-gVTH4W-*(lBEj10Dg)fvK1+-X6h-O9sq
z)K7A0|Lk~#30Jk3tJ!3oNhi$wI#5P6guz<R*VB<%x-wfuO_qRgW*Zc-xCUGd8NHTO
zY;Xl-pKQ_$j3yIO$F8z0XDZ*|nKgGRG?FpP#q{1T2Ojv2pE0#3HSp>2@sD@V>n=)u
zBItRk=0Oy}wZ|m=xTSwfa*OXsOeu6K?TyrnufoK04-R+0z=@3ID31z+9Ls*n+@XHX
z#Y}_PE=hvJWgFmK)A~MuM_@N?k(Pt%vX_zalz;{w{;+0k)XZ!j{xA)#^YDBU4drK=
zaWl_x%Uh`z#x5;q4toVLrL(kl%V+a}Qc2wBI8tktHO@cZ%yU-KuT8y74au=-5kHnZ
zLzQwoyD(VBaLdF?7~Or(0rRM>%c}JRqv)mN<eJx;%LPNMmq-XN&hF6WeOKM1PQPd}
z=c^$6qBZ{VfF8`VYrTURQYI2fWzZ9y;PDjSk=jtoOZ9Wol|vuB`k@1Jz3r%7fAuip
z&K!ZOyz6SCVH$(=z%E0Ese5%Wm8+m1(-!`2b=7L1?C4s~7(8>ZuTTK+m%?3{r4QDI
z_nUOHaL@l@hHT?8Q*W&`+{kWnTCHr-Fi5-P@+Q%o69ke-Yiv3{gm=AL`gTDhT{!;e
zK$}3Aq&;tQ>M*?Zq_d)&QYWtcPj`ohSWA}ojq&>@8mb|M{vMTne1YN8ZkdY=jMYlI
zTGy|ig6V$DS*o_(-&Dz?dcBSYVzN{}OEECsBStGn6>Z{5B{Iw+&e<q;dd%!$^Jmq$
zsgz;8ZE-W?n&BENU^c_6bw-|M=4Ga>J?w!(%hHuqB5Ghaz5IA)WOw3u{<pY&M$D?F
zGu--{b|2f2<^A)YmotM>r+Vj;chT3zR>nww5S}ZI+S|YlkJtEeTqUS$9v=r8p^{q!
z1d4BYyNg7Is$5uZy1Qh(zo67SnU^{~))I{6ysP{TZr45^9wmjcj14yp?KI;Yc}nnO
znS4X%`#`<*{!93uG?o?ezHb8V*mx73ZhIsNd%FWHIZR++p)7dLxc-;y8ggCbG4MS9
z+NjcN)IcgXZGqVkr0e+10_pO6KB>7W5o>x>fep05Cfd%m7BBebFR{gmq?!iOvpbW-
z4t-pHIXfk)*=t~gK}blP<LNK7PI_L<5JV7u{i?U8IT6~~>eyGuLAzbVVG5D{+*9r=
zrU2>af%UP@K;9&V^9oLn>2i<id8u=?Lv5V9C^w*d7Mp*A>28P#!f2qiJxc0m*;<hE
zEVTtSD}IK#bUW35kYo5z7Va*xg`!<um9m6%xuQ*t+svOm+Ip(Siy32VMHeQ;oBPOU
z0PZKVr9y})`;i%pH`9Yr!qe?wW?38Ow6(;1RX>!@<(CI~s9RPl7?7?tf8PxE$TX~+
zl_(r2D914b1_WwfN4GPrBA=JG2TQ%Uj8*Be?DJ40uv1CAN8#+O!(8NksJ(I}8Xyg~
zx0iixyC1cf&frDR$Vbc&*nkjNWX?WAxlSZ1M&Vl3UgsLy9mj;5(846rpAx)BiCdp<
z!;@-x*_&+jIs}zDRlAxZaJ@4ljdtzFFpSS7`<O5Q7Mzxbx~&1veW<Wj!RX}H(%$*C
z;{{wM%9d~6U58mDUQhXeZkYmJ$Z*!nWjX!+LfjWgq0)Ik5u-iicLguT1wC&AeS*#0
zp@H2QL-1k5Xa+RmjZtF4AkFJfhKRF_gZM$QGR`R=0xu#6+^D01uZAv@>q^nJpU&Pp
ze~aQdhBh*Epdizr{bTPeXH6|9fVLg!Ym^a#lf~wBmq%QKXcI@{*HYnznoFd&8PkHW
zeP8lG@9ur`MDnOZ1qqt0r%p8T*KQ%kntpb~n0N65(K<sHEW)QrF++Udbf-~#m?86a
zKj+S0o@a4p`;rAXM-lVhD8nxhQoUoZ>&~bSTsHClzG}HDAKUFXbF{d+32CS5q)hvG
zH9bu4FL6CV7FX{J*MtvGwVf+b37Vt8$q$O@*oVA#_YYDU&R>j<zKoNSUzQZK%xxXM
zYLCmY$%t>_$P$Tgu%8NR;T!u((!M7RtVbhNpk}97)^5oppS|6rw+1vByFkH*wmo3x
zXBwI`0X9V%(m4e@;amQp0&3%VR$1U^B<+i|>0PTd9ZR{*)8YdK(l4*ck0tv1zGHOS
zwqoiBNfDXIYzo}O8Z~|u-9n%Qeq7@Y<hA#GFnKy^+ARrlxH@J=Ip6v%JmL3@xqgnT
zM&-|8iiRI)BM(b1asZ-1d)EJ#J{`4~Ma4d+SWkJna6=&TV+kQ*k}YQT5DTQ1!O9>b
zxZ`7HFM~8puNI<{SeBMXuBZL?1TvI9%N;0i^zgx0N6{`-V6#;4THeUG)nQe=q~mLw
zqhjLs%W=`alP>&rM7<aE3(xb3R?}!}gW;rPE;$<6HVS5&kL&oxYUUnTw8co>G2^FB
ziP;?)WwLtrZp(X?k+EeQnjha2N&KDX;lip6TQPcPtM{=mXy&L>m-EQA=6`2zwKHoF
zy`;{Z;URgT2ge(^lQr+Ss`l`Al#x?gEPVB#>GPq2s^d4bgDiJ=-}H3TQ<kcc<>9L@
z13ho=q<Qr&hD>oE6uZs2WmYfferfu2&X=BqWOE}LdZ-}%!;Y~Ydv<l)9m5MHu($Po
z126M}iAV2U6x8;uD_ywZpQVlYwohyyny5=Tj~$hmgc9r&9c1TRJx7_Y-v^ZwQFf{o
zC-v7k>K(Lh4!3{E@LJGU5y5;jN|%#SV*PvFSk>d`a`K?Y1-oS^mhDzm#Ci}N0%_BB
zS}bQRX~zB|WIZfd66OBY_kIo>hu%0cwuep_ul;dp%BuXtIlKJb-Kyqm<lJm~Y(kPB
z_jcp!`AyRnmAVH9ACzC1Sh5^CR6HYCN6+oA#Wwf!pq?I-3Tw!stKk?szK)riOZU(_
z@3u7jqbm0KcB+$|r!>0<Bll~uelE8JyXkbV!kT`sxp8#IM{5r%q68h)%FZ?TnqL;%
znUn8D2%O(JP`;D9@UFkF>kX9zm@vf1I8jr9=;UH=RLn3dzKoA4y>!n=9h&bIW87_w
zj<(AvulBip#EBZpS+o`=<~>1WgaIV_a&l<y1gAdFz!0c^K1BbSUYYmLpfvE?kN=B5
zArabxu}|9^`W<(5GjKbO8KRXmsKW@GwR}s?z=eFc9)R&o1ygrzm#ruW+I~DKg1xQ)
zr!W6-dmnOgOyXeZAo$)}!KpRvo7Z_AI12Pt7$myrljtc2D|vxOqZrscBTnmo4W=w^
zc<!H1G$z0UUSj{X5#pCPemb@m()r-M!jEw9ptp~6qBv6D|MI4!HzMr*!EsOkPn1ol
zQtHhq2E2W4pM?&)f69Xm=JfMz@lm1k{U(Q}fcI-U_bGu_{m?)Q=|h0@{ymK*9bLh~
zbJCI0vnc#{<swkL%}f;)mbb87tD?ty=dsyr==&)0WE|<)-a~O(Zbru2g;P|sK$26r
zF;aYQHA@xO%lhaMGa0U0$VUb`)Q{|@m7K4`K40colxXHgM4YHnNwM7S&c9c@-H8A0
zL&*R8Pst)Agflk!9Sg`qrDsKVX4t!zVi^~XD9+%h^M^iTR*Qu6Khc!S_#fWhm!eux
z)FLeZ$HcPGcJ@2ItD?d+eK|RqXP79230Vc59-q`&=IZ*FQ1iaB?ilNm4H4CQFmESo
zG1LAMF`iScxEMb5pd;Hr-cV0^7eN&K&;6blt5w*qWpAj+MZ>7aQ*_Y3TFBLy=zh_g
zEy#M>T<DuDbhs<^=OA-2p@%nU6DFS1b-LUt-K#HLASd=dA_96miK*3Oe#b91bJrUt
zF^2rvGqoQ%gOR;v3kx#t4DCBA9$o(-bMhIxi_!}7V`_F%^j7}wkr@^gY;u(ndEDL4
zT^=_ZWo9lY#4NrPacf~=m0%Q=2U8}&PEH&Xqou6}5ux2IKKb#(e7ihkZElk1dY^W4
zHUQYU-BRU@|BPjQ|KB;TkhK2u5rE`Yi07^fGJi^py?_epP<4ckmhkMJcWRpVda}uz
z&nIDJRQLX~^<$gJOtia|?7_?4Kg_Qi>Np0%nti@l^OLA}P6%??V<OR!>k|K`7U1<H
zp2J`#uF^_VZPc1g77ihyI-~>zTLCMS=r8?O-z#+Sw2Y#D&I{1HI)in$nVm9|VFJ;e
zt)gtyBSl(!X68r8Bg<D@HE_y0&I<JJ-^1DBQd_-$BLy}M#NSQ!y^1r7TP_WcW>Ge=
zmDj~YI*Z#^n*A#?=`j@=e5YBQh>-C1X?@tq<9<G6;TO0o&R|J_^QhL^Myb*PfgT-?
zGkGQ+A;);~kjuh<Fn0t~Eu`brSTR@r$N%Y?3a=G?mAzaA<iE!&^BjCMnqM_iz+L^6
zVz}1tDWS+tmG}_7e#CP$V=!<;JoH#fe=Lzqj9(s0Z;~^IRw8vpzFF?S_rZjLKWq2O
z{t*+$sm{t}?-7;WE8P|!T1$2sd8~z66*CP!q({Vfk%H7R8U^E>S_v}$eQsW4Q!=@c
z@!BTdO%^_^HRr+0_k`F3Z0JB+O-zDsDD#-zd7{RLdAJ3C1rPvh^<hfsU)`H%xmEhx
zfBla=d<`t)o9yq{QdQ5sMaK5-N%!YPD2_oot~kv#?mJ1^JT)g-`G!;H)JONn7auSx
zS>hce(qaNW(}Wc}Hq`IjQV^5JK2toONa=xbBY_s$;}_=CJTmzNr+&2pC1!&s8dDgl
z$wfPsw|YyGo2<fL4&(E^La*3`N0N^7{Eb&eDsvK!R%}}g|LQRL9HE7YK}t)X=4pDY
zJL>t8Md337#fB5uy~0ev=2mUmIM16>o0j%+HtVQ=<o^+8QTO@mf{L!(N~oLK1H$|%
z+PJW)`<9y4cfsV*^1M=Zp#O4{TMco{b3dQNJACTh_E)%wec14^rV#r3e?0)CP2jT-
zSws;LvCH<)Iw=a}Jwyk3bo=19>-T><0Q~l`9HeAExTs&^e@MB01&V}&I>cW!3=BH$
zP!B)6FC#42B#1H*wpSu)cqaTD47~(qi7<h-7ge!E_}*(*S?15GMK68`)e*$W%YNU}
zoMlc>Mjp3r@v;#T5B8kx{j15OCMS2Rz}r>q-5);+Ka=OD`Fc=};cph8_h|4Lg`^up
zm9=nN{baqYh5SRLZmj!_hF(ty_@_D5WhJItl8N~LF=@ZkwLLx(4J}_EzU+}9f_py}
zCw@uq{`>l5UoVm}k#+8`_$i$rMUd?oXYtIb44<_)%=;utPt@4$Gs!>t=aETEl=3;>
zHL6!7bP|IkoL6Vo_Dp)W?&{<7dPHN}+t~{r?nxvoF@=+`Cr)4?n~$3X!A6X(>_#~7
zxYtE~D;KY55?ih<q9Oq68htBJPbU_T$-kD@+p1ZWz$UU_K{&zr@SuoqocUmPzNxPM
z$<vnep~{;2uaFU`l#(W7dzwx%6A*4c{eLoS(E8qfr2jX6O33Pmt^#&zPwCc|^l;bW
z$`$ly&D4VFe?A1aLZ4XDgvwdqZqLcsl30gOlPMat1rMhTiG}%A#9l;{p@AamvM^qD
zu?S6d5?3B$b2P7tQ^Rqt)t?^bf86S~`f7G1YH4%w=d4@+^FY#Aom3L&+phx}DL~WG
zEJ5`oXdK#Zt(R4`6V-e7uZSv^P^7(%E3#Q_<^&OcpEHd;#moh{dfG$u&NkM6&Gt7{
z5+>A;pj}wQ(yu2825i&hA4-S_6Dl<zb1d_KEDHI#zCO-l(Jbh%%v3Qb<Fm8R#ObEc
z!9zd)=Svo#-)Oau+Hn}&l5s&=G|O^#x@wkS?(H-B|G5zyYQg9m_}I;lFlf1EMwKDV
zbo=>!7n1g}bz!_@3yVNC<|N@*S$47^t@{1Z!~rIyWOkF~j$lnKEr+wIGjaTGZfk36
z<DZ{jG9^7Vt)j?Rh#Qr(1poYE)K1s$Od;;Yjkx?f8T;TrPRXkg4A|2xc=7Q1FfQ-_
z3|BtdF3y<%k03992ateo{mzfG{86HwYI#Oe(%`oJqV4)`g=;sikQ2y2I-X4l;e=WY
zB_AU819Q}*5F}&ACFSLtO<UQh%YeZFnnVB34ELtXKcyGH8rll8lz{8JRo??{m88DD
zC!l2%J>dE{N5jBCB}dX9_>$@R5kRdl1{lBZ2Y^m}r-274>k?DWnRfd=2YjAPG9C5y
z%9B0Z(XEBYE($Der>{C)1&a#{-$MPruYZ2ty|VwTQodtJd>>{c)yQS7$KAXdg3Y<S
zkz!;Cn#0~FCtE78ffg|*zz|S<wbugCuj@}){2tGxHRMv{2V)U!(rFVpI=ABF7`^9A
zl^TNGtg?X;-U1TF1#!Cpv|MrnH$r?VI5!5Z8$pfb$m+*Xyga~kgi9@WFx7tjp~-dY
zsE1h1oER;83qw5X0cyyVOX%#p+~d8QmhueT^cJ78k1-iaF6;Hhh}wu#4OUU?@zArO
zuUriVT<vgP1zLBK_KOk6qwM2fdXC5kb4%CBpg~pCks57(ZpL?}KFi)}?`nOK6By51
zL%T~7X2R{bC{7n}ZUP>$8}y+*5vLnlG%_$4DN)ZC-uY6KD&%MwzE!SM6*4Ay72Vt{
z%J%5d4EP=p2&eF~i+-u=PKOrQ6L)t3dcOl?&>P1@6zK$-!7Yqzf$;d9g5ns^RcB~8
z8bNq}+W%~!;oKDoEG5bT5bH`0*eh^4N)Z<(RCJtgN<=??{;UF>*iWJ<q`_x#=Zis0
za-HoIug{4!W5Qh^vAJz+bV2XlVQGEQkp`{CpiPwpn~Z%Kuu!1D)p2tCPbnkiLaNOA
zEfrXWW4pWFV(Zr1Q4W|Vc7|f`z5c=a?AbFX10gXP<BOvWa4ntyxK}cl(J1I$gJd5x
zJD+S%Du|T=ynKt1#eRW4Kiu=<->ylh-bUpKiXN)|<M}sT*j!E{0#lWs%M=S68)`cR
zdNl9UjSE_oc`CMUO7de(XKQV6=)^%k{$^zW!iS0)>efYH?j<)`Pq!^M`TOUmT)z5?
zPb8$Ir1M^DF`0fx1A1^>@U=7nTe85*$sB8Y`@;JA<ioZ8;~}H#M8xSF*Hz<IPGI&B
zS9>PZ5^Fh}+B$W<upiJ5Qmj3o%M~=At(Ld!t;Pr~BEi3-0P+DUz*zvDUElrZ%bE2~
zNH4Ikuu4ix%|HukYSB|yS3W7kwwgD{Lsw&ln)42T`YAvTFy93>G}(VC2eE4<e%M3Y
zdVRSAwHS^`-DHCO-yF&4$7IR;NGtGz9~vWltCJ3#S*lR`=-|*0rNHYSmh|*!h#bEo
z@I5D#-rionEwLq&^Me(?!(MWTC?M0R3)Am>B|Ms1i1VryoD>p3u)`X~s*pZ`o^_G~
z05PSmy~6P6M?nuEg`ge9&aX1Fz4<2PbpA-tS}w)axbz<Ai>WRd=-w=LTxx3tECJBT
zmD70NzQradit0tu0j=SHp8_a(VYe+r-vC;YlT?e5h2aO?pu2i=!E?#tV7W8XW$+<r
z8|E?pMgUkaa2EZJRUol=gGo#w>}0%X3z}kk#X)Pia<+I{eyr%bt}dDD%c*u=&_(*I
z++YawsHXbwLR;|Z)5@2cQPXen_cFa!Bf%-v4GhX?1x_?Wfyw`8q4q#+9{Thuk~jsS
z6IhM4W=`laJ<=Z&DAW7yR)KbYzPTMbi1c7g{1^vtVZ2y`ZDqRRA?>#wt1c3~g$5W{
zSXk(L%viLZ&f4nzXah7T$1CCV)%$H%C{94%CL0`$0?xbXd3{gqINRYaL3Vi`Yi5x;
zS>G+U0Y;TOLG<ut46yRC`vGSyGySnt0`KM=+q0ndZKzYzV|RMjzoPwG6kqb3fzN7m
zeXnJIbWzg((k%>+b`TPxMV}oGJc*O%NI?3Le@|7=tBD`J!6F6&kL)mtfWdQ9kglrx
z=rt4Tho8VvXVQ%x)Y{>v)Cw&5E@<rb9zYJN2t@4Q&`V|tE&0`SyN2OVPTUiz1mXj|
z9)ZX>*x1-Wv$qv+IH0BZ#fukRCgHSwI~DD1y|ByKskUSlJhbwrO)4Zn*^nJWj-9sL
zUc=z0p(Ns=rO3m|N)hwq*da&4R}cwey8#AacJ>wMfdqsIn}Fm4Z8<lb?$5z&!=g`V
zBC15}bv6VvpAbZxFxK0920Z*VFkxK2vns5bLcZ8}WZH>{T{D05#c{7r=z&f-yQfH<
z@jwHIu7;$wrxmi|p6@z3h2HVyL<;~3$UxOEM<fMP4J&>AzV4vs7*D$&D;yFCoD?MZ
zfRcr7W)5HU_eZ=q1Z0|3NHX1j0T23VSCTfa|I?>W$P8)99=i1Rrhw#$+Pk&~AarW2
zzqk9BZ|`Oj*NP-4t}oDiad7IV^?a_aono;1{a73*9CGy}y9-2GFw5hcdBE)8+Adbt
z9Ci!S7tUNB5sy3Wbur&fY+yFVL%AIF!4~s{smy4`nOzl5iQ4vW&MwT@+HLPdq20NZ
z1Tt=il$kM%b=v*0!_a|&<_pERp_A`bP*X`sNw-->dOFhY$zuEm4<2}5HE($XdzWxO
zPTVC*p|3VDZrOFcT$7;HV-JxbX4xPgFI@8F3*>FjcvX`2fdz=BlXxYT>t@@uBz>uR
z8y^<T@#2I6@lQ=oCIgfTM<{PVgFBFhL~=XG@`H{~(>ag&eu)(OIT}DfK|)4mN`ame
zcr79zAOL`e{?rIMCF1JLYP!-yFVn81tjuD2qL5v`F+M$=3Yhw7$9BZD@BY_^TwF`*
zRQq8j<3E1%^wS4M?=;f@Dh%3R@eS-zfEncwaA}xFRnmGb2h#vX4H6O(yYEy-6TZ61
zy^>eE-9WLuwz52h!Q#~P%brFF5Dj2sfxARGS0WEF`aRA+VmHC5C<t&E`~Fu^vAvM4
z2#7TSa9D@|W`r5U&tvg-@7{6am_-qz0cZ8iR^xzD`-@IOgX!Y~?~tiFd+M>{h$s0o
zghV&xCf#9GKz1PbVw}z#DaW{9LrP9=i}BuJ<XHUA&W<i%qI!G^|0iy9QvzOK8Q8GG
zygWwg=Pl(!AEiSZcdLhlL92XC90JhEbXcSqJn{DSF20Ws`jd4Uo$f&{bD2xzv;!6Y
zxw?9=h3?GbmfG6)X?#|^w!glN#2N!Ck_V6-K3wh9NtTzEM*gJv4)|b~mAB+a$2m8x
zLty{ZMgZXhSS}2}7zQ@3wLhb10B{#Io=hlR>=@k)#@~~5q6sH`qy=Fxr=7Bf*XhA_
zzaZBV$&&pIDD6;bMcW{7Grvrh3>zG1x!TSdf&CFoP<#`NigC1=;rIdwc!L@sxH+1&
zt@z&HxMcVJ>tn0P&$4(TXhGh`o6c9T22V<UN6rD#`5OQ=U2YI(TPpn*;10o*#!Qb{
zo0&NPBak})ROti6C8>y3FxeL{f!Uw>*B96WPb%s7^)K1kf+sz|WZx}`QAD+TtDTr_
zaJ!cQ2h19I`}v(iuh#C8kEctzohL2EY%wMjI>~Q$nr9NcJ8v3td{TUUxmSx2JPB8#
zuqA5I*lWzhE75u=T1+rB1r8p-Xh5qJVn@&pa028sfg6kHHJz!O18U@CkOEvg^aj}D
z_lPnR(fbb{ECTV9MZ|!=rQ|igTfXx`Pj3h!82O-WI1q+QeLkQTP(5&325=Zedo|3{
zf3U$7n1+ms4opG+e3NSvF#kXUHUh7=0!78S8j;ea0wrCyMjz0{B`Gu4kc&0NW$;>H
zL8$-N7hOnm<p384zIX?%=e)GKIw6Dv>hcv+#K+d&5UUykGM;&$Iy3?2EAWGGDf5%x
z-+N4XK-d7mEdVp}N=hPt$q~!Y6h0p2yXX^Y>oo9Okk{3v1VNL&&vqfwyvLk9?P4kD
zHJ=om_t`1a2UJHi-s=evZ`0`MiI9-c@uKg3mH7aXL(@jGN<i$gmj_Mwq^ej9<evaC
z0*&&onANhQO(<>wOn>0tQlmS|?KfBLB0v?lfgSq$_wRhb6>Nr~q(&XPDgOh?fH`jk
zZZP<hD^OS$U~@swdf!yrX`Njn)N&7@83@TB0DefCQ!v+VJ{}dTxw-($Cj#87sJ}iz
z=9a1gRscSV+g!bv$`?;}-@W%?#)Ae2H%*am=lj=VC$@JFZzIPO<IZovr_u-O14(Is
z&|nF%=UD^jmV}NoT}KC~b;mz2=sNY1YOiw`2m<n?zBu}^zf(r-GiaSb$f#a`JWBYW
z1NGRUeK`ah0&d`SYC%;M4;K$IwSr8h+r-mM@69xRutdONxixlppL|z<R38jlVr~Zv
zVMmfTkgBhQQJ=5i2Ttd*Xft^lp|*DgT39@QsCNJ-jEKCvJiyr#^YhAVAD|cmXt*U;
z{TRSEfHA6_2VfV#htvg517QuowFP0cmED@LJty^Su^y=@Qf>WCz^u>+$dg3YqWP*P
zLji-qV}Ka}4a%;ilma_hcQt!?+vV$Q|5M#jKshitI9UA_b04rM0icj>uK`Sl(1IKB
zlz}HLpnEU{Mw(wx@HTL={lyOLc`F0UISugfI|qv!8ygg1-)#tcAp#jk{~bLDP6MJg
z3C));6O@^OZ|tnM=WPC+w`+EFwqHS8-H&CJ&;>-{ru$eN02E08hdUwWKAVsZcLU(;
zSvfcgfrx_8?zx7Btg|x@WcPr)0Ij?{+gD@z00P!Fh7ZqpT-Fr<@Jvudo2%Dt^Ao|v
z#Ra0rV?ECBE-I=96TO>T2iT_s#e#0dW?%(9gu`&UKLX|oImTQwB?!`O+jpw?g6BgK
ztr%{r>vXv?i~&(jihBN&DnNRM_i}cQTGWkIHXIM;<SS0M1$1i};D9wKP}K{S{LRw@
zY$k{eHUTJE1)<;W$=@5GASml)Lv)SniAs%|>9!D=PR-76lp9sEdB`TUV{3xzs7N?8
zbPK>JBj5-jUkA)*D!&a8DhA%L?;>DX>Er3A=>|iw<N!er6m5cCV+UwlCj2aw&kFCp
zLhSC*SI6dUW<cxJ1J=18Y(o`$)ZpN=zF3evizF7hqD04~>XA5`?}p9<0;3C<W8l!A
zpQST^M^FU3v#Ys}?lMNX1i0WRcudhCe#G(}UcYmvHU!ME#$iDeKv8;dXGbCNAPRDO
zrFj5l1OVv@4i$e%w?DW8h(%@Nto=rU-v5LU1Ze<PLzg5}ph;+8T6;@wIBi0To?x&I
zVoaW8mbxXcHQc@Sn*gn{S3g(u$ET;i-67ZS0E~5UHtBEx$1Q=;1$OlCa;L%-c!wH5
zKL>H!L0nwGg+p!)M6m22HDLv!zu|0&400ns9z9ocU=j44dr2YJKXcr(J5!@uYfT8K
zR))mF!Jkmaj=k0s6po6H09i~_oBx#yaA$ww9^NKww?aN|!@^NL1(yg#jefCvImC-w
zOObeM$Wn%)J+5rkeSW~<MGIz?gF_lY#vo11lXLu{(lQ9JwIHG6(WD9?I`~!4@#%2R
zB_2N`_*S&OR{`Z+&^xoreh})HU`Vwe#yB3tIQr#x6Re!(V*o~#UmLy{aT2?g-NN91
z(JZyrrh8o=iuB%hUT~kXuU6^2!N`ws3Fp<;)T{&nk*Bqlc8&=MjAzy--awJq)!fo3
z-Hic9k`H()OKrPxC+nG?M3HC^;#Z*e=SR{u@)v;OE1^X%Gmly6k7Z>sTiL#`0HOgf
z#CHlvz3@cV<9E+slWI~UfWu5tPft(Kd8$3|#_nqf_+1tN^+}P$EW9AzhG=K#uMaQ`
zy;kll23*;LfQ750!ue?0EY2C&1q+4aR=|5F4y+tf>(SsN$nXV8Jy->tnY31!&SBT7
zP?|!~G0E3CN0G_va=SP<_T>*`I`4HtGIpJK^5+<?q=BHC$|DU2sh#(7>E_xKP(Edn
zgLbN3zwuFWL$6@ZMelI4SP|wHJpa8ND9omD`F1Z87=sKy1klk#$ej%5NJ)u(U_2On
zRDpB=LiF8ASexvi<U{U1f}%>PcdksY{Hqa)8D*GGb5iv+7C-7A7%ysO)UQi;qHz$C
zhaeV_JjKTD-ZkrDIhTZApS9zkM^=J(iQD&d7Y{R}1AcMb1pG`I;O0xVNKh?4mJC!;
z!388OG$3Y}d8q~^vY?O?GGP?~e@l0Hcs=*@N2PB3AX-L4|GzJhRHYDZfYNRR0t7%4
zks*r!v=v<BR;h^5Ky2=JD)NtG;srbCUaz<UxI-ZtNMH#!PUjr=52n>5FWvy^pUuCQ
zo?4_LPILAR7F%59dJVA{bY4h9)#o!~0rfBKaeVm5IbQXE(EYKE%|qAC;Q`xF;DQ4#
zHb5pY8;Ddzw=EiJW@uCy#6$_m!eStbP7nqp0`N}LzXF`=764pG9}5dD!tKw<tgZ~@
zse`r}XLAd+Cw~VU40(v-Oil;9M8yeBCdi(005&M^J8M6UtMX|*bf(hs=6BkL-Vem8
z89Z|qfN5$1h_<=`!(AjK2UIGmg}-6~qIn<bHqqrE1^o6ccOuA=$nd5y(Z#ufyaH+f
zB8x%#0Eiin%>)xL_&K!h<l^Y?H!Zhe(O2c?9bAoyK#XM@;YUNjf4hXQG(&`z5M7{{
zGVSj_d4AXMi=MT^-JLklPX{3ifMk)b39jQLFIh8;6$@X0G{b`mLFss!bKo=~p1ATI
zD0jC`s=|j83QS}Q;IIq@E6d>cN1B}sRg_E-MA?XQ;Cn|L1CFWjq~K1>%BdjzyU7)J
zflbV;Yz(d_R{MCRvg__Nr`Eiw?>Zn&7;f~@kvE_E#1`>9qY{yecEdb`#IkX@HAnzR
zQ`L0Z1N`f`ZMYY#0bYYp(k-|GtfqiPt#P=i_1K1YtuJQBNe|@8fZ-eSH9R;5hixXQ
zNv?pnVdIg}1r`zpjniX~H<?@BfRQNu<~;7^V{>zQ%YIu1uu>ophZsafY2{gORgudH
zF!B*b%7GLt5WY&#Zt31^PSi17WhVB8t&-<4%uC%B*dfTPijU=g`4SE2?V>_LLUs{c
zUr&f2quOQ!IE0`}3Q7|&VL%MI0G|m!X%whwv1~Z|DnPm{NIDe#5G<?doZ#EHZ@=g_
z5d}firy(IIfOjd~e-Y#lYyrCO1c!_QVna1?I;+*5NC?`6;)z;^8#1VT;#|Fyyu<FG
zY@%~x7?)uaWZMEawKIZ^Ggc4Hi+;x6`l=m@2^T|Lqc2a>g$|l#3X}+I&MgDqfdM54
zA#tDi<k9&DxdI@ZdTl9fkYX0~tGKgNmceV>5!AYMN2;{6G#DPz4PvDYz};u|Pv}Vy
zl%y&ed35B`!L=v`WDi(YjUwgmcpT93ojg$hwu=k{p7F}IwWgomxrY#BL0X5lG6OER
zp$K<o3*=}4W>myePUQ{;`Eh3u*6;zksg4R8f=IdQF^Jv?SPh8uv9GUh_ge_aogqOa
zs~|SK*c56{%X(o#fext;$lMfw1fmW*l$r%v>v-B_qwugAC!pQP2rs})AUmaraS>gb
zYi|AC1N~;-hc|snkr8%$_`;sgc>-VMEO!>h=<v8^C@6R%6?KMQy;KJ;h3jZTej;g=
z>kq8<M%65-gJ{ipt*^?s69rf;Bj<~alr4#hn!GkWM%G!|Il&?u%VLB-$o3<7KYF8_
zCHeuv_2M8jhG-~(YT3CMgS@FYAWob~a4pXE`Rwr;de>DTG&8TiF#0a>!-o&<7w7YN
zcAaZNDbFFiEJm9^Kq9FRU>aD{-172RDe&&WTZ!yR{`b}`DBhT^R{=TCQ^eY#u;+q1
zIUs`EoKWI|RipSgJ3B8PsRDWBKq?05K0E|9jV9gNra#-oqQXv9o?aji?MXE*XVhUw
z+soNc^5!Wmj`|NAr&x3+;=3a35Wh<pwl8j;{9P#-40-N<3X%vkK@h7k;PI>J_A~~W
z8VX52Vc_{(!_hjE{Tk|p?54TNI9|yC=NSIb88Y^d7d>}D4jd`a;tgPE+=wAc6?JF3
z-57T~$#Y}aFl4<I9f>p*Rf{KSLtv*N_Y&y?&hJ4`ZG4jOODG*q0?r=(qM0o*$l8ws
ziby~o=>_DY2~hP>O8D?ck#Xb|7CuOzv99`0EkH}_B#5z8yuC%Tx7?d(Rv`ZY_O1!|
z+T%o|Fp~oiUT_1*23X^OMuV(L`Pb7|0AKZv1)SD!7QeDn(}w8|!kY!N^=7JnN;u3I
z)*rrnFRvU!N04C9_boHYw5U^dPFQqsm<fYfliYb>ku-M>{RXjggL~c^7`oqm3c*VV
zD1a3M>S<Q3akrUDy7JTh_nq>VSYX^!ASYr3UnSSn)CBLp27&hgzU*R5wVj+rIkf|1
zyK&+hqF2DHrBOi8IRFS|J3$Vyl=c-AXm8tx6m%qOBJkebT?FS6Kl*l`c2lmtuFe_a
z4D~$^?eNi%pwHk7y!KlRyzxhZAPBf2IagQL#oup$F1`WT+}BZ(R#x^G+iHQ&fLjLm
z84aQXhhl|H03P5|fru%*8)~$v(RS*k6dv4%ziMao^y3(QTU>qr{fyb%XXZT(Sd1(t
zq(HYH9QkZbuRD|DVtJ9K4og_y9b6$UH?!_{8v|(S2jC0Hx|R>EBXyj$ok?T|T-rFa
zfqLB0tPG($xsy@HDomfpM7C(MefLB_h{XdXLm?vvIB-)zED!8lV6hTgwsRjP04(%D
z0Ag7>N}Y_$BJB6Py(3X?-xK>pLZJajl9)$@oTK6i2R$lp3^C_@%9!vwUQ>}dX=Vc_
zG%7YRF%eIR*FFYVS?BNa<my^lPk@|ytUqQ*P<;OKW!(%@Kj3%;675y|HXCB`gNEyZ
z+W<M8P<+YMr%B8__OrFSZzp2)1v$Jn21^D|*REXV=WOd)MdIg%JYW$nK%6IF`b`QC
zk8O#?TdKW>fFW^OYHY(9{PK;cs9<aG!=p4nya-;NF0QZPiS2T8HmxV_zW=(qY6^vU
zoIA)_AU8U*Ea9kO(Ut=tzBWK5epoDphy-b+=g*&a0FY<^lECS^cU$K7tAIyKyMs*z
z-l)!-M6QCQPclgQ{-h^Jz?=HaI8nzJ9PZNo@xeSze{NYB?$|La8`~eJ&RRbIn@JxB
z{aTYG@=5gNCOT5*sJJ%Kcxh)P4Q#mrJ<QRk)DxWfH`T9e^7BIo)yel?j_&e&$b}C#
zJtDSy>LT*tnMKOV0v#~&Aj@pcpDvVD-VpU9ziCp46MgTO7U7Hw$cHO(HF?biN5<^E
z=bzEk3%Oz+$}1>%16uYC(rJ*+XTD*zRqNeTon1KLU8NhI5z72(2g-}?Y!yhReOIS&
zOQYdNW`R`)zLXXeA?W+zyI1{IkN1e3EXw}$iude#M56xVNAO1MUD-1{yOEtJ=*87U
z64#=~dZl;w-QKNTRoL)Xh5)bjfcIo_PP7%W%Z`-Gg$A56HJOg5Tt8yj85b6-{ISok
zBd`=(bh0u%494!0)p!ZNamVmDIkT`T#e|id?K><*-eOC|3Nrn4n4z`vSmPj`fi*A(
zZFlAciqs5WNXr$7wV0US3E$U>NE)Q4wV(<@^^Eyso|9<3*!2A0B8ue`<lbQlt#>Jt
zYnwiFhpD~YYgZOYsLk<Y345H1=90C6QqQmv-_@7nB3v{$t#CL@F5`*pS^udeDkzQG
zZt2(JsppL)X2PW5njG5X63$-_0(2Wr<y5bXogFRR-Xjaw4`pxgZ0_oP@Z2K?#RIZK
zH1AH{GfW;k8E8{8_B;)VusB+7e1_-cFQ96hJ{k7GWhLJ9K%YjaHD#j6=5=|gBA0lD
zURb9M>br~fdu78Ht;l)I?bK7@GQ@WM>JxlDZieJgK~%FyH@Anke89ySOk2G}Rsq&&
zZ^Y#7W+#5WmM2vfF}h>oz7&8s=hqt54K=0F%5FmX`Bdfh9ng&PuQI)U0B1fjNq_5A
z4qd=a)z4T)G|Rl6bSUwi0SFqTW(z)leio>oQ1%0uc>o(#n1~jK1n^7VrDrxuc1qf=
zwwgTd`nev$?P`$1W7fy*wEWl}2dcO#Xz_v7O##pH`c#IODAGD+EuJkhRR<mZ!vk~8
zz1D<cx*j*keoh=Dh(QWaJy0j1>;$}TcLni;)!2@S3KBK?dtGKuPy>_xieRwa<l5UG
zMC{5Lf-2NcZnGWbM3Hi*QDSGsZ`62aCMDHg1ZpPO$Ez0m4tq=FN?FBC{Q6ZC!vF=x
zZdQk0;`?U+8d3K<*P56Y^Qj~O@_6&WxL(s)R>nD(gkKJ?I}cX6>(mSMGN-VyuxizB
zv!~qNM*}P~l_sHrZ+6{4Vh9*bRcaY?u`5t35VzBuF3Bur0BQ`(`eT2F5kPy<^%`U+
zXYYbSsc$Lma0`1hPzI6ve<_1VF#IA2TlJxPQ&3bJuCP?{HqFK%YVG@<##|dIFA_aw
z`Q#<T{?OUlP{O=Zy)bDD%_a+|f&yAlg^R$b9h<w~6E!$u`{(^jS)<O8{~Qb0!^hIK
z81G#m)ngRnvs%8mkwN-1!O17}Ie!|fAHykl!IZ2a?oWQG0#xt^rHP)>>CY%@OyH`8
zPI7w2eJWsg-MrNEPgF<|xQMa~XY4wq(4`q+zMF2J!jd8=9pSTnMJ^9q9A^JEvv_DJ
zuX4vR{JW`%DRA1$(v7e75@-`x{t5SR?zP^<JFeG)xh2G-&D|5{@;SdSy#I#f?{X=t
zCdZU#@omV{gr%IsJc&IfA7Z<XOBRuiuJgpJ{0(lzCjU?FgE@LoN$2&vWRE%FxniT?
z@74B3o8?B%df1y1UTmG@r#;n*O)FGa%NF%bqknF%8WEw0Z{d3os<u;eM_txg`Za~V
zSY9T+q+!=_MVo9-waj;ies>YT{w;hO!R5SvWU2L_J853AnNE9Gy)US-F!aekWmSaR
zkrUbduqeUD>`Z@E#M{ZQX9+i2&kl(kQW3tZhdRRbb@r}@x=GM!4tFd@{C>pajFq(3
z_EVxpeE@}(P#F*?2jcHPzIj{Nl-79>yOyX}C^NZNYi~A(w|w?(?TyWBc6f0Y?|*Jv
zt-Xs65qM8M{*xn7%oEm!Yy}K*_wTM&u?Dr9+xOt8A4+MLQbZ`;zR0|-o4sAW+Zwcg
zY9{`F71Q395kLq3e-zz<?|n^AK*Zgi{KNy)Jtr&<@x$SudW|V~vOJ49$i(<<3F66B
zJ=L2cghRwBDI}ke@Yi=WzYk4uaq8lYW`HgGPzTz(h%1T;n?$}ch$Hh|!*CLZpSIka
zF%ggZONc8@{O?gdko7kWg}t>*mQI4bh)=?yFru>P`e@4dL|9UK_LxG0YUJ9`l+w@J
z3G;!LaEJqbN$jKBN<B!mPlWN~D!Q;=MoE~8be@QyMqT_0%6smQ@`xp^N^gGkXXq7S
zkqkk)2~^p4`+e^6EyB94IqF^HWE<nR+~$9IgX@ud*dM*WinU>V$uMC+iNq`iD5jVc
zv#G+Qd#WpCTpUis@%i|wM+SeSVJaC=&T+p7!*9WM_gGP(4{4QkyDyB_&Nv%{;g<LD
zs^A#Q$`b#L8KJm`sRc%psChpmwRmvdo3>VIxA97uPh0~hFS&Oa8R3Q$>2_@P;Y}#a
zC)dU)YNMg9jzijCg|^4+USzdBRfZmN$<rpUrx;@F>YlR?VBVs}x5|6yf3xZP$<ey&
z?$d<!qGZABwk;pFX9%RL8STZ@++rhUB1{D}jaiRWL3hUDEH4ry_rEd9S7RBh4>@t<
zuMDR0n~ff{BQAc5ddW5O5Kr4SFb?LMJk6F7pM*)8cK{x#o|FjrZs*9?Vx0HSXiz#<
z{8!nd({puUg)^tD8Y{ZD8;ug^y(RmrY>6NpO}aWa&24b>`g*2a>oBn5Cd*oplS$)Y
zMt)nX(dk*e+&k@?BfP`mh!Ae3yI_HS1>c_-0d<2yZ7)<xWZH+P#+{8MBD&l8W5<V0
zk3N3vpoek$+C2b8k9Du~KeBxWI}Fvqb;_HwgL*XV5dws{c)I5XX_fM7m5wzLTIc+1
zOgri!1yIkhiRN-8ak8J#6-4M-Ut}<lC9YS5hrjJyzkd5g*Qh__ZBDr5pAS1tyCUXD
zW~YmxXH+lj6kBpHR8?B&`p+JH)XxVh3=AZ4#1lrj{Dg6m%qY2hZwbC+3!<neGGK7d
zY-R>Gcs-g+`D~||?t4TbszN~vMnHBy<^C5?h+o|HG}mInXP)5tq|aC18DY12?~`<V
zy?vYi=F|D}uax?APN@4rMKXLL9W@4vIe<!F&{Wj=_KKi3F8dp*qDbzmzIqOg=R9f8
zCa0&B`Pnf;KpnmwQP7jQx3_ip!7$U(C%ONcfr)N;j=SICM<K&*5nk-ip4#x-xkn}a
zT!#STBfsh8oH#g|fMbA_Q7n#`Hu4|)@lP!gWCS2x_~-M#+5z#u^F640BEfJN36&e?
zK0ge0jvPH++g_>_R)h#^h*Ya`_0Hb9ISHIu$D{k~4{G*r*JOrNe+sBp`C94TFT;Kq
z0aJ5%)a2gj-k>v}BBqvWFnQg+)6`hrg1xkK+>nEjY=Qm|X*y(i=~$d*s&|;_{d;bx
z2obVNF4Oh0vmV?r@n|LWp;Mg8s>SwQM1KnHG5Y_y9HG+10{iU2I=kOUSHHHKcji&0
z%*RD(EQVKR|MbN+zs+3d5L1itB*>1NR<R!Hyw3{}4l^OB`Vw=MxMP~0ZUMD7K&y%5
zv{_&V3OjIB9ADcqN}GshBM?e!Pk7f(h^t&s+V_S%Xill?v`^0x^B64yY<b__IecCV
z?w0M*pvQo(UeQw}Tk6tZKh-s~$H<4YYkUkicK-aadF@bGC=d)2ZY)<~BDzNw^IYqN
zoVN|;PXrsKsId?_?(ZRo52QG|OXQRMhvI7YF$)xUd2gy@s(pRwbkfYebHfQ`v=oWF
z%55yi?iUf3!}ci9MNvV20~%wZ`@+Unc=<?EN;EMGP1hkY9_G?p-zT7UxTd0AUkp$#
zVk|q}oWGZ9gQGjPo$+8x={@7MSZh=r<)`G*hzn9)K)%nH*wMkup*;D~(c%t9>_Wy`
ztZ>9yta$46OVZiK39~rL7kH`>v-63R1|JlQY)Tvp1n98kTPAJ7f7PQQ{k0)~!Ktc~
z=P=iM>ehN&=Y}GjodT=E+__VUoV$zo#_^ZDPkOI>%Ve*|zrV(wIm?sOmg-d{BN_U2
z1j-}64;`T1A@0)+J^t9x4_YVv7dF)W?l#sA7Kalb#NpxOjepxMoqBt%cH|f&5-~M3
zd$x&j`ziZSv6OEzCmPf!%@knr>{Pf4(J*0X1%uQdsM~Rg-@lt~m%^xI_Kz*-q#wPA
zE7C0S!KcAL<%%L2z=_iT;HH!1#4W=g?eRQV-hDoYD8WTS((V~?(N0Y&80O-(W!N}X
zp#S-dqA%$0j*ga|n!ha(3sL8xGCK}?lAQ#yl!Wo}%8F0G0@{>wr+bxn%02(V-x`2C
z-YZfSWx@;q@rJe+Up0um=k9+(cAD45-wdj{FFWZ0Vnwb;=*!w>rHYL!7cbl;;v}$0
zv2GVG(Rnny?ngo9_=)M%9H)^oh4NBr;L7>c2afqm`TJGz`l!X?jh0>_HDMo6Dg`%=
z>e|b5#fcJ|>O0~??m7k*`V_bASMx`=T}o}Ydm-Okdfme<oib*gWk~zj#PL**EsOTS
z@+wO^BdEe)mr)mg@`WLmoU4P4!)!ca<z&gzDETk_XX2m}9lGZkE|G6oFDRJMUVP&J
z8>5o@jG)K#dc)0Zws6K-7C6yEW9#5p@>{*+m9$Y!QWajIEY5MMa!tn6Pjb_lN*&Y;
z!;*aitKzJ4)^eKtYyDbzk|fcj!)ttX7rw|t?$szT8ZUvjzFdXj{Bgs$+z11g+Pch<
zVP^ITTUt+@z4%ynE&CViH(n-$cY5FISZsLte^xyCw2B~0>7O$mdlMVD-Sncof2E?L
zh(S&R2TLUsw;yk%>B8~&YYrmai=9aW$1WUOO)7Y&qA>`Y?8i}nEE|7*7g7BXh7s+z
z^XgPQs@t0foleiGalb1Ej13LlAxTA-8-D)a?y88r051Ggp^G=HJKv7n?P;>Sa*2(x
zx6QqsOyN5Tta6`u*gW&AbEH36JU5r_LBbBL-AXw6I1r3JS2)AREkW!VDy=c`wn*^B
z%R@ugOfl7d3uTT+a3yik?5XRM$4p!2N8(NcfzgrDjyjE9fc536u$J|!%;DW6jxx+&
z6$WvXiHvW{?I*{7pma1B$cmaI;SH%;wiHObd`JQz7N#vz5sf1Wnpp!kN@)jRiuVgx
zO;Ts~DbxHdx7c{ZpWv~s9BXBm8xTL8dK+;kS>C^=hfBuT*!;Cdqam}X+kDK89ftUg
zw=5g>`Rj}NKtvOavjq2d{m=uS>?s>wV=MCX`RkmhSa~hn8%O%}yV1&BB~$No39!`|
zSy^A`5X7m!P=6w=qs3Ec{|+1Bd`YT4xwL0815=hU_@Z+Q!xIck9we-%>LeCcv6eo=
zs$`GCwMha0UA_;^sO}v2qF>2l8Xh5S(o>+O{{FKLGLN`vcDJN+T<`$%$NhQ(jqu+g
zv)nk6ui$sG$w{2h4k(0N#-8sJhauDd6fkQs6eO#Wh(o>lU_q~up;+_fVS)-pBOT5|
zj*pJgniwB|LxfN=(G&aFq0Dx!6>8tTz9}{(ZZqdUdFZ%!{7r;J<q8wWwxo?|5Bk_q
zvG{HV@6(YMsmxbZ5o2E0HYWF~C(+Z^_G>ltx=YL^4z%LQ3{^M(_1m`5UU~zYMq5a)
zUpt7~+IkbvE_|bwQ<=+=MXRHkQ~jxF`5SmG!Bw&LS%^#{o>Dr&P2WM-jcb%j<njO9
z)w3{ZQ(0^-4g-aicdx{a8}fvI$~%9?GQ){}MIz6^SQKnSF<q`O=x(Dr!_RD=TwAC!
zlgW_affV<*?BQt%`$AFCm(qm}dsn(Kyg(0w6hwct@!+8mtUhBuzt<@KS#ixIgI)hN
zVN0(<-3Zew4eD@LmI?2(DdveUB{s7<Yuz57Y?OAqKpPX}Q{}ooi@#QvDB!YvO=aVe
zXiW9R@#*21H8#)Wa5Tk1)EFgN?>3y#qkiZ6W@DobhTRmsnGwy_x6Ti9Ka&4aQ#D>>
zD?H?QIEX%HrE=-hpi1zbj6AVApMt{6OIU(Q`Z-mdVh<yAF+r_fprj=wL6os25nB27
zWdnp`ZnsF4XKEw$v#`>;&rT{)0%cFNgjf`rCzxn6VtO%LP~`3xBU!P_c<b_@Dq*q4
zy&|FJTw9&fYMMJ1ue7(D8(_M->p<${o4wUQ$gLv9EyzJUB*23cI*OHiHHKMl#QeP)
z*PSe6Hr_iBLy*~_9`^Uh=#gh9(Ij_ebWB4NEGpIX@g`3|NXLQWPW9J5L9U3^{SeqY
zZ7TF<3jf|9<D9yIZHT9&PLB}&G&KvEg?++P6`Sa=+9+x!R9<y!0yV0%jxkV<ryMG&
zo&*}kpG}&LT%uo}g?`X?p8j2ig@lW$+|8t=spy<wR5xEwyPCF*K655`l#@cw?{Mue
z`1ibg541{59ak#5MuqxL9Utfien7o>NS(7KjJHYFmYm{UW=x`BwT(14@-^_dVylrZ
zVR(=PHb=vRaZRgA4jVzR>$TzzL>q6fyNYES$$i%>^!j&i&F9c0`om1(-So-0^oI6l
z6H}81=w5E%l(Lz**$^gJqmA)*9HfX2KN&9%U-70WS1{B2KgfH_s3@cF@0$jt5$O^T
zkdjb31q7r-kWOi7=`IDNQBshQ?jE{CI);!OIwgk?hUPir?|1+2H}`th^WuK@TuatM
zU}moC#6ElPv(M*C%`shf_)<lDHw~sROW>yzDbfPX$M8rdC9mC`47Fw+w>WIibI#uK
z@{g?<<JA!mIPs3zeV59TWE!WOSLA5d!(&okERp7`HgQ-YEE1#SXU6|TROJ~-cJU9q
z!RqVbKi(;6f3yhE_8GDq6h)uG2uAL#;B9Mrh4*h`11vYET45eF+x_|EuSj0plMc^P
zxQBE6OCD2NR#PefM?pg~t~eOqj|f$olOen?oDmIeJS5>G5n3+7+^5}s^ZK^I0%0j=
zAw%N+6LI7HxY+~#U34qsiN}b?NP0gOT2+>ev%7D2M#YkMh-eLkONC=pL3lGA#6q3y
zJ`uj_5(M7*tkK27S}ET~t=S|?h%}4&&$7;x`(UF*T-+HRSN}7`X><mX$|r=FC(!0u
z*eiDAaUx9QVdR@Q+rv#wa`{IjrZ@R4I3mC6J*qrq#2@vJ7-&15E(8xQ-Hx0nG|-hD
zF|%Y~Ko3`*)M0i8Zl#vFm&LMvxn3(GzxU}ysZsss*OgZ#6V+T=rL!G5&=zIBRK`SA
zm+LSq;(KA9wzKaSb*oqc&V=^8ha^nsv9z`S5;N<F9bK2Wx^|yzsOG*I5{39^-X=lZ
zGdypf2cxnYt6?7qekPW|idW9_+CHuKyv&={t#U^{l8tEgchZ&Ct;aK-6K&FIFsFRl
z0!i5@nVy&$lPhKWx$c~EJ{okz>$3ED29ikl6hcOT(lR^KxqE8QS!iqKXBF%@$Fw(t
z_vC(+vD-a<>_nbAKFDxSeSauA(7>4UvNU|)_^fs!MS-~1?!6Kd4R1?u$op4C5g7cf
z@|RB#w~m5tJOM5o6?P70W>#p>j>A=+Fnq4HxK4{G&X{Ej>Qg_gm$PxEGV;NK)S{j?
z8n_&K&DM%dI#O~%9SzlN*o53>ALODQ$vv{40d9LBFl4!sVo=hL=qPdozOL6qB3xn5
zRrX7Z93L%?p=L}kAS9>X&j0WcLwo9Ty8!8^dcq#f>+qW>B1C|{Bodc81}3Dm_<Ovk
zh}=Nl%LdPkf$H8?ri&e@=P@3-{^W={IzQJLQ2;l!R(TlUy>8>sa?|F^)SAOTV(i$Q
zlq2bCl6lg&cU52YZus(g2K7{hE5$->Gt;=yohkET?s<YGw!+?C#7S>vK=QZZ`)!Cw
zpN4(HfSj_NueG~lv-3S=(?&D$QYOc(=xshH3-fmfRvUQKS9e3;1X0LzI2RUiZ(T!g
z2)8oaNn*?pMyf0mQsuD)=L<0EhPB~SP6s=vmfwmF5gAWgOZWsU;l|0=Fy0?R^HvHb
zx~Zcp5W#8Z3EL8##qmVL(AjQl{cH2i_Y%VXJoQ7V$_=p}?A7`pUY-`>+O4%wnE3k*
zM(#tQd)}@mXpv?w-#VW@S`(_wuZ6}jOqZ;so(|8qm^>$}C}lnH_A|OuC#@9Db|>0W
zqwY4989XjQo^wG%9ueEGw2ypFEpT$~QZYXJkTp@2-*yU*T{KPD+K$UMQl_*boEU+Z
zWG{0@@==yOI*ry2TE&SLBgahftEvy{9^Jzgh8(4w1YK@;TTK+XG1uCL9Na{oC)*u^
zUi3OkVld%VGxMThfBrEE&Zk03UJU5TarE5O839QeH6P(On1Tg1&t3$pjlF$WzfthA
zd(R2<G_S5TB!BAc0L#Wjzw^XqnrU|YrH1=!*?JCX_V1g=^q?2o1Hx-}N$z9up(=H3
zyGXs@jEq}z>U=J_1@XCiscq(xPfUj+^-Ni@G~AQ1|9FzRVGp}KH@h*B1A1prVT1v}
z1SZ1R>-{}MdkwL{(~0+Ic;?M}Rxx~bQD!F&h8#;ad&@<2D)XmfMN#ZAfA7=S;kKQO
ze<%;|^4b*V#I>XTV&fz(1|9vhA{aR}p7hF!3U;I0$QfxR`0Ql4dbqaM-C}Wi;G_IU
z^uSJ~2C-^8{zz)k^Q~Sho<_zgD`9#ANa|DZ3o?A#&qG!o)f+>YR1i$*uo3fbS4)c>
zUw6MXtQ9`-^&MicU;povF_-^-QbvEY7qA4*vFX7?Jl7X9j}CEh3;k_{HC_@}EUC+U
z?vCJzL01jTwF1_wynnuD6vBxHk(+ULI%N3X^Qu5sf4a{la?#vH1J^8!aI+#-d=MId
z&_;_1S(~}O>=_hZDU|U^EMOzqjQc9_%7*imm00I$UdeGQjTn^Y{ftQujTk<ZVos!q
zZQu;#WL_jBCJ_ytGT>mItUdgjS1R;U&c@k;5Urx}fnP_}z2wAo$Ep<0h~r-;6;3dE
zx}FoU@p5_gL>3~ath_R~0AH%o1BZ6FX_JVEu+tP?F~`86SXXEh7Fm1qm*NP%@p2`7
zAJ*xcCAg6J9BIsV7pGhC4_2e?f7kWr@20#_(&IwXK0JB)nJ2q6+`5D1033+l|55&2
zm4Sx&yYdPAVVdxbacF@W(12?Tw(q1*c38b6Pqs`lWs~ZFDD-GV(Hp!{QOD`6wpPvc
zGLzvCFshDsVxFm}Ig*C0BH(t+vvYE#E#ZFbX0kMCkc7dXmjVV*q6iGdCb|-;IpLcs
zL9dGdLT<cDPS3jXXs84vynKWigC;nhp~(DOf7d`!Oejy!>!rli{Y%KuRi5~co`~?i
zair&N?Q9eqfh`mHe-|Uz$yvnn>ovF<G5b?g&Xy6O=MVE-1t3*T{HjVmtdUx!!-AtK
zkVi(O_g91jV(dZGJ8qK<DHkf=wV=)pl^w_{YW-@i&&r=4!<zBuM-7K60};c1lkZ(J
z;*65$n=gID|EdMhOiDgXHZ}ql%F}k6Jw{@U<kdU%J^_fN>$3m#I?VWShQcak?7;CC
z@i*GpCV`S-*UaU+>LiiXOSx6QHMNQEh3V8+wLz888ThfJt+a#rCA`G8{LjquW9XxI
zn}&=lKhodq_-A-s;J=e29#D$3v9aCS6}<kzz)%?|`E+UMCUWY<=p#}m1@F*DdFpL$
zqZF!zw6brR6*L&T&1|U}SK2vr=6)-(kuq$1Ut4us0%h-q<WLI3*X0#rt;{ElKT+}G
zl$&TI{c(&lc_VBlA7j`@J}h%~@N?m+Z)>_QDC+PlraI7^QN`^ouQnZD($^lmAAB_C
zhVsG2?R6Q{#`=l(NNA|1A?NTNZ#sjvRE?e~(iZ4ersr4CajHTSGLKU~2z#v(aSLZ9
zfnIOOSQ*YIXuBJDb=%ewW8K$@J6G$$+_2v0(SS&~sm+f3U$*nJ9MfykJZ7zr7P_t?
z1h|Wykurpf$(x>E>Zk2;^U4~1u2Es3fhrx&$2q@zX7z6yZ3HK7sE-#WIZ6g7M5~f)
zy0DLySTplVYg1k`NEw>(=Y&rE)n$P}DBl>~liEK#7{uj^wrAtw9=L;j<;>#9$f_0U
z@yZnc%zw9~(-TdHCB^=IB=&jbW9W&{UZ7v&33l?sD1_4xF*dLLfDnQ3hJlF48cIUq
zFP-ZtzT4(QdxH6ZH@9`h0cVCd=08!<mnjc~-m|;dmYh^PArMJ_Zb6~J@ova}*+Lm|
z^!+~8`B~)PeYD_!^7mQgRUxlA(_3XwqRPULde_bh3VXlf=D#IBi4^{-J3+Pu=h(qH
zGowL6H|csBlm4^Cd127|ofq*my$Bt&17U2W%HzX|{*-jRI5~+N14FIM_T9?2muOW^
z`GTRm%Cwq4{d)J*`nFX$2<Y`jhRhHJf0OU>OT^>psn-ba46~7L11XX!1FC3bu$;TK
zRk!j*Z>4OUqpX>Kxk+B(vqHtcxP5;zfG%(kiVj1C<uheu@y9b~D6e>@?*2wsW#M6q
z_!utt6T_Z*(34_1ub6G%eCN&7Q{<i%7Yo(BiSen<+^U?0onZcY{kF4_@?hH|E>I%g
z{?yH)I{N16HT=lHWlR?nYQOs?^lXINp<z9;6)OaTg0Jw4i|UP#;Pooq&uoSsDRspt
zNr_`u2?m|%c$4n@Bww?|=b@=G1NVOhWmw`rSz9y1J{mn10hXD~<N-YVWIoiAfLf2B
zV?k+3hB<}4sY9jNfZx_YT4;2F%Y}vCdpm!q?oPCiKh&>S3Hcsvf<9uBzJ70S$HQKC
zY(~39V^&<W3r@Wdife!NQ>F3x{O~j8ij=B)$L1>fd#?}nOhJ26-_k_CKKrGiDHl99
z15RzCf<$!RnZea*&CF|U=mOzw<W9t!DZ9e8TkJbUU7IIYaDF;b(jJTuBE0cbysrvA
z%oE%oCD<1&ZSsIXgcK7U0|lmJR-j?G*pM0^cE|YL(Af_5gM+T1_(#-T^CcR~@0gh#
zt{ZOwzN`0z@M5fh0NteF)8zDp_NV@=9V^)I_Q@?tgk<mWclsqcp#+;>FJDL-jV$J&
zE>}(B6S*F5oQic7u;7gZgtonhM=8o&?2fCLSQX%6UThVI`br%Rg<7JCqvKAm{r)Ti
z(d$uvOgs?r(=~UVwr!~66t6&)YTQcLB{r5uz4L)Bl`|)YhU(xuX+P14YL8Y=^S6GM
zQDFy|9lbBL?EMTgp2y^$|G6F#Gh-tVnd|#c<KT%w<Zeft{3|Wci?_b-`>Nlz-QTG>
zh9BtHZczPcZ`gdXL#^+1-cMf0{?x)ngPP0!`!j*(W)|88gAg{Z81*+aIcy|9?_uzn
z5#<^*%|CP7H>`!!PdrrnlKL|aWM$zIW}>Q_)gJ?g5l^p3<VU_`*u!;wJh>L%`5j<G
zPi7p)DbQkSy6Qo&+LaP71G)HS`uuH!IJQD9=al37K1qZ5->Bl${TZg-5Y*N>#yw+0
zyz+i}HyubfY?$_9Yr3xcp3uVDtC^L3`iwmd%C&A4E+J7pmyDIHZ`BfmsXo3=DwMHh
z4u9G(%{olwY_%mA{#Zvo;Jm-F7(;9^X3xDHXu-=?yomN6-@}*TfYM2234c1Mj~}z%
zK`G7d!u~En(%xGjv_EO-Jm(sVWbuH@wmgXLKGww~f_r7%rENH}qFvEVj<QJe+h$C;
zvb1u(RGB@k5G2|RHj7pLi5DxR(~Y0$3L&=ZC`vG9jbVmo_?cNd<4#yrbUVkVlBDdr
zcQJgjpU3TmeK~%6NaW~iLv_f)^sK(Y%Xo$S<5<x*2T4T)-JSOIs^%F_4Ln4=7mvqI
zD?cRTQn2CM-yP<Pr8i={3)>y0cqKF9zR3StM;UijzwvoqCr>V0O9|Olx$$7B9nM~u
zVdS>a7GKw=nHh;EE%vaB8!c{Egi@=|IcpXfGG464P7oq9l!ggn<J{Ajak>W+a35u{
zzF2#}9R0`+VfPVLI?btr{$3b1si;qR>13kC^^Q;Sq2!idSYFALy4JWKZijEGScny$
zw3LKUWyjB-)%L+L6D8V#7P_?%kHbW0#!#N|rE6PIwUr2=ow!N=v!7-Dt;UL_0q9J6
zI?tZfAG|&vt(j@m7qq0h8UfF2ujr8mpZ=zdxU}mQ?@`1FMO#?7<+4_^3or=Fmn{w$
z9IA#~wQ(udPcw#-8N7LYZmYpqr%}>Lr#m@Lmk~=6JhI20tej`F`-eG^;Nr(0&_kYl
z_oU?K%y$-8mlSjH%hir;h}ESS7=1dXdy~djs5moaF$^>0)nhw5Vu~Y(DhHaI(FvW(
zd1;I&`z5&v=X`zMU=nB2{G!I&@DUnHdFIYsZ{YGMvvpbTn;z`*%bzJ3;%IibPBUXA
zO|-E5wxUmJnVzJQms8r+_77?Xxw-z#p2PTtD(xaY8E55TZ2WgBZqcH|Z~nsnEJBdc
zbsWBSi;=}MEP>yXTg_Knw?e*sS=U|nHsn%oPa99CAf7c<-(#bT=ijd(%JO4^CyKEs
zrgMC5&yN`wZFZPNljvDeSysF;?3<@&b)Xb>9Q;uK?leeuiJQHzo#tCOFHSY;m=bjw
z?h6HjSl&EwZYCUTA5bMR_)Qddpf=A#zsb|}t+?u$46}F_YwsivC)r{dEE-d14CUdY
zfRC{~QqP)49+JyZ3cC`m#HpbLyMBupA9~Ct^OGR$2kO#a25a28kJsbqYRKiQ@4%z;
z56Rubre5ju^Z_Cl_7@Ty0f-6<)U=Y`<V(o4&j2y@J~VnsDD0&C`U&V76Q0*ZBztVV
zRYkUTD>-4q>&espty$R!BZ;=J&F=n@S&vBWJ$$x=9y6@3r0u*aZwVq%PENegz%`KL
z0sAs}KS~2&>#CaQ{hHsbjEwqLu=myG+)nWI&G<4m4kX22G1DB3y*x{)f_}VSi;XD_
z)}-QyyYPkBJKv3~7zp`k-cAL~HeZ^Wnd%<G9#bDD6X~GzKkJP?Xxr^v=^F4Vsmf`N
zr>(r`{!wSZV^8XECS&PfdJh%$&QCsg>I|dPe;&W1itG09fQ36#SO2wY?2ry>Tw}DU
zw9Jxuui78=kuAYzTaHDFb7r^sS_E_kujcq?v^?aAdm-nYB^-@w(RDMkCKJ1_j*3ns
zs$uo1aTZ7QA@6-Bn(s3t-_Qx$zT6~BPg3*NqU`F+PDR=G`O0Cf;?q`iQp@}M<s>N)
zat6Q1w}lv&AI<A^^;L5k8%gOposAzaVb9FXjxTI$u+nrlso?u=^?Ixdxwk{lMIkCO
zE*$)b)V}Xa%^!8t%?l)bpt|}=-4V~Jukfgf2M2k2c8Q6Et}l$CAGVVvWjdbhogB96
z!yi%IgRjs1mh7%#)4pAwzTS`7{Va#OYr5Yf;&5{#^Re3O{!Egbtq<!-j40#Xx4ADk
zUF?ywGpQX_oHT>0-EJykjgZew@|u5l#!lWFQU~ZYeKv`e#tDgM2qWZxwu6HhT@Cw_
zZ)e6#STJyeeBp_SL)6+`;j0IHb6PCS4d;y>*_j^bm0XdS)M*#H5hAKo{A|V}2K;{q
zE(R4JPhixn(R=n(LgRiOQMC!!lR`@iS#?MUFvz9D99t?A<W<vE3#}?}Ib1~hmN<0@
z%Okfe5pUxeWFp)H3T<?GL`XW0#tt6W&;5#EjG_>;Cd8v?+CBSY@i>*p#;Gs=naN6y
zsUOZ9)h8+BFvm%)@rSlwvh-`t31agX)efeIAXq8zOmr*X?Z4D2Uufv1E`jHnio37B
zen|+OC*jUeIG*Q-^nOl;ePCMoh=hRO_A&wS2roFY?NCv`&S818nLf9a8Pe5?V<$S%
zN6Y_?t$xYLS=!^mKfp$Wz$ljZ*1mUh<C9XuZcqT9`Rm7A_O$7qzq7&3(lji|&doK4
zYg%weR@$dcleG_?)?a6=Ns2|9j6O<LKKf}Jd9iIsxYv34Obju=$lbW2H6p&hHrw<O
zxK_T1&F0zPqq2&nID-QQB)ndG3IuZjA3A;JJ!ddfea_v`#!TGjQ6iB6U|MX-FaNjQ
zA8FHacaKzESjoi^1fEzNaog%daAl{iFx=^EnXm-jYWALQQ$j0&A-tg~WrpczGmPb#
z&oi>Ccpw&I8Okp3X6i;n+36_si08wG*wyFmT;NOB7QcHDqoO^xQT069;AWgJ-HvBf
zhpecaS(~1(d|CX~$^bv7I8t|EUXhu!qk3hDV+T1Ngncqs5<a2jipv;kf8j7Pc*VC(
zav!^@Y-sCtEDSLO`T7@6zOD9iL{<PdFODc#Bmw<^8T2-J*YQF;J5KW8HcqwKck@?H
z!+B)pYH<BGo!?2vDm)WO0%Fbz_!^OJ1^E9~iH_OirM}Hj%cZ}})(-YoH@Jka4O4|n
zJm4~FPHXl73>1HR%Q;hDId5EKuKlc$xQC2!dNbX0Z}%nw35CRB)w{4yc4X@J$1e1=
zJP<e3?al;g)Zfl;jOl?lS(;U~h^QqqJYwF+<8^2|_RUwJL}U!`eXzza@+4{I6g3bI
zHRtP|u4w-hxHkHGuBZ5JULK@Gf8VRaD=Nw@;gDvg`N<!uAoL5r{an1=f0>w_<CTq_
zLr!^B>gFX&zA~+vpP3SZ0;{!2$I4+pg?@hl;{J!T#9lWweyTsc^(n{avH9^H$GhvN
zxBLO9Uwfk&6Ww(Wr})62%wi_%cD8F;RRkq$z+%@=`(%hX;~-RNes;m_KE4jL?lbjM
zV$JmWi*Jd?o-IO>`y_s|u1R3^Gikh2T<+FN_tzmR!{H5nTc6e~E|+|#Rhf!-g+^H!
z0#%-l0IlavFZ@KY@z@`)>u}m1nA<3kGf!wH^;)YH)Xz~{4aQ&N6A@q*I&QDl-0A&7
z4S7|BgRKLjvoe*T_6#*)I9c_y`XcjE+hw5Po6$zgO<|GmD!gK1Hj$C!-iy&qtBQ(M
zM~>apH^ey{`_+}%-+Ru4viFuFk47-A2uqfKZs(j0LGJ$+>@Fg_VDcR*FPy!3hg;Ep
zBoI_7$xW|RFhS$K+=jyWUa7i8csn!0{z<HA^cyzD`(U@Tpeipqbd&5ZJ;>3fYj@0|
znV+tpu?Q2!iW-#s3V6Vz&l<i)b3My7%*4P59bf%{uvkp8*myeJa5@)R2-dhUD<jL-
z7L?_?_Rn<`nz>`_T^6RHn=?!V-q4NkhjK#OGfd~ZW>6A>FdQs2A&82S5_XSEc4HYs
zZNu5mnug`MyBu%_I?rCPQRMQ|T5U9`a2nU3*7FmT56(Y`F<&30I`CeO_l+f~s43Bw
zdH|1FGsh-V&RYnE5jv=VyoNdZCktKAjNY~fy*v)Td2Vmu<X$z6i5}YJ-Oe;1_X6G-
zxj}AU&{CUnBH_d55}W?^(7$oxFf9kMp$R=apL^LPLPF%E(5*3Kyre7<?IiAGw`!<}
zBTS<u8oor^s@8e6p8i^Y_%Zg>@yRFKKRbObbiMhF;+^fyJN!4<8r-?!XJuviEL5vY
zB|FVa%x^uzf)n~%B;t2%zMKg;Zc;etHh?n>tWRB89+_-47h<3h9BER$;r<$y;DV4x
zY}ZIg3RHumBi4-5Hy)U_)N_xP19?mBtj#Q3tTu0|GeuK}5^U~FA6}oCU7cMtdZcB2
zzwm7{!Lgj)ea$1K?o4b-kBJd)ryEI2jh0lwP3z+N2^53@tU-Fj_`5gTwT@`PmN0HM
zTXkk-U9X~8Z=zh>a{hYzG!r>Wi^sf67#JIQaAU`?`7g0Ae_rogB_DrwY3t^2Y>|2E
z^~8Z*{9aU$r&zxrR5ThIaNJ<16_Xi&uuUc2)2=80NncM!IX2Q?5H1m=h<kq<xE9GJ
zo+oeE3LGL>jm;PGz4i29ik8&XRX=dLUgDcw{yw_CE(WJ7V`F(5n!cF0Uxu2P)-Rbb
zNU5kid|WkDcxf}ekiEznQo^B-XUK_bFl{zpXlmd-Vs$<@DQVVU{oHP5UP@b@T!(DG
z=k;5ofsb?dd{@orSJSrq+)D<t8EKY-*3Pz?w6zXn6HiNR)JA6#Kgx<MNwJV-iBXx$
zat^68aWt-W@cx#S!Gupt5L&8biIA+;J35+J*plxbk$TeQ;$hgo8vLPL#I`xb#Cnoe
zFL0=5$Pu)cy454KGJE_ZP(_hbnHi?wpqLBZNDpjr1Q;3h4oq>sInvB8g~-iS7k-wT
zX>UrQCQR$dJuV9~-P#ik9(%0r-(vF6@wt0t6~CfLMzAN}gb*H9cqlqv>?dJsCrWKv
zv82zlh?=u?vbD6H%tXgRy;-`aG5cSo@B8>Zoh4f?sB$JH_)iLJmKu*%v!8a_c&Meq
zKl$yhz#Bb-k6Wh=H`x|rzvo7hCH&BqE_b$PJC<9QMn);R!XJe+AT}LO{wm?>fIX98
zuKnC`8#L4~DF!JUfkQyNcUtJSa{0T04Yp0msz2;wfrPOWO&T(3)AzS1-Sw~Ci%tf4
zT5lB4Ehov>zaD&QMGu|QbYAE-W}x^(C+qlbzi&V3BBCNa1Mc;e<L-Q{t8t5UX>%o@
zy(9SX;A~}MM!9FO&a;8_Dkfpe5owxA=&$`Lea=5lH>F3u{>MDGb@}mf`1^Ky%0y~D
zAJGLuavtSt^$5L}eSG!SUUaDMyKej6z%k?$l;POXng1;<qfIgh+Br0ZxA5Z~tz=54
z+#|i-M&;tDji-+ka&enq(Q9f95E7)N&=)hTP=%6yI=|h!m;TI>NUhX<_4}skx78<j
zJS{>)dyg!9Km0U#hicWlA?D!vP(HeF*^y(%%gY6~WNx*Vg+IvFf*F*vFd{Qt@l5Qs
zoq(D2ZwF?<W=YZPq8;uZ&ClHy1c4Al{2<0l?Xfd7!##-+56(1nyXD5k&bm^?jDH<&
zMj&Y|TCsn7(-o$?1-_OnbqO*AQbP(<7I&`RzD@XWb~mzbm4#Xmu=+<LlY`^4^PD36
zM(x)b%SQLyMV16|-t1mV#P{u&ODCrq9A?U8Q5#E<5{vL~(+!%jHB}t_5f89k*Z<HD
zF_J1?n?5TuSbuQ~W4CgUFzvNmJg3@>u9z)CfSLqKYrD4~k#s&OZ%N;QpX?iN!Xss-
zM7>YzEHL@}cts}P8Fos(MO}}+on+$!;j}F2>^$I{F2DY$MGv=fKe>YMZPmOM0#(Ad
z`E+1ob`>}EW!aPSZ6N_GFbhj(D^r*5Hk?WNnVCYB*Qa*5f0QzZZh0+OC6*1{^mFZo
zOnijRS)KYk)VaBC26;hXqKM;L87X)GiEm9PFwuUVOaQY&2wvlR%~Uh1x%4YtCVvb2
zvgV57&k#SBS(GD|^uKaw5a=?#*yL>Z+IniNEheomPY$wKB>klQnh~1pD(dPOy^x}!
zVv~(Ie6VpFG}L#u@;cmFNTHPNm5m}N|7t^?%P`nfMy`FbzqEJK^mHKa#Rs)3thS5Y
zN@1GN{7aT^`Y;e}=W=8)#48iyU{rD{6c}<2GE&`p5c%;52UmV|%g5}HVhZthI1g;)
zd1~_`ZdhaAZh*1z@0p0(J^hZ}w4D#9=`_BxKcic_pw71n*t~-UZ_^Erq+GX4F%NyF
z7f!$qaE@lzMiz1+vwP>=g)1S*4ISOTXZQ7B{Nh-v{mlL55w4)F**mopC%S)Wb9BTg
zX<0}h+awMc{%0ilY*b*+^szjhcObjg|GZECuQ}rDy@3<O0>h+=*^Wz<Q>4W9?==!Y
zjFX!?Jq{tnLL(ScNU>&C_EO{pb=P=hMdp7GAK?{qYUKUTGc5U=V$!h0P<H;}WzwWX
zwpVk|!4Wg^8*m}p?mrju*8g2u$b9d=Yn0&=@^5?|a$IF@d_AV>sBTiSg=uxJp&CwW
zdB|N=FV;2wn@0WgGc&1I-7ttG0Wsf3?R|nFJDB`NeW_8HnSb$}@^`Q!8ct8;-+;A7
zv6LX3@#HzEN>|JE)i}@`c>LWRRl%D@1MWYY=Yyq|1}L}41uteqXrI<lN5UqH1t!Xy
ze-$VguRI3!+<E@Q8tNjq`*$P`pJzN88&gbMi+x$Y<>l(?X46GXxy%_Ds51KWii}~$
zye%vkU^l5A(GXnz*xSxhVoP~q`@5<Noh(LeWUwHr{Q18u(>T(shKI_u-NVs)6<Os&
zbC{G&#_#=$C>XVu9^0(wS>EHSYctkm9=r9|uOcIT)&18fLka0}@#W34=;w9GB<Cw4
z!ehNzF{_);bouj-Lk8QU*3y=LS3*DPp>R{`zcL&@QK`Sl<&6q2e8TY0QQJLsQ3IPk
zmz#P}b9Omrb>_SC9Qtqlm7S9_kbsd6xS);Qt+SjBub}^$!3&9PRMRL>3TusgIvQLF
zUgn)N{#lg9itPBV_d+c0agJ@tJg;dY(VS#0Qa)~F?p})H<)KEu`1es5JXoq?JFkrc
zLj72yOqcGN`Rd>Pu>Po_CN_|~8<S`DDjt=?0?-><GsXiYe`=tA<ctg>p!wx;=NDqU
zcqO=GAhOv+*Gn%PS+hyo`&>z1XD+_yDXpN0L6_l`HgCDYFzUZRgsNRf*1Te!%G2Kv
z`=hVxCoc%=%<*&Ti<2Crx9F?BrztWVo}H>xL01xc;^6XA(zwh<<|N_1J<q?Q1U?Ux
z^Zh1*Cb6|Nv(%4PLO$lsw@q_*5N0oMsJ2x!ke*-;KP(|ERUK%F*(*^W7)`6ruok4@
zy<XMvwbW6jj_V6_bkbkW;l<h8WB3$f_J9EMUf4CSLbkm=M*5JD&?8ZOxPOs5tWYrJ
z-{^86Z<$mu``Ilh&;KO(4xDIZP7tQ>aWZ2SY?;{03^9Q^ohC&1N{z;SxCp42lo@5U
zx`;BBRhI%&1TEN*JAtd%42BVcx)e41PInzWILqRKlcDl=>}zEFtxz9R8&vzqXL>K_
zsr?`4WEqkS)M<36B&zf{zc~Gixs%C6I@<EFP$n?3kOdiM0(q!uBGc!^L31^t=D(`=
z5Ix80guxm0v^89Ot=PU3At+0D-bf7;#+w2raZuxpypTD<|6KnwfieD7*dTuxj9l3M
zUDE%DN&WhNUMeL84uAxd?-CRq3&n(NdFk=+F>Fi?2SFCX@yzEcBF}b3>lgmbZ$DI2
z0CGhB9vltd?ocsjO-N2IAu*R?fAWN~^2%+nx;7B`@O#44f(`(u<Sdog?@W`#sr%y-
z6Z&|XW1FIiO0+jPhVRne=CtzS?MFNXc-s{>AV?5wyLw9{mbA3cm_?Drfh9S9Vc_qz
z*SI2U{Hr`HO;oPIA4Ns?q^cV6gaFd$Ya_sMx{b1oLvOr&e45X8`4|=ecM{2Q7z~6}
zV50?PnASpQk*TL<H_<B2!FPb{fPNYHCYp!RdUqXgIb7pde%v(e$?dKKo=(|hiS|`?
zLlZ;=816pqWE8P1TNL_SN(Pgt&(_ZUfqK0BFl8EroEl-qn}rI|o)qhLYNy#dPI|xB
zwvkwc#@tNx-U;(_s|8Ctt(jM+7;#j~^nzb*ZvSmCH~)UId~AnqmGvQrvLjW`fDh${
z%=5v5o;Ik!F`+rUbI)H1<2^503xeewZF|v`@5cUY&Pax@tXSGEQGm2>{uR~0_e1h!
zFRU|4Z|sx`!_UapF|=1aTaxnlf`&$!b&DgG|DI^OKE$1cp|4g8U+7j5=ZiSi{E_T{
z;D&0R;hS08#jhFtznhJlh&(r2F);2>{Bu+^k&6_b!3`bbVpYe$?MZjl5ZT6%YffFI
zpT@03)3Dxo!S$Z2uG<wF@!34&LFKa2SN3q>+nGiRhLi<i)l9w&rS`wT>kKduIq;?n
zqpKuDv5Muiw0HY0xK!jk%_CGhak^)7LW&%s==$xw^w!E=DG6bYBfNPn!Q3^5<w3cT
zTWMsF{)EP>N6f85YMIurynM_TNQqH?i1MqCRyR1j6ZUVS^(`-)D2TCIO(8ztes$|4
zlt73xE-{x$oyAx=q3!>v&MN}BP_z!A<~`cc$Z+Jwb5!)f@cxL(^n<``pbNu~ga{;!
zD-{s4-zx2Sa~YLwI_Q-;_)|s5<Ab;?mRBFEwCzX}Rdf0C<ppb#wRVA%&flau=)(%j
zx17&|uAIlm!kh<--L-lr+AdW0kMXqC*sNq_uiXBVcu~vOiaS>BWE(=i%9?&Xw-$&<
z=#-<;q565Icp;8gH+Hb#@V0N0aH8a%j=!+Db|3ctss#ual(kz3mom>%qV8IN3&`}z
z<G0&M9rk?Lxc)mG!1zU+Lvp!2`Hc;L!R6)<t_kkW>5iu#Lfw39Vk@H9k!=blJVsQI
z%)kvyW)vkS2lb`=O*3`m44F$YW1E2o=LXm};W%xO&)O)lW#Tl36V`E%Bj>{x(flCS
zxd2S4yJ#JX*wYiJnw{v$(_Y8H4z$5B?XbSUt(nw#Yd^+m(I?A7vOgbz>^V*@NL?-|
z5yJ+4H#>%^Rg#`XAbsZTv$dt`Pch{U0WS?=SR1eBH9pJnTP<XIdp&{<8HewmuSKX$
zpPu*ap$pk<wxQJ4x;ahIA}%(iW+-I5Y<P{uZywBfBK-?wQhsrLtKBPZ7=O5Jdt`QM
zZsfu$;|n1UNccPgo4SaL8bNeJzC_@!A#iZ!8qYSHpFDYTCU+rjR!A$M^B>~|?r0X4
zJI-CCNj3Ns9h&#%ZwqVAmRIhZ+wC8juNpnYxj*w@DdZ_$tn;Bm`E$q11!$1A5j074
zJ*Xh~^-kR`^Dl!!0jNHF^r91`TTVS#8vFn!|K2HE30OeP<VeZo{M7xS{J(xQoz?*t
zsc&&W>*0T$rGG)hKU@2M#Uvv=>Hl1yasG4R{BH}7Q+OZQZfA5_XJ7+Y-V1+OC-tWu
znNUo$+0`0Nk`+v6wi=eyzOzur$qCTx|LXl@&EXb!L>|uKYiWEQGpW_?u8*x}=cSZ{
z_F{tz#=$-qN~Q3M#m>UhvV#Qq0gC^w$g;BLjvG2ofBN3NLcbNlbqtLAZu3^Hkanb>
z{*J9PP?A6$2)AsueMJ#+c1p<WmR>SwhpkYthr|d_{C91tuD?G^28GhC4p!6q{OY6u
zUu~RMTvwX!Hy#i#X-F{~qgujJcg5@nk5#<3ox5`_bY~=*v?jNm%XX3mI|w-k2f5LS
z;|-zwv%BYhy6$Uf@HhW@ZM@(zpKOQf8(+?g3uz_ljH6>hQIWN0lY-Kttj*Qt5iuj4
zx7oYi%sbSx3Cfq7e_Gk7Cc$dk{K@{Jaqm9%KOg;k9z-M{yEwI5dL}gq+)6fzAxSz(
zZy|YFf1;p3$?|OFJj0$Cnr{I#(H~RAw}>LlZmnH?dU(v^nnsbH^<EjNN5r1^CThTt
zIFBd~-w}v4#OR|x*hlroNhe-79N5zW2|m{gYi&!m&kr}yQKwI5G?&6HE(&06Ycsz!
z6CLjc^LSjp;a<z66)LpTVu`^rZ<_iXY@d-HOhd8!^Yn;kuL`k)`izE7TI`=e@Il1x
zi%5;)1rZi+YLB~OX`e@CryGc6epYdTgNA4J7xgdKNW=FQ{j!45W+!Tm$&usn@&$by
z=iu@1V`BX^p_qZEjPx^pS^r|2c=6Kg?4Zz4thXQcoWW7=VvP6&li6yE3p!0kpEq)H
zpV+05>%@5iD!`D_6=}Yz<Bc;Px!k|h2tY5bBiiGQW2V}DX`Z}44PePbLG}Ici~~HZ
zPYmc0XS+xf^($cSpzC=gfXlpke%#5sOfbbcxHB6*)?LEbY|?%U2t)5PS<pX~KY#FV
zoiGr&;?F)P0890Ik3EkE=;!=lKBZ}l<C5iSkfVj8#=0ennLz_XQaPl}K&hu_cx60?
zelB$<(+f4d_V3*vFRy{%d-%cMWO*|9&Cor1OXR5?X>49liq(e8jBR}g>!eJ+_DnXi
z1p6R6-|r6BGw+Jlexok<d#X$_RC~N3=O&I9-`LK_RrnRq5}CGoSY<kw9NBlZW=!2;
z=r2Zj-?%%BqrGI{A@S@<!Vr)Ecrum}*&OLJyz_yC*vmMU89mgHmgpl>1_<BQ{pQL_
zz(hTZYdST38yvp(wn?OP;V|RT+tTp9D;XvwP{8!>`X4v#f18f~_xTTO)_<=5uO8n2
z58p2SyOky;h*=4djU^ICMog)2^?aX0PIi$MX-Ln~8EIuB`;2;K@vnZods~9+5irjC
zGHQ6PCSaIw3l!X$-1iUT5o2HW+N`xdVl?`#*%oX4r)1ElzOgO<lMZ^|vt+AaViAy0
z6oLFp^7<Lpea^27f^8_*s*~*h?5;Wd6p`4tq>4?oxOOW@&OP5ZKNoO}NK9hakzEYC
z`PMx@XB7mut8$#9BpXFd?*Ut+OWH;La>zw7d~aiO$^m1<n-D(gmt^m-pC^9H!dP99
zy63-7wWtp-oR`=e94a%+UikLUJCAP_r}%Rq$5VgyXnAvw|3*CJKw=Q<@bFb+T*)Q4
zLEHAE-Ok#Y)|?jb2CAD=w{(4b99vVx|NS&yDT4g1DS!NPM|a7VoPq)GOp6*G8gW%O
z4n{a3$5Nz;mACwcw=5S{TRJ{A=O^_4Bn>1OuqRN#J+F^6e8>7VTh-b6!D`k&uf~p%
z&xFRf*#cgec{XEZ?-85gw_r0=rgCz49~U-W2f7B?4H9hF6HF)^9zFZC%Po8SFcG#X
zT>)ashWkG9hA1sVXFl>9XdiJ?^{>2HxGV8`qqY2i`V&|iFFI(IxBT02&$$+?#xKWD
zIMW8ZILdtMjfmreI$SO%L<a`xLT*zV>r#(&elxxLXYBj1#hCl(#IW0NT9lTo=09Y2
zVlG(C^3YVE+YnPqsnXnA@ObV?E+lw0lyp^V5L;N+kvi)nCH~_-MQ^^Nx~cpQF7nKQ
z(9PQhIiWbw!LypZpxTk@&wU-wJd~k}>h1aPMfFC%dCH+O`{6dD|NTnBe0g;DgD9Bq
z@)x>JsRK?Jm1~lUtZ$8R$-dWa*Hbk3=z=IaIjO<${2VK8J}3uA?R@hvd~ekKhrOJ8
zruqYu)8<t4t#+<E(IhwD+mCu8H^izL=Z-I9>UG;m=4*G9WU&Nbp+(p~e_YT7_+Oa>
zcP1Yj7xah?e>RiE>uB(^OKL5;{IbxWMTb<yTV0Z&TszF%d@MF=;@RE&`XM7@8=^@#
zjg=ZtO|a}&%DcC~wdu&K`N0U&#>2DdJ=o?vraS9P`ow!`;JmvX_05UW__ps6%8V*0
zbdr*Y_=p?^Ic&j*Tb}C8w3?V}Za5ywM4v3CnSEf(P)TX6{ubF|vxzZi<ya(*0_lAY
zW>L7cyw-o1cLZ7@?UUan;D?7B$3EjaY!N8Nip?{f_`nchfvL$`ebx5zWGVI0^2_B%
zlM#I}69jtZS1(b*PLH~os;_?AI|cb2{&tp=^<B^#ytbQJN4<J@tzwS2$|;sYhH>Ru
zqc2=&<exM8n3TED+LhS?KW}eBj9On1`egy?wopdb^`A(Jj>sV3=G4{Qz4%|s4J*$2
z*??gCg-Xl8qDT3Ax!eF(-nzcC@At1I5K_rNA}Kr^PdWQ_xF&m|k0;$BF7nnijGzA3
zUHe}q+z0#gFR7Q~|9EfzC;#yOvQ<Wg@r>x-x4d+c5(DN>JpSF$>~rKw8E6^R37MQ#
zbRTX?;E#Kezn{o~XbI*8e}TMTdg~;MD$8LiJ6I0|64Z-y11M&T-1=iT-tZvv3&@#T
zJtDHp`ASL2*)Vx6Ml=Q)X%{C*cxGsi7S3zn4<c;aZcL-pW%QAbDbitBK*9v?T7Ud_
zf!e9nVx0W$e1(K)u)1M8KMe;f1i8C+buc(4PcP0eaMlX<iv`k?poSSRVRdUPH!_<+
ze82tYPX|R;Up!N)pks!X>wj++ZMmtOJ)3ab_1m;@0Bx@`;Gd7>$O8$QR{71F<H4s7
za4>j!cwGOy1_)#khTT73l(_rv0Nk&DFePxAdg=V-Edmg^Kut6f({BC#J=*f94B=bQ
z3jZEcqHj@Y`VYc(Zs8GC^cU5FCmbACQk>1VZQ6jD$eQWP2ZSG(N$w#bZ4v<24UdN+
zQxxjz>e7H`e0>7lDlxv<39LZ4irlnuE!nHoTQq*@H_vQupS}p`5kUI%U^!|t(<Lk4
z1z`Pl!aqwDHXd?a9Mj=d_Gw7&CTNB&FC++8|2ww_SBr%%ribLA7$FRDBI$EmR^hd|
znRcZQwSjm<xAI~^Pn>92@0w_fdiRGV5vdSPwSzi<!E)Q3ssi+16aj~Mox^YQyY3su
z{eBrh;ieKOL7)I4Z0o|snXkj()>UyBC}%0B5#KGYu$b3iCfPpymT}tH3T@l)y%b%T
zqS{_@YFKtwRJ;e&-BEtMs|w~r)T!?E@$Cvj5w_HzfFAic<7!8q=?cujYeF!wKDZKN
zJOc&ZX2HT37Pt9`<{U(Yf#?sSw)w|V<AyKbBo0Uz*W_hH=Vm~+s9@rm&>Jp9Q=0@V
zDFCL|Fj^C12jebuVYu_Ooj5&By}HytaE2Z-TLFa-Ot`MDZljV+kbPoF0ibQ!kJML)
z^tEoql*9CWhQZ6@!Ci|~F%5d?pKRMM%+FQN>*|+piSIh-yDd(i=$Ds(9!5q&ScO2<
z012x{BHVcZAQI@+dz_d4MUu7w5>M!$VJ$_-nGpyaT!eO?<u9~&@dK(LKT=2K0jsJx
zP!<st+xM~1B`eXcu-q6(ya-iOR8*9p0fcPR^}cw&P%0#A5Ms}7)`JH`aykKLT8)kL
z9e_AR0gfpcLtg-VkeDD0Nuvdb$RV|*@$sJkmlnDvlsjRA1hE-RZxj_|yaQf1P|!I)
zT1y2&8dW-_+990eQh@p&3K&k6K%=Lrsfmr78!&5mk!VMtuY;I9A6gi22JlQkVFcq5
zsNnnn>|mf(gA|5YJre2)#W$O8a7Hqf1Cejybz%u{Y*kfN%i`pmIvqe7HV$5$p;`by
z<s&`>cpz;C?x9q_z8reG4A_4wfXvIB_-bQI6mWH~Q=MTDfJnDGb`t=sZa^|0->QaA
zb|rapb4<G-`S4HJ*;fG@dJCv;<=53E78kSkL{s+yrH6SdCL!$uBpVoDWJ5f~Mu5;!
zihw<pSx?k<UHXSuQBhGZz|hjEdWQ*M;2*n8krFgOkRwIR>sTNp(%?c*Pmcm976JLD
zcR<qS#t5sT<@i3p;v-4PD_vm#Xn4hAF_0iJT{gc2Bz}N|7u8_$Q>%%570(QL0B8q<
zXG_oyehwO7OaP%(SW#gA4eTpjz_l}a3Scn0fXDg<Xwgtkqm~(9;toZo-F$BL!Q(#*
z1fn$pNHQ{zQ1BT7RBL6C?~4bufO9DjSX5ZZgQRF)08JpPMZn4i*#COb{qjSA&00!G
zNJs%qpz-#t1sHHDZf<qJw(uL@T~1%6TY@`dA;L+iuI6(A49_%TrkCr<vw*^FO6T`i
zeN9O9))CyY0MOBas@(x5d#VESJLH=%lkkG5E2<G7?3JMw^!Gmvd2M~S0(`IvZ=RkM
z0CN9{6v1I3z5h@dXrT`Qbe|29#Ap_^T{GhUH-|oTeEbctoc1)myr7{sTTVv!!8luP
zeQ#@fGJ#Y^$uBJ(0{r(>FyD+$h9-^QzD)$lR6q_jtoB6;+i2+NeG7nRoJ7CO!8~!J
z=`Gc(D>k_VP{UN@qZ%1GNFW-ar+R0N-j$ps&<jLeDH65J4<=D?wWhW<Ip<mi^gBR7
zZ-T{uAIvz+0LjZgIm1B6;xZjx{I&!rL!Z?dO`uo;HL}eNpM&i>UzheJ*9jF%q(zuO
z(xmgt%gdJ?mo7#;?r+YX@Y8}>y$NOyAApl_rEJE@5by%%vU$bv=4LBUGjN%;NKOUR
z=~H$O8=E3P19zyfU}jv^4wA~+@;@-3?THBQGkQRTq_edW{2>GqJ1(b_+Gz!u{&RLS
zmD$Uu+pZ!@Y&w-D(iJv9oyWvwBT@SsnA6o+E43D80bshc=6@G<$wvyNS%AWofHZ)t
zKbXo|{t?N&^ZahU9gsL%p*B+$#+d=p7P>vvrNP@kK*pyP3Hu-8i0Z#d6LehE8g%N!
zB>(&E4$Mds0QHB6;G4XRp|Jt1-Z`s_c0hM0Z#e^@GuUiir*ZqOju}vIFM4-z^{u?7
z#wD3E6Zs*=Cjf%)ftwG%m<ROu2@c<x#WTo8wFnRrfaqxhQt6IM8$e6X?mPno$b&kG
z-nAop)vs3`K(^tG9gdU<o16|<eULe`5{Q(d&^PQwjyNSHr3$a!HD?zfhu~O0j|Aoa
zF8){I`9ld$7OVQ+0~RNwObKAVR(T%1`<9mG1^(*^zJOdVQr2R*VJ*fCNOK`Iq)d2H
zGTi|<&cJm{4gfVXftmf9pU<)h&4dDC_XW^Z0KW&HjTC))&6beq4m#c7L;vFz0PIIn
z%bz`ah9rjn`^ure2?ttK9{@%VX?Ou+G=cumUORd&0MP7#K!-dyltmT&0f=j@XATbA
zD(Sl;+QAI>0=<g_kKRYhDgZqq4zO>rUR+X+_K!dcV72y^RupXWv=V@Qi@Xh>-t-AT
znGe8J@%6aLyk2KTiZ=8cI}#!_=>SpkOThI7G#~?@FTt+IPk`RTHRrw(y7@cd1u(om
zi2;M1+hBnq0FKnfXqdz=VEKR$2|$7)f$ST`{6Gl~DZR!(lvPr~AxUqB7e?pJ2KdIq
zgjv&zi;KJb2c>4Q;$(x>$LPF^V6tCasAfuh2lje%3pP8e=;d{xwZuUcO(u5GNat`G
zX&VVWN?W?d3Pi%eLEixVUtmPGka$-uxYps{3AL?c4Y0f;RSa~22t{0@>za1^<wGE4
z;C_1wG%URQsR9AKJrp=)V*vLLv<PHmWYAHpzPphj7sL=O{DJl}pttcP1pzbyoLN-U
z`+%xLAHcV61v@{6Yyia#XoYPEHsK~<9Bv_n(X!gwGLVpQAfixZ@(o~3YfZkV0@(yG
z7EOUb1#rWw0nQj{_vT=*x~_kewOLfSK4EY=kQ{UY@axp4{zs9PU<|QP;E|=@q)GTS
zHLgZ*3;{B&>CVp1{d{`|2LYhL5hx5)yN04D`Muo&mH_yB2*?OJAhE7BKx8Y}1~xZ0
z1X!xpZO4ffDFD>kA<GyaY*V0|`i{Hx@0$}Km2kE*T*+~oz6Fqm8^CJ;%0Sn)KrAns
zw+ARAo_Ps)4^C5>4FRI><ZkH>z^R@<15VPGIE9pMzh+)D-_6Sqifi&LfK|ibH3aIr
z$&U$rYS2)reNoYT&=fq45h8uE970Ie!jkwZ71>~Z{<r!0`N7+c&HE5>i2qqS8~|1q
z_S@R5)Iz|x%Uf>bsA&1nERJ)jr-jst@iBaH1bo+2uz7{h9hrZo?*j!~*oUr@OCMU{
z!WDp4+rE4OEC|otJdokT%@7501;<b~S65Pl`(5tdL@1lUC=Ey=fKcJX^e4EVUSLkJ
zeC-dm;oty8L5+X+++2Z%15%<vy?|CuRvg3tve!gJe`|d&KDYpKeJaoZN(FXenv|S;
zYmn@->=0qVc~_Zg^ZQA5mtI<r6g9sc4a@eMGkzdeg^527I^D8lqTH}~YTj+yaFajt
zB5F3^?ncGN#>Rh6iov7}2rb~H1e)ypE^5uEeP;c-qJ0+9I*GMP-gfxOK4}dYj={T|
z1P#?$%Nz*UT1-QIVT_7Y9I2eVJQS<YCO-!&LGU&lXa^x4Gc+7D?JjCaIZeLRJ{4t1
z0fxSPkTUwEk9feSH1?@59H57D8EAma-|cVPIRNlnFW}(j9=W^8Pjg`Rgo3rb+@uK*
z*qgyeBth^py|KuVG6(kuTFd;p?i}GpW<MH!N(9~|%bAdqt&hve$zuFkB~!9yB$MfQ
zNlf<WW|XC_XOxlRS?-h1Hl>%@-+D;WYK@2VHB{{XK7KBs@<_$@Uh-W28z+4`j@pK9
z+^KiLg|8iO4>Axk{96mNwA_(z#%=@pduCHz+(*(=yTzaxsWP&^Fi?q61L>Os|BR8)
z)5n6<J6*z$syBwV3v@F>eRNFB5700_n}EcSG?>*A7wZqeX-ygce*fb8^XmLu2r$Z7
zOQ|0`2mz2^u>P(Am64fhi^!I|mBm&cAut8gx0w+7T%>w3(}|{sdZk(=(@j9^XmF#v
zoVyY5_`MoiK<`(0udJ*T>*D}rqOd2ira%?&s{^7X(agj|>eS!dd_DVbYu8Fc#pML$
zdw60etYvmf#CE#;bk5kwC}Xp#wzjr#0FaeWjP{42(B$3uuwB@``=k}nwx}sa0L`2U
zhm-L<rPis3q@;z5&2Y8GkpzJ@z{|cy2YDP>Wg;T+fppb^=_X*Rmw|2&5O3GSEY6TF
z)mmu3y~G8AUqHAnY{5Z0F}2u1O^r}bUte8YJ4`8=yT52JVZF;B`=PmN65i-LfizYX
zq6X|BkAwNtYWn$t_>=T6ua#U~|JHs*kL829JpXEz)~^{f;PqiCb8Cxj>l9F9A3)S~
z&(c|5U)(CYIR@zHq(JT6>w&$RyaM!$0Sl;mvm}2A$Pg&t(!Wu7*88OW(8I$cr=&!Q
z1#4dZJ_a^6ldv!)(r4Szevx+ov=D)}ZUIz=a;mC`0q>fRho1sZK~?pox3>sTziHiB
zY;Z<5G&Hofv5^LL7j&P+;R90AkL&B^xp{di7m<iARig9vo}M$d3rhiaT*(Xa%oKQ7
zlsE+V_%{9-K3RV*Oxk+;T`w0~qcSs>WEB0aqbn(BY2UVLzw}f7YckX?p#~o1bX2ZP
zHvQ>zu2tHCJk;9IN{^7lt<k_}3t0-3!ocuDxM#(w!?LE+vIay8#40K(#SD{n0%ky7
zv{J9EZ*Wip-;ti49%z4U9Rc5?|6}l##u*b6(-Bx;m?S0Xf&7^#Aj-a#RsvBpP%ASx
zL>Tvj!Uv!d$G)q_nRG8CHWee~D`wn)fQ4)z(F-gy!Y{*?T|f|L#Z@GGVtaXOD{RUC
zO2xsr!71Co2ndRTU4K-R_Uq%wVn31Z1mGX=ni4i1kdRo4_h)0&*DR=IB$()J3zpq9
zN!A(LWRm1I8Cy+k?EWD_IR`7zThu0_im+izjYM^SXGDwrFNb=jCr|EiCo9oqP7RS}
zOy4_H193ZF|5~f((P}MhesPgIR(sY_Uq7u_*}x#(;LHvIyp@o>Jv%Rl1UUIubTBf(
z>>OhP>o%@d$=ioBR~Hv6zz=<)rA1b%#Q}tyvM3YoKsdz4#&&qW;f;m{Qd~=?Oly5}
z^F3B{YisLFt<Bf;CKb~vow(H0&p>Da{Kk7>_m1HU`YS2%_~m7j;a{C0XW;J55C;p@
z0br@q46^q2ZZhEJ<V0!=xr)$+hK8Cm<O?~keq`ryZx6eXevFgg@7(Tw8|{DW3Pu{G
zliDwk-HImz@qDp+lUl!Enuw^V^Zwj(%Gb>X+(7DaVE>@K&2hd#9Wu7@A-lNvzI-fQ
zqJ5j&Zc_0A2*5vh4S#!Hd;{F|W~8K!`2x_X37E5|qonKv-^C6MDK{PfJ+5Ng!JDLz
z>uX=Yl<#wgFSXNyVEK$n*Q6e1%0r=9KZ?wRJiw5B*8BJG-@bJzJ3CI`JH;g@hh1M^
zKY?Gs_xBU~gHlo`z$iBb`WbytsJyT5b*>wHzLDuGI`Ce9g6P0};n%NUx}rwkRl^Pr
z99&&p`%qj!okuZWGPmK+*%~We0Rb!&3To;J->l^uNKQ@7gR8UskjeMm%gYSF>9$zj
zbrq?y7$np6he7OTYOG@Mc7T8DyxvEQ#JhtLC2L`khj)4E-MZ8Hg)qWr_1DISmET1g
z#U@U6c6eY=*Cg@0KDz=7{4xkRL;B6p4xtK3>FG3YUpE1{<qKO+D=StcL;hEHH*)H<
zwi-VFX#y$de}oZX7^BdDf`T=!C@ZZQ8R>kuEdA%y+ZTA%f8g-s-W7j;Gcz+V6r7G$
z<-jj!MBVW~>sRV<ItkGTc^+o%xFGgsnRXp!Dy8enKrd}?Z-;;W9JJuj3q<WUBoaS-
zt!vE9#XzbTM7?BcyFOOBzP*B1w;eCbb%W{`Wcr}CzFwwcItNtMfLb0;pwvc8LK1v*
z<m~F<u?CpXGj+D)pofsx#NlYZu$ad|x`Lu2akKkA2`~T>ldVoG!N63Jfmi=%Yg;U6
za&I<dW?_i{PP2;d&FQp}#?Ej)P*+e>9nNoA<_&1wf@~|hSPP^*FKAA)-f00@kbdu=
zprB%VAX)tPJpxn*6|p~{q2UB-qq0&`f$wiXDE~b6sv`)!_ycGS3521$``Jf7OsS}-
zsG$L2Sc}(5?p5oDjfmoFQFu!Zd&%^!+`D%_(ql*@R9$99N0(2Yg@N1#S|AQS{zI@N
z9=#6$l7__&E-p1Z6nMpa99pG^?<=MfB7wk2adA|8d%MD`|EH%b4QlF&!Z8gH5hRqQ
zgi=U=BGX|>(ops-d60sc1O<dr3@H^FAydO9%ZMz=RL0suY$&@JoGAkiVA+%wk=jxR
zkrrf-H7vs#b}JAdV0r?y=V#t~=iPU6&UeoJ=Fa^B-$fxIA(FT;keLcek$Fdz{6iuT
ze5|dl(^_`n81SvpRCqkjEqHE}2{)6;<U|=r)3`dD>L--afnaJYCoHb68V$h$@0_fo
z!z%%F2k?Wd>+A1j!P11jn|}aw$hP*ipEA;Ius{z@8yWih`Rz&wK$z%Vy4`tDD3Mt6
zKPihH>v{fMG{S$yp8>guF6frY(+QABstyB;w`#xXQ#M<n?N&I%{}5ZxdHfNDLILF)
z4{1r?>-HZtH8qjM#Ky3;ot-qGCdo0mcWY|o@F1v9J&kM0Xd&9!G0j{AK3L+VXy&eB
zvHxB)kVd7M@stw8*Ei=;rm}TjaT!L=N!dR*@Pj>DaWZHSLQ??K=USmTVu8w6dL}C<
zr~q_lZq(MT^_*efF*N?x@69+0rz!M>{+Loru(ZME<vldP^~S&`KCXXXUte8M@PoiL
z`0;p)bk}*ni<R*<gST&0s)g>P;TGjaQq0wCfI9wU#|NN4T`OOLmC_ef)6ju#h_l~#
zRMsA5_tq^PP_(`aVejebxuH-nXf!nBbLig|L8JEu<N`?G+1c5t`wbnm)Kq7G1_Lms
zpr9Zd)Xn&#%>khkGT{0^dh^6%PQFx1th}OuqY<2(jDUauY3AQy#ZL0@u)ES_Vs3t6
zOD?n&`9wyV0Vk3${{kP`mZ%{V3KO5eRMg71L^x6j(<Noe5Xfc`fDlThGc);XYD0M`
zTdrjf(P!)E$l!7bW~ZMuR}YW>{G}Eejh+J70e%+nJsOal7auR|nWMo8aKN$zoj+go
z%bFzXY+xW1$D5$%1hB^Ca#0}MLr+eoHa0b#i-=fqiqFBJrBdk*6xf8tKH@&8<^a&q
z!^2Hlzm}=hk@;h-x<n#T;8sz)a0=D=SA~23<fI-E1IhIuQ+c(xH})FP18^C}AO7UJ
zCnxJ5rD6SMBZx6MTG`v{4h{}J5=^7_lu9VU@CS&Ci}L{40FPhmIRJS3K64<HzKlhw
z?i-cA94E;}3?xYloC`I;41P8!)F>+}6N|+srYdjrX<#rr5-Q%z?2o=a9F0!5SiY>g
zKD0Xa5DI%pfGA68iZ?g^6Cn9T>>JUX=cIcDYjbOhNFa1ByW)T7A01+jt_WNNinWUf
zgTapA#9xexvJm00dU~p`*jCuv;M`m0tOpYrJh6t_@Ln;hlFwk#mV4|ox;7LqFNdP9
zD7USwtuZhg9{Rz>a@!)UZXWe1k6QaGo#TFIeJkUjId{Ki+i75#N+Jann)SuaAj_6k
zR_W95onjl-btaPEF4y#r2)4~QZ58&-_T;+cJF22V+&1CB5duD{+wy+;l)we$!*Z@}
zSD%b)xz{jduZbWiV~5B*;Nu~3qtnt%fTQGAt7~6>mbMyPQFg|sKoxdaUk$UHm0h3f
zK!P0FNwnGvSafdo@lT!bnj_2k|0?<zq7ek6PPRkCLW_KSBlK=ZUMxwz|83^x9y>HQ
zaI5&f!YA2ce?$Z6sHv;#>g+_h=Yk8~X>tiu&=$FZty02E#rY7J!za~4L^g+~;j_G^
r9ylHP&4?uX`pmvV#JC@A8aF?y(qd#$D~sfwQWRK#9}B6cadZC#+k}gE

literal 0
HcmV?d00001


From 3f26dfa36488c34329e31f44041f5b842248619c Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Thu, 15 Jun 2017 21:04:51 -0400
Subject: [PATCH 0725/1644] ARROW-1096: [C++] CreateFileMapping maximum size
 calculation issue

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #756 from MaxRis/ARROW-1096 and squashes the following commits:

a864fe7f [Max Risuhin] [C++] CreateFileMapping maximum size calculation issue
---
 cpp/src/arrow/io/io-file-test.cc | 35 ++++++++++++++++++++++++++++++++
 cpp/src/arrow/io/mman.h          |  9 +++-----
 cpp/src/arrow/io/test-common.h   |  2 ++
 3 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 1b7e052d651e6..6550c74662298 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -462,6 +462,41 @@ TEST_F(TestMemoryMappedFile, ReadOnly) {
   rommap->Close();
 }
 
+TEST_F(TestMemoryMappedFile, DISABLED_ReadWriteOver4GbFile) {
+  // ARROW-1096
+  const int64_t buffer_size = 1000 * 1000;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(buffer_size, 0, buffer.data());
+
+  const int64_t reps = 5000;
+
+  std::string path = "ipc-read-over-4gb-file-test";
+  std::shared_ptr<MemoryMappedFile> rwmmap;
+  ASSERT_OK(InitMemoryMap(reps * buffer_size, path, &rwmmap));
+  AppendFile(path);
+
+  int64_t position = 0;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(rwmmap->Write(buffer.data(), buffer_size));
+    position += buffer_size;
+  }
+  rwmmap->Close();
+
+  std::shared_ptr<MemoryMappedFile> rommap;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
+
+  position = 0;
+  std::shared_ptr<Buffer> out_buffer;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(rommap->ReadAt(position, buffer_size, &out_buffer));
+
+    ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
+    position += buffer_size;
+  }
+  rommap->Close();
+}
+
 TEST_F(TestMemoryMappedFile, RetainMemoryMapReference) {
   // ARROW-494
 
diff --git a/cpp/src/arrow/io/mman.h b/cpp/src/arrow/io/mman.h
index 27d9736b683fd..e45c88efd564b 100644
--- a/cpp/src/arrow/io/mman.h
+++ b/cpp/src/arrow/io/mman.h
@@ -93,13 +93,10 @@ static void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t
   const DWORD protect = __map_mmap_prot_page(prot);
   const DWORD desiredAccess = __map_mmap_prot_file(prot);
 
-  const off_t maxSize = off + (off_t)len;
+  const size_t maxSize = off + len;
 
-  const DWORD dwMaxSizeLow =
-      (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
-  const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD))
-                                  ? (DWORD)0
-                                  : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
+  const DWORD dwMaxSizeLow = static_cast<DWORD>(maxSize & 0xFFFFFFFFL);
+  const DWORD dwMaxSizeHigh = static_cast<DWORD>((maxSize >> 32) & 0xFFFFFFFFL);
 
 #ifdef _MSC_VER
 #pragma warning(pop)
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index d6ec27048d51e..438f378085f65 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -80,6 +80,8 @@ class MemoryMapFixture {
     return Status::OK();
   }
 
+  void AppendFile(const std::string& path) { tmp_files_.push_back(path); }
+
  private:
   std::vector<std::string> tmp_files_;
 };

From d54bf4829757c21c74e647d1e0baf2ec7a0f18d2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 16 Jun 2017 10:57:20 -0400
Subject: [PATCH 0726/1644] ARROW-1122: [Website] Change timestamp to yield
 correct Jekyll date

Change-Id: Ic221871d3fbd1c51af1544471d80b632e135c7f6
---
 site/_posts/2017-06-16-turbodbc-arrow.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/_posts/2017-06-16-turbodbc-arrow.md b/site/_posts/2017-06-16-turbodbc-arrow.md
index 438e56a0ecd84..be6256de0d4a4 100644
--- a/site/_posts/2017-06-16-turbodbc-arrow.md
+++ b/site/_posts/2017-06-16-turbodbc-arrow.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: "Connecting Relational Databases to the Apache Arrow World with turbodbc"
-date: "2017-06-16 00:00:00 +0200"
+date: "2017-06-16 04:00:00 -0400"
 author: MathMagique
 categories: [application]
 ---

From 1a23419fb1ec1ead90eb45e4b3c2cacd8a85bba9 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 17 Jun 2017 19:27:08 +0200
Subject: [PATCH 0727/1644] ARROW-1124: Increase numpy dependency to >=1.10.x

While we could still build with NumPy>=1.9 for Python 2, Python 3 builds
require >= 1.10 due to a bug in the C-headers.

Change-Id: I0f9e0ad72e4ce4b1c6b44883d5781347d33f7e5b

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #758 from xhochy/ARROW-1124 and squashes the following commits:

5fff1ea [Uwe L. Korn] ARROW-1124: Increase numpy dependency to >=1.10.x
---
 python/requirements.txt | 2 +-
 python/setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/requirements.txt b/python/requirements.txt
index f42c90c5c9b3f..103f490588a5b 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,3 +1,3 @@
 pytest
-numpy>=1.7.0
+numpy>=1.10.0
 six
diff --git a/python/setup.py b/python/setup.py
index e10b4b84b01f5..eddf88ab16b19 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -366,7 +366,7 @@ def has_ext_modules(foo):
     },
     use_scm_version={"root": "..", "relative_to": __file__},
     setup_requires=['setuptools_scm', 'cython >= 0.23'],
-    install_requires=['numpy >= 1.9', 'six >= 1.0.0'],
+    install_requires=['numpy >= 1.10', 'six >= 1.0.0'],
     tests_require=['pytest'],
     description="Python library for Apache Arrow",
     long_description=long_description,

From 5be05ac154843908f87fb40f3e0bde5e256e8626 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sun, 18 Jun 2017 18:43:30 +0200
Subject: [PATCH 0728/1644] =?UTF-8?q?ARROW-742:=20[C++]=20Use=20gflags=20f?=
 =?UTF-8?q?rom=20toolchain;=20Resolve=20cmake=20FindGFlags=20=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…and FindGTest Windows issues.

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #759 from MaxRis/ARROW-784 and squashes the following commits:

358a9cc [Max Risuhin] ARROW-742: [C++] Use gflags from toolchain; Resolve cmake FindGFlags and FindGTest Windows issues.
---
 ci/msvc-build.bat                  |  2 +-
 cpp/CMakeLists.txt                 |  7 ++++++-
 cpp/cmake_modules/FindGFlags.cmake | 12 ++++++++----
 cpp/cmake_modules/FindGTest.cmake  |  9 +++++----
 4 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 263d4bc580508..366fc2818f0f3 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -23,7 +23,7 @@ conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython
 conda install -n arrow -q -y -c conda-forge ^
       flatbuffers rapidjson ^
-      cmake git boost-cpp thrift-cpp snappy zlib brotli
+      cmake git boost-cpp thrift-cpp snappy zlib brotli gflags
 
 call activate arrow
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index c295fa0ba8005..42e1f69b0e4ad 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -412,6 +412,7 @@ if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
   set(FLATBUFFERS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
   set(RAPIDJSON_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
   set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(GFLAGS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
 
   if (NOT DEFINED ENV{BOOST_ROOT})
     # Since we have to set this in the environment, we check whether
@@ -432,6 +433,10 @@ if (DEFINED ENV{JEMALLOC_HOME})
   set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}")
 endif()
 
+if (DEFINED ENV{GFLAGS_HOME})
+  set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
+endif()
+
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
@@ -560,7 +565,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
   endif()
 
   # gflags (formerly Googleflags) command line parsing
-  if("$ENV{GFLAGS_HOME}" STREQUAL "")
+  if("${GFLAGS_HOME}" STREQUAL "")
     set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS})
 
     set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
diff --git a/cpp/cmake_modules/FindGFlags.cmake b/cpp/cmake_modules/FindGFlags.cmake
index eaea83530547b..9eaf82497cd88 100644
--- a/cpp/cmake_modules/FindGFlags.cmake
+++ b/cpp/cmake_modules/FindGFlags.cmake
@@ -22,13 +22,17 @@
 #  GFLAGS_STATIC_LIB, path to libgflags static library
 #  GFLAGS_FOUND, whether gflags has been found
 
-if( NOT "$ENV{GFLAGS_HOME}" STREQUAL "")
-    file( TO_CMAKE_PATH "$ENV{GFLAGS_HOME}" _native_path )
+if( NOT "${GFLAGS_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${GFLAGS_HOME}" _native_path )
     list( APPEND _gflags_roots ${_native_path} )
 elseif ( GFlags_HOME )
     list( APPEND _gflags_roots ${GFlags_HOME} )
 endif()
 
+if (MSVC AND NOT GFLAGS_MSVC_STATIC_LIB_SUFFIX)
+  set(GFLAGS_MSVC_STATIC_LIB_SUFFIX "_static")
+endif()
+
 if ( _gflags_roots )
   find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
     PATHS ${_gflags_roots}
@@ -38,7 +42,7 @@ if ( _gflags_roots )
     PATHS ${_gflags_roots}
     NO_DEFAULT_PATH
     PATH_SUFFIXES "lib" )
-  find_library(GFLAGS_SHARED_LIB NAMES libgflags.a
+  find_library(GFLAGS_STATIC_LIB NAMES ${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_MSVC_STATIC_LIB_SUFFIX}
     PATHS ${_gflags_roots}
     NO_DEFAULT_PATH
     PATH_SUFFIXES "lib" )
@@ -50,7 +54,7 @@ else()
   find_library(GFLAGS_SHARED_LIB gflags
     NO_CMAKE_SYSTEM_PATH
     NO_SYSTEM_ENVIRONMENT_PATH)
-  find_library(GFLAGS_STATIC_LIB libgflags.a
+  find_library(GFLAGS_STATIC_LIB ${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_MSVC_STATIC_LIB_SUFFIX}
     NO_CMAKE_SYSTEM_PATH
     NO_SYSTEM_ENVIRONMENT_PATH)
 endif()
diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake
index 3c5d2b67e4494..0e2af2fbc0941 100644
--- a/cpp/cmake_modules/FindGTest.cmake
+++ b/cpp/cmake_modules/FindGTest.cmake
@@ -43,7 +43,7 @@ if ( _gtest_roots )
     find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h
         PATHS ${_gtest_roots} NO_DEFAULT_PATH
         PATH_SUFFIXES "include" )
-    find_library( GTEST_LIBRARIES NAMES gtest
+    find_library( GTEST_LIBRARIES NAMES gtest gtest_main
         PATHS ${_gtest_roots} NO_DEFAULT_PATH
         PATH_SUFFIXES "lib" )
 else ()
@@ -55,9 +55,10 @@ endif ()
 if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
   set(GTEST_FOUND TRUE)
   get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH )
-  set(GTEST_LIB_NAME libgtest)
-  set(GTEST_STATIC_LIB ${GTEST_LIBS}/${GTEST_LIB_NAME}.a)
-  set(GTEST_SHARED_LIB ${GTEST_LIBS}/${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  set(GTEST_LIB_NAME gtest)
+  set(GTEST_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(GTEST_MAIN_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}_main${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(GTEST_SHARED_LIB ${GTEST_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 else ()
   set(GTEST_FOUND FALSE)
 endif ()

From d874d4e32707f249ce64e21f4694c95ff3a40a82 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 19 Jun 2017 11:37:02 -0400
Subject: [PATCH 0729/1644] ARROW-1081: Fill null_bitmap correctly in TestBase

Change-Id: I103cc31e1400491713b160b429c5491e8c82f810

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #760 from xhochy/ARROW-1081 and squashes the following commits:

e012438 [Uwe L. Korn] ARROW-1081: Fill null_bitmap correctly in TestBase
---
 cpp/src/arrow/test-common.h | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h
index dc11e76edf465..b3e5af86d4b53 100644
--- a/cpp/src/arrow/test-common.h
+++ b/cpp/src/arrow/test-common.h
@@ -52,7 +52,12 @@ class TestBase : public ::testing::Test {
     test::random_bytes(data_nbytes, random_seed_++, data->mutable_data());
 
     auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
-    EXPECT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
+    const int64_t null_nbytes = BitUtil::BytesForBits(length);
+    EXPECT_OK(null_bitmap->Resize(null_nbytes));
+    memset(null_bitmap->mutable_data(), 255, null_nbytes);
+    for (int64_t i = 0; i < null_count; i++) {
+      BitUtil::ClearBit(null_bitmap->mutable_data(), i * (length / null_count));
+    }
     return std::make_shared<ArrayType>(length, data, null_bitmap, null_count);
   }
 

From b5e8a4831339a7750ff48c0080134cd9c4a2f785 Mon Sep 17 00:00:00 2001
From: Kengo Seki <sekikn@apache.org>
Date: Mon, 19 Jun 2017 11:44:36 -0400
Subject: [PATCH 0730/1644] ARROW-1128: [Docs] command to build a wheel is not
 properly rendered

Author: Kengo Seki <sekikn@apache.org>

Closes #762 from sekikn/ARROW-1128 and squashes the following commits:

2179516 [Kengo Seki] ARROW-1128: [Docs] command to build a wheel is not properly rendered
---
 python/doc/source/development.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index cd2440610eb71..910418cf10804 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -174,6 +174,7 @@ You should be able to run the unit tests with:
 You can build a wheel by running:
 
 .. code-block:: shell
+
    python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
           --with-parquet --with-jemalloc --bundle-arrow-cpp bdist_wheel
 

From 86c67d06146dee957c10e200a758be07c85da2d6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 20 Jun 2017 14:05:12 +0200
Subject: [PATCH 0731/1644] ARROW-1129: [C++] Fix gflags issue in Linux/macOS
 toolchain builds

Minor regression introduced in #759

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #763 from wesm/ARROW-1129 and squashes the following commits:

afc4bb6 [Wes McKinney] Fix gflags library name being searched for in toolchain builds on Linux/macOS
---
 cpp/cmake_modules/FindGFlags.cmake | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/FindGFlags.cmake b/cpp/cmake_modules/FindGFlags.cmake
index 9eaf82497cd88..63a55d75133b6 100644
--- a/cpp/cmake_modules/FindGFlags.cmake
+++ b/cpp/cmake_modules/FindGFlags.cmake
@@ -33,6 +33,12 @@ if (MSVC AND NOT GFLAGS_MSVC_STATIC_LIB_SUFFIX)
   set(GFLAGS_MSVC_STATIC_LIB_SUFFIX "_static")
 endif()
 
+set(GFLAGS_STATIC_LIB_SUFFIX
+  "${GFLAGS_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+set(GFLAGS_STATIC_LIB_NAME
+  ${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_STATIC_LIB_SUFFIX})
+
 if ( _gflags_roots )
   find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
     PATHS ${_gflags_roots}
@@ -42,7 +48,7 @@ if ( _gflags_roots )
     PATHS ${_gflags_roots}
     NO_DEFAULT_PATH
     PATH_SUFFIXES "lib" )
-  find_library(GFLAGS_STATIC_LIB NAMES ${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_MSVC_STATIC_LIB_SUFFIX}
+  find_library(GFLAGS_STATIC_LIB NAMES ${GFLAGS_STATIC_LIB_NAME}
     PATHS ${_gflags_roots}
     NO_DEFAULT_PATH
     PATH_SUFFIXES "lib" )
@@ -54,7 +60,7 @@ else()
   find_library(GFLAGS_SHARED_LIB gflags
     NO_CMAKE_SYSTEM_PATH
     NO_SYSTEM_ENVIRONMENT_PATH)
-  find_library(GFLAGS_STATIC_LIB ${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_MSVC_STATIC_LIB_SUFFIX}
+  find_library(GFLAGS_STATIC_LIB ${GFLAGS_STATIC_LIB_NAME}
     NO_CMAKE_SYSTEM_PATH
     NO_SYSTEM_ENVIRONMENT_PATH)
 endif()

From f0f1ca60d8581dc252568c27fa3481aa5e1cf9ac Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 22 Jun 2017 09:12:43 +0200
Subject: [PATCH 0732/1644] ARROW-1138: Travis: Use OpenJDK7 instead of
 OracleJDK7

Will merge on green build to unbreak Travis but it would be very helpful if a Java expert can review this change.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #765 from xhochy/fix-jdk and squashes the following commits:

262f6f9 [Uwe L. Korn] Switch to openjdk7
751444a [Uwe L. Korn] Add JDK 7 to package list
---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7a8b16c2f198b..a32562fdd19a2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,6 +17,7 @@ addons:
     - gtk-doc-tools
     - autoconf-archive
     - libgirepository1.0-dev
+
 services:
   - docker
 
@@ -59,13 +60,13 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
   - language: java
     os: linux
-    jdk: oraclejdk7
+    jdk: openjdk7
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
   - language: java
     os: linux
     env: ARROW_TEST_GROUP=integration
-    jdk: oraclejdk7
+    jdk: openjdk7
     before_script:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"

From ef579ca7eb0a99ff32ae8eed8fc3127c7ef4110b Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 22 Jun 2017 09:33:20 -0400
Subject: [PATCH 0733/1644] ARROW-1123: Make jemalloc the default allocator

Change-Id: Ib03392431851773df3b59b2c9a4d9a7bd672d2cb

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #761 from xhochy/ARROW-1123 and squashes the following commits:

2c75f56d [Uwe L. Korn] Use shared jemalloc always if available
2205586f [Uwe L. Korn] Run parquet tests in manylinux1 build
97f77d08 [Uwe L. Korn] Add pthread to static dependencies
7d01e9eb [Uwe L. Korn] Cpplint
bf478f1c [Uwe L. Korn] Fix allocator
4fbc7bac [Uwe L. Korn] Correct small allocations
c3bacc05 [Uwe L. Korn] Revert "Also link static libs to librt"
44f0cfc5 [Uwe L. Korn] Also link static libs to librt
bac694df [Uwe L. Korn] Also link static libs to librt
35212bc0 [Uwe L. Korn] Remove obsolete import check
4b714a14 [Uwe L. Korn] Don't force optional that is no longer there
e004150c [Uwe L. Korn] Only search the for pthread library, not the headers
e65d0d1d [Uwe L. Korn] ARROW-1123: Make jemalloc the default allocator
---
 ci/travis_script_python.sh                    |   3 +-
 cpp/CMakeLists.txt                            |  74 +++++++++--
 cpp/cmake_modules/BuildUtils.cmake            |  17 ++-
 cpp/src/arrow/allocator-test.cc               |   2 +-
 cpp/src/arrow/jemalloc/CMakeLists.txt         | 120 ------------------
 cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in   |  30 -----
 .../jemalloc/jemalloc-builder-benchmark.cc    |  47 -------
 .../jemalloc/jemalloc-memory_pool-test.cc     |  51 --------
 cpp/src/arrow/jemalloc/memory_pool.cc         |  76 -----------
 cpp/src/arrow/jemalloc/memory_pool.h          |  57 ---------
 cpp/src/arrow/jemalloc/symbols.map            |  30 -----
 cpp/src/arrow/memory_pool.cc                  |  32 ++++-
 python/CMakeLists.txt                         |  17 ---
 python/cmake_modules/FindArrow.cmake          |  10 --
 python/doc/source/api.rst                     |   1 -
 python/doc/source/development.rst             |   5 +-
 python/doc/source/index.rst                   |   6 -
 python/doc/source/jemalloc.rst                |  47 -------
 python/manylinux1/Dockerfile-x86_64           |  10 +-
 python/manylinux1/Dockerfile-x86_64_base      |   3 +-
 python/manylinux1/build_arrow.sh              |   6 +-
 python/pyarrow/__init__.py                    |   9 --
 python/pyarrow/_jemalloc.pyx                  |  28 ----
 python/pyarrow/includes/libarrow_jemalloc.pxd |  27 ----
 python/pyarrow/tests/test_jemalloc.py         |  67 ----------
 python/setup.py                               |  11 --
 site/README.md                                |   4 +-
 27 files changed, 117 insertions(+), 673 deletions(-)
 delete mode 100644 cpp/src/arrow/jemalloc/CMakeLists.txt
 delete mode 100644 cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
 delete mode 100644 cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
 delete mode 100644 cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc
 delete mode 100644 cpp/src/arrow/jemalloc/memory_pool.cc
 delete mode 100644 cpp/src/arrow/jemalloc/memory_pool.h
 delete mode 100644 cpp/src/arrow/jemalloc/symbols.map
 delete mode 100644 python/doc/source/jemalloc.rst
 delete mode 100644 python/pyarrow/_jemalloc.pyx
 delete mode 100644 python/pyarrow/includes/libarrow_jemalloc.pxd
 delete mode 100644 python/pyarrow/tests/test_jemalloc.py

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 904db52a69a1d..6cc760f0ac23f 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -106,10 +106,9 @@ python_version_tests() {
   # Other stuff pip install
   pip install -r requirements.txt
 
-  python setup.py build_ext --inplace --with-parquet --with-jemalloc
+  python setup.py build_ext --inplace --with-parquet
 
   python -c "import pyarrow.parquet"
-  python -c "import pyarrow._jemalloc"
 
   python -m pytest -vv -r sxX pyarrow --parquet
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 42e1f69b0e4ad..962891acf9cf2 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -437,6 +437,14 @@ if (DEFINED ENV{GFLAGS_HOME})
   set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
 endif()
 
+# ----------------------------------------------------------------------
+# Find pthreads
+
+if (NOT MSVC)
+  find_library(PTHREAD_LIBRARY pthread)
+  message(STATUS "Found pthread: ${PTHREAD_LIBRARY}")
+endif()
+
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
@@ -756,7 +764,8 @@ if (ARROW_JEMALLOC)
   include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(jemalloc
     STATIC_LIB ${JEMALLOC_STATIC_LIB}
-    SHARED_LIB ${JEMALLOC_SHARED_LIB})
+    SHARED_LIB ${JEMALLOC_SHARED_LIB}
+    DEPS ${PTHREAD_LIBRARY})
 endif()
 
 ## Google PerfTools
@@ -803,20 +812,16 @@ include_directories(SYSTEM "${HADOOP_HOME}/include")
 # Linker setup
 ############################################################
 set(ARROW_MIN_TEST_LIBS
+  ${ARROW_STATIC_LINK_LIBS}
   arrow_static
   gtest
   gtest_main
   ${ARROW_BASE_LIBS})
 
-if (APPLE)
+if(NOT MSVC)
   set(ARROW_MIN_TEST_LIBS
     ${ARROW_MIN_TEST_LIBS}
     ${CMAKE_DL_LIBS})
-elseif(NOT MSVC)
-  set(ARROW_MIN_TEST_LIBS
-    ${ARROW_MIN_TEST_LIBS}
-    pthread
-    ${CMAKE_DL_LIBS})
 endif()
 
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
@@ -938,8 +943,55 @@ set(ARROW_STATIC_PRIVATE_LINK_LIBS
 if (NOT MSVC)
   set(ARROW_LINK_LIBS
     ${ARROW_LINK_LIBS}
-    ${CMAKE_DL_LIBS}
-    pthread)
+    ${CMAKE_DL_LIBS})
+endif()
+
+if (ARROW_JEMALLOC)
+  add_definitions(-DARROW_JEMALLOC)
+  # In the case that jemalloc is only available as a shared library also use it to
+  # link it in the static requirements. In contrast to other libraries we try in
+  # most cases to use the system provided version of jemalloc to better align with
+  # other potential users of jemalloc.
+  if (JEMALLOC_STATIC_LIB AND NOT ARROW_JEMALLOC_USE_SHARED)
+    set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_static)
+  else()
+    set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_shared)
+  endif()
+
+  if (NOT APPLE)
+      set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} ${PTHREAD_LIBRARY} rt)
+  endif()
+
+  if (ARROW_JEMALLOC_USE_SHARED)
+    set(ARROW_JEMALLOC_SHARED_LINK_LIBS jemalloc_shared)
+  else()
+    if (CMAKE_COMPILER_IS_GNUCXX)
+      set(ARROW_JEMALLOC_SHARED_LINK_LIBS
+        jemalloc_static
+        # For glibc <2.17 we need to link to librt.
+        # As we compile with --as-needed by default, the linker will omit this
+        # dependency if not required.
+        ${PTHREAD_LIBRARY}
+        rt
+        )
+    else()
+      set(ARROW_JEMALLOC_SHARED_LINK_LIBS
+        jemalloc_static
+        )
+    endif()
+  endif()
+  set(ARROW_SHARED_PRIVATE_LINK_LIBS
+    ${ARROW_SHARED_PRIVATE_LINK_LIBS}
+    ${ARROW_JEMALLOC_SHARED_LINK_LIBS})
+  set(ARROW_STATIC_LINK_LIBS
+    ${ARROW_STATIC_LINK_LIBS}
+    ${ARROW_JEMALLOC_STATIC_LINK_LIBS})
+elseif (NOT MSVC)
+  # We need to separate this as otherwise CMake would mess with the library
+  # linking order.
+  set(ARROW_LINK_LIBS
+    ${ARROW_LINK_LIBS}
+    ${PTHREAD_LIBRARY})
 endif()
 
 if(RAPIDJSON_VENDORED)
@@ -1021,10 +1073,6 @@ ADD_ARROW_LIB(arrow
 
 add_subdirectory(src/arrow/util)
 
-if(ARROW_JEMALLOC)
-  add_subdirectory(src/arrow/jemalloc)
-endif()
-
 if(ARROW_PYTHON)
   find_package(PythonLibsNew REQUIRED)
   find_package(NumPy REQUIRED)
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index db83efed35031..6b2be41d24aa7 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -46,6 +46,10 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
         set_target_properties(${AUG_LIB_NAME}
             PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     endif()
+    if(ARG_DEPS)
+      set_target_properties(${AUG_LIB_NAME}
+        PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+    endif()
     message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
   elseif(ARG_STATIC_LIB)
     add_library(${LIB_NAME} STATIC IMPORTED)
@@ -55,6 +59,10 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     add_library(${AUG_LIB_NAME} STATIC IMPORTED)
     set_target_properties(${AUG_LIB_NAME}
       PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    if(ARG_DEPS)
+      set_target_properties(${AUG_LIB_NAME}
+        PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+    endif()
     message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
   elseif(ARG_SHARED_LIB)
     add_library(${LIB_NAME} SHARED IMPORTED)
@@ -72,14 +80,13 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
             PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     endif()
     message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+    if(ARG_DEPS)
+      set_target_properties(${AUG_LIB_NAME}
+        PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+    endif()
   else()
     message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
   endif()
-
-  if(ARG_DEPS)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
-  endif()
 endfunction()
 
 function(ADD_ARROW_LIB LIB_NAME)
diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
index 811ef5a79c2dd..5a4e98d76600f 100644
--- a/cpp/src/arrow/allocator-test.cc
+++ b/cpp/src/arrow/allocator-test.cc
@@ -33,7 +33,7 @@ TEST(stl_allocator, MemoryTracking) {
   ASSERT_EQ(0, pool->bytes_allocated());
 }
 
-#if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER))
+#if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER) || defined(ARROW_JEMALLOC))
 
 TEST(stl_allocator, TestOOM) {
   stl_allocator<uint64_t> alloc;
diff --git a/cpp/src/arrow/jemalloc/CMakeLists.txt b/cpp/src/arrow/jemalloc/CMakeLists.txt
deleted file mode 100644
index 7b627ac97b884..0000000000000
--- a/cpp/src/arrow/jemalloc/CMakeLists.txt
+++ /dev/null
@@ -1,120 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# ----------------------------------------------------------------------
-# arrow_jemalloc : Arrow jemalloc-based allocator
-
-include_directories(SYSTEM "{JEMALLOC_INCLUDE_DIR}")
-
-# In the case that jemalloc is only available as a shared library also use it to
-# link it in the static requirements. In contrast to other libraries we try in
-# most cases to use the system provided version of jemalloc to better align with
-# other potential users of jemalloc.
-if (JEMALLOC_STATIC_LIB)
-  set(ARROW_JEMALLOC_STATIC_LINK_LIBS
-    arrow_static
-    jemalloc_static
-  )
-else()
-  set(ARROW_JEMALLOC_STATIC_LINK_LIBS
-    arrow_static
-    jemalloc_shared
-  )
-endif()
-
-if (NOT APPLE)
-  set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} pthread)
-endif()
-
-if (ARROW_JEMALLOC_USE_SHARED)
-  set(ARROW_JEMALLOC_SHARED_LINK_LIBS
-    arrow_shared
-    jemalloc_shared
-  )
-else()
-  if (CMAKE_COMPILER_IS_GNUCXX)
-    set(ARROW_JEMALLOC_SHARED_LINK_LIBS
-      arrow_shared
-      jemalloc_static
-      # For glibc <2.17 we need to link to librt.
-      # As we compile with --as-needed by default, the linker will omit this
-      # dependency if not required.
-      rt
-    )
-  else()
-    set(ARROW_JEMALLOC_SHARED_LINK_LIBS
-      arrow_shared
-      jemalloc_static
-    )
-  endif()
-endif()
-
-
-if (ARROW_BUILD_STATIC)
-  set(ARROW_JEMALLOC_TEST_LINK_LIBS
-    ${ARROW_JEMALLOC_STATIC_LINK_LIBS}
-    arrow_jemalloc_static)
-else()
-  set(ARROW_JEMALLOC_TEST_LINK_LIBS
-    arrow_jemalloc_shared)
-endif()
-
-set(ARROW_JEMALLOC_SRCS
-  memory_pool.cc
-)
-
-if(NOT APPLE)
-  # Localize thirdparty symbols using a linker version script. This hides them
-  # from the client application. The OS X linker does not support the
-  # version-script option.
-  set(ARROW_JEMALLOC_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
-endif()
-
-if (JEMALLOC_VENDORED)
-  set(JEMALLOC_DEPENDENCIES jemalloc_ep)
-endif()
-
-ADD_ARROW_LIB(arrow_jemalloc
-  SOURCES ${ARROW_JEMALLOC_SRCS}
-  DEPENDENCIES ${JEMALLOC_DEPENDENCIES}
-  SHARED_LINK_FLAGS ${ARROW_JEMALLOC_LINK_FLAGS}
-  SHARED_LINK_LIBS ${ARROW_JEMALLOC_SHARED_LINK_LIBS}
-  SHARED_PRIVATE_LINK_LIBS ${ARROW_JEMALLOC_SHARED_PRIVATE_LINK_LIBS}
-  STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS}
-  STATIC_PRIVATE_LINK_LIBS ${ARROW_JEMALLOC_STATIC_PRIVATE_LINK_LIBS}
-)
-
-ADD_ARROW_TEST(jemalloc-memory_pool-test)
-ARROW_TEST_LINK_LIBRARIES(jemalloc-memory_pool-test
-  ${ARROW_JEMALLOC_TEST_LINK_LIBS})
-
-ADD_ARROW_BENCHMARK(jemalloc-builder-benchmark)
-ARROW_BENCHMARK_LINK_LIBRARIES(jemalloc-builder-benchmark
-  ${ARROW_JEMALLOC_TEST_LINK_LIBS})
-
-# Headers: top level
-install(FILES
-  memory_pool.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/jemalloc")
-
-# pkg-config support
-configure_file(arrow-jemalloc.pc.in
-  "${CMAKE_CURRENT_BINARY_DIR}/arrow-jemalloc.pc"
-  @ONLY)
-install(
-  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-jemalloc.pc"
-  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
diff --git a/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in b/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
deleted file mode 100644
index 8e946d17d8601..0000000000000
--- a/cpp/src/arrow/jemalloc/arrow-jemalloc.pc.in
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
-includedir=${prefix}/include
-
-so_version=@ARROW_SO_VERSION@
-abi_version=@ARROW_ABI_VERSION@
-
-Name: Apache Arrow jemalloc-based allocator
-Description: jemalloc allocator for Arrow.
-Version: @ARROW_VERSION@
-Libs: -L${libdir} -larrow_jemalloc
-Cflags: -I${includedir}
-Requires: arrow
diff --git a/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc b/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
deleted file mode 100644
index d69c3047587bf..0000000000000
--- a/cpp/src/arrow/jemalloc/jemalloc-builder-benchmark.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "benchmark/benchmark.h"
-
-#include "arrow/builder.h"
-#include "arrow/jemalloc/memory_pool.h"
-#include "arrow/test-util.h"
-
-namespace arrow {
-
-constexpr int64_t kFinalSize = 256;
-
-static void BM_BuildPrimitiveArrayNoNulls(
-    benchmark::State& state) {  // NOLINT non-const reference
-  // 2 MiB block
-  std::vector<int64_t> data(256 * 1024, 100);
-  while (state.KeepRunning()) {
-    Int64Builder builder(jemalloc::MemoryPool::default_pool(), arrow::int64());
-    for (int i = 0; i < kFinalSize; i++) {
-      // Build up an array of 512 MiB in size
-      builder.Append(data.data(), data.size(), nullptr);
-    }
-    std::shared_ptr<Array> out;
-    builder.Finish(&out);
-  }
-  state.SetBytesProcessed(
-      state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
-}
-
-BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(5)->Unit(benchmark::kMillisecond);
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc b/cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc
deleted file mode 100644
index a8448abc7d296..0000000000000
--- a/cpp/src/arrow/jemalloc/jemalloc-memory_pool-test.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <limits>
-
-#include "gtest/gtest.h"
-
-#include "arrow/jemalloc/memory_pool.h"
-#include "arrow/memory_pool-test.h"
-
-namespace arrow {
-namespace jemalloc {
-namespace test {
-
-class TestJemallocMemoryPool : public ::arrow::test::TestMemoryPoolBase {
- public:
-  ::arrow::MemoryPool* memory_pool() override {
-    return ::arrow::jemalloc::MemoryPool::default_pool();
-  }
-};
-
-TEST_F(TestJemallocMemoryPool, MemoryTracking) {
-  this->TestMemoryTracking();
-}
-
-TEST_F(TestJemallocMemoryPool, OOM) {
-  this->TestOOM();
-}
-
-TEST_F(TestJemallocMemoryPool, Reallocate) {
-  this->TestReallocate();
-}
-
-}  // namespace test
-}  // namespace jemalloc
-}  // namespace arrow
diff --git a/cpp/src/arrow/jemalloc/memory_pool.cc b/cpp/src/arrow/jemalloc/memory_pool.cc
deleted file mode 100644
index f7a1446a0d27c..0000000000000
--- a/cpp/src/arrow/jemalloc/memory_pool.cc
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/jemalloc/memory_pool.h"
-
-#include <sstream>
-
-// Needed to support jemalloc 3 and 4
-#define JEMALLOC_MANGLE
-#include <jemalloc/jemalloc.h>
-
-#include "arrow/status.h"
-
-constexpr size_t kAlignment = 64;
-
-namespace arrow {
-namespace jemalloc {
-
-MemoryPool* MemoryPool::default_pool() {
-  static MemoryPool pool;
-  return &pool;
-}
-
-MemoryPool::MemoryPool() : allocated_size_(0) {}
-
-MemoryPool::~MemoryPool() {}
-
-Status MemoryPool::Allocate(int64_t size, uint8_t** out) {
-  *out = reinterpret_cast<uint8_t*>(mallocx(size, MALLOCX_ALIGN(kAlignment)));
-  if (*out == NULL) {
-    std::stringstream ss;
-    ss << "malloc of size " << size << " failed";
-    return Status::OutOfMemory(ss.str());
-  }
-  allocated_size_ += size;
-  return Status::OK();
-}
-
-Status MemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
-  *ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
-  if (*ptr == NULL) {
-    std::stringstream ss;
-    ss << "realloc of size " << new_size << " failed";
-    return Status::OutOfMemory(ss.str());
-  }
-
-  allocated_size_ += new_size - old_size;
-
-  return Status::OK();
-}
-
-void MemoryPool::Free(uint8_t* buffer, int64_t size) {
-  allocated_size_ -= size;
-  dallocx(buffer, MALLOCX_ALIGN(kAlignment));
-}
-
-int64_t MemoryPool::bytes_allocated() const {
-  return allocated_size_.load();
-}
-
-}  // namespace jemalloc
-}  // namespace arrow
diff --git a/cpp/src/arrow/jemalloc/memory_pool.h b/cpp/src/arrow/jemalloc/memory_pool.h
deleted file mode 100644
index 0d32b4658e3e8..0000000000000
--- a/cpp/src/arrow/jemalloc/memory_pool.h
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Public API for the jemalloc-based allocator
-
-#ifndef ARROW_JEMALLOC_MEMORY_POOL_H
-#define ARROW_JEMALLOC_MEMORY_POOL_H
-
-#include "arrow/memory_pool.h"
-
-#include <atomic>
-
-namespace arrow {
-
-class Status;
-
-namespace jemalloc {
-
-class ARROW_EXPORT MemoryPool : public ::arrow::MemoryPool {
- public:
-  static MemoryPool* default_pool();
-
-  MemoryPool(MemoryPool const&) = delete;
-  MemoryPool& operator=(MemoryPool const&) = delete;
-
-  virtual ~MemoryPool();
-
-  Status Allocate(int64_t size, uint8_t** out) override;
-  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
-  void Free(uint8_t* buffer, int64_t size) override;
-
-  int64_t bytes_allocated() const override;
-
- private:
-  MemoryPool();
-
-  std::atomic<int64_t> allocated_size_;
-};
-
-}  // namespace jemalloc
-}  // namespace arrow
-
-#endif  // ARROW_JEMALLOC_MEMORY_POOL_H
diff --git a/cpp/src/arrow/jemalloc/symbols.map b/cpp/src/arrow/jemalloc/symbols.map
deleted file mode 100644
index 1e87caef9c8c1..0000000000000
--- a/cpp/src/arrow/jemalloc/symbols.map
+++ /dev/null
@@ -1,30 +0,0 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-{
-  # Symbols marked as 'local' are not exported by the DSO and thus may not
-  # be used by client applications.
-  local:
-    # devtoolset / static-libstdc++ symbols
-    __cxa_*;
-
-    extern "C++" {
-      # boost
-      boost::*;
-
-      # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
-      # links c++11 symbols into binaries so that the result may be executed on
-      # a system with an older libstdc++ which doesn't include the necessary
-      # c++11 symbols.
-      std::*;
-    };
-};
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 7992f229862bf..e7de5c4fc589a 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -27,14 +27,21 @@
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
+#ifdef ARROW_JEMALLOC
+// Needed to support jemalloc 3 and 4
+#define JEMALLOC_MANGLE
+#include <jemalloc/jemalloc.h>
+#endif
+
 namespace arrow {
 
+constexpr size_t kAlignment = 64;
+
 namespace {
 // Allocate memory according to the alignment requirements for Arrow
 // (as of May 2016 64 bytes)
 Status AllocateAligned(int64_t size, uint8_t** out) {
-  // TODO(emkornfield) find something compatible with windows
-  constexpr size_t kAlignment = 64;
+// TODO(emkornfield) find something compatible with windows
 #ifdef _MSC_VER
   // Special code path for MSVC
   *out =
@@ -44,6 +51,14 @@ Status AllocateAligned(int64_t size, uint8_t** out) {
     ss << "malloc of size " << size << " failed";
     return Status::OutOfMemory(ss.str());
   }
+#elif defined(ARROW_JEMALLOC)
+  *out = reinterpret_cast<uint8_t*>(mallocx(
+      std::max(static_cast<size_t>(size), kAlignment), MALLOCX_ALIGN(kAlignment)));
+  if (*out == NULL) {
+    std::stringstream ss;
+    ss << "malloc of size " << size << " failed";
+    return Status::OutOfMemory(ss.str());
+  }
 #else
   const int result = posix_memalign(
       reinterpret_cast<void**>(out), kAlignment, static_cast<size_t>(size));
@@ -87,6 +102,14 @@ Status DefaultMemoryPool::Allocate(int64_t size, uint8_t** out) {
 }
 
 Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
+#ifdef ARROW_JEMALLOC
+  *ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
+  if (*ptr == NULL) {
+    std::stringstream ss;
+    ss << "realloc of size " << new_size << " failed";
+    return Status::OutOfMemory(ss.str());
+  }
+#else
   // Note: We cannot use realloc() here as it doesn't guarantee alignment.
 
   // Allocate new chunk
@@ -98,8 +121,9 @@ Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t
   _aligned_free(*ptr);
 #else
   std::free(*ptr);
-#endif
+#endif  // defined(_MSC_VER)
   *ptr = out;
+#endif  // defined(ARROW_JEMALLOC)
 
   bytes_allocated_ += new_size - old_size;
   {
@@ -118,6 +142,8 @@ void DefaultMemoryPool::Free(uint8_t* buffer, int64_t size) {
   DCHECK_GE(bytes_allocated_, size);
 #ifdef _MSC_VER
   _aligned_free(buffer);
+#elif defined(ARROW_JEMALLOC)
+  dallocx(buffer, MALLOCX_ALIGN(kAlignment));
 #else
   std::free(buffer);
 #endif
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index f35326a642f58..224147d8b5c3b 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -51,9 +51,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUILD_PARQUET
     "Build the PyArrow Parquet integration"
     OFF)
-  option(PYARROW_BUILD_JEMALLOC
-    "Build the PyArrow jemalloc integration"
-    OFF)
   option(PYARROW_BUNDLE_ARROW_CPP
     "Bundle the Arrow C++ libraries"
     OFF)
@@ -336,20 +333,6 @@ if (PYARROW_BUILD_PARQUET)
     _parquet)
 endif()
 
-if (PYARROW_BUILD_JEMALLOC)
-  if (PYARROW_BUNDLE_ARROW_CPP)
-    bundle_arrow_lib(ARROW_JEMALLOC_SHARED_LIB)
-  endif()
-  ADD_THIRDPARTY_LIB(arrow_jemalloc
-    SHARED_LIB ${ARROW_JEMALLOC_SHARED_LIB})
-  set(LINK_LIBS
-    ${LINK_LIBS}
-    arrow_jemalloc_shared)
-  set(CYTHON_EXTENSIONS
-    ${CYTHON_EXTENSIONS}
-    _jemalloc)
-endif()
-
 ############################################################
 # Setup and build Cython modules
 ############################################################
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 4c8ed3dad2cdb..c893a1aadb0e2 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -61,11 +61,6 @@ find_library(ARROW_LIB_PATH NAMES arrow
   NO_DEFAULT_PATH)
 get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
 
-find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
-
 find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
   PATHS
   ${ARROW_SEARCH_LIB_PATH}
@@ -104,10 +99,8 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   else()
     set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_LIB_NAME}.a)
     set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_PYTHON_LIB_NAME}.a)
-    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
 
     set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
     set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
 endif()
@@ -116,7 +109,6 @@ if (ARROW_FOUND)
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
     message(STATUS "Found the Arrow Python library: ${ARROW_PYTHON_LIB_PATH}")
-    message(STATUS "Found the Arrow jemalloc library: ${ARROW_JEMALLOC_LIB_PATH}")
   endif ()
 else ()
   if (NOT Arrow_FIND_QUIETLY)
@@ -138,6 +130,4 @@ mark_as_advanced(
   ARROW_SHARED_LIB
   ARROW_PYTHON_STATIC_LIB
   ARROW_PYTHON_SHARED_LIB
-  ARROW_JEMALLOC_STATIC_LIB
-  ARROW_JEMALLOC_SHARED_LIB
 )
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index c1459358b80d3..4810a31ffb5d5 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -194,7 +194,6 @@ Memory Pools
 
    MemoryPool
    default_memory_pool
-   jemalloc_memory_pool
    total_allocated_bytes
    set_memory_pool
 
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 910418cf10804..2063ba821b68e 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -141,7 +141,7 @@ Now, build pyarrow:
 
    cd arrow/python
    python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
-          --with-parquet --with-jemalloc --inplace
+          --with-parquet --inplace
 
 If you did not build parquet-cpp, you can omit ``--with-parquet``.
 
@@ -162,7 +162,6 @@ You should be able to run the unit tests with:
    pyarrow/tests/test_hdfs.py sssssssssssssss
    pyarrow/tests/test_io.py ..................
    pyarrow/tests/test_ipc.py ........
-   pyarrow/tests/test_jemalloc.py ss
    pyarrow/tests/test_parquet.py ....................
    pyarrow/tests/test_scalars.py ..........
    pyarrow/tests/test_schema.py .........
@@ -176,7 +175,7 @@ You can build a wheel by running:
 .. code-block:: shell
 
    python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
-          --with-parquet --with-jemalloc --bundle-arrow-cpp bdist_wheel
+          --with-parquet --bundle-arrow-cpp bdist_wheel
 
 Again, if you did not build parquet-cpp, you should omit ``--with-parquet``.
 
diff --git a/python/doc/source/index.rst b/python/doc/source/index.rst
index 4bfbe44605767..a12853c448209 100644
--- a/python/doc/source/index.rst
+++ b/python/doc/source/index.rst
@@ -44,9 +44,3 @@ structures.
    parquet
    api
    getting_involved
-
-.. toctree::
-   :maxdepth: 2
-   :caption: Additional Features
-
-   jemalloc MemoryPool <jemalloc.rst>
diff --git a/python/doc/source/jemalloc.rst b/python/doc/source/jemalloc.rst
deleted file mode 100644
index 9389dcbd25cfe..0000000000000
--- a/python/doc/source/jemalloc.rst
+++ /dev/null
@@ -1,47 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
-.. or more contributor license agreements.  See the NOTICE file
-.. distributed with this work for additional information
-.. regarding copyright ownership.  The ASF licenses this file
-.. to you under the Apache License, Version 2.0 (the
-.. "License"); you may not use this file except in compliance
-.. with the License.  You may obtain a copy of the License at
-
-..   http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
-.. software distributed under the License is distributed on an
-.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-.. KIND, either express or implied.  See the License for the
-.. specific language governing permissions and limitations
-.. under the License.
-
-jemalloc MemoryPool
-===================
-
-Arrow's default :class:`~pyarrow.MemoryPool` uses the system's allocator
-through the POSIX APIs. Although this already provides aligned allocation, the
-POSIX interface doesn't support aligned reallocation. The default reallocation
-strategy is to allocate a new region, copy over the old data and free the
-previous region. Using `jemalloc <http://jemalloc.net/>`_ we can simply extend
-the existing memory allocation to the requested size. While this may still be
-linear in the size of allocated memory, it is magnitudes faster as only the page
-mapping in the kernel is touched, not the actual data.
-
-The jemalloc-based allocator is not enabled by default to allow the use of the
-system allocator and/or other allocators like ``tcmalloc``. You can either
-explicitly make it the default allocator or pass it only to single operations.
-
-.. code:: python
-
-    import pyarrow as pa
-
-    jemalloc_pool = pyarrow.jemalloc_memory_pool()
-
-    # Explicitly use jemalloc for allocating memory for an Arrow Table object
-    array = pa.Array.from_pylist([1, 2, 3], memory_pool=jemalloc_pool)
-
-    # Set the global pool
-    pyarrow.set_memory_pool(jemalloc_pool)
-    # This operation has no explicit MemoryPool specified and will thus will
-    # also use jemalloc for its allocations.
-    array = pa.Array.from_pylist([1, 2, 3])
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 11e66f7d0cdb1..13919a2408768 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -9,17 +9,17 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License. See accompanying LICENSE file.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-1024
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
 RUN mkdir build-plain
 WORKDIR /arrow/cpp/build-plain
-RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF ..
-RUN make -j5 install
+RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF ..
+RUN ninja install
 
 WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BOOST_USE_SHARED=OFF .
-RUN make -j5 install
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BOOST_USE_SHARED=OFF -GNinja .
+RUN ninja install
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index 2ae7e0ff4f98f..44a988867f03d 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -25,8 +25,9 @@ RUN /build_jemalloc.sh
 
 WORKDIR /
 # Install cmake manylinux1 package
-RUN /opt/python/cp35-cp35m/bin/pip install cmake
+RUN /opt/python/cp35-cp35m/bin/pip install cmake ninja
 RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
+RUN ln -s /opt/python/cp35-cp35m/bin/ninja /usr/bin/ninja
 
 ADD scripts/build_gtest.sh /
 RUN /build_gtest.sh
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 4565e59bea222..8c6bda9550e87 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -35,7 +35,6 @@ cd /arrow/python
 # PyArrow build configuration
 export PYARROW_BUILD_TYPE='release'
 export PYARROW_WITH_PARQUET=1
-export PYARROW_WITH_JEMALLOC=1
 export PYARROW_BUNDLE_ARROW_CPP=1
 # Need as otherwise arrow_io is sometimes not linked
 export LDFLAGS="-Wl,--no-as-needed"
@@ -60,13 +59,12 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     # Clear output directory
     rm -rf dist/
     echo "=== (${PYTHON}) Building wheel ==="
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-jemalloc --bundle-arrow-cpp
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
     echo "=== (${PYTHON}) Test the existence of optional modules ==="
     $PIPI_IO -r requirements.txt
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow._jemalloc"
 
     echo "=== (${PYTHON}) Tag the wheel with manylinux1 ==="
     mkdir -p repaired_wheels/
@@ -75,7 +73,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     echo "=== (${PYTHON}) Testing manylinux1 wheel ==="
     source /venv-test-${PYTHON}/bin/activate
     pip install repaired_wheels/*.whl
-    py.test /venv-test-${PYTHON}/lib/*/site-packages/pyarrow
+    py.test --parquet /venv-test-${PYTHON}/lib/*/site-packages/pyarrow
     deactivate
 
     mv repaired_wheels/*.whl /io/dist
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 0f34121f653c2..237d44f01ca48 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -90,15 +90,6 @@
                          ArrowTypeError)
 
 
-def jemalloc_memory_pool():
-    """
-    Returns a jemalloc-based memory allocator, which can be passed to
-    pyarrow.set_memory_pool
-    """
-    from pyarrow._jemalloc import default_pool
-    return default_pool()
-
-
 from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
 
 from pyarrow.ipc import (RecordBatchFileReader, RecordBatchFileWriter,
diff --git a/python/pyarrow/_jemalloc.pyx b/python/pyarrow/_jemalloc.pyx
deleted file mode 100644
index 6f00c9d2dded8..0000000000000
--- a/python/pyarrow/_jemalloc.pyx
+++ /dev/null
@@ -1,28 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# cython: profile=False
-# distutils: language = c++
-# cython: embedsignature = True
-
-from pyarrow.includes.libarrow_jemalloc cimport CJemallocMemoryPool
-from pyarrow.lib cimport MemoryPool
-
-def default_pool():
-    cdef MemoryPool pool = MemoryPool()
-    pool.init(CJemallocMemoryPool.default_pool())
-    return pool
diff --git a/python/pyarrow/includes/libarrow_jemalloc.pxd b/python/pyarrow/includes/libarrow_jemalloc.pxd
deleted file mode 100644
index 0609d1907589a..0000000000000
--- a/python/pyarrow/includes/libarrow_jemalloc.pxd
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# distutils: language = c++
-
-from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport *
-    
-cdef extern from "arrow/jemalloc/memory_pool.h" namespace "arrow::jemalloc" nogil:
-    cdef cppclass CJemallocMemoryPool" arrow::jemalloc::MemoryPool":
-        int64_t bytes_allocated()
-        @staticmethod
-        CMemoryPool* default_pool()
diff --git a/python/pyarrow/tests/test_jemalloc.py b/python/pyarrow/tests/test_jemalloc.py
deleted file mode 100644
index 50eb74ae0e2c6..0000000000000
--- a/python/pyarrow/tests/test_jemalloc.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import gc
-import pytest
-
-import pyarrow as pa
-
-
-try:
-    pa.jemalloc_memory_pool()
-    HAVE_JEMALLOC = True
-except ImportError:
-    HAVE_JEMALLOC = False
-
-
-jemalloc = pytest.mark.skipif(not HAVE_JEMALLOC,
-                              reason='jemalloc support not built')
-
-
-@jemalloc
-def test_different_memory_pool():
-    gc.collect()
-    bytes_before_default = pa.total_allocated_bytes()
-    bytes_before_jemalloc = pa.jemalloc_memory_pool().bytes_allocated()
-
-    # it works
-    array = pa.array([1, None, 3, None],  # noqa
-                     memory_pool=pa.jemalloc_memory_pool())
-    gc.collect()
-    assert pa.total_allocated_bytes() == bytes_before_default
-    assert (pa.jemalloc_memory_pool().bytes_allocated() >
-            bytes_before_jemalloc)
-
-
-@jemalloc
-def test_default_memory_pool():
-    gc.collect()
-    bytes_before_default = pa.total_allocated_bytes()
-    bytes_before_jemalloc = pa.jemalloc_memory_pool().bytes_allocated()
-
-    old_memory_pool = pa.default_memory_pool()
-    pa.set_memory_pool(pa.jemalloc_memory_pool())
-
-    array = pa.array([1, None, 3, None])  # noqa
-
-    pa.set_memory_pool(old_memory_pool)
-    gc.collect()
-
-    assert pa.total_allocated_bytes() == bytes_before_default
-
-    assert (pa.jemalloc_memory_pool().bytes_allocated() >
-            bytes_before_jemalloc)
diff --git a/python/setup.py b/python/setup.py
index eddf88ab16b19..1ea57ae2d858d 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -82,7 +82,6 @@ def run(self):
     user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
                      ('build-type=', None, 'build type (debug or release)'),
                      ('with-parquet', None, 'build the Parquet extension'),
-                     ('with-jemalloc', None, 'build the jemalloc extension'),
                      ('bundle-arrow-cpp', None,
                       'bundle the Arrow C++ libraries')] +
                     _build_ext.user_options)
@@ -100,14 +99,11 @@ def initialize_options(self):
 
         self.with_parquet = strtobool(
             os.environ.get('PYARROW_WITH_PARQUET', '0'))
-        self.with_jemalloc = strtobool(
-            os.environ.get('PYARROW_WITH_JEMALLOC', '0'))
         self.bundle_arrow_cpp = strtobool(
             os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
 
     CYTHON_MODULE_NAMES = [
         'lib',
-        '_jemalloc',
         '_parquet']
 
     def _run_cmake(self):
@@ -143,9 +139,6 @@ def _run_cmake(self):
         if self.with_parquet:
             cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
 
-        if self.with_jemalloc:
-            cmake_options.append('-DPYARROW_BUILD_JEMALLOC=on')
-
         if self.bundle_arrow_cpp:
             cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
             # ARROW-1090: work around CMake rough edges
@@ -249,8 +242,6 @@ def move_lib(lib_name):
             shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
             move_lib("arrow")
             move_lib("arrow_python")
-            if self.with_jemalloc:
-                move_lib("arrow_jemalloc")
             if self.with_parquet:
                 move_lib("parquet")
 
@@ -284,8 +275,6 @@ def move_lib(lib_name):
     def _failure_permitted(self, name):
         if name == '_parquet' and not self.with_parquet:
             return True
-        if name == '_jemalloc' and not self.with_jemalloc:
-            return True
         return False
 
     def _get_inplace_dir(self):
diff --git a/site/README.md b/site/README.md
index aeebaa1790ef9..0e052c84aebd9 100644
--- a/site/README.md
+++ b/site/README.md
@@ -73,11 +73,11 @@ rsync -r html/ ../../site/asf-site/docs/cpp
 
 #### Python
 
-First, build PyArrow with all optional extensions (Apache Parquet, jemalloc).
+First, build PyArrow with all optional extensions (Apache Parquet).
 
 ```
 cd ../python
-python setup.py build_ext --inplace --with-parquet --with-jemalloc
+python setup.py build_ext --inplace --with-parquet
 python setup.py build_sphinx -s doc/source
 rsync -r doc/_build/html/ ../site/asf-site/docs/python/
 ```

From 5e343098187cb822017f359748e28c53ece70e75 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Thu, 22 Jun 2017 09:35:34 -0400
Subject: [PATCH 0734/1644] ARROW-1104: Integrate in-memory object store into
 arrow

This supersedes https://github.com/apache/arrow/pull/467

This is ready for review. Next steps are
- Integration with the arrow CI
- Write docs on how to use the object store

There is one remaining compilation error (it doesn't find Python.h for one of the Travis configurations, if anybody has an idea on what is going on, let me know).

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #742 from pcmoritz/plasma-store-2 and squashes the following commits:

c100a453 [Philipp Moritz] fixes
d67160c5 [Philipp Moritz] build dlmalloc with -O3
16d1f716 [Philipp Moritz] fix test hanging
0f321e16 [Philipp Moritz] try to fix tests
80f9df40 [Philipp Moritz] make format
4c474d71 [Philipp Moritz] run plasma_store from the right directory
85aa1710 [Philipp Moritz] fix mac tests
61d421b5 [Philipp Moritz] fix formatting
4497e337 [Philipp Moritz] fix tests
00f17f24 [Philipp Moritz] fix licenses
81437920 [Philipp Moritz] fix linting
5370ae06 [Philipp Moritz] fix plasma protocol
a137e783 [Philipp Moritz] more fixes
b36c6aaa [Philipp Moritz] fix fling.cc
214c426c [Philipp Moritz] fix eviction policy
e7badc48 [Philipp Moritz] fix python extension
6432d3fa [Philipp Moritz] fix formatting
b21f0814 [Philipp Moritz] fix remaining comments about client
27f9c9e8 [Philipp Moritz] fix formatting
7b08fd2a [Philipp Moritz] replace ObjectID pass by value with pass by const reference and fix const correctness
ca80e9a6 [Philipp Moritz] remove plain pointer in plasma client, part II
627b7c75 [Philipp Moritz] fix python extension name
30bd68b7 [Philipp Moritz] remove plain pointer in plasma client, part I
77d98227 [Philipp Moritz] put all the object code into a common library
0fdd4cd5 [Philipp Moritz] link libarrow.a and remove hardcoded optimization flags
8daea699 [Philipp Moritz] fix includes according to google styleguide
65ac7433 [Philipp Moritz] remove offending c++ flag from c flags
7003a4a4 [Philipp Moritz] fix valgrind test by setting working directory
217ff3d8 [Philipp Moritz] add valgrind heuristic
9c703c20 [Philipp Moritz] integrate client tests
9e5ae0e1 [Philipp Moritz] port serialization tests to gtest
0b8593db [Robert Nishihara] Port change from Ray. Change listen backlog size from 5 to 128.
b9a5a06e [Philipp Moritz] fix includes
ed680f97 [Philipp Moritz] reformat the code
f40f85bd [Philipp Moritz] add clang-format exceptions
d6e60d26 [Philipp Moritz] do not compile plasma on windows
f936adb7 [Philipp Moritz] build plasma python client only if python is available
e11b0e86 [Philipp Moritz] fix pthread
74ecb199 [Philipp Moritz] don't link against Python libraries
b1e0335a [Philipp Moritz] fix linting
7f7e7e78 [Philipp Moritz] more linting
79ea0ca7 [Philipp Moritz] fix clang-tidy
99420e8f [Philipp Moritz] add rat exceptions
6cee1e25 [Philipp Moritz] fix
c93034fb [Philipp Moritz] add Apache 2.0 headers
63729130 [Philipp Moritz] fix malloc?
99537c94 [Philipp Moritz] fix compiler warnings
cb3f3a38 [Philipp Moritz] compile C files with CMAKE_C_FLAGS
e649c2af [Philipp Moritz] fix compilation
04c2edb3 [Philipp Moritz] add missing file
51ab9630 [Philipp Moritz] fix compiler warnings
9ef7f412 [Philipp Moritz] make the plasma store compile
e9f9bb4a [Philipp Moritz] Initial commit of the plasma store. Contributors: Philipp Moritz, Robert Nishihara, Richard Shin, Stephanie Wang, Alexey Tumanov, Ion Stoica @ RISElab, UC Berkeley (2017) [from https://github.com/ray-project/ray/commit/b94b4a35e04d8d2c0af4420518a4e9a94c1c9b9f]
---
 LICENSE.txt                                |   96 +
 ci/travis_before_script_cpp.sh             |    5 +-
 ci/travis_script_python.sh                 |    1 +
 cpp/CMakeLists.txt                         |   20 +-
 cpp/src/arrow/status.h                     |   23 +
 cpp/src/arrow/util/logging.h               |    8 +-
 cpp/src/plasma/CMakeLists.txt              |  112 +
 cpp/src/plasma/client.cc                   |  557 ++
 cpp/src/plasma/client.h                    |  343 ++
 cpp/src/plasma/common.cc                   |   83 +
 cpp/src/plasma/common.h                    |   63 +
 cpp/src/plasma/events.cc                   |   81 +
 cpp/src/plasma/events.h                    |   99 +
 cpp/src/plasma/eviction_policy.cc          |  107 +
 cpp/src/plasma/eviction_policy.h           |  134 +
 cpp/src/plasma/extension.cc                |  456 ++
 cpp/src/plasma/extension.h                 |   50 +
 cpp/src/plasma/fling.cc                    |   90 +
 cpp/src/plasma/fling.h                     |   52 +
 cpp/src/plasma/format/common.fbs           |   34 +
 cpp/src/plasma/format/plasma.fbs           |  291 +
 cpp/src/plasma/io.cc                       |  212 +
 cpp/src/plasma/io.h                        |   55 +
 cpp/src/plasma/malloc.cc                   |  178 +
 cpp/src/plasma/malloc.h                    |   26 +
 cpp/src/plasma/plasma.cc                   |   64 +
 cpp/src/plasma/plasma.h                    |  191 +
 cpp/src/plasma/protocol.cc                 |  502 ++
 cpp/src/plasma/protocol.h                  |  170 +
 cpp/src/plasma/store.cc                    |  681 +++
 cpp/src/plasma/store.h                     |  169 +
 cpp/src/plasma/test/client_tests.cc        |  132 +
 cpp/src/plasma/test/run_tests.sh           |   61 +
 cpp/src/plasma/test/run_valgrind.sh        |   27 +
 cpp/src/plasma/test/serialization_tests.cc |  388 ++
 cpp/src/plasma/thirdparty/ae/ae.c          |  465 ++
 cpp/src/plasma/thirdparty/ae/ae.h          |  123 +
 cpp/src/plasma/thirdparty/ae/ae_epoll.c    |  135 +
 cpp/src/plasma/thirdparty/ae/ae_evport.c   |  320 +
 cpp/src/plasma/thirdparty/ae/ae_kqueue.c   |  138 +
 cpp/src/plasma/thirdparty/ae/ae_select.c   |  106 +
 cpp/src/plasma/thirdparty/ae/config.h      |   54 +
 cpp/src/plasma/thirdparty/ae/zmalloc.h     |   45 +
 cpp/src/plasma/thirdparty/dlmalloc.c       | 6281 ++++++++++++++++++++
 cpp/src/plasma/thirdparty/xxhash.cc        |  889 +++
 cpp/src/plasma/thirdparty/xxhash.h         |  293 +
 dev/release/run-rat.sh                     |   11 +
 47 files changed, 14411 insertions(+), 10 deletions(-)
 create mode 100644 cpp/src/plasma/CMakeLists.txt
 create mode 100644 cpp/src/plasma/client.cc
 create mode 100644 cpp/src/plasma/client.h
 create mode 100644 cpp/src/plasma/common.cc
 create mode 100644 cpp/src/plasma/common.h
 create mode 100644 cpp/src/plasma/events.cc
 create mode 100644 cpp/src/plasma/events.h
 create mode 100644 cpp/src/plasma/eviction_policy.cc
 create mode 100644 cpp/src/plasma/eviction_policy.h
 create mode 100644 cpp/src/plasma/extension.cc
 create mode 100644 cpp/src/plasma/extension.h
 create mode 100644 cpp/src/plasma/fling.cc
 create mode 100644 cpp/src/plasma/fling.h
 create mode 100644 cpp/src/plasma/format/common.fbs
 create mode 100644 cpp/src/plasma/format/plasma.fbs
 create mode 100644 cpp/src/plasma/io.cc
 create mode 100644 cpp/src/plasma/io.h
 create mode 100644 cpp/src/plasma/malloc.cc
 create mode 100644 cpp/src/plasma/malloc.h
 create mode 100644 cpp/src/plasma/plasma.cc
 create mode 100644 cpp/src/plasma/plasma.h
 create mode 100644 cpp/src/plasma/protocol.cc
 create mode 100644 cpp/src/plasma/protocol.h
 create mode 100644 cpp/src/plasma/store.cc
 create mode 100644 cpp/src/plasma/store.h
 create mode 100644 cpp/src/plasma/test/client_tests.cc
 create mode 100644 cpp/src/plasma/test/run_tests.sh
 create mode 100644 cpp/src/plasma/test/run_valgrind.sh
 create mode 100644 cpp/src/plasma/test/serialization_tests.cc
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae.h
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae_epoll.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae_evport.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae_kqueue.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae_select.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/config.h
 create mode 100644 cpp/src/plasma/thirdparty/ae/zmalloc.h
 create mode 100644 cpp/src/plasma/thirdparty/dlmalloc.c
 create mode 100644 cpp/src/plasma/thirdparty/xxhash.cc
 create mode 100644 cpp/src/plasma/thirdparty/xxhash.h

diff --git a/LICENSE.txt b/LICENSE.txt
index d645695673349..700073379376a 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -200,3 +200,99 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+--------------------------------------------------------------------------------
+
+src/plasma/fling.cc and src/plasma/fling.h: Apache 2.0
+
+Copyright 2013 Sharvil Nanavati
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+src/plasma/thirdparty/ae: Modified / 3-Clause BSD
+
+Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+ * Neither the name of Redis nor the names of its contributors may be used
+   to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+src/plasma/thirdparty/dlmalloc.c: CC0
+
+This is a version (aka dlmalloc) of malloc/free/realloc written by
+Doug Lea and released to the public domain, as explained at
+http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
+comments, complaints, performance data, etc to dl@cs.oswego.edu
+
+--------------------------------------------------------------------------------
+
+src/plasma/thirdparty/xxhash: BSD 2-Clause License
+
+xxHash - Fast Hash algorithm
+Copyright (C) 2012-2016, Yann Collet
+
+BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at :
+- xxHash homepage: http://www.xxhash.com
+- xxHash source repository : https://github.com/Cyan4973/xxHash
+
+--------------------------------------------------------------------------------
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 99087350689db..a38a0dc132817 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -26,12 +26,11 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 if [ $only_library_mode == "no" ]; then
   # C++ toolchain
   export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
-  export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
   export RAPIDJSON_HOME=$CPP_TOOLCHAIN
 
   # Set up C++ toolchain from conda-forge packages for faster builds
   source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-  conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson
+  conda create -y -q -p $CPP_TOOLCHAIN python=2.7 rapidjson
 fi
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
@@ -73,7 +72,7 @@ else
           $ARROW_CPP_DIR
 fi
 
-make -j4
+make VERBOSE=1 -j4
 make install
 
 popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 6cc760f0ac23f..b82653dfb01b5 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -72,6 +72,7 @@ function build_arrow_libraries() {
 
   cmake -DARROW_BUILD_TESTS=off \
         -DARROW_PYTHON=on \
+        -DPLASMA_PYTHON=on \
         -DCMAKE_INSTALL_PREFIX=$2 \
         $CPP_DIR
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 962891acf9cf2..0897e99ae06e6 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -187,6 +187,8 @@ include(san-config)
 
 # For any C code, use the same flags.
 set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
+# Remove --std=c++11 to avoid errors from C compilers
+string(REPLACE "-std=c++11" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
 
 # Code coverage
 if ("${ARROW_GENERATE_COVERAGE}")
@@ -362,7 +364,7 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
       APPEND_STRING PROPERTY
       COMPILE_FLAGS " -DARROW_VALGRIND")
     add_test(${TEST_NAME}
-      valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
+      bash -c "cd ${EXECUTABLE_OUTPUT_PATH}; valgrind --tool=memcheck --leak-check=full --leak-check-heuristics=stdstring --error-exitcode=1 ${TEST_PATH}")
   elseif(MSVC)
     add_test(${TEST_NAME} ${TEST_PATH})
   else()
@@ -707,6 +709,7 @@ if (ARROW_IPC)
     ExternalProject_Add(flatbuffers_ep
       URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
       CMAKE_ARGS
+      "-DCMAKE_CXX_FLAGS=-fPIC"
       "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
       "-DFLATBUFFERS_BUILD_TESTS=OFF")
 
@@ -871,7 +874,12 @@ if (UNIX)
 
   FOREACH(item ${LINT_FILES})
     IF(NOT ((item MATCHES "_generated.h") OR
-            (item MATCHES "pyarrow_api.h")))
+            (item MATCHES "pyarrow_api.h") OR
+            (item MATCHES "xxhash.h") OR
+            (item MATCHES "xxhash.cc") OR
+            (item MATCHES "config.h") OR
+            (item MATCHES "zmalloc.h") OR
+            (item MATCHES "ae.h")))
       LIST(APPEND FILTERED_LINT_FILES ${item})
     ENDIF()
   ENDFOREACH(item ${LINT_FILES})
@@ -899,7 +907,10 @@ if (${CLANG_FORMAT_FOUND})
     sed -e '/windows_compatibility.h/g' |
     sed -e '/pyarrow_api.h/g' |
     sed -e '/config.h/g' |   # python/config.h
-    sed -e '/platform.h/g'`  # python/platform.h
+    sed -e '/platform.h/g' |  # python/platform.h
+    sed -e '/ae.h/g' |
+    sed -e '/xxhash.cc/g' |
+    sed -e '/xxhash.h/g'`
     )
 
   # runs clang format and exits with a non-zero exit code if any files need to be reformatted
@@ -1002,6 +1013,9 @@ if(FLATBUFFERS_VENDORED)
   set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} flatbuffers_ep)
 endif()
 
+if(NOT WIN32)
+  add_subdirectory(src/plasma)
+endif()
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
 if (ARROW_IPC)
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 1688b96785367..7e7f67ce62800 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -83,6 +83,9 @@ enum class StatusCode : char {
   IOError = 5,
   UnknownError = 9,
   NotImplemented = 10,
+  PlasmaObjectExists = 20,
+  PlasmaObjectNonexistent = 21,
+  PlasmaStoreFull = 22
 };
 
 class ARROW_EXPORT Status {
@@ -129,6 +132,18 @@ class ARROW_EXPORT Status {
     return Status(StatusCode::IOError, msg, -1);
   }
 
+  static Status PlasmaObjectExists(const std::string& msg) {
+    return Status(StatusCode::PlasmaObjectExists, msg, -1);
+  }
+
+  static Status PlasmaObjectNonexistent(const std::string& msg) {
+    return Status(StatusCode::PlasmaObjectNonexistent, msg, -1);
+  }
+
+  static Status PlasmaStoreFull(const std::string& msg) {
+    return Status(StatusCode::PlasmaStoreFull, msg, -1);
+  }
+
   // Returns true iff the status indicates success.
   bool ok() const { return (state_ == NULL); }
 
@@ -139,6 +154,14 @@ class ARROW_EXPORT Status {
   bool IsTypeError() const { return code() == StatusCode::TypeError; }
   bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
   bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
+  // An object with this object ID already exists in the plasma store.
+  bool IsPlasmaObjectExists() const { return code() == StatusCode::PlasmaObjectExists; }
+  // An object was requested that doesn't exist in the plasma store.
+  bool IsPlasmaObjectNonexistent() const {
+    return code() == StatusCode::PlasmaObjectNonexistent;
+  }
+  // An object is too large to fit into the plasma store.
+  bool IsPlasmaStoreFull() const { return code() == StatusCode::PlasmaStoreFull; }
 
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 697d47c541003..49f1699f13623 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -30,6 +30,7 @@ namespace arrow {
 
 // Log levels. LOG ignores them, so their values are abitrary.
 
+#define ARROW_DEBUG (-1)
 #define ARROW_INFO 0
 #define ARROW_WARNING 1
 #define ARROW_ERROR 2
@@ -38,10 +39,9 @@ namespace arrow {
 #define ARROW_LOG_INTERNAL(level) ::arrow::internal::CerrLog(level)
 #define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level)
 
-#define ARROW_CHECK(condition)                           \
-  (condition) ? 0                                        \
-              : ::arrow::internal::FatalLog(ARROW_FATAL) \
-                    << __FILE__ << __LINE__ << " Check failed: " #condition " "
+#define ARROW_CHECK(condition)                               \
+  (condition) ? 0 : ::arrow::internal::FatalLog(ARROW_FATAL) \
+                        << __FILE__ << __LINE__ << " Check failed: " #condition " "
 
 #ifdef NDEBUG
 #define ARROW_DFATAL ARROW_WARNING
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
new file mode 100644
index 0000000000000..992c33ed0fcc9
--- /dev/null
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -0,0 +1,112 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 2.8)
+
+project(plasma)
+
+find_package(PythonLibsNew REQUIRED)
+find_package(Threads)
+
+option(PLASMA_PYTHON
+  "Build the Plasma Python extensions"
+  OFF)
+
+if(APPLE)
+  SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
+endif(APPLE)
+
+include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
+include_directories("${FLATBUFFERS_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/" "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" "${CMAKE_CURRENT_LIST_DIR}/../")
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L")
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-conversion")
+
+# Compile flatbuffers
+
+set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
+set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
+
+set(PLASMA_FBS_OUTPUT_FILES
+  "${OUTPUT_DIR}/common_generated.h"
+  "${OUTPUT_DIR}/plasma_generated.h")
+
+add_custom_command(
+  OUTPUT ${PLASMA_FBS_OUTPUT_FILES}
+  # The --gen-object-api flag generates a C++ class MessageT for each
+  # flatbuffers message Message, which can be used to store deserialized
+  # messages in data structures. This is currently used for ObjectInfo for
+  # example.
+  COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} --gen-object-api
+  DEPENDS ${PLASMA_FBS_SRC}
+  COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}"
+  VERBATIM)
+
+add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
+
+add_dependencies(gen_plasma_fbs flatbuffers_ep)
+
+if(UNIX AND NOT APPLE)
+  link_libraries(rt)
+endif()
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
+set_source_files_properties(thirdparty/dlmalloc.c PROPERTIES COMPILE_FLAGS -Wno-all)
+set_source_files_properties(extension.cc PROPERTIES COMPILE_FLAGS -Wno-strict-aliasing)
+
+set(PLASMA_SRCS
+  client.cc
+  common.cc
+  eviction_policy.cc
+  events.cc
+  fling.cc
+  io.cc
+  malloc.cc
+  plasma.cc
+  protocol.cc
+  thirdparty/ae/ae.c
+  thirdparty/xxhash.cc)
+
+ADD_ARROW_LIB(plasma
+  SOURCES ${PLASMA_SRCS}
+  DEPENDENCIES gen_plasma_fbs
+  SHARED_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static
+  STATIC_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static)
+
+# The optimization flag -O3 is suggested by dlmalloc.c, which is #included in
+# malloc.cc; we set it here regardless of whether we do a debug or release build.
+set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error=conversion -O3")
+
+add_executable(plasma_store store.cc)
+target_link_libraries(plasma_store plasma_static)
+
+ADD_ARROW_TEST(test/serialization_tests)
+ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static)
+ADD_ARROW_TEST(test/client_tests)
+ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static)
+
+if(PLASMA_PYTHON)
+  add_library(plasma_extension SHARED extension.cc)
+
+  if(APPLE)
+    target_link_libraries(plasma_extension plasma_static "-undefined dynamic_lookup")
+  else(APPLE)
+    target_link_libraries(plasma_extension plasma_static -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive)
+  endif(APPLE)
+endif()
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
new file mode 100644
index 0000000000000..dcb78e7ec52c6
--- /dev/null
+++ b/cpp/src/plasma/client.cc
@@ -0,0 +1,557 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// PLASMA CLIENT: Client library for using the plasma store and manager
+
+#include "plasma/client.h"
+
+#ifdef _WIN32
+#include <Win32_Interop/win32_types.h>
+#endif
+
+#include <assert.h>
+#include <fcntl.h>
+#include <netinet/in.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <thread>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/fling.h"
+#include "plasma/io.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+#define XXH_STATIC_LINKING_ONLY
+#include "thirdparty/xxhash.h"
+
+#define XXH64_DEFAULT_SEED 0
+
+// Number of threads used for memcopy and hash computations.
+constexpr int64_t kThreadPoolSize = 8;
+constexpr int64_t kBytesInMB = 1 << 20;
+static std::vector<std::thread> threadpool_(kThreadPoolSize);
+
+// If the file descriptor fd has been mmapped in this client process before,
+// return the pointer that was returned by mmap, otherwise mmap it and store the
+// pointer in a hash table.
+uint8_t* PlasmaClient::lookup_or_mmap(int fd, int store_fd_val, int64_t map_size) {
+  auto entry = mmap_table_.find(store_fd_val);
+  if (entry != mmap_table_.end()) {
+    close(fd);
+    return entry->second.pointer;
+  } else {
+    uint8_t* result = reinterpret_cast<uint8_t*>(
+        mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
+    // TODO(pcm): Don't fail here, instead return a Status.
+    if (result == MAP_FAILED) { ARROW_LOG(FATAL) << "mmap failed"; }
+    close(fd);
+    ClientMmapTableEntry& entry = mmap_table_[store_fd_val];
+    entry.pointer = result;
+    entry.length = map_size;
+    entry.count = 0;
+    return result;
+  }
+}
+
+// Get a pointer to a file that we know has been memory mapped in this client
+// process before.
+uint8_t* PlasmaClient::lookup_mmapped_file(int store_fd_val) {
+  auto entry = mmap_table_.find(store_fd_val);
+  ARROW_CHECK(entry != mmap_table_.end());
+  return entry->second.pointer;
+}
+
+void PlasmaClient::increment_object_count(
+    const ObjectID& object_id, PlasmaObject* object, bool is_sealed) {
+  // Increment the count of the object to track the fact that it is being used.
+  // The corresponding decrement should happen in PlasmaClient::Release.
+  auto elem = objects_in_use_.find(object_id);
+  ObjectInUseEntry* object_entry;
+  if (elem == objects_in_use_.end()) {
+    // Add this object ID to the hash table of object IDs in use. The
+    // corresponding call to free happens in PlasmaClient::Release.
+    objects_in_use_[object_id] =
+        std::unique_ptr<ObjectInUseEntry>(new ObjectInUseEntry());
+    objects_in_use_[object_id]->object = *object;
+    objects_in_use_[object_id]->count = 0;
+    objects_in_use_[object_id]->is_sealed = is_sealed;
+    object_entry = objects_in_use_[object_id].get();
+    // Increment the count of the number of objects in the memory-mapped file
+    // that are being used. The corresponding decrement should happen in
+    // PlasmaClient::Release.
+    auto entry = mmap_table_.find(object->handle.store_fd);
+    ARROW_CHECK(entry != mmap_table_.end());
+    ARROW_CHECK(entry->second.count >= 0);
+    // Update the in_use_object_bytes_.
+    in_use_object_bytes_ +=
+        (object_entry->object.data_size + object_entry->object.metadata_size);
+    entry->second.count += 1;
+  } else {
+    object_entry = elem->second.get();
+    ARROW_CHECK(object_entry->count > 0);
+  }
+  // Increment the count of the number of instances of this object that are
+  // being used by this client. The corresponding decrement should happen in
+  // PlasmaClient::Release.
+  object_entry->count += 1;
+}
+
+Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
+    uint8_t* metadata, int64_t metadata_size, uint8_t** data) {
+  ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
+                   << data_size << " and metadata size " << metadata_size;
+  RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaCreateReply, &buffer));
+  ObjectID id;
+  PlasmaObject object;
+  RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object));
+  // If the CreateReply included an error, then the store will not send a file
+  // descriptor.
+  int fd = recv_fd(store_conn_);
+  ARROW_CHECK(fd >= 0) << "recv not successful";
+  ARROW_CHECK(object.data_size == data_size);
+  ARROW_CHECK(object.metadata_size == metadata_size);
+  // The metadata should come right after the data.
+  ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
+  *data = lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) +
+          object.data_offset;
+  // If plasma_create is being called from a transfer, then we will not copy the
+  // metadata here. The metadata will be written along with the data streamed
+  // from the transfer.
+  if (metadata != NULL) {
+    // Copy the metadata to the buffer.
+    memcpy(*data + object.data_size, metadata, metadata_size);
+  }
+  // Increment the count of the number of instances of this object that this
+  // client is using. A call to PlasmaClient::Release is required to decrement
+  // this
+  // count. Cache the reference to the object.
+  increment_object_count(object_id, &object, false);
+  // We increment the count a second time (and the corresponding decrement will
+  // happen in a PlasmaClient::Release call in plasma_seal) so even if the
+  // buffer
+  // returned by PlasmaClient::Dreate goes out of scope, the object does not get
+  // released before the call to PlasmaClient::Seal happens.
+  increment_object_count(object_id, &object, false);
+  return Status::OK();
+}
+
+Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
+    int64_t timeout_ms, ObjectBuffer* object_buffers) {
+  // Fill out the info for the objects that are already in use locally.
+  bool all_present = true;
+  for (int i = 0; i < num_objects; ++i) {
+    auto object_entry = objects_in_use_.find(object_ids[i]);
+    if (object_entry == objects_in_use_.end()) {
+      // This object is not currently in use by this client, so we need to send
+      // a request to the store.
+      all_present = false;
+      // Make a note to ourselves that the object is not present.
+      object_buffers[i].data_size = -1;
+    } else {
+      // NOTE: If the object is still unsealed, we will deadlock, since we must
+      // have been the one who created it.
+      ARROW_CHECK(object_entry->second->is_sealed)
+          << "Plasma client called get on an unsealed object that it created";
+      PlasmaObject* object = &object_entry->second->object;
+      object_buffers[i].data = lookup_mmapped_file(object->handle.store_fd);
+      object_buffers[i].data = object_buffers[i].data + object->data_offset;
+      object_buffers[i].data_size = object->data_size;
+      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
+      object_buffers[i].metadata_size = object->metadata_size;
+      // Increment the count of the number of instances of this object that this
+      // client is using. A call to PlasmaClient::Release is required to
+      // decrement this
+      // count. Cache the reference to the object.
+      increment_object_count(object_ids[i], object, true);
+    }
+  }
+
+  if (all_present) { return Status::OK(); }
+
+  // If we get here, then the objects aren't all currently in use by this
+  // client, so we need to send a request to the plasma store.
+  RETURN_NOT_OK(SendGetRequest(store_conn_, object_ids, num_objects, timeout_ms));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaGetReply, &buffer));
+  std::vector<ObjectID> received_object_ids(num_objects);
+  std::vector<PlasmaObject> object_data(num_objects);
+  PlasmaObject* object;
+  RETURN_NOT_OK(ReadGetReply(
+      buffer.data(), received_object_ids.data(), object_data.data(), num_objects));
+
+  for (int i = 0; i < num_objects; ++i) {
+    DCHECK(received_object_ids[i] == object_ids[i]);
+    object = &object_data[i];
+    if (object_buffers[i].data_size != -1) {
+      // If the object was already in use by the client, then the store should
+      // have returned it.
+      DCHECK_NE(object->data_size, -1);
+      // We won't use this file descriptor, but the store sent us one, so we
+      // need to receive it and then close it right away so we don't leak file
+      // descriptors.
+      int fd = recv_fd(store_conn_);
+      close(fd);
+      ARROW_CHECK(fd >= 0);
+      // We've already filled out the information for this object, so we can
+      // just continue.
+      continue;
+    }
+    // If we are here, the object was not currently in use, so we need to
+    // process the reply from the object store.
+    if (object->data_size != -1) {
+      // The object was retrieved. The user will be responsible for releasing
+      // this object.
+      int fd = recv_fd(store_conn_);
+      ARROW_CHECK(fd >= 0);
+      object_buffers[i].data =
+          lookup_or_mmap(fd, object->handle.store_fd, object->handle.mmap_size);
+      // Finish filling out the return values.
+      object_buffers[i].data = object_buffers[i].data + object->data_offset;
+      object_buffers[i].data_size = object->data_size;
+      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
+      object_buffers[i].metadata_size = object->metadata_size;
+      // Increment the count of the number of instances of this object that this
+      // client is using. A call to PlasmaClient::Release is required to
+      // decrement this
+      // count. Cache the reference to the object.
+      increment_object_count(received_object_ids[i], object, true);
+    } else {
+      // The object was not retrieved. Make sure we already put a -1 here to
+      // indicate that the object was not retrieved. The caller is not
+      // responsible for releasing this object.
+      DCHECK_EQ(object_buffers[i].data_size, -1);
+      object_buffers[i].data_size = -1;
+    }
+  }
+  return Status::OK();
+}
+
+/// This is a helper method for implementing plasma_release. We maintain a
+/// buffer
+/// of release calls and only perform them once the buffer becomes full (as
+/// judged by the aggregate sizes of the objects). There may be multiple release
+/// calls for the same object ID in the buffer. In this case, the first release
+/// calls will not do anything. The client will only send a message to the store
+/// releasing the object when the client is truly done with the object.
+///
+/// @param conn The plasma connection.
+/// @param object_id The object ID to attempt to release.
+Status PlasmaClient::PerformRelease(const ObjectID& object_id) {
+  // Decrement the count of the number of instances of this object that are
+  // being used by this client. The corresponding increment should have happened
+  // in PlasmaClient::Get.
+  auto object_entry = objects_in_use_.find(object_id);
+  ARROW_CHECK(object_entry != objects_in_use_.end());
+  object_entry->second->count -= 1;
+  ARROW_CHECK(object_entry->second->count >= 0);
+  // Check if the client is no longer using this object.
+  if (object_entry->second->count == 0) {
+    // Decrement the count of the number of objects in this memory-mapped file
+    // that the client is using. The corresponding increment should have
+    // happened in plasma_get.
+    int fd = object_entry->second->object.handle.store_fd;
+    auto entry = mmap_table_.find(fd);
+    ARROW_CHECK(entry != mmap_table_.end());
+    entry->second.count -= 1;
+    ARROW_CHECK(entry->second.count >= 0);
+    // If none are being used then unmap the file.
+    if (entry->second.count == 0) {
+      munmap(entry->second.pointer, entry->second.length);
+      // Remove the corresponding entry from the hash table.
+      mmap_table_.erase(fd);
+    }
+    // Tell the store that the client no longer needs the object.
+    RETURN_NOT_OK(SendReleaseRequest(store_conn_, object_id));
+    // Update the in_use_object_bytes_.
+    in_use_object_bytes_ -= (object_entry->second->object.data_size +
+                             object_entry->second->object.metadata_size);
+    DCHECK_GE(in_use_object_bytes_, 0);
+    // Remove the entry from the hash table of objects currently in use.
+    objects_in_use_.erase(object_id);
+  }
+  return Status::OK();
+}
+
+Status PlasmaClient::Release(const ObjectID& object_id) {
+  // Add the new object to the release history.
+  release_history_.push_front(object_id);
+  // If there are too many bytes in use by the client or if there are too many
+  // pending release calls, and there are at least some pending release calls in
+  // the release_history list, then release some objects.
+  while ((in_use_object_bytes_ > std::min(kL3CacheSizeBytes, store_capacity_ / 100) ||
+             release_history_.size() > config_.release_delay) &&
+         release_history_.size() > 0) {
+    // Perform a release for the object ID for the first pending release.
+    RETURN_NOT_OK(PerformRelease(release_history_.back()));
+    // Remove the last entry from the release history.
+    release_history_.pop_back();
+  }
+  return Status::OK();
+}
+
+// This method is used to query whether the plasma store contains an object.
+Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) {
+  // Check if we already have a reference to the object.
+  if (objects_in_use_.count(object_id) > 0) {
+    *has_object = 1;
+  } else {
+    // If we don't already have a reference to the object, check with the store
+    // to see if we have the object.
+    RETURN_NOT_OK(SendContainsRequest(store_conn_, object_id));
+    std::vector<uint8_t> buffer;
+    RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaContainsReply, &buffer));
+    ObjectID object_id2;
+    RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object));
+  }
+  return Status::OK();
+}
+
+static void ComputeBlockHash(const unsigned char* data, int64_t nbytes, uint64_t* hash) {
+  XXH64_state_t hash_state;
+  XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
+  XXH64_update(&hash_state, data, nbytes);
+  *hash = XXH64_digest(&hash_state);
+}
+
+static inline bool compute_object_hash_parallel(
+    XXH64_state_t* hash_state, const unsigned char* data, int64_t nbytes) {
+  // Note that this function will likely be faster if the address of data is
+  // aligned on a 64-byte boundary.
+  const int num_threads = kThreadPoolSize;
+  uint64_t threadhash[num_threads + 1];
+  const uint64_t data_address = reinterpret_cast<uint64_t>(data);
+  const uint64_t num_blocks = nbytes / BLOCK_SIZE;
+  const uint64_t chunk_size = (num_blocks / num_threads) * BLOCK_SIZE;
+  const uint64_t right_address = data_address + chunk_size * num_threads;
+  const uint64_t suffix = (data_address + nbytes) - right_address;
+  // Now the data layout is | k * num_threads * block_size | suffix | ==
+  // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size.
+  // Each thread gets a "chunk" of k blocks, except the suffix thread.
+
+  for (int i = 0; i < num_threads; i++) {
+    threadpool_[i] = std::thread(ComputeBlockHash,
+        reinterpret_cast<uint8_t*>(data_address) + i * chunk_size, chunk_size,
+        &threadhash[i]);
+  }
+  ComputeBlockHash(
+      reinterpret_cast<uint8_t*>(right_address), suffix, &threadhash[num_threads]);
+
+  // Join the threads.
+  for (auto& t : threadpool_) {
+    if (t.joinable()) { t.join(); }
+  }
+
+  XXH64_update(hash_state, (unsigned char*)threadhash, sizeof(threadhash));
+  return true;
+}
+
+static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
+  XXH64_state_t hash_state;
+  XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
+  if (obj_buffer.data_size >= kBytesInMB) {
+    compute_object_hash_parallel(
+        &hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
+  } else {
+    XXH64_update(&hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
+  }
+  XXH64_update(
+      &hash_state, (unsigned char*)obj_buffer.metadata, obj_buffer.metadata_size);
+  return XXH64_digest(&hash_state);
+}
+
+bool plasma_compute_object_hash(
+    PlasmaClient* conn, ObjectID object_id, unsigned char* digest) {
+  // Get the plasma object data. We pass in a timeout of 0 to indicate that
+  // the operation should timeout immediately.
+  ObjectBuffer object_buffer;
+  ARROW_CHECK_OK(conn->Get(&object_id, 1, 0, &object_buffer));
+  // If the object was not retrieved, return false.
+  if (object_buffer.data_size == -1) { return false; }
+  // Compute the hash.
+  uint64_t hash = compute_object_hash(object_buffer);
+  memcpy(digest, &hash, sizeof(hash));
+  // Release the plasma object.
+  ARROW_CHECK_OK(conn->Release(object_id));
+  return true;
+}
+
+Status PlasmaClient::Seal(const ObjectID& object_id) {
+  // Make sure this client has a reference to the object before sending the
+  // request to Plasma.
+  auto object_entry = objects_in_use_.find(object_id);
+  ARROW_CHECK(object_entry != objects_in_use_.end())
+      << "Plasma client called seal an object without a reference to it";
+  ARROW_CHECK(!object_entry->second->is_sealed)
+      << "Plasma client called seal an already sealed object";
+  object_entry->second->is_sealed = true;
+  /// Send the seal request to Plasma.
+  static unsigned char digest[kDigestSize];
+  ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0]));
+  RETURN_NOT_OK(SendSealRequest(store_conn_, object_id, &digest[0]));
+  // We call PlasmaClient::Release to decrement the number of instances of this
+  // object
+  // that are currently being used by this client. The corresponding increment
+  // happened in plasma_create and was used to ensure that the object was not
+  // released before the call to PlasmaClient::Seal.
+  return Release(object_id);
+}
+
+Status PlasmaClient::Delete(const ObjectID& object_id) {
+  // TODO(rkn): In the future, we can use this method to give hints to the
+  // eviction policy about when an object will no longer be needed.
+  return Status::NotImplemented("PlasmaClient::Delete is not implemented.");
+}
+
+Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) {
+  // Send a request to the store to evict objects.
+  RETURN_NOT_OK(SendEvictRequest(store_conn_, num_bytes));
+  // Wait for a response with the number of bytes actually evicted.
+  std::vector<uint8_t> buffer;
+  int64_t type;
+  RETURN_NOT_OK(ReadMessage(store_conn_, &type, &buffer));
+  return ReadEvictReply(buffer.data(), num_bytes_evicted);
+}
+
+Status PlasmaClient::Subscribe(int* fd) {
+  int sock[2];
+  // Create a non-blocking socket pair. This will only be used to send
+  // notifications from the Plasma store to the client.
+  socketpair(AF_UNIX, SOCK_STREAM, 0, sock);
+  // Make the socket non-blocking.
+  int flags = fcntl(sock[1], F_GETFL, 0);
+  ARROW_CHECK(fcntl(sock[1], F_SETFL, flags | O_NONBLOCK) == 0);
+  // Tell the Plasma store about the subscription.
+  RETURN_NOT_OK(SendSubscribeRequest(store_conn_));
+  // Send the file descriptor that the Plasma store should use to push
+  // notifications about sealed objects to this client.
+  ARROW_CHECK(send_fd(store_conn_, sock[1]) >= 0);
+  close(sock[1]);
+  // Return the file descriptor that the client should use to read notifications
+  // about sealed objects.
+  *fd = sock[0];
+  return Status::OK();
+}
+
+Status PlasmaClient::Connect(const std::string& store_socket_name,
+    const std::string& manager_socket_name, int release_delay) {
+  store_conn_ = connect_ipc_sock_retry(store_socket_name, -1, -1);
+  if (manager_socket_name != "") {
+    manager_conn_ = connect_ipc_sock_retry(manager_socket_name, -1, -1);
+  } else {
+    manager_conn_ = -1;
+  }
+  config_.release_delay = release_delay;
+  in_use_object_bytes_ = 0;
+  // Send a ConnectRequest to the store to get its memory capacity.
+  RETURN_NOT_OK(SendConnectRequest(store_conn_));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaConnectReply, &buffer));
+  RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity_));
+  return Status::OK();
+}
+
+Status PlasmaClient::Disconnect() {
+  // NOTE: We purposefully do not finish sending release calls for objects in
+  // use, so that we don't duplicate PlasmaClient::Release calls (when handling
+  // a SIGTERM, for example).
+
+  // Close the connections to Plasma. The Plasma store will release the objects
+  // that were in use by us when handling the SIGPIPE.
+  close(store_conn_);
+  if (manager_conn_ >= 0) { close(manager_conn_); }
+  return Status::OK();
+}
+
+#define h_addr h_addr_list[0]
+
+Status PlasmaClient::Transfer(const char* address, int port, const ObjectID& object_id) {
+  return SendDataRequest(manager_conn_, object_id, address, port);
+}
+
+Status PlasmaClient::Fetch(int num_object_ids, const ObjectID* object_ids) {
+  ARROW_CHECK(manager_conn_ >= 0);
+  return SendFetchRequest(manager_conn_, object_ids, num_object_ids);
+}
+
+int PlasmaClient::get_manager_fd() {
+  return manager_conn_;
+}
+
+Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
+  ARROW_CHECK(manager_conn_ >= 0);
+
+  RETURN_NOT_OK(SendStatusRequest(manager_conn_, &object_id, 1));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaStatusReply, &buffer));
+  ObjectID id;
+  RETURN_NOT_OK(ReadStatusReply(buffer.data(), &id, object_status, 1));
+  ARROW_CHECK(object_id == id);
+  return Status::OK();
+}
+
+Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_requests,
+    int num_ready_objects, int64_t timeout_ms, int* num_objects_ready) {
+  ARROW_CHECK(manager_conn_ >= 0);
+  ARROW_CHECK(num_object_requests > 0);
+  ARROW_CHECK(num_ready_objects > 0);
+  ARROW_CHECK(num_ready_objects <= num_object_requests);
+
+  for (int i = 0; i < num_object_requests; ++i) {
+    ARROW_CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL ||
+                object_requests[i].type == PLASMA_QUERY_ANYWHERE);
+  }
+
+  RETURN_NOT_OK(SendWaitRequest(manager_conn_, object_requests, num_object_requests,
+      num_ready_objects, timeout_ms));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaWaitReply, &buffer));
+  RETURN_NOT_OK(ReadWaitReply(buffer.data(), object_requests, &num_ready_objects));
+
+  *num_objects_ready = 0;
+  for (int i = 0; i < num_object_requests; ++i) {
+    int type = object_requests[i].type;
+    int status = object_requests[i].status;
+    switch (type) {
+      case PLASMA_QUERY_LOCAL:
+        if (status == ObjectStatus_Local) { *num_objects_ready += 1; }
+        break;
+      case PLASMA_QUERY_ANYWHERE:
+        if (status == ObjectStatus_Local || status == ObjectStatus_Remote) {
+          *num_objects_ready += 1;
+        } else {
+          ARROW_CHECK(status == ObjectStatus_Nonexistent);
+        }
+        break;
+      default:
+        ARROW_LOG(FATAL) << "This code should be unreachable.";
+    }
+  }
+  return Status::OK();
+}
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
new file mode 100644
index 0000000000000..fb3a161795d47
--- /dev/null
+++ b/cpp/src/plasma/client.h
@@ -0,0 +1,343 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_CLIENT_H
+#define PLASMA_CLIENT_H
+
+#include <stdbool.h>
+#include <time.h>
+
+#include <deque>
+#include <string>
+
+#include "plasma/plasma.h"
+
+using arrow::Status;
+
+#define PLASMA_DEFAULT_RELEASE_DELAY 64
+
+// Use 100MB as an overestimate of the L3 cache size.
+constexpr int64_t kL3CacheSizeBytes = 100000000;
+
+/// Object buffer data structure.
+struct ObjectBuffer {
+  /// The size in bytes of the data object.
+  int64_t data_size;
+  /// The address of the data object.
+  uint8_t* data;
+  /// The metadata size in bytes.
+  int64_t metadata_size;
+  /// The address of the metadata.
+  uint8_t* metadata;
+};
+
+/// Configuration options for the plasma client.
+struct PlasmaClientConfig {
+  /// Number of release calls we wait until the object is actually released.
+  /// This allows us to avoid invalidating the cpu cache on workers if objects
+  /// are reused accross tasks.
+  size_t release_delay;
+};
+
+struct ClientMmapTableEntry {
+  /// The result of mmap for this file descriptor.
+  uint8_t* pointer;
+  /// The length of the memory-mapped file.
+  size_t length;
+  /// The number of objects in this memory-mapped file that are currently being
+  /// used by the client. When this count reaches zeros, we unmap the file.
+  int count;
+};
+
+struct ObjectInUseEntry {
+  /// A count of the number of times this client has called PlasmaClient::Create
+  /// or
+  /// PlasmaClient::Get on this object ID minus the number of calls to
+  /// PlasmaClient::Release.
+  /// When this count reaches zero, we remove the entry from the ObjectsInUse
+  /// and decrement a count in the relevant ClientMmapTableEntry.
+  int count;
+  /// Cached information to read the object.
+  PlasmaObject object;
+  /// A flag representing whether the object has been sealed.
+  bool is_sealed;
+};
+
+class PlasmaClient {
+ public:
+  /// Connect to the local plasma store and plasma manager. Return
+  /// the resulting connection.
+  ///
+  /// @param store_socket_name The name of the UNIX domain socket to use to
+  ///        connect to the Plasma store.
+  /// @param manager_socket_name The name of the UNIX domain socket to use to
+  ///        connect to the local Plasma manager. If this is "", then this
+  ///        function will not connect to a manager.
+  /// @param release_delay Number of released objects that are kept around
+  ///        and not evicted to avoid too many munmaps.
+  /// @return The return status.
+  Status Connect(const std::string& store_socket_name,
+      const std::string& manager_socket_name, int release_delay);
+
+  /// Create an object in the Plasma Store. Any metadata for this object must be
+  /// be passed in when the object is created.
+  ///
+  /// @param object_id The ID to use for the newly created object.
+  /// @param data_size The size in bytes of the space to be allocated for this
+  /// object's
+  ///        data (this does not include space used for metadata).
+  /// @param metadata The object's metadata. If there is no metadata, this
+  /// pointer
+  ///        should be NULL.
+  /// @param metadata_size The size in bytes of the metadata. If there is no
+  ///        metadata, this should be 0.
+  /// @param data The address of the newly created object will be written here.
+  /// @return The return status.
+  Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
+      int64_t metadata_size, uint8_t** data);
+
+  /// Get some objects from the Plasma Store. This function will block until the
+  /// objects have all been created and sealed in the Plasma Store or the
+  /// timeout
+  /// expires. The caller is responsible for releasing any retrieved objects,
+  /// but
+  /// the caller should not release objects that were not retrieved.
+  ///
+  /// @param object_ids The IDs of the objects to get.
+  /// @param num_object_ids The number of object IDs to get.
+  /// @param timeout_ms The amount of time in milliseconds to wait before this
+  ///        request times out. If this value is -1, then no timeout is set.
+  /// @param object_buffers An array where the results will be stored. If the
+  /// data
+  ///        size field is -1, then the object was not retrieved.
+  /// @return The return status.
+  Status Get(const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms,
+      ObjectBuffer* object_buffers);
+
+  /// Tell Plasma that the client no longer needs the object. This should be
+  /// called
+  /// after Get when the client is done with the object. After this call,
+  /// the address returned by Get is no longer valid. This should be called
+  /// once for each call to Get (with the same object ID).
+  ///
+  /// @param object_id The ID of the object that is no longer needed.
+  /// @return The return status.
+  Status Release(const ObjectID& object_id);
+
+  /// Check if the object store contains a particular object and the object has
+  /// been sealed. The result will be stored in has_object.
+  ///
+  /// @todo: We may want to indicate if the object has been created but not
+  /// sealed.
+  ///
+  /// @param object_id The ID of the object whose presence we are checking.
+  /// @param has_object The function will write true at this address if
+  ///        the object is present and false if it is not present.
+  /// @return The return status.
+  Status Contains(const ObjectID& object_id, bool* has_object);
+
+  /// Seal an object in the object store. The object will be immutable after
+  /// this
+  /// call.
+  ///
+  /// @param object_id The ID of the object to seal.
+  /// @return The return status.
+  Status Seal(const ObjectID& object_id);
+
+  /// Delete an object from the object store. This currently assumes that the
+  /// object is present and has been sealed.
+  ///
+  /// @todo We may want to allow the deletion of objects that are not present or
+  ///       haven't been sealed.
+  ///
+  /// @param object_id The ID of the object to delete.
+  /// @return The return status.
+  Status Delete(const ObjectID& object_id);
+
+  /// Delete objects until we have freed up num_bytes bytes or there are no more
+  /// released objects that can be deleted.
+  ///
+  /// @param num_bytes The number of bytes to try to free up.
+  /// @param num_bytes_evicted Out parameter for total number of bytes of space
+  /// retrieved.
+  /// @return The return status.
+  Status Evict(int64_t num_bytes, int64_t& num_bytes_evicted);
+
+  /// Subscribe to notifications when objects are sealed in the object store.
+  /// Whenever an object is sealed, a message will be written to the client
+  /// socket
+  /// that is returned by this method.
+  ///
+  /// @param fd Out parameter for the file descriptor the client should use to
+  /// read notifications
+  ///         from the object store about sealed objects.
+  /// @return The return status.
+  Status Subscribe(int* fd);
+
+  /// Disconnect from the local plasma instance, including the local store and
+  /// manager.
+  ///
+  /// @return The return status.
+  Status Disconnect();
+
+  /// Attempt to initiate the transfer of some objects from remote Plasma
+  /// Stores.
+  /// This method does not guarantee that the fetched objects will arrive
+  /// locally.
+  ///
+  /// For an object that is available in the local Plasma Store, this method
+  /// will
+  /// not do anything. For an object that is not available locally, it will
+  /// check
+  /// if the object are already being fetched. If so, it will not do anything.
+  /// If
+  /// not, it will query the object table for a list of Plasma Managers that
+  /// have
+  /// the object. The object table will return a non-empty list, and this Plasma
+  /// Manager will attempt to initiate transfers from one of those Plasma
+  /// Managers.
+  ///
+  /// This function is non-blocking.
+  ///
+  /// This method is idempotent in the sense that it is ok to call it multiple
+  /// times.
+  ///
+  /// @param num_object_ids The number of object IDs fetch is being called on.
+  /// @param object_ids The IDs of the objects that fetch is being called on.
+  /// @return The return status.
+  Status Fetch(int num_object_ids, const ObjectID* object_ids);
+
+  /// Wait for (1) a specified number of objects to be available (sealed) in the
+  /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to
+  /// expire. This is a blocking call.
+  ///
+  /// @param num_object_requests Size of the object_requests array.
+  /// @param object_requests Object event array. Each element contains a request
+  ///        for a particular object_id. The type of request is specified in the
+  ///        "type" field.
+  ///        - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes
+  ///          available in the local Plasma Store. In this case, this function
+  ///          sets the "status" field to ObjectStatus_Local. Note, if the
+  ///          status
+  ///          is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent,
+  ///          but it may exist elsewhere in the system.
+  ///        - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id
+  ///        becomes
+  ///          available either at the local Plasma Store or on a remote Plasma
+  ///          Store. In this case, the functions sets the "status" field to
+  ///          ObjectStatus_Local or ObjectStatus_Remote.
+  /// @param num_ready_objects The number of requests in object_requests array
+  /// that
+  ///        must be satisfied before the function returns, unless it timeouts.
+  ///        The num_ready_objects should be no larger than num_object_requests.
+  /// @param timeout_ms Timeout value in milliseconds. If this timeout expires
+  ///        before min_num_ready_objects of requests are satisfied, the
+  ///        function
+  ///        returns.
+  /// @param num_objects_ready Out parameter for number of satisfied requests in
+  ///        the object_requests list. If the returned number is less than
+  ///        min_num_ready_objects this means that timeout expired.
+  /// @return The return status.
+  Status Wait(int64_t num_object_requests, ObjectRequest* object_requests,
+      int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
+
+  /// Transfer local object to a different plasma manager.
+  ///
+  /// @param conn The object containing the connection state.
+  /// @param addr IP address of the plasma manager we are transfering to.
+  /// @param port Port of the plasma manager we are transfering to.
+  /// @object_id ObjectID of the object we are transfering.
+  /// @return The return status.
+  Status Transfer(const char* addr, int port, const ObjectID& object_id);
+
+  /// Return the status of a given object. This method may query the object
+  /// table.
+  ///
+  /// @param conn The object containing the connection state.
+  /// @param object_id The ID of the object whose status we query.
+  /// @param object_status Out parameter for object status. Can take the
+  ///         following values.
+  ///         - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma
+  ///         Store.
+  ///           has been already scheduled by the Plasma Manager.
+  ///         - PLASMA_CLIENT_TRANSFER, if the object is either currently being
+  ///           transferred or just scheduled.
+  ///         - PLASMA_CLIENT_REMOTE, if the object is stored at a remote
+  ///           Plasma Store.
+  ///         - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the
+  ///           system.
+  /// @return The return status.
+  Status Info(const ObjectID& object_id, int* object_status);
+
+  /// Get the file descriptor for the socket connection to the plasma manager.
+  ///
+  /// @param conn The plasma connection.
+  /// @return The file descriptor for the manager connection. If there is no
+  ///         connection to the manager, this is -1.
+  int get_manager_fd();
+
+ private:
+  Status PerformRelease(const ObjectID& object_id);
+
+  uint8_t* lookup_or_mmap(int fd, int store_fd_val, int64_t map_size);
+
+  uint8_t* lookup_mmapped_file(int store_fd_val);
+
+  void increment_object_count(
+      const ObjectID& object_id, PlasmaObject* object, bool is_sealed);
+
+  /// File descriptor of the Unix domain socket that connects to the store.
+  int store_conn_;
+  /// File descriptor of the Unix domain socket that connects to the manager.
+  int manager_conn_;
+  /// Table of dlmalloc buffer files that have been memory mapped so far. This
+  /// is a hash table mapping a file descriptor to a struct containing the
+  /// address of the corresponding memory-mapped file.
+  std::unordered_map<int, ClientMmapTableEntry> mmap_table_;
+  /// A hash table of the object IDs that are currently being used by this
+  /// client.
+  std::unordered_map<ObjectID, std::unique_ptr<ObjectInUseEntry>, UniqueIDHasher>
+      objects_in_use_;
+  /// Object IDs of the last few release calls. This is a deque and
+  /// is used to delay releasing objects to see if they can be reused by
+  /// subsequent tasks so we do not unneccessarily invalidate cpu caches.
+  /// TODO(pcm): replace this with a proper lru cache using the size of the L3
+  /// cache.
+  std::deque<ObjectID> release_history_;
+  /// The number of bytes in the combined objects that are held in the release
+  /// history doubly-linked list. If this is too large then the client starts
+  /// releasing objects.
+  int64_t in_use_object_bytes_;
+  /// Configuration options for the plasma client.
+  PlasmaClientConfig config_;
+  /// The amount of memory available to the Plasma store. The client needs this
+  /// information to make sure that it does not delay in releasing so much
+  /// memory that the store is unable to evict enough objects to free up space.
+  int64_t store_capacity_;
+};
+
+/// Compute the hash of an object in the object store.
+///
+/// @param conn The object containing the connection state.
+/// @param object_id The ID of the object we want to hash.
+/// @param digest A pointer at which to return the hash digest of the object.
+///        The pointer must have at least DIGEST_SIZE bytes allocated.
+/// @return A boolean representing whether the hash operation succeeded.
+bool plasma_compute_object_hash(
+    PlasmaClient* conn, ObjectID object_id, unsigned char* digest);
+
+#endif  // PLASMA_CLIENT_H
diff --git a/cpp/src/plasma/common.cc b/cpp/src/plasma/common.cc
new file mode 100644
index 0000000000000..a09a963fa4769
--- /dev/null
+++ b/cpp/src/plasma/common.cc
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/common.h"
+
+#include <random>
+
+#include "format/plasma_generated.h"
+
+using arrow::Status;
+
+UniqueID UniqueID::from_random() {
+  UniqueID id;
+  uint8_t* data = id.mutable_data();
+  std::random_device engine;
+  for (int i = 0; i < kUniqueIDSize; i++) {
+    data[i] = static_cast<uint8_t>(engine());
+  }
+  return id;
+}
+
+UniqueID UniqueID::from_binary(const std::string& binary) {
+  UniqueID id;
+  std::memcpy(&id, binary.data(), sizeof(id));
+  return id;
+}
+
+const uint8_t* UniqueID::data() const {
+  return id_;
+}
+
+uint8_t* UniqueID::mutable_data() {
+  return id_;
+}
+
+std::string UniqueID::binary() const {
+  return std::string(reinterpret_cast<const char*>(id_), kUniqueIDSize);
+}
+
+std::string UniqueID::hex() const {
+  constexpr char hex[] = "0123456789abcdef";
+  std::string result;
+  for (int i = 0; i < kUniqueIDSize; i++) {
+    unsigned int val = id_[i];
+    result.push_back(hex[val >> 4]);
+    result.push_back(hex[val & 0xf]);
+  }
+  return result;
+}
+
+bool UniqueID::operator==(const UniqueID& rhs) const {
+  return std::memcmp(data(), rhs.data(), kUniqueIDSize) == 0;
+}
+
+Status plasma_error_status(int plasma_error) {
+  switch (plasma_error) {
+    case PlasmaError_OK:
+      return Status::OK();
+    case PlasmaError_ObjectExists:
+      return Status::PlasmaObjectExists("object already exists in the plasma store");
+    case PlasmaError_ObjectNonexistent:
+      return Status::PlasmaObjectNonexistent("object does not exist in the plasma store");
+    case PlasmaError_OutOfMemory:
+      return Status::PlasmaStoreFull("object does not fit in the plasma store");
+    default:
+      ARROW_LOG(FATAL) << "unknown plasma error code " << plasma_error;
+  }
+  return Status::OK();
+}
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
new file mode 100644
index 0000000000000..85dc74bf86e0d
--- /dev/null
+++ b/cpp/src/plasma/common.h
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_COMMON_H
+#define PLASMA_COMMON_H
+
+#include <cstring>
+#include <string>
+// TODO(pcm): Convert getopt and sscanf in the store to use more idiomatic C++
+// and get rid of the next three lines:
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+constexpr int64_t kUniqueIDSize = 20;
+
+class UniqueID {
+ public:
+  static UniqueID from_random();
+  static UniqueID from_binary(const std::string& binary);
+  bool operator==(const UniqueID& rhs) const;
+  const uint8_t* data() const;
+  uint8_t* mutable_data();
+  std::string binary() const;
+  std::string hex() const;
+
+ private:
+  uint8_t id_[kUniqueIDSize];
+};
+
+static_assert(std::is_pod<UniqueID>::value, "UniqueID must be plain old data");
+
+struct UniqueIDHasher {
+  // ObjectID hashing function.
+  size_t operator()(const UniqueID& id) const {
+    size_t result;
+    std::memcpy(&result, id.data(), sizeof(size_t));
+    return result;
+  }
+};
+
+typedef UniqueID ObjectID;
+
+arrow::Status plasma_error_status(int plasma_error);
+
+#endif  // PLASMA_COMMON_H
diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
new file mode 100644
index 0000000000000..a9f7356e1f67e
--- /dev/null
+++ b/cpp/src/plasma/events.cc
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/events.h"
+
+#include <errno.h>
+
+void EventLoop::file_event_callback(
+    aeEventLoop* loop, int fd, void* context, int events) {
+  FileCallback* callback = reinterpret_cast<FileCallback*>(context);
+  (*callback)(events);
+}
+
+int EventLoop::timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context) {
+  TimerCallback* callback = reinterpret_cast<TimerCallback*>(context);
+  return (*callback)(timer_id);
+}
+
+constexpr int kInitialEventLoopSize = 1024;
+
+EventLoop::EventLoop() {
+  loop_ = aeCreateEventLoop(kInitialEventLoopSize);
+}
+
+bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback) {
+  if (file_callbacks_.find(fd) != file_callbacks_.end()) { return false; }
+  auto data = std::unique_ptr<FileCallback>(new FileCallback(callback));
+  void* context = reinterpret_cast<void*>(data.get());
+  // Try to add the file descriptor.
+  int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
+  // If it cannot be added, increase the size of the event loop.
+  if (err == AE_ERR && errno == ERANGE) {
+    err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2);
+    if (err != AE_OK) { return false; }
+    err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
+  }
+  // In any case, test if there were errors.
+  if (err == AE_OK) {
+    file_callbacks_.emplace(fd, std::move(data));
+    return true;
+  }
+  return false;
+}
+
+void EventLoop::remove_file_event(int fd) {
+  aeDeleteFileEvent(loop_, fd, AE_READABLE | AE_WRITABLE);
+  file_callbacks_.erase(fd);
+}
+
+void EventLoop::run() {
+  aeMain(loop_);
+}
+
+int64_t EventLoop::add_timer(int64_t timeout, const TimerCallback& callback) {
+  auto data = std::unique_ptr<TimerCallback>(new TimerCallback(callback));
+  void* context = reinterpret_cast<void*>(data.get());
+  int64_t timer_id =
+      aeCreateTimeEvent(loop_, timeout, EventLoop::timer_event_callback, context, NULL);
+  timer_callbacks_.emplace(timer_id, std::move(data));
+  return timer_id;
+}
+
+int EventLoop::remove_timer(int64_t timer_id) {
+  int err = aeDeleteTimeEvent(loop_, timer_id);
+  timer_callbacks_.erase(timer_id);
+  return err;
+}
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
new file mode 100644
index 0000000000000..bd93d6bb2a6fd
--- /dev/null
+++ b/cpp/src/plasma/events.h
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EVENTS
+#define PLASMA_EVENTS
+
+#include <functional>
+#include <memory>
+#include <unordered_map>
+
+extern "C" {
+#include "ae/ae.h"
+}
+
+/// Constant specifying that the timer is done and it will be removed.
+constexpr int kEventLoopTimerDone = AE_NOMORE;
+
+/// Read event on the file descriptor.
+constexpr int kEventLoopRead = AE_READABLE;
+
+/// Write event on the file descriptor.
+constexpr int kEventLoopWrite = AE_WRITABLE;
+
+typedef long long TimerID;  // NOLINT
+
+class EventLoop {
+ public:
+  // Signature of the handler that will be called when there is a new event
+  // on the file descriptor that this handler has been registered for.
+  //
+  // The arguments are the event flags (read or write).
+  using FileCallback = std::function<void(int)>;
+
+  // This handler will be called when a timer times out. The timer id is
+  // passed as an argument. The return is the number of milliseconds the timer
+  // shall be reset to or kEventLoopTimerDone if the timer shall not be
+  // triggered again.
+  using TimerCallback = std::function<int(int64_t)>;
+
+  EventLoop();
+
+  /// Add a new file event handler to the event loop.
+  ///
+  /// @param fd The file descriptor we are listening to.
+  /// @param events The flags for events we are listening to (read or write).
+  /// @param callback The callback that will be called when the event happens.
+  /// @return Returns true if the event handler was added successfully.
+  bool add_file_event(int fd, int events, const FileCallback& callback);
+
+  /// Remove a file event handler from the event loop.
+  ///
+  /// @param fd The file descriptor of the event handler.
+  /// @return Void.
+  void remove_file_event(int fd);
+
+  /// Register a handler that will be called after a time slice of
+  ///  "timeout" milliseconds.
+  ///
+  ///  @param timeout The timeout in milliseconds.
+  ///  @param callback The callback for the timeout.
+  ///  @return The ID of the newly created timer.
+  int64_t add_timer(int64_t timeout, const TimerCallback& callback);
+
+  /// Remove a timer handler from the event loop.
+  ///
+  /// @param timer_id The ID of the timer that is to be removed.
+  /// @return The ae.c error code. TODO(pcm): needs to be standardized
+  int remove_timer(int64_t timer_id);
+
+  /// Run the event loop.
+  ///
+  /// @return Void.
+  void run();
+
+ private:
+  static void file_event_callback(aeEventLoop* loop, int fd, void* context, int events);
+
+  static int timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context);
+
+  aeEventLoop* loop_;
+  std::unordered_map<int, std::unique_ptr<FileCallback>> file_callbacks_;
+  std::unordered_map<int64_t, std::unique_ptr<TimerCallback>> timer_callbacks_;
+};
+
+#endif  // PLASMA_EVENTS
diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
new file mode 100644
index 0000000000000..4ae6384d42543
--- /dev/null
+++ b/cpp/src/plasma/eviction_policy.cc
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/eviction_policy.h"
+
+#include <algorithm>
+
+void LRUCache::add(const ObjectID& key, int64_t size) {
+  auto it = item_map_.find(key);
+  ARROW_CHECK(it == item_map_.end());
+  /* Note that it is important to use a list so the iterators stay valid. */
+  item_list_.emplace_front(key, size);
+  item_map_.emplace(key, item_list_.begin());
+}
+
+void LRUCache::remove(const ObjectID& key) {
+  auto it = item_map_.find(key);
+  ARROW_CHECK(it != item_map_.end());
+  item_list_.erase(it->second);
+  item_map_.erase(it);
+}
+
+int64_t LRUCache::choose_objects_to_evict(
+    int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
+  int64_t bytes_evicted = 0;
+  auto it = item_list_.end();
+  while (bytes_evicted < num_bytes_required && it != item_list_.begin()) {
+    it--;
+    objects_to_evict->push_back(it->first);
+    bytes_evicted += it->second;
+  }
+  return bytes_evicted;
+}
+
+EvictionPolicy::EvictionPolicy(PlasmaStoreInfo* store_info)
+    : memory_used_(0), store_info_(store_info) {}
+
+int64_t EvictionPolicy::choose_objects_to_evict(
+    int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
+  int64_t bytes_evicted =
+      cache_.choose_objects_to_evict(num_bytes_required, objects_to_evict);
+  /* Update the LRU cache. */
+  for (auto& object_id : *objects_to_evict) {
+    cache_.remove(object_id);
+  }
+  /* Update the number of bytes used. */
+  memory_used_ -= bytes_evicted;
+  return bytes_evicted;
+}
+
+void EvictionPolicy::object_created(const ObjectID& object_id) {
+  auto entry = store_info_->objects[object_id].get();
+  cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
+}
+
+bool EvictionPolicy::require_space(
+    int64_t size, std::vector<ObjectID>* objects_to_evict) {
+  /* Check if there is enough space to create the object. */
+  int64_t required_space = memory_used_ + size - store_info_->memory_capacity;
+  int64_t num_bytes_evicted;
+  if (required_space > 0) {
+    /* Try to free up at least as much space as we need right now but ideally
+     * up to 20% of the total capacity. */
+    int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5);
+    ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects";
+    /* Choose some objects to evict, and update the return pointers. */
+    num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict);
+    ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting "
+                    << objects_to_evict->size() << " objects to free up "
+                    << num_bytes_evicted << " bytes.";
+  } else {
+    num_bytes_evicted = 0;
+  }
+  if (num_bytes_evicted >= required_space) {
+    /* We only increment the space used if there is enough space to create the
+     * object. */
+    memory_used_ += size;
+  }
+  return num_bytes_evicted >= required_space;
+}
+
+void EvictionPolicy::begin_object_access(
+    const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
+  /* If the object is in the LRU cache, remove it. */
+  cache_.remove(object_id);
+}
+
+void EvictionPolicy::end_object_access(
+    const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
+  auto entry = store_info_->objects[object_id].get();
+  /* Add the object to the LRU cache.*/
+  cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
+}
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
new file mode 100644
index 0000000000000..3815fc6652f0c
--- /dev/null
+++ b/cpp/src/plasma/eviction_policy.h
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EVICTION_POLICY_H
+#define PLASMA_EVICTION_POLICY_H
+
+#include <list>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/plasma.h"
+
+// ==== The eviction policy ====
+//
+// This file contains declaration for all functions and data structures that
+// need to be provided if you want to implement a new eviction algorithm for the
+// Plasma store.
+
+class LRUCache {
+ public:
+  LRUCache() {}
+
+  void add(const ObjectID& key, int64_t size);
+
+  void remove(const ObjectID& key);
+
+  int64_t choose_objects_to_evict(
+      int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
+
+ private:
+  /// A doubly-linked list containing the items in the cache and
+  /// their sizes in LRU order.
+  typedef std::list<std::pair<ObjectID, int64_t>> ItemList;
+  ItemList item_list_;
+  /// A hash table mapping the object ID of an object in the cache to its
+  /// location in the doubly linked list item_list_.
+  std::unordered_map<ObjectID, ItemList::iterator, UniqueIDHasher> item_map_;
+};
+
+/// The eviction policy.
+class EvictionPolicy {
+ public:
+  /// Construct an eviction policy.
+  ///
+  /// @param store_info Information about the Plasma store that is exposed
+  ///        to the eviction policy.
+  explicit EvictionPolicy(PlasmaStoreInfo* store_info);
+
+  /// This method will be called whenever an object is first created in order to
+  /// add it to the LRU cache. This is done so that the first time, the Plasma
+  /// store calls begin_object_access, we can remove the object from the LRU
+  /// cache.
+  ///
+  /// @param object_id The object ID of the object that was created.
+  /// @return Void.
+  void object_created(const ObjectID& object_id);
+
+  /// This method will be called when the Plasma store needs more space, perhaps
+  /// to create a new object. If the required amount of space cannot be freed up,
+  /// then a fatal error will be thrown. When this method is called, the eviction
+  /// policy will assume that the objects chosen to be evicted will in fact be
+  /// evicted from the Plasma store by the caller.
+  ///
+  /// @param size The size in bytes of the new object, including both data and
+  ///        metadata.
+  /// @param objects_to_evict The object IDs that were chosen for eviction will
+  ///        be stored into this vector.
+  /// @return True if enough space can be freed and false otherwise.
+  bool require_space(int64_t size, std::vector<ObjectID>* objects_to_evict);
+
+  /// This method will be called whenever an unused object in the Plasma store
+  /// starts to be used. When this method is called, the eviction policy will
+  /// assume that the objects chosen to be evicted will in fact be evicted from
+  /// the Plasma store by the caller.
+  ///
+  /// @param object_id The ID of the object that is now being used.
+  /// @param objects_to_evict The object IDs that were chosen for eviction will
+  ///        be stored into this vector.
+  /// @return Void.
+  void begin_object_access(
+      const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
+
+  /// This method will be called whenever an object in the Plasma store that was
+  /// being used is no longer being used. When this method is called, the
+  /// eviction policy will assume that the objects chosen to be evicted will in
+  /// fact be evicted from the Plasma store by the caller.
+  ///
+  /// @param object_id The ID of the object that is no longer being used.
+  /// @param objects_to_evict The object IDs that were chosen for eviction will
+  ///        be stored into this vector.
+  /// @return Void.
+  void end_object_access(
+      const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
+
+  /// Choose some objects to evict from the Plasma store. When this method is
+  /// called, the eviction policy will assume that the objects chosen to be
+  /// evicted will in fact be evicted from the Plasma store by the caller.
+  ///
+  /// @note This method is not part of the API. It is exposed in the header file
+  /// only for testing.
+  ///
+  /// @param num_bytes_required The number of bytes of space to try to free up.
+  /// @param objects_to_evict The object IDs that were chosen for eviction will
+  ///        be stored into this vector.
+  /// @return The total number of bytes of space chosen to be evicted.
+  int64_t choose_objects_to_evict(
+      int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
+
+ private:
+  /// The amount of memory (in bytes) currently being used.
+  int64_t memory_used_;
+  /// Pointer to the plasma store info.
+  PlasmaStoreInfo* store_info_;
+  /// Datastructure for the LRU cache.
+  LRUCache cache_;
+};
+
+#endif  // PLASMA_EVICTION_POLICY_H
diff --git a/cpp/src/plasma/extension.cc b/cpp/src/plasma/extension.cc
new file mode 100644
index 0000000000000..5d61e337c108d
--- /dev/null
+++ b/cpp/src/plasma/extension.cc
@@ -0,0 +1,456 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/extension.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+#include "plasma/io.h"
+#include "plasma/protocol.h"
+
+PyObject* PlasmaOutOfMemoryError;
+PyObject* PlasmaObjectExistsError;
+
+PyObject* PyPlasma_connect(PyObject* self, PyObject* args) {
+  const char* store_socket_name;
+  const char* manager_socket_name;
+  int release_delay;
+  if (!PyArg_ParseTuple(
+          args, "ssi", &store_socket_name, &manager_socket_name, &release_delay)) {
+    return NULL;
+  }
+  PlasmaClient* client = new PlasmaClient();
+  ARROW_CHECK_OK(client->Connect(store_socket_name, manager_socket_name, release_delay));
+
+  return PyCapsule_New(client, "plasma", NULL);
+}
+
+PyObject* PyPlasma_disconnect(PyObject* self, PyObject* args) {
+  PyObject* client_capsule;
+  if (!PyArg_ParseTuple(args, "O", &client_capsule)) { return NULL; }
+  PlasmaClient* client;
+  ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client));
+  ARROW_CHECK_OK(client->Disconnect());
+  /* We use the context of the connection capsule to indicate if the connection
+   * is still active (if the context is NULL) or if it is closed (if the context
+   * is (void*) 0x1). This is neccessary because the primary pointer of the
+   * capsule cannot be NULL. */
+  PyCapsule_SetContext(client_capsule, reinterpret_cast<void*>(0x1));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_create(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  Py_ssize_t size;
+  PyObject* metadata;
+  if (!PyArg_ParseTuple(args, "O&O&nO", PyObjectToPlasmaClient, &client,
+          PyStringToUniqueID, &object_id, &size, &metadata)) {
+    return NULL;
+  }
+  if (!PyByteArray_Check(metadata)) {
+    PyErr_SetString(PyExc_TypeError, "metadata must be a bytearray");
+    return NULL;
+  }
+  uint8_t* data;
+  Status s = client->Create(object_id, size,
+      reinterpret_cast<uint8_t*>(PyByteArray_AsString(metadata)),
+      PyByteArray_Size(metadata), &data);
+  if (s.IsPlasmaObjectExists()) {
+    PyErr_SetString(PlasmaObjectExistsError,
+        "An object with this ID already exists in the plasma "
+        "store.");
+    return NULL;
+  }
+  if (s.IsPlasmaStoreFull()) {
+    PyErr_SetString(PlasmaOutOfMemoryError,
+        "The plasma store ran out of memory and could not create "
+        "this object.");
+    return NULL;
+  }
+  ARROW_CHECK(s.ok());
+
+#if PY_MAJOR_VERSION >= 3
+  return PyMemoryView_FromMemory(reinterpret_cast<char*>(data), size, PyBUF_WRITE);
+#else
+  return PyBuffer_FromReadWriteMemory(reinterpret_cast<void*>(data), size);
+#endif
+}
+
+PyObject* PyPlasma_hash(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  unsigned char digest[kDigestSize];
+  bool success = plasma_compute_object_hash(client, object_id, digest);
+  if (success) {
+    PyObject* digest_string =
+        PyBytes_FromStringAndSize(reinterpret_cast<char*>(digest), kDigestSize);
+    return digest_string;
+  } else {
+    Py_RETURN_NONE;
+  }
+}
+
+PyObject* PyPlasma_seal(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  ARROW_CHECK_OK(client->Seal(object_id));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_release(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  ARROW_CHECK_OK(client->Release(object_id));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_get(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  PyObject* object_id_list;
+  Py_ssize_t timeout_ms;
+  if (!PyArg_ParseTuple(
+          args, "O&On", PyObjectToPlasmaClient, &client, &object_id_list, &timeout_ms)) {
+    return NULL;
+  }
+
+  Py_ssize_t num_object_ids = PyList_Size(object_id_list);
+  std::vector<ObjectID> object_ids(num_object_ids);
+  std::vector<ObjectBuffer> object_buffers(num_object_ids);
+
+  for (int i = 0; i < num_object_ids; ++i) {
+    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
+  }
+
+  Py_BEGIN_ALLOW_THREADS;
+  ARROW_CHECK_OK(
+      client->Get(object_ids.data(), num_object_ids, timeout_ms, object_buffers.data()));
+  Py_END_ALLOW_THREADS;
+
+  PyObject* returns = PyList_New(num_object_ids);
+  for (int i = 0; i < num_object_ids; ++i) {
+    if (object_buffers[i].data_size != -1) {
+      /* The object was retrieved, so return the object. */
+      PyObject* t = PyTuple_New(2);
+      Py_ssize_t data_size = static_cast<Py_ssize_t>(object_buffers[i].data_size);
+      Py_ssize_t metadata_size = static_cast<Py_ssize_t>(object_buffers[i].metadata_size);
+#if PY_MAJOR_VERSION >= 3
+      char* data = reinterpret_cast<char*>(object_buffers[i].data);
+      char* metadata = reinterpret_cast<char*>(object_buffers[i].metadata);
+      PyTuple_SET_ITEM(t, 0, PyMemoryView_FromMemory(data, data_size, PyBUF_READ));
+      PyTuple_SET_ITEM(
+          t, 1, PyMemoryView_FromMemory(metadata, metadata_size, PyBUF_READ));
+#else
+      void* data = reinterpret_cast<void*>(object_buffers[i].data);
+      void* metadata = reinterpret_cast<void*>(object_buffers[i].metadata);
+      PyTuple_SET_ITEM(t, 0, PyBuffer_FromMemory(data, data_size));
+      PyTuple_SET_ITEM(t, 1, PyBuffer_FromMemory(metadata, metadata_size));
+#endif
+      ARROW_CHECK(PyList_SetItem(returns, i, t) == 0);
+    } else {
+      /* The object was not retrieved, so just add None to the list of return
+       * values. */
+      Py_INCREF(Py_None);
+      ARROW_CHECK(PyList_SetItem(returns, i, Py_None) == 0);
+    }
+  }
+  return returns;
+}
+
+PyObject* PyPlasma_contains(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  bool has_object;
+  ARROW_CHECK_OK(client->Contains(object_id, &has_object));
+
+  if (has_object) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
+PyObject* PyPlasma_fetch(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  PyObject* object_id_list;
+  if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client, &object_id_list)) {
+    return NULL;
+  }
+  if (client->get_manager_fd() == -1) {
+    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+    return NULL;
+  }
+  Py_ssize_t n = PyList_Size(object_id_list);
+  ObjectID* object_ids = new ObjectID[n];
+  for (int i = 0; i < n; ++i) {
+    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
+  }
+  ARROW_CHECK_OK(client->Fetch(static_cast<int>(n), object_ids));
+  delete[] object_ids;
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_wait(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  PyObject* object_id_list;
+  Py_ssize_t timeout;
+  int num_returns;
+  if (!PyArg_ParseTuple(args, "O&Oni", PyObjectToPlasmaClient, &client, &object_id_list,
+          &timeout, &num_returns)) {
+    return NULL;
+  }
+  Py_ssize_t n = PyList_Size(object_id_list);
+
+  if (client->get_manager_fd() == -1) {
+    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+    return NULL;
+  }
+  if (num_returns < 0) {
+    PyErr_SetString(
+        PyExc_RuntimeError, "The argument num_returns cannot be less than zero.");
+    return NULL;
+  }
+  if (num_returns > n) {
+    PyErr_SetString(PyExc_RuntimeError,
+        "The argument num_returns cannot be greater than len(object_ids)");
+    return NULL;
+  }
+  int64_t threshold = 1 << 30;
+  if (timeout > threshold) {
+    PyErr_SetString(
+        PyExc_RuntimeError, "The argument timeout cannot be greater than 2 ** 30.");
+    return NULL;
+  }
+
+  std::vector<ObjectRequest> object_requests(n);
+  for (int i = 0; i < n; ++i) {
+    ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i),
+                    &object_requests[i].object_id) == 1);
+    object_requests[i].type = PLASMA_QUERY_ANYWHERE;
+  }
+  /* Drop the global interpreter lock while we are waiting, so other threads can
+   * run. */
+  int num_return_objects;
+  Py_BEGIN_ALLOW_THREADS;
+  ARROW_CHECK_OK(
+      client->Wait(n, object_requests.data(), num_returns, timeout, &num_return_objects));
+  Py_END_ALLOW_THREADS;
+
+  int num_to_return = std::min(num_return_objects, num_returns);
+  PyObject* ready_ids = PyList_New(num_to_return);
+  PyObject* waiting_ids = PySet_New(object_id_list);
+  int num_returned = 0;
+  for (int i = 0; i < n; ++i) {
+    if (num_returned == num_to_return) { break; }
+    if (object_requests[i].status == ObjectStatus_Local ||
+        object_requests[i].status == ObjectStatus_Remote) {
+      PyObject* ready = PyBytes_FromStringAndSize(
+          reinterpret_cast<char*>(&object_requests[i].object_id),
+          sizeof(object_requests[i].object_id));
+      PyList_SetItem(ready_ids, num_returned, ready);
+      PySet_Discard(waiting_ids, ready);
+      num_returned += 1;
+    } else {
+      ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent);
+    }
+  }
+  ARROW_CHECK(num_returned == num_to_return);
+  /* Return both the ready IDs and the remaining IDs. */
+  PyObject* t = PyTuple_New(2);
+  PyTuple_SetItem(t, 0, ready_ids);
+  PyTuple_SetItem(t, 1, waiting_ids);
+  return t;
+}
+
+PyObject* PyPlasma_evict(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  Py_ssize_t num_bytes;
+  if (!PyArg_ParseTuple(args, "O&n", PyObjectToPlasmaClient, &client, &num_bytes)) {
+    return NULL;
+  }
+  int64_t evicted_bytes;
+  ARROW_CHECK_OK(client->Evict(static_cast<int64_t>(num_bytes), evicted_bytes));
+  return PyLong_FromSsize_t(static_cast<Py_ssize_t>(evicted_bytes));
+}
+
+PyObject* PyPlasma_delete(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  ARROW_CHECK_OK(client->Delete(object_id));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_transfer(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  const char* addr;
+  int port;
+  if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client,
+          PyStringToUniqueID, &object_id, &addr, &port)) {
+    return NULL;
+  }
+
+  if (client->get_manager_fd() == -1) {
+    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+    return NULL;
+  }
+
+  ARROW_CHECK_OK(client->Transfer(addr, port, object_id));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_subscribe(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) { return NULL; }
+
+  int sock;
+  ARROW_CHECK_OK(client->Subscribe(&sock));
+  return PyLong_FromLong(sock);
+}
+
+PyObject* PyPlasma_receive_notification(PyObject* self, PyObject* args) {
+  int plasma_sock;
+
+  if (!PyArg_ParseTuple(args, "i", &plasma_sock)) { return NULL; }
+  /* Receive object notification from the plasma connection socket. If the
+   * object was added, return a tuple of its fields: ObjectID, data_size,
+   * metadata_size. If the object was deleted, data_size and metadata_size will
+   * be set to -1. */
+  uint8_t* notification = read_message_async(plasma_sock);
+  if (notification == NULL) {
+    PyErr_SetString(
+        PyExc_RuntimeError, "Failed to read object notification from Plasma socket");
+    return NULL;
+  }
+  auto object_info = flatbuffers::GetRoot<ObjectInfo>(notification);
+  /* Construct a tuple from object_info and return. */
+  PyObject* t = PyTuple_New(3);
+  PyTuple_SetItem(t, 0, PyBytes_FromStringAndSize(object_info->object_id()->data(),
+                            object_info->object_id()->size()));
+  if (object_info->is_deletion()) {
+    PyTuple_SetItem(t, 1, PyLong_FromLong(-1));
+    PyTuple_SetItem(t, 2, PyLong_FromLong(-1));
+  } else {
+    PyTuple_SetItem(t, 1, PyLong_FromLong(object_info->data_size()));
+    PyTuple_SetItem(t, 2, PyLong_FromLong(object_info->metadata_size()));
+  }
+
+  delete[] notification;
+  return t;
+}
+
+static PyMethodDef plasma_methods[] = {
+    {"connect", PyPlasma_connect, METH_VARARGS, "Connect to plasma."},
+    {"disconnect", PyPlasma_disconnect, METH_VARARGS, "Disconnect from plasma."},
+    {"create", PyPlasma_create, METH_VARARGS, "Create a new plasma object."},
+    {"hash", PyPlasma_hash, METH_VARARGS, "Compute the hash of a plasma object."},
+    {"seal", PyPlasma_seal, METH_VARARGS, "Seal a plasma object."},
+    {"get", PyPlasma_get, METH_VARARGS, "Get a plasma object."},
+    {"contains", PyPlasma_contains, METH_VARARGS,
+        "Does the plasma store contain this plasma object?"},
+    {"fetch", PyPlasma_fetch, METH_VARARGS,
+        "Fetch the object from another plasma manager instance."},
+    {"wait", PyPlasma_wait, METH_VARARGS,
+        "Wait until num_returns objects in object_ids are ready."},
+    {"evict", PyPlasma_evict, METH_VARARGS,
+        "Evict some objects until we recover some number of bytes."},
+    {"release", PyPlasma_release, METH_VARARGS, "Release the plasma object."},
+    {"delete", PyPlasma_delete, METH_VARARGS, "Delete a plasma object."},
+    {"transfer", PyPlasma_transfer, METH_VARARGS,
+        "Transfer object to another plasma manager."},
+    {"subscribe", PyPlasma_subscribe, METH_VARARGS,
+        "Subscribe to the plasma notification socket."},
+    {"receive_notification", PyPlasma_receive_notification, METH_VARARGS,
+        "Receive next notification from plasma notification socket."},
+    {NULL} /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT, "libplasma",    /* m_name */
+    "A Python client library for plasma.", /* m_doc */
+    0,                                     /* m_size */
+    plasma_methods,                        /* m_methods */
+    NULL,                                  /* m_reload */
+    NULL,                                  /* m_traverse */
+    NULL,                                  /* m_clear */
+    NULL,                                  /* m_free */
+};
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define INITERROR return NULL
+#else
+#define INITERROR return
+#endif
+
+#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
+#define PyMODINIT_FUNC void
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
+#else
+#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
+#endif
+
+MOD_INIT(libplasma) {
+#if PY_MAJOR_VERSION >= 3
+  PyObject* m = PyModule_Create(&moduledef);
+#else
+  PyObject* m =
+      Py_InitModule3("libplasma", plasma_methods, "A Python client library for plasma.");
+#endif
+
+  /* Create a custom exception for when an object ID is reused. */
+  char plasma_object_exists_error[] = "plasma_object_exists.error";
+  PlasmaObjectExistsError = PyErr_NewException(plasma_object_exists_error, NULL, NULL);
+  Py_INCREF(PlasmaObjectExistsError);
+  PyModule_AddObject(m, "plasma_object_exists_error", PlasmaObjectExistsError);
+  /* Create a custom exception for when the plasma store is out of memory. */
+  char plasma_out_of_memory_error[] = "plasma_out_of_memory.error";
+  PlasmaOutOfMemoryError = PyErr_NewException(plasma_out_of_memory_error, NULL, NULL);
+  Py_INCREF(PlasmaOutOfMemoryError);
+  PyModule_AddObject(m, "plasma_out_of_memory_error", PlasmaOutOfMemoryError);
+
+#if PY_MAJOR_VERSION >= 3
+  return m;
+#endif
+}
diff --git a/cpp/src/plasma/extension.h b/cpp/src/plasma/extension.h
new file mode 100644
index 0000000000000..cee30abb3592d
--- /dev/null
+++ b/cpp/src/plasma/extension.h
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EXTENSION_H
+#define PLASMA_EXTENSION_H
+
+#undef _XOPEN_SOURCE
+#undef _POSIX_C_SOURCE
+#include <Python.h>
+
+#include "bytesobject.h"  // NOLINT
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+
+static int PyObjectToPlasmaClient(PyObject* object, PlasmaClient** client) {
+  if (PyCapsule_IsValid(object, "plasma")) {
+    *client = reinterpret_cast<PlasmaClient*>(PyCapsule_GetPointer(object, "plasma"));
+    return 1;
+  } else {
+    PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule");
+    return 0;
+  }
+}
+
+int PyStringToUniqueID(PyObject* object, ObjectID* object_id) {
+  if (PyBytes_Check(object)) {
+    memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID));
+    return 1;
+  } else {
+    PyErr_SetString(PyExc_TypeError, "must be a 20 character string");
+    return 0;
+  }
+}
+
+#endif  // PLASMA_EXTENSION_H
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
new file mode 100644
index 0000000000000..79da4f43a192a
--- /dev/null
+++ b/cpp/src/plasma/fling.cc
@@ -0,0 +1,90 @@
+// Copyright 2013 Sharvil Nanavati
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "plasma/fling.h"
+
+#include <string.h>
+
+void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len) {
+  iov->iov_base = buf;
+  iov->iov_len = 1;
+
+  msg->msg_iov = iov;
+  msg->msg_iovlen = 1;
+  msg->msg_control = buf;
+  msg->msg_controllen = buf_len;
+  msg->msg_name = NULL;
+  msg->msg_namelen = 0;
+}
+
+int send_fd(int conn, int fd) {
+  struct msghdr msg;
+  struct iovec iov;
+  char buf[CMSG_SPACE(sizeof(int))];
+  memset(&buf, 0, CMSG_SPACE(sizeof(int)));
+
+  init_msg(&msg, &iov, buf, sizeof(buf));
+
+  struct cmsghdr* header = CMSG_FIRSTHDR(&msg);
+  header->cmsg_level = SOL_SOCKET;
+  header->cmsg_type = SCM_RIGHTS;
+  header->cmsg_len = CMSG_LEN(sizeof(int));
+  *reinterpret_cast<int*>(CMSG_DATA(header)) = fd;
+
+  // Send file descriptor.
+  ssize_t r = sendmsg(conn, &msg, 0);
+  if (r >= 0) {
+    return 0;
+  } else {
+    return static_cast<int>(r);
+  }
+}
+
+int recv_fd(int conn) {
+  struct msghdr msg;
+  struct iovec iov;
+  char buf[CMSG_SPACE(sizeof(int))];
+  init_msg(&msg, &iov, buf, sizeof(buf));
+
+  if (recvmsg(conn, &msg, 0) == -1) return -1;
+
+  int found_fd = -1;
+  int oh_noes = 0;
+  for (struct cmsghdr* header = CMSG_FIRSTHDR(&msg); header != NULL;
+       header = CMSG_NXTHDR(&msg, header))
+    if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) {
+      ssize_t count =
+          (header->cmsg_len - (CMSG_DATA(header) - (unsigned char*)header)) / sizeof(int);
+      for (int i = 0; i < count; ++i) {
+        int fd = (reinterpret_cast<int*>(CMSG_DATA(header)))[i];
+        if (found_fd == -1) {
+          found_fd = fd;
+        } else {
+          close(fd);
+          oh_noes = 1;
+        }
+      }
+    }
+
+  // The sender sent us more than one file descriptor. We've closed
+  // them all to prevent fd leaks but notify the caller that we got
+  // a bad message.
+  if (oh_noes) {
+    close(found_fd);
+    errno = EBADMSG;
+    return -1;
+  }
+
+  return found_fd;
+}
diff --git a/cpp/src/plasma/fling.h b/cpp/src/plasma/fling.h
new file mode 100644
index 0000000000000..78ac9d17f26fb
--- /dev/null
+++ b/cpp/src/plasma/fling.h
@@ -0,0 +1,52 @@
+// Copyright 2013 Sharvil Nanavati
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// FLING: Exchanging file descriptors over sockets
+//
+// This is a little library for sending file descriptors over a socket
+// between processes. The reason for doing that (as opposed to using
+// filenames to share the files) is so (a) no files remain in the
+// filesystem after all the processes terminate, (b) to make sure that
+// there are no name collisions and (c) to be able to control who has
+// access to the data.
+//
+// Most of the code is from https://github.com/sharvil/flingfd
+
+#include <errno.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+// This is neccessary for Mac OS X, see http://www.apuebook.com/faqs2e.html
+// (10).
+#if !defined(CMSG_SPACE) && !defined(CMSG_LEN)
+#define CMSG_SPACE(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + __DARWIN_ALIGN32(len))
+#define CMSG_LEN(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + (len))
+#endif
+
+void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len);
+
+// Send a file descriptor over a unix domain socket.
+//
+// @param conn Unix domain socket to send the file descriptor over.
+// @param fd File descriptor to send over.
+// @return Status code which is < 0 on failure.
+int send_fd(int conn, int fd);
+
+// Receive a file descriptor over a unix domain socket.
+//
+// @param conn Unix domain socket to receive the file descriptor from.
+// @return File descriptor or a value < 0 on failure.
+int recv_fd(int conn);
diff --git a/cpp/src/plasma/format/common.fbs b/cpp/src/plasma/format/common.fbs
new file mode 100644
index 0000000000000..4d7d2852aec3d
--- /dev/null
+++ b/cpp/src/plasma/format/common.fbs
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Object information data structure.
+table ObjectInfo {
+  // Object ID of this object.
+  object_id: string;
+  // Number of bytes the content of this object occupies in memory.
+  data_size: long;
+  // Number of bytes the metadata of this object occupies in memory.
+  metadata_size: long;
+  // Unix epoch of when this object was created.
+  create_time: long;
+  // How long creation of this object took.
+  construct_duration: long;
+  // Hash of the object content.
+  digest: string;
+  // Specifies if this object was deleted or added.
+  is_deletion: bool;
+}
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
new file mode 100644
index 0000000000000..23782ade539d4
--- /dev/null
+++ b/cpp/src/plasma/format/plasma.fbs
@@ -0,0 +1,291 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Plasma protocol specification
+
+enum MessageType:int {
+  // Create a new object.
+  PlasmaCreateRequest = 1,
+  PlasmaCreateReply,
+  // Seal an object.
+  PlasmaSealRequest,
+  PlasmaSealReply,
+  // Get an object that is stored on the local Plasma store.
+  PlasmaGetRequest,
+  PlasmaGetReply,
+  // Release an object.
+  PlasmaReleaseRequest,
+  PlasmaReleaseReply,
+  // Delete an object.
+  PlasmaDeleteRequest,
+  PlasmaDeleteReply,
+  // Get status of an object.
+  PlasmaStatusRequest,
+  PlasmaStatusReply,
+  // See if the store contains an object (will be deprecated).
+  PlasmaContainsRequest,
+  PlasmaContainsReply,
+  // Get information for a newly connecting client.
+  PlasmaConnectRequest,
+  PlasmaConnectReply,
+  // Make room for new objects in the plasma store.
+  PlasmaEvictRequest,
+  PlasmaEvictReply,
+  // Fetch objects from remote Plasma stores.
+  PlasmaFetchRequest,
+  // Wait for objects to be ready either from local or remote Plasma stores.
+  PlasmaWaitRequest,
+  PlasmaWaitReply,
+  // Subscribe to a list of objects or to all objects.
+  PlasmaSubscribeRequest,
+  // Unsubscribe.
+  PlasmaUnsubscribeRequest,
+  // Sending and receiving data.
+  // PlasmaDataRequest initiates sending the data, there will be one
+  // such message per data transfer.
+  PlasmaDataRequest,
+  // PlasmaDataReply contains the actual data and is sent back to the
+  // object store that requested the data. For each transfer, multiple
+  // reply messages get sent. Each one contains a fixed number of bytes.
+  PlasmaDataReply,
+  // Object notifications.
+  PlasmaNotification
+}
+
+enum PlasmaError:int {
+  // Operation was successful.
+  OK,
+  // Trying to create an object that already exists.
+  ObjectExists,
+  // Trying to access an object that doesn't exist.
+  ObjectNonexistent,
+  // Trying to create an object but there isn't enough space in the store.
+  OutOfMemory
+}
+
+// Plasma store messages
+
+struct PlasmaObjectSpec {
+  // Index of the memory segment (= memory mapped file) that
+  // this object is allocated in.
+  segment_index: int;
+  // Size in bytes of this segment (needed to call mmap).
+  mmap_size: ulong;
+  // The offset in bytes in the memory mapped file of the data.
+  data_offset: ulong;
+  // The size in bytes of the data.
+  data_size: ulong;
+  // The offset in bytes in the memory mapped file of the metadata.
+  metadata_offset: ulong;
+  // The size in bytes of the metadata.
+  metadata_size: ulong;
+}
+
+table PlasmaCreateRequest {
+  // ID of the object to be created.
+  object_id: string;
+  // The size of the object's data in bytes.
+  data_size: ulong;
+  // The size of the object's metadata in bytes.
+  metadata_size: ulong;
+}
+
+table PlasmaCreateReply {
+  // ID of the object that was created.
+  object_id: string;
+  // The object that is returned with this reply.
+  plasma_object: PlasmaObjectSpec;
+  // Error that occurred for this call.
+  error: PlasmaError;
+}
+
+table PlasmaSealRequest {
+  // ID of the object to be sealed.
+  object_id: string;
+  // Hash of the object data.
+  digest: string;
+}
+
+table PlasmaSealReply {
+  // ID of the object that was sealed.
+  object_id: string;
+  // Error code.
+  error: PlasmaError;
+}
+
+table PlasmaGetRequest {
+  // IDs of the objects stored at local Plasma store we are getting.
+  object_ids: [string];
+  // The number of milliseconds before the request should timeout.
+  timeout_ms: long;
+}
+
+table PlasmaGetReply {
+  // IDs of the objects being returned.
+  // This number can be smaller than the number of requested
+  // objects if not all requested objects are stored and sealed
+  // in the local Plasma store.
+  object_ids: [string];
+  // Plasma object information, in the same order as their IDs.
+  plasma_objects: [PlasmaObjectSpec];
+  // The number of elements in both object_ids and plasma_objects arrays must agree.
+}
+
+table PlasmaReleaseRequest {
+  // ID of the object to be released.
+  object_id: string;
+}
+
+table PlasmaReleaseReply {
+  // ID of the object that was released.
+  object_id: string;
+  // Error code.
+  error: PlasmaError;
+}
+
+table PlasmaDeleteRequest {
+  // ID of the object to be deleted.
+  object_id: string;
+}
+
+table PlasmaDeleteReply {
+  // ID of the object that was deleted.
+  object_id: string;
+  // Error code.
+  error: PlasmaError;
+}
+
+table PlasmaStatusRequest {
+  // IDs of the objects stored at local Plasma store we request the status of.
+  object_ids: [string];
+}
+
+enum ObjectStatus:int {
+  // Object is stored in the local Plasma Store.
+  Local = 1,
+  // Object is stored on a remote Plasma store, and it is not stored on the
+  // local Plasma Store.
+  Remote,
+  // Object is not stored in the system.
+  Nonexistent,
+  // Object is currently transferred from a remote Plasma store the the local
+  // Plasma Store.
+  Transfer
+}
+
+table PlasmaStatusReply {
+  // IDs of the objects being returned.
+  object_ids: [string];
+  // Status of the object.
+  status: [ObjectStatus];
+}
+
+// PlasmaContains is a subset of PlasmaStatus which does not
+// involve the plasma manager, only the store. We should consider
+// unifying them in the future and deprecating PlasmaContains.
+
+table PlasmaContainsRequest {
+  // ID of the object we are querying.
+  object_id: string;
+}
+
+table PlasmaContainsReply {
+  // ID of the object we are querying.
+  object_id: string;
+  // 1 if the object is in the store and 0 otherwise.
+  has_object: int;
+}
+
+// PlasmaConnect is used by a plasma client the first time it connects with the
+// store. This is not really necessary, but is used to get some information
+// about the store such as its memory capacity.
+
+table PlasmaConnectRequest {
+}
+
+table PlasmaConnectReply {
+  // The memory capacity of the store.
+  memory_capacity: long;
+}
+
+table PlasmaEvictRequest {
+  // Number of bytes that shall be freed.
+  num_bytes: ulong;
+}
+
+table PlasmaEvictReply {
+  // Number of bytes that have been freed.
+  num_bytes: ulong;
+}
+
+table PlasmaFetchRequest {
+  // IDs of objects to be gotten.
+  object_ids: [string];
+}
+
+table ObjectRequestSpec {
+  // ID of the object.
+  object_id: string;
+  // The type of the object. This specifies whether we
+  // will be waiting for an object store in the local or
+  // global Plasma store.
+  type: int;
+}
+
+table PlasmaWaitRequest {
+  // Array of object requests whose status we are asking for.
+  object_requests: [ObjectRequestSpec];
+  // Number of objects expected to be returned, if available.
+  num_ready_objects: int;
+  // timeout
+  timeout: long;
+}
+
+table ObjectReply {
+  // ID of the object.
+  object_id: string;
+  // The object status. This specifies where the object is stored.
+  status: int;
+}
+
+table PlasmaWaitReply {
+  // Array of object requests being returned.
+  object_requests: [ObjectReply];
+  // Number of objects expected to be returned, if available.
+  num_ready_objects: int;
+}
+
+table PlasmaSubscribeRequest {
+}
+
+table PlasmaDataRequest {
+  // ID of the object that is requested.
+  object_id: string;
+  // The host address where the data shall be sent to.
+  address: string;
+  // The port of the manager the data shall be sent to.
+  port: int;
+}
+
+table PlasmaDataReply {
+  // ID of the object that will be sent.
+  object_id: string;
+  // Size of the object data in bytes.
+  object_size: ulong;
+  // Size of the metadata in bytes.
+  metadata_size: ulong;
+}
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
new file mode 100644
index 0000000000000..5875ebb7ae611
--- /dev/null
+++ b/cpp/src/plasma/io.cc
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/io.h"
+
+#include "plasma/common.h"
+
+using arrow::Status;
+
+/* Number of times we try binding to a socket. */
+#define NUM_BIND_ATTEMPTS 5
+#define BIND_TIMEOUT_MS 100
+
+/* Number of times we try connecting to a socket. */
+#define NUM_CONNECT_ATTEMPTS 50
+#define CONNECT_TIMEOUT_MS 100
+
+Status WriteBytes(int fd, uint8_t* cursor, size_t length) {
+  ssize_t nbytes = 0;
+  size_t bytesleft = length;
+  size_t offset = 0;
+  while (bytesleft > 0) {
+    /* While we haven't written the whole message, write to the file descriptor,
+     * advance the cursor, and decrease the amount left to write. */
+    nbytes = write(fd, cursor + offset, bytesleft);
+    if (nbytes < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
+      return Status::IOError(std::string(strerror(errno)));
+    } else if (nbytes == 0) {
+      return Status::IOError("Encountered unexpected EOF");
+    }
+    ARROW_CHECK(nbytes > 0);
+    bytesleft -= nbytes;
+    offset += nbytes;
+  }
+
+  return Status::OK();
+}
+
+Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes) {
+  int64_t version = PLASMA_PROTOCOL_VERSION;
+  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)));
+  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&type), sizeof(type)));
+  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)));
+  return WriteBytes(fd, bytes, length * sizeof(char));
+}
+
+Status ReadBytes(int fd, uint8_t* cursor, size_t length) {
+  ssize_t nbytes = 0;
+  /* Termination condition: EOF or read 'length' bytes total. */
+  size_t bytesleft = length;
+  size_t offset = 0;
+  while (bytesleft > 0) {
+    nbytes = read(fd, cursor + offset, bytesleft);
+    if (nbytes < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
+      return Status::IOError(std::string(strerror(errno)));
+    } else if (0 == nbytes) {
+      return Status::IOError("Encountered unexpected EOF");
+    }
+    ARROW_CHECK(nbytes > 0);
+    bytesleft -= nbytes;
+    offset += nbytes;
+  }
+
+  return Status::OK();
+}
+
+Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
+  int64_t version;
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)),
+      *type = DISCONNECT_CLIENT);
+  ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version;
+  size_t length;
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(type), sizeof(*type)),
+      *type = DISCONNECT_CLIENT);
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)),
+      *type = DISCONNECT_CLIENT);
+  if (length > buffer->size()) { buffer->resize(length); }
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer->data(), length), *type = DISCONNECT_CLIENT);
+  return Status::OK();
+}
+
+int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
+  struct sockaddr_un socket_address;
+  int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (socket_fd < 0) {
+    ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
+    return -1;
+  }
+  /* Tell the system to allow the port to be reused. */
+  int on = 1;
+  if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&on),
+          sizeof(on)) < 0) {
+    ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname;
+    close(socket_fd);
+    return -1;
+  }
+
+  unlink(pathname.c_str());
+  memset(&socket_address, 0, sizeof(socket_address));
+  socket_address.sun_family = AF_UNIX;
+  if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
+    ARROW_LOG(ERROR) << "Socket pathname is too long.";
+    close(socket_fd);
+    return -1;
+  }
+  strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
+
+  if (bind(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) != 0) {
+    ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname;
+    close(socket_fd);
+    return -1;
+  }
+  if (shall_listen && listen(socket_fd, 128) == -1) {
+    ARROW_LOG(ERROR) << "Could not listen to socket " << pathname;
+    close(socket_fd);
+    return -1;
+  }
+  return socket_fd;
+}
+
+int connect_ipc_sock_retry(
+    const std::string& pathname, int num_retries, int64_t timeout) {
+  /* Pick the default values if the user did not specify. */
+  if (num_retries < 0) { num_retries = NUM_CONNECT_ATTEMPTS; }
+  if (timeout < 0) { timeout = CONNECT_TIMEOUT_MS; }
+
+  int fd = -1;
+  for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
+    fd = connect_ipc_sock(pathname);
+    if (fd >= 0) { break; }
+    if (num_attempts == 0) {
+      ARROW_LOG(ERROR) << "Connection to socket failed for pathname " << pathname;
+    }
+    /* Sleep for timeout milliseconds. */
+    usleep(static_cast<int>(timeout * 1000));
+  }
+  /* If we could not connect to the socket, exit. */
+  if (fd == -1) { ARROW_LOG(FATAL) << "Could not connect to socket " << pathname; }
+  return fd;
+}
+
+int connect_ipc_sock(const std::string& pathname) {
+  struct sockaddr_un socket_address;
+  int socket_fd;
+
+  socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (socket_fd < 0) {
+    ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
+    return -1;
+  }
+
+  memset(&socket_address, 0, sizeof(socket_address));
+  socket_address.sun_family = AF_UNIX;
+  if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
+    ARROW_LOG(ERROR) << "Socket pathname is too long.";
+    return -1;
+  }
+  strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
+
+  if (connect(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) !=
+      0) {
+    close(socket_fd);
+    return -1;
+  }
+
+  return socket_fd;
+}
+
+int AcceptClient(int socket_fd) {
+  int client_fd = accept(socket_fd, NULL, NULL);
+  if (client_fd < 0) {
+    ARROW_LOG(ERROR) << "Error reading from socket.";
+    return -1;
+  }
+  return client_fd;
+}
+
+uint8_t* read_message_async(int sock) {
+  int64_t size;
+  Status s = ReadBytes(sock, reinterpret_cast<uint8_t*>(&size), sizeof(int64_t));
+  if (!s.ok()) {
+    /* The other side has closed the socket. */
+    ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
+    close(sock);
+    return NULL;
+  }
+  uint8_t* message = reinterpret_cast<uint8_t*>(malloc(size));
+  s = ReadBytes(sock, message, size);
+  if (!s.ok()) {
+    /* The other side has closed the socket. */
+    ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
+    close(sock);
+    return NULL;
+  }
+  return message;
+}
diff --git a/cpp/src/plasma/io.h b/cpp/src/plasma/io.h
new file mode 100644
index 0000000000000..43c3fb535497f
--- /dev/null
+++ b/cpp/src/plasma/io.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_IO_H
+#define PLASMA_IO_H
+
+#include <inttypes.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <string>
+#include <vector>
+
+#include "arrow/status.h"
+
+// TODO(pcm): Replace our own custom message header (message type,
+// message length, plasma protocol verion) with one that is serialized
+// using flatbuffers.
+#define PLASMA_PROTOCOL_VERSION 0x0000000000000000
+#define DISCONNECT_CLIENT 0
+
+arrow::Status WriteBytes(int fd, uint8_t* cursor, size_t length);
+
+arrow::Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes);
+
+arrow::Status ReadBytes(int fd, uint8_t* cursor, size_t length);
+
+arrow::Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer);
+
+int bind_ipc_sock(const std::string& pathname, bool shall_listen);
+
+int connect_ipc_sock(const std::string& pathname);
+
+int connect_ipc_sock_retry(const std::string& pathname, int num_retries, int64_t timeout);
+
+int AcceptClient(int socket_fd);
+
+uint8_t* read_message_async(int sock);
+
+#endif  // PLASMA_IO_H
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
new file mode 100644
index 0000000000000..e7ffd1ad37797
--- /dev/null
+++ b/cpp/src/plasma/malloc.cc
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/malloc.h"
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <unordered_map>
+
+#include "plasma/common.h"
+
+extern "C" {
+void* fake_mmap(size_t);
+int fake_munmap(void*, int64_t);
+
+#define MMAP(s) fake_mmap(s)
+#define MUNMAP(a, s) fake_munmap(a, s)
+#define DIRECT_MMAP(s) fake_mmap(s)
+#define DIRECT_MUNMAP(a, s) fake_munmap(a, s)
+#define USE_DL_PREFIX
+#define HAVE_MORECORE 0
+#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
+#define DEFAULT_GRANULARITY ((size_t)128U * 1024U)
+
+#include "thirdparty/dlmalloc.c"
+
+#undef MMAP
+#undef MUNMAP
+#undef DIRECT_MMAP
+#undef DIRECT_MUNMAP
+#undef USE_DL_PREFIX
+#undef HAVE_MORECORE
+#undef DEFAULT_GRANULARITY
+}
+
+struct mmap_record {
+  int fd;
+  int64_t size;
+};
+
+namespace {
+
+/** Hashtable that contains one entry per segment that we got from the OS
+ *  via mmap. Associates the address of that segment with its file descriptor
+ *  and size. */
+std::unordered_map<void*, mmap_record> mmap_records;
+
+} /* namespace */
+
+constexpr int GRANULARITY_MULTIPLIER = 2;
+
+static void* pointer_advance(void* p, ptrdiff_t n) {
+  return (unsigned char*)p + n;
+}
+
+static void* pointer_retreat(void* p, ptrdiff_t n) {
+  return (unsigned char*)p - n;
+}
+
+static ptrdiff_t pointer_distance(void const* pfrom, void const* pto) {
+  return (unsigned char const*)pto - (unsigned char const*)pfrom;
+}
+
+/* Create a buffer. This is creating a temporary file and then
+ * immediately unlinking it so we do not leave traces in the system. */
+int create_buffer(int64_t size) {
+  int fd;
+#ifdef _WIN32
+  if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
+          (DWORD)((uint64_t)size >> (CHAR_BIT * sizeof(DWORD))), (DWORD)(uint64_t)size,
+          NULL)) {
+    fd = -1;
+  }
+#else
+#ifdef __linux__
+  constexpr char file_template[] = "/dev/shm/plasmaXXXXXX";
+#else
+  constexpr char file_template[] = "/tmp/plasmaXXXXXX";
+#endif
+  char file_name[32];
+  strncpy(file_name, file_template, 32);
+  fd = mkstemp(file_name);
+  if (fd < 0) return -1;
+  FILE* file = fdopen(fd, "a+");
+  if (!file) {
+    close(fd);
+    return -1;
+  }
+  if (unlink(file_name) != 0) {
+    ARROW_LOG(FATAL) << "unlink error";
+    return -1;
+  }
+  if (ftruncate(fd, (off_t)size) != 0) {
+    ARROW_LOG(FATAL) << "ftruncate error";
+    return -1;
+  }
+#endif
+  return fd;
+}
+
+void* fake_mmap(size_t size) {
+  /* Add sizeof(size_t) so that the returned pointer is deliberately not
+   * page-aligned. This ensures that the segments of memory returned by
+   * fake_mmap are never contiguous. */
+  size += sizeof(size_t);
+
+  int fd = create_buffer(size);
+  ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
+  void* pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+  if (pointer == MAP_FAILED) { return pointer; }
+
+  /* Increase dlmalloc's allocation granularity directly. */
+  mparams.granularity *= GRANULARITY_MULTIPLIER;
+
+  mmap_record& record = mmap_records[pointer];
+  record.fd = fd;
+  record.size = size;
+
+  /* We lie to dlmalloc about where mapped memory actually lives. */
+  pointer = pointer_advance(pointer, sizeof(size_t));
+  ARROW_LOG(DEBUG) << pointer << " = fake_mmap(" << size << ")";
+  return pointer;
+}
+
+int fake_munmap(void* addr, int64_t size) {
+  ARROW_LOG(DEBUG) << "fake_munmap(" << addr << ", " << size << ")";
+  addr = pointer_retreat(addr, sizeof(size_t));
+  size += sizeof(size_t);
+
+  auto entry = mmap_records.find(addr);
+
+  if (entry == mmap_records.end() || entry->second.size != size) {
+    /* Reject requests to munmap that don't directly match previous
+     * calls to mmap, to prevent dlmalloc from trimming. */
+    return -1;
+  }
+
+  int r = munmap(addr, size);
+  if (r == 0) { close(entry->second.fd); }
+
+  mmap_records.erase(entry);
+  return r;
+}
+
+void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_size, ptrdiff_t* offset) {
+  /* TODO(rshin): Implement a more efficient search through mmap_records. */
+  for (const auto& entry : mmap_records) {
+    if (addr >= entry.first && addr < pointer_advance(entry.first, entry.second.size)) {
+      *fd = entry.second.fd;
+      *map_size = entry.second.size;
+      *offset = pointer_distance(entry.first, addr);
+      return;
+    }
+  }
+  *fd = -1;
+  *map_size = 0;
+  *offset = 0;
+}
diff --git a/cpp/src/plasma/malloc.h b/cpp/src/plasma/malloc.h
new file mode 100644
index 0000000000000..b4af2c826b5c9
--- /dev/null
+++ b/cpp/src/plasma/malloc.h
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_MALLOC_H
+#define PLASMA_MALLOC_H
+
+#include <inttypes.h>
+#include <stddef.h>
+
+void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_length, ptrdiff_t* offset);
+
+#endif  // MALLOC_H
diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc
new file mode 100644
index 0000000000000..559d8e7f2a65e
--- /dev/null
+++ b/cpp/src/plasma/plasma.cc
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/plasma.h"
+
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/common.h"
+#include "plasma/protocol.h"
+
+int warn_if_sigpipe(int status, int client_sock) {
+  if (status >= 0) { return 0; }
+  if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
+    ARROW_LOG(WARNING) << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when "
+                          "sending a message to client on fd "
+                       << client_sock << ". The client on the other end may "
+                                         "have hung up.";
+    return errno;
+  }
+  ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock << ".";
+  return -1;  // This is never reached.
+}
+
+/**
+ * This will create a new ObjectInfo buffer. The first sizeof(int64_t) bytes
+ * of this buffer are the length of the remaining message and the
+ * remaining message is a serialized version of the object info.
+ *
+ * @param object_info The object info to be serialized
+ * @return The object info buffer. It is the caller's responsibility to free
+ *         this buffer with "delete" after it has been used.
+ */
+uint8_t* create_object_info_buffer(ObjectInfoT* object_info) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreateObjectInfo(fbb, object_info);
+  fbb.Finish(message);
+  uint8_t* notification = new uint8_t[sizeof(int64_t) + fbb.GetSize()];
+  *(reinterpret_cast<int64_t*>(notification)) = fbb.GetSize();
+  memcpy(notification + sizeof(int64_t), fbb.GetBufferPointer(), fbb.GetSize());
+  return notification;
+}
+
+ObjectTableEntry* get_object_table_entry(
+    PlasmaStoreInfo* store_info, const ObjectID& object_id) {
+  auto it = store_info->objects.find(object_id);
+  if (it == store_info->objects.end()) { return NULL; }
+  return it->second.get();
+}
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
new file mode 100644
index 0000000000000..275d0c7a41687
--- /dev/null
+++ b/cpp/src/plasma/plasma.h
@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_PLASMA_H
+#define PLASMA_PLASMA_H
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>  // pid_t
+
+#include <unordered_map>
+#include <unordered_set>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+#include "format/common_generated.h"
+#include "plasma/common.h"
+
+#define HANDLE_SIGPIPE(s, fd_)                                              \
+  do {                                                                      \
+    Status _s = (s);                                                        \
+    if (!_s.ok()) {                                                         \
+      if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {        \
+        ARROW_LOG(WARNING)                                                  \
+            << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \
+               "sending a message to client on fd "                         \
+            << fd_ << ". "                                                  \
+                      "The client on the other end may have hung up.";      \
+      } else {                                                              \
+        return _s;                                                          \
+      }                                                                     \
+    }                                                                       \
+  } while (0);
+
+/// Allocation granularity used in plasma for object allocation.
+#define BLOCK_SIZE 64
+
+/// Size of object hash digests.
+constexpr int64_t kDigestSize = sizeof(uint64_t);
+
+struct Client;
+
+/// Object request data structure. Used in the plasma_wait_for_objects()
+/// argument.
+typedef struct {
+  /// The ID of the requested object. If ID_NIL request any object.
+  ObjectID object_id;
+  /// Request associated to the object. It can take one of the following values:
+  ///  - PLASMA_QUERY_LOCAL: return if or when the object is available in the
+  ///    local Plasma Store.
+  ///  - PLASMA_QUERY_ANYWHERE: return if or when the object is available in
+  ///    the system (i.e., either in the local or a remote Plasma Store).
+  int type;
+  /// Object status. Same as the status returned by plasma_status() function
+  /// call. This is filled in by plasma_wait_for_objects1():
+  ///  - ObjectStatus_Local: object is ready at the local Plasma Store.
+  ///  - ObjectStatus_Remote: object is ready at a remote Plasma Store.
+  ///  - ObjectStatus_Nonexistent: object does not exist in the system.
+  ///  - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
+  ///    for being transferred or it is transferring.
+  int status;
+} ObjectRequest;
+
+/// Mapping from object IDs to type and status of the request.
+typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectRequestMap;
+
+/// Handle to access memory mapped file and map it into client address space.
+typedef struct {
+  /// The file descriptor of the memory mapped file in the store. It is used as
+  /// a unique identifier of the file in the client to look up the corresponding
+  /// file descriptor on the client's side.
+  int store_fd;
+  /// The size in bytes of the memory mapped file.
+  int64_t mmap_size;
+} object_handle;
+
+// TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
+typedef struct {
+  /// Handle for memory mapped file the object is stored in.
+  object_handle handle;
+  /// The offset in bytes in the memory mapped file of the data.
+  ptrdiff_t data_offset;
+  /// The offset in bytes in the memory mapped file of the metadata.
+  ptrdiff_t metadata_offset;
+  /// The size in bytes of the data.
+  int64_t data_size;
+  /// The size in bytes of the metadata.
+  int64_t metadata_size;
+} PlasmaObject;
+
+typedef enum {
+  /// Object was created but not sealed in the local Plasma Store.
+  PLASMA_CREATED = 1,
+  /// Object is sealed and stored in the local Plasma Store.
+  PLASMA_SEALED
+} object_state;
+
+typedef enum {
+  /// The object was not found.
+  OBJECT_NOT_FOUND = 0,
+  /// The object was found.
+  OBJECT_FOUND = 1
+} object_status;
+
+typedef enum {
+  /// Query for object in the local plasma store.
+  PLASMA_QUERY_LOCAL = 1,
+  /// Query for object in the local plasma store or in a remote plasma store.
+  PLASMA_QUERY_ANYWHERE
+} object_request_type;
+
+/// This type is used by the Plasma store. It is here because it is exposed to
+/// the eviction policy.
+struct ObjectTableEntry {
+  /// Object id of this object.
+  ObjectID object_id;
+  /// Object info like size, creation time and owner.
+  ObjectInfoT info;
+  /// Memory mapped file containing the object.
+  int fd;
+  /// Size of the underlying map.
+  int64_t map_size;
+  /// Offset from the base of the mmap.
+  ptrdiff_t offset;
+  /// Pointer to the object data. Needed to free the object.
+  uint8_t* pointer;
+  /// Set of clients currently using this object.
+  std::unordered_set<Client*> clients;
+  /// The state of the object, e.g., whether it is open or sealed.
+  object_state state;
+  /// The digest of the object. Used to see if two objects are the same.
+  unsigned char digest[kDigestSize];
+};
+
+/// The plasma store information that is exposed to the eviction policy.
+struct PlasmaStoreInfo {
+  /// Objects that are in the Plasma store.
+  std::unordered_map<ObjectID, std::unique_ptr<ObjectTableEntry>, UniqueIDHasher> objects;
+  /// The amount of memory (in bytes) that we allow to be allocated in the
+  /// store.
+  int64_t memory_capacity;
+};
+
+/// Get an entry from the object table and return NULL if the object_id
+/// is not present.
+///
+/// @param store_info The PlasmaStoreInfo that contains the object table.
+/// @param object_id The object_id of the entry we are looking for.
+/// @return The entry associated with the object_id or NULL if the object_id
+///         is not present.
+ObjectTableEntry* get_object_table_entry(
+    PlasmaStoreInfo* store_info, const ObjectID& object_id);
+
+/// Print a warning if the status is less than zero. This should be used to check
+/// the success of messages sent to plasma clients. We print a warning instead of
+/// failing because the plasma clients are allowed to die. This is used to handle
+/// situations where the store writes to a client file descriptor, and the client
+/// may already have disconnected. If we have processed the disconnection and
+/// closed the file descriptor, we should get a BAD FILE DESCRIPTOR error. If we
+/// have not, then we should get a SIGPIPE. If we write to a TCP socket that
+/// isn't connected yet, then we should get an ECONNRESET.
+///
+/// @param status The status to check. If it is less less than zero, we will
+///        print a warning.
+/// @param client_sock The client socket. This is just used to print some extra
+///        information.
+/// @return The errno set.
+int warn_if_sigpipe(int status, int client_sock);
+
+uint8_t* create_object_info_buffer(ObjectInfoT* object_info);
+
+#endif  // PLASMA_PLASMA_H
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
new file mode 100644
index 0000000000000..246aa29736056
--- /dev/null
+++ b/cpp/src/plasma/protocol.cc
@@ -0,0 +1,502 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/protocol.h"
+
+#include "flatbuffers/flatbuffers.h"
+#include "format/plasma_generated.h"
+
+#include "plasma/common.h"
+#include "plasma/io.h"
+
+using flatbuffers::uoffset_t;
+
+flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
+to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
+    int64_t num_objects) {
+  std::vector<flatbuffers::Offset<flatbuffers::String>> results;
+  for (int64_t i = 0; i < num_objects; i++) {
+    results.push_back(fbb->CreateString(object_ids[i].binary()));
+  }
+  return fbb->CreateVector(results);
+}
+
+Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer) {
+  int64_t type;
+  RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
+  ARROW_CHECK(type == message_type) << "type = " << type
+                                    << ", message_type = " << message_type;
+  return Status::OK();
+}
+
+template <typename Message>
+Status PlasmaSend(int sock, int64_t message_type, flatbuffers::FlatBufferBuilder* fbb,
+    const Message& message) {
+  fbb->Finish(message);
+  return WriteMessage(sock, message_type, fbb->GetSize(), fbb->GetBufferPointer());
+}
+
+// Create messages.
+
+Status SendCreateRequest(
+    int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaCreateRequest(
+      fbb, fbb.CreateString(object_id.binary()), data_size, metadata_size);
+  return PlasmaSend(sock, MessageType_PlasmaCreateRequest, &fbb, message);
+}
+
+Status ReadCreateRequest(
+    uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
+  *data_size = message->data_size();
+  *metadata_size = message->metadata_size();
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendCreateReply(
+    int sock, ObjectID object_id, PlasmaObject* object, int error_code) {
+  flatbuffers::FlatBufferBuilder fbb;
+  PlasmaObjectSpec plasma_object(object->handle.store_fd, object->handle.mmap_size,
+      object->data_offset, object->data_size, object->metadata_offset,
+      object->metadata_size);
+  auto message = CreatePlasmaCreateReply(
+      fbb, fbb.CreateString(object_id.binary()), &plasma_object, (PlasmaError)error_code);
+  return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
+}
+
+Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  object->handle.store_fd = message->plasma_object()->segment_index();
+  object->handle.mmap_size = message->plasma_object()->mmap_size();
+  object->data_offset = message->plasma_object()->data_offset();
+  object->data_size = message->plasma_object()->data_size();
+  object->metadata_offset = message->plasma_object()->metadata_offset();
+  object->metadata_size = message->plasma_object()->metadata_size();
+  return plasma_error_status(message->error());
+}
+
+// Seal messages.
+
+Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto digest_string = fbb.CreateString(reinterpret_cast<char*>(digest), kDigestSize);
+  auto message =
+      CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()), digest_string);
+  return PlasmaSend(sock, MessageType_PlasmaSealRequest, &fbb, message);
+}
+
+Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaSealRequest>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  ARROW_CHECK(message->digest()->size() == kDigestSize);
+  memcpy(digest, message->digest()->data(), kDigestSize);
+  return Status::OK();
+}
+
+Status SendSealReply(int sock, ObjectID object_id, int error) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaSealReply(
+      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  return PlasmaSend(sock, MessageType_PlasmaSealReply, &fbb, message);
+}
+
+Status ReadSealReply(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaSealReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return plasma_error_status(message->error());
+}
+
+// Release messages.
+
+Status SendReleaseRequest(int sock, ObjectID object_id) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()));
+  return PlasmaSend(sock, MessageType_PlasmaReleaseRequest, &fbb, message);
+}
+
+Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaReleaseRequest>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendReleaseReply(int sock, ObjectID object_id, int error) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaReleaseReply(
+      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  return PlasmaSend(sock, MessageType_PlasmaReleaseReply, &fbb, message);
+}
+
+Status ReadReleaseReply(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return plasma_error_status(message->error());
+}
+
+// Delete messages.
+
+Status SendDeleteRequest(int sock, ObjectID object_id) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaDeleteRequest(fbb, fbb.CreateString(object_id.binary()));
+  return PlasmaSend(sock, MessageType_PlasmaDeleteRequest, &fbb, message);
+}
+
+Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendDeleteReply(int sock, ObjectID object_id, int error) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaDeleteReply(
+      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  return PlasmaSend(sock, MessageType_PlasmaDeleteReply, &fbb, message);
+}
+
+Status ReadDeleteReply(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaDeleteReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return plasma_error_status(message->error());
+}
+
+// Satus messages.
+
+Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message =
+      CreatePlasmaStatusRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
+  return PlasmaSend(sock, MessageType_PlasmaStatusRequest, &fbb, message);
+}
+
+Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaStatusRequest>(data);
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+  }
+  return Status::OK();
+}
+
+Status SendStatusReply(
+    int sock, ObjectID object_ids[], int object_status[], int64_t num_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message =
+      CreatePlasmaStatusReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+          fbb.CreateVector(object_status, num_objects));
+  return PlasmaSend(sock, MessageType_PlasmaStatusReply, &fbb, message);
+}
+
+int64_t ReadStatusReply_num_objects(uint8_t* data) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
+  return message->object_ids()->size();
+}
+
+Status ReadStatusReply(
+    uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+  }
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    object_status[i] = message->status()->data()[i];
+  }
+  return Status::OK();
+}
+
+// Contains messages.
+
+Status SendContainsRequest(int sock, ObjectID object_id) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaContainsRequest(fbb, fbb.CreateString(object_id.binary()));
+  return PlasmaSend(sock, MessageType_PlasmaContainsRequest, &fbb, message);
+}
+
+Status ReadContainsRequest(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaContainsRequest>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendContainsReply(int sock, ObjectID object_id, bool has_object) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message =
+      CreatePlasmaContainsReply(fbb, fbb.CreateString(object_id.binary()), has_object);
+  return PlasmaSend(sock, MessageType_PlasmaContainsReply, &fbb, message);
+}
+
+Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaContainsReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  *has_object = message->has_object();
+  return Status::OK();
+}
+
+// Connect messages.
+
+Status SendConnectRequest(int sock) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaConnectRequest(fbb);
+  return PlasmaSend(sock, MessageType_PlasmaConnectRequest, &fbb, message);
+}
+
+Status ReadConnectRequest(uint8_t* data) {
+  return Status::OK();
+}
+
+Status SendConnectReply(int sock, int64_t memory_capacity) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaConnectReply(fbb, memory_capacity);
+  return PlasmaSend(sock, MessageType_PlasmaConnectReply, &fbb, message);
+}
+
+Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaConnectReply>(data);
+  *memory_capacity = message->memory_capacity();
+  return Status::OK();
+}
+
+// Evict messages.
+
+Status SendEvictRequest(int sock, int64_t num_bytes) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaEvictRequest(fbb, num_bytes);
+  return PlasmaSend(sock, MessageType_PlasmaEvictRequest, &fbb, message);
+}
+
+Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaEvictRequest>(data);
+  *num_bytes = message->num_bytes();
+  return Status::OK();
+}
+
+Status SendEvictReply(int sock, int64_t num_bytes) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaEvictReply(fbb, num_bytes);
+  return PlasmaSend(sock, MessageType_PlasmaEvictReply, &fbb, message);
+}
+
+Status ReadEvictReply(uint8_t* data, int64_t& num_bytes) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaEvictReply>(data);
+  num_bytes = message->num_bytes();
+  return Status::OK();
+}
+
+// Get messages.
+
+Status SendGetRequest(
+    int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaGetRequest(
+      fbb, to_flatbuffer(&fbb, object_ids, num_objects), timeout_ms);
+  return PlasmaSend(sock, MessageType_PlasmaGetRequest, &fbb, message);
+}
+
+Status ReadGetRequest(
+    uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
+  for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
+    auto object_id = message->object_ids()->Get(i)->str();
+    object_ids.push_back(ObjectID::from_binary(object_id));
+  }
+  *timeout_ms = message->timeout_ms();
+  return Status::OK();
+}
+
+Status SendGetReply(int sock, ObjectID object_ids[],
+    std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
+    int64_t num_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+  std::vector<PlasmaObjectSpec> objects;
+
+  for (int i = 0; i < num_objects; ++i) {
+    const PlasmaObject& object = plasma_objects[object_ids[i]];
+    objects.push_back(PlasmaObjectSpec(object.handle.store_fd, object.handle.mmap_size,
+        object.data_offset, object.data_size, object.metadata_offset,
+        object.metadata_size));
+  }
+  auto message = CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+      fbb.CreateVectorOfStructs(objects.data(), num_objects));
+  return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
+}
+
+Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
+    int64_t num_objects) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+  }
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    const PlasmaObjectSpec* object = message->plasma_objects()->Get(i);
+    plasma_objects[i].handle.store_fd = object->segment_index();
+    plasma_objects[i].handle.mmap_size = object->mmap_size();
+    plasma_objects[i].data_offset = object->data_offset();
+    plasma_objects[i].data_size = object->data_size();
+    plasma_objects[i].metadata_offset = object->metadata_offset();
+    plasma_objects[i].metadata_size = object->metadata_size();
+  }
+  return Status::OK();
+}
+
+// Fetch messages.
+
+Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message =
+      CreatePlasmaFetchRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
+  return PlasmaSend(sock, MessageType_PlasmaFetchRequest, &fbb, message);
+}
+
+Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaFetchRequest>(data);
+  for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
+    object_ids.push_back(ObjectID::from_binary(message->object_ids()->Get(i)->str()));
+  }
+  return Status::OK();
+}
+
+// Wait messages.
+
+Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
+    int num_ready_objects, int64_t timeout_ms) {
+  flatbuffers::FlatBufferBuilder fbb;
+
+  std::vector<flatbuffers::Offset<ObjectRequestSpec>> object_request_specs;
+  for (int i = 0; i < num_requests; i++) {
+    object_request_specs.push_back(CreateObjectRequestSpec(fbb,
+        fbb.CreateString(object_requests[i].object_id.binary()),
+        object_requests[i].type));
+  }
+
+  auto message = CreatePlasmaWaitRequest(
+      fbb, fbb.CreateVector(object_request_specs), num_ready_objects, timeout_ms);
+  return PlasmaSend(sock, MessageType_PlasmaWaitRequest, &fbb, message);
+}
+
+Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
+    int64_t* timeout_ms, int* num_ready_objects) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaWaitRequest>(data);
+  *num_ready_objects = message->num_ready_objects();
+  *timeout_ms = message->timeout();
+
+  for (uoffset_t i = 0; i < message->object_requests()->size(); i++) {
+    ObjectID object_id =
+        ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
+    ObjectRequest object_request({object_id, message->object_requests()->Get(i)->type(),
+        ObjectStatus_Nonexistent});
+    object_requests[object_id] = object_request;
+  }
+  return Status::OK();
+}
+
+Status SendWaitReply(
+    int sock, const ObjectRequestMap& object_requests, int num_ready_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+
+  std::vector<flatbuffers::Offset<ObjectReply>> object_replies;
+  for (const auto& entry : object_requests) {
+    const auto& object_request = entry.second;
+    object_replies.push_back(CreateObjectReply(
+        fbb, fbb.CreateString(object_request.object_id.binary()), object_request.status));
+  }
+
+  auto message = CreatePlasmaWaitReply(
+      fbb, fbb.CreateVector(object_replies.data(), num_ready_objects), num_ready_objects);
+  return PlasmaSend(sock, MessageType_PlasmaWaitReply, &fbb, message);
+}
+
+Status ReadWaitReply(
+    uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects) {
+  DCHECK(data);
+
+  auto message = flatbuffers::GetRoot<PlasmaWaitReply>(data);
+  *num_ready_objects = message->num_ready_objects();
+  for (int i = 0; i < *num_ready_objects; i++) {
+    object_requests[i].object_id =
+        ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
+    object_requests[i].status = message->object_requests()->Get(i)->status();
+  }
+  return Status::OK();
+}
+
+// Subscribe messages.
+
+Status SendSubscribeRequest(int sock) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaSubscribeRequest(fbb);
+  return PlasmaSend(sock, MessageType_PlasmaSubscribeRequest, &fbb, message);
+}
+
+// Data messages.
+
+Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto addr = fbb.CreateString(address, strlen(address));
+  auto message =
+      CreatePlasmaDataRequest(fbb, fbb.CreateString(object_id.binary()), addr, port);
+  return PlasmaSend(sock, MessageType_PlasmaDataRequest, &fbb, message);
+}
+
+Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaDataRequest>(data);
+  DCHECK(message->object_id()->size() == sizeof(ObjectID));
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  *address = strdup(message->address()->c_str());
+  *port = message->port();
+  return Status::OK();
+}
+
+Status SendDataReply(
+    int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaDataReply(
+      fbb, fbb.CreateString(object_id.binary()), object_size, metadata_size);
+  return PlasmaSend(sock, MessageType_PlasmaDataReply, &fbb, message);
+}
+
+Status ReadDataReply(
+    uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  *object_size = (int64_t)message->object_size();
+  *metadata_size = (int64_t)message->metadata_size();
+  return Status::OK();
+}
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
new file mode 100644
index 0000000000000..5d9d13675144f
--- /dev/null
+++ b/cpp/src/plasma/protocol.h
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_PROTOCOL_H
+#define PLASMA_PROTOCOL_H
+
+#include <vector>
+
+#include "arrow/status.h"
+#include "format/plasma_generated.h"
+#include "plasma/plasma.h"
+
+using arrow::Status;
+
+/* Plasma receive message. */
+
+Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer);
+
+/* Plasma Create message functions. */
+
+Status SendCreateRequest(
+    int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size);
+
+Status ReadCreateRequest(
+    uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size);
+
+Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error);
+
+Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object);
+
+/* Plasma Seal message functions. */
+
+Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest);
+
+Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest);
+
+Status SendSealReply(int sock, ObjectID object_id, int error);
+
+Status ReadSealReply(uint8_t* data, ObjectID* object_id);
+
+/* Plasma Get message functions. */
+
+Status SendGetRequest(
+    int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms);
+
+Status ReadGetRequest(
+    uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms);
+
+Status SendGetReply(int sock, ObjectID object_ids[],
+    std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
+    int64_t num_objects);
+
+Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
+    int64_t num_objects);
+
+/* Plasma Release message functions. */
+
+Status SendReleaseRequest(int sock, ObjectID object_id);
+
+Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendReleaseReply(int sock, ObjectID object_id, int error);
+
+Status ReadReleaseReply(uint8_t* data, ObjectID* object_id);
+
+/* Plasma Delete message functions. */
+
+Status SendDeleteRequest(int sock, ObjectID object_id);
+
+Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendDeleteReply(int sock, ObjectID object_id, int error);
+
+Status ReadDeleteReply(uint8_t* data, ObjectID* object_id);
+
+/* Satus messages. */
+
+Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
+
+Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects);
+
+Status SendStatusReply(
+    int sock, ObjectID object_ids[], int object_status[], int64_t num_objects);
+
+int64_t ReadStatusReply_num_objects(uint8_t* data);
+
+Status ReadStatusReply(
+    uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects);
+
+/* Plasma Constains message functions. */
+
+Status SendContainsRequest(int sock, ObjectID object_id);
+
+Status ReadContainsRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendContainsReply(int sock, ObjectID object_id, bool has_object);
+
+Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object);
+
+/* Plasma Connect message functions. */
+
+Status SendConnectRequest(int sock);
+
+Status ReadConnectRequest(uint8_t* data);
+
+Status SendConnectReply(int sock, int64_t memory_capacity);
+
+Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity);
+
+/* Plasma Evict message functions (no reply so far). */
+
+Status SendEvictRequest(int sock, int64_t num_bytes);
+
+Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes);
+
+Status SendEvictReply(int sock, int64_t num_bytes);
+
+Status ReadEvictReply(uint8_t* data, int64_t& num_bytes);
+
+/* Plasma Fetch Remote message functions. */
+
+Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
+
+Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids);
+
+/* Plasma Wait message functions. */
+
+Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
+    int num_ready_objects, int64_t timeout_ms);
+
+Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
+    int64_t* timeout_ms, int* num_ready_objects);
+
+Status SendWaitReply(
+    int sock, const ObjectRequestMap& object_requests, int num_ready_objects);
+
+Status ReadWaitReply(
+    uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects);
+
+/* Plasma Subscribe message functions. */
+
+Status SendSubscribeRequest(int sock);
+
+/* Data messages. */
+
+Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port);
+
+Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port);
+
+Status SendDataReply(
+    int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size);
+
+Status ReadDataReply(
+    uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size);
+
+#endif /* PLASMA_PROTOCOL */
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
new file mode 100644
index 0000000000000..5151a44d7217d
--- /dev/null
+++ b/cpp/src/plasma/store.cc
@@ -0,0 +1,681 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// PLASMA STORE: This is a simple object store server process
+//
+// It accepts incoming client connections on a unix domain socket
+// (name passed in via the -s option of the executable) and uses a
+// single thread to serve the clients. Each client establishes a
+// connection and can create objects, wait for objects and seal
+// objects through that connection.
+//
+// It keeps a hash table that maps object_ids (which are 20 byte long,
+// just enough to store and SHA1 hash) to memory mapped files.
+
+#include "plasma/store.h"
+
+#include <assert.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <limits.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/statvfs.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <deque>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "format/common_generated.h"
+#include "plasma/common.h"
+#include "plasma/fling.h"
+#include "plasma/io.h"
+#include "plasma/malloc.h"
+
+extern "C" {
+void* dlmalloc(size_t bytes);
+void* dlmemalign(size_t alignment, size_t bytes);
+void dlfree(void* mem);
+size_t dlmalloc_set_footprint_limit(size_t bytes);
+}
+
+struct GetRequest {
+  GetRequest(Client* client, const std::vector<ObjectID>& object_ids);
+  /// The client that called get.
+  Client* client;
+  /// The ID of the timer that will time out and cause this wait to return to
+  ///  the client if it hasn't already returned.
+  int64_t timer;
+  /// The object IDs involved in this request. This is used in the reply.
+  std::vector<ObjectID> object_ids;
+  /// The object information for the objects in this request. This is used in
+  /// the reply.
+  std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> objects;
+  /// The minimum number of objects to wait for in this request.
+  int64_t num_objects_to_wait_for;
+  /// The number of object requests in this wait request that are already
+  /// satisfied.
+  int64_t num_satisfied;
+};
+
+GetRequest::GetRequest(Client* client, const std::vector<ObjectID>& object_ids)
+    : client(client),
+      timer(-1),
+      object_ids(object_ids.begin(), object_ids.end()),
+      objects(object_ids.size()),
+      num_satisfied(0) {
+  std::unordered_set<ObjectID, UniqueIDHasher> unique_ids(
+      object_ids.begin(), object_ids.end());
+  num_objects_to_wait_for = unique_ids.size();
+}
+
+Client::Client(int fd) : fd(fd) {}
+
+PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory)
+    : loop_(loop), eviction_policy_(&store_info_) {
+  store_info_.memory_capacity = system_memory;
+}
+
+// TODO(pcm): Get rid of this destructor by using RAII to clean up data.
+PlasmaStore::~PlasmaStore() {
+  for (const auto& element : pending_notifications_) {
+    auto object_notifications = element.second.object_notifications;
+    for (size_t i = 0; i < object_notifications.size(); ++i) {
+      uint8_t* notification = reinterpret_cast<uint8_t*>(object_notifications.at(i));
+      uint8_t* data = notification;
+      // TODO(pcm): Get rid of this delete.
+      delete[] data;
+    }
+  }
+}
+
+// If this client is not already using the object, add the client to the
+// object's list of clients, otherwise do nothing.
+void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client* client) {
+  // Check if this client is already using the object.
+  if (entry->clients.find(client) != entry->clients.end()) { return; }
+  // If there are no other clients using this object, notify the eviction policy
+  // that the object is being used.
+  if (entry->clients.size() == 0) {
+    // Tell the eviction policy that this object is being used.
+    std::vector<ObjectID> objects_to_evict;
+    eviction_policy_.begin_object_access(entry->object_id, &objects_to_evict);
+    delete_objects(objects_to_evict);
+  }
+  // Add the client pointer to the list of clients using this object.
+  entry->clients.insert(client);
+}
+
+// Create a new object buffer in the hash table.
+int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
+    int64_t metadata_size, Client* client, PlasmaObject* result) {
+  ARROW_LOG(DEBUG) << "creating object " << object_id.hex();
+  if (store_info_.objects.count(object_id) != 0) {
+    // There is already an object with the same ID in the Plasma Store, so
+    // ignore this requst.
+    return PlasmaError_ObjectExists;
+  }
+  // Try to evict objects until there is enough space.
+  uint8_t* pointer;
+  do {
+    // Allocate space for the new object. We use dlmemalign instead of dlmalloc
+    // in order to align the allocated region to a 64-byte boundary. This is not
+    // strictly necessary, but it is an optimization that could speed up the
+    // computation of a hash of the data (see compute_object_hash_parallel in
+    // plasma_client.cc). Note that even though this pointer is 64-byte aligned,
+    // it is not guaranteed that the corresponding pointer in the client will be
+    // 64-byte aligned, but in practice it often will be.
+    pointer =
+        reinterpret_cast<uint8_t*>(dlmemalign(BLOCK_SIZE, data_size + metadata_size));
+    if (pointer == NULL) {
+      // Tell the eviction policy how much space we need to create this object.
+      std::vector<ObjectID> objects_to_evict;
+      bool success =
+          eviction_policy_.require_space(data_size + metadata_size, &objects_to_evict);
+      delete_objects(objects_to_evict);
+      // Return an error to the client if not enough space could be freed to
+      // create the object.
+      if (!success) { return PlasmaError_OutOfMemory; }
+    }
+  } while (pointer == NULL);
+  int fd;
+  int64_t map_size;
+  ptrdiff_t offset;
+  get_malloc_mapinfo(pointer, &fd, &map_size, &offset);
+  assert(fd != -1);
+
+  auto entry = std::unique_ptr<ObjectTableEntry>(new ObjectTableEntry());
+  entry->object_id = object_id;
+  entry->info.object_id = object_id.binary();
+  entry->info.data_size = data_size;
+  entry->info.metadata_size = metadata_size;
+  entry->pointer = pointer;
+  // TODO(pcm): Set the other fields.
+  entry->fd = fd;
+  entry->map_size = map_size;
+  entry->offset = offset;
+  entry->state = PLASMA_CREATED;
+
+  store_info_.objects[object_id] = std::move(entry);
+  result->handle.store_fd = fd;
+  result->handle.mmap_size = map_size;
+  result->data_offset = offset;
+  result->metadata_offset = offset + data_size;
+  result->data_size = data_size;
+  result->metadata_size = metadata_size;
+  // Notify the eviction policy that this object was created. This must be done
+  // immediately before the call to add_client_to_object_clients so that the
+  // eviction policy does not have an opportunity to evict the object.
+  eviction_policy_.object_created(object_id);
+  // Record that this client is using this object.
+  add_client_to_object_clients(store_info_.objects[object_id].get(), client);
+  return PlasmaError_OK;
+}
+
+void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
+  DCHECK(object != NULL);
+  DCHECK(entry != NULL);
+  DCHECK(entry->state == PLASMA_SEALED);
+  object->handle.store_fd = entry->fd;
+  object->handle.mmap_size = entry->map_size;
+  object->data_offset = entry->offset;
+  object->metadata_offset = entry->offset + entry->info.data_size;
+  object->data_size = entry->info.data_size;
+  object->metadata_size = entry->info.metadata_size;
+}
+
+void PlasmaStore::return_from_get(GetRequest* get_req) {
+  // Send the get reply to the client.
+  Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], get_req->objects,
+      get_req->object_ids.size());
+  warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd);
+  // If we successfully sent the get reply message to the client, then also send
+  // the file descriptors.
+  if (s.ok()) {
+    // Send all of the file descriptors for the present objects.
+    for (const auto& object_id : get_req->object_ids) {
+      PlasmaObject& object = get_req->objects[object_id];
+      // We use the data size to indicate whether the object is present or not.
+      if (object.data_size != -1) {
+        int error_code = send_fd(get_req->client->fd, object.handle.store_fd);
+        // If we failed to send the file descriptor, loop until we have sent it
+        // successfully. TODO(rkn): This is problematic for two reasons. First
+        // of all, sending the file descriptor should just succeed without any
+        // errors, but sometimes I see a "Message too long" error number.
+        // Second, looping like this allows a client to potentially block the
+        // plasma store event loop which should never happen.
+        while (error_code < 0) {
+          if (errno == EMSGSIZE) {
+            ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying.";
+            error_code = send_fd(get_req->client->fd, object.handle.store_fd);
+            continue;
+          }
+          warn_if_sigpipe(error_code, get_req->client->fd);
+          break;
+        }
+      }
+    }
+  }
+
+  // Remove the get request from each of the relevant object_get_requests hash
+  // tables if it is present there. It should only be present there if the get
+  // request timed out.
+  for (ObjectID& object_id : get_req->object_ids) {
+    auto& get_requests = object_get_requests_[object_id];
+    // Erase get_req from the vector.
+    auto it = std::find(get_requests.begin(), get_requests.end(), get_req);
+    if (it != get_requests.end()) { get_requests.erase(it); }
+  }
+  // Remove the get request.
+  if (get_req->timer != -1) { ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK); }
+  delete get_req;
+}
+
+void PlasmaStore::update_object_get_requests(const ObjectID& object_id) {
+  std::vector<GetRequest*>& get_requests = object_get_requests_[object_id];
+  size_t index = 0;
+  size_t num_requests = get_requests.size();
+  for (size_t i = 0; i < num_requests; ++i) {
+    GetRequest* get_req = get_requests[index];
+    auto entry = get_object_table_entry(&store_info_, object_id);
+    ARROW_CHECK(entry != NULL);
+
+    PlasmaObject_init(&get_req->objects[object_id], entry);
+    get_req->num_satisfied += 1;
+    // Record the fact that this client will be using this object and will
+    // be responsible for releasing this object.
+    add_client_to_object_clients(entry, get_req->client);
+
+    // If this get request is done, reply to the client.
+    if (get_req->num_satisfied == get_req->num_objects_to_wait_for) {
+      return_from_get(get_req);
+    } else {
+      // The call to return_from_get will remove the current element in the
+      // array, so we only increment the counter in the else branch.
+      index += 1;
+    }
+  }
+
+  DCHECK(index == get_requests.size());
+  // Remove the array of get requests for this object, since no one should be
+  // waiting for this object anymore.
+  object_get_requests_.erase(object_id);
+}
+
+void PlasmaStore::process_get_request(
+    Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms) {
+  // Create a get request for this object.
+  GetRequest* get_req = new GetRequest(client, object_ids);
+
+  for (auto object_id : object_ids) {
+    // Check if this object is already present locally. If so, record that the
+    // object is being used and mark it as accounted for.
+    auto entry = get_object_table_entry(&store_info_, object_id);
+    if (entry && entry->state == PLASMA_SEALED) {
+      // Update the get request to take into account the present object.
+      PlasmaObject_init(&get_req->objects[object_id], entry);
+      get_req->num_satisfied += 1;
+      // If necessary, record that this client is using this object. In the case
+      // where entry == NULL, this will be called from seal_object.
+      add_client_to_object_clients(entry, client);
+    } else {
+      // Add a placeholder plasma object to the get request to indicate that the
+      // object is not present. This will be parsed by the client. We set the
+      // data size to -1 to indicate that the object is not present.
+      get_req->objects[object_id].data_size = -1;
+      // Add the get request to the relevant data structures.
+      object_get_requests_[object_id].push_back(get_req);
+    }
+  }
+
+  // If all of the objects are present already or if the timeout is 0, return to
+  // the client.
+  if (get_req->num_satisfied == get_req->num_objects_to_wait_for || timeout_ms == 0) {
+    return_from_get(get_req);
+  } else if (timeout_ms != -1) {
+    // Set a timer that will cause the get request to return to the client. Note
+    // that a timeout of -1 is used to indicate that no timer should be set.
+    get_req->timer = loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) {
+      return_from_get(get_req);
+      return kEventLoopTimerDone;
+    });
+  }
+}
+
+int PlasmaStore::remove_client_from_object_clients(
+    ObjectTableEntry* entry, Client* client) {
+  auto it = entry->clients.find(client);
+  if (it != entry->clients.end()) {
+    entry->clients.erase(it);
+    // If no more clients are using this object, notify the eviction policy
+    // that the object is no longer being used.
+    if (entry->clients.size() == 0) {
+      // Tell the eviction policy that this object is no longer being used.
+      std::vector<ObjectID> objects_to_evict;
+      eviction_policy_.end_object_access(entry->object_id, &objects_to_evict);
+      delete_objects(objects_to_evict);
+    }
+    // Return 1 to indicate that the client was removed.
+    return 1;
+  } else {
+    // Return 0 to indicate that the client was not removed.
+    return 0;
+  }
+}
+
+void PlasmaStore::release_object(const ObjectID& object_id, Client* client) {
+  auto entry = get_object_table_entry(&store_info_, object_id);
+  ARROW_CHECK(entry != NULL);
+  // Remove the client from the object's array of clients.
+  ARROW_CHECK(remove_client_from_object_clients(entry, client) == 1);
+}
+
+// Check if an object is present.
+int PlasmaStore::contains_object(const ObjectID& object_id) {
+  auto entry = get_object_table_entry(&store_info_, object_id);
+  return entry && (entry->state == PLASMA_SEALED) ? OBJECT_FOUND : OBJECT_NOT_FOUND;
+}
+
+// Seal an object that has been created in the hash table.
+void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[]) {
+  ARROW_LOG(DEBUG) << "sealing object " << object_id.hex();
+  auto entry = get_object_table_entry(&store_info_, object_id);
+  ARROW_CHECK(entry != NULL);
+  ARROW_CHECK(entry->state == PLASMA_CREATED);
+  // Set the state of object to SEALED.
+  entry->state = PLASMA_SEALED;
+  // Set the object digest.
+  entry->info.digest = std::string(reinterpret_cast<char*>(&digest[0]), kDigestSize);
+  // Inform all subscribers that a new object has been sealed.
+  push_notification(&entry->info);
+
+  // Update all get requests that involve this object.
+  update_object_get_requests(object_id);
+}
+
+void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
+  for (const auto& object_id : object_ids) {
+    ARROW_LOG(DEBUG) << "deleting object " << object_id.hex();
+    auto entry = get_object_table_entry(&store_info_, object_id);
+    // TODO(rkn): This should probably not fail, but should instead throw an
+    // error. Maybe we should also support deleting objects that have been
+    // created but not sealed.
+    ARROW_CHECK(entry != NULL) << "To delete an object it must be in the object table.";
+    ARROW_CHECK(entry->state == PLASMA_SEALED)
+        << "To delete an object it must have been sealed.";
+    ARROW_CHECK(entry->clients.size() == 0)
+        << "To delete an object, there must be no clients currently using it.";
+    dlfree(entry->pointer);
+    store_info_.objects.erase(object_id);
+    // Inform all subscribers that the object has been deleted.
+    ObjectInfoT notification;
+    notification.object_id = object_id.binary();
+    notification.is_deletion = true;
+    push_notification(&notification);
+  }
+}
+
+void PlasmaStore::connect_client(int listener_sock) {
+  int client_fd = AcceptClient(listener_sock);
+  // This is freed in disconnect_client.
+  Client* client = new Client(client_fd);
+  // Add a callback to handle events on this socket.
+  // TODO(pcm): Check return value.
+  loop_->add_file_event(
+      client_fd, kEventLoopRead, [this, client](int events) { process_message(client); });
+  ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
+}
+
+void PlasmaStore::disconnect_client(Client* client) {
+  ARROW_CHECK(client != NULL);
+  ARROW_CHECK(client->fd > 0);
+  loop_->remove_file_event(client->fd);
+  // Close the socket.
+  close(client->fd);
+  ARROW_LOG(INFO) << "Disconnecting client on fd " << client->fd;
+  // If this client was using any objects, remove it from the appropriate
+  // lists.
+  for (const auto& entry : store_info_.objects) {
+    remove_client_from_object_clients(entry.second.get(), client);
+  }
+  // Note, the store may still attempt to send a message to the disconnected
+  // client (for example, when an object ID that the client was waiting for
+  // is ready). In these cases, the attempt to send the message will fail, but
+  // the store should just ignore the failure.
+  delete client;
+}
+
+/// Send notifications about sealed objects to the subscribers. This is called
+/// in seal_object. If the socket's send buffer is full, the notification will
+/// be
+/// buffered, and this will be called again when the send buffer has room.
+///
+/// @param client The client to send the notification to.
+/// @return Void.
+void PlasmaStore::send_notifications(int client_fd) {
+  auto it = pending_notifications_.find(client_fd);
+
+  int num_processed = 0;
+  bool closed = false;
+  // Loop over the array of pending notifications and send as many of them as
+  // possible.
+  for (size_t i = 0; i < it->second.object_notifications.size(); ++i) {
+    uint8_t* notification =
+        reinterpret_cast<uint8_t*>(it->second.object_notifications.at(i));
+    // Decode the length, which is the first bytes of the message.
+    int64_t size = *(reinterpret_cast<int64_t*>(notification));
+
+    // Attempt to send a notification about this object ID.
+    ssize_t nbytes = send(client_fd, notification, sizeof(int64_t) + size, 0);
+    if (nbytes >= 0) {
+      ARROW_CHECK(nbytes == static_cast<ssize_t>(sizeof(int64_t)) + size);
+    } else if (nbytes == -1 &&
+               (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
+      ARROW_LOG(DEBUG) << "The socket's send buffer is full, so we are caching this "
+                          "notification and will send it later.";
+      // Add a callback to the event loop to send queued notifications whenever
+      // there is room in the socket's send buffer. Callbacks can be added
+      // more than once here and will be overwritten. The callback is removed
+      // at the end of the method.
+      // TODO(pcm): Introduce status codes and check in case the file descriptor
+      // is added twice.
+      loop_->add_file_event(client_fd, kEventLoopWrite,
+          [this, client_fd](int events) { send_notifications(client_fd); });
+      break;
+    } else {
+      ARROW_LOG(WARNING) << "Failed to send notification to client on fd " << client_fd;
+      if (errno == EPIPE) {
+        closed = true;
+        break;
+      }
+    }
+    num_processed += 1;
+    // The corresponding malloc happened in create_object_info_buffer
+    // within push_notification.
+    delete[] notification;
+  }
+  // Remove the sent notifications from the array.
+  it->second.object_notifications.erase(it->second.object_notifications.begin(),
+      it->second.object_notifications.begin() + num_processed);
+
+  // Stop sending notifications if the pipe was broken.
+  if (closed) {
+    close(client_fd);
+    pending_notifications_.erase(client_fd);
+  }
+
+  // If we have sent all notifications, remove the fd from the event loop.
+  if (it->second.object_notifications.empty()) { loop_->remove_file_event(client_fd); }
+}
+
+void PlasmaStore::push_notification(ObjectInfoT* object_info) {
+  for (auto& element : pending_notifications_) {
+    uint8_t* notification = create_object_info_buffer(object_info);
+    element.second.object_notifications.push_back(notification);
+    send_notifications(element.first);
+    // The notification gets freed in send_notifications when the notification
+    // is sent over the socket.
+  }
+}
+
+// Subscribe to notifications about sealed objects.
+void PlasmaStore::subscribe_to_updates(Client* client) {
+  ARROW_LOG(DEBUG) << "subscribing to updates on fd " << client->fd;
+  // TODO(rkn): The store could block here if the client doesn't send a file
+  // descriptor.
+  int fd = recv_fd(client->fd);
+  if (fd < 0) {
+    // This may mean that the client died before sending the file descriptor.
+    ARROW_LOG(WARNING) << "Failed to receive file descriptor from client on fd "
+                       << client->fd << ".";
+    return;
+  }
+
+  // Create a new array to buffer notifications that can't be sent to the
+  // subscriber yet because the socket send buffer is full. TODO(rkn): the queue
+  // never gets freed.
+  // TODO(pcm): Is the following neccessary?
+  pending_notifications_[fd];
+
+  // Push notifications to the new subscriber about existing objects.
+  for (const auto& entry : store_info_.objects) {
+    push_notification(&entry.second->info);
+  }
+  send_notifications(fd);
+}
+
+Status PlasmaStore::process_message(Client* client) {
+  int64_t type;
+  Status s = ReadMessage(client->fd, &type, &input_buffer_);
+  ARROW_CHECK(s.ok() || s.IsIOError());
+
+  uint8_t* input = input_buffer_.data();
+  ObjectID object_id;
+  PlasmaObject object;
+  // TODO(pcm): Get rid of the following.
+  memset(&object, 0, sizeof(object));
+
+  // Process the different types of requests.
+  switch (type) {
+    case MessageType_PlasmaCreateRequest: {
+      int64_t data_size;
+      int64_t metadata_size;
+      RETURN_NOT_OK(ReadCreateRequest(input, &object_id, &data_size, &metadata_size));
+      int error_code =
+          create_object(object_id, data_size, metadata_size, client, &object);
+      HANDLE_SIGPIPE(
+          SendCreateReply(client->fd, object_id, &object, error_code), client->fd);
+      if (error_code == PlasmaError_OK) {
+        warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
+      }
+    } break;
+    case MessageType_PlasmaGetRequest: {
+      std::vector<ObjectID> object_ids_to_get;
+      int64_t timeout_ms;
+      RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms));
+      process_get_request(client, object_ids_to_get, timeout_ms);
+    } break;
+    case MessageType_PlasmaReleaseRequest:
+      RETURN_NOT_OK(ReadReleaseRequest(input, &object_id));
+      release_object(object_id, client);
+      break;
+    case MessageType_PlasmaContainsRequest:
+      RETURN_NOT_OK(ReadContainsRequest(input, &object_id));
+      if (contains_object(object_id) == OBJECT_FOUND) {
+        HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd);
+      } else {
+        HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd);
+      }
+      break;
+    case MessageType_PlasmaSealRequest: {
+      unsigned char digest[kDigestSize];
+      RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0]));
+      seal_object(object_id, &digest[0]);
+    } break;
+    case MessageType_PlasmaEvictRequest: {
+      // This code path should only be used for testing.
+      int64_t num_bytes;
+      RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes));
+      std::vector<ObjectID> objects_to_evict;
+      int64_t num_bytes_evicted =
+          eviction_policy_.choose_objects_to_evict(num_bytes, &objects_to_evict);
+      delete_objects(objects_to_evict);
+      HANDLE_SIGPIPE(SendEvictReply(client->fd, num_bytes_evicted), client->fd);
+    } break;
+    case MessageType_PlasmaSubscribeRequest:
+      subscribe_to_updates(client);
+      break;
+    case MessageType_PlasmaConnectRequest: {
+      HANDLE_SIGPIPE(
+          SendConnectReply(client->fd, store_info_.memory_capacity), client->fd);
+    } break;
+    case DISCONNECT_CLIENT:
+      ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd;
+      disconnect_client(client);
+      break;
+    default:
+      // This code should be unreachable.
+      ARROW_CHECK(0);
+  }
+  return Status::OK();
+}
+
+// Report "success" to valgrind.
+void signal_handler(int signal) {
+  if (signal == SIGTERM) { exit(0); }
+}
+
+void start_server(char* socket_name, int64_t system_memory) {
+  // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
+  // to a client that has already died, the store could die.
+  signal(SIGPIPE, SIG_IGN);
+  // Create the event loop.
+  EventLoop loop;
+  PlasmaStore store(&loop, system_memory);
+  int socket = bind_ipc_sock(socket_name, true);
+  ARROW_CHECK(socket >= 0);
+  // TODO(pcm): Check return value.
+  loop.add_file_event(socket, kEventLoopRead,
+      [&store, socket](int events) { store.connect_client(socket); });
+  loop.run();
+}
+
+int main(int argc, char* argv[]) {
+  signal(SIGTERM, signal_handler);
+  char* socket_name = NULL;
+  int64_t system_memory = -1;
+  int c;
+  while ((c = getopt(argc, argv, "s:m:")) != -1) {
+    switch (c) {
+      case 's':
+        socket_name = optarg;
+        break;
+      case 'm': {
+        char extra;
+        int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra);
+        ARROW_CHECK(scanned == 1);
+        ARROW_LOG(INFO) << "Allowing the Plasma store to use up to "
+                        << static_cast<double>(system_memory) / 1000000000
+                        << "GB of memory.";
+        break;
+      }
+      default:
+        exit(-1);
+    }
+  }
+  if (!socket_name) {
+    ARROW_LOG(FATAL) << "please specify socket for incoming connections with -s switch";
+  }
+  if (system_memory == -1) {
+    ARROW_LOG(FATAL) << "please specify the amount of system memory with -m switch";
+  }
+#ifdef __linux__
+  // On Linux, check that the amount of memory available in /dev/shm is large
+  // enough to accommodate the request. If it isn't, then fail.
+  int shm_fd = open("/dev/shm", O_RDONLY);
+  struct statvfs shm_vfs_stats;
+  fstatvfs(shm_fd, &shm_vfs_stats);
+  // The value shm_vfs_stats.f_bsize is the block size, and the value
+  // shm_vfs_stats.f_bavail is the number of available blocks.
+  int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail;
+  close(shm_fd);
+  if (system_memory > shm_mem_avail) {
+    ARROW_LOG(FATAL) << "System memory request exceeds memory available in /dev/shm. The "
+                        "request is for "
+                     << system_memory << " bytes, and the amount available is "
+                     << shm_mem_avail
+                     << " bytes. You may be able to free up space by deleting files in "
+                        "/dev/shm. If you are inside a Docker container, you may need to "
+                        "pass "
+                        "an argument with the flag '--shm-size' to 'docker run'.";
+  }
+#endif
+  // Make it so dlmalloc fails if we try to request more memory than is
+  // available.
+  dlmalloc_set_footprint_limit((size_t)system_memory);
+  ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
+  start_server(socket_name, system_memory);
+}
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
new file mode 100644
index 0000000000000..8bd94265410f6
--- /dev/null
+++ b/cpp/src/plasma/store.h
@@ -0,0 +1,169 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_STORE_H
+#define PLASMA_STORE_H
+
+#include <deque>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/events.h"
+#include "plasma/eviction_policy.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+struct GetRequest;
+
+struct NotificationQueue {
+  /// The object notifications for clients. We notify the client about the
+  /// objects in the order that the objects were sealed or deleted.
+  std::deque<uint8_t*> object_notifications;
+};
+
+/// Contains all information that is associated with a Plasma store client.
+struct Client {
+  explicit Client(int fd);
+
+  /// The file descriptor used to communicate with the client.
+  int fd;
+};
+
+class PlasmaStore {
+ public:
+  PlasmaStore(EventLoop* loop, int64_t system_memory);
+
+  ~PlasmaStore();
+
+  /// Create a new object. The client must do a call to release_object to tell
+  /// the store when it is done with the object.
+  ///
+  /// @param object_id Object ID of the object to be created.
+  /// @param data_size Size in bytes of the object to be created.
+  /// @param metadata_size Size in bytes of the object metadata.
+  /// @return One of the following error codes:
+  ///  - PlasmaError_OK, if the object was created successfully.
+  ///  - PlasmaError_ObjectExists, if an object with this ID is already
+  ///    present in the store. In this case, the client should not call
+  ///    plasma_release.
+  ///  - PlasmaError_OutOfMemory, if the store is out of memory and
+  ///    cannot create the object. In this case, the client should not call
+  ///    plasma_release.
+  int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
+      Client* client, PlasmaObject* result);
+
+  /// Delete objects that have been created in the hash table. This should only
+  /// be called on objects that are returned by the eviction policy to evict.
+  ///
+  /// @param object_ids Object IDs of the objects to be deleted.
+  /// @return Void.
+  void delete_objects(const std::vector<ObjectID>& object_ids);
+
+  /// Process a get request from a client. This method assumes that we will
+  /// eventually have these objects sealed. If one of the objects has not yet
+  /// been sealed, the client that requested the object will be notified when it
+  /// is sealed.
+  ///
+  /// For each object, the client must do a call to release_object to tell the
+  /// store when it is done with the object.
+  ///
+  /// @param client The client making this request.
+  /// @param object_ids Object IDs of the objects to be gotten.
+  /// @param timeout_ms The timeout for the get request in milliseconds.
+  /// @return Void.
+  void process_get_request(
+      Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms);
+
+  /// Seal an object. The object is now immutable and can be accessed with get.
+  ///
+  /// @param object_id Object ID of the object to be sealed.
+  /// @param digest The digest of the object. This is used to tell if two
+  /// objects
+  ///        with the same object ID are the same.
+  /// @return Void.
+  void seal_object(const ObjectID& object_id, unsigned char digest[]);
+
+  /// Check if the plasma store contains an object:
+  ///
+  /// @param object_id Object ID that will be checked.
+  /// @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if
+  /// not
+  int contains_object(const ObjectID& object_id);
+
+  /// Record the fact that a particular client is no longer using an object.
+  ///
+  /// @param object_id The object ID of the object that is being released.
+  /// @param client The client making this request.
+  /// @param Void.
+  void release_object(const ObjectID& object_id, Client* client);
+
+  /// Subscribe a file descriptor to updates about new sealed objects.
+  ///
+  /// @param client The client making this request.
+  /// @return Void.
+  void subscribe_to_updates(Client* client);
+
+  /// Connect a new client to the PlasmaStore.
+  ///
+  /// @param listener_sock The socket that is listening to incoming connections.
+  /// @return Void.
+  void connect_client(int listener_sock);
+
+  /// Disconnect a client from the PlasmaStore.
+  ///
+  /// @param client The client that is disconnected.
+  /// @return Void.
+  void disconnect_client(Client* client);
+
+  void send_notifications(int client_fd);
+
+  Status process_message(Client* client);
+
+ private:
+  void push_notification(ObjectInfoT* object_notification);
+
+  void add_client_to_object_clients(ObjectTableEntry* entry, Client* client);
+
+  void return_from_get(GetRequest* get_req);
+
+  void update_object_get_requests(const ObjectID& object_id);
+
+  int remove_client_from_object_clients(ObjectTableEntry* entry, Client* client);
+
+  /// Event loop of the plasma store.
+  EventLoop* loop_;
+  /// The plasma store information, including the object tables, that is exposed
+  /// to the eviction policy.
+  PlasmaStoreInfo store_info_;
+  /// The state that is managed by the eviction policy.
+  EvictionPolicy eviction_policy_;
+  /// Input buffer. This is allocated only once to avoid mallocs for every
+  /// call to process_message.
+  std::vector<uint8_t> input_buffer_;
+  /// A hash table mapping object IDs to a vector of the get requests that are
+  /// waiting for the object to arrive.
+  std::unordered_map<ObjectID, std::vector<GetRequest*>, UniqueIDHasher>
+      object_get_requests_;
+  /// The pending notifications that have not been sent to subscribers because
+  /// the socket send buffers were full. This is a hash table from client file
+  /// descriptor to an array of object_ids to send to that client.
+  /// TODO(pcm): Consider putting this into the Client data structure and
+  /// reorganize the code slightly.
+  std::unordered_map<int, NotificationQueue> pending_notifications_;
+};
+
+#endif  // PLASMA_STORE_H
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
new file mode 100644
index 0000000000000..dc457731b696c
--- /dev/null
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include <assert.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+std::string g_test_executable;
+
+class TestPlasmaStore : public ::testing::Test {
+ public:
+  // TODO(pcm): At the moment, stdout of the test gets mixed up with
+  // stdout of the object store. Consider changing that.
+  void SetUp() {
+    std::string plasma_directory =
+        g_test_executable.substr(0, g_test_executable.find_last_of("/"));
+    std::string plasma_command =
+        plasma_directory +
+        "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &";
+    system(plasma_command.c_str());
+    ARROW_CHECK_OK(client_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
+  }
+  virtual void Finish() {
+    ARROW_CHECK_OK(client_.Disconnect());
+    system("killall plasma_store &");
+  }
+
+ protected:
+  PlasmaClient client_;
+};
+
+TEST_F(TestPlasmaStore, ContainsTest) {
+  ObjectID object_id = ObjectID::from_random();
+
+  // Test for object non-existence.
+  bool has_object;
+  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, false);
+
+  // Test for the object being in local Plasma store.
+  // First create object.
+  int64_t data_size = 100;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  uint8_t* data;
+  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  ARROW_CHECK_OK(client_.Seal(object_id));
+  // Avoid race condition of Plasma Manager waiting for notification.
+  ObjectBuffer object_buffer;
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, true);
+}
+
+TEST_F(TestPlasmaStore, GetTest) {
+  ObjectID object_id = ObjectID::from_random();
+  ObjectBuffer object_buffer;
+
+  // Test for object non-existence.
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer));
+  ASSERT_EQ(object_buffer.data_size, -1);
+
+  // Test for the object being in local Plasma store.
+  // First create object.
+  int64_t data_size = 4;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  uint8_t* data;
+  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  for (int64_t i = 0; i < data_size; i++) {
+    data[i] = static_cast<uint8_t>(i % 4);
+  }
+  ARROW_CHECK_OK(client_.Seal(object_id));
+
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  for (int64_t i = 0; i < data_size; i++) {
+    ASSERT_EQ(data[i], object_buffer.data[i]);
+  }
+}
+
+TEST_F(TestPlasmaStore, MultipleGetTest) {
+  ObjectID object_id1 = ObjectID::from_random();
+  ObjectID object_id2 = ObjectID::from_random();
+  ObjectID object_ids[2] = {object_id1, object_id2};
+  ObjectBuffer object_buffer[2];
+
+  int64_t data_size = 4;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  uint8_t* data;
+  ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data));
+  data[0] = 1;
+  ARROW_CHECK_OK(client_.Seal(object_id1));
+
+  ARROW_CHECK_OK(client_.Create(object_id2, data_size, metadata, metadata_size, &data));
+  data[0] = 2;
+  ARROW_CHECK_OK(client_.Seal(object_id2));
+
+  ARROW_CHECK_OK(client_.Get(object_ids, 2, -1, object_buffer));
+  ASSERT_EQ(object_buffer[0].data[0], 1);
+  ASSERT_EQ(object_buffer[1].data[0], 2);
+}
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  g_test_executable = std::string(argv[0]);
+  return RUN_ALL_TESTS();
+}
diff --git a/cpp/src/plasma/test/run_tests.sh b/cpp/src/plasma/test/run_tests.sh
new file mode 100644
index 0000000000000..958bd08398e23
--- /dev/null
+++ b/cpp/src/plasma/test/run_tests.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cause the script to exit if a single command fails.
+set -e
+
+./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
+sleep 1
+./src/plasma/manager_tests
+killall plasma_store
+./src/plasma/serialization_tests
+
+# Start the Redis shards.
+./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
+redis_pid1=$!
+./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
+redis_pid2=$!
+sleep 1
+
+# Flush the redis server
+./src/common/thirdparty/redis/src/redis-cli flushall
+# Register the shard location with the primary shard.
+./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
+./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
+sleep 1
+./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
+plasma1_pid=$!
+./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
+plasma2_pid=$!
+./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 &
+plasma3_pid=$!
+./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
+plasma4_pid=$!
+sleep 1
+
+./src/plasma/client_tests
+
+kill $plasma4_pid
+kill $plasma3_pid
+kill $plasma2_pid
+kill $plasma1_pid
+kill $redis_pid1
+wait $redis_pid1
+kill $redis_pid2
+wait $redis_pid2
diff --git a/cpp/src/plasma/test/run_valgrind.sh b/cpp/src/plasma/test/run_valgrind.sh
new file mode 100644
index 0000000000000..0472194128679
--- /dev/null
+++ b/cpp/src/plasma/test/run_valgrind.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cause the script to exit if a single command fails.
+set -e
+
+./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
+sleep 1
+valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests
+killall plasma_store
+valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
new file mode 100644
index 0000000000000..325cead06e770
--- /dev/null
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -0,0 +1,388 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/common.h"
+#include "plasma/io.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+/**
+ * Create a temporary file. Needs to be closed by the caller.
+ *
+ * @return File descriptor of the file.
+ */
+int create_temp_file(void) {
+  static char temp[] = "/tmp/tempfileXXXXXX";
+  char file_name[32];
+  strncpy(file_name, temp, 32);
+  return mkstemp(file_name);
+}
+
+/**
+ * Seek to the beginning of a file and read a message from it.
+ *
+ * @param fd File descriptor of the file.
+ * @param message type Message type that we expect in the file.
+ *
+ * @return Pointer to the content of the message. Needs to be freed by the
+ * caller.
+ */
+std::vector<uint8_t> read_message_from_file(int fd, int message_type) {
+  /* Go to the beginning of the file. */
+  lseek(fd, 0, SEEK_SET);
+  int64_t type;
+  std::vector<uint8_t> data;
+  ARROW_CHECK_OK(ReadMessage(fd, &type, &data));
+  ARROW_CHECK(type == message_type);
+  return data;
+}
+
+PlasmaObject random_plasma_object(void) {
+  unsigned int seed = static_cast<unsigned int>(time(NULL));
+  int random = rand_r(&seed);
+  PlasmaObject object;
+  memset(&object, 0, sizeof(object));
+  object.handle.store_fd = random + 7;
+  object.handle.mmap_size = random + 42;
+  object.data_offset = random + 1;
+  object.metadata_offset = random + 2;
+  object.data_size = random + 3;
+  object.metadata_size = random + 4;
+  return object;
+}
+
+TEST(PlasmaSerialization, CreateRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  int64_t data_size1 = 42;
+  int64_t metadata_size1 = 11;
+  ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateRequest);
+  ObjectID object_id2;
+  int64_t data_size2;
+  int64_t metadata_size2;
+  ARROW_CHECK_OK(
+      ReadCreateRequest(data.data(), &object_id2, &data_size2, &metadata_size2));
+  ASSERT_EQ(data_size1, data_size2);
+  ASSERT_EQ(metadata_size1, metadata_size2);
+  ASSERT_EQ(object_id1, object_id2);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, CreateReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  PlasmaObject object1 = random_plasma_object();
+  ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateReply);
+  ObjectID object_id2;
+  PlasmaObject object2;
+  memset(&object2, 0, sizeof(object2));
+  ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2));
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(memcmp(&object1, &object2, sizeof(object1)), 0);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, SealRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  unsigned char digest1[kDigestSize];
+  memset(&digest1[0], 7, kDigestSize);
+  ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0]));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealRequest);
+  ObjectID object_id2;
+  unsigned char digest2[kDigestSize];
+  ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0]));
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(memcmp(&digest1[0], &digest2[0], kDigestSize), 0);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, SealReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealReply);
+  ObjectID object_id2;
+  Status s = ReadSealReply(data.data(), &object_id2);
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_TRUE(s.IsPlasmaObjectExists());
+  close(fd);
+}
+
+TEST(PlasmaSerialization, GetRequest) {
+  int fd = create_temp_file();
+  ObjectID object_ids[2];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  int64_t timeout_ms = 1234;
+  ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetRequest);
+  std::vector<ObjectID> object_ids_return;
+  int64_t timeout_ms_return;
+  ARROW_CHECK_OK(ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return));
+  ASSERT_EQ(object_ids[0], object_ids_return[0]);
+  ASSERT_EQ(object_ids[1], object_ids_return[1]);
+  ASSERT_EQ(timeout_ms, timeout_ms_return);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, GetReply) {
+  int fd = create_temp_file();
+  ObjectID object_ids[2];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> plasma_objects;
+  plasma_objects[object_ids[0]] = random_plasma_object();
+  plasma_objects[object_ids[1]] = random_plasma_object();
+  ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetReply);
+  ObjectID object_ids_return[2];
+  PlasmaObject plasma_objects_return[2];
+  memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
+  ARROW_CHECK_OK(
+      ReadGetReply(data.data(), object_ids_return, &plasma_objects_return[0], 2));
+  ASSERT_EQ(object_ids[0], object_ids_return[0]);
+  ASSERT_EQ(object_ids[1], object_ids_return[1]);
+  ASSERT_EQ(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
+                sizeof(PlasmaObject)),
+      0);
+  ASSERT_EQ(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
+                sizeof(PlasmaObject)),
+      0);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, ReleaseRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1));
+  std::vector<uint8_t> data =
+      read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
+  ObjectID object_id2;
+  ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2));
+  ASSERT_EQ(object_id1, object_id2);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, ReleaseReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaReleaseReply);
+  ObjectID object_id2;
+  Status s = ReadReleaseReply(data.data(), &object_id2);
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_TRUE(s.IsPlasmaObjectExists());
+  close(fd);
+}
+
+TEST(PlasmaSerialization, DeleteRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
+  ObjectID object_id2;
+  ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2));
+  ASSERT_EQ(object_id1, object_id2);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, DeleteReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  int error1 = PlasmaError_ObjectExists;
+  ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteReply);
+  ObjectID object_id2;
+  Status s = ReadDeleteReply(data.data(), &object_id2);
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_TRUE(s.IsPlasmaObjectExists());
+  close(fd);
+}
+
+TEST(PlasmaSerialization, StatusRequest) {
+  int fd = create_temp_file();
+  int64_t num_objects = 2;
+  ObjectID object_ids[num_objects];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusRequest);
+  ObjectID object_ids_read[num_objects];
+  ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects));
+  ASSERT_EQ(object_ids[0], object_ids_read[0]);
+  ASSERT_EQ(object_ids[1], object_ids_read[1]);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, StatusReply) {
+  int fd = create_temp_file();
+  ObjectID object_ids[2];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  int object_statuses[2] = {42, 43};
+  ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusReply);
+  int64_t num_objects = ReadStatusReply_num_objects(data.data());
+  ObjectID object_ids_read[num_objects];
+  int object_statuses_read[num_objects];
+  ARROW_CHECK_OK(
+      ReadStatusReply(data.data(), object_ids_read, object_statuses_read, num_objects));
+  ASSERT_EQ(object_ids[0], object_ids_read[0]);
+  ASSERT_EQ(object_ids[1], object_ids_read[1]);
+  ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
+  ASSERT_EQ(object_statuses[1], object_statuses_read[1]);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, EvictRequest) {
+  int fd = create_temp_file();
+  int64_t num_bytes = 111;
+  ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictRequest);
+  int64_t num_bytes_received;
+  ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received));
+  ASSERT_EQ(num_bytes, num_bytes_received);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, EvictReply) {
+  int fd = create_temp_file();
+  int64_t num_bytes = 111;
+  ARROW_CHECK_OK(SendEvictReply(fd, num_bytes));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictReply);
+  int64_t num_bytes_received;
+  ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received));
+  ASSERT_EQ(num_bytes, num_bytes_received);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, FetchRequest) {
+  int fd = create_temp_file();
+  ObjectID object_ids[2];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaFetchRequest);
+  std::vector<ObjectID> object_ids_read;
+  ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read));
+  ASSERT_EQ(object_ids[0], object_ids_read[0]);
+  ASSERT_EQ(object_ids[1], object_ids_read[1]);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, WaitRequest) {
+  int fd = create_temp_file();
+  const int num_objects_in = 2;
+  ObjectRequest object_requests_in[num_objects_in] = {
+      ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}),
+      ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})};
+  const int num_ready_objects_in = 1;
+  int64_t timeout_ms = 1000;
+
+  ARROW_CHECK_OK(SendWaitRequest(
+      fd, &object_requests_in[0], num_objects_in, num_ready_objects_in, timeout_ms));
+  /* Read message back. */
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitRequest);
+  int num_ready_objects_out;
+  int64_t timeout_ms_read;
+  ObjectRequestMap object_requests_out;
+  ARROW_CHECK_OK(ReadWaitRequest(
+      data.data(), object_requests_out, &timeout_ms_read, &num_ready_objects_out));
+  ASSERT_EQ(num_objects_in, object_requests_out.size());
+  ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
+  for (int i = 0; i < num_objects_in; i++) {
+    const ObjectID& object_id = object_requests_in[i].object_id;
+    ASSERT_EQ(1, object_requests_out.count(object_id));
+    const auto& entry = object_requests_out.find(object_id);
+    ASSERT_TRUE(entry != object_requests_out.end());
+    ASSERT_EQ(entry->second.object_id, object_requests_in[i].object_id);
+    ASSERT_EQ(entry->second.type, object_requests_in[i].type);
+  }
+  close(fd);
+}
+
+TEST(PlasmaSerialization, WaitReply) {
+  int fd = create_temp_file();
+  const int num_objects_in = 2;
+  /* Create a map with two ObjectRequests in it. */
+  ObjectRequestMap objects_in(num_objects_in);
+  ObjectID id1 = ObjectID::from_random();
+  objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local});
+  ObjectID id2 = ObjectID::from_random();
+  objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent});
+
+  ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in));
+  /* Read message back. */
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitReply);
+  ObjectRequest objects_out[2];
+  int num_objects_out;
+  ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out));
+  ASSERT_EQ(num_objects_in, num_objects_out);
+  for (int i = 0; i < num_objects_out; i++) {
+    /* Each object request must appear exactly once. */
+    ASSERT_EQ(objects_in.count(objects_out[i].object_id), 1);
+    const auto& entry = objects_in.find(objects_out[i].object_id);
+    ASSERT_TRUE(entry != objects_in.end());
+    ASSERT_EQ(entry->second.object_id, objects_out[i].object_id);
+    ASSERT_EQ(entry->second.status, objects_out[i].status);
+  }
+  close(fd);
+}
+
+TEST(PlasmaSerialization, DataRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  const char* address1 = "address1";
+  int port1 = 12345;
+  ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1));
+  /* Reading message back. */
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataRequest);
+  ObjectID object_id2;
+  char* address2;
+  int port2;
+  ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2));
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(strcmp(address1, address2), 0);
+  ASSERT_EQ(port1, port2);
+  free(address2);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, DataReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  int64_t object_size1 = 146;
+  int64_t metadata_size1 = 198;
+  ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1));
+  /* Reading message back. */
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataReply);
+  ObjectID object_id2;
+  int64_t object_size2;
+  int64_t metadata_size2;
+  ARROW_CHECK_OK(ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2));
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(object_size1, object_size2);
+  ASSERT_EQ(metadata_size1, metadata_size2);
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae.c b/cpp/src/plasma/thirdparty/ae/ae.c
new file mode 100644
index 0000000000000..e66808a81466d
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae.c
@@ -0,0 +1,465 @@
+/* A simple event-driven programming library. Originally I wrote this code
+ * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
+ * it in form of a library for easy reuse.
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <poll.h>
+#include <string.h>
+#include <time.h>
+#include <errno.h>
+
+#include "ae.h"
+#include "zmalloc.h"
+#include "config.h"
+
+/* Include the best multiplexing layer supported by this system.
+ * The following should be ordered by performances, descending. */
+#ifdef HAVE_EVPORT
+#include "ae_evport.c"
+#else
+    #ifdef HAVE_EPOLL
+    #include "ae_epoll.c"
+    #else
+        #ifdef HAVE_KQUEUE
+        #include "ae_kqueue.c"
+        #else
+        #include "ae_select.c"
+        #endif
+    #endif
+#endif
+
+aeEventLoop *aeCreateEventLoop(int setsize) {
+    aeEventLoop *eventLoop;
+    int i;
+
+    if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
+    eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
+    eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
+    if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
+    eventLoop->setsize = setsize;
+    eventLoop->lastTime = time(NULL);
+    eventLoop->timeEventHead = NULL;
+    eventLoop->timeEventNextId = 0;
+    eventLoop->stop = 0;
+    eventLoop->maxfd = -1;
+    eventLoop->beforesleep = NULL;
+    if (aeApiCreate(eventLoop) == -1) goto err;
+    /* Events with mask == AE_NONE are not set. So let's initialize the
+     * vector with it. */
+    for (i = 0; i < setsize; i++)
+        eventLoop->events[i].mask = AE_NONE;
+    return eventLoop;
+
+err:
+    if (eventLoop) {
+        zfree(eventLoop->events);
+        zfree(eventLoop->fired);
+        zfree(eventLoop);
+    }
+    return NULL;
+}
+
+/* Return the current set size. */
+int aeGetSetSize(aeEventLoop *eventLoop) {
+    return eventLoop->setsize;
+}
+
+/* Resize the maximum set size of the event loop.
+ * If the requested set size is smaller than the current set size, but
+ * there is already a file descriptor in use that is >= the requested
+ * set size minus one, AE_ERR is returned and the operation is not
+ * performed at all.
+ *
+ * Otherwise AE_OK is returned and the operation is successful. */
+int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
+    int i;
+
+    if (setsize == eventLoop->setsize) return AE_OK;
+    if (eventLoop->maxfd >= setsize) return AE_ERR;
+    if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
+
+    eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
+    eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
+    eventLoop->setsize = setsize;
+
+    /* Make sure that if we created new slots, they are initialized with
+     * an AE_NONE mask. */
+    for (i = eventLoop->maxfd+1; i < setsize; i++)
+        eventLoop->events[i].mask = AE_NONE;
+    return AE_OK;
+}
+
+void aeDeleteEventLoop(aeEventLoop *eventLoop) {
+    aeApiFree(eventLoop);
+    zfree(eventLoop->events);
+    zfree(eventLoop->fired);
+    zfree(eventLoop);
+}
+
+void aeStop(aeEventLoop *eventLoop) {
+    eventLoop->stop = 1;
+}
+
+int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
+        aeFileProc *proc, void *clientData)
+{
+    if (fd >= eventLoop->setsize) {
+        errno = ERANGE;
+        return AE_ERR;
+    }
+    aeFileEvent *fe = &eventLoop->events[fd];
+
+    if (aeApiAddEvent(eventLoop, fd, mask) == -1)
+        return AE_ERR;
+    fe->mask |= mask;
+    if (mask & AE_READABLE) fe->rfileProc = proc;
+    if (mask & AE_WRITABLE) fe->wfileProc = proc;
+    fe->clientData = clientData;
+    if (fd > eventLoop->maxfd)
+        eventLoop->maxfd = fd;
+    return AE_OK;
+}
+
+void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
+{
+    if (fd >= eventLoop->setsize) return;
+    aeFileEvent *fe = &eventLoop->events[fd];
+    if (fe->mask == AE_NONE) return;
+
+    aeApiDelEvent(eventLoop, fd, mask);
+    fe->mask = fe->mask & (~mask);
+    if (fd == eventLoop->maxfd && fe->mask == AE_NONE) {
+        /* Update the max fd */
+        int j;
+
+        for (j = eventLoop->maxfd-1; j >= 0; j--)
+            if (eventLoop->events[j].mask != AE_NONE) break;
+        eventLoop->maxfd = j;
+    }
+}
+
+int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
+    if (fd >= eventLoop->setsize) return 0;
+    aeFileEvent *fe = &eventLoop->events[fd];
+
+    return fe->mask;
+}
+
+static void aeGetTime(long *seconds, long *milliseconds)
+{
+    struct timeval tv;
+
+    gettimeofday(&tv, NULL);
+    *seconds = tv.tv_sec;
+    *milliseconds = tv.tv_usec/1000;
+}
+
+static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) {
+    long cur_sec, cur_ms, when_sec, when_ms;
+
+    aeGetTime(&cur_sec, &cur_ms);
+    when_sec = cur_sec + milliseconds/1000;
+    when_ms = cur_ms + milliseconds%1000;
+    if (when_ms >= 1000) {
+        when_sec ++;
+        when_ms -= 1000;
+    }
+    *sec = when_sec;
+    *ms = when_ms;
+}
+
+long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
+        aeTimeProc *proc, void *clientData,
+        aeEventFinalizerProc *finalizerProc)
+{
+    long long id = eventLoop->timeEventNextId++;
+    aeTimeEvent *te;
+
+    te = zmalloc(sizeof(*te));
+    if (te == NULL) return AE_ERR;
+    te->id = id;
+    aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms);
+    te->timeProc = proc;
+    te->finalizerProc = finalizerProc;
+    te->clientData = clientData;
+    te->next = eventLoop->timeEventHead;
+    eventLoop->timeEventHead = te;
+    return id;
+}
+
+int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
+{
+    aeTimeEvent *te = eventLoop->timeEventHead;
+    while(te) {
+        if (te->id == id) {
+            te->id = AE_DELETED_EVENT_ID;
+            return AE_OK;
+        }
+        te = te->next;
+    }
+    return AE_ERR; /* NO event with the specified ID found */
+}
+
+/* Search the first timer to fire.
+ * This operation is useful to know how many time the select can be
+ * put in sleep without to delay any event.
+ * If there are no timers NULL is returned.
+ *
+ * Note that's O(N) since time events are unsorted.
+ * Possible optimizations (not needed by Redis so far, but...):
+ * 1) Insert the event in order, so that the nearest is just the head.
+ *    Much better but still insertion or deletion of timers is O(N).
+ * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)).
+ */
+static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
+{
+    aeTimeEvent *te = eventLoop->timeEventHead;
+    aeTimeEvent *nearest = NULL;
+
+    while(te) {
+        if (!nearest || te->when_sec < nearest->when_sec ||
+                (te->when_sec == nearest->when_sec &&
+                 te->when_ms < nearest->when_ms))
+            nearest = te;
+        te = te->next;
+    }
+    return nearest;
+}
+
+/* Process time events */
+static int processTimeEvents(aeEventLoop *eventLoop) {
+    int processed = 0;
+    aeTimeEvent *te, *prev;
+    long long maxId;
+    time_t now = time(NULL);
+
+    /* If the system clock is moved to the future, and then set back to the
+     * right value, time events may be delayed in a random way. Often this
+     * means that scheduled operations will not be performed soon enough.
+     *
+     * Here we try to detect system clock skews, and force all the time
+     * events to be processed ASAP when this happens: the idea is that
+     * processing events earlier is less dangerous than delaying them
+     * indefinitely, and practice suggests it is. */
+    if (now < eventLoop->lastTime) {
+        te = eventLoop->timeEventHead;
+        while(te) {
+            te->when_sec = 0;
+            te = te->next;
+        }
+    }
+    eventLoop->lastTime = now;
+
+    prev = NULL;
+    te = eventLoop->timeEventHead;
+    maxId = eventLoop->timeEventNextId-1;
+    while(te) {
+        long now_sec, now_ms;
+        long long id;
+
+        /* Remove events scheduled for deletion. */
+        if (te->id == AE_DELETED_EVENT_ID) {
+            aeTimeEvent *next = te->next;
+            if (prev == NULL)
+                eventLoop->timeEventHead = te->next;
+            else
+                prev->next = te->next;
+            if (te->finalizerProc)
+                te->finalizerProc(eventLoop, te->clientData);
+            zfree(te);
+            te = next;
+            continue;
+        }
+
+        /* Make sure we don't process time events created by time events in
+         * this iteration. Note that this check is currently useless: we always
+         * add new timers on the head, however if we change the implementation
+         * detail, this check may be useful again: we keep it here for future
+         * defense. */
+        if (te->id > maxId) {
+            te = te->next;
+            continue;
+        }
+        aeGetTime(&now_sec, &now_ms);
+        if (now_sec > te->when_sec ||
+            (now_sec == te->when_sec && now_ms >= te->when_ms))
+        {
+            int retval;
+
+            id = te->id;
+            retval = te->timeProc(eventLoop, id, te->clientData);
+            processed++;
+            if (retval != AE_NOMORE) {
+                aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
+            } else {
+                te->id = AE_DELETED_EVENT_ID;
+            }
+        }
+        prev = te;
+        te = te->next;
+    }
+    return processed;
+}
+
+/* Process every pending time event, then every pending file event
+ * (that may be registered by time event callbacks just processed).
+ * Without special flags the function sleeps until some file event
+ * fires, or when the next time event occurs (if any).
+ *
+ * If flags is 0, the function does nothing and returns.
+ * if flags has AE_ALL_EVENTS set, all the kind of events are processed.
+ * if flags has AE_FILE_EVENTS set, file events are processed.
+ * if flags has AE_TIME_EVENTS set, time events are processed.
+ * if flags has AE_DONT_WAIT set the function returns ASAP until all
+ * the events that's possible to process without to wait are processed.
+ *
+ * The function returns the number of events processed. */
+int aeProcessEvents(aeEventLoop *eventLoop, int flags)
+{
+    int processed = 0, numevents;
+
+    /* Nothing to do? return ASAP */
+    if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0;
+
+    /* Note that we want call select() even if there are no
+     * file events to process as long as we want to process time
+     * events, in order to sleep until the next time event is ready
+     * to fire. */
+    if (eventLoop->maxfd != -1 ||
+        ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) {
+        int j;
+        aeTimeEvent *shortest = NULL;
+        struct timeval tv, *tvp;
+
+        if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT))
+            shortest = aeSearchNearestTimer(eventLoop);
+        if (shortest) {
+            long now_sec, now_ms;
+
+            aeGetTime(&now_sec, &now_ms);
+            tvp = &tv;
+
+            /* How many milliseconds we need to wait for the next
+             * time event to fire? */
+            long long ms =
+                (shortest->when_sec - now_sec)*1000 +
+                shortest->when_ms - now_ms;
+
+            if (ms > 0) {
+                tvp->tv_sec = ms/1000;
+                tvp->tv_usec = (ms % 1000)*1000;
+            } else {
+                tvp->tv_sec = 0;
+                tvp->tv_usec = 0;
+            }
+        } else {
+            /* If we have to check for events but need to return
+             * ASAP because of AE_DONT_WAIT we need to set the timeout
+             * to zero */
+            if (flags & AE_DONT_WAIT) {
+                tv.tv_sec = tv.tv_usec = 0;
+                tvp = &tv;
+            } else {
+                /* Otherwise we can block */
+                tvp = NULL; /* wait forever */
+            }
+        }
+
+        numevents = aeApiPoll(eventLoop, tvp);
+        for (j = 0; j < numevents; j++) {
+            aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd];
+            int mask = eventLoop->fired[j].mask;
+            int fd = eventLoop->fired[j].fd;
+            int rfired = 0;
+
+	    /* note the fe->mask & mask & ... code: maybe an already processed
+             * event removed an element that fired and we still didn't
+             * processed, so we check if the event is still valid. */
+            if (fe->mask & mask & AE_READABLE) {
+                rfired = 1;
+                fe->rfileProc(eventLoop,fd,fe->clientData,mask);
+            }
+            if (fe->mask & mask & AE_WRITABLE) {
+                if (!rfired || fe->wfileProc != fe->rfileProc)
+                    fe->wfileProc(eventLoop,fd,fe->clientData,mask);
+            }
+            processed++;
+        }
+    }
+    /* Check time events */
+    if (flags & AE_TIME_EVENTS)
+        processed += processTimeEvents(eventLoop);
+
+    return processed; /* return the number of processed file/time events */
+}
+
+/* Wait for milliseconds until the given file descriptor becomes
+ * writable/readable/exception */
+int aeWait(int fd, int mask, long long milliseconds) {
+    struct pollfd pfd;
+    int retmask = 0, retval;
+
+    memset(&pfd, 0, sizeof(pfd));
+    pfd.fd = fd;
+    if (mask & AE_READABLE) pfd.events |= POLLIN;
+    if (mask & AE_WRITABLE) pfd.events |= POLLOUT;
+
+    if ((retval = poll(&pfd, 1, milliseconds))== 1) {
+        if (pfd.revents & POLLIN) retmask |= AE_READABLE;
+        if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE;
+	if (pfd.revents & POLLERR) retmask |= AE_WRITABLE;
+        if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE;
+        return retmask;
+    } else {
+        return retval;
+    }
+}
+
+void aeMain(aeEventLoop *eventLoop) {
+    eventLoop->stop = 0;
+    while (!eventLoop->stop) {
+        if (eventLoop->beforesleep != NULL)
+            eventLoop->beforesleep(eventLoop);
+        aeProcessEvents(eventLoop, AE_ALL_EVENTS);
+    }
+}
+
+char *aeGetApiName(void) {
+    return aeApiName();
+}
+
+void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) {
+    eventLoop->beforesleep = beforesleep;
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae.h b/cpp/src/plasma/thirdparty/ae/ae.h
new file mode 100644
index 0000000000000..827c4c9e4e59e
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae.h
@@ -0,0 +1,123 @@
+/* A simple event-driven programming library. Originally I wrote this code
+ * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
+ * it in form of a library for easy reuse.
+ *
+ * Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __AE_H__
+#define __AE_H__
+
+#include <time.h>
+
+#define AE_OK 0
+#define AE_ERR -1
+
+#define AE_NONE 0
+#define AE_READABLE 1
+#define AE_WRITABLE 2
+
+#define AE_FILE_EVENTS 1
+#define AE_TIME_EVENTS 2
+#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
+#define AE_DONT_WAIT 4
+
+#define AE_NOMORE -1
+#define AE_DELETED_EVENT_ID -1
+
+/* Macros */
+#define AE_NOTUSED(V) ((void) V)
+
+struct aeEventLoop;
+
+/* Types and data structures */
+typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
+typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData);
+typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData);
+typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop);
+
+/* File event structure */
+typedef struct aeFileEvent {
+    int mask; /* one of AE_(READABLE|WRITABLE) */
+    aeFileProc *rfileProc;
+    aeFileProc *wfileProc;
+    void *clientData;
+} aeFileEvent;
+
+/* Time event structure */
+typedef struct aeTimeEvent {
+    long long id; /* time event identifier. */
+    long when_sec; /* seconds */
+    long when_ms; /* milliseconds */
+    aeTimeProc *timeProc;
+    aeEventFinalizerProc *finalizerProc;
+    void *clientData;
+    struct aeTimeEvent *next;
+} aeTimeEvent;
+
+/* A fired event */
+typedef struct aeFiredEvent {
+    int fd;
+    int mask;
+} aeFiredEvent;
+
+/* State of an event based program */
+typedef struct aeEventLoop {
+    int maxfd;   /* highest file descriptor currently registered */
+    int setsize; /* max number of file descriptors tracked */
+    long long timeEventNextId;
+    time_t lastTime;     /* Used to detect system clock skew */
+    aeFileEvent *events; /* Registered events */
+    aeFiredEvent *fired; /* Fired events */
+    aeTimeEvent *timeEventHead;
+    int stop;
+    void *apidata; /* This is used for polling API specific data */
+    aeBeforeSleepProc *beforesleep;
+} aeEventLoop;
+
+/* Prototypes */
+aeEventLoop *aeCreateEventLoop(int setsize);
+void aeDeleteEventLoop(aeEventLoop *eventLoop);
+void aeStop(aeEventLoop *eventLoop);
+int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
+        aeFileProc *proc, void *clientData);
+void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask);
+int aeGetFileEvents(aeEventLoop *eventLoop, int fd);
+long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
+        aeTimeProc *proc, void *clientData,
+        aeEventFinalizerProc *finalizerProc);
+int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id);
+int aeProcessEvents(aeEventLoop *eventLoop, int flags);
+int aeWait(int fd, int mask, long long milliseconds);
+void aeMain(aeEventLoop *eventLoop);
+char *aeGetApiName(void);
+void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
+int aeGetSetSize(aeEventLoop *eventLoop);
+int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
+
+#endif
diff --git a/cpp/src/plasma/thirdparty/ae/ae_epoll.c b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
new file mode 100644
index 0000000000000..410aac70dc5af
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
@@ -0,0 +1,135 @@
+/* Linux epoll(2) based ae.c module
+ *
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/epoll.h>
+
+typedef struct aeApiState {
+    int epfd;
+    struct epoll_event *events;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+    aeApiState *state = zmalloc(sizeof(aeApiState));
+
+    if (!state) return -1;
+    state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize);
+    if (!state->events) {
+        zfree(state);
+        return -1;
+    }
+    state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */
+    if (state->epfd == -1) {
+        zfree(state->events);
+        zfree(state);
+        return -1;
+    }
+    eventLoop->apidata = state;
+    return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    aeApiState *state = eventLoop->apidata;
+
+    state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize);
+    return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+    aeApiState *state = eventLoop->apidata;
+
+    close(state->epfd);
+    zfree(state->events);
+    zfree(state);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    struct epoll_event ee = {0}; /* avoid valgrind warning */
+    /* If the fd was already monitored for some event, we need a MOD
+     * operation. Otherwise we need an ADD operation. */
+    int op = eventLoop->events[fd].mask == AE_NONE ?
+            EPOLL_CTL_ADD : EPOLL_CTL_MOD;
+
+    ee.events = 0;
+    mask |= eventLoop->events[fd].mask; /* Merge old events */
+    if (mask & AE_READABLE) ee.events |= EPOLLIN;
+    if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
+    ee.data.fd = fd;
+    if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1;
+    return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
+    aeApiState *state = eventLoop->apidata;
+    struct epoll_event ee = {0}; /* avoid valgrind warning */
+    int mask = eventLoop->events[fd].mask & (~delmask);
+
+    ee.events = 0;
+    if (mask & AE_READABLE) ee.events |= EPOLLIN;
+    if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
+    ee.data.fd = fd;
+    if (mask != AE_NONE) {
+        epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee);
+    } else {
+        /* Note, Kernel < 2.6.9 requires a non null event pointer even for
+         * EPOLL_CTL_DEL. */
+        epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee);
+    }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+    aeApiState *state = eventLoop->apidata;
+    int retval, numevents = 0;
+
+    retval = epoll_wait(state->epfd,state->events,eventLoop->setsize,
+            tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
+    if (retval > 0) {
+        int j;
+
+        numevents = retval;
+        for (j = 0; j < numevents; j++) {
+            int mask = 0;
+            struct epoll_event *e = state->events+j;
+
+            if (e->events & EPOLLIN) mask |= AE_READABLE;
+            if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
+            if (e->events & EPOLLERR) mask |= AE_WRITABLE;
+            if (e->events & EPOLLHUP) mask |= AE_WRITABLE;
+            eventLoop->fired[j].fd = e->data.fd;
+            eventLoop->fired[j].mask = mask;
+        }
+    }
+    return numevents;
+}
+
+static char *aeApiName(void) {
+    return "epoll";
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_evport.c b/cpp/src/plasma/thirdparty/ae/ae_evport.c
new file mode 100644
index 0000000000000..5c317becb6f7d
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_evport.c
@@ -0,0 +1,320 @@
+/* ae.c module for illumos event ports.
+ *
+ * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <assert.h>
+#include <errno.h>
+#include <port.h>
+#include <poll.h>
+
+#include <sys/types.h>
+#include <sys/time.h>
+
+#include <stdio.h>
+
+static int evport_debug = 0;
+
+/*
+ * This file implements the ae API using event ports, present on Solaris-based
+ * systems since Solaris 10.  Using the event port interface, we associate file
+ * descriptors with the port.  Each association also includes the set of poll(2)
+ * events that the consumer is interested in (e.g., POLLIN and POLLOUT).
+ *
+ * There's one tricky piece to this implementation: when we return events via
+ * aeApiPoll, the corresponding file descriptors become dissociated from the
+ * port.  This is necessary because poll events are level-triggered, so if the
+ * fd didn't become dissociated, it would immediately fire another event since
+ * the underlying state hasn't changed yet.  We must re-associate the file
+ * descriptor, but only after we know that our caller has actually read from it.
+ * The ae API does not tell us exactly when that happens, but we do know that
+ * it must happen by the time aeApiPoll is called again.  Our solution is to
+ * keep track of the last fds returned by aeApiPoll and re-associate them next
+ * time aeApiPoll is invoked.
+ *
+ * To summarize, in this module, each fd association is EITHER (a) represented
+ * only via the in-kernel association OR (b) represented by pending_fds and
+ * pending_masks.  (b) is only true for the last fds we returned from aeApiPoll,
+ * and only until we enter aeApiPoll again (at which point we restore the
+ * in-kernel association).
+ */
+#define MAX_EVENT_BATCHSZ 512
+
+typedef struct aeApiState {
+    int     portfd;                             /* event port */
+    int     npending;                           /* # of pending fds */
+    int     pending_fds[MAX_EVENT_BATCHSZ];     /* pending fds */
+    int     pending_masks[MAX_EVENT_BATCHSZ];   /* pending fds' masks */
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+    int i;
+    aeApiState *state = zmalloc(sizeof(aeApiState));
+    if (!state) return -1;
+
+    state->portfd = port_create();
+    if (state->portfd == -1) {
+        zfree(state);
+        return -1;
+    }
+
+    state->npending = 0;
+
+    for (i = 0; i < MAX_EVENT_BATCHSZ; i++) {
+        state->pending_fds[i] = -1;
+        state->pending_masks[i] = AE_NONE;
+    }
+
+    eventLoop->apidata = state;
+    return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    /* Nothing to resize here. */
+    return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+    aeApiState *state = eventLoop->apidata;
+
+    close(state->portfd);
+    zfree(state);
+}
+
+static int aeApiLookupPending(aeApiState *state, int fd) {
+    int i;
+
+    for (i = 0; i < state->npending; i++) {
+        if (state->pending_fds[i] == fd)
+            return (i);
+    }
+
+    return (-1);
+}
+
+/*
+ * Helper function to invoke port_associate for the given fd and mask.
+ */
+static int aeApiAssociate(const char *where, int portfd, int fd, int mask) {
+    int events = 0;
+    int rv, err;
+
+    if (mask & AE_READABLE)
+        events |= POLLIN;
+    if (mask & AE_WRITABLE)
+        events |= POLLOUT;
+
+    if (evport_debug)
+        fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events);
+
+    rv = port_associate(portfd, PORT_SOURCE_FD, fd, events,
+        (void *)(uintptr_t)mask);
+    err = errno;
+
+    if (evport_debug)
+        fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err));
+
+    if (rv == -1) {
+        fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err));
+
+        if (err == EAGAIN)
+            fprintf(stderr, "aeApiAssociate: event port limit exceeded.");
+    }
+
+    return rv;
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    int fullmask, pfd;
+
+    if (evport_debug)
+        fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask);
+
+    /*
+     * Since port_associate's "events" argument replaces any existing events, we
+     * must be sure to include whatever events are already associated when
+     * we call port_associate() again.
+     */
+    fullmask = mask | eventLoop->events[fd].mask;
+    pfd = aeApiLookupPending(state, fd);
+
+    if (pfd != -1) {
+        /*
+         * This fd was recently returned from aeApiPoll.  It should be safe to
+         * assume that the consumer has processed that poll event, but we play
+         * it safer by simply updating pending_mask.  The fd will be
+         * re-associated as usual when aeApiPoll is called again.
+         */
+        if (evport_debug)
+            fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd);
+        state->pending_masks[pfd] |= fullmask;
+        return 0;
+    }
+
+    return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask));
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    int fullmask, pfd;
+
+    if (evport_debug)
+        fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask);
+
+    pfd = aeApiLookupPending(state, fd);
+
+    if (pfd != -1) {
+        if (evport_debug)
+            fprintf(stderr, "deleting event from pending fd %d\n", fd);
+
+        /*
+         * This fd was just returned from aeApiPoll, so it's not currently
+         * associated with the port.  All we need to do is update
+         * pending_mask appropriately.
+         */
+        state->pending_masks[pfd] &= ~mask;
+
+        if (state->pending_masks[pfd] == AE_NONE)
+            state->pending_fds[pfd] = -1;
+
+        return;
+    }
+
+    /*
+     * The fd is currently associated with the port.  Like with the add case
+     * above, we must look at the full mask for the file descriptor before
+     * updating that association.  We don't have a good way of knowing what the
+     * events are without looking into the eventLoop state directly.  We rely on
+     * the fact that our caller has already updated the mask in the eventLoop.
+     */
+
+    fullmask = eventLoop->events[fd].mask;
+    if (fullmask == AE_NONE) {
+        /*
+         * We're removing *all* events, so use port_dissociate to remove the
+         * association completely.  Failure here indicates a bug.
+         */
+        if (evport_debug)
+            fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd);
+
+        if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) {
+            perror("aeApiDelEvent: port_dissociate");
+            abort(); /* will not return */
+        }
+    } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd,
+        fullmask) != 0) {
+        /*
+         * ENOMEM is a potentially transient condition, but the kernel won't
+         * generally return it unless things are really bad.  EAGAIN indicates
+         * we've reached an resource limit, for which it doesn't make sense to
+         * retry (counter-intuitively).  All other errors indicate a bug.  In any
+         * of these cases, the best we can do is to abort.
+         */
+        abort(); /* will not return */
+    }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+    aeApiState *state = eventLoop->apidata;
+    struct timespec timeout, *tsp;
+    int mask, i;
+    uint_t nevents;
+    port_event_t event[MAX_EVENT_BATCHSZ];
+
+    /*
+     * If we've returned fd events before, we must re-associate them with the
+     * port now, before calling port_get().  See the block comment at the top of
+     * this file for an explanation of why.
+     */
+    for (i = 0; i < state->npending; i++) {
+        if (state->pending_fds[i] == -1)
+            /* This fd has since been deleted. */
+            continue;
+
+        if (aeApiAssociate("aeApiPoll", state->portfd,
+            state->pending_fds[i], state->pending_masks[i]) != 0) {
+            /* See aeApiDelEvent for why this case is fatal. */
+            abort();
+        }
+
+        state->pending_masks[i] = AE_NONE;
+        state->pending_fds[i] = -1;
+    }
+
+    state->npending = 0;
+
+    if (tvp != NULL) {
+        timeout.tv_sec = tvp->tv_sec;
+        timeout.tv_nsec = tvp->tv_usec * 1000;
+        tsp = &timeout;
+    } else {
+        tsp = NULL;
+    }
+
+    /*
+     * port_getn can return with errno == ETIME having returned some events (!).
+     * So if we get ETIME, we check nevents, too.
+     */
+    nevents = 1;
+    if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents,
+        tsp) == -1 && (errno != ETIME || nevents == 0)) {
+        if (errno == ETIME || errno == EINTR)
+            return 0;
+
+        /* Any other error indicates a bug. */
+        perror("aeApiPoll: port_get");
+        abort();
+    }
+
+    state->npending = nevents;
+
+    for (i = 0; i < nevents; i++) {
+            mask = 0;
+            if (event[i].portev_events & POLLIN)
+                mask |= AE_READABLE;
+            if (event[i].portev_events & POLLOUT)
+                mask |= AE_WRITABLE;
+
+            eventLoop->fired[i].fd = event[i].portev_object;
+            eventLoop->fired[i].mask = mask;
+
+            if (evport_debug)
+                fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n",
+                    (int)event[i].portev_object, mask);
+
+            state->pending_fds[i] = event[i].portev_object;
+            state->pending_masks[i] = (uintptr_t)event[i].portev_user;
+    }
+
+    return nevents;
+}
+
+static char *aeApiName(void) {
+    return "evport";
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_kqueue.c b/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
new file mode 100644
index 0000000000000..6796f4ceb5939
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
@@ -0,0 +1,138 @@
+/* Kqueue(2)-based ae.c module
+ *
+ * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/time.h>
+
+typedef struct aeApiState {
+    int kqfd;
+    struct kevent *events;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+    aeApiState *state = zmalloc(sizeof(aeApiState));
+
+    if (!state) return -1;
+    state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize);
+    if (!state->events) {
+        zfree(state);
+        return -1;
+    }
+    state->kqfd = kqueue();
+    if (state->kqfd == -1) {
+        zfree(state->events);
+        zfree(state);
+        return -1;
+    }
+    eventLoop->apidata = state;
+    return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    aeApiState *state = eventLoop->apidata;
+
+    state->events = zrealloc(state->events, sizeof(struct kevent)*setsize);
+    return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+    aeApiState *state = eventLoop->apidata;
+
+    close(state->kqfd);
+    zfree(state->events);
+    zfree(state);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    struct kevent ke;
+
+    if (mask & AE_READABLE) {
+        EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
+        if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
+    }
+    if (mask & AE_WRITABLE) {
+        EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL);
+        if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
+    }
+    return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    struct kevent ke;
+
+    if (mask & AE_READABLE) {
+        EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
+        kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
+    }
+    if (mask & AE_WRITABLE) {
+        EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
+        kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
+    }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+    aeApiState *state = eventLoop->apidata;
+    int retval, numevents = 0;
+
+    if (tvp != NULL) {
+        struct timespec timeout;
+        timeout.tv_sec = tvp->tv_sec;
+        timeout.tv_nsec = tvp->tv_usec * 1000;
+        retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
+                        &timeout);
+    } else {
+        retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
+                        NULL);
+    }
+
+    if (retval > 0) {
+        int j;
+
+        numevents = retval;
+        for(j = 0; j < numevents; j++) {
+            int mask = 0;
+            struct kevent *e = state->events+j;
+
+            if (e->filter == EVFILT_READ) mask |= AE_READABLE;
+            if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
+            eventLoop->fired[j].fd = e->ident;
+            eventLoop->fired[j].mask = mask;
+        }
+    }
+    return numevents;
+}
+
+static char *aeApiName(void) {
+    return "kqueue";
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_select.c b/cpp/src/plasma/thirdparty/ae/ae_select.c
new file mode 100644
index 0000000000000..c039a8ea3128d
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_select.c
@@ -0,0 +1,106 @@
+/* Select()-based ae.c module.
+ *
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/select.h>
+#include <string.h>
+
+typedef struct aeApiState {
+    fd_set rfds, wfds;
+    /* We need to have a copy of the fd sets as it's not safe to reuse
+     * FD sets after select(). */
+    fd_set _rfds, _wfds;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+    aeApiState *state = zmalloc(sizeof(aeApiState));
+
+    if (!state) return -1;
+    FD_ZERO(&state->rfds);
+    FD_ZERO(&state->wfds);
+    eventLoop->apidata = state;
+    return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    /* Just ensure we have enough room in the fd_set type. */
+    if (setsize >= FD_SETSIZE) return -1;
+    return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+    zfree(eventLoop->apidata);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+
+    if (mask & AE_READABLE) FD_SET(fd,&state->rfds);
+    if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds);
+    return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+
+    if (mask & AE_READABLE) FD_CLR(fd,&state->rfds);
+    if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds);
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+    aeApiState *state = eventLoop->apidata;
+    int retval, j, numevents = 0;
+
+    memcpy(&state->_rfds,&state->rfds,sizeof(fd_set));
+    memcpy(&state->_wfds,&state->wfds,sizeof(fd_set));
+
+    retval = select(eventLoop->maxfd+1,
+                &state->_rfds,&state->_wfds,NULL,tvp);
+    if (retval > 0) {
+        for (j = 0; j <= eventLoop->maxfd; j++) {
+            int mask = 0;
+            aeFileEvent *fe = &eventLoop->events[j];
+
+            if (fe->mask == AE_NONE) continue;
+            if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds))
+                mask |= AE_READABLE;
+            if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds))
+                mask |= AE_WRITABLE;
+            eventLoop->fired[numevents].fd = j;
+            eventLoop->fired[numevents].mask = mask;
+            numevents++;
+        }
+    }
+    return numevents;
+}
+
+static char *aeApiName(void) {
+    return "select";
+}
diff --git a/cpp/src/plasma/thirdparty/ae/config.h b/cpp/src/plasma/thirdparty/ae/config.h
new file mode 100644
index 0000000000000..4f8e1ea1bc38c
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/config.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __CONFIG_H
+#define __CONFIG_H
+
+#ifdef __APPLE__
+#include <AvailabilityMacros.h>
+#endif
+
+/* Test for polling API */
+#ifdef __linux__
+#define HAVE_EPOLL 1
+#endif
+
+#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__)
+#define HAVE_KQUEUE 1
+#endif
+
+#ifdef __sun
+#include <sys/feature_tests.h>
+#ifdef _DTRACE_VERSION
+#define HAVE_EVPORT 1
+#endif
+#endif
+
+
+#endif
diff --git a/cpp/src/plasma/thirdparty/ae/zmalloc.h b/cpp/src/plasma/thirdparty/ae/zmalloc.h
new file mode 100644
index 0000000000000..6c27dd4e5c3d3
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/zmalloc.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _ZMALLOC_H
+#define _ZMALLOC_H
+
+#ifndef zmalloc
+#define zmalloc malloc
+#endif
+
+#ifndef zfree
+#define zfree free
+#endif
+
+#ifndef zrealloc
+#define zrealloc realloc
+#endif
+
+#endif /* _ZMALLOC_H */
diff --git a/cpp/src/plasma/thirdparty/dlmalloc.c b/cpp/src/plasma/thirdparty/dlmalloc.c
new file mode 100644
index 0000000000000..84ccbd28fc4ec
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/dlmalloc.c
@@ -0,0 +1,6281 @@
+/*
+  This is a version (aka dlmalloc) of malloc/free/realloc written by
+  Doug Lea and released to the public domain, as explained at
+  http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
+  comments, complaints, performance data, etc to dl@cs.oswego.edu
+
+* Version 2.8.6 Wed Aug 29 06:57:58 2012  Doug Lea
+   Note: There may be an updated version of this malloc obtainable at
+           ftp://gee.cs.oswego.edu/pub/misc/malloc.c
+         Check before installing!
+
+* Quickstart
+
+  This library is all in one file to simplify the most common usage:
+  ftp it, compile it (-O3), and link it into another program. All of
+  the compile-time options default to reasonable values for use on
+  most platforms.  You might later want to step through various
+  compile-time and dynamic tuning options.
+
+  For convenience, an include file for code using this malloc is at:
+     ftp://gee.cs.oswego.edu/pub/misc/malloc-2.8.6.h
+  You don't really need this .h file unless you call functions not
+  defined in your system include files.  The .h file contains only the
+  excerpts from this file needed for using this malloc on ANSI C/C++
+  systems, so long as you haven't changed compile-time options about
+  naming and tuning parameters.  If you do, then you can create your
+  own malloc.h that does include all settings by cutting at the point
+  indicated below. Note that you may already by default be using a C
+  library containing a malloc that is based on some version of this
+  malloc (for example in linux). You might still want to use the one
+  in this file to customize settings or to avoid overheads associated
+  with library versions.
+
+* Vital statistics:
+
+  Supported pointer/size_t representation:       4 or 8 bytes
+       size_t MUST be an unsigned type of the same width as
+       pointers. (If you are using an ancient system that declares
+       size_t as a signed type, or need it to be a different width
+       than pointers, you can use a previous release of this malloc
+       (e.g. 2.7.2) supporting these.)
+
+  Alignment:                                     8 bytes (minimum)
+       This suffices for nearly all current machines and C compilers.
+       However, you can define MALLOC_ALIGNMENT to be wider than this
+       if necessary (up to 128bytes), at the expense of using more space.
+
+  Minimum overhead per allocated chunk:   4 or  8 bytes (if 4byte sizes)
+                                          8 or 16 bytes (if 8byte sizes)
+       Each malloced chunk has a hidden word of overhead holding size
+       and status information, and additional cross-check word
+       if FOOTERS is defined.
+
+  Minimum allocated size: 4-byte ptrs:  16 bytes    (including overhead)
+                          8-byte ptrs:  32 bytes    (including overhead)
+
+       Even a request for zero bytes (i.e., malloc(0)) returns a
+       pointer to something of the minimum allocatable size.
+       The maximum overhead wastage (i.e., number of extra bytes
+       allocated than were requested in malloc) is less than or equal
+       to the minimum size, except for requests >= mmap_threshold that
+       are serviced via mmap(), where the worst case wastage is about
+       32 bytes plus the remainder from a system page (the minimal
+       mmap unit); typically 4096 or 8192 bytes.
+
+  Security: static-safe; optionally more or less
+       The "security" of malloc refers to the ability of malicious
+       code to accentuate the effects of errors (for example, freeing
+       space that is not currently malloc'ed or overwriting past the
+       ends of chunks) in code that calls malloc.  This malloc
+       guarantees not to modify any memory locations below the base of
+       heap, i.e., static variables, even in the presence of usage
+       errors.  The routines additionally detect most improper frees
+       and reallocs.  All this holds as long as the static bookkeeping
+       for malloc itself is not corrupted by some other means.  This
+       is only one aspect of security -- these checks do not, and
+       cannot, detect all possible programming errors.
+
+       If FOOTERS is defined nonzero, then each allocated chunk
+       carries an additional check word to verify that it was malloced
+       from its space.  These check words are the same within each
+       execution of a program using malloc, but differ across
+       executions, so externally crafted fake chunks cannot be
+       freed. This improves security by rejecting frees/reallocs that
+       could corrupt heap memory, in addition to the checks preventing
+       writes to statics that are always on.  This may further improve
+       security at the expense of time and space overhead.  (Note that
+       FOOTERS may also be worth using with MSPACES.)
+
+       By default detected errors cause the program to abort (calling
+       "abort()"). You can override this to instead proceed past
+       errors by defining PROCEED_ON_ERROR.  In this case, a bad free
+       has no effect, and a malloc that encounters a bad address
+       caused by user overwrites will ignore the bad address by
+       dropping pointers and indices to all known memory. This may
+       be appropriate for programs that should continue if at all
+       possible in the face of programming errors, although they may
+       run out of memory because dropped memory is never reclaimed.
+
+       If you don't like either of these options, you can define
+       CORRUPTION_ERROR_ACTION and USAGE_ERROR_ACTION to do anything
+       else. And if if you are sure that your program using malloc has
+       no errors or vulnerabilities, you can define INSECURE to 1,
+       which might (or might not) provide a small performance improvement.
+
+       It is also possible to limit the maximum total allocatable
+       space, using malloc_set_footprint_limit. This is not
+       designed as a security feature in itself (calls to set limits
+       are not screened or privileged), but may be useful as one
+       aspect of a secure implementation.
+
+  Thread-safety: NOT thread-safe unless USE_LOCKS defined non-zero
+       When USE_LOCKS is defined, each public call to malloc, free,
+       etc is surrounded with a lock. By default, this uses a plain
+       pthread mutex, win32 critical section, or a spin-lock if if
+       available for the platform and not disabled by setting
+       USE_SPIN_LOCKS=0.  However, if USE_RECURSIVE_LOCKS is defined,
+       recursive versions are used instead (which are not required for
+       base functionality but may be needed in layered extensions).
+       Using a global lock is not especially fast, and can be a major
+       bottleneck.  It is designed only to provide minimal protection
+       in concurrent environments, and to provide a basis for
+       extensions.  If you are using malloc in a concurrent program,
+       consider instead using nedmalloc
+       (http://www.nedprod.com/programs/portable/nedmalloc/) or
+       ptmalloc (See http://www.malloc.de), which are derived from
+       versions of this malloc.
+
+  System requirements: Any combination of MORECORE and/or MMAP/MUNMAP
+       This malloc can use unix sbrk or any emulation (invoked using
+       the CALL_MORECORE macro) and/or mmap/munmap or any emulation
+       (invoked using CALL_MMAP/CALL_MUNMAP) to get and release system
+       memory.  On most unix systems, it tends to work best if both
+       MORECORE and MMAP are enabled.  On Win32, it uses emulations
+       based on VirtualAlloc. It also uses common C library functions
+       like memset.
+
+  Compliance: I believe it is compliant with the Single Unix Specification
+       (See http://www.unix.org). Also SVID/XPG, ANSI C, and probably
+       others as well.
+
+* Overview of algorithms
+
+  This is not the fastest, most space-conserving, most portable, or
+  most tunable malloc ever written. However it is among the fastest
+  while also being among the most space-conserving, portable and
+  tunable.  Consistent balance across these factors results in a good
+  general-purpose allocator for malloc-intensive programs.
+
+  In most ways, this malloc is a best-fit allocator. Generally, it
+  chooses the best-fitting existing chunk for a request, with ties
+  broken in approximately least-recently-used order. (This strategy
+  normally maintains low fragmentation.) However, for requests less
+  than 256bytes, it deviates from best-fit when there is not an
+  exactly fitting available chunk by preferring to use space adjacent
+  to that used for the previous small request, as well as by breaking
+  ties in approximately most-recently-used order. (These enhance
+  locality of series of small allocations.)  And for very large requests
+  (>= 256Kb by default), it relies on system memory mapping
+  facilities, if supported.  (This helps avoid carrying around and
+  possibly fragmenting memory used only for large chunks.)
+
+  All operations (except malloc_stats and mallinfo) have execution
+  times that are bounded by a constant factor of the number of bits in
+  a size_t, not counting any clearing in calloc or copying in realloc,
+  or actions surrounding MORECORE and MMAP that have times
+  proportional to the number of non-contiguous regions returned by
+  system allocation routines, which is often just 1. In real-time
+  applications, you can optionally suppress segment traversals using
+  NO_SEGMENT_TRAVERSAL, which assures bounded execution even when
+  system allocators return non-contiguous spaces, at the typical
+  expense of carrying around more memory and increased fragmentation.
+
+  The implementation is not very modular and seriously overuses
+  macros. Perhaps someday all C compilers will do as good a job
+  inlining modular code as can now be done by brute-force expansion,
+  but now, enough of them seem not to.
+
+  Some compilers issue a lot of warnings about code that is
+  dead/unreachable only on some platforms, and also about intentional
+  uses of negation on unsigned types. All known cases of each can be
+  ignored.
+
+  For a longer but out of date high-level description, see
+     http://gee.cs.oswego.edu/dl/html/malloc.html
+
+* MSPACES
+  If MSPACES is defined, then in addition to malloc, free, etc.,
+  this file also defines mspace_malloc, mspace_free, etc. These
+  are versions of malloc routines that take an "mspace" argument
+  obtained using create_mspace, to control all internal bookkeeping.
+  If ONLY_MSPACES is defined, only these versions are compiled.
+  So if you would like to use this allocator for only some allocations,
+  and your system malloc for others, you can compile with
+  ONLY_MSPACES and then do something like...
+    static mspace mymspace = create_mspace(0,0); // for example
+    #define mymalloc(bytes)  mspace_malloc(mymspace, bytes)
+
+  (Note: If you only need one instance of an mspace, you can instead
+  use "USE_DL_PREFIX" to relabel the global malloc.)
+
+  You can similarly create thread-local allocators by storing
+  mspaces as thread-locals. For example:
+    static __thread mspace tlms = 0;
+    void*  tlmalloc(size_t bytes) {
+      if (tlms == 0) tlms = create_mspace(0, 0);
+      return mspace_malloc(tlms, bytes);
+    }
+    void  tlfree(void* mem) { mspace_free(tlms, mem); }
+
+  Unless FOOTERS is defined, each mspace is completely independent.
+  You cannot allocate from one and free to another (although
+  conformance is only weakly checked, so usage errors are not always
+  caught). If FOOTERS is defined, then each chunk carries around a tag
+  indicating its originating mspace, and frees are directed to their
+  originating spaces. Normally, this requires use of locks.
+
+ -------------------------  Compile-time options ---------------------------
+
+Be careful in setting #define values for numerical constants of type
+size_t. On some systems, literal values are not automatically extended
+to size_t precision unless they are explicitly casted. You can also
+use the symbolic values MAX_SIZE_T, SIZE_T_ONE, etc below.
+
+WIN32                    default: defined if _WIN32 defined
+  Defining WIN32 sets up defaults for MS environment and compilers.
+  Otherwise defaults are for unix. Beware that there seem to be some
+  cases where this malloc might not be a pure drop-in replacement for
+  Win32 malloc: Random-looking failures from Win32 GDI API's (eg;
+  SetDIBits()) may be due to bugs in some video driver implementations
+  when pixel buffers are malloc()ed, and the region spans more than
+  one VirtualAlloc()ed region. Because dlmalloc uses a small (64Kb)
+  default granularity, pixel buffers may straddle virtual allocation
+  regions more often than when using the Microsoft allocator.  You can
+  avoid this by using VirtualAlloc() and VirtualFree() for all pixel
+  buffers rather than using malloc().  If this is not possible,
+  recompile this malloc with a larger DEFAULT_GRANULARITY. Note:
+  in cases where MSC and gcc (cygwin) are known to differ on WIN32,
+  conditions use _MSC_VER to distinguish them.
+
+DLMALLOC_EXPORT       default: extern
+  Defines how public APIs are declared. If you want to export via a
+  Windows DLL, you might define this as
+    #define DLMALLOC_EXPORT extern  __declspec(dllexport)
+  If you want a POSIX ELF shared object, you might use
+    #define DLMALLOC_EXPORT extern __attribute__((visibility("default")))
+
+MALLOC_ALIGNMENT         default: (size_t)(2 * sizeof(void *))
+  Controls the minimum alignment for malloc'ed chunks.  It must be a
+  power of two and at least 8, even on machines for which smaller
+  alignments would suffice. It may be defined as larger than this
+  though. Note however that code and data structures are optimized for
+  the case of 8-byte alignment.
+
+MSPACES                  default: 0 (false)
+  If true, compile in support for independent allocation spaces.
+  This is only supported if HAVE_MMAP is true.
+
+ONLY_MSPACES             default: 0 (false)
+  If true, only compile in mspace versions, not regular versions.
+
+USE_LOCKS                default: 0 (false)
+  Causes each call to each public routine to be surrounded with
+  pthread or WIN32 mutex lock/unlock. (If set true, this can be
+  overridden on a per-mspace basis for mspace versions.) If set to a
+  non-zero value other than 1, locks are used, but their
+  implementation is left out, so lock functions must be supplied manually,
+  as described below.
+
+USE_SPIN_LOCKS           default: 1 iff USE_LOCKS and spin locks available
+  If true, uses custom spin locks for locking. This is currently
+  supported only gcc >= 4.1, older gccs on x86 platforms, and recent
+  MS compilers.  Otherwise, posix locks or win32 critical sections are
+  used.
+
+USE_RECURSIVE_LOCKS      default: not defined
+  If defined nonzero, uses recursive (aka reentrant) locks, otherwise
+  uses plain mutexes. This is not required for malloc proper, but may
+  be needed for layered allocators such as nedmalloc.
+
+LOCK_AT_FORK            default: not defined
+  If defined nonzero, performs pthread_atfork upon initialization
+  to initialize child lock while holding parent lock. The implementation
+  assumes that pthread locks (not custom locks) are being used. In other
+  cases, you may need to customize the implementation.
+
+FOOTERS                  default: 0
+  If true, provide extra checking and dispatching by placing
+  information in the footers of allocated chunks. This adds
+  space and time overhead.
+
+INSECURE                 default: 0
+  If true, omit checks for usage errors and heap space overwrites.
+
+USE_DL_PREFIX            default: NOT defined
+  Causes compiler to prefix all public routines with the string 'dl'.
+  This can be useful when you only want to use this malloc in one part
+  of a program, using your regular system malloc elsewhere.
+
+MALLOC_INSPECT_ALL       default: NOT defined
+  If defined, compiles malloc_inspect_all and mspace_inspect_all, that
+  perform traversal of all heap space.  Unless access to these
+  functions is otherwise restricted, you probably do not want to
+  include them in secure implementations.
+
+ABORT                    default: defined as abort()
+  Defines how to abort on failed checks.  On most systems, a failed
+  check cannot die with an "assert" or even print an informative
+  message, because the underlying print routines in turn call malloc,
+  which will fail again.  Generally, the best policy is to simply call
+  abort(). It's not very useful to do more than this because many
+  errors due to overwriting will show up as address faults (null, odd
+  addresses etc) rather than malloc-triggered checks, so will also
+  abort.  Also, most compilers know that abort() does not return, so
+  can better optimize code conditionally calling it.
+
+PROCEED_ON_ERROR           default: defined as 0 (false)
+  Controls whether detected bad addresses cause them to bypassed
+  rather than aborting. If set, detected bad arguments to free and
+  realloc are ignored. And all bookkeeping information is zeroed out
+  upon a detected overwrite of freed heap space, thus losing the
+  ability to ever return it from malloc again, but enabling the
+  application to proceed. If PROCEED_ON_ERROR is defined, the
+  static variable malloc_corruption_error_count is compiled in
+  and can be examined to see if errors have occurred. This option
+  generates slower code than the default abort policy.
+
+DEBUG                    default: NOT defined
+  The DEBUG setting is mainly intended for people trying to modify
+  this code or diagnose problems when porting to new platforms.
+  However, it may also be able to better isolate user errors than just
+  using runtime checks.  The assertions in the check routines spell
+  out in more detail the assumptions and invariants underlying the
+  algorithms.  The checking is fairly extensive, and will slow down
+  execution noticeably. Calling malloc_stats or mallinfo with DEBUG
+  set will attempt to check every non-mmapped allocated and free chunk
+  in the course of computing the summaries.
+
+ABORT_ON_ASSERT_FAILURE   default: defined as 1 (true)
+  Debugging assertion failures can be nearly impossible if your
+  version of the assert macro causes malloc to be called, which will
+  lead to a cascade of further failures, blowing the runtime stack.
+  ABORT_ON_ASSERT_FAILURE cause assertions failures to call abort(),
+  which will usually make debugging easier.
+
+MALLOC_FAILURE_ACTION     default: sets errno to ENOMEM, or no-op on win32
+  The action to take before "return 0" when malloc fails to be able to
+  return memory because there is none available.
+
+HAVE_MORECORE             default: 1 (true) unless win32 or ONLY_MSPACES
+  True if this system supports sbrk or an emulation of it.
+
+MORECORE                  default: sbrk
+  The name of the sbrk-style system routine to call to obtain more
+  memory.  See below for guidance on writing custom MORECORE
+  functions. The type of the argument to sbrk/MORECORE varies across
+  systems.  It cannot be size_t, because it supports negative
+  arguments, so it is normally the signed type of the same width as
+  size_t (sometimes declared as "intptr_t").  It doesn't much matter
+  though. Internally, we only call it with arguments less than half
+  the max value of a size_t, which should work across all reasonable
+  possibilities, although sometimes generating compiler warnings.
+
+MORECORE_CONTIGUOUS       default: 1 (true) if HAVE_MORECORE
+  If true, take advantage of fact that consecutive calls to MORECORE
+  with positive arguments always return contiguous increasing
+  addresses.  This is true of unix sbrk. It does not hurt too much to
+  set it true anyway, since malloc copes with non-contiguities.
+  Setting it false when definitely non-contiguous saves time
+  and possibly wasted space it would take to discover this though.
+
+MORECORE_CANNOT_TRIM      default: NOT defined
+  True if MORECORE cannot release space back to the system when given
+  negative arguments. This is generally necessary only if you are
+  using a hand-crafted MORECORE function that cannot handle negative
+  arguments.
+
+NO_SEGMENT_TRAVERSAL       default: 0
+  If non-zero, suppresses traversals of memory segments
+  returned by either MORECORE or CALL_MMAP. This disables
+  merging of segments that are contiguous, and selectively
+  releasing them to the OS if unused, but bounds execution times.
+
+HAVE_MMAP                 default: 1 (true)
+  True if this system supports mmap or an emulation of it.  If so, and
+  HAVE_MORECORE is not true, MMAP is used for all system
+  allocation. If set and HAVE_MORECORE is true as well, MMAP is
+  primarily used to directly allocate very large blocks. It is also
+  used as a backup strategy in cases where MORECORE fails to provide
+  space from system. Note: A single call to MUNMAP is assumed to be
+  able to unmap memory that may have be allocated using multiple calls
+  to MMAP, so long as they are adjacent.
+
+HAVE_MREMAP               default: 1 on linux, else 0
+  If true realloc() uses mremap() to re-allocate large blocks and
+  extend or shrink allocation spaces.
+
+MMAP_CLEARS               default: 1 except on WINCE.
+  True if mmap clears memory so calloc doesn't need to. This is true
+  for standard unix mmap using /dev/zero and on WIN32 except for WINCE.
+
+USE_BUILTIN_FFS            default: 0 (i.e., not used)
+  Causes malloc to use the builtin ffs() function to compute indices.
+  Some compilers may recognize and intrinsify ffs to be faster than the
+  supplied C version. Also, the case of x86 using gcc is special-cased
+  to an asm instruction, so is already as fast as it can be, and so
+  this setting has no effect. Similarly for Win32 under recent MS compilers.
+  (On most x86s, the asm version is only slightly faster than the C version.)
+
+malloc_getpagesize         default: derive from system includes, or 4096.
+  The system page size. To the extent possible, this malloc manages
+  memory from the system in page-size units.  This may be (and
+  usually is) a function rather than a constant. This is ignored
+  if WIN32, where page size is determined using getSystemInfo during
+  initialization.
+
+USE_DEV_RANDOM             default: 0 (i.e., not used)
+  Causes malloc to use /dev/random to initialize secure magic seed for
+  stamping footers. Otherwise, the current time is used.
+
+NO_MALLINFO                default: 0
+  If defined, don't compile "mallinfo". This can be a simple way
+  of dealing with mismatches between system declarations and
+  those in this file.
+
+MALLINFO_FIELD_TYPE        default: size_t
+  The type of the fields in the mallinfo struct. This was originally
+  defined as "int" in SVID etc, but is more usefully defined as
+  size_t. The value is used only if  HAVE_USR_INCLUDE_MALLOC_H is not set
+
+NO_MALLOC_STATS            default: 0
+  If defined, don't compile "malloc_stats". This avoids calls to
+  fprintf and bringing in stdio dependencies you might not want.
+
+REALLOC_ZERO_BYTES_FREES    default: not defined
+  This should be set if a call to realloc with zero bytes should
+  be the same as a call to free. Some people think it should. Otherwise,
+  since this malloc returns a unique pointer for malloc(0), so does
+  realloc(p, 0).
+
+LACKS_UNISTD_H, LACKS_FCNTL_H, LACKS_SYS_PARAM_H, LACKS_SYS_MMAN_H
+LACKS_STRINGS_H, LACKS_STRING_H, LACKS_SYS_TYPES_H,  LACKS_ERRNO_H
+LACKS_STDLIB_H LACKS_SCHED_H LACKS_TIME_H  default: NOT defined unless on WIN32
+  Define these if your system does not have these header files.
+  You might need to manually insert some of the declarations they provide.
+
+DEFAULT_GRANULARITY        default: page size if MORECORE_CONTIGUOUS,
+                                system_info.dwAllocationGranularity in WIN32,
+                                otherwise 64K.
+      Also settable using mallopt(M_GRANULARITY, x)
+  The unit for allocating and deallocating memory from the system.  On
+  most systems with contiguous MORECORE, there is no reason to
+  make this more than a page. However, systems with MMAP tend to
+  either require or encourage larger granularities.  You can increase
+  this value to prevent system allocation functions to be called so
+  often, especially if they are slow.  The value must be at least one
+  page and must be a power of two.  Setting to 0 causes initialization
+  to either page size or win32 region size.  (Note: In previous
+  versions of malloc, the equivalent of this option was called
+  "TOP_PAD")
+
+DEFAULT_TRIM_THRESHOLD    default: 2MB
+      Also settable using mallopt(M_TRIM_THRESHOLD, x)
+  The maximum amount of unused top-most memory to keep before
+  releasing via malloc_trim in free().  Automatic trimming is mainly
+  useful in long-lived programs using contiguous MORECORE.  Because
+  trimming via sbrk can be slow on some systems, and can sometimes be
+  wasteful (in cases where programs immediately afterward allocate
+  more large chunks) the value should be high enough so that your
+  overall system performance would improve by releasing this much
+  memory.  As a rough guide, you might set to a value close to the
+  average size of a process (program) running on your system.
+  Releasing this much memory would allow such a process to run in
+  memory.  Generally, it is worth tuning trim thresholds when a
+  program undergoes phases where several large chunks are allocated
+  and released in ways that can reuse each other's storage, perhaps
+  mixed with phases where there are no such chunks at all. The trim
+  value must be greater than page size to have any useful effect.  To
+  disable trimming completely, you can set to MAX_SIZE_T. Note that the trick
+  some people use of mallocing a huge space and then freeing it at
+  program startup, in an attempt to reserve system memory, doesn't
+  have the intended effect under automatic trimming, since that memory
+  will immediately be returned to the system.
+
+DEFAULT_MMAP_THRESHOLD       default: 256K
+      Also settable using mallopt(M_MMAP_THRESHOLD, x)
+  The request size threshold for using MMAP to directly service a
+  request. Requests of at least this size that cannot be allocated
+  using already-existing space will be serviced via mmap.  (If enough
+  normal freed space already exists it is used instead.)  Using mmap
+  segregates relatively large chunks of memory so that they can be
+  individually obtained and released from the host system. A request
+  serviced through mmap is never reused by any other request (at least
+  not directly; the system may just so happen to remap successive
+  requests to the same locations).  Segregating space in this way has
+  the benefits that: Mmapped space can always be individually released
+  back to the system, which helps keep the system level memory demands
+  of a long-lived program low.  Also, mapped memory doesn't become
+  `locked' between other chunks, as can happen with normally allocated
+  chunks, which means that even trimming via malloc_trim would not
+  release them.  However, it has the disadvantage that the space
+  cannot be reclaimed, consolidated, and then used to service later
+  requests, as happens with normal chunks.  The advantages of mmap
+  nearly always outweigh disadvantages for "large" chunks, but the
+  value of "large" may vary across systems.  The default is an
+  empirically derived value that works well in most systems. You can
+  disable mmap by setting to MAX_SIZE_T.
+
+MAX_RELEASE_CHECK_RATE   default: 4095 unless not HAVE_MMAP
+  The number of consolidated frees between checks to release
+  unused segments when freeing. When using non-contiguous segments,
+  especially with multiple mspaces, checking only for topmost space
+  doesn't always suffice to trigger trimming. To compensate for this,
+  free() will, with a period of MAX_RELEASE_CHECK_RATE (or the
+  current number of segments, if greater) try to release unused
+  segments to the OS when freeing chunks that result in
+  consolidation. The best value for this parameter is a compromise
+  between slowing down frees with relatively costly checks that
+  rarely trigger versus holding on to unused memory. To effectively
+  disable, set to MAX_SIZE_T. This may lead to a very slight speed
+  improvement at the expense of carrying around more memory.
+*/
+
+/* Version identifier to allow people to support multiple versions */
+#ifndef DLMALLOC_VERSION
+#define DLMALLOC_VERSION 20806
+#endif /* DLMALLOC_VERSION */
+
+#ifndef DLMALLOC_EXPORT
+#define DLMALLOC_EXPORT extern
+#endif
+
+#ifndef WIN32
+#ifdef _WIN32
+#define WIN32 1
+#endif  /* _WIN32 */
+#ifdef _WIN32_WCE
+#define LACKS_FCNTL_H
+#define WIN32 1
+#endif /* _WIN32_WCE */
+#endif  /* WIN32 */
+#ifdef WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <tchar.h>
+#define HAVE_MMAP 1
+#define HAVE_MORECORE 0
+#define LACKS_UNISTD_H
+#define LACKS_SYS_PARAM_H
+#define LACKS_SYS_MMAN_H
+#define LACKS_STRING_H
+#define LACKS_STRINGS_H
+#define LACKS_SYS_TYPES_H
+#define LACKS_ERRNO_H
+#define LACKS_SCHED_H
+#ifndef MALLOC_FAILURE_ACTION
+#define MALLOC_FAILURE_ACTION
+#endif /* MALLOC_FAILURE_ACTION */
+#ifndef MMAP_CLEARS
+#ifdef _WIN32_WCE /* WINCE reportedly does not clear */
+#define MMAP_CLEARS 0
+#else
+#define MMAP_CLEARS 1
+#endif /* _WIN32_WCE */
+#endif /*MMAP_CLEARS */
+#endif  /* WIN32 */
+
+#if defined(DARWIN) || defined(_DARWIN)
+/* Mac OSX docs advise not to use sbrk; it seems better to use mmap */
+#ifndef HAVE_MORECORE
+#define HAVE_MORECORE 0
+#define HAVE_MMAP 1
+/* OSX allocators provide 16 byte alignment */
+#ifndef MALLOC_ALIGNMENT
+#define MALLOC_ALIGNMENT ((size_t)16U)
+#endif
+#endif  /* HAVE_MORECORE */
+#endif  /* DARWIN */
+
+#ifndef LACKS_SYS_TYPES_H
+#include <sys/types.h>  /* For size_t */
+#endif  /* LACKS_SYS_TYPES_H */
+
+/* The maximum possible size_t value has all bits set */
+#define MAX_SIZE_T           (~(size_t)0)
+
+#ifndef USE_LOCKS /* ensure true if spin or recursive locks set */
+#define USE_LOCKS  ((defined(USE_SPIN_LOCKS) && USE_SPIN_LOCKS != 0) || \
+                    (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0))
+#endif /* USE_LOCKS */
+
+#if USE_LOCKS /* Spin locks for gcc >= 4.1, older gcc on x86, MSC >= 1310 */
+#if ((defined(__GNUC__) &&                                              \
+      ((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) ||      \
+       defined(__i386__) || defined(__x86_64__))) ||                    \
+     (defined(_MSC_VER) && _MSC_VER>=1310))
+#ifndef USE_SPIN_LOCKS
+#define USE_SPIN_LOCKS 1
+#endif /* USE_SPIN_LOCKS */
+#elif USE_SPIN_LOCKS
+#error "USE_SPIN_LOCKS defined without implementation"
+#endif /* ... locks available... */
+#elif !defined(USE_SPIN_LOCKS)
+#define USE_SPIN_LOCKS 0
+#endif /* USE_LOCKS */
+
+#ifndef ONLY_MSPACES
+#define ONLY_MSPACES 0
+#endif  /* ONLY_MSPACES */
+#ifndef MSPACES
+#if ONLY_MSPACES
+#define MSPACES 1
+#else   /* ONLY_MSPACES */
+#define MSPACES 0
+#endif  /* ONLY_MSPACES */
+#endif  /* MSPACES */
+#ifndef MALLOC_ALIGNMENT
+#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *)))
+#endif  /* MALLOC_ALIGNMENT */
+#ifndef FOOTERS
+#define FOOTERS 0
+#endif  /* FOOTERS */
+#ifndef ABORT
+#define ABORT  abort()
+#endif  /* ABORT */
+#ifndef ABORT_ON_ASSERT_FAILURE
+#define ABORT_ON_ASSERT_FAILURE 1
+#endif  /* ABORT_ON_ASSERT_FAILURE */
+#ifndef PROCEED_ON_ERROR
+#define PROCEED_ON_ERROR 0
+#endif  /* PROCEED_ON_ERROR */
+
+#ifndef INSECURE
+#define INSECURE 0
+#endif  /* INSECURE */
+#ifndef MALLOC_INSPECT_ALL
+#define MALLOC_INSPECT_ALL 0
+#endif  /* MALLOC_INSPECT_ALL */
+#ifndef HAVE_MMAP
+#define HAVE_MMAP 1
+#endif  /* HAVE_MMAP */
+#ifndef MMAP_CLEARS
+#define MMAP_CLEARS 1
+#endif  /* MMAP_CLEARS */
+#ifndef HAVE_MREMAP
+#ifdef linux
+#define HAVE_MREMAP 1
+#define _GNU_SOURCE /* Turns on mremap() definition */
+#else   /* linux */
+#define HAVE_MREMAP 0
+#endif  /* linux */
+#endif  /* HAVE_MREMAP */
+#ifndef MALLOC_FAILURE_ACTION
+#define MALLOC_FAILURE_ACTION  errno = ENOMEM;
+#endif  /* MALLOC_FAILURE_ACTION */
+#ifndef HAVE_MORECORE
+#if ONLY_MSPACES
+#define HAVE_MORECORE 0
+#else   /* ONLY_MSPACES */
+#define HAVE_MORECORE 1
+#endif  /* ONLY_MSPACES */
+#endif  /* HAVE_MORECORE */
+#if !HAVE_MORECORE
+#define MORECORE_CONTIGUOUS 0
+#else   /* !HAVE_MORECORE */
+#define MORECORE_DEFAULT sbrk
+#ifndef MORECORE_CONTIGUOUS
+#define MORECORE_CONTIGUOUS 1
+#endif  /* MORECORE_CONTIGUOUS */
+#endif  /* HAVE_MORECORE */
+#ifndef DEFAULT_GRANULARITY
+#if (MORECORE_CONTIGUOUS || defined(WIN32))
+#define DEFAULT_GRANULARITY (0)  /* 0 means to compute in init_mparams */
+#else   /* MORECORE_CONTIGUOUS */
+#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U)
+#endif  /* MORECORE_CONTIGUOUS */
+#endif  /* DEFAULT_GRANULARITY */
+#ifndef DEFAULT_TRIM_THRESHOLD
+#ifndef MORECORE_CANNOT_TRIM
+#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U)
+#else   /* MORECORE_CANNOT_TRIM */
+#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T
+#endif  /* MORECORE_CANNOT_TRIM */
+#endif  /* DEFAULT_TRIM_THRESHOLD */
+#ifndef DEFAULT_MMAP_THRESHOLD
+#if HAVE_MMAP
+#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U)
+#else   /* HAVE_MMAP */
+#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
+#endif  /* HAVE_MMAP */
+#endif  /* DEFAULT_MMAP_THRESHOLD */
+#ifndef MAX_RELEASE_CHECK_RATE
+#if HAVE_MMAP
+#define MAX_RELEASE_CHECK_RATE 4095
+#else
+#define MAX_RELEASE_CHECK_RATE MAX_SIZE_T
+#endif /* HAVE_MMAP */
+#endif /* MAX_RELEASE_CHECK_RATE */
+#ifndef USE_BUILTIN_FFS
+#define USE_BUILTIN_FFS 0
+#endif  /* USE_BUILTIN_FFS */
+#ifndef USE_DEV_RANDOM
+#define USE_DEV_RANDOM 0
+#endif  /* USE_DEV_RANDOM */
+#ifndef NO_MALLINFO
+#define NO_MALLINFO 0
+#endif  /* NO_MALLINFO */
+#ifndef MALLINFO_FIELD_TYPE
+#define MALLINFO_FIELD_TYPE size_t
+#endif  /* MALLINFO_FIELD_TYPE */
+#ifndef NO_MALLOC_STATS
+#define NO_MALLOC_STATS 0
+#endif  /* NO_MALLOC_STATS */
+#ifndef NO_SEGMENT_TRAVERSAL
+#define NO_SEGMENT_TRAVERSAL 0
+#endif /* NO_SEGMENT_TRAVERSAL */
+
+/*
+  mallopt tuning options.  SVID/XPG defines four standard parameter
+  numbers for mallopt, normally defined in malloc.h.  None of these
+  are used in this malloc, so setting them has no effect. But this
+  malloc does support the following options.
+*/
+
+#define M_TRIM_THRESHOLD     (-1)
+#define M_GRANULARITY        (-2)
+#define M_MMAP_THRESHOLD     (-3)
+
+/* ------------------------ Mallinfo declarations ------------------------ */
+
+#if !NO_MALLINFO
+/*
+  This version of malloc supports the standard SVID/XPG mallinfo
+  routine that returns a struct containing usage properties and
+  statistics. It should work on any system that has a
+  /usr/include/malloc.h defining struct mallinfo.  The main
+  declaration needed is the mallinfo struct that is returned (by-copy)
+  by mallinfo().  The malloinfo struct contains a bunch of fields that
+  are not even meaningful in this version of malloc.  These fields are
+  are instead filled by mallinfo() with other numbers that might be of
+  interest.
+
+  HAVE_USR_INCLUDE_MALLOC_H should be set if you have a
+  /usr/include/malloc.h file that includes a declaration of struct
+  mallinfo.  If so, it is included; else a compliant version is
+  declared below.  These must be precisely the same for mallinfo() to
+  work.  The original SVID version of this struct, defined on most
+  systems with mallinfo, declares all fields as ints. But some others
+  define as unsigned long. If your system defines the fields using a
+  type of different width than listed here, you MUST #include your
+  system version and #define HAVE_USR_INCLUDE_MALLOC_H.
+*/
+
+/* #define HAVE_USR_INCLUDE_MALLOC_H */
+
+#ifdef HAVE_USR_INCLUDE_MALLOC_H
+#include "/usr/include/malloc.h"
+#else /* HAVE_USR_INCLUDE_MALLOC_H */
+#ifndef STRUCT_MALLINFO_DECLARED
+/* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is defined */
+#define _STRUCT_MALLINFO
+#define STRUCT_MALLINFO_DECLARED 1
+struct mallinfo {
+  MALLINFO_FIELD_TYPE arena;    /* non-mmapped space allocated from system */
+  MALLINFO_FIELD_TYPE ordblks;  /* number of free chunks */
+  MALLINFO_FIELD_TYPE smblks;   /* always 0 */
+  MALLINFO_FIELD_TYPE hblks;    /* always 0 */
+  MALLINFO_FIELD_TYPE hblkhd;   /* space in mmapped regions */
+  MALLINFO_FIELD_TYPE usmblks;  /* maximum total allocated space */
+  MALLINFO_FIELD_TYPE fsmblks;  /* always 0 */
+  MALLINFO_FIELD_TYPE uordblks; /* total allocated space */
+  MALLINFO_FIELD_TYPE fordblks; /* total free space */
+  MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */
+};
+#endif /* STRUCT_MALLINFO_DECLARED */
+#endif /* HAVE_USR_INCLUDE_MALLOC_H */
+#endif /* NO_MALLINFO */
+
+/*
+  Try to persuade compilers to inline. The most critical functions for
+  inlining are defined as macros, so these aren't used for them.
+*/
+
+#ifndef FORCEINLINE
+  #if defined(__GNUC__)
+#define FORCEINLINE __inline __attribute__ ((always_inline))
+  #elif defined(_MSC_VER)
+    #define FORCEINLINE __forceinline
+  #endif
+#endif
+#ifndef NOINLINE
+  #if defined(__GNUC__)
+    #define NOINLINE __attribute__ ((noinline))
+  #elif defined(_MSC_VER)
+    #define NOINLINE __declspec(noinline)
+  #else
+    #define NOINLINE
+  #endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#ifndef FORCEINLINE
+ #define FORCEINLINE inline
+#endif
+#endif /* __cplusplus */
+#ifndef FORCEINLINE
+ #define FORCEINLINE
+#endif
+
+#if !ONLY_MSPACES
+
+/* ------------------- Declarations of public routines ------------------- */
+
+#ifndef USE_DL_PREFIX
+#define dlcalloc               calloc
+#define dlfree                 free
+#define dlmalloc               malloc
+#define dlmemalign             memalign
+#define dlposix_memalign       posix_memalign
+#define dlrealloc              realloc
+#define dlrealloc_in_place     realloc_in_place
+#define dlvalloc               valloc
+#define dlpvalloc              pvalloc
+#define dlmallinfo             mallinfo
+#define dlmallopt              mallopt
+#define dlmalloc_trim          malloc_trim
+#define dlmalloc_stats         malloc_stats
+#define dlmalloc_usable_size   malloc_usable_size
+#define dlmalloc_footprint     malloc_footprint
+#define dlmalloc_max_footprint malloc_max_footprint
+#define dlmalloc_footprint_limit malloc_footprint_limit
+#define dlmalloc_set_footprint_limit malloc_set_footprint_limit
+#define dlmalloc_inspect_all   malloc_inspect_all
+#define dlindependent_calloc   independent_calloc
+#define dlindependent_comalloc independent_comalloc
+#define dlbulk_free            bulk_free
+#endif /* USE_DL_PREFIX */
+
+/*
+  malloc(size_t n)
+  Returns a pointer to a newly allocated chunk of at least n bytes, or
+  null if no space is available, in which case errno is set to ENOMEM
+  on ANSI C systems.
+
+  If n is zero, malloc returns a minimum-sized chunk. (The minimum
+  size is 16 bytes on most 32bit systems, and 32 bytes on 64bit
+  systems.)  Note that size_t is an unsigned type, so calls with
+  arguments that would be negative if signed are interpreted as
+  requests for huge amounts of space, which will often fail. The
+  maximum supported value of n differs across systems, but is in all
+  cases less than the maximum representable value of a size_t.
+*/
+DLMALLOC_EXPORT void* dlmalloc(size_t);
+
+/*
+  free(void* p)
+  Releases the chunk of memory pointed to by p, that had been previously
+  allocated using malloc or a related routine such as realloc.
+  It has no effect if p is null. If p was not malloced or already
+  freed, free(p) will by default cause the current program to abort.
+*/
+DLMALLOC_EXPORT void  dlfree(void*);
+
+/*
+  calloc(size_t n_elements, size_t element_size);
+  Returns a pointer to n_elements * element_size bytes, with all locations
+  set to zero.
+*/
+DLMALLOC_EXPORT void* dlcalloc(size_t, size_t);
+
+/*
+  realloc(void* p, size_t n)
+  Returns a pointer to a chunk of size n that contains the same data
+  as does chunk p up to the minimum of (n, p's size) bytes, or null
+  if no space is available.
+
+  The returned pointer may or may not be the same as p. The algorithm
+  prefers extending p in most cases when possible, otherwise it
+  employs the equivalent of a malloc-copy-free sequence.
+
+  If p is null, realloc is equivalent to malloc.
+
+  If space is not available, realloc returns null, errno is set (if on
+  ANSI) and p is NOT freed.
+
+  if n is for fewer bytes than already held by p, the newly unused
+  space is lopped off and freed if possible.  realloc with a size
+  argument of zero (re)allocates a minimum-sized chunk.
+
+  The old unix realloc convention of allowing the last-free'd chunk
+  to be used as an argument to realloc is not supported.
+*/
+DLMALLOC_EXPORT void* dlrealloc(void*, size_t);
+
+/*
+  realloc_in_place(void* p, size_t n)
+  Resizes the space allocated for p to size n, only if this can be
+  done without moving p (i.e., only if there is adjacent space
+  available if n is greater than p's current allocated size, or n is
+  less than or equal to p's size). This may be used instead of plain
+  realloc if an alternative allocation strategy is needed upon failure
+  to expand space; for example, reallocation of a buffer that must be
+  memory-aligned or cleared. You can use realloc_in_place to trigger
+  these alternatives only when needed.
+
+  Returns p if successful; otherwise null.
+*/
+DLMALLOC_EXPORT void* dlrealloc_in_place(void*, size_t);
+
+/*
+  memalign(size_t alignment, size_t n);
+  Returns a pointer to a newly allocated chunk of n bytes, aligned
+  in accord with the alignment argument.
+
+  The alignment argument should be a power of two. If the argument is
+  not a power of two, the nearest greater power is used.
+  8-byte alignment is guaranteed by normal malloc calls, so don't
+  bother calling memalign with an argument of 8 or less.
+
+  Overreliance on memalign is a sure way to fragment space.
+*/
+DLMALLOC_EXPORT void* dlmemalign(size_t, size_t);
+
+/*
+  int posix_memalign(void** pp, size_t alignment, size_t n);
+  Allocates a chunk of n bytes, aligned in accord with the alignment
+  argument. Differs from memalign only in that it (1) assigns the
+  allocated memory to *pp rather than returning it, (2) fails and
+  returns EINVAL if the alignment is not a power of two (3) fails and
+  returns ENOMEM if memory cannot be allocated.
+*/
+DLMALLOC_EXPORT int dlposix_memalign(void**, size_t, size_t);
+
+/*
+  valloc(size_t n);
+  Equivalent to memalign(pagesize, n), where pagesize is the page
+  size of the system. If the pagesize is unknown, 4096 is used.
+*/
+DLMALLOC_EXPORT void* dlvalloc(size_t);
+
+/*
+  mallopt(int parameter_number, int parameter_value)
+  Sets tunable parameters The format is to provide a
+  (parameter-number, parameter-value) pair.  mallopt then sets the
+  corresponding parameter to the argument value if it can (i.e., so
+  long as the value is meaningful), and returns 1 if successful else
+  0.  To workaround the fact that mallopt is specified to use int,
+  not size_t parameters, the value -1 is specially treated as the
+  maximum unsigned size_t value.
+
+  SVID/XPG/ANSI defines four standard param numbers for mallopt,
+  normally defined in malloc.h.  None of these are use in this malloc,
+  so setting them has no effect. But this malloc also supports other
+  options in mallopt. See below for details.  Briefly, supported
+  parameters are as follows (listed defaults are for "typical"
+  configurations).
+
+  Symbol            param #  default    allowed param values
+  M_TRIM_THRESHOLD     -1   2*1024*1024   any   (-1 disables)
+  M_GRANULARITY        -2     page size   any power of 2 >= page size
+  M_MMAP_THRESHOLD     -3      256*1024   any   (or 0 if no MMAP support)
+*/
+DLMALLOC_EXPORT int dlmallopt(int, int);
+
+/*
+  malloc_footprint();
+  Returns the number of bytes obtained from the system.  The total
+  number of bytes allocated by malloc, realloc etc., is less than this
+  value. Unlike mallinfo, this function returns only a precomputed
+  result, so can be called frequently to monitor memory consumption.
+  Even if locks are otherwise defined, this function does not use them,
+  so results might not be up to date.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_footprint(void);
+
+/*
+  malloc_max_footprint();
+  Returns the maximum number of bytes obtained from the system. This
+  value will be greater than current footprint if deallocated space
+  has been reclaimed by the system. The peak number of bytes allocated
+  by malloc, realloc etc., is less than this value. Unlike mallinfo,
+  this function returns only a precomputed result, so can be called
+  frequently to monitor memory consumption.  Even if locks are
+  otherwise defined, this function does not use them, so results might
+  not be up to date.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_max_footprint(void);
+
+/*
+  malloc_footprint_limit();
+  Returns the number of bytes that the heap is allowed to obtain from
+  the system, returning the last value returned by
+  malloc_set_footprint_limit, or the maximum size_t value if
+  never set. The returned value reflects a permission. There is no
+  guarantee that this number of bytes can actually be obtained from
+  the system.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_footprint_limit();
+
+/*
+  malloc_set_footprint_limit();
+  Sets the maximum number of bytes to obtain from the system, causing
+  failure returns from malloc and related functions upon attempts to
+  exceed this value. The argument value may be subject to page
+  rounding to an enforceable limit; this actual value is returned.
+  Using an argument of the maximum possible size_t effectively
+  disables checks. If the argument is less than or equal to the
+  current malloc_footprint, then all future allocations that require
+  additional system memory will fail. However, invocation cannot
+  retroactively deallocate existing used memory.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_set_footprint_limit(size_t bytes);
+
+#if MALLOC_INSPECT_ALL
+/*
+  malloc_inspect_all(void(*handler)(void *start,
+                                    void *end,
+                                    size_t used_bytes,
+                                    void* callback_arg),
+                      void* arg);
+  Traverses the heap and calls the given handler for each managed
+  region, skipping all bytes that are (or may be) used for bookkeeping
+  purposes.  Traversal does not include include chunks that have been
+  directly memory mapped. Each reported region begins at the start
+  address, and continues up to but not including the end address.  The
+  first used_bytes of the region contain allocated data. If
+  used_bytes is zero, the region is unallocated. The handler is
+  invoked with the given callback argument. If locks are defined, they
+  are held during the entire traversal. It is a bad idea to invoke
+  other malloc functions from within the handler.
+
+  For example, to count the number of in-use chunks with size greater
+  than 1000, you could write:
+  static int count = 0;
+  void count_chunks(void* start, void* end, size_t used, void* arg) {
+    if (used >= 1000) ++count;
+  }
+  then:
+    malloc_inspect_all(count_chunks, NULL);
+
+  malloc_inspect_all is compiled only if MALLOC_INSPECT_ALL is defined.
+*/
+DLMALLOC_EXPORT void dlmalloc_inspect_all(void(*handler)(void*, void *, size_t, void*),
+                           void* arg);
+
+#endif /* MALLOC_INSPECT_ALL */
+
+#if !NO_MALLINFO
+/*
+  mallinfo()
+  Returns (by copy) a struct containing various summary statistics:
+
+  arena:     current total non-mmapped bytes allocated from system
+  ordblks:   the number of free chunks
+  smblks:    always zero.
+  hblks:     current number of mmapped regions
+  hblkhd:    total bytes held in mmapped regions
+  usmblks:   the maximum total allocated space. This will be greater
+                than current total if trimming has occurred.
+  fsmblks:   always zero
+  uordblks:  current total allocated space (normal or mmapped)
+  fordblks:  total free space
+  keepcost:  the maximum number of bytes that could ideally be released
+               back to system via malloc_trim. ("ideally" means that
+               it ignores page restrictions etc.)
+
+  Because these fields are ints, but internal bookkeeping may
+  be kept as longs, the reported values may wrap around zero and
+  thus be inaccurate.
+*/
+DLMALLOC_EXPORT struct mallinfo dlmallinfo(void);
+#endif /* NO_MALLINFO */
+
+/*
+  independent_calloc(size_t n_elements, size_t element_size, void* chunks[]);
+
+  independent_calloc is similar to calloc, but instead of returning a
+  single cleared space, it returns an array of pointers to n_elements
+  independent elements that can hold contents of size elem_size, each
+  of which starts out cleared, and can be independently freed,
+  realloc'ed etc. The elements are guaranteed to be adjacently
+  allocated (this is not guaranteed to occur with multiple callocs or
+  mallocs), which may also improve cache locality in some
+  applications.
+
+  The "chunks" argument is optional (i.e., may be null, which is
+  probably the most typical usage). If it is null, the returned array
+  is itself dynamically allocated and should also be freed when it is
+  no longer needed. Otherwise, the chunks array must be of at least
+  n_elements in length. It is filled in with the pointers to the
+  chunks.
+
+  In either case, independent_calloc returns this pointer array, or
+  null if the allocation failed.  If n_elements is zero and "chunks"
+  is null, it returns a chunk representing an array with zero elements
+  (which should be freed if not wanted).
+
+  Each element must be freed when it is no longer needed. This can be
+  done all at once using bulk_free.
+
+  independent_calloc simplifies and speeds up implementations of many
+  kinds of pools.  It may also be useful when constructing large data
+  structures that initially have a fixed number of fixed-sized nodes,
+  but the number is not known at compile time, and some of the nodes
+  may later need to be freed. For example:
+
+  struct Node { int item; struct Node* next; };
+
+  struct Node* build_list() {
+    struct Node** pool;
+    int n = read_number_of_nodes_needed();
+    if (n <= 0) return 0;
+    pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0);
+    if (pool == 0) die();
+    // organize into a linked list...
+    struct Node* first = pool[0];
+    for (i = 0; i < n-1; ++i)
+      pool[i]->next = pool[i+1];
+    free(pool);     // Can now free the array (or not, if it is needed later)
+    return first;
+  }
+*/
+DLMALLOC_EXPORT void** dlindependent_calloc(size_t, size_t, void**);
+
+/*
+  independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
+
+  independent_comalloc allocates, all at once, a set of n_elements
+  chunks with sizes indicated in the "sizes" array.    It returns
+  an array of pointers to these elements, each of which can be
+  independently freed, realloc'ed etc. The elements are guaranteed to
+  be adjacently allocated (this is not guaranteed to occur with
+  multiple callocs or mallocs), which may also improve cache locality
+  in some applications.
+
+  The "chunks" argument is optional (i.e., may be null). If it is null
+  the returned array is itself dynamically allocated and should also
+  be freed when it is no longer needed. Otherwise, the chunks array
+  must be of at least n_elements in length. It is filled in with the
+  pointers to the chunks.
+
+  In either case, independent_comalloc returns this pointer array, or
+  null if the allocation failed.  If n_elements is zero and chunks is
+  null, it returns a chunk representing an array with zero elements
+  (which should be freed if not wanted).
+
+  Each element must be freed when it is no longer needed. This can be
+  done all at once using bulk_free.
+
+  independent_comallac differs from independent_calloc in that each
+  element may have a different size, and also that it does not
+  automatically clear elements.
+
+  independent_comalloc can be used to speed up allocation in cases
+  where several structs or objects must always be allocated at the
+  same time.  For example:
+
+  struct Head { ... }
+  struct Foot { ... }
+
+  void send_message(char* msg) {
+    int msglen = strlen(msg);
+    size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) };
+    void* chunks[3];
+    if (independent_comalloc(3, sizes, chunks) == 0)
+      die();
+    struct Head* head = (struct Head*)(chunks[0]);
+    char*        body = (char*)(chunks[1]);
+    struct Foot* foot = (struct Foot*)(chunks[2]);
+    // ...
+  }
+
+  In general though, independent_comalloc is worth using only for
+  larger values of n_elements. For small values, you probably won't
+  detect enough difference from series of malloc calls to bother.
+
+  Overuse of independent_comalloc can increase overall memory usage,
+  since it cannot reuse existing noncontiguous small chunks that
+  might be available for some of the elements.
+*/
+DLMALLOC_EXPORT void** dlindependent_comalloc(size_t, size_t*, void**);
+
+/*
+  bulk_free(void* array[], size_t n_elements)
+  Frees and clears (sets to null) each non-null pointer in the given
+  array.  This is likely to be faster than freeing them one-by-one.
+  If footers are used, pointers that have been allocated in different
+  mspaces are not freed or cleared, and the count of all such pointers
+  is returned.  For large arrays of pointers with poor locality, it
+  may be worthwhile to sort this array before calling bulk_free.
+*/
+DLMALLOC_EXPORT size_t  dlbulk_free(void**, size_t n_elements);
+
+/*
+  pvalloc(size_t n);
+  Equivalent to valloc(minimum-page-that-holds(n)), that is,
+  round up n to nearest pagesize.
+ */
+DLMALLOC_EXPORT void*  dlpvalloc(size_t);
+
+/*
+  malloc_trim(size_t pad);
+
+  If possible, gives memory back to the system (via negative arguments
+  to sbrk) if there is unused memory at the `high' end of the malloc
+  pool or in unused MMAP segments. You can call this after freeing
+  large blocks of memory to potentially reduce the system-level memory
+  requirements of a program. However, it cannot guarantee to reduce
+  memory. Under some allocation patterns, some large free blocks of
+  memory will be locked between two used chunks, so they cannot be
+  given back to the system.
+
+  The `pad' argument to malloc_trim represents the amount of free
+  trailing space to leave untrimmed. If this argument is zero, only
+  the minimum amount of memory to maintain internal data structures
+  will be left. Non-zero arguments can be supplied to maintain enough
+  trailing space to service future expected allocations without having
+  to re-obtain memory from the system.
+
+  Malloc_trim returns 1 if it actually released any memory, else 0.
+*/
+DLMALLOC_EXPORT int  dlmalloc_trim(size_t);
+
+/*
+  malloc_stats();
+  Prints on stderr the amount of space obtained from the system (both
+  via sbrk and mmap), the maximum amount (which may be more than
+  current if malloc_trim and/or munmap got called), and the current
+  number of bytes allocated via malloc (or realloc, etc) but not yet
+  freed. Note that this is the number of bytes allocated, not the
+  number requested. It will be larger than the number requested
+  because of alignment and bookkeeping overhead. Because it includes
+  alignment wastage as being in use, this figure may be greater than
+  zero even when no user-level chunks are allocated.
+
+  The reported current and maximum system memory can be inaccurate if
+  a program makes other calls to system memory allocation functions
+  (normally sbrk) outside of malloc.
+
+  malloc_stats prints only the most commonly interesting statistics.
+  More information can be obtained by calling mallinfo.
+*/
+DLMALLOC_EXPORT void  dlmalloc_stats(void);
+
+/*
+  malloc_usable_size(void* p);
+
+  Returns the number of bytes you can actually use in
+  an allocated chunk, which may be more than you requested (although
+  often not) due to alignment and minimum size constraints.
+  You can use this many bytes without worrying about
+  overwriting other allocated objects. This is not a particularly great
+  programming practice. malloc_usable_size can be more useful in
+  debugging and assertions, for example:
+
+  p = malloc(n);
+  assert(malloc_usable_size(p) >= 256);
+*/
+size_t dlmalloc_usable_size(void*);
+
+#endif /* ONLY_MSPACES */
+
+#if MSPACES
+
+/*
+  mspace is an opaque type representing an independent
+  region of space that supports mspace_malloc, etc.
+*/
+typedef void* mspace;
+
+/*
+  create_mspace creates and returns a new independent space with the
+  given initial capacity, or, if 0, the default granularity size.  It
+  returns null if there is no system memory available to create the
+  space.  If argument locked is non-zero, the space uses a separate
+  lock to control access. The capacity of the space will grow
+  dynamically as needed to service mspace_malloc requests.  You can
+  control the sizes of incremental increases of this space by
+  compiling with a different DEFAULT_GRANULARITY or dynamically
+  setting with mallopt(M_GRANULARITY, value).
+*/
+DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked);
+
+/*
+  destroy_mspace destroys the given space, and attempts to return all
+  of its memory back to the system, returning the total number of
+  bytes freed. After destruction, the results of access to all memory
+  used by the space become undefined.
+*/
+DLMALLOC_EXPORT size_t destroy_mspace(mspace msp);
+
+/*
+  create_mspace_with_base uses the memory supplied as the initial base
+  of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this
+  space is used for bookkeeping, so the capacity must be at least this
+  large. (Otherwise 0 is returned.) When this initial space is
+  exhausted, additional memory will be obtained from the system.
+  Destroying this space will deallocate all additionally allocated
+  space (if possible) but not the initial base.
+*/
+DLMALLOC_EXPORT mspace create_mspace_with_base(void* base, size_t capacity, int locked);
+
+/*
+  mspace_track_large_chunks controls whether requests for large chunks
+  are allocated in their own untracked mmapped regions, separate from
+  others in this mspace. By default large chunks are not tracked,
+  which reduces fragmentation. However, such chunks are not
+  necessarily released to the system upon destroy_mspace.  Enabling
+  tracking by setting to true may increase fragmentation, but avoids
+  leakage when relying on destroy_mspace to release all memory
+  allocated using this space.  The function returns the previous
+  setting.
+*/
+DLMALLOC_EXPORT int mspace_track_large_chunks(mspace msp, int enable);
+
+
+/*
+  mspace_malloc behaves as malloc, but operates within
+  the given space.
+*/
+DLMALLOC_EXPORT void* mspace_malloc(mspace msp, size_t bytes);
+
+/*
+  mspace_free behaves as free, but operates within
+  the given space.
+
+  If compiled with FOOTERS==1, mspace_free is not actually needed.
+  free may be called instead of mspace_free because freed chunks from
+  any space are handled by their originating spaces.
+*/
+DLMALLOC_EXPORT void mspace_free(mspace msp, void* mem);
+
+/*
+  mspace_realloc behaves as realloc, but operates within
+  the given space.
+
+  If compiled with FOOTERS==1, mspace_realloc is not actually
+  needed.  realloc may be called instead of mspace_realloc because
+  realloced chunks from any space are handled by their originating
+  spaces.
+*/
+DLMALLOC_EXPORT void* mspace_realloc(mspace msp, void* mem, size_t newsize);
+
+/*
+  mspace_calloc behaves as calloc, but operates within
+  the given space.
+*/
+DLMALLOC_EXPORT void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size);
+
+/*
+  mspace_memalign behaves as memalign, but operates within
+  the given space.
+*/
+DLMALLOC_EXPORT void* mspace_memalign(mspace msp, size_t alignment, size_t bytes);
+
+/*
+  mspace_independent_calloc behaves as independent_calloc, but
+  operates within the given space.
+*/
+DLMALLOC_EXPORT void** mspace_independent_calloc(mspace msp, size_t n_elements,
+                                 size_t elem_size, void* chunks[]);
+
+/*
+  mspace_independent_comalloc behaves as independent_comalloc, but
+  operates within the given space.
+*/
+DLMALLOC_EXPORT void** mspace_independent_comalloc(mspace msp, size_t n_elements,
+                                   size_t sizes[], void* chunks[]);
+
+/*
+  mspace_footprint() returns the number of bytes obtained from the
+  system for this space.
+*/
+DLMALLOC_EXPORT size_t mspace_footprint(mspace msp);
+
+/*
+  mspace_max_footprint() returns the peak number of bytes obtained from the
+  system for this space.
+*/
+DLMALLOC_EXPORT size_t mspace_max_footprint(mspace msp);
+
+
+#if !NO_MALLINFO
+/*
+  mspace_mallinfo behaves as mallinfo, but reports properties of
+  the given space.
+*/
+DLMALLOC_EXPORT struct mallinfo mspace_mallinfo(mspace msp);
+#endif /* NO_MALLINFO */
+
+/*
+  malloc_usable_size(void* p) behaves the same as malloc_usable_size;
+*/
+DLMALLOC_EXPORT size_t mspace_usable_size(const void* mem);
+
+/*
+  mspace_malloc_stats behaves as malloc_stats, but reports
+  properties of the given space.
+*/
+DLMALLOC_EXPORT void mspace_malloc_stats(mspace msp);
+
+/*
+  mspace_trim behaves as malloc_trim, but
+  operates within the given space.
+*/
+DLMALLOC_EXPORT int mspace_trim(mspace msp, size_t pad);
+
+/*
+  An alias for mallopt.
+*/
+DLMALLOC_EXPORT int mspace_mallopt(int, int);
+
+#endif /* MSPACES */
+
+#ifdef __cplusplus
+}  /* end of extern "C" */
+#endif /* __cplusplus */
+
+/*
+  ========================================================================
+  To make a fully customizable malloc.h header file, cut everything
+  above this line, put into file malloc.h, edit to suit, and #include it
+  on the next line, as well as in programs that use this malloc.
+  ========================================================================
+*/
+
+/* #include "malloc.h" */
+
+/*------------------------------ internal #includes ---------------------- */
+
+#ifdef _MSC_VER
+#pragma warning( disable : 4146 ) /* no "unsigned" warnings */
+#endif /* _MSC_VER */
+#if !NO_MALLOC_STATS
+#include <stdio.h>       /* for printing in malloc_stats */
+#endif /* NO_MALLOC_STATS */
+#ifndef LACKS_ERRNO_H
+#include <errno.h>       /* for MALLOC_FAILURE_ACTION */
+#endif /* LACKS_ERRNO_H */
+#ifdef DEBUG
+#if ABORT_ON_ASSERT_FAILURE
+#undef assert
+#define assert(x) if(!(x)) ABORT
+#else /* ABORT_ON_ASSERT_FAILURE */
+#include <assert.h>
+#endif /* ABORT_ON_ASSERT_FAILURE */
+#else  /* DEBUG */
+#ifndef assert
+#define assert(x)
+#endif
+#define DEBUG 0
+#endif /* DEBUG */
+#if !defined(WIN32) && !defined(LACKS_TIME_H)
+#include <time.h>        /* for magic initialization */
+#endif /* WIN32 */
+#ifndef LACKS_STDLIB_H
+#include <stdlib.h>      /* for abort() */
+#endif /* LACKS_STDLIB_H */
+#ifndef LACKS_STRING_H
+#include <string.h>      /* for memset etc */
+#endif  /* LACKS_STRING_H */
+#if USE_BUILTIN_FFS
+#ifndef LACKS_STRINGS_H
+#include <strings.h>     /* for ffs */
+#endif /* LACKS_STRINGS_H */
+#endif /* USE_BUILTIN_FFS */
+#if HAVE_MMAP
+#ifndef LACKS_SYS_MMAN_H
+/* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */
+#if (defined(linux) && !defined(__USE_GNU))
+#define __USE_GNU 1
+#include <sys/mman.h>    /* for mmap */
+#undef __USE_GNU
+#else
+#include <sys/mman.h>    /* for mmap */
+#endif /* linux */
+#endif /* LACKS_SYS_MMAN_H */
+#ifndef LACKS_FCNTL_H
+#include <fcntl.h>
+#endif /* LACKS_FCNTL_H */
+#endif /* HAVE_MMAP */
+#ifndef LACKS_UNISTD_H
+#include <unistd.h>     /* for sbrk, sysconf */
+#else /* LACKS_UNISTD_H */
+#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__)
+extern void*     sbrk(ptrdiff_t);
+#endif /* FreeBSD etc */
+#endif /* LACKS_UNISTD_H */
+
+/* Declarations for locking */
+#if USE_LOCKS
+#ifndef WIN32
+#if defined (__SVR4) && defined (__sun)  /* solaris */
+#include <thread.h>
+#elif !defined(LACKS_SCHED_H)
+#include <sched.h>
+#endif /* solaris or LACKS_SCHED_H */
+#if (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0) || !USE_SPIN_LOCKS
+#include <pthread.h>
+#endif /* USE_RECURSIVE_LOCKS ... */
+#elif defined(_MSC_VER)
+#ifndef _M_AMD64
+/* These are already defined on AMD64 builds */
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+LONG __cdecl _InterlockedCompareExchange(LONG volatile *Dest, LONG Exchange, LONG Comp);
+LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value);
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+#endif /* _M_AMD64 */
+#pragma intrinsic (_InterlockedCompareExchange)
+#pragma intrinsic (_InterlockedExchange)
+#define interlockedcompareexchange _InterlockedCompareExchange
+#define interlockedexchange _InterlockedExchange
+#elif defined(WIN32) && defined(__GNUC__)
+#define interlockedcompareexchange(a, b, c) __sync_val_compare_and_swap(a, c, b)
+#define interlockedexchange __sync_lock_test_and_set
+#endif /* Win32 */
+#else /* USE_LOCKS */
+#endif /* USE_LOCKS */
+
+#ifndef LOCK_AT_FORK
+#define LOCK_AT_FORK 0
+#endif
+
+/* Declarations for bit scanning on win32 */
+#if defined(_MSC_VER) && _MSC_VER>=1300
+#ifndef BitScanForward /* Try to avoid pulling in WinNT.h */
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+unsigned char _BitScanForward(unsigned long *index, unsigned long mask);
+unsigned char _BitScanReverse(unsigned long *index, unsigned long mask);
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#define BitScanForward _BitScanForward
+#define BitScanReverse _BitScanReverse
+#pragma intrinsic(_BitScanForward)
+#pragma intrinsic(_BitScanReverse)
+#endif /* BitScanForward */
+#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */
+
+#ifndef WIN32
+#ifndef malloc_getpagesize
+#  ifdef _SC_PAGESIZE         /* some SVR4 systems omit an underscore */
+#    ifndef _SC_PAGE_SIZE
+#      define _SC_PAGE_SIZE _SC_PAGESIZE
+#    endif
+#  endif
+#  ifdef _SC_PAGE_SIZE
+#    define malloc_getpagesize sysconf(_SC_PAGE_SIZE)
+#  else
+#    if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE)
+       extern size_t getpagesize();
+#      define malloc_getpagesize getpagesize()
+#    else
+#      ifdef WIN32 /* use supplied emulation of getpagesize */
+#        define malloc_getpagesize getpagesize()
+#      else
+#        ifndef LACKS_SYS_PARAM_H
+#          include <sys/param.h>
+#        endif
+#        ifdef EXEC_PAGESIZE
+#          define malloc_getpagesize EXEC_PAGESIZE
+#        else
+#          ifdef NBPG
+#            ifndef CLSIZE
+#              define malloc_getpagesize NBPG
+#            else
+#              define malloc_getpagesize (NBPG * CLSIZE)
+#            endif
+#          else
+#            ifdef NBPC
+#              define malloc_getpagesize NBPC
+#            else
+#              ifdef PAGESIZE
+#                define malloc_getpagesize PAGESIZE
+#              else /* just guess */
+#                define malloc_getpagesize ((size_t)4096U)
+#              endif
+#            endif
+#          endif
+#        endif
+#      endif
+#    endif
+#  endif
+#endif
+#endif
+
+/* ------------------- size_t and alignment properties -------------------- */
+
+/* The byte and bit size of a size_t */
+#define SIZE_T_SIZE         (sizeof(size_t))
+#define SIZE_T_BITSIZE      (sizeof(size_t) << 3)
+
+/* Some constants coerced to size_t */
+/* Annoying but necessary to avoid errors on some platforms */
+#define SIZE_T_ZERO         ((size_t)0)
+#define SIZE_T_ONE          ((size_t)1)
+#define SIZE_T_TWO          ((size_t)2)
+#define SIZE_T_FOUR         ((size_t)4)
+#define TWO_SIZE_T_SIZES    (SIZE_T_SIZE<<1)
+#define FOUR_SIZE_T_SIZES   (SIZE_T_SIZE<<2)
+#define SIX_SIZE_T_SIZES    (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES)
+#define HALF_MAX_SIZE_T     (MAX_SIZE_T / 2U)
+
+/* The bit mask value corresponding to MALLOC_ALIGNMENT */
+#define CHUNK_ALIGN_MASK    (MALLOC_ALIGNMENT - SIZE_T_ONE)
+
+/* True if address a has acceptable alignment */
+#define is_aligned(A)       (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0)
+
+/* the number of bytes to offset an address to align it */
+#define align_offset(A)\
+ ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\
+  ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK))
+
+/* -------------------------- MMAP preliminaries ------------------------- */
+
+/*
+   If HAVE_MORECORE or HAVE_MMAP are false, we just define calls and
+   checks to fail so compiler optimizer can delete code rather than
+   using so many "#if"s.
+*/
+
+
+/* MORECORE and MMAP must return MFAIL on failure */
+#define MFAIL                ((void*)(MAX_SIZE_T))
+#define CMFAIL               ((char*)(MFAIL)) /* defined for convenience */
+
+#if HAVE_MMAP
+
+#ifndef WIN32
+#define MUNMAP_DEFAULT(a, s)  munmap((a), (s))
+#define MMAP_PROT            (PROT_READ|PROT_WRITE)
+#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
+#define MAP_ANONYMOUS        MAP_ANON
+#endif /* MAP_ANON */
+#ifdef MAP_ANONYMOUS
+#define MMAP_FLAGS           (MAP_PRIVATE|MAP_ANONYMOUS)
+#define MMAP_DEFAULT(s)       mmap(0, (s), MMAP_PROT, MMAP_FLAGS, -1, 0)
+#else /* MAP_ANONYMOUS */
+/*
+   Nearly all versions of mmap support MAP_ANONYMOUS, so the following
+   is unlikely to be needed, but is supplied just in case.
+*/
+#define MMAP_FLAGS           (MAP_PRIVATE)
+static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */
+#define MMAP_DEFAULT(s) ((dev_zero_fd < 0) ? \
+           (dev_zero_fd = open("/dev/zero", O_RDWR), \
+            mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) : \
+            mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0))
+#endif /* MAP_ANONYMOUS */
+
+#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s)
+
+#else /* WIN32 */
+
+/* Win32 MMAP via VirtualAlloc */
+static FORCEINLINE void* win32mmap(size_t size) {
+  void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
+  return (ptr != 0)? ptr: MFAIL;
+}
+
+/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
+static FORCEINLINE void* win32direct_mmap(size_t size) {
+  void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN,
+                           PAGE_READWRITE);
+  return (ptr != 0)? ptr: MFAIL;
+}
+
+/* This function supports releasing coalesed segments */
+static FORCEINLINE int win32munmap(void* ptr, size_t size) {
+  MEMORY_BASIC_INFORMATION minfo;
+  char* cptr = (char*)ptr;
+  while (size) {
+    if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0)
+      return -1;
+    if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr ||
+        minfo.State != MEM_COMMIT || minfo.RegionSize > size)
+      return -1;
+    if (VirtualFree(cptr, 0, MEM_RELEASE) == 0)
+      return -1;
+    cptr += minfo.RegionSize;
+    size -= minfo.RegionSize;
+  }
+  return 0;
+}
+
+#define MMAP_DEFAULT(s)             win32mmap(s)
+#define MUNMAP_DEFAULT(a, s)        win32munmap((a), (s))
+#define DIRECT_MMAP_DEFAULT(s)      win32direct_mmap(s)
+#endif /* WIN32 */
+#endif /* HAVE_MMAP */
+
+#if HAVE_MREMAP
+#ifndef WIN32
+#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv))
+#endif /* WIN32 */
+#endif /* HAVE_MREMAP */
+
+/**
+ * Define CALL_MORECORE
+ */
+#if HAVE_MORECORE
+    #ifdef MORECORE
+        #define CALL_MORECORE(S)    MORECORE(S)
+    #else  /* MORECORE */
+        #define CALL_MORECORE(S)    MORECORE_DEFAULT(S)
+    #endif /* MORECORE */
+#else  /* HAVE_MORECORE */
+    #define CALL_MORECORE(S)        MFAIL
+#endif /* HAVE_MORECORE */
+
+/**
+ * Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP
+ */
+#if HAVE_MMAP
+    #define USE_MMAP_BIT            (SIZE_T_ONE)
+
+    #ifdef MMAP
+        #define CALL_MMAP(s)        MMAP(s)
+    #else /* MMAP */
+        #define CALL_MMAP(s)        MMAP_DEFAULT(s)
+    #endif /* MMAP */
+    #ifdef MUNMAP
+        #define CALL_MUNMAP(a, s)   MUNMAP((a), (s))
+    #else /* MUNMAP */
+        #define CALL_MUNMAP(a, s)   MUNMAP_DEFAULT((a), (s))
+    #endif /* MUNMAP */
+    #ifdef DIRECT_MMAP
+        #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
+    #else /* DIRECT_MMAP */
+        #define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s)
+    #endif /* DIRECT_MMAP */
+#else  /* HAVE_MMAP */
+    #define USE_MMAP_BIT            (SIZE_T_ZERO)
+
+    #define MMAP(s)                 MFAIL
+    #define MUNMAP(a, s)            (-1)
+    #define DIRECT_MMAP(s)          MFAIL
+    #define CALL_DIRECT_MMAP(s)     DIRECT_MMAP(s)
+    #define CALL_MMAP(s)            MMAP(s)
+    #define CALL_MUNMAP(a, s)       MUNMAP((a), (s))
+#endif /* HAVE_MMAP */
+
+/**
+ * Define CALL_MREMAP
+ */
+#if HAVE_MMAP && HAVE_MREMAP
+    #ifdef MREMAP
+        #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv))
+    #else /* MREMAP */
+        #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP_DEFAULT((addr), (osz), (nsz), (mv))
+    #endif /* MREMAP */
+#else  /* HAVE_MMAP && HAVE_MREMAP */
+    #define CALL_MREMAP(addr, osz, nsz, mv)     MFAIL
+#endif /* HAVE_MMAP && HAVE_MREMAP */
+
+/* mstate bit set if continguous morecore disabled or failed */
+#define USE_NONCONTIGUOUS_BIT (4U)
+
+/* segment bit set in create_mspace_with_base */
+#define EXTERN_BIT            (8U)
+
+
+/* --------------------------- Lock preliminaries ------------------------ */
+
+/*
+  When locks are defined, there is one global lock, plus
+  one per-mspace lock.
+
+  The global lock_ensures that mparams.magic and other unique
+  mparams values are initialized only once. It also protects
+  sequences of calls to MORECORE.  In many cases sys_alloc requires
+  two calls, that should not be interleaved with calls by other
+  threads.  This does not protect against direct calls to MORECORE
+  by other threads not using this lock, so there is still code to
+  cope the best we can on interference.
+
+  Per-mspace locks surround calls to malloc, free, etc.
+  By default, locks are simple non-reentrant mutexes.
+
+  Because lock-protected regions generally have bounded times, it is
+  OK to use the supplied simple spinlocks. Spinlocks are likely to
+  improve performance for lightly contended applications, but worsen
+  performance under heavy contention.
+
+  If USE_LOCKS is > 1, the definitions of lock routines here are
+  bypassed, in which case you will need to define the type MLOCK_T,
+  and at least INITIAL_LOCK, DESTROY_LOCK, ACQUIRE_LOCK, RELEASE_LOCK
+  and TRY_LOCK.  You must also declare a
+    static MLOCK_T malloc_global_mutex = { initialization values };.
+
+*/
+
+#if !USE_LOCKS
+#define USE_LOCK_BIT               (0U)
+#define INITIAL_LOCK(l)            (0)
+#define DESTROY_LOCK(l)            (0)
+#define ACQUIRE_MALLOC_GLOBAL_LOCK()
+#define RELEASE_MALLOC_GLOBAL_LOCK()
+
+#else
+#if USE_LOCKS > 1
+/* -----------------------  User-defined locks ------------------------ */
+/* Define your own lock implementation here */
+/* #define INITIAL_LOCK(lk)  ... */
+/* #define DESTROY_LOCK(lk)  ... */
+/* #define ACQUIRE_LOCK(lk)  ... */
+/* #define RELEASE_LOCK(lk)  ... */
+/* #define TRY_LOCK(lk) ... */
+/* static MLOCK_T malloc_global_mutex = ... */
+
+#elif USE_SPIN_LOCKS
+
+/* First, define CAS_LOCK and CLEAR_LOCK on ints */
+/* Note CAS_LOCK defined to return 0 on success */
+
+#if defined(__GNUC__)&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))
+#define CAS_LOCK(sl)     __sync_lock_test_and_set(sl, 1)
+#define CLEAR_LOCK(sl)   __sync_lock_release(sl)
+
+#elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)))
+/* Custom spin locks for older gcc on x86 */
+static FORCEINLINE int x86_cas_lock(int *sl) {
+  int ret;
+  int val = 1;
+  int cmp = 0;
+  __asm__ __volatile__  ("lock; cmpxchgl %1, %2"
+                         : "=a" (ret)
+                         : "r" (val), "m" (*(sl)), "0"(cmp)
+                         : "memory", "cc");
+  return ret;
+}
+
+static FORCEINLINE void x86_clear_lock(int* sl) {
+  assert(*sl != 0);
+  int prev = 0;
+  int ret;
+  __asm__ __volatile__ ("lock; xchgl %0, %1"
+                        : "=r" (ret)
+                        : "m" (*(sl)), "0"(prev)
+                        : "memory");
+}
+
+#define CAS_LOCK(sl)     x86_cas_lock(sl)
+#define CLEAR_LOCK(sl)   x86_clear_lock(sl)
+
+#else /* Win32 MSC */
+#define CAS_LOCK(sl)     interlockedexchange(sl, (LONG)1)
+#define CLEAR_LOCK(sl)   interlockedexchange (sl, (LONG)0)
+
+#endif /* ... gcc spins locks ... */
+
+/* How to yield for a spin lock */
+#define SPINS_PER_YIELD       63
+#if defined(_MSC_VER)
+#define SLEEP_EX_DURATION     50 /* delay for yield/sleep */
+#define SPIN_LOCK_YIELD  SleepEx(SLEEP_EX_DURATION, FALSE)
+#elif defined (__SVR4) && defined (__sun) /* solaris */
+#define SPIN_LOCK_YIELD   thr_yield();
+#elif !defined(LACKS_SCHED_H)
+#define SPIN_LOCK_YIELD   sched_yield();
+#else
+#define SPIN_LOCK_YIELD
+#endif /* ... yield ... */
+
+#if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0
+/* Plain spin locks use single word (embedded in malloc_states) */
+static int spin_acquire_lock(int *sl) {
+  int spins = 0;
+  while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) {
+    if ((++spins & SPINS_PER_YIELD) == 0) {
+      SPIN_LOCK_YIELD;
+    }
+  }
+  return 0;
+}
+
+#define MLOCK_T               int
+#define TRY_LOCK(sl)          !CAS_LOCK(sl)
+#define RELEASE_LOCK(sl)      CLEAR_LOCK(sl)
+#define ACQUIRE_LOCK(sl)      (CAS_LOCK(sl)? spin_acquire_lock(sl) : 0)
+#define INITIAL_LOCK(sl)      (*sl = 0)
+#define DESTROY_LOCK(sl)      (0)
+static MLOCK_T malloc_global_mutex = 0;
+
+#else /* USE_RECURSIVE_LOCKS */
+/* types for lock owners */
+#ifdef WIN32
+#define THREAD_ID_T           DWORD
+#define CURRENT_THREAD        GetCurrentThreadId()
+#define EQ_OWNER(X,Y)         ((X) == (Y))
+#else
+/*
+  Note: the following assume that pthread_t is a type that can be
+  initialized to (casted) zero. If this is not the case, you will need to
+  somehow redefine these or not use spin locks.
+*/
+#define THREAD_ID_T           pthread_t
+#define CURRENT_THREAD        pthread_self()
+#define EQ_OWNER(X,Y)         pthread_equal(X, Y)
+#endif
+
+struct malloc_recursive_lock {
+  int sl;
+  unsigned int c;
+  THREAD_ID_T threadid;
+};
+
+#define MLOCK_T  struct malloc_recursive_lock
+static MLOCK_T malloc_global_mutex = { 0, 0, (THREAD_ID_T)0};
+
+static FORCEINLINE void recursive_release_lock(MLOCK_T *lk) {
+  assert(lk->sl != 0);
+  if (--lk->c == 0) {
+    CLEAR_LOCK(&lk->sl);
+  }
+}
+
+static FORCEINLINE int recursive_acquire_lock(MLOCK_T *lk) {
+  THREAD_ID_T mythreadid = CURRENT_THREAD;
+  int spins = 0;
+  for (;;) {
+    if (*((volatile int *)(&lk->sl)) == 0) {
+      if (!CAS_LOCK(&lk->sl)) {
+        lk->threadid = mythreadid;
+        lk->c = 1;
+        return 0;
+      }
+    }
+    else if (EQ_OWNER(lk->threadid, mythreadid)) {
+      ++lk->c;
+      return 0;
+    }
+    if ((++spins & SPINS_PER_YIELD) == 0) {
+      SPIN_LOCK_YIELD;
+    }
+  }
+}
+
+static FORCEINLINE int recursive_try_lock(MLOCK_T *lk) {
+  THREAD_ID_T mythreadid = CURRENT_THREAD;
+  if (*((volatile int *)(&lk->sl)) == 0) {
+    if (!CAS_LOCK(&lk->sl)) {
+      lk->threadid = mythreadid;
+      lk->c = 1;
+      return 1;
+    }
+  }
+  else if (EQ_OWNER(lk->threadid, mythreadid)) {
+    ++lk->c;
+    return 1;
+  }
+  return 0;
+}
+
+#define RELEASE_LOCK(lk)      recursive_release_lock(lk)
+#define TRY_LOCK(lk)          recursive_try_lock(lk)
+#define ACQUIRE_LOCK(lk)      recursive_acquire_lock(lk)
+#define INITIAL_LOCK(lk)      ((lk)->threadid = (THREAD_ID_T)0, (lk)->sl = 0, (lk)->c = 0)
+#define DESTROY_LOCK(lk)      (0)
+#endif /* USE_RECURSIVE_LOCKS */
+
+#elif defined(WIN32) /* Win32 critical sections */
+#define MLOCK_T               CRITICAL_SECTION
+#define ACQUIRE_LOCK(lk)      (EnterCriticalSection(lk), 0)
+#define RELEASE_LOCK(lk)      LeaveCriticalSection(lk)
+#define TRY_LOCK(lk)          TryEnterCriticalSection(lk)
+#define INITIAL_LOCK(lk)      (!InitializeCriticalSectionAndSpinCount((lk), 0x80000000|4000))
+#define DESTROY_LOCK(lk)      (DeleteCriticalSection(lk), 0)
+#define NEED_GLOBAL_LOCK_INIT
+
+static MLOCK_T malloc_global_mutex;
+static volatile LONG malloc_global_mutex_status;
+
+/* Use spin loop to initialize global lock */
+static void init_malloc_global_mutex() {
+  for (;;) {
+    long stat = malloc_global_mutex_status;
+    if (stat > 0)
+      return;
+    /* transition to < 0 while initializing, then to > 0) */
+    if (stat == 0 &&
+        interlockedcompareexchange(&malloc_global_mutex_status, (LONG)-1, (LONG)0) == 0) {
+      InitializeCriticalSection(&malloc_global_mutex);
+      interlockedexchange(&malloc_global_mutex_status, (LONG)1);
+      return;
+    }
+    SleepEx(0, FALSE);
+  }
+}
+
+#else /* pthreads-based locks */
+#define MLOCK_T               pthread_mutex_t
+#define ACQUIRE_LOCK(lk)      pthread_mutex_lock(lk)
+#define RELEASE_LOCK(lk)      pthread_mutex_unlock(lk)
+#define TRY_LOCK(lk)          (!pthread_mutex_trylock(lk))
+#define INITIAL_LOCK(lk)      pthread_init_lock(lk)
+#define DESTROY_LOCK(lk)      pthread_mutex_destroy(lk)
+
+#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 && defined(linux) && !defined(PTHREAD_MUTEX_RECURSIVE)
+/* Cope with old-style linux recursive lock initialization by adding */
+/* skipped internal declaration from pthread.h */
+extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr,
+                                              int __kind));
+#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP
+#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y)
+#endif /* USE_RECURSIVE_LOCKS ... */
+
+static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static int pthread_init_lock (MLOCK_T *lk) {
+  pthread_mutexattr_t attr;
+  if (pthread_mutexattr_init(&attr)) return 1;
+#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0
+  if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1;
+#endif
+  if (pthread_mutex_init(lk, &attr)) return 1;
+  if (pthread_mutexattr_destroy(&attr)) return 1;
+  return 0;
+}
+
+#endif /* ... lock types ... */
+
+/* Common code for all lock types */
+#define USE_LOCK_BIT               (2U)
+
+#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK
+#define ACQUIRE_MALLOC_GLOBAL_LOCK()  ACQUIRE_LOCK(&malloc_global_mutex);
+#endif
+
+#ifndef RELEASE_MALLOC_GLOBAL_LOCK
+#define RELEASE_MALLOC_GLOBAL_LOCK()  RELEASE_LOCK(&malloc_global_mutex);
+#endif
+
+#endif /* USE_LOCKS */
+
+/* -----------------------  Chunk representations ------------------------ */
+
+/*
+  (The following includes lightly edited explanations by Colin Plumb.)
+
+  The malloc_chunk declaration below is misleading (but accurate and
+  necessary).  It declares a "view" into memory allowing access to
+  necessary fields at known offsets from a given base.
+
+  Chunks of memory are maintained using a `boundary tag' method as
+  originally described by Knuth.  (See the paper by Paul Wilson
+  ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a survey of such
+  techniques.)  Sizes of free chunks are stored both in the front of
+  each chunk and at the end.  This makes consolidating fragmented
+  chunks into bigger chunks fast.  The head fields also hold bits
+  representing whether chunks are free or in use.
+
+  Here are some pictures to make it clearer.  They are "exploded" to
+  show that the state of a chunk can be thought of as extending from
+  the high 31 bits of the head field of its header through the
+  prev_foot and PINUSE_BIT bit of the following chunk header.
+
+  A chunk that's in use looks like:
+
+   chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+           | Size of previous chunk (if P = 0)                             |
+           +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
+         | Size of this chunk                                         1| +-+
+   mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |                                                               |
+         +-                                                             -+
+         |                                                               |
+         +-                                                             -+
+         |                                                               :
+         +-      size - sizeof(size_t) available payload bytes          -+
+         :                                                               |
+ chunk-> +-                                                             -+
+         |                                                               |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |1|
+       | Size of next chunk (may or may not be in use)               | +-+
+ mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+    And if it's free, it looks like this:
+
+   chunk-> +-                                                             -+
+           | User payload (must be in use, or we would have merged!)       |
+           +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
+         | Size of this chunk                                         0| +-+
+   mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         | Next pointer                                                  |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         | Prev pointer                                                  |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |                                                               :
+         +-      size - sizeof(struct chunk) unused bytes               -+
+         :                                                               |
+ chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         | Size of this chunk                                            |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |0|
+       | Size of next chunk (must be in use, or we would have merged)| +-+
+ mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+       |                                                               :
+       +- User payload                                                -+
+       :                                                               |
+       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+                                                                     |0|
+                                                                     +-+
+  Note that since we always merge adjacent free chunks, the chunks
+  adjacent to a free chunk must be in use.
+
+  Given a pointer to a chunk (which can be derived trivially from the
+  payload pointer) we can, in O(1) time, find out whether the adjacent
+  chunks are free, and if so, unlink them from the lists that they
+  are on and merge them with the current chunk.
+
+  Chunks always begin on even word boundaries, so the mem portion
+  (which is returned to the user) is also on an even word boundary, and
+  thus at least double-word aligned.
+
+  The P (PINUSE_BIT) bit, stored in the unused low-order bit of the
+  chunk size (which is always a multiple of two words), is an in-use
+  bit for the *previous* chunk.  If that bit is *clear*, then the
+  word before the current chunk size contains the previous chunk
+  size, and can be used to find the front of the previous chunk.
+  The very first chunk allocated always has this bit set, preventing
+  access to non-existent (or non-owned) memory. If pinuse is set for
+  any given chunk, then you CANNOT determine the size of the
+  previous chunk, and might even get a memory addressing fault when
+  trying to do so.
+
+  The C (CINUSE_BIT) bit, stored in the unused second-lowest bit of
+  the chunk size redundantly records whether the current chunk is
+  inuse (unless the chunk is mmapped). This redundancy enables usage
+  checks within free and realloc, and reduces indirection when freeing
+  and consolidating chunks.
+
+  Each freshly allocated chunk must have both cinuse and pinuse set.
+  That is, each allocated chunk borders either a previously allocated
+  and still in-use chunk, or the base of its memory arena. This is
+  ensured by making all allocations from the `lowest' part of any
+  found chunk.  Further, no free chunk physically borders another one,
+  so each free chunk is known to be preceded and followed by either
+  inuse chunks or the ends of memory.
+
+  Note that the `foot' of the current chunk is actually represented
+  as the prev_foot of the NEXT chunk. This makes it easier to
+  deal with alignments etc but can be very confusing when trying
+  to extend or adapt this code.
+
+  The exceptions to all this are
+
+     1. The special chunk `top' is the top-most available chunk (i.e.,
+        the one bordering the end of available memory). It is treated
+        specially.  Top is never included in any bin, is used only if
+        no other chunk is available, and is released back to the
+        system if it is very large (see M_TRIM_THRESHOLD).  In effect,
+        the top chunk is treated as larger (and thus less well
+        fitting) than any other available chunk.  The top chunk
+        doesn't update its trailing size field since there is no next
+        contiguous chunk that would have to index off it. However,
+        space is still allocated for it (TOP_FOOT_SIZE) to enable
+        separation or merging when space is extended.
+
+     3. Chunks allocated via mmap, have both cinuse and pinuse bits
+        cleared in their head fields.  Because they are allocated
+        one-by-one, each must carry its own prev_foot field, which is
+        also used to hold the offset this chunk has within its mmapped
+        region, which is needed to preserve alignment. Each mmapped
+        chunk is trailed by the first two fields of a fake next-chunk
+        for sake of usage checks.
+
+*/
+
+struct malloc_chunk {
+  size_t               prev_foot;  /* Size of previous chunk (if free).  */
+  size_t               head;       /* Size and inuse bits. */
+  struct malloc_chunk* fd;         /* double links -- used only if free. */
+  struct malloc_chunk* bk;
+};
+
+typedef struct malloc_chunk  mchunk;
+typedef struct malloc_chunk* mchunkptr;
+typedef struct malloc_chunk* sbinptr;  /* The type of bins of chunks */
+typedef unsigned int bindex_t;         /* Described below */
+typedef unsigned int binmap_t;         /* Described below */
+typedef unsigned int flag_t;           /* The type of various bit flag sets */
+
+/* ------------------- Chunks sizes and alignments ----------------------- */
+
+#define MCHUNK_SIZE         (sizeof(mchunk))
+
+#if FOOTERS
+#define CHUNK_OVERHEAD      (TWO_SIZE_T_SIZES)
+#else /* FOOTERS */
+#define CHUNK_OVERHEAD      (SIZE_T_SIZE)
+#endif /* FOOTERS */
+
+/* MMapped chunks need a second word of overhead ... */
+#define MMAP_CHUNK_OVERHEAD (TWO_SIZE_T_SIZES)
+/* ... and additional padding for fake next-chunk at foot */
+#define MMAP_FOOT_PAD       (FOUR_SIZE_T_SIZES)
+
+/* The smallest size we can malloc is an aligned minimal chunk */
+#define MIN_CHUNK_SIZE\
+  ((MCHUNK_SIZE + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
+
+/* conversion from malloc headers to user pointers, and back */
+#define chunk2mem(p)        ((void*)((char*)(p)       + TWO_SIZE_T_SIZES))
+#define mem2chunk(mem)      ((mchunkptr)((char*)(mem) - TWO_SIZE_T_SIZES))
+/* chunk associated with aligned address A */
+#define align_as_chunk(A)   (mchunkptr)((A) + align_offset(chunk2mem(A)))
+
+/* Bounds on request (not chunk) sizes. */
+#define MAX_REQUEST         ((-MIN_CHUNK_SIZE) << 2)
+#define MIN_REQUEST         (MIN_CHUNK_SIZE - CHUNK_OVERHEAD - SIZE_T_ONE)
+
+/* pad request bytes into a usable size */
+#define pad_request(req) \
+   (((req) + CHUNK_OVERHEAD + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
+
+/* pad request, checking for minimum (but not maximum) */
+#define request2size(req) \
+  (((req) < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(req))
+
+
+/* ------------------ Operations on head and foot fields ----------------- */
+
+/*
+  The head field of a chunk is or'ed with PINUSE_BIT when previous
+  adjacent chunk in use, and or'ed with CINUSE_BIT if this chunk is in
+  use, unless mmapped, in which case both bits are cleared.
+
+  FLAG4_BIT is not used by this malloc, but might be useful in extensions.
+*/
+
+#define PINUSE_BIT          (SIZE_T_ONE)
+#define CINUSE_BIT          (SIZE_T_TWO)
+#define FLAG4_BIT           (SIZE_T_FOUR)
+#define INUSE_BITS          (PINUSE_BIT|CINUSE_BIT)
+#define FLAG_BITS           (PINUSE_BIT|CINUSE_BIT|FLAG4_BIT)
+
+/* Head value for fenceposts */
+#define FENCEPOST_HEAD      (INUSE_BITS|SIZE_T_SIZE)
+
+/* extraction of fields from head words */
+#define cinuse(p)           ((p)->head & CINUSE_BIT)
+#define pinuse(p)           ((p)->head & PINUSE_BIT)
+#define flag4inuse(p)       ((p)->head & FLAG4_BIT)
+#define is_inuse(p)         (((p)->head & INUSE_BITS) != PINUSE_BIT)
+#define is_mmapped(p)       (((p)->head & INUSE_BITS) == 0)
+
+#define chunksize(p)        ((p)->head & ~(FLAG_BITS))
+
+#define clear_pinuse(p)     ((p)->head &= ~PINUSE_BIT)
+#define set_flag4(p)        ((p)->head |= FLAG4_BIT)
+#define clear_flag4(p)      ((p)->head &= ~FLAG4_BIT)
+
+/* Treat space at ptr +/- offset as a chunk */
+#define chunk_plus_offset(p, s)  ((mchunkptr)(((char*)(p)) + (s)))
+#define chunk_minus_offset(p, s) ((mchunkptr)(((char*)(p)) - (s)))
+
+/* Ptr to next or previous physical malloc_chunk. */
+#define next_chunk(p) ((mchunkptr)( ((char*)(p)) + ((p)->head & ~FLAG_BITS)))
+#define prev_chunk(p) ((mchunkptr)( ((char*)(p)) - ((p)->prev_foot) ))
+
+/* extract next chunk's pinuse bit */
+#define next_pinuse(p)  ((next_chunk(p)->head) & PINUSE_BIT)
+
+/* Get/set size at footer */
+#define get_foot(p, s)  (((mchunkptr)((char*)(p) + (s)))->prev_foot)
+#define set_foot(p, s)  (((mchunkptr)((char*)(p) + (s)))->prev_foot = (s))
+
+/* Set size, pinuse bit, and foot */
+#define set_size_and_pinuse_of_free_chunk(p, s)\
+  ((p)->head = (s|PINUSE_BIT), set_foot(p, s))
+
+/* Set size, pinuse bit, foot, and clear next pinuse */
+#define set_free_with_pinuse(p, s, n)\
+  (clear_pinuse(n), set_size_and_pinuse_of_free_chunk(p, s))
+
+/* Get the internal overhead associated with chunk p */
+#define overhead_for(p)\
+ (is_mmapped(p)? MMAP_CHUNK_OVERHEAD : CHUNK_OVERHEAD)
+
+/* Return true if malloced space is not necessarily cleared */
+#if MMAP_CLEARS
+#define calloc_must_clear(p) (!is_mmapped(p))
+#else /* MMAP_CLEARS */
+#define calloc_must_clear(p) (1)
+#endif /* MMAP_CLEARS */
+
+/* ---------------------- Overlaid data structures ----------------------- */
+
+/*
+  When chunks are not in use, they are treated as nodes of either
+  lists or trees.
+
+  "Small"  chunks are stored in circular doubly-linked lists, and look
+  like this:
+
+    chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Size of previous chunk                            |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    `head:' |             Size of chunk, in bytes                         |P|
+      mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Forward pointer to next chunk in list             |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Back pointer to previous chunk in list            |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Unused space (may be 0 bytes long)                .
+            .                                                               .
+            .                                                               |
+nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    `foot:' |             Size of chunk, in bytes                           |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+  Larger chunks are kept in a form of bitwise digital trees (aka
+  tries) keyed on chunksizes.  Because malloc_tree_chunks are only for
+  free chunks greater than 256 bytes, their size doesn't impose any
+  constraints on user chunk sizes.  Each node looks like:
+
+    chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Size of previous chunk                            |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    `head:' |             Size of chunk, in bytes                         |P|
+      mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Forward pointer to next chunk of same size        |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Back pointer to previous chunk of same size       |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Pointer to left child (child[0])                  |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Pointer to right child (child[1])                 |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Pointer to parent                                 |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             bin index of this chunk                           |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Unused space                                      .
+            .                                                               |
+nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    `foot:' |             Size of chunk, in bytes                           |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+  Each tree holding treenodes is a tree of unique chunk sizes.  Chunks
+  of the same size are arranged in a circularly-linked list, with only
+  the oldest chunk (the next to be used, in our FIFO ordering)
+  actually in the tree.  (Tree members are distinguished by a non-null
+  parent pointer.)  If a chunk with the same size an an existing node
+  is inserted, it is linked off the existing node using pointers that
+  work in the same way as fd/bk pointers of small chunks.
+
+  Each tree contains a power of 2 sized range of chunk sizes (the
+  smallest is 0x100 <= x < 0x180), which is is divided in half at each
+  tree level, with the chunks in the smaller half of the range (0x100
+  <= x < 0x140 for the top nose) in the left subtree and the larger
+  half (0x140 <= x < 0x180) in the right subtree.  This is, of course,
+  done by inspecting individual bits.
+
+  Using these rules, each node's left subtree contains all smaller
+  sizes than its right subtree.  However, the node at the root of each
+  subtree has no particular ordering relationship to either.  (The
+  dividing line between the subtree sizes is based on trie relation.)
+  If we remove the last chunk of a given size from the interior of the
+  tree, we need to replace it with a leaf node.  The tree ordering
+  rules permit a node to be replaced by any leaf below it.
+
+  The smallest chunk in a tree (a common operation in a best-fit
+  allocator) can be found by walking a path to the leftmost leaf in
+  the tree.  Unlike a usual binary tree, where we follow left child
+  pointers until we reach a null, here we follow the right child
+  pointer any time the left one is null, until we reach a leaf with
+  both child pointers null. The smallest chunk in the tree will be
+  somewhere along that path.
+
+  The worst case number of steps to add, find, or remove a node is
+  bounded by the number of bits differentiating chunks within
+  bins. Under current bin calculations, this ranges from 6 up to 21
+  (for 32 bit sizes) or up to 53 (for 64 bit sizes). The typical case
+  is of course much better.
+*/
+
+struct malloc_tree_chunk {
+  /* The first four fields must be compatible with malloc_chunk */
+  size_t                    prev_foot;
+  size_t                    head;
+  struct malloc_tree_chunk* fd;
+  struct malloc_tree_chunk* bk;
+
+  struct malloc_tree_chunk* child[2];
+  struct malloc_tree_chunk* parent;
+  bindex_t                  index;
+};
+
+typedef struct malloc_tree_chunk  tchunk;
+typedef struct malloc_tree_chunk* tchunkptr;
+typedef struct malloc_tree_chunk* tbinptr; /* The type of bins of trees */
+
+/* A little helper macro for trees */
+#define leftmost_child(t) ((t)->child[0] != 0? (t)->child[0] : (t)->child[1])
+
+/* ----------------------------- Segments -------------------------------- */
+
+/*
+  Each malloc space may include non-contiguous segments, held in a
+  list headed by an embedded malloc_segment record representing the
+  top-most space. Segments also include flags holding properties of
+  the space. Large chunks that are directly allocated by mmap are not
+  included in this list. They are instead independently created and
+  destroyed without otherwise keeping track of them.
+
+  Segment management mainly comes into play for spaces allocated by
+  MMAP.  Any call to MMAP might or might not return memory that is
+  adjacent to an existing segment.  MORECORE normally contiguously
+  extends the current space, so this space is almost always adjacent,
+  which is simpler and faster to deal with. (This is why MORECORE is
+  used preferentially to MMAP when both are available -- see
+  sys_alloc.)  When allocating using MMAP, we don't use any of the
+  hinting mechanisms (inconsistently) supported in various
+  implementations of unix mmap, or distinguish reserving from
+  committing memory. Instead, we just ask for space, and exploit
+  contiguity when we get it.  It is probably possible to do
+  better than this on some systems, but no general scheme seems
+  to be significantly better.
+
+  Management entails a simpler variant of the consolidation scheme
+  used for chunks to reduce fragmentation -- new adjacent memory is
+  normally prepended or appended to an existing segment. However,
+  there are limitations compared to chunk consolidation that mostly
+  reflect the fact that segment processing is relatively infrequent
+  (occurring only when getting memory from system) and that we
+  don't expect to have huge numbers of segments:
+
+  * Segments are not indexed, so traversal requires linear scans.  (It
+    would be possible to index these, but is not worth the extra
+    overhead and complexity for most programs on most platforms.)
+  * New segments are only appended to old ones when holding top-most
+    memory; if they cannot be prepended to others, they are held in
+    different segments.
+
+  Except for the top-most segment of an mstate, each segment record
+  is kept at the tail of its segment. Segments are added by pushing
+  segment records onto the list headed by &mstate.seg for the
+  containing mstate.
+
+  Segment flags control allocation/merge/deallocation policies:
+  * If EXTERN_BIT set, then we did not allocate this segment,
+    and so should not try to deallocate or merge with others.
+    (This currently holds only for the initial segment passed
+    into create_mspace_with_base.)
+  * If USE_MMAP_BIT set, the segment may be merged with
+    other surrounding mmapped segments and trimmed/de-allocated
+    using munmap.
+  * If neither bit is set, then the segment was obtained using
+    MORECORE so can be merged with surrounding MORECORE'd segments
+    and deallocated/trimmed using MORECORE with negative arguments.
+*/
+
+struct malloc_segment {
+  char*        base;             /* base address */
+  size_t       size;             /* allocated size */
+  struct malloc_segment* next;   /* ptr to next segment */
+  flag_t       sflags;           /* mmap and extern flag */
+};
+
+#define is_mmapped_segment(S)  ((S)->sflags & USE_MMAP_BIT)
+#define is_extern_segment(S)   ((S)->sflags & EXTERN_BIT)
+
+typedef struct malloc_segment  msegment;
+typedef struct malloc_segment* msegmentptr;
+
+/* ---------------------------- malloc_state ----------------------------- */
+
+/*
+   A malloc_state holds all of the bookkeeping for a space.
+   The main fields are:
+
+  Top
+    The topmost chunk of the currently active segment. Its size is
+    cached in topsize.  The actual size of topmost space is
+    topsize+TOP_FOOT_SIZE, which includes space reserved for adding
+    fenceposts and segment records if necessary when getting more
+    space from the system.  The size at which to autotrim top is
+    cached from mparams in trim_check, except that it is disabled if
+    an autotrim fails.
+
+  Designated victim (dv)
+    This is the preferred chunk for servicing small requests that
+    don't have exact fits.  It is normally the chunk split off most
+    recently to service another small request.  Its size is cached in
+    dvsize. The link fields of this chunk are not maintained since it
+    is not kept in a bin.
+
+  SmallBins
+    An array of bin headers for free chunks.  These bins hold chunks
+    with sizes less than MIN_LARGE_SIZE bytes. Each bin contains
+    chunks of all the same size, spaced 8 bytes apart.  To simplify
+    use in double-linked lists, each bin header acts as a malloc_chunk
+    pointing to the real first node, if it exists (else pointing to
+    itself).  This avoids special-casing for headers.  But to avoid
+    waste, we allocate only the fd/bk pointers of bins, and then use
+    repositioning tricks to treat these as the fields of a chunk.
+
+  TreeBins
+    Treebins are pointers to the roots of trees holding a range of
+    sizes. There are 2 equally spaced treebins for each power of two
+    from TREE_SHIFT to TREE_SHIFT+16. The last bin holds anything
+    larger.
+
+  Bin maps
+    There is one bit map for small bins ("smallmap") and one for
+    treebins ("treemap).  Each bin sets its bit when non-empty, and
+    clears the bit when empty.  Bit operations are then used to avoid
+    bin-by-bin searching -- nearly all "search" is done without ever
+    looking at bins that won't be selected.  The bit maps
+    conservatively use 32 bits per map word, even if on 64bit system.
+    For a good description of some of the bit-based techniques used
+    here, see Henry S. Warren Jr's book "Hacker's Delight" (and
+    supplement at http://hackersdelight.org/). Many of these are
+    intended to reduce the branchiness of paths through malloc etc, as
+    well as to reduce the number of memory locations read or written.
+
+  Segments
+    A list of segments headed by an embedded malloc_segment record
+    representing the initial space.
+
+  Address check support
+    The least_addr field is the least address ever obtained from
+    MORECORE or MMAP. Attempted frees and reallocs of any address less
+    than this are trapped (unless INSECURE is defined).
+
+  Magic tag
+    A cross-check field that should always hold same value as mparams.magic.
+
+  Max allowed footprint
+    The maximum allowed bytes to allocate from system (zero means no limit)
+
+  Flags
+    Bits recording whether to use MMAP, locks, or contiguous MORECORE
+
+  Statistics
+    Each space keeps track of current and maximum system memory
+    obtained via MORECORE or MMAP.
+
+  Trim support
+    Fields holding the amount of unused topmost memory that should trigger
+    trimming, and a counter to force periodic scanning to release unused
+    non-topmost segments.
+
+  Locking
+    If USE_LOCKS is defined, the "mutex" lock is acquired and released
+    around every public call using this mspace.
+
+  Extension support
+    A void* pointer and a size_t field that can be used to help implement
+    extensions to this malloc.
+*/
+
+/* Bin types, widths and sizes */
+#define NSMALLBINS        (32U)
+#define NTREEBINS         (32U)
+#define SMALLBIN_SHIFT    (3U)
+#define SMALLBIN_WIDTH    (SIZE_T_ONE << SMALLBIN_SHIFT)
+#define TREEBIN_SHIFT     (8U)
+#define MIN_LARGE_SIZE    (SIZE_T_ONE << TREEBIN_SHIFT)
+#define MAX_SMALL_SIZE    (MIN_LARGE_SIZE - SIZE_T_ONE)
+#define MAX_SMALL_REQUEST (MAX_SMALL_SIZE - CHUNK_ALIGN_MASK - CHUNK_OVERHEAD)
+
+struct malloc_state {
+  binmap_t   smallmap;
+  binmap_t   treemap;
+  size_t     dvsize;
+  size_t     topsize;
+  char*      least_addr;
+  mchunkptr  dv;
+  mchunkptr  top;
+  size_t     trim_check;
+  size_t     release_checks;
+  size_t     magic;
+  mchunkptr  smallbins[(NSMALLBINS+1)*2];
+  tbinptr    treebins[NTREEBINS];
+  size_t     footprint;
+  size_t     max_footprint;
+  size_t     footprint_limit; /* zero means no limit */
+  flag_t     mflags;
+#if USE_LOCKS
+  MLOCK_T    mutex;     /* locate lock among fields that rarely change */
+#endif /* USE_LOCKS */
+  msegment   seg;
+  void*      extp;      /* Unused but available for extensions */
+  size_t     exts;
+};
+
+typedef struct malloc_state*    mstate;
+
+/* ------------- Global malloc_state and malloc_params ------------------- */
+
+/*
+  malloc_params holds global properties, including those that can be
+  dynamically set using mallopt. There is a single instance, mparams,
+  initialized in init_mparams. Note that the non-zeroness of "magic"
+  also serves as an initialization flag.
+*/
+
+struct malloc_params {
+  size_t magic;
+  size_t page_size;
+  size_t granularity;
+  size_t mmap_threshold;
+  size_t trim_threshold;
+  flag_t default_mflags;
+};
+
+static struct malloc_params mparams;
+
+/* Ensure mparams initialized */
+#define ensure_initialization() (void)(mparams.magic != 0 || init_mparams())
+
+#if !ONLY_MSPACES
+
+/* The global malloc_state used for all non-"mspace" calls */
+static struct malloc_state _gm_;
+#define gm                 (&_gm_)
+#define is_global(M)       ((M) == &_gm_)
+
+#endif /* !ONLY_MSPACES */
+
+#define is_initialized(M)  ((M)->top != 0)
+
+/* -------------------------- system alloc setup ------------------------- */
+
+/* Operations on mflags */
+
+#define use_lock(M)           ((M)->mflags &   USE_LOCK_BIT)
+#define enable_lock(M)        ((M)->mflags |=  USE_LOCK_BIT)
+#if USE_LOCKS
+#define disable_lock(M)       ((M)->mflags &= ~USE_LOCK_BIT)
+#else
+#define disable_lock(M)
+#endif
+
+#define use_mmap(M)           ((M)->mflags &   USE_MMAP_BIT)
+#define enable_mmap(M)        ((M)->mflags |=  USE_MMAP_BIT)
+#if HAVE_MMAP
+#define disable_mmap(M)       ((M)->mflags &= ~USE_MMAP_BIT)
+#else
+#define disable_mmap(M)
+#endif
+
+#define use_noncontiguous(M)  ((M)->mflags &   USE_NONCONTIGUOUS_BIT)
+#define disable_contiguous(M) ((M)->mflags |=  USE_NONCONTIGUOUS_BIT)
+
+#define set_lock(M,L)\
+ ((M)->mflags = (L)?\
+  ((M)->mflags | USE_LOCK_BIT) :\
+  ((M)->mflags & ~USE_LOCK_BIT))
+
+/* page-align a size */
+#define page_align(S)\
+ (((S) + (mparams.page_size - SIZE_T_ONE)) & ~(mparams.page_size - SIZE_T_ONE))
+
+/* granularity-align a size */
+#define granularity_align(S)\
+  (((S) + (mparams.granularity - SIZE_T_ONE))\
+   & ~(mparams.granularity - SIZE_T_ONE))
+
+
+/* For mmap, use granularity alignment on windows, else page-align */
+#ifdef WIN32
+#define mmap_align(S) granularity_align(S)
+#else
+#define mmap_align(S) page_align(S)
+#endif
+
+/* For sys_alloc, enough padding to ensure can malloc request on success */
+#define SYS_ALLOC_PADDING (TOP_FOOT_SIZE + MALLOC_ALIGNMENT)
+
+#define is_page_aligned(S)\
+   (((size_t)(S) & (mparams.page_size - SIZE_T_ONE)) == 0)
+#define is_granularity_aligned(S)\
+   (((size_t)(S) & (mparams.granularity - SIZE_T_ONE)) == 0)
+
+/*  True if segment S holds address A */
+#define segment_holds(S, A)\
+  ((char*)(A) >= S->base && (char*)(A) < S->base + S->size)
+
+/* Return segment holding given address */
+static msegmentptr segment_holding(mstate m, char* addr) {
+  msegmentptr sp = &m->seg;
+  for (;;) {
+    if (addr >= sp->base && addr < sp->base + sp->size)
+      return sp;
+    if ((sp = sp->next) == 0)
+      return 0;
+  }
+}
+
+/* Return true if segment contains a segment link */
+static int has_segment_link(mstate m, msegmentptr ss) {
+  msegmentptr sp = &m->seg;
+  for (;;) {
+    if ((char*)sp >= ss->base && (char*)sp < ss->base + ss->size)
+      return 1;
+    if ((sp = sp->next) == 0)
+      return 0;
+  }
+}
+
+#ifndef MORECORE_CANNOT_TRIM
+#define should_trim(M,s)  ((s) > (M)->trim_check)
+#else  /* MORECORE_CANNOT_TRIM */
+#define should_trim(M,s)  (0)
+#endif /* MORECORE_CANNOT_TRIM */
+
+/*
+  TOP_FOOT_SIZE is padding at the end of a segment, including space
+  that may be needed to place segment records and fenceposts when new
+  noncontiguous segments are added.
+*/
+#define TOP_FOOT_SIZE\
+  (align_offset(chunk2mem(0))+pad_request(sizeof(struct malloc_segment))+MIN_CHUNK_SIZE)
+
+
+/* -------------------------------  Hooks -------------------------------- */
+
+/*
+  PREACTION should be defined to return 0 on success, and nonzero on
+  failure. If you are not using locking, you can redefine these to do
+  anything you like.
+*/
+
+#if USE_LOCKS
+#define PREACTION(M)  ((use_lock(M))? ACQUIRE_LOCK(&(M)->mutex) : 0)
+#define POSTACTION(M) { if (use_lock(M)) RELEASE_LOCK(&(M)->mutex); }
+#else /* USE_LOCKS */
+
+#ifndef PREACTION
+#define PREACTION(M) (0)
+#endif  /* PREACTION */
+
+#ifndef POSTACTION
+#define POSTACTION(M)
+#endif  /* POSTACTION */
+
+#endif /* USE_LOCKS */
+
+/*
+  CORRUPTION_ERROR_ACTION is triggered upon detected bad addresses.
+  USAGE_ERROR_ACTION is triggered on detected bad frees and
+  reallocs. The argument p is an address that might have triggered the
+  fault. It is ignored by the two predefined actions, but might be
+  useful in custom actions that try to help diagnose errors.
+*/
+
+#if PROCEED_ON_ERROR
+
+/* A count of the number of corruption errors causing resets */
+int malloc_corruption_error_count;
+
+/* default corruption action */
+static void reset_on_error(mstate m);
+
+#define CORRUPTION_ERROR_ACTION(m)  reset_on_error(m)
+#define USAGE_ERROR_ACTION(m, p)
+
+#else /* PROCEED_ON_ERROR */
+
+#ifndef CORRUPTION_ERROR_ACTION
+#define CORRUPTION_ERROR_ACTION(m) ABORT
+#endif /* CORRUPTION_ERROR_ACTION */
+
+#ifndef USAGE_ERROR_ACTION
+#define USAGE_ERROR_ACTION(m,p) ABORT
+#endif /* USAGE_ERROR_ACTION */
+
+#endif /* PROCEED_ON_ERROR */
+
+
+/* -------------------------- Debugging setup ---------------------------- */
+
+#if ! DEBUG
+
+#define check_free_chunk(M,P)
+#define check_inuse_chunk(M,P)
+#define check_malloced_chunk(M,P,N)
+#define check_mmapped_chunk(M,P)
+#define check_malloc_state(M)
+#define check_top_chunk(M,P)
+
+#else /* DEBUG */
+#define check_free_chunk(M,P)       do_check_free_chunk(M,P)
+#define check_inuse_chunk(M,P)      do_check_inuse_chunk(M,P)
+#define check_top_chunk(M,P)        do_check_top_chunk(M,P)
+#define check_malloced_chunk(M,P,N) do_check_malloced_chunk(M,P,N)
+#define check_mmapped_chunk(M,P)    do_check_mmapped_chunk(M,P)
+#define check_malloc_state(M)       do_check_malloc_state(M)
+
+static void   do_check_any_chunk(mstate m, mchunkptr p);
+static void   do_check_top_chunk(mstate m, mchunkptr p);
+static void   do_check_mmapped_chunk(mstate m, mchunkptr p);
+static void   do_check_inuse_chunk(mstate m, mchunkptr p);
+static void   do_check_free_chunk(mstate m, mchunkptr p);
+static void   do_check_malloced_chunk(mstate m, void* mem, size_t s);
+static void   do_check_tree(mstate m, tchunkptr t);
+static void   do_check_treebin(mstate m, bindex_t i);
+static void   do_check_smallbin(mstate m, bindex_t i);
+static void   do_check_malloc_state(mstate m);
+static int    bin_find(mstate m, mchunkptr x);
+static size_t traverse_and_check(mstate m);
+#endif /* DEBUG */
+
+/* ---------------------------- Indexing Bins ---------------------------- */
+
+#define is_small(s)         (((s) >> SMALLBIN_SHIFT) < NSMALLBINS)
+#define small_index(s)      (bindex_t)((s)  >> SMALLBIN_SHIFT)
+#define small_index2size(i) ((i)  << SMALLBIN_SHIFT)
+#define MIN_SMALL_INDEX     (small_index(MIN_CHUNK_SIZE))
+
+/* addressing by index. See above about smallbin repositioning */
+#define smallbin_at(M, i)   ((sbinptr)((char*)&((M)->smallbins[(i)<<1])))
+#define treebin_at(M,i)     (&((M)->treebins[i]))
+
+/* assign tree index for size S to variable I. Use x86 asm if possible  */
+#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
+#define compute_tree_index(S, I)\
+{\
+  unsigned int X = S >> TREEBIN_SHIFT;\
+  if (X == 0)\
+    I = 0;\
+  else if (X > 0xFFFF)\
+    I = NTREEBINS-1;\
+  else {\
+    unsigned int K = (unsigned) sizeof(X)*__CHAR_BIT__ - 1 - (unsigned) __builtin_clz(X); \
+    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
+  }\
+}
+
+#elif defined (__INTEL_COMPILER)
+#define compute_tree_index(S, I)\
+{\
+  size_t X = S >> TREEBIN_SHIFT;\
+  if (X == 0)\
+    I = 0;\
+  else if (X > 0xFFFF)\
+    I = NTREEBINS-1;\
+  else {\
+    unsigned int K = _bit_scan_reverse (X); \
+    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
+  }\
+}
+
+#elif defined(_MSC_VER) && _MSC_VER>=1300
+#define compute_tree_index(S, I)\
+{\
+  size_t X = S >> TREEBIN_SHIFT;\
+  if (X == 0)\
+    I = 0;\
+  else if (X > 0xFFFF)\
+    I = NTREEBINS-1;\
+  else {\
+    unsigned int K;\
+    _BitScanReverse((DWORD *) &K, (DWORD) X);\
+    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
+  }\
+}
+
+#else /* GNUC */
+#define compute_tree_index(S, I)\
+{\
+  size_t X = S >> TREEBIN_SHIFT;\
+  if (X == 0)\
+    I = 0;\
+  else if (X > 0xFFFF)\
+    I = NTREEBINS-1;\
+  else {\
+    unsigned int Y = (unsigned int)X;\
+    unsigned int N = ((Y - 0x100) >> 16) & 8;\
+    unsigned int K = (((Y <<= N) - 0x1000) >> 16) & 4;\
+    N += K;\
+    N += K = (((Y <<= K) - 0x4000) >> 16) & 2;\
+    K = 14 - N + ((Y <<= K) >> 15);\
+    I = (K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1));\
+  }\
+}
+#endif /* GNUC */
+
+/* Bit representing maximum resolved size in a treebin at i */
+#define bit_for_tree_index(i) \
+   (i == NTREEBINS-1)? (SIZE_T_BITSIZE-1) : (((i) >> 1) + TREEBIN_SHIFT - 2)
+
+/* Shift placing maximum resolved bit in a treebin at i as sign bit */
+#define leftshift_for_tree_index(i) \
+   ((i == NTREEBINS-1)? 0 : \
+    ((SIZE_T_BITSIZE-SIZE_T_ONE) - (((i) >> 1) + TREEBIN_SHIFT - 2)))
+
+/* The size of the smallest chunk held in bin with index i */
+#define minsize_for_tree_index(i) \
+   ((SIZE_T_ONE << (((i) >> 1) + TREEBIN_SHIFT)) |  \
+   (((size_t)((i) & SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1)))
+
+
+/* ------------------------ Operations on bin maps ----------------------- */
+
+/* bit corresponding to given index */
+#define idx2bit(i)              ((binmap_t)(1) << (i))
+
+/* Mark/Clear bits with given index */
+#define mark_smallmap(M,i)      ((M)->smallmap |=  idx2bit(i))
+#define clear_smallmap(M,i)     ((M)->smallmap &= ~idx2bit(i))
+#define smallmap_is_marked(M,i) ((M)->smallmap &   idx2bit(i))
+
+#define mark_treemap(M,i)       ((M)->treemap  |=  idx2bit(i))
+#define clear_treemap(M,i)      ((M)->treemap  &= ~idx2bit(i))
+#define treemap_is_marked(M,i)  ((M)->treemap  &   idx2bit(i))
+
+/* isolate the least set bit of a bitmap */
+#define least_bit(x)         ((x) & -(x))
+
+/* mask with all bits to left of least bit of x on */
+#define left_bits(x)         ((x<<1) | -(x<<1))
+
+/* mask with all bits to left of or equal to least bit of x on */
+#define same_or_left_bits(x) ((x) | -(x))
+
+/* index corresponding to given bit. Use x86 asm if possible */
+
+#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
+#define compute_bit2idx(X, I)\
+{\
+  unsigned int J;\
+  J = __builtin_ctz(X); \
+  I = (bindex_t)J;\
+}
+
+#elif defined (__INTEL_COMPILER)
+#define compute_bit2idx(X, I)\
+{\
+  unsigned int J;\
+  J = _bit_scan_forward (X); \
+  I = (bindex_t)J;\
+}
+
+#elif defined(_MSC_VER) && _MSC_VER>=1300
+#define compute_bit2idx(X, I)\
+{\
+  unsigned int J;\
+  _BitScanForward((DWORD *) &J, X);\
+  I = (bindex_t)J;\
+}
+
+#elif USE_BUILTIN_FFS
+#define compute_bit2idx(X, I) I = ffs(X)-1
+
+#else
+#define compute_bit2idx(X, I)\
+{\
+  unsigned int Y = X - 1;\
+  unsigned int K = Y >> (16-4) & 16;\
+  unsigned int N = K;        Y >>= K;\
+  N += K = Y >> (8-3) &  8;  Y >>= K;\
+  N += K = Y >> (4-2) &  4;  Y >>= K;\
+  N += K = Y >> (2-1) &  2;  Y >>= K;\
+  N += K = Y >> (1-0) &  1;  Y >>= K;\
+  I = (bindex_t)(N + Y);\
+}
+#endif /* GNUC */
+
+
+/* ----------------------- Runtime Check Support ------------------------- */
+
+/*
+  For security, the main invariant is that malloc/free/etc never
+  writes to a static address other than malloc_state, unless static
+  malloc_state itself has been corrupted, which cannot occur via
+  malloc (because of these checks). In essence this means that we
+  believe all pointers, sizes, maps etc held in malloc_state, but
+  check all of those linked or offsetted from other embedded data
+  structures.  These checks are interspersed with main code in a way
+  that tends to minimize their run-time cost.
+
+  When FOOTERS is defined, in addition to range checking, we also
+  verify footer fields of inuse chunks, which can be used guarantee
+  that the mstate controlling malloc/free is intact.  This is a
+  streamlined version of the approach described by William Robertson
+  et al in "Run-time Detection of Heap-based Overflows" LISA'03
+  http://www.usenix.org/events/lisa03/tech/robertson.html The footer
+  of an inuse chunk holds the xor of its mstate and a random seed,
+  that is checked upon calls to free() and realloc().  This is
+  (probabalistically) unguessable from outside the program, but can be
+  computed by any code successfully malloc'ing any chunk, so does not
+  itself provide protection against code that has already broken
+  security through some other means.  Unlike Robertson et al, we
+  always dynamically check addresses of all offset chunks (previous,
+  next, etc). This turns out to be cheaper than relying on hashes.
+*/
+
+#if !INSECURE
+/* Check if address a is at least as high as any from MORECORE or MMAP */
+#define ok_address(M, a) ((char*)(a) >= (M)->least_addr)
+/* Check if address of next chunk n is higher than base chunk p */
+#define ok_next(p, n)    ((char*)(p) < (char*)(n))
+/* Check if p has inuse status */
+#define ok_inuse(p)     is_inuse(p)
+/* Check if p has its pinuse bit on */
+#define ok_pinuse(p)     pinuse(p)
+
+#else /* !INSECURE */
+#define ok_address(M, a) (1)
+#define ok_next(b, n)    (1)
+#define ok_inuse(p)      (1)
+#define ok_pinuse(p)     (1)
+#endif /* !INSECURE */
+
+#if (FOOTERS && !INSECURE)
+/* Check if (alleged) mstate m has expected magic field */
+#define ok_magic(M)      ((M)->magic == mparams.magic)
+#else  /* (FOOTERS && !INSECURE) */
+#define ok_magic(M)      (1)
+#endif /* (FOOTERS && !INSECURE) */
+
+/* In gcc, use __builtin_expect to minimize impact of checks */
+#if !INSECURE
+#if defined(__GNUC__) && __GNUC__ >= 3
+#define RTCHECK(e)  __builtin_expect(e, 1)
+#else /* GNUC */
+#define RTCHECK(e)  (e)
+#endif /* GNUC */
+#else /* !INSECURE */
+#define RTCHECK(e)  (1)
+#endif /* !INSECURE */
+
+/* macros to set up inuse chunks with or without footers */
+
+#if !FOOTERS
+
+#define mark_inuse_foot(M,p,s)
+
+/* Macros for setting head/foot of non-mmapped chunks */
+
+/* Set cinuse bit and pinuse bit of next chunk */
+#define set_inuse(M,p,s)\
+  ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\
+  ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT)
+
+/* Set cinuse and pinuse of this chunk and pinuse of next chunk */
+#define set_inuse_and_pinuse(M,p,s)\
+  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
+  ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT)
+
+/* Set size, cinuse and pinuse bit of this chunk */
+#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\
+  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT))
+
+#else /* FOOTERS */
+
+/* Set foot of inuse chunk to be xor of mstate and seed */
+#define mark_inuse_foot(M,p,s)\
+  (((mchunkptr)((char*)(p) + (s)))->prev_foot = ((size_t)(M) ^ mparams.magic))
+
+#define get_mstate_for(p)\
+  ((mstate)(((mchunkptr)((char*)(p) +\
+    (chunksize(p))))->prev_foot ^ mparams.magic))
+
+#define set_inuse(M,p,s)\
+  ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\
+  (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT), \
+  mark_inuse_foot(M,p,s))
+
+#define set_inuse_and_pinuse(M,p,s)\
+  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
+  (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT),\
+ mark_inuse_foot(M,p,s))
+
+#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\
+  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
+  mark_inuse_foot(M, p, s))
+
+#endif /* !FOOTERS */
+
+/* ---------------------------- setting mparams -------------------------- */
+
+#if LOCK_AT_FORK
+static void pre_fork(void)         { ACQUIRE_LOCK(&(gm)->mutex); }
+static void post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
+static void post_fork_child(void)  { INITIAL_LOCK(&(gm)->mutex); }
+#endif /* LOCK_AT_FORK */
+
+/* Initialize mparams */
+static int init_mparams(void) {
+#ifdef NEED_GLOBAL_LOCK_INIT
+  if (malloc_global_mutex_status <= 0)
+    init_malloc_global_mutex();
+#endif
+
+  ACQUIRE_MALLOC_GLOBAL_LOCK();
+  if (mparams.magic == 0) {
+    size_t magic;
+    size_t psize;
+    size_t gsize;
+
+#ifndef WIN32
+    psize = malloc_getpagesize;
+    gsize = ((DEFAULT_GRANULARITY != 0)? DEFAULT_GRANULARITY : psize);
+#else /* WIN32 */
+    {
+      SYSTEM_INFO system_info;
+      GetSystemInfo(&system_info);
+      psize = system_info.dwPageSize;
+      gsize = ((DEFAULT_GRANULARITY != 0)?
+               DEFAULT_GRANULARITY : system_info.dwAllocationGranularity);
+    }
+#endif /* WIN32 */
+
+    /* Sanity-check configuration:
+       size_t must be unsigned and as wide as pointer type.
+       ints must be at least 4 bytes.
+       alignment must be at least 8.
+       Alignment, min chunk size, and page size must all be powers of 2.
+    */
+    if ((sizeof(size_t) != sizeof(char*)) ||
+        (MAX_SIZE_T < MIN_CHUNK_SIZE)  ||
+        (sizeof(int) < 4)  ||
+        (MALLOC_ALIGNMENT < (size_t)8U) ||
+        ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT-SIZE_T_ONE)) != 0) ||
+        ((MCHUNK_SIZE      & (MCHUNK_SIZE-SIZE_T_ONE))      != 0) ||
+        ((gsize            & (gsize-SIZE_T_ONE))            != 0) ||
+        ((psize            & (psize-SIZE_T_ONE))            != 0))
+      ABORT;
+    mparams.granularity = gsize;
+    mparams.page_size = psize;
+    mparams.mmap_threshold = DEFAULT_MMAP_THRESHOLD;
+    mparams.trim_threshold = DEFAULT_TRIM_THRESHOLD;
+#if MORECORE_CONTIGUOUS
+    mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT;
+#else  /* MORECORE_CONTIGUOUS */
+    mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT|USE_NONCONTIGUOUS_BIT;
+#endif /* MORECORE_CONTIGUOUS */
+
+#if !ONLY_MSPACES
+    /* Set up lock for main malloc area */
+    gm->mflags = mparams.default_mflags;
+    (void)INITIAL_LOCK(&gm->mutex);
+#endif
+#if LOCK_AT_FORK
+    pthread_atfork(&pre_fork, &post_fork_parent, &post_fork_child);
+#endif
+
+    {
+#if USE_DEV_RANDOM
+      int fd;
+      unsigned char buf[sizeof(size_t)];
+      /* Try to use /dev/urandom, else fall back on using time */
+      if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 &&
+          read(fd, buf, sizeof(buf)) == sizeof(buf)) {
+        magic = *((size_t *) buf);
+        close(fd);
+      }
+      else
+#endif /* USE_DEV_RANDOM */
+#ifdef WIN32
+      magic = (size_t)(GetTickCount() ^ (size_t)0x55555555U);
+#elif defined(LACKS_TIME_H)
+      magic = (size_t)&magic ^ (size_t)0x55555555U;
+#else
+      magic = (size_t)(time(0) ^ (size_t)0x55555555U);
+#endif
+      magic |= (size_t)8U;    /* ensure nonzero */
+      magic &= ~(size_t)7U;   /* improve chances of fault for bad values */
+      /* Until memory modes commonly available, use volatile-write */
+      (*(volatile size_t *)(&(mparams.magic))) = magic;
+    }
+  }
+
+  RELEASE_MALLOC_GLOBAL_LOCK();
+  return 1;
+}
+
+/* support for mallopt */
+static int change_mparam(int param_number, int value) {
+  size_t val;
+  ensure_initialization();
+  val = (value == -1)? MAX_SIZE_T : (size_t)value;
+  switch(param_number) {
+  case M_TRIM_THRESHOLD:
+    mparams.trim_threshold = val;
+    return 1;
+  case M_GRANULARITY:
+    if (val >= mparams.page_size && ((val & (val-1)) == 0)) {
+      mparams.granularity = val;
+      return 1;
+    }
+    else
+      return 0;
+  case M_MMAP_THRESHOLD:
+    mparams.mmap_threshold = val;
+    return 1;
+  default:
+    return 0;
+  }
+}
+
+#if DEBUG
+/* ------------------------- Debugging Support --------------------------- */
+
+/* Check properties of any chunk, whether free, inuse, mmapped etc  */
+static void do_check_any_chunk(mstate m, mchunkptr p) {
+  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
+  assert(ok_address(m, p));
+}
+
+/* Check properties of top chunk */
+static void do_check_top_chunk(mstate m, mchunkptr p) {
+  msegmentptr sp = segment_holding(m, (char*)p);
+  size_t  sz = p->head & ~INUSE_BITS; /* third-lowest bit can be set! */
+  assert(sp != 0);
+  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
+  assert(ok_address(m, p));
+  assert(sz == m->topsize);
+  assert(sz > 0);
+  assert(sz == ((sp->base + sp->size) - (char*)p) - TOP_FOOT_SIZE);
+  assert(pinuse(p));
+  assert(!pinuse(chunk_plus_offset(p, sz)));
+}
+
+/* Check properties of (inuse) mmapped chunks */
+static void do_check_mmapped_chunk(mstate m, mchunkptr p) {
+  size_t  sz = chunksize(p);
+  size_t len = (sz + (p->prev_foot) + MMAP_FOOT_PAD);
+  assert(is_mmapped(p));
+  assert(use_mmap(m));
+  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
+  assert(ok_address(m, p));
+  assert(!is_small(sz));
+  assert((len & (mparams.page_size-SIZE_T_ONE)) == 0);
+  assert(chunk_plus_offset(p, sz)->head == FENCEPOST_HEAD);
+  assert(chunk_plus_offset(p, sz+SIZE_T_SIZE)->head == 0);
+}
+
+/* Check properties of inuse chunks */
+static void do_check_inuse_chunk(mstate m, mchunkptr p) {
+  do_check_any_chunk(m, p);
+  assert(is_inuse(p));
+  assert(next_pinuse(p));
+  /* If not pinuse and not mmapped, previous chunk has OK offset */
+  assert(is_mmapped(p) || pinuse(p) || next_chunk(prev_chunk(p)) == p);
+  if (is_mmapped(p))
+    do_check_mmapped_chunk(m, p);
+}
+
+/* Check properties of free chunks */
+static void do_check_free_chunk(mstate m, mchunkptr p) {
+  size_t sz = chunksize(p);
+  mchunkptr next = chunk_plus_offset(p, sz);
+  do_check_any_chunk(m, p);
+  assert(!is_inuse(p));
+  assert(!next_pinuse(p));
+  assert (!is_mmapped(p));
+  if (p != m->dv && p != m->top) {
+    if (sz >= MIN_CHUNK_SIZE) {
+      assert((sz & CHUNK_ALIGN_MASK) == 0);
+      assert(is_aligned(chunk2mem(p)));
+      assert(next->prev_foot == sz);
+      assert(pinuse(p));
+      assert (next == m->top || is_inuse(next));
+      assert(p->fd->bk == p);
+      assert(p->bk->fd == p);
+    }
+    else  /* markers are always of size SIZE_T_SIZE */
+      assert(sz == SIZE_T_SIZE);
+  }
+}
+
+/* Check properties of malloced chunks at the point they are malloced */
+static void do_check_malloced_chunk(mstate m, void* mem, size_t s) {
+  if (mem != 0) {
+    mchunkptr p = mem2chunk(mem);
+    size_t sz = p->head & ~INUSE_BITS;
+    do_check_inuse_chunk(m, p);
+    assert((sz & CHUNK_ALIGN_MASK) == 0);
+    assert(sz >= MIN_CHUNK_SIZE);
+    assert(sz >= s);
+    /* unless mmapped, size is less than MIN_CHUNK_SIZE more than request */
+    assert(is_mmapped(p) || sz < (s + MIN_CHUNK_SIZE));
+  }
+}
+
+/* Check a tree and its subtrees.  */
+static void do_check_tree(mstate m, tchunkptr t) {
+  tchunkptr head = 0;
+  tchunkptr u = t;
+  bindex_t tindex = t->index;
+  size_t tsize = chunksize(t);
+  bindex_t idx;
+  compute_tree_index(tsize, idx);
+  assert(tindex == idx);
+  assert(tsize >= MIN_LARGE_SIZE);
+  assert(tsize >= minsize_for_tree_index(idx));
+  assert((idx == NTREEBINS-1) || (tsize < minsize_for_tree_index((idx+1))));
+
+  do { /* traverse through chain of same-sized nodes */
+    do_check_any_chunk(m, ((mchunkptr)u));
+    assert(u->index == tindex);
+    assert(chunksize(u) == tsize);
+    assert(!is_inuse(u));
+    assert(!next_pinuse(u));
+    assert(u->fd->bk == u);
+    assert(u->bk->fd == u);
+    if (u->parent == 0) {
+      assert(u->child[0] == 0);
+      assert(u->child[1] == 0);
+    }
+    else {
+      assert(head == 0); /* only one node on chain has parent */
+      head = u;
+      assert(u->parent != u);
+      assert (u->parent->child[0] == u ||
+              u->parent->child[1] == u ||
+              *((tbinptr*)(u->parent)) == u);
+      if (u->child[0] != 0) {
+        assert(u->child[0]->parent == u);
+        assert(u->child[0] != u);
+        do_check_tree(m, u->child[0]);
+      }
+      if (u->child[1] != 0) {
+        assert(u->child[1]->parent == u);
+        assert(u->child[1] != u);
+        do_check_tree(m, u->child[1]);
+      }
+      if (u->child[0] != 0 && u->child[1] != 0) {
+        assert(chunksize(u->child[0]) < chunksize(u->child[1]));
+      }
+    }
+    u = u->fd;
+  } while (u != t);
+  assert(head != 0);
+}
+
+/*  Check all the chunks in a treebin.  */
+static void do_check_treebin(mstate m, bindex_t i) {
+  tbinptr* tb = treebin_at(m, i);
+  tchunkptr t = *tb;
+  int empty = (m->treemap & (1U << i)) == 0;
+  if (t == 0)
+    assert(empty);
+  if (!empty)
+    do_check_tree(m, t);
+}
+
+/*  Check all the chunks in a smallbin.  */
+static void do_check_smallbin(mstate m, bindex_t i) {
+  sbinptr b = smallbin_at(m, i);
+  mchunkptr p = b->bk;
+  unsigned int empty = (m->smallmap & (1U << i)) == 0;
+  if (p == b)
+    assert(empty);
+  if (!empty) {
+    for (; p != b; p = p->bk) {
+      size_t size = chunksize(p);
+      mchunkptr q;
+      /* each chunk claims to be free */
+      do_check_free_chunk(m, p);
+      /* chunk belongs in bin */
+      assert(small_index(size) == i);
+      assert(p->bk == b || chunksize(p->bk) == chunksize(p));
+      /* chunk is followed by an inuse chunk */
+      q = next_chunk(p);
+      if (q->head != FENCEPOST_HEAD)
+        do_check_inuse_chunk(m, q);
+    }
+  }
+}
+
+/* Find x in a bin. Used in other check functions. */
+static int bin_find(mstate m, mchunkptr x) {
+  size_t size = chunksize(x);
+  if (is_small(size)) {
+    bindex_t sidx = small_index(size);
+    sbinptr b = smallbin_at(m, sidx);
+    if (smallmap_is_marked(m, sidx)) {
+      mchunkptr p = b;
+      do {
+        if (p == x)
+          return 1;
+      } while ((p = p->fd) != b);
+    }
+  }
+  else {
+    bindex_t tidx;
+    compute_tree_index(size, tidx);
+    if (treemap_is_marked(m, tidx)) {
+      tchunkptr t = *treebin_at(m, tidx);
+      size_t sizebits = size << leftshift_for_tree_index(tidx);
+      while (t != 0 && chunksize(t) != size) {
+        t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1];
+        sizebits <<= 1;
+      }
+      if (t != 0) {
+        tchunkptr u = t;
+        do {
+          if (u == (tchunkptr)x)
+            return 1;
+        } while ((u = u->fd) != t);
+      }
+    }
+  }
+  return 0;
+}
+
+/* Traverse each chunk and check it; return total */
+static size_t traverse_and_check(mstate m) {
+  size_t sum = 0;
+  if (is_initialized(m)) {
+    msegmentptr s = &m->seg;
+    sum += m->topsize + TOP_FOOT_SIZE;
+    while (s != 0) {
+      mchunkptr q = align_as_chunk(s->base);
+      mchunkptr lastq = 0;
+      assert(pinuse(q));
+      while (segment_holds(s, q) &&
+             q != m->top && q->head != FENCEPOST_HEAD) {
+        sum += chunksize(q);
+        if (is_inuse(q)) {
+          assert(!bin_find(m, q));
+          do_check_inuse_chunk(m, q);
+        }
+        else {
+          assert(q == m->dv || bin_find(m, q));
+          assert(lastq == 0 || is_inuse(lastq)); /* Not 2 consecutive free */
+          do_check_free_chunk(m, q);
+        }
+        lastq = q;
+        q = next_chunk(q);
+      }
+      s = s->next;
+    }
+  }
+  return sum;
+}
+
+
+/* Check all properties of malloc_state. */
+static void do_check_malloc_state(mstate m) {
+  bindex_t i;
+  size_t total;
+  /* check bins */
+  for (i = 0; i < NSMALLBINS; ++i)
+    do_check_smallbin(m, i);
+  for (i = 0; i < NTREEBINS; ++i)
+    do_check_treebin(m, i);
+
+  if (m->dvsize != 0) { /* check dv chunk */
+    do_check_any_chunk(m, m->dv);
+    assert(m->dvsize == chunksize(m->dv));
+    assert(m->dvsize >= MIN_CHUNK_SIZE);
+    assert(bin_find(m, m->dv) == 0);
+  }
+
+  if (m->top != 0) {   /* check top chunk */
+    do_check_top_chunk(m, m->top);
+    /*assert(m->topsize == chunksize(m->top)); redundant */
+    assert(m->topsize > 0);
+    assert(bin_find(m, m->top) == 0);
+  }
+
+  total = traverse_and_check(m);
+  assert(total <= m->footprint);
+  assert(m->footprint <= m->max_footprint);
+}
+#endif /* DEBUG */
+
+/* ----------------------------- statistics ------------------------------ */
+
+#if !NO_MALLINFO
+static struct mallinfo internal_mallinfo(mstate m) {
+  struct mallinfo nm = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+  ensure_initialization();
+  if (!PREACTION(m)) {
+    check_malloc_state(m);
+    if (is_initialized(m)) {
+      size_t nfree = SIZE_T_ONE; /* top always free */
+      size_t mfree = m->topsize + TOP_FOOT_SIZE;
+      size_t sum = mfree;
+      msegmentptr s = &m->seg;
+      while (s != 0) {
+        mchunkptr q = align_as_chunk(s->base);
+        while (segment_holds(s, q) &&
+               q != m->top && q->head != FENCEPOST_HEAD) {
+          size_t sz = chunksize(q);
+          sum += sz;
+          if (!is_inuse(q)) {
+            mfree += sz;
+            ++nfree;
+          }
+          q = next_chunk(q);
+        }
+        s = s->next;
+      }
+
+      nm.arena    = sum;
+      nm.ordblks  = nfree;
+      nm.hblkhd   = m->footprint - sum;
+      nm.usmblks  = m->max_footprint;
+      nm.uordblks = m->footprint - mfree;
+      nm.fordblks = mfree;
+      nm.keepcost = m->topsize;
+    }
+
+    POSTACTION(m);
+  }
+  return nm;
+}
+#endif /* !NO_MALLINFO */
+
+#if !NO_MALLOC_STATS
+static void internal_malloc_stats(mstate m) {
+  ensure_initialization();
+  if (!PREACTION(m)) {
+    size_t maxfp = 0;
+    size_t fp = 0;
+    size_t used = 0;
+    check_malloc_state(m);
+    if (is_initialized(m)) {
+      msegmentptr s = &m->seg;
+      maxfp = m->max_footprint;
+      fp = m->footprint;
+      used = fp - (m->topsize + TOP_FOOT_SIZE);
+
+      while (s != 0) {
+        mchunkptr q = align_as_chunk(s->base);
+        while (segment_holds(s, q) &&
+               q != m->top && q->head != FENCEPOST_HEAD) {
+          if (!is_inuse(q))
+            used -= chunksize(q);
+          q = next_chunk(q);
+        }
+        s = s->next;
+      }
+    }
+    POSTACTION(m); /* drop lock */
+    fprintf(stderr, "max system bytes = %10lu\n", (unsigned long)(maxfp));
+    fprintf(stderr, "system bytes     = %10lu\n", (unsigned long)(fp));
+    fprintf(stderr, "in use bytes     = %10lu\n", (unsigned long)(used));
+  }
+}
+#endif /* NO_MALLOC_STATS */
+
+/* ----------------------- Operations on smallbins ----------------------- */
+
+/*
+  Various forms of linking and unlinking are defined as macros.  Even
+  the ones for trees, which are very long but have very short typical
+  paths.  This is ugly but reduces reliance on inlining support of
+  compilers.
+*/
+
+/* Link a free chunk into a smallbin  */
+#define insert_small_chunk(M, P, S) {\
+  bindex_t I  = small_index(S);\
+  mchunkptr B = smallbin_at(M, I);\
+  mchunkptr F = B;\
+  assert(S >= MIN_CHUNK_SIZE);\
+  if (!smallmap_is_marked(M, I))\
+    mark_smallmap(M, I);\
+  else if (RTCHECK(ok_address(M, B->fd)))\
+    F = B->fd;\
+  else {\
+    CORRUPTION_ERROR_ACTION(M);\
+  }\
+  B->fd = P;\
+  F->bk = P;\
+  P->fd = F;\
+  P->bk = B;\
+}
+
+/* Unlink a chunk from a smallbin  */
+#define unlink_small_chunk(M, P, S) {\
+  mchunkptr F = P->fd;\
+  mchunkptr B = P->bk;\
+  bindex_t I = small_index(S);\
+  assert(P != B);\
+  assert(P != F);\
+  assert(chunksize(P) == small_index2size(I));\
+  if (RTCHECK(F == smallbin_at(M,I) || (ok_address(M, F) && F->bk == P))) { \
+    if (B == F) {\
+      clear_smallmap(M, I);\
+    }\
+    else if (RTCHECK(B == smallbin_at(M,I) ||\
+                     (ok_address(M, B) && B->fd == P))) {\
+      F->bk = B;\
+      B->fd = F;\
+    }\
+    else {\
+      CORRUPTION_ERROR_ACTION(M);\
+    }\
+  }\
+  else {\
+    CORRUPTION_ERROR_ACTION(M);\
+  }\
+}
+
+/* Unlink the first chunk from a smallbin */
+#define unlink_first_small_chunk(M, B, P, I) {\
+  mchunkptr F = P->fd;\
+  assert(P != B);\
+  assert(P != F);\
+  assert(chunksize(P) == small_index2size(I));\
+  if (B == F) {\
+    clear_smallmap(M, I);\
+  }\
+  else if (RTCHECK(ok_address(M, F) && F->bk == P)) {\
+    F->bk = B;\
+    B->fd = F;\
+  }\
+  else {\
+    CORRUPTION_ERROR_ACTION(M);\
+  }\
+}
+
+/* Replace dv node, binning the old one */
+/* Used only when dvsize known to be small */
+#define replace_dv(M, P, S) {\
+  size_t DVS = M->dvsize;\
+  assert(is_small(DVS));\
+  if (DVS != 0) {\
+    mchunkptr DV = M->dv;\
+    insert_small_chunk(M, DV, DVS);\
+  }\
+  M->dvsize = S;\
+  M->dv = P;\
+}
+
+/* ------------------------- Operations on trees ------------------------- */
+
+/* Insert chunk into tree */
+#define insert_large_chunk(M, X, S) {\
+  tbinptr* H;\
+  bindex_t I;\
+  compute_tree_index(S, I);\
+  H = treebin_at(M, I);\
+  X->index = I;\
+  X->child[0] = X->child[1] = 0;\
+  if (!treemap_is_marked(M, I)) {\
+    mark_treemap(M, I);\
+    *H = X;\
+    X->parent = (tchunkptr)H;\
+    X->fd = X->bk = X;\
+  }\
+  else {\
+    tchunkptr T = *H;\
+    size_t K = S << leftshift_for_tree_index(I);\
+    for (;;) {\
+      if (chunksize(T) != S) {\
+        tchunkptr* C = &(T->child[(K >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]);\
+        K <<= 1;\
+        if (*C != 0)\
+          T = *C;\
+        else if (RTCHECK(ok_address(M, C))) {\
+          *C = X;\
+          X->parent = T;\
+          X->fd = X->bk = X;\
+          break;\
+        }\
+        else {\
+          CORRUPTION_ERROR_ACTION(M);\
+          break;\
+        }\
+      }\
+      else {\
+        tchunkptr F = T->fd;\
+        if (RTCHECK(ok_address(M, T) && ok_address(M, F))) {\
+          T->fd = F->bk = X;\
+          X->fd = F;\
+          X->bk = T;\
+          X->parent = 0;\
+          break;\
+        }\
+        else {\
+          CORRUPTION_ERROR_ACTION(M);\
+          break;\
+        }\
+      }\
+    }\
+  }\
+}
+
+/*
+  Unlink steps:
+
+  1. If x is a chained node, unlink it from its same-sized fd/bk links
+     and choose its bk node as its replacement.
+  2. If x was the last node of its size, but not a leaf node, it must
+     be replaced with a leaf node (not merely one with an open left or
+     right), to make sure that lefts and rights of descendents
+     correspond properly to bit masks.  We use the rightmost descendent
+     of x.  We could use any other leaf, but this is easy to locate and
+     tends to counteract removal of leftmosts elsewhere, and so keeps
+     paths shorter than minimally guaranteed.  This doesn't loop much
+     because on average a node in a tree is near the bottom.
+  3. If x is the base of a chain (i.e., has parent links) relink
+     x's parent and children to x's replacement (or null if none).
+*/
+
+#define unlink_large_chunk(M, X) {\
+  tchunkptr XP = X->parent;\
+  tchunkptr R;\
+  if (X->bk != X) {\
+    tchunkptr F = X->fd;\
+    R = X->bk;\
+    if (RTCHECK(ok_address(M, F) && F->bk == X && R->fd == X)) {\
+      F->bk = R;\
+      R->fd = F;\
+    }\
+    else {\
+      CORRUPTION_ERROR_ACTION(M);\
+    }\
+  }\
+  else {\
+    tchunkptr* RP;\
+    if (((R = *(RP = &(X->child[1]))) != 0) ||\
+        ((R = *(RP = &(X->child[0]))) != 0)) {\
+      tchunkptr* CP;\
+      while ((*(CP = &(R->child[1])) != 0) ||\
+             (*(CP = &(R->child[0])) != 0)) {\
+        R = *(RP = CP);\
+      }\
+      if (RTCHECK(ok_address(M, RP)))\
+        *RP = 0;\
+      else {\
+        CORRUPTION_ERROR_ACTION(M);\
+      }\
+    }\
+  }\
+  if (XP != 0) {\
+    tbinptr* H = treebin_at(M, X->index);\
+    if (X == *H) {\
+      if ((*H = R) == 0) \
+        clear_treemap(M, X->index);\
+    }\
+    else if (RTCHECK(ok_address(M, XP))) {\
+      if (XP->child[0] == X) \
+        XP->child[0] = R;\
+      else \
+        XP->child[1] = R;\
+    }\
+    else\
+      CORRUPTION_ERROR_ACTION(M);\
+    if (R != 0) {\
+      if (RTCHECK(ok_address(M, R))) {\
+        tchunkptr C0, C1;\
+        R->parent = XP;\
+        if ((C0 = X->child[0]) != 0) {\
+          if (RTCHECK(ok_address(M, C0))) {\
+            R->child[0] = C0;\
+            C0->parent = R;\
+          }\
+          else\
+            CORRUPTION_ERROR_ACTION(M);\
+        }\
+        if ((C1 = X->child[1]) != 0) {\
+          if (RTCHECK(ok_address(M, C1))) {\
+            R->child[1] = C1;\
+            C1->parent = R;\
+          }\
+          else\
+            CORRUPTION_ERROR_ACTION(M);\
+        }\
+      }\
+      else\
+        CORRUPTION_ERROR_ACTION(M);\
+    }\
+  }\
+}
+
+/* Relays to large vs small bin operations */
+
+#define insert_chunk(M, P, S)\
+  if (is_small(S)) insert_small_chunk(M, P, S)\
+  else { tchunkptr TP = (tchunkptr)(P); insert_large_chunk(M, TP, S); }
+
+#define unlink_chunk(M, P, S)\
+  if (is_small(S)) unlink_small_chunk(M, P, S)\
+  else { tchunkptr TP = (tchunkptr)(P); unlink_large_chunk(M, TP); }
+
+
+/* Relays to internal calls to malloc/free from realloc, memalign etc */
+
+#if ONLY_MSPACES
+#define internal_malloc(m, b) mspace_malloc(m, b)
+#define internal_free(m, mem) mspace_free(m,mem);
+#else /* ONLY_MSPACES */
+#if MSPACES
+#define internal_malloc(m, b)\
+  ((m == gm)? dlmalloc(b) : mspace_malloc(m, b))
+#define internal_free(m, mem)\
+   if (m == gm) dlfree(mem); else mspace_free(m,mem);
+#else /* MSPACES */
+#define internal_malloc(m, b) dlmalloc(b)
+#define internal_free(m, mem) dlfree(mem)
+#endif /* MSPACES */
+#endif /* ONLY_MSPACES */
+
+/* -----------------------  Direct-mmapping chunks ----------------------- */
+
+/*
+  Directly mmapped chunks are set up with an offset to the start of
+  the mmapped region stored in the prev_foot field of the chunk. This
+  allows reconstruction of the required argument to MUNMAP when freed,
+  and also allows adjustment of the returned chunk to meet alignment
+  requirements (especially in memalign).
+*/
+
+/* Malloc using mmap */
+static void* mmap_alloc(mstate m, size_t nb) {
+  size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
+  if (m->footprint_limit != 0) {
+    size_t fp = m->footprint + mmsize;
+    if (fp <= m->footprint || fp > m->footprint_limit)
+      return 0;
+  }
+  if (mmsize > nb) {     /* Check for wrap around 0 */
+    char* mm = (char*)(CALL_DIRECT_MMAP(mmsize));
+    if (mm != CMFAIL) {
+      size_t offset = align_offset(chunk2mem(mm));
+      size_t psize = mmsize - offset - MMAP_FOOT_PAD;
+      mchunkptr p = (mchunkptr)(mm + offset);
+      p->prev_foot = offset;
+      p->head = psize;
+      mark_inuse_foot(m, p, psize);
+      chunk_plus_offset(p, psize)->head = FENCEPOST_HEAD;
+      chunk_plus_offset(p, psize+SIZE_T_SIZE)->head = 0;
+
+      if (m->least_addr == 0 || mm < m->least_addr)
+        m->least_addr = mm;
+      if ((m->footprint += mmsize) > m->max_footprint)
+        m->max_footprint = m->footprint;
+      assert(is_aligned(chunk2mem(p)));
+      check_mmapped_chunk(m, p);
+      return chunk2mem(p);
+    }
+  }
+  return 0;
+}
+
+/* Realloc using mmap */
+static mchunkptr mmap_resize(mstate m, mchunkptr oldp, size_t nb, int flags) {
+  size_t oldsize = chunksize(oldp);
+  (void)flags; /* placate people compiling -Wunused */
+  if (is_small(nb)) /* Can't shrink mmap regions below small size */
+    return 0;
+  /* Keep old chunk if big enough but not too big */
+  if (oldsize >= nb + SIZE_T_SIZE &&
+      (oldsize - nb) <= (mparams.granularity << 1))
+    return oldp;
+  else {
+    size_t offset = oldp->prev_foot;
+    size_t oldmmsize = oldsize + offset + MMAP_FOOT_PAD;
+    size_t newmmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
+    char* cp = (char*)CALL_MREMAP((char*)oldp - offset,
+                                  oldmmsize, newmmsize, flags);
+    if (cp != CMFAIL) {
+      mchunkptr newp = (mchunkptr)(cp + offset);
+      size_t psize = newmmsize - offset - MMAP_FOOT_PAD;
+      newp->head = psize;
+      mark_inuse_foot(m, newp, psize);
+      chunk_plus_offset(newp, psize)->head = FENCEPOST_HEAD;
+      chunk_plus_offset(newp, psize+SIZE_T_SIZE)->head = 0;
+
+      if (cp < m->least_addr)
+        m->least_addr = cp;
+      if ((m->footprint += newmmsize - oldmmsize) > m->max_footprint)
+        m->max_footprint = m->footprint;
+      check_mmapped_chunk(m, newp);
+      return newp;
+    }
+  }
+  return 0;
+}
+
+
+/* -------------------------- mspace management -------------------------- */
+
+/* Initialize top chunk and its size */
+static void init_top(mstate m, mchunkptr p, size_t psize) {
+  /* Ensure alignment */
+  size_t offset = align_offset(chunk2mem(p));
+  p = (mchunkptr)((char*)p + offset);
+  psize -= offset;
+
+  m->top = p;
+  m->topsize = psize;
+  p->head = psize | PINUSE_BIT;
+  /* set size of fake trailing chunk holding overhead space only once */
+  chunk_plus_offset(p, psize)->head = TOP_FOOT_SIZE;
+  m->trim_check = mparams.trim_threshold; /* reset on each update */
+}
+
+/* Initialize bins for a new mstate that is otherwise zeroed out */
+static void init_bins(mstate m) {
+  /* Establish circular links for smallbins */
+  bindex_t i;
+  for (i = 0; i < NSMALLBINS; ++i) {
+    sbinptr bin = smallbin_at(m,i);
+    bin->fd = bin->bk = bin;
+  }
+}
+
+#if PROCEED_ON_ERROR
+
+/* default corruption action */
+static void reset_on_error(mstate m) {
+  int i;
+  ++malloc_corruption_error_count;
+  /* Reinitialize fields to forget about all memory */
+  m->smallmap = m->treemap = 0;
+  m->dvsize = m->topsize = 0;
+  m->seg.base = 0;
+  m->seg.size = 0;
+  m->seg.next = 0;
+  m->top = m->dv = 0;
+  for (i = 0; i < NTREEBINS; ++i)
+    *treebin_at(m, i) = 0;
+  init_bins(m);
+}
+#endif /* PROCEED_ON_ERROR */
+
+/* Allocate chunk and prepend remainder with chunk in successor base. */
+static void* prepend_alloc(mstate m, char* newbase, char* oldbase,
+                           size_t nb) {
+  mchunkptr p = align_as_chunk(newbase);
+  mchunkptr oldfirst = align_as_chunk(oldbase);
+  size_t psize = (char*)oldfirst - (char*)p;
+  mchunkptr q = chunk_plus_offset(p, nb);
+  size_t qsize = psize - nb;
+  set_size_and_pinuse_of_inuse_chunk(m, p, nb);
+
+  assert((char*)oldfirst > (char*)q);
+  assert(pinuse(oldfirst));
+  assert(qsize >= MIN_CHUNK_SIZE);
+
+  /* consolidate remainder with first chunk of old base */
+  if (oldfirst == m->top) {
+    size_t tsize = m->topsize += qsize;
+    m->top = q;
+    q->head = tsize | PINUSE_BIT;
+    check_top_chunk(m, q);
+  }
+  else if (oldfirst == m->dv) {
+    size_t dsize = m->dvsize += qsize;
+    m->dv = q;
+    set_size_and_pinuse_of_free_chunk(q, dsize);
+  }
+  else {
+    if (!is_inuse(oldfirst)) {
+      size_t nsize = chunksize(oldfirst);
+      unlink_chunk(m, oldfirst, nsize);
+      oldfirst = chunk_plus_offset(oldfirst, nsize);
+      qsize += nsize;
+    }
+    set_free_with_pinuse(q, qsize, oldfirst);
+    insert_chunk(m, q, qsize);
+    check_free_chunk(m, q);
+  }
+
+  check_malloced_chunk(m, chunk2mem(p), nb);
+  return chunk2mem(p);
+}
+
+/* Add a segment to hold a new noncontiguous region */
+static void add_segment(mstate m, char* tbase, size_t tsize, flag_t mmapped) {
+  /* Determine locations and sizes of segment, fenceposts, old top */
+  char* old_top = (char*)m->top;
+  msegmentptr oldsp = segment_holding(m, old_top);
+  char* old_end = oldsp->base + oldsp->size;
+  size_t ssize = pad_request(sizeof(struct malloc_segment));
+  char* rawsp = old_end - (ssize + FOUR_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
+  size_t offset = align_offset(chunk2mem(rawsp));
+  char* asp = rawsp + offset;
+  char* csp = (asp < (old_top + MIN_CHUNK_SIZE))? old_top : asp;
+  mchunkptr sp = (mchunkptr)csp;
+  msegmentptr ss = (msegmentptr)(chunk2mem(sp));
+  mchunkptr tnext = chunk_plus_offset(sp, ssize);
+  mchunkptr p = tnext;
+  int nfences = 0;
+
+  /* reset top to new space */
+  init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE);
+
+  /* Set up segment record */
+  assert(is_aligned(ss));
+  set_size_and_pinuse_of_inuse_chunk(m, sp, ssize);
+  *ss = m->seg; /* Push current record */
+  m->seg.base = tbase;
+  m->seg.size = tsize;
+  m->seg.sflags = mmapped;
+  m->seg.next = ss;
+
+  /* Insert trailing fenceposts */
+  for (;;) {
+    mchunkptr nextp = chunk_plus_offset(p, SIZE_T_SIZE);
+    p->head = FENCEPOST_HEAD;
+    ++nfences;
+    if ((char*)(&(nextp->head)) < old_end)
+      p = nextp;
+    else
+      break;
+  }
+  assert(nfences >= 2);
+
+  /* Insert the rest of old top into a bin as an ordinary free chunk */
+  if (csp != old_top) {
+    mchunkptr q = (mchunkptr)old_top;
+    size_t psize = csp - old_top;
+    mchunkptr tn = chunk_plus_offset(q, psize);
+    set_free_with_pinuse(q, psize, tn);
+    insert_chunk(m, q, psize);
+  }
+
+  check_top_chunk(m, m->top);
+}
+
+/* -------------------------- System allocation -------------------------- */
+
+/* Get memory from system using MORECORE or MMAP */
+static void* sys_alloc(mstate m, size_t nb) {
+  char* tbase = CMFAIL;
+  size_t tsize = 0;
+  flag_t mmap_flag = 0;
+  size_t asize; /* allocation size */
+
+  ensure_initialization();
+
+  /* Directly map large chunks, but only if already initialized */
+  if (use_mmap(m) && nb >= mparams.mmap_threshold && m->topsize != 0) {
+    void* mem = mmap_alloc(m, nb);
+    if (mem != 0)
+      return mem;
+  }
+
+  asize = granularity_align(nb + SYS_ALLOC_PADDING);
+  if (asize <= nb)
+    return 0; /* wraparound */
+  if (m->footprint_limit != 0) {
+    size_t fp = m->footprint + asize;
+    if (fp <= m->footprint || fp > m->footprint_limit)
+      return 0;
+  }
+
+  /*
+    Try getting memory in any of three ways (in most-preferred to
+    least-preferred order):
+    1. A call to MORECORE that can normally contiguously extend memory.
+       (disabled if not MORECORE_CONTIGUOUS or not HAVE_MORECORE or
+       or main space is mmapped or a previous contiguous call failed)
+    2. A call to MMAP new space (disabled if not HAVE_MMAP).
+       Note that under the default settings, if MORECORE is unable to
+       fulfill a request, and HAVE_MMAP is true, then mmap is
+       used as a noncontiguous system allocator. This is a useful backup
+       strategy for systems with holes in address spaces -- in this case
+       sbrk cannot contiguously expand the heap, but mmap may be able to
+       find space.
+    3. A call to MORECORE that cannot usually contiguously extend memory.
+       (disabled if not HAVE_MORECORE)
+
+   In all cases, we need to request enough bytes from system to ensure
+   we can malloc nb bytes upon success, so pad with enough space for
+   top_foot, plus alignment-pad to make sure we don't lose bytes if
+   not on boundary, and round this up to a granularity unit.
+  */
+
+  if (MORECORE_CONTIGUOUS && !use_noncontiguous(m)) {
+    char* br = CMFAIL;
+    size_t ssize = asize; /* sbrk call size */
+    msegmentptr ss = (m->top == 0)? 0 : segment_holding(m, (char*)m->top);
+    ACQUIRE_MALLOC_GLOBAL_LOCK();
+
+    if (ss == 0) {  /* First time through or recovery */
+      char* base = (char*)CALL_MORECORE(0);
+      if (base != CMFAIL) {
+        size_t fp;
+        /* Adjust to end on a page boundary */
+        if (!is_page_aligned(base))
+          ssize += (page_align((size_t)base) - (size_t)base);
+        fp = m->footprint + ssize; /* recheck limits */
+        if (ssize > nb && ssize < HALF_MAX_SIZE_T &&
+            (m->footprint_limit == 0 ||
+             (fp > m->footprint && fp <= m->footprint_limit)) &&
+            (br = (char*)(CALL_MORECORE(ssize))) == base) {
+          tbase = base;
+          tsize = ssize;
+        }
+      }
+    }
+    else {
+      /* Subtract out existing available top space from MORECORE request. */
+      ssize = granularity_align(nb - m->topsize + SYS_ALLOC_PADDING);
+      /* Use mem here only if it did continuously extend old space */
+      if (ssize < HALF_MAX_SIZE_T &&
+          (br = (char*)(CALL_MORECORE(ssize))) == ss->base+ss->size) {
+        tbase = br;
+        tsize = ssize;
+      }
+    }
+
+    if (tbase == CMFAIL) {    /* Cope with partial failure */
+      if (br != CMFAIL) {    /* Try to use/extend the space we did get */
+        if (ssize < HALF_MAX_SIZE_T &&
+            ssize < nb + SYS_ALLOC_PADDING) {
+          size_t esize = granularity_align(nb + SYS_ALLOC_PADDING - ssize);
+          if (esize < HALF_MAX_SIZE_T) {
+            char* end = (char*)CALL_MORECORE(esize);
+            if (end != CMFAIL)
+              ssize += esize;
+            else {            /* Can't use; try to release */
+              (void) CALL_MORECORE(-ssize);
+              br = CMFAIL;
+            }
+          }
+        }
+      }
+      if (br != CMFAIL) {    /* Use the space we did get */
+        tbase = br;
+        tsize = ssize;
+      }
+      else
+        disable_contiguous(m); /* Don't try contiguous path in the future */
+    }
+
+    RELEASE_MALLOC_GLOBAL_LOCK();
+  }
+
+  if (HAVE_MMAP && tbase == CMFAIL) {  /* Try MMAP */
+    char* mp = (char*)(CALL_MMAP(asize));
+    if (mp != CMFAIL) {
+      tbase = mp;
+      tsize = asize;
+      mmap_flag = USE_MMAP_BIT;
+    }
+  }
+
+  if (HAVE_MORECORE && tbase == CMFAIL) { /* Try noncontiguous MORECORE */
+    if (asize < HALF_MAX_SIZE_T) {
+      char* br = CMFAIL;
+      char* end = CMFAIL;
+      ACQUIRE_MALLOC_GLOBAL_LOCK();
+      br = (char*)(CALL_MORECORE(asize));
+      end = (char*)(CALL_MORECORE(0));
+      RELEASE_MALLOC_GLOBAL_LOCK();
+      if (br != CMFAIL && end != CMFAIL && br < end) {
+        size_t ssize = end - br;
+        if (ssize > nb + TOP_FOOT_SIZE) {
+          tbase = br;
+          tsize = ssize;
+        }
+      }
+    }
+  }
+
+  if (tbase != CMFAIL) {
+
+    if ((m->footprint += tsize) > m->max_footprint)
+      m->max_footprint = m->footprint;
+
+    if (!is_initialized(m)) { /* first-time initialization */
+      if (m->least_addr == 0 || tbase < m->least_addr)
+        m->least_addr = tbase;
+      m->seg.base = tbase;
+      m->seg.size = tsize;
+      m->seg.sflags = mmap_flag;
+      m->magic = mparams.magic;
+      m->release_checks = MAX_RELEASE_CHECK_RATE;
+      init_bins(m);
+#if !ONLY_MSPACES
+      if (is_global(m))
+        init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE);
+      else
+#endif
+      {
+        /* Offset top by embedded malloc_state */
+        mchunkptr mn = next_chunk(mem2chunk(m));
+        init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) -TOP_FOOT_SIZE);
+      }
+    }
+
+    else {
+      /* Try to merge with an existing segment */
+      msegmentptr sp = &m->seg;
+      /* Only consider most recent segment if traversal suppressed */
+      while (sp != 0 && tbase != sp->base + sp->size)
+        sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next;
+      if (sp != 0 &&
+          !is_extern_segment(sp) &&
+          (sp->sflags & USE_MMAP_BIT) == mmap_flag &&
+          segment_holds(sp, m->top)) { /* append */
+        sp->size += tsize;
+        init_top(m, m->top, m->topsize + tsize);
+      }
+      else {
+        if (tbase < m->least_addr)
+          m->least_addr = tbase;
+        sp = &m->seg;
+        while (sp != 0 && sp->base != tbase + tsize)
+          sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next;
+        if (sp != 0 &&
+            !is_extern_segment(sp) &&
+            (sp->sflags & USE_MMAP_BIT) == mmap_flag) {
+          char* oldbase = sp->base;
+          sp->base = tbase;
+          sp->size += tsize;
+          return prepend_alloc(m, tbase, oldbase, nb);
+        }
+        else
+          add_segment(m, tbase, tsize, mmap_flag);
+      }
+    }
+
+    if (nb < m->topsize) { /* Allocate from new or extended top space */
+      size_t rsize = m->topsize -= nb;
+      mchunkptr p = m->top;
+      mchunkptr r = m->top = chunk_plus_offset(p, nb);
+      r->head = rsize | PINUSE_BIT;
+      set_size_and_pinuse_of_inuse_chunk(m, p, nb);
+      check_top_chunk(m, m->top);
+      check_malloced_chunk(m, chunk2mem(p), nb);
+      return chunk2mem(p);
+    }
+  }
+
+  MALLOC_FAILURE_ACTION;
+  return 0;
+}
+
+/* -----------------------  system deallocation -------------------------- */
+
+/* Unmap and unlink any mmapped segments that don't contain used chunks */
+static size_t release_unused_segments(mstate m) {
+  size_t released = 0;
+  int nsegs = 0;
+  msegmentptr pred = &m->seg;
+  msegmentptr sp = pred->next;
+  while (sp != 0) {
+    char* base = sp->base;
+    size_t size = sp->size;
+    msegmentptr next = sp->next;
+    ++nsegs;
+    if (is_mmapped_segment(sp) && !is_extern_segment(sp)) {
+      mchunkptr p = align_as_chunk(base);
+      size_t psize = chunksize(p);
+      /* Can unmap if first chunk holds entire segment and not pinned */
+      if (!is_inuse(p) && (char*)p + psize >= base + size - TOP_FOOT_SIZE) {
+        tchunkptr tp = (tchunkptr)p;
+        assert(segment_holds(sp, (char*)sp));
+        if (p == m->dv) {
+          m->dv = 0;
+          m->dvsize = 0;
+        }
+        else {
+          unlink_large_chunk(m, tp);
+        }
+        if (CALL_MUNMAP(base, size) == 0) {
+          released += size;
+          m->footprint -= size;
+          /* unlink obsoleted record */
+          sp = pred;
+          sp->next = next;
+        }
+        else { /* back out if cannot unmap */
+          insert_large_chunk(m, tp, psize);
+        }
+      }
+    }
+    if (NO_SEGMENT_TRAVERSAL) /* scan only first segment */
+      break;
+    pred = sp;
+    sp = next;
+  }
+  /* Reset check counter */
+  m->release_checks = (((size_t) nsegs > (size_t) MAX_RELEASE_CHECK_RATE)?
+                       (size_t) nsegs : (size_t) MAX_RELEASE_CHECK_RATE);
+  return released;
+}
+
+static int sys_trim(mstate m, size_t pad) {
+  size_t released = 0;
+  ensure_initialization();
+  if (pad < MAX_REQUEST && is_initialized(m)) {
+    pad += TOP_FOOT_SIZE; /* ensure enough room for segment overhead */
+
+    if (m->topsize > pad) {
+      /* Shrink top space in granularity-size units, keeping at least one */
+      size_t unit = mparams.granularity;
+      size_t extra = ((m->topsize - pad + (unit - SIZE_T_ONE)) / unit -
+                      SIZE_T_ONE) * unit;
+      msegmentptr sp = segment_holding(m, (char*)m->top);
+
+      if (!is_extern_segment(sp)) {
+        if (is_mmapped_segment(sp)) {
+          if (HAVE_MMAP &&
+              sp->size >= extra &&
+              !has_segment_link(m, sp)) { /* can't shrink if pinned */
+            size_t newsize = sp->size - extra;
+            (void)newsize; /* placate people compiling -Wunused-variable */
+            /* Prefer mremap, fall back to munmap */
+            if ((CALL_MREMAP(sp->base, sp->size, newsize, 0) != MFAIL) ||
+                (CALL_MUNMAP(sp->base + newsize, extra) == 0)) {
+              released = extra;
+            }
+          }
+        }
+        else if (HAVE_MORECORE) {
+          if (extra >= HALF_MAX_SIZE_T) /* Avoid wrapping negative */
+            extra = (HALF_MAX_SIZE_T) + SIZE_T_ONE - unit;
+          ACQUIRE_MALLOC_GLOBAL_LOCK();
+          {
+            /* Make sure end of memory is where we last set it. */
+            char* old_br = (char*)(CALL_MORECORE(0));
+            if (old_br == sp->base + sp->size) {
+              char* rel_br = (char*)(CALL_MORECORE(-extra));
+              char* new_br = (char*)(CALL_MORECORE(0));
+              if (rel_br != CMFAIL && new_br < old_br)
+                released = old_br - new_br;
+            }
+          }
+          RELEASE_MALLOC_GLOBAL_LOCK();
+        }
+      }
+
+      if (released != 0) {
+        sp->size -= released;
+        m->footprint -= released;
+        init_top(m, m->top, m->topsize - released);
+        check_top_chunk(m, m->top);
+      }
+    }
+
+    /* Unmap any unused mmapped segments */
+    if (HAVE_MMAP)
+      released += release_unused_segments(m);
+
+    /* On failure, disable autotrim to avoid repeated failed future calls */
+    if (released == 0 && m->topsize > m->trim_check)
+      m->trim_check = MAX_SIZE_T;
+  }
+
+  return (released != 0)? 1 : 0;
+}
+
+/* Consolidate and bin a chunk. Differs from exported versions
+   of free mainly in that the chunk need not be marked as inuse.
+*/
+static void dispose_chunk(mstate m, mchunkptr p, size_t psize) {
+  mchunkptr next = chunk_plus_offset(p, psize);
+  if (!pinuse(p)) {
+    mchunkptr prev;
+    size_t prevsize = p->prev_foot;
+    if (is_mmapped(p)) {
+      psize += prevsize + MMAP_FOOT_PAD;
+      if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
+        m->footprint -= psize;
+      return;
+    }
+    prev = chunk_minus_offset(p, prevsize);
+    psize += prevsize;
+    p = prev;
+    if (RTCHECK(ok_address(m, prev))) { /* consolidate backward */
+      if (p != m->dv) {
+        unlink_chunk(m, p, prevsize);
+      }
+      else if ((next->head & INUSE_BITS) == INUSE_BITS) {
+        m->dvsize = psize;
+        set_free_with_pinuse(p, psize, next);
+        return;
+      }
+    }
+    else {
+      CORRUPTION_ERROR_ACTION(m);
+      return;
+    }
+  }
+  if (RTCHECK(ok_address(m, next))) {
+    if (!cinuse(next)) {  /* consolidate forward */
+      if (next == m->top) {
+        size_t tsize = m->topsize += psize;
+        m->top = p;
+        p->head = tsize | PINUSE_BIT;
+        if (p == m->dv) {
+          m->dv = 0;
+          m->dvsize = 0;
+        }
+        return;
+      }
+      else if (next == m->dv) {
+        size_t dsize = m->dvsize += psize;
+        m->dv = p;
+        set_size_and_pinuse_of_free_chunk(p, dsize);
+        return;
+      }
+      else {
+        size_t nsize = chunksize(next);
+        psize += nsize;
+        unlink_chunk(m, next, nsize);
+        set_size_and_pinuse_of_free_chunk(p, psize);
+        if (p == m->dv) {
+          m->dvsize = psize;
+          return;
+        }
+      }
+    }
+    else {
+      set_free_with_pinuse(p, psize, next);
+    }
+    insert_chunk(m, p, psize);
+  }
+  else {
+    CORRUPTION_ERROR_ACTION(m);
+  }
+}
+
+/* ---------------------------- malloc --------------------------- */
+
+/* allocate a large request from the best fitting chunk in a treebin */
+static void* tmalloc_large(mstate m, size_t nb) {
+  tchunkptr v = 0;
+  size_t rsize = -nb; /* Unsigned negation */
+  tchunkptr t;
+  bindex_t idx;
+  compute_tree_index(nb, idx);
+  if ((t = *treebin_at(m, idx)) != 0) {
+    /* Traverse tree for this bin looking for node with size == nb */
+    size_t sizebits = nb << leftshift_for_tree_index(idx);
+    tchunkptr rst = 0;  /* The deepest untaken right subtree */
+    for (;;) {
+      tchunkptr rt;
+      size_t trem = chunksize(t) - nb;
+      if (trem < rsize) {
+        v = t;
+        if ((rsize = trem) == 0)
+          break;
+      }
+      rt = t->child[1];
+      t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1];
+      if (rt != 0 && rt != t)
+        rst = rt;
+      if (t == 0) {
+        t = rst; /* set t to least subtree holding sizes > nb */
+        break;
+      }
+      sizebits <<= 1;
+    }
+  }
+  if (t == 0 && v == 0) { /* set t to root of next non-empty treebin */
+    binmap_t leftbits = left_bits(idx2bit(idx)) & m->treemap;
+    if (leftbits != 0) {
+      bindex_t i;
+      binmap_t leastbit = least_bit(leftbits);
+      compute_bit2idx(leastbit, i);
+      t = *treebin_at(m, i);
+    }
+  }
+
+  while (t != 0) { /* find smallest of tree or subtree */
+    size_t trem = chunksize(t) - nb;
+    if (trem < rsize) {
+      rsize = trem;
+      v = t;
+    }
+    t = leftmost_child(t);
+  }
+
+  /*  If dv is a better fit, return 0 so malloc will use it */
+  if (v != 0 && rsize < (size_t)(m->dvsize - nb)) {
+    if (RTCHECK(ok_address(m, v))) { /* split */
+      mchunkptr r = chunk_plus_offset(v, nb);
+      assert(chunksize(v) == rsize + nb);
+      if (RTCHECK(ok_next(v, r))) {
+        unlink_large_chunk(m, v);
+        if (rsize < MIN_CHUNK_SIZE)
+          set_inuse_and_pinuse(m, v, (rsize + nb));
+        else {
+          set_size_and_pinuse_of_inuse_chunk(m, v, nb);
+          set_size_and_pinuse_of_free_chunk(r, rsize);
+          insert_chunk(m, r, rsize);
+        }
+        return chunk2mem(v);
+      }
+    }
+    CORRUPTION_ERROR_ACTION(m);
+  }
+  return 0;
+}
+
+/* allocate a small request from the best fitting chunk in a treebin */
+static void* tmalloc_small(mstate m, size_t nb) {
+  tchunkptr t, v;
+  size_t rsize;
+  bindex_t i;
+  binmap_t leastbit = least_bit(m->treemap);
+  compute_bit2idx(leastbit, i);
+  v = t = *treebin_at(m, i);
+  rsize = chunksize(t) - nb;
+
+  while ((t = leftmost_child(t)) != 0) {
+    size_t trem = chunksize(t) - nb;
+    if (trem < rsize) {
+      rsize = trem;
+      v = t;
+    }
+  }
+
+  if (RTCHECK(ok_address(m, v))) {
+    mchunkptr r = chunk_plus_offset(v, nb);
+    assert(chunksize(v) == rsize + nb);
+    if (RTCHECK(ok_next(v, r))) {
+      unlink_large_chunk(m, v);
+      if (rsize < MIN_CHUNK_SIZE)
+        set_inuse_and_pinuse(m, v, (rsize + nb));
+      else {
+        set_size_and_pinuse_of_inuse_chunk(m, v, nb);
+        set_size_and_pinuse_of_free_chunk(r, rsize);
+        replace_dv(m, r, rsize);
+      }
+      return chunk2mem(v);
+    }
+  }
+
+  CORRUPTION_ERROR_ACTION(m);
+  return 0;
+}
+
+#if !ONLY_MSPACES
+
+void* dlmalloc(size_t bytes) {
+  /*
+     Basic algorithm:
+     If a small request (< 256 bytes minus per-chunk overhead):
+       1. If one exists, use a remainderless chunk in associated smallbin.
+          (Remainderless means that there are too few excess bytes to
+          represent as a chunk.)
+       2. If it is big enough, use the dv chunk, which is normally the
+          chunk adjacent to the one used for the most recent small request.
+       3. If one exists, split the smallest available chunk in a bin,
+          saving remainder in dv.
+       4. If it is big enough, use the top chunk.
+       5. If available, get memory from system and use it
+     Otherwise, for a large request:
+       1. Find the smallest available binned chunk that fits, and use it
+          if it is better fitting than dv chunk, splitting if necessary.
+       2. If better fitting than any binned chunk, use the dv chunk.
+       3. If it is big enough, use the top chunk.
+       4. If request size >= mmap threshold, try to directly mmap this chunk.
+       5. If available, get memory from system and use it
+
+     The ugly goto's here ensure that postaction occurs along all paths.
+  */
+
+#if USE_LOCKS
+  ensure_initialization(); /* initialize in sys_alloc if not using locks */
+#endif
+
+  if (!PREACTION(gm)) {
+    void* mem;
+    size_t nb;
+    if (bytes <= MAX_SMALL_REQUEST) {
+      bindex_t idx;
+      binmap_t smallbits;
+      nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes);
+      idx = small_index(nb);
+      smallbits = gm->smallmap >> idx;
+
+      if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */
+        mchunkptr b, p;
+        idx += ~smallbits & 1;       /* Uses next bin if idx empty */
+        b = smallbin_at(gm, idx);
+        p = b->fd;
+        assert(chunksize(p) == small_index2size(idx));
+        unlink_first_small_chunk(gm, b, p, idx);
+        set_inuse_and_pinuse(gm, p, small_index2size(idx));
+        mem = chunk2mem(p);
+        check_malloced_chunk(gm, mem, nb);
+        goto postaction;
+      }
+
+      else if (nb > gm->dvsize) {
+        if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
+          mchunkptr b, p, r;
+          size_t rsize;
+          bindex_t i;
+          binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx));
+          binmap_t leastbit = least_bit(leftbits);
+          compute_bit2idx(leastbit, i);
+          b = smallbin_at(gm, i);
+          p = b->fd;
+          assert(chunksize(p) == small_index2size(i));
+          unlink_first_small_chunk(gm, b, p, i);
+          rsize = small_index2size(i) - nb;
+          /* Fit here cannot be remainderless if 4byte sizes */
+          if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE)
+            set_inuse_and_pinuse(gm, p, small_index2size(i));
+          else {
+            set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
+            r = chunk_plus_offset(p, nb);
+            set_size_and_pinuse_of_free_chunk(r, rsize);
+            replace_dv(gm, r, rsize);
+          }
+          mem = chunk2mem(p);
+          check_malloced_chunk(gm, mem, nb);
+          goto postaction;
+        }
+
+        else if (gm->treemap != 0 && (mem = tmalloc_small(gm, nb)) != 0) {
+          check_malloced_chunk(gm, mem, nb);
+          goto postaction;
+        }
+      }
+    }
+    else if (bytes >= MAX_REQUEST)
+      nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */
+    else {
+      nb = pad_request(bytes);
+      if (gm->treemap != 0 && (mem = tmalloc_large(gm, nb)) != 0) {
+        check_malloced_chunk(gm, mem, nb);
+        goto postaction;
+      }
+    }
+
+    if (nb <= gm->dvsize) {
+      size_t rsize = gm->dvsize - nb;
+      mchunkptr p = gm->dv;
+      if (rsize >= MIN_CHUNK_SIZE) { /* split dv */
+        mchunkptr r = gm->dv = chunk_plus_offset(p, nb);
+        gm->dvsize = rsize;
+        set_size_and_pinuse_of_free_chunk(r, rsize);
+        set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
+      }
+      else { /* exhaust dv */
+        size_t dvs = gm->dvsize;
+        gm->dvsize = 0;
+        gm->dv = 0;
+        set_inuse_and_pinuse(gm, p, dvs);
+      }
+      mem = chunk2mem(p);
+      check_malloced_chunk(gm, mem, nb);
+      goto postaction;
+    }
+
+    else if (nb < gm->topsize) { /* Split top */
+      size_t rsize = gm->topsize -= nb;
+      mchunkptr p = gm->top;
+      mchunkptr r = gm->top = chunk_plus_offset(p, nb);
+      r->head = rsize | PINUSE_BIT;
+      set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
+      mem = chunk2mem(p);
+      check_top_chunk(gm, gm->top);
+      check_malloced_chunk(gm, mem, nb);
+      goto postaction;
+    }
+
+    mem = sys_alloc(gm, nb);
+
+  postaction:
+    POSTACTION(gm);
+    return mem;
+  }
+
+  return 0;
+}
+
+/* ---------------------------- free --------------------------- */
+
+void dlfree(void* mem) {
+  /*
+     Consolidate freed chunks with preceeding or succeeding bordering
+     free chunks, if they exist, and then place in a bin.  Intermixed
+     with special cases for top, dv, mmapped chunks, and usage errors.
+  */
+
+  if (mem != 0) {
+    mchunkptr p  = mem2chunk(mem);
+#if FOOTERS
+    mstate fm = get_mstate_for(p);
+    if (!ok_magic(fm)) {
+      USAGE_ERROR_ACTION(fm, p);
+      return;
+    }
+#else /* FOOTERS */
+#define fm gm
+#endif /* FOOTERS */
+    if (!PREACTION(fm)) {
+      check_inuse_chunk(fm, p);
+      if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) {
+        size_t psize = chunksize(p);
+        mchunkptr next = chunk_plus_offset(p, psize);
+        if (!pinuse(p)) {
+          size_t prevsize = p->prev_foot;
+          if (is_mmapped(p)) {
+            psize += prevsize + MMAP_FOOT_PAD;
+            if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
+              fm->footprint -= psize;
+            goto postaction;
+          }
+          else {
+            mchunkptr prev = chunk_minus_offset(p, prevsize);
+            psize += prevsize;
+            p = prev;
+            if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */
+              if (p != fm->dv) {
+                unlink_chunk(fm, p, prevsize);
+              }
+              else if ((next->head & INUSE_BITS) == INUSE_BITS) {
+                fm->dvsize = psize;
+                set_free_with_pinuse(p, psize, next);
+                goto postaction;
+              }
+            }
+            else
+              goto erroraction;
+          }
+        }
+
+        if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) {
+          if (!cinuse(next)) {  /* consolidate forward */
+            if (next == fm->top) {
+              size_t tsize = fm->topsize += psize;
+              fm->top = p;
+              p->head = tsize | PINUSE_BIT;
+              if (p == fm->dv) {
+                fm->dv = 0;
+                fm->dvsize = 0;
+              }
+              if (should_trim(fm, tsize))
+                sys_trim(fm, 0);
+              goto postaction;
+            }
+            else if (next == fm->dv) {
+              size_t dsize = fm->dvsize += psize;
+              fm->dv = p;
+              set_size_and_pinuse_of_free_chunk(p, dsize);
+              goto postaction;
+            }
+            else {
+              size_t nsize = chunksize(next);
+              psize += nsize;
+              unlink_chunk(fm, next, nsize);
+              set_size_and_pinuse_of_free_chunk(p, psize);
+              if (p == fm->dv) {
+                fm->dvsize = psize;
+                goto postaction;
+              }
+            }
+          }
+          else
+            set_free_with_pinuse(p, psize, next);
+
+          if (is_small(psize)) {
+            insert_small_chunk(fm, p, psize);
+            check_free_chunk(fm, p);
+          }
+          else {
+            tchunkptr tp = (tchunkptr)p;
+            insert_large_chunk(fm, tp, psize);
+            check_free_chunk(fm, p);
+            if (--fm->release_checks == 0)
+              release_unused_segments(fm);
+          }
+          goto postaction;
+        }
+      }
+    erroraction:
+      USAGE_ERROR_ACTION(fm, p);
+    postaction:
+      POSTACTION(fm);
+    }
+  }
+#if !FOOTERS
+#undef fm
+#endif /* FOOTERS */
+}
+
+void* dlcalloc(size_t n_elements, size_t elem_size) {
+  void* mem;
+  size_t req = 0;
+  if (n_elements != 0) {
+    req = n_elements * elem_size;
+    if (((n_elements | elem_size) & ~(size_t)0xffff) &&
+        (req / n_elements != elem_size))
+      req = MAX_SIZE_T; /* force downstream failure on overflow */
+  }
+  mem = dlmalloc(req);
+  if (mem != 0 && calloc_must_clear(mem2chunk(mem)))
+    memset(mem, 0, req);
+  return mem;
+}
+
+#endif /* !ONLY_MSPACES */
+
+/* ------------ Internal support for realloc, memalign, etc -------------- */
+
+/* Try to realloc; only in-place unless can_move true */
+static mchunkptr try_realloc_chunk(mstate m, mchunkptr p, size_t nb,
+                                   int can_move) {
+  mchunkptr newp = 0;
+  size_t oldsize = chunksize(p);
+  mchunkptr next = chunk_plus_offset(p, oldsize);
+  if (RTCHECK(ok_address(m, p) && ok_inuse(p) &&
+              ok_next(p, next) && ok_pinuse(next))) {
+    if (is_mmapped(p)) {
+      newp = mmap_resize(m, p, nb, can_move);
+    }
+    else if (oldsize >= nb) {             /* already big enough */
+      size_t rsize = oldsize - nb;
+      if (rsize >= MIN_CHUNK_SIZE) {      /* split off remainder */
+        mchunkptr r = chunk_plus_offset(p, nb);
+        set_inuse(m, p, nb);
+        set_inuse(m, r, rsize);
+        dispose_chunk(m, r, rsize);
+      }
+      newp = p;
+    }
+    else if (next == m->top) {  /* extend into top */
+      if (oldsize + m->topsize > nb) {
+        size_t newsize = oldsize + m->topsize;
+        size_t newtopsize = newsize - nb;
+        mchunkptr newtop = chunk_plus_offset(p, nb);
+        set_inuse(m, p, nb);
+        newtop->head = newtopsize |PINUSE_BIT;
+        m->top = newtop;
+        m->topsize = newtopsize;
+        newp = p;
+      }
+    }
+    else if (next == m->dv) { /* extend into dv */
+      size_t dvs = m->dvsize;
+      if (oldsize + dvs >= nb) {
+        size_t dsize = oldsize + dvs - nb;
+        if (dsize >= MIN_CHUNK_SIZE) {
+          mchunkptr r = chunk_plus_offset(p, nb);
+          mchunkptr n = chunk_plus_offset(r, dsize);
+          set_inuse(m, p, nb);
+          set_size_and_pinuse_of_free_chunk(r, dsize);
+          clear_pinuse(n);
+          m->dvsize = dsize;
+          m->dv = r;
+        }
+        else { /* exhaust dv */
+          size_t newsize = oldsize + dvs;
+          set_inuse(m, p, newsize);
+          m->dvsize = 0;
+          m->dv = 0;
+        }
+        newp = p;
+      }
+    }
+    else if (!cinuse(next)) { /* extend into next free chunk */
+      size_t nextsize = chunksize(next);
+      if (oldsize + nextsize >= nb) {
+        size_t rsize = oldsize + nextsize - nb;
+        unlink_chunk(m, next, nextsize);
+        if (rsize < MIN_CHUNK_SIZE) {
+          size_t newsize = oldsize + nextsize;
+          set_inuse(m, p, newsize);
+        }
+        else {
+          mchunkptr r = chunk_plus_offset(p, nb);
+          set_inuse(m, p, nb);
+          set_inuse(m, r, rsize);
+          dispose_chunk(m, r, rsize);
+        }
+        newp = p;
+      }
+    }
+  }
+  else {
+    USAGE_ERROR_ACTION(m, chunk2mem(p));
+  }
+  return newp;
+}
+
+static void* internal_memalign(mstate m, size_t alignment, size_t bytes) {
+  void* mem = 0;
+  if (alignment <  MIN_CHUNK_SIZE) /* must be at least a minimum chunk size */
+    alignment = MIN_CHUNK_SIZE;
+  if ((alignment & (alignment-SIZE_T_ONE)) != 0) {/* Ensure a power of 2 */
+    size_t a = MALLOC_ALIGNMENT << 1;
+    while (a < alignment) a <<= 1;
+    alignment = a;
+  }
+  if (bytes >= MAX_REQUEST - alignment) {
+    if (m != 0)  { /* Test isn't needed but avoids compiler warning */
+      MALLOC_FAILURE_ACTION;
+    }
+  }
+  else {
+    size_t nb = request2size(bytes);
+    size_t req = nb + alignment + MIN_CHUNK_SIZE - CHUNK_OVERHEAD;
+    mem = internal_malloc(m, req);
+    if (mem != 0) {
+      mchunkptr p = mem2chunk(mem);
+      if (PREACTION(m))
+        return 0;
+      if ((((size_t)(mem)) & (alignment - 1)) != 0) { /* misaligned */
+        /*
+          Find an aligned spot inside chunk.  Since we need to give
+          back leading space in a chunk of at least MIN_CHUNK_SIZE, if
+          the first calculation places us at a spot with less than
+          MIN_CHUNK_SIZE leader, we can move to the next aligned spot.
+          We've allocated enough total room so that this is always
+          possible.
+        */
+        char* br = (char*)mem2chunk((size_t)(((size_t)((char*)mem + alignment -
+                                                       SIZE_T_ONE)) &
+                                             -alignment));
+        char* pos = ((size_t)(br - (char*)(p)) >= MIN_CHUNK_SIZE)?
+          br : br+alignment;
+        mchunkptr newp = (mchunkptr)pos;
+        size_t leadsize = pos - (char*)(p);
+        size_t newsize = chunksize(p) - leadsize;
+
+        if (is_mmapped(p)) { /* For mmapped chunks, just adjust offset */
+          newp->prev_foot = p->prev_foot + leadsize;
+          newp->head = newsize;
+        }
+        else { /* Otherwise, give back leader, use the rest */
+          set_inuse(m, newp, newsize);
+          set_inuse(m, p, leadsize);
+          dispose_chunk(m, p, leadsize);
+        }
+        p = newp;
+      }
+
+      /* Give back spare room at the end */
+      if (!is_mmapped(p)) {
+        size_t size = chunksize(p);
+        if (size > nb + MIN_CHUNK_SIZE) {
+          size_t remainder_size = size - nb;
+          mchunkptr remainder = chunk_plus_offset(p, nb);
+          set_inuse(m, p, nb);
+          set_inuse(m, remainder, remainder_size);
+          dispose_chunk(m, remainder, remainder_size);
+        }
+      }
+
+      mem = chunk2mem(p);
+      assert (chunksize(p) >= nb);
+      assert(((size_t)mem & (alignment - 1)) == 0);
+      check_inuse_chunk(m, p);
+      POSTACTION(m);
+    }
+  }
+  return mem;
+}
+
+/*
+  Common support for independent_X routines, handling
+    all of the combinations that can result.
+  The opts arg has:
+    bit 0 set if all elements are same size (using sizes[0])
+    bit 1 set if elements should be zeroed
+*/
+static void** ialloc(mstate m,
+                     size_t n_elements,
+                     size_t* sizes,
+                     int opts,
+                     void* chunks[]) {
+
+  size_t    element_size;   /* chunksize of each element, if all same */
+  size_t    contents_size;  /* total size of elements */
+  size_t    array_size;     /* request size of pointer array */
+  void*     mem;            /* malloced aggregate space */
+  mchunkptr p;              /* corresponding chunk */
+  size_t    remainder_size; /* remaining bytes while splitting */
+  void**    marray;         /* either "chunks" or malloced ptr array */
+  mchunkptr array_chunk;    /* chunk for malloced ptr array */
+  flag_t    was_enabled;    /* to disable mmap */
+  size_t    size;
+  size_t    i;
+
+  ensure_initialization();
+  /* compute array length, if needed */
+  if (chunks != 0) {
+    if (n_elements == 0)
+      return chunks; /* nothing to do */
+    marray = chunks;
+    array_size = 0;
+  }
+  else {
+    /* if empty req, must still return chunk representing empty array */
+    if (n_elements == 0)
+      return (void**)internal_malloc(m, 0);
+    marray = 0;
+    array_size = request2size(n_elements * (sizeof(void*)));
+  }
+
+  /* compute total element size */
+  if (opts & 0x1) { /* all-same-size */
+    element_size = request2size(*sizes);
+    contents_size = n_elements * element_size;
+  }
+  else { /* add up all the sizes */
+    element_size = 0;
+    contents_size = 0;
+    for (i = 0; i != n_elements; ++i)
+      contents_size += request2size(sizes[i]);
+  }
+
+  size = contents_size + array_size;
+
+  /*
+     Allocate the aggregate chunk.  First disable direct-mmapping so
+     malloc won't use it, since we would not be able to later
+     free/realloc space internal to a segregated mmap region.
+  */
+  was_enabled = use_mmap(m);
+  disable_mmap(m);
+  mem = internal_malloc(m, size - CHUNK_OVERHEAD);
+  if (was_enabled)
+    enable_mmap(m);
+  if (mem == 0)
+    return 0;
+
+  if (PREACTION(m)) return 0;
+  p = mem2chunk(mem);
+  remainder_size = chunksize(p);
+
+  assert(!is_mmapped(p));
+
+  if (opts & 0x2) {       /* optionally clear the elements */
+    memset((size_t*)mem, 0, remainder_size - SIZE_T_SIZE - array_size);
+  }
+
+  /* If not provided, allocate the pointer array as final part of chunk */
+  if (marray == 0) {
+    size_t  array_chunk_size;
+    array_chunk = chunk_plus_offset(p, contents_size);
+    array_chunk_size = remainder_size - contents_size;
+    marray = (void**) (chunk2mem(array_chunk));
+    set_size_and_pinuse_of_inuse_chunk(m, array_chunk, array_chunk_size);
+    remainder_size = contents_size;
+  }
+
+  /* split out elements */
+  for (i = 0; ; ++i) {
+    marray[i] = chunk2mem(p);
+    if (i != n_elements-1) {
+      if (element_size != 0)
+        size = element_size;
+      else
+        size = request2size(sizes[i]);
+      remainder_size -= size;
+      set_size_and_pinuse_of_inuse_chunk(m, p, size);
+      p = chunk_plus_offset(p, size);
+    }
+    else { /* the final element absorbs any overallocation slop */
+      set_size_and_pinuse_of_inuse_chunk(m, p, remainder_size);
+      break;
+    }
+  }
+
+#if DEBUG
+  if (marray != chunks) {
+    /* final element must have exactly exhausted chunk */
+    if (element_size != 0) {
+      assert(remainder_size == element_size);
+    }
+    else {
+      assert(remainder_size == request2size(sizes[i]));
+    }
+    check_inuse_chunk(m, mem2chunk(marray));
+  }
+  for (i = 0; i != n_elements; ++i)
+    check_inuse_chunk(m, mem2chunk(marray[i]));
+
+#endif /* DEBUG */
+
+  POSTACTION(m);
+  return marray;
+}
+
+/* Try to free all pointers in the given array.
+   Note: this could be made faster, by delaying consolidation,
+   at the price of disabling some user integrity checks, We
+   still optimize some consolidations by combining adjacent
+   chunks before freeing, which will occur often if allocated
+   with ialloc or the array is sorted.
+*/
+static size_t internal_bulk_free(mstate m, void* array[], size_t nelem) {
+  size_t unfreed = 0;
+  if (!PREACTION(m)) {
+    void** a;
+    void** fence = &(array[nelem]);
+    for (a = array; a != fence; ++a) {
+      void* mem = *a;
+      if (mem != 0) {
+        mchunkptr p = mem2chunk(mem);
+        size_t psize = chunksize(p);
+#if FOOTERS
+        if (get_mstate_for(p) != m) {
+          ++unfreed;
+          continue;
+        }
+#endif
+        check_inuse_chunk(m, p);
+        *a = 0;
+        if (RTCHECK(ok_address(m, p) && ok_inuse(p))) {
+          void ** b = a + 1; /* try to merge with next chunk */
+          mchunkptr next = next_chunk(p);
+          if (b != fence && *b == chunk2mem(next)) {
+            size_t newsize = chunksize(next) + psize;
+            set_inuse(m, p, newsize);
+            *b = chunk2mem(p);
+          }
+          else
+            dispose_chunk(m, p, psize);
+        }
+        else {
+          CORRUPTION_ERROR_ACTION(m);
+          break;
+        }
+      }
+    }
+    if (should_trim(m, m->topsize))
+      sys_trim(m, 0);
+    POSTACTION(m);
+  }
+  return unfreed;
+}
+
+/* Traversal */
+#if MALLOC_INSPECT_ALL
+static void internal_inspect_all(mstate m,
+                                 void(*handler)(void *start,
+                                                void *end,
+                                                size_t used_bytes,
+                                                void* callback_arg),
+                                 void* arg) {
+  if (is_initialized(m)) {
+    mchunkptr top = m->top;
+    msegmentptr s;
+    for (s = &m->seg; s != 0; s = s->next) {
+      mchunkptr q = align_as_chunk(s->base);
+      while (segment_holds(s, q) && q->head != FENCEPOST_HEAD) {
+        mchunkptr next = next_chunk(q);
+        size_t sz = chunksize(q);
+        size_t used;
+        void* start;
+        if (is_inuse(q)) {
+          used = sz - CHUNK_OVERHEAD; /* must not be mmapped */
+          start = chunk2mem(q);
+        }
+        else {
+          used = 0;
+          if (is_small(sz)) {     /* offset by possible bookkeeping */
+            start = (void*)((char*)q + sizeof(struct malloc_chunk));
+          }
+          else {
+            start = (void*)((char*)q + sizeof(struct malloc_tree_chunk));
+          }
+        }
+        if (start < (void*)next)  /* skip if all space is bookkeeping */
+          handler(start, next, used, arg);
+        if (q == top)
+          break;
+        q = next;
+      }
+    }
+  }
+}
+#endif /* MALLOC_INSPECT_ALL */
+
+/* ------------------ Exported realloc, memalign, etc -------------------- */
+
+#if !ONLY_MSPACES
+
+void* dlrealloc(void* oldmem, size_t bytes) {
+  void* mem = 0;
+  if (oldmem == 0) {
+    mem = dlmalloc(bytes);
+  }
+  else if (bytes >= MAX_REQUEST) {
+    MALLOC_FAILURE_ACTION;
+  }
+#ifdef REALLOC_ZERO_BYTES_FREES
+  else if (bytes == 0) {
+    dlfree(oldmem);
+  }
+#endif /* REALLOC_ZERO_BYTES_FREES */
+  else {
+    size_t nb = request2size(bytes);
+    mchunkptr oldp = mem2chunk(oldmem);
+#if ! FOOTERS
+    mstate m = gm;
+#else /* FOOTERS */
+    mstate m = get_mstate_for(oldp);
+    if (!ok_magic(m)) {
+      USAGE_ERROR_ACTION(m, oldmem);
+      return 0;
+    }
+#endif /* FOOTERS */
+    if (!PREACTION(m)) {
+      mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1);
+      POSTACTION(m);
+      if (newp != 0) {
+        check_inuse_chunk(m, newp);
+        mem = chunk2mem(newp);
+      }
+      else {
+        mem = internal_malloc(m, bytes);
+        if (mem != 0) {
+          size_t oc = chunksize(oldp) - overhead_for(oldp);
+          memcpy(mem, oldmem, (oc < bytes)? oc : bytes);
+          internal_free(m, oldmem);
+        }
+      }
+    }
+  }
+  return mem;
+}
+
+void* dlrealloc_in_place(void* oldmem, size_t bytes) {
+  void* mem = 0;
+  if (oldmem != 0) {
+    if (bytes >= MAX_REQUEST) {
+      MALLOC_FAILURE_ACTION;
+    }
+    else {
+      size_t nb = request2size(bytes);
+      mchunkptr oldp = mem2chunk(oldmem);
+#if ! FOOTERS
+      mstate m = gm;
+#else /* FOOTERS */
+      mstate m = get_mstate_for(oldp);
+      if (!ok_magic(m)) {
+        USAGE_ERROR_ACTION(m, oldmem);
+        return 0;
+      }
+#endif /* FOOTERS */
+      if (!PREACTION(m)) {
+        mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
+        POSTACTION(m);
+        if (newp == oldp) {
+          check_inuse_chunk(m, newp);
+          mem = oldmem;
+        }
+      }
+    }
+  }
+  return mem;
+}
+
+void* dlmemalign(size_t alignment, size_t bytes) {
+  if (alignment <= MALLOC_ALIGNMENT) {
+    return dlmalloc(bytes);
+  }
+  return internal_memalign(gm, alignment, bytes);
+}
+
+int dlposix_memalign(void** pp, size_t alignment, size_t bytes) {
+  void* mem = 0;
+  if (alignment == MALLOC_ALIGNMENT)
+    mem = dlmalloc(bytes);
+  else {
+    size_t d = alignment / sizeof(void*);
+    size_t r = alignment % sizeof(void*);
+    if (r != 0 || d == 0 || (d & (d-SIZE_T_ONE)) != 0)
+      return EINVAL;
+    else if (bytes <= MAX_REQUEST - alignment) {
+      if (alignment <  MIN_CHUNK_SIZE)
+        alignment = MIN_CHUNK_SIZE;
+      mem = internal_memalign(gm, alignment, bytes);
+    }
+  }
+  if (mem == 0)
+    return ENOMEM;
+  else {
+    *pp = mem;
+    return 0;
+  }
+}
+
+void* dlvalloc(size_t bytes) {
+  size_t pagesz;
+  ensure_initialization();
+  pagesz = mparams.page_size;
+  return dlmemalign(pagesz, bytes);
+}
+
+void* dlpvalloc(size_t bytes) {
+  size_t pagesz;
+  ensure_initialization();
+  pagesz = mparams.page_size;
+  return dlmemalign(pagesz, (bytes + pagesz - SIZE_T_ONE) & ~(pagesz - SIZE_T_ONE));
+}
+
+void** dlindependent_calloc(size_t n_elements, size_t elem_size,
+                            void* chunks[]) {
+  size_t sz = elem_size; /* serves as 1-element array */
+  return ialloc(gm, n_elements, &sz, 3, chunks);
+}
+
+void** dlindependent_comalloc(size_t n_elements, size_t sizes[],
+                              void* chunks[]) {
+  return ialloc(gm, n_elements, sizes, 0, chunks);
+}
+
+size_t dlbulk_free(void* array[], size_t nelem) {
+  return internal_bulk_free(gm, array, nelem);
+}
+
+#if MALLOC_INSPECT_ALL
+void dlmalloc_inspect_all(void(*handler)(void *start,
+                                         void *end,
+                                         size_t used_bytes,
+                                         void* callback_arg),
+                          void* arg) {
+  ensure_initialization();
+  if (!PREACTION(gm)) {
+    internal_inspect_all(gm, handler, arg);
+    POSTACTION(gm);
+  }
+}
+#endif /* MALLOC_INSPECT_ALL */
+
+int dlmalloc_trim(size_t pad) {
+  int result = 0;
+  ensure_initialization();
+  if (!PREACTION(gm)) {
+    result = sys_trim(gm, pad);
+    POSTACTION(gm);
+  }
+  return result;
+}
+
+size_t dlmalloc_footprint(void) {
+  return gm->footprint;
+}
+
+size_t dlmalloc_max_footprint(void) {
+  return gm->max_footprint;
+}
+
+size_t dlmalloc_footprint_limit(void) {
+  size_t maf = gm->footprint_limit;
+  return maf == 0 ? MAX_SIZE_T : maf;
+}
+
+size_t dlmalloc_set_footprint_limit(size_t bytes) {
+  ensure_initialization();
+  size_t result;  /* invert sense of 0 */
+  if (bytes == 0)
+    result = granularity_align(1); /* Use minimal size */
+  if (bytes == MAX_SIZE_T)
+    result = 0;                    /* disable */
+  else
+    result = granularity_align(bytes);
+  return gm->footprint_limit = result;
+}
+
+#if !NO_MALLINFO
+struct mallinfo dlmallinfo(void) {
+  return internal_mallinfo(gm);
+}
+#endif /* NO_MALLINFO */
+
+#if !NO_MALLOC_STATS
+void dlmalloc_stats() {
+  internal_malloc_stats(gm);
+}
+#endif /* NO_MALLOC_STATS */
+
+int dlmallopt(int param_number, int value) {
+  return change_mparam(param_number, value);
+}
+
+size_t dlmalloc_usable_size(void* mem) {
+  if (mem != 0) {
+    mchunkptr p = mem2chunk(mem);
+    if (is_inuse(p))
+      return chunksize(p) - overhead_for(p);
+  }
+  return 0;
+}
+
+#endif /* !ONLY_MSPACES */
+
+/* ----------------------------- user mspaces ---------------------------- */
+
+#if MSPACES
+
+static mstate init_user_mstate(char* tbase, size_t tsize) {
+  size_t msize = pad_request(sizeof(struct malloc_state));
+  mchunkptr mn;
+  mchunkptr msp = align_as_chunk(tbase);
+  mstate m = (mstate)(chunk2mem(msp));
+  memset(m, 0, msize);
+  (void)INITIAL_LOCK(&m->mutex);
+  msp->head = (msize|INUSE_BITS);
+  m->seg.base = m->least_addr = tbase;
+  m->seg.size = m->footprint = m->max_footprint = tsize;
+  m->magic = mparams.magic;
+  m->release_checks = MAX_RELEASE_CHECK_RATE;
+  m->mflags = mparams.default_mflags;
+  m->extp = 0;
+  m->exts = 0;
+  disable_contiguous(m);
+  init_bins(m);
+  mn = next_chunk(mem2chunk(m));
+  init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) - TOP_FOOT_SIZE);
+  check_top_chunk(m, m->top);
+  return m;
+}
+
+mspace create_mspace(size_t capacity, int locked) {
+  mstate m = 0;
+  size_t msize;
+  ensure_initialization();
+  msize = pad_request(sizeof(struct malloc_state));
+  if (capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) {
+    size_t rs = ((capacity == 0)? mparams.granularity :
+                 (capacity + TOP_FOOT_SIZE + msize));
+    size_t tsize = granularity_align(rs);
+    char* tbase = (char*)(CALL_MMAP(tsize));
+    if (tbase != CMFAIL) {
+      m = init_user_mstate(tbase, tsize);
+      m->seg.sflags = USE_MMAP_BIT;
+      set_lock(m, locked);
+    }
+  }
+  return (mspace)m;
+}
+
+mspace create_mspace_with_base(void* base, size_t capacity, int locked) {
+  mstate m = 0;
+  size_t msize;
+  ensure_initialization();
+  msize = pad_request(sizeof(struct malloc_state));
+  if (capacity > msize + TOP_FOOT_SIZE &&
+      capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) {
+    m = init_user_mstate((char*)base, capacity);
+    m->seg.sflags = EXTERN_BIT;
+    set_lock(m, locked);
+  }
+  return (mspace)m;
+}
+
+int mspace_track_large_chunks(mspace msp, int enable) {
+  int ret = 0;
+  mstate ms = (mstate)msp;
+  if (!PREACTION(ms)) {
+    if (!use_mmap(ms)) {
+      ret = 1;
+    }
+    if (!enable) {
+      enable_mmap(ms);
+    } else {
+      disable_mmap(ms);
+    }
+    POSTACTION(ms);
+  }
+  return ret;
+}
+
+size_t destroy_mspace(mspace msp) {
+  size_t freed = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    msegmentptr sp = &ms->seg;
+    (void)DESTROY_LOCK(&ms->mutex); /* destroy before unmapped */
+    while (sp != 0) {
+      char* base = sp->base;
+      size_t size = sp->size;
+      flag_t flag = sp->sflags;
+      (void)base; /* placate people compiling -Wunused-variable */
+      sp = sp->next;
+      if ((flag & USE_MMAP_BIT) && !(flag & EXTERN_BIT) &&
+          CALL_MUNMAP(base, size) == 0)
+        freed += size;
+    }
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return freed;
+}
+
+/*
+  mspace versions of routines are near-clones of the global
+  versions. This is not so nice but better than the alternatives.
+*/
+
+void* mspace_malloc(mspace msp, size_t bytes) {
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  if (!PREACTION(ms)) {
+    void* mem;
+    size_t nb;
+    if (bytes <= MAX_SMALL_REQUEST) {
+      bindex_t idx;
+      binmap_t smallbits;
+      nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes);
+      idx = small_index(nb);
+      smallbits = ms->smallmap >> idx;
+
+      if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */
+        mchunkptr b, p;
+        idx += ~smallbits & 1;       /* Uses next bin if idx empty */
+        b = smallbin_at(ms, idx);
+        p = b->fd;
+        assert(chunksize(p) == small_index2size(idx));
+        unlink_first_small_chunk(ms, b, p, idx);
+        set_inuse_and_pinuse(ms, p, small_index2size(idx));
+        mem = chunk2mem(p);
+        check_malloced_chunk(ms, mem, nb);
+        goto postaction;
+      }
+
+      else if (nb > ms->dvsize) {
+        if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
+          mchunkptr b, p, r;
+          size_t rsize;
+          bindex_t i;
+          binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx));
+          binmap_t leastbit = least_bit(leftbits);
+          compute_bit2idx(leastbit, i);
+          b = smallbin_at(ms, i);
+          p = b->fd;
+          assert(chunksize(p) == small_index2size(i));
+          unlink_first_small_chunk(ms, b, p, i);
+          rsize = small_index2size(i) - nb;
+          /* Fit here cannot be remainderless if 4byte sizes */
+          if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE)
+            set_inuse_and_pinuse(ms, p, small_index2size(i));
+          else {
+            set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
+            r = chunk_plus_offset(p, nb);
+            set_size_and_pinuse_of_free_chunk(r, rsize);
+            replace_dv(ms, r, rsize);
+          }
+          mem = chunk2mem(p);
+          check_malloced_chunk(ms, mem, nb);
+          goto postaction;
+        }
+
+        else if (ms->treemap != 0 && (mem = tmalloc_small(ms, nb)) != 0) {
+          check_malloced_chunk(ms, mem, nb);
+          goto postaction;
+        }
+      }
+    }
+    else if (bytes >= MAX_REQUEST)
+      nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */
+    else {
+      nb = pad_request(bytes);
+      if (ms->treemap != 0 && (mem = tmalloc_large(ms, nb)) != 0) {
+        check_malloced_chunk(ms, mem, nb);
+        goto postaction;
+      }
+    }
+
+    if (nb <= ms->dvsize) {
+      size_t rsize = ms->dvsize - nb;
+      mchunkptr p = ms->dv;
+      if (rsize >= MIN_CHUNK_SIZE) { /* split dv */
+        mchunkptr r = ms->dv = chunk_plus_offset(p, nb);
+        ms->dvsize = rsize;
+        set_size_and_pinuse_of_free_chunk(r, rsize);
+        set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
+      }
+      else { /* exhaust dv */
+        size_t dvs = ms->dvsize;
+        ms->dvsize = 0;
+        ms->dv = 0;
+        set_inuse_and_pinuse(ms, p, dvs);
+      }
+      mem = chunk2mem(p);
+      check_malloced_chunk(ms, mem, nb);
+      goto postaction;
+    }
+
+    else if (nb < ms->topsize) { /* Split top */
+      size_t rsize = ms->topsize -= nb;
+      mchunkptr p = ms->top;
+      mchunkptr r = ms->top = chunk_plus_offset(p, nb);
+      r->head = rsize | PINUSE_BIT;
+      set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
+      mem = chunk2mem(p);
+      check_top_chunk(ms, ms->top);
+      check_malloced_chunk(ms, mem, nb);
+      goto postaction;
+    }
+
+    mem = sys_alloc(ms, nb);
+
+  postaction:
+    POSTACTION(ms);
+    return mem;
+  }
+
+  return 0;
+}
+
+void mspace_free(mspace msp, void* mem) {
+  if (mem != 0) {
+    mchunkptr p  = mem2chunk(mem);
+#if FOOTERS
+    mstate fm = get_mstate_for(p);
+    (void)msp; /* placate people compiling -Wunused */
+#else /* FOOTERS */
+    mstate fm = (mstate)msp;
+#endif /* FOOTERS */
+    if (!ok_magic(fm)) {
+      USAGE_ERROR_ACTION(fm, p);
+      return;
+    }
+    if (!PREACTION(fm)) {
+      check_inuse_chunk(fm, p);
+      if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) {
+        size_t psize = chunksize(p);
+        mchunkptr next = chunk_plus_offset(p, psize);
+        if (!pinuse(p)) {
+          size_t prevsize = p->prev_foot;
+          if (is_mmapped(p)) {
+            psize += prevsize + MMAP_FOOT_PAD;
+            if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
+              fm->footprint -= psize;
+            goto postaction;
+          }
+          else {
+            mchunkptr prev = chunk_minus_offset(p, prevsize);
+            psize += prevsize;
+            p = prev;
+            if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */
+              if (p != fm->dv) {
+                unlink_chunk(fm, p, prevsize);
+              }
+              else if ((next->head & INUSE_BITS) == INUSE_BITS) {
+                fm->dvsize = psize;
+                set_free_with_pinuse(p, psize, next);
+                goto postaction;
+              }
+            }
+            else
+              goto erroraction;
+          }
+        }
+
+        if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) {
+          if (!cinuse(next)) {  /* consolidate forward */
+            if (next == fm->top) {
+              size_t tsize = fm->topsize += psize;
+              fm->top = p;
+              p->head = tsize | PINUSE_BIT;
+              if (p == fm->dv) {
+                fm->dv = 0;
+                fm->dvsize = 0;
+              }
+              if (should_trim(fm, tsize))
+                sys_trim(fm, 0);
+              goto postaction;
+            }
+            else if (next == fm->dv) {
+              size_t dsize = fm->dvsize += psize;
+              fm->dv = p;
+              set_size_and_pinuse_of_free_chunk(p, dsize);
+              goto postaction;
+            }
+            else {
+              size_t nsize = chunksize(next);
+              psize += nsize;
+              unlink_chunk(fm, next, nsize);
+              set_size_and_pinuse_of_free_chunk(p, psize);
+              if (p == fm->dv) {
+                fm->dvsize = psize;
+                goto postaction;
+              }
+            }
+          }
+          else
+            set_free_with_pinuse(p, psize, next);
+
+          if (is_small(psize)) {
+            insert_small_chunk(fm, p, psize);
+            check_free_chunk(fm, p);
+          }
+          else {
+            tchunkptr tp = (tchunkptr)p;
+            insert_large_chunk(fm, tp, psize);
+            check_free_chunk(fm, p);
+            if (--fm->release_checks == 0)
+              release_unused_segments(fm);
+          }
+          goto postaction;
+        }
+      }
+    erroraction:
+      USAGE_ERROR_ACTION(fm, p);
+    postaction:
+      POSTACTION(fm);
+    }
+  }
+}
+
+void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size) {
+  void* mem;
+  size_t req = 0;
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  if (n_elements != 0) {
+    req = n_elements * elem_size;
+    if (((n_elements | elem_size) & ~(size_t)0xffff) &&
+        (req / n_elements != elem_size))
+      req = MAX_SIZE_T; /* force downstream failure on overflow */
+  }
+  mem = internal_malloc(ms, req);
+  if (mem != 0 && calloc_must_clear(mem2chunk(mem)))
+    memset(mem, 0, req);
+  return mem;
+}
+
+void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) {
+  void* mem = 0;
+  if (oldmem == 0) {
+    mem = mspace_malloc(msp, bytes);
+  }
+  else if (bytes >= MAX_REQUEST) {
+    MALLOC_FAILURE_ACTION;
+  }
+#ifdef REALLOC_ZERO_BYTES_FREES
+  else if (bytes == 0) {
+    mspace_free(msp, oldmem);
+  }
+#endif /* REALLOC_ZERO_BYTES_FREES */
+  else {
+    size_t nb = request2size(bytes);
+    mchunkptr oldp = mem2chunk(oldmem);
+#if ! FOOTERS
+    mstate m = (mstate)msp;
+#else /* FOOTERS */
+    mstate m = get_mstate_for(oldp);
+    if (!ok_magic(m)) {
+      USAGE_ERROR_ACTION(m, oldmem);
+      return 0;
+    }
+#endif /* FOOTERS */
+    if (!PREACTION(m)) {
+      mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1);
+      POSTACTION(m);
+      if (newp != 0) {
+        check_inuse_chunk(m, newp);
+        mem = chunk2mem(newp);
+      }
+      else {
+        mem = mspace_malloc(m, bytes);
+        if (mem != 0) {
+          size_t oc = chunksize(oldp) - overhead_for(oldp);
+          memcpy(mem, oldmem, (oc < bytes)? oc : bytes);
+          mspace_free(m, oldmem);
+        }
+      }
+    }
+  }
+  return mem;
+}
+
+void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) {
+  void* mem = 0;
+  if (oldmem != 0) {
+    if (bytes >= MAX_REQUEST) {
+      MALLOC_FAILURE_ACTION;
+    }
+    else {
+      size_t nb = request2size(bytes);
+      mchunkptr oldp = mem2chunk(oldmem);
+#if ! FOOTERS
+      mstate m = (mstate)msp;
+#else /* FOOTERS */
+      mstate m = get_mstate_for(oldp);
+      (void)msp; /* placate people compiling -Wunused */
+      if (!ok_magic(m)) {
+        USAGE_ERROR_ACTION(m, oldmem);
+        return 0;
+      }
+#endif /* FOOTERS */
+      if (!PREACTION(m)) {
+        mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
+        POSTACTION(m);
+        if (newp == oldp) {
+          check_inuse_chunk(m, newp);
+          mem = oldmem;
+        }
+      }
+    }
+  }
+  return mem;
+}
+
+void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) {
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  if (alignment <= MALLOC_ALIGNMENT)
+    return mspace_malloc(msp, bytes);
+  return internal_memalign(ms, alignment, bytes);
+}
+
+void** mspace_independent_calloc(mspace msp, size_t n_elements,
+                                 size_t elem_size, void* chunks[]) {
+  size_t sz = elem_size; /* serves as 1-element array */
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  return ialloc(ms, n_elements, &sz, 3, chunks);
+}
+
+void** mspace_independent_comalloc(mspace msp, size_t n_elements,
+                                   size_t sizes[], void* chunks[]) {
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  return ialloc(ms, n_elements, sizes, 0, chunks);
+}
+
+size_t mspace_bulk_free(mspace msp, void* array[], size_t nelem) {
+  return internal_bulk_free((mstate)msp, array, nelem);
+}
+
+#if MALLOC_INSPECT_ALL
+void mspace_inspect_all(mspace msp,
+                        void(*handler)(void *start,
+                                       void *end,
+                                       size_t used_bytes,
+                                       void* callback_arg),
+                        void* arg) {
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    if (!PREACTION(ms)) {
+      internal_inspect_all(ms, handler, arg);
+      POSTACTION(ms);
+    }
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+}
+#endif /* MALLOC_INSPECT_ALL */
+
+int mspace_trim(mspace msp, size_t pad) {
+  int result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    if (!PREACTION(ms)) {
+      result = sys_trim(ms, pad);
+      POSTACTION(ms);
+    }
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+#if !NO_MALLOC_STATS
+void mspace_malloc_stats(mspace msp) {
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    internal_malloc_stats(ms);
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+}
+#endif /* NO_MALLOC_STATS */
+
+size_t mspace_footprint(mspace msp) {
+  size_t result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    result = ms->footprint;
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+size_t mspace_max_footprint(mspace msp) {
+  size_t result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    result = ms->max_footprint;
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+size_t mspace_footprint_limit(mspace msp) {
+  size_t result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    size_t maf = ms->footprint_limit;
+    result = (maf == 0) ? MAX_SIZE_T : maf;
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+size_t mspace_set_footprint_limit(mspace msp, size_t bytes) {
+  size_t result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    if (bytes == 0)
+      result = granularity_align(1); /* Use minimal size */
+    if (bytes == MAX_SIZE_T)
+      result = 0;                    /* disable */
+    else
+      result = granularity_align(bytes);
+    ms->footprint_limit = result;
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+#if !NO_MALLINFO
+struct mallinfo mspace_mallinfo(mspace msp) {
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return internal_mallinfo(ms);
+}
+#endif /* NO_MALLINFO */
+
+size_t mspace_usable_size(const void* mem) {
+  if (mem != 0) {
+    mchunkptr p = mem2chunk(mem);
+    if (is_inuse(p))
+      return chunksize(p) - overhead_for(p);
+  }
+  return 0;
+}
+
+int mspace_mallopt(int param_number, int value) {
+  return change_mparam(param_number, value);
+}
+
+#endif /* MSPACES */
+
+
+/* -------------------- Alternative MORECORE functions ------------------- */
+
+/*
+  Guidelines for creating a custom version of MORECORE:
+
+  * For best performance, MORECORE should allocate in multiples of pagesize.
+  * MORECORE may allocate more memory than requested. (Or even less,
+      but this will usually result in a malloc failure.)
+  * MORECORE must not allocate memory when given argument zero, but
+      instead return one past the end address of memory from previous
+      nonzero call.
+  * For best performance, consecutive calls to MORECORE with positive
+      arguments should return increasing addresses, indicating that
+      space has been contiguously extended.
+  * Even though consecutive calls to MORECORE need not return contiguous
+      addresses, it must be OK for malloc'ed chunks to span multiple
+      regions in those cases where they do happen to be contiguous.
+  * MORECORE need not handle negative arguments -- it may instead
+      just return MFAIL when given negative arguments.
+      Negative arguments are always multiples of pagesize. MORECORE
+      must not misinterpret negative args as large positive unsigned
+      args. You can suppress all such calls from even occurring by defining
+      MORECORE_CANNOT_TRIM,
+
+  As an example alternative MORECORE, here is a custom allocator
+  kindly contributed for pre-OSX macOS.  It uses virtually but not
+  necessarily physically contiguous non-paged memory (locked in,
+  present and won't get swapped out).  You can use it by uncommenting
+  this section, adding some #includes, and setting up the appropriate
+  defines above:
+
+      #define MORECORE osMoreCore
+
+  There is also a shutdown routine that should somehow be called for
+  cleanup upon program exit.
+
+  #define MAX_POOL_ENTRIES 100
+  #define MINIMUM_MORECORE_SIZE  (64 * 1024U)
+  static int next_os_pool;
+  void *our_os_pools[MAX_POOL_ENTRIES];
+
+  void *osMoreCore(int size)
+  {
+    void *ptr = 0;
+    static void *sbrk_top = 0;
+
+    if (size > 0)
+    {
+      if (size < MINIMUM_MORECORE_SIZE)
+         size = MINIMUM_MORECORE_SIZE;
+      if (CurrentExecutionLevel() == kTaskLevel)
+         ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
+      if (ptr == 0)
+      {
+        return (void *) MFAIL;
+      }
+      // save ptrs so they can be freed during cleanup
+      our_os_pools[next_os_pool] = ptr;
+      next_os_pool++;
+      ptr = (void *) ((((size_t) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
+      sbrk_top = (char *) ptr + size;
+      return ptr;
+    }
+    else if (size < 0)
+    {
+      // we don't currently support shrink behavior
+      return (void *) MFAIL;
+    }
+    else
+    {
+      return sbrk_top;
+    }
+  }
+
+  // cleanup any allocated memory pools
+  // called as last thing before shutting down driver
+
+  void osCleanupMem(void)
+  {
+    void **ptr;
+
+    for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
+      if (*ptr)
+      {
+         PoolDeallocate(*ptr);
+         *ptr = 0;
+      }
+  }
+
+*/
+
+
+/* -----------------------------------------------------------------------
+History:
+    v2.8.6 Wed Aug 29 06:57:58 2012  Doug Lea
+      * fix bad comparison in dlposix_memalign
+      * don't reuse adjusted asize in sys_alloc
+      * add LOCK_AT_FORK -- thanks to Kirill Artamonov for the suggestion
+      * reduce compiler warnings -- thanks to all who reported/suggested these
+
+    v2.8.5 Sun May 22 10:26:02 2011  Doug Lea  (dl at gee)
+      * Always perform unlink checks unless INSECURE
+      * Add posix_memalign.
+      * Improve realloc to expand in more cases; expose realloc_in_place.
+        Thanks to Peter Buhr for the suggestion.
+      * Add footprint_limit, inspect_all, bulk_free. Thanks
+        to Barry Hayes and others for the suggestions.
+      * Internal refactorings to avoid calls while holding locks
+      * Use non-reentrant locks by default. Thanks to Roland McGrath
+        for the suggestion.
+      * Small fixes to mspace_destroy, reset_on_error.
+      * Various configuration extensions/changes. Thanks
+         to all who contributed these.
+
+    V2.8.4a Thu Apr 28 14:39:43 2011 (dl at gee.cs.oswego.edu)
+      * Update Creative Commons URL
+
+    V2.8.4 Wed May 27 09:56:23 2009  Doug Lea  (dl at gee)
+      * Use zeros instead of prev foot for is_mmapped
+      * Add mspace_track_large_chunks; thanks to Jean Brouwers
+      * Fix set_inuse in internal_realloc; thanks to Jean Brouwers
+      * Fix insufficient sys_alloc padding when using 16byte alignment
+      * Fix bad error check in mspace_footprint
+      * Adaptations for ptmalloc; thanks to Wolfram Gloger.
+      * Reentrant spin locks; thanks to Earl Chew and others
+      * Win32 improvements; thanks to Niall Douglas and Earl Chew
+      * Add NO_SEGMENT_TRAVERSAL and MAX_RELEASE_CHECK_RATE options
+      * Extension hook in malloc_state
+      * Various small adjustments to reduce warnings on some compilers
+      * Various configuration extensions/changes for more platforms. Thanks
+         to all who contributed these.
+
+    V2.8.3 Thu Sep 22 11:16:32 2005  Doug Lea  (dl at gee)
+      * Add max_footprint functions
+      * Ensure all appropriate literals are size_t
+      * Fix conditional compilation problem for some #define settings
+      * Avoid concatenating segments with the one provided
+        in create_mspace_with_base
+      * Rename some variables to avoid compiler shadowing warnings
+      * Use explicit lock initialization.
+      * Better handling of sbrk interference.
+      * Simplify and fix segment insertion, trimming and mspace_destroy
+      * Reinstate REALLOC_ZERO_BYTES_FREES option from 2.7.x
+      * Thanks especially to Dennis Flanagan for help on these.
+
+    V2.8.2 Sun Jun 12 16:01:10 2005  Doug Lea  (dl at gee)
+      * Fix memalign brace error.
+
+    V2.8.1 Wed Jun  8 16:11:46 2005  Doug Lea  (dl at gee)
+      * Fix improper #endif nesting in C++
+      * Add explicit casts needed for C++
+
+    V2.8.0 Mon May 30 14:09:02 2005  Doug Lea  (dl at gee)
+      * Use trees for large bins
+      * Support mspaces
+      * Use segments to unify sbrk-based and mmap-based system allocation,
+        removing need for emulation on most platforms without sbrk.
+      * Default safety checks
+      * Optional footer checks. Thanks to William Robertson for the idea.
+      * Internal code refactoring
+      * Incorporate suggestions and platform-specific changes.
+        Thanks to Dennis Flanagan, Colin Plumb, Niall Douglas,
+        Aaron Bachmann,  Emery Berger, and others.
+      * Speed up non-fastbin processing enough to remove fastbins.
+      * Remove useless cfree() to avoid conflicts with other apps.
+      * Remove internal memcpy, memset. Compilers handle builtins better.
+      * Remove some options that no one ever used and rename others.
+
+    V2.7.2 Sat Aug 17 09:07:30 2002  Doug Lea  (dl at gee)
+      * Fix malloc_state bitmap array misdeclaration
+
+    V2.7.1 Thu Jul 25 10:58:03 2002  Doug Lea  (dl at gee)
+      * Allow tuning of FIRST_SORTED_BIN_SIZE
+      * Use PTR_UINT as type for all ptr->int casts. Thanks to John Belmonte.
+      * Better detection and support for non-contiguousness of MORECORE.
+        Thanks to Andreas Mueller, Conal Walsh, and Wolfram Gloger
+      * Bypass most of malloc if no frees. Thanks To Emery Berger.
+      * Fix freeing of old top non-contiguous chunk im sysmalloc.
+      * Raised default trim and map thresholds to 256K.
+      * Fix mmap-related #defines. Thanks to Lubos Lunak.
+      * Fix copy macros; added LACKS_FCNTL_H. Thanks to Neal Walfield.
+      * Branch-free bin calculation
+      * Default trim and mmap thresholds now 256K.
+
+    V2.7.0 Sun Mar 11 14:14:06 2001  Doug Lea  (dl at gee)
+      * Introduce independent_comalloc and independent_calloc.
+        Thanks to Michael Pachos for motivation and help.
+      * Make optional .h file available
+      * Allow > 2GB requests on 32bit systems.
+      * new WIN32 sbrk, mmap, munmap, lock code from <Walter@GeNeSys-e.de>.
+        Thanks also to Andreas Mueller <a.mueller at paradatec.de>,
+        and Anonymous.
+      * Allow override of MALLOC_ALIGNMENT (Thanks to Ruud Waij for
+        helping test this.)
+      * memalign: check alignment arg
+      * realloc: don't try to shift chunks backwards, since this
+        leads to  more fragmentation in some programs and doesn't
+        seem to help in any others.
+      * Collect all cases in malloc requiring system memory into sysmalloc
+      * Use mmap as backup to sbrk
+      * Place all internal state in malloc_state
+      * Introduce fastbins (although similar to 2.5.1)
+      * Many minor tunings and cosmetic improvements
+      * Introduce USE_PUBLIC_MALLOC_WRAPPERS, USE_MALLOC_LOCK
+      * Introduce MALLOC_FAILURE_ACTION, MORECORE_CONTIGUOUS
+        Thanks to Tony E. Bennett <tbennett@nvidia.com> and others.
+      * Include errno.h to support default failure action.
+
+    V2.6.6 Sun Dec  5 07:42:19 1999  Doug Lea  (dl at gee)
+      * return null for negative arguments
+      * Added Several WIN32 cleanups from Martin C. Fong <mcfong at yahoo.com>
+         * Add 'LACKS_SYS_PARAM_H' for those systems without 'sys/param.h'
+          (e.g. WIN32 platforms)
+         * Cleanup header file inclusion for WIN32 platforms
+         * Cleanup code to avoid Microsoft Visual C++ compiler complaints
+         * Add 'USE_DL_PREFIX' to quickly allow co-existence with existing
+           memory allocation routines
+         * Set 'malloc_getpagesize' for WIN32 platforms (needs more work)
+         * Use 'assert' rather than 'ASSERT' in WIN32 code to conform to
+           usage of 'assert' in non-WIN32 code
+         * Improve WIN32 'sbrk()' emulation's 'findRegion()' routine to
+           avoid infinite loop
+      * Always call 'fREe()' rather than 'free()'
+
+    V2.6.5 Wed Jun 17 15:57:31 1998  Doug Lea  (dl at gee)
+      * Fixed ordering problem with boundary-stamping
+
+    V2.6.3 Sun May 19 08:17:58 1996  Doug Lea  (dl at gee)
+      * Added pvalloc, as recommended by H.J. Liu
+      * Added 64bit pointer support mainly from Wolfram Gloger
+      * Added anonymously donated WIN32 sbrk emulation
+      * Malloc, calloc, getpagesize: add optimizations from Raymond Nijssen
+      * malloc_extend_top: fix mask error that caused wastage after
+        foreign sbrks
+      * Add linux mremap support code from HJ Liu
+
+    V2.6.2 Tue Dec  5 06:52:55 1995  Doug Lea  (dl at gee)
+      * Integrated most documentation with the code.
+      * Add support for mmap, with help from
+        Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
+      * Use last_remainder in more cases.
+      * Pack bins using idea from  colin@nyx10.cs.du.edu
+      * Use ordered bins instead of best-fit threshhold
+      * Eliminate block-local decls to simplify tracing and debugging.
+      * Support another case of realloc via move into top
+      * Fix error occuring when initial sbrk_base not word-aligned.
+      * Rely on page size for units instead of SBRK_UNIT to
+        avoid surprises about sbrk alignment conventions.
+      * Add mallinfo, mallopt. Thanks to Raymond Nijssen
+        (raymond@es.ele.tue.nl) for the suggestion.
+      * Add `pad' argument to malloc_trim and top_pad mallopt parameter.
+      * More precautions for cases where other routines call sbrk,
+        courtesy of Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
+      * Added macros etc., allowing use in linux libc from
+        H.J. Lu (hjl@gnu.ai.mit.edu)
+      * Inverted this history list
+
+    V2.6.1 Sat Dec  2 14:10:57 1995  Doug Lea  (dl at gee)
+      * Re-tuned and fixed to behave more nicely with V2.6.0 changes.
+      * Removed all preallocation code since under current scheme
+        the work required to undo bad preallocations exceeds
+        the work saved in good cases for most test programs.
+      * No longer use return list or unconsolidated bins since
+        no scheme using them consistently outperforms those that don't
+        given above changes.
+      * Use best fit for very large chunks to prevent some worst-cases.
+      * Added some support for debugging
+
+    V2.6.0 Sat Nov  4 07:05:23 1995  Doug Lea  (dl at gee)
+      * Removed footers when chunks are in use. Thanks to
+        Paul Wilson (wilson@cs.texas.edu) for the suggestion.
+
+    V2.5.4 Wed Nov  1 07:54:51 1995  Doug Lea  (dl at gee)
+      * Added malloc_trim, with help from Wolfram Gloger
+        (wmglo@Dent.MED.Uni-Muenchen.DE).
+
+    V2.5.3 Tue Apr 26 10:16:01 1994  Doug Lea  (dl at g)
+
+    V2.5.2 Tue Apr  5 16:20:40 1994  Doug Lea  (dl at g)
+      * realloc: try to expand in both directions
+      * malloc: swap order of clean-bin strategy;
+      * realloc: only conditionally expand backwards
+      * Try not to scavenge used bins
+      * Use bin counts as a guide to preallocation
+      * Occasionally bin return list chunks in first scan
+      * Add a few optimizations from colin@nyx10.cs.du.edu
+
+    V2.5.1 Sat Aug 14 15:40:43 1993  Doug Lea  (dl at g)
+      * faster bin computation & slightly different binning
+      * merged all consolidations to one part of malloc proper
+         (eliminating old malloc_find_space & malloc_clean_bin)
+      * Scan 2 returns chunks (not just 1)
+      * Propagate failure in realloc if malloc returns 0
+      * Add stuff to allow compilation on non-ANSI compilers
+          from kpv@research.att.com
+
+    V2.5 Sat Aug  7 07:41:59 1993  Doug Lea  (dl at g.oswego.edu)
+      * removed potential for odd address access in prev_chunk
+      * removed dependency on getpagesize.h
+      * misc cosmetics and a bit more internal documentation
+      * anticosmetics: mangled names in macros to evade debugger strangeness
+      * tested on sparc, hp-700, dec-mips, rs6000
+          with gcc & native cc (hp, dec only) allowing
+          Detlefs & Zorn comparison study (in SIGPLAN Notices.)
+
+    Trial version Fri Aug 28 13:14:29 1992  Doug Lea  (dl at g.oswego.edu)
+      * Based loosely on libg++-1.2X malloc. (It retains some of the overall
+         structure of old version,  but most details differ.)
+
+*/
diff --git a/cpp/src/plasma/thirdparty/xxhash.cc b/cpp/src/plasma/thirdparty/xxhash.cc
new file mode 100644
index 0000000000000..f74880b0de71d
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/xxhash.cc
@@ -0,0 +1,889 @@
+/*
+*  xxHash - Fast Hash algorithm
+*  Copyright (C) 2012-2016, Yann Collet
+*
+*  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+*
+*  Redistribution and use in source and binary forms, with or without
+*  modification, are permitted provided that the following conditions are
+*  met:
+*
+*  * Redistributions of source code must retain the above copyright
+*  notice, this list of conditions and the following disclaimer.
+*  * Redistributions in binary form must reproduce the above
+*  copyright notice, this list of conditions and the following disclaimer
+*  in the documentation and/or other materials provided with the
+*  distribution.
+*
+*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+*  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+*  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+*  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+*  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+*  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+*  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+*  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+*  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+*  You can contact the author at :
+*  - xxHash homepage: http://www.xxhash.com
+*  - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+
+/* *************************************
+*  Tuning parameters
+***************************************/
+/*!XXH_FORCE_MEMORY_ACCESS :
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
+ *            It can generate buggy code on targets which do not support unaligned memory accesses.
+ *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See http://stackoverflow.com/a/32095106/646947 for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
+ */
+#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#    define XXH_FORCE_MEMORY_ACCESS 2
+#  elif defined(__INTEL_COMPILER) || \
+  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+#    define XXH_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+/*!XXH_ACCEPT_NULL_INPUT_POINTER :
+ * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
+ * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
+ * By default, this option is disabled. To enable it, uncomment below define :
+ */
+/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
+
+/*!XXH_FORCE_NATIVE_FORMAT :
+ * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
+ * Results are therefore identical for little-endian and big-endian CPU.
+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+ * Should endian-independence be of no importance for your application, you may set the #define below to 1,
+ * to improve speed for Big-endian CPU.
+ * This option has no impact on Little_Endian CPU.
+ */
+#ifndef XXH_FORCE_NATIVE_FORMAT   /* can be defined externally */
+#  define XXH_FORCE_NATIVE_FORMAT 0
+#endif
+
+/*!XXH_FORCE_ALIGN_CHECK :
+ * This is a minor performance trick, only useful with lots of very small keys.
+ * It means : check for aligned/unaligned input.
+ * The check costs one initial branch per hash; set to 0 when the input data
+ * is guaranteed to be aligned.
+ */
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+#  if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+#    define XXH_FORCE_ALIGN_CHECK 0
+#  else
+#    define XXH_FORCE_ALIGN_CHECK 1
+#  endif
+#endif
+
+
+/* *************************************
+*  Includes & Memory related functions
+***************************************/
+/*! Modify the local functions below should you wish to use some other memory routines
+*   for malloc(), free() */
+#include <stdlib.h>
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void  XXH_free  (void* p)  { free(p); }
+/*! and for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
+
+
+/* *************************************
+*  Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
+#  define FORCE_INLINE static __forceinline
+#else
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
+#endif
+
+
+/* *************************************
+*  Basic Types
+***************************************/
+#ifndef MEM_MODULE
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+    typedef uint8_t  BYTE;
+    typedef uint16_t U16;
+    typedef uint32_t U32;
+    typedef  int32_t S32;
+# else
+    typedef unsigned char      BYTE;
+    typedef unsigned short     U16;
+    typedef unsigned int       U32;
+    typedef   signed int       S32;
+# endif
+#endif
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; } __attribute__((packed)) unalign;
+static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+static U32 XXH_read32(const void* memPtr)
+{
+    U32 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* ****************************************
+*  Compiler-specific Functions and Macros
+******************************************/
+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
+#if defined(_MSC_VER)
+#  define XXH_rotl32(x,r) _rotl(x,r)
+#  define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+#  define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#  define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
+#endif
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap32 _byteswap_ulong
+#elif XXH_GCC_VERSION >= 403
+#  define XXH_swap32 __builtin_bswap32
+#else
+static U32 XXH_swap32 (U32 x)
+{
+    return  ((x << 24) & 0xff000000 ) |
+            ((x <<  8) & 0x00ff0000 ) |
+            ((x >>  8) & 0x0000ff00 ) |
+            ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* *************************************
+*  Architecture Macros
+***************************************/
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+
+/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+    static const int g_one = 1;
+#   define XXH_CPU_LITTLE_ENDIAN   (*(const char*)(&g_one))
+#endif
+
+
+/* ***************************
+*  Memory reads
+*****************************/
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
+}
+
+FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE32_align(ptr, endian, XXH_unaligned);
+}
+
+static U32 XXH_readBE32(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+
+
+/* *************************************
+*  Macros
+***************************************/
+#define XXH_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(int)(!!(c)) }; }    /* use only *after* variable declarations */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+*  32-bits hash functions
+*********************************************************************/
+static const U32 PRIME32_1 = 2654435761U;
+static const U32 PRIME32_2 = 2246822519U;
+static const U32 PRIME32_3 = 3266489917U;
+static const U32 PRIME32_4 =  668265263U;
+static const U32 PRIME32_5 =  374761393U;
+
+static U32 XXH32_round(U32 seed, U32 input)
+{
+    seed += input * PRIME32_2;
+    seed  = XXH_rotl32(seed, 13);
+    seed *= PRIME32_1;
+    return seed;
+}
+
+FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U32 h32;
+#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (p==NULL) {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)16;
+    }
+#endif
+
+    if (len>=16) {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = seed + PRIME32_1 + PRIME32_2;
+        U32 v2 = seed + PRIME32_2;
+        U32 v3 = seed + 0;
+        U32 v4 = seed - PRIME32_1;
+
+        do {
+            v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
+            v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
+            v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
+            v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
+        } while (p<=limit);
+
+        h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+    } else {
+        h32  = seed + PRIME32_5;
+    }
+
+    h32 += (U32) len;
+
+    while (p+4<=bEnd) {
+        h32 += XXH_get32bits(p) * PRIME32_3;
+        h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
+        p+=4;
+    }
+
+    while (p<bEnd) {
+        h32 += (*p) * PRIME32_5;
+        h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+        p++;
+    }
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH32_state_t state;
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, input, len);
+    return XXH32_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
+            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+                return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+            else
+                return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }   }
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+
+
+/*======   Hash streaming   ======*/
+
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
+{
+    XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state)-4);   /* do not write into reserved, for future removal */
+    state.v1 = seed + PRIME32_1 + PRIME32_2;
+    state.v2 = seed + PRIME32_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - PRIME32_1;
+    memcpy(statePtr, &state, sizeof(state));
+    return XXH_OK;
+}
+
+
+FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (input==NULL) return XXH_ERROR;
+#endif
+
+    state->total_len_32 += (unsigned)len;
+    state->large_len |= (len>=16) | (state->total_len_32>=16);
+
+    if (state->memsize + len < 16)  {   /* fill in tmp buffer */
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
+        state->memsize += (unsigned)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize) {   /* some data left from previous update */
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
+        {   const U32* p32 = state->mem32;
+            state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
+            state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
+            state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
+            state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
+        }
+        p += 16-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p <= bEnd-16) {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = state->v1;
+        U32 v2 = state->v2;
+        U32 v3 = state->v3;
+        U32 v4 = state->v4;
+
+        do {
+            v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
+            v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
+            v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
+            v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
+        } while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd) {
+        XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
+{
+    const BYTE * p = (const BYTE*)state->mem32;
+    const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
+    U32 h32;
+
+    if (state->large_len) {
+        h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
+    } else {
+        h32 = state->v3 /* == seed */ + PRIME32_5;
+    }
+
+    h32 += state->total_len_32;
+
+    while (p+4<=bEnd) {
+        h32 += XXH_readLE32(p, endian) * PRIME32_3;
+        h32  = XXH_rotl32(h32, 17) * PRIME32_4;
+        p+=4;
+    }
+
+    while (p<bEnd) {
+        h32 += (*p) * PRIME32_5;
+        h32  = XXH_rotl32(h32, 11) * PRIME32_1;
+        p++;
+    }
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH32_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*======   Canonical representation   ======*/
+
+/*! Default XXH result types are basic unsigned 32 and 64 bits.
+*   The canonical representation follows human-readable write convention, aka big-endian (large digits first).
+*   These functions allow transformation of hash result into and from its canonical format.
+*   This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
+*/
+
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+    return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+*  64-bits hash functions
+*********************************************************************/
+
+/*======   Memory access   ======*/
+
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+    typedef uint64_t U64;
+# else
+    typedef unsigned long long U64;   /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
+# endif
+#endif
+
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
+static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+
+static U64 XXH_read64(const void* memPtr)
+{
+    U64 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap64 _byteswap_uint64
+#elif XXH_GCC_VERSION >= 403
+#  define XXH_swap64 __builtin_bswap64
+#else
+static U64 XXH_swap64 (U64 x)
+{
+    return  ((x << 56) & 0xff00000000000000ULL) |
+            ((x << 40) & 0x00ff000000000000ULL) |
+            ((x << 24) & 0x0000ff0000000000ULL) |
+            ((x << 8)  & 0x000000ff00000000ULL) |
+            ((x >> 8)  & 0x00000000ff000000ULL) |
+            ((x >> 24) & 0x0000000000ff0000ULL) |
+            ((x >> 40) & 0x000000000000ff00ULL) |
+            ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
+}
+
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+}
+
+static U64 XXH_readBE64(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+
+
+/*======   xxh64   ======*/
+
+static const U64 PRIME64_1 = 11400714785074694791ULL;
+static const U64 PRIME64_2 = 14029467366897019727ULL;
+static const U64 PRIME64_3 =  1609587929392839161ULL;
+static const U64 PRIME64_4 =  9650029242287828579ULL;
+static const U64 PRIME64_5 =  2870177450012600261ULL;
+
+static U64 XXH64_round(U64 acc, U64 input)
+{
+    acc += input * PRIME64_2;
+    acc  = XXH_rotl64(acc, 31);
+    acc *= PRIME64_1;
+    return acc;
+}
+
+static U64 XXH64_mergeRound(U64 acc, U64 val)
+{
+    val  = XXH64_round(0, val);
+    acc ^= val;
+    acc  = acc * PRIME64_1 + PRIME64_4;
+    return acc;
+}
+
+FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U64 h64;
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (p==NULL) {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)32;
+    }
+#endif
+
+    if (len>=32) {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = seed + PRIME64_1 + PRIME64_2;
+        U64 v2 = seed + PRIME64_2;
+        U64 v3 = seed + 0;
+        U64 v4 = seed - PRIME64_1;
+
+        do {
+            v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
+            v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
+            v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
+            v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
+        } while (p<=limit);
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+
+    } else {
+        h64  = seed + PRIME64_5;
+    }
+
+    h64 += (U64) len;
+
+    while (p+8<=bEnd) {
+        U64 const k1 = XXH64_round(0, XXH_get64bits(p));
+        h64 ^= k1;
+        h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+        p+=8;
+    }
+
+    if (p+4<=bEnd) {
+        h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
+        h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+        p+=4;
+    }
+
+    while (p<bEnd) {
+        h64 ^= (*p) * PRIME64_5;
+        h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+        p++;
+    }
+
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+
+    return h64;
+}
+
+
+XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH64_state_t state;
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, input, len);
+    return XXH64_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
+            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+                return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+            else
+                return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }   }
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+/*======   Hash Streaming   ======*/
+
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
+{
+    XXH64_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state)-8);   /* do not write into reserved, for future removal */
+    state.v1 = seed + PRIME64_1 + PRIME64_2;
+    state.v2 = seed + PRIME64_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - PRIME64_1;
+    memcpy(statePtr, &state, sizeof(state));
+    return XXH_OK;
+}
+
+FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (input==NULL) return XXH_ERROR;
+#endif
+
+    state->total_len += len;
+
+    if (state->memsize + len < 32) {  /* fill in tmp buffer */
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
+        state->memsize += (U32)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize) {   /* tmp buffer is full */
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
+        state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
+        state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
+        state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
+        state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
+        p += 32-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p+32 <= bEnd) {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = state->v1;
+        U64 v2 = state->v2;
+        U64 v3 = state->v3;
+        U64 v4 = state->v4;
+
+        do {
+            v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
+            v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
+            v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
+            v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
+        } while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd) {
+        XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
+{
+    const BYTE * p = (const BYTE*)state->mem64;
+    const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
+    U64 h64;
+
+    if (state->total_len >= 32) {
+        U64 const v1 = state->v1;
+        U64 const v2 = state->v2;
+        U64 const v3 = state->v3;
+        U64 const v4 = state->v4;
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+    } else {
+        h64  = state->v3 + PRIME64_5;
+    }
+
+    h64 += (U64) state->total_len;
+
+    while (p+8<=bEnd) {
+        U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
+        h64 ^= k1;
+        h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+        p+=8;
+    }
+
+    if (p+4<=bEnd) {
+        h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
+        h64  = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+        p+=4;
+    }
+
+    while (p<bEnd) {
+        h64 ^= (*p) * PRIME64_5;
+        h64  = XXH_rotl64(h64, 11) * PRIME64_1;
+        p++;
+    }
+
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+
+    return h64;
+}
+
+XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH64_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*====== Canonical representation   ======*/
+
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+    return XXH_readBE64(src);
+}
+
+#endif  /* XXH_NO_LONG_LONG */
diff --git a/cpp/src/plasma/thirdparty/xxhash.h b/cpp/src/plasma/thirdparty/xxhash.h
new file mode 100644
index 0000000000000..9d831e03b35f6
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/xxhash.h
@@ -0,0 +1,293 @@
+/*
+   xxHash - Extremely Fast Hash algorithm
+   Header File
+   Copyright (C) 2012-2016, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name            Speed       Q.Score   Author
+xxHash          5.4 GB/s     10
+CrapWow         3.2 GB/s      2       Andrew
+MumurHash 3a    2.7 GB/s     10       Austin Appleby
+SpookyHash      2.0 GB/s     10       Bob Jenkins
+SBox            1.4 GB/s      9       Bret Mulvey
+Lookup3         1.2 GB/s      9       Bob Jenkins
+SuperFastHash   1.2 GB/s      1       Paul Hsieh
+CityHash64      1.05 GB/s    10       Pike & Alakuijala
+FNV             0.55 GB/s     5       Fowler, Noll, Vo
+CRC32           0.43 GB/s     9
+MD5-32          0.33 GB/s    10       Ronald L. Rivest
+SHA1-32         0.28 GB/s    10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+A 64-bits version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bits applications only.
+Name     Speed on 64 bits    Speed on 32 bits
+XXH64       13.8 GB/s            1.9 GB/s
+XXH32        6.8 GB/s            6.0 GB/s
+*/
+
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************
+*  Definitions
+******************************/
+#include <stddef.h>   /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/* ****************************
+*  API modifier
+******************************/
+/** XXH_PRIVATE_API
+*   This is useful to include xxhash functions in `static` mode
+*   in order to inline them, and remove their symbol from the public list.
+*   Methodology :
+*     #define XXH_PRIVATE_API
+*     #include "xxhash.h"
+*   `xxhash.c` is automatically included.
+*   It's not useful to compile and link it as a separate module.
+*/
+#ifdef XXH_PRIVATE_API
+#  ifndef XXH_STATIC_LINKING_ONLY
+#    define XXH_STATIC_LINKING_ONLY
+#  endif
+#  if defined(__GNUC__)
+#    define XXH_PUBLIC_API static __inline __attribute__((unused))
+#  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#    define XXH_PUBLIC_API static inline
+#  elif defined(_MSC_VER)
+#    define XXH_PUBLIC_API static __inline
+#  else
+#    define XXH_PUBLIC_API static   /* this version may generate warnings for unused static functions; disable the relevant warning */
+#  endif
+#else
+#  define XXH_PUBLIC_API   /* do nothing */
+#endif /* XXH_PRIVATE_API */
+
+/*!XXH_NAMESPACE, aka Namespace Emulation :
+
+If you want to include _and expose_ xxHash functions from within your own library,
+but also want to avoid symbol collisions with other libraries which may also include xxHash,
+
+you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
+with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
+
+Note that no change is required within the calling program as long as it includes `xxhash.h` :
+regular symbol name will be automatically translated by this header.
+*/
+#ifdef XXH_NAMESPACE
+#  define XXH_CAT(A,B) A##B
+#  define XXH_NAME2(A,B) XXH_CAT(A,B)
+#  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+#endif
+
+
+/* *************************************
+*  Version
+***************************************/
+#define XXH_VERSION_MAJOR    0
+#define XXH_VERSION_MINOR    6
+#define XXH_VERSION_RELEASE  2
+#define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+XXH_PUBLIC_API unsigned XXH_versionNumber (void);
+
+
+/*-**********************************************************************
+*  32-bits hash
+************************************************************************/
+typedef unsigned int XXH32_hash_t;
+
+/*! XXH32() :
+    Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
+    The memory between input & input+length must be valid (allocated and read-accessible).
+    "seed" can be used to alter the result predictably.
+    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
+
+/*======   Streaming   ======*/
+typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
+XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned int seed);
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH32_hash_t  XXH32_digest (const XXH32_state_t* statePtr);
+
+/*
+These functions generate the xxHash of an input provided in multiple segments.
+Note that, for small input, they are slower than single-call functions, due to state management.
+For small input, prefer `XXH32()` and `XXH64()` .
+
+XXH state must first be allocated, using XXH*_createState() .
+
+Start a new hash by initializing state with a seed, using XXH*_reset().
+
+Then, feed the hash state by calling XXH*_update() as many times as necessary.
+Obviously, input must be allocated and read accessible.
+The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+
+Finally, a hash value can be produced anytime, by using XXH*_digest().
+This function returns the nn-bits hash as an int or long long.
+
+It's still possible to continue inserting input into the hash state after a digest,
+and generate some new hashes later on, by calling again XXH*_digest().
+
+When done, free XXH state space if it was allocated dynamically.
+*/
+
+/*======   Canonical representation   ======*/
+
+typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+*  These functions allow transformation of hash result into and from its canonical format.
+*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+*/
+
+
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+*  64-bits hash
+************************************************************************/
+typedef unsigned long long XXH64_hash_t;
+
+/*! XXH64() :
+    Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
+    "seed" can be used to alter the result predictably.
+    This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
+*/
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
+
+/*======   Streaming   ======*/
+typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
+XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
+
+/*======   Canonical representation   ======*/
+typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+#endif  /* XXH_NO_LONG_LONG */
+
+
+#ifdef XXH_STATIC_LINKING_ONLY
+
+/* ================================================================================================
+   This section contains definitions which are not guaranteed to remain stable.
+   They may change in future versions, becoming incompatible with a different version of the library.
+   They shall only be used with static linking.
+   Never use these definitions in association with dynamic linking !
+=================================================================================================== */
+
+/* These definitions are only meant to make possible
+   static allocation of XXH state, on stack or in a struct for example.
+   Never use members directly. */
+
+struct XXH32_state_s {
+   unsigned total_len_32;
+   unsigned large_len;
+   unsigned v1;
+   unsigned v2;
+   unsigned v3;
+   unsigned v4;
+   unsigned mem32[4];   /* buffer defined as U32 for alignment */
+   unsigned memsize;
+   unsigned reserved;   /* never read nor write, will be removed in a future version */
+};   /* typedef'd to XXH32_state_t */
+
+#ifndef XXH_NO_LONG_LONG   /* remove 64-bits support */
+struct XXH64_state_s {
+   unsigned long long total_len;
+   unsigned long long v1;
+   unsigned long long v2;
+   unsigned long long v3;
+   unsigned long long v4;
+   unsigned long long mem64[4];   /* buffer defined as U64 for alignment */
+   unsigned memsize;
+   unsigned reserved[2];          /* never read nor write, will be removed in a future version */
+};   /* typedef'd to XXH64_state_t */
+#endif
+
+#ifdef XXH_PRIVATE_API
+#  include "xxhash.c"   /* include xxhash function bodies as `static`, for inlining */
+#endif
+
+#endif /* XXH_STATIC_LINKING_ONLY */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* XXHASH_H_5627135585666179 */
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index f225c66268a61..757604f38590e 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -59,6 +59,17 @@ $RAT $1 \
   -e arrow-glib-sections.txt \
   -e arrow-glib-overrides.txt \
   -e gtk-doc.make \
+  -e ae.c \
+  -e ae.h \
+  -e ae_epoll.c \
+  -e ae_evport.c \
+  -e ae_kqueue.c \
+  -e ae_select.c \
+  -e config.h \
+  -e zmalloc.h \
+  -e dlmalloc.c \
+  -e xxhash.cc \
+  -e xxhash.h \
   -e "*.html" \
   -e "*.sgml" \
   -e "*.css" \

From 222628c9d86c27800e94311fb33c2edd36ee623b Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 22 Jun 2017 17:30:33 -0400
Subject: [PATCH 0735/1644] ARROW-1140: [C++] Allow optional build of plasma

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #767 from cpcloud/ARROW-1140 and squashes the following commits:

b046215c [Phillip Cloud] ARROW-1140: [C++] Allow optional build of plasma
---
 ci/travis_before_script_cpp.sh | 3 ++-
 ci/travis_script_python.sh     | 1 +
 cpp/CMakeLists.txt             | 8 +++++++-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index a38a0dc132817..3a1836ca3a4fa 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -44,7 +44,8 @@ pushd $ARROW_CPP_BUILD_DIR
 
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
--DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL
+-DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \
+-DARROW_PLASMA=ON \
 -DARROW_NO_DEPRECATED_API=ON"
 CMAKE_LINUX_FLAGS=""
 CMAKE_OSX_FLAGS=""
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index b82653dfb01b5..85855276abec4 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -73,6 +73,7 @@ function build_arrow_libraries() {
   cmake -DARROW_BUILD_TESTS=off \
         -DARROW_PYTHON=on \
         -DPLASMA_PYTHON=on \
+        -DARROW_PLASMA=on \
         -DCMAKE_INSTALL_PREFIX=$2 \
         $CPP_DIR
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0897e99ae06e6..5ba56e5963213 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -132,6 +132,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(ARROW_RPATH_ORIGIN
     "Build Arrow libraries with RATH set to \$ORIGIN"
     OFF)
+
+  option(ARROW_PLASMA
+    "Build the plasma object store along with Arrow"
+    OFF)
 endif()
 
 if(ARROW_BUILD_TESTS)
@@ -1013,11 +1017,13 @@ if(FLATBUFFERS_VENDORED)
   set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} flatbuffers_ep)
 endif()
 
-if(NOT WIN32)
+if(NOT WIN32 AND ARROW_PLASMA)
   add_subdirectory(src/plasma)
 endif()
+
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
+
 if (ARROW_IPC)
   add_subdirectory(src/arrow/ipc)
   set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} metadata_fbs)

From 608b89e1648ef8a116c3139606b0440122d2b75a Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 22 Jun 2017 17:32:41 -0400
Subject: [PATCH 0736/1644] ARROW-1073: C++: Adapative integer builder

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #723 from xhochy/ARROW-1073 and squashes the following commits:

5bab9c2f [Uwe L. Korn] ARROW-1073: C++: Adapative integer builder
---
 cpp/src/arrow/array-test.cc        | 220 +++++++++++++++++++
 cpp/src/arrow/builder-benchmark.cc |  66 ++++++
 cpp/src/arrow/builder.cc           | 331 +++++++++++++++++++++++++++++
 cpp/src/arrow/builder.h            | 190 ++++++++++++++++-
 4 files changed, 806 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 636d97f9d059f..beffa1b11cbe2 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1257,6 +1257,226 @@ TEST_F(TestFWBinaryArray, Slice) {
   ASSERT_TRUE(array->RangeEquals(1, 3, 0, slice));
 }
 
+// ----------------------------------------------------------------------
+// AdaptiveInt tests
+
+class TestAdaptiveIntBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+    builder_ = std::make_shared<AdaptiveIntBuilder>(pool_);
+  }
+
+  void Done() { EXPECT_OK(builder_->Finish(&result_)); }
+
+ protected:
+  std::shared_ptr<AdaptiveIntBuilder> builder_;
+
+  std::shared_ptr<Array> expected_;
+  std::shared_ptr<Array> result_;
+};
+
+TEST_F(TestAdaptiveIntBuilder, TestInt8) {
+  builder_->Append(0);
+  builder_->Append(127);
+  builder_->Append(-128);
+
+  Done();
+
+  std::vector<int8_t> expected_values({0, 127, -128});
+  ArrayFromVector<Int8Type, int8_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+TEST_F(TestAdaptiveIntBuilder, TestInt16) {
+  builder_->Append(0);
+  builder_->Append(128);
+  Done();
+
+  std::vector<int16_t> expected_values({0, 128});
+  ArrayFromVector<Int16Type, int16_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(-129);
+  expected_values = {-129};
+  Done();
+
+  ArrayFromVector<Int16Type, int16_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(std::numeric_limits<int16_t>::max());
+  builder_->Append(std::numeric_limits<int16_t>::min());
+  expected_values = {
+      std::numeric_limits<int16_t>::max(), std::numeric_limits<int16_t>::min()};
+  Done();
+
+  ArrayFromVector<Int16Type, int16_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+TEST_F(TestAdaptiveIntBuilder, TestInt32) {
+  builder_->Append(0);
+  builder_->Append(static_cast<int64_t>(std::numeric_limits<int16_t>::max()) + 1);
+  Done();
+
+  std::vector<int32_t> expected_values(
+      {0, static_cast<int32_t>(std::numeric_limits<int16_t>::max()) + 1});
+  ArrayFromVector<Int32Type, int32_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(static_cast<int64_t>(std::numeric_limits<int16_t>::min()) - 1);
+  expected_values = {static_cast<int32_t>(std::numeric_limits<int16_t>::min()) - 1};
+  Done();
+
+  ArrayFromVector<Int32Type, int32_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(std::numeric_limits<int32_t>::max());
+  builder_->Append(std::numeric_limits<int32_t>::min());
+  expected_values = {
+      std::numeric_limits<int32_t>::max(), std::numeric_limits<int32_t>::min()};
+  Done();
+
+  ArrayFromVector<Int32Type, int32_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+TEST_F(TestAdaptiveIntBuilder, TestInt64) {
+  builder_->Append(0);
+  builder_->Append(static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1);
+  Done();
+
+  std::vector<int64_t> expected_values(
+      {0, static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1});
+  ArrayFromVector<Int64Type, int64_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(static_cast<int64_t>(std::numeric_limits<int32_t>::min()) - 1);
+  expected_values = {static_cast<int64_t>(std::numeric_limits<int32_t>::min()) - 1};
+  Done();
+
+  ArrayFromVector<Int64Type, int64_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(std::numeric_limits<int64_t>::max());
+  builder_->Append(std::numeric_limits<int64_t>::min());
+  expected_values = {
+      std::numeric_limits<int64_t>::max(), std::numeric_limits<int64_t>::min()};
+  Done();
+
+  ArrayFromVector<Int64Type, int64_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+TEST_F(TestAdaptiveIntBuilder, TestAppendVector) {
+  std::vector<int64_t> expected_values(
+      {0, static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1});
+  builder_->Append(expected_values.data(), expected_values.size());
+  Done();
+
+  ArrayFromVector<Int64Type, int64_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+class TestAdaptiveUIntBuilder : public TestBuilder {
+ public:
+  void SetUp() {
+    TestBuilder::SetUp();
+    builder_ = std::make_shared<AdaptiveUIntBuilder>(pool_);
+  }
+
+  void Done() { EXPECT_OK(builder_->Finish(&result_)); }
+
+ protected:
+  std::shared_ptr<AdaptiveUIntBuilder> builder_;
+
+  std::shared_ptr<Array> expected_;
+  std::shared_ptr<Array> result_;
+};
+
+TEST_F(TestAdaptiveUIntBuilder, TestUInt8) {
+  builder_->Append(0);
+  builder_->Append(255);
+
+  Done();
+
+  std::vector<uint8_t> expected_values({0, 255});
+  ArrayFromVector<UInt8Type, uint8_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+TEST_F(TestAdaptiveUIntBuilder, TestUInt16) {
+  builder_->Append(0);
+  builder_->Append(256);
+  Done();
+
+  std::vector<uint16_t> expected_values({0, 256});
+  ArrayFromVector<UInt16Type, uint16_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(std::numeric_limits<uint16_t>::max());
+  expected_values = {std::numeric_limits<uint16_t>::max()};
+  Done();
+
+  ArrayFromVector<UInt16Type, uint16_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+TEST_F(TestAdaptiveUIntBuilder, TestUInt32) {
+  builder_->Append(0);
+  builder_->Append(static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 1);
+  Done();
+
+  std::vector<uint32_t> expected_values(
+      {0, static_cast<uint32_t>(std::numeric_limits<uint16_t>::max()) + 1});
+  ArrayFromVector<UInt32Type, uint32_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(std::numeric_limits<uint32_t>::max());
+  expected_values = {std::numeric_limits<uint32_t>::max()};
+  Done();
+
+  ArrayFromVector<UInt32Type, uint32_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+TEST_F(TestAdaptiveUIntBuilder, TestUInt64) {
+  builder_->Append(0);
+  builder_->Append(static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
+  Done();
+
+  std::vector<uint64_t> expected_values(
+      {0, static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1});
+  ArrayFromVector<UInt64Type, uint64_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+
+  SetUp();
+  builder_->Append(std::numeric_limits<uint64_t>::max());
+  expected_values = {std::numeric_limits<uint64_t>::max()};
+  Done();
+
+  ArrayFromVector<UInt64Type, uint64_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
+TEST_F(TestAdaptiveUIntBuilder, TestAppendVector) {
+  std::vector<uint64_t> expected_values(
+      {0, static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1});
+  builder_->Append(expected_values.data(), expected_values.size());
+  Done();
+
+  ArrayFromVector<UInt64Type, uint64_t>(expected_values, &expected_);
+  ASSERT_TRUE(expected_->Equals(result_));
+}
+
 // ----------------------------------------------------------------------
 // List tests
 
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index b0c3cd19064de..62f2fd620a969 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -61,4 +61,70 @@ static void BM_BuildVectorNoNulls(
 
 BENCHMARK(BM_BuildVectorNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
 
+static void BM_BuildAdaptiveIntNoNulls(
+    benchmark::State& state) {  // NOLINT non-const reference
+  int64_t size = static_cast<int64_t>(std::numeric_limits<int16_t>::max()) * 256;
+  int64_t chunk_size = size / 8;
+  std::vector<int64_t> data;
+  for (int64_t i = 0; i < size; i++) {
+    data.push_back(i);
+  }
+  while (state.KeepRunning()) {
+    AdaptiveIntBuilder builder(default_memory_pool());
+    for (int64_t i = 0; i < size; i += chunk_size) {
+      // Build up an array of 512 MiB in size
+      builder.Append(data.data() + i, chunk_size, nullptr);
+    }
+    std::shared_ptr<Array> out;
+    builder.Finish(&out);
+  }
+  state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t));
+}
+
+BENCHMARK(BM_BuildAdaptiveIntNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
+
+static void BM_BuildAdaptiveIntNoNullsScalarAppend(
+    benchmark::State& state) {  // NOLINT non-const reference
+  int64_t size = static_cast<int64_t>(std::numeric_limits<int16_t>::max()) * 256;
+  std::vector<int64_t> data;
+  for (int64_t i = 0; i < size; i++) {
+    data.push_back(i);
+  }
+  while (state.KeepRunning()) {
+    AdaptiveIntBuilder builder(default_memory_pool());
+    for (int64_t i = 0; i < size; i++) {
+      builder.Append(data[i]);
+    }
+    std::shared_ptr<Array> out;
+    builder.Finish(&out);
+  }
+  state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t));
+}
+
+BENCHMARK(BM_BuildAdaptiveIntNoNullsScalarAppend)
+    ->Repetitions(3)
+    ->Unit(benchmark::kMillisecond);
+
+static void BM_BuildAdaptiveUIntNoNulls(
+    benchmark::State& state) {  // NOLINT non-const reference
+  int64_t size = static_cast<int64_t>(std::numeric_limits<uint16_t>::max()) * 256;
+  int64_t chunk_size = size / 8;
+  std::vector<uint64_t> data;
+  for (uint64_t i = 0; i < static_cast<uint64_t>(size); i++) {
+    data.push_back(i);
+  }
+  while (state.KeepRunning()) {
+    AdaptiveUIntBuilder builder(default_memory_pool());
+    for (int64_t i = 0; i < size; i += chunk_size) {
+      // Build up an array of 512 MiB in size
+      builder.Append(data.data() + i, chunk_size, nullptr);
+    }
+    std::shared_ptr<Array> out;
+    builder.Finish(&out);
+  }
+  state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t));
+}
+
+BENCHMARK(BM_BuildAdaptiveUIntNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index ab43c2a51baf4..6762e172676cb 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/builder.h"
 
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <limits>
@@ -248,6 +249,336 @@ template class PrimitiveBuilder<HalfFloatType>;
 template class PrimitiveBuilder<FloatType>;
 template class PrimitiveBuilder<DoubleType>;
 
+AdaptiveIntBuilderBase::AdaptiveIntBuilderBase(MemoryPool* pool)
+    : ArrayBuilder(pool, int64()), data_(nullptr), raw_data_(nullptr), int_size_(1) {}
+
+Status AdaptiveIntBuilderBase::Init(int64_t capacity) {
+  RETURN_NOT_OK(ArrayBuilder::Init(capacity));
+  data_ = std::make_shared<PoolBuffer>(pool_);
+
+  int64_t nbytes = capacity * int_size_;
+  RETURN_NOT_OK(data_->Resize(nbytes));
+  // TODO(emkornfield) valgrind complains without this
+  memset(data_->mutable_data(), 0, static_cast<size_t>(nbytes));
+
+  raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
+  return Status::OK();
+}
+
+Status AdaptiveIntBuilderBase::Resize(int64_t capacity) {
+  // XXX: Set floor size for now
+  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+
+  if (capacity_ == 0) {
+    RETURN_NOT_OK(Init(capacity));
+  } else {
+    RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
+    const int64_t old_bytes = data_->size();
+    const int64_t new_bytes = capacity * int_size_;
+    RETURN_NOT_OK(data_->Resize(new_bytes));
+    raw_data_ = data_->mutable_data();
+    // TODO(emkornfield) valgrind complains without this
+    memset(
+        data_->mutable_data() + old_bytes, 0, static_cast<size_t>(new_bytes - old_bytes));
+  }
+  return Status::OK();
+}
+
+AdaptiveIntBuilder::AdaptiveIntBuilder(MemoryPool* pool) : AdaptiveIntBuilderBase(pool) {}
+
+Status AdaptiveIntBuilder::Finish(std::shared_ptr<Array>* out) {
+  const int64_t bytes_required = length_ * int_size_;
+  if (bytes_required > 0 && bytes_required < data_->size()) {
+    // Trim buffers
+    RETURN_NOT_OK(data_->Resize(bytes_required));
+  }
+  switch (int_size_) {
+    case 1:
+      *out =
+          std::make_shared<Int8Array>(int8(), length_, data_, null_bitmap_, null_count_);
+      break;
+    case 2:
+      *out = std::make_shared<Int16Array>(
+          int16(), length_, data_, null_bitmap_, null_count_);
+      break;
+    case 4:
+      *out = std::make_shared<Int32Array>(
+          int32(), length_, data_, null_bitmap_, null_count_);
+      break;
+    case 8:
+      *out = std::make_shared<Int64Array>(
+          int64(), length_, data_, null_bitmap_, null_count_);
+      break;
+    default:
+      DCHECK(false);
+      return Status::NotImplemented("Only ints of size 1,2,4,8 are supported");
+  }
+
+  data_ = null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
+  return Status::OK();
+}
+
+Status AdaptiveIntBuilder::Append(
+    const int64_t* values, int64_t length, const uint8_t* valid_bytes) {
+  RETURN_NOT_OK(Reserve(length));
+
+  if (length > 0) {
+    if (int_size_ < 8) {
+      uint8_t new_int_size = int_size_;
+      for (int64_t i = 0; i < length; i++) {
+        if (valid_bytes == nullptr || valid_bytes[i]) {
+          new_int_size = expanded_int_size(values[i], new_int_size);
+        }
+      }
+      if (new_int_size != int_size_) { RETURN_NOT_OK(ExpandIntSize(new_int_size)); }
+    }
+  }
+
+  if (int_size_ == 8) {
+    std::memcpy(reinterpret_cast<int64_t*>(raw_data_) + length_, values,
+        sizeof(int64_t) * length);
+  } else {
+    // int_size_ may have changed, so we need to recheck
+    switch (int_size_) {
+      case 1: {
+        int8_t* data_ptr = reinterpret_cast<int8_t*>(raw_data_) + length_;
+        std::transform(values, values + length, data_ptr,
+            [](int64_t x) { return static_cast<int8_t>(x); });
+      } break;
+      case 2: {
+        int16_t* data_ptr = reinterpret_cast<int16_t*>(raw_data_) + length_;
+        std::transform(values, values + length, data_ptr,
+            [](int64_t x) { return static_cast<int16_t>(x); });
+      } break;
+      case 4: {
+        int32_t* data_ptr = reinterpret_cast<int32_t*>(raw_data_) + length_;
+        std::transform(values, values + length, data_ptr,
+            [](int64_t x) { return static_cast<int32_t>(x); });
+      } break;
+      default:
+        DCHECK(false);
+    }
+  }
+
+  // length_ is update by these
+  ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
+
+  return Status::OK();
+}
+
+template <typename new_type, typename old_type>
+typename std::enable_if<sizeof(old_type) >= sizeof(new_type), Status>::type
+AdaptiveIntBuilder::ExpandIntSizeInternal() {
+  return Status::OK();
+}
+
+#define __LESS(a, b) (a) < (b)
+template <typename new_type, typename old_type>
+typename std::enable_if<__LESS(sizeof(old_type), sizeof(new_type)), Status>::type
+AdaptiveIntBuilder::ExpandIntSizeInternal() {
+  int_size_ = sizeof(new_type);
+  RETURN_NOT_OK(Resize(data_->size() / sizeof(old_type)));
+
+  old_type* src = reinterpret_cast<old_type*>(raw_data_);
+  new_type* dst = reinterpret_cast<new_type*>(raw_data_);
+  // By doing the backward copy, we ensure that no element is overriden during
+  // the copy process and the copy stays in-place.
+  std::copy_backward(src, src + length_, dst + length_);
+
+  return Status::OK();
+}
+#undef __LESS
+
+template <typename new_type>
+Status AdaptiveIntBuilder::ExpandIntSizeN() {
+  switch (int_size_) {
+    case 1:
+      RETURN_NOT_OK((ExpandIntSizeInternal<new_type, int8_t>()));
+      break;
+    case 2:
+      RETURN_NOT_OK((ExpandIntSizeInternal<new_type, int16_t>()));
+      break;
+    case 4:
+      RETURN_NOT_OK((ExpandIntSizeInternal<new_type, int32_t>()));
+      break;
+    case 8:
+      RETURN_NOT_OK((ExpandIntSizeInternal<new_type, int64_t>()));
+      break;
+    default:
+      DCHECK(false);
+  }
+  return Status::OK();
+}
+
+Status AdaptiveIntBuilder::ExpandIntSize(uint8_t new_int_size) {
+  switch (new_int_size) {
+    case 1:
+      RETURN_NOT_OK((ExpandIntSizeN<int8_t>()));
+      break;
+    case 2:
+      RETURN_NOT_OK((ExpandIntSizeN<int16_t>()));
+      break;
+    case 4:
+      RETURN_NOT_OK((ExpandIntSizeN<int32_t>()));
+      break;
+    case 8:
+      RETURN_NOT_OK((ExpandIntSizeN<int64_t>()));
+      break;
+    default:
+      DCHECK(false);
+  }
+  return Status::OK();
+}
+
+AdaptiveUIntBuilder::AdaptiveUIntBuilder(MemoryPool* pool)
+    : AdaptiveIntBuilderBase(pool) {}
+
+Status AdaptiveUIntBuilder::Finish(std::shared_ptr<Array>* out) {
+  const int64_t bytes_required = length_ * int_size_;
+  if (bytes_required > 0 && bytes_required < data_->size()) {
+    // Trim buffers
+    RETURN_NOT_OK(data_->Resize(bytes_required));
+  }
+  switch (int_size_) {
+    case 1:
+      *out = std::make_shared<UInt8Array>(
+          uint8(), length_, data_, null_bitmap_, null_count_);
+      break;
+    case 2:
+      *out = std::make_shared<UInt16Array>(
+          uint16(), length_, data_, null_bitmap_, null_count_);
+      break;
+    case 4:
+      *out = std::make_shared<UInt32Array>(
+          uint32(), length_, data_, null_bitmap_, null_count_);
+      break;
+    case 8:
+      *out = std::make_shared<UInt64Array>(
+          uint64(), length_, data_, null_bitmap_, null_count_);
+      break;
+    default:
+      DCHECK(false);
+      return Status::NotImplemented("Only ints of size 1,2,4,8 are supported");
+  }
+
+  data_ = null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
+  return Status::OK();
+}
+
+Status AdaptiveUIntBuilder::Append(
+    const uint64_t* values, int64_t length, const uint8_t* valid_bytes) {
+  RETURN_NOT_OK(Reserve(length));
+
+  if (length > 0) {
+    if (int_size_ < 8) {
+      uint8_t new_int_size = int_size_;
+      for (int64_t i = 0; i < length; i++) {
+        if (valid_bytes == nullptr || valid_bytes[i]) {
+          new_int_size = expanded_uint_size(values[i], new_int_size);
+        }
+      }
+      if (new_int_size != int_size_) { RETURN_NOT_OK(ExpandIntSize(new_int_size)); }
+    }
+  }
+
+  if (int_size_ == 8) {
+    std::memcpy(reinterpret_cast<uint64_t*>(raw_data_) + length_, values,
+        sizeof(uint64_t) * length);
+  } else {
+    // int_size_ may have changed, so we need to recheck
+    switch (int_size_) {
+      case 1: {
+        uint8_t* data_ptr = reinterpret_cast<uint8_t*>(raw_data_) + length_;
+        std::transform(values, values + length, data_ptr,
+            [](uint64_t x) { return static_cast<uint8_t>(x); });
+      } break;
+      case 2: {
+        uint16_t* data_ptr = reinterpret_cast<uint16_t*>(raw_data_) + length_;
+        std::transform(values, values + length, data_ptr,
+            [](uint64_t x) { return static_cast<uint16_t>(x); });
+      } break;
+      case 4: {
+        uint32_t* data_ptr = reinterpret_cast<uint32_t*>(raw_data_) + length_;
+        std::transform(values, values + length, data_ptr,
+            [](uint64_t x) { return static_cast<uint32_t>(x); });
+      } break;
+      default:
+        DCHECK(false);
+    }
+  }
+
+  // length_ is update by these
+  ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
+
+  return Status::OK();
+}
+
+template <typename new_type, typename old_type>
+typename std::enable_if<sizeof(old_type) >= sizeof(new_type), Status>::type
+AdaptiveUIntBuilder::ExpandIntSizeInternal() {
+  return Status::OK();
+}
+
+#define __LESS(a, b) (a) < (b)
+template <typename new_type, typename old_type>
+typename std::enable_if<__LESS(sizeof(old_type), sizeof(new_type)), Status>::type
+AdaptiveUIntBuilder::ExpandIntSizeInternal() {
+  int_size_ = sizeof(new_type);
+  RETURN_NOT_OK(Resize(data_->size() / sizeof(old_type)));
+
+  old_type* src = reinterpret_cast<old_type*>(raw_data_);
+  new_type* dst = reinterpret_cast<new_type*>(raw_data_);
+  // By doing the backward copy, we ensure that no element is overriden during
+  // the copy process and the copy stays in-place.
+  std::copy_backward(src, src + length_, dst + length_);
+
+  return Status::OK();
+}
+#undef __LESS
+
+template <typename new_type>
+Status AdaptiveUIntBuilder::ExpandIntSizeN() {
+  switch (int_size_) {
+    case 1:
+      RETURN_NOT_OK((ExpandIntSizeInternal<new_type, uint8_t>()));
+      break;
+    case 2:
+      RETURN_NOT_OK((ExpandIntSizeInternal<new_type, uint16_t>()));
+      break;
+    case 4:
+      RETURN_NOT_OK((ExpandIntSizeInternal<new_type, uint32_t>()));
+      break;
+    case 8:
+      RETURN_NOT_OK((ExpandIntSizeInternal<new_type, uint64_t>()));
+      break;
+    default:
+      DCHECK(false);
+  }
+  return Status::OK();
+}
+
+Status AdaptiveUIntBuilder::ExpandIntSize(uint8_t new_int_size) {
+  switch (new_int_size) {
+    case 1:
+      RETURN_NOT_OK((ExpandIntSizeN<uint8_t>()));
+      break;
+    case 2:
+      RETURN_NOT_OK((ExpandIntSizeN<uint16_t>()));
+      break;
+    case 4:
+      RETURN_NOT_OK((ExpandIntSizeN<uint32_t>()));
+      break;
+    case 8:
+      RETURN_NOT_OK((ExpandIntSizeN<uint64_t>()));
+      break;
+    default:
+      DCHECK(false);
+  }
+  return Status::OK();
+}
+
 BooleanBuilder::BooleanBuilder(MemoryPool* pool)
     : ArrayBuilder(pool, boolean()), data_(nullptr), raw_data_(nullptr) {}
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 68769165b02c0..d77223e788378 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -20,6 +20,7 @@
 
 #include <cstdint>
 #include <functional>
+#include <limits>
 #include <memory>
 #include <string>
 #include <vector>
@@ -247,6 +248,193 @@ using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
 using FloatBuilder = NumericBuilder<FloatType>;
 using DoubleBuilder = NumericBuilder<DoubleType>;
 
+class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
+ public:
+  explicit AdaptiveIntBuilderBase(MemoryPool* pool);
+
+  /// Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+  Status AppendNulls(const uint8_t* valid_bytes, int64_t length) {
+    RETURN_NOT_OK(Reserve(length));
+    UnsafeAppendToBitmap(valid_bytes, length);
+    return Status::OK();
+  }
+
+  Status AppendNull() {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(false);
+    return Status::OK();
+  }
+
+  std::shared_ptr<Buffer> data() const { return data_; }
+
+  Status Init(int64_t capacity) override;
+
+  /// Increase the capacity of the builder to accommodate at least the indicated
+  /// number of elements
+  Status Resize(int64_t capacity) override;
+
+ protected:
+  std::shared_ptr<PoolBuffer> data_;
+  uint8_t* raw_data_;
+
+  uint8_t int_size_;
+};
+
+// Check if we would need to expand the underlying storage type
+inline uint8_t expanded_uint_size(uint64_t val, uint8_t current_int_size) {
+  if (current_int_size == 8 ||
+      (current_int_size < 8 &&
+          (val > static_cast<uint64_t>(std::numeric_limits<uint32_t>::max())))) {
+    return 8;
+  } else if (current_int_size == 4 ||
+             (current_int_size < 4 &&
+                 (val > static_cast<uint64_t>(std::numeric_limits<uint16_t>::max())))) {
+    return 4;
+  } else if (current_int_size == 2 ||
+             (current_int_size == 1 &&
+                 (val > static_cast<uint64_t>(std::numeric_limits<uint8_t>::max())))) {
+    return 2;
+  } else {
+    return 1;
+  }
+}
+
+class ARROW_EXPORT AdaptiveUIntBuilder : public AdaptiveIntBuilderBase {
+ public:
+  explicit AdaptiveUIntBuilder(MemoryPool* pool);
+
+  using ArrayBuilder::Advance;
+
+  /// Scalar append
+  Status Append(uint64_t val) {
+    RETURN_NOT_OK(Reserve(1));
+    BitUtil::SetBit(null_bitmap_data_, length_);
+
+    uint8_t new_int_size = expanded_uint_size(val, int_size_);
+    if (new_int_size != int_size_) { RETURN_NOT_OK(ExpandIntSize(new_int_size)); }
+
+    switch (int_size_) {
+      case 1:
+        reinterpret_cast<uint8_t*>(raw_data_)[length_++] = static_cast<uint8_t>(val);
+        break;
+      case 2:
+        reinterpret_cast<uint16_t*>(raw_data_)[length_++] = static_cast<uint16_t>(val);
+        break;
+      case 4:
+        reinterpret_cast<uint32_t*>(raw_data_)[length_++] = static_cast<uint32_t>(val);
+        break;
+      case 8:
+        reinterpret_cast<uint64_t*>(raw_data_)[length_++] = val;
+        break;
+      default:
+        return Status::NotImplemented("This code shall never be reached");
+    }
+    return Status::OK();
+  }
+
+  /// Vector append
+  ///
+  /// If passed, valid_bytes is of equal length to values, and any zero byte
+  /// will be considered as a null for that slot
+  Status Append(
+      const uint64_t* values, int64_t length, const uint8_t* valid_bytes = nullptr);
+
+  Status ExpandIntSize(uint8_t new_int_size);
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+ protected:
+  template <typename new_type, typename old_type>
+  typename std::enable_if<sizeof(old_type) >= sizeof(new_type), Status>::type
+  ExpandIntSizeInternal();
+#define __LESS(a, b) (a) < (b)
+  template <typename new_type, typename old_type>
+  typename std::enable_if<__LESS(sizeof(old_type), sizeof(new_type)), Status>::type
+  ExpandIntSizeInternal();
+#undef __LESS
+
+  template <typename new_type>
+  Status ExpandIntSizeN();
+};
+
+// Check if we would need to expand the underlying storage type
+inline uint8_t expanded_int_size(int64_t val, uint8_t current_int_size) {
+  if (current_int_size == 8 ||
+      (current_int_size < 8 &&
+          (val > static_cast<int64_t>(std::numeric_limits<int32_t>::max()) ||
+              val < static_cast<int64_t>(std::numeric_limits<int32_t>::min())))) {
+    return 8;
+  } else if (current_int_size == 4 ||
+             (current_int_size < 4 &&
+                 (val > static_cast<int64_t>(std::numeric_limits<int16_t>::max()) ||
+                     val < static_cast<int64_t>(std::numeric_limits<int16_t>::min())))) {
+    return 4;
+  } else if (current_int_size == 2 ||
+             (current_int_size == 1 &&
+                 (val > static_cast<int64_t>(std::numeric_limits<int8_t>::max()) ||
+                     val < static_cast<int64_t>(std::numeric_limits<int8_t>::min())))) {
+    return 2;
+  } else {
+    return 1;
+  }
+}
+
+class ARROW_EXPORT AdaptiveIntBuilder : public AdaptiveIntBuilderBase {
+ public:
+  explicit AdaptiveIntBuilder(MemoryPool* pool);
+
+  using ArrayBuilder::Advance;
+
+  /// Scalar append
+  Status Append(int64_t val) {
+    RETURN_NOT_OK(Reserve(1));
+    BitUtil::SetBit(null_bitmap_data_, length_);
+
+    uint8_t new_int_size = expanded_int_size(val, int_size_);
+    if (new_int_size != int_size_) { RETURN_NOT_OK(ExpandIntSize(new_int_size)); }
+
+    switch (int_size_) {
+      case 1:
+        reinterpret_cast<int8_t*>(raw_data_)[length_++] = static_cast<int8_t>(val);
+        break;
+      case 2:
+        reinterpret_cast<int16_t*>(raw_data_)[length_++] = static_cast<int16_t>(val);
+        break;
+      case 4:
+        reinterpret_cast<int32_t*>(raw_data_)[length_++] = static_cast<int32_t>(val);
+        break;
+      case 8:
+        reinterpret_cast<int64_t*>(raw_data_)[length_++] = val;
+        break;
+      default:
+        return Status::NotImplemented("This code shall never be reached");
+    }
+    return Status::OK();
+  }
+
+  /// Vector append
+  ///
+  /// If passed, valid_bytes is of equal length to values, and any zero byte
+  /// will be considered as a null for that slot
+  Status Append(
+      const int64_t* values, int64_t length, const uint8_t* valid_bytes = nullptr);
+
+  Status ExpandIntSize(uint8_t new_int_size);
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+ protected:
+  template <typename new_type, typename old_type>
+  typename std::enable_if<sizeof(old_type) >= sizeof(new_type), Status>::type
+  ExpandIntSizeInternal();
+#define __LESS(a, b) (a) < (b)
+  template <typename new_type, typename old_type>
+  typename std::enable_if<__LESS(sizeof(old_type), sizeof(new_type)), Status>::type
+  ExpandIntSizeInternal();
+#undef __LESS
+
+  template <typename new_type>
+  Status ExpandIntSizeN();
+};
+
 class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
  public:
   explicit BooleanBuilder(MemoryPool* pool);
@@ -271,7 +459,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 
   /// Scalar append
   Status Append(bool val) {
-    Reserve(1);
+    RETURN_NOT_OK(Reserve(1));
     BitUtil::SetBit(null_bitmap_data_, length_);
     if (val) {
       BitUtil::SetBit(raw_data_, length_);

From a16c1246ec25a020cacc1330650ac7d4cfd6d230 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 22 Jun 2017 18:06:12 -0400
Subject: [PATCH 0737/1644] ARROW-1137: Python: Ensure Pandas roundtrip of
 all-None column

Change-Id: Ib815d3fa42f0a0ae6c0d9850e9b0b435bad1c331

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #764 from xhochy/ARROW-1137 and squashes the following commits:

59c0df89 [Uwe L. Korn] Remove unused variables
1d11513f [Uwe L. Korn] ARROW-1137: Python: Ensure Pandas roundtrip of all-None column
---
 cpp/src/arrow/python/pandas_convert.cc      | 63 ++++++++++++++++++---
 python/pyarrow/tests/test_convert_pandas.py |  5 ++
 2 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index ac61cbc13c6ef..6b0e342908104 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -1305,6 +1305,22 @@ inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values)
   return Status::OK();
 }
 
+inline Status ConvertNulls(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    std::shared_ptr<Array> arr = data.chunk(c);
+
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      // All values are null
+      Py_INCREF(Py_None);
+      *out_values = Py_None;
+      ++out_values;
+    }
+  }
+  return Status::OK();
+}
+
 inline Status ConvertFixedSizeBinary(const ChunkedArray& data, PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
@@ -1457,6 +1473,8 @@ class ObjectBlock : public PandasBlock {
       RETURN_NOT_OK(ConvertFixedSizeBinary(data, out_buffer));
     } else if (type == Type::DECIMAL) {
       RETURN_NOT_OK(ConvertDecimals(data, out_buffer));
+    } else if (type == Type::NA) {
+      RETURN_NOT_OK(ConvertNulls(data, out_buffer));
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
       switch (list_type->value_type()->id()) {
@@ -1506,7 +1524,12 @@ class IntBlock : public PandasBlock {
 
     const ChunkedArray& data = *col->data().get();
 
-    if (type != ARROW_TYPE) { return Status::NotImplemented(col->type()->ToString()); }
+    if (type != ARROW_TYPE) {
+      std::stringstream ss;
+      ss << "Cannot write Arrow data of type " << col->type()->ToString();
+      ss << " to a Pandas int" << sizeof(C_TYPE) << " block.";
+      return Status::NotImplemented(ss.str());
+    }
 
     ConvertIntegerNoNullsSameType<C_TYPE>(data, out_buffer);
     placement_data_[rel_placement] = abs_placement;
@@ -1532,7 +1555,12 @@ class Float32Block : public PandasBlock {
       int64_t rel_placement) override {
     Type::type type = col->type()->id();
 
-    if (type != Type::FLOAT) { return Status::NotImplemented(col->type()->ToString()); }
+    if (type != Type::FLOAT) {
+      std::stringstream ss;
+      ss << "Cannot write Arrow data of type " << col->type()->ToString();
+      ss << " to a Pandas float32 block.";
+      return Status::NotImplemented(ss.str());
+    }
 
     float* out_buffer = reinterpret_cast<float*>(block_data_) + rel_placement * num_rows_;
 
@@ -1584,7 +1612,10 @@ class Float64Block : public PandasBlock {
         ConvertNumericNullable<double>(data, NAN, out_buffer);
         break;
       default:
-        return Status::NotImplemented(col->type()->ToString());
+        std::stringstream ss;
+        ss << "Cannot write Arrow data of type " << col->type()->ToString();
+        ss << " to a Pandas float64 block.";
+        return Status::NotImplemented(ss.str());
     }
 
 #undef INTEGER_CASE
@@ -1603,7 +1634,12 @@ class BoolBlock : public PandasBlock {
       int64_t rel_placement) override {
     Type::type type = col->type()->id();
 
-    if (type != Type::BOOL) { return Status::NotImplemented(col->type()->ToString()); }
+    if (type != Type::BOOL) {
+      std::stringstream ss;
+      ss << "Cannot write Arrow data of type " << col->type()->ToString();
+      ss << " to a Pandas boolean block.";
+      return Status::NotImplemented(ss.str());
+    }
 
     uint8_t* out_buffer =
         reinterpret_cast<uint8_t*>(block_data_) + rel_placement * num_rows_;
@@ -1660,7 +1696,10 @@ class DatetimeBlock : public PandasBlock {
         return Status::NotImplemented("Unsupported time unit");
       }
     } else {
-      return Status::NotImplemented(col->type()->ToString());
+      std::stringstream ss;
+      ss << "Cannot write Arrow data of type " << col->type()->ToString();
+      ss << " to a Pandas datetime block.";
+      return Status::NotImplemented(ss.str());
     }
 
     placement_data_[rel_placement] = abs_placement;
@@ -1917,8 +1956,14 @@ class DataFrameBlockCreator {
         case Type::DECIMAL:
           output_type = PandasBlock::DECIMAL;
           break;
+        case Type::NA:
+          output_type = PandasBlock::OBJECT;
+          break;
         default:
-          return Status::NotImplemented(col->type()->ToString());
+          std::stringstream ss;
+          ss << "No known equivalent Pandas block for Arrow data of type ";
+          ss << col->type()->ToString() << " is known.";
+          return Status::NotImplemented(ss.str());
       }
 
       int block_placement = 0;
@@ -2301,7 +2346,11 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
-  Status Visit(const NullType& type) { return Status::NotImplemented("null type"); }
+  Status Visit(const NullType& type) {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    return ConvertNulls(data_, out_values);
+  }
 
   Status Visit(const StructType& type) { return Status::NotImplemented("struct type"); }
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index ca304558c5fd3..d17ef3c0ad1c4 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -98,6 +98,11 @@ def _check_array_roundtrip(self, values, expected=None, mask=None,
             tm.assert_series_equal(pd.Series(result), expected,
                                    check_names=False)
 
+    def test_all_none_objects(self):
+        df = pd.DataFrame({'a': [None, None, None]})
+        self._check_pandas_roundtrip(df)
+
+
     def test_float_no_nulls(self):
         data = {}
         fields = []

From c1ec0c723e43b9450ef2655e6415eb40301c4ce8 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 23 Jun 2017 15:49:55 +0200
Subject: [PATCH 0738/1644] ARROW-1039: Python: Remove duplicate column

Note that part of this problem was related to the fix I made in https://github.com/apache/parquet-cpp/pull/358/files#diff-2f5ceebd1726b16db561185cc620d18e

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #773 from xhochy/ARROW-1039 and squashes the following commits:

44a002a [Uwe L. Korn] ARROW-1039: Python: Remove duplicate column
---
 python/pyarrow/array.pxi             | 29 ++++++++++++++++++++++++++++
 python/pyarrow/includes/libarrow.pxd |  6 ++++++
 python/pyarrow/tests/test_parquet.py |  4 ++--
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 5930de392715b..c7563c8536f61 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -432,6 +432,7 @@ cdef set PRIMITIVE_TYPES = set([
     _Type_UINT32, _Type_INT32,
     _Type_UINT64, _Type_INT64,
     _Type_TIMESTAMP, _Type_DATE32,
+    _Type_TIME32, _Type_TIME64,
     _Type_DATE64,
     _Type_HALF_FLOAT,
     _Type_FLOAT,
@@ -816,6 +817,32 @@ cdef class Date64Value(ArrayValue):
             ap.Value(self.index) / 1000).date()
 
 
+cdef class Time32Value(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CTime32Array* ap = <CTime32Array*> self.sp_array.get()
+            CTime32Type* dtype = <CTime32Type*> ap.type().get()
+
+        if dtype.unit() == TimeUnit_SECOND:
+            return (datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ap.Value(self.index))).time()
+        else:
+            return (datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=ap.Value(self.index))).time()
+
+
+cdef class Time64Value(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CTime64Array* ap = <CTime64Array*> self.sp_array.get()
+            CTime64Type* dtype = <CTime64Type*> ap.type().get()
+
+        if dtype.unit() == TimeUnit_MICRO:
+            return (datetime.datetime(1970, 1, 1) + datetime.timedelta(microseconds=ap.Value(self.index))).time()
+        else:
+            return (datetime.datetime(1970, 1, 1) + datetime.timedelta(microseconds=ap.Value(self.index) / 1000)).time()
+
+
 cdef dict DATETIME_CONVERSION_FUNCTIONS
 
 try:
@@ -975,6 +1002,8 @@ cdef dict _scalar_classes = {
     _Type_INT64: Int64Value,
     _Type_DATE32: Date32Value,
     _Type_DATE64: Date64Value,
+    _Type_TIME32: Time32Value,
+    _Type_TIME64: Time64Value,
     _Type_TIMESTAMP: TimestampValue,
     _Type_FLOAT: FloatValue,
     _Type_DOUBLE: DoubleValue,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 9df31c80ccf62..f712274426d56 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -249,6 +249,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CDate64Array" arrow::Date64Array"(CArray):
         int64_t Value(int i)
 
+    cdef cppclass CTime32Array" arrow::Time32Array"(CArray):
+        int32_t Value(int i)
+
+    cdef cppclass CTime64Array" arrow::Time64Array"(CArray):
+        int64_t Value(int i)
+
     cdef cppclass CTimestampArray" arrow::TimestampArray"(CArray):
         int64_t Value(int i)
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 052d395b9811c..3c2b73e3a535f 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -449,13 +449,13 @@ def test_date_time_types():
 
     table = pa.Table.from_arrays([a1, a2, a3, a4, a5, a6],
                                  ['date32', 'date64', 'timestamp[us]',
-                                  'time32[s]', 'time64[us]', 'time32[s]'])
+                                  'time32[s]', 'time64[us]', 'time32_from64[s]'])
 
     # date64 as date32
     # time32[s] to time32[ms]
     expected = pa.Table.from_arrays([a1, a1, a3, a4, a5, ex_a6],
                                     ['date32', 'date64', 'timestamp[us]',
-                                     'time32[s]', 'time64[us]', 'time32[s]'])
+                                     'time32[s]', 'time64[us]', 'time32_from64[s]'])
 
     _check_roundtrip(table, expected=expected, version='2.0')
 

From 074dde41c6ba70daeac07b2f88413e2eb21ec9f0 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 23 Jun 2017 10:12:07 -0400
Subject: [PATCH 0739/1644] ARROW-1143: C++: Fix comparison of NullArray

Change-Id: Ib18dc6b00c9806aaf541c61cb63673ac51b0525c

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #772 from xhochy/ARROW-1143 and squashes the following commits:

eff44306 [Uwe L. Korn] ARROW-1143: C++: Fix comparison of NullArray
---
 cpp/src/arrow/array-test.cc                 | 10 ++++++++++
 cpp/src/arrow/compare.cc                    |  7 +++++--
 python/pyarrow/tests/test_convert_pandas.py |  6 ++++++
 python/pyarrow/tests/test_parquet.py        |  6 +++++-
 4 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index beffa1b11cbe2..8f6323b7555ba 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -95,6 +95,16 @@ TEST_F(TestArray, TestEquality) {
   EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
 }
 
+TEST_F(TestArray, TestNullArrayEquality) {
+  auto array_1 = std::make_shared<NullArray>(10);
+  auto array_2 = std::make_shared<NullArray>(10);
+  auto array_3 = std::make_shared<NullArray>(20);
+
+  EXPECT_TRUE(array_1->Equals(array_1));
+  EXPECT_TRUE(array_1->Equals(array_2));
+  EXPECT_FALSE(array_1->Equals(array_3));
+}
+
 TEST_F(TestArray, SliceRecomputeNullCount) {
   vector<uint8_t> valid_bytes = {1, 0, 1, 1, 0, 1, 0, 0, 0};
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 562d4e1b4ddff..c2f4f845baf34 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -322,7 +322,10 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
   explicit ArrayEqualsVisitor(const Array& right)
       : RangeEqualsVisitor(right, 0, right.length(), 0) {}
 
-  Status Visit(const NullArray& left) { return Status::OK(); }
+  Status Visit(const NullArray& left) {
+      result_ = true;
+      return Status::OK();
+  }
 
   Status Visit(const BooleanArray& left) {
     const auto& right = static_cast<const BooleanArray&>(right_);
@@ -529,7 +532,7 @@ static bool BaseDataEquals(const Array& left, const Array& right) {
       left.type_id() != right.type_id()) {
     return false;
   }
-  if (left.null_count() > 0) {
+  if (left.null_count() > 0 && left.null_count() < left.length()) {
     return BitmapEquals(left.null_bitmap()->data(), left.offset(),
         right.null_bitmap()->data(), right.offset(), left.length());
   }
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index d17ef3c0ad1c4..f6ada09cc2cdf 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -103,6 +103,12 @@ def test_all_none_objects(self):
         self._check_pandas_roundtrip(df)
 
 
+    def test_all_none_category(self):
+        df = pd.DataFrame({'a': [None, None, None]})
+        df['a'] = df['a'].astype('category')
+        self._check_pandas_roundtrip(df)
+
+
     def test_float_no_nulls(self):
         data = {}
         fields = []
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 3c2b73e3a535f..1cf54959da073 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -225,8 +225,12 @@ def _test_dataframe(size=10000, seed=0):
         'float32': np.random.randn(size).astype(np.float32),
         'float64': np.arange(size, dtype=np.float64),
         'bool': np.random.randn(size) > 0,
-        'strings': [tm.rands(10) for i in range(size)]
+        'strings': [tm.rands(10) for i in range(size)],
+        'all_none': [None] * size,
+        'all_none_category': [None] * size
     })
+    # TODO(PARQUET-1015)
+    # df['all_none_category'] = df['all_none_category'].astype('category')
     return df
 
 

From e209e5865ea58e57925cae24d4bf3f63d58ee21d Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 23 Jun 2017 11:10:57 -0400
Subject: [PATCH 0740/1644] ARROW-1144: [C++] Remove unused variable

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #774 from cpcloud/ARROW-1144 and squashes the following commits:

a79aebaa [Phillip Cloud] ARROW-1144: [C++] Remove unused variable
---
 cpp/src/arrow/python/pandas_convert.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 6b0e342908104..654c3921e8b61 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -1310,7 +1310,6 @@ inline Status ConvertNulls(const ChunkedArray& data, PyObject** out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     std::shared_ptr<Array> arr = data.chunk(c);
 
-    const bool has_nulls = data.null_count() > 0;
     for (int64_t i = 0; i < arr->length(); ++i) {
       // All values are null
       Py_INCREF(Py_None);

From 6768f5268700aebdd4e5fe6a99ca5c912640ca82 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 23 Jun 2017 12:56:36 -0400
Subject: [PATCH 0741/1644] ARROW-1139: Silence dlmalloc warning on clang-4.0

An alternative would be to patch dlmalloc similarly to https://github.com/greg7mdp/dlmalloc

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #769 from pcmoritz/dlmalloc-fix and squashes the following commits:

6602c7f5 [Philipp Moritz] clean up
672ec2ee [Philipp Moritz] fix on g++
fd91ccdb [Philipp Moritz] silence dlmalloc warnings on clang-4.0
---
 cpp/src/plasma/CMakeLists.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 992c33ed0fcc9..8f2d81fba78bd 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -67,7 +67,6 @@ endif()
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
 
-set_source_files_properties(thirdparty/dlmalloc.c PROPERTIES COMPILE_FLAGS -Wno-all)
 set_source_files_properties(extension.cc PROPERTIES COMPILE_FLAGS -Wno-strict-aliasing)
 
 set(PLASMA_SRCS
@@ -91,7 +90,7 @@ ADD_ARROW_LIB(plasma
 
 # The optimization flag -O3 is suggested by dlmalloc.c, which is #included in
 # malloc.cc; we set it here regardless of whether we do a debug or release build.
-set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error=conversion -O3")
+set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error -O3")
 
 add_executable(plasma_store store.cc)
 target_link_libraries(plasma_store plasma_static)

From 8bf567e636e6f8a7e779a6f89ad3169f3ffa9fba Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 23 Jun 2017 14:26:12 -0400
Subject: [PATCH 0742/1644] ARROW-1136: [C++] Add null checks for invalid
 streams

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #770 from wesm/ARROW-1136 and squashes the following commits:

6ae5cd82 [Wes McKinney] Centralize null checking
bc3ec207 [Wes McKinney] Add null checks for invalid streams
---
 cpp/src/arrow/ipc/reader.cc      | 20 ++++++++++++++------
 python/pyarrow/tests/test_ipc.py | 10 ++++++++++
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 2b7b90f2f2ea7..7fef847c9f213 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -162,7 +162,7 @@ static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
   return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
 }
 
-static inline std::string message_type_name(Message::Type type) {
+static inline std::string FormatMessageType(Message::Type type) {
   switch (type) {
     case Message::SCHEMA:
       return "schema";
@@ -188,14 +188,22 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
     return ReadSchema();
   }
 
-  Status ReadNextMessage(Message::Type expected_type, std::shared_ptr<Message>* message) {
+  Status ReadNextMessage(Message::Type expected_type, bool allow_null,
+      std::shared_ptr<Message>* message) {
     RETURN_NOT_OK(ReadMessage(stream_.get(), message));
 
+    if (!(*message) && !allow_null) {
+      std::stringstream ss;
+      ss << "Expected " << FormatMessageType(expected_type)
+         << " message in stream, was null or length 0";
+      return Status::Invalid(ss.str());
+    }
+
     if ((*message) == nullptr) { return Status::OK(); }
 
     if ((*message)->type() != expected_type) {
       std::stringstream ss;
-      ss << "Message not expected type: " << message_type_name(expected_type)
+      ss << "Message not expected type: " << FormatMessageType(expected_type)
          << ", was: " << (*message)->type();
       return Status::IOError(ss.str());
     }
@@ -213,7 +221,7 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
 
   Status ReadNextDictionary() {
     std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::DICTIONARY_BATCH, &message));
+    RETURN_NOT_OK(ReadNextMessage(Message::DICTIONARY_BATCH, false, &message));
 
     std::shared_ptr<Buffer> batch_body;
     RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body))
@@ -227,7 +235,7 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
 
   Status ReadSchema() {
     std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::SCHEMA, &message));
+    RETURN_NOT_OK(ReadNextMessage(Message::SCHEMA, false, &message));
 
     RETURN_NOT_OK(GetDictionaryTypes(message->header(), &dictionary_types_));
 
@@ -243,7 +251,7 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
 
   Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
     std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::RECORD_BATCH, &message));
+    RETURN_NOT_OK(ReadNextMessage(Message::RECORD_BATCH, true, &message));
 
     if (message == nullptr) {
       // End of stream
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index eeea39ab19416..47ef75602bc65 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -72,6 +72,11 @@ class TestFile(MessagingTest, unittest.TestCase):
     def _get_writer(self, sink, schema):
         return pa.RecordBatchFileWriter(sink, schema)
 
+    def test_empty_file(self):
+        buf = io.BytesIO(b'')
+        with pytest.raises(pa.ArrowInvalid):
+            pa.open_file(buf)
+
     def test_simple_roundtrip(self):
         batches = self.write_batches()
         file_contents = self._get_source()
@@ -101,6 +106,11 @@ class TestStream(MessagingTest, unittest.TestCase):
     def _get_writer(self, sink, schema):
         return pa.RecordBatchStreamWriter(sink, schema)
 
+    def test_empty_stream(self):
+        buf = io.BytesIO(b'')
+        with pytest.raises(pa.ArrowInvalid):
+            pa.open_stream(buf)
+
     def test_simple_roundtrip(self):
         batches = self.write_batches()
         file_contents = self._get_source()

From b7befeb7c9e34d86b7341ba4d8710104f42efc0b Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 23 Jun 2017 15:29:42 -0400
Subject: [PATCH 0743/1644] ARROW-1132: [Python] Unable to write pandas
 DataFrame w/MultiIndex containing duplicate values to parquet

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #768 from cpcloud/ARROW-1132 and squashes the following commits:

4b42f643 [Phillip Cloud] Add test for parquet roundtripping with dups
49684fd1 [Phillip Cloud] ARROW-1132: [Python] Unable to write pandas DataFrame w/MultiIndex containing duplicate values to parquet
---
 python/pyarrow/table.pxi                    | 15 +++++++++------
 python/pyarrow/tests/test_convert_pandas.py | 14 ++++++++++++++
 python/pyarrow/tests/test_parquet.py        | 21 +++++++++++++++++++++
 3 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 3f67ba40c7321..7d44f2e46eec6 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -320,12 +320,15 @@ cdef tuple _dataframe_to_arrays(
     cdef:
         list names = []
         list arrays = []
-        list index_levels = []
+        list index_columns = []
         DataType type = None
         dict metadata
+        Py_ssize_t i
+        Py_ssize_t n
 
     if preserve_index:
-        index_levels.extend(getattr(df.index, 'levels', [df.index]))
+        n = len(getattr(df.index, 'levels', [df.index]))
+        index_columns.extend(df.index.get_level_values(i) for i in range(n))
 
     for name in df.columns:
         col = df[name]
@@ -339,13 +342,13 @@ cdef tuple _dataframe_to_arrays(
         )
         names.append(name)
 
-    for i, level in enumerate(index_levels):
+    for i, column in enumerate(index_columns):
         arrays.append(
-            Array.from_pandas(level, timestamps_to_ms=timestamps_to_ms)
+            Array.from_pandas(column, timestamps_to_ms=timestamps_to_ms)
         )
-        names.append(pdcompat.index_level_name(level, i))
+        names.append(pdcompat.index_level_name(column, i))
 
-    metadata = pdcompat.construct_metadata(df, index_levels, preserve_index)
+    metadata = pdcompat.construct_metadata(df, index_columns, preserve_index)
     return names, arrays, metadata
 
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index f6ada09cc2cdf..49b7eb73df09b 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -640,3 +640,17 @@ def _check_series(s):
         _check_series(pd.Series([None] * 3, dtype=object))
         _check_series(pd.Series([np.nan] * 3, dtype=object))
         _check_series(pd.Series([np.sqrt(-1)] * 3, dtype=object))
+
+    def test_multiindex_duplicate_values(self):
+        num_rows = 3
+        numbers = list(range(num_rows))
+        index = pd.MultiIndex.from_arrays(
+            [['foo', 'foo', 'bar'], numbers],
+            names=['foobar', 'some_numbers'],
+        )
+
+        df = pd.DataFrame({'numbers': numbers}, index=index)
+
+        table = pa.Table.from_pandas(df)
+        result_df = table.to_pandas()
+        tm.assert_frame_equal(result_df, df)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 1cf54959da073..4e8f0db046c62 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -847,3 +847,24 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
 
     with pytest.raises(ValueError):
         read_multiple_files(mixed_paths)
+
+
+def test_multiindex_duplicate_values(tmpdir):
+    num_rows = 3
+    numbers = list(range(num_rows))
+    index = pd.MultiIndex.from_arrays(
+        [['foo', 'foo', 'bar'], numbers],
+        names=['foobar', 'some_numbers'],
+    )
+
+    df = pd.DataFrame({'numbers': numbers}, index=index)
+    table = pa.Table.from_pandas(df)
+
+    filename = tmpdir.join('dup_multi_index_levels.parquet').strpath
+
+    _write_table(table, filename)
+    result_table = _read_table(filename)
+    assert table.equals(result_table)
+
+    result_df = result_table.to_pandas()
+    tm.assert_frame_equal(result_df, df)

From 1514016a72385f28235fc36b19274b46e25c389a Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 23 Jun 2017 17:06:33 -0400
Subject: [PATCH 0744/1644] ARROW-1146: Add .gitignore for *_generated.h files
 in src/plasma/format

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #777 from cpcloud/ARROW-1146 and squashes the following commits:

f97de967 [Phillip Cloud] ARROW-1146: Add .gitignore for *_generated.h files in src/plasma/format
---
 cpp/src/arrow/ipc/.gitignore     | 2 +-
 cpp/src/plasma/format/.gitignore | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 cpp/src/plasma/format/.gitignore

diff --git a/cpp/src/arrow/ipc/.gitignore b/cpp/src/arrow/ipc/.gitignore
index 8150d7efe33c4..b2ddb055dcbc6 100644
--- a/cpp/src/arrow/ipc/.gitignore
+++ b/cpp/src/arrow/ipc/.gitignore
@@ -1 +1 @@
-*_generated.h
\ No newline at end of file
+*_generated.h
diff --git a/cpp/src/plasma/format/.gitignore b/cpp/src/plasma/format/.gitignore
new file mode 100644
index 0000000000000..b2ddb055dcbc6
--- /dev/null
+++ b/cpp/src/plasma/format/.gitignore
@@ -0,0 +1 @@
+*_generated.h

From 98f7cac6e162d9775d615d07b9867c1ec0030f82 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 23 Jun 2017 19:06:55 -0400
Subject: [PATCH 0745/1644] ARROW-1142: [C++] Port over compression toolchain
 and interfaces from parquet-cpp, use Arrow-style error handling

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #771 from wesm/import-parquet-compression and squashes the following commits:

b7609f8e [Wes McKinney] Boost toolchain tweaks
44e77a64 [Wes McKinney] Install compression.h
732e426e [Wes McKinney] Revert bash equality test
31b2705a [Wes McKinney] cpplint
2ef43de2 [Wes McKinney] Clean up RAT exclusions, use absolute paths
cbbaecf0 [Wes McKinney] Add some license headers, use Apache Kudu approach for managing a lot of RAT exclusions
198dee16 [Wes McKinney] Clean up build dependencies
da31c2cb [Wes McKinney] Port over compression toolchain and interfaces from parquet-cpp, adapt to Arrow-style error handling
---
 .readthedocs.yml                            |  17 +
 .travis.yml                                 |  17 +
 ci/travis_script_cpp.sh                     |   2 +-
 cpp/.clang-format                           |  28 +-
 cpp/.clang-tidy                             |  19 +-
 cpp/.clang-tidy-ignore                      |  16 +
 cpp/CMakeLists.txt                          | 505 ++-------------
 cpp/cmake_modules/FindBrotli.cmake          | 116 ++++
 cpp/cmake_modules/FindSnappy.cmake          |  94 +++
 cpp/cmake_modules/FindZLIB.cmake            | 105 ++++
 cpp/cmake_modules/SnappyCMakeLists.txt      |  85 +++
 cpp/cmake_modules/SnappyConfig.h            |  36 ++
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 641 ++++++++++++++++++++
 cpp/src/arrow/python/CMakeLists.txt         |   4 +-
 cpp/src/arrow/util/CMakeLists.txt           |   7 +-
 cpp/src/arrow/util/compression-test.cc      |  89 +++
 cpp/src/arrow/util/compression.cc           | 327 ++++++++++
 cpp/src/arrow/util/compression.h            | 109 ++++
 cpp/src/arrow/util/logging.h                |   7 +-
 cpp/src/plasma/malloc.cc                    |   2 +-
 cpp/src/plasma/test/client_tests.cc         |   2 +-
 dev/release/check-rat-report.py             |  59 ++
 dev/release/rat_exclude_files.txt           |  66 ++
 dev/release/run-rat.sh                      |  64 +-
 24 files changed, 1890 insertions(+), 527 deletions(-)
 create mode 100644 cpp/cmake_modules/FindBrotli.cmake
 create mode 100644 cpp/cmake_modules/FindSnappy.cmake
 create mode 100644 cpp/cmake_modules/FindZLIB.cmake
 create mode 100644 cpp/cmake_modules/SnappyCMakeLists.txt
 create mode 100644 cpp/cmake_modules/SnappyConfig.h
 create mode 100644 cpp/cmake_modules/ThirdpartyToolchain.cmake
 create mode 100644 cpp/src/arrow/util/compression-test.cc
 create mode 100644 cpp/src/arrow/util/compression.cc
 create mode 100644 cpp/src/arrow/util/compression.h
 create mode 100644 dev/release/check-rat-report.py
 create mode 100644 dev/release/rat_exclude_files.txt

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 2e1fe3fbc251a..11a7d70c25131 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -1,2 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 conda:
     file: python/doc/environment.yml
diff --git a/.travis.yml b/.travis.yml
index a32562fdd19a2..315cbd21f2e19 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 sudo: required
 dist: trusty
 addons:
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index d555cab3e640c..c368a1daeddb9 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -17,7 +17,7 @@ set -e
 : ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
 
 # Check licenses according to Apache policy
-git archive HEAD -o arrow-src.tar.gz
+git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
 ./dev/release/run-rat.sh arrow-src.tar.gz
 
 pushd $CPP_BUILD_DIR
diff --git a/cpp/.clang-format b/cpp/.clang-format
index 7d5b3cf30ef51..33f282a20de20 100644
--- a/cpp/.clang-format
+++ b/cpp/.clang-format
@@ -1,34 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 ---
 Language:        Cpp
 # BasedOnStyle:  Google
 AccessModifierOffset: -1
-AlignAfterOpenBracket: false 
+AlignAfterOpenBracket: false
 AlignConsecutiveAssignments: false
 AlignEscapedNewlinesLeft: true
 AlignOperands:   true
 AlignTrailingComments: true
 AllowAllParametersOfDeclarationOnNextLine: true
-AllowShortBlocksOnASingleLine: true 
+AllowShortBlocksOnASingleLine: true
 AllowShortCaseLabelsOnASingleLine: false
 AllowShortFunctionsOnASingleLine: Inline
 AllowShortIfStatementsOnASingleLine: true
-AllowShortLoopsOnASingleLine: false 
+AllowShortLoopsOnASingleLine: false
 AlwaysBreakAfterDefinitionReturnType: None
 AlwaysBreakBeforeMultilineStrings: true
 AlwaysBreakTemplateDeclarations: true
 BinPackArguments: true
-BinPackParameters: true 
+BinPackParameters: true
 BreakBeforeBinaryOperators: None
 BreakBeforeBraces: Attach
 BreakBeforeTernaryOperators: true
 BreakConstructorInitializersBeforeComma: false
-ColumnLimit: 90 
+ColumnLimit: 90
 CommentPragmas:  '^ IWYU pragma:'
 ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
-DerivePointerAlignment: false 
+DerivePointerAlignment: false
 DisableFormat:   false
 ExperimentalAutoDetectBinPacking: false
 ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
diff --git a/cpp/.clang-tidy b/cpp/.clang-tidy
index deaa9bdf97fa1..b6b5a818f0a0e 100644
--- a/cpp/.clang-tidy
+++ b/cpp/.clang-tidy
@@ -1,8 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 ---
 Checks:          'clang-diagnostic-*,clang-analyzer-*,-clang-analyzer-alpha*,google-.*,modernize-.*,readablity-.*'
 HeaderFilterRegex: 'arrow/.*'
 AnalyzeTemporaryDtors: true
-CheckOptions:    
+CheckOptions:
   - key:             google-readability-braces-around-statements.ShortStatementLines
     value:           '1'
   - key:             google-readability-function-size.StatementThreshold
@@ -11,4 +27,3 @@ CheckOptions:
     value:           '10'
   - key:             google-readability-namespace-comments.SpacesBeforeComments
     value:           '2'
-
diff --git a/cpp/.clang-tidy-ignore b/cpp/.clang-tidy-ignore
index 5ab4d20d61942..3270b973f2e9a 100644
--- a/cpp/.clang-tidy-ignore
+++ b/cpp/.clang-tidy-ignore
@@ -1,2 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 ipc-adapter-test.cc
 memory-pool-test.cc
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5ba56e5963213..49e1d97510e26 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -136,6 +136,18 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(ARROW_PLASMA
     "Build the plasma object store along with Arrow"
     OFF)
+
+  option(ARROW_ZLIB_VENDORED
+    "Build our own zlib (some libz.a aren't configured for static linking)"
+    ON)
+  if (MSVC)
+    set(BROTLI_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+      "Brotli static lib suffix used on Windows with MSVC (default _static)")
+    set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "" CACHE STRING
+      "Snappy static lib suffix used on Windows with MSVC (default is empty string)")
+    set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING
+      "Zlib static lib suffix used on Windows with MSVC (default libstatic)")
+  endif()
 endif()
 
 if(ARROW_BUILD_TESTS)
@@ -166,9 +178,14 @@ if (ARROW_NO_DEPRECATED_API)
   add_definitions(-DARROW_NO_DEPRECATED_API)
 endif()
 
+############################################################
+# Dependencies
+############################################################
+
+include(ThirdpartyToolchain)
+
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMMON_FLAGS}")
-set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}")
 
 message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
@@ -400,444 +417,6 @@ endfunction()
 
 enable_testing()
 
-############################################################
-# Dependencies
-############################################################
-
-# ----------------------------------------------------------------------
-# Thirdparty toolchain
-
-set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
-set(GFLAGS_VERSION "2.1.2")
-set(GTEST_VERSION "1.8.0")
-set(GBENCHMARK_VERSION "1.1.0")
-set(FLATBUFFERS_VERSION "1.6.0")
-set(JEMALLOC_VERSION "4.4.0")
-
-if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
-  set(FLATBUFFERS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
-  set(RAPIDJSON_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
-  set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
-  set(GFLAGS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
-
-  if (NOT DEFINED ENV{BOOST_ROOT})
-    # Since we have to set this in the environment, we check whether
-    # $BOOST_ROOT is defined inside here
-    set(ENV{BOOST_ROOT} "$ENV{ARROW_BUILD_TOOLCHAIN}")
-  endif()
-endif()
-
-if (DEFINED ENV{FLATBUFFERS_HOME})
-  set(FLATBUFFERS_HOME "$ENV{FLATBUFFERS_HOME}")
-endif()
-
-if (DEFINED ENV{RAPIDJSON_HOME})
-  set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}")
-endif()
-
-if (DEFINED ENV{JEMALLOC_HOME})
-  set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}")
-endif()
-
-if (DEFINED ENV{GFLAGS_HOME})
-  set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
-endif()
-
-# ----------------------------------------------------------------------
-# Find pthreads
-
-if (NOT MSVC)
-  find_library(PTHREAD_LIBRARY pthread)
-  message(STATUS "Found pthread: ${PTHREAD_LIBRARY}")
-endif()
-
-# ----------------------------------------------------------------------
-# Add Boost dependencies (code adapted from Apache Kudu (incubating))
-
-set(Boost_DEBUG TRUE)
-set(Boost_USE_MULTITHREADED ON)
-set(Boost_ADDITIONAL_VERSIONS
-  "1.63.0" "1.63"
-  "1.62.0" "1.61"
-  "1.61.0" "1.62"
-  "1.60.0" "1.60")
-
-if (ARROW_BOOST_USE_SHARED)
-  # Find shared Boost libraries.
-  set(Boost_USE_STATIC_LIBS OFF)
-
-  if(MSVC)
-    # disable autolinking in boost
-    add_definitions(-DBOOST_ALL_NO_LIB)
-
-    # force all boost libraries to dynamic link
-    add_definitions(-DBOOST_ALL_DYN_LINK)
-  endif()
-
-  if (ARROW_BOOST_HEADER_ONLY)
-    find_package(Boost)
-  else()
-    find_package(Boost COMPONENTS system filesystem REQUIRED)
-    if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-      set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-      set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
-    else()
-      set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-      set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
-    endif()
-    set(BOOST_SYSTEM_LIBRARY boost_system_shared)
-    set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
-  endif()
-else()
-  # Find static boost headers and libs
-  # TODO Differentiate here between release and debug builds
-  set(Boost_USE_STATIC_LIBS ON)
-  if (ARROW_BOOST_HEADER_ONLY)
-    find_package(Boost)
-  else()
-    find_package(Boost COMPONENTS system filesystem regex REQUIRED)
-    if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-      set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-      set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
-    else()
-      set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-      set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
-    endif()
-    set(BOOST_SYSTEM_LIBRARY boost_system_static)
-    set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
-  endif()
-endif()
-
-message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
-message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
-
-if (NOT ARROW_BOOST_HEADER_ONLY)
-  ADD_THIRDPARTY_LIB(boost_system
-      STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}"
-      SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
-
-  ADD_THIRDPARTY_LIB(boost_filesystem
-      STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
-      SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
-
-  SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
-endif()
-
-include_directories(SYSTEM ${Boost_INCLUDE_DIR})
-
-if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
-  add_custom_target(unittest ctest -L unittest)
-
-  if("$ENV{GTEST_HOME}" STREQUAL "")
-    if(APPLE)
-      set(GTEST_CMAKE_CXX_FLAGS "-fPIC -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
-    elseif(NOT MSVC)
-      set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
-    endif()
-    string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
-    set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}")
-
-    set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
-    set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
-    set(GTEST_STATIC_LIB
-      "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(GTEST_MAIN_STATIC_LIB
-      "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(GTEST_VENDORED 1)
-    set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-                         -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}
-                         -Dgtest_force_shared_crt=ON
-                         -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
-
-    if (CMAKE_VERSION VERSION_GREATER "3.2")
-      # BUILD_BYPRODUCTS is a 3.2+ feature
-      ExternalProject_Add(googletest_ep
-        URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-        BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB}
-        CMAKE_ARGS ${GTEST_CMAKE_ARGS})
-    else()
-      ExternalProject_Add(googletest_ep
-        URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-        CMAKE_ARGS ${GTEST_CMAKE_ARGS})
-    endif()
-  else()
-    find_package(GTest REQUIRED)
-    set(GTEST_VENDORED 0)
-  endif()
-
-  message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}")
-  message(STATUS "GTest static library: ${GTEST_STATIC_LIB}")
-  include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(gtest
-    STATIC_LIB ${GTEST_STATIC_LIB})
-  ADD_THIRDPARTY_LIB(gtest_main
-    STATIC_LIB ${GTEST_MAIN_STATIC_LIB})
-
-  if(GTEST_VENDORED)
-    add_dependencies(gtest googletest_ep)
-    add_dependencies(gtest_main googletest_ep)
-  endif()
-
-  # gflags (formerly Googleflags) command line parsing
-  if("${GFLAGS_HOME}" STREQUAL "")
-    set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS})
-
-    set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
-    set(GFLAGS_HOME "${GFLAGS_PREFIX}")
-    set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
-    if(MSVC)
-      set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/gflags_static.lib")
-    else()
-      set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
-    endif()
-    set(GFLAGS_VENDORED 1)
-    set(GFLAGS_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-                          -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
-                          -DBUILD_SHARED_LIBS=OFF
-                          -DBUILD_STATIC_LIBS=ON
-                          -DBUILD_PACKAGING=OFF
-                          -DBUILD_TESTING=OFF
-                          -BUILD_CONFIG_TESTS=OFF
-                          -DINSTALL_HEADERS=ON
-                          -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
-    if (CMAKE_VERSION VERSION_GREATER "3.2")
-      # BUILD_BYPRODUCTS is a 3.2+ feature
-      ExternalProject_Add(gflags_ep
-        GIT_REPOSITORY https://github.com/gflags/gflags.git
-        GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
-        BUILD_IN_SOURCE 1
-        BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}"
-        CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
-    else()
-      ExternalProject_Add(gflags_ep
-        GIT_REPOSITORY https://github.com/gflags/gflags.git
-        GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
-        BUILD_IN_SOURCE 1
-        CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
-    endif()
-  else()
-    set(GFLAGS_VENDORED 0)
-    find_package(GFlags REQUIRED)
-  endif()
-
-  message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}")
-  message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}")
-  include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(gflags
-    STATIC_LIB ${GFLAGS_STATIC_LIB})
-  if(MSVC)
-    set_target_properties(gflags
-      PROPERTIES
-      IMPORTED_LINK_INTERFACE_LIBRARIES "shlwapi.lib")
-  endif()
-
-  if(GFLAGS_VENDORED)
-    add_dependencies(gflags gflags_ep)
-  endif()
-endif()
-
-if(ARROW_BUILD_BENCHMARKS)
-  add_custom_target(runbenchmark ctest -L benchmark)
-
-  if("$ENV{GBENCHMARK_HOME}" STREQUAL "")
-    if(APPLE)
-      set(GBENCHMARK_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++")
-    elseif(NOT MSVC)
-      set(GBENCHMARK_CMAKE_CXX_FLAGS "-fPIC --std=c++11")
-    endif()
-
-    set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install")
-    set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include")
-    set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(GBENCHMARK_VENDORED 1)
-    set(GBENCHMARK_CMAKE_ARGS
-          "-DCMAKE_BUILD_TYPE=Release"
-          "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}"
-          "-DBENCHMARK_ENABLE_TESTING=OFF"
-          "-DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}")
-    if (APPLE)
-      set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON")
-    endif()
-    if (CMAKE_VERSION VERSION_GREATER "3.2")
-      # BUILD_BYPRODUCTS is a 3.2+ feature
-      ExternalProject_Add(gbenchmark_ep
-        URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
-        BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}"
-        CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
-    else()
-      ExternalProject_Add(gbenchmark_ep
-        URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
-        CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
-    endif()
-  else()
-    find_package(GBenchmark REQUIRED)
-    set(GBENCHMARK_VENDORED 0)
-  endif()
-
-  message(STATUS "GBenchmark include dir: ${GBENCHMARK_INCLUDE_DIR}")
-  message(STATUS "GBenchmark static library: ${GBENCHMARK_STATIC_LIB}")
-  include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(benchmark
-    STATIC_LIB ${GBENCHMARK_STATIC_LIB})
-
-  if(GBENCHMARK_VENDORED)
-    add_dependencies(benchmark gbenchmark_ep)
-  endif()
-endif()
-
-if (ARROW_IPC)
-  # RapidJSON, header only dependency
-  if("${RAPIDJSON_HOME}" STREQUAL "")
-    ExternalProject_Add(rapidjson_ep
-      PREFIX "${CMAKE_BINARY_DIR}"
-      URL "https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz"
-      URL_MD5 "badd12c511e081fec6c89c43a7027bce"
-      CONFIGURE_COMMAND ""
-      BUILD_COMMAND ""
-      BUILD_IN_SOURCE 1
-      INSTALL_COMMAND "")
-
-    ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR)
-    set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include")
-    set(RAPIDJSON_VENDORED 1)
-  else()
-    set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_HOME}/include")
-    set(RAPIDJSON_VENDORED 0)
-  endif()
-  message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
-  include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
-
-  ## Flatbuffers
-  if("${FLATBUFFERS_HOME}" STREQUAL "")
-    set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
-    ExternalProject_Add(flatbuffers_ep
-      URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
-      CMAKE_ARGS
-      "-DCMAKE_CXX_FLAGS=-fPIC"
-      "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
-      "-DFLATBUFFERS_BUILD_TESTS=OFF")
-
-    set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
-    set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
-    set(FLATBUFFERS_VENDORED 1)
-  else()
-    find_package(Flatbuffers REQUIRED)
-    set(FLATBUFFERS_VENDORED 0)
-  endif()
-
-  message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
-  message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
-  include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
-endif()
-#----------------------------------------------------------------------
-
-if (MSVC)
-  # jemalloc is not supported on Windows
-  set(ARROW_JEMALLOC off)
-endif()
-
-if (ARROW_JEMALLOC)
-  find_package(jemalloc)
-
-  if(NOT JEMALLOC_FOUND)
-    set(ARROW_JEMALLOC_USE_SHARED OFF)
-    set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
-    set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
-    set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
-    set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
-    set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(JEMALLOC_VENDORED 1)
-    if (CMAKE_VERSION VERSION_GREATER "3.2")
-      # BUILD_BYPRODUCTS is a 3.2+ feature
-      ExternalProject_Add(jemalloc_ep
-        URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
-        CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
-        BUILD_IN_SOURCE 1
-        BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
-        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
-        INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j1 install)
-    else()
-      ExternalProject_Add(jemalloc_ep
-        URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
-        CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
-        BUILD_IN_SOURCE 1
-        BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
-        INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j1 install)
-    endif()
-  else()
-    set(JEMALLOC_VENDORED 0)
-  endif()
-
-  include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(jemalloc
-    STATIC_LIB ${JEMALLOC_STATIC_LIB}
-    SHARED_LIB ${JEMALLOC_SHARED_LIB}
-    DEPS ${PTHREAD_LIBRARY})
-endif()
-
-## Google PerfTools
-##
-## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment
-## near definition of ARROW_USING_GOLD).
-# find_package(GPerf REQUIRED)
-# if (NOT "${ARROW_USE_ASAN}" AND
-#     NOT "${ARROW_USE_TSAN}" AND
-#     NOT ("${ARROW_USING_GOLD}" AND "${ARROW_LINK}" STREQUAL "d"))
-#   ADD_THIRDPARTY_LIB(tcmalloc
-#     STATIC_LIB "${TCMALLOC_STATIC_LIB}"
-#     SHARED_LIB "${TCMALLOC_SHARED_LIB}")
-#   ADD_THIRDPARTY_LIB(profiler
-#     STATIC_LIB "${PROFILER_STATIC_LIB}"
-#     SHARED_LIB "${PROFILER_SHARED_LIB}")
-#   list(APPEND ARROW_BASE_LIBS tcmalloc profiler)
-#   add_definitions("-DTCMALLOC_ENABLED")
-#   set(ARROW_TCMALLOC_AVAILABLE 1)
-# endif()
-
-########################################################################
-# HDFS thirdparty setup
-
-if (DEFINED ENV{HADOOP_HOME})
-  set(HADOOP_HOME $ENV{HADOOP_HOME})
-  if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
-    message(STATUS "Did not find hdfs.h in expected location, using vendored one")
-    set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
-  endif()
-else()
-  set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
-endif()
-
-set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
-if (NOT EXISTS ${HDFS_H_PATH})
-  message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
-endif()
-message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
-
-include_directories(SYSTEM "${HADOOP_HOME}/include")
-
-############################################################
-# Linker setup
-############################################################
-set(ARROW_MIN_TEST_LIBS
-  ${ARROW_STATIC_LINK_LIBS}
-  arrow_static
-  gtest
-  gtest_main
-  ${ARROW_BASE_LIBS})
-
-if(NOT MSVC)
-  set(ARROW_MIN_TEST_LIBS
-    ${ARROW_MIN_TEST_LIBS}
-    ${CMAKE_DL_LIBS})
-endif()
-
-set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
-
-set(ARROW_BENCHMARK_LINK_LIBS
-  arrow_static
-  arrow_benchmark_main
-  ${ARROW_BASE_LIBS})
-
 ############################################################
 # "make ctags" target
 ############################################################
@@ -936,16 +515,41 @@ if (${CLANG_TIDY_FOUND})
 
 endif()
 
-
-
 ############################################################
-# Subdirectories
+# Linker and Dependencies
 ############################################################
 
-set(ARROW_LINK_LIBS
-    )
+set(ARROW_STATIC_LINK_LIBS
+  brotli_dec
+  brotli_enc
+  brotli_common
+  snappy
+  zlib)
+
+set(ARROW_DEPENDENCIES
+  ${ARROW_STATIC_LINK_LIBS})
 
-set(ARROW_STATIC_LINK_LIBS)
+set(ARROW_MIN_TEST_LIBS
+  arrow_static
+  ${ARROW_STATIC_LINK_LIBS}
+  gtest
+  gtest_main)
+
+if(NOT MSVC)
+  set(ARROW_MIN_TEST_LIBS
+    ${ARROW_MIN_TEST_LIBS}
+    ${CMAKE_DL_LIBS})
+endif()
+
+set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
+
+set(ARROW_BENCHMARK_LINK_LIBS
+  arrow_static
+  arrow_benchmark_main
+  ${ARROW_STATIC_LINK_LIBS})
+
+set(ARROW_LINK_LIBS
+  ${ARROW_STATIC_LINK_LIBS})
 
 set(ARROW_SHARED_PRIVATE_LINK_LIBS
   ${BOOST_SYSTEM_LIBRARY}
@@ -1009,13 +613,9 @@ elseif (NOT MSVC)
     ${PTHREAD_LIBRARY})
 endif()
 
-if(RAPIDJSON_VENDORED)
-  set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} rapidjson_ep)
-endif()
-
-if(FLATBUFFERS_VENDORED)
-  set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} flatbuffers_ep)
-endif()
+############################################################
+# Subdirectories
+############################################################
 
 if(NOT WIN32 AND ARROW_PLASMA)
   add_subdirectory(src/plasma)
@@ -1048,6 +648,7 @@ set(ARROW_SRCS
   src/arrow/io/memory.cc
 
   src/arrow/util/bit-util.cc
+  src/arrow/util/compression.cc
   src/arrow/util/decimal.cc
   src/arrow/util/key_value_metadata.cc
 )
diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake
new file mode 100644
index 0000000000000..f2e714c6fc870
--- /dev/null
+++ b/cpp/cmake_modules/FindBrotli.cmake
@@ -0,0 +1,116 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find Brotli headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(Brotli)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  Brotli_HOME - When set, this path is inspected instead of standard library
+#                locations as the root of the Brotli installation.
+#                The environment variable BROTLI_HOME overrides this veriable.
+#
+# This module defines
+#  BROTLI_INCLUDE_DIR, directory containing headers
+#  BROTLI_LIBS, directory containing brotli libraries
+#  BROTLI_STATIC_LIB, path to libbrotli.a
+#  BROTLI_SHARED_LIB, path to libbrotli's shared library
+#  BROTLI_FOUND, whether brotli has been found
+
+if( NOT "${BROTLI_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${BROTLI_HOME}" _native_path )
+    list( APPEND _brotli_roots ${_native_path} )
+elseif ( Brotli_HOME )
+    list( APPEND _brotli_roots ${Brotli_HOME} )
+endif()
+
+find_path( BROTLI_INCLUDE_DIR NAMES brotli/decode.h
+  PATHS ${_brotli_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "include" )
+
+find_library( BROTLI_LIBRARY_ENC NAMES libbrotlienc.a brotlienc
+  PATHS ${_brotli_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
+
+find_library( BROTLI_LIBRARY_DEC NAMES libbrotlidec.a brotlidec
+  PATHS ${_brotli_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
+
+find_library( BROTLI_LIBRARY_COMMON NAMES libbrotlicommon.a brotlicommon
+  PATHS ${_brotli_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
+
+set(BROTLI_LIBRARIES ${BROTLI_LIBRARY_ENC} ${BROTLI_LIBRARY_DEC}
+    ${BROTLI_LIBRARY_COMMON})
+
+if (BROTLI_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR BROTLI_LIBRARIES))
+  set(BROTLI_FOUND TRUE)
+  get_filename_component( BROTLI_LIBS ${BROTLI_LIBRARY_ENC} PATH )
+  set(BROTLI_LIB_NAME brotli)
+  if (MSVC AND NOT BROTLI_MSVC_STATIC_LIB_SUFFIX)
+    set(BROTLI_MSVC_STATIC_LIB_SUFFIX _static)
+  endif()
+  set(BROTLI_STATIC_LIB
+      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
+      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
+      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(BROTLI_STATIC_LIBRARY_ENC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(BROTLI_STATIC_LIBRARY_DEC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(BROTLI_STATIC_LIBRARY_COMMON ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(BROTLI_SHARED_LIB
+      ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${CMAKE_SHARED_LIBRARY_SUFFIX}
+      ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${CMAKE_SHARED_LIBRARY_SUFFIX}
+      ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${CMAKE_SHARED_LIBRARY_SUFFIX})
+else ()
+  set(BROTLI_FOUND FALSE)
+endif ()
+
+if (BROTLI_FOUND)
+  if (NOT Brotli_FIND_QUIETLY)
+    if (PARQUET_MINIMAL_DEPENDENCY)
+      message(STATUS "Found the Brotli headers: ${BROTLI_INCLUDE_DIR}")
+    else ()
+      message(STATUS "Found the Brotli library: ${BROTLI_LIBRARIES}")
+    endif ()
+  endif ()
+else ()
+  if (NOT Brotli_FIND_QUIETLY)
+    set(BROTLI_ERR_MSG "Could not find the Brotli library. Looked in ")
+    if ( _brotli_roots )
+      set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} in ${_brotli_roots}.")
+    else ()
+      set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} system search paths.")
+    endif ()
+    if (Brotli_FIND_REQUIRED)
+      message(FATAL_ERROR "${BROTLI_ERR_MSG}")
+    else (Brotli_FIND_REQUIRED)
+      message(STATUS "${BROTLI_ERR_MSG}")
+    endif (Brotli_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  BROTLI_INCLUDE_DIR
+  BROTLI_LIBS
+  BROTLI_LIBRARIES
+  BROTLI_STATIC_LIB
+  BROTLI_SHARED_LIB
+)
diff --git a/cpp/cmake_modules/FindSnappy.cmake b/cpp/cmake_modules/FindSnappy.cmake
new file mode 100644
index 0000000000000..867963c103c4b
--- /dev/null
+++ b/cpp/cmake_modules/FindSnappy.cmake
@@ -0,0 +1,94 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find Snappy headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(Snappy)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  Snappy_HOME - When set, this path is inspected instead of standard library
+#                locations as the root of the Snappy installation.
+#                The environment variable SNAPPY_HOME overrides this variable.
+#
+# This module defines
+#  SNAPPY_INCLUDE_DIR, directory containing headers
+#  SNAPPY_LIBS, directory containing snappy libraries
+#  SNAPPY_STATIC_LIB, path to libsnappy.a
+#  SNAPPY_SHARED_LIB, path to libsnappy's shared library
+#  SNAPPY_FOUND, whether snappy has been found
+
+if( NOT "${SNAPPY_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${SNAPPY_HOME}" _native_path )
+    list( APPEND _snappy_roots ${_native_path} )
+elseif ( Snappy_HOME )
+    list( APPEND _snappy_roots ${Snappy_HOME} )
+endif()
+
+message(STATUS "SNAPPY_HOME: ${SNAPPY_HOME}")
+find_path(SNAPPY_INCLUDE_DIR snappy.h HINTS
+  ${_snappy_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "include")
+
+find_library( SNAPPY_LIBRARIES NAMES snappy PATHS
+  ${_snappy_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib")
+
+if (SNAPPY_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR SNAPPY_LIBRARIES))
+  set(SNAPPY_FOUND TRUE)
+  get_filename_component( SNAPPY_LIBS ${SNAPPY_LIBRARIES} PATH )
+  set(SNAPPY_HEADER_NAME snappy.h)
+  set(SNAPPY_HEADER ${SNAPPY_INCLUDE_DIR}/${SNAPPY_HEADER_NAME})
+  set(SNAPPY_LIB_NAME snappy)
+  set(SNAPPY_STATIC_LIB ${SNAPPY_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_LIB_NAME}${SNAPPY_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(SNAPPY_SHARED_LIB ${SNAPPY_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${SNAPPY_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+else ()
+  set(SNAPPY_FOUND FALSE)
+endif ()
+
+if (SNAPPY_FOUND)
+  if (NOT Snappy_FIND_QUIETLY)
+    if (PARQUET_MINIMAL_DEPENDENCY)
+      message(STATUS "Found the Snappy header: ${SNAPPY_HEADER}")
+    else ()
+      message(STATUS "Found the Snappy library: ${SNAPPY_LIBRARIES}")
+    endif ()
+  endif ()
+else ()
+  if (NOT Snappy_FIND_QUIETLY)
+    set(SNAPPY_ERR_MSG "Could not find the Snappy library. Looked in ")
+    if ( _snappy_roots )
+      set(SNAPPY_ERR_MSG "${SNAPPY_ERR_MSG} in ${_snappy_roots}.")
+    else ()
+      set(SNAPPY_ERR_MSG "${SNAPPY_ERR_MSG} system search paths.")
+    endif ()
+    if (Snappy_FIND_REQUIRED)
+      message(FATAL_ERROR "${SNAPPY_ERR_MSG}")
+    else (Snappy_FIND_REQUIRED)
+      message(STATUS "${SNAPPY_ERR_MSG}")
+    endif (Snappy_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  SNAPPY_INCLUDE_DIR
+  SNAPPY_LIBS
+  SNAPPY_LIBRARIES
+  SNAPPY_STATIC_LIB
+  SNAPPY_SHARED_LIB
+)
diff --git a/cpp/cmake_modules/FindZLIB.cmake b/cpp/cmake_modules/FindZLIB.cmake
new file mode 100644
index 0000000000000..78b84f2073ff6
--- /dev/null
+++ b/cpp/cmake_modules/FindZLIB.cmake
@@ -0,0 +1,105 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find ZLIB headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(ZLIB)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  ZLIB_HOME - When set, this path is inspected instead of standard library
+#             locations as the root of the ZLIB installation.
+#             The environment variable ZLIB_HOME overrides this variable.
+#
+# - Find ZLIB (zlib.h, libz.a, libz.so, and libz.so.1)
+# This module defines
+#  ZLIB_INCLUDE_DIR, directory containing headers
+#  ZLIB_LIBS, directory containing zlib libraries
+#  ZLIB_STATIC_LIB, path to libz.a
+#  ZLIB_SHARED_LIB, path to libz's shared library
+#  ZLIB_FOUND, whether zlib has been found
+
+if( NOT "${ZLIB_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${ZLIB_HOME}" _native_path )
+    list( APPEND _zlib_roots ${_native_path} )
+elseif ( ZLIB_HOME )
+    list( APPEND _zlib_roots ${ZLIB_HOME} )
+endif()
+
+# Try the parameterized roots, if they exist
+if ( _zlib_roots )
+    find_path( ZLIB_INCLUDE_DIR NAMES zlib.h
+        PATHS ${_zlib_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "include" )
+    find_library( ZLIB_LIBRARIES NAMES libz.a zlib
+        PATHS ${_zlib_roots} NO_DEFAULT_PATH
+        PATH_SUFFIXES "lib" )
+else ()
+    find_path( ZLIB_INCLUDE_DIR NAMES zlib.h )
+    # Only look for the static library
+    find_library( ZLIB_LIBRARIES NAMES libz.a zlib )
+endif ()
+
+
+if (ZLIB_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR ZLIB_LIBRARIES))
+  set(ZLIB_FOUND TRUE)
+  get_filename_component( ZLIB_LIBS ${ZLIB_LIBRARIES} PATH )
+  set(ZLIB_HEADER_NAME zlib.h)
+  set(ZLIB_HEADER ${ZLIB_INCLUDE_DIR}/${ZLIB_HEADER_NAME})
+  set(ZLIB_LIB_NAME z)
+  if (MSVC)
+    if (NOT ZLIB_MSVC_STATIC_LIB_SUFFIX)
+      set(ZLIB_MSVC_STATIC_LIB_SUFFIX libstatic)
+    endif()
+    set(ZLIB_MSVC_SHARED_LIB_SUFFIX lib)
+  endif()
+  set(ZLIB_STATIC_LIB ${ZLIB_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${ZLIB_LIB_NAME}${ZLIB_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(ZLIB_SHARED_LIB ${ZLIB_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${ZLIB_LIB_NAME}${ZLIB_MSVC_SHARED_LIB_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX})
+else ()
+  set(ZLIB_FOUND FALSE)
+endif ()
+
+if (ZLIB_FOUND)
+  if (NOT ZLIB_FIND_QUIETLY)
+    if (PARQUET_MINIMAL_DEPENDENCY)
+      message(STATUS "Found the ZLIB header: ${ZLIB_HEADER}")
+    else()
+      message(STATUS "Found the ZLIB library: ${ZLIB_LIBRARIES}")
+    endif ()
+  endif ()
+else ()
+  if (NOT ZLIB_FIND_QUIETLY)
+    set(ZLIB_ERR_MSG "Could not find the ZLIB library. Looked in ")
+    if ( _zlib_roots )
+      set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} in ${_zlib_roots}.")
+    else ()
+      set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} system search paths.")
+    endif ()
+    if (ZLIB_FIND_REQUIRED)
+      message(FATAL_ERROR "${ZLIB_ERR_MSG}")
+    else (ZLIB_FIND_REQUIRED)
+      message(STATUS "${ZLIB_ERR_MSG}")
+    endif (ZLIB_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+  ZLIB_INCLUDE_DIR
+  ZLIB_LIBS
+  ZLIB_LIBRARIES
+  ZLIB_STATIC_LIB
+  ZLIB_SHARED_LIB
+)
diff --git a/cpp/cmake_modules/SnappyCMakeLists.txt b/cpp/cmake_modules/SnappyCMakeLists.txt
new file mode 100644
index 0000000000000..9d0a166064ea6
--- /dev/null
+++ b/cpp/cmake_modules/SnappyCMakeLists.txt
@@ -0,0 +1,85 @@
+# Copyright 2008 Google Inc. All Rights Reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+CMAKE_MINIMUM_REQUIRED(VERSION 2.6)
+PROJECT(snappy)
+
+INCLUDE(CheckIncludeFiles)
+INCLUDE(CMakePackageConfigHelpers)
+
+CHECK_INCLUDE_FILES("stdint.h" HAVE_STDINT_H)
+CHECK_INCLUDE_FILES("stddef.h" HAVE_STDDEF_H)
+CHECK_INCLUDE_FILES("sys/uio.h" HAVE_SYS_UIO_H)
+
+if (NOT HAVE_SYS_UIO_H)
+  set(HAVE_SYS_UIO_H 0)
+endif()
+
+if (NOT HAVE_STDINT_H)
+  set(HAVE_STDINT_H 0)
+endif()
+
+if (NOT HAVE_STDDEF_H)
+  set(HAVE_STDDEF_H 0)
+endif()
+
+set(ac_cv_have_stdint_h ${HAVE_STDINT_H})
+set(ac_cv_have_stddef_h ${HAVE_STDDEF_H})
+set(ac_cv_have_sys_uio_h ${HAVE_SYS_UIO_H})
+CONFIGURE_FILE(${snappy_SOURCE_DIR}/snappy-stubs-public.h.in
+               snappy-stubs-public.h)
+
+if (WIN32)
+  ADD_DEFINITIONS(-D_CRT_SECURE_NO_WARNINGS)
+endif()
+
+set(SNAPPY_SRCS snappy.cc
+  snappy-c.cc
+  snappy-stubs-internal.cc
+  snappy-sinksource.cc
+  snappy.h
+  snappy-c.h
+  snappy-sinksource.h
+  snappy-stubs-public.h)
+
+add_library(snappy SHARED ${SNAPPY_SRCS})
+add_library(snappystatic STATIC ${SNAPPY_SRCS})
+
+TARGET_COMPILE_DEFINITIONS(snappy PRIVATE -DHAVE_CONFIG_H)
+TARGET_COMPILE_DEFINITIONS(snappystatic PRIVATE -DHAVE_CONFIG_H)
+
+install(FILES snappy.h
+  snappy-c.h
+  snappy-sinksource.h
+  ${snappy_BINARY_DIR}/snappy-stubs-public.h
+  DESTINATION include)
+
+install(TARGETS snappy snappystatic
+  RUNTIME DESTINATION bin
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib)
diff --git a/cpp/cmake_modules/SnappyConfig.h b/cpp/cmake_modules/SnappyConfig.h
new file mode 100644
index 0000000000000..74eb77621626b
--- /dev/null
+++ b/cpp/cmake_modules/SnappyConfig.h
@@ -0,0 +1,36 @@
+// Copyright 2008 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef SNAPPY_CONFIG_H
+#define SNAPPY_CONFIG_H 1
+
+#if defined(_MSC_VER) && (_MSC_VER <= 1900)
+typedef __int64 ssize_t;
+#endif
+
+#endif
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
new file mode 100644
index 0000000000000..f6a9bb4f80e80
--- /dev/null
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -0,0 +1,641 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# ----------------------------------------------------------------------
+# Thirdparty toolchain
+
+set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
+set(GFLAGS_VERSION "2.1.2")
+set(GTEST_VERSION "1.8.0")
+set(GBENCHMARK_VERSION "1.1.0")
+set(FLATBUFFERS_VERSION "1.6.0")
+set(JEMALLOC_VERSION "4.4.0")
+set(SNAPPY_VERSION "1.1.3")
+set(BROTLI_VERSION "v0.6.0")
+
+string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
+
+set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}")
+set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}")
+
+if (NOT MSVC)
+  # Set -fPIC on all external projects
+  set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC")
+  set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC")
+endif()
+
+if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
+  set(FLATBUFFERS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(RAPIDJSON_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(GFLAGS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(SNAPPY_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(ZLIB_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(BROTLI_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+
+  if (NOT DEFINED ENV{BOOST_ROOT})
+    # Since we have to set this in the environment, we check whether
+    # $BOOST_ROOT is defined inside here
+    set(ENV{BOOST_ROOT} "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  endif()
+endif()
+
+if (DEFINED ENV{FLATBUFFERS_HOME})
+  set(FLATBUFFERS_HOME "$ENV{FLATBUFFERS_HOME}")
+endif()
+
+if (DEFINED ENV{RAPIDJSON_HOME})
+  set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}")
+endif()
+
+if (DEFINED ENV{JEMALLOC_HOME})
+  set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}")
+endif()
+
+if (DEFINED ENV{GFLAGS_HOME})
+  set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
+endif()
+
+if (DEFINED ENV{SNAPPY_HOME})
+  set(SNAPPY_HOME "$ENV{SNAPPY_HOME}")
+endif()
+
+if (DEFINED ENV{ZLIB_HOME})
+  set(ZLIB_HOME "$ENV{ZLIB_HOME}")
+endif()
+
+if (DEFINED ENV{BROTLI_HOME})
+  set(BROTLI_HOME "$ENV{BROTLI_HOME}")
+endif()
+
+# ----------------------------------------------------------------------
+# Find pthreads
+
+if (NOT MSVC)
+  find_library(PTHREAD_LIBRARY pthread)
+  message(STATUS "Found pthread: ${PTHREAD_LIBRARY}")
+endif()
+
+# ----------------------------------------------------------------------
+# Add Boost dependencies (code adapted from Apache Kudu (incubating))
+
+set(Boost_DEBUG TRUE)
+set(Boost_USE_MULTITHREADED ON)
+set(Boost_ADDITIONAL_VERSIONS
+  "1.64.0" "1.64"
+  "1.63.0" "1.63"
+  "1.62.0" "1.61"
+  "1.61.0" "1.62"
+  "1.60.0" "1.60")
+
+if (ARROW_BOOST_USE_SHARED)
+  # Find shared Boost libraries.
+  set(Boost_USE_STATIC_LIBS OFF)
+
+  if(MSVC)
+    # disable autolinking in boost
+    add_definitions(-DBOOST_ALL_NO_LIB)
+
+    # force all boost libraries to dynamic link
+    add_definitions(-DBOOST_ALL_DYN_LINK)
+  endif()
+
+  if (ARROW_BOOST_HEADER_ONLY)
+    find_package(Boost)
+  else()
+    find_package(Boost COMPONENTS system filesystem REQUIRED)
+    if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+      set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+      set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+    else()
+      set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+      set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+    endif()
+    set(BOOST_SYSTEM_LIBRARY boost_system_shared)
+    set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
+  endif()
+else()
+  # Find static boost headers and libs
+  # TODO Differentiate here between release and debug builds
+  set(Boost_USE_STATIC_LIBS ON)
+  if (ARROW_BOOST_HEADER_ONLY)
+    find_package(Boost)
+  else()
+    find_package(Boost COMPONENTS system filesystem REQUIRED)
+    if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+      set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+      set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+    else()
+      set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+      set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+    endif()
+    set(BOOST_SYSTEM_LIBRARY boost_system_static)
+    set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
+  endif()
+endif()
+
+message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
+message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
+
+if (NOT ARROW_BOOST_HEADER_ONLY)
+  ADD_THIRDPARTY_LIB(boost_system
+      STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}"
+      SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
+
+  ADD_THIRDPARTY_LIB(boost_filesystem
+      STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
+      SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
+
+  SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
+endif()
+
+include_directories(SYSTEM ${Boost_INCLUDE_DIR})
+
+if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
+  add_custom_target(unittest ctest -L unittest)
+
+  if("$ENV{GTEST_HOME}" STREQUAL "")
+    if(APPLE)
+      set(GTEST_CMAKE_CXX_FLAGS "-fPIC -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
+    elseif(NOT MSVC)
+      set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
+    endif()
+    string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
+    set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}")
+
+    set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
+    set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
+    set(GTEST_STATIC_LIB
+      "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GTEST_MAIN_STATIC_LIB
+      "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GTEST_VENDORED 1)
+    set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                         -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}
+                         -Dgtest_force_shared_crt=ON
+                         -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
+
+    if (CMAKE_VERSION VERSION_GREATER "3.2")
+      # BUILD_BYPRODUCTS is a 3.2+ feature
+      ExternalProject_Add(googletest_ep
+        URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+        BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB}
+        CMAKE_ARGS ${GTEST_CMAKE_ARGS})
+    else()
+      ExternalProject_Add(googletest_ep
+        URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+        CMAKE_ARGS ${GTEST_CMAKE_ARGS})
+    endif()
+  else()
+    find_package(GTest REQUIRED)
+    set(GTEST_VENDORED 0)
+  endif()
+
+  message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}")
+  message(STATUS "GTest static library: ${GTEST_STATIC_LIB}")
+  include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(gtest
+    STATIC_LIB ${GTEST_STATIC_LIB})
+  ADD_THIRDPARTY_LIB(gtest_main
+    STATIC_LIB ${GTEST_MAIN_STATIC_LIB})
+
+  if(GTEST_VENDORED)
+    add_dependencies(gtest googletest_ep)
+    add_dependencies(gtest_main googletest_ep)
+  endif()
+
+  # gflags (formerly Googleflags) command line parsing
+  if("${GFLAGS_HOME}" STREQUAL "")
+    set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS})
+
+    set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
+    set(GFLAGS_HOME "${GFLAGS_PREFIX}")
+    set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
+    if(MSVC)
+      set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/gflags_static.lib")
+    else()
+      set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
+    endif()
+    set(GFLAGS_VENDORED 1)
+    set(GFLAGS_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                          -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
+                          -DBUILD_SHARED_LIBS=OFF
+                          -DBUILD_STATIC_LIBS=ON
+                          -DBUILD_PACKAGING=OFF
+                          -DBUILD_TESTING=OFF
+                          -BUILD_CONFIG_TESTS=OFF
+                          -DINSTALL_HEADERS=ON
+                          -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
+    if (CMAKE_VERSION VERSION_GREATER "3.2")
+      # BUILD_BYPRODUCTS is a 3.2+ feature
+      ExternalProject_Add(gflags_ep
+        GIT_REPOSITORY https://github.com/gflags/gflags.git
+        GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+        BUILD_IN_SOURCE 1
+        BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}"
+        CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
+    else()
+      ExternalProject_Add(gflags_ep
+        GIT_REPOSITORY https://github.com/gflags/gflags.git
+        GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+        BUILD_IN_SOURCE 1
+        CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
+    endif()
+  else()
+    set(GFLAGS_VENDORED 0)
+    find_package(GFlags REQUIRED)
+  endif()
+
+  message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}")
+  message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}")
+  include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(gflags
+    STATIC_LIB ${GFLAGS_STATIC_LIB})
+  if(MSVC)
+    set_target_properties(gflags
+      PROPERTIES
+      IMPORTED_LINK_INTERFACE_LIBRARIES "shlwapi.lib")
+  endif()
+
+  if(GFLAGS_VENDORED)
+    add_dependencies(gflags gflags_ep)
+  endif()
+endif()
+
+if(ARROW_BUILD_BENCHMARKS)
+  add_custom_target(runbenchmark ctest -L benchmark)
+
+  if("$ENV{GBENCHMARK_HOME}" STREQUAL "")
+    if(APPLE)
+      set(GBENCHMARK_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++")
+    elseif(NOT MSVC)
+      set(GBENCHMARK_CMAKE_CXX_FLAGS "-fPIC --std=c++11")
+    endif()
+
+    set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install")
+    set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include")
+    set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GBENCHMARK_VENDORED 1)
+    set(GBENCHMARK_CMAKE_ARGS
+          "-DCMAKE_BUILD_TYPE=Release"
+          "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}"
+          "-DBENCHMARK_ENABLE_TESTING=OFF"
+          "-DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}")
+    if (APPLE)
+      set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON")
+    endif()
+    if (CMAKE_VERSION VERSION_GREATER "3.2")
+      # BUILD_BYPRODUCTS is a 3.2+ feature
+      ExternalProject_Add(gbenchmark_ep
+        URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
+        BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}"
+        CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
+    else()
+      ExternalProject_Add(gbenchmark_ep
+        URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
+        CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
+    endif()
+  else()
+    find_package(GBenchmark REQUIRED)
+    set(GBENCHMARK_VENDORED 0)
+  endif()
+
+  message(STATUS "GBenchmark include dir: ${GBENCHMARK_INCLUDE_DIR}")
+  message(STATUS "GBenchmark static library: ${GBENCHMARK_STATIC_LIB}")
+  include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(benchmark
+    STATIC_LIB ${GBENCHMARK_STATIC_LIB})
+
+  if(GBENCHMARK_VENDORED)
+    add_dependencies(benchmark gbenchmark_ep)
+  endif()
+endif()
+
+
+if (ARROW_IPC)
+  # RapidJSON, header only dependency
+  if("${RAPIDJSON_HOME}" STREQUAL "")
+    ExternalProject_Add(rapidjson_ep
+      PREFIX "${CMAKE_BINARY_DIR}"
+      URL "https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz"
+      URL_MD5 "badd12c511e081fec6c89c43a7027bce"
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND ""
+      BUILD_IN_SOURCE 1
+      INSTALL_COMMAND "")
+
+    ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR)
+    set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include")
+    set(RAPIDJSON_VENDORED 1)
+  else()
+    set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_HOME}/include")
+    set(RAPIDJSON_VENDORED 0)
+  endif()
+  message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
+  include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
+
+  ## Flatbuffers
+  if("${FLATBUFFERS_HOME}" STREQUAL "")
+    set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
+    ExternalProject_Add(flatbuffers_ep
+      URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
+      CMAKE_ARGS
+      "-DCMAKE_CXX_FLAGS=-fPIC"
+      "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
+      "-DFLATBUFFERS_BUILD_TESTS=OFF")
+
+    set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
+    set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
+    set(FLATBUFFERS_VENDORED 1)
+  else()
+    find_package(Flatbuffers REQUIRED)
+    set(FLATBUFFERS_VENDORED 0)
+  endif()
+
+  if(RAPIDJSON_VENDORED)
+    set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} rapidjson_ep)
+  endif()
+
+  if(FLATBUFFERS_VENDORED)
+    set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} flatbuffers_ep)
+  endif()
+
+  message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
+  message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
+  include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
+endif()
+#----------------------------------------------------------------------
+
+if (MSVC)
+  # jemalloc is not supported on Windows
+  set(ARROW_JEMALLOC off)
+endif()
+
+if (ARROW_JEMALLOC)
+  find_package(jemalloc)
+
+  if(NOT JEMALLOC_FOUND)
+    set(ARROW_JEMALLOC_USE_SHARED OFF)
+    set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
+    set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
+    set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
+    set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(JEMALLOC_VENDORED 1)
+    if (CMAKE_VERSION VERSION_GREATER "3.2")
+      # BUILD_BYPRODUCTS is a 3.2+ feature
+      ExternalProject_Add(jemalloc_ep
+        URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+        CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
+        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
+        INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j1 install)
+    else()
+      ExternalProject_Add(jemalloc_ep
+        URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+        CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
+        INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j1 install)
+    endif()
+  else()
+    set(JEMALLOC_VENDORED 0)
+  endif()
+
+  include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(jemalloc
+    STATIC_LIB ${JEMALLOC_STATIC_LIB}
+    SHARED_LIB ${JEMALLOC_SHARED_LIB}
+    DEPS ${PTHREAD_LIBRARY})
+endif()
+
+## Google PerfTools
+##
+## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment
+## near definition of ARROW_USING_GOLD).
+# find_package(GPerf REQUIRED)
+# if (NOT "${ARROW_USE_ASAN}" AND
+#     NOT "${ARROW_USE_TSAN}" AND
+#     NOT ("${ARROW_USING_GOLD}" AND "${ARROW_LINK}" STREQUAL "d"))
+#   ADD_THIRDPARTY_LIB(tcmalloc
+#     STATIC_LIB "${TCMALLOC_STATIC_LIB}"
+#     SHARED_LIB "${TCMALLOC_SHARED_LIB}")
+#   ADD_THIRDPARTY_LIB(profiler
+#     STATIC_LIB "${PROFILER_STATIC_LIB}"
+#     SHARED_LIB "${PROFILER_SHARED_LIB}")
+#   list(APPEND ARROW_BASE_LIBS tcmalloc profiler)
+#   add_definitions("-DTCMALLOC_ENABLED")
+#   set(ARROW_TCMALLOC_AVAILABLE 1)
+# endif()
+
+########################################################################
+# HDFS thirdparty setup
+
+if (DEFINED ENV{HADOOP_HOME})
+  set(HADOOP_HOME $ENV{HADOOP_HOME})
+  if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h")
+    message(STATUS "Did not find hdfs.h in expected location, using vendored one")
+    set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+  endif()
+else()
+  set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
+endif()
+
+set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
+if (NOT EXISTS ${HDFS_H_PATH})
+  message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
+endif()
+message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
+
+include_directories(SYSTEM "${HADOOP_HOME}/include")
+
+# ----------------------------------------------------------------------
+# ZLIB
+
+if (NOT ARROW_ZLIB_VENDORED)
+  find_package(ZLIB)
+endif()
+
+if (NOT ZLIB_FOUND)
+  set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install")
+  set(ZLIB_HOME "${ZLIB_PREFIX}")
+  set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include")
+  if (MSVC)
+    if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+      set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib)
+    else()
+      set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
+    endif()
+  else()
+    set(ZLIB_STATIC_LIB_NAME libz.a)
+  endif()
+  set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}")
+  set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                      -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}
+                      -DCMAKE_C_FLAGS=${EP_C_FLAGS}
+                      -DBUILD_SHARED_LIBS=OFF)
+
+  if (CMAKE_VERSION VERSION_GREATER "3.2")
+    set(ZLIB_BUILD_BYPRODUCTS BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}")
+  endif()
+  ExternalProject_Add(zlib_ep
+    URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz"
+    ${ZLIB_BUILD_BYPRODUCTS}
+    CMAKE_ARGS ${ZLIB_CMAKE_ARGS})
+  set(ZLIB_VENDORED 1)
+else()
+  set(ZLIB_VENDORED 0)
+endif()
+
+include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(zlib
+  STATIC_LIB ${ZLIB_STATIC_LIB})
+
+if (ZLIB_VENDORED)
+  add_dependencies(zlib zlib_ep)
+endif()
+
+# ----------------------------------------------------------------------
+# Snappy
+
+## Snappy
+find_package(Snappy)
+if (NOT SNAPPY_FOUND)
+  set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep/src/snappy_ep-install")
+  set(SNAPPY_HOME "${SNAPPY_PREFIX}")
+  set(SNAPPY_INCLUDE_DIR "${SNAPPY_PREFIX}/include")
+  if (MSVC)
+    set(SNAPPY_STATIC_LIB_NAME snappystatic)
+  else()
+    set(SNAPPY_STATIC_LIB_NAME snappy)
+  endif()
+  set(SNAPPY_STATIC_LIB "${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(SNAPPY_SRC_URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz")
+
+  if (${UPPERCASE_BUILD_TYPE} EQUAL "RELEASE")
+    if (APPLE)
+      set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O1'")
+    else()
+      set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O2'")
+    endif()
+  endif()
+
+  if (CMAKE_VERSION VERSION_GREATER "3.2")
+    set(SNAPPY_BUILD_BYPRODUCTS BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}")
+  endif()
+
+  if (MSVC)
+    set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                          "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
+                          "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                          "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}")
+    set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy
+                      ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt
+                      ./CMakeLists.txt &&
+                      ${CMAKE_COMMAND} -E copy
+                      ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyConfig.h
+                      ./config.h)
+    ExternalProject_Add(snappy_ep
+      UPDATE_COMMAND ${SNAPPY_UPDATE_COMMAND}
+      BUILD_IN_SOURCE 1
+      BUILD_COMMAND ${MAKE}
+      INSTALL_DIR ${SNAPPY_PREFIX}
+      URL ${SNAPPY_SRC_URL}
+      CMAKE_ARGS ${SNAPPY_CMAKE_ARGS}
+      ${SNAPPY_BUILD_BYPRODUCTS})
+  else()
+    ExternalProject_Add(snappy_ep
+      CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS}
+      BUILD_IN_SOURCE 1
+      BUILD_COMMAND ${MAKE}
+      INSTALL_DIR ${SNAPPY_PREFIX}
+      URL ${SNAPPY_SRC_URL}
+      ${SNAPPY_BUILD_BYPRODUCTS})
+  endif()
+  set(SNAPPY_VENDORED 1)
+else()
+  set(SNAPPY_VENDORED 0)
+endif()
+
+include_directories(SYSTEM ${SNAPPY_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(snappy
+  STATIC_LIB ${SNAPPY_STATIC_LIB})
+
+if (SNAPPY_VENDORED)
+  add_dependencies(snappy snappy_ep)
+endif()
+
+# ----------------------------------------------------------------------
+# Brotli
+
+find_package(Brotli)
+if (NOT BROTLI_FOUND)
+  set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install")
+  set(BROTLI_HOME "${BROTLI_PREFIX}")
+  set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include")
+  if (MSVC)
+    set(BROTLI_LIB_DIR bin)
+  else()
+    set(BROTLI_LIB_DIR lib)
+  endif()
+  set(BROTLI_STATIC_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(BROTLI_STATIC_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                        "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
+                        "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                        -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}
+                        -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE}
+                        -DBUILD_SHARED_LIBS=OFF)
+
+  if (CMAKE_VERSION VERSION_GREATER "3.2")
+    set(BROTLI_BUILD_BYPRODUCTS BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}")
+  endif()
+
+  ExternalProject_Add(brotli_ep
+    URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz"
+    ${BROTLI_BUILD_BYPRODUCTS}
+    CMAKE_ARGS ${BROTLI_CMAKE_ARGS}
+    STEP_TARGETS headers_copy)
+  if (MSVC)
+    ExternalProject_Get_Property(brotli_ep SOURCE_DIR)
+
+    ExternalProject_Add_Step(brotli_ep headers_copy
+      COMMAND xcopy /E /I include ..\\..\\..\\brotli_ep\\src\\brotli_ep-install\\include /Y
+      DEPENDEES build
+      WORKING_DIRECTORY ${SOURCE_DIR})
+  endif()
+  set(BROTLI_VENDORED 1)
+else()
+  set(BROTLI_VENDORED 0)
+endif()
+
+include_directories(SYSTEM ${BROTLI_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(brotli_enc
+  STATIC_LIB ${BROTLI_STATIC_LIBRARY_ENC})
+ADD_THIRDPARTY_LIB(brotli_dec
+  STATIC_LIB ${BROTLI_STATIC_LIBRARY_DEC})
+ADD_THIRDPARTY_LIB(brotli_common
+  STATIC_LIB ${BROTLI_STATIC_LIBRARY_COMMON})
+
+if (BROTLI_VENDORED)
+  add_dependencies(brotli_enc brotli_ep)
+  add_dependencies(brotli_dec brotli_ep)
+  add_dependencies(brotli_common brotli_ep)
+endif()
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 30852291d1a8d..bc2a815097d05 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -34,8 +34,8 @@ endif()
 
 set(ARROW_PYTHON_MIN_TEST_LIBS
   arrow_python_test_main
-  arrow_python_static
-  arrow_static)
+  arrow_python_shared
+  arrow_shared)
 
 set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
 
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index ac7e86615eb40..1abcce49db6bf 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -22,11 +22,13 @@
 # Headers: top level
 install(FILES
   bit-util.h
+  compression.h
+  key_value_metadata.h
   logging.h
   macros.h
   random.h
+  stl.h
   visibility.h
-  key_value_metadata.h
   DESTINATION include/arrow/util)
 
 #######################################
@@ -51,6 +53,7 @@ if (ARROW_BUILD_BENCHMARKS)
 endif()
 
 ADD_ARROW_TEST(bit-util-test)
-ADD_ARROW_TEST(stl-util-test)
+ADD_ARROW_TEST(compression-test)
 ADD_ARROW_TEST(decimal-test)
 ADD_ARROW_TEST(key-value-metadata-test)
+ADD_ARROW_TEST(stl-util-test)
diff --git a/cpp/src/arrow/util/compression-test.cc b/cpp/src/arrow/util/compression-test.cc
new file mode 100644
index 0000000000000..1a0e5d7023cf1
--- /dev/null
+++ b/cpp/src/arrow/util/compression-test.cc
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <gtest/gtest.h>
+#include <string>
+#include <vector>
+
+#include "arrow/status.h"
+#include "arrow/test-common.h"
+#include "arrow/util/compression.h"
+
+using std::string;
+using std::vector;
+
+namespace arrow {
+
+template <typename T>
+void CheckCodecRoundtrip(const vector<uint8_t>& data) {
+  // create multiple compressors to try to break them
+  T c1;
+  T c2;
+
+  int max_compressed_len = static_cast<int>(c1.MaxCompressedLen(data.size(), &data[0]));
+  std::vector<uint8_t> compressed(max_compressed_len);
+  std::vector<uint8_t> decompressed(data.size());
+
+  // compress with c1
+  int64_t actual_size;
+  ASSERT_OK(c1.Compress(
+      data.size(), &data[0], max_compressed_len, &compressed[0], &actual_size));
+  compressed.resize(actual_size);
+
+  // decompress with c2
+  ASSERT_OK(c2.Decompress(
+      compressed.size(), &compressed[0], decompressed.size(), &decompressed[0]));
+
+  ASSERT_EQ(data, decompressed);
+
+  // compress with c2
+  int64_t actual_size2;
+  ASSERT_OK(c2.Compress(
+      data.size(), &data[0], max_compressed_len, &compressed[0], &actual_size2));
+  ASSERT_EQ(actual_size2, actual_size);
+
+  // decompress with c1
+  ASSERT_OK(c1.Decompress(
+      compressed.size(), &compressed[0], decompressed.size(), &decompressed[0]));
+
+  ASSERT_EQ(data, decompressed);
+}
+
+template <typename T>
+void CheckCodec() {
+  int sizes[] = {10000, 100000};
+  for (int data_size : sizes) {
+    vector<uint8_t> data(data_size);
+    test::random_bytes(data_size, 1234, data.data());
+    CheckCodecRoundtrip<T>(data);
+  }
+}
+
+TEST(TestCompressors, Snappy) {
+  CheckCodec<SnappyCodec>();
+}
+
+TEST(TestCompressors, Brotli) {
+  CheckCodec<BrotliCodec>();
+}
+
+TEST(TestCompressors, GZip) {
+  CheckCodec<GZipCodec>();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/compression.cc b/cpp/src/arrow/util/compression.cc
new file mode 100644
index 0000000000000..f82ae5cac1150
--- /dev/null
+++ b/cpp/src/arrow/util/compression.cc
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/compression.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include <brotli/decode.h>
+#include <brotli/encode.h>
+#include <snappy.h>
+#include <zlib.h>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+Codec::~Codec() {}
+
+Status Codec::Create(Compression::type codec_type, std::unique_ptr<Codec>* result) {
+  switch (codec_type) {
+    case Compression::UNCOMPRESSED:
+      break;
+    case Compression::SNAPPY:
+      result->reset(new SnappyCodec());
+      break;
+    case Compression::GZIP:
+      result->reset(new GZipCodec());
+      break;
+    case Compression::LZO:
+      return Status::NotImplemented("LZO codec not implemented");
+    case Compression::BROTLI:
+      result->reset(new BrotliCodec());
+      break;
+    default:
+      return Status::Invalid("Unrecognized codec");
+  }
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// gzip implementation
+
+// These are magic numbers from zlib.h.  Not clear why they are not defined
+// there.
+
+// Maximum window size
+static constexpr int WINDOW_BITS = 15;
+
+// Output Gzip.
+static constexpr int GZIP_CODEC = 16;
+
+// Determine if this is libz or gzip from header.
+static constexpr int DETECT_CODEC = 32;
+
+class GZipCodec::GZipCodecImpl {
+ public:
+  explicit GZipCodecImpl(GZipCodec::Format format)
+      : format_(format),
+        compressor_initialized_(false),
+        decompressor_initialized_(false) {}
+
+  ~GZipCodecImpl() {
+    EndCompressor();
+    EndDecompressor();
+  }
+
+  Status InitCompressor() {
+    EndDecompressor();
+    memset(&stream_, 0, sizeof(stream_));
+
+    int ret;
+    // Initialize to run specified format
+    int window_bits = WINDOW_BITS;
+    if (format_ == DEFLATE) {
+      window_bits = -window_bits;
+    } else if (format_ == GZIP) {
+      window_bits += GZIP_CODEC;
+    }
+    if ((ret = deflateInit2(&stream_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits, 9,
+             Z_DEFAULT_STRATEGY)) != Z_OK) {
+      std::stringstream ss;
+      ss << "zlib deflateInit failed: " << std::string(stream_.msg);
+      return Status::IOError(ss.str());
+    }
+    compressor_initialized_ = true;
+    return Status::OK();
+  }
+
+  void EndCompressor() {
+    if (compressor_initialized_) { (void)deflateEnd(&stream_); }
+    compressor_initialized_ = false;
+  }
+
+  Status InitDecompressor() {
+    EndCompressor();
+    memset(&stream_, 0, sizeof(stream_));
+    int ret;
+
+    // Initialize to run either deflate or zlib/gzip format
+    int window_bits = format_ == DEFLATE ? -WINDOW_BITS : WINDOW_BITS | DETECT_CODEC;
+    if ((ret = inflateInit2(&stream_, window_bits)) != Z_OK) {
+      std::stringstream ss;
+      ss << "zlib inflateInit failed: " << std::string(stream_.msg);
+      return Status::IOError(ss.str());
+    }
+    decompressor_initialized_ = true;
+    return Status::OK();
+  }
+
+  void EndDecompressor() {
+    if (decompressor_initialized_) { (void)inflateEnd(&stream_); }
+    decompressor_initialized_ = false;
+  }
+
+  Status Decompress(int64_t input_length, const uint8_t* input, int64_t output_length,
+      uint8_t* output) {
+    if (!decompressor_initialized_) { RETURN_NOT_OK(InitDecompressor()); }
+    if (output_length == 0) {
+      // The zlib library does not allow *output to be NULL, even when output_length
+      // is 0 (inflate() will return Z_STREAM_ERROR). We don't consider this an
+      // error, so bail early if no output is expected. Note that we don't signal
+      // an error if the input actually contains compressed data.
+      return Status::OK();
+    }
+
+    // Reset the stream for this block
+    if (inflateReset(&stream_) != Z_OK) {
+      std::stringstream ss;
+      ss << "zlib inflateReset failed: " << std::string(stream_.msg);
+      return Status::IOError(ss.str());
+    }
+
+    int ret = 0;
+    // gzip can run in streaming mode or non-streaming mode.  We only
+    // support the non-streaming use case where we present it the entire
+    // compressed input and a buffer big enough to contain the entire
+    // compressed output.  In the case where we don't know the output,
+    // we just make a bigger buffer and try the non-streaming mode
+    // from the beginning again.
+    while (ret != Z_STREAM_END) {
+      stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input));
+      stream_.avail_in = static_cast<uInt>(input_length);
+      stream_.next_out = reinterpret_cast<Bytef*>(output);
+      stream_.avail_out = static_cast<uInt>(output_length);
+
+      // We know the output size.  In this case, we can use Z_FINISH
+      // which is more efficient.
+      ret = inflate(&stream_, Z_FINISH);
+      if (ret == Z_STREAM_END || ret != Z_OK) break;
+
+      // Failure, buffer was too small
+      std::stringstream ss;
+      ss << "Too small a buffer passed to GZipCodec. InputLength=" << input_length
+         << " OutputLength=" << output_length;
+      return Status::IOError(ss.str());
+    }
+
+    // Failure for some other reason
+    if (ret != Z_STREAM_END) {
+      std::stringstream ss;
+      ss << "GZipCodec failed: ";
+      if (stream_.msg != NULL) ss << stream_.msg;
+      return Status::IOError(ss.str());
+    }
+    return Status::OK();
+  }
+
+  int64_t MaxCompressedLen(int64_t input_length, const uint8_t* input) {
+    // Most be in compression mode
+    if (!compressor_initialized_) {
+      Status s = InitCompressor();
+      DCHECK(s.ok());
+    }
+    // TODO(wesm): deal with zlib < 1.2.3 (see Impala codebase)
+    return deflateBound(&stream_, static_cast<uLong>(input_length));
+  }
+
+  Status Compress(int64_t input_length, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output, int64_t* output_length) {
+    if (!compressor_initialized_) { RETURN_NOT_OK(InitCompressor()); }
+    stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input));
+    stream_.avail_in = static_cast<uInt>(input_length);
+    stream_.next_out = reinterpret_cast<Bytef*>(output);
+    stream_.avail_out = static_cast<uInt>(output_buffer_len);
+
+    int64_t ret = 0;
+    if ((ret = deflate(&stream_, Z_FINISH)) != Z_STREAM_END) {
+      if (ret == Z_OK) {
+        // will return Z_OK (and stream.msg NOT set) if stream.avail_out is too
+        // small
+        return Status::IOError("zlib deflate failed, output buffer too small");
+      }
+      std::stringstream ss;
+      ss << "zlib deflate failed: " << stream_.msg;
+      return Status::IOError(ss.str());
+    }
+
+    if (deflateReset(&stream_) != Z_OK) {
+      std::stringstream ss;
+      ss << "zlib deflateReset failed: " << std::string(stream_.msg);
+      return Status::IOError(ss.str());
+    }
+
+    // Actual output length
+    *output_length = output_buffer_len - stream_.avail_out;
+    return Status::OK();
+  }
+
+ private:
+  // zlib is stateful and the z_stream state variable must be initialized
+  // before
+  z_stream stream_;
+
+  // Realistically, this will always be GZIP, but we leave the option open to
+  // configure
+  GZipCodec::Format format_;
+
+  // These variables are mutually exclusive. When the codec is in "compressor"
+  // state, compressor_initialized_ is true while decompressor_initialized_ is
+  // false. When it's decompressing, the opposite is true.
+  //
+  // Indeed, this is slightly hacky, but the alternative is having separate
+  // Compressor and Decompressor classes. If this ever becomes an issue, we can
+  // perform the refactoring then
+  bool compressor_initialized_;
+  bool decompressor_initialized_;
+};
+
+GZipCodec::GZipCodec(Format format) {
+  impl_.reset(new GZipCodecImpl(format));
+}
+
+GZipCodec::~GZipCodec() {}
+
+Status GZipCodec::Decompress(int64_t input_length, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output) {
+  return impl_->Decompress(input_length, input, output_buffer_len, output);
+}
+
+int64_t GZipCodec::MaxCompressedLen(int64_t input_length, const uint8_t* input) {
+  return impl_->MaxCompressedLen(input_length, input);
+}
+
+Status GZipCodec::Compress(int64_t input_length, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output, int64_t* output_length) {
+  return impl_->Compress(input_length, input, output_buffer_len, output, output_length);
+}
+
+const char* GZipCodec::name() const {
+  return "gzip";
+}
+
+// ----------------------------------------------------------------------
+// Snappy implementation
+
+Status SnappyCodec::Decompress(
+    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+  if (!snappy::RawUncompress(reinterpret_cast<const char*>(input),
+          static_cast<size_t>(input_len), reinterpret_cast<char*>(output_buffer))) {
+    return Status::IOError("Corrupt snappy compressed data.");
+  }
+  return Status::OK();
+}
+
+int64_t SnappyCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+  return snappy::MaxCompressedLength(input_len);
+}
+
+Status SnappyCodec::Compress(int64_t input_len, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+  size_t output_len;
+  snappy::RawCompress(reinterpret_cast<const char*>(input),
+      static_cast<size_t>(input_len), reinterpret_cast<char*>(output_buffer),
+      &output_len);
+  *output_length = static_cast<int64_t>(output_len);
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Brotli implementation
+
+Status BrotliCodec::Decompress(
+    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+  size_t output_size = output_len;
+  if (BrotliDecoderDecompress(input_len, input, &output_size, output_buffer) !=
+      BROTLI_DECODER_RESULT_SUCCESS) {
+    return Status::IOError("Corrupt brotli compressed data.");
+  }
+  return Status::OK();
+}
+
+int64_t BrotliCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+  return BrotliEncoderMaxCompressedSize(input_len);
+}
+
+Status BrotliCodec::Compress(int64_t input_len, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+  size_t output_len = output_buffer_len;
+  // TODO: Make quality configurable. We use 8 as a default as it is the best
+  //       trade-off for Parquet workload
+  if (BrotliEncoderCompress(8, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, input_len,
+          input, &output_len, output_buffer) == BROTLI_FALSE) {
+    return Status::IOError("Brotli compression failure.");
+  }
+  *output_length = output_len;
+  return Status::OK();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h
new file mode 100644
index 0000000000000..6886d04c7620d
--- /dev/null
+++ b/cpp/src/arrow/util/compression.h
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_COMPRESSION_H
+#define ARROW_UTIL_COMPRESSION_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/status.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+struct Compression {
+  enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO, BROTLI };
+};
+
+class ARROW_EXPORT Codec {
+ public:
+  virtual ~Codec();
+
+  static Status Create(Compression::type codec, std::unique_ptr<Codec>* out);
+
+  virtual Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) = 0;
+
+  virtual Status Compress(int64_t input_len, const uint8_t* input,
+      int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) = 0;
+
+  virtual int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) = 0;
+
+  virtual const char* name() const = 0;
+};
+
+// Snappy codec.
+class ARROW_EXPORT SnappyCodec : public Codec {
+ public:
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override { return "snappy"; }
+};
+
+// Brotli codec.
+class ARROW_EXPORT BrotliCodec : public Codec {
+ public:
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override { return "brotli"; }
+};
+
+// GZip codec.
+class ARROW_EXPORT GZipCodec : public Codec {
+ public:
+  /// Compression formats supported by the zlib library
+  enum Format {
+    ZLIB,
+    DEFLATE,
+    GZIP,
+  };
+
+  explicit GZipCodec(Format format = GZIP);
+  virtual ~GZipCodec();
+
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override;
+
+ private:
+  // The gzip compressor is stateful
+  class GZipCodecImpl;
+  std::unique_ptr<GZipCodecImpl> impl_;
+};
+
+}  // namespace arrow
+
+#endif
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 49f1699f13623..8a929da0e0231 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -39,9 +39,10 @@ namespace arrow {
 #define ARROW_LOG_INTERNAL(level) ::arrow::internal::CerrLog(level)
 #define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level)
 
-#define ARROW_CHECK(condition)                               \
-  (condition) ? 0 : ::arrow::internal::FatalLog(ARROW_FATAL) \
-                        << __FILE__ << __LINE__ << " Check failed: " #condition " "
+#define ARROW_CHECK(condition)                           \
+  (condition) ? 0                                        \
+              : ::arrow::internal::FatalLog(ARROW_FATAL) \
+                    << __FILE__ << __LINE__ << " Check failed: " #condition " "
 
 #ifdef NDEBUG
 #define ARROW_DFATAL ARROW_WARNING
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
index e7ffd1ad37797..97c9a16c0c0bd 100644
--- a/cpp/src/plasma/malloc.cc
+++ b/cpp/src/plasma/malloc.cc
@@ -42,7 +42,7 @@ int fake_munmap(void*, int64_t);
 #define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
 #define DEFAULT_GRANULARITY ((size_t)128U * 1024U)
 
-#include "thirdparty/dlmalloc.c"
+#include "thirdparty/dlmalloc.c"  // NOLINT
 
 #undef MMAP
 #undef MUNMAP
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index dc457731b696c..29b5b135144c3 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -29,7 +29,7 @@
 #include "plasma/plasma.h"
 #include "plasma/protocol.h"
 
-std::string g_test_executable;
+std::string g_test_executable;  // NOLINT
 
 class TestPlasmaStore : public ::testing::Test {
  public:
diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py
new file mode 100644
index 0000000000000..e30d72bddd7f8
--- /dev/null
+++ b/dev/release/check-rat-report.py
@@ -0,0 +1,59 @@
+#!/usr/bin/python
+##############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##############################################################################
+import fnmatch
+import re
+import sys
+import xml.etree.ElementTree as ET
+
+if len(sys.argv) != 3:
+    sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
+                     sys.argv[0])
+    sys.exit(1)
+
+exclude_globs_filename = sys.argv[1]
+xml_filename = sys.argv[2]
+
+globs = [line.strip() for line in open(exclude_globs_filename, "r")]
+
+tree = ET.parse(xml_filename)
+root = tree.getroot()
+resources = root.findall('resource')
+
+all_ok = True
+for r in resources:
+    approvals = r.findall('license-approval')
+    if not approvals or approvals[0].attrib['name'] == 'true':
+        continue
+    clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
+    excluded = False
+    for g in globs:
+        if fnmatch.fnmatch(clean_name, g):
+            excluded = True
+            break
+    if not excluded:
+        sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
+            clean_name, r.attrib['name'], approvals[0].attrib['name']))
+        all_ok = False
+
+if not all_ok:
+    sys.exit(1)
+
+print('OK')
+sys.exit(0)
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
new file mode 100644
index 0000000000000..286793e85f146
--- /dev/null
+++ b/dev/release/rat_exclude_files.txt
@@ -0,0 +1,66 @@
+*.gitignore
+*_generated.h
+*.json
+cpp/src/arrow/io/mman.h
+cpp/src/arrow/util/random.h
+cpp/src/arrow/status.cc
+cpp/src/arrow/status.h
+cpp/build-support/asan_symbolize.py
+cpp/build-support/cpplint.py
+cpp/cmake_modules/BuildUtils.cmake
+cpp/cmake_modules/FindPythonLibsNew.cmake
+cpp/cmake_modules/FindNumPy.cmake
+cpp/cmake_modules/SetupCxxFlags.cmake
+cpp/cmake_modules/SnappyCMakeLists.txt
+cpp/cmake_modules/SnappyConfig.h
+cpp/cmake_modules/CompilerInfo.cmake
+cpp/src/plasma/thirdparty/ae/ae.c
+cpp/src/plasma/thirdparty/ae/ae.h
+cpp/src/plasma/thirdparty/ae/ae_epoll.c
+cpp/src/plasma/thirdparty/ae/ae_evport.c
+cpp/src/plasma/thirdparty/ae/ae_kqueue.c
+cpp/src/plasma/thirdparty/ae/ae_select.c
+cpp/src/plasma/thirdparty/ae/config.h
+cpp/src/plasma/thirdparty/ae/zmalloc.h
+cpp/src/plasma/thirdparty/dlmalloc.c
+cpp/src/plasma/thirdparty/xxhash.cc
+cpp/src/plasma/thirdparty/xxhash.h
+dev/release/rat_exclude_files.txt
+js/.npmignore
+python/cmake_modules/BuildUtils.cmake
+python/cmake_modules/FindPythonLibsNew.cmake
+python/cmake_modules/FindNumPy.cmake
+python/cmake_modules/SetupCxxFlags.cmake
+python/cmake_modules/CompilerInfo.cmake
+python/doc/requirements.txt
+python/MANIFEST.in
+python/pyarrow/includes/__init__.pxd
+python/pyarrow/tests/__init__.py
+python/requirements.txt
+pax_global_header
+MANIFEST.in
+__init__.pxd
+__init__.py
+requirements.txt
+version
+*.m4
+configure
+config.sub
+config.h.in
+compile
+missing
+install-sh
+config.guess
+depcomp
+ltmain.sh
+arrow-glib.types
+arrow-glib-sections.txt
+arrow-glib-overrides.txt
+gtk-doc.make
+*.html
+*.sgml
+*.css
+*.png
+*.svg
+*.devhelp2
+*.scss
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index 757604f38590e..53a322a969718 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -21,65 +21,15 @@
 # download apache rat
 curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.12/apache-rat-0.12.jar > apache-rat-0.12.jar
 
-RAT="java -jar apache-rat-0.12.jar -d "
+RAT="java -jar apache-rat-0.12.jar -x "
+
+RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
 
 # generate the rat report
-$RAT $1 \
-  -e ".*" \
-  -e mman.h \
-  -e "*_generated.h" \
-  -e "*.json" \
-  -e random.h \
-  -e status.cc \
-  -e status.h \
-  -e asan_symbolize.py \
-  -e cpplint.py \
-  -e BuildUtils.cmake \
-  -e FindPythonLibsNew.cmake \
-  -e FindNumPy.cmake \
-  -e SetupCxxFlags.cmake \
-  -e CompilerInfo.cmake \
-  -e pax_global_header \
-  -e MANIFEST.in \
-  -e __init__.pxd \
-  -e __init__.py \
-  -e requirements.txt \
-  -e version \
-  -e "*.m4" \
-  -e configure \
-  -e config.sub \
-  -e config.h.in \
-  -e compile \
-  -e missing \
-  -e install-sh \
-  -e config.guess \
-  -e depcomp \
-  -e ltmain.sh \
-  -e arrow-glib.types \
-  -e arrow-glib-sections.txt \
-  -e arrow-glib-overrides.txt \
-  -e gtk-doc.make \
-  -e ae.c \
-  -e ae.h \
-  -e ae_epoll.c \
-  -e ae_evport.c \
-  -e ae_kqueue.c \
-  -e ae_select.c \
-  -e config.h \
-  -e zmalloc.h \
-  -e dlmalloc.c \
-  -e xxhash.cc \
-  -e xxhash.h \
-  -e "*.html" \
-  -e "*.sgml" \
-  -e "*.css" \
-  -e "*.png" \
-  -e "*.svg" \
-  -e "*.devhelp2" \
-  -e "*.scss" \
-  > rat.txt
-cat rat.txt
-UNAPPROVED=`cat rat.txt  | grep "Unknown Licenses" | head -n 1 | cut -d " " -f 1`
+$RAT $1 > rat.txt
+python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt
+cat filtered_rat.txt
+UNAPPROVED=`cat filtered_rat.txt  | grep "NOT APPROVED" | wc -l`
 
 if [ "0" -eq "${UNAPPROVED}" ]; then
   echo "No unapproved licenses"

From 73007de727f07efdf91c4832b1284079fba59d77 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 23 Jun 2017 19:33:01 -0400
Subject: [PATCH 0746/1644] ARROW-1147: [C++] Allow optional vendoring of
 flatbuffers in plasma

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #780 from cpcloud/ARROW-1147 and squashes the following commits:

2deec3f2 [Phillip Cloud] Move target
ea9df3a4 [Phillip Cloud] ARROW-1147: [C++] Allow optional vendoring of flatbuffers in plasma
---
 cpp/src/plasma/CMakeLists.txt | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 8f2d81fba78bd..4ff3beba779c2 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -46,6 +46,12 @@ set(PLASMA_FBS_OUTPUT_FILES
   "${OUTPUT_DIR}/common_generated.h"
   "${OUTPUT_DIR}/plasma_generated.h")
 
+add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
+
+if(FLATBUFFERS_VENDORED)
+  add_dependencies(gen_plasma_fbs flatbuffers_ep)
+endif()
+
 add_custom_command(
   OUTPUT ${PLASMA_FBS_OUTPUT_FILES}
   # The --gen-object-api flag generates a C++ class MessageT for each
@@ -57,10 +63,6 @@ add_custom_command(
   COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}"
   VERBATIM)
 
-add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
-
-add_dependencies(gen_plasma_fbs flatbuffers_ep)
-
 if(UNIX AND NOT APPLE)
   link_libraries(rt)
 endif()

From 41524d6d280b284ca5be31de426928fd6eea97c7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 24 Jun 2017 11:05:08 -0400
Subject: [PATCH 0747/1644] ARROW-1135: [C++] Use clang 4.0 in one of the Linux
 builds

This will help us keep the Linux build clean on latest clang

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #779 from wesm/ARROW-1135 and squashes the following commits:

ca8ba32b [Wes McKinney] Fix dependency graph problems. Disable sign-conversion warnings
e5d3efb1 [Wes McKinney] Upgrade Flatbuffers to 1.7.1, because build fails in 1.6.0 on clang 4.0
f186b926 [Wes McKinney] Add clang installer script
24bb6c63 [Wes McKinney] Use clang 4.0 in one of the Linux builds
---
 .travis.yml                                 | 10 +++-------
 ci/travis_before_script_cpp.sh              |  2 +-
 ci/travis_env_common.sh                     |  2 ++
 ci/travis_install_clang_tools.sh            | 19 +++++++++++++++++++
 cpp/CMakeLists.txt                          |  9 +++++----
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 13 ++++++-------
 6 files changed, 36 insertions(+), 19 deletions(-)
 create mode 100644 ci/travis_install_clang_tools.sh

diff --git a/.travis.yml b/.travis.yml
index 315cbd21f2e19..ff2cafede9cde 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -85,8 +85,9 @@ matrix:
     env: ARROW_TEST_GROUP=integration
     jdk: openjdk7
     before_script:
-    - export CC="gcc-4.9"
-    - export CXX="g++-4.9"
+    - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
+    - export CC="clang-4.0"
+    - export CXX="clang++-4.0"
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
@@ -122,11 +123,6 @@ matrix:
 
 before_install:
 - ulimit -c unlimited -S
-- export CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
-- export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
-
-after_script:
-- rm -rf $CPP_BUILD_DIR
 
 after_failure:
 - COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1)
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 3a1836ca3a4fa..36966da32f3bc 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -64,7 +64,7 @@ fi
 if [ $TRAVIS_OS_NAME == "linux" ]; then
     cmake $CMAKE_COMMON_FLAGS \
           $CMAKE_LINUX_FLAGS \
-          -DARROW_CXXFLAGS="-Wconversion -Werror" \
+          -DARROW_CXXFLAGS="-Wconversion -Wno-sign-conversion -Werror" \
           $ARROW_CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index 9ded3e07e7ef7..a2e591014cfd3 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -23,6 +23,8 @@ export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java
 export ARROW_JS_DIR=${TRAVIS_BUILD_DIR}/js
 export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
 
+export CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
+
 export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
 export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
 export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
new file mode 100644
index 0000000000000..a4fd0e24619b1
--- /dev/null
+++ b/ci/travis_install_clang_tools.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
+sudo apt-add-repository -y \
+     "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-4.0 main"
+sudo apt-get update
+sudo apt-get install clang-4.0 clang-format-4.0 clang-tidy-4.0
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 49e1d97510e26..ca3410197343c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -182,6 +182,8 @@ endif()
 # Dependencies
 ############################################################
 
+add_custom_target(arrow_dependencies)
+
 include(ThirdpartyToolchain)
 
 # Add common flags
@@ -526,8 +528,7 @@ set(ARROW_STATIC_LINK_LIBS
   snappy
   zlib)
 
-set(ARROW_DEPENDENCIES
-  ${ARROW_STATIC_LINK_LIBS})
+add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
 
 set(ARROW_MIN_TEST_LIBS
   arrow_static
@@ -626,7 +627,7 @@ add_subdirectory(src/arrow/io)
 
 if (ARROW_IPC)
   add_subdirectory(src/arrow/ipc)
-  set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} metadata_fbs)
+  add_dependencies(arrow_dependencies metadata_fbs)
 endif()
 
 set(ARROW_SRCS
@@ -684,7 +685,7 @@ set(ARROW_ALL_SRCS
 
 ADD_ARROW_LIB(arrow
   SOURCES ${ARROW_ALL_SRCS}
-  DEPENDENCIES ${ARROW_DEPENDENCIES}
+  DEPENDENCIES arrow_dependencies
   SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
   SHARED_LINK_LIBS ${ARROW_LINK_LIBS}
   SHARED_PRIVATE_LINK_LIBS ${ARROW_SHARED_PRIVATE_LINK_LIBS}
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index f6a9bb4f80e80..f51f6ffdb58e8 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -23,7 +23,7 @@ set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 set(GFLAGS_VERSION "2.1.2")
 set(GTEST_VERSION "1.8.0")
 set(GBENCHMARK_VERSION "1.1.0")
-set(FLATBUFFERS_VERSION "1.6.0")
+set(FLATBUFFERS_VERSION "1.7.1")
 set(JEMALLOC_VERSION "4.4.0")
 set(SNAPPY_VERSION "1.1.3")
 set(BROTLI_VERSION "v0.6.0")
@@ -326,7 +326,6 @@ if(ARROW_BUILD_BENCHMARKS)
   endif()
 endif()
 
-
 if (ARROW_IPC)
   # RapidJSON, header only dependency
   if("${RAPIDJSON_HOME}" STREQUAL "")
@@ -349,6 +348,10 @@ if (ARROW_IPC)
   message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
   include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
 
+  if(RAPIDJSON_VENDORED)
+    add_dependencies(arrow_dependencies rapidjson_ep)
+  endif()
+
   ## Flatbuffers
   if("${FLATBUFFERS_HOME}" STREQUAL "")
     set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
@@ -367,12 +370,8 @@ if (ARROW_IPC)
     set(FLATBUFFERS_VENDORED 0)
   endif()
 
-  if(RAPIDJSON_VENDORED)
-    set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} rapidjson_ep)
-  endif()
-
   if(FLATBUFFERS_VENDORED)
-    set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} flatbuffers_ep)
+    add_dependencies(arrow_dependencies flatbuffers_ep)
   endif()
 
   message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")

From f3bcf76f2c5eb11b9f750994b97c3006b6371fec Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 24 Jun 2017 12:20:34 -0400
Subject: [PATCH 0748/1644] ARROW-1145: [GLib] Add get_values()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #775 from kou/glib-get-values and squashes the following commits:

7a02ab73 [Kouhei Sutou] [GLib] Use template for common code
ebb24f9e [Kouhei Sutou] [GLib] Add get_values()
---
 c_glib/arrow-glib/array.cpp      | 168 ++++++++++++++++++++++++++++++-
 c_glib/arrow-glib/array.h        |  20 ++++
 c_glib/test/test-double-array.rb |  11 ++
 c_glib/test/test-float-array.rb  |  11 ++
 c_glib/test/test-int16-array.rb  |  11 ++
 c_glib/test/test-int32-array.rb  |   9 ++
 c_glib/test/test-int64-array.rb  |   9 ++
 c_glib/test/test-int8-array.rb   |   9 ++
 c_glib/test/test-uint16-array.rb |  11 ++
 c_glib/test/test-uint32-array.rb |  11 ++
 c_glib/test/test-uint64-array.rb |  11 ++
 c_glib/test/test-uint8-array.rb  |   9 ++
 12 files changed, 289 insertions(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index e9a6a494c0faf..92a748de7a068 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -30,6 +30,17 @@
 #include <iostream>
 #include <sstream>
 
+template <typename T>
+const typename T::c_type *
+garrow_array_get_values_raw(std::shared_ptr<arrow::Array> arrow_array,
+                            gint64 *length)
+{
+  auto arrow_specific_array =
+    std::static_pointer_cast<typename arrow::TypeTraits<T>::ArrayType>(arrow_array);
+  *length = arrow_specific_array->length();
+  return arrow_specific_array->raw_data();
+};
+
 G_BEGIN_DECLS
 
 /**
@@ -609,6 +620,21 @@ garrow_int8_array_get_value(GArrowInt8Array *array,
   return static_cast<arrow::Int8Array *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_int8_array_get_values:
+ * @array: A #GArrowInt8Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const gint8 *
+garrow_int8_array_get_values(GArrowInt8Array *array,
+                             gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::Int8Type>(arrow_array, length);
+}
+
 
 G_DEFINE_TYPE(GArrowUInt8Array,               \
               garrow_uint8_array,             \
@@ -672,6 +698,21 @@ garrow_uint8_array_get_value(GArrowUInt8Array *array,
   return static_cast<arrow::UInt8Array *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_uint8_array_get_values:
+ * @array: A #GArrowUInt8Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const guint8 *
+garrow_uint8_array_get_values(GArrowUInt8Array *array,
+                              gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::UInt8Type>(arrow_array, length);
+}
+
 
 G_DEFINE_TYPE(GArrowInt16Array,               \
               garrow_int16_array,             \
@@ -735,6 +776,21 @@ garrow_int16_array_get_value(GArrowInt16Array *array,
   return static_cast<arrow::Int16Array *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_int16_array_get_values:
+ * @array: A #GArrowInt16Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const gint16 *
+garrow_int16_array_get_values(GArrowInt16Array *array,
+                              gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::Int16Type>(arrow_array, length);
+}
+
 
 G_DEFINE_TYPE(GArrowUInt16Array,               \
               garrow_uint16_array,             \
@@ -792,12 +848,27 @@ garrow_uint16_array_new(gint64 length,
  */
 guint16
 garrow_uint16_array_get_value(GArrowUInt16Array *array,
-                             gint64 i)
+                              gint64 i)
 {
   auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
   return static_cast<arrow::UInt16Array *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_uint16_array_get_values:
+ * @array: A #GArrowUInt16Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const guint16 *
+garrow_uint16_array_get_values(GArrowUInt16Array *array,
+                               gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::UInt16Type>(arrow_array, length);
+}
+
 
 G_DEFINE_TYPE(GArrowInt32Array,               \
               garrow_int32_array,             \
@@ -861,6 +932,21 @@ garrow_int32_array_get_value(GArrowInt32Array *array,
   return static_cast<arrow::Int32Array *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_int32_array_get_values:
+ * @array: A #GArrowInt32Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const gint32 *
+garrow_int32_array_get_values(GArrowInt32Array *array,
+                              gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::Int32Type>(arrow_array, length);
+}
+
 
 G_DEFINE_TYPE(GArrowUInt32Array,               \
               garrow_uint32_array,             \
@@ -924,6 +1010,21 @@ garrow_uint32_array_get_value(GArrowUInt32Array *array,
   return static_cast<arrow::UInt32Array *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_uint32_array_get_values:
+ * @array: A #GArrowUInt32Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const guint32 *
+garrow_uint32_array_get_values(GArrowUInt32Array *array,
+                               gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::UInt32Type>(arrow_array, length);
+}
+
 
 G_DEFINE_TYPE(GArrowInt64Array,               \
               garrow_int64_array,             \
@@ -987,6 +1088,23 @@ garrow_int64_array_get_value(GArrowInt64Array *array,
   return static_cast<arrow::Int64Array *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_int64_array_get_values:
+ * @array: A #GArrowInt64Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const gint64 *
+garrow_int64_array_get_values(GArrowInt64Array *array,
+                              gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto values =
+    garrow_array_get_values_raw<arrow::Int64Type>(arrow_array, length);
+  return reinterpret_cast<const gint64 *>(values);
+}
+
 
 G_DEFINE_TYPE(GArrowUInt64Array,               \
               garrow_uint64_array,             \
@@ -1050,6 +1168,24 @@ garrow_uint64_array_get_value(GArrowUInt64Array *array,
   return static_cast<arrow::UInt64Array *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_uint64_array_get_values:
+ * @array: A #GArrowUInt64Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const guint64 *
+garrow_uint64_array_get_values(GArrowUInt64Array *array,
+                               gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto values =
+    garrow_array_get_values_raw<arrow::UInt64Type>(arrow_array, length);
+  return reinterpret_cast<const guint64 *>(values);
+}
+
+
 G_DEFINE_TYPE(GArrowFloatArray,               \
               garrow_float_array,             \
               GARROW_TYPE_PRIMITIVE_ARRAY)
@@ -1112,6 +1248,21 @@ garrow_float_array_get_value(GArrowFloatArray *array,
   return static_cast<arrow::FloatArray *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_float_array_get_values:
+ * @array: A #GArrowFloatArray.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const gfloat *
+garrow_float_array_get_values(GArrowFloatArray *array,
+                              gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::FloatType>(arrow_array, length);
+}
+
 
 G_DEFINE_TYPE(GArrowDoubleArray,               \
               garrow_double_array,             \
@@ -1175,6 +1326,21 @@ garrow_double_array_get_value(GArrowDoubleArray *array,
   return static_cast<arrow::DoubleArray *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_double_array_get_values:
+ * @array: A #GArrowDoubleArray.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ */
+const gdouble *
+garrow_double_array_get_values(GArrowDoubleArray *array,
+                               gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::DoubleType>(arrow_array, length);
+}
+
 
 G_DEFINE_TYPE(GArrowBinaryArray,               \
               garrow_binary_array,             \
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index d32157b74cbc9..6043e904ef2df 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -272,6 +272,8 @@ GArrowInt8Array *garrow_int8_array_new(gint64 length,
 
 gint8 garrow_int8_array_get_value(GArrowInt8Array *array,
                                   gint64 i);
+const gint8 *garrow_int8_array_get_values(GArrowInt8Array *array,
+                                          gint64 *length);
 
 
 #define GARROW_TYPE_UINT8_ARRAY                 \
@@ -323,6 +325,8 @@ GArrowUInt8Array *garrow_uint8_array_new(gint64 length,
 
 guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array,
                                     gint64 i);
+const guint8 *garrow_uint8_array_get_values(GArrowUInt8Array *array,
+                                            gint64 *length);
 
 
 #define GARROW_TYPE_INT16_ARRAY                  \
@@ -374,6 +378,8 @@ GArrowInt16Array *garrow_int16_array_new(gint64 length,
 
 gint16 garrow_int16_array_get_value(GArrowInt16Array *array,
                                     gint64 i);
+const gint16 *garrow_int16_array_get_values(GArrowInt16Array *array,
+                                            gint64 *length);
 
 
 #define GARROW_TYPE_UINT16_ARRAY                 \
@@ -425,6 +431,8 @@ GArrowUInt16Array *garrow_uint16_array_new(gint64 length,
 
 guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array,
                                       gint64 i);
+const guint16 *garrow_uint16_array_get_values(GArrowUInt16Array *array,
+                                              gint64 *length);
 
 
 #define GARROW_TYPE_INT32_ARRAY                 \
@@ -476,6 +484,8 @@ GArrowInt32Array *garrow_int32_array_new(gint64 length,
 
 gint32 garrow_int32_array_get_value(GArrowInt32Array *array,
                                     gint64 i);
+const gint32 *garrow_int32_array_get_values(GArrowInt32Array *array,
+                                            gint64 *length);
 
 
 #define GARROW_TYPE_UINT32_ARRAY                \
@@ -527,6 +537,8 @@ GArrowUInt32Array *garrow_uint32_array_new(gint64 length,
 
 guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array,
                                       gint64 i);
+const guint32 *garrow_uint32_array_get_values(GArrowUInt32Array *array,
+                                              gint64 *length);
 
 
 #define GARROW_TYPE_INT64_ARRAY                 \
@@ -578,6 +590,8 @@ GArrowInt64Array *garrow_int64_array_new(gint64 length,
 
 gint64 garrow_int64_array_get_value(GArrowInt64Array *array,
                                     gint64 i);
+const gint64 *garrow_int64_array_get_values(GArrowInt64Array *array,
+                                            gint64 *length);
 
 
 #define GARROW_TYPE_UINT64_ARRAY                \
@@ -629,6 +643,8 @@ GArrowUInt64Array *garrow_uint64_array_new(gint64 length,
 
 guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array,
                                       gint64 i);
+const guint64 *garrow_uint64_array_get_values(GArrowUInt64Array *array,
+                                              gint64 *length);
 
 
 #define GARROW_TYPE_FLOAT_ARRAY                 \
@@ -680,6 +696,8 @@ GArrowFloatArray *garrow_float_array_new(gint64 length,
 
 gfloat garrow_float_array_get_value(GArrowFloatArray *array,
                                     gint64 i);
+const gfloat *garrow_float_array_get_values(GArrowFloatArray *array,
+                                            gint64 *length);
 
 
 #define GARROW_TYPE_DOUBLE_ARRAY                \
@@ -731,6 +749,8 @@ GArrowDoubleArray *garrow_double_array_new(gint64 length,
 
 gdouble garrow_double_array_get_value(GArrowDoubleArray *array,
                                       gint64 i);
+const gdouble *garrow_double_array_get_values(GArrowDoubleArray *array,
+                                              gint64 *length);
 
 
 #define GARROW_TYPE_BINARY_ARRAY                \
diff --git a/c_glib/test/test-double-array.rb b/c_glib/test/test-double-array.rb
index 935fbe5b93dd9..c4a2d3017b6d0 100644
--- a/c_glib/test/test-double-array.rb
+++ b/c_glib/test/test-double-array.rb
@@ -17,6 +17,7 @@
 
 class TestDoubleArray < Test::Unit::TestCase
   include Helper::Buildable
+  include Helper::Omittable
 
   def test_new
     assert_equal(build_double_array([-1.1, 2.2, nil]),
@@ -41,4 +42,14 @@ def test_value
     array = builder.finish
     assert_in_delta(1.5, array.get_value(0))
   end
+
+  def test_values
+    require_gi(3, 1, 7)
+    builder = Arrow::DoubleArrayBuilder.new
+    builder.append(1.5)
+    builder.append(3)
+    builder.append(4.5)
+    array = builder.finish
+    assert_equal([1.5, 3.0, 4.5], array.values)
+  end
 end
diff --git a/c_glib/test/test-float-array.rb b/c_glib/test/test-float-array.rb
index fcac9021e56d2..ce77c8b3defba 100644
--- a/c_glib/test/test-float-array.rb
+++ b/c_glib/test/test-float-array.rb
@@ -17,6 +17,7 @@
 
 class TestFloatArray < Test::Unit::TestCase
   include Helper::Buildable
+  include Helper::Omittable
 
   def test_new
     assert_equal(build_float_array([-1.1, 2.2, nil]),
@@ -41,4 +42,14 @@ def test_value
     array = builder.finish
     assert_in_delta(1.5, array.get_value(0))
   end
+
+  def test_values
+    require_gi(3, 1, 7)
+    builder = Arrow::FloatArrayBuilder.new
+    builder.append(1.5)
+    builder.append(3)
+    builder.append(4.5)
+    array = builder.finish
+    assert_equal([1.5, 3.0, 4.5], array.values)
+  end
 end
diff --git a/c_glib/test/test-int16-array.rb b/c_glib/test/test-int16-array.rb
index 6bc7f8815c26e..8703e803ff0f4 100644
--- a/c_glib/test/test-int16-array.rb
+++ b/c_glib/test/test-int16-array.rb
@@ -17,6 +17,7 @@
 
 class TestInt16Array < Test::Unit::TestCase
   include Helper::Buildable
+  include Helper::Omittable
 
   def test_new
     assert_equal(build_int16_array([-1, 2, nil]),
@@ -41,4 +42,14 @@ def test_value
     array = builder.finish
     assert_equal(-1, array.get_value(0))
   end
+
+  def test_values
+    require_gi(3, 1, 7)
+    builder = Arrow::Int16ArrayBuilder.new
+    builder.append(-1)
+    builder.append(2)
+    builder.append(-4)
+    array = builder.finish
+    assert_equal([-1, 2, -4], array.values)
+  end
 end
diff --git a/c_glib/test/test-int32-array.rb b/c_glib/test/test-int32-array.rb
index 0b68273aca7dd..d1579a8eba881 100644
--- a/c_glib/test/test-int32-array.rb
+++ b/c_glib/test/test-int32-array.rb
@@ -41,4 +41,13 @@ def test_value
     array = builder.finish
     assert_equal(-1, array.get_value(0))
   end
+
+  def test_values
+    builder = Arrow::Int32ArrayBuilder.new
+    builder.append(-1)
+    builder.append(2)
+    builder.append(-4)
+    array = builder.finish
+    assert_equal([-1, 2, -4], array.values)
+  end
 end
diff --git a/c_glib/test/test-int64-array.rb b/c_glib/test/test-int64-array.rb
index c2174345746c2..5d9c37a55c084 100644
--- a/c_glib/test/test-int64-array.rb
+++ b/c_glib/test/test-int64-array.rb
@@ -41,4 +41,13 @@ def test_value
     array = builder.finish
     assert_equal(-1, array.get_value(0))
   end
+
+  def test_values
+    builder = Arrow::Int64ArrayBuilder.new
+    builder.append(-1)
+    builder.append(2)
+    builder.append(-4)
+    array = builder.finish
+    assert_equal([-1, 2, -4], array.values)
+  end
 end
diff --git a/c_glib/test/test-int8-array.rb b/c_glib/test/test-int8-array.rb
index c7ff2165056cb..e17c10c53611e 100644
--- a/c_glib/test/test-int8-array.rb
+++ b/c_glib/test/test-int8-array.rb
@@ -41,4 +41,13 @@ def test_value
     array = builder.finish
     assert_equal(-1, array.get_value(0))
   end
+
+  def test_values
+    builder = Arrow::Int8ArrayBuilder.new
+    builder.append(-1)
+    builder.append(2)
+    builder.append(-4)
+    array = builder.finish
+    assert_equal([-1, 2, -4], array.values)
+  end
 end
diff --git a/c_glib/test/test-uint16-array.rb b/c_glib/test/test-uint16-array.rb
index e3ffa5d28b6fa..1c79c999c30cf 100644
--- a/c_glib/test/test-uint16-array.rb
+++ b/c_glib/test/test-uint16-array.rb
@@ -17,6 +17,7 @@
 
 class TestUInt16Array < Test::Unit::TestCase
   include Helper::Buildable
+  include Helper::Omittable
 
   def test_new
     assert_equal(build_uint16_array([1, 2, nil]),
@@ -41,4 +42,14 @@ def test_value
     array = builder.finish
     assert_equal(1, array.get_value(0))
   end
+
+  def test_values
+    require_gi(3, 1, 7)
+    builder = Arrow::UInt16ArrayBuilder.new
+    builder.append(1)
+    builder.append(2)
+    builder.append(4)
+    array = builder.finish
+    assert_equal([1, 2, 4], array.values)
+  end
 end
diff --git a/c_glib/test/test-uint32-array.rb b/c_glib/test/test-uint32-array.rb
index 95aee79921929..a4d1e3257c98e 100644
--- a/c_glib/test/test-uint32-array.rb
+++ b/c_glib/test/test-uint32-array.rb
@@ -17,6 +17,7 @@
 
 class TestUInt32Array < Test::Unit::TestCase
   include Helper::Buildable
+  include Helper::Omittable
 
   def test_new
     assert_equal(build_uint32_array([1, 2, nil]),
@@ -41,4 +42,14 @@ def test_value
     array = builder.finish
     assert_equal(1, array.get_value(0))
   end
+
+  def test_values
+    require_gi(3, 1, 7)
+    builder = Arrow::UInt32ArrayBuilder.new
+    builder.append(1)
+    builder.append(2)
+    builder.append(4)
+    array = builder.finish
+    assert_equal([1, 2, 4], array.values)
+  end
 end
diff --git a/c_glib/test/test-uint64-array.rb b/c_glib/test/test-uint64-array.rb
index 7d9185459b295..b7f9c403e2494 100644
--- a/c_glib/test/test-uint64-array.rb
+++ b/c_glib/test/test-uint64-array.rb
@@ -17,6 +17,7 @@
 
 class TestUInt64Array < Test::Unit::TestCase
   include Helper::Buildable
+  include Helper::Omittable
 
   def test_new
     assert_equal(build_uint64_array([1, 2, nil]),
@@ -41,4 +42,14 @@ def test_value
     array = builder.finish
     assert_equal(1, array.get_value(0))
   end
+
+  def test_values
+    require_gi(3, 1, 7)
+    builder = Arrow::UInt64ArrayBuilder.new
+    builder.append(1)
+    builder.append(2)
+    builder.append(4)
+    array = builder.finish
+    assert_equal([1, 2, 4], array.values)
+  end
 end
diff --git a/c_glib/test/test-uint8-array.rb b/c_glib/test/test-uint8-array.rb
index 9c93abe7c349e..9137e53be70e5 100644
--- a/c_glib/test/test-uint8-array.rb
+++ b/c_glib/test/test-uint8-array.rb
@@ -41,4 +41,13 @@ def test_value
     array = builder.finish
     assert_equal(1, array.get_value(0))
   end
+
+  def test_values
+    builder = Arrow::UInt8ArrayBuilder.new
+    builder.append(1)
+    builder.append(2)
+    builder.append(4)
+    array = builder.finish
+    assert_equal([1, 2, 4], array.values)
+  end
 end

From bea30d6c24ac07b9c4b29ba1523a13dcd27d578b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 24 Jun 2017 12:21:09 -0400
Subject: [PATCH 0749/1644] ARROW-1113: [C++] Upgrade to gflags 2.2.0, use
 tarball instead of git tag

This fixes the weirdness I was seeing with make and ninja where the EP was re-running (but doing nothing)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #776 from wesm/gflags-ep-make-oddness and squashes the following commits:

82b163b2 [Wes McKinney] Upgrade to gflags 2.2.0, use tarball to avoid odd no-op work on subsequent builds with make or ninja
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index f51f6ffdb58e8..d77aa53f9c026 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -20,7 +20,7 @@
 # Thirdparty toolchain
 
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
-set(GFLAGS_VERSION "2.1.2")
+set(GFLAGS_VERSION "2.2.0")
 set(GTEST_VERSION "1.8.0")
 set(GBENCHMARK_VERSION "1.1.0")
 set(FLATBUFFERS_VERSION "1.7.1")
@@ -223,6 +223,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
   if("${GFLAGS_HOME}" STREQUAL "")
     set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS})
 
+    set(GFLAGS_URL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz")
     set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
     set(GFLAGS_HOME "${GFLAGS_PREFIX}")
     set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
@@ -244,15 +245,13 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
     if (CMAKE_VERSION VERSION_GREATER "3.2")
       # BUILD_BYPRODUCTS is a 3.2+ feature
       ExternalProject_Add(gflags_ep
-        GIT_REPOSITORY https://github.com/gflags/gflags.git
-        GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+        URL ${GFLAGS_URL}
         BUILD_IN_SOURCE 1
         BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}"
         CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
     else()
       ExternalProject_Add(gflags_ep
-        GIT_REPOSITORY https://github.com/gflags/gflags.git
-        GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+        URL ${GFLAGS_URL}
         BUILD_IN_SOURCE 1
         CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
     endif()

From fc3f8c2a05a2e3b2aec9b319f2686fe678e307f4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 26 Jun 2017 08:59:35 +0200
Subject: [PATCH 0750/1644] ARROW-1131: [Python] Enable the Parquet unit tests
 by default if the extension imports

@xhochy what do you think about this?

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #781 from wesm/ARROW-1131 and squashes the following commits:

8803e52 [Wes McKinney] Enable the Parquet unit tests by default if the extension imports
---
 python/pyarrow/tests/conftest.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index 9b767fc7d52a7..fa9608f8afb7a 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -20,6 +20,17 @@
 
 groups = ['hdfs', 'parquet']
 
+defaults = {
+    'hdfs': False,
+    'parquet': False
+}
+
+try:
+    import pyarrow.parquet
+    defaults['parquet'] = True
+except ImportError:
+    pass
+
 
 def pytest_configure(config):
     pass
@@ -28,7 +39,7 @@ def pytest_configure(config):
 def pytest_addoption(parser):
     for group in groups:
         parser.addoption('--{0}'.format(group), action='store_true',
-                         default=False,
+                         default=defaults[group],
                          help=('Enable the {0} test group'.format(group)))
 
     for group in groups:

From 5de6eb5ffed91d1d06dca27ee6613636c7f87e41 Mon Sep 17 00:00:00 2001
From: Jeff Knupp <jeff.knupp@enigma.com>
Date: Mon, 26 Jun 2017 09:52:29 -0400
Subject: [PATCH 0751/1644] ARROW-978: [Python] - Change python documentation
 sphinx theme to bootstrap

This sets the default HTML theme to the sphinx-bootstrap-theme. There were vestiges of the (improperly configured and thus unused) sphinx-rtd-theme which I removed.

Author: Jeff Knupp <jeff.knupp@enigma.com>

Closes #784 from jeffknupp/master and squashes the following commits:

f4a1cb64 [Jeff Knupp] ARROW-978: Python - Change sphinx theme to bootstrap
---
 python/doc/requirements.txt | 2 +-
 python/doc/source/conf.py   | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/python/doc/requirements.txt b/python/doc/requirements.txt
index f3c3414a4be9a..ee9a4d8e7a732 100644
--- a/python/doc/requirements.txt
+++ b/python/doc/requirements.txt
@@ -2,4 +2,4 @@ ipython
 matplotlib
 numpydoc
 sphinx
-sphinx_rtd_theme
+sphinx_bootstrap_theme
diff --git a/python/doc/source/conf.py b/python/doc/source/conf.py
index 7f98979e88ff8..4a72fd0d6f958 100644
--- a/python/doc/source/conf.py
+++ b/python/doc/source/conf.py
@@ -28,7 +28,7 @@
 import os
 import sys
 
-import sphinx_rtd_theme
+import sphinx_bootstrap_theme
 
 sys.path.extend([
     os.path.join(os.path.dirname(__file__),
@@ -150,7 +150,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinxdoc'
+html_theme = 'bootstrap'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -159,7 +159,7 @@
 # html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
 
 # The name for this set of Sphinx documents.
 # "<project> v<release> documentation" by default.

From ec6e183ee31411a18c531507216c21cc80060c8b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 26 Jun 2017 14:01:59 -0400
Subject: [PATCH 0752/1644] ARROW-1151: [C++] Add branch prediction to
 RETURN_NOT_OK

Also added some missing status checks to builder-benchmark

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #782 from wesm/ARROW-1151 and squashes the following commits:

9b488a0e [Wes McKinney] Try to fix snappy warning
06276119 [Wes McKinney] Restore check macros used in libplasma
83b3f36d [Wes McKinney] Add branch prediction to RETURN_NOT_OK
---
 LICENSE.txt                        | 34 +++++++++++++++++++++++++
 cpp/src/arrow/builder-benchmark.cc | 40 ++++++++++++++++--------------
 cpp/src/arrow/status.h             | 36 ++++++++-------------------
 cpp/src/arrow/util/compression.cc  |  5 ++++
 cpp/src/arrow/util/macros.h        | 14 +++++++++++
 5 files changed, 84 insertions(+), 45 deletions(-)

diff --git a/LICENSE.txt b/LICENSE.txt
index 700073379376a..95c506f22bb6f 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -296,3 +296,37 @@ You can contact the author at :
 - xxHash source repository : https://github.com/Cyan4973/xxHash
 
 --------------------------------------------------------------------------------
+
+src/arrow/util (some portions): Apache 2.0, and 3-clause BSD
+
+Some portions of this module are derived from code in the Chromium project,
+copyright (c) Google inc and (c) The Chromium Authors and licensed under the
+Apache 2.0 License or the under the 3-clause BSD license:
+
+  Copyright (c) 2013 The Chromium Authors. All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are
+  met:
+
+     * Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+     * Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following disclaimer
+  in the documentation and/or other materials provided with the
+  distribution.
+     * Neither the name of Google Inc. nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index 62f2fd620a969..3c49c6375448f 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -25,6 +25,12 @@ namespace arrow {
 
 constexpr int64_t kFinalSize = 256;
 
+#define ABORT_NOT_OK(s)                                 \
+  do {                                                  \
+    ::arrow::Status _s = (s);                           \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { exit(-1); }    \
+  } while (0);
+
 static void BM_BuildPrimitiveArrayNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
   // 2 MiB block
@@ -33,17 +39,15 @@ static void BM_BuildPrimitiveArrayNoNulls(
     Int64Builder builder(default_memory_pool());
     for (int i = 0; i < kFinalSize; i++) {
       // Build up an array of 512 MiB in size
-      builder.Append(data.data(), data.size(), nullptr);
+      ABORT_NOT_OK(builder.Append(data.data(), data.size(), nullptr));
     }
     std::shared_ptr<Array> out;
-    builder.Finish(&out);
+    ABORT_NOT_OK(builder.Finish(&out));
   }
   state.SetBytesProcessed(
       state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
 }
 
-BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
-
 static void BM_BuildVectorNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
   // 2 MiB block
@@ -59,8 +63,6 @@ static void BM_BuildVectorNoNulls(
       state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
 }
 
-BENCHMARK(BM_BuildVectorNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
-
 static void BM_BuildAdaptiveIntNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
   int64_t size = static_cast<int64_t>(std::numeric_limits<int16_t>::max()) * 256;
@@ -73,16 +75,14 @@ static void BM_BuildAdaptiveIntNoNulls(
     AdaptiveIntBuilder builder(default_memory_pool());
     for (int64_t i = 0; i < size; i += chunk_size) {
       // Build up an array of 512 MiB in size
-      builder.Append(data.data() + i, chunk_size, nullptr);
+      ABORT_NOT_OK(builder.Append(data.data() + i, chunk_size, nullptr));
     }
     std::shared_ptr<Array> out;
-    builder.Finish(&out);
+    ABORT_NOT_OK(builder.Finish(&out));
   }
   state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t));
 }
 
-BENCHMARK(BM_BuildAdaptiveIntNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
-
 static void BM_BuildAdaptiveIntNoNullsScalarAppend(
     benchmark::State& state) {  // NOLINT non-const reference
   int64_t size = static_cast<int64_t>(std::numeric_limits<int16_t>::max()) * 256;
@@ -93,18 +93,14 @@ static void BM_BuildAdaptiveIntNoNullsScalarAppend(
   while (state.KeepRunning()) {
     AdaptiveIntBuilder builder(default_memory_pool());
     for (int64_t i = 0; i < size; i++) {
-      builder.Append(data[i]);
+      ABORT_NOT_OK(builder.Append(data[i]));
     }
     std::shared_ptr<Array> out;
-    builder.Finish(&out);
+    ABORT_NOT_OK(builder.Finish(&out));
   }
   state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t));
 }
 
-BENCHMARK(BM_BuildAdaptiveIntNoNullsScalarAppend)
-    ->Repetitions(3)
-    ->Unit(benchmark::kMillisecond);
-
 static void BM_BuildAdaptiveUIntNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
   int64_t size = static_cast<int64_t>(std::numeric_limits<uint16_t>::max()) * 256;
@@ -117,14 +113,20 @@ static void BM_BuildAdaptiveUIntNoNulls(
     AdaptiveUIntBuilder builder(default_memory_pool());
     for (int64_t i = 0; i < size; i += chunk_size) {
       // Build up an array of 512 MiB in size
-      builder.Append(data.data() + i, chunk_size, nullptr);
+      ABORT_NOT_OK(builder.Append(data.data() + i, chunk_size, nullptr));
     }
     std::shared_ptr<Array> out;
-    builder.Finish(&out);
+    ABORT_NOT_OK(builder.Finish(&out));
   }
   state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t));
 }
 
-BENCHMARK(BM_BuildAdaptiveUIntNoNulls)->Repetitions(3)->Unit(benchmark::kMillisecond);
+BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildVectorNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildAdaptiveIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildAdaptiveIntNoNullsScalarAppend)
+    ->Repetitions(3)
+    ->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildAdaptiveUIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 7e7f67ce62800..9a75a58dd91ec 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -10,7 +10,7 @@
 // non-const method, all threads accessing the same Status must use
 // external synchronization.
 
-// Adapted from Kudu github.com/cloudera/kudu
+// Adapted from Kudu github.com/apache/kudu
 
 #ifndef ARROW_STATUS_H_
 #define ARROW_STATUS_H_
@@ -19,30 +19,14 @@
 #include <cstring>
 #include <string>
 
+#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 // Return the given status if it is not OK.
-#define ARROW_RETURN_NOT_OK(s)   \
-  do {                           \
-    ::arrow::Status _s = (s);    \
-    if (!_s.ok()) { return _s; } \
-  } while (0);
-
-// Return the given status if it is not OK, but first clone it and
-// prepend the given message.
-#define ARROW_RETURN_NOT_OK_PREPEND(s, msg)                               \
-  do {                                                                    \
-    ::arrow::Status _s = (s);                                             \
-    if (::gutil::PREDICT_FALSE(!_s.ok())) return _s.CloneAndPrepend(msg); \
-  } while (0);
-
-// Return 'to_return' if 'to_call' returns a bad status.
-// The substitution for 'to_return' may reference the variable
-// 's' for the bad status.
-#define ARROW_RETURN_NOT_OK_RET(to_call, to_return)          \
-  do {                                                       \
-    ::arrow::Status s = (to_call);                           \
-    if (::gutil::PREDICT_FALSE(!s.ok())) return (to_return); \
+#define ARROW_RETURN_NOT_OK(s)                          \
+  do {                                                  \
+    ::arrow::Status _s = (s);                           \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; }   \
   } while (0);
 
 // If 'to_call' returns a bad status, CHECK immediately with a logged message
@@ -59,10 +43,10 @@
 
 namespace arrow {
 
-#define RETURN_NOT_OK(s)         \
-  do {                           \
-    Status _s = (s);             \
-    if (!_s.ok()) { return _s; } \
+#define RETURN_NOT_OK(s)                                \
+  do {                                                  \
+    Status _s = (s);                                    \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; }   \
   } while (0);
 
 #define RETURN_NOT_OK_ELSE(s, else_) \
diff --git a/cpp/src/arrow/util/compression.cc b/cpp/src/arrow/util/compression.cc
index f82ae5cac1150..070f857e69643 100644
--- a/cpp/src/arrow/util/compression.cc
+++ b/cpp/src/arrow/util/compression.cc
@@ -17,6 +17,11 @@
 
 #include "arrow/util/compression.h"
 
+// Work around warning caused by Snappy include
+#ifdef DISALLOW_COPY_AND_ASSIGN
+#undef DISALLOW_COPY_AND_ASSIGN
+#endif
+
 #include <cstdint>
 #include <memory>
 #include <sstream>
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index c4a62a475b92f..affe8bca4f207 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -27,4 +27,18 @@
 
 #define UNUSED(x) (void)x
 
+//
+// GCC can be told that a certain branch is not likely to be taken (for
+// instance, a CHECK failure), and use that information in static analysis.
+// Giving it this information can help it optimize for the common case in
+// the absence of better information (ie. -fprofile-arcs).
+//
+#if defined(__GNUC__)
+#define ARROW_PREDICT_FALSE(x) (__builtin_expect(x, 0))
+#define ARROW_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
+#else
+#define ARROW_PREDICT_FALSE(x) x
+#define ARROW_PREDICT_TRUE(x) x
+#endif
+
 #endif  // ARROW_UTIL_MACROS_H

From bfe15dbf91dce9a1dd810951db522de50e376c9f Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 26 Jun 2017 16:43:38 -0400
Subject: [PATCH 0753/1644] ARROW-1152: [Cython] read_tensor should work with a
 readable file

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #783 from pcmoritz/fix-read-tensor and squashes the following commits:

a03b59fc [Philipp Moritz] add unit test
e6fb7104 [Philipp Moritz] fix read_tensor
---
 python/pyarrow/io.pxi               |  2 +-
 python/pyarrow/tests/test_tensor.py | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 23eb6ef10823e..9f0ad7e4bf3a3 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -1254,7 +1254,7 @@ def read_tensor(NativeFile source):
     cdef:
         shared_ptr[CTensor] sp_tensor
 
-    source._assert_writeable()
+    source._assert_readable()
 
     cdef int64_t offset = source.tell()
     with nogil:
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index b0924e3504ff7..a83f6f2b5eacd 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -114,3 +114,16 @@ def test_tensor_size():
     data = np.random.randn(10, 4)
     tensor = pa.Tensor.from_numpy(data)
     assert pa.get_tensor_size(tensor) > (data.size * 8)
+
+def test_read_tensor(tmpdir):
+    # Create and write tensor tensor
+    data = np.random.randn(10, 4)
+    tensor = pa.Tensor.from_numpy(data)
+    data_size = pa.get_tensor_size(tensor)
+    path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-read-tensor')
+    write_mmap = pa.create_memory_map(path, data_size)
+    pa.write_tensor(tensor, write_mmap)
+    # Try to read tensor
+    read_mmap = pa.memory_map(path, mode='r')
+    array = pa.read_tensor(read_mmap).to_numpy()
+    np.testing.assert_equal(data, array)

From 3e754a0d0b36967f0df7a14135b8be5da3d6cf77 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 26 Jun 2017 16:46:05 -0400
Subject: [PATCH 0754/1644] ARROW-1155: [Python] Add null check when user
 improperly instantiates ArrayValue instances

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #786 from wesm/ARROW-1155 and squashes the following commits:

e496e919 [Wes McKinney] Add null check when user improperly instantiates ArrayValue instances
---
 python/pyarrow/array.pxi             |  7 +++++++
 python/pyarrow/tests/test_scalars.py | 13 +++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index c7563c8536f61..2a0b0a7c5257f 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -729,7 +729,14 @@ cdef class ArrayValue(Scalar):
     cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
         self.sp_array = sp_array
 
+    def _check_null(self):
+        if self.sp_array.get() == NULL:
+            raise ReferenceError(
+                'ArrayValue instance not propertly initialized '
+                '(references NULL pointer)')
+
     def __repr__(self):
+        self._check_null()
         if hasattr(self, 'as_py'):
             return repr(self.as_py())
         else:
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 149973b7831cb..0aa9466936503 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -16,6 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import pytest
+
 import pandas as pd
 
 from pyarrow.compat import unittest, u, unicode_type
@@ -28,6 +30,17 @@ def test_null_singleton(self):
         with self.assertRaises(Exception):
             pa.NAType()
 
+    def test_ctor_null_check(self):
+        # ARROW-1155
+        with pytest.raises(ReferenceError):
+            repr(pa.Int16Value())
+
+        with pytest.raises(ReferenceError):
+            str(pa.Int16Value())
+
+        with pytest.raises(ReferenceError):
+            repr(pa.StringValue())
+
     def test_bool(self):
         arr = pa.array([True, None, False, None])
 

From cb5f2b9538e2ca11f65bdbe96451e34259899e26 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 27 Jun 2017 10:48:37 +0200
Subject: [PATCH 0755/1644] ARROW-1157: C++/Python: Decimal templates are not
 correctly exported on OSX

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #787 from xhochy/ARROW-1157 and squashes the following commits:

1eec942 [Uwe L. Korn] ARROW-1157: C++/Python: Decimal templates are not correctly exported on OSX
---
 ci/travis_script_python.sh      | 2 +-
 cpp/src/arrow/util/visibility.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 85855276abec4..ac64c548d8225 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -52,7 +52,7 @@ build_parquet_cpp() {
       -DPARQUET_BUILD_BENCHMARKS=off \
       -DPARQUET_BUILD_EXECUTABLES=off \
       -DPARQUET_ZLIB_VENDORED=off \
-      -DPARQUET_BUILD_TESTS=off \
+      -DPARQUET_BUILD_TESTS=on \
       ..
 
   make -j${CPU_COUNT}
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
index e84cc45aadf01..0018ce0989ad6 100644
--- a/cpp/src/arrow/util/visibility.h
+++ b/cpp/src/arrow/util/visibility.h
@@ -46,7 +46,7 @@
 
 // This is a complicated topic, some reading on it:
 // http://www.codesynthesis.com/~boris/blog/2010/01/18/dll-export-cxx-templates/
-#if defined(_MSC_VER)
+#if defined(_MSC_VER) || defined(__clang__)
 #define ARROW_TEMPLATE_EXPORT ARROW_EXPORT
 #else
 #define ARROW_TEMPLATE_EXPORT

From b06522870cb212989dc619b3c50e080b05772bce Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 27 Jun 2017 08:13:59 -0400
Subject: [PATCH 0756/1644] ARROW-1154: [C++] Import miscellaneous
 computational utility code from parquet-cpp

I will make a corresponding PR to parquet-cpp to ensure that this code migration is complete enough.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #785 from wesm/ARROW-1154 and squashes the following commits:

08b54c98 [Wes McKinney] Fix variety of compiler warnings
ddc7354b [Wes McKinney] Fixes to get PARQUET-1045 working
f5cd0259 [Wes McKinney] Import miscellaneous computational utility code from parquet-cpp
---
 LICENSE.txt                             |   11 +
 cpp/CMakeLists.txt                      |    5 +
 cpp/src/arrow/builder-benchmark.cc      |    8 +-
 cpp/src/arrow/compare.cc                |    4 +-
 cpp/src/arrow/ipc/reader.cc             |    4 +-
 cpp/src/arrow/status.h                  |   16 +-
 cpp/src/arrow/util/CMakeLists.txt       |    8 +
 cpp/src/arrow/util/bit-stream-utils.h   |  397 +++
 cpp/src/arrow/util/bit-util-test.cc     |  166 +-
 cpp/src/arrow/util/bit-util.h           |  329 ++-
 cpp/src/arrow/util/bpacking.h           | 3342 +++++++++++++++++++++++
 cpp/src/arrow/util/compiler-util.h      |   59 +
 cpp/src/arrow/util/cpu-info.cc          |  206 ++
 cpp/src/arrow/util/cpu-info.h           |   92 +
 cpp/src/arrow/util/hash-util.h          |  258 ++
 cpp/src/arrow/util/logging.h            |   12 +-
 cpp/src/arrow/util/rle-encoding-test.cc |  460 ++++
 cpp/src/arrow/util/rle-encoding.h       |  598 ++++
 cpp/src/arrow/util/sse-util.h           |  237 ++
 19 files changed, 6191 insertions(+), 21 deletions(-)
 create mode 100644 cpp/src/arrow/util/bit-stream-utils.h
 create mode 100644 cpp/src/arrow/util/bpacking.h
 create mode 100644 cpp/src/arrow/util/compiler-util.h
 create mode 100644 cpp/src/arrow/util/cpu-info.cc
 create mode 100644 cpp/src/arrow/util/cpu-info.h
 create mode 100644 cpp/src/arrow/util/hash-util.h
 create mode 100644 cpp/src/arrow/util/rle-encoding-test.cc
 create mode 100644 cpp/src/arrow/util/rle-encoding.h
 create mode 100644 cpp/src/arrow/util/sse-util.h

diff --git a/LICENSE.txt b/LICENSE.txt
index 95c506f22bb6f..55823cb4374bd 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -330,3 +330,14 @@ Apache 2.0 License or the under the 3-clause BSD license:
   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from Daniel Lemire's FrameOfReference project.
+
+https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
+
+Copyright: 2013 Daniel Lemire
+Home page: http://lemire.me/en/
+Project page: https://github.com/lemire/FrameOfReference
+License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index ca3410197343c..1d1ffbea652df 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -137,6 +137,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the plasma object store along with Arrow"
     OFF)
 
+  option(ARROW_USE_SSE
+    "Build with SSE4 optimizations"
+    OFF)
+
   option(ARROW_ZLIB_VENDORED
     "Build our own zlib (some libz.a aren't configured for static linking)"
     ON)
@@ -650,6 +654,7 @@ set(ARROW_SRCS
 
   src/arrow/util/bit-util.cc
   src/arrow/util/compression.cc
+  src/arrow/util/cpu-info.cc
   src/arrow/util/decimal.cc
   src/arrow/util/key_value_metadata.cc
 )
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index 3c49c6375448f..bdfba8b6c997b 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -25,10 +25,10 @@ namespace arrow {
 
 constexpr int64_t kFinalSize = 256;
 
-#define ABORT_NOT_OK(s)                                 \
-  do {                                                  \
-    ::arrow::Status _s = (s);                           \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) { exit(-1); }    \
+#define ABORT_NOT_OK(s)                              \
+  do {                                               \
+    ::arrow::Status _s = (s);                        \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { exit(-1); } \
   } while (0);
 
 static void BM_BuildPrimitiveArrayNoNulls(
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index c2f4f845baf34..390a406c19742 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -323,8 +323,8 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       : RangeEqualsVisitor(right, 0, right.length(), 0) {}
 
   Status Visit(const NullArray& left) {
-      result_ = true;
-      return Status::OK();
+    result_ = true;
+    return Status::OK();
   }
 
   Status Visit(const BooleanArray& left) {
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 7fef847c9f213..cb747b681f799 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -188,8 +188,8 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
     return ReadSchema();
   }
 
-  Status ReadNextMessage(Message::Type expected_type, bool allow_null,
-      std::shared_ptr<Message>* message) {
+  Status ReadNextMessage(
+      Message::Type expected_type, bool allow_null, std::shared_ptr<Message>* message) {
     RETURN_NOT_OK(ReadMessage(stream_.get(), message));
 
     if (!(*message) && !allow_null) {
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 9a75a58dd91ec..bfb945c498e64 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -23,10 +23,10 @@
 #include "arrow/util/visibility.h"
 
 // Return the given status if it is not OK.
-#define ARROW_RETURN_NOT_OK(s)                          \
-  do {                                                  \
-    ::arrow::Status _s = (s);                           \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; }   \
+#define ARROW_RETURN_NOT_OK(s)                        \
+  do {                                                \
+    ::arrow::Status _s = (s);                         \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; } \
   } while (0);
 
 // If 'to_call' returns a bad status, CHECK immediately with a logged message
@@ -43,10 +43,10 @@
 
 namespace arrow {
 
-#define RETURN_NOT_OK(s)                                \
-  do {                                                  \
-    Status _s = (s);                                    \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; }   \
+#define RETURN_NOT_OK(s)                              \
+  do {                                                \
+    Status _s = (s);                                  \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; } \
   } while (0);
 
 #define RETURN_NOT_OK_ELSE(s, else_) \
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 1abcce49db6bf..bc1eeb2d98167 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -22,11 +22,18 @@
 # Headers: top level
 install(FILES
   bit-util.h
+  bit-stream-utils.h
+  bpacking.h
+  compiler-util.h
   compression.h
+  cpu-info.h
   key_value_metadata.h
+  hash-util.h
   logging.h
   macros.h
   random.h
+  rle-encoding.h
+  sse-util.h
   stl.h
   visibility.h
   DESTINATION include/arrow/util)
@@ -56,4 +63,5 @@ ADD_ARROW_TEST(bit-util-test)
 ADD_ARROW_TEST(compression-test)
 ADD_ARROW_TEST(decimal-test)
 ADD_ARROW_TEST(key-value-metadata-test)
+ADD_ARROW_TEST(rle-encoding-test)
 ADD_ARROW_TEST(stl-util-test)
diff --git a/cpp/src/arrow/util/bit-stream-utils.h b/cpp/src/arrow/util/bit-stream-utils.h
new file mode 100644
index 0000000000000..537fdc3045ca5
--- /dev/null
+++ b/cpp/src/arrow/util/bit-stream-utils.h
@@ -0,0 +1,397 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// From Apache Impala (incubating) as of 2016-01-29
+
+#ifndef ARROW_UTIL_BIT_STREAM_UTILS_H
+#define ARROW_UTIL_BIT_STREAM_UTILS_H
+
+#include <algorithm>
+#include <cstdint>
+#include <string.h>
+
+#include "arrow/util/bit-util.h"
+#include "arrow/util/bpacking.h"
+#include "arrow/util/compiler-util.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+/// Utility class to write bit/byte streams.  This class can write data to either be
+/// bit packed or byte aligned (and a single stream that has a mix of both).
+/// This class does not allocate memory.
+class BitWriter {
+ public:
+  /// buffer: buffer to write bits to.  Buffer should be preallocated with
+  /// 'buffer_len' bytes.
+  BitWriter(uint8_t* buffer, int buffer_len) : buffer_(buffer), max_bytes_(buffer_len) {
+    Clear();
+  }
+
+  void Clear() {
+    buffered_values_ = 0;
+    byte_offset_ = 0;
+    bit_offset_ = 0;
+  }
+
+  /// The number of current bytes written, including the current byte (i.e. may include a
+  /// fraction of a byte). Includes buffered values.
+  int bytes_written() const {
+    return byte_offset_ + static_cast<int>(BitUtil::Ceil(bit_offset_, 8));
+  }
+  uint8_t* buffer() const { return buffer_; }
+  int buffer_len() const { return max_bytes_; }
+
+  /// Writes a value to buffered_values_, flushing to buffer_ if necessary.  This is bit
+  /// packed.  Returns false if there was not enough space. num_bits must be <= 32.
+  bool PutValue(uint64_t v, int num_bits);
+
+  /// Writes v to the next aligned byte using num_bytes. If T is larger than
+  /// num_bytes, the extra high-order bytes will be ignored. Returns false if
+  /// there was not enough space.
+  template <typename T>
+  bool PutAligned(T v, int num_bytes);
+
+  /// Write a Vlq encoded int to the buffer.  Returns false if there was not enough
+  /// room.  The value is written byte aligned.
+  /// For more details on vlq:
+  /// en.wikipedia.org/wiki/Variable-length_quantity
+  bool PutVlqInt(uint32_t v);
+
+  // Writes an int zigzag encoded.
+  bool PutZigZagVlqInt(int32_t v);
+
+  /// Get a pointer to the next aligned byte and advance the underlying buffer
+  /// by num_bytes.
+  /// Returns NULL if there was not enough space.
+  uint8_t* GetNextBytePtr(int num_bytes = 1);
+
+  /// Flushes all buffered values to the buffer. Call this when done writing to
+  /// the buffer.  If 'align' is true, buffered_values_ is reset and any future
+  /// writes will be written to the next byte boundary.
+  void Flush(bool align = false);
+
+ private:
+  uint8_t* buffer_;
+  int max_bytes_;
+
+  /// Bit-packed values are initially written to this variable before being memcpy'd to
+  /// buffer_. This is faster than writing values byte by byte directly to buffer_.
+  uint64_t buffered_values_;
+
+  int byte_offset_;  // Offset in buffer_
+  int bit_offset_;   // Offset in buffered_values_
+};
+
+/// Utility class to read bit/byte stream.  This class can read bits or bytes
+/// that are either byte aligned or not.  It also has utilities to read multiple
+/// bytes in one read (e.g. encoded int).
+class BitReader {
+ public:
+  /// 'buffer' is the buffer to read from.  The buffer's length is 'buffer_len'.
+  BitReader(const uint8_t* buffer, int buffer_len)
+      : buffer_(buffer), max_bytes_(buffer_len), byte_offset_(0), bit_offset_(0) {
+    int num_bytes = std::min(8, max_bytes_ - byte_offset_);
+    memcpy(&buffered_values_, buffer_ + byte_offset_, num_bytes);
+  }
+
+  BitReader() : buffer_(NULL), max_bytes_(0) {}
+
+  void Reset(const uint8_t* buffer, int buffer_len) {
+    buffer_ = buffer;
+    max_bytes_ = buffer_len;
+    byte_offset_ = 0;
+    bit_offset_ = 0;
+    int num_bytes = std::min(8, max_bytes_ - byte_offset_);
+    memcpy(&buffered_values_, buffer_ + byte_offset_, num_bytes);
+  }
+
+  /// Gets the next value from the buffer.  Returns true if 'v' could be read or false if
+  /// there are not enough bytes left. num_bits must be <= 32.
+  template <typename T>
+  bool GetValue(int num_bits, T* v);
+
+  /// Get a number of values from the buffer. Return the number of values actually read.
+  template <typename T>
+  int GetBatch(int num_bits, T* v, int batch_size);
+
+  /// Reads a 'num_bytes'-sized value from the buffer and stores it in 'v'. T
+  /// needs to be a little-endian native type and big enough to store
+  /// 'num_bytes'. The value is assumed to be byte-aligned so the stream will
+  /// be advanced to the start of the next byte before 'v' is read. Returns
+  /// false if there are not enough bytes left.
+  template <typename T>
+  bool GetAligned(int num_bytes, T* v);
+
+  /// Reads a vlq encoded int from the stream.  The encoded int must start at
+  /// the beginning of a byte. Return false if there were not enough bytes in
+  /// the buffer.
+  bool GetVlqInt(int32_t* v);
+
+  // Reads a zigzag encoded int `into` v.
+  bool GetZigZagVlqInt(int32_t* v);
+
+  /// Returns the number of bytes left in the stream, not including the current
+  /// byte (i.e., there may be an additional fraction of a byte).
+  int bytes_left() {
+    return max_bytes_ - (byte_offset_ + static_cast<int>(BitUtil::Ceil(bit_offset_, 8)));
+  }
+
+  /// Maximum byte length of a vlq encoded int
+  static const int MAX_VLQ_BYTE_LEN = 5;
+
+ private:
+  const uint8_t* buffer_;
+  int max_bytes_;
+
+  /// Bytes are memcpy'd from buffer_ and values are read from this variable. This is
+  /// faster than reading values byte by byte directly from buffer_.
+  uint64_t buffered_values_;
+
+  int byte_offset_;  // Offset in buffer_
+  int bit_offset_;   // Offset in buffered_values_
+};
+
+inline bool BitWriter::PutValue(uint64_t v, int num_bits) {
+  // TODO: revisit this limit if necessary (can be raised to 64 by fixing some edge cases)
+  DCHECK_LE(num_bits, 32);
+  DCHECK_EQ(v >> num_bits, 0) << "v = " << v << ", num_bits = " << num_bits;
+
+  if (UNLIKELY(byte_offset_ * 8 + bit_offset_ + num_bits > max_bytes_ * 8)) return false;
+
+  buffered_values_ |= v << bit_offset_;
+  bit_offset_ += num_bits;
+
+  if (UNLIKELY(bit_offset_ >= 64)) {
+    // Flush buffered_values_ and write out bits of v that did not fit
+    memcpy(buffer_ + byte_offset_, &buffered_values_, 8);
+    buffered_values_ = 0;
+    byte_offset_ += 8;
+    bit_offset_ -= 64;
+    buffered_values_ = v >> (num_bits - bit_offset_);
+  }
+  DCHECK_LT(bit_offset_, 64);
+  return true;
+}
+
+inline void BitWriter::Flush(bool align) {
+  int num_bytes = static_cast<int>(BitUtil::Ceil(bit_offset_, 8));
+  DCHECK_LE(byte_offset_ + num_bytes, max_bytes_);
+  memcpy(buffer_ + byte_offset_, &buffered_values_, num_bytes);
+
+  if (align) {
+    buffered_values_ = 0;
+    byte_offset_ += num_bytes;
+    bit_offset_ = 0;
+  }
+}
+
+inline uint8_t* BitWriter::GetNextBytePtr(int num_bytes) {
+  Flush(/* align */ true);
+  DCHECK_LE(byte_offset_, max_bytes_);
+  if (byte_offset_ + num_bytes > max_bytes_) return NULL;
+  uint8_t* ptr = buffer_ + byte_offset_;
+  byte_offset_ += num_bytes;
+  return ptr;
+}
+
+template <typename T>
+inline bool BitWriter::PutAligned(T val, int num_bytes) {
+  uint8_t* ptr = GetNextBytePtr(num_bytes);
+  if (ptr == NULL) return false;
+  memcpy(ptr, &val, num_bytes);
+  return true;
+}
+
+inline bool BitWriter::PutVlqInt(uint32_t v) {
+  bool result = true;
+  while ((v & 0xFFFFFF80) != 0L) {
+    result &= PutAligned<uint8_t>(static_cast<uint8_t>((v & 0x7F) | 0x80), 1);
+    v >>= 7;
+  }
+  result &= PutAligned<uint8_t>(static_cast<uint8_t>(v & 0x7F), 1);
+  return result;
+}
+
+template <typename T>
+inline void GetValue_(int num_bits, T* v, int max_bytes, const uint8_t* buffer,
+    int* bit_offset, int* byte_offset, uint64_t* buffered_values) {
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4800)
+#endif
+  *v = static_cast<T>(
+      BitUtil::TrailingBits(*buffered_values, *bit_offset + num_bits) >> *bit_offset);
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+  *bit_offset += num_bits;
+  if (*bit_offset >= 64) {
+    *byte_offset += 8;
+    *bit_offset -= 64;
+
+    int bytes_remaining = max_bytes - *byte_offset;
+    if (LIKELY(bytes_remaining >= 8)) {
+      memcpy(buffered_values, buffer + *byte_offset, 8);
+    } else {
+      memcpy(buffered_values, buffer + *byte_offset, bytes_remaining);
+    }
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4800 4805)
+#endif
+    // Read bits of v that crossed into new buffered_values_
+    *v = *v | static_cast<T>(BitUtil::TrailingBits(*buffered_values, *bit_offset)
+                             << (num_bits - *bit_offset));
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+    DCHECK_LE(*bit_offset, 64);
+  }
+}
+
+template <typename T>
+inline bool BitReader::GetValue(int num_bits, T* v) {
+  return GetBatch(num_bits, v, 1) == 1;
+}
+
+template <typename T>
+inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
+  DCHECK(buffer_ != NULL);
+  // TODO: revisit this limit if necessary
+  DCHECK_LE(num_bits, 32);
+  DCHECK_LE(num_bits, static_cast<int>(sizeof(T) * 8));
+
+  int bit_offset = bit_offset_;
+  int byte_offset = byte_offset_;
+  uint64_t buffered_values = buffered_values_;
+  int max_bytes = max_bytes_;
+  const uint8_t* buffer = buffer_;
+
+  uint64_t needed_bits = num_bits * batch_size;
+  uint64_t remaining_bits = (max_bytes - byte_offset) * 8 - bit_offset;
+  if (remaining_bits < needed_bits) {
+    batch_size = static_cast<int>(remaining_bits) / num_bits;
+  }
+
+  int i = 0;
+  if (UNLIKELY(bit_offset != 0)) {
+    for (; i < batch_size && bit_offset != 0; ++i) {
+      GetValue_(num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset,
+          &buffered_values);
+    }
+  }
+
+  if (sizeof(T) == 4) {
+    int num_unpacked = unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
+        reinterpret_cast<uint32_t*>(v + i), batch_size - i, num_bits);
+    i += num_unpacked;
+    byte_offset += num_unpacked * num_bits / 8;
+  } else {
+    const int buffer_size = 1024;
+    uint32_t unpack_buffer[buffer_size];
+    while (i < batch_size) {
+      int unpack_size = std::min(buffer_size, batch_size - i);
+      int num_unpacked = unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
+          unpack_buffer, unpack_size, num_bits);
+      if (num_unpacked == 0) { break; }
+      for (int k = 0; k < num_unpacked; ++k) {
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4800)
+#endif
+        v[i + k] = static_cast<T>(unpack_buffer[k]);
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+      }
+      i += num_unpacked;
+      byte_offset += num_unpacked * num_bits / 8;
+    }
+  }
+
+  int bytes_remaining = max_bytes - byte_offset;
+  if (bytes_remaining >= 8) {
+    memcpy(&buffered_values, buffer + byte_offset, 8);
+  } else {
+    memcpy(&buffered_values, buffer + byte_offset, bytes_remaining);
+  }
+
+  for (; i < batch_size; ++i) {
+    GetValue_(
+        num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset, &buffered_values);
+  }
+
+  bit_offset_ = bit_offset;
+  byte_offset_ = byte_offset;
+  buffered_values_ = buffered_values;
+
+  return batch_size;
+}
+
+template <typename T>
+inline bool BitReader::GetAligned(int num_bytes, T* v) {
+  DCHECK_LE(num_bytes, static_cast<int>(sizeof(T)));
+  int bytes_read = static_cast<int>(BitUtil::Ceil(bit_offset_, 8));
+  if (UNLIKELY(byte_offset_ + bytes_read + num_bytes > max_bytes_)) return false;
+
+  // Advance byte_offset to next unread byte and read num_bytes
+  byte_offset_ += bytes_read;
+  memcpy(v, buffer_ + byte_offset_, num_bytes);
+  byte_offset_ += num_bytes;
+
+  // Reset buffered_values_
+  bit_offset_ = 0;
+  int bytes_remaining = max_bytes_ - byte_offset_;
+  if (LIKELY(bytes_remaining >= 8)) {
+    memcpy(&buffered_values_, buffer_ + byte_offset_, 8);
+  } else {
+    memcpy(&buffered_values_, buffer_ + byte_offset_, bytes_remaining);
+  }
+  return true;
+}
+
+inline bool BitReader::GetVlqInt(int32_t* v) {
+  *v = 0;
+  int shift = 0;
+  int num_bytes = 0;
+  uint8_t byte = 0;
+  do {
+    if (!GetAligned<uint8_t>(1, &byte)) return false;
+    *v |= (byte & 0x7F) << shift;
+    shift += 7;
+    DCHECK_LE(++num_bytes, MAX_VLQ_BYTE_LEN);
+  } while ((byte & 0x80) != 0);
+  return true;
+}
+
+inline bool BitWriter::PutZigZagVlqInt(int32_t v) {
+  uint32_t u = (v << 1) ^ (v >> 31);
+  return PutVlqInt(u);
+}
+
+inline bool BitReader::GetZigZagVlqInt(int32_t* v) {
+  int32_t u_signed;
+  if (!GetVlqInt(&u_signed)) return false;
+  uint32_t u = static_cast<uint32_t>(u_signed);
+  *reinterpret_cast<uint32_t*>(v) = (u >> 1) ^ -(static_cast<int32_t>(u & 1));
+  return true;
+}
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_BIT_STREAM_UTILS_H
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index cb2fd1ab276ad..cd945585ba210 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -15,18 +15,29 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/bit-util.h"
-
 #include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <limits>
 #include <vector>
 
 #include "gtest/gtest.h"
 
+#include <boost/utility.hpp>
+
 #include "arrow/buffer.h"
 #include "arrow/test-util.h"
+#include "arrow/util/bit-stream-utils.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/cpu-info.h"
 
 namespace arrow {
 
+static void EnsureCpuInfoInitialized() {
+  if (!CpuInfo::initialized()) { CpuInfo::Init(); }
+}
+
 TEST(BitUtilTests, TestIsMultipleOf64) {
   using BitUtil::IsMultipleOf64;
   EXPECT_TRUE(IsMultipleOf64(64));
@@ -109,4 +120,155 @@ TEST(BitUtilTests, TestCopyBitmap) {
   }
 }
 
+TEST(BitUtil, Ceil) {
+  EXPECT_EQ(BitUtil::Ceil(0, 1), 0);
+  EXPECT_EQ(BitUtil::Ceil(1, 1), 1);
+  EXPECT_EQ(BitUtil::Ceil(1, 2), 1);
+  EXPECT_EQ(BitUtil::Ceil(1, 8), 1);
+  EXPECT_EQ(BitUtil::Ceil(7, 8), 1);
+  EXPECT_EQ(BitUtil::Ceil(8, 8), 1);
+  EXPECT_EQ(BitUtil::Ceil(9, 8), 2);
+  EXPECT_EQ(BitUtil::Ceil(9, 9), 1);
+  EXPECT_EQ(BitUtil::Ceil(10000000000, 10), 1000000000);
+  EXPECT_EQ(BitUtil::Ceil(10, 10000000000), 1);
+  EXPECT_EQ(BitUtil::Ceil(100000000000, 10000000000), 10);
+}
+
+TEST(BitUtil, RoundUp) {
+  EXPECT_EQ(BitUtil::RoundUp(0, 1), 0);
+  EXPECT_EQ(BitUtil::RoundUp(1, 1), 1);
+  EXPECT_EQ(BitUtil::RoundUp(1, 2), 2);
+  EXPECT_EQ(BitUtil::RoundUp(6, 2), 6);
+  EXPECT_EQ(BitUtil::RoundUp(7, 3), 9);
+  EXPECT_EQ(BitUtil::RoundUp(9, 9), 9);
+  EXPECT_EQ(BitUtil::RoundUp(10000000001, 10), 10000000010);
+  EXPECT_EQ(BitUtil::RoundUp(10, 10000000000), 10000000000);
+  EXPECT_EQ(BitUtil::RoundUp(100000000000, 10000000000), 100000000000);
+}
+
+TEST(BitUtil, RoundDown) {
+  EXPECT_EQ(BitUtil::RoundDown(0, 1), 0);
+  EXPECT_EQ(BitUtil::RoundDown(1, 1), 1);
+  EXPECT_EQ(BitUtil::RoundDown(1, 2), 0);
+  EXPECT_EQ(BitUtil::RoundDown(6, 2), 6);
+  EXPECT_EQ(BitUtil::RoundDown(7, 3), 6);
+  EXPECT_EQ(BitUtil::RoundDown(9, 9), 9);
+  EXPECT_EQ(BitUtil::RoundDown(10000000001, 10), 10000000000);
+  EXPECT_EQ(BitUtil::RoundDown(10, 10000000000), 0);
+  EXPECT_EQ(BitUtil::RoundDown(100000000000, 10000000000), 100000000000);
+}
+
+TEST(BitUtil, Popcount) {
+  EnsureCpuInfoInitialized();
+
+  EXPECT_EQ(BitUtil::Popcount(BOOST_BINARY(0 1 0 1 0 1 0 1)), 4);
+  EXPECT_EQ(BitUtil::PopcountNoHw(BOOST_BINARY(0 1 0 1 0 1 0 1)), 4);
+  EXPECT_EQ(BitUtil::Popcount(BOOST_BINARY(1 1 1 1 0 1 0 1)), 6);
+  EXPECT_EQ(BitUtil::PopcountNoHw(BOOST_BINARY(1 1 1 1 0 1 0 1)), 6);
+  EXPECT_EQ(BitUtil::Popcount(BOOST_BINARY(1 1 1 1 1 1 1 1)), 8);
+  EXPECT_EQ(BitUtil::PopcountNoHw(BOOST_BINARY(1 1 1 1 1 1 1 1)), 8);
+  EXPECT_EQ(BitUtil::Popcount(0), 0);
+  EXPECT_EQ(BitUtil::PopcountNoHw(0), 0);
+}
+
+TEST(BitUtil, TrailingBits) {
+  EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 0), 0);
+  EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 1), 1);
+  EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 64),
+      BOOST_BINARY(1 1 1 1 1 1 1 1));
+  EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 100),
+      BOOST_BINARY(1 1 1 1 1 1 1 1));
+  EXPECT_EQ(BitUtil::TrailingBits(0, 1), 0);
+  EXPECT_EQ(BitUtil::TrailingBits(0, 64), 0);
+  EXPECT_EQ(BitUtil::TrailingBits(1LL << 63, 0), 0);
+  EXPECT_EQ(BitUtil::TrailingBits(1LL << 63, 63), 0);
+  EXPECT_EQ(BitUtil::TrailingBits(1LL << 63, 64), 1LL << 63);
+}
+
+TEST(BitUtil, ByteSwap) {
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<uint32_t>(0)), 0);
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<uint32_t>(0x11223344)), 0x44332211);
+
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<int32_t>(0)), 0);
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<int32_t>(0x11223344)), 0x44332211);
+
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<uint64_t>(0)), 0);
+  EXPECT_EQ(
+      BitUtil::ByteSwap(static_cast<uint64_t>(0x1122334455667788)), 0x8877665544332211);
+
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<int64_t>(0)), 0);
+  EXPECT_EQ(
+      BitUtil::ByteSwap(static_cast<int64_t>(0x1122334455667788)), 0x8877665544332211);
+
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<int16_t>(0)), 0);
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<int16_t>(0x1122)), 0x2211);
+
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<uint16_t>(0)), 0);
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<uint16_t>(0x1122)), 0x2211);
+}
+
+TEST(BitUtil, Log2) {
+  EXPECT_EQ(BitUtil::Log2(1), 0);
+  EXPECT_EQ(BitUtil::Log2(2), 1);
+  EXPECT_EQ(BitUtil::Log2(3), 2);
+  EXPECT_EQ(BitUtil::Log2(4), 2);
+  EXPECT_EQ(BitUtil::Log2(5), 3);
+  EXPECT_EQ(BitUtil::Log2(INT_MAX), 31);
+  EXPECT_EQ(BitUtil::Log2(UINT_MAX), 32);
+  EXPECT_EQ(BitUtil::Log2(ULLONG_MAX), 64);
+}
+
+TEST(BitUtil, RoundUpToPowerOf2) {
+  EXPECT_EQ(BitUtil::RoundUpToPowerOf2(7, 8), 8);
+  EXPECT_EQ(BitUtil::RoundUpToPowerOf2(8, 8), 8);
+  EXPECT_EQ(BitUtil::RoundUpToPowerOf2(9, 8), 16);
+}
+
+TEST(BitUtil, RoundDownToPowerOf2) {
+  EXPECT_EQ(BitUtil::RoundDownToPowerOf2(7, 8), 0);
+  EXPECT_EQ(BitUtil::RoundDownToPowerOf2(8, 8), 8);
+  EXPECT_EQ(BitUtil::RoundDownToPowerOf2(9, 8), 8);
+}
+
+TEST(BitUtil, RoundUpDown) {
+  EXPECT_EQ(BitUtil::RoundUpNumBytes(7), 1);
+  EXPECT_EQ(BitUtil::RoundUpNumBytes(8), 1);
+  EXPECT_EQ(BitUtil::RoundUpNumBytes(9), 2);
+  EXPECT_EQ(BitUtil::RoundDownNumBytes(7), 0);
+  EXPECT_EQ(BitUtil::RoundDownNumBytes(8), 1);
+  EXPECT_EQ(BitUtil::RoundDownNumBytes(9), 1);
+
+  EXPECT_EQ(BitUtil::RoundUpNumi32(31), 1);
+  EXPECT_EQ(BitUtil::RoundUpNumi32(32), 1);
+  EXPECT_EQ(BitUtil::RoundUpNumi32(33), 2);
+  EXPECT_EQ(BitUtil::RoundDownNumi32(31), 0);
+  EXPECT_EQ(BitUtil::RoundDownNumi32(32), 1);
+  EXPECT_EQ(BitUtil::RoundDownNumi32(33), 1);
+
+  EXPECT_EQ(BitUtil::RoundUpNumi64(63), 1);
+  EXPECT_EQ(BitUtil::RoundUpNumi64(64), 1);
+  EXPECT_EQ(BitUtil::RoundUpNumi64(65), 2);
+  EXPECT_EQ(BitUtil::RoundDownNumi64(63), 0);
+  EXPECT_EQ(BitUtil::RoundDownNumi64(64), 1);
+  EXPECT_EQ(BitUtil::RoundDownNumi64(65), 1);
+}
+
+void TestZigZag(int32_t v) {
+  uint8_t buffer[BitReader::MAX_VLQ_BYTE_LEN];
+  BitWriter writer(buffer, sizeof(buffer));
+  BitReader reader(buffer, sizeof(buffer));
+  writer.PutZigZagVlqInt(v);
+  int32_t result;
+  EXPECT_TRUE(reader.GetZigZagVlqInt(&result));
+  EXPECT_EQ(v, result);
+}
+
+TEST(BitStreamUtil, ZigZag) {
+  TestZigZag(0);
+  TestZigZag(1);
+  TestZigZag(-1);
+  TestZigZag(std::numeric_limits<int32_t>::max());
+  TestZigZag(-std::numeric_limits<int32_t>::max());
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 90a1c3eab9266..bba9d2dbb9d21 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -18,15 +18,77 @@
 #ifndef ARROW_UTIL_BIT_UTIL_H
 #define ARROW_UTIL_BIT_UTIL_H
 
+#if defined(__APPLE__)
+#include <machine/endian.h>
+#elif defined(_WIN32)
+#define __LITTLE_ENDIAN 1
+#else
+#include <endian.h>
+#endif
+
+#if defined(_MSC_VER)
+#define ARROW_BYTE_SWAP64 _byteswap_uint64
+#define ARROW_BYTE_SWAP32 _byteswap_ulong
+#else
+#define ARROW_BYTE_SWAP64 __builtin_bswap64
+#define ARROW_BYTE_SWAP32 __builtin_bswap32
+#endif
+
 #include <cstdint>
 #include <limits>
 #include <memory>
 #include <vector>
 
+#include "arrow/util/compiler-util.h"
 #include "arrow/util/visibility.h"
 
+#ifdef ARROW_USE_SSE
+#include "arrow/util/cpu-info.h"
+#include "arrow/util/sse-util.h"
+#endif
+
 namespace arrow {
 
+#define INIT_BITSET(valid_bits_vector, valid_bits_index)        \
+  int byte_offset_##valid_bits_vector = (valid_bits_index) / 8; \
+  int bit_offset_##valid_bits_vector = (valid_bits_index) % 8;  \
+  uint8_t bitset_##valid_bits_vector = valid_bits_vector[byte_offset_##valid_bits_vector];
+
+#define READ_NEXT_BITSET(valid_bits_vector)                                          \
+  bit_offset_##valid_bits_vector++;                                                  \
+  if (bit_offset_##valid_bits_vector == 8) {                                         \
+    bit_offset_##valid_bits_vector = 0;                                              \
+    byte_offset_##valid_bits_vector++;                                               \
+    bitset_##valid_bits_vector = valid_bits_vector[byte_offset_##valid_bits_vector]; \
+  }
+
+// TODO(wesm): The source from Impala was depending on boost::make_unsigned
+//
+// We add a partial stub implementation here
+
+template <typename T>
+struct make_unsigned {};
+
+template <>
+struct make_unsigned<int8_t> {
+  typedef uint8_t type;
+};
+
+template <>
+struct make_unsigned<int16_t> {
+  typedef uint16_t type;
+};
+
+template <>
+struct make_unsigned<int32_t> {
+  typedef uint32_t type;
+};
+
+template <>
+struct make_unsigned<int64_t> {
+  typedef uint64_t type;
+};
+
 class Buffer;
 class MemoryPool;
 class MutableBuffer;
@@ -67,6 +129,11 @@ static inline void SetBit(uint8_t* bits, int64_t i) {
   bits[i / 8] |= kBitmask[i % 8];
 }
 
+/// Set bit if is_set is true, but cannot clear bit
+static inline void SetArrayBit(uint8_t* bits, int i, bool is_set) {
+  if (is_set) { SetBit(bits, i); }
+}
+
 static inline void SetBitTo(uint8_t* bits, int64_t i, bool bit_is_set) {
   // TODO: speed up. See https://graphics.stanford.edu/~seander/bithacks.html
   // "Conditionally set or clear bits without branching"
@@ -77,6 +144,18 @@ static inline void SetBitTo(uint8_t* bits, int64_t i, bool bit_is_set) {
   }
 }
 
+// Returns the minimum number of bits needed to represent the value of 'x'
+static inline int NumRequiredBits(uint64_t x) {
+  for (int i = 63; i >= 0; --i) {
+    if (x & (UINT64_C(1) << i)) return i + 1;
+  }
+  return 0;
+}
+
+/// Returns the smallest power of two that contains v. Taken from
+/// http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+/// TODO: Pick a better name, as it is not clear what happens when the input is
+/// already a power of two.
 static inline int64_t NextPower2(int64_t n) {
   n--;
   n |= n >> 1;
@@ -97,12 +176,66 @@ static inline bool IsMultipleOf8(int64_t n) {
   return (n & 7) == 0;
 }
 
+/// Returns the ceil of value/divisor
+static inline int64_t Ceil(int64_t value, int64_t divisor) {
+  return value / divisor + (value % divisor != 0);
+}
+
 /// Returns 'value' rounded up to the nearest multiple of 'factor'
 inline int64_t RoundUp(int64_t value, int64_t factor) {
   return (value + (factor - 1)) / factor * factor;
 }
 
-inline int64_t RoundUpToMultipleOf64(int64_t num) {
+/// Returns 'value' rounded down to the nearest multiple of 'factor'
+static inline int64_t RoundDown(int64_t value, int64_t factor) {
+  return (value / factor) * factor;
+}
+
+/// Returns 'value' rounded up to the nearest multiple of 'factor' when factor is
+/// a power of two
+static inline int RoundUpToPowerOf2(int value, int factor) {
+  // DCHECK((factor > 0) && ((factor & (factor - 1)) == 0));
+  return (value + (factor - 1)) & ~(factor - 1);
+}
+
+static inline int RoundDownToPowerOf2(int value, int factor) {
+  // DCHECK((factor > 0) && ((factor & (factor - 1)) == 0));
+  return value & ~(factor - 1);
+}
+
+/// Specialized round up and down functions for frequently used factors,
+/// like 8 (bits->bytes), 32 (bits->i32), and 64 (bits->i64).
+/// Returns the rounded up number of bytes that fit the number of bits.
+static inline uint32_t RoundUpNumBytes(uint32_t bits) {
+  return (bits + 7) >> 3;
+}
+
+/// Returns the rounded down number of bytes that fit the number of bits.
+static inline uint32_t RoundDownNumBytes(uint32_t bits) {
+  return bits >> 3;
+}
+
+/// Returns the rounded up to 32 multiple. Used for conversions of bits to i32.
+static inline uint32_t RoundUpNumi32(uint32_t bits) {
+  return (bits + 31) >> 5;
+}
+
+/// Returns the rounded up 32 multiple.
+static inline uint32_t RoundDownNumi32(uint32_t bits) {
+  return bits >> 5;
+}
+
+/// Returns the rounded up to 64 multiple. Used for conversions of bits to i64.
+static inline uint32_t RoundUpNumi64(uint32_t bits) {
+  return (bits + 63) >> 6;
+}
+
+/// Returns the rounded down to 64 multiple.
+static inline uint32_t RoundDownNumi64(uint32_t bits) {
+  return bits >> 6;
+}
+
+static inline int64_t RoundUpToMultipleOf64(int64_t num) {
   // TODO(wesm): is this definitely needed?
   // DCHECK_GE(num, 0);
   constexpr int64_t round_to = 64;
@@ -114,6 +247,200 @@ inline int64_t RoundUpToMultipleOf64(int64_t num) {
   return num;
 }
 
+/// Non hw accelerated pop count.
+/// TODO: we don't use this in any perf sensitive code paths currently.  There
+/// might be a much faster way to implement this.
+static inline int PopcountNoHw(uint64_t x) {
+  int count = 0;
+  for (; x != 0; ++count)
+    x &= x - 1;
+  return count;
+}
+
+/// Returns the number of set bits in x
+static inline int Popcount(uint64_t x) {
+#ifdef ARROW_USE_SSE
+  if (LIKELY(CpuInfo::IsSupported(CpuInfo::POPCNT))) {
+    return POPCNT_popcnt_u64(x);
+  } else {
+    return PopcountNoHw(x);
+  }
+#else
+  return PopcountNoHw(x);
+#endif
+}
+
+// Compute correct population count for various-width signed integers
+template <typename T>
+static inline int PopcountSigned(T v) {
+  // Converting to same-width unsigned then extending preserves the bit pattern.
+  return BitUtil::Popcount(static_cast<typename make_unsigned<T>::type>(v));
+}
+
+/// Returns the 'num_bits' least-significant bits of 'v'.
+static inline uint64_t TrailingBits(uint64_t v, int num_bits) {
+  if (UNLIKELY(num_bits == 0)) return 0;
+  if (UNLIKELY(num_bits >= 64)) return v;
+  int n = 64 - num_bits;
+  return (v << n) >> n;
+}
+
+/// Returns ceil(log2(x)).
+/// TODO: this could be faster if we use __builtin_clz.  Fix this if this ever shows up
+/// in a hot path.
+static inline int Log2(uint64_t x) {
+  // DCHECK_GT(x, 0);
+  if (x == 1) return 0;
+  // Compute result = ceil(log2(x))
+  //                = floor(log2(x - 1)) + 1, for x > 1
+  // by finding the position of the most significant bit (1-indexed) of x - 1
+  // (floor(log2(n)) = MSB(n) (0-indexed))
+  --x;
+  int result = 1;
+  while (x >>= 1)
+    ++result;
+  return result;
+}
+
+/// Swaps the byte order (i.e. endianess)
+static inline int64_t ByteSwap(int64_t value) {
+  return ARROW_BYTE_SWAP64(value);
+}
+static inline uint64_t ByteSwap(uint64_t value) {
+  return static_cast<uint64_t>(ARROW_BYTE_SWAP64(value));
+}
+static inline int32_t ByteSwap(int32_t value) {
+  return ARROW_BYTE_SWAP32(value);
+}
+static inline uint32_t ByteSwap(uint32_t value) {
+  return static_cast<uint32_t>(ARROW_BYTE_SWAP32(value));
+}
+static inline int16_t ByteSwap(int16_t value) {
+  constexpr int16_t m = static_cast<int16_t>(0xff);
+  return static_cast<int16_t>(((value >> 8) & m) | ((value & m) << 8));
+}
+static inline uint16_t ByteSwap(uint16_t value) {
+  return static_cast<uint16_t>(ByteSwap(static_cast<int16_t>(value)));
+}
+
+/// Write the swapped bytes into dst. Src and st cannot overlap.
+static inline void ByteSwap(void* dst, const void* src, int len) {
+  switch (len) {
+    case 1:
+      *reinterpret_cast<int8_t*>(dst) = *reinterpret_cast<const int8_t*>(src);
+      return;
+    case 2:
+      *reinterpret_cast<int16_t*>(dst) = ByteSwap(*reinterpret_cast<const int16_t*>(src));
+      return;
+    case 4:
+      *reinterpret_cast<int32_t*>(dst) = ByteSwap(*reinterpret_cast<const int32_t*>(src));
+      return;
+    case 8:
+      *reinterpret_cast<int64_t*>(dst) = ByteSwap(*reinterpret_cast<const int64_t*>(src));
+      return;
+    default:
+      break;
+  }
+
+  uint8_t* d = reinterpret_cast<uint8_t*>(dst);
+  const uint8_t* s = reinterpret_cast<const uint8_t*>(src);
+  for (int i = 0; i < len; ++i) {
+    d[i] = s[len - i - 1];
+  }
+}
+
+/// Converts to big endian format (if not already in big endian) from the
+/// machine's native endian format.
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline int64_t ToBigEndian(int64_t value) {
+  return ByteSwap(value);
+}
+static inline uint64_t ToBigEndian(uint64_t value) {
+  return ByteSwap(value);
+}
+static inline int32_t ToBigEndian(int32_t value) {
+  return ByteSwap(value);
+}
+static inline uint32_t ToBigEndian(uint32_t value) {
+  return ByteSwap(value);
+}
+static inline int16_t ToBigEndian(int16_t value) {
+  return ByteSwap(value);
+}
+static inline uint16_t ToBigEndian(uint16_t value) {
+  return ByteSwap(value);
+}
+#else
+static inline int64_t ToBigEndian(int64_t val) {
+  return val;
+}
+static inline uint64_t ToBigEndian(uint64_t val) {
+  return val;
+}
+static inline int32_t ToBigEndian(int32_t val) {
+  return val;
+}
+static inline uint32_t ToBigEndian(uint32_t val) {
+  return val;
+}
+static inline int16_t ToBigEndian(int16_t val) {
+  return val;
+}
+static inline uint16_t ToBigEndian(uint16_t val) {
+  return val;
+}
+#endif
+
+/// Converts from big endian format to the machine's native endian format.
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline int64_t FromBigEndian(int64_t value) {
+  return ByteSwap(value);
+}
+static inline uint64_t FromBigEndian(uint64_t value) {
+  return ByteSwap(value);
+}
+static inline int32_t FromBigEndian(int32_t value) {
+  return ByteSwap(value);
+}
+static inline uint32_t FromBigEndian(uint32_t value) {
+  return ByteSwap(value);
+}
+static inline int16_t FromBigEndian(int16_t value) {
+  return ByteSwap(value);
+}
+static inline uint16_t FromBigEndian(uint16_t value) {
+  return ByteSwap(value);
+}
+#else
+static inline int64_t FromBigEndian(int64_t val) {
+  return val;
+}
+static inline uint64_t FromBigEndian(uint64_t val) {
+  return val;
+}
+static inline int32_t FromBigEndian(int32_t val) {
+  return val;
+}
+static inline uint32_t FromBigEndian(uint32_t val) {
+  return val;
+}
+static inline int16_t FromBigEndian(int16_t val) {
+  return val;
+}
+static inline uint16_t FromBigEndian(uint16_t val) {
+  return val;
+}
+#endif
+
+// Logical right shift for signed integer types
+// This is needed because the C >> operator does arithmetic right shift
+// Negative shift amounts lead to undefined behavior
+template <typename T>
+static T ShiftRightLogical(T v, int shift) {
+  // Conversion to unsigned ensures most significant bits always filled with 0's
+  return static_cast<typename make_unsigned<T>::type>(v) >> shift;
+}
+
 void BytesToBits(const std::vector<uint8_t>& bytes, uint8_t* bits);
 ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
diff --git a/cpp/src/arrow/util/bpacking.h b/cpp/src/arrow/util/bpacking.h
new file mode 100644
index 0000000000000..fce5f55224cd4
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking.h
@@ -0,0 +1,3342 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// This file was modified from its original version for inclusion in parquet-cpp.
+// Original source:
+// https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
+// The original copyright notice follows.
+
+/**
+*
+* This code is released under the
+* Apache License Version 2.0 http://www.apache.org/licenses/.
+* (c) Daniel Lemire 2013
+*/
+
+#ifndef ARROW_UTIL_BPACKING_H
+#define ARROW_UTIL_BPACKING_H
+
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+inline const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) & 1;
+  out++;
+  *out = ((*in) >> 1) & 1;
+  out++;
+  *out = ((*in) >> 2) & 1;
+  out++;
+  *out = ((*in) >> 3) & 1;
+  out++;
+  *out = ((*in) >> 4) & 1;
+  out++;
+  *out = ((*in) >> 5) & 1;
+  out++;
+  *out = ((*in) >> 6) & 1;
+  out++;
+  *out = ((*in) >> 7) & 1;
+  out++;
+  *out = ((*in) >> 8) & 1;
+  out++;
+  *out = ((*in) >> 9) & 1;
+  out++;
+  *out = ((*in) >> 10) & 1;
+  out++;
+  *out = ((*in) >> 11) & 1;
+  out++;
+  *out = ((*in) >> 12) & 1;
+  out++;
+  *out = ((*in) >> 13) & 1;
+  out++;
+  *out = ((*in) >> 14) & 1;
+  out++;
+  *out = ((*in) >> 15) & 1;
+  out++;
+  *out = ((*in) >> 16) & 1;
+  out++;
+  *out = ((*in) >> 17) & 1;
+  out++;
+  *out = ((*in) >> 18) & 1;
+  out++;
+  *out = ((*in) >> 19) & 1;
+  out++;
+  *out = ((*in) >> 20) & 1;
+  out++;
+  *out = ((*in) >> 21) & 1;
+  out++;
+  *out = ((*in) >> 22) & 1;
+  out++;
+  *out = ((*in) >> 23) & 1;
+  out++;
+  *out = ((*in) >> 24) & 1;
+  out++;
+  *out = ((*in) >> 25) & 1;
+  out++;
+  *out = ((*in) >> 26) & 1;
+  out++;
+  *out = ((*in) >> 27) & 1;
+  out++;
+  *out = ((*in) >> 28) & 1;
+  out++;
+  *out = ((*in) >> 29) & 1;
+  out++;
+  *out = ((*in) >> 30) & 1;
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack2_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 2);
+  out++;
+  *out = ((*in) >> 4) % (1U << 2);
+  out++;
+  *out = ((*in) >> 6) % (1U << 2);
+  out++;
+  *out = ((*in) >> 8) % (1U << 2);
+  out++;
+  *out = ((*in) >> 10) % (1U << 2);
+  out++;
+  *out = ((*in) >> 12) % (1U << 2);
+  out++;
+  *out = ((*in) >> 14) % (1U << 2);
+  out++;
+  *out = ((*in) >> 16) % (1U << 2);
+  out++;
+  *out = ((*in) >> 18) % (1U << 2);
+  out++;
+  *out = ((*in) >> 20) % (1U << 2);
+  out++;
+  *out = ((*in) >> 22) % (1U << 2);
+  out++;
+  *out = ((*in) >> 24) % (1U << 2);
+  out++;
+  *out = ((*in) >> 26) % (1U << 2);
+  out++;
+  *out = ((*in) >> 28) % (1U << 2);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 2);
+  out++;
+  *out = ((*in) >> 4) % (1U << 2);
+  out++;
+  *out = ((*in) >> 6) % (1U << 2);
+  out++;
+  *out = ((*in) >> 8) % (1U << 2);
+  out++;
+  *out = ((*in) >> 10) % (1U << 2);
+  out++;
+  *out = ((*in) >> 12) % (1U << 2);
+  out++;
+  *out = ((*in) >> 14) % (1U << 2);
+  out++;
+  *out = ((*in) >> 16) % (1U << 2);
+  out++;
+  *out = ((*in) >> 18) % (1U << 2);
+  out++;
+  *out = ((*in) >> 20) % (1U << 2);
+  out++;
+  *out = ((*in) >> 22) % (1U << 2);
+  out++;
+  *out = ((*in) >> 24) % (1U << 2);
+  out++;
+  *out = ((*in) >> 26) % (1U << 2);
+  out++;
+  *out = ((*in) >> 28) % (1U << 2);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack3_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 3);
+  out++;
+  *out = ((*in) >> 6) % (1U << 3);
+  out++;
+  *out = ((*in) >> 9) % (1U << 3);
+  out++;
+  *out = ((*in) >> 12) % (1U << 3);
+  out++;
+  *out = ((*in) >> 15) % (1U << 3);
+  out++;
+  *out = ((*in) >> 18) % (1U << 3);
+  out++;
+  *out = ((*in) >> 21) % (1U << 3);
+  out++;
+  *out = ((*in) >> 24) % (1U << 3);
+  out++;
+  *out = ((*in) >> 27) % (1U << 3);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (3 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 3);
+  out++;
+  *out = ((*in) >> 4) % (1U << 3);
+  out++;
+  *out = ((*in) >> 7) % (1U << 3);
+  out++;
+  *out = ((*in) >> 10) % (1U << 3);
+  out++;
+  *out = ((*in) >> 13) % (1U << 3);
+  out++;
+  *out = ((*in) >> 16) % (1U << 3);
+  out++;
+  *out = ((*in) >> 19) % (1U << 3);
+  out++;
+  *out = ((*in) >> 22) % (1U << 3);
+  out++;
+  *out = ((*in) >> 25) % (1U << 3);
+  out++;
+  *out = ((*in) >> 28) % (1U << 3);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (3 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 3);
+  out++;
+  *out = ((*in) >> 5) % (1U << 3);
+  out++;
+  *out = ((*in) >> 8) % (1U << 3);
+  out++;
+  *out = ((*in) >> 11) % (1U << 3);
+  out++;
+  *out = ((*in) >> 14) % (1U << 3);
+  out++;
+  *out = ((*in) >> 17) % (1U << 3);
+  out++;
+  *out = ((*in) >> 20) % (1U << 3);
+  out++;
+  *out = ((*in) >> 23) % (1U << 3);
+  out++;
+  *out = ((*in) >> 26) % (1U << 3);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack4_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 4);
+  out++;
+  *out = ((*in) >> 8) % (1U << 4);
+  out++;
+  *out = ((*in) >> 12) % (1U << 4);
+  out++;
+  *out = ((*in) >> 16) % (1U << 4);
+  out++;
+  *out = ((*in) >> 20) % (1U << 4);
+  out++;
+  *out = ((*in) >> 24) % (1U << 4);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 4);
+  out++;
+  *out = ((*in) >> 8) % (1U << 4);
+  out++;
+  *out = ((*in) >> 12) % (1U << 4);
+  out++;
+  *out = ((*in) >> 16) % (1U << 4);
+  out++;
+  *out = ((*in) >> 20) % (1U << 4);
+  out++;
+  *out = ((*in) >> 24) % (1U << 4);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 4);
+  out++;
+  *out = ((*in) >> 8) % (1U << 4);
+  out++;
+  *out = ((*in) >> 12) % (1U << 4);
+  out++;
+  *out = ((*in) >> 16) % (1U << 4);
+  out++;
+  *out = ((*in) >> 20) % (1U << 4);
+  out++;
+  *out = ((*in) >> 24) % (1U << 4);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 4);
+  out++;
+  *out = ((*in) >> 8) % (1U << 4);
+  out++;
+  *out = ((*in) >> 12) % (1U << 4);
+  out++;
+  *out = ((*in) >> 16) % (1U << 4);
+  out++;
+  *out = ((*in) >> 20) % (1U << 4);
+  out++;
+  *out = ((*in) >> 24) % (1U << 4);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack5_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 5);
+  out++;
+  *out = ((*in) >> 10) % (1U << 5);
+  out++;
+  *out = ((*in) >> 15) % (1U << 5);
+  out++;
+  *out = ((*in) >> 20) % (1U << 5);
+  out++;
+  *out = ((*in) >> 25) % (1U << 5);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (5 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 5);
+  out++;
+  *out = ((*in) >> 8) % (1U << 5);
+  out++;
+  *out = ((*in) >> 13) % (1U << 5);
+  out++;
+  *out = ((*in) >> 18) % (1U << 5);
+  out++;
+  *out = ((*in) >> 23) % (1U << 5);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (5 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 5);
+  out++;
+  *out = ((*in) >> 6) % (1U << 5);
+  out++;
+  *out = ((*in) >> 11) % (1U << 5);
+  out++;
+  *out = ((*in) >> 16) % (1U << 5);
+  out++;
+  *out = ((*in) >> 21) % (1U << 5);
+  out++;
+  *out = ((*in) >> 26) % (1U << 5);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (5 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 5);
+  out++;
+  *out = ((*in) >> 9) % (1U << 5);
+  out++;
+  *out = ((*in) >> 14) % (1U << 5);
+  out++;
+  *out = ((*in) >> 19) % (1U << 5);
+  out++;
+  *out = ((*in) >> 24) % (1U << 5);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (5 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 5);
+  out++;
+  *out = ((*in) >> 7) % (1U << 5);
+  out++;
+  *out = ((*in) >> 12) % (1U << 5);
+  out++;
+  *out = ((*in) >> 17) % (1U << 5);
+  out++;
+  *out = ((*in) >> 22) % (1U << 5);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack6_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 6);
+  out++;
+  *out = ((*in) >> 12) % (1U << 6);
+  out++;
+  *out = ((*in) >> 18) % (1U << 6);
+  out++;
+  *out = ((*in) >> 24) % (1U << 6);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (6 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 6);
+  out++;
+  *out = ((*in) >> 10) % (1U << 6);
+  out++;
+  *out = ((*in) >> 16) % (1U << 6);
+  out++;
+  *out = ((*in) >> 22) % (1U << 6);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (6 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 6);
+  out++;
+  *out = ((*in) >> 8) % (1U << 6);
+  out++;
+  *out = ((*in) >> 14) % (1U << 6);
+  out++;
+  *out = ((*in) >> 20) % (1U << 6);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 6);
+  out++;
+  *out = ((*in) >> 12) % (1U << 6);
+  out++;
+  *out = ((*in) >> 18) % (1U << 6);
+  out++;
+  *out = ((*in) >> 24) % (1U << 6);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (6 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 6);
+  out++;
+  *out = ((*in) >> 10) % (1U << 6);
+  out++;
+  *out = ((*in) >> 16) % (1U << 6);
+  out++;
+  *out = ((*in) >> 22) % (1U << 6);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (6 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 6);
+  out++;
+  *out = ((*in) >> 8) % (1U << 6);
+  out++;
+  *out = ((*in) >> 14) % (1U << 6);
+  out++;
+  *out = ((*in) >> 20) % (1U << 6);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack7_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 7);
+  out++;
+  *out = ((*in) >> 14) % (1U << 7);
+  out++;
+  *out = ((*in) >> 21) % (1U << 7);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (7 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 7);
+  out++;
+  *out = ((*in) >> 10) % (1U << 7);
+  out++;
+  *out = ((*in) >> 17) % (1U << 7);
+  out++;
+  *out = ((*in) >> 24) % (1U << 7);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (7 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 7);
+  out++;
+  *out = ((*in) >> 13) % (1U << 7);
+  out++;
+  *out = ((*in) >> 20) % (1U << 7);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (7 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 7);
+  out++;
+  *out = ((*in) >> 9) % (1U << 7);
+  out++;
+  *out = ((*in) >> 16) % (1U << 7);
+  out++;
+  *out = ((*in) >> 23) % (1U << 7);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (7 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 7);
+  out++;
+  *out = ((*in) >> 12) % (1U << 7);
+  out++;
+  *out = ((*in) >> 19) % (1U << 7);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (7 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 7);
+  out++;
+  *out = ((*in) >> 8) % (1U << 7);
+  out++;
+  *out = ((*in) >> 15) % (1U << 7);
+  out++;
+  *out = ((*in) >> 22) % (1U << 7);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (7 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 7);
+  out++;
+  *out = ((*in) >> 11) % (1U << 7);
+  out++;
+  *out = ((*in) >> 18) % (1U << 7);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack8_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 8);
+  out++;
+  *out = ((*in) >> 16) % (1U << 8);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 8);
+  out++;
+  *out = ((*in) >> 16) % (1U << 8);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 8);
+  out++;
+  *out = ((*in) >> 16) % (1U << 8);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 8);
+  out++;
+  *out = ((*in) >> 16) % (1U << 8);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 8);
+  out++;
+  *out = ((*in) >> 16) % (1U << 8);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 8);
+  out++;
+  *out = ((*in) >> 16) % (1U << 8);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 8);
+  out++;
+  *out = ((*in) >> 16) % (1U << 8);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 8);
+  out++;
+  *out = ((*in) >> 16) % (1U << 8);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack9_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 9);
+  out++;
+  *out = ((*in) >> 9) % (1U << 9);
+  out++;
+  *out = ((*in) >> 18) % (1U << 9);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (9 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 9);
+  out++;
+  *out = ((*in) >> 13) % (1U << 9);
+  out++;
+  *out = ((*in) >> 22) % (1U << 9);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (9 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 9);
+  out++;
+  *out = ((*in) >> 17) % (1U << 9);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (9 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 9);
+  out++;
+  *out = ((*in) >> 12) % (1U << 9);
+  out++;
+  *out = ((*in) >> 21) % (1U << 9);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (9 - 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 9);
+  out++;
+  *out = ((*in) >> 16) % (1U << 9);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (9 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 9);
+  out++;
+  *out = ((*in) >> 11) % (1U << 9);
+  out++;
+  *out = ((*in) >> 20) % (1U << 9);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (9 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 9);
+  out++;
+  *out = ((*in) >> 15) % (1U << 9);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (9 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 9);
+  out++;
+  *out = ((*in) >> 10) % (1U << 9);
+  out++;
+  *out = ((*in) >> 19) % (1U << 9);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (9 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 9);
+  out++;
+  *out = ((*in) >> 14) % (1U << 9);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack10_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 10);
+  out++;
+  *out = ((*in) >> 20) % (1U << 10);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (10 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 10);
+  out++;
+  *out = ((*in) >> 18) % (1U << 10);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (10 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 10);
+  out++;
+  *out = ((*in) >> 16) % (1U << 10);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (10 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 10);
+  out++;
+  *out = ((*in) >> 14) % (1U << 10);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (10 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 10);
+  out++;
+  *out = ((*in) >> 12) % (1U << 10);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 10);
+  out++;
+  *out = ((*in) >> 20) % (1U << 10);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (10 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 10);
+  out++;
+  *out = ((*in) >> 18) % (1U << 10);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (10 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 10);
+  out++;
+  *out = ((*in) >> 16) % (1U << 10);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (10 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 10);
+  out++;
+  *out = ((*in) >> 14) % (1U << 10);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (10 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 10);
+  out++;
+  *out = ((*in) >> 12) % (1U << 10);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack11_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 11);
+  out++;
+  *out = ((*in) >> 11) % (1U << 11);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (11 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 11);
+  out++;
+  *out = ((*in) >> 12) % (1U << 11);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (11 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 11);
+  out++;
+  *out = ((*in) >> 13) % (1U << 11);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (11 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 11);
+  out++;
+  *out = ((*in) >> 14) % (1U << 11);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (11 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 11);
+  out++;
+  *out = ((*in) >> 15) % (1U << 11);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (11 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 11);
+  out++;
+  *out = ((*in) >> 16) % (1U << 11);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (11 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 11);
+  out++;
+  *out = ((*in) >> 17) % (1U << 11);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (11 - 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 11);
+  out++;
+  *out = ((*in) >> 18) % (1U << 11);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (11 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 11);
+  out++;
+  *out = ((*in) >> 19) % (1U << 11);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (11 - 9);
+  out++;
+  *out = ((*in) >> 9) % (1U << 11);
+  out++;
+  *out = ((*in) >> 20) % (1U << 11);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (11 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 11);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack12_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 12);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (12 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 12);
+  out++;
+  *out = ((*in) >> 16) % (1U << 12);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (12 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 12);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 12);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (12 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 12);
+  out++;
+  *out = ((*in) >> 16) % (1U << 12);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (12 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 12);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 12);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (12 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 12);
+  out++;
+  *out = ((*in) >> 16) % (1U << 12);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (12 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 12);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 12);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (12 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 12);
+  out++;
+  *out = ((*in) >> 16) % (1U << 12);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (12 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 12);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack13_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 13);
+  out++;
+  *out = ((*in) >> 13) % (1U << 13);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (13 - 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 13);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (13 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 13);
+  out++;
+  *out = ((*in) >> 14) % (1U << 13);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (13 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 13);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (13 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 13);
+  out++;
+  *out = ((*in) >> 15) % (1U << 13);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (13 - 9);
+  out++;
+  *out = ((*in) >> 9) % (1U << 13);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (13 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 13);
+  out++;
+  *out = ((*in) >> 16) % (1U << 13);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (13 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 13);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (13 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 13);
+  out++;
+  *out = ((*in) >> 17) % (1U << 13);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (13 - 11);
+  out++;
+  *out = ((*in) >> 11) % (1U << 13);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (13 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 13);
+  out++;
+  *out = ((*in) >> 18) % (1U << 13);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (13 - 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 13);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (13 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 13);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack14_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 14);
+  out++;
+  *out = ((*in) >> 14) % (1U << 14);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (14 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 14);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (14 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 14);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (14 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 14);
+  out++;
+  *out = ((*in) >> 16) % (1U << 14);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (14 - 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 14);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (14 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 14);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (14 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 14);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 14);
+  out++;
+  *out = ((*in) >> 14) % (1U << 14);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (14 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 14);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (14 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 14);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (14 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 14);
+  out++;
+  *out = ((*in) >> 16) % (1U << 14);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (14 - 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 14);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (14 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 14);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (14 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 14);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack15_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 15);
+  out++;
+  *out = ((*in) >> 15) % (1U << 15);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (15 - 13);
+  out++;
+  *out = ((*in) >> 13) % (1U << 15);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (15 - 11);
+  out++;
+  *out = ((*in) >> 11) % (1U << 15);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (15 - 9);
+  out++;
+  *out = ((*in) >> 9) % (1U << 15);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (15 - 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 15);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (15 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 15);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (15 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 15);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (15 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 15);
+  out++;
+  *out = ((*in) >> 16) % (1U << 15);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (15 - 14);
+  out++;
+  *out = ((*in) >> 14) % (1U << 15);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (15 - 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 15);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (15 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 15);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (15 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 15);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (15 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 15);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (15 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 15);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (15 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 15);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack16_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack17_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 17);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (17 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 17);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (17 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 17);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (17 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 17);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (17 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 17);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (17 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 17);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (17 - 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 17);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (17 - 14);
+  out++;
+  *out = ((*in) >> 14) % (1U << 17);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (17 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (17 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 17);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (17 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 17);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (17 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 17);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (17 - 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 17);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (17 - 9);
+  out++;
+  *out = ((*in) >> 9) % (1U << 17);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (17 - 11);
+  out++;
+  *out = ((*in) >> 11) % (1U << 17);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (17 - 13);
+  out++;
+  *out = ((*in) >> 13) % (1U << 17);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 15)) << (17 - 15);
+  out++;
+  *out = ((*in) >> 15);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack18_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (18 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 18);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (18 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 18);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (18 - 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 18);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (18 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (18 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 18);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (18 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 18);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (18 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 18);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (18 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (18 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 18);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (18 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 18);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (18 - 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 18);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (18 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (18 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 18);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (18 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 18);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (18 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 18);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (18 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack19_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 19);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (19 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 19);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (19 - 12);
+  out++;
+  *out = ((*in) >> 12) % (1U << 19);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (19 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (19 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 19);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (19 - 11);
+  out++;
+  *out = ((*in) >> 11) % (1U << 19);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 17)) << (19 - 17);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (19 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 19);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (19 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 19);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (19 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (19 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 19);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (19 - 9);
+  out++;
+  *out = ((*in) >> 9) % (1U << 19);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 15)) << (19 - 15);
+  out++;
+  *out = ((*in) >> 15);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (19 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 19);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (19 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 19);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (19 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (19 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 19);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (19 - 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 19);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (19 - 13);
+  out++;
+  *out = ((*in) >> 13);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack20_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (20 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 20);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (20 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (20 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 20);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (20 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (20 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 20);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (20 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (20 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 20);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (20 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (20 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 20);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (20 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (20 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 20);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (20 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (20 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 20);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (20 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (20 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 20);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (20 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack21_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 21);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (21 - 10);
+  out++;
+  *out = ((*in) >> 10) % (1U << 21);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (21 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (21 - 9);
+  out++;
+  *out = ((*in) >> 9) % (1U << 21);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 19)) << (21 - 19);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (21 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 21);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (21 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (21 - 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 21);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 17)) << (21 - 17);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (21 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 21);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (21 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (21 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 21);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 15)) << (21 - 15);
+  out++;
+  *out = ((*in) >> 15);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (21 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 21);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (21 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (21 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 21);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (21 - 13);
+  out++;
+  *out = ((*in) >> 13);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (21 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 21);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (21 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (21 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 21);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (21 - 11);
+  out++;
+  *out = ((*in) >> 11);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack22_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (22 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (22 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 22);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (22 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (22 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 22);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (22 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (22 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 22);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (22 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (22 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 22);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (22 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (22 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (22 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (22 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 22);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (22 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (22 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 22);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (22 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (22 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 22);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (22 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (22 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 22);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (22 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (22 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack23_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 23);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (23 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (23 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 23);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 19)) << (23 - 19);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (23 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (23 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 23);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 15)) << (23 - 15);
+  out++;
+  *out = ((*in) >> 15);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (23 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 23);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (23 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (23 - 11);
+  out++;
+  *out = ((*in) >> 11);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (23 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 23);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (23 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (23 - 7);
+  out++;
+  *out = ((*in) >> 7) % (1U << 23);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 21)) << (23 - 21);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (23 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (23 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 23);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 17)) << (23 - 17);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (23 - 8);
+  out++;
+  *out = ((*in) >> 8) % (1U << 23);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (23 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (23 - 13);
+  out++;
+  *out = ((*in) >> 13);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (23 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 23);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (23 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (23 - 9);
+  out++;
+  *out = ((*in) >> 9);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack24_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (24 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (24 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (24 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (24 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (24 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (24 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (24 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (24 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (24 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (24 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (24 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (24 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (24 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (24 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (24 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (24 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack25_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 25);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (25 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (25 - 11);
+  out++;
+  *out = ((*in) >> 11);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (25 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 25);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (25 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 15)) << (25 - 15);
+  out++;
+  *out = ((*in) >> 15);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (25 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (25 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 25);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 19)) << (25 - 19);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (25 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (25 - 5);
+  out++;
+  *out = ((*in) >> 5) % (1U << 25);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 23)) << (25 - 23);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (25 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (25 - 9);
+  out++;
+  *out = ((*in) >> 9);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (25 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 25);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (25 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (25 - 13);
+  out++;
+  *out = ((*in) >> 13);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (25 - 6);
+  out++;
+  *out = ((*in) >> 6) % (1U << 25);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (25 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 17)) << (25 - 17);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (25 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (25 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 25);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 21)) << (25 - 21);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (25 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (25 - 7);
+  out++;
+  *out = ((*in) >> 7);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack26_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 26);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (26 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (26 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (26 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (26 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 26);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (26 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (26 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (26 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (26 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 26);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (26 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (26 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (26 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (26 - 6);
+  out++;
+  *out = ((*in) >> 6);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 26);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (26 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (26 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (26 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (26 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 26);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (26 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (26 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (26 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (26 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 26);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (26 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (26 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (26 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (26 - 6);
+  out++;
+  *out = ((*in) >> 6);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack27_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 27);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (27 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 17)) << (27 - 17);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (27 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (27 - 7);
+  out++;
+  *out = ((*in) >> 7);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (27 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 27);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (27 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 19)) << (27 - 19);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (27 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (27 - 9);
+  out++;
+  *out = ((*in) >> 9);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (27 - 4);
+  out++;
+  *out = ((*in) >> 4) % (1U << 27);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 26)) << (27 - 26);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 21)) << (27 - 21);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (27 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (27 - 11);
+  out++;
+  *out = ((*in) >> 11);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (27 - 6);
+  out++;
+  *out = ((*in) >> 6);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (27 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 27);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 23)) << (27 - 23);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (27 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (27 - 13);
+  out++;
+  *out = ((*in) >> 13);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (27 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (27 - 3);
+  out++;
+  *out = ((*in) >> 3) % (1U << 27);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 25)) << (27 - 25);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (27 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 15)) << (27 - 15);
+  out++;
+  *out = ((*in) >> 15);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (27 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (27 - 5);
+  out++;
+  *out = ((*in) >> 5);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack28_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 28);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (28 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (28 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (28 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (28 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (28 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (28 - 4);
+  out++;
+  *out = ((*in) >> 4);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 28);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (28 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (28 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (28 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (28 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (28 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (28 - 4);
+  out++;
+  *out = ((*in) >> 4);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 28);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (28 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (28 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (28 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (28 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (28 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (28 - 4);
+  out++;
+  *out = ((*in) >> 4);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 28);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (28 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (28 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (28 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (28 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (28 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (28 - 4);
+  out++;
+  *out = ((*in) >> 4);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack29_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 29);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 26)) << (29 - 26);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 23)) << (29 - 23);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (29 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 17)) << (29 - 17);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (29 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (29 - 11);
+  out++;
+  *out = ((*in) >> 11);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (29 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (29 - 5);
+  out++;
+  *out = ((*in) >> 5);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (29 - 2);
+  out++;
+  *out = ((*in) >> 2) % (1U << 29);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 28)) << (29 - 28);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 25)) << (29 - 25);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (29 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 19)) << (29 - 19);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (29 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (29 - 13);
+  out++;
+  *out = ((*in) >> 13);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (29 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (29 - 7);
+  out++;
+  *out = ((*in) >> 7);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (29 - 4);
+  out++;
+  *out = ((*in) >> 4);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (29 - 1);
+  out++;
+  *out = ((*in) >> 1) % (1U << 29);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 27)) << (29 - 27);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (29 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 21)) << (29 - 21);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (29 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 15)) << (29 - 15);
+  out++;
+  *out = ((*in) >> 15);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (29 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (29 - 9);
+  out++;
+  *out = ((*in) >> 9);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (29 - 6);
+  out++;
+  *out = ((*in) >> 6);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (29 - 3);
+  out++;
+  *out = ((*in) >> 3);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack30_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 30);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 28)) << (30 - 28);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 26)) << (30 - 26);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (30 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (30 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (30 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (30 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (30 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (30 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (30 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (30 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (30 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (30 - 6);
+  out++;
+  *out = ((*in) >> 6);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (30 - 4);
+  out++;
+  *out = ((*in) >> 4);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (30 - 2);
+  out++;
+  *out = ((*in) >> 2);
+  ++in;
+  out++;
+  *out = ((*in) >> 0) % (1U << 30);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 28)) << (30 - 28);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 26)) << (30 - 26);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (30 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (30 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (30 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (30 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (30 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (30 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (30 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (30 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (30 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (30 - 6);
+  out++;
+  *out = ((*in) >> 6);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (30 - 4);
+  out++;
+  *out = ((*in) >> 4);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (30 - 2);
+  out++;
+  *out = ((*in) >> 2);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack31_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0) % (1U << 31);
+  out++;
+  *out = ((*in) >> 31);
+  ++in;
+  *out |= ((*in) % (1U << 30)) << (31 - 30);
+  out++;
+  *out = ((*in) >> 30);
+  ++in;
+  *out |= ((*in) % (1U << 29)) << (31 - 29);
+  out++;
+  *out = ((*in) >> 29);
+  ++in;
+  *out |= ((*in) % (1U << 28)) << (31 - 28);
+  out++;
+  *out = ((*in) >> 28);
+  ++in;
+  *out |= ((*in) % (1U << 27)) << (31 - 27);
+  out++;
+  *out = ((*in) >> 27);
+  ++in;
+  *out |= ((*in) % (1U << 26)) << (31 - 26);
+  out++;
+  *out = ((*in) >> 26);
+  ++in;
+  *out |= ((*in) % (1U << 25)) << (31 - 25);
+  out++;
+  *out = ((*in) >> 25);
+  ++in;
+  *out |= ((*in) % (1U << 24)) << (31 - 24);
+  out++;
+  *out = ((*in) >> 24);
+  ++in;
+  *out |= ((*in) % (1U << 23)) << (31 - 23);
+  out++;
+  *out = ((*in) >> 23);
+  ++in;
+  *out |= ((*in) % (1U << 22)) << (31 - 22);
+  out++;
+  *out = ((*in) >> 22);
+  ++in;
+  *out |= ((*in) % (1U << 21)) << (31 - 21);
+  out++;
+  *out = ((*in) >> 21);
+  ++in;
+  *out |= ((*in) % (1U << 20)) << (31 - 20);
+  out++;
+  *out = ((*in) >> 20);
+  ++in;
+  *out |= ((*in) % (1U << 19)) << (31 - 19);
+  out++;
+  *out = ((*in) >> 19);
+  ++in;
+  *out |= ((*in) % (1U << 18)) << (31 - 18);
+  out++;
+  *out = ((*in) >> 18);
+  ++in;
+  *out |= ((*in) % (1U << 17)) << (31 - 17);
+  out++;
+  *out = ((*in) >> 17);
+  ++in;
+  *out |= ((*in) % (1U << 16)) << (31 - 16);
+  out++;
+  *out = ((*in) >> 16);
+  ++in;
+  *out |= ((*in) % (1U << 15)) << (31 - 15);
+  out++;
+  *out = ((*in) >> 15);
+  ++in;
+  *out |= ((*in) % (1U << 14)) << (31 - 14);
+  out++;
+  *out = ((*in) >> 14);
+  ++in;
+  *out |= ((*in) % (1U << 13)) << (31 - 13);
+  out++;
+  *out = ((*in) >> 13);
+  ++in;
+  *out |= ((*in) % (1U << 12)) << (31 - 12);
+  out++;
+  *out = ((*in) >> 12);
+  ++in;
+  *out |= ((*in) % (1U << 11)) << (31 - 11);
+  out++;
+  *out = ((*in) >> 11);
+  ++in;
+  *out |= ((*in) % (1U << 10)) << (31 - 10);
+  out++;
+  *out = ((*in) >> 10);
+  ++in;
+  *out |= ((*in) % (1U << 9)) << (31 - 9);
+  out++;
+  *out = ((*in) >> 9);
+  ++in;
+  *out |= ((*in) % (1U << 8)) << (31 - 8);
+  out++;
+  *out = ((*in) >> 8);
+  ++in;
+  *out |= ((*in) % (1U << 7)) << (31 - 7);
+  out++;
+  *out = ((*in) >> 7);
+  ++in;
+  *out |= ((*in) % (1U << 6)) << (31 - 6);
+  out++;
+  *out = ((*in) >> 6);
+  ++in;
+  *out |= ((*in) % (1U << 5)) << (31 - 5);
+  out++;
+  *out = ((*in) >> 5);
+  ++in;
+  *out |= ((*in) % (1U << 4)) << (31 - 4);
+  out++;
+  *out = ((*in) >> 4);
+  ++in;
+  *out |= ((*in) % (1U << 3)) << (31 - 3);
+  out++;
+  *out = ((*in) >> 3);
+  ++in;
+  *out |= ((*in) % (1U << 2)) << (31 - 2);
+  out++;
+  *out = ((*in) >> 2);
+  ++in;
+  *out |= ((*in) % (1U << 1)) << (31 - 1);
+  out++;
+  *out = ((*in) >> 1);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* unpack32_32(const uint32_t* in, uint32_t* out) {
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+  *out = ((*in) >> 0);
+  ++in;
+  out++;
+
+  return in;
+}
+
+inline const uint32_t* nullunpacker32(const uint32_t* in, uint32_t* out) {
+  for (int k = 0; k < 32; ++k) {
+    out[k] = 0;
+  }
+  return in;
+}
+
+inline int unpack32(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
+  batch_size = batch_size / 32 * 32;
+  int num_loops = batch_size / 32;
+
+  switch (num_bits) {
+    case 0:
+      for (int i = 0; i < num_loops; ++i)
+        in = nullunpacker32(in, out + i * 32);
+      break;
+    case 1:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack1_32(in, out + i * 32);
+      break;
+    case 2:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack2_32(in, out + i * 32);
+      break;
+    case 3:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack3_32(in, out + i * 32);
+      break;
+    case 4:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack4_32(in, out + i * 32);
+      break;
+    case 5:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack5_32(in, out + i * 32);
+      break;
+    case 6:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack6_32(in, out + i * 32);
+      break;
+    case 7:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack7_32(in, out + i * 32);
+      break;
+    case 8:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack8_32(in, out + i * 32);
+      break;
+    case 9:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack9_32(in, out + i * 32);
+      break;
+    case 10:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack10_32(in, out + i * 32);
+      break;
+    case 11:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack11_32(in, out + i * 32);
+      break;
+    case 12:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack12_32(in, out + i * 32);
+      break;
+    case 13:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack13_32(in, out + i * 32);
+      break;
+    case 14:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack14_32(in, out + i * 32);
+      break;
+    case 15:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack15_32(in, out + i * 32);
+      break;
+    case 16:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack16_32(in, out + i * 32);
+      break;
+    case 17:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack17_32(in, out + i * 32);
+      break;
+    case 18:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack18_32(in, out + i * 32);
+      break;
+    case 19:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack19_32(in, out + i * 32);
+      break;
+    case 20:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack20_32(in, out + i * 32);
+      break;
+    case 21:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack21_32(in, out + i * 32);
+      break;
+    case 22:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack22_32(in, out + i * 32);
+      break;
+    case 23:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack23_32(in, out + i * 32);
+      break;
+    case 24:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack24_32(in, out + i * 32);
+      break;
+    case 25:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack25_32(in, out + i * 32);
+      break;
+    case 26:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack26_32(in, out + i * 32);
+      break;
+    case 27:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack27_32(in, out + i * 32);
+      break;
+    case 28:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack28_32(in, out + i * 32);
+      break;
+    case 29:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack29_32(in, out + i * 32);
+      break;
+    case 30:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack30_32(in, out + i * 32);
+      break;
+    case 31:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack31_32(in, out + i * 32);
+      break;
+    case 32:
+      for (int i = 0; i < num_loops; ++i)
+        in = unpack32_32(in, out + i * 32);
+      break;
+    default:
+      DCHECK(false) << "Unsupported num_bits";
+  }
+
+  return batch_size;
+}
+
+};  // namespace arrow
+
+#endif  // ARROW_UTIL_BPACKING_H
diff --git a/cpp/src/arrow/util/compiler-util.h b/cpp/src/arrow/util/compiler-util.h
new file mode 100644
index 0000000000000..ccbe54584b40c
--- /dev/null
+++ b/cpp/src/arrow/util/compiler-util.h
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_COMPILER_UTIL_H
+#define ARROW_UTIL_COMPILER_UTIL_H
+
+// Branch prediction macro hints for GCC
+#ifdef LIKELY
+#undef LIKELY
+#endif
+
+#ifdef UNLIKELY
+#undef UNLIKELY
+#endif
+
+#ifdef _MSC_VER
+#define LIKELY(expr) expr
+#define UNLIKELY(expr) expr
+#else
+#define LIKELY(expr) __builtin_expect(!!(expr), 1)
+#define UNLIKELY(expr) __builtin_expect(!!(expr), 0)
+#endif
+
+#define PREFETCH(addr) __builtin_prefetch(addr)
+
+// macros to disable padding
+// these macros are portable across different compilers and platforms
+//[https://github.com/google/flatbuffers/blob/master/include/flatbuffers/flatbuffers.h#L1355]
+#if defined(_MSC_VER)
+#define MANUALLY_ALIGNED_STRUCT(alignment) \
+  __pragma(pack(1));                       \
+  struct __declspec(align(alignment))
+#define STRUCT_END(name, size) \
+  __pragma(pack());            \
+  static_assert(sizeof(name) == size, "compiler breaks packing rules")
+#elif defined(__GNUC__) || defined(__clang__)
+#define MANUALLY_ALIGNED_STRUCT(alignment) \
+  _Pragma("pack(1)") struct __attribute__((aligned(alignment)))
+#define STRUCT_END(name, size) \
+  _Pragma("pack()") static_assert(sizeof(name) == size, "compiler breaks packing rules")
+#else
+#error Unknown compiler, please define structure alignment macros
+#endif
+
+#endif  // ARROW_UTIL_COMPILER_UTIL_H
diff --git a/cpp/src/arrow/util/cpu-info.cc b/cpp/src/arrow/util/cpu-info.cc
new file mode 100644
index 0000000000000..c0fc8bdddf4bf
--- /dev/null
+++ b/cpp/src/arrow/util/cpu-info.cc
@@ -0,0 +1,206 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// From Apache Impala (incubating) as of 2016-01-29.
+
+#include "arrow/util/cpu-info.h"
+
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef _MSC_VER
+#include <unistd.h>
+#endif
+
+#include <boost/algorithm/string.hpp>
+
+#include <algorithm>
+#include <cstdint>
+#include <fstream>
+#include <iostream>
+#include <mutex>
+#include <sstream>
+#include <string>
+
+#include "arrow/util/logging.h"
+
+using boost::algorithm::contains;
+using boost::algorithm::trim;
+using std::max;
+using std::string;
+
+namespace arrow {
+
+bool CpuInfo::initialized_ = false;
+int64_t CpuInfo::hardware_flags_ = 0;
+int64_t CpuInfo::original_hardware_flags_;
+int64_t CpuInfo::cache_sizes_[L3_CACHE + 1];
+int64_t CpuInfo::cycles_per_ms_;
+int CpuInfo::num_cores_ = 1;
+string CpuInfo::model_name_ = "unknown";  // NOLINT
+static std::mutex cpuinfo_mutex;
+
+static struct {
+  string name;
+  int64_t flag;
+} flag_mappings[] = {
+    {"ssse3", CpuInfo::SSSE3}, {"sse4_1", CpuInfo::SSE4_1}, {"sse4_2", CpuInfo::SSE4_2},
+    {"popcnt", CpuInfo::POPCNT},
+};
+static const int64_t num_flags = sizeof(flag_mappings) / sizeof(flag_mappings[0]);
+
+// Helper function to parse for hardware flags.
+// values contains a list of space-seperated flags.  check to see if the flags we
+// care about are present.
+// Returns a bitmap of flags.
+int64_t ParseCPUFlags(const string& values) {
+  int64_t flags = 0;
+  for (int i = 0; i < num_flags; ++i) {
+    if (contains(values, flag_mappings[i].name)) { flags |= flag_mappings[i].flag; }
+  }
+  return flags;
+}
+
+void CpuInfo::Init() {
+  std::lock_guard<std::mutex> cpuinfo_lock(cpuinfo_mutex);
+
+  if (initialized()) { return; }
+
+  string line;
+  string name;
+  string value;
+
+  float max_mhz = 0;
+  int num_cores = 0;
+
+  memset(&cache_sizes_, 0, sizeof(cache_sizes_));
+
+  // Read from /proc/cpuinfo
+  std::ifstream cpuinfo("/proc/cpuinfo", std::ios::in);
+  while (cpuinfo) {
+    getline(cpuinfo, line);
+    size_t colon = line.find(':');
+    if (colon != string::npos) {
+      name = line.substr(0, colon - 1);
+      value = line.substr(colon + 1, string::npos);
+      trim(name);
+      trim(value);
+      if (name.compare("flags") == 0) {
+        hardware_flags_ |= ParseCPUFlags(value);
+      } else if (name.compare("cpu MHz") == 0) {
+        // Every core will report a different speed.  We'll take the max, assuming
+        // that when impala is running, the core will not be in a lower power state.
+        // TODO: is there a more robust way to do this, such as
+        // Window's QueryPerformanceFrequency()
+        float mhz = static_cast<float>(atof(value.c_str()));
+        max_mhz = max(mhz, max_mhz);
+      } else if (name.compare("processor") == 0) {
+        ++num_cores;
+      } else if (name.compare("model name") == 0) {
+        model_name_ = value;
+      }
+    }
+  }
+  if (cpuinfo.is_open()) cpuinfo.close();
+
+#ifdef __APPLE__
+  // On Mac OS X use sysctl() to get the cache sizes
+  size_t len = 0;
+  sysctlbyname("hw.cachesize", NULL, &len, NULL, 0);
+  uint64_t* data = static_cast<uint64_t*>(malloc(len));
+  sysctlbyname("hw.cachesize", data, &len, NULL, 0);
+  DCHECK_GE(len / sizeof(uint64_t), 3);
+  for (size_t i = 0; i < 3; ++i) {
+    cache_sizes_[i] = data[i];
+  }
+#else
+#ifndef _SC_LEVEL1_DCACHE_SIZE
+  // Provide reasonable default values if no info
+  cache_sizes_[0] = 32 * 1024;    // Level 1: 32k
+  cache_sizes_[1] = 256 * 1024;   // Level 2: 256k
+  cache_sizes_[2] = 3072 * 1024;  // Level 3: 3M
+#else
+  // Call sysconf to query for the cache sizes
+  cache_sizes_[0] = sysconf(_SC_LEVEL1_DCACHE_SIZE);
+  cache_sizes_[1] = sysconf(_SC_LEVEL2_CACHE_SIZE);
+  cache_sizes_[2] = sysconf(_SC_LEVEL3_CACHE_SIZE);
+#endif
+#endif
+
+  if (max_mhz != 0) {
+    cycles_per_ms_ = static_cast<int64_t>(max_mhz) * 1000;
+  } else {
+    cycles_per_ms_ = 1000000;
+  }
+  original_hardware_flags_ = hardware_flags_;
+
+  if (num_cores > 0) {
+    num_cores_ = num_cores;
+  } else {
+    num_cores_ = 1;
+  }
+
+  initialized_ = true;
+}
+
+void CpuInfo::VerifyCpuRequirements() {
+  if (!CpuInfo::IsSupported(CpuInfo::SSSE3)) {
+    DCHECK(false) << "CPU does not support the Supplemental SSE3 instruction set";
+  }
+}
+
+void CpuInfo::EnableFeature(int64_t flag, bool enable) {
+  DCHECK(initialized_);
+  if (!enable) {
+    hardware_flags_ &= ~flag;
+  } else {
+    // Can't turn something on that can't be supported
+    DCHECK_NE(original_hardware_flags_ & flag, 0);
+    hardware_flags_ |= flag;
+  }
+}
+
+int64_t CpuInfo::hardware_flags() {
+  DCHECK(initialized_);
+  return hardware_flags_;
+}
+
+int64_t CpuInfo::CacheSize(CacheLevel level) {
+  DCHECK(initialized_);
+  return cache_sizes_[level];
+}
+
+int64_t CpuInfo::cycles_per_ms() {
+  DCHECK(initialized_);
+  return cycles_per_ms_;
+}
+
+int CpuInfo::num_cores() {
+  DCHECK(initialized_);
+  return num_cores_;
+}
+
+std::string CpuInfo::model_name() {
+  DCHECK(initialized_);
+  return model_name_;
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/cpu-info.h b/cpp/src/arrow/util/cpu-info.h
new file mode 100644
index 0000000000000..06800fc275572
--- /dev/null
+++ b/cpp/src/arrow/util/cpu-info.h
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// From Apache Impala (incubating) as of 2016-01-29. Pared down to a minimal
+// set of functions needed for Apache Arrow / Apache parquet-cpp
+
+#ifndef ARROW_UTIL_CPU_INFO_H
+#define ARROW_UTIL_CPU_INFO_H
+
+#include <cstdint>
+#include <string>
+
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+/// CpuInfo is an interface to query for cpu information at runtime.  The caller can
+/// ask for the sizes of the caches and what hardware features are supported.
+/// On Linux, this information is pulled from a couple of sys files (/proc/cpuinfo and
+/// /sys/devices)
+class ARROW_EXPORT CpuInfo {
+ public:
+  static const int64_t SSSE3 = (1 << 1);
+  static const int64_t SSE4_1 = (1 << 2);
+  static const int64_t SSE4_2 = (1 << 3);
+  static const int64_t POPCNT = (1 << 4);
+
+  /// Cache enums for L1 (data), L2 and L3
+  enum CacheLevel {
+    L1_CACHE = 0,
+    L2_CACHE = 1,
+    L3_CACHE = 2,
+  };
+
+  /// Initialize CpuInfo.
+  static void Init();
+
+  /// Determine if the CPU meets the minimum CPU requirements and if not, issue an error
+  /// and terminate.
+  static void VerifyCpuRequirements();
+
+  /// Returns all the flags for this cpu
+  static int64_t hardware_flags();
+
+  /// Returns whether of not the cpu supports this flag
+  inline static bool IsSupported(int64_t flag) { return (hardware_flags_ & flag) != 0; }
+
+  /// Toggle a hardware feature on and off.  It is not valid to turn on a feature
+  /// that the underlying hardware cannot support. This is useful for testing.
+  static void EnableFeature(int64_t flag, bool enable);
+
+  /// Returns the size of the cache in KB at this cache level
+  static int64_t CacheSize(CacheLevel level);
+
+  /// Returns the number of cpu cycles per millisecond
+  static int64_t cycles_per_ms();
+
+  /// Returns the number of cores (including hyper-threaded) on this machine.
+  static int num_cores();
+
+  /// Returns the model name of the cpu (e.g. Intel i7-2600)
+  static std::string model_name();
+
+  static bool initialized() { return initialized_; }
+
+ private:
+  static bool initialized_;
+  static int64_t hardware_flags_;
+  static int64_t original_hardware_flags_;
+  static int64_t cache_sizes_[L3_CACHE + 1];
+  static int64_t cycles_per_ms_;
+  static int num_cores_;
+  static std::string model_name_;  // NOLINT
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_CPU_INFO_H
diff --git a/cpp/src/arrow/util/hash-util.h b/cpp/src/arrow/util/hash-util.h
new file mode 100644
index 0000000000000..ffe1a9d01b4d3
--- /dev/null
+++ b/cpp/src/arrow/util/hash-util.h
@@ -0,0 +1,258 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// From Apache Impala (incubating) as of 2016-02-22
+
+#ifndef ARROW_UTIL_HASH_UTIL_H
+#define ARROW_UTIL_HASH_UTIL_H
+
+#include <cstdint>
+
+#include "arrow/util/compiler-util.h"
+#include "arrow/util/cpu-info.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/sse-util.h"
+
+namespace arrow {
+
+/// Utility class to compute hash values.
+class HashUtil {
+ public:
+  /// Compute the Crc32 hash for data using SSE4 instructions.  The input hash
+  /// parameter is the current hash/seed value.
+  /// This should only be called if SSE is supported.
+  /// This is ~4x faster than Fnv/Boost Hash.
+  /// TODO: crc32 hashes with different seeds do not result in different hash functions.
+  /// The resulting hashes are correlated.
+  /// TODO: update this to also use SSE4_crc32_u64 and SSE4_crc32_u16 where appropriate.
+  static uint32_t CrcHash(const void* data, int32_t bytes, uint32_t hash) {
+    DCHECK(CpuInfo::IsSupported(CpuInfo::SSE4_2));
+    uint32_t words = bytes / sizeof(uint32_t);
+    bytes = bytes % sizeof(uint32_t);
+
+    const uint32_t* p = reinterpret_cast<const uint32_t*>(data);
+    while (words--) {
+      hash = SSE4_crc32_u32(hash, *p);
+      ++p;
+    }
+
+    const uint8_t* s = reinterpret_cast<const uint8_t*>(p);
+    while (bytes--) {
+      hash = SSE4_crc32_u8(hash, *s);
+      ++s;
+    }
+
+    // The lower half of the CRC hash has has poor uniformity, so swap the halves
+    // for anyone who only uses the first several bits of the hash.
+    hash = (hash << 16) | (hash >> 16);
+    return hash;
+  }
+
+  /// CrcHash() specialized for 1-byte data
+  static inline uint32_t CrcHash1(const void* v, uint32_t hash) {
+    DCHECK(CpuInfo::IsSupported(CpuInfo::SSE4_2));
+    const uint8_t* s = reinterpret_cast<const uint8_t*>(v);
+    hash = SSE4_crc32_u8(hash, *s);
+    hash = (hash << 16) | (hash >> 16);
+    return hash;
+  }
+
+  /// CrcHash() specialized for 2-byte data
+  static inline uint32_t CrcHash2(const void* v, uint32_t hash) {
+    DCHECK(CpuInfo::IsSupported(CpuInfo::SSE4_2));
+    const uint16_t* s = reinterpret_cast<const uint16_t*>(v);
+    hash = SSE4_crc32_u16(hash, *s);
+    hash = (hash << 16) | (hash >> 16);
+    return hash;
+  }
+
+  /// CrcHash() specialized for 4-byte data
+  static inline uint32_t CrcHash4(const void* v, uint32_t hash) {
+    DCHECK(CpuInfo::IsSupported(CpuInfo::SSE4_2));
+    const uint32_t* p = reinterpret_cast<const uint32_t*>(v);
+    hash = SSE4_crc32_u32(hash, *p);
+    hash = (hash << 16) | (hash >> 16);
+    return hash;
+  }
+
+  /// CrcHash() specialized for 8-byte data
+  static inline uint32_t CrcHash8(const void* v, uint32_t hash) {
+    DCHECK(CpuInfo::IsSupported(CpuInfo::SSE4_2));
+    const uint64_t* p = reinterpret_cast<const uint64_t*>(v);
+    hash = SSE4_crc32_u64(hash, *p);
+    hash = (hash << 16) | (hash >> 16);
+    return hash;
+  }
+
+  /// CrcHash() specialized for 12-byte data
+  static inline uint32_t CrcHash12(const void* v, uint32_t hash) {
+    DCHECK(CpuInfo::IsSupported(CpuInfo::SSE4_2));
+    const uint64_t* p = reinterpret_cast<const uint64_t*>(v);
+    hash = SSE4_crc32_u64(hash, *p);
+    ++p;
+    hash = SSE4_crc32_u32(hash, *reinterpret_cast<const uint32_t*>(p));
+    hash = (hash << 16) | (hash >> 16);
+    return hash;
+  }
+
+  /// CrcHash() specialized for 16-byte data
+  static inline uint32_t CrcHash16(const void* v, uint32_t hash) {
+    DCHECK(CpuInfo::IsSupported(CpuInfo::SSE4_2));
+    const uint64_t* p = reinterpret_cast<const uint64_t*>(v);
+    hash = SSE4_crc32_u64(hash, *p);
+    ++p;
+    hash = SSE4_crc32_u64(hash, *p);
+    hash = (hash << 16) | (hash >> 16);
+    return hash;
+  }
+
+  static const uint64_t MURMUR_PRIME = 0xc6a4a7935bd1e995;
+  static const int MURMUR_R = 47;
+
+  /// Murmur2 hash implementation returning 64-bit hashes.
+  static uint64_t MurmurHash2_64(const void* input, int len, uint64_t seed) {
+    uint64_t h = seed ^ (len * MURMUR_PRIME);
+
+    const uint64_t* data = reinterpret_cast<const uint64_t*>(input);
+    const uint64_t* end = data + (len / sizeof(uint64_t));
+
+    while (data != end) {
+      uint64_t k = *data++;
+      k *= MURMUR_PRIME;
+      k ^= k >> MURMUR_R;
+      k *= MURMUR_PRIME;
+      h ^= k;
+      h *= MURMUR_PRIME;
+    }
+
+    const uint8_t* data2 = reinterpret_cast<const uint8_t*>(data);
+    switch (len & 7) {
+      case 7:
+        h ^= uint64_t(data2[6]) << 48;
+      case 6:
+        h ^= uint64_t(data2[5]) << 40;
+      case 5:
+        h ^= uint64_t(data2[4]) << 32;
+      case 4:
+        h ^= uint64_t(data2[3]) << 24;
+      case 3:
+        h ^= uint64_t(data2[2]) << 16;
+      case 2:
+        h ^= uint64_t(data2[1]) << 8;
+      case 1:
+        h ^= uint64_t(data2[0]);
+        h *= MURMUR_PRIME;
+    }
+
+    h ^= h >> MURMUR_R;
+    h *= MURMUR_PRIME;
+    h ^= h >> MURMUR_R;
+    return h;
+  }
+
+  /// default values recommended by http://isthe.com/chongo/tech/comp/fnv/
+  static const uint32_t FNV_PRIME = 0x01000193;  //   16777619
+  static const uint32_t FNV_SEED = 0x811C9DC5;   // 2166136261
+  static const uint64_t FNV64_PRIME = 1099511628211UL;
+  static const uint64_t FNV64_SEED = 14695981039346656037UL;
+
+  /// Implementation of the Fowler-Noll-Vo hash function. This is not as performant
+  /// as boost's hash on int types (2x slower) but has bit entropy.
+  /// For ints, boost just returns the value of the int which can be pathological.
+  /// For example, if the data is <1000, 2000, 3000, 4000, ..> and then the mod of 1000
+  /// is taken on the hash, all values will collide to the same bucket.
+  /// For string values, Fnv is slightly faster than boost.
+  /// IMPORTANT: FNV hash suffers from poor diffusion of the least significant bit,
+  /// which can lead to poor results when input bytes are duplicated.
+  /// See FnvHash64to32() for how this can be mitigated.
+  static uint64_t FnvHash64(const void* data, int32_t bytes, uint64_t hash) {
+    const uint8_t* ptr = reinterpret_cast<const uint8_t*>(data);
+    while (bytes--) {
+      hash = (*ptr ^ hash) * FNV64_PRIME;
+      ++ptr;
+    }
+    return hash;
+  }
+
+  /// Return a 32-bit hash computed by invoking FNV-64 and folding the result to 32-bits.
+  /// This technique is recommended instead of FNV-32 since the LSB of an FNV hash is the
+  /// XOR of the LSBs of its input bytes, leading to poor results for duplicate inputs.
+  /// The input seed 'hash' is duplicated so the top half of the seed is not all zero.
+  /// Data length must be at least 1 byte: zero-length data should be handled separately,
+  /// for example using CombineHash with a unique constant value to avoid returning the
+  /// hash argument. Zero-length data gives terrible results: the initial hash value is
+  /// xored with itself cancelling all bits.
+  static uint32_t FnvHash64to32(const void* data, int32_t bytes, uint32_t hash) {
+    // IMPALA-2270: this function should never be used for zero-byte inputs.
+    DCHECK_GT(bytes, 0);
+    uint64_t hash_u64 = hash | ((uint64_t)hash << 32);
+    hash_u64 = FnvHash64(data, bytes, hash_u64);
+    return (hash_u64 >> 32) ^ (hash_u64 & 0xFFFFFFFF);
+  }
+
+  /// Computes the hash value for data.  Will call either CrcHash or MurmurHash
+  /// depending on hardware capabilities.
+  /// Seed values for different steps of the query execution should use different seeds
+  /// to prevent accidental key collisions. (See IMPALA-219 for more details).
+  static uint32_t Hash(const void* data, int32_t bytes, uint32_t seed) {
+#ifdef ARROW_USE_SSE
+    if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
+      return CrcHash(data, bytes, seed);
+    } else {
+      return MurmurHash2_64(data, bytes, seed);
+    }
+#else
+    return static_cast<uint32_t>(MurmurHash2_64(data, bytes, seed));
+#endif
+  }
+
+  /// The magic number (used in hash_combine()) 0x9e3779b9 = 2^32 / (golden ratio).
+  static const uint32_t HASH_COMBINE_SEED = 0x9e3779b9;
+
+  /// Combine hashes 'value' and 'seed' to get a new hash value.  Similar to
+  /// boost::hash_combine(), but for uint32_t. This function should be used with a
+  /// constant first argument to update the hash value for zero-length values such as
+  /// NULL, boolean, and empty strings.
+  static inline uint32_t HashCombine32(uint32_t value, uint32_t seed) {
+    return seed ^ (HASH_COMBINE_SEED + value + (seed << 6) + (seed >> 2));
+  }
+
+  // Get 32 more bits of randomness from a 32-bit hash:
+  static inline uint32_t Rehash32to32(const uint32_t hash) {
+    // Constants generated by uuidgen(1) with the -r flag
+    static const uint64_t m = 0x7850f11ec6d14889ull, a = 0x6773610597ca4c63ull;
+    // This is strongly universal hashing following Dietzfelbinger's "Universal hashing
+    // and k-wise independent random variables via integer arithmetic without primes". As
+    // such, for any two distinct uint32_t's hash1 and hash2, the probability (over the
+    // randomness of the constants) that any subset of bit positions of
+    // Rehash32to32(hash1) is equal to the same subset of bit positions
+    // Rehash32to32(hash2) is minimal.
+    return (static_cast<uint64_t>(hash) * m + a) >> 32;
+  }
+
+  static inline uint64_t Rehash32to64(const uint32_t hash) {
+    static const uint64_t m1 = 0x47b6137a44974d91ull, m2 = 0x8824ad5ba2b7289cull,
+                          a1 = 0x705495c62df1424aull, a2 = 0x9efc49475c6bfb31ull;
+    const uint64_t hash1 = (static_cast<uint64_t>(hash) * m1 + a1) >> 32;
+    const uint64_t hash2 = (static_cast<uint64_t>(hash) * m2 + a2) >> 32;
+    return hash1 | (hash2 << 32);
+  }
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_HASH_UTIL_H
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 8a929da0e0231..b6181219dbae6 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -38,6 +38,7 @@ namespace arrow {
 
 #define ARROW_LOG_INTERNAL(level) ::arrow::internal::CerrLog(level)
 #define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level)
+#define ARROW_IGNORE_EXPR(expr) ((void)(expr));
 
 #define ARROW_CHECK(condition)                           \
   (condition) ? 0                                        \
@@ -47,25 +48,32 @@ namespace arrow {
 #ifdef NDEBUG
 #define ARROW_DFATAL ARROW_WARNING
 
-#define DCHECK(condition) \
-  while (false)           \
+#define DCHECK(condition)      \
+  ARROW_IGNORE_EXPR(condition) \
+  while (false)                \
   ::arrow::internal::NullLog()
 #define DCHECK_EQ(val1, val2) \
+  ARROW_IGNORE_EXPR(val1)     \
   while (false)               \
   ::arrow::internal::NullLog()
 #define DCHECK_NE(val1, val2) \
+  ARROW_IGNORE_EXPR(val1)     \
   while (false)               \
   ::arrow::internal::NullLog()
 #define DCHECK_LE(val1, val2) \
+  ARROW_IGNORE_EXPR(val1)     \
   while (false)               \
   ::arrow::internal::NullLog()
 #define DCHECK_LT(val1, val2) \
+  ARROW_IGNORE_EXPR(val1)     \
   while (false)               \
   ::arrow::internal::NullLog()
 #define DCHECK_GE(val1, val2) \
+  ARROW_IGNORE_EXPR(val1)     \
   while (false)               \
   ::arrow::internal::NullLog()
 #define DCHECK_GT(val1, val2) \
+  ARROW_IGNORE_EXPR(val1)     \
   while (false)               \
   ::arrow::internal::NullLog()
 
diff --git a/cpp/src/arrow/util/rle-encoding-test.cc b/cpp/src/arrow/util/rle-encoding-test.cc
new file mode 100644
index 0000000000000..7c9b33c349496
--- /dev/null
+++ b/cpp/src/arrow/util/rle-encoding-test.cc
@@ -0,0 +1,460 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// From Apache Impala as of 2016-01-29
+
+#include <gtest/gtest.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <boost/utility.hpp>
+
+#include <cstdint>
+#include <iostream>
+#include <random>
+#include <vector>
+
+#include "arrow/util/bit-stream-utils.h"
+#include "arrow/util/rle-encoding.h"
+
+using std::vector;
+
+namespace arrow {
+
+const int MAX_WIDTH = 32;
+
+TEST(BitArray, TestBool) {
+  const int len = 8;
+  uint8_t buffer[len];
+
+  BitWriter writer(buffer, len);
+
+  // Write alternating 0's and 1's
+  for (int i = 0; i < 8; ++i) {
+    bool result = writer.PutValue(i % 2, 1);
+    EXPECT_TRUE(result);
+  }
+  writer.Flush();
+  EXPECT_EQ((int)buffer[0], BOOST_BINARY(1 0 1 0 1 0 1 0));
+
+  // Write 00110011
+  for (int i = 0; i < 8; ++i) {
+    bool result = false;
+    switch (i) {
+      case 0:
+      case 1:
+      case 4:
+      case 5:
+        result = writer.PutValue(false, 1);
+        break;
+      default:
+        result = writer.PutValue(true, 1);
+        break;
+    }
+    EXPECT_TRUE(result);
+  }
+  writer.Flush();
+
+  // Validate the exact bit value
+  EXPECT_EQ((int)buffer[0], BOOST_BINARY(1 0 1 0 1 0 1 0));
+  EXPECT_EQ((int)buffer[1], BOOST_BINARY(1 1 0 0 1 1 0 0));
+
+  // Use the reader and validate
+  BitReader reader(buffer, len);
+  for (int i = 0; i < 8; ++i) {
+    bool val = false;
+    bool result = reader.GetValue(1, &val);
+    EXPECT_TRUE(result);
+    EXPECT_EQ(val, (i % 2) != 0);
+  }
+
+  for (int i = 0; i < 8; ++i) {
+    bool val = false;
+    bool result = reader.GetValue(1, &val);
+    EXPECT_TRUE(result);
+    switch (i) {
+      case 0:
+      case 1:
+      case 4:
+      case 5:
+        EXPECT_EQ(val, false);
+        break;
+      default:
+        EXPECT_EQ(val, true);
+        break;
+    }
+  }
+}
+
+// Writes 'num_vals' values with width 'bit_width' and reads them back.
+void TestBitArrayValues(int bit_width, int num_vals) {
+  int len = static_cast<int>(BitUtil::Ceil(bit_width * num_vals, 8));
+  EXPECT_GT(len, 0);
+  const uint64_t mod = bit_width == 64 ? 1 : 1LL << bit_width;
+
+  std::vector<uint8_t> buffer(len);
+  BitWriter writer(buffer.data(), len);
+  for (int i = 0; i < num_vals; ++i) {
+    bool result = writer.PutValue(i % mod, bit_width);
+    EXPECT_TRUE(result);
+  }
+  writer.Flush();
+  EXPECT_EQ(writer.bytes_written(), len);
+
+  BitReader reader(buffer.data(), len);
+  for (int i = 0; i < num_vals; ++i) {
+    int64_t val = 0;
+    bool result = reader.GetValue(bit_width, &val);
+    EXPECT_TRUE(result);
+    EXPECT_EQ(val, i % mod);
+  }
+  EXPECT_EQ(reader.bytes_left(), 0);
+}
+
+TEST(BitArray, TestValues) {
+  for (int width = 1; width <= MAX_WIDTH; ++width) {
+    TestBitArrayValues(width, 1);
+    TestBitArrayValues(width, 2);
+    // Don't write too many values
+    TestBitArrayValues(width, (width < 12) ? (1 << width) : 4096);
+    TestBitArrayValues(width, 1024);
+  }
+}
+
+// Test some mixed values
+TEST(BitArray, TestMixed) {
+  const int len = 1024;
+  uint8_t buffer[len];
+  bool parity = true;
+
+  BitWriter writer(buffer, len);
+  for (int i = 0; i < len; ++i) {
+    bool result;
+    if (i % 2 == 0) {
+      result = writer.PutValue(parity, 1);
+      parity = !parity;
+    } else {
+      result = writer.PutValue(i, 10);
+    }
+    EXPECT_TRUE(result);
+  }
+  writer.Flush();
+
+  parity = true;
+  BitReader reader(buffer, len);
+  for (int i = 0; i < len; ++i) {
+    bool result;
+    if (i % 2 == 0) {
+      bool val;
+      result = reader.GetValue(1, &val);
+      EXPECT_EQ(val, parity);
+      parity = !parity;
+    } else {
+      int val;
+      result = reader.GetValue(10, &val);
+      EXPECT_EQ(val, i);
+    }
+    EXPECT_TRUE(result);
+  }
+}
+
+// Validates encoding of values by encoding and decoding them.  If
+// expected_encoding != NULL, also validates that the encoded buffer is
+// exactly 'expected_encoding'.
+// if expected_len is not -1, it will validate the encoded size is correct.
+void ValidateRle(const vector<int>& values, int bit_width, uint8_t* expected_encoding,
+    int expected_len) {
+  const int len = 64 * 1024;
+  uint8_t buffer[len];
+  EXPECT_LE(expected_len, len);
+
+  RleEncoder encoder(buffer, len, bit_width);
+  for (size_t i = 0; i < values.size(); ++i) {
+    bool result = encoder.Put(values[i]);
+    EXPECT_TRUE(result);
+  }
+  int encoded_len = encoder.Flush();
+
+  if (expected_len != -1) { EXPECT_EQ(encoded_len, expected_len); }
+  if (expected_encoding != NULL) {
+    EXPECT_EQ(memcmp(buffer, expected_encoding, expected_len), 0);
+  }
+
+  // Verify read
+  {
+    RleDecoder decoder(buffer, len, bit_width);
+    for (size_t i = 0; i < values.size(); ++i) {
+      uint64_t val;
+      bool result = decoder.Get(&val);
+      EXPECT_TRUE(result);
+      EXPECT_EQ(values[i], val);
+    }
+  }
+
+  // Verify batch read
+  {
+    RleDecoder decoder(buffer, len, bit_width);
+    vector<int> values_read(values.size());
+    ASSERT_EQ(values.size(),
+        decoder.GetBatch(values_read.data(), static_cast<int>(values.size())));
+    EXPECT_EQ(values, values_read);
+  }
+}
+
+// A version of ValidateRle that round-trips the values and returns false if
+// the returned values are not all the same
+bool CheckRoundTrip(const vector<int>& values, int bit_width) {
+  const int len = 64 * 1024;
+  uint8_t buffer[len];
+  RleEncoder encoder(buffer, len, bit_width);
+  for (size_t i = 0; i < values.size(); ++i) {
+    bool result = encoder.Put(values[i]);
+    if (!result) { return false; }
+  }
+  int encoded_len = encoder.Flush();
+  int out = 0;
+
+  {
+    RleDecoder decoder(buffer, encoded_len, bit_width);
+    for (size_t i = 0; i < values.size(); ++i) {
+      EXPECT_TRUE(decoder.Get(&out));
+      if (values[i] != out) { return false; }
+    }
+  }
+
+  // Verify batch read
+  {
+    RleDecoder decoder(buffer, len, bit_width);
+    vector<int> values_read(values.size());
+    if (static_cast<int>(values.size()) !=
+        decoder.GetBatch(values_read.data(), static_cast<int>(values.size()))) {
+      return false;
+    }
+    if (values != values_read) { return false; }
+  }
+
+  return true;
+}
+
+TEST(Rle, SpecificSequences) {
+  const int len = 1024;
+  uint8_t expected_buffer[len];
+  vector<int> values;
+
+  // Test 50 0' followed by 50 1's
+  values.resize(100);
+  for (int i = 0; i < 50; ++i) {
+    values[i] = 0;
+  }
+  for (int i = 50; i < 100; ++i) {
+    values[i] = 1;
+  }
+
+  // expected_buffer valid for bit width <= 1 byte
+  expected_buffer[0] = (50 << 1);
+  expected_buffer[1] = 0;
+  expected_buffer[2] = (50 << 1);
+  expected_buffer[3] = 1;
+  for (int width = 1; width <= 8; ++width) {
+    ValidateRle(values, width, expected_buffer, 4);
+  }
+
+  for (int width = 9; width <= MAX_WIDTH; ++width) {
+    ValidateRle(values, width, NULL, 2 * (1 + static_cast<int>(BitUtil::Ceil(width, 8))));
+  }
+
+  // Test 100 0's and 1's alternating
+  for (int i = 0; i < 100; ++i) {
+    values[i] = i % 2;
+  }
+  int num_groups = static_cast<int>(BitUtil::Ceil(100, 8));
+  expected_buffer[0] = static_cast<uint8_t>((num_groups << 1) | 1);
+  for (int i = 1; i <= 100 / 8; ++i) {
+    expected_buffer[i] = BOOST_BINARY(1 0 1 0 1 0 1 0);
+  }
+  // Values for the last 4 0 and 1's. The upper 4 bits should be padded to 0.
+  expected_buffer[100 / 8 + 1] = BOOST_BINARY(0 0 0 0 1 0 1 0);
+
+  // num_groups and expected_buffer only valid for bit width = 1
+  ValidateRle(values, 1, expected_buffer, 1 + num_groups);
+  for (int width = 2; width <= MAX_WIDTH; ++width) {
+    int num_values = static_cast<int>(BitUtil::Ceil(100, 8)) * 8;
+    ValidateRle(
+        values, width, NULL, 1 + static_cast<int>(BitUtil::Ceil(width * num_values, 8)));
+  }
+}
+
+// ValidateRle on 'num_vals' values with width 'bit_width'. If 'value' != -1, that value
+// is used, otherwise alternating values are used.
+void TestRleValues(int bit_width, int num_vals, int value = -1) {
+  const uint64_t mod = (bit_width == 64) ? 1 : 1LL << bit_width;
+  vector<int> values;
+  for (int v = 0; v < num_vals; ++v) {
+    values.push_back((value != -1) ? value : static_cast<int>(v % mod));
+  }
+  ValidateRle(values, bit_width, NULL, -1);
+}
+
+TEST(Rle, TestValues) {
+  for (int width = 1; width <= MAX_WIDTH; ++width) {
+    TestRleValues(width, 1);
+    TestRleValues(width, 1024);
+    TestRleValues(width, 1024, 0);
+    TestRleValues(width, 1024, 1);
+  }
+}
+
+TEST(Rle, BitWidthZeroRepeated) {
+  uint8_t buffer[1];
+  const int num_values = 15;
+  buffer[0] = num_values << 1;  // repeated indicator byte
+  RleDecoder decoder(buffer, sizeof(buffer), 0);
+  uint8_t val;
+  for (int i = 0; i < num_values; ++i) {
+    bool result = decoder.Get(&val);
+    EXPECT_TRUE(result);
+    EXPECT_EQ(val, 0);  // can only encode 0s with bit width 0
+  }
+  EXPECT_FALSE(decoder.Get(&val));
+}
+
+TEST(Rle, BitWidthZeroLiteral) {
+  uint8_t buffer[1];
+  const int num_groups = 4;
+  buffer[0] = num_groups << 1 | 1;  // literal indicator byte
+  RleDecoder decoder = RleDecoder(buffer, sizeof(buffer), 0);
+  const int num_values = num_groups * 8;
+  uint8_t val;
+  for (int i = 0; i < num_values; ++i) {
+    bool result = decoder.Get(&val);
+    EXPECT_TRUE(result);
+    EXPECT_EQ(val, 0);  // can only encode 0s with bit width 0
+  }
+  EXPECT_FALSE(decoder.Get(&val));
+}
+
+// Test that writes out a repeated group and then a literal
+// group but flush before finishing.
+TEST(BitRle, Flush) {
+  vector<int> values;
+  for (int i = 0; i < 16; ++i)
+    values.push_back(1);
+  values.push_back(0);
+  ValidateRle(values, 1, NULL, -1);
+  values.push_back(1);
+  ValidateRle(values, 1, NULL, -1);
+  values.push_back(1);
+  ValidateRle(values, 1, NULL, -1);
+  values.push_back(1);
+  ValidateRle(values, 1, NULL, -1);
+}
+
+// Test some random sequences.
+TEST(BitRle, Random) {
+  int niters = 50;
+  int ngroups = 1000;
+  int max_group_size = 16;
+  vector<int> values(ngroups + max_group_size);
+
+  // prng setup
+  std::random_device rd;
+  std::uniform_int_distribution<int> dist(1, 20);
+
+  for (int iter = 0; iter < niters; ++iter) {
+    // generate a seed with device entropy
+    uint32_t seed = rd();
+    std::mt19937 gen(seed);
+
+    bool parity = 0;
+    values.resize(0);
+
+    for (int i = 0; i < ngroups; ++i) {
+      int group_size = dist(gen);
+      if (group_size > max_group_size) { group_size = 1; }
+      for (int i = 0; i < group_size; ++i) {
+        values.push_back(parity);
+      }
+      parity = !parity;
+    }
+    if (!CheckRoundTrip(values, BitUtil::NumRequiredBits(values.size()))) {
+      FAIL() << "failing seed: " << seed;
+    }
+  }
+}
+
+// Test a sequence of 1 0's, 2 1's, 3 0's. etc
+// e.g. 011000111100000
+TEST(BitRle, RepeatedPattern) {
+  vector<int> values;
+  const int min_run = 1;
+  const int max_run = 32;
+
+  for (int i = min_run; i <= max_run; ++i) {
+    int v = i % 2;
+    for (int j = 0; j < i; ++j) {
+      values.push_back(v);
+    }
+  }
+
+  // And go back down again
+  for (int i = max_run; i >= min_run; --i) {
+    int v = i % 2;
+    for (int j = 0; j < i; ++j) {
+      values.push_back(v);
+    }
+  }
+
+  ValidateRle(values, 1, NULL, -1);
+}
+
+TEST(BitRle, Overflow) {
+  for (int bit_width = 1; bit_width < 32; bit_width += 3) {
+    int len = RleEncoder::MinBufferSize(bit_width);
+    std::vector<uint8_t> buffer(len);
+    int num_added = 0;
+    bool parity = true;
+
+    RleEncoder encoder(buffer.data(), len, bit_width);
+    // Insert alternating true/false until there is no space left
+    while (true) {
+      bool result = encoder.Put(parity);
+      parity = !parity;
+      if (!result) break;
+      ++num_added;
+    }
+
+    int bytes_written = encoder.Flush();
+    EXPECT_LE(bytes_written, len);
+    EXPECT_GT(num_added, 0);
+
+    RleDecoder decoder(buffer.data(), bytes_written, bit_width);
+    parity = true;
+    uint32_t v;
+    for (int i = 0; i < num_added; ++i) {
+      bool result = decoder.Get(&v);
+      EXPECT_TRUE(result);
+      EXPECT_EQ(v != 0, parity);
+      parity = !parity;
+    }
+    // Make sure we get false when reading past end a couple times.
+    EXPECT_FALSE(decoder.Get(&v));
+    EXPECT_FALSE(decoder.Get(&v));
+  }
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/rle-encoding.h b/cpp/src/arrow/util/rle-encoding.h
new file mode 100644
index 0000000000000..9ec6235144665
--- /dev/null
+++ b/cpp/src/arrow/util/rle-encoding.h
@@ -0,0 +1,598 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Imported from Apache Impala (incubating) on 2016-01-29 and modified for use
+// in parquet-cpp, Arrow
+
+#ifndef ARROW_UTIL_RLE_ENCODING_H
+#define ARROW_UTIL_RLE_ENCODING_H
+
+#include <algorithm>
+#include <math.h>
+
+#include "arrow/util/bit-stream-utils.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/compiler-util.h"
+
+namespace arrow {
+
+/// Utility classes to do run length encoding (RLE) for fixed bit width values.  If runs
+/// are sufficiently long, RLE is used, otherwise, the values are just bit-packed
+/// (literal encoding).
+/// For both types of runs, there is a byte-aligned indicator which encodes the length
+/// of the run and the type of the run.
+/// This encoding has the benefit that when there aren't any long enough runs, values
+/// are always decoded at fixed (can be precomputed) bit offsets OR both the value and
+/// the run length are byte aligned. This allows for very efficient decoding
+/// implementations.
+/// The encoding is:
+///    encoded-block := run*
+///    run := literal-run | repeated-run
+///    literal-run := literal-indicator < literal bytes >
+///    repeated-run := repeated-indicator < repeated value. padded to byte boundary >
+///    literal-indicator := varint_encode( number_of_groups << 1 | 1)
+///    repeated-indicator := varint_encode( number_of_repetitions << 1 )
+//
+/// Each run is preceded by a varint. The varint's least significant bit is
+/// used to indicate whether the run is a literal run or a repeated run. The rest
+/// of the varint is used to determine the length of the run (eg how many times the
+/// value repeats).
+//
+/// In the case of literal runs, the run length is always a multiple of 8 (i.e. encode
+/// in groups of 8), so that no matter the bit-width of the value, the sequence will end
+/// on a byte boundary without padding.
+/// Given that we know it is a multiple of 8, we store the number of 8-groups rather than
+/// the actual number of encoded ints. (This means that the total number of encoded values
+/// can not be determined from the encoded data, since the number of values in the last
+/// group may not be a multiple of 8). For the last group of literal runs, we pad
+/// the group to 8 with zeros. This allows for 8 at a time decoding on the read side
+/// without the need for additional checks.
+//
+/// There is a break-even point when it is more storage efficient to do run length
+/// encoding.  For 1 bit-width values, that point is 8 values.  They require 2 bytes
+/// for both the repeated encoding or the literal encoding.  This value can always
+/// be computed based on the bit-width.
+/// TODO: think about how to use this for strings.  The bit packing isn't quite the same.
+//
+/// Examples with bit-width 1 (eg encoding booleans):
+/// ----------------------------------------
+/// 100 1s followed by 100 0s:
+/// <varint(100 << 1)> <1, padded to 1 byte>  <varint(100 << 1)> <0, padded to 1 byte>
+///  - (total 4 bytes)
+//
+/// alternating 1s and 0s (200 total):
+/// 200 ints = 25 groups of 8
+/// <varint((25 << 1) | 1)> <25 bytes of values, bitpacked>
+/// (total 26 bytes, 1 byte overhead)
+//
+
+/// Decoder class for RLE encoded data.
+class RleDecoder {
+ public:
+  /// Create a decoder object. buffer/buffer_len is the decoded data.
+  /// bit_width is the width of each value (before encoding).
+  RleDecoder(const uint8_t* buffer, int buffer_len, int bit_width)
+      : bit_reader_(buffer, buffer_len),
+        bit_width_(bit_width),
+        current_value_(0),
+        repeat_count_(0),
+        literal_count_(0) {
+    DCHECK_GE(bit_width_, 0);
+    DCHECK_LE(bit_width_, 64);
+  }
+
+  RleDecoder() : bit_width_(-1) {}
+
+  void Reset(const uint8_t* buffer, int buffer_len, int bit_width) {
+    DCHECK_GE(bit_width, 0);
+    DCHECK_LE(bit_width, 64);
+    bit_reader_.Reset(buffer, buffer_len);
+    bit_width_ = bit_width;
+    current_value_ = 0;
+    repeat_count_ = 0;
+    literal_count_ = 0;
+  }
+
+  /// Gets the next value.  Returns false if there are no more.
+  template <typename T>
+  bool Get(T* val);
+
+  /// Gets a batch of values.  Returns the number of decoded elements.
+  template <typename T>
+  int GetBatch(T* values, int batch_size);
+
+  /// Like GetBatch but the values are then decoded using the provided dictionary
+  template <typename T>
+  int GetBatchWithDict(const T* dictionary, T* values, int batch_size);
+
+  /// Like GetBatchWithDict but add spacing for null entries
+  template <typename T>
+  int GetBatchWithDictSpaced(const T* dictionary, T* values, int batch_size,
+      int null_count, const uint8_t* valid_bits, int64_t valid_bits_offset);
+
+ protected:
+  BitReader bit_reader_;
+  /// Number of bits needed to encode the value. Must be between 0 and 64.
+  int bit_width_;
+  uint64_t current_value_;
+  uint32_t repeat_count_;
+  uint32_t literal_count_;
+
+ private:
+  /// Fills literal_count_ and repeat_count_ with next values. Returns false if there
+  /// are no more.
+  template <typename T>
+  bool NextCounts();
+};
+
+/// Class to incrementally build the rle data.   This class does not allocate any memory.
+/// The encoding has two modes: encoding repeated runs and literal runs.
+/// If the run is sufficiently short, it is more efficient to encode as a literal run.
+/// This class does so by buffering 8 values at a time.  If they are not all the same
+/// they are added to the literal run.  If they are the same, they are added to the
+/// repeated run.  When we switch modes, the previous run is flushed out.
+class RleEncoder {
+ public:
+  /// buffer/buffer_len: preallocated output buffer.
+  /// bit_width: max number of bits for value.
+  /// TODO: consider adding a min_repeated_run_length so the caller can control
+  /// when values should be encoded as repeated runs.  Currently this is derived
+  /// based on the bit_width, which can determine a storage optimal choice.
+  /// TODO: allow 0 bit_width (and have dict encoder use it)
+  RleEncoder(uint8_t* buffer, int buffer_len, int bit_width)
+      : bit_width_(bit_width), bit_writer_(buffer, buffer_len) {
+    DCHECK_GE(bit_width_, 0);
+    DCHECK_LE(bit_width_, 64);
+    max_run_byte_size_ = MinBufferSize(bit_width);
+    DCHECK_GE(buffer_len, max_run_byte_size_) << "Input buffer not big enough.";
+    Clear();
+  }
+
+  /// Returns the minimum buffer size needed to use the encoder for 'bit_width'
+  /// This is the maximum length of a single run for 'bit_width'.
+  /// It is not valid to pass a buffer less than this length.
+  static int MinBufferSize(int bit_width) {
+    /// 1 indicator byte and MAX_VALUES_PER_LITERAL_RUN 'bit_width' values.
+    int max_literal_run_size =
+        1 + static_cast<int>(BitUtil::Ceil(MAX_VALUES_PER_LITERAL_RUN * bit_width, 8));
+    /// Up to MAX_VLQ_BYTE_LEN indicator and a single 'bit_width' value.
+    int max_repeated_run_size =
+        BitReader::MAX_VLQ_BYTE_LEN + static_cast<int>(BitUtil::Ceil(bit_width, 8));
+    return std::max(max_literal_run_size, max_repeated_run_size);
+  }
+
+  /// Returns the maximum byte size it could take to encode 'num_values'.
+  static int MaxBufferSize(int bit_width, int num_values) {
+    // For a bit_width > 1, the worst case is the repetition of "literal run of length 8
+    // and then a repeated run of length 8".
+    // 8 values per smallest run, 8 bits per byte
+    // int bytes_per_run = BitUtil::Ceil(bit_width * 8, 8);
+    int bytes_per_run = bit_width;
+    int num_runs = static_cast<int>(BitUtil::Ceil(num_values, 8));
+    int literal_max_size = num_runs + num_runs * bytes_per_run;
+
+    // In the very worst case scenario, the data is a concatenation of repeated
+    // runs of 8 values. Repeated run has a 1 byte varint followed by the
+    // bit-packed repeated value
+    int min_repeated_run_size = 1 + static_cast<int>(BitUtil::Ceil(bit_width, 8));
+    int repeated_max_size =
+        static_cast<int>(BitUtil::Ceil(num_values, 8)) * min_repeated_run_size;
+
+    return std::max(literal_max_size, repeated_max_size);
+  }
+
+  /// Encode value.  Returns true if the value fits in buffer, false otherwise.
+  /// This value must be representable with bit_width_ bits.
+  bool Put(uint64_t value);
+
+  /// Flushes any pending values to the underlying buffer.
+  /// Returns the total number of bytes written
+  int Flush();
+
+  /// Resets all the state in the encoder.
+  void Clear();
+
+  /// Returns pointer to underlying buffer
+  uint8_t* buffer() { return bit_writer_.buffer(); }
+  int32_t len() { return bit_writer_.bytes_written(); }
+
+ private:
+  /// Flushes any buffered values.  If this is part of a repeated run, this is largely
+  /// a no-op.
+  /// If it is part of a literal run, this will call FlushLiteralRun, which writes
+  /// out the buffered literal values.
+  /// If 'done' is true, the current run would be written even if it would normally
+  /// have been buffered more.  This should only be called at the end, when the
+  /// encoder has received all values even if it would normally continue to be
+  /// buffered.
+  void FlushBufferedValues(bool done);
+
+  /// Flushes literal values to the underlying buffer.  If update_indicator_byte,
+  /// then the current literal run is complete and the indicator byte is updated.
+  void FlushLiteralRun(bool update_indicator_byte);
+
+  /// Flushes a repeated run to the underlying buffer.
+  void FlushRepeatedRun();
+
+  /// Checks and sets buffer_full_. This must be called after flushing a run to
+  /// make sure there are enough bytes remaining to encode the next run.
+  void CheckBufferFull();
+
+  /// The maximum number of values in a single literal run
+  /// (number of groups encodable by a 1-byte indicator * 8)
+  static const int MAX_VALUES_PER_LITERAL_RUN = (1 << 6) * 8;
+
+  /// Number of bits needed to encode the value. Must be between 0 and 64.
+  const int bit_width_;
+
+  /// Underlying buffer.
+  BitWriter bit_writer_;
+
+  /// If true, the buffer is full and subsequent Put()'s will fail.
+  bool buffer_full_;
+
+  /// The maximum byte size a single run can take.
+  int max_run_byte_size_;
+
+  /// We need to buffer at most 8 values for literals.  This happens when the
+  /// bit_width is 1 (so 8 values fit in one byte).
+  /// TODO: generalize this to other bit widths
+  int64_t buffered_values_[8];
+
+  /// Number of values in buffered_values_
+  int num_buffered_values_;
+
+  /// The current (also last) value that was written and the count of how
+  /// many times in a row that value has been seen.  This is maintained even
+  /// if we are in a literal run.  If the repeat_count_ get high enough, we switch
+  /// to encoding repeated runs.
+  uint64_t current_value_;
+  int repeat_count_;
+
+  /// Number of literals in the current run.  This does not include the literals
+  /// that might be in buffered_values_.  Only after we've got a group big enough
+  /// can we decide if they should part of the literal_count_ or repeat_count_
+  int literal_count_;
+
+  /// Pointer to a byte in the underlying buffer that stores the indicator byte.
+  /// This is reserved as soon as we need a literal run but the value is written
+  /// when the literal run is complete.
+  uint8_t* literal_indicator_byte_;
+};
+
+template <typename T>
+inline bool RleDecoder::Get(T* val) {
+  return GetBatch(val, 1) == 1;
+}
+
+template <typename T>
+inline int RleDecoder::GetBatch(T* values, int batch_size) {
+  DCHECK_GE(bit_width_, 0);
+  int values_read = 0;
+
+  while (values_read < batch_size) {
+    if (repeat_count_ > 0) {
+      int repeat_batch =
+          std::min(batch_size - values_read, static_cast<int>(repeat_count_));
+      std::fill(values + values_read, values + values_read + repeat_batch,
+          static_cast<T>(current_value_));
+      repeat_count_ -= repeat_batch;
+      values_read += repeat_batch;
+    } else if (literal_count_ > 0) {
+      int literal_batch =
+          std::min(batch_size - values_read, static_cast<int>(literal_count_));
+      int actual_read =
+          bit_reader_.GetBatch(bit_width_, values + values_read, literal_batch);
+      DCHECK_EQ(actual_read, literal_batch);
+      literal_count_ -= literal_batch;
+      values_read += literal_batch;
+    } else {
+      if (!NextCounts<T>()) return values_read;
+    }
+  }
+
+  return values_read;
+}
+
+template <typename T>
+inline int RleDecoder::GetBatchWithDict(const T* dictionary, T* values, int batch_size) {
+  DCHECK_GE(bit_width_, 0);
+  int values_read = 0;
+
+  while (values_read < batch_size) {
+    if (repeat_count_ > 0) {
+      int repeat_batch =
+          std::min(batch_size - values_read, static_cast<int>(repeat_count_));
+      std::fill(values + values_read, values + values_read + repeat_batch,
+          dictionary[current_value_]);
+      repeat_count_ -= repeat_batch;
+      values_read += repeat_batch;
+    } else if (literal_count_ > 0) {
+      int literal_batch =
+          std::min(batch_size - values_read, static_cast<int>(literal_count_));
+
+      const int buffer_size = 1024;
+      int indices[buffer_size];
+      literal_batch = std::min(literal_batch, buffer_size);
+      int actual_read = bit_reader_.GetBatch(bit_width_, &indices[0], literal_batch);
+      DCHECK_EQ(actual_read, literal_batch);
+      for (int i = 0; i < literal_batch; ++i) {
+        values[values_read + i] = dictionary[indices[i]];
+      }
+      literal_count_ -= literal_batch;
+      values_read += literal_batch;
+    } else {
+      if (!NextCounts<T>()) return values_read;
+    }
+  }
+
+  return values_read;
+}
+
+template <typename T>
+inline int RleDecoder::GetBatchWithDictSpaced(const T* dictionary, T* values,
+    int batch_size, int null_count, const uint8_t* valid_bits,
+    int64_t valid_bits_offset) {
+  DCHECK_GE(bit_width_, 0);
+  int values_read = 0;
+  int remaining_nulls = null_count;
+  INIT_BITSET(valid_bits, static_cast<int>(valid_bits_offset));
+
+  while (values_read < batch_size) {
+    bool is_valid = (bitset_valid_bits & (1 << bit_offset_valid_bits)) != 0;
+    READ_NEXT_BITSET(valid_bits);
+
+    if (is_valid) {
+      if ((repeat_count_ == 0) && (literal_count_ == 0)) {
+        if (!NextCounts<T>()) return values_read;
+      }
+      if (repeat_count_ > 0) {
+        T value = dictionary[current_value_];
+        // The current index is already valid, we don't need to check that again
+        int repeat_batch = 1;
+        repeat_count_--;
+
+        while (repeat_count_ > 0 && (values_read + repeat_batch) < batch_size) {
+          if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+            repeat_count_--;
+          } else {
+            remaining_nulls--;
+          }
+          repeat_batch++;
+
+          READ_NEXT_BITSET(valid_bits);
+        }
+        std::fill(values + values_read, values + values_read + repeat_batch, value);
+        values_read += repeat_batch;
+      } else if (literal_count_ > 0) {
+        int literal_batch = std::min(
+            batch_size - values_read - remaining_nulls, static_cast<int>(literal_count_));
+
+        // Decode the literals
+        constexpr int kBufferSize = 1024;
+        int indices[kBufferSize];
+        literal_batch = std::min(literal_batch, kBufferSize);
+        int actual_read = bit_reader_.GetBatch(bit_width_, &indices[0], literal_batch);
+        DCHECK_EQ(actual_read, literal_batch);
+
+        int skipped = 0;
+        int literals_read = 1;
+        values[values_read] = dictionary[indices[0]];
+
+        // Read the first bitset to the end
+        while (literals_read < literal_batch) {
+          if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+            values[values_read + literals_read + skipped] =
+                dictionary[indices[literals_read]];
+            literals_read++;
+          } else {
+            skipped++;
+          }
+
+          READ_NEXT_BITSET(valid_bits);
+        }
+        literal_count_ -= literal_batch;
+        values_read += literal_batch + skipped;
+        remaining_nulls -= skipped;
+      }
+    } else {
+      values_read++;
+      remaining_nulls--;
+    }
+  }
+
+  return values_read;
+}
+
+template <typename T>
+bool RleDecoder::NextCounts() {
+  // Read the next run's indicator int, it could be a literal or repeated run.
+  // The int is encoded as a vlq-encoded value.
+  int32_t indicator_value = 0;
+  bool result = bit_reader_.GetVlqInt(&indicator_value);
+  if (!result) return false;
+
+  // lsb indicates if it is a literal run or repeated run
+  bool is_literal = indicator_value & 1;
+  if (is_literal) {
+    literal_count_ = (indicator_value >> 1) * 8;
+  } else {
+    repeat_count_ = indicator_value >> 1;
+    bool result =
+        bit_reader_.GetAligned<T>(static_cast<int>(BitUtil::Ceil(bit_width_, 8)),
+            reinterpret_cast<T*>(&current_value_));
+    DCHECK(result);
+  }
+  return true;
+}
+
+/// This function buffers input values 8 at a time.  After seeing all 8 values,
+/// it decides whether they should be encoded as a literal or repeated run.
+inline bool RleEncoder::Put(uint64_t value) {
+  DCHECK(bit_width_ == 64 || value < (1ULL << bit_width_));
+  if (UNLIKELY(buffer_full_)) return false;
+
+  if (LIKELY(current_value_ == value)) {
+    ++repeat_count_;
+    if (repeat_count_ > 8) {
+      // This is just a continuation of the current run, no need to buffer the
+      // values.
+      // Note that this is the fast path for long repeated runs.
+      return true;
+    }
+  } else {
+    if (repeat_count_ >= 8) {
+      // We had a run that was long enough but it has ended.  Flush the
+      // current repeated run.
+      DCHECK_EQ(literal_count_, 0);
+      FlushRepeatedRun();
+    }
+    repeat_count_ = 1;
+    current_value_ = value;
+  }
+
+  buffered_values_[num_buffered_values_] = value;
+  if (++num_buffered_values_ == 8) {
+    DCHECK_EQ(literal_count_ % 8, 0);
+    FlushBufferedValues(false);
+  }
+  return true;
+}
+
+inline void RleEncoder::FlushLiteralRun(bool update_indicator_byte) {
+  if (literal_indicator_byte_ == NULL) {
+    // The literal indicator byte has not been reserved yet, get one now.
+    literal_indicator_byte_ = bit_writer_.GetNextBytePtr();
+    DCHECK(literal_indicator_byte_ != NULL);
+  }
+
+  // Write all the buffered values as bit packed literals
+  for (int i = 0; i < num_buffered_values_; ++i) {
+    bool success = bit_writer_.PutValue(buffered_values_[i], bit_width_);
+    DCHECK(success) << "There is a bug in using CheckBufferFull()";
+  }
+  num_buffered_values_ = 0;
+
+  if (update_indicator_byte) {
+    // At this point we need to write the indicator byte for the literal run.
+    // We only reserve one byte, to allow for streaming writes of literal values.
+    // The logic makes sure we flush literal runs often enough to not overrun
+    // the 1 byte.
+    DCHECK_EQ(literal_count_ % 8, 0);
+    int num_groups = literal_count_ / 8;
+    int32_t indicator_value = (num_groups << 1) | 1;
+    DCHECK_EQ(indicator_value & 0xFFFFFF00, 0);
+    *literal_indicator_byte_ = static_cast<uint8_t>(indicator_value);
+    literal_indicator_byte_ = NULL;
+    literal_count_ = 0;
+    CheckBufferFull();
+  }
+}
+
+inline void RleEncoder::FlushRepeatedRun() {
+  DCHECK_GT(repeat_count_, 0);
+  bool result = true;
+  // The lsb of 0 indicates this is a repeated run
+  int32_t indicator_value = repeat_count_ << 1 | 0;
+  result &= bit_writer_.PutVlqInt(indicator_value);
+  result &= bit_writer_.PutAligned(
+      current_value_, static_cast<int>(BitUtil::Ceil(bit_width_, 8)));
+  DCHECK(result);
+  num_buffered_values_ = 0;
+  repeat_count_ = 0;
+  CheckBufferFull();
+}
+
+/// Flush the values that have been buffered.  At this point we decide whether
+/// we need to switch between the run types or continue the current one.
+inline void RleEncoder::FlushBufferedValues(bool done) {
+  if (repeat_count_ >= 8) {
+    // Clear the buffered values.  They are part of the repeated run now and we
+    // don't want to flush them out as literals.
+    num_buffered_values_ = 0;
+    if (literal_count_ != 0) {
+      // There was a current literal run.  All the values in it have been flushed
+      // but we still need to update the indicator byte.
+      DCHECK_EQ(literal_count_ % 8, 0);
+      DCHECK_EQ(repeat_count_, 8);
+      FlushLiteralRun(true);
+    }
+    DCHECK_EQ(literal_count_, 0);
+    return;
+  }
+
+  literal_count_ += num_buffered_values_;
+  DCHECK_EQ(literal_count_ % 8, 0);
+  int num_groups = literal_count_ / 8;
+  if (num_groups + 1 >= (1 << 6)) {
+    // We need to start a new literal run because the indicator byte we've reserved
+    // cannot store more values.
+    DCHECK(literal_indicator_byte_ != NULL);
+    FlushLiteralRun(true);
+  } else {
+    FlushLiteralRun(done);
+  }
+  repeat_count_ = 0;
+}
+
+inline int RleEncoder::Flush() {
+  if (literal_count_ > 0 || repeat_count_ > 0 || num_buffered_values_ > 0) {
+    bool all_repeat = literal_count_ == 0 && (repeat_count_ == num_buffered_values_ ||
+                                                 num_buffered_values_ == 0);
+    // There is something pending, figure out if it's a repeated or literal run
+    if (repeat_count_ > 0 && all_repeat) {
+      FlushRepeatedRun();
+    } else {
+      DCHECK_EQ(literal_count_ % 8, 0);
+      // Buffer the last group of literals to 8 by padding with 0s.
+      for (; num_buffered_values_ != 0 && num_buffered_values_ < 8;
+           ++num_buffered_values_) {
+        buffered_values_[num_buffered_values_] = 0;
+      }
+      literal_count_ += num_buffered_values_;
+      FlushLiteralRun(true);
+      repeat_count_ = 0;
+    }
+  }
+  bit_writer_.Flush();
+  DCHECK_EQ(num_buffered_values_, 0);
+  DCHECK_EQ(literal_count_, 0);
+  DCHECK_EQ(repeat_count_, 0);
+
+  return bit_writer_.bytes_written();
+}
+
+inline void RleEncoder::CheckBufferFull() {
+  int bytes_written = bit_writer_.bytes_written();
+  if (bytes_written + max_run_byte_size_ > bit_writer_.buffer_len()) {
+    buffer_full_ = true;
+  }
+}
+
+inline void RleEncoder::Clear() {
+  buffer_full_ = false;
+  current_value_ = 0;
+  repeat_count_ = 0;
+  num_buffered_values_ = 0;
+  literal_count_ = 0;
+  literal_indicator_byte_ = NULL;
+  bit_writer_.Clear();
+}
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_RLE_ENCODING_H
diff --git a/cpp/src/arrow/util/sse-util.h b/cpp/src/arrow/util/sse-util.h
new file mode 100644
index 0000000000000..570c4057a7573
--- /dev/null
+++ b/cpp/src/arrow/util/sse-util.h
@@ -0,0 +1,237 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// From Apache Impala as of 2016-01-29. Pared down to a minimal set of
+// functions needed for parquet-cpp
+
+#ifndef ARROW_UTIL_SSE_UTIL_H
+#define ARROW_UTIL_SSE_UTIL_H
+
+#ifdef ARROW_USE_SSE
+#include <emmintrin.h>
+#endif
+
+namespace arrow {
+
+/// This class contains constants useful for text processing with SSE4.2 intrinsics.
+namespace SSEUtil {
+/// Number of characters that fit in 64/128 bit register.  SSE provides instructions
+/// for loading 64 or 128 bits into a register at a time.
+static const int CHARS_PER_64_BIT_REGISTER = 8;
+static const int CHARS_PER_128_BIT_REGISTER = 16;
+
+/// SSE4.2 adds instructions for text processing.  The instructions have a control
+/// byte that determines some of functionality of the instruction.  (Equivalent to
+/// GCC's _SIDD_CMP_EQUAL_ANY, etc).
+static const int PCMPSTR_EQUAL_ANY = 0x00;     // strchr
+static const int PCMPSTR_EQUAL_EACH = 0x08;    // strcmp
+static const int PCMPSTR_UBYTE_OPS = 0x00;     // unsigned char (8-bits, rather than 16)
+static const int PCMPSTR_NEG_POLARITY = 0x10;  // see Intel SDM chapter 4.1.4.
+
+/// In this mode, SSE text processing functions will return a mask of all the
+/// characters that matched.
+static const int STRCHR_MODE = PCMPSTR_EQUAL_ANY | PCMPSTR_UBYTE_OPS;
+
+/// In this mode, SSE text processing functions will return the number of
+/// bytes that match consecutively from the beginning.
+static const int STRCMP_MODE =
+    PCMPSTR_EQUAL_EACH | PCMPSTR_UBYTE_OPS | PCMPSTR_NEG_POLARITY;
+
+/// Precomputed mask values up to 16 bits.
+static const int SSE_BITMASK[CHARS_PER_128_BIT_REGISTER] = {
+    1 << 0, 1 << 1, 1 << 2, 1 << 3, 1 << 4, 1 << 5, 1 << 6, 1 << 7, 1 << 8, 1 << 9,
+    1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15,
+};
+}  // namespace SSEUtil
+
+#ifdef ARROW_USE_SSE
+
+/// Define the SSE 4.2 intrinsics.  The caller must first verify at runtime (or codegen
+/// IR load time) that the processor supports SSE 4.2 before calling these.  These are
+/// defined outside the namespace because the IR w/ SSE 4.2 case needs to use macros.
+#ifndef IR_COMPILE
+/// When compiling to native code (i.e. not IR), we cannot use the -msse4.2 compiler
+/// flag.  Otherwise, the compiler will emit SSE 4.2 instructions outside of the runtime
+/// SSE 4.2 checks and Impala will crash on CPUs that don't support SSE 4.2
+/// (IMPALA-1399/1646).  The compiler intrinsics cannot be used without -msse4.2, so we
+/// define our own implementations of the intrinsics instead.
+
+/// The PCMPxSTRy instructions require that the control byte 'mode' be encoded as an
+/// immediate.  So, those need to be always inlined in order to always propagate the
+/// mode constant into the inline asm.
+#define SSE_ALWAYS_INLINE inline __attribute__((__always_inline__))
+
+template <int MODE>
+static inline __m128i SSE4_cmpestrm(__m128i str1, int len1, __m128i str2, int len2) {
+#ifdef __clang__
+  /// Use asm reg rather than Yz output constraint to workaround LLVM bug 13199 -
+  /// clang doesn't support Y-prefixed asm constraints.
+  register volatile __m128i result asm("xmm0");
+  __asm__ volatile("pcmpestrm %5, %2, %1"
+                   : "=x"(result)
+                   : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE)
+                   : "cc");
+#else
+  __m128i result;
+  __asm__ volatile("pcmpestrm %5, %2, %1"
+                   : "=Yz"(result)
+                   : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE)
+                   : "cc");
+#endif
+  return result;
+}
+
+template <int MODE>
+static inline int SSE4_cmpestri(__m128i str1, int len1, __m128i str2, int len2) {
+  int result;
+  __asm__("pcmpestri %5, %2, %1"
+          : "=c"(result)
+          : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE)
+          : "cc");
+  return result;
+}
+
+static inline uint32_t SSE4_crc32_u8(uint32_t crc, uint8_t v) {
+  __asm__("crc32b %1, %0" : "+r"(crc) : "rm"(v));
+  return crc;
+}
+
+static inline uint32_t SSE4_crc32_u16(uint32_t crc, uint16_t v) {
+  __asm__("crc32w %1, %0" : "+r"(crc) : "rm"(v));
+  return crc;
+}
+
+static inline uint32_t SSE4_crc32_u32(uint32_t crc, uint32_t v) {
+  __asm__("crc32l %1, %0" : "+r"(crc) : "rm"(v));
+  return crc;
+}
+
+static inline uint32_t SSE4_crc32_u64(uint32_t crc, uint64_t v) {
+  uint64_t result = crc;
+  __asm__("crc32q %1, %0" : "+r"(result) : "rm"(v));
+  return result;
+}
+
+static inline int64_t POPCNT_popcnt_u64(uint64_t a) {
+  int64_t result;
+  __asm__("popcntq %1, %0" : "=r"(result) : "mr"(a) : "cc");
+  return result;
+}
+
+#undef SSE_ALWAYS_INLINE
+
+#elif defined(__SSE4_2__)  // IR_COMPILE for SSE 4.2.
+/// When cross-compiling to IR, we cannot use inline asm because LLVM JIT does not
+/// support it.  However, the cross-compiled IR is compiled twice: with and without
+/// -msse4.2.  When -msse4.2 is enabled in the cross-compile, we can just use the
+/// compiler intrinsics.
+
+#include <smmintrin.h>
+
+template <int MODE>
+static inline __m128i SSE4_cmpestrm(__m128i str1, int len1, __m128i str2, int len2) {
+  return _mm_cmpestrm(str1, len1, str2, len2, MODE);
+}
+
+template <int MODE>
+static inline int SSE4_cmpestri(__m128i str1, int len1, __m128i str2, int len2) {
+  return _mm_cmpestri(str1, len1, str2, len2, MODE);
+}
+
+#define SSE4_crc32_u8 _mm_crc32_u8
+#define SSE4_crc32_u16 _mm_crc32_u16
+#define SSE4_crc32_u32 _mm_crc32_u32
+#define SSE4_crc32_u64 _mm_crc32_u64
+#define POPCNT_popcnt_u64 _mm_popcnt_u64
+
+#else  // IR_COMPILE without SSE 4.2.
+/// When cross-compiling to IR without SSE 4.2 support (i.e. no -msse4.2), we cannot use
+/// SSE 4.2 instructions.  Otherwise, the IR loading will fail on CPUs that don't
+/// support SSE 4.2.  However, because the caller isn't allowed to call these routines
+/// on CPUs that lack SSE 4.2 anyway, we can implement stubs for this case.
+
+template <int MODE>
+static inline __m128i SSE4_cmpestrm(__m128i str1, int len1, __m128i str2, int len2) {
+  DCHECK(false) << "CPU doesn't support SSE 4.2";
+  return (__m128i){0};  // NOLINT
+}
+
+template <int MODE>
+static inline int SSE4_cmpestri(__m128i str1, int len1, __m128i str2, int len2) {
+  DCHECK(false) << "CPU doesn't support SSE 4.2";
+  return 0;
+}
+
+static inline uint32_t SSE4_crc32_u8(uint32_t crc, uint8_t v) {
+  DCHECK(false) << "CPU doesn't support SSE 4.2";
+  return 0;
+}
+
+static inline uint32_t SSE4_crc32_u16(uint32_t crc, uint16_t v) {
+  DCHECK(false) << "CPU doesn't support SSE 4.2";
+  return 0;
+}
+
+static inline uint32_t SSE4_crc32_u32(uint32_t crc, uint32_t v) {
+  DCHECK(false) << "CPU doesn't support SSE 4.2";
+  return 0;
+}
+
+static inline uint32_t SSE4_crc32_u64(uint32_t crc, uint64_t v) {
+  DCHECK(false) << "CPU doesn't support SSE 4.2";
+  return 0;
+}
+
+static inline int64_t POPCNT_popcnt_u64(uint64_t a) {
+  DCHECK(false) << "CPU doesn't support SSE 4.2";
+  return 0;
+}
+
+#endif  // IR_COMPILE
+
+#else
+
+static inline uint32_t SSE4_crc32_u8(uint32_t crc, uint8_t v) {
+  DCHECK(false) << "SSE support is not enabled";
+  return 0;
+}
+
+static inline uint32_t SSE4_crc32_u16(uint32_t crc, uint16_t v) {
+  DCHECK(false) << "SSE support is not enabled";
+  return 0;
+}
+
+static inline uint32_t SSE4_crc32_u32(uint32_t crc, uint32_t v) {
+  DCHECK(false) << "SSE support is not enabled";
+  return 0;
+}
+
+static inline uint32_t SSE4_crc32_u64(uint32_t crc, uint64_t v) {
+  DCHECK(false) << "SSE support is not enabled";
+  return 0;
+}
+
+static inline int64_t POPCNT_popcnt_u64(uint64_t a) {
+  DCHECK(false) << "SSE support is not enabled";
+  return 0;
+}
+
+#endif  // ARROW_USE_SSE
+
+}  // namespace arrow
+
+#endif  //  ARROW_UTIL_SSE_UTIL_H

From a58893882ac8acd1ac4a5036685cbf09a9a09673 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 27 Jun 2017 16:06:43 -0400
Subject: [PATCH 0757/1644] ARROW-1159: [C++] Use dllimport for visibility when
 not building Arrow library

This should fix the linking issues currently in https://ci.appveyor.com/project/ApacheSoftwareFoundation/parquet-cpp/build/1.0.173/job/90ahlb7eoxlch3yj

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #789 from wesm/ARROW-1154-post and squashes the following commits:

985cedee [Wes McKinney] Use dllimport for visibility when not building Arrow library, this enables static members to be visible from inline functions like in cpu-info.h
---
 cpp/cmake_modules/SetupCxxFlags.cmake | 4 ++++
 cpp/src/arrow/util/visibility.h       | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index e2106559ba028..6e92c4b1c1b0d 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -30,6 +30,10 @@ if (MSVC)
   # insecure, like std::getenv
   add_definitions(-D_CRT_SECURE_NO_WARNINGS)
 
+  # Use __declspec(dllexport) during library build, other users of the Arrow
+  # headers will see dllimport
+  add_definitions(-DARROW_EXPORTING)
+
   if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     # clang-cl
     set(CXX_COMMON_FLAGS "-EHsc")
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
index 0018ce0989ad6..ea78e57204efc 100644
--- a/cpp/src/arrow/util/visibility.h
+++ b/cpp/src/arrow/util/visibility.h
@@ -24,7 +24,13 @@
 #else
 #pragma GCC diagnostic ignored "-Wattributes"
 #endif
+
+#ifdef ARROW_EXPORTING
 #define ARROW_EXPORT __declspec(dllexport)
+#else
+#define ARROW_EXPORT __declspec(dllimport)
+#endif
+
 #define ARROW_NO_EXPORT
 #else  // Not Windows
 #ifndef ARROW_EXPORT

From bddb2197df4e3cba4a27da27cd15917fe30d3d45 Mon Sep 17 00:00:00 2001
From: Holden Karau <holden@us.ibm.com>
Date: Wed, 28 Jun 2017 11:35:45 -0400
Subject: [PATCH 0758/1644] ARROW-834: Python Support creating from iterables

Support creating arrow arrays from iterables.
Possible follow up TODO (or possibly belongs in this issue); throw a clear exception when passed an iterator rather than an iterable.

Author: Holden Karau <holden@us.ibm.com>

Closes #602 from holdenk/ARROW-834-csupport-creating-from-iterables and squashes the following commits:

750e7f4c [Holden Karau] Switch AppendItem to pure virtual for TypedConverterVisitor
0b72e956 [Holden Karau] Remove unecessary file after merge
2ed00d91 [Holden Karau] Fix long line
ee2afaa4 [Holden Karau] Comment the built in converter type inferance code a bit.
dddf57db [Holden Karau] Make a note about the resize/realloc in underflow with size
1fd9588a [Holden Karau] Do dynamic resize on the array buffer if size ended up being larger (e.g. support underflow from iterator constructors).
ad935e9d [Holden Karau] Have size override the size of the iterator if the iterator is larger.
42f06996 [Holden Karau] Style fix
fa0abcc2 [Holden Karau] Add ConvertPySequence to other side
01e462c2 [Holden Karau] Naive merge, lets see if it works
9eb3f106 [Holden Karau] Return the append inside of the decimal convert case/switch business
a571ad4b [Holden Karau] Merge in changes to timestamp/datetime builtin converter
8c42fdc2 [Holden Karau] Feedback from wes (fix some previously unchecked appends, fix long line )
389976cb [Holden Karau] Use CRTP in the iterator
52b03e3e [Holden Karau] Use a const ownedref
1d970bdb [Holden Karau] Switch the SeqVisitor to use OwnedRef
c429f9a5 [Holden Karau] Style fixes
d392daa8 [Holden Karau] Add limmited pure iterator support and a note
be58bc0f [Holden Karau] Restore ArrowBlock (unreleated change)
3a55e824 [Holden Karau] Update array function description
80cc971e [Holden Karau] Cleanup debugging
63c0b7fa [Holden Karau] Tests pass (TODO cleanup debugging)
82ec3c3d [Holden Karau] revert changes to _array.pyx
ca0d5303 [Holden Karau] In theory this works ok now for iterables as well
b6c72f5c [Holden Karau] Make TypedConverterVisitor work on PySequence or Python Iterators
48b08aa5 [Holden Karau] Switch remaining converters
a1bf4bd1 [Holden Karau] Move over timestamp and byte converters
15cdfe34 [Holden Karau] Move more of the convertors to the visitor version
76e08ca5 [Holden Karau] Part of the way along adding iterable support
77c935b9 [Holden Karau] Revert accidently java change
5c0fa0b5 [Holden Karau] Start adding iterable support
---
 cpp/src/arrow/python/builtin_convert.cc      | 434 ++++++++++---------
 cpp/src/arrow/python/builtin_convert.h       |   3 +-
 cpp/src/arrow/python/pandas_convert.cc       |   4 +-
 python/pyarrow/array.pxi                     |  29 +-
 python/pyarrow/includes/libarrow.pxd         |   4 +
 python/pyarrow/tests/test_convert_builtin.py |  50 +++
 6 files changed, 307 insertions(+), 217 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 3197c2ade4bae..d3bfb37fdfbad 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -145,48 +145,50 @@ class ScalarVisitor {
 
 static constexpr int MAX_NESTING_LEVELS = 32;
 
+// SeqVisitor is used to infer the type.
 class SeqVisitor {
  public:
   SeqVisitor() : max_nesting_level_(0) {
     memset(nesting_histogram_, 0, MAX_NESTING_LEVELS * sizeof(int));
   }
 
+  // co-recursive with VisitElem
   Status Visit(PyObject* obj, int level = 0) {
-    Py_ssize_t size = PySequence_Size(obj);
-
     if (level > max_nesting_level_) { max_nesting_level_ = level; }
 
-    for (int64_t i = 0; i < size; ++i) {
-      // TODO(wesm): Error checking?
-      // TODO(wesm): Specialize for PyList_GET_ITEM?
-      OwnedRef item_ref(PySequence_GetItem(obj, i));
-      PyObject* item = item_ref.obj();
-
-      if (PyList_Check(item)) {
-        RETURN_NOT_OK(Visit(item, level + 1));
-      } else if (PyDict_Check(item)) {
-        return Status::NotImplemented("No type inference for dicts");
-      } else {
-        // We permit nulls at any level of nesting
-        if (item == Py_None) {
-          // TODO
-        } else {
-          ++nesting_histogram_[level];
-          scalars_.Visit(item);
-        }
+    // Loop through either a sequence or an iterator.
+    if (PySequence_Check(obj)) {
+      Py_ssize_t size = PySequence_Size(obj);
+      for (int64_t i = 0; i < size; ++i) {
+	// TODO(wesm): Specialize for PyList_GET_ITEM?
+	OwnedRef ref = OwnedRef(PySequence_GetItem(obj, i));
+	RETURN_NOT_OK(VisitElem(ref, level));
       }
+    } else if (PyObject_HasAttrString(obj, "__iter__")) {
+      OwnedRef iter = OwnedRef(PyObject_GetIter(obj));
+      PyObject* item;
+      while ((item = PyIter_Next(iter.obj()))) {
+	OwnedRef ref = OwnedRef(item);
+	RETURN_NOT_OK(VisitElem(ref, level));
+      }
+    } else {
+      return Status::TypeError("Object is not a sequence or iterable");
     }
     return Status::OK();
   }
 
   std::shared_ptr<DataType> GetType() {
+    // If all the non-list inputs were null (or there were no inputs)
     if (scalars_.total_count() == 0) {
       if (max_nesting_level_ == 0) {
+	// If its just a single empty list or list of nulls, return null.
         return null();
       } else {
+	// Error, if we have nesting but no concrete base type.
         return nullptr;
       }
     } else {
+      // Lists of Lists of [X]
       std::shared_ptr<DataType> result = scalars_.GetType();
       for (int i = 0; i < max_nesting_level_; ++i) {
         result = std::make_shared<ListType>(result);
@@ -199,6 +201,7 @@ class SeqVisitor {
     if (scalars_.total_count() > 0) {
       if (num_nesting_levels() > 1) {
         return Status::Invalid("Mixed nesting levels not supported");
+      // If the nesting goes deeper than the deepest scalar
       } else if (max_observed_level() < max_nesting_level_) {
         return Status::Invalid("Mixed nesting levels not supported");
       }
@@ -206,6 +209,7 @@ class SeqVisitor {
     return Status::OK();
   }
 
+  // Returns the deepest level which has scalar elements.
   int max_observed_level() const {
     int result = 0;
     for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
@@ -214,6 +218,7 @@ class SeqVisitor {
     return result;
   }
 
+  // Returns the number of nesting levels which have scalar elements.
   int num_nesting_levels() const {
     int result = 0;
     for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
@@ -226,16 +231,50 @@ class SeqVisitor {
   ScalarVisitor scalars_;
 
   // Track observed
+  // Deapest nesting level (irregardless of scalars)
   int max_nesting_level_;
+  // Number of scalar elements at each nesting level.
+  // (TOOD: We really only need to know if a scalar is present, not the count).
   int nesting_histogram_[MAX_NESTING_LEVELS];
+
+  // Visits a specific element (inner part of the loop).
+  Status VisitElem(const OwnedRef &item_ref, int level) {
+    if (PyList_Check(item_ref.obj())) {
+      RETURN_NOT_OK(Visit(item_ref.obj(), level + 1));
+    } else if (PyDict_Check(item_ref.obj())) {
+      return Status::NotImplemented("No type inference for dicts");
+    } else {
+      // We permit nulls at any level of nesting
+      if (item_ref.obj() == Py_None) {
+	// TODO
+      } else {
+	++nesting_histogram_[level];
+	scalars_.Visit(item_ref.obj());
+      }
+    }
+    return Status::OK();
+  }
 };
 
 Status InferArrowSize(PyObject* obj, int64_t* size) {
-  *size = static_cast<int64_t>(PySequence_Size(obj));
+  if (PySequence_Check(obj)) {
+    *size = static_cast<int64_t>(PySequence_Size(obj));
+  } else if (PyObject_HasAttrString(obj, "__iter__")) {
+    PyObject* iter = PyObject_GetIter(obj);
+    OwnedRef iter_ref(iter);
+    *size = 0;
+    PyObject* item;
+    while ((item = PyIter_Next(iter))) {
+      OwnedRef item_ref(item);
+      *size += 1;
+    }
+  } else {
+    return Status::TypeError("Object is not a sequence or iterable");
+  }
   if (PyErr_Occurred()) {
     // Not a sequence
     PyErr_Clear();
-    return Status::TypeError("Object is not a sequence");
+    return Status::TypeError("Object is not a sequence or iterable");
   }
   return Status::OK();
 }
@@ -243,6 +282,7 @@ Status InferArrowSize(PyObject* obj, int64_t* size) {
 // Non-exhaustive type inference
 Status InferArrowTypeAndSize(
     PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
+
   RETURN_NOT_OK(InferArrowSize(obj, size));
 
   // For 0-length sequences, refuse to guess
@@ -268,7 +308,9 @@ class SeqConverter {
     return Status::OK();
   }
 
-  virtual Status AppendData(PyObject* seq) = 0;
+  virtual Status AppendData(PyObject* seq, int64_t size) = 0;
+
+  virtual ~SeqConverter() {}
 
  protected:
   std::shared_ptr<ArrayBuilder> builder_;
@@ -287,221 +329,210 @@ class TypedConverter : public SeqConverter {
   BuilderType* typed_builder_;
 };
 
-class BoolConverter : public TypedConverter<BooleanBuilder> {
+template <typename BuilderType, class Derived>
+class TypedConverterVisitor : public TypedConverter<BuilderType> {
  public:
-  Status AppendData(PyObject* seq) override {
-    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
-    for (int64_t i = 0; i < size; ++i) {
-      OwnedRef item(PySequence_GetItem(seq, i));
-      if (item.obj() == Py_None) {
-        typed_builder_->AppendNull();
-      } else {
-        if (item.obj() == Py_True) {
-          typed_builder_->Append(true);
-        } else {
-          typed_builder_->Append(false);
-        }
+  Status AppendData(PyObject* obj, int64_t size) override {
+    /// Ensure we've allocated enough space
+    RETURN_NOT_OK(this->typed_builder_->Reserve(size));
+    // Iterate over the items adding each one
+    if (PySequence_Check(obj)) {
+      for (int64_t i = 0; i < size; ++i) {
+	OwnedRef ref(PySequence_GetItem(obj, i));
+	RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
+      }
+    } else if (PyObject_HasAttrString(obj, "__iter__")) {
+      PyObject* iter = PyObject_GetIter(obj);
+      OwnedRef iter_ref(iter);
+      PyObject* item;
+      int64_t i = 0;
+      // To allow people with long generators to only convert a subset, stop
+      // consuming at size.
+      while ((item = PyIter_Next(iter)) && i < size) {
+	OwnedRef ref(item);
+	RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
+	++i;
       }
+      if (size != i) {
+	RETURN_NOT_OK(this->typed_builder_->Resize(i));
+      }
+    } else {
+      return Status::TypeError("Object is not a sequence or iterable");
     }
     return Status::OK();
   }
+
+  virtual Status AppendItem(const OwnedRef& item) = 0;
 };
 
-class Int64Converter : public TypedConverter<Int64Builder> {
+class BoolConverter : public TypedConverterVisitor<
+  BooleanBuilder, BoolConverter> {
  public:
-  Status AppendData(PyObject* seq) override {
-    int64_t val;
-    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
-    for (int64_t i = 0; i < size; ++i) {
-      OwnedRef item(PySequence_GetItem(seq, i));
-      if (item.obj() == Py_None) {
-        typed_builder_->AppendNull();
+  inline Status AppendItem(const OwnedRef& item) {
+    if (item.obj() == Py_None) {
+      return typed_builder_->AppendNull();
+    } else {
+      if (item.obj() == Py_True) {
+	return typed_builder_->Append(true);
       } else {
-        val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
-        RETURN_IF_PYERROR();
-        typed_builder_->Append(val);
+	return typed_builder_->Append(false);
       }
     }
-    return Status::OK();
   }
 };
 
-class DateConverter : public TypedConverter<Date64Builder> {
+class Int64Converter : public TypedConverterVisitor<
+  Int64Builder, Int64Converter> {
  public:
-  Status AppendData(PyObject* seq) override {
-    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
-    for (int64_t i = 0; i < size; ++i) {
-      OwnedRef item(PySequence_GetItem(seq, i));
-      if (item.obj() == Py_None) {
-        typed_builder_->AppendNull();
-      } else {
-        PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
-        typed_builder_->Append(PyDate_to_ms(pydate));
-      }
+  inline Status AppendItem(const OwnedRef& item) {
+    int64_t val;
+    if (item.obj() == Py_None) {
+      return typed_builder_->AppendNull();
+    } else {
+      val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      RETURN_IF_PYERROR();
+      return typed_builder_->Append(val);
     }
-    return Status::OK();
   }
 };
 
-class TimestampConverter : public TypedConverter<TimestampBuilder> {
+class DateConverter : public TypedConverterVisitor<
+  Date64Builder, DateConverter> {
  public:
-  Status AppendData(PyObject* seq) override {
-    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
-    for (int64_t i = 0; i < size; ++i) {
-      OwnedRef item(PySequence_GetItem(seq, i));
-      if (item.obj() == Py_None) {
-        typed_builder_->AppendNull();
-      } else {
-        PyDateTime_DateTime* pydatetime =
-            reinterpret_cast<PyDateTime_DateTime*>(item.obj());
-        typed_builder_->Append(PyDateTime_to_us(pydatetime));
-        RETURN_IF_PYERROR();
-      }
+  inline Status AppendItem(const OwnedRef& item) {
+    if (item.obj() == Py_None) {
+      return typed_builder_->AppendNull();
+    } else {
+      PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
+      return typed_builder_->Append(PyDate_to_ms(pydate));
+    }
+  }
+};
+
+class TimestampConverter : public TypedConverterVisitor<
+  Date64Builder, TimestampConverter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    if (item.obj() == Py_None) {
+      return typed_builder_->AppendNull();
+    } else {
+      PyDateTime_DateTime* pydatetime =
+	reinterpret_cast<PyDateTime_DateTime*>(item.obj());
+      return typed_builder_->Append(PyDateTime_to_us(pydatetime));
     }
-    return Status::OK();
   }
 };
 
-class DoubleConverter : public TypedConverter<DoubleBuilder> {
+class DoubleConverter : public TypedConverterVisitor<
+  DoubleBuilder, DoubleConverter> {
  public:
-  Status AppendData(PyObject* seq) override {
+  inline Status AppendItem(const OwnedRef& item) {
     double val;
-    int64_t size = static_cast<int64_t>(PySequence_Size(seq));
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
-    for (int64_t i = 0; i < size; ++i) {
-      OwnedRef item(PySequence_GetItem(seq, i));
-      if (item.obj() == Py_None) {
-        typed_builder_->AppendNull();
-      } else {
-        val = PyFloat_AsDouble(item.obj());
-        RETURN_IF_PYERROR();
-        typed_builder_->Append(val);
-      }
+    if (item.obj() == Py_None) {
+      return typed_builder_->AppendNull();
+    } else {
+      val = PyFloat_AsDouble(item.obj());
+      RETURN_IF_PYERROR();
+      return typed_builder_->Append(val);
     }
-    return Status::OK();
   }
 };
 
-class BytesConverter : public TypedConverter<BinaryBuilder> {
+class BytesConverter : public TypedConverterVisitor<
+  BinaryBuilder, BytesConverter> {
  public:
-  Status AppendData(PyObject* seq) override {
-    PyObject* item;
+  inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
-    OwnedRef tmp;
     const char* bytes;
     Py_ssize_t length;
-    Py_ssize_t size = PySequence_Size(seq);
-    for (int64_t i = 0; i < size; ++i) {
-      item = PySequence_GetItem(seq, i);
-      OwnedRef holder(item);
-
-      if (item == Py_None) {
-        RETURN_NOT_OK(typed_builder_->AppendNull());
-        continue;
-      } else if (PyUnicode_Check(item)) {
-        tmp.reset(PyUnicode_AsUTF8String(item));
-        RETURN_IF_PYERROR();
-        bytes_obj = tmp.obj();
-      } else if (PyBytes_Check(item)) {
-        bytes_obj = item;
-      } else {
-        return InvalidConversion(item, "bytes");
-      }
-      // No error checking
-      length = PyBytes_GET_SIZE(bytes_obj);
-      bytes = PyBytes_AS_STRING(bytes_obj);
-      RETURN_NOT_OK(typed_builder_->Append(bytes, static_cast<int32_t>(length)));
+    OwnedRef tmp;
+
+    if (item.obj() == Py_None) {
+      RETURN_NOT_OK(typed_builder_->AppendNull());
+      return Status::OK();
+    } else if (PyUnicode_Check(item.obj())) {
+      tmp.reset(PyUnicode_AsUTF8String(item.obj()));
+      RETURN_IF_PYERROR();
+      bytes_obj = tmp.obj();
+    } else if (PyBytes_Check(item.obj())) {
+      bytes_obj = item.obj();
+    } else {
+      return InvalidConversion(item.obj(), "bytes");
     }
-    return Status::OK();
+    // No error checking
+    length = PyBytes_GET_SIZE(bytes_obj);
+    bytes = PyBytes_AS_STRING(bytes_obj);
+    return typed_builder_->Append(bytes, static_cast<int32_t>(length));
   }
 };
 
-class FixedWidthBytesConverter : public TypedConverter<FixedSizeBinaryBuilder> {
+class FixedWidthBytesConverter : public TypedConverterVisitor<
+  FixedSizeBinaryBuilder, FixedWidthBytesConverter> {
  public:
-  Status AppendData(PyObject* seq) override {
-    PyObject* item;
+  inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
     Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedSizeBinaryType>(
         typed_builder_->type())->byte_width();
-    Py_ssize_t size = PySequence_Size(seq);
-    for (int64_t i = 0; i < size; ++i) {
-      item = PySequence_GetItem(seq, i);
-      OwnedRef holder(item);
-
-      if (item == Py_None) {
-        RETURN_NOT_OK(typed_builder_->AppendNull());
-        continue;
-      } else if (PyUnicode_Check(item)) {
-        tmp.reset(PyUnicode_AsUTF8String(item));
-        RETURN_IF_PYERROR();
-        bytes_obj = tmp.obj();
-      } else if (PyBytes_Check(item)) {
-        bytes_obj = item;
-      } else {
-        return InvalidConversion(item, "bytes");
-      }
-      // No error checking
-      RETURN_NOT_OK(CheckPythonBytesAreFixedLength(bytes_obj, expected_length));
-      RETURN_NOT_OK(typed_builder_->Append(
-          reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(bytes_obj))));
+    if (item.obj() == Py_None) {
+      RETURN_NOT_OK(typed_builder_->AppendNull());
+      return Status::OK();
+    } else if (PyUnicode_Check(item.obj())) {
+      tmp.reset(PyUnicode_AsUTF8String(item.obj()));
+      RETURN_IF_PYERROR();
+      bytes_obj = tmp.obj();
+    } else if (PyBytes_Check(item.obj())) {
+      bytes_obj = item.obj();
+    } else {
+      return InvalidConversion(item.obj(), "bytes");
     }
-    return Status::OK();
+    // No error checking
+    RETURN_NOT_OK(CheckPythonBytesAreFixedLength(bytes_obj, expected_length));
+    return typed_builder_->Append(
+	reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(bytes_obj)));
   }
 };
 
-class UTF8Converter : public TypedConverter<StringBuilder> {
+class UTF8Converter : public TypedConverterVisitor<
+  StringBuilder, UTF8Converter> {
  public:
-  Status AppendData(PyObject* seq) override {
-    PyObject* item;
+  inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
     const char* bytes;
     Py_ssize_t length;
-    Py_ssize_t size = PySequence_Size(seq);
-    for (int64_t i = 0; i < size; ++i) {
-      item = PySequence_GetItem(seq, i);
-      OwnedRef holder(item);
-
-      if (item == Py_None) {
-        RETURN_NOT_OK(typed_builder_->AppendNull());
-        continue;
-      } else if (!PyUnicode_Check(item)) {
-        return Status::Invalid("Non-unicode value encountered");
-      }
-      tmp.reset(PyUnicode_AsUTF8String(item));
-      RETURN_IF_PYERROR();
-      bytes_obj = tmp.obj();
 
-      // No error checking
-      length = PyBytes_GET_SIZE(bytes_obj);
-      bytes = PyBytes_AS_STRING(bytes_obj);
-      RETURN_NOT_OK(typed_builder_->Append(bytes, static_cast<int32_t>(length)));
+    if (item.obj() == Py_None) {
+      return typed_builder_->AppendNull();
+    } else if (!PyUnicode_Check(item.obj())) {
+      return Status::Invalid("Non-unicode value encountered");
     }
-    return Status::OK();
+    tmp.reset(PyUnicode_AsUTF8String(item.obj()));
+    RETURN_IF_PYERROR();
+    bytes_obj = tmp.obj();
+
+    // No error checking
+    length = PyBytes_GET_SIZE(bytes_obj);
+    bytes = PyBytes_AS_STRING(bytes_obj);
+    return typed_builder_->Append(bytes, static_cast<int32_t>(length));
   }
 };
 
-class ListConverter : public TypedConverter<ListBuilder> {
+class ListConverter : public TypedConverterVisitor<
+  ListBuilder, ListConverter> {
  public:
   Status Init(const std::shared_ptr<ArrayBuilder>& builder) override;
 
-  Status AppendData(PyObject* seq) override {
-    Py_ssize_t size = PySequence_Size(seq);
-    for (int64_t i = 0; i < size; ++i) {
-      OwnedRef item(PySequence_GetItem(seq, i));
-      if (item.obj() == Py_None) {
-        RETURN_NOT_OK(typed_builder_->AppendNull());
-      } else {
-        typed_builder_->Append();
-        RETURN_NOT_OK(value_converter_->AppendData(item.obj()));
-      }
+  inline Status AppendItem(const OwnedRef& item) {
+    if (item.obj() == Py_None) {
+      return typed_builder_->AppendNull();
+    } else {
+      typed_builder_->Append();
+      PyObject* item_obj = item.obj();
+      int64_t list_size =
+	static_cast<int64_t>(PySequence_Size(item_obj));
+      return value_converter_->AppendData(item_obj, list_size);
     }
-    return Status::OK();
   }
 
  protected:
@@ -512,45 +543,33 @@ class ListConverter : public TypedConverter<ListBuilder> {
   case bit_width: {                                           \
     arrow::decimal::Decimal##bit_width out;                   \
     RETURN_NOT_OK(PythonDecimalToArrowDecimal((item), &out)); \
-    RETURN_NOT_OK((builder)->Append(out));                    \
+    return ((builder)->Append(out));                          \
     break;                                                    \
   }
 
-class DecimalConverter : public TypedConverter<arrow::DecimalBuilder> {
+class DecimalConverter : public TypedConverterVisitor<
+  arrow::DecimalBuilder, DecimalConverter> {
  public:
-  Status AppendData(PyObject* seq) override {
-    /// Ensure we've allocated enough space
-    Py_ssize_t size = PySequence_Size(seq);
-    RETURN_NOT_OK(typed_builder_->Reserve(size));
-
+  inline Status AppendItem(const OwnedRef& item) {
     /// Can the compiler figure out that the case statement below isn't necessary
     /// once we're running?
     const int bit_width =
         std::dynamic_pointer_cast<arrow::DecimalType>(typed_builder_->type())
             ->bit_width();
 
-    OwnedRef ref;
-    PyObject* item = nullptr;
-    for (int64_t i = 0; i < size; ++i) {
-      ref.reset(PySequence_GetItem(seq, i));
-      item = ref.obj();
-
-      /// TODO(phillipc): Check for nan?
-      if (item != Py_None) {
-        switch (bit_width) {
-          DECIMAL_CONVERT_CASE(32, item, typed_builder_)
-          DECIMAL_CONVERT_CASE(64, item, typed_builder_)
-          DECIMAL_CONVERT_CASE(128, item, typed_builder_)
-          default:
-            break;
-        }
-        RETURN_IF_PYERROR();
-      } else {
-        RETURN_NOT_OK(typed_builder_->AppendNull());
+    /// TODO(phillipc): Check for nan?
+    if (item.obj() != Py_None) {
+      switch (bit_width) {
+	  DECIMAL_CONVERT_CASE(32, item.obj(), typed_builder_)
+	  DECIMAL_CONVERT_CASE(64, item.obj(), typed_builder_)
+	  DECIMAL_CONVERT_CASE(128, item.obj(), typed_builder_)
+      default:
+	  return Status::OK();
       }
+      RETURN_IF_PYERROR();
+    } else {
+      return typed_builder_->AppendNull();
     }
-
-    return Status::OK();
   }
 };
 
@@ -601,7 +620,8 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
 }
 
 Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
-    const std::shared_ptr<ArrayBuilder>& builder) {
+			const std::shared_ptr<ArrayBuilder>& builder,
+			int64_t size) {
   PyDateTime_IMPORT;
   std::shared_ptr<SeqConverter> converter = GetConverter(type);
   if (converter == nullptr) {
@@ -611,7 +631,7 @@ Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
   }
   converter->Init(builder);
 
-  return converter->AppendData(obj);
+  return converter->AppendData(obj, size);
 }
 
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out) {
@@ -632,7 +652,7 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
   // Give the sequence converter an array builder
   std::shared_ptr<ArrayBuilder> builder;
   RETURN_NOT_OK(MakeBuilder(pool, type, &builder));
-  RETURN_NOT_OK(AppendPySequence(obj, type, builder));
+  RETURN_NOT_OK(AppendPySequence(obj, type, builder, size));
   return builder->Finish(out);
 }
 
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index a6180d496a920..7f42c334cd7f7 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -44,7 +44,8 @@ ARROW_EXPORT arrow::Status InferArrowSize(PyObject* obj, int64_t* size);
 
 ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj,
     const std::shared_ptr<arrow::DataType>& type,
-    const std::shared_ptr<arrow::ArrayBuilder>& builder);
+    const std::shared_ptr<arrow::ArrayBuilder>& builder,
+    int64_t size);
 
 // Type and size inference
 ARROW_EXPORT
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 654c3921e8b61..ea23496266cd8 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -953,7 +953,7 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
         ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
         return Status::TypeError(ss.str());
       }
-      RETURN_NOT_OK(AppendPySequence(objects[i], type, value_builder));
+      RETURN_NOT_OK(AppendPySequence(objects[i], type, value_builder, size));
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }
@@ -1002,7 +1002,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
         ss << inferred_type->ToString() << " cannot be converted to STRING.";
         return Status::TypeError(ss.str());
       }
-      RETURN_NOT_OK(AppendPySequence(objects[i], inferred_type, value_builder));
+      RETURN_NOT_OK(AppendPySequence(objects[i], inferred_type, value_builder, size));
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 2a0b0a7c5257f..e1be5b1f63d8b 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -1059,18 +1059,26 @@ cdef maybe_coerce_datetime64(values, dtype, DataType type,
 
 
 
-def array(object sequence, DataType type=None, MemoryPool memory_pool=None):
+def array(object sequence, DataType type=None, MemoryPool memory_pool=None,
+          size=None):
     """
     Create pyarrow.Array instance from a Python sequence
 
     Parameters
     ----------
-    sequence : sequence-like object of Python objects
+    sequence : sequence-like or iterable object of Python objects.
+        If both type and size are specified may be a single use iterable.
     type : pyarrow.DataType, optional
         If not passed, will be inferred from the data
     memory_pool : pyarrow.MemoryPool, optional
         If not passed, will allocate memory from the currently-set default
         memory pool
+    size : int64, optional
+        Size of the elements. If the imput is larger than size bail at this
+        length. For iterators, if size is larger than the input iterator this
+        will be treated as a "max size", but will involve an initial allocation
+        of size followed by a resize to the actual size (so if you know the
+        exact size specifying it correctly will give you better performance).
 
     Returns
     -------
@@ -1084,11 +1092,18 @@ def array(object sequence, DataType type=None, MemoryPool memory_pool=None):
     if type is None:
         check_status(ConvertPySequence(sequence, pool, &sp_array))
     else:
-        check_status(
-            ConvertPySequence(
-                sequence, pool, &sp_array, type.sp_type
-            )
-        )
+        if size is None:
+            check_status(
+                ConvertPySequence(
+                    sequence, pool, &sp_array, type.sp_type
+                )
+             )
+        else:
+            check_status(
+                ConvertPySequence(
+                    sequence, pool, &sp_array, type.sp_type, size
+                )
+             )
 
     return pyarrow_wrap_array(sp_array)
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index f712274426d56..741d8321158f6 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -650,6 +650,10 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     CStatus ConvertPySequence(object obj, CMemoryPool* pool,
                               shared_ptr[CArray]* out,
                               const shared_ptr[CDataType]& type)
+    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
+                              shared_ptr[CArray]* out,
+                              const shared_ptr[CDataType]& type,
+			      int64_t size)
 
     CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
 
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index d25055d828062..bf14c4f232852 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -22,6 +22,44 @@
 import datetime
 import decimal
 
+class StrangeIterable:
+    def __init__(self, lst):
+        self.lst = lst
+
+    def __iter__(self):
+        return self.lst.__iter__()
+
+class TestConvertIterable(unittest.TestCase):
+
+    def test_iterable_types(self):
+        arr1 = pa.array(StrangeIterable([0, 1, 2, 3]))
+        arr2 = pa.array((0, 1, 2, 3))
+
+        assert arr1.equals(arr2)
+
+    def test_empty_iterable(self):
+        arr = pa.array(StrangeIterable([]))
+        assert len(arr) == 0
+        assert arr.null_count == 0
+        assert arr.type == pa.null()
+        assert arr.to_pylist() == []
+
+
+class TestLimitedConvertIterator(unittest.TestCase):
+    def test_iterator_types(self):
+        arr1 = pa.array(iter(range(3)), type=pa.int64(), size=3)
+        arr2 = pa.array((0, 1, 2))
+        assert arr1.equals(arr2)
+
+    def test_iterator_size_overflow(self):
+        arr1 = pa.array(iter(range(3)), type=pa.int64(), size=2)
+        arr2 = pa.array((0, 1))
+        assert arr1.equals(arr2)
+
+    def test_iterator_size_underflow(self):
+        arr1 = pa.array(iter(range(3)), type=pa.int64(), size=10)
+        arr2 = pa.array((0, 1, 2))
+        assert arr1.equals(arr2)
 
 class TestConvertSequence(unittest.TestCase):
 
@@ -208,3 +246,15 @@ def test_decimal_large_integer(self):
         type = pa.decimal(precision=23, scale=5)
         arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
+
+    def test_range_types(self):
+        arr1 = pa.array(range(3))
+        arr2 = pa.array((0, 1, 2))
+        assert arr1.equals(arr2)
+
+    def test_empty_range(self):
+        arr = pa.array(range(0))
+        assert len(arr) == 0
+        assert arr.null_count == 0
+        assert arr.type == pa.null()
+        assert arr.to_pylist() == []

From 65558dbb927c045f104988c1c7632872cb95752b Mon Sep 17 00:00:00 2001
From: Sudheesh Katkam <sudheesh@dremio.com>
Date: Thu, 29 Jun 2017 08:22:07 -0700
Subject: [PATCH 0759/1644] =?UTF-8?q?ARROW-1162:=20Empty=20data=20vector?=
 =?UTF-8?q?=20transfer=20between=20list=20vectors=20should=20no=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…t trigger callback

@adeneche please review.

Author: Sudheesh Katkam <sudheesh@dremio.com>

Closes #791 from sudheeshkatkam/ARROW-1162 and squashes the following commits:

390458e [Sudheesh Katkam] ARROW-1162: Empty data vector transfer between list vectors should not trigger callback
---
 .../complex/BaseRepeatedValueVector.java      |  5 ++-
 .../vector/TestBufferOwnershipTransfer.java   | 39 +++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 5ff4c2c8172c3..3bfa8e7f7ce67 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -27,6 +27,7 @@
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
@@ -164,7 +165,9 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldT
       vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack);
       // returned vector must have the same field
       created = true;
-      if (callBack != null) {
+      if (callBack != null &&
+        // not a schema change if changing from ZeroVector to ZeroVector
+        (fieldType.getType().getTypeID() != ArrowTypeID.Null)) {
         callBack.doWork();
       }
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
index fa657875d6d92..08e3700daeebf 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
@@ -18,9 +18,15 @@
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
 import org.junit.Test;
 
 public class TestBufferOwnershipTransfer {
@@ -62,4 +68,37 @@ public void testTransferVariableidth() {
     int expected = 8*4096 + 4*4096 + 4096;
     assertEquals(expected, childAllocator2.getAllocatedMemory());
   }
+
+  private static class Pointer<T> {
+    T value;
+  }
+
+  private static CallBack newTriggerCallback(final Pointer<Boolean> trigger) {
+    trigger.value = false;
+    return new CallBack() {
+      @Override
+      public void doWork() {
+        trigger.value = true;
+      }
+    };
+  }
+
+  @Test
+  public void emptyListTransferShouldNotTriggerSchemaChange() {
+    final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+
+    final Pointer<Boolean> trigger1 = new Pointer<>();
+    final Pointer<Boolean> trigger2 = new Pointer<>();
+    final ListVector v1 = new ListVector("v1", allocator,
+            FieldType.nullable(ArrowType.Null.INSTANCE),
+            newTriggerCallback(trigger1));
+    final ListVector v2 = new ListVector("v2", allocator,
+            FieldType.nullable(ArrowType.Null.INSTANCE),
+            newTriggerCallback(trigger2));
+
+    v1.makeTransferPair(v2).transfer();
+
+    assertFalse(trigger1.value);
+    assertFalse(trigger2.value);
+  }
 }

From 69582528e00f465b6f2b16d39ebe20dc6811ec6c Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 29 Jun 2017 22:01:00 +0200
Subject: [PATCH 0760/1644] ARROW-1165: [C++] Refactor
 PythonDecimalToArrowDecimal to not use templates

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #794 from cpcloud/ARROW-1165 and squashes the following commits:

53565ea [Phillip Cloud] ARROW-1165: [C++] Refactor PythonDecimalToArrowDecimal to not use templates
---
 cpp/src/arrow/python/builtin_convert.cc | 112 +++++++++++-------------
 cpp/src/arrow/python/builtin_convert.h  |   3 +-
 cpp/src/arrow/python/helpers.cc         |  14 +--
 cpp/src/arrow/python/helpers.h          |  11 +--
 cpp/src/arrow/python/pandas_convert.cc  |  14 +--
 cpp/src/arrow/python/python-test.cc     |  12 +--
 6 files changed, 71 insertions(+), 95 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index d3bfb37fdfbad..97e2bb7072e30 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -160,16 +160,16 @@ class SeqVisitor {
     if (PySequence_Check(obj)) {
       Py_ssize_t size = PySequence_Size(obj);
       for (int64_t i = 0; i < size; ++i) {
-	// TODO(wesm): Specialize for PyList_GET_ITEM?
-	OwnedRef ref = OwnedRef(PySequence_GetItem(obj, i));
-	RETURN_NOT_OK(VisitElem(ref, level));
+        // TODO(wesm): Specialize for PyList_GET_ITEM?
+        OwnedRef ref = OwnedRef(PySequence_GetItem(obj, i));
+        RETURN_NOT_OK(VisitElem(ref, level));
       }
     } else if (PyObject_HasAttrString(obj, "__iter__")) {
       OwnedRef iter = OwnedRef(PyObject_GetIter(obj));
       PyObject* item;
       while ((item = PyIter_Next(iter.obj()))) {
-	OwnedRef ref = OwnedRef(item);
-	RETURN_NOT_OK(VisitElem(ref, level));
+        OwnedRef ref = OwnedRef(item);
+        RETURN_NOT_OK(VisitElem(ref, level));
       }
     } else {
       return Status::TypeError("Object is not a sequence or iterable");
@@ -181,10 +181,10 @@ class SeqVisitor {
     // If all the non-list inputs were null (or there were no inputs)
     if (scalars_.total_count() == 0) {
       if (max_nesting_level_ == 0) {
-	// If its just a single empty list or list of nulls, return null.
+        // If its just a single empty list or list of nulls, return null.
         return null();
       } else {
-	// Error, if we have nesting but no concrete base type.
+        // Error, if we have nesting but no concrete base type.
         return nullptr;
       }
     } else {
@@ -201,7 +201,7 @@ class SeqVisitor {
     if (scalars_.total_count() > 0) {
       if (num_nesting_levels() > 1) {
         return Status::Invalid("Mixed nesting levels not supported");
-      // If the nesting goes deeper than the deepest scalar
+        // If the nesting goes deeper than the deepest scalar
       } else if (max_observed_level() < max_nesting_level_) {
         return Status::Invalid("Mixed nesting levels not supported");
       }
@@ -238,7 +238,7 @@ class SeqVisitor {
   int nesting_histogram_[MAX_NESTING_LEVELS];
 
   // Visits a specific element (inner part of the loop).
-  Status VisitElem(const OwnedRef &item_ref, int level) {
+  Status VisitElem(const OwnedRef& item_ref, int level) {
     if (PyList_Check(item_ref.obj())) {
       RETURN_NOT_OK(Visit(item_ref.obj(), level + 1));
     } else if (PyDict_Check(item_ref.obj())) {
@@ -246,10 +246,10 @@ class SeqVisitor {
     } else {
       // We permit nulls at any level of nesting
       if (item_ref.obj() == Py_None) {
-	// TODO
+        // TODO
       } else {
-	++nesting_histogram_[level];
-	scalars_.Visit(item_ref.obj());
+        ++nesting_histogram_[level];
+        scalars_.Visit(item_ref.obj());
       }
     }
     return Status::OK();
@@ -282,7 +282,6 @@ Status InferArrowSize(PyObject* obj, int64_t* size) {
 // Non-exhaustive type inference
 Status InferArrowTypeAndSize(
     PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
-
   RETURN_NOT_OK(InferArrowSize(obj, size));
 
   // For 0-length sequences, refuse to guess
@@ -338,8 +337,8 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
     // Iterate over the items adding each one
     if (PySequence_Check(obj)) {
       for (int64_t i = 0; i < size; ++i) {
-	OwnedRef ref(PySequence_GetItem(obj, i));
-	RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
+        OwnedRef ref(PySequence_GetItem(obj, i));
+        RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
       }
     } else if (PyObject_HasAttrString(obj, "__iter__")) {
       PyObject* iter = PyObject_GetIter(obj);
@@ -349,13 +348,11 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
       // To allow people with long generators to only convert a subset, stop
       // consuming at size.
       while ((item = PyIter_Next(iter)) && i < size) {
-	OwnedRef ref(item);
-	RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
-	++i;
-      }
-      if (size != i) {
-	RETURN_NOT_OK(this->typed_builder_->Resize(i));
+        OwnedRef ref(item);
+        RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
+        ++i;
       }
+      if (size != i) { RETURN_NOT_OK(this->typed_builder_->Resize(i)); }
     } else {
       return Status::TypeError("Object is not a sequence or iterable");
     }
@@ -365,24 +362,22 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
   virtual Status AppendItem(const OwnedRef& item) = 0;
 };
 
-class BoolConverter : public TypedConverterVisitor<
-  BooleanBuilder, BoolConverter> {
+class BoolConverter : public TypedConverterVisitor<BooleanBuilder, BoolConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     if (item.obj() == Py_None) {
       return typed_builder_->AppendNull();
     } else {
       if (item.obj() == Py_True) {
-	return typed_builder_->Append(true);
+        return typed_builder_->Append(true);
       } else {
-	return typed_builder_->Append(false);
+        return typed_builder_->Append(false);
       }
     }
   }
 };
 
-class Int64Converter : public TypedConverterVisitor<
-  Int64Builder, Int64Converter> {
+class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     int64_t val;
@@ -396,8 +391,7 @@ class Int64Converter : public TypedConverterVisitor<
   }
 };
 
-class DateConverter : public TypedConverterVisitor<
-  Date64Builder, DateConverter> {
+class DateConverter : public TypedConverterVisitor<Date64Builder, DateConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     if (item.obj() == Py_None) {
@@ -409,22 +403,21 @@ class DateConverter : public TypedConverterVisitor<
   }
 };
 
-class TimestampConverter : public TypedConverterVisitor<
-  Date64Builder, TimestampConverter> {
+class TimestampConverter
+    : public TypedConverterVisitor<Date64Builder, TimestampConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     if (item.obj() == Py_None) {
       return typed_builder_->AppendNull();
     } else {
       PyDateTime_DateTime* pydatetime =
-	reinterpret_cast<PyDateTime_DateTime*>(item.obj());
+          reinterpret_cast<PyDateTime_DateTime*>(item.obj());
       return typed_builder_->Append(PyDateTime_to_us(pydatetime));
     }
   }
 };
 
-class DoubleConverter : public TypedConverterVisitor<
-  DoubleBuilder, DoubleConverter> {
+class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     double val;
@@ -438,8 +431,7 @@ class DoubleConverter : public TypedConverterVisitor<
   }
 };
 
-class BytesConverter : public TypedConverterVisitor<
-  BinaryBuilder, BytesConverter> {
+class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
@@ -466,8 +458,8 @@ class BytesConverter : public TypedConverterVisitor<
   }
 };
 
-class FixedWidthBytesConverter : public TypedConverterVisitor<
-  FixedSizeBinaryBuilder, FixedWidthBytesConverter> {
+class FixedWidthBytesConverter
+    : public TypedConverterVisitor<FixedSizeBinaryBuilder, FixedWidthBytesConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
@@ -489,12 +481,11 @@ class FixedWidthBytesConverter : public TypedConverterVisitor<
     // No error checking
     RETURN_NOT_OK(CheckPythonBytesAreFixedLength(bytes_obj, expected_length));
     return typed_builder_->Append(
-	reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(bytes_obj)));
+        reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(bytes_obj)));
   }
 };
 
-class UTF8Converter : public TypedConverterVisitor<
-  StringBuilder, UTF8Converter> {
+class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
@@ -518,19 +509,17 @@ class UTF8Converter : public TypedConverterVisitor<
   }
 };
 
-class ListConverter : public TypedConverterVisitor<
-  ListBuilder, ListConverter> {
+class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
  public:
   Status Init(const std::shared_ptr<ArrayBuilder>& builder) override;
 
-  inline Status AppendItem(const OwnedRef& item) {
+  inline Status AppendItem(const OwnedRef& item) override {
     if (item.obj() == Py_None) {
       return typed_builder_->AppendNull();
     } else {
       typed_builder_->Append();
       PyObject* item_obj = item.obj();
-      int64_t list_size =
-	static_cast<int64_t>(PySequence_Size(item_obj));
+      int64_t list_size = static_cast<int64_t>(PySequence_Size(item_obj));
       return value_converter_->AppendData(item_obj, list_size);
     }
   }
@@ -539,16 +528,18 @@ class ListConverter : public TypedConverterVisitor<
   std::shared_ptr<SeqConverter> value_converter_;
 };
 
-#define DECIMAL_CONVERT_CASE(bit_width, item, builder)        \
-  case bit_width: {                                           \
-    arrow::decimal::Decimal##bit_width out;                   \
-    RETURN_NOT_OK(PythonDecimalToArrowDecimal((item), &out)); \
-    return ((builder)->Append(out));                          \
-    break;                                                    \
+#define DECIMAL_CONVERT_CASE(bit_width, item, builder)         \
+  case bit_width: {                                            \
+    arrow::decimal::Decimal##bit_width out;                    \
+    std::string string_out;                                    \
+    RETURN_NOT_OK(PythonDecimalToString((item), &string_out)); \
+    RETURN_NOT_OK(FromString(string_out, &out));               \
+    return ((builder)->Append(out));                           \
+    break;                                                     \
   }
 
-class DecimalConverter : public TypedConverterVisitor<
-  arrow::DecimalBuilder, DecimalConverter> {
+class DecimalConverter
+    : public TypedConverterVisitor<arrow::DecimalBuilder, DecimalConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     /// Can the compiler figure out that the case statement below isn't necessary
@@ -560,11 +551,11 @@ class DecimalConverter : public TypedConverterVisitor<
     /// TODO(phillipc): Check for nan?
     if (item.obj() != Py_None) {
       switch (bit_width) {
-	  DECIMAL_CONVERT_CASE(32, item.obj(), typed_builder_)
-	  DECIMAL_CONVERT_CASE(64, item.obj(), typed_builder_)
-	  DECIMAL_CONVERT_CASE(128, item.obj(), typed_builder_)
-      default:
-	  return Status::OK();
+        DECIMAL_CONVERT_CASE(32, item.obj(), typed_builder_)
+        DECIMAL_CONVERT_CASE(64, item.obj(), typed_builder_)
+        DECIMAL_CONVERT_CASE(128, item.obj(), typed_builder_)
+        default:
+          return Status::OK();
       }
       RETURN_IF_PYERROR();
     } else {
@@ -620,8 +611,7 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
 }
 
 Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
-			const std::shared_ptr<ArrayBuilder>& builder,
-			int64_t size) {
+    const std::shared_ptr<ArrayBuilder>& builder, int64_t size) {
   PyDateTime_IMPORT;
   std::shared_ptr<SeqConverter> converter = GetConverter(type);
   if (converter == nullptr) {
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index 7f42c334cd7f7..54860b36c1986 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -44,8 +44,7 @@ ARROW_EXPORT arrow::Status InferArrowSize(PyObject* obj, int64_t* size);
 
 ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj,
     const std::shared_ptr<arrow::DataType>& type,
-    const std::shared_ptr<arrow::ArrayBuilder>& builder,
-    int64_t size);
+    const std::shared_ptr<arrow::ArrayBuilder>& builder, int64_t size);
 
 // Type and size inference
 ARROW_EXPORT
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index e5d1d388cb54c..76ec3a1ba8746 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -71,9 +71,7 @@ Status ImportFromModule(const OwnedRef& module, const std::string& name, OwnedRe
   return Status::OK();
 }
 
-template <typename T>
-Status PythonDecimalToArrowDecimal(
-    PyObject* python_decimal, decimal::Decimal<T>* arrow_decimal) {
+Status PythonDecimalToString(PyObject* python_decimal, std::string* out) {
   // Call Python's str(decimal_object)
   OwnedRef str_obj(PyObject_Str(python_decimal));
   RETURN_IF_PYERROR();
@@ -87,16 +85,10 @@ Status PythonDecimalToArrowDecimal(
   Py_ssize_t size = str.size;
 
   std::string c_string(bytes, size);
-  return FromString(c_string, arrow_decimal);
+  *out = c_string;
+  return Status::OK();
 }
 
-template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal(
-    PyObject* python_decimal, decimal::Decimal32* arrow_decimal);
-template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal(
-    PyObject* python_decimal, decimal::Decimal64* arrow_decimal);
-template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal(
-    PyObject* python_decimal, decimal::Decimal128* arrow_decimal);
-
 Status InferDecimalPrecisionAndScale(
     PyObject* python_decimal, int* precision, int* scale) {
   // Call Python's str(decimal_object)
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index c6402d8796fe2..e0656699ce4c2 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -29,13 +29,6 @@
 
 namespace arrow {
 
-namespace decimal {
-
-template <typename T>
-struct Decimal;
-
-}  // namespace decimal
-
 namespace py {
 
 class OwnedRef;
@@ -46,9 +39,7 @@ Status ARROW_EXPORT ImportModule(const std::string& module_name, OwnedRef* ref);
 Status ARROW_EXPORT ImportFromModule(
     const OwnedRef& module, const std::string& module_name, OwnedRef* ref);
 
-template <typename T>
-Status ARROW_EXPORT PythonDecimalToArrowDecimal(
-    PyObject* python_decimal, decimal::Decimal<T>* arrow_decimal);
+Status ARROW_EXPORT PythonDecimalToString(PyObject* python_decimal, std::string* out);
 
 Status ARROW_EXPORT InferDecimalPrecisionAndScale(
     PyObject* python_decimal, int* precision = nullptr, int* scale = nullptr);
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index ea23496266cd8..2918f9e6b10d0 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -545,12 +545,14 @@ Status PandasConverter::ConvertDates() {
   return date_builder.Finish(&out_);
 }
 
-#define CONVERT_DECIMAL_CASE(bit_width, builder, object)      \
-  case bit_width: {                                           \
-    decimal::Decimal##bit_width d;                            \
-    RETURN_NOT_OK(PythonDecimalToArrowDecimal((object), &d)); \
-    RETURN_NOT_OK((builder).Append(d));                       \
-    break;                                                    \
+#define CONVERT_DECIMAL_CASE(bit_width, builder, object)         \
+  case bit_width: {                                              \
+    decimal::Decimal##bit_width d;                               \
+    std::string string_out;                                      \
+    RETURN_NOT_OK(PythonDecimalToString((object), &string_out)); \
+    RETURN_NOT_OK(FromString(string_out, &d));                   \
+    RETURN_NOT_OK((builder).Append(d));                          \
+    break;                                                       \
   }
 
 Status PandasConverter::ConvertDecimals() {
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index cbc93776f98ef..43c7f60aaf36b 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -40,7 +40,7 @@ TEST(PyBuffer, InvalidInputObject) {
   PyBuffer buffer(Py_None);
 }
 
-TEST(DecimalTest, TestPythonDecimalToArrowDecimal128) {
+TEST(DecimalTest, TestPythonDecimalToString) {
   PyAcquireGIL lock;
 
   OwnedRef decimal;
@@ -63,11 +63,13 @@ TEST(DecimalTest, TestPythonDecimalToArrowDecimal128) {
   ASSERT_NE(pydecimal.obj(), nullptr);
   ASSERT_EQ(PyErr_Occurred(), nullptr);
 
-  decimal::Decimal128 arrow_decimal;
   boost::multiprecision::int128_t boost_decimal(decimal_string);
-  PyObject* obj = pydecimal.obj();
-  ASSERT_OK(PythonDecimalToArrowDecimal(obj, &arrow_decimal));
-  ASSERT_EQ(boost_decimal, arrow_decimal.value);
+  PyObject* python_object = pydecimal.obj();
+  ASSERT_NE(python_object, nullptr);
+
+  std::string string_result;
+  ASSERT_OK(PythonDecimalToString(python_object, &string_result));
+  ASSERT_EQ(boost_decimal.str(), string_result);
 }
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {

From af83c451983b3c5b231aea04875767de5336d380 Mon Sep 17 00:00:00 2001
From: Fang Zheng <zhengfang.xjtu@gmail.com>
Date: Thu, 29 Jun 2017 16:01:14 -0400
Subject: [PATCH 0761/1644] ARROW-1166: Fix errors in example and missing
 reference in Layout.md

Author: Fang Zheng <zhengfang.xjtu@gmail.com>

Closes #795 from fangzheng/master and squashes the following commits:

4f9d8d8 [Fang Zheng] ARROW-1166: Fix errors in Struct type's example and missing reference in Layout.md
---
 format/Layout.md | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index b073854c5cd61..1b85abf942bf6 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -384,37 +384,42 @@ The layout for [{'joe', 1}, {null, 2}, null, {'mark', 4}] would be:
 
 * Children arrays:
   * field-0 array (`List<char>`):
-    * Length: 4, Null count: 1
+    * Length: 4, Null count: 2
     * Null bitmap buffer:
 
       | Byte 0 (validity bitmap) | Bytes 1-63            |
       |--------------------------|-----------------------|
-      | 00001101                 | 0 (padding)           |
+      | 00001001                 | 0 (padding)           |
 
     * Offsets buffer:
 
       | Bytes 0-19     |
       |----------------|
-      | 0, 3, 3, 6, 10 |
+      | 0, 3, 3, 3, 7  |
 
      * Values array:
-        * Length: 10, Null count: 0
+        * Length: 7, Null count: 0
         * Null bitmap buffer: Not required
 
         * Value buffer:
 
-          | Bytes 0-9      |
+          | Bytes 0-6      |
           |----------------|
-          | joebobmark     |
+          | joemark        |
 
   * field-1 array (int32 array):
-    * Length: 4, Null count: 0
-    * Null bitmap buffer: Not required
+    * Length: 4, Null count: 1
+    * Null bitmap buffer:
+
+      | Byte 0 (validity bitmap) | Bytes 1-63            |
+      |--------------------------|-----------------------|
+      | 00001011                 | 0 (padding)           |
+
     * Value Buffer:
 
-      | Bytes 0-15     |
-      |----------------|
-      | 1, 2, 3, 4     |
+      |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-63 |
+      |------------|-------------|-------------|-------------|-------------|
+      | 1          | 2           | unspecified | 4           | unspecified |
 
 ```
 
@@ -600,7 +605,7 @@ the the types array indicates that a slot contains a different type at the index
 ## Dictionary encoding
 
 When a field is dictionary encoded, the values are represented by an array of Int32 representing the index of the value in the dictionary.
-The Dictionary is received as a DictionaryBacth whose id is referenced by a dictionary attribute defined in the metadata (Message.fbs) in the Field table.
+The Dictionary is received as a DictionaryBacth whose id is referenced by a dictionary attribute defined in the metadata ([Message.fbs][7]) in the Field table.
 The dictionary has the same layout as the type of the field would dictate. Each entry in the dictionary can be accessed by its index in the DictionaryBatch.
 When a Schema references a Dictionary id, it must send a DictionaryBatch for this id before any RecordBatch.
 
@@ -644,3 +649,4 @@ Apache Drill Documentation - [Value Vectors][6]
 [4]: https://software.intel.com/en-us/node/600110
 [5]: https://parquet.apache.org/documentation/latest/
 [6]: https://drill.apache.org/docs/value-vectors/
+[7]: https://github.com/apache/arrow/blob/master/format/Message.fbs

From 9f500aff84f59db073ec68b52592a0821a79797c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 30 Jun 2017 11:35:52 +0200
Subject: [PATCH 0762/1644] ARROW-1170: C++: Link to pthread on
 ARROW_JEMALLOC=OFF

Also test this on Travis.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #798 from xhochy/ARROW-1170 and squashes the following commits:

f247422 [Uwe L. Korn] C++: Link to pthread on ARROW_JEMALLOC=OFF
---
 ci/travis_before_script_cpp.sh | 3 +++
 cpp/CMakeLists.txt             | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 36966da32f3bc..e250e705f1fbd 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -56,8 +56,11 @@ $CMAKE_COMMON_FLAGS \
 -DARROW_BUILD_TESTS=OFF \
 -DARROW_BUILD_UTILITIES=OFF"
 else
+  # Deactivate jemalloc on Linux builds. We check the jemalloc+Linux build
+  # also in the manylinux1 image.
   CMAKE_LINUX_FLAGS="\
 $CMAKE_LINUX_FLAGS \
+-DARROW_JEMALLOC=ON \
 -DARROW_TEST_MEMCHECK=ON"
 fi
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 1d1ffbea652df..57453382df26c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -616,6 +616,9 @@ elseif (NOT MSVC)
   set(ARROW_LINK_LIBS
     ${ARROW_LINK_LIBS}
     ${PTHREAD_LIBRARY})
+  set(ARROW_STATIC_LINK_LIBS
+    ${ARROW_STATIC_LINK_LIBS}
+    ${PTHREAD_LIBRARY})
 endif()
 
 ############################################################

From 456330f6d2a84ffa28a325e32f2786469cdad820 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sat, 1 Jul 2017 18:00:53 -0400
Subject: [PATCH 0763/1644] ARROW-599: CMake support of LZ4 compression lib

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #799 from MaxRis/ARROW-599 and squashes the following commits:

2e905dae [Max Risuhin] ARROW-599: CMake support of LZ4 compression lib
---
 cpp/cmake_modules/FindLz4.cmake             | 70 +++++++++++++++++++++
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 45 +++++++++++++
 2 files changed, 115 insertions(+)
 create mode 100644 cpp/cmake_modules/FindLz4.cmake

diff --git a/cpp/cmake_modules/FindLz4.cmake b/cpp/cmake_modules/FindLz4.cmake
new file mode 100644
index 0000000000000..e25b0137f9776
--- /dev/null
+++ b/cpp/cmake_modules/FindLz4.cmake
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find LZ4 (lz4.h, liblz4.a, liblz4.so, and liblz4.so.0)
+# This module defines
+#  LZ4_INCLUDE_DIR, directory containing headers
+#  LZ4_SHARED_LIB, path to liblz4 shared library
+#  LZ4_STATIC_LIB, path to liblz4 static library
+#  LZ4_FOUND, whether lz4 has been found
+
+if( NOT "${LZ4_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${LZ4_HOME}" _native_path )
+    list( APPEND _lz4_roots ${_native_path} )
+elseif ( Lz4_HOME )
+    list( APPEND _lz4_roots ${Lz4_HOME} )
+endif()
+
+if (MSVC AND NOT LZ4_MSVC_STATIC_LIB_SUFFIX)
+  set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static")
+endif()
+
+set(LZ4_STATIC_LIB_SUFFIX
+  "${LZ4_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+set(LZ4_STATIC_LIB_NAME
+  ${CMAKE_STATIC_LIBRARY_PREFIX}lz4${LZ4_STATIC_LIB_SUFFIX})
+
+if ( _lz4_roots )
+  find_path(LZ4_INCLUDE_DIR NAMES lz4.h
+    PATHS ${_lz4_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "include" )
+  find_library(LZ4_SHARED_LIB NAMES lz4
+    PATHS ${_lz4_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+  find_library(LZ4_STATIC_LIB NAMES ${LZ4_STATIC_LIB_NAME}
+    PATHS ${_lz4_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+else()
+  find_path(LZ4_INCLUDE_DIR lz4.h
+    # make sure we don't accidentally pick up a different version
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(LZ4_SHARED_LIB lz4
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(LZ4_STATIC_LIB ${LZ4_STATIC_LIB_NAME}
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LZ4 REQUIRED_VARS
+  LZ4_SHARED_LIB LZ4_STATIC_LIB LZ4_INCLUDE_DIR)
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index d77aa53f9c026..8da20874aabd1 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -27,6 +27,7 @@ set(FLATBUFFERS_VERSION "1.7.1")
 set(JEMALLOC_VERSION "4.4.0")
 set(SNAPPY_VERSION "1.1.3")
 set(BROTLI_VERSION "v0.6.0")
+set(LZ4_VERSION "1.7.5")
 
 string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
 
@@ -47,6 +48,7 @@ if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
   set(SNAPPY_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
   set(ZLIB_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
   set(BROTLI_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(LZ4_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
 
   if (NOT DEFINED ENV{BOOST_ROOT})
     # Since we have to set this in the environment, we check whether
@@ -83,6 +85,10 @@ if (DEFINED ENV{BROTLI_HOME})
   set(BROTLI_HOME "$ENV{BROTLI_HOME}")
 endif()
 
+if (DEFINED ENV{LZ4_HOME})
+  set(LZ4_HOME "$ENV{LZ4_HOME}")
+endif()
+
 # ----------------------------------------------------------------------
 # Find pthreads
 
@@ -637,3 +643,42 @@ if (BROTLI_VENDORED)
   add_dependencies(brotli_dec brotli_ep)
   add_dependencies(brotli_common brotli_ep)
 endif()
+
+# ----------------------------------------------------------------------
+# Lz4
+
+find_package(Lz4)
+if (NOT LZ4_FOUND)
+  set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep")
+  set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
+
+  if (MSVC)
+    set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_Release/liblz4_static.lib")
+    set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=Release /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
+  else()
+    set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
+    set(LZ4_BUILD_COMMAND BUILD_COMMAND make -j4)
+  endif()
+
+  ExternalProject_Add(lz4_ep
+      URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz"
+      UPDATE_COMMAND ""
+      PATCH_COMMAND ""
+      CONFIGURE_COMMAND ""
+      INSTALL_COMMAND ""
+      BINARY_DIR ${LZ4_BUILD_DIR}
+      ${LZ4_BUILD_COMMAND}
+      )
+
+  set(LZ4_VENDORED 1)
+else()
+  set(LZ4_VENDORED 0)
+endif()
+
+include_directories(SYSTEM ${LZ4_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(lz4_static
+  STATIC_LIB ${LZ4_STATIC_LIB})
+
+if (LZ4_VENDORED)
+  add_dependencies(lz4_static lz4_ep)
+endif()

From 930db87d63571756118c6edc2e4df3a678d92393 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 29 Jun 2017 23:27:02 +0200
Subject: [PATCH 0764/1644] ARROW-1169: [C++] jemalloc externalproject doesn't
 build with CMake's ninja generator

Closes #796

Change-Id: I3cbd06d774feca254183e0ed1b0e75111f7cb389
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 8da20874aabd1..857334579819c 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -407,16 +407,16 @@ if (ARROW_JEMALLOC)
         URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
         CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
         BUILD_IN_SOURCE 1
-        BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
+        BUILD_COMMAND ${MAKE}
         BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
-        INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j1 install)
+        INSTALL_COMMAND ${MAKE} -j1 install)
     else()
       ExternalProject_Add(jemalloc_ep
         URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
         CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
         BUILD_IN_SOURCE 1
-        BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
-        INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j1 install)
+        BUILD_COMMAND ${MAKE}
+        INSTALL_COMMAND ${MAKE} -j1 install)
     endif()
   else()
     set(JEMALLOC_VENDORED 0)

From c294ec3dbdd3f5cfa55a6eb5c7b27535b240ccf0 Mon Sep 17 00:00:00 2001
From: fjetter <florian.jetter@blue-yonder.com>
Date: Sat, 1 Jul 2017 18:12:09 -0400
Subject: [PATCH 0765/1644] ARROW-1125: partial schemas for Table.from_pandas

Author: fjetter <florian.jetter@blue-yonder.com>

Closes #790 from fjetter/ARROW-1125-partial-schemas and squashes the following commits:

0a58b708 [fjetter] Remove trailing whitespaces
87ccb0c4 [fjetter] Fix indentation to respect max line length
92001422 [fjetter] Remove template from TypeNotImplemented status message
67dbba5d [fjetter] Remove range from test due to pandas bug on Windows
4890b5af [fjetter] Refactor TypeNotImplemented message
9de8611c [fjetter] Partial schema test in test_convert_pandas.py
dcf44f09 [fjetter] Allow partial schemas in Table.from_pandas again
66671a27 [fjetter] Improved NotImplemented messages in PandasConverter
---
 cpp/src/arrow/python/pandas_convert.cc      | 22 ++++++++++---------
 cpp/src/arrow/type.h                        |  3 +++
 python/pyarrow/table.pxi                    |  3 ++-
 python/pyarrow/tests/test_convert_pandas.py | 24 +++++++++++++++++++--
 python/pyarrow/tests/test_table.py          | 18 ----------------
 5 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 2918f9e6b10d0..9b655708b4dd6 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -362,23 +362,25 @@ class PandasConverter {
   Status Visit(const Time32Type& type) { return VisitNative<Int32Type>(); }
   Status Visit(const Time64Type& type) { return VisitNative<Int64Type>(); }
 
-  Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
+  Status TypeNotImplemented(std::string type_name) {
+    std::stringstream ss;
+    ss << "PandasConverter doesn't implement <" << type_name << "> conversion. ";
+    return Status::NotImplemented(ss.str());
+  }
 
-  Status Visit(const BinaryType& type) { return Status::NotImplemented(type.ToString()); }
+  Status Visit(const NullType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const BinaryType& type) { return TypeNotImplemented(type.ToString()); }
 
   Status Visit(const FixedSizeBinaryType& type) {
-    return Status::NotImplemented(type.ToString());
+    return TypeNotImplemented(type.ToString());
   }
 
-  Status Visit(const DecimalType& type) {
-    return Status::NotImplemented(type.ToString());
-  }
+  Status Visit(const DecimalType& type) { return TypeNotImplemented(type.ToString()); }
 
-  Status Visit(const DictionaryType& type) {
-    return Status::NotImplemented(type.ToString());
-  }
+  Status Visit(const DictionaryType& type) { return TypeNotImplemented(type.ToString()); }
 
-  Status Visit(const NestedType& type) { return Status::NotImplemented(type.ToString()); }
+  Status Visit(const NestedType& type) { return TypeNotImplemented(type.ToString()); }
 
   Status Convert() {
     if (PyArray_NDIM(arr_) != 1) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 3e85291f043e4..8338800225912 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -194,6 +194,7 @@ class ARROW_EXPORT FloatingPoint : public PrimitiveCType {
 class ARROW_EXPORT NestedType : public DataType {
  public:
   using DataType::DataType;
+  static std::string name() { return "nested"; }
 };
 
 class NoExtraMeta {};
@@ -406,6 +407,7 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+  static std::string name() {return "fixed_size_binary"; }
 
   std::vector<BufferDescr> GetBufferLayout() const override;
 
@@ -674,6 +676,7 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+  static std::string name() { return "dictionary"; }
 
   bool ordered() const { return ordered_; }
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 7d44f2e46eec6..ef83636daff84 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -333,7 +333,8 @@ cdef tuple _dataframe_to_arrays(
     for name in df.columns:
         col = df[name]
         if schema is not None:
-            type = schema.field_by_name(name).type
+            field = schema.field_by_name(name)
+            type = getattr(field, "type", None)
 
         arr = arrays.append(
             Array.from_pandas(
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 49b7eb73df09b..9cce7bbdc7a52 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -102,13 +102,11 @@ def test_all_none_objects(self):
         df = pd.DataFrame({'a': [None, None, None]})
         self._check_pandas_roundtrip(df)
 
-
     def test_all_none_category(self):
         df = pd.DataFrame({'a': [None, None, None]})
         df['a'] = df['a'].astype('category')
         self._check_pandas_roundtrip(df)
 
-
     def test_float_no_nulls(self):
         data = {}
         fields = []
@@ -654,3 +652,25 @@ def test_multiindex_duplicate_values(self):
         table = pa.Table.from_pandas(df)
         result_df = table.to_pandas()
         tm.assert_frame_equal(result_df, df)
+
+    def test_partial_schema(self):
+        data = OrderedDict([
+            ('a', [0, 1, 2, 3, 4]),
+            ('b', np.array([-10, -5, 0, 5, 10], dtype=np.int32)),
+            ('c', [-10, -5, 0, 5, 10])
+        ])
+        df = pd.DataFrame(data)
+
+        partial_schema = pa.schema([
+            pa.field('a', pa.int64()),
+            pa.field('b', pa.int32())
+        ])
+
+        expected_schema = pa.schema([
+            pa.field('a', pa.int64()),
+            pa.field('b', pa.int32()),
+            pa.field('c', pa.int64())
+        ])
+
+        self._check_pandas_roundtrip(df, schema=partial_schema,
+                                     expected_schema=expected_schema)
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index afc95205927be..3198941c0b752 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -238,24 +238,6 @@ def test_concat_tables():
     assert result.equals(expected)
 
 
-def test_table_pandas():
-    data = [
-        pa.array(range(5)),
-        pa.array([-10, -5, 0, 5, 10])
-    ]
-    table = pa.Table.from_arrays(data, names=('a', 'b'))
-
-    # TODO: Use this part once from_pandas is implemented
-    # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}
-    # df = pd.DataFrame(data)
-    # pa.Table.from_pandas(df)
-
-    df = table.to_pandas()
-    assert set(df.columns) == set(('a', 'b'))
-    assert df.shape == (5, 2)
-    assert df.loc[0, 'b'] == -10
-
-
 def test_table_negative_indexing():
     data = [
         pa.array(range(5)),

From 96e7e9979bd522b2231cc33c4196c2418a24e0fc Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 27 Jun 2017 16:41:06 +0200
Subject: [PATCH 0766/1644] ARROW-960: Add section on how to develop with pip

Closes #788

Change-Id: Ia904d5e065c15ba83cf39f57ef97a4f6710aa60f
---
 python/doc/source/development.rst | 93 +++++++++++++++++++++++--------
 1 file changed, 71 insertions(+), 22 deletions(-)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 2063ba821b68e..8a70180130ca0 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -22,14 +22,11 @@
 Development
 ***********
 
-Developing with conda
-=====================
-
-Linux and macOS
----------------
+Developing on Linux and MacOS
+=============================
 
 System Requirements
-~~~~~~~~~~~~~~~~~~~
+-------------------
 
 On macOS, any modern XCode (6.4 or higher; the current version is 8.3.1) is
 sufficient.
@@ -55,20 +52,9 @@ Finally, set gcc 4.9 as the active compiler using:
    export CXX=g++-4.9
 
 Environment Setup and Build
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-First, let's create a conda environment with all the C++ build and Python
-dependencies from conda-forge:
-
-.. code-block:: shell
-
-   conda create -y -q -n pyarrow-dev \
-         python=3.6 numpy six setuptools cython pandas pytest \
-         cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
-         brotli jemalloc -c conda-forge
-   source activate pyarrow-dev
+---------------------------
 
-Now, let's clone the Arrow and Parquet git repositories:
+First, let's clone the Arrow and Parquet git repositories:
 
 .. code-block:: shell
 
@@ -87,6 +73,21 @@ You should now see
    drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 arrow/
    drwxrwxr-x 12 wesm wesm 4096 Apr 15 19:19 parquet-cpp/
 
+Using Conda
+~~~~~~~~~~~
+
+Let's create a conda environment with all the C++ build and Python dependencies
+from conda-forge:
+
+.. code-block:: shell
+
+   conda create -y -q -n pyarrow-dev \
+         python=3.6 numpy six setuptools cython pandas pytest \
+         cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
+         brotli jemalloc -c conda-forge
+   source activate pyarrow-dev
+
+
 We need to set some environment variables to let Arrow's build system know
 about our build toolchain:
 
@@ -99,6 +100,55 @@ about our build toolchain:
    export ARROW_HOME=$CONDA_PREFIX
    export PARQUET_HOME=$CONDA_PREFIX
 
+Using pip
+~~~~~~~~~
+
+On macOS, install all dependencies through Homebrew that are required for
+building Arrow C++:
+
+.. code-block:: shell
+
+   brew install ccache jemalloc boost thrift
+
+On Debian/Ubuntu, you need the following minimal set of dependencies. All other
+dependencies will be automatically built by Arrow' thrid-party toolchain.
+
+.. code-block:: shell
+
+   $ sudo apt-get install libjemalloc-dev libboost-dev \
+                          libboost-filesystem-dev \
+                          libboost-system-dev
+
+Now, let's create a Python virtualenv with all Python dependencies in the same
+folder as the repositories and a target installation folder:
+
+.. code-block:: shell
+
+   virtualenv pyarrow
+   source ./pyarrow/bin/activate
+   pip install six numpy pandas cython pytest
+
+   # This is the folder where we will install Arrow and Parquet to during
+   # development
+   mkdir dist
+
+If your cmake version is too old on Linux, you could get a newer one via ``pip
+install cmake``.
+
+We need to set some environment variables to let Arrow's build system know
+about our build toolchain:
+
+.. code-block:: shell
+
+   export ARROW_BUILD_TYPE=release
+
+   export ARROW_HOME=$(pwd)/dist
+   export PARQUET_HOME=$(pwd)/dist
+   export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH
+
+Build and test
+--------------
+
 Now build and install the Arrow C++ libraries:
 
 .. code-block:: shell
@@ -127,7 +177,6 @@ toolchain:
          -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
          -DPARQUET_BUILD_BENCHMARKS=off \
          -DPARQUET_BUILD_EXECUTABLES=off \
-         -DPARQUET_ZLIB_VENDORED=off \
          -DPARQUET_BUILD_TESTS=off \
          ..
 
@@ -179,8 +228,8 @@ You can build a wheel by running:
 
 Again, if you did not build parquet-cpp, you should omit ``--with-parquet``.
 
-Windows
-=======
+Developing on Windows
+=====================
 
 First, we bootstrap a conda environment similar to the `C++ build instructions
 <https://github.com/apache/arrow/blob/master/cpp/apidoc/Windows.md>`_. This

From e268ce87e5d8f93ff6ecc871d218a58605b9692f Mon Sep 17 00:00:00 2001
From: Itai Incze <itai.in@gmail.com>
Date: Sun, 2 Jul 2017 13:38:46 -0400
Subject: [PATCH 0767/1644] ARROW-915: [Python] Struct Array reads limited
 support

Add limited struct array reading support in pyarrow.

This is done to complement parquet-cpp struct reader.

cc @wesm

Author: Itai Incze <itai.in@gmail.com>

Closes #615 from itaiin/ARROW-915 and squashes the following commits:

d8f2636e [Itai Incze] convert struct field names using frombytes
e654abfa [Itai Incze] fix python3 tests & msvc build
3a4edf43 [Itai Incze] fix lint errors
bef46447 [Itai Incze] Refactor due to review
e2a697ff [Itai Incze] Further fixes due to review
eecbb32f [Itai Incze] fix per code review
4c255391 [Itai Incze] Add basic StructArray read support
---
 cpp/src/arrow/python/pandas_convert.cc       | 69 +++++++++++++++++++-
 python/pyarrow/__init__.py                   |  2 +-
 python/pyarrow/_parquet.pxd                  |  1 +
 python/pyarrow/_parquet.pyx                  | 11 ++++
 python/pyarrow/array.pxi                     | 53 +++++++++++++++
 python/pyarrow/includes/libarrow.pxd         | 15 +++++
 python/pyarrow/lib.pxd                       |  1 +
 python/pyarrow/tests/test_convert_builtin.py | 17 +++++
 python/pyarrow/tests/test_convert_pandas.py  | 17 +++++
 9 files changed, 184 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 9b655708b4dd6..60ae1b23844b6 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -1354,6 +1354,66 @@ inline Status ConvertFixedSizeBinary(const ChunkedArray& data, PyObject** out_va
   return Status::OK();
 }
 
+inline Status ConvertStruct(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  if (data.num_chunks() <= 0) { return Status::OK(); }
+  // ChunkedArray has at least one chunk
+  auto arr = static_cast<const StructArray*>(data.chunk(0).get());
+  // Use it to cache the struct type and number of fields for all chunks
+  auto num_fields = arr->fields().size();
+  auto array_type = arr->type();
+  std::vector<OwnedRef> fields_data(num_fields);
+  OwnedRef dict_item;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto arr = static_cast<const StructArray*>(data.chunk(c).get());
+    // Convert the struct arrays first
+    for (size_t i = 0; i < num_fields; i++) {
+      PyObject* numpy_array;
+      RETURN_NOT_OK(
+          ConvertArrayToPandas(arr->field(static_cast<int>(i)), nullptr, &numpy_array));
+      fields_data[i].reset(numpy_array);
+    }
+
+    // Construct a dictionary for each row
+    const bool has_nulls = data.null_count() > 0;
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (has_nulls && arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values = Py_None;
+      } else {
+        // Build the new dict object for the row
+        dict_item.reset(PyDict_New());
+        RETURN_IF_PYERROR();
+        for (size_t field_idx = 0; field_idx < num_fields; ++field_idx) {
+          OwnedRef field_value;
+          auto name = array_type->child(static_cast<int>(field_idx))->name();
+          if (!arr->field(static_cast<int>(field_idx))->IsNull(i)) {
+            // Value exists in child array, obtain it
+            auto array = reinterpret_cast<PyArrayObject*>(fields_data[field_idx].obj());
+            auto ptr = reinterpret_cast<const char*>(PyArray_GETPTR1(array, i));
+            field_value.reset(PyArray_GETITEM(array, ptr));
+            RETURN_IF_PYERROR();
+          } else {
+            // Translate the Null to a None
+            Py_INCREF(Py_None);
+            field_value.reset(Py_None);
+          }
+          // PyDict_SetItemString does not steal the value reference
+          auto setitem_result =
+              PyDict_SetItemString(dict_item.obj(), name.c_str(), field_value.obj());
+          RETURN_IF_PYERROR();
+          DCHECK_EQ(setitem_result, 0);
+        }
+        *out_values = dict_item.obj();
+        // Grant ownership to the resulting array
+        Py_INCREF(*out_values);
+      }
+      ++out_values;
+    }
+  }
+  return Status::OK();
+}
+
 template <typename ArrowType>
 inline Status ConvertListsLike(
     const std::shared_ptr<Column>& col, PyObject** out_values) {
@@ -1499,6 +1559,8 @@ class ObjectBlock : public PandasBlock {
           return Status::NotImplemented(ss.str());
         }
       }
+    } else if (type == Type::STRUCT) {
+      RETURN_NOT_OK(ConvertStruct(data, out_buffer));
     } else {
       std::stringstream ss;
       ss << "Unsupported type for object array output: " << col->type()->ToString();
@@ -1960,6 +2022,7 @@ class DataFrameBlockCreator {
           output_type = PandasBlock::DECIMAL;
           break;
         case Type::NA:
+        case Type::STRUCT:
           output_type = PandasBlock::OBJECT;
           break;
         default:
@@ -2355,7 +2418,11 @@ class ArrowDeserializer {
     return ConvertNulls(data_, out_values);
   }
 
-  Status Visit(const StructType& type) { return Status::NotImplemented("struct type"); }
+  Status Visit(const StructType& type) {
+    AllocateOutput(NPY_OBJECT);
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    return ConvertStruct(data_, out_values);
+  }
 
   Status Visit(const UnionType& type) { return Status::NotImplemented("union type"); }
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 237d44f01ca48..771a516e147a5 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -59,7 +59,7 @@
                          DictionaryArray,
                          Date32Array, Date64Array,
                          TimestampArray, Time32Array, Time64Array,
-                         DecimalArray,
+                         DecimalArray, StructArray,
                          ArrayValue, Scalar, NA, NAType,
                          BooleanValue,
                          Int8Value, Int16Value, Int32Value, Int64Value,
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 2f6b9a9d64898..3d2d0c86ce021 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -214,6 +214,7 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
     cdef cppclass FileReader:
         FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
         CStatus ReadColumn(int i, shared_ptr[CArray]* out)
+        CStatus ReadSchemaField(int i, shared_ptr[CArray]* out)
 
         int num_row_groups()
         CStatus ReadRowGroup(int i, shared_ptr[CTable]* out)
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 77ef7adadef7e..5d446a81044b9 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -501,6 +501,17 @@ cdef class ParquetReader:
         array.init(carray)
         return array
 
+    def read_schema_field(self, int field_index):
+        cdef:
+            Array array = Array()
+            shared_ptr[CArray] carray
+
+        with nogil:
+            check_status(self.reader.get()
+                         .ReadSchemaField(field_index, &carray));
+
+        array.init(carray)
+        return array
 
 cdef int check_compression_name(name) except -1:
     if name.upper() not in ['NONE', 'SNAPPY', 'GZIP', 'LZO', 'BROTLI']:
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index e1be5b1f63d8b..79e88fc954549 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -996,6 +996,21 @@ cdef class FixedSizeBinaryValue(ArrayValue):
         return cp.PyBytes_FromStringAndSize(data, length)
 
 
+cdef class StructValue(ArrayValue):
+    def as_py(self):
+        cdef:
+            CStructArray* ap
+            vector[shared_ptr[CField]] child_fields = self.type.type.children()
+        ap = <CStructArray*> self.sp_array.get()
+        child_arrays = ap.fields()
+        wrapped_arrays = (pyarrow_wrap_array(child) for child in child_arrays)
+        child_names = (child.get().name() for child in child_fields)
+        # Return the struct as a dict
+        return {
+            frombytes(name): child_array[self.index].as_py()
+            for name, child_array in
+            zip(child_names, wrapped_arrays)
+        }
 
 cdef dict _scalar_classes = {
     _Type_BOOL: BooleanValue,
@@ -1019,6 +1034,7 @@ cdef dict _scalar_classes = {
     _Type_STRING: StringValue,
     _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
     _Type_DECIMAL: DecimalValue,
+    _Type_STRUCT: StructValue,
 }
 
 cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
@@ -1589,6 +1605,42 @@ cdef class DictionaryArray(Array):
         result.init(c_result)
         return result
 
+cdef class StructArray(Array):
+    @staticmethod
+    def from_arrays(field_names, arrays):
+        cdef:
+            Array array
+            shared_ptr[CArray] c_array
+            vector[shared_ptr[CArray]] c_arrays
+            shared_ptr[CArray] c_result
+            ssize_t num_arrays
+            ssize_t length
+            ssize_t i
+
+        num_arrays = len(arrays)
+        if num_arrays == 0:
+            raise ValueError("arrays list is empty")
+
+        length = len(arrays[0])
+
+        c_arrays.resize(num_arrays)
+        for i in range(num_arrays):
+            array = arrays[i]
+            if len(array) != length:
+                raise ValueError("All arrays must have the same length")
+            c_arrays[i] = array.sp_array
+
+        cdef DataType struct_type = struct([
+            field(name, array.type)
+            for name, array in
+            zip(field_names, arrays)
+        ])
+
+        c_result.reset(new CStructArray(struct_type.sp_type, length, c_arrays))
+        result = StructArray()
+        result.init(c_result)
+        return result
+
 
 cdef dict _array_classes = {
     _Type_NA: NullArray,
@@ -1614,6 +1666,7 @@ cdef dict _array_classes = {
     _Type_DICTIONARY: DictionaryArray,
     _Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
     _Type_DECIMAL: DecimalArray,
+    _Type_STRUCT: StructArray,
 }
 
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 741d8321158f6..fb101fe1c7dd6 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -76,6 +76,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_bool Equals(const CDataType& other)
 
+        shared_ptr[CField] child(int i)
+
+        const vector[shared_ptr[CField]] children()
+
+        int num_children()
+
         c_string ToString()
 
     cdef cppclass CArray" arrow::Array":
@@ -283,6 +289,15 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CStringArray" arrow::StringArray"(CBinaryArray):
         c_string GetString(int i)
 
+    cdef cppclass CStructArray" arrow::StructArray"(CArray):
+        CStructArray(shared_ptr[CDataType] type, int64_t length,
+            vector[shared_ptr[CArray]] children,
+            shared_ptr[CBuffer] null_bitmap = nullptr, int64_t null_count = 0,
+            int64_t offset = 0)
+
+        shared_ptr[CArray] field(int pos)
+        const vector[shared_ptr[CArray]] fields()
+
     cdef cppclass CChunkedArray" arrow::ChunkedArray":
         int64_t length()
         int64_t null_count()
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 4a2ab8698f693..8fa7fd9d98df0 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -18,6 +18,7 @@
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 from cpython cimport PyObject
+from libcpp cimport nullptr
 
 cdef extern from "Python.h":
     int PySlice_Check(object)
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index bf14c4f232852..62592f9700536 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -258,3 +258,20 @@ def test_empty_range(self):
         assert arr.null_count == 0
         assert arr.type == pa.null()
         assert arr.to_pylist() == []
+
+    def test_structarray(self):
+        ints = pa.array([None, 2, 3], type=pa.int64())
+        strs = pa.array([u'a', None, u'c'], type=pa.string())
+        bools = pa.array([True, False, None], type=pa.bool_())
+        arr = pa.StructArray.from_arrays(
+            ['ints', 'strs', 'bools'],
+            [ints, strs, bools])
+
+        expected = [
+            {'ints': None, 'strs': u'a', 'bools': True},
+            {'ints': 2, 'strs': None, 'bools': False},
+            {'ints': 3, 'strs': u'c', 'bools': None},
+        ]
+
+        pylist = arr.to_pylist()
+        assert pylist == expected, (pylist, expected)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 9cce7bbdc7a52..b952d4a5ffd13 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -674,3 +674,20 @@ def test_partial_schema(self):
 
         self._check_pandas_roundtrip(df, schema=partial_schema,
                                      expected_schema=expected_schema)
+
+    def test_structarray(self):
+        ints = pa.array([None, 2, 3], type=pa.int64())
+        strs = pa.array([u'a', None, u'c'], type=pa.string())
+        bools = pa.array([True, False, None], type=pa.bool_())
+        arr = pa.StructArray.from_arrays(
+            ['ints', 'strs', 'bools'],
+            [ints, strs, bools])
+
+        expected = pd.Series([
+            {'ints': None, 'strs': u'a', 'bools': True},
+            {'ints': 2, 'strs': None, 'bools': False},
+            {'ints': 3, 'strs': u'c', 'bools': None},
+        ])
+
+        series = pd.Series(arr.to_pandas())
+        tm.assert_series_equal(series, expected)

From 9e4906f7ff911c3a869976b2add30d3066cbe9cf Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 2 Jul 2017 17:41:34 -0400
Subject: [PATCH 0768/1644] ARROW-1160: C++: Implement DictionaryBuilder

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #792 from xhochy/ARROW-1160 and squashes the following commits:

cf59f3a1 [Uwe L. Korn] Add benchmarks
809cdc71 [Uwe L. Korn] Incorporate review comments
69cc6a92 [Uwe L. Korn] ninja lint
957a1a9b [Uwe L. Korn] Generalize DictionaryBuilder
c97b423e [Uwe L. Korn] Fix conversion warning.
ea0d41ae [Uwe L. Korn] ARROW-1160: C++: Implement DictionaryBuilder
---
 cpp/src/arrow/array-decimal-test.cc           |  12 +-
 cpp/src/arrow/array-test.cc                   | 145 ++++++++++++
 cpp/src/arrow/buffer.h                        |   5 +-
 cpp/src/arrow/builder-benchmark.cc            |  43 ++++
 cpp/src/arrow/builder.cc                      | 211 ++++++++++++++++++
 cpp/src/arrow/builder.h                       |  85 +++++++
 cpp/src/arrow/io/io-hdfs-test.cc              |   7 +-
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc |   4 +-
 cpp/src/arrow/python/builtin_convert.cc       |   5 +-
 cpp/src/arrow/type.h                          |   2 +-
 cpp/src/arrow/util/hash-util.h                |   8 +-
 11 files changed, 507 insertions(+), 20 deletions(-)

diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index fb4b8d983d345..2a3d0a905acc6 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -170,14 +170,14 @@ TEST_P(Decimal128BuilderTest, WithNulls) {
 }
 
 INSTANTIATE_TEST_CASE_P(Decimal32BuilderTest, Decimal32BuilderTest,
-    ::testing::Range(
-        DecimalPrecision<int32_t>::minimum, DecimalPrecision<int32_t>::maximum));
+    ::testing::Range(DecimalPrecision<int32_t>::minimum,
+                            DecimalPrecision<int32_t>::maximum));
 INSTANTIATE_TEST_CASE_P(Decimal64BuilderTest, Decimal64BuilderTest,
-    ::testing::Range(
-        DecimalPrecision<int64_t>::minimum, DecimalPrecision<int64_t>::maximum));
+    ::testing::Range(DecimalPrecision<int64_t>::minimum,
+                            DecimalPrecision<int64_t>::maximum));
 INSTANTIATE_TEST_CASE_P(Decimal128BuilderTest, Decimal128BuilderTest,
-    ::testing::Range(
-        DecimalPrecision<int128_t>::minimum, DecimalPrecision<int128_t>::maximum));
+    ::testing::Range(DecimalPrecision<int128_t>::minimum,
+                            DecimalPrecision<int128_t>::maximum));
 
 }  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 8f6323b7555ba..b28a3a3630205 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1487,6 +1487,151 @@ TEST_F(TestAdaptiveUIntBuilder, TestAppendVector) {
   ASSERT_TRUE(expected_->Equals(result_));
 }
 
+// ----------------------------------------------------------------------
+// Dictionary tests
+
+template <typename Type>
+class TestDictionaryBuilder : public TestBuilder {};
+
+typedef ::testing::Types<Int8Type, UInt8Type, Int16Type, UInt16Type, Int32Type,
+    UInt32Type, Int64Type, UInt64Type, FloatType, DoubleType>
+    PrimitiveDictionaries;
+
+TYPED_TEST_CASE(TestDictionaryBuilder, PrimitiveDictionaries);
+
+TYPED_TEST(TestDictionaryBuilder, Basic) {
+  DictionaryBuilder<TypeParam> builder(default_memory_pool());
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  NumericBuilder<TypeParam> dict_builder(default_memory_pool());
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> dict_array;
+  ASSERT_OK(dict_builder.Finish(&dict_array));
+  auto dtype =
+      std::make_shared<DictionaryType>(std::make_shared<TypeParam>(), dict_array);
+
+  UInt8Builder int_builder(default_memory_pool());
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
+  using Scalar = typename TypeParam::c_type;
+  // Skip this test for (u)int8
+  if (sizeof(Scalar) > 1) {
+    // Build the dictionary Array
+    DictionaryBuilder<TypeParam> builder(default_memory_pool());
+    // Build expected data
+    NumericBuilder<TypeParam> dict_builder(default_memory_pool());
+    UInt16Builder int_builder(default_memory_pool());
+
+    // Fill with 1024 different values
+    for (int64_t i = 0; i < 1024; i++) {
+      ASSERT_OK(builder.Append(static_cast<Scalar>(i)));
+      ASSERT_OK(dict_builder.Append(static_cast<Scalar>(i)));
+      ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+    }
+    // Fill with an already existing value
+    for (int64_t i = 0; i < 1024; i++) {
+      ASSERT_OK(builder.Append(static_cast<Scalar>(1)));
+      ASSERT_OK(int_builder.Append(1));
+    }
+
+    // Finalize result
+    std::shared_ptr<Array> result;
+    ASSERT_OK(builder.Finish(&result));
+
+    // Finalize expected data
+    std::shared_ptr<Array> dict_array;
+    ASSERT_OK(dict_builder.Finish(&dict_array));
+    auto dtype =
+        std::make_shared<DictionaryType>(std::make_shared<TypeParam>(), dict_array);
+    std::shared_ptr<Array> int_array;
+    ASSERT_OK(int_builder.Finish(&int_array));
+
+    DictionaryArray expected(dtype, int_array);
+    ASSERT_TRUE(expected.Equals(result));
+  }
+}
+
+TEST(TestStringDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  StringDictionaryBuilder builder(default_memory_pool());
+  ASSERT_OK(builder.Append("test"));
+  ASSERT_OK(builder.Append("test2"));
+  ASSERT_OK(builder.Append("test"));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  StringBuilder str_builder(default_memory_pool());
+  ASSERT_OK(str_builder.Append("test"));
+  ASSERT_OK(str_builder.Append("test2"));
+  std::shared_ptr<Array> str_array;
+  ASSERT_OK(str_builder.Finish(&str_array));
+  auto dtype = std::make_shared<DictionaryType>(utf8(), str_array);
+
+  UInt8Builder int_builder(default_memory_pool());
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestStringDictionaryBuilder, DoubleTableSize) {
+  // Build the dictionary Array
+  StringDictionaryBuilder builder(default_memory_pool());
+  // Build expected data
+  StringBuilder str_builder(default_memory_pool());
+  UInt16Builder int_builder(default_memory_pool());
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    std::stringstream ss;
+    ss << "test" << i;
+    ASSERT_OK(builder.Append(ss.str()));
+    ASSERT_OK(str_builder.Append(ss.str()));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append("test1"));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> str_array;
+  ASSERT_OK(str_builder.Finish(&str_array));
+  auto dtype = std::make_shared<DictionaryType>(utf8(), str_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
 // ----------------------------------------------------------------------
 // List tests
 
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index a02ce3cbe8107..bfbea776f1a5f 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -241,8 +241,9 @@ class ARROW_EXPORT BufferBuilder {
     capacity_ = size_ = 0;
     return result;
   }
-  int64_t capacity() { return capacity_; }
-  int64_t length() { return size_; }
+  int64_t capacity() const { return capacity_; }
+  int64_t length() const { return size_; }
+  const uint8_t* data() const { return data_; }
 
  private:
   std::shared_ptr<PoolBuffer> buffer_;
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index bdfba8b6c997b..5eae9ab5c917f 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -121,6 +121,47 @@ static void BM_BuildAdaptiveUIntNoNulls(
   state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t));
 }
 
+static void BM_BuildDictionary(benchmark::State& state) {  // NOLINT non-const reference
+  const int64_t iterations = 1024;
+  while (state.KeepRunning()) {
+    DictionaryBuilder<Int64Type> builder(default_memory_pool());
+    for (int64_t i = 0; i < iterations; i++) {
+      for (int64_t j = 0; j < i; j++) {
+        ABORT_NOT_OK(builder.Append(j));
+      }
+    }
+    std::shared_ptr<Array> out;
+    ABORT_NOT_OK(builder.Finish(&out));
+  }
+  state.SetBytesProcessed(
+      state.iterations() * iterations * (iterations + 1) / 2 * sizeof(int64_t));
+}
+
+static void BM_BuildStringDictionary(
+    benchmark::State& state) {  // NOLINT non-const reference
+  const int64_t iterations = 1024;
+  // Pre-render strings
+  std::vector<std::string> data;
+  for (int64_t i = 0; i < iterations; i++) {
+    std::stringstream ss;
+    ss << i;
+    data.push_back(ss.str());
+  }
+  while (state.KeepRunning()) {
+    StringDictionaryBuilder builder(default_memory_pool());
+    for (int64_t i = 0; i < iterations; i++) {
+      for (int64_t j = 0; j < i; j++) {
+        ABORT_NOT_OK(builder.Append(data[j]));
+      }
+    }
+    std::shared_ptr<Array> out;
+    ABORT_NOT_OK(builder.Finish(&out));
+  }
+  // Assuming a string here needs on average 2 bytes
+  state.SetBytesProcessed(
+      state.iterations() * iterations * (iterations + 1) / 2 * sizeof(int32_t));
+}
+
 BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildVectorNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildAdaptiveIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
@@ -128,5 +169,7 @@ BENCHMARK(BM_BuildAdaptiveIntNoNullsScalarAppend)
     ->Repetitions(3)
     ->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildAdaptiveUIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 6762e172676cb..e61258c91a232 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -28,7 +28,9 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/cpu-info.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/hash-util.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -655,6 +657,204 @@ Status BooleanBuilder::Append(
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// DictionaryBuilder
+
+template <typename T, typename Scalar>
+DictionaryBuilder<T, Scalar>::DictionaryBuilder(
+    MemoryPool* pool, const std::shared_ptr<DataType>& type)
+    : ArrayBuilder(pool, type),
+      hash_table_(new PoolBuffer(pool)),
+      hash_slots_(nullptr),
+      dict_builder_(pool, type),
+      values_builder_(pool) {
+  if (!::arrow::CpuInfo::initialized()) { ::arrow::CpuInfo::Init(); }
+}
+
+template <typename T, typename Scalar>
+Status DictionaryBuilder<T, Scalar>::Init(int64_t elements) {
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+
+  // Fill the initial hash table
+  RETURN_NOT_OK(hash_table_->Resize(sizeof(hash_slot_t) * kInitialHashTableSize));
+  hash_slots_ = reinterpret_cast<int32_t*>(hash_table_->mutable_data());
+  std::fill(hash_slots_, hash_slots_ + kInitialHashTableSize, kHashSlotEmpty);
+  hash_table_size_ = kInitialHashTableSize;
+  mod_bitmask_ = kInitialHashTableSize - 1;
+
+  return values_builder_.Init(elements);
+}
+
+template <typename T, typename Scalar>
+Status DictionaryBuilder<T, Scalar>::Resize(int64_t capacity) {
+  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+
+  if (capacity_ == 0) {
+    return Init(capacity);
+  } else {
+    return ArrayBuilder::Resize(capacity);
+  }
+}
+
+template <typename T, typename Scalar>
+Status DictionaryBuilder<T, Scalar>::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<Array> dictionary;
+  RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
+  auto type = std::make_shared<DictionaryType>(type_, dictionary);
+
+  std::shared_ptr<Array> values;
+  RETURN_NOT_OK(values_builder_.Finish(&values));
+
+  *out = std::make_shared<DictionaryArray>(type, values);
+  return Status::OK();
+}
+
+template <typename T, typename Scalar>
+Status DictionaryBuilder<T, Scalar>::Append(const Scalar& value) {
+  RETURN_NOT_OK(Reserve(1));
+  // Based on DictEncoder<DType>::Put
+  int j = HashValue(value) & mod_bitmask_;
+  hash_slot_t index = hash_slots_[j];
+
+  // Find an empty slot
+  while (kHashSlotEmpty != index && SlotDifferent(index, value)) {
+    // Linear probing
+    ++j;
+    if (j == hash_table_size_) { j = 0; }
+    index = hash_slots_[j];
+  }
+
+  if (index == kHashSlotEmpty) {
+    // Not in the hash table, so we insert it now
+    index = static_cast<hash_slot_t>(dict_builder_.length());
+    hash_slots_[j] = index;
+    RETURN_NOT_OK(AppendDictionary(value));
+
+    if (UNLIKELY(static_cast<int32_t>(dict_builder_.length()) >
+                 hash_table_size_ * kMaxHashTableLoad)) {
+      RETURN_NOT_OK(DoubleTableSize());
+    }
+  }
+
+  RETURN_NOT_OK(values_builder_.Append(index));
+
+  return Status::OK();
+}
+
+template <typename T, typename Scalar>
+Status DictionaryBuilder<T, Scalar>::DoubleTableSize() {
+  int new_size = hash_table_size_ * 2;
+  auto new_hash_table = std::make_shared<PoolBuffer>(pool_);
+
+  RETURN_NOT_OK(new_hash_table->Resize(sizeof(hash_slot_t) * new_size));
+  int32_t* new_hash_slots = reinterpret_cast<int32_t*>(new_hash_table->mutable_data());
+  std::fill(new_hash_slots, new_hash_slots + new_size, kHashSlotEmpty);
+  int new_mod_bitmask = new_size - 1;
+
+  for (int i = 0; i < hash_table_size_; ++i) {
+    hash_slot_t index = hash_slots_[i];
+
+    if (index == kHashSlotEmpty) { continue; }
+
+    // Compute the hash value mod the new table size to start looking for an
+    // empty slot
+    Scalar value = GetDictionaryValue(static_cast<int64_t>(index));
+
+    // Find an empty slot in the new hash table
+    int j = HashValue(value) & new_mod_bitmask;
+    hash_slot_t slot = new_hash_slots[j];
+
+    while (kHashSlotEmpty != slot && SlotDifferent(slot, value)) {
+      ++j;
+      if (j == new_size) { j = 0; }
+      slot = new_hash_slots[j];
+    }
+
+    // Copy the old slot index to the new hash table
+    new_hash_slots[j] = index;
+  }
+
+  hash_table_ = new_hash_table;
+  hash_slots_ = reinterpret_cast<int32_t*>(hash_table_->mutable_data());
+  hash_table_size_ = new_size;
+  mod_bitmask_ = new_size - 1;
+
+  return Status::OK();
+}
+
+template <typename T, typename Scalar>
+Scalar DictionaryBuilder<T, Scalar>::GetDictionaryValue(int64_t index) {
+  const Scalar* data = reinterpret_cast<const Scalar*>(dict_builder_.data()->data());
+  return data[index];
+}
+
+template <typename T, typename Scalar>
+int DictionaryBuilder<T, Scalar>::HashValue(const Scalar& value) {
+  return HashUtil::Hash(&value, sizeof(Scalar), 0);
+}
+
+template <typename T, typename Scalar>
+bool DictionaryBuilder<T, Scalar>::SlotDifferent(hash_slot_t index, const Scalar& value) {
+  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
+  return other != value;
+}
+
+template <typename T, typename Scalar>
+Status DictionaryBuilder<T, Scalar>::AppendDictionary(const Scalar& value) {
+  return dict_builder_.Append(value);
+}
+
+#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                       \
+  template <>                                                                         \
+  WrappedBinary DictionaryBuilder<Type, WrappedBinary>::GetDictionaryValue(           \
+      int64_t index) {                                                                \
+    int32_t v_len;                                                                    \
+    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len);   \
+    return WrappedBinary(v, v_len);                                                   \
+  }                                                                                   \
+                                                                                      \
+  template <>                                                                         \
+  int DictionaryBuilder<Type, WrappedBinary>::HashValue(const WrappedBinary& value) { \
+    return HashUtil::Hash(value.ptr_, value.length_, 0);                              \
+  }                                                                                   \
+                                                                                      \
+  template <>                                                                         \
+  bool DictionaryBuilder<Type, WrappedBinary>::SlotDifferent(                         \
+      hash_slot_t index, const WrappedBinary& value) {                                \
+    int32_t other_length;                                                             \
+    const uint8_t* other_value =                                                      \
+        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);           \
+    return !(other_length == value.length_ &&                                         \
+             0 == memcmp(other_value, value.ptr_, value.length_));                    \
+  }                                                                                   \
+                                                                                      \
+  template <>                                                                         \
+  Status DictionaryBuilder<Type, WrappedBinary>::AppendDictionary(                    \
+      const WrappedBinary& value) {                                                   \
+    return dict_builder_.Append(value.ptr_, value.length_);                           \
+  }
+
+BINARY_DICTIONARY_SPECIALIZATIONS(StringType);
+BINARY_DICTIONARY_SPECIALIZATIONS(BinaryType);
+
+template class DictionaryBuilder<UInt8Type>;
+template class DictionaryBuilder<UInt16Type>;
+template class DictionaryBuilder<UInt32Type>;
+template class DictionaryBuilder<UInt64Type>;
+template class DictionaryBuilder<Int8Type>;
+template class DictionaryBuilder<Int16Type>;
+template class DictionaryBuilder<Int32Type>;
+template class DictionaryBuilder<Int64Type>;
+template class DictionaryBuilder<Date32Type>;
+template class DictionaryBuilder<Date64Type>;
+template class DictionaryBuilder<Time32Type>;
+template class DictionaryBuilder<Time64Type>;
+template class DictionaryBuilder<TimestampType>;
+template class DictionaryBuilder<FloatType>;
+template class DictionaryBuilder<DoubleType>;
+template class DictionaryBuilder<BinaryType, WrappedBinary>;
+template class DictionaryBuilder<StringType, WrappedBinary>;
+
 // ----------------------------------------------------------------------
 // DecimalBuilder
 DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
@@ -830,6 +1030,17 @@ Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
+const uint8_t* BinaryBuilder::GetValue(int64_t i, int32_t* out_length) const {
+  const int32_t* offsets = reinterpret_cast<const int32_t*>(offset_builder_.data());
+  int32_t offset = offsets[i];
+  if (i == (length_ - 1)) {
+    *out_length = static_cast<int32_t>(value_builder_->length()) - offset;
+  } else {
+    *out_length = offsets[i + 1] - offset;
+  }
+  return byte_builder_->data()->data() + offset;
+}
+
 StringBuilder::StringBuilder(MemoryPool* pool) : BinaryBuilder(pool, utf8()) {}
 
 Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index d77223e788378..75fb6efffcc58 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -569,6 +569,11 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
 
   Status Finish(std::shared_ptr<Array>* out) override;
 
+  /// Temporary access to a value.
+  ///
+  /// This pointer becomes invalid on the next modifying operation.
+  const uint8_t* GetValue(int64_t i, int32_t* out_length) const;
+
  protected:
   UInt8Builder* byte_builder_;
 };
@@ -674,6 +679,86 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
   std::vector<std::shared_ptr<ArrayBuilder>> field_builders_;
 };
 
+// ----------------------------------------------------------------------
+// Dictionary builder
+
+// Based on Apache Parquet-cpp's DictEncoder
+
+// Initially 1024 elements
+static constexpr int kInitialHashTableSize = 1 << 10;
+
+typedef int32_t hash_slot_t;
+static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits<int32_t>::max();
+
+// The maximum load factor for the hash table before resizing.
+static constexpr double kMaxHashTableLoad = 0.7;
+
+template <typename T, typename Scalar = typename T::c_type>
+class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
+ public:
+  explicit DictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+
+  template <typename T1 = T, typename Scalar1 = Scalar>
+  explicit DictionaryBuilder(
+      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool)
+      : DictionaryBuilder<T1, Scalar1>(pool, TypeTraits<T1>::type_singleton()) {}
+
+  Status Append(const Scalar& value);
+
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
+  Status Finish(std::shared_ptr<Array>* out) override;
+
+ protected:
+  Status DoubleTableSize();
+  Scalar GetDictionaryValue(int64_t index);
+  int HashValue(const Scalar& value);
+  bool SlotDifferent(hash_slot_t slot, const Scalar& value);
+  Status AppendDictionary(const Scalar& value);
+
+  std::shared_ptr<PoolBuffer> hash_table_;
+  int32_t* hash_slots_;
+
+  /// Size of the table. Must be a power of 2.
+  int hash_table_size_;
+
+  // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j %
+  // hash_table_size_, but uses far fewer CPU cycles
+  int mod_bitmask_;
+
+  typename TypeTraits<T>::BuilderType dict_builder_;
+  AdaptiveUIntBuilder values_builder_;
+};
+
+// TODO(ARROW-1176): Use Tensorflow's StringPiece instead of this here.
+struct WrappedBinary {
+  WrappedBinary(const uint8_t* ptr, int32_t length) : ptr_(ptr), length_(length) {}
+
+  const uint8_t* ptr_;
+  int32_t length_;
+};
+
+class ARROW_EXPORT StringDictionaryBuilder
+    : public DictionaryBuilder<StringType, WrappedBinary> {
+ public:
+  using DictionaryBuilder::DictionaryBuilder;
+
+  using DictionaryBuilder::Append;
+
+  Status Append(const uint8_t* value, int32_t length) {
+    return Append(WrappedBinary(value, length));
+  }
+
+  Status Append(const char* value, int32_t length) {
+    return Append(WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
+  }
+
+  Status Append(const std::string& value) {
+    return Append(WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
+        static_cast<int32_t>(value.size())));
+  }
+};
+
 // ----------------------------------------------------------------------
 // Helper functions
 
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 74f80428c4561..b8203f0daa1a4 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -87,9 +87,10 @@ class TestHdfsClient : public ::testing::Test {
     LibHdfsShim* driver_shim;
 
     client_ = nullptr;
-    scratch_dir_ = boost::filesystem::unique_path(
-        boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
-                       .string();
+    scratch_dir_ =
+        boost::filesystem::unique_path(
+            boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
+            .string();
 
     loaded_driver_ = false;
 
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
index b385929d8b10a..1aecdbc633190 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -80,7 +80,7 @@ static void BM_WriteRecordBatch(benchmark::State& state) {  // NOLINT non-const
     int32_t metadata_length;
     int64_t body_length;
     if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
-            default_memory_pool())
+             default_memory_pool())
              .ok()) {
       state.SkipWithError("Failed to write!");
     }
@@ -101,7 +101,7 @@ static void BM_ReadRecordBatch(benchmark::State& state) {  // NOLINT non-const r
   int32_t metadata_length;
   int64_t body_length;
   if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
-          default_memory_pool())
+           default_memory_pool())
            .ok()) {
     state.SkipWithError("Failed to write!");
   }
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 97e2bb7072e30..2f334f0ed0206 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -464,8 +464,9 @@ class FixedWidthBytesConverter
   inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
-    Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedSizeBinaryType>(
-        typed_builder_->type())->byte_width();
+    Py_ssize_t expected_length =
+        std::dynamic_pointer_cast<FixedSizeBinaryType>(typed_builder_->type())
+            ->byte_width();
     if (item.obj() == Py_None) {
       RETURN_NOT_OK(typed_builder_->AppendNull());
       return Status::OK();
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 8338800225912..9ae83743e5754 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -407,7 +407,7 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() {return "fixed_size_binary"; }
+  static std::string name() { return "fixed_size_binary"; }
 
   std::vector<BufferDescr> GetBufferLayout() const override;
 
diff --git a/cpp/src/arrow/util/hash-util.h b/cpp/src/arrow/util/hash-util.h
index ffe1a9d01b4d3..4c049c1697b72 100644
--- a/cpp/src/arrow/util/hash-util.h
+++ b/cpp/src/arrow/util/hash-util.h
@@ -41,8 +41,8 @@ class HashUtil {
   /// TODO: update this to also use SSE4_crc32_u64 and SSE4_crc32_u16 where appropriate.
   static uint32_t CrcHash(const void* data, int32_t bytes, uint32_t hash) {
     DCHECK(CpuInfo::IsSupported(CpuInfo::SSE4_2));
-    uint32_t words = bytes / sizeof(uint32_t);
-    bytes = bytes % sizeof(uint32_t);
+    uint32_t words = static_cast<uint32_t>(bytes / sizeof(uint32_t));
+    bytes = static_cast<int32_t>(bytes % sizeof(uint32_t));
 
     const uint32_t* p = reinterpret_cast<const uint32_t*>(data);
     while (words--) {
@@ -201,7 +201,7 @@ class HashUtil {
     DCHECK_GT(bytes, 0);
     uint64_t hash_u64 = hash | ((uint64_t)hash << 32);
     hash_u64 = FnvHash64(data, bytes, hash_u64);
-    return (hash_u64 >> 32) ^ (hash_u64 & 0xFFFFFFFF);
+    return static_cast<uint32_t>((hash_u64 >> 32) ^ (hash_u64 & 0xFFFFFFFF));
   }
 
   /// Computes the hash value for data.  Will call either CrcHash or MurmurHash
@@ -241,7 +241,7 @@ class HashUtil {
     // randomness of the constants) that any subset of bit positions of
     // Rehash32to32(hash1) is equal to the same subset of bit positions
     // Rehash32to32(hash2) is minimal.
-    return (static_cast<uint64_t>(hash) * m + a) >> 32;
+    return static_cast<uint32_t>((static_cast<uint64_t>(hash) * m + a) >> 32);
   }
 
   static inline uint64_t Rehash32to64(const uint32_t hash) {

From 2e5ddfe7dc7c653ebf97b95a62e7da8909fc7ac8 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 3 Jul 2017 12:16:15 -0400
Subject: [PATCH 0769/1644] ARROW-1179: C++: Add missing virtual destructors

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #803 from xhochy/ARROW-1179 and squashes the following commits:

69f283f1 [Uwe L. Korn] Add another destructor
ce71464e [Uwe L. Korn] Add virtual destructor to KVMetadata
---
 cpp/src/arrow/type.h                    | 1 +
 cpp/src/arrow/util/key_value_metadata.h | 1 +
 2 files changed, 2 insertions(+)

diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 9ae83743e5754..70e8644c11746 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -694,6 +694,7 @@ class ARROW_EXPORT Schema {
  public:
   explicit Schema(const std::vector<std::shared_ptr<Field>>& fields,
       const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+  virtual ~Schema() = default;
 
   // Returns true if all of the schema fields are equal
   bool Equals(const Schema& other) const;
diff --git a/cpp/src/arrow/util/key_value_metadata.h b/cpp/src/arrow/util/key_value_metadata.h
index bae4ad806dd62..a2a4623aee7cc 100644
--- a/cpp/src/arrow/util/key_value_metadata.h
+++ b/cpp/src/arrow/util/key_value_metadata.h
@@ -35,6 +35,7 @@ class ARROW_EXPORT KeyValueMetadata {
   KeyValueMetadata(
       const std::vector<std::string>& keys, const std::vector<std::string>& values);
   explicit KeyValueMetadata(const std::unordered_map<std::string, std::string>& map);
+  virtual ~KeyValueMetadata() = default;
 
   void ToUnorderedMap(std::unordered_map<std::string, std::string>* out) const;
 

From 2c3e8b09d168597260e28c331ed6cef7a744f82c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 3 Jul 2017 12:35:01 -0400
Subject: [PATCH 0770/1644] ARROW-692: Integration test data generator for
 dictionary types

cc @BryanCutler -- sorry to take a little while to get to this. I suspect we may have a little bit of slippage in the JSON we are generating, so this will help resolve the discrepancies

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #800 from wesm/ARROW-692 and squashes the following commits:

d04d0a60 [Wes McKinney] Disable dictionary case for now
acd9ea4c [Wes McKinney] Start dictionary test cases from 0, since C++ does not preserve the ids through stream-to-file
0f30d0b0 [Wes McKinney] Comment out generate_dictionary_case to get passing CI build
63a9ad5b [Wes McKinney] Add hack to be able to generate a column with a different name
548452d9 [Wes McKinney] Do not write dictionaries if length-0
4779db0c [Wes McKinney] Add dictionary JSON test data generation, passes C++ tests
---
 cpp/src/arrow/ipc/metadata.cc   |   2 +-
 integration/integration_test.py | 177 +++++++++++++++++++++++++-------
 2 files changed, 142 insertions(+), 37 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 706ab2e8aab0e..54f0547b5991b 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -798,7 +798,7 @@ int64_t DictionaryMemo::GetId(const std::shared_ptr<Array>& dictionary) {
     // Dictionary already observed, return the id
     return it->second;
   } else {
-    int64_t new_id = static_cast<int64_t>(dictionary_to_id_.size()) + 1;
+    int64_t new_id = static_cast<int64_t>(dictionary_to_id_.size());
     dictionary_to_id_[address] = new_id;
     id_to_dictionary_[new_id] = dictionary;
     return new_id;
diff --git a/integration/integration_test.py b/integration/integration_test.py
index cc59593dbdc1b..9532d4e7ec7e1 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -119,6 +119,9 @@ def __init__(self, name, count):
         self.name = name
         self.count = count
 
+    def __len__(self):
+        return self.count
+
     def _get_children(self):
         return []
 
@@ -195,15 +198,21 @@ def _get_type(self):
             ('bitWidth', self.bit_width)
         ])
 
-    def generate_column(self, size):
+    def generate_column(self, size, name=None):
         iinfo = np.iinfo(self.numpy_type)
+        lower_bound = max(iinfo.min, self.min_value)
+        upper_bound = min(iinfo.max, self.max_value)
+        return self.generate_range(size, lower_bound, upper_bound, name=name)
+
+    def generate_range(self, size, lower, upper, name=None):
         values = [int(x) for x in
-                  np.random.randint(max(iinfo.min, self.min_value),
-                                    min(iinfo.max, self.max_value),
-                                    size=size)]
+                  np.random.randint(lower, upper, size=size)]
 
         is_valid = self._make_is_valid(size)
-        return PrimitiveColumn(self.name, size, is_valid, values)
+
+        if name is None:
+            name = self.name
+        return PrimitiveColumn(name, size, is_valid, values)
 
 
 class DateType(IntegerType):
@@ -294,12 +303,14 @@ def _get_type(self):
             ('precision', self.precision)
         ])
 
-    def generate_column(self, size):
+    def generate_column(self, size, name=None):
         values = np.random.randn(size) * 1000
         values = np.round(values, 3)
 
         is_valid = self._make_is_valid(size)
-        return PrimitiveColumn(self.name, size, is_valid, values)
+        if name is None:
+            name = self.name
+        return PrimitiveColumn(name, size, is_valid, values)
 
 
 class BooleanType(PrimitiveType):
@@ -313,10 +324,12 @@ def _get_type(self):
     def numpy_type(self):
         return 'bool'
 
-    def generate_column(self, size):
+    def generate_column(self, size, name=None):
         values = list(map(bool, np.random.randint(0, 2, size=size)))
         is_valid = self._make_is_valid(size)
-        return PrimitiveColumn(self.name, size, is_valid, values)
+        if name is None:
+            name = self.name
+        return PrimitiveColumn(name, size, is_valid, values)
 
 
 class BinaryType(PrimitiveType):
@@ -342,7 +355,7 @@ def _get_type_layout(self):
               OrderedDict([('type', 'DATA'),
                            ('typeBitWidth', 8)])])])
 
-    def generate_column(self, size):
+    def generate_column(self, size, name=None):
         K = 7
         is_valid = self._make_is_valid(size)
         values = []
@@ -356,7 +369,9 @@ def generate_column(self, size):
             else:
                 values.append("")
 
-        return self.column_class(self.name, size, is_valid, values)
+        if name is None:
+            name = self.name
+        return self.column_class(name, size, is_valid, values)
 
 
 class StringType(BinaryType):
@@ -368,7 +383,7 @@ def column_class(self):
     def _get_type(self):
         return OrderedDict([('name', 'utf8')])
 
-    def generate_column(self, size):
+    def generate_column(self, size, name=None):
         K = 7
         is_valid = self._make_is_valid(size)
         values = []
@@ -379,10 +394,12 @@ def generate_column(self, size):
             else:
                 values.append("")
 
-        return self.column_class(self.name, size, is_valid, values)
+        if name is None:
+            name = self.name
+        return self.column_class(name, size, is_valid, values)
 
 
-class JSONSchema(object):
+class JsonSchema(object):
 
     def __init__(self, fields):
         self.fields = fields
@@ -447,7 +464,7 @@ def _get_type_layout(self):
               OrderedDict([('type', 'OFFSET'),
                            ('typeBitWidth', 32)])])])
 
-    def generate_column(self, size):
+    def generate_column(self, size, name=None):
         MAX_LIST_SIZE = 4
 
         is_valid = self._make_is_valid(size)
@@ -463,7 +480,9 @@ def generate_column(self, size):
         # The offset now is the total number of elements in the child array
         values = self.value_type.generate_column(offset)
 
-        return ListColumn(self.name, size, is_valid, offsets, values)
+        if name is None:
+            name = self.name
+        return ListColumn(name, size, is_valid, offsets, values)
 
 
 class ListColumn(Column):
@@ -504,13 +523,66 @@ def _get_type_layout(self):
              [OrderedDict([('type', 'VALIDITY'),
                            ('typeBitWidth', 1)])])])
 
-    def generate_column(self, size):
+    def generate_column(self, size, name=None):
         is_valid = self._make_is_valid(size)
 
         field_values = [type_.generate_column(size)
                         for type_ in self.field_types]
+        if name is None:
+            name = self.name
+        return StructColumn(name, size, is_valid, field_values)
+
+
+class Dictionary(object):
+
+    def __init__(self, id_, field, values, ordered=False):
+        self.id_ = id_
+        self.field = field
+        self.values = values
+        self.ordered = ordered
+
+    def __len__(self):
+        return len(self.values)
+
+    def get_json(self):
+        dummy_batch = JsonRecordBatch(len(self.values), [self.values])
+        return OrderedDict([
+            ('id', self.id_),
+            ('data', dummy_batch.get_json())
+        ])
+
 
-        return StructColumn(self.name, size, is_valid, field_values)
+class DictionaryType(DataType):
+
+    def __init__(self, name, index_type, dictionary, nullable=True):
+        DataType.__init__(self, name, nullable=nullable)
+        assert isinstance(index_type, IntegerType)
+        assert isinstance(dictionary, Dictionary)
+
+        self.index_type = index_type
+        self.dictionary = dictionary
+
+    def get_json(self):
+        dict_field = self.dictionary.field
+        return OrderedDict([
+            ('name', self.name),
+            ('type', dict_field._get_type()),
+            ('nullable', self.nullable),
+            ('children', dict_field._get_children()),
+            ('dictionary', OrderedDict([
+                ('id', self.dictionary.id_),
+                ('indexType', self.index_type._get_type()),
+                ('isOrdered', self.dictionary.ordered)
+            ])),
+            ('typeLayout', self.index_type._get_type_layout())
+        ])
+
+    def _get_type_layout(self):
+        return self.index_type._get_type_layout()
+
+    def generate_column(self, size, name=None):
+        return self.index_type.generate_range(size, 0, len(self.dictionary),
+                                              name=name)
 
 
 class StructColumn(Column):
@@ -529,7 +601,7 @@ def _get_children(self):
         return [field.get_json() for field in self.field_values]
 
 
-class JSONRecordBatch(object):
+class JsonRecordBatch(object):
 
     def __init__(self, count, columns):
         self.count = count
@@ -542,18 +614,27 @@ def get_json(self):
         ])
 
 
-class JSONFile(object):
+class JsonFile(object):
 
-    def __init__(self, name, schema, batches):
+    def __init__(self, name, schema, batches, dictionaries=None):
         self.name = name
         self.schema = schema
+        self.dictionaries = dictionaries or []
         self.batches = batches
 
     def get_json(self):
-        return OrderedDict([
-            ('schema', self.schema.get_json()),
-            ('batches', [batch.get_json() for batch in self.batches])
-        ])
+        entries = [
+            ('schema', self.schema.get_json())
+        ]
+
+        if len(self.dictionaries) > 0:
+            entries.append(('dictionaries',
+                            [dictionary.get_json()
+                             for dictionary in self.dictionaries]))
+
+        entries.append(('batches', [batch.get_json()
+                                    for batch in self.batches]))
+        return OrderedDict(entries)
 
     def write(self, path):
         with open(path, 'wb') as f:
@@ -580,8 +661,8 @@ def get_field(name, type_, nullable=True):
         raise TypeError(dtype)
 
 
-def _generate_file(name, fields, batch_sizes):
-    schema = JSONSchema(fields)
+def _generate_file(name, fields, batch_sizes, dictionaries=None):
+    schema = JsonSchema(fields)
     batches = []
     for size in batch_sizes:
         columns = []
@@ -589,9 +670,9 @@ def _generate_file(name, fields, batch_sizes):
             col = field.generate_column(size)
             columns.append(col)
 
-        batches.append(JSONRecordBatch(size, columns))
+        batches.append(JsonRecordBatch(size, columns))
 
-    return JSONFile(name, schema, batches)
+    return JsonFile(name, schema, batches, dictionaries)
 
 
 def generate_primitive_case(batch_sizes):
@@ -645,6 +726,25 @@ def generate_nested_case():
     return _generate_file("nested", fields, batch_sizes)
 
 
+def generate_dictionary_case():
+    dict_type1 = StringType('dictionary1')
+    dict_type2 = get_field('dictionary2', 'int64')
+
+    dict1 = Dictionary(0, dict_type1,
+                       dict_type1.generate_column(10, name='DICT0'))
+    dict2 = Dictionary(1, dict_type2,
+                       dict_type2.generate_column(50, name='DICT1'))
+
+    fields = [
+        DictionaryType('dict1_0', get_field('', 'int8'), dict1),
+        DictionaryType('dict1_1', get_field('', 'int32'), dict1),
+        DictionaryType('dict2_0', get_field('', 'int16'), dict2)
+    ]
+    batch_sizes = [7, 10]
+    return _generate_file("dictionary", fields, batch_sizes,
+                          dictionaries=[dict1, dict2])
+
+
 def get_generated_json_files():
     temp_dir = tempfile.mkdtemp()
 
@@ -655,12 +755,14 @@ def _temp_path():
         generate_primitive_case([7, 10]),
         generate_primitive_case([0, 0, 0]),
         generate_datetime_case(),
-        generate_nested_case()
+        generate_nested_case(),
+        # generate_dictionary_case()
     ]
 
     generated_paths = []
     for file_obj in file_objs:
-        out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
+        out_path = os.path.join(temp_dir, 'generated_' +
+                                file_obj.name + '.json')
         file_obj.write(out_path)
         generated_paths.append(out_path)
 
@@ -689,15 +791,16 @@ def _compare_implementations(self, producer, consumer):
                                                        consumer.name))
 
         for json_path in self.json_files:
-            print('=====================================================================================')
+            print('==========================================================')
             print('Testing file {0}'.format(json_path))
-            print('=====================================================================================')
+            print('==========================================================')
 
             name = os.path.splitext(os.path.basename(json_path))[0]
 
             # Make the random access file
             print('-- Creating binary inputs')
-            producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
+            producer_file_path = os.path.join(self.temp_dir, guid() + '_' +
+                                              name + '.json_to_arrow')
             producer.json_to_file(json_path, producer_file_path)
 
             # Validate the file
@@ -705,8 +808,10 @@ def _compare_implementations(self, producer, consumer):
             consumer.validate(json_path, producer_file_path)
 
             print('-- Validating stream')
-            producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
-            consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
+            producer_stream_path = os.path.join(self.temp_dir, guid() + '_' +
+                                                name + '.arrow_to_stream')
+            consumer_file_path = os.path.join(self.temp_dir, guid() + '_' +
+                                              name + '.stream_to_arrow')
             producer.file_to_stream(producer_file_path,
                                     producer_stream_path)
             consumer.stream_to_file(producer_stream_path,

From a6d0c2654742b1cec4aa186ec9a430e28fd3d4af Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 3 Jul 2017 13:44:28 -0400
Subject: [PATCH 0771/1644] ARROW-1180: [GLib] Fix a returning invalid address
 bug in garrow_tensor_get_dimension_name()

See also: https://github.com/apache/arrow/pull/783#pullrequestreview-46377983

Author: Kouhei Sutou <kou@clear-code.com>

Closes #804 from kou/glib-fix-tensor-dimension-name and squashes the following commits:

5740be3f [Kouhei Sutou] [GLib] Fix a returning invalid address bug in garrow_tensor_get_dimension_name()
---
 c_glib/arrow-glib/tensor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp
index 89e971c726ecb..3325f85113375 100644
--- a/c_glib/arrow-glib/tensor.cpp
+++ b/c_glib/arrow-glib/tensor.cpp
@@ -314,8 +314,8 @@ const gchar *
 garrow_tensor_get_dimension_name(GArrowTensor *tensor, gint i)
 {
   auto arrow_tensor = garrow_tensor_get_raw(tensor);
-  auto arrow_dimension_name = arrow_tensor->dim_name(i);
-  return arrow_dimension_name.c_str();
+  auto arrow_dimension_name = &(arrow_tensor->dim_name(i));
+  return arrow_dimension_name->c_str();
 }
 
 /**

From e18abac1919fc223f995e85e012480221054df45 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 3 Jul 2017 22:09:10 +0200
Subject: [PATCH 0772/1644] ARROW-1181: [Python] Parquet multiindex test should
 be optional

Test `test_multiindex_duplicate_values()` fails if parquet not installed.  Added annotation which makes the test optional.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #806 from BryanCutler/pyarrow-test-parquet-make-optional and squashes the following commits:

55ea49f [Bryan Cutler] parquet multiindex test should be optional
---
 python/pyarrow/tests/test_parquet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 4e8f0db046c62..94a0e38cfb090 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -848,7 +848,7 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
     with pytest.raises(ValueError):
         read_multiple_files(mixed_paths)
 
-
+@parquet
 def test_multiindex_duplicate_values(tmpdir):
     num_rows = 3
     numbers = list(range(num_rows))

From cdee23c27ac36f957512e33cc1ee49674c515dc8 Mon Sep 17 00:00:00 2001
From: Max Risukhin <risuhin.max@gmail.com>
Date: Mon, 3 Jul 2017 22:39:09 +0200
Subject: [PATCH 0773/1644] ARROW-600: ZSTD compression lib support

Author: Max Risukhin <risuhin.max@gmail.com>

Closes #807 from MaxRis/ARROW-600 and squashes the following commits:

2fc4578 [Max Risukhin] ARROW-600: ZSTD compression lib support
---
 cpp/CMakeLists.txt                          |  3 +-
 cpp/build-support/build-zstd-lib.sh         | 16 +++++
 cpp/cmake_modules/FindZSTD.cmake            | 70 +++++++++++++++++++++
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 49 ++++++++++++++-
 cpp/src/arrow/util/compression-test.cc      |  4 ++
 cpp/src/arrow/util/compression.cc           | 28 +++++++++
 cpp/src/arrow/util/compression.h            | 16 ++++-
 python/manylinux1/Dockerfile-x86_64_base    |  8 +++
 python/manylinux1/scripts/build_lz4.sh      | 24 +++++++
 python/manylinux1/scripts/build_zstd.sh     | 25 ++++++++
 10 files changed, 239 insertions(+), 4 deletions(-)
 create mode 100755 cpp/build-support/build-zstd-lib.sh
 create mode 100644 cpp/cmake_modules/FindZSTD.cmake
 create mode 100755 python/manylinux1/scripts/build_lz4.sh
 create mode 100755 python/manylinux1/scripts/build_zstd.sh

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 57453382df26c..28a3bb0266271 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -530,7 +530,8 @@ set(ARROW_STATIC_LINK_LIBS
   brotli_enc
   brotli_common
   snappy
-  zlib)
+  zlib
+  zstd_static)
 
 add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
 
diff --git a/cpp/build-support/build-zstd-lib.sh b/cpp/build-support/build-zstd-lib.sh
new file mode 100755
index 0000000000000..62805bae286a2
--- /dev/null
+++ b/cpp/build-support/build-zstd-lib.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+export CFLAGS="${CFLAGS} -O3 -fPIC"
+make -j4
\ No newline at end of file
diff --git a/cpp/cmake_modules/FindZSTD.cmake b/cpp/cmake_modules/FindZSTD.cmake
new file mode 100644
index 0000000000000..1fda29e458c31
--- /dev/null
+++ b/cpp/cmake_modules/FindZSTD.cmake
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find ZSTD (zstd.h, libzstd.a, libzstd.so, and libzstd.so.0)
+# This module defines
+#  ZSTD_INCLUDE_DIR, directory containing headers
+#  ZSTD_SHARED_LIB, path to libzstd shared library
+#  ZSTD_STATIC_LIB, path to libzstd static library
+#  ZSTD_FOUND, whether zstd has been found
+
+if( NOT "${ZSTD_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${ZSTD_HOME}" _native_path )
+    list( APPEND _zstd_roots ${_native_path} )
+elseif ( ZStd_HOME )
+    list( APPEND _zstd_roots ${ZStd_HOME} )
+endif()
+
+if (MSVC AND NOT ZSTD_MSVC_STATIC_LIB_SUFFIX)
+  set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static")
+endif()
+
+set(ZSTD_STATIC_LIB_SUFFIX
+  "${ZSTD_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+set(ZSTD_STATIC_LIB_NAME
+  ${CMAKE_STATIC_LIBRARY_PREFIX}zstd${ZSTD_STATIC_LIB_SUFFIX})
+
+if ( _zstd_roots )
+  find_path(ZSTD_INCLUDE_DIR NAMES zstd.h
+    PATHS ${_zstd_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "include" )
+  find_library(ZSTD_SHARED_LIB NAMES zstd
+    PATHS ${_zstd_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+  find_library(ZSTD_STATIC_LIB NAMES ${ZSTD_STATIC_LIB_NAME}
+    PATHS ${_zstd_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+else()
+  find_path(ZSTD_INCLUDE_DIR zstd.h
+    # make sure we don't accidentally pick up a different version
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(ZSTD_SHARED_LIB zstd
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(ZSTD_STATIC_LIB ${ZSTD_STATIC_LIB_NAME}
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ZSTD REQUIRED_VARS
+  ZSTD_SHARED_LIB ZSTD_STATIC_LIB ZSTD_INCLUDE_DIR)
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 857334579819c..33447ae4a7f56 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -28,6 +28,7 @@ set(JEMALLOC_VERSION "4.4.0")
 set(SNAPPY_VERSION "1.1.3")
 set(BROTLI_VERSION "v0.6.0")
 set(LZ4_VERSION "1.7.5")
+set(ZSTD_VERSION "1.2.0")
 
 string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
 
@@ -49,6 +50,7 @@ if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
   set(ZLIB_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
   set(BROTLI_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
   set(LZ4_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+  set(ZSTD_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
 
   if (NOT DEFINED ENV{BOOST_ROOT})
     # Since we have to set this in the environment, we check whether
@@ -89,6 +91,10 @@ if (DEFINED ENV{LZ4_HOME})
   set(LZ4_HOME "$ENV{LZ4_HOME}")
 endif()
 
+if (DEFINED ENV{ZSTD_HOME})
+  set(ZSTD_HOME "$ENV{ZSTD_HOME}")
+endif()
+
 # ----------------------------------------------------------------------
 # Find pthreads
 
@@ -653,8 +659,8 @@ if (NOT LZ4_FOUND)
   set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
 
   if (MSVC)
-    set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_Release/liblz4_static.lib")
-    set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=Release /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
+    set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
+    set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
   else()
     set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
     set(LZ4_BUILD_COMMAND BUILD_COMMAND make -j4)
@@ -682,3 +688,42 @@ ADD_THIRDPARTY_LIB(lz4_static
 if (LZ4_VENDORED)
   add_dependencies(lz4_static lz4_ep)
 endif()
+
+# ----------------------------------------------------------------------
+# ZSTD
+
+find_package(ZSTD)
+if (NOT ZSTD_FOUND)
+  set(ZSTD_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix/src/zstd_ep")
+  set(ZSTD_INCLUDE_DIR "${ZSTD_BUILD_DIR}/lib")
+
+  if (MSVC)
+    set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib")
+    set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
+  else()
+    set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a")
+    set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh)
+  endif()
+
+  ExternalProject_Add(zstd_ep
+      URL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz"
+      UPDATE_COMMAND ""
+      PATCH_COMMAND ""
+      CONFIGURE_COMMAND ""
+      INSTALL_COMMAND ""
+      BINARY_DIR ${ZSTD_BUILD_DIR}
+      ${ZSTD_BUILD_COMMAND}
+      )
+
+  set(ZSTD_VENDORED 1)
+else()
+  set(ZSTD_VENDORED 0)
+endif()
+
+include_directories(SYSTEM ${ZSTD_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(zstd_static
+  STATIC_LIB ${ZSTD_STATIC_LIB})
+
+if (ZSTD_VENDORED)
+  add_dependencies(zstd_static zstd_ep)
+endif()
diff --git a/cpp/src/arrow/util/compression-test.cc b/cpp/src/arrow/util/compression-test.cc
index 1a0e5d7023cf1..3b19a6d89a209 100644
--- a/cpp/src/arrow/util/compression-test.cc
+++ b/cpp/src/arrow/util/compression-test.cc
@@ -86,4 +86,8 @@ TEST(TestCompressors, GZip) {
   CheckCodec<GZipCodec>();
 }
 
+TEST(TestCompressors, ZSTD) {
+  CheckCodec<ZSTDCodec>();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression.cc b/cpp/src/arrow/util/compression.cc
index 070f857e69643..df1afa317ceae 100644
--- a/cpp/src/arrow/util/compression.cc
+++ b/cpp/src/arrow/util/compression.cc
@@ -31,6 +31,7 @@
 #include <brotli/encode.h>
 #include <snappy.h>
 #include <zlib.h>
+#include <zstd.h>
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
@@ -329,4 +330,31 @@ Status BrotliCodec::Compress(int64_t input_len, const uint8_t* input,
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// ZSTD implementation
+
+Status ZSTDCodec::Decompress(
+    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+  int64_t decompressed_size = ZSTD_decompress(output_buffer,
+      static_cast<size_t>(output_len), input, static_cast<size_t>(input_len));
+  if (decompressed_size != output_len) {
+    return Status::IOError("Corrupt ZSTD compressed data.");
+  }
+  return Status::OK();
+}
+
+int64_t ZSTDCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+  return ZSTD_compressBound(input_len);
+}
+
+Status ZSTDCodec::Compress(int64_t input_len, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+  *output_length = ZSTD_compress(output_buffer, static_cast<size_t>(output_buffer_len),
+      input, static_cast<size_t>(input_len), 1);
+  if (ZSTD_isError(*output_length)) {
+    return Status::IOError("ZSTD compression failure.");
+  }
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h
index 6886d04c7620d..9e581d8029f96 100644
--- a/cpp/src/arrow/util/compression.h
+++ b/cpp/src/arrow/util/compression.h
@@ -27,7 +27,7 @@
 namespace arrow {
 
 struct Compression {
-  enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO, BROTLI };
+  enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO, BROTLI, ZSTD };
 };
 
 class ARROW_EXPORT Codec {
@@ -104,6 +104,20 @@ class ARROW_EXPORT GZipCodec : public Codec {
   std::unique_ptr<GZipCodecImpl> impl_;
 };
 
+// ZSTD codec.
+class ARROW_EXPORT ZSTDCodec : public Codec {
+ public:
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override { return "zstd"; }
+};
+
 }  // namespace arrow
 
 #endif
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index 44a988867f03d..cdd13e2e93304 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -49,6 +49,14 @@ ADD scripts/build_snappy.sh /
 RUN /build_snappy.sh
 ENV SNAPPY_HOME /usr
 
+ADD scripts/build_lz4.sh /
+RUN /build_lz4.sh
+ENV LZ4_HOME /usr
+
+ADD scripts/build_zstd.sh /
+RUN /build_zstd.sh
+ENV ZSTD_HOME /usr
+
 ADD scripts/build_ccache.sh /
 RUN /build_ccache.sh
 
diff --git a/python/manylinux1/scripts/build_lz4.sh b/python/manylinux1/scripts/build_lz4.sh
new file mode 100755
index 0000000000000..5a25d3de42515
--- /dev/null
+++ b/python/manylinux1/scripts/build_lz4.sh
@@ -0,0 +1,24 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+export LZ4_VERSION="1.7.5"
+export PREFIX="/usr"
+export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib -L${PREFIX}/lib"
+wget "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -O lz4-${LZ4_VERSION}.tar.gz
+tar xf lz4-${LZ4_VERSION}.tar.gz
+pushd lz4-${LZ4_VERSION}
+
+make -j5 PREFIX=${PREFIX}
+make install PREFIX=$PREFIX
+popd
+rm -rf lz4-${LZ4_VERSION}.tar.gz lz4-${LZ4_VERSION}
diff --git a/python/manylinux1/scripts/build_zstd.sh b/python/manylinux1/scripts/build_zstd.sh
new file mode 100755
index 0000000000000..268e5c8894ccf
--- /dev/null
+++ b/python/manylinux1/scripts/build_zstd.sh
@@ -0,0 +1,25 @@
+#!/bin/bash -ex
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+export ZSTD_VERSION="1.2.0"
+export CFLAGS="${CFLAGS} -O3 -fPIC"
+export PREFIX="/usr"
+export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib"
+wget "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -O zstd-${ZSTD_VERSION}.tar.gz
+tar xf zstd-${ZSTD_VERSION}.tar.gz
+pushd zstd-${ZSTD_VERSION}
+
+make -j5
+make install PREFIX=$PREFIX
+popd
+rm -rf zstd-${ZSTD_VERSION}.tar.gz zstd-${ZSTD_VERSION}

From 681479dc77a6099391b1190e7c875a905b63e138 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 3 Jul 2017 18:05:13 -0400
Subject: [PATCH 0774/1644] ARROW-1182: C++: Specify BUILD_BYPRODUCTS for zlib
 and zstd

Change-Id: I7f4ce23ead7ab1b27741b80a382c0cd4d0019895

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #808 from xhochy/ARROW-1182 and squashes the following commits:

8dfe7b62 [Uwe L. Korn] ARROW-1182: C++: Specify BUILD_BYPRODUCTS for zlib and zstd
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 33447ae4a7f56..3e09006d7c680 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -673,6 +673,7 @@ if (NOT LZ4_FOUND)
       CONFIGURE_COMMAND ""
       INSTALL_COMMAND ""
       BINARY_DIR ${LZ4_BUILD_DIR}
+      BUILD_BYPRODUCTS ${LZ4_STATIC_LIB}
       ${LZ4_BUILD_COMMAND}
       )
 
@@ -712,6 +713,7 @@ if (NOT ZSTD_FOUND)
       CONFIGURE_COMMAND ""
       INSTALL_COMMAND ""
       BINARY_DIR ${ZSTD_BUILD_DIR}
+      BUILD_BYPRODUCTS ${ZSTD_STATIC_LIB}
       ${ZSTD_BUILD_COMMAND}
       )
 

From e5a08dd8f707663a725ac8d002a9484e7b992847 Mon Sep 17 00:00:00 2001
From: LynnYuan <yuanxiaolong@inspur.com>
Date: Tue, 4 Jul 2017 11:26:34 -0400
Subject: [PATCH 0775/1644] ARROW-1098. [Format] modify document mistake

Author: LynnYuan <yuanxiaolong@inspur.com>

Closes #811 from LynnYuanInspur/lynn-doc and squashes the following commits:

b6ec6c05 [LynnYuan] ARROW-1098. [Format] modify document mistake
---
 format/Layout.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index 1b85abf942bf6..1e817ff1375ac 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -552,7 +552,7 @@ will have the following layout:
 
       |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  | Bytes 24-63           |
       |------------|-------------|-------------|-------------|-------------|--------------|-----------------------|
-      | 1          | unspecified | unspecified | unspecified | 4           |  unspecified | unspecified (padding) |
+      | 5          | unspecified | unspecified | unspecified | 4           |  unspecified | unspecified (padding) |
 
   * u1 (float):
     * Length: 6, Null count: 4
@@ -605,7 +605,7 @@ the the types array indicates that a slot contains a different type at the index
 ## Dictionary encoding
 
 When a field is dictionary encoded, the values are represented by an array of Int32 representing the index of the value in the dictionary.
-The Dictionary is received as a DictionaryBacth whose id is referenced by a dictionary attribute defined in the metadata ([Message.fbs][7]) in the Field table.
+The Dictionary is received as a DictionaryBatch whose id is referenced by a dictionary attribute defined in the metadata ([Message.fbs][7]) in the Field table.
 The dictionary has the same layout as the type of the field would dictate. Each entry in the dictionary can be accessed by its index in the DictionaryBatch.
 When a Schema references a Dictionary id, it must send a DictionaryBatch for this id before any RecordBatch.
 

From 7c18ddd4e6bc4b29cd4dd23786760f5c00137bf9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 4 Jul 2017 11:32:20 -0400
Subject: [PATCH 0776/1644] ARROW-966: [Python] Also accept Field instance in
 pyarrow.list_

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #809 from wesm/ARROW-966 and squashes the following commits:

a20f54dd [Wes McKinney] Also accept Field instance in pyarrow.list_
---
 python/pyarrow/array.pxi             | 60 ++++++++++++++++++++++++----
 python/pyarrow/includes/libarrow.pxd |  1 +
 python/pyarrow/tests/test_schema.py  |  4 ++
 3 files changed, 57 insertions(+), 8 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 79e88fc954549..6a4925603d67b 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -622,17 +622,46 @@ def binary(int length=-1):
     return pyarrow_wrap_data_type(fixed_size_binary_type)
 
 
-def list_(DataType value_type):
-    cdef DataType out = DataType()
+def list_(value_type):
+    """
+    Create ListType instance from child data type or field
+
+    Parameters
+    ----------
+    value_type : DataType or Field
+
+    Returns
+    -------
+    list_type : DataType
+    """
+    cdef:
+        DataType data_type
+        Field field
+
     cdef shared_ptr[CDataType] list_type
-    list_type.reset(new CListType(value_type.sp_type))
-    out.init(list_type)
-    return out
+
+    if isinstance(value_type, DataType):
+        list_type.reset(new CListType((<DataType> value_type).sp_type))
+    elif isinstance(value_type, Field):
+        list_type.reset(new CListType((<Field> value_type).sp_field))
+    else:
+        raise ValueError('List requires DataType or Field')
+
+    return pyarrow_wrap_data_type(list_type)
 
 
 def dictionary(DataType index_type, Array dictionary):
     """
     Dictionary (categorical, or simply encoded) type
+
+    Parameters
+    ----------
+    index_type : DataType
+    dictionary : Array
+
+    Returns
+    -------
+    type : DictionaryType
     """
     cdef DictionaryType out = DictionaryType()
     cdef shared_ptr[CDataType] dict_type
@@ -644,10 +673,26 @@ def dictionary(DataType index_type, Array dictionary):
 
 def struct(fields):
     """
+    Create StructType instance from fields
 
+    Parameters
+    ----------
+    fields : sequence of Field values
+
+    Examples
+    --------
+    import pyarrow as pa
+    fields = [
+        pa.field('f1', pa.int32()),
+        pa.field('f2', pa.string())
+    ]
+    struct_type = pa.struct(fields)
+
+    Returns
+    -------
+    type : DataType
     """
     cdef:
-        DataType out = DataType()
         Field field
         vector[shared_ptr[CField]] c_fields
         cdef shared_ptr[CDataType] struct_type
@@ -656,8 +701,7 @@ def struct(fields):
         c_fields.push_back(field.sp_field)
 
     struct_type.reset(new CStructType(c_fields))
-    out.init(struct_type)
-    return out
+    return pyarrow_wrap_data_type(struct_type)
 
 
 def schema(fields):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index fb101fe1c7dd6..902f98e34d67f 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -160,6 +160,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CListType" arrow::ListType"(CDataType):
         CListType(const shared_ptr[CDataType]& value_type)
+        CListType(const shared_ptr[CField]& field)
 
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index a6fe1a5df0e2c..f920e8d703bad 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -64,6 +64,10 @@ def test_type_list():
     list_type = pa.list_(value_type)
     assert str(list_type) == 'list<item: int32>'
 
+    field = pa.field('my_item', pa.string())
+    l2 = pa.list_(field)
+    assert str(l2) == 'list<my_item: string>'
+
 
 def test_type_string():
     t = pa.string()

From edcded3c6996a812188259c2cd4f8b75778defe4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 4 Jul 2017 11:35:50 -0400
Subject: [PATCH 0777/1644] ARROW-1148: [C++] Raise minimum CMake version to
 3.2

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #805 from wesm/ARROW-1148 and squashes the following commits:

db36f510 [Wes McKinney] Restore logic stripping -std=c++11 from cflags
b30f70ca [Wes McKinney] Fix cmake syntax errors
4a202399 [Wes McKinney] Raise minimum CMake version to 3.2
---
 cpp/CMakeLists.txt                          | 16 ++--
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 92 ++++++---------------
 2 files changed, 35 insertions(+), 73 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 28a3bb0266271..76d89ce010514 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-cmake_minimum_required(VERSION 2.7)
+cmake_minimum_required(VERSION 3.2)
 project(arrow)
 
 file(READ "${CMAKE_CURRENT_SOURCE_DIR}/../java/pom.xml" POM_XML)
@@ -29,12 +29,13 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 include(CMakeParseArguments)
 include(ExternalProject)
 
-if(CMAKE_MAJOR_VERSION LESS 3)
-  set(CMAKE_INSTALL_INCLUDEDIR "include")
-  set(CMAKE_INSTALL_LIBDIR "lib")
-else()
-  include(GNUInstallDirs)
-endif()
+# This ensures that things like gnu++11 get passed correctly
+set(CMAKE_CXX_STANDARD 11)
+
+# We require a C++11 compliant compiler
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+include(GNUInstallDirs)
 
 # Compatibility with CMake 3.1
 if(POLICY CMP0054)
@@ -214,6 +215,7 @@ include(san-config)
 
 # For any C code, use the same flags.
 set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
+
 # Remove --std=c++11 to avoid errors from C compilers
 string(REPLACE "-std=c++11" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
 
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 3e09006d7c680..2be7b5a1ea656 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -202,17 +202,10 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
                          -Dgtest_force_shared_crt=ON
                          -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
 
-    if (CMAKE_VERSION VERSION_GREATER "3.2")
-      # BUILD_BYPRODUCTS is a 3.2+ feature
-      ExternalProject_Add(googletest_ep
-        URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-        BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB}
-        CMAKE_ARGS ${GTEST_CMAKE_ARGS})
-    else()
-      ExternalProject_Add(googletest_ep
-        URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-        CMAKE_ARGS ${GTEST_CMAKE_ARGS})
-    endif()
+    ExternalProject_Add(googletest_ep
+      URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+      BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB}
+      CMAKE_ARGS ${GTEST_CMAKE_ARGS})
   else()
     find_package(GTest REQUIRED)
     set(GTEST_VENDORED 0)
@@ -254,19 +247,12 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
                           -BUILD_CONFIG_TESTS=OFF
                           -DINSTALL_HEADERS=ON
                           -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
-    if (CMAKE_VERSION VERSION_GREATER "3.2")
-      # BUILD_BYPRODUCTS is a 3.2+ feature
-      ExternalProject_Add(gflags_ep
-        URL ${GFLAGS_URL}
-        BUILD_IN_SOURCE 1
-        BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}"
-        CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
-    else()
-      ExternalProject_Add(gflags_ep
-        URL ${GFLAGS_URL}
-        BUILD_IN_SOURCE 1
-        CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
-    endif()
+
+    ExternalProject_Add(gflags_ep
+      URL ${GFLAGS_URL}
+      BUILD_IN_SOURCE 1
+      BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}"
+      CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
   else()
     set(GFLAGS_VENDORED 0)
     find_package(GFlags REQUIRED)
@@ -310,17 +296,11 @@ if(ARROW_BUILD_BENCHMARKS)
     if (APPLE)
       set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON")
     endif()
-    if (CMAKE_VERSION VERSION_GREATER "3.2")
-      # BUILD_BYPRODUCTS is a 3.2+ feature
-      ExternalProject_Add(gbenchmark_ep
-        URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
-        BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}"
-        CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
-    else()
-      ExternalProject_Add(gbenchmark_ep
-        URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
-        CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
-    endif()
+
+    ExternalProject_Add(gbenchmark_ep
+      URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
+      BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}"
+      CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
   else()
     find_package(GBenchmark REQUIRED)
     set(GBENCHMARK_VENDORED 0)
@@ -407,23 +387,13 @@ if (ARROW_JEMALLOC)
     set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
     set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(JEMALLOC_VENDORED 1)
-    if (CMAKE_VERSION VERSION_GREATER "3.2")
-      # BUILD_BYPRODUCTS is a 3.2+ feature
-      ExternalProject_Add(jemalloc_ep
-        URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
-        CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
-        BUILD_IN_SOURCE 1
-        BUILD_COMMAND ${MAKE}
-        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
-        INSTALL_COMMAND ${MAKE} -j1 install)
-    else()
-      ExternalProject_Add(jemalloc_ep
-        URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
-        CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
-        BUILD_IN_SOURCE 1
-        BUILD_COMMAND ${MAKE}
-        INSTALL_COMMAND ${MAKE} -j1 install)
-    endif()
+    ExternalProject_Add(jemalloc_ep
+      URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+      CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
+      BUILD_IN_SOURCE 1
+      BUILD_COMMAND ${MAKE}
+      BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
+      INSTALL_COMMAND ${MAKE} -j1 install)
   else()
     set(JEMALLOC_VENDORED 0)
   endif()
@@ -501,12 +471,9 @@ if (NOT ZLIB_FOUND)
                       -DCMAKE_C_FLAGS=${EP_C_FLAGS}
                       -DBUILD_SHARED_LIBS=OFF)
 
-  if (CMAKE_VERSION VERSION_GREATER "3.2")
-    set(ZLIB_BUILD_BYPRODUCTS BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}")
-  endif()
   ExternalProject_Add(zlib_ep
     URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz"
-    ${ZLIB_BUILD_BYPRODUCTS}
+    BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}"
     CMAKE_ARGS ${ZLIB_CMAKE_ARGS})
   set(ZLIB_VENDORED 1)
 else()
@@ -546,10 +513,6 @@ if (NOT SNAPPY_FOUND)
     endif()
   endif()
 
-  if (CMAKE_VERSION VERSION_GREATER "3.2")
-    set(SNAPPY_BUILD_BYPRODUCTS BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}")
-  endif()
-
   if (MSVC)
     set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                           "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
@@ -568,7 +531,7 @@ if (NOT SNAPPY_FOUND)
       INSTALL_DIR ${SNAPPY_PREFIX}
       URL ${SNAPPY_SRC_URL}
       CMAKE_ARGS ${SNAPPY_CMAKE_ARGS}
-      ${SNAPPY_BUILD_BYPRODUCTS})
+      BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}")
   else()
     ExternalProject_Add(snappy_ep
       CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS}
@@ -576,7 +539,7 @@ if (NOT SNAPPY_FOUND)
       BUILD_COMMAND ${MAKE}
       INSTALL_DIR ${SNAPPY_PREFIX}
       URL ${SNAPPY_SRC_URL}
-      ${SNAPPY_BUILD_BYPRODUCTS})
+      BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}")
   endif()
   set(SNAPPY_VENDORED 1)
 else()
@@ -614,12 +577,9 @@ if (NOT BROTLI_FOUND)
                         -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE}
                         -DBUILD_SHARED_LIBS=OFF)
 
-  if (CMAKE_VERSION VERSION_GREATER "3.2")
-    set(BROTLI_BUILD_BYPRODUCTS BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}")
-  endif()
-
   ExternalProject_Add(brotli_ep
     URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz"
+    BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}"
     ${BROTLI_BUILD_BYPRODUCTS}
     CMAKE_ARGS ${BROTLI_CMAKE_ARGS}
     STEP_TARGETS headers_copy)

From cbbd04bfa8bc206630522f577069843fdf4d80e0 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 4 Jul 2017 18:15:53 -0400
Subject: [PATCH 0778/1644] ARROW-1172: [C++] Refactor to use unique_ptr for
 builders

The intent is to make the API simpler and perhaps in some cases more performant (since sharing ownership of builders is not too common, there won't be a need to deal with atomic reference counts). I happened on this when looking at ARROW-1114 where there might be quite a bit of overhead dealing with `std::vector<std::shared_ptr<ArrayBuilder>>`

Author: Kouhei Sutou <kou@clear-code.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #801 from wesm/builder-unique-ptr and squashes the following commits:

9590bfd3 [Kouhei Sutou] Remove needless GError from builder constructor
59ed8cf0 [Kouhei Sutou] Change GLib to follow builder API change
3bf684b1 [Wes McKinney] Refactor to use unique_ptr for builders
---
 c_glib/arrow-glib/array-builder.cpp     | 382 ++++++++++++++----------
 c_glib/arrow-glib/array-builder.h       |   5 +-
 c_glib/arrow-glib/array-builder.hpp     |   4 +-
 c_glib/test/helper/buildable.rb         |  15 +-
 c_glib/test/test-list-array.rb          |  10 +-
 c_glib/test/test-struct-array.rb        |   6 +-
 cpp/src/arrow/array-test.cc             |  65 ++--
 cpp/src/arrow/builder.cc                |  39 +--
 cpp/src/arrow/builder.h                 |  21 +-
 cpp/src/arrow/python/builtin_convert.cc |  22 +-
 cpp/src/arrow/python/builtin_convert.h  |   5 +-
 cpp/src/arrow/python/pandas_convert.cc  |  18 +-
 integration/integration_test.py         |   4 +-
 13 files changed, 333 insertions(+), 263 deletions(-)

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 30158b07b11d3..17b2ec8b55069 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -85,7 +85,7 @@ G_BEGIN_DECLS
  */
 
 typedef struct GArrowArrayBuilderPrivate_ {
-  std::shared_ptr<arrow::ArrayBuilder> array_builder;
+  arrow::ArrayBuilder *array_builder;
 } GArrowArrayBuilderPrivate;
 
 enum {
@@ -109,7 +109,7 @@ garrow_array_builder_finalize(GObject *object)
 
   priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(object);
 
-  priv->array_builder = nullptr;
+  delete priv->array_builder;
 
   G_OBJECT_CLASS(garrow_array_builder_parent_class)->finalize(object);
 }
@@ -127,7 +127,7 @@ garrow_array_builder_set_property(GObject *object,
   switch (prop_id) {
   case PROP_ARRAY_BUILDER:
     priv->array_builder =
-      *static_cast<std::shared_ptr<arrow::ArrayBuilder> *>(g_value_get_pointer(value));
+      static_cast<arrow::ArrayBuilder *>(g_value_get_pointer(value));
     break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
@@ -167,12 +167,26 @@ garrow_array_builder_class_init(GArrowArrayBuilderClass *klass)
 
   spec = g_param_spec_pointer("array-builder",
                               "Array builder",
-                              "The raw std::shared<arrow::ArrayBuilder> *",
+                              "The raw arrow::ArrayBuilder *",
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property(gobject_class, PROP_ARRAY_BUILDER, spec);
 }
 
+static GArrowArrayBuilder *
+garrow_array_builder_new(const std::shared_ptr<arrow::DataType> &type,
+                         GError **error,
+                         const char *context)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  std::unique_ptr<arrow::ArrayBuilder> arrow_builder;
+  auto status = arrow::MakeBuilder(memory_pool, type, &arrow_builder);
+  if (!garrow_error_check(error, status, context)) {
+    return NULL;
+  }
+  return garrow_array_builder_new_raw(arrow_builder.release());
+};
+
 /**
  * garrow_array_builder_finish:
  * @builder: A #GArrowArrayBuilder.
@@ -211,11 +225,9 @@ garrow_boolean_array_builder_class_init(GArrowBooleanArrayBuilderClass *klass)
 GArrowBooleanArrayBuilder *
 garrow_boolean_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_boolean_builder =
-    std::make_shared<arrow::BooleanBuilder>(memory_pool);
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_boolean_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::boolean(),
+                                          NULL,
+                                          "[boolean-array-builder][new]");
   return GARROW_BOOLEAN_ARRAY_BUILDER(builder);
 }
 
@@ -234,7 +246,7 @@ garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::BooleanBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(static_cast<bool>(value));
   return garrow_error_check(error, status, "[boolean-array-builder][append]");
@@ -253,7 +265,7 @@ garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::BooleanBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error,
@@ -284,11 +296,9 @@ garrow_int8_array_builder_class_init(GArrowInt8ArrayBuilderClass *klass)
 GArrowInt8ArrayBuilder *
 garrow_int8_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_int8_builder =
-    std::make_shared<arrow::Int8Builder>(memory_pool, arrow::int8());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_int8_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::int8(),
+                                          NULL,
+                                          "[int8-array-builder][new]");
   return GARROW_INT8_ARRAY_BUILDER(builder);
 }
 
@@ -307,7 +317,7 @@ garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::Int8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[int8-array-builder][append]");
@@ -326,7 +336,7 @@ garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::Int8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error, status, "[int8-array-builder][append-null]");
@@ -355,11 +365,9 @@ garrow_uint8_array_builder_class_init(GArrowUInt8ArrayBuilderClass *klass)
 GArrowUInt8ArrayBuilder *
 garrow_uint8_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_uint8_builder =
-    std::make_shared<arrow::UInt8Builder>(memory_pool, arrow::uint8());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_uint8_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::uint8(),
+                                          NULL,
+                                          "[uint8-array-builder][new]");
   return GARROW_UINT8_ARRAY_BUILDER(builder);
 }
 
@@ -378,7 +386,7 @@ garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::UInt8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[uint8-array-builder][append]");
@@ -397,7 +405,7 @@ garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::UInt8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error, status, "[uint8-array-builder][append-null]");
@@ -426,11 +434,9 @@ garrow_int16_array_builder_class_init(GArrowInt16ArrayBuilderClass *klass)
 GArrowInt16ArrayBuilder *
 garrow_int16_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_int16_builder =
-    std::make_shared<arrow::Int16Builder>(memory_pool, arrow::int16());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_int16_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::int16(),
+                                          NULL,
+                                          "[int16-array-builder][new]");
   return GARROW_INT16_ARRAY_BUILDER(builder);
 }
 
@@ -449,7 +455,7 @@ garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::Int16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[int16-array-builder][append]");
@@ -468,7 +474,7 @@ garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::Int16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error, status, "[int16-array-builder][append-null]");
@@ -497,11 +503,9 @@ garrow_uint16_array_builder_class_init(GArrowUInt16ArrayBuilderClass *klass)
 GArrowUInt16ArrayBuilder *
 garrow_uint16_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_uint16_builder =
-    std::make_shared<arrow::UInt16Builder>(memory_pool, arrow::uint16());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_uint16_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::uint16(),
+                                          NULL,
+                                          "[uint16-array-builder][new]");
   return GARROW_UINT16_ARRAY_BUILDER(builder);
 }
 
@@ -520,7 +524,7 @@ garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::UInt16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[uint16-array-builder][append]");
@@ -539,7 +543,7 @@ garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::UInt16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error,
@@ -570,11 +574,9 @@ garrow_int32_array_builder_class_init(GArrowInt32ArrayBuilderClass *klass)
 GArrowInt32ArrayBuilder *
 garrow_int32_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_int32_builder =
-    std::make_shared<arrow::Int32Builder>(memory_pool, arrow::int32());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_int32_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::int32(),
+                                          NULL,
+                                          "[int32-array-builder][new]");
   return GARROW_INT32_ARRAY_BUILDER(builder);
 }
 
@@ -593,7 +595,7 @@ garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::Int32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[int32-array-builder][append]");
@@ -612,7 +614,7 @@ garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::Int32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error, status, "[int32-array-builder][append-null]");
@@ -641,11 +643,9 @@ garrow_uint32_array_builder_class_init(GArrowUInt32ArrayBuilderClass *klass)
 GArrowUInt32ArrayBuilder *
 garrow_uint32_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_uint32_builder =
-    std::make_shared<arrow::UInt32Builder>(memory_pool, arrow::uint32());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_uint32_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::uint32(),
+                                          NULL,
+                                          "[uint32-array-builder][new]");
   return GARROW_UINT32_ARRAY_BUILDER(builder);
 }
 
@@ -664,7 +664,7 @@ garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::UInt32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[uint32-array-builder][append]");
@@ -683,7 +683,7 @@ garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::UInt32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error,
@@ -714,11 +714,9 @@ garrow_int64_array_builder_class_init(GArrowInt64ArrayBuilderClass *klass)
 GArrowInt64ArrayBuilder *
 garrow_int64_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_int64_builder =
-    std::make_shared<arrow::Int64Builder>(memory_pool, arrow::int64());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_int64_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::int64(),
+                                          NULL,
+                                          "[int64-array-builder][new]");
   return GARROW_INT64_ARRAY_BUILDER(builder);
 }
 
@@ -737,7 +735,7 @@ garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::Int64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[int64-array-builder][append]");
@@ -756,7 +754,7 @@ garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::Int64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error, status, "[int64-array-builder][append-null]");
@@ -785,11 +783,9 @@ garrow_uint64_array_builder_class_init(GArrowUInt64ArrayBuilderClass *klass)
 GArrowUInt64ArrayBuilder *
 garrow_uint64_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_uint64_builder =
-    std::make_shared<arrow::UInt64Builder>(memory_pool, arrow::uint64());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_uint64_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::uint64(),
+                                          NULL,
+                                          "[uint64-array-builder][new]");
   return GARROW_UINT64_ARRAY_BUILDER(builder);
 }
 
@@ -808,7 +804,7 @@ garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::UInt64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[uint64-array-builder][append]");
@@ -827,7 +823,7 @@ garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::UInt64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   if (status.ok()) {
@@ -861,11 +857,9 @@ garrow_float_array_builder_class_init(GArrowFloatArrayBuilderClass *klass)
 GArrowFloatArrayBuilder *
 garrow_float_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_float_builder =
-    std::make_shared<arrow::FloatBuilder>(memory_pool, arrow::float32());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_float_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::float32(),
+                                          NULL,
+                                          "[float-array-builder][new]");
   return GARROW_FLOAT_ARRAY_BUILDER(builder);
 }
 
@@ -884,7 +878,7 @@ garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::FloatBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[float-array-builder][append]");
@@ -903,7 +897,7 @@ garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::FloatBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error, status, "[float-array-builder][append-null]");
@@ -932,11 +926,9 @@ garrow_double_array_builder_class_init(GArrowDoubleArrayBuilderClass *klass)
 GArrowDoubleArrayBuilder *
 garrow_double_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_double_builder =
-    std::make_shared<arrow::DoubleBuilder>(memory_pool, arrow::float64());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_double_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::float64(),
+                                          NULL,
+                                          "[double-array-builder][new]");
   return GARROW_DOUBLE_ARRAY_BUILDER(builder);
 }
 
@@ -955,7 +947,7 @@ garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::DoubleBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value);
   return garrow_error_check(error, status, "[double-array-builder][append]");
@@ -974,7 +966,7 @@ garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::DoubleBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error,
@@ -1005,11 +997,9 @@ garrow_binary_array_builder_class_init(GArrowBinaryArrayBuilderClass *klass)
 GArrowBinaryArrayBuilder *
 garrow_binary_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_binary_builder =
-    std::make_shared<arrow::BinaryBuilder>(memory_pool, arrow::binary());
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_binary_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::binary(),
+                                          NULL,
+                                          "[binary-array-builder][new]");
   return GARROW_BINARY_ARRAY_BUILDER(builder);
 }
 
@@ -1030,7 +1020,7 @@ garrow_binary_array_builder_append(GArrowBinaryArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::BinaryBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value, length);
   return garrow_error_check(error, status, "[binary-array-builder][append]");
@@ -1049,7 +1039,7 @@ garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::BinaryBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error,
@@ -1080,11 +1070,9 @@ garrow_string_array_builder_class_init(GArrowStringArrayBuilderClass *klass)
 GArrowStringArrayBuilder *
 garrow_string_array_builder_new(void)
 {
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_string_builder =
-    std::make_shared<arrow::StringBuilder>(memory_pool);
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_string_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto builder = garrow_array_builder_new(arrow::utf8(),
+                                          NULL,
+                                          "[string-array-builder][new]");
   return GARROW_STRING_ARRAY_BUILDER(builder);
 }
 
@@ -1103,7 +1091,7 @@ garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::StringBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append(value,
                                       static_cast<gint32>(strlen(value)));
@@ -1111,9 +1099,36 @@ garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
 }
 
 
-G_DEFINE_TYPE(GArrowListArrayBuilder,
-              garrow_list_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
+typedef struct GArrowListArrayBuilderPrivate_ {
+  GArrowArrayBuilder *value_builder;
+} GArrowListArrayBuilderPrivate;
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowListArrayBuilder,
+                           garrow_list_array_builder,
+                           GARROW_TYPE_ARRAY_BUILDER)
+
+#define GARROW_LIST_ARRAY_BUILDER_GET_PRIVATE(obj)              \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                           \
+                               GARROW_TYPE_LIST_ARRAY_BUILDER,  \
+                               GArrowListArrayBuilderPrivate))
+
+static void
+garrow_list_array_builder_dispose(GObject *object)
+{
+  GArrowListArrayBuilderPrivate *priv;
+
+  priv = GARROW_LIST_ARRAY_BUILDER_GET_PRIVATE(object);
+
+  if (priv->value_builder) {
+    GArrowArrayBuilderPrivate *value_builder_priv;
+    value_builder_priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(priv->value_builder);
+    value_builder_priv->array_builder = nullptr;
+    g_object_unref(priv->value_builder);
+    priv->value_builder = NULL;
+  }
+
+  G_OBJECT_CLASS(garrow_list_array_builder_parent_class)->dispose(object);
+}
 
 static void
 garrow_list_array_builder_init(GArrowListArrayBuilder *builder)
@@ -1123,23 +1138,37 @@ garrow_list_array_builder_init(GArrowListArrayBuilder *builder)
 static void
 garrow_list_array_builder_class_init(GArrowListArrayBuilderClass *klass)
 {
+  GObjectClass *gobject_class;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->dispose = garrow_list_array_builder_dispose;
 }
 
 /**
  * garrow_list_array_builder_new:
- * @value_builder: A #GArrowArrayBuilder for value array.
+ * @data_type: A #GArrowListDataType for value.
+ * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: A newly created #GArrowListArrayBuilder.
  */
 GArrowListArrayBuilder *
-garrow_list_array_builder_new(GArrowArrayBuilder *value_builder)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_value_builder = garrow_array_builder_get_raw(value_builder);
-  auto arrow_list_builder =
-    std::make_shared<arrow::ListBuilder>(memory_pool, arrow_value_builder);
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_list_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+garrow_list_array_builder_new(GArrowListDataType *data_type,
+                              GError **error)
+{
+  if (!GARROW_IS_LIST_DATA_TYPE(data_type)) {
+    g_set_error(error,
+                GARROW_ERROR,
+                GARROW_ERROR_INVALID,
+                "[list-array-builder][new] data type must be list data type");
+    return NULL;
+  }
+
+  auto arrow_data_type =
+    garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto builder = garrow_array_builder_new(arrow_data_type,
+                                          error,
+                                          "[list-array-builder][new]");
   return GARROW_LIST_ARRAY_BUILDER(builder);
 }
 
@@ -1192,7 +1221,7 @@ garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::ListBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append();
   return garrow_error_check(error, status, "[list-array-builder][append]");
@@ -1213,7 +1242,7 @@ garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::ListBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error, status, "[list-array-builder][append-null]");
@@ -1223,22 +1252,59 @@ garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
  * garrow_list_array_builder_get_value_builder:
  * @builder: A #GArrowListArrayBuilder.
  *
- * Returns: (transfer full): The #GArrowArrayBuilder for values.
+ * Returns: (transfer none): The #GArrowArrayBuilder for values.
  */
 GArrowArrayBuilder *
 garrow_list_array_builder_get_value_builder(GArrowListArrayBuilder *builder)
 {
-  auto arrow_builder =
-    static_cast<arrow::ListBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-  auto arrow_value_builder = arrow_builder->value_builder();
-  return garrow_array_builder_new_raw(&arrow_value_builder);
+  GArrowListArrayBuilderPrivate *priv;
+
+  priv = GARROW_LIST_ARRAY_BUILDER_GET_PRIVATE(builder);
+  if (!priv->value_builder) {
+    auto arrow_builder =
+      static_cast<arrow::ListBuilder *>(
+        garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
+    auto arrow_value_builder = arrow_builder->value_builder();
+    priv->value_builder = garrow_array_builder_new_raw(arrow_value_builder);
+  }
+  return priv->value_builder;
 }
 
 
-G_DEFINE_TYPE(GArrowStructArrayBuilder,
-              garrow_struct_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
+typedef struct GArrowStructArrayBuilderPrivate_ {
+  GList *field_builders;
+} GArrowStructArrayBuilderPrivate;
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowStructArrayBuilder,
+                           garrow_struct_array_builder,
+                           GARROW_TYPE_ARRAY_BUILDER)
+
+#define GARROW_STRUCT_ARRAY_BUILDER_GET_PRIVATE(obj)                    \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
+                               GARROW_TYPE_STRUCT_ARRAY_BUILDER,        \
+                               GArrowStructArrayBuilderPrivate))
+
+static void
+garrow_struct_array_builder_dispose(GObject *object)
+{
+  GArrowStructArrayBuilderPrivate *priv;
+  GList *node;
+
+  priv = GARROW_STRUCT_ARRAY_BUILDER_GET_PRIVATE(object);
+
+  for (node = priv->field_builders; node; node = g_list_next(node)) {
+    auto field_builder = static_cast<GArrowArrayBuilder *>(node->data);
+    GArrowArrayBuilderPrivate *field_builder_priv;
+
+    field_builder_priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(field_builder);
+    field_builder_priv->array_builder = nullptr;
+    g_object_unref(field_builder);
+  }
+  g_list_free(priv->field_builders);
+  priv->field_builders = NULL;
+
+  G_OBJECT_CLASS(garrow_struct_array_builder_parent_class)->dispose(object);
+}
 
 static void
 garrow_struct_array_builder_init(GArrowStructArrayBuilder *builder)
@@ -1248,35 +1314,36 @@ garrow_struct_array_builder_init(GArrowStructArrayBuilder *builder)
 static void
 garrow_struct_array_builder_class_init(GArrowStructArrayBuilderClass *klass)
 {
+  GObjectClass *gobject_class;
+
+  gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->dispose = garrow_struct_array_builder_dispose;
 }
 
 /**
  * garrow_struct_array_builder_new:
  * @data_type: #GArrowStructDataType for the struct.
- * @field_builders: (element-type GArrowArray): #GArrowArrayBuilders
- *   for fields.
+ * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: A newly created #GArrowStructArrayBuilder.
  */
 GArrowStructArrayBuilder *
 garrow_struct_array_builder_new(GArrowStructDataType *data_type,
-                                GList *field_builders)
-{
-  auto memory_pool = arrow::default_memory_pool();
-  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
-  std::vector<std::shared_ptr<arrow::ArrayBuilder>> arrow_field_builders;
-  for (GList *node = field_builders; node; node = g_list_next(node)) {
-    auto field_builder = static_cast<GArrowArrayBuilder *>(node->data);
-    auto arrow_field_builder = garrow_array_builder_get_raw(field_builder);
-    arrow_field_builders.push_back(arrow_field_builder);
+                                GError **error)
+{
+  if (!GARROW_IS_STRUCT_DATA_TYPE(data_type)) {
+    g_set_error(error,
+                GARROW_ERROR,
+                GARROW_ERROR_INVALID,
+                "[struct-array-builder][new] data type must be struct data type");
+    return NULL;
   }
 
-  auto arrow_struct_builder =
-    std::make_shared<arrow::StructBuilder>(memory_pool,
-                                           arrow_data_type,
-                                           arrow_field_builders);
-  std::shared_ptr<arrow::ArrayBuilder> arrow_builder = arrow_struct_builder;
-  auto builder = garrow_array_builder_new_raw(&arrow_builder);
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto builder = garrow_array_builder_new(arrow_data_type,
+                                          error,
+                                          "[struct-array-builder][new]");
   return GARROW_STRUCT_ARRAY_BUILDER(builder);
 }
 
@@ -1304,7 +1371,7 @@ garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->Append();
   return garrow_error_check(error, status, "[struct-array-builder][append]");
@@ -1325,7 +1392,7 @@ garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
 {
   auto arrow_builder =
     static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
+      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
   auto status = arrow_builder->AppendNull();
   return garrow_error_check(error,
@@ -1338,51 +1405,56 @@ garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
  * @builder: A #GArrowStructArrayBuilder.
  * @i: The index of the field in the struct.
  *
- * Returns: (transfer full): The #GArrowArrayBuilder for the i-th field.
+ * Returns: (transfer none): The #GArrowArrayBuilder for the i-th field.
  */
 GArrowArrayBuilder *
 garrow_struct_array_builder_get_field_builder(GArrowStructArrayBuilder *builder,
                                               gint i)
 {
-  auto arrow_builder =
-    static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-  auto arrow_field_builder = arrow_builder->field_builder(i);
-  return garrow_array_builder_new_raw(&arrow_field_builder);
+  auto field_builders = garrow_struct_array_builder_get_field_builders(builder);
+  auto field_builder = g_list_nth_data(field_builders, i);
+  return static_cast<GArrowArrayBuilder *>(field_builder);
 }
 
 /**
  * garrow_struct_array_builder_get_field_builders:
  * @builder: A #GArrowStructArrayBuilder.
  *
- * Returns: (element-type GArrowArray) (transfer full):
+ * Returns: (element-type GArrowArray) (transfer none):
  *   The #GArrowArrayBuilder for all fields.
  */
 GList *
 garrow_struct_array_builder_get_field_builders(GArrowStructArrayBuilder *builder)
 {
-  auto arrow_struct_builder =
-    static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)).get());
-
-  GList *field_builders = NULL;
-  for (auto arrow_field_builder : arrow_struct_builder->field_builders()) {
-    auto field_builder = garrow_array_builder_new_raw(&arrow_field_builder);
-    field_builders = g_list_prepend(field_builders, field_builder);
+  GArrowStructArrayBuilderPrivate *priv;
+
+  priv = GARROW_STRUCT_ARRAY_BUILDER_GET_PRIVATE(builder);
+  if (!priv->field_builders) {
+    auto arrow_struct_builder =
+      static_cast<arrow::StructBuilder *>(
+        garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
+
+    GList *field_builders = NULL;
+    for (int i = 0; i < arrow_struct_builder->num_fields(); ++i) {
+      auto arrow_field_builder = arrow_struct_builder->field_builder(i);
+      auto field_builder = garrow_array_builder_new_raw(arrow_field_builder);
+      field_builders = g_list_prepend(field_builders, field_builder);
+    }
+    priv->field_builders = g_list_reverse(field_builders);
   }
 
-  return g_list_reverse(field_builders);
+  return priv->field_builders;
 }
 
 
 G_END_DECLS
 
 GArrowArrayBuilder *
-garrow_array_builder_new_raw(std::shared_ptr<arrow::ArrayBuilder> *arrow_builder)
+garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder)
 {
   GType type;
 
-  switch ((*arrow_builder)->type()->id()) {
+  switch (arrow_builder->type()->id()) {
   case arrow::Type::type::BOOL:
     type = GARROW_TYPE_BOOLEAN_ARRAY_BUILDER;
     break;
@@ -1440,7 +1512,7 @@ garrow_array_builder_new_raw(std::shared_ptr<arrow::ArrayBuilder> *arrow_builder
   return builder;
 }
 
-std::shared_ptr<arrow::ArrayBuilder>
+arrow::ArrayBuilder *
 garrow_array_builder_get_raw(GArrowArrayBuilder *builder)
 {
   GArrowArrayBuilderPrivate *priv;
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index ad72f9ae8488b..f5a8ac73d630a 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -772,7 +772,8 @@ struct _GArrowListArrayBuilderClass
 
 GType garrow_list_array_builder_get_type(void) G_GNUC_CONST;
 
-GArrowListArrayBuilder *garrow_list_array_builder_new(GArrowArrayBuilder *value_builder);
+GArrowListArrayBuilder *garrow_list_array_builder_new(GArrowListDataType *data_type,
+                                                      GError **error);
 
 gboolean garrow_list_array_builder_append(GArrowListArrayBuilder *builder,
                                           GError **error);
@@ -825,7 +826,7 @@ struct _GArrowStructArrayBuilderClass
 GType garrow_struct_array_builder_get_type(void) G_GNUC_CONST;
 
 GArrowStructArrayBuilder *garrow_struct_array_builder_new(GArrowStructDataType *data_type,
-                                                          GList *field_builders);
+                                                          GError **error);
 
 gboolean garrow_struct_array_builder_append(GArrowStructArrayBuilder *builder,
                                             GError **error);
diff --git a/c_glib/arrow-glib/array-builder.hpp b/c_glib/arrow-glib/array-builder.hpp
index becebb23f9bb0..e65ad005c12fd 100644
--- a/c_glib/arrow-glib/array-builder.hpp
+++ b/c_glib/arrow-glib/array-builder.hpp
@@ -22,5 +22,5 @@
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/array-builder.h>
 
-GArrowArrayBuilder *garrow_array_builder_new_raw(std::shared_ptr<arrow::ArrayBuilder> *arrow_builder);
-std::shared_ptr<arrow::ArrayBuilder> garrow_array_builder_get_raw(GArrowArrayBuilder *builder);
+GArrowArrayBuilder *garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder);
+arrow::ArrayBuilder *garrow_array_builder_get_raw(GArrowArrayBuilder *builder);
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index 4120eed6fe2d0..f1bac47d6c7b9 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -69,9 +69,11 @@ def build_string_array(values)
       build_array(Arrow::StringArrayBuilder, values)
     end
 
-    def build_list_array(value_builder_class, values_list)
-      value_builder = value_builder_class.new
-      builder = Arrow::ListArrayBuilder.new(value_builder)
+    def build_list_array(value_data_type, values_list)
+      value_field = Arrow::Field.new("value", value_data_type)
+      data_type = Arrow::ListDataType.new(value_field)
+      builder = Arrow::ListArrayBuilder.new(data_type)
+      value_builder = builder.value_builder
       values_list.each do |values|
         if values.nil?
           builder.append_null
@@ -90,13 +92,8 @@ def build_list_array(value_builder_class, values_list)
     end
 
     def build_struct_array(fields, structs)
-      field_builders = fields.collect do |field|
-        data_type_name = field.data_type.class.name
-        builder_name = data_type_name.gsub(/DataType/, "ArrayBuilder")
-        Arrow.const_get(builder_name).new
-      end
       data_type = Arrow::StructDataType.new(fields)
-      builder = Arrow::StructArrayBuilder.new(data_type, field_builders)
+      builder = Arrow::StructArrayBuilder.new(data_type)
       structs.each do |struct|
         if struct.nil?
           builder.append_null
diff --git a/c_glib/test/test-list-array.rb b/c_glib/test/test-list-array.rb
index 8e481e8367053..7f2c96787a975 100644
--- a/c_glib/test/test-list-array.rb
+++ b/c_glib/test/test-list-array.rb
@@ -22,7 +22,7 @@ def test_new
     value_offsets = Arrow::Buffer.new([0, 2, 5, 5].pack("l*"))
     data = Arrow::Buffer.new([1, 2, 3, 4, 5].pack("c*"))
     values = Arrow::Int8Array.new(5, data, nil, 0)
-    assert_equal(build_list_array(Arrow::Int8ArrayBuilder,
+    assert_equal(build_list_array(Arrow::Int8DataType.new,
                                   [[1, 2], [3, 4, 5], nil]),
                  Arrow::ListArray.new(3,
                                       value_offsets,
@@ -32,7 +32,9 @@ def test_new
   end
 
   def test_value
-    builder = Arrow::ListArrayBuilder.new(Arrow::Int8ArrayBuilder.new)
+    field = Arrow::Field.new("value", Arrow::Int8DataType.new)
+    data_type = Arrow::ListDataType.new(field)
+    builder = Arrow::ListArrayBuilder.new(data_type)
     value_builder = builder.value_builder
 
     builder.append
@@ -51,7 +53,9 @@ def test_value
   end
 
   def test_value_type
-    builder = Arrow::ListArrayBuilder.new(Arrow::Int8ArrayBuilder.new)
+    field = Arrow::Field.new("value", Arrow::Int8DataType.new)
+    data_type = Arrow::ListDataType.new(field)
+    builder = Arrow::ListArrayBuilder.new(data_type)
     array = builder.finish
     assert_equal(Arrow::Int8DataType.new, array.value_type)
   end
diff --git a/c_glib/test/test-struct-array.rb b/c_glib/test/test-struct-array.rb
index ef0bc7179f290..29219f863013d 100644
--- a/c_glib/test/test-struct-array.rb
+++ b/c_glib/test/test-struct-array.rb
@@ -55,11 +55,7 @@ def test_fields
       Arrow::Field.new("enabled", Arrow::BooleanDataType.new),
     ]
     data_type = Arrow::StructDataType.new(fields)
-    field_builders = [
-      Arrow::Int8ArrayBuilder.new,
-      Arrow::BooleanArrayBuilder.new,
-    ]
-    builder = Arrow::StructArrayBuilder.new(data_type, field_builders)
+    builder = Arrow::StructArrayBuilder.new(data_type)
 
     builder.append
     builder.get_field_builder(0).append(-29)
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index b28a3a3630205..76d6bbed33fca 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -197,12 +197,12 @@ class TestPrimitiveBuilder : public TestBuilder {
 
     type_ = Attrs::type();
 
-    std::shared_ptr<ArrayBuilder> tmp;
+    std::unique_ptr<ArrayBuilder> tmp;
     ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-    builder_ = std::dynamic_pointer_cast<BuilderType>(tmp);
+    builder_.reset(static_cast<BuilderType*>(tmp.release()));
 
     ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-    builder_nn_ = std::dynamic_pointer_cast<BuilderType>(tmp);
+    builder_nn_.reset(static_cast<BuilderType*>(tmp.release()));
   }
 
   void RandomData(int64_t N, double pct_null = 0.1) {
@@ -212,7 +212,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     test::random_null_bytes(N, pct_null, valid_bytes_.data());
   }
 
-  void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
+  void Check(const std::unique_ptr<BuilderType>& builder, bool nullable) {
     int64_t size = builder->length();
 
     auto ex_data = std::make_shared<Buffer>(
@@ -248,8 +248,8 @@ class TestPrimitiveBuilder : public TestBuilder {
 
  protected:
   std::shared_ptr<DataType> type_;
-  std::shared_ptr<BuilderType> builder_;
-  std::shared_ptr<BuilderType> builder_nn_;
+  std::unique_ptr<BuilderType> builder_;
+  std::unique_ptr<BuilderType> builder_nn_;
 
   vector<T> draws_;
   vector<uint8_t> valid_bytes_;
@@ -309,7 +309,7 @@ void TestPrimitiveBuilder<PBoolean>::RandomData(int64_t N, double pct_null) {
 
 template <>
 void TestPrimitiveBuilder<PBoolean>::Check(
-    const std::shared_ptr<BooleanBuilder>& builder, bool nullable) {
+    const std::unique_ptr<BooleanBuilder>& builder, bool nullable) {
   int64_t size = builder->length();
 
   auto ex_data = test::bytes_to_null_buffer(draws_);
@@ -1643,9 +1643,9 @@ class TestListBuilder : public TestBuilder {
     value_type_ = int32();
     type_ = list(value_type_);
 
-    std::shared_ptr<ArrayBuilder> tmp;
+    std::unique_ptr<ArrayBuilder> tmp;
     ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-    builder_ = std::dynamic_pointer_cast<ListBuilder>(tmp);
+    builder_.reset(static_cast<ListBuilder*>(tmp.release()));
   }
 
   void Done() {
@@ -1663,7 +1663,7 @@ class TestListBuilder : public TestBuilder {
 };
 
 TEST_F(TestListBuilder, Equality) {
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
 
   std::shared_ptr<Array> array, equal_array, unequal_array;
   vector<int32_t> equal_offsets = {0, 1, 2, 5, 6, 7, 8, 10};
@@ -1770,7 +1770,7 @@ TEST_F(TestListBuilder, TestBasics) {
   vector<int> lengths = {3, 0, 4};
   vector<uint8_t> is_valid = {1, 0, 1};
 
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
 
   ASSERT_OK(builder_->Reserve(lengths.size()));
   ASSERT_OK(vb->Reserve(values.size()));
@@ -1793,7 +1793,7 @@ TEST_F(TestListBuilder, BulkAppend) {
   vector<uint8_t> is_valid = {1, 0, 1};
   vector<int32_t> offsets = {0, 3, 3};
 
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
   ASSERT_OK(vb->Reserve(values.size()));
 
   builder_->Append(offsets.data(), offsets.size(), is_valid.data());
@@ -1811,7 +1811,7 @@ TEST_F(TestListBuilder, BulkAppendInvalid) {
   vector<uint8_t> is_valid = {1, 0, 1};
   vector<int32_t> offsets = {0, 2, 4};  // should be 0, 3, 3 given the is_null array
 
-  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+  Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
   ASSERT_OK(vb->Reserve(values.size()));
 
   builder_->Append(offsets.data(), offsets.size(), is_valid.data());
@@ -2003,11 +2003,10 @@ class TestStructBuilder : public TestBuilder {
     type_ = struct_(fields);
     value_fields_ = fields;
 
-    std::shared_ptr<ArrayBuilder> tmp;
+    std::unique_ptr<ArrayBuilder> tmp;
     ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-
-    builder_ = std::dynamic_pointer_cast<StructBuilder>(tmp);
-    ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+    builder_.reset(static_cast<StructBuilder*>(tmp.release()));
+    ASSERT_EQ(2, static_cast<int>(builder_->num_fields()));
   }
 
   void Done() {
@@ -2027,14 +2026,14 @@ class TestStructBuilder : public TestBuilder {
 TEST_F(TestStructBuilder, TestAppendNull) {
   ASSERT_OK(builder_->AppendNull());
   ASSERT_OK(builder_->AppendNull());
-  ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+  ASSERT_EQ(2, static_cast<int>(builder_->num_fields()));
 
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0));
   ASSERT_OK(list_vb->AppendNull());
   ASSERT_OK(list_vb->AppendNull());
   ASSERT_EQ(2, list_vb->length());
 
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1));
   ASSERT_OK(int_vb->AppendNull());
   ASSERT_OK(int_vb->AppendNull());
   ASSERT_EQ(2, int_vb->length());
@@ -2066,10 +2065,10 @@ TEST_F(TestStructBuilder, TestBasics) {
   vector<uint8_t> list_is_valid = {1, 0, 1, 1};
   vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
 
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-  ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0));
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1));
+  ASSERT_EQ(2, static_cast<int>(builder_->num_fields()));
 
   EXPECT_OK(builder_->Resize(list_lengths.size()));
   EXPECT_OK(char_vb->Resize(list_values.size()));
@@ -2102,9 +2101,9 @@ TEST_F(TestStructBuilder, BulkAppend) {
   vector<uint8_t> list_is_valid = {1, 0, 1, 1};
   vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
 
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0));
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1));
 
   ASSERT_OK(builder_->Resize(list_lengths.size()));
   ASSERT_OK(char_vb->Resize(list_values.size()));
@@ -2133,9 +2132,9 @@ TEST_F(TestStructBuilder, BulkAppendInvalid) {
   vector<uint8_t> list_is_valid = {1, 0, 1, 1};
   vector<uint8_t> struct_is_valid = {1, 0, 1, 1};  // should be 1, 1, 1, 1
 
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0));
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1));
 
   ASSERT_OK(builder_->Reserve(list_lengths.size()));
   ASSERT_OK(char_vb->Reserve(list_values.size()));
@@ -2174,9 +2173,9 @@ TEST_F(TestStructBuilder, TestEquality) {
   vector<uint8_t> unequal_list_is_valid = {1, 1, 1, 1};
   vector<uint8_t> unequal_struct_is_valid = {1, 0, 0, 1};
 
-  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
-  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
-  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0));
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1));
   ASSERT_OK(builder_->Reserve(list_lengths.size()));
   ASSERT_OK(char_vb->Reserve(list_values.size()));
   ASSERT_OK(int_vb->Reserve(int_values.size()));
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index e61258c91a232..f3ba2dddb9158 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -935,13 +935,13 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
 // ----------------------------------------------------------------------
 // ListBuilder
 
-ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
+ListBuilder::ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
     const std::shared_ptr<DataType>& type)
     : ArrayBuilder(
           pool, type ? type : std::static_pointer_cast<DataType>(
                                   std::make_shared<ListType>(value_builder->type()))),
       offset_builder_(pool),
-      value_builder_(value_builder) {}
+      value_builder_(std::move(value_builder)) {}
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<Array> values,
     const std::shared_ptr<DataType>& type)
@@ -1000,21 +1000,23 @@ void ListBuilder::Reset() {
   null_bitmap_ = nullptr;
 }
 
-std::shared_ptr<ArrayBuilder> ListBuilder::value_builder() const {
+ArrayBuilder* ListBuilder::value_builder() const {
   DCHECK(!values_) << "Using value builder is pointless when values_ is set";
-  return value_builder_;
+  return value_builder_.get();
 }
 
 // ----------------------------------------------------------------------
 // String and binary
 
 BinaryBuilder::BinaryBuilder(MemoryPool* pool)
-    : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, uint8()), binary()) {
+    : ListBuilder(pool, std::unique_ptr<ArrayBuilder>(new UInt8Builder(pool, uint8())),
+          binary()) {
   byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
 }
 
 BinaryBuilder::BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-    : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, uint8()), type) {
+    : ListBuilder(
+          pool, std::unique_ptr<ArrayBuilder>(new UInt8Builder(pool, uint8())), type) {
   byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
 }
 
@@ -1109,6 +1111,12 @@ Status FixedSizeBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
 // ----------------------------------------------------------------------
 // Struct
 
+StructBuilder::StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+    std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders)
+    : ArrayBuilder(pool, type) {
+  field_builders_ = std::move(field_builders);
+}
+
 Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
   std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
   for (size_t i = 0; i < field_builders_.size(); ++i) {
@@ -1123,11 +1131,6 @@ Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
-std::shared_ptr<ArrayBuilder> StructBuilder::field_builder(int pos) const {
-  DCHECK_GT(field_builders_.size(), 0);
-  return field_builders_[pos];
-}
-
 // ----------------------------------------------------------------------
 // Helper functions
 
@@ -1141,7 +1144,7 @@ std::shared_ptr<ArrayBuilder> StructBuilder::field_builder(int pos) const {
 //
 // TODO(wesm): come up with a less monolithic strategy
 Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-    std::shared_ptr<ArrayBuilder>* out) {
+    std::unique_ptr<ArrayBuilder>* out) {
   switch (type->id()) {
     BUILDER_CASE(UINT8, UInt8Builder);
     BUILDER_CASE(INT8, Int8Builder);
@@ -1164,24 +1167,24 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder);
     BUILDER_CASE(DECIMAL, DecimalBuilder);
     case Type::LIST: {
-      std::shared_ptr<ArrayBuilder> value_builder;
+      std::unique_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
           static_cast<ListType*>(type.get())->value_type();
       RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
-      out->reset(new ListBuilder(pool, value_builder));
+      out->reset(new ListBuilder(pool, std::move(value_builder)));
       return Status::OK();
     }
 
     case Type::STRUCT: {
       const std::vector<FieldPtr>& fields = type->children();
-      std::vector<std::shared_ptr<ArrayBuilder>> values_builder;
+      std::vector<std::unique_ptr<ArrayBuilder>> values_builder;
 
       for (auto it : fields) {
-        std::shared_ptr<ArrayBuilder> builder;
+        std::unique_ptr<ArrayBuilder> builder;
         RETURN_NOT_OK(MakeBuilder(pool, it->type(), &builder));
-        values_builder.push_back(builder);
+        values_builder.emplace_back(std::move(builder));
       }
-      out->reset(new StructBuilder(pool, type, values_builder));
+      out->reset(new StructBuilder(pool, type, std::move(values_builder)));
       return Status::OK();
     }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 75fb6efffcc58..5acefa1fc6298 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -510,7 +510,7 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
  public:
   /// Use this constructor to incrementally build the value array along with offsets and
   /// null bitmap.
-  ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
+  ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
       const std::shared_ptr<DataType>& type = nullptr);
 
   /// Use this constructor to build the list with a pre-existing values array
@@ -536,11 +536,11 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
 
   Status AppendNull() { return Append(false); }
 
-  std::shared_ptr<ArrayBuilder> value_builder() const;
+  ArrayBuilder* value_builder() const;
 
  protected:
   BufferBuilder offset_builder_;
-  std::shared_ptr<ArrayBuilder> value_builder_;
+  std::unique_ptr<ArrayBuilder> value_builder_;
   std::shared_ptr<Array> values_;
 
   void Reset();
@@ -642,10 +642,7 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
 class ARROW_EXPORT StructBuilder : public ArrayBuilder {
  public:
   StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-      const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders)
-      : ArrayBuilder(pool, type) {
-    field_builders_ = field_builders;
-  }
+      std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders);
 
   Status Finish(std::shared_ptr<Array>* out) override;
 
@@ -669,14 +666,12 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
 
   Status AppendNull() { return Append(false); }
 
-  std::shared_ptr<ArrayBuilder> field_builder(int pos) const;
+  ArrayBuilder* field_builder(int i) const { return field_builders_[i].get(); }
 
-  const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders() const {
-    return field_builders_;
-  }
+  int num_fields() const { return static_cast<int>(field_builders_.size()); }
 
  protected:
-  std::vector<std::shared_ptr<ArrayBuilder>> field_builders_;
+  std::vector<std::unique_ptr<ArrayBuilder>> field_builders_;
 };
 
 // ----------------------------------------------------------------------
@@ -763,7 +758,7 @@ class ARROW_EXPORT StringDictionaryBuilder
 // Helper functions
 
 Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-    std::shared_ptr<ArrayBuilder>* out);
+    std::unique_ptr<ArrayBuilder>* out);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 2f334f0ed0206..c7af5679e3879 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -302,7 +302,7 @@ Status InferArrowTypeAndSize(
 // Marshal Python sequence (list, tuple, etc.) to Arrow array
 class SeqConverter {
  public:
-  virtual Status Init(const std::shared_ptr<ArrayBuilder>& builder) {
+  virtual Status Init(ArrayBuilder* builder) {
     builder_ = builder;
     return Status::OK();
   }
@@ -312,15 +312,15 @@ class SeqConverter {
   virtual ~SeqConverter() {}
 
  protected:
-  std::shared_ptr<ArrayBuilder> builder_;
+  ArrayBuilder* builder_;
 };
 
 template <typename BuilderType>
 class TypedConverter : public SeqConverter {
  public:
-  Status Init(const std::shared_ptr<ArrayBuilder>& builder) override {
+  Status Init(ArrayBuilder* builder) override {
     builder_ = builder;
-    typed_builder_ = static_cast<BuilderType*>(builder.get());
+    typed_builder_ = static_cast<BuilderType*>(builder);
     return Status::OK();
   }
 
@@ -512,7 +512,7 @@ class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter>
 
 class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
  public:
-  Status Init(const std::shared_ptr<ArrayBuilder>& builder) override;
+  Status Init(ArrayBuilder* builder) override;
 
   inline Status AppendItem(const OwnedRef& item) override {
     if (item.obj() == Py_None) {
@@ -597,9 +597,9 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
   }
 }
 
-Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
+Status ListConverter::Init(ArrayBuilder* builder) {
   builder_ = builder;
-  typed_builder_ = static_cast<ListBuilder*>(builder.get());
+  typed_builder_ = static_cast<ListBuilder*>(builder);
 
   value_converter_ =
       GetConverter(static_cast<ListType*>(builder->type().get())->value_type());
@@ -611,8 +611,8 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>& builder) {
   return Status::OK();
 }
 
-Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
-    const std::shared_ptr<ArrayBuilder>& builder, int64_t size) {
+Status AppendPySequence(PyObject* obj, int64_t size,
+    const std::shared_ptr<DataType>& type, ArrayBuilder* builder) {
   PyDateTime_IMPORT;
   std::shared_ptr<SeqConverter> converter = GetConverter(type);
   if (converter == nullptr) {
@@ -641,9 +641,9 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
   }
 
   // Give the sequence converter an array builder
-  std::shared_ptr<ArrayBuilder> builder;
+  std::unique_ptr<ArrayBuilder> builder;
   RETURN_NOT_OK(MakeBuilder(pool, type, &builder));
-  RETURN_NOT_OK(AppendPySequence(obj, type, builder, size));
+  RETURN_NOT_OK(AppendPySequence(obj, size, type, builder.get()));
   return builder->Finish(out);
 }
 
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index 54860b36c1986..7a84cbefef6a1 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -42,9 +42,8 @@ ARROW_EXPORT arrow::Status InferArrowTypeAndSize(
     PyObject* obj, int64_t* size, std::shared_ptr<arrow::DataType>* out_type);
 ARROW_EXPORT arrow::Status InferArrowSize(PyObject* obj, int64_t* size);
 
-ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj,
-    const std::shared_ptr<arrow::DataType>& type,
-    const std::shared_ptr<arrow::ArrayBuilder>& builder, int64_t size);
+ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj, int64_t size,
+    const std::shared_ptr<arrow::DataType>& type, arrow::ArrayBuilder* builder);
 
 // Type and size inference
 ARROW_EXPORT
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 60ae1b23844b6..0858d06a0efde 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -921,8 +921,10 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
     return Status::NotImplemented("strided arrays not implemented for lists");
   }
 
-  auto value_builder = std::make_shared<BuilderT>(pool_, type);
-  ListBuilder list_builder(pool_, value_builder);
+  ListBuilder list_builder(
+      pool_, std::unique_ptr<ArrayBuilder>(new BuilderT(pool_, type)));
+  BuilderT* value_builder = static_cast<BuilderT*>(list_builder.value_builder());
+
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
   for (int64_t i = 0; i < length_; ++i) {
     if (PandasObjectIsNull(objects[i])) {
@@ -957,7 +959,7 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
         ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
         return Status::TypeError(ss.str());
       }
-      RETURN_NOT_OK(AppendPySequence(objects[i], type, value_builder, size));
+      RETURN_NOT_OK(AppendPySequence(objects[i], size, type, value_builder));
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }
@@ -981,8 +983,10 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     return Status::NotImplemented("strided arrays not implemented for lists");
   }
 
-  auto value_builder = std::make_shared<StringBuilder>(pool_);
-  ListBuilder list_builder(pool_, value_builder);
+  ListBuilder list_builder(
+      pool_, std::unique_ptr<ArrayBuilder>(new StringBuilder(pool_)));
+  auto value_builder = static_cast<StringBuilder*>(list_builder.value_builder());
+
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
   for (int64_t i = 0; i < length_; ++i) {
     if (PandasObjectIsNull(objects[i])) {
@@ -995,7 +999,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
       RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
 
       RETURN_NOT_OK(
-          AppendObjectStrings(numpy_array, nullptr, value_builder.get(), &have_bytes));
+          AppendObjectStrings(numpy_array, nullptr, value_builder, &have_bytes));
     } else if (PyList_Check(objects[i])) {
       int64_t size;
       std::shared_ptr<DataType> inferred_type;
@@ -1006,7 +1010,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
         ss << inferred_type->ToString() << " cannot be converted to STRING.";
         return Status::TypeError(ss.str());
       }
-      RETURN_NOT_OK(AppendPySequence(objects[i], inferred_type, value_builder, size));
+      RETURN_NOT_OK(AppendPySequence(objects[i], size, inferred_type, value_builder));
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 9532d4e7ec7e1..76099669ad879 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -756,7 +756,7 @@ def _temp_path():
         generate_primitive_case([0, 0, 0]),
         generate_datetime_case(),
         generate_nested_case(),
-        # generate_dictionary_case()
+        generate_dictionary_case()
     ]
 
     generated_paths = []
@@ -943,7 +943,7 @@ def get_static_json_files():
 
 
 def run_all_tests(debug=False):
-    testers = [CPPTester(debug=debug), JavaTester(debug=debug)]
+    testers = [CPPTester(debug=debug)]  # , JavaTester(debug=debug)]
     static_json_files = get_static_json_files()
     generated_json_files = get_generated_json_files()
     json_files = static_json_files + generated_json_files

From 7d86c28e09f6c402ea1dbb9e67dcee2507b62986 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Tue, 4 Jul 2017 18:19:00 -0400
Subject: [PATCH 0779/1644] ARROW-693: [Java] Add dictionary support to JSON
 reader and writer

Added support for Dictionary encoded vectors for reading and writing JSON.  Also added validation to compare DictionaryEncodings and DictionaryBatches.

New unit test for round-trip of flat dictionary type.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #752 from BryanCutler/java-json-dict-support-ARROW-693 and squashes the following commits:

9f9eae92 [Bryan Cutler] Adding back in Dictionary.equals, this should be a separate issue
923c3d02 [Bryan Cutler] Changed integration.py script to produce correct Field name for dictionary encoded fields, enabled dictionary_generation
90c59090 [Bryan Cutler] Removed equals from Dictionary since it does not compare the FieldVector properly, fixed Integration to check FieldVectors of Dictionary
e60b7f18 [Bryan Cutler] Merge remote-tracking branch 'upstream/master' into java-json-dict-support-ARROW-693
b8fb5430 [Bryan Cutler] Removed need for dictionary fields to have a specific name, now name is not checked
577ec665 [Bryan Cutler] Merge remote-tracking branch 'upstream/master' into java-json-dict-support-ARROW-693
7c56c0ca [Bryan Cutler] fixed up Validator error messages
552c26cd [Bryan Cutler] expanded dictionary test to use multiple dictionaries, and 2 fields using same dict
a7c67447 [Bryan Cutler] Tests working with nested dictionary
8ab4dd38 [Bryan Cutler] cleaned up validator for integration tests
8a323cf1 [Bryan Cutler] cleanup of unused imports
22e849ae [Bryan Cutler] Removed WIP on DictionaryWriter for now
933a78c6 [Bryan Cutler] JsonFileReader and JsonFileWriter are working for Java, tested with flat Dictionary type
---
 integration/integration_test.py               |   2 +
 .../org/apache/arrow/tools/Integration.java   |  27 +-
 .../vector/dictionary/DictionaryProvider.java |   5 +
 .../apache/arrow/vector/file/ArrowReader.java |  52 +---
 .../apache/arrow/vector/file/ArrowWriter.java |  62 ++---
 .../vector/file/json/JsonFileReader.java      |  78 +++++-
 .../vector/file/json/JsonFileWriter.java      |  54 +++-
 .../vector/types/pojo/DictionaryEncoding.java |  10 +-
 .../arrow/vector/util/DictionaryUtility.java  | 113 +++++++++
 .../apache/arrow/vector/util/Validator.java   | 237 ++++++++++-------
 .../arrow/vector/file/BaseFileTest.java       | 238 ++++++++++++++++++
 .../arrow/vector/file/TestArrowFile.java      | 153 ++---------
 .../arrow/vector/file/json/TestJSONFile.java  |  93 ++++++-
 13 files changed, 790 insertions(+), 334 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 76099669ad879..215ba58232a00 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -581,6 +581,8 @@ def _get_type_layout(self):
         return self.index_type._get_type_layout()
 
     def generate_column(self, size, name=None):
+        if name is None:
+            name = self.name
         return self.index_type.generate_range(size, 0, len(self.dictionary),
                                               name=name)
 
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index e8266d50786d3..7d4c86f81670f 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -27,6 +27,8 @@
 import org.apache.arrow.vector.file.ArrowFileWriter;
 import org.apache.arrow.vector.file.json.JsonFileReader;
 import org.apache.arrow.vector.file.json.JsonFileWriter;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.Validator;
 import org.apache.commons.cli.CommandLine;
@@ -41,6 +43,7 @@
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
@@ -90,11 +93,21 @@ private File validateFile(String type, String fileName, boolean shouldExist) {
     return f;
   }
 
+  static void extractDictionaryEncodings(List<Field> fields, List<DictionaryEncoding> encodings) {
+    for (Field field: fields) {
+      DictionaryEncoding encoding = field.getDictionary();
+      if (encoding != null) {
+        encodings.add(encoding);
+      }
+
+      extractDictionaryEncodings(field.getChildren(), encodings);
+    }
+  }
+
   void run(String[] args) throws ParseException, IOException {
     CommandLineParser parser = new PosixParser();
     CommandLine cmd = parser.parse(options, args, false);
 
-
     Command command = toCommand(cmd.getOptionValue("command"));
     File arrowFile = validateFile("arrow", cmd.getOptionValue("arrow"), command.arrowExists);
     File jsonFile = validateFile("json", cmd.getOptionValue("json"), command.jsonExists);
@@ -124,7 +137,7 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
           LOGGER.debug("Found schema: " + schema);
           try (JsonFileWriter writer = new JsonFileWriter(jsonFile, JsonFileWriter.config()
               .pretty(true))) {
-            writer.start(schema);
+            writer.start(schema, arrowReader);
             for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
               if (!arrowReader.loadRecordBatch(rbBlock)) {
                 throw new IOException("Expected to load record batch");
@@ -147,7 +160,7 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
           try (FileOutputStream fileOutputStream = new FileOutputStream(arrowFile);
                VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
                // TODO json dictionaries
-               ArrowFileWriter arrowWriter = new ArrowFileWriter(root, null, fileOutputStream
+               ArrowFileWriter arrowWriter = new ArrowFileWriter(root, reader, fileOutputStream
                    .getChannel())) {
             arrowWriter.start();
             while (reader.read(root)) {
@@ -189,6 +202,14 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
             jsonRoot.close();
             totalBatches++;
           }
+
+          // Validate Dictionaries after ArrowFileReader has read batches
+          List<DictionaryEncoding> encodingsJson = new ArrayList<>();
+          extractDictionaryEncodings(jsonSchema.getFields(), encodingsJson);
+          List<DictionaryEncoding> encodingsArrow = new ArrayList<>();
+          extractDictionaryEncodings(arrowSchema.getFields(), encodingsArrow);
+          Validator.compareDictionaries(encodingsJson, encodingsArrow, jsonReader, arrowReader);
+
           boolean hasMoreJSON = jsonRoot != null;
           boolean hasMoreArrow = iterator.hasNext();
           if (hasMoreJSON || hasMoreArrow) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
index 63fde2536da8b..87516c9a8fc5b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
@@ -19,6 +19,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 public interface DictionaryProvider {
 
@@ -39,6 +40,10 @@ public void put(Dictionary dictionary) {
       map.put(dictionary.getEncoding().getId(), dictionary);
     }
 
+    public final Set<Long> getDictionaryIds() {
+      return map.keySet();
+    }
+
     @Override
     public Dictionary lookup(long id) {
       return map.get(id);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index b331ea776cafc..f6b104145527c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -36,12 +36,9 @@
 import org.apache.arrow.vector.schema.ArrowMessage;
 import org.apache.arrow.vector.schema.ArrowMessage.ArrowMessageVisitor;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Int;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.DictionaryUtility;
 
 public abstract class ArrowReader<T extends ReadChannel> implements DictionaryProvider, AutoCloseable {
 
@@ -83,11 +80,11 @@ public Map<Long, Dictionary> getDictionaryVectors() throws IOException {
 
   @Override
   public Dictionary lookup(long id) {
-    if (initialized) {
-      return dictionaries.get(id);
-    } else {
-      return null;
+    if (!initialized) {
+      throw new IllegalStateException("Unable to lookup until reader has been initialized");
     }
+
+    return dictionaries.get(id);
   }
 
   // Returns true if a batch was read, false on EOS
@@ -155,8 +152,9 @@ private void initialize() throws IOException {
     List<FieldVector> vectors = new ArrayList<>();
     Map<Long, Dictionary> dictionaries = new HashMap<>();
 
+    // Convert fields with dictionaries to have the index type
     for (Field field: originalSchema.getFields()) {
-      Field updated = toMemoryFormat(field, dictionaries);
+      Field updated = DictionaryUtility.toMemoryFormat(field, allocator, dictionaries);
       fields.add(updated);
       vectors.add(updated.createVector(allocator));
     }
@@ -167,42 +165,6 @@ private void initialize() throws IOException {
     this.dictionaries = Collections.unmodifiableMap(dictionaries);
   }
 
-  // in the message format, fields have the dictionary type
-  // in the memory format, they have the index type
-  private Field toMemoryFormat(Field field, Map<Long, Dictionary> dictionaries) {
-    DictionaryEncoding encoding = field.getDictionary();
-    List<Field> children = field.getChildren();
-
-    if (encoding == null && children.isEmpty()) {
-      return field;
-    }
-
-    List<Field> updatedChildren = new ArrayList<>(children.size());
-    for (Field child: children) {
-      updatedChildren.add(toMemoryFormat(child, dictionaries));
-    }
-
-    ArrowType type;
-    if (encoding == null) {
-      type = field.getType();
-    } else {
-      // re-type the field for in-memory format
-      type = encoding.getIndexType();
-      if (type == null) {
-        type = new Int(32, true);
-      }
-      // get existing or create dictionary vector
-      if (!dictionaries.containsKey(encoding.getId())) {
-        // create a new dictionary vector for the values
-        Field dictionaryField = new Field(field.getName(), new FieldType(field.isNullable(), field.getType(), null, null), children);
-        FieldVector dictionaryVector = dictionaryField.createVector(allocator);
-        dictionaries.put(encoding.getId(), new Dictionary(dictionaryVector, encoding));
-      }
-    }
-
-    return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren);
-  }
-
   private void load(ArrowDictionaryBatch dictionaryBatch) {
     long id = dictionaryBatch.getDictionaryId();
     Dictionary dictionary = dictionaries.get(id);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 4abaed0ee7608..3b37071382ff6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -20,10 +20,9 @@
 import java.io.IOException;
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
+import java.util.Set;
 
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
@@ -33,11 +32,9 @@
 import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.stream.MessageSerializer;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.DictionaryUtility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -72,53 +69,26 @@ protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, Writab
     this.out = new WriteChannel(out);
 
     List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
-    Map<Long, ArrowDictionaryBatch> dictionaryBatches = new HashMap<>();
+    Set<Long> dictionaryIdsUsed = new HashSet<>();
 
+    // Convert fields with dictionaries to have dictionary type
     for (Field field: root.getSchema().getFields()) {
-      fields.add(toMessageFormat(field, provider, dictionaryBatches));
+      fields.add(DictionaryUtility.toMessageFormat(field, provider, dictionaryIdsUsed));
     }
 
-    this.schema = new Schema(fields, root.getSchema().getCustomMetadata());
-    this.dictionaries = Collections.unmodifiableList(new ArrayList<>(dictionaryBatches.values()));
-  }
-
-  // in the message format, fields have the dictionary type
-  // in the memory format, they have the index type
-  private Field toMessageFormat(Field field, DictionaryProvider provider, Map<Long, ArrowDictionaryBatch> batches) {
-    DictionaryEncoding encoding = field.getDictionary();
-    List<Field> children = field.getChildren();
-
-    if (encoding == null && children.isEmpty()) {
-      return field;
-    }
-
-    List<Field> updatedChildren = new ArrayList<>(children.size());
-    for (Field child: children) {
-      updatedChildren.add(toMessageFormat(child, provider, batches));
-    }
-
-    ArrowType type;
-    if (encoding == null) {
-      type = field.getType();
-    } else {
-      long id = encoding.getId();
+    // Create a record batch for each dictionary
+    this.dictionaries = new ArrayList<>(dictionaryIdsUsed.size());
+    for (long id: dictionaryIdsUsed) {
       Dictionary dictionary = provider.lookup(id);
-      if (dictionary == null) {
-        throw new IllegalArgumentException("Could not find dictionary with ID " + id);
-      }
-      type = dictionary.getVectorType();
-
-      if (!batches.containsKey(id)) {
-        FieldVector vector = dictionary.getVector();
-        int count = vector.getAccessor().getValueCount();
-        VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(field), ImmutableList.of(vector), count);
-        VectorUnloader unloader = new VectorUnloader(root);
-        ArrowRecordBatch batch = unloader.getRecordBatch();
-        batches.put(id, new ArrowDictionaryBatch(id, batch));
-      }
+      FieldVector vector = dictionary.getVector();
+      int count = vector.getAccessor().getValueCount();
+      VectorSchemaRoot dictRoot = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), count);
+      VectorUnloader unloader = new VectorUnloader(dictRoot);
+      ArrowRecordBatch batch = unloader.getRecordBatch();
+      this.dictionaries.add(new ArrowDictionaryBatch(id, batch));
     }
 
-    return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren);
+    this.schema = new Schema(fields, root.getSchema().getCustomMetadata());
   }
 
   public void start() throws IOException {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 3ef1484e7eaba..364d273fadae4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -26,8 +26,12 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.BitVector;
@@ -62,9 +66,12 @@
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.DictionaryUtility;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
 
@@ -74,11 +81,13 @@
 import com.fasterxml.jackson.databind.MappingJsonFactory;
 import com.google.common.base.Objects;
 
-public class JsonFileReader implements AutoCloseable {
+public class JsonFileReader implements AutoCloseable, DictionaryProvider {
   private final File inputFile;
   private final JsonParser parser;
   private final BufferAllocator allocator;
   private Schema schema;
+  private Map<Long, Dictionary> dictionaries;
+  private Boolean started = false;
 
   public JsonFileReader(File inputFile, BufferAllocator allocator) throws JsonParseException, IOException {
     super();
@@ -88,13 +97,68 @@ public JsonFileReader(File inputFile, BufferAllocator allocator) throws JsonPars
     this.parser = jsonFactory.createParser(inputFile);
   }
 
+  @Override
+  public Dictionary lookup(long id) {
+    if (!started) {
+      throw new IllegalStateException("Unable to lookup until after read() has started");
+    }
+
+    return dictionaries.get(id);
+  }
+
   public Schema start() throws JsonParseException, IOException {
     readToken(START_OBJECT);
     {
-      this.schema = readNextField("schema", Schema.class);
+      Schema originalSchema = readNextField("schema", Schema.class);
+      List<Field> fields = new ArrayList<>();
+      dictionaries = new HashMap<>();
+
+      // Convert fields with dictionaries to have the index type
+      for (Field field: originalSchema.getFields()) {
+        fields.add(DictionaryUtility.toMemoryFormat(field, allocator, dictionaries));
+      }
+      this.schema = new Schema(fields, originalSchema.getCustomMetadata());
+
+      if (!dictionaries.isEmpty()) {
+        nextFieldIs("dictionaries");
+        readDictionaryBatches();
+      }
+
       nextFieldIs("batches");
       readToken(START_ARRAY);
-      return schema;
+      started = true;
+      return this.schema;
+    }
+  }
+
+  private void readDictionaryBatches() throws JsonParseException, IOException {
+    readToken(START_ARRAY);
+    JsonToken token = parser.nextToken();
+    boolean haveDictionaryBatch = token == START_OBJECT;
+    while (haveDictionaryBatch) {
+
+      // Lookup what dictionary for the batch about to be read
+      long id = readNextField("id", Long.class);
+      Dictionary dict = dictionaries.get(id);
+      if (dict == null) {
+        throw new IllegalArgumentException("Dictionary with id: " + id + " missing encoding from schema Field");
+      }
+
+      // Read the dictionary record batch
+      nextFieldIs("data");
+      FieldVector vector = dict.getVector();
+      List<Field> fields = ImmutableList.of(vector.getField());
+      List<FieldVector> vectors = ImmutableList.of(vector);
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount());
+      read(root);
+
+      readToken(END_OBJECT);
+      token = parser.nextToken();
+      haveDictionaryBatch = token == START_OBJECT;
+    }
+
+    if (token != END_ARRAY) {
+      throw new IllegalArgumentException("Invalid token: " + token + " expected end of array at " + parser.getTokenLocation());
     }
   }
 
@@ -107,7 +171,7 @@ public boolean read(VectorSchemaRoot root) throws IOException {
         nextFieldIs("columns");
         readToken(START_ARRAY);
         {
-          for (Field field : schema.getFields()) {
+          for (Field field : root.getSchema().getFields()) {
             FieldVector vector = root.getVector(field.getName());
             readVector(field, vector);
           }
@@ -158,8 +222,9 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
     }
     readToken(START_OBJECT);
     {
+      // If currently reading dictionaries, field name is not important so don't check
       String name = readNextField("name", String.class);
-      if (!Objects.equal(field.getName(), name)) {
+      if (started && !Objects.equal(field.getName(), name)) {
         throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name);
       }
       int count = readNextField("count", Integer.class);
@@ -302,6 +367,9 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
   @Override
   public void close() throws IOException {
     parser.close();
+    for (Dictionary dictionary: dictionaries.values()) {
+      dictionary.getVector().close();
+    }
   }
 
   private <T> T readNextField(String expectedFieldName, Class<T> c) throws IOException, JsonParseException {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index d86b3de3b9da3..befa92251f0f9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -19,8 +19,12 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.BufferBacked;
 import org.apache.arrow.vector.DateDayVector;
@@ -38,6 +42,8 @@
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -47,6 +53,7 @@
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
 import com.fasterxml.jackson.databind.MappingJsonFactory;
+import org.apache.arrow.vector.util.DictionaryUtility;
 import org.apache.commons.codec.binary.Hex;
 
 public class JsonFileWriter implements AutoCloseable {
@@ -85,22 +92,61 @@ public JsonFileWriter(File outputFile, JSONWriteConfig config) throws IOExceptio
     }
   }
 
-  public void start(Schema schema) throws IOException {
-    this.schema = schema;
+  public void start(Schema schema, DictionaryProvider provider) throws IOException {
+    List<Field> fields = new ArrayList<>(schema.getFields().size());
+    Set<Long> dictionaryIdsUsed = new HashSet<>();
+    this.schema = schema;  // Store original Schema to ensure batches written match
+
+    // Convert fields with dictionaries to have dictionary type
+    for (Field field: schema.getFields()) {
+      fields.add(DictionaryUtility.toMessageFormat(field, provider, dictionaryIdsUsed));
+    }
+    Schema updatedSchema = new Schema(fields, schema.getCustomMetadata());
+
     generator.writeStartObject();
-    generator.writeObjectField("schema", schema);
+    generator.writeObjectField("schema", updatedSchema);
+
+    // Write all dictionaries that were used
+    if (!dictionaryIdsUsed.isEmpty()) {
+      writeDictionaryBatches(generator, dictionaryIdsUsed, provider);
+    }
+
+    // Start writing of record batches
     generator.writeArrayFieldStart("batches");
   }
 
+  private void writeDictionaryBatches(JsonGenerator generator, Set<Long> dictionaryIdsUsed, DictionaryProvider provider) throws IOException {
+    generator.writeArrayFieldStart("dictionaries");
+    for (Long id: dictionaryIdsUsed) {
+      generator.writeStartObject();
+      generator.writeObjectField("id", id);
+
+      generator.writeFieldName("data");
+      Dictionary dictionary = provider.lookup(id);
+      FieldVector vector = dictionary.getVector();
+      List<Field> fields = ImmutableList.of(vector.getField());
+      List<FieldVector> vectors = ImmutableList.of(vector);
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount());
+      writeBatch(root);
+
+      generator.writeEndObject();
+    }
+    generator.writeEndArray();
+  }
+
   public void write(VectorSchemaRoot recordBatch) throws IOException {
     if (!recordBatch.getSchema().equals(schema)) {
       throw new IllegalArgumentException("record batches must have the same schema: " + schema);
     }
+    writeBatch(recordBatch);
+  }
+
+  private void writeBatch(VectorSchemaRoot recordBatch) throws IOException {
     generator.writeStartObject();
     {
       generator.writeObjectField("count", recordBatch.getRowCount());
       generator.writeArrayFieldStart("columns");
-      for (Field field : schema.getFields()) {
+      for (Field field : recordBatch.getSchema().getFields()) {
         FieldVector vector = recordBatch.getVector(field.getName());
         writeVector(field, vector);
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
index 32568d34ba495..8a0224d5564b0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
@@ -20,6 +20,9 @@
 
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 
 public class DictionaryEncoding {
@@ -28,7 +31,11 @@ public class DictionaryEncoding {
   private final boolean ordered;
   private final Int indexType;
 
-  public DictionaryEncoding(long id, boolean ordered, Int indexType) {
+  @JsonCreator
+  public DictionaryEncoding(
+      @JsonProperty("id") long id,
+      @JsonProperty("isOrdered") boolean ordered,
+      @JsonProperty("indexType") Int indexType) {
     this.id = id;
     this.ordered = ordered;
     this.indexType = indexType == null ? new Int(32, true) : indexType;
@@ -38,6 +45,7 @@ public long getId() {
     return id;
   }
 
+  @JsonGetter("isOrdered")
   public boolean isOrdered() {
     return ordered;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
new file mode 100644
index 0000000000000..4108dc4610838
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+
+
+public class DictionaryUtility {
+
+  /**
+   * Convert field and child fields that have a dictionary encoding to message format, so fields
+   * have the dictionary type
+   *
+   * NOTE: in the message format, fields have the dictionary type
+   *       in the memory format, they have the index type
+   */
+  public static Field toMessageFormat(Field field, DictionaryProvider provider, Set<Long> dictionaryIdsUsed) {
+    DictionaryEncoding encoding = field.getDictionary();
+    List<Field> children = field.getChildren();
+
+    if (encoding == null && children.isEmpty()) {
+      return field;
+    }
+
+    List<Field> updatedChildren = new ArrayList<>(children.size());
+    for (Field child: children) {
+      updatedChildren.add(toMessageFormat(child, provider, dictionaryIdsUsed));
+    }
+
+    ArrowType type;
+    if (encoding == null) {
+      type = field.getType();
+    } else {
+      long id = encoding.getId();
+      Dictionary dictionary = provider.lookup(id);
+      if (dictionary == null) {
+        throw new IllegalArgumentException("Could not find dictionary with ID " + id);
+      }
+      type = dictionary.getVectorType();
+
+      dictionaryIdsUsed.add(id);
+    }
+
+    return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren);
+  }
+
+  /**
+   * Convert field and child fields that have a dictionary encoding to message format, so fields
+   * have the index type
+   */
+  public static Field toMemoryFormat(Field field, BufferAllocator allocator, Map<Long, Dictionary> dictionaries) {
+    DictionaryEncoding encoding = field.getDictionary();
+    List<Field> children = field.getChildren();
+
+    if (encoding == null && children.isEmpty()) {
+      return field;
+    }
+
+    List<Field> updatedChildren = new ArrayList<>(children.size());
+    for (Field child: children) {
+      updatedChildren.add(toMemoryFormat(child, allocator, dictionaries));
+    }
+
+    ArrowType type;
+    if (encoding == null) {
+      type = field.getType();
+    } else {
+      // re-type the field for in-memory format
+      type = encoding.getIndexType();
+      if (type == null) {
+        type = new ArrowType.Int(32, true);
+      }
+      // get existing or create dictionary vector
+      if (!dictionaries.containsKey(encoding.getId())) {
+        // create a new dictionary vector for the values
+        String dictName = "DICT" + encoding.getId();
+        Field dictionaryField = new Field(dictName, new FieldType(false, field.getType(), null, null), children);
+        FieldVector dictionaryVector = dictionaryField.createVector(allocator);
+        dictionaries.put(encoding.getId(), new Dictionary(dictionaryVector, encoding));
+      }
+    }
+
+    return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
index 3035576da3327..b70a63fe7dd02 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
@@ -22,7 +22,10 @@
 
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 
@@ -33,101 +36,143 @@
  */
 public class Validator {
 
-    /**
-     * Validate two arrow schemas are equal.
-     *
-     * @param schema1 the 1st shema to compare
-     * @param schema2 the 2nd shema to compare
-     * @throws IllegalArgumentException if they are different.
-     */
-    public static void compareSchemas(Schema schema1, Schema schema2) {
-        if (!schema2.equals(schema1)) {
-            throw new IllegalArgumentException("Different schemas:\n" + schema2 + "\n" + schema1);
-        }
-    }
-
-    /**
-     * Validate two arrow vectorSchemaRoot are equal.
-     *
-     * @param root1 the 1st shema to compare
-     * @param root2 the 2nd shema to compare
-     * @throws IllegalArgumentException if they are different.
-     */
-    public static void compareVectorSchemaRoot(VectorSchemaRoot root1, VectorSchemaRoot root2) {
-        compareSchemas(root2.getSchema(), root1.getSchema());
-        if (root1.getRowCount() != root2.getRowCount()) {
-            throw new IllegalArgumentException("Different row count:\n" + root1.getRowCount() + "\n" + root2.getRowCount());
-        }
-        List<FieldVector> arrowVectors = root1.getFieldVectors();
-        List<FieldVector> jsonVectors = root2.getFieldVectors();
-        if (arrowVectors.size() != jsonVectors.size()) {
-            throw new IllegalArgumentException("Different column count:\n" + arrowVectors.size() + "\n" + jsonVectors.size());
-        }
-        for (int i = 0; i < arrowVectors.size(); i++) {
-            Field field = root1.getSchema().getFields().get(i);
-            FieldVector arrowVector = arrowVectors.get(i);
-            FieldVector jsonVector = jsonVectors.get(i);
-            int valueCount = arrowVector.getAccessor().getValueCount();
-            if (valueCount != jsonVector.getAccessor().getValueCount()) {
-                throw new IllegalArgumentException("Different value count for field " + field + " : " + valueCount + " != " + jsonVector.getAccessor().getValueCount());
-            }
-            for (int j = 0; j < valueCount; j++) {
-                Object arrow = arrowVector.getAccessor().getObject(j);
-                Object json = jsonVector.getAccessor().getObject(j);
-                if (!equals(field.getType(), arrow, json)) {
-                    throw new IllegalArgumentException(
-                            "Different values in column:\n" + field + " at index " + j + ": " + arrow + " != " + json);
-                }
-            }
-        }
-    }
-
-    static boolean equals(ArrowType type, final Object o1, final Object o2) {
-        if (type instanceof ArrowType.FloatingPoint) {
-            ArrowType.FloatingPoint fpType = (ArrowType.FloatingPoint) type;
-            switch (fpType.getPrecision()) {
-                case DOUBLE:
-                    return equalEnough((Double)o1, (Double)o2);
-                case SINGLE:
-                    return equalEnough((Float)o1, (Float)o2);
-                case HALF:
-                default:
-                    throw new UnsupportedOperationException("unsupported precision: " + fpType);
-            }
-        } else if (type instanceof ArrowType.Binary) {
-            return Arrays.equals((byte[]) o1, (byte[]) o2);
-        }
-
-        return Objects.equal(o1, o2);
-    }
-
-    static boolean equalEnough(Float f1, Float f2) {
-        if (f1 == null || f2 == null) {
-            return f1 == null && f2 == null;
-        }
-        if (f1.isNaN()) {
-            return f2.isNaN();
-        }
-        if (f1.isInfinite()) {
-            return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
-        }
-        float average = Math.abs((f1 + f2) / 2);
-        float differenceScaled = Math.abs(f1 - f2) / (average == 0.0f ? 1f : average);
-        return differenceScaled < 1.0E-6f;
-    }
-
-    static boolean equalEnough(Double f1, Double f2) {
-        if (f1 == null || f2 == null) {
-            return f1 == null && f2 == null;
-        }
-        if (f1.isNaN()) {
-            return f2.isNaN();
-        }
-        if (f1.isInfinite()) {
-            return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
-        }
-        double average = Math.abs((f1 + f2) / 2);
-        double differenceScaled = Math.abs(f1 - f2) / (average == 0.0d ? 1d : average);
-        return differenceScaled < 1.0E-12d;
+  /**
+   * Validate two arrow schemas are equal.
+   *
+   * @param schema1 the 1st schema to compare
+   * @param schema2 the 2nd schema to compare
+   * @throws IllegalArgumentException if they are different.
+   */
+  public static void compareSchemas(Schema schema1, Schema schema2) {
+    if (!schema2.equals(schema1)) {
+      throw new IllegalArgumentException("Different schemas:\n" + schema2 + "\n" + schema1);
     }
+  }
+
+  /**
+   * Validate two Dictionary encodings and dictionaries with id's from the encodings
+   */
+  public static void compareDictionaries(List<DictionaryEncoding> encodings1, List<DictionaryEncoding> encodings2, DictionaryProvider provider1, DictionaryProvider provider2) {
+
+    if (encodings1.size() != encodings2.size()) {
+      throw new IllegalArgumentException("Different dictionary encoding count:\n" + encodings1.size() + "\n" + encodings2.size());
+    }
+
+    for (int i = 0; i < encodings1.size(); i++) {
+      if (!encodings1.get(i).equals(encodings2.get(i))) {
+        throw new IllegalArgumentException("Different dictionary encodings:\n" + encodings1.get(i) + "\n" + encodings2.get(i));
+      }
+
+      long id = encodings1.get(i).getId();
+      Dictionary dict1 = provider1.lookup(id);
+      Dictionary dict2 = provider2.lookup(id);
+
+      if (dict1 == null || dict2 == null) {
+        throw new IllegalArgumentException("The DictionaryProvider did not contain the required dictionary with id: " + id +"\n" + dict1 + "\n" + dict2);
+      }
+
+      try {
+        compareFieldVectors(dict1.getVector(), dict2.getVector());
+      } catch (IllegalArgumentException e) {
+        throw new IllegalArgumentException("Different dictionaries:\n" + dict1 + "\n" + dict2, e);
+      }
+    }
+  }
+
+  /**
+   * Validate two arrow vectorSchemaRoot are equal.
+   *
+   * @param root1 the 1st schema to compare
+   * @param root2 the 2nd schema to compare
+   * @throws IllegalArgumentException if they are different.
+   */
+  public static void compareVectorSchemaRoot(VectorSchemaRoot root1, VectorSchemaRoot root2) {
+    compareSchemas(root2.getSchema(), root1.getSchema());
+    if (root1.getRowCount() != root2.getRowCount()) {
+      throw new IllegalArgumentException("Different row count:\n" + root1.getRowCount() + " != " + root2.getRowCount());
+    }
+    List<FieldVector> vectors1 = root1.getFieldVectors();
+    List<FieldVector> vectors2 = root2.getFieldVectors();
+    if (vectors1.size() != vectors2.size()) {
+      throw new IllegalArgumentException("Different column count:\n" + vectors1.toString() + "\n!=\n" + vectors2.toString());
+    }
+    for (int i = 0; i < vectors1.size(); i++) {
+      compareFieldVectors(vectors1.get(i), vectors2.get(i));
+    }
+  }
+
+  /**
+   * Validate two arrow FieldVectors are equal.
+   *
+   * @param vector1 the 1st VectorField to compare
+   * @param vector2 the 2nd VectorField to compare
+   * @throws IllegalArgumentException if they are different
+   */
+  public static void compareFieldVectors(FieldVector vector1, FieldVector vector2) {
+    Field field1 = vector1.getField();
+    if (!field1.equals(vector2.getField())) {
+      throw new IllegalArgumentException("Different Fields:\n" + field1 + "\n!=\n" + vector2.getField());
+    }
+    int valueCount = vector1.getAccessor().getValueCount();
+    if (valueCount != vector2.getAccessor().getValueCount()) {
+      throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getAccessor().getValueCount());
+    }
+    for (int j = 0; j < valueCount; j++) {
+      Object obj1 = vector1.getAccessor().getObject(j);
+      Object obj2 = vector2.getAccessor().getObject(j);
+      if (!equals(field1.getType(), obj1, obj2)) {
+        throw new IllegalArgumentException(
+            "Different values in column:\n" + field1 + " at index " + j + ": " + obj1 + " != " + obj2);
+      }
+    }
+  }
+
+  static boolean equals(ArrowType type, final Object o1, final Object o2) {
+    if (type instanceof ArrowType.FloatingPoint) {
+      ArrowType.FloatingPoint fpType = (ArrowType.FloatingPoint) type;
+      switch (fpType.getPrecision()) {
+        case DOUBLE:
+          return equalEnough((Double)o1, (Double)o2);
+        case SINGLE:
+          return equalEnough((Float)o1, (Float)o2);
+        case HALF:
+        default:
+          throw new UnsupportedOperationException("unsupported precision: " + fpType);
+      }
+    } else if (type instanceof ArrowType.Binary) {
+      return Arrays.equals((byte[]) o1, (byte[]) o2);
+    }
+
+    return Objects.equal(o1, o2);
+  }
+
+  static boolean equalEnough(Float f1, Float f2) {
+    if (f1 == null || f2 == null) {
+      return f1 == null && f2 == null;
+    }
+    if (f1.isNaN()) {
+      return f2.isNaN();
+    }
+    if (f1.isInfinite()) {
+      return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
+    }
+    float average = Math.abs((f1 + f2) / 2);
+    float differenceScaled = Math.abs(f1 - f2) / (average == 0.0f ? 1f : average);
+    return differenceScaled < 1.0E-6f;
+  }
+
+  static boolean equalEnough(Double f1, Double f2) {
+    if (f1 == null || f2 == null) {
+      return f1 == null && f2 == null;
+    }
+    if (f1.isNaN()) {
+      return f2.isNaN();
+    }
+    if (f1.isInfinite()) {
+      return f2.isInfinite() && Math.signum(f1) == Math.signum(f2);
+    }
+    double average = Math.abs((f1 + f2) / 2);
+    double differenceScaled = Math.abs(f1 - f2) / (average == 0.0d ? 1d : average);
+    return differenceScaled < 1.0E-12d;
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 63027e6c4633b..3f717cbc18b6e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -17,18 +17,25 @@
  */
 package org.apache.arrow.vector.file;
 
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.NullableDateMilliVector;
+import org.apache.arrow.vector.NullableIntVector;
 import org.apache.arrow.vector.NullableTimeMilliVector;
+import org.apache.arrow.vector.NullableVarCharVector;
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
@@ -39,8 +46,17 @@
 import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
 import org.apache.arrow.vector.complex.writer.TimeStampMilliTZWriter;
 import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter;
+import org.apache.arrow.vector.dictionary.Dictionary;
+import org.apache.arrow.vector.dictionary.DictionaryEncoder;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.DateUtility;
+import org.apache.arrow.vector.util.DictionaryUtility;
+import org.apache.arrow.vector.util.Text;
 import org.joda.time.DateTimeZone;
 import org.joda.time.LocalDateTime;
 import org.junit.After;
@@ -51,6 +67,8 @@
 
 import io.netty.buffer.ArrowBuf;
 
+import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
+
 /**
  * Helps testing the file formats
  */
@@ -193,6 +211,226 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
     }
   }
 
+  protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) {
+
+    // Define dictionaries and add to provider
+    NullableVarCharVector dictionary1Vector = newNullableVarCharVector("D1", bufferAllocator);
+    dictionary1Vector.allocateNewSafe();
+    NullableVarCharVector.Mutator mutator = dictionary1Vector.getMutator();
+    mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+    mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+    mutator.set(2, "baz".getBytes(StandardCharsets.UTF_8));
+    mutator.setValueCount(3);
+
+    Dictionary dictionary1 = new Dictionary(dictionary1Vector, new DictionaryEncoding(1L, false, null));
+    provider.put(dictionary1);
+
+    NullableVarCharVector dictionary2Vector = newNullableVarCharVector("D2", bufferAllocator);
+    dictionary2Vector.allocateNewSafe();
+    mutator = dictionary2Vector.getMutator();
+    mutator.set(0, "micro".getBytes(StandardCharsets.UTF_8));
+    mutator.set(1, "small".getBytes(StandardCharsets.UTF_8));
+    mutator.set(2, "large".getBytes(StandardCharsets.UTF_8));
+    mutator.setValueCount(3);
+
+    Dictionary dictionary2 = new Dictionary(dictionary2Vector, new DictionaryEncoding(2L, false, null));
+    provider.put(dictionary2);
+
+    // Populate the vectors
+    NullableVarCharVector vector1A = newNullableVarCharVector("varcharA", bufferAllocator);
+    vector1A.allocateNewSafe();
+    mutator = vector1A.getMutator();
+    mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+    mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+    mutator.set(3, "baz".getBytes(StandardCharsets.UTF_8));
+    mutator.set(4, "bar".getBytes(StandardCharsets.UTF_8));
+    mutator.set(5, "baz".getBytes(StandardCharsets.UTF_8));
+    mutator.setValueCount(6);
+
+    FieldVector encodedVector1A = (FieldVector) DictionaryEncoder.encode(vector1A, dictionary1);
+    vector1A.close();  // Done with this vector after encoding
+
+    // Write this vector using indices instead of encoding
+    NullableIntVector encodedVector1B = new NullableIntVector("varcharB", bufferAllocator);
+    encodedVector1B.allocateNewSafe();
+    NullableIntVector.Mutator mutator1B = encodedVector1B.getMutator();
+    mutator1B.set(0, 2);  // "baz"
+    mutator1B.set(1, 1);  // "bar"
+    mutator1B.set(2, 2);  // "baz"
+    mutator1B.set(4, 1);  // "bar"
+    mutator1B.set(5, 0);  // "foo"
+    mutator1B.setValueCount(6);
+
+    NullableVarCharVector vector2 = newNullableVarCharVector("sizes", bufferAllocator);
+    vector2.allocateNewSafe();
+    mutator = vector2.getMutator();
+    mutator.set(1, "large".getBytes(StandardCharsets.UTF_8));
+    mutator.set(2, "small".getBytes(StandardCharsets.UTF_8));
+    mutator.set(3, "small".getBytes(StandardCharsets.UTF_8));
+    mutator.set(4, "large".getBytes(StandardCharsets.UTF_8));
+    mutator.setValueCount(6);
+
+    FieldVector encodedVector2 = (FieldVector) DictionaryEncoder.encode(vector2, dictionary2);
+    vector2.close();  // Done with this vector after encoding
+
+    List<Field> fields = ImmutableList.of(encodedVector1A.getField(), encodedVector1B.getField(), encodedVector2.getField());
+    List<FieldVector> vectors = ImmutableList.of(encodedVector1A, encodedVector1B, encodedVector2);
+
+    return new VectorSchemaRoot(fields, vectors, encodedVector1A.getAccessor().getValueCount());
+  }
+
+  protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider provider) {
+    FieldVector vector1A = root.getVector("varcharA");
+    Assert.assertNotNull(vector1A);
+
+    DictionaryEncoding encoding1A = vector1A.getField().getDictionary();
+    Assert.assertNotNull(encoding1A);
+    Assert.assertEquals(1L, encoding1A.getId());
+
+    FieldVector.Accessor accessor = vector1A.getAccessor();
+    Assert.assertEquals(6, accessor.getValueCount());
+    Assert.assertEquals(0, accessor.getObject(0));
+    Assert.assertEquals(1, accessor.getObject(1));
+    Assert.assertEquals(null, accessor.getObject(2));
+    Assert.assertEquals(2, accessor.getObject(3));
+    Assert.assertEquals(1, accessor.getObject(4));
+    Assert.assertEquals(2, accessor.getObject(5));
+
+    FieldVector vector1B = root.getVector("varcharB");
+    Assert.assertNotNull(vector1B);
+
+    DictionaryEncoding encoding1B = vector1A.getField().getDictionary();
+    Assert.assertNotNull(encoding1B);
+    Assert.assertTrue(encoding1A.equals(encoding1B));
+    Assert.assertEquals(1L, encoding1B.getId());
+
+    accessor = vector1B.getAccessor();
+    Assert.assertEquals(6, accessor.getValueCount());
+    Assert.assertEquals(2, accessor.getObject(0));
+    Assert.assertEquals(1, accessor.getObject(1));
+    Assert.assertEquals(2, accessor.getObject(2));
+    Assert.assertEquals(null, accessor.getObject(3));
+    Assert.assertEquals(1, accessor.getObject(4));
+    Assert.assertEquals(0, accessor.getObject(5));
+
+
+    FieldVector vector2 = root.getVector("sizes");
+    Assert.assertNotNull(vector2);
+
+    DictionaryEncoding encoding2 = vector2.getField().getDictionary();
+    Assert.assertNotNull(encoding2);
+    Assert.assertEquals(2L, encoding2.getId());
+
+    accessor = vector2.getAccessor();
+    Assert.assertEquals(6, accessor.getValueCount());
+    Assert.assertEquals(null, accessor.getObject(0));
+    Assert.assertEquals(2, accessor.getObject(1));
+    Assert.assertEquals(1, accessor.getObject(2));
+    Assert.assertEquals(1, accessor.getObject(3));
+    Assert.assertEquals(2, accessor.getObject(4));
+    Assert.assertEquals(null, accessor.getObject(5));
+
+    Dictionary dictionary1 = provider.lookup(1L);
+    Assert.assertNotNull(dictionary1);
+    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary1.getVector()).getAccessor();
+    Assert.assertEquals(3, dictionaryAccessor.getValueCount());
+    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
+    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+    Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2));
+
+    Dictionary dictionary2 = provider.lookup(2L);
+    Assert.assertNotNull(dictionary2);
+    dictionaryAccessor = ((NullableVarCharVector) dictionary2.getVector()).getAccessor();
+    Assert.assertEquals(3, dictionaryAccessor.getValueCount());
+    Assert.assertEquals(new Text("micro"), dictionaryAccessor.getObject(0));
+    Assert.assertEquals(new Text("small"), dictionaryAccessor.getObject(1));
+    Assert.assertEquals(new Text("large"), dictionaryAccessor.getObject(2));
+  }
+
+  protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) {
+
+    // Define the dictionary and add to the provider
+    NullableVarCharVector dictionaryVector = newNullableVarCharVector("D2", bufferAllocator);
+    dictionaryVector.allocateNewSafe();
+    dictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8));
+    dictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
+    dictionaryVector.getMutator().setValueCount(2);
+
+    Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(2L, false, null));
+    provider.put(dictionary);
+
+    // Write the vector data using dictionary indices
+    ListVector listVector = ListVector.empty("list", bufferAllocator);
+    DictionaryEncoding encoding = dictionary.getEncoding();
+    listVector.addOrGetVector(new FieldType(true, encoding.getIndexType(), encoding));
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    listWriter.startList();
+    listWriter.writeInt(0);
+    listWriter.writeInt(1);
+    listWriter.endList();
+    listWriter.startList();
+    listWriter.writeInt(0);
+    listWriter.endList();
+    listWriter.startList();
+    listWriter.writeInt(1);
+    listWriter.endList();
+    listWriter.setValueCount(3);
+
+    List<Field> fields = ImmutableList.of(listVector.getField());
+    List<FieldVector> vectors = ImmutableList.<FieldVector>of(listVector);
+    return new VectorSchemaRoot(fields, vectors, 3);
+  }
+
+  protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvider provider) {
+    FieldVector vector = root.getFieldVectors().get(0);
+    Assert.assertNotNull(vector);
+    Assert.assertNull(vector.getField().getDictionary());
+    Field nestedField = vector.getField().getChildren().get(0);
+
+    DictionaryEncoding encoding = nestedField.getDictionary();
+    Assert.assertNotNull(encoding);
+    Assert.assertEquals(2L, encoding.getId());
+    Assert.assertEquals(new ArrowType.Int(32, true), encoding.getIndexType());
+
+    FieldVector.Accessor accessor = vector.getAccessor();
+    Assert.assertEquals(3, accessor.getValueCount());
+    Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
+    Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
+    Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
+
+    Dictionary dictionary = provider.lookup(2L);
+    Assert.assertNotNull(dictionary);
+    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
+    Assert.assertEquals(2, dictionaryAccessor.getValueCount());
+    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
+    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+  }
+
+  protected void validateNestedDictionary(ListVector vector, DictionaryProvider provider) {
+    Assert.assertNotNull(vector);
+    Assert.assertNull(vector.getField().getDictionary());
+    Field nestedField = vector.getField().getChildren().get(0);
+
+    DictionaryEncoding encoding = nestedField.getDictionary();
+    Assert.assertNotNull(encoding);
+    Assert.assertEquals(2L, encoding.getId());
+    Assert.assertEquals(new ArrowType.Int(32, true), encoding.getIndexType());
+
+    ListVector.Accessor accessor = vector.getAccessor();
+    Assert.assertEquals(3, accessor.getValueCount());
+    Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
+    Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
+    Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
+
+    Dictionary dictionary = provider.lookup(2L);
+    Assert.assertNotNull(dictionary);
+    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
+    Assert.assertEquals(2, dictionaryAccessor.getValueCount());
+    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
+    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+  }
+
   protected void writeData(int count, MapVector parent) {
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 3353112d2b793..90fb5d252d68d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -17,8 +17,6 @@
  */
 package org.apache.arrow.vector.file;
 
-import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -26,9 +24,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -42,16 +38,10 @@
 import org.apache.arrow.vector.NullableFloat4Vector;
 import org.apache.arrow.vector.NullableIntVector;
 import org.apache.arrow.vector.NullableTinyIntVector;
-import org.apache.arrow.vector.NullableVarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
-import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
-import org.apache.arrow.vector.complex.impl.UnionListWriter;
-import org.apache.arrow.vector.dictionary.Dictionary;
-import org.apache.arrow.vector.dictionary.DictionaryEncoder;
-import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.schema.ArrowBuffer;
 import org.apache.arrow.vector.schema.ArrowMessage;
@@ -64,11 +54,9 @@
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
-import org.apache.arrow.vector.util.Text;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -469,36 +457,12 @@ public void testWriteReadDictionary() throws IOException {
     ByteArrayOutputStream stream = new ByteArrayOutputStream();
 
     // write
-    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableVarCharVector vector = newNullableVarCharVector("varchar", originalVectorAllocator);
-         NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", originalVectorAllocator)) {
-      vector.allocateNewSafe();
-      NullableVarCharVector.Mutator mutator = vector.getMutator();
-      mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
-      mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
-      mutator.set(3, "baz".getBytes(StandardCharsets.UTF_8));
-      mutator.set(4, "bar".getBytes(StandardCharsets.UTF_8));
-      mutator.set(5, "baz".getBytes(StandardCharsets.UTF_8));
-      mutator.setValueCount(6);
-
-      dictionaryVector.allocateNewSafe();
-      mutator = dictionaryVector.getMutator();
-      mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
-      mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
-      mutator.set(2, "baz".getBytes(StandardCharsets.UTF_8));
-      mutator.setValueCount(3);
-
-      Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null));
-      MapDictionaryProvider provider = new MapDictionaryProvider();
-      provider.put(dictionary);
-
-      FieldVector encodedVector = (FieldVector) DictionaryEncoder.encode(vector, dictionary);
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE)) {
 
-      List<Field> fields = ImmutableList.of(encodedVector.getField());
-      List<FieldVector> vectors = ImmutableList.of(encodedVector);
-      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 6);
+      MapDictionaryProvider provider = new MapDictionaryProvider();
 
-      try (FileOutputStream fileOutputStream = new FileOutputStream(file);
+      try (VectorSchemaRoot root = writeFlatDictionaryData(originalVectorAllocator, provider);
+           FileOutputStream fileOutputStream = new FileOutputStream(file);
            ArrowFileWriter fileWriter = new ArrowFileWriter(root, provider, fileOutputStream.getChannel());
            ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, provider, stream)) {
         LOGGER.debug("writing schema: " + root.getSchema());
@@ -510,8 +474,10 @@ public void testWriteReadDictionary() throws IOException {
         streamWriter.end();
       }
 
-      dictionaryVector.close();
-      encodedVector.close();
+      // Need to close dictionary vectors
+      for (long id: provider.getDictionaryIds()) {
+        provider.lookup(id).getVector().close();
+      }
     }
 
     // read from file
@@ -522,7 +488,7 @@ public void testWriteReadDictionary() throws IOException {
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
       Assert.assertTrue(arrowReader.loadNextBatch());
-      validateFlatDictionary(root.getFieldVectors().get(0), arrowReader);
+      validateFlatDictionary(root, arrowReader);
     }
 
     // Read from stream
@@ -533,81 +499,31 @@ public void testWriteReadDictionary() throws IOException {
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
       Assert.assertTrue(arrowReader.loadNextBatch());
-      validateFlatDictionary(root.getFieldVectors().get(0), arrowReader);
+      validateFlatDictionary(root, arrowReader);
     }
   }
 
-  private void validateFlatDictionary(FieldVector vector, DictionaryProvider provider) {
-    Assert.assertNotNull(vector);
-
-    DictionaryEncoding encoding = vector.getField().getDictionary();
-    Assert.assertNotNull(encoding);
-    Assert.assertEquals(1L, encoding.getId());
-
-    FieldVector.Accessor accessor = vector.getAccessor();
-    Assert.assertEquals(6, accessor.getValueCount());
-    Assert.assertEquals(0, accessor.getObject(0));
-    Assert.assertEquals(1, accessor.getObject(1));
-    Assert.assertEquals(null, accessor.getObject(2));
-    Assert.assertEquals(2, accessor.getObject(3));
-    Assert.assertEquals(1, accessor.getObject(4));
-    Assert.assertEquals(2, accessor.getObject(5));
-
-    Dictionary dictionary = provider.lookup(1L);
-    Assert.assertNotNull(dictionary);
-    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
-    Assert.assertEquals(3, dictionaryAccessor.getValueCount());
-    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
-    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
-    Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2));
-  }
-
   @Test
   public void testWriteReadNestedDictionary() throws IOException {
     File file = new File("target/mytest_dict_nested.arrow");
     ByteArrayOutputStream stream = new ByteArrayOutputStream();
 
-    DictionaryEncoding encoding = new DictionaryEncoding(2L, false, null);
-
     // data being written:
     // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
 
     // write
-    try (NullableVarCharVector dictionaryVector = newNullableVarCharVector("dictionary", allocator);
-         ListVector listVector = ListVector.empty("list", allocator)) {
-
-      Dictionary dictionary = new Dictionary(dictionaryVector, encoding);
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE)
+    ) {
       MapDictionaryProvider provider = new MapDictionaryProvider();
-      provider.put(dictionary);
-
-      dictionaryVector.allocateNew();
-      dictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8));
-      dictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
-      dictionaryVector.getMutator().setValueCount(2);
-
-      listVector.addOrGetVector(new FieldType(true, new Int(32, true), encoding));
-      listVector.allocateNew();
-      UnionListWriter listWriter = new UnionListWriter(listVector);
-      listWriter.startList();
-      listWriter.writeInt(0);
-      listWriter.writeInt(1);
-      listWriter.endList();
-      listWriter.startList();
-      listWriter.writeInt(0);
-      listWriter.endList();
-      listWriter.startList();
-      listWriter.writeInt(1);
-      listWriter.endList();
-      listWriter.setValueCount(3);
-
-      List<Field> fields = ImmutableList.of(listVector.getField());
-      List<FieldVector> vectors = ImmutableList.<FieldVector>of(listVector);
-      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 3);
-
-      try (
+
+      try (VectorSchemaRoot root = writeNestedDictionaryData(vectorAllocator, provider);
            FileOutputStream fileOutputStream = new FileOutputStream(file);
            ArrowFileWriter fileWriter = new ArrowFileWriter(root, provider, fileOutputStream.getChannel());
            ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, provider, stream)) {
+
+        validateNestedDictionary(root, provider);
+
         LOGGER.debug("writing schema: " + root.getSchema());
         fileWriter.start();
         streamWriter.start();
@@ -616,6 +532,11 @@ public void testWriteReadNestedDictionary() throws IOException {
         fileWriter.end();
         streamWriter.end();
       }
+
+      // Need to close dictionary vectors
+      for (long id: provider.getDictionaryIds()) {
+        provider.lookup(id).getVector().close();
+      }
     }
 
     // read from file
@@ -626,7 +547,7 @@ public void testWriteReadNestedDictionary() throws IOException {
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
       Assert.assertTrue(arrowReader.loadNextBatch());
-      validateNestedDictionary((ListVector) root.getFieldVectors().get(0), arrowReader);
+      validateNestedDictionary(root, arrowReader);
     }
 
     // Read from stream
@@ -637,34 +558,10 @@ public void testWriteReadNestedDictionary() throws IOException {
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
       Assert.assertTrue(arrowReader.loadNextBatch());
-      validateNestedDictionary((ListVector) root.getFieldVectors().get(0), arrowReader);
+      validateNestedDictionary(root, arrowReader);
     }
   }
 
-  private void validateNestedDictionary(ListVector vector, DictionaryProvider provider) {
-    Assert.assertNotNull(vector);
-    Assert.assertNull(vector.getField().getDictionary());
-    Field nestedField = vector.getField().getChildren().get(0);
-
-    DictionaryEncoding encoding = nestedField.getDictionary();
-    Assert.assertNotNull(encoding);
-    Assert.assertEquals(2L, encoding.getId());
-    Assert.assertEquals(new Int(32, true), encoding.getIndexType());
-
-    ListVector.Accessor accessor = vector.getAccessor();
-    Assert.assertEquals(3, accessor.getValueCount());
-    Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
-    Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
-    Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
-
-    Dictionary dictionary = provider.lookup(2L);
-    Assert.assertNotNull(dictionary);
-    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
-    Assert.assertEquals(2, dictionaryAccessor.getValueCount());
-    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
-    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
-  }
-
   @Test
   public void testWriteReadFixedSizeList() throws IOException {
     File file = new File("target/mytest_fixed_list.arrow");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 6c29cab1a0723..311cada194eaf 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -25,6 +25,8 @@
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.file.BaseFileTest;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Assert;
@@ -45,7 +47,7 @@ public void testWriteReadComplexJSON() throws IOException {
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
       writeComplexData(count, parent);
-      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")));
+      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
     }
 
     // read
@@ -74,13 +76,13 @@ public void testWriteComplexJSON() throws IOException {
       writeComplexData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateComplexContent(root.getRowCount(), root);
-      writeJSON(file, root);
+      writeJSON(file, root, null);
     }
   }
 
-  public void writeJSON(File file, VectorSchemaRoot root) throws IOException {
+  public void writeJSON(File file, VectorSchemaRoot root, DictionaryProvider provider) throws IOException {
     JsonFileWriter writer = new JsonFileWriter(file, JsonFileWriter.config().pretty(true));
-    writer.start(root.getSchema());
+    writer.start(root.getSchema(), provider);
     writer.write(root);
     writer.close();
   }
@@ -101,7 +103,7 @@ public void testWriteReadUnionJSON() throws IOException {
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateUnionData(count, root);
 
-      writeJSON(file, root);
+      writeJSON(file, root, null);
     }
     // read
     try (
@@ -136,7 +138,7 @@ public void testWriteReadDateTimeJSON() throws IOException {
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateDateTimeContent(count, root);
 
-      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")));
+      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
     }
 
     // read
@@ -155,6 +157,85 @@ public void testWriteReadDateTimeJSON() throws IOException {
     }
   }
 
+  @Test
+  public void testWriteReadDictionaryJSON() throws IOException {
+    File file = new File("target/mytest_dictionary.json");
+
+    // write
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE)
+    ) {
+      MapDictionaryProvider provider = new MapDictionaryProvider();
+
+      try (VectorSchemaRoot root = writeFlatDictionaryData(vectorAllocator, provider)) {
+        printVectors(root.getFieldVectors());
+        validateFlatDictionary(root, provider);
+        writeJSON(file, root, provider);
+      }
+
+      // Need to close dictionary vectors
+      for (long id: provider.getDictionaryIds()) {
+        provider.lookup(id).getVector().close();
+      }
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+    ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateFlatDictionary(root, reader);
+      }
+      reader.close();
+    }
+  }
+
+  @Test
+  public void testWriteReadNestedDictionaryJSON() throws IOException {
+    File file = new File("target/mytest_dict_nested.json");
+
+    // data being written:
+    // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
+
+    // write
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE)
+    ) {
+      MapDictionaryProvider provider = new MapDictionaryProvider();
+
+      try (VectorSchemaRoot root = writeNestedDictionaryData(vectorAllocator, provider)) {
+        printVectors(root.getFieldVectors());
+        validateNestedDictionary(root, provider);
+        writeJSON(file, root, provider);
+      }
+
+      // Need to close dictionary vectors
+      for (long id: provider.getDictionaryIds()) {
+        provider.lookup(id).getVector().close();
+      }
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+    ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateNestedDictionary(root, reader);
+      }
+      reader.close();
+    }
+  }
+
   @Test
   public void testSetStructLength() throws IOException {
     File file = new File("../../integration/data/struct_example.json");

From 00a7d55cc06ab54af860be38fea701169b8f8aa3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 5 Jul 2017 17:46:41 -0400
Subject: [PATCH 0780/1644] ARROW-1185: [C++] Status class cleanup,
 warn_unused_result attribute and Clang warning fixes

This was tedious, but overdue. The Status class in Arrow as originally imported from Apache Kudu, which had been modified from standard use in Google projects. I simplified the implementation to bring it more in line with the Status implementation used in TensorFlow.

This also addresses ARROW-111 by providing an attribute to warn in Clang if a Status is ignored

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #814 from wesm/status-cleaning and squashes the following commits:

7b7e6517 [Wes McKinney] Bring Status implementation somewhat more in line with TensorFlow and other Google codebases, remove unused posix code. Add warn_unused_result attribute and fix clang warnings
---
 LICENSE.txt                                   |  18 ++
 cpp/src/arrow/array-decimal-test.cc           |  24 +-
 cpp/src/arrow/array-test.cc                   | 249 ++++++++++--------
 cpp/src/arrow/builder-benchmark.cc            |   6 -
 cpp/src/arrow/builder.cc                      |   2 +-
 cpp/src/arrow/column-benchmark.cc             |  13 +-
 cpp/src/arrow/io/file.cc                      |   7 +-
 cpp/src/arrow/io/hdfs.cc                      |   5 +-
 cpp/src/arrow/io/io-file-test.cc              |  16 +-
 cpp/src/arrow/io/io-hdfs-test.cc              |   7 +-
 cpp/src/arrow/io/io-memory-benchmark.cc       |  12 +-
 cpp/src/arrow/io/io-memory-test.cc            |   6 +-
 cpp/src/arrow/io/memory.cc                    |   2 +-
 cpp/src/arrow/ipc/feather-test.cc             |  20 +-
 cpp/src/arrow/ipc/feather.cc                  |   5 +-
 cpp/src/arrow/ipc/ipc-json-test.cc            |   4 +-
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc |  14 +-
 cpp/src/arrow/ipc/json-internal.cc            |  10 +-
 cpp/src/arrow/ipc/test-common.h               |  14 +-
 cpp/src/arrow/pretty_print-test.cc            |   8 +-
 cpp/src/arrow/python/builtin_convert.cc       |  13 +-
 cpp/src/arrow/python/pandas_convert.cc        |  10 +-
 cpp/src/arrow/python/python-test.cc           |   2 +-
 cpp/src/arrow/status.cc                       |  38 ++-
 cpp/src/arrow/status.h                        |  77 +++---
 cpp/src/arrow/test-util.h                     |  14 +-
 cpp/src/arrow/type-test.cc                    |   2 +-
 cpp/src/arrow/util/bit-util.cc                |   4 +-
 cpp/src/arrow/util/bit-util.h                 |   2 +-
 cpp/src/arrow/util/decimal-test.cc            |   6 +-
 cpp/src/arrow/util/decimal.h                  |   2 +-
 cpp/src/arrow/util/macros.h                   |   8 +
 cpp/src/plasma/store.cc                       |   6 +-
 33 files changed, 328 insertions(+), 298 deletions(-)

diff --git a/LICENSE.txt b/LICENSE.txt
index 55823cb4374bd..34879f61dc80f 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -341,3 +341,21 @@ Copyright: 2013 Daniel Lemire
 Home page: http://lemire.me/en/
 Project page: https://github.com/lemire/FrameOfReference
 License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from the TensorFlow project
+
+Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 2a3d0a905acc6..0959d686498d5 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -41,25 +41,27 @@ class DecimalTestBase {
     size_t null_count = 0;
 
     size_t size = draw.size();
-    builder->Reserve(size);
+    ASSERT_OK(builder->Reserve(size));
 
     for (size_t i = 0; i < size; ++i) {
       if (valid_bytes[i]) {
-        builder->Append(draw[i]);
+        ASSERT_OK(builder->Append(draw[i]));
       } else {
-        builder->AppendNull();
+        ASSERT_OK(builder->AppendNull());
         ++null_count;
       }
     }
 
     std::shared_ptr<Buffer> expected_sign_bitmap;
     if (!sign_bitmap.empty()) {
-      BitUtil::BytesToBits(sign_bitmap, &expected_sign_bitmap);
+      ASSERT_OK(BitUtil::BytesToBits(sign_bitmap, &expected_sign_bitmap));
     }
 
     auto raw_bytes = data(draw, byte_width);
     auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), size * byte_width);
-    auto expected_null_bitmap = test::bytes_to_null_buffer(valid_bytes);
+    std::shared_ptr<Buffer> expected_null_bitmap;
+    ASSERT_OK(BitUtil::BytesToBits(valid_bytes, &expected_null_bitmap));
+
     int64_t expected_null_count = test::null_count(valid_bytes);
     auto expected = std::make_shared<DecimalArray>(type, size, expected_data,
         expected_null_bitmap, expected_null_count, offset, expected_sign_bitmap);
@@ -170,14 +172,14 @@ TEST_P(Decimal128BuilderTest, WithNulls) {
 }
 
 INSTANTIATE_TEST_CASE_P(Decimal32BuilderTest, Decimal32BuilderTest,
-    ::testing::Range(DecimalPrecision<int32_t>::minimum,
-                            DecimalPrecision<int32_t>::maximum));
+    ::testing::Range(
+        DecimalPrecision<int32_t>::minimum, DecimalPrecision<int32_t>::maximum));
 INSTANTIATE_TEST_CASE_P(Decimal64BuilderTest, Decimal64BuilderTest,
-    ::testing::Range(DecimalPrecision<int64_t>::minimum,
-                            DecimalPrecision<int64_t>::maximum));
+    ::testing::Range(
+        DecimalPrecision<int64_t>::minimum, DecimalPrecision<int64_t>::maximum));
 INSTANTIATE_TEST_CASE_P(Decimal128BuilderTest, Decimal128BuilderTest,
-    ::testing::Range(DecimalPrecision<int128_t>::minimum,
-                            DecimalPrecision<int128_t>::maximum));
+    ::testing::Range(
+        DecimalPrecision<int128_t>::minimum, DecimalPrecision<int128_t>::maximum));
 
 }  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 76d6bbed33fca..64672d4eed804 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -64,25 +64,29 @@ TEST_F(TestArray, TestLength) {
   ASSERT_EQ(arr->length(), 100);
 }
 
-std::shared_ptr<Array> MakeArrayFromValidBytes(
-    const vector<uint8_t>& v, MemoryPool* pool) {
+Status MakeArrayFromValidBytes(
+    const vector<uint8_t>& v, MemoryPool* pool, std::shared_ptr<Array>* out) {
   int64_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
-  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(v);
+
+  std::shared_ptr<Buffer> null_buf;
+  RETURN_NOT_OK(BitUtil::BytesToBits(v, &null_buf));
 
   BufferBuilder value_builder(pool);
   for (size_t i = 0; i < v.size(); ++i) {
-    value_builder.Append<int32_t>(0);
+    RETURN_NOT_OK(value_builder.Append<int32_t>(0));
   }
 
-  std::shared_ptr<Array> arr(
-      new Int32Array(v.size(), value_builder.Finish(), null_buf, null_count));
-  return arr;
+  *out = std::make_shared<Int32Array>(
+      v.size(), value_builder.Finish(), null_buf, null_count);
+  return Status::OK();
 }
 
 TEST_F(TestArray, TestEquality) {
-  auto array = MakeArrayFromValidBytes({1, 0, 1, 1, 0, 1, 0, 0}, pool_);
-  auto equal_array = MakeArrayFromValidBytes({1, 0, 1, 1, 0, 1, 0, 0}, pool_);
-  auto unequal_array = MakeArrayFromValidBytes({1, 1, 1, 1, 0, 1, 0, 0}, pool_);
+  std::shared_ptr<Array> array, equal_array, unequal_array;
+
+  ASSERT_OK(MakeArrayFromValidBytes({1, 0, 1, 1, 0, 1, 0, 0}, pool_, &array));
+  ASSERT_OK(MakeArrayFromValidBytes({1, 0, 1, 1, 0, 1, 0, 0}, pool_, &equal_array));
+  ASSERT_OK(MakeArrayFromValidBytes({1, 1, 1, 1, 0, 1, 0, 0}, pool_, &unequal_array));
 
   EXPECT_TRUE(array->Equals(array));
   EXPECT_TRUE(array->Equals(equal_array));
@@ -108,7 +112,8 @@ TEST_F(TestArray, TestNullArrayEquality) {
 TEST_F(TestArray, SliceRecomputeNullCount) {
   vector<uint8_t> valid_bytes = {1, 0, 1, 1, 0, 1, 0, 0, 0};
 
-  auto array = MakeArrayFromValidBytes(valid_bytes, pool_);
+  std::shared_ptr<Array> array;
+  ASSERT_OK(MakeArrayFromValidBytes(valid_bytes, pool_, &array));
 
   ASSERT_EQ(5, array->null_count());
 
@@ -144,7 +149,9 @@ TEST_F(TestArray, TestIsNull) {
     if (x == 0) { ++null_count; }
   }
 
-  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(null_bitmap);
+  std::shared_ptr<Buffer> null_buf;
+  ASSERT_OK(BitUtil::BytesToBits(null_bitmap, &null_buf));
+
   std::unique_ptr<Array> arr;
   arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_buf, null_count));
 
@@ -177,10 +184,10 @@ TEST_F(TestArray, TestCopy) {}
 // Primitive type tests
 
 TEST_F(TestBuilder, TestReserve) {
-  builder_->Init(10);
+  ASSERT_OK(builder_->Init(10));
   ASSERT_EQ(2, builder_->null_bitmap()->size());
 
-  builder_->Reserve(30);
+  ASSERT_OK(builder_->Reserve(30));
   ASSERT_EQ(4, builder_->null_bitmap()->size());
 }
 
@@ -222,7 +229,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     int64_t ex_null_count = 0;
 
     if (nullable) {
-      ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
+      ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &ex_null_bitmap));
       ex_null_count = test::null_count(valid_bytes_);
     } else {
       ex_null_bitmap = nullptr;
@@ -312,13 +319,14 @@ void TestPrimitiveBuilder<PBoolean>::Check(
     const std::unique_ptr<BooleanBuilder>& builder, bool nullable) {
   int64_t size = builder->length();
 
-  auto ex_data = test::bytes_to_null_buffer(draws_);
+  std::shared_ptr<Buffer> ex_data;
+  ASSERT_OK(BitUtil::BytesToBits(draws_, &ex_data));
 
   std::shared_ptr<Buffer> ex_null_bitmap;
   int64_t ex_null_count = 0;
 
   if (nullable) {
-    ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
+    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &ex_null_bitmap));
     ex_null_count = test::null_count(valid_bytes_);
   } else {
     ex_null_bitmap = nullptr;
@@ -399,15 +407,14 @@ TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
   int64_t memory_before = this->pool_->bytes_allocated();
 
   this->RandomData(size);
-
-  this->builder_->Reserve(size);
+  ASSERT_OK(this->builder_->Reserve(size));
 
   int64_t i;
   for (i = 0; i < size; ++i) {
     if (valid_bytes[i] > 0) {
-      this->builder_->Append(draws[i]);
+      ASSERT_OK(this->builder_->Append(draws[i]));
     } else {
-      this->builder_->AppendNull();
+      ASSERT_OK(this->builder_->AppendNull());
     }
   }
 
@@ -499,19 +506,19 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
 
   this->RandomData(size);
 
-  this->builder_->Reserve(1000);
-  this->builder_nn_->Reserve(1000);
+  ASSERT_OK(this->builder_->Reserve(1000));
+  ASSERT_OK(this->builder_nn_->Reserve(1000));
 
   int64_t null_count = 0;
   // Append the first 1000
   for (size_t i = 0; i < 1000; ++i) {
     if (valid_bytes[i] > 0) {
-      this->builder_->Append(draws[i]);
+      ASSERT_OK(this->builder_->Append(draws[i]));
     } else {
-      this->builder_->AppendNull();
+      ASSERT_OK(this->builder_->AppendNull());
       ++null_count;
     }
-    this->builder_nn_->Append(draws[i]);
+    ASSERT_OK(this->builder_nn_->Append(draws[i]));
   }
 
   ASSERT_EQ(null_count, this->builder_->null_count());
@@ -522,17 +529,17 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   ASSERT_EQ(1000, this->builder_nn_->length());
   ASSERT_EQ(1024, this->builder_nn_->capacity());
 
-  this->builder_->Reserve(size - 1000);
-  this->builder_nn_->Reserve(size - 1000);
+  ASSERT_OK(this->builder_->Reserve(size - 1000));
+  ASSERT_OK(this->builder_nn_->Reserve(size - 1000));
 
   // Append the next 9000
   for (size_t i = 1000; i < size; ++i) {
     if (valid_bytes[i] > 0) {
-      this->builder_->Append(draws[i]);
+      ASSERT_OK(this->builder_->Append(draws[i]));
     } else {
-      this->builder_->AppendNull();
+      ASSERT_OK(this->builder_->AppendNull());
     }
-    this->builder_nn_->Append(draws[i]);
+    ASSERT_OK(this->builder_nn_->Append(draws[i]));
   }
 
   ASSERT_EQ(size, this->builder_->length());
@@ -668,7 +675,7 @@ class TestStringArray : public ::testing::Test {
     length_ = static_cast<int64_t>(offsets_.size()) - 1;
     value_buf_ = test::GetBufferFromVector(chars_);
     offsets_buf_ = test::GetBufferFromVector(offsets_);
-    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
+    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_));
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<StringArray>(
@@ -746,19 +753,19 @@ TEST_F(TestStringArray, CompareNullByteSlots) {
   StringBuilder builder2(default_memory_pool());
   StringBuilder builder3(default_memory_pool());
 
-  builder.Append("foo");
-  builder2.Append("foo");
-  builder3.Append("foo");
+  ASSERT_OK(builder.Append("foo"));
+  ASSERT_OK(builder2.Append("foo"));
+  ASSERT_OK(builder3.Append("foo"));
 
-  builder.Append("bar");
-  builder2.AppendNull();
+  ASSERT_OK(builder.Append("bar"));
+  ASSERT_OK(builder2.AppendNull());
 
   // same length, but different
-  builder3.Append("xyz");
+  ASSERT_OK(builder3.Append("xyz"));
 
-  builder.Append("baz");
-  builder2.Append("baz");
-  builder3.Append("baz");
+  ASSERT_OK(builder.Append("baz"));
+  ASSERT_OK(builder2.Append("baz"));
+  ASSERT_OK(builder3.Append("baz"));
 
   std::shared_ptr<Array> array, array2, array3;
   ASSERT_OK(builder.Finish(&array));
@@ -785,9 +792,9 @@ TEST_F(TestStringArray, CompareNullByteSlots) {
 TEST_F(TestStringArray, TestSliceGetString) {
   StringBuilder builder(default_memory_pool());
 
-  builder.Append("a");
-  builder.Append("b");
-  builder.Append("c");
+  ASSERT_OK(builder.Append("a"));
+  ASSERT_OK(builder.Append("b"));
+  ASSERT_OK(builder.Append("c"));
 
   std::shared_ptr<Array> array;
   ASSERT_OK(builder.Finish(&array));
@@ -829,9 +836,9 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
   for (int j = 0; j < reps; ++j) {
     for (int i = 0; i < N; ++i) {
       if (is_null[i]) {
-        builder_->AppendNull();
+        ASSERT_OK(builder_->AppendNull());
       } else {
-        builder_->Append(strings[i]);
+        ASSERT_OK(builder_->Append(strings[i]));
       }
     }
   }
@@ -882,7 +889,7 @@ class TestBinaryArray : public ::testing::Test {
     value_buf_ = test::GetBufferFromVector(chars_);
     offsets_buf_ = test::GetBufferFromVector(offsets_);
 
-    null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
+    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_));
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<BinaryArray>(
@@ -949,12 +956,9 @@ TEST_F(TestBinaryArray, TestEqualsEmptyStrings) {
   BinaryBuilder builder(default_memory_pool(), arrow::binary());
 
   string empty_string("");
-
-  builder.Append(empty_string);
-  builder.Append(empty_string);
-  builder.Append(empty_string);
-  builder.Append(empty_string);
-  builder.Append(empty_string);
+  for (int i = 0; i < 5; ++i) {
+    ASSERT_OK(builder.Append(empty_string));
+  }
 
   std::shared_ptr<Array> left_arr;
   ASSERT_OK(builder.Finish(&left_arr));
@@ -997,9 +1001,9 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
   for (int j = 0; j < reps; ++j) {
     for (int i = 0; i < N; ++i) {
       if (is_null[i]) {
-        builder_->AppendNull();
+        ASSERT_OK(builder_->AppendNull());
       } else {
-        builder_->Append(strings[i]);
+        ASSERT_OK(builder_->Append(strings[i]));
       }
     }
   }
@@ -1046,9 +1050,9 @@ void CheckSliceEquality() {
   for (int j = 0; j < reps; ++j) {
     for (int i = 0; i < N; ++i) {
       if (is_null[i]) {
-        builder.AppendNull();
+        ASSERT_OK(builder.AppendNull());
       } else {
-        builder.Append(strings[i]);
+        ASSERT_OK(builder.Append(strings[i]));
       }
     }
   }
@@ -1142,9 +1146,9 @@ TEST_F(TestFWBinaryArray, Builder) {
   InitBuilder(byte_width);
   for (int64_t i = 0; i < length; ++i) {
     if (is_valid[i]) {
-      builder_->Append(raw_data + byte_width * i);
+      ASSERT_OK(builder_->Append(raw_data + byte_width * i));
     } else {
-      builder_->AppendNull();
+      ASSERT_OK(builder_->AppendNull());
     }
   }
 
@@ -1165,10 +1169,10 @@ TEST_F(TestFWBinaryArray, Builder) {
   InitBuilder(byte_width);
   for (int64_t i = 0; i < length; ++i) {
     if (is_valid[i]) {
-      builder_->Append(
-          string(reinterpret_cast<const char*>(raw_data + byte_width * i), byte_width));
+      ASSERT_OK(builder_->Append(
+          string(reinterpret_cast<const char*>(raw_data + byte_width * i), byte_width)));
     } else {
-      builder_->AppendNull();
+      ASSERT_OK(builder_->AppendNull());
     }
   }
 
@@ -1236,9 +1240,9 @@ TEST_F(TestFWBinaryArray, Slice) {
 
   for (int i = 0; i < 5; ++i) {
     if (is_null[i]) {
-      builder.AppendNull();
+      ASSERT_OK(builder.AppendNull());
     } else {
-      builder.Append(strings[i]);
+      ASSERT_OK(builder.Append(strings[i]));
     }
   }
 
@@ -1287,9 +1291,9 @@ class TestAdaptiveIntBuilder : public TestBuilder {
 };
 
 TEST_F(TestAdaptiveIntBuilder, TestInt8) {
-  builder_->Append(0);
-  builder_->Append(127);
-  builder_->Append(-128);
+  ASSERT_OK(builder_->Append(0));
+  ASSERT_OK(builder_->Append(127));
+  ASSERT_OK(builder_->Append(-128));
 
   Done();
 
@@ -1299,8 +1303,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt8) {
 }
 
 TEST_F(TestAdaptiveIntBuilder, TestInt16) {
-  builder_->Append(0);
-  builder_->Append(128);
+  ASSERT_OK(builder_->Append(0));
+  ASSERT_OK(builder_->Append(128));
   Done();
 
   std::vector<int16_t> expected_values({0, 128});
@@ -1308,7 +1312,7 @@ TEST_F(TestAdaptiveIntBuilder, TestInt16) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(-129);
+  ASSERT_OK(builder_->Append(-129));
   expected_values = {-129};
   Done();
 
@@ -1316,8 +1320,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt16) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(std::numeric_limits<int16_t>::max());
-  builder_->Append(std::numeric_limits<int16_t>::min());
+  ASSERT_OK(builder_->Append(std::numeric_limits<int16_t>::max()));
+  ASSERT_OK(builder_->Append(std::numeric_limits<int16_t>::min()));
   expected_values = {
       std::numeric_limits<int16_t>::max(), std::numeric_limits<int16_t>::min()};
   Done();
@@ -1327,8 +1331,9 @@ TEST_F(TestAdaptiveIntBuilder, TestInt16) {
 }
 
 TEST_F(TestAdaptiveIntBuilder, TestInt32) {
-  builder_->Append(0);
-  builder_->Append(static_cast<int64_t>(std::numeric_limits<int16_t>::max()) + 1);
+  ASSERT_OK(builder_->Append(0));
+  ASSERT_OK(
+      builder_->Append(static_cast<int64_t>(std::numeric_limits<int16_t>::max()) + 1));
   Done();
 
   std::vector<int32_t> expected_values(
@@ -1337,7 +1342,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt32) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(static_cast<int64_t>(std::numeric_limits<int16_t>::min()) - 1);
+  ASSERT_OK(
+      builder_->Append(static_cast<int64_t>(std::numeric_limits<int16_t>::min()) - 1));
   expected_values = {static_cast<int32_t>(std::numeric_limits<int16_t>::min()) - 1};
   Done();
 
@@ -1345,8 +1351,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt32) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(std::numeric_limits<int32_t>::max());
-  builder_->Append(std::numeric_limits<int32_t>::min());
+  ASSERT_OK(builder_->Append(std::numeric_limits<int32_t>::max()));
+  ASSERT_OK(builder_->Append(std::numeric_limits<int32_t>::min()));
   expected_values = {
       std::numeric_limits<int32_t>::max(), std::numeric_limits<int32_t>::min()};
   Done();
@@ -1356,8 +1362,9 @@ TEST_F(TestAdaptiveIntBuilder, TestInt32) {
 }
 
 TEST_F(TestAdaptiveIntBuilder, TestInt64) {
-  builder_->Append(0);
-  builder_->Append(static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1);
+  ASSERT_OK(builder_->Append(0));
+  ASSERT_OK(
+      builder_->Append(static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1));
   Done();
 
   std::vector<int64_t> expected_values(
@@ -1366,7 +1373,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt64) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(static_cast<int64_t>(std::numeric_limits<int32_t>::min()) - 1);
+  ASSERT_OK(
+      builder_->Append(static_cast<int64_t>(std::numeric_limits<int32_t>::min()) - 1));
   expected_values = {static_cast<int64_t>(std::numeric_limits<int32_t>::min()) - 1};
   Done();
 
@@ -1374,8 +1382,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt64) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(std::numeric_limits<int64_t>::max());
-  builder_->Append(std::numeric_limits<int64_t>::min());
+  ASSERT_OK(builder_->Append(std::numeric_limits<int64_t>::max()));
+  ASSERT_OK(builder_->Append(std::numeric_limits<int64_t>::min()));
   expected_values = {
       std::numeric_limits<int64_t>::max(), std::numeric_limits<int64_t>::min()};
   Done();
@@ -1387,7 +1395,7 @@ TEST_F(TestAdaptiveIntBuilder, TestInt64) {
 TEST_F(TestAdaptiveIntBuilder, TestAppendVector) {
   std::vector<int64_t> expected_values(
       {0, static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1});
-  builder_->Append(expected_values.data(), expected_values.size());
+  ASSERT_OK(builder_->Append(expected_values.data(), expected_values.size()));
   Done();
 
   ArrayFromVector<Int64Type, int64_t>(expected_values, &expected_);
@@ -1411,8 +1419,8 @@ class TestAdaptiveUIntBuilder : public TestBuilder {
 };
 
 TEST_F(TestAdaptiveUIntBuilder, TestUInt8) {
-  builder_->Append(0);
-  builder_->Append(255);
+  ASSERT_OK(builder_->Append(0));
+  ASSERT_OK(builder_->Append(255));
 
   Done();
 
@@ -1422,8 +1430,8 @@ TEST_F(TestAdaptiveUIntBuilder, TestUInt8) {
 }
 
 TEST_F(TestAdaptiveUIntBuilder, TestUInt16) {
-  builder_->Append(0);
-  builder_->Append(256);
+  ASSERT_OK(builder_->Append(0));
+  ASSERT_OK(builder_->Append(256));
   Done();
 
   std::vector<uint16_t> expected_values({0, 256});
@@ -1431,7 +1439,7 @@ TEST_F(TestAdaptiveUIntBuilder, TestUInt16) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(std::numeric_limits<uint16_t>::max());
+  ASSERT_OK(builder_->Append(std::numeric_limits<uint16_t>::max()));
   expected_values = {std::numeric_limits<uint16_t>::max()};
   Done();
 
@@ -1440,8 +1448,9 @@ TEST_F(TestAdaptiveUIntBuilder, TestUInt16) {
 }
 
 TEST_F(TestAdaptiveUIntBuilder, TestUInt32) {
-  builder_->Append(0);
-  builder_->Append(static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 1);
+  ASSERT_OK(builder_->Append(0));
+  ASSERT_OK(
+      builder_->Append(static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) + 1));
   Done();
 
   std::vector<uint32_t> expected_values(
@@ -1450,7 +1459,7 @@ TEST_F(TestAdaptiveUIntBuilder, TestUInt32) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(std::numeric_limits<uint32_t>::max());
+  ASSERT_OK(builder_->Append(std::numeric_limits<uint32_t>::max()));
   expected_values = {std::numeric_limits<uint32_t>::max()};
   Done();
 
@@ -1459,8 +1468,9 @@ TEST_F(TestAdaptiveUIntBuilder, TestUInt32) {
 }
 
 TEST_F(TestAdaptiveUIntBuilder, TestUInt64) {
-  builder_->Append(0);
-  builder_->Append(static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
+  ASSERT_OK(builder_->Append(0));
+  ASSERT_OK(
+      builder_->Append(static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1));
   Done();
 
   std::vector<uint64_t> expected_values(
@@ -1469,7 +1479,7 @@ TEST_F(TestAdaptiveUIntBuilder, TestUInt64) {
   ASSERT_TRUE(expected_->Equals(result_));
 
   SetUp();
-  builder_->Append(std::numeric_limits<uint64_t>::max());
+  ASSERT_OK(builder_->Append(std::numeric_limits<uint64_t>::max()));
   expected_values = {std::numeric_limits<uint64_t>::max()};
   Done();
 
@@ -1480,7 +1490,7 @@ TEST_F(TestAdaptiveUIntBuilder, TestUInt64) {
 TEST_F(TestAdaptiveUIntBuilder, TestAppendVector) {
   std::vector<uint64_t> expected_values(
       {0, static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1});
-  builder_->Append(expected_values.data(), expected_values.size());
+  ASSERT_OK(builder_->Append(expected_values.data(), expected_values.size()));
   Done();
 
   ArrayFromVector<UInt64Type, uint64_t>(expected_values, &expected_);
@@ -1779,7 +1789,7 @@ TEST_F(TestListBuilder, TestBasics) {
   for (size_t i = 0; i < lengths.size(); ++i) {
     ASSERT_OK(builder_->Append(is_valid[i] > 0));
     for (int j = 0; j < lengths[i]; ++j) {
-      vb->Append(values[pos++]);
+      ASSERT_OK(vb->Append(values[pos++]));
     }
   }
 
@@ -1796,9 +1806,9 @@ TEST_F(TestListBuilder, BulkAppend) {
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
   ASSERT_OK(vb->Reserve(values.size()));
 
-  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+  ASSERT_OK(builder_->Append(offsets.data(), offsets.size(), is_valid.data()));
   for (int32_t value : values) {
-    vb->Append(value);
+    ASSERT_OK(vb->Append(value));
   }
   Done();
   ValidateBasicListArray(result_.get(), values, is_valid);
@@ -1814,10 +1824,10 @@ TEST_F(TestListBuilder, BulkAppendInvalid) {
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
   ASSERT_OK(vb->Reserve(values.size()));
 
-  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
-  builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+  ASSERT_OK(builder_->Append(offsets.data(), offsets.size(), is_valid.data()));
+  ASSERT_OK(builder_->Append(offsets.data(), offsets.size(), is_valid.data()));
   for (int32_t value : values) {
-    vb->Append(value);
+    ASSERT_OK(vb->Append(value));
   }
 
   Done();
@@ -2109,9 +2119,10 @@ TEST_F(TestStructBuilder, BulkAppend) {
   ASSERT_OK(char_vb->Resize(list_values.size()));
   ASSERT_OK(int_vb->Resize(int_values.size()));
 
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  ASSERT_OK(builder_->Append(struct_is_valid.size(), struct_is_valid.data()));
 
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  ASSERT_OK(
+      list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data()));
   for (int8_t value : list_values) {
     char_vb->UnsafeAppend(value);
   }
@@ -2140,9 +2151,10 @@ TEST_F(TestStructBuilder, BulkAppendInvalid) {
   ASSERT_OK(char_vb->Reserve(list_values.size()));
   ASSERT_OK(int_vb->Reserve(int_values.size()));
 
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+  ASSERT_OK(builder_->Append(struct_is_valid.size(), struct_is_valid.data()));
 
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  ASSERT_OK(
+      list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data()));
   for (int8_t value : list_values) {
     char_vb->UnsafeAppend(value);
   }
@@ -2181,8 +2193,9 @@ TEST_F(TestStructBuilder, TestEquality) {
   ASSERT_OK(int_vb->Reserve(int_values.size()));
 
   // setup two equal arrays, one of which takes an unequal bitmap
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  ASSERT_OK(builder_->Append(struct_is_valid.size(), struct_is_valid.data()));
+  ASSERT_OK(
+      list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data()));
   for (int8_t value : list_values) {
     char_vb->UnsafeAppend(value);
   }
@@ -2196,8 +2209,9 @@ TEST_F(TestStructBuilder, TestEquality) {
   ASSERT_OK(char_vb->Resize(list_values.size()));
   ASSERT_OK(int_vb->Resize(int_values.size()));
 
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  ASSERT_OK(builder_->Append(struct_is_valid.size(), struct_is_valid.data()));
+  ASSERT_OK(
+      list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data()));
   for (int8_t value : list_values) {
     char_vb->UnsafeAppend(value);
   }
@@ -2212,8 +2226,10 @@ TEST_F(TestStructBuilder, TestEquality) {
   ASSERT_OK(int_vb->Resize(int_values.size()));
 
   // setup an unequal one with the unequal bitmap
-  builder_->Append(unequal_struct_is_valid.size(), unequal_struct_is_valid.data());
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  ASSERT_OK(
+      builder_->Append(unequal_struct_is_valid.size(), unequal_struct_is_valid.data()));
+  ASSERT_OK(
+      list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data()));
   for (int8_t value : list_values) {
     char_vb->UnsafeAppend(value);
   }
@@ -2228,9 +2244,9 @@ TEST_F(TestStructBuilder, TestEquality) {
   ASSERT_OK(int_vb->Resize(int_values.size()));
 
   // setup an unequal one with unequal offsets
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-  list_vb->Append(unequal_list_offsets.data(), unequal_list_offsets.size(),
-      unequal_list_is_valid.data());
+  ASSERT_OK(builder_->Append(struct_is_valid.size(), struct_is_valid.data()));
+  ASSERT_OK(list_vb->Append(unequal_list_offsets.data(), unequal_list_offsets.size(),
+      unequal_list_is_valid.data()));
   for (int8_t value : list_values) {
     char_vb->UnsafeAppend(value);
   }
@@ -2245,8 +2261,9 @@ TEST_F(TestStructBuilder, TestEquality) {
   ASSERT_OK(int_vb->Resize(int_values.size()));
 
   // setup anunequal one with unequal values
-  builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-  list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+  ASSERT_OK(builder_->Append(struct_is_valid.size(), struct_is_valid.data()));
+  ASSERT_OK(
+      list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data()));
   for (int8_t value : unequal_list_values) {
     char_vb->UnsafeAppend(value);
   }
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index 5eae9ab5c917f..7ca7bb4999801 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -25,12 +25,6 @@ namespace arrow {
 
 constexpr int64_t kFinalSize = 256;
 
-#define ABORT_NOT_OK(s)                              \
-  do {                                               \
-    ::arrow::Status _s = (s);                        \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) { exit(-1); } \
-  } while (0);
-
 static void BM_BuildPrimitiveArrayNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
   // 2 MiB block
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index f3ba2dddb9158..16f252c3d6c84 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -897,7 +897,7 @@ ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal128& value) {
 Status DecimalBuilder::Init(int64_t capacity) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Init(capacity));
   if (byte_width_ == 16) {
-    AllocateResizableBuffer(pool_, null_bitmap_->size(), &sign_bitmap_);
+    RETURN_NOT_OK(AllocateResizableBuffer(pool_, null_bitmap_->size(), &sign_bitmap_));
     sign_bitmap_data_ = sign_bitmap_->mutable_data();
     memset(sign_bitmap_data_, 0, static_cast<size_t>(sign_bitmap_->capacity()));
   }
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index 13076a4788689..e50ddf6d7032d 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -24,13 +24,14 @@
 namespace arrow {
 namespace {
 template <typename ArrayType>
-std::shared_ptr<Array> MakePrimitive(int64_t length, int64_t null_count = 0) {
+Status MakePrimitive(int64_t length, int64_t null_count, std::shared_ptr<Array>* out) {
   auto pool = default_memory_pool();
   auto data = std::make_shared<PoolBuffer>(pool);
   auto null_bitmap = std::make_shared<PoolBuffer>(pool);
-  data->Resize(length * sizeof(typename ArrayType::value_type));
-  null_bitmap->Resize(BitUtil::BytesForBits(length));
-  return std::make_shared<ArrayType>(length, data, null_bitmap, 10);
+  RETURN_NOT_OK(data->Resize(length * sizeof(typename ArrayType::value_type)));
+  RETURN_NOT_OK(null_bitmap->Resize(BitUtil::BytesForBits(length)));
+  *out = std::make_shared<ArrayType>(length, data, null_bitmap, null_count);
+  return Status::OK();
 }
 }  // anonymous namespace
 
@@ -38,7 +39,9 @@ static void BM_BuildInt32ColumnByChunk(
     benchmark::State& state) {  // NOLINT non-const reference
   ArrayVector arrays;
   for (int chunk_n = 0; chunk_n < state.range(0); ++chunk_n) {
-    arrays.push_back(MakePrimitive<Int32Array>(100, 10));
+    std::shared_ptr<Array> array;
+    ABORT_NOT_OK(MakePrimitive<Int32Array>(100, 10, &array));
+    arrays.push_back(array);
   }
   const auto INT32 = std::make_shared<Int32Type>();
   const auto field = std::make_shared<Field>("c0", INT32);
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index b2b880159fb6d..936655f26dbda 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -111,6 +111,7 @@
 #include "arrow/buffer.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace io {
@@ -434,7 +435,7 @@ ReadableFile::ReadableFile(MemoryPool* pool) {
 }
 
 ReadableFile::~ReadableFile() {
-  impl_->Close();
+  DCHECK(impl_->Close().ok());
 }
 
 Status ReadableFile::Open(const std::string& path, std::shared_ptr<ReadableFile>* file) {
@@ -497,7 +498,7 @@ FileOutputStream::FileOutputStream() {
 
 FileOutputStream::~FileOutputStream() {
   // This can fail; better to explicitly call close
-  impl_->Close();
+  DCHECK(impl_->Close().ok());
 }
 
 Status FileOutputStream::Open(
@@ -538,7 +539,7 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
   ~MemoryMap() {
     if (file_->is_open()) {
       munmap(mutable_data_, static_cast<size_t>(size_));
-      file_->Close();
+      DCHECK(file_->Close().ok());
     }
   }
 
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index ba9c2c207a013..9ded9bc3f9902 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -27,6 +27,7 @@
 #include "arrow/io/hdfs.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace io {
@@ -191,7 +192,7 @@ HdfsReadableFile::HdfsReadableFile(MemoryPool* pool) {
 }
 
 HdfsReadableFile::~HdfsReadableFile() {
-  impl_->Close();
+  DCHECK(impl_->Close().ok());
 }
 
 Status HdfsReadableFile::Close() {
@@ -271,7 +272,7 @@ HdfsOutputStream::HdfsOutputStream() {
 }
 
 HdfsOutputStream::~HdfsOutputStream() {
-  impl_->Close();
+  DCHECK(impl_->Close().ok());
 }
 
 Status HdfsOutputStream::Close() {
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 6550c74662298..a077f8cb921c7 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -125,12 +125,12 @@ TEST_F(TestFileOutputStream, Close) {
   ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), strlen(data)));
 
   int fd = file_->file_descriptor();
-  file_->Close();
+  ASSERT_OK(file_->Close());
 
   ASSERT_TRUE(FileIsClosed(fd));
 
   // Idempotent
-  file_->Close();
+  ASSERT_OK(file_->Close());
 
   std::shared_ptr<ReadableFile> rd_file;
   ASSERT_OK(ReadableFile::Open(path_, &rd_file));
@@ -215,12 +215,12 @@ TEST_F(TestReadableFile, Close) {
   OpenFile();
 
   int fd = file_->file_descriptor();
-  file_->Close();
+  ASSERT_OK(file_->Close());
 
   ASSERT_TRUE(FileIsClosed(fd));
 
   // Idempotent
-  file_->Close();
+  ASSERT_OK(file_->Close());
 }
 
 TEST_F(TestReadableFile, SeekTellSize) {
@@ -446,7 +446,7 @@ TEST_F(TestMemoryMappedFile, ReadOnly) {
     ASSERT_OK(rwmmap->Write(buffer.data(), buffer_size));
     position += buffer_size;
   }
-  rwmmap->Close();
+  ASSERT_OK(rwmmap->Close());
 
   std::shared_ptr<MemoryMappedFile> rommap;
   ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
@@ -459,7 +459,7 @@ TEST_F(TestMemoryMappedFile, ReadOnly) {
     ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
     position += buffer_size;
   }
-  rommap->Close();
+  ASSERT_OK(rommap->Close());
 }
 
 TEST_F(TestMemoryMappedFile, DISABLED_ReadWriteOver4GbFile) {
@@ -481,7 +481,7 @@ TEST_F(TestMemoryMappedFile, DISABLED_ReadWriteOver4GbFile) {
     ASSERT_OK(rwmmap->Write(buffer.data(), buffer_size));
     position += buffer_size;
   }
-  rwmmap->Close();
+  ASSERT_OK(rwmmap->Close());
 
   std::shared_ptr<MemoryMappedFile> rommap;
   ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
@@ -494,7 +494,7 @@ TEST_F(TestMemoryMappedFile, DISABLED_ReadWriteOver4GbFile) {
     ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
     position += buffer_size;
   }
-  rommap->Close();
+  ASSERT_OK(rommap->Close());
 }
 
 TEST_F(TestMemoryMappedFile, RetainMemoryMapReference) {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index b8203f0daa1a4..74f80428c4561 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -87,10 +87,9 @@ class TestHdfsClient : public ::testing::Test {
     LibHdfsShim* driver_shim;
 
     client_ = nullptr;
-    scratch_dir_ =
-        boost::filesystem::unique_path(
-            boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
-            .string();
+    scratch_dir_ = boost::filesystem::unique_path(
+        boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
+                       .string();
 
     loaded_driver_ = false;
 
diff --git a/cpp/src/arrow/io/io-memory-benchmark.cc b/cpp/src/arrow/io/io-memory-benchmark.cc
index 6aa9577f0a1d9..45cf427759525 100644
--- a/cpp/src/arrow/io/io-memory-benchmark.cc
+++ b/cpp/src/arrow/io/io-memory-benchmark.cc
@@ -29,15 +29,15 @@ static void BM_SerialMemcopy(benchmark::State& state) {  // NOLINT non-const ref
   constexpr int64_t kTotalSize = 100 * 1024 * 1024;      // 100MB
 
   auto buffer1 = std::make_shared<PoolBuffer>(default_memory_pool());
-  buffer1->Resize(kTotalSize);
+  ABORT_NOT_OK(buffer1->Resize(kTotalSize));
 
   auto buffer2 = std::make_shared<PoolBuffer>(default_memory_pool());
-  buffer2->Resize(kTotalSize);
+  ABORT_NOT_OK(buffer2->Resize(kTotalSize));
   test::random_bytes(kTotalSize, 0, buffer2->mutable_data());
 
   while (state.KeepRunning()) {
     io::FixedSizeBufferWriter writer(buffer1);
-    writer.Write(buffer2->data(), buffer2->size());
+    ABORT_NOT_OK(writer.Write(buffer2->data(), buffer2->size()));
   }
   state.SetBytesProcessed(int64_t(state.iterations()) * kTotalSize);
 }
@@ -46,16 +46,16 @@ static void BM_ParallelMemcopy(benchmark::State& state) {  // NOLINT non-const r
   constexpr int64_t kTotalSize = 100 * 1024 * 1024;        // 100MB
 
   auto buffer1 = std::make_shared<PoolBuffer>(default_memory_pool());
-  buffer1->Resize(kTotalSize);
+  ABORT_NOT_OK(buffer1->Resize(kTotalSize));
 
   auto buffer2 = std::make_shared<PoolBuffer>(default_memory_pool());
-  buffer2->Resize(kTotalSize);
+  ABORT_NOT_OK(buffer2->Resize(kTotalSize));
   test::random_bytes(kTotalSize, 0, buffer2->mutable_data());
 
   while (state.KeepRunning()) {
     io::FixedSizeBufferWriter writer(buffer1);
     writer.set_memcopy_threads(4);
-    writer.Write(buffer2->data(), buffer2->size());
+    ABORT_NOT_OK(writer.Write(buffer2->data(), buffer2->size()));
   }
   state.SetBytesProcessed(int64_t(state.iterations()) * kTotalSize);
 }
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 33249cb27f200..75365df6b189d 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -121,16 +121,16 @@ TEST(TestMemcopy, ParallelMemcopy) {
     int64_t total_size = 3 * 1024 * 1024 + std::rand() % 100;
 
     auto buffer1 = std::make_shared<PoolBuffer>(default_memory_pool());
-    buffer1->Resize(total_size);
+    ASSERT_OK(buffer1->Resize(total_size));
 
     auto buffer2 = std::make_shared<PoolBuffer>(default_memory_pool());
-    buffer2->Resize(total_size);
+    ASSERT_OK(buffer2->Resize(total_size));
     test::random_bytes(total_size, 0, buffer2->mutable_data());
 
     io::FixedSizeBufferWriter writer(buffer1);
     writer.set_memcopy_threads(4);
     writer.set_memcopy_threshold(1024 * 1024);
-    writer.Write(buffer2->data(), buffer2->size());
+    ASSERT_OK(writer.Write(buffer2->data(), buffer2->size()));
 
     ASSERT_EQ(0, memcmp(buffer1->data(), buffer2->data(), buffer1->size()));
   }
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index faf02d2fa22f7..22721f1dd6871 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -55,7 +55,7 @@ Status BufferOutputStream::Create(int64_t initial_capacity, MemoryPool* pool,
 
 BufferOutputStream::~BufferOutputStream() {
   // This can fail, better to explicitly call close
-  if (buffer_) { Close(); }
+  if (buffer_) { DCHECK(Close().ok()); }
 }
 
 Status BufferOutputStream::Close() {
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index fb26df6e130f2..807ea4e746270 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -50,7 +50,7 @@ class TestTableBuilder : public ::testing::Test {
   void SetUp() { tb_.reset(new TableBuilder(1000)); }
 
   virtual void Finish() {
-    tb_->Finish();
+    ASSERT_OK(tb_->Finish());
 
     table_.reset(new TableMetadata());
     ASSERT_OK(table_->Open(tb_->GetBuffer()));
@@ -107,7 +107,7 @@ TEST_F(TestTableBuilder, AddPrimitiveColumn) {
   std::string user_meta = "as you wish";
   cb->SetUserMetadata(user_meta);
 
-  cb->Finish();
+  ASSERT_OK(cb->Finish());
 
   cb = tb_->AddColumn("f1");
 
@@ -118,7 +118,7 @@ TEST_F(TestTableBuilder, AddPrimitiveColumn) {
   values2.total_bytes = 10000;
 
   cb->SetValues(values2);
-  cb->Finish();
+  ASSERT_OK(cb->Finish());
 
   Finish();
 
@@ -148,12 +148,12 @@ TEST_F(TestTableBuilder, AddCategoryColumn) {
   std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("c0");
   cb->SetValues(values1);
   cb->SetCategory(levels);
-  cb->Finish();
+  ASSERT_OK(cb->Finish());
 
   cb = tb_->AddColumn("c1");
   cb->SetValues(values1);
   cb->SetCategory(levels, true);
-  cb->Finish();
+  ASSERT_OK(cb->Finish());
 
   Finish();
 
@@ -182,7 +182,7 @@ TEST_F(TestTableBuilder, AddTimestampColumn) {
   std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("c0");
   cb->SetValues(values1);
   cb->SetTimestamp(TimeUnit::MILLI);
-  cb->Finish();
+  ASSERT_OK(cb->Finish());
 
   cb = tb_->AddColumn("c1");
 
@@ -190,7 +190,7 @@ TEST_F(TestTableBuilder, AddTimestampColumn) {
 
   cb->SetValues(values1);
   cb->SetTimestamp(TimeUnit::SECOND, tz);
-  cb->Finish();
+  ASSERT_OK(cb->Finish());
 
   Finish();
 
@@ -216,7 +216,7 @@ TEST_F(TestTableBuilder, AddDateColumn) {
   std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("d0");
   cb->SetValues(values1);
   cb->SetDate();
-  cb->Finish();
+  ASSERT_OK(cb->Finish());
 
   Finish();
 
@@ -233,7 +233,7 @@ TEST_F(TestTableBuilder, AddTimeColumn) {
   std::unique_ptr<ColumnBuilder> cb = tb_->AddColumn("c0");
   cb->SetValues(values1);
   cb->SetTime(TimeUnit::SECOND);
-  cb->Finish();
+  ASSERT_OK(cb->Finish());
   Finish();
 
   auto col = table_->column(0);
@@ -379,7 +379,7 @@ TEST_F(TestTableWriter, TimeTypes) {
 
   for (int i = 1; i < schema->num_fields(); ++i) {
     std::shared_ptr<Array> arr;
-    LoadArray(schema->field(i)->type(), fields, buffers, &arr);
+    ASSERT_OK(LoadArray(schema->field(i)->type(), fields, buffers, &arr));
     arrays.push_back(arr);
   }
 
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 023e7ec6e98b7..7f577a09b8a46 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -498,7 +498,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
 
   Status Finalize() {
     RETURN_NOT_OK(CheckStarted());
-    metadata_.Finish();
+    RETURN_NOT_OK(metadata_.Finish());
 
     auto buffer = metadata_.GetBuffer();
 
@@ -655,8 +655,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
   Status Append(const std::string& name, const Array& values) {
     current_column_ = metadata_.AddColumn(name);
     RETURN_NOT_OK(values.Accept(this));
-    current_column_->Finish();
-    return Status::OK();
+    return current_column_->Finish();
   }
 
  private:
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index e06af9f965e6f..92971468964c3 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -224,10 +224,10 @@ void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
   StringBuilder string_builder(default_memory_pool());
   for (int i = 0; i < num_rows; ++i) {
     if (!is_valid[i]) {
-      string_builder.AppendNull();
+      ASSERT_OK(string_builder.AppendNull());
     } else {
       test::random_ascii(kBufferSize, seed++, buffer);
-      string_builder.Append(buffer, kBufferSize);
+      ASSERT_OK(string_builder.Append(buffer, kBufferSize));
     }
   }
   std::shared_ptr<Array> v3;
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
index 1aecdbc633190..c890d829849fd 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -46,13 +46,13 @@ std::shared_ptr<RecordBatch> MakeRecordBatch(int64_t total_size, int64_t num_fie
   typename TypeTraits<TYPE>::BuilderType builder(pool, type);
   for (size_t i = 0; i < values.size(); ++i) {
     if (is_valid[i]) {
-      builder.Append(values[i]);
+      ABORT_NOT_OK(builder.Append(values[i]));
     } else {
-      builder.AppendNull();
+      ABORT_NOT_OK(builder.AppendNull());
     }
   }
   std::shared_ptr<Array> array;
-  builder.Finish(&array);
+  ABORT_NOT_OK(builder.Finish(&array));
 
   ArrayVector arrays;
   std::vector<std::shared_ptr<Field>> fields;
@@ -72,7 +72,7 @@ static void BM_WriteRecordBatch(benchmark::State& state) {  // NOLINT non-const
   constexpr int64_t kTotalSize = 1 << 20;
 
   auto buffer = std::make_shared<PoolBuffer>(default_memory_pool());
-  buffer->Resize(kTotalSize & 2);
+  ABORT_NOT_OK(buffer->Resize(kTotalSize & 2));
   auto record_batch = MakeRecordBatch<Int64Type>(kTotalSize, state.range(0));
 
   while (state.KeepRunning()) {
@@ -80,7 +80,7 @@ static void BM_WriteRecordBatch(benchmark::State& state) {  // NOLINT non-const
     int32_t metadata_length;
     int64_t body_length;
     if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
-             default_memory_pool())
+            default_memory_pool())
              .ok()) {
       state.SkipWithError("Failed to write!");
     }
@@ -93,7 +93,7 @@ static void BM_ReadRecordBatch(benchmark::State& state) {  // NOLINT non-const r
   constexpr int64_t kTotalSize = 1 << 20;
 
   auto buffer = std::make_shared<PoolBuffer>(default_memory_pool());
-  buffer->Resize(kTotalSize & 2);
+  ABORT_NOT_OK(buffer->Resize(kTotalSize & 2));
   auto record_batch = MakeRecordBatch<Int64Type>(kTotalSize, state.range(0));
 
   io::BufferOutputStream stream(buffer);
@@ -101,7 +101,7 @@ static void BM_ReadRecordBatch(benchmark::State& state) {  // NOLINT non-const r
   int32_t metadata_length;
   int64_t body_length;
   if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
-           default_memory_pool())
+          default_memory_pool())
            .ok()) {
     state.SkipWithError("Failed to write!");
   }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index bf2c194e4fb5a..beebb4f0e2808 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -974,12 +974,12 @@ class ArrayReader {
     DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
     for (int i = 0; i < length_; ++i) {
       if (!is_valid_[i]) {
-        builder.AppendNull();
+        RETURN_NOT_OK(builder.AppendNull());
         continue;
       }
 
       const rj::Value& val = json_data_arr[i];
-      builder.Append(UnboxValue<T>(val));
+      RETURN_NOT_OK(builder.Append(UnboxValue<T>(val)));
     }
 
     return builder.Finish(&result_);
@@ -1000,14 +1000,14 @@ class ArrayReader {
     auto byte_buffer = std::make_shared<PoolBuffer>(pool_);
     for (int i = 0; i < length_; ++i) {
       if (!is_valid_[i]) {
-        builder.AppendNull();
+        RETURN_NOT_OK(builder.AppendNull());
         continue;
       }
 
       const rj::Value& val = json_data_arr[i];
       DCHECK(val.IsString());
       if (std::is_base_of<StringType, T>::value) {
-        builder.Append(val.GetString());
+        RETURN_NOT_OK(builder.Append(val.GetString()));
       } else {
         std::string hex_string = val.GetString();
 
@@ -1048,7 +1048,7 @@ class ArrayReader {
 
     for (int i = 0; i < length_; ++i) {
       if (!is_valid_[i]) {
-        builder.AppendNull();
+        RETURN_NOT_OK(builder.AppendNull());
         continue;
       }
 
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index deaeb597f9095..747aca0873df9 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -96,11 +96,11 @@ const auto kListListInt32 = list(kListInt32);
 Status MakeRandomInt32Array(
     int64_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
   std::shared_ptr<PoolBuffer> data;
-  test::MakeRandomInt32PoolBuffer(length, pool, &data);
+  RETURN_NOT_OK(test::MakeRandomInt32PoolBuffer(length, pool, &data));
   Int32Builder builder(pool, int32());
   if (include_nulls) {
     std::shared_ptr<PoolBuffer> valid_bytes;
-    test::MakeRandomBytePoolBuffer(length, pool, &valid_bytes);
+    RETURN_NOT_OK(test::MakeRandomBytePoolBuffer(length, pool, &valid_bytes));
     RETURN_NOT_OK(builder.Append(
         reinterpret_cast<const int32_t*>(data->data()), length, valid_bytes->data()));
     return builder.Finish(out);
@@ -149,11 +149,13 @@ Status MakeRandomBooleanArray(
     const int length, bool include_nulls, std::shared_ptr<Array>* out) {
   std::vector<uint8_t> values(length);
   test::random_null_bytes(length, 0.5, values.data());
-  auto data = test::bytes_to_null_buffer(values);
+  std::shared_ptr<Buffer> data;
+  RETURN_NOT_OK(BitUtil::BytesToBits(values, &data));
 
   if (include_nulls) {
     std::vector<uint8_t> valid_bytes(length);
-    auto null_bitmap = test::bytes_to_null_buffer(valid_bytes);
+    std::shared_ptr<Buffer> null_bitmap;
+    RETURN_NOT_OK(BitUtil::BytesToBits(valid_bytes, &null_bitmap));
     test::random_null_bytes(length, 0.1, valid_bytes.data());
     *out = std::make_shared<BooleanArray>(length, data, null_bitmap, -1);
   } else {
@@ -611,9 +613,9 @@ void AppendValues(const std::vector<bool>& is_valid, const std::vector<T>& value
     BuilderType* builder) {
   for (size_t i = 0; i < values.size(); ++i) {
     if (is_valid[i]) {
-      builder->Append(values[i]);
+      ASSERT_OK(builder->Append(values[i]));
     } else {
-      builder->AppendNull();
+      ASSERT_OK(builder->AppendNull());
     }
   }
 }
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index 80cd9cfe6ac6d..72bd0a8045580 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -87,10 +87,10 @@ TEST_F(TestPrettyPrint, FixedSizeBinaryType) {
   auto type = fixed_size_binary(3);
   FixedSizeBinaryBuilder builder(default_memory_pool(), type);
 
-  builder.Append(values[0]);
-  builder.Append(values[1]);
-  builder.Append(values[2]);
-  builder.Finish(&array);
+  ASSERT_OK(builder.Append(values[0]));
+  ASSERT_OK(builder.Append(values[1]));
+  ASSERT_OK(builder.Append(values[2]));
+  ASSERT_OK(builder.Finish(&array));
 
   CheckArray(*array, 0, ex);
 }
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index c7af5679e3879..11114b0230ab1 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -464,9 +464,8 @@ class FixedWidthBytesConverter
   inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
-    Py_ssize_t expected_length =
-        std::dynamic_pointer_cast<FixedSizeBinaryType>(typed_builder_->type())
-            ->byte_width();
+    Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedSizeBinaryType>(
+        typed_builder_->type())->byte_width();
     if (item.obj() == Py_None) {
       RETURN_NOT_OK(typed_builder_->AppendNull());
       return Status::OK();
@@ -518,7 +517,7 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
     if (item.obj() == Py_None) {
       return typed_builder_->AppendNull();
     } else {
-      typed_builder_->Append();
+      RETURN_NOT_OK(typed_builder_->Append());
       PyObject* item_obj = item.obj();
       int64_t list_size = static_cast<int64_t>(PySequence_Size(item_obj));
       return value_converter_->AppendData(item_obj, list_size);
@@ -607,8 +606,7 @@ Status ListConverter::Init(ArrayBuilder* builder) {
     return Status::NotImplemented("value type not implemented");
   }
 
-  value_converter_->Init(typed_builder_->value_builder());
-  return Status::OK();
+  return value_converter_->Init(typed_builder_->value_builder());
 }
 
 Status AppendPySequence(PyObject* obj, int64_t size,
@@ -620,8 +618,7 @@ Status AppendPySequence(PyObject* obj, int64_t size,
     ss << "No type converter implemented for " << type->ToString();
     return Status::NotImplemented(ss.str());
   }
-  converter->Init(builder);
-
+  RETURN_NOT_OK(converter->Init(builder));
   return converter->AppendData(obj, size);
 }
 
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 0858d06a0efde..f75a2ba921b0d 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -537,9 +537,9 @@ Status PandasConverter::ConvertDates() {
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
     if (PyDate_CheckExact(obj)) {
-      date_builder.Append(UnboxDate<ArrowType>::Unbox(obj));
+      RETURN_NOT_OK(date_builder.Append(UnboxDate<ArrowType>::Unbox(obj)));
     } else if (PandasObjectIsNull(obj)) {
-      date_builder.AppendNull();
+      RETURN_NOT_OK(date_builder.AppendNull());
     } else {
       return InvalidConversion(obj, "date");
     }
@@ -592,7 +592,7 @@ Status PandasConverter::ConvertDecimals() {
           break;
       }
     } else if (PandasObjectIsNull(object)) {
-      decimal_builder.AppendNull();
+      RETURN_NOT_OK(decimal_builder.AppendNull());
     } else {
       return InvalidConversion(object, "decimal.Decimal");
     }
@@ -939,7 +939,7 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
       int64_t size = PyArray_DIM(numpy_array, 0);
       auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
       if (traits::supports_nulls) {
-        null_bitmap_->Resize(size, false);
+        RETURN_NOT_OK(null_bitmap_->Resize(size, false));
         // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
         // currently support this.
         // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
@@ -2423,7 +2423,7 @@ class ArrowDeserializer {
   }
 
   Status Visit(const StructType& type) {
-    AllocateOutput(NPY_OBJECT);
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     return ConvertStruct(data_, out_values);
   }
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 43c7f60aaf36b..592a5e6de56c9 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -77,7 +77,7 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   const char value[] = {'\xf1', '\0'};
 
   for (int i = 0; i < 1000; ++i) {
-    builder.Append(value, static_cast<int32_t>(strlen(value)));
+    ASSERT_OK(builder.Append(value, static_cast<int32_t>(strlen(value))));
   }
 
   std::shared_ptr<Array> arr;
diff --git a/cpp/src/arrow/status.cc b/cpp/src/arrow/status.cc
index 3a39c8409a5f7..77ecb9357b73c 100644
--- a/cpp/src/arrow/status.cc
+++ b/cpp/src/arrow/status.cc
@@ -16,23 +16,25 @@
 
 namespace arrow {
 
-Status::Status(StatusCode code, const std::string& msg, int16_t posix_code) {
+Status::Status(StatusCode code, const std::string& msg) {
   assert(code != StatusCode::OK);
-  const uint32_t size = static_cast<uint32_t>(msg.size());
-  char* result = new char[size + 7];
-  memcpy(result, &size, sizeof(size));
-  result[4] = static_cast<char>(code);
-  memcpy(result + 5, &posix_code, sizeof(posix_code));
-  memcpy(result + 7, msg.c_str(), msg.size());
-  state_ = result;
+  state_ = new State;
+  state_->code = code;
+  state_->msg = msg;
 }
 
-const char* Status::CopyState(const char* state) {
-  uint32_t size;
-  memcpy(&size, state, sizeof(size));
-  char* result = new char[size + 7];
-  memcpy(result, state, size + 7);
-  return result;
+void Status::CopyFrom(const State* state) {
+  delete state_;
+  if (state == nullptr) {
+    state_ = nullptr;
+  } else {
+    state_ = new State(*state);
+  }
+}
+
+std::ostream& operator<<(std::ostream& os, const Status& x) {
+  os << x.ToString();
+  return os;
 }
 
 std::string Status::CodeAsString() const {
@@ -74,12 +76,8 @@ std::string Status::CodeAsString() const {
 std::string Status::ToString() const {
   std::string result(CodeAsString());
   if (state_ == NULL) { return result; }
-
-  result.append(": ");
-
-  uint32_t length;
-  memcpy(&length, state_, sizeof(length));
-  result.append(reinterpret_cast<const char*>(state_ + 7), length);
+  result += ": ";
+  result += state_->msg;
   return result;
 }
 
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index bfb945c498e64..448f4fa822059 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -10,7 +10,7 @@
 // non-const method, all threads accessing the same Status must use
 // external synchronization.
 
-// Adapted from Kudu github.com/apache/kudu
+// Adapted from Apache Kudu, TensorFlow
 
 #ifndef ARROW_STATUS_H_
 #define ARROW_STATUS_H_
@@ -72,13 +72,18 @@ enum class StatusCode : char {
   PlasmaStoreFull = 22
 };
 
+#if defined(__clang__)
+// Only clang supports warn_unused_result as a type annotation.
+class ARROW_MUST_USE_RESULT ARROW_EXPORT Status;
+#endif
+
 class ARROW_EXPORT Status {
  public:
   // Create a success status.
   Status() : state_(NULL) {}
-  ~Status() { delete[] state_; }
+  ~Status() { delete state_; }
 
-  Status(StatusCode code, const std::string& msg) : Status(code, msg, -1) {}
+  Status(StatusCode code, const std::string& msg);
 
   // Copy the specified status.
   Status(const Status& s);
@@ -88,44 +93,44 @@ class ARROW_EXPORT Status {
   static Status OK() { return Status(); }
 
   // Return error status of an appropriate type.
-  static Status OutOfMemory(const std::string& msg, int16_t posix_code = -1) {
-    return Status(StatusCode::OutOfMemory, msg, posix_code);
+  static Status OutOfMemory(const std::string& msg) {
+    return Status(StatusCode::OutOfMemory, msg);
   }
 
   static Status KeyError(const std::string& msg) {
-    return Status(StatusCode::KeyError, msg, -1);
+    return Status(StatusCode::KeyError, msg);
   }
 
   static Status TypeError(const std::string& msg) {
-    return Status(StatusCode::TypeError, msg, -1);
+    return Status(StatusCode::TypeError, msg);
   }
 
   static Status UnknownError(const std::string& msg) {
-    return Status(StatusCode::UnknownError, msg, -1);
+    return Status(StatusCode::UnknownError, msg);
   }
 
   static Status NotImplemented(const std::string& msg) {
-    return Status(StatusCode::NotImplemented, msg, -1);
+    return Status(StatusCode::NotImplemented, msg);
   }
 
   static Status Invalid(const std::string& msg) {
-    return Status(StatusCode::Invalid, msg, -1);
+    return Status(StatusCode::Invalid, msg);
   }
 
   static Status IOError(const std::string& msg) {
-    return Status(StatusCode::IOError, msg, -1);
+    return Status(StatusCode::IOError, msg);
   }
 
   static Status PlasmaObjectExists(const std::string& msg) {
-    return Status(StatusCode::PlasmaObjectExists, msg, -1);
+    return Status(StatusCode::PlasmaObjectExists, msg);
   }
 
   static Status PlasmaObjectNonexistent(const std::string& msg) {
-    return Status(StatusCode::PlasmaObjectNonexistent, msg, -1);
+    return Status(StatusCode::PlasmaObjectNonexistent, msg);
   }
 
   static Status PlasmaStoreFull(const std::string& msg) {
-    return Status(StatusCode::PlasmaStoreFull, msg, -1);
+    return Status(StatusCode::PlasmaStoreFull, msg);
   }
 
   // Returns true iff the status indicates success.
@@ -155,45 +160,31 @@ class ARROW_EXPORT Status {
   // text or posix code information.
   std::string CodeAsString() const;
 
-  // Get the POSIX code associated with this Status, or -1 if there is none.
-  int16_t posix_code() const;
+  StatusCode code() const { return ok() ? StatusCode::OK : state_->code; }
 
-  StatusCode code() const {
-    return ((state_ == NULL) ? StatusCode::OK : static_cast<StatusCode>(state_[4]));
-  }
+  std::string message() const { return ok() ? "" : state_->msg; }
 
-  std::string message() const {
-    uint32_t length;
-    memcpy(&length, state_, sizeof(length));
+ private:
+  struct State {
+    StatusCode code;
     std::string msg;
-    msg.append((state_ + 7), length);
-    return msg;
-  }
+  };
+  // OK status has a `NULL` state_.  Otherwise, `state_` points to
+  // a `State` structure containing the error code and message(s)
+  State* state_;
 
- private:
-  // OK status has a NULL state_.  Otherwise, state_ is a new[] array
-  // of the following form:
-  //    state_[0..3] == length of message
-  //    state_[4]    == code
-  //    state_[5..6] == posix_code
-  //    state_[7..]  == message
-  const char* state_;
-
-  Status(StatusCode code, const std::string& msg, int16_t posix_code);
-  static const char* CopyState(const char* s);
+  void CopyFrom(const State* s);
 };
 
-inline Status::Status(const Status& s) {
-  state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-}
+std::ostream& operator<<(std::ostream& os, const Status& x);
+
+inline Status::Status(const Status& s)
+    : state_((s.state_ == NULL) ? NULL : new State(*s.state_)) {}
 
 inline void Status::operator=(const Status& s) {
   // The following condition catches both aliasing (when this == &s),
   // and the common case where both s and *this are ok.
-  if (state_ != s.state_) {
-    delete[] state_;
-    state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-  }
+  if (state_ != s.state_) { CopyFrom(s.state_); }
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 94937b592cc33..2cff97a60a514 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -59,6 +59,12 @@
     EXPECT_TRUE(s.ok());        \
   } while (0)
 
+#define ABORT_NOT_OK(s)                              \
+  do {                                               \
+    ::arrow::Status _s = (s);                        \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { exit(-1); } \
+  } while (0);
+
 namespace arrow {
 
 using ArrayVector = std::vector<std::shared_ptr<Array>>;
@@ -176,14 +182,6 @@ static inline int64_t null_count(const std::vector<uint8_t>& valid_bytes) {
   return result;
 }
 
-std::shared_ptr<Buffer> bytes_to_null_buffer(const std::vector<uint8_t>& bytes) {
-  std::shared_ptr<Buffer> out;
-
-  // TODO(wesm): error checking
-  BitUtil::BytesToBits(bytes, &out);
-  return out;
-}
-
 Status MakeRandomInt32PoolBuffer(int64_t length, MemoryPool* pool,
     std::shared_ptr<PoolBuffer>* pool_buffer, uint32_t seed = 0) {
   DCHECK(pool);
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 1fbb683da6cf4..7f3adef633767 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -188,7 +188,7 @@ TEST_F(TestSchema, TestAddMetadata) {
       new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
   auto schema = std::make_shared<Schema>(fields);
   std::shared_ptr<Schema> new_schema;
-  schema->AddMetadata(metadata, &new_schema);
+  ASSERT_OK(schema->AddMetadata(metadata, &new_schema));
   ASSERT_TRUE(metadata->Equals(*new_schema->metadata()));
 
   // Not copied
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index ba0bfd7a9e387..5bbec6f23111e 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -34,7 +34,7 @@
 
 namespace arrow {
 
-void BitUtil::BytesToBits(const std::vector<uint8_t>& bytes, uint8_t* bits) {
+void BitUtil::FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits) {
   for (size_t i = 0; i < bytes.size(); ++i) {
     if (bytes[i] > 0) { SetBit(bits, i); }
   }
@@ -48,7 +48,7 @@ Status BitUtil::BytesToBits(
   RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), bit_length, &buffer));
 
   memset(buffer->mutable_data(), 0, static_cast<size_t>(bit_length));
-  BytesToBits(bytes, buffer->mutable_data());
+  FillBitsFromBytes(bytes, buffer->mutable_data());
 
   *out = buffer;
   return Status::OK();
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index bba9d2dbb9d21..d055c751d16fa 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -441,7 +441,7 @@ static T ShiftRightLogical(T v, int shift) {
   return static_cast<typename make_unsigned<T>::type>(v) >> shift;
 }
 
-void BytesToBits(const std::vector<uint8_t>& bytes, uint8_t* bits);
+void FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits);
 ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
 
 }  // namespace BitUtil
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 72107a21448d1..e3fd4802627ae 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -249,7 +249,7 @@ TYPED_TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
   Decimal<TypeParam> d;
   int precision;
   int scale;
-  FromString(string_value, &d, &precision, &scale);
+  ASSERT_OK(FromString(string_value, &d, &precision, &scale));
   ASSERT_EQ(precision, 7);
   ASSERT_EQ(scale, 0);
   ASSERT_EQ(d.value, 0);
@@ -260,7 +260,7 @@ TYPED_TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
   Decimal<TypeParam> d;
   int precision;
   int scale;
-  FromString(string_value, &d, &precision, &scale);
+  ASSERT_OK(FromString(string_value, &d, &precision, &scale));
   // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
   ASSERT_NE(precision, 7);
 
@@ -273,7 +273,7 @@ TYPED_TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
   Decimal<TypeParam> d;
   int precision;
   int scale;
-  FromString(string_value, &d, &precision, &scale);
+  ASSERT_OK(FromString(string_value, &d, &precision, &scale));
   ASSERT_EQ(precision, 5);
   ASSERT_EQ(scale, 5);
   ASSERT_EQ(d.value, 0);
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index f113c3359eaeb..0d84ba89db973 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -51,7 +51,7 @@ ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out,
 template <typename T>
 struct ARROW_EXPORT Decimal {
   Decimal() : value() {}
-  explicit Decimal(const std::string& s) : value() { FromString(s, this); }
+  explicit Decimal(const std::string& s) : value() { DCHECK(FromString(s, this).ok()); }
   explicit Decimal(const char* s) : Decimal(std::string(s)) {}
   explicit Decimal(const T& value) : value(value) {}
 
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index affe8bca4f207..a2f704f0c68bc 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -41,4 +41,12 @@
 #define ARROW_PREDICT_TRUE(x) x
 #endif
 
+#if (defined(__GNUC__) || defined(__APPLE__))
+#define ARROW_MUST_USE_RESULT __attribute__((warn_unused_result))
+#elif defined(_MSC_VER)
+#define ARROW_MUST_USE_RESULT
+#else
+#define ARROW_MUST_USE_RESULT
+#endif
+
 #endif  // ARROW_UTIL_MACROS_H
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 5151a44d7217d..9394e3de310b2 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -404,8 +404,10 @@ void PlasmaStore::connect_client(int listener_sock) {
   Client* client = new Client(client_fd);
   // Add a callback to handle events on this socket.
   // TODO(pcm): Check return value.
-  loop_->add_file_event(
-      client_fd, kEventLoopRead, [this, client](int events) { process_message(client); });
+  loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) {
+    Status s = process_message(client);
+    if (!s.ok()) { ARROW_LOG(FATAL) << "Failed to process file event: " << s; }
+  });
   ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
 }
 

From 83a4405ea0bd1696aeec7677edaef8671faed1ea Mon Sep 17 00:00:00 2001
From: Max Risukhin <risuhin.max@gmail.com>
Date: Thu, 6 Jul 2017 14:33:52 +0200
Subject: [PATCH 0781/1644] ARROW-599: [C++] Lz4 compression codec support

Author: Max Risukhin <risuhin.max@gmail.com>

Closes #813 from MaxRis/ARROW-599 and squashes the following commits:

434a238 [Max Risukhin] ARROW-599: [C++] Lz4 compression codec support
---
 cpp/CMakeLists.txt                          |  3 ++-
 cpp/build-support/build-lz4-lib.sh          | 16 +++++++++++++
 cpp/cmake_modules/ThirdpartyToolchain.cmake |  2 +-
 cpp/src/arrow/util/compression-test.cc      |  4 ++++
 cpp/src/arrow/util/compression.cc           | 26 +++++++++++++++++++++
 cpp/src/arrow/util/compression.h            | 16 ++++++++++++-
 python/manylinux1/scripts/build_lz4.sh      |  1 +
 7 files changed, 65 insertions(+), 3 deletions(-)
 create mode 100755 cpp/build-support/build-lz4-lib.sh

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 76d89ce010514..6d01fd9063e78 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -533,7 +533,8 @@ set(ARROW_STATIC_LINK_LIBS
   brotli_common
   snappy
   zlib
-  zstd_static)
+  zstd_static
+  lz4_static)
 
 add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
 
diff --git a/cpp/build-support/build-lz4-lib.sh b/cpp/build-support/build-lz4-lib.sh
new file mode 100755
index 0000000000000..62805bae286a2
--- /dev/null
+++ b/cpp/build-support/build-lz4-lib.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+export CFLAGS="${CFLAGS} -O3 -fPIC"
+make -j4
\ No newline at end of file
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 2be7b5a1ea656..3eef2f711572f 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -623,7 +623,7 @@ if (NOT LZ4_FOUND)
     set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
   else()
     set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
-    set(LZ4_BUILD_COMMAND BUILD_COMMAND make -j4)
+    set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh)
   endif()
 
   ExternalProject_Add(lz4_ep
diff --git a/cpp/src/arrow/util/compression-test.cc b/cpp/src/arrow/util/compression-test.cc
index 3b19a6d89a209..56dcc2a9ec42b 100644
--- a/cpp/src/arrow/util/compression-test.cc
+++ b/cpp/src/arrow/util/compression-test.cc
@@ -90,4 +90,8 @@ TEST(TestCompressors, ZSTD) {
   CheckCodec<ZSTDCodec>();
 }
 
+TEST(TestCompressors, Lz4) {
+  CheckCodec<Lz4Codec>();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression.cc b/cpp/src/arrow/util/compression.cc
index df1afa317ceae..0f17e7cd0e932 100644
--- a/cpp/src/arrow/util/compression.cc
+++ b/cpp/src/arrow/util/compression.cc
@@ -29,6 +29,7 @@
 
 #include <brotli/decode.h>
 #include <brotli/encode.h>
+#include <lz4.h>
 #include <snappy.h>
 #include <zlib.h>
 #include <zstd.h>
@@ -357,4 +358,29 @@ Status ZSTDCodec::Compress(int64_t input_len, const uint8_t* input,
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// Lz4 implementation
+
+Status Lz4Codec::Decompress(
+    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+  int64_t decompressed_size = LZ4_decompress_safe(reinterpret_cast<const char*>(input),
+      reinterpret_cast<char*>(output_buffer), static_cast<int>(input_len),
+      static_cast<int>(output_len));
+  if (decompressed_size < 1) { return Status::IOError("Corrupt Lz4 compressed data."); }
+  return Status::OK();
+}
+
+int64_t Lz4Codec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+  return LZ4_compressBound(static_cast<int>(input_len));
+}
+
+Status Lz4Codec::Compress(int64_t input_len, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+  *output_length = LZ4_compress_default(reinterpret_cast<const char*>(input),
+      reinterpret_cast<char*>(output_buffer), static_cast<int>(input_len),
+      static_cast<int>(output_buffer_len));
+  if (*output_length < 1) { return Status::IOError("Lz4 compression failure."); }
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h
index 9e581d8029f96..d38215320b70b 100644
--- a/cpp/src/arrow/util/compression.h
+++ b/cpp/src/arrow/util/compression.h
@@ -27,7 +27,7 @@
 namespace arrow {
 
 struct Compression {
-  enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO, BROTLI, ZSTD };
+  enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO, BROTLI, ZSTD, LZ4 };
 };
 
 class ARROW_EXPORT Codec {
@@ -118,6 +118,20 @@ class ARROW_EXPORT ZSTDCodec : public Codec {
   const char* name() const override { return "zstd"; }
 };
 
+// Lz4 codec.
+class ARROW_EXPORT Lz4Codec : public Codec {
+ public:
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override { return "lz4"; }
+};
+
 }  // namespace arrow
 
 #endif
diff --git a/python/manylinux1/scripts/build_lz4.sh b/python/manylinux1/scripts/build_lz4.sh
index 5a25d3de42515..975a3015412c3 100755
--- a/python/manylinux1/scripts/build_lz4.sh
+++ b/python/manylinux1/scripts/build_lz4.sh
@@ -13,6 +13,7 @@
 
 export LZ4_VERSION="1.7.5"
 export PREFIX="/usr"
+export CFLAGS="${CFLAGS} -O3 -fPIC"
 export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib -L${PREFIX}/lib"
 wget "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -O lz4-${LZ4_VERSION}.tar.gz
 tar xf lz4-${LZ4_VERSION}.tar.gz

From c398fda75a90fc83bafb2aafb895760b44ecd436 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 6 Jul 2017 20:18:38 -0400
Subject: [PATCH 0782/1644] ARROW-462: [C++] Implement in-memory conversions
 between non-nested primitive types and DictionaryArray equivalent

Simple usage:

```
Array primitive_array;
DictionaryBuilder<Type> builder;
RETURN_NOT_OK(builder.AppendArray(primitive_array));
std::shared_ptr<Array> dict_encoded_array;
RETURN_NOT_OK(builder.Finish(&dict_encoded_array));
```

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #812 from xhochy/ARROW-462 and squashes the following commits:

9e289b6a [Wes McKinney] Add a couple doxygen strings. Place WrappedBinary, DictionaryScalar in internal namespace. Add BinaryDictionaryBuilder convenience
4147bdbe [Uwe L. Korn] ARROW-462: [C++] Implement in-memory conversions between non-nested primitive types and DictionaryArray equivalent
---
 cpp/src/arrow/array-test.cc |  34 ++++++++
 cpp/src/arrow/array.h       |   2 +-
 cpp/src/arrow/builder.cc    | 165 +++++++++++++++++++++++++-----------
 cpp/src/arrow/builder.h     |  80 ++++++++++++++---
 4 files changed, 216 insertions(+), 65 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 64672d4eed804..7ae03cfb88ad4 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1538,6 +1538,40 @@ TYPED_TEST(TestDictionaryBuilder, Basic) {
   ASSERT_TRUE(expected.Equals(result));
 }
 
+TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
+  NumericBuilder<TypeParam> builder(default_memory_pool());
+  // DictionaryBuilder<TypeParam> builder(default_memory_pool());
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+
+  std::shared_ptr<Array> intermediate_result;
+  ASSERT_OK(builder.Finish(&intermediate_result));
+  DictionaryBuilder<TypeParam> dictionary_builder(default_memory_pool());
+  ASSERT_OK(dictionary_builder.AppendArray(*intermediate_result));
+  std::shared_ptr<Array> result;
+  ASSERT_OK(dictionary_builder.Finish(&result));
+
+  // Build expected data
+  NumericBuilder<TypeParam> dict_builder(default_memory_pool());
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> dict_array;
+  ASSERT_OK(dict_builder.Finish(&dict_array));
+  auto dtype =
+      std::make_shared<DictionaryType>(std::make_shared<TypeParam>(), dict_array);
+
+  UInt8Builder int_builder(default_memory_pool());
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
 TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
   using Scalar = typename TypeParam::c_type;
   // Skip this test for (u)int8
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 1c9769fff4615..59269adcb913e 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -478,7 +478,7 @@ class ARROW_EXPORT DictionaryArray : public Array {
   std::shared_ptr<Array> indices() const { return indices_; }
   std::shared_ptr<Array> dictionary() const;
 
-  const DictionaryType* dict_type() { return dict_type_; }
+  const DictionaryType* dict_type() const { return dict_type_; }
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 16f252c3d6c84..a57f75a1075e1 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -660,8 +660,8 @@ Status BooleanBuilder::Append(
 // ----------------------------------------------------------------------
 // DictionaryBuilder
 
-template <typename T, typename Scalar>
-DictionaryBuilder<T, Scalar>::DictionaryBuilder(
+template <typename T>
+DictionaryBuilder<T>::DictionaryBuilder(
     MemoryPool* pool, const std::shared_ptr<DataType>& type)
     : ArrayBuilder(pool, type),
       hash_table_(new PoolBuffer(pool)),
@@ -671,8 +671,8 @@ DictionaryBuilder<T, Scalar>::DictionaryBuilder(
   if (!::arrow::CpuInfo::initialized()) { ::arrow::CpuInfo::Init(); }
 }
 
-template <typename T, typename Scalar>
-Status DictionaryBuilder<T, Scalar>::Init(int64_t elements) {
+template <typename T>
+Status DictionaryBuilder<T>::Init(int64_t elements) {
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
 
   // Fill the initial hash table
@@ -685,8 +685,8 @@ Status DictionaryBuilder<T, Scalar>::Init(int64_t elements) {
   return values_builder_.Init(elements);
 }
 
-template <typename T, typename Scalar>
-Status DictionaryBuilder<T, Scalar>::Resize(int64_t capacity) {
+template <typename T>
+Status DictionaryBuilder<T>::Resize(int64_t capacity) {
   if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
 
   if (capacity_ == 0) {
@@ -696,8 +696,8 @@ Status DictionaryBuilder<T, Scalar>::Resize(int64_t capacity) {
   }
 }
 
-template <typename T, typename Scalar>
-Status DictionaryBuilder<T, Scalar>::Finish(std::shared_ptr<Array>* out) {
+template <typename T>
+Status DictionaryBuilder<T>::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<Array> dictionary;
   RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
   auto type = std::make_shared<DictionaryType>(type_, dictionary);
@@ -709,8 +709,8 @@ Status DictionaryBuilder<T, Scalar>::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
-template <typename T, typename Scalar>
-Status DictionaryBuilder<T, Scalar>::Append(const Scalar& value) {
+template <typename T>
+Status DictionaryBuilder<T>::Append(const Scalar& value) {
   RETURN_NOT_OK(Reserve(1));
   // Based on DictEncoder<DType>::Put
   int j = HashValue(value) & mod_bitmask_;
@@ -741,8 +741,26 @@ Status DictionaryBuilder<T, Scalar>::Append(const Scalar& value) {
   return Status::OK();
 }
 
-template <typename T, typename Scalar>
-Status DictionaryBuilder<T, Scalar>::DoubleTableSize() {
+template <typename T>
+Status DictionaryBuilder<T>::AppendArray(const Array& array) {
+  const NumericArray<T>& numeric_array = static_cast<const NumericArray<T>&>(array);
+  for (int64_t i = 0; i < array.length(); i++) {
+    if (array.IsNull(i)) {
+      RETURN_NOT_OK(AppendNull());
+    } else {
+      RETURN_NOT_OK(Append(numeric_array.Value(i)));
+    }
+  }
+  return Status::OK();
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::AppendNull() {
+  return values_builder_.AppendNull();
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::DoubleTableSize() {
   int new_size = hash_table_size_ * 2;
   auto new_hash_table = std::make_shared<PoolBuffer>(pool_);
 
@@ -782,56 +800,71 @@ Status DictionaryBuilder<T, Scalar>::DoubleTableSize() {
   return Status::OK();
 }
 
-template <typename T, typename Scalar>
-Scalar DictionaryBuilder<T, Scalar>::GetDictionaryValue(int64_t index) {
+template <typename T>
+typename DictionaryBuilder<T>::Scalar DictionaryBuilder<T>::GetDictionaryValue(
+    int64_t index) {
   const Scalar* data = reinterpret_cast<const Scalar*>(dict_builder_.data()->data());
   return data[index];
 }
 
-template <typename T, typename Scalar>
-int DictionaryBuilder<T, Scalar>::HashValue(const Scalar& value) {
+template <typename T>
+int DictionaryBuilder<T>::HashValue(const Scalar& value) {
   return HashUtil::Hash(&value, sizeof(Scalar), 0);
 }
 
-template <typename T, typename Scalar>
-bool DictionaryBuilder<T, Scalar>::SlotDifferent(hash_slot_t index, const Scalar& value) {
+template <typename T>
+bool DictionaryBuilder<T>::SlotDifferent(hash_slot_t index, const Scalar& value) {
   const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
   return other != value;
 }
 
-template <typename T, typename Scalar>
-Status DictionaryBuilder<T, Scalar>::AppendDictionary(const Scalar& value) {
+template <typename T>
+Status DictionaryBuilder<T>::AppendDictionary(const Scalar& value) {
   return dict_builder_.Append(value);
 }
 
-#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                       \
-  template <>                                                                         \
-  WrappedBinary DictionaryBuilder<Type, WrappedBinary>::GetDictionaryValue(           \
-      int64_t index) {                                                                \
-    int32_t v_len;                                                                    \
-    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len);   \
-    return WrappedBinary(v, v_len);                                                   \
-  }                                                                                   \
-                                                                                      \
-  template <>                                                                         \
-  int DictionaryBuilder<Type, WrappedBinary>::HashValue(const WrappedBinary& value) { \
-    return HashUtil::Hash(value.ptr_, value.length_, 0);                              \
-  }                                                                                   \
-                                                                                      \
-  template <>                                                                         \
-  bool DictionaryBuilder<Type, WrappedBinary>::SlotDifferent(                         \
-      hash_slot_t index, const WrappedBinary& value) {                                \
-    int32_t other_length;                                                             \
-    const uint8_t* other_value =                                                      \
-        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);           \
-    return !(other_length == value.length_ &&                                         \
-             0 == memcmp(other_value, value.ptr_, value.length_));                    \
-  }                                                                                   \
-                                                                                      \
-  template <>                                                                         \
-  Status DictionaryBuilder<Type, WrappedBinary>::AppendDictionary(                    \
-      const WrappedBinary& value) {                                                   \
-    return dict_builder_.Append(value.ptr_, value.length_);                           \
+#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                        \
+  template <>                                                                          \
+  internal::WrappedBinary DictionaryBuilder<Type>::GetDictionaryValue(int64_t index) { \
+    int32_t v_len;                                                                     \
+    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len);    \
+    return internal::WrappedBinary(v, v_len);                                          \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  Status DictionaryBuilder<Type>::AppendDictionary(                                    \
+      const internal::WrappedBinary& value) {                                          \
+    return dict_builder_.Append(value.ptr_, value.length_);                            \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  Status DictionaryBuilder<Type>::AppendArray(const Array& array) {                    \
+    const BinaryArray& binary_array = static_cast<const BinaryArray&>(array);          \
+    internal::WrappedBinary value(nullptr, 0);                                         \
+    for (int64_t i = 0; i < array.length(); i++) {                                     \
+      if (array.IsNull(i)) {                                                           \
+        RETURN_NOT_OK(AppendNull());                                                   \
+      } else {                                                                         \
+        value.ptr_ = binary_array.GetValue(i, &value.length_);                         \
+        RETURN_NOT_OK(Append(value));                                                  \
+      }                                                                                \
+    }                                                                                  \
+    return Status::OK();                                                               \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  int DictionaryBuilder<Type>::HashValue(const internal::WrappedBinary& value) {       \
+    return HashUtil::Hash(value.ptr_, value.length_, 0);                               \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  bool DictionaryBuilder<Type>::SlotDifferent(                                         \
+      hash_slot_t index, const internal::WrappedBinary& value) {                       \
+    int32_t other_length;                                                              \
+    const uint8_t* other_value =                                                       \
+        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);            \
+    return !(other_length == value.length_ &&                                          \
+             0 == memcmp(other_value, value.ptr_, value.length_));                     \
   }
 
 BINARY_DICTIONARY_SPECIALIZATIONS(StringType);
@@ -852,8 +885,8 @@ template class DictionaryBuilder<Time64Type>;
 template class DictionaryBuilder<TimestampType>;
 template class DictionaryBuilder<FloatType>;
 template class DictionaryBuilder<DoubleType>;
-template class DictionaryBuilder<BinaryType, WrappedBinary>;
-template class DictionaryBuilder<StringType, WrappedBinary>;
+template class DictionaryBuilder<BinaryType>;
+template class DictionaryBuilder<StringType>;
 
 // ----------------------------------------------------------------------
 // DecimalBuilder
@@ -1193,4 +1226,36 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
   }
 }
 
+#define DICTIONARY_BUILDER_CASE(ENUM, BuilderType) \
+  case Type::ENUM:                                 \
+    out->reset(new BuilderType(pool, type));       \
+    return Status::OK();
+
+Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+    std::shared_ptr<ArrayBuilder>* out) {
+  switch (type->id()) {
+    DICTIONARY_BUILDER_CASE(UINT8, DictionaryBuilder<UInt8Type>);
+    DICTIONARY_BUILDER_CASE(INT8, DictionaryBuilder<Int8Type>);
+    DICTIONARY_BUILDER_CASE(UINT16, DictionaryBuilder<UInt16Type>);
+    DICTIONARY_BUILDER_CASE(INT16, DictionaryBuilder<Int16Type>);
+    DICTIONARY_BUILDER_CASE(UINT32, DictionaryBuilder<UInt32Type>);
+    DICTIONARY_BUILDER_CASE(INT32, DictionaryBuilder<Int32Type>);
+    DICTIONARY_BUILDER_CASE(UINT64, DictionaryBuilder<UInt64Type>);
+    DICTIONARY_BUILDER_CASE(INT64, DictionaryBuilder<Int64Type>);
+    DICTIONARY_BUILDER_CASE(DATE32, DictionaryBuilder<Date32Type>);
+    DICTIONARY_BUILDER_CASE(DATE64, DictionaryBuilder<Date64Type>);
+    DICTIONARY_BUILDER_CASE(TIME32, DictionaryBuilder<Time32Type>);
+    DICTIONARY_BUILDER_CASE(TIME64, DictionaryBuilder<Time64Type>);
+    DICTIONARY_BUILDER_CASE(TIMESTAMP, DictionaryBuilder<TimestampType>);
+    DICTIONARY_BUILDER_CASE(FLOAT, DictionaryBuilder<FloatType>);
+    DICTIONARY_BUILDER_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
+    DICTIONARY_BUILDER_CASE(STRING, StringDictionaryBuilder);
+    DICTIONARY_BUILDER_CASE(BINARY, BinaryDictionaryBuilder);
+    // DICTIONARY_BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder);
+    // DICTIONARY_BUILDER_CASE(DECIMAL, DecimalBuilder);
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 5acefa1fc6298..12f3683ec0d41 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -688,18 +688,55 @@ static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits<int32_t>::max(
 // The maximum load factor for the hash table before resizing.
 static constexpr double kMaxHashTableLoad = 0.7;
 
-template <typename T, typename Scalar = typename T::c_type>
+namespace internal {
+
+// TODO(ARROW-1176): Use Tensorflow's StringPiece instead of this here.
+struct WrappedBinary {
+  WrappedBinary(const uint8_t* ptr, int32_t length) : ptr_(ptr), length_(length) {}
+
+  const uint8_t* ptr_;
+  int32_t length_;
+};
+
+template <typename T>
+struct DictionaryScalar {
+  using type = typename T::c_type;
+};
+
+template <>
+struct DictionaryScalar<BinaryType> {
+  using type = WrappedBinary;
+};
+
+template <>
+struct DictionaryScalar<StringType> {
+  using type = WrappedBinary;
+};
+
+}  // namespace internal
+
+/// \brief Array builder for created encoded DictionaryArray from dense array
+/// data
+template <typename T>
 class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
  public:
+  using Scalar = typename internal::DictionaryScalar<T>::type;
   explicit DictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 
-  template <typename T1 = T, typename Scalar1 = Scalar>
+  template <typename T1 = T>
   explicit DictionaryBuilder(
       typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool)
-      : DictionaryBuilder<T1, Scalar1>(pool, TypeTraits<T1>::type_singleton()) {}
+      : DictionaryBuilder<T1>(pool, TypeTraits<T1>::type_singleton()) {}
 
+  /// \brief Append a scalar value
   Status Append(const Scalar& value);
 
+  /// \brief Append a scalar null value
+  Status AppendNull();
+
+  /// \brief Append a whole dense array to the builder
+  Status AppendArray(const Array& array);
+
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
@@ -725,31 +762,43 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
   AdaptiveUIntBuilder values_builder_;
 };
 
-// TODO(ARROW-1176): Use Tensorflow's StringPiece instead of this here.
-struct WrappedBinary {
-  WrappedBinary(const uint8_t* ptr, int32_t length) : ptr_(ptr), length_(length) {}
+class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
+ public:
+  using DictionaryBuilder::DictionaryBuilder;
+  using DictionaryBuilder::Append;
 
-  const uint8_t* ptr_;
-  int32_t length_;
+  Status Append(const uint8_t* value, int32_t length) {
+    return Append(internal::WrappedBinary(value, length));
+  }
+
+  Status Append(const char* value, int32_t length) {
+    return Append(
+        internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
+  }
+
+  Status Append(const std::string& value) {
+    return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
+        static_cast<int32_t>(value.size())));
+  }
 };
 
-class ARROW_EXPORT StringDictionaryBuilder
-    : public DictionaryBuilder<StringType, WrappedBinary> {
+/// \brief Dictionary array builder with convenience methods for strings
+class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
  public:
   using DictionaryBuilder::DictionaryBuilder;
-
   using DictionaryBuilder::Append;
 
   Status Append(const uint8_t* value, int32_t length) {
-    return Append(WrappedBinary(value, length));
+    return Append(internal::WrappedBinary(value, length));
   }
 
   Status Append(const char* value, int32_t length) {
-    return Append(WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
+    return Append(
+        internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
   }
 
   Status Append(const std::string& value) {
-    return Append(WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
+    return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
         static_cast<int32_t>(value.size())));
   }
 };
@@ -760,6 +809,9 @@ class ARROW_EXPORT StringDictionaryBuilder
 Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     std::unique_ptr<ArrayBuilder>* out);
 
+Status ARROW_EXPORT MakeDictionaryBuilder(MemoryPool* pool,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<ArrayBuilder>* out);
+
 }  // namespace arrow
 
 #endif  // ARROW_BUILDER_H_

From 3309d126802f957b691a86044e32ae1828df33a7 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 7 Jul 2017 12:31:17 -0400
Subject: [PATCH 0783/1644] ARROW-1174: [GLib] Fix ListArray test failure

Data passed into `Arrow::Buffer` isn't owned by `Arrow::Buffer`. It's GLib bindings spec for optimization.
We should keep `Arrow::Buffer` data until `Arrow::Buffer` is alive.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #820 from kou/glib-fix-list-array-test-failure and squashes the following commits:

dfe9890e [Kouhei Sutou] [GLib] Fix ListArray test failure
---
 c_glib/test/run-test.rb | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb
index 50f548f3f5b3b..43bb22029d2a0 100755
--- a/c_glib/test/run-test.rb
+++ b/c_glib/test/run-test.rb
@@ -32,6 +32,15 @@
 require "gi"
 
 Arrow = GI.load("Arrow")
+module Arrow
+  class Buffer
+    alias_method :initialize_raw, :initialize
+    def initialize(data)
+      initialize_raw(data)
+      @data = data
+    end
+  end
+end
 
 require "tempfile"
 require_relative "helper/buildable"

From b6b876c52effa11e623fda464c3230fb9f4f26d3 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 9 Jul 2017 00:37:21 -0400
Subject: [PATCH 0784/1644] ARROW-1193: [C++] Support pkg-config for
 arrow_python.so

I want it to create a Ruby library that integrates Python via Apache Arrow object in the same process.
[pycall](https://rubygems.org/gems/pycall) gem provides a feature to use Python objects in Ruby script in the same process. In this situation, we can change Apache Arrow data between Ruby and Python as Ruby/Python object instead of writing/reading Apache Arrow objects.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #821 from kou/cpp-python-support-pkg-config and squashes the following commits:

12979fca [Kouhei Sutou] [C++] Don't link libpython
f473e667 [Kouhei Sutou] [C++] Support pkg-config for arrow_python.so
---
 cpp/src/arrow/python/CMakeLists.txt     |  8 ++++++++
 cpp/src/arrow/python/arrow-python.pc.in | 27 +++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 cpp/src/arrow/python/arrow-python.pc.in

diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index bc2a815097d05..d5e980bac441f 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -95,6 +95,14 @@ install(FILES
   type_traits.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python")
 
+# pkg-config support
+configure_file(arrow-python.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/arrow-python.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-python.pc"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
+
 if (ARROW_BUILD_TESTS)
   ADD_ARROW_TEST(python-test
     STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}")
diff --git a/cpp/src/arrow/python/arrow-python.pc.in b/cpp/src/arrow/python/arrow-python.pc.in
new file mode 100644
index 0000000000000..11e33c512c7d9
--- /dev/null
+++ b/cpp/src/arrow/python/arrow-python.pc.in
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
+includedir=${prefix}/include
+
+Name: Apache Arrow Python
+Description: Python integration library for Apache Arrow
+Version: @ARROW_VERSION@
+Requires: arrow
+Libs: -L${libdir} -larrow_python
+Cflags: -I${includedir} -I@PYTHON_INCLUDE_DIRS@

From e8945325e187e2f9667840497e22cc05c53ae123 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 9 Jul 2017 00:38:05 -0400
Subject: [PATCH 0785/1644] ARROW-1197: [GLib] Fix a bug that record batch
 related functions for C++ aren't included

Author: Kouhei Sutou <kou@clear-code.com>

Closes #822 from kou/glib-fix-typo and squashes the following commits:

452dba8f [Kouhei Sutou] [GLib] Fix a bug that record batch related functions for C++ isn't included
---
 c_glib/arrow-glib/arrow-glib.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
index ae550702bed92..7fc6c4828d8c5 100644
--- a/c_glib/arrow-glib/arrow-glib.hpp
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -29,7 +29,7 @@
 #include <arrow-glib/data-type.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/field.hpp>
-#include <arrow-glib/record-batch.h>
+#include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 #include <arrow-glib/table.hpp>
 #include <arrow-glib/tensor.hpp>

From 7870804e0ea370b0e56811769e5252e6aba69e34 Mon Sep 17 00:00:00 2001
From: fjetter <florian.jetter@blue-yonder.com>
Date: Mon, 10 Jul 2017 16:22:05 +0200
Subject: [PATCH 0786/1644] ARROW-1074: Support lists and arrays in pandas
 DataFrames without explicit schema

This introduces automatic type inference for lists and numpy arrays in a pandas data frame.

Partial implementation for: https://issues.apache.org/jira/browse/ARROW-575

Author: fjetter <florian.jetter@blue-yonder.com>

Closes #825 from fjetter/feature/pandas_converter_lists and squashes the following commits:

8bde4e7 [fjetter] Use unicode instead of str in tests
6d262e9 [fjetter] Use OwnedRef reset
037cc77 [fjetter] apply clang-format
331f8a7 [fjetter] Fix bus error
506666f [fjetter] Support numpy array in sequential visitor
b54c1f5 [fjetter] Factor out InferArrowType
4a61585 [fjetter] Add vscode config files to gitignore
6dee516 [fjetter] infer lists in pandas converter
---
 .gitignore                                  |  3 +-
 cpp/src/arrow/python/builtin_convert.cc     | 42 ++++++++++++++-------
 cpp/src/arrow/python/builtin_convert.h      |  2 +
 cpp/src/arrow/python/pandas_convert.cc      | 13 +++++--
 python/pyarrow/tests/test_convert_pandas.py | 30 +++++++++++++++
 5 files changed, 72 insertions(+), 18 deletions(-)

diff --git a/.gitignore b/.gitignore
index 5e28b3685e465..dd69b6cec9c5f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,4 +26,5 @@
 MANIFEST
 
 cpp/.idea/
-python/.eggs/
\ No newline at end of file
+python/.eggs/
+.vscode
\ No newline at end of file
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 11114b0230ab1..f10dac75c67c8 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -155,14 +155,20 @@ class SeqVisitor {
   // co-recursive with VisitElem
   Status Visit(PyObject* obj, int level = 0) {
     if (level > max_nesting_level_) { max_nesting_level_ = level; }
-
     // Loop through either a sequence or an iterator.
     if (PySequence_Check(obj)) {
       Py_ssize_t size = PySequence_Size(obj);
       for (int64_t i = 0; i < size; ++i) {
-        // TODO(wesm): Specialize for PyList_GET_ITEM?
-        OwnedRef ref = OwnedRef(PySequence_GetItem(obj, i));
-        RETURN_NOT_OK(VisitElem(ref, level));
+        OwnedRef ref;
+        if (PyArray_Check(obj)) {
+          auto array = reinterpret_cast<PyArrayObject*>(obj);
+          auto ptr = reinterpret_cast<const char*>(PyArray_GETPTR1(array, i));
+          ref.reset(PyArray_GETITEM(array, ptr));
+          RETURN_NOT_OK(VisitElem(ref, level));
+        } else {
+          ref.reset(PySequence_GetItem(obj, i));
+          RETURN_NOT_OK(VisitElem(ref, level));
+        }
       }
     } else if (PyObject_HasAttrString(obj, "__iter__")) {
       OwnedRef iter = OwnedRef(PyObject_GetIter(obj));
@@ -280,25 +286,32 @@ Status InferArrowSize(PyObject* obj, int64_t* size) {
 }
 
 // Non-exhaustive type inference
-Status InferArrowTypeAndSize(
-    PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
-  RETURN_NOT_OK(InferArrowSize(obj, size));
-
-  // For 0-length sequences, refuse to guess
-  if (*size == 0) { *out_type = null(); }
-
+Status InferArrowType(PyObject* obj, std::shared_ptr<DataType>* out_type) {
   PyDateTime_IMPORT;
   SeqVisitor seq_visitor;
   RETURN_NOT_OK(seq_visitor.Visit(obj));
   RETURN_NOT_OK(seq_visitor.Validate());
 
   *out_type = seq_visitor.GetType();
-
   if (*out_type == nullptr) { return Status::TypeError("Unable to determine data type"); }
 
   return Status::OK();
 }
 
+Status InferArrowTypeAndSize(
+    PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
+  RETURN_NOT_OK(InferArrowSize(obj, size));
+
+  // For 0-length sequences, refuse to guess
+  if (*size == 0) {
+    *out_type = null();
+    return Status::OK();
+  }
+  RETURN_NOT_OK(InferArrowType(obj, out_type));
+
+  return Status::OK();
+}
+
 // Marshal Python sequence (list, tuple, etc.) to Arrow array
 class SeqConverter {
  public:
@@ -464,8 +477,9 @@ class FixedWidthBytesConverter
   inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
-    Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedSizeBinaryType>(
-        typed_builder_->type())->byte_width();
+    Py_ssize_t expected_length =
+        std::dynamic_pointer_cast<FixedSizeBinaryType>(typed_builder_->type())
+            ->byte_width();
     if (item.obj() == Py_None) {
       RETURN_NOT_OK(typed_builder_->AppendNull());
       return Status::OK();
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index 7a84cbefef6a1..dd878b2d95847 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -38,6 +38,8 @@ class Status;
 
 namespace py {
 
+ARROW_EXPORT arrow::Status InferArrowType(
+    PyObject* obj, std::shared_ptr<arrow::DataType>* out_type);
 ARROW_EXPORT arrow::Status InferArrowTypeAndSize(
     PyObject* obj, int64_t* size, std::shared_ptr<arrow::DataType>* out_type);
 ARROW_EXPORT arrow::Status InferArrowSize(PyObject* obj, int64_t* size);
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index f75a2ba921b0d..2364f135b343f 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -893,9 +893,13 @@ Status PandasConverter::ConvertObjects() {
         return ConvertDates<Date32Type>();
       } else if (PyObject_IsInstance(const_cast<PyObject*>(objects[i]), Decimal.obj())) {
         return ConvertDecimals();
+      } else if (PyList_Check(objects[i]) || PyArray_Check(objects[i])) {
+        std::shared_ptr<DataType> inferred_type;
+        RETURN_NOT_OK(InferArrowType(objects[i], &inferred_type));
+        return ConvertLists(inferred_type);
       } else {
-        return InvalidConversion(
-            const_cast<PyObject*>(objects[i]), "string, bool, float, int, date, decimal");
+        return InvalidConversion(const_cast<PyObject*>(objects[i]),
+            "string, bool, float, int, date, decimal, list, array");
       }
     }
   }
@@ -1038,7 +1042,10 @@ Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
     LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
     LIST_CASE(STRING, NPY_OBJECT, StringType)
     default:
-      return Status::TypeError("Unknown list item type");
+      std::stringstream ss;
+      ss << "Unknown list item type: ";
+      ss << type->ToString();
+      return Status::TypeError(ss.str());
   }
 
   return Status::TypeError("Unknown list type");
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index b952d4a5ffd13..fb69cac9dc339 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -691,3 +691,33 @@ def test_structarray(self):
 
         series = pd.Series(arr.to_pandas())
         tm.assert_series_equal(series, expected)
+
+    def test_infer_lists(self):
+        data = OrderedDict([
+            ('nan_ints', [[None, 1], [2, 3]]),
+            ('ints', [[0, 1], [2, 3]]),
+            ('strs', [[None, u'b'], [u'c', u'd']])
+        ])
+        df = pd.DataFrame(data)
+
+        expected_schema = pa.schema([
+            pa.field('nan_ints', pa.list_(pa.int64())),
+            pa.field('ints', pa.list_(pa.int64())),
+            pa.field('strs', pa.list_(pa.string()))
+        ])
+
+        self._check_pandas_roundtrip(df, expected_schema=expected_schema)
+
+    def test_infer_numpy_array(self):
+        data = OrderedDict([
+            ('ints', [
+                np.array([0, 1], dtype=np.int64),
+                np.array([2, 3], dtype=np.int64)
+            ])
+        ])
+        df = pd.DataFrame(data)
+        expected_schema = pa.schema([
+            pa.field('ints', pa.list_(pa.int64()))
+        ])
+
+        self._check_pandas_roundtrip(df, expected_schema=expected_schema)

From f73c1c3bb69c2edf67561569ce5f0a8ae79462db Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 10 Jul 2017 17:00:42 +0200
Subject: [PATCH 0787/1644] ARROW-1201: [Python] Incomplete Python types cause
 a core dump when repr-ing

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #826 from cpcloud/ARROW-1201 and squashes the following commits:

fbf93c3 [Phillip Cloud] ARROW-1201: [Python] Incomplete Python types cause a core dump when repr-ing
---
 python/pyarrow/__init__.py         | 9 +--------
 python/pyarrow/array.pxi           | 8 ++++++++
 python/pyarrow/tests/test_array.py | 6 ++++++
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 771a516e147a5..43109548bc148 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -33,13 +33,6 @@
                          float16, float32, float64,
                          binary, string, decimal,
                          list_, struct, dictionary, field,
-                         DataType,
-                         DecimalType,
-                         DictionaryType,
-                         FixedSizeBinaryType,
-                         TimestampType,
-                         Time32Type,
-                         Time64Type,
                          Field,
                          Schema,
                          schema,
@@ -60,7 +53,7 @@
                          Date32Array, Date64Array,
                          TimestampArray, Time32Array, Time64Array,
                          DecimalArray, StructArray,
-                         ArrayValue, Scalar, NA, NAType,
+                         ArrayValue, Scalar, NA,
                          BooleanValue,
                          Int8Value, Int16Value, Int32Value, Int64Value,
                          UInt8Value, UInt16Value, UInt32Value, UInt64Value,
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 6a4925603d67b..9e6ac8d347538 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -52,6 +52,14 @@ cdef class DataType:
         self.type = type.get()
 
     def __str__(self):
+        if self.type is NULL:
+            raise TypeError(
+                '{} is incomplete. The correct way to construct types is '
+                'through public API functions named '
+                'pyarrow.int64, pyarrow.list_, etc.'.format(
+                    type(self).__name__
+                )
+            )
         return frombytes(self.type.ToString())
 
     def __repr__(self):
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 7c91785e12b2a..af21741bf5e4f 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -189,3 +189,9 @@ def test_dictionary_with_pandas():
                                            categories=dictionary)
 
     tm.assert_series_equal(pd.Series(pandas2), pd.Series(ex_pandas2))
+
+
+def test_simple_type_construction():
+    result = pa.lib.TimestampType()
+    with pytest.raises(TypeError):
+        str(result)

From cab07c2bd655160930199160ac52c91b3ff0fba4 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 10 Jul 2017 19:27:22 +0200
Subject: [PATCH 0788/1644] ARROW-1202: [C++] Remove semicolons from status
 macros

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #828 from cpcloud/ARROW-1202 and squashes the following commits:

50ba53a [Phillip Cloud] ARROW-1202: [C++] Remove semicolons from status macros
---
 cpp/src/arrow/ipc/feather.cc | 2 +-
 cpp/src/arrow/ipc/reader.cc  | 2 +-
 cpp/src/arrow/status.h       | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 7f577a09b8a46..bc7c431969cac 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -562,7 +562,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
       // Write the variable-length offsets
       RETURN_NOT_OK(WritePadded(stream_.get(),
           reinterpret_cast<const uint8_t*>(bin_values.raw_value_offsets()), offset_bytes,
-          &bytes_written))
+          &bytes_written));
       meta->total_bytes += bytes_written;
 
       if (bin_values.data()) { values_buffer = bin_values.data()->data(); }
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index cb747b681f799..ea16bf0335e9d 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -224,7 +224,7 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
     RETURN_NOT_OK(ReadNextMessage(Message::DICTIONARY_BATCH, false, &message));
 
     std::shared_ptr<Buffer> batch_body;
-    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body))
+    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body));
     io::BufferReader reader(batch_body);
 
     std::shared_ptr<Array> dictionary;
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 448f4fa822059..8491ac84730ec 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -27,7 +27,7 @@
   do {                                                \
     ::arrow::Status _s = (s);                         \
     if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; } \
-  } while (0);
+  } while (0)
 
 // If 'to_call' returns a bad status, CHECK immediately with a logged message
 // of 'msg' followed by the status.
@@ -35,7 +35,7 @@
   do {                                                      \
     ::arrow::Status _s = (to_call);                         \
     ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString(); \
-  } while (0);
+  } while (0)
 
 // If the status is bad, CHECK immediately, appending the status to the
 // logged message.
@@ -47,7 +47,7 @@ namespace arrow {
   do {                                                \
     Status _s = (s);                                  \
     if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; } \
-  } while (0);
+  } while (0)
 
 #define RETURN_NOT_OK_ELSE(s, else_) \
   do {                               \
@@ -56,7 +56,7 @@ namespace arrow {
       else_;                         \
       return _s;                     \
     }                                \
-  } while (0);
+  } while (0)
 
 enum class StatusCode : char {
   OK = 0,

From bc16e0e04a3d4ea9c96579ac59cee4b16e771850 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Mon, 10 Jul 2017 19:32:46 +0200
Subject: [PATCH 0789/1644] =?UTF-8?q?ARROW-1196:=20[C++]=20Release,=20Debu?=
 =?UTF-8?q?g,=20Toolchain,=20NMake=20Generator=20Appveyor=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… jobs; Windows.md updated.

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #823 from MaxRis/ARROW-1196 and squashes the following commits:

1ca1155 [Max Risuhin] ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor jobs; Windows.md updated.
---
 appveyor.yml                                |  15 ++
 ci/msvc-build.bat                           |  31 ++++-
 cpp/CMakeLists.txt                          |   4 +
 cpp/apidoc/Windows.md                       | 146 +++++++++++++++++---
 cpp/cmake_modules/ThirdpartyToolchain.cmake |   7 +-
 5 files changed, 181 insertions(+), 22 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 802d43bad4ee8..f3ce5a544375f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -20,9 +20,23 @@ os: Visual Studio 2015
 
 environment:
   matrix:
+    - GENERATOR: NMake Makefiles
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Release"
+    - GENERATOR: Visual Studio 14 2015 Win64
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Debug"
     - GENERATOR: Visual Studio 14 2015 Win64
       PYTHON: "3.5"
       ARCH: "64"
+      CONFIGURATION: "Release"
+    - GENERATOR: Visual Studio 14 2015 Win64
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Toolchain"
+
   MSVC_DEFAULT_OPTIONS: ON
   BOOST_ROOT: C:\Libraries\boost_1_63_0
   BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
@@ -30,6 +44,7 @@ environment:
 init:
   - set MINICONDA=C:\Miniconda35-x64
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
+  - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64
 
 build_script:
   - git config core.symlinks true
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 366fc2818f0f3..c84d12d97d7d8 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -17,18 +17,41 @@
 
 @echo on
 
+if "%CONFIGURATION%" == "Debug" (
+  mkdir cpp\build-debug
+  pushd cpp\build-debug
+
+  cmake -G "%GENERATOR%" ^
+        -DARROW_BOOST_USE_SHARED=OFF ^
+        -DCMAKE_BUILD_TYPE=Debug ^
+        -DARROW_CXXFLAGS="/MP" ^
+        ..  || exit /B
+
+  cmake --build . --config Debug || exit /B
+  popd
+
+  @rem Finish Debug build successfully
+  exit /B 0
+)
+
 conda update --yes --quiet conda
 
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython
-conda install -n arrow -q -y -c conda-forge ^
-      flatbuffers rapidjson ^
-      cmake git boost-cpp thrift-cpp snappy zlib brotli gflags
+
+if "%CONFIGURATION%" == "Toolchain" (
+  conda install -n arrow -q -y -c conda-forge ^
+      flatbuffers rapidjson cmake git boost-cpp ^
+      thrift-cpp snappy zlib brotli gflags lz4-c zstd
+)
 
 call activate arrow
 
+if "%CONFIGURATION%" == "Toolchain" (
+  set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+)
+
 set ARROW_HOME=%CONDA_PREFIX%\Library
-set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 
 @rem Build and test Arrow C++ libraries
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6d01fd9063e78..002a07e922c74 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -152,6 +152,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
       "Snappy static lib suffix used on Windows with MSVC (default is empty string)")
     set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING
       "Zlib static lib suffix used on Windows with MSVC (default libstatic)")
+    set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+      "Lz4 static lib suffix used on Windows with MSVC (default _static)")
+    set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+      "ZStd static lib suffix used on Windows with MSVC (default _static)")
   endif()
 endif()
 
diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index b9a6143a744a4..6bfb951548ace 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -21,24 +21,13 @@ investing time in maintaining the thirdparty build dependencies for Arrow and
 related C++ libraries using the conda package manager. Others are free to add
 other development instructions for Windows here.
 
-### Visual Studio
-
-Microsoft provides the free Visual Studio 2017 Community edition. When doing
-development, you must launch the developer command prompt using
-
-```
-"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64
-```
-
-It's easiest to configure a console emulator like [cmder][3] to automatically
-launch this when starting a new development console.
-
 ### conda and package toolchain
 
 [Miniconda][1] is a minimal Python distribution including the conda package
 manager. To get started, download and install a 64-bit distribution.
 
-We recommend using packages from [conda-forge][2]
+We recommend using packages from [conda-forge][2].  
+Launch cmd.exe and run following commands:
 
 ```shell
 conda config --add channels conda-forge
@@ -47,9 +36,92 @@ conda config --add channels conda-forge
 Now, you can bootstrap a build environment
 
 ```shell
-conda create -n arrow-dev cmake git boost
+conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy zlib brotli gflags lz4-c zstd
 ```
 
+Activate just created conda environment with pre-installed packages from
+previous step:
+
+```shell
+activate arrow-dev
+```
+
+We are using [cmake][4] tool to support Windows builds.  
+To allow cmake to pick up 3rd party dependencies, you should set
+`ARROW_BUILD_TOOLCHAIN` environment variable to contain `Library` folder
+path of new created on previous step `arrow-dev` conda environment.
+For instance, if `Miniconda` was installed to default destination, `Library`
+folder path for `arrow-dev` conda environment will be as following:
+
+```shell
+C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library
+```
+
+To set `ARROW_BUILD_TOOLCHAIN` environment variable visible only for current terminal session you can run following:
+```shell
+set ARROW_BUILD_TOOLCHAIN=C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library
+```
+
+To validate value of `ARROW_BUILD_TOOLCHAIN` environment variable you can run following terminal command:
+```shell
+echo %ARROW_BUILD_TOOLCHAIN%
+```
+
+As alternative to `ARROW_BUILD_TOOLCHAIN`, it's possible to configure path
+to each 3rd party dependency separately by setting appropriate environment
+variable:
+
+`FLATBUFFERS_HOME` variable with path to `flatbuffers` installation  
+`RAPIDJSON_HOME` variable with path to `rapidjson` installation  
+`GFLAGS_HOME` variable with path to `gflags` installation  
+`SNAPPY_HOME` variable with path to `snappy` installation  
+`ZLIB_HOME` variable with path to `zlib` installation  
+`BROTLI_HOME` variable with path to `brotli` installation  
+`LZ4_HOME` variable with path to `lz4` installation  
+`ZSTD_HOME` variable with path to `zstd` installation
+
+### Customize static libraries names lookup of 3rd party dependencies 
+
+If you decided to use pre-built 3rd party dependencies libs, it's possible to
+configure Arrow's cmake build script to search for customized names of 3rd
+party static libs.
+
+`zlib`. Pass `-DARROW_ZLIB_VENDORED=OFF` to enable lookup of custom zlib
+build. Set `ZLIB_HOME` environment variable. Pass
+`-DZLIB_MSVC_STATIC_LIB_SUFFIX=%ZLIB_SUFFIX%` to link with z%ZLIB_SUFFIX%.lib
+
+`brotli`. Set `BROTLY_HOME` environment variable. Pass
+`-DBROTLI_MSVC_STATIC_LIB_SUFFIX=%BROTLI_SUFFIX%` to link with
+brotli*%BROTLI_SUFFIX%.lib.
+
+`snappy`. Set `SNAPPY_HOME` environment variable. Pass
+`-DSNAPPY_MSVC_STATIC_LIB_SUFFIX=%SNAPPY_SUFFIX%` to link with
+snappy%SNAPPY_SUFFIX%.lib.
+
+`lz4`. Set `LZ4_HOME` environment variable. Pass
+`-LZ4_MSVC_STATIC_LIB_SUFFIX=%LZ4_SUFFIX%` to link with
+lz4%LZ4_SUFFIX%.lib.
+
+`zstd`. Set `ZSTD_HOME` environment variable. Pass
+`-ZSTD_MSVC_STATIC_LIB_SUFFIX=%ZSTD_SUFFIX%` to link with
+zstd%ZSTD_SUFFIX%.lib.
+
+### Visual Studio
+
+Microsoft provides the free Visual Studio Community edition. When doing
+development, you must launch the developer command prompt using
+
+#### Visual Studio 2015
+
+```"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64```
+
+#### Visual Studio 2017
+
+```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64```
+
+It's easiest to configure a console emulator like [cmder][3] to automatically
+launch this when starting a new development console.
+
 ## Building with NMake
 
 Activate your conda build environment:
@@ -58,7 +130,8 @@ Activate your conda build environment:
 activate arrow-dev
 ```
 
-Now, do an out of source build using `nmake`:
+Change working directory in cmd.exe to the root directory of Arrow and
+do an out of source build using `nmake`:
 
 ```
 cd cpp
@@ -72,14 +145,53 @@ When using conda, only release builds are currently supported.
 
 ## Build using Visual Studio (MSVC) Solution Files
 
-To build on the command line by instead generating a MSVC solution, instead
-run:
+Activate your conda build environment:
 
 ```
+activate arrow-dev
+```
+
+Change working directory in cmd.exe to the root directory of Arrow and
+do an out of source build by generating a MSVC solution:
+
+```
+cd cpp
+mkdir build
+cd build
 cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=Release ..
 cmake --build . --config Release
 ```
 
+## Debug build
+
+To build Debug version of Arrow you should have pre-insalled Debug version of
+boost libs.
+
+It's recommended to configure cmake build with following variables for Debug build:
+
+`-DARROW_BOOST_USE_SHARED=OFF` - enables static linking with boost debug libs and
+simplifies run-time loading of 3rd parties. (Recommended)
+
+`-DBOOST_ROOT` - sets the root directory of boost libs. (Optional)
+
+`-DBOOST_LIBRARYDIR` - sets the directory with boost lib files. (Optional)
+
+Command line to build Arrow in Debug might look as following:
+
+```
+cd cpp
+mkdir build
+cd build
+cmake -G "Visual Studio 14 2015 Win64" -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=OFF -DCMAKE_BUILD_TYPE=Debug -DBOOST_ROOT=C:/local/boost_1_63_0  -DBOOST_LIBRARYDIR=C:/local/boost_1_63_0/lib64-msvc-14.0 ..
+cmake --build . --config Debug
+```
+
+To get the latest build instructions, you can reference [msvc-build.bat][5], which is used by automated Appveyor builds.
+
+
+
 [1]: https://conda.io/miniconda.html
 [2]: https://conda-forge.github.io/
 [3]: http://cmder.net/
+[4]: https://cmake.org/
+[5]: https://github.com/apache/arrow/blob/master/ci/msvc-build.bat
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 3eef2f711572f..74939acabfffb 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -346,10 +346,15 @@ if (ARROW_IPC)
   ## Flatbuffers
   if("${FLATBUFFERS_HOME}" STREQUAL "")
     set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
+    if (MSVC)
+      set(FLATBUFFERS_CMAKE_CXX_FLAGS /EHsc)
+    else()
+      set(FLATBUFFERS_CMAKE_CXX_FLAGS -fPIC)
+    endif()
     ExternalProject_Add(flatbuffers_ep
       URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
       CMAKE_ARGS
-      "-DCMAKE_CXX_FLAGS=-fPIC"
+      "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}"
       "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
       "-DFLATBUFFERS_BUILD_TESTS=OFF")
 

From 471a85fd77b25f6be4b7557349d1d32b042f0dd9 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 10 Jul 2017 17:24:55 -0400
Subject: [PATCH 0790/1644] ARROW-1168: [Python] pandas metadata may contain
 "mixed" data types

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #817 from cpcloud/ARROW-1168 and squashes the following commits:

4b85f3cb [Phillip Cloud] ARROW-1168: [Python] pandas metadata may contain "mixed" data types
---
 python/pyarrow/array.pxi                    |  44 +++++--
 python/pyarrow/includes/libarrow.pxd        |   1 +
 python/pyarrow/lib.pxd                      |   5 +
 python/pyarrow/lib.pyx                      |  27 ++++
 python/pyarrow/pandas_compat.py             | 133 +++++++++++++++++---
 python/pyarrow/public-api.pxi               |   2 +
 python/pyarrow/table.pxi                    |  20 +--
 python/pyarrow/tests/test_array.py          |  34 +++++
 python/pyarrow/tests/test_convert_pandas.py |  44 ++++++-
 9 files changed, 277 insertions(+), 33 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 9e6ac8d347538..bf87173781d53 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -19,7 +19,7 @@ from pyarrow.includes.libarrow cimport *
 
 # These are imprecise because the type (in pandas 0.x) depends on the presence
 # of nulls
-_pandas_type_map = {
+cdef dict _pandas_type_map = {
     _Type_NA: np.float64,  # NaNs
     _Type_BOOL: np.bool_,
     _Type_INT8: np.int8,
@@ -39,9 +39,11 @@ _pandas_type_map = {
     _Type_BINARY: np.object_,
     _Type_FIXED_SIZE_BINARY: np.object_,
     _Type_STRING: np.object_,
-    _Type_LIST: np.object_
+    _Type_LIST: np.object_,
+    _Type_DECIMAL: np.object_,
 }
 
+
 cdef class DataType:
 
     def __cinit__(self):
@@ -51,6 +53,11 @@ cdef class DataType:
         self.sp_type = type
         self.type = type.get()
 
+    property id:
+
+        def __get__(self):
+            return self.type.id()
+
     def __str__(self):
         if self.type is NULL:
             raise TypeError(
@@ -91,6 +98,18 @@ cdef class DictionaryType(DataType):
         self.dict_type = <const CDictionaryType*> type.get()
 
 
+cdef class ListType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.list_type = <const CListType*> type.get()
+
+    property value_type:
+
+        def __get__(self):
+            return pyarrow_wrap_data_type(self.list_type.value_type())
+
+
 cdef class TimestampType(DataType):
 
     cdef void init(self, const shared_ptr[CDataType]& type):
@@ -154,6 +173,16 @@ cdef class DecimalType(FixedSizeBinaryType):
         DataType.init(self, type)
         self.decimal_type = <const CDecimalType*> type.get()
 
+    property precision:
+
+        def __get__(self):
+            return self.decimal_type.precision()
+
+    property scale:
+
+        def __get__(self):
+            return self.decimal_type.scale()
+
 
 cdef class Field:
     """
@@ -630,7 +659,7 @@ def binary(int length=-1):
     return pyarrow_wrap_data_type(fixed_size_binary_type)
 
 
-def list_(value_type):
+cpdef ListType list_(value_type):
     """
     Create ListType instance from child data type or field
 
@@ -645,8 +674,8 @@ def list_(value_type):
     cdef:
         DataType data_type
         Field field
-
-    cdef shared_ptr[CDataType] list_type
+        shared_ptr[CDataType] list_type
+        ListType out = ListType()
 
     if isinstance(value_type, DataType):
         list_type.reset(new CListType((<DataType> value_type).sp_type))
@@ -655,10 +684,11 @@ def list_(value_type):
     else:
         raise ValueError('List requires DataType or Field')
 
-    return pyarrow_wrap_data_type(list_type)
+    out.init(list_type)
+    return out
 
 
-def dictionary(DataType index_type, Array dictionary):
+cpdef DictionaryType dictionary(DataType index_type, Array dictionary):
     """
     Dictionary (categorical, or simply encoded) type
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 902f98e34d67f..cc46c76d3f614 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -161,6 +161,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CListType" arrow::ListType"(CDataType):
         CListType(const shared_ptr[CDataType]& value_type)
         CListType(const shared_ptr[CField]& field)
+        shared_ptr[CDataType] value_type()
 
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 8fa7fd9d98df0..3e1419bdfc072 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -52,6 +52,11 @@ cdef class DataType:
     cdef void init(self, const shared_ptr[CDataType]& type)
 
 
+cdef class ListType(DataType):
+    cdef:
+        const CListType* list_type
+
+
 cdef class DictionaryType(DataType):
     cdef:
         const CDictionaryType* dict_type
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index ae311aca8d0cc..13c1822b25e56 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -67,6 +67,33 @@ def set_cpu_count(count):
     CPU_COUNT = max(int(count), 1)
 
 
+Type_NA = _Type_NA
+Type_BOOL = _Type_BOOL
+Type_UINT8 = _Type_UINT8
+Type_INT8 = _Type_INT8
+Type_UINT16 = _Type_UINT16
+Type_INT16 = _Type_INT16
+Type_UINT32 = _Type_UINT32
+Type_INT32 = _Type_INT32
+Type_UINT64 = _Type_UINT64
+Type_INT64 = _Type_INT64
+Type_HALF_FLOAT = _Type_HALF_FLOAT
+Type_FLOAT = _Type_FLOAT
+Type_DOUBLE = _Type_DOUBLE
+Type_DECIMAL = _Type_DECIMAL
+Type_DATE32 = _Type_DATE32
+Type_DATE64 = _Type_DATE64
+Type_TIMESTAMP = _Type_TIMESTAMP
+Type_TIME32 = _Type_TIME32
+Type_TIME64 = _Type_TIME64
+Type_BINARY = _Type_BINARY
+Type_STRING = _Type_STRING
+Type_FIXED_SIZE_BINARY = _Type_FIXED_SIZE_BINARY
+Type_LIST = _Type_LIST
+Type_STRUCT = _Type_STRUCT
+Type_DICTIONARY = _Type_DICTIONARY
+
+
 # Exception types
 include "error.pxi"
 
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 9711b722c2f8e..a9569b2d73a46 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -21,6 +21,7 @@
 
 import six
 
+import pyarrow as pa
 from pyarrow.compat import PY2
 
 
@@ -38,17 +39,83 @@ def infer_dtype(column):
         return pd.lib.infer_dtype(column)
 
 
-def get_column_metadata(column, name):
-    inferred_dtype = infer_dtype(column)
+_logical_type_map = {}
+
+
+def get_logical_type_map():
+    global _logical_type_map
+
+    if not _logical_type_map:
+        _logical_type_map.update({
+            pa.lib.Type_NA: 'float64',  # NaNs
+            pa.lib.Type_BOOL: 'bool',
+            pa.lib.Type_INT8: 'int8',
+            pa.lib.Type_INT16: 'int16',
+            pa.lib.Type_INT32: 'int32',
+            pa.lib.Type_INT64: 'int64',
+            pa.lib.Type_UINT8: 'uint8',
+            pa.lib.Type_UINT16: 'uint16',
+            pa.lib.Type_UINT32: 'uint32',
+            pa.lib.Type_UINT64: 'uint64',
+            pa.lib.Type_HALF_FLOAT: 'float16',
+            pa.lib.Type_FLOAT: 'float32',
+            pa.lib.Type_DOUBLE: 'float64',
+            pa.lib.Type_DATE32: 'date',
+            pa.lib.Type_DATE64: 'date',
+            pa.lib.Type_BINARY: 'bytes',
+            pa.lib.Type_FIXED_SIZE_BINARY: 'bytes',
+            pa.lib.Type_STRING: 'unicode',
+        })
+    return _logical_type_map
+
+
+def get_logical_type(arrow_type):
+    logical_type_map = get_logical_type_map()
+
+    try:
+        return logical_type_map[arrow_type.id]
+    except KeyError:
+        if isinstance(arrow_type, pa.lib.DictionaryType):
+            return 'categorical'
+        elif isinstance(arrow_type, pa.lib.ListType):
+            return 'list[{}]'.format(get_logical_type(arrow_type.value_type))
+        elif isinstance(arrow_type, pa.lib.TimestampType):
+            return 'datetimetz' if arrow_type.tz is not None else 'datetime'
+        elif isinstance(arrow_type, pa.lib.DecimalType):
+            return 'decimal'
+        raise NotImplementedError(str(arrow_type))
+
+
+def get_column_metadata(column, name, arrow_type):
+    """Construct the metadata for a given column
+
+    Parameters
+    ----------
+    column : pandas.Series
+    name : str
+    arrow_type : pyarrow.DataType
+
+    Returns
+    -------
+    dict
+    """
     dtype = column.dtype
+    logical_type = get_logical_type(arrow_type)
 
     if hasattr(dtype, 'categories'):
+        assert logical_type == 'categorical'
         extra_metadata = {
             'num_categories': len(column.cat.categories),
             'ordered': column.cat.ordered,
         }
     elif hasattr(dtype, 'tz'):
+        assert logical_type == 'datetimetz'
         extra_metadata = {'timezone': str(dtype.tz)}
+    elif logical_type == 'decimal':
+        extra_metadata = {
+            'precision': arrow_type.precision,
+            'scale': arrow_type.scale,
+        }
     else:
         extra_metadata = None
 
@@ -61,25 +128,49 @@ def get_column_metadata(column, name):
 
     return {
         'name': name,
-        'pandas_type': {
-            'string': 'bytes' if PY2 else 'unicode',
-            'datetime64': (
-                'datetimetz' if hasattr(dtype, 'tz')
-                else 'datetime'
-            ),
-            'integer': str(dtype),
-            'floating': str(dtype),
-        }.get(inferred_dtype, inferred_dtype),
+        'pandas_type': logical_type,
         'numpy_type': str(dtype),
         'metadata': extra_metadata,
     }
 
 
 def index_level_name(index, i):
-    return index.name or '__index_level_{:d}__'.format(i)
+    """Return the name of an index level or a default name if `index.name` is
+    None.
 
+    Parameters
+    ----------
+    index : pandas.Index
+    i : int
+
+    Returns
+    -------
+    name : str
+    """
+    if index.name is not None:
+        return index.name
+    else:
+        return '__index_level_{:d}__'.format(i)
 
-def construct_metadata(df, index_levels, preserve_index):
+
+def construct_metadata(df, index_levels, preserve_index, types):
+    """Returns a dictionary containing enough metadata to reconstruct a pandas
+    DataFrame as an Arrow Table, including index columns.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+    index_levels : List[pd.Index]
+    presere_index : bool
+    types : List[pyarrow.DataType]
+
+    Returns
+    -------
+    dict
+    """
+    ncolumns = len(df.columns)
+    df_types = types[:ncolumns]
+    index_types = types[ncolumns:ncolumns + len(index_levels)]
     return {
         b'pandas': json.dumps(
             {
@@ -88,14 +179,22 @@ def construct_metadata(df, index_levels, preserve_index):
                     for i, level in enumerate(index_levels)
                 ] if preserve_index else [],
                 'columns': [
-                    get_column_metadata(df[name], name=name)
-                    for name in df.columns
+                    get_column_metadata(
+                        df[name],
+                        name=name,
+                        arrow_type=arrow_type
+                    )
+                    for name, arrow_type in zip(df.columns, df_types)
                 ] + (
                     [
                         get_column_metadata(
-                            level, name=index_level_name(level, i)
+                            level,
+                            name=index_level_name(level, i),
+                            arrow_type=arrow_type
+                        )
+                        for i, (level, arrow_type) in enumerate(
+                            zip(index_levels, index_types)
                         )
-                        for i, level in enumerate(index_levels)
                     ] if preserve_index else []
                 ),
                 'pandas_version': pd.__version__,
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 7b5565124339b..637b7498ecb59 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -37,6 +37,8 @@ cdef public api object pyarrow_wrap_data_type(
 
     if type.get().id() == _Type_DICTIONARY:
         out = DictionaryType()
+    elif type.get().id() == _Type_LIST:
+        out = ListType()
     elif type.get().id() == _Type_TIMESTAMP:
         out = TimestampType()
     elif type.get().id() == _Type_FIXED_SIZE_BINARY:
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index ef83636daff84..01e530651ddd3 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -321,7 +321,9 @@ cdef tuple _dataframe_to_arrays(
         list names = []
         list arrays = []
         list index_columns = []
+        list types = []
         DataType type = None
+        Array array
         dict metadata
         Py_ssize_t i
         Py_ssize_t n
@@ -336,20 +338,22 @@ cdef tuple _dataframe_to_arrays(
             field = schema.field_by_name(name)
             type = getattr(field, "type", None)
 
-        arr = arrays.append(
-            Array.from_pandas(
-                col, type=type, timestamps_to_ms=timestamps_to_ms
-            )
+        array = Array.from_pandas(
+            col, type=type, timestamps_to_ms=timestamps_to_ms
         )
+        arrays.append(array)
         names.append(name)
+        types.append(array.type)
 
     for i, column in enumerate(index_columns):
-        arrays.append(
-            Array.from_pandas(column, timestamps_to_ms=timestamps_to_ms)
-        )
+        array = Array.from_pandas(column, timestamps_to_ms=timestamps_to_ms)
+        arrays.append(array)
         names.append(pdcompat.index_level_name(column, i))
+        types.append(array.type)
 
-    metadata = pdcompat.construct_metadata(df, index_columns, preserve_index)
+    metadata = pdcompat.construct_metadata(
+        df, index_columns, preserve_index, types
+    )
     return names, arrays, metadata
 
 
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index af21741bf5e4f..e0a7416af3d3f 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -23,6 +23,7 @@
 import pandas.util.testing as tm
 
 import pyarrow as pa
+from pyarrow.pandas_compat import get_logical_type
 import pyarrow.formatting as fmt
 
 
@@ -195,3 +196,36 @@ def test_simple_type_construction():
     result = pa.lib.TimestampType()
     with pytest.raises(TypeError):
         str(result)
+
+
+@pytest.mark.parametrize(
+    ('type', 'expected'),
+    [
+        (pa.null(), 'float64'),
+        (pa.bool_(), 'bool'),
+        (pa.int8(), 'int8'),
+        (pa.int16(), 'int16'),
+        (pa.int32(), 'int32'),
+        (pa.int64(), 'int64'),
+        (pa.uint8(), 'uint8'),
+        (pa.uint16(), 'uint16'),
+        (pa.uint32(), 'uint32'),
+        (pa.uint64(), 'uint64'),
+        (pa.float16(), 'float16'),
+        (pa.float32(), 'float32'),
+        (pa.float64(), 'float64'),
+        (pa.date32(), 'date'),
+        (pa.date64(), 'date'),
+        (pa.binary(), 'bytes'),
+        (pa.binary(length=4), 'bytes'),
+        (pa.string(), 'unicode'),
+        (pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
+        (pa.decimal(18, 3), 'decimal'),
+        (pa.timestamp('ms'), 'datetime'),
+        (pa.timestamp('us', 'UTC'), 'datetimetz'),
+        pytest.mark.xfail((pa.time32('s'), None), raises=NotImplementedError),
+        pytest.mark.xfail((pa.time64('us'), None), raises=NotImplementedError),
+   ]
+)
+def test_logical_type(type, expected):
+    assert get_logical_type(type) == expected
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index fb69cac9dc339..ac4ad82f16b11 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -18,10 +18,12 @@
 
 from collections import OrderedDict
 
-import pytest
 import datetime
 import unittest
 import decimal
+import json
+
+import pytest
 
 import numpy as np
 
@@ -721,3 +723,43 @@ def test_infer_numpy_array(self):
         ])
 
         self._check_pandas_roundtrip(df, expected_schema=expected_schema)
+
+    def test_metadata_with_mixed_types(self):
+        df = pd.DataFrame({'data': [b'some_bytes', u'some_unicode']})
+        table = pa.Table.from_pandas(df)
+        metadata = table.schema.metadata
+        assert b'mixed' not in metadata[b'pandas']
+
+        js = json.loads(metadata[b'pandas'].decode('utf8'))
+        data_column = js['columns'][0]
+        assert data_column['pandas_type'] == 'bytes'
+        assert data_column['numpy_type'] == 'object'
+
+    def test_list_metadata(self):
+        df = pd.DataFrame({'data': [[1], [2, 3, 4], [5] * 7]})
+        schema = pa.schema([pa.field('data', type=pa.list_(pa.int64()))])
+        table = pa.Table.from_pandas(df, schema=schema)
+        metadata = table.schema.metadata
+        assert b'mixed' not in metadata[b'pandas']
+
+        js = json.loads(metadata[b'pandas'].decode('utf8'))
+        data_column = js['columns'][0]
+        assert data_column['pandas_type'] == 'list[int64]'
+        assert data_column['numpy_type'] == 'object'
+
+    def test_decimal_metadata(self):
+        expected = pd.DataFrame({
+            'decimals': [
+                decimal.Decimal('394092382910493.12341234678'),
+                -decimal.Decimal('314292388910493.12343437128'),
+            ]
+        })
+        table = pa.Table.from_pandas(expected)
+        metadata = table.schema.metadata
+        assert b'mixed' not in metadata[b'pandas']
+
+        js = json.loads(metadata[b'pandas'].decode('utf8'))
+        data_column = js['columns'][0]
+        assert data_column['pandas_type'] == 'decimal'
+        assert data_column['numpy_type'] == 'object'
+        assert data_column['metadata'] == {'precision': 26, 'scale': 11}

From ad57ea8ecabc38af0dc680780c33fb3cb130eb26 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 10 Jul 2017 22:27:56 -0400
Subject: [PATCH 0791/1644] ARROW-1125: Python: Add public C++ API to unwrap
 PyArrow object

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #827 from xhochy/ARROW-1125 and squashes the following commits:

5738c893 [Uwe L. Korn] ninja format
cd80f880 [Uwe L. Korn] Explicitly cast to boolean
6abad750 [Uwe L. Korn] Add to arrow::py
83d55e9f [Uwe L. Korn] ARROW-1125: Python: Add public C++ API to unwrap PyArrow object
---
 cpp/src/arrow/python/pyarrow.cc    | 117 ++++++++++++++++++++++++++++
 cpp/src/arrow/python/pyarrow.h     |  29 +++++++
 cpp/src/arrow/python/pyarrow_api.h |  54 +++++++++++++
 python/pyarrow/includes/common.pxd |   1 +
 python/pyarrow/public-api.pxi      | 120 +++++++++++++++++++++++++++++
 5 files changed, 321 insertions(+)

diff --git a/cpp/src/arrow/python/pyarrow.cc b/cpp/src/arrow/python/pyarrow.cc
index 56c0381957f01..5d88051117b78 100644
--- a/cpp/src/arrow/python/pyarrow.cc
+++ b/cpp/src/arrow/python/pyarrow.cc
@@ -35,38 +35,155 @@ int import_pyarrow() {
   return ::import_pyarrow__lib();
 }
 
+bool is_buffer(PyObject* buffer) {
+  return ::pyarrow_is_buffer(buffer) != 0;
+}
+
+Status unwrap_buffer(PyObject* buffer, std::shared_ptr<Buffer>* out) {
+  *out = ::pyarrow_unwrap_buffer(buffer);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap Buffer from the passed Python object.");
+  }
+}
+
 PyObject* wrap_buffer(const std::shared_ptr<Buffer>& buffer) {
   return ::pyarrow_wrap_buffer(buffer);
 }
 
+bool is_data_type(PyObject* data_type) {
+  return ::pyarrow_is_data_type(data_type) != 0;
+}
+
+Status unwrap_data_type(PyObject* object, std::shared_ptr<DataType>* out) {
+  *out = ::pyarrow_unwrap_data_type(object);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap DataType from the passed Python object.");
+  }
+}
+
 PyObject* wrap_data_type(const std::shared_ptr<DataType>& type) {
   return ::pyarrow_wrap_data_type(type);
 }
 
+bool is_field(PyObject* field) {
+  return ::pyarrow_is_field(field) != 0;
+}
+
+Status unwrap_field(PyObject* field, std::shared_ptr<Field>* out) {
+  *out = ::pyarrow_unwrap_field(field);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap Field from the passed Python object.");
+  }
+}
+
 PyObject* wrap_field(const std::shared_ptr<Field>& field) {
   return ::pyarrow_wrap_field(field);
 }
 
+bool is_schema(PyObject* schema) {
+  return ::pyarrow_is_schema(schema) != 0;
+}
+
+Status unwrap_schema(PyObject* schema, std::shared_ptr<Schema>* out) {
+  *out = ::pyarrow_unwrap_schema(schema);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap Schema from the passed Python object.");
+  }
+}
+
 PyObject* wrap_schema(const std::shared_ptr<Schema>& schema) {
   return ::pyarrow_wrap_schema(schema);
 }
 
+bool is_array(PyObject* array) {
+  return ::pyarrow_is_array(array) != 0;
+}
+
+Status unwrap_array(PyObject* array, std::shared_ptr<Array>* out) {
+  *out = ::pyarrow_unwrap_array(array);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap Array from the passed Python object.");
+  }
+}
+
 PyObject* wrap_array(const std::shared_ptr<Array>& array) {
   return ::pyarrow_wrap_array(array);
 }
 
+bool is_tensor(PyObject* tensor) {
+  return ::pyarrow_is_tensor(tensor) != 0;
+}
+
+Status unwrap_tensor(PyObject* tensor, std::shared_ptr<Tensor>* out) {
+  *out = ::pyarrow_unwrap_tensor(tensor);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap Tensor from the passed Python object.");
+  }
+}
+
 PyObject* wrap_tensor(const std::shared_ptr<Tensor>& tensor) {
   return ::pyarrow_wrap_tensor(tensor);
 }
 
+bool is_column(PyObject* column) {
+  return ::pyarrow_is_column(column) != 0;
+}
+
+Status unwrap_column(PyObject* column, std::shared_ptr<Column>* out) {
+  *out = ::pyarrow_unwrap_column(column);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap Column from the passed Python object.");
+  }
+}
+
 PyObject* wrap_column(const std::shared_ptr<Column>& column) {
   return ::pyarrow_wrap_column(column);
 }
 
+bool is_table(PyObject* table) {
+  return ::pyarrow_is_table(table) != 0;
+}
+
+Status unwrap_table(PyObject* table, std::shared_ptr<Table>* out) {
+  *out = ::pyarrow_unwrap_table(table);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap Table from the passed Python object.");
+  }
+}
+
 PyObject* wrap_table(const std::shared_ptr<Table>& table) {
   return ::pyarrow_wrap_table(table);
 }
 
+bool is_record_batch(PyObject* batch) {
+  return ::pyarrow_is_batch(batch) != 0;
+}
+
+Status unwrap_record_batch(PyObject* batch, std::shared_ptr<RecordBatch>* out) {
+  *out = ::pyarrow_unwrap_batch(batch);
+  if (*out) {
+    return Status::OK();
+  } else {
+    return Status::Invalid("Could not unwrap RecordBatch from the passed Python object.");
+  }
+}
+
 PyObject* wrap_record_batch(const std::shared_ptr<RecordBatch>& batch) {
   return ::pyarrow_wrap_batch(batch);
 }
diff --git a/cpp/src/arrow/python/pyarrow.h b/cpp/src/arrow/python/pyarrow.h
index 7c618ce192575..7278d1c285785 100644
--- a/cpp/src/arrow/python/pyarrow.h
+++ b/cpp/src/arrow/python/pyarrow.h
@@ -33,20 +33,49 @@ class DataType;
 class Field;
 class RecordBatch;
 class Schema;
+class Status;
 class Table;
 class Tensor;
 
 namespace py {
 
 ARROW_EXPORT int import_pyarrow();
+
+ARROW_EXPORT bool is_buffer(PyObject* buffer);
+ARROW_EXPORT Status unwrap_buffer(PyObject* buffer, std::shared_ptr<Buffer>* out);
 ARROW_EXPORT PyObject* wrap_buffer(const std::shared_ptr<Buffer>& buffer);
+
+ARROW_EXPORT bool is_data_type(PyObject* data_type);
+ARROW_EXPORT Status unwrap_data_type(PyObject* data_type, std::shared_ptr<DataType>* out);
 ARROW_EXPORT PyObject* wrap_data_type(const std::shared_ptr<DataType>& type);
+
+ARROW_EXPORT bool is_field(PyObject* field);
+ARROW_EXPORT Status unwrap_field(PyObject* field, std::shared_ptr<Field>* out);
 ARROW_EXPORT PyObject* wrap_field(const std::shared_ptr<Field>& field);
+
+ARROW_EXPORT bool is_schema(PyObject* schema);
+ARROW_EXPORT Status unwrap_schema(PyObject* schema, std::shared_ptr<Schema>* out);
 ARROW_EXPORT PyObject* wrap_schema(const std::shared_ptr<Schema>& schema);
+
+ARROW_EXPORT bool is_array(PyObject* array);
+ARROW_EXPORT Status unwrap_array(PyObject* array, std::shared_ptr<Array>* out);
 ARROW_EXPORT PyObject* wrap_array(const std::shared_ptr<Array>& array);
+
+ARROW_EXPORT bool is_tensor(PyObject* tensor);
+ARROW_EXPORT Status unwrap_tensor(PyObject* tensor, std::shared_ptr<Tensor>* out);
 ARROW_EXPORT PyObject* wrap_tensor(const std::shared_ptr<Tensor>& tensor);
+
+ARROW_EXPORT bool is_column(PyObject* column);
+ARROW_EXPORT Status unwrap_column(PyObject* column, std::shared_ptr<Column>* out);
 ARROW_EXPORT PyObject* wrap_column(const std::shared_ptr<Column>& column);
+
+ARROW_EXPORT bool is_table(PyObject* table);
+ARROW_EXPORT Status unwrap_table(PyObject* table, std::shared_ptr<Table>* out);
 ARROW_EXPORT PyObject* wrap_table(const std::shared_ptr<Table>& table);
+
+ARROW_EXPORT bool is_record_batch(PyObject* batch);
+ARROW_EXPORT Status unwrap_record_batch(
+    PyObject* batch, std::shared_ptr<RecordBatch>* out);
 ARROW_EXPORT PyObject* wrap_record_batch(const std::shared_ptr<RecordBatch>& batch);
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/pyarrow_api.h b/cpp/src/arrow/python/pyarrow_api.h
index 7b7084466109b..423595f127a56 100644
--- a/cpp/src/arrow/python/pyarrow_api.h
+++ b/cpp/src/arrow/python/pyarrow_api.h
@@ -41,6 +41,42 @@ static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_table)(std::shared_ptr
 #define pyarrow_wrap_table __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_table
 static PyObject *(*__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_batch)(std::shared_ptr< arrow::RecordBatch>  const &) = 0;
 #define pyarrow_wrap_batch __pyx_api_f_7pyarrow_3lib_pyarrow_wrap_batch
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_buffer)(PyObject *) = 0;
+#define pyarrow_is_buffer __pyx_api_f_7pyarrow_3lib_pyarrow_is_buffer
+static std::shared_ptr< arrow::Buffer>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_buffer)(PyObject *) = 0;
+#define pyarrow_unwrap_buffer __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_buffer
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_data_type)(PyObject *) = 0;
+#define pyarrow_is_data_type __pyx_api_f_7pyarrow_3lib_pyarrow_is_data_type
+static std::shared_ptr< arrow::DataType>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_data_type)(PyObject *) = 0;
+#define pyarrow_unwrap_data_type __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_data_type
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_field)(PyObject *) = 0;
+#define pyarrow_is_field __pyx_api_f_7pyarrow_3lib_pyarrow_is_field
+static std::shared_ptr< arrow::Field>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_field)(PyObject *) = 0;
+#define pyarrow_unwrap_field __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_field
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_schema)(PyObject *) = 0;
+#define pyarrow_is_schema __pyx_api_f_7pyarrow_3lib_pyarrow_is_schema
+static std::shared_ptr< arrow::Schema>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_schema)(PyObject *) = 0;
+#define pyarrow_unwrap_schema __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_schema
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_array)(PyObject *) = 0;
+#define pyarrow_is_array __pyx_api_f_7pyarrow_3lib_pyarrow_is_array
+static std::shared_ptr< arrow::Array>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_array)(PyObject *) = 0;
+#define pyarrow_unwrap_array __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_array
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_tensor)(PyObject *) = 0;
+#define pyarrow_is_tensor __pyx_api_f_7pyarrow_3lib_pyarrow_is_tensor
+static std::shared_ptr< arrow::Tensor>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_tensor)(PyObject *) = 0;
+#define pyarrow_unwrap_tensor __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_tensor
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_column)(PyObject *) = 0;
+#define pyarrow_is_column __pyx_api_f_7pyarrow_3lib_pyarrow_is_column
+static std::shared_ptr< arrow::Column>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_column)(PyObject *) = 0;
+#define pyarrow_unwrap_column __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_column
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_table)(PyObject *) = 0;
+#define pyarrow_is_table __pyx_api_f_7pyarrow_3lib_pyarrow_is_table
+static std::shared_ptr< arrow::Table>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_table)(PyObject *) = 0;
+#define pyarrow_unwrap_table __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_table
+static int (*__pyx_api_f_7pyarrow_3lib_pyarrow_is_batch)(PyObject *) = 0;
+#define pyarrow_is_batch __pyx_api_f_7pyarrow_3lib_pyarrow_is_batch
+static std::shared_ptr< arrow::RecordBatch>  (*__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_batch)(PyObject *) = 0;
+#define pyarrow_unwrap_batch __pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_batch
 #if !defined(__Pyx_PyIdentifier_FromString)
 #if PY_MAJOR_VERSION < 3
   #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s)
@@ -133,6 +169,24 @@ static int import_pyarrow__lib(void) {
   if (__Pyx_ImportFunction(module, "pyarrow_wrap_column", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_column, "PyObject *(std::shared_ptr< arrow::Column>  const &)") < 0) goto bad;
   if (__Pyx_ImportFunction(module, "pyarrow_wrap_table", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_table, "PyObject *(std::shared_ptr< arrow::Table>  const &)") < 0) goto bad;
   if (__Pyx_ImportFunction(module, "pyarrow_wrap_batch", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_wrap_batch, "PyObject *(std::shared_ptr< arrow::RecordBatch>  const &)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_buffer", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_buffer, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_buffer", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_buffer, "std::shared_ptr< arrow::Buffer>  (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_data_type", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_data_type, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_data_type", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_data_type, "std::shared_ptr< arrow::DataType>  (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_field", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_field, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_field", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_field, "std::shared_ptr< arrow::Field>  (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_schema", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_schema, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_schema", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_schema, "std::shared_ptr< arrow::Schema>  (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_array", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_array, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_array", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_array, "std::shared_ptr< arrow::Array>  (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_tensor", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_tensor, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_tensor", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_tensor, "std::shared_ptr< arrow::Tensor>  (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_column", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_column, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_column", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_column, "std::shared_ptr< arrow::Column>  (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_table", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_table, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_table", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_table, "std::shared_ptr< arrow::Table>  (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_is_batch", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_is_batch, "int (PyObject *)") < 0) goto bad;
+  if (__Pyx_ImportFunction(module, "pyarrow_unwrap_batch", (void (**)(void))&__pyx_api_f_7pyarrow_3lib_pyarrow_unwrap_batch, "std::shared_ptr< arrow::RecordBatch>  (PyObject *)") < 0) goto bad;
   Py_DECREF(module); module = 0;
   return 0;
   bad:
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 73bfb4ff50896..3487d48ce9b52 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -36,6 +36,7 @@ cdef extern from "<Python.h>":
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
     # We can later add more of the common status factory methods as needed
     cdef CStatus CStatus_OK "Status::OK"()
+    cdef CStatus CStatus_Invalid "Status::Invalid"()
 
     cdef cppclass CStatus "arrow::Status":
         CStatus()
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 637b7498ecb59..8c2083d379061 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -20,6 +20,22 @@ from pyarrow.includes.libarrow cimport (CArray, CColumn, CDataType, CField,
                                         CRecordBatch, CSchema,
                                         CTable, CTensor)
 
+# You cannot assign something to a dereferenced pointer in Cython thus these
+# methods don't use Status to indicate a successful operation.
+
+
+cdef public api bint pyarrow_is_buffer(object buffer):
+    return isinstance(buffer, Buffer)
+
+
+cdef public api shared_ptr[CBuffer] pyarrow_unwrap_buffer(object buffer):
+    cdef Buffer buf
+    if pyarrow_is_buffer(buffer):
+        buf = <Buffer>(buffer)
+        return buf.buffer
+
+    return shared_ptr[CBuffer]()
+
 
 cdef public api object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf):
     cdef Buffer result = Buffer()
@@ -27,6 +43,19 @@ cdef public api object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf):
     return result
 
 
+cdef public api bint pyarrow_is_data_type(object type_):
+    return isinstance(type_, DataType)
+
+
+cdef public api shared_ptr[CDataType] pyarrow_unwrap_data_type(object data_type):
+    cdef DataType type_
+    if pyarrow_is_data_type(data_type):
+        type_ = <DataType>(data_type)
+        return type_.sp_type
+
+    return shared_ptr[CDataType]()
+
+
 cdef public api object pyarrow_wrap_data_type(
     const shared_ptr[CDataType]& type):
     cdef:
@@ -52,6 +81,19 @@ cdef public api object pyarrow_wrap_data_type(
     return out
 
 
+cdef public api bint pyarrow_is_field(object field):
+    return isinstance(field, Field)
+
+
+cdef public api shared_ptr[CField] pyarrow_unwrap_field(object field):
+    cdef Field field_
+    if pyarrow_is_field(field):
+        field_ = <Field>(field)
+        return field_.sp_field
+
+    return shared_ptr[CField]()
+
+
 cdef public api object pyarrow_wrap_field(const shared_ptr[CField]& field):
     if field.get() == NULL:
         return None
@@ -60,12 +102,38 @@ cdef public api object pyarrow_wrap_field(const shared_ptr[CField]& field):
     return out
 
 
+cdef public api bint pyarrow_is_schema(object schema):
+    return isinstance(schema, Schema)
+
+
+cdef public api shared_ptr[CSchema] pyarrow_unwrap_schema(object schema):
+    cdef Schema sch
+    if pyarrow_is_schema(schema):
+        sch = <Schema>(schema)
+        return sch.sp_schema
+
+    return shared_ptr[CSchema]()
+
+
 cdef public api object pyarrow_wrap_schema(const shared_ptr[CSchema]& type):
     cdef Schema out = Schema()
     out.init_schema(type)
     return out
 
 
+cdef public api bint pyarrow_is_array(object array):
+    return isinstance(array, Array)
+
+
+cdef public api shared_ptr[CArray] pyarrow_unwrap_array(object array):
+    cdef Array arr
+    if pyarrow_is_array(array):
+        arr = <Array>(array)
+        return arr.sp_array
+
+    return shared_ptr[CArray]()
+
+
 cdef public api object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array):
     if sp_array.get() == NULL:
         raise ValueError('Array was NULL')
@@ -80,6 +148,19 @@ cdef public api object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array):
     return arr
 
 
+cdef public api bint pyarrow_is_tensor(object tensor):
+    return isinstance(tensor, Tensor)
+
+
+cdef public api shared_ptr[CTensor] pyarrow_unwrap_tensor(object tensor):
+    cdef Tensor ten
+    if pyarrow_is_tensor(tensor):
+        ten = <Tensor>(tensor)
+        return ten.sp_tensor
+
+    return shared_ptr[CTensor]()
+
+
 cdef public api object pyarrow_wrap_tensor(
     const shared_ptr[CTensor]& sp_tensor):
     if sp_tensor.get() == NULL:
@@ -90,18 +171,57 @@ cdef public api object pyarrow_wrap_tensor(
     return tensor
 
 
+cdef public api bint pyarrow_is_column(object column):
+    return isinstance(column, Column)
+
+
+cdef public api shared_ptr[CColumn] pyarrow_unwrap_column(object column):
+    cdef Column col
+    if pyarrow_is_column(column):
+        col = <Column>(column)
+        return col.sp_column
+
+    return shared_ptr[CColumn]()
+
+
 cdef public api object pyarrow_wrap_column(const shared_ptr[CColumn]& ccolumn):
     cdef Column column = Column()
     column.init(ccolumn)
     return column
 
 
+cdef public api bint pyarrow_is_table(object table):
+    return isinstance(table, Table)
+
+
+cdef public api shared_ptr[CTable] pyarrow_unwrap_table(object table):
+    cdef Table tab
+    if pyarrow_is_table(table):
+        tab = <Table>(table)
+        return tab.sp_table
+
+    return shared_ptr[CTable]()
+
+
 cdef public api object pyarrow_wrap_table(const shared_ptr[CTable]& ctable):
     cdef Table table = Table()
     table.init(ctable)
     return table
 
 
+cdef public api bint pyarrow_is_batch(object batch):
+    return isinstance(batch, RecordBatch)
+
+
+cdef public api shared_ptr[CRecordBatch] pyarrow_unwrap_batch(object batch):
+    cdef RecordBatch bat
+    if pyarrow_is_batch(batch):
+        bat = <RecordBatch>(batch)
+        return bat.sp_batch
+
+    return shared_ptr[CRecordBatch]()
+
+
 cdef public api object pyarrow_wrap_batch(
     const shared_ptr[CRecordBatch]& cbatch):
     cdef RecordBatch batch = RecordBatch()

From 8452071180c075d7d829d9c0a49376adb45971e0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 11 Jul 2017 01:39:20 -0400
Subject: [PATCH 0792/1644] ARROW-1199: [C++] Implement mutable POD struct for
 Array data

This data structure provides a new internal data structure that is a self-contained representation of the memory and metadata inside an Arrow array data structure.

This class is designed for easy internal data manipulation, analytical data processing, and data transport to and from IPC messages. For example, we could cast from int64 to float64 like so:

```c++
Int64Array arr = GetMyData();
std::shared_ptr<internal::ArrayData> new_data = arr->data()->ShallowCopy();
new_data->type = arrow::float64();
Float64Array double_arr(new_data);
```

This object is also useful in an analytics setting where memory may be reused. For example, if we had a group of operations all returning doubles, say:

```
Log(Sqrt(Expr(arr))
```

Then the low-level implementations of each of these functions could have the signatures

void Log(const ArrayData& values, ArrayData* out);

As another example a function may consume one or more memory buffers in an input array and replace them with newly-allocated data, changing the output data type as well.

I did quite a bit of refactoring and code simplification that was enabled by this patch. I note that performance in IPC loading of very wide record batches is about 15% slower, but in smaller record batches it is about the same in microbenchmarks. This code path could possibly be made faster with some performance analysis work.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #824 from wesm/array-data-internals and squashes the following commits:

f1acbae1 [Wes McKinney] MSVC fixes
dcdf2b29 [Wes McKinney] Fix glib per C++ API changes
d0a8ee2b [Wes McKinney] Fix logic error in UnsafeSetNotNull
d17f886c [Wes McKinney] Construct dictionary indices in ctor
bba42530 [Wes McKinney] Set correct type when creating BinaryArray
ba3b2992 [Wes McKinney] Various fixes, Python fixes, add Array operator<< to std::ostream for debugging
0b8af24a [Wes McKinney] Write field metadata directly into output object
05058638 [Wes McKinney] Fix up cmake
75bc6b4f [Wes McKinney] Delete cruft from array/loader.h and consolidate in arrow/ipc
24df1b97 [Wes McKinney] Review comments, add some doxygen comments
6e2e5720 [Wes McKinney] Preallocate vector of shared_ptr
05b806b2 [Wes McKinney] Tests passing again
5bdd6a99 [Wes McKinney] bug fixes
7894496e [Wes McKinney] Some fixes
bf91a75a [Wes McKinney] Refactor to use shared_ptr, not yet working
130f0c1a [Wes McKinney] Use std::move instead of std::forward
a9b4031b [Wes McKinney] Add move constructors to reduce unnecessary copying
475a3db6 [Wes McKinney] Bug fixes, test suite passing again
16918279 [Wes McKinney] Array internals refactoring to use POD struct for all buffers, auxiliary metadata
---
 c_glib/arrow-glib/array.cpp                 |   9 +-
 c_glib/arrow-glib/record-batch.cpp          |   3 +-
 cpp/CMakeLists.txt                          |   1 -
 cpp/cmake_modules/FindLz4.cmake             |  35 +-
 cpp/cmake_modules/FindZSTD.cmake            |  35 +-
 cpp/src/arrow/CMakeLists.txt                |   1 -
 cpp/src/arrow/api.h                         |   1 -
 cpp/src/arrow/array-test.cc                 |  71 ++--
 cpp/src/arrow/array.cc                      | 375 ++++++++++++++------
 cpp/src/arrow/array.h                       | 345 +++++++++++++-----
 cpp/src/arrow/builder.cc                    |  13 +-
 cpp/src/arrow/compare.cc                    | 155 +++++---
 cpp/src/arrow/ipc/feather-test.cc           |  19 +-
 cpp/src/arrow/ipc/feather.cc                |   5 +-
 cpp/src/arrow/ipc/json-internal.cc          |  20 +-
 cpp/src/arrow/ipc/metadata.h                |  37 +-
 cpp/src/arrow/ipc/reader.cc                 | 204 ++++++++++-
 cpp/src/arrow/ipc/test-common.h             |   8 +-
 cpp/src/arrow/ipc/writer.cc                 |  14 +-
 cpp/src/arrow/loader.cc                     | 297 ----------------
 cpp/src/arrow/loader.h                      | 124 -------
 cpp/src/arrow/pretty_print.cc               |  23 +-
 cpp/src/arrow/pretty_print.h                |   3 +
 cpp/src/arrow/python/pandas_convert.cc      |  41 +--
 cpp/src/arrow/table.cc                      |  46 ++-
 cpp/src/arrow/table.h                       |  32 +-
 python/doc/source/development.rst           |   2 +-
 python/pyarrow/array.pxi                    |   7 +-
 python/pyarrow/includes/libarrow.pxd        |   4 +
 python/pyarrow/tests/test_convert_pandas.py |  17 +
 30 files changed, 1101 insertions(+), 846 deletions(-)
 delete mode 100644 cpp/src/arrow/loader.cc
 delete mode 100644 cpp/src/arrow/loader.h

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 92a748de7a068..ab62bba3f2aba 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -38,7 +38,7 @@ garrow_array_get_values_raw(std::shared_ptr<arrow::Array> arrow_array,
   auto arrow_specific_array =
     std::static_pointer_cast<typename arrow::TypeTraits<T>::ArrayType>(arrow_array);
   *length = arrow_specific_array->length();
-  return arrow_specific_array->raw_data();
+  return arrow_specific_array->raw_values();
 };
 
 G_BEGIN_DECLS
@@ -490,7 +490,7 @@ garrow_primitive_array_get_buffer(GArrowPrimitiveArray *array)
   auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
   auto arrow_primitive_array =
     static_cast<arrow::PrimitiveArray *>(arrow_array.get());
-  auto arrow_data = arrow_primitive_array->data();
+  auto arrow_data = arrow_primitive_array->values();
   return garrow_buffer_new_raw(&arrow_data);
 }
 
@@ -1425,7 +1425,7 @@ garrow_binary_array_get_buffer(GArrowBinaryArray *array)
   auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
   auto arrow_binary_array =
     static_cast<arrow::BinaryArray *>(arrow_array.get());
-  auto arrow_data = arrow_binary_array->data();
+  auto arrow_data = arrow_binary_array->value_data();
   return garrow_buffer_new_raw(&arrow_data);
 }
 
@@ -1681,7 +1681,8 @@ garrow_struct_array_get_fields(GArrowStructArray *array)
     static_cast<const arrow::StructArray *>(arrow_array.get());
 
   GList *fields = NULL;
-  for (auto arrow_field : arrow_struct_array->fields()) {
+  for (int i = 0; i < arrow_struct_array->num_fields(); ++i) {
+    auto arrow_field = arrow_struct_array->field(i);
     GArrowArray *field = garrow_array_new_raw(&arrow_field);
     fields = g_list_prepend(fields, field);
   }
diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp
index cd030de526f4c..f381af0a2c2c7 100644
--- a/c_glib/arrow-glib/record-batch.cpp
+++ b/c_glib/arrow-glib/record-batch.cpp
@@ -219,7 +219,8 @@ garrow_record_batch_get_columns(GArrowRecordBatch *record_batch)
   const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
 
   GList *columns = NULL;
-  for (auto arrow_column : arrow_record_batch->columns()) {
+  for (int i = 0; i < arrow_record_batch->num_columns(); ++i) {
+    auto arrow_column = arrow_record_batch->column(i);
     GArrowArray *column = garrow_array_new_raw(&arrow_column);
     columns = g_list_prepend(columns, column);
   }
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 002a07e922c74..e67c7f66265f2 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -650,7 +650,6 @@ set(ARROW_SRCS
   src/arrow/buffer.cc
   src/arrow/builder.cc
   src/arrow/compare.cc
-  src/arrow/loader.cc
   src/arrow/memory_pool.cc
   src/arrow/pretty_print.cc
   src/arrow/status.cc
diff --git a/cpp/cmake_modules/FindLz4.cmake b/cpp/cmake_modules/FindLz4.cmake
index e25b0137f9776..07707cf7c6fa9 100644
--- a/cpp/cmake_modules/FindLz4.cmake
+++ b/cpp/cmake_modules/FindLz4.cmake
@@ -39,32 +39,15 @@ set(LZ4_STATIC_LIB_SUFFIX
 set(LZ4_STATIC_LIB_NAME
   ${CMAKE_STATIC_LIBRARY_PREFIX}lz4${LZ4_STATIC_LIB_SUFFIX})
 
-if ( _lz4_roots )
-  find_path(LZ4_INCLUDE_DIR NAMES lz4.h
-    PATHS ${_lz4_roots}
-    NO_DEFAULT_PATH
-    PATH_SUFFIXES "include" )
-  find_library(LZ4_SHARED_LIB NAMES lz4
-    PATHS ${_lz4_roots}
-    NO_DEFAULT_PATH
-    PATH_SUFFIXES "lib" )
-  find_library(LZ4_STATIC_LIB NAMES ${LZ4_STATIC_LIB_NAME}
-    PATHS ${_lz4_roots}
-    NO_DEFAULT_PATH
-    PATH_SUFFIXES "lib" )
-else()
-  find_path(LZ4_INCLUDE_DIR lz4.h
-    # make sure we don't accidentally pick up a different version
-    NO_CMAKE_SYSTEM_PATH
-    NO_SYSTEM_ENVIRONMENT_PATH)
-  find_library(LZ4_SHARED_LIB lz4
-    NO_CMAKE_SYSTEM_PATH
-    NO_SYSTEM_ENVIRONMENT_PATH)
-  find_library(LZ4_STATIC_LIB ${LZ4_STATIC_LIB_NAME}
-    NO_CMAKE_SYSTEM_PATH
-    NO_SYSTEM_ENVIRONMENT_PATH)
-endif()
+find_path(LZ4_INCLUDE_DIR NAMES lz4.h
+  PATHS ${_lz4_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "include" )
+find_library(LZ4_STATIC_LIB NAMES ${LZ4_STATIC_LIB_NAME} lib${LZ4_STATIC_LIB_NAME}
+  PATHS ${_lz4_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib" )
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(LZ4 REQUIRED_VARS
-  LZ4_SHARED_LIB LZ4_STATIC_LIB LZ4_INCLUDE_DIR)
+  LZ4_STATIC_LIB LZ4_INCLUDE_DIR)
diff --git a/cpp/cmake_modules/FindZSTD.cmake b/cpp/cmake_modules/FindZSTD.cmake
index 1fda29e458c31..02a0c39bb7193 100644
--- a/cpp/cmake_modules/FindZSTD.cmake
+++ b/cpp/cmake_modules/FindZSTD.cmake
@@ -39,32 +39,15 @@ set(ZSTD_STATIC_LIB_SUFFIX
 set(ZSTD_STATIC_LIB_NAME
   ${CMAKE_STATIC_LIBRARY_PREFIX}zstd${ZSTD_STATIC_LIB_SUFFIX})
 
-if ( _zstd_roots )
-  find_path(ZSTD_INCLUDE_DIR NAMES zstd.h
-    PATHS ${_zstd_roots}
-    NO_DEFAULT_PATH
-    PATH_SUFFIXES "include" )
-  find_library(ZSTD_SHARED_LIB NAMES zstd
-    PATHS ${_zstd_roots}
-    NO_DEFAULT_PATH
-    PATH_SUFFIXES "lib" )
-  find_library(ZSTD_STATIC_LIB NAMES ${ZSTD_STATIC_LIB_NAME}
-    PATHS ${_zstd_roots}
-    NO_DEFAULT_PATH
-    PATH_SUFFIXES "lib" )
-else()
-  find_path(ZSTD_INCLUDE_DIR zstd.h
-    # make sure we don't accidentally pick up a different version
-    NO_CMAKE_SYSTEM_PATH
-    NO_SYSTEM_ENVIRONMENT_PATH)
-  find_library(ZSTD_SHARED_LIB zstd
-    NO_CMAKE_SYSTEM_PATH
-    NO_SYSTEM_ENVIRONMENT_PATH)
-  find_library(ZSTD_STATIC_LIB ${ZSTD_STATIC_LIB_NAME}
-    NO_CMAKE_SYSTEM_PATH
-    NO_SYSTEM_ENVIRONMENT_PATH)
-endif()
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h
+  PATHS ${_zstd_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "include" )
+find_library(ZSTD_STATIC_LIB NAMES ${ZSTD_STATIC_LIB_NAME} lib${ZSTD_STATIC_LIB_NAME}
+  PATHS ${_zstd_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib" )
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(ZSTD REQUIRED_VARS
-  ZSTD_SHARED_LIB ZSTD_STATIC_LIB ZSTD_INCLUDE_DIR)
+  ZSTD_STATIC_LIB ZSTD_INCLUDE_DIR)
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index cb5282cbf1eff..55fab2dd18db3 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -23,7 +23,6 @@ install(FILES
   buffer.h
   builder.h
   compare.h
-  loader.h
   memory_pool.h
   pretty_print.h
   status.h
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index aa0da7580244a..731f23918e4a4 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -24,7 +24,6 @@
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
 #include "arrow/compare.h"
-#include "arrow/loader.h"
 #include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
 #include "arrow/status.h"
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 7ae03cfb88ad4..bfdb9231b5602 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -351,7 +351,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 
   for (int64_t i = 0; i < result->length(); ++i) {
     if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
-    bool actual = BitUtil::GetBit(result->data()->data(), i);
+    bool actual = BitUtil::GetBit(result->values()->data(), i);
     ASSERT_EQ(draws_[i] != 0, actual) << i;
   }
   ASSERT_TRUE(result->Equals(*expected));
@@ -778,8 +778,8 @@ TEST_F(TestStringArray, CompareNullByteSlots) {
 
   // The validity bitmaps are the same, the data is different, but the unequal
   // portion is masked out
-  StringArray equal_array(3, a1.value_offsets(), a1.data(), a2.null_bitmap(), 1);
-  StringArray equal_array2(3, a3.value_offsets(), a3.data(), a2.null_bitmap(), 1);
+  StringArray equal_array(3, a1.value_offsets(), a1.value_data(), a2.null_bitmap(), 1);
+  StringArray equal_array2(3, a3.value_offsets(), a3.value_data(), a2.null_bitmap(), 1);
 
   ASSERT_TRUE(equal_array.Equals(equal_array2));
   ASSERT_TRUE(a2.RangeEquals(equal_array2, 0, 3, 0));
@@ -846,7 +846,7 @@ TEST_F(TestStringBuilder, TestScalarAppend) {
 
   ASSERT_EQ(reps * N, result_->length());
   ASSERT_EQ(reps, result_->null_count());
-  ASSERT_EQ(reps * 6, result_->data()->size());
+  ASSERT_EQ(reps * 6, result_->value_data()->size());
 
   int32_t length;
   int32_t pos = 0;
@@ -1011,7 +1011,7 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
   ASSERT_OK(ValidateArray(*result_));
   ASSERT_EQ(reps * N, result_->length());
   ASSERT_EQ(reps, result_->null_count());
-  ASSERT_EQ(reps * 6, result_->data()->size());
+  ASSERT_EQ(reps * 6, result_->value_data()->size());
 
   int32_t length;
   for (int i = 0; i < N * reps; ++i) {
@@ -1200,8 +1200,8 @@ TEST_F(TestFWBinaryArray, EqualsRangeEquals) {
   const auto& a1 = static_cast<const FixedSizeBinaryArray&>(*array1);
   const auto& a2 = static_cast<const FixedSizeBinaryArray&>(*array2);
 
-  FixedSizeBinaryArray equal1(type, 2, a1.data(), a1.null_bitmap(), 1);
-  FixedSizeBinaryArray equal2(type, 2, a2.data(), a1.null_bitmap(), 1);
+  FixedSizeBinaryArray equal1(type, 2, a1.values(), a1.null_bitmap(), 1);
+  FixedSizeBinaryArray equal2(type, 2, a2.values(), a1.null_bitmap(), 1);
 
   ASSERT_TRUE(equal1.Equals(equal2));
   ASSERT_TRUE(equal1.RangeEquals(equal2, 0, 2, 0));
@@ -1224,7 +1224,7 @@ TEST_F(TestFWBinaryArray, ZeroSize) {
   const auto& fw_array = static_cast<const FixedSizeBinaryArray&>(*array);
 
   // data is never allocated
-  ASSERT_TRUE(fw_array.data() == nullptr);
+  ASSERT_TRUE(fw_array.values() == nullptr);
   ASSERT_EQ(0, fw_array.byte_width());
 
   ASSERT_EQ(6, array->length());
@@ -1524,8 +1524,7 @@ TYPED_TEST(TestDictionaryBuilder, Basic) {
   ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
   std::shared_ptr<Array> dict_array;
   ASSERT_OK(dict_builder.Finish(&dict_array));
-  auto dtype =
-      std::make_shared<DictionaryType>(std::make_shared<TypeParam>(), dict_array);
+  auto dtype = std::make_shared<DictionaryType>(uint8(), dict_array);
 
   UInt8Builder int_builder(default_memory_pool());
   ASSERT_OK(int_builder.Append(0));
@@ -1558,8 +1557,7 @@ TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
   ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
   std::shared_ptr<Array> dict_array;
   ASSERT_OK(dict_builder.Finish(&dict_array));
-  auto dtype =
-      std::make_shared<DictionaryType>(std::make_shared<TypeParam>(), dict_array);
+  auto dtype = std::make_shared<DictionaryType>(uint8(), dict_array);
 
   UInt8Builder int_builder(default_memory_pool());
   ASSERT_OK(int_builder.Append(0));
@@ -1601,8 +1599,7 @@ TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
     // Finalize expected data
     std::shared_ptr<Array> dict_array;
     ASSERT_OK(dict_builder.Finish(&dict_array));
-    auto dtype =
-        std::make_shared<DictionaryType>(std::make_shared<TypeParam>(), dict_array);
+    auto dtype = std::make_shared<DictionaryType>(uint16(), dict_array);
     std::shared_ptr<Array> int_array;
     ASSERT_OK(int_builder.Finish(&int_array));
 
@@ -1627,7 +1624,7 @@ TEST(TestStringDictionaryBuilder, Basic) {
   ASSERT_OK(str_builder.Append("test2"));
   std::shared_ptr<Array> str_array;
   ASSERT_OK(str_builder.Finish(&str_array));
-  auto dtype = std::make_shared<DictionaryType>(utf8(), str_array);
+  auto dtype = std::make_shared<DictionaryType>(uint8(), str_array);
 
   UInt8Builder int_builder(default_memory_pool());
   ASSERT_OK(int_builder.Append(0));
@@ -1668,7 +1665,7 @@ TEST(TestStringDictionaryBuilder, DoubleTableSize) {
   // Finalize expected data
   std::shared_ptr<Array> str_array;
   ASSERT_OK(str_builder.Finish(&str_array));
-  auto dtype = std::make_shared<DictionaryType>(utf8(), str_array);
+  auto dtype = std::make_shared<DictionaryType>(uint16(), str_array);
   std::shared_ptr<Array> int_array;
   ASSERT_OK(int_builder.Finish(&int_array));
 
@@ -1781,7 +1778,7 @@ TEST_F(TestListBuilder, TestAppendNull) {
   ASSERT_EQ(0, result_->value_offset(1));
   ASSERT_EQ(0, result_->value_offset(2));
 
-  Int32Array* values = static_cast<Int32Array*>(result_->values().get());
+  auto values = result_->values();
   ASSERT_EQ(0, values->length());
 }
 
@@ -1802,7 +1799,7 @@ void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& valu
   }
 
   ASSERT_EQ(7, result->values()->length());
-  Int32Array* varr = static_cast<Int32Array*>(result->values().get());
+  auto varr = std::dynamic_pointer_cast<Int32Array>(result->values());
 
   for (size_t i = 0; i < values.size(); ++i) {
     ASSERT_EQ(values[i], varr->Value(i));
@@ -1972,25 +1969,27 @@ TEST(TestDictionary, Validate) {
   std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
 
   std::shared_ptr<Array> indices;
-  vector<uint8_t> indices_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<UInt8Type, uint8_t>(is_valid, indices_values, &indices);
-
-  std::shared_ptr<Array> indices2;
-  vector<float> indices2_values = {1., 2., 0., 0., 2., 0.};
-  ArrayFromVector<FloatType, float>(is_valid, indices2_values, &indices2);
-
-  std::shared_ptr<Array> indices3;
-  vector<int64_t> indices3_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<Int64Type, int64_t>(is_valid, indices3_values, &indices3);
+  vector<int16_t> indices_values = {1, 2, 0, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices_values, &indices);
 
   std::shared_ptr<Array> arr = std::make_shared<DictionaryArray>(dict_type, indices);
-  std::shared_ptr<Array> arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
-  std::shared_ptr<Array> arr3 = std::make_shared<DictionaryArray>(dict_type, indices3);
 
   // Only checking index type for now
   ASSERT_OK(ValidateArray(*arr));
-  ASSERT_RAISES(Invalid, ValidateArray(*arr2));
-  ASSERT_OK(ValidateArray(*arr3));
+
+  // TODO(wesm) In ARROW-1199, there is now a DCHECK to compare the indices
+  // type with the dict_type. How can we test for this?
+
+  // std::shared_ptr<Array> indices2;
+  // vector<float> indices2_values = {1., 2., 0., 0., 2., 0.};
+  // ArrayFromVector<FloatType, float>(is_valid, indices2_values, &indices2);
+
+  // std::shared_ptr<Array> indices3;
+  // vector<int64_t> indices3_values = {1, 2, 0, 0, 2, 0};
+  // ArrayFromVector<Int64Type, int64_t>(is_valid, indices3_values, &indices3);
+  // std::shared_ptr<Array> arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
+  // std::shared_ptr<Array> arr3 = std::make_shared<DictionaryArray>(dict_type, indices3);
+  // ASSERT_OK(ValidateArray(*arr3));
 }
 
 // ----------------------------------------------------------------------
@@ -2003,9 +2002,9 @@ void ValidateBasicStructArray(const StructArray* result,
   ASSERT_EQ(4, result->length());
   ASSERT_OK(ValidateArray(*result));
 
-  auto list_char_arr = static_cast<ListArray*>(result->field(0).get());
-  auto char_arr = static_cast<Int8Array*>(list_char_arr->values().get());
-  auto int32_arr = static_cast<Int32Array*>(result->field(1).get());
+  auto list_char_arr = std::dynamic_pointer_cast<ListArray>(result->field(0));
+  auto char_arr = std::dynamic_pointer_cast<Int8Array>(list_char_arr->values());
+  auto int32_arr = std::dynamic_pointer_cast<Int32Array>(result->field(1));
 
   ASSERT_EQ(0, result->null_count());
   ASSERT_EQ(1, list_char_arr->null_count());
@@ -2086,7 +2085,7 @@ TEST_F(TestStructBuilder, TestAppendNull) {
 
   ASSERT_OK(ValidateArray(*result_));
 
-  ASSERT_EQ(2, static_cast<int>(result_->fields().size()));
+  ASSERT_EQ(2, static_cast<int>(result_->num_fields()));
   ASSERT_EQ(2, result_->length());
   ASSERT_EQ(2, result_->field(0)->length());
   ASSERT_EQ(2, result_->field(1)->length());
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index c5acf3ee6aa4b..48a3bd5a635a4 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -24,46 +24,34 @@
 
 #include "arrow/buffer.h"
 #include "arrow/compare.h"
+#include "arrow/pretty_print.h"
 #include "arrow/status.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
 #include "arrow/visitor.h"
 #include "arrow/visitor_inline.h"
 
 namespace arrow {
 
-// When slicing, we do not know the null count of the sliced range without
-// doing some computation. To avoid doing this eagerly, we set the null count
-// to -1 (any negative number will do). When Array::null_count is called the
-// first time, the null count will be computed. See ARROW-33
-constexpr int64_t kUnknownNullCount = -1;
+using internal::ArrayData;
 
 // ----------------------------------------------------------------------
 // Base array class
 
-Array::Array(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
-    : type_(type),
-      length_(length),
-      offset_(offset),
-      null_count_(null_count),
-      null_bitmap_(null_bitmap),
-      null_bitmap_data_(nullptr) {
-  if (null_count_ == 0) { null_bitmap_ = nullptr; }
-  if (null_bitmap_) { null_bitmap_data_ = null_bitmap_->data(); }
-}
-
 int64_t Array::null_count() const {
-  if (null_count_ < 0) {
-    if (null_bitmap_) {
-      null_count_ = length_ - CountSetBits(null_bitmap_data_, offset_, length_);
+  if (ARROW_PREDICT_FALSE(data_->null_count < 0)) {
+    if (data_->buffers[0]) {
+      data_->null_count =
+          data_->length - CountSetBits(null_bitmap_data_, data_->offset, data_->length);
+
     } else {
-      null_count_ = 0;
+      data_->null_count = 0;
     }
   }
-  return null_count_;
+  return data_->null_count;
 }
 
 bool Array::Equals(const Array& arr) const {
@@ -115,15 +103,34 @@ static inline void ConformSliceParams(
 }
 
 std::shared_ptr<Array> Array::Slice(int64_t offset) const {
-  int64_t slice_length = length_ - offset;
+  int64_t slice_length = data_->length - offset;
   return Slice(offset, slice_length);
 }
 
-NullArray::NullArray(int64_t length) : Array(null(), length, nullptr, length) {}
+std::ostream& operator<<(std::ostream& os, const Array& x) {
+  DCHECK(PrettyPrint(x, 0, &os).ok());
+  return os;
+}
+
+static inline std::shared_ptr<ArrayData> SliceData(
+    const ArrayData& data, int64_t offset, int64_t length) {
+  ConformSliceParams(data.offset, data.length, &offset, &length);
+
+  auto new_data = data.ShallowCopy();
+  new_data->length = length;
+  new_data->offset = offset;
+  new_data->null_count = kUnknownNullCount;
+  return new_data;
+}
+
+NullArray::NullArray(int64_t length) {
+  BufferVector buffers = {nullptr};
+  SetData(std::make_shared<ArrayData>(null(), length, std::move(buffers), length));
+}
 
 std::shared_ptr<Array> NullArray::Slice(int64_t offset, int64_t length) const {
-  DCHECK_LE(offset, length_);
-  length = std::min(length_ - offset, length);
+  DCHECK_LE(offset, data_->length);
+  length = std::min(data_->length - offset, length);
   return std::make_shared<NullArray>(length);
 }
 
@@ -132,40 +139,78 @@ std::shared_ptr<Array> NullArray::Slice(int64_t offset, int64_t length) const {
 
 PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset)
-    : Array(type, length, null_bitmap, null_count, offset) {
-  data_ = data;
-  raw_data_ = data == nullptr ? nullptr : data_->data();
+    int64_t null_count, int64_t offset) {
+  BufferVector buffers = {null_bitmap, data};
+  SetData(
+      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+}
+
+template <typename T>
+NumericArray<T>::NumericArray(const std::shared_ptr<internal::ArrayData>& data)
+    : PrimitiveArray(data) {
+  DCHECK_EQ(data->type->id(), T::type_id);
 }
 
 template <typename T>
 std::shared_ptr<Array> NumericArray<T>::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<NumericArray<T>>(
-      type_, length, data_, null_bitmap_, kUnknownNullCount, offset);
+  return std::make_shared<NumericArray<T>>(SliceData(*data_, offset, length));
 }
 
 // ----------------------------------------------------------------------
 // BooleanArray
 
+BooleanArray::BooleanArray(const std::shared_ptr<internal::ArrayData>& data)
+    : PrimitiveArray(data) {
+  DCHECK_EQ(data->type->id(), Type::BOOL);
+}
+
 BooleanArray::BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
     const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
-    : PrimitiveArray(std::make_shared<BooleanType>(), length, data, null_bitmap,
-          null_count, offset) {}
+    : PrimitiveArray(boolean(), length, data, null_bitmap, null_count, offset) {}
 
 std::shared_ptr<Array> BooleanArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<BooleanArray>(
-      length, data_, null_bitmap_, kUnknownNullCount, offset);
+  return std::make_shared<BooleanArray>(SliceData(*data_, offset, length));
 }
 
 // ----------------------------------------------------------------------
 // ListArray
 
+ListArray::ListArray(const std::shared_ptr<ArrayData>& data) {
+  DCHECK_EQ(data->type->id(), Type::LIST);
+  SetData(data);
+}
+
+ListArray::ListArray(const std::shared_ptr<DataType>& type, int64_t length,
+    const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Array>& values,
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset) {
+  BufferVector buffers = {null_bitmap, value_offsets};
+  auto internal_data =
+      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
+  internal_data->child_data.emplace_back(values->data());
+  SetData(internal_data);
+}
+
+void ListArray::SetData(const std::shared_ptr<ArrayData>& data) {
+  this->Array::SetData(data);
+  auto value_offsets = data->buffers[1];
+  raw_value_offsets_ = value_offsets == nullptr
+                           ? nullptr
+                           : reinterpret_cast<const int32_t*>(value_offsets->data());
+  DCHECK(internal::MakeArray(data_->child_data[0], &values_).ok());
+}
+
+std::shared_ptr<DataType> ListArray::value_type() const {
+  return static_cast<const ListType&>(*type()).value_type();
+}
+
+std::shared_ptr<Array> ListArray::values() const {
+  return values_;
+}
+
 std::shared_ptr<Array> ListArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<ListArray>(
-      type_, length, value_offsets_, values_, null_bitmap_, kUnknownNullCount, offset);
+  ConformSliceParams(data_->offset, data_->length, &offset, &length);
+  return std::make_shared<ListArray>(type(), length, value_offsets(), values(),
+      null_bitmap(), kUnknownNullCount, offset);
 }
 
 // ----------------------------------------------------------------------
@@ -174,6 +219,21 @@ std::shared_ptr<Array> ListArray::Slice(int64_t offset, int64_t length) const {
 static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
 static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
 
+BinaryArray::BinaryArray(const std::shared_ptr<internal::ArrayData>& data) {
+  DCHECK_EQ(data->type->id(), Type::BINARY);
+  SetData(data);
+}
+
+void BinaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
+  auto value_offsets = data->buffers[1];
+  auto value_data = data->buffers[2];
+  this->Array::SetData(data);
+  raw_data_ = value_data == nullptr ? nullptr : value_data->data();
+  raw_value_offsets_ = value_offsets == nullptr
+                           ? nullptr
+                           : reinterpret_cast<const int32_t*>(value_offsets->data());
+}
+
 BinaryArray::BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
     const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
     int64_t null_count, int64_t offset)
@@ -182,22 +242,19 @@ BinaryArray::BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_of
 
 BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
-    : Array(type, length, null_bitmap, null_count, offset),
-      value_offsets_(value_offsets),
-      raw_value_offsets_(nullptr),
-      data_(data),
-      raw_data_(nullptr) {
-  if (value_offsets_ != nullptr) {
-    raw_value_offsets_ = reinterpret_cast<const int32_t*>(value_offsets_->data());
-  }
-  if (data_ != nullptr) { raw_data_ = data_->data(); }
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset) {
+  BufferVector buffers = {null_bitmap, value_offsets, data};
+  SetData(
+      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
 }
 
 std::shared_ptr<Array> BinaryArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<BinaryArray>(
-      length, value_offsets_, data_, null_bitmap_, kUnknownNullCount, offset);
+  return std::make_shared<BinaryArray>(SliceData(*data_, offset, length));
+}
+
+StringArray::StringArray(const std::shared_ptr<internal::ArrayData>& data) {
+  DCHECK_EQ(data->type->id(), Type::STRING);
+  SetData(data);
 }
 
 StringArray::StringArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
@@ -207,14 +264,18 @@ StringArray::StringArray(int64_t length, const std::shared_ptr<Buffer>& value_of
 }
 
 std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<StringArray>(
-      length, value_offsets_, data_, null_bitmap_, kUnknownNullCount, offset);
+  return std::make_shared<StringArray>(SliceData(*data_, offset, length));
 }
 
 // ----------------------------------------------------------------------
 // Fixed width binary
 
+FixedSizeBinaryArray::FixedSizeBinaryArray(
+    const std::shared_ptr<internal::ArrayData>& data) {
+  DCHECK_EQ(data->type->id(), Type::FIXED_SIZE_BINARY);
+  SetData(data);
+}
+
 FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<DataType>& type,
     int64_t length, const std::shared_ptr<Buffer>& data,
     const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
@@ -222,34 +283,52 @@ FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<DataType>& type
       byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {}
 
 std::shared_ptr<Array> FixedSizeBinaryArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<FixedSizeBinaryArray>(
-      type_, length, data_, null_bitmap_, kUnknownNullCount, offset);
+  return std::make_shared<FixedSizeBinaryArray>(SliceData(*data_, offset, length));
 }
 
 const uint8_t* FixedSizeBinaryArray::GetValue(int64_t i) const {
-  return raw_data_ + (i + offset_) * byte_width_;
+  return raw_values_ + (i + data_->offset) * byte_width_;
 }
 
 // ----------------------------------------------------------------------
 // Decimal
+
+DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data) {
+  DCHECK_EQ(data->type->id(), Type::DECIMAL);
+  SetData(data);
+}
+
+void DecimalArray::SetData(const std::shared_ptr<ArrayData>& data) {
+  auto fixed_size_data = data->buffers[1];
+  auto sign_bitmap = data->buffers[2];
+  this->Array::SetData(data);
+
+  raw_values_ = fixed_size_data != nullptr ? fixed_size_data->data() : nullptr;
+  sign_bitmap_data_ = sign_bitmap != nullptr ? sign_bitmap->data() : nullptr;
+}
+
 DecimalArray::DecimalArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset, const std::shared_ptr<Buffer>& sign_bitmap)
-    : FixedSizeBinaryArray(type, length, data, null_bitmap, null_count, offset),
-      sign_bitmap_(sign_bitmap),
-      sign_bitmap_data_(sign_bitmap != nullptr ? sign_bitmap->data() : nullptr) {}
+    int64_t null_count, int64_t offset, const std::shared_ptr<Buffer>& sign_bitmap) {
+  BufferVector buffers = {null_bitmap, data, sign_bitmap};
+  SetData(
+      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+}
 
 bool DecimalArray::IsNegative(int64_t i) const {
   return sign_bitmap_data_ != nullptr ? BitUtil::GetBit(sign_bitmap_data_, i) : false;
 }
 
+const uint8_t* DecimalArray::GetValue(int64_t i) const {
+  return raw_values_ + (i + data_->offset) * byte_width();
+}
+
 std::string DecimalArray::FormatValue(int64_t i) const {
-  const auto type_ = std::dynamic_pointer_cast<DecimalType>(type());
-  const int precision = type_->precision();
-  const int scale = type_->scale();
-  const int byte_width = byte_width_;
-  const uint8_t* bytes = GetValue(i);
+  const auto& type_ = static_cast<const DecimalType&>(*type());
+  const int precision = type_.precision();
+  const int scale = type_.scale();
+  const int byte_width = type_.byte_width();
+  const uint8_t* bytes = raw_values_ + (i + data_->offset) * byte_width;
   switch (byte_width) {
     case 4: {
       decimal::Decimal32 value;
@@ -274,73 +353,110 @@ std::string DecimalArray::FormatValue(int64_t i) const {
 }
 
 std::shared_ptr<Array> DecimalArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<DecimalArray>(
-      type_, length, data_, null_bitmap_, kUnknownNullCount, offset, sign_bitmap_);
+  return std::make_shared<DecimalArray>(SliceData(*data_, offset, length));
 }
 
 // ----------------------------------------------------------------------
 // Struct
 
+StructArray::StructArray(const std::shared_ptr<ArrayData>& data) {
+  DCHECK_EQ(data->type->id(), Type::STRUCT);
+  SetData(data);
+}
+
 StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::vector<std::shared_ptr<Array>>& children,
-    std::shared_ptr<Buffer> null_bitmap, int64_t null_count, int64_t offset)
-    : Array(type, length, null_bitmap, null_count, offset) {
-  type_ = type;
-  children_ = children;
+    std::shared_ptr<Buffer> null_bitmap, int64_t null_count, int64_t offset) {
+  BufferVector buffers = {null_bitmap};
+  SetData(
+      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+  for (const auto& child : children) {
+    data_->child_data.push_back(child->data());
+  }
 }
 
 std::shared_ptr<Array> StructArray::field(int pos) const {
-  DCHECK_GT(children_.size(), 0);
-  return children_[pos];
+  std::shared_ptr<Array> result;
+  DCHECK(internal::MakeArray(data_->child_data[pos], &result).ok());
+  return result;
 }
 
 std::shared_ptr<Array> StructArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<StructArray>(
-      type_, length, children_, null_bitmap_, kUnknownNullCount, offset);
+  return std::make_shared<StructArray>(SliceData(*data_, offset, length));
 }
 
 // ----------------------------------------------------------------------
 // UnionArray
 
+void UnionArray::SetData(const std::shared_ptr<ArrayData>& data) {
+  this->Array::SetData(data);
+
+  auto type_ids = data_->buffers[1];
+  auto value_offsets = data_->buffers[2];
+  raw_type_ids_ =
+      type_ids == nullptr ? nullptr : reinterpret_cast<const uint8_t*>(type_ids->data());
+  raw_value_offsets_ = value_offsets == nullptr
+                           ? nullptr
+                           : reinterpret_cast<const int32_t*>(value_offsets->data());
+}
+
+UnionArray::UnionArray(const std::shared_ptr<ArrayData>& data) {
+  DCHECK_EQ(data->type->id(), Type::UNION);
+  SetData(data);
+}
+
 UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
     const std::vector<std::shared_ptr<Array>>& children,
     const std::shared_ptr<Buffer>& type_ids, const std::shared_ptr<Buffer>& value_offsets,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
-    : Array(type, length, null_bitmap, null_count, offset),
-      children_(children),
-      type_ids_(type_ids),
-      raw_type_ids_(nullptr),
-      value_offsets_(value_offsets),
-      raw_value_offsets_(nullptr) {
-  if (type_ids) { raw_type_ids_ = reinterpret_cast<const uint8_t*>(type_ids->data()); }
-  if (value_offsets) {
-    raw_value_offsets_ = reinterpret_cast<const int32_t*>(value_offsets->data());
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset) {
+  BufferVector buffers = {null_bitmap, type_ids, value_offsets};
+  auto internal_data =
+      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
+  for (const auto& child : children) {
+    internal_data->child_data.push_back(child->data());
   }
+  SetData(internal_data);
 }
 
 std::shared_ptr<Array> UnionArray::child(int pos) const {
-  DCHECK_GT(children_.size(), 0);
-  return children_[pos];
+  std::shared_ptr<Array> result;
+  DCHECK(internal::MakeArray(data_->child_data[pos], &result).ok());
+  return result;
 }
 
 std::shared_ptr<Array> UnionArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(offset_, length_, &offset, &length);
-  return std::make_shared<UnionArray>(type_, length, children_, type_ids_, value_offsets_,
-      null_bitmap_, kUnknownNullCount, offset);
+  return std::make_shared<UnionArray>(SliceData(*data_, offset, length));
 }
 
 // ----------------------------------------------------------------------
 // DictionaryArray
 
+DictionaryArray::DictionaryArray(const std::shared_ptr<ArrayData>& data)
+    : dict_type_(static_cast<const DictionaryType*>(data->type.get())) {
+  DCHECK_EQ(data->type->id(), Type::DICTIONARY);
+  SetData(data);
+}
+
 DictionaryArray::DictionaryArray(
     const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices)
-    : Array(type, indices->length(), indices->null_bitmap(), indices->null_count(),
-          indices->offset()),
-      dict_type_(static_cast<const DictionaryType*>(type.get())),
-      indices_(indices) {
+    : dict_type_(static_cast<const DictionaryType*>(type.get())) {
   DCHECK_EQ(type->id(), Type::DICTIONARY);
+  DCHECK_EQ(indices->type_id(), dict_type_->index_type()->id());
+  auto data = indices->data()->ShallowCopy();
+  data->type = type;
+  SetData(data);
+}
+
+void DictionaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
+  this->Array::SetData(data);
+  auto indices_data = data_->ShallowCopy();
+  indices_data->type = dict_type_->index_type();
+  std::shared_ptr<Array> result;
+  DCHECK(internal::MakeArray(indices_data, &indices_).ok());
+}
+
+std::shared_ptr<Array> DictionaryArray::indices() const {
+  return indices_;
 }
 
 std::shared_ptr<Array> DictionaryArray::dictionary() const {
@@ -348,8 +464,7 @@ std::shared_ptr<Array> DictionaryArray::dictionary() const {
 }
 
 std::shared_ptr<Array> DictionaryArray::Slice(int64_t offset, int64_t length) const {
-  std::shared_ptr<Array> sliced_indices = indices_->Slice(offset, length);
-  return std::make_shared<DictionaryArray>(type_, sliced_indices);
+  return std::make_shared<DictionaryArray>(SliceData(*data_, offset, length));
 }
 
 // ----------------------------------------------------------------------
@@ -367,6 +482,8 @@ struct ValidateVisitor {
 
   Status Visit(const PrimitiveArray& array) { return Status::OK(); }
 
+  Status Visit(const DecimalArray& array) { return Status::OK(); }
+
   Status Visit(const BinaryArray& array) {
     // TODO(wesm): what to do here?
     return Status::OK();
@@ -435,11 +552,12 @@ struct ValidateVisitor {
       return Status::Invalid("Null count exceeds the length of this struct");
     }
 
-    if (array.fields().size() > 0) {
+    if (array.num_fields() > 0) {
       // Validate fields
-      int64_t array_length = array.fields()[0]->length();
+      int64_t array_length = array.field(0)->length();
       size_t idx = 0;
-      for (auto it : array.fields()) {
+      for (int i = 0; i < array.num_fields(); ++i) {
+        auto it = array.field(i);
         if (it->length() != array_length) {
           std::stringstream ss;
           ss << "Length is not equal from field " << it->type()->ToString()
@@ -487,6 +605,51 @@ Status ValidateArray(const Array& array) {
   return VisitArrayInline(array, &validate_visitor);
 }
 
+// ----------------------------------------------------------------------
+// Loading from ArrayData
+
+namespace internal {
+
+class ArrayDataWrapper {
+ public:
+  ArrayDataWrapper(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out)
+      : data_(data), out_(out) {}
+
+  template <typename T>
+  Status Visit(const T& type) {
+    using ArrayType = typename TypeTraits<T>::ArrayType;
+    *out_ = std::make_shared<ArrayType>(data_);
+    return Status::OK();
+  }
+
+  const std::shared_ptr<ArrayData>& data_;
+  std::shared_ptr<Array>* out_;
+};
+
+Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out) {
+  ArrayDataWrapper wrapper_visitor(data, out);
+  return VisitTypeInline(*data->type, &wrapper_visitor);
+}
+
+}  // namespace internal
+
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
+    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
+    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
+  BufferVector buffers = {null_bitmap, data};
+  auto internal_data = std::make_shared<internal::ArrayData>(
+      type, length, std::move(buffers), null_count, offset);
+  return internal::MakeArray(internal_data, out);
+}
+
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
+    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
+  auto internal_data =
+      std::make_shared<internal::ArrayData>(type, length, buffers, null_count, offset);
+  return internal::MakeArray(internal_data, out);
+}
+
 // ----------------------------------------------------------------------
 // Instantiate templates
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 59269adcb913e..80284cdc2dff7 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -35,13 +35,125 @@
 
 namespace arrow {
 
+using BufferVector = std::vector<std::shared_ptr<Buffer>>;
+
+// When slicing, we do not know the null count of the sliced range without
+// doing some computation. To avoid doing this eagerly, we set the null count
+// to -1 (any negative number will do). When Array::null_count is called the
+// first time, the null count will be computed. See ARROW-33
+constexpr int64_t kUnknownNullCount = -1;
+
 class MemoryPool;
-class MutableBuffer;
 class Status;
 
 template <typename T>
 struct Decimal;
 
+// ----------------------------------------------------------------------
+// Generic array data container
+
+namespace internal {
+
+/// \brief Mutable internal container for generic Arrow array data
+///
+/// This data structure is a self-contained representation of the memory and
+/// metadata inside an Arrow array data structure (called vectors in Java). The
+/// classes arrow::Array and its subclasses provide strongly-typed accessors
+/// with support for the visitor pattern and other affordances.
+///
+/// This class is designed for easy internal data manipulation, analytical data
+/// processing, and data transport to and from IPC messages. For example, we
+/// could cast from int64 to float64 like so:
+///
+/// Int64Array arr = GetMyData();
+/// auto new_data = arr->data()->ShallowCopy();
+/// new_data->type = arrow::float64();
+/// Float64Array double_arr(new_data);
+///
+/// This object is also useful in an analytics setting where memory may be
+/// reused. For example, if we had a group of operations all returning doubles,
+/// say:
+///
+/// Log(Sqrt(Expr(arr))
+///
+/// Then the low-level implementations of each of these functions could have
+/// the signatures
+///
+/// void Log(const ArrayData& values, ArrayData* out);
+///
+/// As another example a function may consume one or more memory buffers in an
+/// input array and replace them with newly-allocated data, changing the output
+/// data type as well.
+struct ARROW_EXPORT ArrayData {
+  ArrayData() {}
+
+  ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
+      const std::vector<std::shared_ptr<Buffer>>& buffers,
+      int64_t null_count = kUnknownNullCount, int64_t offset = 0)
+      : type(type),
+        length(length),
+        buffers(buffers),
+        null_count(null_count),
+        offset(offset) {}
+
+  ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
+      std::vector<std::shared_ptr<Buffer>>&& buffers,
+      int64_t null_count = kUnknownNullCount, int64_t offset = 0)
+      : type(type),
+        length(length),
+        buffers(std::move(buffers)),
+        null_count(null_count),
+        offset(offset) {}
+
+  // Move constructor
+  ArrayData(ArrayData&& other) noexcept
+      : type(std::move(other.type)),
+        length(other.length),
+        buffers(std::move(other.buffers)),
+        null_count(other.null_count),
+        offset(other.offset),
+        child_data(std::move(other.child_data)) {}
+
+  ArrayData(const ArrayData& other) noexcept
+      : type(other.type),
+        length(other.length),
+        buffers(other.buffers),
+        null_count(other.null_count),
+        offset(other.offset),
+        child_data(other.child_data) {}
+
+  // Move assignment
+  ArrayData& operator=(ArrayData&& other) {
+    type = std::move(other.type);
+    length = other.length;
+    buffers = std::move(other.buffers);
+    null_count = other.null_count;
+    offset = other.offset;
+    child_data = std::move(other.child_data);
+    return *this;
+  }
+
+  std::shared_ptr<ArrayData> ShallowCopy() const {
+    return std::make_shared<ArrayData>(*this);
+  }
+
+  std::shared_ptr<DataType> type;
+  int64_t length;
+  std::vector<std::shared_ptr<Buffer>> buffers;
+  int64_t null_count;
+  int64_t offset;
+  std::vector<std::shared_ptr<ArrayData>> child_data;
+};
+
+Status ARROW_EXPORT MakeArray(
+    const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out);
+
+}  // namespace internal
+
+// ----------------------------------------------------------------------
+// User array accessor types
+
+/// \brief Array base type
 /// Immutable data array with some logical type and some length.
 ///
 /// Any memory is owned by the respective Buffer instance (or its parents).
@@ -54,24 +166,20 @@ struct Decimal;
 /// be computed on the first call to null_count()
 class ARROW_EXPORT Array {
  public:
-  Array(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
-
   virtual ~Array() = default;
 
   /// Determine if a slot is null. For inner loops. Does *not* boundscheck
   bool IsNull(int64_t i) const {
     return null_bitmap_data_ != nullptr &&
-           BitUtil::BitNotSet(null_bitmap_data_, i + offset_);
+           BitUtil::BitNotSet(null_bitmap_data_, i + data_->offset);
   }
 
   /// Size in the number of elements this array contains.
-  int64_t length() const { return length_; }
+  int64_t length() const { return data_->length; }
 
   /// A relative position into another array's data, to enable zero-copy
   /// slicing. This value defaults to zero
-  int64_t offset() const { return offset_; }
+  int64_t offset() const { return data_->offset; }
 
   /// The number of null entries in the array. If the null count was not known
   /// at time of construction (and set to a negative value), then the null
@@ -79,14 +187,14 @@ class ARROW_EXPORT Array {
   /// function
   int64_t null_count() const;
 
-  std::shared_ptr<DataType> type() const { return type_; }
-  Type::type type_id() const { return type_->id(); }
+  std::shared_ptr<DataType> type() const { return data_->type; }
+  Type::type type_id() const { return data_->type->id(); }
 
   /// Buffer for the null bitmap.
   ///
   /// Note that for `null_count == 0`, this can be a `nullptr`.
   /// This buffer does not account for any slice offset
-  std::shared_ptr<Buffer> null_bitmap() const { return null_bitmap_; }
+  std::shared_ptr<Buffer> null_bitmap() const { return data_->buffers[0]; }
 
   /// Raw pointer to the null bitmap.
   ///
@@ -124,49 +232,77 @@ class ARROW_EXPORT Array {
   /// Slice from offset until end of the array
   std::shared_ptr<Array> Slice(int64_t offset) const;
 
- protected:
-  std::shared_ptr<DataType> type_;
-  int64_t length_;
-  int64_t offset_;
+  std::shared_ptr<internal::ArrayData> data() const { return data_; }
+
+  int num_fields() const { return static_cast<int>(data_->child_data.size()); }
 
-  // This member is marked mutable so that it can be modified when null_count()
-  // is called from a const context and the null count has to be computed (if
-  // it is not already known)
-  mutable int64_t null_count_;
+ protected:
+  Array() {}
 
-  std::shared_ptr<Buffer> null_bitmap_;
+  std::shared_ptr<internal::ArrayData> data_;
   const uint8_t* null_bitmap_data_;
 
+  /// Protected method for constructors
+  inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
+    if (data->buffers.size() > 0 && data->buffers[0]) {
+      null_bitmap_data_ = data->buffers[0]->data();
+    } else {
+      null_bitmap_data_ = nullptr;
+    }
+    data_ = data;
+  }
+
  private:
-  Array() {}
   DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
+ARROW_EXPORT std::ostream& operator<<(std::ostream& os, const Array& x);
+
+class ARROW_EXPORT FlatArray : public Array {
+ protected:
+  using Array::Array;
+};
+
 /// Degenerate null type Array
-class ARROW_EXPORT NullArray : public Array {
+class ARROW_EXPORT NullArray : public FlatArray {
  public:
   using TypeClass = NullType;
 
+  explicit NullArray(const std::shared_ptr<internal::ArrayData>& data) { SetData(data); }
+
   explicit NullArray(int64_t length);
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
 /// Base class for fixed-size logical types
-class ARROW_EXPORT PrimitiveArray : public Array {
+class ARROW_EXPORT PrimitiveArray : public FlatArray {
  public:
   PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& data,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0);
 
-  /// The memory containing this array's data
-  /// This buffer does not account for any slice offset
-  std::shared_ptr<Buffer> data() const { return data_; }
+  /// Does not account for any slice offset
+  std::shared_ptr<Buffer> values() const { return data_->buffers[1]; }
+
+  /// Does not account for any slice offset
+  const uint8_t* raw_values() const { return raw_values_; }
 
  protected:
-  std::shared_ptr<Buffer> data_;
-  const uint8_t* raw_data_;
+  PrimitiveArray() {}
+
+  inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
+    auto values = data->buffers[1];
+    this->Array::SetData(data);
+    raw_values_ = values == nullptr ? nullptr : values->data();
+  }
+
+  explicit inline PrimitiveArray(const std::shared_ptr<internal::ArrayData>& data) {
+    SetData(data);
+  }
+
+  const uint8_t* raw_values_;
 };
 
 template <typename TYPE>
@@ -175,7 +311,7 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   using TypeClass = TYPE;
   using value_type = typename TypeClass::c_type;
 
-  using PrimitiveArray::PrimitiveArray;
+  explicit NumericArray(const std::shared_ptr<internal::ArrayData>& data);
 
   // Only enable this constructor without a type argument for types without additional
   // metadata
@@ -188,20 +324,23 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
       : PrimitiveArray(TypeTraits<T1>::type_singleton(), length, data, null_bitmap,
             null_count, offset) {}
 
-  const value_type* raw_data() const {
-    return reinterpret_cast<const value_type*>(raw_data_) + offset_;
+  const value_type* raw_values() const {
+    return reinterpret_cast<const value_type*>(raw_values_) + data_->offset;
   }
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
-  value_type Value(int64_t i) const { return raw_data()[i]; }
+  value_type Value(int64_t i) const { return raw_values()[i]; }
+
+ protected:
+  using PrimitiveArray::PrimitiveArray;
 };
 
 class ARROW_EXPORT BooleanArray : public PrimitiveArray {
  public:
   using TypeClass = BooleanType;
 
-  using PrimitiveArray::PrimitiveArray;
+  explicit BooleanArray(const std::shared_ptr<internal::ArrayData>& data);
 
   BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
@@ -210,8 +349,12 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
   bool Value(int64_t i) const {
-    return BitUtil::GetBit(reinterpret_cast<const uint8_t*>(raw_data_), i + offset_);
+    return BitUtil::GetBit(
+        reinterpret_cast<const uint8_t*>(raw_values_), i + data_->offset);
   }
+
+ protected:
+  using PrimitiveArray::PrimitiveArray;
 };
 
 // ----------------------------------------------------------------------
@@ -221,52 +364,50 @@ class ARROW_EXPORT ListArray : public Array {
  public:
   using TypeClass = ListType;
 
+  explicit ListArray(const std::shared_ptr<internal::ArrayData>& data);
+
   ListArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Array>& values,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0)
-      : Array(type, length, null_bitmap, null_count, offset) {
-    value_offsets_ = value_offsets;
-    raw_value_offsets_ = value_offsets == nullptr
-                             ? nullptr
-                             : reinterpret_cast<const int32_t*>(value_offsets_->data());
-    values_ = values;
-  }
+      int64_t offset = 0);
 
-  // Return a shared pointer in case the requestor desires to share ownership
-  // with this array.
-  std::shared_ptr<Array> values() const { return values_; }
+  /// \brief Return array object containing the list's values
+  std::shared_ptr<Array> values() const;
 
   /// Note that this buffer does not account for any slice offset
-  std::shared_ptr<Buffer> value_offsets() const { return value_offsets_; }
+  std::shared_ptr<Buffer> value_offsets() const { return data_->buffers[1]; }
 
-  std::shared_ptr<DataType> value_type() const { return values_->type(); }
+  std::shared_ptr<DataType> value_type() const;
 
   /// Return pointer to raw value offsets accounting for any slice offset
-  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
+  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + data_->offset; }
 
   // Neither of these functions will perform boundschecking
-  int32_t value_offset(int64_t i) const { return raw_value_offsets_[i + offset_]; }
+  int32_t value_offset(int64_t i) const { return raw_value_offsets_[i + data_->offset]; }
   int32_t value_length(int64_t i) const {
-    i += offset_;
+    i += data_->offset;
     return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
   }
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
-  std::shared_ptr<Buffer> value_offsets_;
+  void SetData(const std::shared_ptr<internal::ArrayData>& data);
   const int32_t* raw_value_offsets_;
+
+ private:
   std::shared_ptr<Array> values_;
 };
 
 // ----------------------------------------------------------------------
 // Binary and String
 
-class ARROW_EXPORT BinaryArray : public Array {
+class ARROW_EXPORT BinaryArray : public FlatArray {
  public:
   using TypeClass = BinaryType;
 
+  explicit BinaryArray(const std::shared_ptr<internal::ArrayData>& data);
+
   BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
       const std::shared_ptr<Buffer>& data,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
@@ -277,7 +418,7 @@ class ARROW_EXPORT BinaryArray : public Array {
   // pointer + offset
   const uint8_t* GetValue(int64_t i, int32_t* out_length) const {
     // Account for base offset
-    i += offset_;
+    i += data_->offset;
 
     const int32_t pos = raw_value_offsets_[i];
     *out_length = raw_value_offsets_[i + 1] - pos;
@@ -285,23 +426,29 @@ class ARROW_EXPORT BinaryArray : public Array {
   }
 
   /// Note that this buffer does not account for any slice offset
-  std::shared_ptr<Buffer> data() const { return data_; }
+  std::shared_ptr<Buffer> value_offsets() const { return data_->buffers[1]; }
 
   /// Note that this buffer does not account for any slice offset
-  std::shared_ptr<Buffer> value_offsets() const { return value_offsets_; }
+  std::shared_ptr<Buffer> value_data() const { return data_->buffers[2]; }
 
-  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
+  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + data_->offset; }
 
   // Neither of these functions will perform boundschecking
-  int32_t value_offset(int64_t i) const { return raw_value_offsets_[i + offset_]; }
+  int32_t value_offset(int64_t i) const { return raw_value_offsets_[i + data_->offset]; }
   int32_t value_length(int64_t i) const {
-    i += offset_;
+    i += data_->offset;
     return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
   }
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
+  // For subclasses
+  BinaryArray() {}
+
+  /// Protected method for constructors
+  void SetData(const std::shared_ptr<internal::ArrayData>& data);
+
   // Constructor that allows sub-classes/builders to propagate there logical type up the
   // class hierarchy.
   BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
@@ -309,10 +456,7 @@ class ARROW_EXPORT BinaryArray : public Array {
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0);
 
-  std::shared_ptr<Buffer> value_offsets_;
   const int32_t* raw_value_offsets_;
-
-  std::shared_ptr<Buffer> data_;
   const uint8_t* raw_data_;
 };
 
@@ -320,6 +464,8 @@ class ARROW_EXPORT StringArray : public BinaryArray {
  public:
   using TypeClass = StringType;
 
+  explicit StringArray(const std::shared_ptr<internal::ArrayData>& data);
+
   StringArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
       const std::shared_ptr<Buffer>& data,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
@@ -343,6 +489,8 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
  public:
   using TypeClass = FixedSizeBinaryType;
 
+  explicit FixedSizeBinaryArray(const std::shared_ptr<internal::ArrayData>& data);
+
   FixedSizeBinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& data,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
@@ -352,20 +500,28 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 
   int32_t byte_width() const { return byte_width_; }
 
-  const uint8_t* raw_data() const { return raw_data_; }
+  const uint8_t* raw_values() const { return raw_values_; }
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
+  inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
+    this->PrimitiveArray::SetData(data);
+    byte_width_ = static_cast<const FixedSizeBinaryType&>(*type()).byte_width();
+  }
+
   int32_t byte_width_;
 };
 
 // ----------------------------------------------------------------------
 // DecimalArray
-class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray {
+class ARROW_EXPORT DecimalArray : public FlatArray {
  public:
   using TypeClass = Type;
 
+  /// \brief Construct DecimalArray from internal::ArrayData instance
+  explicit DecimalArray(const std::shared_ptr<internal::ArrayData>& data);
+
   DecimalArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::shared_ptr<Buffer>& data,
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
@@ -373,13 +529,27 @@ class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray {
 
   bool IsNegative(int64_t i) const;
 
+  const uint8_t* GetValue(int64_t i) const;
+
   std::string FormatValue(int64_t i) const;
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
- private:
+  /// \brief The main decimal data
+  /// For 32/64-bit decimal this is everything
+  std::shared_ptr<Buffer> values() const { return data_->buffers[1]; }
+
   /// Only needed for 128 bit Decimals
-  std::shared_ptr<Buffer> sign_bitmap_;
+  std::shared_ptr<Buffer> sign_bitmap() const { return data_->buffers[2]; }
+
+  int32_t byte_width() const {
+    return static_cast<const DecimalType&>(*type()).byte_width();
+  }
+  const uint8_t* raw_values() const { return raw_values_; }
+
+ private:
+  void SetData(const std::shared_ptr<internal::ArrayData>& data);
+  const uint8_t* raw_values_;
   const uint8_t* sign_bitmap_data_;
 };
 
@@ -390,6 +560,8 @@ class ARROW_EXPORT StructArray : public Array {
  public:
   using TypeClass = StructType;
 
+  explicit StructArray(const std::shared_ptr<internal::ArrayData>& data);
+
   StructArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::vector<std::shared_ptr<Array>>& children,
       std::shared_ptr<Buffer> null_bitmap = nullptr, int64_t null_count = 0,
@@ -399,13 +571,7 @@ class ARROW_EXPORT StructArray : public Array {
   // with this array.
   std::shared_ptr<Array> field(int pos) const;
 
-  const std::vector<std::shared_ptr<Array>>& fields() const { return children_; }
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
- protected:
-  // The child arrays corresponding to each field of the struct data type.
-  std::vector<std::shared_ptr<Array>> children_;
 };
 
 // ----------------------------------------------------------------------
@@ -416,6 +582,8 @@ class ARROW_EXPORT UnionArray : public Array {
   using TypeClass = UnionType;
   using type_id_t = uint8_t;
 
+  explicit UnionArray(const std::shared_ptr<internal::ArrayData>& data);
+
   UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
       const std::vector<std::shared_ptr<Array>>& children,
       const std::shared_ptr<Buffer>& type_ids,
@@ -424,29 +592,24 @@ class ARROW_EXPORT UnionArray : public Array {
       int64_t offset = 0);
 
   /// Note that this buffer does not account for any slice offset
-  std::shared_ptr<Buffer> type_ids() const { return type_ids_; }
+  std::shared_ptr<Buffer> type_ids() const { return data_->buffers[1]; }
 
   /// Note that this buffer does not account for any slice offset
-  std::shared_ptr<Buffer> value_offsets() const { return value_offsets_; }
+  std::shared_ptr<Buffer> value_offsets() const { return data_->buffers[2]; }
 
-  const type_id_t* raw_type_ids() const { return raw_type_ids_ + offset_; }
-  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + offset_; }
+  const type_id_t* raw_type_ids() const { return raw_type_ids_ + data_->offset; }
+  const int32_t* raw_value_offsets() const { return raw_value_offsets_ + data_->offset; }
 
-  UnionMode mode() const { return static_cast<const UnionType&>(*type_.get()).mode(); }
+  UnionMode mode() const { return static_cast<const UnionType&>(*type()).mode(); }
 
   std::shared_ptr<Array> child(int pos) const;
 
-  const std::vector<std::shared_ptr<Array>>& children() const { return children_; }
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
-  std::vector<std::shared_ptr<Array>> children_;
+  void SetData(const std::shared_ptr<internal::ArrayData>& data);
 
-  std::shared_ptr<Buffer> type_ids_;
   const type_id_t* raw_type_ids_;
-
-  std::shared_ptr<Buffer> value_offsets_;
   const int32_t* raw_value_offsets_;
 };
 
@@ -472,17 +635,21 @@ class ARROW_EXPORT DictionaryArray : public Array {
  public:
   using TypeClass = DictionaryType;
 
+  explicit DictionaryArray(const std::shared_ptr<internal::ArrayData>& data);
+
   DictionaryArray(
       const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices);
 
-  std::shared_ptr<Array> indices() const { return indices_; }
+  std::shared_ptr<Array> indices() const;
   std::shared_ptr<Array> dictionary() const;
 
   const DictionaryType* dict_type() const { return dict_type_; }
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
- protected:
+ private:
+  void SetData(const std::shared_ptr<internal::ArrayData>& data);
+
   const DictionaryType* dict_type_;
   std::shared_ptr<Array> indices_;
 };
@@ -517,6 +684,16 @@ ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
 /// \return Status
 Status ARROW_EXPORT ValidateArray(const Array& array);
 
+/// Create new arrays for logical types that are backed by primitive arrays.
+Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    int64_t length, const std::shared_ptr<Buffer>& data,
+    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset,
+    std::shared_ptr<Array>* out);
+
+Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
+    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out);
+
 }  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index a57f75a1075e1..c3bc7459953f9 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -150,15 +150,15 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
   const int64_t new_length = length + length_;
 
   // Fill up the bytes until we have a byte alignment
-  int64_t pad_to_byte = 8 - (length_ % 8);
+  int64_t pad_to_byte = std::min<int64_t>(8 - (length_ % 8), length);
   if (pad_to_byte == 8) { pad_to_byte = 0; }
-  for (int64_t i = 0; i < pad_to_byte; ++i) {
+  for (int64_t i = length_; i < length_ + pad_to_byte; ++i) {
     BitUtil::SetBit(null_bitmap_data_, i);
   }
 
   // Fast bitsetting
   int64_t fast_length = (length - pad_to_byte) / 8;
-  memset(null_bitmap_data_ + ((length_ + pad_to_byte) / 8), 255,
+  memset(null_bitmap_data_ + ((length_ + pad_to_byte) / 8), 0xFF,
       static_cast<size_t>(fast_length));
 
   // Trailing bytes
@@ -700,11 +700,11 @@ template <typename T>
 Status DictionaryBuilder<T>::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<Array> dictionary;
   RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
-  auto type = std::make_shared<DictionaryType>(type_, dictionary);
 
   std::shared_ptr<Array> values;
   RETURN_NOT_OK(values_builder_.Finish(&values));
 
+  auto type = std::make_shared<DictionaryType>(values->type(), dictionary);
   *out = std::make_shared<DictionaryArray>(type, values);
   return Status::OK();
 }
@@ -1031,6 +1031,7 @@ Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
 void ListBuilder::Reset() {
   capacity_ = length_ = null_count_ = 0;
   null_bitmap_ = nullptr;
+  values_ = nullptr;
 }
 
 ArrayBuilder* ListBuilder::value_builder() const {
@@ -1061,7 +1062,7 @@ Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
   auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
 
   *out = std::make_shared<BinaryArray>(list->length(), list->value_offsets(),
-      values->data(), list->null_bitmap(), list->null_count());
+      values->values(), list->null_bitmap(), list->null_count());
   return Status::OK();
 }
 
@@ -1086,7 +1087,7 @@ Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
   auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
 
   *out = std::make_shared<StringArray>(list->length(), list->value_offsets(),
-      values->data(), list->null_bitmap(), list->null_count());
+      values->values(), list->null_bitmap(), list->null_count());
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 390a406c19742..23f5a1945b4c1 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -83,8 +83,8 @@ class RangeEqualsVisitor {
       }
 
       if (end_offset - begin_offset > 0 &&
-          std::memcmp(left.data()->data() + begin_offset,
-              right.data()->data() + right_begin_offset,
+          std::memcmp(left.value_data()->data() + begin_offset,
+              right.value_data()->data() + right_begin_offset,
               static_cast<size_t>(end_offset - begin_offset))) {
         return false;
       }
@@ -126,7 +126,7 @@ class RangeEqualsVisitor {
          ++i, ++o_i) {
       if (left.IsNull(i) != right.IsNull(o_i)) { return false; }
       if (left.IsNull(i)) continue;
-      for (int j = 0; j < static_cast<int>(left.fields().size()); ++j) {
+      for (int j = 0; j < left.num_fields(); ++j) {
         // TODO: really we should be comparing stretches of non-null data rather
         // than looking at one value at a time.
         const int64_t left_abs_index = i + left.offset();
@@ -188,7 +188,7 @@ class RangeEqualsVisitor {
         }
       } else {
         const int32_t offset = left.raw_value_offsets()[i];
-        const int32_t o_offset = right.raw_value_offsets()[i];
+        const int32_t o_offset = right.raw_value_offsets()[o_i];
         if (!left.child(child_num)->RangeEquals(
                 offset, offset + 1, o_offset, right.child(child_num))) {
           return false;
@@ -211,9 +211,9 @@ class RangeEqualsVisitor {
     const uint8_t* left_data = nullptr;
     const uint8_t* right_data = nullptr;
 
-    if (left.data()) { left_data = left.raw_data() + left.offset() * width; }
+    if (left.values()) { left_data = left.raw_values() + left.offset() * width; }
 
-    if (right.data()) { right_data = right.raw_data() + right.offset() * width; }
+    if (right.values()) { right_data = right.raw_values() + right.offset() * width; }
 
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
@@ -241,9 +241,9 @@ class RangeEqualsVisitor {
     const uint8_t* left_data = nullptr;
     const uint8_t* right_data = nullptr;
 
-    if (left.data()) { left_data = left.raw_data() + left.offset() * width; }
+    if (left.values()) { left_data = left.raw_values() + left.offset() * width; }
 
-    if (right.data()) { right_data = right.raw_data() + right.offset() * width; }
+    if (right.values()) { right_data = right.raw_values() + right.offset() * width; }
 
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
@@ -317,6 +317,95 @@ class RangeEqualsVisitor {
   bool result_;
 };
 
+static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& right) {
+  const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
+  const int byte_width = size_meta.bit_width() / 8;
+
+  const uint8_t* left_data = nullptr;
+  const uint8_t* right_data = nullptr;
+
+  if (left.values()) { left_data = left.values()->data() + left.offset() * byte_width; }
+  if (right.values()) {
+    right_data = right.values()->data() + right.offset() * byte_width;
+  }
+
+  if (left.null_count() > 0) {
+    for (int64_t i = 0; i < left.length(); ++i) {
+      bool left_null = left.IsNull(i);
+      if (!left_null && (memcmp(left_data, right_data, byte_width) || right.IsNull(i))) {
+        return false;
+      }
+      left_data += byte_width;
+      right_data += byte_width;
+    }
+    return true;
+  } else {
+    return memcmp(left_data, right_data,
+               static_cast<size_t>(byte_width * left.length())) == 0;
+  }
+}
+
+template <typename T>
+static inline bool CompareBuiltIn(
+    const Array& left, const Array& right, const T* ldata, const T* rdata) {
+  if (left.null_count() > 0) {
+    for (int64_t i = 0; i < left.length(); ++i) {
+      if (left.IsNull(i) != right.IsNull(i)) {
+        return false;
+      } else if (!left.IsNull(i) && (ldata[i] != rdata[i])) {
+        return false;
+      }
+    }
+    return true;
+  } else {
+    return memcmp(ldata, rdata, sizeof(T) * left.length()) == 0;
+  }
+}
+
+static bool IsEqualDecimal(const DecimalArray& left, const DecimalArray& right) {
+  const int64_t loffset = left.offset();
+  const int64_t roffset = right.offset();
+
+  const uint8_t* left_data = nullptr;
+  const uint8_t* right_data = nullptr;
+
+  if (left.values()) { left_data = left.values()->data(); }
+  if (right.values()) { right_data = right.values()->data(); }
+
+  const int32_t byte_width = left.byte_width();
+  if (byte_width == 4) {
+    return CompareBuiltIn<int32_t>(left, right,
+        reinterpret_cast<const int32_t*>(left_data) + loffset,
+        reinterpret_cast<const int32_t*>(right_data) + roffset);
+  } else if (byte_width == 8) {
+    return CompareBuiltIn<int64_t>(left, right,
+        reinterpret_cast<const int64_t*>(left_data) + loffset,
+        reinterpret_cast<const int64_t*>(right_data) + roffset);
+  } else {
+    // 128-bit
+
+    // Must also compare sign bitmap
+    const uint8_t* left_sign = nullptr;
+    const uint8_t* right_sign = nullptr;
+    if (left.sign_bitmap()) { left_sign = left.sign_bitmap()->data(); }
+    if (right.sign_bitmap()) { right_sign = right.sign_bitmap()->data(); }
+
+    for (int64_t i = 0; i < left.length(); ++i) {
+      bool left_null = left.IsNull(i);
+      if (!left_null && (memcmp(left_data, right_data, byte_width) || right.IsNull(i))) {
+        return false;
+      }
+      if (BitUtil::GetBit(left_sign, i + loffset) !=
+          BitUtil::GetBit(right_sign, i + roffset)) {
+        return false;
+      }
+      left_data += byte_width;
+      right_data += byte_width;
+    }
+    return true;
+  }
+}
+
 class ArrayEqualsVisitor : public RangeEqualsVisitor {
  public:
   explicit ArrayEqualsVisitor(const Array& right)
@@ -331,8 +420,8 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     const auto& right = static_cast<const BooleanArray&>(right_);
 
     if (left.null_count() > 0) {
-      const uint8_t* left_data = left.data()->data();
-      const uint8_t* right_data = right.data()->data();
+      const uint8_t* left_data = left.values()->data();
+      const uint8_t* right_data = right.values()->data();
 
       for (int64_t i = 0; i < left.length(); ++i) {
         if (!left.IsNull(i) &&
@@ -344,45 +433,23 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       }
       result_ = true;
     } else {
-      result_ = BitmapEquals(left.data()->data(), left.offset(), right.data()->data(),
+      result_ = BitmapEquals(left.values()->data(), left.offset(), right.values()->data(),
           right.offset(), left.length());
     }
     return Status::OK();
   }
 
-  bool IsEqualPrimitive(const PrimitiveArray& left) {
-    const auto& right = static_cast<const PrimitiveArray&>(right_);
-    const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
-    const int byte_width = size_meta.bit_width() / 8;
-
-    const uint8_t* left_data = nullptr;
-    const uint8_t* right_data = nullptr;
-
-    if (left.data()) { left_data = left.data()->data() + left.offset() * byte_width; }
-
-    if (right.data()) { right_data = right.data()->data() + right.offset() * byte_width; }
-
-    if (left.null_count() > 0) {
-      for (int64_t i = 0; i < left.length(); ++i) {
-        if (!left.IsNull(i) && memcmp(left_data, right_data, byte_width)) {
-          return false;
-        }
-        left_data += byte_width;
-        right_data += byte_width;
-      }
-      return true;
-    } else {
-      return memcmp(left_data, right_data,
-                 static_cast<size_t>(byte_width * left.length())) == 0;
-    }
-  }
-
   template <typename T>
   typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value &&
                               !std::is_base_of<BooleanArray, T>::value,
       Status>::type
   Visit(const T& left) {
-    result_ = IsEqualPrimitive(left);
+    result_ = IsEqualPrimitive(left, static_cast<const PrimitiveArray&>(right_));
+    return Status::OK();
+  }
+
+  Status Visit(const DecimalArray& left) {
+    result_ = IsEqualDecimal(left, static_cast<const DecimalArray&>(right_));
     return Status::OK();
   }
 
@@ -417,11 +484,11 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     bool equal_offsets = ValueOffsetsEqual<BinaryArray>(left);
     if (!equal_offsets) { return false; }
 
-    if (!left.data() && !(right.data())) { return true; }
+    if (!left.value_data() && !(right.value_data())) { return true; }
     if (left.value_offset(left.length()) == 0) { return true; }
 
-    const uint8_t* left_data = left.data()->data();
-    const uint8_t* right_data = right.data()->data();
+    const uint8_t* left_data = left.value_data()->data();
+    const uint8_t* right_data = right.value_data()->data();
 
     if (left.null_count() == 0) {
       // Fast path for null count 0, single memcmp
@@ -491,8 +558,8 @@ inline bool FloatingApproxEquals(
     const NumericArray<TYPE>& left, const NumericArray<TYPE>& right) {
   using T = typename TYPE::c_type;
 
-  const T* left_data = left.raw_data();
-  const T* right_data = right.raw_data();
+  const T* left_data = left.raw_values();
+  const T* right_data = right.raw_values();
 
   static constexpr T EPSILON = static_cast<T>(1E-5);
 
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index 807ea4e746270..a7793f20183e7 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -28,7 +28,6 @@
 #include "arrow/ipc/feather-internal.h"
 #include "arrow/ipc/feather.h"
 #include "arrow/ipc/test-common.h"
-#include "arrow/loader.h"
 #include "arrow/pretty_print.h"
 #include "arrow/test-util.h"
 
@@ -365,25 +364,19 @@ TEST_F(TestTableWriter, TimeTypes) {
   std::shared_ptr<Array> date_array;
   ArrayFromVector<Date32Type, int32_t>(is_valid, date_values_vec, &date_array);
 
-  std::vector<FieldMetadata> fields(1);
-  fields[0].length = values->length();
-  fields[0].null_count = values->null_count();
-  fields[0].offset = 0;
-
   const auto& prim_values = static_cast<const PrimitiveArray&>(*values);
   std::vector<std::shared_ptr<Buffer>> buffers = {
-      prim_values.null_bitmap(), prim_values.data()};
+      prim_values.null_bitmap(), prim_values.values()};
 
-  std::vector<std::shared_ptr<Array>> arrays;
-  arrays.push_back(date_array);
+  std::vector<std::shared_ptr<internal::ArrayData>> arrays;
+  arrays.push_back(date_array->data());
 
   for (int i = 1; i < schema->num_fields(); ++i) {
-    std::shared_ptr<Array> arr;
-    ASSERT_OK(LoadArray(schema->field(i)->type(), fields, buffers, &arr));
-    arrays.push_back(arr);
+    arrays.emplace_back(std::make_shared<internal::ArrayData>(
+        schema->field(i)->type(), values->length(), buffers, values->null_count(), 0));
   }
 
-  RecordBatch batch(schema, values->length(), arrays);
+  RecordBatch batch(schema, values->length(), std::move(arrays));
   CheckBatch(batch);
 }
 
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index bc7c431969cac..37b01c5191992 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -33,7 +33,6 @@
 #include "arrow/io/file.h"
 #include "arrow/ipc/feather-internal.h"
 #include "arrow/ipc/feather_generated.h"
-#include "arrow/loader.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/util/bit-util.h"
@@ -565,7 +564,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
           &bytes_written));
       meta->total_bytes += bytes_written;
 
-      if (bin_values.data()) { values_buffer = bin_values.data()->data(); }
+      if (bin_values.value_data()) { values_buffer = bin_values.value_data()->data(); }
     } else {
       const auto& prim_values = static_cast<const PrimitiveArray&>(values);
       const auto& fw_type = static_cast<const FixedWidthType&>(*values.type());
@@ -577,7 +576,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
         values_bytes = values.length() * fw_type.bit_width() / 8;
       }
 
-      if (prim_values.data()) { values_buffer = prim_values.data()->data(); }
+      if (prim_values.values()) { values_buffer = prim_values.values()->data(); }
     }
     RETURN_NOT_OK(
         WritePadded(stream_.get(), values_buffer, values_bytes, &bytes_written));
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index beebb4f0e2808..69e4ae8d14a04 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -414,7 +414,7 @@ class ArrayWriter {
   template <typename T>
   typename std::enable_if<IsSignedInt<T>::value, void>::type WriteDataValues(
       const T& arr) {
-    const auto data = arr.raw_data();
+    const auto data = arr.raw_values();
     for (int i = 0; i < arr.length(); ++i) {
       writer_->Int64(data[i]);
     }
@@ -423,7 +423,7 @@ class ArrayWriter {
   template <typename T>
   typename std::enable_if<IsUnsignedInt<T>::value, void>::type WriteDataValues(
       const T& arr) {
-    const auto data = arr.raw_data();
+    const auto data = arr.raw_values();
     for (int i = 0; i < arr.length(); ++i) {
       writer_->Uint64(data[i]);
     }
@@ -432,7 +432,7 @@ class ArrayWriter {
   template <typename T>
   typename std::enable_if<IsFloatingPoint<T>::value, void>::type WriteDataValues(
       const T& arr) {
-    const auto data = arr.raw_data();
+    const auto data = arr.raw_values();
     for (int i = 0; i < arr.length(); ++i) {
       writer_->Double(data[i]);
     }
@@ -558,7 +558,12 @@ class ArrayWriter {
   Status Visit(const StructArray& array) {
     WriteValidityField(array);
     const auto& type = static_cast<const StructType&>(*array.type());
-    return WriteChildren(type.children(), array.fields());
+    std::vector<std::shared_ptr<Array>> children;
+    children.reserve(array.num_fields());
+    for (int i = 0; i < array.num_fields(); ++i) {
+      children.emplace_back(array.field(i));
+    }
+    return WriteChildren(type.children(), children);
   }
 
   Status Visit(const UnionArray& array) {
@@ -569,7 +574,12 @@ class ArrayWriter {
     if (type.mode() == UnionMode::DENSE) {
       WriteIntegerField("OFFSET", array.raw_value_offsets(), array.length());
     }
-    return WriteChildren(type.children(), array.children());
+    std::vector<std::shared_ptr<Array>> children;
+    children.reserve(array.num_fields());
+    for (int i = 0; i < array.num_fields(); ++i) {
+      children.emplace_back(array.child(i));
+    }
+    return WriteChildren(type.children(), children);
   }
 
  private:
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index ec7bc392f6fb8..257bbd8673a8f 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -26,7 +26,6 @@
 #include <unordered_map>
 #include <vector>
 
-#include "arrow/loader.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -54,6 +53,42 @@ enum class MetadataVersion : char { V1, V2, V3 };
 
 static constexpr const char* kArrowMagicBytes = "ARROW1";
 
+// ARROW-109: We set this number arbitrarily to help catch user mistakes. For
+// deeply nested schemas, it is expected the user will indicate explicitly the
+// maximum allowed recursion depth
+constexpr int kMaxNestingDepth = 64;
+
+struct ARROW_EXPORT FieldMetadata {
+  FieldMetadata() {}
+  FieldMetadata(int64_t length, int64_t null_count, int64_t offset)
+      : length(length), null_count(null_count), offset(offset) {}
+
+  FieldMetadata(const FieldMetadata& other) {
+    this->length = other.length;
+    this->null_count = other.null_count;
+    this->offset = other.offset;
+  }
+
+  int64_t length;
+  int64_t null_count;
+  int64_t offset;
+};
+
+struct ARROW_EXPORT BufferMetadata {
+  BufferMetadata() {}
+  BufferMetadata(int32_t page, int64_t offset, int64_t length)
+      : page(page), offset(offset), length(length) {}
+
+  /// The shared memory page id where to find this. Set to -1 if unused
+  int32_t page;
+
+  /// The relative offset into the memory page to the starting byte of the buffer
+  int64_t offset;
+
+  /// Absolute length in bytes of the buffer
+  int64_t length;
+};
+
 struct FileBlock {
   FileBlock() {}
   FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index ea16bf0335e9d..ae46207d2fbc1 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -35,6 +35,7 @@
 #include "arrow/tensor.h"
 #include "arrow/type.h"
 #include "arrow/util/logging.h"
+#include "arrow/visitor_inline.h"
 
 namespace arrow {
 
@@ -45,12 +46,13 @@ namespace ipc {
 // ----------------------------------------------------------------------
 // Record batch read path
 
-class IpcComponentSource : public ArrayComponentSource {
+/// Accessor class for flatbuffers metadata
+class IpcComponentSource {
  public:
   IpcComponentSource(const flatbuf::RecordBatch* metadata, io::RandomAccessFile* file)
       : metadata_(metadata), file_(file) {}
 
-  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
     const flatbuf::Buffer* buffer = metadata_->buffers()->Get(buffer_index);
 
     if (buffer->length() == 0) {
@@ -61,7 +63,7 @@ class IpcComponentSource : public ArrayComponentSource {
     }
   }
 
-  Status GetFieldMetadata(int field_index, FieldMetadata* field) override {
+  Status GetFieldMetadata(int field_index, internal::ArrayData* out) {
     auto nodes = metadata_->nodes();
     // pop off a field
     if (field_index >= static_cast<int>(nodes->size())) {
@@ -69,9 +71,9 @@ class IpcComponentSource : public ArrayComponentSource {
     }
     const flatbuf::FieldNode* node = nodes->Get(field_index);
 
-    field->length = node->length();
-    field->null_count = node->null_count();
-    field->offset = 0;
+    out->length = node->length();
+    out->null_count = node->null_count();
+    out->offset = 0;
     return Status::OK();
   }
 
@@ -80,26 +82,204 @@ class IpcComponentSource : public ArrayComponentSource {
   io::RandomAccessFile* file_;
 };
 
+/// Bookkeeping struct for loading array objects from their constituent pieces of raw data
+///
+/// The field_index and buffer_index are incremented in the ArrayLoader
+/// based on how much of the batch is "consumed" (through nested data
+/// reconstruction, for example)
+struct ArrayLoaderContext {
+  IpcComponentSource* source;
+  int buffer_index;
+  int field_index;
+  int max_recursion_depth;
+};
+
+static Status LoadArray(const std::shared_ptr<DataType>& type,
+    ArrayLoaderContext* context, internal::ArrayData* out);
+
+class ArrayLoader {
+ public:
+  ArrayLoader(const std::shared_ptr<DataType>& type, internal::ArrayData* out,
+      ArrayLoaderContext* context)
+      : type_(type), context_(context), out_(out) {}
+
+  Status Load() {
+    if (context_->max_recursion_depth <= 0) {
+      return Status::Invalid("Max recursion depth reached");
+    }
+
+    out_->type = type_;
+
+    RETURN_NOT_OK(VisitTypeInline(*type_, this));
+    return Status::OK();
+  }
+
+  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
+    return context_->source->GetBuffer(buffer_index, out);
+  }
+
+  Status LoadCommon() {
+    // This only contains the length and null count, which we need to figure
+    // out what to do with the buffers. For example, if null_count == 0, then
+    // we can skip that buffer without reading from shared memory
+    RETURN_NOT_OK(context_->source->GetFieldMetadata(context_->field_index++, out_));
+
+    // extract null_bitmap which is common to all arrays
+    if (out_->null_count == 0) {
+      out_->buffers[0] = nullptr;
+    } else {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &out_->buffers[0]));
+    }
+    context_->buffer_index++;
+    return Status::OK();
+  }
+
+  template <typename TYPE>
+  Status LoadPrimitive() {
+    out_->buffers.resize(2);
+
+    RETURN_NOT_OK(LoadCommon());
+    if (out_->length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &out_->buffers[1]));
+    } else {
+      context_->buffer_index++;
+      out_->buffers[1].reset(new Buffer(nullptr, 0));
+    }
+    return Status::OK();
+  }
+
+  template <typename TYPE>
+  Status LoadBinary() {
+    out_->buffers.resize(3);
+
+    RETURN_NOT_OK(LoadCommon());
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &out_->buffers[1]));
+    return GetBuffer(context_->buffer_index++, &out_->buffers[2]);
+  }
+
+  Status LoadChild(const Field& field, internal::ArrayData* out) {
+    ArrayLoader loader(field.type(), out, context_);
+    --context_->max_recursion_depth;
+    RETURN_NOT_OK(loader.Load());
+    ++context_->max_recursion_depth;
+    return Status::OK();
+  }
+
+  Status LoadChildren(std::vector<std::shared_ptr<Field>> child_fields) {
+    out_->child_data.reserve(static_cast<int>(child_fields.size()));
+
+    for (const auto& child_field : child_fields) {
+      auto field_array = std::make_shared<internal::ArrayData>();
+      RETURN_NOT_OK(LoadChild(*child_field.get(), field_array.get()));
+      out_->child_data.emplace_back(field_array);
+    }
+    return Status::OK();
+  }
+
+  Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
+
+  Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
+                              !std::is_base_of<FixedSizeBinaryType, T>::value &&
+                              !std::is_base_of<DictionaryType, T>::value,
+      Status>::type
+  Visit(const T& type) {
+    return LoadPrimitive<T>();
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BinaryType, T>::value, Status>::type Visit(
+      const T& type) {
+    return LoadBinary<T>();
+  }
+
+  Status Visit(const FixedSizeBinaryType& type) {
+    out_->buffers.resize(2);
+    RETURN_NOT_OK(LoadCommon());
+    return GetBuffer(context_->buffer_index++, &out_->buffers[1]);
+  }
+
+  Status Visit(const ListType& type) {
+    out_->buffers.resize(2);
+
+    RETURN_NOT_OK(LoadCommon());
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &out_->buffers[1]));
+
+    const int num_children = type.num_children();
+    if (num_children != 1) {
+      std::stringstream ss;
+      ss << "Wrong number of children: " << num_children;
+      return Status::Invalid(ss.str());
+    }
+
+    return LoadChildren(type.children());
+  }
+
+  Status Visit(const StructType& type) {
+    out_->buffers.resize(1);
+    RETURN_NOT_OK(LoadCommon());
+    return LoadChildren(type.children());
+  }
+
+  Status Visit(const UnionType& type) {
+    out_->buffers.resize(3);
+
+    RETURN_NOT_OK(LoadCommon());
+    if (out_->length > 0) {
+      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &out_->buffers[1]));
+      if (type.mode() == UnionMode::DENSE) {
+        RETURN_NOT_OK(GetBuffer(context_->buffer_index + 1, &out_->buffers[2]));
+      }
+    }
+    context_->buffer_index += type.mode() == UnionMode::DENSE ? 2 : 1;
+    return LoadChildren(type.children());
+  }
+
+  Status Visit(const DictionaryType& type) {
+    RETURN_NOT_OK(LoadArray(type.index_type(), context_, out_));
+    out_->type = type_;
+    return Status::OK();
+  }
+
+ private:
+  const std::shared_ptr<DataType>& type_;
+  ArrayLoaderContext* context_;
+
+  // Used in visitor pattern
+  internal::ArrayData* out_;
+};
+
+static Status LoadArray(const std::shared_ptr<DataType>& type,
+    ArrayLoaderContext* context, internal::ArrayData* out) {
+  ArrayLoader loader(type, out, context);
+  return loader.Load();
+}
+
 Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
   return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
 }
 
+// ----------------------------------------------------------------------
+// Array loading
+
 static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
-    int64_t num_rows, int max_recursion_depth, ArrayComponentSource* source,
+    int64_t num_rows, int max_recursion_depth, IpcComponentSource* source,
     std::shared_ptr<RecordBatch>* out) {
-  std::vector<std::shared_ptr<Array>> arrays(schema->num_fields());
-
   ArrayLoaderContext context;
   context.source = source;
   context.field_index = 0;
   context.buffer_index = 0;
   context.max_recursion_depth = max_recursion_depth;
 
+  std::vector<std::shared_ptr<internal::ArrayData>> arrays(schema->num_fields());
   for (int i = 0; i < schema->num_fields(); ++i) {
-    RETURN_NOT_OK(LoadArray(schema->field(i)->type(), &context, &arrays[i]));
-    DCHECK_EQ(num_rows, arrays[i]->length())
-        << "Array length did not match record batch length";
+    auto arr = std::make_shared<internal::ArrayData>();
+    RETURN_NOT_OK(LoadArray(schema->field(i)->type(), &context, arr.get()));
+    DCHECK_EQ(num_rows, arr->length) << "Array length did not match record batch length";
+    arrays[i] = std::move(arr);
   }
 
   *out = std::make_shared<RecordBatch>(schema, num_rows, std::move(arrays));
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 747aca0873df9..6fdf1cc8a8f39 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -86,7 +86,9 @@ static inline void CompareArraysDetailed(
 static inline void CompareBatchColumnsDetailed(
     const RecordBatch& result, const RecordBatch& expected) {
   for (int i = 0; i < expected.num_columns(); ++i) {
-    CompareArraysDetailed(i, *result.column(i), *expected.column(i));
+    auto left = result.column(i);
+    auto right = expected.column(i);
+    CompareArraysDetailed(i, *left, *right);
   }
 }
 
@@ -471,7 +473,7 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   RETURN_NOT_OK(test::GetBitmapFromBoolVector(is_valid, &null_bitmap));
 
   std::shared_ptr<Array> a3 = std::make_shared<ListArray>(f3_type, length,
-      std::static_pointer_cast<PrimitiveArray>(offsets)->data(),
+      std::static_pointer_cast<PrimitiveArray>(offsets)->values(),
       std::make_shared<DictionaryArray>(f1_type, indices3), null_bitmap, 1);
 
   // Dictionary-encoded list of integer
@@ -487,7 +489,7 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   ArrayFromVector<Int8Type, int8_t>(std::vector<bool>(3, true), list_values4, &values4);
 
   auto dict3 = std::make_shared<ListArray>(f4_value_type, 3,
-      std::static_pointer_cast<PrimitiveArray>(offsets4)->data(), values4);
+      std::static_pointer_cast<PrimitiveArray>(offsets4)->values(), values4);
 
   std::vector<int8_t> indices4_values = {0, 1, 2, 0, 1, 2};
   ArrayFromVector<Int8Type, int8_t>(is_valid, indices4_values, &indices4);
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 60b1f474a39c6..592bca222f70b 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -30,7 +30,6 @@
 #include "arrow/io/memory.h"
 #include "arrow/ipc/metadata.h"
 #include "arrow/ipc/util.h"
-#include "arrow/loader.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
@@ -233,7 +232,7 @@ class RecordBatchSerializer : public ArrayVisitor {
  protected:
   template <typename ArrayType>
   Status VisitFixedWidth(const ArrayType& array) {
-    std::shared_ptr<Buffer> data = array.data();
+    std::shared_ptr<Buffer> data = array.values();
 
     const auto& fw_type = static_cast<const FixedWidthType&>(*array.type());
     const int64_t type_width = fw_type.bit_width() / 8;
@@ -287,7 +286,7 @@ class RecordBatchSerializer : public ArrayVisitor {
   Status VisitBinary(const BinaryArray& array) {
     std::shared_ptr<Buffer> value_offsets;
     RETURN_NOT_OK(GetZeroBasedValueOffsets<BinaryArray>(array, &value_offsets));
-    auto data = array.data();
+    auto data = array.value_data();
 
     int64_t total_data_bytes = 0;
     if (value_offsets) {
@@ -309,7 +308,7 @@ class RecordBatchSerializer : public ArrayVisitor {
   Status Visit(const BooleanArray& array) override {
     std::shared_ptr<Buffer> data;
     RETURN_NOT_OK(
-        GetTruncatedBitmap(array.offset(), array.length(), array.data(), pool_, &data));
+        GetTruncatedBitmap(array.offset(), array.length(), array.values(), pool_, &data));
     buffers_.push_back(data);
     return Status::OK();
   }
@@ -367,7 +366,8 @@ class RecordBatchSerializer : public ArrayVisitor {
 
   Status Visit(const StructArray& array) override {
     --max_recursion_depth_;
-    for (std::shared_ptr<Array> field : array.fields()) {
+    for (int i = 0; i < array.num_fields(); ++i) {
+      std::shared_ptr<Array> field = array.field(i);
       if (array.offset() != 0 || array.length() < field->length()) {
         // If offset is non-zero, slice the child array
         field = field->Slice(array.offset(), array.length());
@@ -450,7 +450,9 @@ class RecordBatchSerializer : public ArrayVisitor {
         RETURN_NOT_OK(VisitArray(*child));
       }
     } else {
-      for (std::shared_ptr<Array> child : array.children()) {
+      for (int i = 0; i < array.num_fields(); ++i) {
+        std::shared_ptr<Array> child = array.child(i);
+
         // Sparse union, slicing is simpler
         if (offset != 0 || length < child->length()) {
           // If offset is non-zero, slice the child array
diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc
deleted file mode 100644
index e4e1ba42ff600..0000000000000
--- a/cpp/src/arrow/loader.cc
+++ /dev/null
@@ -1,297 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/loader.h"
-
-#include <cstdint>
-#include <memory>
-#include <sstream>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/status.h"
-#include "arrow/type.h"
-#include "arrow/type_traits.h"
-#include "arrow/util/logging.h"
-#include "arrow/util/visibility.h"
-#include "arrow/visitor_inline.h"
-
-namespace arrow {
-
-class ArrayLoader {
- public:
-  ArrayLoader(const std::shared_ptr<DataType>& type, ArrayLoaderContext* context)
-      : type_(type), context_(context) {}
-
-  Status Load(std::shared_ptr<Array>* out) {
-    if (context_->max_recursion_depth <= 0) {
-      return Status::Invalid("Max recursion depth reached");
-    }
-
-    RETURN_NOT_OK(VisitTypeInline(*type_, this));
-
-    *out = std::move(result_);
-    return Status::OK();
-  }
-
-  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
-    return context_->source->GetBuffer(buffer_index, out);
-  }
-
-  Status LoadCommon(FieldMetadata* field_meta, std::shared_ptr<Buffer>* null_bitmap) {
-    // This only contains the length and null count, which we need to figure
-    // out what to do with the buffers. For example, if null_count == 0, then
-    // we can skip that buffer without reading from shared memory
-    RETURN_NOT_OK(
-        context_->source->GetFieldMetadata(context_->field_index++, field_meta));
-
-    // extract null_bitmap which is common to all arrays
-    if (field_meta->null_count == 0) {
-      *null_bitmap = nullptr;
-    } else {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index, null_bitmap));
-    }
-    context_->buffer_index++;
-    return Status::OK();
-  }
-
-  template <typename TYPE>
-  Status LoadPrimitive() {
-    using ArrayType = typename TypeTraits<TYPE>::ArrayType;
-
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, data;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    if (field_meta.length > 0) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &data));
-    } else {
-      context_->buffer_index++;
-      data.reset(new Buffer(nullptr, 0));
-    }
-    result_ = std::make_shared<ArrayType>(type_, field_meta.length, data, null_bitmap,
-        field_meta.null_count, field_meta.offset);
-    return Status::OK();
-  }
-
-  template <typename TYPE>
-  Status LoadBinary() {
-    using CONTAINER = typename TypeTraits<TYPE>::ArrayType;
-
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, offsets, values;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &values));
-
-    result_ = std::make_shared<CONTAINER>(
-        field_meta.length, offsets, values, null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status LoadChild(const Field& field, std::shared_ptr<Array>* out) {
-    ArrayLoader loader(field.type(), context_);
-    --context_->max_recursion_depth;
-    RETURN_NOT_OK(loader.Load(out));
-    ++context_->max_recursion_depth;
-    return Status::OK();
-  }
-
-  Status LoadChildren(std::vector<std::shared_ptr<Field>> child_fields,
-      std::vector<std::shared_ptr<Array>>* arrays) {
-    arrays->reserve(static_cast<int>(child_fields.size()));
-
-    for (const auto& child_field : child_fields) {
-      std::shared_ptr<Array> field_array;
-      RETURN_NOT_OK(LoadChild(*child_field.get(), &field_array));
-      arrays->emplace_back(field_array);
-    }
-    return Status::OK();
-  }
-
-  Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
-
-  Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
-
-  template <typename T>
-  typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
-                              !std::is_base_of<FixedSizeBinaryType, T>::value &&
-                              !std::is_base_of<DictionaryType, T>::value,
-      Status>::type
-  Visit(const T& type) {
-    return LoadPrimitive<T>();
-  }
-
-  template <typename T>
-  typename std::enable_if<std::is_base_of<BinaryType, T>::value, Status>::type Visit(
-      const T& type) {
-    return LoadBinary<T>();
-  }
-
-  Status Visit(const FixedSizeBinaryType& type) {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, data;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &data));
-
-    result_ = std::make_shared<FixedSizeBinaryArray>(
-        type_, field_meta.length, data, null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status Visit(const ListType& type) {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, offsets;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &offsets));
-
-    const int num_children = type.num_children();
-    if (num_children != 1) {
-      std::stringstream ss;
-      ss << "Wrong number of children: " << num_children;
-      return Status::Invalid(ss.str());
-    }
-    std::shared_ptr<Array> values_array;
-
-    RETURN_NOT_OK(LoadChild(*type.child(0).get(), &values_array));
-
-    result_ = std::make_shared<ListArray>(type_, field_meta.length, offsets, values_array,
-        null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status Visit(const StructType& type) {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap;
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-
-    std::vector<std::shared_ptr<Array>> fields;
-    RETURN_NOT_OK(LoadChildren(type.children(), &fields));
-
-    result_ = std::make_shared<StructArray>(
-        type_, field_meta.length, fields, null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status Visit(const UnionType& type) {
-    FieldMetadata field_meta;
-    std::shared_ptr<Buffer> null_bitmap, type_ids, offsets;
-
-    RETURN_NOT_OK(LoadCommon(&field_meta, &null_bitmap));
-    if (field_meta.length > 0) {
-      RETURN_NOT_OK(GetBuffer(context_->buffer_index, &type_ids));
-      if (type.mode() == UnionMode::DENSE) {
-        RETURN_NOT_OK(GetBuffer(context_->buffer_index + 1, &offsets));
-      }
-    }
-    context_->buffer_index += type.mode() == UnionMode::DENSE ? 2 : 1;
-
-    std::vector<std::shared_ptr<Array>> fields;
-    RETURN_NOT_OK(LoadChildren(type.children(), &fields));
-
-    result_ = std::make_shared<UnionArray>(type_, field_meta.length, fields, type_ids,
-        offsets, null_bitmap, field_meta.null_count);
-    return Status::OK();
-  }
-
-  Status Visit(const DictionaryType& type) {
-    std::shared_ptr<Array> indices;
-    RETURN_NOT_OK(LoadArray(type.index_type(), context_, &indices));
-    result_ = std::make_shared<DictionaryArray>(type_, indices);
-    return Status::OK();
-  }
-
-  std::shared_ptr<Array> result() const { return result_; }
-
- private:
-  const std::shared_ptr<DataType> type_;
-  ArrayLoaderContext* context_;
-
-  // Used in visitor pattern
-  std::shared_ptr<Array> result_;
-};
-
-Status LoadArray(const std::shared_ptr<DataType>& type, ArrayComponentSource* source,
-    std::shared_ptr<Array>* out) {
-  ArrayLoaderContext context;
-  context.source = source;
-  context.field_index = context.buffer_index = 0;
-  context.max_recursion_depth = kMaxNestingDepth;
-  return LoadArray(type, &context, out);
-}
-
-Status LoadArray(const std::shared_ptr<DataType>& type, ArrayLoaderContext* context,
-    std::shared_ptr<Array>* out) {
-  ArrayLoader loader(type, context);
-  RETURN_NOT_OK(loader.Load(out));
-
-  return Status::OK();
-}
-
-class InMemorySource : public ArrayComponentSource {
- public:
-  InMemorySource(const std::vector<FieldMetadata>& fields,
-      const std::vector<std::shared_ptr<Buffer>>& buffers)
-      : fields_(fields), buffers_(buffers) {}
-
-  Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
-    DCHECK(buffer_index < static_cast<int>(buffers_.size()));
-    *out = buffers_[buffer_index];
-    return Status::OK();
-  }
-
-  Status GetFieldMetadata(int field_index, FieldMetadata* metadata) {
-    DCHECK(field_index < static_cast<int>(fields_.size()));
-    *metadata = fields_[field_index];
-    return Status::OK();
-  }
-
- private:
-  const std::vector<FieldMetadata>& fields_;
-  const std::vector<std::shared_ptr<Buffer>>& buffers_;
-};
-
-Status LoadArray(const std::shared_ptr<DataType>& type,
-    const std::vector<FieldMetadata>& fields,
-    const std::vector<std::shared_ptr<Buffer>>& buffers, std::shared_ptr<Array>* out) {
-  InMemorySource source(fields, buffers);
-  return LoadArray(type, &source, out);
-}
-
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
-  std::vector<std::shared_ptr<Buffer>> buffers = {null_bitmap, data};
-  return MakePrimitiveArray(type, buffers, length, null_count, offset, out);
-}
-
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
-    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
-  std::vector<FieldMetadata> fields(1);
-  fields[0].length = length;
-  fields[0].null_count = null_count;
-  fields[0].offset = offset;
-
-  return LoadArray(type, fields, buffers, out);
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/loader.h b/cpp/src/arrow/loader.h
deleted file mode 100644
index f5e399537fd7b..0000000000000
--- a/cpp/src/arrow/loader.h
+++ /dev/null
@@ -1,124 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Function for constructing Array array objects from metadata and raw memory
-// buffers
-
-#ifndef ARROW_LOADER_H
-#define ARROW_LOADER_H
-
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Array;
-class Buffer;
-class DataType;
-
-// ARROW-109: We set this number arbitrarily to help catch user mistakes. For
-// deeply nested schemas, it is expected the user will indicate explicitly the
-// maximum allowed recursion depth
-constexpr int kMaxNestingDepth = 64;
-
-struct ARROW_EXPORT FieldMetadata {
-  FieldMetadata() {}
-  FieldMetadata(int64_t length, int64_t null_count, int64_t offset)
-      : length(length), null_count(null_count), offset(offset) {}
-
-  FieldMetadata(const FieldMetadata& other) {
-    this->length = other.length;
-    this->null_count = other.null_count;
-    this->offset = other.offset;
-  }
-
-  int64_t length;
-  int64_t null_count;
-  int64_t offset;
-};
-
-struct ARROW_EXPORT BufferMetadata {
-  BufferMetadata() {}
-  BufferMetadata(int32_t page, int64_t offset, int64_t length)
-      : page(page), offset(offset), length(length) {}
-
-  /// The shared memory page id where to find this. Set to -1 if unused
-  int32_t page;
-
-  /// The relative offset into the memory page to the starting byte of the buffer
-  int64_t offset;
-
-  /// Absolute length in bytes of the buffer
-  int64_t length;
-};
-
-/// Implement this to create new types of Arrow data loaders
-class ARROW_EXPORT ArrayComponentSource {
- public:
-  virtual ~ArrayComponentSource() = default;
-
-  virtual Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) = 0;
-  virtual Status GetFieldMetadata(int field_index, FieldMetadata* metadata) = 0;
-};
-
-/// Bookkeeping struct for loading array objects from their constituent pieces of raw data
-///
-/// The field_index and buffer_index are incremented in the ArrayLoader
-/// based on how much of the batch is "consumed" (through nested data
-/// reconstruction, for example)
-struct ArrayLoaderContext {
-  ArrayComponentSource* source;
-  int buffer_index;
-  int field_index;
-  int max_recursion_depth;
-};
-
-/// Construct an Array container from type metadata and a collection of memory
-/// buffers
-///
-/// \param[in] field the data type of the array being loaded
-/// \param[in] source an implementation of ArrayComponentSource
-/// \param[out] out the constructed array
-/// \return Status indicating success or failure
-Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
-    ArrayComponentSource* source, std::shared_ptr<Array>* out);
-
-Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& field,
-    ArrayLoaderContext* context, std::shared_ptr<Array>* out);
-
-Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type,
-    const std::vector<FieldMetadata>& fields,
-    const std::vector<std::shared_ptr<Buffer>>& buffers, std::shared_ptr<Array>* out);
-
-/// Create new arrays for logical types that are backed by primitive arrays.
-Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    int64_t length, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset,
-    std::shared_ptr<Array>* out);
-
-Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
-    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out);
-
-}  // namespace arrow
-
-#endif  // ARROW_LOADER_H
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 1f4bfa9acd357..93f6ff0f363f0 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -26,6 +26,7 @@
 #include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/string.h"
 #include "arrow/visitor_inline.h"
 
@@ -39,7 +40,7 @@ class ArrayPrinter {
   template <typename T>
   inline typename std::enable_if<IsInteger<T>::value, void>::type WriteDataValues(
       const T& array) {
-    const auto data = array.raw_data();
+    const auto data = array.raw_values();
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) { (*sink_) << ", "; }
       if (array.IsNull(i)) {
@@ -53,7 +54,7 @@ class ArrayPrinter {
   template <typename T>
   inline typename std::enable_if<IsFloatingPoint<T>::value, void>::type WriteDataValues(
       const T& array) {
-    const auto data = array.raw_data();
+    const auto data = array.raw_values();
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) { (*sink_) << ", "; }
       if (array.IsNull(i)) {
@@ -187,7 +188,12 @@ class ArrayPrinter {
 
   Status Visit(const StructArray& array) {
     RETURN_NOT_OK(WriteValidityBitmap(array));
-    return PrintChildren(array.fields(), array.offset(), array.length());
+    std::vector<std::shared_ptr<Array>> children;
+    children.reserve(array.num_fields());
+    for (int i = 0; i < array.num_fields(); ++i) {
+      children.emplace_back(array.field(i));
+    }
+    return PrintChildren(children, array.offset(), array.length());
   }
 
   Status Visit(const UnionArray& array) {
@@ -207,7 +213,12 @@ class ArrayPrinter {
     }
 
     // Print the children without any offset, because the type ids are absolute
-    return PrintChildren(array.children(), 0, array.length() + array.offset());
+    std::vector<std::shared_ptr<Array>> children;
+    children.reserve(array.num_fields());
+    for (int i = 0; i < array.num_fields(); ++i) {
+      children.emplace_back(array.child(i));
+    }
+    return PrintChildren(children, 0, array.length() + array.offset());
   }
 
   Status Visit(const DictionaryArray& array) {
@@ -286,4 +297,8 @@ Status PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink) {
   return Status::OK();
 }
 
+Status ARROW_EXPORT DebugPrint(const Array& arr, int indent) {
+  return PrettyPrint(arr, indent, &std::cout);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/pretty_print.h b/cpp/src/arrow/pretty_print.h
index f508aa042945a..a45c8a8275288 100644
--- a/cpp/src/arrow/pretty_print.h
+++ b/cpp/src/arrow/pretty_print.h
@@ -25,6 +25,7 @@
 
 namespace arrow {
 
+class Array;
 class Status;
 
 struct PrettyPrintOptions {
@@ -34,6 +35,8 @@ struct PrettyPrintOptions {
 Status ARROW_EXPORT PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink);
 Status ARROW_EXPORT PrettyPrint(const Array& arr, int indent, std::ostream* sink);
 
+Status ARROW_EXPORT DebugPrint(const Array& arr, int indent);
+
 }  // namespace arrow
 
 #endif  // ARROW_PRETTY_PRINT_H
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 2364f135b343f..cdd3f58e422ed 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -34,7 +34,6 @@
 #include <vector>
 
 #include "arrow/array.h"
-#include "arrow/loader.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type_fwd.h"
@@ -340,12 +339,10 @@ class PandasConverter {
       null_count = ValuesToBitmap<traits::npy_type>(arr_, null_bitmap_data_);
     }
 
-    std::vector<FieldMetadata> fields(1);
-    fields[0].length = length_;
-    fields[0].null_count = null_count;
-    fields[0].offset = 0;
-
-    return LoadArray(type_, fields, {null_bitmap_, data}, &out_);
+    BufferVector buffers = {null_bitmap_, data};
+    auto array_data = std::make_shared<internal::ArrayData>(
+        type_, length_, std::move(buffers), null_count, 0);
+    return internal::MakeArray(array_data, &out_);
   }
 
   template <typename T>
@@ -617,9 +614,9 @@ Status PandasConverter::ConvertObjectStrings() {
   RETURN_NOT_OK(builder.Finish(&out_));
 
   if (have_bytes) {
-    const auto& arr = static_cast<const StringArray&>(*out_);
-    out_ = std::make_shared<BinaryArray>(arr.length(), arr.value_offsets(), arr.data(),
-        arr.null_bitmap(), arr.null_count());
+    auto binary_data = out_->data()->ShallowCopy();
+    binary_data->type = ::arrow::binary();
+    out_ = std::make_shared<BinaryArray>(binary_data);
   }
   return Status::OK();
 }
@@ -1223,7 +1220,7 @@ inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
     // Upcast to double, set NaN as appropriate
 
     for (int i = 0; i < arr->length(); ++i) {
@@ -1237,7 +1234,7 @@ inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_value
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
     memcpy(out_values, in_values, sizeof(T) * arr->length());
     out_values += arr->length();
   }
@@ -1248,7 +1245,7 @@ inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_val
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->raw_values());
     for (int64_t i = 0; i < arr->length(); ++i) {
       *out_values = in_values[i];
     }
@@ -1371,14 +1368,14 @@ inline Status ConvertStruct(const ChunkedArray& data, PyObject** out_values) {
   // ChunkedArray has at least one chunk
   auto arr = static_cast<const StructArray*>(data.chunk(0).get());
   // Use it to cache the struct type and number of fields for all chunks
-  auto num_fields = arr->fields().size();
+  int32_t num_fields = arr->num_fields();
   auto array_type = arr->type();
   std::vector<OwnedRef> fields_data(num_fields);
   OwnedRef dict_item;
   for (int c = 0; c < data.num_chunks(); c++) {
     auto arr = static_cast<const StructArray*>(data.chunk(c).get());
     // Convert the struct arrays first
-    for (size_t i = 0; i < num_fields; i++) {
+    for (int32_t i = 0; i < num_fields; i++) {
       PyObject* numpy_array;
       RETURN_NOT_OK(
           ConvertArrayToPandas(arr->field(static_cast<int>(i)), nullptr, &numpy_array));
@@ -1395,7 +1392,7 @@ inline Status ConvertStruct(const ChunkedArray& data, PyObject** out_values) {
         // Build the new dict object for the row
         dict_item.reset(PyDict_New());
         RETURN_IF_PYERROR();
-        for (size_t field_idx = 0; field_idx < num_fields; ++field_idx) {
+        for (int32_t field_idx = 0; field_idx < num_fields; ++field_idx) {
           OwnedRef field_value;
           auto name = array_type->child(static_cast<int>(field_idx))->name();
           if (!arr->field(static_cast<int>(field_idx))->IsNull(i)) {
@@ -1475,7 +1472,7 @@ inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
 
     const uint8_t* valid_bits = arr->null_bitmap_data();
 
@@ -1496,7 +1493,7 @@ inline void ConvertNumericNullableCast(
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->raw_values());
 
     for (int64_t i = 0; i < arr->length(); ++i) {
       *out_values++ = arr->IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
@@ -1509,7 +1506,7 @@ inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values)
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->data()->data());
+    auto in_values = reinterpret_cast<const InType*>(prim_arr->raw_values());
 
     for (int64_t i = 0; i < arr->length(); ++i) {
       *out_values++ = arr->IsNull(i) ? kPandasTimestampNull
@@ -1838,7 +1835,7 @@ class CategoricalBlock : public PandasBlock {
       const std::shared_ptr<Array> arr = data.chunk(c);
       const auto& dict_arr = static_cast<const DictionaryArray&>(*arr);
       const auto& indices = static_cast<const PrimitiveArray&>(*dict_arr.indices());
-      auto in_values = reinterpret_cast<const T*>(indices.data()->data());
+      auto in_values = reinterpret_cast<const T*>(indices.raw_values());
 
       // Null is -1 in CategoricalBlock
       for (int i = 0; i < arr->length(); ++i) {
@@ -2214,7 +2211,7 @@ class ArrowDeserializer {
     typedef typename arrow_traits<TYPE>::T T;
 
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+    auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
 
     // Zero-Copy. We can pass the data pointer directly to NumPy.
     void* data = const_cast<T*>(in_values);
@@ -2290,7 +2287,7 @@ class ArrowDeserializer {
     for (int c = 0; c < data_.num_chunks(); c++) {
       const std::shared_ptr<Array> arr = data_.chunk(c);
       auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-      auto in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+      auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
 
       for (int64_t i = 0; i < arr->length(); ++i) {
         *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / kShift;
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index c110ec16a0494..aa042433c4486 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -146,12 +146,30 @@ void AssertBatchValid(const RecordBatch& batch) {
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
     const std::vector<std::shared_ptr<Array>>& columns)
-    : schema_(schema), num_rows_(num_rows), columns_(columns) {}
+    : schema_(schema), num_rows_(num_rows), columns_(columns.size()) {
+  for (size_t i = 0; i < columns.size(); ++i) {
+    columns_[i] = columns[i]->data();
+  }
+}
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
     std::vector<std::shared_ptr<Array>>&& columns)
+    : schema_(schema), num_rows_(num_rows), columns_(columns.size()) {
+  for (size_t i = 0; i < columns.size(); ++i) {
+    columns_[i] = columns[i]->data();
+  }
+}
+
+RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+    std::vector<std::shared_ptr<internal::ArrayData>>&& columns)
     : schema_(schema), num_rows_(num_rows), columns_(std::move(columns)) {}
 
+std::shared_ptr<Array> RecordBatch::column(int i) const {
+  std::shared_ptr<Array> result;
+  DCHECK(MakeArray(columns_[i], &result).ok());
+  return result;
+}
+
 const std::string& RecordBatch::column_name(int i) const {
   return schema_->field(i)->name();
 }
@@ -185,30 +203,36 @@ std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset) const {
 }
 
 std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length) const {
-  std::vector<std::shared_ptr<Array>> arrays;
+  std::vector<std::shared_ptr<internal::ArrayData>> arrays;
   arrays.reserve(num_columns());
   for (const auto& field : columns_) {
-    arrays.emplace_back(field->Slice(offset, length));
+    int64_t col_length = std::min(field->length - offset, length);
+    int64_t col_offset = field->offset + offset;
+
+    auto new_data = std::make_shared<internal::ArrayData>(*field);
+    new_data->length = col_length;
+    new_data->offset = col_offset;
+    new_data->null_count = kUnknownNullCount;
+    arrays.emplace_back(new_data);
   }
-
   int64_t num_rows = std::min(num_rows_ - offset, length);
-  return std::make_shared<RecordBatch>(schema_, num_rows, arrays);
+  return std::make_shared<RecordBatch>(schema_, num_rows, std::move(arrays));
 }
 
 Status RecordBatch::Validate() const {
   for (int i = 0; i < num_columns(); ++i) {
-    const Array& arr = *columns_[i];
-    if (arr.length() != num_rows_) {
+    const internal::ArrayData& arr = *columns_[i];
+    if (arr.length != num_rows_) {
       std::stringstream ss;
-      ss << "Number of rows in column " << i << " did not match batch: " << arr.length()
+      ss << "Number of rows in column " << i << " did not match batch: " << arr.length
          << " vs " << num_rows_;
       return Status::Invalid(ss.str());
     }
     const auto& schema_type = *schema_->field(i)->type();
-    if (!arr.type()->Equals(schema_type)) {
+    if (!arr.type->Equals(schema_type)) {
       std::stringstream ss;
-      ss << "Column " << i << " type not match schema: " << arr.type()->ToString()
-         << " vs " << schema_type.ToString();
+      ss << "Column " << i << " type not match schema: " << arr.type->ToString() << " vs "
+         << schema_type.ToString();
       return Status::Invalid(ss.str());
     }
   }
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 67710a8216010..18315f32013c0 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -28,6 +28,12 @@
 
 namespace arrow {
 
+namespace internal {
+
+struct ArrayData;
+
+}  // namespace internal
+
 class Array;
 class Column;
 class Schema;
@@ -106,15 +112,29 @@ class ARROW_EXPORT Column {
 // corresponding sequence of equal-length Arrow arrays
 class ARROW_EXPORT RecordBatch {
  public:
-  // num_rows is a parameter to allow for record batches of a particular size not
-  // having any materialized columns. Each array should have the same length as
-  // num_rows
+  /// num_rows is a parameter to allow for record batches of a particular size not
+  /// having any materialized columns. Each array should have the same length as
+  /// num_rows
+
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
       const std::vector<std::shared_ptr<Array>>& columns);
 
+  /// \brief Deprecated move constructor for a vector of Array instances
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
       std::vector<std::shared_ptr<Array>>&& columns);
 
+  /// \brief Construct record batch from vector of internal data structures
+  ///
+  /// This class is only provided with an rvalue-reference for the input data,
+  /// and is intended for internal use, or advanced users.
+  ///
+  /// \param schema the record batch schema
+  /// \param num_rows the number of semantic rows in the record batch. This
+  /// should be equal to the length of each field
+  /// \param columns the data for the batch's columns
+  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+      std::vector<std::shared_ptr<internal::ArrayData>>&& columns);
+
   bool Equals(const RecordBatch& other) const;
 
   bool ApproxEquals(const RecordBatch& other) const;
@@ -124,9 +144,9 @@ class ARROW_EXPORT RecordBatch {
 
   // @returns: the i-th column
   // Note: Does not boundscheck
-  std::shared_ptr<Array> column(int i) const { return columns_[i]; }
+  std::shared_ptr<Array> column(int i) const;
 
-  const std::vector<std::shared_ptr<Array>>& columns() const { return columns_; }
+  std::shared_ptr<internal::ArrayData> column_data(int i) const { return columns_[i]; }
 
   const std::string& column_name(int i) const;
 
@@ -147,7 +167,7 @@ class ARROW_EXPORT RecordBatch {
  private:
   std::shared_ptr<Schema> schema_;
   int64_t num_rows_;
-  std::vector<std::shared_ptr<Array>> columns_;
+  std::vector<std::shared_ptr<internal::ArrayData>> columns_;
 };
 
 // Immutable container of fixed-length columns conforming to a particular schema
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 8a70180130ca0..b5aba6c53efb9 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -267,6 +267,7 @@ Now, we build and install Arrow C++ libraries
          -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
          -DCMAKE_BUILD_TYPE=Release ^
          -DARROW_BUILD_TESTS=off ^
+         -DARROW_ZLIB_VENDORED=off ^
          -DARROW_PYTHON=on ..
    cmake --build . --target INSTALL --config Release
    cd ..\..
@@ -282,7 +283,6 @@ Now, we build parquet-cpp and install the result in the same place:
    cmake -G "Visual Studio 14 2015 Win64" ^
          -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
          -DCMAKE_BUILD_TYPE=Release ^
-         -DPARQUET_ZLIB_VENDORED=off ^
          -DPARQUET_BUILD_TESTS=off ..
    cmake --build . --target INSTALL --config Release
    popd
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index bf87173781d53..ae9ff885c4c8b 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -1084,8 +1084,8 @@ cdef class StructValue(ArrayValue):
             CStructArray* ap
             vector[shared_ptr[CField]] child_fields = self.type.type.children()
         ap = <CStructArray*> self.sp_array.get()
-        child_arrays = ap.fields()
-        wrapped_arrays = (pyarrow_wrap_array(child) for child in child_arrays)
+        wrapped_arrays = (pyarrow_wrap_array(ap.field(i))
+                          for i in range(ap.num_fields()))
         child_names = (child.get().name() for child in child_fields)
         # Return the struct as a dict
         return {
@@ -1214,6 +1214,9 @@ cdef class Array:
         self.ap = sp_array.get()
         self.type = pyarrow_wrap_data_type(self.sp_array.get().type())
 
+    def _debug_print(self):
+        check_status(DebugPrint(deref(self.ap), 0))
+
     @staticmethod
     def from_pandas(obj, mask=None, DataType type=None,
                     timestamps_to_ms=False,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index cc46c76d3f614..2db1dd163d8fb 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -91,12 +91,16 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int64_t null_count()
         Type type_id()
 
+        int num_fields()
+
         c_bool Equals(const CArray& arr)
         c_bool IsNull(int i)
 
         shared_ptr[CArray] Slice(int64_t offset)
         shared_ptr[CArray] Slice(int64_t offset, int64_t length)
 
+    CStatus DebugPrint(const CArray& arr, int indent)
+
     cdef cppclass CFixedWidthType" arrow::FixedWidthType"(CDataType):
         int bit_width()
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index ac4ad82f16b11..4c51d71f1f202 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -477,6 +477,23 @@ def test_column_of_arrays(self):
             field = schema.field_by_name(column)
             self._check_array_roundtrip(df[column], type=field.type)
 
+    def test_column_of_arrays_to_py(self):
+        # Test regression in ARROW-1199 not caught in above test
+        dtype = 'i1'
+        arr = np.array([
+            np.arange(10, dtype=dtype),
+            np.arange(5, dtype=dtype),
+            None,
+            np.arange(1, dtype=dtype)
+        ])
+        type_ = pa.list_(pa.int8())
+        parr = pa.Array.from_pandas(arr, type=type_)
+
+        assert parr[0].as_py() == list(range(10))
+        assert parr[1].as_py() == list(range(5))
+        assert parr[2].as_py() is None
+        assert parr[3].as_py() == [0]
+
     def test_column_of_lists(self):
         df, schema = dataframe_with_lists()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)

From dbedc8d20d9936076cbf47d8ef1ef893b8e05f45 Mon Sep 17 00:00:00 2001
From: Deepak Majeti <deepak.majeti@hpe.com>
Date: Tue, 11 Jul 2017 01:57:55 -0400
Subject: [PATCH 0793/1644] ARROW-1186: [C++] Add support to build only Parquet
 dependencies

Author: Deepak Majeti <deepak.majeti@hpe.com>

Closes #815 from majetideepak/ARROW-1186 and squashes the following commits:

1d3ac224 [Deepak Majeti] fix srcs and libs
ed7db98b [Deepak Majeti] Remove extra headers
f1761920 [Deepak Majeti] Review comments
46a52dd2 [Deepak Majeti] disable minimal parquet build if tests and benchmarks are built
1640f209 [Deepak Majeti] Add support to build only Parquet dependencies
---
 cpp/CMakeLists.txt                          |  36 ++-
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 132 ++++----
 cpp/src/arrow/util/compression.cc           | 333 --------------------
 cpp/src/arrow/util/compression_brotli.cc    |  63 ++++
 cpp/src/arrow/util/compression_lz4.cc       |  57 ++++
 cpp/src/arrow/util/compression_snappy.cc    |  63 ++++
 cpp/src/arrow/util/compression_zlib.cc      | 245 ++++++++++++++
 cpp/src/arrow/util/compression_zstd.cc      |  59 ++++
 8 files changed, 588 insertions(+), 400 deletions(-)
 create mode 100644 cpp/src/arrow/util/compression_brotli.cc
 create mode 100644 cpp/src/arrow/util/compression_lz4.cc
 create mode 100644 cpp/src/arrow/util/compression_snappy.cc
 create mode 100644 cpp/src/arrow/util/compression_zlib.cc
 create mode 100644 cpp/src/arrow/util/compression_zstd.cc

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index e67c7f66265f2..77f1e593f1cac 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -142,6 +142,14 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build with SSE4 optimizations"
     OFF)
 
+  option(ARROW_WITH_LZ4
+    "Build with lz4 compression"
+    ON)
+
+  option(ARROW_WITH_ZSTD
+    "Build with zstd compression"
+    ON)
+
   option(ARROW_ZLIB_VENDORED
     "Build our own zlib (some libz.a aren't configured for static linking)"
     ON)
@@ -161,12 +169,17 @@ endif()
 
 if(ARROW_BUILD_TESTS)
   set(ARROW_BUILD_STATIC ON)
+  set(ARROW_WITH_LZ4 ON)
+  set(ARROW_WITH_ZSTD ON)
 else()
   set(NO_TESTS 1)
 endif()
 
 if(NOT ARROW_BUILD_BENCHMARKS)
   set(NO_BENCHMARKS 1)
+else()
+  set(ARROW_WITH_LZ4 ON)
+  set(ARROW_WITH_ZSTD ON)
 endif()
 
 if(ARROW_HDFS)
@@ -536,9 +549,15 @@ set(ARROW_STATIC_LINK_LIBS
   brotli_enc
   brotli_common
   snappy
-  zlib
-  zstd_static
-  lz4_static)
+  zlib)
+
+if (ARROW_WITH_LZ4)
+  SET(ARROW_STATIC_LINK_LIBS lz4_static ${ARROW_STATIC_LINK_LIBS})
+endif()
+
+if (ARROW_WITH_ZSTD)
+  SET(ARROW_STATIC_LINK_LIBS zstd_static ${ARROW_STATIC_LINK_LIBS})
+endif()
 
 add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
 
@@ -664,11 +683,22 @@ set(ARROW_SRCS
 
   src/arrow/util/bit-util.cc
   src/arrow/util/compression.cc
+  src/arrow/util/compression_brotli.cc
+  src/arrow/util/compression_snappy.cc
+  src/arrow/util/compression_zlib.cc
   src/arrow/util/cpu-info.cc
   src/arrow/util/decimal.cc
   src/arrow/util/key_value_metadata.cc
 )
 
+if (ARROW_WITH_LZ4)
+  SET(ARROW_SRCS src/arrow/util/compression_lz4.cc ${ARROW_SRCS})
+endif()
+
+if (ARROW_WITH_ZSTD)
+  SET(ARROW_SRCS src/arrow/util/compression_zstd.cc ${ARROW_SRCS})
+endif()
+
 if (NOT ARROW_BOOST_HEADER_ONLY)
   set(ARROW_SRCS ${ARROW_SRCS}
     src/arrow/io/hdfs.cc
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 74939acabfffb..ab8fc5df8d27b 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -615,82 +615,86 @@ if (BROTLI_VENDORED)
   add_dependencies(brotli_common brotli_ep)
 endif()
 
+if (ARROW_WITH_LZ4)
 # ----------------------------------------------------------------------
 # Lz4
 
-find_package(Lz4)
-if (NOT LZ4_FOUND)
-  set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep")
-  set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
-
-  if (MSVC)
-    set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
-    set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
+  find_package(Lz4)
+  if (NOT LZ4_FOUND)
+    set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep")
+    set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
+  
+    if (MSVC)
+      set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
+      set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
+    else()
+      set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
+      set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh)
+    endif()
+  
+    ExternalProject_Add(lz4_ep
+        URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz"
+        UPDATE_COMMAND ""
+        PATCH_COMMAND ""
+        CONFIGURE_COMMAND ""
+        INSTALL_COMMAND ""
+        BINARY_DIR ${LZ4_BUILD_DIR}
+        BUILD_BYPRODUCTS ${LZ4_STATIC_LIB}
+        ${LZ4_BUILD_COMMAND}
+        )
+  
+    set(LZ4_VENDORED 1)
   else()
-    set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
-    set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh)
+    set(LZ4_VENDORED 0)
+  endif()
+  
+  include_directories(SYSTEM ${LZ4_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(lz4_static
+    STATIC_LIB ${LZ4_STATIC_LIB})
+  
+  if (LZ4_VENDORED)
+    add_dependencies(lz4_static lz4_ep)
   endif()
-
-  ExternalProject_Add(lz4_ep
-      URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz"
-      UPDATE_COMMAND ""
-      PATCH_COMMAND ""
-      CONFIGURE_COMMAND ""
-      INSTALL_COMMAND ""
-      BINARY_DIR ${LZ4_BUILD_DIR}
-      BUILD_BYPRODUCTS ${LZ4_STATIC_LIB}
-      ${LZ4_BUILD_COMMAND}
-      )
-
-  set(LZ4_VENDORED 1)
-else()
-  set(LZ4_VENDORED 0)
-endif()
-
-include_directories(SYSTEM ${LZ4_INCLUDE_DIR})
-ADD_THIRDPARTY_LIB(lz4_static
-  STATIC_LIB ${LZ4_STATIC_LIB})
-
-if (LZ4_VENDORED)
-  add_dependencies(lz4_static lz4_ep)
 endif()
-
+  
+if (ARROW_WITH_ZSTD)
 # ----------------------------------------------------------------------
 # ZSTD
 
-find_package(ZSTD)
-if (NOT ZSTD_FOUND)
-  set(ZSTD_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix/src/zstd_ep")
-  set(ZSTD_INCLUDE_DIR "${ZSTD_BUILD_DIR}/lib")
+  find_package(ZSTD)
+  if (NOT ZSTD_FOUND)
+    set(ZSTD_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix/src/zstd_ep")
+    set(ZSTD_INCLUDE_DIR "${ZSTD_BUILD_DIR}/lib")
 
-  if (MSVC)
-    set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib")
-    set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
+    if (MSVC)
+      set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib")
+      set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
+    else()
+      set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a")
+      set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh)
+    endif()
+
+    ExternalProject_Add(zstd_ep
+        URL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz"
+        UPDATE_COMMAND ""
+        PATCH_COMMAND ""
+        CONFIGURE_COMMAND ""
+        INSTALL_COMMAND ""
+        BINARY_DIR ${ZSTD_BUILD_DIR}
+        BUILD_BYPRODUCTS ${ZSTD_STATIC_LIB}
+        ${ZSTD_BUILD_COMMAND}
+        )
+
+    set(ZSTD_VENDORED 1)
   else()
-    set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a")
-    set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh)
+    set(ZSTD_VENDORED 0)
   endif()
 
-  ExternalProject_Add(zstd_ep
-      URL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz"
-      UPDATE_COMMAND ""
-      PATCH_COMMAND ""
-      CONFIGURE_COMMAND ""
-      INSTALL_COMMAND ""
-      BINARY_DIR ${ZSTD_BUILD_DIR}
-      BUILD_BYPRODUCTS ${ZSTD_STATIC_LIB}
-      ${ZSTD_BUILD_COMMAND}
-      )
-
-  set(ZSTD_VENDORED 1)
-else()
-  set(ZSTD_VENDORED 0)
-endif()
-
-include_directories(SYSTEM ${ZSTD_INCLUDE_DIR})
-ADD_THIRDPARTY_LIB(zstd_static
-  STATIC_LIB ${ZSTD_STATIC_LIB})
+  include_directories(SYSTEM ${ZSTD_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(zstd_static
+    STATIC_LIB ${ZSTD_STATIC_LIB})
 
-if (ZSTD_VENDORED)
-  add_dependencies(zstd_static zstd_ep)
+  if (ZSTD_VENDORED)
+    add_dependencies(zstd_static zstd_ep)
+  endif()
 endif()
diff --git a/cpp/src/arrow/util/compression.cc b/cpp/src/arrow/util/compression.cc
index 0f17e7cd0e932..4681488c54b5b 100644
--- a/cpp/src/arrow/util/compression.cc
+++ b/cpp/src/arrow/util/compression.cc
@@ -17,23 +17,11 @@
 
 #include "arrow/util/compression.h"
 
-// Work around warning caused by Snappy include
-#ifdef DISALLOW_COPY_AND_ASSIGN
-#undef DISALLOW_COPY_AND_ASSIGN
-#endif
-
 #include <cstdint>
 #include <memory>
 #include <sstream>
 #include <string>
 
-#include <brotli/decode.h>
-#include <brotli/encode.h>
-#include <lz4.h>
-#include <snappy.h>
-#include <zlib.h>
-#include <zstd.h>
-
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
@@ -62,325 +50,4 @@ Status Codec::Create(Compression::type codec_type, std::unique_ptr<Codec>* resul
   return Status::OK();
 }
 
-// ----------------------------------------------------------------------
-// gzip implementation
-
-// These are magic numbers from zlib.h.  Not clear why they are not defined
-// there.
-
-// Maximum window size
-static constexpr int WINDOW_BITS = 15;
-
-// Output Gzip.
-static constexpr int GZIP_CODEC = 16;
-
-// Determine if this is libz or gzip from header.
-static constexpr int DETECT_CODEC = 32;
-
-class GZipCodec::GZipCodecImpl {
- public:
-  explicit GZipCodecImpl(GZipCodec::Format format)
-      : format_(format),
-        compressor_initialized_(false),
-        decompressor_initialized_(false) {}
-
-  ~GZipCodecImpl() {
-    EndCompressor();
-    EndDecompressor();
-  }
-
-  Status InitCompressor() {
-    EndDecompressor();
-    memset(&stream_, 0, sizeof(stream_));
-
-    int ret;
-    // Initialize to run specified format
-    int window_bits = WINDOW_BITS;
-    if (format_ == DEFLATE) {
-      window_bits = -window_bits;
-    } else if (format_ == GZIP) {
-      window_bits += GZIP_CODEC;
-    }
-    if ((ret = deflateInit2(&stream_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits, 9,
-             Z_DEFAULT_STRATEGY)) != Z_OK) {
-      std::stringstream ss;
-      ss << "zlib deflateInit failed: " << std::string(stream_.msg);
-      return Status::IOError(ss.str());
-    }
-    compressor_initialized_ = true;
-    return Status::OK();
-  }
-
-  void EndCompressor() {
-    if (compressor_initialized_) { (void)deflateEnd(&stream_); }
-    compressor_initialized_ = false;
-  }
-
-  Status InitDecompressor() {
-    EndCompressor();
-    memset(&stream_, 0, sizeof(stream_));
-    int ret;
-
-    // Initialize to run either deflate or zlib/gzip format
-    int window_bits = format_ == DEFLATE ? -WINDOW_BITS : WINDOW_BITS | DETECT_CODEC;
-    if ((ret = inflateInit2(&stream_, window_bits)) != Z_OK) {
-      std::stringstream ss;
-      ss << "zlib inflateInit failed: " << std::string(stream_.msg);
-      return Status::IOError(ss.str());
-    }
-    decompressor_initialized_ = true;
-    return Status::OK();
-  }
-
-  void EndDecompressor() {
-    if (decompressor_initialized_) { (void)inflateEnd(&stream_); }
-    decompressor_initialized_ = false;
-  }
-
-  Status Decompress(int64_t input_length, const uint8_t* input, int64_t output_length,
-      uint8_t* output) {
-    if (!decompressor_initialized_) { RETURN_NOT_OK(InitDecompressor()); }
-    if (output_length == 0) {
-      // The zlib library does not allow *output to be NULL, even when output_length
-      // is 0 (inflate() will return Z_STREAM_ERROR). We don't consider this an
-      // error, so bail early if no output is expected. Note that we don't signal
-      // an error if the input actually contains compressed data.
-      return Status::OK();
-    }
-
-    // Reset the stream for this block
-    if (inflateReset(&stream_) != Z_OK) {
-      std::stringstream ss;
-      ss << "zlib inflateReset failed: " << std::string(stream_.msg);
-      return Status::IOError(ss.str());
-    }
-
-    int ret = 0;
-    // gzip can run in streaming mode or non-streaming mode.  We only
-    // support the non-streaming use case where we present it the entire
-    // compressed input and a buffer big enough to contain the entire
-    // compressed output.  In the case where we don't know the output,
-    // we just make a bigger buffer and try the non-streaming mode
-    // from the beginning again.
-    while (ret != Z_STREAM_END) {
-      stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input));
-      stream_.avail_in = static_cast<uInt>(input_length);
-      stream_.next_out = reinterpret_cast<Bytef*>(output);
-      stream_.avail_out = static_cast<uInt>(output_length);
-
-      // We know the output size.  In this case, we can use Z_FINISH
-      // which is more efficient.
-      ret = inflate(&stream_, Z_FINISH);
-      if (ret == Z_STREAM_END || ret != Z_OK) break;
-
-      // Failure, buffer was too small
-      std::stringstream ss;
-      ss << "Too small a buffer passed to GZipCodec. InputLength=" << input_length
-         << " OutputLength=" << output_length;
-      return Status::IOError(ss.str());
-    }
-
-    // Failure for some other reason
-    if (ret != Z_STREAM_END) {
-      std::stringstream ss;
-      ss << "GZipCodec failed: ";
-      if (stream_.msg != NULL) ss << stream_.msg;
-      return Status::IOError(ss.str());
-    }
-    return Status::OK();
-  }
-
-  int64_t MaxCompressedLen(int64_t input_length, const uint8_t* input) {
-    // Most be in compression mode
-    if (!compressor_initialized_) {
-      Status s = InitCompressor();
-      DCHECK(s.ok());
-    }
-    // TODO(wesm): deal with zlib < 1.2.3 (see Impala codebase)
-    return deflateBound(&stream_, static_cast<uLong>(input_length));
-  }
-
-  Status Compress(int64_t input_length, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output, int64_t* output_length) {
-    if (!compressor_initialized_) { RETURN_NOT_OK(InitCompressor()); }
-    stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input));
-    stream_.avail_in = static_cast<uInt>(input_length);
-    stream_.next_out = reinterpret_cast<Bytef*>(output);
-    stream_.avail_out = static_cast<uInt>(output_buffer_len);
-
-    int64_t ret = 0;
-    if ((ret = deflate(&stream_, Z_FINISH)) != Z_STREAM_END) {
-      if (ret == Z_OK) {
-        // will return Z_OK (and stream.msg NOT set) if stream.avail_out is too
-        // small
-        return Status::IOError("zlib deflate failed, output buffer too small");
-      }
-      std::stringstream ss;
-      ss << "zlib deflate failed: " << stream_.msg;
-      return Status::IOError(ss.str());
-    }
-
-    if (deflateReset(&stream_) != Z_OK) {
-      std::stringstream ss;
-      ss << "zlib deflateReset failed: " << std::string(stream_.msg);
-      return Status::IOError(ss.str());
-    }
-
-    // Actual output length
-    *output_length = output_buffer_len - stream_.avail_out;
-    return Status::OK();
-  }
-
- private:
-  // zlib is stateful and the z_stream state variable must be initialized
-  // before
-  z_stream stream_;
-
-  // Realistically, this will always be GZIP, but we leave the option open to
-  // configure
-  GZipCodec::Format format_;
-
-  // These variables are mutually exclusive. When the codec is in "compressor"
-  // state, compressor_initialized_ is true while decompressor_initialized_ is
-  // false. When it's decompressing, the opposite is true.
-  //
-  // Indeed, this is slightly hacky, but the alternative is having separate
-  // Compressor and Decompressor classes. If this ever becomes an issue, we can
-  // perform the refactoring then
-  bool compressor_initialized_;
-  bool decompressor_initialized_;
-};
-
-GZipCodec::GZipCodec(Format format) {
-  impl_.reset(new GZipCodecImpl(format));
-}
-
-GZipCodec::~GZipCodec() {}
-
-Status GZipCodec::Decompress(int64_t input_length, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output) {
-  return impl_->Decompress(input_length, input, output_buffer_len, output);
-}
-
-int64_t GZipCodec::MaxCompressedLen(int64_t input_length, const uint8_t* input) {
-  return impl_->MaxCompressedLen(input_length, input);
-}
-
-Status GZipCodec::Compress(int64_t input_length, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output, int64_t* output_length) {
-  return impl_->Compress(input_length, input, output_buffer_len, output, output_length);
-}
-
-const char* GZipCodec::name() const {
-  return "gzip";
-}
-
-// ----------------------------------------------------------------------
-// Snappy implementation
-
-Status SnappyCodec::Decompress(
-    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
-  if (!snappy::RawUncompress(reinterpret_cast<const char*>(input),
-          static_cast<size_t>(input_len), reinterpret_cast<char*>(output_buffer))) {
-    return Status::IOError("Corrupt snappy compressed data.");
-  }
-  return Status::OK();
-}
-
-int64_t SnappyCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
-  return snappy::MaxCompressedLength(input_len);
-}
-
-Status SnappyCodec::Compress(int64_t input_len, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
-  size_t output_len;
-  snappy::RawCompress(reinterpret_cast<const char*>(input),
-      static_cast<size_t>(input_len), reinterpret_cast<char*>(output_buffer),
-      &output_len);
-  *output_length = static_cast<int64_t>(output_len);
-  return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// Brotli implementation
-
-Status BrotliCodec::Decompress(
-    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
-  size_t output_size = output_len;
-  if (BrotliDecoderDecompress(input_len, input, &output_size, output_buffer) !=
-      BROTLI_DECODER_RESULT_SUCCESS) {
-    return Status::IOError("Corrupt brotli compressed data.");
-  }
-  return Status::OK();
-}
-
-int64_t BrotliCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
-  return BrotliEncoderMaxCompressedSize(input_len);
-}
-
-Status BrotliCodec::Compress(int64_t input_len, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
-  size_t output_len = output_buffer_len;
-  // TODO: Make quality configurable. We use 8 as a default as it is the best
-  //       trade-off for Parquet workload
-  if (BrotliEncoderCompress(8, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, input_len,
-          input, &output_len, output_buffer) == BROTLI_FALSE) {
-    return Status::IOError("Brotli compression failure.");
-  }
-  *output_length = output_len;
-  return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// ZSTD implementation
-
-Status ZSTDCodec::Decompress(
-    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
-  int64_t decompressed_size = ZSTD_decompress(output_buffer,
-      static_cast<size_t>(output_len), input, static_cast<size_t>(input_len));
-  if (decompressed_size != output_len) {
-    return Status::IOError("Corrupt ZSTD compressed data.");
-  }
-  return Status::OK();
-}
-
-int64_t ZSTDCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
-  return ZSTD_compressBound(input_len);
-}
-
-Status ZSTDCodec::Compress(int64_t input_len, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
-  *output_length = ZSTD_compress(output_buffer, static_cast<size_t>(output_buffer_len),
-      input, static_cast<size_t>(input_len), 1);
-  if (ZSTD_isError(*output_length)) {
-    return Status::IOError("ZSTD compression failure.");
-  }
-  return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// Lz4 implementation
-
-Status Lz4Codec::Decompress(
-    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
-  int64_t decompressed_size = LZ4_decompress_safe(reinterpret_cast<const char*>(input),
-      reinterpret_cast<char*>(output_buffer), static_cast<int>(input_len),
-      static_cast<int>(output_len));
-  if (decompressed_size < 1) { return Status::IOError("Corrupt Lz4 compressed data."); }
-  return Status::OK();
-}
-
-int64_t Lz4Codec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
-  return LZ4_compressBound(static_cast<int>(input_len));
-}
-
-Status Lz4Codec::Compress(int64_t input_len, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
-  *output_length = LZ4_compress_default(reinterpret_cast<const char*>(input),
-      reinterpret_cast<char*>(output_buffer), static_cast<int>(input_len),
-      static_cast<int>(output_buffer_len));
-  if (*output_length < 1) { return Status::IOError("Lz4 compression failure."); }
-  return Status::OK();
-}
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression_brotli.cc b/cpp/src/arrow/util/compression_brotli.cc
new file mode 100644
index 0000000000000..f5f9f571bcd01
--- /dev/null
+++ b/cpp/src/arrow/util/compression_brotli.cc
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/compression.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include <brotli/decode.h>
+#include <brotli/encode.h>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// Brotli implementation
+
+Status BrotliCodec::Decompress(
+    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+  size_t output_size = output_len;
+  if (BrotliDecoderDecompress(input_len, input, &output_size, output_buffer) !=
+      BROTLI_DECODER_RESULT_SUCCESS) {
+    return Status::IOError("Corrupt brotli compressed data.");
+  }
+  return Status::OK();
+}
+
+int64_t BrotliCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+  return BrotliEncoderMaxCompressedSize(input_len);
+}
+
+Status BrotliCodec::Compress(int64_t input_len, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+  size_t output_len = output_buffer_len;
+  // TODO: Make quality configurable. We use 8 as a default as it is the best
+  //       trade-off for Parquet workload
+  if (BrotliEncoderCompress(8, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, input_len,
+          input, &output_len, output_buffer) == BROTLI_FALSE) {
+    return Status::IOError("Brotli compression failure.");
+  }
+  *output_length = output_len;
+  return Status::OK();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/compression_lz4.cc b/cpp/src/arrow/util/compression_lz4.cc
new file mode 100644
index 0000000000000..27094f1360fe1
--- /dev/null
+++ b/cpp/src/arrow/util/compression_lz4.cc
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/compression.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include <lz4.h>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// Lz4 implementation
+
+Status Lz4Codec::Decompress(
+    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+  int64_t decompressed_size = LZ4_decompress_safe(reinterpret_cast<const char*>(input),
+      reinterpret_cast<char*>(output_buffer), static_cast<int>(input_len),
+      static_cast<int>(output_len));
+  if (decompressed_size < 1) { return Status::IOError("Corrupt Lz4 compressed data."); }
+  return Status::OK();
+}
+
+int64_t Lz4Codec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+  return LZ4_compressBound(static_cast<int>(input_len));
+}
+
+Status Lz4Codec::Compress(int64_t input_len, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+  *output_length = LZ4_compress_default(reinterpret_cast<const char*>(input),
+      reinterpret_cast<char*>(output_buffer), static_cast<int>(input_len),
+      static_cast<int>(output_buffer_len));
+  if (*output_length < 1) { return Status::IOError("Lz4 compression failure."); }
+  return Status::OK();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/compression_snappy.cc b/cpp/src/arrow/util/compression_snappy.cc
new file mode 100644
index 0000000000000..ab418b06f5fc5
--- /dev/null
+++ b/cpp/src/arrow/util/compression_snappy.cc
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/compression.h"
+
+// Work around warning caused by Snappy include
+#ifdef DISALLOW_COPY_AND_ASSIGN
+#undef DISALLOW_COPY_AND_ASSIGN
+#endif
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include <snappy.h>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// Snappy implementation
+
+Status SnappyCodec::Decompress(
+    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+  if (!snappy::RawUncompress(reinterpret_cast<const char*>(input),
+          static_cast<size_t>(input_len), reinterpret_cast<char*>(output_buffer))) {
+    return Status::IOError("Corrupt snappy compressed data.");
+  }
+  return Status::OK();
+}
+
+int64_t SnappyCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+  return snappy::MaxCompressedLength(input_len);
+}
+
+Status SnappyCodec::Compress(int64_t input_len, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+  size_t output_len;
+  snappy::RawCompress(reinterpret_cast<const char*>(input),
+      static_cast<size_t>(input_len), reinterpret_cast<char*>(output_buffer),
+      &output_len);
+  *output_length = static_cast<int64_t>(output_len);
+  return Status::OK();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/compression_zlib.cc b/cpp/src/arrow/util/compression_zlib.cc
new file mode 100644
index 0000000000000..934ea1b258c21
--- /dev/null
+++ b/cpp/src/arrow/util/compression_zlib.cc
@@ -0,0 +1,245 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/compression.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include <zlib.h>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// gzip implementation
+
+// These are magic numbers from zlib.h.  Not clear why they are not defined
+// there.
+
+// Maximum window size
+static constexpr int WINDOW_BITS = 15;
+
+// Output Gzip.
+static constexpr int GZIP_CODEC = 16;
+
+// Determine if this is libz or gzip from header.
+static constexpr int DETECT_CODEC = 32;
+
+class GZipCodec::GZipCodecImpl {
+ public:
+  explicit GZipCodecImpl(GZipCodec::Format format)
+      : format_(format),
+        compressor_initialized_(false),
+        decompressor_initialized_(false) {}
+
+  ~GZipCodecImpl() {
+    EndCompressor();
+    EndDecompressor();
+  }
+
+  Status InitCompressor() {
+    EndDecompressor();
+    memset(&stream_, 0, sizeof(stream_));
+
+    int ret;
+    // Initialize to run specified format
+    int window_bits = WINDOW_BITS;
+    if (format_ == DEFLATE) {
+      window_bits = -window_bits;
+    } else if (format_ == GZIP) {
+      window_bits += GZIP_CODEC;
+    }
+    if ((ret = deflateInit2(&stream_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits, 9,
+             Z_DEFAULT_STRATEGY)) != Z_OK) {
+      std::stringstream ss;
+      ss << "zlib deflateInit failed: " << std::string(stream_.msg);
+      return Status::IOError(ss.str());
+    }
+    compressor_initialized_ = true;
+    return Status::OK();
+  }
+
+  void EndCompressor() {
+    if (compressor_initialized_) { (void)deflateEnd(&stream_); }
+    compressor_initialized_ = false;
+  }
+
+  Status InitDecompressor() {
+    EndCompressor();
+    memset(&stream_, 0, sizeof(stream_));
+    int ret;
+
+    // Initialize to run either deflate or zlib/gzip format
+    int window_bits = format_ == DEFLATE ? -WINDOW_BITS : WINDOW_BITS | DETECT_CODEC;
+    if ((ret = inflateInit2(&stream_, window_bits)) != Z_OK) {
+      std::stringstream ss;
+      ss << "zlib inflateInit failed: " << std::string(stream_.msg);
+      return Status::IOError(ss.str());
+    }
+    decompressor_initialized_ = true;
+    return Status::OK();
+  }
+
+  void EndDecompressor() {
+    if (decompressor_initialized_) { (void)inflateEnd(&stream_); }
+    decompressor_initialized_ = false;
+  }
+
+  Status Decompress(int64_t input_length, const uint8_t* input, int64_t output_length,
+      uint8_t* output) {
+    if (!decompressor_initialized_) { RETURN_NOT_OK(InitDecompressor()); }
+    if (output_length == 0) {
+      // The zlib library does not allow *output to be NULL, even when output_length
+      // is 0 (inflate() will return Z_STREAM_ERROR). We don't consider this an
+      // error, so bail early if no output is expected. Note that we don't signal
+      // an error if the input actually contains compressed data.
+      return Status::OK();
+    }
+
+    // Reset the stream for this block
+    if (inflateReset(&stream_) != Z_OK) {
+      std::stringstream ss;
+      ss << "zlib inflateReset failed: " << std::string(stream_.msg);
+      return Status::IOError(ss.str());
+    }
+
+    int ret = 0;
+    // gzip can run in streaming mode or non-streaming mode.  We only
+    // support the non-streaming use case where we present it the entire
+    // compressed input and a buffer big enough to contain the entire
+    // compressed output.  In the case where we don't know the output,
+    // we just make a bigger buffer and try the non-streaming mode
+    // from the beginning again.
+    while (ret != Z_STREAM_END) {
+      stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input));
+      stream_.avail_in = static_cast<uInt>(input_length);
+      stream_.next_out = reinterpret_cast<Bytef*>(output);
+      stream_.avail_out = static_cast<uInt>(output_length);
+
+      // We know the output size.  In this case, we can use Z_FINISH
+      // which is more efficient.
+      ret = inflate(&stream_, Z_FINISH);
+      if (ret == Z_STREAM_END || ret != Z_OK) break;
+
+      // Failure, buffer was too small
+      std::stringstream ss;
+      ss << "Too small a buffer passed to GZipCodec. InputLength=" << input_length
+         << " OutputLength=" << output_length;
+      return Status::IOError(ss.str());
+    }
+
+    // Failure for some other reason
+    if (ret != Z_STREAM_END) {
+      std::stringstream ss;
+      ss << "GZipCodec failed: ";
+      if (stream_.msg != NULL) ss << stream_.msg;
+      return Status::IOError(ss.str());
+    }
+    return Status::OK();
+  }
+
+  int64_t MaxCompressedLen(int64_t input_length, const uint8_t* input) {
+    // Most be in compression mode
+    if (!compressor_initialized_) {
+      Status s = InitCompressor();
+      DCHECK(s.ok());
+    }
+    // TODO(wesm): deal with zlib < 1.2.3 (see Impala codebase)
+    return deflateBound(&stream_, static_cast<uLong>(input_length));
+  }
+
+  Status Compress(int64_t input_length, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output, int64_t* output_length) {
+    if (!compressor_initialized_) { RETURN_NOT_OK(InitCompressor()); }
+    stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input));
+    stream_.avail_in = static_cast<uInt>(input_length);
+    stream_.next_out = reinterpret_cast<Bytef*>(output);
+    stream_.avail_out = static_cast<uInt>(output_buffer_len);
+
+    int64_t ret = 0;
+    if ((ret = deflate(&stream_, Z_FINISH)) != Z_STREAM_END) {
+      if (ret == Z_OK) {
+        // will return Z_OK (and stream.msg NOT set) if stream.avail_out is too
+        // small
+        return Status::IOError("zlib deflate failed, output buffer too small");
+      }
+      std::stringstream ss;
+      ss << "zlib deflate failed: " << stream_.msg;
+      return Status::IOError(ss.str());
+    }
+
+    if (deflateReset(&stream_) != Z_OK) {
+      std::stringstream ss;
+      ss << "zlib deflateReset failed: " << std::string(stream_.msg);
+      return Status::IOError(ss.str());
+    }
+
+    // Actual output length
+    *output_length = output_buffer_len - stream_.avail_out;
+    return Status::OK();
+  }
+
+ private:
+  // zlib is stateful and the z_stream state variable must be initialized
+  // before
+  z_stream stream_;
+
+  // Realistically, this will always be GZIP, but we leave the option open to
+  // configure
+  GZipCodec::Format format_;
+
+  // These variables are mutually exclusive. When the codec is in "compressor"
+  // state, compressor_initialized_ is true while decompressor_initialized_ is
+  // false. When it's decompressing, the opposite is true.
+  //
+  // Indeed, this is slightly hacky, but the alternative is having separate
+  // Compressor and Decompressor classes. If this ever becomes an issue, we can
+  // perform the refactoring then
+  bool compressor_initialized_;
+  bool decompressor_initialized_;
+};
+
+GZipCodec::GZipCodec(Format format) {
+  impl_.reset(new GZipCodecImpl(format));
+}
+
+GZipCodec::~GZipCodec() {}
+
+Status GZipCodec::Decompress(int64_t input_length, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output) {
+  return impl_->Decompress(input_length, input, output_buffer_len, output);
+}
+
+int64_t GZipCodec::MaxCompressedLen(int64_t input_length, const uint8_t* input) {
+  return impl_->MaxCompressedLen(input_length, input);
+}
+
+Status GZipCodec::Compress(int64_t input_length, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output, int64_t* output_length) {
+  return impl_->Compress(input_length, input, output_buffer_len, output, output_length);
+}
+
+const char* GZipCodec::name() const {
+  return "gzip";
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/compression_zstd.cc b/cpp/src/arrow/util/compression_zstd.cc
new file mode 100644
index 0000000000000..a03a3fc336493
--- /dev/null
+++ b/cpp/src/arrow/util/compression_zstd.cc
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/compression.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include <zstd.h>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// ZSTD implementation
+
+Status ZSTDCodec::Decompress(
+    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+  int64_t decompressed_size = ZSTD_decompress(output_buffer,
+      static_cast<size_t>(output_len), input, static_cast<size_t>(input_len));
+  if (decompressed_size != output_len) {
+    return Status::IOError("Corrupt ZSTD compressed data.");
+  }
+  return Status::OK();
+}
+
+int64_t ZSTDCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+  return ZSTD_compressBound(input_len);
+}
+
+Status ZSTDCodec::Compress(int64_t input_len, const uint8_t* input,
+    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+  *output_length = ZSTD_compress(output_buffer, static_cast<size_t>(output_buffer_len),
+      input, static_cast<size_t>(input_len), 1);
+  if (ZSTD_isError(*output_length)) {
+    return Status::IOError("ZSTD compression failure.");
+  }
+  return Status::OK();
+}
+
+}  // namespace arrow

From e8c09c625ba7f00ce8d3a78aee5675f4eb635bfc Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 11 Jul 2017 18:30:39 +0200
Subject: [PATCH 0794/1644] ARROW-1205: C++: Reference to type objects in
 ArrayLoader may cause segmentation faults

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #831 from xhochy/ARROW-1205 and squashes the following commits:

081a843 [Uwe L. Korn] ARROW-1205: C++: Reference to type objects in ArrayLoader may cause segmentation faults.
---
 cpp/src/arrow/ipc/reader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index ae46207d2fbc1..8ca4d82714a75 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -244,7 +244,7 @@ class ArrayLoader {
   }
 
  private:
-  const std::shared_ptr<DataType>& type_;
+  const std::shared_ptr<DataType> type_;
   ArrayLoaderContext* context_;
 
   // Used in visitor pattern

From afb192824a75ab81fbc8dcd2da56409186bb23e0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 11 Jul 2017 14:13:04 -0400
Subject: [PATCH 0795/1644] ARROW-1206: [C++] Add finer grained control of
 compression library support, do not expose symbols which may not be built in
 compression.h

MSVC will fail to link arrow.dll if one or more of the compression libraries is disabled. This moves those symbols into their own header files. I also added additional options so any of the compression libraries can be disabled in user builds. If they are disabled then `arrow::Codec::Create` will fail with a helpful message

This build will still fail while the parquet-cpp build is broken

Close #810

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #832 from wesm/fix-compressors-windows and squashes the following commits:

9c914b3d [Wes McKinney] Put compressor implementations in own headers, options to toggle each compressor on/off, and add defines so that Codec::Create will work regardless of what compression library support is built
---
 cpp/CMakeLists.txt                       | 67 +++++++++++++++----
 cpp/src/arrow/util/CMakeLists.txt        |  5 ++
 cpp/src/arrow/util/compression-test.cc   | 32 ++++-----
 cpp/src/arrow/util/compression.cc        | 46 +++++++++++++
 cpp/src/arrow/util/compression.h         | 85 ------------------------
 cpp/src/arrow/util/compression_brotli.cc |  2 +-
 cpp/src/arrow/util/compression_brotli.h  | 45 +++++++++++++
 cpp/src/arrow/util/compression_lz4.cc    |  2 +-
 cpp/src/arrow/util/compression_lz4.h     | 45 +++++++++++++
 cpp/src/arrow/util/compression_snappy.cc |  2 +-
 cpp/src/arrow/util/compression_snappy.h  | 44 ++++++++++++
 cpp/src/arrow/util/compression_zlib.cc   |  2 +-
 cpp/src/arrow/util/compression_zlib.h    | 60 +++++++++++++++++
 cpp/src/arrow/util/compression_zstd.cc   |  2 +-
 cpp/src/arrow/util/compression_zstd.h    | 45 +++++++++++++
 15 files changed, 365 insertions(+), 119 deletions(-)
 create mode 100644 cpp/src/arrow/util/compression_brotli.h
 create mode 100644 cpp/src/arrow/util/compression_lz4.h
 create mode 100644 cpp/src/arrow/util/compression_snappy.h
 create mode 100644 cpp/src/arrow/util/compression_zlib.h
 create mode 100644 cpp/src/arrow/util/compression_zstd.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 77f1e593f1cac..d84611c1c8cb4 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -142,10 +142,22 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build with SSE4 optimizations"
     OFF)
 
+  option(ARROW_WITH_BROTLI
+    "Build with Brotli compression"
+    ON)
+
   option(ARROW_WITH_LZ4
     "Build with lz4 compression"
     ON)
 
+  option(ARROW_WITH_SNAPPY
+    "Build with Snappy compression"
+    ON)
+
+  option(ARROW_WITH_ZLIB
+    "Build with zlib compression"
+    ON)
+
   option(ARROW_WITH_ZSTD
     "Build with zstd compression"
     ON)
@@ -167,19 +179,21 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   endif()
 endif()
 
-if(ARROW_BUILD_TESTS)
+if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
   set(ARROW_BUILD_STATIC ON)
+  set(ARROW_WITH_BROTLI ON)
   set(ARROW_WITH_LZ4 ON)
+  set(ARROW_WITH_SNAPPY ON)
+  set(ARROW_WITH_ZLIB ON)
   set(ARROW_WITH_ZSTD ON)
-else()
+endif()
+
+if(NOT ARROW_BUILD_TESTS)
   set(NO_TESTS 1)
 endif()
 
 if(NOT ARROW_BUILD_BENCHMARKS)
   set(NO_BENCHMARKS 1)
-else()
-  set(ARROW_WITH_LZ4 ON)
-  set(ARROW_WITH_ZSTD ON)
 endif()
 
 if(ARROW_HDFS)
@@ -544,17 +558,28 @@ endif()
 # Linker and Dependencies
 ############################################################
 
-set(ARROW_STATIC_LINK_LIBS
-  brotli_dec
-  brotli_enc
-  brotli_common
-  snappy
-  zlib)
+set(ARROW_STATIC_LINK_LIBS)
+
+if (ARROW_WITH_BROTLI)
+  SET(ARROW_STATIC_LINK_LIBS
+    brotli_dec
+    brotli_enc
+    brotli_common
+    ${ARROW_STATIC_LINK_LIBS})
+endif()
 
 if (ARROW_WITH_LZ4)
   SET(ARROW_STATIC_LINK_LIBS lz4_static ${ARROW_STATIC_LINK_LIBS})
 endif()
 
+if (ARROW_WITH_SNAPPY)
+  SET(ARROW_STATIC_LINK_LIBS snappy ${ARROW_STATIC_LINK_LIBS})
+endif()
+
+if (ARROW_WITH_ZLIB)
+  SET(ARROW_STATIC_LINK_LIBS zlib ${ARROW_STATIC_LINK_LIBS})
+endif()
+
 if (ARROW_WITH_ZSTD)
   SET(ARROW_STATIC_LINK_LIBS zstd_static ${ARROW_STATIC_LINK_LIBS})
 endif()
@@ -683,19 +708,33 @@ set(ARROW_SRCS
 
   src/arrow/util/bit-util.cc
   src/arrow/util/compression.cc
-  src/arrow/util/compression_brotli.cc
-  src/arrow/util/compression_snappy.cc
-  src/arrow/util/compression_zlib.cc
   src/arrow/util/cpu-info.cc
   src/arrow/util/decimal.cc
   src/arrow/util/key_value_metadata.cc
 )
 
+if (ARROW_WITH_BROTLI)
+  add_definitions(-DARROW_WITH_BROTLI)
+  SET(ARROW_SRCS src/arrow/util/compression_brotli.cc ${ARROW_SRCS})
+endif()
+
 if (ARROW_WITH_LZ4)
+  add_definitions(-DARROW_WITH_LZ4)
   SET(ARROW_SRCS src/arrow/util/compression_lz4.cc ${ARROW_SRCS})
 endif()
 
+if (ARROW_WITH_SNAPPY)
+  add_definitions(-DARROW_WITH_SNAPPY)
+  SET(ARROW_SRCS src/arrow/util/compression_snappy.cc ${ARROW_SRCS})
+endif()
+
+if (ARROW_WITH_ZLIB)
+  add_definitions(-DARROW_WITH_ZLIB)
+  SET(ARROW_SRCS src/arrow/util/compression_zlib.cc ${ARROW_SRCS})
+endif()
+
 if (ARROW_WITH_ZSTD)
+  add_definitions(-DARROW_WITH_ZSTD)
   SET(ARROW_SRCS src/arrow/util/compression_zstd.cc ${ARROW_SRCS})
 endif()
 
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index bc1eeb2d98167..8c1a23d62f0cd 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -26,6 +26,11 @@ install(FILES
   bpacking.h
   compiler-util.h
   compression.h
+  compression_brotli.h
+  compression_lz4.h
+  compression_snappy.h
+  compression_zlib.h
+  compression_zstd.h
   cpu-info.h
   key_value_metadata.h
   hash-util.h
diff --git a/cpp/src/arrow/util/compression-test.cc b/cpp/src/arrow/util/compression-test.cc
index 56dcc2a9ec42b..f7739fc6dd7c3 100644
--- a/cpp/src/arrow/util/compression-test.cc
+++ b/cpp/src/arrow/util/compression-test.cc
@@ -29,69 +29,71 @@ using std::vector;
 
 namespace arrow {
 
-template <typename T>
+template <Compression::type CODEC>
 void CheckCodecRoundtrip(const vector<uint8_t>& data) {
   // create multiple compressors to try to break them
-  T c1;
-  T c2;
+  std::unique_ptr<Codec> c1, c2;
 
-  int max_compressed_len = static_cast<int>(c1.MaxCompressedLen(data.size(), &data[0]));
+  ASSERT_OK(Codec::Create(CODEC, &c1));
+  ASSERT_OK(Codec::Create(CODEC, &c2));
+
+  int max_compressed_len = static_cast<int>(c1->MaxCompressedLen(data.size(), &data[0]));
   std::vector<uint8_t> compressed(max_compressed_len);
   std::vector<uint8_t> decompressed(data.size());
 
   // compress with c1
   int64_t actual_size;
-  ASSERT_OK(c1.Compress(
+  ASSERT_OK(c1->Compress(
       data.size(), &data[0], max_compressed_len, &compressed[0], &actual_size));
   compressed.resize(actual_size);
 
   // decompress with c2
-  ASSERT_OK(c2.Decompress(
+  ASSERT_OK(c2->Decompress(
       compressed.size(), &compressed[0], decompressed.size(), &decompressed[0]));
 
   ASSERT_EQ(data, decompressed);
 
   // compress with c2
   int64_t actual_size2;
-  ASSERT_OK(c2.Compress(
+  ASSERT_OK(c2->Compress(
       data.size(), &data[0], max_compressed_len, &compressed[0], &actual_size2));
   ASSERT_EQ(actual_size2, actual_size);
 
   // decompress with c1
-  ASSERT_OK(c1.Decompress(
+  ASSERT_OK(c1->Decompress(
       compressed.size(), &compressed[0], decompressed.size(), &decompressed[0]));
 
   ASSERT_EQ(data, decompressed);
 }
 
-template <typename T>
+template <Compression::type CODEC>
 void CheckCodec() {
   int sizes[] = {10000, 100000};
   for (int data_size : sizes) {
     vector<uint8_t> data(data_size);
     test::random_bytes(data_size, 1234, data.data());
-    CheckCodecRoundtrip<T>(data);
+    CheckCodecRoundtrip<CODEC>(data);
   }
 }
 
 TEST(TestCompressors, Snappy) {
-  CheckCodec<SnappyCodec>();
+  CheckCodec<Compression::SNAPPY>();
 }
 
 TEST(TestCompressors, Brotli) {
-  CheckCodec<BrotliCodec>();
+  CheckCodec<Compression::BROTLI>();
 }
 
 TEST(TestCompressors, GZip) {
-  CheckCodec<GZipCodec>();
+  CheckCodec<Compression::GZIP>();
 }
 
 TEST(TestCompressors, ZSTD) {
-  CheckCodec<ZSTDCodec>();
+  CheckCodec<Compression::ZSTD>();
 }
 
 TEST(TestCompressors, Lz4) {
-  CheckCodec<Lz4Codec>();
+  CheckCodec<Compression::LZ4>();
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression.cc b/cpp/src/arrow/util/compression.cc
index 4681488c54b5b..b4e5373c5b019 100644
--- a/cpp/src/arrow/util/compression.cc
+++ b/cpp/src/arrow/util/compression.cc
@@ -22,6 +22,26 @@
 #include <sstream>
 #include <string>
 
+#ifdef ARROW_WITH_BROTLI
+#include "arrow/util/compression_brotli.h"
+#endif
+
+#ifdef ARROW_WITH_SNAPPY
+#include "arrow/util/compression_snappy.h"
+#endif
+
+#ifdef ARROW_WITH_LZ4
+#include "arrow/util/compression_lz4.h"
+#endif
+
+#ifdef ARROW_WITH_ZLIB
+#include "arrow/util/compression_zlib.h"
+#endif
+
+#ifdef ARROW_WITH_ZSTD
+#include "arrow/util/compression_zstd.h"
+#endif
+
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
@@ -34,15 +54,41 @@ Status Codec::Create(Compression::type codec_type, std::unique_ptr<Codec>* resul
     case Compression::UNCOMPRESSED:
       break;
     case Compression::SNAPPY:
+#ifdef ARROW_WITH_SNAPPY
       result->reset(new SnappyCodec());
+#else
+      return Status::NotImplemented("Snappy codec support not built");
+#endif
       break;
     case Compression::GZIP:
+#ifdef ARROW_WITH_ZLIB
       result->reset(new GZipCodec());
+#else
+      return Status::NotImplemented("Gzip codec support not built");
+#endif
       break;
     case Compression::LZO:
       return Status::NotImplemented("LZO codec not implemented");
     case Compression::BROTLI:
+#ifdef ARROW_WITH_BROTLI
       result->reset(new BrotliCodec());
+#else
+      return Status::NotImplemented("Brotli codec support not built");
+#endif
+      break;
+    case Compression::LZ4:
+#ifdef ARROW_WITH_LZ4
+      result->reset(new Lz4Codec());
+#else
+      return Status::NotImplemented("LZ4 codec support not built");
+#endif
+      break;
+    case Compression::ZSTD:
+#ifdef ARROW_WITH_ZSTD
+      result->reset(new ZSTDCodec());
+#else
+      return Status::NotImplemented("ZSTD codec support not built");
+#endif
       break;
     default:
       return Status::Invalid("Unrecognized codec");
diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h
index d38215320b70b..19c61179a502a 100644
--- a/cpp/src/arrow/util/compression.h
+++ b/cpp/src/arrow/util/compression.h
@@ -47,91 +47,6 @@ class ARROW_EXPORT Codec {
   virtual const char* name() const = 0;
 };
 
-// Snappy codec.
-class ARROW_EXPORT SnappyCodec : public Codec {
- public:
-  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
-
-  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
-
-  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
-
-  const char* name() const override { return "snappy"; }
-};
-
-// Brotli codec.
-class ARROW_EXPORT BrotliCodec : public Codec {
- public:
-  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
-
-  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
-
-  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
-
-  const char* name() const override { return "brotli"; }
-};
-
-// GZip codec.
-class ARROW_EXPORT GZipCodec : public Codec {
- public:
-  /// Compression formats supported by the zlib library
-  enum Format {
-    ZLIB,
-    DEFLATE,
-    GZIP,
-  };
-
-  explicit GZipCodec(Format format = GZIP);
-  virtual ~GZipCodec();
-
-  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
-
-  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
-
-  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
-
-  const char* name() const override;
-
- private:
-  // The gzip compressor is stateful
-  class GZipCodecImpl;
-  std::unique_ptr<GZipCodecImpl> impl_;
-};
-
-// ZSTD codec.
-class ARROW_EXPORT ZSTDCodec : public Codec {
- public:
-  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
-
-  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
-
-  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
-
-  const char* name() const override { return "zstd"; }
-};
-
-// Lz4 codec.
-class ARROW_EXPORT Lz4Codec : public Codec {
- public:
-  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
-
-  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
-
-  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
-
-  const char* name() const override { return "lz4"; }
-};
-
 }  // namespace arrow
 
 #endif
diff --git a/cpp/src/arrow/util/compression_brotli.cc b/cpp/src/arrow/util/compression_brotli.cc
index f5f9f571bcd01..c03573bc46c1c 100644
--- a/cpp/src/arrow/util/compression_brotli.cc
+++ b/cpp/src/arrow/util/compression_brotli.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/compression.h"
+#include "arrow/util/compression_brotli.h"
 
 #include <cstdint>
 #include <memory>
diff --git a/cpp/src/arrow/util/compression_brotli.h b/cpp/src/arrow/util/compression_brotli.h
new file mode 100644
index 0000000000000..08bd3379e3489
--- /dev/null
+++ b/cpp/src/arrow/util/compression_brotli.h
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_COMPRESSION_BROTLI_H
+#define ARROW_UTIL_COMPRESSION_BROTLI_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/status.h"
+#include "arrow/util/compression.h"
+
+namespace arrow {
+
+// Brotli codec.
+class ARROW_EXPORT BrotliCodec : public Codec {
+ public:
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override { return "brotli"; }
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_COMPRESSION_BROTLI_H
diff --git a/cpp/src/arrow/util/compression_lz4.cc b/cpp/src/arrow/util/compression_lz4.cc
index 27094f1360fe1..65eaa08946e79 100644
--- a/cpp/src/arrow/util/compression_lz4.cc
+++ b/cpp/src/arrow/util/compression_lz4.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/compression.h"
+#include "arrow/util/compression_lz4.h"
 
 #include <cstdint>
 #include <memory>
diff --git a/cpp/src/arrow/util/compression_lz4.h b/cpp/src/arrow/util/compression_lz4.h
new file mode 100644
index 0000000000000..9668fec126b12
--- /dev/null
+++ b/cpp/src/arrow/util/compression_lz4.h
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_COMPRESSION_LZ4_H
+#define ARROW_UTIL_COMPRESSION_LZ4_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/status.h"
+#include "arrow/util/compression.h"
+
+namespace arrow {
+
+// Lz4 codec.
+class ARROW_EXPORT Lz4Codec : public Codec {
+ public:
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override { return "lz4"; }
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_COMPRESSION_LZ4_H
diff --git a/cpp/src/arrow/util/compression_snappy.cc b/cpp/src/arrow/util/compression_snappy.cc
index ab418b06f5fc5..db2b67355109a 100644
--- a/cpp/src/arrow/util/compression_snappy.cc
+++ b/cpp/src/arrow/util/compression_snappy.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/compression.h"
+#include "arrow/util/compression_snappy.h"
 
 // Work around warning caused by Snappy include
 #ifdef DISALLOW_COPY_AND_ASSIGN
diff --git a/cpp/src/arrow/util/compression_snappy.h b/cpp/src/arrow/util/compression_snappy.h
new file mode 100644
index 0000000000000..25281e1a97a16
--- /dev/null
+++ b/cpp/src/arrow/util/compression_snappy.h
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_COMPRESSION_SNAPPY_H
+#define ARROW_UTIL_COMPRESSION_SNAPPY_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/status.h"
+#include "arrow/util/compression.h"
+
+namespace arrow {
+
+class ARROW_EXPORT SnappyCodec : public Codec {
+ public:
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override { return "snappy"; }
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_COMPRESSION_SNAPPY_H
diff --git a/cpp/src/arrow/util/compression_zlib.cc b/cpp/src/arrow/util/compression_zlib.cc
index 934ea1b258c21..3ff33b82028e8 100644
--- a/cpp/src/arrow/util/compression_zlib.cc
+++ b/cpp/src/arrow/util/compression_zlib.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/compression.h"
+#include "arrow/util/compression_zlib.h"
 
 #include <cstdint>
 #include <memory>
diff --git a/cpp/src/arrow/util/compression_zlib.h b/cpp/src/arrow/util/compression_zlib.h
new file mode 100644
index 0000000000000..517a06175ec8f
--- /dev/null
+++ b/cpp/src/arrow/util/compression_zlib.h
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_COMPRESSION_ZLIB_H
+#define ARROW_UTIL_COMPRESSION_ZLIB_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/status.h"
+#include "arrow/util/compression.h"
+
+namespace arrow {
+
+// GZip codec.
+class ARROW_EXPORT GZipCodec : public Codec {
+ public:
+  /// Compression formats supported by the zlib library
+  enum Format {
+    ZLIB,
+    DEFLATE,
+    GZIP,
+  };
+
+  explicit GZipCodec(Format format = GZIP);
+  virtual ~GZipCodec();
+
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override;
+
+ private:
+  // The gzip compressor is stateful
+  class GZipCodecImpl;
+  std::unique_ptr<GZipCodecImpl> impl_;
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_COMPRESSION_ZLIB_H
diff --git a/cpp/src/arrow/util/compression_zstd.cc b/cpp/src/arrow/util/compression_zstd.cc
index a03a3fc336493..5511cb9dd8f37 100644
--- a/cpp/src/arrow/util/compression_zstd.cc
+++ b/cpp/src/arrow/util/compression_zstd.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/compression.h"
+#include "arrow/util/compression_zstd.h"
 
 #include <cstdint>
 #include <memory>
diff --git a/cpp/src/arrow/util/compression_zstd.h b/cpp/src/arrow/util/compression_zstd.h
new file mode 100644
index 0000000000000..2356d5862e01a
--- /dev/null
+++ b/cpp/src/arrow/util/compression_zstd.h
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_COMPRESSION_ZSTD_H
+#define ARROW_UTIL_COMPRESSION_ZSTD_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/status.h"
+#include "arrow/util/compression.h"
+
+namespace arrow {
+
+// ZSTD codec.
+class ARROW_EXPORT ZSTDCodec : public Codec {
+ public:
+  Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+      uint8_t* output_buffer) override;
+
+  Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+      uint8_t* output_buffer, int64_t* output_length) override;
+
+  int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
+
+  const char* name() const override { return "zstd"; }
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_COMPRESSION_ZSTD_H

From f0ecc06735ec3f12b778d767b5d7c9f312188d95 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Wed, 12 Jul 2017 13:23:27 +0200
Subject: [PATCH 0796/1644] =?UTF-8?q?ARROW-1208:=20[C++]=20Temporary=20rem?=
 =?UTF-8?q?ove=20conda's=20build=20of=20zstd=20from=20Toolcha=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…in Appveyor build

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #835 from MaxRis/ARROW-1208 and squashes the following commits:

e12f836 [Max Risuhin] ARROW-1208: [C++] Temporary remove conda's build of zstd from Toolchain Appveyor build
---
 ci/msvc-build.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index c84d12d97d7d8..d4f7c8b32a3f6 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -42,7 +42,7 @@ conda create -n arrow -q -y python=%PYTHON% ^
 if "%CONFIGURATION%" == "Toolchain" (
   conda install -n arrow -q -y -c conda-forge ^
       flatbuffers rapidjson cmake git boost-cpp ^
-      thrift-cpp snappy zlib brotli gflags lz4-c zstd
+      thrift-cpp snappy zlib brotli gflags lz4-c
 )
 
 call activate arrow

From 28e06d870b6159a9dc3b40d88244a5a7ce08b1b8 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Thu, 13 Jul 2017 08:26:13 -0400
Subject: [PATCH 0797/1644] ARROW-1194: [Python] Expose MockOutputStream in
 pyarrow.

This allows you to get the size of a record batch and schema through pyarrow by writing to a mock output stream. You can then use the resulting size to allocate an appropriately sized buffer to actually write to.

Example usage.

```python
import pyarrow as pa
import pandas as pd

val = pd.DataFrame({'a': [1, 2, 3]})
record_batch = pa.RecordBatch.from_pandas(val)

# Get the size of the record batch and schema
sink = pa.MockOutputStream()
stream_writer = pa.RecordBatchStreamWriter(sink, record_batch.schema)
stream_writer.write_batch(record_batch)
size = sink.size()
```

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #830 from robertnishihara/mockoutputstream and squashes the following commits:

4e15cd9 [Robert Nishihara] Expose MockOutputStream to Python.
---
 cpp/src/arrow/io/memory.cc               | 17 +++++++++++
 cpp/src/arrow/io/memory.h                | 16 ++++++++++
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  2 +-
 cpp/src/arrow/ipc/util.h                 | 23 --------------
 cpp/src/arrow/ipc/writer.cc              |  4 +--
 python/pyarrow/__init__.py               |  2 +-
 python/pyarrow/includes/libarrow.pxd     |  5 +++
 python/pyarrow/io.pxi                    | 12 ++++++++
 python/pyarrow/tests/test_io.py          | 39 ++++++++++++++++++++++++
 9 files changed, 93 insertions(+), 27 deletions(-)

diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 22721f1dd6871..4d8bf63757d64 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -99,6 +99,23 @@ Status BufferOutputStream::Reserve(int64_t nbytes) {
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// OutputStream that doesn't write anything
+
+Status MockOutputStream::Close() {
+  return Status::OK();
+}
+
+Status MockOutputStream::Tell(int64_t* position) {
+  *position = extent_bytes_written_;
+  return Status::OK();
+}
+
+Status MockOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+  extent_bytes_written_ += nbytes;
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // In-memory buffer writer
 
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index f1b59905d8a3a..06384f0d4c4b7 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -67,6 +67,22 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   uint8_t* mutable_data_;
 };
 
+// A helper class to tracks the size of allocations
+class ARROW_EXPORT MockOutputStream : public OutputStream {
+ public:
+  MockOutputStream() : extent_bytes_written_(0) {}
+
+  // Implement the OutputStream interface
+  Status Close() override;
+  Status Tell(int64_t* position) override;
+  Status Write(const uint8_t* data, int64_t nbytes) override;
+
+  int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
+
+ private:
+  int64_t extent_bytes_written_;
+};
+
 /// \brief Enables random writes into a fixed-size mutable buffer
 ///
 class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index c99816c760e6f..c71d0467b9736 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -340,7 +340,7 @@ TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) {
 }
 
 void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
-  ipc::MockOutputStream mock;
+  io::MockOutputStream mock;
   int32_t mock_metadata_length = -1;
   int64_t mock_body_length = -1;
   int64_t size = -1;
diff --git a/cpp/src/arrow/ipc/util.h b/cpp/src/arrow/ipc/util.h
index 2000c61e7ed57..49a7d01ac6f35 100644
--- a/cpp/src/arrow/ipc/util.h
+++ b/cpp/src/arrow/ipc/util.h
@@ -37,29 +37,6 @@ static inline int64_t PaddedLength(int64_t nbytes, int64_t alignment = kArrowAli
   return ((nbytes + alignment - 1) / alignment) * alignment;
 }
 
-// A helper class to tracks the size of allocations
-class MockOutputStream : public io::OutputStream {
- public:
-  MockOutputStream() : extent_bytes_written_(0) {}
-
-  Status Close() override { return Status::OK(); }
-
-  Status Write(const uint8_t* data, int64_t nbytes) override {
-    extent_bytes_written_ += nbytes;
-    return Status::OK();
-  }
-
-  Status Tell(int64_t* position) override {
-    *position = extent_bytes_written_;
-    return Status::OK();
-  }
-
-  int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
-
- private:
-  int64_t extent_bytes_written_;
-};
-
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 592bca222f70b..7563343554c7c 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -566,7 +566,7 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
   // emulates the behavior of Write without actually writing
   int32_t metadata_length = 0;
   int64_t body_length = 0;
-  MockOutputStream dst;
+  io::MockOutputStream dst;
   RETURN_NOT_OK(WriteRecordBatch(batch, 0, &dst, &metadata_length, &body_length,
       default_memory_pool(), kMaxNestingDepth, true));
   *size = dst.GetExtentBytesWritten();
@@ -577,7 +577,7 @@ Status GetTensorSize(const Tensor& tensor, int64_t* size) {
   // emulates the behavior of Write without actually writing
   int32_t metadata_length = 0;
   int64_t body_length = 0;
-  MockOutputStream dst;
+  io::MockOutputStream dst;
   RETURN_NOT_OK(WriteTensor(tensor, &dst, &metadata_length, &body_length));
   *size = dst.GetExtentBytesWritten();
   return Status::OK();
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 43109548bc148..37aec6c8803dc 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -68,7 +68,7 @@
                          frombuffer, read_tensor, write_tensor,
                          memory_map, create_memory_map,
                          get_record_batch_size, get_tensor_size,
-                         have_libhdfs, have_libhdfs3)
+                         have_libhdfs, have_libhdfs3, MockOutputStream)
 
 from pyarrow.lib import (MemoryPool, total_allocated_bytes,
                          set_memory_pool, default_memory_pool)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 2db1dd163d8fb..9fad8244442aa 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -540,6 +540,11 @@ cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
         (OutputStream):
         CBufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
 
+    cdef cppclass CMockOutputStream" arrow::io::MockOutputStream"\
+        (OutputStream):
+        CMockOutputStream()
+        int64_t GetExtentBytesWritten()
+
 
 cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
     cdef cppclass SchemaMessage:
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 9f0ad7e4bf3a3..cfa751d500069 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -534,6 +534,18 @@ cdef class BufferOutputStream(NativeFile):
         return pyarrow_wrap_buffer(<shared_ptr[CBuffer]> self.buffer)
 
 
+cdef class MockOutputStream(NativeFile):
+
+    def __cinit__(self):
+        self.wr_file.reset(new CMockOutputStream())
+        self.is_readable = 0
+        self.is_writeable = 1
+        self.is_open = True
+
+    def size(self):
+        return (<CMockOutputStream*>self.wr_file.get()).GetExtentBytesWritten()
+
+
 cdef class BufferReader(NativeFile):
     """
     Zero-copy reader from objects convertible to Arrow buffer
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 397b7a895c1e7..cadf7868fc177 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -23,6 +23,8 @@
 
 import numpy as np
 
+import pandas as pd
+
 from pyarrow.compat import u, guid
 import pyarrow as pa
 
@@ -232,6 +234,43 @@ def test_nativefile_write_memoryview():
     assert buf.to_pybytes() == data * 2
 
 
+# ----------------------------------------------------------------------
+# Mock output stream
+
+
+def test_mock_output_stream():
+    # Make sure that the MockOutputStream and the BufferOutputStream record the
+    # same size
+
+    # 10 bytes
+    val = b'dataabcdef'
+
+    f1 = pa.MockOutputStream()
+    f2 = pa.BufferOutputStream()
+
+    K = 1000
+    for i in range(K):
+        f1.write(val)
+        f2.write(val)
+
+    assert f1.size() == len(f2.get_result())
+
+    # Do the same test with a pandas DataFrame
+    val = pd.DataFrame({'a': [1, 2, 3]})
+    record_batch = pa.RecordBatch.from_pandas(val)
+
+    f1 = pa.MockOutputStream()
+    f2 = pa.BufferOutputStream()
+
+    stream_writer1 = pa.RecordBatchStreamWriter(f1, record_batch.schema)
+    stream_writer2 = pa.RecordBatchStreamWriter(f2, record_batch.schema)
+
+    stream_writer1.write_batch(record_batch)
+    stream_writer2.write_batch(record_batch)
+
+    assert f1.size() == len(f2.get_result())
+
+
 # ----------------------------------------------------------------------
 # OS files and memory maps
 

From 74bc8735b258f9437fd0e239d892da857c83b780 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 13 Jul 2017 08:43:01 -0400
Subject: [PATCH 0798/1644] ARROW-1150: Silence AdaptiveIntBuilder compiler
 warning on MSVC

We need to explicitly mark those sections as safe.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #834 from xhochy/ARROW-1150 and squashes the following commits:

a03df6c [Uwe L. Korn] ARROW-1150: Silence AdaptiveIntBuilder compiler warning on MSVC
---
 cpp/src/arrow/builder.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index c3bc7459953f9..155d81ae7faf6 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -341,6 +341,10 @@ Status AdaptiveIntBuilder::Append(
     std::memcpy(reinterpret_cast<int64_t*>(raw_data_) + length_, values,
         sizeof(int64_t) * length);
   } else {
+#ifdef _MSC_VER
+#  pragma warning(push)
+#  pragma warning(disable:4996)
+#endif
     // int_size_ may have changed, so we need to recheck
     switch (int_size_) {
       case 1: {
@@ -361,6 +365,9 @@ Status AdaptiveIntBuilder::Append(
       default:
         DCHECK(false);
     }
+#ifdef _MSC_VER
+#  pragma warning(pop)
+#endif
   }
 
   // length_ is update by these
@@ -489,6 +496,10 @@ Status AdaptiveUIntBuilder::Append(
     std::memcpy(reinterpret_cast<uint64_t*>(raw_data_) + length_, values,
         sizeof(uint64_t) * length);
   } else {
+#ifdef _MSC_VER
+#  pragma warning(push)
+#  pragma warning(disable:4996)
+#endif
     // int_size_ may have changed, so we need to recheck
     switch (int_size_) {
       case 1: {
@@ -509,6 +520,9 @@ Status AdaptiveUIntBuilder::Append(
       default:
         DCHECK(false);
     }
+#ifdef _MSC_VER
+#  pragma warning(pop)
+#endif
   }
 
   // length_ is update by these

From 85892a28835ec61dd589a63874bf01b6d23d4655 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 13 Jul 2017 08:44:54 -0400
Subject: [PATCH 0799/1644] ARROW-1187: Python: Feather: Serialize a DataFrame
 with None column

Change-Id: Id489a4fdc203849b747f754f0b48d64a56b2ff8f

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #833 from xhochy/ARROW-1187 and squashes the following commits:

74a09a1 [Uwe L. Korn] ARROW-1187: Python: Feather: Serialize a DataFrame with None column
---
 cpp/src/arrow/compare.cc             |  2 ++
 cpp/src/arrow/ipc/feather-test.cc    | 19 ++++++++++
 cpp/src/arrow/ipc/feather.cc         | 54 +++++++++++++++++++++++-----
 cpp/src/arrow/ipc/test-common.h      |  9 +++++
 python/pyarrow/tests/test_feather.py |  4 +++
 5 files changed, 79 insertions(+), 9 deletions(-)

diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 23f5a1945b4c1..1465e0b414fe3 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -615,6 +615,8 @@ inline Status ArrayEqualsImpl(const Array& left, const Array& right, bool* are_e
     *are_equal = false;
   } else if (left.length() == 0) {
     *are_equal = true;
+  } else if (left.null_count() == left.length()) {
+    *are_equal = true;
   } else {
     VISITOR visitor(right);
     RETURN_NOT_OK(VisitArrayInline(left, &visitor));
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index a7793f20183e7..029aae31ff52c 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -386,6 +386,25 @@ TEST_F(TestTableWriter, VLenPrimitiveRoundTrip) {
   CheckBatch(*batch);
 }
 
+TEST_F(TestTableWriter, PrimitiveNullRoundTrip) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeNullRecordBatch(&batch));
+
+  for (int i = 0; i < batch->num_columns(); ++i) {
+    ASSERT_OK(writer_->Append(batch->column_name(i), *batch->column(i)));
+  }
+  Finish();
+
+  std::shared_ptr<Column> col;
+  for (int i = 0; i < batch->num_columns(); ++i) {
+    ASSERT_OK(reader_->GetColumn(i, &col));
+    ASSERT_EQ(batch->column_name(i), col->name());
+    StringArray str_values(batch->column(i)->length(), nullptr, nullptr,
+        batch->column(i)->null_bitmap(), batch->column(i)->null_count());
+    CheckArrays(str_values, *col->data()->chunk(0));
+  }
+}
+
 }  // namespace feather
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 37b01c5191992..1bcd50520b61a 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -70,6 +70,21 @@ static Status WritePadded(io::OutputStream* stream, const uint8_t* data, int64_t
   return Status::OK();
 }
 
+/// For compability, we need to write any data sometimes just to keep producing
+/// files that can be read with an older reader.
+static Status WritePaddedBlank(
+    io::OutputStream* stream, int64_t length, int64_t* bytes_written) {
+  const uint8_t null = 0;
+  for (int64_t i = 0; i < length; i++) {
+    RETURN_NOT_OK(stream->Write(&null, 1));
+  }
+
+  int64_t remainder = PaddedLength(length) - length;
+  if (remainder != 0) { RETURN_NOT_OK(stream->Write(kPaddingBytes, remainder)); }
+  *bytes_written = length + remainder;
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // TableBuilder
 
@@ -542,8 +557,13 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     if (values.null_count() > 0) {
       // We assume there is one bit for each value in values.nulls, aligned on a
       // byte boundary, and we write this much data into the stream
-      RETURN_NOT_OK(WritePadded(stream_.get(), values.null_bitmap()->data(),
-          values.null_bitmap()->size(), &bytes_written));
+      if (values.null_bitmap()) {
+        RETURN_NOT_OK(WritePadded(stream_.get(), values.null_bitmap()->data(),
+            values.null_bitmap()->size(), &bytes_written));
+      } else {
+        RETURN_NOT_OK(WritePaddedBlank(
+            stream_.get(), BitUtil::BytesForBits(values.length()), &bytes_written));
+      }
       meta->total_bytes += bytes_written;
     }
 
@@ -556,12 +576,16 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
 
       int64_t offset_bytes = sizeof(int32_t) * (values.length() + 1);
 
-      values_bytes = bin_values.raw_value_offsets()[values.length()];
+      if (bin_values.value_offsets()) {
+        values_bytes = bin_values.raw_value_offsets()[values.length()];
 
-      // Write the variable-length offsets
-      RETURN_NOT_OK(WritePadded(stream_.get(),
-          reinterpret_cast<const uint8_t*>(bin_values.raw_value_offsets()), offset_bytes,
-          &bytes_written));
+        // Write the variable-length offsets
+        RETURN_NOT_OK(WritePadded(stream_.get(),
+            reinterpret_cast<const uint8_t*>(bin_values.raw_value_offsets()),
+            offset_bytes, &bytes_written));
+      } else {
+        RETURN_NOT_OK(WritePaddedBlank(stream_.get(), offset_bytes, &bytes_written));
+      }
       meta->total_bytes += bytes_written;
 
       if (bin_values.value_data()) { values_buffer = bin_values.value_data()->data(); }
@@ -578,8 +602,12 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
 
       if (prim_values.values()) { values_buffer = prim_values.values()->data(); }
     }
-    RETURN_NOT_OK(
-        WritePadded(stream_.get(), values_buffer, values_bytes, &bytes_written));
+    if (values_buffer) {
+      RETURN_NOT_OK(
+          WritePadded(stream_.get(), values_buffer, values_bytes, &bytes_written));
+    } else {
+      RETURN_NOT_OK(WritePaddedBlank(stream_.get(), values_bytes, &bytes_written));
+    }
     meta->total_bytes += bytes_written;
 
     return Status::OK();
@@ -593,6 +621,14 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     return Status::OK();
   }
 
+  Status Visit(const NullArray& values) override {
+    // As long as R doesn't support NA, we write this as a StringColumn
+    // to ensure stable roundtrips.
+    StringArray str_values(
+        values.length(), nullptr, nullptr, values.null_bitmap(), values.null_count());
+    return WritePrimitiveValues(str_values);
+  }
+
 #define VISIT_PRIMITIVE(TYPE) \
   Status Visit(const TYPE& values) override { return WritePrimitiveValues(values); }
 
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 6fdf1cc8a8f39..a542c878e056e 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -247,6 +247,15 @@ Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
+Status MakeNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
+  const int64_t length = 500;
+  auto f0 = field("f0", null());
+  std::shared_ptr<Schema> schema(new Schema({f0}));
+  std::shared_ptr<Array> a0 = std::make_shared<NullArray>(length);
+  out->reset(new RecordBatch(schema, length, {a0}));
+  return Status::OK();
+}
+
 Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out) {
   // Make the schema
   auto f0 = field("f0", kListInt32);
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 71e4fee83dc01..91bf56bd8e649 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -294,6 +294,10 @@ def test_empty_strings(self):
         df = pd.DataFrame({'strings': [''] * 10})
         self._check_pandas_roundtrip(df)
 
+    def test_all_none(self):
+        df = pd.DataFrame({'all_none': [None] * 10})
+        self._check_pandas_roundtrip(df, null_counts=[10])
+
     def test_multithreaded_read(self):
         data = {'c{0}'.format(i): [''] * 10
                 for i in range(100)}

From 248a9d8303101c76be7bceb1254855121a188b33 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 13 Jul 2017 10:41:12 -0400
Subject: [PATCH 0800/1644] ARROW-1212: [GLib] Add
 garrow_binary_array_get_offsets_buffer()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #836 from kou/glib-add-binary-get-offsets-buffer and squashes the following commits:

3b15321 [Kouhei Sutou] Add garrow_binary_array_get_offsets_buffer()
---
 c_glib/arrow-glib/array.cpp      | 16 ++++++++++++++++
 c_glib/arrow-glib/array.h        |  1 +
 c_glib/test/test-binary-array.rb | 12 ++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index ab62bba3f2aba..30e51fb309f97 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -1429,6 +1429,22 @@ garrow_binary_array_get_buffer(GArrowBinaryArray *array)
   return garrow_buffer_new_raw(&arrow_data);
 }
 
+/**
+ * garrow_binary_array_get_offsets_buffer:
+ * @array: A #GArrowBinaryArray.
+ *
+ * Returns: (transfer full): The offsets of the array as #GArrowBuffer.
+ */
+GArrowBuffer *
+garrow_binary_array_get_offsets_buffer(GArrowBinaryArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_binary_array =
+    static_cast<arrow::BinaryArray *>(arrow_array.get());
+  auto arrow_offsets = arrow_binary_array->value_offsets();
+  return garrow_buffer_new_raw(&arrow_offsets);
+}
+
 
 G_DEFINE_TYPE(GArrowStringArray,               \
               garrow_string_array,             \
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 6043e904ef2df..1b2ba9fc1cc39 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -804,6 +804,7 @@ GArrowBinaryArray *garrow_binary_array_new(gint64 length,
 GBytes *garrow_binary_array_get_value(GArrowBinaryArray *array,
                                       gint64 i);
 GArrowBuffer *garrow_binary_array_get_buffer(GArrowBinaryArray *array);
+GArrowBuffer *garrow_binary_array_get_offsets_buffer(GArrowBinaryArray *array);
 
 #define GARROW_TYPE_STRING_ARRAY                \
   (garrow_string_array_get_type())
diff --git a/c_glib/test/test-binary-array.rb b/c_glib/test/test-binary-array.rb
index 9ae122a9a742b..2dfd9cfbaaf14 100644
--- a/c_glib/test/test-binary-array.rb
+++ b/c_glib/test/test-binary-array.rb
@@ -46,4 +46,16 @@ def test_buffer
     array = builder.finish
     assert_equal(data1 + data2, array.buffer.data.to_s)
   end
+
+  def test_offsets_buffer
+    data1 = "\x00\x01"
+    data2 = "\x02\x03\x04"
+    builder = Arrow::BinaryArrayBuilder.new
+    builder.append(data1)
+    builder.append(data2)
+    array = builder.finish
+    byte_per_offset = 4
+    assert_equal([0, 2, 5].pack("l*"),
+                 array.offsets_buffer.data.to_s[0, byte_per_offset * 3])
+  end
 end

From c7e09952083b70b62d2aa6f3cd4ad89e68a5ae88 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Fri, 14 Jul 2017 19:51:14 +0200
Subject: [PATCH 0801/1644] =?UTF-8?q?ARROW-1208:=20[C++]=20Install=20zstd?=
 =?UTF-8?q?=20from=20conda=20for=20Toolchain=20Appveyor=20buil=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…d job

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #837 from MaxRis/ARROW-1208 and squashes the following commits:

080026b [Max Risuhin] ARROW-1208: [C++] Install zstd from conda for Toolchain Appveyor build job
---
 ci/msvc-build.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index d4f7c8b32a3f6..c84d12d97d7d8 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -42,7 +42,7 @@ conda create -n arrow -q -y python=%PYTHON% ^
 if "%CONFIGURATION%" == "Toolchain" (
   conda install -n arrow -q -y -c conda-forge ^
       flatbuffers rapidjson cmake git boost-cpp ^
-      thrift-cpp snappy zlib brotli gflags lz4-c
+      thrift-cpp snappy zlib brotli gflags lz4-c zstd
 )
 
 call activate arrow

From 8cad26e3a33752892767d63acea2186cd62d3c8e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 14 Jul 2017 15:32:20 -0400
Subject: [PATCH 0802/1644] ARROW-1200: C++: Switch DictionaryBuilder to signed
 integers

Change-Id: I26e06bf361e032072d7cc1478c673da29e403b04

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #838 from xhochy/ARROW-1200 and squashes the following commits:

41b6afe7 [Uwe L. Korn] ARROW-1200: C++: Switch DictionaryBuilder to signed integers
---
 cpp/src/arrow/array-test.cc | 20 ++++++++++----------
 cpp/src/arrow/builder.h     |  2 +-
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index bfdb9231b5602..01bef030cc3d4 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1524,9 +1524,9 @@ TYPED_TEST(TestDictionaryBuilder, Basic) {
   ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
   std::shared_ptr<Array> dict_array;
   ASSERT_OK(dict_builder.Finish(&dict_array));
-  auto dtype = std::make_shared<DictionaryType>(uint8(), dict_array);
+  auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
 
-  UInt8Builder int_builder(default_memory_pool());
+  Int8Builder int_builder(default_memory_pool());
   ASSERT_OK(int_builder.Append(0));
   ASSERT_OK(int_builder.Append(1));
   ASSERT_OK(int_builder.Append(0));
@@ -1557,9 +1557,9 @@ TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
   ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
   std::shared_ptr<Array> dict_array;
   ASSERT_OK(dict_builder.Finish(&dict_array));
-  auto dtype = std::make_shared<DictionaryType>(uint8(), dict_array);
+  auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
 
-  UInt8Builder int_builder(default_memory_pool());
+  Int8Builder int_builder(default_memory_pool());
   ASSERT_OK(int_builder.Append(0));
   ASSERT_OK(int_builder.Append(1));
   ASSERT_OK(int_builder.Append(0));
@@ -1578,7 +1578,7 @@ TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
     DictionaryBuilder<TypeParam> builder(default_memory_pool());
     // Build expected data
     NumericBuilder<TypeParam> dict_builder(default_memory_pool());
-    UInt16Builder int_builder(default_memory_pool());
+    Int16Builder int_builder(default_memory_pool());
 
     // Fill with 1024 different values
     for (int64_t i = 0; i < 1024; i++) {
@@ -1599,7 +1599,7 @@ TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
     // Finalize expected data
     std::shared_ptr<Array> dict_array;
     ASSERT_OK(dict_builder.Finish(&dict_array));
-    auto dtype = std::make_shared<DictionaryType>(uint16(), dict_array);
+    auto dtype = std::make_shared<DictionaryType>(int16(), dict_array);
     std::shared_ptr<Array> int_array;
     ASSERT_OK(int_builder.Finish(&int_array));
 
@@ -1624,9 +1624,9 @@ TEST(TestStringDictionaryBuilder, Basic) {
   ASSERT_OK(str_builder.Append("test2"));
   std::shared_ptr<Array> str_array;
   ASSERT_OK(str_builder.Finish(&str_array));
-  auto dtype = std::make_shared<DictionaryType>(uint8(), str_array);
+  auto dtype = std::make_shared<DictionaryType>(int8(), str_array);
 
-  UInt8Builder int_builder(default_memory_pool());
+  Int8Builder int_builder(default_memory_pool());
   ASSERT_OK(int_builder.Append(0));
   ASSERT_OK(int_builder.Append(1));
   ASSERT_OK(int_builder.Append(0));
@@ -1642,7 +1642,7 @@ TEST(TestStringDictionaryBuilder, DoubleTableSize) {
   StringDictionaryBuilder builder(default_memory_pool());
   // Build expected data
   StringBuilder str_builder(default_memory_pool());
-  UInt16Builder int_builder(default_memory_pool());
+  Int16Builder int_builder(default_memory_pool());
 
   // Fill with 1024 different values
   for (int64_t i = 0; i < 1024; i++) {
@@ -1665,7 +1665,7 @@ TEST(TestStringDictionaryBuilder, DoubleTableSize) {
   // Finalize expected data
   std::shared_ptr<Array> str_array;
   ASSERT_OK(str_builder.Finish(&str_array));
-  auto dtype = std::make_shared<DictionaryType>(uint16(), str_array);
+  auto dtype = std::make_shared<DictionaryType>(int16(), str_array);
   std::shared_ptr<Array> int_array;
   ASSERT_OK(int_builder.Finish(&int_array));
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 12f3683ec0d41..c0a075b7d6c57 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -759,7 +759,7 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
   int mod_bitmask_;
 
   typename TypeTraits<T>::BuilderType dict_builder_;
-  AdaptiveUIntBuilder values_builder_;
+  AdaptiveIntBuilder values_builder_;
 };
 
 class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {

From cb31b8bc17560a5ac770c8529bfb859dffb5b98c Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 14 Jul 2017 17:49:02 -0400
Subject: [PATCH 0803/1644] ARROW-1215: [Python] Generate documentation for
 class members in API Reference

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #840 from pcmoritz/pyarrow-doc-members and squashes the following commits:

b09f9df1 [Philipp Moritz] generate documentation for class members in API Reference
---
 python/doc/source/conf.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/python/doc/source/conf.py b/python/doc/source/conf.py
index 4a72fd0d6f958..c7f098fc0d5cd 100644
--- a/python/doc/source/conf.py
+++ b/python/doc/source/conf.py
@@ -56,6 +56,9 @@
     'IPython.sphinxext.ipython_console_highlighting'
 ]
 
+# Show members for classes in .. autosummary
+autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance', 'inherited-members']
+
 # numpydoc configuration
 napoleon_use_rtype = False
 

From bfe39590670c54483391d2d6a127ff7a98d513a0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 15 Jul 2017 13:25:10 +0200
Subject: [PATCH 0804/1644] ARROW-962: [Python] Add schema attribute to
 RecordBatchFileReader

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #843 from wesm/ARROW-962 and squashes the following commits:

b7144aa [Wes McKinney] Add schema attribute to RecordBatchFileReader
---
 python/pyarrow/io.pxi            | 8 ++++++--
 python/pyarrow/tests/test_ipc.py | 1 +
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index cfa751d500069..64cce03f4c997 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -990,8 +990,7 @@ cdef class _RecordBatchReader:
             check_status(CRecordBatchStreamReader.Open(in_stream, &reader))
 
         self.reader = <shared_ptr[CRecordBatchReader]> reader
-        self.schema = Schema()
-        self.schema.init_schema(self.reader.get().schema())
+        self.schema = pyarrow_wrap_schema(self.reader.get().schema())
 
     def get_next_batch(self):
         """
@@ -1049,6 +1048,9 @@ cdef class _RecordBatchFileReader:
     cdef:
         shared_ptr[CRecordBatchFileReader] reader
 
+    cdef readonly:
+        Schema schema
+
     def __cinit__(self):
         pass
 
@@ -1067,6 +1069,8 @@ cdef class _RecordBatchFileReader:
             else:
                 check_status(CRecordBatchFileReader.Open(reader, &self.reader))
 
+        self.schema = pyarrow_wrap_schema(self.reader.get().schema())
+
     property num_record_batches:
 
         def __get__(self):
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 47ef75602bc65..b91a8e9608df5 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -89,6 +89,7 @@ def test_simple_roundtrip(self):
             # it works. Must convert back to DataFrame
             batch = reader.get_batch(i)
             assert batches[i].equals(batch)
+            assert reader.schema.equals(batches[0].schema)
 
     def test_read_all(self):
         batches = self.write_batches()

From f62db836a0eb606b1e92d3e29fef8012fc9d000f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 15 Jul 2017 13:30:54 +0200
Subject: [PATCH 0805/1644] ARROW-1100: [Python] Add mode property to
 NativeFile

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #842 from wesm/ARROW-1100 and squashes the following commits:

ac245e6 [Wes McKinney] Add docstring
64831de [Wes McKinney] Add mode property to pyarrow.NativeFile
---
 python/pyarrow/io.pxi           | 26 ++++++++++++++++++++++---
 python/pyarrow/tests/test_io.py | 34 +++++++++++++++++++++++++++++++--
 2 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 64cce03f4c997..c15be5e435764 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -54,6 +54,26 @@ cdef class NativeFile:
     def __exit__(self, exc_type, exc_value, tb):
         self.close()
 
+    property mode:
+        """
+        The file mode. Currently instances of NativeFile may support:
+
+        * rb: binary read
+        * wb: binary write
+        * rb+: binary read and write
+        """
+
+        def __get__(self):
+            # Emulate built-in file modes
+            if self.is_readable and self.is_writeable:
+                return 'rb+'
+            elif self.is_readable:
+                return 'rb'
+            elif self.is_writeable:
+                return 'wb'
+            else:
+                raise ValueError('File object is malformed, has no mode')
+
     def close(self):
         if self.is_open:
             with nogil:
@@ -346,7 +366,7 @@ cdef class MemoryMappedFile(NativeFile):
 
         return result
 
-    def open(self, path, mode='r'):
+    def _open(self, path, mode='r'):
         self.path = path
 
         cdef:
@@ -360,7 +380,7 @@ cdef class MemoryMappedFile(NativeFile):
         elif mode in ('w', 'wb'):
             c_mode = FileMode_WRITE
             self.is_writeable = 1
-        elif mode == 'r+w':
+        elif mode in ('r+', 'r+b', 'rb+'):
             c_mode = FileMode_READWRITE
             self.is_readable = 1
             self.is_writeable = 1
@@ -388,7 +408,7 @@ def memory_map(path, mode='r'):
     mmap : MemoryMappedFile
     """
     cdef MemoryMappedFile mmap = MemoryMappedFile()
-    mmap.open(path, mode)
+    mmap._open(path, mode)
     return mmap
 
 
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index cadf7868fc177..6258f6de18191 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -354,7 +354,7 @@ def test_memory_map_writer(tmpdir):
     with open(path, 'wb') as f:
         f.write(data)
 
-    f = pa.memory_map(path, mode='r+w')
+    f = pa.memory_map(path, mode='r+b')
 
     f.seek(10)
     f.write('peekaboo')
@@ -363,7 +363,7 @@ def test_memory_map_writer(tmpdir):
     f.seek(10)
     assert f.read(8) == b'peekaboo'
 
-    f2 = pa.memory_map(path, mode='r+w')
+    f2 = pa.memory_map(path, mode='r+b')
 
     f2.seek(10)
     f2.write(b'booapeak')
@@ -404,3 +404,33 @@ def test_os_file_writer(tmpdir):
 
     with pytest.raises(IOError):
         f2.read(5)
+
+
+def test_native_file_modes(tmpdir):
+    path = os.path.join(str(tmpdir), guid())
+    with open(path, 'wb') as f:
+        f.write(b'foooo')
+
+    with pa.OSFile(path, mode='r') as f:
+        assert f.mode == 'rb'
+
+    with pa.OSFile(path, mode='rb') as f:
+        assert f.mode == 'rb'
+
+    with pa.OSFile(path, mode='w') as f:
+        assert f.mode == 'wb'
+
+    with pa.OSFile(path, mode='wb') as f:
+        assert f.mode == 'wb'
+
+    with open(path, 'wb') as f:
+        f.write(b'foooo')
+
+    with pa.memory_map(path, 'r') as f:
+        assert f.mode == 'rb'
+
+    with pa.memory_map(path, 'r+') as f:
+        assert f.mode == 'rb+'
+
+    with pa.memory_map(path, 'r+b') as f:
+        assert f.mode == 'rb+'

From d46b7ea3e8c59778146d4587ab9b09461cf58634 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 15 Jul 2017 13:35:19 +0200
Subject: [PATCH 0806/1644] ARROW-992: [Python] Try to set a __version__ in
 in-place local builds

This is better than having no version at all

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #841 from wesm/ARROW-992 and squashes the following commits:

40b86fa [Wes McKinney] Use setuptools_scm to determine __version__ in dev builds if possible
---
 python/pyarrow/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 37aec6c8803dc..434722c29baa3 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -22,7 +22,11 @@
     __version__ = get_distribution(__name__).version
 except DistributionNotFound:
    # package is not installed
-   pass
+    try:
+        import setuptools_scm
+        __version__ = setuptools_scm.get_version('../')
+    except (ImportError, LookupError):
+        __version__ = None
 
 
 from pyarrow.lib import cpu_count, set_cpu_count

From 9ff39f3e9d711bebddbe452ef4f4a3b9b48d5396 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 15 Jul 2017 22:09:16 +0200
Subject: [PATCH 0807/1644] ARROW-1216: [Python] Fix creating numpy array from
 arrow buffers on python 2

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #844 from pcmoritz/numpy-buffer and squashes the following commits:

a891971 [Philipp Moritz] test that base object is set correctly
22bb6b5 [Philipp Moritz] fix creating numpy buffer from arrow buffer on python 2
---
 python/pyarrow/io.pxi           | 12 ++++++++++++
 python/pyarrow/tests/test_io.py | 10 ++++++++++
 2 files changed, 22 insertions(+)

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index c15be5e435764..3221185c27bc5 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -528,6 +528,18 @@ cdef class Buffer:
         buffer.strides = self.strides
         buffer.suboffsets = NULL
 
+    def __getsegcount__(self, Py_ssize_t *len_out):
+        if len_out != NULL:
+            len_out[0] = <Py_ssize_t>self.size
+        return 1
+
+    def __getreadbuffer__(self, Py_ssize_t idx, void **p):
+        if idx != 0:
+            raise SystemError("accessing non-existent buffer segment")
+        if p != NULL:
+            p[0] = <void*> self.buffer.get().data()
+        return self.size
+
 
 cdef shared_ptr[PoolBuffer] allocate_buffer(CMemoryPool* pool):
     cdef shared_ptr[PoolBuffer] result
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 6258f6de18191..835f50874f7fd 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -160,6 +160,16 @@ def test_buffer_bytearray():
     assert result == val
 
 
+def test_buffer_numpy():
+    # Make sure creating a numpy array from an arrow buffer works
+    byte_array = bytearray(20)
+    byte_array[0] = 42
+    buf = pa.frombuffer(byte_array)
+    array = np.frombuffer(buf, dtype="uint8")
+    assert array[0] == byte_array[0]
+    assert array.base == buf
+
+
 def test_buffer_memoryview_is_immutable():
     val = b'some data'
 

From 099f61ce5b5007230788a470f53c04315263ff8e Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 15 Jul 2017 22:11:13 +0200
Subject: [PATCH 0808/1644] ARROW-1218: [C++] Fix arrow build if no compression
 library is used

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #846 from pcmoritz/arrow-no-compression-build and squashes the following commits:

75b7ebb [Philipp Moritz] fix arrow build if no compression library is used
---
 cpp/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d84611c1c8cb4..c6098502e3621 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -584,7 +584,9 @@ if (ARROW_WITH_ZSTD)
   SET(ARROW_STATIC_LINK_LIBS zstd_static ${ARROW_STATIC_LINK_LIBS})
 endif()
 
-add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
+if (ARROW_STATIC_LINK_LIBS)
+  add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
+endif()
 
 set(ARROW_MIN_TEST_LIBS
   arrow_static

From bb0a75885f2655ac54be47bd238811b74782532e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 15 Jul 2017 16:51:51 -0400
Subject: [PATCH 0809/1644] ARROW-1214: [Python/C++] Add C++ functionality to
 more easily handle encapsulated IPC messages, Python bindings

This patch does a bunch of things:

* Decouples the RecordBatchStreamReader from the actual message iteration (which is handled by a new `arrow::ipc::MessageReader` interface
* Enables `arrow::ipc::Message` to hold all of the memory for a complete unit of data: metadata plus body
* Renames some IPC methods for better consistency (GetNextRecordBatch -> ReadNextRecordBatch)
* Adds function to serialize a complete encapsulated message to an `arrow::io::OutputStream*
* Add Python bindings for all of the above, introduce `pyarrow.Message`, `pyarrow.MessageReader`. Add `read_message` and `Message.serialize` functions for efficient memory round trips
* Add `pyarrow.read_record_batch` for reading a single record batch given a message and a known schema

Later we will want to add `pyarrow.read_schema`, but it seemed like a bit of work to make it work for dictionaries.

This implements the C++ analogue to ARROW-1047, which was for Java. Not sure why I didn't create a JIRA about this. cc @icexelloss

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #839 from wesm/ARROW-1214 and squashes the following commits:

07f1820a [Wes McKinney] Refactor to introduce MessageReader abstract type, use unique_ptr for messages instead of shared_ptr. First cut at Message, MessageReader Python API. Add read_message, C++/Python machinery for message roundtrips to Buffer, comparison. Add function to read RecordBatch from encapsulated message given schema.
---
 c_glib/arrow-glib/reader.cpp                 |   4 +-
 cpp/src/arrow/buffer.cc                      |   2 -
 cpp/src/arrow/buffer.h                       |  15 +-
 cpp/src/arrow/builder.cc                     |  12 +-
 cpp/src/arrow/ipc/file-to-stream.cc          |   2 +-
 cpp/src/arrow/ipc/ipc-json-test.cc           |   6 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc     |  70 +--
 cpp/src/arrow/ipc/json-integration-test.cc   |   8 +-
 cpp/src/arrow/ipc/json.cc                    |   6 +-
 cpp/src/arrow/ipc/json.h                     |   2 +-
 cpp/src/arrow/ipc/metadata.cc                | 150 ++++--
 cpp/src/arrow/ipc/metadata.h                 | 100 ++--
 cpp/src/arrow/ipc/reader.cc                  | 206 ++++----
 cpp/src/arrow/ipc/reader.h                   |  35 +-
 cpp/src/arrow/ipc/stream-to-file.cc          |   2 +-
 cpp/src/arrow/ipc/writer.cc                  |   4 +-
 cpp/src/arrow/python/builtin_convert.cc      |   5 +-
 python/doc/source/api.rst                    |  12 +-
 python/pyarrow/__init__.py                   |   9 +-
 python/pyarrow/feather.pxi                   | 109 +++++
 python/pyarrow/includes/libarrow.pxd         |  66 +--
 python/pyarrow/io.pxi                        | 353 --------------
 python/pyarrow/ipc.pxi                       | 480 +++++++++++++++++++
 python/pyarrow/ipc.py                        |  13 +-
 python/pyarrow/lib.pyx                       |   8 +-
 python/pyarrow/pandas_compat.py              |   2 +-
 python/pyarrow/table.pxi                     |  16 +-
 python/pyarrow/tests/conftest.py             |   2 +-
 python/pyarrow/tests/test_array.py           |   2 +-
 python/pyarrow/tests/test_convert_builtin.py |   3 +
 python/pyarrow/tests/test_feather.py         |   3 +-
 python/pyarrow/tests/test_ipc.py             |  59 ++-
 python/pyarrow/tests/test_parquet.py         |   7 +-
 python/pyarrow/tests/test_table.py           |   4 +
 python/pyarrow/tests/test_tensor.py          |   1 +
 35 files changed, 1135 insertions(+), 643 deletions(-)
 create mode 100644 python/pyarrow/feather.pxi
 create mode 100644 python/pyarrow/ipc.pxi

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index 3ff6ba1bb5294..523bdeef77db3 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -173,7 +173,7 @@ garrow_record_batch_reader_get_next_record_batch(GArrowRecordBatchReader *reader
 {
   auto arrow_reader = garrow_record_batch_reader_get_raw(reader);
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
-  auto status = arrow_reader->GetNextRecordBatch(&arrow_record_batch);
+  auto status = arrow_reader->ReadNextRecordBatch(&arrow_record_batch);
 
   if (garrow_error_check(error,
                          status,
@@ -410,7 +410,7 @@ garrow_record_batch_file_reader_get_record_batch(GArrowRecordBatchFileReader *re
 {
   auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader);
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
-  auto status = arrow_reader->GetRecordBatch(i, &arrow_record_batch);
+  auto status = arrow_reader->ReadRecordBatch(i, &arrow_record_batch);
 
   if (garrow_error_check(error,
                          status,
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index fb6379894c3b0..a1d119ecdcae5 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -27,8 +27,6 @@
 
 namespace arrow {
 
-Buffer::~Buffer() {}
-
 Status Buffer::Copy(
     int64_t start, int64_t nbytes, MemoryPool* pool, std::shared_ptr<Buffer>* out) const {
   // Sanity checks
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index bfbea776f1a5f..b117b2414d0fd 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -22,6 +22,7 @@
 #include <cstdint>
 #include <cstring>
 #include <memory>
+#include <string>
 
 #include "arrow/status.h"
 #include "arrow/util/macros.h"
@@ -47,7 +48,8 @@ class ARROW_EXPORT Buffer {
  public:
   Buffer(const uint8_t* data, int64_t size)
       : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
-  virtual ~Buffer();
+
+  virtual ~Buffer() = default;
 
   /// An offset into data that is owned by another buffer, but we want to be
   /// able to retain a valid pointer to it even after other shared_ptr's to the
@@ -97,6 +99,17 @@ class ARROW_EXPORT Buffer {
   DISALLOW_COPY_AND_ASSIGN(Buffer);
 };
 
+/// \brief Create Buffer referencing std::string memory
+///
+/// Warning: string instance must stay alive
+///
+/// \param str std::string instance
+/// \return std::shared_ptr<Buffer>
+static inline std::shared_ptr<Buffer> GetBufferFromString(const std::string& str) {
+  return std::make_shared<Buffer>(
+      reinterpret_cast<const uint8_t*>(str.c_str()), static_cast<int64_t>(str.size()));
+}
+
 /// Construct a view on passed buffer at the indicated offset and length. This
 /// function cannot fail and does not error checking (except in debug builds)
 static inline std::shared_ptr<Buffer> SliceBuffer(
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 155d81ae7faf6..e466838c37539 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -342,8 +342,8 @@ Status AdaptiveIntBuilder::Append(
         sizeof(int64_t) * length);
   } else {
 #ifdef _MSC_VER
-#  pragma warning(push)
-#  pragma warning(disable:4996)
+#pragma warning(push)
+#pragma warning(disable : 4996)
 #endif
     // int_size_ may have changed, so we need to recheck
     switch (int_size_) {
@@ -366,7 +366,7 @@ Status AdaptiveIntBuilder::Append(
         DCHECK(false);
     }
 #ifdef _MSC_VER
-#  pragma warning(pop)
+#pragma warning(pop)
 #endif
   }
 
@@ -497,8 +497,8 @@ Status AdaptiveUIntBuilder::Append(
         sizeof(uint64_t) * length);
   } else {
 #ifdef _MSC_VER
-#  pragma warning(push)
-#  pragma warning(disable:4996)
+#pragma warning(push)
+#pragma warning(disable : 4996)
 #endif
     // int_size_ may have changed, so we need to recheck
     switch (int_size_) {
@@ -521,7 +521,7 @@ Status AdaptiveUIntBuilder::Append(
         DCHECK(false);
     }
 #ifdef _MSC_VER
-#  pragma warning(pop)
+#pragma warning(pop)
 #endif
   }
 
diff --git a/cpp/src/arrow/ipc/file-to-stream.cc b/cpp/src/arrow/ipc/file-to-stream.cc
index 39c720cb35aa5..a1feedc212618 100644
--- a/cpp/src/arrow/ipc/file-to-stream.cc
+++ b/cpp/src/arrow/ipc/file-to-stream.cc
@@ -39,7 +39,7 @@ Status ConvertToStream(const char* path) {
   RETURN_NOT_OK(RecordBatchStreamWriter::Open(&sink, reader->schema(), &writer));
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> chunk;
-    RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
+    RETURN_NOT_OK(reader->ReadRecordBatch(i, &chunk));
     RETURN_NOT_OK(writer->WriteRecordBatch(*chunk));
   }
   return writer->Close();
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 92971468964c3..318e3184ba78f 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -276,7 +276,7 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
 
   for (int i = 0; i < nbatches; ++i) {
     std::shared_ptr<RecordBatch> batch;
-    ASSERT_OK(reader->GetRecordBatch(i, &batch));
+    ASSERT_OK(reader->ReadRecordBatch(i, &batch));
     ASSERT_TRUE(batch->Equals(*batches[i]));
   }
 }
@@ -344,7 +344,7 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
   ASSERT_EQ(1, reader->num_record_batches());
 
   std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK(reader->GetRecordBatch(0, &batch));
+  ASSERT_OK(reader->ReadRecordBatch(0, &batch));
 
   std::vector<bool> foo_valid = {true, false, true, true, true};
   std::vector<int32_t> foo_values = {1, 2, 3, 4, 5};
@@ -388,7 +388,7 @@ void CheckRoundtrip(const RecordBatch& batch) {
   ASSERT_OK(JsonReader::Open(buffer, &reader));
 
   std::shared_ptr<RecordBatch> result_batch;
-  ASSERT_OK(reader->GetRecordBatch(0, &result_batch));
+  ASSERT_OK(reader->ReadRecordBatch(0, &result_batch));
 
   CompareBatch(batch, *result_batch);
 }
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index c71d0467b9736..42f14b0dd1507 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -51,8 +51,8 @@ class TestSchemaMetadata : public ::testing::Test {
     std::shared_ptr<Buffer> buffer;
     ASSERT_OK(WriteSchemaMessage(schema, memo, &buffer));
 
-    std::shared_ptr<Message> message;
-    ASSERT_OK(Message::Open(buffer, 0, &message));
+    std::unique_ptr<Message> message;
+    ASSERT_OK(Message::Open(buffer, nullptr, &message));
 
     ASSERT_EQ(Message::SCHEMA, message->type());
 
@@ -65,6 +65,32 @@ class TestSchemaMetadata : public ::testing::Test {
   }
 };
 
+TEST(TestMessage, Equals) {
+  std::string metadata = "foo";
+  std::string body = "bar";
+
+  auto b1 = GetBufferFromString(metadata);
+  auto b2 = GetBufferFromString(metadata);
+  auto b3 = GetBufferFromString(body);
+  auto b4 = GetBufferFromString(body);
+
+  Message msg1(b1, b3);
+  Message msg2(b2, b4);
+  Message msg3(b1, nullptr);
+  Message msg4(b2, nullptr);
+
+  ASSERT_TRUE(msg1.Equals(msg2));
+  ASSERT_TRUE(msg3.Equals(msg4));
+
+  ASSERT_FALSE(msg1.Equals(msg3));
+  ASSERT_FALSE(msg3.Equals(msg1));
+
+  // same metadata as msg1, different body
+  Message msg5(b2, b1);
+  ASSERT_FALSE(msg1.Equals(msg5));
+  ASSERT_FALSE(msg5.Equals(msg1));
+}
+
 const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
 
 TEST_F(TestSchemaMetadata, PrimitiveFields) {
@@ -123,16 +149,12 @@ class IpcTestFixture : public io::MemoryMapFixture {
     RETURN_NOT_OK(WriteRecordBatch(
         batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
 
-    std::shared_ptr<Message> message;
+    std::unique_ptr<Message> message;
     RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
 
-    // The buffer offsets start at 0, so we must construct a
-    // RandomAccessFile according to that frame of reference
-    std::shared_ptr<Buffer> buffer_payload;
-    RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
-    io::BufferReader buffer_reader(buffer_payload);
-
-    return ReadRecordBatch(*message, batch.schema(), &buffer_reader, batch_result);
+    io::BufferReader buffer_reader(message->body());
+    return ReadRecordBatch(
+        *message->metadata(), batch.schema(), &buffer_reader, batch_result);
   }
 
   Status DoLargeRoundTrip(
@@ -151,7 +173,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
     std::shared_ptr<RecordBatchFileReader> file_reader;
     RETURN_NOT_OK(RecordBatchFileReader::Open(mmap_, offset, &file_reader));
 
-    return file_reader->GetRecordBatch(0, result);
+    return file_reader->ReadRecordBatch(0, result);
   }
 
   void CheckReadResult(const RecordBatch& result, const RecordBatch& expected) {
@@ -225,7 +247,7 @@ TEST_F(TestIpcRoundTrip, MetadataVersion) {
   ASSERT_OK(WriteRecordBatch(
       *batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
 
-  std::shared_ptr<Message> message;
+  std::unique_ptr<Message> message;
   ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
 
   ASSERT_EQ(MetadataVersion::V3, message->metadata_version());
@@ -434,16 +456,13 @@ TEST_F(RecursionLimits, ReadLimit) {
   ASSERT_OK(WriteToMmap(
       recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
 
-  std::shared_ptr<Message> message;
+  std::unique_ptr<Message> message;
   ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
 
-  std::shared_ptr<Buffer> payload;
-  ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
-
-  io::BufferReader reader(payload);
+  io::BufferReader reader(message->body());
 
   std::shared_ptr<RecordBatch> result;
-  ASSERT_RAISES(Invalid, ReadRecordBatch(*message, schema, &reader, &result));
+  ASSERT_RAISES(Invalid, ReadRecordBatch(*message->metadata(), schema, &reader, &result));
 }
 
 TEST_F(RecursionLimits, StressLimit) {
@@ -455,16 +474,13 @@ TEST_F(RecursionLimits, StressLimit) {
     ASSERT_OK(WriteToMmap(
         recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
 
-    std::shared_ptr<Message> message;
+    std::unique_ptr<Message> message;
     ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
 
-    std::shared_ptr<Buffer> payload;
-    ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
-
-    io::BufferReader reader(payload);
-
+    io::BufferReader reader(message->body());
     std::shared_ptr<RecordBatch> result;
-    ASSERT_OK(ReadRecordBatch(*message, schema, recursion_depth + 1, &reader, &result));
+    ASSERT_OK(ReadRecordBatch(
+        *message->metadata(), schema, recursion_depth + 1, &reader, &result));
     *it_works = result->Equals(*batch);
   };
 
@@ -511,7 +527,7 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
     EXPECT_EQ(num_batches, reader->num_record_batches());
     for (int i = 0; i < num_batches; ++i) {
       std::shared_ptr<RecordBatch> chunk;
-      RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
+      RETURN_NOT_OK(reader->ReadRecordBatch(i, &chunk));
       out_batches->emplace_back(chunk);
     }
 
@@ -571,7 +587,7 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
 
     std::shared_ptr<RecordBatch> chunk;
     while (true) {
-      RETURN_NOT_OK(reader->GetNextRecordBatch(&chunk));
+      RETURN_NOT_OK(reader->ReadNextRecordBatch(&chunk));
       if (chunk == nullptr) { break; }
       out_batches->emplace_back(chunk);
     }
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 424755a94068b..18f5dfaf57098 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -82,7 +82,7 @@ static Status ConvertJsonToArrow(
 
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
-    RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
+    RETURN_NOT_OK(reader->ReadRecordBatch(i, &batch));
     RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
   }
   return writer->Close();
@@ -109,7 +109,7 @@ static Status ConvertArrowToJson(
 
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
-    RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
+    RETURN_NOT_OK(reader->ReadRecordBatch(i, &batch));
     RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
   }
 
@@ -168,8 +168,8 @@ static Status ValidateArrowVsJson(
   std::shared_ptr<RecordBatch> arrow_batch;
   std::shared_ptr<RecordBatch> json_batch;
   for (int i = 0; i < json_nbatches; ++i) {
-    RETURN_NOT_OK(json_reader->GetRecordBatch(i, &json_batch));
-    RETURN_NOT_OK(arrow_reader->GetRecordBatch(i, &arrow_batch));
+    RETURN_NOT_OK(json_reader->ReadRecordBatch(i, &json_batch));
+    RETURN_NOT_OK(arrow_reader->ReadRecordBatch(i, &arrow_batch));
 
     if (!json_batch->ApproxEquals(*arrow_batch)) {
       std::stringstream ss;
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index f8c0b62ca9a86..36e343e5fb5bc 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -115,7 +115,7 @@ class JsonReader::JsonReaderImpl {
     return Status::OK();
   }
 
-  Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const {
+  Status ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const {
     DCHECK_GE(i, 0) << "i out of bounds";
     DCHECK_LT(i, static_cast<int>(record_batches_->GetArray().Size()))
         << "i out of bounds";
@@ -164,8 +164,8 @@ int JsonReader::num_record_batches() const {
   return impl_->num_record_batches();
 }
 
-Status JsonReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const {
-  return impl_->GetRecordBatch(i, batch);
+Status JsonReader::ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const {
+  return impl_->ReadRecordBatch(i, batch);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
index ad94defdeabf3..2ba27c7f2c37d 100644
--- a/cpp/src/arrow/ipc/json.h
+++ b/cpp/src/arrow/ipc/json.h
@@ -72,7 +72,7 @@ class ARROW_EXPORT JsonReader {
   int num_record_batches() const;
 
   // Read a record batch from the file
-  Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const;
+  Status ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const;
 
  private:
   JsonReader(MemoryPool* pool, const std::shared_ptr<Buffer>& data);
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 54f0547b5991b..5b2ca3b9c7dc6 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/ipc/metadata.h"
 
+#include <algorithm>
 #include <cstdint>
 #include <memory>
 #include <sstream>
@@ -834,11 +835,12 @@ Status DictionaryMemo::AddDictionary(
 
 class Message::MessageImpl {
  public:
-  explicit MessageImpl(const std::shared_ptr<Buffer>& buffer, int64_t offset)
-      : buffer_(buffer), offset_(offset), message_(nullptr) {}
+  explicit MessageImpl(
+      const std::shared_ptr<Buffer>& metadata, const std::shared_ptr<Buffer>& body)
+      : metadata_(metadata), message_(nullptr), body_(body) {}
 
   Status Open() {
-    message_ = flatbuf::GetMessage(buffer_->data() + offset_);
+    message_ = flatbuf::GetMessage(metadata_->data());
 
     // Check that the metadata version is supported
     if (message_->version() < kMinMetadataVersion) {
@@ -872,7 +874,7 @@ class Message::MessageImpl {
         // Arrow 0.2
         return MetadataVersion::V2;
       case flatbuf::MetadataVersion_V3:
-        // Arrow 0.3
+        // Arrow >= 0.3
         return MetadataVersion::V3;
       // Add cases as other versions become available
       default:
@@ -882,28 +884,38 @@ class Message::MessageImpl {
 
   const void* header() const { return message_->header(); }
 
-  int64_t body_length() const { return message_->bodyLength(); }
+  std::shared_ptr<Buffer> body() const { return body_; }
 
- private:
-  // Retain reference to memory
-  std::shared_ptr<Buffer> buffer_;
-  int64_t offset_;
+  std::shared_ptr<Buffer> metadata() const { return metadata_; }
 
+ private:
+  // The Flatbuffer metadata
+  std::shared_ptr<Buffer> metadata_;
   const flatbuf::Message* message_;
+
+  // The message body, if any
+  std::shared_ptr<Buffer> body_;
 };
 
-Message::Message(const std::shared_ptr<Buffer>& buffer, int64_t offset) {
-  impl_.reset(new MessageImpl(buffer, offset));
+Message::Message(
+    const std::shared_ptr<Buffer>& metadata, const std::shared_ptr<Buffer>& body) {
+  impl_.reset(new MessageImpl(metadata, body));
+}
+
+Status Message::Open(const std::shared_ptr<Buffer>& metadata,
+    const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out) {
+  out->reset(new Message(metadata, body));
+  return (*out)->impl_->Open();
 }
 
 Message::~Message() {}
 
-Status Message::Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
-    std::shared_ptr<Message>* out) {
-  // ctor is private
+std::shared_ptr<Buffer> Message::body() const {
+  return impl_->body();
+}
 
-  *out = std::shared_ptr<Message>(new Message(buffer, offset));
-  return (*out)->impl_->Open();
+std::shared_ptr<Buffer> Message::metadata() const {
+  return impl_->metadata();
 }
 
 Message::Type Message::type() const {
@@ -914,14 +926,64 @@ MetadataVersion Message::metadata_version() const {
   return impl_->version();
 }
 
-int64_t Message::body_length() const {
-  return impl_->body_length();
-}
-
 const void* Message::header() const {
   return impl_->header();
 }
 
+bool Message::Equals(const Message& other) const {
+  int64_t metadata_bytes = std::min(metadata()->size(), other.metadata()->size());
+
+  if (!metadata()->Equals(*other.metadata(), metadata_bytes)) {
+    return false;
+  }
+
+  // Compare bodies, if they have them
+  auto this_body = body();
+  auto other_body = other.body();
+
+  const bool this_has_body = (this_body != nullptr) && (this_body->size() > 0);
+  const bool other_has_body = (other_body != nullptr) && (other_body->size() > 0);
+
+  if (this_has_body && other_has_body) {
+    return this_body->Equals(*other_body);
+  } else if (this_has_body ^ other_has_body) {
+    // One has a body but not the other
+    return false;
+  } else {
+    // Neither has a body
+    return true;
+  }
+}
+
+Status Message::SerializeTo(io::OutputStream* file, int64_t* output_length) const {
+  int32_t metadata_length = 0;
+  RETURN_NOT_OK(WriteMessage(*metadata(), file, &metadata_length));
+
+  *output_length = metadata_length;
+
+  auto body_buffer = body();
+  if (body_buffer) {
+    RETURN_NOT_OK(file->Write(body_buffer->data(), body_buffer->size()));
+    *output_length += body_buffer->size();
+  }
+
+  return Status::OK();
+}
+
+std::string FormatMessageType(Message::Type type) {
+  switch (type) {
+    case Message::SCHEMA:
+      return "schema";
+    case Message::RECORD_BATCH:
+      return "record batch";
+    case Message::DICTIONARY_BATCH:
+      return "dictionary";
+    default:
+      break;
+  }
+  return "unknown";
+}
+
 // ----------------------------------------------------------------------
 
 static Status VisitField(const flatbuf::Field* field, DictionaryTypeMap* id_to_field) {
@@ -975,10 +1037,11 @@ Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_mem
   return Status::OK();
 }
 
-Status GetTensorMetadata(const void* opaque_tensor, std::shared_ptr<DataType>* type,
+Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>* type,
     std::vector<int64_t>* shape, std::vector<int64_t>* strides,
     std::vector<std::string>* dim_names) {
-  auto tensor = static_cast<const flatbuf::Tensor*>(opaque_tensor);
+  auto message = flatbuf::GetMessage(metadata.data());
+  auto tensor = reinterpret_cast<const flatbuf::Tensor*>(message->header());
 
   int ndim = static_cast<int>(tensor->shape()->size());
 
@@ -1006,8 +1069,27 @@ Status GetTensorMetadata(const void* opaque_tensor, std::shared_ptr<DataType>* t
 // ----------------------------------------------------------------------
 // Read and write messages
 
+static Status ReadFullMessage(const std::shared_ptr<Buffer>& metadata,
+    io::InputStream* stream, std::unique_ptr<Message>* message) {
+  auto fb_message = flatbuf::GetMessage(metadata->data());
+
+  int64_t body_length = fb_message->bodyLength();
+
+  std::shared_ptr<Buffer> body;
+  RETURN_NOT_OK(stream->Read(body_length, &body));
+
+  if (body->size() < body_length) {
+    std::stringstream ss;
+    ss << "Expected to be able to read " << body_length << " bytes for message body, got "
+       << body->size();
+    return Status::IOError(ss.str());
+  }
+
+  return Message::Open(metadata, body, message);
+}
+
 Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
-    std::shared_ptr<Message>* message) {
+    std::unique_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
 
@@ -1019,13 +1101,15 @@ Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile
        << ", metadata length: " << metadata_length;
     return Status::Invalid(ss.str());
   }
-  return Message::Open(buffer, 4, message);
+
+  auto metadata = SliceBuffer(buffer, 4, buffer->size() - 4);
+  return ReadFullMessage(metadata, file, message);
 }
 
-Status ReadMessage(io::InputStream* file, std::shared_ptr<Message>* message) {
+Status ReadMessage(io::InputStream* file, std::unique_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
 
+  RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
   if (buffer->size() != sizeof(int32_t)) {
     *message = nullptr;
     return Status::OK();
@@ -1044,9 +1128,21 @@ Status ReadMessage(io::InputStream* file, std::shared_ptr<Message>* message) {
     return Status::IOError("Unexpected end of stream trying to read message");
   }
 
-  return Message::Open(buffer, 0, message);
+  return ReadFullMessage(buffer, file, message);
+}
+
+// ----------------------------------------------------------------------
+// Implement InputStream message reader
+
+Status InputStreamMessageReader::ReadNextMessage(std::unique_ptr<Message>* message) {
+  return ReadMessage(stream_.get(), message);
 }
 
+InputStreamMessageReader::~InputStreamMessageReader() {}
+
+// ----------------------------------------------------------------------
+// Implement message writing
+
 Status WriteMessage(
     const Buffer& message, io::OutputStream* file, int32_t* message_length) {
   // Need to write 4 bytes (message size), the message, plus padding to
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 257bbd8673a8f..64b2571ed464d 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -59,26 +59,12 @@ static constexpr const char* kArrowMagicBytes = "ARROW1";
 constexpr int kMaxNestingDepth = 64;
 
 struct ARROW_EXPORT FieldMetadata {
-  FieldMetadata() {}
-  FieldMetadata(int64_t length, int64_t null_count, int64_t offset)
-      : length(length), null_count(null_count), offset(offset) {}
-
-  FieldMetadata(const FieldMetadata& other) {
-    this->length = other.length;
-    this->null_count = other.null_count;
-    this->offset = other.offset;
-  }
-
   int64_t length;
   int64_t null_count;
   int64_t offset;
 };
 
 struct ARROW_EXPORT BufferMetadata {
-  BufferMetadata() {}
-  BufferMetadata(int32_t page, int64_t offset, int64_t length)
-      : page(page), offset(offset), length(length) {}
-
   /// The shared memory page id where to find this. Set to -1 if unused
   int32_t page;
 
@@ -90,10 +76,6 @@ struct ARROW_EXPORT BufferMetadata {
 };
 
 struct FileBlock {
-  FileBlock() {}
-  FileBlock(int64_t offset, int32_t metadata_length, int64_t body_length)
-      : offset(offset), metadata_length(metadata_length), body_length(body_length) {}
-
   int64_t offset;
   int32_t metadata_length;
   int64_t body_length;
@@ -153,20 +135,46 @@ Status GetDictionaryTypes(const void* opaque_schema, DictionaryTypeMap* id_to_fi
 Status ARROW_EXPORT GetSchema(const void* opaque_schema,
     const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out);
 
-Status ARROW_EXPORT GetTensorMetadata(const void* opaque_tensor,
+Status ARROW_EXPORT GetTensorMetadata(const Buffer& metadata,
     std::shared_ptr<DataType>* type, std::vector<int64_t>* shape,
     std::vector<int64_t>* strides, std::vector<std::string>* dim_names);
 
+/// \brief An IPC message including metadata and body
 class ARROW_EXPORT Message {
  public:
   enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH, TENSOR };
 
-  ~Message();
+  /// \brief Construct message, but do not validate
+  ///
+  /// Use at your own risk; Message::Open has more metadata validation
+  Message(const std::shared_ptr<Buffer>& metadata, const std::shared_ptr<Buffer>& body);
 
-  static Status Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
-      std::shared_ptr<Message>* out);
+  ~Message();
 
-  int64_t body_length() const;
+  /// \brief Create and validate a Message instance from two buffers
+  ///
+  /// \param[in] metadata a buffer containing the Flatbuffer metadata
+  /// \param[in] body a buffer containing the message body, which may be nullptr
+  /// \param[out] out the created message
+  static Status Open(const std::shared_ptr<Buffer>& metadata,
+      const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out);
+
+  /// \brief Write length-prefixed metadata and body to output stream
+  ///
+  /// \param[in] file output stream to write to
+  /// \param[out] output_length the number of bytes written
+  /// \return Status
+  bool Equals(const Message& other) const;
+
+  /// \brief the Message metadata
+  ///
+  /// \return buffer
+  std::shared_ptr<Buffer> metadata() const;
+
+  /// \brief the Message body, if any
+  ///
+  /// \return buffer is nullptr if no body
+  std::shared_ptr<Buffer> body() const;
 
   Type type() const;
 
@@ -174,9 +182,14 @@ class ARROW_EXPORT Message {
 
   const void* header() const;
 
- private:
-  Message(const std::shared_ptr<Buffer>& buffer, int64_t offset);
+  /// \brief Write length-prefixed metadata and body to output stream
+  ///
+  /// \param[in] file output stream to write to
+  /// \param[out] output_length the number of bytes written
+  /// \return Status
+  Status SerializeTo(io::OutputStream* file, int64_t* output_length) const;
 
+ private:
   // Hide serialization details from user API
   class MessageImpl;
   std::unique_ptr<MessageImpl> impl_;
@@ -184,8 +197,34 @@ class ARROW_EXPORT Message {
   DISALLOW_COPY_AND_ASSIGN(Message);
 };
 
+ARROW_EXPORT std::string FormatMessageType(Message::Type type);
+
+/// \brief Abstract interface for a sequence of messages
+class ARROW_EXPORT MessageReader {
+ public:
+  virtual ~MessageReader() = default;
+
+  virtual Status ReadNextMessage(std::unique_ptr<Message>* message) = 0;
+};
+
+class ARROW_EXPORT InputStreamMessageReader : public MessageReader {
+ public:
+  explicit InputStreamMessageReader(const std::shared_ptr<io::InputStream>& stream)
+      : stream_(stream) {}
+
+  ~InputStreamMessageReader();
+
+  Status ReadNextMessage(std::unique_ptr<Message>* message) override;
+
+ private:
+  std::shared_ptr<io::InputStream> stream_;
+};
+
+/// \brief Read encapulated RPC message from position in file
+///
 /// Read a length-prefixed message flatbuffer starting at the indicated file
-/// offset
+/// offset. If the message has a body with non-zero length, it will also be
+/// read
 ///
 /// The metadata_length includes at least the length prefix and the flatbuffer
 ///
@@ -196,15 +235,18 @@ class ARROW_EXPORT Message {
 /// \param[out] message the message read
 /// \return Status success or failure
 Status ARROW_EXPORT ReadMessage(int64_t offset, int32_t metadata_length,
-    io::RandomAccessFile* file, std::shared_ptr<Message>* message);
+    io::RandomAccessFile* file, std::unique_ptr<Message>* message);
 
+/// \brief Read encapulated RPC message (metadata and body) from InputStream
+///
 /// Read length-prefixed message with as-yet unknown length. Returns nullptr if
 /// there are not enough bytes available or the message length is 0 (e.g. EOS
 /// in a stream)
 Status ARROW_EXPORT ReadMessage(
-    io::InputStream* stream, std::shared_ptr<Message>* message);
+    io::InputStream* stream, std::unique_ptr<Message>* message);
 
-/// Write a serialized message with a length-prefix and padding to an 8-byte offset
+/// Write a serialized message metadata with a length-prefix and padding to an
+/// 8-byte offset
 ///
 /// <message_size: int32><message: const void*><padding>
 Status ARROW_EXPORT WriteMessage(
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 8ca4d82714a75..88ab33087b637 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -257,11 +257,18 @@ static Status LoadArray(const std::shared_ptr<DataType>& type,
   return loader.Load();
 }
 
-Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
+Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
   return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
 }
 
+Status ReadRecordBatch(const Message& message, const std::shared_ptr<Schema>& schema,
+    std::shared_ptr<RecordBatch>* out) {
+  io::BufferReader reader(message.body());
+  DCHECK_EQ(message.type(), Message::RECORD_BATCH);
+  return ReadRecordBatch(*message.metadata(), schema, kMaxNestingDepth, &reader, out);
+}
+
 // ----------------------------------------------------------------------
 // Array loading
 
@@ -294,18 +301,22 @@ static inline Status ReadRecordBatch(const flatbuf::RecordBatch* metadata,
       schema, metadata->length(), max_recursion_depth, &source, out);
 }
 
-Status ReadRecordBatch(const Message& metadata, const std::shared_ptr<Schema>& schema,
+Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
     int max_recursion_depth, io::RandomAccessFile* file,
     std::shared_ptr<RecordBatch>* out) {
-  DCHECK_EQ(metadata.type(), Message::RECORD_BATCH);
-  auto batch = reinterpret_cast<const flatbuf::RecordBatch*>(metadata.header());
+  auto message = flatbuf::GetMessage(metadata.data());
+  if (message->header_type() != flatbuf::MessageHeader_RecordBatch) {
+    DCHECK_EQ(message->header_type(), flatbuf::MessageHeader_RecordBatch);
+  }
+  auto batch = reinterpret_cast<const flatbuf::RecordBatch*>(message->header());
   return ReadRecordBatch(batch, schema, max_recursion_depth, file, out);
 }
 
-Status ReadDictionary(const Message& metadata, const DictionaryTypeMap& dictionary_types,
+Status ReadDictionary(const Buffer& metadata, const DictionaryTypeMap& dictionary_types,
     io::RandomAccessFile* file, int64_t* dictionary_id, std::shared_ptr<Array>* out) {
+  auto message = flatbuf::GetMessage(metadata.data());
   auto dictionary_batch =
-      reinterpret_cast<const flatbuf::DictionaryBatch*>(metadata.header());
+      reinterpret_cast<const flatbuf::DictionaryBatch*>(message->header());
 
   int64_t id = *dictionary_id = dictionary_batch->id();
   auto it = dictionary_types.find(id);
@@ -335,25 +346,33 @@ Status ReadDictionary(const Message& metadata, const DictionaryTypeMap& dictiona
   return Status::OK();
 }
 
+static Status ReadMessageAndValidate(MessageReader* reader, Message::Type expected_type,
+    bool allow_null, std::unique_ptr<Message>* message) {
+  RETURN_NOT_OK(reader->ReadNextMessage(message));
+
+  if (!(*message) && !allow_null) {
+    std::stringstream ss;
+    ss << "Expected " << FormatMessageType(expected_type)
+       << " message in stream, was null or length 0";
+    return Status::Invalid(ss.str());
+  }
+
+  if ((*message) == nullptr) { return Status::OK(); }
+
+  if ((*message)->type() != expected_type) {
+    std::stringstream ss;
+    ss << "Message not expected type: " << FormatMessageType(expected_type)
+       << ", was: " << (*message)->type();
+    return Status::IOError(ss.str());
+  }
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // RecordBatchStreamReader implementation
 
 static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
-  return FileBlock(block->offset(), block->metaDataLength(), block->bodyLength());
-}
-
-static inline std::string FormatMessageType(Message::Type type) {
-  switch (type) {
-    case Message::SCHEMA:
-      return "schema";
-    case Message::RECORD_BATCH:
-      return "record batch";
-    case Message::DICTIONARY_BATCH:
-      return "dictionary";
-    default:
-      break;
-  }
-  return "unknown";
+  return FileBlock{block->offset(), block->metaDataLength(), block->bodyLength()};
 }
 
 RecordBatchReader::~RecordBatchReader() {}
@@ -363,59 +382,29 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
   RecordBatchStreamReaderImpl() {}
   ~RecordBatchStreamReaderImpl() {}
 
-  Status Open(const std::shared_ptr<io::InputStream>& stream) {
-    stream_ = stream;
+  Status Open(std::unique_ptr<MessageReader> message_reader) {
+    message_reader_ = std::move(message_reader);
     return ReadSchema();
   }
 
-  Status ReadNextMessage(
-      Message::Type expected_type, bool allow_null, std::shared_ptr<Message>* message) {
-    RETURN_NOT_OK(ReadMessage(stream_.get(), message));
-
-    if (!(*message) && !allow_null) {
-      std::stringstream ss;
-      ss << "Expected " << FormatMessageType(expected_type)
-         << " message in stream, was null or length 0";
-      return Status::Invalid(ss.str());
-    }
-
-    if ((*message) == nullptr) { return Status::OK(); }
-
-    if ((*message)->type() != expected_type) {
-      std::stringstream ss;
-      ss << "Message not expected type: " << FormatMessageType(expected_type)
-         << ", was: " << (*message)->type();
-      return Status::IOError(ss.str());
-    }
-    return Status::OK();
-  }
-
-  Status ReadExact(int64_t size, std::shared_ptr<Buffer>* buffer) {
-    RETURN_NOT_OK(stream_->Read(size, buffer));
-
-    if ((*buffer)->size() < size) {
-      return Status::IOError("Unexpected EOS when reading buffer");
-    }
-    return Status::OK();
-  }
-
   Status ReadNextDictionary() {
-    std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::DICTIONARY_BATCH, false, &message));
+    std::unique_ptr<Message> message;
+    RETURN_NOT_OK(ReadMessageAndValidate(
+        message_reader_.get(), Message::DICTIONARY_BATCH, false, &message));
 
-    std::shared_ptr<Buffer> batch_body;
-    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body));
-    io::BufferReader reader(batch_body);
+    io::BufferReader reader(message->body());
 
     std::shared_ptr<Array> dictionary;
     int64_t id;
-    RETURN_NOT_OK(ReadDictionary(*message, dictionary_types_, &reader, &id, &dictionary));
+    RETURN_NOT_OK(ReadDictionary(
+        *message->metadata(), dictionary_types_, &reader, &id, &dictionary));
     return dictionary_memo_.AddDictionary(id, dictionary);
   }
 
   Status ReadSchema() {
-    std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::SCHEMA, false, &message));
+    std::unique_ptr<Message> message;
+    RETURN_NOT_OK(
+        ReadMessageAndValidate(message_reader_.get(), Message::SCHEMA, false, &message));
 
     RETURN_NOT_OK(GetDictionaryTypes(message->header(), &dictionary_types_));
 
@@ -429,9 +418,10 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
     return GetSchema(message->header(), dictionary_memo_, &schema_);
   }
 
-  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
-    std::shared_ptr<Message> message;
-    RETURN_NOT_OK(ReadNextMessage(Message::RECORD_BATCH, true, &message));
+  Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+    std::unique_ptr<Message> message;
+    RETURN_NOT_OK(ReadMessageAndValidate(
+        message_reader_.get(), Message::RECORD_BATCH, true, &message));
 
     if (message == nullptr) {
       // End of stream
@@ -439,21 +429,18 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
       return Status::OK();
     }
 
-    std::shared_ptr<Buffer> batch_body;
-    RETURN_NOT_OK(ReadExact(message->body_length(), &batch_body));
-    io::BufferReader reader(batch_body);
-    return ReadRecordBatch(*message, schema_, &reader, batch);
+    io::BufferReader reader(message->body());
+    return ReadRecordBatch(*message->metadata(), schema_, &reader, batch);
   }
 
   std::shared_ptr<Schema> schema() const { return schema_; }
 
  private:
+  std::unique_ptr<MessageReader> message_reader_;
+
   // dictionary_id -> type
   DictionaryTypeMap dictionary_types_;
-
   DictionaryMemo dictionary_memo_;
-
-  std::shared_ptr<io::InputStream> stream_;
   std::shared_ptr<Schema> schema_;
 };
 
@@ -463,19 +450,25 @@ RecordBatchStreamReader::RecordBatchStreamReader() {
 
 RecordBatchStreamReader::~RecordBatchStreamReader() {}
 
-Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
+Status RecordBatchStreamReader::Open(std::unique_ptr<MessageReader> message_reader,
     std::shared_ptr<RecordBatchStreamReader>* reader) {
   // Private ctor
   *reader = std::shared_ptr<RecordBatchStreamReader>(new RecordBatchStreamReader());
-  return (*reader)->impl_->Open(stream);
+  return (*reader)->impl_->Open(std::move(message_reader));
+}
+
+Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
+    std::shared_ptr<RecordBatchStreamReader>* out) {
+  std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
+  return Open(std::move(message_reader), out);
 }
 
 std::shared_ptr<Schema> RecordBatchStreamReader::schema() const {
   return impl_->schema();
 }
 
-Status RecordBatchStreamReader::GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
-  return impl_->GetNextRecordBatch(batch);
+Status RecordBatchStreamReader::ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+  return impl_->ReadNextRecordBatch(batch);
 }
 
 // ----------------------------------------------------------------------
@@ -547,22 +540,17 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
     return FileBlockFromFlatbuffer(footer_->dictionaries()->Get(i));
   }
 
-  Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
+  Status ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
     DCHECK_GE(i, 0);
     DCHECK_LT(i, num_record_batches());
     FileBlock block = record_batch(i);
 
-    std::shared_ptr<Message> message;
+    std::unique_ptr<Message> message;
     RETURN_NOT_OK(
         ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
 
-    // TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
-    // ARROW-384).
-    std::shared_ptr<Buffer> buffer_block;
-    RETURN_NOT_OK(file_->Read(block.body_length, &buffer_block));
-    io::BufferReader reader(buffer_block);
-
-    return ReadRecordBatch(*message, schema_, &reader, batch);
+    io::BufferReader reader(message->body());
+    return ::arrow::ipc::ReadRecordBatch(*message->metadata(), schema_, &reader, batch);
   }
 
   Status ReadSchema() {
@@ -571,23 +559,16 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
     // Read all the dictionaries
     for (int i = 0; i < num_dictionaries(); ++i) {
       FileBlock block = dictionary(i);
-      std::shared_ptr<Message> message;
+      std::unique_ptr<Message> message;
       RETURN_NOT_OK(
           ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
 
-      // TODO(wesm): ARROW-577: This code is a bit duplicated, can be fixed
-      // with a more invasive refactor
-
-      // TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
-      // ARROW-384).
-      std::shared_ptr<Buffer> buffer_block;
-      RETURN_NOT_OK(file_->Read(block.body_length, &buffer_block));
-      io::BufferReader reader(buffer_block);
+      io::BufferReader reader(message->body());
 
       std::shared_ptr<Array> dictionary;
       int64_t dictionary_id;
-      RETURN_NOT_OK(ReadDictionary(
-          *message, dictionary_fields_, &reader, &dictionary_id, &dictionary));
+      RETURN_NOT_OK(ReadDictionary(*message->metadata(), dictionary_fields_, &reader,
+          &dictionary_id, &dictionary));
       RETURN_NOT_OK(dictionary_memo_->AddDictionary(dictionary_id, dictionary));
     }
 
@@ -653,12 +634,13 @@ MetadataVersion RecordBatchFileReader::version() const {
   return impl_->version();
 }
 
-Status RecordBatchFileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
-  return impl_->GetRecordBatch(i, batch);
+Status RecordBatchFileReader::ReadRecordBatch(
+    int i, std::shared_ptr<RecordBatch>* batch) {
+  return impl_->ReadRecordBatch(i, batch);
 }
 
-static Status ReadContiguousPayload(int64_t offset, io::RandomAccessFile* file,
-    std::shared_ptr<Message>* message, std::shared_ptr<Buffer>* payload) {
+static Status ReadContiguousPayload(
+    int64_t offset, io::RandomAccessFile* file, std::unique_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->Seek(offset));
   RETURN_NOT_OK(ReadMessage(file, message));
@@ -666,38 +648,32 @@ static Status ReadContiguousPayload(int64_t offset, io::RandomAccessFile* file,
   if (*message == nullptr) {
     return Status::Invalid("Unable to read metadata at offset");
   }
-
-  // TODO(ARROW-388): The buffer offsets start at 0, so we must construct a
-  // RandomAccessFile according to that frame of reference
-  RETURN_NOT_OK(file->Read((*message)->body_length(), payload));
   return Status::OK();
 }
 
 Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
-  std::shared_ptr<Buffer> payload;
-  std::shared_ptr<Message> message;
-
-  RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message, &payload));
-  io::BufferReader buffer_reader(payload);
-  return ReadRecordBatch(*message, schema, kMaxNestingDepth, &buffer_reader, out);
+  std::unique_ptr<Message> message;
+  RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message));
+  io::BufferReader buffer_reader(message->body());
+  return ReadRecordBatch(
+      *message->metadata(), schema, kMaxNestingDepth, &buffer_reader, out);
 }
 
 Status ReadTensor(
     int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out) {
   // Respect alignment of Tensor messages (see WriteTensor)
   offset = PaddedLength(offset);
-  std::shared_ptr<Message> message;
-  std::shared_ptr<Buffer> data;
-  RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message, &data));
+  std::unique_ptr<Message> message;
+  RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message));
 
   std::shared_ptr<DataType> type;
   std::vector<int64_t> shape;
   std::vector<int64_t> strides;
   std::vector<std::string> dim_names;
   RETURN_NOT_OK(
-      GetTensorMetadata(message->header(), &type, &shape, &strides, &dim_names));
-  *out = std::make_shared<Tensor>(type, data, shape, strides, dim_names);
+      GetTensorMetadata(*message->metadata(), &type, &shape, &strides, &dim_names));
+  *out = std::make_shared<Tensor>(type, message->body(), shape, strides, dim_names);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index dd29a36d40593..d6c261475014c 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -57,7 +57,7 @@ class ARROW_EXPORT RecordBatchReader {
   ///
   /// \param(out) batch the next loaded batch, nullptr at end of stream
   /// \return Status
-  virtual Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) = 0;
+  virtual Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) = 0;
 };
 
 /// \class RecordBatchStreamReader
@@ -66,16 +66,24 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
  public:
   virtual ~RecordBatchStreamReader();
 
-  /// Create batch reader from InputStream
+  /// Create batch reader from generic MessageReader
+  ///
+  /// \param(in) message_reader a MessageReader implementation
+  /// \param(out) out the created RecordBatchStreamReader object
+  /// \return Status
+  static Status Open(std::unique_ptr<MessageReader> message_reader,
+      std::shared_ptr<RecordBatchStreamReader>* out);
+
+  /// \Create Record batch stream reader from InputStream
   ///
   /// \param(in) stream an input stream instance
-  /// \param(out) reader the created reader object
+  /// \param(out) out the created RecordBatchStreamReader object
   /// \return Status
   static Status Open(const std::shared_ptr<io::InputStream>& stream,
-      std::shared_ptr<RecordBatchStreamReader>* reader);
+      std::shared_ptr<RecordBatchStreamReader>* out);
 
   std::shared_ptr<Schema> schema() const override;
-  Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) override;
+  Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) override;
 
  private:
   RecordBatchStreamReader();
@@ -122,7 +130,7 @@ class ARROW_EXPORT RecordBatchFileReader {
   /// \param(in) i the index of the record batch to return
   /// \param(out) batch the read batch
   /// \return Status
-  Status GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch);
+  Status ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch);
 
  private:
   RecordBatchFileReader();
@@ -133,16 +141,25 @@ class ARROW_EXPORT RecordBatchFileReader {
 
 // Generic read functions; does not copy data if the input supports zero copy reads
 
-/// Read record batch from file given metadata and schema
+/// \brief Read record batch from file given metadata and schema
 ///
 /// \param(in) metadata a Message containing the record batch metadata
 /// \param(in) schema the record batch schema
 /// \param(in) file a random access file
 /// \param(out) out the read record batch
-Status ARROW_EXPORT ReadRecordBatch(const Message& metadata,
+Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
     const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
     std::shared_ptr<RecordBatch>* out);
 
+/// \brief Read record batch from fully encapulated Message
+///
+/// \param[in] message a message instance containing metadata and body
+/// \param[in] schema
+/// \param[out] out the resulting RecordBatch
+/// \return Status
+Status ARROW_EXPORT ReadRecordBatch(const Message& message,
+    const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out);
+
 /// Read record batch from file given metadata and schema
 ///
 /// \param(in) metadata a Message containing the record batch metadata
@@ -150,7 +167,7 @@ Status ARROW_EXPORT ReadRecordBatch(const Message& metadata,
 /// \param(in) file a random access file
 /// \param(in) max_recursion_depth the maximum permitted nesting depth
 /// \param(out) out the read record batch
-Status ARROW_EXPORT ReadRecordBatch(const Message& metadata,
+Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
     const std::shared_ptr<Schema>& schema, int max_recursion_depth,
     io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
diff --git a/cpp/src/arrow/ipc/stream-to-file.cc b/cpp/src/arrow/ipc/stream-to-file.cc
index b94205427dfe0..de65883910120 100644
--- a/cpp/src/arrow/ipc/stream-to-file.cc
+++ b/cpp/src/arrow/ipc/stream-to-file.cc
@@ -40,7 +40,7 @@ Status ConvertToFile() {
 
   std::shared_ptr<RecordBatch> batch;
   while (true) {
-    RETURN_NOT_OK(reader->GetNextRecordBatch(&batch));
+    RETURN_NOT_OK(reader->ReadNextRecordBatch(&batch));
     if (batch == nullptr) break;
     RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
   }
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 7563343554c7c..14708a1e7a032 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -110,7 +110,7 @@ class RecordBatchSerializer : public ArrayVisitor {
     }
 
     // push back all common elements
-    field_nodes_.emplace_back(arr.length(), arr.null_count(), 0);
+    field_nodes_.push_back({arr.length(), arr.null_count(), 0});
 
     if (arr.null_count() > 0) {
       std::shared_ptr<Buffer> bitmap;
@@ -680,7 +680,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
 
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit) {
     // Push an empty FileBlock. Can be written in the footer later
-    record_batches_.emplace_back(0, 0, 0);
+    record_batches_.push_back({0, 0, 0});
     return WriteRecordBatch(
         batch, allow_64bit, &record_batches_[record_batches_.size() - 1]);
   }
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index f10dac75c67c8..816f95a42c0d7 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -477,9 +477,8 @@ class FixedWidthBytesConverter
   inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
-    Py_ssize_t expected_length =
-        std::dynamic_pointer_cast<FixedSizeBinaryType>(typed_builder_->type())
-            ->byte_width();
+    Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedSizeBinaryType>(
+        typed_builder_->type())->byte_width();
     if (item.obj() == Py_None) {
       RETURN_NOT_OK(typed_builder_->AppendNull());
       return Status::OK();
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 4810a31ffb5d5..400614d6e7b97 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -137,7 +137,6 @@ Tables and Record Batches
    Column
    RecordBatch
    Table
-   get_record_batch_size
 
 .. _api.tensor:
 
@@ -148,9 +147,6 @@ Tensor type and Functions
    :toctree: generated/
 
    Tensor
-   write_tensor
-   get_tensor_size
-   read_tensor
 
 .. _api.io:
 
@@ -177,12 +173,20 @@ Interprocess Communication and Messaging
 .. autosummary::
    :toctree: generated/
 
+   Message
+   MessageReader
    RecordBatchFileReader
    RecordBatchFileWriter
    RecordBatchStreamReader
    RecordBatchStreamWriter
    open_file
    open_stream
+   read_message
+   read_record_batch
+   get_record_batch_size
+   read_tensor
+   write_tensor
+   get_tensor_size
 
 .. _api.memory_pool:
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 434722c29baa3..f7cddd0e71057 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -69,9 +69,8 @@
 from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
                          Buffer, BufferReader, BufferOutputStream,
                          OSFile, MemoryMappedFile, memory_map,
-                         frombuffer, read_tensor, write_tensor,
+                         frombuffer,
                          memory_map, create_memory_map,
-                         get_record_batch_size, get_tensor_size,
                          have_libhdfs, have_libhdfs3, MockOutputStream)
 
 from pyarrow.lib import (MemoryPool, total_allocated_bytes,
@@ -89,8 +88,12 @@
 
 from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
 
-from pyarrow.ipc import (RecordBatchFileReader, RecordBatchFileWriter,
+from pyarrow.ipc import (Message, MessageReader,
+                         RecordBatchFileReader, RecordBatchFileWriter,
                          RecordBatchStreamReader, RecordBatchStreamWriter,
+                         read_message, read_record_batch, read_tensor,
+                         write_tensor,
+                         get_record_batch_size, get_tensor_size,
                          open_stream,
                          open_file,
                          serialize_pandas, deserialize_pandas)
diff --git a/python/pyarrow/feather.pxi b/python/pyarrow/feather.pxi
new file mode 100644
index 0000000000000..2e7cf6c9bd1b8
--- /dev/null
+++ b/python/pyarrow/feather.pxi
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#----------------------------------------------------------------------
+# Implement legacy Feather file format
+
+
+class FeatherError(Exception):
+    pass
+
+
+cdef class FeatherWriter:
+    cdef:
+        unique_ptr[CFeatherWriter] writer
+
+    cdef public:
+        int64_t num_rows
+
+    def __cinit__(self):
+        self.num_rows = -1
+
+    def open(self, object dest):
+        cdef shared_ptr[OutputStream] sink
+        get_writer(dest, &sink)
+
+        with nogil:
+            check_status(CFeatherWriter.Open(sink, &self.writer))
+
+    def close(self):
+        if self.num_rows < 0:
+            self.num_rows = 0
+        self.writer.get().SetNumRows(self.num_rows)
+        check_status(self.writer.get().Finalize())
+
+    def write_array(self, object name, object col, object mask=None):
+        cdef Array arr
+
+        if self.num_rows >= 0:
+            if len(col) != self.num_rows:
+                raise ValueError('prior column had a different number of rows')
+        else:
+            self.num_rows = len(col)
+
+        if isinstance(col, Array):
+            arr = col
+        else:
+            arr = Array.from_pandas(col, mask=mask)
+
+        cdef c_string c_name = tobytes(name)
+
+        with nogil:
+            check_status(
+                self.writer.get().Append(c_name, deref(arr.sp_array)))
+
+
+cdef class FeatherReader:
+    cdef:
+        unique_ptr[CFeatherReader] reader
+
+    def __cinit__(self):
+        pass
+
+    def open(self, source):
+        cdef shared_ptr[RandomAccessFile] reader
+        get_reader(source, &reader)
+
+        with nogil:
+            check_status(CFeatherReader.Open(reader, &self.reader))
+
+    property num_rows:
+
+        def __get__(self):
+            return self.reader.get().num_rows()
+
+    property num_columns:
+
+        def __get__(self):
+            return self.reader.get().num_columns()
+
+    def get_column_name(self, int i):
+        cdef c_string name = self.reader.get().GetColumnName(i)
+        return frombytes(name)
+
+    def get_column(self, int i):
+        if i < 0 or i >= self.num_columns:
+            raise IndexError(i)
+
+        cdef shared_ptr[CColumn] sp_column
+        with nogil:
+            check_status(self.reader.get()
+                         .GetColumn(i, &sp_column))
+
+        cdef Column col = Column()
+        col.init(sp_column)
+        return col
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 9fad8244442aa..dd791cdefb73e 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -546,41 +546,41 @@ cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
         int64_t GetExtentBytesWritten()
 
 
-cdef extern from "arrow/ipc/metadata.h" namespace "arrow::ipc" nogil:
-    cdef cppclass SchemaMessage:
-        int num_fields()
-        CStatus GetField(int i, shared_ptr[CField]* out)
-        CStatus GetSchema(shared_ptr[CSchema]* out)
-
-    cdef cppclass FieldMetadata:
-        pass
-
-    cdef cppclass BufferMetadata:
-        pass
-
-    cdef cppclass RecordBatchMessage:
-        pass
-
-    cdef cppclass DictionaryBatchMessage:
-        pass
-
+cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
     enum MessageType" arrow::ipc::Message::Type":
         MessageType_SCHEMA" arrow::ipc::Message::SCHEMA"
         MessageType_RECORD_BATCH" arrow::ipc::Message::RECORD_BATCH"
         MessageType_DICTIONARY_BATCH" arrow::ipc::Message::DICTIONARY_BATCH"
 
-    cdef cppclass Message:
-        CStatus Open(const shared_ptr[CBuffer]& buf,
-                     shared_ptr[Message]* out)
-        int64_t body_length()
+    enum MetadataVersion" arrow::ipc::MetadataVersion":
+        MessageType_V1" arrow::ipc::MetadataVersion::V1"
+        MessageType_V2" arrow::ipc::MetadataVersion::V2"
+        MessageType_V3" arrow::ipc::MetadataVersion::V3"
+
+    cdef cppclass CMessage" arrow::ipc::Message":
+        CStatus Open(const shared_ptr[CBuffer]& metadata,
+                     const shared_ptr[CBuffer]& body,
+                     unique_ptr[CMessage]* out)
+
+        shared_ptr[CBuffer] body()
+
+        c_bool Equals(const CMessage& other)
+
+        shared_ptr[CBuffer] metadata()
+        MetadataVersion metadata_version()
         MessageType type()
 
-        shared_ptr[SchemaMessage] GetSchema()
-        shared_ptr[RecordBatchMessage] GetRecordBatch()
-        shared_ptr[DictionaryBatchMessage] GetDictionaryBatch()
+        CStatus SerializeTo(OutputStream* stream, int64_t* output_length)
 
+    c_string FormatMessageType(MessageType type)
 
-cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
+    cdef cppclass CMessageReader \
+        " arrow::ipc::MessageReader":
+        CStatus ReadNextMessage(unique_ptr[CMessage]* out)
+
+    cdef cppclass CInputStreamMessageReader \
+        " arrow::ipc::InputStreamMessageReader":
+        CInputStreamMessageReader(const shared_ptr[InputStream]& stream)
 
     cdef cppclass CRecordBatchWriter \
         " arrow::ipc::RecordBatchWriter":
@@ -590,7 +590,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
     cdef cppclass CRecordBatchReader \
         " arrow::ipc::RecordBatchReader":
         shared_ptr[CSchema] schema()
-        CStatus GetNextRecordBatch(shared_ptr[CRecordBatch]* batch)
+        CStatus ReadNextRecordBatch(shared_ptr[CRecordBatch]* batch)
 
     cdef cppclass CRecordBatchStreamReader \
         " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
@@ -598,6 +598,10 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
         CStatus Open(const shared_ptr[InputStream]& stream,
                      shared_ptr[CRecordBatchStreamReader]* out)
 
+        @staticmethod
+        CStatus Open2" Open"(unique_ptr[CMessageReader] message_reader,
+                     shared_ptr[CRecordBatchStreamReader]* out)
+
     cdef cppclass CRecordBatchStreamWriter \
         " arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter):
         @staticmethod
@@ -625,7 +629,9 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
 
         int num_record_batches()
 
-        CStatus GetRecordBatch(int i, shared_ptr[CRecordBatch]* batch)
+        CStatus ReadRecordBatch(int i, shared_ptr[CRecordBatch]* batch)
+
+    CStatus ReadMessage(InputStream* stream, unique_ptr[CMessage]* message)
 
     CStatus GetRecordBatchSize(const CRecordBatch& batch, int64_t* size)
     CStatus GetTensorSize(const CTensor& tensor, int64_t* size)
@@ -637,6 +643,10 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
     CStatus ReadTensor(int64_t offset, RandomAccessFile* file,
                        shared_ptr[CTensor]* out)
 
+    CStatus ReadRecordBatch(const CMessage& message,
+                            const shared_ptr[CSchema]& schema,
+                            shared_ptr[CRecordBatch]* out)
+
 
 cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
 
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 3221185c27bc5..8b213a33053d4 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -956,356 +956,3 @@ cdef class HdfsFile(NativeFile):
 
     def __dealloc__(self):
         self.parent = None
-
-# ----------------------------------------------------------------------
-# File and stream readers and writers
-
-cdef class _RecordBatchWriter:
-    cdef:
-        shared_ptr[CRecordBatchWriter] writer
-        shared_ptr[OutputStream] sink
-        bint closed
-
-    def __cinit__(self):
-        self.closed = True
-
-    def __dealloc__(self):
-        if not self.closed:
-            self.close()
-
-    def _open(self, sink, Schema schema):
-        cdef:
-            shared_ptr[CRecordBatchStreamWriter] writer
-
-        get_writer(sink, &self.sink)
-
-        with nogil:
-            check_status(
-                CRecordBatchStreamWriter.Open(self.sink.get(),
-                                              schema.sp_schema,
-                                              &writer))
-
-        self.writer = <shared_ptr[CRecordBatchWriter]> writer
-        self.closed = False
-
-    def write_batch(self, RecordBatch batch):
-        with nogil:
-            check_status(self.writer.get()
-                         .WriteRecordBatch(deref(batch.batch)))
-
-    def close(self):
-        with nogil:
-            check_status(self.writer.get().Close())
-        self.closed = True
-
-
-cdef class _RecordBatchReader:
-    cdef:
-        shared_ptr[CRecordBatchReader] reader
-
-    cdef readonly:
-        Schema schema
-
-    def __cinit__(self):
-        pass
-
-    def _open(self, source):
-        cdef:
-            shared_ptr[RandomAccessFile] file_handle
-            shared_ptr[InputStream] in_stream
-            shared_ptr[CRecordBatchStreamReader] reader
-
-        get_reader(source, &file_handle)
-        in_stream = <shared_ptr[InputStream]> file_handle
-
-        with nogil:
-            check_status(CRecordBatchStreamReader.Open(in_stream, &reader))
-
-        self.reader = <shared_ptr[CRecordBatchReader]> reader
-        self.schema = pyarrow_wrap_schema(self.reader.get().schema())
-
-    def get_next_batch(self):
-        """
-        Read next RecordBatch from the stream. Raises StopIteration at end of
-        stream
-        """
-        cdef shared_ptr[CRecordBatch] batch
-
-        with nogil:
-            check_status(self.reader.get().GetNextRecordBatch(&batch))
-
-        if batch.get() == NULL:
-            raise StopIteration
-
-        return pyarrow_wrap_batch(batch)
-
-    def read_all(self):
-        """
-        Read all record batches as a pyarrow.Table
-        """
-        cdef:
-            vector[shared_ptr[CRecordBatch]] batches
-            shared_ptr[CRecordBatch] batch
-            shared_ptr[CTable] table
-
-        with nogil:
-            while True:
-                check_status(self.reader.get().GetNextRecordBatch(&batch))
-                if batch.get() == NULL:
-                    break
-                batches.push_back(batch)
-
-            check_status(CTable.FromRecordBatches(batches, &table))
-
-        return pyarrow_wrap_table(table)
-
-
-cdef class _RecordBatchFileWriter(_RecordBatchWriter):
-
-    def _open(self, sink, Schema schema):
-        cdef shared_ptr[CRecordBatchFileWriter] writer
-        get_writer(sink, &self.sink)
-
-        with nogil:
-            check_status(
-                CRecordBatchFileWriter.Open(self.sink.get(), schema.sp_schema,
-                                      &writer))
-
-        # Cast to base class, because has same interface
-        self.writer = <shared_ptr[CRecordBatchWriter]> writer
-        self.closed = False
-
-
-cdef class _RecordBatchFileReader:
-    cdef:
-        shared_ptr[CRecordBatchFileReader] reader
-
-    cdef readonly:
-        Schema schema
-
-    def __cinit__(self):
-        pass
-
-    def _open(self, source, footer_offset=None):
-        cdef shared_ptr[RandomAccessFile] reader
-        get_reader(source, &reader)
-
-        cdef int64_t offset = 0
-        if footer_offset is not None:
-            offset = footer_offset
-
-        with nogil:
-            if offset != 0:
-                check_status(CRecordBatchFileReader.Open2(
-                    reader, offset, &self.reader))
-            else:
-                check_status(CRecordBatchFileReader.Open(reader, &self.reader))
-
-        self.schema = pyarrow_wrap_schema(self.reader.get().schema())
-
-    property num_record_batches:
-
-        def __get__(self):
-            return self.reader.get().num_record_batches()
-
-    def get_batch(self, int i):
-        cdef shared_ptr[CRecordBatch] batch
-
-        if i < 0 or i >= self.num_record_batches:
-            raise ValueError('Batch number {0} out of range'.format(i))
-
-        with nogil:
-            check_status(self.reader.get().GetRecordBatch(i, &batch))
-
-        return pyarrow_wrap_batch(batch)
-
-    # TODO(wesm): ARROW-503: Function was renamed. Remove after a period of
-    # time has passed
-    get_record_batch = get_batch
-
-    def read_all(self):
-        """
-        Read all record batches as a pyarrow.Table
-        """
-        cdef:
-            vector[shared_ptr[CRecordBatch]] batches
-            shared_ptr[CTable] table
-            int i, nbatches
-
-        nbatches = self.num_record_batches
-
-        batches.resize(nbatches)
-        with nogil:
-            for i in range(nbatches):
-                check_status(self.reader.get().GetRecordBatch(i, &batches[i]))
-            check_status(CTable.FromRecordBatches(batches, &table))
-
-        return pyarrow_wrap_table(table)
-
-
-#----------------------------------------------------------------------
-# Implement legacy Feather file format
-
-
-class FeatherError(Exception):
-    pass
-
-
-cdef class FeatherWriter:
-    cdef:
-        unique_ptr[CFeatherWriter] writer
-
-    cdef public:
-        int64_t num_rows
-
-    def __cinit__(self):
-        self.num_rows = -1
-
-    def open(self, object dest):
-        cdef shared_ptr[OutputStream] sink
-        get_writer(dest, &sink)
-
-        with nogil:
-            check_status(CFeatherWriter.Open(sink, &self.writer))
-
-    def close(self):
-        if self.num_rows < 0:
-            self.num_rows = 0
-        self.writer.get().SetNumRows(self.num_rows)
-        check_status(self.writer.get().Finalize())
-
-    def write_array(self, object name, object col, object mask=None):
-        cdef Array arr
-
-        if self.num_rows >= 0:
-            if len(col) != self.num_rows:
-                raise ValueError('prior column had a different number of rows')
-        else:
-            self.num_rows = len(col)
-
-        if isinstance(col, Array):
-            arr = col
-        else:
-            arr = Array.from_pandas(col, mask=mask)
-
-        cdef c_string c_name = tobytes(name)
-
-        with nogil:
-            check_status(
-                self.writer.get().Append(c_name, deref(arr.sp_array)))
-
-
-cdef class FeatherReader:
-    cdef:
-        unique_ptr[CFeatherReader] reader
-
-    def __cinit__(self):
-        pass
-
-    def open(self, source):
-        cdef shared_ptr[RandomAccessFile] reader
-        get_reader(source, &reader)
-
-        with nogil:
-            check_status(CFeatherReader.Open(reader, &self.reader))
-
-    property num_rows:
-
-        def __get__(self):
-            return self.reader.get().num_rows()
-
-    property num_columns:
-
-        def __get__(self):
-            return self.reader.get().num_columns()
-
-    def get_column_name(self, int i):
-        cdef c_string name = self.reader.get().GetColumnName(i)
-        return frombytes(name)
-
-    def get_column(self, int i):
-        if i < 0 or i >= self.num_columns:
-            raise IndexError(i)
-
-        cdef shared_ptr[CColumn] sp_column
-        with nogil:
-            check_status(self.reader.get()
-                         .GetColumn(i, &sp_column))
-
-        cdef Column col = Column()
-        col.init(sp_column)
-        return col
-
-
-def get_tensor_size(Tensor tensor):
-    """
-    Return total size of serialized Tensor including metadata and padding
-    """
-    cdef int64_t size
-    with nogil:
-        check_status(GetTensorSize(deref(tensor.tp), &size))
-    return size
-
-
-def get_record_batch_size(RecordBatch batch):
-    """
-    Return total size of serialized RecordBatch including metadata and padding
-    """
-    cdef int64_t size
-    with nogil:
-        check_status(GetRecordBatchSize(deref(batch.batch), &size))
-    return size
-
-
-def write_tensor(Tensor tensor, NativeFile dest):
-    """
-    Write pyarrow.Tensor to pyarrow.NativeFile object its current position
-
-    Parameters
-    ----------
-    tensor : pyarrow.Tensor
-    dest : pyarrow.NativeFile
-
-    Returns
-    -------
-    bytes_written : int
-        Total number of bytes written to the file
-    """
-    cdef:
-        int32_t metadata_length
-        int64_t body_length
-
-    dest._assert_writeable()
-
-    with nogil:
-        check_status(
-            WriteTensor(deref(tensor.tp), dest.wr_file.get(),
-                        &metadata_length, &body_length))
-
-    return metadata_length + body_length
-
-
-def read_tensor(NativeFile source):
-    """
-    Read pyarrow.Tensor from pyarrow.NativeFile object from current
-    position. If the file source supports zero copy (e.g. a memory map), then
-    this operation does not allocate any memory
-
-    Parameters
-    ----------
-    source : pyarrow.NativeFile
-
-    Returns
-    -------
-    tensor : Tensor
-    """
-    cdef:
-        shared_ptr[CTensor] sp_tensor
-
-    source._assert_readable()
-
-    cdef int64_t offset = source.tell()
-    with nogil:
-        check_status(ReadTensor(offset, source.rd_file.get(), &sp_tensor))
-
-    return pyarrow_wrap_tensor(sp_tensor)
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
new file mode 100644
index 0000000000000..d6df30b8619db
--- /dev/null
+++ b/python/pyarrow/ipc.pxi
@@ -0,0 +1,480 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cdef class Message:
+    cdef:
+        unique_ptr[CMessage] message
+
+    def __cinit__(self):
+        pass
+
+    def __null_check(self):
+        if self.message.get() == NULL:
+            raise TypeError('Message improperly initialized (null)')
+
+    property type:
+
+        def __get__(self):
+            self.__null_check()
+            return frombytes(FormatMessageType(self.message.get().type()))
+
+    property metadata:
+
+        def __get__(self):
+            self.__null_check()
+            return pyarrow_wrap_buffer(self.message.get().metadata())
+
+    property body:
+
+        def __get__(self):
+            self.__null_check()
+            cdef shared_ptr[CBuffer] body = self.message.get().body()
+            if body.get() == NULL:
+                return None
+            else:
+                return pyarrow_wrap_buffer(body)
+
+    def equals(self, Message other):
+        """
+        Returns True if the message contents (metadata and body) are identical
+
+        Parameters
+        ----------
+        other : Message
+
+        Returns
+        -------
+        are_equal : bool
+        """
+        cdef c_bool result
+        with nogil:
+            result = self.message.get().Equals(deref(other.message.get()))
+        return result
+
+    def serialize(self, memory_pool=None):
+        """
+        Write message to Buffer with length-prefixed metadata, then body
+
+        Parameters
+        ----------
+        memory_pool : MemoryPool, default None
+            Uses default memory pool if not specified
+
+        Returns
+        -------
+        serialized : Buffer
+        """
+        cdef:
+            BufferOutputStream stream = BufferOutputStream(memory_pool)
+            int64_t output_length = 0
+
+        with nogil:
+            check_status(self.message.get()
+                         .SerializeTo(stream.wr_file.get(),
+                                      &output_length))
+        return stream.get_result()
+
+    def __repr__(self):
+        metadata_len = self.metadata.size
+        body = self.body
+        body_len = 0 if body is None else body.size
+
+        return """pyarrow.Message
+type: {0}
+metadata length: {1}
+body length: {2}""".format(self.type, metadata_len, body_len)
+
+
+cdef class MessageReader:
+    cdef:
+        unique_ptr[CMessageReader] reader
+
+    def __cinit__(self):
+        pass
+
+    def __null_check(self):
+        if self.reader.get() == NULL:
+            raise TypeError('Message improperly initialized (null)')
+
+    def __repr__(self):
+        self.__null_check()
+        return object.__repr__(self)
+
+    @staticmethod
+    def open_stream(source):
+        cdef MessageReader result = MessageReader()
+        cdef shared_ptr[InputStream] in_stream
+        get_input_stream(source, &in_stream)
+        with nogil:
+            result.reader.reset(new CInputStreamMessageReader(in_stream))
+
+        return result
+
+    def __iter__(self):
+        while True:
+            yield self.read_next_message()
+
+    def read_next_message(self):
+        """
+        Read next Message from the stream. Raises StopIteration at end of
+        stream
+        """
+        cdef Message result = Message()
+
+        with nogil:
+            check_status(self.reader.get().ReadNextMessage(&result.message))
+
+        if result.message.get() == NULL:
+            raise StopIteration
+
+        return result
+
+# ----------------------------------------------------------------------
+# File and stream readers and writers
+
+cdef class _RecordBatchWriter:
+    cdef:
+        shared_ptr[CRecordBatchWriter] writer
+        shared_ptr[OutputStream] sink
+        bint closed
+
+    def __cinit__(self):
+        self.closed = True
+
+    def __dealloc__(self):
+        if not self.closed:
+            self.close()
+
+    def _open(self, sink, Schema schema):
+        cdef:
+            shared_ptr[CRecordBatchStreamWriter] writer
+
+        get_writer(sink, &self.sink)
+
+        with nogil:
+            check_status(
+                CRecordBatchStreamWriter.Open(self.sink.get(),
+                                              schema.sp_schema,
+                                              &writer))
+
+        self.writer = <shared_ptr[CRecordBatchWriter]> writer
+        self.closed = False
+
+    def write_batch(self, RecordBatch batch):
+        with nogil:
+            check_status(self.writer.get()
+                         .WriteRecordBatch(deref(batch.batch)))
+
+    def close(self):
+        with nogil:
+            check_status(self.writer.get().Close())
+        self.closed = True
+
+
+cdef get_input_stream(object source, shared_ptr[InputStream]* out):
+    cdef:
+        shared_ptr[RandomAccessFile] file_handle
+
+    get_reader(source, &file_handle)
+    out[0] = <shared_ptr[InputStream]> file_handle
+
+
+cdef class _RecordBatchReader:
+    cdef:
+        shared_ptr[CRecordBatchReader] reader
+
+    cdef readonly:
+        Schema schema
+
+    def __cinit__(self):
+        pass
+
+    def _open(self, source):
+        cdef:
+            shared_ptr[InputStream] in_stream
+            shared_ptr[CRecordBatchStreamReader] reader
+
+        get_input_stream(source, &in_stream)
+
+        with nogil:
+            check_status(CRecordBatchStreamReader.Open(in_stream, &reader))
+
+        self.reader = <shared_ptr[CRecordBatchReader]> reader
+        self.schema = Schema()
+        self.schema.init_schema(self.reader.get().schema())
+
+    def __iter__(self):
+        while True:
+            yield self.read_next_batch()
+
+    def get_next_batch(self):
+        import warnings
+        warnings.warn('Please use read_next_batch instead of '
+                      'get_next_batch', FutureWarning)
+        return self.read_next_batch()
+
+    def read_next_batch(self):
+        """
+        Read next RecordBatch from the stream. Raises StopIteration at end of
+        stream
+        """
+        cdef shared_ptr[CRecordBatch] batch
+
+        with nogil:
+            check_status(self.reader.get().ReadNextRecordBatch(&batch))
+
+        if batch.get() == NULL:
+            raise StopIteration
+
+        return pyarrow_wrap_batch(batch)
+
+    def read_all(self):
+        """
+        Read all record batches as a pyarrow.Table
+        """
+        cdef:
+            vector[shared_ptr[CRecordBatch]] batches
+            shared_ptr[CRecordBatch] batch
+            shared_ptr[CTable] table
+
+        with nogil:
+            while True:
+                check_status(self.reader.get().ReadNextRecordBatch(&batch))
+                if batch.get() == NULL:
+                    break
+                batches.push_back(batch)
+
+            check_status(CTable.FromRecordBatches(batches, &table))
+
+        return pyarrow_wrap_table(table)
+
+
+cdef class _RecordBatchFileWriter(_RecordBatchWriter):
+
+    def _open(self, sink, Schema schema):
+        cdef shared_ptr[CRecordBatchFileWriter] writer
+        get_writer(sink, &self.sink)
+
+        with nogil:
+            check_status(
+                CRecordBatchFileWriter.Open(self.sink.get(), schema.sp_schema,
+                                      &writer))
+
+        # Cast to base class, because has same interface
+        self.writer = <shared_ptr[CRecordBatchWriter]> writer
+        self.closed = False
+
+
+cdef class _RecordBatchFileReader:
+    cdef:
+        shared_ptr[CRecordBatchFileReader] reader
+
+    cdef readonly:
+        Schema schema
+
+    def __cinit__(self):
+        pass
+
+    def _open(self, source, footer_offset=None):
+        cdef shared_ptr[RandomAccessFile] reader
+        get_reader(source, &reader)
+
+        cdef int64_t offset = 0
+        if footer_offset is not None:
+            offset = footer_offset
+
+        with nogil:
+            if offset != 0:
+                check_status(CRecordBatchFileReader.Open2(
+                    reader, offset, &self.reader))
+            else:
+                check_status(CRecordBatchFileReader.Open(reader, &self.reader))
+
+        self.schema = pyarrow_wrap_schema(self.reader.get().schema())
+
+    property num_record_batches:
+
+        def __get__(self):
+            return self.reader.get().num_record_batches()
+
+    def get_batch(self, int i):
+        cdef shared_ptr[CRecordBatch] batch
+
+        if i < 0 or i >= self.num_record_batches:
+            raise ValueError('Batch number {0} out of range'.format(i))
+
+        with nogil:
+            check_status(self.reader.get().ReadRecordBatch(i, &batch))
+
+        return pyarrow_wrap_batch(batch)
+
+    # TODO(wesm): ARROW-503: Function was renamed. Remove after a period of
+    # time has passed
+    get_record_batch = get_batch
+
+    def read_all(self):
+        """
+        Read all record batches as a pyarrow.Table
+        """
+        cdef:
+            vector[shared_ptr[CRecordBatch]] batches
+            shared_ptr[CTable] table
+            int i, nbatches
+
+        nbatches = self.num_record_batches
+
+        batches.resize(nbatches)
+        with nogil:
+            for i in range(nbatches):
+                check_status(self.reader.get().ReadRecordBatch(i, &batches[i]))
+            check_status(CTable.FromRecordBatches(batches, &table))
+
+        return pyarrow_wrap_table(table)
+
+
+def get_tensor_size(Tensor tensor):
+    """
+    Return total size of serialized Tensor including metadata and padding
+    """
+    cdef int64_t size
+    with nogil:
+        check_status(GetTensorSize(deref(tensor.tp), &size))
+    return size
+
+
+def get_record_batch_size(RecordBatch batch):
+    """
+    Return total size of serialized RecordBatch including metadata and padding
+    """
+    cdef int64_t size
+    with nogil:
+        check_status(GetRecordBatchSize(deref(batch.batch), &size))
+    return size
+
+
+def write_tensor(Tensor tensor, NativeFile dest):
+    """
+    Write pyarrow.Tensor to pyarrow.NativeFile object its current position
+
+    Parameters
+    ----------
+    tensor : pyarrow.Tensor
+    dest : pyarrow.NativeFile
+
+    Returns
+    -------
+    bytes_written : int
+        Total number of bytes written to the file
+    """
+    cdef:
+        int32_t metadata_length
+        int64_t body_length
+
+    dest._assert_writeable()
+
+    with nogil:
+        check_status(
+            WriteTensor(deref(tensor.tp), dest.wr_file.get(),
+                        &metadata_length, &body_length))
+
+    return metadata_length + body_length
+
+
+def read_tensor(NativeFile source):
+    """
+    Read pyarrow.Tensor from pyarrow.NativeFile object from current
+    position. If the file source supports zero copy (e.g. a memory map), then
+    this operation does not allocate any memory
+
+    Parameters
+    ----------
+    source : pyarrow.NativeFile
+
+    Returns
+    -------
+    tensor : Tensor
+    """
+    cdef:
+        shared_ptr[CTensor] sp_tensor
+
+    source._assert_readable()
+
+    cdef int64_t offset = source.tell()
+    with nogil:
+        check_status(ReadTensor(offset, source.rd_file.get(), &sp_tensor))
+
+    return pyarrow_wrap_tensor(sp_tensor)
+
+
+def read_message(source):
+    """
+    Read length-prefixed message from file or buffer-like object
+
+    Parameters
+    ----------
+    source : pyarrow.NativeFile, file-like object, or buffer-like object
+
+    Returns
+    -------
+    message : Message
+    """
+    cdef:
+        Message result = Message()
+        NativeFile cpp_file
+
+    if not isinstance(source, NativeFile):
+        if hasattr(source, 'read'):
+            source = PythonFile(source)
+        else:
+            source = BufferReader(source)
+
+    if not isinstance(source, NativeFile):
+        raise ValueError('Unable to read message from object with type: {0}'
+                         .format(type(source)))
+
+    source._assert_readable()
+
+    cpp_file = source
+
+    with nogil:
+        check_status(ReadMessage(cpp_file.rd_file.get(),
+                                 &result.message))
+
+    return result
+
+
+def read_record_batch(Message batch_message, Schema schema):
+    """
+    Read RecordBatch from message, given a known schema
+
+    Parameters
+    ----------
+    batch_message : Message
+        Such as that obtained from read_message
+    schema : Schema
+
+    Returns
+    -------
+    batch : RecordBatch
+    """
+    cdef shared_ptr[CRecordBatch] result
+
+    with nogil:
+        check_status(ReadRecordBatch(deref(batch_message.message.get()),
+                                     schema.sp_schema, &result))
+
+    return pyarrow_wrap_batch(result)
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index 6173299bca6f5..e8ea3acb3c397 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -18,6 +18,11 @@
 # Arrow file and stream reader/writer classes, and other messaging tools
 
 import pyarrow as pa
+
+from pyarrow.lib import (Message, MessageReader,  # noqa
+                         read_message, read_record_batch,
+                         read_tensor, write_tensor,
+                         get_record_batch_size, get_tensor_size)
 import pyarrow.lib as lib
 
 
@@ -33,10 +38,6 @@ class RecordBatchStreamReader(lib._RecordBatchReader):
     def __init__(self, source):
         self._open(source)
 
-    def __iter__(self):
-        while True:
-            yield self.get_next_batch()
-
 
 class RecordBatchStreamWriter(lib._RecordBatchWriter):
     """
@@ -136,7 +137,7 @@ def serialize_pandas(df):
     """
     batch = pa.RecordBatch.from_pandas(df)
     sink = pa.InMemoryOutputStream()
-    writer = pa.RecordBatchFileWriter(sink, batch.schema)
+    writer = pa.RecordBatchStreamWriter(sink, batch.schema)
     writer.write_batch(batch)
     writer.close()
     return sink.get_result()
@@ -157,6 +158,6 @@ def deserialize_pandas(buf, nthreads=1):
     df : pandas.DataFrame
     """
     buffer_reader = pa.BufferReader(buf)
-    reader = pa.RecordBatchFileReader(buffer_reader)
+    reader = pa.RecordBatchStreamReader(buffer_reader)
     table = reader.read_all()
     return table.to_pandas(nthreads=nthreads)
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 13c1822b25e56..cf8e4df444594 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -106,9 +106,15 @@ include "array.pxi"
 # Column, Table, Record Batch
 include "table.pxi"
 
-# File IO, IPC
+# File IO
 include "io.pxi"
 
+# IPC / Messaging
+include "ipc.pxi"
+
+# Feather format
+include "feather.pxi"
+
 #----------------------------------------------------------------------
 # Public API
 
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index a9569b2d73a46..c909b3ec856b6 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -22,7 +22,7 @@
 import six
 
 import pyarrow as pa
-from pyarrow.compat import PY2
+from pyarrow.compat import PY2  # noqa
 
 
 INDEX_LEVEL_NAME_REGEX = re.compile(r'^__index_level_\d+__$')
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 01e530651ddd3..575755debc589 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -432,9 +432,23 @@ cdef class RecordBatch:
 
         return self._schema
 
-    def __getitem__(self, i):
+    def column(self, i):
+        """
+        Select single column from record batcha
+
+        Returns
+        -------
+        column : pyarrow.Array
+        """
+        if not -self.num_columns <= i < self.num_columns:
+            raise IndexError(
+                'Record batch column index {:d} is out of range'.format(i)
+            )
         return pyarrow_wrap_array(self.batch.column(i))
 
+    def __getitem__(self, i):
+        return self.column(i)
+
     def slice(self, offset=0, length=None):
         """
         Compute zero-copy slice of this RecordBatch
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index fa9608f8afb7a..da94da9fc7499 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -26,7 +26,7 @@
 }
 
 try:
-    import pyarrow.parquet
+    import pyarrow.parquet  # noqa
     defaults['parquet'] = True
 except ImportError:
     pass
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index e0a7416af3d3f..ed8153189175b 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -225,7 +225,7 @@ def test_simple_type_construction():
         (pa.timestamp('us', 'UTC'), 'datetimetz'),
         pytest.mark.xfail((pa.time32('s'), None), raises=NotImplementedError),
         pytest.mark.xfail((pa.time64('us'), None), raises=NotImplementedError),
-   ]
+    ]
 )
 def test_logical_type(type, expected):
     assert get_logical_type(type) == expected
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 62592f9700536..ec261595585c2 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -22,6 +22,7 @@
 import datetime
 import decimal
 
+
 class StrangeIterable:
     def __init__(self, lst):
         self.lst = lst
@@ -29,6 +30,7 @@ def __init__(self, lst):
     def __iter__(self):
         return self.lst.__iter__()
 
+
 class TestConvertIterable(unittest.TestCase):
 
     def test_iterable_types(self):
@@ -61,6 +63,7 @@ def test_iterator_size_underflow(self):
         arr2 = pa.array((0, 1, 2))
         assert arr1.equals(arr2)
 
+
 class TestConvertSequence(unittest.TestCase):
 
     def test_sequence_types(self):
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 91bf56bd8e649..7978ace1c4302 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -359,7 +359,8 @@ def test_non_string_columns(self):
         expected = df.rename(columns=str)
         self._check_pandas_roundtrip(df, expected)
 
-    @pytest.mark.skipif(not os.path.supports_unicode_filenames, reason='unicode filenames not supported')
+    @pytest.mark.skipif(not os.path.supports_unicode_filenames,
+                        reason='unicode filenames not supported')
     def test_unicode_filename(self):
         # GH #209
         name = (b'Besa_Kavaj\xc3\xab.feather').decode('utf-8')
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index b91a8e9608df5..b2b90d41bfa08 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -38,7 +38,7 @@ def _get_sink(self):
         return io.BytesIO()
 
     def _get_source(self):
-        return pa.BufferReader(self.sink.getvalue())
+        return self.sink.getvalue()
 
     def write_batches(self):
         nrows = 5
@@ -79,7 +79,7 @@ def test_empty_file(self):
 
     def test_simple_roundtrip(self):
         batches = self.write_batches()
-        file_contents = self._get_source()
+        file_contents = pa.BufferReader(self._get_source())
 
         reader = pa.open_file(file_contents)
 
@@ -93,7 +93,7 @@ def test_simple_roundtrip(self):
 
     def test_read_all(self):
         batches = self.write_batches()
-        file_contents = self._get_source()
+        file_contents = pa.BufferReader(self._get_source())
 
         reader = pa.open_file(file_contents)
 
@@ -114,7 +114,7 @@ def test_empty_stream(self):
 
     def test_simple_roundtrip(self):
         batches = self.write_batches()
-        file_contents = self._get_source()
+        file_contents = pa.BufferReader(self._get_source())
         reader = pa.open_stream(file_contents)
 
         assert reader.schema.equals(batches[0].schema)
@@ -131,7 +131,7 @@ def test_simple_roundtrip(self):
 
     def test_read_all(self):
         batches = self.write_batches()
-        file_contents = self._get_source()
+        file_contents = pa.BufferReader(self._get_source())
         reader = pa.open_stream(file_contents)
 
         result = reader.read_all()
@@ -139,6 +139,55 @@ def test_read_all(self):
         assert result.equals(expected)
 
 
+class TestMessageReader(MessagingTest, unittest.TestCase):
+
+    def _get_example_messages(self):
+        batches = self.write_batches()
+        file_contents = self._get_source()
+        buf_reader = pa.BufferReader(file_contents)
+        reader = pa.MessageReader.open_stream(buf_reader)
+        return batches, list(reader)
+
+    def _get_writer(self, sink, schema):
+        return pa.RecordBatchStreamWriter(sink, schema)
+
+    def test_ctors_no_segfault(self):
+        with pytest.raises(TypeError):
+            repr(pa.Message())
+
+        with pytest.raises(TypeError):
+            repr(pa.MessageReader())
+
+    def test_message_reader(self):
+        _, messages = self._get_example_messages()
+
+        assert len(messages) == 6
+        assert messages[0].type == 'schema'
+        for msg in messages[1:]:
+            assert msg.type == 'record batch'
+
+    def test_serialize_read_message(self):
+        _, messages = self._get_example_messages()
+
+        msg = messages[0]
+        buf = msg.serialize()
+
+        restored = pa.read_message(buf)
+        restored2 = pa.read_message(pa.BufferReader(buf))
+        restored3 = pa.read_message(buf.to_pybytes())
+
+        assert msg.equals(restored)
+        assert msg.equals(restored2)
+        assert msg.equals(restored3)
+
+    def test_read_record_batch(self):
+        batches, messages = self._get_example_messages()
+
+        for batch, message in zip(batches, messages[1:]):
+            read_batch = pa.read_record_batch(message, batch.schema)
+            assert read_batch.equals(batch)
+
+
 class TestSocket(MessagingTest, unittest.TestCase):
 
     class StreamReaderServer(threading.Thread):
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 94a0e38cfb090..f606a7fa0c670 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -453,13 +453,15 @@ def test_date_time_types():
 
     table = pa.Table.from_arrays([a1, a2, a3, a4, a5, a6],
                                  ['date32', 'date64', 'timestamp[us]',
-                                  'time32[s]', 'time64[us]', 'time32_from64[s]'])
+                                  'time32[s]', 'time64[us]',
+                                  'time32_from64[s]'])
 
     # date64 as date32
     # time32[s] to time32[ms]
     expected = pa.Table.from_arrays([a1, a1, a3, a4, a5, ex_a6],
                                     ['date32', 'date64', 'timestamp[us]',
-                                     'time32[s]', 'time64[us]', 'time32_from64[s]'])
+                                     'time32[s]', 'time64[us]',
+                                     'time32_from64[s]'])
 
     _check_roundtrip(table, expected=expected, version='2.0')
 
@@ -848,6 +850,7 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
     with pytest.raises(ValueError):
         read_multiple_files(mixed_paths)
 
+
 @parquet
 def test_multiindex_duplicate_values(tmpdir):
     num_rows = 3
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 3198941c0b752..c2aeda9b2df62 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -75,6 +75,10 @@ def test_recordbatch_basics():
         ('c1', [-10, -5, 0, 5, 10])
     ])
 
+    with pytest.raises(IndexError):
+        # bounds checking
+        batch[2]
+
 
 def test_recordbatch_slice():
     data = [
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index a83f6f2b5eacd..c4958342a6f47 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -115,6 +115,7 @@ def test_tensor_size():
     tensor = pa.Tensor.from_numpy(data)
     assert pa.get_tensor_size(tensor) > (data.size * 8)
 
+
 def test_read_tensor(tmpdir):
     # Create and write tensor tensor
     data = np.random.randn(10, 4)

From dc4216f0216bb04bea86efe8b5d1f9158c6e698b Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 15 Jul 2017 16:58:03 -0400
Subject: [PATCH 0810/1644] ARROW-575: Python: Auto-detect nested lists and
 nested numpy arrays in Pandas

cc @fjetter

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #847 from xhochy/ARROW-575 and squashes the following commits:

9318b19e [Uwe L. Korn] ARROW-575: Python: Auto-detect nested lists and nested numpy arrays in Pandas
---
 cpp/src/arrow/python/pandas_convert.cc      | 149 ++++++++++++++------
 python/pyarrow/tests/test_convert_pandas.py |   6 +-
 2 files changed, 108 insertions(+), 47 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index cdd3f58e422ed..83cd35abf8c4b 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -261,8 +261,8 @@ struct WrapBytes<FixedSizeBinaryArray> {
   }
 };
 
-static inline bool ListTypeSupported(const Type::type type_id) {
-  switch (type_id) {
+static inline bool ListTypeSupported(const DataType& type) {
+  switch (type.id()) {
     case Type::UINT8:
     case Type::INT8:
     case Type::UINT16:
@@ -277,6 +277,10 @@ static inline bool ListTypeSupported(const Type::type type_id) {
     case Type::TIMESTAMP:
       // The above types are all supported.
       return true;
+    case Type::LIST: {
+      const ListType& list_type = static_cast<const ListType&>(type);
+      return ListTypeSupported(*list_type.value_type());
+    }
     default:
       break;
   }
@@ -396,7 +400,8 @@ class PandasConverter {
   // Conversion logic for various object dtype arrays
 
   template <int ITEM_TYPE, typename ArrowType>
-  Status ConvertTypedLists(const std::shared_ptr<DataType>& type);
+  Status ConvertTypedLists(
+      const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list);
 
   template <typename ArrowType>
   Status ConvertDates();
@@ -407,6 +412,8 @@ class PandasConverter {
   Status ConvertObjectFixedWidthBytes(const std::shared_ptr<DataType>& type);
   Status ConvertObjectIntegers();
   Status ConvertLists(const std::shared_ptr<DataType>& type);
+  Status ConvertLists(
+      const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list);
   Status ConvertObjects();
   Status ConvertDecimals();
 
@@ -905,8 +912,40 @@ Status PandasConverter::ConvertObjects() {
   return Status::OK();
 }
 
+template <typename T>
+Status LoopPySequence(PyObject* sequence, T func) {
+  if (PySequence_Check(sequence)) {
+    OwnedRef ref;
+    Py_ssize_t size = PySequence_Size(sequence);
+    if (PyArray_Check(sequence)) {
+      auto array = reinterpret_cast<PyArrayObject*>(sequence);
+      PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(array));
+      for (int64_t i = 0; i < size; ++i) {
+        RETURN_NOT_OK(func(objects[i]));
+      }
+    } else {
+      for (int64_t i = 0; i < size; ++i) {
+        ref.reset(PySequence_GetItem(sequence, i));
+        RETURN_NOT_OK(func(ref.obj()));
+      }
+    }
+  } else if (PyObject_HasAttrString(sequence, "__iter__")) {
+    OwnedRef iter = OwnedRef(PyObject_GetIter(sequence));
+    PyObject* item;
+    while ((item = PyIter_Next(iter.obj()))) {
+      OwnedRef ref = OwnedRef(item);
+      RETURN_NOT_OK(func(ref.obj()));
+    }
+  } else {
+    return Status::TypeError("Object is not a sequence or iterable");
+  }
+
+  return Status::OK();
+}
+
 template <int ITEM_TYPE, typename ArrowType>
-inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>& type) {
+inline Status PandasConverter::ConvertTypedLists(
+    const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list) {
   typedef npy_traits<ITEM_TYPE> traits;
   typedef typename traits::value_type T;
   typedef typename traits::BuilderClass BuilderT;
@@ -922,16 +961,13 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
     return Status::NotImplemented("strided arrays not implemented for lists");
   }
 
-  ListBuilder list_builder(
-      pool_, std::unique_ptr<ArrayBuilder>(new BuilderT(pool_, type)));
   BuilderT* value_builder = static_cast<BuilderT*>(list_builder.value_builder());
 
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  for (int64_t i = 0; i < length_; ++i) {
-    if (PandasObjectIsNull(objects[i])) {
-      RETURN_NOT_OK(list_builder.AppendNull());
-    } else if (PyArray_Check(objects[i])) {
-      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
+  auto foreach_item = [&](PyObject* object) {
+    if (PandasObjectIsNull(object)) {
+      return list_builder.AppendNull();
+    } else if (PyArray_Check(object)) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
       RETURN_NOT_OK(list_builder.Append(true));
 
       // TODO(uwe): Support more complex numpy array structures
@@ -945,32 +981,32 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
         // currently support this.
         // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
         ValuesToValidBytes<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        RETURN_NOT_OK(value_builder->Append(data, size, null_bitmap_->data()));
+        return value_builder->Append(data, size, null_bitmap_->data());
       } else {
-        RETURN_NOT_OK(value_builder->Append(data, size));
+        return value_builder->Append(data, size);
       }
-
-    } else if (PyList_Check(objects[i])) {
+    } else if (PyList_Check(object)) {
       int64_t size;
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(list_builder.Append(true));
-      RETURN_NOT_OK(InferArrowTypeAndSize(objects[i], &size, &inferred_type));
+      RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
       if (inferred_type->id() != type->id()) {
         std::stringstream ss;
         ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
         return Status::TypeError(ss.str());
       }
-      RETURN_NOT_OK(AppendPySequence(objects[i], size, type, value_builder));
+      return AppendPySequence(object, size, type, value_builder);
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }
-  }
-  return list_builder.Finish(&out_);
+  };
+
+  return LoopPySequence(list, foreach_item);
 }
 
 template <>
 inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
-    const std::shared_ptr<DataType>& type) {
+    const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list) {
   PyAcquireGIL lock;
   // TODO: If there are bytes involed, convert to Binary representation
   bool have_bytes = false;
@@ -984,47 +1020,45 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     return Status::NotImplemented("strided arrays not implemented for lists");
   }
 
-  ListBuilder list_builder(
-      pool_, std::unique_ptr<ArrayBuilder>(new StringBuilder(pool_)));
   auto value_builder = static_cast<StringBuilder*>(list_builder.value_builder());
 
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  for (int64_t i = 0; i < length_; ++i) {
-    if (PandasObjectIsNull(objects[i])) {
-      RETURN_NOT_OK(list_builder.AppendNull());
-    } else if (PyArray_Check(objects[i])) {
-      auto numpy_array = reinterpret_cast<PyArrayObject*>(objects[i]);
+  auto foreach_item = [&](PyObject* object) {
+    if (PandasObjectIsNull(object)) {
+      return list_builder.AppendNull();
+    } else if (PyArray_Check(object)) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
       RETURN_NOT_OK(list_builder.Append(true));
 
       // TODO(uwe): Support more complex numpy array structures
       RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
 
-      RETURN_NOT_OK(
-          AppendObjectStrings(numpy_array, nullptr, value_builder, &have_bytes));
-    } else if (PyList_Check(objects[i])) {
+      return AppendObjectStrings(numpy_array, nullptr, value_builder, &have_bytes);
+    } else if (PyList_Check(object)) {
       int64_t size;
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(list_builder.Append(true));
-      RETURN_NOT_OK(InferArrowTypeAndSize(objects[i], &size, &inferred_type));
+      RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
       if (inferred_type->id() != Type::STRING) {
         std::stringstream ss;
         ss << inferred_type->ToString() << " cannot be converted to STRING.";
         return Status::TypeError(ss.str());
       }
-      RETURN_NOT_OK(AppendPySequence(objects[i], size, inferred_type, value_builder));
+      return AppendPySequence(object, size, inferred_type, value_builder);
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }
-  }
-  return list_builder.Finish(&out_);
+  };
+
+  return LoopPySequence(list, foreach_item);
 }
 
-#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)             \
-  case Type::TYPE: {                                       \
-    return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type); \
+#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                                 \
+  case Type::TYPE: {                                                           \
+    return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type, list_builder, list); \
   }
 
-Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
+Status PandasConverter::ConvertLists(
+    const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list) {
   switch (type->id()) {
     LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
     LIST_CASE(INT8, NPY_INT8, Int8Type)
@@ -1038,14 +1072,36 @@ Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
     LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
     LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
     LIST_CASE(STRING, NPY_OBJECT, StringType)
-    default:
+    case Type::LIST: {
+      const ListType& list_type = static_cast<const ListType&>(*type);
+      auto value_builder = static_cast<ListBuilder*>(list_builder.value_builder());
+
+      auto foreach_item = [&](PyObject* object) {
+        if (PandasObjectIsNull(object)) {
+          return list_builder.AppendNull();
+        } else {
+          RETURN_NOT_OK(list_builder.Append(true));
+          return ConvertLists(list_type.value_type(), *value_builder, object);
+        }
+      };
+
+      return LoopPySequence(list, foreach_item);
+    }
+    default: {
       std::stringstream ss;
       ss << "Unknown list item type: ";
       ss << type->ToString();
       return Status::TypeError(ss.str());
+    }
   }
+}
 
-  return Status::TypeError("Unknown list type");
+Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
+  std::unique_ptr<ArrayBuilder> array_builder;
+  RETURN_NOT_OK(MakeBuilder(pool_, arrow::list(type), &array_builder));
+  ListBuilder& list_builder = static_cast<ListBuilder&>(*array_builder);
+  RETURN_NOT_OK(ConvertLists(type, list_builder, reinterpret_cast<PyObject*>(arr_)));
+  return list_builder.Finish(&out_);
 }
 
 Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
@@ -1561,9 +1617,11 @@ class ObjectBlock : public PandasBlock {
         CONVERTLISTSLIKE_CASE(FloatType, FLOAT)
         CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE)
         CONVERTLISTSLIKE_CASE(StringType, STRING)
+        CONVERTLISTSLIKE_CASE(ListType, LIST)
         default: {
           std::stringstream ss;
-          ss << "Not implemented type for lists: " << list_type->value_type()->ToString();
+          ss << "Not implemented type for conversion from List to Pandas ObjectBlock: "
+             << list_type->value_type()->ToString();
           return Status::NotImplemented(ss.str());
         }
       }
@@ -2015,9 +2073,9 @@ class DataFrameBlockCreator {
         } break;
         case Type::LIST: {
           auto list_type = std::static_pointer_cast<ListType>(col->type());
-          if (!ListTypeSupported(list_type->value_type()->id())) {
+          if (!ListTypeSupported(*list_type->value_type())) {
             std::stringstream ss;
-            ss << "Not implemented type for lists: "
+            ss << "Not implemented type for list in DataFrameBlock: "
                << list_type->value_type()->ToString();
             return Status::NotImplemented(ss.str());
           }
@@ -2386,6 +2444,7 @@ class ArrowDeserializer {
       CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE)
       CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING)
       CONVERTVALUES_LISTSLIKE_CASE(DecimalType, DECIMAL)
+      CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST)
       default: {
         std::stringstream ss;
         ss << "Not implemented type for lists: " << list_type->value_type()->ToString();
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 4c51d71f1f202..b8b85ca7632a1 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -715,14 +715,16 @@ def test_infer_lists(self):
         data = OrderedDict([
             ('nan_ints', [[None, 1], [2, 3]]),
             ('ints', [[0, 1], [2, 3]]),
-            ('strs', [[None, u'b'], [u'c', u'd']])
+            ('strs', [[None, u'b'], [u'c', u'd']]),
+            ('nested_strs', [[[None, u'b'], [u'c', u'd']], None])
         ])
         df = pd.DataFrame(data)
 
         expected_schema = pa.schema([
             pa.field('nan_ints', pa.list_(pa.int64())),
             pa.field('ints', pa.list_(pa.int64())),
-            pa.field('strs', pa.list_(pa.string()))
+            pa.field('strs', pa.list_(pa.string())),
+            pa.field('nested_strs', pa.list_(pa.list_(pa.string())))
         ])
 
         self._check_pandas_roundtrip(df, expected_schema=expected_schema)

From e438e15172b94b5bad80ba39d838195b4748d7c3 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 15 Jul 2017 23:46:31 -0400
Subject: [PATCH 0811/1644] ARROW-1217: [GLib] Add GInputStream based
 arrow::io::RandomAccessFile

`GInputStream` provides many input types such as local file, in memory data, socket including TLS support. It's useful to read data from many data sources.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #845 from kou/glib-support-ginput-stream and squashes the following commits:

59247ba7 [Kouhei Sutou] [GLib] Disable GSettings related codes in Go example
268a5fb3 [Kouhei Sutou] [GLib] Remove needless gdk-3.0 dependency in Go example
9e54e7b6 [Kouhei Sutou] [GLib] Update dependency in Go example
ec2ddaca [Kouhei Sutou] [GLib] Add GInputStream based arrow::io::RandomAccessFile
---
 c_glib/arrow-glib/Makefile.am           |   7 +-
 c_glib/arrow-glib/input-stream.cpp      | 207 ++++++++++++++++++++++++
 c_glib/arrow-glib/input-stream.h        |  52 ++++++
 c_glib/configure.ac                     |   2 +-
 c_glib/example/go/arrow-1.0/arrow.go.in |   1 +
 c_glib/test/run-test.rb                 |   1 +
 c_glib/test/test-gio-input-stream.rb    |  53 ++++++
 ci/travis_before_script_c_glib.sh       |   6 +
 8 files changed, 326 insertions(+), 3 deletions(-)
 create mode 100644 c_glib/test/test-gio-input-stream.rb

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index f0bb6011b65c7..f33a8d197875c 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -194,9 +194,12 @@ INTROSPECTION_COMPILER_ARGS =
 if HAVE_INTROSPECTION
 Arrow-1.0.gir: libarrow-glib.la
 Arrow_1_0_gir_PACKAGES =			\
-	gobject-2.0
+	gobject-2.0				\
+	gio-2.0
 Arrow_1_0_gir_EXPORT_PACKAGES = arrow
-Arrow_1_0_gir_INCLUDES = GObject-2.0
+Arrow_1_0_gir_INCLUDES =			\
+	GObject-2.0				\
+	Gio-2.0
 Arrow_1_0_gir_CFLAGS =				\
 	$(AM_CPPFLAGS)
 Arrow_1_0_gir_LIBS = libarrow-glib.la
diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index d81e4a32d5db4..37ac2fad4eca1 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -32,6 +32,9 @@
 #include <arrow-glib/readable.hpp>
 #include <arrow-glib/tensor.hpp>
 
+#include <iostream>
+#include <sstream>
+
 G_BEGIN_DECLS
 
 /**
@@ -373,6 +376,210 @@ garrow_memory_mapped_input_stream_new(const gchar *path,
 }
 
 
+G_END_DECLS
+
+namespace garrow {
+  class GIOInputStream : public arrow::io::RandomAccessFile {
+  public:
+    GIOInputStream(GInputStream *input_stream) :
+      input_stream_(input_stream) {
+      g_object_ref(input_stream_);
+    }
+
+    ~GIOInputStream() {
+      g_object_unref(input_stream_);
+    }
+
+    GInputStream *get_input_stream() {
+      return input_stream_;
+    }
+
+    arrow::Status Close() override {
+      GError *error = NULL;
+      if (g_input_stream_close(input_stream_, NULL, &error)) {
+        return arrow::Status::OK();
+      } else {
+        return io_error_to_status(error, "[gio-input-stream][close]");
+      }
+    }
+
+    arrow::Status Tell(int64_t *position) override {
+      if (!G_IS_SEEKABLE(input_stream_)) {
+        std::string message("[gio-input-stream][tell] "
+                            "not seekable input stream: <");
+        message += G_OBJECT_CLASS_NAME(G_OBJECT_GET_CLASS(input_stream_));
+        message += ">";
+        return arrow::Status::NotImplemented(message);
+      }
+
+      *position = g_seekable_tell(G_SEEKABLE(input_stream_));
+      return arrow::Status::OK();
+    }
+
+    arrow::Status Read(int64_t n_bytes,
+                       int64_t *n_read_bytes,
+                       uint8_t *out) override {
+      GError *error = NULL;
+      *n_read_bytes = g_input_stream_read(input_stream_,
+                                          out,
+                                          n_bytes,
+                                          NULL,
+                                          &error);
+      if (*n_read_bytes == -1) {
+        return io_error_to_status(error, "[gio-input-stream][read]");
+      } else {
+        return arrow::Status::OK();
+      }
+    }
+
+    arrow::Status Read(int64_t n_bytes,
+                       std::shared_ptr<arrow::Buffer> *out) override {
+      arrow::MemoryPool *pool = arrow::default_memory_pool();
+      std::shared_ptr<arrow::ResizableBuffer> buffer;
+      ARROW_RETURN_NOT_OK(AllocateResizableBuffer(pool, n_bytes, &buffer));
+
+      GError *error = NULL;
+      auto n_read_bytes = g_input_stream_read(input_stream_,
+                                              buffer->mutable_data(),
+                                              n_bytes,
+                                              NULL,
+                                              &error);
+      if (n_read_bytes == -1) {
+        return io_error_to_status(error, "[gio-input-stream][read][buffer]");
+      } else {
+        if (n_read_bytes < n_bytes) {
+          ARROW_RETURN_NOT_OK(buffer->Resize(n_read_bytes));
+        }
+        *out = buffer;
+        return arrow::Status::OK();
+      }
+    }
+
+    arrow::Status Seek(int64_t position) override {
+      if (!G_IS_SEEKABLE(input_stream_)) {
+        std::string message("[gio-input-stream][seek] "
+                            "not seekable input stream: <");
+        message += G_OBJECT_CLASS_NAME(G_OBJECT_GET_CLASS(input_stream_));
+        message += ">";
+        return arrow::Status::NotImplemented(message);
+      }
+
+      GError *error = NULL;
+      if (g_seekable_seek(G_SEEKABLE(input_stream_),
+                          position,
+                          G_SEEK_SET,
+                          NULL,
+                          &error)) {
+        return arrow::Status::OK();
+      } else {
+        return io_error_to_status(error, "[gio-input-stream][seek]");
+      }
+    }
+
+    arrow::Status GetSize(int64_t *size) override {
+      if (!G_IS_SEEKABLE(input_stream_)) {
+        std::string message("[gio-input-stream][size] "
+                            "not seekable input stream: <");
+        message += G_OBJECT_CLASS_NAME(G_OBJECT_GET_CLASS(input_stream_));
+        message += ">";
+        return arrow::Status::NotImplemented(message);
+      }
+
+      auto current_position = g_seekable_tell(G_SEEKABLE(input_stream_));
+      GError *error = NULL;
+      if (!g_seekable_seek(G_SEEKABLE(input_stream_),
+                           0,
+                           G_SEEK_END,
+                           NULL,
+                           &error)) {
+        return io_error_to_status(error, "[gio-input-stream][size][seek]");
+      }
+      *size = g_seekable_tell(G_SEEKABLE(input_stream_));
+      if (!g_seekable_seek(G_SEEKABLE(input_stream_),
+                           current_position,
+                           G_SEEK_SET,
+                           NULL,
+                           &error)) {
+        return io_error_to_status(error,
+                                  "[gio-input-stream][size][seek][restore]");
+      }
+      return arrow::Status::OK();
+    }
+
+    bool supports_zero_copy() const override {
+      return false;
+    }
+
+  private:
+    GInputStream *input_stream_;
+
+    arrow::Status io_error_to_status(GError *error, const char *context) {
+      std::stringstream message;
+      message << context << ": " << g_quark_to_string(error->domain);
+      message << "(" << error->code << "): ";
+      message << error->message;
+      g_error_free(error);
+      return arrow::Status::IOError(message.str());
+    }
+  };
+};
+
+G_BEGIN_DECLS
+
+G_DEFINE_TYPE(GArrowGIOInputStream,                \
+              garrow_gio_input_stream,             \
+              GARROW_TYPE_SEEKABLE_INPUT_STREAM);
+
+static void
+garrow_gio_input_stream_init(GArrowGIOInputStream *object)
+{
+}
+
+static void
+garrow_gio_input_stream_class_init(GArrowGIOInputStreamClass *klass)
+{
+}
+
+/**
+ * garrow_gio_input_stream_new:
+ * @gio_input_stream: The stream to be read.
+ *
+ * Returns: A newly created #GArrowGIOInputStream.
+ *
+ * Since: 0.5.0
+ */
+GArrowGIOInputStream *
+garrow_gio_input_stream_new(GInputStream *gio_input_stream)
+{
+  auto arrow_input_stream =
+    std::make_shared<garrow::GIOInputStream>(gio_input_stream);
+  auto object = g_object_new(GARROW_TYPE_GIO_INPUT_STREAM,
+                             "input-stream", &arrow_input_stream,
+                             NULL);
+  auto input_stream = GARROW_GIO_INPUT_STREAM(object);
+  return input_stream;
+}
+
+/**
+ * garrow_gio_input_stream_get_gio_input_stream:
+ * @input_stream: A #GArrowGIOInputStream.
+ *
+ * Returns: (transfer none): The wrapped #GInputStream.
+ *
+ * Since: 0.5.0
+ */
+GInputStream *
+garrow_gio_input_stream_get_gio_input_stream(GArrowGIOInputStream *input_stream)
+{
+  auto arrow_input_stream =
+    garrow_input_stream_get_raw(GARROW_INPUT_STREAM(input_stream));
+  auto arrow_gio_input_stream =
+    std::static_pointer_cast<garrow::GIOInputStream>(arrow_input_stream);
+  auto gio_input_stream = arrow_gio_input_stream->get_input_stream();
+  return gio_input_stream;
+}
+
+
 G_END_DECLS
 
 GArrowInputStream *
diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h
index 8a4d362189f7d..120839d9fcade 100644
--- a/c_glib/arrow-glib/input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include <gio/gio.h>
+
 #include <arrow-glib/buffer.h>
 #include <arrow-glib/tensor.h>
 
@@ -229,4 +231,54 @@ GType garrow_memory_mapped_input_stream_get_type(void) G_GNUC_CONST;
 GArrowMemoryMappedInputStream *garrow_memory_mapped_input_stream_new(const gchar *path,
                                                                      GError **error);
 
+
+#define GARROW_TYPE_GIO_INPUT_STREAM            \
+  (garrow_gio_input_stream_get_type())
+#define GARROW_GIO_INPUT_STREAM(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_GIO_INPUT_STREAM,     \
+                              GArrowGIOInputStream))
+#define GARROW_GIO_INPUT_STREAM_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_GIO_INPUT_STREAM,        \
+                           GArrowGIOInputStreamClass))
+#define GARROW_IS_GIO_INPUT_STREAM(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_GIO_INPUT_STREAM))
+#define GARROW_IS_GIO_INPUT_STREAM_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_GIO_INPUT_STREAM))
+#define GARROW_GIO_INPUT_STREAM_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_GIO_INPUT_STREAM,      \
+                             GArrowGIOInputStreamClass))
+
+typedef struct _GArrowGIOInputStream         GArrowGIOInputStream;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowGIOInputStreamClass    GArrowGIOInputStreamClass;
+#endif
+
+/**
+ * GArrowGIOInputStream:
+ *
+ * It's an input stream for `GInputStream`.
+ */
+struct _GArrowGIOInputStream
+{
+  /*< private >*/
+  GArrowSeekableInputStream parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowGIOInputStreamClass
+{
+  GArrowSeekableInputStreamClass parent_class;
+};
+#endif
+
+GType garrow_gio_input_stream_get_type(void) G_GNUC_CONST;
+
+GArrowGIOInputStream *garrow_gio_input_stream_new(GInputStream *gio_input_stream);
+GInputStream *garrow_gio_input_stream_get_gio_input_stream(GArrowGIOInputStream *input_stream);
+
 G_END_DECLS
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index e010d962f377a..d4e828ba55c1b 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -56,7 +56,7 @@ fi
 AC_SUBST(GARROW_CFLAGS)
 AC_SUBST(GARROW_CXXFLAGS)
 
-AM_PATH_GLIB_2_0([2.32.4], [], [], [gobject])
+AM_PATH_GLIB_2_0([2.32.4], [], [], [gobject gio])
 
 GOBJECT_INTROSPECTION_REQUIRE([1.32.1])
 GTK_DOC_CHECK([1.18-2])
diff --git a/c_glib/example/go/arrow-1.0/arrow.go.in b/c_glib/example/go/arrow-1.0/arrow.go.in
index bd124afd596b2..08ee13d25d0d5 100644
--- a/c_glib/example/go/arrow-1.0/arrow.go.in
+++ b/c_glib/example/go/arrow-1.0/arrow.go.in
@@ -31,6 +31,7 @@ import "unsafe"
 import (
 	"gir/glib-2.0"
 	"gir/gobject-2.0"
+	"gir/gio-2.0"
 )
 
 [<.go_utils>]
diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb
index 43bb22029d2a0..75ff34ff47281 100755
--- a/c_glib/test/run-test.rb
+++ b/c_glib/test/run-test.rb
@@ -31,6 +31,7 @@
 
 require "gi"
 
+Gio = GI.load("Gio")
 Arrow = GI.load("Arrow")
 module Arrow
   class Buffer
diff --git a/c_glib/test/test-gio-input-stream.rb b/c_glib/test/test-gio-input-stream.rb
new file mode 100644
index 0000000000000..baa978e10f754
--- /dev/null
+++ b/c_glib/test/test-gio-input-stream.rb
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestGIOInputStream < Test::Unit::TestCase
+  def test_reader_backend
+    tempfile = Tempfile.open("arrow-gio-input-stream")
+    output = Arrow::FileOutputStream.new(tempfile.path, false)
+    begin
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      schema = Arrow::Schema.new([field])
+      file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
+      begin
+        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        file_writer.write_record_batch(record_batch)
+      ensure
+        file_writer.close
+      end
+    ensure
+      output.close
+    end
+
+    file = Gio::File.new_for_path(tempfile.path)
+    input_stream = file.read
+    input = Arrow::GIOInputStream.new(input_stream)
+    begin
+      file_reader = Arrow::RecordBatchFileReader.new(input)
+      assert_equal(["enabled"],
+                   file_reader.schema.fields.collect(&:name))
+    ensure
+      input.close
+    end
+  end
+
+  def test_getter
+    input_stream = Gio::MemoryInputStream.new("Hello")
+    input = Arrow::GIOInputStream.new(input_stream)
+    assert_equal(input_stream, input.gio_input_stream)
+  end
+end
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 6387f4ddcc902..6547ea4e53709 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -44,6 +44,12 @@ luarocks install lgi
 
 go get github.com/linuxdeepin/go-gir-generator || :
 pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
+rm lib.in/gio-2.0/gdk_workaround.go
+mv lib.in/gio-2.0/config.json{,.orig}
+sed \
+  -e 's/\("Settings",\)/\/\/ \1/g' \
+  -e 's/\("SettingsBackend",\)/\/\/ \1/g' \
+  lib.in/gio-2.0/config.json.orig > lib.in/gio-2.0/config.json
 mv Makefile{,.orig}
 sed -e 's/ gudev-1.0//' Makefile.orig > Makefile
 mkdir -p out/src/gir/gudev-1.0

From f266f17e023c7ea8391a2d33c2a8f9e5593e3a19 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sun, 16 Jul 2017 14:40:19 -0400
Subject: [PATCH 0812/1644] =?UTF-8?q?ARROW-1220:=20[C++]=20Cmake=20script?=
 =?UTF-8?q?=20errors=20out=20if=20lib=20is=20not=20found=20under=20*?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…_HOME directory

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #850 from MaxRis/ARROW-1220 and squashes the following commits:

ef855c9e [Max Risuhin] ARROW-1220: [C++] Cmake script errors out if lib is not found under *_HOME directory
---
 appveyor.yml                                |  22 +-
 ci/msvc-build.bat                           |  16 +-
 ci/test-cmake-build-script.bat              | 113 ++++++++
 cpp/CMakeLists.txt                          |   3 -
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 276 ++++++++++----------
 5 files changed, 276 insertions(+), 154 deletions(-)
 create mode 100644 ci/test-cmake-build-script.bat

diff --git a/appveyor.yml b/appveyor.yml
index f3ce5a544375f..c58e1dab7d8fd 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -20,22 +20,31 @@ os: Visual Studio 2015
 
 environment:
   matrix:
-    - GENERATOR: NMake Makefiles
+    - JOB: "Cmake_Script_Tests"
+      GENERATOR: NMake Makefiles
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Release"
-    - GENERATOR: Visual Studio 14 2015 Win64
+    - JOB: "Build"
+      GENERATOR: NMake Makefiles
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Release"
+    - JOB: "Build_Debug"
+      GENERATOR: Visual Studio 14 2015 Win64
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Debug"
-    - GENERATOR: Visual Studio 14 2015 Win64
+    - JOB: "Build"
+      GENERATOR: Visual Studio 14 2015 Win64
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Release"
-    - GENERATOR: Visual Studio 14 2015 Win64
+    - JOB: "Toolchain"
+      GENERATOR: Visual Studio 14 2015 Win64
       PYTHON: "3.5"
       ARCH: "64"
-      CONFIGURATION: "Toolchain"
+      CONFIGURATION: "Release"
 
   MSVC_DEFAULT_OPTIONS: ON
   BOOST_ROOT: C:\Libraries\boost_1_63_0
@@ -49,4 +58,5 @@ init:
 build_script:
   - git config core.symlinks true
   - git reset --hard
-  - call ci\msvc-build.bat
+  - if "%JOB%"=="Cmake_Script_Tests" call ci\test-cmake-build-script.bat
+  - if NOT "%JOB%"=="Cmake_Script_Tests" call ci\msvc-build.bat
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index c84d12d97d7d8..be5fd7b35f468 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -17,13 +17,13 @@
 
 @echo on
 
-if "%CONFIGURATION%" == "Debug" (
+if "%JOB%" == "Build_Debug" (
   mkdir cpp\build-debug
   pushd cpp\build-debug
 
   cmake -G "%GENERATOR%" ^
         -DARROW_BOOST_USE_SHARED=OFF ^
-        -DCMAKE_BUILD_TYPE=Debug ^
+        -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
         -DARROW_CXXFLAGS="/MP" ^
         ..  || exit /B
 
@@ -39,7 +39,7 @@ conda update --yes --quiet conda
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython
 
-if "%CONFIGURATION%" == "Toolchain" (
+if "%JOB%" == "Toolchain" (
   conda install -n arrow -q -y -c conda-forge ^
       flatbuffers rapidjson cmake git boost-cpp ^
       thrift-cpp snappy zlib brotli gflags lz4-c zstd
@@ -47,7 +47,7 @@ if "%CONFIGURATION%" == "Toolchain" (
 
 call activate arrow
 
-if "%CONFIGURATION%" == "Toolchain" (
+if "%JOB%" == "Toolchain" (
   set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 )
 
@@ -61,11 +61,11 @@ pushd cpp\build
 cmake -G "%GENERATOR%" ^
       -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
       -DARROW_BOOST_USE_SHARED=OFF ^
-      -DCMAKE_BUILD_TYPE=Release ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
       -DARROW_CXXFLAGS="/WX /MP" ^
       -DARROW_PYTHON=ON ^
       ..  || exit /B
-cmake --build . --target INSTALL --config Release  || exit /B
+cmake --build . --target INSTALL --config %CONFIGURATION%  || exit /B
 
 @rem Needed so python-test.exe works
 set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
@@ -83,11 +83,11 @@ set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 set PARQUET_HOME=%CONDA_PREFIX%\Library
 cmake -G "%GENERATOR%" ^
      -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
-     -DCMAKE_BUILD_TYPE=Release ^
+     -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
      -DPARQUET_BOOST_USE_SHARED=OFF ^
      -DPARQUET_ZLIB_VENDORED=off ^
      -DPARQUET_BUILD_TESTS=off .. || exit /B
-cmake --build . --target INSTALL --config Release || exit /B
+cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
 popd
 
 @rem Build and import pyarrow
diff --git a/ci/test-cmake-build-script.bat b/ci/test-cmake-build-script.bat
new file mode 100644
index 0000000000000..e37a8117739f1
--- /dev/null
+++ b/ci/test-cmake-build-script.bat
@@ -0,0 +1,113 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+conda update --yes --quiet conda
+conda create -n arrow-cmake-script-tests -q -y
+conda install -n arrow-cmake-script-tests -q -y -c conda-forge ^
+cmake git boost-cpp
+call activate arrow-cmake-script-tests
+
+mkdir cpp\build-cmake-test
+pushd cpp\build-cmake-test
+
+echo Test cmake script errors out on flatbuffers missed
+set FLATBUFFERS_HOME=WrongPath
+
+cmake -G "%GENERATOR%" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/MP" ^
+      .. >nul 2>error.txt
+
+FINDSTR /M /C:"Could not find the Flatbuffers library" error.txt || exit /B
+set FLATBUFFERS_HOME=
+
+echo Test cmake script errors out on gflags missed
+set GFLAGS_HOME=WrongPath
+
+cmake -G "%GENERATOR%" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/MP" ^
+      .. >nul 2>error.txt
+
+FINDSTR /M /C:"No static or shared library provided for gflags" error.txt || exit /B
+set GFLAGS_HOME=
+
+echo Test cmake script errors out on snappy missed
+set SNAPPY_HOME=WrongPath
+
+cmake -G "%GENERATOR%" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/MP" ^
+      .. >nul 2>error.txt
+
+FINDSTR /M /C:"Could not find the Snappy library" error.txt || exit /B
+set SNAPPY_HOME=
+
+echo Test cmake script errors out on zlib missed
+set ZLIB_HOME=WrongPath
+
+cmake -G "%GENERATOR%" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/MP" ^
+      .. >nul 2>error.txt
+
+FINDSTR /M /C:"Could not find the ZLIB library" error.txt || exit /B
+set ZLIB_HOME=
+
+echo Test cmake script errors out on brotli missed
+set BROTLI_HOME=WrongPath
+
+cmake -G "%GENERATOR%" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/MP" ^
+      .. >nul 2>error.txt
+
+FINDSTR /M /C:"Could not find the Brotli library" error.txt || exit /B
+set BROTLI_HOME=
+
+echo Test cmake script errors out on lz4 missed
+set LZ4_HOME=WrongPath
+
+cmake -G "%GENERATOR%" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/MP" ^
+      .. >nul 2>error.txt
+
+FINDSTR /M /C:"No static or shared library provided for lz4_static" error.txt || exit /B
+set LZ4_HOME=
+
+echo Test cmake script errors out on zstd missed
+set ZSTD_HOME=WrongPath
+
+cmake -G "%GENERATOR%" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/MP" ^
+      .. >nul 2>error.txt
+
+FINDSTR /M /C:"Could NOT find ZSTD" error.txt || exit /B
+set ZSTD_HOME=
+
+popd
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index c6098502e3621..3146b87b678e0 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -162,9 +162,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build with zstd compression"
     ON)
 
-  option(ARROW_ZLIB_VENDORED
-    "Build our own zlib (some libz.a aren't configured for static linking)"
-    ON)
   if (MSVC)
     set(BROTLI_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Brotli static lib suffix used on Windows with MSVC (default _static)")
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index ab8fc5df8d27b..b9d9823e80ce3 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -450,177 +450,178 @@ message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
 
 include_directories(SYSTEM "${HADOOP_HOME}/include")
 
+if (ARROW_WITH_ZLIB)
 # ----------------------------------------------------------------------
 # ZLIB
 
-if (NOT ARROW_ZLIB_VENDORED)
-  find_package(ZLIB)
-endif()
-
-if (NOT ZLIB_FOUND)
-  set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install")
-  set(ZLIB_HOME "${ZLIB_PREFIX}")
-  set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include")
-  if (MSVC)
-    if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
-      set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib)
+  if("${ZLIB_HOME}" STREQUAL "")
+    set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install")
+    set(ZLIB_HOME "${ZLIB_PREFIX}")
+    set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include")
+    if (MSVC)
+      if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+        set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib)
+      else()
+        set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
+      endif()
     else()
-      set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
+      set(ZLIB_STATIC_LIB_NAME libz.a)
     endif()
+    set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}")
+    set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                        -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}
+                        -DCMAKE_C_FLAGS=${EP_C_FLAGS}
+                        -DBUILD_SHARED_LIBS=OFF)
+
+    ExternalProject_Add(zlib_ep
+      URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz"
+      BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}"
+      CMAKE_ARGS ${ZLIB_CMAKE_ARGS})
+    set(ZLIB_VENDORED 1)
   else()
-    set(ZLIB_STATIC_LIB_NAME libz.a)
+    find_package(ZLIB REQUIRED)
+    set(ZLIB_VENDORED 0)
   endif()
-  set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}")
-  set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-                      -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}
-                      -DCMAKE_C_FLAGS=${EP_C_FLAGS}
-                      -DBUILD_SHARED_LIBS=OFF)
-
-  ExternalProject_Add(zlib_ep
-    URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz"
-    BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}"
-    CMAKE_ARGS ${ZLIB_CMAKE_ARGS})
-  set(ZLIB_VENDORED 1)
-else()
-  set(ZLIB_VENDORED 0)
-endif()
 
-include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
-ADD_THIRDPARTY_LIB(zlib
-  STATIC_LIB ${ZLIB_STATIC_LIB})
+  include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(zlib
+    STATIC_LIB ${ZLIB_STATIC_LIB})
 
-if (ZLIB_VENDORED)
-  add_dependencies(zlib zlib_ep)
+  if (ZLIB_VENDORED)
+    add_dependencies(zlib zlib_ep)
+  endif()
 endif()
 
+if (ARROW_WITH_SNAPPY)
 # ----------------------------------------------------------------------
 # Snappy
 
-## Snappy
-find_package(Snappy)
-if (NOT SNAPPY_FOUND)
-  set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep/src/snappy_ep-install")
-  set(SNAPPY_HOME "${SNAPPY_PREFIX}")
-  set(SNAPPY_INCLUDE_DIR "${SNAPPY_PREFIX}/include")
-  if (MSVC)
-    set(SNAPPY_STATIC_LIB_NAME snappystatic)
-  else()
-    set(SNAPPY_STATIC_LIB_NAME snappy)
-  endif()
-  set(SNAPPY_STATIC_LIB "${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  set(SNAPPY_SRC_URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz")
-
-  if (${UPPERCASE_BUILD_TYPE} EQUAL "RELEASE")
-    if (APPLE)
-      set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O1'")
+  if("${SNAPPY_HOME}" STREQUAL "")
+    set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep/src/snappy_ep-install")
+    set(SNAPPY_HOME "${SNAPPY_PREFIX}")
+    set(SNAPPY_INCLUDE_DIR "${SNAPPY_PREFIX}/include")
+    if (MSVC)
+      set(SNAPPY_STATIC_LIB_NAME snappystatic)
     else()
-      set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O2'")
+      set(SNAPPY_STATIC_LIB_NAME snappy)
+    endif()
+    set(SNAPPY_STATIC_LIB "${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(SNAPPY_SRC_URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz")
+
+    if (${UPPERCASE_BUILD_TYPE} EQUAL "RELEASE")
+      if (APPLE)
+        set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O1'")
+      else()
+        set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O2'")
+      endif()
     endif()
-  endif()
 
-  if (MSVC)
-    set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-                          "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
-                          "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
-                          "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}")
-    set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy
-                      ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt
-                      ./CMakeLists.txt &&
-                      ${CMAKE_COMMAND} -E copy
-                      ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyConfig.h
-                      ./config.h)
-    ExternalProject_Add(snappy_ep
-      UPDATE_COMMAND ${SNAPPY_UPDATE_COMMAND}
-      BUILD_IN_SOURCE 1
-      BUILD_COMMAND ${MAKE}
-      INSTALL_DIR ${SNAPPY_PREFIX}
-      URL ${SNAPPY_SRC_URL}
-      CMAKE_ARGS ${SNAPPY_CMAKE_ARGS}
-      BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}")
+    if (MSVC)
+      set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                            "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
+                            "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                            "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}")
+      set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy
+                        ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt
+                        ./CMakeLists.txt &&
+                        ${CMAKE_COMMAND} -E copy
+                        ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyConfig.h
+                        ./config.h)
+      ExternalProject_Add(snappy_ep
+        UPDATE_COMMAND ${SNAPPY_UPDATE_COMMAND}
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND ${MAKE}
+        INSTALL_DIR ${SNAPPY_PREFIX}
+        URL ${SNAPPY_SRC_URL}
+        CMAKE_ARGS ${SNAPPY_CMAKE_ARGS}
+        BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}")
+    else()
+      ExternalProject_Add(snappy_ep
+        CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS}
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND ${MAKE}
+        INSTALL_DIR ${SNAPPY_PREFIX}
+        URL ${SNAPPY_SRC_URL}
+        BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}")
+    endif()
+    set(SNAPPY_VENDORED 1)
   else()
-    ExternalProject_Add(snappy_ep
-      CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS}
-      BUILD_IN_SOURCE 1
-      BUILD_COMMAND ${MAKE}
-      INSTALL_DIR ${SNAPPY_PREFIX}
-      URL ${SNAPPY_SRC_URL}
-      BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}")
+    find_package(Snappy REQUIRED)
+    set(SNAPPY_VENDORED 0)
   endif()
-  set(SNAPPY_VENDORED 1)
-else()
-  set(SNAPPY_VENDORED 0)
-endif()
 
-include_directories(SYSTEM ${SNAPPY_INCLUDE_DIR})
-ADD_THIRDPARTY_LIB(snappy
-  STATIC_LIB ${SNAPPY_STATIC_LIB})
+  include_directories(SYSTEM ${SNAPPY_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(snappy
+    STATIC_LIB ${SNAPPY_STATIC_LIB})
 
-if (SNAPPY_VENDORED)
-  add_dependencies(snappy snappy_ep)
+  if (SNAPPY_VENDORED)
+    add_dependencies(snappy snappy_ep)
+  endif()
 endif()
 
+if (ARROW_WITH_BROTLI)
 # ----------------------------------------------------------------------
 # Brotli
 
-find_package(Brotli)
-if (NOT BROTLI_FOUND)
-  set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install")
-  set(BROTLI_HOME "${BROTLI_PREFIX}")
-  set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include")
-  if (MSVC)
-    set(BROTLI_LIB_DIR bin)
+  if("${BROTLI_HOME}" STREQUAL "")
+    set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install")
+    set(BROTLI_HOME "${BROTLI_PREFIX}")
+    set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include")
+    if (MSVC)
+      set(BROTLI_LIB_DIR bin)
+    else()
+      set(BROTLI_LIB_DIR lib)
+    endif()
+    set(BROTLI_STATIC_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(BROTLI_STATIC_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                          "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
+                          "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                          -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}
+                          -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE}
+                          -DBUILD_SHARED_LIBS=OFF)
+
+    ExternalProject_Add(brotli_ep
+      URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz"
+      BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}"
+      ${BROTLI_BUILD_BYPRODUCTS}
+      CMAKE_ARGS ${BROTLI_CMAKE_ARGS}
+      STEP_TARGETS headers_copy)
+    if (MSVC)
+      ExternalProject_Get_Property(brotli_ep SOURCE_DIR)
+
+      ExternalProject_Add_Step(brotli_ep headers_copy
+        COMMAND xcopy /E /I include ..\\..\\..\\brotli_ep\\src\\brotli_ep-install\\include /Y
+        DEPENDEES build
+        WORKING_DIRECTORY ${SOURCE_DIR})
+    endif()
+    set(BROTLI_VENDORED 1)
   else()
-    set(BROTLI_LIB_DIR lib)
+    find_package(Brotli REQUIRED)
+    set(BROTLI_VENDORED 0)
   endif()
-  set(BROTLI_STATIC_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  set(BROTLI_STATIC_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-                        "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
-                        "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
-                        -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}
-                        -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE}
-                        -DBUILD_SHARED_LIBS=OFF)
 
-  ExternalProject_Add(brotli_ep
-    URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz"
-    BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}"
-    ${BROTLI_BUILD_BYPRODUCTS}
-    CMAKE_ARGS ${BROTLI_CMAKE_ARGS}
-    STEP_TARGETS headers_copy)
-  if (MSVC)
-    ExternalProject_Get_Property(brotli_ep SOURCE_DIR)
-
-    ExternalProject_Add_Step(brotli_ep headers_copy
-      COMMAND xcopy /E /I include ..\\..\\..\\brotli_ep\\src\\brotli_ep-install\\include /Y
-      DEPENDEES build
-      WORKING_DIRECTORY ${SOURCE_DIR})
+  include_directories(SYSTEM ${BROTLI_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(brotli_enc
+    STATIC_LIB ${BROTLI_STATIC_LIBRARY_ENC})
+  ADD_THIRDPARTY_LIB(brotli_dec
+    STATIC_LIB ${BROTLI_STATIC_LIBRARY_DEC})
+  ADD_THIRDPARTY_LIB(brotli_common
+    STATIC_LIB ${BROTLI_STATIC_LIBRARY_COMMON})
+
+  if (BROTLI_VENDORED)
+    add_dependencies(brotli_enc brotli_ep)
+    add_dependencies(brotli_dec brotli_ep)
+    add_dependencies(brotli_common brotli_ep)
   endif()
-  set(BROTLI_VENDORED 1)
-else()
-  set(BROTLI_VENDORED 0)
-endif()
-
-include_directories(SYSTEM ${BROTLI_INCLUDE_DIR})
-ADD_THIRDPARTY_LIB(brotli_enc
-  STATIC_LIB ${BROTLI_STATIC_LIBRARY_ENC})
-ADD_THIRDPARTY_LIB(brotli_dec
-  STATIC_LIB ${BROTLI_STATIC_LIBRARY_DEC})
-ADD_THIRDPARTY_LIB(brotli_common
-  STATIC_LIB ${BROTLI_STATIC_LIBRARY_COMMON})
-
-if (BROTLI_VENDORED)
-  add_dependencies(brotli_enc brotli_ep)
-  add_dependencies(brotli_dec brotli_ep)
-  add_dependencies(brotli_common brotli_ep)
 endif()
 
 if (ARROW_WITH_LZ4)
 # ----------------------------------------------------------------------
 # Lz4
 
-  find_package(Lz4)
-  if (NOT LZ4_FOUND)
+  if("${LZ4_HOME}" STREQUAL "")
     set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep")
     set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
   
@@ -645,6 +646,7 @@ if (ARROW_WITH_LZ4)
   
     set(LZ4_VENDORED 1)
   else()
+    find_package(Lz4 REQUIRED)
     set(LZ4_VENDORED 0)
   endif()
   
@@ -661,8 +663,7 @@ if (ARROW_WITH_ZSTD)
 # ----------------------------------------------------------------------
 # ZSTD
 
-  find_package(ZSTD)
-  if (NOT ZSTD_FOUND)
+  if("${ZSTD_HOME}" STREQUAL "")
     set(ZSTD_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix/src/zstd_ep")
     set(ZSTD_INCLUDE_DIR "${ZSTD_BUILD_DIR}/lib")
 
@@ -687,6 +688,7 @@ if (ARROW_WITH_ZSTD)
 
     set(ZSTD_VENDORED 1)
   else()
+    find_package(ZSTD REQUIRED)
     set(ZSTD_VENDORED 0)
   endif()
 

From bf01966b460b11e65b4040aa816f88b7af2d7820 Mon Sep 17 00:00:00 2001
From: Mark Lavrynenko <marklavrinenko@gmail.com>
Date: Sun, 16 Jul 2017 20:53:32 +0300
Subject: [PATCH 0813/1644] [Python] Correct function name in use with pandas
 documentation

Contributed by Mark Lavrynenko <marklavrinenko@gmail.com>

Closes #851 on GitHub

Change-Id: I37118a04a8fa59b5432394ffbbadc73593889c01
---
 python/doc/source/pandas.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/doc/source/pandas.rst b/python/doc/source/pandas.rst
index cb7a56d020d19..d234e78b7ccb5 100644
--- a/python/doc/source/pandas.rst
+++ b/python/doc/source/pandas.rst
@@ -55,7 +55,7 @@ Series
 
 In Arrow, the most similar structure to a pandas Series is an Array.
 It is a vector that contains data of the same type as linear memory. You can
-convert a pandas Series to an Arrow Array using :meth:`pyarrow.array.from_pandas_series`.
+convert a pandas Series to an Arrow Array using :meth:`pyarrow.Array.from_pandas`.
 As Arrow Arrays are always nullable, you can supply an optional mask using
 the ``mask`` parameter to mark all null-entries.
 

From 50b518afcb03172e6133643eb5cbecd03c63368c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 16 Jul 2017 17:29:43 -0400
Subject: [PATCH 0814/1644] ARROW-1183: [Python] Implement pandas conversions
 between Time32, Time64 types and datetime.time

There's also a little bit of code reorganization; sorry about the large diff.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #849 from wesm/ARROW-1183 and squashes the following commits:

7e94e25c [Wes McKinney] Improve error messages to add extra context when using InvalidConversion
9659b903 [Wes McKinney] Always install thrift-cpp in toolchain Parquet MSVC build
41ce47ea [Wes McKinney] Fix MSVC compiler warning
681459c9 [Wes McKinney] Add missing PyDateTime_IMPORT
7024a366 [Wes McKinney] Finish roundtrip of time32/time64 to array of pytime
58fe4c00 [Wes McKinney] Add time to_pandas test
58f99f60 [Wes McKinney] Test from_pandas conversions from pytime
9228ec46 [Wes McKinney] Start in on time conversions
---
 ci/msvc-build.bat                           |   16 +-
 cpp/src/arrow/builder.cc                    |   11 +-
 cpp/src/arrow/ipc/metadata.cc               |    4 +-
 cpp/src/arrow/python/builtin_convert.cc     |   21 +-
 cpp/src/arrow/python/builtin_convert.h      |    4 +-
 cpp/src/arrow/python/pandas_convert.cc      |  216 ++--
 cpp/src/arrow/python/util/datetime.h        |   45 +
 cpp/src/arrow/util/logging.h                |    3 +-
 cpp/src/plasma/protocol.cc                  |    4 +-
 python/pyarrow/array.pxi                    | 1119 +------------------
 python/pyarrow/lib.pyx                      |    6 +
 python/pyarrow/pandas_compat.py             |    2 +
 python/pyarrow/scalar.pxi                   |  376 +++++++
 python/pyarrow/tests/test_array.py          |    4 +-
 python/pyarrow/tests/test_convert_pandas.py |  106 ++
 python/pyarrow/types.pxi                    |  776 +++++++++++++
 16 files changed, 1506 insertions(+), 1207 deletions(-)
 create mode 100644 python/pyarrow/scalar.pxi
 create mode 100644 python/pyarrow/types.pxi

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index be5fd7b35f468..22108abdd3b97 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -35,14 +35,26 @@ if "%JOB%" == "Build_Debug" (
 )
 
 conda update --yes --quiet conda
+conda config --set auto_update_conda false
+conda info -a
+
+conda config --set show_channel_urls True
+
+# Help with SSL timeouts to S3
+conda config --set remote_connect_timeout_secs 12
+
+conda config --add channels https://repo.continuum.io/pkgs/free
+conda config --add channels conda-forge
+conda info -a
 
 conda create -n arrow -q -y python=%PYTHON% ^
-      six pytest setuptools numpy pandas cython
+      six pytest setuptools numpy pandas cython ^
+      thrift-cpp
 
 if "%JOB%" == "Toolchain" (
   conda install -n arrow -q -y -c conda-forge ^
       flatbuffers rapidjson cmake git boost-cpp ^
-      thrift-cpp snappy zlib brotli gflags lz4-c zstd
+      snappy zlib brotli gflags lz4-c zstd
 )
 
 call activate arrow
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index e466838c37539..885c65029637a 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -984,16 +984,17 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
     const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(
-          pool, type ? type : std::static_pointer_cast<DataType>(
-                                  std::make_shared<ListType>(value_builder->type()))),
+    : ArrayBuilder(pool,
+          type ? type : std::static_pointer_cast<DataType>(
+                            std::make_shared<ListType>(value_builder->type()))),
       offset_builder_(pool),
       value_builder_(std::move(value_builder)) {}
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<Array> values,
     const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
-                                           std::make_shared<ListType>(values->type()))),
+    : ArrayBuilder(pool,
+          type ? type : std::static_pointer_cast<DataType>(
+                            std::make_shared<ListType>(values->type()))),
       offset_builder_(pool),
       values_(values) {}
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 5b2ca3b9c7dc6..49c24c72727b7 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -933,9 +933,7 @@ const void* Message::header() const {
 bool Message::Equals(const Message& other) const {
   int64_t metadata_bytes = std::min(metadata()->size(), other.metadata()->size());
 
-  if (!metadata()->Equals(*other.metadata(), metadata_bytes)) {
-    return false;
-  }
+  if (!metadata()->Equals(*other.metadata(), metadata_bytes)) { return false; }
 
   // Compare bodies, if they have them
   auto this_body = body();
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 816f95a42c0d7..83154bb1675b2 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -44,7 +44,8 @@ static inline bool IsPyInteger(PyObject* obj) {
 #endif
 }
 
-Status InvalidConversion(PyObject* obj, const std::string& expected_type_name) {
+Status InvalidConversion(PyObject* obj, const std::string& expected_types,
+    std::ostream* out) {
   OwnedRef type(PyObject_Type(obj));
   RETURN_IF_PYERROR();
   DCHECK_NE(type.obj(), nullptr);
@@ -63,10 +64,10 @@ Status InvalidConversion(PyObject* obj, const std::string& expected_type_name) {
 
   std::string cpp_type_name(bytes, size);
 
-  std::stringstream ss;
-  ss << "Python object of type " << cpp_type_name << " is not None and is not a "
-     << expected_type_name << " object";
-  return Status::Invalid(ss.str());
+  (*out) << "Got Python object of type " << cpp_type_name
+         << " but can only handle these types: "
+         << expected_types;
+  return Status::OK();
 }
 
 class ScalarVisitor {
@@ -462,7 +463,10 @@ class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverte
     } else if (PyBytes_Check(item.obj())) {
       bytes_obj = item.obj();
     } else {
-      return InvalidConversion(item.obj(), "bytes");
+      std::stringstream ss;
+      ss << "Error converting to Binary type: ";
+      RETURN_NOT_OK(InvalidConversion(item.obj(), "bytes", &ss));
+      return Status::Invalid(ss.str());
     }
     // No error checking
     length = PyBytes_GET_SIZE(bytes_obj);
@@ -489,7 +493,10 @@ class FixedWidthBytesConverter
     } else if (PyBytes_Check(item.obj())) {
       bytes_obj = item.obj();
     } else {
-      return InvalidConversion(item.obj(), "bytes");
+      std::stringstream ss;
+      ss << "Error converting to FixedSizeBinary type: ";
+      RETURN_NOT_OK(InvalidConversion(item.obj(), "bytes", &ss));
+      return Status::Invalid(ss.str());
     }
     // No error checking
     RETURN_NOT_OK(CheckPythonBytesAreFixedLength(bytes_obj, expected_length));
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index dd878b2d95847..4f84fbb7caca9 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -24,6 +24,7 @@
 #include "arrow/python/platform.h"
 
 #include <memory>
+#include <ostream>
 #include <string>
 
 #include "arrow/type.h"
@@ -62,7 +63,8 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
     const std::shared_ptr<DataType>& type, int64_t size);
 
 ARROW_EXPORT
-Status InvalidConversion(PyObject* obj, const std::string& expected_type_name);
+Status InvalidConversion(
+    PyObject* obj, const std::string& expected_type_name, std::ostream* out);
 
 ARROW_EXPORT Status CheckPythonBytesAreFixedLength(
     PyObject* obj, Py_ssize_t expected_length);
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 83cd35abf8c4b..c520d8d242a47 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -189,7 +189,10 @@ static Status AppendObjectStrings(
       const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
       RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
     } else {
-      return InvalidConversion(obj, "string or bytes");
+      std::stringstream ss;
+      ss << "Error converting to Python objects to String/UTF8: ";
+      RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
+      return Status::Invalid(ss.str());
     }
   }
 
@@ -230,7 +233,10 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
       RETURN_NOT_OK(
           builder->Append(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj))));
     } else {
-      return InvalidConversion(obj, "string or bytes");
+      std::stringstream ss;
+      ss << "Error converting to Python objects to FixedSizeBinary: ";
+      RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
+      return Status::Invalid(ss.str());
     }
   }
 
@@ -416,6 +422,7 @@ class PandasConverter {
       const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list);
   Status ConvertObjects();
   Status ConvertDecimals();
+  Status ConvertTimes();
 
  protected:
   MemoryPool* pool_;
@@ -536,7 +543,6 @@ Status PandasConverter::ConvertDates() {
   /// datetime API otherwise
   PyDateTime_IMPORT;
 
-  Status s;
   PyObject* obj;
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
@@ -545,7 +551,11 @@ Status PandasConverter::ConvertDates() {
     } else if (PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(date_builder.AppendNull());
     } else {
-      return InvalidConversion(obj, "date");
+      std::stringstream ss;
+      ss << "Error converting from Python objects to "
+         << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "datetime.date", &ss));
+      return Status::Invalid(ss.str());
     }
   }
   return date_builder.Finish(&out_);
@@ -582,7 +592,6 @@ Status PandasConverter::ConvertDecimals() {
 
   const int bit_width = std::dynamic_pointer_cast<DecimalType>(type_)->bit_width();
   DecimalBuilder decimal_builder(pool_, type_);
-
   RETURN_NOT_OK(decimal_builder.Resize(length_));
 
   for (int64_t i = 0; i < length_; ++i) {
@@ -598,12 +607,45 @@ Status PandasConverter::ConvertDecimals() {
     } else if (PandasObjectIsNull(object)) {
       RETURN_NOT_OK(decimal_builder.AppendNull());
     } else {
-      return InvalidConversion(object, "decimal.Decimal");
+      std::stringstream ss;
+      ss << "Error converting from Python objects to "
+         << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(object, "decimal.Decimal", &ss));
+      return Status::Invalid(ss.str());
     }
   }
   return decimal_builder.Finish(&out_);
 }
 
+Status PandasConverter::ConvertTimes() {
+  // Convert array of datetime.time objects to Arrow
+  PyAcquireGIL lock;
+  PyDateTime_IMPORT;
+
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+
+  // datetime.time stores microsecond resolution
+  Time64Builder builder(pool_, ::arrow::time64(TimeUnit::MICRO));
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  PyObject* obj;
+  for (int64_t i = 0; i < length_; ++i) {
+    obj = objects[i];
+    if (PyTime_Check(obj)) {
+      RETURN_NOT_OK(builder.Append(PyTime_to_us(obj)));
+    } else if (PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else {
+      std::stringstream ss;
+      ss << "Error converting from Python objects to "
+         << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "datetime.time", &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+  return builder.Finish(&out_);
+}
+
 #undef CONVERT_DECIMAL_CASE
 
 Status PandasConverter::ConvertObjectStrings() {
@@ -631,9 +673,6 @@ Status PandasConverter::ConvertObjectStrings() {
 Status PandasConverter::ConvertObjectFloats() {
   PyAcquireGIL lock;
 
-  DoubleBuilder builder(pool_);
-  RETURN_NOT_OK(builder.Resize(length_));
-
   Ndarray1DIndexer<PyObject*> objects(arr_);
   Ndarray1DIndexer<uint8_t> mask_values;
 
@@ -643,6 +682,9 @@ Status PandasConverter::ConvertObjectFloats() {
     have_mask = true;
   }
 
+  DoubleBuilder builder(pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
   PyObject* obj;
   for (int64_t i = 0; i < objects.size(); ++i) {
     obj = objects[i];
@@ -653,7 +695,11 @@ Status PandasConverter::ConvertObjectFloats() {
       RETURN_IF_PYERROR();
       RETURN_NOT_OK(builder.Append(val));
     } else {
-      return InvalidConversion(obj, "float");
+      std::stringstream ss;
+      ss << "Error converting from Python objects to "
+         << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "float", &ss));
+      return Status::Invalid(ss.str());
     }
   }
 
@@ -685,7 +731,11 @@ Status PandasConverter::ConvertObjectIntegers() {
       RETURN_IF_PYERROR();
       RETURN_NOT_OK(builder.Append(val));
     } else {
-      return InvalidConversion(obj, "integer");
+      std::stringstream ss;
+      ss << "Error converting from Python objects to "
+         << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "integer", &ss));
+      return Status::Invalid(ss.str());
     }
   }
 
@@ -791,6 +841,36 @@ static Status ConvertDecimals(const ChunkedArray& data, PyObject** out_values) {
   return Status::OK();
 }
 
+template <typename TYPE>
+static Status ConvertTimes(const ChunkedArray& data, PyObject** out_values) {
+  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
+
+  PyAcquireGIL lock;
+  OwnedRef time_ref;
+
+  PyDateTime_IMPORT;
+
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const auto& arr = static_cast<const ArrayType&>(*data.chunk(c));
+    auto type = std::dynamic_pointer_cast<TYPE>(arr.type());
+    DCHECK(type);
+
+    const TimeUnit::type unit = type->unit();
+
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (arr.IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values++ = Py_None;
+      } else {
+        RETURN_NOT_OK(PyTime_from_int(arr.Value(i), unit, out_values++));
+        RETURN_IF_PYERROR();
+      }
+    }
+  }
+
+  return Status::OK();
+}
+
 Status PandasConverter::ConvertBooleans() {
   PyAcquireGIL lock;
 
@@ -821,7 +901,11 @@ Status PandasConverter::ConvertBooleans() {
     } else if (obj == Py_False) {
       BitUtil::SetBit(null_bitmap_data_, i);
     } else {
-      return InvalidConversion(obj, "bool");
+      std::stringstream ss;
+      ss << "Error converting from Python objects to "
+         << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "bool", &ss));
+      return Status::Invalid(ss.str());
     }
   }
 
@@ -882,28 +966,35 @@ Status PandasConverter::ConvertObjects() {
     RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
 
     for (int64_t i = 0; i < length_; ++i) {
-      if (PandasObjectIsNull(objects[i])) {
+      PyObject* obj = objects[i];
+      if (PandasObjectIsNull(obj)) {
         continue;
-      } else if (PyObject_is_string(objects[i])) {
+      } else if (PyObject_is_string(obj)) {
         return ConvertObjectStrings();
-      } else if (PyObject_is_float(objects[i])) {
+      } else if (PyObject_is_float(obj)) {
         return ConvertObjectFloats();
-      } else if (PyBool_Check(objects[i])) {
+      } else if (PyBool_Check(obj)) {
         return ConvertBooleans();
-      } else if (PyObject_is_integer(objects[i])) {
+      } else if (PyObject_is_integer(obj)) {
         return ConvertObjectIntegers();
-      } else if (PyDate_CheckExact(objects[i])) {
+      } else if (PyDate_CheckExact(obj)) {
         // We could choose Date32 or Date64
         return ConvertDates<Date32Type>();
-      } else if (PyObject_IsInstance(const_cast<PyObject*>(objects[i]), Decimal.obj())) {
+      } else if (PyTime_Check(obj)) {
+        return ConvertTimes();
+      } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
         return ConvertDecimals();
-      } else if (PyList_Check(objects[i]) || PyArray_Check(objects[i])) {
+      } else if (PyList_Check(obj) || PyArray_Check(obj)) {
         std::shared_ptr<DataType> inferred_type;
-        RETURN_NOT_OK(InferArrowType(objects[i], &inferred_type));
+        RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
         return ConvertLists(inferred_type);
       } else {
-        return InvalidConversion(const_cast<PyObject*>(objects[i]),
-            "string, bool, float, int, date, decimal, list, array");
+        const std::string supported_types =
+          "string, bool, float, int, date, time, decimal, list, array";
+        std::stringstream ss;
+        ss << "Error inferring Arrow type for Python object array. ";
+        RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
+        return Status::Invalid(ss.str());
       }
     }
   }
@@ -1187,7 +1278,6 @@ class PandasBlock {
     INT64,
     FLOAT,
     DOUBLE,
-    DECIMAL,
     BOOL,
     DATETIME,
     DATETIME_WITH_TZ,
@@ -1598,6 +1688,10 @@ class ObjectBlock : public PandasBlock {
       RETURN_NOT_OK(ConvertBinaryLike<StringType>(data, out_buffer));
     } else if (type == Type::FIXED_SIZE_BINARY) {
       RETURN_NOT_OK(ConvertFixedSizeBinary(data, out_buffer));
+    } else if (type == Type::TIME32) {
+      RETURN_NOT_OK(ConvertTimes<Time32Type>(data, out_buffer));
+    } else if (type == Type::TIME64) {
+      RETURN_NOT_OK(ConvertTimes<Time64Type>(data, out_buffer));
     } else if (type == Type::DECIMAL) {
       RETURN_NOT_OK(ConvertDecimals(data, out_buffer));
     } else if (type == Type::NA) {
@@ -1950,7 +2044,6 @@ Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
     BLOCK_CASE(DOUBLE, Float64Block);
     BLOCK_CASE(BOOL, BoolBlock);
     BLOCK_CASE(DATETIME, DatetimeBlock);
-    BLOCK_CASE(DECIMAL, ObjectBlock);
     default:
       return Status::NotImplemented("Unsupported block type");
   }
@@ -2052,9 +2145,14 @@ class DataFrameBlockCreator {
         case Type::DOUBLE:
           output_type = PandasBlock::DOUBLE;
           break;
+        case Type::NA:
         case Type::STRING:
         case Type::BINARY:
         case Type::FIXED_SIZE_BINARY:
+        case Type::STRUCT:
+        case Type::TIME32:
+        case Type::TIME64:
+        case Type::DECIMAL:
           output_type = PandasBlock::OBJECT;
           break;
         case Type::DATE32:
@@ -2084,13 +2182,6 @@ class DataFrameBlockCreator {
         case Type::DICTIONARY:
           output_type = PandasBlock::CATEGORICAL;
           break;
-        case Type::DECIMAL:
-          output_type = PandasBlock::DECIMAL;
-          break;
-        case Type::NA:
-        case Type::STRUCT:
-          output_type = PandasBlock::OBJECT;
-          break;
         default:
           std::stringstream ss;
           ss << "No known equivalent Pandas block for Arrow data of type ";
@@ -2386,40 +2477,45 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
-  // Boolean specialization
-  Status Visit(const BooleanType& type) {
-    if (data_.null_count() > 0) {
-      RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-      auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-      RETURN_NOT_OK(ConvertBooleanWithNulls(data_, out_values));
-    } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<Type::BOOL>::npy_type));
-      auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(arr_));
-      ConvertBooleanNoNulls(data_, out_values);
-    }
-    return Status::OK();
+  template <typename FUNCTOR>
+  inline Status VisitObjects(FUNCTOR func) {
+    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
+    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+    return func(data_, out_values);
   }
 
   // UTF8 strings
   template <typename Type>
   typename std::enable_if<std::is_base_of<BinaryType, Type>::value, Status>::type Visit(
       const Type& type) {
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertBinaryLike<Type>(data_, out_values);
+    return VisitObjects(ConvertBinaryLike<Type>);
   }
 
+  Status Visit(const NullType& type) { return VisitObjects(ConvertNulls); }
+
   // Fixed length binary strings
   Status Visit(const FixedSizeBinaryType& type) {
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertFixedSizeBinary(data_, out_values);
+    return VisitObjects(ConvertFixedSizeBinary);
   }
 
-  Status Visit(const DecimalType& type) {
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertDecimals(data_, out_values);
+  Status Visit(const DecimalType& type) { return VisitObjects(ConvertDecimals); }
+
+  Status Visit(const Time32Type& type) { return VisitObjects(ConvertTimes<Time32Type>); }
+
+  Status Visit(const Time64Type& type) { return VisitObjects(ConvertTimes<Time64Type>); }
+
+  Status Visit(const StructType& type) { return VisitObjects(ConvertStruct); }
+
+  // Boolean specialization
+  Status Visit(const BooleanType& type) {
+    if (data_.null_count() > 0) {
+      return VisitObjects(ConvertBooleanWithNulls);
+    } else {
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<Type::BOOL>::npy_type));
+      auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(arr_));
+      ConvertBooleanNoNulls(data_, out_values);
+    }
+    return Status::OK();
   }
 
   Status Visit(const ListType& type) {
@@ -2479,18 +2575,6 @@ class ArrowDeserializer {
     return Status::OK();
   }
 
-  Status Visit(const NullType& type) {
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertNulls(data_, out_values);
-  }
-
-  Status Visit(const StructType& type) {
-    RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
-    auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return ConvertStruct(data_, out_values);
-  }
-
   Status Visit(const UnionType& type) { return Status::NotImplemented("union type"); }
 
   Status Convert(PyObject** out) {
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index 7ebf46a92fd5c..d32421e8e3652 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -24,6 +24,51 @@
 namespace arrow {
 namespace py {
 
+static inline int64_t PyTime_to_us(PyObject* pytime) {
+  return (static_cast<int64_t>(PyDateTime_TIME_GET_HOUR(pytime)) * 3600000000LL +
+          static_cast<int64_t>(PyDateTime_TIME_GET_MINUTE(pytime)) * 60000000LL +
+          static_cast<int64_t>(PyDateTime_TIME_GET_SECOND(pytime)) * 1000000LL +
+          PyDateTime_TIME_GET_MICROSECOND(pytime));
+}
+
+static inline Status PyTime_from_int(
+    int64_t val, const TimeUnit::type unit, PyObject** out) {
+  int64_t hour = 0, minute = 0, second = 0, microsecond = 0;
+  switch (unit) {
+    case TimeUnit::NANO:
+      if (val % 1000 != 0) {
+        std::stringstream ss;
+        ss << "Value " << val << " has non-zero nanoseconds";
+        return Status::Invalid(ss.str());
+      }
+      val /= 1000;
+    // fall through
+    case TimeUnit::MICRO:
+      microsecond = val - (val / 1000000LL) * 1000000LL;
+      val /= 1000000LL;
+      second = val - (val / 60) * 60;
+      val /= 60;
+      minute = val - (val / 60) * 60;
+      hour = val / 60;
+      break;
+    case TimeUnit::MILLI:
+      microsecond = (val - (val / 1000) * 1000) * 1000;
+      val /= 1000;
+    // fall through
+    case TimeUnit::SECOND:
+      second = val - (val / 60) * 60;
+      val /= 60;
+      minute = val - (val / 60) * 60;
+      hour = val / 60;
+      break;
+    default:
+      break;
+  }
+  *out = PyTime_FromTime(static_cast<int32_t>(hour), static_cast<int32_t>(minute),
+      static_cast<int32_t>(second), static_cast<int32_t>(microsecond));
+  return Status::OK();
+}
+
 static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   struct tm date = {0};
   date.tm_year = PyDateTime_GET_YEAR(pydate) - 1900;
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index b6181219dbae6..2fec4fa9e3c3a 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -103,8 +103,7 @@ class NullLog {
 class CerrLog {
  public:
   CerrLog(int severity)  // NOLINT(runtime/explicit)
-      : severity_(severity),
-        has_logged_(false) {}
+      : severity_(severity), has_logged_(false) {}
 
   virtual ~CerrLog() {
     if (has_logged_) { std::cerr << std::endl; }
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 246aa29736056..9739d7725a166 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -38,8 +38,8 @@ to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
 Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer) {
   int64_t type;
   RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
-  ARROW_CHECK(type == message_type) << "type = " << type
-                                    << ", message_type = " << message_type;
+  ARROW_CHECK(type == message_type)
+      << "type = " << type << ", message_type = " << message_type;
   return Status::OK();
 }
 
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index ae9ff885c4c8b..f8bccc772912b 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -15,1122 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.includes.libarrow cimport *
-
-# These are imprecise because the type (in pandas 0.x) depends on the presence
-# of nulls
-cdef dict _pandas_type_map = {
-    _Type_NA: np.float64,  # NaNs
-    _Type_BOOL: np.bool_,
-    _Type_INT8: np.int8,
-    _Type_INT16: np.int16,
-    _Type_INT32: np.int32,
-    _Type_INT64: np.int64,
-    _Type_UINT8: np.uint8,
-    _Type_UINT16: np.uint16,
-    _Type_UINT32: np.uint32,
-    _Type_UINT64: np.uint64,
-    _Type_HALF_FLOAT: np.float16,
-    _Type_FLOAT: np.float32,
-    _Type_DOUBLE: np.float64,
-    _Type_DATE32: np.dtype('datetime64[ns]'),
-    _Type_DATE64: np.dtype('datetime64[ns]'),
-    _Type_TIMESTAMP: np.dtype('datetime64[ns]'),
-    _Type_BINARY: np.object_,
-    _Type_FIXED_SIZE_BINARY: np.object_,
-    _Type_STRING: np.object_,
-    _Type_LIST: np.object_,
-    _Type_DECIMAL: np.object_,
-}
-
-
-cdef class DataType:
-
-    def __cinit__(self):
-        pass
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        self.sp_type = type
-        self.type = type.get()
-
-    property id:
-
-        def __get__(self):
-            return self.type.id()
-
-    def __str__(self):
-        if self.type is NULL:
-            raise TypeError(
-                '{} is incomplete. The correct way to construct types is '
-                'through public API functions named '
-                'pyarrow.int64, pyarrow.list_, etc.'.format(
-                    type(self).__name__
-                )
-            )
-        return frombytes(self.type.ToString())
-
-    def __repr__(self):
-        return '{0.__class__.__name__}({0})'.format(self)
-
-    def __richcmp__(DataType self, DataType other, int op):
-        if op == cp.Py_EQ:
-            return self.type.Equals(deref(other.type))
-        elif op == cp.Py_NE:
-            return not self.type.Equals(deref(other.type))
-        else:
-            raise TypeError('Invalid comparison')
-
-    def to_pandas_dtype(self):
-        """
-        Return the NumPy dtype that would be used for storing this
-        """
-        cdef Type type_id = self.type.id()
-        if type_id in _pandas_type_map:
-            return _pandas_type_map[type_id]
-        else:
-            raise NotImplementedError(str(self))
-
-
-cdef class DictionaryType(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.dict_type = <const CDictionaryType*> type.get()
-
-
-cdef class ListType(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.list_type = <const CListType*> type.get()
-
-    property value_type:
-
-        def __get__(self):
-            return pyarrow_wrap_data_type(self.list_type.value_type())
-
-
-cdef class TimestampType(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.ts_type = <const CTimestampType*> type.get()
-
-    property unit:
-
-        def __get__(self):
-            return timeunit_to_string(self.ts_type.unit())
-
-    property tz:
-
-        def __get__(self):
-            if self.ts_type.timezone().size() > 0:
-                return frombytes(self.ts_type.timezone())
-            else:
-                return None
-
-
-cdef class Time32Type(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.time_type = <const CTime32Type*> type.get()
-
-    property unit:
-
-        def __get__(self):
-            return timeunit_to_string(self.time_type.unit())
-
-
-cdef class Time64Type(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.time_type = <const CTime64Type*> type.get()
-
-    property unit:
-
-        def __get__(self):
-            return timeunit_to_string(self.time_type.unit())
-
-
-cdef class FixedSizeBinaryType(DataType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.fixed_size_binary_type = (
-            <const CFixedSizeBinaryType*> type.get())
-
-    property byte_width:
-
-        def __get__(self):
-            return self.fixed_size_binary_type.byte_width()
-
-
-cdef class DecimalType(FixedSizeBinaryType):
-
-    cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
-        self.decimal_type = <const CDecimalType*> type.get()
-
-    property precision:
-
-        def __get__(self):
-            return self.decimal_type.precision()
-
-    property scale:
-
-        def __get__(self):
-            return self.decimal_type.scale()
-
-
-cdef class Field:
-    """
-    Represents a named field, with a data type, nullability, and optional
-    metadata
-
-    Notes
-    -----
-    Do not use this class's constructor directly; use pyarrow.field
-    """
-    def __cinit__(self):
-        pass
-
-    cdef void init(self, const shared_ptr[CField]& field):
-        self.sp_field = field
-        self.field = field.get()
-        self.type = pyarrow_wrap_data_type(field.get().type())
-
-    def equals(self, Field other):
-        """
-        Test if this field is equal to the other
-        """
-        return self.field.Equals(deref(other.field))
-
-    def __str__(self):
-        self._check_null()
-        return 'pyarrow.Field<{0}>'.format(frombytes(self.field.ToString()))
-
-    def __repr__(self):
-        return self.__str__()
-
-    property nullable:
-
-        def __get__(self):
-            self._check_null()
-            return self.field.nullable()
-
-    property name:
-
-        def __get__(self):
-            self._check_null()
-            return frombytes(self.field.name())
-
-    property metadata:
-
-        def __get__(self):
-            self._check_null()
-            return box_metadata(self.field.metadata().get())
-
-    def _check_null(self):
-        if self.field == NULL:
-            raise ReferenceError(
-                'Field not initialized (references NULL pointer)')
-
-    def add_metadata(self, dict metadata):
-        """
-        Add metadata as dict of string keys and values to Field
-
-        Parameters
-        ----------
-        metadata : dict
-            Keys and values must be string-like / coercible to bytes
-
-        Returns
-        -------
-        field : pyarrow.Field
-        """
-        cdef shared_ptr[CKeyValueMetadata] c_meta
-        convert_metadata(metadata, &c_meta)
-
-        cdef shared_ptr[CField] new_field
-        with nogil:
-            check_status(self.field.AddMetadata(c_meta, &new_field))
-
-        return pyarrow_wrap_field(new_field)
-
-    def remove_metadata(self):
-        """
-        Create new field without metadata, if any
-
-        Returns
-        -------
-        field : pyarrow.Field
-        """
-        cdef shared_ptr[CField] new_field
-        with nogil:
-            new_field = self.field.RemoveMetadata()
-        return pyarrow_wrap_field(new_field)
-
-
-cdef class Schema:
-
-    def __cinit__(self):
-        pass
-
-    def __len__(self):
-        return self.schema.num_fields()
-
-    def __getitem__(self, int64_t i):
-
-        cdef:
-            Field result = Field()
-            int64_t num_fields = self.schema.num_fields()
-            int64_t index
-
-        if not -num_fields <= i < num_fields:
-            raise IndexError(
-                'Schema field index {:d} is out of range'.format(i)
-            )
-
-        index = i if i >= 0 else num_fields + i
-        assert index >= 0
-
-        result.init(self.schema.field(index))
-        result.type = pyarrow_wrap_data_type(result.field.type())
-
-        return result
-
-    cdef void init(self, const vector[shared_ptr[CField]]& fields):
-        self.schema = new CSchema(fields)
-        self.sp_schema.reset(self.schema)
-
-    cdef void init_schema(self, const shared_ptr[CSchema]& schema):
-        self.schema = schema.get()
-        self.sp_schema = schema
-
-    property names:
-
-        def __get__(self):
-            cdef int i
-            result = []
-            for i in range(self.schema.num_fields()):
-                name = frombytes(self.schema.field(i).get().name())
-                result.append(name)
-            return result
-
-    property metadata:
-
-        def __get__(self):
-            return box_metadata(self.schema.metadata().get())
-
-    def equals(self, other):
-        """
-        Test if this schema is equal to the other
-        """
-        cdef Schema _other
-        _other = other
-
-        return self.sp_schema.get().Equals(deref(_other.schema))
-
-    def field_by_name(self, name):
-        """
-        Access a field by its name rather than the column index.
-
-        Parameters
-        ----------
-        name: str
-
-        Returns
-        -------
-        field: pyarrow.Field
-        """
-        return pyarrow_wrap_field(self.schema.GetFieldByName(tobytes(name)))
-
-    def get_field_index(self, name):
-        return self.schema.GetFieldIndex(tobytes(name))
-
-    def add_metadata(self, dict metadata):
-        """
-        Add metadata as dict of string keys and values to Schema
-
-        Parameters
-        ----------
-        metadata : dict
-            Keys and values must be string-like / coercible to bytes
-
-        Returns
-        -------
-        schema : pyarrow.Schema
-        """
-        cdef shared_ptr[CKeyValueMetadata] c_meta
-        convert_metadata(metadata, &c_meta)
-
-        cdef shared_ptr[CSchema] new_schema
-        with nogil:
-            check_status(self.schema.AddMetadata(c_meta, &new_schema))
-
-        return pyarrow_wrap_schema(new_schema)
-
-    def remove_metadata(self):
-        """
-        Create new schema without metadata, if any
-
-        Returns
-        -------
-        schema : pyarrow.Schema
-        """
-        cdef shared_ptr[CSchema] new_schema
-        with nogil:
-            new_schema = self.schema.RemoveMetadata()
-        return pyarrow_wrap_schema(new_schema)
-
-    def __str__(self):
-        return frombytes(self.schema.ToString())
-
-    def __repr__(self):
-        return self.__str__()
-
-
-cdef dict box_metadata(const CKeyValueMetadata* metadata):
-    cdef unordered_map[c_string, c_string] result
-    if metadata != nullptr:
-        metadata.ToUnorderedMap(&result)
-        return result
-    else:
-        return None
-
-
-cdef dict _type_cache = {}
-
-
-cdef DataType primitive_type(Type type):
-    if type in _type_cache:
-        return _type_cache[type]
-
-    cdef DataType out = DataType()
-    out.init(GetPrimitiveType(type))
-
-    _type_cache[type] = out
-    return out
-
-#------------------------------------------------------------
-# Type factory functions
-
-cdef int convert_metadata(dict metadata,
-                          shared_ptr[CKeyValueMetadata]* out) except -1:
-    cdef:
-        shared_ptr[CKeyValueMetadata] meta = (
-            make_shared[CKeyValueMetadata]())
-        c_string key, value
-
-    for py_key, py_value in metadata.items():
-        key = tobytes(py_key)
-        value = tobytes(py_value)
-        meta.get().Append(key, value)
-    out[0] = meta
-    return 0
-
-
-def field(name, DataType type, bint nullable=True, dict metadata=None):
-    """
-    Create a pyarrow.Field instance
-
-    Parameters
-    ----------
-    name : string or bytes
-    type : pyarrow.DataType
-    nullable : boolean, default True
-    metadata : dict, default None
-        Keys and values must be coercible to bytes
-
-    Returns
-    -------
-    field : pyarrow.Field
-    """
-    cdef:
-        shared_ptr[CKeyValueMetadata] c_meta
-        Field result = Field()
-
-    if metadata is not None:
-        convert_metadata(metadata, &c_meta)
-
-    result.sp_field.reset(new CField(tobytes(name), type.sp_type,
-                                     nullable, c_meta))
-    result.field = result.sp_field.get()
-    result.type = type
-    return result
-
-
-cdef set PRIMITIVE_TYPES = set([
-    _Type_NA, _Type_BOOL,
-    _Type_UINT8, _Type_INT8,
-    _Type_UINT16, _Type_INT16,
-    _Type_UINT32, _Type_INT32,
-    _Type_UINT64, _Type_INT64,
-    _Type_TIMESTAMP, _Type_DATE32,
-    _Type_TIME32, _Type_TIME64,
-    _Type_DATE64,
-    _Type_HALF_FLOAT,
-    _Type_FLOAT,
-    _Type_DOUBLE])
-
-
-def null():
-    return primitive_type(_Type_NA)
-
-
-def bool_():
-    return primitive_type(_Type_BOOL)
-
-
-def uint8():
-    return primitive_type(_Type_UINT8)
-
-
-def int8():
-    return primitive_type(_Type_INT8)
-
-
-def uint16():
-    return primitive_type(_Type_UINT16)
-
-
-def int16():
-    return primitive_type(_Type_INT16)
-
-
-def uint32():
-    return primitive_type(_Type_UINT32)
-
-
-def int32():
-    return primitive_type(_Type_INT32)
-
-
-def uint64():
-    return primitive_type(_Type_UINT64)
-
-
-def int64():
-    return primitive_type(_Type_INT64)
-
-
-cdef dict _timestamp_type_cache = {}
-cdef dict _time_type_cache = {}
-
-
-cdef timeunit_to_string(TimeUnit unit):
-    if unit == TimeUnit_SECOND:
-        return 's'
-    elif unit == TimeUnit_MILLI:
-        return 'ms'
-    elif unit == TimeUnit_MICRO:
-        return 'us'
-    elif unit == TimeUnit_NANO:
-        return 'ns'
-
-
-def timestamp(unit_str, tz=None):
-    cdef:
-        TimeUnit unit
-        c_string c_timezone
-
-    if unit_str == "s":
-        unit = TimeUnit_SECOND
-    elif unit_str == 'ms':
-        unit = TimeUnit_MILLI
-    elif unit_str == 'us':
-        unit = TimeUnit_MICRO
-    elif unit_str == 'ns':
-        unit = TimeUnit_NANO
-    else:
-        raise ValueError('Invalid TimeUnit string')
-
-    cdef TimestampType out = TimestampType()
-
-    if tz is None:
-        out.init(ctimestamp(unit))
-        if unit in _timestamp_type_cache:
-            return _timestamp_type_cache[unit]
-        _timestamp_type_cache[unit] = out
-    else:
-        if not isinstance(tz, six.string_types):
-            tz = tz.zone
-
-        c_timezone = tobytes(tz)
-        out.init(ctimestamp(unit, c_timezone))
-
-    return out
-
-
-def time32(unit_str):
-    cdef:
-        TimeUnit unit
-        c_string c_timezone
-
-    if unit_str == "s":
-        unit = TimeUnit_SECOND
-    elif unit_str == 'ms':
-        unit = TimeUnit_MILLI
-    else:
-        raise ValueError('Invalid TimeUnit for time32: {}'.format(unit_str))
-
-    cdef Time32Type out
-    if unit in _time_type_cache:
-        return _time_type_cache[unit]
-    else:
-        out = Time32Type()
-        out.init(ctime32(unit))
-        _time_type_cache[unit] = out
-        return out
-
-
-def time64(unit_str):
-    cdef:
-        TimeUnit unit
-        c_string c_timezone
-
-    if unit_str == "us":
-        unit = TimeUnit_MICRO
-    elif unit_str == 'ns':
-        unit = TimeUnit_NANO
-    else:
-        raise ValueError('Invalid TimeUnit for time64: {}'.format(unit_str))
-
-    cdef Time64Type out
-    if unit in _time_type_cache:
-        return _time_type_cache[unit]
-    else:
-        out = Time64Type()
-        out.init(ctime64(unit))
-        _time_type_cache[unit] = out
-        return out
-
-
-def date32():
-    return primitive_type(_Type_DATE32)
-
-
-def date64():
-    return primitive_type(_Type_DATE64)
-
-
-def float16():
-    return primitive_type(_Type_HALF_FLOAT)
-
-
-def float32():
-    return primitive_type(_Type_FLOAT)
-
-
-def float64():
-    return primitive_type(_Type_DOUBLE)
-
-
-cpdef DataType decimal(int precision, int scale=0):
-    cdef shared_ptr[CDataType] decimal_type
-    decimal_type.reset(new CDecimalType(precision, scale))
-    return pyarrow_wrap_data_type(decimal_type)
-
-
-def string():
-    """
-    UTF8 string
-    """
-    return primitive_type(_Type_STRING)
-
-
-def binary(int length=-1):
-    """Binary (PyBytes-like) type
-
-    Parameters
-    ----------
-    length : int, optional, default -1
-        If length == -1 then return a variable length binary type. If length is
-        greater than or equal to 0 then return a fixed size binary type of
-        width `length`.
-    """
-    if length == -1:
-        return primitive_type(_Type_BINARY)
-
-    cdef shared_ptr[CDataType] fixed_size_binary_type
-    fixed_size_binary_type.reset(new CFixedSizeBinaryType(length))
-    return pyarrow_wrap_data_type(fixed_size_binary_type)
-
-
-cpdef ListType list_(value_type):
-    """
-    Create ListType instance from child data type or field
-
-    Parameters
-    ----------
-    value_type : DataType or Field
-
-    Returns
-    -------
-    list_type : DataType
-    """
-    cdef:
-        DataType data_type
-        Field field
-        shared_ptr[CDataType] list_type
-        ListType out = ListType()
-
-    if isinstance(value_type, DataType):
-        list_type.reset(new CListType((<DataType> value_type).sp_type))
-    elif isinstance(value_type, Field):
-        list_type.reset(new CListType((<Field> value_type).sp_field))
-    else:
-        raise ValueError('List requires DataType or Field')
-
-    out.init(list_type)
-    return out
-
-
-cpdef DictionaryType dictionary(DataType index_type, Array dictionary):
-    """
-    Dictionary (categorical, or simply encoded) type
-
-    Parameters
-    ----------
-    index_type : DataType
-    dictionary : Array
-
-    Returns
-    -------
-    type : DictionaryType
-    """
-    cdef DictionaryType out = DictionaryType()
-    cdef shared_ptr[CDataType] dict_type
-    dict_type.reset(new CDictionaryType(index_type.sp_type,
-                                        dictionary.sp_array))
-    out.init(dict_type)
-    return out
-
-
-def struct(fields):
-    """
-    Create StructType instance from fields
-
-    Parameters
-    ----------
-    fields : sequence of Field values
-
-    Examples
-    --------
-    import pyarrow as pa
-    fields = [
-        pa.field('f1', pa.int32()),
-        pa.field('f2', pa.string())
-    ]
-    struct_type = pa.struct(fields)
-
-    Returns
-    -------
-    type : DataType
-    """
-    cdef:
-        Field field
-        vector[shared_ptr[CField]] c_fields
-        cdef shared_ptr[CDataType] struct_type
-
-    for field in fields:
-        c_fields.push_back(field.sp_field)
-
-    struct_type.reset(new CStructType(c_fields))
-    return pyarrow_wrap_data_type(struct_type)
-
-
-def schema(fields):
-    """
-    Construct pyarrow.Schema from collection of fields
-
-    Parameters
-    ----------
-    field : list or iterable
-
-    Returns
-    -------
-    schema : pyarrow.Schema
-    """
-    cdef:
-        Schema result
-        Field field
-        vector[shared_ptr[CField]] c_fields
-
-    for i, field in enumerate(fields):
-        c_fields.push_back(field.sp_field)
-
-    result = Schema()
-    result.init(c_fields)
-    return result
-
-
-def from_numpy_dtype(object dtype):
-    """
-    Convert NumPy dtype to pyarrow.DataType
-    """
-    cdef shared_ptr[CDataType] c_type
-    with nogil:
-        check_status(NumPyDtypeToArrow(dtype, &c_type))
-
-    return pyarrow_wrap_data_type(c_type)
-
-
-NA = None
-
-
-cdef class NAType(Scalar):
-
-    def __cinit__(self):
-        global NA
-        if NA is not None:
-            raise Exception('Cannot create multiple NAType instances')
-
-        self.type = null()
-
-    def __repr__(self):
-        return 'NA'
-
-    def as_py(self):
-        return None
-
-
-NA = NAType()
-
-
-cdef class ArrayValue(Scalar):
-
-    cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
-                   int64_t index):
-        self.type = type
-        self.index = index
-        self._set_array(sp_array)
-
-    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
-        self.sp_array = sp_array
-
-    def _check_null(self):
-        if self.sp_array.get() == NULL:
-            raise ReferenceError(
-                'ArrayValue instance not propertly initialized '
-                '(references NULL pointer)')
-
-    def __repr__(self):
-        self._check_null()
-        if hasattr(self, 'as_py'):
-            return repr(self.as_py())
-        else:
-            return super(Scalar, self).__repr__()
-
-
-cdef class BooleanValue(ArrayValue):
-
-    def as_py(self):
-        cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Int8Value(ArrayValue):
-
-    def as_py(self):
-        cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class UInt8Value(ArrayValue):
-
-    def as_py(self):
-        cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Int16Value(ArrayValue):
-
-    def as_py(self):
-        cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class UInt16Value(ArrayValue):
-
-    def as_py(self):
-        cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Int32Value(ArrayValue):
-
-    def as_py(self):
-        cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class UInt32Value(ArrayValue):
-
-    def as_py(self):
-        cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Int64Value(ArrayValue):
-
-    def as_py(self):
-        cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class UInt64Value(ArrayValue):
-
-    def as_py(self):
-        cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class Date32Value(ArrayValue):
-
-    def as_py(self):
-        cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
-
-        # Shift to seconds since epoch
-        return datetime.datetime.utcfromtimestamp(
-            int(ap.Value(self.index)) * 86400).date()
-
-
-cdef class Date64Value(ArrayValue):
-
-    def as_py(self):
-        cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
-        return datetime.datetime.utcfromtimestamp(
-            ap.Value(self.index) / 1000).date()
-
-
-cdef class Time32Value(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            CTime32Array* ap = <CTime32Array*> self.sp_array.get()
-            CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
-        if dtype.unit() == TimeUnit_SECOND:
-            return (datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ap.Value(self.index))).time()
-        else:
-            return (datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=ap.Value(self.index))).time()
-
-
-cdef class Time64Value(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            CTime64Array* ap = <CTime64Array*> self.sp_array.get()
-            CTime64Type* dtype = <CTime64Type*> ap.type().get()
-
-        if dtype.unit() == TimeUnit_MICRO:
-            return (datetime.datetime(1970, 1, 1) + datetime.timedelta(microseconds=ap.Value(self.index))).time()
-        else:
-            return (datetime.datetime(1970, 1, 1) + datetime.timedelta(microseconds=ap.Value(self.index) / 1000)).time()
-
-
-cdef dict DATETIME_CONVERSION_FUNCTIONS
-
-try:
-    import pandas as pd
-except ImportError:
-    DATETIME_CONVERSION_FUNCTIONS = {
-        TimeUnit_SECOND: lambda x, tzinfo: (
-            datetime.datetime.utcfromtimestamp(x).replace(tzinfo=tzinfo)
-        ),
-        TimeUnit_MILLI: lambda x, tzinfo: (
-            datetime.datetime.utcfromtimestamp(x / 1e3).replace(tzinfo=tzinfo)
-        ),
-        TimeUnit_MICRO: lambda x, tzinfo: (
-            datetime.datetime.utcfromtimestamp(x / 1e6).replace(tzinfo=tzinfo)
-        ),
-    }
-else:
-    DATETIME_CONVERSION_FUNCTIONS = {
-        TimeUnit_SECOND: lambda x, tzinfo: pd.Timestamp(
-            x * 1000000000, tz=tzinfo, unit='ns',
-        ),
-        TimeUnit_MILLI: lambda x, tzinfo: pd.Timestamp(
-            x * 1000000, tz=tzinfo, unit='ns',
-        ),
-        TimeUnit_MICRO: lambda x, tzinfo: pd.Timestamp(
-            x * 1000, tz=tzinfo, unit='ns',
-        ),
-        TimeUnit_NANO: lambda x, tzinfo: pd.Timestamp(
-            x, tz=tzinfo, unit='ns',
-        )
-    }
-
-
-cdef class TimestampValue(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
-            CTimestampType* dtype = <CTimestampType*> ap.type().get()
-            int64_t value = ap.Value(self.index)
-
-        if not dtype.timezone().empty():
-            import pytz
-            tzinfo = pytz.timezone(frombytes(dtype.timezone()))
-        else:
-            tzinfo = None
-
-        try:
-            converter = DATETIME_CONVERSION_FUNCTIONS[dtype.unit()]
-        except KeyError:
-            raise ValueError(
-                'Cannot convert nanosecond timestamps without pandas'
-            )
-        return converter(value, tzinfo=tzinfo)
-
-
-cdef class FloatValue(ArrayValue):
-
-    def as_py(self):
-        cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class DoubleValue(ArrayValue):
-
-    def as_py(self):
-        cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
-        return ap.Value(self.index)
-
-
-cdef class DecimalValue(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            CDecimalArray* ap = <CDecimalArray*> self.sp_array.get()
-            c_string s = ap.FormatValue(self.index)
-        return _pydecimal.Decimal(s.decode('utf8'))
-
-
-cdef class StringValue(ArrayValue):
-
-    def as_py(self):
-        cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
-        return ap.GetString(self.index).decode('utf-8')
-
-
-cdef class BinaryValue(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            const uint8_t* ptr
-            int32_t length
-            CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
-
-        ptr = ap.GetValue(self.index, &length)
-        return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
-
-cdef class ListValue(ArrayValue):
-
-    def __len__(self):
-        return self.ap.value_length(self.index)
-
-    def __getitem__(self, i):
-        return self.getitem(i)
-
-    def __iter__(self):
-        for i in range(len(self)):
-            yield self.getitem(i)
-        raise StopIteration
-
-    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
-        self.sp_array = sp_array
-        self.ap = <CListArray*> sp_array.get()
-        self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
-    cdef getitem(self, int64_t i):
-        cdef int64_t j = self.ap.value_offset(self.index) + i
-        return box_scalar(self.value_type, self.ap.values(), j)
-
-    def as_py(self):
-        cdef:
-            int64_t j
-            list result = []
-
-        for j in range(len(self)):
-            result.append(self.getitem(j).as_py())
-
-        return result
-
-
-cdef class FixedSizeBinaryValue(ArrayValue):
-
-    def as_py(self):
-        cdef:
-            CFixedSizeBinaryArray* ap
-            CFixedSizeBinaryType* ap_type
-            int32_t length
-            const char* data
-        ap = <CFixedSizeBinaryArray*> self.sp_array.get()
-        ap_type = <CFixedSizeBinaryType*> ap.type().get()
-        length = ap_type.byte_width()
-        data = <const char*> ap.GetValue(self.index)
-        return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
-    def as_py(self):
-        cdef:
-            CStructArray* ap
-            vector[shared_ptr[CField]] child_fields = self.type.type.children()
-        ap = <CStructArray*> self.sp_array.get()
-        wrapped_arrays = (pyarrow_wrap_array(ap.field(i))
-                          for i in range(ap.num_fields()))
-        child_names = (child.get().name() for child in child_fields)
-        # Return the struct as a dict
-        return {
-            frombytes(name): child_array[self.index].as_py()
-            for name, child_array in
-            zip(child_names, wrapped_arrays)
-        }
-
-cdef dict _scalar_classes = {
-    _Type_BOOL: BooleanValue,
-    _Type_UINT8: Int8Value,
-    _Type_UINT16: Int16Value,
-    _Type_UINT32: Int32Value,
-    _Type_UINT64: Int64Value,
-    _Type_INT8: Int8Value,
-    _Type_INT16: Int16Value,
-    _Type_INT32: Int32Value,
-    _Type_INT64: Int64Value,
-    _Type_DATE32: Date32Value,
-    _Type_DATE64: Date64Value,
-    _Type_TIME32: Time32Value,
-    _Type_TIME64: Time64Value,
-    _Type_TIMESTAMP: TimestampValue,
-    _Type_FLOAT: FloatValue,
-    _Type_DOUBLE: DoubleValue,
-    _Type_LIST: ListValue,
-    _Type_BINARY: BinaryValue,
-    _Type_STRING: StringValue,
-    _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
-    _Type_DECIMAL: DecimalValue,
-    _Type_STRUCT: StructValue,
-}
-
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
-                       int64_t index):
-    cdef ArrayValue val
-    if type.type.id() == _Type_NA:
-        return NA
-    elif sp_array.get().IsNull(index):
-        return NA
-    else:
-        val = _scalar_classes[type.type.id()]()
-        val.init(type, sp_array, index)
-        return val
-
 
 cdef maybe_coerce_datetime64(values, dtype, DataType type,
                              timestamps_to_ms=False):
@@ -1229,7 +113,7 @@ cdef class Array:
         series : pandas.Series or numpy.ndarray
 
         mask : pandas.Series or numpy.ndarray, optional
-            boolean mask if the object is valid or null
+            boolean mask if the object is null (True) or valid (False)
 
         type : pyarrow.DataType
             Explicit type to attempt to coerce to
@@ -1690,6 +574,7 @@ cdef class DictionaryArray(Array):
         result.init(c_result)
         return result
 
+
 cdef class StructArray(Array):
     @staticmethod
     def from_arrays(field_names, arrays):
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index cf8e4df444594..599030888178b 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -100,6 +100,12 @@ include "error.pxi"
 # Memory pools and allocation
 include "memory.pxi"
 
+# DataType, Field, Schema
+include "types.pxi"
+
+# Array scalar values
+include "scalar.pxi"
+
 # Array types
 include "array.pxi"
 
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index c909b3ec856b6..9b2a5c4c60dbf 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -62,6 +62,8 @@ def get_logical_type_map():
             pa.lib.Type_DOUBLE: 'float64',
             pa.lib.Type_DATE32: 'date',
             pa.lib.Type_DATE64: 'date',
+            pa.lib.Type_TIME32: 'time',
+            pa.lib.Type_TIME64: 'time',
             pa.lib.Type_BINARY: 'bytes',
             pa.lib.Type_FIXED_SIZE_BINARY: 'bytes',
             pa.lib.Type_STRING: 'unicode',
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
new file mode 100644
index 0000000000000..11ed0efb39b97
--- /dev/null
+++ b/python/pyarrow/scalar.pxi
@@ -0,0 +1,376 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+NA = None
+
+
+cdef class NAType(Scalar):
+
+    def __cinit__(self):
+        global NA
+        if NA is not None:
+            raise Exception('Cannot create multiple NAType instances')
+
+        self.type = null()
+
+    def __repr__(self):
+        return 'NA'
+
+    def as_py(self):
+        return None
+
+
+NA = NAType()
+
+
+cdef class ArrayValue(Scalar):
+
+    cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
+                   int64_t index):
+        self.type = type
+        self.index = index
+        self._set_array(sp_array)
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+
+    def _check_null(self):
+        if self.sp_array.get() == NULL:
+            raise ReferenceError(
+                'ArrayValue instance not propertly initialized '
+                '(references NULL pointer)')
+
+    def __repr__(self):
+        self._check_null()
+        if hasattr(self, 'as_py'):
+            return repr(self.as_py())
+        else:
+            return super(Scalar, self).__repr__()
+
+
+cdef class BooleanValue(ArrayValue):
+
+    def as_py(self):
+        cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int8Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt8Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int16Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt16Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int32Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt32Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Int64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class UInt64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class Date32Value(ArrayValue):
+
+    def as_py(self):
+        cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+
+        # Shift to seconds since epoch
+        return datetime.datetime.utcfromtimestamp(
+            int(ap.Value(self.index)) * 86400).date()
+
+
+cdef class Date64Value(ArrayValue):
+
+    def as_py(self):
+        cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
+        return datetime.datetime.utcfromtimestamp(
+            ap.Value(self.index) / 1000).date()
+
+
+cdef class Time32Value(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CTime32Array* ap = <CTime32Array*> self.sp_array.get()
+            CTime32Type* dtype = <CTime32Type*> ap.type().get()
+
+        if dtype.unit() == TimeUnit_SECOND:
+            return (datetime.datetime(1970, 1, 1) +
+                    datetime.timedelta(seconds=ap.Value(self.index))).time()
+        else:
+            return (datetime.datetime(1970, 1, 1) +
+                    datetime.timedelta(milliseconds=ap.Value(self.index))).time()
+
+
+cdef class Time64Value(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CTime64Array* ap = <CTime64Array*> self.sp_array.get()
+            CTime64Type* dtype = <CTime64Type*> ap.type().get()
+
+        cdef int64_t val = ap.Value(self.index)
+        print(val)
+        if dtype.unit() == TimeUnit_MICRO:
+            return (datetime.datetime(1970, 1, 1) +
+                    datetime.timedelta(microseconds=val)).time()
+        else:
+            return (datetime.datetime(1970, 1, 1) +
+                    datetime.timedelta(microseconds=val / 1000)).time()
+
+
+cdef dict DATETIME_CONVERSION_FUNCTIONS
+
+try:
+    import pandas as pd
+except ImportError:
+    DATETIME_CONVERSION_FUNCTIONS = {
+        TimeUnit_SECOND: lambda x, tzinfo: (
+            datetime.datetime.utcfromtimestamp(x).replace(tzinfo=tzinfo)
+        ),
+        TimeUnit_MILLI: lambda x, tzinfo: (
+            datetime.datetime.utcfromtimestamp(x / 1e3).replace(tzinfo=tzinfo)
+        ),
+        TimeUnit_MICRO: lambda x, tzinfo: (
+            datetime.datetime.utcfromtimestamp(x / 1e6).replace(tzinfo=tzinfo)
+        ),
+    }
+else:
+    DATETIME_CONVERSION_FUNCTIONS = {
+        TimeUnit_SECOND: lambda x, tzinfo: pd.Timestamp(
+            x * 1000000000, tz=tzinfo, unit='ns',
+        ),
+        TimeUnit_MILLI: lambda x, tzinfo: pd.Timestamp(
+            x * 1000000, tz=tzinfo, unit='ns',
+        ),
+        TimeUnit_MICRO: lambda x, tzinfo: pd.Timestamp(
+            x * 1000, tz=tzinfo, unit='ns',
+        ),
+        TimeUnit_NANO: lambda x, tzinfo: pd.Timestamp(
+            x, tz=tzinfo, unit='ns',
+        )
+    }
+
+
+cdef class TimestampValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
+            CTimestampType* dtype = <CTimestampType*> ap.type().get()
+            int64_t value = ap.Value(self.index)
+
+        if not dtype.timezone().empty():
+            import pytz
+            tzinfo = pytz.timezone(frombytes(dtype.timezone()))
+        else:
+            tzinfo = None
+
+        try:
+            converter = DATETIME_CONVERSION_FUNCTIONS[dtype.unit()]
+        except KeyError:
+            raise ValueError(
+                'Cannot convert nanosecond timestamps without pandas'
+            )
+        return converter(value, tzinfo=tzinfo)
+
+
+cdef class FloatValue(ArrayValue):
+
+    def as_py(self):
+        cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class DoubleValue(ArrayValue):
+
+    def as_py(self):
+        cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
+        return ap.Value(self.index)
+
+
+cdef class DecimalValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CDecimalArray* ap = <CDecimalArray*> self.sp_array.get()
+            c_string s = ap.FormatValue(self.index)
+        return _pydecimal.Decimal(s.decode('utf8'))
+
+
+cdef class StringValue(ArrayValue):
+
+    def as_py(self):
+        cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
+        return ap.GetString(self.index).decode('utf-8')
+
+
+cdef class BinaryValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            const uint8_t* ptr
+            int32_t length
+            CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+
+        ptr = ap.GetValue(self.index, &length)
+        return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+
+
+cdef class ListValue(ArrayValue):
+
+    def __len__(self):
+        return self.ap.value_length(self.index)
+
+    def __getitem__(self, i):
+        return self.getitem(i)
+
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self.getitem(i)
+        raise StopIteration
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+        self.ap = <CListArray*> sp_array.get()
+        self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
+
+    cdef getitem(self, int64_t i):
+        cdef int64_t j = self.ap.value_offset(self.index) + i
+        return box_scalar(self.value_type, self.ap.values(), j)
+
+    def as_py(self):
+        cdef:
+            int64_t j
+            list result = []
+
+        for j in range(len(self)):
+            result.append(self.getitem(j).as_py())
+
+        return result
+
+
+cdef class FixedSizeBinaryValue(ArrayValue):
+
+    def as_py(self):
+        cdef:
+            CFixedSizeBinaryArray* ap
+            CFixedSizeBinaryType* ap_type
+            int32_t length
+            const char* data
+        ap = <CFixedSizeBinaryArray*> self.sp_array.get()
+        ap_type = <CFixedSizeBinaryType*> ap.type().get()
+        length = ap_type.byte_width()
+        data = <const char*> ap.GetValue(self.index)
+        return cp.PyBytes_FromStringAndSize(data, length)
+
+
+cdef class StructValue(ArrayValue):
+    def as_py(self):
+        cdef:
+            CStructArray* ap
+            vector[shared_ptr[CField]] child_fields = self.type.type.children()
+        ap = <CStructArray*> self.sp_array.get()
+        wrapped_arrays = (pyarrow_wrap_array(ap.field(i))
+                          for i in range(ap.num_fields()))
+        child_names = (child.get().name() for child in child_fields)
+        # Return the struct as a dict
+        return {
+            frombytes(name): child_array[self.index].as_py()
+            for name, child_array in
+            zip(child_names, wrapped_arrays)
+        }
+
+cdef dict _scalar_classes = {
+    _Type_BOOL: BooleanValue,
+    _Type_UINT8: Int8Value,
+    _Type_UINT16: Int16Value,
+    _Type_UINT32: Int32Value,
+    _Type_UINT64: Int64Value,
+    _Type_INT8: Int8Value,
+    _Type_INT16: Int16Value,
+    _Type_INT32: Int32Value,
+    _Type_INT64: Int64Value,
+    _Type_DATE32: Date32Value,
+    _Type_DATE64: Date64Value,
+    _Type_TIME32: Time32Value,
+    _Type_TIME64: Time64Value,
+    _Type_TIMESTAMP: TimestampValue,
+    _Type_FLOAT: FloatValue,
+    _Type_DOUBLE: DoubleValue,
+    _Type_LIST: ListValue,
+    _Type_BINARY: BinaryValue,
+    _Type_STRING: StringValue,
+    _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
+    _Type_DECIMAL: DecimalValue,
+    _Type_STRUCT: StructValue,
+}
+
+cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
+                       int64_t index):
+    cdef ArrayValue val
+    if type.type.id() == _Type_NA:
+        return NA
+    elif sp_array.get().IsNull(index):
+        return NA
+    else:
+        val = _scalar_classes[type.type.id()]()
+        val.init(type, sp_array, index)
+        return val
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index ed8153189175b..413a3be7ae728 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -223,8 +223,8 @@ def test_simple_type_construction():
         (pa.decimal(18, 3), 'decimal'),
         (pa.timestamp('ms'), 'datetime'),
         (pa.timestamp('us', 'UTC'), 'datetimetz'),
-        pytest.mark.xfail((pa.time32('s'), None), raises=NotImplementedError),
-        pytest.mark.xfail((pa.time64('us'), None), raises=NotImplementedError),
+        (pa.time32('s'), 'time'),
+        (pa.time64('us'), 'time')
     ]
 )
 def test_logical_type(type, expected):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index b8b85ca7632a1..5b84817a7f6b2 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -646,6 +646,95 @@ def test_decimal_128_to_pandas(self):
         df = converted.to_pandas()
         tm.assert_frame_equal(df, expected)
 
+    def test_pytime_from_pandas(self):
+        pytimes = [datetime.time(1, 2, 3, 1356),
+                   datetime.time(4, 5, 6, 1356)]
+
+        # microseconds
+        t1 = pa.time64('us')
+
+        aobjs = np.array(pytimes + [None], dtype=object)
+        parr = pa.Array.from_pandas(aobjs)
+        assert parr.type == t1
+        assert parr[0].as_py() == pytimes[0]
+        assert parr[1].as_py() == pytimes[1]
+        assert parr[2] is pa.NA
+
+        # DataFrame
+        df = pd.DataFrame({'times': aobjs})
+        batch = pa.RecordBatch.from_pandas(df)
+        assert batch[0].equals(parr)
+
+        # Test ndarray of int64 values
+        arr = np.array([_pytime_to_micros(v) for v in pytimes],
+                       dtype='int64')
+
+        a1 = pa.Array.from_pandas(arr, type=pa.time64('us'))
+        assert a1[0].as_py() == pytimes[0]
+
+        a2 = pa.Array.from_pandas(arr * 1000, type=pa.time64('ns'))
+        assert a2[0].as_py() == pytimes[0]
+
+        a3 = pa.Array.from_pandas((arr / 1000).astype('i4'),
+                                  type=pa.time32('ms'))
+        assert a3[0].as_py() == pytimes[0].replace(microsecond=1000)
+
+        a4 = pa.Array.from_pandas((arr / 1000000).astype('i4'),
+                                  type=pa.time32('s'))
+        assert a4[0].as_py() == pytimes[0].replace(microsecond=0)
+
+    def test_arrow_time_to_pandas(self):
+        pytimes = [datetime.time(1, 2, 3, 1356),
+                   datetime.time(4, 5, 6, 1356),
+                   datetime.time(0, 0, 0)]
+
+        expected = np.array(pytimes[:2] + [None])
+        expected_ms = np.array([x.replace(microsecond=1000)
+                                for x in pytimes[:2]] +
+                               [None])
+        expected_s = np.array([x.replace(microsecond=0)
+                               for x in pytimes[:2]] +
+                              [None])
+
+        arr = np.array([_pytime_to_micros(v) for v in pytimes],
+                       dtype='int64')
+        arr = np.array([_pytime_to_micros(v) for v in pytimes],
+                       dtype='int64')
+
+        null_mask = np.array([False, False, True], dtype=bool)
+
+        a1 = pa.Array.from_pandas(arr, mask=null_mask, type=pa.time64('us'))
+        a2 = pa.Array.from_pandas(arr * 1000, mask=null_mask,
+                                  type=pa.time64('ns'))
+
+        a3 = pa.Array.from_pandas((arr / 1000).astype('i4'), mask=null_mask,
+                                  type=pa.time32('ms'))
+        a4 = pa.Array.from_pandas((arr / 1000000).astype('i4'), mask=null_mask,
+                                  type=pa.time32('s'))
+
+        names = ['time64[us]', 'time64[ns]', 'time32[ms]', 'time32[s]']
+        batch = pa.RecordBatch.from_arrays([a1, a2, a3, a4], names)
+        arr = a1.to_pandas()
+        assert (arr == expected).all()
+
+        arr = a2.to_pandas()
+        assert (arr == expected).all()
+
+        arr = a3.to_pandas()
+        assert (arr == expected_ms).all()
+
+        arr = a4.to_pandas()
+        assert (arr == expected_s).all()
+
+        df = batch.to_pandas()
+        expected_df = pd.DataFrame({'time64[us]': expected,
+                                    'time64[ns]': expected,
+                                    'time32[ms]': expected_ms,
+                                    'time32[s]': expected_s},
+                                   columns=names)
+
+        tm.assert_frame_equal(df, expected_df)
+
     def test_all_nones(self):
         def _check_series(s):
             converted = pa.Array.from_pandas(s)
@@ -782,3 +871,20 @@ def test_decimal_metadata(self):
         assert data_column['pandas_type'] == 'decimal'
         assert data_column['numpy_type'] == 'object'
         assert data_column['metadata'] == {'precision': 26, 'scale': 11}
+
+
+def _pytime_from_micros(val):
+    microseconds = val % 1000000
+    val //= 1000000
+    seconds = val % 60
+    val //= 60
+    minutes = val % 60
+    hours = val // 60
+    return datetime.time(hours, minutes, seconds, microseconds)
+
+
+def _pytime_to_micros(pytime):
+    return (pytime.hour * 3600000000 +
+            pytime.minute * 60000000 +
+            pytime.second * 1000000 +
+            pytime.microsecond)
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
new file mode 100644
index 0000000000000..0ae25c8655ef8
--- /dev/null
+++ b/python/pyarrow/types.pxi
@@ -0,0 +1,776 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# These are imprecise because the type (in pandas 0.x) depends on the presence
+# of nulls
+cdef dict _pandas_type_map = {
+    _Type_NA: np.float64,  # NaNs
+    _Type_BOOL: np.bool_,
+    _Type_INT8: np.int8,
+    _Type_INT16: np.int16,
+    _Type_INT32: np.int32,
+    _Type_INT64: np.int64,
+    _Type_UINT8: np.uint8,
+    _Type_UINT16: np.uint16,
+    _Type_UINT32: np.uint32,
+    _Type_UINT64: np.uint64,
+    _Type_HALF_FLOAT: np.float16,
+    _Type_FLOAT: np.float32,
+    _Type_DOUBLE: np.float64,
+    _Type_DATE32: np.dtype('datetime64[ns]'),
+    _Type_DATE64: np.dtype('datetime64[ns]'),
+    _Type_TIMESTAMP: np.dtype('datetime64[ns]'),
+    _Type_BINARY: np.object_,
+    _Type_FIXED_SIZE_BINARY: np.object_,
+    _Type_STRING: np.object_,
+    _Type_LIST: np.object_,
+    _Type_DECIMAL: np.object_,
+}
+
+
+cdef class DataType:
+
+    def __cinit__(self):
+        pass
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        self.sp_type = type
+        self.type = type.get()
+
+    property id:
+
+        def __get__(self):
+            return self.type.id()
+
+    def __str__(self):
+        if self.type is NULL:
+            raise TypeError(
+                '{} is incomplete. The correct way to construct types is '
+                'through public API functions named '
+                'pyarrow.int64, pyarrow.list_, etc.'.format(
+                    type(self).__name__
+                )
+            )
+        return frombytes(self.type.ToString())
+
+    def __repr__(self):
+        return '{0.__class__.__name__}({0})'.format(self)
+
+    def __richcmp__(DataType self, DataType other, int op):
+        if op == cp.Py_EQ:
+            return self.type.Equals(deref(other.type))
+        elif op == cp.Py_NE:
+            return not self.type.Equals(deref(other.type))
+        else:
+            raise TypeError('Invalid comparison')
+
+    def to_pandas_dtype(self):
+        """
+        Return the NumPy dtype that would be used for storing this
+        """
+        cdef Type type_id = self.type.id()
+        if type_id in _pandas_type_map:
+            return _pandas_type_map[type_id]
+        else:
+            raise NotImplementedError(str(self))
+
+
+cdef class DictionaryType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.dict_type = <const CDictionaryType*> type.get()
+
+
+cdef class ListType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.list_type = <const CListType*> type.get()
+
+    property value_type:
+
+        def __get__(self):
+            return pyarrow_wrap_data_type(self.list_type.value_type())
+
+
+cdef class TimestampType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.ts_type = <const CTimestampType*> type.get()
+
+    property unit:
+
+        def __get__(self):
+            return timeunit_to_string(self.ts_type.unit())
+
+    property tz:
+
+        def __get__(self):
+            if self.ts_type.timezone().size() > 0:
+                return frombytes(self.ts_type.timezone())
+            else:
+                return None
+
+
+cdef class Time32Type(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.time_type = <const CTime32Type*> type.get()
+
+    property unit:
+
+        def __get__(self):
+            return timeunit_to_string(self.time_type.unit())
+
+
+cdef class Time64Type(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.time_type = <const CTime64Type*> type.get()
+
+    property unit:
+
+        def __get__(self):
+            return timeunit_to_string(self.time_type.unit())
+
+
+cdef class FixedSizeBinaryType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.fixed_size_binary_type = (
+            <const CFixedSizeBinaryType*> type.get())
+
+    property byte_width:
+
+        def __get__(self):
+            return self.fixed_size_binary_type.byte_width()
+
+
+cdef class DecimalType(FixedSizeBinaryType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+        self.decimal_type = <const CDecimalType*> type.get()
+
+    property precision:
+
+        def __get__(self):
+            return self.decimal_type.precision()
+
+    property scale:
+
+        def __get__(self):
+            return self.decimal_type.scale()
+
+
+cdef class Field:
+    """
+    Represents a named field, with a data type, nullability, and optional
+    metadata
+
+    Notes
+    -----
+    Do not use this class's constructor directly; use pyarrow.field
+    """
+    def __cinit__(self):
+        pass
+
+    cdef void init(self, const shared_ptr[CField]& field):
+        self.sp_field = field
+        self.field = field.get()
+        self.type = pyarrow_wrap_data_type(field.get().type())
+
+    def equals(self, Field other):
+        """
+        Test if this field is equal to the other
+        """
+        return self.field.Equals(deref(other.field))
+
+    def __str__(self):
+        self._check_null()
+        return 'pyarrow.Field<{0}>'.format(frombytes(self.field.ToString()))
+
+    def __repr__(self):
+        return self.__str__()
+
+    property nullable:
+
+        def __get__(self):
+            self._check_null()
+            return self.field.nullable()
+
+    property name:
+
+        def __get__(self):
+            self._check_null()
+            return frombytes(self.field.name())
+
+    property metadata:
+
+        def __get__(self):
+            self._check_null()
+            return box_metadata(self.field.metadata().get())
+
+    def _check_null(self):
+        if self.field == NULL:
+            raise ReferenceError(
+                'Field not initialized (references NULL pointer)')
+
+    def add_metadata(self, dict metadata):
+        """
+        Add metadata as dict of string keys and values to Field
+
+        Parameters
+        ----------
+        metadata : dict
+            Keys and values must be string-like / coercible to bytes
+
+        Returns
+        -------
+        field : pyarrow.Field
+        """
+        cdef shared_ptr[CKeyValueMetadata] c_meta
+        convert_metadata(metadata, &c_meta)
+
+        cdef shared_ptr[CField] new_field
+        with nogil:
+            check_status(self.field.AddMetadata(c_meta, &new_field))
+
+        return pyarrow_wrap_field(new_field)
+
+    def remove_metadata(self):
+        """
+        Create new field without metadata, if any
+
+        Returns
+        -------
+        field : pyarrow.Field
+        """
+        cdef shared_ptr[CField] new_field
+        with nogil:
+            new_field = self.field.RemoveMetadata()
+        return pyarrow_wrap_field(new_field)
+
+
+cdef class Schema:
+
+    def __cinit__(self):
+        pass
+
+    def __len__(self):
+        return self.schema.num_fields()
+
+    def __getitem__(self, int64_t i):
+
+        cdef:
+            Field result = Field()
+            int64_t num_fields = self.schema.num_fields()
+            int64_t index
+
+        if not -num_fields <= i < num_fields:
+            raise IndexError(
+                'Schema field index {:d} is out of range'.format(i)
+            )
+
+        index = i if i >= 0 else num_fields + i
+        assert index >= 0
+
+        result.init(self.schema.field(index))
+        result.type = pyarrow_wrap_data_type(result.field.type())
+
+        return result
+
+    cdef void init(self, const vector[shared_ptr[CField]]& fields):
+        self.schema = new CSchema(fields)
+        self.sp_schema.reset(self.schema)
+
+    cdef void init_schema(self, const shared_ptr[CSchema]& schema):
+        self.schema = schema.get()
+        self.sp_schema = schema
+
+    property names:
+
+        def __get__(self):
+            cdef int i
+            result = []
+            for i in range(self.schema.num_fields()):
+                name = frombytes(self.schema.field(i).get().name())
+                result.append(name)
+            return result
+
+    property metadata:
+
+        def __get__(self):
+            return box_metadata(self.schema.metadata().get())
+
+    def equals(self, other):
+        """
+        Test if this schema is equal to the other
+        """
+        cdef Schema _other
+        _other = other
+
+        return self.sp_schema.get().Equals(deref(_other.schema))
+
+    def field_by_name(self, name):
+        """
+        Access a field by its name rather than the column index.
+
+        Parameters
+        ----------
+        name: str
+
+        Returns
+        -------
+        field: pyarrow.Field
+        """
+        return pyarrow_wrap_field(self.schema.GetFieldByName(tobytes(name)))
+
+    def get_field_index(self, name):
+        return self.schema.GetFieldIndex(tobytes(name))
+
+    def add_metadata(self, dict metadata):
+        """
+        Add metadata as dict of string keys and values to Schema
+
+        Parameters
+        ----------
+        metadata : dict
+            Keys and values must be string-like / coercible to bytes
+
+        Returns
+        -------
+        schema : pyarrow.Schema
+        """
+        cdef shared_ptr[CKeyValueMetadata] c_meta
+        convert_metadata(metadata, &c_meta)
+
+        cdef shared_ptr[CSchema] new_schema
+        with nogil:
+            check_status(self.schema.AddMetadata(c_meta, &new_schema))
+
+        return pyarrow_wrap_schema(new_schema)
+
+    def remove_metadata(self):
+        """
+        Create new schema without metadata, if any
+
+        Returns
+        -------
+        schema : pyarrow.Schema
+        """
+        cdef shared_ptr[CSchema] new_schema
+        with nogil:
+            new_schema = self.schema.RemoveMetadata()
+        return pyarrow_wrap_schema(new_schema)
+
+    def __str__(self):
+        return frombytes(self.schema.ToString())
+
+    def __repr__(self):
+        return self.__str__()
+
+
+cdef dict box_metadata(const CKeyValueMetadata* metadata):
+    cdef unordered_map[c_string, c_string] result
+    if metadata != nullptr:
+        metadata.ToUnorderedMap(&result)
+        return result
+    else:
+        return None
+
+
+cdef dict _type_cache = {}
+
+
+cdef DataType primitive_type(Type type):
+    if type in _type_cache:
+        return _type_cache[type]
+
+    cdef DataType out = DataType()
+    out.init(GetPrimitiveType(type))
+
+    _type_cache[type] = out
+    return out
+
+#------------------------------------------------------------
+# Type factory functions
+
+cdef int convert_metadata(dict metadata,
+                          shared_ptr[CKeyValueMetadata]* out) except -1:
+    cdef:
+        shared_ptr[CKeyValueMetadata] meta = (
+            make_shared[CKeyValueMetadata]())
+        c_string key, value
+
+    for py_key, py_value in metadata.items():
+        key = tobytes(py_key)
+        value = tobytes(py_value)
+        meta.get().Append(key, value)
+    out[0] = meta
+    return 0
+
+
+def field(name, DataType type, bint nullable=True, dict metadata=None):
+    """
+    Create a pyarrow.Field instance
+
+    Parameters
+    ----------
+    name : string or bytes
+    type : pyarrow.DataType
+    nullable : boolean, default True
+    metadata : dict, default None
+        Keys and values must be coercible to bytes
+
+    Returns
+    -------
+    field : pyarrow.Field
+    """
+    cdef:
+        shared_ptr[CKeyValueMetadata] c_meta
+        Field result = Field()
+
+    if metadata is not None:
+        convert_metadata(metadata, &c_meta)
+
+    result.sp_field.reset(new CField(tobytes(name), type.sp_type,
+                                     nullable, c_meta))
+    result.field = result.sp_field.get()
+    result.type = type
+    return result
+
+
+cdef set PRIMITIVE_TYPES = set([
+    _Type_NA, _Type_BOOL,
+    _Type_UINT8, _Type_INT8,
+    _Type_UINT16, _Type_INT16,
+    _Type_UINT32, _Type_INT32,
+    _Type_UINT64, _Type_INT64,
+    _Type_TIMESTAMP, _Type_DATE32,
+    _Type_TIME32, _Type_TIME64,
+    _Type_DATE64,
+    _Type_HALF_FLOAT,
+    _Type_FLOAT,
+    _Type_DOUBLE])
+
+
+def null():
+    return primitive_type(_Type_NA)
+
+
+def bool_():
+    return primitive_type(_Type_BOOL)
+
+
+def uint8():
+    return primitive_type(_Type_UINT8)
+
+
+def int8():
+    return primitive_type(_Type_INT8)
+
+
+def uint16():
+    return primitive_type(_Type_UINT16)
+
+
+def int16():
+    return primitive_type(_Type_INT16)
+
+
+def uint32():
+    return primitive_type(_Type_UINT32)
+
+
+def int32():
+    return primitive_type(_Type_INT32)
+
+
+def uint64():
+    return primitive_type(_Type_UINT64)
+
+
+def int64():
+    return primitive_type(_Type_INT64)
+
+
+cdef dict _timestamp_type_cache = {}
+cdef dict _time_type_cache = {}
+
+
+cdef timeunit_to_string(TimeUnit unit):
+    if unit == TimeUnit_SECOND:
+        return 's'
+    elif unit == TimeUnit_MILLI:
+        return 'ms'
+    elif unit == TimeUnit_MICRO:
+        return 'us'
+    elif unit == TimeUnit_NANO:
+        return 'ns'
+
+
+def timestamp(unit_str, tz=None):
+    cdef:
+        TimeUnit unit
+        c_string c_timezone
+
+    if unit_str == "s":
+        unit = TimeUnit_SECOND
+    elif unit_str == 'ms':
+        unit = TimeUnit_MILLI
+    elif unit_str == 'us':
+        unit = TimeUnit_MICRO
+    elif unit_str == 'ns':
+        unit = TimeUnit_NANO
+    else:
+        raise ValueError('Invalid TimeUnit string')
+
+    cdef TimestampType out = TimestampType()
+
+    if tz is None:
+        out.init(ctimestamp(unit))
+        if unit in _timestamp_type_cache:
+            return _timestamp_type_cache[unit]
+        _timestamp_type_cache[unit] = out
+    else:
+        if not isinstance(tz, six.string_types):
+            tz = tz.zone
+
+        c_timezone = tobytes(tz)
+        out.init(ctimestamp(unit, c_timezone))
+
+    return out
+
+
+def time32(unit_str):
+    cdef:
+        TimeUnit unit
+        c_string c_timezone
+
+    if unit_str == "s":
+        unit = TimeUnit_SECOND
+    elif unit_str == 'ms':
+        unit = TimeUnit_MILLI
+    else:
+        raise ValueError('Invalid TimeUnit for time32: {}'.format(unit_str))
+
+    cdef Time32Type out
+    if unit in _time_type_cache:
+        return _time_type_cache[unit]
+    else:
+        out = Time32Type()
+        out.init(ctime32(unit))
+        _time_type_cache[unit] = out
+        return out
+
+
+def time64(unit_str):
+    cdef:
+        TimeUnit unit
+        c_string c_timezone
+
+    if unit_str == "us":
+        unit = TimeUnit_MICRO
+    elif unit_str == 'ns':
+        unit = TimeUnit_NANO
+    else:
+        raise ValueError('Invalid TimeUnit for time64: {}'.format(unit_str))
+
+    cdef Time64Type out
+    if unit in _time_type_cache:
+        return _time_type_cache[unit]
+    else:
+        out = Time64Type()
+        out.init(ctime64(unit))
+        _time_type_cache[unit] = out
+        return out
+
+
+def date32():
+    return primitive_type(_Type_DATE32)
+
+
+def date64():
+    return primitive_type(_Type_DATE64)
+
+
+def float16():
+    return primitive_type(_Type_HALF_FLOAT)
+
+
+def float32():
+    return primitive_type(_Type_FLOAT)
+
+
+def float64():
+    return primitive_type(_Type_DOUBLE)
+
+
+cpdef DataType decimal(int precision, int scale=0):
+    cdef shared_ptr[CDataType] decimal_type
+    decimal_type.reset(new CDecimalType(precision, scale))
+    return pyarrow_wrap_data_type(decimal_type)
+
+
+def string():
+    """
+    UTF8 string
+    """
+    return primitive_type(_Type_STRING)
+
+
+def binary(int length=-1):
+    """Binary (PyBytes-like) type
+
+    Parameters
+    ----------
+    length : int, optional, default -1
+        If length == -1 then return a variable length binary type. If length is
+        greater than or equal to 0 then return a fixed size binary type of
+        width `length`.
+    """
+    if length == -1:
+        return primitive_type(_Type_BINARY)
+
+    cdef shared_ptr[CDataType] fixed_size_binary_type
+    fixed_size_binary_type.reset(new CFixedSizeBinaryType(length))
+    return pyarrow_wrap_data_type(fixed_size_binary_type)
+
+
+cpdef ListType list_(value_type):
+    """
+    Create ListType instance from child data type or field
+
+    Parameters
+    ----------
+    value_type : DataType or Field
+
+    Returns
+    -------
+    list_type : DataType
+    """
+    cdef:
+        DataType data_type
+        Field field
+        shared_ptr[CDataType] list_type
+        ListType out = ListType()
+
+    if isinstance(value_type, DataType):
+        list_type.reset(new CListType((<DataType> value_type).sp_type))
+    elif isinstance(value_type, Field):
+        list_type.reset(new CListType((<Field> value_type).sp_field))
+    else:
+        raise ValueError('List requires DataType or Field')
+
+    out.init(list_type)
+    return out
+
+
+cpdef DictionaryType dictionary(DataType index_type, Array dictionary):
+    """
+    Dictionary (categorical, or simply encoded) type
+
+    Parameters
+    ----------
+    index_type : DataType
+    dictionary : Array
+
+    Returns
+    -------
+    type : DictionaryType
+    """
+    cdef DictionaryType out = DictionaryType()
+    cdef shared_ptr[CDataType] dict_type
+    dict_type.reset(new CDictionaryType(index_type.sp_type,
+                                        dictionary.sp_array))
+    out.init(dict_type)
+    return out
+
+
+def struct(fields):
+    """
+    Create StructType instance from fields
+
+    Parameters
+    ----------
+    fields : sequence of Field values
+
+    Examples
+    --------
+    import pyarrow as pa
+    fields = [
+        pa.field('f1', pa.int32()),
+        pa.field('f2', pa.string())
+    ]
+    struct_type = pa.struct(fields)
+
+    Returns
+    -------
+    type : DataType
+    """
+    cdef:
+        Field field
+        vector[shared_ptr[CField]] c_fields
+        cdef shared_ptr[CDataType] struct_type
+
+    for field in fields:
+        c_fields.push_back(field.sp_field)
+
+    struct_type.reset(new CStructType(c_fields))
+    return pyarrow_wrap_data_type(struct_type)
+
+
+def schema(fields):
+    """
+    Construct pyarrow.Schema from collection of fields
+
+    Parameters
+    ----------
+    field : list or iterable
+
+    Returns
+    -------
+    schema : pyarrow.Schema
+    """
+    cdef:
+        Schema result
+        Field field
+        vector[shared_ptr[CField]] c_fields
+
+    for i, field in enumerate(fields):
+        c_fields.push_back(field.sp_field)
+
+    result = Schema()
+    result.init(c_fields)
+    return result
+
+
+def from_numpy_dtype(object dtype):
+    """
+    Convert NumPy dtype to pyarrow.DataType
+    """
+    cdef shared_ptr[CDataType] c_type
+    with nogil:
+        check_status(NumPyDtypeToArrow(dtype, &c_type))
+
+    return pyarrow_wrap_data_type(c_type)

From cdf7db9105e713e64de1a063ffe36e3f8d65227f Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 16 Jul 2017 17:30:33 -0400
Subject: [PATCH 0815/1644] ARROW-1223: [GLib] Fix function name that returns
 wrapped object

We're using `XXX_get_raw()` as name for function that returns wrapped object.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #852 from kou/glib-use-get-raw and squashes the following commits:

e7d84575 [Kouhei Sutou] [GLib] Fix function name that returns wrapped object
---
 c_glib/arrow-glib/input-stream.cpp   | 4 ++--
 c_glib/arrow-glib/input-stream.h     | 2 +-
 c_glib/test/test-gio-input-stream.rb | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 37ac2fad4eca1..40f0c949f73a3 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -561,7 +561,7 @@ garrow_gio_input_stream_new(GInputStream *gio_input_stream)
 }
 
 /**
- * garrow_gio_input_stream_get_gio_input_stream:
+ * garrow_gio_input_stream_get_raw:
  * @input_stream: A #GArrowGIOInputStream.
  *
  * Returns: (transfer none): The wrapped #GInputStream.
@@ -569,7 +569,7 @@ garrow_gio_input_stream_new(GInputStream *gio_input_stream)
  * Since: 0.5.0
  */
 GInputStream *
-garrow_gio_input_stream_get_gio_input_stream(GArrowGIOInputStream *input_stream)
+garrow_gio_input_stream_get_raw(GArrowGIOInputStream *input_stream)
 {
   auto arrow_input_stream =
     garrow_input_stream_get_raw(GARROW_INPUT_STREAM(input_stream));
diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h
index 120839d9fcade..12c7ae700f79d 100644
--- a/c_glib/arrow-glib/input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -279,6 +279,6 @@ struct _GArrowGIOInputStreamClass
 GType garrow_gio_input_stream_get_type(void) G_GNUC_CONST;
 
 GArrowGIOInputStream *garrow_gio_input_stream_new(GInputStream *gio_input_stream);
-GInputStream *garrow_gio_input_stream_get_gio_input_stream(GArrowGIOInputStream *input_stream);
+GInputStream *garrow_gio_input_stream_get_raw(GArrowGIOInputStream *input_stream);
 
 G_END_DECLS
diff --git a/c_glib/test/test-gio-input-stream.rb b/c_glib/test/test-gio-input-stream.rb
index baa978e10f754..a71a370430e6d 100644
--- a/c_glib/test/test-gio-input-stream.rb
+++ b/c_glib/test/test-gio-input-stream.rb
@@ -48,6 +48,6 @@ def test_reader_backend
   def test_getter
     input_stream = Gio::MemoryInputStream.new("Hello")
     input = Arrow::GIOInputStream.new(input_stream)
-    assert_equal(input_stream, input.gio_input_stream)
+    assert_equal(input_stream, input.raw)
   end
 end

From d538426918939efd96f67472a9d66f599d6c6692 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 17 Jul 2017 14:31:31 +0200
Subject: [PATCH 0816/1644] ARROW-1228: [GLib] Fix test file name

The test file is for Arrow::BufferOutputStream. So test file name should
be the same.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #857 from kou/glib-test-fix-file-name and squashes the following commits:

9225e75 [Kouhei Sutou] [GLib] Fix test file name
---
 .../{test-buffer-output-file.rb => test-buffer-output-stream.rb}  | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename c_glib/test/{test-buffer-output-file.rb => test-buffer-output-stream.rb} (100%)

diff --git a/c_glib/test/test-buffer-output-file.rb b/c_glib/test/test-buffer-output-stream.rb
similarity index 100%
rename from c_glib/test/test-buffer-output-file.rb
rename to c_glib/test/test-buffer-output-stream.rb

From 8644ee1778b3512d15ced89adacde1a3cafb50c0 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 17 Jul 2017 14:33:28 +0200
Subject: [PATCH 0817/1644] ARROW-1227: [GLib] Support GOutputStream

Author: Kouhei Sutou <kou@clear-code.com>

Closes #856 from kou/glib-support-goutput-stream and squashes the following commits:

955fecc [Kouhei Sutou] [GLib] Support GOutputStream
---
 c_glib/arrow-glib/error.cpp           |  16 +++
 c_glib/arrow-glib/error.hpp           |   3 +
 c_glib/arrow-glib/input-stream.cpp    |  42 ++++----
 c_glib/arrow-glib/output-stream.cpp   | 142 ++++++++++++++++++++++++++
 c_glib/arrow-glib/output-stream.h     |  52 +++++++++-
 c_glib/test/test-gio-output-stream.rb |  53 ++++++++++
 6 files changed, 287 insertions(+), 21 deletions(-)
 create mode 100644 c_glib/test/test-gio-output-stream.rb

diff --git a/c_glib/arrow-glib/error.cpp b/c_glib/arrow-glib/error.cpp
index e5d2ad6f6eb40..298f5dfd8f734 100644
--- a/c_glib/arrow-glib/error.cpp
+++ b/c_glib/arrow-glib/error.cpp
@@ -23,6 +23,9 @@
 
 #include <arrow-glib/error.hpp>
 
+#include <iostream>
+#include <sstream>
+
 G_BEGIN_DECLS
 
 /**
@@ -80,3 +83,16 @@ garrow_error_check(GError **error,
     return FALSE;
   }
 }
+
+arrow::Status
+garrow_error_to_status(GError *error,
+                       arrow::StatusCode code,
+                       const char *context)
+{
+  std::stringstream message;
+  message << context << ": " << g_quark_to_string(error->domain);
+  message << "(" << error->code << "): ";
+  message << error->message;
+  g_error_free(error);
+  return arrow::Status(code, message.str());
+}
diff --git a/c_glib/arrow-glib/error.hpp b/c_glib/arrow-glib/error.hpp
index dad27bd5c9b5a..4149365bbab9e 100644
--- a/c_glib/arrow-glib/error.hpp
+++ b/c_glib/arrow-glib/error.hpp
@@ -26,3 +26,6 @@
 gboolean garrow_error_check(GError **error,
                             const arrow::Status &status,
                             const char *context);
+arrow::Status garrow_error_to_status(GError *error,
+                                     arrow::StatusCode code,
+                                     const char *context);
diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 40f0c949f73a3..645ce8aa2295c 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -32,9 +32,6 @@
 #include <arrow-glib/readable.hpp>
 #include <arrow-glib/tensor.hpp>
 
-#include <iostream>
-#include <sstream>
-
 G_BEGIN_DECLS
 
 /**
@@ -52,6 +49,9 @@ G_BEGIN_DECLS
  *
  * #GArrowMemoryMappedFile is a class to read data in file by mapping
  * the file on memory. It supports zero copy.
+ *
+ * #GArrowGIOInputStream is a class for `GInputStream` based input
+ * stream.
  */
 
 typedef struct GArrowInputStreamPrivate_ {
@@ -399,7 +399,9 @@ namespace garrow {
       if (g_input_stream_close(input_stream_, NULL, &error)) {
         return arrow::Status::OK();
       } else {
-        return io_error_to_status(error, "[gio-input-stream][close]");
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      "[gio-input-stream][close]");
       }
     }
 
@@ -426,7 +428,9 @@ namespace garrow {
                                           NULL,
                                           &error);
       if (*n_read_bytes == -1) {
-        return io_error_to_status(error, "[gio-input-stream][read]");
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      "[gio-input-stream][read]");
       } else {
         return arrow::Status::OK();
       }
@@ -445,7 +449,9 @@ namespace garrow {
                                               NULL,
                                               &error);
       if (n_read_bytes == -1) {
-        return io_error_to_status(error, "[gio-input-stream][read][buffer]");
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      "[gio-input-stream][read][buffer]");
       } else {
         if (n_read_bytes < n_bytes) {
           ARROW_RETURN_NOT_OK(buffer->Resize(n_read_bytes));
@@ -472,7 +478,9 @@ namespace garrow {
                           &error)) {
         return arrow::Status::OK();
       } else {
-        return io_error_to_status(error, "[gio-input-stream][seek]");
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      "[gio-input-stream][seek]");
       }
     }
 
@@ -492,7 +500,9 @@ namespace garrow {
                            G_SEEK_END,
                            NULL,
                            &error)) {
-        return io_error_to_status(error, "[gio-input-stream][size][seek]");
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      "[gio-input-stream][size][seek]");
       }
       *size = g_seekable_tell(G_SEEKABLE(input_stream_));
       if (!g_seekable_seek(G_SEEKABLE(input_stream_),
@@ -500,8 +510,9 @@ namespace garrow {
                            G_SEEK_SET,
                            NULL,
                            &error)) {
-        return io_error_to_status(error,
-                                  "[gio-input-stream][size][seek][restore]");
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      "[gio-input-stream][size][seek][restore]");
       }
       return arrow::Status::OK();
     }
@@ -512,15 +523,6 @@ namespace garrow {
 
   private:
     GInputStream *input_stream_;
-
-    arrow::Status io_error_to_status(GError *error, const char *context) {
-      std::stringstream message;
-      message << context << ": " << g_quark_to_string(error->domain);
-      message << "(" << error->code << "): ";
-      message << error->message;
-      g_error_free(error);
-      return arrow::Status::IOError(message.str());
-    }
   };
 };
 
@@ -544,7 +546,7 @@ garrow_gio_input_stream_class_init(GArrowGIOInputStreamClass *klass)
  * garrow_gio_input_stream_new:
  * @gio_input_stream: The stream to be read.
  *
- * Returns: A newly created #GArrowGIOInputStream.
+ * Returns: (transfer full): A newly created #GArrowGIOInputStream.
  *
  * Since: 0.5.0
  */
diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index ffb6fecb5c16b..7f37864028651 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -31,6 +31,9 @@
 #include <arrow-glib/tensor.hpp>
 #include <arrow-glib/writeable.hpp>
 
+#include <iostream>
+#include <sstream>
+
 G_BEGIN_DECLS
 
 /**
@@ -45,6 +48,9 @@ G_BEGIN_DECLS
  * #GArrowFileOutputStream is a class for file output stream.
  *
  * #GArrowBufferOutputStream is a class for buffer output stream.
+ *
+ * #GArrowGIOOutputStream is a class for `GOutputStream` based output
+ * stream.
  */
 
 typedef struct GArrowOutputStreamPrivate_ {
@@ -275,8 +281,144 @@ garrow_buffer_output_stream_new(GArrowResizableBuffer *buffer)
     std::make_shared<arrow::io::BufferOutputStream>(arrow_resizable_buffer);
   return garrow_buffer_output_stream_new_raw(&arrow_buffer_output_stream);
 }
+
 G_END_DECLS
 
+
+namespace garrow {
+  class GIOOutputStream : public arrow::io::OutputStream {
+  public:
+    GIOOutputStream(GOutputStream *output_stream) :
+      output_stream_(output_stream) {
+      g_object_ref(output_stream_);
+    }
+
+    ~GIOOutputStream() {
+      g_object_unref(output_stream_);
+    }
+
+    GOutputStream *get_output_stream() {
+      return output_stream_;
+    }
+
+    arrow::Status Close() override {
+      GError *error = NULL;
+      if (g_output_stream_close(output_stream_, NULL, &error)) {
+        return arrow::Status::OK();
+      } else {
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      "[gio-output-stream][close]");
+      }
+    }
+
+    arrow::Status Tell(int64_t *position) override {
+      if (!G_IS_SEEKABLE(output_stream_)) {
+        std::string message("[gio-output-stream][tell] "
+                            "not seekable output stream: <");
+        message += G_OBJECT_CLASS_NAME(G_OBJECT_GET_CLASS(output_stream_));
+        message += ">";
+        return arrow::Status::NotImplemented(message);
+      }
+
+      *position = g_seekable_tell(G_SEEKABLE(output_stream_));
+      return arrow::Status::OK();
+    }
+
+    arrow::Status Write(const uint8_t *data,
+                        int64_t n_bytes) override {
+      GError *error = NULL;
+      gsize n_written_bytes;
+      auto successed = g_output_stream_write_all(output_stream_,
+                                                 data,
+                                                 n_bytes,
+                                                 &n_written_bytes,
+                                                 NULL,
+                                                 &error);
+      if (successed) {
+        return arrow::Status::OK();
+      } else {
+        std::stringstream message("[gio-output-stream][write]");
+        message << "[" << n_written_bytes << "/" << n_bytes << "]";
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      message.str().c_str());
+      }
+    }
+
+    arrow::Status Flush() override {
+      GError *error = NULL;
+      auto successed = g_output_stream_flush(output_stream_, NULL, &error);
+      if (successed) {
+        return arrow::Status::OK();
+      } else {
+        return garrow_error_to_status(error,
+                                      arrow::StatusCode::IOError,
+                                      "[gio-output-stream][flush]");
+      }
+    }
+
+  private:
+    GOutputStream *output_stream_;
+  };
+};
+
+G_BEGIN_DECLS
+
+G_DEFINE_TYPE(GArrowGIOOutputStream,
+              garrow_gio_output_stream,
+              GARROW_TYPE_OUTPUT_STREAM);
+
+static void
+garrow_gio_output_stream_init(GArrowGIOOutputStream *gio_output_stream)
+{
+}
+
+static void
+garrow_gio_output_stream_class_init(GArrowGIOOutputStreamClass *klass)
+{
+}
+
+/**
+ * garrow_gio_output_stream_new:
+ * @gio_output_stream: The stream to be output.
+ *
+ * Returns: (transfer full): A newly created #GArrowGIOOutputStream.
+ */
+GArrowGIOOutputStream *
+garrow_gio_output_stream_new(GOutputStream *gio_output_stream)
+{
+  auto arrow_output_stream =
+    std::make_shared<garrow::GIOOutputStream>(gio_output_stream);
+  auto object = g_object_new(GARROW_TYPE_GIO_OUTPUT_STREAM,
+                             "output-stream", &arrow_output_stream,
+                             NULL);
+  auto output_stream = GARROW_GIO_OUTPUT_STREAM(object);
+  return output_stream;
+}
+
+/**
+ * garrow_gio_output_stream_get_raw:
+ * @output_stream: A #GArrowGIOOutputStream.
+ *
+ * Returns: (transfer none): The wrapped #GOutputStream.
+ *
+ * Since: 0.5.0
+ */
+GOutputStream *
+garrow_gio_output_stream_get_raw(GArrowGIOOutputStream *output_stream)
+{
+  auto arrow_output_stream =
+    garrow_output_stream_get_raw(GARROW_OUTPUT_STREAM(output_stream));
+  auto arrow_gio_output_stream =
+    std::static_pointer_cast<garrow::GIOOutputStream>(arrow_output_stream);
+  auto gio_output_stream = arrow_gio_output_stream->get_output_stream();
+  return gio_output_stream;
+}
+
+G_END_DECLS
+
+
 GArrowOutputStream *
 garrow_output_stream_new_raw(std::shared_ptr<arrow::io::OutputStream> *arrow_output_stream)
 {
diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h
index c86597bdd26c2..e42ebcde47d6b 100644
--- a/c_glib/arrow-glib/output-stream.h
+++ b/c_glib/arrow-glib/output-stream.h
@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <glib-object.h>
+#include <gio/gio.h>
 
 #include <arrow-glib/buffer.h>
 #include <arrow-glib/tensor.h>
@@ -176,4 +176,54 @@ GType garrow_buffer_output_stream_get_type(void) G_GNUC_CONST;
 
 GArrowBufferOutputStream *garrow_buffer_output_stream_new(GArrowResizableBuffer *buffer);
 
+
+#define GARROW_TYPE_GIO_OUTPUT_STREAM           \
+  (garrow_gio_output_stream_get_type())
+#define GARROW_GIO_OUTPUT_STREAM(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_GIO_OUTPUT_STREAM,    \
+                              GArrowGIOOutputStream))
+#define GARROW_GIO_OUTPUT_STREAM_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_GIO_OUTPUT_STREAM,       \
+                           GArrowGIOOutputStreamClass))
+#define GARROW_IS_GIO_OUTPUT_STREAM(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_GIO_OUTPUT_STREAM))
+#define GARROW_IS_GIO_OUTPUT_STREAM_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_GIO_OUTPUT_STREAM))
+#define GARROW_GIO_OUTPUT_STREAM_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_GIO_OUTPUT_STREAM,     \
+                             GArrowGIOOutputStreamClass))
+
+typedef struct _GArrowGIOOutputStream         GArrowGIOOutputStream;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowGIOOutputStreamClass    GArrowGIOOutputStreamClass;
+#endif
+
+/**
+ * GArrowGIOOutputStream:
+ *
+ * It's an output stream for `GOutputStream`.
+ */
+struct _GArrowGIOOutputStream
+{
+  /*< private >*/
+  GArrowOutputStream parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowGIOOutputStreamClass
+{
+  GArrowOutputStreamClass parent_class;
+};
+#endif
+
+GType garrow_gio_output_stream_get_type(void) G_GNUC_CONST;
+
+GArrowGIOOutputStream *garrow_gio_output_stream_new(GOutputStream *gio_output_stream);
+GOutputStream *garrow_gio_output_stream_get_raw(GArrowGIOOutputStream *output_stream);
+
 G_END_DECLS
diff --git a/c_glib/test/test-gio-output-stream.rb b/c_glib/test/test-gio-output-stream.rb
new file mode 100644
index 0000000000000..adaa8c1b7b2b0
--- /dev/null
+++ b/c_glib/test/test-gio-output-stream.rb
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestGIOOutputStream < Test::Unit::TestCase
+  def test_writer_backend
+    tempfile = Tempfile.open("arrow-gio-output-stream")
+    file = Gio::File.new_for_path(tempfile.path)
+    output_stream = file.append_to(:none)
+    output = Arrow::GIOOutputStream.new(output_stream)
+    begin
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      schema = Arrow::Schema.new([field])
+      file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
+      begin
+        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        file_writer.write_record_batch(record_batch)
+      ensure
+        file_writer.close
+      end
+    ensure
+      output.close
+    end
+
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      file_reader = Arrow::RecordBatchFileReader.new(input)
+      assert_equal(["enabled"],
+                   file_reader.schema.fields.collect(&:name))
+    ensure
+      input.close
+    end
+  end
+
+  def test_getter
+    output_stream = Gio::MemoryOutputStream.new
+    output = Arrow::GIOOutputStream.new(output_stream)
+    assert_equal(output_stream, output.raw)
+  end
+end

From e370174dd4568a25264da1e531af359c205546ab Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Jul 2017 09:00:54 -0400
Subject: [PATCH 0818/1644] ARROW-1222: [Python] Raise exception when passing
 unsupported Python object type to pyarrow.array

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #854 from wesm/ARROW-1222 and squashes the following commits:

f1a9fc40 [Wes McKinney] Raise exception when passing unsupported Python object type to pyarrow.array
---
 cpp/src/arrow/python/builtin_convert.cc | 11 +++++++++--
 python/pyarrow/tests/test_array.py      |  7 +++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 83154bb1675b2..fa1c7348b82ba 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -83,7 +83,7 @@ class ScalarVisitor {
         binary_count_(0),
         unicode_count_(0) {}
 
-  void Visit(PyObject* obj) {
+  Status Visit(PyObject* obj) {
     ++total_count_;
     if (obj == Py_None) {
       ++none_count_;
@@ -103,7 +103,14 @@ class ScalarVisitor {
       ++unicode_count_;
     } else {
       // TODO(wesm): accumulate error information somewhere
+      static std::string supported_types =
+        "bool, float, integer, date, datetime, bytes, unicode";
+      std::stringstream ss;
+      ss << "Error inferring Arrow data type for collection of Python objects. ";
+      RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
+      return Status::Invalid(ss.str());
     }
+    return Status::OK();
   }
 
   std::shared_ptr<DataType> GetType() {
@@ -256,7 +263,7 @@ class SeqVisitor {
         // TODO
       } else {
         ++nesting_histogram_[level];
-        scalars_.Visit(item_ref.obj());
+        return scalars_.Visit(item_ref.obj());
       }
     }
     return Status::OK();
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 413a3be7ae728..1a0ee617be9bb 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import datetime
 import pytest
 import sys
 
@@ -142,6 +143,12 @@ def test_array_slice():
         arr[::2]
 
 
+def test_array_factory_invalid_type():
+    arr = np.array([datetime.timedelta(1), datetime.timedelta(2)])
+    with pytest.raises(ValueError):
+        pa.array(arr)
+
+
 def test_dictionary_from_numpy():
     indices = np.repeat([0, 1, 2], 2)
     dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)

From 5fbfd8e58392439c754cdf19f6f3b27b28303e0a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Jul 2017 09:03:20 -0400
Subject: [PATCH 0819/1644] ARROW-597: [Python] Add read_pandas convenience to
 stream and file reader classes. Add some data type docstrings

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #855 from wesm/ARROW-597 and squashes the following commits:

1c9c3e20 [Wes McKinney] Add read_pandas convenience to stream and file reader classes. Add a bunch of missing API docstrings
---
 python/doc/source/api.rst        |   7 --
 python/doc/source/ipc.rst        |  12 ++
 python/doc/source/pandas.rst     |   2 +
 python/pyarrow/__init__.py       |   1 +
 python/pyarrow/ipc.pxi           |   7 ++
 python/pyarrow/ipc.py            |  23 +++-
 python/pyarrow/scalar.pxi        |   4 +-
 python/pyarrow/tests/test_ipc.py |  35 ++++--
 python/pyarrow/types.pxi         | 206 ++++++++++++++++++++++++-------
 9 files changed, 233 insertions(+), 64 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 400614d6e7b97..c52d400cef1c7 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -67,7 +67,6 @@ Scalar Value Types
    :toctree: generated/
 
    NA
-   NAType
    Scalar
    ArrayValue
    BooleanValue
@@ -210,12 +209,6 @@ Type Classes
    :toctree: generated/
 
    DataType
-   DecimalType
-   DictionaryType
-   FixedSizeBinaryType
-   Time32Type
-   Time64Type
-   TimestampType
    Field
    Schema
 
diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index f0844cd2a9cb5..dca776b792059 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -136,3 +136,15 @@ batches in the file, and can read any at random:
    reader.num_record_batches
    b = reader.get_batch(3)
    b.equals(batch)
+
+Reading from Stream and File Format for pandas
+----------------------------------------------
+
+The stream and file reader classes have a special ``read_pandas`` method to
+simplify reading multiple record batches and converting them to a single
+DataFrame output:
+
+.. ipython:: python
+
+   df = pa.open_file(buf).read_pandas()
+   df
diff --git a/python/doc/source/pandas.rst b/python/doc/source/pandas.rst
index d234e78b7ccb5..765d62a8a6ab9 100644
--- a/python/doc/source/pandas.rst
+++ b/python/doc/source/pandas.rst
@@ -15,6 +15,8 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
+.. _pandas:
+
 Using PyArrow with pandas
 =========================
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index f7cddd0e71057..e3d783aee58b4 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -37,6 +37,7 @@
                          float16, float32, float64,
                          binary, string, decimal,
                          list_, struct, dictionary, field,
+                         DataType, NAType,
                          Field,
                          Schema,
                          schema,
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index d6df30b8619db..31ee578920eae 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -16,6 +16,9 @@
 # under the License.
 
 cdef class Message:
+    """
+    Container for an Arrow IPC message with metadata and optional body
+    """
     cdef:
         unique_ptr[CMessage] message
 
@@ -100,6 +103,10 @@ body length: {2}""".format(self.type, metadata_len, body_len)
 
 
 cdef class MessageReader:
+    """
+    Interface for reading Message objects from some source (like an
+    InputStream)
+    """
     cdef:
         unique_ptr[CMessageReader] reader
 
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index e8ea3acb3c397..f86312867a9f8 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -26,7 +26,26 @@
 import pyarrow.lib as lib
 
 
-class RecordBatchStreamReader(lib._RecordBatchReader):
+class _ReadPandasOption(object):
+
+    def read_pandas(self, **options):
+        """
+        Read contents of stream and convert to pandas.DataFrame using
+        Table.to_pandas
+
+        Parameters
+        ----------
+        **options : arguments to forward to Table.to_pandas
+
+        Returns
+        -------
+        df : pandas.DataFrame
+        """
+        table = self.read_all()
+        return table.to_pandas(**options)
+
+
+class RecordBatchStreamReader(lib._RecordBatchReader, _ReadPandasOption):
     """
     Reader for the Arrow streaming binary format
 
@@ -54,7 +73,7 @@ def __init__(self, sink, schema):
         self._open(sink, schema)
 
 
-class RecordBatchFileReader(lib._RecordBatchFileReader):
+class RecordBatchFileReader(lib._RecordBatchFileReader, _ReadPandasOption):
     """
     Class for reading Arrow record batch data from the Arrow binary file format
 
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index 11ed0efb39b97..dec5341ca4ad1 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -20,7 +20,9 @@ NA = None
 
 
 cdef class NAType(Scalar):
-
+    """
+    Null (NA) value singleton
+    """
     def __cinit__(self):
         global NA
         if NA is not None:
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index b2b90d41bfa08..bcaca6df777b8 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -63,7 +63,7 @@ def write_batches(self):
             batches.append(batch)
 
         writer.close()
-        return batches
+        return frames, batches
 
 
 class TestFile(MessagingTest, unittest.TestCase):
@@ -78,7 +78,7 @@ def test_empty_file(self):
             pa.open_file(buf)
 
     def test_simple_roundtrip(self):
-        batches = self.write_batches()
+        _, batches = self.write_batches()
         file_contents = pa.BufferReader(self._get_source())
 
         reader = pa.open_file(file_contents)
@@ -92,7 +92,7 @@ def test_simple_roundtrip(self):
             assert reader.schema.equals(batches[0].schema)
 
     def test_read_all(self):
-        batches = self.write_batches()
+        _, batches = self.write_batches()
         file_contents = pa.BufferReader(self._get_source())
 
         reader = pa.open_file(file_contents)
@@ -101,6 +101,16 @@ def test_read_all(self):
         expected = pa.Table.from_batches(batches)
         assert result.equals(expected)
 
+    def test_read_pandas(self):
+        frames, _ = self.write_batches()
+
+        file_contents = pa.BufferReader(self._get_source())
+        reader = pa.open_file(file_contents)
+        result = reader.read_pandas()
+
+        expected = pd.concat(frames)
+        assert_frame_equal(result, expected)
+
 
 class TestStream(MessagingTest, unittest.TestCase):
 
@@ -113,7 +123,7 @@ def test_empty_stream(self):
             pa.open_stream(buf)
 
     def test_simple_roundtrip(self):
-        batches = self.write_batches()
+        _, batches = self.write_batches()
         file_contents = pa.BufferReader(self._get_source())
         reader = pa.open_stream(file_contents)
 
@@ -130,7 +140,7 @@ def test_simple_roundtrip(self):
             reader.get_next_batch()
 
     def test_read_all(self):
-        batches = self.write_batches()
+        _, batches = self.write_batches()
         file_contents = pa.BufferReader(self._get_source())
         reader = pa.open_stream(file_contents)
 
@@ -142,7 +152,7 @@ def test_read_all(self):
 class TestMessageReader(MessagingTest, unittest.TestCase):
 
     def _get_example_messages(self):
-        batches = self.write_batches()
+        _, batches = self.write_batches()
         file_contents = self._get_source()
         buf_reader = pa.BufferReader(file_contents)
         reader = pa.MessageReader.open_stream(buf_reader)
@@ -187,6 +197,15 @@ def test_read_record_batch(self):
             read_batch = pa.read_record_batch(message, batch.schema)
             assert read_batch.equals(batch)
 
+    def test_read_pandas(self):
+        frames, _ = self.write_batches()
+        file_contents = pa.BufferReader(self._get_source())
+        reader = pa.open_stream(file_contents)
+        result = reader.read_pandas()
+
+        expected = pd.concat(frames)
+        assert_frame_equal(result, expected)
+
 
 class TestSocket(MessagingTest, unittest.TestCase):
 
@@ -249,7 +268,7 @@ def _get_writer(self, sink, schema):
 
     def test_simple_roundtrip(self):
         self.start_server(do_read_all=False)
-        writer_batches = self.write_batches()
+        _, writer_batches = self.write_batches()
         reader_schema, reader_batches = self.stop_and_get_result()
 
         assert reader_schema.equals(writer_batches[0].schema)
@@ -259,7 +278,7 @@ def test_simple_roundtrip(self):
 
     def test_read_all(self):
         self.start_server(do_read_all=True)
-        writer_batches = self.write_batches()
+        _, writer_batches = self.write_batches()
         _, result = self.stop_and_get_result()
 
         expected = pa.Table.from_batches(writer_batches)
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 0ae25c8655ef8..95bfbfbb0b607 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -43,7 +43,9 @@ cdef dict _pandas_type_map = {
 
 
 cdef class DataType:
-
+    """
+    Base type for Apache Arrow data type instances. Wraps C++ arrow::DataType
+    """
     def __cinit__(self):
         pass
 
@@ -475,42 +477,72 @@ cdef set PRIMITIVE_TYPES = set([
 
 
 def null():
+    """
+    Create instance of null type
+    """
     return primitive_type(_Type_NA)
 
 
 def bool_():
+    """
+    Create instance of boolean type
+    """
     return primitive_type(_Type_BOOL)
 
 
 def uint8():
+    """
+    Create instance of boolean type
+    """
     return primitive_type(_Type_UINT8)
 
 
 def int8():
+    """
+    Create instance of signed int8 type
+    """
     return primitive_type(_Type_INT8)
 
 
 def uint16():
+    """
+    Create instance of unsigned uint16 type
+    """
     return primitive_type(_Type_UINT16)
 
 
 def int16():
+    """
+    Create instance of signed int16 type
+    """
     return primitive_type(_Type_INT16)
 
 
 def uint32():
+    """
+    Create instance of unsigned uint32 type
+    """
     return primitive_type(_Type_UINT32)
 
 
 def int32():
+    """
+    Create instance of signed int32 type
+    """
     return primitive_type(_Type_INT32)
 
 
 def uint64():
+    """
+    Create instance of unsigned uint64 type
+    """
     return primitive_type(_Type_UINT64)
 
 
 def int64():
+    """
+    Create instance of signed int64 type
+    """
     return primitive_type(_Type_INT64)
 
 
@@ -529,104 +561,183 @@ cdef timeunit_to_string(TimeUnit unit):
         return 'ns'
 
 
-def timestamp(unit_str, tz=None):
+def timestamp(unit, tz=None):
+    """
+    Create instance of timestamp type with resolution and optional time zone
+
+    Parameters
+    ----------
+    unit : string
+        one of 's' [second], 'ms' [millisecond], 'us' [microsecond], or 'ns'
+        [nanosecond]
+    tz : string, default None
+        Time zone name. None indicates time zone naive
+
+    Examples
+    --------
+    ::
+
+        t1 = pa.timestamp('us')
+        t2 = pa.timestamp('s', tz='America/New_York')
+
+    Returns
+    -------
+    timestamp_type : TimestampType
+    """
     cdef:
-        TimeUnit unit
+        TimeUnit unit_code
         c_string c_timezone
 
-    if unit_str == "s":
-        unit = TimeUnit_SECOND
-    elif unit_str == 'ms':
-        unit = TimeUnit_MILLI
-    elif unit_str == 'us':
-        unit = TimeUnit_MICRO
-    elif unit_str == 'ns':
-        unit = TimeUnit_NANO
+    if unit == "s":
+        unit_code = TimeUnit_SECOND
+    elif unit == 'ms':
+        unit_code = TimeUnit_MILLI
+    elif unit == 'us':
+        unit_code = TimeUnit_MICRO
+    elif unit == 'ns':
+        unit_code = TimeUnit_NANO
     else:
         raise ValueError('Invalid TimeUnit string')
 
     cdef TimestampType out = TimestampType()
 
     if tz is None:
-        out.init(ctimestamp(unit))
-        if unit in _timestamp_type_cache:
-            return _timestamp_type_cache[unit]
-        _timestamp_type_cache[unit] = out
+        out.init(ctimestamp(unit_code))
+        if unit_code in _timestamp_type_cache:
+            return _timestamp_type_cache[unit_code]
+        _timestamp_type_cache[unit_code] = out
     else:
         if not isinstance(tz, six.string_types):
             tz = tz.zone
 
         c_timezone = tobytes(tz)
-        out.init(ctimestamp(unit, c_timezone))
+        out.init(ctimestamp(unit_code, c_timezone))
 
     return out
 
 
-def time32(unit_str):
+def time32(unit):
+    """
+    Create instance of 32-bit time (time of day) type with unit resolution
+
+    Parameters
+    ----------
+    unit : string
+        one of 's' [second], or 'ms' [millisecond]
+
+    Examples
+    --------
+    ::
+
+        t1 = pa.time32('s')
+        t2 = pa.time32('ms')
+    """
     cdef:
-        TimeUnit unit
+        TimeUnit unit_code
         c_string c_timezone
 
-    if unit_str == "s":
-        unit = TimeUnit_SECOND
-    elif unit_str == 'ms':
-        unit = TimeUnit_MILLI
+    if unit == "s":
+        unit_code = TimeUnit_SECOND
+    elif unit == 'ms':
+        unit_code = TimeUnit_MILLI
     else:
-        raise ValueError('Invalid TimeUnit for time32: {}'.format(unit_str))
+        raise ValueError('Invalid TimeUnit for time32: {}'.format(unit))
 
     cdef Time32Type out
-    if unit in _time_type_cache:
-        return _time_type_cache[unit]
+    if unit_code in _time_type_cache:
+        return _time_type_cache[unit_code]
     else:
         out = Time32Type()
-        out.init(ctime32(unit))
-        _time_type_cache[unit] = out
+        out.init(ctime32(unit_code))
+        _time_type_cache[unit_code] = out
         return out
 
 
-def time64(unit_str):
+def time64(unit):
+    """
+    Create instance of 64-bit time (time of day) type with unit resolution
+
+    Parameters
+    ----------
+    unit : string
+        one of 'us' [microsecond], or 'ns' [nanosecond]
+
+    Examples
+    --------
+    ::
+
+        t1 = pa.time64('us')
+        t2 = pa.time64('ns')
+    """
     cdef:
-        TimeUnit unit
+        TimeUnit unit_code
         c_string c_timezone
 
-    if unit_str == "us":
-        unit = TimeUnit_MICRO
-    elif unit_str == 'ns':
-        unit = TimeUnit_NANO
+    if unit == "us":
+        unit_code = TimeUnit_MICRO
+    elif unit == 'ns':
+        unit_code = TimeUnit_NANO
     else:
-        raise ValueError('Invalid TimeUnit for time64: {}'.format(unit_str))
+        raise ValueError('Invalid TimeUnit for time64: {}'.format(unit))
 
     cdef Time64Type out
-    if unit in _time_type_cache:
-        return _time_type_cache[unit]
+    if unit_code in _time_type_cache:
+        return _time_type_cache[unit_code]
     else:
         out = Time64Type()
-        out.init(ctime64(unit))
-        _time_type_cache[unit] = out
+        out.init(ctime64(unit_code))
+        _time_type_cache[unit_code] = out
         return out
 
 
 def date32():
+    """
+    Create instance of 32-bit date (days since UNIX epoch 1970-01-01)
+    """
     return primitive_type(_Type_DATE32)
 
 
 def date64():
+    """
+    Create instance of 64-bit date (milliseconds since UNIX epoch 1970-01-01)
+    """
     return primitive_type(_Type_DATE64)
 
 
 def float16():
+    """
+    Create half-precision floating point type
+    """
     return primitive_type(_Type_HALF_FLOAT)
 
 
 def float32():
+    """
+    Create single-precision floating point type
+    """
     return primitive_type(_Type_FLOAT)
 
 
 def float64():
+    """
+    Create double-precision floating point type
+    """
     return primitive_type(_Type_DOUBLE)
 
 
 cpdef DataType decimal(int precision, int scale=0):
+    """
+    Create decimal type with precision and scale
+
+    Parameters
+    ----------
+    precision : int
+    scale : int
+
+    Returns
+    -------
+    decimal_type : DecimalType
+    """
     cdef shared_ptr[CDataType] decimal_type
     decimal_type.reset(new CDecimalType(precision, scale))
     return pyarrow_wrap_data_type(decimal_type)
@@ -634,13 +745,14 @@ cpdef DataType decimal(int precision, int scale=0):
 
 def string():
     """
-    UTF8 string
+    Create UTF8 variable-length string type
     """
     return primitive_type(_Type_STRING)
 
 
 def binary(int length=-1):
-    """Binary (PyBytes-like) type
+    """
+    Create variable-length binary type
 
     Parameters
     ----------
@@ -717,12 +829,14 @@ def struct(fields):
 
     Examples
     --------
-    import pyarrow as pa
-    fields = [
-        pa.field('f1', pa.int32()),
-        pa.field('f2', pa.string())
-    ]
-    struct_type = pa.struct(fields)
+    ::
+
+        import pyarrow as pa
+        fields = [
+            pa.field('f1', pa.int32()),
+            pa.field('f2', pa.string())
+        ]
+        struct_type = pa.struct(fields)
 
     Returns
     -------

From b474cacf77050b7bba242340d5a469ddeb67e92d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Jul 2017 18:11:03 +0200
Subject: [PATCH 0820/1644] ARROW-1221: [C++] Add run_clang_format.py script,
 exclusions file. Pin clang-format-3.9

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #859 from wesm/ARROW-1221 and squashes the following commits:

dc0c997 [Wes McKinney] Add run_clang_format.py script, exclusions file. Pin clang-format-3.9
---
 cpp/CMakeLists.txt                            | 31 +++++-----
 cpp/build-support/clang_format_exclusions.txt |  8 +++
 cpp/build-support/run-clang-format.sh         | 42 -------------
 cpp/build-support/run_clang_format.py         | 61 +++++++++++++++++++
 dev/release/rat_exclude_files.txt             |  1 +
 5 files changed, 84 insertions(+), 59 deletions(-)
 create mode 100644 cpp/build-support/clang_format_exclusions.txt
 delete mode 100755 cpp/build-support/run-clang-format.sh
 create mode 100755 cpp/build-support/run_clang_format.py

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 3146b87b678e0..2891a5d76186c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -518,25 +518,22 @@ endif (UNIX)
 ############################################################
 # "make format" and "make check-format" targets
 ############################################################
-if (${CLANG_FORMAT_FOUND})
-  # runs clang format and updates files in place.
-  add_custom_target(format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1
-    `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h |
-    sed -e '/_generated/g' |
-    sed -e '/windows_compatibility.h/g' |
-    sed -e '/pyarrow_api.h/g' |
-    sed -e '/config.h/g' |   # python/config.h
-    sed -e '/platform.h/g' |  # python/platform.h
-    sed -e '/ae.h/g' |
-    sed -e '/xxhash.cc/g' |
-    sed -e '/xxhash.h/g'`
-    )
 
-  # runs clang format and exits with a non-zero exit code if any files need to be reformatted
-  add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0
-  `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`)
-endif()
+set(CLANG_FORMAT_VERSION 3.9)
+
+# runs clang format and updates files in place.
+add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py
+  ${CLANG_FORMAT_VERSION}
+  ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
+  ${CMAKE_CURRENT_SOURCE_DIR}/src)
+
+# runs clang format and exits with a non-zero exit code if any files need to be reformatted
 
+# TODO(wesm): Make this work in run_clang_format.py
+# add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py
+#   ${CLANG_FORMAT_VERSION}
+#   ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
+#   ${CMAKE_CURRENT_SOURCE_DIR}/src 1)
 
 ############################################################
 # "make clang-tidy" and "make check-clang-tidy" targets
diff --git a/cpp/build-support/clang_format_exclusions.txt b/cpp/build-support/clang_format_exclusions.txt
new file mode 100644
index 0000000000000..2d5d86d4e4cde
--- /dev/null
+++ b/cpp/build-support/clang_format_exclusions.txt
@@ -0,0 +1,8 @@
+*_generated*
+*windows_compatibility.h
+*pyarrow_api.h
+*python/config.h
+*python/platform.h
+*thirdparty/ae/*
+*xxhash.cc
+*xxhash.h
diff --git a/cpp/build-support/run-clang-format.sh b/cpp/build-support/run-clang-format.sh
deleted file mode 100755
index ba525dfc33c69..0000000000000
--- a/cpp/build-support/run-clang-format.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Runs clang format in the given directory
-# Arguments:
-#   $1 - Path to the source tree
-#   $2 - Path to the clang format binary
-#   $3 - Apply fixes (will raise an error if false and not there where changes)
-#   $ARGN - Files to run clang format on
-#
-SOURCE_DIR=$1
-shift
-CLANG_FORMAT=$1
-shift
-APPLY_FIXES=$1
-shift
-
-# clang format will only find its configuration if we are in 
-# the source tree or in a path relative to the source tree
-pushd $SOURCE_DIR
-if [ "$APPLY_FIXES" == "1" ]; then
-  $CLANG_FORMAT -i $@
-else
-
-  NUM_CORRECTIONS=`$CLANG_FORMAT -output-replacements-xml  $@ | grep offset | wc -l`
-  if [ "$NUM_CORRECTIONS" -gt "0" ]; then
-    echo "clang-format suggested changes, please run 'make format'!!!!"
-    exit 1
-  fi
-fi 
-popd
diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py
new file mode 100755
index 0000000000000..ab800e641b5d2
--- /dev/null
+++ b/cpp/build-support/run_clang_format.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import fnmatch
+import os
+import subprocess
+import sys
+
+if len(sys.argv) < 4:
+    sys.stderr.write("Usage: %s $CLANG_FORMAT_VERSION exclude_globs.txt "
+                     "$source_dir\n" %
+                     sys.argv[0])
+    sys.exit(1)
+
+CLANG_FORMAT = 'clang-format-{0}'.format(sys.argv[1])
+EXCLUDE_GLOBS_FILENAME = sys.argv[2]
+SOURCE_DIR = sys.argv[3]
+
+exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")]
+
+files_to_format = []
+matches = []
+for directory, subdirs, files in os.walk(SOURCE_DIR):
+    for name in files:
+        name = os.path.join(directory, name)
+        if not (name.endswith('.h') or name.endswith('.cc')):
+            continue
+
+        excluded = False
+        for g in exclude_globs:
+            if fnmatch.fnmatch(name, g):
+                excluded = True
+                break
+        if not excluded:
+            files_to_format.append(name)
+
+# TODO(wesm): Port this to work with Python, for check-format
+# NUM_CORRECTIONS=`$CLANG_FORMAT -output-replacements-xml  $@ |
+# grep offset | wc -l`
+# if [ "$NUM_CORRECTIONS" -gt "0" ]; then
+#   echo "clang-format suggested changes, please run 'make format'!!!!"
+#   exit 1
+# fi
+
+subprocess.check_output([CLANG_FORMAT, '-i'] + files_to_format,
+                        stderr=subprocess.STDOUT)
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 286793e85f146..5a0b8a3838fd6 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -7,6 +7,7 @@ cpp/src/arrow/status.cc
 cpp/src/arrow/status.h
 cpp/build-support/asan_symbolize.py
 cpp/build-support/cpplint.py
+cpp/build-support/clang_format_exclusions.txt
 cpp/cmake_modules/BuildUtils.cmake
 cpp/cmake_modules/FindPythonLibsNew.cmake
 cpp/cmake_modules/FindNumPy.cmake

From ea9bc8378abfbe6e15a4e338bd7024841dc2a37c Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 17 Jul 2017 18:17:45 +0200
Subject: [PATCH 0821/1644] ARROW-1229: [GLib] Use "read" instead of "get" for
 reading record batch

It follows C++ API change.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #858 from kou/glib-use-read and squashes the following commits:

6fc499e [Kouhei Sutou] [GLib] Use "read" instead of "get" for reading record batch
---
 c_glib/arrow-glib/reader.cpp                  | 46 ++++++++++++++++++-
 c_glib/arrow-glib/reader.h                    | 13 ++++++
 c_glib/example/Makefile.am                    |  3 +-
 c_glib/example/go/read-batch.go               |  4 +-
 c_glib/example/go/read-stream.go              |  4 +-
 c_glib/example/lua/read-batch.lua             |  2 +-
 c_glib/example/lua/read-stream.lua            |  2 +-
 c_glib/example/lua/stream-to-torch-tensor.lua |  2 +-
 c_glib/example/read-batch.c                   |  2 +-
 c_glib/example/read-stream.c                  |  2 +-
 c_glib/test/test-stream-writer.rb             |  4 +-
 11 files changed, 70 insertions(+), 14 deletions(-)

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index 523bdeef77db3..25c8d67d95f60 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -166,10 +166,30 @@ garrow_record_batch_reader_get_schema(GArrowRecordBatchReader *reader)
  *   The next record batch in the stream or %NULL on end of stream.
  *
  * Since: 0.4.0
+ *
+ * Deprecated: 0.5.0:
+ *   Use garrow_record_batch_reader_read_next_record_batch() instead.
  */
 GArrowRecordBatch *
 garrow_record_batch_reader_get_next_record_batch(GArrowRecordBatchReader *reader,
                                                  GError **error)
+{
+  return garrow_record_batch_reader_read_next_record_batch(reader, error);
+}
+
+/**
+ * garrow_record_batch_reader_read_next_record_batch:
+ * @reader: A #GArrowRecordBatchReader.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The next record batch in the stream or %NULL on end of stream.
+ *
+ * Since: 0.5.0
+ */
+GArrowRecordBatch *
+garrow_record_batch_reader_read_next_record_batch(GArrowRecordBatchReader *reader,
+                                                  GError **error)
 {
   auto arrow_reader = garrow_record_batch_reader_get_raw(reader);
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
@@ -177,7 +197,7 @@ garrow_record_batch_reader_get_next_record_batch(GArrowRecordBatchReader *reader
 
   if (garrow_error_check(error,
                          status,
-                         "[record-batch-reader][get-next-record-batch]")) {
+                         "[record-batch-reader][read-next-record-batch]")) {
     if (arrow_record_batch == nullptr) {
       return NULL;
     } else {
@@ -402,11 +422,33 @@ garrow_record_batch_file_reader_get_version(GArrowRecordBatchFileReader *reader)
  *   The i-th record batch in the file or %NULL on error.
  *
  * Since: 0.4.0
+ *
+ * Deprecated: 0.5.0:
+ *   Use garrow_record_batch_file_reader_read_record_batch() instead.
  */
 GArrowRecordBatch *
 garrow_record_batch_file_reader_get_record_batch(GArrowRecordBatchFileReader *reader,
                                                  guint i,
                                                  GError **error)
+{
+  return garrow_record_batch_file_reader_read_record_batch(reader, i, error);
+}
+
+/**
+ * garrow_record_batch_file_reader_read_record_batch:
+ * @reader: A #GArrowRecordBatchFileReader.
+ * @i: The index of the target record batch.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The i-th record batch in the file or %NULL on error.
+ *
+ * Since: 0.5.0
+ */
+GArrowRecordBatch *
+garrow_record_batch_file_reader_read_record_batch(GArrowRecordBatchFileReader *reader,
+                                                  guint i,
+                                                  GError **error)
 {
   auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader);
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
@@ -414,7 +456,7 @@ garrow_record_batch_file_reader_get_record_batch(GArrowRecordBatchFileReader *re
 
   if (garrow_error_check(error,
                          status,
-                         "[record-batch-file-reader][get-record-batch]")) {
+                         "[record-batch-file-reader][read-record-batch]")) {
     return garrow_record_batch_new_raw(&arrow_record_batch);
   } else {
     return NULL;
diff --git a/c_glib/arrow-glib/reader.h b/c_glib/arrow-glib/reader.h
index b8bdbe221d646..f5195e4a434f6 100644
--- a/c_glib/arrow-glib/reader.h
+++ b/c_glib/arrow-glib/reader.h
@@ -76,9 +76,15 @@ GType garrow_record_batch_reader_get_type(void) G_GNUC_CONST;
 
 GArrowSchema *garrow_record_batch_reader_get_schema(
   GArrowRecordBatchReader *reader);
+#ifndef GARROW_DISABLE_DEPRECATED
+G_GNUC_DEPRECATED_FOR(garrow_record_batch_reader_read_next_record_batch)
 GArrowRecordBatch *garrow_record_batch_reader_get_next_record_batch(
   GArrowRecordBatchReader *reader,
   GError **error);
+#endif
+GArrowRecordBatch *garrow_record_batch_reader_read_next_record_batch(
+  GArrowRecordBatchReader *reader,
+  GError **error);
 
 
 #define GARROW_TYPE_RECORD_BATCH_STREAM_READER          \
@@ -188,10 +194,17 @@ guint garrow_record_batch_file_reader_get_n_record_batches(
   GArrowRecordBatchFileReader *reader);
 GArrowMetadataVersion garrow_record_batch_file_reader_get_version(
   GArrowRecordBatchFileReader *reader);
+#ifndef GARROW_DISABLE_DEPRECATED
+G_GNUC_DEPRECATED_FOR(garrow_record_batch_file_reader_read_record_batch)
 GArrowRecordBatch *garrow_record_batch_file_reader_get_record_batch(
   GArrowRecordBatchFileReader *reader,
   guint i,
   GError **error);
+#endif
+GArrowRecordBatch *garrow_record_batch_file_reader_read_record_batch(
+  GArrowRecordBatchFileReader *reader,
+  guint i,
+  GError **error);
 
 
 #define GARROW_TYPE_FEATHER_FILE_READER         \
diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am
index 66d2cddcac5fb..6c177954c6cb9 100644
--- a/c_glib/example/Makefile.am
+++ b/c_glib/example/Makefile.am
@@ -20,7 +20,8 @@ SUBDIRS	=					\
 
 AM_CPPFLAGS =					\
 	-I$(top_builddir)			\
-	-I$(top_srcdir)
+	-I$(top_srcdir)				\
+	-DGARROW_DISABLE_DEPRECATED
 
 AM_CFLAGS =					\
 	$(GLIB_CFLAGS)				\
diff --git a/c_glib/example/go/read-batch.go b/c_glib/example/go/read-batch.go
index f47d3e20bc42e..ef1a7fb5a69f9 100644
--- a/c_glib/example/go/read-batch.go
+++ b/c_glib/example/go/read-batch.go
@@ -89,9 +89,9 @@ func main() {
 	var i uint32
 	nRecordBatches := reader.GetNRecordBatches()
 	for i = 0; i < nRecordBatches; i++ {
-		recordBatch, err := reader.GetRecordBatch(i)
+		recordBatch, err := reader.ReadRecordBatch(i)
 		if err != nil {
-			log.Fatalf("Failed to get record batch[%d]: %v", i, err)
+			log.Fatalf("Failed to read record batch[%d]: %v", i, err)
 		}
 		fmt.Println(strings.Repeat("=", 40))
 		fmt.Printf("record-batch[%d]:\n", i)
diff --git a/c_glib/example/go/read-stream.go b/c_glib/example/go/read-stream.go
index 789b87bad6b1d..fdb7ba1b164d3 100644
--- a/c_glib/example/go/read-stream.go
+++ b/c_glib/example/go/read-stream.go
@@ -87,9 +87,9 @@ func main() {
 		log.Fatalf("Failed to parse data: %v", err)
 	}
 	for i := 0; true; i++ {
-		recordBatch, err := reader.GetNextRecordBatch()
+		recordBatch, err := reader.ReadNextRecordBatch()
 		if err != nil {
-			log.Fatalf("Failed to get next record batch: %v", err)
+			log.Fatalf("Failed to read next record batch: %v", err)
 		}
 		if recordBatch == nil {
 			break
diff --git a/c_glib/example/lua/read-batch.lua b/c_glib/example/lua/read-batch.lua
index 5e28d7efd5033..8dc2fd82b3965 100644
--- a/c_glib/example/lua/read-batch.lua
+++ b/c_glib/example/lua/read-batch.lua
@@ -24,7 +24,7 @@ local input = Arrow.MemoryMappedInputStream.new(input_path)
 local reader = Arrow.RecordBatchFileReader.new(input)
 
 for i = 0, reader:get_n_record_batches() - 1 do
-   local record_batch = reader:get_record_batch(i)
+   local record_batch = reader:read_record_batch(i)
    print(string.rep("=", 40))
    print("record-batch["..i.."]:")
    for j = 0, record_batch:get_n_columns() - 1 do
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index 95cf2e7a8a9f3..7dc9617d05a61 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -25,7 +25,7 @@ local reader = Arrow.RecordBatchStreamReader.new(input)
 
 local i = 0
 while true do
-   local record_batch = reader:get_next_record_batch()
+   local record_batch = reader:read_next_record_batch()
    if not record_batch then
       break
    end
diff --git a/c_glib/example/lua/stream-to-torch-tensor.lua b/c_glib/example/lua/stream-to-torch-tensor.lua
index 9825f105bd7cb..fc765e3c96872 100644
--- a/c_glib/example/lua/stream-to-torch-tensor.lua
+++ b/c_glib/example/lua/stream-to-torch-tensor.lua
@@ -81,7 +81,7 @@ local reader = Arrow.RecordBatchStreamReader.new(input)
 
 local i = 0
 while true do
-   local record_batch = reader:get_next_record_batch()
+   local record_batch = reader:read_next_record_batch()
    if not record_batch then
       break
    end
diff --git a/c_glib/example/read-batch.c b/c_glib/example/read-batch.c
index 4461c43c84777..4382816e048fa 100644
--- a/c_glib/example/read-batch.c
+++ b/c_glib/example/read-batch.c
@@ -121,7 +121,7 @@ main(int argc, char **argv)
         GArrowRecordBatch *record_batch;
 
         record_batch =
-          garrow_record_batch_file_reader_get_record_batch(reader, i, &error);
+          garrow_record_batch_file_reader_read_record_batch(reader, i, &error);
         if (!record_batch) {
           g_print("failed to open file reader: %s\n", error->message);
           g_error_free(error);
diff --git a/c_glib/example/read-stream.c b/c_glib/example/read-stream.c
index 412be35135350..a624f72949118 100644
--- a/c_glib/example/read-stream.c
+++ b/c_glib/example/read-stream.c
@@ -118,7 +118,7 @@ main(int argc, char **argv)
       GArrowRecordBatch *record_batch;
 
       record_batch =
-        garrow_record_batch_reader_get_next_record_batch(reader, &error);
+        garrow_record_batch_reader_read_next_record_batch(reader, &error);
       if (error) {
         g_print("failed to get record batch: %s\n", error->message);
         g_error_free(error);
diff --git a/c_glib/test/test-stream-writer.rb b/c_glib/test/test-stream-writer.rb
index 41fd14718e66e..833e08b145a52 100644
--- a/c_glib/test/test-stream-writer.rb
+++ b/c_glib/test/test-stream-writer.rb
@@ -44,8 +44,8 @@ def test_write_record_batch
       assert_equal(["enabled"],
                    stream_reader.schema.fields.collect(&:name))
       assert_equal(true,
-                   stream_reader.next_record_batch.get_column(0).get_value(0))
-      assert_nil(stream_reader.next_record_batch)
+                   stream_reader.read_next_record_batch.get_column(0).get_value(0))
+      assert_nil(stream_reader.read_next_record_batch)
     ensure
       input.close
     end

From 0396240b55ab1d74c6b36ffaf95290135d8da389 Mon Sep 17 00:00:00 2001
From: Antony Mayi <antonymayi@yahoo.com>
Date: Mon, 17 Jul 2017 18:18:56 +0200
Subject: [PATCH 0822/1644] ARROW-1190: [JAVA] Fixing VectorLoader for
 duplicate field names

VectorLoader was corrupting data when some of the fields had same name in which case only one of that vectors got properly loaded. This PR resolves the problem by avoiding by-name field lookups.

Author: Antony Mayi <antonymayi@yahoo.com>

Closes #816 from antonymayi/master and squashes the following commits:

38c1837 [Antony Mayi] adding unit test for unload-loading vectors with duplicate field names
a723416 [Antony Mayi] ARROW-1190 - fixing VectorLoader for duplicate field names
---
 .../org/apache/arrow/vector/VectorLoader.java |  6 +--
 .../arrow/vector/TestVectorUnloadLoad.java    | 47 +++++++++++++++++++
 2 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 33a608cd92922..e640c7cb78418 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -55,10 +55,8 @@ public VectorLoader(VectorSchemaRoot root) {
   public void load(ArrowRecordBatch recordBatch) {
     Iterator<ArrowBuf> buffers = recordBatch.getBuffers().iterator();
     Iterator<ArrowFieldNode> nodes = recordBatch.getNodes().iterator();
-    List<Field> fields = root.getSchema().getFields();
-    for (Field field: fields) {
-      FieldVector fieldVector = root.getVector(field.getName());
-      loadBuffers(fieldVector, field, buffers, nodes);
+    for (FieldVector fieldVector: root.getFieldVectors()) {
+      loadBuffers(fieldVector, fieldVector.getField(), buffers, nodes);
     }
     root.setRowCount(recordBatch.getLength());
     if (nodes.hasNext() || buffers.hasNext()) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 6fb559ca0d728..f3694659a8f51 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -240,6 +240,53 @@ public void testLoadEmptyValidityBuffer() throws IOException {
     }
   }
 
+  @Test
+  public void testUnloadLoadDuplicates() throws IOException {
+    int count = 10;
+    Schema schema = new Schema(asList(
+      new Field("duplicate", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList()),
+      new Field("duplicate", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList())
+    ));
+
+    try (
+      BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+    ) {
+      List<FieldVector> sources = new ArrayList<>();
+      for (Field field: schema.getFields()) {
+        FieldVector vector = field.createVector(originalVectorsAllocator);
+        vector.allocateNew();
+        sources.add(vector);
+        NullableIntVector.Mutator mutator = (NullableIntVector.Mutator) vector.getMutator();
+        for (int i =  0; i < count; i++) {
+          mutator.set(i, i);
+        }
+        mutator.setValueCount(count);
+      }
+
+      try (VectorSchemaRoot root = new VectorSchemaRoot(schema.getFields(), sources, count)) {
+        VectorUnloader vectorUnloader = new VectorUnloader(root);
+        try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+                BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+                VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator);) {
+          // load it
+          VectorLoader vectorLoader = new VectorLoader(newRoot);
+          vectorLoader.load(recordBatch);
+
+          List<FieldVector> targets = newRoot.getFieldVectors();
+          Assert.assertEquals(sources.size(), targets.size());
+          for (int k = 0; k < sources.size(); k++) {
+            NullableIntVector.Accessor src = (NullableIntVector.Accessor) sources.get(k).getAccessor();
+            NullableIntVector.Accessor tgt = (NullableIntVector.Accessor) targets.get(k).getAccessor();
+            Assert.assertEquals(src.getValueCount(), tgt.getValueCount());
+            for (int i = 0; i < count; i++) {
+              Assert.assertEquals(src.get(i), tgt.get(i));
+            }
+          }
+        }
+      }
+    }
+  }
+
   public static VectorUnloader newVectorUnloader(FieldVector root) {
     Schema schema = new Schema(root.getField().getChildren());
     int valueCount = root.getAccessor().getValueCount();

From 1541a08c795f03b2ce5884a00fc83d1fb79a448e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Jul 2017 18:32:30 +0200
Subject: [PATCH 0823/1644] ARROW-1177: [C++] Check for int32 offset overflow
 in ListBuilder, BinaryBuilder

I also refactored BinaryBuilder to not inherit from ListBuilder, which is a bit cleaner. I added a draft of ARROW-507; it needs a unit test and to handle the case where some passed offsets are null (so they need to be sanitized)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #853 from wesm/ARROW-1177 and squashes the following commits:

f6be04f [Wes McKinney] Fix DCHECKs in ListBuilder, BinaryBuilder
28f17ab [Wes McKinney] Use binary strings for py2.7
c9e7502 [Wes McKinney] Fix some off-by-one errors
5a8be84 [Wes McKinney] Fix another warning
23adefc [Wes McKinney] Fix compiler warning
35ab4f2 [Wes McKinney] Refactoring BinaryBuilder. Add check for int32 offset overflow for List, Binary, String. Add basic ListArray::FromArrays method, add Python binding
---
 cpp/src/arrow/array-test.cc              |  11 +-
 cpp/src/arrow/array.cc                   |  26 ++++
 cpp/src/arrow/array.h                    |  13 ++
 cpp/src/arrow/buffer.h                   |  73 ++++++----
 cpp/src/arrow/builder.cc                 | 168 ++++++++++++++---------
 cpp/src/arrow/builder.h                  |  60 +++++---
 cpp/src/arrow/ipc/ipc-json-test.cc       |   4 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |   5 +-
 cpp/src/arrow/ipc/test-common.h          |  21 ++-
 cpp/src/arrow/test-util.h                |   9 +-
 python/pyarrow/array.pxi                 |  36 ++++-
 python/pyarrow/includes/libarrow.pxd     |   6 +
 python/pyarrow/tests/test_array.py       |  12 ++
 13 files changed, 313 insertions(+), 131 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 01bef030cc3d4..acb4819dd0949 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -71,13 +71,14 @@ Status MakeArrayFromValidBytes(
   std::shared_ptr<Buffer> null_buf;
   RETURN_NOT_OK(BitUtil::BytesToBits(v, &null_buf));
 
-  BufferBuilder value_builder(pool);
+  TypedBufferBuilder<int32_t> value_builder(pool);
   for (size_t i = 0; i < v.size(); ++i) {
-    RETURN_NOT_OK(value_builder.Append<int32_t>(0));
+    RETURN_NOT_OK(value_builder.Append(0));
   }
 
-  *out = std::make_shared<Int32Array>(
-      v.size(), value_builder.Finish(), null_buf, null_count);
+  std::shared_ptr<Buffer> values;
+  RETURN_NOT_OK(value_builder.Finish(&values));
+  *out = std::make_shared<Int32Array>(v.size(), values, null_buf, null_count);
   return Status::OK();
 }
 
@@ -996,7 +997,7 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
   vector<uint8_t> is_null = {0, 0, 0, 1, 0};
 
   int N = static_cast<int>(strings.size());
-  int reps = 1000;
+  int reps = 10;
 
   for (int j = 0; j < reps; ++j) {
     for (int i = 0; i < N; ++i) {
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 48a3bd5a635a4..f20b8493b17f6 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -190,6 +190,32 @@ ListArray::ListArray(const std::shared_ptr<DataType>& type, int64_t length,
   SetData(internal_data);
 }
 
+Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPool* pool,
+    std::shared_ptr<Array>* out) {
+  if (ARROW_PREDICT_FALSE(offsets.length() == 0)) {
+    return Status::Invalid("List offsets must have non-zero length");
+  }
+
+  if (ARROW_PREDICT_FALSE(offsets.null_count() > 0)) {
+    return Status::Invalid("Null offsets in ListArray::FromArrays not yet implemented");
+  }
+
+  if (ARROW_PREDICT_FALSE(offsets.type_id() != Type::INT32)) {
+    return Status::Invalid("List offsets must be signed int32");
+  }
+
+  BufferVector buffers = {
+      offsets.null_bitmap(), static_cast<const Int32Array&>(offsets).values()};
+
+  auto list_type = list(values.type());
+  auto internal_data = std::make_shared<internal::ArrayData>(list_type,
+      offsets.length() - 1, std::move(buffers), offsets.null_count(), offsets.offset());
+  internal_data->child_data.push_back(values.data());
+
+  *out = std::make_shared<ListArray>(internal_data);
+  return Status::OK();
+}
+
 void ListArray::SetData(const std::shared_ptr<ArrayData>& data) {
   this->Array::SetData(data);
   auto value_offsets = data->buffers[1];
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 80284cdc2dff7..2da0b54b73b35 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -371,6 +371,19 @@ class ARROW_EXPORT ListArray : public Array {
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0);
 
+  /// \brief Construct ListArray from array of offsets and child value array
+  ///
+  /// Note: does not validate input beyond sanity checks. Use
+  /// arrow::ValidateArray if you need stronger validation of inputs
+  ///
+  /// \param[in] offsets Array containing n + 1 offsets encoding length and size
+  /// \param[in] values Array containing
+  /// \param[in] pool MemoryPool in case new offsets array needs to be
+  /// allocated because of null values
+  /// \param[out] out Will have length equal to offsets.length() - 1
+  static Status FromArrays(const Array& offsets, const Array& values, MemoryPool* pool,
+      std::shared_ptr<Array>* out);
+
   /// \brief Return array object containing the list's values
   std::shared_ptr<Array> values() const;
 
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index b117b2414d0fd..488a4c05334d5 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -212,58 +212,71 @@ class ARROW_EXPORT BufferBuilder {
     return Status::OK();
   }
 
-  template <typename T>
+  // Unsafe methods don't check existing size
+  void UnsafeAppend(const uint8_t* data, int64_t length) {
+    memcpy(data_ + size_, data, static_cast<size_t>(length));
+    size_ += length;
+  }
+
+  Status Finish(std::shared_ptr<Buffer>* out) {
+    // Do not shrink to fit to avoid unneeded realloc
+    if (size_ > 0) { RETURN_NOT_OK(buffer_->Resize(size_, false)); }
+    *out = buffer_;
+    Reset();
+    return Status::OK();
+  }
+
+  void Reset() {
+    buffer_ = nullptr;
+    capacity_ = size_ = 0;
+  }
+
+  int64_t capacity() const { return capacity_; }
+  int64_t length() const { return size_; }
+  const uint8_t* data() const { return data_; }
+
+ protected:
+  std::shared_ptr<PoolBuffer> buffer_;
+  MemoryPool* pool_;
+  uint8_t* data_;
+  int64_t capacity_;
+  int64_t size_;
+};
+
+template <typename T>
+class ARROW_EXPORT TypedBufferBuilder : public BufferBuilder {
+ public:
+  explicit TypedBufferBuilder(MemoryPool* pool) : BufferBuilder(pool) {}
+
   Status Append(T arithmetic_value) {
     static_assert(std::is_arithmetic<T>::value,
         "Convenience buffer append only supports arithmetic types");
-    return Append(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
+    return BufferBuilder::Append(
+        reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
   }
 
-  template <typename T>
   Status Append(const T* arithmetic_values, int64_t num_elements) {
     static_assert(std::is_arithmetic<T>::value,
         "Convenience buffer append only supports arithmetic types");
-    return Append(
+    return BufferBuilder::Append(
         reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
   }
 
-  // Unsafe methods don't check existing size
-  void UnsafeAppend(const uint8_t* data, int64_t length) {
-    memcpy(data_ + size_, data, static_cast<size_t>(length));
-    size_ += length;
-  }
-
-  template <typename T>
   void UnsafeAppend(T arithmetic_value) {
     static_assert(std::is_arithmetic<T>::value,
         "Convenience buffer append only supports arithmetic types");
-    UnsafeAppend(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
+    BufferBuilder::UnsafeAppend(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
   }
 
-  template <typename T>
   void UnsafeAppend(const T* arithmetic_values, int64_t num_elements) {
     static_assert(std::is_arithmetic<T>::value,
         "Convenience buffer append only supports arithmetic types");
-    UnsafeAppend(
+    BufferBuilder::UnsafeAppend(
         reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
   }
 
-  std::shared_ptr<Buffer> Finish() {
-    auto result = buffer_;
-    buffer_ = nullptr;
-    capacity_ = size_ = 0;
-    return result;
-  }
-  int64_t capacity() const { return capacity_; }
-  int64_t length() const { return size_; }
-  const uint8_t* data() const { return data_; }
-
- private:
-  std::shared_ptr<PoolBuffer> buffer_;
-  MemoryPool* pool_;
-  uint8_t* data_;
-  int64_t capacity_;
-  int64_t size_;
+  const T* data() const { return reinterpret_cast<const T*>(data_); }
+  int64_t length() const { return size_ / sizeof(T); }
 };
 
 /// Allocate a new mutable buffer from a memory pool
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 885c65029637a..a2f24a7ce9cce 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -21,6 +21,8 @@
 #include <cstdint>
 #include <cstring>
 #include <limits>
+#include <sstream>
+#include <vector>
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
@@ -99,21 +101,17 @@ Status ArrayBuilder::Reserve(int64_t elements) {
   return Status::OK();
 }
 
+void ArrayBuilder::Reset() {
+  capacity_ = length_ = null_count_ = 0;
+  null_bitmap_ = nullptr;
+}
+
 Status ArrayBuilder::SetNotNull(int64_t length) {
   RETURN_NOT_OK(Reserve(length));
   UnsafeSetNotNull(length);
   return Status::OK();
 }
 
-void ArrayBuilder::UnsafeAppendToBitmap(bool is_valid) {
-  if (is_valid) {
-    BitUtil::SetBit(null_bitmap_data_, length_);
-  } else {
-    ++null_count_;
-  }
-  ++length_;
-}
-
 void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t length) {
   if (valid_bytes == nullptr) {
     UnsafeSetNotNull(length);
@@ -971,7 +969,8 @@ Status DecimalBuilder::Resize(int64_t capacity) {
 }
 
 Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<Buffer> data = byte_builder_.Finish();
+  std::shared_ptr<Buffer> data;
+  RETURN_NOT_OK(byte_builder_.Finish(&data));
 
   /// TODO(phillipc): not sure where to get the offset argument here
   *out = std::make_shared<DecimalArray>(
@@ -987,65 +986,66 @@ ListBuilder::ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_b
     : ArrayBuilder(pool,
           type ? type : std::static_pointer_cast<DataType>(
                             std::make_shared<ListType>(value_builder->type()))),
-      offset_builder_(pool),
+      offsets_builder_(pool),
       value_builder_(std::move(value_builder)) {}
 
-ListBuilder::ListBuilder(MemoryPool* pool, std::shared_ptr<Array> values,
-    const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool,
-          type ? type : std::static_pointer_cast<DataType>(
-                            std::make_shared<ListType>(values->type()))),
-      offset_builder_(pool),
-      values_(values) {}
-
 Status ListBuilder::Append(
     const int32_t* offsets, int64_t length, const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
   UnsafeAppendToBitmap(valid_bytes, length);
-  offset_builder_.UnsafeAppend<int32_t>(offsets, length);
+  offsets_builder_.UnsafeAppend(offsets, length);
   return Status::OK();
 }
 
+Status ListBuilder::AppendNextOffset() {
+  int64_t num_values = value_builder_->length();
+  if (ARROW_PREDICT_FALSE(num_values >= std::numeric_limits<int32_t>::max())) {
+    std::stringstream ss;
+    ss << "ListArray cannot contain more then INT32_MAX - 1 child elements,"
+       << " have " << num_values;
+    return Status::Invalid(ss.str());
+  }
+  return offsets_builder_.Append(static_cast<int32_t>(num_values));
+}
+
 Status ListBuilder::Append(bool is_valid) {
   RETURN_NOT_OK(Reserve(1));
   UnsafeAppendToBitmap(is_valid);
-  RETURN_NOT_OK(
-      offset_builder_.Append<int32_t>(static_cast<int32_t>(value_builder_->length())));
-  return Status::OK();
+  return AppendNextOffset();
 }
 
 Status ListBuilder::Init(int64_t elements) {
-  DCHECK_LT(elements, std::numeric_limits<int64_t>::max());
+  DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
   // one more then requested for offsets
-  return offset_builder_.Resize((elements + 1) * sizeof(int64_t));
+  return offsets_builder_.Resize((elements + 1) * sizeof(int64_t));
 }
 
 Status ListBuilder::Resize(int64_t capacity) {
-  DCHECK_LT(capacity, std::numeric_limits<int64_t>::max());
+  DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
   // one more then requested for offsets
-  RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int64_t)));
+  RETURN_NOT_OK(offsets_builder_.Resize((capacity + 1) * sizeof(int64_t)));
   return ArrayBuilder::Resize(capacity);
 }
 
 Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
+  RETURN_NOT_OK(AppendNextOffset());
+
+  std::shared_ptr<Buffer> offsets;
+  RETURN_NOT_OK(offsets_builder_.Finish(&offsets));
+
   std::shared_ptr<Array> items = values_;
   if (!items) { RETURN_NOT_OK(value_builder_->Finish(&items)); }
 
-  RETURN_NOT_OK(offset_builder_.Append<int64_t>(items->length()));
-  std::shared_ptr<Buffer> offsets = offset_builder_.Finish();
-
   *out = std::make_shared<ListArray>(
       type_, length_, offsets, items, null_bitmap_, null_count_);
 
   Reset();
-
   return Status::OK();
 }
 
 void ListBuilder::Reset() {
-  capacity_ = length_ = null_count_ = 0;
-  null_bitmap_ = nullptr;
+  ArrayBuilder::Reset();
   values_ = nullptr;
 }
 
@@ -1057,52 +1057,97 @@ ArrayBuilder* ListBuilder::value_builder() const {
 // ----------------------------------------------------------------------
 // String and binary
 
-BinaryBuilder::BinaryBuilder(MemoryPool* pool)
-    : ListBuilder(pool, std::unique_ptr<ArrayBuilder>(new UInt8Builder(pool, uint8())),
-          binary()) {
-  byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
+BinaryBuilder::BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
+    : ArrayBuilder(pool, type), offsets_builder_(pool), value_data_builder_(pool) {}
+
+BinaryBuilder::BinaryBuilder(MemoryPool* pool) : BinaryBuilder(pool, binary()) {}
+
+Status BinaryBuilder::Init(int64_t elements) {
+  DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+  // one more then requested for offsets
+  return offsets_builder_.Resize((elements + 1) * sizeof(int64_t));
+}
+
+Status BinaryBuilder::Resize(int64_t capacity) {
+  DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
+  // one more then requested for offsets
+  RETURN_NOT_OK(offsets_builder_.Resize((capacity + 1) * sizeof(int64_t)));
+  return ArrayBuilder::Resize(capacity);
 }
 
-BinaryBuilder::BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-    : ListBuilder(
-          pool, std::unique_ptr<ArrayBuilder>(new UInt8Builder(pool, uint8())), type) {
-  byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
+Status BinaryBuilder::AppendNextOffset() {
+  const int64_t num_bytes = value_data_builder_.length();
+  if (ARROW_PREDICT_FALSE(num_bytes > kMaximumCapacity)) {
+    std::stringstream ss;
+    ss << "BinaryArray cannot contain more than " << kMaximumCapacity << " bytes, have "
+       << num_bytes;
+    return Status::Invalid(ss.str());
+  }
+  return offsets_builder_.Append(static_cast<int32_t>(num_bytes));
 }
 
-Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<Array> result;
-  RETURN_NOT_OK(ListBuilder::Finish(&result));
+Status BinaryBuilder::Append(const uint8_t* value, int32_t length) {
+  RETURN_NOT_OK(Reserve(1));
+  RETURN_NOT_OK(AppendNextOffset());
+  RETURN_NOT_OK(value_data_builder_.Append(value, length));
+  UnsafeAppendToBitmap(true);
+  return Status::OK();
+}
 
-  const auto list = std::dynamic_pointer_cast<ListArray>(result);
-  auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
+Status BinaryBuilder::AppendNull() {
+  RETURN_NOT_OK(AppendNextOffset());
+  RETURN_NOT_OK(Reserve(1));
+  UnsafeAppendToBitmap(false);
+  return Status::OK();
+}
+
+Status BinaryBuilder::FinishInternal(std::shared_ptr<internal::ArrayData>* out) {
+  // Write final offset (values length)
+  RETURN_NOT_OK(AppendNextOffset());
+  std::shared_ptr<Buffer> offsets, value_data;
 
-  *out = std::make_shared<BinaryArray>(list->length(), list->value_offsets(),
-      values->values(), list->null_bitmap(), list->null_count());
+  RETURN_NOT_OK(offsets_builder_.Finish(&offsets));
+  RETURN_NOT_OK(value_data_builder_.Finish(&value_data));
+
+  BufferVector buffers = {null_bitmap_, offsets, value_data};
+  *out = std::make_shared<internal::ArrayData>(
+      type_, length_, std::move(buffers), null_count_, 0);
   return Status::OK();
 }
 
+Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<internal::ArrayData> data;
+  RETURN_NOT_OK(FinishInternal(&data));
+  *out = std::make_shared<BinaryArray>(data);
+  Reset();
+  return Status::OK();
+}
+
+void BinaryBuilder::Reset() {
+  ArrayBuilder::Reset();
+  offsets_builder_.Reset();
+  value_data_builder_.Reset();
+}
+
 const uint8_t* BinaryBuilder::GetValue(int64_t i, int32_t* out_length) const {
-  const int32_t* offsets = reinterpret_cast<const int32_t*>(offset_builder_.data());
+  const int32_t* offsets = offsets_builder_.data();
   int32_t offset = offsets[i];
   if (i == (length_ - 1)) {
-    *out_length = static_cast<int32_t>(value_builder_->length()) - offset;
+    *out_length = static_cast<int32_t>(value_data_builder_.length()) - offset;
   } else {
     *out_length = offsets[i + 1] - offset;
   }
-  return byte_builder_->data()->data() + offset;
+  return value_data_builder_.data() + offset;
 }
 
 StringBuilder::StringBuilder(MemoryPool* pool) : BinaryBuilder(pool, utf8()) {}
 
 Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<Array> result;
-  RETURN_NOT_OK(ListBuilder::Finish(&result));
-
-  const auto list = std::dynamic_pointer_cast<ListArray>(result);
-  auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
-
-  *out = std::make_shared<StringArray>(list->length(), list->value_offsets(),
-      values->values(), list->null_bitmap(), list->null_count());
+  std::shared_ptr<internal::ArrayData> data;
+  RETURN_NOT_OK(FinishInternal(&data));
+  *out = std::make_shared<StringArray>(data);
+  Reset();
   return Status::OK();
 }
 
@@ -1139,19 +1184,18 @@ Status FixedSizeBinaryBuilder::AppendNull() {
 }
 
 Status FixedSizeBinaryBuilder::Init(int64_t elements) {
-  DCHECK_LT(elements, std::numeric_limits<int64_t>::max());
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
   return byte_builder_.Resize(elements * byte_width_);
 }
 
 Status FixedSizeBinaryBuilder::Resize(int64_t capacity) {
-  DCHECK_LT(capacity, std::numeric_limits<int64_t>::max());
   RETURN_NOT_OK(byte_builder_.Resize(capacity * byte_width_));
   return ArrayBuilder::Resize(capacity);
 }
 
 Status FixedSizeBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<Buffer> data = byte_builder_.Finish();
+  std::shared_ptr<Buffer> data;
+  RETURN_NOT_OK(byte_builder_.Finish(&data));
   *out = std::make_shared<FixedSizeBinaryArray>(
       type_, length_, data, null_bitmap_, null_count_);
   return Status::OK();
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index c0a075b7d6c57..6b54c9f77f3f2 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -38,6 +38,12 @@ namespace arrow {
 
 class Array;
 
+namespace internal {
+
+struct ArrayData;
+
+}  // namespace internal
+
 namespace decimal {
 
 template <typename T>
@@ -127,12 +133,20 @@ class ARROW_EXPORT ArrayBuilder {
   // Child value array builders. These are owned by this class
   std::vector<std::unique_ptr<ArrayBuilder>> children_;
 
-  //
+  void Reset();
+
   // Unsafe operations (don't check capacity/don't resize)
-  //
 
   // Append to null bitmap.
-  void UnsafeAppendToBitmap(bool is_valid);
+  void UnsafeAppendToBitmap(bool is_valid) {
+    if (is_valid) {
+      BitUtil::SetBit(null_bitmap_data_, length_);
+    } else {
+      ++null_count_;
+    }
+    ++length_;
+  }
+
   // Vector append. Treat each zero byte as a nullzero. If valid_bytes is null
   // assume all of length bits are valid.
   void UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t length);
@@ -494,7 +508,8 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 // ----------------------------------------------------------------------
 // List builder
 
-/// Builder class for variable-length list array value types
+/// \class ListBuilder
+/// \brief Builder class for variable-length list array value types
 ///
 /// To use this class, you must append values to the child array builder and use
 /// the Append function to delimit each distinct list value (once the values
@@ -513,22 +528,18 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
       const std::shared_ptr<DataType>& type = nullptr);
 
-  /// Use this constructor to build the list with a pre-existing values array
-  ListBuilder(MemoryPool* pool, std::shared_ptr<Array> values,
-      const std::shared_ptr<DataType>& type = nullptr);
-
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
 
-  /// Vector append
+  /// \brief Vector append
   ///
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
   Status Append(
       const int32_t* offsets, int64_t length, const uint8_t* valid_bytes = nullptr);
 
-  /// Start a new variable-length list slot
+  /// \brief Start a new variable-length list slot
   ///
   /// This function should be called before beginning to append elements to the
   /// value builder
@@ -539,25 +550,26 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   ArrayBuilder* value_builder() const;
 
  protected:
-  BufferBuilder offset_builder_;
+  TypedBufferBuilder<int32_t> offsets_builder_;
   std::unique_ptr<ArrayBuilder> value_builder_;
   std::shared_ptr<Array> values_;
 
+  Status AppendNextOffset();
+
   void Reset();
 };
 
 // ----------------------------------------------------------------------
 // Binary and String
 
-class ARROW_EXPORT BinaryBuilder : public ListBuilder {
+/// \class BinaryBuilder
+/// \brief Builder class for variable-length binary data
+class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
  public:
   explicit BinaryBuilder(MemoryPool* pool);
   explicit BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 
-  Status Append(const uint8_t* value, int32_t length) {
-    RETURN_NOT_OK(ListBuilder::Append());
-    return byte_builder_->Append(value, length);
-  }
+  Status Append(const uint8_t* value, int32_t length);
 
   Status Append(const char* value, int32_t length) {
     return Append(reinterpret_cast<const uint8_t*>(value), length);
@@ -567,6 +579,10 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
     return Append(value.c_str(), static_cast<int32_t>(value.size()));
   }
 
+  Status AppendNull();
+
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
 
   /// Temporary access to a value.
@@ -575,10 +591,18 @@ class ARROW_EXPORT BinaryBuilder : public ListBuilder {
   const uint8_t* GetValue(int64_t i, int32_t* out_length) const;
 
  protected:
-  UInt8Builder* byte_builder_;
+  TypedBufferBuilder<int32_t> offsets_builder_;
+  TypedBufferBuilder<uint8_t> value_data_builder_;
+
+  static constexpr int64_t kMaximumCapacity = std::numeric_limits<int32_t>::max() - 1;
+
+  Status AppendNextOffset();
+  Status FinishInternal(std::shared_ptr<internal::ArrayData>* out);
+  void Reset();
 };
 
-// String builder
+/// \class StringBuilder
+/// \brief Builder class for UTF8 strings
 class ARROW_EXPORT StringBuilder : public BinaryBuilder {
  public:
   using BinaryBuilder::BinaryBuilder;
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 318e3184ba78f..79344df46b243 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -169,7 +169,7 @@ TEST(TestJsonArrayWriter, NestedTypes) {
   std::vector<int32_t> offsets = {0, 0, 0, 1, 4, 7};
 
   std::shared_ptr<Buffer> list_bitmap;
-  ASSERT_OK(test::GetBitmapFromBoolVector(list_is_valid, &list_bitmap));
+  ASSERT_OK(test::GetBitmapFromVector(list_is_valid, &list_bitmap));
   std::shared_ptr<Buffer> offsets_buffer = test::GetBufferFromVector(offsets);
 
   ListArray list_array(list(value_type), 5, offsets_buffer, values_array, list_bitmap, 1);
@@ -179,7 +179,7 @@ TEST(TestJsonArrayWriter, NestedTypes) {
   // Struct
   std::vector<bool> struct_is_valid = {true, false, true, true, true, false, true};
   std::shared_ptr<Buffer> struct_bitmap;
-  ASSERT_OK(test::GetBitmapFromBoolVector(struct_is_valid, &struct_bitmap));
+  ASSERT_OK(test::GetBitmapFromVector(struct_is_valid, &struct_bitmap));
 
   auto struct_type =
       struct_({field("f1", int32()), field("f2", int32()), field("f3", int32())});
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 42f14b0dd1507..2119ff74056f2 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -343,7 +343,7 @@ TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) {
   auto union_type = union_({field("f0", a0->type())}, {0});
   std::vector<int32_t> type_ids(a0->length());
   std::shared_ptr<Buffer> ids_buffer;
-  ASSERT_OK(test::CopyBufferFromVector(type_ids, &ids_buffer));
+  ASSERT_OK(test::CopyBufferFromVector(type_ids, default_memory_pool(), &ids_buffer));
   a1 =
       std::make_shared<UnionArray>(union_type, a0->length(), struct_children, ids_buffer);
   CheckArray(a1);
@@ -355,7 +355,8 @@ TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) {
     type_offsets.push_back(i);
   }
   std::shared_ptr<Buffer> offsets_buffer;
-  ASSERT_OK(test::CopyBufferFromVector(type_offsets, &offsets_buffer));
+  ASSERT_OK(
+      test::CopyBufferFromVector(type_offsets, default_memory_pool(), &offsets_buffer));
   a1 = std::make_shared<UnionArray>(
       dense_union_type, a0->length(), struct_children, ids_buffer, offsets_buffer);
   CheckArray(a1);
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index a542c878e056e..67a41ba086b75 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -139,9 +139,16 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
     std::replace_if(offsets.begin(), offsets.end(),
         [child_length](int32_t offset) { return offset > child_length; }, child_length);
   }
-  ListBuilder builder(pool, child_array);
-  RETURN_NOT_OK(builder.Append(offsets.data(), num_lists, valid_lists.data()));
-  RETURN_NOT_OK(builder.Finish(out));
+
+  offsets[num_lists] = static_cast<int32_t>(child_array->length());
+
+  /// TODO(wesm): Implement support for nulls in ListArray::FromArrays
+  std::shared_ptr<Buffer> null_bitmap, offsets_buffer;
+  RETURN_NOT_OK(test::GetBitmapFromVector(valid_lists, &null_bitmap));
+  RETURN_NOT_OK(test::CopyBufferFromVector(offsets, pool, &offsets_buffer));
+
+  *out = std::make_shared<ListArray>(list(child_array->type()), num_lists, offsets_buffer,
+      child_array, null_bitmap, kUnknownNullCount);
   return ValidateArray(**out);
 }
 
@@ -398,7 +405,8 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
 
   std::shared_ptr<Buffer> type_ids_buffer;
   std::vector<uint8_t> type_ids = {5, 10, 5, 5, 10, 10, 5};
-  RETURN_NOT_OK(test::CopyBufferFromVector(type_ids, &type_ids_buffer));
+  RETURN_NOT_OK(
+      test::CopyBufferFromVector(type_ids, default_memory_pool(), &type_ids_buffer));
 
   std::vector<int32_t> u0_values = {0, 1, 2, 3, 4, 5, 6};
   ArrayFromVector<Int32Type, int32_t>(u0_values, &sparse_children[0]);
@@ -415,7 +423,8 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
 
   std::shared_ptr<Buffer> offsets_buffer;
   std::vector<int32_t> offsets = {0, 0, 1, 2, 1, 2, 3};
-  RETURN_NOT_OK(test::CopyBufferFromVector(offsets, &offsets_buffer));
+  RETURN_NOT_OK(
+      test::CopyBufferFromVector(offsets, default_memory_pool(), &offsets_buffer));
 
   std::vector<uint8_t> null_bytes(length, 1);
   null_bytes[2] = 0;
@@ -479,7 +488,7 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   ArrayFromVector<Int8Type, int8_t>(is_valid3, indices3_values, &indices3);
 
   std::shared_ptr<Buffer> null_bitmap;
-  RETURN_NOT_OK(test::GetBitmapFromBoolVector(is_valid, &null_bitmap));
+  RETURN_NOT_OK(test::GetBitmapFromVector(is_valid, &null_bitmap));
 
   std::shared_ptr<Array> a3 = std::make_shared<ListArray>(f3_type, length,
       std::static_pointer_cast<PrimitiveArray>(offsets)->values(),
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 2cff97a60a514..2bc662526713e 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -102,10 +102,10 @@ std::shared_ptr<Buffer> GetBufferFromVector(const std::vector<T>& values) {
 
 template <typename T>
 inline Status CopyBufferFromVector(
-    const std::vector<T>& values, std::shared_ptr<Buffer>* result) {
+    const std::vector<T>& values, MemoryPool* pool, std::shared_ptr<Buffer>* result) {
   int64_t nbytes = static_cast<int>(values.size()) * sizeof(T);
 
-  auto buffer = std::make_shared<PoolBuffer>(default_memory_pool());
+  auto buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(buffer->Resize(nbytes));
   memcpy(buffer->mutable_data(), values.data(), nbytes);
 
@@ -113,8 +113,9 @@ inline Status CopyBufferFromVector(
   return Status::OK();
 }
 
-static inline Status GetBitmapFromBoolVector(
-    const std::vector<bool>& is_valid, std::shared_ptr<Buffer>* result) {
+template <typename T>
+static inline Status GetBitmapFromVector(
+    const std::vector<T>& is_valid, std::shared_ptr<Buffer>* result) {
   size_t length = is_valid.size();
 
   std::shared_ptr<MutableBuffer> buffer;
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index f8bccc772912b..c7a441597b2e7 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -40,7 +40,6 @@ cdef maybe_coerce_datetime64(values, dtype, DataType type,
     return values, type
 
 
-
 def array(object sequence, DataType type=None, MemoryPool memory_pool=None,
           size=None):
     """
@@ -302,6 +301,19 @@ cdef class Array:
         """
         return [x.as_py() for x in self]
 
+    def validate(self):
+        """
+        Perform any validation checks implemented by
+        arrow::ValidateArray. Raises exception with error message if array does
+        not validate
+
+        Raises
+        ------
+        ArrowInvalid
+        """
+        with nogil:
+            check_status(ValidateArray(deref(self.ap)))
+
 
 cdef class Tensor:
 
@@ -478,7 +490,27 @@ cdef class DecimalArray(FixedSizeBinaryArray):
 
 
 cdef class ListArray(Array):
-    pass
+
+    @staticmethod
+    def from_arrays(Array offsets, Array values, MemoryPool pool=None):
+        """
+        Construct ListArray from arrays of int32 offsets and values
+
+        Parameters
+        ----------
+        offset : Array (int32 type)
+        values : Array (any type)
+
+        Returns
+        -------
+        list_array : ListArray
+        """
+        cdef shared_ptr[CArray] out
+        cdef CMemoryPool* cpool = maybe_unbox_memory_pool(pool)
+        with nogil:
+            check_status(CListArray.FromArrays(
+                deref(offsets.ap), deref(values.ap), cpool, &out))
+        return pyarrow_wrap_array(out)
 
 
 cdef class StringArray(Array):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index dd791cdefb73e..44d83da7e7569 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -283,6 +283,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_string FormatValue(int i)
 
     cdef cppclass CListArray" arrow::ListArray"(CArray):
+        @staticmethod
+        CStatus FromArrays(const CArray& offsets, const CArray& values,
+                           CMemoryPool* pool, shared_ptr[CArray]* out)
+
         const int32_t* raw_value_offsets()
         int32_t value_offset(int i)
         int32_t value_length(int i)
@@ -304,6 +308,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CArray] field(int pos)
         const vector[shared_ptr[CArray]] fields()
 
+    CStatus ValidateArray(const CArray& array)
+
     cdef cppclass CChunkedArray" arrow::ChunkedArray":
         int64_t length()
         int64_t null_count()
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 1a0ee617be9bb..5a373b49a515d 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -199,6 +199,18 @@ def test_dictionary_with_pandas():
     tm.assert_series_equal(pd.Series(pandas2), pd.Series(ex_pandas2))
 
 
+def test_list_from_arrays():
+    offsets_arr = np.array([0, 2, 5, 8], dtype='i4')
+    offsets = pa.Array.from_pandas(offsets_arr, type=pa.int32())
+    pyvalues = [b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h']
+    values = pa.array(pyvalues, type=pa.binary())
+
+    result = pa.ListArray.from_arrays(offsets, values)
+    expected = pa.array([pyvalues[:2], pyvalues[2:5], pyvalues[5:8]])
+
+    assert result.equals(expected)
+
+
 def test_simple_type_construction():
     result = pa.lib.TimestampType()
     with pytest.raises(TypeError):

From b4d34f8fde52df14f087cedcca8b7276b5cf0697 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Mon, 17 Jul 2017 18:33:25 +0200
Subject: [PATCH 0824/1644] ARROW-1191: [JAVA] Implement getField() method for
 complex readers

Author: Steven Phillips <steven@dremio.com>

Closes #818 from StevenMPhillips/bugs/arrow-1191 and squashes the following commits:

5225bb6 [Steven Phillips] ARROW-1191: [JAVA] Implement getField() method for complex readers
---
 java/vector/src/main/codegen/templates/UnionReader.java     | 6 ++++++
 .../arrow/vector/complex/impl/NullableMapReaderImpl.java    | 6 ++++++
 .../arrow/vector/complex/impl/SingleMapReaderImpl.java      | 6 ++++++
 .../apache/arrow/vector/complex/impl/UnionListReader.java   | 6 ++++++
 4 files changed, 24 insertions(+)

diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index 2246fb36642b8..fd3a766fc2cb4 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -18,6 +18,7 @@
 
 
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionReader.java" />
@@ -53,6 +54,11 @@ public MinorType getMinorType() {
     }
   }
 
+  @Override
+  public Field getField() {
+    return data.getField();
+  }
+
   public boolean isSet(){
     return !data.getAccessor().isNull(idx());
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
index 7c389e61ae202..067716e8ea290 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
@@ -21,6 +21,7 @@
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.Field;
 
 public class NullableMapReaderImpl extends SingleMapReaderImpl {
 
@@ -31,6 +32,11 @@ public NullableMapReaderImpl(MapVector vector) {
     this.nullableMapVector = (NullableMapVector)vector;
   }
 
+  @Override
+  public Field getField() {
+    return nullableMapVector.getField();
+  }
+
   @Override
   public void copyAsValue(MapWriter writer){
     NullableMapWriter impl = (NullableMapWriter) writer;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index ae17b4bbb10dd..48019093e387f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -26,6 +26,7 @@
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 
 import com.google.common.collect.Maps;
 
@@ -45,6 +46,11 @@ private void setChildrenPosition(int index){
     }
   }
 
+  @Override
+  public Field getField() {
+    return vector.getField();
+  }
+
   @Override
   public FieldReader reader(String name){
     FieldReader reader = fields.get(name);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
index 6c7c230226ea3..2bd0ca87cd074 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
@@ -26,6 +26,7 @@
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.holders.UnionHolder;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 
 public class UnionListReader extends AbstractFieldReader {
 
@@ -39,6 +40,11 @@ public UnionListReader(ListVector vector) {
     this.offsets = vector.getOffsetVector();
   }
 
+  @Override
+  public Field getField() {
+    return vector.getField();
+  }
+
   @Override
   public boolean isSet() {
     return !vector.getAccessor().isNull(idx());

From a1c8b83b49192230bd2c91bd009e2ff272d89310 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 17 Jul 2017 20:34:51 -0400
Subject: [PATCH 0825/1644] ARROW-1079: [Python] Filter out private directories
 when building Parquet dataset manifest

Some systems like Hive and Impala use special files or directories to signal to other readers that a dataset modification is in progress. If such directories (starting with an underscore) exist in a flat Parquet directory, it currently breaks the dataset reader.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #860 from wesm/ARROW-1079 and squashes the following commits:

c1c445b4 [Wes McKinney] Filter out private directories when building Parquet dataset manifest
---
 python/pyarrow/parquet.py            |  9 +++++++
 python/pyarrow/tests/test_parquet.py | 35 ++++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index dc26daba90597..aa2352c894411 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
 import json
 
 import six
@@ -414,6 +415,9 @@ def _visit_level(self, level, base_path, part_keys):
             elif fs.isdir(path):
                 directories.append(path)
 
+        # ARROW-1079: Filter out "private" directories starting with underscore
+        directories = [x for x in directories if not _is_private_directory(x)]
+
         if len(files) > 0 and len(directories) > 0:
             raise ValueError('Found files in an intermediate '
                              'directory: {0}'.format(base_path))
@@ -456,6 +460,11 @@ def _parse_hive_partition(value):
     return value.split('=', 1)
 
 
+def _is_private_directory(x):
+    _, tail = os.path.split(x)
+    return tail.startswith('_') and '=' not in tail
+
+
 def _path_split(path, sep):
     i = path.rfind(sep) + 1
     head, tail = path[:i], path[i:]
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index f606a7fa0c670..0f44d164ca612 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -36,9 +36,14 @@
 parquet = pytest.mark.parquet
 
 
-def _write_table(*args, **kwargs):
+def _write_table(table, path, **kwargs):
     import pyarrow.parquet as pq
-    return pq.write_table(*args, **kwargs)
+
+    if isinstance(table, pd.DataFrame):
+        table = pa.Table.from_pandas(table)
+
+    pq.write_table(table, path, **kwargs)
+    return table
 
 
 def _read_table(*args, **kwargs):
@@ -851,6 +856,32 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
         read_multiple_files(mixed_paths)
 
 
+@parquet
+def test_ignore_private_directories(tmpdir):
+    import pyarrow.parquet as pq
+
+    nfiles = 10
+    size = 5
+
+    dirpath = tmpdir.join(guid()).strpath
+    os.mkdir(dirpath)
+
+    test_data = []
+    paths = []
+    for i in range(nfiles):
+        df = _test_dataframe(size, seed=i)
+        path = pjoin(dirpath, '{0}.parquet'.format(i))
+
+        test_data.append(_write_table(df, path))
+        paths.append(path)
+
+    # private directory
+    os.mkdir(pjoin(dirpath, '_impala_staging'))
+
+    dataset = pq.ParquetDataset(dirpath)
+    assert set(paths) == set(x.path for x in dataset.pieces)
+
+
 @parquet
 def test_multiindex_duplicate_values(tmpdir):
     num_rows = 3

From 6035d9bd481877154d66c13e4ba8eb2fd8954105 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Mon, 17 Jul 2017 22:35:17 -0400
Subject: [PATCH 0826/1644] ARROW-1233: [C++] Validate libs availability in
 conda toolchain

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #861 from MaxRis/ARROW-1233 and squashes the following commits:

ddc96af1 [Max Risuhin] ARROW-1233: [C++] Validate libs availability in conda toolchain
---
 ci/test-cmake-build-script.bat | 43 ++++++++++++++++++++++++++++++----
 1 file changed, 38 insertions(+), 5 deletions(-)

diff --git a/ci/test-cmake-build-script.bat b/ci/test-cmake-build-script.bat
index e37a8117739f1..54d892fd40ef1 100644
--- a/ci/test-cmake-build-script.bat
+++ b/ci/test-cmake-build-script.bat
@@ -18,13 +18,17 @@
 @echo on
 
 conda update --yes --quiet conda
-conda create -n arrow-cmake-script-tests -q -y
-conda install -n arrow-cmake-script-tests -q -y -c conda-forge ^
+
+@rem Validate cmake script behaviour on missed lib in toolchain
+set CONDA_ENV=arrow-cmake-tests-libs
+conda create -n %CONDA_ENV% -q -y
+conda install -n %CONDA_ENV% -q -y -c conda-forge ^
 cmake git boost-cpp
-call activate arrow-cmake-script-tests
+call activate %CONDA_ENV%
 
-mkdir cpp\build-cmake-test
-pushd cpp\build-cmake-test
+set BUILD_DIR=cpp\build-cmake-test
+mkdir %BUILD_DIR%
+pushd %BUILD_DIR%
 
 echo Test cmake script errors out on flatbuffers missed
 set FLATBUFFERS_HOME=WrongPath
@@ -111,3 +115,32 @@ FINDSTR /M /C:"Could NOT find ZSTD" error.txt || exit /B
 set ZSTD_HOME=
 
 popd
+rmdir /S /Q %BUILD_DIR%
+call deactivate
+
+@rem Validate libs availability in conda toolchain
+mkdir %BUILD_DIR%
+pushd %BUILD_DIR%
+
+set CONDA_ENV=arrow-cmake-tests-toolchain
+conda create -n %CONDA_ENV% -q -y
+conda install -n %CONDA_ENV% -q -y -c conda-forge ^
+      flatbuffers rapidjson cmake git boost-cpp ^
+      thrift-cpp snappy zlib brotli gflags lz4-c zstd
+call activate %CONDA_ENV%
+
+set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+cmake -G "%GENERATOR%" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/MP" ^
+      .. 2>output.txt
+
+set LIBRARY_FOUND_MSG=Added static library dependency
+for %%x in (snappy gflags zlib brotli_enc brotli_dec brotli_common lz4_static zstd_static) do (
+    echo Checking %%x library path
+    FINDSTR /C:"%LIBRARY_FOUND_MSG% %%x: %CONDA_PREFIX:\=/%" output.txt || exit /B
+)
+
+popd
+rmdir /S /Q %BUILD_DIR%
\ No newline at end of file

From 8152433e7815630bf225aade1ffe53b20b393553 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 18 Jul 2017 09:37:06 -0400
Subject: [PATCH 0827/1644] ARROW-1188: [Python] Handle Feather case where
 category values are null type

We already had handled a dense null array in ARROW-1187, but the dictionary write path was circumventing that code

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #863 from wesm/ARROW-1188 and squashes the following commits:

43e970d1 [Wes McKinney] Intercept NullArray in Dictionary Feather write path so that written as an all-null StringArray as with non-dictionary NullArray
a31e3358 [Wes McKinney] Add failing test case
---
 cpp/src/arrow/ipc/feather.cc         | 26 ++++++++++++++++++++------
 python/pyarrow/tests/test_feather.py |  6 ++++++
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 1bcd50520b61a..d5b938b68af71 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -497,6 +497,19 @@ fbs::Type ToFlatbufferType(Type::type type) {
   return fbs::Type_MIN;
 }
 
+static Status SanitizeUnsupportedTypes(
+    const Array& values, std::shared_ptr<Array>* out) {
+  if (values.type_id() == Type::NA) {
+    // As long as R doesn't support NA, we write this as a StringColumn
+    // to ensure stable roundtrips.
+    *out = std::make_shared<StringArray>(
+        values.length(), nullptr, nullptr, values.null_bitmap(), values.null_count());
+    return Status::OK();
+  } else {
+    return MakeArray(values.data(), out);
+  }
+}
+
 class TableWriter::TableWriterImpl : public ArrayVisitor {
  public:
   TableWriterImpl() : initialized_stream_(false), metadata_(0) {}
@@ -622,11 +635,9 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
   }
 
   Status Visit(const NullArray& values) override {
-    // As long as R doesn't support NA, we write this as a StringColumn
-    // to ensure stable roundtrips.
-    StringArray str_values(
-        values.length(), nullptr, nullptr, values.null_bitmap(), values.null_count());
-    return WritePrimitiveValues(str_values);
+    std::shared_ptr<Array> sanitized_nulls;
+    RETURN_NOT_OK(SanitizeUnsupportedTypes(values, &sanitized_nulls));
+    return WritePrimitiveValues(*sanitized_nulls);
   }
 
 #define VISIT_PRIMITIVE(TYPE) \
@@ -658,7 +669,10 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     RETURN_NOT_OK(WritePrimitiveValues(*values.indices()));
 
     ArrayMetadata levels_meta;
-    RETURN_NOT_OK(WriteArray(*dict_type.dictionary(), &levels_meta));
+    std::shared_ptr<Array> sanitized_dictionary;
+    RETURN_NOT_OK(
+        SanitizeUnsupportedTypes(*dict_type.dictionary(), &sanitized_dictionary));
+    RETURN_NOT_OK(WriteArray(*sanitized_dictionary, &levels_meta));
     current_column_->SetCategory(levels_meta, dict_type.ordered());
     return Status::OK();
   }
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 7978ace1c4302..93d67365c8ac5 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -298,6 +298,12 @@ def test_all_none(self):
         df = pd.DataFrame({'all_none': [None] * 10})
         self._check_pandas_roundtrip(df, null_counts=[10])
 
+    def test_all_null_category(self):
+        # ARROW-1188
+        df = pd.DataFrame({"A": (1, 2, 3), "B": (None, None, None)})
+        df = df.assign(B=df.B.astype("category"))
+        self._check_pandas_roundtrip(df, null_counts=[0, 3])
+
     def test_multithreaded_read(self):
         data = {'c{0}'.format(i): [''] * 10
                 for i in range(100)}

From a73252d0e1e8e6e4ca9474571de657c38aedf9a3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 18 Jul 2017 13:51:14 -0400
Subject: [PATCH 0828/1644] ARROW-1235: [C++] Make operator<< for Array/Status
 and std::ostream inline

I'm unable to reproduce the linker failure, but I think this will fix it.

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Wes McKinney <wesmckinn@gmail.com>

Closes #864 from wesm/xcode-6.4-fixes and squashes the following commits:

43bb0b8c [Wes McKinney] Add tests for operator<< with Status and Array
f7524379 [Wes McKinney] Also make operator<<(std::ostream& for Array inline, add Array::ToString
cf2b4719 [Wes McKinney] Make operator<< for Status inline
---
 cpp/src/arrow/array.cc             | 7 ++++---
 cpp/src/arrow/array.h              | 8 +++++++-
 cpp/src/arrow/pretty_print-test.cc | 4 ++++
 cpp/src/arrow/status-test.cc       | 6 ++++++
 cpp/src/arrow/status.cc            | 5 -----
 cpp/src/arrow/status.h             | 5 ++++-
 6 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index f20b8493b17f6..4a405f24342fb 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -107,9 +107,10 @@ std::shared_ptr<Array> Array::Slice(int64_t offset) const {
   return Slice(offset, slice_length);
 }
 
-std::ostream& operator<<(std::ostream& os, const Array& x) {
-  DCHECK(PrettyPrint(x, 0, &os).ok());
-  return os;
+std::string Array::ToString() const {
+  std::stringstream ss;
+  DCHECK(PrettyPrint(*this, 0, &ss).ok());
+  return ss.str();
 }
 
 static inline std::shared_ptr<ArrayData> SliceData(
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 2da0b54b73b35..c32d5e1c93ffd 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -236,6 +236,9 @@ class ARROW_EXPORT Array {
 
   int num_fields() const { return static_cast<int>(data_->child_data.size()); }
 
+  /// \return PrettyPrint representation of array suitable for debugging
+  std::string ToString() const;
+
  protected:
   Array() {}
 
@@ -256,7 +259,10 @@ class ARROW_EXPORT Array {
   DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
-ARROW_EXPORT std::ostream& operator<<(std::ostream& os, const Array& x);
+static inline std::ostream& operator<<(std::ostream& os, const Array& x) {
+  os << x.ToString();
+  return os;
+}
 
 class ARROW_EXPORT FlatArray : public Array {
  protected:
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index 72bd0a8045580..10a91f5e4e461 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -49,6 +49,10 @@ void CheckArray(const Array& arr, int indent, const char* expected) {
   ASSERT_OK(PrettyPrint(arr, indent, &sink));
   std::string result = sink.str();
   ASSERT_EQ(std::string(expected, strlen(expected)), result);
+
+  std::stringstream ss;
+  ss << arr;
+  ASSERT_EQ(result, ss.str());
 }
 
 template <typename TYPE, typename C_TYPE>
diff --git a/cpp/src/arrow/status-test.cc b/cpp/src/arrow/status-test.cc
index 969ba970c154f..f32eb152d13ec 100644
--- a/cpp/src/arrow/status-test.cc
+++ b/cpp/src/arrow/status-test.cc
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <sstream>
+
 #include "gtest/gtest.h"
 
 #include "arrow/status.h"
@@ -33,6 +35,10 @@ TEST(StatusTest, TestCodeAndMessage) {
 TEST(StatusTest, TestToString) {
   Status file_error = Status::IOError("file error");
   ASSERT_EQ("IOError: file error", file_error.ToString());
+
+  std::stringstream ss;
+  ss << file_error;
+  ASSERT_EQ(file_error.ToString(), ss.str());
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/status.cc b/cpp/src/arrow/status.cc
index 77ecb9357b73c..99897428eaed3 100644
--- a/cpp/src/arrow/status.cc
+++ b/cpp/src/arrow/status.cc
@@ -32,11 +32,6 @@ void Status::CopyFrom(const State* state) {
   }
 }
 
-std::ostream& operator<<(std::ostream& os, const Status& x) {
-  os << x.ToString();
-  return os;
-}
-
 std::string Status::CodeAsString() const {
   if (state_ == NULL) { return "OK"; }
 
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 8491ac84730ec..1bea1fca84ebb 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -176,7 +176,10 @@ class ARROW_EXPORT Status {
   void CopyFrom(const State* s);
 };
 
-std::ostream& operator<<(std::ostream& os, const Status& x);
+static inline std::ostream& operator<<(std::ostream& os, const Status& x) {
+  os << x.ToString();
+  return os;
+}
 
 inline Status::Status(const Status& s)
     : state_((s.state_ == NULL) ? NULL : new State(*s.state_)) {}

From 362e754b395787fd935d0b66a0cdcbec8aa13f85 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 18 Jul 2017 13:52:17 -0400
Subject: [PATCH 0829/1644] ARROW-1103: [Python] Support read_pandas (with
 index metadata) on directory of Parquet files

Also fixes ARROW-1041, a case where the `_metadata` file contains the pandas schema metadata but the individual dataset fragments do not.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #862 from wesm/ARROW-1103 and squashes the following commits:

3f309166 [Wes McKinney] Add test for esoteric case where _metadata has pandas metadata but the individual Parquet dataset pieces don't
5985fc13 [Wes McKinney] Add experimental replace_schema_metadata functions, get basic read_pandas with metadata working on a multi-file dataset
b362d60c [Wes McKinney] Initial refactor to support common metadata, read_pandas on a dataset
---
 cpp/src/arrow/builder.cc                |   6 +-
 cpp/src/arrow/ipc/metadata.h            |   7 ++
 cpp/src/arrow/python/builtin_convert.cc |   9 +-
 cpp/src/arrow/python/pandas_convert.cc  |  20 ++-
 cpp/src/arrow/table.cc                  |  16 +++
 cpp/src/arrow/table.h                   |  68 ++++++----
 cpp/src/arrow/type.cc                   |  14 ++-
 cpp/src/arrow/type.h                    |  30 ++++-
 cpp/src/arrow/util/logging.h            |   3 +-
 cpp/src/plasma/protocol.cc              |   4 +-
 python/pyarrow/_parquet.pyx             |  10 +-
 python/pyarrow/filesystem.py            |   8 +-
 python/pyarrow/includes/libarrow.pxd    |  14 ++-
 python/pyarrow/parquet.py               | 161 ++++++++++++++++--------
 python/pyarrow/table.pxi                |  48 +++++++
 python/pyarrow/tests/test_parquet.py    |  85 ++++++++++++-
 python/pyarrow/types.pxi                |   4 +-
 17 files changed, 390 insertions(+), 117 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index a2f24a7ce9cce..ee363b91d8fcc 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -983,9 +983,9 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
     const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool,
-          type ? type : std::static_pointer_cast<DataType>(
-                            std::make_shared<ListType>(value_builder->type()))),
+    : ArrayBuilder(
+          pool, type ? type : std::static_pointer_cast<DataType>(
+                                  std::make_shared<ListType>(value_builder->type()))),
       offsets_builder_(pool),
       value_builder_(std::move(value_builder)) {}
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 64b2571ed464d..614f7a6a922cc 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -200,13 +200,20 @@ class ARROW_EXPORT Message {
 ARROW_EXPORT std::string FormatMessageType(Message::Type type);
 
 /// \brief Abstract interface for a sequence of messages
+/// \since 0.5.0
 class ARROW_EXPORT MessageReader {
  public:
   virtual ~MessageReader() = default;
 
+  /// \brief Read next Message from the interface
+  ///
+  /// \param[out] message an arrow::ipc::Message instance
+  /// \return Status
   virtual Status ReadNextMessage(std::unique_ptr<Message>* message) = 0;
 };
 
+/// \brief Implementation of MessageReader that reads from InputStream
+/// \since 0.5.0
 class ARROW_EXPORT InputStreamMessageReader : public MessageReader {
  public:
   explicit InputStreamMessageReader(const std::shared_ptr<io::InputStream>& stream)
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index fa1c7348b82ba..a76b6ba25531c 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -44,8 +44,8 @@ static inline bool IsPyInteger(PyObject* obj) {
 #endif
 }
 
-Status InvalidConversion(PyObject* obj, const std::string& expected_types,
-    std::ostream* out) {
+Status InvalidConversion(
+    PyObject* obj, const std::string& expected_types, std::ostream* out) {
   OwnedRef type(PyObject_Type(obj));
   RETURN_IF_PYERROR();
   DCHECK_NE(type.obj(), nullptr);
@@ -65,8 +65,7 @@ Status InvalidConversion(PyObject* obj, const std::string& expected_types,
   std::string cpp_type_name(bytes, size);
 
   (*out) << "Got Python object of type " << cpp_type_name
-         << " but can only handle these types: "
-         << expected_types;
+         << " but can only handle these types: " << expected_types;
   return Status::OK();
 }
 
@@ -104,7 +103,7 @@ class ScalarVisitor {
     } else {
       // TODO(wesm): accumulate error information somewhere
       static std::string supported_types =
-        "bool, float, integer, date, datetime, bytes, unicode";
+          "bool, float, integer, date, datetime, bytes, unicode";
       std::stringstream ss;
       ss << "Error inferring Arrow data type for collection of Python objects. ";
       RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index c520d8d242a47..282b3a90b4059 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -552,8 +552,7 @@ Status PandasConverter::ConvertDates() {
       RETURN_NOT_OK(date_builder.AppendNull());
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to "
-         << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
       RETURN_NOT_OK(InvalidConversion(obj, "datetime.date", &ss));
       return Status::Invalid(ss.str());
     }
@@ -608,8 +607,7 @@ Status PandasConverter::ConvertDecimals() {
       RETURN_NOT_OK(decimal_builder.AppendNull());
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to "
-         << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
       RETURN_NOT_OK(InvalidConversion(object, "decimal.Decimal", &ss));
       return Status::Invalid(ss.str());
     }
@@ -637,8 +635,7 @@ Status PandasConverter::ConvertTimes() {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to "
-         << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
       RETURN_NOT_OK(InvalidConversion(obj, "datetime.time", &ss));
       return Status::Invalid(ss.str());
     }
@@ -696,8 +693,7 @@ Status PandasConverter::ConvertObjectFloats() {
       RETURN_NOT_OK(builder.Append(val));
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to "
-         << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
       RETURN_NOT_OK(InvalidConversion(obj, "float", &ss));
       return Status::Invalid(ss.str());
     }
@@ -732,8 +728,7 @@ Status PandasConverter::ConvertObjectIntegers() {
       RETURN_NOT_OK(builder.Append(val));
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to "
-         << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
       RETURN_NOT_OK(InvalidConversion(obj, "integer", &ss));
       return Status::Invalid(ss.str());
     }
@@ -902,8 +897,7 @@ Status PandasConverter::ConvertBooleans() {
       BitUtil::SetBit(null_bitmap_data_, i);
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to "
-         << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
       RETURN_NOT_OK(InvalidConversion(obj, "bool", &ss));
       return Status::Invalid(ss.str());
     }
@@ -990,7 +984,7 @@ Status PandasConverter::ConvertObjects() {
         return ConvertLists(inferred_type);
       } else {
         const std::string supported_types =
-          "string, bool, float, int, date, time, decimal, list, array";
+            "string, bool, float, int, date, time, decimal, list, array";
         std::stringstream ss;
         ss << "Error inferring Arrow type for Python object array. ";
         RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index aa042433c4486..c09628ed395c4 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -164,6 +164,10 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
     std::vector<std::shared_ptr<internal::ArrayData>>&& columns)
     : schema_(schema), num_rows_(num_rows), columns_(std::move(columns)) {}
 
+RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+    const std::vector<std::shared_ptr<internal::ArrayData>>& columns)
+    : schema_(schema), num_rows_(num_rows), columns_(columns) {}
+
 std::shared_ptr<Array> RecordBatch::column(int i) const {
   std::shared_ptr<Array> result;
   DCHECK(MakeArray(columns_[i], &result).ok());
@@ -198,6 +202,12 @@ bool RecordBatch::ApproxEquals(const RecordBatch& other) const {
   return true;
 }
 
+std::shared_ptr<RecordBatch> RecordBatch::ReplaceSchemaMetadata(
+    const std::shared_ptr<const KeyValueMetadata>& metadata) const {
+  auto new_schema = schema_->AddMetadata(metadata);
+  return std::make_shared<RecordBatch>(new_schema, num_rows_, columns_);
+}
+
 std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset) const {
   return Slice(offset, this->num_rows() - offset);
 }
@@ -256,6 +266,12 @@ Table::Table(const std::shared_ptr<Schema>& schema,
     const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
     : schema_(schema), columns_(columns), num_rows_(num_rows) {}
 
+std::shared_ptr<Table> Table::ReplaceSchemaMetadata(
+    const std::shared_ptr<const KeyValueMetadata>& metadata) const {
+  auto new_schema = schema_->AddMetadata(metadata);
+  return std::make_shared<Table>(new_schema, columns_);
+}
+
 Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>& batches,
     std::shared_ptr<Table>* table) {
   if (batches.size() == 0) {
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 18315f32013c0..3ec1df9a5b883 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -41,13 +41,13 @@ class Status;
 
 using ArrayVector = std::vector<std::shared_ptr<Array>>;
 
-// A data structure managing a list of primitive Arrow arrays logically as one
-// large array
+/// \brief A data structure managing a list of primitive Arrow arrays logically
+/// as one large array
 class ARROW_EXPORT ChunkedArray {
  public:
   explicit ChunkedArray(const ArrayVector& chunks);
 
-  // @returns: the total length of the chunked array; computed on construction
+  // \return the total length of the chunked array; computed on construction
   int64_t length() const { return length_; }
 
   int64_t null_count() const { return null_count_; }
@@ -67,9 +67,8 @@ class ARROW_EXPORT ChunkedArray {
   int64_t null_count_;
 };
 
-// An immutable column data structure consisting of a field (type metadata) and
-// a logical chunked data array (which can be validated as all being the same
-// type).
+/// \brief An immutable column data structure consisting of a field (type
+/// metadata) and a logical chunked data array
 class ARROW_EXPORT Column {
  public:
   Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
@@ -86,13 +85,13 @@ class ARROW_EXPORT Column {
 
   std::shared_ptr<Field> field() const { return field_; }
 
-  // @returns: the column's name in the passed metadata
+  // \return the column's name in the passed metadata
   const std::string& name() const { return field_->name(); }
 
-  // @returns: the column's type according to the metadata
+  // \return the column's type according to the metadata
   std::shared_ptr<DataType> type() const { return field_->type(); }
 
-  // @returns: the column's data as a chunked logical array
+  // \return the column's data as a chunked logical array
   std::shared_ptr<ChunkedArray> data() const { return data_; }
 
   bool Equals(const Column& other) const;
@@ -107,9 +106,11 @@ class ARROW_EXPORT Column {
   std::shared_ptr<ChunkedArray> data_;
 };
 
-// A record batch is a simpler and more rigid table data structure intended for
-// use primarily in shared memory IPC. It contains a schema (metadata) and a
-// corresponding sequence of equal-length Arrow arrays
+/// \class RecordBatch
+/// \brief Collection of equal-length arrays matching a particular Schema
+///
+/// A record batch is table-like data structure consisting of an internal
+/// sequence of fields, each a contiguous Arrow array
 class ARROW_EXPORT RecordBatch {
  public:
   /// num_rows is a parameter to allow for record batches of a particular size not
@@ -124,6 +125,7 @@ class ARROW_EXPORT RecordBatch {
       std::vector<std::shared_ptr<Array>>&& columns);
 
   /// \brief Construct record batch from vector of internal data structures
+  /// \since 0.5.0
   ///
   /// This class is only provided with an rvalue-reference for the input data,
   /// and is intended for internal use, or advanced users.
@@ -135,14 +137,19 @@ class ARROW_EXPORT RecordBatch {
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
       std::vector<std::shared_ptr<internal::ArrayData>>&& columns);
 
+  /// \brief Construct record batch by copying vector of array data
+  /// \since 0.5.0
+  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+      const std::vector<std::shared_ptr<internal::ArrayData>>& columns);
+
   bool Equals(const RecordBatch& other) const;
 
   bool ApproxEquals(const RecordBatch& other) const;
 
-  // @returns: the table's schema
+  // \return the table's schema
   std::shared_ptr<Schema> schema() const { return schema_; }
 
-  // @returns: the i-th column
+  // \return the i-th column
   // Note: Does not boundscheck
   std::shared_ptr<Array> column(int i) const;
 
@@ -150,18 +157,27 @@ class ARROW_EXPORT RecordBatch {
 
   const std::string& column_name(int i) const;
 
-  // @returns: the number of columns in the table
+  // \return the number of columns in the table
   int num_columns() const { return static_cast<int>(columns_.size()); }
 
-  // @returns: the number of rows (the corresponding length of each column)
+  // \return the number of rows (the corresponding length of each column)
   int64_t num_rows() const { return num_rows_; }
 
+  /// \brief Replace schema key-value metadata with new metadata (EXPERIMENTAL)
+  /// \since 0.5.0
+  ///
+  /// \param[in] metadata new KeyValueMetadata
+  /// \return new RecordBatch
+  std::shared_ptr<RecordBatch> ReplaceSchemaMetadata(
+      const std::shared_ptr<const KeyValueMetadata>& metadata) const;
+
   /// Slice each of the arrays in the record batch and construct a new RecordBatch object
   std::shared_ptr<RecordBatch> Slice(int64_t offset) const;
   std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length) const;
 
-  /// Returns error status is there is something wrong with the record batch
-  /// contents, like a schema/array mismatch or inconsistent lengths
+  /// \brief Check for schema or length inconsistencies
+  ///
+  /// \return Status
   Status Validate() const;
 
  private:
@@ -190,11 +206,11 @@ class ARROW_EXPORT Table {
       const std::vector<std::shared_ptr<RecordBatch>>& batches,
       std::shared_ptr<Table>* table);
 
-  // @returns: the table's schema
+  // \return the table's schema
   std::shared_ptr<Schema> schema() const { return schema_; }
 
   // Note: Does not boundscheck
-  // @returns: the i-th column
+  // \return the i-th column
   std::shared_ptr<Column> column(int i) const { return columns_[i]; }
 
   /// Remove column from the table, producing a new Table (because tables and
@@ -205,10 +221,18 @@ class ARROW_EXPORT Table {
   Status AddColumn(
       int i, const std::shared_ptr<Column>& column, std::shared_ptr<Table>* out) const;
 
-  // @returns: the number of columns in the table
+  /// \brief Replace schema key-value metadata with new metadata (EXPERIMENTAL)
+  /// \since 0.5.0
+  ///
+  /// \param[in] metadata new KeyValueMetadata
+  /// \return new Table
+  std::shared_ptr<Table> ReplaceSchemaMetadata(
+      const std::shared_ptr<const KeyValueMetadata>& metadata) const;
+
+  // \return the number of columns in the table
   int num_columns() const { return static_cast<int>(columns_.size()); }
 
-  // @returns: the number of rows (the corresponding length of each column)
+  // \return the number of rows (the corresponding length of each column)
   int64_t num_rows() const { return num_rows_; }
 
   bool Equals(const Table& other) const;
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 891045e689b51..623c1934f875e 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -31,9 +31,14 @@
 
 namespace arrow {
 
+std::shared_ptr<Field> Field::AddMetadata(
+    const std::shared_ptr<const KeyValueMetadata>& metadata) const {
+  return std::make_shared<Field>(name_, type_, nullable_, metadata);
+}
+
 Status Field::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
     std::shared_ptr<Field>* out) const {
-  *out = std::make_shared<Field>(name_, type_, nullable_, metadata);
+  *out = AddMetadata(metadata);
   return Status::OK();
 }
 
@@ -294,9 +299,14 @@ Status Schema::AddField(
   return Status::OK();
 }
 
+std::shared_ptr<Schema> Schema::AddMetadata(
+    const std::shared_ptr<const KeyValueMetadata>& metadata) const {
+  return std::make_shared<Schema>(fields_, metadata);
+}
+
 Status Schema::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
     std::shared_ptr<Schema>* out) const {
-  *out = std::make_shared<Schema>(fields_, metadata);
+  *out = AddMetadata(metadata);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 70e8644c11746..fffb840e3cef7 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -210,9 +210,12 @@ class ARROW_EXPORT Field {
 
   std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
 
+  /// \deprecated
   Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
       std::shared_ptr<Field>* out) const;
 
+  std::shared_ptr<Field> AddMetadata(
+      const std::shared_ptr<const KeyValueMetadata>& metadata) const;
   std::shared_ptr<Field> RemoveMetadata() const;
 
   bool Equals(const Field& other) const;
@@ -690,39 +693,56 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 // ----------------------------------------------------------------------
 // Schema
 
+/// \class Schema
+/// \brief Sequence of arrow::Field objects describing the columns of a record
+/// batch or table data structure
 class ARROW_EXPORT Schema {
  public:
   explicit Schema(const std::vector<std::shared_ptr<Field>>& fields,
       const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
   virtual ~Schema() = default;
 
-  // Returns true if all of the schema fields are equal
+  /// Returns true if all of the schema fields are equal
   bool Equals(const Schema& other) const;
 
-  // Return the ith schema element. Does not boundscheck
+  /// Return the ith schema element. Does not boundscheck
   std::shared_ptr<Field> field(int i) const { return fields_[i]; }
 
-  // Returns nullptr if name not found
+  /// Returns nullptr if name not found
   std::shared_ptr<Field> GetFieldByName(const std::string& name) const;
 
-  // Returns -1 if name not found
+  /// Returns -1 if name not found
   int64_t GetFieldIndex(const std::string& name) const;
 
   const std::vector<std::shared_ptr<Field>>& fields() const { return fields_; }
+
+  /// \brief The custom key-value metadata, if any
+  ///
+  /// \return metadata may be nullptr
   std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
 
-  // Render a string representation of the schema suitable for debugging
+  /// \brief Render a string representation of the schema suitable for debugging
   std::string ToString() const;
 
   Status AddField(
       int i, const std::shared_ptr<Field>& field, std::shared_ptr<Schema>* out) const;
   Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
 
+  /// \deprecated
   Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
       std::shared_ptr<Schema>* out) const;
 
+  /// \brief Replace key-value metadata with new metadata
+  ///
+  /// \param[in] metadata new KeyValueMetadata
+  /// \return new Schema
+  std::shared_ptr<Schema> AddMetadata(
+      const std::shared_ptr<const KeyValueMetadata>& metadata) const;
+
+  /// \brief Return copy of Schema without the KeyValueMetadata
   std::shared_ptr<Schema> RemoveMetadata() const;
 
+  /// \brief Return the number of fields (columns) in the schema
   int num_fields() const { return static_cast<int>(fields_.size()); }
 
  private:
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 2fec4fa9e3c3a..b6181219dbae6 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -103,7 +103,8 @@ class NullLog {
 class CerrLog {
  public:
   CerrLog(int severity)  // NOLINT(runtime/explicit)
-      : severity_(severity), has_logged_(false) {}
+      : severity_(severity),
+        has_logged_(false) {}
 
   virtual ~CerrLog() {
     if (has_logged_) { std::cerr << std::endl; }
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 9739d7725a166..246aa29736056 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -38,8 +38,8 @@ to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
 Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer) {
   int64_t type;
   RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
-  ARROW_CHECK(type == message_type)
-      << "type = " << type << ", message_type = " << message_type;
+  ARROW_CHECK(type == message_type) << "type = " << type
+                                    << ", message_type = " << message_type;
   return Status::OK();
 }
 
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 5d446a81044b9..0e0d58e4bf6a0 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -423,11 +423,14 @@ cdef class ParquetReader:
     def set_num_threads(self, int nthreads):
         self.reader.get().set_num_threads(nthreads)
 
-    def read_row_group(self, int i, column_indices=None):
+    def read_row_group(self, int i, column_indices=None, nthreads=None):
         cdef:
             shared_ptr[CTable] ctable
             vector[int] c_column_indices
 
+        if nthreads:
+            self.set_num_threads(nthreads)
+
         if column_indices is not None:
             for index in column_indices:
                 c_column_indices.push_back(index)
@@ -442,11 +445,14 @@ cdef class ParquetReader:
                              .ReadRowGroup(i, &ctable))
         return pyarrow_wrap_table(ctable)
 
-    def read_all(self, column_indices=None):
+    def read_all(self, column_indices=None, nthreads=None):
         cdef:
             shared_ptr[CTable] ctable
             vector[int] c_column_indices
 
+        if nthreads:
+            self.set_num_threads(nthreads)
+
         if column_indices is not None:
             for index in column_indices:
                 c_column_indices.push_back(index)
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 798d96b2cac8d..9fa4f7627775a 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -63,7 +63,7 @@ def isfile(self, path):
         raise NotImplementedError
 
     def read_parquet(self, path, columns=None, metadata=None, schema=None,
-                     nthreads=1):
+                     nthreads=1, use_pandas_metadata=False):
         """
         Read Parquet data from path in file system. Can read from a single file
         or a directory of files
@@ -82,6 +82,9 @@ def read_parquet(self, path, columns=None, metadata=None, schema=None,
         nthreads : int, default 1
             Number of columns to read in parallel. If > 1, requires that the
             underlying file source is threadsafe
+        use_pandas_metadata : boolean, default False
+            If True and file has custom pandas schema metadata, ensure that
+            index columns are also loaded
 
         Returns
         -------
@@ -90,7 +93,8 @@ def read_parquet(self, path, columns=None, metadata=None, schema=None,
         from pyarrow.parquet import ParquetDataset
         dataset = ParquetDataset(path, schema=schema, metadata=metadata,
                                  filesystem=self)
-        return dataset.read(columns=columns, nthreads=nthreads)
+        return dataset.read(columns=columns, nthreads=nthreads,
+                            use_pandas_metadata=use_pandas_metadata)
 
     @property
     def pathsep(self):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 44d83da7e7569..e1fe0c0282411 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -197,8 +197,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
                c_bool nullable, const shared_ptr[CKeyValueMetadata]& metadata)
 
         # Removed const in Cython so don't have to cast to get code to generate
-        CStatus AddMetadata(const shared_ptr[CKeyValueMetadata]& metadata,
-                            shared_ptr[CField]* out)
+        shared_ptr[CField] AddMetadata(
+            const shared_ptr[CKeyValueMetadata]& metadata)
         shared_ptr[CField] RemoveMetadata()
 
 
@@ -224,8 +224,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_string ToString()
 
         # Removed const in Cython so don't have to cast to get code to generate
-        CStatus AddMetadata(const shared_ptr[CKeyValueMetadata]& metadata,
-                            shared_ptr[CSchema]* out)
+        shared_ptr[CSchema] AddMetadata(
+            const shared_ptr[CKeyValueMetadata]& metadata)
         shared_ptr[CSchema] RemoveMetadata()
 
     cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray):
@@ -346,6 +346,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int num_columns()
         int64_t num_rows()
 
+        shared_ptr[CRecordBatch] ReplaceSchemaMetadata(
+            const shared_ptr[CKeyValueMetadata]& metadata)
+
         shared_ptr[CRecordBatch] Slice(int64_t offset)
         shared_ptr[CRecordBatch] Slice(int64_t offset, int64_t length)
 
@@ -370,6 +373,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
                           shared_ptr[CTable]* out)
         CStatus RemoveColumn(int i, shared_ptr[CTable]* out)
 
+        shared_ptr[CTable] ReplaceSchemaMetadata(
+            const shared_ptr[CKeyValueMetadata]& metadata)
+
     cdef cppclass CTensor" arrow::Tensor":
         shared_ptr[CDataType] type()
         shared_ptr[CBuffer] data()
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index aa2352c894411..06b3a3d1e043c 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -45,10 +45,14 @@ class ParquetFile(object):
         see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
     metadata : ParquetFileMetadata, default None
         Use existing metadata object, rather than reading from file.
+    common_metadata : ParquetFileMetadata, default None
+        Will be used in reads for pandas schema metadata if not found in the
+        main file's metadata, no other uses at the moment
     """
-    def __init__(self, source, metadata=None):
+    def __init__(self, source, metadata=None, common_metadata=None):
         self.reader = ParquetReader()
         self.reader.open(source, metadata=metadata)
+        self.common_metadata = common_metadata
 
     @property
     def metadata(self):
@@ -62,7 +66,8 @@ def schema(self):
     def num_row_groups(self):
         return self.reader.num_row_groups
 
-    def read_row_group(self, i, columns=None, nthreads=1):
+    def read_row_group(self, i, columns=None, nthreads=1,
+                       use_pandas_metadata=False):
         """
         Read a single row group from a Parquet file
 
@@ -73,18 +78,21 @@ def read_row_group(self, i, columns=None, nthreads=1):
         nthreads : int, default 1
             Number of columns to read in parallel. If > 1, requires that the
             underlying file source is threadsafe
+        use_pandas_metadata : boolean, default False
+            If True and file has custom pandas schema metadata, ensure that
+            index columns are also loaded
 
         Returns
         -------
         pyarrow.table.Table
             Content of the row group as a table (of columns)
         """
-        column_indices = self._get_column_indices(columns)
-        if nthreads is not None:
-            self.reader.set_num_threads(nthreads)
-        return self.reader.read_row_group(i, column_indices=column_indices)
+        column_indices = self._get_column_indices(
+            columns, use_pandas_metadata=use_pandas_metadata)
+        return self.reader.read_row_group(i, column_indices=column_indices,
+                                          nthreads=nthreads)
 
-    def read(self, columns=None, nthreads=1):
+    def read(self, columns=None, nthreads=1, use_pandas_metadata=False):
         """
         Read a Table from Parquet format
 
@@ -95,40 +103,48 @@ def read(self, columns=None, nthreads=1):
         nthreads : int, default 1
             Number of columns to read in parallel. If > 1, requires that the
             underlying file source is threadsafe
+        use_pandas_metadata : boolean, default False
+            If True and file has custom pandas schema metadata, ensure that
+            index columns are also loaded
 
         Returns
         -------
         pyarrow.table.Table
             Content of the file as a table (of columns)
         """
-        column_indices = self._get_column_indices(columns)
-        if nthreads is not None:
-            self.reader.set_num_threads(nthreads)
+        column_indices = self._get_column_indices(
+            columns, use_pandas_metadata=use_pandas_metadata)
+        return self.reader.read_all(column_indices=column_indices,
+                                    nthreads=nthreads)
 
-        return self.reader.read_all(column_indices=column_indices)
+    def _get_column_indices(self, column_names, use_pandas_metadata=False):
+        if column_names is None:
+            return None
 
-    def read_pandas(self, columns=None, nthreads=1):
-        column_indices = self._get_column_indices(columns)
-        custom_metadata = self.metadata.metadata
+        indices = list(map(self.reader.column_name_idx, column_names))
 
-        if custom_metadata and b'pandas' in custom_metadata:
-            index_columns = json.loads(
-                custom_metadata[b'pandas'].decode('utf8')
-            )['index_columns']
-        else:
-            index_columns = []
+        if use_pandas_metadata:
+            file_keyvalues = self.metadata.metadata
+            common_keyvalues = (self.common_metadata.metadata
+                                if self.common_metadata is not None
+                                else None)
 
-        if column_indices is not None and index_columns:
-            column_indices += map(self.reader.column_name_idx, index_columns)
+            if file_keyvalues and b'pandas' in file_keyvalues:
+                index_columns = _get_pandas_index_columns(file_keyvalues)
+            elif common_keyvalues and b'pandas' in common_keyvalues:
+                index_columns = _get_pandas_index_columns(common_keyvalues)
+            else:
+                index_columns = []
 
-        if nthreads is not None:
-            self.reader.set_num_threads(nthreads)
-        return self.reader.read_all(column_indices=column_indices)
+            if indices is not None and index_columns:
+                indices += map(self.reader.column_name_idx, index_columns)
+
+        return indices
 
-    def _get_column_indices(self, column_names):
-        if column_names is None:
-            return None
-        return list(map(self.reader.column_name_idx, column_names))
+
+def _get_pandas_index_columns(keyvalues):
+    return (json.loads(keyvalues[b'pandas'].decode('utf8'))
+            ['index_columns'])
 
 
 # ----------------------------------------------------------------------
@@ -205,7 +221,7 @@ def _open(self, open_file_func=None):
         return reader
 
     def read(self, columns=None, nthreads=1, partitions=None,
-             open_file_func=None, file=None):
+             open_file_func=None, file=None, use_pandas_metadata=False):
         """
         Read this piece as a pyarrow.Table
 
@@ -218,6 +234,8 @@ def read(self, columns=None, nthreads=1, partitions=None,
         open_file_func : function, default None
             A function that knows how to construct a ParquetFile object given
             the file path in this piece
+        file : file-like object
+            passed to ParquetFile
 
         Returns
         -------
@@ -231,11 +249,14 @@ def read(self, columns=None, nthreads=1, partitions=None,
             # try to read the local path
             reader = ParquetFile(self.path)
 
+        options = dict(columns=columns,
+                       nthreads=nthreads,
+                       use_pandas_metadata=use_pandas_metadata)
+
         if self.row_group is not None:
-            table = reader.read_row_group(self.row_group, columns=columns,
-                                          nthreads=nthreads)
+            table = reader.read_row_group(self.row_group, **options)
         else:
-            table = reader.read(columns=columns, nthreads=nthreads)
+            table = reader.read(**options)
 
         if len(self.partition_keys) > 0:
             if partitions is None:
@@ -509,6 +530,11 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
         (self.pieces, self.partitions,
          self.metadata_path) = _make_manifest(path_or_paths, self.fs)
 
+        if self.metadata_path is not None:
+            self.common_metadata = ParquetFile(self.metadata_path).metadata
+        else:
+            self.common_metadata = None
+
         self.metadata = metadata
         self.schema = schema
 
@@ -540,7 +566,7 @@ def validate_schemas(self):
                                  .format(piece, file_metadata.schema,
                                          self.schema))
 
-    def read(self, columns=None, nthreads=1):
+    def read(self, columns=None, nthreads=1, use_pandas_metadata=False):
         """
         Read multiple Parquet files as a single pyarrow.Table
 
@@ -551,6 +577,8 @@ def read(self, columns=None, nthreads=1):
         nthreads : int, default 1
             Number of columns to read in parallel. Requires that the underlying
             file source is threadsafe
+        use_pandas_metadata : bool, default False
+            Passed through to each dataset piece
 
         Returns
         -------
@@ -563,20 +591,54 @@ def read(self, columns=None, nthreads=1):
         for piece in self.pieces:
             table = piece.read(columns=columns, nthreads=nthreads,
                                partitions=self.partitions,
-                               open_file_func=open_file)
+                               open_file_func=open_file,
+                               use_pandas_metadata=use_pandas_metadata)
             tables.append(table)
 
         all_data = lib.concat_tables(tables)
+
+        if use_pandas_metadata:
+            # We need to ensure that this metadata is set in the Table's schema
+            # so that Table.to_pandas will construct pandas.DataFrame with the
+            # right index
+            common_metadata = self._get_common_pandas_metadata()
+            current_metadata = all_data.schema.metadata or {}
+
+            if common_metadata and b'pandas' not in current_metadata:
+                all_data = all_data.replace_schema_metadata({
+                    b'pandas': common_metadata})
+
         return all_data
 
+    def read_pandas(self, **kwargs):
+        """
+        Read dataset including pandas metadata, if any. Other arguments passed
+        through to ParquetDataset.read, see docstring for further details
+
+        Returns
+        -------
+        pyarrow.Table
+            Content of the file as a table (of columns)
+        """
+        return self.read(use_pandas_metadata=True, **kwargs)
+
+    def _get_common_pandas_metadata(self):
+        if self.common_metadata is None:
+            return None
+
+        keyvalues = self.common_metadata.metadata
+        return keyvalues.get(b'pandas', None)
+
     def _get_open_file_func(self):
         if self.fs is None or isinstance(self.fs, LocalFilesystem):
             def open_file(path, meta=None):
-                return ParquetFile(path, metadata=meta)
+                return ParquetFile(path, metadata=meta,
+                                   common_metadata=self.common_metadata)
         else:
             def open_file(path, meta=None):
                 return ParquetFile(self.fs.open(path, mode='rb'),
-                                   metadata=meta)
+                                   metadata=meta,
+                                   common_metadata=self.common_metadata)
         return open_file
 
 
@@ -613,7 +675,8 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
     return pieces, partitions, metadata_path
 
 
-def read_table(source, columns=None, nthreads=1, metadata=None):
+def read_table(source, columns=None, nthreads=1, metadata=None,
+               use_pandas_metadata=False):
     """
     Read a Table from Parquet format
 
@@ -630,6 +693,9 @@ def read_table(source, columns=None, nthreads=1, metadata=None):
         file source is threadsafe
     metadata : FileMetaData
         If separately computed
+    use_pandas_metadata : boolean, default False
+        If True and file has custom pandas schema metadata, ensure that
+        index columns are also loaded
 
     Returns
     -------
@@ -643,13 +709,14 @@ def read_table(source, columns=None, nthreads=1, metadata=None):
                                    metadata=metadata)
 
     pf = ParquetFile(source, metadata=metadata)
-    return pf.read(columns=columns, nthreads=nthreads)
+    return pf.read(columns=columns, nthreads=nthreads,
+                   use_pandas_metadata=use_pandas_metadata)
 
 
 def read_pandas(source, columns=None, nthreads=1, metadata=None):
     """
-    Read a Table from Parquet format, reconstructing the index values if
-    available.
+    Read a Table from Parquet format, also reading DataFrame index values if
+    known in the file metadata
 
     Parameters
     ----------
@@ -671,16 +738,8 @@ def read_pandas(source, columns=None, nthreads=1, metadata=None):
         Content of the file as a Table of Columns, including DataFrame indexes
         as Columns.
     """
-    if is_string(source):
-        fs = LocalFilesystem.get_instance()
-        if fs.isdir(source):
-            raise NotImplementedError(
-                'Reading a directory of Parquet files with DataFrame index '
-                'metadata is not yet supported'
-            )
-
-    pf = ParquetFile(source, metadata=metadata)
-    return pf.read_pandas(columns=columns, nthreads=nthreads)
+    return read_table(source, columns=columns, nthreads=nthreads,
+                      metadata=metadata, use_pandas_metadata=True)
 
 
 def write_table(table, where, row_group_size=None, version='1.0',
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 575755debc589..d7a6060c4383f 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -388,6 +388,30 @@ cdef class RecordBatch:
         self._check_nullptr()
         return self.batch.num_rows()
 
+    def replace_schema_metadata(self, dict metadata=None):
+        """
+        EXPERIMENTAL: Create shallow copy of record batch by replacing schema
+        key-value metadata with the indicated new metadata (which may be None,
+        which deletes any existing metadata
+
+        Parameters
+        ----------
+        metadata : dict, default None
+
+        Returns
+        -------
+        shallow_copy : RecordBatch
+        """
+        cdef shared_ptr[CKeyValueMetadata] c_meta
+        if metadata is not None:
+            convert_metadata(metadata, &c_meta)
+
+        cdef shared_ptr[CRecordBatch] new_batch
+        with nogil:
+            new_batch = self.batch.ReplaceSchemaMetadata(c_meta)
+
+        return pyarrow_wrap_batch(new_batch)
+
     @property
     def num_columns(self):
         """
@@ -624,6 +648,30 @@ cdef class Table:
             )
         return 0
 
+    def replace_schema_metadata(self, dict metadata=None):
+        """
+        EXPERIMENTAL: Create shallow copy of table by replacing schema
+        key-value metadata with the indicated new metadata (which may be None,
+        which deletes any existing metadata
+
+        Parameters
+        ----------
+        metadata : dict, default None
+
+        Returns
+        -------
+        shallow_copy : Table
+        """
+        cdef shared_ptr[CKeyValueMetadata] c_meta
+        if metadata is not None:
+            convert_metadata(metadata, &c_meta)
+
+        cdef shared_ptr[CTable] new_table
+        with nogil:
+            new_table = self.table.ReplaceSchemaMetadata(c_meta)
+
+        return pyarrow_wrap_table(new_table)
+
     def equals(self, Table other):
         """
         Check if contents of two tables are equal
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 0f44d164ca612..d17eb14be75fc 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -809,9 +809,6 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
 
     assert result.equals(expected)
 
-    with pytest.raises(NotImplementedError):
-        pq.read_pandas(dirpath)
-
     # Read with provided metadata
     metadata = pq.ParquetFile(paths[0]).metadata
 
@@ -856,6 +853,88 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
         read_multiple_files(mixed_paths)
 
 
+@parquet
+def test_dataset_read_pandas(tmpdir):
+    import pyarrow.parquet as pq
+
+    nfiles = 5
+    size = 5
+
+    dirpath = tmpdir.join(guid()).strpath
+    os.mkdir(dirpath)
+
+    test_data = []
+    frames = []
+    paths = []
+    for i in range(nfiles):
+        df = _test_dataframe(size, seed=i)
+        df.index = np.arange(i * size, (i + 1) * size)
+        df.index.name = 'index'
+
+        path = pjoin(dirpath, '{0}.parquet'.format(i))
+
+        table = pa.Table.from_pandas(df)
+        _write_table(table, path)
+        test_data.append(table)
+        frames.append(df)
+        paths.append(path)
+
+    dataset = pq.ParquetDataset(dirpath)
+    columns = ['uint8', 'strings']
+    result = dataset.read_pandas(columns=columns).to_pandas()
+    expected = pd.concat([x[columns] for x in frames])
+
+    tm.assert_frame_equal(result, expected)
+
+
+@parquet
+def test_dataset_read_pandas_common_metadata(tmpdir):
+    # ARROW-1103
+    import pyarrow.parquet as pq
+
+    nfiles = 5
+    size = 5
+
+    dirpath = tmpdir.join(guid()).strpath
+    os.mkdir(dirpath)
+
+    test_data = []
+    frames = []
+    paths = []
+    for i in range(nfiles):
+        df = _test_dataframe(size, seed=i)
+        df.index = pd.Index(np.arange(i * size, (i + 1) * size))
+        df.index.name = 'index'
+
+        path = pjoin(dirpath, '{0}.parquet'.format(i))
+
+        df_ex_index = df.reset_index(drop=True)
+        df_ex_index['index'] = df.index
+        table = pa.Table.from_pandas(df_ex_index,
+                                     preserve_index=False)
+
+        # Obliterate metadata
+        table = table.replace_schema_metadata(None)
+        assert table.schema.metadata is None
+
+        _write_table(table, path)
+        test_data.append(table)
+        frames.append(df)
+        paths.append(path)
+
+    # Write _metadata common file
+    table_for_metadata = pa.Table.from_pandas(df)
+    pq.write_metadata(table_for_metadata.schema,
+                      pjoin(dirpath, '_metadata'))
+
+    dataset = pq.ParquetDataset(dirpath)
+    columns = ['uint8', 'strings']
+    result = dataset.read_pandas(columns=columns).to_pandas()
+    expected = pd.concat([x[columns] for x in frames])
+
+    tm.assert_frame_equal(result, expected)
+
+
 @parquet
 def test_ignore_private_directories(tmpdir):
     import pyarrow.parquet as pq
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 95bfbfbb0b607..a8d7aa0ee8122 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -255,7 +255,7 @@ cdef class Field:
 
         cdef shared_ptr[CField] new_field
         with nogil:
-            check_status(self.field.AddMetadata(c_meta, &new_field))
+            new_field = self.field.AddMetadata(c_meta)
 
         return pyarrow_wrap_field(new_field)
 
@@ -368,7 +368,7 @@ cdef class Schema:
 
         cdef shared_ptr[CSchema] new_schema
         with nogil:
-            check_status(self.schema.AddMetadata(c_meta, &new_schema))
+            new_schema = self.schema.AddMetadata(c_meta)
 
         return pyarrow_wrap_schema(new_schema)
 

From c5a89b7b4fb94c3988677c5d80405ff7e9cfbd18 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 18 Jul 2017 15:18:03 -0400
Subject: [PATCH 0830/1644] ARROW-1120: Support for writing timestamp(ns) to
 Int96

cc @c-nichols

Author: Uwe L. Korn <uwelk@xhochy.com>
Author: Colin Nichols <nichols01@gmail.com>

Closes #865 from xhochy/ARROW-1120 and squashes the following commits:

ff70832f [Uwe L. Korn] Use integer division
99f825d3 [Uwe L. Korn] Add flag for timestamp[ns] roundtrips
7c28835b [Colin Nichols] ARROW-1120 Support for writing timestamp(ns) to Int96
---
 python/pyarrow/_parquet.pxd          |  9 ++++++++
 python/pyarrow/_parquet.pyx          | 17 ++++++++++++--
 python/pyarrow/parquet.py            | 11 ++++++----
 python/pyarrow/tests/test_parquet.py | 33 ++++++++++++++++++++++++----
 4 files changed, 60 insertions(+), 10 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 3d2d0c86ce021..b1cd5eb2c2be0 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -247,8 +247,17 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
         CStatus Open(const CSchema& schema, CMemoryPool* pool,
                      const shared_ptr[OutputStream]& sink,
                      const shared_ptr[WriterProperties]& properties,
+                     const shared_ptr[ArrowWriterProperties]& arrow_properties,
                      unique_ptr[FileWriter]* writer)
 
         CStatus WriteTable(const CTable& table, int64_t chunk_size)
         CStatus NewRowGroup(int64_t chunk_size)
         CStatus Close()
+
+    cdef cppclass ArrowWriterProperties:
+        cppclass Builder:
+            Builder()
+            Builder* disable_deprecated_int96_timestamps()
+            Builder* enable_deprecated_int96_timestamps()
+            shared_ptr[ArrowWriterProperties] build()
+        c_bool support_deprecated_int96_timestamps()
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 0e0d58e4bf6a0..bbe5203352627 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -545,13 +545,14 @@ cdef class ParquetWriter:
 
     cdef readonly:
         object use_dictionary
+        object use_deprecated_int96_timestamps
         object compression
         object version
         int row_group_size
 
     def __cinit__(self, where, Schema schema, use_dictionary=None,
                   compression=None, version=None,
-                  MemoryPool memory_pool=None):
+                  MemoryPool memory_pool=None, use_deprecated_int96_timestamps=False):
         cdef:
             shared_ptr[FileOutputStream] filestream
             shared_ptr[OutputStream] sink
@@ -566,6 +567,7 @@ cdef class ParquetWriter:
         self.use_dictionary = use_dictionary
         self.compression = compression
         self.version = version
+        self.use_deprecated_int96_timestamps = use_deprecated_int96_timestamps
 
         cdef WriterProperties.Builder properties_builder
         self._set_version(&properties_builder)
@@ -573,10 +575,21 @@ cdef class ParquetWriter:
         self._set_dictionary_props(&properties_builder)
         properties = properties_builder.build()
 
+        cdef ArrowWriterProperties.Builder arrow_properties_builder
+        self._set_int96_support(&arrow_properties_builder)
+        arrow_properties = arrow_properties_builder.build()
+
         check_status(
             FileWriter.Open(deref(schema.schema),
                             maybe_unbox_memory_pool(memory_pool),
-                            sink, properties, &self.writer))
+                            sink, properties, arrow_properties,
+                            &self.writer))
+
+    cdef void _set_int96_support(self, ArrowWriterProperties.Builder* props):
+        if self.use_deprecated_int96_timestamps:
+            props.enable_deprecated_int96_timestamps()
+        else:
+            props.disable_deprecated_int96_timestamps()
 
     cdef void _set_version(self, WriterProperties.Builder* props):
         if self.version is not None:
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 06b3a3d1e043c..64cf3300ba19f 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -743,7 +743,8 @@ def read_pandas(source, columns=None, nthreads=1, metadata=None):
 
 
 def write_table(table, where, row_group_size=None, version='1.0',
-                use_dictionary=True, compression='snappy', **kwargs):
+                use_dictionary=True, compression='snappy',
+                use_deprecated_int96_timestamps=False, **kwargs):
     """
     Write a Table to Parquet format
 
@@ -766,12 +767,13 @@ def write_table(table, where, row_group_size=None, version='1.0',
     writer = ParquetWriter(where, table.schema,
                            use_dictionary=use_dictionary,
                            compression=compression,
-                           version=version)
+                           version=version,
+                           use_deprecated_int96_timestamps=use_deprecated_int96_timestamps)
     writer.write_table(table, row_group_size=row_group_size)
     writer.close()
 
 
-def write_metadata(schema, where, version='1.0'):
+def write_metadata(schema, where, version='1.0', use_deprecated_int96_timestamps=False):
     """
     Write metadata-only Parquet file from schema
 
@@ -782,5 +784,6 @@ def write_metadata(schema, where, version='1.0'):
     version : {"1.0", "2.0"}, default "1.0"
         The Parquet format version, defaults to 1.0
     """
-    writer = ParquetWriter(where, schema, version=version)
+    writer = ParquetWriter(where, schema, version=version,
+                           use_deprecated_int96_timestamps=use_deprecated_int96_timestamps)
     writer.close()
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index d17eb14be75fc..40e44b352ace3 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -456,20 +456,45 @@ def test_date_time_types():
     ex_t6 = pa.time32('ms')
     ex_a6 = pa.Array.from_pandas(data4 * 1000, type=ex_t6)
 
-    table = pa.Table.from_arrays([a1, a2, a3, a4, a5, a6],
+    t7 = pa.timestamp('ns')
+    start = pd.Timestamp('2001-01-01').value
+    data7 = np.array([start, start + 1, start + 2], dtype='int64')
+    a7 = pa.Array.from_pandas(data7, type=t7)
+
+    t7_us = pa.timestamp('us')
+    start = pd.Timestamp('2001-01-01').value
+    data7_us = np.array([start, start + 1, start + 2], dtype='int64') // 1000
+    a7_us = pa.Array.from_pandas(data7_us, type=t7_us)
+
+    table = pa.Table.from_arrays([a1, a2, a3, a4, a5, a6, a7],
                                  ['date32', 'date64', 'timestamp[us]',
                                   'time32[s]', 'time64[us]',
-                                  'time32_from64[s]'])
+                                  'time32_from64[s]',
+                                  'timestamp[ns]'])
 
     # date64 as date32
     # time32[s] to time32[ms]
-    expected = pa.Table.from_arrays([a1, a1, a3, a4, a5, ex_a6],
+    # 'timestamp[ns]' to 'timestamp[us]'
+    expected = pa.Table.from_arrays([a1, a1, a3, a4, a5, ex_a6, a7_us],
                                     ['date32', 'date64', 'timestamp[us]',
                                      'time32[s]', 'time64[us]',
-                                     'time32_from64[s]'])
+                                     'time32_from64[s]',
+                                     'timestamp[ns]'])
 
     _check_roundtrip(table, expected=expected, version='2.0')
 
+    # date64 as date32
+    # time32[s] to time32[ms]
+    # 'timestamp[ns]' is saved as INT96 timestamp
+    expected = pa.Table.from_arrays([a1, a1, a3, a4, a5, ex_a6, a7],
+                                    ['date32', 'date64', 'timestamp[us]',
+                                     'time32[s]', 'time64[us]',
+                                     'time32_from64[s]',
+                                     'timestamp[ns]'])
+
+    _check_roundtrip(table, expected=expected, version='2.0',
+                     use_deprecated_int96_timestamps=True)
+
     # Unsupported stuff
     def _assert_unsupported(array):
         table = pa.Table.from_arrays([array], ['unsupported'])

From fe9c7ef50e54354d9037a3175214365c8cc7d85f Mon Sep 17 00:00:00 2001
From: Zahari <zaharid@gmail.com>
Date: Wed, 19 Jul 2017 10:08:27 +0900
Subject: [PATCH 0831/1644] ARROW-1236: Fix lib path in pkg-config file

The `CMAKE_INSTALL_LIBDIR` already contains the prefix on a default
installation (such as in the conda-forge package) and therefore
prepending the prefix results in the wrong path.

While apparently it is not clear what this variable should be
(https://bugzilla.redhat.com/show_bug.cgi?id=795542), it seems that
it's used as an absolute path in the rest of the project.

Author: Zahari <zaharid@gmail.com>

Closes #866 from Zaharid/fixpc and squashes the following commits:

231238c [Zahari] Use better cmake variables
4d1c69a [Zahari] Fix lib path in pkg-config file
---
 cpp/src/arrow/arrow.pc.in               | 5 ++---
 cpp/src/arrow/python/arrow-python.pc.in | 5 ++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/arrow.pc.in b/cpp/src/arrow/arrow.pc.in
index 0debee32a243a..fc16267c7bcc8 100644
--- a/cpp/src/arrow/arrow.pc.in
+++ b/cpp/src/arrow/arrow.pc.in
@@ -15,9 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
-includedir=${prefix}/include
+libdir=@CMAKE_INSTALL_FULL_LIBDIR@
+includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
 
 so_version=@ARROW_SO_VERSION@
 abi_version=@ARROW_ABI_VERSION@
diff --git a/cpp/src/arrow/python/arrow-python.pc.in b/cpp/src/arrow/python/arrow-python.pc.in
index 11e33c512c7d9..52939519874b1 100644
--- a/cpp/src/arrow/python/arrow-python.pc.in
+++ b/cpp/src/arrow/python/arrow-python.pc.in
@@ -15,9 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
-includedir=${prefix}/include
+libdir=@CMAKE_INSTALL_FULL_LIBDIR@
+includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
 
 Name: Apache Arrow Python
 Description: Python integration library for Apache Arrow

From 6999dbd1e1186149693676ce2701763d7bc72c12 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 19 Jul 2017 00:57:25 -0400
Subject: [PATCH 0832/1644] ARROW-935: [Java] Build Javadoc and site with
 OpenJDK8 in Java CI build

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #778 from wesm/ARROW-935 and squashes the following commits:

54d54535 [Wes McKinney] Set simpleLogger.defaultLogLevel to warn
69878e03 [Wes McKinney] Run Maven commands in batch mode
7cc74694 [Wes McKinney] Build Javadoc and site with OpenJDK8
---
 .travis.yml              | 2 +-
 ci/travis_script_java.sh | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index ff2cafede9cde..cdf787c831b0f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -77,7 +77,7 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
   - language: java
     os: linux
-    jdk: openjdk7
+    jdk: openjdk8
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
   - language: java
diff --git a/ci/travis_script_java.sh b/ci/travis_script_java.sh
index 4679f9c6daf87..259b73ec24e11 100755
--- a/ci/travis_script_java.sh
+++ b/ci/travis_script_java.sh
@@ -18,6 +18,9 @@ JAVA_DIR=${TRAVIS_BUILD_DIR}/java
 
 pushd $JAVA_DIR
 
+export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
 mvn -B test
+mvn -B install
+mvn -B site
 
 popd

From 2c5b412c2866b6561d35ba3399036c22b646d699 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 19 Jul 2017 08:16:25 -0400
Subject: [PATCH 0833/1644] ARROW-1167: [Python] Support chunking string
 columns in Table.from_pandas

This resolves the error with converting the dataset in ARROW-1167, which only takes up 4.5 GB in memory but has a single column with over 2GB in binary data.

The unit test for this is not run in CI because of large memory allocation, but can be run with

```
py.test pyarrow --large_memory
```

cc @jeffknupp

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #867 from wesm/ARROW-1167 and squashes the following commits:

dae62326 [Wes McKinney] cpplint
dcdec91a [Wes McKinney] Support ChunkedArray outputs of Array.from_pandas
150e9fc9 [Wes McKinney] Produced ChunkedArray when exceeding 2GB in a single BinaryArray column
707555f8 [Wes McKinney] Split up pandas_convert, make PandasObjectsToArrow return ChunkedArray to accommodate large string data
---
 cpp/src/arrow/builder.h                       |    6 +
 cpp/src/arrow/ipc/feather.cc                  |    3 +-
 cpp/src/arrow/python/CMakeLists.txt           |    6 +-
 cpp/src/arrow/python/api.h                    |    3 +-
 .../{pandas_convert.cc => arrow_to_pandas.cc} | 1284 ++---------------
 .../{pandas_convert.h => arrow_to_pandas.h}   |   10 -
 cpp/src/arrow/python/pandas_to_arrow.cc       | 1099 ++++++++++++++
 cpp/src/arrow/python/pandas_to_arrow.h        |   58 +
 cpp/src/arrow/python/python-test.cc           |    2 +-
 cpp/src/arrow/table.h                         |    3 +
 python/pyarrow/array.pxi                      |   11 +-
 python/pyarrow/includes/libarrow.pxd          |    3 +-
 python/pyarrow/parquet.py                     |   21 +-
 python/pyarrow/public-api.pxi                 |   15 +
 python/pyarrow/table.pxi                      |   38 +-
 python/pyarrow/tests/conftest.py              |    5 +-
 python/pyarrow/tests/test_convert_pandas.py   |   14 +
 17 files changed, 1406 insertions(+), 1175 deletions(-)
 rename cpp/src/arrow/python/{pandas_convert.cc => arrow_to_pandas.cc} (62%)
 rename cpp/src/arrow/python/{pandas_convert.h => arrow_to_pandas.h} (83%)
 create mode 100644 cpp/src/arrow/python/pandas_to_arrow.cc
 create mode 100644 cpp/src/arrow/python/pandas_to_arrow.h

diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 6b54c9f77f3f2..065e115ac5872 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -585,6 +585,9 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
   Status Resize(int64_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
 
+  /// \return size of values buffer so far
+  int64_t value_data_length() const { return value_data_builder_.length(); }
+
   /// Temporary access to a value.
   ///
   /// This pointer becomes invalid on the next modifying operation.
@@ -632,6 +635,9 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
   Status Resize(int64_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
 
+  /// \return size of values buffer so far
+  int64_t value_data_length() const { return byte_builder_.length(); }
+
  protected:
   int32_t byte_width_;
   BufferBuilder byte_builder_;
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index d5b938b68af71..61b96e0c1dc3b 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -497,8 +497,7 @@ fbs::Type ToFlatbufferType(Type::type type) {
   return fbs::Type_MIN;
 }
 
-static Status SanitizeUnsupportedTypes(
-    const Array& values, std::shared_ptr<Array>* out) {
+static Status SanitizeUnsupportedTypes(const Array& values, std::shared_ptr<Array>* out) {
   if (values.type_id() == Type::NA) {
     // As long as R doesn't support NA, we write this as a StringColumn
     // to ensure stable roundtrips.
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index d5e980bac441f..0fdf81e7aa9d7 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -42,6 +42,7 @@ set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
 # ----------------------------------------------------------------------
 
 set(ARROW_PYTHON_SRCS
+  arrow_to_pandas.cc
   builtin_convert.cc
   common.cc
   config.cc
@@ -49,7 +50,7 @@ set(ARROW_PYTHON_SRCS
   init.cc
   io.cc
   numpy_convert.cc
-  pandas_convert.cc
+  pandas_to_arrow.cc
   pyarrow.cc
 )
 
@@ -81,6 +82,7 @@ endif()
 
 install(FILES
   api.h
+  arrow_to_pandas.h
   builtin_convert.h
   common.h
   config.h
@@ -89,7 +91,7 @@ install(FILES
   io.h
   numpy_convert.h
   numpy_interop.h
-  pandas_convert.h
+  pandas_to_arrow.h
   platform.h
   pyarrow.h
   type_traits.h
diff --git a/cpp/src/arrow/python/api.h b/cpp/src/arrow/python/api.h
index 895d1f447ff58..7cb36ad636fc9 100644
--- a/cpp/src/arrow/python/api.h
+++ b/cpp/src/arrow/python/api.h
@@ -18,11 +18,12 @@
 #ifndef ARROW_PYTHON_API_H
 #define ARROW_PYTHON_API_H
 
+#include "arrow/python/arrow_to_pandas.h"
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/io.h"
 #include "arrow/python/numpy_convert.h"
-#include "arrow/python/pandas_convert.h"
+#include "arrow/python/pandas_to_arrow.h"
 
 #endif  // ARROW_PYTHON_API_H
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
similarity index 62%
rename from cpp/src/arrow/python/pandas_convert.cc
rename to cpp/src/arrow/python/arrow_to_pandas.cc
index 282b3a90b4059..d40609fe3fad2 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -19,7 +19,7 @@
 
 #include "arrow/python/numpy_interop.h"
 
-#include "arrow/python/pandas_convert.h"
+#include "arrow/python/arrow_to_pandas.h"
 
 #include <algorithm>
 #include <atomic>
@@ -59,1150 +59,54 @@ namespace py {
 // ----------------------------------------------------------------------
 // Utility code
 
-static inline bool PyFloat_isnan(const PyObject* obj) {
-  if (PyFloat_Check(obj)) {
-    double val = PyFloat_AS_DOUBLE(obj);
-    return val != val;
-  } else {
-    return false;
-  }
-}
-static inline bool PandasObjectIsNull(const PyObject* obj) {
-  return obj == Py_None || obj == numpy_nan || PyFloat_isnan(obj);
-}
-
-static inline bool PyObject_is_string(const PyObject* obj) {
-#if PY_MAJOR_VERSION >= 3
-  return PyUnicode_Check(obj) || PyBytes_Check(obj);
-#else
-  return PyString_Check(obj) || PyUnicode_Check(obj);
-#endif
-}
-
-static inline bool PyObject_is_float(const PyObject* obj) {
-  return PyFloat_Check(obj);
-}
-
-static inline bool PyObject_is_integer(const PyObject* obj) {
-  return (!PyBool_Check(obj)) && PyArray_IsIntegerScalar(obj);
-}
-
-template <int TYPE>
-static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
-  typedef npy_traits<TYPE> traits;
-  typedef typename traits::value_type T;
-
-  int64_t null_count = 0;
-
-  Ndarray1DIndexer<T> values(arr);
-
-  // TODO(wesm): striding
-  for (int i = 0; i < values.size(); ++i) {
-    if (traits::isnull(values[i])) {
-      ++null_count;
-    } else {
-      BitUtil::SetBit(bitmap, i);
-    }
-  }
-
-  return null_count;
-}
-
-// Returns null count
-static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
-  int64_t null_count = 0;
-
-  Ndarray1DIndexer<uint8_t> mask_values(mask);
-  for (int i = 0; i < length; ++i) {
-    if (mask_values[i]) {
-      ++null_count;
-    } else {
-      BitUtil::SetBit(bitmap, i);
-    }
-  }
-  return null_count;
-}
-
-template <int TYPE>
-static int64_t ValuesToValidBytes(
-    const void* data, int64_t length, uint8_t* valid_bytes) {
-  typedef npy_traits<TYPE> traits;
-  typedef typename traits::value_type T;
-
-  int64_t null_count = 0;
-  const T* values = reinterpret_cast<const T*>(data);
-
-  // TODO(wesm): striding
-  for (int i = 0; i < length; ++i) {
-    valid_bytes[i] = !traits::isnull(values[i]);
-    if (traits::isnull(values[i])) null_count++;
-  }
-
-  return null_count;
-}
-
-Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
-  if (PyArray_NDIM(numpy_array) != 1) {
-    return Status::Invalid("only handle 1-dimensional arrays");
-  }
-
-  if (PyArray_DESCR(numpy_array)->type_num != np_type) {
-    return Status::Invalid("can only handle exact conversions");
-  }
-
-  npy_intp* astrides = PyArray_STRIDES(numpy_array);
-  if (astrides[0] != PyArray_DESCR(numpy_array)->elsize) {
-    return Status::Invalid("No support for strided arrays in lists yet");
-  }
-  return Status::OK();
-}
-
-static Status AppendObjectStrings(
-    PyArrayObject* arr, PyArrayObject* mask, StringBuilder* builder, bool* have_bytes) {
-  PyObject* obj;
-
-  Ndarray1DIndexer<PyObject*> objects(arr);
-  Ndarray1DIndexer<uint8_t> mask_values;
-
-  bool have_mask = false;
-  if (mask != nullptr) {
-    mask_values.Init(mask);
-    have_mask = true;
-  }
-
-  for (int64_t i = 0; i < objects.size(); ++i) {
-    obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
-      RETURN_NOT_OK(builder->AppendNull());
-    } else if (PyUnicode_Check(obj)) {
-      obj = PyUnicode_AsUTF8String(obj);
-      if (obj == NULL) {
-        PyErr_Clear();
-        return Status::Invalid("failed converting unicode to UTF8");
-      }
-      const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
-      Status s = builder->Append(PyBytes_AS_STRING(obj), length);
-      Py_DECREF(obj);
-      if (!s.ok()) { return s; }
-    } else if (PyBytes_Check(obj)) {
-      *have_bytes = true;
-      const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
-      RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
-    } else {
-      std::stringstream ss;
-      ss << "Error converting to Python objects to String/UTF8: ";
-      RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
-      return Status::Invalid(ss.str());
-    }
-  }
-
-  return Status::OK();
-}
-
-static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mask,
-    int byte_width, FixedSizeBinaryBuilder* builder) {
-  PyObject* obj;
-
-  Ndarray1DIndexer<PyObject*> objects(arr);
-  Ndarray1DIndexer<uint8_t> mask_values;
-
-  bool have_mask = false;
-  if (mask != nullptr) {
-    mask_values.Init(mask);
-    have_mask = true;
-  }
-
-  for (int64_t i = 0; i < objects.size(); ++i) {
-    obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
-      RETURN_NOT_OK(builder->AppendNull());
-    } else if (PyUnicode_Check(obj)) {
-      obj = PyUnicode_AsUTF8String(obj);
-      if (obj == NULL) {
-        PyErr_Clear();
-        return Status::Invalid("failed converting unicode to UTF8");
-      }
-
-      RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
-      Status s =
-          builder->Append(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj)));
-      Py_DECREF(obj);
-      RETURN_NOT_OK(s);
-    } else if (PyBytes_Check(obj)) {
-      RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
-      RETURN_NOT_OK(
-          builder->Append(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj))));
-    } else {
-      std::stringstream ss;
-      ss << "Error converting to Python objects to FixedSizeBinary: ";
-      RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
-      return Status::Invalid(ss.str());
-    }
-  }
-
-  return Status::OK();
-}
-
-template <typename T>
-struct WrapBytes {};
-
-template <>
-struct WrapBytes<StringArray> {
-  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
-    return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
-  }
-};
-
-template <>
-struct WrapBytes<BinaryArray> {
-  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
-    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
-  }
-};
-
-template <>
-struct WrapBytes<FixedSizeBinaryArray> {
-  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
-    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
-  }
-};
-
-static inline bool ListTypeSupported(const DataType& type) {
-  switch (type.id()) {
-    case Type::UINT8:
-    case Type::INT8:
-    case Type::UINT16:
-    case Type::INT16:
-    case Type::UINT32:
-    case Type::INT32:
-    case Type::INT64:
-    case Type::UINT64:
-    case Type::FLOAT:
-    case Type::DOUBLE:
-    case Type::STRING:
-    case Type::TIMESTAMP:
-      // The above types are all supported.
-      return true;
-    case Type::LIST: {
-      const ListType& list_type = static_cast<const ListType&>(type);
-      return ListTypeSupported(*list_type.value_type());
-    }
-    default:
-      break;
-  }
-  return false;
-}
-
-// ----------------------------------------------------------------------
-// Conversion from NumPy-in-Pandas to Arrow
-
-class PandasConverter {
- public:
-  PandasConverter(
-      MemoryPool* pool, PyObject* ao, PyObject* mo, const std::shared_ptr<DataType>& type)
-      : pool_(pool),
-        type_(type),
-        arr_(reinterpret_cast<PyArrayObject*>(ao)),
-        mask_(nullptr) {
-    if (mo != nullptr && mo != Py_None) { mask_ = reinterpret_cast<PyArrayObject*>(mo); }
-    length_ = static_cast<int64_t>(PyArray_SIZE(arr_));
-  }
-
-  bool is_strided() const {
-    npy_intp* astrides = PyArray_STRIDES(arr_);
-    return astrides[0] != PyArray_DESCR(arr_)->elsize;
-  }
-
-  Status InitNullBitmap() {
-    int64_t null_bytes = BitUtil::BytesForBits(length_);
-
-    null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
-
-    null_bitmap_data_ = null_bitmap_->mutable_data();
-    memset(null_bitmap_data_, 0, static_cast<size_t>(null_bytes));
-
-    return Status::OK();
-  }
-
-  // ----------------------------------------------------------------------
-  // Traditional visitor conversion for non-object arrays
-
-  template <typename ArrowType>
-  Status ConvertData(std::shared_ptr<Buffer>* data);
-
-  template <typename ArrowType>
-  Status VisitNative() {
-    using traits = arrow_traits<ArrowType::type_id>;
-
-    if (mask_ != nullptr || traits::supports_nulls) { RETURN_NOT_OK(InitNullBitmap()); }
-
-    std::shared_ptr<Buffer> data;
-    RETURN_NOT_OK(ConvertData<ArrowType>(&data));
-
-    int64_t null_count = 0;
-    if (mask_ != nullptr) {
-      null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
-    } else if (traits::supports_nulls) {
-      // TODO(wesm): this presumes the NumPy C type and arrow C type are the
-      // same
-      null_count = ValuesToBitmap<traits::npy_type>(arr_, null_bitmap_data_);
-    }
-
-    BufferVector buffers = {null_bitmap_, data};
-    auto array_data = std::make_shared<internal::ArrayData>(
-        type_, length_, std::move(buffers), null_count, 0);
-    return internal::MakeArray(array_data, &out_);
-  }
-
-  template <typename T>
-  typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
-                              std::is_same<BooleanType, T>::value,
-      Status>::type
-  Visit(const T& type) {
-    return VisitNative<T>();
-  }
-
-  Status Visit(const Date32Type& type) { return VisitNative<Int32Type>(); }
-  Status Visit(const Date64Type& type) { return VisitNative<Int64Type>(); }
-  Status Visit(const TimestampType& type) { return VisitNative<TimestampType>(); }
-  Status Visit(const Time32Type& type) { return VisitNative<Int32Type>(); }
-  Status Visit(const Time64Type& type) { return VisitNative<Int64Type>(); }
-
-  Status TypeNotImplemented(std::string type_name) {
-    std::stringstream ss;
-    ss << "PandasConverter doesn't implement <" << type_name << "> conversion. ";
-    return Status::NotImplemented(ss.str());
-  }
-
-  Status Visit(const NullType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Visit(const BinaryType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Visit(const FixedSizeBinaryType& type) {
-    return TypeNotImplemented(type.ToString());
-  }
-
-  Status Visit(const DecimalType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Visit(const DictionaryType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Visit(const NestedType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Convert() {
-    if (PyArray_NDIM(arr_) != 1) {
-      return Status::Invalid("only handle 1-dimensional arrays");
-    }
-
-    if (type_ == nullptr) { return Status::Invalid("Must pass data type"); }
-
-    // Visit the type to perform conversion
-    return VisitTypeInline(*type_, this);
-  }
-
-  std::shared_ptr<Array> result() const { return out_; }
-
-  // ----------------------------------------------------------------------
-  // Conversion logic for various object dtype arrays
-
-  template <int ITEM_TYPE, typename ArrowType>
-  Status ConvertTypedLists(
-      const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list);
-
-  template <typename ArrowType>
-  Status ConvertDates();
-
-  Status ConvertBooleans();
-  Status ConvertObjectStrings();
-  Status ConvertObjectFloats();
-  Status ConvertObjectFixedWidthBytes(const std::shared_ptr<DataType>& type);
-  Status ConvertObjectIntegers();
-  Status ConvertLists(const std::shared_ptr<DataType>& type);
-  Status ConvertLists(
-      const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list);
-  Status ConvertObjects();
-  Status ConvertDecimals();
-  Status ConvertTimes();
-
- protected:
-  MemoryPool* pool_;
-  std::shared_ptr<DataType> type_;
-  PyArrayObject* arr_;
-  PyArrayObject* mask_;
-  int64_t length_;
-
-  // Used in visitor pattern
-  std::shared_ptr<Array> out_;
-
-  std::shared_ptr<ResizableBuffer> null_bitmap_;
-  uint8_t* null_bitmap_data_;
-};
-
-template <typename T>
-void CopyStrided(T* input_data, int64_t length, int64_t stride, T* output_data) {
-  // Passing input_data as non-const is a concession to PyObject*
-  int64_t j = 0;
-  for (int64_t i = 0; i < length; ++i) {
-    output_data[i] = input_data[j];
-    j += stride;
-  }
-}
-
-template <>
-void CopyStrided<PyObject*>(
-    PyObject** input_data, int64_t length, int64_t stride, PyObject** output_data) {
-  int64_t j = 0;
-  for (int64_t i = 0; i < length; ++i) {
-    output_data[i] = input_data[j];
-    if (output_data[i] != nullptr) { Py_INCREF(output_data[i]); }
-    j += stride;
-  }
-}
-
-template <typename ArrowType>
-inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
-  using traits = arrow_traits<ArrowType::type_id>;
-  using T = typename traits::T;
-
-  // Handle LONGLONG->INT64 and other fun things
-  int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
-
-  if (numpy_type_size(traits::npy_type) != numpy_type_size(type_num_compat)) {
-    return Status::NotImplemented("NumPy type casts not yet implemented");
-  }
-
-  if (is_strided()) {
-    // Strided, must copy into new contiguous memory
-    const int64_t stride = PyArray_STRIDES(arr_)[0];
-    const int64_t stride_elements = stride / sizeof(T);
-
-    auto new_buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length_));
-    CopyStrided(reinterpret_cast<T*>(PyArray_DATA(arr_)), length_, stride_elements,
-        reinterpret_cast<T*>(new_buffer->mutable_data()));
-    *data = new_buffer;
-  } else {
-    // Can zero-copy
-    *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
-  }
-  return Status::OK();
-}
-
-template <>
-inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>* data) {
-  int64_t nbytes = BitUtil::BytesForBits(length_);
-  auto buffer = std::make_shared<PoolBuffer>(pool_);
-  RETURN_NOT_OK(buffer->Resize(nbytes));
-
-  Ndarray1DIndexer<uint8_t> values(arr_);
-
-  uint8_t* bitmap = buffer->mutable_data();
-
-  memset(bitmap, 0, nbytes);
-  for (int i = 0; i < length_; ++i) {
-    if (values[i] > 0) { BitUtil::SetBit(bitmap, i); }
-  }
-
-  *data = buffer;
-  return Status::OK();
-}
-
-template <typename T>
-struct UnboxDate {};
-
-template <>
-struct UnboxDate<Date32Type> {
-  static int32_t Unbox(PyObject* obj) {
-    return PyDate_to_days(reinterpret_cast<PyDateTime_Date*>(obj));
-  }
-};
-
-template <>
-struct UnboxDate<Date64Type> {
-  static int64_t Unbox(PyObject* obj) {
-    return PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(obj));
-  }
-};
-
-template <typename ArrowType>
-Status PandasConverter::ConvertDates() {
-  PyAcquireGIL lock;
-
-  using BuilderType = typename TypeTraits<ArrowType>::BuilderType;
-
-  Ndarray1DIndexer<PyObject*> objects(arr_);
-
-  if (mask_ != nullptr) {
-    return Status::NotImplemented("mask not supported in object conversions yet");
-  }
-
-  BuilderType date_builder(pool_);
-  RETURN_NOT_OK(date_builder.Resize(length_));
-
-  /// We have to run this in this compilation unit, since we cannot use the
-  /// datetime API otherwise
-  PyDateTime_IMPORT;
-
-  PyObject* obj;
-  for (int64_t i = 0; i < length_; ++i) {
-    obj = objects[i];
-    if (PyDate_CheckExact(obj)) {
-      RETURN_NOT_OK(date_builder.Append(UnboxDate<ArrowType>::Unbox(obj)));
-    } else if (PandasObjectIsNull(obj)) {
-      RETURN_NOT_OK(date_builder.AppendNull());
-    } else {
-      std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
-      RETURN_NOT_OK(InvalidConversion(obj, "datetime.date", &ss));
-      return Status::Invalid(ss.str());
-    }
-  }
-  return date_builder.Finish(&out_);
-}
-
-#define CONVERT_DECIMAL_CASE(bit_width, builder, object)         \
-  case bit_width: {                                              \
-    decimal::Decimal##bit_width d;                               \
-    std::string string_out;                                      \
-    RETURN_NOT_OK(PythonDecimalToString((object), &string_out)); \
-    RETURN_NOT_OK(FromString(string_out, &d));                   \
-    RETURN_NOT_OK((builder).Append(d));                          \
-    break;                                                       \
-  }
-
-Status PandasConverter::ConvertDecimals() {
-  PyAcquireGIL lock;
-
-  // Import the decimal module and Decimal class
-  OwnedRef decimal;
-  OwnedRef Decimal;
-  RETURN_NOT_OK(ImportModule("decimal", &decimal));
-  RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
-
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-  PyObject* object = objects[0];
-
-  int precision;
-  int scale;
-
-  RETURN_NOT_OK(InferDecimalPrecisionAndScale(object, &precision, &scale));
-
-  type_ = std::make_shared<DecimalType>(precision, scale);
-
-  const int bit_width = std::dynamic_pointer_cast<DecimalType>(type_)->bit_width();
-  DecimalBuilder decimal_builder(pool_, type_);
-  RETURN_NOT_OK(decimal_builder.Resize(length_));
-
-  for (int64_t i = 0; i < length_; ++i) {
-    object = objects[i];
-    if (PyObject_IsInstance(object, Decimal.obj())) {
-      switch (bit_width) {
-        CONVERT_DECIMAL_CASE(32, decimal_builder, object)
-        CONVERT_DECIMAL_CASE(64, decimal_builder, object)
-        CONVERT_DECIMAL_CASE(128, decimal_builder, object)
-        default:
-          break;
-      }
-    } else if (PandasObjectIsNull(object)) {
-      RETURN_NOT_OK(decimal_builder.AppendNull());
-    } else {
-      std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
-      RETURN_NOT_OK(InvalidConversion(object, "decimal.Decimal", &ss));
-      return Status::Invalid(ss.str());
-    }
-  }
-  return decimal_builder.Finish(&out_);
-}
-
-Status PandasConverter::ConvertTimes() {
-  // Convert array of datetime.time objects to Arrow
-  PyAcquireGIL lock;
-  PyDateTime_IMPORT;
-
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-
-  // datetime.time stores microsecond resolution
-  Time64Builder builder(pool_, ::arrow::time64(TimeUnit::MICRO));
-  RETURN_NOT_OK(builder.Resize(length_));
-
-  PyObject* obj;
-  for (int64_t i = 0; i < length_; ++i) {
-    obj = objects[i];
-    if (PyTime_Check(obj)) {
-      RETURN_NOT_OK(builder.Append(PyTime_to_us(obj)));
-    } else if (PandasObjectIsNull(obj)) {
-      RETURN_NOT_OK(builder.AppendNull());
-    } else {
-      std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
-      RETURN_NOT_OK(InvalidConversion(obj, "datetime.time", &ss));
-      return Status::Invalid(ss.str());
-    }
-  }
-  return builder.Finish(&out_);
-}
-
-#undef CONVERT_DECIMAL_CASE
-
-Status PandasConverter::ConvertObjectStrings() {
-  PyAcquireGIL lock;
-
-  // The output type at this point is inconclusive because there may be bytes
-  // and unicode mixed in the object array
-
-  StringBuilder builder(pool_);
-  RETURN_NOT_OK(builder.Resize(length_));
-
-  Status s;
-  bool have_bytes = false;
-  RETURN_NOT_OK(AppendObjectStrings(arr_, mask_, &builder, &have_bytes));
-  RETURN_NOT_OK(builder.Finish(&out_));
-
-  if (have_bytes) {
-    auto binary_data = out_->data()->ShallowCopy();
-    binary_data->type = ::arrow::binary();
-    out_ = std::make_shared<BinaryArray>(binary_data);
-  }
-  return Status::OK();
-}
-
-Status PandasConverter::ConvertObjectFloats() {
-  PyAcquireGIL lock;
-
-  Ndarray1DIndexer<PyObject*> objects(arr_);
-  Ndarray1DIndexer<uint8_t> mask_values;
-
-  bool have_mask = false;
-  if (mask_ != nullptr) {
-    mask_values.Init(mask_);
-    have_mask = true;
-  }
-
-  DoubleBuilder builder(pool_);
-  RETURN_NOT_OK(builder.Resize(length_));
-
-  PyObject* obj;
-  for (int64_t i = 0; i < objects.size(); ++i) {
-    obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
-      RETURN_NOT_OK(builder.AppendNull());
-    } else if (PyFloat_Check(obj)) {
-      double val = PyFloat_AsDouble(obj);
-      RETURN_IF_PYERROR();
-      RETURN_NOT_OK(builder.Append(val));
-    } else {
-      std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
-      RETURN_NOT_OK(InvalidConversion(obj, "float", &ss));
-      return Status::Invalid(ss.str());
-    }
-  }
-
-  return builder.Finish(&out_);
-}
-
-Status PandasConverter::ConvertObjectIntegers() {
-  PyAcquireGIL lock;
-
-  Int64Builder builder(pool_);
-  RETURN_NOT_OK(builder.Resize(length_));
-
-  Ndarray1DIndexer<PyObject*> objects(arr_);
-  Ndarray1DIndexer<uint8_t> mask_values;
-
-  bool have_mask = false;
-  if (mask_ != nullptr) {
-    mask_values.Init(mask_);
-    have_mask = true;
-  }
-
-  PyObject* obj;
-  for (int64_t i = 0; i < objects.size(); ++i) {
-    obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
-      RETURN_NOT_OK(builder.AppendNull());
-    } else if (PyObject_is_integer(obj)) {
-      const int64_t val = static_cast<int64_t>(PyLong_AsLong(obj));
-      RETURN_IF_PYERROR();
-      RETURN_NOT_OK(builder.Append(val));
-    } else {
-      std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
-      RETURN_NOT_OK(InvalidConversion(obj, "integer", &ss));
-      return Status::Invalid(ss.str());
-    }
-  }
-
-  return builder.Finish(&out_);
-}
-
-Status PandasConverter::ConvertObjectFixedWidthBytes(
-    const std::shared_ptr<DataType>& type) {
-  PyAcquireGIL lock;
-
-  Ndarray1DIndexer<PyObject*> objects(arr_);
-
-  int32_t value_size = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
-
-  FixedSizeBinaryBuilder builder(pool_, type);
-  RETURN_NOT_OK(builder.Resize(length_));
-  RETURN_NOT_OK(AppendObjectFixedWidthBytes(arr_, mask_, value_size, &builder));
-  RETURN_NOT_OK(builder.Finish(&out_));
-  return Status::OK();
-}
-
-template <typename T>
-Status validate_precision(int precision) {
-  constexpr static const int maximum_precision = decimal::DecimalPrecision<T>::maximum;
-  if (!(precision > 0 && precision <= maximum_precision)) {
-    std::stringstream ss;
-    ss << "Invalid precision: " << precision << ". Minimum is 1, maximum is "
-       << maximum_precision;
-    return Status::Invalid(ss.str());
-  }
-  return Status::OK();
-}
-
-template <typename T>
-Status RawDecimalToString(
-    const uint8_t* bytes, int precision, int scale, std::string* result) {
-  DCHECK_NE(bytes, nullptr);
-  DCHECK_NE(result, nullptr);
-  RETURN_NOT_OK(validate_precision<T>(precision));
-  decimal::Decimal<T> decimal;
-  FromBytes(bytes, &decimal);
-  *result = ToString(decimal, precision, scale);
-  return Status::OK();
-}
-
-template Status RawDecimalToString<int32_t>(
-    const uint8_t*, int, int, std::string* result);
-template Status RawDecimalToString<int64_t>(
-    const uint8_t*, int, int, std::string* result);
-
-Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
-    bool is_negative, std::string* result) {
-  DCHECK_NE(bytes, nullptr);
-  DCHECK_NE(result, nullptr);
-  RETURN_NOT_OK(validate_precision<boost::multiprecision::int128_t>(precision));
-  decimal::Decimal128 decimal;
-  FromBytes(bytes, is_negative, &decimal);
-  *result = ToString(decimal, precision, scale);
-  return Status::OK();
-}
-
-static Status ConvertDecimals(const ChunkedArray& data, PyObject** out_values) {
-  PyAcquireGIL lock;
-  OwnedRef decimal_ref;
-  OwnedRef Decimal_ref;
-  RETURN_NOT_OK(ImportModule("decimal", &decimal_ref));
-  RETURN_NOT_OK(ImportFromModule(decimal_ref, "Decimal", &Decimal_ref));
-  PyObject* Decimal = Decimal_ref.obj();
-
-  for (int c = 0; c < data.num_chunks(); c++) {
-    auto* arr(static_cast<arrow::DecimalArray*>(data.chunk(c).get()));
-    auto type(std::dynamic_pointer_cast<arrow::DecimalType>(arr->type()));
-    const int precision = type->precision();
-    const int scale = type->scale();
-    const int bit_width = type->bit_width();
-
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (arr->IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values++ = Py_None;
-      } else {
-        const uint8_t* raw_value = arr->GetValue(i);
-        std::string s;
-        switch (bit_width) {
-          case 32:
-            RETURN_NOT_OK(RawDecimalToString<int32_t>(raw_value, precision, scale, &s));
-            break;
-          case 64:
-            RETURN_NOT_OK(RawDecimalToString<int64_t>(raw_value, precision, scale, &s));
-            break;
-          case 128:
-            RETURN_NOT_OK(
-                RawDecimalToString(raw_value, precision, scale, arr->IsNegative(i), &s));
-            break;
-          default:
-            break;
-        }
-        RETURN_NOT_OK(DecimalFromString(Decimal, s, out_values++));
-      }
-    }
-  }
-
-  return Status::OK();
-}
-
-template <typename TYPE>
-static Status ConvertTimes(const ChunkedArray& data, PyObject** out_values) {
-  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
-
-  PyAcquireGIL lock;
-  OwnedRef time_ref;
-
-  PyDateTime_IMPORT;
-
-  for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const ArrayType&>(*data.chunk(c));
-    auto type = std::dynamic_pointer_cast<TYPE>(arr.type());
-    DCHECK(type);
-
-    const TimeUnit::type unit = type->unit();
-
-    for (int64_t i = 0; i < arr.length(); ++i) {
-      if (arr.IsNull(i)) {
-        Py_INCREF(Py_None);
-        *out_values++ = Py_None;
-      } else {
-        RETURN_NOT_OK(PyTime_from_int(arr.Value(i), unit, out_values++));
-        RETURN_IF_PYERROR();
-      }
-    }
-  }
-
-  return Status::OK();
-}
-
-Status PandasConverter::ConvertBooleans() {
-  PyAcquireGIL lock;
-
-  Ndarray1DIndexer<PyObject*> objects(arr_);
-  Ndarray1DIndexer<uint8_t> mask_values;
-
-  bool have_mask = false;
-  if (mask_ != nullptr) {
-    mask_values.Init(mask_);
-    have_mask = true;
-  }
-
-  int64_t nbytes = BitUtil::BytesForBits(length_);
-  auto data = std::make_shared<PoolBuffer>(pool_);
-  RETURN_NOT_OK(data->Resize(nbytes));
-  uint8_t* bitmap = data->mutable_data();
-  memset(bitmap, 0, nbytes);
-
-  int64_t null_count = 0;
-  PyObject* obj;
-  for (int64_t i = 0; i < length_; ++i) {
-    obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
-      ++null_count;
-    } else if (obj == Py_True) {
-      BitUtil::SetBit(bitmap, i);
-      BitUtil::SetBit(null_bitmap_data_, i);
-    } else if (obj == Py_False) {
-      BitUtil::SetBit(null_bitmap_data_, i);
-    } else {
-      std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
-      RETURN_NOT_OK(InvalidConversion(obj, "bool", &ss));
-      return Status::Invalid(ss.str());
-    }
-  }
-
-  out_ = std::make_shared<BooleanArray>(length_, data, null_bitmap_, null_count);
-
-  return Status::OK();
-}
-
-Status PandasConverter::ConvertObjects() {
-  // Python object arrays are annoying, since we could have one of:
-  //
-  // * Strings
-  // * Booleans with nulls
-  // * decimal.Decimals
-  // * Mixed type (not supported at the moment by arrow format)
-  //
-  // Additionally, nulls may be encoded either as np.nan or None. So we have to
-  // do some type inference and conversion
-
-  RETURN_NOT_OK(InitNullBitmap());
-
-  Ndarray1DIndexer<PyObject*> objects;
-
-  PyAcquireGIL lock;
-  objects.Init(arr_);
-  PyDateTime_IMPORT;
-  lock.release();
-
-  // This means we received an explicit type from the user
-  if (type_) {
-    switch (type_->id()) {
-      case Type::STRING:
-        return ConvertObjectStrings();
-      case Type::FIXED_SIZE_BINARY:
-        return ConvertObjectFixedWidthBytes(type_);
-      case Type::BOOL:
-        return ConvertBooleans();
-      case Type::DATE32:
-        return ConvertDates<Date32Type>();
-      case Type::DATE64:
-        return ConvertDates<Date64Type>();
-      case Type::LIST: {
-        const auto& list_field = static_cast<const ListType&>(*type_);
-        return ConvertLists(list_field.value_field()->type());
-      }
-      case Type::DECIMAL:
-        return ConvertDecimals();
-      default:
-        return Status::TypeError("No known conversion to Arrow type");
-    }
-  } else {
-    // Re-acquire GIL
-    lock.acquire();
-
-    OwnedRef decimal;
-    OwnedRef Decimal;
-    RETURN_NOT_OK(ImportModule("decimal", &decimal));
-    RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
-
-    for (int64_t i = 0; i < length_; ++i) {
-      PyObject* obj = objects[i];
-      if (PandasObjectIsNull(obj)) {
-        continue;
-      } else if (PyObject_is_string(obj)) {
-        return ConvertObjectStrings();
-      } else if (PyObject_is_float(obj)) {
-        return ConvertObjectFloats();
-      } else if (PyBool_Check(obj)) {
-        return ConvertBooleans();
-      } else if (PyObject_is_integer(obj)) {
-        return ConvertObjectIntegers();
-      } else if (PyDate_CheckExact(obj)) {
-        // We could choose Date32 or Date64
-        return ConvertDates<Date32Type>();
-      } else if (PyTime_Check(obj)) {
-        return ConvertTimes();
-      } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
-        return ConvertDecimals();
-      } else if (PyList_Check(obj) || PyArray_Check(obj)) {
-        std::shared_ptr<DataType> inferred_type;
-        RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
-        return ConvertLists(inferred_type);
-      } else {
-        const std::string supported_types =
-            "string, bool, float, int, date, time, decimal, list, array";
-        std::stringstream ss;
-        ss << "Error inferring Arrow type for Python object array. ";
-        RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
-        return Status::Invalid(ss.str());
-      }
-    }
-  }
-
-  out_ = std::make_shared<NullArray>(length_);
-  return Status::OK();
-}
-
 template <typename T>
-Status LoopPySequence(PyObject* sequence, T func) {
-  if (PySequence_Check(sequence)) {
-    OwnedRef ref;
-    Py_ssize_t size = PySequence_Size(sequence);
-    if (PyArray_Check(sequence)) {
-      auto array = reinterpret_cast<PyArrayObject*>(sequence);
-      PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(array));
-      for (int64_t i = 0; i < size; ++i) {
-        RETURN_NOT_OK(func(objects[i]));
-      }
-    } else {
-      for (int64_t i = 0; i < size; ++i) {
-        ref.reset(PySequence_GetItem(sequence, i));
-        RETURN_NOT_OK(func(ref.obj()));
-      }
-    }
-  } else if (PyObject_HasAttrString(sequence, "__iter__")) {
-    OwnedRef iter = OwnedRef(PyObject_GetIter(sequence));
-    PyObject* item;
-    while ((item = PyIter_Next(iter.obj()))) {
-      OwnedRef ref = OwnedRef(item);
-      RETURN_NOT_OK(func(ref.obj()));
-    }
-  } else {
-    return Status::TypeError("Object is not a sequence or iterable");
-  }
-
-  return Status::OK();
-}
-
-template <int ITEM_TYPE, typename ArrowType>
-inline Status PandasConverter::ConvertTypedLists(
-    const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list) {
-  typedef npy_traits<ITEM_TYPE> traits;
-  typedef typename traits::value_type T;
-  typedef typename traits::BuilderClass BuilderT;
-
-  PyAcquireGIL lock;
-
-  // TODO: mask not supported here
-  if (mask_ != nullptr) {
-    return Status::NotImplemented("mask not supported in object conversions yet");
-  }
-
-  if (is_strided()) {
-    return Status::NotImplemented("strided arrays not implemented for lists");
-  }
-
-  BuilderT* value_builder = static_cast<BuilderT*>(list_builder.value_builder());
-
-  auto foreach_item = [&](PyObject* object) {
-    if (PandasObjectIsNull(object)) {
-      return list_builder.AppendNull();
-    } else if (PyArray_Check(object)) {
-      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
-      RETURN_NOT_OK(list_builder.Append(true));
-
-      // TODO(uwe): Support more complex numpy array structures
-      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
-
-      int64_t size = PyArray_DIM(numpy_array, 0);
-      auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
-      if (traits::supports_nulls) {
-        RETURN_NOT_OK(null_bitmap_->Resize(size, false));
-        // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
-        // currently support this.
-        // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        ValuesToValidBytes<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        return value_builder->Append(data, size, null_bitmap_->data());
-      } else {
-        return value_builder->Append(data, size);
-      }
-    } else if (PyList_Check(object)) {
-      int64_t size;
-      std::shared_ptr<DataType> inferred_type;
-      RETURN_NOT_OK(list_builder.Append(true));
-      RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
-      if (inferred_type->id() != type->id()) {
-        std::stringstream ss;
-        ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
-        return Status::TypeError(ss.str());
-      }
-      return AppendPySequence(object, size, type, value_builder);
-    } else {
-      return Status::TypeError("Unsupported Python type for list items");
-    }
-  };
-
-  return LoopPySequence(list, foreach_item);
-}
+struct WrapBytes {};
 
 template <>
-inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
-    const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list) {
-  PyAcquireGIL lock;
-  // TODO: If there are bytes involed, convert to Binary representation
-  bool have_bytes = false;
-
-  // TODO: mask not supported here
-  if (mask_ != nullptr) {
-    return Status::NotImplemented("mask not supported in object conversions yet");
+struct WrapBytes<StringArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(data), length);
   }
+};
 
-  if (is_strided()) {
-    return Status::NotImplemented("strided arrays not implemented for lists");
+template <>
+struct WrapBytes<BinaryArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
   }
+};
 
-  auto value_builder = static_cast<StringBuilder*>(list_builder.value_builder());
-
-  auto foreach_item = [&](PyObject* object) {
-    if (PandasObjectIsNull(object)) {
-      return list_builder.AppendNull();
-    } else if (PyArray_Check(object)) {
-      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
-      RETURN_NOT_OK(list_builder.Append(true));
-
-      // TODO(uwe): Support more complex numpy array structures
-      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
-
-      return AppendObjectStrings(numpy_array, nullptr, value_builder, &have_bytes);
-    } else if (PyList_Check(object)) {
-      int64_t size;
-      std::shared_ptr<DataType> inferred_type;
-      RETURN_NOT_OK(list_builder.Append(true));
-      RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
-      if (inferred_type->id() != Type::STRING) {
-        std::stringstream ss;
-        ss << inferred_type->ToString() << " cannot be converted to STRING.";
-        return Status::TypeError(ss.str());
-      }
-      return AppendPySequence(object, size, inferred_type, value_builder);
-    } else {
-      return Status::TypeError("Unsupported Python type for list items");
-    }
-  };
-
-  return LoopPySequence(list, foreach_item);
-}
-
-#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                                 \
-  case Type::TYPE: {                                                           \
-    return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type, list_builder, list); \
+template <>
+struct WrapBytes<FixedSizeBinaryArray> {
+  static inline PyObject* Wrap(const uint8_t* data, int64_t length) {
+    return PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), length);
   }
+};
 
-Status PandasConverter::ConvertLists(
-    const std::shared_ptr<DataType>& type, ListBuilder& list_builder, PyObject* list) {
-  switch (type->id()) {
-    LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
-    LIST_CASE(INT8, NPY_INT8, Int8Type)
-    LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
-    LIST_CASE(INT16, NPY_INT16, Int16Type)
-    LIST_CASE(UINT32, NPY_UINT32, UInt32Type)
-    LIST_CASE(INT32, NPY_INT32, Int32Type)
-    LIST_CASE(UINT64, NPY_UINT64, UInt64Type)
-    LIST_CASE(INT64, NPY_INT64, Int64Type)
-    LIST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
-    LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
-    LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
-    LIST_CASE(STRING, NPY_OBJECT, StringType)
+static inline bool ListTypeSupported(const DataType& type) {
+  switch (type.id()) {
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::INT64:
+    case Type::UINT64:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+    case Type::STRING:
+    case Type::TIMESTAMP:
+      // The above types are all supported.
+      return true;
     case Type::LIST: {
-      const ListType& list_type = static_cast<const ListType&>(*type);
-      auto value_builder = static_cast<ListBuilder*>(list_builder.value_builder());
-
-      auto foreach_item = [&](PyObject* object) {
-        if (PandasObjectIsNull(object)) {
-          return list_builder.AppendNull();
-        } else {
-          RETURN_NOT_OK(list_builder.Append(true));
-          return ConvertLists(list_type.value_type(), *value_builder, object);
-        }
-      };
-
-      return LoopPySequence(list, foreach_item);
-    }
-    default: {
-      std::stringstream ss;
-      ss << "Unknown list item type: ";
-      ss << type->ToString();
-      return Status::TypeError(ss.str());
+      const ListType& list_type = static_cast<const ListType&>(type);
+      return ListTypeSupported(*list_type.value_type());
     }
+    default:
+      break;
   }
-}
-
-Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
-  std::unique_ptr<ArrayBuilder> array_builder;
-  RETURN_NOT_OK(MakeBuilder(pool_, arrow::list(type), &array_builder));
-  ListBuilder& list_builder = static_cast<ListBuilder&>(*array_builder);
-  RETURN_NOT_OK(ConvertLists(type, list_builder, reinterpret_cast<PyObject*>(arr_)));
-  return list_builder.Finish(&out_);
-}
-
-Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
-  PandasConverter converter(pool, ao, mo, type);
-  RETURN_NOT_OK(converter.Convert());
-  *out = converter.result();
-  return Status::OK();
-}
-
-Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
-  PandasConverter converter(pool, ao, mo, type);
-  RETURN_NOT_OK(converter.ConvertObjects());
-  *out = converter.result();
-  return Status::OK();
+  return false;
 }
 
 // ----------------------------------------------------------------------
@@ -1655,6 +559,120 @@ inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values)
   }
 }
 
+template <typename TYPE>
+static Status ConvertTimes(const ChunkedArray& data, PyObject** out_values) {
+  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
+
+  PyAcquireGIL lock;
+  OwnedRef time_ref;
+
+  PyDateTime_IMPORT;
+
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const auto& arr = static_cast<const ArrayType&>(*data.chunk(c));
+    auto type = std::dynamic_pointer_cast<TYPE>(arr.type());
+    DCHECK(type);
+
+    const TimeUnit::type unit = type->unit();
+
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (arr.IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values++ = Py_None;
+      } else {
+        RETURN_NOT_OK(PyTime_from_int(arr.Value(i), unit, out_values++));
+        RETURN_IF_PYERROR();
+      }
+    }
+  }
+
+  return Status::OK();
+}
+
+template <typename T>
+Status ValidateDecimalPrecision(int precision) {
+  constexpr static const int maximum_precision = decimal::DecimalPrecision<T>::maximum;
+  if (!(precision > 0 && precision <= maximum_precision)) {
+    std::stringstream ss;
+    ss << "Invalid precision: " << precision << ". Minimum is 1, maximum is "
+       << maximum_precision;
+    return Status::Invalid(ss.str());
+  }
+  return Status::OK();
+}
+
+template <typename T>
+Status RawDecimalToString(
+    const uint8_t* bytes, int precision, int scale, std::string* result) {
+  DCHECK_NE(bytes, nullptr);
+  DCHECK_NE(result, nullptr);
+  RETURN_NOT_OK(ValidateDecimalPrecision<T>(precision));
+  decimal::Decimal<T> decimal;
+  FromBytes(bytes, &decimal);
+  *result = ToString(decimal, precision, scale);
+  return Status::OK();
+}
+
+template Status RawDecimalToString<int32_t>(
+    const uint8_t*, int, int, std::string* result);
+template Status RawDecimalToString<int64_t>(
+    const uint8_t*, int, int, std::string* result);
+
+Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
+    bool is_negative, std::string* result) {
+  DCHECK_NE(bytes, nullptr);
+  DCHECK_NE(result, nullptr);
+  RETURN_NOT_OK(ValidateDecimalPrecision<boost::multiprecision::int128_t>(precision));
+  decimal::Decimal128 decimal;
+  FromBytes(bytes, is_negative, &decimal);
+  *result = ToString(decimal, precision, scale);
+  return Status::OK();
+}
+
+static Status ConvertDecimals(const ChunkedArray& data, PyObject** out_values) {
+  PyAcquireGIL lock;
+  OwnedRef decimal_ref;
+  OwnedRef Decimal_ref;
+  RETURN_NOT_OK(ImportModule("decimal", &decimal_ref));
+  RETURN_NOT_OK(ImportFromModule(decimal_ref, "Decimal", &Decimal_ref));
+  PyObject* Decimal = Decimal_ref.obj();
+
+  for (int c = 0; c < data.num_chunks(); c++) {
+    auto* arr(static_cast<arrow::DecimalArray*>(data.chunk(c).get()));
+    auto type(std::dynamic_pointer_cast<arrow::DecimalType>(arr->type()));
+    const int precision = type->precision();
+    const int scale = type->scale();
+    const int bit_width = type->bit_width();
+
+    for (int64_t i = 0; i < arr->length(); ++i) {
+      if (arr->IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values++ = Py_None;
+      } else {
+        const uint8_t* raw_value = arr->GetValue(i);
+        std::string s;
+        switch (bit_width) {
+          case 32:
+            RETURN_NOT_OK(RawDecimalToString<int32_t>(raw_value, precision, scale, &s));
+            break;
+          case 64:
+            RETURN_NOT_OK(RawDecimalToString<int64_t>(raw_value, precision, scale, &s));
+            break;
+          case 128:
+            RETURN_NOT_OK(
+                RawDecimalToString(raw_value, precision, scale, arr->IsNegative(i), &s));
+            break;
+          default:
+            break;
+        }
+        RETURN_NOT_OK(DecimalFromString(Decimal, s, out_values++));
+      }
+    }
+  }
+
+  return Status::OK();
+}
+
 #define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum)                \
   case Type::ArrowEnum:                                            \
     RETURN_NOT_OK((ConvertListsLike<ArrowType>(col, out_buffer))); \
diff --git a/cpp/src/arrow/python/pandas_convert.h b/cpp/src/arrow/python/arrow_to_pandas.h
similarity index 83%
rename from cpp/src/arrow/python/pandas_convert.h
rename to cpp/src/arrow/python/arrow_to_pandas.h
index 45c8a1a21fe20..c606dcbbe0aa7 100644
--- a/cpp/src/arrow/python/pandas_convert.h
+++ b/cpp/src/arrow/python/arrow_to_pandas.h
@@ -61,16 +61,6 @@ ARROW_EXPORT
 Status ConvertTableToPandas(
     const std::shared_ptr<Table>& table, int nthreads, PyObject** out);
 
-ARROW_EXPORT
-Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
-
-/// Convert dtype=object arrays. If target data type is not known, pass a type
-/// with nullptr
-ARROW_EXPORT
-Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
-
 }  // namespace py
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
new file mode 100644
index 0000000000000..1368c3605a4e3
--- /dev/null
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -0,0 +1,1099 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Functions for pandas conversion via NumPy
+
+#include "arrow/python/numpy_interop.h"
+
+#include "arrow/python/pandas_to_arrow.h"
+
+#include <cmath>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/status.h"
+#include "arrow/table.h"
+#include "arrow/type_fwd.h"
+#include "arrow/type_traits.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
+#include "arrow/visitor_inline.h"
+
+#include "arrow/python/builtin_convert.h"
+#include "arrow/python/common.h"
+#include "arrow/python/config.h"
+#include "arrow/python/helpers.h"
+#include "arrow/python/numpy-internal.h"
+#include "arrow/python/numpy_convert.h"
+#include "arrow/python/type_traits.h"
+#include "arrow/python/util/datetime.h"
+
+namespace arrow {
+namespace py {
+
+// ----------------------------------------------------------------------
+// Conversion utilities
+
+static inline bool PyFloat_isnan(const PyObject* obj) {
+  if (PyFloat_Check(obj)) {
+    double val = PyFloat_AS_DOUBLE(obj);
+    return val != val;
+  } else {
+    return false;
+  }
+}
+static inline bool PandasObjectIsNull(const PyObject* obj) {
+  return obj == Py_None || obj == numpy_nan || PyFloat_isnan(obj);
+}
+
+static inline bool PyObject_is_string(const PyObject* obj) {
+#if PY_MAJOR_VERSION >= 3
+  return PyUnicode_Check(obj) || PyBytes_Check(obj);
+#else
+  return PyString_Check(obj) || PyUnicode_Check(obj);
+#endif
+}
+
+static inline bool PyObject_is_float(const PyObject* obj) {
+  return PyFloat_Check(obj);
+}
+
+static inline bool PyObject_is_integer(const PyObject* obj) {
+  return (!PyBool_Check(obj)) && PyArray_IsIntegerScalar(obj);
+}
+
+template <int TYPE>
+static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
+  typedef npy_traits<TYPE> traits;
+  typedef typename traits::value_type T;
+
+  int64_t null_count = 0;
+
+  Ndarray1DIndexer<T> values(arr);
+
+  // TODO(wesm): striding
+  for (int i = 0; i < values.size(); ++i) {
+    if (traits::isnull(values[i])) {
+      ++null_count;
+    } else {
+      BitUtil::SetBit(bitmap, i);
+    }
+  }
+
+  return null_count;
+}
+
+// Returns null count
+static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
+  int64_t null_count = 0;
+
+  Ndarray1DIndexer<uint8_t> mask_values(mask);
+  for (int i = 0; i < length; ++i) {
+    if (mask_values[i]) {
+      ++null_count;
+    } else {
+      BitUtil::SetBit(bitmap, i);
+    }
+  }
+  return null_count;
+}
+
+template <int TYPE>
+static int64_t ValuesToValidBytes(
+    const void* data, int64_t length, uint8_t* valid_bytes) {
+  typedef npy_traits<TYPE> traits;
+  typedef typename traits::value_type T;
+
+  int64_t null_count = 0;
+  const T* values = reinterpret_cast<const T*>(data);
+
+  // TODO(wesm): striding
+  for (int i = 0; i < length; ++i) {
+    valid_bytes[i] = !traits::isnull(values[i]);
+    if (traits::isnull(values[i])) null_count++;
+  }
+
+  return null_count;
+}
+
+Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
+  if (PyArray_NDIM(numpy_array) != 1) {
+    return Status::Invalid("only handle 1-dimensional arrays");
+  }
+
+  if (PyArray_DESCR(numpy_array)->type_num != np_type) {
+    return Status::Invalid("can only handle exact conversions");
+  }
+
+  npy_intp* astrides = PyArray_STRIDES(numpy_array);
+  if (astrides[0] != PyArray_DESCR(numpy_array)->elsize) {
+    return Status::Invalid("No support for strided arrays in lists yet");
+  }
+  return Status::OK();
+}
+
+constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max();
+
+/// Append as many string objects from NumPy arrays to a `StringBuilder` as we
+/// can fit
+///
+/// \param[in] offset starting offset for appending
+/// \param[out] values_consumed ending offset where we stopped appending. Will
+/// be length of arr if fully consumed
+/// \param[out] have_bytes true if we encountered any PyBytes object
+static Status AppendObjectStrings(PyArrayObject* arr, PyArrayObject* mask, int64_t offset,
+    StringBuilder* builder, int64_t* end_offset, bool* have_bytes) {
+  PyObject* obj;
+
+  Ndarray1DIndexer<PyObject*> objects(arr);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask != nullptr) {
+    mask_values.Init(mask);
+    have_mask = true;
+  }
+
+  for (; offset < objects.size(); ++offset) {
+    OwnedRef tmp_obj;
+    obj = objects[offset];
+    if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder->AppendNull());
+      continue;
+    } else if (PyUnicode_Check(obj)) {
+      obj = PyUnicode_AsUTF8String(obj);
+      if (obj == NULL) {
+        PyErr_Clear();
+        return Status::Invalid("failed converting unicode to UTF8");
+      }
+      tmp_obj.reset(obj);
+    } else if (PyBytes_Check(obj)) {
+      *have_bytes = true;
+    } else {
+      std::stringstream ss;
+      ss << "Error converting to Python objects to String/UTF8: ";
+      RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
+      return Status::Invalid(ss.str());
+    }
+
+    const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
+    if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
+      break;
+    }
+    RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
+  }
+
+  // If we consumed the whole array, this will be the length of arr
+  *end_offset = offset;
+  return Status::OK();
+}
+
+static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mask,
+    int byte_width, int64_t offset, FixedSizeBinaryBuilder* builder,
+    int64_t* end_offset) {
+  PyObject* obj;
+
+  Ndarray1DIndexer<PyObject*> objects(arr);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask != nullptr) {
+    mask_values.Init(mask);
+    have_mask = true;
+  }
+
+  for (; offset < objects.size(); ++offset) {
+    OwnedRef tmp_obj;
+    obj = objects[offset];
+    if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder->AppendNull());
+      continue;
+    } else if (PyUnicode_Check(obj)) {
+      obj = PyUnicode_AsUTF8String(obj);
+      if (obj == NULL) {
+        PyErr_Clear();
+        return Status::Invalid("failed converting unicode to UTF8");
+      }
+
+      tmp_obj.reset(obj);
+    } else if (!PyBytes_Check(obj)) {
+      std::stringstream ss;
+      ss << "Error converting to Python objects to FixedSizeBinary: ";
+      RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
+      return Status::Invalid(ss.str());
+    }
+
+    RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
+    if (ARROW_PREDICT_FALSE(
+            builder->value_data_length() + byte_width > kBinaryMemoryLimit)) {
+      break;
+    }
+    RETURN_NOT_OK(
+        builder->Append(reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(obj))));
+  }
+
+  // If we consumed the whole array, this will be the length of arr
+  *end_offset = offset;
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Conversion from NumPy-in-Pandas to Arrow
+
+class PandasConverter {
+ public:
+  PandasConverter(
+      MemoryPool* pool, PyObject* ao, PyObject* mo, const std::shared_ptr<DataType>& type)
+      : pool_(pool),
+        type_(type),
+        arr_(reinterpret_cast<PyArrayObject*>(ao)),
+        mask_(nullptr) {
+    if (mo != nullptr && mo != Py_None) { mask_ = reinterpret_cast<PyArrayObject*>(mo); }
+    length_ = static_cast<int64_t>(PyArray_SIZE(arr_));
+  }
+
+  bool is_strided() const {
+    npy_intp* astrides = PyArray_STRIDES(arr_);
+    return astrides[0] != PyArray_DESCR(arr_)->elsize;
+  }
+
+  Status InitNullBitmap() {
+    int64_t null_bytes = BitUtil::BytesForBits(length_);
+
+    null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
+
+    null_bitmap_data_ = null_bitmap_->mutable_data();
+    memset(null_bitmap_data_, 0, static_cast<size_t>(null_bytes));
+
+    return Status::OK();
+  }
+
+  // ----------------------------------------------------------------------
+  // Traditional visitor conversion for non-object arrays
+
+  template <typename ArrowType>
+  Status ConvertData(std::shared_ptr<Buffer>* data);
+
+  template <typename T>
+  Status PushBuilderResult(T* builder) {
+    std::shared_ptr<Array> out;
+    RETURN_NOT_OK(builder->Finish(&out));
+    out_arrays_.emplace_back(out);
+    return Status::OK();
+  }
+
+  Status PushArray(const std::shared_ptr<internal::ArrayData>& data) {
+    std::shared_ptr<Array> result;
+    RETURN_NOT_OK(internal::MakeArray(data, &result));
+    out_arrays_.emplace_back(std::move(result));
+    return Status::OK();
+  }
+
+  template <typename ArrowType>
+  Status VisitNative() {
+    using traits = arrow_traits<ArrowType::type_id>;
+
+    if (mask_ != nullptr || traits::supports_nulls) { RETURN_NOT_OK(InitNullBitmap()); }
+
+    std::shared_ptr<Buffer> data;
+    RETURN_NOT_OK(ConvertData<ArrowType>(&data));
+
+    int64_t null_count = 0;
+    if (mask_ != nullptr) {
+      null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
+    } else if (traits::supports_nulls) {
+      // TODO(wesm): this presumes the NumPy C type and arrow C type are the
+      // same
+      null_count = ValuesToBitmap<traits::npy_type>(arr_, null_bitmap_data_);
+    }
+
+    BufferVector buffers = {null_bitmap_, data};
+    return PushArray(std::make_shared<internal::ArrayData>(
+        type_, length_, std::move(buffers), null_count, 0));
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
+                              std::is_same<BooleanType, T>::value,
+      Status>::type
+  Visit(const T& type) {
+    return VisitNative<T>();
+  }
+
+  Status Visit(const Date32Type& type) { return VisitNative<Int32Type>(); }
+  Status Visit(const Date64Type& type) { return VisitNative<Int64Type>(); }
+  Status Visit(const TimestampType& type) { return VisitNative<TimestampType>(); }
+  Status Visit(const Time32Type& type) { return VisitNative<Int32Type>(); }
+  Status Visit(const Time64Type& type) { return VisitNative<Int64Type>(); }
+
+  Status TypeNotImplemented(std::string type_name) {
+    std::stringstream ss;
+    ss << "PandasConverter doesn't implement <" << type_name << "> conversion. ";
+    return Status::NotImplemented(ss.str());
+  }
+
+  Status Visit(const NullType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const BinaryType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const FixedSizeBinaryType& type) {
+    return TypeNotImplemented(type.ToString());
+  }
+
+  Status Visit(const DecimalType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const DictionaryType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const NestedType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Convert() {
+    if (PyArray_NDIM(arr_) != 1) {
+      return Status::Invalid("only handle 1-dimensional arrays");
+    }
+
+    if (type_ == nullptr) { return Status::Invalid("Must pass data type"); }
+
+    // Visit the type to perform conversion
+    return VisitTypeInline(*type_, this);
+  }
+
+  const std::vector<std::shared_ptr<Array>>& result() const { return out_arrays_; }
+
+  // ----------------------------------------------------------------------
+  // Conversion logic for various object dtype arrays
+
+  template <int ITEM_TYPE, typename ArrowType>
+  Status ConvertTypedLists(
+      const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list);
+
+  template <typename ArrowType>
+  Status ConvertDates();
+
+  Status ConvertBooleans();
+  Status ConvertObjectStrings();
+  Status ConvertObjectFloats();
+  Status ConvertObjectFixedWidthBytes(const std::shared_ptr<DataType>& type);
+  Status ConvertObjectIntegers();
+  Status ConvertLists(const std::shared_ptr<DataType>& type);
+  Status ConvertLists(
+      const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list);
+  Status ConvertObjects();
+  Status ConvertDecimals();
+  Status ConvertTimes();
+
+ protected:
+  MemoryPool* pool_;
+  std::shared_ptr<DataType> type_;
+  PyArrayObject* arr_;
+  PyArrayObject* mask_;
+  int64_t length_;
+
+  // Used in visitor pattern
+  std::vector<std::shared_ptr<Array>> out_arrays_;
+
+  std::shared_ptr<ResizableBuffer> null_bitmap_;
+  uint8_t* null_bitmap_data_;
+};
+
+template <typename T>
+void CopyStrided(T* input_data, int64_t length, int64_t stride, T* output_data) {
+  // Passing input_data as non-const is a concession to PyObject*
+  int64_t j = 0;
+  for (int64_t i = 0; i < length; ++i) {
+    output_data[i] = input_data[j];
+    j += stride;
+  }
+}
+
+template <>
+void CopyStrided<PyObject*>(
+    PyObject** input_data, int64_t length, int64_t stride, PyObject** output_data) {
+  int64_t j = 0;
+  for (int64_t i = 0; i < length; ++i) {
+    output_data[i] = input_data[j];
+    if (output_data[i] != nullptr) { Py_INCREF(output_data[i]); }
+    j += stride;
+  }
+}
+
+template <typename ArrowType>
+inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
+  using traits = arrow_traits<ArrowType::type_id>;
+  using T = typename traits::T;
+
+  // Handle LONGLONG->INT64 and other fun things
+  int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
+
+  if (numpy_type_size(traits::npy_type) != numpy_type_size(type_num_compat)) {
+    return Status::NotImplemented("NumPy type casts not yet implemented");
+  }
+
+  if (is_strided()) {
+    // Strided, must copy into new contiguous memory
+    const int64_t stride = PyArray_STRIDES(arr_)[0];
+    const int64_t stride_elements = stride / sizeof(T);
+
+    auto new_buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length_));
+    CopyStrided(reinterpret_cast<T*>(PyArray_DATA(arr_)), length_, stride_elements,
+        reinterpret_cast<T*>(new_buffer->mutable_data()));
+    *data = new_buffer;
+  } else {
+    // Can zero-copy
+    *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
+  }
+  return Status::OK();
+}
+
+template <>
+inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>* data) {
+  int64_t nbytes = BitUtil::BytesForBits(length_);
+  auto buffer = std::make_shared<PoolBuffer>(pool_);
+  RETURN_NOT_OK(buffer->Resize(nbytes));
+
+  Ndarray1DIndexer<uint8_t> values(arr_);
+
+  uint8_t* bitmap = buffer->mutable_data();
+
+  memset(bitmap, 0, nbytes);
+  for (int i = 0; i < length_; ++i) {
+    if (values[i] > 0) { BitUtil::SetBit(bitmap, i); }
+  }
+
+  *data = buffer;
+  return Status::OK();
+}
+
+template <typename T>
+struct UnboxDate {};
+
+template <>
+struct UnboxDate<Date32Type> {
+  static int32_t Unbox(PyObject* obj) {
+    return PyDate_to_days(reinterpret_cast<PyDateTime_Date*>(obj));
+  }
+};
+
+template <>
+struct UnboxDate<Date64Type> {
+  static int64_t Unbox(PyObject* obj) {
+    return PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(obj));
+  }
+};
+
+template <typename ArrowType>
+Status PandasConverter::ConvertDates() {
+  PyAcquireGIL lock;
+
+  using BuilderType = typename TypeTraits<ArrowType>::BuilderType;
+
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+
+  if (mask_ != nullptr) {
+    return Status::NotImplemented("mask not supported in object conversions yet");
+  }
+
+  BuilderType builder(pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  /// We have to run this in this compilation unit, since we cannot use the
+  /// datetime API otherwise
+  PyDateTime_IMPORT;
+
+  PyObject* obj;
+  for (int64_t i = 0; i < length_; ++i) {
+    obj = objects[i];
+    if (PyDate_CheckExact(obj)) {
+      RETURN_NOT_OK(builder.Append(UnboxDate<ArrowType>::Unbox(obj)));
+    } else if (PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else {
+      std::stringstream ss;
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "datetime.date", &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  return PushBuilderResult(&builder);
+}
+
+#define CONVERT_DECIMAL_CASE(bit_width, builder, object)         \
+  case bit_width: {                                              \
+    decimal::Decimal##bit_width d;                               \
+    std::string string_out;                                      \
+    RETURN_NOT_OK(PythonDecimalToString((object), &string_out)); \
+    RETURN_NOT_OK(FromString(string_out, &d));                   \
+    RETURN_NOT_OK((builder).Append(d));                          \
+    break;                                                       \
+  }
+
+Status PandasConverter::ConvertDecimals() {
+  PyAcquireGIL lock;
+
+  // Import the decimal module and Decimal class
+  OwnedRef decimal;
+  OwnedRef Decimal;
+  RETURN_NOT_OK(ImportModule("decimal", &decimal));
+  RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  PyObject* object = objects[0];
+
+  int precision;
+  int scale;
+
+  RETURN_NOT_OK(InferDecimalPrecisionAndScale(object, &precision, &scale));
+
+  type_ = std::make_shared<DecimalType>(precision, scale);
+
+  const int bit_width = std::dynamic_pointer_cast<DecimalType>(type_)->bit_width();
+  DecimalBuilder builder(pool_, type_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  for (int64_t i = 0; i < length_; ++i) {
+    object = objects[i];
+    if (PyObject_IsInstance(object, Decimal.obj())) {
+      switch (bit_width) {
+        CONVERT_DECIMAL_CASE(32, builder, object)
+        CONVERT_DECIMAL_CASE(64, builder, object)
+        CONVERT_DECIMAL_CASE(128, builder, object)
+        default:
+          break;
+      }
+    } else if (PandasObjectIsNull(object)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else {
+      std::stringstream ss;
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(object, "decimal.Decimal", &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+  return PushBuilderResult(&builder);
+}
+
+Status PandasConverter::ConvertTimes() {
+  // Convert array of datetime.time objects to Arrow
+  PyAcquireGIL lock;
+  PyDateTime_IMPORT;
+
+  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+
+  // datetime.time stores microsecond resolution
+  Time64Builder builder(pool_, ::arrow::time64(TimeUnit::MICRO));
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  PyObject* obj;
+  for (int64_t i = 0; i < length_; ++i) {
+    obj = objects[i];
+    if (PyTime_Check(obj)) {
+      RETURN_NOT_OK(builder.Append(PyTime_to_us(obj)));
+    } else if (PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else {
+      std::stringstream ss;
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "datetime.time", &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+  return PushBuilderResult(&builder);
+}
+
+#undef CONVERT_DECIMAL_CASE
+
+Status PandasConverter::ConvertObjectStrings() {
+  PyAcquireGIL lock;
+
+  // The output type at this point is inconclusive because there may be bytes
+  // and unicode mixed in the object array
+  StringBuilder builder(pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  bool global_have_bytes = false;
+  int64_t offset = 0;
+  while (offset < length_) {
+    bool chunk_have_bytes = false;
+    RETURN_NOT_OK(
+        AppendObjectStrings(arr_, mask_, offset, &builder, &offset, &chunk_have_bytes));
+
+    global_have_bytes = global_have_bytes | chunk_have_bytes;
+    std::shared_ptr<Array> chunk;
+    RETURN_NOT_OK(builder.Finish(&chunk));
+    out_arrays_.emplace_back(std::move(chunk));
+  }
+
+  // If we saw PyBytes, convert everything to BinaryArray
+  if (global_have_bytes) {
+    for (size_t i = 0; i < out_arrays_.size(); ++i) {
+      auto binary_data = out_arrays_[i]->data()->ShallowCopy();
+      binary_data->type = ::arrow::binary();
+      out_arrays_[i] = std::make_shared<BinaryArray>(binary_data);
+    }
+  }
+  return Status::OK();
+}
+
+Status PandasConverter::ConvertObjectFloats() {
+  PyAcquireGIL lock;
+
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask_ != nullptr) {
+    mask_values.Init(mask_);
+    have_mask = true;
+  }
+
+  DoubleBuilder builder(pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  PyObject* obj;
+  for (int64_t i = 0; i < objects.size(); ++i) {
+    obj = objects[i];
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else if (PyFloat_Check(obj)) {
+      double val = PyFloat_AsDouble(obj);
+      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(builder.Append(val));
+    } else {
+      std::stringstream ss;
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "float", &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  return PushBuilderResult(&builder);
+}
+
+Status PandasConverter::ConvertObjectIntegers() {
+  PyAcquireGIL lock;
+
+  Int64Builder builder(pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask_ != nullptr) {
+    mask_values.Init(mask_);
+    have_mask = true;
+  }
+
+  PyObject* obj;
+  for (int64_t i = 0; i < objects.size(); ++i) {
+    obj = objects[i];
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else if (PyObject_is_integer(obj)) {
+      const int64_t val = static_cast<int64_t>(PyLong_AsLong(obj));
+      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(builder.Append(val));
+    } else {
+      std::stringstream ss;
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "integer", &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  return PushBuilderResult(&builder);
+}
+
+Status PandasConverter::ConvertObjectFixedWidthBytes(
+    const std::shared_ptr<DataType>& type) {
+  PyAcquireGIL lock;
+
+  int32_t byte_width = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
+
+  // The output type at this point is inconclusive because there may be bytes
+  // and unicode mixed in the object array
+  FixedSizeBinaryBuilder builder(pool_, type);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  int64_t offset = 0;
+  while (offset < length_) {
+    RETURN_NOT_OK(
+        AppendObjectFixedWidthBytes(arr_, mask_, byte_width, offset, &builder, &offset));
+
+    std::shared_ptr<Array> chunk;
+    RETURN_NOT_OK(builder.Finish(&chunk));
+    out_arrays_.emplace_back(std::move(chunk));
+  }
+  return Status::OK();
+}
+
+Status PandasConverter::ConvertBooleans() {
+  PyAcquireGIL lock;
+
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask_ != nullptr) {
+    mask_values.Init(mask_);
+    have_mask = true;
+  }
+
+  int64_t nbytes = BitUtil::BytesForBits(length_);
+  auto data = std::make_shared<PoolBuffer>(pool_);
+  RETURN_NOT_OK(data->Resize(nbytes));
+  uint8_t* bitmap = data->mutable_data();
+  memset(bitmap, 0, nbytes);
+
+  int64_t null_count = 0;
+  PyObject* obj;
+  for (int64_t i = 0; i < length_; ++i) {
+    obj = objects[i];
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+      ++null_count;
+    } else if (obj == Py_True) {
+      BitUtil::SetBit(bitmap, i);
+      BitUtil::SetBit(null_bitmap_data_, i);
+    } else if (obj == Py_False) {
+      BitUtil::SetBit(null_bitmap_data_, i);
+    } else {
+      std::stringstream ss;
+      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      RETURN_NOT_OK(InvalidConversion(obj, "bool", &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  out_arrays_.push_back(
+      std::make_shared<BooleanArray>(length_, data, null_bitmap_, null_count));
+  return Status::OK();
+}
+
+Status PandasConverter::ConvertObjects() {
+  // Python object arrays are annoying, since we could have one of:
+  //
+  // * Strings
+  // * Booleans with nulls
+  // * decimal.Decimals
+  // * Mixed type (not supported at the moment by arrow format)
+  //
+  // Additionally, nulls may be encoded either as np.nan or None. So we have to
+  // do some type inference and conversion
+
+  RETURN_NOT_OK(InitNullBitmap());
+
+  Ndarray1DIndexer<PyObject*> objects;
+
+  PyAcquireGIL lock;
+  objects.Init(arr_);
+  PyDateTime_IMPORT;
+  lock.release();
+
+  // This means we received an explicit type from the user
+  if (type_) {
+    switch (type_->id()) {
+      case Type::STRING:
+        return ConvertObjectStrings();
+      case Type::FIXED_SIZE_BINARY:
+        return ConvertObjectFixedWidthBytes(type_);
+      case Type::BOOL:
+        return ConvertBooleans();
+      case Type::DATE32:
+        return ConvertDates<Date32Type>();
+      case Type::DATE64:
+        return ConvertDates<Date64Type>();
+      case Type::LIST: {
+        const auto& list_field = static_cast<const ListType&>(*type_);
+        return ConvertLists(list_field.value_field()->type());
+      }
+      case Type::DECIMAL:
+        return ConvertDecimals();
+      default:
+        return Status::TypeError("No known conversion to Arrow type");
+    }
+  } else {
+    // Re-acquire GIL
+    lock.acquire();
+
+    OwnedRef decimal;
+    OwnedRef Decimal;
+    RETURN_NOT_OK(ImportModule("decimal", &decimal));
+    RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+
+    for (int64_t i = 0; i < length_; ++i) {
+      PyObject* obj = objects[i];
+      if (PandasObjectIsNull(obj)) {
+        continue;
+      } else if (PyObject_is_string(obj)) {
+        return ConvertObjectStrings();
+      } else if (PyObject_is_float(obj)) {
+        return ConvertObjectFloats();
+      } else if (PyBool_Check(obj)) {
+        return ConvertBooleans();
+      } else if (PyObject_is_integer(obj)) {
+        return ConvertObjectIntegers();
+      } else if (PyDate_CheckExact(obj)) {
+        // We could choose Date32 or Date64
+        return ConvertDates<Date32Type>();
+      } else if (PyTime_Check(obj)) {
+        return ConvertTimes();
+      } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
+        return ConvertDecimals();
+      } else if (PyList_Check(obj) || PyArray_Check(obj)) {
+        std::shared_ptr<DataType> inferred_type;
+        RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
+        return ConvertLists(inferred_type);
+      } else {
+        const std::string supported_types =
+            "string, bool, float, int, date, time, decimal, list, array";
+        std::stringstream ss;
+        ss << "Error inferring Arrow type for Python object array. ";
+        RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
+        return Status::Invalid(ss.str());
+      }
+    }
+  }
+
+  out_arrays_.push_back(std::make_shared<NullArray>(length_));
+  return Status::OK();
+}
+
+template <typename T>
+Status LoopPySequence(PyObject* sequence, T func) {
+  if (PySequence_Check(sequence)) {
+    OwnedRef ref;
+    Py_ssize_t size = PySequence_Size(sequence);
+    if (PyArray_Check(sequence)) {
+      auto array = reinterpret_cast<PyArrayObject*>(sequence);
+      PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(array));
+      for (int64_t i = 0; i < size; ++i) {
+        RETURN_NOT_OK(func(objects[i]));
+      }
+    } else {
+      for (int64_t i = 0; i < size; ++i) {
+        ref.reset(PySequence_GetItem(sequence, i));
+        RETURN_NOT_OK(func(ref.obj()));
+      }
+    }
+  } else if (PyObject_HasAttrString(sequence, "__iter__")) {
+    OwnedRef iter = OwnedRef(PyObject_GetIter(sequence));
+    PyObject* item;
+    while ((item = PyIter_Next(iter.obj()))) {
+      OwnedRef ref = OwnedRef(item);
+      RETURN_NOT_OK(func(ref.obj()));
+    }
+  } else {
+    return Status::TypeError("Object is not a sequence or iterable");
+  }
+
+  return Status::OK();
+}
+
+template <int ITEM_TYPE, typename ArrowType>
+inline Status PandasConverter::ConvertTypedLists(
+    const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
+  typedef npy_traits<ITEM_TYPE> traits;
+  typedef typename traits::value_type T;
+  typedef typename traits::BuilderClass BuilderT;
+
+  PyAcquireGIL lock;
+
+  // TODO: mask not supported here
+  if (mask_ != nullptr) {
+    return Status::NotImplemented("mask not supported in object conversions yet");
+  }
+
+  if (is_strided()) {
+    return Status::NotImplemented("strided arrays not implemented for lists");
+  }
+
+  BuilderT* value_builder = static_cast<BuilderT*>(builder->value_builder());
+
+  auto foreach_item = [&](PyObject* object) {
+    if (PandasObjectIsNull(object)) {
+      return builder->AppendNull();
+    } else if (PyArray_Check(object)) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
+      RETURN_NOT_OK(builder->Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
+
+      int64_t size = PyArray_DIM(numpy_array, 0);
+      auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
+      if (traits::supports_nulls) {
+        RETURN_NOT_OK(null_bitmap_->Resize(size, false));
+        // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
+        // currently support this.
+        // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
+        ValuesToValidBytes<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
+        return value_builder->Append(data, size, null_bitmap_->data());
+      } else {
+        return value_builder->Append(data, size);
+      }
+    } else if (PyList_Check(object)) {
+      int64_t size;
+      std::shared_ptr<DataType> inferred_type;
+      RETURN_NOT_OK(builder->Append(true));
+      RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
+      if (inferred_type->id() != type->id()) {
+        std::stringstream ss;
+        ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
+        return Status::TypeError(ss.str());
+      }
+      return AppendPySequence(object, size, type, value_builder);
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
+  };
+
+  return LoopPySequence(list, foreach_item);
+}
+
+template <>
+inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
+    const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
+  PyAcquireGIL lock;
+  // TODO: If there are bytes involed, convert to Binary representation
+  bool have_bytes = false;
+
+  // TODO: mask not supported here
+  if (mask_ != nullptr) {
+    return Status::NotImplemented("mask not supported in object conversions yet");
+  }
+
+  if (is_strided()) {
+    return Status::NotImplemented("strided arrays not implemented for lists");
+  }
+
+  auto value_builder = static_cast<StringBuilder*>(builder->value_builder());
+
+  auto foreach_item = [&](PyObject* object) {
+    if (PandasObjectIsNull(object)) {
+      return builder->AppendNull();
+    } else if (PyArray_Check(object)) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
+      RETURN_NOT_OK(builder->Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+
+      int64_t offset = 0;
+      RETURN_NOT_OK(AppendObjectStrings(
+          numpy_array, nullptr, 0, value_builder, &offset, &have_bytes));
+      if (offset < PyArray_SIZE(numpy_array)) {
+        return Status::Invalid("Array cell value exceeded 2GB");
+      }
+      return Status::OK();
+    } else if (PyList_Check(object)) {
+      int64_t size;
+      std::shared_ptr<DataType> inferred_type;
+      RETURN_NOT_OK(builder->Append(true));
+      RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
+      if (inferred_type->id() != Type::STRING) {
+        std::stringstream ss;
+        ss << inferred_type->ToString() << " cannot be converted to STRING.";
+        return Status::TypeError(ss.str());
+      }
+      return AppendPySequence(object, size, inferred_type, value_builder);
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
+  };
+
+  return LoopPySequence(list, foreach_item);
+}
+
+#define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                            \
+  case Type::TYPE: {                                                      \
+    return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type, builder, list); \
+  }
+
+Status PandasConverter::ConvertLists(
+    const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
+  switch (type->id()) {
+    LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
+    LIST_CASE(INT8, NPY_INT8, Int8Type)
+    LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
+    LIST_CASE(INT16, NPY_INT16, Int16Type)
+    LIST_CASE(UINT32, NPY_UINT32, UInt32Type)
+    LIST_CASE(INT32, NPY_INT32, Int32Type)
+    LIST_CASE(UINT64, NPY_UINT64, UInt64Type)
+    LIST_CASE(INT64, NPY_INT64, Int64Type)
+    LIST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
+    LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
+    LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
+    LIST_CASE(STRING, NPY_OBJECT, StringType)
+    case Type::LIST: {
+      const ListType& list_type = static_cast<const ListType&>(*type);
+      auto value_builder = static_cast<ListBuilder*>(builder->value_builder());
+
+      auto foreach_item = [&](PyObject* object) {
+        if (PandasObjectIsNull(object)) {
+          return builder->AppendNull();
+        } else {
+          RETURN_NOT_OK(builder->Append(true));
+          return ConvertLists(list_type.value_type(), value_builder, object);
+        }
+      };
+
+      return LoopPySequence(list, foreach_item);
+    }
+    default: {
+      std::stringstream ss;
+      ss << "Unknown list item type: ";
+      ss << type->ToString();
+      return Status::TypeError(ss.str());
+    }
+  }
+}
+
+Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
+  std::unique_ptr<ArrayBuilder> array_builder;
+  RETURN_NOT_OK(MakeBuilder(pool_, arrow::list(type), &array_builder));
+  ListBuilder* list_builder = static_cast<ListBuilder*>(array_builder.get());
+  RETURN_NOT_OK(ConvertLists(type, list_builder, reinterpret_cast<PyObject*>(arr_)));
+  return PushBuilderResult(list_builder);
+}
+
+Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+  PandasConverter converter(pool, ao, mo, type);
+  RETURN_NOT_OK(converter.Convert());
+  *out = converter.result()[0];
+  return Status::OK();
+}
+
+Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<ChunkedArray>* out) {
+  PandasConverter converter(pool, ao, mo, type);
+  RETURN_NOT_OK(converter.ConvertObjects());
+  *out = std::make_shared<ChunkedArray>(converter.result());
+  return Status::OK();
+}
+
+}  // namespace py
+}  // namespace arrow
diff --git a/cpp/src/arrow/python/pandas_to_arrow.h b/cpp/src/arrow/python/pandas_to_arrow.h
new file mode 100644
index 0000000000000..8f1862470bc94
--- /dev/null
+++ b/cpp/src/arrow/python/pandas_to_arrow.h
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Converting from pandas memory representation to Arrow data structures
+
+#ifndef ARROW_PYTHON_PANDAS_TO_ARROW_H
+#define ARROW_PYTHON_PANDAS_TO_ARROW_H
+
+#include "arrow/python/platform.h"
+
+#include <memory>
+
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+class ChunkedArray;
+class DataType;
+class MemoryPool;
+class Status;
+
+namespace py {
+
+ARROW_EXPORT
+Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
+
+/// Convert dtype=object arrays. If target data type is not known, pass a type
+/// with nullptr
+///
+/// \param[in] pool Memory pool for any memory allocations
+/// \param[in] ao an ndarray with the array data
+/// \param[in] mo an ndarray with a null mask (True is null), optional
+/// \param[in] type
+/// \param[out] out a ChunkedArray, to accommodate chunked output
+ARROW_EXPORT
+Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+    const std::shared_ptr<DataType>& type, std::shared_ptr<ChunkedArray>* out);
+
+}  // namespace py
+}  // namespace arrow
+
+#endif  // ARROW_PYTHON_PANDAS_TO_ARROW_H
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 592a5e6de56c9..c0e555d4904d5 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -26,10 +26,10 @@
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 
+#include "arrow/python/arrow_to_pandas.h"
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
-#include "arrow/python/pandas_convert.h"
 
 #include "arrow/util/decimal.h"
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 3ec1df9a5b883..7ada0e9709f05 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+#include "arrow/array.h"
 #include "arrow/type.h"
 #include "arrow/util/visibility.h"
 
@@ -58,6 +59,8 @@ class ARROW_EXPORT ChunkedArray {
 
   const ArrayVector& chunks() const { return chunks_; }
 
+  std::shared_ptr<DataType> type() const { return chunks_[0]->type(); }
+
   bool Equals(const ChunkedArray& other) const;
   bool Equals(const std::shared_ptr<ChunkedArray>& other) const;
 
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index c7a441597b2e7..efbe36f80b3ea 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -154,10 +154,12 @@ cdef class Array:
 
         Returns
         -------
-        pyarrow.array.Array
+        array : pyarrow.Array or pyarrow.ChunkedArray (if object data
+        overflowed binary storage)
         """
         cdef:
             shared_ptr[CArray] out
+            shared_ptr[CChunkedArray] chunked_out
             shared_ptr[CDataType] c_type
             CMemoryPool* pool
 
@@ -178,7 +180,12 @@ cdef class Array:
                 c_type = type.sp_type
             with nogil:
                 check_status(PandasObjectsToArrow(
-                    pool, values, mask, c_type, &out))
+                    pool, values, mask, c_type, &chunked_out))
+
+            if chunked_out.get().num_chunks() > 1:
+                return pyarrow_wrap_chunked_array(chunked_out)
+            else:
+                out = chunked_out.get().chunk(0)
         else:
             values, type = maybe_coerce_datetime64(
                 values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index e1fe0c0282411..edf50ad54e787 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -315,6 +315,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int64_t null_count()
         int num_chunks()
         shared_ptr[CArray] chunk(int i)
+        shared_ptr[CDataType] type()
 
     cdef cppclass CColumn" arrow::Column":
         CColumn(const shared_ptr[CField]& field,
@@ -711,7 +712,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
 
     CStatus PandasObjectsToArrow(CMemoryPool* pool, object ao, object mo,
                                  const shared_ptr[CDataType]& type,
-                                 shared_ptr[CArray]* out)
+                                 shared_ptr[CChunkedArray]* out)
 
     CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
                             shared_ptr[CTensor]* out);
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 64cf3300ba19f..fea73978e3ecf 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -764,16 +764,18 @@ def write_table(table, where, row_group_size=None, version='1.0',
         Specify the compression codec, either on a general basis or per-column.
     """
     row_group_size = kwargs.get('chunk_size', row_group_size)
-    writer = ParquetWriter(where, table.schema,
-                           use_dictionary=use_dictionary,
-                           compression=compression,
-                           version=version,
-                           use_deprecated_int96_timestamps=use_deprecated_int96_timestamps)
+    options = dict(
+        use_dictionary=use_dictionary,
+        compression=compression,
+        version=version,
+        use_deprecated_int96_timestamps=use_deprecated_int96_timestamps)
+    writer = ParquetWriter(where, table.schema, **options)
     writer.write_table(table, row_group_size=row_group_size)
     writer.close()
 
 
-def write_metadata(schema, where, version='1.0', use_deprecated_int96_timestamps=False):
+def write_metadata(schema, where, version='1.0',
+                   use_deprecated_int96_timestamps=False):
     """
     Write metadata-only Parquet file from schema
 
@@ -784,6 +786,9 @@ def write_metadata(schema, where, version='1.0', use_deprecated_int96_timestamps
     version : {"1.0", "2.0"}, default "1.0"
         The Parquet format version, defaults to 1.0
     """
-    writer = ParquetWriter(where, schema, version=version,
-                           use_deprecated_int96_timestamps=use_deprecated_int96_timestamps)
+    options = dict(
+        version=version,
+        use_deprecated_int96_timestamps=use_deprecated_int96_timestamps
+    )
+    writer = ParquetWriter(where, schema, **options)
     writer.close()
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 8c2083d379061..28e07ffc37dc3 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -148,6 +148,21 @@ cdef public api object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array):
     return arr
 
 
+cdef public api object pyarrow_wrap_chunked_array(
+    const shared_ptr[CChunkedArray]& sp_array):
+    if sp_array.get() == NULL:
+        raise ValueError('ChunkedArray was NULL')
+
+    cdef CDataType* data_type = sp_array.get().type().get()
+
+    if data_type == NULL:
+        raise ValueError('ChunkedArray data type was NULL')
+
+    cdef ChunkedArray arr = ChunkedArray()
+    arr.init(sp_array)
+    return arr
+
+
 cdef public api bint pyarrow_is_tensor(object tensor):
     return isinstance(tensor, Tensor)
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index d7a6060c4383f..6188e90616ba2 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -44,6 +44,11 @@ cdef class ChunkedArray:
         self.sp_chunked_array = chunked_array
         self.chunked_array = chunked_array.get()
 
+    property type:
+
+        def __get__(self):
+            return pyarrow_wrap_data_type(self.sp_chunked_array.get().type())
+
     cdef int _check_nullptr(self) except -1:
         if self.chunked_array == NULL:
             raise ReferenceError(
@@ -277,7 +282,6 @@ cdef shared_ptr[const CKeyValueMetadata] unbox_metadata(dict metadata):
 cdef int _schema_from_arrays(
         arrays, names, dict metadata, shared_ptr[CSchema]* schema) except -1:
     cdef:
-        Array arr
         Column col
         c_string c_name
         vector[shared_ptr[CField]] fields
@@ -289,23 +293,25 @@ cdef int _schema_from_arrays(
     if len(arrays) == 0:
         raise ValueError('Must pass at least one array')
 
-    if isinstance(arrays[0], Array):
-        if names is None:
-            raise ValueError('Must pass names when constructing '
-                             'from Array objects')
-        for i in range(K):
-            arr = arrays[i]
-            type_ = arr.type.sp_type
-            c_name = tobytes(names[i])
-            fields[i].reset(new CField(c_name, type_, True))
-    elif isinstance(arrays[0], Column):
+    if isinstance(arrays[0], Column):
         for i in range(K):
             col = arrays[i]
             type_ = col.sp_column.get().type()
             c_name = tobytes(col.name)
             fields[i].reset(new CField(c_name, type_, True))
     else:
-        raise TypeError(type(arrays[0]))
+        if names is None:
+            raise ValueError('Must pass names when constructing '
+                             'from Array objects')
+        for i in range(K):
+            val = arrays[i]
+            if isinstance(val, (Array, ChunkedArray)):
+                type_ = (<DataType> val.type).sp_type
+            else:
+                raise TypeError(type(val))
+
+            c_name = tobytes(names[i])
+            fields[i].reset(new CField(c_name, type_, True))
 
     schema.reset(new CSchema(fields, unbox_metadata(metadata)))
     return 0
@@ -323,7 +329,6 @@ cdef tuple _dataframe_to_arrays(
         list index_columns = []
         list types = []
         DataType type = None
-        Array array
         dict metadata
         Py_ssize_t i
         Py_ssize_t n
@@ -782,6 +787,13 @@ cdef class Table:
                         (<Array> arrays[i]).sp_array
                     )
                 )
+            elif isinstance(arrays[i], ChunkedArray):
+                columns.push_back(
+                    make_shared[CColumn](
+                        schema.get().field(i),
+                        (<ChunkedArray> arrays[i]).sp_chunked_array
+                    )
+                )
             elif isinstance(arrays[i], Column):
                 columns.push_back((<Column> arrays[i]).sp_column)
             else:
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index da94da9fc7499..2aeeab7294ccc 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -18,11 +18,12 @@
 from pytest import skip
 
 
-groups = ['hdfs', 'parquet']
+groups = ['hdfs', 'parquet', 'large_memory']
 
 defaults = {
     'hdfs': False,
-    'parquet': False
+    'parquet': False,
+    'large_memory': False
 }
 
 try:
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 5b84817a7f6b2..43e0bad5e3d2d 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -284,6 +284,20 @@ def test_bytes_to_binary(self):
         expected = pd.DataFrame({'strings': values2})
         self._check_pandas_roundtrip(df, expected)
 
+    @pytest.mark.large_memory
+    def test_bytes_exceed_2gb(self):
+        val = 'x' * (1 << 20)
+        df = pd.DataFrame({
+            'strings': np.array([val] * 4000, dtype=object)
+        })
+        arr = pa.Array.from_pandas(df['strings'])
+        assert isinstance(arr, pa.ChunkedArray)
+        assert arr.num_chunks == 2
+        arr = None
+
+        table = pa.Table.from_pandas(df)
+        assert table[0].data.num_chunks == 2
+
     def test_fixed_size_bytes(self):
         values = [b'foo', None, b'bar', None, None, b'hey']
         df = pd.DataFrame({'strings': values})

From 5aa0809961210b2cd0c302a7d8cd74c4363f5eb5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 19 Jul 2017 17:10:49 -0400
Subject: [PATCH 0834/1644] [GLib] Update rat_exclusion_files.txt

Change-Id: Id21cfaaefcf9f664e13741bd61a20e8aac84a050
---
 dev/release/rat_exclude_files.txt | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 5a0b8a3838fd6..1a3461383da1b 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -43,21 +43,21 @@ MANIFEST.in
 __init__.pxd
 __init__.py
 requirements.txt
-version
-*.m4
-configure
-config.sub
-config.h.in
-compile
-missing
-install-sh
-config.guess
-depcomp
-ltmain.sh
-arrow-glib.types
-arrow-glib-sections.txt
-arrow-glib-overrides.txt
-gtk-doc.make
+c_glib/configure
+c_glib/version
+c_glib/m4/*.m4
+c_glib/config/config.sub
+c_glib/config.h.in
+c_glib/config/compile
+c_glib/config/missing
+c_glib/config/install-sh
+c_glib/config/config.guess
+c_glib/config/depcomp
+c_glib/config/ltmain.sh
+c_glib/doc/reference/arrow-glib.types
+c_glib/doc/reference/arrow-glib-sections.txt
+c_glib/doc/reference/arrow-glib-overrides.txt
+c_glib/doc/reference/gtk-doc.make
 *.html
 *.sgml
 *.css

From db181d19bfc17bc4c253fc6e171cbe444cd637a7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 20 Jul 2017 12:39:18 -0400
Subject: [PATCH 0835/1644] ARROW-1244: Exclude C++ Plasma source tree when
 creating source release

We must undergo an IP clearance process with Apache Incubator before this code can be included in an Apache Arrow release.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #874 from wesm/ARROW-1244 and squashes the following commits:

4a1b28b7 [Wes McKinney] Remove Plasma source tree when creating source release until after IP clearance
---
 dev/release/02-source.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index 508141e7198b2..6e8e7e4f4e3ca 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -88,6 +88,12 @@ rm -rf ${tag}
 git archive $release_hash --prefix ${tag}/ | tar xf -
 rm -rf ${tag}/c_glib
 mv tmp-c_glib ${tag}/c_glib
+
+# ARROW-1244 Remove cpp/src/plasma from source release until after IP clearance
+# is resolved
+rm -rf ${tag}/cpp/src/plasma
+
+# Create new tarball from modified source directory
 tar czf ${tarball} ${tag}
 rm -rf ${tag}
 

From 62ef2cd8a39fc93e7fa4bb790d7cd92adb77571f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 20 Jul 2017 12:58:35 -0400
Subject: [PATCH 0836/1644] [C++] Remove Plasma source tree for 0.5.0 release
 pending IP Clearance

Change-Id: I1244ada7297d2f6f567d679452d21a26b91c0360
---
 cpp/src/plasma/CMakeLists.txt              |  113 -
 cpp/src/plasma/client.cc                   |  557 --
 cpp/src/plasma/client.h                    |  343 --
 cpp/src/plasma/common.cc                   |   83 -
 cpp/src/plasma/common.h                    |   63 -
 cpp/src/plasma/events.cc                   |   81 -
 cpp/src/plasma/events.h                    |   99 -
 cpp/src/plasma/eviction_policy.cc          |  107 -
 cpp/src/plasma/eviction_policy.h           |  134 -
 cpp/src/plasma/extension.cc                |  456 --
 cpp/src/plasma/extension.h                 |   50 -
 cpp/src/plasma/fling.cc                    |   90 -
 cpp/src/plasma/fling.h                     |   52 -
 cpp/src/plasma/format/.gitignore           |    1 -
 cpp/src/plasma/format/common.fbs           |   34 -
 cpp/src/plasma/format/plasma.fbs           |  291 -
 cpp/src/plasma/io.cc                       |  212 -
 cpp/src/plasma/io.h                        |   55 -
 cpp/src/plasma/malloc.cc                   |  178 -
 cpp/src/plasma/malloc.h                    |   26 -
 cpp/src/plasma/plasma.cc                   |   64 -
 cpp/src/plasma/plasma.h                    |  191 -
 cpp/src/plasma/protocol.cc                 |  502 --
 cpp/src/plasma/protocol.h                  |  170 -
 cpp/src/plasma/store.cc                    |  683 ---
 cpp/src/plasma/store.h                     |  169 -
 cpp/src/plasma/test/client_tests.cc        |  132 -
 cpp/src/plasma/test/run_tests.sh           |   61 -
 cpp/src/plasma/test/run_valgrind.sh        |   27 -
 cpp/src/plasma/test/serialization_tests.cc |  388 --
 cpp/src/plasma/thirdparty/ae/ae.c          |  465 --
 cpp/src/plasma/thirdparty/ae/ae.h          |  123 -
 cpp/src/plasma/thirdparty/ae/ae_epoll.c    |  135 -
 cpp/src/plasma/thirdparty/ae/ae_evport.c   |  320 -
 cpp/src/plasma/thirdparty/ae/ae_kqueue.c   |  138 -
 cpp/src/plasma/thirdparty/ae/ae_select.c   |  106 -
 cpp/src/plasma/thirdparty/ae/config.h      |   54 -
 cpp/src/plasma/thirdparty/ae/zmalloc.h     |   45 -
 cpp/src/plasma/thirdparty/dlmalloc.c       | 6281 --------------------
 cpp/src/plasma/thirdparty/xxhash.cc        |  889 ---
 cpp/src/plasma/thirdparty/xxhash.h         |  293 -
 41 files changed, 14261 deletions(-)
 delete mode 100644 cpp/src/plasma/CMakeLists.txt
 delete mode 100644 cpp/src/plasma/client.cc
 delete mode 100644 cpp/src/plasma/client.h
 delete mode 100644 cpp/src/plasma/common.cc
 delete mode 100644 cpp/src/plasma/common.h
 delete mode 100644 cpp/src/plasma/events.cc
 delete mode 100644 cpp/src/plasma/events.h
 delete mode 100644 cpp/src/plasma/eviction_policy.cc
 delete mode 100644 cpp/src/plasma/eviction_policy.h
 delete mode 100644 cpp/src/plasma/extension.cc
 delete mode 100644 cpp/src/plasma/extension.h
 delete mode 100644 cpp/src/plasma/fling.cc
 delete mode 100644 cpp/src/plasma/fling.h
 delete mode 100644 cpp/src/plasma/format/.gitignore
 delete mode 100644 cpp/src/plasma/format/common.fbs
 delete mode 100644 cpp/src/plasma/format/plasma.fbs
 delete mode 100644 cpp/src/plasma/io.cc
 delete mode 100644 cpp/src/plasma/io.h
 delete mode 100644 cpp/src/plasma/malloc.cc
 delete mode 100644 cpp/src/plasma/malloc.h
 delete mode 100644 cpp/src/plasma/plasma.cc
 delete mode 100644 cpp/src/plasma/plasma.h
 delete mode 100644 cpp/src/plasma/protocol.cc
 delete mode 100644 cpp/src/plasma/protocol.h
 delete mode 100644 cpp/src/plasma/store.cc
 delete mode 100644 cpp/src/plasma/store.h
 delete mode 100644 cpp/src/plasma/test/client_tests.cc
 delete mode 100644 cpp/src/plasma/test/run_tests.sh
 delete mode 100644 cpp/src/plasma/test/run_valgrind.sh
 delete mode 100644 cpp/src/plasma/test/serialization_tests.cc
 delete mode 100644 cpp/src/plasma/thirdparty/ae/ae.c
 delete mode 100644 cpp/src/plasma/thirdparty/ae/ae.h
 delete mode 100644 cpp/src/plasma/thirdparty/ae/ae_epoll.c
 delete mode 100644 cpp/src/plasma/thirdparty/ae/ae_evport.c
 delete mode 100644 cpp/src/plasma/thirdparty/ae/ae_kqueue.c
 delete mode 100644 cpp/src/plasma/thirdparty/ae/ae_select.c
 delete mode 100644 cpp/src/plasma/thirdparty/ae/config.h
 delete mode 100644 cpp/src/plasma/thirdparty/ae/zmalloc.h
 delete mode 100644 cpp/src/plasma/thirdparty/dlmalloc.c
 delete mode 100644 cpp/src/plasma/thirdparty/xxhash.cc
 delete mode 100644 cpp/src/plasma/thirdparty/xxhash.h

diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
deleted file mode 100644
index 4ff3beba779c2..0000000000000
--- a/cpp/src/plasma/CMakeLists.txt
+++ /dev/null
@@ -1,113 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-cmake_minimum_required(VERSION 2.8)
-
-project(plasma)
-
-find_package(PythonLibsNew REQUIRED)
-find_package(Threads)
-
-option(PLASMA_PYTHON
-  "Build the Plasma Python extensions"
-  OFF)
-
-if(APPLE)
-  SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
-endif(APPLE)
-
-include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
-include_directories("${FLATBUFFERS_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/" "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" "${CMAKE_CURRENT_LIST_DIR}/../")
-
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L")
-
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-conversion")
-
-# Compile flatbuffers
-
-set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
-set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
-
-set(PLASMA_FBS_OUTPUT_FILES
-  "${OUTPUT_DIR}/common_generated.h"
-  "${OUTPUT_DIR}/plasma_generated.h")
-
-add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
-
-if(FLATBUFFERS_VENDORED)
-  add_dependencies(gen_plasma_fbs flatbuffers_ep)
-endif()
-
-add_custom_command(
-  OUTPUT ${PLASMA_FBS_OUTPUT_FILES}
-  # The --gen-object-api flag generates a C++ class MessageT for each
-  # flatbuffers message Message, which can be used to store deserialized
-  # messages in data structures. This is currently used for ObjectInfo for
-  # example.
-  COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} --gen-object-api
-  DEPENDS ${PLASMA_FBS_SRC}
-  COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}"
-  VERBATIM)
-
-if(UNIX AND NOT APPLE)
-  link_libraries(rt)
-endif()
-
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
-
-set_source_files_properties(extension.cc PROPERTIES COMPILE_FLAGS -Wno-strict-aliasing)
-
-set(PLASMA_SRCS
-  client.cc
-  common.cc
-  eviction_policy.cc
-  events.cc
-  fling.cc
-  io.cc
-  malloc.cc
-  plasma.cc
-  protocol.cc
-  thirdparty/ae/ae.c
-  thirdparty/xxhash.cc)
-
-ADD_ARROW_LIB(plasma
-  SOURCES ${PLASMA_SRCS}
-  DEPENDENCIES gen_plasma_fbs
-  SHARED_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static
-  STATIC_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static)
-
-# The optimization flag -O3 is suggested by dlmalloc.c, which is #included in
-# malloc.cc; we set it here regardless of whether we do a debug or release build.
-set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error -O3")
-
-add_executable(plasma_store store.cc)
-target_link_libraries(plasma_store plasma_static)
-
-ADD_ARROW_TEST(test/serialization_tests)
-ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static)
-ADD_ARROW_TEST(test/client_tests)
-ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static)
-
-if(PLASMA_PYTHON)
-  add_library(plasma_extension SHARED extension.cc)
-
-  if(APPLE)
-    target_link_libraries(plasma_extension plasma_static "-undefined dynamic_lookup")
-  else(APPLE)
-    target_link_libraries(plasma_extension plasma_static -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive)
-  endif(APPLE)
-endif()
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
deleted file mode 100644
index dcb78e7ec52c6..0000000000000
--- a/cpp/src/plasma/client.cc
+++ /dev/null
@@ -1,557 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// PLASMA CLIENT: Client library for using the plasma store and manager
-
-#include "plasma/client.h"
-
-#ifdef _WIN32
-#include <Win32_Interop/win32_types.h>
-#endif
-
-#include <assert.h>
-#include <fcntl.h>
-#include <netinet/in.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include <algorithm>
-#include <thread>
-#include <vector>
-
-#include "plasma/common.h"
-#include "plasma/fling.h"
-#include "plasma/io.h"
-#include "plasma/plasma.h"
-#include "plasma/protocol.h"
-
-#define XXH_STATIC_LINKING_ONLY
-#include "thirdparty/xxhash.h"
-
-#define XXH64_DEFAULT_SEED 0
-
-// Number of threads used for memcopy and hash computations.
-constexpr int64_t kThreadPoolSize = 8;
-constexpr int64_t kBytesInMB = 1 << 20;
-static std::vector<std::thread> threadpool_(kThreadPoolSize);
-
-// If the file descriptor fd has been mmapped in this client process before,
-// return the pointer that was returned by mmap, otherwise mmap it and store the
-// pointer in a hash table.
-uint8_t* PlasmaClient::lookup_or_mmap(int fd, int store_fd_val, int64_t map_size) {
-  auto entry = mmap_table_.find(store_fd_val);
-  if (entry != mmap_table_.end()) {
-    close(fd);
-    return entry->second.pointer;
-  } else {
-    uint8_t* result = reinterpret_cast<uint8_t*>(
-        mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
-    // TODO(pcm): Don't fail here, instead return a Status.
-    if (result == MAP_FAILED) { ARROW_LOG(FATAL) << "mmap failed"; }
-    close(fd);
-    ClientMmapTableEntry& entry = mmap_table_[store_fd_val];
-    entry.pointer = result;
-    entry.length = map_size;
-    entry.count = 0;
-    return result;
-  }
-}
-
-// Get a pointer to a file that we know has been memory mapped in this client
-// process before.
-uint8_t* PlasmaClient::lookup_mmapped_file(int store_fd_val) {
-  auto entry = mmap_table_.find(store_fd_val);
-  ARROW_CHECK(entry != mmap_table_.end());
-  return entry->second.pointer;
-}
-
-void PlasmaClient::increment_object_count(
-    const ObjectID& object_id, PlasmaObject* object, bool is_sealed) {
-  // Increment the count of the object to track the fact that it is being used.
-  // The corresponding decrement should happen in PlasmaClient::Release.
-  auto elem = objects_in_use_.find(object_id);
-  ObjectInUseEntry* object_entry;
-  if (elem == objects_in_use_.end()) {
-    // Add this object ID to the hash table of object IDs in use. The
-    // corresponding call to free happens in PlasmaClient::Release.
-    objects_in_use_[object_id] =
-        std::unique_ptr<ObjectInUseEntry>(new ObjectInUseEntry());
-    objects_in_use_[object_id]->object = *object;
-    objects_in_use_[object_id]->count = 0;
-    objects_in_use_[object_id]->is_sealed = is_sealed;
-    object_entry = objects_in_use_[object_id].get();
-    // Increment the count of the number of objects in the memory-mapped file
-    // that are being used. The corresponding decrement should happen in
-    // PlasmaClient::Release.
-    auto entry = mmap_table_.find(object->handle.store_fd);
-    ARROW_CHECK(entry != mmap_table_.end());
-    ARROW_CHECK(entry->second.count >= 0);
-    // Update the in_use_object_bytes_.
-    in_use_object_bytes_ +=
-        (object_entry->object.data_size + object_entry->object.metadata_size);
-    entry->second.count += 1;
-  } else {
-    object_entry = elem->second.get();
-    ARROW_CHECK(object_entry->count > 0);
-  }
-  // Increment the count of the number of instances of this object that are
-  // being used by this client. The corresponding decrement should happen in
-  // PlasmaClient::Release.
-  object_entry->count += 1;
-}
-
-Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
-    uint8_t* metadata, int64_t metadata_size, uint8_t** data) {
-  ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
-                   << data_size << " and metadata size " << metadata_size;
-  RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaCreateReply, &buffer));
-  ObjectID id;
-  PlasmaObject object;
-  RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object));
-  // If the CreateReply included an error, then the store will not send a file
-  // descriptor.
-  int fd = recv_fd(store_conn_);
-  ARROW_CHECK(fd >= 0) << "recv not successful";
-  ARROW_CHECK(object.data_size == data_size);
-  ARROW_CHECK(object.metadata_size == metadata_size);
-  // The metadata should come right after the data.
-  ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
-  *data = lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) +
-          object.data_offset;
-  // If plasma_create is being called from a transfer, then we will not copy the
-  // metadata here. The metadata will be written along with the data streamed
-  // from the transfer.
-  if (metadata != NULL) {
-    // Copy the metadata to the buffer.
-    memcpy(*data + object.data_size, metadata, metadata_size);
-  }
-  // Increment the count of the number of instances of this object that this
-  // client is using. A call to PlasmaClient::Release is required to decrement
-  // this
-  // count. Cache the reference to the object.
-  increment_object_count(object_id, &object, false);
-  // We increment the count a second time (and the corresponding decrement will
-  // happen in a PlasmaClient::Release call in plasma_seal) so even if the
-  // buffer
-  // returned by PlasmaClient::Dreate goes out of scope, the object does not get
-  // released before the call to PlasmaClient::Seal happens.
-  increment_object_count(object_id, &object, false);
-  return Status::OK();
-}
-
-Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
-    int64_t timeout_ms, ObjectBuffer* object_buffers) {
-  // Fill out the info for the objects that are already in use locally.
-  bool all_present = true;
-  for (int i = 0; i < num_objects; ++i) {
-    auto object_entry = objects_in_use_.find(object_ids[i]);
-    if (object_entry == objects_in_use_.end()) {
-      // This object is not currently in use by this client, so we need to send
-      // a request to the store.
-      all_present = false;
-      // Make a note to ourselves that the object is not present.
-      object_buffers[i].data_size = -1;
-    } else {
-      // NOTE: If the object is still unsealed, we will deadlock, since we must
-      // have been the one who created it.
-      ARROW_CHECK(object_entry->second->is_sealed)
-          << "Plasma client called get on an unsealed object that it created";
-      PlasmaObject* object = &object_entry->second->object;
-      object_buffers[i].data = lookup_mmapped_file(object->handle.store_fd);
-      object_buffers[i].data = object_buffers[i].data + object->data_offset;
-      object_buffers[i].data_size = object->data_size;
-      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
-      object_buffers[i].metadata_size = object->metadata_size;
-      // Increment the count of the number of instances of this object that this
-      // client is using. A call to PlasmaClient::Release is required to
-      // decrement this
-      // count. Cache the reference to the object.
-      increment_object_count(object_ids[i], object, true);
-    }
-  }
-
-  if (all_present) { return Status::OK(); }
-
-  // If we get here, then the objects aren't all currently in use by this
-  // client, so we need to send a request to the plasma store.
-  RETURN_NOT_OK(SendGetRequest(store_conn_, object_ids, num_objects, timeout_ms));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaGetReply, &buffer));
-  std::vector<ObjectID> received_object_ids(num_objects);
-  std::vector<PlasmaObject> object_data(num_objects);
-  PlasmaObject* object;
-  RETURN_NOT_OK(ReadGetReply(
-      buffer.data(), received_object_ids.data(), object_data.data(), num_objects));
-
-  for (int i = 0; i < num_objects; ++i) {
-    DCHECK(received_object_ids[i] == object_ids[i]);
-    object = &object_data[i];
-    if (object_buffers[i].data_size != -1) {
-      // If the object was already in use by the client, then the store should
-      // have returned it.
-      DCHECK_NE(object->data_size, -1);
-      // We won't use this file descriptor, but the store sent us one, so we
-      // need to receive it and then close it right away so we don't leak file
-      // descriptors.
-      int fd = recv_fd(store_conn_);
-      close(fd);
-      ARROW_CHECK(fd >= 0);
-      // We've already filled out the information for this object, so we can
-      // just continue.
-      continue;
-    }
-    // If we are here, the object was not currently in use, so we need to
-    // process the reply from the object store.
-    if (object->data_size != -1) {
-      // The object was retrieved. The user will be responsible for releasing
-      // this object.
-      int fd = recv_fd(store_conn_);
-      ARROW_CHECK(fd >= 0);
-      object_buffers[i].data =
-          lookup_or_mmap(fd, object->handle.store_fd, object->handle.mmap_size);
-      // Finish filling out the return values.
-      object_buffers[i].data = object_buffers[i].data + object->data_offset;
-      object_buffers[i].data_size = object->data_size;
-      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
-      object_buffers[i].metadata_size = object->metadata_size;
-      // Increment the count of the number of instances of this object that this
-      // client is using. A call to PlasmaClient::Release is required to
-      // decrement this
-      // count. Cache the reference to the object.
-      increment_object_count(received_object_ids[i], object, true);
-    } else {
-      // The object was not retrieved. Make sure we already put a -1 here to
-      // indicate that the object was not retrieved. The caller is not
-      // responsible for releasing this object.
-      DCHECK_EQ(object_buffers[i].data_size, -1);
-      object_buffers[i].data_size = -1;
-    }
-  }
-  return Status::OK();
-}
-
-/// This is a helper method for implementing plasma_release. We maintain a
-/// buffer
-/// of release calls and only perform them once the buffer becomes full (as
-/// judged by the aggregate sizes of the objects). There may be multiple release
-/// calls for the same object ID in the buffer. In this case, the first release
-/// calls will not do anything. The client will only send a message to the store
-/// releasing the object when the client is truly done with the object.
-///
-/// @param conn The plasma connection.
-/// @param object_id The object ID to attempt to release.
-Status PlasmaClient::PerformRelease(const ObjectID& object_id) {
-  // Decrement the count of the number of instances of this object that are
-  // being used by this client. The corresponding increment should have happened
-  // in PlasmaClient::Get.
-  auto object_entry = objects_in_use_.find(object_id);
-  ARROW_CHECK(object_entry != objects_in_use_.end());
-  object_entry->second->count -= 1;
-  ARROW_CHECK(object_entry->second->count >= 0);
-  // Check if the client is no longer using this object.
-  if (object_entry->second->count == 0) {
-    // Decrement the count of the number of objects in this memory-mapped file
-    // that the client is using. The corresponding increment should have
-    // happened in plasma_get.
-    int fd = object_entry->second->object.handle.store_fd;
-    auto entry = mmap_table_.find(fd);
-    ARROW_CHECK(entry != mmap_table_.end());
-    entry->second.count -= 1;
-    ARROW_CHECK(entry->second.count >= 0);
-    // If none are being used then unmap the file.
-    if (entry->second.count == 0) {
-      munmap(entry->second.pointer, entry->second.length);
-      // Remove the corresponding entry from the hash table.
-      mmap_table_.erase(fd);
-    }
-    // Tell the store that the client no longer needs the object.
-    RETURN_NOT_OK(SendReleaseRequest(store_conn_, object_id));
-    // Update the in_use_object_bytes_.
-    in_use_object_bytes_ -= (object_entry->second->object.data_size +
-                             object_entry->second->object.metadata_size);
-    DCHECK_GE(in_use_object_bytes_, 0);
-    // Remove the entry from the hash table of objects currently in use.
-    objects_in_use_.erase(object_id);
-  }
-  return Status::OK();
-}
-
-Status PlasmaClient::Release(const ObjectID& object_id) {
-  // Add the new object to the release history.
-  release_history_.push_front(object_id);
-  // If there are too many bytes in use by the client or if there are too many
-  // pending release calls, and there are at least some pending release calls in
-  // the release_history list, then release some objects.
-  while ((in_use_object_bytes_ > std::min(kL3CacheSizeBytes, store_capacity_ / 100) ||
-             release_history_.size() > config_.release_delay) &&
-         release_history_.size() > 0) {
-    // Perform a release for the object ID for the first pending release.
-    RETURN_NOT_OK(PerformRelease(release_history_.back()));
-    // Remove the last entry from the release history.
-    release_history_.pop_back();
-  }
-  return Status::OK();
-}
-
-// This method is used to query whether the plasma store contains an object.
-Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) {
-  // Check if we already have a reference to the object.
-  if (objects_in_use_.count(object_id) > 0) {
-    *has_object = 1;
-  } else {
-    // If we don't already have a reference to the object, check with the store
-    // to see if we have the object.
-    RETURN_NOT_OK(SendContainsRequest(store_conn_, object_id));
-    std::vector<uint8_t> buffer;
-    RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaContainsReply, &buffer));
-    ObjectID object_id2;
-    RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object));
-  }
-  return Status::OK();
-}
-
-static void ComputeBlockHash(const unsigned char* data, int64_t nbytes, uint64_t* hash) {
-  XXH64_state_t hash_state;
-  XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
-  XXH64_update(&hash_state, data, nbytes);
-  *hash = XXH64_digest(&hash_state);
-}
-
-static inline bool compute_object_hash_parallel(
-    XXH64_state_t* hash_state, const unsigned char* data, int64_t nbytes) {
-  // Note that this function will likely be faster if the address of data is
-  // aligned on a 64-byte boundary.
-  const int num_threads = kThreadPoolSize;
-  uint64_t threadhash[num_threads + 1];
-  const uint64_t data_address = reinterpret_cast<uint64_t>(data);
-  const uint64_t num_blocks = nbytes / BLOCK_SIZE;
-  const uint64_t chunk_size = (num_blocks / num_threads) * BLOCK_SIZE;
-  const uint64_t right_address = data_address + chunk_size * num_threads;
-  const uint64_t suffix = (data_address + nbytes) - right_address;
-  // Now the data layout is | k * num_threads * block_size | suffix | ==
-  // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size.
-  // Each thread gets a "chunk" of k blocks, except the suffix thread.
-
-  for (int i = 0; i < num_threads; i++) {
-    threadpool_[i] = std::thread(ComputeBlockHash,
-        reinterpret_cast<uint8_t*>(data_address) + i * chunk_size, chunk_size,
-        &threadhash[i]);
-  }
-  ComputeBlockHash(
-      reinterpret_cast<uint8_t*>(right_address), suffix, &threadhash[num_threads]);
-
-  // Join the threads.
-  for (auto& t : threadpool_) {
-    if (t.joinable()) { t.join(); }
-  }
-
-  XXH64_update(hash_state, (unsigned char*)threadhash, sizeof(threadhash));
-  return true;
-}
-
-static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
-  XXH64_state_t hash_state;
-  XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
-  if (obj_buffer.data_size >= kBytesInMB) {
-    compute_object_hash_parallel(
-        &hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
-  } else {
-    XXH64_update(&hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
-  }
-  XXH64_update(
-      &hash_state, (unsigned char*)obj_buffer.metadata, obj_buffer.metadata_size);
-  return XXH64_digest(&hash_state);
-}
-
-bool plasma_compute_object_hash(
-    PlasmaClient* conn, ObjectID object_id, unsigned char* digest) {
-  // Get the plasma object data. We pass in a timeout of 0 to indicate that
-  // the operation should timeout immediately.
-  ObjectBuffer object_buffer;
-  ARROW_CHECK_OK(conn->Get(&object_id, 1, 0, &object_buffer));
-  // If the object was not retrieved, return false.
-  if (object_buffer.data_size == -1) { return false; }
-  // Compute the hash.
-  uint64_t hash = compute_object_hash(object_buffer);
-  memcpy(digest, &hash, sizeof(hash));
-  // Release the plasma object.
-  ARROW_CHECK_OK(conn->Release(object_id));
-  return true;
-}
-
-Status PlasmaClient::Seal(const ObjectID& object_id) {
-  // Make sure this client has a reference to the object before sending the
-  // request to Plasma.
-  auto object_entry = objects_in_use_.find(object_id);
-  ARROW_CHECK(object_entry != objects_in_use_.end())
-      << "Plasma client called seal an object without a reference to it";
-  ARROW_CHECK(!object_entry->second->is_sealed)
-      << "Plasma client called seal an already sealed object";
-  object_entry->second->is_sealed = true;
-  /// Send the seal request to Plasma.
-  static unsigned char digest[kDigestSize];
-  ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0]));
-  RETURN_NOT_OK(SendSealRequest(store_conn_, object_id, &digest[0]));
-  // We call PlasmaClient::Release to decrement the number of instances of this
-  // object
-  // that are currently being used by this client. The corresponding increment
-  // happened in plasma_create and was used to ensure that the object was not
-  // released before the call to PlasmaClient::Seal.
-  return Release(object_id);
-}
-
-Status PlasmaClient::Delete(const ObjectID& object_id) {
-  // TODO(rkn): In the future, we can use this method to give hints to the
-  // eviction policy about when an object will no longer be needed.
-  return Status::NotImplemented("PlasmaClient::Delete is not implemented.");
-}
-
-Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) {
-  // Send a request to the store to evict objects.
-  RETURN_NOT_OK(SendEvictRequest(store_conn_, num_bytes));
-  // Wait for a response with the number of bytes actually evicted.
-  std::vector<uint8_t> buffer;
-  int64_t type;
-  RETURN_NOT_OK(ReadMessage(store_conn_, &type, &buffer));
-  return ReadEvictReply(buffer.data(), num_bytes_evicted);
-}
-
-Status PlasmaClient::Subscribe(int* fd) {
-  int sock[2];
-  // Create a non-blocking socket pair. This will only be used to send
-  // notifications from the Plasma store to the client.
-  socketpair(AF_UNIX, SOCK_STREAM, 0, sock);
-  // Make the socket non-blocking.
-  int flags = fcntl(sock[1], F_GETFL, 0);
-  ARROW_CHECK(fcntl(sock[1], F_SETFL, flags | O_NONBLOCK) == 0);
-  // Tell the Plasma store about the subscription.
-  RETURN_NOT_OK(SendSubscribeRequest(store_conn_));
-  // Send the file descriptor that the Plasma store should use to push
-  // notifications about sealed objects to this client.
-  ARROW_CHECK(send_fd(store_conn_, sock[1]) >= 0);
-  close(sock[1]);
-  // Return the file descriptor that the client should use to read notifications
-  // about sealed objects.
-  *fd = sock[0];
-  return Status::OK();
-}
-
-Status PlasmaClient::Connect(const std::string& store_socket_name,
-    const std::string& manager_socket_name, int release_delay) {
-  store_conn_ = connect_ipc_sock_retry(store_socket_name, -1, -1);
-  if (manager_socket_name != "") {
-    manager_conn_ = connect_ipc_sock_retry(manager_socket_name, -1, -1);
-  } else {
-    manager_conn_ = -1;
-  }
-  config_.release_delay = release_delay;
-  in_use_object_bytes_ = 0;
-  // Send a ConnectRequest to the store to get its memory capacity.
-  RETURN_NOT_OK(SendConnectRequest(store_conn_));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaConnectReply, &buffer));
-  RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity_));
-  return Status::OK();
-}
-
-Status PlasmaClient::Disconnect() {
-  // NOTE: We purposefully do not finish sending release calls for objects in
-  // use, so that we don't duplicate PlasmaClient::Release calls (when handling
-  // a SIGTERM, for example).
-
-  // Close the connections to Plasma. The Plasma store will release the objects
-  // that were in use by us when handling the SIGPIPE.
-  close(store_conn_);
-  if (manager_conn_ >= 0) { close(manager_conn_); }
-  return Status::OK();
-}
-
-#define h_addr h_addr_list[0]
-
-Status PlasmaClient::Transfer(const char* address, int port, const ObjectID& object_id) {
-  return SendDataRequest(manager_conn_, object_id, address, port);
-}
-
-Status PlasmaClient::Fetch(int num_object_ids, const ObjectID* object_ids) {
-  ARROW_CHECK(manager_conn_ >= 0);
-  return SendFetchRequest(manager_conn_, object_ids, num_object_ids);
-}
-
-int PlasmaClient::get_manager_fd() {
-  return manager_conn_;
-}
-
-Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
-  ARROW_CHECK(manager_conn_ >= 0);
-
-  RETURN_NOT_OK(SendStatusRequest(manager_conn_, &object_id, 1));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaStatusReply, &buffer));
-  ObjectID id;
-  RETURN_NOT_OK(ReadStatusReply(buffer.data(), &id, object_status, 1));
-  ARROW_CHECK(object_id == id);
-  return Status::OK();
-}
-
-Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_requests,
-    int num_ready_objects, int64_t timeout_ms, int* num_objects_ready) {
-  ARROW_CHECK(manager_conn_ >= 0);
-  ARROW_CHECK(num_object_requests > 0);
-  ARROW_CHECK(num_ready_objects > 0);
-  ARROW_CHECK(num_ready_objects <= num_object_requests);
-
-  for (int i = 0; i < num_object_requests; ++i) {
-    ARROW_CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL ||
-                object_requests[i].type == PLASMA_QUERY_ANYWHERE);
-  }
-
-  RETURN_NOT_OK(SendWaitRequest(manager_conn_, object_requests, num_object_requests,
-      num_ready_objects, timeout_ms));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaWaitReply, &buffer));
-  RETURN_NOT_OK(ReadWaitReply(buffer.data(), object_requests, &num_ready_objects));
-
-  *num_objects_ready = 0;
-  for (int i = 0; i < num_object_requests; ++i) {
-    int type = object_requests[i].type;
-    int status = object_requests[i].status;
-    switch (type) {
-      case PLASMA_QUERY_LOCAL:
-        if (status == ObjectStatus_Local) { *num_objects_ready += 1; }
-        break;
-      case PLASMA_QUERY_ANYWHERE:
-        if (status == ObjectStatus_Local || status == ObjectStatus_Remote) {
-          *num_objects_ready += 1;
-        } else {
-          ARROW_CHECK(status == ObjectStatus_Nonexistent);
-        }
-        break;
-      default:
-        ARROW_LOG(FATAL) << "This code should be unreachable.";
-    }
-  }
-  return Status::OK();
-}
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
deleted file mode 100644
index fb3a161795d47..0000000000000
--- a/cpp/src/plasma/client.h
+++ /dev/null
@@ -1,343 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_CLIENT_H
-#define PLASMA_CLIENT_H
-
-#include <stdbool.h>
-#include <time.h>
-
-#include <deque>
-#include <string>
-
-#include "plasma/plasma.h"
-
-using arrow::Status;
-
-#define PLASMA_DEFAULT_RELEASE_DELAY 64
-
-// Use 100MB as an overestimate of the L3 cache size.
-constexpr int64_t kL3CacheSizeBytes = 100000000;
-
-/// Object buffer data structure.
-struct ObjectBuffer {
-  /// The size in bytes of the data object.
-  int64_t data_size;
-  /// The address of the data object.
-  uint8_t* data;
-  /// The metadata size in bytes.
-  int64_t metadata_size;
-  /// The address of the metadata.
-  uint8_t* metadata;
-};
-
-/// Configuration options for the plasma client.
-struct PlasmaClientConfig {
-  /// Number of release calls we wait until the object is actually released.
-  /// This allows us to avoid invalidating the cpu cache on workers if objects
-  /// are reused accross tasks.
-  size_t release_delay;
-};
-
-struct ClientMmapTableEntry {
-  /// The result of mmap for this file descriptor.
-  uint8_t* pointer;
-  /// The length of the memory-mapped file.
-  size_t length;
-  /// The number of objects in this memory-mapped file that are currently being
-  /// used by the client. When this count reaches zeros, we unmap the file.
-  int count;
-};
-
-struct ObjectInUseEntry {
-  /// A count of the number of times this client has called PlasmaClient::Create
-  /// or
-  /// PlasmaClient::Get on this object ID minus the number of calls to
-  /// PlasmaClient::Release.
-  /// When this count reaches zero, we remove the entry from the ObjectsInUse
-  /// and decrement a count in the relevant ClientMmapTableEntry.
-  int count;
-  /// Cached information to read the object.
-  PlasmaObject object;
-  /// A flag representing whether the object has been sealed.
-  bool is_sealed;
-};
-
-class PlasmaClient {
- public:
-  /// Connect to the local plasma store and plasma manager. Return
-  /// the resulting connection.
-  ///
-  /// @param store_socket_name The name of the UNIX domain socket to use to
-  ///        connect to the Plasma store.
-  /// @param manager_socket_name The name of the UNIX domain socket to use to
-  ///        connect to the local Plasma manager. If this is "", then this
-  ///        function will not connect to a manager.
-  /// @param release_delay Number of released objects that are kept around
-  ///        and not evicted to avoid too many munmaps.
-  /// @return The return status.
-  Status Connect(const std::string& store_socket_name,
-      const std::string& manager_socket_name, int release_delay);
-
-  /// Create an object in the Plasma Store. Any metadata for this object must be
-  /// be passed in when the object is created.
-  ///
-  /// @param object_id The ID to use for the newly created object.
-  /// @param data_size The size in bytes of the space to be allocated for this
-  /// object's
-  ///        data (this does not include space used for metadata).
-  /// @param metadata The object's metadata. If there is no metadata, this
-  /// pointer
-  ///        should be NULL.
-  /// @param metadata_size The size in bytes of the metadata. If there is no
-  ///        metadata, this should be 0.
-  /// @param data The address of the newly created object will be written here.
-  /// @return The return status.
-  Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
-      int64_t metadata_size, uint8_t** data);
-
-  /// Get some objects from the Plasma Store. This function will block until the
-  /// objects have all been created and sealed in the Plasma Store or the
-  /// timeout
-  /// expires. The caller is responsible for releasing any retrieved objects,
-  /// but
-  /// the caller should not release objects that were not retrieved.
-  ///
-  /// @param object_ids The IDs of the objects to get.
-  /// @param num_object_ids The number of object IDs to get.
-  /// @param timeout_ms The amount of time in milliseconds to wait before this
-  ///        request times out. If this value is -1, then no timeout is set.
-  /// @param object_buffers An array where the results will be stored. If the
-  /// data
-  ///        size field is -1, then the object was not retrieved.
-  /// @return The return status.
-  Status Get(const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms,
-      ObjectBuffer* object_buffers);
-
-  /// Tell Plasma that the client no longer needs the object. This should be
-  /// called
-  /// after Get when the client is done with the object. After this call,
-  /// the address returned by Get is no longer valid. This should be called
-  /// once for each call to Get (with the same object ID).
-  ///
-  /// @param object_id The ID of the object that is no longer needed.
-  /// @return The return status.
-  Status Release(const ObjectID& object_id);
-
-  /// Check if the object store contains a particular object and the object has
-  /// been sealed. The result will be stored in has_object.
-  ///
-  /// @todo: We may want to indicate if the object has been created but not
-  /// sealed.
-  ///
-  /// @param object_id The ID of the object whose presence we are checking.
-  /// @param has_object The function will write true at this address if
-  ///        the object is present and false if it is not present.
-  /// @return The return status.
-  Status Contains(const ObjectID& object_id, bool* has_object);
-
-  /// Seal an object in the object store. The object will be immutable after
-  /// this
-  /// call.
-  ///
-  /// @param object_id The ID of the object to seal.
-  /// @return The return status.
-  Status Seal(const ObjectID& object_id);
-
-  /// Delete an object from the object store. This currently assumes that the
-  /// object is present and has been sealed.
-  ///
-  /// @todo We may want to allow the deletion of objects that are not present or
-  ///       haven't been sealed.
-  ///
-  /// @param object_id The ID of the object to delete.
-  /// @return The return status.
-  Status Delete(const ObjectID& object_id);
-
-  /// Delete objects until we have freed up num_bytes bytes or there are no more
-  /// released objects that can be deleted.
-  ///
-  /// @param num_bytes The number of bytes to try to free up.
-  /// @param num_bytes_evicted Out parameter for total number of bytes of space
-  /// retrieved.
-  /// @return The return status.
-  Status Evict(int64_t num_bytes, int64_t& num_bytes_evicted);
-
-  /// Subscribe to notifications when objects are sealed in the object store.
-  /// Whenever an object is sealed, a message will be written to the client
-  /// socket
-  /// that is returned by this method.
-  ///
-  /// @param fd Out parameter for the file descriptor the client should use to
-  /// read notifications
-  ///         from the object store about sealed objects.
-  /// @return The return status.
-  Status Subscribe(int* fd);
-
-  /// Disconnect from the local plasma instance, including the local store and
-  /// manager.
-  ///
-  /// @return The return status.
-  Status Disconnect();
-
-  /// Attempt to initiate the transfer of some objects from remote Plasma
-  /// Stores.
-  /// This method does not guarantee that the fetched objects will arrive
-  /// locally.
-  ///
-  /// For an object that is available in the local Plasma Store, this method
-  /// will
-  /// not do anything. For an object that is not available locally, it will
-  /// check
-  /// if the object are already being fetched. If so, it will not do anything.
-  /// If
-  /// not, it will query the object table for a list of Plasma Managers that
-  /// have
-  /// the object. The object table will return a non-empty list, and this Plasma
-  /// Manager will attempt to initiate transfers from one of those Plasma
-  /// Managers.
-  ///
-  /// This function is non-blocking.
-  ///
-  /// This method is idempotent in the sense that it is ok to call it multiple
-  /// times.
-  ///
-  /// @param num_object_ids The number of object IDs fetch is being called on.
-  /// @param object_ids The IDs of the objects that fetch is being called on.
-  /// @return The return status.
-  Status Fetch(int num_object_ids, const ObjectID* object_ids);
-
-  /// Wait for (1) a specified number of objects to be available (sealed) in the
-  /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to
-  /// expire. This is a blocking call.
-  ///
-  /// @param num_object_requests Size of the object_requests array.
-  /// @param object_requests Object event array. Each element contains a request
-  ///        for a particular object_id. The type of request is specified in the
-  ///        "type" field.
-  ///        - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes
-  ///          available in the local Plasma Store. In this case, this function
-  ///          sets the "status" field to ObjectStatus_Local. Note, if the
-  ///          status
-  ///          is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent,
-  ///          but it may exist elsewhere in the system.
-  ///        - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id
-  ///        becomes
-  ///          available either at the local Plasma Store or on a remote Plasma
-  ///          Store. In this case, the functions sets the "status" field to
-  ///          ObjectStatus_Local or ObjectStatus_Remote.
-  /// @param num_ready_objects The number of requests in object_requests array
-  /// that
-  ///        must be satisfied before the function returns, unless it timeouts.
-  ///        The num_ready_objects should be no larger than num_object_requests.
-  /// @param timeout_ms Timeout value in milliseconds. If this timeout expires
-  ///        before min_num_ready_objects of requests are satisfied, the
-  ///        function
-  ///        returns.
-  /// @param num_objects_ready Out parameter for number of satisfied requests in
-  ///        the object_requests list. If the returned number is less than
-  ///        min_num_ready_objects this means that timeout expired.
-  /// @return The return status.
-  Status Wait(int64_t num_object_requests, ObjectRequest* object_requests,
-      int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
-
-  /// Transfer local object to a different plasma manager.
-  ///
-  /// @param conn The object containing the connection state.
-  /// @param addr IP address of the plasma manager we are transfering to.
-  /// @param port Port of the plasma manager we are transfering to.
-  /// @object_id ObjectID of the object we are transfering.
-  /// @return The return status.
-  Status Transfer(const char* addr, int port, const ObjectID& object_id);
-
-  /// Return the status of a given object. This method may query the object
-  /// table.
-  ///
-  /// @param conn The object containing the connection state.
-  /// @param object_id The ID of the object whose status we query.
-  /// @param object_status Out parameter for object status. Can take the
-  ///         following values.
-  ///         - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma
-  ///         Store.
-  ///           has been already scheduled by the Plasma Manager.
-  ///         - PLASMA_CLIENT_TRANSFER, if the object is either currently being
-  ///           transferred or just scheduled.
-  ///         - PLASMA_CLIENT_REMOTE, if the object is stored at a remote
-  ///           Plasma Store.
-  ///         - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the
-  ///           system.
-  /// @return The return status.
-  Status Info(const ObjectID& object_id, int* object_status);
-
-  /// Get the file descriptor for the socket connection to the plasma manager.
-  ///
-  /// @param conn The plasma connection.
-  /// @return The file descriptor for the manager connection. If there is no
-  ///         connection to the manager, this is -1.
-  int get_manager_fd();
-
- private:
-  Status PerformRelease(const ObjectID& object_id);
-
-  uint8_t* lookup_or_mmap(int fd, int store_fd_val, int64_t map_size);
-
-  uint8_t* lookup_mmapped_file(int store_fd_val);
-
-  void increment_object_count(
-      const ObjectID& object_id, PlasmaObject* object, bool is_sealed);
-
-  /// File descriptor of the Unix domain socket that connects to the store.
-  int store_conn_;
-  /// File descriptor of the Unix domain socket that connects to the manager.
-  int manager_conn_;
-  /// Table of dlmalloc buffer files that have been memory mapped so far. This
-  /// is a hash table mapping a file descriptor to a struct containing the
-  /// address of the corresponding memory-mapped file.
-  std::unordered_map<int, ClientMmapTableEntry> mmap_table_;
-  /// A hash table of the object IDs that are currently being used by this
-  /// client.
-  std::unordered_map<ObjectID, std::unique_ptr<ObjectInUseEntry>, UniqueIDHasher>
-      objects_in_use_;
-  /// Object IDs of the last few release calls. This is a deque and
-  /// is used to delay releasing objects to see if they can be reused by
-  /// subsequent tasks so we do not unneccessarily invalidate cpu caches.
-  /// TODO(pcm): replace this with a proper lru cache using the size of the L3
-  /// cache.
-  std::deque<ObjectID> release_history_;
-  /// The number of bytes in the combined objects that are held in the release
-  /// history doubly-linked list. If this is too large then the client starts
-  /// releasing objects.
-  int64_t in_use_object_bytes_;
-  /// Configuration options for the plasma client.
-  PlasmaClientConfig config_;
-  /// The amount of memory available to the Plasma store. The client needs this
-  /// information to make sure that it does not delay in releasing so much
-  /// memory that the store is unable to evict enough objects to free up space.
-  int64_t store_capacity_;
-};
-
-/// Compute the hash of an object in the object store.
-///
-/// @param conn The object containing the connection state.
-/// @param object_id The ID of the object we want to hash.
-/// @param digest A pointer at which to return the hash digest of the object.
-///        The pointer must have at least DIGEST_SIZE bytes allocated.
-/// @return A boolean representing whether the hash operation succeeded.
-bool plasma_compute_object_hash(
-    PlasmaClient* conn, ObjectID object_id, unsigned char* digest);
-
-#endif  // PLASMA_CLIENT_H
diff --git a/cpp/src/plasma/common.cc b/cpp/src/plasma/common.cc
deleted file mode 100644
index a09a963fa4769..0000000000000
--- a/cpp/src/plasma/common.cc
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/common.h"
-
-#include <random>
-
-#include "format/plasma_generated.h"
-
-using arrow::Status;
-
-UniqueID UniqueID::from_random() {
-  UniqueID id;
-  uint8_t* data = id.mutable_data();
-  std::random_device engine;
-  for (int i = 0; i < kUniqueIDSize; i++) {
-    data[i] = static_cast<uint8_t>(engine());
-  }
-  return id;
-}
-
-UniqueID UniqueID::from_binary(const std::string& binary) {
-  UniqueID id;
-  std::memcpy(&id, binary.data(), sizeof(id));
-  return id;
-}
-
-const uint8_t* UniqueID::data() const {
-  return id_;
-}
-
-uint8_t* UniqueID::mutable_data() {
-  return id_;
-}
-
-std::string UniqueID::binary() const {
-  return std::string(reinterpret_cast<const char*>(id_), kUniqueIDSize);
-}
-
-std::string UniqueID::hex() const {
-  constexpr char hex[] = "0123456789abcdef";
-  std::string result;
-  for (int i = 0; i < kUniqueIDSize; i++) {
-    unsigned int val = id_[i];
-    result.push_back(hex[val >> 4]);
-    result.push_back(hex[val & 0xf]);
-  }
-  return result;
-}
-
-bool UniqueID::operator==(const UniqueID& rhs) const {
-  return std::memcmp(data(), rhs.data(), kUniqueIDSize) == 0;
-}
-
-Status plasma_error_status(int plasma_error) {
-  switch (plasma_error) {
-    case PlasmaError_OK:
-      return Status::OK();
-    case PlasmaError_ObjectExists:
-      return Status::PlasmaObjectExists("object already exists in the plasma store");
-    case PlasmaError_ObjectNonexistent:
-      return Status::PlasmaObjectNonexistent("object does not exist in the plasma store");
-    case PlasmaError_OutOfMemory:
-      return Status::PlasmaStoreFull("object does not fit in the plasma store");
-    default:
-      ARROW_LOG(FATAL) << "unknown plasma error code " << plasma_error;
-  }
-  return Status::OK();
-}
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
deleted file mode 100644
index 85dc74bf86e0d..0000000000000
--- a/cpp/src/plasma/common.h
+++ /dev/null
@@ -1,63 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_COMMON_H
-#define PLASMA_COMMON_H
-
-#include <cstring>
-#include <string>
-// TODO(pcm): Convert getopt and sscanf in the store to use more idiomatic C++
-// and get rid of the next three lines:
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include "arrow/status.h"
-#include "arrow/util/logging.h"
-
-constexpr int64_t kUniqueIDSize = 20;
-
-class UniqueID {
- public:
-  static UniqueID from_random();
-  static UniqueID from_binary(const std::string& binary);
-  bool operator==(const UniqueID& rhs) const;
-  const uint8_t* data() const;
-  uint8_t* mutable_data();
-  std::string binary() const;
-  std::string hex() const;
-
- private:
-  uint8_t id_[kUniqueIDSize];
-};
-
-static_assert(std::is_pod<UniqueID>::value, "UniqueID must be plain old data");
-
-struct UniqueIDHasher {
-  // ObjectID hashing function.
-  size_t operator()(const UniqueID& id) const {
-    size_t result;
-    std::memcpy(&result, id.data(), sizeof(size_t));
-    return result;
-  }
-};
-
-typedef UniqueID ObjectID;
-
-arrow::Status plasma_error_status(int plasma_error);
-
-#endif  // PLASMA_COMMON_H
diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
deleted file mode 100644
index a9f7356e1f67e..0000000000000
--- a/cpp/src/plasma/events.cc
+++ /dev/null
@@ -1,81 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/events.h"
-
-#include <errno.h>
-
-void EventLoop::file_event_callback(
-    aeEventLoop* loop, int fd, void* context, int events) {
-  FileCallback* callback = reinterpret_cast<FileCallback*>(context);
-  (*callback)(events);
-}
-
-int EventLoop::timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context) {
-  TimerCallback* callback = reinterpret_cast<TimerCallback*>(context);
-  return (*callback)(timer_id);
-}
-
-constexpr int kInitialEventLoopSize = 1024;
-
-EventLoop::EventLoop() {
-  loop_ = aeCreateEventLoop(kInitialEventLoopSize);
-}
-
-bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback) {
-  if (file_callbacks_.find(fd) != file_callbacks_.end()) { return false; }
-  auto data = std::unique_ptr<FileCallback>(new FileCallback(callback));
-  void* context = reinterpret_cast<void*>(data.get());
-  // Try to add the file descriptor.
-  int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
-  // If it cannot be added, increase the size of the event loop.
-  if (err == AE_ERR && errno == ERANGE) {
-    err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2);
-    if (err != AE_OK) { return false; }
-    err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
-  }
-  // In any case, test if there were errors.
-  if (err == AE_OK) {
-    file_callbacks_.emplace(fd, std::move(data));
-    return true;
-  }
-  return false;
-}
-
-void EventLoop::remove_file_event(int fd) {
-  aeDeleteFileEvent(loop_, fd, AE_READABLE | AE_WRITABLE);
-  file_callbacks_.erase(fd);
-}
-
-void EventLoop::run() {
-  aeMain(loop_);
-}
-
-int64_t EventLoop::add_timer(int64_t timeout, const TimerCallback& callback) {
-  auto data = std::unique_ptr<TimerCallback>(new TimerCallback(callback));
-  void* context = reinterpret_cast<void*>(data.get());
-  int64_t timer_id =
-      aeCreateTimeEvent(loop_, timeout, EventLoop::timer_event_callback, context, NULL);
-  timer_callbacks_.emplace(timer_id, std::move(data));
-  return timer_id;
-}
-
-int EventLoop::remove_timer(int64_t timer_id) {
-  int err = aeDeleteTimeEvent(loop_, timer_id);
-  timer_callbacks_.erase(timer_id);
-  return err;
-}
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
deleted file mode 100644
index bd93d6bb2a6fd..0000000000000
--- a/cpp/src/plasma/events.h
+++ /dev/null
@@ -1,99 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_EVENTS
-#define PLASMA_EVENTS
-
-#include <functional>
-#include <memory>
-#include <unordered_map>
-
-extern "C" {
-#include "ae/ae.h"
-}
-
-/// Constant specifying that the timer is done and it will be removed.
-constexpr int kEventLoopTimerDone = AE_NOMORE;
-
-/// Read event on the file descriptor.
-constexpr int kEventLoopRead = AE_READABLE;
-
-/// Write event on the file descriptor.
-constexpr int kEventLoopWrite = AE_WRITABLE;
-
-typedef long long TimerID;  // NOLINT
-
-class EventLoop {
- public:
-  // Signature of the handler that will be called when there is a new event
-  // on the file descriptor that this handler has been registered for.
-  //
-  // The arguments are the event flags (read or write).
-  using FileCallback = std::function<void(int)>;
-
-  // This handler will be called when a timer times out. The timer id is
-  // passed as an argument. The return is the number of milliseconds the timer
-  // shall be reset to or kEventLoopTimerDone if the timer shall not be
-  // triggered again.
-  using TimerCallback = std::function<int(int64_t)>;
-
-  EventLoop();
-
-  /// Add a new file event handler to the event loop.
-  ///
-  /// @param fd The file descriptor we are listening to.
-  /// @param events The flags for events we are listening to (read or write).
-  /// @param callback The callback that will be called when the event happens.
-  /// @return Returns true if the event handler was added successfully.
-  bool add_file_event(int fd, int events, const FileCallback& callback);
-
-  /// Remove a file event handler from the event loop.
-  ///
-  /// @param fd The file descriptor of the event handler.
-  /// @return Void.
-  void remove_file_event(int fd);
-
-  /// Register a handler that will be called after a time slice of
-  ///  "timeout" milliseconds.
-  ///
-  ///  @param timeout The timeout in milliseconds.
-  ///  @param callback The callback for the timeout.
-  ///  @return The ID of the newly created timer.
-  int64_t add_timer(int64_t timeout, const TimerCallback& callback);
-
-  /// Remove a timer handler from the event loop.
-  ///
-  /// @param timer_id The ID of the timer that is to be removed.
-  /// @return The ae.c error code. TODO(pcm): needs to be standardized
-  int remove_timer(int64_t timer_id);
-
-  /// Run the event loop.
-  ///
-  /// @return Void.
-  void run();
-
- private:
-  static void file_event_callback(aeEventLoop* loop, int fd, void* context, int events);
-
-  static int timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context);
-
-  aeEventLoop* loop_;
-  std::unordered_map<int, std::unique_ptr<FileCallback>> file_callbacks_;
-  std::unordered_map<int64_t, std::unique_ptr<TimerCallback>> timer_callbacks_;
-};
-
-#endif  // PLASMA_EVENTS
diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
deleted file mode 100644
index 4ae6384d42543..0000000000000
--- a/cpp/src/plasma/eviction_policy.cc
+++ /dev/null
@@ -1,107 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/eviction_policy.h"
-
-#include <algorithm>
-
-void LRUCache::add(const ObjectID& key, int64_t size) {
-  auto it = item_map_.find(key);
-  ARROW_CHECK(it == item_map_.end());
-  /* Note that it is important to use a list so the iterators stay valid. */
-  item_list_.emplace_front(key, size);
-  item_map_.emplace(key, item_list_.begin());
-}
-
-void LRUCache::remove(const ObjectID& key) {
-  auto it = item_map_.find(key);
-  ARROW_CHECK(it != item_map_.end());
-  item_list_.erase(it->second);
-  item_map_.erase(it);
-}
-
-int64_t LRUCache::choose_objects_to_evict(
-    int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
-  int64_t bytes_evicted = 0;
-  auto it = item_list_.end();
-  while (bytes_evicted < num_bytes_required && it != item_list_.begin()) {
-    it--;
-    objects_to_evict->push_back(it->first);
-    bytes_evicted += it->second;
-  }
-  return bytes_evicted;
-}
-
-EvictionPolicy::EvictionPolicy(PlasmaStoreInfo* store_info)
-    : memory_used_(0), store_info_(store_info) {}
-
-int64_t EvictionPolicy::choose_objects_to_evict(
-    int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
-  int64_t bytes_evicted =
-      cache_.choose_objects_to_evict(num_bytes_required, objects_to_evict);
-  /* Update the LRU cache. */
-  for (auto& object_id : *objects_to_evict) {
-    cache_.remove(object_id);
-  }
-  /* Update the number of bytes used. */
-  memory_used_ -= bytes_evicted;
-  return bytes_evicted;
-}
-
-void EvictionPolicy::object_created(const ObjectID& object_id) {
-  auto entry = store_info_->objects[object_id].get();
-  cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
-}
-
-bool EvictionPolicy::require_space(
-    int64_t size, std::vector<ObjectID>* objects_to_evict) {
-  /* Check if there is enough space to create the object. */
-  int64_t required_space = memory_used_ + size - store_info_->memory_capacity;
-  int64_t num_bytes_evicted;
-  if (required_space > 0) {
-    /* Try to free up at least as much space as we need right now but ideally
-     * up to 20% of the total capacity. */
-    int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5);
-    ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects";
-    /* Choose some objects to evict, and update the return pointers. */
-    num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict);
-    ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting "
-                    << objects_to_evict->size() << " objects to free up "
-                    << num_bytes_evicted << " bytes.";
-  } else {
-    num_bytes_evicted = 0;
-  }
-  if (num_bytes_evicted >= required_space) {
-    /* We only increment the space used if there is enough space to create the
-     * object. */
-    memory_used_ += size;
-  }
-  return num_bytes_evicted >= required_space;
-}
-
-void EvictionPolicy::begin_object_access(
-    const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
-  /* If the object is in the LRU cache, remove it. */
-  cache_.remove(object_id);
-}
-
-void EvictionPolicy::end_object_access(
-    const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
-  auto entry = store_info_->objects[object_id].get();
-  /* Add the object to the LRU cache.*/
-  cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
-}
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
deleted file mode 100644
index 3815fc6652f0c..0000000000000
--- a/cpp/src/plasma/eviction_policy.h
+++ /dev/null
@@ -1,134 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_EVICTION_POLICY_H
-#define PLASMA_EVICTION_POLICY_H
-
-#include <list>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-#include "plasma/common.h"
-#include "plasma/plasma.h"
-
-// ==== The eviction policy ====
-//
-// This file contains declaration for all functions and data structures that
-// need to be provided if you want to implement a new eviction algorithm for the
-// Plasma store.
-
-class LRUCache {
- public:
-  LRUCache() {}
-
-  void add(const ObjectID& key, int64_t size);
-
-  void remove(const ObjectID& key);
-
-  int64_t choose_objects_to_evict(
-      int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
-
- private:
-  /// A doubly-linked list containing the items in the cache and
-  /// their sizes in LRU order.
-  typedef std::list<std::pair<ObjectID, int64_t>> ItemList;
-  ItemList item_list_;
-  /// A hash table mapping the object ID of an object in the cache to its
-  /// location in the doubly linked list item_list_.
-  std::unordered_map<ObjectID, ItemList::iterator, UniqueIDHasher> item_map_;
-};
-
-/// The eviction policy.
-class EvictionPolicy {
- public:
-  /// Construct an eviction policy.
-  ///
-  /// @param store_info Information about the Plasma store that is exposed
-  ///        to the eviction policy.
-  explicit EvictionPolicy(PlasmaStoreInfo* store_info);
-
-  /// This method will be called whenever an object is first created in order to
-  /// add it to the LRU cache. This is done so that the first time, the Plasma
-  /// store calls begin_object_access, we can remove the object from the LRU
-  /// cache.
-  ///
-  /// @param object_id The object ID of the object that was created.
-  /// @return Void.
-  void object_created(const ObjectID& object_id);
-
-  /// This method will be called when the Plasma store needs more space, perhaps
-  /// to create a new object. If the required amount of space cannot be freed up,
-  /// then a fatal error will be thrown. When this method is called, the eviction
-  /// policy will assume that the objects chosen to be evicted will in fact be
-  /// evicted from the Plasma store by the caller.
-  ///
-  /// @param size The size in bytes of the new object, including both data and
-  ///        metadata.
-  /// @param objects_to_evict The object IDs that were chosen for eviction will
-  ///        be stored into this vector.
-  /// @return True if enough space can be freed and false otherwise.
-  bool require_space(int64_t size, std::vector<ObjectID>* objects_to_evict);
-
-  /// This method will be called whenever an unused object in the Plasma store
-  /// starts to be used. When this method is called, the eviction policy will
-  /// assume that the objects chosen to be evicted will in fact be evicted from
-  /// the Plasma store by the caller.
-  ///
-  /// @param object_id The ID of the object that is now being used.
-  /// @param objects_to_evict The object IDs that were chosen for eviction will
-  ///        be stored into this vector.
-  /// @return Void.
-  void begin_object_access(
-      const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
-
-  /// This method will be called whenever an object in the Plasma store that was
-  /// being used is no longer being used. When this method is called, the
-  /// eviction policy will assume that the objects chosen to be evicted will in
-  /// fact be evicted from the Plasma store by the caller.
-  ///
-  /// @param object_id The ID of the object that is no longer being used.
-  /// @param objects_to_evict The object IDs that were chosen for eviction will
-  ///        be stored into this vector.
-  /// @return Void.
-  void end_object_access(
-      const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
-
-  /// Choose some objects to evict from the Plasma store. When this method is
-  /// called, the eviction policy will assume that the objects chosen to be
-  /// evicted will in fact be evicted from the Plasma store by the caller.
-  ///
-  /// @note This method is not part of the API. It is exposed in the header file
-  /// only for testing.
-  ///
-  /// @param num_bytes_required The number of bytes of space to try to free up.
-  /// @param objects_to_evict The object IDs that were chosen for eviction will
-  ///        be stored into this vector.
-  /// @return The total number of bytes of space chosen to be evicted.
-  int64_t choose_objects_to_evict(
-      int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
-
- private:
-  /// The amount of memory (in bytes) currently being used.
-  int64_t memory_used_;
-  /// Pointer to the plasma store info.
-  PlasmaStoreInfo* store_info_;
-  /// Datastructure for the LRU cache.
-  LRUCache cache_;
-};
-
-#endif  // PLASMA_EVICTION_POLICY_H
diff --git a/cpp/src/plasma/extension.cc b/cpp/src/plasma/extension.cc
deleted file mode 100644
index 5d61e337c108d..0000000000000
--- a/cpp/src/plasma/extension.cc
+++ /dev/null
@@ -1,456 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/extension.h"
-
-#include <algorithm>
-#include <vector>
-
-#include "plasma/client.h"
-#include "plasma/common.h"
-#include "plasma/io.h"
-#include "plasma/protocol.h"
-
-PyObject* PlasmaOutOfMemoryError;
-PyObject* PlasmaObjectExistsError;
-
-PyObject* PyPlasma_connect(PyObject* self, PyObject* args) {
-  const char* store_socket_name;
-  const char* manager_socket_name;
-  int release_delay;
-  if (!PyArg_ParseTuple(
-          args, "ssi", &store_socket_name, &manager_socket_name, &release_delay)) {
-    return NULL;
-  }
-  PlasmaClient* client = new PlasmaClient();
-  ARROW_CHECK_OK(client->Connect(store_socket_name, manager_socket_name, release_delay));
-
-  return PyCapsule_New(client, "plasma", NULL);
-}
-
-PyObject* PyPlasma_disconnect(PyObject* self, PyObject* args) {
-  PyObject* client_capsule;
-  if (!PyArg_ParseTuple(args, "O", &client_capsule)) { return NULL; }
-  PlasmaClient* client;
-  ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client));
-  ARROW_CHECK_OK(client->Disconnect());
-  /* We use the context of the connection capsule to indicate if the connection
-   * is still active (if the context is NULL) or if it is closed (if the context
-   * is (void*) 0x1). This is neccessary because the primary pointer of the
-   * capsule cannot be NULL. */
-  PyCapsule_SetContext(client_capsule, reinterpret_cast<void*>(0x1));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_create(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  Py_ssize_t size;
-  PyObject* metadata;
-  if (!PyArg_ParseTuple(args, "O&O&nO", PyObjectToPlasmaClient, &client,
-          PyStringToUniqueID, &object_id, &size, &metadata)) {
-    return NULL;
-  }
-  if (!PyByteArray_Check(metadata)) {
-    PyErr_SetString(PyExc_TypeError, "metadata must be a bytearray");
-    return NULL;
-  }
-  uint8_t* data;
-  Status s = client->Create(object_id, size,
-      reinterpret_cast<uint8_t*>(PyByteArray_AsString(metadata)),
-      PyByteArray_Size(metadata), &data);
-  if (s.IsPlasmaObjectExists()) {
-    PyErr_SetString(PlasmaObjectExistsError,
-        "An object with this ID already exists in the plasma "
-        "store.");
-    return NULL;
-  }
-  if (s.IsPlasmaStoreFull()) {
-    PyErr_SetString(PlasmaOutOfMemoryError,
-        "The plasma store ran out of memory and could not create "
-        "this object.");
-    return NULL;
-  }
-  ARROW_CHECK(s.ok());
-
-#if PY_MAJOR_VERSION >= 3
-  return PyMemoryView_FromMemory(reinterpret_cast<char*>(data), size, PyBUF_WRITE);
-#else
-  return PyBuffer_FromReadWriteMemory(reinterpret_cast<void*>(data), size);
-#endif
-}
-
-PyObject* PyPlasma_hash(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  unsigned char digest[kDigestSize];
-  bool success = plasma_compute_object_hash(client, object_id, digest);
-  if (success) {
-    PyObject* digest_string =
-        PyBytes_FromStringAndSize(reinterpret_cast<char*>(digest), kDigestSize);
-    return digest_string;
-  } else {
-    Py_RETURN_NONE;
-  }
-}
-
-PyObject* PyPlasma_seal(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Seal(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_release(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Release(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_get(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  PyObject* object_id_list;
-  Py_ssize_t timeout_ms;
-  if (!PyArg_ParseTuple(
-          args, "O&On", PyObjectToPlasmaClient, &client, &object_id_list, &timeout_ms)) {
-    return NULL;
-  }
-
-  Py_ssize_t num_object_ids = PyList_Size(object_id_list);
-  std::vector<ObjectID> object_ids(num_object_ids);
-  std::vector<ObjectBuffer> object_buffers(num_object_ids);
-
-  for (int i = 0; i < num_object_ids; ++i) {
-    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
-  }
-
-  Py_BEGIN_ALLOW_THREADS;
-  ARROW_CHECK_OK(
-      client->Get(object_ids.data(), num_object_ids, timeout_ms, object_buffers.data()));
-  Py_END_ALLOW_THREADS;
-
-  PyObject* returns = PyList_New(num_object_ids);
-  for (int i = 0; i < num_object_ids; ++i) {
-    if (object_buffers[i].data_size != -1) {
-      /* The object was retrieved, so return the object. */
-      PyObject* t = PyTuple_New(2);
-      Py_ssize_t data_size = static_cast<Py_ssize_t>(object_buffers[i].data_size);
-      Py_ssize_t metadata_size = static_cast<Py_ssize_t>(object_buffers[i].metadata_size);
-#if PY_MAJOR_VERSION >= 3
-      char* data = reinterpret_cast<char*>(object_buffers[i].data);
-      char* metadata = reinterpret_cast<char*>(object_buffers[i].metadata);
-      PyTuple_SET_ITEM(t, 0, PyMemoryView_FromMemory(data, data_size, PyBUF_READ));
-      PyTuple_SET_ITEM(
-          t, 1, PyMemoryView_FromMemory(metadata, metadata_size, PyBUF_READ));
-#else
-      void* data = reinterpret_cast<void*>(object_buffers[i].data);
-      void* metadata = reinterpret_cast<void*>(object_buffers[i].metadata);
-      PyTuple_SET_ITEM(t, 0, PyBuffer_FromMemory(data, data_size));
-      PyTuple_SET_ITEM(t, 1, PyBuffer_FromMemory(metadata, metadata_size));
-#endif
-      ARROW_CHECK(PyList_SetItem(returns, i, t) == 0);
-    } else {
-      /* The object was not retrieved, so just add None to the list of return
-       * values. */
-      Py_INCREF(Py_None);
-      ARROW_CHECK(PyList_SetItem(returns, i, Py_None) == 0);
-    }
-  }
-  return returns;
-}
-
-PyObject* PyPlasma_contains(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  bool has_object;
-  ARROW_CHECK_OK(client->Contains(object_id, &has_object));
-
-  if (has_object) {
-    Py_RETURN_TRUE;
-  } else {
-    Py_RETURN_FALSE;
-  }
-}
-
-PyObject* PyPlasma_fetch(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  PyObject* object_id_list;
-  if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client, &object_id_list)) {
-    return NULL;
-  }
-  if (client->get_manager_fd() == -1) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-  Py_ssize_t n = PyList_Size(object_id_list);
-  ObjectID* object_ids = new ObjectID[n];
-  for (int i = 0; i < n; ++i) {
-    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
-  }
-  ARROW_CHECK_OK(client->Fetch(static_cast<int>(n), object_ids));
-  delete[] object_ids;
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_wait(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  PyObject* object_id_list;
-  Py_ssize_t timeout;
-  int num_returns;
-  if (!PyArg_ParseTuple(args, "O&Oni", PyObjectToPlasmaClient, &client, &object_id_list,
-          &timeout, &num_returns)) {
-    return NULL;
-  }
-  Py_ssize_t n = PyList_Size(object_id_list);
-
-  if (client->get_manager_fd() == -1) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-  if (num_returns < 0) {
-    PyErr_SetString(
-        PyExc_RuntimeError, "The argument num_returns cannot be less than zero.");
-    return NULL;
-  }
-  if (num_returns > n) {
-    PyErr_SetString(PyExc_RuntimeError,
-        "The argument num_returns cannot be greater than len(object_ids)");
-    return NULL;
-  }
-  int64_t threshold = 1 << 30;
-  if (timeout > threshold) {
-    PyErr_SetString(
-        PyExc_RuntimeError, "The argument timeout cannot be greater than 2 ** 30.");
-    return NULL;
-  }
-
-  std::vector<ObjectRequest> object_requests(n);
-  for (int i = 0; i < n; ++i) {
-    ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i),
-                    &object_requests[i].object_id) == 1);
-    object_requests[i].type = PLASMA_QUERY_ANYWHERE;
-  }
-  /* Drop the global interpreter lock while we are waiting, so other threads can
-   * run. */
-  int num_return_objects;
-  Py_BEGIN_ALLOW_THREADS;
-  ARROW_CHECK_OK(
-      client->Wait(n, object_requests.data(), num_returns, timeout, &num_return_objects));
-  Py_END_ALLOW_THREADS;
-
-  int num_to_return = std::min(num_return_objects, num_returns);
-  PyObject* ready_ids = PyList_New(num_to_return);
-  PyObject* waiting_ids = PySet_New(object_id_list);
-  int num_returned = 0;
-  for (int i = 0; i < n; ++i) {
-    if (num_returned == num_to_return) { break; }
-    if (object_requests[i].status == ObjectStatus_Local ||
-        object_requests[i].status == ObjectStatus_Remote) {
-      PyObject* ready = PyBytes_FromStringAndSize(
-          reinterpret_cast<char*>(&object_requests[i].object_id),
-          sizeof(object_requests[i].object_id));
-      PyList_SetItem(ready_ids, num_returned, ready);
-      PySet_Discard(waiting_ids, ready);
-      num_returned += 1;
-    } else {
-      ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent);
-    }
-  }
-  ARROW_CHECK(num_returned == num_to_return);
-  /* Return both the ready IDs and the remaining IDs. */
-  PyObject* t = PyTuple_New(2);
-  PyTuple_SetItem(t, 0, ready_ids);
-  PyTuple_SetItem(t, 1, waiting_ids);
-  return t;
-}
-
-PyObject* PyPlasma_evict(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  Py_ssize_t num_bytes;
-  if (!PyArg_ParseTuple(args, "O&n", PyObjectToPlasmaClient, &client, &num_bytes)) {
-    return NULL;
-  }
-  int64_t evicted_bytes;
-  ARROW_CHECK_OK(client->Evict(static_cast<int64_t>(num_bytes), evicted_bytes));
-  return PyLong_FromSsize_t(static_cast<Py_ssize_t>(evicted_bytes));
-}
-
-PyObject* PyPlasma_delete(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Delete(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_transfer(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  const char* addr;
-  int port;
-  if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client,
-          PyStringToUniqueID, &object_id, &addr, &port)) {
-    return NULL;
-  }
-
-  if (client->get_manager_fd() == -1) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-
-  ARROW_CHECK_OK(client->Transfer(addr, port, object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_subscribe(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) { return NULL; }
-
-  int sock;
-  ARROW_CHECK_OK(client->Subscribe(&sock));
-  return PyLong_FromLong(sock);
-}
-
-PyObject* PyPlasma_receive_notification(PyObject* self, PyObject* args) {
-  int plasma_sock;
-
-  if (!PyArg_ParseTuple(args, "i", &plasma_sock)) { return NULL; }
-  /* Receive object notification from the plasma connection socket. If the
-   * object was added, return a tuple of its fields: ObjectID, data_size,
-   * metadata_size. If the object was deleted, data_size and metadata_size will
-   * be set to -1. */
-  uint8_t* notification = read_message_async(plasma_sock);
-  if (notification == NULL) {
-    PyErr_SetString(
-        PyExc_RuntimeError, "Failed to read object notification from Plasma socket");
-    return NULL;
-  }
-  auto object_info = flatbuffers::GetRoot<ObjectInfo>(notification);
-  /* Construct a tuple from object_info and return. */
-  PyObject* t = PyTuple_New(3);
-  PyTuple_SetItem(t, 0, PyBytes_FromStringAndSize(object_info->object_id()->data(),
-                            object_info->object_id()->size()));
-  if (object_info->is_deletion()) {
-    PyTuple_SetItem(t, 1, PyLong_FromLong(-1));
-    PyTuple_SetItem(t, 2, PyLong_FromLong(-1));
-  } else {
-    PyTuple_SetItem(t, 1, PyLong_FromLong(object_info->data_size()));
-    PyTuple_SetItem(t, 2, PyLong_FromLong(object_info->metadata_size()));
-  }
-
-  delete[] notification;
-  return t;
-}
-
-static PyMethodDef plasma_methods[] = {
-    {"connect", PyPlasma_connect, METH_VARARGS, "Connect to plasma."},
-    {"disconnect", PyPlasma_disconnect, METH_VARARGS, "Disconnect from plasma."},
-    {"create", PyPlasma_create, METH_VARARGS, "Create a new plasma object."},
-    {"hash", PyPlasma_hash, METH_VARARGS, "Compute the hash of a plasma object."},
-    {"seal", PyPlasma_seal, METH_VARARGS, "Seal a plasma object."},
-    {"get", PyPlasma_get, METH_VARARGS, "Get a plasma object."},
-    {"contains", PyPlasma_contains, METH_VARARGS,
-        "Does the plasma store contain this plasma object?"},
-    {"fetch", PyPlasma_fetch, METH_VARARGS,
-        "Fetch the object from another plasma manager instance."},
-    {"wait", PyPlasma_wait, METH_VARARGS,
-        "Wait until num_returns objects in object_ids are ready."},
-    {"evict", PyPlasma_evict, METH_VARARGS,
-        "Evict some objects until we recover some number of bytes."},
-    {"release", PyPlasma_release, METH_VARARGS, "Release the plasma object."},
-    {"delete", PyPlasma_delete, METH_VARARGS, "Delete a plasma object."},
-    {"transfer", PyPlasma_transfer, METH_VARARGS,
-        "Transfer object to another plasma manager."},
-    {"subscribe", PyPlasma_subscribe, METH_VARARGS,
-        "Subscribe to the plasma notification socket."},
-    {"receive_notification", PyPlasma_receive_notification, METH_VARARGS,
-        "Receive next notification from plasma notification socket."},
-    {NULL} /* Sentinel */
-};
-
-#if PY_MAJOR_VERSION >= 3
-static struct PyModuleDef moduledef = {
-    PyModuleDef_HEAD_INIT, "libplasma",    /* m_name */
-    "A Python client library for plasma.", /* m_doc */
-    0,                                     /* m_size */
-    plasma_methods,                        /* m_methods */
-    NULL,                                  /* m_reload */
-    NULL,                                  /* m_traverse */
-    NULL,                                  /* m_clear */
-    NULL,                                  /* m_free */
-};
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define INITERROR return NULL
-#else
-#define INITERROR return
-#endif
-
-#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
-#define PyMODINIT_FUNC void
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
-#else
-#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
-#endif
-
-MOD_INIT(libplasma) {
-#if PY_MAJOR_VERSION >= 3
-  PyObject* m = PyModule_Create(&moduledef);
-#else
-  PyObject* m =
-      Py_InitModule3("libplasma", plasma_methods, "A Python client library for plasma.");
-#endif
-
-  /* Create a custom exception for when an object ID is reused. */
-  char plasma_object_exists_error[] = "plasma_object_exists.error";
-  PlasmaObjectExistsError = PyErr_NewException(plasma_object_exists_error, NULL, NULL);
-  Py_INCREF(PlasmaObjectExistsError);
-  PyModule_AddObject(m, "plasma_object_exists_error", PlasmaObjectExistsError);
-  /* Create a custom exception for when the plasma store is out of memory. */
-  char plasma_out_of_memory_error[] = "plasma_out_of_memory.error";
-  PlasmaOutOfMemoryError = PyErr_NewException(plasma_out_of_memory_error, NULL, NULL);
-  Py_INCREF(PlasmaOutOfMemoryError);
-  PyModule_AddObject(m, "plasma_out_of_memory_error", PlasmaOutOfMemoryError);
-
-#if PY_MAJOR_VERSION >= 3
-  return m;
-#endif
-}
diff --git a/cpp/src/plasma/extension.h b/cpp/src/plasma/extension.h
deleted file mode 100644
index cee30abb3592d..0000000000000
--- a/cpp/src/plasma/extension.h
+++ /dev/null
@@ -1,50 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_EXTENSION_H
-#define PLASMA_EXTENSION_H
-
-#undef _XOPEN_SOURCE
-#undef _POSIX_C_SOURCE
-#include <Python.h>
-
-#include "bytesobject.h"  // NOLINT
-
-#include "plasma/client.h"
-#include "plasma/common.h"
-
-static int PyObjectToPlasmaClient(PyObject* object, PlasmaClient** client) {
-  if (PyCapsule_IsValid(object, "plasma")) {
-    *client = reinterpret_cast<PlasmaClient*>(PyCapsule_GetPointer(object, "plasma"));
-    return 1;
-  } else {
-    PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule");
-    return 0;
-  }
-}
-
-int PyStringToUniqueID(PyObject* object, ObjectID* object_id) {
-  if (PyBytes_Check(object)) {
-    memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID));
-    return 1;
-  } else {
-    PyErr_SetString(PyExc_TypeError, "must be a 20 character string");
-    return 0;
-  }
-}
-
-#endif  // PLASMA_EXTENSION_H
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
deleted file mode 100644
index 79da4f43a192a..0000000000000
--- a/cpp/src/plasma/fling.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright 2013 Sharvil Nanavati
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "plasma/fling.h"
-
-#include <string.h>
-
-void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len) {
-  iov->iov_base = buf;
-  iov->iov_len = 1;
-
-  msg->msg_iov = iov;
-  msg->msg_iovlen = 1;
-  msg->msg_control = buf;
-  msg->msg_controllen = buf_len;
-  msg->msg_name = NULL;
-  msg->msg_namelen = 0;
-}
-
-int send_fd(int conn, int fd) {
-  struct msghdr msg;
-  struct iovec iov;
-  char buf[CMSG_SPACE(sizeof(int))];
-  memset(&buf, 0, CMSG_SPACE(sizeof(int)));
-
-  init_msg(&msg, &iov, buf, sizeof(buf));
-
-  struct cmsghdr* header = CMSG_FIRSTHDR(&msg);
-  header->cmsg_level = SOL_SOCKET;
-  header->cmsg_type = SCM_RIGHTS;
-  header->cmsg_len = CMSG_LEN(sizeof(int));
-  *reinterpret_cast<int*>(CMSG_DATA(header)) = fd;
-
-  // Send file descriptor.
-  ssize_t r = sendmsg(conn, &msg, 0);
-  if (r >= 0) {
-    return 0;
-  } else {
-    return static_cast<int>(r);
-  }
-}
-
-int recv_fd(int conn) {
-  struct msghdr msg;
-  struct iovec iov;
-  char buf[CMSG_SPACE(sizeof(int))];
-  init_msg(&msg, &iov, buf, sizeof(buf));
-
-  if (recvmsg(conn, &msg, 0) == -1) return -1;
-
-  int found_fd = -1;
-  int oh_noes = 0;
-  for (struct cmsghdr* header = CMSG_FIRSTHDR(&msg); header != NULL;
-       header = CMSG_NXTHDR(&msg, header))
-    if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) {
-      ssize_t count =
-          (header->cmsg_len - (CMSG_DATA(header) - (unsigned char*)header)) / sizeof(int);
-      for (int i = 0; i < count; ++i) {
-        int fd = (reinterpret_cast<int*>(CMSG_DATA(header)))[i];
-        if (found_fd == -1) {
-          found_fd = fd;
-        } else {
-          close(fd);
-          oh_noes = 1;
-        }
-      }
-    }
-
-  // The sender sent us more than one file descriptor. We've closed
-  // them all to prevent fd leaks but notify the caller that we got
-  // a bad message.
-  if (oh_noes) {
-    close(found_fd);
-    errno = EBADMSG;
-    return -1;
-  }
-
-  return found_fd;
-}
diff --git a/cpp/src/plasma/fling.h b/cpp/src/plasma/fling.h
deleted file mode 100644
index 78ac9d17f26fb..0000000000000
--- a/cpp/src/plasma/fling.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2013 Sharvil Nanavati
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// FLING: Exchanging file descriptors over sockets
-//
-// This is a little library for sending file descriptors over a socket
-// between processes. The reason for doing that (as opposed to using
-// filenames to share the files) is so (a) no files remain in the
-// filesystem after all the processes terminate, (b) to make sure that
-// there are no name collisions and (c) to be able to control who has
-// access to the data.
-//
-// Most of the code is from https://github.com/sharvil/flingfd
-
-#include <errno.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-// This is neccessary for Mac OS X, see http://www.apuebook.com/faqs2e.html
-// (10).
-#if !defined(CMSG_SPACE) && !defined(CMSG_LEN)
-#define CMSG_SPACE(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + __DARWIN_ALIGN32(len))
-#define CMSG_LEN(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + (len))
-#endif
-
-void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len);
-
-// Send a file descriptor over a unix domain socket.
-//
-// @param conn Unix domain socket to send the file descriptor over.
-// @param fd File descriptor to send over.
-// @return Status code which is < 0 on failure.
-int send_fd(int conn, int fd);
-
-// Receive a file descriptor over a unix domain socket.
-//
-// @param conn Unix domain socket to receive the file descriptor from.
-// @return File descriptor or a value < 0 on failure.
-int recv_fd(int conn);
diff --git a/cpp/src/plasma/format/.gitignore b/cpp/src/plasma/format/.gitignore
deleted file mode 100644
index b2ddb055dcbc6..0000000000000
--- a/cpp/src/plasma/format/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*_generated.h
diff --git a/cpp/src/plasma/format/common.fbs b/cpp/src/plasma/format/common.fbs
deleted file mode 100644
index 4d7d2852aec3d..0000000000000
--- a/cpp/src/plasma/format/common.fbs
+++ /dev/null
@@ -1,34 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Object information data structure.
-table ObjectInfo {
-  // Object ID of this object.
-  object_id: string;
-  // Number of bytes the content of this object occupies in memory.
-  data_size: long;
-  // Number of bytes the metadata of this object occupies in memory.
-  metadata_size: long;
-  // Unix epoch of when this object was created.
-  create_time: long;
-  // How long creation of this object took.
-  construct_duration: long;
-  // Hash of the object content.
-  digest: string;
-  // Specifies if this object was deleted or added.
-  is_deletion: bool;
-}
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
deleted file mode 100644
index 23782ade539d4..0000000000000
--- a/cpp/src/plasma/format/plasma.fbs
+++ /dev/null
@@ -1,291 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Plasma protocol specification
-
-enum MessageType:int {
-  // Create a new object.
-  PlasmaCreateRequest = 1,
-  PlasmaCreateReply,
-  // Seal an object.
-  PlasmaSealRequest,
-  PlasmaSealReply,
-  // Get an object that is stored on the local Plasma store.
-  PlasmaGetRequest,
-  PlasmaGetReply,
-  // Release an object.
-  PlasmaReleaseRequest,
-  PlasmaReleaseReply,
-  // Delete an object.
-  PlasmaDeleteRequest,
-  PlasmaDeleteReply,
-  // Get status of an object.
-  PlasmaStatusRequest,
-  PlasmaStatusReply,
-  // See if the store contains an object (will be deprecated).
-  PlasmaContainsRequest,
-  PlasmaContainsReply,
-  // Get information for a newly connecting client.
-  PlasmaConnectRequest,
-  PlasmaConnectReply,
-  // Make room for new objects in the plasma store.
-  PlasmaEvictRequest,
-  PlasmaEvictReply,
-  // Fetch objects from remote Plasma stores.
-  PlasmaFetchRequest,
-  // Wait for objects to be ready either from local or remote Plasma stores.
-  PlasmaWaitRequest,
-  PlasmaWaitReply,
-  // Subscribe to a list of objects or to all objects.
-  PlasmaSubscribeRequest,
-  // Unsubscribe.
-  PlasmaUnsubscribeRequest,
-  // Sending and receiving data.
-  // PlasmaDataRequest initiates sending the data, there will be one
-  // such message per data transfer.
-  PlasmaDataRequest,
-  // PlasmaDataReply contains the actual data and is sent back to the
-  // object store that requested the data. For each transfer, multiple
-  // reply messages get sent. Each one contains a fixed number of bytes.
-  PlasmaDataReply,
-  // Object notifications.
-  PlasmaNotification
-}
-
-enum PlasmaError:int {
-  // Operation was successful.
-  OK,
-  // Trying to create an object that already exists.
-  ObjectExists,
-  // Trying to access an object that doesn't exist.
-  ObjectNonexistent,
-  // Trying to create an object but there isn't enough space in the store.
-  OutOfMemory
-}
-
-// Plasma store messages
-
-struct PlasmaObjectSpec {
-  // Index of the memory segment (= memory mapped file) that
-  // this object is allocated in.
-  segment_index: int;
-  // Size in bytes of this segment (needed to call mmap).
-  mmap_size: ulong;
-  // The offset in bytes in the memory mapped file of the data.
-  data_offset: ulong;
-  // The size in bytes of the data.
-  data_size: ulong;
-  // The offset in bytes in the memory mapped file of the metadata.
-  metadata_offset: ulong;
-  // The size in bytes of the metadata.
-  metadata_size: ulong;
-}
-
-table PlasmaCreateRequest {
-  // ID of the object to be created.
-  object_id: string;
-  // The size of the object's data in bytes.
-  data_size: ulong;
-  // The size of the object's metadata in bytes.
-  metadata_size: ulong;
-}
-
-table PlasmaCreateReply {
-  // ID of the object that was created.
-  object_id: string;
-  // The object that is returned with this reply.
-  plasma_object: PlasmaObjectSpec;
-  // Error that occurred for this call.
-  error: PlasmaError;
-}
-
-table PlasmaSealRequest {
-  // ID of the object to be sealed.
-  object_id: string;
-  // Hash of the object data.
-  digest: string;
-}
-
-table PlasmaSealReply {
-  // ID of the object that was sealed.
-  object_id: string;
-  // Error code.
-  error: PlasmaError;
-}
-
-table PlasmaGetRequest {
-  // IDs of the objects stored at local Plasma store we are getting.
-  object_ids: [string];
-  // The number of milliseconds before the request should timeout.
-  timeout_ms: long;
-}
-
-table PlasmaGetReply {
-  // IDs of the objects being returned.
-  // This number can be smaller than the number of requested
-  // objects if not all requested objects are stored and sealed
-  // in the local Plasma store.
-  object_ids: [string];
-  // Plasma object information, in the same order as their IDs.
-  plasma_objects: [PlasmaObjectSpec];
-  // The number of elements in both object_ids and plasma_objects arrays must agree.
-}
-
-table PlasmaReleaseRequest {
-  // ID of the object to be released.
-  object_id: string;
-}
-
-table PlasmaReleaseReply {
-  // ID of the object that was released.
-  object_id: string;
-  // Error code.
-  error: PlasmaError;
-}
-
-table PlasmaDeleteRequest {
-  // ID of the object to be deleted.
-  object_id: string;
-}
-
-table PlasmaDeleteReply {
-  // ID of the object that was deleted.
-  object_id: string;
-  // Error code.
-  error: PlasmaError;
-}
-
-table PlasmaStatusRequest {
-  // IDs of the objects stored at local Plasma store we request the status of.
-  object_ids: [string];
-}
-
-enum ObjectStatus:int {
-  // Object is stored in the local Plasma Store.
-  Local = 1,
-  // Object is stored on a remote Plasma store, and it is not stored on the
-  // local Plasma Store.
-  Remote,
-  // Object is not stored in the system.
-  Nonexistent,
-  // Object is currently transferred from a remote Plasma store the the local
-  // Plasma Store.
-  Transfer
-}
-
-table PlasmaStatusReply {
-  // IDs of the objects being returned.
-  object_ids: [string];
-  // Status of the object.
-  status: [ObjectStatus];
-}
-
-// PlasmaContains is a subset of PlasmaStatus which does not
-// involve the plasma manager, only the store. We should consider
-// unifying them in the future and deprecating PlasmaContains.
-
-table PlasmaContainsRequest {
-  // ID of the object we are querying.
-  object_id: string;
-}
-
-table PlasmaContainsReply {
-  // ID of the object we are querying.
-  object_id: string;
-  // 1 if the object is in the store and 0 otherwise.
-  has_object: int;
-}
-
-// PlasmaConnect is used by a plasma client the first time it connects with the
-// store. This is not really necessary, but is used to get some information
-// about the store such as its memory capacity.
-
-table PlasmaConnectRequest {
-}
-
-table PlasmaConnectReply {
-  // The memory capacity of the store.
-  memory_capacity: long;
-}
-
-table PlasmaEvictRequest {
-  // Number of bytes that shall be freed.
-  num_bytes: ulong;
-}
-
-table PlasmaEvictReply {
-  // Number of bytes that have been freed.
-  num_bytes: ulong;
-}
-
-table PlasmaFetchRequest {
-  // IDs of objects to be gotten.
-  object_ids: [string];
-}
-
-table ObjectRequestSpec {
-  // ID of the object.
-  object_id: string;
-  // The type of the object. This specifies whether we
-  // will be waiting for an object store in the local or
-  // global Plasma store.
-  type: int;
-}
-
-table PlasmaWaitRequest {
-  // Array of object requests whose status we are asking for.
-  object_requests: [ObjectRequestSpec];
-  // Number of objects expected to be returned, if available.
-  num_ready_objects: int;
-  // timeout
-  timeout: long;
-}
-
-table ObjectReply {
-  // ID of the object.
-  object_id: string;
-  // The object status. This specifies where the object is stored.
-  status: int;
-}
-
-table PlasmaWaitReply {
-  // Array of object requests being returned.
-  object_requests: [ObjectReply];
-  // Number of objects expected to be returned, if available.
-  num_ready_objects: int;
-}
-
-table PlasmaSubscribeRequest {
-}
-
-table PlasmaDataRequest {
-  // ID of the object that is requested.
-  object_id: string;
-  // The host address where the data shall be sent to.
-  address: string;
-  // The port of the manager the data shall be sent to.
-  port: int;
-}
-
-table PlasmaDataReply {
-  // ID of the object that will be sent.
-  object_id: string;
-  // Size of the object data in bytes.
-  object_size: ulong;
-  // Size of the metadata in bytes.
-  metadata_size: ulong;
-}
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
deleted file mode 100644
index 5875ebb7ae611..0000000000000
--- a/cpp/src/plasma/io.cc
+++ /dev/null
@@ -1,212 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/io.h"
-
-#include "plasma/common.h"
-
-using arrow::Status;
-
-/* Number of times we try binding to a socket. */
-#define NUM_BIND_ATTEMPTS 5
-#define BIND_TIMEOUT_MS 100
-
-/* Number of times we try connecting to a socket. */
-#define NUM_CONNECT_ATTEMPTS 50
-#define CONNECT_TIMEOUT_MS 100
-
-Status WriteBytes(int fd, uint8_t* cursor, size_t length) {
-  ssize_t nbytes = 0;
-  size_t bytesleft = length;
-  size_t offset = 0;
-  while (bytesleft > 0) {
-    /* While we haven't written the whole message, write to the file descriptor,
-     * advance the cursor, and decrease the amount left to write. */
-    nbytes = write(fd, cursor + offset, bytesleft);
-    if (nbytes < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
-      return Status::IOError(std::string(strerror(errno)));
-    } else if (nbytes == 0) {
-      return Status::IOError("Encountered unexpected EOF");
-    }
-    ARROW_CHECK(nbytes > 0);
-    bytesleft -= nbytes;
-    offset += nbytes;
-  }
-
-  return Status::OK();
-}
-
-Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes) {
-  int64_t version = PLASMA_PROTOCOL_VERSION;
-  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)));
-  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&type), sizeof(type)));
-  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)));
-  return WriteBytes(fd, bytes, length * sizeof(char));
-}
-
-Status ReadBytes(int fd, uint8_t* cursor, size_t length) {
-  ssize_t nbytes = 0;
-  /* Termination condition: EOF or read 'length' bytes total. */
-  size_t bytesleft = length;
-  size_t offset = 0;
-  while (bytesleft > 0) {
-    nbytes = read(fd, cursor + offset, bytesleft);
-    if (nbytes < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
-      return Status::IOError(std::string(strerror(errno)));
-    } else if (0 == nbytes) {
-      return Status::IOError("Encountered unexpected EOF");
-    }
-    ARROW_CHECK(nbytes > 0);
-    bytesleft -= nbytes;
-    offset += nbytes;
-  }
-
-  return Status::OK();
-}
-
-Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
-  int64_t version;
-  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)),
-      *type = DISCONNECT_CLIENT);
-  ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version;
-  size_t length;
-  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(type), sizeof(*type)),
-      *type = DISCONNECT_CLIENT);
-  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)),
-      *type = DISCONNECT_CLIENT);
-  if (length > buffer->size()) { buffer->resize(length); }
-  RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer->data(), length), *type = DISCONNECT_CLIENT);
-  return Status::OK();
-}
-
-int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
-  struct sockaddr_un socket_address;
-  int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
-  if (socket_fd < 0) {
-    ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
-    return -1;
-  }
-  /* Tell the system to allow the port to be reused. */
-  int on = 1;
-  if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&on),
-          sizeof(on)) < 0) {
-    ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname;
-    close(socket_fd);
-    return -1;
-  }
-
-  unlink(pathname.c_str());
-  memset(&socket_address, 0, sizeof(socket_address));
-  socket_address.sun_family = AF_UNIX;
-  if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
-    ARROW_LOG(ERROR) << "Socket pathname is too long.";
-    close(socket_fd);
-    return -1;
-  }
-  strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
-
-  if (bind(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) != 0) {
-    ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname;
-    close(socket_fd);
-    return -1;
-  }
-  if (shall_listen && listen(socket_fd, 128) == -1) {
-    ARROW_LOG(ERROR) << "Could not listen to socket " << pathname;
-    close(socket_fd);
-    return -1;
-  }
-  return socket_fd;
-}
-
-int connect_ipc_sock_retry(
-    const std::string& pathname, int num_retries, int64_t timeout) {
-  /* Pick the default values if the user did not specify. */
-  if (num_retries < 0) { num_retries = NUM_CONNECT_ATTEMPTS; }
-  if (timeout < 0) { timeout = CONNECT_TIMEOUT_MS; }
-
-  int fd = -1;
-  for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
-    fd = connect_ipc_sock(pathname);
-    if (fd >= 0) { break; }
-    if (num_attempts == 0) {
-      ARROW_LOG(ERROR) << "Connection to socket failed for pathname " << pathname;
-    }
-    /* Sleep for timeout milliseconds. */
-    usleep(static_cast<int>(timeout * 1000));
-  }
-  /* If we could not connect to the socket, exit. */
-  if (fd == -1) { ARROW_LOG(FATAL) << "Could not connect to socket " << pathname; }
-  return fd;
-}
-
-int connect_ipc_sock(const std::string& pathname) {
-  struct sockaddr_un socket_address;
-  int socket_fd;
-
-  socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
-  if (socket_fd < 0) {
-    ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
-    return -1;
-  }
-
-  memset(&socket_address, 0, sizeof(socket_address));
-  socket_address.sun_family = AF_UNIX;
-  if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
-    ARROW_LOG(ERROR) << "Socket pathname is too long.";
-    return -1;
-  }
-  strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
-
-  if (connect(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) !=
-      0) {
-    close(socket_fd);
-    return -1;
-  }
-
-  return socket_fd;
-}
-
-int AcceptClient(int socket_fd) {
-  int client_fd = accept(socket_fd, NULL, NULL);
-  if (client_fd < 0) {
-    ARROW_LOG(ERROR) << "Error reading from socket.";
-    return -1;
-  }
-  return client_fd;
-}
-
-uint8_t* read_message_async(int sock) {
-  int64_t size;
-  Status s = ReadBytes(sock, reinterpret_cast<uint8_t*>(&size), sizeof(int64_t));
-  if (!s.ok()) {
-    /* The other side has closed the socket. */
-    ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
-    close(sock);
-    return NULL;
-  }
-  uint8_t* message = reinterpret_cast<uint8_t*>(malloc(size));
-  s = ReadBytes(sock, message, size);
-  if (!s.ok()) {
-    /* The other side has closed the socket. */
-    ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
-    close(sock);
-    return NULL;
-  }
-  return message;
-}
diff --git a/cpp/src/plasma/io.h b/cpp/src/plasma/io.h
deleted file mode 100644
index 43c3fb535497f..0000000000000
--- a/cpp/src/plasma/io.h
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_IO_H
-#define PLASMA_IO_H
-
-#include <inttypes.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include <string>
-#include <vector>
-
-#include "arrow/status.h"
-
-// TODO(pcm): Replace our own custom message header (message type,
-// message length, plasma protocol verion) with one that is serialized
-// using flatbuffers.
-#define PLASMA_PROTOCOL_VERSION 0x0000000000000000
-#define DISCONNECT_CLIENT 0
-
-arrow::Status WriteBytes(int fd, uint8_t* cursor, size_t length);
-
-arrow::Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes);
-
-arrow::Status ReadBytes(int fd, uint8_t* cursor, size_t length);
-
-arrow::Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer);
-
-int bind_ipc_sock(const std::string& pathname, bool shall_listen);
-
-int connect_ipc_sock(const std::string& pathname);
-
-int connect_ipc_sock_retry(const std::string& pathname, int num_retries, int64_t timeout);
-
-int AcceptClient(int socket_fd);
-
-uint8_t* read_message_async(int sock);
-
-#endif  // PLASMA_IO_H
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
deleted file mode 100644
index 97c9a16c0c0bd..0000000000000
--- a/cpp/src/plasma/malloc.cc
+++ /dev/null
@@ -1,178 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/malloc.h"
-
-#include <assert.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <unistd.h>
-
-#include <unordered_map>
-
-#include "plasma/common.h"
-
-extern "C" {
-void* fake_mmap(size_t);
-int fake_munmap(void*, int64_t);
-
-#define MMAP(s) fake_mmap(s)
-#define MUNMAP(a, s) fake_munmap(a, s)
-#define DIRECT_MMAP(s) fake_mmap(s)
-#define DIRECT_MUNMAP(a, s) fake_munmap(a, s)
-#define USE_DL_PREFIX
-#define HAVE_MORECORE 0
-#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
-#define DEFAULT_GRANULARITY ((size_t)128U * 1024U)
-
-#include "thirdparty/dlmalloc.c"  // NOLINT
-
-#undef MMAP
-#undef MUNMAP
-#undef DIRECT_MMAP
-#undef DIRECT_MUNMAP
-#undef USE_DL_PREFIX
-#undef HAVE_MORECORE
-#undef DEFAULT_GRANULARITY
-}
-
-struct mmap_record {
-  int fd;
-  int64_t size;
-};
-
-namespace {
-
-/** Hashtable that contains one entry per segment that we got from the OS
- *  via mmap. Associates the address of that segment with its file descriptor
- *  and size. */
-std::unordered_map<void*, mmap_record> mmap_records;
-
-} /* namespace */
-
-constexpr int GRANULARITY_MULTIPLIER = 2;
-
-static void* pointer_advance(void* p, ptrdiff_t n) {
-  return (unsigned char*)p + n;
-}
-
-static void* pointer_retreat(void* p, ptrdiff_t n) {
-  return (unsigned char*)p - n;
-}
-
-static ptrdiff_t pointer_distance(void const* pfrom, void const* pto) {
-  return (unsigned char const*)pto - (unsigned char const*)pfrom;
-}
-
-/* Create a buffer. This is creating a temporary file and then
- * immediately unlinking it so we do not leave traces in the system. */
-int create_buffer(int64_t size) {
-  int fd;
-#ifdef _WIN32
-  if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
-          (DWORD)((uint64_t)size >> (CHAR_BIT * sizeof(DWORD))), (DWORD)(uint64_t)size,
-          NULL)) {
-    fd = -1;
-  }
-#else
-#ifdef __linux__
-  constexpr char file_template[] = "/dev/shm/plasmaXXXXXX";
-#else
-  constexpr char file_template[] = "/tmp/plasmaXXXXXX";
-#endif
-  char file_name[32];
-  strncpy(file_name, file_template, 32);
-  fd = mkstemp(file_name);
-  if (fd < 0) return -1;
-  FILE* file = fdopen(fd, "a+");
-  if (!file) {
-    close(fd);
-    return -1;
-  }
-  if (unlink(file_name) != 0) {
-    ARROW_LOG(FATAL) << "unlink error";
-    return -1;
-  }
-  if (ftruncate(fd, (off_t)size) != 0) {
-    ARROW_LOG(FATAL) << "ftruncate error";
-    return -1;
-  }
-#endif
-  return fd;
-}
-
-void* fake_mmap(size_t size) {
-  /* Add sizeof(size_t) so that the returned pointer is deliberately not
-   * page-aligned. This ensures that the segments of memory returned by
-   * fake_mmap are never contiguous. */
-  size += sizeof(size_t);
-
-  int fd = create_buffer(size);
-  ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
-  void* pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-  if (pointer == MAP_FAILED) { return pointer; }
-
-  /* Increase dlmalloc's allocation granularity directly. */
-  mparams.granularity *= GRANULARITY_MULTIPLIER;
-
-  mmap_record& record = mmap_records[pointer];
-  record.fd = fd;
-  record.size = size;
-
-  /* We lie to dlmalloc about where mapped memory actually lives. */
-  pointer = pointer_advance(pointer, sizeof(size_t));
-  ARROW_LOG(DEBUG) << pointer << " = fake_mmap(" << size << ")";
-  return pointer;
-}
-
-int fake_munmap(void* addr, int64_t size) {
-  ARROW_LOG(DEBUG) << "fake_munmap(" << addr << ", " << size << ")";
-  addr = pointer_retreat(addr, sizeof(size_t));
-  size += sizeof(size_t);
-
-  auto entry = mmap_records.find(addr);
-
-  if (entry == mmap_records.end() || entry->second.size != size) {
-    /* Reject requests to munmap that don't directly match previous
-     * calls to mmap, to prevent dlmalloc from trimming. */
-    return -1;
-  }
-
-  int r = munmap(addr, size);
-  if (r == 0) { close(entry->second.fd); }
-
-  mmap_records.erase(entry);
-  return r;
-}
-
-void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_size, ptrdiff_t* offset) {
-  /* TODO(rshin): Implement a more efficient search through mmap_records. */
-  for (const auto& entry : mmap_records) {
-    if (addr >= entry.first && addr < pointer_advance(entry.first, entry.second.size)) {
-      *fd = entry.second.fd;
-      *map_size = entry.second.size;
-      *offset = pointer_distance(entry.first, addr);
-      return;
-    }
-  }
-  *fd = -1;
-  *map_size = 0;
-  *offset = 0;
-}
diff --git a/cpp/src/plasma/malloc.h b/cpp/src/plasma/malloc.h
deleted file mode 100644
index b4af2c826b5c9..0000000000000
--- a/cpp/src/plasma/malloc.h
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_MALLOC_H
-#define PLASMA_MALLOC_H
-
-#include <inttypes.h>
-#include <stddef.h>
-
-void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_length, ptrdiff_t* offset);
-
-#endif  // MALLOC_H
diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc
deleted file mode 100644
index 559d8e7f2a65e..0000000000000
--- a/cpp/src/plasma/plasma.cc
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/plasma.h"
-
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "plasma/common.h"
-#include "plasma/protocol.h"
-
-int warn_if_sigpipe(int status, int client_sock) {
-  if (status >= 0) { return 0; }
-  if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
-    ARROW_LOG(WARNING) << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when "
-                          "sending a message to client on fd "
-                       << client_sock << ". The client on the other end may "
-                                         "have hung up.";
-    return errno;
-  }
-  ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock << ".";
-  return -1;  // This is never reached.
-}
-
-/**
- * This will create a new ObjectInfo buffer. The first sizeof(int64_t) bytes
- * of this buffer are the length of the remaining message and the
- * remaining message is a serialized version of the object info.
- *
- * @param object_info The object info to be serialized
- * @return The object info buffer. It is the caller's responsibility to free
- *         this buffer with "delete" after it has been used.
- */
-uint8_t* create_object_info_buffer(ObjectInfoT* object_info) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreateObjectInfo(fbb, object_info);
-  fbb.Finish(message);
-  uint8_t* notification = new uint8_t[sizeof(int64_t) + fbb.GetSize()];
-  *(reinterpret_cast<int64_t*>(notification)) = fbb.GetSize();
-  memcpy(notification + sizeof(int64_t), fbb.GetBufferPointer(), fbb.GetSize());
-  return notification;
-}
-
-ObjectTableEntry* get_object_table_entry(
-    PlasmaStoreInfo* store_info, const ObjectID& object_id) {
-  auto it = store_info->objects.find(object_id);
-  if (it == store_info->objects.end()) { return NULL; }
-  return it->second.get();
-}
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
deleted file mode 100644
index 275d0c7a41687..0000000000000
--- a/cpp/src/plasma/plasma.h
+++ /dev/null
@@ -1,191 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_PLASMA_H
-#define PLASMA_PLASMA_H
-
-#include <errno.h>
-#include <inttypes.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>  // pid_t
-
-#include <unordered_map>
-#include <unordered_set>
-
-#include "arrow/status.h"
-#include "arrow/util/logging.h"
-#include "format/common_generated.h"
-#include "plasma/common.h"
-
-#define HANDLE_SIGPIPE(s, fd_)                                              \
-  do {                                                                      \
-    Status _s = (s);                                                        \
-    if (!_s.ok()) {                                                         \
-      if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {        \
-        ARROW_LOG(WARNING)                                                  \
-            << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \
-               "sending a message to client on fd "                         \
-            << fd_ << ". "                                                  \
-                      "The client on the other end may have hung up.";      \
-      } else {                                                              \
-        return _s;                                                          \
-      }                                                                     \
-    }                                                                       \
-  } while (0);
-
-/// Allocation granularity used in plasma for object allocation.
-#define BLOCK_SIZE 64
-
-/// Size of object hash digests.
-constexpr int64_t kDigestSize = sizeof(uint64_t);
-
-struct Client;
-
-/// Object request data structure. Used in the plasma_wait_for_objects()
-/// argument.
-typedef struct {
-  /// The ID of the requested object. If ID_NIL request any object.
-  ObjectID object_id;
-  /// Request associated to the object. It can take one of the following values:
-  ///  - PLASMA_QUERY_LOCAL: return if or when the object is available in the
-  ///    local Plasma Store.
-  ///  - PLASMA_QUERY_ANYWHERE: return if or when the object is available in
-  ///    the system (i.e., either in the local or a remote Plasma Store).
-  int type;
-  /// Object status. Same as the status returned by plasma_status() function
-  /// call. This is filled in by plasma_wait_for_objects1():
-  ///  - ObjectStatus_Local: object is ready at the local Plasma Store.
-  ///  - ObjectStatus_Remote: object is ready at a remote Plasma Store.
-  ///  - ObjectStatus_Nonexistent: object does not exist in the system.
-  ///  - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
-  ///    for being transferred or it is transferring.
-  int status;
-} ObjectRequest;
-
-/// Mapping from object IDs to type and status of the request.
-typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectRequestMap;
-
-/// Handle to access memory mapped file and map it into client address space.
-typedef struct {
-  /// The file descriptor of the memory mapped file in the store. It is used as
-  /// a unique identifier of the file in the client to look up the corresponding
-  /// file descriptor on the client's side.
-  int store_fd;
-  /// The size in bytes of the memory mapped file.
-  int64_t mmap_size;
-} object_handle;
-
-// TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
-typedef struct {
-  /// Handle for memory mapped file the object is stored in.
-  object_handle handle;
-  /// The offset in bytes in the memory mapped file of the data.
-  ptrdiff_t data_offset;
-  /// The offset in bytes in the memory mapped file of the metadata.
-  ptrdiff_t metadata_offset;
-  /// The size in bytes of the data.
-  int64_t data_size;
-  /// The size in bytes of the metadata.
-  int64_t metadata_size;
-} PlasmaObject;
-
-typedef enum {
-  /// Object was created but not sealed in the local Plasma Store.
-  PLASMA_CREATED = 1,
-  /// Object is sealed and stored in the local Plasma Store.
-  PLASMA_SEALED
-} object_state;
-
-typedef enum {
-  /// The object was not found.
-  OBJECT_NOT_FOUND = 0,
-  /// The object was found.
-  OBJECT_FOUND = 1
-} object_status;
-
-typedef enum {
-  /// Query for object in the local plasma store.
-  PLASMA_QUERY_LOCAL = 1,
-  /// Query for object in the local plasma store or in a remote plasma store.
-  PLASMA_QUERY_ANYWHERE
-} object_request_type;
-
-/// This type is used by the Plasma store. It is here because it is exposed to
-/// the eviction policy.
-struct ObjectTableEntry {
-  /// Object id of this object.
-  ObjectID object_id;
-  /// Object info like size, creation time and owner.
-  ObjectInfoT info;
-  /// Memory mapped file containing the object.
-  int fd;
-  /// Size of the underlying map.
-  int64_t map_size;
-  /// Offset from the base of the mmap.
-  ptrdiff_t offset;
-  /// Pointer to the object data. Needed to free the object.
-  uint8_t* pointer;
-  /// Set of clients currently using this object.
-  std::unordered_set<Client*> clients;
-  /// The state of the object, e.g., whether it is open or sealed.
-  object_state state;
-  /// The digest of the object. Used to see if two objects are the same.
-  unsigned char digest[kDigestSize];
-};
-
-/// The plasma store information that is exposed to the eviction policy.
-struct PlasmaStoreInfo {
-  /// Objects that are in the Plasma store.
-  std::unordered_map<ObjectID, std::unique_ptr<ObjectTableEntry>, UniqueIDHasher> objects;
-  /// The amount of memory (in bytes) that we allow to be allocated in the
-  /// store.
-  int64_t memory_capacity;
-};
-
-/// Get an entry from the object table and return NULL if the object_id
-/// is not present.
-///
-/// @param store_info The PlasmaStoreInfo that contains the object table.
-/// @param object_id The object_id of the entry we are looking for.
-/// @return The entry associated with the object_id or NULL if the object_id
-///         is not present.
-ObjectTableEntry* get_object_table_entry(
-    PlasmaStoreInfo* store_info, const ObjectID& object_id);
-
-/// Print a warning if the status is less than zero. This should be used to check
-/// the success of messages sent to plasma clients. We print a warning instead of
-/// failing because the plasma clients are allowed to die. This is used to handle
-/// situations where the store writes to a client file descriptor, and the client
-/// may already have disconnected. If we have processed the disconnection and
-/// closed the file descriptor, we should get a BAD FILE DESCRIPTOR error. If we
-/// have not, then we should get a SIGPIPE. If we write to a TCP socket that
-/// isn't connected yet, then we should get an ECONNRESET.
-///
-/// @param status The status to check. If it is less less than zero, we will
-///        print a warning.
-/// @param client_sock The client socket. This is just used to print some extra
-///        information.
-/// @return The errno set.
-int warn_if_sigpipe(int status, int client_sock);
-
-uint8_t* create_object_info_buffer(ObjectInfoT* object_info);
-
-#endif  // PLASMA_PLASMA_H
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
deleted file mode 100644
index 246aa29736056..0000000000000
--- a/cpp/src/plasma/protocol.cc
+++ /dev/null
@@ -1,502 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/protocol.h"
-
-#include "flatbuffers/flatbuffers.h"
-#include "format/plasma_generated.h"
-
-#include "plasma/common.h"
-#include "plasma/io.h"
-
-using flatbuffers::uoffset_t;
-
-flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
-to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
-    int64_t num_objects) {
-  std::vector<flatbuffers::Offset<flatbuffers::String>> results;
-  for (int64_t i = 0; i < num_objects; i++) {
-    results.push_back(fbb->CreateString(object_ids[i].binary()));
-  }
-  return fbb->CreateVector(results);
-}
-
-Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer) {
-  int64_t type;
-  RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
-  ARROW_CHECK(type == message_type) << "type = " << type
-                                    << ", message_type = " << message_type;
-  return Status::OK();
-}
-
-template <typename Message>
-Status PlasmaSend(int sock, int64_t message_type, flatbuffers::FlatBufferBuilder* fbb,
-    const Message& message) {
-  fbb->Finish(message);
-  return WriteMessage(sock, message_type, fbb->GetSize(), fbb->GetBufferPointer());
-}
-
-// Create messages.
-
-Status SendCreateRequest(
-    int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaCreateRequest(
-      fbb, fbb.CreateString(object_id.binary()), data_size, metadata_size);
-  return PlasmaSend(sock, MessageType_PlasmaCreateRequest, &fbb, message);
-}
-
-Status ReadCreateRequest(
-    uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
-  *data_size = message->data_size();
-  *metadata_size = message->metadata_size();
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  return Status::OK();
-}
-
-Status SendCreateReply(
-    int sock, ObjectID object_id, PlasmaObject* object, int error_code) {
-  flatbuffers::FlatBufferBuilder fbb;
-  PlasmaObjectSpec plasma_object(object->handle.store_fd, object->handle.mmap_size,
-      object->data_offset, object->data_size, object->metadata_offset,
-      object->metadata_size);
-  auto message = CreatePlasmaCreateReply(
-      fbb, fbb.CreateString(object_id.binary()), &plasma_object, (PlasmaError)error_code);
-  return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
-}
-
-Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  object->handle.store_fd = message->plasma_object()->segment_index();
-  object->handle.mmap_size = message->plasma_object()->mmap_size();
-  object->data_offset = message->plasma_object()->data_offset();
-  object->data_size = message->plasma_object()->data_size();
-  object->metadata_offset = message->plasma_object()->metadata_offset();
-  object->metadata_size = message->plasma_object()->metadata_size();
-  return plasma_error_status(message->error());
-}
-
-// Seal messages.
-
-Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto digest_string = fbb.CreateString(reinterpret_cast<char*>(digest), kDigestSize);
-  auto message =
-      CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()), digest_string);
-  return PlasmaSend(sock, MessageType_PlasmaSealRequest, &fbb, message);
-}
-
-Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaSealRequest>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  ARROW_CHECK(message->digest()->size() == kDigestSize);
-  memcpy(digest, message->digest()->data(), kDigestSize);
-  return Status::OK();
-}
-
-Status SendSealReply(int sock, ObjectID object_id, int error) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaSealReply(
-      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
-  return PlasmaSend(sock, MessageType_PlasmaSealReply, &fbb, message);
-}
-
-Status ReadSealReply(uint8_t* data, ObjectID* object_id) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaSealReply>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  return plasma_error_status(message->error());
-}
-
-// Release messages.
-
-Status SendReleaseRequest(int sock, ObjectID object_id) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()));
-  return PlasmaSend(sock, MessageType_PlasmaReleaseRequest, &fbb, message);
-}
-
-Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaReleaseRequest>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  return Status::OK();
-}
-
-Status SendReleaseReply(int sock, ObjectID object_id, int error) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaReleaseReply(
-      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
-  return PlasmaSend(sock, MessageType_PlasmaReleaseReply, &fbb, message);
-}
-
-Status ReadReleaseReply(uint8_t* data, ObjectID* object_id) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  return plasma_error_status(message->error());
-}
-
-// Delete messages.
-
-Status SendDeleteRequest(int sock, ObjectID object_id) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaDeleteRequest(fbb, fbb.CreateString(object_id.binary()));
-  return PlasmaSend(sock, MessageType_PlasmaDeleteRequest, &fbb, message);
-}
-
-Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  return Status::OK();
-}
-
-Status SendDeleteReply(int sock, ObjectID object_id, int error) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaDeleteReply(
-      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
-  return PlasmaSend(sock, MessageType_PlasmaDeleteReply, &fbb, message);
-}
-
-Status ReadDeleteReply(uint8_t* data, ObjectID* object_id) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaDeleteReply>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  return plasma_error_status(message->error());
-}
-
-// Satus messages.
-
-Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message =
-      CreatePlasmaStatusRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
-  return PlasmaSend(sock, MessageType_PlasmaStatusRequest, &fbb, message);
-}
-
-Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaStatusRequest>(data);
-  for (uoffset_t i = 0; i < num_objects; ++i) {
-    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
-  }
-  return Status::OK();
-}
-
-Status SendStatusReply(
-    int sock, ObjectID object_ids[], int object_status[], int64_t num_objects) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message =
-      CreatePlasmaStatusReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
-          fbb.CreateVector(object_status, num_objects));
-  return PlasmaSend(sock, MessageType_PlasmaStatusReply, &fbb, message);
-}
-
-int64_t ReadStatusReply_num_objects(uint8_t* data) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
-  return message->object_ids()->size();
-}
-
-Status ReadStatusReply(
-    uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
-  for (uoffset_t i = 0; i < num_objects; ++i) {
-    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
-  }
-  for (uoffset_t i = 0; i < num_objects; ++i) {
-    object_status[i] = message->status()->data()[i];
-  }
-  return Status::OK();
-}
-
-// Contains messages.
-
-Status SendContainsRequest(int sock, ObjectID object_id) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaContainsRequest(fbb, fbb.CreateString(object_id.binary()));
-  return PlasmaSend(sock, MessageType_PlasmaContainsRequest, &fbb, message);
-}
-
-Status ReadContainsRequest(uint8_t* data, ObjectID* object_id) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaContainsRequest>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  return Status::OK();
-}
-
-Status SendContainsReply(int sock, ObjectID object_id, bool has_object) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message =
-      CreatePlasmaContainsReply(fbb, fbb.CreateString(object_id.binary()), has_object);
-  return PlasmaSend(sock, MessageType_PlasmaContainsReply, &fbb, message);
-}
-
-Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaContainsReply>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  *has_object = message->has_object();
-  return Status::OK();
-}
-
-// Connect messages.
-
-Status SendConnectRequest(int sock) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaConnectRequest(fbb);
-  return PlasmaSend(sock, MessageType_PlasmaConnectRequest, &fbb, message);
-}
-
-Status ReadConnectRequest(uint8_t* data) {
-  return Status::OK();
-}
-
-Status SendConnectReply(int sock, int64_t memory_capacity) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaConnectReply(fbb, memory_capacity);
-  return PlasmaSend(sock, MessageType_PlasmaConnectReply, &fbb, message);
-}
-
-Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaConnectReply>(data);
-  *memory_capacity = message->memory_capacity();
-  return Status::OK();
-}
-
-// Evict messages.
-
-Status SendEvictRequest(int sock, int64_t num_bytes) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaEvictRequest(fbb, num_bytes);
-  return PlasmaSend(sock, MessageType_PlasmaEvictRequest, &fbb, message);
-}
-
-Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaEvictRequest>(data);
-  *num_bytes = message->num_bytes();
-  return Status::OK();
-}
-
-Status SendEvictReply(int sock, int64_t num_bytes) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaEvictReply(fbb, num_bytes);
-  return PlasmaSend(sock, MessageType_PlasmaEvictReply, &fbb, message);
-}
-
-Status ReadEvictReply(uint8_t* data, int64_t& num_bytes) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaEvictReply>(data);
-  num_bytes = message->num_bytes();
-  return Status::OK();
-}
-
-// Get messages.
-
-Status SendGetRequest(
-    int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaGetRequest(
-      fbb, to_flatbuffer(&fbb, object_ids, num_objects), timeout_ms);
-  return PlasmaSend(sock, MessageType_PlasmaGetRequest, &fbb, message);
-}
-
-Status ReadGetRequest(
-    uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
-  for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
-    auto object_id = message->object_ids()->Get(i)->str();
-    object_ids.push_back(ObjectID::from_binary(object_id));
-  }
-  *timeout_ms = message->timeout_ms();
-  return Status::OK();
-}
-
-Status SendGetReply(int sock, ObjectID object_ids[],
-    std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
-    int64_t num_objects) {
-  flatbuffers::FlatBufferBuilder fbb;
-  std::vector<PlasmaObjectSpec> objects;
-
-  for (int i = 0; i < num_objects; ++i) {
-    const PlasmaObject& object = plasma_objects[object_ids[i]];
-    objects.push_back(PlasmaObjectSpec(object.handle.store_fd, object.handle.mmap_size,
-        object.data_offset, object.data_size, object.metadata_offset,
-        object.metadata_size));
-  }
-  auto message = CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
-      fbb.CreateVectorOfStructs(objects.data(), num_objects));
-  return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
-}
-
-Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
-    int64_t num_objects) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
-  for (uoffset_t i = 0; i < num_objects; ++i) {
-    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
-  }
-  for (uoffset_t i = 0; i < num_objects; ++i) {
-    const PlasmaObjectSpec* object = message->plasma_objects()->Get(i);
-    plasma_objects[i].handle.store_fd = object->segment_index();
-    plasma_objects[i].handle.mmap_size = object->mmap_size();
-    plasma_objects[i].data_offset = object->data_offset();
-    plasma_objects[i].data_size = object->data_size();
-    plasma_objects[i].metadata_offset = object->metadata_offset();
-    plasma_objects[i].metadata_size = object->metadata_size();
-  }
-  return Status::OK();
-}
-
-// Fetch messages.
-
-Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message =
-      CreatePlasmaFetchRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
-  return PlasmaSend(sock, MessageType_PlasmaFetchRequest, &fbb, message);
-}
-
-Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaFetchRequest>(data);
-  for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
-    object_ids.push_back(ObjectID::from_binary(message->object_ids()->Get(i)->str()));
-  }
-  return Status::OK();
-}
-
-// Wait messages.
-
-Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
-    int num_ready_objects, int64_t timeout_ms) {
-  flatbuffers::FlatBufferBuilder fbb;
-
-  std::vector<flatbuffers::Offset<ObjectRequestSpec>> object_request_specs;
-  for (int i = 0; i < num_requests; i++) {
-    object_request_specs.push_back(CreateObjectRequestSpec(fbb,
-        fbb.CreateString(object_requests[i].object_id.binary()),
-        object_requests[i].type));
-  }
-
-  auto message = CreatePlasmaWaitRequest(
-      fbb, fbb.CreateVector(object_request_specs), num_ready_objects, timeout_ms);
-  return PlasmaSend(sock, MessageType_PlasmaWaitRequest, &fbb, message);
-}
-
-Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
-    int64_t* timeout_ms, int* num_ready_objects) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaWaitRequest>(data);
-  *num_ready_objects = message->num_ready_objects();
-  *timeout_ms = message->timeout();
-
-  for (uoffset_t i = 0; i < message->object_requests()->size(); i++) {
-    ObjectID object_id =
-        ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
-    ObjectRequest object_request({object_id, message->object_requests()->Get(i)->type(),
-        ObjectStatus_Nonexistent});
-    object_requests[object_id] = object_request;
-  }
-  return Status::OK();
-}
-
-Status SendWaitReply(
-    int sock, const ObjectRequestMap& object_requests, int num_ready_objects) {
-  flatbuffers::FlatBufferBuilder fbb;
-
-  std::vector<flatbuffers::Offset<ObjectReply>> object_replies;
-  for (const auto& entry : object_requests) {
-    const auto& object_request = entry.second;
-    object_replies.push_back(CreateObjectReply(
-        fbb, fbb.CreateString(object_request.object_id.binary()), object_request.status));
-  }
-
-  auto message = CreatePlasmaWaitReply(
-      fbb, fbb.CreateVector(object_replies.data(), num_ready_objects), num_ready_objects);
-  return PlasmaSend(sock, MessageType_PlasmaWaitReply, &fbb, message);
-}
-
-Status ReadWaitReply(
-    uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects) {
-  DCHECK(data);
-
-  auto message = flatbuffers::GetRoot<PlasmaWaitReply>(data);
-  *num_ready_objects = message->num_ready_objects();
-  for (int i = 0; i < *num_ready_objects; i++) {
-    object_requests[i].object_id =
-        ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
-    object_requests[i].status = message->object_requests()->Get(i)->status();
-  }
-  return Status::OK();
-}
-
-// Subscribe messages.
-
-Status SendSubscribeRequest(int sock) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaSubscribeRequest(fbb);
-  return PlasmaSend(sock, MessageType_PlasmaSubscribeRequest, &fbb, message);
-}
-
-// Data messages.
-
-Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto addr = fbb.CreateString(address, strlen(address));
-  auto message =
-      CreatePlasmaDataRequest(fbb, fbb.CreateString(object_id.binary()), addr, port);
-  return PlasmaSend(sock, MessageType_PlasmaDataRequest, &fbb, message);
-}
-
-Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaDataRequest>(data);
-  DCHECK(message->object_id()->size() == sizeof(ObjectID));
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  *address = strdup(message->address()->c_str());
-  *port = message->port();
-  return Status::OK();
-}
-
-Status SendDataReply(
-    int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaDataReply(
-      fbb, fbb.CreateString(object_id.binary()), object_size, metadata_size);
-  return PlasmaSend(sock, MessageType_PlasmaDataReply, &fbb, message);
-}
-
-Status ReadDataReply(
-    uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size) {
-  DCHECK(data);
-  auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
-  *object_id = ObjectID::from_binary(message->object_id()->str());
-  *object_size = (int64_t)message->object_size();
-  *metadata_size = (int64_t)message->metadata_size();
-  return Status::OK();
-}
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
deleted file mode 100644
index 5d9d13675144f..0000000000000
--- a/cpp/src/plasma/protocol.h
+++ /dev/null
@@ -1,170 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_PROTOCOL_H
-#define PLASMA_PROTOCOL_H
-
-#include <vector>
-
-#include "arrow/status.h"
-#include "format/plasma_generated.h"
-#include "plasma/plasma.h"
-
-using arrow::Status;
-
-/* Plasma receive message. */
-
-Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer);
-
-/* Plasma Create message functions. */
-
-Status SendCreateRequest(
-    int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size);
-
-Status ReadCreateRequest(
-    uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size);
-
-Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error);
-
-Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object);
-
-/* Plasma Seal message functions. */
-
-Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest);
-
-Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest);
-
-Status SendSealReply(int sock, ObjectID object_id, int error);
-
-Status ReadSealReply(uint8_t* data, ObjectID* object_id);
-
-/* Plasma Get message functions. */
-
-Status SendGetRequest(
-    int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms);
-
-Status ReadGetRequest(
-    uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms);
-
-Status SendGetReply(int sock, ObjectID object_ids[],
-    std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
-    int64_t num_objects);
-
-Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
-    int64_t num_objects);
-
-/* Plasma Release message functions. */
-
-Status SendReleaseRequest(int sock, ObjectID object_id);
-
-Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id);
-
-Status SendReleaseReply(int sock, ObjectID object_id, int error);
-
-Status ReadReleaseReply(uint8_t* data, ObjectID* object_id);
-
-/* Plasma Delete message functions. */
-
-Status SendDeleteRequest(int sock, ObjectID object_id);
-
-Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id);
-
-Status SendDeleteReply(int sock, ObjectID object_id, int error);
-
-Status ReadDeleteReply(uint8_t* data, ObjectID* object_id);
-
-/* Satus messages. */
-
-Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
-
-Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects);
-
-Status SendStatusReply(
-    int sock, ObjectID object_ids[], int object_status[], int64_t num_objects);
-
-int64_t ReadStatusReply_num_objects(uint8_t* data);
-
-Status ReadStatusReply(
-    uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects);
-
-/* Plasma Constains message functions. */
-
-Status SendContainsRequest(int sock, ObjectID object_id);
-
-Status ReadContainsRequest(uint8_t* data, ObjectID* object_id);
-
-Status SendContainsReply(int sock, ObjectID object_id, bool has_object);
-
-Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object);
-
-/* Plasma Connect message functions. */
-
-Status SendConnectRequest(int sock);
-
-Status ReadConnectRequest(uint8_t* data);
-
-Status SendConnectReply(int sock, int64_t memory_capacity);
-
-Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity);
-
-/* Plasma Evict message functions (no reply so far). */
-
-Status SendEvictRequest(int sock, int64_t num_bytes);
-
-Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes);
-
-Status SendEvictReply(int sock, int64_t num_bytes);
-
-Status ReadEvictReply(uint8_t* data, int64_t& num_bytes);
-
-/* Plasma Fetch Remote message functions. */
-
-Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
-
-Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids);
-
-/* Plasma Wait message functions. */
-
-Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
-    int num_ready_objects, int64_t timeout_ms);
-
-Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
-    int64_t* timeout_ms, int* num_ready_objects);
-
-Status SendWaitReply(
-    int sock, const ObjectRequestMap& object_requests, int num_ready_objects);
-
-Status ReadWaitReply(
-    uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects);
-
-/* Plasma Subscribe message functions. */
-
-Status SendSubscribeRequest(int sock);
-
-/* Data messages. */
-
-Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port);
-
-Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port);
-
-Status SendDataReply(
-    int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size);
-
-Status ReadDataReply(
-    uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size);
-
-#endif /* PLASMA_PROTOCOL */
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
deleted file mode 100644
index 9394e3de310b2..0000000000000
--- a/cpp/src/plasma/store.cc
+++ /dev/null
@@ -1,683 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// PLASMA STORE: This is a simple object store server process
-//
-// It accepts incoming client connections on a unix domain socket
-// (name passed in via the -s option of the executable) and uses a
-// single thread to serve the clients. Each client establishes a
-// connection and can create objects, wait for objects and seal
-// objects through that connection.
-//
-// It keeps a hash table that maps object_ids (which are 20 byte long,
-// just enough to store and SHA1 hash) to memory mapped files.
-
-#include "plasma/store.h"
-
-#include <assert.h>
-#include <fcntl.h>
-#include <getopt.h>
-#include <limits.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/statvfs.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include <deque>
-#include <string>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-
-#include "format/common_generated.h"
-#include "plasma/common.h"
-#include "plasma/fling.h"
-#include "plasma/io.h"
-#include "plasma/malloc.h"
-
-extern "C" {
-void* dlmalloc(size_t bytes);
-void* dlmemalign(size_t alignment, size_t bytes);
-void dlfree(void* mem);
-size_t dlmalloc_set_footprint_limit(size_t bytes);
-}
-
-struct GetRequest {
-  GetRequest(Client* client, const std::vector<ObjectID>& object_ids);
-  /// The client that called get.
-  Client* client;
-  /// The ID of the timer that will time out and cause this wait to return to
-  ///  the client if it hasn't already returned.
-  int64_t timer;
-  /// The object IDs involved in this request. This is used in the reply.
-  std::vector<ObjectID> object_ids;
-  /// The object information for the objects in this request. This is used in
-  /// the reply.
-  std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> objects;
-  /// The minimum number of objects to wait for in this request.
-  int64_t num_objects_to_wait_for;
-  /// The number of object requests in this wait request that are already
-  /// satisfied.
-  int64_t num_satisfied;
-};
-
-GetRequest::GetRequest(Client* client, const std::vector<ObjectID>& object_ids)
-    : client(client),
-      timer(-1),
-      object_ids(object_ids.begin(), object_ids.end()),
-      objects(object_ids.size()),
-      num_satisfied(0) {
-  std::unordered_set<ObjectID, UniqueIDHasher> unique_ids(
-      object_ids.begin(), object_ids.end());
-  num_objects_to_wait_for = unique_ids.size();
-}
-
-Client::Client(int fd) : fd(fd) {}
-
-PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory)
-    : loop_(loop), eviction_policy_(&store_info_) {
-  store_info_.memory_capacity = system_memory;
-}
-
-// TODO(pcm): Get rid of this destructor by using RAII to clean up data.
-PlasmaStore::~PlasmaStore() {
-  for (const auto& element : pending_notifications_) {
-    auto object_notifications = element.second.object_notifications;
-    for (size_t i = 0; i < object_notifications.size(); ++i) {
-      uint8_t* notification = reinterpret_cast<uint8_t*>(object_notifications.at(i));
-      uint8_t* data = notification;
-      // TODO(pcm): Get rid of this delete.
-      delete[] data;
-    }
-  }
-}
-
-// If this client is not already using the object, add the client to the
-// object's list of clients, otherwise do nothing.
-void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client* client) {
-  // Check if this client is already using the object.
-  if (entry->clients.find(client) != entry->clients.end()) { return; }
-  // If there are no other clients using this object, notify the eviction policy
-  // that the object is being used.
-  if (entry->clients.size() == 0) {
-    // Tell the eviction policy that this object is being used.
-    std::vector<ObjectID> objects_to_evict;
-    eviction_policy_.begin_object_access(entry->object_id, &objects_to_evict);
-    delete_objects(objects_to_evict);
-  }
-  // Add the client pointer to the list of clients using this object.
-  entry->clients.insert(client);
-}
-
-// Create a new object buffer in the hash table.
-int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
-    int64_t metadata_size, Client* client, PlasmaObject* result) {
-  ARROW_LOG(DEBUG) << "creating object " << object_id.hex();
-  if (store_info_.objects.count(object_id) != 0) {
-    // There is already an object with the same ID in the Plasma Store, so
-    // ignore this requst.
-    return PlasmaError_ObjectExists;
-  }
-  // Try to evict objects until there is enough space.
-  uint8_t* pointer;
-  do {
-    // Allocate space for the new object. We use dlmemalign instead of dlmalloc
-    // in order to align the allocated region to a 64-byte boundary. This is not
-    // strictly necessary, but it is an optimization that could speed up the
-    // computation of a hash of the data (see compute_object_hash_parallel in
-    // plasma_client.cc). Note that even though this pointer is 64-byte aligned,
-    // it is not guaranteed that the corresponding pointer in the client will be
-    // 64-byte aligned, but in practice it often will be.
-    pointer =
-        reinterpret_cast<uint8_t*>(dlmemalign(BLOCK_SIZE, data_size + metadata_size));
-    if (pointer == NULL) {
-      // Tell the eviction policy how much space we need to create this object.
-      std::vector<ObjectID> objects_to_evict;
-      bool success =
-          eviction_policy_.require_space(data_size + metadata_size, &objects_to_evict);
-      delete_objects(objects_to_evict);
-      // Return an error to the client if not enough space could be freed to
-      // create the object.
-      if (!success) { return PlasmaError_OutOfMemory; }
-    }
-  } while (pointer == NULL);
-  int fd;
-  int64_t map_size;
-  ptrdiff_t offset;
-  get_malloc_mapinfo(pointer, &fd, &map_size, &offset);
-  assert(fd != -1);
-
-  auto entry = std::unique_ptr<ObjectTableEntry>(new ObjectTableEntry());
-  entry->object_id = object_id;
-  entry->info.object_id = object_id.binary();
-  entry->info.data_size = data_size;
-  entry->info.metadata_size = metadata_size;
-  entry->pointer = pointer;
-  // TODO(pcm): Set the other fields.
-  entry->fd = fd;
-  entry->map_size = map_size;
-  entry->offset = offset;
-  entry->state = PLASMA_CREATED;
-
-  store_info_.objects[object_id] = std::move(entry);
-  result->handle.store_fd = fd;
-  result->handle.mmap_size = map_size;
-  result->data_offset = offset;
-  result->metadata_offset = offset + data_size;
-  result->data_size = data_size;
-  result->metadata_size = metadata_size;
-  // Notify the eviction policy that this object was created. This must be done
-  // immediately before the call to add_client_to_object_clients so that the
-  // eviction policy does not have an opportunity to evict the object.
-  eviction_policy_.object_created(object_id);
-  // Record that this client is using this object.
-  add_client_to_object_clients(store_info_.objects[object_id].get(), client);
-  return PlasmaError_OK;
-}
-
-void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
-  DCHECK(object != NULL);
-  DCHECK(entry != NULL);
-  DCHECK(entry->state == PLASMA_SEALED);
-  object->handle.store_fd = entry->fd;
-  object->handle.mmap_size = entry->map_size;
-  object->data_offset = entry->offset;
-  object->metadata_offset = entry->offset + entry->info.data_size;
-  object->data_size = entry->info.data_size;
-  object->metadata_size = entry->info.metadata_size;
-}
-
-void PlasmaStore::return_from_get(GetRequest* get_req) {
-  // Send the get reply to the client.
-  Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], get_req->objects,
-      get_req->object_ids.size());
-  warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd);
-  // If we successfully sent the get reply message to the client, then also send
-  // the file descriptors.
-  if (s.ok()) {
-    // Send all of the file descriptors for the present objects.
-    for (const auto& object_id : get_req->object_ids) {
-      PlasmaObject& object = get_req->objects[object_id];
-      // We use the data size to indicate whether the object is present or not.
-      if (object.data_size != -1) {
-        int error_code = send_fd(get_req->client->fd, object.handle.store_fd);
-        // If we failed to send the file descriptor, loop until we have sent it
-        // successfully. TODO(rkn): This is problematic for two reasons. First
-        // of all, sending the file descriptor should just succeed without any
-        // errors, but sometimes I see a "Message too long" error number.
-        // Second, looping like this allows a client to potentially block the
-        // plasma store event loop which should never happen.
-        while (error_code < 0) {
-          if (errno == EMSGSIZE) {
-            ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying.";
-            error_code = send_fd(get_req->client->fd, object.handle.store_fd);
-            continue;
-          }
-          warn_if_sigpipe(error_code, get_req->client->fd);
-          break;
-        }
-      }
-    }
-  }
-
-  // Remove the get request from each of the relevant object_get_requests hash
-  // tables if it is present there. It should only be present there if the get
-  // request timed out.
-  for (ObjectID& object_id : get_req->object_ids) {
-    auto& get_requests = object_get_requests_[object_id];
-    // Erase get_req from the vector.
-    auto it = std::find(get_requests.begin(), get_requests.end(), get_req);
-    if (it != get_requests.end()) { get_requests.erase(it); }
-  }
-  // Remove the get request.
-  if (get_req->timer != -1) { ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK); }
-  delete get_req;
-}
-
-void PlasmaStore::update_object_get_requests(const ObjectID& object_id) {
-  std::vector<GetRequest*>& get_requests = object_get_requests_[object_id];
-  size_t index = 0;
-  size_t num_requests = get_requests.size();
-  for (size_t i = 0; i < num_requests; ++i) {
-    GetRequest* get_req = get_requests[index];
-    auto entry = get_object_table_entry(&store_info_, object_id);
-    ARROW_CHECK(entry != NULL);
-
-    PlasmaObject_init(&get_req->objects[object_id], entry);
-    get_req->num_satisfied += 1;
-    // Record the fact that this client will be using this object and will
-    // be responsible for releasing this object.
-    add_client_to_object_clients(entry, get_req->client);
-
-    // If this get request is done, reply to the client.
-    if (get_req->num_satisfied == get_req->num_objects_to_wait_for) {
-      return_from_get(get_req);
-    } else {
-      // The call to return_from_get will remove the current element in the
-      // array, so we only increment the counter in the else branch.
-      index += 1;
-    }
-  }
-
-  DCHECK(index == get_requests.size());
-  // Remove the array of get requests for this object, since no one should be
-  // waiting for this object anymore.
-  object_get_requests_.erase(object_id);
-}
-
-void PlasmaStore::process_get_request(
-    Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms) {
-  // Create a get request for this object.
-  GetRequest* get_req = new GetRequest(client, object_ids);
-
-  for (auto object_id : object_ids) {
-    // Check if this object is already present locally. If so, record that the
-    // object is being used and mark it as accounted for.
-    auto entry = get_object_table_entry(&store_info_, object_id);
-    if (entry && entry->state == PLASMA_SEALED) {
-      // Update the get request to take into account the present object.
-      PlasmaObject_init(&get_req->objects[object_id], entry);
-      get_req->num_satisfied += 1;
-      // If necessary, record that this client is using this object. In the case
-      // where entry == NULL, this will be called from seal_object.
-      add_client_to_object_clients(entry, client);
-    } else {
-      // Add a placeholder plasma object to the get request to indicate that the
-      // object is not present. This will be parsed by the client. We set the
-      // data size to -1 to indicate that the object is not present.
-      get_req->objects[object_id].data_size = -1;
-      // Add the get request to the relevant data structures.
-      object_get_requests_[object_id].push_back(get_req);
-    }
-  }
-
-  // If all of the objects are present already or if the timeout is 0, return to
-  // the client.
-  if (get_req->num_satisfied == get_req->num_objects_to_wait_for || timeout_ms == 0) {
-    return_from_get(get_req);
-  } else if (timeout_ms != -1) {
-    // Set a timer that will cause the get request to return to the client. Note
-    // that a timeout of -1 is used to indicate that no timer should be set.
-    get_req->timer = loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) {
-      return_from_get(get_req);
-      return kEventLoopTimerDone;
-    });
-  }
-}
-
-int PlasmaStore::remove_client_from_object_clients(
-    ObjectTableEntry* entry, Client* client) {
-  auto it = entry->clients.find(client);
-  if (it != entry->clients.end()) {
-    entry->clients.erase(it);
-    // If no more clients are using this object, notify the eviction policy
-    // that the object is no longer being used.
-    if (entry->clients.size() == 0) {
-      // Tell the eviction policy that this object is no longer being used.
-      std::vector<ObjectID> objects_to_evict;
-      eviction_policy_.end_object_access(entry->object_id, &objects_to_evict);
-      delete_objects(objects_to_evict);
-    }
-    // Return 1 to indicate that the client was removed.
-    return 1;
-  } else {
-    // Return 0 to indicate that the client was not removed.
-    return 0;
-  }
-}
-
-void PlasmaStore::release_object(const ObjectID& object_id, Client* client) {
-  auto entry = get_object_table_entry(&store_info_, object_id);
-  ARROW_CHECK(entry != NULL);
-  // Remove the client from the object's array of clients.
-  ARROW_CHECK(remove_client_from_object_clients(entry, client) == 1);
-}
-
-// Check if an object is present.
-int PlasmaStore::contains_object(const ObjectID& object_id) {
-  auto entry = get_object_table_entry(&store_info_, object_id);
-  return entry && (entry->state == PLASMA_SEALED) ? OBJECT_FOUND : OBJECT_NOT_FOUND;
-}
-
-// Seal an object that has been created in the hash table.
-void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[]) {
-  ARROW_LOG(DEBUG) << "sealing object " << object_id.hex();
-  auto entry = get_object_table_entry(&store_info_, object_id);
-  ARROW_CHECK(entry != NULL);
-  ARROW_CHECK(entry->state == PLASMA_CREATED);
-  // Set the state of object to SEALED.
-  entry->state = PLASMA_SEALED;
-  // Set the object digest.
-  entry->info.digest = std::string(reinterpret_cast<char*>(&digest[0]), kDigestSize);
-  // Inform all subscribers that a new object has been sealed.
-  push_notification(&entry->info);
-
-  // Update all get requests that involve this object.
-  update_object_get_requests(object_id);
-}
-
-void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
-  for (const auto& object_id : object_ids) {
-    ARROW_LOG(DEBUG) << "deleting object " << object_id.hex();
-    auto entry = get_object_table_entry(&store_info_, object_id);
-    // TODO(rkn): This should probably not fail, but should instead throw an
-    // error. Maybe we should also support deleting objects that have been
-    // created but not sealed.
-    ARROW_CHECK(entry != NULL) << "To delete an object it must be in the object table.";
-    ARROW_CHECK(entry->state == PLASMA_SEALED)
-        << "To delete an object it must have been sealed.";
-    ARROW_CHECK(entry->clients.size() == 0)
-        << "To delete an object, there must be no clients currently using it.";
-    dlfree(entry->pointer);
-    store_info_.objects.erase(object_id);
-    // Inform all subscribers that the object has been deleted.
-    ObjectInfoT notification;
-    notification.object_id = object_id.binary();
-    notification.is_deletion = true;
-    push_notification(&notification);
-  }
-}
-
-void PlasmaStore::connect_client(int listener_sock) {
-  int client_fd = AcceptClient(listener_sock);
-  // This is freed in disconnect_client.
-  Client* client = new Client(client_fd);
-  // Add a callback to handle events on this socket.
-  // TODO(pcm): Check return value.
-  loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) {
-    Status s = process_message(client);
-    if (!s.ok()) { ARROW_LOG(FATAL) << "Failed to process file event: " << s; }
-  });
-  ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
-}
-
-void PlasmaStore::disconnect_client(Client* client) {
-  ARROW_CHECK(client != NULL);
-  ARROW_CHECK(client->fd > 0);
-  loop_->remove_file_event(client->fd);
-  // Close the socket.
-  close(client->fd);
-  ARROW_LOG(INFO) << "Disconnecting client on fd " << client->fd;
-  // If this client was using any objects, remove it from the appropriate
-  // lists.
-  for (const auto& entry : store_info_.objects) {
-    remove_client_from_object_clients(entry.second.get(), client);
-  }
-  // Note, the store may still attempt to send a message to the disconnected
-  // client (for example, when an object ID that the client was waiting for
-  // is ready). In these cases, the attempt to send the message will fail, but
-  // the store should just ignore the failure.
-  delete client;
-}
-
-/// Send notifications about sealed objects to the subscribers. This is called
-/// in seal_object. If the socket's send buffer is full, the notification will
-/// be
-/// buffered, and this will be called again when the send buffer has room.
-///
-/// @param client The client to send the notification to.
-/// @return Void.
-void PlasmaStore::send_notifications(int client_fd) {
-  auto it = pending_notifications_.find(client_fd);
-
-  int num_processed = 0;
-  bool closed = false;
-  // Loop over the array of pending notifications and send as many of them as
-  // possible.
-  for (size_t i = 0; i < it->second.object_notifications.size(); ++i) {
-    uint8_t* notification =
-        reinterpret_cast<uint8_t*>(it->second.object_notifications.at(i));
-    // Decode the length, which is the first bytes of the message.
-    int64_t size = *(reinterpret_cast<int64_t*>(notification));
-
-    // Attempt to send a notification about this object ID.
-    ssize_t nbytes = send(client_fd, notification, sizeof(int64_t) + size, 0);
-    if (nbytes >= 0) {
-      ARROW_CHECK(nbytes == static_cast<ssize_t>(sizeof(int64_t)) + size);
-    } else if (nbytes == -1 &&
-               (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
-      ARROW_LOG(DEBUG) << "The socket's send buffer is full, so we are caching this "
-                          "notification and will send it later.";
-      // Add a callback to the event loop to send queued notifications whenever
-      // there is room in the socket's send buffer. Callbacks can be added
-      // more than once here and will be overwritten. The callback is removed
-      // at the end of the method.
-      // TODO(pcm): Introduce status codes and check in case the file descriptor
-      // is added twice.
-      loop_->add_file_event(client_fd, kEventLoopWrite,
-          [this, client_fd](int events) { send_notifications(client_fd); });
-      break;
-    } else {
-      ARROW_LOG(WARNING) << "Failed to send notification to client on fd " << client_fd;
-      if (errno == EPIPE) {
-        closed = true;
-        break;
-      }
-    }
-    num_processed += 1;
-    // The corresponding malloc happened in create_object_info_buffer
-    // within push_notification.
-    delete[] notification;
-  }
-  // Remove the sent notifications from the array.
-  it->second.object_notifications.erase(it->second.object_notifications.begin(),
-      it->second.object_notifications.begin() + num_processed);
-
-  // Stop sending notifications if the pipe was broken.
-  if (closed) {
-    close(client_fd);
-    pending_notifications_.erase(client_fd);
-  }
-
-  // If we have sent all notifications, remove the fd from the event loop.
-  if (it->second.object_notifications.empty()) { loop_->remove_file_event(client_fd); }
-}
-
-void PlasmaStore::push_notification(ObjectInfoT* object_info) {
-  for (auto& element : pending_notifications_) {
-    uint8_t* notification = create_object_info_buffer(object_info);
-    element.second.object_notifications.push_back(notification);
-    send_notifications(element.first);
-    // The notification gets freed in send_notifications when the notification
-    // is sent over the socket.
-  }
-}
-
-// Subscribe to notifications about sealed objects.
-void PlasmaStore::subscribe_to_updates(Client* client) {
-  ARROW_LOG(DEBUG) << "subscribing to updates on fd " << client->fd;
-  // TODO(rkn): The store could block here if the client doesn't send a file
-  // descriptor.
-  int fd = recv_fd(client->fd);
-  if (fd < 0) {
-    // This may mean that the client died before sending the file descriptor.
-    ARROW_LOG(WARNING) << "Failed to receive file descriptor from client on fd "
-                       << client->fd << ".";
-    return;
-  }
-
-  // Create a new array to buffer notifications that can't be sent to the
-  // subscriber yet because the socket send buffer is full. TODO(rkn): the queue
-  // never gets freed.
-  // TODO(pcm): Is the following neccessary?
-  pending_notifications_[fd];
-
-  // Push notifications to the new subscriber about existing objects.
-  for (const auto& entry : store_info_.objects) {
-    push_notification(&entry.second->info);
-  }
-  send_notifications(fd);
-}
-
-Status PlasmaStore::process_message(Client* client) {
-  int64_t type;
-  Status s = ReadMessage(client->fd, &type, &input_buffer_);
-  ARROW_CHECK(s.ok() || s.IsIOError());
-
-  uint8_t* input = input_buffer_.data();
-  ObjectID object_id;
-  PlasmaObject object;
-  // TODO(pcm): Get rid of the following.
-  memset(&object, 0, sizeof(object));
-
-  // Process the different types of requests.
-  switch (type) {
-    case MessageType_PlasmaCreateRequest: {
-      int64_t data_size;
-      int64_t metadata_size;
-      RETURN_NOT_OK(ReadCreateRequest(input, &object_id, &data_size, &metadata_size));
-      int error_code =
-          create_object(object_id, data_size, metadata_size, client, &object);
-      HANDLE_SIGPIPE(
-          SendCreateReply(client->fd, object_id, &object, error_code), client->fd);
-      if (error_code == PlasmaError_OK) {
-        warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
-      }
-    } break;
-    case MessageType_PlasmaGetRequest: {
-      std::vector<ObjectID> object_ids_to_get;
-      int64_t timeout_ms;
-      RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms));
-      process_get_request(client, object_ids_to_get, timeout_ms);
-    } break;
-    case MessageType_PlasmaReleaseRequest:
-      RETURN_NOT_OK(ReadReleaseRequest(input, &object_id));
-      release_object(object_id, client);
-      break;
-    case MessageType_PlasmaContainsRequest:
-      RETURN_NOT_OK(ReadContainsRequest(input, &object_id));
-      if (contains_object(object_id) == OBJECT_FOUND) {
-        HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd);
-      } else {
-        HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd);
-      }
-      break;
-    case MessageType_PlasmaSealRequest: {
-      unsigned char digest[kDigestSize];
-      RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0]));
-      seal_object(object_id, &digest[0]);
-    } break;
-    case MessageType_PlasmaEvictRequest: {
-      // This code path should only be used for testing.
-      int64_t num_bytes;
-      RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes));
-      std::vector<ObjectID> objects_to_evict;
-      int64_t num_bytes_evicted =
-          eviction_policy_.choose_objects_to_evict(num_bytes, &objects_to_evict);
-      delete_objects(objects_to_evict);
-      HANDLE_SIGPIPE(SendEvictReply(client->fd, num_bytes_evicted), client->fd);
-    } break;
-    case MessageType_PlasmaSubscribeRequest:
-      subscribe_to_updates(client);
-      break;
-    case MessageType_PlasmaConnectRequest: {
-      HANDLE_SIGPIPE(
-          SendConnectReply(client->fd, store_info_.memory_capacity), client->fd);
-    } break;
-    case DISCONNECT_CLIENT:
-      ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd;
-      disconnect_client(client);
-      break;
-    default:
-      // This code should be unreachable.
-      ARROW_CHECK(0);
-  }
-  return Status::OK();
-}
-
-// Report "success" to valgrind.
-void signal_handler(int signal) {
-  if (signal == SIGTERM) { exit(0); }
-}
-
-void start_server(char* socket_name, int64_t system_memory) {
-  // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
-  // to a client that has already died, the store could die.
-  signal(SIGPIPE, SIG_IGN);
-  // Create the event loop.
-  EventLoop loop;
-  PlasmaStore store(&loop, system_memory);
-  int socket = bind_ipc_sock(socket_name, true);
-  ARROW_CHECK(socket >= 0);
-  // TODO(pcm): Check return value.
-  loop.add_file_event(socket, kEventLoopRead,
-      [&store, socket](int events) { store.connect_client(socket); });
-  loop.run();
-}
-
-int main(int argc, char* argv[]) {
-  signal(SIGTERM, signal_handler);
-  char* socket_name = NULL;
-  int64_t system_memory = -1;
-  int c;
-  while ((c = getopt(argc, argv, "s:m:")) != -1) {
-    switch (c) {
-      case 's':
-        socket_name = optarg;
-        break;
-      case 'm': {
-        char extra;
-        int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra);
-        ARROW_CHECK(scanned == 1);
-        ARROW_LOG(INFO) << "Allowing the Plasma store to use up to "
-                        << static_cast<double>(system_memory) / 1000000000
-                        << "GB of memory.";
-        break;
-      }
-      default:
-        exit(-1);
-    }
-  }
-  if (!socket_name) {
-    ARROW_LOG(FATAL) << "please specify socket for incoming connections with -s switch";
-  }
-  if (system_memory == -1) {
-    ARROW_LOG(FATAL) << "please specify the amount of system memory with -m switch";
-  }
-#ifdef __linux__
-  // On Linux, check that the amount of memory available in /dev/shm is large
-  // enough to accommodate the request. If it isn't, then fail.
-  int shm_fd = open("/dev/shm", O_RDONLY);
-  struct statvfs shm_vfs_stats;
-  fstatvfs(shm_fd, &shm_vfs_stats);
-  // The value shm_vfs_stats.f_bsize is the block size, and the value
-  // shm_vfs_stats.f_bavail is the number of available blocks.
-  int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail;
-  close(shm_fd);
-  if (system_memory > shm_mem_avail) {
-    ARROW_LOG(FATAL) << "System memory request exceeds memory available in /dev/shm. The "
-                        "request is for "
-                     << system_memory << " bytes, and the amount available is "
-                     << shm_mem_avail
-                     << " bytes. You may be able to free up space by deleting files in "
-                        "/dev/shm. If you are inside a Docker container, you may need to "
-                        "pass "
-                        "an argument with the flag '--shm-size' to 'docker run'.";
-  }
-#endif
-  // Make it so dlmalloc fails if we try to request more memory than is
-  // available.
-  dlmalloc_set_footprint_limit((size_t)system_memory);
-  ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
-  start_server(socket_name, system_memory);
-}
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
deleted file mode 100644
index 8bd94265410f6..0000000000000
--- a/cpp/src/plasma/store.h
+++ /dev/null
@@ -1,169 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_STORE_H
-#define PLASMA_STORE_H
-
-#include <deque>
-#include <vector>
-
-#include "plasma/common.h"
-#include "plasma/events.h"
-#include "plasma/eviction_policy.h"
-#include "plasma/plasma.h"
-#include "plasma/protocol.h"
-
-struct GetRequest;
-
-struct NotificationQueue {
-  /// The object notifications for clients. We notify the client about the
-  /// objects in the order that the objects were sealed or deleted.
-  std::deque<uint8_t*> object_notifications;
-};
-
-/// Contains all information that is associated with a Plasma store client.
-struct Client {
-  explicit Client(int fd);
-
-  /// The file descriptor used to communicate with the client.
-  int fd;
-};
-
-class PlasmaStore {
- public:
-  PlasmaStore(EventLoop* loop, int64_t system_memory);
-
-  ~PlasmaStore();
-
-  /// Create a new object. The client must do a call to release_object to tell
-  /// the store when it is done with the object.
-  ///
-  /// @param object_id Object ID of the object to be created.
-  /// @param data_size Size in bytes of the object to be created.
-  /// @param metadata_size Size in bytes of the object metadata.
-  /// @return One of the following error codes:
-  ///  - PlasmaError_OK, if the object was created successfully.
-  ///  - PlasmaError_ObjectExists, if an object with this ID is already
-  ///    present in the store. In this case, the client should not call
-  ///    plasma_release.
-  ///  - PlasmaError_OutOfMemory, if the store is out of memory and
-  ///    cannot create the object. In this case, the client should not call
-  ///    plasma_release.
-  int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
-      Client* client, PlasmaObject* result);
-
-  /// Delete objects that have been created in the hash table. This should only
-  /// be called on objects that are returned by the eviction policy to evict.
-  ///
-  /// @param object_ids Object IDs of the objects to be deleted.
-  /// @return Void.
-  void delete_objects(const std::vector<ObjectID>& object_ids);
-
-  /// Process a get request from a client. This method assumes that we will
-  /// eventually have these objects sealed. If one of the objects has not yet
-  /// been sealed, the client that requested the object will be notified when it
-  /// is sealed.
-  ///
-  /// For each object, the client must do a call to release_object to tell the
-  /// store when it is done with the object.
-  ///
-  /// @param client The client making this request.
-  /// @param object_ids Object IDs of the objects to be gotten.
-  /// @param timeout_ms The timeout for the get request in milliseconds.
-  /// @return Void.
-  void process_get_request(
-      Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms);
-
-  /// Seal an object. The object is now immutable and can be accessed with get.
-  ///
-  /// @param object_id Object ID of the object to be sealed.
-  /// @param digest The digest of the object. This is used to tell if two
-  /// objects
-  ///        with the same object ID are the same.
-  /// @return Void.
-  void seal_object(const ObjectID& object_id, unsigned char digest[]);
-
-  /// Check if the plasma store contains an object:
-  ///
-  /// @param object_id Object ID that will be checked.
-  /// @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if
-  /// not
-  int contains_object(const ObjectID& object_id);
-
-  /// Record the fact that a particular client is no longer using an object.
-  ///
-  /// @param object_id The object ID of the object that is being released.
-  /// @param client The client making this request.
-  /// @param Void.
-  void release_object(const ObjectID& object_id, Client* client);
-
-  /// Subscribe a file descriptor to updates about new sealed objects.
-  ///
-  /// @param client The client making this request.
-  /// @return Void.
-  void subscribe_to_updates(Client* client);
-
-  /// Connect a new client to the PlasmaStore.
-  ///
-  /// @param listener_sock The socket that is listening to incoming connections.
-  /// @return Void.
-  void connect_client(int listener_sock);
-
-  /// Disconnect a client from the PlasmaStore.
-  ///
-  /// @param client The client that is disconnected.
-  /// @return Void.
-  void disconnect_client(Client* client);
-
-  void send_notifications(int client_fd);
-
-  Status process_message(Client* client);
-
- private:
-  void push_notification(ObjectInfoT* object_notification);
-
-  void add_client_to_object_clients(ObjectTableEntry* entry, Client* client);
-
-  void return_from_get(GetRequest* get_req);
-
-  void update_object_get_requests(const ObjectID& object_id);
-
-  int remove_client_from_object_clients(ObjectTableEntry* entry, Client* client);
-
-  /// Event loop of the plasma store.
-  EventLoop* loop_;
-  /// The plasma store information, including the object tables, that is exposed
-  /// to the eviction policy.
-  PlasmaStoreInfo store_info_;
-  /// The state that is managed by the eviction policy.
-  EvictionPolicy eviction_policy_;
-  /// Input buffer. This is allocated only once to avoid mallocs for every
-  /// call to process_message.
-  std::vector<uint8_t> input_buffer_;
-  /// A hash table mapping object IDs to a vector of the get requests that are
-  /// waiting for the object to arrive.
-  std::unordered_map<ObjectID, std::vector<GetRequest*>, UniqueIDHasher>
-      object_get_requests_;
-  /// The pending notifications that have not been sent to subscribers because
-  /// the socket send buffers were full. This is a hash table from client file
-  /// descriptor to an array of object_ids to send to that client.
-  /// TODO(pcm): Consider putting this into the Client data structure and
-  /// reorganize the code slightly.
-  std::unordered_map<int, NotificationQueue> pending_notifications_;
-};
-
-#endif  // PLASMA_STORE_H
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
deleted file mode 100644
index 29b5b135144c3..0000000000000
--- a/cpp/src/plasma/test/client_tests.cc
+++ /dev/null
@@ -1,132 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-#include <assert.h>
-#include <signal.h>
-#include <stdlib.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "plasma/client.h"
-#include "plasma/common.h"
-#include "plasma/plasma.h"
-#include "plasma/protocol.h"
-
-std::string g_test_executable;  // NOLINT
-
-class TestPlasmaStore : public ::testing::Test {
- public:
-  // TODO(pcm): At the moment, stdout of the test gets mixed up with
-  // stdout of the object store. Consider changing that.
-  void SetUp() {
-    std::string plasma_directory =
-        g_test_executable.substr(0, g_test_executable.find_last_of("/"));
-    std::string plasma_command =
-        plasma_directory +
-        "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &";
-    system(plasma_command.c_str());
-    ARROW_CHECK_OK(client_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
-  }
-  virtual void Finish() {
-    ARROW_CHECK_OK(client_.Disconnect());
-    system("killall plasma_store &");
-  }
-
- protected:
-  PlasmaClient client_;
-};
-
-TEST_F(TestPlasmaStore, ContainsTest) {
-  ObjectID object_id = ObjectID::from_random();
-
-  // Test for object non-existence.
-  bool has_object;
-  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
-  ASSERT_EQ(has_object, false);
-
-  // Test for the object being in local Plasma store.
-  // First create object.
-  int64_t data_size = 100;
-  uint8_t metadata[] = {5};
-  int64_t metadata_size = sizeof(metadata);
-  uint8_t* data;
-  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
-  ARROW_CHECK_OK(client_.Seal(object_id));
-  // Avoid race condition of Plasma Manager waiting for notification.
-  ObjectBuffer object_buffer;
-  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
-  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
-  ASSERT_EQ(has_object, true);
-}
-
-TEST_F(TestPlasmaStore, GetTest) {
-  ObjectID object_id = ObjectID::from_random();
-  ObjectBuffer object_buffer;
-
-  // Test for object non-existence.
-  ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer));
-  ASSERT_EQ(object_buffer.data_size, -1);
-
-  // Test for the object being in local Plasma store.
-  // First create object.
-  int64_t data_size = 4;
-  uint8_t metadata[] = {5};
-  int64_t metadata_size = sizeof(metadata);
-  uint8_t* data;
-  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
-  for (int64_t i = 0; i < data_size; i++) {
-    data[i] = static_cast<uint8_t>(i % 4);
-  }
-  ARROW_CHECK_OK(client_.Seal(object_id));
-
-  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
-  for (int64_t i = 0; i < data_size; i++) {
-    ASSERT_EQ(data[i], object_buffer.data[i]);
-  }
-}
-
-TEST_F(TestPlasmaStore, MultipleGetTest) {
-  ObjectID object_id1 = ObjectID::from_random();
-  ObjectID object_id2 = ObjectID::from_random();
-  ObjectID object_ids[2] = {object_id1, object_id2};
-  ObjectBuffer object_buffer[2];
-
-  int64_t data_size = 4;
-  uint8_t metadata[] = {5};
-  int64_t metadata_size = sizeof(metadata);
-  uint8_t* data;
-  ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data));
-  data[0] = 1;
-  ARROW_CHECK_OK(client_.Seal(object_id1));
-
-  ARROW_CHECK_OK(client_.Create(object_id2, data_size, metadata, metadata_size, &data));
-  data[0] = 2;
-  ARROW_CHECK_OK(client_.Seal(object_id2));
-
-  ARROW_CHECK_OK(client_.Get(object_ids, 2, -1, object_buffer));
-  ASSERT_EQ(object_buffer[0].data[0], 1);
-  ASSERT_EQ(object_buffer[1].data[0], 2);
-}
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  g_test_executable = std::string(argv[0]);
-  return RUN_ALL_TESTS();
-}
diff --git a/cpp/src/plasma/test/run_tests.sh b/cpp/src/plasma/test/run_tests.sh
deleted file mode 100644
index 958bd08398e23..0000000000000
--- a/cpp/src/plasma/test/run_tests.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Cause the script to exit if a single command fails.
-set -e
-
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
-sleep 1
-./src/plasma/manager_tests
-killall plasma_store
-./src/plasma/serialization_tests
-
-# Start the Redis shards.
-./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
-redis_pid1=$!
-./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
-redis_pid2=$!
-sleep 1
-
-# Flush the redis server
-./src/common/thirdparty/redis/src/redis-cli flushall
-# Register the shard location with the primary shard.
-./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
-./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
-sleep 1
-./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
-plasma1_pid=$!
-./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
-plasma2_pid=$!
-./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 &
-plasma3_pid=$!
-./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
-plasma4_pid=$!
-sleep 1
-
-./src/plasma/client_tests
-
-kill $plasma4_pid
-kill $plasma3_pid
-kill $plasma2_pid
-kill $plasma1_pid
-kill $redis_pid1
-wait $redis_pid1
-kill $redis_pid2
-wait $redis_pid2
diff --git a/cpp/src/plasma/test/run_valgrind.sh b/cpp/src/plasma/test/run_valgrind.sh
deleted file mode 100644
index 0472194128679..0000000000000
--- a/cpp/src/plasma/test/run_valgrind.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Cause the script to exit if a single command fails.
-set -e
-
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
-sleep 1
-valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests
-killall plasma_store
-valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
deleted file mode 100644
index 325cead06e770..0000000000000
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ /dev/null
@@ -1,388 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "plasma/common.h"
-#include "plasma/io.h"
-#include "plasma/plasma.h"
-#include "plasma/protocol.h"
-
-/**
- * Create a temporary file. Needs to be closed by the caller.
- *
- * @return File descriptor of the file.
- */
-int create_temp_file(void) {
-  static char temp[] = "/tmp/tempfileXXXXXX";
-  char file_name[32];
-  strncpy(file_name, temp, 32);
-  return mkstemp(file_name);
-}
-
-/**
- * Seek to the beginning of a file and read a message from it.
- *
- * @param fd File descriptor of the file.
- * @param message type Message type that we expect in the file.
- *
- * @return Pointer to the content of the message. Needs to be freed by the
- * caller.
- */
-std::vector<uint8_t> read_message_from_file(int fd, int message_type) {
-  /* Go to the beginning of the file. */
-  lseek(fd, 0, SEEK_SET);
-  int64_t type;
-  std::vector<uint8_t> data;
-  ARROW_CHECK_OK(ReadMessage(fd, &type, &data));
-  ARROW_CHECK(type == message_type);
-  return data;
-}
-
-PlasmaObject random_plasma_object(void) {
-  unsigned int seed = static_cast<unsigned int>(time(NULL));
-  int random = rand_r(&seed);
-  PlasmaObject object;
-  memset(&object, 0, sizeof(object));
-  object.handle.store_fd = random + 7;
-  object.handle.mmap_size = random + 42;
-  object.data_offset = random + 1;
-  object.metadata_offset = random + 2;
-  object.data_size = random + 3;
-  object.metadata_size = random + 4;
-  return object;
-}
-
-TEST(PlasmaSerialization, CreateRequest) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  int64_t data_size1 = 42;
-  int64_t metadata_size1 = 11;
-  ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateRequest);
-  ObjectID object_id2;
-  int64_t data_size2;
-  int64_t metadata_size2;
-  ARROW_CHECK_OK(
-      ReadCreateRequest(data.data(), &object_id2, &data_size2, &metadata_size2));
-  ASSERT_EQ(data_size1, data_size2);
-  ASSERT_EQ(metadata_size1, metadata_size2);
-  ASSERT_EQ(object_id1, object_id2);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, CreateReply) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  PlasmaObject object1 = random_plasma_object();
-  ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateReply);
-  ObjectID object_id2;
-  PlasmaObject object2;
-  memset(&object2, 0, sizeof(object2));
-  ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2));
-  ASSERT_EQ(object_id1, object_id2);
-  ASSERT_EQ(memcmp(&object1, &object2, sizeof(object1)), 0);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, SealRequest) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  unsigned char digest1[kDigestSize];
-  memset(&digest1[0], 7, kDigestSize);
-  ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0]));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealRequest);
-  ObjectID object_id2;
-  unsigned char digest2[kDigestSize];
-  ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0]));
-  ASSERT_EQ(object_id1, object_id2);
-  ASSERT_EQ(memcmp(&digest1[0], &digest2[0], kDigestSize), 0);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, SealReply) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealReply);
-  ObjectID object_id2;
-  Status s = ReadSealReply(data.data(), &object_id2);
-  ASSERT_EQ(object_id1, object_id2);
-  ASSERT_TRUE(s.IsPlasmaObjectExists());
-  close(fd);
-}
-
-TEST(PlasmaSerialization, GetRequest) {
-  int fd = create_temp_file();
-  ObjectID object_ids[2];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  int64_t timeout_ms = 1234;
-  ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetRequest);
-  std::vector<ObjectID> object_ids_return;
-  int64_t timeout_ms_return;
-  ARROW_CHECK_OK(ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return));
-  ASSERT_EQ(object_ids[0], object_ids_return[0]);
-  ASSERT_EQ(object_ids[1], object_ids_return[1]);
-  ASSERT_EQ(timeout_ms, timeout_ms_return);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, GetReply) {
-  int fd = create_temp_file();
-  ObjectID object_ids[2];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> plasma_objects;
-  plasma_objects[object_ids[0]] = random_plasma_object();
-  plasma_objects[object_ids[1]] = random_plasma_object();
-  ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetReply);
-  ObjectID object_ids_return[2];
-  PlasmaObject plasma_objects_return[2];
-  memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
-  ARROW_CHECK_OK(
-      ReadGetReply(data.data(), object_ids_return, &plasma_objects_return[0], 2));
-  ASSERT_EQ(object_ids[0], object_ids_return[0]);
-  ASSERT_EQ(object_ids[1], object_ids_return[1]);
-  ASSERT_EQ(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
-                sizeof(PlasmaObject)),
-      0);
-  ASSERT_EQ(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
-                sizeof(PlasmaObject)),
-      0);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, ReleaseRequest) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
-  ObjectID object_id2;
-  ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2));
-  ASSERT_EQ(object_id1, object_id2);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, ReleaseReply) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaReleaseReply);
-  ObjectID object_id2;
-  Status s = ReadReleaseReply(data.data(), &object_id2);
-  ASSERT_EQ(object_id1, object_id2);
-  ASSERT_TRUE(s.IsPlasmaObjectExists());
-  close(fd);
-}
-
-TEST(PlasmaSerialization, DeleteRequest) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
-  ObjectID object_id2;
-  ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2));
-  ASSERT_EQ(object_id1, object_id2);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, DeleteReply) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  int error1 = PlasmaError_ObjectExists;
-  ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteReply);
-  ObjectID object_id2;
-  Status s = ReadDeleteReply(data.data(), &object_id2);
-  ASSERT_EQ(object_id1, object_id2);
-  ASSERT_TRUE(s.IsPlasmaObjectExists());
-  close(fd);
-}
-
-TEST(PlasmaSerialization, StatusRequest) {
-  int fd = create_temp_file();
-  int64_t num_objects = 2;
-  ObjectID object_ids[num_objects];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusRequest);
-  ObjectID object_ids_read[num_objects];
-  ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects));
-  ASSERT_EQ(object_ids[0], object_ids_read[0]);
-  ASSERT_EQ(object_ids[1], object_ids_read[1]);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, StatusReply) {
-  int fd = create_temp_file();
-  ObjectID object_ids[2];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  int object_statuses[2] = {42, 43};
-  ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusReply);
-  int64_t num_objects = ReadStatusReply_num_objects(data.data());
-  ObjectID object_ids_read[num_objects];
-  int object_statuses_read[num_objects];
-  ARROW_CHECK_OK(
-      ReadStatusReply(data.data(), object_ids_read, object_statuses_read, num_objects));
-  ASSERT_EQ(object_ids[0], object_ids_read[0]);
-  ASSERT_EQ(object_ids[1], object_ids_read[1]);
-  ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
-  ASSERT_EQ(object_statuses[1], object_statuses_read[1]);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, EvictRequest) {
-  int fd = create_temp_file();
-  int64_t num_bytes = 111;
-  ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictRequest);
-  int64_t num_bytes_received;
-  ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received));
-  ASSERT_EQ(num_bytes, num_bytes_received);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, EvictReply) {
-  int fd = create_temp_file();
-  int64_t num_bytes = 111;
-  ARROW_CHECK_OK(SendEvictReply(fd, num_bytes));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictReply);
-  int64_t num_bytes_received;
-  ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received));
-  ASSERT_EQ(num_bytes, num_bytes_received);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, FetchRequest) {
-  int fd = create_temp_file();
-  ObjectID object_ids[2];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2));
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaFetchRequest);
-  std::vector<ObjectID> object_ids_read;
-  ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read));
-  ASSERT_EQ(object_ids[0], object_ids_read[0]);
-  ASSERT_EQ(object_ids[1], object_ids_read[1]);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, WaitRequest) {
-  int fd = create_temp_file();
-  const int num_objects_in = 2;
-  ObjectRequest object_requests_in[num_objects_in] = {
-      ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}),
-      ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})};
-  const int num_ready_objects_in = 1;
-  int64_t timeout_ms = 1000;
-
-  ARROW_CHECK_OK(SendWaitRequest(
-      fd, &object_requests_in[0], num_objects_in, num_ready_objects_in, timeout_ms));
-  /* Read message back. */
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitRequest);
-  int num_ready_objects_out;
-  int64_t timeout_ms_read;
-  ObjectRequestMap object_requests_out;
-  ARROW_CHECK_OK(ReadWaitRequest(
-      data.data(), object_requests_out, &timeout_ms_read, &num_ready_objects_out));
-  ASSERT_EQ(num_objects_in, object_requests_out.size());
-  ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
-  for (int i = 0; i < num_objects_in; i++) {
-    const ObjectID& object_id = object_requests_in[i].object_id;
-    ASSERT_EQ(1, object_requests_out.count(object_id));
-    const auto& entry = object_requests_out.find(object_id);
-    ASSERT_TRUE(entry != object_requests_out.end());
-    ASSERT_EQ(entry->second.object_id, object_requests_in[i].object_id);
-    ASSERT_EQ(entry->second.type, object_requests_in[i].type);
-  }
-  close(fd);
-}
-
-TEST(PlasmaSerialization, WaitReply) {
-  int fd = create_temp_file();
-  const int num_objects_in = 2;
-  /* Create a map with two ObjectRequests in it. */
-  ObjectRequestMap objects_in(num_objects_in);
-  ObjectID id1 = ObjectID::from_random();
-  objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local});
-  ObjectID id2 = ObjectID::from_random();
-  objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent});
-
-  ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in));
-  /* Read message back. */
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitReply);
-  ObjectRequest objects_out[2];
-  int num_objects_out;
-  ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out));
-  ASSERT_EQ(num_objects_in, num_objects_out);
-  for (int i = 0; i < num_objects_out; i++) {
-    /* Each object request must appear exactly once. */
-    ASSERT_EQ(objects_in.count(objects_out[i].object_id), 1);
-    const auto& entry = objects_in.find(objects_out[i].object_id);
-    ASSERT_TRUE(entry != objects_in.end());
-    ASSERT_EQ(entry->second.object_id, objects_out[i].object_id);
-    ASSERT_EQ(entry->second.status, objects_out[i].status);
-  }
-  close(fd);
-}
-
-TEST(PlasmaSerialization, DataRequest) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  const char* address1 = "address1";
-  int port1 = 12345;
-  ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1));
-  /* Reading message back. */
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataRequest);
-  ObjectID object_id2;
-  char* address2;
-  int port2;
-  ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2));
-  ASSERT_EQ(object_id1, object_id2);
-  ASSERT_EQ(strcmp(address1, address2), 0);
-  ASSERT_EQ(port1, port2);
-  free(address2);
-  close(fd);
-}
-
-TEST(PlasmaSerialization, DataReply) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  int64_t object_size1 = 146;
-  int64_t metadata_size1 = 198;
-  ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1));
-  /* Reading message back. */
-  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataReply);
-  ObjectID object_id2;
-  int64_t object_size2;
-  int64_t metadata_size2;
-  ARROW_CHECK_OK(ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2));
-  ASSERT_EQ(object_id1, object_id2);
-  ASSERT_EQ(object_size1, object_size2);
-  ASSERT_EQ(metadata_size1, metadata_size2);
-}
diff --git a/cpp/src/plasma/thirdparty/ae/ae.c b/cpp/src/plasma/thirdparty/ae/ae.c
deleted file mode 100644
index e66808a81466d..0000000000000
--- a/cpp/src/plasma/thirdparty/ae/ae.c
+++ /dev/null
@@ -1,465 +0,0 @@
-/* A simple event-driven programming library. Originally I wrote this code
- * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
- * it in form of a library for easy reuse.
- *
- * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <stdio.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <poll.h>
-#include <string.h>
-#include <time.h>
-#include <errno.h>
-
-#include "ae.h"
-#include "zmalloc.h"
-#include "config.h"
-
-/* Include the best multiplexing layer supported by this system.
- * The following should be ordered by performances, descending. */
-#ifdef HAVE_EVPORT
-#include "ae_evport.c"
-#else
-    #ifdef HAVE_EPOLL
-    #include "ae_epoll.c"
-    #else
-        #ifdef HAVE_KQUEUE
-        #include "ae_kqueue.c"
-        #else
-        #include "ae_select.c"
-        #endif
-    #endif
-#endif
-
-aeEventLoop *aeCreateEventLoop(int setsize) {
-    aeEventLoop *eventLoop;
-    int i;
-
-    if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
-    eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
-    eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
-    if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
-    eventLoop->setsize = setsize;
-    eventLoop->lastTime = time(NULL);
-    eventLoop->timeEventHead = NULL;
-    eventLoop->timeEventNextId = 0;
-    eventLoop->stop = 0;
-    eventLoop->maxfd = -1;
-    eventLoop->beforesleep = NULL;
-    if (aeApiCreate(eventLoop) == -1) goto err;
-    /* Events with mask == AE_NONE are not set. So let's initialize the
-     * vector with it. */
-    for (i = 0; i < setsize; i++)
-        eventLoop->events[i].mask = AE_NONE;
-    return eventLoop;
-
-err:
-    if (eventLoop) {
-        zfree(eventLoop->events);
-        zfree(eventLoop->fired);
-        zfree(eventLoop);
-    }
-    return NULL;
-}
-
-/* Return the current set size. */
-int aeGetSetSize(aeEventLoop *eventLoop) {
-    return eventLoop->setsize;
-}
-
-/* Resize the maximum set size of the event loop.
- * If the requested set size is smaller than the current set size, but
- * there is already a file descriptor in use that is >= the requested
- * set size minus one, AE_ERR is returned and the operation is not
- * performed at all.
- *
- * Otherwise AE_OK is returned and the operation is successful. */
-int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
-    int i;
-
-    if (setsize == eventLoop->setsize) return AE_OK;
-    if (eventLoop->maxfd >= setsize) return AE_ERR;
-    if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
-
-    eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
-    eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
-    eventLoop->setsize = setsize;
-
-    /* Make sure that if we created new slots, they are initialized with
-     * an AE_NONE mask. */
-    for (i = eventLoop->maxfd+1; i < setsize; i++)
-        eventLoop->events[i].mask = AE_NONE;
-    return AE_OK;
-}
-
-void aeDeleteEventLoop(aeEventLoop *eventLoop) {
-    aeApiFree(eventLoop);
-    zfree(eventLoop->events);
-    zfree(eventLoop->fired);
-    zfree(eventLoop);
-}
-
-void aeStop(aeEventLoop *eventLoop) {
-    eventLoop->stop = 1;
-}
-
-int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
-        aeFileProc *proc, void *clientData)
-{
-    if (fd >= eventLoop->setsize) {
-        errno = ERANGE;
-        return AE_ERR;
-    }
-    aeFileEvent *fe = &eventLoop->events[fd];
-
-    if (aeApiAddEvent(eventLoop, fd, mask) == -1)
-        return AE_ERR;
-    fe->mask |= mask;
-    if (mask & AE_READABLE) fe->rfileProc = proc;
-    if (mask & AE_WRITABLE) fe->wfileProc = proc;
-    fe->clientData = clientData;
-    if (fd > eventLoop->maxfd)
-        eventLoop->maxfd = fd;
-    return AE_OK;
-}
-
-void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
-{
-    if (fd >= eventLoop->setsize) return;
-    aeFileEvent *fe = &eventLoop->events[fd];
-    if (fe->mask == AE_NONE) return;
-
-    aeApiDelEvent(eventLoop, fd, mask);
-    fe->mask = fe->mask & (~mask);
-    if (fd == eventLoop->maxfd && fe->mask == AE_NONE) {
-        /* Update the max fd */
-        int j;
-
-        for (j = eventLoop->maxfd-1; j >= 0; j--)
-            if (eventLoop->events[j].mask != AE_NONE) break;
-        eventLoop->maxfd = j;
-    }
-}
-
-int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
-    if (fd >= eventLoop->setsize) return 0;
-    aeFileEvent *fe = &eventLoop->events[fd];
-
-    return fe->mask;
-}
-
-static void aeGetTime(long *seconds, long *milliseconds)
-{
-    struct timeval tv;
-
-    gettimeofday(&tv, NULL);
-    *seconds = tv.tv_sec;
-    *milliseconds = tv.tv_usec/1000;
-}
-
-static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) {
-    long cur_sec, cur_ms, when_sec, when_ms;
-
-    aeGetTime(&cur_sec, &cur_ms);
-    when_sec = cur_sec + milliseconds/1000;
-    when_ms = cur_ms + milliseconds%1000;
-    if (when_ms >= 1000) {
-        when_sec ++;
-        when_ms -= 1000;
-    }
-    *sec = when_sec;
-    *ms = when_ms;
-}
-
-long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
-        aeTimeProc *proc, void *clientData,
-        aeEventFinalizerProc *finalizerProc)
-{
-    long long id = eventLoop->timeEventNextId++;
-    aeTimeEvent *te;
-
-    te = zmalloc(sizeof(*te));
-    if (te == NULL) return AE_ERR;
-    te->id = id;
-    aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms);
-    te->timeProc = proc;
-    te->finalizerProc = finalizerProc;
-    te->clientData = clientData;
-    te->next = eventLoop->timeEventHead;
-    eventLoop->timeEventHead = te;
-    return id;
-}
-
-int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
-{
-    aeTimeEvent *te = eventLoop->timeEventHead;
-    while(te) {
-        if (te->id == id) {
-            te->id = AE_DELETED_EVENT_ID;
-            return AE_OK;
-        }
-        te = te->next;
-    }
-    return AE_ERR; /* NO event with the specified ID found */
-}
-
-/* Search the first timer to fire.
- * This operation is useful to know how many time the select can be
- * put in sleep without to delay any event.
- * If there are no timers NULL is returned.
- *
- * Note that's O(N) since time events are unsorted.
- * Possible optimizations (not needed by Redis so far, but...):
- * 1) Insert the event in order, so that the nearest is just the head.
- *    Much better but still insertion or deletion of timers is O(N).
- * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)).
- */
-static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
-{
-    aeTimeEvent *te = eventLoop->timeEventHead;
-    aeTimeEvent *nearest = NULL;
-
-    while(te) {
-        if (!nearest || te->when_sec < nearest->when_sec ||
-                (te->when_sec == nearest->when_sec &&
-                 te->when_ms < nearest->when_ms))
-            nearest = te;
-        te = te->next;
-    }
-    return nearest;
-}
-
-/* Process time events */
-static int processTimeEvents(aeEventLoop *eventLoop) {
-    int processed = 0;
-    aeTimeEvent *te, *prev;
-    long long maxId;
-    time_t now = time(NULL);
-
-    /* If the system clock is moved to the future, and then set back to the
-     * right value, time events may be delayed in a random way. Often this
-     * means that scheduled operations will not be performed soon enough.
-     *
-     * Here we try to detect system clock skews, and force all the time
-     * events to be processed ASAP when this happens: the idea is that
-     * processing events earlier is less dangerous than delaying them
-     * indefinitely, and practice suggests it is. */
-    if (now < eventLoop->lastTime) {
-        te = eventLoop->timeEventHead;
-        while(te) {
-            te->when_sec = 0;
-            te = te->next;
-        }
-    }
-    eventLoop->lastTime = now;
-
-    prev = NULL;
-    te = eventLoop->timeEventHead;
-    maxId = eventLoop->timeEventNextId-1;
-    while(te) {
-        long now_sec, now_ms;
-        long long id;
-
-        /* Remove events scheduled for deletion. */
-        if (te->id == AE_DELETED_EVENT_ID) {
-            aeTimeEvent *next = te->next;
-            if (prev == NULL)
-                eventLoop->timeEventHead = te->next;
-            else
-                prev->next = te->next;
-            if (te->finalizerProc)
-                te->finalizerProc(eventLoop, te->clientData);
-            zfree(te);
-            te = next;
-            continue;
-        }
-
-        /* Make sure we don't process time events created by time events in
-         * this iteration. Note that this check is currently useless: we always
-         * add new timers on the head, however if we change the implementation
-         * detail, this check may be useful again: we keep it here for future
-         * defense. */
-        if (te->id > maxId) {
-            te = te->next;
-            continue;
-        }
-        aeGetTime(&now_sec, &now_ms);
-        if (now_sec > te->when_sec ||
-            (now_sec == te->when_sec && now_ms >= te->when_ms))
-        {
-            int retval;
-
-            id = te->id;
-            retval = te->timeProc(eventLoop, id, te->clientData);
-            processed++;
-            if (retval != AE_NOMORE) {
-                aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
-            } else {
-                te->id = AE_DELETED_EVENT_ID;
-            }
-        }
-        prev = te;
-        te = te->next;
-    }
-    return processed;
-}
-
-/* Process every pending time event, then every pending file event
- * (that may be registered by time event callbacks just processed).
- * Without special flags the function sleeps until some file event
- * fires, or when the next time event occurs (if any).
- *
- * If flags is 0, the function does nothing and returns.
- * if flags has AE_ALL_EVENTS set, all the kind of events are processed.
- * if flags has AE_FILE_EVENTS set, file events are processed.
- * if flags has AE_TIME_EVENTS set, time events are processed.
- * if flags has AE_DONT_WAIT set the function returns ASAP until all
- * the events that's possible to process without to wait are processed.
- *
- * The function returns the number of events processed. */
-int aeProcessEvents(aeEventLoop *eventLoop, int flags)
-{
-    int processed = 0, numevents;
-
-    /* Nothing to do? return ASAP */
-    if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0;
-
-    /* Note that we want call select() even if there are no
-     * file events to process as long as we want to process time
-     * events, in order to sleep until the next time event is ready
-     * to fire. */
-    if (eventLoop->maxfd != -1 ||
-        ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) {
-        int j;
-        aeTimeEvent *shortest = NULL;
-        struct timeval tv, *tvp;
-
-        if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT))
-            shortest = aeSearchNearestTimer(eventLoop);
-        if (shortest) {
-            long now_sec, now_ms;
-
-            aeGetTime(&now_sec, &now_ms);
-            tvp = &tv;
-
-            /* How many milliseconds we need to wait for the next
-             * time event to fire? */
-            long long ms =
-                (shortest->when_sec - now_sec)*1000 +
-                shortest->when_ms - now_ms;
-
-            if (ms > 0) {
-                tvp->tv_sec = ms/1000;
-                tvp->tv_usec = (ms % 1000)*1000;
-            } else {
-                tvp->tv_sec = 0;
-                tvp->tv_usec = 0;
-            }
-        } else {
-            /* If we have to check for events but need to return
-             * ASAP because of AE_DONT_WAIT we need to set the timeout
-             * to zero */
-            if (flags & AE_DONT_WAIT) {
-                tv.tv_sec = tv.tv_usec = 0;
-                tvp = &tv;
-            } else {
-                /* Otherwise we can block */
-                tvp = NULL; /* wait forever */
-            }
-        }
-
-        numevents = aeApiPoll(eventLoop, tvp);
-        for (j = 0; j < numevents; j++) {
-            aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd];
-            int mask = eventLoop->fired[j].mask;
-            int fd = eventLoop->fired[j].fd;
-            int rfired = 0;
-
-	    /* note the fe->mask & mask & ... code: maybe an already processed
-             * event removed an element that fired and we still didn't
-             * processed, so we check if the event is still valid. */
-            if (fe->mask & mask & AE_READABLE) {
-                rfired = 1;
-                fe->rfileProc(eventLoop,fd,fe->clientData,mask);
-            }
-            if (fe->mask & mask & AE_WRITABLE) {
-                if (!rfired || fe->wfileProc != fe->rfileProc)
-                    fe->wfileProc(eventLoop,fd,fe->clientData,mask);
-            }
-            processed++;
-        }
-    }
-    /* Check time events */
-    if (flags & AE_TIME_EVENTS)
-        processed += processTimeEvents(eventLoop);
-
-    return processed; /* return the number of processed file/time events */
-}
-
-/* Wait for milliseconds until the given file descriptor becomes
- * writable/readable/exception */
-int aeWait(int fd, int mask, long long milliseconds) {
-    struct pollfd pfd;
-    int retmask = 0, retval;
-
-    memset(&pfd, 0, sizeof(pfd));
-    pfd.fd = fd;
-    if (mask & AE_READABLE) pfd.events |= POLLIN;
-    if (mask & AE_WRITABLE) pfd.events |= POLLOUT;
-
-    if ((retval = poll(&pfd, 1, milliseconds))== 1) {
-        if (pfd.revents & POLLIN) retmask |= AE_READABLE;
-        if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE;
-	if (pfd.revents & POLLERR) retmask |= AE_WRITABLE;
-        if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE;
-        return retmask;
-    } else {
-        return retval;
-    }
-}
-
-void aeMain(aeEventLoop *eventLoop) {
-    eventLoop->stop = 0;
-    while (!eventLoop->stop) {
-        if (eventLoop->beforesleep != NULL)
-            eventLoop->beforesleep(eventLoop);
-        aeProcessEvents(eventLoop, AE_ALL_EVENTS);
-    }
-}
-
-char *aeGetApiName(void) {
-    return aeApiName();
-}
-
-void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) {
-    eventLoop->beforesleep = beforesleep;
-}
diff --git a/cpp/src/plasma/thirdparty/ae/ae.h b/cpp/src/plasma/thirdparty/ae/ae.h
deleted file mode 100644
index 827c4c9e4e59e..0000000000000
--- a/cpp/src/plasma/thirdparty/ae/ae.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/* A simple event-driven programming library. Originally I wrote this code
- * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
- * it in form of a library for easy reuse.
- *
- * Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __AE_H__
-#define __AE_H__
-
-#include <time.h>
-
-#define AE_OK 0
-#define AE_ERR -1
-
-#define AE_NONE 0
-#define AE_READABLE 1
-#define AE_WRITABLE 2
-
-#define AE_FILE_EVENTS 1
-#define AE_TIME_EVENTS 2
-#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
-#define AE_DONT_WAIT 4
-
-#define AE_NOMORE -1
-#define AE_DELETED_EVENT_ID -1
-
-/* Macros */
-#define AE_NOTUSED(V) ((void) V)
-
-struct aeEventLoop;
-
-/* Types and data structures */
-typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
-typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData);
-typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData);
-typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop);
-
-/* File event structure */
-typedef struct aeFileEvent {
-    int mask; /* one of AE_(READABLE|WRITABLE) */
-    aeFileProc *rfileProc;
-    aeFileProc *wfileProc;
-    void *clientData;
-} aeFileEvent;
-
-/* Time event structure */
-typedef struct aeTimeEvent {
-    long long id; /* time event identifier. */
-    long when_sec; /* seconds */
-    long when_ms; /* milliseconds */
-    aeTimeProc *timeProc;
-    aeEventFinalizerProc *finalizerProc;
-    void *clientData;
-    struct aeTimeEvent *next;
-} aeTimeEvent;
-
-/* A fired event */
-typedef struct aeFiredEvent {
-    int fd;
-    int mask;
-} aeFiredEvent;
-
-/* State of an event based program */
-typedef struct aeEventLoop {
-    int maxfd;   /* highest file descriptor currently registered */
-    int setsize; /* max number of file descriptors tracked */
-    long long timeEventNextId;
-    time_t lastTime;     /* Used to detect system clock skew */
-    aeFileEvent *events; /* Registered events */
-    aeFiredEvent *fired; /* Fired events */
-    aeTimeEvent *timeEventHead;
-    int stop;
-    void *apidata; /* This is used for polling API specific data */
-    aeBeforeSleepProc *beforesleep;
-} aeEventLoop;
-
-/* Prototypes */
-aeEventLoop *aeCreateEventLoop(int setsize);
-void aeDeleteEventLoop(aeEventLoop *eventLoop);
-void aeStop(aeEventLoop *eventLoop);
-int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
-        aeFileProc *proc, void *clientData);
-void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask);
-int aeGetFileEvents(aeEventLoop *eventLoop, int fd);
-long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
-        aeTimeProc *proc, void *clientData,
-        aeEventFinalizerProc *finalizerProc);
-int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id);
-int aeProcessEvents(aeEventLoop *eventLoop, int flags);
-int aeWait(int fd, int mask, long long milliseconds);
-void aeMain(aeEventLoop *eventLoop);
-char *aeGetApiName(void);
-void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
-int aeGetSetSize(aeEventLoop *eventLoop);
-int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
-
-#endif
diff --git a/cpp/src/plasma/thirdparty/ae/ae_epoll.c b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
deleted file mode 100644
index 410aac70dc5af..0000000000000
--- a/cpp/src/plasma/thirdparty/ae/ae_epoll.c
+++ /dev/null
@@ -1,135 +0,0 @@
-/* Linux epoll(2) based ae.c module
- *
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include <sys/epoll.h>
-
-typedef struct aeApiState {
-    int epfd;
-    struct epoll_event *events;
-} aeApiState;
-
-static int aeApiCreate(aeEventLoop *eventLoop) {
-    aeApiState *state = zmalloc(sizeof(aeApiState));
-
-    if (!state) return -1;
-    state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize);
-    if (!state->events) {
-        zfree(state);
-        return -1;
-    }
-    state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */
-    if (state->epfd == -1) {
-        zfree(state->events);
-        zfree(state);
-        return -1;
-    }
-    eventLoop->apidata = state;
-    return 0;
-}
-
-static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
-    aeApiState *state = eventLoop->apidata;
-
-    state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize);
-    return 0;
-}
-
-static void aeApiFree(aeEventLoop *eventLoop) {
-    aeApiState *state = eventLoop->apidata;
-
-    close(state->epfd);
-    zfree(state->events);
-    zfree(state);
-}
-
-static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee = {0}; /* avoid valgrind warning */
-    /* If the fd was already monitored for some event, we need a MOD
-     * operation. Otherwise we need an ADD operation. */
-    int op = eventLoop->events[fd].mask == AE_NONE ?
-            EPOLL_CTL_ADD : EPOLL_CTL_MOD;
-
-    ee.events = 0;
-    mask |= eventLoop->events[fd].mask; /* Merge old events */
-    if (mask & AE_READABLE) ee.events |= EPOLLIN;
-    if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
-    ee.data.fd = fd;
-    if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1;
-    return 0;
-}
-
-static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
-    aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee = {0}; /* avoid valgrind warning */
-    int mask = eventLoop->events[fd].mask & (~delmask);
-
-    ee.events = 0;
-    if (mask & AE_READABLE) ee.events |= EPOLLIN;
-    if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
-    ee.data.fd = fd;
-    if (mask != AE_NONE) {
-        epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee);
-    } else {
-        /* Note, Kernel < 2.6.9 requires a non null event pointer even for
-         * EPOLL_CTL_DEL. */
-        epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee);
-    }
-}
-
-static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
-    aeApiState *state = eventLoop->apidata;
-    int retval, numevents = 0;
-
-    retval = epoll_wait(state->epfd,state->events,eventLoop->setsize,
-            tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
-    if (retval > 0) {
-        int j;
-
-        numevents = retval;
-        for (j = 0; j < numevents; j++) {
-            int mask = 0;
-            struct epoll_event *e = state->events+j;
-
-            if (e->events & EPOLLIN) mask |= AE_READABLE;
-            if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
-            if (e->events & EPOLLERR) mask |= AE_WRITABLE;
-            if (e->events & EPOLLHUP) mask |= AE_WRITABLE;
-            eventLoop->fired[j].fd = e->data.fd;
-            eventLoop->fired[j].mask = mask;
-        }
-    }
-    return numevents;
-}
-
-static char *aeApiName(void) {
-    return "epoll";
-}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_evport.c b/cpp/src/plasma/thirdparty/ae/ae_evport.c
deleted file mode 100644
index 5c317becb6f7d..0000000000000
--- a/cpp/src/plasma/thirdparty/ae/ae_evport.c
+++ /dev/null
@@ -1,320 +0,0 @@
-/* ae.c module for illumos event ports.
- *
- * Copyright (c) 2012, Joyent, Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include <assert.h>
-#include <errno.h>
-#include <port.h>
-#include <poll.h>
-
-#include <sys/types.h>
-#include <sys/time.h>
-
-#include <stdio.h>
-
-static int evport_debug = 0;
-
-/*
- * This file implements the ae API using event ports, present on Solaris-based
- * systems since Solaris 10.  Using the event port interface, we associate file
- * descriptors with the port.  Each association also includes the set of poll(2)
- * events that the consumer is interested in (e.g., POLLIN and POLLOUT).
- *
- * There's one tricky piece to this implementation: when we return events via
- * aeApiPoll, the corresponding file descriptors become dissociated from the
- * port.  This is necessary because poll events are level-triggered, so if the
- * fd didn't become dissociated, it would immediately fire another event since
- * the underlying state hasn't changed yet.  We must re-associate the file
- * descriptor, but only after we know that our caller has actually read from it.
- * The ae API does not tell us exactly when that happens, but we do know that
- * it must happen by the time aeApiPoll is called again.  Our solution is to
- * keep track of the last fds returned by aeApiPoll and re-associate them next
- * time aeApiPoll is invoked.
- *
- * To summarize, in this module, each fd association is EITHER (a) represented
- * only via the in-kernel association OR (b) represented by pending_fds and
- * pending_masks.  (b) is only true for the last fds we returned from aeApiPoll,
- * and only until we enter aeApiPoll again (at which point we restore the
- * in-kernel association).
- */
-#define MAX_EVENT_BATCHSZ 512
-
-typedef struct aeApiState {
-    int     portfd;                             /* event port */
-    int     npending;                           /* # of pending fds */
-    int     pending_fds[MAX_EVENT_BATCHSZ];     /* pending fds */
-    int     pending_masks[MAX_EVENT_BATCHSZ];   /* pending fds' masks */
-} aeApiState;
-
-static int aeApiCreate(aeEventLoop *eventLoop) {
-    int i;
-    aeApiState *state = zmalloc(sizeof(aeApiState));
-    if (!state) return -1;
-
-    state->portfd = port_create();
-    if (state->portfd == -1) {
-        zfree(state);
-        return -1;
-    }
-
-    state->npending = 0;
-
-    for (i = 0; i < MAX_EVENT_BATCHSZ; i++) {
-        state->pending_fds[i] = -1;
-        state->pending_masks[i] = AE_NONE;
-    }
-
-    eventLoop->apidata = state;
-    return 0;
-}
-
-static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
-    /* Nothing to resize here. */
-    return 0;
-}
-
-static void aeApiFree(aeEventLoop *eventLoop) {
-    aeApiState *state = eventLoop->apidata;
-
-    close(state->portfd);
-    zfree(state);
-}
-
-static int aeApiLookupPending(aeApiState *state, int fd) {
-    int i;
-
-    for (i = 0; i < state->npending; i++) {
-        if (state->pending_fds[i] == fd)
-            return (i);
-    }
-
-    return (-1);
-}
-
-/*
- * Helper function to invoke port_associate for the given fd and mask.
- */
-static int aeApiAssociate(const char *where, int portfd, int fd, int mask) {
-    int events = 0;
-    int rv, err;
-
-    if (mask & AE_READABLE)
-        events |= POLLIN;
-    if (mask & AE_WRITABLE)
-        events |= POLLOUT;
-
-    if (evport_debug)
-        fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events);
-
-    rv = port_associate(portfd, PORT_SOURCE_FD, fd, events,
-        (void *)(uintptr_t)mask);
-    err = errno;
-
-    if (evport_debug)
-        fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err));
-
-    if (rv == -1) {
-        fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err));
-
-        if (err == EAGAIN)
-            fprintf(stderr, "aeApiAssociate: event port limit exceeded.");
-    }
-
-    return rv;
-}
-
-static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-    int fullmask, pfd;
-
-    if (evport_debug)
-        fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask);
-
-    /*
-     * Since port_associate's "events" argument replaces any existing events, we
-     * must be sure to include whatever events are already associated when
-     * we call port_associate() again.
-     */
-    fullmask = mask | eventLoop->events[fd].mask;
-    pfd = aeApiLookupPending(state, fd);
-
-    if (pfd != -1) {
-        /*
-         * This fd was recently returned from aeApiPoll.  It should be safe to
-         * assume that the consumer has processed that poll event, but we play
-         * it safer by simply updating pending_mask.  The fd will be
-         * re-associated as usual when aeApiPoll is called again.
-         */
-        if (evport_debug)
-            fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd);
-        state->pending_masks[pfd] |= fullmask;
-        return 0;
-    }
-
-    return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask));
-}
-
-static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-    int fullmask, pfd;
-
-    if (evport_debug)
-        fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask);
-
-    pfd = aeApiLookupPending(state, fd);
-
-    if (pfd != -1) {
-        if (evport_debug)
-            fprintf(stderr, "deleting event from pending fd %d\n", fd);
-
-        /*
-         * This fd was just returned from aeApiPoll, so it's not currently
-         * associated with the port.  All we need to do is update
-         * pending_mask appropriately.
-         */
-        state->pending_masks[pfd] &= ~mask;
-
-        if (state->pending_masks[pfd] == AE_NONE)
-            state->pending_fds[pfd] = -1;
-
-        return;
-    }
-
-    /*
-     * The fd is currently associated with the port.  Like with the add case
-     * above, we must look at the full mask for the file descriptor before
-     * updating that association.  We don't have a good way of knowing what the
-     * events are without looking into the eventLoop state directly.  We rely on
-     * the fact that our caller has already updated the mask in the eventLoop.
-     */
-
-    fullmask = eventLoop->events[fd].mask;
-    if (fullmask == AE_NONE) {
-        /*
-         * We're removing *all* events, so use port_dissociate to remove the
-         * association completely.  Failure here indicates a bug.
-         */
-        if (evport_debug)
-            fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd);
-
-        if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) {
-            perror("aeApiDelEvent: port_dissociate");
-            abort(); /* will not return */
-        }
-    } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd,
-        fullmask) != 0) {
-        /*
-         * ENOMEM is a potentially transient condition, but the kernel won't
-         * generally return it unless things are really bad.  EAGAIN indicates
-         * we've reached an resource limit, for which it doesn't make sense to
-         * retry (counter-intuitively).  All other errors indicate a bug.  In any
-         * of these cases, the best we can do is to abort.
-         */
-        abort(); /* will not return */
-    }
-}
-
-static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
-    aeApiState *state = eventLoop->apidata;
-    struct timespec timeout, *tsp;
-    int mask, i;
-    uint_t nevents;
-    port_event_t event[MAX_EVENT_BATCHSZ];
-
-    /*
-     * If we've returned fd events before, we must re-associate them with the
-     * port now, before calling port_get().  See the block comment at the top of
-     * this file for an explanation of why.
-     */
-    for (i = 0; i < state->npending; i++) {
-        if (state->pending_fds[i] == -1)
-            /* This fd has since been deleted. */
-            continue;
-
-        if (aeApiAssociate("aeApiPoll", state->portfd,
-            state->pending_fds[i], state->pending_masks[i]) != 0) {
-            /* See aeApiDelEvent for why this case is fatal. */
-            abort();
-        }
-
-        state->pending_masks[i] = AE_NONE;
-        state->pending_fds[i] = -1;
-    }
-
-    state->npending = 0;
-
-    if (tvp != NULL) {
-        timeout.tv_sec = tvp->tv_sec;
-        timeout.tv_nsec = tvp->tv_usec * 1000;
-        tsp = &timeout;
-    } else {
-        tsp = NULL;
-    }
-
-    /*
-     * port_getn can return with errno == ETIME having returned some events (!).
-     * So if we get ETIME, we check nevents, too.
-     */
-    nevents = 1;
-    if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents,
-        tsp) == -1 && (errno != ETIME || nevents == 0)) {
-        if (errno == ETIME || errno == EINTR)
-            return 0;
-
-        /* Any other error indicates a bug. */
-        perror("aeApiPoll: port_get");
-        abort();
-    }
-
-    state->npending = nevents;
-
-    for (i = 0; i < nevents; i++) {
-            mask = 0;
-            if (event[i].portev_events & POLLIN)
-                mask |= AE_READABLE;
-            if (event[i].portev_events & POLLOUT)
-                mask |= AE_WRITABLE;
-
-            eventLoop->fired[i].fd = event[i].portev_object;
-            eventLoop->fired[i].mask = mask;
-
-            if (evport_debug)
-                fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n",
-                    (int)event[i].portev_object, mask);
-
-            state->pending_fds[i] = event[i].portev_object;
-            state->pending_masks[i] = (uintptr_t)event[i].portev_user;
-    }
-
-    return nevents;
-}
-
-static char *aeApiName(void) {
-    return "evport";
-}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_kqueue.c b/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
deleted file mode 100644
index 6796f4ceb5939..0000000000000
--- a/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
+++ /dev/null
@@ -1,138 +0,0 @@
-/* Kqueue(2)-based ae.c module
- *
- * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include <sys/types.h>
-#include <sys/event.h>
-#include <sys/time.h>
-
-typedef struct aeApiState {
-    int kqfd;
-    struct kevent *events;
-} aeApiState;
-
-static int aeApiCreate(aeEventLoop *eventLoop) {
-    aeApiState *state = zmalloc(sizeof(aeApiState));
-
-    if (!state) return -1;
-    state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize);
-    if (!state->events) {
-        zfree(state);
-        return -1;
-    }
-    state->kqfd = kqueue();
-    if (state->kqfd == -1) {
-        zfree(state->events);
-        zfree(state);
-        return -1;
-    }
-    eventLoop->apidata = state;
-    return 0;
-}
-
-static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
-    aeApiState *state = eventLoop->apidata;
-
-    state->events = zrealloc(state->events, sizeof(struct kevent)*setsize);
-    return 0;
-}
-
-static void aeApiFree(aeEventLoop *eventLoop) {
-    aeApiState *state = eventLoop->apidata;
-
-    close(state->kqfd);
-    zfree(state->events);
-    zfree(state);
-}
-
-static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-    struct kevent ke;
-
-    if (mask & AE_READABLE) {
-        EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
-        if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
-    }
-    if (mask & AE_WRITABLE) {
-        EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL);
-        if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
-    }
-    return 0;
-}
-
-static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-    struct kevent ke;
-
-    if (mask & AE_READABLE) {
-        EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
-        kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
-    }
-    if (mask & AE_WRITABLE) {
-        EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
-        kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
-    }
-}
-
-static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
-    aeApiState *state = eventLoop->apidata;
-    int retval, numevents = 0;
-
-    if (tvp != NULL) {
-        struct timespec timeout;
-        timeout.tv_sec = tvp->tv_sec;
-        timeout.tv_nsec = tvp->tv_usec * 1000;
-        retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
-                        &timeout);
-    } else {
-        retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
-                        NULL);
-    }
-
-    if (retval > 0) {
-        int j;
-
-        numevents = retval;
-        for(j = 0; j < numevents; j++) {
-            int mask = 0;
-            struct kevent *e = state->events+j;
-
-            if (e->filter == EVFILT_READ) mask |= AE_READABLE;
-            if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
-            eventLoop->fired[j].fd = e->ident;
-            eventLoop->fired[j].mask = mask;
-        }
-    }
-    return numevents;
-}
-
-static char *aeApiName(void) {
-    return "kqueue";
-}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_select.c b/cpp/src/plasma/thirdparty/ae/ae_select.c
deleted file mode 100644
index c039a8ea3128d..0000000000000
--- a/cpp/src/plasma/thirdparty/ae/ae_select.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/* Select()-based ae.c module.
- *
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include <sys/select.h>
-#include <string.h>
-
-typedef struct aeApiState {
-    fd_set rfds, wfds;
-    /* We need to have a copy of the fd sets as it's not safe to reuse
-     * FD sets after select(). */
-    fd_set _rfds, _wfds;
-} aeApiState;
-
-static int aeApiCreate(aeEventLoop *eventLoop) {
-    aeApiState *state = zmalloc(sizeof(aeApiState));
-
-    if (!state) return -1;
-    FD_ZERO(&state->rfds);
-    FD_ZERO(&state->wfds);
-    eventLoop->apidata = state;
-    return 0;
-}
-
-static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
-    /* Just ensure we have enough room in the fd_set type. */
-    if (setsize >= FD_SETSIZE) return -1;
-    return 0;
-}
-
-static void aeApiFree(aeEventLoop *eventLoop) {
-    zfree(eventLoop->apidata);
-}
-
-static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-
-    if (mask & AE_READABLE) FD_SET(fd,&state->rfds);
-    if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds);
-    return 0;
-}
-
-static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-
-    if (mask & AE_READABLE) FD_CLR(fd,&state->rfds);
-    if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds);
-}
-
-static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
-    aeApiState *state = eventLoop->apidata;
-    int retval, j, numevents = 0;
-
-    memcpy(&state->_rfds,&state->rfds,sizeof(fd_set));
-    memcpy(&state->_wfds,&state->wfds,sizeof(fd_set));
-
-    retval = select(eventLoop->maxfd+1,
-                &state->_rfds,&state->_wfds,NULL,tvp);
-    if (retval > 0) {
-        for (j = 0; j <= eventLoop->maxfd; j++) {
-            int mask = 0;
-            aeFileEvent *fe = &eventLoop->events[j];
-
-            if (fe->mask == AE_NONE) continue;
-            if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds))
-                mask |= AE_READABLE;
-            if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds))
-                mask |= AE_WRITABLE;
-            eventLoop->fired[numevents].fd = j;
-            eventLoop->fired[numevents].mask = mask;
-            numevents++;
-        }
-    }
-    return numevents;
-}
-
-static char *aeApiName(void) {
-    return "select";
-}
diff --git a/cpp/src/plasma/thirdparty/ae/config.h b/cpp/src/plasma/thirdparty/ae/config.h
deleted file mode 100644
index 4f8e1ea1bc38c..0000000000000
--- a/cpp/src/plasma/thirdparty/ae/config.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __CONFIG_H
-#define __CONFIG_H
-
-#ifdef __APPLE__
-#include <AvailabilityMacros.h>
-#endif
-
-/* Test for polling API */
-#ifdef __linux__
-#define HAVE_EPOLL 1
-#endif
-
-#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__)
-#define HAVE_KQUEUE 1
-#endif
-
-#ifdef __sun
-#include <sys/feature_tests.h>
-#ifdef _DTRACE_VERSION
-#define HAVE_EVPORT 1
-#endif
-#endif
-
-
-#endif
diff --git a/cpp/src/plasma/thirdparty/ae/zmalloc.h b/cpp/src/plasma/thirdparty/ae/zmalloc.h
deleted file mode 100644
index 6c27dd4e5c3d3..0000000000000
--- a/cpp/src/plasma/thirdparty/ae/zmalloc.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _ZMALLOC_H
-#define _ZMALLOC_H
-
-#ifndef zmalloc
-#define zmalloc malloc
-#endif
-
-#ifndef zfree
-#define zfree free
-#endif
-
-#ifndef zrealloc
-#define zrealloc realloc
-#endif
-
-#endif /* _ZMALLOC_H */
diff --git a/cpp/src/plasma/thirdparty/dlmalloc.c b/cpp/src/plasma/thirdparty/dlmalloc.c
deleted file mode 100644
index 84ccbd28fc4ec..0000000000000
--- a/cpp/src/plasma/thirdparty/dlmalloc.c
+++ /dev/null
@@ -1,6281 +0,0 @@
-/*
-  This is a version (aka dlmalloc) of malloc/free/realloc written by
-  Doug Lea and released to the public domain, as explained at
-  http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
-  comments, complaints, performance data, etc to dl@cs.oswego.edu
-
-* Version 2.8.6 Wed Aug 29 06:57:58 2012  Doug Lea
-   Note: There may be an updated version of this malloc obtainable at
-           ftp://gee.cs.oswego.edu/pub/misc/malloc.c
-         Check before installing!
-
-* Quickstart
-
-  This library is all in one file to simplify the most common usage:
-  ftp it, compile it (-O3), and link it into another program. All of
-  the compile-time options default to reasonable values for use on
-  most platforms.  You might later want to step through various
-  compile-time and dynamic tuning options.
-
-  For convenience, an include file for code using this malloc is at:
-     ftp://gee.cs.oswego.edu/pub/misc/malloc-2.8.6.h
-  You don't really need this .h file unless you call functions not
-  defined in your system include files.  The .h file contains only the
-  excerpts from this file needed for using this malloc on ANSI C/C++
-  systems, so long as you haven't changed compile-time options about
-  naming and tuning parameters.  If you do, then you can create your
-  own malloc.h that does include all settings by cutting at the point
-  indicated below. Note that you may already by default be using a C
-  library containing a malloc that is based on some version of this
-  malloc (for example in linux). You might still want to use the one
-  in this file to customize settings or to avoid overheads associated
-  with library versions.
-
-* Vital statistics:
-
-  Supported pointer/size_t representation:       4 or 8 bytes
-       size_t MUST be an unsigned type of the same width as
-       pointers. (If you are using an ancient system that declares
-       size_t as a signed type, or need it to be a different width
-       than pointers, you can use a previous release of this malloc
-       (e.g. 2.7.2) supporting these.)
-
-  Alignment:                                     8 bytes (minimum)
-       This suffices for nearly all current machines and C compilers.
-       However, you can define MALLOC_ALIGNMENT to be wider than this
-       if necessary (up to 128bytes), at the expense of using more space.
-
-  Minimum overhead per allocated chunk:   4 or  8 bytes (if 4byte sizes)
-                                          8 or 16 bytes (if 8byte sizes)
-       Each malloced chunk has a hidden word of overhead holding size
-       and status information, and additional cross-check word
-       if FOOTERS is defined.
-
-  Minimum allocated size: 4-byte ptrs:  16 bytes    (including overhead)
-                          8-byte ptrs:  32 bytes    (including overhead)
-
-       Even a request for zero bytes (i.e., malloc(0)) returns a
-       pointer to something of the minimum allocatable size.
-       The maximum overhead wastage (i.e., number of extra bytes
-       allocated than were requested in malloc) is less than or equal
-       to the minimum size, except for requests >= mmap_threshold that
-       are serviced via mmap(), where the worst case wastage is about
-       32 bytes plus the remainder from a system page (the minimal
-       mmap unit); typically 4096 or 8192 bytes.
-
-  Security: static-safe; optionally more or less
-       The "security" of malloc refers to the ability of malicious
-       code to accentuate the effects of errors (for example, freeing
-       space that is not currently malloc'ed or overwriting past the
-       ends of chunks) in code that calls malloc.  This malloc
-       guarantees not to modify any memory locations below the base of
-       heap, i.e., static variables, even in the presence of usage
-       errors.  The routines additionally detect most improper frees
-       and reallocs.  All this holds as long as the static bookkeeping
-       for malloc itself is not corrupted by some other means.  This
-       is only one aspect of security -- these checks do not, and
-       cannot, detect all possible programming errors.
-
-       If FOOTERS is defined nonzero, then each allocated chunk
-       carries an additional check word to verify that it was malloced
-       from its space.  These check words are the same within each
-       execution of a program using malloc, but differ across
-       executions, so externally crafted fake chunks cannot be
-       freed. This improves security by rejecting frees/reallocs that
-       could corrupt heap memory, in addition to the checks preventing
-       writes to statics that are always on.  This may further improve
-       security at the expense of time and space overhead.  (Note that
-       FOOTERS may also be worth using with MSPACES.)
-
-       By default detected errors cause the program to abort (calling
-       "abort()"). You can override this to instead proceed past
-       errors by defining PROCEED_ON_ERROR.  In this case, a bad free
-       has no effect, and a malloc that encounters a bad address
-       caused by user overwrites will ignore the bad address by
-       dropping pointers and indices to all known memory. This may
-       be appropriate for programs that should continue if at all
-       possible in the face of programming errors, although they may
-       run out of memory because dropped memory is never reclaimed.
-
-       If you don't like either of these options, you can define
-       CORRUPTION_ERROR_ACTION and USAGE_ERROR_ACTION to do anything
-       else. And if if you are sure that your program using malloc has
-       no errors or vulnerabilities, you can define INSECURE to 1,
-       which might (or might not) provide a small performance improvement.
-
-       It is also possible to limit the maximum total allocatable
-       space, using malloc_set_footprint_limit. This is not
-       designed as a security feature in itself (calls to set limits
-       are not screened or privileged), but may be useful as one
-       aspect of a secure implementation.
-
-  Thread-safety: NOT thread-safe unless USE_LOCKS defined non-zero
-       When USE_LOCKS is defined, each public call to malloc, free,
-       etc is surrounded with a lock. By default, this uses a plain
-       pthread mutex, win32 critical section, or a spin-lock if if
-       available for the platform and not disabled by setting
-       USE_SPIN_LOCKS=0.  However, if USE_RECURSIVE_LOCKS is defined,
-       recursive versions are used instead (which are not required for
-       base functionality but may be needed in layered extensions).
-       Using a global lock is not especially fast, and can be a major
-       bottleneck.  It is designed only to provide minimal protection
-       in concurrent environments, and to provide a basis for
-       extensions.  If you are using malloc in a concurrent program,
-       consider instead using nedmalloc
-       (http://www.nedprod.com/programs/portable/nedmalloc/) or
-       ptmalloc (See http://www.malloc.de), which are derived from
-       versions of this malloc.
-
-  System requirements: Any combination of MORECORE and/or MMAP/MUNMAP
-       This malloc can use unix sbrk or any emulation (invoked using
-       the CALL_MORECORE macro) and/or mmap/munmap or any emulation
-       (invoked using CALL_MMAP/CALL_MUNMAP) to get and release system
-       memory.  On most unix systems, it tends to work best if both
-       MORECORE and MMAP are enabled.  On Win32, it uses emulations
-       based on VirtualAlloc. It also uses common C library functions
-       like memset.
-
-  Compliance: I believe it is compliant with the Single Unix Specification
-       (See http://www.unix.org). Also SVID/XPG, ANSI C, and probably
-       others as well.
-
-* Overview of algorithms
-
-  This is not the fastest, most space-conserving, most portable, or
-  most tunable malloc ever written. However it is among the fastest
-  while also being among the most space-conserving, portable and
-  tunable.  Consistent balance across these factors results in a good
-  general-purpose allocator for malloc-intensive programs.
-
-  In most ways, this malloc is a best-fit allocator. Generally, it
-  chooses the best-fitting existing chunk for a request, with ties
-  broken in approximately least-recently-used order. (This strategy
-  normally maintains low fragmentation.) However, for requests less
-  than 256bytes, it deviates from best-fit when there is not an
-  exactly fitting available chunk by preferring to use space adjacent
-  to that used for the previous small request, as well as by breaking
-  ties in approximately most-recently-used order. (These enhance
-  locality of series of small allocations.)  And for very large requests
-  (>= 256Kb by default), it relies on system memory mapping
-  facilities, if supported.  (This helps avoid carrying around and
-  possibly fragmenting memory used only for large chunks.)
-
-  All operations (except malloc_stats and mallinfo) have execution
-  times that are bounded by a constant factor of the number of bits in
-  a size_t, not counting any clearing in calloc or copying in realloc,
-  or actions surrounding MORECORE and MMAP that have times
-  proportional to the number of non-contiguous regions returned by
-  system allocation routines, which is often just 1. In real-time
-  applications, you can optionally suppress segment traversals using
-  NO_SEGMENT_TRAVERSAL, which assures bounded execution even when
-  system allocators return non-contiguous spaces, at the typical
-  expense of carrying around more memory and increased fragmentation.
-
-  The implementation is not very modular and seriously overuses
-  macros. Perhaps someday all C compilers will do as good a job
-  inlining modular code as can now be done by brute-force expansion,
-  but now, enough of them seem not to.
-
-  Some compilers issue a lot of warnings about code that is
-  dead/unreachable only on some platforms, and also about intentional
-  uses of negation on unsigned types. All known cases of each can be
-  ignored.
-
-  For a longer but out of date high-level description, see
-     http://gee.cs.oswego.edu/dl/html/malloc.html
-
-* MSPACES
-  If MSPACES is defined, then in addition to malloc, free, etc.,
-  this file also defines mspace_malloc, mspace_free, etc. These
-  are versions of malloc routines that take an "mspace" argument
-  obtained using create_mspace, to control all internal bookkeeping.
-  If ONLY_MSPACES is defined, only these versions are compiled.
-  So if you would like to use this allocator for only some allocations,
-  and your system malloc for others, you can compile with
-  ONLY_MSPACES and then do something like...
-    static mspace mymspace = create_mspace(0,0); // for example
-    #define mymalloc(bytes)  mspace_malloc(mymspace, bytes)
-
-  (Note: If you only need one instance of an mspace, you can instead
-  use "USE_DL_PREFIX" to relabel the global malloc.)
-
-  You can similarly create thread-local allocators by storing
-  mspaces as thread-locals. For example:
-    static __thread mspace tlms = 0;
-    void*  tlmalloc(size_t bytes) {
-      if (tlms == 0) tlms = create_mspace(0, 0);
-      return mspace_malloc(tlms, bytes);
-    }
-    void  tlfree(void* mem) { mspace_free(tlms, mem); }
-
-  Unless FOOTERS is defined, each mspace is completely independent.
-  You cannot allocate from one and free to another (although
-  conformance is only weakly checked, so usage errors are not always
-  caught). If FOOTERS is defined, then each chunk carries around a tag
-  indicating its originating mspace, and frees are directed to their
-  originating spaces. Normally, this requires use of locks.
-
- -------------------------  Compile-time options ---------------------------
-
-Be careful in setting #define values for numerical constants of type
-size_t. On some systems, literal values are not automatically extended
-to size_t precision unless they are explicitly casted. You can also
-use the symbolic values MAX_SIZE_T, SIZE_T_ONE, etc below.
-
-WIN32                    default: defined if _WIN32 defined
-  Defining WIN32 sets up defaults for MS environment and compilers.
-  Otherwise defaults are for unix. Beware that there seem to be some
-  cases where this malloc might not be a pure drop-in replacement for
-  Win32 malloc: Random-looking failures from Win32 GDI API's (eg;
-  SetDIBits()) may be due to bugs in some video driver implementations
-  when pixel buffers are malloc()ed, and the region spans more than
-  one VirtualAlloc()ed region. Because dlmalloc uses a small (64Kb)
-  default granularity, pixel buffers may straddle virtual allocation
-  regions more often than when using the Microsoft allocator.  You can
-  avoid this by using VirtualAlloc() and VirtualFree() for all pixel
-  buffers rather than using malloc().  If this is not possible,
-  recompile this malloc with a larger DEFAULT_GRANULARITY. Note:
-  in cases where MSC and gcc (cygwin) are known to differ on WIN32,
-  conditions use _MSC_VER to distinguish them.
-
-DLMALLOC_EXPORT       default: extern
-  Defines how public APIs are declared. If you want to export via a
-  Windows DLL, you might define this as
-    #define DLMALLOC_EXPORT extern  __declspec(dllexport)
-  If you want a POSIX ELF shared object, you might use
-    #define DLMALLOC_EXPORT extern __attribute__((visibility("default")))
-
-MALLOC_ALIGNMENT         default: (size_t)(2 * sizeof(void *))
-  Controls the minimum alignment for malloc'ed chunks.  It must be a
-  power of two and at least 8, even on machines for which smaller
-  alignments would suffice. It may be defined as larger than this
-  though. Note however that code and data structures are optimized for
-  the case of 8-byte alignment.
-
-MSPACES                  default: 0 (false)
-  If true, compile in support for independent allocation spaces.
-  This is only supported if HAVE_MMAP is true.
-
-ONLY_MSPACES             default: 0 (false)
-  If true, only compile in mspace versions, not regular versions.
-
-USE_LOCKS                default: 0 (false)
-  Causes each call to each public routine to be surrounded with
-  pthread or WIN32 mutex lock/unlock. (If set true, this can be
-  overridden on a per-mspace basis for mspace versions.) If set to a
-  non-zero value other than 1, locks are used, but their
-  implementation is left out, so lock functions must be supplied manually,
-  as described below.
-
-USE_SPIN_LOCKS           default: 1 iff USE_LOCKS and spin locks available
-  If true, uses custom spin locks for locking. This is currently
-  supported only gcc >= 4.1, older gccs on x86 platforms, and recent
-  MS compilers.  Otherwise, posix locks or win32 critical sections are
-  used.
-
-USE_RECURSIVE_LOCKS      default: not defined
-  If defined nonzero, uses recursive (aka reentrant) locks, otherwise
-  uses plain mutexes. This is not required for malloc proper, but may
-  be needed for layered allocators such as nedmalloc.
-
-LOCK_AT_FORK            default: not defined
-  If defined nonzero, performs pthread_atfork upon initialization
-  to initialize child lock while holding parent lock. The implementation
-  assumes that pthread locks (not custom locks) are being used. In other
-  cases, you may need to customize the implementation.
-
-FOOTERS                  default: 0
-  If true, provide extra checking and dispatching by placing
-  information in the footers of allocated chunks. This adds
-  space and time overhead.
-
-INSECURE                 default: 0
-  If true, omit checks for usage errors and heap space overwrites.
-
-USE_DL_PREFIX            default: NOT defined
-  Causes compiler to prefix all public routines with the string 'dl'.
-  This can be useful when you only want to use this malloc in one part
-  of a program, using your regular system malloc elsewhere.
-
-MALLOC_INSPECT_ALL       default: NOT defined
-  If defined, compiles malloc_inspect_all and mspace_inspect_all, that
-  perform traversal of all heap space.  Unless access to these
-  functions is otherwise restricted, you probably do not want to
-  include them in secure implementations.
-
-ABORT                    default: defined as abort()
-  Defines how to abort on failed checks.  On most systems, a failed
-  check cannot die with an "assert" or even print an informative
-  message, because the underlying print routines in turn call malloc,
-  which will fail again.  Generally, the best policy is to simply call
-  abort(). It's not very useful to do more than this because many
-  errors due to overwriting will show up as address faults (null, odd
-  addresses etc) rather than malloc-triggered checks, so will also
-  abort.  Also, most compilers know that abort() does not return, so
-  can better optimize code conditionally calling it.
-
-PROCEED_ON_ERROR           default: defined as 0 (false)
-  Controls whether detected bad addresses cause them to bypassed
-  rather than aborting. If set, detected bad arguments to free and
-  realloc are ignored. And all bookkeeping information is zeroed out
-  upon a detected overwrite of freed heap space, thus losing the
-  ability to ever return it from malloc again, but enabling the
-  application to proceed. If PROCEED_ON_ERROR is defined, the
-  static variable malloc_corruption_error_count is compiled in
-  and can be examined to see if errors have occurred. This option
-  generates slower code than the default abort policy.
-
-DEBUG                    default: NOT defined
-  The DEBUG setting is mainly intended for people trying to modify
-  this code or diagnose problems when porting to new platforms.
-  However, it may also be able to better isolate user errors than just
-  using runtime checks.  The assertions in the check routines spell
-  out in more detail the assumptions and invariants underlying the
-  algorithms.  The checking is fairly extensive, and will slow down
-  execution noticeably. Calling malloc_stats or mallinfo with DEBUG
-  set will attempt to check every non-mmapped allocated and free chunk
-  in the course of computing the summaries.
-
-ABORT_ON_ASSERT_FAILURE   default: defined as 1 (true)
-  Debugging assertion failures can be nearly impossible if your
-  version of the assert macro causes malloc to be called, which will
-  lead to a cascade of further failures, blowing the runtime stack.
-  ABORT_ON_ASSERT_FAILURE cause assertions failures to call abort(),
-  which will usually make debugging easier.
-
-MALLOC_FAILURE_ACTION     default: sets errno to ENOMEM, or no-op on win32
-  The action to take before "return 0" when malloc fails to be able to
-  return memory because there is none available.
-
-HAVE_MORECORE             default: 1 (true) unless win32 or ONLY_MSPACES
-  True if this system supports sbrk or an emulation of it.
-
-MORECORE                  default: sbrk
-  The name of the sbrk-style system routine to call to obtain more
-  memory.  See below for guidance on writing custom MORECORE
-  functions. The type of the argument to sbrk/MORECORE varies across
-  systems.  It cannot be size_t, because it supports negative
-  arguments, so it is normally the signed type of the same width as
-  size_t (sometimes declared as "intptr_t").  It doesn't much matter
-  though. Internally, we only call it with arguments less than half
-  the max value of a size_t, which should work across all reasonable
-  possibilities, although sometimes generating compiler warnings.
-
-MORECORE_CONTIGUOUS       default: 1 (true) if HAVE_MORECORE
-  If true, take advantage of fact that consecutive calls to MORECORE
-  with positive arguments always return contiguous increasing
-  addresses.  This is true of unix sbrk. It does not hurt too much to
-  set it true anyway, since malloc copes with non-contiguities.
-  Setting it false when definitely non-contiguous saves time
-  and possibly wasted space it would take to discover this though.
-
-MORECORE_CANNOT_TRIM      default: NOT defined
-  True if MORECORE cannot release space back to the system when given
-  negative arguments. This is generally necessary only if you are
-  using a hand-crafted MORECORE function that cannot handle negative
-  arguments.
-
-NO_SEGMENT_TRAVERSAL       default: 0
-  If non-zero, suppresses traversals of memory segments
-  returned by either MORECORE or CALL_MMAP. This disables
-  merging of segments that are contiguous, and selectively
-  releasing them to the OS if unused, but bounds execution times.
-
-HAVE_MMAP                 default: 1 (true)
-  True if this system supports mmap or an emulation of it.  If so, and
-  HAVE_MORECORE is not true, MMAP is used for all system
-  allocation. If set and HAVE_MORECORE is true as well, MMAP is
-  primarily used to directly allocate very large blocks. It is also
-  used as a backup strategy in cases where MORECORE fails to provide
-  space from system. Note: A single call to MUNMAP is assumed to be
-  able to unmap memory that may have be allocated using multiple calls
-  to MMAP, so long as they are adjacent.
-
-HAVE_MREMAP               default: 1 on linux, else 0
-  If true realloc() uses mremap() to re-allocate large blocks and
-  extend or shrink allocation spaces.
-
-MMAP_CLEARS               default: 1 except on WINCE.
-  True if mmap clears memory so calloc doesn't need to. This is true
-  for standard unix mmap using /dev/zero and on WIN32 except for WINCE.
-
-USE_BUILTIN_FFS            default: 0 (i.e., not used)
-  Causes malloc to use the builtin ffs() function to compute indices.
-  Some compilers may recognize and intrinsify ffs to be faster than the
-  supplied C version. Also, the case of x86 using gcc is special-cased
-  to an asm instruction, so is already as fast as it can be, and so
-  this setting has no effect. Similarly for Win32 under recent MS compilers.
-  (On most x86s, the asm version is only slightly faster than the C version.)
-
-malloc_getpagesize         default: derive from system includes, or 4096.
-  The system page size. To the extent possible, this malloc manages
-  memory from the system in page-size units.  This may be (and
-  usually is) a function rather than a constant. This is ignored
-  if WIN32, where page size is determined using getSystemInfo during
-  initialization.
-
-USE_DEV_RANDOM             default: 0 (i.e., not used)
-  Causes malloc to use /dev/random to initialize secure magic seed for
-  stamping footers. Otherwise, the current time is used.
-
-NO_MALLINFO                default: 0
-  If defined, don't compile "mallinfo". This can be a simple way
-  of dealing with mismatches between system declarations and
-  those in this file.
-
-MALLINFO_FIELD_TYPE        default: size_t
-  The type of the fields in the mallinfo struct. This was originally
-  defined as "int" in SVID etc, but is more usefully defined as
-  size_t. The value is used only if  HAVE_USR_INCLUDE_MALLOC_H is not set
-
-NO_MALLOC_STATS            default: 0
-  If defined, don't compile "malloc_stats". This avoids calls to
-  fprintf and bringing in stdio dependencies you might not want.
-
-REALLOC_ZERO_BYTES_FREES    default: not defined
-  This should be set if a call to realloc with zero bytes should
-  be the same as a call to free. Some people think it should. Otherwise,
-  since this malloc returns a unique pointer for malloc(0), so does
-  realloc(p, 0).
-
-LACKS_UNISTD_H, LACKS_FCNTL_H, LACKS_SYS_PARAM_H, LACKS_SYS_MMAN_H
-LACKS_STRINGS_H, LACKS_STRING_H, LACKS_SYS_TYPES_H,  LACKS_ERRNO_H
-LACKS_STDLIB_H LACKS_SCHED_H LACKS_TIME_H  default: NOT defined unless on WIN32
-  Define these if your system does not have these header files.
-  You might need to manually insert some of the declarations they provide.
-
-DEFAULT_GRANULARITY        default: page size if MORECORE_CONTIGUOUS,
-                                system_info.dwAllocationGranularity in WIN32,
-                                otherwise 64K.
-      Also settable using mallopt(M_GRANULARITY, x)
-  The unit for allocating and deallocating memory from the system.  On
-  most systems with contiguous MORECORE, there is no reason to
-  make this more than a page. However, systems with MMAP tend to
-  either require or encourage larger granularities.  You can increase
-  this value to prevent system allocation functions to be called so
-  often, especially if they are slow.  The value must be at least one
-  page and must be a power of two.  Setting to 0 causes initialization
-  to either page size or win32 region size.  (Note: In previous
-  versions of malloc, the equivalent of this option was called
-  "TOP_PAD")
-
-DEFAULT_TRIM_THRESHOLD    default: 2MB
-      Also settable using mallopt(M_TRIM_THRESHOLD, x)
-  The maximum amount of unused top-most memory to keep before
-  releasing via malloc_trim in free().  Automatic trimming is mainly
-  useful in long-lived programs using contiguous MORECORE.  Because
-  trimming via sbrk can be slow on some systems, and can sometimes be
-  wasteful (in cases where programs immediately afterward allocate
-  more large chunks) the value should be high enough so that your
-  overall system performance would improve by releasing this much
-  memory.  As a rough guide, you might set to a value close to the
-  average size of a process (program) running on your system.
-  Releasing this much memory would allow such a process to run in
-  memory.  Generally, it is worth tuning trim thresholds when a
-  program undergoes phases where several large chunks are allocated
-  and released in ways that can reuse each other's storage, perhaps
-  mixed with phases where there are no such chunks at all. The trim
-  value must be greater than page size to have any useful effect.  To
-  disable trimming completely, you can set to MAX_SIZE_T. Note that the trick
-  some people use of mallocing a huge space and then freeing it at
-  program startup, in an attempt to reserve system memory, doesn't
-  have the intended effect under automatic trimming, since that memory
-  will immediately be returned to the system.
-
-DEFAULT_MMAP_THRESHOLD       default: 256K
-      Also settable using mallopt(M_MMAP_THRESHOLD, x)
-  The request size threshold for using MMAP to directly service a
-  request. Requests of at least this size that cannot be allocated
-  using already-existing space will be serviced via mmap.  (If enough
-  normal freed space already exists it is used instead.)  Using mmap
-  segregates relatively large chunks of memory so that they can be
-  individually obtained and released from the host system. A request
-  serviced through mmap is never reused by any other request (at least
-  not directly; the system may just so happen to remap successive
-  requests to the same locations).  Segregating space in this way has
-  the benefits that: Mmapped space can always be individually released
-  back to the system, which helps keep the system level memory demands
-  of a long-lived program low.  Also, mapped memory doesn't become
-  `locked' between other chunks, as can happen with normally allocated
-  chunks, which means that even trimming via malloc_trim would not
-  release them.  However, it has the disadvantage that the space
-  cannot be reclaimed, consolidated, and then used to service later
-  requests, as happens with normal chunks.  The advantages of mmap
-  nearly always outweigh disadvantages for "large" chunks, but the
-  value of "large" may vary across systems.  The default is an
-  empirically derived value that works well in most systems. You can
-  disable mmap by setting to MAX_SIZE_T.
-
-MAX_RELEASE_CHECK_RATE   default: 4095 unless not HAVE_MMAP
-  The number of consolidated frees between checks to release
-  unused segments when freeing. When using non-contiguous segments,
-  especially with multiple mspaces, checking only for topmost space
-  doesn't always suffice to trigger trimming. To compensate for this,
-  free() will, with a period of MAX_RELEASE_CHECK_RATE (or the
-  current number of segments, if greater) try to release unused
-  segments to the OS when freeing chunks that result in
-  consolidation. The best value for this parameter is a compromise
-  between slowing down frees with relatively costly checks that
-  rarely trigger versus holding on to unused memory. To effectively
-  disable, set to MAX_SIZE_T. This may lead to a very slight speed
-  improvement at the expense of carrying around more memory.
-*/
-
-/* Version identifier to allow people to support multiple versions */
-#ifndef DLMALLOC_VERSION
-#define DLMALLOC_VERSION 20806
-#endif /* DLMALLOC_VERSION */
-
-#ifndef DLMALLOC_EXPORT
-#define DLMALLOC_EXPORT extern
-#endif
-
-#ifndef WIN32
-#ifdef _WIN32
-#define WIN32 1
-#endif  /* _WIN32 */
-#ifdef _WIN32_WCE
-#define LACKS_FCNTL_H
-#define WIN32 1
-#endif /* _WIN32_WCE */
-#endif  /* WIN32 */
-#ifdef WIN32
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <tchar.h>
-#define HAVE_MMAP 1
-#define HAVE_MORECORE 0
-#define LACKS_UNISTD_H
-#define LACKS_SYS_PARAM_H
-#define LACKS_SYS_MMAN_H
-#define LACKS_STRING_H
-#define LACKS_STRINGS_H
-#define LACKS_SYS_TYPES_H
-#define LACKS_ERRNO_H
-#define LACKS_SCHED_H
-#ifndef MALLOC_FAILURE_ACTION
-#define MALLOC_FAILURE_ACTION
-#endif /* MALLOC_FAILURE_ACTION */
-#ifndef MMAP_CLEARS
-#ifdef _WIN32_WCE /* WINCE reportedly does not clear */
-#define MMAP_CLEARS 0
-#else
-#define MMAP_CLEARS 1
-#endif /* _WIN32_WCE */
-#endif /*MMAP_CLEARS */
-#endif  /* WIN32 */
-
-#if defined(DARWIN) || defined(_DARWIN)
-/* Mac OSX docs advise not to use sbrk; it seems better to use mmap */
-#ifndef HAVE_MORECORE
-#define HAVE_MORECORE 0
-#define HAVE_MMAP 1
-/* OSX allocators provide 16 byte alignment */
-#ifndef MALLOC_ALIGNMENT
-#define MALLOC_ALIGNMENT ((size_t)16U)
-#endif
-#endif  /* HAVE_MORECORE */
-#endif  /* DARWIN */
-
-#ifndef LACKS_SYS_TYPES_H
-#include <sys/types.h>  /* For size_t */
-#endif  /* LACKS_SYS_TYPES_H */
-
-/* The maximum possible size_t value has all bits set */
-#define MAX_SIZE_T           (~(size_t)0)
-
-#ifndef USE_LOCKS /* ensure true if spin or recursive locks set */
-#define USE_LOCKS  ((defined(USE_SPIN_LOCKS) && USE_SPIN_LOCKS != 0) || \
-                    (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0))
-#endif /* USE_LOCKS */
-
-#if USE_LOCKS /* Spin locks for gcc >= 4.1, older gcc on x86, MSC >= 1310 */
-#if ((defined(__GNUC__) &&                                              \
-      ((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) ||      \
-       defined(__i386__) || defined(__x86_64__))) ||                    \
-     (defined(_MSC_VER) && _MSC_VER>=1310))
-#ifndef USE_SPIN_LOCKS
-#define USE_SPIN_LOCKS 1
-#endif /* USE_SPIN_LOCKS */
-#elif USE_SPIN_LOCKS
-#error "USE_SPIN_LOCKS defined without implementation"
-#endif /* ... locks available... */
-#elif !defined(USE_SPIN_LOCKS)
-#define USE_SPIN_LOCKS 0
-#endif /* USE_LOCKS */
-
-#ifndef ONLY_MSPACES
-#define ONLY_MSPACES 0
-#endif  /* ONLY_MSPACES */
-#ifndef MSPACES
-#if ONLY_MSPACES
-#define MSPACES 1
-#else   /* ONLY_MSPACES */
-#define MSPACES 0
-#endif  /* ONLY_MSPACES */
-#endif  /* MSPACES */
-#ifndef MALLOC_ALIGNMENT
-#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *)))
-#endif  /* MALLOC_ALIGNMENT */
-#ifndef FOOTERS
-#define FOOTERS 0
-#endif  /* FOOTERS */
-#ifndef ABORT
-#define ABORT  abort()
-#endif  /* ABORT */
-#ifndef ABORT_ON_ASSERT_FAILURE
-#define ABORT_ON_ASSERT_FAILURE 1
-#endif  /* ABORT_ON_ASSERT_FAILURE */
-#ifndef PROCEED_ON_ERROR
-#define PROCEED_ON_ERROR 0
-#endif  /* PROCEED_ON_ERROR */
-
-#ifndef INSECURE
-#define INSECURE 0
-#endif  /* INSECURE */
-#ifndef MALLOC_INSPECT_ALL
-#define MALLOC_INSPECT_ALL 0
-#endif  /* MALLOC_INSPECT_ALL */
-#ifndef HAVE_MMAP
-#define HAVE_MMAP 1
-#endif  /* HAVE_MMAP */
-#ifndef MMAP_CLEARS
-#define MMAP_CLEARS 1
-#endif  /* MMAP_CLEARS */
-#ifndef HAVE_MREMAP
-#ifdef linux
-#define HAVE_MREMAP 1
-#define _GNU_SOURCE /* Turns on mremap() definition */
-#else   /* linux */
-#define HAVE_MREMAP 0
-#endif  /* linux */
-#endif  /* HAVE_MREMAP */
-#ifndef MALLOC_FAILURE_ACTION
-#define MALLOC_FAILURE_ACTION  errno = ENOMEM;
-#endif  /* MALLOC_FAILURE_ACTION */
-#ifndef HAVE_MORECORE
-#if ONLY_MSPACES
-#define HAVE_MORECORE 0
-#else   /* ONLY_MSPACES */
-#define HAVE_MORECORE 1
-#endif  /* ONLY_MSPACES */
-#endif  /* HAVE_MORECORE */
-#if !HAVE_MORECORE
-#define MORECORE_CONTIGUOUS 0
-#else   /* !HAVE_MORECORE */
-#define MORECORE_DEFAULT sbrk
-#ifndef MORECORE_CONTIGUOUS
-#define MORECORE_CONTIGUOUS 1
-#endif  /* MORECORE_CONTIGUOUS */
-#endif  /* HAVE_MORECORE */
-#ifndef DEFAULT_GRANULARITY
-#if (MORECORE_CONTIGUOUS || defined(WIN32))
-#define DEFAULT_GRANULARITY (0)  /* 0 means to compute in init_mparams */
-#else   /* MORECORE_CONTIGUOUS */
-#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U)
-#endif  /* MORECORE_CONTIGUOUS */
-#endif  /* DEFAULT_GRANULARITY */
-#ifndef DEFAULT_TRIM_THRESHOLD
-#ifndef MORECORE_CANNOT_TRIM
-#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U)
-#else   /* MORECORE_CANNOT_TRIM */
-#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T
-#endif  /* MORECORE_CANNOT_TRIM */
-#endif  /* DEFAULT_TRIM_THRESHOLD */
-#ifndef DEFAULT_MMAP_THRESHOLD
-#if HAVE_MMAP
-#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U)
-#else   /* HAVE_MMAP */
-#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
-#endif  /* HAVE_MMAP */
-#endif  /* DEFAULT_MMAP_THRESHOLD */
-#ifndef MAX_RELEASE_CHECK_RATE
-#if HAVE_MMAP
-#define MAX_RELEASE_CHECK_RATE 4095
-#else
-#define MAX_RELEASE_CHECK_RATE MAX_SIZE_T
-#endif /* HAVE_MMAP */
-#endif /* MAX_RELEASE_CHECK_RATE */
-#ifndef USE_BUILTIN_FFS
-#define USE_BUILTIN_FFS 0
-#endif  /* USE_BUILTIN_FFS */
-#ifndef USE_DEV_RANDOM
-#define USE_DEV_RANDOM 0
-#endif  /* USE_DEV_RANDOM */
-#ifndef NO_MALLINFO
-#define NO_MALLINFO 0
-#endif  /* NO_MALLINFO */
-#ifndef MALLINFO_FIELD_TYPE
-#define MALLINFO_FIELD_TYPE size_t
-#endif  /* MALLINFO_FIELD_TYPE */
-#ifndef NO_MALLOC_STATS
-#define NO_MALLOC_STATS 0
-#endif  /* NO_MALLOC_STATS */
-#ifndef NO_SEGMENT_TRAVERSAL
-#define NO_SEGMENT_TRAVERSAL 0
-#endif /* NO_SEGMENT_TRAVERSAL */
-
-/*
-  mallopt tuning options.  SVID/XPG defines four standard parameter
-  numbers for mallopt, normally defined in malloc.h.  None of these
-  are used in this malloc, so setting them has no effect. But this
-  malloc does support the following options.
-*/
-
-#define M_TRIM_THRESHOLD     (-1)
-#define M_GRANULARITY        (-2)
-#define M_MMAP_THRESHOLD     (-3)
-
-/* ------------------------ Mallinfo declarations ------------------------ */
-
-#if !NO_MALLINFO
-/*
-  This version of malloc supports the standard SVID/XPG mallinfo
-  routine that returns a struct containing usage properties and
-  statistics. It should work on any system that has a
-  /usr/include/malloc.h defining struct mallinfo.  The main
-  declaration needed is the mallinfo struct that is returned (by-copy)
-  by mallinfo().  The malloinfo struct contains a bunch of fields that
-  are not even meaningful in this version of malloc.  These fields are
-  are instead filled by mallinfo() with other numbers that might be of
-  interest.
-
-  HAVE_USR_INCLUDE_MALLOC_H should be set if you have a
-  /usr/include/malloc.h file that includes a declaration of struct
-  mallinfo.  If so, it is included; else a compliant version is
-  declared below.  These must be precisely the same for mallinfo() to
-  work.  The original SVID version of this struct, defined on most
-  systems with mallinfo, declares all fields as ints. But some others
-  define as unsigned long. If your system defines the fields using a
-  type of different width than listed here, you MUST #include your
-  system version and #define HAVE_USR_INCLUDE_MALLOC_H.
-*/
-
-/* #define HAVE_USR_INCLUDE_MALLOC_H */
-
-#ifdef HAVE_USR_INCLUDE_MALLOC_H
-#include "/usr/include/malloc.h"
-#else /* HAVE_USR_INCLUDE_MALLOC_H */
-#ifndef STRUCT_MALLINFO_DECLARED
-/* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is defined */
-#define _STRUCT_MALLINFO
-#define STRUCT_MALLINFO_DECLARED 1
-struct mallinfo {
-  MALLINFO_FIELD_TYPE arena;    /* non-mmapped space allocated from system */
-  MALLINFO_FIELD_TYPE ordblks;  /* number of free chunks */
-  MALLINFO_FIELD_TYPE smblks;   /* always 0 */
-  MALLINFO_FIELD_TYPE hblks;    /* always 0 */
-  MALLINFO_FIELD_TYPE hblkhd;   /* space in mmapped regions */
-  MALLINFO_FIELD_TYPE usmblks;  /* maximum total allocated space */
-  MALLINFO_FIELD_TYPE fsmblks;  /* always 0 */
-  MALLINFO_FIELD_TYPE uordblks; /* total allocated space */
-  MALLINFO_FIELD_TYPE fordblks; /* total free space */
-  MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */
-};
-#endif /* STRUCT_MALLINFO_DECLARED */
-#endif /* HAVE_USR_INCLUDE_MALLOC_H */
-#endif /* NO_MALLINFO */
-
-/*
-  Try to persuade compilers to inline. The most critical functions for
-  inlining are defined as macros, so these aren't used for them.
-*/
-
-#ifndef FORCEINLINE
-  #if defined(__GNUC__)
-#define FORCEINLINE __inline __attribute__ ((always_inline))
-  #elif defined(_MSC_VER)
-    #define FORCEINLINE __forceinline
-  #endif
-#endif
-#ifndef NOINLINE
-  #if defined(__GNUC__)
-    #define NOINLINE __attribute__ ((noinline))
-  #elif defined(_MSC_VER)
-    #define NOINLINE __declspec(noinline)
-  #else
-    #define NOINLINE
-  #endif
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#ifndef FORCEINLINE
- #define FORCEINLINE inline
-#endif
-#endif /* __cplusplus */
-#ifndef FORCEINLINE
- #define FORCEINLINE
-#endif
-
-#if !ONLY_MSPACES
-
-/* ------------------- Declarations of public routines ------------------- */
-
-#ifndef USE_DL_PREFIX
-#define dlcalloc               calloc
-#define dlfree                 free
-#define dlmalloc               malloc
-#define dlmemalign             memalign
-#define dlposix_memalign       posix_memalign
-#define dlrealloc              realloc
-#define dlrealloc_in_place     realloc_in_place
-#define dlvalloc               valloc
-#define dlpvalloc              pvalloc
-#define dlmallinfo             mallinfo
-#define dlmallopt              mallopt
-#define dlmalloc_trim          malloc_trim
-#define dlmalloc_stats         malloc_stats
-#define dlmalloc_usable_size   malloc_usable_size
-#define dlmalloc_footprint     malloc_footprint
-#define dlmalloc_max_footprint malloc_max_footprint
-#define dlmalloc_footprint_limit malloc_footprint_limit
-#define dlmalloc_set_footprint_limit malloc_set_footprint_limit
-#define dlmalloc_inspect_all   malloc_inspect_all
-#define dlindependent_calloc   independent_calloc
-#define dlindependent_comalloc independent_comalloc
-#define dlbulk_free            bulk_free
-#endif /* USE_DL_PREFIX */
-
-/*
-  malloc(size_t n)
-  Returns a pointer to a newly allocated chunk of at least n bytes, or
-  null if no space is available, in which case errno is set to ENOMEM
-  on ANSI C systems.
-
-  If n is zero, malloc returns a minimum-sized chunk. (The minimum
-  size is 16 bytes on most 32bit systems, and 32 bytes on 64bit
-  systems.)  Note that size_t is an unsigned type, so calls with
-  arguments that would be negative if signed are interpreted as
-  requests for huge amounts of space, which will often fail. The
-  maximum supported value of n differs across systems, but is in all
-  cases less than the maximum representable value of a size_t.
-*/
-DLMALLOC_EXPORT void* dlmalloc(size_t);
-
-/*
-  free(void* p)
-  Releases the chunk of memory pointed to by p, that had been previously
-  allocated using malloc or a related routine such as realloc.
-  It has no effect if p is null. If p was not malloced or already
-  freed, free(p) will by default cause the current program to abort.
-*/
-DLMALLOC_EXPORT void  dlfree(void*);
-
-/*
-  calloc(size_t n_elements, size_t element_size);
-  Returns a pointer to n_elements * element_size bytes, with all locations
-  set to zero.
-*/
-DLMALLOC_EXPORT void* dlcalloc(size_t, size_t);
-
-/*
-  realloc(void* p, size_t n)
-  Returns a pointer to a chunk of size n that contains the same data
-  as does chunk p up to the minimum of (n, p's size) bytes, or null
-  if no space is available.
-
-  The returned pointer may or may not be the same as p. The algorithm
-  prefers extending p in most cases when possible, otherwise it
-  employs the equivalent of a malloc-copy-free sequence.
-
-  If p is null, realloc is equivalent to malloc.
-
-  If space is not available, realloc returns null, errno is set (if on
-  ANSI) and p is NOT freed.
-
-  if n is for fewer bytes than already held by p, the newly unused
-  space is lopped off and freed if possible.  realloc with a size
-  argument of zero (re)allocates a minimum-sized chunk.
-
-  The old unix realloc convention of allowing the last-free'd chunk
-  to be used as an argument to realloc is not supported.
-*/
-DLMALLOC_EXPORT void* dlrealloc(void*, size_t);
-
-/*
-  realloc_in_place(void* p, size_t n)
-  Resizes the space allocated for p to size n, only if this can be
-  done without moving p (i.e., only if there is adjacent space
-  available if n is greater than p's current allocated size, or n is
-  less than or equal to p's size). This may be used instead of plain
-  realloc if an alternative allocation strategy is needed upon failure
-  to expand space; for example, reallocation of a buffer that must be
-  memory-aligned or cleared. You can use realloc_in_place to trigger
-  these alternatives only when needed.
-
-  Returns p if successful; otherwise null.
-*/
-DLMALLOC_EXPORT void* dlrealloc_in_place(void*, size_t);
-
-/*
-  memalign(size_t alignment, size_t n);
-  Returns a pointer to a newly allocated chunk of n bytes, aligned
-  in accord with the alignment argument.
-
-  The alignment argument should be a power of two. If the argument is
-  not a power of two, the nearest greater power is used.
-  8-byte alignment is guaranteed by normal malloc calls, so don't
-  bother calling memalign with an argument of 8 or less.
-
-  Overreliance on memalign is a sure way to fragment space.
-*/
-DLMALLOC_EXPORT void* dlmemalign(size_t, size_t);
-
-/*
-  int posix_memalign(void** pp, size_t alignment, size_t n);
-  Allocates a chunk of n bytes, aligned in accord with the alignment
-  argument. Differs from memalign only in that it (1) assigns the
-  allocated memory to *pp rather than returning it, (2) fails and
-  returns EINVAL if the alignment is not a power of two (3) fails and
-  returns ENOMEM if memory cannot be allocated.
-*/
-DLMALLOC_EXPORT int dlposix_memalign(void**, size_t, size_t);
-
-/*
-  valloc(size_t n);
-  Equivalent to memalign(pagesize, n), where pagesize is the page
-  size of the system. If the pagesize is unknown, 4096 is used.
-*/
-DLMALLOC_EXPORT void* dlvalloc(size_t);
-
-/*
-  mallopt(int parameter_number, int parameter_value)
-  Sets tunable parameters The format is to provide a
-  (parameter-number, parameter-value) pair.  mallopt then sets the
-  corresponding parameter to the argument value if it can (i.e., so
-  long as the value is meaningful), and returns 1 if successful else
-  0.  To workaround the fact that mallopt is specified to use int,
-  not size_t parameters, the value -1 is specially treated as the
-  maximum unsigned size_t value.
-
-  SVID/XPG/ANSI defines four standard param numbers for mallopt,
-  normally defined in malloc.h.  None of these are use in this malloc,
-  so setting them has no effect. But this malloc also supports other
-  options in mallopt. See below for details.  Briefly, supported
-  parameters are as follows (listed defaults are for "typical"
-  configurations).
-
-  Symbol            param #  default    allowed param values
-  M_TRIM_THRESHOLD     -1   2*1024*1024   any   (-1 disables)
-  M_GRANULARITY        -2     page size   any power of 2 >= page size
-  M_MMAP_THRESHOLD     -3      256*1024   any   (or 0 if no MMAP support)
-*/
-DLMALLOC_EXPORT int dlmallopt(int, int);
-
-/*
-  malloc_footprint();
-  Returns the number of bytes obtained from the system.  The total
-  number of bytes allocated by malloc, realloc etc., is less than this
-  value. Unlike mallinfo, this function returns only a precomputed
-  result, so can be called frequently to monitor memory consumption.
-  Even if locks are otherwise defined, this function does not use them,
-  so results might not be up to date.
-*/
-DLMALLOC_EXPORT size_t dlmalloc_footprint(void);
-
-/*
-  malloc_max_footprint();
-  Returns the maximum number of bytes obtained from the system. This
-  value will be greater than current footprint if deallocated space
-  has been reclaimed by the system. The peak number of bytes allocated
-  by malloc, realloc etc., is less than this value. Unlike mallinfo,
-  this function returns only a precomputed result, so can be called
-  frequently to monitor memory consumption.  Even if locks are
-  otherwise defined, this function does not use them, so results might
-  not be up to date.
-*/
-DLMALLOC_EXPORT size_t dlmalloc_max_footprint(void);
-
-/*
-  malloc_footprint_limit();
-  Returns the number of bytes that the heap is allowed to obtain from
-  the system, returning the last value returned by
-  malloc_set_footprint_limit, or the maximum size_t value if
-  never set. The returned value reflects a permission. There is no
-  guarantee that this number of bytes can actually be obtained from
-  the system.
-*/
-DLMALLOC_EXPORT size_t dlmalloc_footprint_limit();
-
-/*
-  malloc_set_footprint_limit();
-  Sets the maximum number of bytes to obtain from the system, causing
-  failure returns from malloc and related functions upon attempts to
-  exceed this value. The argument value may be subject to page
-  rounding to an enforceable limit; this actual value is returned.
-  Using an argument of the maximum possible size_t effectively
-  disables checks. If the argument is less than or equal to the
-  current malloc_footprint, then all future allocations that require
-  additional system memory will fail. However, invocation cannot
-  retroactively deallocate existing used memory.
-*/
-DLMALLOC_EXPORT size_t dlmalloc_set_footprint_limit(size_t bytes);
-
-#if MALLOC_INSPECT_ALL
-/*
-  malloc_inspect_all(void(*handler)(void *start,
-                                    void *end,
-                                    size_t used_bytes,
-                                    void* callback_arg),
-                      void* arg);
-  Traverses the heap and calls the given handler for each managed
-  region, skipping all bytes that are (or may be) used for bookkeeping
-  purposes.  Traversal does not include include chunks that have been
-  directly memory mapped. Each reported region begins at the start
-  address, and continues up to but not including the end address.  The
-  first used_bytes of the region contain allocated data. If
-  used_bytes is zero, the region is unallocated. The handler is
-  invoked with the given callback argument. If locks are defined, they
-  are held during the entire traversal. It is a bad idea to invoke
-  other malloc functions from within the handler.
-
-  For example, to count the number of in-use chunks with size greater
-  than 1000, you could write:
-  static int count = 0;
-  void count_chunks(void* start, void* end, size_t used, void* arg) {
-    if (used >= 1000) ++count;
-  }
-  then:
-    malloc_inspect_all(count_chunks, NULL);
-
-  malloc_inspect_all is compiled only if MALLOC_INSPECT_ALL is defined.
-*/
-DLMALLOC_EXPORT void dlmalloc_inspect_all(void(*handler)(void*, void *, size_t, void*),
-                           void* arg);
-
-#endif /* MALLOC_INSPECT_ALL */
-
-#if !NO_MALLINFO
-/*
-  mallinfo()
-  Returns (by copy) a struct containing various summary statistics:
-
-  arena:     current total non-mmapped bytes allocated from system
-  ordblks:   the number of free chunks
-  smblks:    always zero.
-  hblks:     current number of mmapped regions
-  hblkhd:    total bytes held in mmapped regions
-  usmblks:   the maximum total allocated space. This will be greater
-                than current total if trimming has occurred.
-  fsmblks:   always zero
-  uordblks:  current total allocated space (normal or mmapped)
-  fordblks:  total free space
-  keepcost:  the maximum number of bytes that could ideally be released
-               back to system via malloc_trim. ("ideally" means that
-               it ignores page restrictions etc.)
-
-  Because these fields are ints, but internal bookkeeping may
-  be kept as longs, the reported values may wrap around zero and
-  thus be inaccurate.
-*/
-DLMALLOC_EXPORT struct mallinfo dlmallinfo(void);
-#endif /* NO_MALLINFO */
-
-/*
-  independent_calloc(size_t n_elements, size_t element_size, void* chunks[]);
-
-  independent_calloc is similar to calloc, but instead of returning a
-  single cleared space, it returns an array of pointers to n_elements
-  independent elements that can hold contents of size elem_size, each
-  of which starts out cleared, and can be independently freed,
-  realloc'ed etc. The elements are guaranteed to be adjacently
-  allocated (this is not guaranteed to occur with multiple callocs or
-  mallocs), which may also improve cache locality in some
-  applications.
-
-  The "chunks" argument is optional (i.e., may be null, which is
-  probably the most typical usage). If it is null, the returned array
-  is itself dynamically allocated and should also be freed when it is
-  no longer needed. Otherwise, the chunks array must be of at least
-  n_elements in length. It is filled in with the pointers to the
-  chunks.
-
-  In either case, independent_calloc returns this pointer array, or
-  null if the allocation failed.  If n_elements is zero and "chunks"
-  is null, it returns a chunk representing an array with zero elements
-  (which should be freed if not wanted).
-
-  Each element must be freed when it is no longer needed. This can be
-  done all at once using bulk_free.
-
-  independent_calloc simplifies and speeds up implementations of many
-  kinds of pools.  It may also be useful when constructing large data
-  structures that initially have a fixed number of fixed-sized nodes,
-  but the number is not known at compile time, and some of the nodes
-  may later need to be freed. For example:
-
-  struct Node { int item; struct Node* next; };
-
-  struct Node* build_list() {
-    struct Node** pool;
-    int n = read_number_of_nodes_needed();
-    if (n <= 0) return 0;
-    pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0);
-    if (pool == 0) die();
-    // organize into a linked list...
-    struct Node* first = pool[0];
-    for (i = 0; i < n-1; ++i)
-      pool[i]->next = pool[i+1];
-    free(pool);     // Can now free the array (or not, if it is needed later)
-    return first;
-  }
-*/
-DLMALLOC_EXPORT void** dlindependent_calloc(size_t, size_t, void**);
-
-/*
-  independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
-
-  independent_comalloc allocates, all at once, a set of n_elements
-  chunks with sizes indicated in the "sizes" array.    It returns
-  an array of pointers to these elements, each of which can be
-  independently freed, realloc'ed etc. The elements are guaranteed to
-  be adjacently allocated (this is not guaranteed to occur with
-  multiple callocs or mallocs), which may also improve cache locality
-  in some applications.
-
-  The "chunks" argument is optional (i.e., may be null). If it is null
-  the returned array is itself dynamically allocated and should also
-  be freed when it is no longer needed. Otherwise, the chunks array
-  must be of at least n_elements in length. It is filled in with the
-  pointers to the chunks.
-
-  In either case, independent_comalloc returns this pointer array, or
-  null if the allocation failed.  If n_elements is zero and chunks is
-  null, it returns a chunk representing an array with zero elements
-  (which should be freed if not wanted).
-
-  Each element must be freed when it is no longer needed. This can be
-  done all at once using bulk_free.
-
-  independent_comallac differs from independent_calloc in that each
-  element may have a different size, and also that it does not
-  automatically clear elements.
-
-  independent_comalloc can be used to speed up allocation in cases
-  where several structs or objects must always be allocated at the
-  same time.  For example:
-
-  struct Head { ... }
-  struct Foot { ... }
-
-  void send_message(char* msg) {
-    int msglen = strlen(msg);
-    size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) };
-    void* chunks[3];
-    if (independent_comalloc(3, sizes, chunks) == 0)
-      die();
-    struct Head* head = (struct Head*)(chunks[0]);
-    char*        body = (char*)(chunks[1]);
-    struct Foot* foot = (struct Foot*)(chunks[2]);
-    // ...
-  }
-
-  In general though, independent_comalloc is worth using only for
-  larger values of n_elements. For small values, you probably won't
-  detect enough difference from series of malloc calls to bother.
-
-  Overuse of independent_comalloc can increase overall memory usage,
-  since it cannot reuse existing noncontiguous small chunks that
-  might be available for some of the elements.
-*/
-DLMALLOC_EXPORT void** dlindependent_comalloc(size_t, size_t*, void**);
-
-/*
-  bulk_free(void* array[], size_t n_elements)
-  Frees and clears (sets to null) each non-null pointer in the given
-  array.  This is likely to be faster than freeing them one-by-one.
-  If footers are used, pointers that have been allocated in different
-  mspaces are not freed or cleared, and the count of all such pointers
-  is returned.  For large arrays of pointers with poor locality, it
-  may be worthwhile to sort this array before calling bulk_free.
-*/
-DLMALLOC_EXPORT size_t  dlbulk_free(void**, size_t n_elements);
-
-/*
-  pvalloc(size_t n);
-  Equivalent to valloc(minimum-page-that-holds(n)), that is,
-  round up n to nearest pagesize.
- */
-DLMALLOC_EXPORT void*  dlpvalloc(size_t);
-
-/*
-  malloc_trim(size_t pad);
-
-  If possible, gives memory back to the system (via negative arguments
-  to sbrk) if there is unused memory at the `high' end of the malloc
-  pool or in unused MMAP segments. You can call this after freeing
-  large blocks of memory to potentially reduce the system-level memory
-  requirements of a program. However, it cannot guarantee to reduce
-  memory. Under some allocation patterns, some large free blocks of
-  memory will be locked between two used chunks, so they cannot be
-  given back to the system.
-
-  The `pad' argument to malloc_trim represents the amount of free
-  trailing space to leave untrimmed. If this argument is zero, only
-  the minimum amount of memory to maintain internal data structures
-  will be left. Non-zero arguments can be supplied to maintain enough
-  trailing space to service future expected allocations without having
-  to re-obtain memory from the system.
-
-  Malloc_trim returns 1 if it actually released any memory, else 0.
-*/
-DLMALLOC_EXPORT int  dlmalloc_trim(size_t);
-
-/*
-  malloc_stats();
-  Prints on stderr the amount of space obtained from the system (both
-  via sbrk and mmap), the maximum amount (which may be more than
-  current if malloc_trim and/or munmap got called), and the current
-  number of bytes allocated via malloc (or realloc, etc) but not yet
-  freed. Note that this is the number of bytes allocated, not the
-  number requested. It will be larger than the number requested
-  because of alignment and bookkeeping overhead. Because it includes
-  alignment wastage as being in use, this figure may be greater than
-  zero even when no user-level chunks are allocated.
-
-  The reported current and maximum system memory can be inaccurate if
-  a program makes other calls to system memory allocation functions
-  (normally sbrk) outside of malloc.
-
-  malloc_stats prints only the most commonly interesting statistics.
-  More information can be obtained by calling mallinfo.
-*/
-DLMALLOC_EXPORT void  dlmalloc_stats(void);
-
-/*
-  malloc_usable_size(void* p);
-
-  Returns the number of bytes you can actually use in
-  an allocated chunk, which may be more than you requested (although
-  often not) due to alignment and minimum size constraints.
-  You can use this many bytes without worrying about
-  overwriting other allocated objects. This is not a particularly great
-  programming practice. malloc_usable_size can be more useful in
-  debugging and assertions, for example:
-
-  p = malloc(n);
-  assert(malloc_usable_size(p) >= 256);
-*/
-size_t dlmalloc_usable_size(void*);
-
-#endif /* ONLY_MSPACES */
-
-#if MSPACES
-
-/*
-  mspace is an opaque type representing an independent
-  region of space that supports mspace_malloc, etc.
-*/
-typedef void* mspace;
-
-/*
-  create_mspace creates and returns a new independent space with the
-  given initial capacity, or, if 0, the default granularity size.  It
-  returns null if there is no system memory available to create the
-  space.  If argument locked is non-zero, the space uses a separate
-  lock to control access. The capacity of the space will grow
-  dynamically as needed to service mspace_malloc requests.  You can
-  control the sizes of incremental increases of this space by
-  compiling with a different DEFAULT_GRANULARITY or dynamically
-  setting with mallopt(M_GRANULARITY, value).
-*/
-DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked);
-
-/*
-  destroy_mspace destroys the given space, and attempts to return all
-  of its memory back to the system, returning the total number of
-  bytes freed. After destruction, the results of access to all memory
-  used by the space become undefined.
-*/
-DLMALLOC_EXPORT size_t destroy_mspace(mspace msp);
-
-/*
-  create_mspace_with_base uses the memory supplied as the initial base
-  of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this
-  space is used for bookkeeping, so the capacity must be at least this
-  large. (Otherwise 0 is returned.) When this initial space is
-  exhausted, additional memory will be obtained from the system.
-  Destroying this space will deallocate all additionally allocated
-  space (if possible) but not the initial base.
-*/
-DLMALLOC_EXPORT mspace create_mspace_with_base(void* base, size_t capacity, int locked);
-
-/*
-  mspace_track_large_chunks controls whether requests for large chunks
-  are allocated in their own untracked mmapped regions, separate from
-  others in this mspace. By default large chunks are not tracked,
-  which reduces fragmentation. However, such chunks are not
-  necessarily released to the system upon destroy_mspace.  Enabling
-  tracking by setting to true may increase fragmentation, but avoids
-  leakage when relying on destroy_mspace to release all memory
-  allocated using this space.  The function returns the previous
-  setting.
-*/
-DLMALLOC_EXPORT int mspace_track_large_chunks(mspace msp, int enable);
-
-
-/*
-  mspace_malloc behaves as malloc, but operates within
-  the given space.
-*/
-DLMALLOC_EXPORT void* mspace_malloc(mspace msp, size_t bytes);
-
-/*
-  mspace_free behaves as free, but operates within
-  the given space.
-
-  If compiled with FOOTERS==1, mspace_free is not actually needed.
-  free may be called instead of mspace_free because freed chunks from
-  any space are handled by their originating spaces.
-*/
-DLMALLOC_EXPORT void mspace_free(mspace msp, void* mem);
-
-/*
-  mspace_realloc behaves as realloc, but operates within
-  the given space.
-
-  If compiled with FOOTERS==1, mspace_realloc is not actually
-  needed.  realloc may be called instead of mspace_realloc because
-  realloced chunks from any space are handled by their originating
-  spaces.
-*/
-DLMALLOC_EXPORT void* mspace_realloc(mspace msp, void* mem, size_t newsize);
-
-/*
-  mspace_calloc behaves as calloc, but operates within
-  the given space.
-*/
-DLMALLOC_EXPORT void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size);
-
-/*
-  mspace_memalign behaves as memalign, but operates within
-  the given space.
-*/
-DLMALLOC_EXPORT void* mspace_memalign(mspace msp, size_t alignment, size_t bytes);
-
-/*
-  mspace_independent_calloc behaves as independent_calloc, but
-  operates within the given space.
-*/
-DLMALLOC_EXPORT void** mspace_independent_calloc(mspace msp, size_t n_elements,
-                                 size_t elem_size, void* chunks[]);
-
-/*
-  mspace_independent_comalloc behaves as independent_comalloc, but
-  operates within the given space.
-*/
-DLMALLOC_EXPORT void** mspace_independent_comalloc(mspace msp, size_t n_elements,
-                                   size_t sizes[], void* chunks[]);
-
-/*
-  mspace_footprint() returns the number of bytes obtained from the
-  system for this space.
-*/
-DLMALLOC_EXPORT size_t mspace_footprint(mspace msp);
-
-/*
-  mspace_max_footprint() returns the peak number of bytes obtained from the
-  system for this space.
-*/
-DLMALLOC_EXPORT size_t mspace_max_footprint(mspace msp);
-
-
-#if !NO_MALLINFO
-/*
-  mspace_mallinfo behaves as mallinfo, but reports properties of
-  the given space.
-*/
-DLMALLOC_EXPORT struct mallinfo mspace_mallinfo(mspace msp);
-#endif /* NO_MALLINFO */
-
-/*
-  malloc_usable_size(void* p) behaves the same as malloc_usable_size;
-*/
-DLMALLOC_EXPORT size_t mspace_usable_size(const void* mem);
-
-/*
-  mspace_malloc_stats behaves as malloc_stats, but reports
-  properties of the given space.
-*/
-DLMALLOC_EXPORT void mspace_malloc_stats(mspace msp);
-
-/*
-  mspace_trim behaves as malloc_trim, but
-  operates within the given space.
-*/
-DLMALLOC_EXPORT int mspace_trim(mspace msp, size_t pad);
-
-/*
-  An alias for mallopt.
-*/
-DLMALLOC_EXPORT int mspace_mallopt(int, int);
-
-#endif /* MSPACES */
-
-#ifdef __cplusplus
-}  /* end of extern "C" */
-#endif /* __cplusplus */
-
-/*
-  ========================================================================
-  To make a fully customizable malloc.h header file, cut everything
-  above this line, put into file malloc.h, edit to suit, and #include it
-  on the next line, as well as in programs that use this malloc.
-  ========================================================================
-*/
-
-/* #include "malloc.h" */
-
-/*------------------------------ internal #includes ---------------------- */
-
-#ifdef _MSC_VER
-#pragma warning( disable : 4146 ) /* no "unsigned" warnings */
-#endif /* _MSC_VER */
-#if !NO_MALLOC_STATS
-#include <stdio.h>       /* for printing in malloc_stats */
-#endif /* NO_MALLOC_STATS */
-#ifndef LACKS_ERRNO_H
-#include <errno.h>       /* for MALLOC_FAILURE_ACTION */
-#endif /* LACKS_ERRNO_H */
-#ifdef DEBUG
-#if ABORT_ON_ASSERT_FAILURE
-#undef assert
-#define assert(x) if(!(x)) ABORT
-#else /* ABORT_ON_ASSERT_FAILURE */
-#include <assert.h>
-#endif /* ABORT_ON_ASSERT_FAILURE */
-#else  /* DEBUG */
-#ifndef assert
-#define assert(x)
-#endif
-#define DEBUG 0
-#endif /* DEBUG */
-#if !defined(WIN32) && !defined(LACKS_TIME_H)
-#include <time.h>        /* for magic initialization */
-#endif /* WIN32 */
-#ifndef LACKS_STDLIB_H
-#include <stdlib.h>      /* for abort() */
-#endif /* LACKS_STDLIB_H */
-#ifndef LACKS_STRING_H
-#include <string.h>      /* for memset etc */
-#endif  /* LACKS_STRING_H */
-#if USE_BUILTIN_FFS
-#ifndef LACKS_STRINGS_H
-#include <strings.h>     /* for ffs */
-#endif /* LACKS_STRINGS_H */
-#endif /* USE_BUILTIN_FFS */
-#if HAVE_MMAP
-#ifndef LACKS_SYS_MMAN_H
-/* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */
-#if (defined(linux) && !defined(__USE_GNU))
-#define __USE_GNU 1
-#include <sys/mman.h>    /* for mmap */
-#undef __USE_GNU
-#else
-#include <sys/mman.h>    /* for mmap */
-#endif /* linux */
-#endif /* LACKS_SYS_MMAN_H */
-#ifndef LACKS_FCNTL_H
-#include <fcntl.h>
-#endif /* LACKS_FCNTL_H */
-#endif /* HAVE_MMAP */
-#ifndef LACKS_UNISTD_H
-#include <unistd.h>     /* for sbrk, sysconf */
-#else /* LACKS_UNISTD_H */
-#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__)
-extern void*     sbrk(ptrdiff_t);
-#endif /* FreeBSD etc */
-#endif /* LACKS_UNISTD_H */
-
-/* Declarations for locking */
-#if USE_LOCKS
-#ifndef WIN32
-#if defined (__SVR4) && defined (__sun)  /* solaris */
-#include <thread.h>
-#elif !defined(LACKS_SCHED_H)
-#include <sched.h>
-#endif /* solaris or LACKS_SCHED_H */
-#if (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0) || !USE_SPIN_LOCKS
-#include <pthread.h>
-#endif /* USE_RECURSIVE_LOCKS ... */
-#elif defined(_MSC_VER)
-#ifndef _M_AMD64
-/* These are already defined on AMD64 builds */
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-LONG __cdecl _InterlockedCompareExchange(LONG volatile *Dest, LONG Exchange, LONG Comp);
-LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value);
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-#endif /* _M_AMD64 */
-#pragma intrinsic (_InterlockedCompareExchange)
-#pragma intrinsic (_InterlockedExchange)
-#define interlockedcompareexchange _InterlockedCompareExchange
-#define interlockedexchange _InterlockedExchange
-#elif defined(WIN32) && defined(__GNUC__)
-#define interlockedcompareexchange(a, b, c) __sync_val_compare_and_swap(a, c, b)
-#define interlockedexchange __sync_lock_test_and_set
-#endif /* Win32 */
-#else /* USE_LOCKS */
-#endif /* USE_LOCKS */
-
-#ifndef LOCK_AT_FORK
-#define LOCK_AT_FORK 0
-#endif
-
-/* Declarations for bit scanning on win32 */
-#if defined(_MSC_VER) && _MSC_VER>=1300
-#ifndef BitScanForward /* Try to avoid pulling in WinNT.h */
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-unsigned char _BitScanForward(unsigned long *index, unsigned long mask);
-unsigned char _BitScanReverse(unsigned long *index, unsigned long mask);
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-
-#define BitScanForward _BitScanForward
-#define BitScanReverse _BitScanReverse
-#pragma intrinsic(_BitScanForward)
-#pragma intrinsic(_BitScanReverse)
-#endif /* BitScanForward */
-#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */
-
-#ifndef WIN32
-#ifndef malloc_getpagesize
-#  ifdef _SC_PAGESIZE         /* some SVR4 systems omit an underscore */
-#    ifndef _SC_PAGE_SIZE
-#      define _SC_PAGE_SIZE _SC_PAGESIZE
-#    endif
-#  endif
-#  ifdef _SC_PAGE_SIZE
-#    define malloc_getpagesize sysconf(_SC_PAGE_SIZE)
-#  else
-#    if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE)
-       extern size_t getpagesize();
-#      define malloc_getpagesize getpagesize()
-#    else
-#      ifdef WIN32 /* use supplied emulation of getpagesize */
-#        define malloc_getpagesize getpagesize()
-#      else
-#        ifndef LACKS_SYS_PARAM_H
-#          include <sys/param.h>
-#        endif
-#        ifdef EXEC_PAGESIZE
-#          define malloc_getpagesize EXEC_PAGESIZE
-#        else
-#          ifdef NBPG
-#            ifndef CLSIZE
-#              define malloc_getpagesize NBPG
-#            else
-#              define malloc_getpagesize (NBPG * CLSIZE)
-#            endif
-#          else
-#            ifdef NBPC
-#              define malloc_getpagesize NBPC
-#            else
-#              ifdef PAGESIZE
-#                define malloc_getpagesize PAGESIZE
-#              else /* just guess */
-#                define malloc_getpagesize ((size_t)4096U)
-#              endif
-#            endif
-#          endif
-#        endif
-#      endif
-#    endif
-#  endif
-#endif
-#endif
-
-/* ------------------- size_t and alignment properties -------------------- */
-
-/* The byte and bit size of a size_t */
-#define SIZE_T_SIZE         (sizeof(size_t))
-#define SIZE_T_BITSIZE      (sizeof(size_t) << 3)
-
-/* Some constants coerced to size_t */
-/* Annoying but necessary to avoid errors on some platforms */
-#define SIZE_T_ZERO         ((size_t)0)
-#define SIZE_T_ONE          ((size_t)1)
-#define SIZE_T_TWO          ((size_t)2)
-#define SIZE_T_FOUR         ((size_t)4)
-#define TWO_SIZE_T_SIZES    (SIZE_T_SIZE<<1)
-#define FOUR_SIZE_T_SIZES   (SIZE_T_SIZE<<2)
-#define SIX_SIZE_T_SIZES    (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES)
-#define HALF_MAX_SIZE_T     (MAX_SIZE_T / 2U)
-
-/* The bit mask value corresponding to MALLOC_ALIGNMENT */
-#define CHUNK_ALIGN_MASK    (MALLOC_ALIGNMENT - SIZE_T_ONE)
-
-/* True if address a has acceptable alignment */
-#define is_aligned(A)       (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0)
-
-/* the number of bytes to offset an address to align it */
-#define align_offset(A)\
- ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\
-  ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK))
-
-/* -------------------------- MMAP preliminaries ------------------------- */
-
-/*
-   If HAVE_MORECORE or HAVE_MMAP are false, we just define calls and
-   checks to fail so compiler optimizer can delete code rather than
-   using so many "#if"s.
-*/
-
-
-/* MORECORE and MMAP must return MFAIL on failure */
-#define MFAIL                ((void*)(MAX_SIZE_T))
-#define CMFAIL               ((char*)(MFAIL)) /* defined for convenience */
-
-#if HAVE_MMAP
-
-#ifndef WIN32
-#define MUNMAP_DEFAULT(a, s)  munmap((a), (s))
-#define MMAP_PROT            (PROT_READ|PROT_WRITE)
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS        MAP_ANON
-#endif /* MAP_ANON */
-#ifdef MAP_ANONYMOUS
-#define MMAP_FLAGS           (MAP_PRIVATE|MAP_ANONYMOUS)
-#define MMAP_DEFAULT(s)       mmap(0, (s), MMAP_PROT, MMAP_FLAGS, -1, 0)
-#else /* MAP_ANONYMOUS */
-/*
-   Nearly all versions of mmap support MAP_ANONYMOUS, so the following
-   is unlikely to be needed, but is supplied just in case.
-*/
-#define MMAP_FLAGS           (MAP_PRIVATE)
-static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */
-#define MMAP_DEFAULT(s) ((dev_zero_fd < 0) ? \
-           (dev_zero_fd = open("/dev/zero", O_RDWR), \
-            mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) : \
-            mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0))
-#endif /* MAP_ANONYMOUS */
-
-#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s)
-
-#else /* WIN32 */
-
-/* Win32 MMAP via VirtualAlloc */
-static FORCEINLINE void* win32mmap(size_t size) {
-  void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
-  return (ptr != 0)? ptr: MFAIL;
-}
-
-/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
-static FORCEINLINE void* win32direct_mmap(size_t size) {
-  void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN,
-                           PAGE_READWRITE);
-  return (ptr != 0)? ptr: MFAIL;
-}
-
-/* This function supports releasing coalesed segments */
-static FORCEINLINE int win32munmap(void* ptr, size_t size) {
-  MEMORY_BASIC_INFORMATION minfo;
-  char* cptr = (char*)ptr;
-  while (size) {
-    if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0)
-      return -1;
-    if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr ||
-        minfo.State != MEM_COMMIT || minfo.RegionSize > size)
-      return -1;
-    if (VirtualFree(cptr, 0, MEM_RELEASE) == 0)
-      return -1;
-    cptr += minfo.RegionSize;
-    size -= minfo.RegionSize;
-  }
-  return 0;
-}
-
-#define MMAP_DEFAULT(s)             win32mmap(s)
-#define MUNMAP_DEFAULT(a, s)        win32munmap((a), (s))
-#define DIRECT_MMAP_DEFAULT(s)      win32direct_mmap(s)
-#endif /* WIN32 */
-#endif /* HAVE_MMAP */
-
-#if HAVE_MREMAP
-#ifndef WIN32
-#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv))
-#endif /* WIN32 */
-#endif /* HAVE_MREMAP */
-
-/**
- * Define CALL_MORECORE
- */
-#if HAVE_MORECORE
-    #ifdef MORECORE
-        #define CALL_MORECORE(S)    MORECORE(S)
-    #else  /* MORECORE */
-        #define CALL_MORECORE(S)    MORECORE_DEFAULT(S)
-    #endif /* MORECORE */
-#else  /* HAVE_MORECORE */
-    #define CALL_MORECORE(S)        MFAIL
-#endif /* HAVE_MORECORE */
-
-/**
- * Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP
- */
-#if HAVE_MMAP
-    #define USE_MMAP_BIT            (SIZE_T_ONE)
-
-    #ifdef MMAP
-        #define CALL_MMAP(s)        MMAP(s)
-    #else /* MMAP */
-        #define CALL_MMAP(s)        MMAP_DEFAULT(s)
-    #endif /* MMAP */
-    #ifdef MUNMAP
-        #define CALL_MUNMAP(a, s)   MUNMAP((a), (s))
-    #else /* MUNMAP */
-        #define CALL_MUNMAP(a, s)   MUNMAP_DEFAULT((a), (s))
-    #endif /* MUNMAP */
-    #ifdef DIRECT_MMAP
-        #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
-    #else /* DIRECT_MMAP */
-        #define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s)
-    #endif /* DIRECT_MMAP */
-#else  /* HAVE_MMAP */
-    #define USE_MMAP_BIT            (SIZE_T_ZERO)
-
-    #define MMAP(s)                 MFAIL
-    #define MUNMAP(a, s)            (-1)
-    #define DIRECT_MMAP(s)          MFAIL
-    #define CALL_DIRECT_MMAP(s)     DIRECT_MMAP(s)
-    #define CALL_MMAP(s)            MMAP(s)
-    #define CALL_MUNMAP(a, s)       MUNMAP((a), (s))
-#endif /* HAVE_MMAP */
-
-/**
- * Define CALL_MREMAP
- */
-#if HAVE_MMAP && HAVE_MREMAP
-    #ifdef MREMAP
-        #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv))
-    #else /* MREMAP */
-        #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP_DEFAULT((addr), (osz), (nsz), (mv))
-    #endif /* MREMAP */
-#else  /* HAVE_MMAP && HAVE_MREMAP */
-    #define CALL_MREMAP(addr, osz, nsz, mv)     MFAIL
-#endif /* HAVE_MMAP && HAVE_MREMAP */
-
-/* mstate bit set if continguous morecore disabled or failed */
-#define USE_NONCONTIGUOUS_BIT (4U)
-
-/* segment bit set in create_mspace_with_base */
-#define EXTERN_BIT            (8U)
-
-
-/* --------------------------- Lock preliminaries ------------------------ */
-
-/*
-  When locks are defined, there is one global lock, plus
-  one per-mspace lock.
-
-  The global lock_ensures that mparams.magic and other unique
-  mparams values are initialized only once. It also protects
-  sequences of calls to MORECORE.  In many cases sys_alloc requires
-  two calls, that should not be interleaved with calls by other
-  threads.  This does not protect against direct calls to MORECORE
-  by other threads not using this lock, so there is still code to
-  cope the best we can on interference.
-
-  Per-mspace locks surround calls to malloc, free, etc.
-  By default, locks are simple non-reentrant mutexes.
-
-  Because lock-protected regions generally have bounded times, it is
-  OK to use the supplied simple spinlocks. Spinlocks are likely to
-  improve performance for lightly contended applications, but worsen
-  performance under heavy contention.
-
-  If USE_LOCKS is > 1, the definitions of lock routines here are
-  bypassed, in which case you will need to define the type MLOCK_T,
-  and at least INITIAL_LOCK, DESTROY_LOCK, ACQUIRE_LOCK, RELEASE_LOCK
-  and TRY_LOCK.  You must also declare a
-    static MLOCK_T malloc_global_mutex = { initialization values };.
-
-*/
-
-#if !USE_LOCKS
-#define USE_LOCK_BIT               (0U)
-#define INITIAL_LOCK(l)            (0)
-#define DESTROY_LOCK(l)            (0)
-#define ACQUIRE_MALLOC_GLOBAL_LOCK()
-#define RELEASE_MALLOC_GLOBAL_LOCK()
-
-#else
-#if USE_LOCKS > 1
-/* -----------------------  User-defined locks ------------------------ */
-/* Define your own lock implementation here */
-/* #define INITIAL_LOCK(lk)  ... */
-/* #define DESTROY_LOCK(lk)  ... */
-/* #define ACQUIRE_LOCK(lk)  ... */
-/* #define RELEASE_LOCK(lk)  ... */
-/* #define TRY_LOCK(lk) ... */
-/* static MLOCK_T malloc_global_mutex = ... */
-
-#elif USE_SPIN_LOCKS
-
-/* First, define CAS_LOCK and CLEAR_LOCK on ints */
-/* Note CAS_LOCK defined to return 0 on success */
-
-#if defined(__GNUC__)&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))
-#define CAS_LOCK(sl)     __sync_lock_test_and_set(sl, 1)
-#define CLEAR_LOCK(sl)   __sync_lock_release(sl)
-
-#elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)))
-/* Custom spin locks for older gcc on x86 */
-static FORCEINLINE int x86_cas_lock(int *sl) {
-  int ret;
-  int val = 1;
-  int cmp = 0;
-  __asm__ __volatile__  ("lock; cmpxchgl %1, %2"
-                         : "=a" (ret)
-                         : "r" (val), "m" (*(sl)), "0"(cmp)
-                         : "memory", "cc");
-  return ret;
-}
-
-static FORCEINLINE void x86_clear_lock(int* sl) {
-  assert(*sl != 0);
-  int prev = 0;
-  int ret;
-  __asm__ __volatile__ ("lock; xchgl %0, %1"
-                        : "=r" (ret)
-                        : "m" (*(sl)), "0"(prev)
-                        : "memory");
-}
-
-#define CAS_LOCK(sl)     x86_cas_lock(sl)
-#define CLEAR_LOCK(sl)   x86_clear_lock(sl)
-
-#else /* Win32 MSC */
-#define CAS_LOCK(sl)     interlockedexchange(sl, (LONG)1)
-#define CLEAR_LOCK(sl)   interlockedexchange (sl, (LONG)0)
-
-#endif /* ... gcc spins locks ... */
-
-/* How to yield for a spin lock */
-#define SPINS_PER_YIELD       63
-#if defined(_MSC_VER)
-#define SLEEP_EX_DURATION     50 /* delay for yield/sleep */
-#define SPIN_LOCK_YIELD  SleepEx(SLEEP_EX_DURATION, FALSE)
-#elif defined (__SVR4) && defined (__sun) /* solaris */
-#define SPIN_LOCK_YIELD   thr_yield();
-#elif !defined(LACKS_SCHED_H)
-#define SPIN_LOCK_YIELD   sched_yield();
-#else
-#define SPIN_LOCK_YIELD
-#endif /* ... yield ... */
-
-#if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0
-/* Plain spin locks use single word (embedded in malloc_states) */
-static int spin_acquire_lock(int *sl) {
-  int spins = 0;
-  while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) {
-    if ((++spins & SPINS_PER_YIELD) == 0) {
-      SPIN_LOCK_YIELD;
-    }
-  }
-  return 0;
-}
-
-#define MLOCK_T               int
-#define TRY_LOCK(sl)          !CAS_LOCK(sl)
-#define RELEASE_LOCK(sl)      CLEAR_LOCK(sl)
-#define ACQUIRE_LOCK(sl)      (CAS_LOCK(sl)? spin_acquire_lock(sl) : 0)
-#define INITIAL_LOCK(sl)      (*sl = 0)
-#define DESTROY_LOCK(sl)      (0)
-static MLOCK_T malloc_global_mutex = 0;
-
-#else /* USE_RECURSIVE_LOCKS */
-/* types for lock owners */
-#ifdef WIN32
-#define THREAD_ID_T           DWORD
-#define CURRENT_THREAD        GetCurrentThreadId()
-#define EQ_OWNER(X,Y)         ((X) == (Y))
-#else
-/*
-  Note: the following assume that pthread_t is a type that can be
-  initialized to (casted) zero. If this is not the case, you will need to
-  somehow redefine these or not use spin locks.
-*/
-#define THREAD_ID_T           pthread_t
-#define CURRENT_THREAD        pthread_self()
-#define EQ_OWNER(X,Y)         pthread_equal(X, Y)
-#endif
-
-struct malloc_recursive_lock {
-  int sl;
-  unsigned int c;
-  THREAD_ID_T threadid;
-};
-
-#define MLOCK_T  struct malloc_recursive_lock
-static MLOCK_T malloc_global_mutex = { 0, 0, (THREAD_ID_T)0};
-
-static FORCEINLINE void recursive_release_lock(MLOCK_T *lk) {
-  assert(lk->sl != 0);
-  if (--lk->c == 0) {
-    CLEAR_LOCK(&lk->sl);
-  }
-}
-
-static FORCEINLINE int recursive_acquire_lock(MLOCK_T *lk) {
-  THREAD_ID_T mythreadid = CURRENT_THREAD;
-  int spins = 0;
-  for (;;) {
-    if (*((volatile int *)(&lk->sl)) == 0) {
-      if (!CAS_LOCK(&lk->sl)) {
-        lk->threadid = mythreadid;
-        lk->c = 1;
-        return 0;
-      }
-    }
-    else if (EQ_OWNER(lk->threadid, mythreadid)) {
-      ++lk->c;
-      return 0;
-    }
-    if ((++spins & SPINS_PER_YIELD) == 0) {
-      SPIN_LOCK_YIELD;
-    }
-  }
-}
-
-static FORCEINLINE int recursive_try_lock(MLOCK_T *lk) {
-  THREAD_ID_T mythreadid = CURRENT_THREAD;
-  if (*((volatile int *)(&lk->sl)) == 0) {
-    if (!CAS_LOCK(&lk->sl)) {
-      lk->threadid = mythreadid;
-      lk->c = 1;
-      return 1;
-    }
-  }
-  else if (EQ_OWNER(lk->threadid, mythreadid)) {
-    ++lk->c;
-    return 1;
-  }
-  return 0;
-}
-
-#define RELEASE_LOCK(lk)      recursive_release_lock(lk)
-#define TRY_LOCK(lk)          recursive_try_lock(lk)
-#define ACQUIRE_LOCK(lk)      recursive_acquire_lock(lk)
-#define INITIAL_LOCK(lk)      ((lk)->threadid = (THREAD_ID_T)0, (lk)->sl = 0, (lk)->c = 0)
-#define DESTROY_LOCK(lk)      (0)
-#endif /* USE_RECURSIVE_LOCKS */
-
-#elif defined(WIN32) /* Win32 critical sections */
-#define MLOCK_T               CRITICAL_SECTION
-#define ACQUIRE_LOCK(lk)      (EnterCriticalSection(lk), 0)
-#define RELEASE_LOCK(lk)      LeaveCriticalSection(lk)
-#define TRY_LOCK(lk)          TryEnterCriticalSection(lk)
-#define INITIAL_LOCK(lk)      (!InitializeCriticalSectionAndSpinCount((lk), 0x80000000|4000))
-#define DESTROY_LOCK(lk)      (DeleteCriticalSection(lk), 0)
-#define NEED_GLOBAL_LOCK_INIT
-
-static MLOCK_T malloc_global_mutex;
-static volatile LONG malloc_global_mutex_status;
-
-/* Use spin loop to initialize global lock */
-static void init_malloc_global_mutex() {
-  for (;;) {
-    long stat = malloc_global_mutex_status;
-    if (stat > 0)
-      return;
-    /* transition to < 0 while initializing, then to > 0) */
-    if (stat == 0 &&
-        interlockedcompareexchange(&malloc_global_mutex_status, (LONG)-1, (LONG)0) == 0) {
-      InitializeCriticalSection(&malloc_global_mutex);
-      interlockedexchange(&malloc_global_mutex_status, (LONG)1);
-      return;
-    }
-    SleepEx(0, FALSE);
-  }
-}
-
-#else /* pthreads-based locks */
-#define MLOCK_T               pthread_mutex_t
-#define ACQUIRE_LOCK(lk)      pthread_mutex_lock(lk)
-#define RELEASE_LOCK(lk)      pthread_mutex_unlock(lk)
-#define TRY_LOCK(lk)          (!pthread_mutex_trylock(lk))
-#define INITIAL_LOCK(lk)      pthread_init_lock(lk)
-#define DESTROY_LOCK(lk)      pthread_mutex_destroy(lk)
-
-#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 && defined(linux) && !defined(PTHREAD_MUTEX_RECURSIVE)
-/* Cope with old-style linux recursive lock initialization by adding */
-/* skipped internal declaration from pthread.h */
-extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr,
-                                              int __kind));
-#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP
-#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y)
-#endif /* USE_RECURSIVE_LOCKS ... */
-
-static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-static int pthread_init_lock (MLOCK_T *lk) {
-  pthread_mutexattr_t attr;
-  if (pthread_mutexattr_init(&attr)) return 1;
-#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0
-  if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1;
-#endif
-  if (pthread_mutex_init(lk, &attr)) return 1;
-  if (pthread_mutexattr_destroy(&attr)) return 1;
-  return 0;
-}
-
-#endif /* ... lock types ... */
-
-/* Common code for all lock types */
-#define USE_LOCK_BIT               (2U)
-
-#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK
-#define ACQUIRE_MALLOC_GLOBAL_LOCK()  ACQUIRE_LOCK(&malloc_global_mutex);
-#endif
-
-#ifndef RELEASE_MALLOC_GLOBAL_LOCK
-#define RELEASE_MALLOC_GLOBAL_LOCK()  RELEASE_LOCK(&malloc_global_mutex);
-#endif
-
-#endif /* USE_LOCKS */
-
-/* -----------------------  Chunk representations ------------------------ */
-
-/*
-  (The following includes lightly edited explanations by Colin Plumb.)
-
-  The malloc_chunk declaration below is misleading (but accurate and
-  necessary).  It declares a "view" into memory allowing access to
-  necessary fields at known offsets from a given base.
-
-  Chunks of memory are maintained using a `boundary tag' method as
-  originally described by Knuth.  (See the paper by Paul Wilson
-  ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a survey of such
-  techniques.)  Sizes of free chunks are stored both in the front of
-  each chunk and at the end.  This makes consolidating fragmented
-  chunks into bigger chunks fast.  The head fields also hold bits
-  representing whether chunks are free or in use.
-
-  Here are some pictures to make it clearer.  They are "exploded" to
-  show that the state of a chunk can be thought of as extending from
-  the high 31 bits of the head field of its header through the
-  prev_foot and PINUSE_BIT bit of the following chunk header.
-
-  A chunk that's in use looks like:
-
-   chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-           | Size of previous chunk (if P = 0)                             |
-           +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
-         | Size of this chunk                                         1| +-+
-   mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |                                                               |
-         +-                                                             -+
-         |                                                               |
-         +-                                                             -+
-         |                                                               :
-         +-      size - sizeof(size_t) available payload bytes          -+
-         :                                                               |
- chunk-> +-                                                             -+
-         |                                                               |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |1|
-       | Size of next chunk (may or may not be in use)               | +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-    And if it's free, it looks like this:
-
-   chunk-> +-                                                             -+
-           | User payload (must be in use, or we would have merged!)       |
-           +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
-         | Size of this chunk                                         0| +-+
-   mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         | Next pointer                                                  |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         | Prev pointer                                                  |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |                                                               :
-         +-      size - sizeof(struct chunk) unused bytes               -+
-         :                                                               |
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         | Size of this chunk                                            |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |0|
-       | Size of next chunk (must be in use, or we would have merged)| +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-       |                                                               :
-       +- User payload                                                -+
-       :                                                               |
-       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-                                                                     |0|
-                                                                     +-+
-  Note that since we always merge adjacent free chunks, the chunks
-  adjacent to a free chunk must be in use.
-
-  Given a pointer to a chunk (which can be derived trivially from the
-  payload pointer) we can, in O(1) time, find out whether the adjacent
-  chunks are free, and if so, unlink them from the lists that they
-  are on and merge them with the current chunk.
-
-  Chunks always begin on even word boundaries, so the mem portion
-  (which is returned to the user) is also on an even word boundary, and
-  thus at least double-word aligned.
-
-  The P (PINUSE_BIT) bit, stored in the unused low-order bit of the
-  chunk size (which is always a multiple of two words), is an in-use
-  bit for the *previous* chunk.  If that bit is *clear*, then the
-  word before the current chunk size contains the previous chunk
-  size, and can be used to find the front of the previous chunk.
-  The very first chunk allocated always has this bit set, preventing
-  access to non-existent (or non-owned) memory. If pinuse is set for
-  any given chunk, then you CANNOT determine the size of the
-  previous chunk, and might even get a memory addressing fault when
-  trying to do so.
-
-  The C (CINUSE_BIT) bit, stored in the unused second-lowest bit of
-  the chunk size redundantly records whether the current chunk is
-  inuse (unless the chunk is mmapped). This redundancy enables usage
-  checks within free and realloc, and reduces indirection when freeing
-  and consolidating chunks.
-
-  Each freshly allocated chunk must have both cinuse and pinuse set.
-  That is, each allocated chunk borders either a previously allocated
-  and still in-use chunk, or the base of its memory arena. This is
-  ensured by making all allocations from the `lowest' part of any
-  found chunk.  Further, no free chunk physically borders another one,
-  so each free chunk is known to be preceded and followed by either
-  inuse chunks or the ends of memory.
-
-  Note that the `foot' of the current chunk is actually represented
-  as the prev_foot of the NEXT chunk. This makes it easier to
-  deal with alignments etc but can be very confusing when trying
-  to extend or adapt this code.
-
-  The exceptions to all this are
-
-     1. The special chunk `top' is the top-most available chunk (i.e.,
-        the one bordering the end of available memory). It is treated
-        specially.  Top is never included in any bin, is used only if
-        no other chunk is available, and is released back to the
-        system if it is very large (see M_TRIM_THRESHOLD).  In effect,
-        the top chunk is treated as larger (and thus less well
-        fitting) than any other available chunk.  The top chunk
-        doesn't update its trailing size field since there is no next
-        contiguous chunk that would have to index off it. However,
-        space is still allocated for it (TOP_FOOT_SIZE) to enable
-        separation or merging when space is extended.
-
-     3. Chunks allocated via mmap, have both cinuse and pinuse bits
-        cleared in their head fields.  Because they are allocated
-        one-by-one, each must carry its own prev_foot field, which is
-        also used to hold the offset this chunk has within its mmapped
-        region, which is needed to preserve alignment. Each mmapped
-        chunk is trailed by the first two fields of a fake next-chunk
-        for sake of usage checks.
-
-*/
-
-struct malloc_chunk {
-  size_t               prev_foot;  /* Size of previous chunk (if free).  */
-  size_t               head;       /* Size and inuse bits. */
-  struct malloc_chunk* fd;         /* double links -- used only if free. */
-  struct malloc_chunk* bk;
-};
-
-typedef struct malloc_chunk  mchunk;
-typedef struct malloc_chunk* mchunkptr;
-typedef struct malloc_chunk* sbinptr;  /* The type of bins of chunks */
-typedef unsigned int bindex_t;         /* Described below */
-typedef unsigned int binmap_t;         /* Described below */
-typedef unsigned int flag_t;           /* The type of various bit flag sets */
-
-/* ------------------- Chunks sizes and alignments ----------------------- */
-
-#define MCHUNK_SIZE         (sizeof(mchunk))
-
-#if FOOTERS
-#define CHUNK_OVERHEAD      (TWO_SIZE_T_SIZES)
-#else /* FOOTERS */
-#define CHUNK_OVERHEAD      (SIZE_T_SIZE)
-#endif /* FOOTERS */
-
-/* MMapped chunks need a second word of overhead ... */
-#define MMAP_CHUNK_OVERHEAD (TWO_SIZE_T_SIZES)
-/* ... and additional padding for fake next-chunk at foot */
-#define MMAP_FOOT_PAD       (FOUR_SIZE_T_SIZES)
-
-/* The smallest size we can malloc is an aligned minimal chunk */
-#define MIN_CHUNK_SIZE\
-  ((MCHUNK_SIZE + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
-
-/* conversion from malloc headers to user pointers, and back */
-#define chunk2mem(p)        ((void*)((char*)(p)       + TWO_SIZE_T_SIZES))
-#define mem2chunk(mem)      ((mchunkptr)((char*)(mem) - TWO_SIZE_T_SIZES))
-/* chunk associated with aligned address A */
-#define align_as_chunk(A)   (mchunkptr)((A) + align_offset(chunk2mem(A)))
-
-/* Bounds on request (not chunk) sizes. */
-#define MAX_REQUEST         ((-MIN_CHUNK_SIZE) << 2)
-#define MIN_REQUEST         (MIN_CHUNK_SIZE - CHUNK_OVERHEAD - SIZE_T_ONE)
-
-/* pad request bytes into a usable size */
-#define pad_request(req) \
-   (((req) + CHUNK_OVERHEAD + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
-
-/* pad request, checking for minimum (but not maximum) */
-#define request2size(req) \
-  (((req) < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(req))
-
-
-/* ------------------ Operations on head and foot fields ----------------- */
-
-/*
-  The head field of a chunk is or'ed with PINUSE_BIT when previous
-  adjacent chunk in use, and or'ed with CINUSE_BIT if this chunk is in
-  use, unless mmapped, in which case both bits are cleared.
-
-  FLAG4_BIT is not used by this malloc, but might be useful in extensions.
-*/
-
-#define PINUSE_BIT          (SIZE_T_ONE)
-#define CINUSE_BIT          (SIZE_T_TWO)
-#define FLAG4_BIT           (SIZE_T_FOUR)
-#define INUSE_BITS          (PINUSE_BIT|CINUSE_BIT)
-#define FLAG_BITS           (PINUSE_BIT|CINUSE_BIT|FLAG4_BIT)
-
-/* Head value for fenceposts */
-#define FENCEPOST_HEAD      (INUSE_BITS|SIZE_T_SIZE)
-
-/* extraction of fields from head words */
-#define cinuse(p)           ((p)->head & CINUSE_BIT)
-#define pinuse(p)           ((p)->head & PINUSE_BIT)
-#define flag4inuse(p)       ((p)->head & FLAG4_BIT)
-#define is_inuse(p)         (((p)->head & INUSE_BITS) != PINUSE_BIT)
-#define is_mmapped(p)       (((p)->head & INUSE_BITS) == 0)
-
-#define chunksize(p)        ((p)->head & ~(FLAG_BITS))
-
-#define clear_pinuse(p)     ((p)->head &= ~PINUSE_BIT)
-#define set_flag4(p)        ((p)->head |= FLAG4_BIT)
-#define clear_flag4(p)      ((p)->head &= ~FLAG4_BIT)
-
-/* Treat space at ptr +/- offset as a chunk */
-#define chunk_plus_offset(p, s)  ((mchunkptr)(((char*)(p)) + (s)))
-#define chunk_minus_offset(p, s) ((mchunkptr)(((char*)(p)) - (s)))
-
-/* Ptr to next or previous physical malloc_chunk. */
-#define next_chunk(p) ((mchunkptr)( ((char*)(p)) + ((p)->head & ~FLAG_BITS)))
-#define prev_chunk(p) ((mchunkptr)( ((char*)(p)) - ((p)->prev_foot) ))
-
-/* extract next chunk's pinuse bit */
-#define next_pinuse(p)  ((next_chunk(p)->head) & PINUSE_BIT)
-
-/* Get/set size at footer */
-#define get_foot(p, s)  (((mchunkptr)((char*)(p) + (s)))->prev_foot)
-#define set_foot(p, s)  (((mchunkptr)((char*)(p) + (s)))->prev_foot = (s))
-
-/* Set size, pinuse bit, and foot */
-#define set_size_and_pinuse_of_free_chunk(p, s)\
-  ((p)->head = (s|PINUSE_BIT), set_foot(p, s))
-
-/* Set size, pinuse bit, foot, and clear next pinuse */
-#define set_free_with_pinuse(p, s, n)\
-  (clear_pinuse(n), set_size_and_pinuse_of_free_chunk(p, s))
-
-/* Get the internal overhead associated with chunk p */
-#define overhead_for(p)\
- (is_mmapped(p)? MMAP_CHUNK_OVERHEAD : CHUNK_OVERHEAD)
-
-/* Return true if malloced space is not necessarily cleared */
-#if MMAP_CLEARS
-#define calloc_must_clear(p) (!is_mmapped(p))
-#else /* MMAP_CLEARS */
-#define calloc_must_clear(p) (1)
-#endif /* MMAP_CLEARS */
-
-/* ---------------------- Overlaid data structures ----------------------- */
-
-/*
-  When chunks are not in use, they are treated as nodes of either
-  lists or trees.
-
-  "Small"  chunks are stored in circular doubly-linked lists, and look
-  like this:
-
-    chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Size of previous chunk                            |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    `head:' |             Size of chunk, in bytes                         |P|
-      mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Forward pointer to next chunk in list             |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Back pointer to previous chunk in list            |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Unused space (may be 0 bytes long)                .
-            .                                                               .
-            .                                                               |
-nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    `foot:' |             Size of chunk, in bytes                           |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-  Larger chunks are kept in a form of bitwise digital trees (aka
-  tries) keyed on chunksizes.  Because malloc_tree_chunks are only for
-  free chunks greater than 256 bytes, their size doesn't impose any
-  constraints on user chunk sizes.  Each node looks like:
-
-    chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Size of previous chunk                            |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    `head:' |             Size of chunk, in bytes                         |P|
-      mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Forward pointer to next chunk of same size        |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Back pointer to previous chunk of same size       |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Pointer to left child (child[0])                  |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Pointer to right child (child[1])                 |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Pointer to parent                                 |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             bin index of this chunk                           |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-            |             Unused space                                      .
-            .                                                               |
-nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    `foot:' |             Size of chunk, in bytes                           |
-            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-  Each tree holding treenodes is a tree of unique chunk sizes.  Chunks
-  of the same size are arranged in a circularly-linked list, with only
-  the oldest chunk (the next to be used, in our FIFO ordering)
-  actually in the tree.  (Tree members are distinguished by a non-null
-  parent pointer.)  If a chunk with the same size an an existing node
-  is inserted, it is linked off the existing node using pointers that
-  work in the same way as fd/bk pointers of small chunks.
-
-  Each tree contains a power of 2 sized range of chunk sizes (the
-  smallest is 0x100 <= x < 0x180), which is is divided in half at each
-  tree level, with the chunks in the smaller half of the range (0x100
-  <= x < 0x140 for the top nose) in the left subtree and the larger
-  half (0x140 <= x < 0x180) in the right subtree.  This is, of course,
-  done by inspecting individual bits.
-
-  Using these rules, each node's left subtree contains all smaller
-  sizes than its right subtree.  However, the node at the root of each
-  subtree has no particular ordering relationship to either.  (The
-  dividing line between the subtree sizes is based on trie relation.)
-  If we remove the last chunk of a given size from the interior of the
-  tree, we need to replace it with a leaf node.  The tree ordering
-  rules permit a node to be replaced by any leaf below it.
-
-  The smallest chunk in a tree (a common operation in a best-fit
-  allocator) can be found by walking a path to the leftmost leaf in
-  the tree.  Unlike a usual binary tree, where we follow left child
-  pointers until we reach a null, here we follow the right child
-  pointer any time the left one is null, until we reach a leaf with
-  both child pointers null. The smallest chunk in the tree will be
-  somewhere along that path.
-
-  The worst case number of steps to add, find, or remove a node is
-  bounded by the number of bits differentiating chunks within
-  bins. Under current bin calculations, this ranges from 6 up to 21
-  (for 32 bit sizes) or up to 53 (for 64 bit sizes). The typical case
-  is of course much better.
-*/
-
-struct malloc_tree_chunk {
-  /* The first four fields must be compatible with malloc_chunk */
-  size_t                    prev_foot;
-  size_t                    head;
-  struct malloc_tree_chunk* fd;
-  struct malloc_tree_chunk* bk;
-
-  struct malloc_tree_chunk* child[2];
-  struct malloc_tree_chunk* parent;
-  bindex_t                  index;
-};
-
-typedef struct malloc_tree_chunk  tchunk;
-typedef struct malloc_tree_chunk* tchunkptr;
-typedef struct malloc_tree_chunk* tbinptr; /* The type of bins of trees */
-
-/* A little helper macro for trees */
-#define leftmost_child(t) ((t)->child[0] != 0? (t)->child[0] : (t)->child[1])
-
-/* ----------------------------- Segments -------------------------------- */
-
-/*
-  Each malloc space may include non-contiguous segments, held in a
-  list headed by an embedded malloc_segment record representing the
-  top-most space. Segments also include flags holding properties of
-  the space. Large chunks that are directly allocated by mmap are not
-  included in this list. They are instead independently created and
-  destroyed without otherwise keeping track of them.
-
-  Segment management mainly comes into play for spaces allocated by
-  MMAP.  Any call to MMAP might or might not return memory that is
-  adjacent to an existing segment.  MORECORE normally contiguously
-  extends the current space, so this space is almost always adjacent,
-  which is simpler and faster to deal with. (This is why MORECORE is
-  used preferentially to MMAP when both are available -- see
-  sys_alloc.)  When allocating using MMAP, we don't use any of the
-  hinting mechanisms (inconsistently) supported in various
-  implementations of unix mmap, or distinguish reserving from
-  committing memory. Instead, we just ask for space, and exploit
-  contiguity when we get it.  It is probably possible to do
-  better than this on some systems, but no general scheme seems
-  to be significantly better.
-
-  Management entails a simpler variant of the consolidation scheme
-  used for chunks to reduce fragmentation -- new adjacent memory is
-  normally prepended or appended to an existing segment. However,
-  there are limitations compared to chunk consolidation that mostly
-  reflect the fact that segment processing is relatively infrequent
-  (occurring only when getting memory from system) and that we
-  don't expect to have huge numbers of segments:
-
-  * Segments are not indexed, so traversal requires linear scans.  (It
-    would be possible to index these, but is not worth the extra
-    overhead and complexity for most programs on most platforms.)
-  * New segments are only appended to old ones when holding top-most
-    memory; if they cannot be prepended to others, they are held in
-    different segments.
-
-  Except for the top-most segment of an mstate, each segment record
-  is kept at the tail of its segment. Segments are added by pushing
-  segment records onto the list headed by &mstate.seg for the
-  containing mstate.
-
-  Segment flags control allocation/merge/deallocation policies:
-  * If EXTERN_BIT set, then we did not allocate this segment,
-    and so should not try to deallocate or merge with others.
-    (This currently holds only for the initial segment passed
-    into create_mspace_with_base.)
-  * If USE_MMAP_BIT set, the segment may be merged with
-    other surrounding mmapped segments and trimmed/de-allocated
-    using munmap.
-  * If neither bit is set, then the segment was obtained using
-    MORECORE so can be merged with surrounding MORECORE'd segments
-    and deallocated/trimmed using MORECORE with negative arguments.
-*/
-
-struct malloc_segment {
-  char*        base;             /* base address */
-  size_t       size;             /* allocated size */
-  struct malloc_segment* next;   /* ptr to next segment */
-  flag_t       sflags;           /* mmap and extern flag */
-};
-
-#define is_mmapped_segment(S)  ((S)->sflags & USE_MMAP_BIT)
-#define is_extern_segment(S)   ((S)->sflags & EXTERN_BIT)
-
-typedef struct malloc_segment  msegment;
-typedef struct malloc_segment* msegmentptr;
-
-/* ---------------------------- malloc_state ----------------------------- */
-
-/*
-   A malloc_state holds all of the bookkeeping for a space.
-   The main fields are:
-
-  Top
-    The topmost chunk of the currently active segment. Its size is
-    cached in topsize.  The actual size of topmost space is
-    topsize+TOP_FOOT_SIZE, which includes space reserved for adding
-    fenceposts and segment records if necessary when getting more
-    space from the system.  The size at which to autotrim top is
-    cached from mparams in trim_check, except that it is disabled if
-    an autotrim fails.
-
-  Designated victim (dv)
-    This is the preferred chunk for servicing small requests that
-    don't have exact fits.  It is normally the chunk split off most
-    recently to service another small request.  Its size is cached in
-    dvsize. The link fields of this chunk are not maintained since it
-    is not kept in a bin.
-
-  SmallBins
-    An array of bin headers for free chunks.  These bins hold chunks
-    with sizes less than MIN_LARGE_SIZE bytes. Each bin contains
-    chunks of all the same size, spaced 8 bytes apart.  To simplify
-    use in double-linked lists, each bin header acts as a malloc_chunk
-    pointing to the real first node, if it exists (else pointing to
-    itself).  This avoids special-casing for headers.  But to avoid
-    waste, we allocate only the fd/bk pointers of bins, and then use
-    repositioning tricks to treat these as the fields of a chunk.
-
-  TreeBins
-    Treebins are pointers to the roots of trees holding a range of
-    sizes. There are 2 equally spaced treebins for each power of two
-    from TREE_SHIFT to TREE_SHIFT+16. The last bin holds anything
-    larger.
-
-  Bin maps
-    There is one bit map for small bins ("smallmap") and one for
-    treebins ("treemap).  Each bin sets its bit when non-empty, and
-    clears the bit when empty.  Bit operations are then used to avoid
-    bin-by-bin searching -- nearly all "search" is done without ever
-    looking at bins that won't be selected.  The bit maps
-    conservatively use 32 bits per map word, even if on 64bit system.
-    For a good description of some of the bit-based techniques used
-    here, see Henry S. Warren Jr's book "Hacker's Delight" (and
-    supplement at http://hackersdelight.org/). Many of these are
-    intended to reduce the branchiness of paths through malloc etc, as
-    well as to reduce the number of memory locations read or written.
-
-  Segments
-    A list of segments headed by an embedded malloc_segment record
-    representing the initial space.
-
-  Address check support
-    The least_addr field is the least address ever obtained from
-    MORECORE or MMAP. Attempted frees and reallocs of any address less
-    than this are trapped (unless INSECURE is defined).
-
-  Magic tag
-    A cross-check field that should always hold same value as mparams.magic.
-
-  Max allowed footprint
-    The maximum allowed bytes to allocate from system (zero means no limit)
-
-  Flags
-    Bits recording whether to use MMAP, locks, or contiguous MORECORE
-
-  Statistics
-    Each space keeps track of current and maximum system memory
-    obtained via MORECORE or MMAP.
-
-  Trim support
-    Fields holding the amount of unused topmost memory that should trigger
-    trimming, and a counter to force periodic scanning to release unused
-    non-topmost segments.
-
-  Locking
-    If USE_LOCKS is defined, the "mutex" lock is acquired and released
-    around every public call using this mspace.
-
-  Extension support
-    A void* pointer and a size_t field that can be used to help implement
-    extensions to this malloc.
-*/
-
-/* Bin types, widths and sizes */
-#define NSMALLBINS        (32U)
-#define NTREEBINS         (32U)
-#define SMALLBIN_SHIFT    (3U)
-#define SMALLBIN_WIDTH    (SIZE_T_ONE << SMALLBIN_SHIFT)
-#define TREEBIN_SHIFT     (8U)
-#define MIN_LARGE_SIZE    (SIZE_T_ONE << TREEBIN_SHIFT)
-#define MAX_SMALL_SIZE    (MIN_LARGE_SIZE - SIZE_T_ONE)
-#define MAX_SMALL_REQUEST (MAX_SMALL_SIZE - CHUNK_ALIGN_MASK - CHUNK_OVERHEAD)
-
-struct malloc_state {
-  binmap_t   smallmap;
-  binmap_t   treemap;
-  size_t     dvsize;
-  size_t     topsize;
-  char*      least_addr;
-  mchunkptr  dv;
-  mchunkptr  top;
-  size_t     trim_check;
-  size_t     release_checks;
-  size_t     magic;
-  mchunkptr  smallbins[(NSMALLBINS+1)*2];
-  tbinptr    treebins[NTREEBINS];
-  size_t     footprint;
-  size_t     max_footprint;
-  size_t     footprint_limit; /* zero means no limit */
-  flag_t     mflags;
-#if USE_LOCKS
-  MLOCK_T    mutex;     /* locate lock among fields that rarely change */
-#endif /* USE_LOCKS */
-  msegment   seg;
-  void*      extp;      /* Unused but available for extensions */
-  size_t     exts;
-};
-
-typedef struct malloc_state*    mstate;
-
-/* ------------- Global malloc_state and malloc_params ------------------- */
-
-/*
-  malloc_params holds global properties, including those that can be
-  dynamically set using mallopt. There is a single instance, mparams,
-  initialized in init_mparams. Note that the non-zeroness of "magic"
-  also serves as an initialization flag.
-*/
-
-struct malloc_params {
-  size_t magic;
-  size_t page_size;
-  size_t granularity;
-  size_t mmap_threshold;
-  size_t trim_threshold;
-  flag_t default_mflags;
-};
-
-static struct malloc_params mparams;
-
-/* Ensure mparams initialized */
-#define ensure_initialization() (void)(mparams.magic != 0 || init_mparams())
-
-#if !ONLY_MSPACES
-
-/* The global malloc_state used for all non-"mspace" calls */
-static struct malloc_state _gm_;
-#define gm                 (&_gm_)
-#define is_global(M)       ((M) == &_gm_)
-
-#endif /* !ONLY_MSPACES */
-
-#define is_initialized(M)  ((M)->top != 0)
-
-/* -------------------------- system alloc setup ------------------------- */
-
-/* Operations on mflags */
-
-#define use_lock(M)           ((M)->mflags &   USE_LOCK_BIT)
-#define enable_lock(M)        ((M)->mflags |=  USE_LOCK_BIT)
-#if USE_LOCKS
-#define disable_lock(M)       ((M)->mflags &= ~USE_LOCK_BIT)
-#else
-#define disable_lock(M)
-#endif
-
-#define use_mmap(M)           ((M)->mflags &   USE_MMAP_BIT)
-#define enable_mmap(M)        ((M)->mflags |=  USE_MMAP_BIT)
-#if HAVE_MMAP
-#define disable_mmap(M)       ((M)->mflags &= ~USE_MMAP_BIT)
-#else
-#define disable_mmap(M)
-#endif
-
-#define use_noncontiguous(M)  ((M)->mflags &   USE_NONCONTIGUOUS_BIT)
-#define disable_contiguous(M) ((M)->mflags |=  USE_NONCONTIGUOUS_BIT)
-
-#define set_lock(M,L)\
- ((M)->mflags = (L)?\
-  ((M)->mflags | USE_LOCK_BIT) :\
-  ((M)->mflags & ~USE_LOCK_BIT))
-
-/* page-align a size */
-#define page_align(S)\
- (((S) + (mparams.page_size - SIZE_T_ONE)) & ~(mparams.page_size - SIZE_T_ONE))
-
-/* granularity-align a size */
-#define granularity_align(S)\
-  (((S) + (mparams.granularity - SIZE_T_ONE))\
-   & ~(mparams.granularity - SIZE_T_ONE))
-
-
-/* For mmap, use granularity alignment on windows, else page-align */
-#ifdef WIN32
-#define mmap_align(S) granularity_align(S)
-#else
-#define mmap_align(S) page_align(S)
-#endif
-
-/* For sys_alloc, enough padding to ensure can malloc request on success */
-#define SYS_ALLOC_PADDING (TOP_FOOT_SIZE + MALLOC_ALIGNMENT)
-
-#define is_page_aligned(S)\
-   (((size_t)(S) & (mparams.page_size - SIZE_T_ONE)) == 0)
-#define is_granularity_aligned(S)\
-   (((size_t)(S) & (mparams.granularity - SIZE_T_ONE)) == 0)
-
-/*  True if segment S holds address A */
-#define segment_holds(S, A)\
-  ((char*)(A) >= S->base && (char*)(A) < S->base + S->size)
-
-/* Return segment holding given address */
-static msegmentptr segment_holding(mstate m, char* addr) {
-  msegmentptr sp = &m->seg;
-  for (;;) {
-    if (addr >= sp->base && addr < sp->base + sp->size)
-      return sp;
-    if ((sp = sp->next) == 0)
-      return 0;
-  }
-}
-
-/* Return true if segment contains a segment link */
-static int has_segment_link(mstate m, msegmentptr ss) {
-  msegmentptr sp = &m->seg;
-  for (;;) {
-    if ((char*)sp >= ss->base && (char*)sp < ss->base + ss->size)
-      return 1;
-    if ((sp = sp->next) == 0)
-      return 0;
-  }
-}
-
-#ifndef MORECORE_CANNOT_TRIM
-#define should_trim(M,s)  ((s) > (M)->trim_check)
-#else  /* MORECORE_CANNOT_TRIM */
-#define should_trim(M,s)  (0)
-#endif /* MORECORE_CANNOT_TRIM */
-
-/*
-  TOP_FOOT_SIZE is padding at the end of a segment, including space
-  that may be needed to place segment records and fenceposts when new
-  noncontiguous segments are added.
-*/
-#define TOP_FOOT_SIZE\
-  (align_offset(chunk2mem(0))+pad_request(sizeof(struct malloc_segment))+MIN_CHUNK_SIZE)
-
-
-/* -------------------------------  Hooks -------------------------------- */
-
-/*
-  PREACTION should be defined to return 0 on success, and nonzero on
-  failure. If you are not using locking, you can redefine these to do
-  anything you like.
-*/
-
-#if USE_LOCKS
-#define PREACTION(M)  ((use_lock(M))? ACQUIRE_LOCK(&(M)->mutex) : 0)
-#define POSTACTION(M) { if (use_lock(M)) RELEASE_LOCK(&(M)->mutex); }
-#else /* USE_LOCKS */
-
-#ifndef PREACTION
-#define PREACTION(M) (0)
-#endif  /* PREACTION */
-
-#ifndef POSTACTION
-#define POSTACTION(M)
-#endif  /* POSTACTION */
-
-#endif /* USE_LOCKS */
-
-/*
-  CORRUPTION_ERROR_ACTION is triggered upon detected bad addresses.
-  USAGE_ERROR_ACTION is triggered on detected bad frees and
-  reallocs. The argument p is an address that might have triggered the
-  fault. It is ignored by the two predefined actions, but might be
-  useful in custom actions that try to help diagnose errors.
-*/
-
-#if PROCEED_ON_ERROR
-
-/* A count of the number of corruption errors causing resets */
-int malloc_corruption_error_count;
-
-/* default corruption action */
-static void reset_on_error(mstate m);
-
-#define CORRUPTION_ERROR_ACTION(m)  reset_on_error(m)
-#define USAGE_ERROR_ACTION(m, p)
-
-#else /* PROCEED_ON_ERROR */
-
-#ifndef CORRUPTION_ERROR_ACTION
-#define CORRUPTION_ERROR_ACTION(m) ABORT
-#endif /* CORRUPTION_ERROR_ACTION */
-
-#ifndef USAGE_ERROR_ACTION
-#define USAGE_ERROR_ACTION(m,p) ABORT
-#endif /* USAGE_ERROR_ACTION */
-
-#endif /* PROCEED_ON_ERROR */
-
-
-/* -------------------------- Debugging setup ---------------------------- */
-
-#if ! DEBUG
-
-#define check_free_chunk(M,P)
-#define check_inuse_chunk(M,P)
-#define check_malloced_chunk(M,P,N)
-#define check_mmapped_chunk(M,P)
-#define check_malloc_state(M)
-#define check_top_chunk(M,P)
-
-#else /* DEBUG */
-#define check_free_chunk(M,P)       do_check_free_chunk(M,P)
-#define check_inuse_chunk(M,P)      do_check_inuse_chunk(M,P)
-#define check_top_chunk(M,P)        do_check_top_chunk(M,P)
-#define check_malloced_chunk(M,P,N) do_check_malloced_chunk(M,P,N)
-#define check_mmapped_chunk(M,P)    do_check_mmapped_chunk(M,P)
-#define check_malloc_state(M)       do_check_malloc_state(M)
-
-static void   do_check_any_chunk(mstate m, mchunkptr p);
-static void   do_check_top_chunk(mstate m, mchunkptr p);
-static void   do_check_mmapped_chunk(mstate m, mchunkptr p);
-static void   do_check_inuse_chunk(mstate m, mchunkptr p);
-static void   do_check_free_chunk(mstate m, mchunkptr p);
-static void   do_check_malloced_chunk(mstate m, void* mem, size_t s);
-static void   do_check_tree(mstate m, tchunkptr t);
-static void   do_check_treebin(mstate m, bindex_t i);
-static void   do_check_smallbin(mstate m, bindex_t i);
-static void   do_check_malloc_state(mstate m);
-static int    bin_find(mstate m, mchunkptr x);
-static size_t traverse_and_check(mstate m);
-#endif /* DEBUG */
-
-/* ---------------------------- Indexing Bins ---------------------------- */
-
-#define is_small(s)         (((s) >> SMALLBIN_SHIFT) < NSMALLBINS)
-#define small_index(s)      (bindex_t)((s)  >> SMALLBIN_SHIFT)
-#define small_index2size(i) ((i)  << SMALLBIN_SHIFT)
-#define MIN_SMALL_INDEX     (small_index(MIN_CHUNK_SIZE))
-
-/* addressing by index. See above about smallbin repositioning */
-#define smallbin_at(M, i)   ((sbinptr)((char*)&((M)->smallbins[(i)<<1])))
-#define treebin_at(M,i)     (&((M)->treebins[i]))
-
-/* assign tree index for size S to variable I. Use x86 asm if possible  */
-#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
-#define compute_tree_index(S, I)\
-{\
-  unsigned int X = S >> TREEBIN_SHIFT;\
-  if (X == 0)\
-    I = 0;\
-  else if (X > 0xFFFF)\
-    I = NTREEBINS-1;\
-  else {\
-    unsigned int K = (unsigned) sizeof(X)*__CHAR_BIT__ - 1 - (unsigned) __builtin_clz(X); \
-    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
-  }\
-}
-
-#elif defined (__INTEL_COMPILER)
-#define compute_tree_index(S, I)\
-{\
-  size_t X = S >> TREEBIN_SHIFT;\
-  if (X == 0)\
-    I = 0;\
-  else if (X > 0xFFFF)\
-    I = NTREEBINS-1;\
-  else {\
-    unsigned int K = _bit_scan_reverse (X); \
-    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
-  }\
-}
-
-#elif defined(_MSC_VER) && _MSC_VER>=1300
-#define compute_tree_index(S, I)\
-{\
-  size_t X = S >> TREEBIN_SHIFT;\
-  if (X == 0)\
-    I = 0;\
-  else if (X > 0xFFFF)\
-    I = NTREEBINS-1;\
-  else {\
-    unsigned int K;\
-    _BitScanReverse((DWORD *) &K, (DWORD) X);\
-    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
-  }\
-}
-
-#else /* GNUC */
-#define compute_tree_index(S, I)\
-{\
-  size_t X = S >> TREEBIN_SHIFT;\
-  if (X == 0)\
-    I = 0;\
-  else if (X > 0xFFFF)\
-    I = NTREEBINS-1;\
-  else {\
-    unsigned int Y = (unsigned int)X;\
-    unsigned int N = ((Y - 0x100) >> 16) & 8;\
-    unsigned int K = (((Y <<= N) - 0x1000) >> 16) & 4;\
-    N += K;\
-    N += K = (((Y <<= K) - 0x4000) >> 16) & 2;\
-    K = 14 - N + ((Y <<= K) >> 15);\
-    I = (K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1));\
-  }\
-}
-#endif /* GNUC */
-
-/* Bit representing maximum resolved size in a treebin at i */
-#define bit_for_tree_index(i) \
-   (i == NTREEBINS-1)? (SIZE_T_BITSIZE-1) : (((i) >> 1) + TREEBIN_SHIFT - 2)
-
-/* Shift placing maximum resolved bit in a treebin at i as sign bit */
-#define leftshift_for_tree_index(i) \
-   ((i == NTREEBINS-1)? 0 : \
-    ((SIZE_T_BITSIZE-SIZE_T_ONE) - (((i) >> 1) + TREEBIN_SHIFT - 2)))
-
-/* The size of the smallest chunk held in bin with index i */
-#define minsize_for_tree_index(i) \
-   ((SIZE_T_ONE << (((i) >> 1) + TREEBIN_SHIFT)) |  \
-   (((size_t)((i) & SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1)))
-
-
-/* ------------------------ Operations on bin maps ----------------------- */
-
-/* bit corresponding to given index */
-#define idx2bit(i)              ((binmap_t)(1) << (i))
-
-/* Mark/Clear bits with given index */
-#define mark_smallmap(M,i)      ((M)->smallmap |=  idx2bit(i))
-#define clear_smallmap(M,i)     ((M)->smallmap &= ~idx2bit(i))
-#define smallmap_is_marked(M,i) ((M)->smallmap &   idx2bit(i))
-
-#define mark_treemap(M,i)       ((M)->treemap  |=  idx2bit(i))
-#define clear_treemap(M,i)      ((M)->treemap  &= ~idx2bit(i))
-#define treemap_is_marked(M,i)  ((M)->treemap  &   idx2bit(i))
-
-/* isolate the least set bit of a bitmap */
-#define least_bit(x)         ((x) & -(x))
-
-/* mask with all bits to left of least bit of x on */
-#define left_bits(x)         ((x<<1) | -(x<<1))
-
-/* mask with all bits to left of or equal to least bit of x on */
-#define same_or_left_bits(x) ((x) | -(x))
-
-/* index corresponding to given bit. Use x86 asm if possible */
-
-#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
-#define compute_bit2idx(X, I)\
-{\
-  unsigned int J;\
-  J = __builtin_ctz(X); \
-  I = (bindex_t)J;\
-}
-
-#elif defined (__INTEL_COMPILER)
-#define compute_bit2idx(X, I)\
-{\
-  unsigned int J;\
-  J = _bit_scan_forward (X); \
-  I = (bindex_t)J;\
-}
-
-#elif defined(_MSC_VER) && _MSC_VER>=1300
-#define compute_bit2idx(X, I)\
-{\
-  unsigned int J;\
-  _BitScanForward((DWORD *) &J, X);\
-  I = (bindex_t)J;\
-}
-
-#elif USE_BUILTIN_FFS
-#define compute_bit2idx(X, I) I = ffs(X)-1
-
-#else
-#define compute_bit2idx(X, I)\
-{\
-  unsigned int Y = X - 1;\
-  unsigned int K = Y >> (16-4) & 16;\
-  unsigned int N = K;        Y >>= K;\
-  N += K = Y >> (8-3) &  8;  Y >>= K;\
-  N += K = Y >> (4-2) &  4;  Y >>= K;\
-  N += K = Y >> (2-1) &  2;  Y >>= K;\
-  N += K = Y >> (1-0) &  1;  Y >>= K;\
-  I = (bindex_t)(N + Y);\
-}
-#endif /* GNUC */
-
-
-/* ----------------------- Runtime Check Support ------------------------- */
-
-/*
-  For security, the main invariant is that malloc/free/etc never
-  writes to a static address other than malloc_state, unless static
-  malloc_state itself has been corrupted, which cannot occur via
-  malloc (because of these checks). In essence this means that we
-  believe all pointers, sizes, maps etc held in malloc_state, but
-  check all of those linked or offsetted from other embedded data
-  structures.  These checks are interspersed with main code in a way
-  that tends to minimize their run-time cost.
-
-  When FOOTERS is defined, in addition to range checking, we also
-  verify footer fields of inuse chunks, which can be used guarantee
-  that the mstate controlling malloc/free is intact.  This is a
-  streamlined version of the approach described by William Robertson
-  et al in "Run-time Detection of Heap-based Overflows" LISA'03
-  http://www.usenix.org/events/lisa03/tech/robertson.html The footer
-  of an inuse chunk holds the xor of its mstate and a random seed,
-  that is checked upon calls to free() and realloc().  This is
-  (probabalistically) unguessable from outside the program, but can be
-  computed by any code successfully malloc'ing any chunk, so does not
-  itself provide protection against code that has already broken
-  security through some other means.  Unlike Robertson et al, we
-  always dynamically check addresses of all offset chunks (previous,
-  next, etc). This turns out to be cheaper than relying on hashes.
-*/
-
-#if !INSECURE
-/* Check if address a is at least as high as any from MORECORE or MMAP */
-#define ok_address(M, a) ((char*)(a) >= (M)->least_addr)
-/* Check if address of next chunk n is higher than base chunk p */
-#define ok_next(p, n)    ((char*)(p) < (char*)(n))
-/* Check if p has inuse status */
-#define ok_inuse(p)     is_inuse(p)
-/* Check if p has its pinuse bit on */
-#define ok_pinuse(p)     pinuse(p)
-
-#else /* !INSECURE */
-#define ok_address(M, a) (1)
-#define ok_next(b, n)    (1)
-#define ok_inuse(p)      (1)
-#define ok_pinuse(p)     (1)
-#endif /* !INSECURE */
-
-#if (FOOTERS && !INSECURE)
-/* Check if (alleged) mstate m has expected magic field */
-#define ok_magic(M)      ((M)->magic == mparams.magic)
-#else  /* (FOOTERS && !INSECURE) */
-#define ok_magic(M)      (1)
-#endif /* (FOOTERS && !INSECURE) */
-
-/* In gcc, use __builtin_expect to minimize impact of checks */
-#if !INSECURE
-#if defined(__GNUC__) && __GNUC__ >= 3
-#define RTCHECK(e)  __builtin_expect(e, 1)
-#else /* GNUC */
-#define RTCHECK(e)  (e)
-#endif /* GNUC */
-#else /* !INSECURE */
-#define RTCHECK(e)  (1)
-#endif /* !INSECURE */
-
-/* macros to set up inuse chunks with or without footers */
-
-#if !FOOTERS
-
-#define mark_inuse_foot(M,p,s)
-
-/* Macros for setting head/foot of non-mmapped chunks */
-
-/* Set cinuse bit and pinuse bit of next chunk */
-#define set_inuse(M,p,s)\
-  ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\
-  ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT)
-
-/* Set cinuse and pinuse of this chunk and pinuse of next chunk */
-#define set_inuse_and_pinuse(M,p,s)\
-  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
-  ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT)
-
-/* Set size, cinuse and pinuse bit of this chunk */
-#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\
-  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT))
-
-#else /* FOOTERS */
-
-/* Set foot of inuse chunk to be xor of mstate and seed */
-#define mark_inuse_foot(M,p,s)\
-  (((mchunkptr)((char*)(p) + (s)))->prev_foot = ((size_t)(M) ^ mparams.magic))
-
-#define get_mstate_for(p)\
-  ((mstate)(((mchunkptr)((char*)(p) +\
-    (chunksize(p))))->prev_foot ^ mparams.magic))
-
-#define set_inuse(M,p,s)\
-  ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\
-  (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT), \
-  mark_inuse_foot(M,p,s))
-
-#define set_inuse_and_pinuse(M,p,s)\
-  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
-  (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT),\
- mark_inuse_foot(M,p,s))
-
-#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\
-  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
-  mark_inuse_foot(M, p, s))
-
-#endif /* !FOOTERS */
-
-/* ---------------------------- setting mparams -------------------------- */
-
-#if LOCK_AT_FORK
-static void pre_fork(void)         { ACQUIRE_LOCK(&(gm)->mutex); }
-static void post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
-static void post_fork_child(void)  { INITIAL_LOCK(&(gm)->mutex); }
-#endif /* LOCK_AT_FORK */
-
-/* Initialize mparams */
-static int init_mparams(void) {
-#ifdef NEED_GLOBAL_LOCK_INIT
-  if (malloc_global_mutex_status <= 0)
-    init_malloc_global_mutex();
-#endif
-
-  ACQUIRE_MALLOC_GLOBAL_LOCK();
-  if (mparams.magic == 0) {
-    size_t magic;
-    size_t psize;
-    size_t gsize;
-
-#ifndef WIN32
-    psize = malloc_getpagesize;
-    gsize = ((DEFAULT_GRANULARITY != 0)? DEFAULT_GRANULARITY : psize);
-#else /* WIN32 */
-    {
-      SYSTEM_INFO system_info;
-      GetSystemInfo(&system_info);
-      psize = system_info.dwPageSize;
-      gsize = ((DEFAULT_GRANULARITY != 0)?
-               DEFAULT_GRANULARITY : system_info.dwAllocationGranularity);
-    }
-#endif /* WIN32 */
-
-    /* Sanity-check configuration:
-       size_t must be unsigned and as wide as pointer type.
-       ints must be at least 4 bytes.
-       alignment must be at least 8.
-       Alignment, min chunk size, and page size must all be powers of 2.
-    */
-    if ((sizeof(size_t) != sizeof(char*)) ||
-        (MAX_SIZE_T < MIN_CHUNK_SIZE)  ||
-        (sizeof(int) < 4)  ||
-        (MALLOC_ALIGNMENT < (size_t)8U) ||
-        ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT-SIZE_T_ONE)) != 0) ||
-        ((MCHUNK_SIZE      & (MCHUNK_SIZE-SIZE_T_ONE))      != 0) ||
-        ((gsize            & (gsize-SIZE_T_ONE))            != 0) ||
-        ((psize            & (psize-SIZE_T_ONE))            != 0))
-      ABORT;
-    mparams.granularity = gsize;
-    mparams.page_size = psize;
-    mparams.mmap_threshold = DEFAULT_MMAP_THRESHOLD;
-    mparams.trim_threshold = DEFAULT_TRIM_THRESHOLD;
-#if MORECORE_CONTIGUOUS
-    mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT;
-#else  /* MORECORE_CONTIGUOUS */
-    mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT|USE_NONCONTIGUOUS_BIT;
-#endif /* MORECORE_CONTIGUOUS */
-
-#if !ONLY_MSPACES
-    /* Set up lock for main malloc area */
-    gm->mflags = mparams.default_mflags;
-    (void)INITIAL_LOCK(&gm->mutex);
-#endif
-#if LOCK_AT_FORK
-    pthread_atfork(&pre_fork, &post_fork_parent, &post_fork_child);
-#endif
-
-    {
-#if USE_DEV_RANDOM
-      int fd;
-      unsigned char buf[sizeof(size_t)];
-      /* Try to use /dev/urandom, else fall back on using time */
-      if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 &&
-          read(fd, buf, sizeof(buf)) == sizeof(buf)) {
-        magic = *((size_t *) buf);
-        close(fd);
-      }
-      else
-#endif /* USE_DEV_RANDOM */
-#ifdef WIN32
-      magic = (size_t)(GetTickCount() ^ (size_t)0x55555555U);
-#elif defined(LACKS_TIME_H)
-      magic = (size_t)&magic ^ (size_t)0x55555555U;
-#else
-      magic = (size_t)(time(0) ^ (size_t)0x55555555U);
-#endif
-      magic |= (size_t)8U;    /* ensure nonzero */
-      magic &= ~(size_t)7U;   /* improve chances of fault for bad values */
-      /* Until memory modes commonly available, use volatile-write */
-      (*(volatile size_t *)(&(mparams.magic))) = magic;
-    }
-  }
-
-  RELEASE_MALLOC_GLOBAL_LOCK();
-  return 1;
-}
-
-/* support for mallopt */
-static int change_mparam(int param_number, int value) {
-  size_t val;
-  ensure_initialization();
-  val = (value == -1)? MAX_SIZE_T : (size_t)value;
-  switch(param_number) {
-  case M_TRIM_THRESHOLD:
-    mparams.trim_threshold = val;
-    return 1;
-  case M_GRANULARITY:
-    if (val >= mparams.page_size && ((val & (val-1)) == 0)) {
-      mparams.granularity = val;
-      return 1;
-    }
-    else
-      return 0;
-  case M_MMAP_THRESHOLD:
-    mparams.mmap_threshold = val;
-    return 1;
-  default:
-    return 0;
-  }
-}
-
-#if DEBUG
-/* ------------------------- Debugging Support --------------------------- */
-
-/* Check properties of any chunk, whether free, inuse, mmapped etc  */
-static void do_check_any_chunk(mstate m, mchunkptr p) {
-  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
-  assert(ok_address(m, p));
-}
-
-/* Check properties of top chunk */
-static void do_check_top_chunk(mstate m, mchunkptr p) {
-  msegmentptr sp = segment_holding(m, (char*)p);
-  size_t  sz = p->head & ~INUSE_BITS; /* third-lowest bit can be set! */
-  assert(sp != 0);
-  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
-  assert(ok_address(m, p));
-  assert(sz == m->topsize);
-  assert(sz > 0);
-  assert(sz == ((sp->base + sp->size) - (char*)p) - TOP_FOOT_SIZE);
-  assert(pinuse(p));
-  assert(!pinuse(chunk_plus_offset(p, sz)));
-}
-
-/* Check properties of (inuse) mmapped chunks */
-static void do_check_mmapped_chunk(mstate m, mchunkptr p) {
-  size_t  sz = chunksize(p);
-  size_t len = (sz + (p->prev_foot) + MMAP_FOOT_PAD);
-  assert(is_mmapped(p));
-  assert(use_mmap(m));
-  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
-  assert(ok_address(m, p));
-  assert(!is_small(sz));
-  assert((len & (mparams.page_size-SIZE_T_ONE)) == 0);
-  assert(chunk_plus_offset(p, sz)->head == FENCEPOST_HEAD);
-  assert(chunk_plus_offset(p, sz+SIZE_T_SIZE)->head == 0);
-}
-
-/* Check properties of inuse chunks */
-static void do_check_inuse_chunk(mstate m, mchunkptr p) {
-  do_check_any_chunk(m, p);
-  assert(is_inuse(p));
-  assert(next_pinuse(p));
-  /* If not pinuse and not mmapped, previous chunk has OK offset */
-  assert(is_mmapped(p) || pinuse(p) || next_chunk(prev_chunk(p)) == p);
-  if (is_mmapped(p))
-    do_check_mmapped_chunk(m, p);
-}
-
-/* Check properties of free chunks */
-static void do_check_free_chunk(mstate m, mchunkptr p) {
-  size_t sz = chunksize(p);
-  mchunkptr next = chunk_plus_offset(p, sz);
-  do_check_any_chunk(m, p);
-  assert(!is_inuse(p));
-  assert(!next_pinuse(p));
-  assert (!is_mmapped(p));
-  if (p != m->dv && p != m->top) {
-    if (sz >= MIN_CHUNK_SIZE) {
-      assert((sz & CHUNK_ALIGN_MASK) == 0);
-      assert(is_aligned(chunk2mem(p)));
-      assert(next->prev_foot == sz);
-      assert(pinuse(p));
-      assert (next == m->top || is_inuse(next));
-      assert(p->fd->bk == p);
-      assert(p->bk->fd == p);
-    }
-    else  /* markers are always of size SIZE_T_SIZE */
-      assert(sz == SIZE_T_SIZE);
-  }
-}
-
-/* Check properties of malloced chunks at the point they are malloced */
-static void do_check_malloced_chunk(mstate m, void* mem, size_t s) {
-  if (mem != 0) {
-    mchunkptr p = mem2chunk(mem);
-    size_t sz = p->head & ~INUSE_BITS;
-    do_check_inuse_chunk(m, p);
-    assert((sz & CHUNK_ALIGN_MASK) == 0);
-    assert(sz >= MIN_CHUNK_SIZE);
-    assert(sz >= s);
-    /* unless mmapped, size is less than MIN_CHUNK_SIZE more than request */
-    assert(is_mmapped(p) || sz < (s + MIN_CHUNK_SIZE));
-  }
-}
-
-/* Check a tree and its subtrees.  */
-static void do_check_tree(mstate m, tchunkptr t) {
-  tchunkptr head = 0;
-  tchunkptr u = t;
-  bindex_t tindex = t->index;
-  size_t tsize = chunksize(t);
-  bindex_t idx;
-  compute_tree_index(tsize, idx);
-  assert(tindex == idx);
-  assert(tsize >= MIN_LARGE_SIZE);
-  assert(tsize >= minsize_for_tree_index(idx));
-  assert((idx == NTREEBINS-1) || (tsize < minsize_for_tree_index((idx+1))));
-
-  do { /* traverse through chain of same-sized nodes */
-    do_check_any_chunk(m, ((mchunkptr)u));
-    assert(u->index == tindex);
-    assert(chunksize(u) == tsize);
-    assert(!is_inuse(u));
-    assert(!next_pinuse(u));
-    assert(u->fd->bk == u);
-    assert(u->bk->fd == u);
-    if (u->parent == 0) {
-      assert(u->child[0] == 0);
-      assert(u->child[1] == 0);
-    }
-    else {
-      assert(head == 0); /* only one node on chain has parent */
-      head = u;
-      assert(u->parent != u);
-      assert (u->parent->child[0] == u ||
-              u->parent->child[1] == u ||
-              *((tbinptr*)(u->parent)) == u);
-      if (u->child[0] != 0) {
-        assert(u->child[0]->parent == u);
-        assert(u->child[0] != u);
-        do_check_tree(m, u->child[0]);
-      }
-      if (u->child[1] != 0) {
-        assert(u->child[1]->parent == u);
-        assert(u->child[1] != u);
-        do_check_tree(m, u->child[1]);
-      }
-      if (u->child[0] != 0 && u->child[1] != 0) {
-        assert(chunksize(u->child[0]) < chunksize(u->child[1]));
-      }
-    }
-    u = u->fd;
-  } while (u != t);
-  assert(head != 0);
-}
-
-/*  Check all the chunks in a treebin.  */
-static void do_check_treebin(mstate m, bindex_t i) {
-  tbinptr* tb = treebin_at(m, i);
-  tchunkptr t = *tb;
-  int empty = (m->treemap & (1U << i)) == 0;
-  if (t == 0)
-    assert(empty);
-  if (!empty)
-    do_check_tree(m, t);
-}
-
-/*  Check all the chunks in a smallbin.  */
-static void do_check_smallbin(mstate m, bindex_t i) {
-  sbinptr b = smallbin_at(m, i);
-  mchunkptr p = b->bk;
-  unsigned int empty = (m->smallmap & (1U << i)) == 0;
-  if (p == b)
-    assert(empty);
-  if (!empty) {
-    for (; p != b; p = p->bk) {
-      size_t size = chunksize(p);
-      mchunkptr q;
-      /* each chunk claims to be free */
-      do_check_free_chunk(m, p);
-      /* chunk belongs in bin */
-      assert(small_index(size) == i);
-      assert(p->bk == b || chunksize(p->bk) == chunksize(p));
-      /* chunk is followed by an inuse chunk */
-      q = next_chunk(p);
-      if (q->head != FENCEPOST_HEAD)
-        do_check_inuse_chunk(m, q);
-    }
-  }
-}
-
-/* Find x in a bin. Used in other check functions. */
-static int bin_find(mstate m, mchunkptr x) {
-  size_t size = chunksize(x);
-  if (is_small(size)) {
-    bindex_t sidx = small_index(size);
-    sbinptr b = smallbin_at(m, sidx);
-    if (smallmap_is_marked(m, sidx)) {
-      mchunkptr p = b;
-      do {
-        if (p == x)
-          return 1;
-      } while ((p = p->fd) != b);
-    }
-  }
-  else {
-    bindex_t tidx;
-    compute_tree_index(size, tidx);
-    if (treemap_is_marked(m, tidx)) {
-      tchunkptr t = *treebin_at(m, tidx);
-      size_t sizebits = size << leftshift_for_tree_index(tidx);
-      while (t != 0 && chunksize(t) != size) {
-        t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1];
-        sizebits <<= 1;
-      }
-      if (t != 0) {
-        tchunkptr u = t;
-        do {
-          if (u == (tchunkptr)x)
-            return 1;
-        } while ((u = u->fd) != t);
-      }
-    }
-  }
-  return 0;
-}
-
-/* Traverse each chunk and check it; return total */
-static size_t traverse_and_check(mstate m) {
-  size_t sum = 0;
-  if (is_initialized(m)) {
-    msegmentptr s = &m->seg;
-    sum += m->topsize + TOP_FOOT_SIZE;
-    while (s != 0) {
-      mchunkptr q = align_as_chunk(s->base);
-      mchunkptr lastq = 0;
-      assert(pinuse(q));
-      while (segment_holds(s, q) &&
-             q != m->top && q->head != FENCEPOST_HEAD) {
-        sum += chunksize(q);
-        if (is_inuse(q)) {
-          assert(!bin_find(m, q));
-          do_check_inuse_chunk(m, q);
-        }
-        else {
-          assert(q == m->dv || bin_find(m, q));
-          assert(lastq == 0 || is_inuse(lastq)); /* Not 2 consecutive free */
-          do_check_free_chunk(m, q);
-        }
-        lastq = q;
-        q = next_chunk(q);
-      }
-      s = s->next;
-    }
-  }
-  return sum;
-}
-
-
-/* Check all properties of malloc_state. */
-static void do_check_malloc_state(mstate m) {
-  bindex_t i;
-  size_t total;
-  /* check bins */
-  for (i = 0; i < NSMALLBINS; ++i)
-    do_check_smallbin(m, i);
-  for (i = 0; i < NTREEBINS; ++i)
-    do_check_treebin(m, i);
-
-  if (m->dvsize != 0) { /* check dv chunk */
-    do_check_any_chunk(m, m->dv);
-    assert(m->dvsize == chunksize(m->dv));
-    assert(m->dvsize >= MIN_CHUNK_SIZE);
-    assert(bin_find(m, m->dv) == 0);
-  }
-
-  if (m->top != 0) {   /* check top chunk */
-    do_check_top_chunk(m, m->top);
-    /*assert(m->topsize == chunksize(m->top)); redundant */
-    assert(m->topsize > 0);
-    assert(bin_find(m, m->top) == 0);
-  }
-
-  total = traverse_and_check(m);
-  assert(total <= m->footprint);
-  assert(m->footprint <= m->max_footprint);
-}
-#endif /* DEBUG */
-
-/* ----------------------------- statistics ------------------------------ */
-
-#if !NO_MALLINFO
-static struct mallinfo internal_mallinfo(mstate m) {
-  struct mallinfo nm = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
-  ensure_initialization();
-  if (!PREACTION(m)) {
-    check_malloc_state(m);
-    if (is_initialized(m)) {
-      size_t nfree = SIZE_T_ONE; /* top always free */
-      size_t mfree = m->topsize + TOP_FOOT_SIZE;
-      size_t sum = mfree;
-      msegmentptr s = &m->seg;
-      while (s != 0) {
-        mchunkptr q = align_as_chunk(s->base);
-        while (segment_holds(s, q) &&
-               q != m->top && q->head != FENCEPOST_HEAD) {
-          size_t sz = chunksize(q);
-          sum += sz;
-          if (!is_inuse(q)) {
-            mfree += sz;
-            ++nfree;
-          }
-          q = next_chunk(q);
-        }
-        s = s->next;
-      }
-
-      nm.arena    = sum;
-      nm.ordblks  = nfree;
-      nm.hblkhd   = m->footprint - sum;
-      nm.usmblks  = m->max_footprint;
-      nm.uordblks = m->footprint - mfree;
-      nm.fordblks = mfree;
-      nm.keepcost = m->topsize;
-    }
-
-    POSTACTION(m);
-  }
-  return nm;
-}
-#endif /* !NO_MALLINFO */
-
-#if !NO_MALLOC_STATS
-static void internal_malloc_stats(mstate m) {
-  ensure_initialization();
-  if (!PREACTION(m)) {
-    size_t maxfp = 0;
-    size_t fp = 0;
-    size_t used = 0;
-    check_malloc_state(m);
-    if (is_initialized(m)) {
-      msegmentptr s = &m->seg;
-      maxfp = m->max_footprint;
-      fp = m->footprint;
-      used = fp - (m->topsize + TOP_FOOT_SIZE);
-
-      while (s != 0) {
-        mchunkptr q = align_as_chunk(s->base);
-        while (segment_holds(s, q) &&
-               q != m->top && q->head != FENCEPOST_HEAD) {
-          if (!is_inuse(q))
-            used -= chunksize(q);
-          q = next_chunk(q);
-        }
-        s = s->next;
-      }
-    }
-    POSTACTION(m); /* drop lock */
-    fprintf(stderr, "max system bytes = %10lu\n", (unsigned long)(maxfp));
-    fprintf(stderr, "system bytes     = %10lu\n", (unsigned long)(fp));
-    fprintf(stderr, "in use bytes     = %10lu\n", (unsigned long)(used));
-  }
-}
-#endif /* NO_MALLOC_STATS */
-
-/* ----------------------- Operations on smallbins ----------------------- */
-
-/*
-  Various forms of linking and unlinking are defined as macros.  Even
-  the ones for trees, which are very long but have very short typical
-  paths.  This is ugly but reduces reliance on inlining support of
-  compilers.
-*/
-
-/* Link a free chunk into a smallbin  */
-#define insert_small_chunk(M, P, S) {\
-  bindex_t I  = small_index(S);\
-  mchunkptr B = smallbin_at(M, I);\
-  mchunkptr F = B;\
-  assert(S >= MIN_CHUNK_SIZE);\
-  if (!smallmap_is_marked(M, I))\
-    mark_smallmap(M, I);\
-  else if (RTCHECK(ok_address(M, B->fd)))\
-    F = B->fd;\
-  else {\
-    CORRUPTION_ERROR_ACTION(M);\
-  }\
-  B->fd = P;\
-  F->bk = P;\
-  P->fd = F;\
-  P->bk = B;\
-}
-
-/* Unlink a chunk from a smallbin  */
-#define unlink_small_chunk(M, P, S) {\
-  mchunkptr F = P->fd;\
-  mchunkptr B = P->bk;\
-  bindex_t I = small_index(S);\
-  assert(P != B);\
-  assert(P != F);\
-  assert(chunksize(P) == small_index2size(I));\
-  if (RTCHECK(F == smallbin_at(M,I) || (ok_address(M, F) && F->bk == P))) { \
-    if (B == F) {\
-      clear_smallmap(M, I);\
-    }\
-    else if (RTCHECK(B == smallbin_at(M,I) ||\
-                     (ok_address(M, B) && B->fd == P))) {\
-      F->bk = B;\
-      B->fd = F;\
-    }\
-    else {\
-      CORRUPTION_ERROR_ACTION(M);\
-    }\
-  }\
-  else {\
-    CORRUPTION_ERROR_ACTION(M);\
-  }\
-}
-
-/* Unlink the first chunk from a smallbin */
-#define unlink_first_small_chunk(M, B, P, I) {\
-  mchunkptr F = P->fd;\
-  assert(P != B);\
-  assert(P != F);\
-  assert(chunksize(P) == small_index2size(I));\
-  if (B == F) {\
-    clear_smallmap(M, I);\
-  }\
-  else if (RTCHECK(ok_address(M, F) && F->bk == P)) {\
-    F->bk = B;\
-    B->fd = F;\
-  }\
-  else {\
-    CORRUPTION_ERROR_ACTION(M);\
-  }\
-}
-
-/* Replace dv node, binning the old one */
-/* Used only when dvsize known to be small */
-#define replace_dv(M, P, S) {\
-  size_t DVS = M->dvsize;\
-  assert(is_small(DVS));\
-  if (DVS != 0) {\
-    mchunkptr DV = M->dv;\
-    insert_small_chunk(M, DV, DVS);\
-  }\
-  M->dvsize = S;\
-  M->dv = P;\
-}
-
-/* ------------------------- Operations on trees ------------------------- */
-
-/* Insert chunk into tree */
-#define insert_large_chunk(M, X, S) {\
-  tbinptr* H;\
-  bindex_t I;\
-  compute_tree_index(S, I);\
-  H = treebin_at(M, I);\
-  X->index = I;\
-  X->child[0] = X->child[1] = 0;\
-  if (!treemap_is_marked(M, I)) {\
-    mark_treemap(M, I);\
-    *H = X;\
-    X->parent = (tchunkptr)H;\
-    X->fd = X->bk = X;\
-  }\
-  else {\
-    tchunkptr T = *H;\
-    size_t K = S << leftshift_for_tree_index(I);\
-    for (;;) {\
-      if (chunksize(T) != S) {\
-        tchunkptr* C = &(T->child[(K >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]);\
-        K <<= 1;\
-        if (*C != 0)\
-          T = *C;\
-        else if (RTCHECK(ok_address(M, C))) {\
-          *C = X;\
-          X->parent = T;\
-          X->fd = X->bk = X;\
-          break;\
-        }\
-        else {\
-          CORRUPTION_ERROR_ACTION(M);\
-          break;\
-        }\
-      }\
-      else {\
-        tchunkptr F = T->fd;\
-        if (RTCHECK(ok_address(M, T) && ok_address(M, F))) {\
-          T->fd = F->bk = X;\
-          X->fd = F;\
-          X->bk = T;\
-          X->parent = 0;\
-          break;\
-        }\
-        else {\
-          CORRUPTION_ERROR_ACTION(M);\
-          break;\
-        }\
-      }\
-    }\
-  }\
-}
-
-/*
-  Unlink steps:
-
-  1. If x is a chained node, unlink it from its same-sized fd/bk links
-     and choose its bk node as its replacement.
-  2. If x was the last node of its size, but not a leaf node, it must
-     be replaced with a leaf node (not merely one with an open left or
-     right), to make sure that lefts and rights of descendents
-     correspond properly to bit masks.  We use the rightmost descendent
-     of x.  We could use any other leaf, but this is easy to locate and
-     tends to counteract removal of leftmosts elsewhere, and so keeps
-     paths shorter than minimally guaranteed.  This doesn't loop much
-     because on average a node in a tree is near the bottom.
-  3. If x is the base of a chain (i.e., has parent links) relink
-     x's parent and children to x's replacement (or null if none).
-*/
-
-#define unlink_large_chunk(M, X) {\
-  tchunkptr XP = X->parent;\
-  tchunkptr R;\
-  if (X->bk != X) {\
-    tchunkptr F = X->fd;\
-    R = X->bk;\
-    if (RTCHECK(ok_address(M, F) && F->bk == X && R->fd == X)) {\
-      F->bk = R;\
-      R->fd = F;\
-    }\
-    else {\
-      CORRUPTION_ERROR_ACTION(M);\
-    }\
-  }\
-  else {\
-    tchunkptr* RP;\
-    if (((R = *(RP = &(X->child[1]))) != 0) ||\
-        ((R = *(RP = &(X->child[0]))) != 0)) {\
-      tchunkptr* CP;\
-      while ((*(CP = &(R->child[1])) != 0) ||\
-             (*(CP = &(R->child[0])) != 0)) {\
-        R = *(RP = CP);\
-      }\
-      if (RTCHECK(ok_address(M, RP)))\
-        *RP = 0;\
-      else {\
-        CORRUPTION_ERROR_ACTION(M);\
-      }\
-    }\
-  }\
-  if (XP != 0) {\
-    tbinptr* H = treebin_at(M, X->index);\
-    if (X == *H) {\
-      if ((*H = R) == 0) \
-        clear_treemap(M, X->index);\
-    }\
-    else if (RTCHECK(ok_address(M, XP))) {\
-      if (XP->child[0] == X) \
-        XP->child[0] = R;\
-      else \
-        XP->child[1] = R;\
-    }\
-    else\
-      CORRUPTION_ERROR_ACTION(M);\
-    if (R != 0) {\
-      if (RTCHECK(ok_address(M, R))) {\
-        tchunkptr C0, C1;\
-        R->parent = XP;\
-        if ((C0 = X->child[0]) != 0) {\
-          if (RTCHECK(ok_address(M, C0))) {\
-            R->child[0] = C0;\
-            C0->parent = R;\
-          }\
-          else\
-            CORRUPTION_ERROR_ACTION(M);\
-        }\
-        if ((C1 = X->child[1]) != 0) {\
-          if (RTCHECK(ok_address(M, C1))) {\
-            R->child[1] = C1;\
-            C1->parent = R;\
-          }\
-          else\
-            CORRUPTION_ERROR_ACTION(M);\
-        }\
-      }\
-      else\
-        CORRUPTION_ERROR_ACTION(M);\
-    }\
-  }\
-}
-
-/* Relays to large vs small bin operations */
-
-#define insert_chunk(M, P, S)\
-  if (is_small(S)) insert_small_chunk(M, P, S)\
-  else { tchunkptr TP = (tchunkptr)(P); insert_large_chunk(M, TP, S); }
-
-#define unlink_chunk(M, P, S)\
-  if (is_small(S)) unlink_small_chunk(M, P, S)\
-  else { tchunkptr TP = (tchunkptr)(P); unlink_large_chunk(M, TP); }
-
-
-/* Relays to internal calls to malloc/free from realloc, memalign etc */
-
-#if ONLY_MSPACES
-#define internal_malloc(m, b) mspace_malloc(m, b)
-#define internal_free(m, mem) mspace_free(m,mem);
-#else /* ONLY_MSPACES */
-#if MSPACES
-#define internal_malloc(m, b)\
-  ((m == gm)? dlmalloc(b) : mspace_malloc(m, b))
-#define internal_free(m, mem)\
-   if (m == gm) dlfree(mem); else mspace_free(m,mem);
-#else /* MSPACES */
-#define internal_malloc(m, b) dlmalloc(b)
-#define internal_free(m, mem) dlfree(mem)
-#endif /* MSPACES */
-#endif /* ONLY_MSPACES */
-
-/* -----------------------  Direct-mmapping chunks ----------------------- */
-
-/*
-  Directly mmapped chunks are set up with an offset to the start of
-  the mmapped region stored in the prev_foot field of the chunk. This
-  allows reconstruction of the required argument to MUNMAP when freed,
-  and also allows adjustment of the returned chunk to meet alignment
-  requirements (especially in memalign).
-*/
-
-/* Malloc using mmap */
-static void* mmap_alloc(mstate m, size_t nb) {
-  size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
-  if (m->footprint_limit != 0) {
-    size_t fp = m->footprint + mmsize;
-    if (fp <= m->footprint || fp > m->footprint_limit)
-      return 0;
-  }
-  if (mmsize > nb) {     /* Check for wrap around 0 */
-    char* mm = (char*)(CALL_DIRECT_MMAP(mmsize));
-    if (mm != CMFAIL) {
-      size_t offset = align_offset(chunk2mem(mm));
-      size_t psize = mmsize - offset - MMAP_FOOT_PAD;
-      mchunkptr p = (mchunkptr)(mm + offset);
-      p->prev_foot = offset;
-      p->head = psize;
-      mark_inuse_foot(m, p, psize);
-      chunk_plus_offset(p, psize)->head = FENCEPOST_HEAD;
-      chunk_plus_offset(p, psize+SIZE_T_SIZE)->head = 0;
-
-      if (m->least_addr == 0 || mm < m->least_addr)
-        m->least_addr = mm;
-      if ((m->footprint += mmsize) > m->max_footprint)
-        m->max_footprint = m->footprint;
-      assert(is_aligned(chunk2mem(p)));
-      check_mmapped_chunk(m, p);
-      return chunk2mem(p);
-    }
-  }
-  return 0;
-}
-
-/* Realloc using mmap */
-static mchunkptr mmap_resize(mstate m, mchunkptr oldp, size_t nb, int flags) {
-  size_t oldsize = chunksize(oldp);
-  (void)flags; /* placate people compiling -Wunused */
-  if (is_small(nb)) /* Can't shrink mmap regions below small size */
-    return 0;
-  /* Keep old chunk if big enough but not too big */
-  if (oldsize >= nb + SIZE_T_SIZE &&
-      (oldsize - nb) <= (mparams.granularity << 1))
-    return oldp;
-  else {
-    size_t offset = oldp->prev_foot;
-    size_t oldmmsize = oldsize + offset + MMAP_FOOT_PAD;
-    size_t newmmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
-    char* cp = (char*)CALL_MREMAP((char*)oldp - offset,
-                                  oldmmsize, newmmsize, flags);
-    if (cp != CMFAIL) {
-      mchunkptr newp = (mchunkptr)(cp + offset);
-      size_t psize = newmmsize - offset - MMAP_FOOT_PAD;
-      newp->head = psize;
-      mark_inuse_foot(m, newp, psize);
-      chunk_plus_offset(newp, psize)->head = FENCEPOST_HEAD;
-      chunk_plus_offset(newp, psize+SIZE_T_SIZE)->head = 0;
-
-      if (cp < m->least_addr)
-        m->least_addr = cp;
-      if ((m->footprint += newmmsize - oldmmsize) > m->max_footprint)
-        m->max_footprint = m->footprint;
-      check_mmapped_chunk(m, newp);
-      return newp;
-    }
-  }
-  return 0;
-}
-
-
-/* -------------------------- mspace management -------------------------- */
-
-/* Initialize top chunk and its size */
-static void init_top(mstate m, mchunkptr p, size_t psize) {
-  /* Ensure alignment */
-  size_t offset = align_offset(chunk2mem(p));
-  p = (mchunkptr)((char*)p + offset);
-  psize -= offset;
-
-  m->top = p;
-  m->topsize = psize;
-  p->head = psize | PINUSE_BIT;
-  /* set size of fake trailing chunk holding overhead space only once */
-  chunk_plus_offset(p, psize)->head = TOP_FOOT_SIZE;
-  m->trim_check = mparams.trim_threshold; /* reset on each update */
-}
-
-/* Initialize bins for a new mstate that is otherwise zeroed out */
-static void init_bins(mstate m) {
-  /* Establish circular links for smallbins */
-  bindex_t i;
-  for (i = 0; i < NSMALLBINS; ++i) {
-    sbinptr bin = smallbin_at(m,i);
-    bin->fd = bin->bk = bin;
-  }
-}
-
-#if PROCEED_ON_ERROR
-
-/* default corruption action */
-static void reset_on_error(mstate m) {
-  int i;
-  ++malloc_corruption_error_count;
-  /* Reinitialize fields to forget about all memory */
-  m->smallmap = m->treemap = 0;
-  m->dvsize = m->topsize = 0;
-  m->seg.base = 0;
-  m->seg.size = 0;
-  m->seg.next = 0;
-  m->top = m->dv = 0;
-  for (i = 0; i < NTREEBINS; ++i)
-    *treebin_at(m, i) = 0;
-  init_bins(m);
-}
-#endif /* PROCEED_ON_ERROR */
-
-/* Allocate chunk and prepend remainder with chunk in successor base. */
-static void* prepend_alloc(mstate m, char* newbase, char* oldbase,
-                           size_t nb) {
-  mchunkptr p = align_as_chunk(newbase);
-  mchunkptr oldfirst = align_as_chunk(oldbase);
-  size_t psize = (char*)oldfirst - (char*)p;
-  mchunkptr q = chunk_plus_offset(p, nb);
-  size_t qsize = psize - nb;
-  set_size_and_pinuse_of_inuse_chunk(m, p, nb);
-
-  assert((char*)oldfirst > (char*)q);
-  assert(pinuse(oldfirst));
-  assert(qsize >= MIN_CHUNK_SIZE);
-
-  /* consolidate remainder with first chunk of old base */
-  if (oldfirst == m->top) {
-    size_t tsize = m->topsize += qsize;
-    m->top = q;
-    q->head = tsize | PINUSE_BIT;
-    check_top_chunk(m, q);
-  }
-  else if (oldfirst == m->dv) {
-    size_t dsize = m->dvsize += qsize;
-    m->dv = q;
-    set_size_and_pinuse_of_free_chunk(q, dsize);
-  }
-  else {
-    if (!is_inuse(oldfirst)) {
-      size_t nsize = chunksize(oldfirst);
-      unlink_chunk(m, oldfirst, nsize);
-      oldfirst = chunk_plus_offset(oldfirst, nsize);
-      qsize += nsize;
-    }
-    set_free_with_pinuse(q, qsize, oldfirst);
-    insert_chunk(m, q, qsize);
-    check_free_chunk(m, q);
-  }
-
-  check_malloced_chunk(m, chunk2mem(p), nb);
-  return chunk2mem(p);
-}
-
-/* Add a segment to hold a new noncontiguous region */
-static void add_segment(mstate m, char* tbase, size_t tsize, flag_t mmapped) {
-  /* Determine locations and sizes of segment, fenceposts, old top */
-  char* old_top = (char*)m->top;
-  msegmentptr oldsp = segment_holding(m, old_top);
-  char* old_end = oldsp->base + oldsp->size;
-  size_t ssize = pad_request(sizeof(struct malloc_segment));
-  char* rawsp = old_end - (ssize + FOUR_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
-  size_t offset = align_offset(chunk2mem(rawsp));
-  char* asp = rawsp + offset;
-  char* csp = (asp < (old_top + MIN_CHUNK_SIZE))? old_top : asp;
-  mchunkptr sp = (mchunkptr)csp;
-  msegmentptr ss = (msegmentptr)(chunk2mem(sp));
-  mchunkptr tnext = chunk_plus_offset(sp, ssize);
-  mchunkptr p = tnext;
-  int nfences = 0;
-
-  /* reset top to new space */
-  init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE);
-
-  /* Set up segment record */
-  assert(is_aligned(ss));
-  set_size_and_pinuse_of_inuse_chunk(m, sp, ssize);
-  *ss = m->seg; /* Push current record */
-  m->seg.base = tbase;
-  m->seg.size = tsize;
-  m->seg.sflags = mmapped;
-  m->seg.next = ss;
-
-  /* Insert trailing fenceposts */
-  for (;;) {
-    mchunkptr nextp = chunk_plus_offset(p, SIZE_T_SIZE);
-    p->head = FENCEPOST_HEAD;
-    ++nfences;
-    if ((char*)(&(nextp->head)) < old_end)
-      p = nextp;
-    else
-      break;
-  }
-  assert(nfences >= 2);
-
-  /* Insert the rest of old top into a bin as an ordinary free chunk */
-  if (csp != old_top) {
-    mchunkptr q = (mchunkptr)old_top;
-    size_t psize = csp - old_top;
-    mchunkptr tn = chunk_plus_offset(q, psize);
-    set_free_with_pinuse(q, psize, tn);
-    insert_chunk(m, q, psize);
-  }
-
-  check_top_chunk(m, m->top);
-}
-
-/* -------------------------- System allocation -------------------------- */
-
-/* Get memory from system using MORECORE or MMAP */
-static void* sys_alloc(mstate m, size_t nb) {
-  char* tbase = CMFAIL;
-  size_t tsize = 0;
-  flag_t mmap_flag = 0;
-  size_t asize; /* allocation size */
-
-  ensure_initialization();
-
-  /* Directly map large chunks, but only if already initialized */
-  if (use_mmap(m) && nb >= mparams.mmap_threshold && m->topsize != 0) {
-    void* mem = mmap_alloc(m, nb);
-    if (mem != 0)
-      return mem;
-  }
-
-  asize = granularity_align(nb + SYS_ALLOC_PADDING);
-  if (asize <= nb)
-    return 0; /* wraparound */
-  if (m->footprint_limit != 0) {
-    size_t fp = m->footprint + asize;
-    if (fp <= m->footprint || fp > m->footprint_limit)
-      return 0;
-  }
-
-  /*
-    Try getting memory in any of three ways (in most-preferred to
-    least-preferred order):
-    1. A call to MORECORE that can normally contiguously extend memory.
-       (disabled if not MORECORE_CONTIGUOUS or not HAVE_MORECORE or
-       or main space is mmapped or a previous contiguous call failed)
-    2. A call to MMAP new space (disabled if not HAVE_MMAP).
-       Note that under the default settings, if MORECORE is unable to
-       fulfill a request, and HAVE_MMAP is true, then mmap is
-       used as a noncontiguous system allocator. This is a useful backup
-       strategy for systems with holes in address spaces -- in this case
-       sbrk cannot contiguously expand the heap, but mmap may be able to
-       find space.
-    3. A call to MORECORE that cannot usually contiguously extend memory.
-       (disabled if not HAVE_MORECORE)
-
-   In all cases, we need to request enough bytes from system to ensure
-   we can malloc nb bytes upon success, so pad with enough space for
-   top_foot, plus alignment-pad to make sure we don't lose bytes if
-   not on boundary, and round this up to a granularity unit.
-  */
-
-  if (MORECORE_CONTIGUOUS && !use_noncontiguous(m)) {
-    char* br = CMFAIL;
-    size_t ssize = asize; /* sbrk call size */
-    msegmentptr ss = (m->top == 0)? 0 : segment_holding(m, (char*)m->top);
-    ACQUIRE_MALLOC_GLOBAL_LOCK();
-
-    if (ss == 0) {  /* First time through or recovery */
-      char* base = (char*)CALL_MORECORE(0);
-      if (base != CMFAIL) {
-        size_t fp;
-        /* Adjust to end on a page boundary */
-        if (!is_page_aligned(base))
-          ssize += (page_align((size_t)base) - (size_t)base);
-        fp = m->footprint + ssize; /* recheck limits */
-        if (ssize > nb && ssize < HALF_MAX_SIZE_T &&
-            (m->footprint_limit == 0 ||
-             (fp > m->footprint && fp <= m->footprint_limit)) &&
-            (br = (char*)(CALL_MORECORE(ssize))) == base) {
-          tbase = base;
-          tsize = ssize;
-        }
-      }
-    }
-    else {
-      /* Subtract out existing available top space from MORECORE request. */
-      ssize = granularity_align(nb - m->topsize + SYS_ALLOC_PADDING);
-      /* Use mem here only if it did continuously extend old space */
-      if (ssize < HALF_MAX_SIZE_T &&
-          (br = (char*)(CALL_MORECORE(ssize))) == ss->base+ss->size) {
-        tbase = br;
-        tsize = ssize;
-      }
-    }
-
-    if (tbase == CMFAIL) {    /* Cope with partial failure */
-      if (br != CMFAIL) {    /* Try to use/extend the space we did get */
-        if (ssize < HALF_MAX_SIZE_T &&
-            ssize < nb + SYS_ALLOC_PADDING) {
-          size_t esize = granularity_align(nb + SYS_ALLOC_PADDING - ssize);
-          if (esize < HALF_MAX_SIZE_T) {
-            char* end = (char*)CALL_MORECORE(esize);
-            if (end != CMFAIL)
-              ssize += esize;
-            else {            /* Can't use; try to release */
-              (void) CALL_MORECORE(-ssize);
-              br = CMFAIL;
-            }
-          }
-        }
-      }
-      if (br != CMFAIL) {    /* Use the space we did get */
-        tbase = br;
-        tsize = ssize;
-      }
-      else
-        disable_contiguous(m); /* Don't try contiguous path in the future */
-    }
-
-    RELEASE_MALLOC_GLOBAL_LOCK();
-  }
-
-  if (HAVE_MMAP && tbase == CMFAIL) {  /* Try MMAP */
-    char* mp = (char*)(CALL_MMAP(asize));
-    if (mp != CMFAIL) {
-      tbase = mp;
-      tsize = asize;
-      mmap_flag = USE_MMAP_BIT;
-    }
-  }
-
-  if (HAVE_MORECORE && tbase == CMFAIL) { /* Try noncontiguous MORECORE */
-    if (asize < HALF_MAX_SIZE_T) {
-      char* br = CMFAIL;
-      char* end = CMFAIL;
-      ACQUIRE_MALLOC_GLOBAL_LOCK();
-      br = (char*)(CALL_MORECORE(asize));
-      end = (char*)(CALL_MORECORE(0));
-      RELEASE_MALLOC_GLOBAL_LOCK();
-      if (br != CMFAIL && end != CMFAIL && br < end) {
-        size_t ssize = end - br;
-        if (ssize > nb + TOP_FOOT_SIZE) {
-          tbase = br;
-          tsize = ssize;
-        }
-      }
-    }
-  }
-
-  if (tbase != CMFAIL) {
-
-    if ((m->footprint += tsize) > m->max_footprint)
-      m->max_footprint = m->footprint;
-
-    if (!is_initialized(m)) { /* first-time initialization */
-      if (m->least_addr == 0 || tbase < m->least_addr)
-        m->least_addr = tbase;
-      m->seg.base = tbase;
-      m->seg.size = tsize;
-      m->seg.sflags = mmap_flag;
-      m->magic = mparams.magic;
-      m->release_checks = MAX_RELEASE_CHECK_RATE;
-      init_bins(m);
-#if !ONLY_MSPACES
-      if (is_global(m))
-        init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE);
-      else
-#endif
-      {
-        /* Offset top by embedded malloc_state */
-        mchunkptr mn = next_chunk(mem2chunk(m));
-        init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) -TOP_FOOT_SIZE);
-      }
-    }
-
-    else {
-      /* Try to merge with an existing segment */
-      msegmentptr sp = &m->seg;
-      /* Only consider most recent segment if traversal suppressed */
-      while (sp != 0 && tbase != sp->base + sp->size)
-        sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next;
-      if (sp != 0 &&
-          !is_extern_segment(sp) &&
-          (sp->sflags & USE_MMAP_BIT) == mmap_flag &&
-          segment_holds(sp, m->top)) { /* append */
-        sp->size += tsize;
-        init_top(m, m->top, m->topsize + tsize);
-      }
-      else {
-        if (tbase < m->least_addr)
-          m->least_addr = tbase;
-        sp = &m->seg;
-        while (sp != 0 && sp->base != tbase + tsize)
-          sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next;
-        if (sp != 0 &&
-            !is_extern_segment(sp) &&
-            (sp->sflags & USE_MMAP_BIT) == mmap_flag) {
-          char* oldbase = sp->base;
-          sp->base = tbase;
-          sp->size += tsize;
-          return prepend_alloc(m, tbase, oldbase, nb);
-        }
-        else
-          add_segment(m, tbase, tsize, mmap_flag);
-      }
-    }
-
-    if (nb < m->topsize) { /* Allocate from new or extended top space */
-      size_t rsize = m->topsize -= nb;
-      mchunkptr p = m->top;
-      mchunkptr r = m->top = chunk_plus_offset(p, nb);
-      r->head = rsize | PINUSE_BIT;
-      set_size_and_pinuse_of_inuse_chunk(m, p, nb);
-      check_top_chunk(m, m->top);
-      check_malloced_chunk(m, chunk2mem(p), nb);
-      return chunk2mem(p);
-    }
-  }
-
-  MALLOC_FAILURE_ACTION;
-  return 0;
-}
-
-/* -----------------------  system deallocation -------------------------- */
-
-/* Unmap and unlink any mmapped segments that don't contain used chunks */
-static size_t release_unused_segments(mstate m) {
-  size_t released = 0;
-  int nsegs = 0;
-  msegmentptr pred = &m->seg;
-  msegmentptr sp = pred->next;
-  while (sp != 0) {
-    char* base = sp->base;
-    size_t size = sp->size;
-    msegmentptr next = sp->next;
-    ++nsegs;
-    if (is_mmapped_segment(sp) && !is_extern_segment(sp)) {
-      mchunkptr p = align_as_chunk(base);
-      size_t psize = chunksize(p);
-      /* Can unmap if first chunk holds entire segment and not pinned */
-      if (!is_inuse(p) && (char*)p + psize >= base + size - TOP_FOOT_SIZE) {
-        tchunkptr tp = (tchunkptr)p;
-        assert(segment_holds(sp, (char*)sp));
-        if (p == m->dv) {
-          m->dv = 0;
-          m->dvsize = 0;
-        }
-        else {
-          unlink_large_chunk(m, tp);
-        }
-        if (CALL_MUNMAP(base, size) == 0) {
-          released += size;
-          m->footprint -= size;
-          /* unlink obsoleted record */
-          sp = pred;
-          sp->next = next;
-        }
-        else { /* back out if cannot unmap */
-          insert_large_chunk(m, tp, psize);
-        }
-      }
-    }
-    if (NO_SEGMENT_TRAVERSAL) /* scan only first segment */
-      break;
-    pred = sp;
-    sp = next;
-  }
-  /* Reset check counter */
-  m->release_checks = (((size_t) nsegs > (size_t) MAX_RELEASE_CHECK_RATE)?
-                       (size_t) nsegs : (size_t) MAX_RELEASE_CHECK_RATE);
-  return released;
-}
-
-static int sys_trim(mstate m, size_t pad) {
-  size_t released = 0;
-  ensure_initialization();
-  if (pad < MAX_REQUEST && is_initialized(m)) {
-    pad += TOP_FOOT_SIZE; /* ensure enough room for segment overhead */
-
-    if (m->topsize > pad) {
-      /* Shrink top space in granularity-size units, keeping at least one */
-      size_t unit = mparams.granularity;
-      size_t extra = ((m->topsize - pad + (unit - SIZE_T_ONE)) / unit -
-                      SIZE_T_ONE) * unit;
-      msegmentptr sp = segment_holding(m, (char*)m->top);
-
-      if (!is_extern_segment(sp)) {
-        if (is_mmapped_segment(sp)) {
-          if (HAVE_MMAP &&
-              sp->size >= extra &&
-              !has_segment_link(m, sp)) { /* can't shrink if pinned */
-            size_t newsize = sp->size - extra;
-            (void)newsize; /* placate people compiling -Wunused-variable */
-            /* Prefer mremap, fall back to munmap */
-            if ((CALL_MREMAP(sp->base, sp->size, newsize, 0) != MFAIL) ||
-                (CALL_MUNMAP(sp->base + newsize, extra) == 0)) {
-              released = extra;
-            }
-          }
-        }
-        else if (HAVE_MORECORE) {
-          if (extra >= HALF_MAX_SIZE_T) /* Avoid wrapping negative */
-            extra = (HALF_MAX_SIZE_T) + SIZE_T_ONE - unit;
-          ACQUIRE_MALLOC_GLOBAL_LOCK();
-          {
-            /* Make sure end of memory is where we last set it. */
-            char* old_br = (char*)(CALL_MORECORE(0));
-            if (old_br == sp->base + sp->size) {
-              char* rel_br = (char*)(CALL_MORECORE(-extra));
-              char* new_br = (char*)(CALL_MORECORE(0));
-              if (rel_br != CMFAIL && new_br < old_br)
-                released = old_br - new_br;
-            }
-          }
-          RELEASE_MALLOC_GLOBAL_LOCK();
-        }
-      }
-
-      if (released != 0) {
-        sp->size -= released;
-        m->footprint -= released;
-        init_top(m, m->top, m->topsize - released);
-        check_top_chunk(m, m->top);
-      }
-    }
-
-    /* Unmap any unused mmapped segments */
-    if (HAVE_MMAP)
-      released += release_unused_segments(m);
-
-    /* On failure, disable autotrim to avoid repeated failed future calls */
-    if (released == 0 && m->topsize > m->trim_check)
-      m->trim_check = MAX_SIZE_T;
-  }
-
-  return (released != 0)? 1 : 0;
-}
-
-/* Consolidate and bin a chunk. Differs from exported versions
-   of free mainly in that the chunk need not be marked as inuse.
-*/
-static void dispose_chunk(mstate m, mchunkptr p, size_t psize) {
-  mchunkptr next = chunk_plus_offset(p, psize);
-  if (!pinuse(p)) {
-    mchunkptr prev;
-    size_t prevsize = p->prev_foot;
-    if (is_mmapped(p)) {
-      psize += prevsize + MMAP_FOOT_PAD;
-      if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
-        m->footprint -= psize;
-      return;
-    }
-    prev = chunk_minus_offset(p, prevsize);
-    psize += prevsize;
-    p = prev;
-    if (RTCHECK(ok_address(m, prev))) { /* consolidate backward */
-      if (p != m->dv) {
-        unlink_chunk(m, p, prevsize);
-      }
-      else if ((next->head & INUSE_BITS) == INUSE_BITS) {
-        m->dvsize = psize;
-        set_free_with_pinuse(p, psize, next);
-        return;
-      }
-    }
-    else {
-      CORRUPTION_ERROR_ACTION(m);
-      return;
-    }
-  }
-  if (RTCHECK(ok_address(m, next))) {
-    if (!cinuse(next)) {  /* consolidate forward */
-      if (next == m->top) {
-        size_t tsize = m->topsize += psize;
-        m->top = p;
-        p->head = tsize | PINUSE_BIT;
-        if (p == m->dv) {
-          m->dv = 0;
-          m->dvsize = 0;
-        }
-        return;
-      }
-      else if (next == m->dv) {
-        size_t dsize = m->dvsize += psize;
-        m->dv = p;
-        set_size_and_pinuse_of_free_chunk(p, dsize);
-        return;
-      }
-      else {
-        size_t nsize = chunksize(next);
-        psize += nsize;
-        unlink_chunk(m, next, nsize);
-        set_size_and_pinuse_of_free_chunk(p, psize);
-        if (p == m->dv) {
-          m->dvsize = psize;
-          return;
-        }
-      }
-    }
-    else {
-      set_free_with_pinuse(p, psize, next);
-    }
-    insert_chunk(m, p, psize);
-  }
-  else {
-    CORRUPTION_ERROR_ACTION(m);
-  }
-}
-
-/* ---------------------------- malloc --------------------------- */
-
-/* allocate a large request from the best fitting chunk in a treebin */
-static void* tmalloc_large(mstate m, size_t nb) {
-  tchunkptr v = 0;
-  size_t rsize = -nb; /* Unsigned negation */
-  tchunkptr t;
-  bindex_t idx;
-  compute_tree_index(nb, idx);
-  if ((t = *treebin_at(m, idx)) != 0) {
-    /* Traverse tree for this bin looking for node with size == nb */
-    size_t sizebits = nb << leftshift_for_tree_index(idx);
-    tchunkptr rst = 0;  /* The deepest untaken right subtree */
-    for (;;) {
-      tchunkptr rt;
-      size_t trem = chunksize(t) - nb;
-      if (trem < rsize) {
-        v = t;
-        if ((rsize = trem) == 0)
-          break;
-      }
-      rt = t->child[1];
-      t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1];
-      if (rt != 0 && rt != t)
-        rst = rt;
-      if (t == 0) {
-        t = rst; /* set t to least subtree holding sizes > nb */
-        break;
-      }
-      sizebits <<= 1;
-    }
-  }
-  if (t == 0 && v == 0) { /* set t to root of next non-empty treebin */
-    binmap_t leftbits = left_bits(idx2bit(idx)) & m->treemap;
-    if (leftbits != 0) {
-      bindex_t i;
-      binmap_t leastbit = least_bit(leftbits);
-      compute_bit2idx(leastbit, i);
-      t = *treebin_at(m, i);
-    }
-  }
-
-  while (t != 0) { /* find smallest of tree or subtree */
-    size_t trem = chunksize(t) - nb;
-    if (trem < rsize) {
-      rsize = trem;
-      v = t;
-    }
-    t = leftmost_child(t);
-  }
-
-  /*  If dv is a better fit, return 0 so malloc will use it */
-  if (v != 0 && rsize < (size_t)(m->dvsize - nb)) {
-    if (RTCHECK(ok_address(m, v))) { /* split */
-      mchunkptr r = chunk_plus_offset(v, nb);
-      assert(chunksize(v) == rsize + nb);
-      if (RTCHECK(ok_next(v, r))) {
-        unlink_large_chunk(m, v);
-        if (rsize < MIN_CHUNK_SIZE)
-          set_inuse_and_pinuse(m, v, (rsize + nb));
-        else {
-          set_size_and_pinuse_of_inuse_chunk(m, v, nb);
-          set_size_and_pinuse_of_free_chunk(r, rsize);
-          insert_chunk(m, r, rsize);
-        }
-        return chunk2mem(v);
-      }
-    }
-    CORRUPTION_ERROR_ACTION(m);
-  }
-  return 0;
-}
-
-/* allocate a small request from the best fitting chunk in a treebin */
-static void* tmalloc_small(mstate m, size_t nb) {
-  tchunkptr t, v;
-  size_t rsize;
-  bindex_t i;
-  binmap_t leastbit = least_bit(m->treemap);
-  compute_bit2idx(leastbit, i);
-  v = t = *treebin_at(m, i);
-  rsize = chunksize(t) - nb;
-
-  while ((t = leftmost_child(t)) != 0) {
-    size_t trem = chunksize(t) - nb;
-    if (trem < rsize) {
-      rsize = trem;
-      v = t;
-    }
-  }
-
-  if (RTCHECK(ok_address(m, v))) {
-    mchunkptr r = chunk_plus_offset(v, nb);
-    assert(chunksize(v) == rsize + nb);
-    if (RTCHECK(ok_next(v, r))) {
-      unlink_large_chunk(m, v);
-      if (rsize < MIN_CHUNK_SIZE)
-        set_inuse_and_pinuse(m, v, (rsize + nb));
-      else {
-        set_size_and_pinuse_of_inuse_chunk(m, v, nb);
-        set_size_and_pinuse_of_free_chunk(r, rsize);
-        replace_dv(m, r, rsize);
-      }
-      return chunk2mem(v);
-    }
-  }
-
-  CORRUPTION_ERROR_ACTION(m);
-  return 0;
-}
-
-#if !ONLY_MSPACES
-
-void* dlmalloc(size_t bytes) {
-  /*
-     Basic algorithm:
-     If a small request (< 256 bytes minus per-chunk overhead):
-       1. If one exists, use a remainderless chunk in associated smallbin.
-          (Remainderless means that there are too few excess bytes to
-          represent as a chunk.)
-       2. If it is big enough, use the dv chunk, which is normally the
-          chunk adjacent to the one used for the most recent small request.
-       3. If one exists, split the smallest available chunk in a bin,
-          saving remainder in dv.
-       4. If it is big enough, use the top chunk.
-       5. If available, get memory from system and use it
-     Otherwise, for a large request:
-       1. Find the smallest available binned chunk that fits, and use it
-          if it is better fitting than dv chunk, splitting if necessary.
-       2. If better fitting than any binned chunk, use the dv chunk.
-       3. If it is big enough, use the top chunk.
-       4. If request size >= mmap threshold, try to directly mmap this chunk.
-       5. If available, get memory from system and use it
-
-     The ugly goto's here ensure that postaction occurs along all paths.
-  */
-
-#if USE_LOCKS
-  ensure_initialization(); /* initialize in sys_alloc if not using locks */
-#endif
-
-  if (!PREACTION(gm)) {
-    void* mem;
-    size_t nb;
-    if (bytes <= MAX_SMALL_REQUEST) {
-      bindex_t idx;
-      binmap_t smallbits;
-      nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes);
-      idx = small_index(nb);
-      smallbits = gm->smallmap >> idx;
-
-      if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */
-        mchunkptr b, p;
-        idx += ~smallbits & 1;       /* Uses next bin if idx empty */
-        b = smallbin_at(gm, idx);
-        p = b->fd;
-        assert(chunksize(p) == small_index2size(idx));
-        unlink_first_small_chunk(gm, b, p, idx);
-        set_inuse_and_pinuse(gm, p, small_index2size(idx));
-        mem = chunk2mem(p);
-        check_malloced_chunk(gm, mem, nb);
-        goto postaction;
-      }
-
-      else if (nb > gm->dvsize) {
-        if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
-          mchunkptr b, p, r;
-          size_t rsize;
-          bindex_t i;
-          binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx));
-          binmap_t leastbit = least_bit(leftbits);
-          compute_bit2idx(leastbit, i);
-          b = smallbin_at(gm, i);
-          p = b->fd;
-          assert(chunksize(p) == small_index2size(i));
-          unlink_first_small_chunk(gm, b, p, i);
-          rsize = small_index2size(i) - nb;
-          /* Fit here cannot be remainderless if 4byte sizes */
-          if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE)
-            set_inuse_and_pinuse(gm, p, small_index2size(i));
-          else {
-            set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
-            r = chunk_plus_offset(p, nb);
-            set_size_and_pinuse_of_free_chunk(r, rsize);
-            replace_dv(gm, r, rsize);
-          }
-          mem = chunk2mem(p);
-          check_malloced_chunk(gm, mem, nb);
-          goto postaction;
-        }
-
-        else if (gm->treemap != 0 && (mem = tmalloc_small(gm, nb)) != 0) {
-          check_malloced_chunk(gm, mem, nb);
-          goto postaction;
-        }
-      }
-    }
-    else if (bytes >= MAX_REQUEST)
-      nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */
-    else {
-      nb = pad_request(bytes);
-      if (gm->treemap != 0 && (mem = tmalloc_large(gm, nb)) != 0) {
-        check_malloced_chunk(gm, mem, nb);
-        goto postaction;
-      }
-    }
-
-    if (nb <= gm->dvsize) {
-      size_t rsize = gm->dvsize - nb;
-      mchunkptr p = gm->dv;
-      if (rsize >= MIN_CHUNK_SIZE) { /* split dv */
-        mchunkptr r = gm->dv = chunk_plus_offset(p, nb);
-        gm->dvsize = rsize;
-        set_size_and_pinuse_of_free_chunk(r, rsize);
-        set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
-      }
-      else { /* exhaust dv */
-        size_t dvs = gm->dvsize;
-        gm->dvsize = 0;
-        gm->dv = 0;
-        set_inuse_and_pinuse(gm, p, dvs);
-      }
-      mem = chunk2mem(p);
-      check_malloced_chunk(gm, mem, nb);
-      goto postaction;
-    }
-
-    else if (nb < gm->topsize) { /* Split top */
-      size_t rsize = gm->topsize -= nb;
-      mchunkptr p = gm->top;
-      mchunkptr r = gm->top = chunk_plus_offset(p, nb);
-      r->head = rsize | PINUSE_BIT;
-      set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
-      mem = chunk2mem(p);
-      check_top_chunk(gm, gm->top);
-      check_malloced_chunk(gm, mem, nb);
-      goto postaction;
-    }
-
-    mem = sys_alloc(gm, nb);
-
-  postaction:
-    POSTACTION(gm);
-    return mem;
-  }
-
-  return 0;
-}
-
-/* ---------------------------- free --------------------------- */
-
-void dlfree(void* mem) {
-  /*
-     Consolidate freed chunks with preceeding or succeeding bordering
-     free chunks, if they exist, and then place in a bin.  Intermixed
-     with special cases for top, dv, mmapped chunks, and usage errors.
-  */
-
-  if (mem != 0) {
-    mchunkptr p  = mem2chunk(mem);
-#if FOOTERS
-    mstate fm = get_mstate_for(p);
-    if (!ok_magic(fm)) {
-      USAGE_ERROR_ACTION(fm, p);
-      return;
-    }
-#else /* FOOTERS */
-#define fm gm
-#endif /* FOOTERS */
-    if (!PREACTION(fm)) {
-      check_inuse_chunk(fm, p);
-      if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) {
-        size_t psize = chunksize(p);
-        mchunkptr next = chunk_plus_offset(p, psize);
-        if (!pinuse(p)) {
-          size_t prevsize = p->prev_foot;
-          if (is_mmapped(p)) {
-            psize += prevsize + MMAP_FOOT_PAD;
-            if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
-              fm->footprint -= psize;
-            goto postaction;
-          }
-          else {
-            mchunkptr prev = chunk_minus_offset(p, prevsize);
-            psize += prevsize;
-            p = prev;
-            if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */
-              if (p != fm->dv) {
-                unlink_chunk(fm, p, prevsize);
-              }
-              else if ((next->head & INUSE_BITS) == INUSE_BITS) {
-                fm->dvsize = psize;
-                set_free_with_pinuse(p, psize, next);
-                goto postaction;
-              }
-            }
-            else
-              goto erroraction;
-          }
-        }
-
-        if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) {
-          if (!cinuse(next)) {  /* consolidate forward */
-            if (next == fm->top) {
-              size_t tsize = fm->topsize += psize;
-              fm->top = p;
-              p->head = tsize | PINUSE_BIT;
-              if (p == fm->dv) {
-                fm->dv = 0;
-                fm->dvsize = 0;
-              }
-              if (should_trim(fm, tsize))
-                sys_trim(fm, 0);
-              goto postaction;
-            }
-            else if (next == fm->dv) {
-              size_t dsize = fm->dvsize += psize;
-              fm->dv = p;
-              set_size_and_pinuse_of_free_chunk(p, dsize);
-              goto postaction;
-            }
-            else {
-              size_t nsize = chunksize(next);
-              psize += nsize;
-              unlink_chunk(fm, next, nsize);
-              set_size_and_pinuse_of_free_chunk(p, psize);
-              if (p == fm->dv) {
-                fm->dvsize = psize;
-                goto postaction;
-              }
-            }
-          }
-          else
-            set_free_with_pinuse(p, psize, next);
-
-          if (is_small(psize)) {
-            insert_small_chunk(fm, p, psize);
-            check_free_chunk(fm, p);
-          }
-          else {
-            tchunkptr tp = (tchunkptr)p;
-            insert_large_chunk(fm, tp, psize);
-            check_free_chunk(fm, p);
-            if (--fm->release_checks == 0)
-              release_unused_segments(fm);
-          }
-          goto postaction;
-        }
-      }
-    erroraction:
-      USAGE_ERROR_ACTION(fm, p);
-    postaction:
-      POSTACTION(fm);
-    }
-  }
-#if !FOOTERS
-#undef fm
-#endif /* FOOTERS */
-}
-
-void* dlcalloc(size_t n_elements, size_t elem_size) {
-  void* mem;
-  size_t req = 0;
-  if (n_elements != 0) {
-    req = n_elements * elem_size;
-    if (((n_elements | elem_size) & ~(size_t)0xffff) &&
-        (req / n_elements != elem_size))
-      req = MAX_SIZE_T; /* force downstream failure on overflow */
-  }
-  mem = dlmalloc(req);
-  if (mem != 0 && calloc_must_clear(mem2chunk(mem)))
-    memset(mem, 0, req);
-  return mem;
-}
-
-#endif /* !ONLY_MSPACES */
-
-/* ------------ Internal support for realloc, memalign, etc -------------- */
-
-/* Try to realloc; only in-place unless can_move true */
-static mchunkptr try_realloc_chunk(mstate m, mchunkptr p, size_t nb,
-                                   int can_move) {
-  mchunkptr newp = 0;
-  size_t oldsize = chunksize(p);
-  mchunkptr next = chunk_plus_offset(p, oldsize);
-  if (RTCHECK(ok_address(m, p) && ok_inuse(p) &&
-              ok_next(p, next) && ok_pinuse(next))) {
-    if (is_mmapped(p)) {
-      newp = mmap_resize(m, p, nb, can_move);
-    }
-    else if (oldsize >= nb) {             /* already big enough */
-      size_t rsize = oldsize - nb;
-      if (rsize >= MIN_CHUNK_SIZE) {      /* split off remainder */
-        mchunkptr r = chunk_plus_offset(p, nb);
-        set_inuse(m, p, nb);
-        set_inuse(m, r, rsize);
-        dispose_chunk(m, r, rsize);
-      }
-      newp = p;
-    }
-    else if (next == m->top) {  /* extend into top */
-      if (oldsize + m->topsize > nb) {
-        size_t newsize = oldsize + m->topsize;
-        size_t newtopsize = newsize - nb;
-        mchunkptr newtop = chunk_plus_offset(p, nb);
-        set_inuse(m, p, nb);
-        newtop->head = newtopsize |PINUSE_BIT;
-        m->top = newtop;
-        m->topsize = newtopsize;
-        newp = p;
-      }
-    }
-    else if (next == m->dv) { /* extend into dv */
-      size_t dvs = m->dvsize;
-      if (oldsize + dvs >= nb) {
-        size_t dsize = oldsize + dvs - nb;
-        if (dsize >= MIN_CHUNK_SIZE) {
-          mchunkptr r = chunk_plus_offset(p, nb);
-          mchunkptr n = chunk_plus_offset(r, dsize);
-          set_inuse(m, p, nb);
-          set_size_and_pinuse_of_free_chunk(r, dsize);
-          clear_pinuse(n);
-          m->dvsize = dsize;
-          m->dv = r;
-        }
-        else { /* exhaust dv */
-          size_t newsize = oldsize + dvs;
-          set_inuse(m, p, newsize);
-          m->dvsize = 0;
-          m->dv = 0;
-        }
-        newp = p;
-      }
-    }
-    else if (!cinuse(next)) { /* extend into next free chunk */
-      size_t nextsize = chunksize(next);
-      if (oldsize + nextsize >= nb) {
-        size_t rsize = oldsize + nextsize - nb;
-        unlink_chunk(m, next, nextsize);
-        if (rsize < MIN_CHUNK_SIZE) {
-          size_t newsize = oldsize + nextsize;
-          set_inuse(m, p, newsize);
-        }
-        else {
-          mchunkptr r = chunk_plus_offset(p, nb);
-          set_inuse(m, p, nb);
-          set_inuse(m, r, rsize);
-          dispose_chunk(m, r, rsize);
-        }
-        newp = p;
-      }
-    }
-  }
-  else {
-    USAGE_ERROR_ACTION(m, chunk2mem(p));
-  }
-  return newp;
-}
-
-static void* internal_memalign(mstate m, size_t alignment, size_t bytes) {
-  void* mem = 0;
-  if (alignment <  MIN_CHUNK_SIZE) /* must be at least a minimum chunk size */
-    alignment = MIN_CHUNK_SIZE;
-  if ((alignment & (alignment-SIZE_T_ONE)) != 0) {/* Ensure a power of 2 */
-    size_t a = MALLOC_ALIGNMENT << 1;
-    while (a < alignment) a <<= 1;
-    alignment = a;
-  }
-  if (bytes >= MAX_REQUEST - alignment) {
-    if (m != 0)  { /* Test isn't needed but avoids compiler warning */
-      MALLOC_FAILURE_ACTION;
-    }
-  }
-  else {
-    size_t nb = request2size(bytes);
-    size_t req = nb + alignment + MIN_CHUNK_SIZE - CHUNK_OVERHEAD;
-    mem = internal_malloc(m, req);
-    if (mem != 0) {
-      mchunkptr p = mem2chunk(mem);
-      if (PREACTION(m))
-        return 0;
-      if ((((size_t)(mem)) & (alignment - 1)) != 0) { /* misaligned */
-        /*
-          Find an aligned spot inside chunk.  Since we need to give
-          back leading space in a chunk of at least MIN_CHUNK_SIZE, if
-          the first calculation places us at a spot with less than
-          MIN_CHUNK_SIZE leader, we can move to the next aligned spot.
-          We've allocated enough total room so that this is always
-          possible.
-        */
-        char* br = (char*)mem2chunk((size_t)(((size_t)((char*)mem + alignment -
-                                                       SIZE_T_ONE)) &
-                                             -alignment));
-        char* pos = ((size_t)(br - (char*)(p)) >= MIN_CHUNK_SIZE)?
-          br : br+alignment;
-        mchunkptr newp = (mchunkptr)pos;
-        size_t leadsize = pos - (char*)(p);
-        size_t newsize = chunksize(p) - leadsize;
-
-        if (is_mmapped(p)) { /* For mmapped chunks, just adjust offset */
-          newp->prev_foot = p->prev_foot + leadsize;
-          newp->head = newsize;
-        }
-        else { /* Otherwise, give back leader, use the rest */
-          set_inuse(m, newp, newsize);
-          set_inuse(m, p, leadsize);
-          dispose_chunk(m, p, leadsize);
-        }
-        p = newp;
-      }
-
-      /* Give back spare room at the end */
-      if (!is_mmapped(p)) {
-        size_t size = chunksize(p);
-        if (size > nb + MIN_CHUNK_SIZE) {
-          size_t remainder_size = size - nb;
-          mchunkptr remainder = chunk_plus_offset(p, nb);
-          set_inuse(m, p, nb);
-          set_inuse(m, remainder, remainder_size);
-          dispose_chunk(m, remainder, remainder_size);
-        }
-      }
-
-      mem = chunk2mem(p);
-      assert (chunksize(p) >= nb);
-      assert(((size_t)mem & (alignment - 1)) == 0);
-      check_inuse_chunk(m, p);
-      POSTACTION(m);
-    }
-  }
-  return mem;
-}
-
-/*
-  Common support for independent_X routines, handling
-    all of the combinations that can result.
-  The opts arg has:
-    bit 0 set if all elements are same size (using sizes[0])
-    bit 1 set if elements should be zeroed
-*/
-static void** ialloc(mstate m,
-                     size_t n_elements,
-                     size_t* sizes,
-                     int opts,
-                     void* chunks[]) {
-
-  size_t    element_size;   /* chunksize of each element, if all same */
-  size_t    contents_size;  /* total size of elements */
-  size_t    array_size;     /* request size of pointer array */
-  void*     mem;            /* malloced aggregate space */
-  mchunkptr p;              /* corresponding chunk */
-  size_t    remainder_size; /* remaining bytes while splitting */
-  void**    marray;         /* either "chunks" or malloced ptr array */
-  mchunkptr array_chunk;    /* chunk for malloced ptr array */
-  flag_t    was_enabled;    /* to disable mmap */
-  size_t    size;
-  size_t    i;
-
-  ensure_initialization();
-  /* compute array length, if needed */
-  if (chunks != 0) {
-    if (n_elements == 0)
-      return chunks; /* nothing to do */
-    marray = chunks;
-    array_size = 0;
-  }
-  else {
-    /* if empty req, must still return chunk representing empty array */
-    if (n_elements == 0)
-      return (void**)internal_malloc(m, 0);
-    marray = 0;
-    array_size = request2size(n_elements * (sizeof(void*)));
-  }
-
-  /* compute total element size */
-  if (opts & 0x1) { /* all-same-size */
-    element_size = request2size(*sizes);
-    contents_size = n_elements * element_size;
-  }
-  else { /* add up all the sizes */
-    element_size = 0;
-    contents_size = 0;
-    for (i = 0; i != n_elements; ++i)
-      contents_size += request2size(sizes[i]);
-  }
-
-  size = contents_size + array_size;
-
-  /*
-     Allocate the aggregate chunk.  First disable direct-mmapping so
-     malloc won't use it, since we would not be able to later
-     free/realloc space internal to a segregated mmap region.
-  */
-  was_enabled = use_mmap(m);
-  disable_mmap(m);
-  mem = internal_malloc(m, size - CHUNK_OVERHEAD);
-  if (was_enabled)
-    enable_mmap(m);
-  if (mem == 0)
-    return 0;
-
-  if (PREACTION(m)) return 0;
-  p = mem2chunk(mem);
-  remainder_size = chunksize(p);
-
-  assert(!is_mmapped(p));
-
-  if (opts & 0x2) {       /* optionally clear the elements */
-    memset((size_t*)mem, 0, remainder_size - SIZE_T_SIZE - array_size);
-  }
-
-  /* If not provided, allocate the pointer array as final part of chunk */
-  if (marray == 0) {
-    size_t  array_chunk_size;
-    array_chunk = chunk_plus_offset(p, contents_size);
-    array_chunk_size = remainder_size - contents_size;
-    marray = (void**) (chunk2mem(array_chunk));
-    set_size_and_pinuse_of_inuse_chunk(m, array_chunk, array_chunk_size);
-    remainder_size = contents_size;
-  }
-
-  /* split out elements */
-  for (i = 0; ; ++i) {
-    marray[i] = chunk2mem(p);
-    if (i != n_elements-1) {
-      if (element_size != 0)
-        size = element_size;
-      else
-        size = request2size(sizes[i]);
-      remainder_size -= size;
-      set_size_and_pinuse_of_inuse_chunk(m, p, size);
-      p = chunk_plus_offset(p, size);
-    }
-    else { /* the final element absorbs any overallocation slop */
-      set_size_and_pinuse_of_inuse_chunk(m, p, remainder_size);
-      break;
-    }
-  }
-
-#if DEBUG
-  if (marray != chunks) {
-    /* final element must have exactly exhausted chunk */
-    if (element_size != 0) {
-      assert(remainder_size == element_size);
-    }
-    else {
-      assert(remainder_size == request2size(sizes[i]));
-    }
-    check_inuse_chunk(m, mem2chunk(marray));
-  }
-  for (i = 0; i != n_elements; ++i)
-    check_inuse_chunk(m, mem2chunk(marray[i]));
-
-#endif /* DEBUG */
-
-  POSTACTION(m);
-  return marray;
-}
-
-/* Try to free all pointers in the given array.
-   Note: this could be made faster, by delaying consolidation,
-   at the price of disabling some user integrity checks, We
-   still optimize some consolidations by combining adjacent
-   chunks before freeing, which will occur often if allocated
-   with ialloc or the array is sorted.
-*/
-static size_t internal_bulk_free(mstate m, void* array[], size_t nelem) {
-  size_t unfreed = 0;
-  if (!PREACTION(m)) {
-    void** a;
-    void** fence = &(array[nelem]);
-    for (a = array; a != fence; ++a) {
-      void* mem = *a;
-      if (mem != 0) {
-        mchunkptr p = mem2chunk(mem);
-        size_t psize = chunksize(p);
-#if FOOTERS
-        if (get_mstate_for(p) != m) {
-          ++unfreed;
-          continue;
-        }
-#endif
-        check_inuse_chunk(m, p);
-        *a = 0;
-        if (RTCHECK(ok_address(m, p) && ok_inuse(p))) {
-          void ** b = a + 1; /* try to merge with next chunk */
-          mchunkptr next = next_chunk(p);
-          if (b != fence && *b == chunk2mem(next)) {
-            size_t newsize = chunksize(next) + psize;
-            set_inuse(m, p, newsize);
-            *b = chunk2mem(p);
-          }
-          else
-            dispose_chunk(m, p, psize);
-        }
-        else {
-          CORRUPTION_ERROR_ACTION(m);
-          break;
-        }
-      }
-    }
-    if (should_trim(m, m->topsize))
-      sys_trim(m, 0);
-    POSTACTION(m);
-  }
-  return unfreed;
-}
-
-/* Traversal */
-#if MALLOC_INSPECT_ALL
-static void internal_inspect_all(mstate m,
-                                 void(*handler)(void *start,
-                                                void *end,
-                                                size_t used_bytes,
-                                                void* callback_arg),
-                                 void* arg) {
-  if (is_initialized(m)) {
-    mchunkptr top = m->top;
-    msegmentptr s;
-    for (s = &m->seg; s != 0; s = s->next) {
-      mchunkptr q = align_as_chunk(s->base);
-      while (segment_holds(s, q) && q->head != FENCEPOST_HEAD) {
-        mchunkptr next = next_chunk(q);
-        size_t sz = chunksize(q);
-        size_t used;
-        void* start;
-        if (is_inuse(q)) {
-          used = sz - CHUNK_OVERHEAD; /* must not be mmapped */
-          start = chunk2mem(q);
-        }
-        else {
-          used = 0;
-          if (is_small(sz)) {     /* offset by possible bookkeeping */
-            start = (void*)((char*)q + sizeof(struct malloc_chunk));
-          }
-          else {
-            start = (void*)((char*)q + sizeof(struct malloc_tree_chunk));
-          }
-        }
-        if (start < (void*)next)  /* skip if all space is bookkeeping */
-          handler(start, next, used, arg);
-        if (q == top)
-          break;
-        q = next;
-      }
-    }
-  }
-}
-#endif /* MALLOC_INSPECT_ALL */
-
-/* ------------------ Exported realloc, memalign, etc -------------------- */
-
-#if !ONLY_MSPACES
-
-void* dlrealloc(void* oldmem, size_t bytes) {
-  void* mem = 0;
-  if (oldmem == 0) {
-    mem = dlmalloc(bytes);
-  }
-  else if (bytes >= MAX_REQUEST) {
-    MALLOC_FAILURE_ACTION;
-  }
-#ifdef REALLOC_ZERO_BYTES_FREES
-  else if (bytes == 0) {
-    dlfree(oldmem);
-  }
-#endif /* REALLOC_ZERO_BYTES_FREES */
-  else {
-    size_t nb = request2size(bytes);
-    mchunkptr oldp = mem2chunk(oldmem);
-#if ! FOOTERS
-    mstate m = gm;
-#else /* FOOTERS */
-    mstate m = get_mstate_for(oldp);
-    if (!ok_magic(m)) {
-      USAGE_ERROR_ACTION(m, oldmem);
-      return 0;
-    }
-#endif /* FOOTERS */
-    if (!PREACTION(m)) {
-      mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1);
-      POSTACTION(m);
-      if (newp != 0) {
-        check_inuse_chunk(m, newp);
-        mem = chunk2mem(newp);
-      }
-      else {
-        mem = internal_malloc(m, bytes);
-        if (mem != 0) {
-          size_t oc = chunksize(oldp) - overhead_for(oldp);
-          memcpy(mem, oldmem, (oc < bytes)? oc : bytes);
-          internal_free(m, oldmem);
-        }
-      }
-    }
-  }
-  return mem;
-}
-
-void* dlrealloc_in_place(void* oldmem, size_t bytes) {
-  void* mem = 0;
-  if (oldmem != 0) {
-    if (bytes >= MAX_REQUEST) {
-      MALLOC_FAILURE_ACTION;
-    }
-    else {
-      size_t nb = request2size(bytes);
-      mchunkptr oldp = mem2chunk(oldmem);
-#if ! FOOTERS
-      mstate m = gm;
-#else /* FOOTERS */
-      mstate m = get_mstate_for(oldp);
-      if (!ok_magic(m)) {
-        USAGE_ERROR_ACTION(m, oldmem);
-        return 0;
-      }
-#endif /* FOOTERS */
-      if (!PREACTION(m)) {
-        mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
-        POSTACTION(m);
-        if (newp == oldp) {
-          check_inuse_chunk(m, newp);
-          mem = oldmem;
-        }
-      }
-    }
-  }
-  return mem;
-}
-
-void* dlmemalign(size_t alignment, size_t bytes) {
-  if (alignment <= MALLOC_ALIGNMENT) {
-    return dlmalloc(bytes);
-  }
-  return internal_memalign(gm, alignment, bytes);
-}
-
-int dlposix_memalign(void** pp, size_t alignment, size_t bytes) {
-  void* mem = 0;
-  if (alignment == MALLOC_ALIGNMENT)
-    mem = dlmalloc(bytes);
-  else {
-    size_t d = alignment / sizeof(void*);
-    size_t r = alignment % sizeof(void*);
-    if (r != 0 || d == 0 || (d & (d-SIZE_T_ONE)) != 0)
-      return EINVAL;
-    else if (bytes <= MAX_REQUEST - alignment) {
-      if (alignment <  MIN_CHUNK_SIZE)
-        alignment = MIN_CHUNK_SIZE;
-      mem = internal_memalign(gm, alignment, bytes);
-    }
-  }
-  if (mem == 0)
-    return ENOMEM;
-  else {
-    *pp = mem;
-    return 0;
-  }
-}
-
-void* dlvalloc(size_t bytes) {
-  size_t pagesz;
-  ensure_initialization();
-  pagesz = mparams.page_size;
-  return dlmemalign(pagesz, bytes);
-}
-
-void* dlpvalloc(size_t bytes) {
-  size_t pagesz;
-  ensure_initialization();
-  pagesz = mparams.page_size;
-  return dlmemalign(pagesz, (bytes + pagesz - SIZE_T_ONE) & ~(pagesz - SIZE_T_ONE));
-}
-
-void** dlindependent_calloc(size_t n_elements, size_t elem_size,
-                            void* chunks[]) {
-  size_t sz = elem_size; /* serves as 1-element array */
-  return ialloc(gm, n_elements, &sz, 3, chunks);
-}
-
-void** dlindependent_comalloc(size_t n_elements, size_t sizes[],
-                              void* chunks[]) {
-  return ialloc(gm, n_elements, sizes, 0, chunks);
-}
-
-size_t dlbulk_free(void* array[], size_t nelem) {
-  return internal_bulk_free(gm, array, nelem);
-}
-
-#if MALLOC_INSPECT_ALL
-void dlmalloc_inspect_all(void(*handler)(void *start,
-                                         void *end,
-                                         size_t used_bytes,
-                                         void* callback_arg),
-                          void* arg) {
-  ensure_initialization();
-  if (!PREACTION(gm)) {
-    internal_inspect_all(gm, handler, arg);
-    POSTACTION(gm);
-  }
-}
-#endif /* MALLOC_INSPECT_ALL */
-
-int dlmalloc_trim(size_t pad) {
-  int result = 0;
-  ensure_initialization();
-  if (!PREACTION(gm)) {
-    result = sys_trim(gm, pad);
-    POSTACTION(gm);
-  }
-  return result;
-}
-
-size_t dlmalloc_footprint(void) {
-  return gm->footprint;
-}
-
-size_t dlmalloc_max_footprint(void) {
-  return gm->max_footprint;
-}
-
-size_t dlmalloc_footprint_limit(void) {
-  size_t maf = gm->footprint_limit;
-  return maf == 0 ? MAX_SIZE_T : maf;
-}
-
-size_t dlmalloc_set_footprint_limit(size_t bytes) {
-  ensure_initialization();
-  size_t result;  /* invert sense of 0 */
-  if (bytes == 0)
-    result = granularity_align(1); /* Use minimal size */
-  if (bytes == MAX_SIZE_T)
-    result = 0;                    /* disable */
-  else
-    result = granularity_align(bytes);
-  return gm->footprint_limit = result;
-}
-
-#if !NO_MALLINFO
-struct mallinfo dlmallinfo(void) {
-  return internal_mallinfo(gm);
-}
-#endif /* NO_MALLINFO */
-
-#if !NO_MALLOC_STATS
-void dlmalloc_stats() {
-  internal_malloc_stats(gm);
-}
-#endif /* NO_MALLOC_STATS */
-
-int dlmallopt(int param_number, int value) {
-  return change_mparam(param_number, value);
-}
-
-size_t dlmalloc_usable_size(void* mem) {
-  if (mem != 0) {
-    mchunkptr p = mem2chunk(mem);
-    if (is_inuse(p))
-      return chunksize(p) - overhead_for(p);
-  }
-  return 0;
-}
-
-#endif /* !ONLY_MSPACES */
-
-/* ----------------------------- user mspaces ---------------------------- */
-
-#if MSPACES
-
-static mstate init_user_mstate(char* tbase, size_t tsize) {
-  size_t msize = pad_request(sizeof(struct malloc_state));
-  mchunkptr mn;
-  mchunkptr msp = align_as_chunk(tbase);
-  mstate m = (mstate)(chunk2mem(msp));
-  memset(m, 0, msize);
-  (void)INITIAL_LOCK(&m->mutex);
-  msp->head = (msize|INUSE_BITS);
-  m->seg.base = m->least_addr = tbase;
-  m->seg.size = m->footprint = m->max_footprint = tsize;
-  m->magic = mparams.magic;
-  m->release_checks = MAX_RELEASE_CHECK_RATE;
-  m->mflags = mparams.default_mflags;
-  m->extp = 0;
-  m->exts = 0;
-  disable_contiguous(m);
-  init_bins(m);
-  mn = next_chunk(mem2chunk(m));
-  init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) - TOP_FOOT_SIZE);
-  check_top_chunk(m, m->top);
-  return m;
-}
-
-mspace create_mspace(size_t capacity, int locked) {
-  mstate m = 0;
-  size_t msize;
-  ensure_initialization();
-  msize = pad_request(sizeof(struct malloc_state));
-  if (capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) {
-    size_t rs = ((capacity == 0)? mparams.granularity :
-                 (capacity + TOP_FOOT_SIZE + msize));
-    size_t tsize = granularity_align(rs);
-    char* tbase = (char*)(CALL_MMAP(tsize));
-    if (tbase != CMFAIL) {
-      m = init_user_mstate(tbase, tsize);
-      m->seg.sflags = USE_MMAP_BIT;
-      set_lock(m, locked);
-    }
-  }
-  return (mspace)m;
-}
-
-mspace create_mspace_with_base(void* base, size_t capacity, int locked) {
-  mstate m = 0;
-  size_t msize;
-  ensure_initialization();
-  msize = pad_request(sizeof(struct malloc_state));
-  if (capacity > msize + TOP_FOOT_SIZE &&
-      capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) {
-    m = init_user_mstate((char*)base, capacity);
-    m->seg.sflags = EXTERN_BIT;
-    set_lock(m, locked);
-  }
-  return (mspace)m;
-}
-
-int mspace_track_large_chunks(mspace msp, int enable) {
-  int ret = 0;
-  mstate ms = (mstate)msp;
-  if (!PREACTION(ms)) {
-    if (!use_mmap(ms)) {
-      ret = 1;
-    }
-    if (!enable) {
-      enable_mmap(ms);
-    } else {
-      disable_mmap(ms);
-    }
-    POSTACTION(ms);
-  }
-  return ret;
-}
-
-size_t destroy_mspace(mspace msp) {
-  size_t freed = 0;
-  mstate ms = (mstate)msp;
-  if (ok_magic(ms)) {
-    msegmentptr sp = &ms->seg;
-    (void)DESTROY_LOCK(&ms->mutex); /* destroy before unmapped */
-    while (sp != 0) {
-      char* base = sp->base;
-      size_t size = sp->size;
-      flag_t flag = sp->sflags;
-      (void)base; /* placate people compiling -Wunused-variable */
-      sp = sp->next;
-      if ((flag & USE_MMAP_BIT) && !(flag & EXTERN_BIT) &&
-          CALL_MUNMAP(base, size) == 0)
-        freed += size;
-    }
-  }
-  else {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-  return freed;
-}
-
-/*
-  mspace versions of routines are near-clones of the global
-  versions. This is not so nice but better than the alternatives.
-*/
-
-void* mspace_malloc(mspace msp, size_t bytes) {
-  mstate ms = (mstate)msp;
-  if (!ok_magic(ms)) {
-    USAGE_ERROR_ACTION(ms,ms);
-    return 0;
-  }
-  if (!PREACTION(ms)) {
-    void* mem;
-    size_t nb;
-    if (bytes <= MAX_SMALL_REQUEST) {
-      bindex_t idx;
-      binmap_t smallbits;
-      nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes);
-      idx = small_index(nb);
-      smallbits = ms->smallmap >> idx;
-
-      if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */
-        mchunkptr b, p;
-        idx += ~smallbits & 1;       /* Uses next bin if idx empty */
-        b = smallbin_at(ms, idx);
-        p = b->fd;
-        assert(chunksize(p) == small_index2size(idx));
-        unlink_first_small_chunk(ms, b, p, idx);
-        set_inuse_and_pinuse(ms, p, small_index2size(idx));
-        mem = chunk2mem(p);
-        check_malloced_chunk(ms, mem, nb);
-        goto postaction;
-      }
-
-      else if (nb > ms->dvsize) {
-        if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
-          mchunkptr b, p, r;
-          size_t rsize;
-          bindex_t i;
-          binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx));
-          binmap_t leastbit = least_bit(leftbits);
-          compute_bit2idx(leastbit, i);
-          b = smallbin_at(ms, i);
-          p = b->fd;
-          assert(chunksize(p) == small_index2size(i));
-          unlink_first_small_chunk(ms, b, p, i);
-          rsize = small_index2size(i) - nb;
-          /* Fit here cannot be remainderless if 4byte sizes */
-          if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE)
-            set_inuse_and_pinuse(ms, p, small_index2size(i));
-          else {
-            set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
-            r = chunk_plus_offset(p, nb);
-            set_size_and_pinuse_of_free_chunk(r, rsize);
-            replace_dv(ms, r, rsize);
-          }
-          mem = chunk2mem(p);
-          check_malloced_chunk(ms, mem, nb);
-          goto postaction;
-        }
-
-        else if (ms->treemap != 0 && (mem = tmalloc_small(ms, nb)) != 0) {
-          check_malloced_chunk(ms, mem, nb);
-          goto postaction;
-        }
-      }
-    }
-    else if (bytes >= MAX_REQUEST)
-      nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */
-    else {
-      nb = pad_request(bytes);
-      if (ms->treemap != 0 && (mem = tmalloc_large(ms, nb)) != 0) {
-        check_malloced_chunk(ms, mem, nb);
-        goto postaction;
-      }
-    }
-
-    if (nb <= ms->dvsize) {
-      size_t rsize = ms->dvsize - nb;
-      mchunkptr p = ms->dv;
-      if (rsize >= MIN_CHUNK_SIZE) { /* split dv */
-        mchunkptr r = ms->dv = chunk_plus_offset(p, nb);
-        ms->dvsize = rsize;
-        set_size_and_pinuse_of_free_chunk(r, rsize);
-        set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
-      }
-      else { /* exhaust dv */
-        size_t dvs = ms->dvsize;
-        ms->dvsize = 0;
-        ms->dv = 0;
-        set_inuse_and_pinuse(ms, p, dvs);
-      }
-      mem = chunk2mem(p);
-      check_malloced_chunk(ms, mem, nb);
-      goto postaction;
-    }
-
-    else if (nb < ms->topsize) { /* Split top */
-      size_t rsize = ms->topsize -= nb;
-      mchunkptr p = ms->top;
-      mchunkptr r = ms->top = chunk_plus_offset(p, nb);
-      r->head = rsize | PINUSE_BIT;
-      set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
-      mem = chunk2mem(p);
-      check_top_chunk(ms, ms->top);
-      check_malloced_chunk(ms, mem, nb);
-      goto postaction;
-    }
-
-    mem = sys_alloc(ms, nb);
-
-  postaction:
-    POSTACTION(ms);
-    return mem;
-  }
-
-  return 0;
-}
-
-void mspace_free(mspace msp, void* mem) {
-  if (mem != 0) {
-    mchunkptr p  = mem2chunk(mem);
-#if FOOTERS
-    mstate fm = get_mstate_for(p);
-    (void)msp; /* placate people compiling -Wunused */
-#else /* FOOTERS */
-    mstate fm = (mstate)msp;
-#endif /* FOOTERS */
-    if (!ok_magic(fm)) {
-      USAGE_ERROR_ACTION(fm, p);
-      return;
-    }
-    if (!PREACTION(fm)) {
-      check_inuse_chunk(fm, p);
-      if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) {
-        size_t psize = chunksize(p);
-        mchunkptr next = chunk_plus_offset(p, psize);
-        if (!pinuse(p)) {
-          size_t prevsize = p->prev_foot;
-          if (is_mmapped(p)) {
-            psize += prevsize + MMAP_FOOT_PAD;
-            if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
-              fm->footprint -= psize;
-            goto postaction;
-          }
-          else {
-            mchunkptr prev = chunk_minus_offset(p, prevsize);
-            psize += prevsize;
-            p = prev;
-            if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */
-              if (p != fm->dv) {
-                unlink_chunk(fm, p, prevsize);
-              }
-              else if ((next->head & INUSE_BITS) == INUSE_BITS) {
-                fm->dvsize = psize;
-                set_free_with_pinuse(p, psize, next);
-                goto postaction;
-              }
-            }
-            else
-              goto erroraction;
-          }
-        }
-
-        if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) {
-          if (!cinuse(next)) {  /* consolidate forward */
-            if (next == fm->top) {
-              size_t tsize = fm->topsize += psize;
-              fm->top = p;
-              p->head = tsize | PINUSE_BIT;
-              if (p == fm->dv) {
-                fm->dv = 0;
-                fm->dvsize = 0;
-              }
-              if (should_trim(fm, tsize))
-                sys_trim(fm, 0);
-              goto postaction;
-            }
-            else if (next == fm->dv) {
-              size_t dsize = fm->dvsize += psize;
-              fm->dv = p;
-              set_size_and_pinuse_of_free_chunk(p, dsize);
-              goto postaction;
-            }
-            else {
-              size_t nsize = chunksize(next);
-              psize += nsize;
-              unlink_chunk(fm, next, nsize);
-              set_size_and_pinuse_of_free_chunk(p, psize);
-              if (p == fm->dv) {
-                fm->dvsize = psize;
-                goto postaction;
-              }
-            }
-          }
-          else
-            set_free_with_pinuse(p, psize, next);
-
-          if (is_small(psize)) {
-            insert_small_chunk(fm, p, psize);
-            check_free_chunk(fm, p);
-          }
-          else {
-            tchunkptr tp = (tchunkptr)p;
-            insert_large_chunk(fm, tp, psize);
-            check_free_chunk(fm, p);
-            if (--fm->release_checks == 0)
-              release_unused_segments(fm);
-          }
-          goto postaction;
-        }
-      }
-    erroraction:
-      USAGE_ERROR_ACTION(fm, p);
-    postaction:
-      POSTACTION(fm);
-    }
-  }
-}
-
-void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size) {
-  void* mem;
-  size_t req = 0;
-  mstate ms = (mstate)msp;
-  if (!ok_magic(ms)) {
-    USAGE_ERROR_ACTION(ms,ms);
-    return 0;
-  }
-  if (n_elements != 0) {
-    req = n_elements * elem_size;
-    if (((n_elements | elem_size) & ~(size_t)0xffff) &&
-        (req / n_elements != elem_size))
-      req = MAX_SIZE_T; /* force downstream failure on overflow */
-  }
-  mem = internal_malloc(ms, req);
-  if (mem != 0 && calloc_must_clear(mem2chunk(mem)))
-    memset(mem, 0, req);
-  return mem;
-}
-
-void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) {
-  void* mem = 0;
-  if (oldmem == 0) {
-    mem = mspace_malloc(msp, bytes);
-  }
-  else if (bytes >= MAX_REQUEST) {
-    MALLOC_FAILURE_ACTION;
-  }
-#ifdef REALLOC_ZERO_BYTES_FREES
-  else if (bytes == 0) {
-    mspace_free(msp, oldmem);
-  }
-#endif /* REALLOC_ZERO_BYTES_FREES */
-  else {
-    size_t nb = request2size(bytes);
-    mchunkptr oldp = mem2chunk(oldmem);
-#if ! FOOTERS
-    mstate m = (mstate)msp;
-#else /* FOOTERS */
-    mstate m = get_mstate_for(oldp);
-    if (!ok_magic(m)) {
-      USAGE_ERROR_ACTION(m, oldmem);
-      return 0;
-    }
-#endif /* FOOTERS */
-    if (!PREACTION(m)) {
-      mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1);
-      POSTACTION(m);
-      if (newp != 0) {
-        check_inuse_chunk(m, newp);
-        mem = chunk2mem(newp);
-      }
-      else {
-        mem = mspace_malloc(m, bytes);
-        if (mem != 0) {
-          size_t oc = chunksize(oldp) - overhead_for(oldp);
-          memcpy(mem, oldmem, (oc < bytes)? oc : bytes);
-          mspace_free(m, oldmem);
-        }
-      }
-    }
-  }
-  return mem;
-}
-
-void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) {
-  void* mem = 0;
-  if (oldmem != 0) {
-    if (bytes >= MAX_REQUEST) {
-      MALLOC_FAILURE_ACTION;
-    }
-    else {
-      size_t nb = request2size(bytes);
-      mchunkptr oldp = mem2chunk(oldmem);
-#if ! FOOTERS
-      mstate m = (mstate)msp;
-#else /* FOOTERS */
-      mstate m = get_mstate_for(oldp);
-      (void)msp; /* placate people compiling -Wunused */
-      if (!ok_magic(m)) {
-        USAGE_ERROR_ACTION(m, oldmem);
-        return 0;
-      }
-#endif /* FOOTERS */
-      if (!PREACTION(m)) {
-        mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
-        POSTACTION(m);
-        if (newp == oldp) {
-          check_inuse_chunk(m, newp);
-          mem = oldmem;
-        }
-      }
-    }
-  }
-  return mem;
-}
-
-void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) {
-  mstate ms = (mstate)msp;
-  if (!ok_magic(ms)) {
-    USAGE_ERROR_ACTION(ms,ms);
-    return 0;
-  }
-  if (alignment <= MALLOC_ALIGNMENT)
-    return mspace_malloc(msp, bytes);
-  return internal_memalign(ms, alignment, bytes);
-}
-
-void** mspace_independent_calloc(mspace msp, size_t n_elements,
-                                 size_t elem_size, void* chunks[]) {
-  size_t sz = elem_size; /* serves as 1-element array */
-  mstate ms = (mstate)msp;
-  if (!ok_magic(ms)) {
-    USAGE_ERROR_ACTION(ms,ms);
-    return 0;
-  }
-  return ialloc(ms, n_elements, &sz, 3, chunks);
-}
-
-void** mspace_independent_comalloc(mspace msp, size_t n_elements,
-                                   size_t sizes[], void* chunks[]) {
-  mstate ms = (mstate)msp;
-  if (!ok_magic(ms)) {
-    USAGE_ERROR_ACTION(ms,ms);
-    return 0;
-  }
-  return ialloc(ms, n_elements, sizes, 0, chunks);
-}
-
-size_t mspace_bulk_free(mspace msp, void* array[], size_t nelem) {
-  return internal_bulk_free((mstate)msp, array, nelem);
-}
-
-#if MALLOC_INSPECT_ALL
-void mspace_inspect_all(mspace msp,
-                        void(*handler)(void *start,
-                                       void *end,
-                                       size_t used_bytes,
-                                       void* callback_arg),
-                        void* arg) {
-  mstate ms = (mstate)msp;
-  if (ok_magic(ms)) {
-    if (!PREACTION(ms)) {
-      internal_inspect_all(ms, handler, arg);
-      POSTACTION(ms);
-    }
-  }
-  else {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-}
-#endif /* MALLOC_INSPECT_ALL */
-
-int mspace_trim(mspace msp, size_t pad) {
-  int result = 0;
-  mstate ms = (mstate)msp;
-  if (ok_magic(ms)) {
-    if (!PREACTION(ms)) {
-      result = sys_trim(ms, pad);
-      POSTACTION(ms);
-    }
-  }
-  else {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-  return result;
-}
-
-#if !NO_MALLOC_STATS
-void mspace_malloc_stats(mspace msp) {
-  mstate ms = (mstate)msp;
-  if (ok_magic(ms)) {
-    internal_malloc_stats(ms);
-  }
-  else {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-}
-#endif /* NO_MALLOC_STATS */
-
-size_t mspace_footprint(mspace msp) {
-  size_t result = 0;
-  mstate ms = (mstate)msp;
-  if (ok_magic(ms)) {
-    result = ms->footprint;
-  }
-  else {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-  return result;
-}
-
-size_t mspace_max_footprint(mspace msp) {
-  size_t result = 0;
-  mstate ms = (mstate)msp;
-  if (ok_magic(ms)) {
-    result = ms->max_footprint;
-  }
-  else {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-  return result;
-}
-
-size_t mspace_footprint_limit(mspace msp) {
-  size_t result = 0;
-  mstate ms = (mstate)msp;
-  if (ok_magic(ms)) {
-    size_t maf = ms->footprint_limit;
-    result = (maf == 0) ? MAX_SIZE_T : maf;
-  }
-  else {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-  return result;
-}
-
-size_t mspace_set_footprint_limit(mspace msp, size_t bytes) {
-  size_t result = 0;
-  mstate ms = (mstate)msp;
-  if (ok_magic(ms)) {
-    if (bytes == 0)
-      result = granularity_align(1); /* Use minimal size */
-    if (bytes == MAX_SIZE_T)
-      result = 0;                    /* disable */
-    else
-      result = granularity_align(bytes);
-    ms->footprint_limit = result;
-  }
-  else {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-  return result;
-}
-
-#if !NO_MALLINFO
-struct mallinfo mspace_mallinfo(mspace msp) {
-  mstate ms = (mstate)msp;
-  if (!ok_magic(ms)) {
-    USAGE_ERROR_ACTION(ms,ms);
-  }
-  return internal_mallinfo(ms);
-}
-#endif /* NO_MALLINFO */
-
-size_t mspace_usable_size(const void* mem) {
-  if (mem != 0) {
-    mchunkptr p = mem2chunk(mem);
-    if (is_inuse(p))
-      return chunksize(p) - overhead_for(p);
-  }
-  return 0;
-}
-
-int mspace_mallopt(int param_number, int value) {
-  return change_mparam(param_number, value);
-}
-
-#endif /* MSPACES */
-
-
-/* -------------------- Alternative MORECORE functions ------------------- */
-
-/*
-  Guidelines for creating a custom version of MORECORE:
-
-  * For best performance, MORECORE should allocate in multiples of pagesize.
-  * MORECORE may allocate more memory than requested. (Or even less,
-      but this will usually result in a malloc failure.)
-  * MORECORE must not allocate memory when given argument zero, but
-      instead return one past the end address of memory from previous
-      nonzero call.
-  * For best performance, consecutive calls to MORECORE with positive
-      arguments should return increasing addresses, indicating that
-      space has been contiguously extended.
-  * Even though consecutive calls to MORECORE need not return contiguous
-      addresses, it must be OK for malloc'ed chunks to span multiple
-      regions in those cases where they do happen to be contiguous.
-  * MORECORE need not handle negative arguments -- it may instead
-      just return MFAIL when given negative arguments.
-      Negative arguments are always multiples of pagesize. MORECORE
-      must not misinterpret negative args as large positive unsigned
-      args. You can suppress all such calls from even occurring by defining
-      MORECORE_CANNOT_TRIM,
-
-  As an example alternative MORECORE, here is a custom allocator
-  kindly contributed for pre-OSX macOS.  It uses virtually but not
-  necessarily physically contiguous non-paged memory (locked in,
-  present and won't get swapped out).  You can use it by uncommenting
-  this section, adding some #includes, and setting up the appropriate
-  defines above:
-
-      #define MORECORE osMoreCore
-
-  There is also a shutdown routine that should somehow be called for
-  cleanup upon program exit.
-
-  #define MAX_POOL_ENTRIES 100
-  #define MINIMUM_MORECORE_SIZE  (64 * 1024U)
-  static int next_os_pool;
-  void *our_os_pools[MAX_POOL_ENTRIES];
-
-  void *osMoreCore(int size)
-  {
-    void *ptr = 0;
-    static void *sbrk_top = 0;
-
-    if (size > 0)
-    {
-      if (size < MINIMUM_MORECORE_SIZE)
-         size = MINIMUM_MORECORE_SIZE;
-      if (CurrentExecutionLevel() == kTaskLevel)
-         ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
-      if (ptr == 0)
-      {
-        return (void *) MFAIL;
-      }
-      // save ptrs so they can be freed during cleanup
-      our_os_pools[next_os_pool] = ptr;
-      next_os_pool++;
-      ptr = (void *) ((((size_t) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
-      sbrk_top = (char *) ptr + size;
-      return ptr;
-    }
-    else if (size < 0)
-    {
-      // we don't currently support shrink behavior
-      return (void *) MFAIL;
-    }
-    else
-    {
-      return sbrk_top;
-    }
-  }
-
-  // cleanup any allocated memory pools
-  // called as last thing before shutting down driver
-
-  void osCleanupMem(void)
-  {
-    void **ptr;
-
-    for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
-      if (*ptr)
-      {
-         PoolDeallocate(*ptr);
-         *ptr = 0;
-      }
-  }
-
-*/
-
-
-/* -----------------------------------------------------------------------
-History:
-    v2.8.6 Wed Aug 29 06:57:58 2012  Doug Lea
-      * fix bad comparison in dlposix_memalign
-      * don't reuse adjusted asize in sys_alloc
-      * add LOCK_AT_FORK -- thanks to Kirill Artamonov for the suggestion
-      * reduce compiler warnings -- thanks to all who reported/suggested these
-
-    v2.8.5 Sun May 22 10:26:02 2011  Doug Lea  (dl at gee)
-      * Always perform unlink checks unless INSECURE
-      * Add posix_memalign.
-      * Improve realloc to expand in more cases; expose realloc_in_place.
-        Thanks to Peter Buhr for the suggestion.
-      * Add footprint_limit, inspect_all, bulk_free. Thanks
-        to Barry Hayes and others for the suggestions.
-      * Internal refactorings to avoid calls while holding locks
-      * Use non-reentrant locks by default. Thanks to Roland McGrath
-        for the suggestion.
-      * Small fixes to mspace_destroy, reset_on_error.
-      * Various configuration extensions/changes. Thanks
-         to all who contributed these.
-
-    V2.8.4a Thu Apr 28 14:39:43 2011 (dl at gee.cs.oswego.edu)
-      * Update Creative Commons URL
-
-    V2.8.4 Wed May 27 09:56:23 2009  Doug Lea  (dl at gee)
-      * Use zeros instead of prev foot for is_mmapped
-      * Add mspace_track_large_chunks; thanks to Jean Brouwers
-      * Fix set_inuse in internal_realloc; thanks to Jean Brouwers
-      * Fix insufficient sys_alloc padding when using 16byte alignment
-      * Fix bad error check in mspace_footprint
-      * Adaptations for ptmalloc; thanks to Wolfram Gloger.
-      * Reentrant spin locks; thanks to Earl Chew and others
-      * Win32 improvements; thanks to Niall Douglas and Earl Chew
-      * Add NO_SEGMENT_TRAVERSAL and MAX_RELEASE_CHECK_RATE options
-      * Extension hook in malloc_state
-      * Various small adjustments to reduce warnings on some compilers
-      * Various configuration extensions/changes for more platforms. Thanks
-         to all who contributed these.
-
-    V2.8.3 Thu Sep 22 11:16:32 2005  Doug Lea  (dl at gee)
-      * Add max_footprint functions
-      * Ensure all appropriate literals are size_t
-      * Fix conditional compilation problem for some #define settings
-      * Avoid concatenating segments with the one provided
-        in create_mspace_with_base
-      * Rename some variables to avoid compiler shadowing warnings
-      * Use explicit lock initialization.
-      * Better handling of sbrk interference.
-      * Simplify and fix segment insertion, trimming and mspace_destroy
-      * Reinstate REALLOC_ZERO_BYTES_FREES option from 2.7.x
-      * Thanks especially to Dennis Flanagan for help on these.
-
-    V2.8.2 Sun Jun 12 16:01:10 2005  Doug Lea  (dl at gee)
-      * Fix memalign brace error.
-
-    V2.8.1 Wed Jun  8 16:11:46 2005  Doug Lea  (dl at gee)
-      * Fix improper #endif nesting in C++
-      * Add explicit casts needed for C++
-
-    V2.8.0 Mon May 30 14:09:02 2005  Doug Lea  (dl at gee)
-      * Use trees for large bins
-      * Support mspaces
-      * Use segments to unify sbrk-based and mmap-based system allocation,
-        removing need for emulation on most platforms without sbrk.
-      * Default safety checks
-      * Optional footer checks. Thanks to William Robertson for the idea.
-      * Internal code refactoring
-      * Incorporate suggestions and platform-specific changes.
-        Thanks to Dennis Flanagan, Colin Plumb, Niall Douglas,
-        Aaron Bachmann,  Emery Berger, and others.
-      * Speed up non-fastbin processing enough to remove fastbins.
-      * Remove useless cfree() to avoid conflicts with other apps.
-      * Remove internal memcpy, memset. Compilers handle builtins better.
-      * Remove some options that no one ever used and rename others.
-
-    V2.7.2 Sat Aug 17 09:07:30 2002  Doug Lea  (dl at gee)
-      * Fix malloc_state bitmap array misdeclaration
-
-    V2.7.1 Thu Jul 25 10:58:03 2002  Doug Lea  (dl at gee)
-      * Allow tuning of FIRST_SORTED_BIN_SIZE
-      * Use PTR_UINT as type for all ptr->int casts. Thanks to John Belmonte.
-      * Better detection and support for non-contiguousness of MORECORE.
-        Thanks to Andreas Mueller, Conal Walsh, and Wolfram Gloger
-      * Bypass most of malloc if no frees. Thanks To Emery Berger.
-      * Fix freeing of old top non-contiguous chunk im sysmalloc.
-      * Raised default trim and map thresholds to 256K.
-      * Fix mmap-related #defines. Thanks to Lubos Lunak.
-      * Fix copy macros; added LACKS_FCNTL_H. Thanks to Neal Walfield.
-      * Branch-free bin calculation
-      * Default trim and mmap thresholds now 256K.
-
-    V2.7.0 Sun Mar 11 14:14:06 2001  Doug Lea  (dl at gee)
-      * Introduce independent_comalloc and independent_calloc.
-        Thanks to Michael Pachos for motivation and help.
-      * Make optional .h file available
-      * Allow > 2GB requests on 32bit systems.
-      * new WIN32 sbrk, mmap, munmap, lock code from <Walter@GeNeSys-e.de>.
-        Thanks also to Andreas Mueller <a.mueller at paradatec.de>,
-        and Anonymous.
-      * Allow override of MALLOC_ALIGNMENT (Thanks to Ruud Waij for
-        helping test this.)
-      * memalign: check alignment arg
-      * realloc: don't try to shift chunks backwards, since this
-        leads to  more fragmentation in some programs and doesn't
-        seem to help in any others.
-      * Collect all cases in malloc requiring system memory into sysmalloc
-      * Use mmap as backup to sbrk
-      * Place all internal state in malloc_state
-      * Introduce fastbins (although similar to 2.5.1)
-      * Many minor tunings and cosmetic improvements
-      * Introduce USE_PUBLIC_MALLOC_WRAPPERS, USE_MALLOC_LOCK
-      * Introduce MALLOC_FAILURE_ACTION, MORECORE_CONTIGUOUS
-        Thanks to Tony E. Bennett <tbennett@nvidia.com> and others.
-      * Include errno.h to support default failure action.
-
-    V2.6.6 Sun Dec  5 07:42:19 1999  Doug Lea  (dl at gee)
-      * return null for negative arguments
-      * Added Several WIN32 cleanups from Martin C. Fong <mcfong at yahoo.com>
-         * Add 'LACKS_SYS_PARAM_H' for those systems without 'sys/param.h'
-          (e.g. WIN32 platforms)
-         * Cleanup header file inclusion for WIN32 platforms
-         * Cleanup code to avoid Microsoft Visual C++ compiler complaints
-         * Add 'USE_DL_PREFIX' to quickly allow co-existence with existing
-           memory allocation routines
-         * Set 'malloc_getpagesize' for WIN32 platforms (needs more work)
-         * Use 'assert' rather than 'ASSERT' in WIN32 code to conform to
-           usage of 'assert' in non-WIN32 code
-         * Improve WIN32 'sbrk()' emulation's 'findRegion()' routine to
-           avoid infinite loop
-      * Always call 'fREe()' rather than 'free()'
-
-    V2.6.5 Wed Jun 17 15:57:31 1998  Doug Lea  (dl at gee)
-      * Fixed ordering problem with boundary-stamping
-
-    V2.6.3 Sun May 19 08:17:58 1996  Doug Lea  (dl at gee)
-      * Added pvalloc, as recommended by H.J. Liu
-      * Added 64bit pointer support mainly from Wolfram Gloger
-      * Added anonymously donated WIN32 sbrk emulation
-      * Malloc, calloc, getpagesize: add optimizations from Raymond Nijssen
-      * malloc_extend_top: fix mask error that caused wastage after
-        foreign sbrks
-      * Add linux mremap support code from HJ Liu
-
-    V2.6.2 Tue Dec  5 06:52:55 1995  Doug Lea  (dl at gee)
-      * Integrated most documentation with the code.
-      * Add support for mmap, with help from
-        Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
-      * Use last_remainder in more cases.
-      * Pack bins using idea from  colin@nyx10.cs.du.edu
-      * Use ordered bins instead of best-fit threshhold
-      * Eliminate block-local decls to simplify tracing and debugging.
-      * Support another case of realloc via move into top
-      * Fix error occuring when initial sbrk_base not word-aligned.
-      * Rely on page size for units instead of SBRK_UNIT to
-        avoid surprises about sbrk alignment conventions.
-      * Add mallinfo, mallopt. Thanks to Raymond Nijssen
-        (raymond@es.ele.tue.nl) for the suggestion.
-      * Add `pad' argument to malloc_trim and top_pad mallopt parameter.
-      * More precautions for cases where other routines call sbrk,
-        courtesy of Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
-      * Added macros etc., allowing use in linux libc from
-        H.J. Lu (hjl@gnu.ai.mit.edu)
-      * Inverted this history list
-
-    V2.6.1 Sat Dec  2 14:10:57 1995  Doug Lea  (dl at gee)
-      * Re-tuned and fixed to behave more nicely with V2.6.0 changes.
-      * Removed all preallocation code since under current scheme
-        the work required to undo bad preallocations exceeds
-        the work saved in good cases for most test programs.
-      * No longer use return list or unconsolidated bins since
-        no scheme using them consistently outperforms those that don't
-        given above changes.
-      * Use best fit for very large chunks to prevent some worst-cases.
-      * Added some support for debugging
-
-    V2.6.0 Sat Nov  4 07:05:23 1995  Doug Lea  (dl at gee)
-      * Removed footers when chunks are in use. Thanks to
-        Paul Wilson (wilson@cs.texas.edu) for the suggestion.
-
-    V2.5.4 Wed Nov  1 07:54:51 1995  Doug Lea  (dl at gee)
-      * Added malloc_trim, with help from Wolfram Gloger
-        (wmglo@Dent.MED.Uni-Muenchen.DE).
-
-    V2.5.3 Tue Apr 26 10:16:01 1994  Doug Lea  (dl at g)
-
-    V2.5.2 Tue Apr  5 16:20:40 1994  Doug Lea  (dl at g)
-      * realloc: try to expand in both directions
-      * malloc: swap order of clean-bin strategy;
-      * realloc: only conditionally expand backwards
-      * Try not to scavenge used bins
-      * Use bin counts as a guide to preallocation
-      * Occasionally bin return list chunks in first scan
-      * Add a few optimizations from colin@nyx10.cs.du.edu
-
-    V2.5.1 Sat Aug 14 15:40:43 1993  Doug Lea  (dl at g)
-      * faster bin computation & slightly different binning
-      * merged all consolidations to one part of malloc proper
-         (eliminating old malloc_find_space & malloc_clean_bin)
-      * Scan 2 returns chunks (not just 1)
-      * Propagate failure in realloc if malloc returns 0
-      * Add stuff to allow compilation on non-ANSI compilers
-          from kpv@research.att.com
-
-    V2.5 Sat Aug  7 07:41:59 1993  Doug Lea  (dl at g.oswego.edu)
-      * removed potential for odd address access in prev_chunk
-      * removed dependency on getpagesize.h
-      * misc cosmetics and a bit more internal documentation
-      * anticosmetics: mangled names in macros to evade debugger strangeness
-      * tested on sparc, hp-700, dec-mips, rs6000
-          with gcc & native cc (hp, dec only) allowing
-          Detlefs & Zorn comparison study (in SIGPLAN Notices.)
-
-    Trial version Fri Aug 28 13:14:29 1992  Doug Lea  (dl at g.oswego.edu)
-      * Based loosely on libg++-1.2X malloc. (It retains some of the overall
-         structure of old version,  but most details differ.)
-
-*/
diff --git a/cpp/src/plasma/thirdparty/xxhash.cc b/cpp/src/plasma/thirdparty/xxhash.cc
deleted file mode 100644
index f74880b0de71d..0000000000000
--- a/cpp/src/plasma/thirdparty/xxhash.cc
+++ /dev/null
@@ -1,889 +0,0 @@
-/*
-*  xxHash - Fast Hash algorithm
-*  Copyright (C) 2012-2016, Yann Collet
-*
-*  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-*
-*  Redistribution and use in source and binary forms, with or without
-*  modification, are permitted provided that the following conditions are
-*  met:
-*
-*  * Redistributions of source code must retain the above copyright
-*  notice, this list of conditions and the following disclaimer.
-*  * Redistributions in binary form must reproduce the above
-*  copyright notice, this list of conditions and the following disclaimer
-*  in the documentation and/or other materials provided with the
-*  distribution.
-*
-*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-*  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-*  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-*  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-*  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-*  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-*  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-*  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-*  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*
-*  You can contact the author at :
-*  - xxHash homepage: http://www.xxhash.com
-*  - xxHash source repository : https://github.com/Cyan4973/xxHash
-*/
-
-
-/* *************************************
-*  Tuning parameters
-***************************************/
-/*!XXH_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
- *            It can generate buggy code on targets which do not support unaligned memory accesses.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://stackoverflow.com/a/32095106/646947 for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
-#    define XXH_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
-  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
-#    define XXH_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
-
-/*!XXH_ACCEPT_NULL_INPUT_POINTER :
- * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
- * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
- * By default, this option is disabled. To enable it, uncomment below define :
- */
-/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
-
-/*!XXH_FORCE_NATIVE_FORMAT :
- * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
- * Results are therefore identical for little-endian and big-endian CPU.
- * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
- * Should endian-independence be of no importance for your application, you may set the #define below to 1,
- * to improve speed for Big-endian CPU.
- * This option has no impact on Little_Endian CPU.
- */
-#ifndef XXH_FORCE_NATIVE_FORMAT   /* can be defined externally */
-#  define XXH_FORCE_NATIVE_FORMAT 0
-#endif
-
-/*!XXH_FORCE_ALIGN_CHECK :
- * This is a minor performance trick, only useful with lots of very small keys.
- * It means : check for aligned/unaligned input.
- * The check costs one initial branch per hash; set to 0 when the input data
- * is guaranteed to be aligned.
- */
-#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
-#  if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
-#    define XXH_FORCE_ALIGN_CHECK 0
-#  else
-#    define XXH_FORCE_ALIGN_CHECK 1
-#  endif
-#endif
-
-
-/* *************************************
-*  Includes & Memory related functions
-***************************************/
-/*! Modify the local functions below should you wish to use some other memory routines
-*   for malloc(), free() */
-#include <stdlib.h>
-static void* XXH_malloc(size_t s) { return malloc(s); }
-static void  XXH_free  (void* p)  { free(p); }
-/*! and for memcpy() */
-#include <string.h>
-static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
-
-#define XXH_STATIC_LINKING_ONLY
-#include "xxhash.h"
-
-
-/* *************************************
-*  Compiler Specific Options
-***************************************/
-#ifdef _MSC_VER    /* Visual Studio */
-#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
-#  define FORCE_INLINE static __forceinline
-#else
-#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
-#    ifdef __GNUC__
-#      define FORCE_INLINE static inline __attribute__((always_inline))
-#    else
-#      define FORCE_INLINE static inline
-#    endif
-#  else
-#    define FORCE_INLINE static
-#  endif /* __STDC_VERSION__ */
-#endif
-
-
-/* *************************************
-*  Basic Types
-***************************************/
-#ifndef MEM_MODULE
-# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-#   include <stdint.h>
-    typedef uint8_t  BYTE;
-    typedef uint16_t U16;
-    typedef uint32_t U32;
-    typedef  int32_t S32;
-# else
-    typedef unsigned char      BYTE;
-    typedef unsigned short     U16;
-    typedef unsigned int       U32;
-    typedef   signed int       S32;
-# endif
-#endif
-
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
-
-/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
-static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
-
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U32 u32; } __attribute__((packed)) unalign;
-static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-
-#else
-
-/* portable and safe solution. Generally efficient.
- * see : http://stackoverflow.com/a/32095106/646947
- */
-static U32 XXH_read32(const void* memPtr)
-{
-    U32 val;
-    memcpy(&val, memPtr, sizeof(val));
-    return val;
-}
-
-#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
-
-
-/* ****************************************
-*  Compiler-specific Functions and Macros
-******************************************/
-#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
-/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
-#if defined(_MSC_VER)
-#  define XXH_rotl32(x,r) _rotl(x,r)
-#  define XXH_rotl64(x,r) _rotl64(x,r)
-#else
-#  define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-#  define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
-#endif
-
-#if defined(_MSC_VER)     /* Visual Studio */
-#  define XXH_swap32 _byteswap_ulong
-#elif XXH_GCC_VERSION >= 403
-#  define XXH_swap32 __builtin_bswap32
-#else
-static U32 XXH_swap32 (U32 x)
-{
-    return  ((x << 24) & 0xff000000 ) |
-            ((x <<  8) & 0x00ff0000 ) |
-            ((x >>  8) & 0x0000ff00 ) |
-            ((x >> 24) & 0x000000ff );
-}
-#endif
-
-
-/* *************************************
-*  Architecture Macros
-***************************************/
-typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
-
-/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
-#ifndef XXH_CPU_LITTLE_ENDIAN
-    static const int g_one = 1;
-#   define XXH_CPU_LITTLE_ENDIAN   (*(const char*)(&g_one))
-#endif
-
-
-/* ***************************
-*  Memory reads
-*****************************/
-typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
-
-FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
-{
-    if (align==XXH_unaligned)
-        return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
-    else
-        return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
-}
-
-FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
-{
-    return XXH_readLE32_align(ptr, endian, XXH_unaligned);
-}
-
-static U32 XXH_readBE32(const void* ptr)
-{
-    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
-}
-
-
-/* *************************************
-*  Macros
-***************************************/
-#define XXH_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(int)(!!(c)) }; }    /* use only *after* variable declarations */
-XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
-
-
-/* *******************************************************************
-*  32-bits hash functions
-*********************************************************************/
-static const U32 PRIME32_1 = 2654435761U;
-static const U32 PRIME32_2 = 2246822519U;
-static const U32 PRIME32_3 = 3266489917U;
-static const U32 PRIME32_4 =  668265263U;
-static const U32 PRIME32_5 =  374761393U;
-
-static U32 XXH32_round(U32 seed, U32 input)
-{
-    seed += input * PRIME32_2;
-    seed  = XXH_rotl32(seed, 13);
-    seed *= PRIME32_1;
-    return seed;
-}
-
-FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
-{
-    const BYTE* p = (const BYTE*)input;
-    const BYTE* bEnd = p + len;
-    U32 h32;
-#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
-    if (p==NULL) {
-        len=0;
-        bEnd=p=(const BYTE*)(size_t)16;
-    }
-#endif
-
-    if (len>=16) {
-        const BYTE* const limit = bEnd - 16;
-        U32 v1 = seed + PRIME32_1 + PRIME32_2;
-        U32 v2 = seed + PRIME32_2;
-        U32 v3 = seed + 0;
-        U32 v4 = seed - PRIME32_1;
-
-        do {
-            v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
-            v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
-            v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
-            v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
-        } while (p<=limit);
-
-        h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
-    } else {
-        h32  = seed + PRIME32_5;
-    }
-
-    h32 += (U32) len;
-
-    while (p+4<=bEnd) {
-        h32 += XXH_get32bits(p) * PRIME32_3;
-        h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
-        p+=4;
-    }
-
-    while (p<bEnd) {
-        h32 += (*p) * PRIME32_5;
-        h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
-        p++;
-    }
-
-    h32 ^= h32 >> 15;
-    h32 *= PRIME32_2;
-    h32 ^= h32 >> 13;
-    h32 *= PRIME32_3;
-    h32 ^= h32 >> 16;
-
-    return h32;
-}
-
-
-XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
-{
-#if 0
-    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
-    XXH32_state_t state;
-    XXH32_reset(&state, seed);
-    XXH32_update(&state, input, len);
-    return XXH32_digest(&state);
-#else
-    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
-    if (XXH_FORCE_ALIGN_CHECK) {
-        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
-            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
-                return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
-            else
-                return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
-    }   }
-
-    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
-        return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
-    else
-        return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
-#endif
-}
-
-
-
-/*======   Hash streaming   ======*/
-
-XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
-{
-    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
-}
-XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
-{
-    XXH_free(statePtr);
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
-{
-    memcpy(dstState, srcState, sizeof(*dstState));
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
-{
-    XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
-    memset(&state, 0, sizeof(state)-4);   /* do not write into reserved, for future removal */
-    state.v1 = seed + PRIME32_1 + PRIME32_2;
-    state.v2 = seed + PRIME32_2;
-    state.v3 = seed + 0;
-    state.v4 = seed - PRIME32_1;
-    memcpy(statePtr, &state, sizeof(state));
-    return XXH_OK;
-}
-
-
-FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
-{
-    const BYTE* p = (const BYTE*)input;
-    const BYTE* const bEnd = p + len;
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
-    if (input==NULL) return XXH_ERROR;
-#endif
-
-    state->total_len_32 += (unsigned)len;
-    state->large_len |= (len>=16) | (state->total_len_32>=16);
-
-    if (state->memsize + len < 16)  {   /* fill in tmp buffer */
-        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
-        state->memsize += (unsigned)len;
-        return XXH_OK;
-    }
-
-    if (state->memsize) {   /* some data left from previous update */
-        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
-        {   const U32* p32 = state->mem32;
-            state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
-            state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
-            state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
-            state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
-        }
-        p += 16-state->memsize;
-        state->memsize = 0;
-    }
-
-    if (p <= bEnd-16) {
-        const BYTE* const limit = bEnd - 16;
-        U32 v1 = state->v1;
-        U32 v2 = state->v2;
-        U32 v3 = state->v3;
-        U32 v4 = state->v4;
-
-        do {
-            v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
-            v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
-            v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
-            v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
-        } while (p<=limit);
-
-        state->v1 = v1;
-        state->v2 = v2;
-        state->v3 = v3;
-        state->v4 = v4;
-    }
-
-    if (p < bEnd) {
-        XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
-        state->memsize = (unsigned)(bEnd-p);
-    }
-
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
-{
-    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
-    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
-        return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
-    else
-        return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
-}
-
-
-
-FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
-{
-    const BYTE * p = (const BYTE*)state->mem32;
-    const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
-    U32 h32;
-
-    if (state->large_len) {
-        h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
-    } else {
-        h32 = state->v3 /* == seed */ + PRIME32_5;
-    }
-
-    h32 += state->total_len_32;
-
-    while (p+4<=bEnd) {
-        h32 += XXH_readLE32(p, endian) * PRIME32_3;
-        h32  = XXH_rotl32(h32, 17) * PRIME32_4;
-        p+=4;
-    }
-
-    while (p<bEnd) {
-        h32 += (*p) * PRIME32_5;
-        h32  = XXH_rotl32(h32, 11) * PRIME32_1;
-        p++;
-    }
-
-    h32 ^= h32 >> 15;
-    h32 *= PRIME32_2;
-    h32 ^= h32 >> 13;
-    h32 *= PRIME32_3;
-    h32 ^= h32 >> 16;
-
-    return h32;
-}
-
-
-XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
-{
-    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
-    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
-        return XXH32_digest_endian(state_in, XXH_littleEndian);
-    else
-        return XXH32_digest_endian(state_in, XXH_bigEndian);
-}
-
-
-/*======   Canonical representation   ======*/
-
-/*! Default XXH result types are basic unsigned 32 and 64 bits.
-*   The canonical representation follows human-readable write convention, aka big-endian (large digits first).
-*   These functions allow transformation of hash result into and from its canonical format.
-*   This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
-*/
-
-XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
-{
-    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
-    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
-    memcpy(dst, &hash, sizeof(*dst));
-}
-
-XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
-{
-    return XXH_readBE32(src);
-}
-
-
-#ifndef XXH_NO_LONG_LONG
-
-/* *******************************************************************
-*  64-bits hash functions
-*********************************************************************/
-
-/*======   Memory access   ======*/
-
-#ifndef MEM_MODULE
-# define MEM_MODULE
-# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-#   include <stdint.h>
-    typedef uint64_t U64;
-# else
-    typedef unsigned long long U64;   /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
-# endif
-#endif
-
-
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
-
-/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
-static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
-static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
-
-#else
-
-/* portable and safe solution. Generally efficient.
- * see : http://stackoverflow.com/a/32095106/646947
- */
-
-static U64 XXH_read64(const void* memPtr)
-{
-    U64 val;
-    memcpy(&val, memPtr, sizeof(val));
-    return val;
-}
-
-#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
-
-#if defined(_MSC_VER)     /* Visual Studio */
-#  define XXH_swap64 _byteswap_uint64
-#elif XXH_GCC_VERSION >= 403
-#  define XXH_swap64 __builtin_bswap64
-#else
-static U64 XXH_swap64 (U64 x)
-{
-    return  ((x << 56) & 0xff00000000000000ULL) |
-            ((x << 40) & 0x00ff000000000000ULL) |
-            ((x << 24) & 0x0000ff0000000000ULL) |
-            ((x << 8)  & 0x000000ff00000000ULL) |
-            ((x >> 8)  & 0x00000000ff000000ULL) |
-            ((x >> 24) & 0x0000000000ff0000ULL) |
-            ((x >> 40) & 0x000000000000ff00ULL) |
-            ((x >> 56) & 0x00000000000000ffULL);
-}
-#endif
-
-FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
-{
-    if (align==XXH_unaligned)
-        return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
-    else
-        return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
-}
-
-FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
-{
-    return XXH_readLE64_align(ptr, endian, XXH_unaligned);
-}
-
-static U64 XXH_readBE64(const void* ptr)
-{
-    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
-}
-
-
-/*======   xxh64   ======*/
-
-static const U64 PRIME64_1 = 11400714785074694791ULL;
-static const U64 PRIME64_2 = 14029467366897019727ULL;
-static const U64 PRIME64_3 =  1609587929392839161ULL;
-static const U64 PRIME64_4 =  9650029242287828579ULL;
-static const U64 PRIME64_5 =  2870177450012600261ULL;
-
-static U64 XXH64_round(U64 acc, U64 input)
-{
-    acc += input * PRIME64_2;
-    acc  = XXH_rotl64(acc, 31);
-    acc *= PRIME64_1;
-    return acc;
-}
-
-static U64 XXH64_mergeRound(U64 acc, U64 val)
-{
-    val  = XXH64_round(0, val);
-    acc ^= val;
-    acc  = acc * PRIME64_1 + PRIME64_4;
-    return acc;
-}
-
-FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
-{
-    const BYTE* p = (const BYTE*)input;
-    const BYTE* bEnd = p + len;
-    U64 h64;
-#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
-    if (p==NULL) {
-        len=0;
-        bEnd=p=(const BYTE*)(size_t)32;
-    }
-#endif
-
-    if (len>=32) {
-        const BYTE* const limit = bEnd - 32;
-        U64 v1 = seed + PRIME64_1 + PRIME64_2;
-        U64 v2 = seed + PRIME64_2;
-        U64 v3 = seed + 0;
-        U64 v4 = seed - PRIME64_1;
-
-        do {
-            v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
-            v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
-            v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
-            v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
-        } while (p<=limit);
-
-        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
-        h64 = XXH64_mergeRound(h64, v1);
-        h64 = XXH64_mergeRound(h64, v2);
-        h64 = XXH64_mergeRound(h64, v3);
-        h64 = XXH64_mergeRound(h64, v4);
-
-    } else {
-        h64  = seed + PRIME64_5;
-    }
-
-    h64 += (U64) len;
-
-    while (p+8<=bEnd) {
-        U64 const k1 = XXH64_round(0, XXH_get64bits(p));
-        h64 ^= k1;
-        h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
-        p+=8;
-    }
-
-    if (p+4<=bEnd) {
-        h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
-        h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
-        p+=4;
-    }
-
-    while (p<bEnd) {
-        h64 ^= (*p) * PRIME64_5;
-        h64 = XXH_rotl64(h64, 11) * PRIME64_1;
-        p++;
-    }
-
-    h64 ^= h64 >> 33;
-    h64 *= PRIME64_2;
-    h64 ^= h64 >> 29;
-    h64 *= PRIME64_3;
-    h64 ^= h64 >> 32;
-
-    return h64;
-}
-
-
-XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
-{
-#if 0
-    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
-    XXH64_state_t state;
-    XXH64_reset(&state, seed);
-    XXH64_update(&state, input, len);
-    return XXH64_digest(&state);
-#else
-    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
-    if (XXH_FORCE_ALIGN_CHECK) {
-        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
-            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
-                return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
-            else
-                return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
-    }   }
-
-    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
-        return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
-    else
-        return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
-#endif
-}
-
-/*======   Hash Streaming   ======*/
-
-XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
-{
-    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
-}
-XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
-{
-    XXH_free(statePtr);
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
-{
-    memcpy(dstState, srcState, sizeof(*dstState));
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
-{
-    XXH64_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
-    memset(&state, 0, sizeof(state)-8);   /* do not write into reserved, for future removal */
-    state.v1 = seed + PRIME64_1 + PRIME64_2;
-    state.v2 = seed + PRIME64_2;
-    state.v3 = seed + 0;
-    state.v4 = seed - PRIME64_1;
-    memcpy(statePtr, &state, sizeof(state));
-    return XXH_OK;
-}
-
-FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
-{
-    const BYTE* p = (const BYTE*)input;
-    const BYTE* const bEnd = p + len;
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
-    if (input==NULL) return XXH_ERROR;
-#endif
-
-    state->total_len += len;
-
-    if (state->memsize + len < 32) {  /* fill in tmp buffer */
-        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
-        state->memsize += (U32)len;
-        return XXH_OK;
-    }
-
-    if (state->memsize) {   /* tmp buffer is full */
-        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
-        state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
-        state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
-        state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
-        state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
-        p += 32-state->memsize;
-        state->memsize = 0;
-    }
-
-    if (p+32 <= bEnd) {
-        const BYTE* const limit = bEnd - 32;
-        U64 v1 = state->v1;
-        U64 v2 = state->v2;
-        U64 v3 = state->v3;
-        U64 v4 = state->v4;
-
-        do {
-            v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
-            v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
-            v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
-            v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
-        } while (p<=limit);
-
-        state->v1 = v1;
-        state->v2 = v2;
-        state->v3 = v3;
-        state->v4 = v4;
-    }
-
-    if (p < bEnd) {
-        XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
-        state->memsize = (unsigned)(bEnd-p);
-    }
-
-    return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
-{
-    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
-    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
-        return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
-    else
-        return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
-}
-
-FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
-{
-    const BYTE * p = (const BYTE*)state->mem64;
-    const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
-    U64 h64;
-
-    if (state->total_len >= 32) {
-        U64 const v1 = state->v1;
-        U64 const v2 = state->v2;
-        U64 const v3 = state->v3;
-        U64 const v4 = state->v4;
-
-        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
-        h64 = XXH64_mergeRound(h64, v1);
-        h64 = XXH64_mergeRound(h64, v2);
-        h64 = XXH64_mergeRound(h64, v3);
-        h64 = XXH64_mergeRound(h64, v4);
-    } else {
-        h64  = state->v3 + PRIME64_5;
-    }
-
-    h64 += (U64) state->total_len;
-
-    while (p+8<=bEnd) {
-        U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
-        h64 ^= k1;
-        h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
-        p+=8;
-    }
-
-    if (p+4<=bEnd) {
-        h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
-        h64  = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
-        p+=4;
-    }
-
-    while (p<bEnd) {
-        h64 ^= (*p) * PRIME64_5;
-        h64  = XXH_rotl64(h64, 11) * PRIME64_1;
-        p++;
-    }
-
-    h64 ^= h64 >> 33;
-    h64 *= PRIME64_2;
-    h64 ^= h64 >> 29;
-    h64 *= PRIME64_3;
-    h64 ^= h64 >> 32;
-
-    return h64;
-}
-
-XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
-{
-    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
-    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
-        return XXH64_digest_endian(state_in, XXH_littleEndian);
-    else
-        return XXH64_digest_endian(state_in, XXH_bigEndian);
-}
-
-
-/*====== Canonical representation   ======*/
-
-XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
-{
-    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
-    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
-    memcpy(dst, &hash, sizeof(*dst));
-}
-
-XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
-{
-    return XXH_readBE64(src);
-}
-
-#endif  /* XXH_NO_LONG_LONG */
diff --git a/cpp/src/plasma/thirdparty/xxhash.h b/cpp/src/plasma/thirdparty/xxhash.h
deleted file mode 100644
index 9d831e03b35f6..0000000000000
--- a/cpp/src/plasma/thirdparty/xxhash.h
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
-   xxHash - Extremely Fast Hash algorithm
-   Header File
-   Copyright (C) 2012-2016, Yann Collet.
-
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - xxHash source repository : https://github.com/Cyan4973/xxHash
-*/
-
-/* Notice extracted from xxHash homepage :
-
-xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
-It also successfully passes all tests from the SMHasher suite.
-
-Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
-
-Name            Speed       Q.Score   Author
-xxHash          5.4 GB/s     10
-CrapWow         3.2 GB/s      2       Andrew
-MumurHash 3a    2.7 GB/s     10       Austin Appleby
-SpookyHash      2.0 GB/s     10       Bob Jenkins
-SBox            1.4 GB/s      9       Bret Mulvey
-Lookup3         1.2 GB/s      9       Bob Jenkins
-SuperFastHash   1.2 GB/s      1       Paul Hsieh
-CityHash64      1.05 GB/s    10       Pike & Alakuijala
-FNV             0.55 GB/s     5       Fowler, Noll, Vo
-CRC32           0.43 GB/s     9
-MD5-32          0.33 GB/s    10       Ronald L. Rivest
-SHA1-32         0.28 GB/s    10
-
-Q.Score is a measure of quality of the hash function.
-It depends on successfully passing SMHasher test set.
-10 is a perfect score.
-
-A 64-bits version, named XXH64, is available since r35.
-It offers much better speed, but for 64-bits applications only.
-Name     Speed on 64 bits    Speed on 32 bits
-XXH64       13.8 GB/s            1.9 GB/s
-XXH32        6.8 GB/s            6.0 GB/s
-*/
-
-#ifndef XXHASH_H_5627135585666179
-#define XXHASH_H_5627135585666179 1
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************
-*  Definitions
-******************************/
-#include <stddef.h>   /* size_t */
-typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
-
-
-/* ****************************
-*  API modifier
-******************************/
-/** XXH_PRIVATE_API
-*   This is useful to include xxhash functions in `static` mode
-*   in order to inline them, and remove their symbol from the public list.
-*   Methodology :
-*     #define XXH_PRIVATE_API
-*     #include "xxhash.h"
-*   `xxhash.c` is automatically included.
-*   It's not useful to compile and link it as a separate module.
-*/
-#ifdef XXH_PRIVATE_API
-#  ifndef XXH_STATIC_LINKING_ONLY
-#    define XXH_STATIC_LINKING_ONLY
-#  endif
-#  if defined(__GNUC__)
-#    define XXH_PUBLIC_API static __inline __attribute__((unused))
-#  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-#    define XXH_PUBLIC_API static inline
-#  elif defined(_MSC_VER)
-#    define XXH_PUBLIC_API static __inline
-#  else
-#    define XXH_PUBLIC_API static   /* this version may generate warnings for unused static functions; disable the relevant warning */
-#  endif
-#else
-#  define XXH_PUBLIC_API   /* do nothing */
-#endif /* XXH_PRIVATE_API */
-
-/*!XXH_NAMESPACE, aka Namespace Emulation :
-
-If you want to include _and expose_ xxHash functions from within your own library,
-but also want to avoid symbol collisions with other libraries which may also include xxHash,
-
-you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
-with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
-
-Note that no change is required within the calling program as long as it includes `xxhash.h` :
-regular symbol name will be automatically translated by this header.
-*/
-#ifdef XXH_NAMESPACE
-#  define XXH_CAT(A,B) A##B
-#  define XXH_NAME2(A,B) XXH_CAT(A,B)
-#  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
-#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
-#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
-#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
-#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
-#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
-#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
-#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
-#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
-#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
-#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
-#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
-#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
-#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
-#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
-#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
-#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
-#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
-#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
-#endif
-
-
-/* *************************************
-*  Version
-***************************************/
-#define XXH_VERSION_MAJOR    0
-#define XXH_VERSION_MINOR    6
-#define XXH_VERSION_RELEASE  2
-#define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
-XXH_PUBLIC_API unsigned XXH_versionNumber (void);
-
-
-/*-**********************************************************************
-*  32-bits hash
-************************************************************************/
-typedef unsigned int XXH32_hash_t;
-
-/*! XXH32() :
-    Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
-    The memory between input & input+length must be valid (allocated and read-accessible).
-    "seed" can be used to alter the result predictably.
-    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
-XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
-
-/*======   Streaming   ======*/
-typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
-XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
-XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
-XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
-
-XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned int seed);
-XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH32_hash_t  XXH32_digest (const XXH32_state_t* statePtr);
-
-/*
-These functions generate the xxHash of an input provided in multiple segments.
-Note that, for small input, they are slower than single-call functions, due to state management.
-For small input, prefer `XXH32()` and `XXH64()` .
-
-XXH state must first be allocated, using XXH*_createState() .
-
-Start a new hash by initializing state with a seed, using XXH*_reset().
-
-Then, feed the hash state by calling XXH*_update() as many times as necessary.
-Obviously, input must be allocated and read accessible.
-The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
-
-Finally, a hash value can be produced anytime, by using XXH*_digest().
-This function returns the nn-bits hash as an int or long long.
-
-It's still possible to continue inserting input into the hash state after a digest,
-and generate some new hashes later on, by calling again XXH*_digest().
-
-When done, free XXH state space if it was allocated dynamically.
-*/
-
-/*======   Canonical representation   ======*/
-
-typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
-XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
-XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
-
-/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
-*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
-*  These functions allow transformation of hash result into and from its canonical format.
-*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
-*/
-
-
-#ifndef XXH_NO_LONG_LONG
-/*-**********************************************************************
-*  64-bits hash
-************************************************************************/
-typedef unsigned long long XXH64_hash_t;
-
-/*! XXH64() :
-    Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
-    "seed" can be used to alter the result predictably.
-    This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
-*/
-XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
-
-/*======   Streaming   ======*/
-typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
-XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
-XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
-XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
-
-XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
-XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
-
-/*======   Canonical representation   ======*/
-typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
-XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
-XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
-#endif  /* XXH_NO_LONG_LONG */
-
-
-#ifdef XXH_STATIC_LINKING_ONLY
-
-/* ================================================================================================
-   This section contains definitions which are not guaranteed to remain stable.
-   They may change in future versions, becoming incompatible with a different version of the library.
-   They shall only be used with static linking.
-   Never use these definitions in association with dynamic linking !
-=================================================================================================== */
-
-/* These definitions are only meant to make possible
-   static allocation of XXH state, on stack or in a struct for example.
-   Never use members directly. */
-
-struct XXH32_state_s {
-   unsigned total_len_32;
-   unsigned large_len;
-   unsigned v1;
-   unsigned v2;
-   unsigned v3;
-   unsigned v4;
-   unsigned mem32[4];   /* buffer defined as U32 for alignment */
-   unsigned memsize;
-   unsigned reserved;   /* never read nor write, will be removed in a future version */
-};   /* typedef'd to XXH32_state_t */
-
-#ifndef XXH_NO_LONG_LONG   /* remove 64-bits support */
-struct XXH64_state_s {
-   unsigned long long total_len;
-   unsigned long long v1;
-   unsigned long long v2;
-   unsigned long long v3;
-   unsigned long long v4;
-   unsigned long long mem64[4];   /* buffer defined as U64 for alignment */
-   unsigned memsize;
-   unsigned reserved[2];          /* never read nor write, will be removed in a future version */
-};   /* typedef'd to XXH64_state_t */
-#endif
-
-#ifdef XXH_PRIVATE_API
-#  include "xxhash.c"   /* include xxhash function bodies as `static`, for inlining */
-#endif
-
-#endif /* XXH_STATIC_LINKING_ONLY */
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* XXHASH_H_5627135585666179 */

From e9f76e125b836d0fdc0a533e2fee3fca8bf4c1a1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 20 Jul 2017 13:02:41 -0400
Subject: [PATCH 0837/1644] [maven-release-plugin] prepare release
 apache-arrow-0.5.0

Change-Id: Icb6a8ac2d92a38190fbc89d88a0a97120646f843
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index e09275476d175..6b962e2b5ca40 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.5.0-SNAPSHOT</version>
+  <version>0.5.0</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index dc4d0daf93d92..e204cd66fd04d 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.5.0-SNAPSHOT</version>
+    <version>0.5.0</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 1ec3d561121c3..7eaf27db813dc 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.5.0-SNAPSHOT</version>
+  <version>0.5.0</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>HEAD</tag>
+    <tag>apache-arrow-0.5.0</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 53331f2f24541..6d28c9f556c7e 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.5.0-SNAPSHOT</version>
+        <version>0.5.0</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index a117a2fb3b7cb..c484935b4c3f2 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.5.0-SNAPSHOT</version>
+    <version>0.5.0</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 9b26ed84e144a2c251a8e163d88302a5bdaf7dfd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 20 Jul 2017 13:02:51 -0400
Subject: [PATCH 0838/1644] [maven-release-plugin] prepare for next development
 iteration

Change-Id: I8fff2322e3b3feb6e80cfc2be1d5743e9ec9bbb7
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 6b962e2b5ca40..af8ff609dbc9c 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.5.0</version>
+  <version>0.6.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index e204cd66fd04d..9a8d2d7c9b56f 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.5.0</version>
+    <version>0.6.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 7eaf27db813dc..2613a44104576 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.5.0</version>
+  <version>0.6.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.5.0</tag>
+    <tag>HEAD</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 6d28c9f556c7e..9d067ef1e9bc2 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.5.0</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index c484935b4c3f2..e15ab9a2497fc 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.5.0</version>
+    <version>0.6.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 2c8101515e2a0ab515c03f82dc84b02ca6c466da Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 23 Jul 2017 14:37:54 -0400
Subject: [PATCH 0839/1644] [C++] Restore Plasma source tree after 0.5.0
 release

This reverts commit 62ef2cd8a39fc93e7fa4bb790d7cd92adb77571f.
---
 cpp/src/plasma/CMakeLists.txt              |  113 +
 cpp/src/plasma/client.cc                   |  557 ++
 cpp/src/plasma/client.h                    |  343 ++
 cpp/src/plasma/common.cc                   |   83 +
 cpp/src/plasma/common.h                    |   63 +
 cpp/src/plasma/events.cc                   |   81 +
 cpp/src/plasma/events.h                    |   99 +
 cpp/src/plasma/eviction_policy.cc          |  107 +
 cpp/src/plasma/eviction_policy.h           |  134 +
 cpp/src/plasma/extension.cc                |  456 ++
 cpp/src/plasma/extension.h                 |   50 +
 cpp/src/plasma/fling.cc                    |   90 +
 cpp/src/plasma/fling.h                     |   52 +
 cpp/src/plasma/format/.gitignore           |    1 +
 cpp/src/plasma/format/common.fbs           |   34 +
 cpp/src/plasma/format/plasma.fbs           |  291 +
 cpp/src/plasma/io.cc                       |  212 +
 cpp/src/plasma/io.h                        |   55 +
 cpp/src/plasma/malloc.cc                   |  178 +
 cpp/src/plasma/malloc.h                    |   26 +
 cpp/src/plasma/plasma.cc                   |   64 +
 cpp/src/plasma/plasma.h                    |  191 +
 cpp/src/plasma/protocol.cc                 |  502 ++
 cpp/src/plasma/protocol.h                  |  170 +
 cpp/src/plasma/store.cc                    |  683 +++
 cpp/src/plasma/store.h                     |  169 +
 cpp/src/plasma/test/client_tests.cc        |  132 +
 cpp/src/plasma/test/run_tests.sh           |   61 +
 cpp/src/plasma/test/run_valgrind.sh        |   27 +
 cpp/src/plasma/test/serialization_tests.cc |  388 ++
 cpp/src/plasma/thirdparty/ae/ae.c          |  465 ++
 cpp/src/plasma/thirdparty/ae/ae.h          |  123 +
 cpp/src/plasma/thirdparty/ae/ae_epoll.c    |  135 +
 cpp/src/plasma/thirdparty/ae/ae_evport.c   |  320 +
 cpp/src/plasma/thirdparty/ae/ae_kqueue.c   |  138 +
 cpp/src/plasma/thirdparty/ae/ae_select.c   |  106 +
 cpp/src/plasma/thirdparty/ae/config.h      |   54 +
 cpp/src/plasma/thirdparty/ae/zmalloc.h     |   45 +
 cpp/src/plasma/thirdparty/dlmalloc.c       | 6281 ++++++++++++++++++++
 cpp/src/plasma/thirdparty/xxhash.cc        |  889 +++
 cpp/src/plasma/thirdparty/xxhash.h         |  293 +
 41 files changed, 14261 insertions(+)
 create mode 100644 cpp/src/plasma/CMakeLists.txt
 create mode 100644 cpp/src/plasma/client.cc
 create mode 100644 cpp/src/plasma/client.h
 create mode 100644 cpp/src/plasma/common.cc
 create mode 100644 cpp/src/plasma/common.h
 create mode 100644 cpp/src/plasma/events.cc
 create mode 100644 cpp/src/plasma/events.h
 create mode 100644 cpp/src/plasma/eviction_policy.cc
 create mode 100644 cpp/src/plasma/eviction_policy.h
 create mode 100644 cpp/src/plasma/extension.cc
 create mode 100644 cpp/src/plasma/extension.h
 create mode 100644 cpp/src/plasma/fling.cc
 create mode 100644 cpp/src/plasma/fling.h
 create mode 100644 cpp/src/plasma/format/.gitignore
 create mode 100644 cpp/src/plasma/format/common.fbs
 create mode 100644 cpp/src/plasma/format/plasma.fbs
 create mode 100644 cpp/src/plasma/io.cc
 create mode 100644 cpp/src/plasma/io.h
 create mode 100644 cpp/src/plasma/malloc.cc
 create mode 100644 cpp/src/plasma/malloc.h
 create mode 100644 cpp/src/plasma/plasma.cc
 create mode 100644 cpp/src/plasma/plasma.h
 create mode 100644 cpp/src/plasma/protocol.cc
 create mode 100644 cpp/src/plasma/protocol.h
 create mode 100644 cpp/src/plasma/store.cc
 create mode 100644 cpp/src/plasma/store.h
 create mode 100644 cpp/src/plasma/test/client_tests.cc
 create mode 100644 cpp/src/plasma/test/run_tests.sh
 create mode 100644 cpp/src/plasma/test/run_valgrind.sh
 create mode 100644 cpp/src/plasma/test/serialization_tests.cc
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae.h
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae_epoll.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae_evport.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae_kqueue.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/ae_select.c
 create mode 100644 cpp/src/plasma/thirdparty/ae/config.h
 create mode 100644 cpp/src/plasma/thirdparty/ae/zmalloc.h
 create mode 100644 cpp/src/plasma/thirdparty/dlmalloc.c
 create mode 100644 cpp/src/plasma/thirdparty/xxhash.cc
 create mode 100644 cpp/src/plasma/thirdparty/xxhash.h

diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
new file mode 100644
index 0000000000000..4ff3beba779c2
--- /dev/null
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 2.8)
+
+project(plasma)
+
+find_package(PythonLibsNew REQUIRED)
+find_package(Threads)
+
+option(PLASMA_PYTHON
+  "Build the Plasma Python extensions"
+  OFF)
+
+if(APPLE)
+  SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
+endif(APPLE)
+
+include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
+include_directories("${FLATBUFFERS_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/" "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" "${CMAKE_CURRENT_LIST_DIR}/../")
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L")
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-conversion")
+
+# Compile flatbuffers
+
+set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
+set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
+
+set(PLASMA_FBS_OUTPUT_FILES
+  "${OUTPUT_DIR}/common_generated.h"
+  "${OUTPUT_DIR}/plasma_generated.h")
+
+add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
+
+if(FLATBUFFERS_VENDORED)
+  add_dependencies(gen_plasma_fbs flatbuffers_ep)
+endif()
+
+add_custom_command(
+  OUTPUT ${PLASMA_FBS_OUTPUT_FILES}
+  # The --gen-object-api flag generates a C++ class MessageT for each
+  # flatbuffers message Message, which can be used to store deserialized
+  # messages in data structures. This is currently used for ObjectInfo for
+  # example.
+  COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} --gen-object-api
+  DEPENDS ${PLASMA_FBS_SRC}
+  COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}"
+  VERBATIM)
+
+if(UNIX AND NOT APPLE)
+  link_libraries(rt)
+endif()
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
+set_source_files_properties(extension.cc PROPERTIES COMPILE_FLAGS -Wno-strict-aliasing)
+
+set(PLASMA_SRCS
+  client.cc
+  common.cc
+  eviction_policy.cc
+  events.cc
+  fling.cc
+  io.cc
+  malloc.cc
+  plasma.cc
+  protocol.cc
+  thirdparty/ae/ae.c
+  thirdparty/xxhash.cc)
+
+ADD_ARROW_LIB(plasma
+  SOURCES ${PLASMA_SRCS}
+  DEPENDENCIES gen_plasma_fbs
+  SHARED_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static
+  STATIC_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static)
+
+# The optimization flag -O3 is suggested by dlmalloc.c, which is #included in
+# malloc.cc; we set it here regardless of whether we do a debug or release build.
+set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error -O3")
+
+add_executable(plasma_store store.cc)
+target_link_libraries(plasma_store plasma_static)
+
+ADD_ARROW_TEST(test/serialization_tests)
+ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static)
+ADD_ARROW_TEST(test/client_tests)
+ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static)
+
+if(PLASMA_PYTHON)
+  add_library(plasma_extension SHARED extension.cc)
+
+  if(APPLE)
+    target_link_libraries(plasma_extension plasma_static "-undefined dynamic_lookup")
+  else(APPLE)
+    target_link_libraries(plasma_extension plasma_static -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive)
+  endif(APPLE)
+endif()
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
new file mode 100644
index 0000000000000..dcb78e7ec52c6
--- /dev/null
+++ b/cpp/src/plasma/client.cc
@@ -0,0 +1,557 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// PLASMA CLIENT: Client library for using the plasma store and manager
+
+#include "plasma/client.h"
+
+#ifdef _WIN32
+#include <Win32_Interop/win32_types.h>
+#endif
+
+#include <assert.h>
+#include <fcntl.h>
+#include <netinet/in.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <thread>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/fling.h"
+#include "plasma/io.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+#define XXH_STATIC_LINKING_ONLY
+#include "thirdparty/xxhash.h"
+
+#define XXH64_DEFAULT_SEED 0
+
+// Number of threads used for memcopy and hash computations.
+constexpr int64_t kThreadPoolSize = 8;
+constexpr int64_t kBytesInMB = 1 << 20;
+static std::vector<std::thread> threadpool_(kThreadPoolSize);
+
+// If the file descriptor fd has been mmapped in this client process before,
+// return the pointer that was returned by mmap, otherwise mmap it and store the
+// pointer in a hash table.
+uint8_t* PlasmaClient::lookup_or_mmap(int fd, int store_fd_val, int64_t map_size) {
+  auto entry = mmap_table_.find(store_fd_val);
+  if (entry != mmap_table_.end()) {
+    close(fd);
+    return entry->second.pointer;
+  } else {
+    uint8_t* result = reinterpret_cast<uint8_t*>(
+        mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
+    // TODO(pcm): Don't fail here, instead return a Status.
+    if (result == MAP_FAILED) { ARROW_LOG(FATAL) << "mmap failed"; }
+    close(fd);
+    ClientMmapTableEntry& entry = mmap_table_[store_fd_val];
+    entry.pointer = result;
+    entry.length = map_size;
+    entry.count = 0;
+    return result;
+  }
+}
+
+// Get a pointer to a file that we know has been memory mapped in this client
+// process before.
+uint8_t* PlasmaClient::lookup_mmapped_file(int store_fd_val) {
+  auto entry = mmap_table_.find(store_fd_val);
+  ARROW_CHECK(entry != mmap_table_.end());
+  return entry->second.pointer;
+}
+
+void PlasmaClient::increment_object_count(
+    const ObjectID& object_id, PlasmaObject* object, bool is_sealed) {
+  // Increment the count of the object to track the fact that it is being used.
+  // The corresponding decrement should happen in PlasmaClient::Release.
+  auto elem = objects_in_use_.find(object_id);
+  ObjectInUseEntry* object_entry;
+  if (elem == objects_in_use_.end()) {
+    // Add this object ID to the hash table of object IDs in use. The
+    // corresponding call to free happens in PlasmaClient::Release.
+    objects_in_use_[object_id] =
+        std::unique_ptr<ObjectInUseEntry>(new ObjectInUseEntry());
+    objects_in_use_[object_id]->object = *object;
+    objects_in_use_[object_id]->count = 0;
+    objects_in_use_[object_id]->is_sealed = is_sealed;
+    object_entry = objects_in_use_[object_id].get();
+    // Increment the count of the number of objects in the memory-mapped file
+    // that are being used. The corresponding decrement should happen in
+    // PlasmaClient::Release.
+    auto entry = mmap_table_.find(object->handle.store_fd);
+    ARROW_CHECK(entry != mmap_table_.end());
+    ARROW_CHECK(entry->second.count >= 0);
+    // Update the in_use_object_bytes_.
+    in_use_object_bytes_ +=
+        (object_entry->object.data_size + object_entry->object.metadata_size);
+    entry->second.count += 1;
+  } else {
+    object_entry = elem->second.get();
+    ARROW_CHECK(object_entry->count > 0);
+  }
+  // Increment the count of the number of instances of this object that are
+  // being used by this client. The corresponding decrement should happen in
+  // PlasmaClient::Release.
+  object_entry->count += 1;
+}
+
+Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
+    uint8_t* metadata, int64_t metadata_size, uint8_t** data) {
+  ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
+                   << data_size << " and metadata size " << metadata_size;
+  RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaCreateReply, &buffer));
+  ObjectID id;
+  PlasmaObject object;
+  RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object));
+  // If the CreateReply included an error, then the store will not send a file
+  // descriptor.
+  int fd = recv_fd(store_conn_);
+  ARROW_CHECK(fd >= 0) << "recv not successful";
+  ARROW_CHECK(object.data_size == data_size);
+  ARROW_CHECK(object.metadata_size == metadata_size);
+  // The metadata should come right after the data.
+  ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
+  *data = lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) +
+          object.data_offset;
+  // If plasma_create is being called from a transfer, then we will not copy the
+  // metadata here. The metadata will be written along with the data streamed
+  // from the transfer.
+  if (metadata != NULL) {
+    // Copy the metadata to the buffer.
+    memcpy(*data + object.data_size, metadata, metadata_size);
+  }
+  // Increment the count of the number of instances of this object that this
+  // client is using. A call to PlasmaClient::Release is required to decrement
+  // this
+  // count. Cache the reference to the object.
+  increment_object_count(object_id, &object, false);
+  // We increment the count a second time (and the corresponding decrement will
+  // happen in a PlasmaClient::Release call in plasma_seal) so even if the
+  // buffer
+  // returned by PlasmaClient::Dreate goes out of scope, the object does not get
+  // released before the call to PlasmaClient::Seal happens.
+  increment_object_count(object_id, &object, false);
+  return Status::OK();
+}
+
+Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
+    int64_t timeout_ms, ObjectBuffer* object_buffers) {
+  // Fill out the info for the objects that are already in use locally.
+  bool all_present = true;
+  for (int i = 0; i < num_objects; ++i) {
+    auto object_entry = objects_in_use_.find(object_ids[i]);
+    if (object_entry == objects_in_use_.end()) {
+      // This object is not currently in use by this client, so we need to send
+      // a request to the store.
+      all_present = false;
+      // Make a note to ourselves that the object is not present.
+      object_buffers[i].data_size = -1;
+    } else {
+      // NOTE: If the object is still unsealed, we will deadlock, since we must
+      // have been the one who created it.
+      ARROW_CHECK(object_entry->second->is_sealed)
+          << "Plasma client called get on an unsealed object that it created";
+      PlasmaObject* object = &object_entry->second->object;
+      object_buffers[i].data = lookup_mmapped_file(object->handle.store_fd);
+      object_buffers[i].data = object_buffers[i].data + object->data_offset;
+      object_buffers[i].data_size = object->data_size;
+      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
+      object_buffers[i].metadata_size = object->metadata_size;
+      // Increment the count of the number of instances of this object that this
+      // client is using. A call to PlasmaClient::Release is required to
+      // decrement this
+      // count. Cache the reference to the object.
+      increment_object_count(object_ids[i], object, true);
+    }
+  }
+
+  if (all_present) { return Status::OK(); }
+
+  // If we get here, then the objects aren't all currently in use by this
+  // client, so we need to send a request to the plasma store.
+  RETURN_NOT_OK(SendGetRequest(store_conn_, object_ids, num_objects, timeout_ms));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaGetReply, &buffer));
+  std::vector<ObjectID> received_object_ids(num_objects);
+  std::vector<PlasmaObject> object_data(num_objects);
+  PlasmaObject* object;
+  RETURN_NOT_OK(ReadGetReply(
+      buffer.data(), received_object_ids.data(), object_data.data(), num_objects));
+
+  for (int i = 0; i < num_objects; ++i) {
+    DCHECK(received_object_ids[i] == object_ids[i]);
+    object = &object_data[i];
+    if (object_buffers[i].data_size != -1) {
+      // If the object was already in use by the client, then the store should
+      // have returned it.
+      DCHECK_NE(object->data_size, -1);
+      // We won't use this file descriptor, but the store sent us one, so we
+      // need to receive it and then close it right away so we don't leak file
+      // descriptors.
+      int fd = recv_fd(store_conn_);
+      close(fd);
+      ARROW_CHECK(fd >= 0);
+      // We've already filled out the information for this object, so we can
+      // just continue.
+      continue;
+    }
+    // If we are here, the object was not currently in use, so we need to
+    // process the reply from the object store.
+    if (object->data_size != -1) {
+      // The object was retrieved. The user will be responsible for releasing
+      // this object.
+      int fd = recv_fd(store_conn_);
+      ARROW_CHECK(fd >= 0);
+      object_buffers[i].data =
+          lookup_or_mmap(fd, object->handle.store_fd, object->handle.mmap_size);
+      // Finish filling out the return values.
+      object_buffers[i].data = object_buffers[i].data + object->data_offset;
+      object_buffers[i].data_size = object->data_size;
+      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
+      object_buffers[i].metadata_size = object->metadata_size;
+      // Increment the count of the number of instances of this object that this
+      // client is using. A call to PlasmaClient::Release is required to
+      // decrement this
+      // count. Cache the reference to the object.
+      increment_object_count(received_object_ids[i], object, true);
+    } else {
+      // The object was not retrieved. Make sure we already put a -1 here to
+      // indicate that the object was not retrieved. The caller is not
+      // responsible for releasing this object.
+      DCHECK_EQ(object_buffers[i].data_size, -1);
+      object_buffers[i].data_size = -1;
+    }
+  }
+  return Status::OK();
+}
+
+/// This is a helper method for implementing plasma_release. We maintain a
+/// buffer
+/// of release calls and only perform them once the buffer becomes full (as
+/// judged by the aggregate sizes of the objects). There may be multiple release
+/// calls for the same object ID in the buffer. In this case, the first release
+/// calls will not do anything. The client will only send a message to the store
+/// releasing the object when the client is truly done with the object.
+///
+/// @param conn The plasma connection.
+/// @param object_id The object ID to attempt to release.
+Status PlasmaClient::PerformRelease(const ObjectID& object_id) {
+  // Decrement the count of the number of instances of this object that are
+  // being used by this client. The corresponding increment should have happened
+  // in PlasmaClient::Get.
+  auto object_entry = objects_in_use_.find(object_id);
+  ARROW_CHECK(object_entry != objects_in_use_.end());
+  object_entry->second->count -= 1;
+  ARROW_CHECK(object_entry->second->count >= 0);
+  // Check if the client is no longer using this object.
+  if (object_entry->second->count == 0) {
+    // Decrement the count of the number of objects in this memory-mapped file
+    // that the client is using. The corresponding increment should have
+    // happened in plasma_get.
+    int fd = object_entry->second->object.handle.store_fd;
+    auto entry = mmap_table_.find(fd);
+    ARROW_CHECK(entry != mmap_table_.end());
+    entry->second.count -= 1;
+    ARROW_CHECK(entry->second.count >= 0);
+    // If none are being used then unmap the file.
+    if (entry->second.count == 0) {
+      munmap(entry->second.pointer, entry->second.length);
+      // Remove the corresponding entry from the hash table.
+      mmap_table_.erase(fd);
+    }
+    // Tell the store that the client no longer needs the object.
+    RETURN_NOT_OK(SendReleaseRequest(store_conn_, object_id));
+    // Update the in_use_object_bytes_.
+    in_use_object_bytes_ -= (object_entry->second->object.data_size +
+                             object_entry->second->object.metadata_size);
+    DCHECK_GE(in_use_object_bytes_, 0);
+    // Remove the entry from the hash table of objects currently in use.
+    objects_in_use_.erase(object_id);
+  }
+  return Status::OK();
+}
+
+Status PlasmaClient::Release(const ObjectID& object_id) {
+  // Add the new object to the release history.
+  release_history_.push_front(object_id);
+  // If there are too many bytes in use by the client or if there are too many
+  // pending release calls, and there are at least some pending release calls in
+  // the release_history list, then release some objects.
+  while ((in_use_object_bytes_ > std::min(kL3CacheSizeBytes, store_capacity_ / 100) ||
+             release_history_.size() > config_.release_delay) &&
+         release_history_.size() > 0) {
+    // Perform a release for the object ID for the first pending release.
+    RETURN_NOT_OK(PerformRelease(release_history_.back()));
+    // Remove the last entry from the release history.
+    release_history_.pop_back();
+  }
+  return Status::OK();
+}
+
+// This method is used to query whether the plasma store contains an object.
+Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) {
+  // Check if we already have a reference to the object.
+  if (objects_in_use_.count(object_id) > 0) {
+    *has_object = 1;
+  } else {
+    // If we don't already have a reference to the object, check with the store
+    // to see if we have the object.
+    RETURN_NOT_OK(SendContainsRequest(store_conn_, object_id));
+    std::vector<uint8_t> buffer;
+    RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaContainsReply, &buffer));
+    ObjectID object_id2;
+    RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object));
+  }
+  return Status::OK();
+}
+
+static void ComputeBlockHash(const unsigned char* data, int64_t nbytes, uint64_t* hash) {
+  XXH64_state_t hash_state;
+  XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
+  XXH64_update(&hash_state, data, nbytes);
+  *hash = XXH64_digest(&hash_state);
+}
+
+static inline bool compute_object_hash_parallel(
+    XXH64_state_t* hash_state, const unsigned char* data, int64_t nbytes) {
+  // Note that this function will likely be faster if the address of data is
+  // aligned on a 64-byte boundary.
+  const int num_threads = kThreadPoolSize;
+  uint64_t threadhash[num_threads + 1];
+  const uint64_t data_address = reinterpret_cast<uint64_t>(data);
+  const uint64_t num_blocks = nbytes / BLOCK_SIZE;
+  const uint64_t chunk_size = (num_blocks / num_threads) * BLOCK_SIZE;
+  const uint64_t right_address = data_address + chunk_size * num_threads;
+  const uint64_t suffix = (data_address + nbytes) - right_address;
+  // Now the data layout is | k * num_threads * block_size | suffix | ==
+  // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size.
+  // Each thread gets a "chunk" of k blocks, except the suffix thread.
+
+  for (int i = 0; i < num_threads; i++) {
+    threadpool_[i] = std::thread(ComputeBlockHash,
+        reinterpret_cast<uint8_t*>(data_address) + i * chunk_size, chunk_size,
+        &threadhash[i]);
+  }
+  ComputeBlockHash(
+      reinterpret_cast<uint8_t*>(right_address), suffix, &threadhash[num_threads]);
+
+  // Join the threads.
+  for (auto& t : threadpool_) {
+    if (t.joinable()) { t.join(); }
+  }
+
+  XXH64_update(hash_state, (unsigned char*)threadhash, sizeof(threadhash));
+  return true;
+}
+
+static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
+  XXH64_state_t hash_state;
+  XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
+  if (obj_buffer.data_size >= kBytesInMB) {
+    compute_object_hash_parallel(
+        &hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
+  } else {
+    XXH64_update(&hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
+  }
+  XXH64_update(
+      &hash_state, (unsigned char*)obj_buffer.metadata, obj_buffer.metadata_size);
+  return XXH64_digest(&hash_state);
+}
+
+bool plasma_compute_object_hash(
+    PlasmaClient* conn, ObjectID object_id, unsigned char* digest) {
+  // Get the plasma object data. We pass in a timeout of 0 to indicate that
+  // the operation should timeout immediately.
+  ObjectBuffer object_buffer;
+  ARROW_CHECK_OK(conn->Get(&object_id, 1, 0, &object_buffer));
+  // If the object was not retrieved, return false.
+  if (object_buffer.data_size == -1) { return false; }
+  // Compute the hash.
+  uint64_t hash = compute_object_hash(object_buffer);
+  memcpy(digest, &hash, sizeof(hash));
+  // Release the plasma object.
+  ARROW_CHECK_OK(conn->Release(object_id));
+  return true;
+}
+
+Status PlasmaClient::Seal(const ObjectID& object_id) {
+  // Make sure this client has a reference to the object before sending the
+  // request to Plasma.
+  auto object_entry = objects_in_use_.find(object_id);
+  ARROW_CHECK(object_entry != objects_in_use_.end())
+      << "Plasma client called seal an object without a reference to it";
+  ARROW_CHECK(!object_entry->second->is_sealed)
+      << "Plasma client called seal an already sealed object";
+  object_entry->second->is_sealed = true;
+  /// Send the seal request to Plasma.
+  static unsigned char digest[kDigestSize];
+  ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0]));
+  RETURN_NOT_OK(SendSealRequest(store_conn_, object_id, &digest[0]));
+  // We call PlasmaClient::Release to decrement the number of instances of this
+  // object
+  // that are currently being used by this client. The corresponding increment
+  // happened in plasma_create and was used to ensure that the object was not
+  // released before the call to PlasmaClient::Seal.
+  return Release(object_id);
+}
+
+Status PlasmaClient::Delete(const ObjectID& object_id) {
+  // TODO(rkn): In the future, we can use this method to give hints to the
+  // eviction policy about when an object will no longer be needed.
+  return Status::NotImplemented("PlasmaClient::Delete is not implemented.");
+}
+
+Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) {
+  // Send a request to the store to evict objects.
+  RETURN_NOT_OK(SendEvictRequest(store_conn_, num_bytes));
+  // Wait for a response with the number of bytes actually evicted.
+  std::vector<uint8_t> buffer;
+  int64_t type;
+  RETURN_NOT_OK(ReadMessage(store_conn_, &type, &buffer));
+  return ReadEvictReply(buffer.data(), num_bytes_evicted);
+}
+
+Status PlasmaClient::Subscribe(int* fd) {
+  int sock[2];
+  // Create a non-blocking socket pair. This will only be used to send
+  // notifications from the Plasma store to the client.
+  socketpair(AF_UNIX, SOCK_STREAM, 0, sock);
+  // Make the socket non-blocking.
+  int flags = fcntl(sock[1], F_GETFL, 0);
+  ARROW_CHECK(fcntl(sock[1], F_SETFL, flags | O_NONBLOCK) == 0);
+  // Tell the Plasma store about the subscription.
+  RETURN_NOT_OK(SendSubscribeRequest(store_conn_));
+  // Send the file descriptor that the Plasma store should use to push
+  // notifications about sealed objects to this client.
+  ARROW_CHECK(send_fd(store_conn_, sock[1]) >= 0);
+  close(sock[1]);
+  // Return the file descriptor that the client should use to read notifications
+  // about sealed objects.
+  *fd = sock[0];
+  return Status::OK();
+}
+
+Status PlasmaClient::Connect(const std::string& store_socket_name,
+    const std::string& manager_socket_name, int release_delay) {
+  store_conn_ = connect_ipc_sock_retry(store_socket_name, -1, -1);
+  if (manager_socket_name != "") {
+    manager_conn_ = connect_ipc_sock_retry(manager_socket_name, -1, -1);
+  } else {
+    manager_conn_ = -1;
+  }
+  config_.release_delay = release_delay;
+  in_use_object_bytes_ = 0;
+  // Send a ConnectRequest to the store to get its memory capacity.
+  RETURN_NOT_OK(SendConnectRequest(store_conn_));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaConnectReply, &buffer));
+  RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity_));
+  return Status::OK();
+}
+
+Status PlasmaClient::Disconnect() {
+  // NOTE: We purposefully do not finish sending release calls for objects in
+  // use, so that we don't duplicate PlasmaClient::Release calls (when handling
+  // a SIGTERM, for example).
+
+  // Close the connections to Plasma. The Plasma store will release the objects
+  // that were in use by us when handling the SIGPIPE.
+  close(store_conn_);
+  if (manager_conn_ >= 0) { close(manager_conn_); }
+  return Status::OK();
+}
+
+#define h_addr h_addr_list[0]
+
+Status PlasmaClient::Transfer(const char* address, int port, const ObjectID& object_id) {
+  return SendDataRequest(manager_conn_, object_id, address, port);
+}
+
+Status PlasmaClient::Fetch(int num_object_ids, const ObjectID* object_ids) {
+  ARROW_CHECK(manager_conn_ >= 0);
+  return SendFetchRequest(manager_conn_, object_ids, num_object_ids);
+}
+
+int PlasmaClient::get_manager_fd() {
+  return manager_conn_;
+}
+
+Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
+  ARROW_CHECK(manager_conn_ >= 0);
+
+  RETURN_NOT_OK(SendStatusRequest(manager_conn_, &object_id, 1));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaStatusReply, &buffer));
+  ObjectID id;
+  RETURN_NOT_OK(ReadStatusReply(buffer.data(), &id, object_status, 1));
+  ARROW_CHECK(object_id == id);
+  return Status::OK();
+}
+
+Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_requests,
+    int num_ready_objects, int64_t timeout_ms, int* num_objects_ready) {
+  ARROW_CHECK(manager_conn_ >= 0);
+  ARROW_CHECK(num_object_requests > 0);
+  ARROW_CHECK(num_ready_objects > 0);
+  ARROW_CHECK(num_ready_objects <= num_object_requests);
+
+  for (int i = 0; i < num_object_requests; ++i) {
+    ARROW_CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL ||
+                object_requests[i].type == PLASMA_QUERY_ANYWHERE);
+  }
+
+  RETURN_NOT_OK(SendWaitRequest(manager_conn_, object_requests, num_object_requests,
+      num_ready_objects, timeout_ms));
+  std::vector<uint8_t> buffer;
+  RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaWaitReply, &buffer));
+  RETURN_NOT_OK(ReadWaitReply(buffer.data(), object_requests, &num_ready_objects));
+
+  *num_objects_ready = 0;
+  for (int i = 0; i < num_object_requests; ++i) {
+    int type = object_requests[i].type;
+    int status = object_requests[i].status;
+    switch (type) {
+      case PLASMA_QUERY_LOCAL:
+        if (status == ObjectStatus_Local) { *num_objects_ready += 1; }
+        break;
+      case PLASMA_QUERY_ANYWHERE:
+        if (status == ObjectStatus_Local || status == ObjectStatus_Remote) {
+          *num_objects_ready += 1;
+        } else {
+          ARROW_CHECK(status == ObjectStatus_Nonexistent);
+        }
+        break;
+      default:
+        ARROW_LOG(FATAL) << "This code should be unreachable.";
+    }
+  }
+  return Status::OK();
+}
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
new file mode 100644
index 0000000000000..fb3a161795d47
--- /dev/null
+++ b/cpp/src/plasma/client.h
@@ -0,0 +1,343 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_CLIENT_H
+#define PLASMA_CLIENT_H
+
+#include <stdbool.h>
+#include <time.h>
+
+#include <deque>
+#include <string>
+
+#include "plasma/plasma.h"
+
+using arrow::Status;
+
+#define PLASMA_DEFAULT_RELEASE_DELAY 64
+
+// Use 100MB as an overestimate of the L3 cache size.
+constexpr int64_t kL3CacheSizeBytes = 100000000;
+
+/// Object buffer data structure.
+struct ObjectBuffer {
+  /// The size in bytes of the data object.
+  int64_t data_size;
+  /// The address of the data object.
+  uint8_t* data;
+  /// The metadata size in bytes.
+  int64_t metadata_size;
+  /// The address of the metadata.
+  uint8_t* metadata;
+};
+
+/// Configuration options for the plasma client.
+struct PlasmaClientConfig {
+  /// Number of release calls we wait until the object is actually released.
+  /// This allows us to avoid invalidating the cpu cache on workers if objects
+  /// are reused accross tasks.
+  size_t release_delay;
+};
+
+struct ClientMmapTableEntry {
+  /// The result of mmap for this file descriptor.
+  uint8_t* pointer;
+  /// The length of the memory-mapped file.
+  size_t length;
+  /// The number of objects in this memory-mapped file that are currently being
+  /// used by the client. When this count reaches zeros, we unmap the file.
+  int count;
+};
+
+struct ObjectInUseEntry {
+  /// A count of the number of times this client has called PlasmaClient::Create
+  /// or
+  /// PlasmaClient::Get on this object ID minus the number of calls to
+  /// PlasmaClient::Release.
+  /// When this count reaches zero, we remove the entry from the ObjectsInUse
+  /// and decrement a count in the relevant ClientMmapTableEntry.
+  int count;
+  /// Cached information to read the object.
+  PlasmaObject object;
+  /// A flag representing whether the object has been sealed.
+  bool is_sealed;
+};
+
+class PlasmaClient {
+ public:
+  /// Connect to the local plasma store and plasma manager. Return
+  /// the resulting connection.
+  ///
+  /// @param store_socket_name The name of the UNIX domain socket to use to
+  ///        connect to the Plasma store.
+  /// @param manager_socket_name The name of the UNIX domain socket to use to
+  ///        connect to the local Plasma manager. If this is "", then this
+  ///        function will not connect to a manager.
+  /// @param release_delay Number of released objects that are kept around
+  ///        and not evicted to avoid too many munmaps.
+  /// @return The return status.
+  Status Connect(const std::string& store_socket_name,
+      const std::string& manager_socket_name, int release_delay);
+
+  /// Create an object in the Plasma Store. Any metadata for this object must be
+  /// be passed in when the object is created.
+  ///
+  /// @param object_id The ID to use for the newly created object.
+  /// @param data_size The size in bytes of the space to be allocated for this
+  /// object's
+  ///        data (this does not include space used for metadata).
+  /// @param metadata The object's metadata. If there is no metadata, this
+  /// pointer
+  ///        should be NULL.
+  /// @param metadata_size The size in bytes of the metadata. If there is no
+  ///        metadata, this should be 0.
+  /// @param data The address of the newly created object will be written here.
+  /// @return The return status.
+  Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
+      int64_t metadata_size, uint8_t** data);
+
+  /// Get some objects from the Plasma Store. This function will block until the
+  /// objects have all been created and sealed in the Plasma Store or the
+  /// timeout
+  /// expires. The caller is responsible for releasing any retrieved objects,
+  /// but
+  /// the caller should not release objects that were not retrieved.
+  ///
+  /// @param object_ids The IDs of the objects to get.
+  /// @param num_object_ids The number of object IDs to get.
+  /// @param timeout_ms The amount of time in milliseconds to wait before this
+  ///        request times out. If this value is -1, then no timeout is set.
+  /// @param object_buffers An array where the results will be stored. If the
+  /// data
+  ///        size field is -1, then the object was not retrieved.
+  /// @return The return status.
+  Status Get(const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms,
+      ObjectBuffer* object_buffers);
+
+  /// Tell Plasma that the client no longer needs the object. This should be
+  /// called
+  /// after Get when the client is done with the object. After this call,
+  /// the address returned by Get is no longer valid. This should be called
+  /// once for each call to Get (with the same object ID).
+  ///
+  /// @param object_id The ID of the object that is no longer needed.
+  /// @return The return status.
+  Status Release(const ObjectID& object_id);
+
+  /// Check if the object store contains a particular object and the object has
+  /// been sealed. The result will be stored in has_object.
+  ///
+  /// @todo: We may want to indicate if the object has been created but not
+  /// sealed.
+  ///
+  /// @param object_id The ID of the object whose presence we are checking.
+  /// @param has_object The function will write true at this address if
+  ///        the object is present and false if it is not present.
+  /// @return The return status.
+  Status Contains(const ObjectID& object_id, bool* has_object);
+
+  /// Seal an object in the object store. The object will be immutable after
+  /// this
+  /// call.
+  ///
+  /// @param object_id The ID of the object to seal.
+  /// @return The return status.
+  Status Seal(const ObjectID& object_id);
+
+  /// Delete an object from the object store. This currently assumes that the
+  /// object is present and has been sealed.
+  ///
+  /// @todo We may want to allow the deletion of objects that are not present or
+  ///       haven't been sealed.
+  ///
+  /// @param object_id The ID of the object to delete.
+  /// @return The return status.
+  Status Delete(const ObjectID& object_id);
+
+  /// Delete objects until we have freed up num_bytes bytes or there are no more
+  /// released objects that can be deleted.
+  ///
+  /// @param num_bytes The number of bytes to try to free up.
+  /// @param num_bytes_evicted Out parameter for total number of bytes of space
+  /// retrieved.
+  /// @return The return status.
+  Status Evict(int64_t num_bytes, int64_t& num_bytes_evicted);
+
+  /// Subscribe to notifications when objects are sealed in the object store.
+  /// Whenever an object is sealed, a message will be written to the client
+  /// socket
+  /// that is returned by this method.
+  ///
+  /// @param fd Out parameter for the file descriptor the client should use to
+  /// read notifications
+  ///         from the object store about sealed objects.
+  /// @return The return status.
+  Status Subscribe(int* fd);
+
+  /// Disconnect from the local plasma instance, including the local store and
+  /// manager.
+  ///
+  /// @return The return status.
+  Status Disconnect();
+
+  /// Attempt to initiate the transfer of some objects from remote Plasma
+  /// Stores.
+  /// This method does not guarantee that the fetched objects will arrive
+  /// locally.
+  ///
+  /// For an object that is available in the local Plasma Store, this method
+  /// will
+  /// not do anything. For an object that is not available locally, it will
+  /// check
+  /// if the object are already being fetched. If so, it will not do anything.
+  /// If
+  /// not, it will query the object table for a list of Plasma Managers that
+  /// have
+  /// the object. The object table will return a non-empty list, and this Plasma
+  /// Manager will attempt to initiate transfers from one of those Plasma
+  /// Managers.
+  ///
+  /// This function is non-blocking.
+  ///
+  /// This method is idempotent in the sense that it is ok to call it multiple
+  /// times.
+  ///
+  /// @param num_object_ids The number of object IDs fetch is being called on.
+  /// @param object_ids The IDs of the objects that fetch is being called on.
+  /// @return The return status.
+  Status Fetch(int num_object_ids, const ObjectID* object_ids);
+
+  /// Wait for (1) a specified number of objects to be available (sealed) in the
+  /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to
+  /// expire. This is a blocking call.
+  ///
+  /// @param num_object_requests Size of the object_requests array.
+  /// @param object_requests Object event array. Each element contains a request
+  ///        for a particular object_id. The type of request is specified in the
+  ///        "type" field.
+  ///        - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes
+  ///          available in the local Plasma Store. In this case, this function
+  ///          sets the "status" field to ObjectStatus_Local. Note, if the
+  ///          status
+  ///          is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent,
+  ///          but it may exist elsewhere in the system.
+  ///        - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id
+  ///        becomes
+  ///          available either at the local Plasma Store or on a remote Plasma
+  ///          Store. In this case, the functions sets the "status" field to
+  ///          ObjectStatus_Local or ObjectStatus_Remote.
+  /// @param num_ready_objects The number of requests in object_requests array
+  /// that
+  ///        must be satisfied before the function returns, unless it timeouts.
+  ///        The num_ready_objects should be no larger than num_object_requests.
+  /// @param timeout_ms Timeout value in milliseconds. If this timeout expires
+  ///        before min_num_ready_objects of requests are satisfied, the
+  ///        function
+  ///        returns.
+  /// @param num_objects_ready Out parameter for number of satisfied requests in
+  ///        the object_requests list. If the returned number is less than
+  ///        min_num_ready_objects this means that timeout expired.
+  /// @return The return status.
+  Status Wait(int64_t num_object_requests, ObjectRequest* object_requests,
+      int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
+
+  /// Transfer local object to a different plasma manager.
+  ///
+  /// @param conn The object containing the connection state.
+  /// @param addr IP address of the plasma manager we are transfering to.
+  /// @param port Port of the plasma manager we are transfering to.
+  /// @object_id ObjectID of the object we are transfering.
+  /// @return The return status.
+  Status Transfer(const char* addr, int port, const ObjectID& object_id);
+
+  /// Return the status of a given object. This method may query the object
+  /// table.
+  ///
+  /// @param conn The object containing the connection state.
+  /// @param object_id The ID of the object whose status we query.
+  /// @param object_status Out parameter for object status. Can take the
+  ///         following values.
+  ///         - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma
+  ///         Store.
+  ///           has been already scheduled by the Plasma Manager.
+  ///         - PLASMA_CLIENT_TRANSFER, if the object is either currently being
+  ///           transferred or just scheduled.
+  ///         - PLASMA_CLIENT_REMOTE, if the object is stored at a remote
+  ///           Plasma Store.
+  ///         - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the
+  ///           system.
+  /// @return The return status.
+  Status Info(const ObjectID& object_id, int* object_status);
+
+  /// Get the file descriptor for the socket connection to the plasma manager.
+  ///
+  /// @param conn The plasma connection.
+  /// @return The file descriptor for the manager connection. If there is no
+  ///         connection to the manager, this is -1.
+  int get_manager_fd();
+
+ private:
+  Status PerformRelease(const ObjectID& object_id);
+
+  uint8_t* lookup_or_mmap(int fd, int store_fd_val, int64_t map_size);
+
+  uint8_t* lookup_mmapped_file(int store_fd_val);
+
+  void increment_object_count(
+      const ObjectID& object_id, PlasmaObject* object, bool is_sealed);
+
+  /// File descriptor of the Unix domain socket that connects to the store.
+  int store_conn_;
+  /// File descriptor of the Unix domain socket that connects to the manager.
+  int manager_conn_;
+  /// Table of dlmalloc buffer files that have been memory mapped so far. This
+  /// is a hash table mapping a file descriptor to a struct containing the
+  /// address of the corresponding memory-mapped file.
+  std::unordered_map<int, ClientMmapTableEntry> mmap_table_;
+  /// A hash table of the object IDs that are currently being used by this
+  /// client.
+  std::unordered_map<ObjectID, std::unique_ptr<ObjectInUseEntry>, UniqueIDHasher>
+      objects_in_use_;
+  /// Object IDs of the last few release calls. This is a deque and
+  /// is used to delay releasing objects to see if they can be reused by
+  /// subsequent tasks so we do not unneccessarily invalidate cpu caches.
+  /// TODO(pcm): replace this with a proper lru cache using the size of the L3
+  /// cache.
+  std::deque<ObjectID> release_history_;
+  /// The number of bytes in the combined objects that are held in the release
+  /// history doubly-linked list. If this is too large then the client starts
+  /// releasing objects.
+  int64_t in_use_object_bytes_;
+  /// Configuration options for the plasma client.
+  PlasmaClientConfig config_;
+  /// The amount of memory available to the Plasma store. The client needs this
+  /// information to make sure that it does not delay in releasing so much
+  /// memory that the store is unable to evict enough objects to free up space.
+  int64_t store_capacity_;
+};
+
+/// Compute the hash of an object in the object store.
+///
+/// @param conn The object containing the connection state.
+/// @param object_id The ID of the object we want to hash.
+/// @param digest A pointer at which to return the hash digest of the object.
+///        The pointer must have at least DIGEST_SIZE bytes allocated.
+/// @return A boolean representing whether the hash operation succeeded.
+bool plasma_compute_object_hash(
+    PlasmaClient* conn, ObjectID object_id, unsigned char* digest);
+
+#endif  // PLASMA_CLIENT_H
diff --git a/cpp/src/plasma/common.cc b/cpp/src/plasma/common.cc
new file mode 100644
index 0000000000000..a09a963fa4769
--- /dev/null
+++ b/cpp/src/plasma/common.cc
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/common.h"
+
+#include <random>
+
+#include "format/plasma_generated.h"
+
+using arrow::Status;
+
+UniqueID UniqueID::from_random() {
+  UniqueID id;
+  uint8_t* data = id.mutable_data();
+  std::random_device engine;
+  for (int i = 0; i < kUniqueIDSize; i++) {
+    data[i] = static_cast<uint8_t>(engine());
+  }
+  return id;
+}
+
+UniqueID UniqueID::from_binary(const std::string& binary) {
+  UniqueID id;
+  std::memcpy(&id, binary.data(), sizeof(id));
+  return id;
+}
+
+const uint8_t* UniqueID::data() const {
+  return id_;
+}
+
+uint8_t* UniqueID::mutable_data() {
+  return id_;
+}
+
+std::string UniqueID::binary() const {
+  return std::string(reinterpret_cast<const char*>(id_), kUniqueIDSize);
+}
+
+std::string UniqueID::hex() const {
+  constexpr char hex[] = "0123456789abcdef";
+  std::string result;
+  for (int i = 0; i < kUniqueIDSize; i++) {
+    unsigned int val = id_[i];
+    result.push_back(hex[val >> 4]);
+    result.push_back(hex[val & 0xf]);
+  }
+  return result;
+}
+
+bool UniqueID::operator==(const UniqueID& rhs) const {
+  return std::memcmp(data(), rhs.data(), kUniqueIDSize) == 0;
+}
+
+Status plasma_error_status(int plasma_error) {
+  switch (plasma_error) {
+    case PlasmaError_OK:
+      return Status::OK();
+    case PlasmaError_ObjectExists:
+      return Status::PlasmaObjectExists("object already exists in the plasma store");
+    case PlasmaError_ObjectNonexistent:
+      return Status::PlasmaObjectNonexistent("object does not exist in the plasma store");
+    case PlasmaError_OutOfMemory:
+      return Status::PlasmaStoreFull("object does not fit in the plasma store");
+    default:
+      ARROW_LOG(FATAL) << "unknown plasma error code " << plasma_error;
+  }
+  return Status::OK();
+}
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
new file mode 100644
index 0000000000000..85dc74bf86e0d
--- /dev/null
+++ b/cpp/src/plasma/common.h
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_COMMON_H
+#define PLASMA_COMMON_H
+
+#include <cstring>
+#include <string>
+// TODO(pcm): Convert getopt and sscanf in the store to use more idiomatic C++
+// and get rid of the next three lines:
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+constexpr int64_t kUniqueIDSize = 20;
+
+class UniqueID {
+ public:
+  static UniqueID from_random();
+  static UniqueID from_binary(const std::string& binary);
+  bool operator==(const UniqueID& rhs) const;
+  const uint8_t* data() const;
+  uint8_t* mutable_data();
+  std::string binary() const;
+  std::string hex() const;
+
+ private:
+  uint8_t id_[kUniqueIDSize];
+};
+
+static_assert(std::is_pod<UniqueID>::value, "UniqueID must be plain old data");
+
+struct UniqueIDHasher {
+  // ObjectID hashing function.
+  size_t operator()(const UniqueID& id) const {
+    size_t result;
+    std::memcpy(&result, id.data(), sizeof(size_t));
+    return result;
+  }
+};
+
+typedef UniqueID ObjectID;
+
+arrow::Status plasma_error_status(int plasma_error);
+
+#endif  // PLASMA_COMMON_H
diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
new file mode 100644
index 0000000000000..a9f7356e1f67e
--- /dev/null
+++ b/cpp/src/plasma/events.cc
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/events.h"
+
+#include <errno.h>
+
+void EventLoop::file_event_callback(
+    aeEventLoop* loop, int fd, void* context, int events) {
+  FileCallback* callback = reinterpret_cast<FileCallback*>(context);
+  (*callback)(events);
+}
+
+int EventLoop::timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context) {
+  TimerCallback* callback = reinterpret_cast<TimerCallback*>(context);
+  return (*callback)(timer_id);
+}
+
+constexpr int kInitialEventLoopSize = 1024;
+
+EventLoop::EventLoop() {
+  loop_ = aeCreateEventLoop(kInitialEventLoopSize);
+}
+
+bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback) {
+  if (file_callbacks_.find(fd) != file_callbacks_.end()) { return false; }
+  auto data = std::unique_ptr<FileCallback>(new FileCallback(callback));
+  void* context = reinterpret_cast<void*>(data.get());
+  // Try to add the file descriptor.
+  int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
+  // If it cannot be added, increase the size of the event loop.
+  if (err == AE_ERR && errno == ERANGE) {
+    err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2);
+    if (err != AE_OK) { return false; }
+    err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
+  }
+  // In any case, test if there were errors.
+  if (err == AE_OK) {
+    file_callbacks_.emplace(fd, std::move(data));
+    return true;
+  }
+  return false;
+}
+
+void EventLoop::remove_file_event(int fd) {
+  aeDeleteFileEvent(loop_, fd, AE_READABLE | AE_WRITABLE);
+  file_callbacks_.erase(fd);
+}
+
+void EventLoop::run() {
+  aeMain(loop_);
+}
+
+int64_t EventLoop::add_timer(int64_t timeout, const TimerCallback& callback) {
+  auto data = std::unique_ptr<TimerCallback>(new TimerCallback(callback));
+  void* context = reinterpret_cast<void*>(data.get());
+  int64_t timer_id =
+      aeCreateTimeEvent(loop_, timeout, EventLoop::timer_event_callback, context, NULL);
+  timer_callbacks_.emplace(timer_id, std::move(data));
+  return timer_id;
+}
+
+int EventLoop::remove_timer(int64_t timer_id) {
+  int err = aeDeleteTimeEvent(loop_, timer_id);
+  timer_callbacks_.erase(timer_id);
+  return err;
+}
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
new file mode 100644
index 0000000000000..bd93d6bb2a6fd
--- /dev/null
+++ b/cpp/src/plasma/events.h
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EVENTS
+#define PLASMA_EVENTS
+
+#include <functional>
+#include <memory>
+#include <unordered_map>
+
+extern "C" {
+#include "ae/ae.h"
+}
+
+/// Constant specifying that the timer is done and it will be removed.
+constexpr int kEventLoopTimerDone = AE_NOMORE;
+
+/// Read event on the file descriptor.
+constexpr int kEventLoopRead = AE_READABLE;
+
+/// Write event on the file descriptor.
+constexpr int kEventLoopWrite = AE_WRITABLE;
+
+typedef long long TimerID;  // NOLINT
+
+class EventLoop {
+ public:
+  // Signature of the handler that will be called when there is a new event
+  // on the file descriptor that this handler has been registered for.
+  //
+  // The arguments are the event flags (read or write).
+  using FileCallback = std::function<void(int)>;
+
+  // This handler will be called when a timer times out. The timer id is
+  // passed as an argument. The return is the number of milliseconds the timer
+  // shall be reset to or kEventLoopTimerDone if the timer shall not be
+  // triggered again.
+  using TimerCallback = std::function<int(int64_t)>;
+
+  EventLoop();
+
+  /// Add a new file event handler to the event loop.
+  ///
+  /// @param fd The file descriptor we are listening to.
+  /// @param events The flags for events we are listening to (read or write).
+  /// @param callback The callback that will be called when the event happens.
+  /// @return Returns true if the event handler was added successfully.
+  bool add_file_event(int fd, int events, const FileCallback& callback);
+
+  /// Remove a file event handler from the event loop.
+  ///
+  /// @param fd The file descriptor of the event handler.
+  /// @return Void.
+  void remove_file_event(int fd);
+
+  /// Register a handler that will be called after a time slice of
+  ///  "timeout" milliseconds.
+  ///
+  ///  @param timeout The timeout in milliseconds.
+  ///  @param callback The callback for the timeout.
+  ///  @return The ID of the newly created timer.
+  int64_t add_timer(int64_t timeout, const TimerCallback& callback);
+
+  /// Remove a timer handler from the event loop.
+  ///
+  /// @param timer_id The ID of the timer that is to be removed.
+  /// @return The ae.c error code. TODO(pcm): needs to be standardized
+  int remove_timer(int64_t timer_id);
+
+  /// Run the event loop.
+  ///
+  /// @return Void.
+  void run();
+
+ private:
+  static void file_event_callback(aeEventLoop* loop, int fd, void* context, int events);
+
+  static int timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context);
+
+  aeEventLoop* loop_;
+  std::unordered_map<int, std::unique_ptr<FileCallback>> file_callbacks_;
+  std::unordered_map<int64_t, std::unique_ptr<TimerCallback>> timer_callbacks_;
+};
+
+#endif  // PLASMA_EVENTS
diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
new file mode 100644
index 0000000000000..4ae6384d42543
--- /dev/null
+++ b/cpp/src/plasma/eviction_policy.cc
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/eviction_policy.h"
+
+#include <algorithm>
+
+void LRUCache::add(const ObjectID& key, int64_t size) {
+  auto it = item_map_.find(key);
+  ARROW_CHECK(it == item_map_.end());
+  /* Note that it is important to use a list so the iterators stay valid. */
+  item_list_.emplace_front(key, size);
+  item_map_.emplace(key, item_list_.begin());
+}
+
+void LRUCache::remove(const ObjectID& key) {
+  auto it = item_map_.find(key);
+  ARROW_CHECK(it != item_map_.end());
+  item_list_.erase(it->second);
+  item_map_.erase(it);
+}
+
+int64_t LRUCache::choose_objects_to_evict(
+    int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
+  int64_t bytes_evicted = 0;
+  auto it = item_list_.end();
+  while (bytes_evicted < num_bytes_required && it != item_list_.begin()) {
+    it--;
+    objects_to_evict->push_back(it->first);
+    bytes_evicted += it->second;
+  }
+  return bytes_evicted;
+}
+
+EvictionPolicy::EvictionPolicy(PlasmaStoreInfo* store_info)
+    : memory_used_(0), store_info_(store_info) {}
+
+int64_t EvictionPolicy::choose_objects_to_evict(
+    int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
+  int64_t bytes_evicted =
+      cache_.choose_objects_to_evict(num_bytes_required, objects_to_evict);
+  /* Update the LRU cache. */
+  for (auto& object_id : *objects_to_evict) {
+    cache_.remove(object_id);
+  }
+  /* Update the number of bytes used. */
+  memory_used_ -= bytes_evicted;
+  return bytes_evicted;
+}
+
+void EvictionPolicy::object_created(const ObjectID& object_id) {
+  auto entry = store_info_->objects[object_id].get();
+  cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
+}
+
+bool EvictionPolicy::require_space(
+    int64_t size, std::vector<ObjectID>* objects_to_evict) {
+  /* Check if there is enough space to create the object. */
+  int64_t required_space = memory_used_ + size - store_info_->memory_capacity;
+  int64_t num_bytes_evicted;
+  if (required_space > 0) {
+    /* Try to free up at least as much space as we need right now but ideally
+     * up to 20% of the total capacity. */
+    int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5);
+    ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects";
+    /* Choose some objects to evict, and update the return pointers. */
+    num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict);
+    ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting "
+                    << objects_to_evict->size() << " objects to free up "
+                    << num_bytes_evicted << " bytes.";
+  } else {
+    num_bytes_evicted = 0;
+  }
+  if (num_bytes_evicted >= required_space) {
+    /* We only increment the space used if there is enough space to create the
+     * object. */
+    memory_used_ += size;
+  }
+  return num_bytes_evicted >= required_space;
+}
+
+void EvictionPolicy::begin_object_access(
+    const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
+  /* If the object is in the LRU cache, remove it. */
+  cache_.remove(object_id);
+}
+
+void EvictionPolicy::end_object_access(
+    const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
+  auto entry = store_info_->objects[object_id].get();
+  /* Add the object to the LRU cache.*/
+  cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
+}
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
new file mode 100644
index 0000000000000..3815fc6652f0c
--- /dev/null
+++ b/cpp/src/plasma/eviction_policy.h
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EVICTION_POLICY_H
+#define PLASMA_EVICTION_POLICY_H
+
+#include <list>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/plasma.h"
+
+// ==== The eviction policy ====
+//
+// This file contains declaration for all functions and data structures that
+// need to be provided if you want to implement a new eviction algorithm for the
+// Plasma store.
+
+class LRUCache {
+ public:
+  LRUCache() {}
+
+  void add(const ObjectID& key, int64_t size);
+
+  void remove(const ObjectID& key);
+
+  int64_t choose_objects_to_evict(
+      int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
+
+ private:
+  /// A doubly-linked list containing the items in the cache and
+  /// their sizes in LRU order.
+  typedef std::list<std::pair<ObjectID, int64_t>> ItemList;
+  ItemList item_list_;
+  /// A hash table mapping the object ID of an object in the cache to its
+  /// location in the doubly linked list item_list_.
+  std::unordered_map<ObjectID, ItemList::iterator, UniqueIDHasher> item_map_;
+};
+
+/// The eviction policy.
+class EvictionPolicy {
+ public:
+  /// Construct an eviction policy.
+  ///
+  /// @param store_info Information about the Plasma store that is exposed
+  ///        to the eviction policy.
+  explicit EvictionPolicy(PlasmaStoreInfo* store_info);
+
+  /// This method will be called whenever an object is first created in order to
+  /// add it to the LRU cache. This is done so that the first time, the Plasma
+  /// store calls begin_object_access, we can remove the object from the LRU
+  /// cache.
+  ///
+  /// @param object_id The object ID of the object that was created.
+  /// @return Void.
+  void object_created(const ObjectID& object_id);
+
+  /// This method will be called when the Plasma store needs more space, perhaps
+  /// to create a new object. If the required amount of space cannot be freed up,
+  /// then a fatal error will be thrown. When this method is called, the eviction
+  /// policy will assume that the objects chosen to be evicted will in fact be
+  /// evicted from the Plasma store by the caller.
+  ///
+  /// @param size The size in bytes of the new object, including both data and
+  ///        metadata.
+  /// @param objects_to_evict The object IDs that were chosen for eviction will
+  ///        be stored into this vector.
+  /// @return True if enough space can be freed and false otherwise.
+  bool require_space(int64_t size, std::vector<ObjectID>* objects_to_evict);
+
+  /// This method will be called whenever an unused object in the Plasma store
+  /// starts to be used. When this method is called, the eviction policy will
+  /// assume that the objects chosen to be evicted will in fact be evicted from
+  /// the Plasma store by the caller.
+  ///
+  /// @param object_id The ID of the object that is now being used.
+  /// @param objects_to_evict The object IDs that were chosen for eviction will
+  ///        be stored into this vector.
+  /// @return Void.
+  void begin_object_access(
+      const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
+
+  /// This method will be called whenever an object in the Plasma store that was
+  /// being used is no longer being used. When this method is called, the
+  /// eviction policy will assume that the objects chosen to be evicted will in
+  /// fact be evicted from the Plasma store by the caller.
+  ///
+  /// @param object_id The ID of the object that is no longer being used.
+  /// @param objects_to_evict The object IDs that were chosen for eviction will
+  ///        be stored into this vector.
+  /// @return Void.
+  void end_object_access(
+      const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
+
+  /// Choose some objects to evict from the Plasma store. When this method is
+  /// called, the eviction policy will assume that the objects chosen to be
+  /// evicted will in fact be evicted from the Plasma store by the caller.
+  ///
+  /// @note This method is not part of the API. It is exposed in the header file
+  /// only for testing.
+  ///
+  /// @param num_bytes_required The number of bytes of space to try to free up.
+  /// @param objects_to_evict The object IDs that were chosen for eviction will
+  ///        be stored into this vector.
+  /// @return The total number of bytes of space chosen to be evicted.
+  int64_t choose_objects_to_evict(
+      int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
+
+ private:
+  /// The amount of memory (in bytes) currently being used.
+  int64_t memory_used_;
+  /// Pointer to the plasma store info.
+  PlasmaStoreInfo* store_info_;
+  /// Datastructure for the LRU cache.
+  LRUCache cache_;
+};
+
+#endif  // PLASMA_EVICTION_POLICY_H
diff --git a/cpp/src/plasma/extension.cc b/cpp/src/plasma/extension.cc
new file mode 100644
index 0000000000000..5d61e337c108d
--- /dev/null
+++ b/cpp/src/plasma/extension.cc
@@ -0,0 +1,456 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/extension.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+#include "plasma/io.h"
+#include "plasma/protocol.h"
+
+PyObject* PlasmaOutOfMemoryError;
+PyObject* PlasmaObjectExistsError;
+
+PyObject* PyPlasma_connect(PyObject* self, PyObject* args) {
+  const char* store_socket_name;
+  const char* manager_socket_name;
+  int release_delay;
+  if (!PyArg_ParseTuple(
+          args, "ssi", &store_socket_name, &manager_socket_name, &release_delay)) {
+    return NULL;
+  }
+  PlasmaClient* client = new PlasmaClient();
+  ARROW_CHECK_OK(client->Connect(store_socket_name, manager_socket_name, release_delay));
+
+  return PyCapsule_New(client, "plasma", NULL);
+}
+
+PyObject* PyPlasma_disconnect(PyObject* self, PyObject* args) {
+  PyObject* client_capsule;
+  if (!PyArg_ParseTuple(args, "O", &client_capsule)) { return NULL; }
+  PlasmaClient* client;
+  ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client));
+  ARROW_CHECK_OK(client->Disconnect());
+  /* We use the context of the connection capsule to indicate if the connection
+   * is still active (if the context is NULL) or if it is closed (if the context
+   * is (void*) 0x1). This is neccessary because the primary pointer of the
+   * capsule cannot be NULL. */
+  PyCapsule_SetContext(client_capsule, reinterpret_cast<void*>(0x1));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_create(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  Py_ssize_t size;
+  PyObject* metadata;
+  if (!PyArg_ParseTuple(args, "O&O&nO", PyObjectToPlasmaClient, &client,
+          PyStringToUniqueID, &object_id, &size, &metadata)) {
+    return NULL;
+  }
+  if (!PyByteArray_Check(metadata)) {
+    PyErr_SetString(PyExc_TypeError, "metadata must be a bytearray");
+    return NULL;
+  }
+  uint8_t* data;
+  Status s = client->Create(object_id, size,
+      reinterpret_cast<uint8_t*>(PyByteArray_AsString(metadata)),
+      PyByteArray_Size(metadata), &data);
+  if (s.IsPlasmaObjectExists()) {
+    PyErr_SetString(PlasmaObjectExistsError,
+        "An object with this ID already exists in the plasma "
+        "store.");
+    return NULL;
+  }
+  if (s.IsPlasmaStoreFull()) {
+    PyErr_SetString(PlasmaOutOfMemoryError,
+        "The plasma store ran out of memory and could not create "
+        "this object.");
+    return NULL;
+  }
+  ARROW_CHECK(s.ok());
+
+#if PY_MAJOR_VERSION >= 3
+  return PyMemoryView_FromMemory(reinterpret_cast<char*>(data), size, PyBUF_WRITE);
+#else
+  return PyBuffer_FromReadWriteMemory(reinterpret_cast<void*>(data), size);
+#endif
+}
+
+PyObject* PyPlasma_hash(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  unsigned char digest[kDigestSize];
+  bool success = plasma_compute_object_hash(client, object_id, digest);
+  if (success) {
+    PyObject* digest_string =
+        PyBytes_FromStringAndSize(reinterpret_cast<char*>(digest), kDigestSize);
+    return digest_string;
+  } else {
+    Py_RETURN_NONE;
+  }
+}
+
+PyObject* PyPlasma_seal(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  ARROW_CHECK_OK(client->Seal(object_id));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_release(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  ARROW_CHECK_OK(client->Release(object_id));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_get(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  PyObject* object_id_list;
+  Py_ssize_t timeout_ms;
+  if (!PyArg_ParseTuple(
+          args, "O&On", PyObjectToPlasmaClient, &client, &object_id_list, &timeout_ms)) {
+    return NULL;
+  }
+
+  Py_ssize_t num_object_ids = PyList_Size(object_id_list);
+  std::vector<ObjectID> object_ids(num_object_ids);
+  std::vector<ObjectBuffer> object_buffers(num_object_ids);
+
+  for (int i = 0; i < num_object_ids; ++i) {
+    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
+  }
+
+  Py_BEGIN_ALLOW_THREADS;
+  ARROW_CHECK_OK(
+      client->Get(object_ids.data(), num_object_ids, timeout_ms, object_buffers.data()));
+  Py_END_ALLOW_THREADS;
+
+  PyObject* returns = PyList_New(num_object_ids);
+  for (int i = 0; i < num_object_ids; ++i) {
+    if (object_buffers[i].data_size != -1) {
+      /* The object was retrieved, so return the object. */
+      PyObject* t = PyTuple_New(2);
+      Py_ssize_t data_size = static_cast<Py_ssize_t>(object_buffers[i].data_size);
+      Py_ssize_t metadata_size = static_cast<Py_ssize_t>(object_buffers[i].metadata_size);
+#if PY_MAJOR_VERSION >= 3
+      char* data = reinterpret_cast<char*>(object_buffers[i].data);
+      char* metadata = reinterpret_cast<char*>(object_buffers[i].metadata);
+      PyTuple_SET_ITEM(t, 0, PyMemoryView_FromMemory(data, data_size, PyBUF_READ));
+      PyTuple_SET_ITEM(
+          t, 1, PyMemoryView_FromMemory(metadata, metadata_size, PyBUF_READ));
+#else
+      void* data = reinterpret_cast<void*>(object_buffers[i].data);
+      void* metadata = reinterpret_cast<void*>(object_buffers[i].metadata);
+      PyTuple_SET_ITEM(t, 0, PyBuffer_FromMemory(data, data_size));
+      PyTuple_SET_ITEM(t, 1, PyBuffer_FromMemory(metadata, metadata_size));
+#endif
+      ARROW_CHECK(PyList_SetItem(returns, i, t) == 0);
+    } else {
+      /* The object was not retrieved, so just add None to the list of return
+       * values. */
+      Py_INCREF(Py_None);
+      ARROW_CHECK(PyList_SetItem(returns, i, Py_None) == 0);
+    }
+  }
+  return returns;
+}
+
+PyObject* PyPlasma_contains(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  bool has_object;
+  ARROW_CHECK_OK(client->Contains(object_id, &has_object));
+
+  if (has_object) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
+PyObject* PyPlasma_fetch(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  PyObject* object_id_list;
+  if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client, &object_id_list)) {
+    return NULL;
+  }
+  if (client->get_manager_fd() == -1) {
+    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+    return NULL;
+  }
+  Py_ssize_t n = PyList_Size(object_id_list);
+  ObjectID* object_ids = new ObjectID[n];
+  for (int i = 0; i < n; ++i) {
+    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
+  }
+  ARROW_CHECK_OK(client->Fetch(static_cast<int>(n), object_ids));
+  delete[] object_ids;
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_wait(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  PyObject* object_id_list;
+  Py_ssize_t timeout;
+  int num_returns;
+  if (!PyArg_ParseTuple(args, "O&Oni", PyObjectToPlasmaClient, &client, &object_id_list,
+          &timeout, &num_returns)) {
+    return NULL;
+  }
+  Py_ssize_t n = PyList_Size(object_id_list);
+
+  if (client->get_manager_fd() == -1) {
+    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+    return NULL;
+  }
+  if (num_returns < 0) {
+    PyErr_SetString(
+        PyExc_RuntimeError, "The argument num_returns cannot be less than zero.");
+    return NULL;
+  }
+  if (num_returns > n) {
+    PyErr_SetString(PyExc_RuntimeError,
+        "The argument num_returns cannot be greater than len(object_ids)");
+    return NULL;
+  }
+  int64_t threshold = 1 << 30;
+  if (timeout > threshold) {
+    PyErr_SetString(
+        PyExc_RuntimeError, "The argument timeout cannot be greater than 2 ** 30.");
+    return NULL;
+  }
+
+  std::vector<ObjectRequest> object_requests(n);
+  for (int i = 0; i < n; ++i) {
+    ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i),
+                    &object_requests[i].object_id) == 1);
+    object_requests[i].type = PLASMA_QUERY_ANYWHERE;
+  }
+  /* Drop the global interpreter lock while we are waiting, so other threads can
+   * run. */
+  int num_return_objects;
+  Py_BEGIN_ALLOW_THREADS;
+  ARROW_CHECK_OK(
+      client->Wait(n, object_requests.data(), num_returns, timeout, &num_return_objects));
+  Py_END_ALLOW_THREADS;
+
+  int num_to_return = std::min(num_return_objects, num_returns);
+  PyObject* ready_ids = PyList_New(num_to_return);
+  PyObject* waiting_ids = PySet_New(object_id_list);
+  int num_returned = 0;
+  for (int i = 0; i < n; ++i) {
+    if (num_returned == num_to_return) { break; }
+    if (object_requests[i].status == ObjectStatus_Local ||
+        object_requests[i].status == ObjectStatus_Remote) {
+      PyObject* ready = PyBytes_FromStringAndSize(
+          reinterpret_cast<char*>(&object_requests[i].object_id),
+          sizeof(object_requests[i].object_id));
+      PyList_SetItem(ready_ids, num_returned, ready);
+      PySet_Discard(waiting_ids, ready);
+      num_returned += 1;
+    } else {
+      ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent);
+    }
+  }
+  ARROW_CHECK(num_returned == num_to_return);
+  /* Return both the ready IDs and the remaining IDs. */
+  PyObject* t = PyTuple_New(2);
+  PyTuple_SetItem(t, 0, ready_ids);
+  PyTuple_SetItem(t, 1, waiting_ids);
+  return t;
+}
+
+PyObject* PyPlasma_evict(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  Py_ssize_t num_bytes;
+  if (!PyArg_ParseTuple(args, "O&n", PyObjectToPlasmaClient, &client, &num_bytes)) {
+    return NULL;
+  }
+  int64_t evicted_bytes;
+  ARROW_CHECK_OK(client->Evict(static_cast<int64_t>(num_bytes), evicted_bytes));
+  return PyLong_FromSsize_t(static_cast<Py_ssize_t>(evicted_bytes));
+}
+
+PyObject* PyPlasma_delete(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+          &object_id)) {
+    return NULL;
+  }
+  ARROW_CHECK_OK(client->Delete(object_id));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_transfer(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  ObjectID object_id;
+  const char* addr;
+  int port;
+  if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client,
+          PyStringToUniqueID, &object_id, &addr, &port)) {
+    return NULL;
+  }
+
+  if (client->get_manager_fd() == -1) {
+    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+    return NULL;
+  }
+
+  ARROW_CHECK_OK(client->Transfer(addr, port, object_id));
+  Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_subscribe(PyObject* self, PyObject* args) {
+  PlasmaClient* client;
+  if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) { return NULL; }
+
+  int sock;
+  ARROW_CHECK_OK(client->Subscribe(&sock));
+  return PyLong_FromLong(sock);
+}
+
+PyObject* PyPlasma_receive_notification(PyObject* self, PyObject* args) {
+  int plasma_sock;
+
+  if (!PyArg_ParseTuple(args, "i", &plasma_sock)) { return NULL; }
+  /* Receive object notification from the plasma connection socket. If the
+   * object was added, return a tuple of its fields: ObjectID, data_size,
+   * metadata_size. If the object was deleted, data_size and metadata_size will
+   * be set to -1. */
+  uint8_t* notification = read_message_async(plasma_sock);
+  if (notification == NULL) {
+    PyErr_SetString(
+        PyExc_RuntimeError, "Failed to read object notification from Plasma socket");
+    return NULL;
+  }
+  auto object_info = flatbuffers::GetRoot<ObjectInfo>(notification);
+  /* Construct a tuple from object_info and return. */
+  PyObject* t = PyTuple_New(3);
+  PyTuple_SetItem(t, 0, PyBytes_FromStringAndSize(object_info->object_id()->data(),
+                            object_info->object_id()->size()));
+  if (object_info->is_deletion()) {
+    PyTuple_SetItem(t, 1, PyLong_FromLong(-1));
+    PyTuple_SetItem(t, 2, PyLong_FromLong(-1));
+  } else {
+    PyTuple_SetItem(t, 1, PyLong_FromLong(object_info->data_size()));
+    PyTuple_SetItem(t, 2, PyLong_FromLong(object_info->metadata_size()));
+  }
+
+  delete[] notification;
+  return t;
+}
+
+static PyMethodDef plasma_methods[] = {
+    {"connect", PyPlasma_connect, METH_VARARGS, "Connect to plasma."},
+    {"disconnect", PyPlasma_disconnect, METH_VARARGS, "Disconnect from plasma."},
+    {"create", PyPlasma_create, METH_VARARGS, "Create a new plasma object."},
+    {"hash", PyPlasma_hash, METH_VARARGS, "Compute the hash of a plasma object."},
+    {"seal", PyPlasma_seal, METH_VARARGS, "Seal a plasma object."},
+    {"get", PyPlasma_get, METH_VARARGS, "Get a plasma object."},
+    {"contains", PyPlasma_contains, METH_VARARGS,
+        "Does the plasma store contain this plasma object?"},
+    {"fetch", PyPlasma_fetch, METH_VARARGS,
+        "Fetch the object from another plasma manager instance."},
+    {"wait", PyPlasma_wait, METH_VARARGS,
+        "Wait until num_returns objects in object_ids are ready."},
+    {"evict", PyPlasma_evict, METH_VARARGS,
+        "Evict some objects until we recover some number of bytes."},
+    {"release", PyPlasma_release, METH_VARARGS, "Release the plasma object."},
+    {"delete", PyPlasma_delete, METH_VARARGS, "Delete a plasma object."},
+    {"transfer", PyPlasma_transfer, METH_VARARGS,
+        "Transfer object to another plasma manager."},
+    {"subscribe", PyPlasma_subscribe, METH_VARARGS,
+        "Subscribe to the plasma notification socket."},
+    {"receive_notification", PyPlasma_receive_notification, METH_VARARGS,
+        "Receive next notification from plasma notification socket."},
+    {NULL} /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT, "libplasma",    /* m_name */
+    "A Python client library for plasma.", /* m_doc */
+    0,                                     /* m_size */
+    plasma_methods,                        /* m_methods */
+    NULL,                                  /* m_reload */
+    NULL,                                  /* m_traverse */
+    NULL,                                  /* m_clear */
+    NULL,                                  /* m_free */
+};
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define INITERROR return NULL
+#else
+#define INITERROR return
+#endif
+
+#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
+#define PyMODINIT_FUNC void
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
+#else
+#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
+#endif
+
+MOD_INIT(libplasma) {
+#if PY_MAJOR_VERSION >= 3
+  PyObject* m = PyModule_Create(&moduledef);
+#else
+  PyObject* m =
+      Py_InitModule3("libplasma", plasma_methods, "A Python client library for plasma.");
+#endif
+
+  /* Create a custom exception for when an object ID is reused. */
+  char plasma_object_exists_error[] = "plasma_object_exists.error";
+  PlasmaObjectExistsError = PyErr_NewException(plasma_object_exists_error, NULL, NULL);
+  Py_INCREF(PlasmaObjectExistsError);
+  PyModule_AddObject(m, "plasma_object_exists_error", PlasmaObjectExistsError);
+  /* Create a custom exception for when the plasma store is out of memory. */
+  char plasma_out_of_memory_error[] = "plasma_out_of_memory.error";
+  PlasmaOutOfMemoryError = PyErr_NewException(plasma_out_of_memory_error, NULL, NULL);
+  Py_INCREF(PlasmaOutOfMemoryError);
+  PyModule_AddObject(m, "plasma_out_of_memory_error", PlasmaOutOfMemoryError);
+
+#if PY_MAJOR_VERSION >= 3
+  return m;
+#endif
+}
diff --git a/cpp/src/plasma/extension.h b/cpp/src/plasma/extension.h
new file mode 100644
index 0000000000000..cee30abb3592d
--- /dev/null
+++ b/cpp/src/plasma/extension.h
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EXTENSION_H
+#define PLASMA_EXTENSION_H
+
+#undef _XOPEN_SOURCE
+#undef _POSIX_C_SOURCE
+#include <Python.h>
+
+#include "bytesobject.h"  // NOLINT
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+
+static int PyObjectToPlasmaClient(PyObject* object, PlasmaClient** client) {
+  if (PyCapsule_IsValid(object, "plasma")) {
+    *client = reinterpret_cast<PlasmaClient*>(PyCapsule_GetPointer(object, "plasma"));
+    return 1;
+  } else {
+    PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule");
+    return 0;
+  }
+}
+
+int PyStringToUniqueID(PyObject* object, ObjectID* object_id) {
+  if (PyBytes_Check(object)) {
+    memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID));
+    return 1;
+  } else {
+    PyErr_SetString(PyExc_TypeError, "must be a 20 character string");
+    return 0;
+  }
+}
+
+#endif  // PLASMA_EXTENSION_H
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
new file mode 100644
index 0000000000000..79da4f43a192a
--- /dev/null
+++ b/cpp/src/plasma/fling.cc
@@ -0,0 +1,90 @@
+// Copyright 2013 Sharvil Nanavati
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "plasma/fling.h"
+
+#include <string.h>
+
+void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len) {
+  iov->iov_base = buf;
+  iov->iov_len = 1;
+
+  msg->msg_iov = iov;
+  msg->msg_iovlen = 1;
+  msg->msg_control = buf;
+  msg->msg_controllen = buf_len;
+  msg->msg_name = NULL;
+  msg->msg_namelen = 0;
+}
+
+int send_fd(int conn, int fd) {
+  struct msghdr msg;
+  struct iovec iov;
+  char buf[CMSG_SPACE(sizeof(int))];
+  memset(&buf, 0, CMSG_SPACE(sizeof(int)));
+
+  init_msg(&msg, &iov, buf, sizeof(buf));
+
+  struct cmsghdr* header = CMSG_FIRSTHDR(&msg);
+  header->cmsg_level = SOL_SOCKET;
+  header->cmsg_type = SCM_RIGHTS;
+  header->cmsg_len = CMSG_LEN(sizeof(int));
+  *reinterpret_cast<int*>(CMSG_DATA(header)) = fd;
+
+  // Send file descriptor.
+  ssize_t r = sendmsg(conn, &msg, 0);
+  if (r >= 0) {
+    return 0;
+  } else {
+    return static_cast<int>(r);
+  }
+}
+
+int recv_fd(int conn) {
+  struct msghdr msg;
+  struct iovec iov;
+  char buf[CMSG_SPACE(sizeof(int))];
+  init_msg(&msg, &iov, buf, sizeof(buf));
+
+  if (recvmsg(conn, &msg, 0) == -1) return -1;
+
+  int found_fd = -1;
+  int oh_noes = 0;
+  for (struct cmsghdr* header = CMSG_FIRSTHDR(&msg); header != NULL;
+       header = CMSG_NXTHDR(&msg, header))
+    if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) {
+      ssize_t count =
+          (header->cmsg_len - (CMSG_DATA(header) - (unsigned char*)header)) / sizeof(int);
+      for (int i = 0; i < count; ++i) {
+        int fd = (reinterpret_cast<int*>(CMSG_DATA(header)))[i];
+        if (found_fd == -1) {
+          found_fd = fd;
+        } else {
+          close(fd);
+          oh_noes = 1;
+        }
+      }
+    }
+
+  // The sender sent us more than one file descriptor. We've closed
+  // them all to prevent fd leaks but notify the caller that we got
+  // a bad message.
+  if (oh_noes) {
+    close(found_fd);
+    errno = EBADMSG;
+    return -1;
+  }
+
+  return found_fd;
+}
diff --git a/cpp/src/plasma/fling.h b/cpp/src/plasma/fling.h
new file mode 100644
index 0000000000000..78ac9d17f26fb
--- /dev/null
+++ b/cpp/src/plasma/fling.h
@@ -0,0 +1,52 @@
+// Copyright 2013 Sharvil Nanavati
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// FLING: Exchanging file descriptors over sockets
+//
+// This is a little library for sending file descriptors over a socket
+// between processes. The reason for doing that (as opposed to using
+// filenames to share the files) is so (a) no files remain in the
+// filesystem after all the processes terminate, (b) to make sure that
+// there are no name collisions and (c) to be able to control who has
+// access to the data.
+//
+// Most of the code is from https://github.com/sharvil/flingfd
+
+#include <errno.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+// This is neccessary for Mac OS X, see http://www.apuebook.com/faqs2e.html
+// (10).
+#if !defined(CMSG_SPACE) && !defined(CMSG_LEN)
+#define CMSG_SPACE(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + __DARWIN_ALIGN32(len))
+#define CMSG_LEN(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + (len))
+#endif
+
+void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len);
+
+// Send a file descriptor over a unix domain socket.
+//
+// @param conn Unix domain socket to send the file descriptor over.
+// @param fd File descriptor to send over.
+// @return Status code which is < 0 on failure.
+int send_fd(int conn, int fd);
+
+// Receive a file descriptor over a unix domain socket.
+//
+// @param conn Unix domain socket to receive the file descriptor from.
+// @return File descriptor or a value < 0 on failure.
+int recv_fd(int conn);
diff --git a/cpp/src/plasma/format/.gitignore b/cpp/src/plasma/format/.gitignore
new file mode 100644
index 0000000000000..b2ddb055dcbc6
--- /dev/null
+++ b/cpp/src/plasma/format/.gitignore
@@ -0,0 +1 @@
+*_generated.h
diff --git a/cpp/src/plasma/format/common.fbs b/cpp/src/plasma/format/common.fbs
new file mode 100644
index 0000000000000..4d7d2852aec3d
--- /dev/null
+++ b/cpp/src/plasma/format/common.fbs
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Object information data structure.
+table ObjectInfo {
+  // Object ID of this object.
+  object_id: string;
+  // Number of bytes the content of this object occupies in memory.
+  data_size: long;
+  // Number of bytes the metadata of this object occupies in memory.
+  metadata_size: long;
+  // Unix epoch of when this object was created.
+  create_time: long;
+  // How long creation of this object took.
+  construct_duration: long;
+  // Hash of the object content.
+  digest: string;
+  // Specifies if this object was deleted or added.
+  is_deletion: bool;
+}
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
new file mode 100644
index 0000000000000..23782ade539d4
--- /dev/null
+++ b/cpp/src/plasma/format/plasma.fbs
@@ -0,0 +1,291 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Plasma protocol specification
+
+enum MessageType:int {
+  // Create a new object.
+  PlasmaCreateRequest = 1,
+  PlasmaCreateReply,
+  // Seal an object.
+  PlasmaSealRequest,
+  PlasmaSealReply,
+  // Get an object that is stored on the local Plasma store.
+  PlasmaGetRequest,
+  PlasmaGetReply,
+  // Release an object.
+  PlasmaReleaseRequest,
+  PlasmaReleaseReply,
+  // Delete an object.
+  PlasmaDeleteRequest,
+  PlasmaDeleteReply,
+  // Get status of an object.
+  PlasmaStatusRequest,
+  PlasmaStatusReply,
+  // See if the store contains an object (will be deprecated).
+  PlasmaContainsRequest,
+  PlasmaContainsReply,
+  // Get information for a newly connecting client.
+  PlasmaConnectRequest,
+  PlasmaConnectReply,
+  // Make room for new objects in the plasma store.
+  PlasmaEvictRequest,
+  PlasmaEvictReply,
+  // Fetch objects from remote Plasma stores.
+  PlasmaFetchRequest,
+  // Wait for objects to be ready either from local or remote Plasma stores.
+  PlasmaWaitRequest,
+  PlasmaWaitReply,
+  // Subscribe to a list of objects or to all objects.
+  PlasmaSubscribeRequest,
+  // Unsubscribe.
+  PlasmaUnsubscribeRequest,
+  // Sending and receiving data.
+  // PlasmaDataRequest initiates sending the data, there will be one
+  // such message per data transfer.
+  PlasmaDataRequest,
+  // PlasmaDataReply contains the actual data and is sent back to the
+  // object store that requested the data. For each transfer, multiple
+  // reply messages get sent. Each one contains a fixed number of bytes.
+  PlasmaDataReply,
+  // Object notifications.
+  PlasmaNotification
+}
+
+enum PlasmaError:int {
+  // Operation was successful.
+  OK,
+  // Trying to create an object that already exists.
+  ObjectExists,
+  // Trying to access an object that doesn't exist.
+  ObjectNonexistent,
+  // Trying to create an object but there isn't enough space in the store.
+  OutOfMemory
+}
+
+// Plasma store messages
+
+struct PlasmaObjectSpec {
+  // Index of the memory segment (= memory mapped file) that
+  // this object is allocated in.
+  segment_index: int;
+  // Size in bytes of this segment (needed to call mmap).
+  mmap_size: ulong;
+  // The offset in bytes in the memory mapped file of the data.
+  data_offset: ulong;
+  // The size in bytes of the data.
+  data_size: ulong;
+  // The offset in bytes in the memory mapped file of the metadata.
+  metadata_offset: ulong;
+  // The size in bytes of the metadata.
+  metadata_size: ulong;
+}
+
+table PlasmaCreateRequest {
+  // ID of the object to be created.
+  object_id: string;
+  // The size of the object's data in bytes.
+  data_size: ulong;
+  // The size of the object's metadata in bytes.
+  metadata_size: ulong;
+}
+
+table PlasmaCreateReply {
+  // ID of the object that was created.
+  object_id: string;
+  // The object that is returned with this reply.
+  plasma_object: PlasmaObjectSpec;
+  // Error that occurred for this call.
+  error: PlasmaError;
+}
+
+table PlasmaSealRequest {
+  // ID of the object to be sealed.
+  object_id: string;
+  // Hash of the object data.
+  digest: string;
+}
+
+table PlasmaSealReply {
+  // ID of the object that was sealed.
+  object_id: string;
+  // Error code.
+  error: PlasmaError;
+}
+
+table PlasmaGetRequest {
+  // IDs of the objects stored at local Plasma store we are getting.
+  object_ids: [string];
+  // The number of milliseconds before the request should timeout.
+  timeout_ms: long;
+}
+
+table PlasmaGetReply {
+  // IDs of the objects being returned.
+  // This number can be smaller than the number of requested
+  // objects if not all requested objects are stored and sealed
+  // in the local Plasma store.
+  object_ids: [string];
+  // Plasma object information, in the same order as their IDs.
+  plasma_objects: [PlasmaObjectSpec];
+  // The number of elements in both object_ids and plasma_objects arrays must agree.
+}
+
+table PlasmaReleaseRequest {
+  // ID of the object to be released.
+  object_id: string;
+}
+
+table PlasmaReleaseReply {
+  // ID of the object that was released.
+  object_id: string;
+  // Error code.
+  error: PlasmaError;
+}
+
+table PlasmaDeleteRequest {
+  // ID of the object to be deleted.
+  object_id: string;
+}
+
+table PlasmaDeleteReply {
+  // ID of the object that was deleted.
+  object_id: string;
+  // Error code.
+  error: PlasmaError;
+}
+
+table PlasmaStatusRequest {
+  // IDs of the objects stored at local Plasma store we request the status of.
+  object_ids: [string];
+}
+
+enum ObjectStatus:int {
+  // Object is stored in the local Plasma Store.
+  Local = 1,
+  // Object is stored on a remote Plasma store, and it is not stored on the
+  // local Plasma Store.
+  Remote,
+  // Object is not stored in the system.
+  Nonexistent,
+  // Object is currently transferred from a remote Plasma store the the local
+  // Plasma Store.
+  Transfer
+}
+
+table PlasmaStatusReply {
+  // IDs of the objects being returned.
+  object_ids: [string];
+  // Status of the object.
+  status: [ObjectStatus];
+}
+
+// PlasmaContains is a subset of PlasmaStatus which does not
+// involve the plasma manager, only the store. We should consider
+// unifying them in the future and deprecating PlasmaContains.
+
+table PlasmaContainsRequest {
+  // ID of the object we are querying.
+  object_id: string;
+}
+
+table PlasmaContainsReply {
+  // ID of the object we are querying.
+  object_id: string;
+  // 1 if the object is in the store and 0 otherwise.
+  has_object: int;
+}
+
+// PlasmaConnect is used by a plasma client the first time it connects with the
+// store. This is not really necessary, but is used to get some information
+// about the store such as its memory capacity.
+
+table PlasmaConnectRequest {
+}
+
+table PlasmaConnectReply {
+  // The memory capacity of the store.
+  memory_capacity: long;
+}
+
+table PlasmaEvictRequest {
+  // Number of bytes that shall be freed.
+  num_bytes: ulong;
+}
+
+table PlasmaEvictReply {
+  // Number of bytes that have been freed.
+  num_bytes: ulong;
+}
+
+table PlasmaFetchRequest {
+  // IDs of objects to be gotten.
+  object_ids: [string];
+}
+
+table ObjectRequestSpec {
+  // ID of the object.
+  object_id: string;
+  // The type of the object. This specifies whether we
+  // will be waiting for an object store in the local or
+  // global Plasma store.
+  type: int;
+}
+
+table PlasmaWaitRequest {
+  // Array of object requests whose status we are asking for.
+  object_requests: [ObjectRequestSpec];
+  // Number of objects expected to be returned, if available.
+  num_ready_objects: int;
+  // timeout
+  timeout: long;
+}
+
+table ObjectReply {
+  // ID of the object.
+  object_id: string;
+  // The object status. This specifies where the object is stored.
+  status: int;
+}
+
+table PlasmaWaitReply {
+  // Array of object requests being returned.
+  object_requests: [ObjectReply];
+  // Number of objects expected to be returned, if available.
+  num_ready_objects: int;
+}
+
+table PlasmaSubscribeRequest {
+}
+
+table PlasmaDataRequest {
+  // ID of the object that is requested.
+  object_id: string;
+  // The host address where the data shall be sent to.
+  address: string;
+  // The port of the manager the data shall be sent to.
+  port: int;
+}
+
+table PlasmaDataReply {
+  // ID of the object that will be sent.
+  object_id: string;
+  // Size of the object data in bytes.
+  object_size: ulong;
+  // Size of the metadata in bytes.
+  metadata_size: ulong;
+}
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
new file mode 100644
index 0000000000000..5875ebb7ae611
--- /dev/null
+++ b/cpp/src/plasma/io.cc
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/io.h"
+
+#include "plasma/common.h"
+
+using arrow::Status;
+
+/* Number of times we try binding to a socket. */
+#define NUM_BIND_ATTEMPTS 5
+#define BIND_TIMEOUT_MS 100
+
+/* Number of times we try connecting to a socket. */
+#define NUM_CONNECT_ATTEMPTS 50
+#define CONNECT_TIMEOUT_MS 100
+
+Status WriteBytes(int fd, uint8_t* cursor, size_t length) {
+  ssize_t nbytes = 0;
+  size_t bytesleft = length;
+  size_t offset = 0;
+  while (bytesleft > 0) {
+    /* While we haven't written the whole message, write to the file descriptor,
+     * advance the cursor, and decrease the amount left to write. */
+    nbytes = write(fd, cursor + offset, bytesleft);
+    if (nbytes < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
+      return Status::IOError(std::string(strerror(errno)));
+    } else if (nbytes == 0) {
+      return Status::IOError("Encountered unexpected EOF");
+    }
+    ARROW_CHECK(nbytes > 0);
+    bytesleft -= nbytes;
+    offset += nbytes;
+  }
+
+  return Status::OK();
+}
+
+Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes) {
+  int64_t version = PLASMA_PROTOCOL_VERSION;
+  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)));
+  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&type), sizeof(type)));
+  RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)));
+  return WriteBytes(fd, bytes, length * sizeof(char));
+}
+
+Status ReadBytes(int fd, uint8_t* cursor, size_t length) {
+  ssize_t nbytes = 0;
+  /* Termination condition: EOF or read 'length' bytes total. */
+  size_t bytesleft = length;
+  size_t offset = 0;
+  while (bytesleft > 0) {
+    nbytes = read(fd, cursor + offset, bytesleft);
+    if (nbytes < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
+      return Status::IOError(std::string(strerror(errno)));
+    } else if (0 == nbytes) {
+      return Status::IOError("Encountered unexpected EOF");
+    }
+    ARROW_CHECK(nbytes > 0);
+    bytesleft -= nbytes;
+    offset += nbytes;
+  }
+
+  return Status::OK();
+}
+
+Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
+  int64_t version;
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)),
+      *type = DISCONNECT_CLIENT);
+  ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version;
+  size_t length;
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(type), sizeof(*type)),
+      *type = DISCONNECT_CLIENT);
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)),
+      *type = DISCONNECT_CLIENT);
+  if (length > buffer->size()) { buffer->resize(length); }
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer->data(), length), *type = DISCONNECT_CLIENT);
+  return Status::OK();
+}
+
+int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
+  struct sockaddr_un socket_address;
+  int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (socket_fd < 0) {
+    ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
+    return -1;
+  }
+  /* Tell the system to allow the port to be reused. */
+  int on = 1;
+  if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&on),
+          sizeof(on)) < 0) {
+    ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname;
+    close(socket_fd);
+    return -1;
+  }
+
+  unlink(pathname.c_str());
+  memset(&socket_address, 0, sizeof(socket_address));
+  socket_address.sun_family = AF_UNIX;
+  if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
+    ARROW_LOG(ERROR) << "Socket pathname is too long.";
+    close(socket_fd);
+    return -1;
+  }
+  strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
+
+  if (bind(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) != 0) {
+    ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname;
+    close(socket_fd);
+    return -1;
+  }
+  if (shall_listen && listen(socket_fd, 128) == -1) {
+    ARROW_LOG(ERROR) << "Could not listen to socket " << pathname;
+    close(socket_fd);
+    return -1;
+  }
+  return socket_fd;
+}
+
+int connect_ipc_sock_retry(
+    const std::string& pathname, int num_retries, int64_t timeout) {
+  /* Pick the default values if the user did not specify. */
+  if (num_retries < 0) { num_retries = NUM_CONNECT_ATTEMPTS; }
+  if (timeout < 0) { timeout = CONNECT_TIMEOUT_MS; }
+
+  int fd = -1;
+  for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
+    fd = connect_ipc_sock(pathname);
+    if (fd >= 0) { break; }
+    if (num_attempts == 0) {
+      ARROW_LOG(ERROR) << "Connection to socket failed for pathname " << pathname;
+    }
+    /* Sleep for timeout milliseconds. */
+    usleep(static_cast<int>(timeout * 1000));
+  }
+  /* If we could not connect to the socket, exit. */
+  if (fd == -1) { ARROW_LOG(FATAL) << "Could not connect to socket " << pathname; }
+  return fd;
+}
+
+int connect_ipc_sock(const std::string& pathname) {
+  struct sockaddr_un socket_address;
+  int socket_fd;
+
+  socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (socket_fd < 0) {
+    ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
+    return -1;
+  }
+
+  memset(&socket_address, 0, sizeof(socket_address));
+  socket_address.sun_family = AF_UNIX;
+  if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
+    ARROW_LOG(ERROR) << "Socket pathname is too long.";
+    return -1;
+  }
+  strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
+
+  if (connect(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) !=
+      0) {
+    close(socket_fd);
+    return -1;
+  }
+
+  return socket_fd;
+}
+
+int AcceptClient(int socket_fd) {
+  int client_fd = accept(socket_fd, NULL, NULL);
+  if (client_fd < 0) {
+    ARROW_LOG(ERROR) << "Error reading from socket.";
+    return -1;
+  }
+  return client_fd;
+}
+
+uint8_t* read_message_async(int sock) {
+  int64_t size;
+  Status s = ReadBytes(sock, reinterpret_cast<uint8_t*>(&size), sizeof(int64_t));
+  if (!s.ok()) {
+    /* The other side has closed the socket. */
+    ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
+    close(sock);
+    return NULL;
+  }
+  uint8_t* message = reinterpret_cast<uint8_t*>(malloc(size));
+  s = ReadBytes(sock, message, size);
+  if (!s.ok()) {
+    /* The other side has closed the socket. */
+    ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
+    close(sock);
+    return NULL;
+  }
+  return message;
+}
diff --git a/cpp/src/plasma/io.h b/cpp/src/plasma/io.h
new file mode 100644
index 0000000000000..43c3fb535497f
--- /dev/null
+++ b/cpp/src/plasma/io.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_IO_H
+#define PLASMA_IO_H
+
+#include <inttypes.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <string>
+#include <vector>
+
+#include "arrow/status.h"
+
+// TODO(pcm): Replace our own custom message header (message type,
+// message length, plasma protocol verion) with one that is serialized
+// using flatbuffers.
+#define PLASMA_PROTOCOL_VERSION 0x0000000000000000
+#define DISCONNECT_CLIENT 0
+
+arrow::Status WriteBytes(int fd, uint8_t* cursor, size_t length);
+
+arrow::Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes);
+
+arrow::Status ReadBytes(int fd, uint8_t* cursor, size_t length);
+
+arrow::Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer);
+
+int bind_ipc_sock(const std::string& pathname, bool shall_listen);
+
+int connect_ipc_sock(const std::string& pathname);
+
+int connect_ipc_sock_retry(const std::string& pathname, int num_retries, int64_t timeout);
+
+int AcceptClient(int socket_fd);
+
+uint8_t* read_message_async(int sock);
+
+#endif  // PLASMA_IO_H
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
new file mode 100644
index 0000000000000..97c9a16c0c0bd
--- /dev/null
+++ b/cpp/src/plasma/malloc.cc
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/malloc.h"
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <unordered_map>
+
+#include "plasma/common.h"
+
+extern "C" {
+void* fake_mmap(size_t);
+int fake_munmap(void*, int64_t);
+
+#define MMAP(s) fake_mmap(s)
+#define MUNMAP(a, s) fake_munmap(a, s)
+#define DIRECT_MMAP(s) fake_mmap(s)
+#define DIRECT_MUNMAP(a, s) fake_munmap(a, s)
+#define USE_DL_PREFIX
+#define HAVE_MORECORE 0
+#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
+#define DEFAULT_GRANULARITY ((size_t)128U * 1024U)
+
+#include "thirdparty/dlmalloc.c"  // NOLINT
+
+#undef MMAP
+#undef MUNMAP
+#undef DIRECT_MMAP
+#undef DIRECT_MUNMAP
+#undef USE_DL_PREFIX
+#undef HAVE_MORECORE
+#undef DEFAULT_GRANULARITY
+}
+
+struct mmap_record {
+  int fd;
+  int64_t size;
+};
+
+namespace {
+
+/** Hashtable that contains one entry per segment that we got from the OS
+ *  via mmap. Associates the address of that segment with its file descriptor
+ *  and size. */
+std::unordered_map<void*, mmap_record> mmap_records;
+
+} /* namespace */
+
+constexpr int GRANULARITY_MULTIPLIER = 2;
+
+static void* pointer_advance(void* p, ptrdiff_t n) {
+  return (unsigned char*)p + n;
+}
+
+static void* pointer_retreat(void* p, ptrdiff_t n) {
+  return (unsigned char*)p - n;
+}
+
+static ptrdiff_t pointer_distance(void const* pfrom, void const* pto) {
+  return (unsigned char const*)pto - (unsigned char const*)pfrom;
+}
+
+/* Create a buffer. This is creating a temporary file and then
+ * immediately unlinking it so we do not leave traces in the system. */
+int create_buffer(int64_t size) {
+  int fd;
+#ifdef _WIN32
+  if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
+          (DWORD)((uint64_t)size >> (CHAR_BIT * sizeof(DWORD))), (DWORD)(uint64_t)size,
+          NULL)) {
+    fd = -1;
+  }
+#else
+#ifdef __linux__
+  constexpr char file_template[] = "/dev/shm/plasmaXXXXXX";
+#else
+  constexpr char file_template[] = "/tmp/plasmaXXXXXX";
+#endif
+  char file_name[32];
+  strncpy(file_name, file_template, 32);
+  fd = mkstemp(file_name);
+  if (fd < 0) return -1;
+  FILE* file = fdopen(fd, "a+");
+  if (!file) {
+    close(fd);
+    return -1;
+  }
+  if (unlink(file_name) != 0) {
+    ARROW_LOG(FATAL) << "unlink error";
+    return -1;
+  }
+  if (ftruncate(fd, (off_t)size) != 0) {
+    ARROW_LOG(FATAL) << "ftruncate error";
+    return -1;
+  }
+#endif
+  return fd;
+}
+
+void* fake_mmap(size_t size) {
+  /* Add sizeof(size_t) so that the returned pointer is deliberately not
+   * page-aligned. This ensures that the segments of memory returned by
+   * fake_mmap are never contiguous. */
+  size += sizeof(size_t);
+
+  int fd = create_buffer(size);
+  ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
+  void* pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+  if (pointer == MAP_FAILED) { return pointer; }
+
+  /* Increase dlmalloc's allocation granularity directly. */
+  mparams.granularity *= GRANULARITY_MULTIPLIER;
+
+  mmap_record& record = mmap_records[pointer];
+  record.fd = fd;
+  record.size = size;
+
+  /* We lie to dlmalloc about where mapped memory actually lives. */
+  pointer = pointer_advance(pointer, sizeof(size_t));
+  ARROW_LOG(DEBUG) << pointer << " = fake_mmap(" << size << ")";
+  return pointer;
+}
+
+int fake_munmap(void* addr, int64_t size) {
+  ARROW_LOG(DEBUG) << "fake_munmap(" << addr << ", " << size << ")";
+  addr = pointer_retreat(addr, sizeof(size_t));
+  size += sizeof(size_t);
+
+  auto entry = mmap_records.find(addr);
+
+  if (entry == mmap_records.end() || entry->second.size != size) {
+    /* Reject requests to munmap that don't directly match previous
+     * calls to mmap, to prevent dlmalloc from trimming. */
+    return -1;
+  }
+
+  int r = munmap(addr, size);
+  if (r == 0) { close(entry->second.fd); }
+
+  mmap_records.erase(entry);
+  return r;
+}
+
+void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_size, ptrdiff_t* offset) {
+  /* TODO(rshin): Implement a more efficient search through mmap_records. */
+  for (const auto& entry : mmap_records) {
+    if (addr >= entry.first && addr < pointer_advance(entry.first, entry.second.size)) {
+      *fd = entry.second.fd;
+      *map_size = entry.second.size;
+      *offset = pointer_distance(entry.first, addr);
+      return;
+    }
+  }
+  *fd = -1;
+  *map_size = 0;
+  *offset = 0;
+}
diff --git a/cpp/src/plasma/malloc.h b/cpp/src/plasma/malloc.h
new file mode 100644
index 0000000000000..b4af2c826b5c9
--- /dev/null
+++ b/cpp/src/plasma/malloc.h
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_MALLOC_H
+#define PLASMA_MALLOC_H
+
+#include <inttypes.h>
+#include <stddef.h>
+
+void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_length, ptrdiff_t* offset);
+
+#endif  // MALLOC_H
diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc
new file mode 100644
index 0000000000000..559d8e7f2a65e
--- /dev/null
+++ b/cpp/src/plasma/plasma.cc
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/plasma.h"
+
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/common.h"
+#include "plasma/protocol.h"
+
+int warn_if_sigpipe(int status, int client_sock) {
+  if (status >= 0) { return 0; }
+  if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
+    ARROW_LOG(WARNING) << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when "
+                          "sending a message to client on fd "
+                       << client_sock << ". The client on the other end may "
+                                         "have hung up.";
+    return errno;
+  }
+  ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock << ".";
+  return -1;  // This is never reached.
+}
+
+/**
+ * This will create a new ObjectInfo buffer. The first sizeof(int64_t) bytes
+ * of this buffer are the length of the remaining message and the
+ * remaining message is a serialized version of the object info.
+ *
+ * @param object_info The object info to be serialized
+ * @return The object info buffer. It is the caller's responsibility to free
+ *         this buffer with "delete" after it has been used.
+ */
+uint8_t* create_object_info_buffer(ObjectInfoT* object_info) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreateObjectInfo(fbb, object_info);
+  fbb.Finish(message);
+  uint8_t* notification = new uint8_t[sizeof(int64_t) + fbb.GetSize()];
+  *(reinterpret_cast<int64_t*>(notification)) = fbb.GetSize();
+  memcpy(notification + sizeof(int64_t), fbb.GetBufferPointer(), fbb.GetSize());
+  return notification;
+}
+
+ObjectTableEntry* get_object_table_entry(
+    PlasmaStoreInfo* store_info, const ObjectID& object_id) {
+  auto it = store_info->objects.find(object_id);
+  if (it == store_info->objects.end()) { return NULL; }
+  return it->second.get();
+}
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
new file mode 100644
index 0000000000000..275d0c7a41687
--- /dev/null
+++ b/cpp/src/plasma/plasma.h
@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_PLASMA_H
+#define PLASMA_PLASMA_H
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>  // pid_t
+
+#include <unordered_map>
+#include <unordered_set>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+#include "format/common_generated.h"
+#include "plasma/common.h"
+
+#define HANDLE_SIGPIPE(s, fd_)                                              \
+  do {                                                                      \
+    Status _s = (s);                                                        \
+    if (!_s.ok()) {                                                         \
+      if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {        \
+        ARROW_LOG(WARNING)                                                  \
+            << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \
+               "sending a message to client on fd "                         \
+            << fd_ << ". "                                                  \
+                      "The client on the other end may have hung up.";      \
+      } else {                                                              \
+        return _s;                                                          \
+      }                                                                     \
+    }                                                                       \
+  } while (0);
+
+/// Allocation granularity used in plasma for object allocation.
+#define BLOCK_SIZE 64
+
+/// Size of object hash digests.
+constexpr int64_t kDigestSize = sizeof(uint64_t);
+
+struct Client;
+
+/// Object request data structure. Used in the plasma_wait_for_objects()
+/// argument.
+typedef struct {
+  /// The ID of the requested object. If ID_NIL request any object.
+  ObjectID object_id;
+  /// Request associated to the object. It can take one of the following values:
+  ///  - PLASMA_QUERY_LOCAL: return if or when the object is available in the
+  ///    local Plasma Store.
+  ///  - PLASMA_QUERY_ANYWHERE: return if or when the object is available in
+  ///    the system (i.e., either in the local or a remote Plasma Store).
+  int type;
+  /// Object status. Same as the status returned by plasma_status() function
+  /// call. This is filled in by plasma_wait_for_objects1():
+  ///  - ObjectStatus_Local: object is ready at the local Plasma Store.
+  ///  - ObjectStatus_Remote: object is ready at a remote Plasma Store.
+  ///  - ObjectStatus_Nonexistent: object does not exist in the system.
+  ///  - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
+  ///    for being transferred or it is transferring.
+  int status;
+} ObjectRequest;
+
+/// Mapping from object IDs to type and status of the request.
+typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectRequestMap;
+
+/// Handle to access memory mapped file and map it into client address space.
+typedef struct {
+  /// The file descriptor of the memory mapped file in the store. It is used as
+  /// a unique identifier of the file in the client to look up the corresponding
+  /// file descriptor on the client's side.
+  int store_fd;
+  /// The size in bytes of the memory mapped file.
+  int64_t mmap_size;
+} object_handle;
+
+// TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
+typedef struct {
+  /// Handle for memory mapped file the object is stored in.
+  object_handle handle;
+  /// The offset in bytes in the memory mapped file of the data.
+  ptrdiff_t data_offset;
+  /// The offset in bytes in the memory mapped file of the metadata.
+  ptrdiff_t metadata_offset;
+  /// The size in bytes of the data.
+  int64_t data_size;
+  /// The size in bytes of the metadata.
+  int64_t metadata_size;
+} PlasmaObject;
+
+typedef enum {
+  /// Object was created but not sealed in the local Plasma Store.
+  PLASMA_CREATED = 1,
+  /// Object is sealed and stored in the local Plasma Store.
+  PLASMA_SEALED
+} object_state;
+
+typedef enum {
+  /// The object was not found.
+  OBJECT_NOT_FOUND = 0,
+  /// The object was found.
+  OBJECT_FOUND = 1
+} object_status;
+
+typedef enum {
+  /// Query for object in the local plasma store.
+  PLASMA_QUERY_LOCAL = 1,
+  /// Query for object in the local plasma store or in a remote plasma store.
+  PLASMA_QUERY_ANYWHERE
+} object_request_type;
+
+/// This type is used by the Plasma store. It is here because it is exposed to
+/// the eviction policy.
+struct ObjectTableEntry {
+  /// Object id of this object.
+  ObjectID object_id;
+  /// Object info like size, creation time and owner.
+  ObjectInfoT info;
+  /// Memory mapped file containing the object.
+  int fd;
+  /// Size of the underlying map.
+  int64_t map_size;
+  /// Offset from the base of the mmap.
+  ptrdiff_t offset;
+  /// Pointer to the object data. Needed to free the object.
+  uint8_t* pointer;
+  /// Set of clients currently using this object.
+  std::unordered_set<Client*> clients;
+  /// The state of the object, e.g., whether it is open or sealed.
+  object_state state;
+  /// The digest of the object. Used to see if two objects are the same.
+  unsigned char digest[kDigestSize];
+};
+
+/// The plasma store information that is exposed to the eviction policy.
+struct PlasmaStoreInfo {
+  /// Objects that are in the Plasma store.
+  std::unordered_map<ObjectID, std::unique_ptr<ObjectTableEntry>, UniqueIDHasher> objects;
+  /// The amount of memory (in bytes) that we allow to be allocated in the
+  /// store.
+  int64_t memory_capacity;
+};
+
+/// Get an entry from the object table and return NULL if the object_id
+/// is not present.
+///
+/// @param store_info The PlasmaStoreInfo that contains the object table.
+/// @param object_id The object_id of the entry we are looking for.
+/// @return The entry associated with the object_id or NULL if the object_id
+///         is not present.
+ObjectTableEntry* get_object_table_entry(
+    PlasmaStoreInfo* store_info, const ObjectID& object_id);
+
+/// Print a warning if the status is less than zero. This should be used to check
+/// the success of messages sent to plasma clients. We print a warning instead of
+/// failing because the plasma clients are allowed to die. This is used to handle
+/// situations where the store writes to a client file descriptor, and the client
+/// may already have disconnected. If we have processed the disconnection and
+/// closed the file descriptor, we should get a BAD FILE DESCRIPTOR error. If we
+/// have not, then we should get a SIGPIPE. If we write to a TCP socket that
+/// isn't connected yet, then we should get an ECONNRESET.
+///
+/// @param status The status to check. If it is less less than zero, we will
+///        print a warning.
+/// @param client_sock The client socket. This is just used to print some extra
+///        information.
+/// @return The errno set.
+int warn_if_sigpipe(int status, int client_sock);
+
+uint8_t* create_object_info_buffer(ObjectInfoT* object_info);
+
+#endif  // PLASMA_PLASMA_H
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
new file mode 100644
index 0000000000000..246aa29736056
--- /dev/null
+++ b/cpp/src/plasma/protocol.cc
@@ -0,0 +1,502 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/protocol.h"
+
+#include "flatbuffers/flatbuffers.h"
+#include "format/plasma_generated.h"
+
+#include "plasma/common.h"
+#include "plasma/io.h"
+
+using flatbuffers::uoffset_t;
+
+flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
+to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
+    int64_t num_objects) {
+  std::vector<flatbuffers::Offset<flatbuffers::String>> results;
+  for (int64_t i = 0; i < num_objects; i++) {
+    results.push_back(fbb->CreateString(object_ids[i].binary()));
+  }
+  return fbb->CreateVector(results);
+}
+
+Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer) {
+  int64_t type;
+  RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
+  ARROW_CHECK(type == message_type) << "type = " << type
+                                    << ", message_type = " << message_type;
+  return Status::OK();
+}
+
+template <typename Message>
+Status PlasmaSend(int sock, int64_t message_type, flatbuffers::FlatBufferBuilder* fbb,
+    const Message& message) {
+  fbb->Finish(message);
+  return WriteMessage(sock, message_type, fbb->GetSize(), fbb->GetBufferPointer());
+}
+
+// Create messages.
+
+Status SendCreateRequest(
+    int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaCreateRequest(
+      fbb, fbb.CreateString(object_id.binary()), data_size, metadata_size);
+  return PlasmaSend(sock, MessageType_PlasmaCreateRequest, &fbb, message);
+}
+
+Status ReadCreateRequest(
+    uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
+  *data_size = message->data_size();
+  *metadata_size = message->metadata_size();
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendCreateReply(
+    int sock, ObjectID object_id, PlasmaObject* object, int error_code) {
+  flatbuffers::FlatBufferBuilder fbb;
+  PlasmaObjectSpec plasma_object(object->handle.store_fd, object->handle.mmap_size,
+      object->data_offset, object->data_size, object->metadata_offset,
+      object->metadata_size);
+  auto message = CreatePlasmaCreateReply(
+      fbb, fbb.CreateString(object_id.binary()), &plasma_object, (PlasmaError)error_code);
+  return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
+}
+
+Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  object->handle.store_fd = message->plasma_object()->segment_index();
+  object->handle.mmap_size = message->plasma_object()->mmap_size();
+  object->data_offset = message->plasma_object()->data_offset();
+  object->data_size = message->plasma_object()->data_size();
+  object->metadata_offset = message->plasma_object()->metadata_offset();
+  object->metadata_size = message->plasma_object()->metadata_size();
+  return plasma_error_status(message->error());
+}
+
+// Seal messages.
+
+Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto digest_string = fbb.CreateString(reinterpret_cast<char*>(digest), kDigestSize);
+  auto message =
+      CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()), digest_string);
+  return PlasmaSend(sock, MessageType_PlasmaSealRequest, &fbb, message);
+}
+
+Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaSealRequest>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  ARROW_CHECK(message->digest()->size() == kDigestSize);
+  memcpy(digest, message->digest()->data(), kDigestSize);
+  return Status::OK();
+}
+
+Status SendSealReply(int sock, ObjectID object_id, int error) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaSealReply(
+      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  return PlasmaSend(sock, MessageType_PlasmaSealReply, &fbb, message);
+}
+
+Status ReadSealReply(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaSealReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return plasma_error_status(message->error());
+}
+
+// Release messages.
+
+Status SendReleaseRequest(int sock, ObjectID object_id) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()));
+  return PlasmaSend(sock, MessageType_PlasmaReleaseRequest, &fbb, message);
+}
+
+Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaReleaseRequest>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendReleaseReply(int sock, ObjectID object_id, int error) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaReleaseReply(
+      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  return PlasmaSend(sock, MessageType_PlasmaReleaseReply, &fbb, message);
+}
+
+Status ReadReleaseReply(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return plasma_error_status(message->error());
+}
+
+// Delete messages.
+
+Status SendDeleteRequest(int sock, ObjectID object_id) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaDeleteRequest(fbb, fbb.CreateString(object_id.binary()));
+  return PlasmaSend(sock, MessageType_PlasmaDeleteRequest, &fbb, message);
+}
+
+Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendDeleteReply(int sock, ObjectID object_id, int error) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaDeleteReply(
+      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  return PlasmaSend(sock, MessageType_PlasmaDeleteReply, &fbb, message);
+}
+
+Status ReadDeleteReply(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaDeleteReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return plasma_error_status(message->error());
+}
+
+// Satus messages.
+
+Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message =
+      CreatePlasmaStatusRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
+  return PlasmaSend(sock, MessageType_PlasmaStatusRequest, &fbb, message);
+}
+
+Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaStatusRequest>(data);
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+  }
+  return Status::OK();
+}
+
+Status SendStatusReply(
+    int sock, ObjectID object_ids[], int object_status[], int64_t num_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message =
+      CreatePlasmaStatusReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+          fbb.CreateVector(object_status, num_objects));
+  return PlasmaSend(sock, MessageType_PlasmaStatusReply, &fbb, message);
+}
+
+int64_t ReadStatusReply_num_objects(uint8_t* data) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
+  return message->object_ids()->size();
+}
+
+Status ReadStatusReply(
+    uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+  }
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    object_status[i] = message->status()->data()[i];
+  }
+  return Status::OK();
+}
+
+// Contains messages.
+
+Status SendContainsRequest(int sock, ObjectID object_id) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaContainsRequest(fbb, fbb.CreateString(object_id.binary()));
+  return PlasmaSend(sock, MessageType_PlasmaContainsRequest, &fbb, message);
+}
+
+Status ReadContainsRequest(uint8_t* data, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaContainsRequest>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendContainsReply(int sock, ObjectID object_id, bool has_object) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message =
+      CreatePlasmaContainsReply(fbb, fbb.CreateString(object_id.binary()), has_object);
+  return PlasmaSend(sock, MessageType_PlasmaContainsReply, &fbb, message);
+}
+
+Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaContainsReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  *has_object = message->has_object();
+  return Status::OK();
+}
+
+// Connect messages.
+
+Status SendConnectRequest(int sock) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaConnectRequest(fbb);
+  return PlasmaSend(sock, MessageType_PlasmaConnectRequest, &fbb, message);
+}
+
+Status ReadConnectRequest(uint8_t* data) {
+  return Status::OK();
+}
+
+Status SendConnectReply(int sock, int64_t memory_capacity) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaConnectReply(fbb, memory_capacity);
+  return PlasmaSend(sock, MessageType_PlasmaConnectReply, &fbb, message);
+}
+
+Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaConnectReply>(data);
+  *memory_capacity = message->memory_capacity();
+  return Status::OK();
+}
+
+// Evict messages.
+
+Status SendEvictRequest(int sock, int64_t num_bytes) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaEvictRequest(fbb, num_bytes);
+  return PlasmaSend(sock, MessageType_PlasmaEvictRequest, &fbb, message);
+}
+
+Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaEvictRequest>(data);
+  *num_bytes = message->num_bytes();
+  return Status::OK();
+}
+
+Status SendEvictReply(int sock, int64_t num_bytes) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaEvictReply(fbb, num_bytes);
+  return PlasmaSend(sock, MessageType_PlasmaEvictReply, &fbb, message);
+}
+
+Status ReadEvictReply(uint8_t* data, int64_t& num_bytes) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaEvictReply>(data);
+  num_bytes = message->num_bytes();
+  return Status::OK();
+}
+
+// Get messages.
+
+Status SendGetRequest(
+    int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaGetRequest(
+      fbb, to_flatbuffer(&fbb, object_ids, num_objects), timeout_ms);
+  return PlasmaSend(sock, MessageType_PlasmaGetRequest, &fbb, message);
+}
+
+Status ReadGetRequest(
+    uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
+  for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
+    auto object_id = message->object_ids()->Get(i)->str();
+    object_ids.push_back(ObjectID::from_binary(object_id));
+  }
+  *timeout_ms = message->timeout_ms();
+  return Status::OK();
+}
+
+Status SendGetReply(int sock, ObjectID object_ids[],
+    std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
+    int64_t num_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+  std::vector<PlasmaObjectSpec> objects;
+
+  for (int i = 0; i < num_objects; ++i) {
+    const PlasmaObject& object = plasma_objects[object_ids[i]];
+    objects.push_back(PlasmaObjectSpec(object.handle.store_fd, object.handle.mmap_size,
+        object.data_offset, object.data_size, object.metadata_offset,
+        object.metadata_size));
+  }
+  auto message = CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+      fbb.CreateVectorOfStructs(objects.data(), num_objects));
+  return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
+}
+
+Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
+    int64_t num_objects) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+  }
+  for (uoffset_t i = 0; i < num_objects; ++i) {
+    const PlasmaObjectSpec* object = message->plasma_objects()->Get(i);
+    plasma_objects[i].handle.store_fd = object->segment_index();
+    plasma_objects[i].handle.mmap_size = object->mmap_size();
+    plasma_objects[i].data_offset = object->data_offset();
+    plasma_objects[i].data_size = object->data_size();
+    plasma_objects[i].metadata_offset = object->metadata_offset();
+    plasma_objects[i].metadata_size = object->metadata_size();
+  }
+  return Status::OK();
+}
+
+// Fetch messages.
+
+Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message =
+      CreatePlasmaFetchRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
+  return PlasmaSend(sock, MessageType_PlasmaFetchRequest, &fbb, message);
+}
+
+Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaFetchRequest>(data);
+  for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
+    object_ids.push_back(ObjectID::from_binary(message->object_ids()->Get(i)->str()));
+  }
+  return Status::OK();
+}
+
+// Wait messages.
+
+Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
+    int num_ready_objects, int64_t timeout_ms) {
+  flatbuffers::FlatBufferBuilder fbb;
+
+  std::vector<flatbuffers::Offset<ObjectRequestSpec>> object_request_specs;
+  for (int i = 0; i < num_requests; i++) {
+    object_request_specs.push_back(CreateObjectRequestSpec(fbb,
+        fbb.CreateString(object_requests[i].object_id.binary()),
+        object_requests[i].type));
+  }
+
+  auto message = CreatePlasmaWaitRequest(
+      fbb, fbb.CreateVector(object_request_specs), num_ready_objects, timeout_ms);
+  return PlasmaSend(sock, MessageType_PlasmaWaitRequest, &fbb, message);
+}
+
+Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
+    int64_t* timeout_ms, int* num_ready_objects) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaWaitRequest>(data);
+  *num_ready_objects = message->num_ready_objects();
+  *timeout_ms = message->timeout();
+
+  for (uoffset_t i = 0; i < message->object_requests()->size(); i++) {
+    ObjectID object_id =
+        ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
+    ObjectRequest object_request({object_id, message->object_requests()->Get(i)->type(),
+        ObjectStatus_Nonexistent});
+    object_requests[object_id] = object_request;
+  }
+  return Status::OK();
+}
+
+Status SendWaitReply(
+    int sock, const ObjectRequestMap& object_requests, int num_ready_objects) {
+  flatbuffers::FlatBufferBuilder fbb;
+
+  std::vector<flatbuffers::Offset<ObjectReply>> object_replies;
+  for (const auto& entry : object_requests) {
+    const auto& object_request = entry.second;
+    object_replies.push_back(CreateObjectReply(
+        fbb, fbb.CreateString(object_request.object_id.binary()), object_request.status));
+  }
+
+  auto message = CreatePlasmaWaitReply(
+      fbb, fbb.CreateVector(object_replies.data(), num_ready_objects), num_ready_objects);
+  return PlasmaSend(sock, MessageType_PlasmaWaitReply, &fbb, message);
+}
+
+Status ReadWaitReply(
+    uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects) {
+  DCHECK(data);
+
+  auto message = flatbuffers::GetRoot<PlasmaWaitReply>(data);
+  *num_ready_objects = message->num_ready_objects();
+  for (int i = 0; i < *num_ready_objects; i++) {
+    object_requests[i].object_id =
+        ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
+    object_requests[i].status = message->object_requests()->Get(i)->status();
+  }
+  return Status::OK();
+}
+
+// Subscribe messages.
+
+Status SendSubscribeRequest(int sock) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaSubscribeRequest(fbb);
+  return PlasmaSend(sock, MessageType_PlasmaSubscribeRequest, &fbb, message);
+}
+
+// Data messages.
+
+Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto addr = fbb.CreateString(address, strlen(address));
+  auto message =
+      CreatePlasmaDataRequest(fbb, fbb.CreateString(object_id.binary()), addr, port);
+  return PlasmaSend(sock, MessageType_PlasmaDataRequest, &fbb, message);
+}
+
+Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaDataRequest>(data);
+  DCHECK(message->object_id()->size() == sizeof(ObjectID));
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  *address = strdup(message->address()->c_str());
+  *port = message->port();
+  return Status::OK();
+}
+
+Status SendDataReply(
+    int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaDataReply(
+      fbb, fbb.CreateString(object_id.binary()), object_size, metadata_size);
+  return PlasmaSend(sock, MessageType_PlasmaDataReply, &fbb, message);
+}
+
+Status ReadDataReply(
+    uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  *object_size = (int64_t)message->object_size();
+  *metadata_size = (int64_t)message->metadata_size();
+  return Status::OK();
+}
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
new file mode 100644
index 0000000000000..5d9d13675144f
--- /dev/null
+++ b/cpp/src/plasma/protocol.h
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_PROTOCOL_H
+#define PLASMA_PROTOCOL_H
+
+#include <vector>
+
+#include "arrow/status.h"
+#include "format/plasma_generated.h"
+#include "plasma/plasma.h"
+
+using arrow::Status;
+
+/* Plasma receive message. */
+
+Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer);
+
+/* Plasma Create message functions. */
+
+Status SendCreateRequest(
+    int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size);
+
+Status ReadCreateRequest(
+    uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size);
+
+Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error);
+
+Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object);
+
+/* Plasma Seal message functions. */
+
+Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest);
+
+Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest);
+
+Status SendSealReply(int sock, ObjectID object_id, int error);
+
+Status ReadSealReply(uint8_t* data, ObjectID* object_id);
+
+/* Plasma Get message functions. */
+
+Status SendGetRequest(
+    int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms);
+
+Status ReadGetRequest(
+    uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms);
+
+Status SendGetReply(int sock, ObjectID object_ids[],
+    std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
+    int64_t num_objects);
+
+Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
+    int64_t num_objects);
+
+/* Plasma Release message functions. */
+
+Status SendReleaseRequest(int sock, ObjectID object_id);
+
+Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendReleaseReply(int sock, ObjectID object_id, int error);
+
+Status ReadReleaseReply(uint8_t* data, ObjectID* object_id);
+
+/* Plasma Delete message functions. */
+
+Status SendDeleteRequest(int sock, ObjectID object_id);
+
+Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendDeleteReply(int sock, ObjectID object_id, int error);
+
+Status ReadDeleteReply(uint8_t* data, ObjectID* object_id);
+
+/* Satus messages. */
+
+Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
+
+Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects);
+
+Status SendStatusReply(
+    int sock, ObjectID object_ids[], int object_status[], int64_t num_objects);
+
+int64_t ReadStatusReply_num_objects(uint8_t* data);
+
+Status ReadStatusReply(
+    uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects);
+
+/* Plasma Constains message functions. */
+
+Status SendContainsRequest(int sock, ObjectID object_id);
+
+Status ReadContainsRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendContainsReply(int sock, ObjectID object_id, bool has_object);
+
+Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object);
+
+/* Plasma Connect message functions. */
+
+Status SendConnectRequest(int sock);
+
+Status ReadConnectRequest(uint8_t* data);
+
+Status SendConnectReply(int sock, int64_t memory_capacity);
+
+Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity);
+
+/* Plasma Evict message functions (no reply so far). */
+
+Status SendEvictRequest(int sock, int64_t num_bytes);
+
+Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes);
+
+Status SendEvictReply(int sock, int64_t num_bytes);
+
+Status ReadEvictReply(uint8_t* data, int64_t& num_bytes);
+
+/* Plasma Fetch Remote message functions. */
+
+Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
+
+Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids);
+
+/* Plasma Wait message functions. */
+
+Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
+    int num_ready_objects, int64_t timeout_ms);
+
+Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
+    int64_t* timeout_ms, int* num_ready_objects);
+
+Status SendWaitReply(
+    int sock, const ObjectRequestMap& object_requests, int num_ready_objects);
+
+Status ReadWaitReply(
+    uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects);
+
+/* Plasma Subscribe message functions. */
+
+Status SendSubscribeRequest(int sock);
+
+/* Data messages. */
+
+Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port);
+
+Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port);
+
+Status SendDataReply(
+    int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size);
+
+Status ReadDataReply(
+    uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size);
+
+#endif /* PLASMA_PROTOCOL */
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
new file mode 100644
index 0000000000000..9394e3de310b2
--- /dev/null
+++ b/cpp/src/plasma/store.cc
@@ -0,0 +1,683 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// PLASMA STORE: This is a simple object store server process
+//
+// It accepts incoming client connections on a unix domain socket
+// (name passed in via the -s option of the executable) and uses a
+// single thread to serve the clients. Each client establishes a
+// connection and can create objects, wait for objects and seal
+// objects through that connection.
+//
+// It keeps a hash table that maps object_ids (which are 20 byte long,
+// just enough to store and SHA1 hash) to memory mapped files.
+
+#include "plasma/store.h"
+
+#include <assert.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <limits.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/statvfs.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <deque>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "format/common_generated.h"
+#include "plasma/common.h"
+#include "plasma/fling.h"
+#include "plasma/io.h"
+#include "plasma/malloc.h"
+
+extern "C" {
+void* dlmalloc(size_t bytes);
+void* dlmemalign(size_t alignment, size_t bytes);
+void dlfree(void* mem);
+size_t dlmalloc_set_footprint_limit(size_t bytes);
+}
+
+struct GetRequest {
+  GetRequest(Client* client, const std::vector<ObjectID>& object_ids);
+  /// The client that called get.
+  Client* client;
+  /// The ID of the timer that will time out and cause this wait to return to
+  ///  the client if it hasn't already returned.
+  int64_t timer;
+  /// The object IDs involved in this request. This is used in the reply.
+  std::vector<ObjectID> object_ids;
+  /// The object information for the objects in this request. This is used in
+  /// the reply.
+  std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> objects;
+  /// The minimum number of objects to wait for in this request.
+  int64_t num_objects_to_wait_for;
+  /// The number of object requests in this wait request that are already
+  /// satisfied.
+  int64_t num_satisfied;
+};
+
+GetRequest::GetRequest(Client* client, const std::vector<ObjectID>& object_ids)
+    : client(client),
+      timer(-1),
+      object_ids(object_ids.begin(), object_ids.end()),
+      objects(object_ids.size()),
+      num_satisfied(0) {
+  std::unordered_set<ObjectID, UniqueIDHasher> unique_ids(
+      object_ids.begin(), object_ids.end());
+  num_objects_to_wait_for = unique_ids.size();
+}
+
+Client::Client(int fd) : fd(fd) {}
+
+PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory)
+    : loop_(loop), eviction_policy_(&store_info_) {
+  store_info_.memory_capacity = system_memory;
+}
+
+// TODO(pcm): Get rid of this destructor by using RAII to clean up data.
+PlasmaStore::~PlasmaStore() {
+  for (const auto& element : pending_notifications_) {
+    auto object_notifications = element.second.object_notifications;
+    for (size_t i = 0; i < object_notifications.size(); ++i) {
+      uint8_t* notification = reinterpret_cast<uint8_t*>(object_notifications.at(i));
+      uint8_t* data = notification;
+      // TODO(pcm): Get rid of this delete.
+      delete[] data;
+    }
+  }
+}
+
+// If this client is not already using the object, add the client to the
+// object's list of clients, otherwise do nothing.
+void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client* client) {
+  // Check if this client is already using the object.
+  if (entry->clients.find(client) != entry->clients.end()) { return; }
+  // If there are no other clients using this object, notify the eviction policy
+  // that the object is being used.
+  if (entry->clients.size() == 0) {
+    // Tell the eviction policy that this object is being used.
+    std::vector<ObjectID> objects_to_evict;
+    eviction_policy_.begin_object_access(entry->object_id, &objects_to_evict);
+    delete_objects(objects_to_evict);
+  }
+  // Add the client pointer to the list of clients using this object.
+  entry->clients.insert(client);
+}
+
+// Create a new object buffer in the hash table.
+int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
+    int64_t metadata_size, Client* client, PlasmaObject* result) {
+  ARROW_LOG(DEBUG) << "creating object " << object_id.hex();
+  if (store_info_.objects.count(object_id) != 0) {
+    // There is already an object with the same ID in the Plasma Store, so
+    // ignore this requst.
+    return PlasmaError_ObjectExists;
+  }
+  // Try to evict objects until there is enough space.
+  uint8_t* pointer;
+  do {
+    // Allocate space for the new object. We use dlmemalign instead of dlmalloc
+    // in order to align the allocated region to a 64-byte boundary. This is not
+    // strictly necessary, but it is an optimization that could speed up the
+    // computation of a hash of the data (see compute_object_hash_parallel in
+    // plasma_client.cc). Note that even though this pointer is 64-byte aligned,
+    // it is not guaranteed that the corresponding pointer in the client will be
+    // 64-byte aligned, but in practice it often will be.
+    pointer =
+        reinterpret_cast<uint8_t*>(dlmemalign(BLOCK_SIZE, data_size + metadata_size));
+    if (pointer == NULL) {
+      // Tell the eviction policy how much space we need to create this object.
+      std::vector<ObjectID> objects_to_evict;
+      bool success =
+          eviction_policy_.require_space(data_size + metadata_size, &objects_to_evict);
+      delete_objects(objects_to_evict);
+      // Return an error to the client if not enough space could be freed to
+      // create the object.
+      if (!success) { return PlasmaError_OutOfMemory; }
+    }
+  } while (pointer == NULL);
+  int fd;
+  int64_t map_size;
+  ptrdiff_t offset;
+  get_malloc_mapinfo(pointer, &fd, &map_size, &offset);
+  assert(fd != -1);
+
+  auto entry = std::unique_ptr<ObjectTableEntry>(new ObjectTableEntry());
+  entry->object_id = object_id;
+  entry->info.object_id = object_id.binary();
+  entry->info.data_size = data_size;
+  entry->info.metadata_size = metadata_size;
+  entry->pointer = pointer;
+  // TODO(pcm): Set the other fields.
+  entry->fd = fd;
+  entry->map_size = map_size;
+  entry->offset = offset;
+  entry->state = PLASMA_CREATED;
+
+  store_info_.objects[object_id] = std::move(entry);
+  result->handle.store_fd = fd;
+  result->handle.mmap_size = map_size;
+  result->data_offset = offset;
+  result->metadata_offset = offset + data_size;
+  result->data_size = data_size;
+  result->metadata_size = metadata_size;
+  // Notify the eviction policy that this object was created. This must be done
+  // immediately before the call to add_client_to_object_clients so that the
+  // eviction policy does not have an opportunity to evict the object.
+  eviction_policy_.object_created(object_id);
+  // Record that this client is using this object.
+  add_client_to_object_clients(store_info_.objects[object_id].get(), client);
+  return PlasmaError_OK;
+}
+
+void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
+  DCHECK(object != NULL);
+  DCHECK(entry != NULL);
+  DCHECK(entry->state == PLASMA_SEALED);
+  object->handle.store_fd = entry->fd;
+  object->handle.mmap_size = entry->map_size;
+  object->data_offset = entry->offset;
+  object->metadata_offset = entry->offset + entry->info.data_size;
+  object->data_size = entry->info.data_size;
+  object->metadata_size = entry->info.metadata_size;
+}
+
+void PlasmaStore::return_from_get(GetRequest* get_req) {
+  // Send the get reply to the client.
+  Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], get_req->objects,
+      get_req->object_ids.size());
+  warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd);
+  // If we successfully sent the get reply message to the client, then also send
+  // the file descriptors.
+  if (s.ok()) {
+    // Send all of the file descriptors for the present objects.
+    for (const auto& object_id : get_req->object_ids) {
+      PlasmaObject& object = get_req->objects[object_id];
+      // We use the data size to indicate whether the object is present or not.
+      if (object.data_size != -1) {
+        int error_code = send_fd(get_req->client->fd, object.handle.store_fd);
+        // If we failed to send the file descriptor, loop until we have sent it
+        // successfully. TODO(rkn): This is problematic for two reasons. First
+        // of all, sending the file descriptor should just succeed without any
+        // errors, but sometimes I see a "Message too long" error number.
+        // Second, looping like this allows a client to potentially block the
+        // plasma store event loop which should never happen.
+        while (error_code < 0) {
+          if (errno == EMSGSIZE) {
+            ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying.";
+            error_code = send_fd(get_req->client->fd, object.handle.store_fd);
+            continue;
+          }
+          warn_if_sigpipe(error_code, get_req->client->fd);
+          break;
+        }
+      }
+    }
+  }
+
+  // Remove the get request from each of the relevant object_get_requests hash
+  // tables if it is present there. It should only be present there if the get
+  // request timed out.
+  for (ObjectID& object_id : get_req->object_ids) {
+    auto& get_requests = object_get_requests_[object_id];
+    // Erase get_req from the vector.
+    auto it = std::find(get_requests.begin(), get_requests.end(), get_req);
+    if (it != get_requests.end()) { get_requests.erase(it); }
+  }
+  // Remove the get request.
+  if (get_req->timer != -1) { ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK); }
+  delete get_req;
+}
+
+void PlasmaStore::update_object_get_requests(const ObjectID& object_id) {
+  std::vector<GetRequest*>& get_requests = object_get_requests_[object_id];
+  size_t index = 0;
+  size_t num_requests = get_requests.size();
+  for (size_t i = 0; i < num_requests; ++i) {
+    GetRequest* get_req = get_requests[index];
+    auto entry = get_object_table_entry(&store_info_, object_id);
+    ARROW_CHECK(entry != NULL);
+
+    PlasmaObject_init(&get_req->objects[object_id], entry);
+    get_req->num_satisfied += 1;
+    // Record the fact that this client will be using this object and will
+    // be responsible for releasing this object.
+    add_client_to_object_clients(entry, get_req->client);
+
+    // If this get request is done, reply to the client.
+    if (get_req->num_satisfied == get_req->num_objects_to_wait_for) {
+      return_from_get(get_req);
+    } else {
+      // The call to return_from_get will remove the current element in the
+      // array, so we only increment the counter in the else branch.
+      index += 1;
+    }
+  }
+
+  DCHECK(index == get_requests.size());
+  // Remove the array of get requests for this object, since no one should be
+  // waiting for this object anymore.
+  object_get_requests_.erase(object_id);
+}
+
+void PlasmaStore::process_get_request(
+    Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms) {
+  // Create a get request for this object.
+  GetRequest* get_req = new GetRequest(client, object_ids);
+
+  for (auto object_id : object_ids) {
+    // Check if this object is already present locally. If so, record that the
+    // object is being used and mark it as accounted for.
+    auto entry = get_object_table_entry(&store_info_, object_id);
+    if (entry && entry->state == PLASMA_SEALED) {
+      // Update the get request to take into account the present object.
+      PlasmaObject_init(&get_req->objects[object_id], entry);
+      get_req->num_satisfied += 1;
+      // If necessary, record that this client is using this object. In the case
+      // where entry == NULL, this will be called from seal_object.
+      add_client_to_object_clients(entry, client);
+    } else {
+      // Add a placeholder plasma object to the get request to indicate that the
+      // object is not present. This will be parsed by the client. We set the
+      // data size to -1 to indicate that the object is not present.
+      get_req->objects[object_id].data_size = -1;
+      // Add the get request to the relevant data structures.
+      object_get_requests_[object_id].push_back(get_req);
+    }
+  }
+
+  // If all of the objects are present already or if the timeout is 0, return to
+  // the client.
+  if (get_req->num_satisfied == get_req->num_objects_to_wait_for || timeout_ms == 0) {
+    return_from_get(get_req);
+  } else if (timeout_ms != -1) {
+    // Set a timer that will cause the get request to return to the client. Note
+    // that a timeout of -1 is used to indicate that no timer should be set.
+    get_req->timer = loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) {
+      return_from_get(get_req);
+      return kEventLoopTimerDone;
+    });
+  }
+}
+
+int PlasmaStore::remove_client_from_object_clients(
+    ObjectTableEntry* entry, Client* client) {
+  auto it = entry->clients.find(client);
+  if (it != entry->clients.end()) {
+    entry->clients.erase(it);
+    // If no more clients are using this object, notify the eviction policy
+    // that the object is no longer being used.
+    if (entry->clients.size() == 0) {
+      // Tell the eviction policy that this object is no longer being used.
+      std::vector<ObjectID> objects_to_evict;
+      eviction_policy_.end_object_access(entry->object_id, &objects_to_evict);
+      delete_objects(objects_to_evict);
+    }
+    // Return 1 to indicate that the client was removed.
+    return 1;
+  } else {
+    // Return 0 to indicate that the client was not removed.
+    return 0;
+  }
+}
+
+void PlasmaStore::release_object(const ObjectID& object_id, Client* client) {
+  auto entry = get_object_table_entry(&store_info_, object_id);
+  ARROW_CHECK(entry != NULL);
+  // Remove the client from the object's array of clients.
+  ARROW_CHECK(remove_client_from_object_clients(entry, client) == 1);
+}
+
+// Check if an object is present.
+int PlasmaStore::contains_object(const ObjectID& object_id) {
+  auto entry = get_object_table_entry(&store_info_, object_id);
+  return entry && (entry->state == PLASMA_SEALED) ? OBJECT_FOUND : OBJECT_NOT_FOUND;
+}
+
+// Seal an object that has been created in the hash table.
+void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[]) {
+  ARROW_LOG(DEBUG) << "sealing object " << object_id.hex();
+  auto entry = get_object_table_entry(&store_info_, object_id);
+  ARROW_CHECK(entry != NULL);
+  ARROW_CHECK(entry->state == PLASMA_CREATED);
+  // Set the state of object to SEALED.
+  entry->state = PLASMA_SEALED;
+  // Set the object digest.
+  entry->info.digest = std::string(reinterpret_cast<char*>(&digest[0]), kDigestSize);
+  // Inform all subscribers that a new object has been sealed.
+  push_notification(&entry->info);
+
+  // Update all get requests that involve this object.
+  update_object_get_requests(object_id);
+}
+
+void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
+  for (const auto& object_id : object_ids) {
+    ARROW_LOG(DEBUG) << "deleting object " << object_id.hex();
+    auto entry = get_object_table_entry(&store_info_, object_id);
+    // TODO(rkn): This should probably not fail, but should instead throw an
+    // error. Maybe we should also support deleting objects that have been
+    // created but not sealed.
+    ARROW_CHECK(entry != NULL) << "To delete an object it must be in the object table.";
+    ARROW_CHECK(entry->state == PLASMA_SEALED)
+        << "To delete an object it must have been sealed.";
+    ARROW_CHECK(entry->clients.size() == 0)
+        << "To delete an object, there must be no clients currently using it.";
+    dlfree(entry->pointer);
+    store_info_.objects.erase(object_id);
+    // Inform all subscribers that the object has been deleted.
+    ObjectInfoT notification;
+    notification.object_id = object_id.binary();
+    notification.is_deletion = true;
+    push_notification(&notification);
+  }
+}
+
+void PlasmaStore::connect_client(int listener_sock) {
+  int client_fd = AcceptClient(listener_sock);
+  // This is freed in disconnect_client.
+  Client* client = new Client(client_fd);
+  // Add a callback to handle events on this socket.
+  // TODO(pcm): Check return value.
+  loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) {
+    Status s = process_message(client);
+    if (!s.ok()) { ARROW_LOG(FATAL) << "Failed to process file event: " << s; }
+  });
+  ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
+}
+
+void PlasmaStore::disconnect_client(Client* client) {
+  ARROW_CHECK(client != NULL);
+  ARROW_CHECK(client->fd > 0);
+  loop_->remove_file_event(client->fd);
+  // Close the socket.
+  close(client->fd);
+  ARROW_LOG(INFO) << "Disconnecting client on fd " << client->fd;
+  // If this client was using any objects, remove it from the appropriate
+  // lists.
+  for (const auto& entry : store_info_.objects) {
+    remove_client_from_object_clients(entry.second.get(), client);
+  }
+  // Note, the store may still attempt to send a message to the disconnected
+  // client (for example, when an object ID that the client was waiting for
+  // is ready). In these cases, the attempt to send the message will fail, but
+  // the store should just ignore the failure.
+  delete client;
+}
+
+/// Send notifications about sealed objects to the subscribers. This is called
+/// in seal_object. If the socket's send buffer is full, the notification will
+/// be
+/// buffered, and this will be called again when the send buffer has room.
+///
+/// @param client The client to send the notification to.
+/// @return Void.
+void PlasmaStore::send_notifications(int client_fd) {
+  auto it = pending_notifications_.find(client_fd);
+
+  int num_processed = 0;
+  bool closed = false;
+  // Loop over the array of pending notifications and send as many of them as
+  // possible.
+  for (size_t i = 0; i < it->second.object_notifications.size(); ++i) {
+    uint8_t* notification =
+        reinterpret_cast<uint8_t*>(it->second.object_notifications.at(i));
+    // Decode the length, which is the first bytes of the message.
+    int64_t size = *(reinterpret_cast<int64_t*>(notification));
+
+    // Attempt to send a notification about this object ID.
+    ssize_t nbytes = send(client_fd, notification, sizeof(int64_t) + size, 0);
+    if (nbytes >= 0) {
+      ARROW_CHECK(nbytes == static_cast<ssize_t>(sizeof(int64_t)) + size);
+    } else if (nbytes == -1 &&
+               (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
+      ARROW_LOG(DEBUG) << "The socket's send buffer is full, so we are caching this "
+                          "notification and will send it later.";
+      // Add a callback to the event loop to send queued notifications whenever
+      // there is room in the socket's send buffer. Callbacks can be added
+      // more than once here and will be overwritten. The callback is removed
+      // at the end of the method.
+      // TODO(pcm): Introduce status codes and check in case the file descriptor
+      // is added twice.
+      loop_->add_file_event(client_fd, kEventLoopWrite,
+          [this, client_fd](int events) { send_notifications(client_fd); });
+      break;
+    } else {
+      ARROW_LOG(WARNING) << "Failed to send notification to client on fd " << client_fd;
+      if (errno == EPIPE) {
+        closed = true;
+        break;
+      }
+    }
+    num_processed += 1;
+    // The corresponding malloc happened in create_object_info_buffer
+    // within push_notification.
+    delete[] notification;
+  }
+  // Remove the sent notifications from the array.
+  it->second.object_notifications.erase(it->second.object_notifications.begin(),
+      it->second.object_notifications.begin() + num_processed);
+
+  // Stop sending notifications if the pipe was broken.
+  if (closed) {
+    close(client_fd);
+    pending_notifications_.erase(client_fd);
+  }
+
+  // If we have sent all notifications, remove the fd from the event loop.
+  if (it->second.object_notifications.empty()) { loop_->remove_file_event(client_fd); }
+}
+
+void PlasmaStore::push_notification(ObjectInfoT* object_info) {
+  for (auto& element : pending_notifications_) {
+    uint8_t* notification = create_object_info_buffer(object_info);
+    element.second.object_notifications.push_back(notification);
+    send_notifications(element.first);
+    // The notification gets freed in send_notifications when the notification
+    // is sent over the socket.
+  }
+}
+
+// Subscribe to notifications about sealed objects.
+void PlasmaStore::subscribe_to_updates(Client* client) {
+  ARROW_LOG(DEBUG) << "subscribing to updates on fd " << client->fd;
+  // TODO(rkn): The store could block here if the client doesn't send a file
+  // descriptor.
+  int fd = recv_fd(client->fd);
+  if (fd < 0) {
+    // This may mean that the client died before sending the file descriptor.
+    ARROW_LOG(WARNING) << "Failed to receive file descriptor from client on fd "
+                       << client->fd << ".";
+    return;
+  }
+
+  // Create a new array to buffer notifications that can't be sent to the
+  // subscriber yet because the socket send buffer is full. TODO(rkn): the queue
+  // never gets freed.
+  // TODO(pcm): Is the following neccessary?
+  pending_notifications_[fd];
+
+  // Push notifications to the new subscriber about existing objects.
+  for (const auto& entry : store_info_.objects) {
+    push_notification(&entry.second->info);
+  }
+  send_notifications(fd);
+}
+
+Status PlasmaStore::process_message(Client* client) {
+  int64_t type;
+  Status s = ReadMessage(client->fd, &type, &input_buffer_);
+  ARROW_CHECK(s.ok() || s.IsIOError());
+
+  uint8_t* input = input_buffer_.data();
+  ObjectID object_id;
+  PlasmaObject object;
+  // TODO(pcm): Get rid of the following.
+  memset(&object, 0, sizeof(object));
+
+  // Process the different types of requests.
+  switch (type) {
+    case MessageType_PlasmaCreateRequest: {
+      int64_t data_size;
+      int64_t metadata_size;
+      RETURN_NOT_OK(ReadCreateRequest(input, &object_id, &data_size, &metadata_size));
+      int error_code =
+          create_object(object_id, data_size, metadata_size, client, &object);
+      HANDLE_SIGPIPE(
+          SendCreateReply(client->fd, object_id, &object, error_code), client->fd);
+      if (error_code == PlasmaError_OK) {
+        warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
+      }
+    } break;
+    case MessageType_PlasmaGetRequest: {
+      std::vector<ObjectID> object_ids_to_get;
+      int64_t timeout_ms;
+      RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms));
+      process_get_request(client, object_ids_to_get, timeout_ms);
+    } break;
+    case MessageType_PlasmaReleaseRequest:
+      RETURN_NOT_OK(ReadReleaseRequest(input, &object_id));
+      release_object(object_id, client);
+      break;
+    case MessageType_PlasmaContainsRequest:
+      RETURN_NOT_OK(ReadContainsRequest(input, &object_id));
+      if (contains_object(object_id) == OBJECT_FOUND) {
+        HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd);
+      } else {
+        HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd);
+      }
+      break;
+    case MessageType_PlasmaSealRequest: {
+      unsigned char digest[kDigestSize];
+      RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0]));
+      seal_object(object_id, &digest[0]);
+    } break;
+    case MessageType_PlasmaEvictRequest: {
+      // This code path should only be used for testing.
+      int64_t num_bytes;
+      RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes));
+      std::vector<ObjectID> objects_to_evict;
+      int64_t num_bytes_evicted =
+          eviction_policy_.choose_objects_to_evict(num_bytes, &objects_to_evict);
+      delete_objects(objects_to_evict);
+      HANDLE_SIGPIPE(SendEvictReply(client->fd, num_bytes_evicted), client->fd);
+    } break;
+    case MessageType_PlasmaSubscribeRequest:
+      subscribe_to_updates(client);
+      break;
+    case MessageType_PlasmaConnectRequest: {
+      HANDLE_SIGPIPE(
+          SendConnectReply(client->fd, store_info_.memory_capacity), client->fd);
+    } break;
+    case DISCONNECT_CLIENT:
+      ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd;
+      disconnect_client(client);
+      break;
+    default:
+      // This code should be unreachable.
+      ARROW_CHECK(0);
+  }
+  return Status::OK();
+}
+
+// Report "success" to valgrind.
+void signal_handler(int signal) {
+  if (signal == SIGTERM) { exit(0); }
+}
+
+void start_server(char* socket_name, int64_t system_memory) {
+  // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
+  // to a client that has already died, the store could die.
+  signal(SIGPIPE, SIG_IGN);
+  // Create the event loop.
+  EventLoop loop;
+  PlasmaStore store(&loop, system_memory);
+  int socket = bind_ipc_sock(socket_name, true);
+  ARROW_CHECK(socket >= 0);
+  // TODO(pcm): Check return value.
+  loop.add_file_event(socket, kEventLoopRead,
+      [&store, socket](int events) { store.connect_client(socket); });
+  loop.run();
+}
+
+int main(int argc, char* argv[]) {
+  signal(SIGTERM, signal_handler);
+  char* socket_name = NULL;
+  int64_t system_memory = -1;
+  int c;
+  while ((c = getopt(argc, argv, "s:m:")) != -1) {
+    switch (c) {
+      case 's':
+        socket_name = optarg;
+        break;
+      case 'm': {
+        char extra;
+        int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra);
+        ARROW_CHECK(scanned == 1);
+        ARROW_LOG(INFO) << "Allowing the Plasma store to use up to "
+                        << static_cast<double>(system_memory) / 1000000000
+                        << "GB of memory.";
+        break;
+      }
+      default:
+        exit(-1);
+    }
+  }
+  if (!socket_name) {
+    ARROW_LOG(FATAL) << "please specify socket for incoming connections with -s switch";
+  }
+  if (system_memory == -1) {
+    ARROW_LOG(FATAL) << "please specify the amount of system memory with -m switch";
+  }
+#ifdef __linux__
+  // On Linux, check that the amount of memory available in /dev/shm is large
+  // enough to accommodate the request. If it isn't, then fail.
+  int shm_fd = open("/dev/shm", O_RDONLY);
+  struct statvfs shm_vfs_stats;
+  fstatvfs(shm_fd, &shm_vfs_stats);
+  // The value shm_vfs_stats.f_bsize is the block size, and the value
+  // shm_vfs_stats.f_bavail is the number of available blocks.
+  int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail;
+  close(shm_fd);
+  if (system_memory > shm_mem_avail) {
+    ARROW_LOG(FATAL) << "System memory request exceeds memory available in /dev/shm. The "
+                        "request is for "
+                     << system_memory << " bytes, and the amount available is "
+                     << shm_mem_avail
+                     << " bytes. You may be able to free up space by deleting files in "
+                        "/dev/shm. If you are inside a Docker container, you may need to "
+                        "pass "
+                        "an argument with the flag '--shm-size' to 'docker run'.";
+  }
+#endif
+  // Make it so dlmalloc fails if we try to request more memory than is
+  // available.
+  dlmalloc_set_footprint_limit((size_t)system_memory);
+  ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
+  start_server(socket_name, system_memory);
+}
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
new file mode 100644
index 0000000000000..8bd94265410f6
--- /dev/null
+++ b/cpp/src/plasma/store.h
@@ -0,0 +1,169 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_STORE_H
+#define PLASMA_STORE_H
+
+#include <deque>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/events.h"
+#include "plasma/eviction_policy.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+struct GetRequest;
+
+struct NotificationQueue {
+  /// The object notifications for clients. We notify the client about the
+  /// objects in the order that the objects were sealed or deleted.
+  std::deque<uint8_t*> object_notifications;
+};
+
+/// Contains all information that is associated with a Plasma store client.
+struct Client {
+  explicit Client(int fd);
+
+  /// The file descriptor used to communicate with the client.
+  int fd;
+};
+
+class PlasmaStore {
+ public:
+  PlasmaStore(EventLoop* loop, int64_t system_memory);
+
+  ~PlasmaStore();
+
+  /// Create a new object. The client must do a call to release_object to tell
+  /// the store when it is done with the object.
+  ///
+  /// @param object_id Object ID of the object to be created.
+  /// @param data_size Size in bytes of the object to be created.
+  /// @param metadata_size Size in bytes of the object metadata.
+  /// @return One of the following error codes:
+  ///  - PlasmaError_OK, if the object was created successfully.
+  ///  - PlasmaError_ObjectExists, if an object with this ID is already
+  ///    present in the store. In this case, the client should not call
+  ///    plasma_release.
+  ///  - PlasmaError_OutOfMemory, if the store is out of memory and
+  ///    cannot create the object. In this case, the client should not call
+  ///    plasma_release.
+  int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
+      Client* client, PlasmaObject* result);
+
+  /// Delete objects that have been created in the hash table. This should only
+  /// be called on objects that are returned by the eviction policy to evict.
+  ///
+  /// @param object_ids Object IDs of the objects to be deleted.
+  /// @return Void.
+  void delete_objects(const std::vector<ObjectID>& object_ids);
+
+  /// Process a get request from a client. This method assumes that we will
+  /// eventually have these objects sealed. If one of the objects has not yet
+  /// been sealed, the client that requested the object will be notified when it
+  /// is sealed.
+  ///
+  /// For each object, the client must do a call to release_object to tell the
+  /// store when it is done with the object.
+  ///
+  /// @param client The client making this request.
+  /// @param object_ids Object IDs of the objects to be gotten.
+  /// @param timeout_ms The timeout for the get request in milliseconds.
+  /// @return Void.
+  void process_get_request(
+      Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms);
+
+  /// Seal an object. The object is now immutable and can be accessed with get.
+  ///
+  /// @param object_id Object ID of the object to be sealed.
+  /// @param digest The digest of the object. This is used to tell if two
+  /// objects
+  ///        with the same object ID are the same.
+  /// @return Void.
+  void seal_object(const ObjectID& object_id, unsigned char digest[]);
+
+  /// Check if the plasma store contains an object:
+  ///
+  /// @param object_id Object ID that will be checked.
+  /// @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if
+  /// not
+  int contains_object(const ObjectID& object_id);
+
+  /// Record the fact that a particular client is no longer using an object.
+  ///
+  /// @param object_id The object ID of the object that is being released.
+  /// @param client The client making this request.
+  /// @param Void.
+  void release_object(const ObjectID& object_id, Client* client);
+
+  /// Subscribe a file descriptor to updates about new sealed objects.
+  ///
+  /// @param client The client making this request.
+  /// @return Void.
+  void subscribe_to_updates(Client* client);
+
+  /// Connect a new client to the PlasmaStore.
+  ///
+  /// @param listener_sock The socket that is listening to incoming connections.
+  /// @return Void.
+  void connect_client(int listener_sock);
+
+  /// Disconnect a client from the PlasmaStore.
+  ///
+  /// @param client The client that is disconnected.
+  /// @return Void.
+  void disconnect_client(Client* client);
+
+  void send_notifications(int client_fd);
+
+  Status process_message(Client* client);
+
+ private:
+  void push_notification(ObjectInfoT* object_notification);
+
+  void add_client_to_object_clients(ObjectTableEntry* entry, Client* client);
+
+  void return_from_get(GetRequest* get_req);
+
+  void update_object_get_requests(const ObjectID& object_id);
+
+  int remove_client_from_object_clients(ObjectTableEntry* entry, Client* client);
+
+  /// Event loop of the plasma store.
+  EventLoop* loop_;
+  /// The plasma store information, including the object tables, that is exposed
+  /// to the eviction policy.
+  PlasmaStoreInfo store_info_;
+  /// The state that is managed by the eviction policy.
+  EvictionPolicy eviction_policy_;
+  /// Input buffer. This is allocated only once to avoid mallocs for every
+  /// call to process_message.
+  std::vector<uint8_t> input_buffer_;
+  /// A hash table mapping object IDs to a vector of the get requests that are
+  /// waiting for the object to arrive.
+  std::unordered_map<ObjectID, std::vector<GetRequest*>, UniqueIDHasher>
+      object_get_requests_;
+  /// The pending notifications that have not been sent to subscribers because
+  /// the socket send buffers were full. This is a hash table from client file
+  /// descriptor to an array of object_ids to send to that client.
+  /// TODO(pcm): Consider putting this into the Client data structure and
+  /// reorganize the code slightly.
+  std::unordered_map<int, NotificationQueue> pending_notifications_;
+};
+
+#endif  // PLASMA_STORE_H
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
new file mode 100644
index 0000000000000..29b5b135144c3
--- /dev/null
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include <assert.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+std::string g_test_executable;  // NOLINT
+
+class TestPlasmaStore : public ::testing::Test {
+ public:
+  // TODO(pcm): At the moment, stdout of the test gets mixed up with
+  // stdout of the object store. Consider changing that.
+  void SetUp() {
+    std::string plasma_directory =
+        g_test_executable.substr(0, g_test_executable.find_last_of("/"));
+    std::string plasma_command =
+        plasma_directory +
+        "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &";
+    system(plasma_command.c_str());
+    ARROW_CHECK_OK(client_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
+  }
+  virtual void Finish() {
+    ARROW_CHECK_OK(client_.Disconnect());
+    system("killall plasma_store &");
+  }
+
+ protected:
+  PlasmaClient client_;
+};
+
+TEST_F(TestPlasmaStore, ContainsTest) {
+  ObjectID object_id = ObjectID::from_random();
+
+  // Test for object non-existence.
+  bool has_object;
+  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, false);
+
+  // Test for the object being in local Plasma store.
+  // First create object.
+  int64_t data_size = 100;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  uint8_t* data;
+  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  ARROW_CHECK_OK(client_.Seal(object_id));
+  // Avoid race condition of Plasma Manager waiting for notification.
+  ObjectBuffer object_buffer;
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, true);
+}
+
+TEST_F(TestPlasmaStore, GetTest) {
+  ObjectID object_id = ObjectID::from_random();
+  ObjectBuffer object_buffer;
+
+  // Test for object non-existence.
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer));
+  ASSERT_EQ(object_buffer.data_size, -1);
+
+  // Test for the object being in local Plasma store.
+  // First create object.
+  int64_t data_size = 4;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  uint8_t* data;
+  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  for (int64_t i = 0; i < data_size; i++) {
+    data[i] = static_cast<uint8_t>(i % 4);
+  }
+  ARROW_CHECK_OK(client_.Seal(object_id));
+
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  for (int64_t i = 0; i < data_size; i++) {
+    ASSERT_EQ(data[i], object_buffer.data[i]);
+  }
+}
+
+TEST_F(TestPlasmaStore, MultipleGetTest) {
+  ObjectID object_id1 = ObjectID::from_random();
+  ObjectID object_id2 = ObjectID::from_random();
+  ObjectID object_ids[2] = {object_id1, object_id2};
+  ObjectBuffer object_buffer[2];
+
+  int64_t data_size = 4;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  uint8_t* data;
+  ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data));
+  data[0] = 1;
+  ARROW_CHECK_OK(client_.Seal(object_id1));
+
+  ARROW_CHECK_OK(client_.Create(object_id2, data_size, metadata, metadata_size, &data));
+  data[0] = 2;
+  ARROW_CHECK_OK(client_.Seal(object_id2));
+
+  ARROW_CHECK_OK(client_.Get(object_ids, 2, -1, object_buffer));
+  ASSERT_EQ(object_buffer[0].data[0], 1);
+  ASSERT_EQ(object_buffer[1].data[0], 2);
+}
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  g_test_executable = std::string(argv[0]);
+  return RUN_ALL_TESTS();
+}
diff --git a/cpp/src/plasma/test/run_tests.sh b/cpp/src/plasma/test/run_tests.sh
new file mode 100644
index 0000000000000..958bd08398e23
--- /dev/null
+++ b/cpp/src/plasma/test/run_tests.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cause the script to exit if a single command fails.
+set -e
+
+./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
+sleep 1
+./src/plasma/manager_tests
+killall plasma_store
+./src/plasma/serialization_tests
+
+# Start the Redis shards.
+./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
+redis_pid1=$!
+./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
+redis_pid2=$!
+sleep 1
+
+# Flush the redis server
+./src/common/thirdparty/redis/src/redis-cli flushall
+# Register the shard location with the primary shard.
+./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
+./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
+sleep 1
+./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
+plasma1_pid=$!
+./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
+plasma2_pid=$!
+./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 &
+plasma3_pid=$!
+./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
+plasma4_pid=$!
+sleep 1
+
+./src/plasma/client_tests
+
+kill $plasma4_pid
+kill $plasma3_pid
+kill $plasma2_pid
+kill $plasma1_pid
+kill $redis_pid1
+wait $redis_pid1
+kill $redis_pid2
+wait $redis_pid2
diff --git a/cpp/src/plasma/test/run_valgrind.sh b/cpp/src/plasma/test/run_valgrind.sh
new file mode 100644
index 0000000000000..0472194128679
--- /dev/null
+++ b/cpp/src/plasma/test/run_valgrind.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cause the script to exit if a single command fails.
+set -e
+
+./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
+sleep 1
+valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests
+killall plasma_store
+valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
new file mode 100644
index 0000000000000..325cead06e770
--- /dev/null
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -0,0 +1,388 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/common.h"
+#include "plasma/io.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+/**
+ * Create a temporary file. Needs to be closed by the caller.
+ *
+ * @return File descriptor of the file.
+ */
+int create_temp_file(void) {
+  static char temp[] = "/tmp/tempfileXXXXXX";
+  char file_name[32];
+  strncpy(file_name, temp, 32);
+  return mkstemp(file_name);
+}
+
+/**
+ * Seek to the beginning of a file and read a message from it.
+ *
+ * @param fd File descriptor of the file.
+ * @param message type Message type that we expect in the file.
+ *
+ * @return Pointer to the content of the message. Needs to be freed by the
+ * caller.
+ */
+std::vector<uint8_t> read_message_from_file(int fd, int message_type) {
+  /* Go to the beginning of the file. */
+  lseek(fd, 0, SEEK_SET);
+  int64_t type;
+  std::vector<uint8_t> data;
+  ARROW_CHECK_OK(ReadMessage(fd, &type, &data));
+  ARROW_CHECK(type == message_type);
+  return data;
+}
+
+PlasmaObject random_plasma_object(void) {
+  unsigned int seed = static_cast<unsigned int>(time(NULL));
+  int random = rand_r(&seed);
+  PlasmaObject object;
+  memset(&object, 0, sizeof(object));
+  object.handle.store_fd = random + 7;
+  object.handle.mmap_size = random + 42;
+  object.data_offset = random + 1;
+  object.metadata_offset = random + 2;
+  object.data_size = random + 3;
+  object.metadata_size = random + 4;
+  return object;
+}
+
+TEST(PlasmaSerialization, CreateRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  int64_t data_size1 = 42;
+  int64_t metadata_size1 = 11;
+  ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateRequest);
+  ObjectID object_id2;
+  int64_t data_size2;
+  int64_t metadata_size2;
+  ARROW_CHECK_OK(
+      ReadCreateRequest(data.data(), &object_id2, &data_size2, &metadata_size2));
+  ASSERT_EQ(data_size1, data_size2);
+  ASSERT_EQ(metadata_size1, metadata_size2);
+  ASSERT_EQ(object_id1, object_id2);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, CreateReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  PlasmaObject object1 = random_plasma_object();
+  ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateReply);
+  ObjectID object_id2;
+  PlasmaObject object2;
+  memset(&object2, 0, sizeof(object2));
+  ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2));
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(memcmp(&object1, &object2, sizeof(object1)), 0);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, SealRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  unsigned char digest1[kDigestSize];
+  memset(&digest1[0], 7, kDigestSize);
+  ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0]));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealRequest);
+  ObjectID object_id2;
+  unsigned char digest2[kDigestSize];
+  ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0]));
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(memcmp(&digest1[0], &digest2[0], kDigestSize), 0);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, SealReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealReply);
+  ObjectID object_id2;
+  Status s = ReadSealReply(data.data(), &object_id2);
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_TRUE(s.IsPlasmaObjectExists());
+  close(fd);
+}
+
+TEST(PlasmaSerialization, GetRequest) {
+  int fd = create_temp_file();
+  ObjectID object_ids[2];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  int64_t timeout_ms = 1234;
+  ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetRequest);
+  std::vector<ObjectID> object_ids_return;
+  int64_t timeout_ms_return;
+  ARROW_CHECK_OK(ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return));
+  ASSERT_EQ(object_ids[0], object_ids_return[0]);
+  ASSERT_EQ(object_ids[1], object_ids_return[1]);
+  ASSERT_EQ(timeout_ms, timeout_ms_return);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, GetReply) {
+  int fd = create_temp_file();
+  ObjectID object_ids[2];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> plasma_objects;
+  plasma_objects[object_ids[0]] = random_plasma_object();
+  plasma_objects[object_ids[1]] = random_plasma_object();
+  ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetReply);
+  ObjectID object_ids_return[2];
+  PlasmaObject plasma_objects_return[2];
+  memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
+  ARROW_CHECK_OK(
+      ReadGetReply(data.data(), object_ids_return, &plasma_objects_return[0], 2));
+  ASSERT_EQ(object_ids[0], object_ids_return[0]);
+  ASSERT_EQ(object_ids[1], object_ids_return[1]);
+  ASSERT_EQ(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
+                sizeof(PlasmaObject)),
+      0);
+  ASSERT_EQ(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
+                sizeof(PlasmaObject)),
+      0);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, ReleaseRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1));
+  std::vector<uint8_t> data =
+      read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
+  ObjectID object_id2;
+  ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2));
+  ASSERT_EQ(object_id1, object_id2);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, ReleaseReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaReleaseReply);
+  ObjectID object_id2;
+  Status s = ReadReleaseReply(data.data(), &object_id2);
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_TRUE(s.IsPlasmaObjectExists());
+  close(fd);
+}
+
+TEST(PlasmaSerialization, DeleteRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
+  ObjectID object_id2;
+  ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2));
+  ASSERT_EQ(object_id1, object_id2);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, DeleteReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  int error1 = PlasmaError_ObjectExists;
+  ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteReply);
+  ObjectID object_id2;
+  Status s = ReadDeleteReply(data.data(), &object_id2);
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_TRUE(s.IsPlasmaObjectExists());
+  close(fd);
+}
+
+TEST(PlasmaSerialization, StatusRequest) {
+  int fd = create_temp_file();
+  int64_t num_objects = 2;
+  ObjectID object_ids[num_objects];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusRequest);
+  ObjectID object_ids_read[num_objects];
+  ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects));
+  ASSERT_EQ(object_ids[0], object_ids_read[0]);
+  ASSERT_EQ(object_ids[1], object_ids_read[1]);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, StatusReply) {
+  int fd = create_temp_file();
+  ObjectID object_ids[2];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  int object_statuses[2] = {42, 43};
+  ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusReply);
+  int64_t num_objects = ReadStatusReply_num_objects(data.data());
+  ObjectID object_ids_read[num_objects];
+  int object_statuses_read[num_objects];
+  ARROW_CHECK_OK(
+      ReadStatusReply(data.data(), object_ids_read, object_statuses_read, num_objects));
+  ASSERT_EQ(object_ids[0], object_ids_read[0]);
+  ASSERT_EQ(object_ids[1], object_ids_read[1]);
+  ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
+  ASSERT_EQ(object_statuses[1], object_statuses_read[1]);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, EvictRequest) {
+  int fd = create_temp_file();
+  int64_t num_bytes = 111;
+  ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictRequest);
+  int64_t num_bytes_received;
+  ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received));
+  ASSERT_EQ(num_bytes, num_bytes_received);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, EvictReply) {
+  int fd = create_temp_file();
+  int64_t num_bytes = 111;
+  ARROW_CHECK_OK(SendEvictReply(fd, num_bytes));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictReply);
+  int64_t num_bytes_received;
+  ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received));
+  ASSERT_EQ(num_bytes, num_bytes_received);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, FetchRequest) {
+  int fd = create_temp_file();
+  ObjectID object_ids[2];
+  object_ids[0] = ObjectID::from_random();
+  object_ids[1] = ObjectID::from_random();
+  ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2));
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaFetchRequest);
+  std::vector<ObjectID> object_ids_read;
+  ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read));
+  ASSERT_EQ(object_ids[0], object_ids_read[0]);
+  ASSERT_EQ(object_ids[1], object_ids_read[1]);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, WaitRequest) {
+  int fd = create_temp_file();
+  const int num_objects_in = 2;
+  ObjectRequest object_requests_in[num_objects_in] = {
+      ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}),
+      ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})};
+  const int num_ready_objects_in = 1;
+  int64_t timeout_ms = 1000;
+
+  ARROW_CHECK_OK(SendWaitRequest(
+      fd, &object_requests_in[0], num_objects_in, num_ready_objects_in, timeout_ms));
+  /* Read message back. */
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitRequest);
+  int num_ready_objects_out;
+  int64_t timeout_ms_read;
+  ObjectRequestMap object_requests_out;
+  ARROW_CHECK_OK(ReadWaitRequest(
+      data.data(), object_requests_out, &timeout_ms_read, &num_ready_objects_out));
+  ASSERT_EQ(num_objects_in, object_requests_out.size());
+  ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
+  for (int i = 0; i < num_objects_in; i++) {
+    const ObjectID& object_id = object_requests_in[i].object_id;
+    ASSERT_EQ(1, object_requests_out.count(object_id));
+    const auto& entry = object_requests_out.find(object_id);
+    ASSERT_TRUE(entry != object_requests_out.end());
+    ASSERT_EQ(entry->second.object_id, object_requests_in[i].object_id);
+    ASSERT_EQ(entry->second.type, object_requests_in[i].type);
+  }
+  close(fd);
+}
+
+TEST(PlasmaSerialization, WaitReply) {
+  int fd = create_temp_file();
+  const int num_objects_in = 2;
+  /* Create a map with two ObjectRequests in it. */
+  ObjectRequestMap objects_in(num_objects_in);
+  ObjectID id1 = ObjectID::from_random();
+  objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local});
+  ObjectID id2 = ObjectID::from_random();
+  objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent});
+
+  ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in));
+  /* Read message back. */
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitReply);
+  ObjectRequest objects_out[2];
+  int num_objects_out;
+  ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out));
+  ASSERT_EQ(num_objects_in, num_objects_out);
+  for (int i = 0; i < num_objects_out; i++) {
+    /* Each object request must appear exactly once. */
+    ASSERT_EQ(objects_in.count(objects_out[i].object_id), 1);
+    const auto& entry = objects_in.find(objects_out[i].object_id);
+    ASSERT_TRUE(entry != objects_in.end());
+    ASSERT_EQ(entry->second.object_id, objects_out[i].object_id);
+    ASSERT_EQ(entry->second.status, objects_out[i].status);
+  }
+  close(fd);
+}
+
+TEST(PlasmaSerialization, DataRequest) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  const char* address1 = "address1";
+  int port1 = 12345;
+  ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1));
+  /* Reading message back. */
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataRequest);
+  ObjectID object_id2;
+  char* address2;
+  int port2;
+  ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2));
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(strcmp(address1, address2), 0);
+  ASSERT_EQ(port1, port2);
+  free(address2);
+  close(fd);
+}
+
+TEST(PlasmaSerialization, DataReply) {
+  int fd = create_temp_file();
+  ObjectID object_id1 = ObjectID::from_random();
+  int64_t object_size1 = 146;
+  int64_t metadata_size1 = 198;
+  ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1));
+  /* Reading message back. */
+  std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataReply);
+  ObjectID object_id2;
+  int64_t object_size2;
+  int64_t metadata_size2;
+  ARROW_CHECK_OK(ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2));
+  ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(object_size1, object_size2);
+  ASSERT_EQ(metadata_size1, metadata_size2);
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae.c b/cpp/src/plasma/thirdparty/ae/ae.c
new file mode 100644
index 0000000000000..e66808a81466d
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae.c
@@ -0,0 +1,465 @@
+/* A simple event-driven programming library. Originally I wrote this code
+ * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
+ * it in form of a library for easy reuse.
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <poll.h>
+#include <string.h>
+#include <time.h>
+#include <errno.h>
+
+#include "ae.h"
+#include "zmalloc.h"
+#include "config.h"
+
+/* Include the best multiplexing layer supported by this system.
+ * The following should be ordered by performances, descending. */
+#ifdef HAVE_EVPORT
+#include "ae_evport.c"
+#else
+    #ifdef HAVE_EPOLL
+    #include "ae_epoll.c"
+    #else
+        #ifdef HAVE_KQUEUE
+        #include "ae_kqueue.c"
+        #else
+        #include "ae_select.c"
+        #endif
+    #endif
+#endif
+
+aeEventLoop *aeCreateEventLoop(int setsize) {
+    aeEventLoop *eventLoop;
+    int i;
+
+    if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
+    eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
+    eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
+    if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
+    eventLoop->setsize = setsize;
+    eventLoop->lastTime = time(NULL);
+    eventLoop->timeEventHead = NULL;
+    eventLoop->timeEventNextId = 0;
+    eventLoop->stop = 0;
+    eventLoop->maxfd = -1;
+    eventLoop->beforesleep = NULL;
+    if (aeApiCreate(eventLoop) == -1) goto err;
+    /* Events with mask == AE_NONE are not set. So let's initialize the
+     * vector with it. */
+    for (i = 0; i < setsize; i++)
+        eventLoop->events[i].mask = AE_NONE;
+    return eventLoop;
+
+err:
+    if (eventLoop) {
+        zfree(eventLoop->events);
+        zfree(eventLoop->fired);
+        zfree(eventLoop);
+    }
+    return NULL;
+}
+
+/* Return the current set size. */
+int aeGetSetSize(aeEventLoop *eventLoop) {
+    return eventLoop->setsize;
+}
+
+/* Resize the maximum set size of the event loop.
+ * If the requested set size is smaller than the current set size, but
+ * there is already a file descriptor in use that is >= the requested
+ * set size minus one, AE_ERR is returned and the operation is not
+ * performed at all.
+ *
+ * Otherwise AE_OK is returned and the operation is successful. */
+int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
+    int i;
+
+    if (setsize == eventLoop->setsize) return AE_OK;
+    if (eventLoop->maxfd >= setsize) return AE_ERR;
+    if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
+
+    eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
+    eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
+    eventLoop->setsize = setsize;
+
+    /* Make sure that if we created new slots, they are initialized with
+     * an AE_NONE mask. */
+    for (i = eventLoop->maxfd+1; i < setsize; i++)
+        eventLoop->events[i].mask = AE_NONE;
+    return AE_OK;
+}
+
+void aeDeleteEventLoop(aeEventLoop *eventLoop) {
+    aeApiFree(eventLoop);
+    zfree(eventLoop->events);
+    zfree(eventLoop->fired);
+    zfree(eventLoop);
+}
+
+void aeStop(aeEventLoop *eventLoop) {
+    eventLoop->stop = 1;
+}
+
+int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
+        aeFileProc *proc, void *clientData)
+{
+    if (fd >= eventLoop->setsize) {
+        errno = ERANGE;
+        return AE_ERR;
+    }
+    aeFileEvent *fe = &eventLoop->events[fd];
+
+    if (aeApiAddEvent(eventLoop, fd, mask) == -1)
+        return AE_ERR;
+    fe->mask |= mask;
+    if (mask & AE_READABLE) fe->rfileProc = proc;
+    if (mask & AE_WRITABLE) fe->wfileProc = proc;
+    fe->clientData = clientData;
+    if (fd > eventLoop->maxfd)
+        eventLoop->maxfd = fd;
+    return AE_OK;
+}
+
+void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
+{
+    if (fd >= eventLoop->setsize) return;
+    aeFileEvent *fe = &eventLoop->events[fd];
+    if (fe->mask == AE_NONE) return;
+
+    aeApiDelEvent(eventLoop, fd, mask);
+    fe->mask = fe->mask & (~mask);
+    if (fd == eventLoop->maxfd && fe->mask == AE_NONE) {
+        /* Update the max fd */
+        int j;
+
+        for (j = eventLoop->maxfd-1; j >= 0; j--)
+            if (eventLoop->events[j].mask != AE_NONE) break;
+        eventLoop->maxfd = j;
+    }
+}
+
+int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
+    if (fd >= eventLoop->setsize) return 0;
+    aeFileEvent *fe = &eventLoop->events[fd];
+
+    return fe->mask;
+}
+
+static void aeGetTime(long *seconds, long *milliseconds)
+{
+    struct timeval tv;
+
+    gettimeofday(&tv, NULL);
+    *seconds = tv.tv_sec;
+    *milliseconds = tv.tv_usec/1000;
+}
+
+static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) {
+    long cur_sec, cur_ms, when_sec, when_ms;
+
+    aeGetTime(&cur_sec, &cur_ms);
+    when_sec = cur_sec + milliseconds/1000;
+    when_ms = cur_ms + milliseconds%1000;
+    if (when_ms >= 1000) {
+        when_sec ++;
+        when_ms -= 1000;
+    }
+    *sec = when_sec;
+    *ms = when_ms;
+}
+
+long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
+        aeTimeProc *proc, void *clientData,
+        aeEventFinalizerProc *finalizerProc)
+{
+    long long id = eventLoop->timeEventNextId++;
+    aeTimeEvent *te;
+
+    te = zmalloc(sizeof(*te));
+    if (te == NULL) return AE_ERR;
+    te->id = id;
+    aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms);
+    te->timeProc = proc;
+    te->finalizerProc = finalizerProc;
+    te->clientData = clientData;
+    te->next = eventLoop->timeEventHead;
+    eventLoop->timeEventHead = te;
+    return id;
+}
+
+int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
+{
+    aeTimeEvent *te = eventLoop->timeEventHead;
+    while(te) {
+        if (te->id == id) {
+            te->id = AE_DELETED_EVENT_ID;
+            return AE_OK;
+        }
+        te = te->next;
+    }
+    return AE_ERR; /* NO event with the specified ID found */
+}
+
+/* Search the first timer to fire.
+ * This operation is useful to know how many time the select can be
+ * put in sleep without to delay any event.
+ * If there are no timers NULL is returned.
+ *
+ * Note that's O(N) since time events are unsorted.
+ * Possible optimizations (not needed by Redis so far, but...):
+ * 1) Insert the event in order, so that the nearest is just the head.
+ *    Much better but still insertion or deletion of timers is O(N).
+ * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)).
+ */
+static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
+{
+    aeTimeEvent *te = eventLoop->timeEventHead;
+    aeTimeEvent *nearest = NULL;
+
+    while(te) {
+        if (!nearest || te->when_sec < nearest->when_sec ||
+                (te->when_sec == nearest->when_sec &&
+                 te->when_ms < nearest->when_ms))
+            nearest = te;
+        te = te->next;
+    }
+    return nearest;
+}
+
+/* Process time events */
+static int processTimeEvents(aeEventLoop *eventLoop) {
+    int processed = 0;
+    aeTimeEvent *te, *prev;
+    long long maxId;
+    time_t now = time(NULL);
+
+    /* If the system clock is moved to the future, and then set back to the
+     * right value, time events may be delayed in a random way. Often this
+     * means that scheduled operations will not be performed soon enough.
+     *
+     * Here we try to detect system clock skews, and force all the time
+     * events to be processed ASAP when this happens: the idea is that
+     * processing events earlier is less dangerous than delaying them
+     * indefinitely, and practice suggests it is. */
+    if (now < eventLoop->lastTime) {
+        te = eventLoop->timeEventHead;
+        while(te) {
+            te->when_sec = 0;
+            te = te->next;
+        }
+    }
+    eventLoop->lastTime = now;
+
+    prev = NULL;
+    te = eventLoop->timeEventHead;
+    maxId = eventLoop->timeEventNextId-1;
+    while(te) {
+        long now_sec, now_ms;
+        long long id;
+
+        /* Remove events scheduled for deletion. */
+        if (te->id == AE_DELETED_EVENT_ID) {
+            aeTimeEvent *next = te->next;
+            if (prev == NULL)
+                eventLoop->timeEventHead = te->next;
+            else
+                prev->next = te->next;
+            if (te->finalizerProc)
+                te->finalizerProc(eventLoop, te->clientData);
+            zfree(te);
+            te = next;
+            continue;
+        }
+
+        /* Make sure we don't process time events created by time events in
+         * this iteration. Note that this check is currently useless: we always
+         * add new timers on the head, however if we change the implementation
+         * detail, this check may be useful again: we keep it here for future
+         * defense. */
+        if (te->id > maxId) {
+            te = te->next;
+            continue;
+        }
+        aeGetTime(&now_sec, &now_ms);
+        if (now_sec > te->when_sec ||
+            (now_sec == te->when_sec && now_ms >= te->when_ms))
+        {
+            int retval;
+
+            id = te->id;
+            retval = te->timeProc(eventLoop, id, te->clientData);
+            processed++;
+            if (retval != AE_NOMORE) {
+                aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
+            } else {
+                te->id = AE_DELETED_EVENT_ID;
+            }
+        }
+        prev = te;
+        te = te->next;
+    }
+    return processed;
+}
+
+/* Process every pending time event, then every pending file event
+ * (that may be registered by time event callbacks just processed).
+ * Without special flags the function sleeps until some file event
+ * fires, or when the next time event occurs (if any).
+ *
+ * If flags is 0, the function does nothing and returns.
+ * if flags has AE_ALL_EVENTS set, all the kind of events are processed.
+ * if flags has AE_FILE_EVENTS set, file events are processed.
+ * if flags has AE_TIME_EVENTS set, time events are processed.
+ * if flags has AE_DONT_WAIT set the function returns ASAP until all
+ * the events that's possible to process without to wait are processed.
+ *
+ * The function returns the number of events processed. */
+int aeProcessEvents(aeEventLoop *eventLoop, int flags)
+{
+    int processed = 0, numevents;
+
+    /* Nothing to do? return ASAP */
+    if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0;
+
+    /* Note that we want call select() even if there are no
+     * file events to process as long as we want to process time
+     * events, in order to sleep until the next time event is ready
+     * to fire. */
+    if (eventLoop->maxfd != -1 ||
+        ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) {
+        int j;
+        aeTimeEvent *shortest = NULL;
+        struct timeval tv, *tvp;
+
+        if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT))
+            shortest = aeSearchNearestTimer(eventLoop);
+        if (shortest) {
+            long now_sec, now_ms;
+
+            aeGetTime(&now_sec, &now_ms);
+            tvp = &tv;
+
+            /* How many milliseconds we need to wait for the next
+             * time event to fire? */
+            long long ms =
+                (shortest->when_sec - now_sec)*1000 +
+                shortest->when_ms - now_ms;
+
+            if (ms > 0) {
+                tvp->tv_sec = ms/1000;
+                tvp->tv_usec = (ms % 1000)*1000;
+            } else {
+                tvp->tv_sec = 0;
+                tvp->tv_usec = 0;
+            }
+        } else {
+            /* If we have to check for events but need to return
+             * ASAP because of AE_DONT_WAIT we need to set the timeout
+             * to zero */
+            if (flags & AE_DONT_WAIT) {
+                tv.tv_sec = tv.tv_usec = 0;
+                tvp = &tv;
+            } else {
+                /* Otherwise we can block */
+                tvp = NULL; /* wait forever */
+            }
+        }
+
+        numevents = aeApiPoll(eventLoop, tvp);
+        for (j = 0; j < numevents; j++) {
+            aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd];
+            int mask = eventLoop->fired[j].mask;
+            int fd = eventLoop->fired[j].fd;
+            int rfired = 0;
+
+	    /* note the fe->mask & mask & ... code: maybe an already processed
+             * event removed an element that fired and we still didn't
+             * processed, so we check if the event is still valid. */
+            if (fe->mask & mask & AE_READABLE) {
+                rfired = 1;
+                fe->rfileProc(eventLoop,fd,fe->clientData,mask);
+            }
+            if (fe->mask & mask & AE_WRITABLE) {
+                if (!rfired || fe->wfileProc != fe->rfileProc)
+                    fe->wfileProc(eventLoop,fd,fe->clientData,mask);
+            }
+            processed++;
+        }
+    }
+    /* Check time events */
+    if (flags & AE_TIME_EVENTS)
+        processed += processTimeEvents(eventLoop);
+
+    return processed; /* return the number of processed file/time events */
+}
+
+/* Wait for milliseconds until the given file descriptor becomes
+ * writable/readable/exception */
+int aeWait(int fd, int mask, long long milliseconds) {
+    struct pollfd pfd;
+    int retmask = 0, retval;
+
+    memset(&pfd, 0, sizeof(pfd));
+    pfd.fd = fd;
+    if (mask & AE_READABLE) pfd.events |= POLLIN;
+    if (mask & AE_WRITABLE) pfd.events |= POLLOUT;
+
+    if ((retval = poll(&pfd, 1, milliseconds))== 1) {
+        if (pfd.revents & POLLIN) retmask |= AE_READABLE;
+        if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE;
+	if (pfd.revents & POLLERR) retmask |= AE_WRITABLE;
+        if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE;
+        return retmask;
+    } else {
+        return retval;
+    }
+}
+
+void aeMain(aeEventLoop *eventLoop) {
+    eventLoop->stop = 0;
+    while (!eventLoop->stop) {
+        if (eventLoop->beforesleep != NULL)
+            eventLoop->beforesleep(eventLoop);
+        aeProcessEvents(eventLoop, AE_ALL_EVENTS);
+    }
+}
+
+char *aeGetApiName(void) {
+    return aeApiName();
+}
+
+void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) {
+    eventLoop->beforesleep = beforesleep;
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae.h b/cpp/src/plasma/thirdparty/ae/ae.h
new file mode 100644
index 0000000000000..827c4c9e4e59e
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae.h
@@ -0,0 +1,123 @@
+/* A simple event-driven programming library. Originally I wrote this code
+ * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
+ * it in form of a library for easy reuse.
+ *
+ * Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __AE_H__
+#define __AE_H__
+
+#include <time.h>
+
+#define AE_OK 0
+#define AE_ERR -1
+
+#define AE_NONE 0
+#define AE_READABLE 1
+#define AE_WRITABLE 2
+
+#define AE_FILE_EVENTS 1
+#define AE_TIME_EVENTS 2
+#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
+#define AE_DONT_WAIT 4
+
+#define AE_NOMORE -1
+#define AE_DELETED_EVENT_ID -1
+
+/* Macros */
+#define AE_NOTUSED(V) ((void) V)
+
+struct aeEventLoop;
+
+/* Types and data structures */
+typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
+typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData);
+typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData);
+typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop);
+
+/* File event structure */
+typedef struct aeFileEvent {
+    int mask; /* one of AE_(READABLE|WRITABLE) */
+    aeFileProc *rfileProc;
+    aeFileProc *wfileProc;
+    void *clientData;
+} aeFileEvent;
+
+/* Time event structure */
+typedef struct aeTimeEvent {
+    long long id; /* time event identifier. */
+    long when_sec; /* seconds */
+    long when_ms; /* milliseconds */
+    aeTimeProc *timeProc;
+    aeEventFinalizerProc *finalizerProc;
+    void *clientData;
+    struct aeTimeEvent *next;
+} aeTimeEvent;
+
+/* A fired event */
+typedef struct aeFiredEvent {
+    int fd;
+    int mask;
+} aeFiredEvent;
+
+/* State of an event based program */
+typedef struct aeEventLoop {
+    int maxfd;   /* highest file descriptor currently registered */
+    int setsize; /* max number of file descriptors tracked */
+    long long timeEventNextId;
+    time_t lastTime;     /* Used to detect system clock skew */
+    aeFileEvent *events; /* Registered events */
+    aeFiredEvent *fired; /* Fired events */
+    aeTimeEvent *timeEventHead;
+    int stop;
+    void *apidata; /* This is used for polling API specific data */
+    aeBeforeSleepProc *beforesleep;
+} aeEventLoop;
+
+/* Prototypes */
+aeEventLoop *aeCreateEventLoop(int setsize);
+void aeDeleteEventLoop(aeEventLoop *eventLoop);
+void aeStop(aeEventLoop *eventLoop);
+int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
+        aeFileProc *proc, void *clientData);
+void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask);
+int aeGetFileEvents(aeEventLoop *eventLoop, int fd);
+long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
+        aeTimeProc *proc, void *clientData,
+        aeEventFinalizerProc *finalizerProc);
+int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id);
+int aeProcessEvents(aeEventLoop *eventLoop, int flags);
+int aeWait(int fd, int mask, long long milliseconds);
+void aeMain(aeEventLoop *eventLoop);
+char *aeGetApiName(void);
+void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
+int aeGetSetSize(aeEventLoop *eventLoop);
+int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
+
+#endif
diff --git a/cpp/src/plasma/thirdparty/ae/ae_epoll.c b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
new file mode 100644
index 0000000000000..410aac70dc5af
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
@@ -0,0 +1,135 @@
+/* Linux epoll(2) based ae.c module
+ *
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/epoll.h>
+
+typedef struct aeApiState {
+    int epfd;
+    struct epoll_event *events;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+    aeApiState *state = zmalloc(sizeof(aeApiState));
+
+    if (!state) return -1;
+    state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize);
+    if (!state->events) {
+        zfree(state);
+        return -1;
+    }
+    state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */
+    if (state->epfd == -1) {
+        zfree(state->events);
+        zfree(state);
+        return -1;
+    }
+    eventLoop->apidata = state;
+    return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    aeApiState *state = eventLoop->apidata;
+
+    state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize);
+    return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+    aeApiState *state = eventLoop->apidata;
+
+    close(state->epfd);
+    zfree(state->events);
+    zfree(state);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    struct epoll_event ee = {0}; /* avoid valgrind warning */
+    /* If the fd was already monitored for some event, we need a MOD
+     * operation. Otherwise we need an ADD operation. */
+    int op = eventLoop->events[fd].mask == AE_NONE ?
+            EPOLL_CTL_ADD : EPOLL_CTL_MOD;
+
+    ee.events = 0;
+    mask |= eventLoop->events[fd].mask; /* Merge old events */
+    if (mask & AE_READABLE) ee.events |= EPOLLIN;
+    if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
+    ee.data.fd = fd;
+    if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1;
+    return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
+    aeApiState *state = eventLoop->apidata;
+    struct epoll_event ee = {0}; /* avoid valgrind warning */
+    int mask = eventLoop->events[fd].mask & (~delmask);
+
+    ee.events = 0;
+    if (mask & AE_READABLE) ee.events |= EPOLLIN;
+    if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
+    ee.data.fd = fd;
+    if (mask != AE_NONE) {
+        epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee);
+    } else {
+        /* Note, Kernel < 2.6.9 requires a non null event pointer even for
+         * EPOLL_CTL_DEL. */
+        epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee);
+    }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+    aeApiState *state = eventLoop->apidata;
+    int retval, numevents = 0;
+
+    retval = epoll_wait(state->epfd,state->events,eventLoop->setsize,
+            tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
+    if (retval > 0) {
+        int j;
+
+        numevents = retval;
+        for (j = 0; j < numevents; j++) {
+            int mask = 0;
+            struct epoll_event *e = state->events+j;
+
+            if (e->events & EPOLLIN) mask |= AE_READABLE;
+            if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
+            if (e->events & EPOLLERR) mask |= AE_WRITABLE;
+            if (e->events & EPOLLHUP) mask |= AE_WRITABLE;
+            eventLoop->fired[j].fd = e->data.fd;
+            eventLoop->fired[j].mask = mask;
+        }
+    }
+    return numevents;
+}
+
+static char *aeApiName(void) {
+    return "epoll";
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_evport.c b/cpp/src/plasma/thirdparty/ae/ae_evport.c
new file mode 100644
index 0000000000000..5c317becb6f7d
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_evport.c
@@ -0,0 +1,320 @@
+/* ae.c module for illumos event ports.
+ *
+ * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <assert.h>
+#include <errno.h>
+#include <port.h>
+#include <poll.h>
+
+#include <sys/types.h>
+#include <sys/time.h>
+
+#include <stdio.h>
+
+static int evport_debug = 0;
+
+/*
+ * This file implements the ae API using event ports, present on Solaris-based
+ * systems since Solaris 10.  Using the event port interface, we associate file
+ * descriptors with the port.  Each association also includes the set of poll(2)
+ * events that the consumer is interested in (e.g., POLLIN and POLLOUT).
+ *
+ * There's one tricky piece to this implementation: when we return events via
+ * aeApiPoll, the corresponding file descriptors become dissociated from the
+ * port.  This is necessary because poll events are level-triggered, so if the
+ * fd didn't become dissociated, it would immediately fire another event since
+ * the underlying state hasn't changed yet.  We must re-associate the file
+ * descriptor, but only after we know that our caller has actually read from it.
+ * The ae API does not tell us exactly when that happens, but we do know that
+ * it must happen by the time aeApiPoll is called again.  Our solution is to
+ * keep track of the last fds returned by aeApiPoll and re-associate them next
+ * time aeApiPoll is invoked.
+ *
+ * To summarize, in this module, each fd association is EITHER (a) represented
+ * only via the in-kernel association OR (b) represented by pending_fds and
+ * pending_masks.  (b) is only true for the last fds we returned from aeApiPoll,
+ * and only until we enter aeApiPoll again (at which point we restore the
+ * in-kernel association).
+ */
+#define MAX_EVENT_BATCHSZ 512
+
+typedef struct aeApiState {
+    int     portfd;                             /* event port */
+    int     npending;                           /* # of pending fds */
+    int     pending_fds[MAX_EVENT_BATCHSZ];     /* pending fds */
+    int     pending_masks[MAX_EVENT_BATCHSZ];   /* pending fds' masks */
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+    int i;
+    aeApiState *state = zmalloc(sizeof(aeApiState));
+    if (!state) return -1;
+
+    state->portfd = port_create();
+    if (state->portfd == -1) {
+        zfree(state);
+        return -1;
+    }
+
+    state->npending = 0;
+
+    for (i = 0; i < MAX_EVENT_BATCHSZ; i++) {
+        state->pending_fds[i] = -1;
+        state->pending_masks[i] = AE_NONE;
+    }
+
+    eventLoop->apidata = state;
+    return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    /* Nothing to resize here. */
+    return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+    aeApiState *state = eventLoop->apidata;
+
+    close(state->portfd);
+    zfree(state);
+}
+
+static int aeApiLookupPending(aeApiState *state, int fd) {
+    int i;
+
+    for (i = 0; i < state->npending; i++) {
+        if (state->pending_fds[i] == fd)
+            return (i);
+    }
+
+    return (-1);
+}
+
+/*
+ * Helper function to invoke port_associate for the given fd and mask.
+ */
+static int aeApiAssociate(const char *where, int portfd, int fd, int mask) {
+    int events = 0;
+    int rv, err;
+
+    if (mask & AE_READABLE)
+        events |= POLLIN;
+    if (mask & AE_WRITABLE)
+        events |= POLLOUT;
+
+    if (evport_debug)
+        fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events);
+
+    rv = port_associate(portfd, PORT_SOURCE_FD, fd, events,
+        (void *)(uintptr_t)mask);
+    err = errno;
+
+    if (evport_debug)
+        fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err));
+
+    if (rv == -1) {
+        fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err));
+
+        if (err == EAGAIN)
+            fprintf(stderr, "aeApiAssociate: event port limit exceeded.");
+    }
+
+    return rv;
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    int fullmask, pfd;
+
+    if (evport_debug)
+        fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask);
+
+    /*
+     * Since port_associate's "events" argument replaces any existing events, we
+     * must be sure to include whatever events are already associated when
+     * we call port_associate() again.
+     */
+    fullmask = mask | eventLoop->events[fd].mask;
+    pfd = aeApiLookupPending(state, fd);
+
+    if (pfd != -1) {
+        /*
+         * This fd was recently returned from aeApiPoll.  It should be safe to
+         * assume that the consumer has processed that poll event, but we play
+         * it safer by simply updating pending_mask.  The fd will be
+         * re-associated as usual when aeApiPoll is called again.
+         */
+        if (evport_debug)
+            fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd);
+        state->pending_masks[pfd] |= fullmask;
+        return 0;
+    }
+
+    return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask));
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    int fullmask, pfd;
+
+    if (evport_debug)
+        fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask);
+
+    pfd = aeApiLookupPending(state, fd);
+
+    if (pfd != -1) {
+        if (evport_debug)
+            fprintf(stderr, "deleting event from pending fd %d\n", fd);
+
+        /*
+         * This fd was just returned from aeApiPoll, so it's not currently
+         * associated with the port.  All we need to do is update
+         * pending_mask appropriately.
+         */
+        state->pending_masks[pfd] &= ~mask;
+
+        if (state->pending_masks[pfd] == AE_NONE)
+            state->pending_fds[pfd] = -1;
+
+        return;
+    }
+
+    /*
+     * The fd is currently associated with the port.  Like with the add case
+     * above, we must look at the full mask for the file descriptor before
+     * updating that association.  We don't have a good way of knowing what the
+     * events are without looking into the eventLoop state directly.  We rely on
+     * the fact that our caller has already updated the mask in the eventLoop.
+     */
+
+    fullmask = eventLoop->events[fd].mask;
+    if (fullmask == AE_NONE) {
+        /*
+         * We're removing *all* events, so use port_dissociate to remove the
+         * association completely.  Failure here indicates a bug.
+         */
+        if (evport_debug)
+            fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd);
+
+        if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) {
+            perror("aeApiDelEvent: port_dissociate");
+            abort(); /* will not return */
+        }
+    } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd,
+        fullmask) != 0) {
+        /*
+         * ENOMEM is a potentially transient condition, but the kernel won't
+         * generally return it unless things are really bad.  EAGAIN indicates
+         * we've reached an resource limit, for which it doesn't make sense to
+         * retry (counter-intuitively).  All other errors indicate a bug.  In any
+         * of these cases, the best we can do is to abort.
+         */
+        abort(); /* will not return */
+    }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+    aeApiState *state = eventLoop->apidata;
+    struct timespec timeout, *tsp;
+    int mask, i;
+    uint_t nevents;
+    port_event_t event[MAX_EVENT_BATCHSZ];
+
+    /*
+     * If we've returned fd events before, we must re-associate them with the
+     * port now, before calling port_get().  See the block comment at the top of
+     * this file for an explanation of why.
+     */
+    for (i = 0; i < state->npending; i++) {
+        if (state->pending_fds[i] == -1)
+            /* This fd has since been deleted. */
+            continue;
+
+        if (aeApiAssociate("aeApiPoll", state->portfd,
+            state->pending_fds[i], state->pending_masks[i]) != 0) {
+            /* See aeApiDelEvent for why this case is fatal. */
+            abort();
+        }
+
+        state->pending_masks[i] = AE_NONE;
+        state->pending_fds[i] = -1;
+    }
+
+    state->npending = 0;
+
+    if (tvp != NULL) {
+        timeout.tv_sec = tvp->tv_sec;
+        timeout.tv_nsec = tvp->tv_usec * 1000;
+        tsp = &timeout;
+    } else {
+        tsp = NULL;
+    }
+
+    /*
+     * port_getn can return with errno == ETIME having returned some events (!).
+     * So if we get ETIME, we check nevents, too.
+     */
+    nevents = 1;
+    if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents,
+        tsp) == -1 && (errno != ETIME || nevents == 0)) {
+        if (errno == ETIME || errno == EINTR)
+            return 0;
+
+        /* Any other error indicates a bug. */
+        perror("aeApiPoll: port_get");
+        abort();
+    }
+
+    state->npending = nevents;
+
+    for (i = 0; i < nevents; i++) {
+            mask = 0;
+            if (event[i].portev_events & POLLIN)
+                mask |= AE_READABLE;
+            if (event[i].portev_events & POLLOUT)
+                mask |= AE_WRITABLE;
+
+            eventLoop->fired[i].fd = event[i].portev_object;
+            eventLoop->fired[i].mask = mask;
+
+            if (evport_debug)
+                fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n",
+                    (int)event[i].portev_object, mask);
+
+            state->pending_fds[i] = event[i].portev_object;
+            state->pending_masks[i] = (uintptr_t)event[i].portev_user;
+    }
+
+    return nevents;
+}
+
+static char *aeApiName(void) {
+    return "evport";
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_kqueue.c b/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
new file mode 100644
index 0000000000000..6796f4ceb5939
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
@@ -0,0 +1,138 @@
+/* Kqueue(2)-based ae.c module
+ *
+ * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/time.h>
+
+typedef struct aeApiState {
+    int kqfd;
+    struct kevent *events;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+    aeApiState *state = zmalloc(sizeof(aeApiState));
+
+    if (!state) return -1;
+    state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize);
+    if (!state->events) {
+        zfree(state);
+        return -1;
+    }
+    state->kqfd = kqueue();
+    if (state->kqfd == -1) {
+        zfree(state->events);
+        zfree(state);
+        return -1;
+    }
+    eventLoop->apidata = state;
+    return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    aeApiState *state = eventLoop->apidata;
+
+    state->events = zrealloc(state->events, sizeof(struct kevent)*setsize);
+    return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+    aeApiState *state = eventLoop->apidata;
+
+    close(state->kqfd);
+    zfree(state->events);
+    zfree(state);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    struct kevent ke;
+
+    if (mask & AE_READABLE) {
+        EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
+        if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
+    }
+    if (mask & AE_WRITABLE) {
+        EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL);
+        if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
+    }
+    return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+    struct kevent ke;
+
+    if (mask & AE_READABLE) {
+        EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
+        kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
+    }
+    if (mask & AE_WRITABLE) {
+        EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
+        kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
+    }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+    aeApiState *state = eventLoop->apidata;
+    int retval, numevents = 0;
+
+    if (tvp != NULL) {
+        struct timespec timeout;
+        timeout.tv_sec = tvp->tv_sec;
+        timeout.tv_nsec = tvp->tv_usec * 1000;
+        retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
+                        &timeout);
+    } else {
+        retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
+                        NULL);
+    }
+
+    if (retval > 0) {
+        int j;
+
+        numevents = retval;
+        for(j = 0; j < numevents; j++) {
+            int mask = 0;
+            struct kevent *e = state->events+j;
+
+            if (e->filter == EVFILT_READ) mask |= AE_READABLE;
+            if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
+            eventLoop->fired[j].fd = e->ident;
+            eventLoop->fired[j].mask = mask;
+        }
+    }
+    return numevents;
+}
+
+static char *aeApiName(void) {
+    return "kqueue";
+}
diff --git a/cpp/src/plasma/thirdparty/ae/ae_select.c b/cpp/src/plasma/thirdparty/ae/ae_select.c
new file mode 100644
index 0000000000000..c039a8ea3128d
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_select.c
@@ -0,0 +1,106 @@
+/* Select()-based ae.c module.
+ *
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/select.h>
+#include <string.h>
+
+typedef struct aeApiState {
+    fd_set rfds, wfds;
+    /* We need to have a copy of the fd sets as it's not safe to reuse
+     * FD sets after select(). */
+    fd_set _rfds, _wfds;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+    aeApiState *state = zmalloc(sizeof(aeApiState));
+
+    if (!state) return -1;
+    FD_ZERO(&state->rfds);
+    FD_ZERO(&state->wfds);
+    eventLoop->apidata = state;
+    return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    /* Just ensure we have enough room in the fd_set type. */
+    if (setsize >= FD_SETSIZE) return -1;
+    return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+    zfree(eventLoop->apidata);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+
+    if (mask & AE_READABLE) FD_SET(fd,&state->rfds);
+    if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds);
+    return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+    aeApiState *state = eventLoop->apidata;
+
+    if (mask & AE_READABLE) FD_CLR(fd,&state->rfds);
+    if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds);
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+    aeApiState *state = eventLoop->apidata;
+    int retval, j, numevents = 0;
+
+    memcpy(&state->_rfds,&state->rfds,sizeof(fd_set));
+    memcpy(&state->_wfds,&state->wfds,sizeof(fd_set));
+
+    retval = select(eventLoop->maxfd+1,
+                &state->_rfds,&state->_wfds,NULL,tvp);
+    if (retval > 0) {
+        for (j = 0; j <= eventLoop->maxfd; j++) {
+            int mask = 0;
+            aeFileEvent *fe = &eventLoop->events[j];
+
+            if (fe->mask == AE_NONE) continue;
+            if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds))
+                mask |= AE_READABLE;
+            if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds))
+                mask |= AE_WRITABLE;
+            eventLoop->fired[numevents].fd = j;
+            eventLoop->fired[numevents].mask = mask;
+            numevents++;
+        }
+    }
+    return numevents;
+}
+
+static char *aeApiName(void) {
+    return "select";
+}
diff --git a/cpp/src/plasma/thirdparty/ae/config.h b/cpp/src/plasma/thirdparty/ae/config.h
new file mode 100644
index 0000000000000..4f8e1ea1bc38c
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/config.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __CONFIG_H
+#define __CONFIG_H
+
+#ifdef __APPLE__
+#include <AvailabilityMacros.h>
+#endif
+
+/* Test for polling API */
+#ifdef __linux__
+#define HAVE_EPOLL 1
+#endif
+
+#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__)
+#define HAVE_KQUEUE 1
+#endif
+
+#ifdef __sun
+#include <sys/feature_tests.h>
+#ifdef _DTRACE_VERSION
+#define HAVE_EVPORT 1
+#endif
+#endif
+
+
+#endif
diff --git a/cpp/src/plasma/thirdparty/ae/zmalloc.h b/cpp/src/plasma/thirdparty/ae/zmalloc.h
new file mode 100644
index 0000000000000..6c27dd4e5c3d3
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/zmalloc.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _ZMALLOC_H
+#define _ZMALLOC_H
+
+#ifndef zmalloc
+#define zmalloc malloc
+#endif
+
+#ifndef zfree
+#define zfree free
+#endif
+
+#ifndef zrealloc
+#define zrealloc realloc
+#endif
+
+#endif /* _ZMALLOC_H */
diff --git a/cpp/src/plasma/thirdparty/dlmalloc.c b/cpp/src/plasma/thirdparty/dlmalloc.c
new file mode 100644
index 0000000000000..84ccbd28fc4ec
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/dlmalloc.c
@@ -0,0 +1,6281 @@
+/*
+  This is a version (aka dlmalloc) of malloc/free/realloc written by
+  Doug Lea and released to the public domain, as explained at
+  http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
+  comments, complaints, performance data, etc to dl@cs.oswego.edu
+
+* Version 2.8.6 Wed Aug 29 06:57:58 2012  Doug Lea
+   Note: There may be an updated version of this malloc obtainable at
+           ftp://gee.cs.oswego.edu/pub/misc/malloc.c
+         Check before installing!
+
+* Quickstart
+
+  This library is all in one file to simplify the most common usage:
+  ftp it, compile it (-O3), and link it into another program. All of
+  the compile-time options default to reasonable values for use on
+  most platforms.  You might later want to step through various
+  compile-time and dynamic tuning options.
+
+  For convenience, an include file for code using this malloc is at:
+     ftp://gee.cs.oswego.edu/pub/misc/malloc-2.8.6.h
+  You don't really need this .h file unless you call functions not
+  defined in your system include files.  The .h file contains only the
+  excerpts from this file needed for using this malloc on ANSI C/C++
+  systems, so long as you haven't changed compile-time options about
+  naming and tuning parameters.  If you do, then you can create your
+  own malloc.h that does include all settings by cutting at the point
+  indicated below. Note that you may already by default be using a C
+  library containing a malloc that is based on some version of this
+  malloc (for example in linux). You might still want to use the one
+  in this file to customize settings or to avoid overheads associated
+  with library versions.
+
+* Vital statistics:
+
+  Supported pointer/size_t representation:       4 or 8 bytes
+       size_t MUST be an unsigned type of the same width as
+       pointers. (If you are using an ancient system that declares
+       size_t as a signed type, or need it to be a different width
+       than pointers, you can use a previous release of this malloc
+       (e.g. 2.7.2) supporting these.)
+
+  Alignment:                                     8 bytes (minimum)
+       This suffices for nearly all current machines and C compilers.
+       However, you can define MALLOC_ALIGNMENT to be wider than this
+       if necessary (up to 128bytes), at the expense of using more space.
+
+  Minimum overhead per allocated chunk:   4 or  8 bytes (if 4byte sizes)
+                                          8 or 16 bytes (if 8byte sizes)
+       Each malloced chunk has a hidden word of overhead holding size
+       and status information, and additional cross-check word
+       if FOOTERS is defined.
+
+  Minimum allocated size: 4-byte ptrs:  16 bytes    (including overhead)
+                          8-byte ptrs:  32 bytes    (including overhead)
+
+       Even a request for zero bytes (i.e., malloc(0)) returns a
+       pointer to something of the minimum allocatable size.
+       The maximum overhead wastage (i.e., number of extra bytes
+       allocated than were requested in malloc) is less than or equal
+       to the minimum size, except for requests >= mmap_threshold that
+       are serviced via mmap(), where the worst case wastage is about
+       32 bytes plus the remainder from a system page (the minimal
+       mmap unit); typically 4096 or 8192 bytes.
+
+  Security: static-safe; optionally more or less
+       The "security" of malloc refers to the ability of malicious
+       code to accentuate the effects of errors (for example, freeing
+       space that is not currently malloc'ed or overwriting past the
+       ends of chunks) in code that calls malloc.  This malloc
+       guarantees not to modify any memory locations below the base of
+       heap, i.e., static variables, even in the presence of usage
+       errors.  The routines additionally detect most improper frees
+       and reallocs.  All this holds as long as the static bookkeeping
+       for malloc itself is not corrupted by some other means.  This
+       is only one aspect of security -- these checks do not, and
+       cannot, detect all possible programming errors.
+
+       If FOOTERS is defined nonzero, then each allocated chunk
+       carries an additional check word to verify that it was malloced
+       from its space.  These check words are the same within each
+       execution of a program using malloc, but differ across
+       executions, so externally crafted fake chunks cannot be
+       freed. This improves security by rejecting frees/reallocs that
+       could corrupt heap memory, in addition to the checks preventing
+       writes to statics that are always on.  This may further improve
+       security at the expense of time and space overhead.  (Note that
+       FOOTERS may also be worth using with MSPACES.)
+
+       By default detected errors cause the program to abort (calling
+       "abort()"). You can override this to instead proceed past
+       errors by defining PROCEED_ON_ERROR.  In this case, a bad free
+       has no effect, and a malloc that encounters a bad address
+       caused by user overwrites will ignore the bad address by
+       dropping pointers and indices to all known memory. This may
+       be appropriate for programs that should continue if at all
+       possible in the face of programming errors, although they may
+       run out of memory because dropped memory is never reclaimed.
+
+       If you don't like either of these options, you can define
+       CORRUPTION_ERROR_ACTION and USAGE_ERROR_ACTION to do anything
+       else. And if if you are sure that your program using malloc has
+       no errors or vulnerabilities, you can define INSECURE to 1,
+       which might (or might not) provide a small performance improvement.
+
+       It is also possible to limit the maximum total allocatable
+       space, using malloc_set_footprint_limit. This is not
+       designed as a security feature in itself (calls to set limits
+       are not screened or privileged), but may be useful as one
+       aspect of a secure implementation.
+
+  Thread-safety: NOT thread-safe unless USE_LOCKS defined non-zero
+       When USE_LOCKS is defined, each public call to malloc, free,
+       etc is surrounded with a lock. By default, this uses a plain
+       pthread mutex, win32 critical section, or a spin-lock if if
+       available for the platform and not disabled by setting
+       USE_SPIN_LOCKS=0.  However, if USE_RECURSIVE_LOCKS is defined,
+       recursive versions are used instead (which are not required for
+       base functionality but may be needed in layered extensions).
+       Using a global lock is not especially fast, and can be a major
+       bottleneck.  It is designed only to provide minimal protection
+       in concurrent environments, and to provide a basis for
+       extensions.  If you are using malloc in a concurrent program,
+       consider instead using nedmalloc
+       (http://www.nedprod.com/programs/portable/nedmalloc/) or
+       ptmalloc (See http://www.malloc.de), which are derived from
+       versions of this malloc.
+
+  System requirements: Any combination of MORECORE and/or MMAP/MUNMAP
+       This malloc can use unix sbrk or any emulation (invoked using
+       the CALL_MORECORE macro) and/or mmap/munmap or any emulation
+       (invoked using CALL_MMAP/CALL_MUNMAP) to get and release system
+       memory.  On most unix systems, it tends to work best if both
+       MORECORE and MMAP are enabled.  On Win32, it uses emulations
+       based on VirtualAlloc. It also uses common C library functions
+       like memset.
+
+  Compliance: I believe it is compliant with the Single Unix Specification
+       (See http://www.unix.org). Also SVID/XPG, ANSI C, and probably
+       others as well.
+
+* Overview of algorithms
+
+  This is not the fastest, most space-conserving, most portable, or
+  most tunable malloc ever written. However it is among the fastest
+  while also being among the most space-conserving, portable and
+  tunable.  Consistent balance across these factors results in a good
+  general-purpose allocator for malloc-intensive programs.
+
+  In most ways, this malloc is a best-fit allocator. Generally, it
+  chooses the best-fitting existing chunk for a request, with ties
+  broken in approximately least-recently-used order. (This strategy
+  normally maintains low fragmentation.) However, for requests less
+  than 256bytes, it deviates from best-fit when there is not an
+  exactly fitting available chunk by preferring to use space adjacent
+  to that used for the previous small request, as well as by breaking
+  ties in approximately most-recently-used order. (These enhance
+  locality of series of small allocations.)  And for very large requests
+  (>= 256Kb by default), it relies on system memory mapping
+  facilities, if supported.  (This helps avoid carrying around and
+  possibly fragmenting memory used only for large chunks.)
+
+  All operations (except malloc_stats and mallinfo) have execution
+  times that are bounded by a constant factor of the number of bits in
+  a size_t, not counting any clearing in calloc or copying in realloc,
+  or actions surrounding MORECORE and MMAP that have times
+  proportional to the number of non-contiguous regions returned by
+  system allocation routines, which is often just 1. In real-time
+  applications, you can optionally suppress segment traversals using
+  NO_SEGMENT_TRAVERSAL, which assures bounded execution even when
+  system allocators return non-contiguous spaces, at the typical
+  expense of carrying around more memory and increased fragmentation.
+
+  The implementation is not very modular and seriously overuses
+  macros. Perhaps someday all C compilers will do as good a job
+  inlining modular code as can now be done by brute-force expansion,
+  but now, enough of them seem not to.
+
+  Some compilers issue a lot of warnings about code that is
+  dead/unreachable only on some platforms, and also about intentional
+  uses of negation on unsigned types. All known cases of each can be
+  ignored.
+
+  For a longer but out of date high-level description, see
+     http://gee.cs.oswego.edu/dl/html/malloc.html
+
+* MSPACES
+  If MSPACES is defined, then in addition to malloc, free, etc.,
+  this file also defines mspace_malloc, mspace_free, etc. These
+  are versions of malloc routines that take an "mspace" argument
+  obtained using create_mspace, to control all internal bookkeeping.
+  If ONLY_MSPACES is defined, only these versions are compiled.
+  So if you would like to use this allocator for only some allocations,
+  and your system malloc for others, you can compile with
+  ONLY_MSPACES and then do something like...
+    static mspace mymspace = create_mspace(0,0); // for example
+    #define mymalloc(bytes)  mspace_malloc(mymspace, bytes)
+
+  (Note: If you only need one instance of an mspace, you can instead
+  use "USE_DL_PREFIX" to relabel the global malloc.)
+
+  You can similarly create thread-local allocators by storing
+  mspaces as thread-locals. For example:
+    static __thread mspace tlms = 0;
+    void*  tlmalloc(size_t bytes) {
+      if (tlms == 0) tlms = create_mspace(0, 0);
+      return mspace_malloc(tlms, bytes);
+    }
+    void  tlfree(void* mem) { mspace_free(tlms, mem); }
+
+  Unless FOOTERS is defined, each mspace is completely independent.
+  You cannot allocate from one and free to another (although
+  conformance is only weakly checked, so usage errors are not always
+  caught). If FOOTERS is defined, then each chunk carries around a tag
+  indicating its originating mspace, and frees are directed to their
+  originating spaces. Normally, this requires use of locks.
+
+ -------------------------  Compile-time options ---------------------------
+
+Be careful in setting #define values for numerical constants of type
+size_t. On some systems, literal values are not automatically extended
+to size_t precision unless they are explicitly casted. You can also
+use the symbolic values MAX_SIZE_T, SIZE_T_ONE, etc below.
+
+WIN32                    default: defined if _WIN32 defined
+  Defining WIN32 sets up defaults for MS environment and compilers.
+  Otherwise defaults are for unix. Beware that there seem to be some
+  cases where this malloc might not be a pure drop-in replacement for
+  Win32 malloc: Random-looking failures from Win32 GDI API's (eg;
+  SetDIBits()) may be due to bugs in some video driver implementations
+  when pixel buffers are malloc()ed, and the region spans more than
+  one VirtualAlloc()ed region. Because dlmalloc uses a small (64Kb)
+  default granularity, pixel buffers may straddle virtual allocation
+  regions more often than when using the Microsoft allocator.  You can
+  avoid this by using VirtualAlloc() and VirtualFree() for all pixel
+  buffers rather than using malloc().  If this is not possible,
+  recompile this malloc with a larger DEFAULT_GRANULARITY. Note:
+  in cases where MSC and gcc (cygwin) are known to differ on WIN32,
+  conditions use _MSC_VER to distinguish them.
+
+DLMALLOC_EXPORT       default: extern
+  Defines how public APIs are declared. If you want to export via a
+  Windows DLL, you might define this as
+    #define DLMALLOC_EXPORT extern  __declspec(dllexport)
+  If you want a POSIX ELF shared object, you might use
+    #define DLMALLOC_EXPORT extern __attribute__((visibility("default")))
+
+MALLOC_ALIGNMENT         default: (size_t)(2 * sizeof(void *))
+  Controls the minimum alignment for malloc'ed chunks.  It must be a
+  power of two and at least 8, even on machines for which smaller
+  alignments would suffice. It may be defined as larger than this
+  though. Note however that code and data structures are optimized for
+  the case of 8-byte alignment.
+
+MSPACES                  default: 0 (false)
+  If true, compile in support for independent allocation spaces.
+  This is only supported if HAVE_MMAP is true.
+
+ONLY_MSPACES             default: 0 (false)
+  If true, only compile in mspace versions, not regular versions.
+
+USE_LOCKS                default: 0 (false)
+  Causes each call to each public routine to be surrounded with
+  pthread or WIN32 mutex lock/unlock. (If set true, this can be
+  overridden on a per-mspace basis for mspace versions.) If set to a
+  non-zero value other than 1, locks are used, but their
+  implementation is left out, so lock functions must be supplied manually,
+  as described below.
+
+USE_SPIN_LOCKS           default: 1 iff USE_LOCKS and spin locks available
+  If true, uses custom spin locks for locking. This is currently
+  supported only gcc >= 4.1, older gccs on x86 platforms, and recent
+  MS compilers.  Otherwise, posix locks or win32 critical sections are
+  used.
+
+USE_RECURSIVE_LOCKS      default: not defined
+  If defined nonzero, uses recursive (aka reentrant) locks, otherwise
+  uses plain mutexes. This is not required for malloc proper, but may
+  be needed for layered allocators such as nedmalloc.
+
+LOCK_AT_FORK            default: not defined
+  If defined nonzero, performs pthread_atfork upon initialization
+  to initialize child lock while holding parent lock. The implementation
+  assumes that pthread locks (not custom locks) are being used. In other
+  cases, you may need to customize the implementation.
+
+FOOTERS                  default: 0
+  If true, provide extra checking and dispatching by placing
+  information in the footers of allocated chunks. This adds
+  space and time overhead.
+
+INSECURE                 default: 0
+  If true, omit checks for usage errors and heap space overwrites.
+
+USE_DL_PREFIX            default: NOT defined
+  Causes compiler to prefix all public routines with the string 'dl'.
+  This can be useful when you only want to use this malloc in one part
+  of a program, using your regular system malloc elsewhere.
+
+MALLOC_INSPECT_ALL       default: NOT defined
+  If defined, compiles malloc_inspect_all and mspace_inspect_all, that
+  perform traversal of all heap space.  Unless access to these
+  functions is otherwise restricted, you probably do not want to
+  include them in secure implementations.
+
+ABORT                    default: defined as abort()
+  Defines how to abort on failed checks.  On most systems, a failed
+  check cannot die with an "assert" or even print an informative
+  message, because the underlying print routines in turn call malloc,
+  which will fail again.  Generally, the best policy is to simply call
+  abort(). It's not very useful to do more than this because many
+  errors due to overwriting will show up as address faults (null, odd
+  addresses etc) rather than malloc-triggered checks, so will also
+  abort.  Also, most compilers know that abort() does not return, so
+  can better optimize code conditionally calling it.
+
+PROCEED_ON_ERROR           default: defined as 0 (false)
+  Controls whether detected bad addresses cause them to bypassed
+  rather than aborting. If set, detected bad arguments to free and
+  realloc are ignored. And all bookkeeping information is zeroed out
+  upon a detected overwrite of freed heap space, thus losing the
+  ability to ever return it from malloc again, but enabling the
+  application to proceed. If PROCEED_ON_ERROR is defined, the
+  static variable malloc_corruption_error_count is compiled in
+  and can be examined to see if errors have occurred. This option
+  generates slower code than the default abort policy.
+
+DEBUG                    default: NOT defined
+  The DEBUG setting is mainly intended for people trying to modify
+  this code or diagnose problems when porting to new platforms.
+  However, it may also be able to better isolate user errors than just
+  using runtime checks.  The assertions in the check routines spell
+  out in more detail the assumptions and invariants underlying the
+  algorithms.  The checking is fairly extensive, and will slow down
+  execution noticeably. Calling malloc_stats or mallinfo with DEBUG
+  set will attempt to check every non-mmapped allocated and free chunk
+  in the course of computing the summaries.
+
+ABORT_ON_ASSERT_FAILURE   default: defined as 1 (true)
+  Debugging assertion failures can be nearly impossible if your
+  version of the assert macro causes malloc to be called, which will
+  lead to a cascade of further failures, blowing the runtime stack.
+  ABORT_ON_ASSERT_FAILURE cause assertions failures to call abort(),
+  which will usually make debugging easier.
+
+MALLOC_FAILURE_ACTION     default: sets errno to ENOMEM, or no-op on win32
+  The action to take before "return 0" when malloc fails to be able to
+  return memory because there is none available.
+
+HAVE_MORECORE             default: 1 (true) unless win32 or ONLY_MSPACES
+  True if this system supports sbrk or an emulation of it.
+
+MORECORE                  default: sbrk
+  The name of the sbrk-style system routine to call to obtain more
+  memory.  See below for guidance on writing custom MORECORE
+  functions. The type of the argument to sbrk/MORECORE varies across
+  systems.  It cannot be size_t, because it supports negative
+  arguments, so it is normally the signed type of the same width as
+  size_t (sometimes declared as "intptr_t").  It doesn't much matter
+  though. Internally, we only call it with arguments less than half
+  the max value of a size_t, which should work across all reasonable
+  possibilities, although sometimes generating compiler warnings.
+
+MORECORE_CONTIGUOUS       default: 1 (true) if HAVE_MORECORE
+  If true, take advantage of fact that consecutive calls to MORECORE
+  with positive arguments always return contiguous increasing
+  addresses.  This is true of unix sbrk. It does not hurt too much to
+  set it true anyway, since malloc copes with non-contiguities.
+  Setting it false when definitely non-contiguous saves time
+  and possibly wasted space it would take to discover this though.
+
+MORECORE_CANNOT_TRIM      default: NOT defined
+  True if MORECORE cannot release space back to the system when given
+  negative arguments. This is generally necessary only if you are
+  using a hand-crafted MORECORE function that cannot handle negative
+  arguments.
+
+NO_SEGMENT_TRAVERSAL       default: 0
+  If non-zero, suppresses traversals of memory segments
+  returned by either MORECORE or CALL_MMAP. This disables
+  merging of segments that are contiguous, and selectively
+  releasing them to the OS if unused, but bounds execution times.
+
+HAVE_MMAP                 default: 1 (true)
+  True if this system supports mmap or an emulation of it.  If so, and
+  HAVE_MORECORE is not true, MMAP is used for all system
+  allocation. If set and HAVE_MORECORE is true as well, MMAP is
+  primarily used to directly allocate very large blocks. It is also
+  used as a backup strategy in cases where MORECORE fails to provide
+  space from system. Note: A single call to MUNMAP is assumed to be
+  able to unmap memory that may have be allocated using multiple calls
+  to MMAP, so long as they are adjacent.
+
+HAVE_MREMAP               default: 1 on linux, else 0
+  If true realloc() uses mremap() to re-allocate large blocks and
+  extend or shrink allocation spaces.
+
+MMAP_CLEARS               default: 1 except on WINCE.
+  True if mmap clears memory so calloc doesn't need to. This is true
+  for standard unix mmap using /dev/zero and on WIN32 except for WINCE.
+
+USE_BUILTIN_FFS            default: 0 (i.e., not used)
+  Causes malloc to use the builtin ffs() function to compute indices.
+  Some compilers may recognize and intrinsify ffs to be faster than the
+  supplied C version. Also, the case of x86 using gcc is special-cased
+  to an asm instruction, so is already as fast as it can be, and so
+  this setting has no effect. Similarly for Win32 under recent MS compilers.
+  (On most x86s, the asm version is only slightly faster than the C version.)
+
+malloc_getpagesize         default: derive from system includes, or 4096.
+  The system page size. To the extent possible, this malloc manages
+  memory from the system in page-size units.  This may be (and
+  usually is) a function rather than a constant. This is ignored
+  if WIN32, where page size is determined using getSystemInfo during
+  initialization.
+
+USE_DEV_RANDOM             default: 0 (i.e., not used)
+  Causes malloc to use /dev/random to initialize secure magic seed for
+  stamping footers. Otherwise, the current time is used.
+
+NO_MALLINFO                default: 0
+  If defined, don't compile "mallinfo". This can be a simple way
+  of dealing with mismatches between system declarations and
+  those in this file.
+
+MALLINFO_FIELD_TYPE        default: size_t
+  The type of the fields in the mallinfo struct. This was originally
+  defined as "int" in SVID etc, but is more usefully defined as
+  size_t. The value is used only if  HAVE_USR_INCLUDE_MALLOC_H is not set
+
+NO_MALLOC_STATS            default: 0
+  If defined, don't compile "malloc_stats". This avoids calls to
+  fprintf and bringing in stdio dependencies you might not want.
+
+REALLOC_ZERO_BYTES_FREES    default: not defined
+  This should be set if a call to realloc with zero bytes should
+  be the same as a call to free. Some people think it should. Otherwise,
+  since this malloc returns a unique pointer for malloc(0), so does
+  realloc(p, 0).
+
+LACKS_UNISTD_H, LACKS_FCNTL_H, LACKS_SYS_PARAM_H, LACKS_SYS_MMAN_H
+LACKS_STRINGS_H, LACKS_STRING_H, LACKS_SYS_TYPES_H,  LACKS_ERRNO_H
+LACKS_STDLIB_H LACKS_SCHED_H LACKS_TIME_H  default: NOT defined unless on WIN32
+  Define these if your system does not have these header files.
+  You might need to manually insert some of the declarations they provide.
+
+DEFAULT_GRANULARITY        default: page size if MORECORE_CONTIGUOUS,
+                                system_info.dwAllocationGranularity in WIN32,
+                                otherwise 64K.
+      Also settable using mallopt(M_GRANULARITY, x)
+  The unit for allocating and deallocating memory from the system.  On
+  most systems with contiguous MORECORE, there is no reason to
+  make this more than a page. However, systems with MMAP tend to
+  either require or encourage larger granularities.  You can increase
+  this value to prevent system allocation functions to be called so
+  often, especially if they are slow.  The value must be at least one
+  page and must be a power of two.  Setting to 0 causes initialization
+  to either page size or win32 region size.  (Note: In previous
+  versions of malloc, the equivalent of this option was called
+  "TOP_PAD")
+
+DEFAULT_TRIM_THRESHOLD    default: 2MB
+      Also settable using mallopt(M_TRIM_THRESHOLD, x)
+  The maximum amount of unused top-most memory to keep before
+  releasing via malloc_trim in free().  Automatic trimming is mainly
+  useful in long-lived programs using contiguous MORECORE.  Because
+  trimming via sbrk can be slow on some systems, and can sometimes be
+  wasteful (in cases where programs immediately afterward allocate
+  more large chunks) the value should be high enough so that your
+  overall system performance would improve by releasing this much
+  memory.  As a rough guide, you might set to a value close to the
+  average size of a process (program) running on your system.
+  Releasing this much memory would allow such a process to run in
+  memory.  Generally, it is worth tuning trim thresholds when a
+  program undergoes phases where several large chunks are allocated
+  and released in ways that can reuse each other's storage, perhaps
+  mixed with phases where there are no such chunks at all. The trim
+  value must be greater than page size to have any useful effect.  To
+  disable trimming completely, you can set to MAX_SIZE_T. Note that the trick
+  some people use of mallocing a huge space and then freeing it at
+  program startup, in an attempt to reserve system memory, doesn't
+  have the intended effect under automatic trimming, since that memory
+  will immediately be returned to the system.
+
+DEFAULT_MMAP_THRESHOLD       default: 256K
+      Also settable using mallopt(M_MMAP_THRESHOLD, x)
+  The request size threshold for using MMAP to directly service a
+  request. Requests of at least this size that cannot be allocated
+  using already-existing space will be serviced via mmap.  (If enough
+  normal freed space already exists it is used instead.)  Using mmap
+  segregates relatively large chunks of memory so that they can be
+  individually obtained and released from the host system. A request
+  serviced through mmap is never reused by any other request (at least
+  not directly; the system may just so happen to remap successive
+  requests to the same locations).  Segregating space in this way has
+  the benefits that: Mmapped space can always be individually released
+  back to the system, which helps keep the system level memory demands
+  of a long-lived program low.  Also, mapped memory doesn't become
+  `locked' between other chunks, as can happen with normally allocated
+  chunks, which means that even trimming via malloc_trim would not
+  release them.  However, it has the disadvantage that the space
+  cannot be reclaimed, consolidated, and then used to service later
+  requests, as happens with normal chunks.  The advantages of mmap
+  nearly always outweigh disadvantages for "large" chunks, but the
+  value of "large" may vary across systems.  The default is an
+  empirically derived value that works well in most systems. You can
+  disable mmap by setting to MAX_SIZE_T.
+
+MAX_RELEASE_CHECK_RATE   default: 4095 unless not HAVE_MMAP
+  The number of consolidated frees between checks to release
+  unused segments when freeing. When using non-contiguous segments,
+  especially with multiple mspaces, checking only for topmost space
+  doesn't always suffice to trigger trimming. To compensate for this,
+  free() will, with a period of MAX_RELEASE_CHECK_RATE (or the
+  current number of segments, if greater) try to release unused
+  segments to the OS when freeing chunks that result in
+  consolidation. The best value for this parameter is a compromise
+  between slowing down frees with relatively costly checks that
+  rarely trigger versus holding on to unused memory. To effectively
+  disable, set to MAX_SIZE_T. This may lead to a very slight speed
+  improvement at the expense of carrying around more memory.
+*/
+
+/* Version identifier to allow people to support multiple versions */
+#ifndef DLMALLOC_VERSION
+#define DLMALLOC_VERSION 20806
+#endif /* DLMALLOC_VERSION */
+
+#ifndef DLMALLOC_EXPORT
+#define DLMALLOC_EXPORT extern
+#endif
+
+#ifndef WIN32
+#ifdef _WIN32
+#define WIN32 1
+#endif  /* _WIN32 */
+#ifdef _WIN32_WCE
+#define LACKS_FCNTL_H
+#define WIN32 1
+#endif /* _WIN32_WCE */
+#endif  /* WIN32 */
+#ifdef WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <tchar.h>
+#define HAVE_MMAP 1
+#define HAVE_MORECORE 0
+#define LACKS_UNISTD_H
+#define LACKS_SYS_PARAM_H
+#define LACKS_SYS_MMAN_H
+#define LACKS_STRING_H
+#define LACKS_STRINGS_H
+#define LACKS_SYS_TYPES_H
+#define LACKS_ERRNO_H
+#define LACKS_SCHED_H
+#ifndef MALLOC_FAILURE_ACTION
+#define MALLOC_FAILURE_ACTION
+#endif /* MALLOC_FAILURE_ACTION */
+#ifndef MMAP_CLEARS
+#ifdef _WIN32_WCE /* WINCE reportedly does not clear */
+#define MMAP_CLEARS 0
+#else
+#define MMAP_CLEARS 1
+#endif /* _WIN32_WCE */
+#endif /*MMAP_CLEARS */
+#endif  /* WIN32 */
+
+#if defined(DARWIN) || defined(_DARWIN)
+/* Mac OSX docs advise not to use sbrk; it seems better to use mmap */
+#ifndef HAVE_MORECORE
+#define HAVE_MORECORE 0
+#define HAVE_MMAP 1
+/* OSX allocators provide 16 byte alignment */
+#ifndef MALLOC_ALIGNMENT
+#define MALLOC_ALIGNMENT ((size_t)16U)
+#endif
+#endif  /* HAVE_MORECORE */
+#endif  /* DARWIN */
+
+#ifndef LACKS_SYS_TYPES_H
+#include <sys/types.h>  /* For size_t */
+#endif  /* LACKS_SYS_TYPES_H */
+
+/* The maximum possible size_t value has all bits set */
+#define MAX_SIZE_T           (~(size_t)0)
+
+#ifndef USE_LOCKS /* ensure true if spin or recursive locks set */
+#define USE_LOCKS  ((defined(USE_SPIN_LOCKS) && USE_SPIN_LOCKS != 0) || \
+                    (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0))
+#endif /* USE_LOCKS */
+
+#if USE_LOCKS /* Spin locks for gcc >= 4.1, older gcc on x86, MSC >= 1310 */
+#if ((defined(__GNUC__) &&                                              \
+      ((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) ||      \
+       defined(__i386__) || defined(__x86_64__))) ||                    \
+     (defined(_MSC_VER) && _MSC_VER>=1310))
+#ifndef USE_SPIN_LOCKS
+#define USE_SPIN_LOCKS 1
+#endif /* USE_SPIN_LOCKS */
+#elif USE_SPIN_LOCKS
+#error "USE_SPIN_LOCKS defined without implementation"
+#endif /* ... locks available... */
+#elif !defined(USE_SPIN_LOCKS)
+#define USE_SPIN_LOCKS 0
+#endif /* USE_LOCKS */
+
+#ifndef ONLY_MSPACES
+#define ONLY_MSPACES 0
+#endif  /* ONLY_MSPACES */
+#ifndef MSPACES
+#if ONLY_MSPACES
+#define MSPACES 1
+#else   /* ONLY_MSPACES */
+#define MSPACES 0
+#endif  /* ONLY_MSPACES */
+#endif  /* MSPACES */
+#ifndef MALLOC_ALIGNMENT
+#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *)))
+#endif  /* MALLOC_ALIGNMENT */
+#ifndef FOOTERS
+#define FOOTERS 0
+#endif  /* FOOTERS */
+#ifndef ABORT
+#define ABORT  abort()
+#endif  /* ABORT */
+#ifndef ABORT_ON_ASSERT_FAILURE
+#define ABORT_ON_ASSERT_FAILURE 1
+#endif  /* ABORT_ON_ASSERT_FAILURE */
+#ifndef PROCEED_ON_ERROR
+#define PROCEED_ON_ERROR 0
+#endif  /* PROCEED_ON_ERROR */
+
+#ifndef INSECURE
+#define INSECURE 0
+#endif  /* INSECURE */
+#ifndef MALLOC_INSPECT_ALL
+#define MALLOC_INSPECT_ALL 0
+#endif  /* MALLOC_INSPECT_ALL */
+#ifndef HAVE_MMAP
+#define HAVE_MMAP 1
+#endif  /* HAVE_MMAP */
+#ifndef MMAP_CLEARS
+#define MMAP_CLEARS 1
+#endif  /* MMAP_CLEARS */
+#ifndef HAVE_MREMAP
+#ifdef linux
+#define HAVE_MREMAP 1
+#define _GNU_SOURCE /* Turns on mremap() definition */
+#else   /* linux */
+#define HAVE_MREMAP 0
+#endif  /* linux */
+#endif  /* HAVE_MREMAP */
+#ifndef MALLOC_FAILURE_ACTION
+#define MALLOC_FAILURE_ACTION  errno = ENOMEM;
+#endif  /* MALLOC_FAILURE_ACTION */
+#ifndef HAVE_MORECORE
+#if ONLY_MSPACES
+#define HAVE_MORECORE 0
+#else   /* ONLY_MSPACES */
+#define HAVE_MORECORE 1
+#endif  /* ONLY_MSPACES */
+#endif  /* HAVE_MORECORE */
+#if !HAVE_MORECORE
+#define MORECORE_CONTIGUOUS 0
+#else   /* !HAVE_MORECORE */
+#define MORECORE_DEFAULT sbrk
+#ifndef MORECORE_CONTIGUOUS
+#define MORECORE_CONTIGUOUS 1
+#endif  /* MORECORE_CONTIGUOUS */
+#endif  /* HAVE_MORECORE */
+#ifndef DEFAULT_GRANULARITY
+#if (MORECORE_CONTIGUOUS || defined(WIN32))
+#define DEFAULT_GRANULARITY (0)  /* 0 means to compute in init_mparams */
+#else   /* MORECORE_CONTIGUOUS */
+#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U)
+#endif  /* MORECORE_CONTIGUOUS */
+#endif  /* DEFAULT_GRANULARITY */
+#ifndef DEFAULT_TRIM_THRESHOLD
+#ifndef MORECORE_CANNOT_TRIM
+#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U)
+#else   /* MORECORE_CANNOT_TRIM */
+#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T
+#endif  /* MORECORE_CANNOT_TRIM */
+#endif  /* DEFAULT_TRIM_THRESHOLD */
+#ifndef DEFAULT_MMAP_THRESHOLD
+#if HAVE_MMAP
+#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U)
+#else   /* HAVE_MMAP */
+#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
+#endif  /* HAVE_MMAP */
+#endif  /* DEFAULT_MMAP_THRESHOLD */
+#ifndef MAX_RELEASE_CHECK_RATE
+#if HAVE_MMAP
+#define MAX_RELEASE_CHECK_RATE 4095
+#else
+#define MAX_RELEASE_CHECK_RATE MAX_SIZE_T
+#endif /* HAVE_MMAP */
+#endif /* MAX_RELEASE_CHECK_RATE */
+#ifndef USE_BUILTIN_FFS
+#define USE_BUILTIN_FFS 0
+#endif  /* USE_BUILTIN_FFS */
+#ifndef USE_DEV_RANDOM
+#define USE_DEV_RANDOM 0
+#endif  /* USE_DEV_RANDOM */
+#ifndef NO_MALLINFO
+#define NO_MALLINFO 0
+#endif  /* NO_MALLINFO */
+#ifndef MALLINFO_FIELD_TYPE
+#define MALLINFO_FIELD_TYPE size_t
+#endif  /* MALLINFO_FIELD_TYPE */
+#ifndef NO_MALLOC_STATS
+#define NO_MALLOC_STATS 0
+#endif  /* NO_MALLOC_STATS */
+#ifndef NO_SEGMENT_TRAVERSAL
+#define NO_SEGMENT_TRAVERSAL 0
+#endif /* NO_SEGMENT_TRAVERSAL */
+
+/*
+  mallopt tuning options.  SVID/XPG defines four standard parameter
+  numbers for mallopt, normally defined in malloc.h.  None of these
+  are used in this malloc, so setting them has no effect. But this
+  malloc does support the following options.
+*/
+
+#define M_TRIM_THRESHOLD     (-1)
+#define M_GRANULARITY        (-2)
+#define M_MMAP_THRESHOLD     (-3)
+
+/* ------------------------ Mallinfo declarations ------------------------ */
+
+#if !NO_MALLINFO
+/*
+  This version of malloc supports the standard SVID/XPG mallinfo
+  routine that returns a struct containing usage properties and
+  statistics. It should work on any system that has a
+  /usr/include/malloc.h defining struct mallinfo.  The main
+  declaration needed is the mallinfo struct that is returned (by-copy)
+  by mallinfo().  The malloinfo struct contains a bunch of fields that
+  are not even meaningful in this version of malloc.  These fields are
+  are instead filled by mallinfo() with other numbers that might be of
+  interest.
+
+  HAVE_USR_INCLUDE_MALLOC_H should be set if you have a
+  /usr/include/malloc.h file that includes a declaration of struct
+  mallinfo.  If so, it is included; else a compliant version is
+  declared below.  These must be precisely the same for mallinfo() to
+  work.  The original SVID version of this struct, defined on most
+  systems with mallinfo, declares all fields as ints. But some others
+  define as unsigned long. If your system defines the fields using a
+  type of different width than listed here, you MUST #include your
+  system version and #define HAVE_USR_INCLUDE_MALLOC_H.
+*/
+
+/* #define HAVE_USR_INCLUDE_MALLOC_H */
+
+#ifdef HAVE_USR_INCLUDE_MALLOC_H
+#include "/usr/include/malloc.h"
+#else /* HAVE_USR_INCLUDE_MALLOC_H */
+#ifndef STRUCT_MALLINFO_DECLARED
+/* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is defined */
+#define _STRUCT_MALLINFO
+#define STRUCT_MALLINFO_DECLARED 1
+struct mallinfo {
+  MALLINFO_FIELD_TYPE arena;    /* non-mmapped space allocated from system */
+  MALLINFO_FIELD_TYPE ordblks;  /* number of free chunks */
+  MALLINFO_FIELD_TYPE smblks;   /* always 0 */
+  MALLINFO_FIELD_TYPE hblks;    /* always 0 */
+  MALLINFO_FIELD_TYPE hblkhd;   /* space in mmapped regions */
+  MALLINFO_FIELD_TYPE usmblks;  /* maximum total allocated space */
+  MALLINFO_FIELD_TYPE fsmblks;  /* always 0 */
+  MALLINFO_FIELD_TYPE uordblks; /* total allocated space */
+  MALLINFO_FIELD_TYPE fordblks; /* total free space */
+  MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */
+};
+#endif /* STRUCT_MALLINFO_DECLARED */
+#endif /* HAVE_USR_INCLUDE_MALLOC_H */
+#endif /* NO_MALLINFO */
+
+/*
+  Try to persuade compilers to inline. The most critical functions for
+  inlining are defined as macros, so these aren't used for them.
+*/
+
+#ifndef FORCEINLINE
+  #if defined(__GNUC__)
+#define FORCEINLINE __inline __attribute__ ((always_inline))
+  #elif defined(_MSC_VER)
+    #define FORCEINLINE __forceinline
+  #endif
+#endif
+#ifndef NOINLINE
+  #if defined(__GNUC__)
+    #define NOINLINE __attribute__ ((noinline))
+  #elif defined(_MSC_VER)
+    #define NOINLINE __declspec(noinline)
+  #else
+    #define NOINLINE
+  #endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#ifndef FORCEINLINE
+ #define FORCEINLINE inline
+#endif
+#endif /* __cplusplus */
+#ifndef FORCEINLINE
+ #define FORCEINLINE
+#endif
+
+#if !ONLY_MSPACES
+
+/* ------------------- Declarations of public routines ------------------- */
+
+#ifndef USE_DL_PREFIX
+#define dlcalloc               calloc
+#define dlfree                 free
+#define dlmalloc               malloc
+#define dlmemalign             memalign
+#define dlposix_memalign       posix_memalign
+#define dlrealloc              realloc
+#define dlrealloc_in_place     realloc_in_place
+#define dlvalloc               valloc
+#define dlpvalloc              pvalloc
+#define dlmallinfo             mallinfo
+#define dlmallopt              mallopt
+#define dlmalloc_trim          malloc_trim
+#define dlmalloc_stats         malloc_stats
+#define dlmalloc_usable_size   malloc_usable_size
+#define dlmalloc_footprint     malloc_footprint
+#define dlmalloc_max_footprint malloc_max_footprint
+#define dlmalloc_footprint_limit malloc_footprint_limit
+#define dlmalloc_set_footprint_limit malloc_set_footprint_limit
+#define dlmalloc_inspect_all   malloc_inspect_all
+#define dlindependent_calloc   independent_calloc
+#define dlindependent_comalloc independent_comalloc
+#define dlbulk_free            bulk_free
+#endif /* USE_DL_PREFIX */
+
+/*
+  malloc(size_t n)
+  Returns a pointer to a newly allocated chunk of at least n bytes, or
+  null if no space is available, in which case errno is set to ENOMEM
+  on ANSI C systems.
+
+  If n is zero, malloc returns a minimum-sized chunk. (The minimum
+  size is 16 bytes on most 32bit systems, and 32 bytes on 64bit
+  systems.)  Note that size_t is an unsigned type, so calls with
+  arguments that would be negative if signed are interpreted as
+  requests for huge amounts of space, which will often fail. The
+  maximum supported value of n differs across systems, but is in all
+  cases less than the maximum representable value of a size_t.
+*/
+DLMALLOC_EXPORT void* dlmalloc(size_t);
+
+/*
+  free(void* p)
+  Releases the chunk of memory pointed to by p, that had been previously
+  allocated using malloc or a related routine such as realloc.
+  It has no effect if p is null. If p was not malloced or already
+  freed, free(p) will by default cause the current program to abort.
+*/
+DLMALLOC_EXPORT void  dlfree(void*);
+
+/*
+  calloc(size_t n_elements, size_t element_size);
+  Returns a pointer to n_elements * element_size bytes, with all locations
+  set to zero.
+*/
+DLMALLOC_EXPORT void* dlcalloc(size_t, size_t);
+
+/*
+  realloc(void* p, size_t n)
+  Returns a pointer to a chunk of size n that contains the same data
+  as does chunk p up to the minimum of (n, p's size) bytes, or null
+  if no space is available.
+
+  The returned pointer may or may not be the same as p. The algorithm
+  prefers extending p in most cases when possible, otherwise it
+  employs the equivalent of a malloc-copy-free sequence.
+
+  If p is null, realloc is equivalent to malloc.
+
+  If space is not available, realloc returns null, errno is set (if on
+  ANSI) and p is NOT freed.
+
+  if n is for fewer bytes than already held by p, the newly unused
+  space is lopped off and freed if possible.  realloc with a size
+  argument of zero (re)allocates a minimum-sized chunk.
+
+  The old unix realloc convention of allowing the last-free'd chunk
+  to be used as an argument to realloc is not supported.
+*/
+DLMALLOC_EXPORT void* dlrealloc(void*, size_t);
+
+/*
+  realloc_in_place(void* p, size_t n)
+  Resizes the space allocated for p to size n, only if this can be
+  done without moving p (i.e., only if there is adjacent space
+  available if n is greater than p's current allocated size, or n is
+  less than or equal to p's size). This may be used instead of plain
+  realloc if an alternative allocation strategy is needed upon failure
+  to expand space; for example, reallocation of a buffer that must be
+  memory-aligned or cleared. You can use realloc_in_place to trigger
+  these alternatives only when needed.
+
+  Returns p if successful; otherwise null.
+*/
+DLMALLOC_EXPORT void* dlrealloc_in_place(void*, size_t);
+
+/*
+  memalign(size_t alignment, size_t n);
+  Returns a pointer to a newly allocated chunk of n bytes, aligned
+  in accord with the alignment argument.
+
+  The alignment argument should be a power of two. If the argument is
+  not a power of two, the nearest greater power is used.
+  8-byte alignment is guaranteed by normal malloc calls, so don't
+  bother calling memalign with an argument of 8 or less.
+
+  Overreliance on memalign is a sure way to fragment space.
+*/
+DLMALLOC_EXPORT void* dlmemalign(size_t, size_t);
+
+/*
+  int posix_memalign(void** pp, size_t alignment, size_t n);
+  Allocates a chunk of n bytes, aligned in accord with the alignment
+  argument. Differs from memalign only in that it (1) assigns the
+  allocated memory to *pp rather than returning it, (2) fails and
+  returns EINVAL if the alignment is not a power of two (3) fails and
+  returns ENOMEM if memory cannot be allocated.
+*/
+DLMALLOC_EXPORT int dlposix_memalign(void**, size_t, size_t);
+
+/*
+  valloc(size_t n);
+  Equivalent to memalign(pagesize, n), where pagesize is the page
+  size of the system. If the pagesize is unknown, 4096 is used.
+*/
+DLMALLOC_EXPORT void* dlvalloc(size_t);
+
+/*
+  mallopt(int parameter_number, int parameter_value)
+  Sets tunable parameters The format is to provide a
+  (parameter-number, parameter-value) pair.  mallopt then sets the
+  corresponding parameter to the argument value if it can (i.e., so
+  long as the value is meaningful), and returns 1 if successful else
+  0.  To workaround the fact that mallopt is specified to use int,
+  not size_t parameters, the value -1 is specially treated as the
+  maximum unsigned size_t value.
+
+  SVID/XPG/ANSI defines four standard param numbers for mallopt,
+  normally defined in malloc.h.  None of these are use in this malloc,
+  so setting them has no effect. But this malloc also supports other
+  options in mallopt. See below for details.  Briefly, supported
+  parameters are as follows (listed defaults are for "typical"
+  configurations).
+
+  Symbol            param #  default    allowed param values
+  M_TRIM_THRESHOLD     -1   2*1024*1024   any   (-1 disables)
+  M_GRANULARITY        -2     page size   any power of 2 >= page size
+  M_MMAP_THRESHOLD     -3      256*1024   any   (or 0 if no MMAP support)
+*/
+DLMALLOC_EXPORT int dlmallopt(int, int);
+
+/*
+  malloc_footprint();
+  Returns the number of bytes obtained from the system.  The total
+  number of bytes allocated by malloc, realloc etc., is less than this
+  value. Unlike mallinfo, this function returns only a precomputed
+  result, so can be called frequently to monitor memory consumption.
+  Even if locks are otherwise defined, this function does not use them,
+  so results might not be up to date.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_footprint(void);
+
+/*
+  malloc_max_footprint();
+  Returns the maximum number of bytes obtained from the system. This
+  value will be greater than current footprint if deallocated space
+  has been reclaimed by the system. The peak number of bytes allocated
+  by malloc, realloc etc., is less than this value. Unlike mallinfo,
+  this function returns only a precomputed result, so can be called
+  frequently to monitor memory consumption.  Even if locks are
+  otherwise defined, this function does not use them, so results might
+  not be up to date.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_max_footprint(void);
+
+/*
+  malloc_footprint_limit();
+  Returns the number of bytes that the heap is allowed to obtain from
+  the system, returning the last value returned by
+  malloc_set_footprint_limit, or the maximum size_t value if
+  never set. The returned value reflects a permission. There is no
+  guarantee that this number of bytes can actually be obtained from
+  the system.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_footprint_limit();
+
+/*
+  malloc_set_footprint_limit();
+  Sets the maximum number of bytes to obtain from the system, causing
+  failure returns from malloc and related functions upon attempts to
+  exceed this value. The argument value may be subject to page
+  rounding to an enforceable limit; this actual value is returned.
+  Using an argument of the maximum possible size_t effectively
+  disables checks. If the argument is less than or equal to the
+  current malloc_footprint, then all future allocations that require
+  additional system memory will fail. However, invocation cannot
+  retroactively deallocate existing used memory.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_set_footprint_limit(size_t bytes);
+
+#if MALLOC_INSPECT_ALL
+/*
+  malloc_inspect_all(void(*handler)(void *start,
+                                    void *end,
+                                    size_t used_bytes,
+                                    void* callback_arg),
+                      void* arg);
+  Traverses the heap and calls the given handler for each managed
+  region, skipping all bytes that are (or may be) used for bookkeeping
+  purposes.  Traversal does not include include chunks that have been
+  directly memory mapped. Each reported region begins at the start
+  address, and continues up to but not including the end address.  The
+  first used_bytes of the region contain allocated data. If
+  used_bytes is zero, the region is unallocated. The handler is
+  invoked with the given callback argument. If locks are defined, they
+  are held during the entire traversal. It is a bad idea to invoke
+  other malloc functions from within the handler.
+
+  For example, to count the number of in-use chunks with size greater
+  than 1000, you could write:
+  static int count = 0;
+  void count_chunks(void* start, void* end, size_t used, void* arg) {
+    if (used >= 1000) ++count;
+  }
+  then:
+    malloc_inspect_all(count_chunks, NULL);
+
+  malloc_inspect_all is compiled only if MALLOC_INSPECT_ALL is defined.
+*/
+DLMALLOC_EXPORT void dlmalloc_inspect_all(void(*handler)(void*, void *, size_t, void*),
+                           void* arg);
+
+#endif /* MALLOC_INSPECT_ALL */
+
+#if !NO_MALLINFO
+/*
+  mallinfo()
+  Returns (by copy) a struct containing various summary statistics:
+
+  arena:     current total non-mmapped bytes allocated from system
+  ordblks:   the number of free chunks
+  smblks:    always zero.
+  hblks:     current number of mmapped regions
+  hblkhd:    total bytes held in mmapped regions
+  usmblks:   the maximum total allocated space. This will be greater
+                than current total if trimming has occurred.
+  fsmblks:   always zero
+  uordblks:  current total allocated space (normal or mmapped)
+  fordblks:  total free space
+  keepcost:  the maximum number of bytes that could ideally be released
+               back to system via malloc_trim. ("ideally" means that
+               it ignores page restrictions etc.)
+
+  Because these fields are ints, but internal bookkeeping may
+  be kept as longs, the reported values may wrap around zero and
+  thus be inaccurate.
+*/
+DLMALLOC_EXPORT struct mallinfo dlmallinfo(void);
+#endif /* NO_MALLINFO */
+
+/*
+  independent_calloc(size_t n_elements, size_t element_size, void* chunks[]);
+
+  independent_calloc is similar to calloc, but instead of returning a
+  single cleared space, it returns an array of pointers to n_elements
+  independent elements that can hold contents of size elem_size, each
+  of which starts out cleared, and can be independently freed,
+  realloc'ed etc. The elements are guaranteed to be adjacently
+  allocated (this is not guaranteed to occur with multiple callocs or
+  mallocs), which may also improve cache locality in some
+  applications.
+
+  The "chunks" argument is optional (i.e., may be null, which is
+  probably the most typical usage). If it is null, the returned array
+  is itself dynamically allocated and should also be freed when it is
+  no longer needed. Otherwise, the chunks array must be of at least
+  n_elements in length. It is filled in with the pointers to the
+  chunks.
+
+  In either case, independent_calloc returns this pointer array, or
+  null if the allocation failed.  If n_elements is zero and "chunks"
+  is null, it returns a chunk representing an array with zero elements
+  (which should be freed if not wanted).
+
+  Each element must be freed when it is no longer needed. This can be
+  done all at once using bulk_free.
+
+  independent_calloc simplifies and speeds up implementations of many
+  kinds of pools.  It may also be useful when constructing large data
+  structures that initially have a fixed number of fixed-sized nodes,
+  but the number is not known at compile time, and some of the nodes
+  may later need to be freed. For example:
+
+  struct Node { int item; struct Node* next; };
+
+  struct Node* build_list() {
+    struct Node** pool;
+    int n = read_number_of_nodes_needed();
+    if (n <= 0) return 0;
+    pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0);
+    if (pool == 0) die();
+    // organize into a linked list...
+    struct Node* first = pool[0];
+    for (i = 0; i < n-1; ++i)
+      pool[i]->next = pool[i+1];
+    free(pool);     // Can now free the array (or not, if it is needed later)
+    return first;
+  }
+*/
+DLMALLOC_EXPORT void** dlindependent_calloc(size_t, size_t, void**);
+
+/*
+  independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
+
+  independent_comalloc allocates, all at once, a set of n_elements
+  chunks with sizes indicated in the "sizes" array.    It returns
+  an array of pointers to these elements, each of which can be
+  independently freed, realloc'ed etc. The elements are guaranteed to
+  be adjacently allocated (this is not guaranteed to occur with
+  multiple callocs or mallocs), which may also improve cache locality
+  in some applications.
+
+  The "chunks" argument is optional (i.e., may be null). If it is null
+  the returned array is itself dynamically allocated and should also
+  be freed when it is no longer needed. Otherwise, the chunks array
+  must be of at least n_elements in length. It is filled in with the
+  pointers to the chunks.
+
+  In either case, independent_comalloc returns this pointer array, or
+  null if the allocation failed.  If n_elements is zero and chunks is
+  null, it returns a chunk representing an array with zero elements
+  (which should be freed if not wanted).
+
+  Each element must be freed when it is no longer needed. This can be
+  done all at once using bulk_free.
+
+  independent_comallac differs from independent_calloc in that each
+  element may have a different size, and also that it does not
+  automatically clear elements.
+
+  independent_comalloc can be used to speed up allocation in cases
+  where several structs or objects must always be allocated at the
+  same time.  For example:
+
+  struct Head { ... }
+  struct Foot { ... }
+
+  void send_message(char* msg) {
+    int msglen = strlen(msg);
+    size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) };
+    void* chunks[3];
+    if (independent_comalloc(3, sizes, chunks) == 0)
+      die();
+    struct Head* head = (struct Head*)(chunks[0]);
+    char*        body = (char*)(chunks[1]);
+    struct Foot* foot = (struct Foot*)(chunks[2]);
+    // ...
+  }
+
+  In general though, independent_comalloc is worth using only for
+  larger values of n_elements. For small values, you probably won't
+  detect enough difference from series of malloc calls to bother.
+
+  Overuse of independent_comalloc can increase overall memory usage,
+  since it cannot reuse existing noncontiguous small chunks that
+  might be available for some of the elements.
+*/
+DLMALLOC_EXPORT void** dlindependent_comalloc(size_t, size_t*, void**);
+
+/*
+  bulk_free(void* array[], size_t n_elements)
+  Frees and clears (sets to null) each non-null pointer in the given
+  array.  This is likely to be faster than freeing them one-by-one.
+  If footers are used, pointers that have been allocated in different
+  mspaces are not freed or cleared, and the count of all such pointers
+  is returned.  For large arrays of pointers with poor locality, it
+  may be worthwhile to sort this array before calling bulk_free.
+*/
+DLMALLOC_EXPORT size_t  dlbulk_free(void**, size_t n_elements);
+
+/*
+  pvalloc(size_t n);
+  Equivalent to valloc(minimum-page-that-holds(n)), that is,
+  round up n to nearest pagesize.
+ */
+DLMALLOC_EXPORT void*  dlpvalloc(size_t);
+
+/*
+  malloc_trim(size_t pad);
+
+  If possible, gives memory back to the system (via negative arguments
+  to sbrk) if there is unused memory at the `high' end of the malloc
+  pool or in unused MMAP segments. You can call this after freeing
+  large blocks of memory to potentially reduce the system-level memory
+  requirements of a program. However, it cannot guarantee to reduce
+  memory. Under some allocation patterns, some large free blocks of
+  memory will be locked between two used chunks, so they cannot be
+  given back to the system.
+
+  The `pad' argument to malloc_trim represents the amount of free
+  trailing space to leave untrimmed. If this argument is zero, only
+  the minimum amount of memory to maintain internal data structures
+  will be left. Non-zero arguments can be supplied to maintain enough
+  trailing space to service future expected allocations without having
+  to re-obtain memory from the system.
+
+  Malloc_trim returns 1 if it actually released any memory, else 0.
+*/
+DLMALLOC_EXPORT int  dlmalloc_trim(size_t);
+
+/*
+  malloc_stats();
+  Prints on stderr the amount of space obtained from the system (both
+  via sbrk and mmap), the maximum amount (which may be more than
+  current if malloc_trim and/or munmap got called), and the current
+  number of bytes allocated via malloc (or realloc, etc) but not yet
+  freed. Note that this is the number of bytes allocated, not the
+  number requested. It will be larger than the number requested
+  because of alignment and bookkeeping overhead. Because it includes
+  alignment wastage as being in use, this figure may be greater than
+  zero even when no user-level chunks are allocated.
+
+  The reported current and maximum system memory can be inaccurate if
+  a program makes other calls to system memory allocation functions
+  (normally sbrk) outside of malloc.
+
+  malloc_stats prints only the most commonly interesting statistics.
+  More information can be obtained by calling mallinfo.
+*/
+DLMALLOC_EXPORT void  dlmalloc_stats(void);
+
+/*
+  malloc_usable_size(void* p);
+
+  Returns the number of bytes you can actually use in
+  an allocated chunk, which may be more than you requested (although
+  often not) due to alignment and minimum size constraints.
+  You can use this many bytes without worrying about
+  overwriting other allocated objects. This is not a particularly great
+  programming practice. malloc_usable_size can be more useful in
+  debugging and assertions, for example:
+
+  p = malloc(n);
+  assert(malloc_usable_size(p) >= 256);
+*/
+size_t dlmalloc_usable_size(void*);
+
+#endif /* ONLY_MSPACES */
+
+#if MSPACES
+
+/*
+  mspace is an opaque type representing an independent
+  region of space that supports mspace_malloc, etc.
+*/
+typedef void* mspace;
+
+/*
+  create_mspace creates and returns a new independent space with the
+  given initial capacity, or, if 0, the default granularity size.  It
+  returns null if there is no system memory available to create the
+  space.  If argument locked is non-zero, the space uses a separate
+  lock to control access. The capacity of the space will grow
+  dynamically as needed to service mspace_malloc requests.  You can
+  control the sizes of incremental increases of this space by
+  compiling with a different DEFAULT_GRANULARITY or dynamically
+  setting with mallopt(M_GRANULARITY, value).
+*/
+DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked);
+
+/*
+  destroy_mspace destroys the given space, and attempts to return all
+  of its memory back to the system, returning the total number of
+  bytes freed. After destruction, the results of access to all memory
+  used by the space become undefined.
+*/
+DLMALLOC_EXPORT size_t destroy_mspace(mspace msp);
+
+/*
+  create_mspace_with_base uses the memory supplied as the initial base
+  of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this
+  space is used for bookkeeping, so the capacity must be at least this
+  large. (Otherwise 0 is returned.) When this initial space is
+  exhausted, additional memory will be obtained from the system.
+  Destroying this space will deallocate all additionally allocated
+  space (if possible) but not the initial base.
+*/
+DLMALLOC_EXPORT mspace create_mspace_with_base(void* base, size_t capacity, int locked);
+
+/*
+  mspace_track_large_chunks controls whether requests for large chunks
+  are allocated in their own untracked mmapped regions, separate from
+  others in this mspace. By default large chunks are not tracked,
+  which reduces fragmentation. However, such chunks are not
+  necessarily released to the system upon destroy_mspace.  Enabling
+  tracking by setting to true may increase fragmentation, but avoids
+  leakage when relying on destroy_mspace to release all memory
+  allocated using this space.  The function returns the previous
+  setting.
+*/
+DLMALLOC_EXPORT int mspace_track_large_chunks(mspace msp, int enable);
+
+
+/*
+  mspace_malloc behaves as malloc, but operates within
+  the given space.
+*/
+DLMALLOC_EXPORT void* mspace_malloc(mspace msp, size_t bytes);
+
+/*
+  mspace_free behaves as free, but operates within
+  the given space.
+
+  If compiled with FOOTERS==1, mspace_free is not actually needed.
+  free may be called instead of mspace_free because freed chunks from
+  any space are handled by their originating spaces.
+*/
+DLMALLOC_EXPORT void mspace_free(mspace msp, void* mem);
+
+/*
+  mspace_realloc behaves as realloc, but operates within
+  the given space.
+
+  If compiled with FOOTERS==1, mspace_realloc is not actually
+  needed.  realloc may be called instead of mspace_realloc because
+  realloced chunks from any space are handled by their originating
+  spaces.
+*/
+DLMALLOC_EXPORT void* mspace_realloc(mspace msp, void* mem, size_t newsize);
+
+/*
+  mspace_calloc behaves as calloc, but operates within
+  the given space.
+*/
+DLMALLOC_EXPORT void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size);
+
+/*
+  mspace_memalign behaves as memalign, but operates within
+  the given space.
+*/
+DLMALLOC_EXPORT void* mspace_memalign(mspace msp, size_t alignment, size_t bytes);
+
+/*
+  mspace_independent_calloc behaves as independent_calloc, but
+  operates within the given space.
+*/
+DLMALLOC_EXPORT void** mspace_independent_calloc(mspace msp, size_t n_elements,
+                                 size_t elem_size, void* chunks[]);
+
+/*
+  mspace_independent_comalloc behaves as independent_comalloc, but
+  operates within the given space.
+*/
+DLMALLOC_EXPORT void** mspace_independent_comalloc(mspace msp, size_t n_elements,
+                                   size_t sizes[], void* chunks[]);
+
+/*
+  mspace_footprint() returns the number of bytes obtained from the
+  system for this space.
+*/
+DLMALLOC_EXPORT size_t mspace_footprint(mspace msp);
+
+/*
+  mspace_max_footprint() returns the peak number of bytes obtained from the
+  system for this space.
+*/
+DLMALLOC_EXPORT size_t mspace_max_footprint(mspace msp);
+
+
+#if !NO_MALLINFO
+/*
+  mspace_mallinfo behaves as mallinfo, but reports properties of
+  the given space.
+*/
+DLMALLOC_EXPORT struct mallinfo mspace_mallinfo(mspace msp);
+#endif /* NO_MALLINFO */
+
+/*
+  malloc_usable_size(void* p) behaves the same as malloc_usable_size;
+*/
+DLMALLOC_EXPORT size_t mspace_usable_size(const void* mem);
+
+/*
+  mspace_malloc_stats behaves as malloc_stats, but reports
+  properties of the given space.
+*/
+DLMALLOC_EXPORT void mspace_malloc_stats(mspace msp);
+
+/*
+  mspace_trim behaves as malloc_trim, but
+  operates within the given space.
+*/
+DLMALLOC_EXPORT int mspace_trim(mspace msp, size_t pad);
+
+/*
+  An alias for mallopt.
+*/
+DLMALLOC_EXPORT int mspace_mallopt(int, int);
+
+#endif /* MSPACES */
+
+#ifdef __cplusplus
+}  /* end of extern "C" */
+#endif /* __cplusplus */
+
+/*
+  ========================================================================
+  To make a fully customizable malloc.h header file, cut everything
+  above this line, put into file malloc.h, edit to suit, and #include it
+  on the next line, as well as in programs that use this malloc.
+  ========================================================================
+*/
+
+/* #include "malloc.h" */
+
+/*------------------------------ internal #includes ---------------------- */
+
+#ifdef _MSC_VER
+#pragma warning( disable : 4146 ) /* no "unsigned" warnings */
+#endif /* _MSC_VER */
+#if !NO_MALLOC_STATS
+#include <stdio.h>       /* for printing in malloc_stats */
+#endif /* NO_MALLOC_STATS */
+#ifndef LACKS_ERRNO_H
+#include <errno.h>       /* for MALLOC_FAILURE_ACTION */
+#endif /* LACKS_ERRNO_H */
+#ifdef DEBUG
+#if ABORT_ON_ASSERT_FAILURE
+#undef assert
+#define assert(x) if(!(x)) ABORT
+#else /* ABORT_ON_ASSERT_FAILURE */
+#include <assert.h>
+#endif /* ABORT_ON_ASSERT_FAILURE */
+#else  /* DEBUG */
+#ifndef assert
+#define assert(x)
+#endif
+#define DEBUG 0
+#endif /* DEBUG */
+#if !defined(WIN32) && !defined(LACKS_TIME_H)
+#include <time.h>        /* for magic initialization */
+#endif /* WIN32 */
+#ifndef LACKS_STDLIB_H
+#include <stdlib.h>      /* for abort() */
+#endif /* LACKS_STDLIB_H */
+#ifndef LACKS_STRING_H
+#include <string.h>      /* for memset etc */
+#endif  /* LACKS_STRING_H */
+#if USE_BUILTIN_FFS
+#ifndef LACKS_STRINGS_H
+#include <strings.h>     /* for ffs */
+#endif /* LACKS_STRINGS_H */
+#endif /* USE_BUILTIN_FFS */
+#if HAVE_MMAP
+#ifndef LACKS_SYS_MMAN_H
+/* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */
+#if (defined(linux) && !defined(__USE_GNU))
+#define __USE_GNU 1
+#include <sys/mman.h>    /* for mmap */
+#undef __USE_GNU
+#else
+#include <sys/mman.h>    /* for mmap */
+#endif /* linux */
+#endif /* LACKS_SYS_MMAN_H */
+#ifndef LACKS_FCNTL_H
+#include <fcntl.h>
+#endif /* LACKS_FCNTL_H */
+#endif /* HAVE_MMAP */
+#ifndef LACKS_UNISTD_H
+#include <unistd.h>     /* for sbrk, sysconf */
+#else /* LACKS_UNISTD_H */
+#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__)
+extern void*     sbrk(ptrdiff_t);
+#endif /* FreeBSD etc */
+#endif /* LACKS_UNISTD_H */
+
+/* Declarations for locking */
+#if USE_LOCKS
+#ifndef WIN32
+#if defined (__SVR4) && defined (__sun)  /* solaris */
+#include <thread.h>
+#elif !defined(LACKS_SCHED_H)
+#include <sched.h>
+#endif /* solaris or LACKS_SCHED_H */
+#if (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0) || !USE_SPIN_LOCKS
+#include <pthread.h>
+#endif /* USE_RECURSIVE_LOCKS ... */
+#elif defined(_MSC_VER)
+#ifndef _M_AMD64
+/* These are already defined on AMD64 builds */
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+LONG __cdecl _InterlockedCompareExchange(LONG volatile *Dest, LONG Exchange, LONG Comp);
+LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value);
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+#endif /* _M_AMD64 */
+#pragma intrinsic (_InterlockedCompareExchange)
+#pragma intrinsic (_InterlockedExchange)
+#define interlockedcompareexchange _InterlockedCompareExchange
+#define interlockedexchange _InterlockedExchange
+#elif defined(WIN32) && defined(__GNUC__)
+#define interlockedcompareexchange(a, b, c) __sync_val_compare_and_swap(a, c, b)
+#define interlockedexchange __sync_lock_test_and_set
+#endif /* Win32 */
+#else /* USE_LOCKS */
+#endif /* USE_LOCKS */
+
+#ifndef LOCK_AT_FORK
+#define LOCK_AT_FORK 0
+#endif
+
+/* Declarations for bit scanning on win32 */
+#if defined(_MSC_VER) && _MSC_VER>=1300
+#ifndef BitScanForward /* Try to avoid pulling in WinNT.h */
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+unsigned char _BitScanForward(unsigned long *index, unsigned long mask);
+unsigned char _BitScanReverse(unsigned long *index, unsigned long mask);
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#define BitScanForward _BitScanForward
+#define BitScanReverse _BitScanReverse
+#pragma intrinsic(_BitScanForward)
+#pragma intrinsic(_BitScanReverse)
+#endif /* BitScanForward */
+#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */
+
+#ifndef WIN32
+#ifndef malloc_getpagesize
+#  ifdef _SC_PAGESIZE         /* some SVR4 systems omit an underscore */
+#    ifndef _SC_PAGE_SIZE
+#      define _SC_PAGE_SIZE _SC_PAGESIZE
+#    endif
+#  endif
+#  ifdef _SC_PAGE_SIZE
+#    define malloc_getpagesize sysconf(_SC_PAGE_SIZE)
+#  else
+#    if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE)
+       extern size_t getpagesize();
+#      define malloc_getpagesize getpagesize()
+#    else
+#      ifdef WIN32 /* use supplied emulation of getpagesize */
+#        define malloc_getpagesize getpagesize()
+#      else
+#        ifndef LACKS_SYS_PARAM_H
+#          include <sys/param.h>
+#        endif
+#        ifdef EXEC_PAGESIZE
+#          define malloc_getpagesize EXEC_PAGESIZE
+#        else
+#          ifdef NBPG
+#            ifndef CLSIZE
+#              define malloc_getpagesize NBPG
+#            else
+#              define malloc_getpagesize (NBPG * CLSIZE)
+#            endif
+#          else
+#            ifdef NBPC
+#              define malloc_getpagesize NBPC
+#            else
+#              ifdef PAGESIZE
+#                define malloc_getpagesize PAGESIZE
+#              else /* just guess */
+#                define malloc_getpagesize ((size_t)4096U)
+#              endif
+#            endif
+#          endif
+#        endif
+#      endif
+#    endif
+#  endif
+#endif
+#endif
+
+/* ------------------- size_t and alignment properties -------------------- */
+
+/* The byte and bit size of a size_t */
+#define SIZE_T_SIZE         (sizeof(size_t))
+#define SIZE_T_BITSIZE      (sizeof(size_t) << 3)
+
+/* Some constants coerced to size_t */
+/* Annoying but necessary to avoid errors on some platforms */
+#define SIZE_T_ZERO         ((size_t)0)
+#define SIZE_T_ONE          ((size_t)1)
+#define SIZE_T_TWO          ((size_t)2)
+#define SIZE_T_FOUR         ((size_t)4)
+#define TWO_SIZE_T_SIZES    (SIZE_T_SIZE<<1)
+#define FOUR_SIZE_T_SIZES   (SIZE_T_SIZE<<2)
+#define SIX_SIZE_T_SIZES    (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES)
+#define HALF_MAX_SIZE_T     (MAX_SIZE_T / 2U)
+
+/* The bit mask value corresponding to MALLOC_ALIGNMENT */
+#define CHUNK_ALIGN_MASK    (MALLOC_ALIGNMENT - SIZE_T_ONE)
+
+/* True if address a has acceptable alignment */
+#define is_aligned(A)       (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0)
+
+/* the number of bytes to offset an address to align it */
+#define align_offset(A)\
+ ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\
+  ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK))
+
+/* -------------------------- MMAP preliminaries ------------------------- */
+
+/*
+   If HAVE_MORECORE or HAVE_MMAP are false, we just define calls and
+   checks to fail so compiler optimizer can delete code rather than
+   using so many "#if"s.
+*/
+
+
+/* MORECORE and MMAP must return MFAIL on failure */
+#define MFAIL                ((void*)(MAX_SIZE_T))
+#define CMFAIL               ((char*)(MFAIL)) /* defined for convenience */
+
+#if HAVE_MMAP
+
+#ifndef WIN32
+#define MUNMAP_DEFAULT(a, s)  munmap((a), (s))
+#define MMAP_PROT            (PROT_READ|PROT_WRITE)
+#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
+#define MAP_ANONYMOUS        MAP_ANON
+#endif /* MAP_ANON */
+#ifdef MAP_ANONYMOUS
+#define MMAP_FLAGS           (MAP_PRIVATE|MAP_ANONYMOUS)
+#define MMAP_DEFAULT(s)       mmap(0, (s), MMAP_PROT, MMAP_FLAGS, -1, 0)
+#else /* MAP_ANONYMOUS */
+/*
+   Nearly all versions of mmap support MAP_ANONYMOUS, so the following
+   is unlikely to be needed, but is supplied just in case.
+*/
+#define MMAP_FLAGS           (MAP_PRIVATE)
+static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */
+#define MMAP_DEFAULT(s) ((dev_zero_fd < 0) ? \
+           (dev_zero_fd = open("/dev/zero", O_RDWR), \
+            mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) : \
+            mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0))
+#endif /* MAP_ANONYMOUS */
+
+#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s)
+
+#else /* WIN32 */
+
+/* Win32 MMAP via VirtualAlloc */
+static FORCEINLINE void* win32mmap(size_t size) {
+  void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
+  return (ptr != 0)? ptr: MFAIL;
+}
+
+/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
+static FORCEINLINE void* win32direct_mmap(size_t size) {
+  void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN,
+                           PAGE_READWRITE);
+  return (ptr != 0)? ptr: MFAIL;
+}
+
+/* This function supports releasing coalesed segments */
+static FORCEINLINE int win32munmap(void* ptr, size_t size) {
+  MEMORY_BASIC_INFORMATION minfo;
+  char* cptr = (char*)ptr;
+  while (size) {
+    if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0)
+      return -1;
+    if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr ||
+        minfo.State != MEM_COMMIT || minfo.RegionSize > size)
+      return -1;
+    if (VirtualFree(cptr, 0, MEM_RELEASE) == 0)
+      return -1;
+    cptr += minfo.RegionSize;
+    size -= minfo.RegionSize;
+  }
+  return 0;
+}
+
+#define MMAP_DEFAULT(s)             win32mmap(s)
+#define MUNMAP_DEFAULT(a, s)        win32munmap((a), (s))
+#define DIRECT_MMAP_DEFAULT(s)      win32direct_mmap(s)
+#endif /* WIN32 */
+#endif /* HAVE_MMAP */
+
+#if HAVE_MREMAP
+#ifndef WIN32
+#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv))
+#endif /* WIN32 */
+#endif /* HAVE_MREMAP */
+
+/**
+ * Define CALL_MORECORE
+ */
+#if HAVE_MORECORE
+    #ifdef MORECORE
+        #define CALL_MORECORE(S)    MORECORE(S)
+    #else  /* MORECORE */
+        #define CALL_MORECORE(S)    MORECORE_DEFAULT(S)
+    #endif /* MORECORE */
+#else  /* HAVE_MORECORE */
+    #define CALL_MORECORE(S)        MFAIL
+#endif /* HAVE_MORECORE */
+
+/**
+ * Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP
+ */
+#if HAVE_MMAP
+    #define USE_MMAP_BIT            (SIZE_T_ONE)
+
+    #ifdef MMAP
+        #define CALL_MMAP(s)        MMAP(s)
+    #else /* MMAP */
+        #define CALL_MMAP(s)        MMAP_DEFAULT(s)
+    #endif /* MMAP */
+    #ifdef MUNMAP
+        #define CALL_MUNMAP(a, s)   MUNMAP((a), (s))
+    #else /* MUNMAP */
+        #define CALL_MUNMAP(a, s)   MUNMAP_DEFAULT((a), (s))
+    #endif /* MUNMAP */
+    #ifdef DIRECT_MMAP
+        #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
+    #else /* DIRECT_MMAP */
+        #define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s)
+    #endif /* DIRECT_MMAP */
+#else  /* HAVE_MMAP */
+    #define USE_MMAP_BIT            (SIZE_T_ZERO)
+
+    #define MMAP(s)                 MFAIL
+    #define MUNMAP(a, s)            (-1)
+    #define DIRECT_MMAP(s)          MFAIL
+    #define CALL_DIRECT_MMAP(s)     DIRECT_MMAP(s)
+    #define CALL_MMAP(s)            MMAP(s)
+    #define CALL_MUNMAP(a, s)       MUNMAP((a), (s))
+#endif /* HAVE_MMAP */
+
+/**
+ * Define CALL_MREMAP
+ */
+#if HAVE_MMAP && HAVE_MREMAP
+    #ifdef MREMAP
+        #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv))
+    #else /* MREMAP */
+        #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP_DEFAULT((addr), (osz), (nsz), (mv))
+    #endif /* MREMAP */
+#else  /* HAVE_MMAP && HAVE_MREMAP */
+    #define CALL_MREMAP(addr, osz, nsz, mv)     MFAIL
+#endif /* HAVE_MMAP && HAVE_MREMAP */
+
+/* mstate bit set if continguous morecore disabled or failed */
+#define USE_NONCONTIGUOUS_BIT (4U)
+
+/* segment bit set in create_mspace_with_base */
+#define EXTERN_BIT            (8U)
+
+
+/* --------------------------- Lock preliminaries ------------------------ */
+
+/*
+  When locks are defined, there is one global lock, plus
+  one per-mspace lock.
+
+  The global lock_ensures that mparams.magic and other unique
+  mparams values are initialized only once. It also protects
+  sequences of calls to MORECORE.  In many cases sys_alloc requires
+  two calls, that should not be interleaved with calls by other
+  threads.  This does not protect against direct calls to MORECORE
+  by other threads not using this lock, so there is still code to
+  cope the best we can on interference.
+
+  Per-mspace locks surround calls to malloc, free, etc.
+  By default, locks are simple non-reentrant mutexes.
+
+  Because lock-protected regions generally have bounded times, it is
+  OK to use the supplied simple spinlocks. Spinlocks are likely to
+  improve performance for lightly contended applications, but worsen
+  performance under heavy contention.
+
+  If USE_LOCKS is > 1, the definitions of lock routines here are
+  bypassed, in which case you will need to define the type MLOCK_T,
+  and at least INITIAL_LOCK, DESTROY_LOCK, ACQUIRE_LOCK, RELEASE_LOCK
+  and TRY_LOCK.  You must also declare a
+    static MLOCK_T malloc_global_mutex = { initialization values };.
+
+*/
+
+#if !USE_LOCKS
+#define USE_LOCK_BIT               (0U)
+#define INITIAL_LOCK(l)            (0)
+#define DESTROY_LOCK(l)            (0)
+#define ACQUIRE_MALLOC_GLOBAL_LOCK()
+#define RELEASE_MALLOC_GLOBAL_LOCK()
+
+#else
+#if USE_LOCKS > 1
+/* -----------------------  User-defined locks ------------------------ */
+/* Define your own lock implementation here */
+/* #define INITIAL_LOCK(lk)  ... */
+/* #define DESTROY_LOCK(lk)  ... */
+/* #define ACQUIRE_LOCK(lk)  ... */
+/* #define RELEASE_LOCK(lk)  ... */
+/* #define TRY_LOCK(lk) ... */
+/* static MLOCK_T malloc_global_mutex = ... */
+
+#elif USE_SPIN_LOCKS
+
+/* First, define CAS_LOCK and CLEAR_LOCK on ints */
+/* Note CAS_LOCK defined to return 0 on success */
+
+#if defined(__GNUC__)&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))
+#define CAS_LOCK(sl)     __sync_lock_test_and_set(sl, 1)
+#define CLEAR_LOCK(sl)   __sync_lock_release(sl)
+
+#elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)))
+/* Custom spin locks for older gcc on x86 */
+static FORCEINLINE int x86_cas_lock(int *sl) {
+  int ret;
+  int val = 1;
+  int cmp = 0;
+  __asm__ __volatile__  ("lock; cmpxchgl %1, %2"
+                         : "=a" (ret)
+                         : "r" (val), "m" (*(sl)), "0"(cmp)
+                         : "memory", "cc");
+  return ret;
+}
+
+static FORCEINLINE void x86_clear_lock(int* sl) {
+  assert(*sl != 0);
+  int prev = 0;
+  int ret;
+  __asm__ __volatile__ ("lock; xchgl %0, %1"
+                        : "=r" (ret)
+                        : "m" (*(sl)), "0"(prev)
+                        : "memory");
+}
+
+#define CAS_LOCK(sl)     x86_cas_lock(sl)
+#define CLEAR_LOCK(sl)   x86_clear_lock(sl)
+
+#else /* Win32 MSC */
+#define CAS_LOCK(sl)     interlockedexchange(sl, (LONG)1)
+#define CLEAR_LOCK(sl)   interlockedexchange (sl, (LONG)0)
+
+#endif /* ... gcc spins locks ... */
+
+/* How to yield for a spin lock */
+#define SPINS_PER_YIELD       63
+#if defined(_MSC_VER)
+#define SLEEP_EX_DURATION     50 /* delay for yield/sleep */
+#define SPIN_LOCK_YIELD  SleepEx(SLEEP_EX_DURATION, FALSE)
+#elif defined (__SVR4) && defined (__sun) /* solaris */
+#define SPIN_LOCK_YIELD   thr_yield();
+#elif !defined(LACKS_SCHED_H)
+#define SPIN_LOCK_YIELD   sched_yield();
+#else
+#define SPIN_LOCK_YIELD
+#endif /* ... yield ... */
+
+#if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0
+/* Plain spin locks use single word (embedded in malloc_states) */
+static int spin_acquire_lock(int *sl) {
+  int spins = 0;
+  while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) {
+    if ((++spins & SPINS_PER_YIELD) == 0) {
+      SPIN_LOCK_YIELD;
+    }
+  }
+  return 0;
+}
+
+#define MLOCK_T               int
+#define TRY_LOCK(sl)          !CAS_LOCK(sl)
+#define RELEASE_LOCK(sl)      CLEAR_LOCK(sl)
+#define ACQUIRE_LOCK(sl)      (CAS_LOCK(sl)? spin_acquire_lock(sl) : 0)
+#define INITIAL_LOCK(sl)      (*sl = 0)
+#define DESTROY_LOCK(sl)      (0)
+static MLOCK_T malloc_global_mutex = 0;
+
+#else /* USE_RECURSIVE_LOCKS */
+/* types for lock owners */
+#ifdef WIN32
+#define THREAD_ID_T           DWORD
+#define CURRENT_THREAD        GetCurrentThreadId()
+#define EQ_OWNER(X,Y)         ((X) == (Y))
+#else
+/*
+  Note: the following assume that pthread_t is a type that can be
+  initialized to (casted) zero. If this is not the case, you will need to
+  somehow redefine these or not use spin locks.
+*/
+#define THREAD_ID_T           pthread_t
+#define CURRENT_THREAD        pthread_self()
+#define EQ_OWNER(X,Y)         pthread_equal(X, Y)
+#endif
+
+struct malloc_recursive_lock {
+  int sl;
+  unsigned int c;
+  THREAD_ID_T threadid;
+};
+
+#define MLOCK_T  struct malloc_recursive_lock
+static MLOCK_T malloc_global_mutex = { 0, 0, (THREAD_ID_T)0};
+
+static FORCEINLINE void recursive_release_lock(MLOCK_T *lk) {
+  assert(lk->sl != 0);
+  if (--lk->c == 0) {
+    CLEAR_LOCK(&lk->sl);
+  }
+}
+
+static FORCEINLINE int recursive_acquire_lock(MLOCK_T *lk) {
+  THREAD_ID_T mythreadid = CURRENT_THREAD;
+  int spins = 0;
+  for (;;) {
+    if (*((volatile int *)(&lk->sl)) == 0) {
+      if (!CAS_LOCK(&lk->sl)) {
+        lk->threadid = mythreadid;
+        lk->c = 1;
+        return 0;
+      }
+    }
+    else if (EQ_OWNER(lk->threadid, mythreadid)) {
+      ++lk->c;
+      return 0;
+    }
+    if ((++spins & SPINS_PER_YIELD) == 0) {
+      SPIN_LOCK_YIELD;
+    }
+  }
+}
+
+static FORCEINLINE int recursive_try_lock(MLOCK_T *lk) {
+  THREAD_ID_T mythreadid = CURRENT_THREAD;
+  if (*((volatile int *)(&lk->sl)) == 0) {
+    if (!CAS_LOCK(&lk->sl)) {
+      lk->threadid = mythreadid;
+      lk->c = 1;
+      return 1;
+    }
+  }
+  else if (EQ_OWNER(lk->threadid, mythreadid)) {
+    ++lk->c;
+    return 1;
+  }
+  return 0;
+}
+
+#define RELEASE_LOCK(lk)      recursive_release_lock(lk)
+#define TRY_LOCK(lk)          recursive_try_lock(lk)
+#define ACQUIRE_LOCK(lk)      recursive_acquire_lock(lk)
+#define INITIAL_LOCK(lk)      ((lk)->threadid = (THREAD_ID_T)0, (lk)->sl = 0, (lk)->c = 0)
+#define DESTROY_LOCK(lk)      (0)
+#endif /* USE_RECURSIVE_LOCKS */
+
+#elif defined(WIN32) /* Win32 critical sections */
+#define MLOCK_T               CRITICAL_SECTION
+#define ACQUIRE_LOCK(lk)      (EnterCriticalSection(lk), 0)
+#define RELEASE_LOCK(lk)      LeaveCriticalSection(lk)
+#define TRY_LOCK(lk)          TryEnterCriticalSection(lk)
+#define INITIAL_LOCK(lk)      (!InitializeCriticalSectionAndSpinCount((lk), 0x80000000|4000))
+#define DESTROY_LOCK(lk)      (DeleteCriticalSection(lk), 0)
+#define NEED_GLOBAL_LOCK_INIT
+
+static MLOCK_T malloc_global_mutex;
+static volatile LONG malloc_global_mutex_status;
+
+/* Use spin loop to initialize global lock */
+static void init_malloc_global_mutex() {
+  for (;;) {
+    long stat = malloc_global_mutex_status;
+    if (stat > 0)
+      return;
+    /* transition to < 0 while initializing, then to > 0) */
+    if (stat == 0 &&
+        interlockedcompareexchange(&malloc_global_mutex_status, (LONG)-1, (LONG)0) == 0) {
+      InitializeCriticalSection(&malloc_global_mutex);
+      interlockedexchange(&malloc_global_mutex_status, (LONG)1);
+      return;
+    }
+    SleepEx(0, FALSE);
+  }
+}
+
+#else /* pthreads-based locks */
+#define MLOCK_T               pthread_mutex_t
+#define ACQUIRE_LOCK(lk)      pthread_mutex_lock(lk)
+#define RELEASE_LOCK(lk)      pthread_mutex_unlock(lk)
+#define TRY_LOCK(lk)          (!pthread_mutex_trylock(lk))
+#define INITIAL_LOCK(lk)      pthread_init_lock(lk)
+#define DESTROY_LOCK(lk)      pthread_mutex_destroy(lk)
+
+#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 && defined(linux) && !defined(PTHREAD_MUTEX_RECURSIVE)
+/* Cope with old-style linux recursive lock initialization by adding */
+/* skipped internal declaration from pthread.h */
+extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr,
+                                              int __kind));
+#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP
+#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y)
+#endif /* USE_RECURSIVE_LOCKS ... */
+
+static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static int pthread_init_lock (MLOCK_T *lk) {
+  pthread_mutexattr_t attr;
+  if (pthread_mutexattr_init(&attr)) return 1;
+#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0
+  if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1;
+#endif
+  if (pthread_mutex_init(lk, &attr)) return 1;
+  if (pthread_mutexattr_destroy(&attr)) return 1;
+  return 0;
+}
+
+#endif /* ... lock types ... */
+
+/* Common code for all lock types */
+#define USE_LOCK_BIT               (2U)
+
+#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK
+#define ACQUIRE_MALLOC_GLOBAL_LOCK()  ACQUIRE_LOCK(&malloc_global_mutex);
+#endif
+
+#ifndef RELEASE_MALLOC_GLOBAL_LOCK
+#define RELEASE_MALLOC_GLOBAL_LOCK()  RELEASE_LOCK(&malloc_global_mutex);
+#endif
+
+#endif /* USE_LOCKS */
+
+/* -----------------------  Chunk representations ------------------------ */
+
+/*
+  (The following includes lightly edited explanations by Colin Plumb.)
+
+  The malloc_chunk declaration below is misleading (but accurate and
+  necessary).  It declares a "view" into memory allowing access to
+  necessary fields at known offsets from a given base.
+
+  Chunks of memory are maintained using a `boundary tag' method as
+  originally described by Knuth.  (See the paper by Paul Wilson
+  ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a survey of such
+  techniques.)  Sizes of free chunks are stored both in the front of
+  each chunk and at the end.  This makes consolidating fragmented
+  chunks into bigger chunks fast.  The head fields also hold bits
+  representing whether chunks are free or in use.
+
+  Here are some pictures to make it clearer.  They are "exploded" to
+  show that the state of a chunk can be thought of as extending from
+  the high 31 bits of the head field of its header through the
+  prev_foot and PINUSE_BIT bit of the following chunk header.
+
+  A chunk that's in use looks like:
+
+   chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+           | Size of previous chunk (if P = 0)                             |
+           +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
+         | Size of this chunk                                         1| +-+
+   mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |                                                               |
+         +-                                                             -+
+         |                                                               |
+         +-                                                             -+
+         |                                                               :
+         +-      size - sizeof(size_t) available payload bytes          -+
+         :                                                               |
+ chunk-> +-                                                             -+
+         |                                                               |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |1|
+       | Size of next chunk (may or may not be in use)               | +-+
+ mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+    And if it's free, it looks like this:
+
+   chunk-> +-                                                             -+
+           | User payload (must be in use, or we would have merged!)       |
+           +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
+         | Size of this chunk                                         0| +-+
+   mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         | Next pointer                                                  |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         | Prev pointer                                                  |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |                                                               :
+         +-      size - sizeof(struct chunk) unused bytes               -+
+         :                                                               |
+ chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         | Size of this chunk                                            |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |0|
+       | Size of next chunk (must be in use, or we would have merged)| +-+
+ mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+       |                                                               :
+       +- User payload                                                -+
+       :                                                               |
+       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+                                                                     |0|
+                                                                     +-+
+  Note that since we always merge adjacent free chunks, the chunks
+  adjacent to a free chunk must be in use.
+
+  Given a pointer to a chunk (which can be derived trivially from the
+  payload pointer) we can, in O(1) time, find out whether the adjacent
+  chunks are free, and if so, unlink them from the lists that they
+  are on and merge them with the current chunk.
+
+  Chunks always begin on even word boundaries, so the mem portion
+  (which is returned to the user) is also on an even word boundary, and
+  thus at least double-word aligned.
+
+  The P (PINUSE_BIT) bit, stored in the unused low-order bit of the
+  chunk size (which is always a multiple of two words), is an in-use
+  bit for the *previous* chunk.  If that bit is *clear*, then the
+  word before the current chunk size contains the previous chunk
+  size, and can be used to find the front of the previous chunk.
+  The very first chunk allocated always has this bit set, preventing
+  access to non-existent (or non-owned) memory. If pinuse is set for
+  any given chunk, then you CANNOT determine the size of the
+  previous chunk, and might even get a memory addressing fault when
+  trying to do so.
+
+  The C (CINUSE_BIT) bit, stored in the unused second-lowest bit of
+  the chunk size redundantly records whether the current chunk is
+  inuse (unless the chunk is mmapped). This redundancy enables usage
+  checks within free and realloc, and reduces indirection when freeing
+  and consolidating chunks.
+
+  Each freshly allocated chunk must have both cinuse and pinuse set.
+  That is, each allocated chunk borders either a previously allocated
+  and still in-use chunk, or the base of its memory arena. This is
+  ensured by making all allocations from the `lowest' part of any
+  found chunk.  Further, no free chunk physically borders another one,
+  so each free chunk is known to be preceded and followed by either
+  inuse chunks or the ends of memory.
+
+  Note that the `foot' of the current chunk is actually represented
+  as the prev_foot of the NEXT chunk. This makes it easier to
+  deal with alignments etc but can be very confusing when trying
+  to extend or adapt this code.
+
+  The exceptions to all this are
+
+     1. The special chunk `top' is the top-most available chunk (i.e.,
+        the one bordering the end of available memory). It is treated
+        specially.  Top is never included in any bin, is used only if
+        no other chunk is available, and is released back to the
+        system if it is very large (see M_TRIM_THRESHOLD).  In effect,
+        the top chunk is treated as larger (and thus less well
+        fitting) than any other available chunk.  The top chunk
+        doesn't update its trailing size field since there is no next
+        contiguous chunk that would have to index off it. However,
+        space is still allocated for it (TOP_FOOT_SIZE) to enable
+        separation or merging when space is extended.
+
+     3. Chunks allocated via mmap, have both cinuse and pinuse bits
+        cleared in their head fields.  Because they are allocated
+        one-by-one, each must carry its own prev_foot field, which is
+        also used to hold the offset this chunk has within its mmapped
+        region, which is needed to preserve alignment. Each mmapped
+        chunk is trailed by the first two fields of a fake next-chunk
+        for sake of usage checks.
+
+*/
+
+struct malloc_chunk {
+  size_t               prev_foot;  /* Size of previous chunk (if free).  */
+  size_t               head;       /* Size and inuse bits. */
+  struct malloc_chunk* fd;         /* double links -- used only if free. */
+  struct malloc_chunk* bk;
+};
+
+typedef struct malloc_chunk  mchunk;
+typedef struct malloc_chunk* mchunkptr;
+typedef struct malloc_chunk* sbinptr;  /* The type of bins of chunks */
+typedef unsigned int bindex_t;         /* Described below */
+typedef unsigned int binmap_t;         /* Described below */
+typedef unsigned int flag_t;           /* The type of various bit flag sets */
+
+/* ------------------- Chunks sizes and alignments ----------------------- */
+
+#define MCHUNK_SIZE         (sizeof(mchunk))
+
+#if FOOTERS
+#define CHUNK_OVERHEAD      (TWO_SIZE_T_SIZES)
+#else /* FOOTERS */
+#define CHUNK_OVERHEAD      (SIZE_T_SIZE)
+#endif /* FOOTERS */
+
+/* MMapped chunks need a second word of overhead ... */
+#define MMAP_CHUNK_OVERHEAD (TWO_SIZE_T_SIZES)
+/* ... and additional padding for fake next-chunk at foot */
+#define MMAP_FOOT_PAD       (FOUR_SIZE_T_SIZES)
+
+/* The smallest size we can malloc is an aligned minimal chunk */
+#define MIN_CHUNK_SIZE\
+  ((MCHUNK_SIZE + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
+
+/* conversion from malloc headers to user pointers, and back */
+#define chunk2mem(p)        ((void*)((char*)(p)       + TWO_SIZE_T_SIZES))
+#define mem2chunk(mem)      ((mchunkptr)((char*)(mem) - TWO_SIZE_T_SIZES))
+/* chunk associated with aligned address A */
+#define align_as_chunk(A)   (mchunkptr)((A) + align_offset(chunk2mem(A)))
+
+/* Bounds on request (not chunk) sizes. */
+#define MAX_REQUEST         ((-MIN_CHUNK_SIZE) << 2)
+#define MIN_REQUEST         (MIN_CHUNK_SIZE - CHUNK_OVERHEAD - SIZE_T_ONE)
+
+/* pad request bytes into a usable size */
+#define pad_request(req) \
+   (((req) + CHUNK_OVERHEAD + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
+
+/* pad request, checking for minimum (but not maximum) */
+#define request2size(req) \
+  (((req) < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(req))
+
+
+/* ------------------ Operations on head and foot fields ----------------- */
+
+/*
+  The head field of a chunk is or'ed with PINUSE_BIT when previous
+  adjacent chunk in use, and or'ed with CINUSE_BIT if this chunk is in
+  use, unless mmapped, in which case both bits are cleared.
+
+  FLAG4_BIT is not used by this malloc, but might be useful in extensions.
+*/
+
+#define PINUSE_BIT          (SIZE_T_ONE)
+#define CINUSE_BIT          (SIZE_T_TWO)
+#define FLAG4_BIT           (SIZE_T_FOUR)
+#define INUSE_BITS          (PINUSE_BIT|CINUSE_BIT)
+#define FLAG_BITS           (PINUSE_BIT|CINUSE_BIT|FLAG4_BIT)
+
+/* Head value for fenceposts */
+#define FENCEPOST_HEAD      (INUSE_BITS|SIZE_T_SIZE)
+
+/* extraction of fields from head words */
+#define cinuse(p)           ((p)->head & CINUSE_BIT)
+#define pinuse(p)           ((p)->head & PINUSE_BIT)
+#define flag4inuse(p)       ((p)->head & FLAG4_BIT)
+#define is_inuse(p)         (((p)->head & INUSE_BITS) != PINUSE_BIT)
+#define is_mmapped(p)       (((p)->head & INUSE_BITS) == 0)
+
+#define chunksize(p)        ((p)->head & ~(FLAG_BITS))
+
+#define clear_pinuse(p)     ((p)->head &= ~PINUSE_BIT)
+#define set_flag4(p)        ((p)->head |= FLAG4_BIT)
+#define clear_flag4(p)      ((p)->head &= ~FLAG4_BIT)
+
+/* Treat space at ptr +/- offset as a chunk */
+#define chunk_plus_offset(p, s)  ((mchunkptr)(((char*)(p)) + (s)))
+#define chunk_minus_offset(p, s) ((mchunkptr)(((char*)(p)) - (s)))
+
+/* Ptr to next or previous physical malloc_chunk. */
+#define next_chunk(p) ((mchunkptr)( ((char*)(p)) + ((p)->head & ~FLAG_BITS)))
+#define prev_chunk(p) ((mchunkptr)( ((char*)(p)) - ((p)->prev_foot) ))
+
+/* extract next chunk's pinuse bit */
+#define next_pinuse(p)  ((next_chunk(p)->head) & PINUSE_BIT)
+
+/* Get/set size at footer */
+#define get_foot(p, s)  (((mchunkptr)((char*)(p) + (s)))->prev_foot)
+#define set_foot(p, s)  (((mchunkptr)((char*)(p) + (s)))->prev_foot = (s))
+
+/* Set size, pinuse bit, and foot */
+#define set_size_and_pinuse_of_free_chunk(p, s)\
+  ((p)->head = (s|PINUSE_BIT), set_foot(p, s))
+
+/* Set size, pinuse bit, foot, and clear next pinuse */
+#define set_free_with_pinuse(p, s, n)\
+  (clear_pinuse(n), set_size_and_pinuse_of_free_chunk(p, s))
+
+/* Get the internal overhead associated with chunk p */
+#define overhead_for(p)\
+ (is_mmapped(p)? MMAP_CHUNK_OVERHEAD : CHUNK_OVERHEAD)
+
+/* Return true if malloced space is not necessarily cleared */
+#if MMAP_CLEARS
+#define calloc_must_clear(p) (!is_mmapped(p))
+#else /* MMAP_CLEARS */
+#define calloc_must_clear(p) (1)
+#endif /* MMAP_CLEARS */
+
+/* ---------------------- Overlaid data structures ----------------------- */
+
+/*
+  When chunks are not in use, they are treated as nodes of either
+  lists or trees.
+
+  "Small"  chunks are stored in circular doubly-linked lists, and look
+  like this:
+
+    chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Size of previous chunk                            |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    `head:' |             Size of chunk, in bytes                         |P|
+      mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Forward pointer to next chunk in list             |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Back pointer to previous chunk in list            |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Unused space (may be 0 bytes long)                .
+            .                                                               .
+            .                                                               |
+nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    `foot:' |             Size of chunk, in bytes                           |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+  Larger chunks are kept in a form of bitwise digital trees (aka
+  tries) keyed on chunksizes.  Because malloc_tree_chunks are only for
+  free chunks greater than 256 bytes, their size doesn't impose any
+  constraints on user chunk sizes.  Each node looks like:
+
+    chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Size of previous chunk                            |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    `head:' |             Size of chunk, in bytes                         |P|
+      mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Forward pointer to next chunk of same size        |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Back pointer to previous chunk of same size       |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Pointer to left child (child[0])                  |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Pointer to right child (child[1])                 |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Pointer to parent                                 |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             bin index of this chunk                           |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |             Unused space                                      .
+            .                                                               |
+nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    `foot:' |             Size of chunk, in bytes                           |
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+  Each tree holding treenodes is a tree of unique chunk sizes.  Chunks
+  of the same size are arranged in a circularly-linked list, with only
+  the oldest chunk (the next to be used, in our FIFO ordering)
+  actually in the tree.  (Tree members are distinguished by a non-null
+  parent pointer.)  If a chunk with the same size an an existing node
+  is inserted, it is linked off the existing node using pointers that
+  work in the same way as fd/bk pointers of small chunks.
+
+  Each tree contains a power of 2 sized range of chunk sizes (the
+  smallest is 0x100 <= x < 0x180), which is is divided in half at each
+  tree level, with the chunks in the smaller half of the range (0x100
+  <= x < 0x140 for the top nose) in the left subtree and the larger
+  half (0x140 <= x < 0x180) in the right subtree.  This is, of course,
+  done by inspecting individual bits.
+
+  Using these rules, each node's left subtree contains all smaller
+  sizes than its right subtree.  However, the node at the root of each
+  subtree has no particular ordering relationship to either.  (The
+  dividing line between the subtree sizes is based on trie relation.)
+  If we remove the last chunk of a given size from the interior of the
+  tree, we need to replace it with a leaf node.  The tree ordering
+  rules permit a node to be replaced by any leaf below it.
+
+  The smallest chunk in a tree (a common operation in a best-fit
+  allocator) can be found by walking a path to the leftmost leaf in
+  the tree.  Unlike a usual binary tree, where we follow left child
+  pointers until we reach a null, here we follow the right child
+  pointer any time the left one is null, until we reach a leaf with
+  both child pointers null. The smallest chunk in the tree will be
+  somewhere along that path.
+
+  The worst case number of steps to add, find, or remove a node is
+  bounded by the number of bits differentiating chunks within
+  bins. Under current bin calculations, this ranges from 6 up to 21
+  (for 32 bit sizes) or up to 53 (for 64 bit sizes). The typical case
+  is of course much better.
+*/
+
+struct malloc_tree_chunk {
+  /* The first four fields must be compatible with malloc_chunk */
+  size_t                    prev_foot;
+  size_t                    head;
+  struct malloc_tree_chunk* fd;
+  struct malloc_tree_chunk* bk;
+
+  struct malloc_tree_chunk* child[2];
+  struct malloc_tree_chunk* parent;
+  bindex_t                  index;
+};
+
+typedef struct malloc_tree_chunk  tchunk;
+typedef struct malloc_tree_chunk* tchunkptr;
+typedef struct malloc_tree_chunk* tbinptr; /* The type of bins of trees */
+
+/* A little helper macro for trees */
+#define leftmost_child(t) ((t)->child[0] != 0? (t)->child[0] : (t)->child[1])
+
+/* ----------------------------- Segments -------------------------------- */
+
+/*
+  Each malloc space may include non-contiguous segments, held in a
+  list headed by an embedded malloc_segment record representing the
+  top-most space. Segments also include flags holding properties of
+  the space. Large chunks that are directly allocated by mmap are not
+  included in this list. They are instead independently created and
+  destroyed without otherwise keeping track of them.
+
+  Segment management mainly comes into play for spaces allocated by
+  MMAP.  Any call to MMAP might or might not return memory that is
+  adjacent to an existing segment.  MORECORE normally contiguously
+  extends the current space, so this space is almost always adjacent,
+  which is simpler and faster to deal with. (This is why MORECORE is
+  used preferentially to MMAP when both are available -- see
+  sys_alloc.)  When allocating using MMAP, we don't use any of the
+  hinting mechanisms (inconsistently) supported in various
+  implementations of unix mmap, or distinguish reserving from
+  committing memory. Instead, we just ask for space, and exploit
+  contiguity when we get it.  It is probably possible to do
+  better than this on some systems, but no general scheme seems
+  to be significantly better.
+
+  Management entails a simpler variant of the consolidation scheme
+  used for chunks to reduce fragmentation -- new adjacent memory is
+  normally prepended or appended to an existing segment. However,
+  there are limitations compared to chunk consolidation that mostly
+  reflect the fact that segment processing is relatively infrequent
+  (occurring only when getting memory from system) and that we
+  don't expect to have huge numbers of segments:
+
+  * Segments are not indexed, so traversal requires linear scans.  (It
+    would be possible to index these, but is not worth the extra
+    overhead and complexity for most programs on most platforms.)
+  * New segments are only appended to old ones when holding top-most
+    memory; if they cannot be prepended to others, they are held in
+    different segments.
+
+  Except for the top-most segment of an mstate, each segment record
+  is kept at the tail of its segment. Segments are added by pushing
+  segment records onto the list headed by &mstate.seg for the
+  containing mstate.
+
+  Segment flags control allocation/merge/deallocation policies:
+  * If EXTERN_BIT set, then we did not allocate this segment,
+    and so should not try to deallocate or merge with others.
+    (This currently holds only for the initial segment passed
+    into create_mspace_with_base.)
+  * If USE_MMAP_BIT set, the segment may be merged with
+    other surrounding mmapped segments and trimmed/de-allocated
+    using munmap.
+  * If neither bit is set, then the segment was obtained using
+    MORECORE so can be merged with surrounding MORECORE'd segments
+    and deallocated/trimmed using MORECORE with negative arguments.
+*/
+
+struct malloc_segment {
+  char*        base;             /* base address */
+  size_t       size;             /* allocated size */
+  struct malloc_segment* next;   /* ptr to next segment */
+  flag_t       sflags;           /* mmap and extern flag */
+};
+
+#define is_mmapped_segment(S)  ((S)->sflags & USE_MMAP_BIT)
+#define is_extern_segment(S)   ((S)->sflags & EXTERN_BIT)
+
+typedef struct malloc_segment  msegment;
+typedef struct malloc_segment* msegmentptr;
+
+/* ---------------------------- malloc_state ----------------------------- */
+
+/*
+   A malloc_state holds all of the bookkeeping for a space.
+   The main fields are:
+
+  Top
+    The topmost chunk of the currently active segment. Its size is
+    cached in topsize.  The actual size of topmost space is
+    topsize+TOP_FOOT_SIZE, which includes space reserved for adding
+    fenceposts and segment records if necessary when getting more
+    space from the system.  The size at which to autotrim top is
+    cached from mparams in trim_check, except that it is disabled if
+    an autotrim fails.
+
+  Designated victim (dv)
+    This is the preferred chunk for servicing small requests that
+    don't have exact fits.  It is normally the chunk split off most
+    recently to service another small request.  Its size is cached in
+    dvsize. The link fields of this chunk are not maintained since it
+    is not kept in a bin.
+
+  SmallBins
+    An array of bin headers for free chunks.  These bins hold chunks
+    with sizes less than MIN_LARGE_SIZE bytes. Each bin contains
+    chunks of all the same size, spaced 8 bytes apart.  To simplify
+    use in double-linked lists, each bin header acts as a malloc_chunk
+    pointing to the real first node, if it exists (else pointing to
+    itself).  This avoids special-casing for headers.  But to avoid
+    waste, we allocate only the fd/bk pointers of bins, and then use
+    repositioning tricks to treat these as the fields of a chunk.
+
+  TreeBins
+    Treebins are pointers to the roots of trees holding a range of
+    sizes. There are 2 equally spaced treebins for each power of two
+    from TREE_SHIFT to TREE_SHIFT+16. The last bin holds anything
+    larger.
+
+  Bin maps
+    There is one bit map for small bins ("smallmap") and one for
+    treebins ("treemap).  Each bin sets its bit when non-empty, and
+    clears the bit when empty.  Bit operations are then used to avoid
+    bin-by-bin searching -- nearly all "search" is done without ever
+    looking at bins that won't be selected.  The bit maps
+    conservatively use 32 bits per map word, even if on 64bit system.
+    For a good description of some of the bit-based techniques used
+    here, see Henry S. Warren Jr's book "Hacker's Delight" (and
+    supplement at http://hackersdelight.org/). Many of these are
+    intended to reduce the branchiness of paths through malloc etc, as
+    well as to reduce the number of memory locations read or written.
+
+  Segments
+    A list of segments headed by an embedded malloc_segment record
+    representing the initial space.
+
+  Address check support
+    The least_addr field is the least address ever obtained from
+    MORECORE or MMAP. Attempted frees and reallocs of any address less
+    than this are trapped (unless INSECURE is defined).
+
+  Magic tag
+    A cross-check field that should always hold same value as mparams.magic.
+
+  Max allowed footprint
+    The maximum allowed bytes to allocate from system (zero means no limit)
+
+  Flags
+    Bits recording whether to use MMAP, locks, or contiguous MORECORE
+
+  Statistics
+    Each space keeps track of current and maximum system memory
+    obtained via MORECORE or MMAP.
+
+  Trim support
+    Fields holding the amount of unused topmost memory that should trigger
+    trimming, and a counter to force periodic scanning to release unused
+    non-topmost segments.
+
+  Locking
+    If USE_LOCKS is defined, the "mutex" lock is acquired and released
+    around every public call using this mspace.
+
+  Extension support
+    A void* pointer and a size_t field that can be used to help implement
+    extensions to this malloc.
+*/
+
+/* Bin types, widths and sizes */
+#define NSMALLBINS        (32U)
+#define NTREEBINS         (32U)
+#define SMALLBIN_SHIFT    (3U)
+#define SMALLBIN_WIDTH    (SIZE_T_ONE << SMALLBIN_SHIFT)
+#define TREEBIN_SHIFT     (8U)
+#define MIN_LARGE_SIZE    (SIZE_T_ONE << TREEBIN_SHIFT)
+#define MAX_SMALL_SIZE    (MIN_LARGE_SIZE - SIZE_T_ONE)
+#define MAX_SMALL_REQUEST (MAX_SMALL_SIZE - CHUNK_ALIGN_MASK - CHUNK_OVERHEAD)
+
+struct malloc_state {
+  binmap_t   smallmap;
+  binmap_t   treemap;
+  size_t     dvsize;
+  size_t     topsize;
+  char*      least_addr;
+  mchunkptr  dv;
+  mchunkptr  top;
+  size_t     trim_check;
+  size_t     release_checks;
+  size_t     magic;
+  mchunkptr  smallbins[(NSMALLBINS+1)*2];
+  tbinptr    treebins[NTREEBINS];
+  size_t     footprint;
+  size_t     max_footprint;
+  size_t     footprint_limit; /* zero means no limit */
+  flag_t     mflags;
+#if USE_LOCKS
+  MLOCK_T    mutex;     /* locate lock among fields that rarely change */
+#endif /* USE_LOCKS */
+  msegment   seg;
+  void*      extp;      /* Unused but available for extensions */
+  size_t     exts;
+};
+
+typedef struct malloc_state*    mstate;
+
+/* ------------- Global malloc_state and malloc_params ------------------- */
+
+/*
+  malloc_params holds global properties, including those that can be
+  dynamically set using mallopt. There is a single instance, mparams,
+  initialized in init_mparams. Note that the non-zeroness of "magic"
+  also serves as an initialization flag.
+*/
+
+struct malloc_params {
+  size_t magic;
+  size_t page_size;
+  size_t granularity;
+  size_t mmap_threshold;
+  size_t trim_threshold;
+  flag_t default_mflags;
+};
+
+static struct malloc_params mparams;
+
+/* Ensure mparams initialized */
+#define ensure_initialization() (void)(mparams.magic != 0 || init_mparams())
+
+#if !ONLY_MSPACES
+
+/* The global malloc_state used for all non-"mspace" calls */
+static struct malloc_state _gm_;
+#define gm                 (&_gm_)
+#define is_global(M)       ((M) == &_gm_)
+
+#endif /* !ONLY_MSPACES */
+
+#define is_initialized(M)  ((M)->top != 0)
+
+/* -------------------------- system alloc setup ------------------------- */
+
+/* Operations on mflags */
+
+#define use_lock(M)           ((M)->mflags &   USE_LOCK_BIT)
+#define enable_lock(M)        ((M)->mflags |=  USE_LOCK_BIT)
+#if USE_LOCKS
+#define disable_lock(M)       ((M)->mflags &= ~USE_LOCK_BIT)
+#else
+#define disable_lock(M)
+#endif
+
+#define use_mmap(M)           ((M)->mflags &   USE_MMAP_BIT)
+#define enable_mmap(M)        ((M)->mflags |=  USE_MMAP_BIT)
+#if HAVE_MMAP
+#define disable_mmap(M)       ((M)->mflags &= ~USE_MMAP_BIT)
+#else
+#define disable_mmap(M)
+#endif
+
+#define use_noncontiguous(M)  ((M)->mflags &   USE_NONCONTIGUOUS_BIT)
+#define disable_contiguous(M) ((M)->mflags |=  USE_NONCONTIGUOUS_BIT)
+
+#define set_lock(M,L)\
+ ((M)->mflags = (L)?\
+  ((M)->mflags | USE_LOCK_BIT) :\
+  ((M)->mflags & ~USE_LOCK_BIT))
+
+/* page-align a size */
+#define page_align(S)\
+ (((S) + (mparams.page_size - SIZE_T_ONE)) & ~(mparams.page_size - SIZE_T_ONE))
+
+/* granularity-align a size */
+#define granularity_align(S)\
+  (((S) + (mparams.granularity - SIZE_T_ONE))\
+   & ~(mparams.granularity - SIZE_T_ONE))
+
+
+/* For mmap, use granularity alignment on windows, else page-align */
+#ifdef WIN32
+#define mmap_align(S) granularity_align(S)
+#else
+#define mmap_align(S) page_align(S)
+#endif
+
+/* For sys_alloc, enough padding to ensure can malloc request on success */
+#define SYS_ALLOC_PADDING (TOP_FOOT_SIZE + MALLOC_ALIGNMENT)
+
+#define is_page_aligned(S)\
+   (((size_t)(S) & (mparams.page_size - SIZE_T_ONE)) == 0)
+#define is_granularity_aligned(S)\
+   (((size_t)(S) & (mparams.granularity - SIZE_T_ONE)) == 0)
+
+/*  True if segment S holds address A */
+#define segment_holds(S, A)\
+  ((char*)(A) >= S->base && (char*)(A) < S->base + S->size)
+
+/* Return segment holding given address */
+static msegmentptr segment_holding(mstate m, char* addr) {
+  msegmentptr sp = &m->seg;
+  for (;;) {
+    if (addr >= sp->base && addr < sp->base + sp->size)
+      return sp;
+    if ((sp = sp->next) == 0)
+      return 0;
+  }
+}
+
+/* Return true if segment contains a segment link */
+static int has_segment_link(mstate m, msegmentptr ss) {
+  msegmentptr sp = &m->seg;
+  for (;;) {
+    if ((char*)sp >= ss->base && (char*)sp < ss->base + ss->size)
+      return 1;
+    if ((sp = sp->next) == 0)
+      return 0;
+  }
+}
+
+#ifndef MORECORE_CANNOT_TRIM
+#define should_trim(M,s)  ((s) > (M)->trim_check)
+#else  /* MORECORE_CANNOT_TRIM */
+#define should_trim(M,s)  (0)
+#endif /* MORECORE_CANNOT_TRIM */
+
+/*
+  TOP_FOOT_SIZE is padding at the end of a segment, including space
+  that may be needed to place segment records and fenceposts when new
+  noncontiguous segments are added.
+*/
+#define TOP_FOOT_SIZE\
+  (align_offset(chunk2mem(0))+pad_request(sizeof(struct malloc_segment))+MIN_CHUNK_SIZE)
+
+
+/* -------------------------------  Hooks -------------------------------- */
+
+/*
+  PREACTION should be defined to return 0 on success, and nonzero on
+  failure. If you are not using locking, you can redefine these to do
+  anything you like.
+*/
+
+#if USE_LOCKS
+#define PREACTION(M)  ((use_lock(M))? ACQUIRE_LOCK(&(M)->mutex) : 0)
+#define POSTACTION(M) { if (use_lock(M)) RELEASE_LOCK(&(M)->mutex); }
+#else /* USE_LOCKS */
+
+#ifndef PREACTION
+#define PREACTION(M) (0)
+#endif  /* PREACTION */
+
+#ifndef POSTACTION
+#define POSTACTION(M)
+#endif  /* POSTACTION */
+
+#endif /* USE_LOCKS */
+
+/*
+  CORRUPTION_ERROR_ACTION is triggered upon detected bad addresses.
+  USAGE_ERROR_ACTION is triggered on detected bad frees and
+  reallocs. The argument p is an address that might have triggered the
+  fault. It is ignored by the two predefined actions, but might be
+  useful in custom actions that try to help diagnose errors.
+*/
+
+#if PROCEED_ON_ERROR
+
+/* A count of the number of corruption errors causing resets */
+int malloc_corruption_error_count;
+
+/* default corruption action */
+static void reset_on_error(mstate m);
+
+#define CORRUPTION_ERROR_ACTION(m)  reset_on_error(m)
+#define USAGE_ERROR_ACTION(m, p)
+
+#else /* PROCEED_ON_ERROR */
+
+#ifndef CORRUPTION_ERROR_ACTION
+#define CORRUPTION_ERROR_ACTION(m) ABORT
+#endif /* CORRUPTION_ERROR_ACTION */
+
+#ifndef USAGE_ERROR_ACTION
+#define USAGE_ERROR_ACTION(m,p) ABORT
+#endif /* USAGE_ERROR_ACTION */
+
+#endif /* PROCEED_ON_ERROR */
+
+
+/* -------------------------- Debugging setup ---------------------------- */
+
+#if ! DEBUG
+
+#define check_free_chunk(M,P)
+#define check_inuse_chunk(M,P)
+#define check_malloced_chunk(M,P,N)
+#define check_mmapped_chunk(M,P)
+#define check_malloc_state(M)
+#define check_top_chunk(M,P)
+
+#else /* DEBUG */
+#define check_free_chunk(M,P)       do_check_free_chunk(M,P)
+#define check_inuse_chunk(M,P)      do_check_inuse_chunk(M,P)
+#define check_top_chunk(M,P)        do_check_top_chunk(M,P)
+#define check_malloced_chunk(M,P,N) do_check_malloced_chunk(M,P,N)
+#define check_mmapped_chunk(M,P)    do_check_mmapped_chunk(M,P)
+#define check_malloc_state(M)       do_check_malloc_state(M)
+
+static void   do_check_any_chunk(mstate m, mchunkptr p);
+static void   do_check_top_chunk(mstate m, mchunkptr p);
+static void   do_check_mmapped_chunk(mstate m, mchunkptr p);
+static void   do_check_inuse_chunk(mstate m, mchunkptr p);
+static void   do_check_free_chunk(mstate m, mchunkptr p);
+static void   do_check_malloced_chunk(mstate m, void* mem, size_t s);
+static void   do_check_tree(mstate m, tchunkptr t);
+static void   do_check_treebin(mstate m, bindex_t i);
+static void   do_check_smallbin(mstate m, bindex_t i);
+static void   do_check_malloc_state(mstate m);
+static int    bin_find(mstate m, mchunkptr x);
+static size_t traverse_and_check(mstate m);
+#endif /* DEBUG */
+
+/* ---------------------------- Indexing Bins ---------------------------- */
+
+#define is_small(s)         (((s) >> SMALLBIN_SHIFT) < NSMALLBINS)
+#define small_index(s)      (bindex_t)((s)  >> SMALLBIN_SHIFT)
+#define small_index2size(i) ((i)  << SMALLBIN_SHIFT)
+#define MIN_SMALL_INDEX     (small_index(MIN_CHUNK_SIZE))
+
+/* addressing by index. See above about smallbin repositioning */
+#define smallbin_at(M, i)   ((sbinptr)((char*)&((M)->smallbins[(i)<<1])))
+#define treebin_at(M,i)     (&((M)->treebins[i]))
+
+/* assign tree index for size S to variable I. Use x86 asm if possible  */
+#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
+#define compute_tree_index(S, I)\
+{\
+  unsigned int X = S >> TREEBIN_SHIFT;\
+  if (X == 0)\
+    I = 0;\
+  else if (X > 0xFFFF)\
+    I = NTREEBINS-1;\
+  else {\
+    unsigned int K = (unsigned) sizeof(X)*__CHAR_BIT__ - 1 - (unsigned) __builtin_clz(X); \
+    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
+  }\
+}
+
+#elif defined (__INTEL_COMPILER)
+#define compute_tree_index(S, I)\
+{\
+  size_t X = S >> TREEBIN_SHIFT;\
+  if (X == 0)\
+    I = 0;\
+  else if (X > 0xFFFF)\
+    I = NTREEBINS-1;\
+  else {\
+    unsigned int K = _bit_scan_reverse (X); \
+    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
+  }\
+}
+
+#elif defined(_MSC_VER) && _MSC_VER>=1300
+#define compute_tree_index(S, I)\
+{\
+  size_t X = S >> TREEBIN_SHIFT;\
+  if (X == 0)\
+    I = 0;\
+  else if (X > 0xFFFF)\
+    I = NTREEBINS-1;\
+  else {\
+    unsigned int K;\
+    _BitScanReverse((DWORD *) &K, (DWORD) X);\
+    I =  (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
+  }\
+}
+
+#else /* GNUC */
+#define compute_tree_index(S, I)\
+{\
+  size_t X = S >> TREEBIN_SHIFT;\
+  if (X == 0)\
+    I = 0;\
+  else if (X > 0xFFFF)\
+    I = NTREEBINS-1;\
+  else {\
+    unsigned int Y = (unsigned int)X;\
+    unsigned int N = ((Y - 0x100) >> 16) & 8;\
+    unsigned int K = (((Y <<= N) - 0x1000) >> 16) & 4;\
+    N += K;\
+    N += K = (((Y <<= K) - 0x4000) >> 16) & 2;\
+    K = 14 - N + ((Y <<= K) >> 15);\
+    I = (K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1));\
+  }\
+}
+#endif /* GNUC */
+
+/* Bit representing maximum resolved size in a treebin at i */
+#define bit_for_tree_index(i) \
+   (i == NTREEBINS-1)? (SIZE_T_BITSIZE-1) : (((i) >> 1) + TREEBIN_SHIFT - 2)
+
+/* Shift placing maximum resolved bit in a treebin at i as sign bit */
+#define leftshift_for_tree_index(i) \
+   ((i == NTREEBINS-1)? 0 : \
+    ((SIZE_T_BITSIZE-SIZE_T_ONE) - (((i) >> 1) + TREEBIN_SHIFT - 2)))
+
+/* The size of the smallest chunk held in bin with index i */
+#define minsize_for_tree_index(i) \
+   ((SIZE_T_ONE << (((i) >> 1) + TREEBIN_SHIFT)) |  \
+   (((size_t)((i) & SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1)))
+
+
+/* ------------------------ Operations on bin maps ----------------------- */
+
+/* bit corresponding to given index */
+#define idx2bit(i)              ((binmap_t)(1) << (i))
+
+/* Mark/Clear bits with given index */
+#define mark_smallmap(M,i)      ((M)->smallmap |=  idx2bit(i))
+#define clear_smallmap(M,i)     ((M)->smallmap &= ~idx2bit(i))
+#define smallmap_is_marked(M,i) ((M)->smallmap &   idx2bit(i))
+
+#define mark_treemap(M,i)       ((M)->treemap  |=  idx2bit(i))
+#define clear_treemap(M,i)      ((M)->treemap  &= ~idx2bit(i))
+#define treemap_is_marked(M,i)  ((M)->treemap  &   idx2bit(i))
+
+/* isolate the least set bit of a bitmap */
+#define least_bit(x)         ((x) & -(x))
+
+/* mask with all bits to left of least bit of x on */
+#define left_bits(x)         ((x<<1) | -(x<<1))
+
+/* mask with all bits to left of or equal to least bit of x on */
+#define same_or_left_bits(x) ((x) | -(x))
+
+/* index corresponding to given bit. Use x86 asm if possible */
+
+#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
+#define compute_bit2idx(X, I)\
+{\
+  unsigned int J;\
+  J = __builtin_ctz(X); \
+  I = (bindex_t)J;\
+}
+
+#elif defined (__INTEL_COMPILER)
+#define compute_bit2idx(X, I)\
+{\
+  unsigned int J;\
+  J = _bit_scan_forward (X); \
+  I = (bindex_t)J;\
+}
+
+#elif defined(_MSC_VER) && _MSC_VER>=1300
+#define compute_bit2idx(X, I)\
+{\
+  unsigned int J;\
+  _BitScanForward((DWORD *) &J, X);\
+  I = (bindex_t)J;\
+}
+
+#elif USE_BUILTIN_FFS
+#define compute_bit2idx(X, I) I = ffs(X)-1
+
+#else
+#define compute_bit2idx(X, I)\
+{\
+  unsigned int Y = X - 1;\
+  unsigned int K = Y >> (16-4) & 16;\
+  unsigned int N = K;        Y >>= K;\
+  N += K = Y >> (8-3) &  8;  Y >>= K;\
+  N += K = Y >> (4-2) &  4;  Y >>= K;\
+  N += K = Y >> (2-1) &  2;  Y >>= K;\
+  N += K = Y >> (1-0) &  1;  Y >>= K;\
+  I = (bindex_t)(N + Y);\
+}
+#endif /* GNUC */
+
+
+/* ----------------------- Runtime Check Support ------------------------- */
+
+/*
+  For security, the main invariant is that malloc/free/etc never
+  writes to a static address other than malloc_state, unless static
+  malloc_state itself has been corrupted, which cannot occur via
+  malloc (because of these checks). In essence this means that we
+  believe all pointers, sizes, maps etc held in malloc_state, but
+  check all of those linked or offsetted from other embedded data
+  structures.  These checks are interspersed with main code in a way
+  that tends to minimize their run-time cost.
+
+  When FOOTERS is defined, in addition to range checking, we also
+  verify footer fields of inuse chunks, which can be used guarantee
+  that the mstate controlling malloc/free is intact.  This is a
+  streamlined version of the approach described by William Robertson
+  et al in "Run-time Detection of Heap-based Overflows" LISA'03
+  http://www.usenix.org/events/lisa03/tech/robertson.html The footer
+  of an inuse chunk holds the xor of its mstate and a random seed,
+  that is checked upon calls to free() and realloc().  This is
+  (probabalistically) unguessable from outside the program, but can be
+  computed by any code successfully malloc'ing any chunk, so does not
+  itself provide protection against code that has already broken
+  security through some other means.  Unlike Robertson et al, we
+  always dynamically check addresses of all offset chunks (previous,
+  next, etc). This turns out to be cheaper than relying on hashes.
+*/
+
+#if !INSECURE
+/* Check if address a is at least as high as any from MORECORE or MMAP */
+#define ok_address(M, a) ((char*)(a) >= (M)->least_addr)
+/* Check if address of next chunk n is higher than base chunk p */
+#define ok_next(p, n)    ((char*)(p) < (char*)(n))
+/* Check if p has inuse status */
+#define ok_inuse(p)     is_inuse(p)
+/* Check if p has its pinuse bit on */
+#define ok_pinuse(p)     pinuse(p)
+
+#else /* !INSECURE */
+#define ok_address(M, a) (1)
+#define ok_next(b, n)    (1)
+#define ok_inuse(p)      (1)
+#define ok_pinuse(p)     (1)
+#endif /* !INSECURE */
+
+#if (FOOTERS && !INSECURE)
+/* Check if (alleged) mstate m has expected magic field */
+#define ok_magic(M)      ((M)->magic == mparams.magic)
+#else  /* (FOOTERS && !INSECURE) */
+#define ok_magic(M)      (1)
+#endif /* (FOOTERS && !INSECURE) */
+
+/* In gcc, use __builtin_expect to minimize impact of checks */
+#if !INSECURE
+#if defined(__GNUC__) && __GNUC__ >= 3
+#define RTCHECK(e)  __builtin_expect(e, 1)
+#else /* GNUC */
+#define RTCHECK(e)  (e)
+#endif /* GNUC */
+#else /* !INSECURE */
+#define RTCHECK(e)  (1)
+#endif /* !INSECURE */
+
+/* macros to set up inuse chunks with or without footers */
+
+#if !FOOTERS
+
+#define mark_inuse_foot(M,p,s)
+
+/* Macros for setting head/foot of non-mmapped chunks */
+
+/* Set cinuse bit and pinuse bit of next chunk */
+#define set_inuse(M,p,s)\
+  ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\
+  ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT)
+
+/* Set cinuse and pinuse of this chunk and pinuse of next chunk */
+#define set_inuse_and_pinuse(M,p,s)\
+  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
+  ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT)
+
+/* Set size, cinuse and pinuse bit of this chunk */
+#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\
+  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT))
+
+#else /* FOOTERS */
+
+/* Set foot of inuse chunk to be xor of mstate and seed */
+#define mark_inuse_foot(M,p,s)\
+  (((mchunkptr)((char*)(p) + (s)))->prev_foot = ((size_t)(M) ^ mparams.magic))
+
+#define get_mstate_for(p)\
+  ((mstate)(((mchunkptr)((char*)(p) +\
+    (chunksize(p))))->prev_foot ^ mparams.magic))
+
+#define set_inuse(M,p,s)\
+  ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\
+  (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT), \
+  mark_inuse_foot(M,p,s))
+
+#define set_inuse_and_pinuse(M,p,s)\
+  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
+  (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT),\
+ mark_inuse_foot(M,p,s))
+
+#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\
+  ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
+  mark_inuse_foot(M, p, s))
+
+#endif /* !FOOTERS */
+
+/* ---------------------------- setting mparams -------------------------- */
+
+#if LOCK_AT_FORK
+static void pre_fork(void)         { ACQUIRE_LOCK(&(gm)->mutex); }
+static void post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
+static void post_fork_child(void)  { INITIAL_LOCK(&(gm)->mutex); }
+#endif /* LOCK_AT_FORK */
+
+/* Initialize mparams */
+static int init_mparams(void) {
+#ifdef NEED_GLOBAL_LOCK_INIT
+  if (malloc_global_mutex_status <= 0)
+    init_malloc_global_mutex();
+#endif
+
+  ACQUIRE_MALLOC_GLOBAL_LOCK();
+  if (mparams.magic == 0) {
+    size_t magic;
+    size_t psize;
+    size_t gsize;
+
+#ifndef WIN32
+    psize = malloc_getpagesize;
+    gsize = ((DEFAULT_GRANULARITY != 0)? DEFAULT_GRANULARITY : psize);
+#else /* WIN32 */
+    {
+      SYSTEM_INFO system_info;
+      GetSystemInfo(&system_info);
+      psize = system_info.dwPageSize;
+      gsize = ((DEFAULT_GRANULARITY != 0)?
+               DEFAULT_GRANULARITY : system_info.dwAllocationGranularity);
+    }
+#endif /* WIN32 */
+
+    /* Sanity-check configuration:
+       size_t must be unsigned and as wide as pointer type.
+       ints must be at least 4 bytes.
+       alignment must be at least 8.
+       Alignment, min chunk size, and page size must all be powers of 2.
+    */
+    if ((sizeof(size_t) != sizeof(char*)) ||
+        (MAX_SIZE_T < MIN_CHUNK_SIZE)  ||
+        (sizeof(int) < 4)  ||
+        (MALLOC_ALIGNMENT < (size_t)8U) ||
+        ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT-SIZE_T_ONE)) != 0) ||
+        ((MCHUNK_SIZE      & (MCHUNK_SIZE-SIZE_T_ONE))      != 0) ||
+        ((gsize            & (gsize-SIZE_T_ONE))            != 0) ||
+        ((psize            & (psize-SIZE_T_ONE))            != 0))
+      ABORT;
+    mparams.granularity = gsize;
+    mparams.page_size = psize;
+    mparams.mmap_threshold = DEFAULT_MMAP_THRESHOLD;
+    mparams.trim_threshold = DEFAULT_TRIM_THRESHOLD;
+#if MORECORE_CONTIGUOUS
+    mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT;
+#else  /* MORECORE_CONTIGUOUS */
+    mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT|USE_NONCONTIGUOUS_BIT;
+#endif /* MORECORE_CONTIGUOUS */
+
+#if !ONLY_MSPACES
+    /* Set up lock for main malloc area */
+    gm->mflags = mparams.default_mflags;
+    (void)INITIAL_LOCK(&gm->mutex);
+#endif
+#if LOCK_AT_FORK
+    pthread_atfork(&pre_fork, &post_fork_parent, &post_fork_child);
+#endif
+
+    {
+#if USE_DEV_RANDOM
+      int fd;
+      unsigned char buf[sizeof(size_t)];
+      /* Try to use /dev/urandom, else fall back on using time */
+      if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 &&
+          read(fd, buf, sizeof(buf)) == sizeof(buf)) {
+        magic = *((size_t *) buf);
+        close(fd);
+      }
+      else
+#endif /* USE_DEV_RANDOM */
+#ifdef WIN32
+      magic = (size_t)(GetTickCount() ^ (size_t)0x55555555U);
+#elif defined(LACKS_TIME_H)
+      magic = (size_t)&magic ^ (size_t)0x55555555U;
+#else
+      magic = (size_t)(time(0) ^ (size_t)0x55555555U);
+#endif
+      magic |= (size_t)8U;    /* ensure nonzero */
+      magic &= ~(size_t)7U;   /* improve chances of fault for bad values */
+      /* Until memory modes commonly available, use volatile-write */
+      (*(volatile size_t *)(&(mparams.magic))) = magic;
+    }
+  }
+
+  RELEASE_MALLOC_GLOBAL_LOCK();
+  return 1;
+}
+
+/* support for mallopt */
+static int change_mparam(int param_number, int value) {
+  size_t val;
+  ensure_initialization();
+  val = (value == -1)? MAX_SIZE_T : (size_t)value;
+  switch(param_number) {
+  case M_TRIM_THRESHOLD:
+    mparams.trim_threshold = val;
+    return 1;
+  case M_GRANULARITY:
+    if (val >= mparams.page_size && ((val & (val-1)) == 0)) {
+      mparams.granularity = val;
+      return 1;
+    }
+    else
+      return 0;
+  case M_MMAP_THRESHOLD:
+    mparams.mmap_threshold = val;
+    return 1;
+  default:
+    return 0;
+  }
+}
+
+#if DEBUG
+/* ------------------------- Debugging Support --------------------------- */
+
+/* Check properties of any chunk, whether free, inuse, mmapped etc  */
+static void do_check_any_chunk(mstate m, mchunkptr p) {
+  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
+  assert(ok_address(m, p));
+}
+
+/* Check properties of top chunk */
+static void do_check_top_chunk(mstate m, mchunkptr p) {
+  msegmentptr sp = segment_holding(m, (char*)p);
+  size_t  sz = p->head & ~INUSE_BITS; /* third-lowest bit can be set! */
+  assert(sp != 0);
+  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
+  assert(ok_address(m, p));
+  assert(sz == m->topsize);
+  assert(sz > 0);
+  assert(sz == ((sp->base + sp->size) - (char*)p) - TOP_FOOT_SIZE);
+  assert(pinuse(p));
+  assert(!pinuse(chunk_plus_offset(p, sz)));
+}
+
+/* Check properties of (inuse) mmapped chunks */
+static void do_check_mmapped_chunk(mstate m, mchunkptr p) {
+  size_t  sz = chunksize(p);
+  size_t len = (sz + (p->prev_foot) + MMAP_FOOT_PAD);
+  assert(is_mmapped(p));
+  assert(use_mmap(m));
+  assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
+  assert(ok_address(m, p));
+  assert(!is_small(sz));
+  assert((len & (mparams.page_size-SIZE_T_ONE)) == 0);
+  assert(chunk_plus_offset(p, sz)->head == FENCEPOST_HEAD);
+  assert(chunk_plus_offset(p, sz+SIZE_T_SIZE)->head == 0);
+}
+
+/* Check properties of inuse chunks */
+static void do_check_inuse_chunk(mstate m, mchunkptr p) {
+  do_check_any_chunk(m, p);
+  assert(is_inuse(p));
+  assert(next_pinuse(p));
+  /* If not pinuse and not mmapped, previous chunk has OK offset */
+  assert(is_mmapped(p) || pinuse(p) || next_chunk(prev_chunk(p)) == p);
+  if (is_mmapped(p))
+    do_check_mmapped_chunk(m, p);
+}
+
+/* Check properties of free chunks */
+static void do_check_free_chunk(mstate m, mchunkptr p) {
+  size_t sz = chunksize(p);
+  mchunkptr next = chunk_plus_offset(p, sz);
+  do_check_any_chunk(m, p);
+  assert(!is_inuse(p));
+  assert(!next_pinuse(p));
+  assert (!is_mmapped(p));
+  if (p != m->dv && p != m->top) {
+    if (sz >= MIN_CHUNK_SIZE) {
+      assert((sz & CHUNK_ALIGN_MASK) == 0);
+      assert(is_aligned(chunk2mem(p)));
+      assert(next->prev_foot == sz);
+      assert(pinuse(p));
+      assert (next == m->top || is_inuse(next));
+      assert(p->fd->bk == p);
+      assert(p->bk->fd == p);
+    }
+    else  /* markers are always of size SIZE_T_SIZE */
+      assert(sz == SIZE_T_SIZE);
+  }
+}
+
+/* Check properties of malloced chunks at the point they are malloced */
+static void do_check_malloced_chunk(mstate m, void* mem, size_t s) {
+  if (mem != 0) {
+    mchunkptr p = mem2chunk(mem);
+    size_t sz = p->head & ~INUSE_BITS;
+    do_check_inuse_chunk(m, p);
+    assert((sz & CHUNK_ALIGN_MASK) == 0);
+    assert(sz >= MIN_CHUNK_SIZE);
+    assert(sz >= s);
+    /* unless mmapped, size is less than MIN_CHUNK_SIZE more than request */
+    assert(is_mmapped(p) || sz < (s + MIN_CHUNK_SIZE));
+  }
+}
+
+/* Check a tree and its subtrees.  */
+static void do_check_tree(mstate m, tchunkptr t) {
+  tchunkptr head = 0;
+  tchunkptr u = t;
+  bindex_t tindex = t->index;
+  size_t tsize = chunksize(t);
+  bindex_t idx;
+  compute_tree_index(tsize, idx);
+  assert(tindex == idx);
+  assert(tsize >= MIN_LARGE_SIZE);
+  assert(tsize >= minsize_for_tree_index(idx));
+  assert((idx == NTREEBINS-1) || (tsize < minsize_for_tree_index((idx+1))));
+
+  do { /* traverse through chain of same-sized nodes */
+    do_check_any_chunk(m, ((mchunkptr)u));
+    assert(u->index == tindex);
+    assert(chunksize(u) == tsize);
+    assert(!is_inuse(u));
+    assert(!next_pinuse(u));
+    assert(u->fd->bk == u);
+    assert(u->bk->fd == u);
+    if (u->parent == 0) {
+      assert(u->child[0] == 0);
+      assert(u->child[1] == 0);
+    }
+    else {
+      assert(head == 0); /* only one node on chain has parent */
+      head = u;
+      assert(u->parent != u);
+      assert (u->parent->child[0] == u ||
+              u->parent->child[1] == u ||
+              *((tbinptr*)(u->parent)) == u);
+      if (u->child[0] != 0) {
+        assert(u->child[0]->parent == u);
+        assert(u->child[0] != u);
+        do_check_tree(m, u->child[0]);
+      }
+      if (u->child[1] != 0) {
+        assert(u->child[1]->parent == u);
+        assert(u->child[1] != u);
+        do_check_tree(m, u->child[1]);
+      }
+      if (u->child[0] != 0 && u->child[1] != 0) {
+        assert(chunksize(u->child[0]) < chunksize(u->child[1]));
+      }
+    }
+    u = u->fd;
+  } while (u != t);
+  assert(head != 0);
+}
+
+/*  Check all the chunks in a treebin.  */
+static void do_check_treebin(mstate m, bindex_t i) {
+  tbinptr* tb = treebin_at(m, i);
+  tchunkptr t = *tb;
+  int empty = (m->treemap & (1U << i)) == 0;
+  if (t == 0)
+    assert(empty);
+  if (!empty)
+    do_check_tree(m, t);
+}
+
+/*  Check all the chunks in a smallbin.  */
+static void do_check_smallbin(mstate m, bindex_t i) {
+  sbinptr b = smallbin_at(m, i);
+  mchunkptr p = b->bk;
+  unsigned int empty = (m->smallmap & (1U << i)) == 0;
+  if (p == b)
+    assert(empty);
+  if (!empty) {
+    for (; p != b; p = p->bk) {
+      size_t size = chunksize(p);
+      mchunkptr q;
+      /* each chunk claims to be free */
+      do_check_free_chunk(m, p);
+      /* chunk belongs in bin */
+      assert(small_index(size) == i);
+      assert(p->bk == b || chunksize(p->bk) == chunksize(p));
+      /* chunk is followed by an inuse chunk */
+      q = next_chunk(p);
+      if (q->head != FENCEPOST_HEAD)
+        do_check_inuse_chunk(m, q);
+    }
+  }
+}
+
+/* Find x in a bin. Used in other check functions. */
+static int bin_find(mstate m, mchunkptr x) {
+  size_t size = chunksize(x);
+  if (is_small(size)) {
+    bindex_t sidx = small_index(size);
+    sbinptr b = smallbin_at(m, sidx);
+    if (smallmap_is_marked(m, sidx)) {
+      mchunkptr p = b;
+      do {
+        if (p == x)
+          return 1;
+      } while ((p = p->fd) != b);
+    }
+  }
+  else {
+    bindex_t tidx;
+    compute_tree_index(size, tidx);
+    if (treemap_is_marked(m, tidx)) {
+      tchunkptr t = *treebin_at(m, tidx);
+      size_t sizebits = size << leftshift_for_tree_index(tidx);
+      while (t != 0 && chunksize(t) != size) {
+        t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1];
+        sizebits <<= 1;
+      }
+      if (t != 0) {
+        tchunkptr u = t;
+        do {
+          if (u == (tchunkptr)x)
+            return 1;
+        } while ((u = u->fd) != t);
+      }
+    }
+  }
+  return 0;
+}
+
+/* Traverse each chunk and check it; return total */
+static size_t traverse_and_check(mstate m) {
+  size_t sum = 0;
+  if (is_initialized(m)) {
+    msegmentptr s = &m->seg;
+    sum += m->topsize + TOP_FOOT_SIZE;
+    while (s != 0) {
+      mchunkptr q = align_as_chunk(s->base);
+      mchunkptr lastq = 0;
+      assert(pinuse(q));
+      while (segment_holds(s, q) &&
+             q != m->top && q->head != FENCEPOST_HEAD) {
+        sum += chunksize(q);
+        if (is_inuse(q)) {
+          assert(!bin_find(m, q));
+          do_check_inuse_chunk(m, q);
+        }
+        else {
+          assert(q == m->dv || bin_find(m, q));
+          assert(lastq == 0 || is_inuse(lastq)); /* Not 2 consecutive free */
+          do_check_free_chunk(m, q);
+        }
+        lastq = q;
+        q = next_chunk(q);
+      }
+      s = s->next;
+    }
+  }
+  return sum;
+}
+
+
+/* Check all properties of malloc_state. */
+static void do_check_malloc_state(mstate m) {
+  bindex_t i;
+  size_t total;
+  /* check bins */
+  for (i = 0; i < NSMALLBINS; ++i)
+    do_check_smallbin(m, i);
+  for (i = 0; i < NTREEBINS; ++i)
+    do_check_treebin(m, i);
+
+  if (m->dvsize != 0) { /* check dv chunk */
+    do_check_any_chunk(m, m->dv);
+    assert(m->dvsize == chunksize(m->dv));
+    assert(m->dvsize >= MIN_CHUNK_SIZE);
+    assert(bin_find(m, m->dv) == 0);
+  }
+
+  if (m->top != 0) {   /* check top chunk */
+    do_check_top_chunk(m, m->top);
+    /*assert(m->topsize == chunksize(m->top)); redundant */
+    assert(m->topsize > 0);
+    assert(bin_find(m, m->top) == 0);
+  }
+
+  total = traverse_and_check(m);
+  assert(total <= m->footprint);
+  assert(m->footprint <= m->max_footprint);
+}
+#endif /* DEBUG */
+
+/* ----------------------------- statistics ------------------------------ */
+
+#if !NO_MALLINFO
+static struct mallinfo internal_mallinfo(mstate m) {
+  struct mallinfo nm = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+  ensure_initialization();
+  if (!PREACTION(m)) {
+    check_malloc_state(m);
+    if (is_initialized(m)) {
+      size_t nfree = SIZE_T_ONE; /* top always free */
+      size_t mfree = m->topsize + TOP_FOOT_SIZE;
+      size_t sum = mfree;
+      msegmentptr s = &m->seg;
+      while (s != 0) {
+        mchunkptr q = align_as_chunk(s->base);
+        while (segment_holds(s, q) &&
+               q != m->top && q->head != FENCEPOST_HEAD) {
+          size_t sz = chunksize(q);
+          sum += sz;
+          if (!is_inuse(q)) {
+            mfree += sz;
+            ++nfree;
+          }
+          q = next_chunk(q);
+        }
+        s = s->next;
+      }
+
+      nm.arena    = sum;
+      nm.ordblks  = nfree;
+      nm.hblkhd   = m->footprint - sum;
+      nm.usmblks  = m->max_footprint;
+      nm.uordblks = m->footprint - mfree;
+      nm.fordblks = mfree;
+      nm.keepcost = m->topsize;
+    }
+
+    POSTACTION(m);
+  }
+  return nm;
+}
+#endif /* !NO_MALLINFO */
+
+#if !NO_MALLOC_STATS
+static void internal_malloc_stats(mstate m) {
+  ensure_initialization();
+  if (!PREACTION(m)) {
+    size_t maxfp = 0;
+    size_t fp = 0;
+    size_t used = 0;
+    check_malloc_state(m);
+    if (is_initialized(m)) {
+      msegmentptr s = &m->seg;
+      maxfp = m->max_footprint;
+      fp = m->footprint;
+      used = fp - (m->topsize + TOP_FOOT_SIZE);
+
+      while (s != 0) {
+        mchunkptr q = align_as_chunk(s->base);
+        while (segment_holds(s, q) &&
+               q != m->top && q->head != FENCEPOST_HEAD) {
+          if (!is_inuse(q))
+            used -= chunksize(q);
+          q = next_chunk(q);
+        }
+        s = s->next;
+      }
+    }
+    POSTACTION(m); /* drop lock */
+    fprintf(stderr, "max system bytes = %10lu\n", (unsigned long)(maxfp));
+    fprintf(stderr, "system bytes     = %10lu\n", (unsigned long)(fp));
+    fprintf(stderr, "in use bytes     = %10lu\n", (unsigned long)(used));
+  }
+}
+#endif /* NO_MALLOC_STATS */
+
+/* ----------------------- Operations on smallbins ----------------------- */
+
+/*
+  Various forms of linking and unlinking are defined as macros.  Even
+  the ones for trees, which are very long but have very short typical
+  paths.  This is ugly but reduces reliance on inlining support of
+  compilers.
+*/
+
+/* Link a free chunk into a smallbin  */
+#define insert_small_chunk(M, P, S) {\
+  bindex_t I  = small_index(S);\
+  mchunkptr B = smallbin_at(M, I);\
+  mchunkptr F = B;\
+  assert(S >= MIN_CHUNK_SIZE);\
+  if (!smallmap_is_marked(M, I))\
+    mark_smallmap(M, I);\
+  else if (RTCHECK(ok_address(M, B->fd)))\
+    F = B->fd;\
+  else {\
+    CORRUPTION_ERROR_ACTION(M);\
+  }\
+  B->fd = P;\
+  F->bk = P;\
+  P->fd = F;\
+  P->bk = B;\
+}
+
+/* Unlink a chunk from a smallbin  */
+#define unlink_small_chunk(M, P, S) {\
+  mchunkptr F = P->fd;\
+  mchunkptr B = P->bk;\
+  bindex_t I = small_index(S);\
+  assert(P != B);\
+  assert(P != F);\
+  assert(chunksize(P) == small_index2size(I));\
+  if (RTCHECK(F == smallbin_at(M,I) || (ok_address(M, F) && F->bk == P))) { \
+    if (B == F) {\
+      clear_smallmap(M, I);\
+    }\
+    else if (RTCHECK(B == smallbin_at(M,I) ||\
+                     (ok_address(M, B) && B->fd == P))) {\
+      F->bk = B;\
+      B->fd = F;\
+    }\
+    else {\
+      CORRUPTION_ERROR_ACTION(M);\
+    }\
+  }\
+  else {\
+    CORRUPTION_ERROR_ACTION(M);\
+  }\
+}
+
+/* Unlink the first chunk from a smallbin */
+#define unlink_first_small_chunk(M, B, P, I) {\
+  mchunkptr F = P->fd;\
+  assert(P != B);\
+  assert(P != F);\
+  assert(chunksize(P) == small_index2size(I));\
+  if (B == F) {\
+    clear_smallmap(M, I);\
+  }\
+  else if (RTCHECK(ok_address(M, F) && F->bk == P)) {\
+    F->bk = B;\
+    B->fd = F;\
+  }\
+  else {\
+    CORRUPTION_ERROR_ACTION(M);\
+  }\
+}
+
+/* Replace dv node, binning the old one */
+/* Used only when dvsize known to be small */
+#define replace_dv(M, P, S) {\
+  size_t DVS = M->dvsize;\
+  assert(is_small(DVS));\
+  if (DVS != 0) {\
+    mchunkptr DV = M->dv;\
+    insert_small_chunk(M, DV, DVS);\
+  }\
+  M->dvsize = S;\
+  M->dv = P;\
+}
+
+/* ------------------------- Operations on trees ------------------------- */
+
+/* Insert chunk into tree */
+#define insert_large_chunk(M, X, S) {\
+  tbinptr* H;\
+  bindex_t I;\
+  compute_tree_index(S, I);\
+  H = treebin_at(M, I);\
+  X->index = I;\
+  X->child[0] = X->child[1] = 0;\
+  if (!treemap_is_marked(M, I)) {\
+    mark_treemap(M, I);\
+    *H = X;\
+    X->parent = (tchunkptr)H;\
+    X->fd = X->bk = X;\
+  }\
+  else {\
+    tchunkptr T = *H;\
+    size_t K = S << leftshift_for_tree_index(I);\
+    for (;;) {\
+      if (chunksize(T) != S) {\
+        tchunkptr* C = &(T->child[(K >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]);\
+        K <<= 1;\
+        if (*C != 0)\
+          T = *C;\
+        else if (RTCHECK(ok_address(M, C))) {\
+          *C = X;\
+          X->parent = T;\
+          X->fd = X->bk = X;\
+          break;\
+        }\
+        else {\
+          CORRUPTION_ERROR_ACTION(M);\
+          break;\
+        }\
+      }\
+      else {\
+        tchunkptr F = T->fd;\
+        if (RTCHECK(ok_address(M, T) && ok_address(M, F))) {\
+          T->fd = F->bk = X;\
+          X->fd = F;\
+          X->bk = T;\
+          X->parent = 0;\
+          break;\
+        }\
+        else {\
+          CORRUPTION_ERROR_ACTION(M);\
+          break;\
+        }\
+      }\
+    }\
+  }\
+}
+
+/*
+  Unlink steps:
+
+  1. If x is a chained node, unlink it from its same-sized fd/bk links
+     and choose its bk node as its replacement.
+  2. If x was the last node of its size, but not a leaf node, it must
+     be replaced with a leaf node (not merely one with an open left or
+     right), to make sure that lefts and rights of descendents
+     correspond properly to bit masks.  We use the rightmost descendent
+     of x.  We could use any other leaf, but this is easy to locate and
+     tends to counteract removal of leftmosts elsewhere, and so keeps
+     paths shorter than minimally guaranteed.  This doesn't loop much
+     because on average a node in a tree is near the bottom.
+  3. If x is the base of a chain (i.e., has parent links) relink
+     x's parent and children to x's replacement (or null if none).
+*/
+
+#define unlink_large_chunk(M, X) {\
+  tchunkptr XP = X->parent;\
+  tchunkptr R;\
+  if (X->bk != X) {\
+    tchunkptr F = X->fd;\
+    R = X->bk;\
+    if (RTCHECK(ok_address(M, F) && F->bk == X && R->fd == X)) {\
+      F->bk = R;\
+      R->fd = F;\
+    }\
+    else {\
+      CORRUPTION_ERROR_ACTION(M);\
+    }\
+  }\
+  else {\
+    tchunkptr* RP;\
+    if (((R = *(RP = &(X->child[1]))) != 0) ||\
+        ((R = *(RP = &(X->child[0]))) != 0)) {\
+      tchunkptr* CP;\
+      while ((*(CP = &(R->child[1])) != 0) ||\
+             (*(CP = &(R->child[0])) != 0)) {\
+        R = *(RP = CP);\
+      }\
+      if (RTCHECK(ok_address(M, RP)))\
+        *RP = 0;\
+      else {\
+        CORRUPTION_ERROR_ACTION(M);\
+      }\
+    }\
+  }\
+  if (XP != 0) {\
+    tbinptr* H = treebin_at(M, X->index);\
+    if (X == *H) {\
+      if ((*H = R) == 0) \
+        clear_treemap(M, X->index);\
+    }\
+    else if (RTCHECK(ok_address(M, XP))) {\
+      if (XP->child[0] == X) \
+        XP->child[0] = R;\
+      else \
+        XP->child[1] = R;\
+    }\
+    else\
+      CORRUPTION_ERROR_ACTION(M);\
+    if (R != 0) {\
+      if (RTCHECK(ok_address(M, R))) {\
+        tchunkptr C0, C1;\
+        R->parent = XP;\
+        if ((C0 = X->child[0]) != 0) {\
+          if (RTCHECK(ok_address(M, C0))) {\
+            R->child[0] = C0;\
+            C0->parent = R;\
+          }\
+          else\
+            CORRUPTION_ERROR_ACTION(M);\
+        }\
+        if ((C1 = X->child[1]) != 0) {\
+          if (RTCHECK(ok_address(M, C1))) {\
+            R->child[1] = C1;\
+            C1->parent = R;\
+          }\
+          else\
+            CORRUPTION_ERROR_ACTION(M);\
+        }\
+      }\
+      else\
+        CORRUPTION_ERROR_ACTION(M);\
+    }\
+  }\
+}
+
+/* Relays to large vs small bin operations */
+
+#define insert_chunk(M, P, S)\
+  if (is_small(S)) insert_small_chunk(M, P, S)\
+  else { tchunkptr TP = (tchunkptr)(P); insert_large_chunk(M, TP, S); }
+
+#define unlink_chunk(M, P, S)\
+  if (is_small(S)) unlink_small_chunk(M, P, S)\
+  else { tchunkptr TP = (tchunkptr)(P); unlink_large_chunk(M, TP); }
+
+
+/* Relays to internal calls to malloc/free from realloc, memalign etc */
+
+#if ONLY_MSPACES
+#define internal_malloc(m, b) mspace_malloc(m, b)
+#define internal_free(m, mem) mspace_free(m,mem);
+#else /* ONLY_MSPACES */
+#if MSPACES
+#define internal_malloc(m, b)\
+  ((m == gm)? dlmalloc(b) : mspace_malloc(m, b))
+#define internal_free(m, mem)\
+   if (m == gm) dlfree(mem); else mspace_free(m,mem);
+#else /* MSPACES */
+#define internal_malloc(m, b) dlmalloc(b)
+#define internal_free(m, mem) dlfree(mem)
+#endif /* MSPACES */
+#endif /* ONLY_MSPACES */
+
+/* -----------------------  Direct-mmapping chunks ----------------------- */
+
+/*
+  Directly mmapped chunks are set up with an offset to the start of
+  the mmapped region stored in the prev_foot field of the chunk. This
+  allows reconstruction of the required argument to MUNMAP when freed,
+  and also allows adjustment of the returned chunk to meet alignment
+  requirements (especially in memalign).
+*/
+
+/* Malloc using mmap */
+static void* mmap_alloc(mstate m, size_t nb) {
+  size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
+  if (m->footprint_limit != 0) {
+    size_t fp = m->footprint + mmsize;
+    if (fp <= m->footprint || fp > m->footprint_limit)
+      return 0;
+  }
+  if (mmsize > nb) {     /* Check for wrap around 0 */
+    char* mm = (char*)(CALL_DIRECT_MMAP(mmsize));
+    if (mm != CMFAIL) {
+      size_t offset = align_offset(chunk2mem(mm));
+      size_t psize = mmsize - offset - MMAP_FOOT_PAD;
+      mchunkptr p = (mchunkptr)(mm + offset);
+      p->prev_foot = offset;
+      p->head = psize;
+      mark_inuse_foot(m, p, psize);
+      chunk_plus_offset(p, psize)->head = FENCEPOST_HEAD;
+      chunk_plus_offset(p, psize+SIZE_T_SIZE)->head = 0;
+
+      if (m->least_addr == 0 || mm < m->least_addr)
+        m->least_addr = mm;
+      if ((m->footprint += mmsize) > m->max_footprint)
+        m->max_footprint = m->footprint;
+      assert(is_aligned(chunk2mem(p)));
+      check_mmapped_chunk(m, p);
+      return chunk2mem(p);
+    }
+  }
+  return 0;
+}
+
+/* Realloc using mmap */
+static mchunkptr mmap_resize(mstate m, mchunkptr oldp, size_t nb, int flags) {
+  size_t oldsize = chunksize(oldp);
+  (void)flags; /* placate people compiling -Wunused */
+  if (is_small(nb)) /* Can't shrink mmap regions below small size */
+    return 0;
+  /* Keep old chunk if big enough but not too big */
+  if (oldsize >= nb + SIZE_T_SIZE &&
+      (oldsize - nb) <= (mparams.granularity << 1))
+    return oldp;
+  else {
+    size_t offset = oldp->prev_foot;
+    size_t oldmmsize = oldsize + offset + MMAP_FOOT_PAD;
+    size_t newmmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
+    char* cp = (char*)CALL_MREMAP((char*)oldp - offset,
+                                  oldmmsize, newmmsize, flags);
+    if (cp != CMFAIL) {
+      mchunkptr newp = (mchunkptr)(cp + offset);
+      size_t psize = newmmsize - offset - MMAP_FOOT_PAD;
+      newp->head = psize;
+      mark_inuse_foot(m, newp, psize);
+      chunk_plus_offset(newp, psize)->head = FENCEPOST_HEAD;
+      chunk_plus_offset(newp, psize+SIZE_T_SIZE)->head = 0;
+
+      if (cp < m->least_addr)
+        m->least_addr = cp;
+      if ((m->footprint += newmmsize - oldmmsize) > m->max_footprint)
+        m->max_footprint = m->footprint;
+      check_mmapped_chunk(m, newp);
+      return newp;
+    }
+  }
+  return 0;
+}
+
+
+/* -------------------------- mspace management -------------------------- */
+
+/* Initialize top chunk and its size */
+static void init_top(mstate m, mchunkptr p, size_t psize) {
+  /* Ensure alignment */
+  size_t offset = align_offset(chunk2mem(p));
+  p = (mchunkptr)((char*)p + offset);
+  psize -= offset;
+
+  m->top = p;
+  m->topsize = psize;
+  p->head = psize | PINUSE_BIT;
+  /* set size of fake trailing chunk holding overhead space only once */
+  chunk_plus_offset(p, psize)->head = TOP_FOOT_SIZE;
+  m->trim_check = mparams.trim_threshold; /* reset on each update */
+}
+
+/* Initialize bins for a new mstate that is otherwise zeroed out */
+static void init_bins(mstate m) {
+  /* Establish circular links for smallbins */
+  bindex_t i;
+  for (i = 0; i < NSMALLBINS; ++i) {
+    sbinptr bin = smallbin_at(m,i);
+    bin->fd = bin->bk = bin;
+  }
+}
+
+#if PROCEED_ON_ERROR
+
+/* default corruption action */
+static void reset_on_error(mstate m) {
+  int i;
+  ++malloc_corruption_error_count;
+  /* Reinitialize fields to forget about all memory */
+  m->smallmap = m->treemap = 0;
+  m->dvsize = m->topsize = 0;
+  m->seg.base = 0;
+  m->seg.size = 0;
+  m->seg.next = 0;
+  m->top = m->dv = 0;
+  for (i = 0; i < NTREEBINS; ++i)
+    *treebin_at(m, i) = 0;
+  init_bins(m);
+}
+#endif /* PROCEED_ON_ERROR */
+
+/* Allocate chunk and prepend remainder with chunk in successor base. */
+static void* prepend_alloc(mstate m, char* newbase, char* oldbase,
+                           size_t nb) {
+  mchunkptr p = align_as_chunk(newbase);
+  mchunkptr oldfirst = align_as_chunk(oldbase);
+  size_t psize = (char*)oldfirst - (char*)p;
+  mchunkptr q = chunk_plus_offset(p, nb);
+  size_t qsize = psize - nb;
+  set_size_and_pinuse_of_inuse_chunk(m, p, nb);
+
+  assert((char*)oldfirst > (char*)q);
+  assert(pinuse(oldfirst));
+  assert(qsize >= MIN_CHUNK_SIZE);
+
+  /* consolidate remainder with first chunk of old base */
+  if (oldfirst == m->top) {
+    size_t tsize = m->topsize += qsize;
+    m->top = q;
+    q->head = tsize | PINUSE_BIT;
+    check_top_chunk(m, q);
+  }
+  else if (oldfirst == m->dv) {
+    size_t dsize = m->dvsize += qsize;
+    m->dv = q;
+    set_size_and_pinuse_of_free_chunk(q, dsize);
+  }
+  else {
+    if (!is_inuse(oldfirst)) {
+      size_t nsize = chunksize(oldfirst);
+      unlink_chunk(m, oldfirst, nsize);
+      oldfirst = chunk_plus_offset(oldfirst, nsize);
+      qsize += nsize;
+    }
+    set_free_with_pinuse(q, qsize, oldfirst);
+    insert_chunk(m, q, qsize);
+    check_free_chunk(m, q);
+  }
+
+  check_malloced_chunk(m, chunk2mem(p), nb);
+  return chunk2mem(p);
+}
+
+/* Add a segment to hold a new noncontiguous region */
+static void add_segment(mstate m, char* tbase, size_t tsize, flag_t mmapped) {
+  /* Determine locations and sizes of segment, fenceposts, old top */
+  char* old_top = (char*)m->top;
+  msegmentptr oldsp = segment_holding(m, old_top);
+  char* old_end = oldsp->base + oldsp->size;
+  size_t ssize = pad_request(sizeof(struct malloc_segment));
+  char* rawsp = old_end - (ssize + FOUR_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
+  size_t offset = align_offset(chunk2mem(rawsp));
+  char* asp = rawsp + offset;
+  char* csp = (asp < (old_top + MIN_CHUNK_SIZE))? old_top : asp;
+  mchunkptr sp = (mchunkptr)csp;
+  msegmentptr ss = (msegmentptr)(chunk2mem(sp));
+  mchunkptr tnext = chunk_plus_offset(sp, ssize);
+  mchunkptr p = tnext;
+  int nfences = 0;
+
+  /* reset top to new space */
+  init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE);
+
+  /* Set up segment record */
+  assert(is_aligned(ss));
+  set_size_and_pinuse_of_inuse_chunk(m, sp, ssize);
+  *ss = m->seg; /* Push current record */
+  m->seg.base = tbase;
+  m->seg.size = tsize;
+  m->seg.sflags = mmapped;
+  m->seg.next = ss;
+
+  /* Insert trailing fenceposts */
+  for (;;) {
+    mchunkptr nextp = chunk_plus_offset(p, SIZE_T_SIZE);
+    p->head = FENCEPOST_HEAD;
+    ++nfences;
+    if ((char*)(&(nextp->head)) < old_end)
+      p = nextp;
+    else
+      break;
+  }
+  assert(nfences >= 2);
+
+  /* Insert the rest of old top into a bin as an ordinary free chunk */
+  if (csp != old_top) {
+    mchunkptr q = (mchunkptr)old_top;
+    size_t psize = csp - old_top;
+    mchunkptr tn = chunk_plus_offset(q, psize);
+    set_free_with_pinuse(q, psize, tn);
+    insert_chunk(m, q, psize);
+  }
+
+  check_top_chunk(m, m->top);
+}
+
+/* -------------------------- System allocation -------------------------- */
+
+/* Get memory from system using MORECORE or MMAP */
+static void* sys_alloc(mstate m, size_t nb) {
+  char* tbase = CMFAIL;
+  size_t tsize = 0;
+  flag_t mmap_flag = 0;
+  size_t asize; /* allocation size */
+
+  ensure_initialization();
+
+  /* Directly map large chunks, but only if already initialized */
+  if (use_mmap(m) && nb >= mparams.mmap_threshold && m->topsize != 0) {
+    void* mem = mmap_alloc(m, nb);
+    if (mem != 0)
+      return mem;
+  }
+
+  asize = granularity_align(nb + SYS_ALLOC_PADDING);
+  if (asize <= nb)
+    return 0; /* wraparound */
+  if (m->footprint_limit != 0) {
+    size_t fp = m->footprint + asize;
+    if (fp <= m->footprint || fp > m->footprint_limit)
+      return 0;
+  }
+
+  /*
+    Try getting memory in any of three ways (in most-preferred to
+    least-preferred order):
+    1. A call to MORECORE that can normally contiguously extend memory.
+       (disabled if not MORECORE_CONTIGUOUS or not HAVE_MORECORE or
+       or main space is mmapped or a previous contiguous call failed)
+    2. A call to MMAP new space (disabled if not HAVE_MMAP).
+       Note that under the default settings, if MORECORE is unable to
+       fulfill a request, and HAVE_MMAP is true, then mmap is
+       used as a noncontiguous system allocator. This is a useful backup
+       strategy for systems with holes in address spaces -- in this case
+       sbrk cannot contiguously expand the heap, but mmap may be able to
+       find space.
+    3. A call to MORECORE that cannot usually contiguously extend memory.
+       (disabled if not HAVE_MORECORE)
+
+   In all cases, we need to request enough bytes from system to ensure
+   we can malloc nb bytes upon success, so pad with enough space for
+   top_foot, plus alignment-pad to make sure we don't lose bytes if
+   not on boundary, and round this up to a granularity unit.
+  */
+
+  if (MORECORE_CONTIGUOUS && !use_noncontiguous(m)) {
+    char* br = CMFAIL;
+    size_t ssize = asize; /* sbrk call size */
+    msegmentptr ss = (m->top == 0)? 0 : segment_holding(m, (char*)m->top);
+    ACQUIRE_MALLOC_GLOBAL_LOCK();
+
+    if (ss == 0) {  /* First time through or recovery */
+      char* base = (char*)CALL_MORECORE(0);
+      if (base != CMFAIL) {
+        size_t fp;
+        /* Adjust to end on a page boundary */
+        if (!is_page_aligned(base))
+          ssize += (page_align((size_t)base) - (size_t)base);
+        fp = m->footprint + ssize; /* recheck limits */
+        if (ssize > nb && ssize < HALF_MAX_SIZE_T &&
+            (m->footprint_limit == 0 ||
+             (fp > m->footprint && fp <= m->footprint_limit)) &&
+            (br = (char*)(CALL_MORECORE(ssize))) == base) {
+          tbase = base;
+          tsize = ssize;
+        }
+      }
+    }
+    else {
+      /* Subtract out existing available top space from MORECORE request. */
+      ssize = granularity_align(nb - m->topsize + SYS_ALLOC_PADDING);
+      /* Use mem here only if it did continuously extend old space */
+      if (ssize < HALF_MAX_SIZE_T &&
+          (br = (char*)(CALL_MORECORE(ssize))) == ss->base+ss->size) {
+        tbase = br;
+        tsize = ssize;
+      }
+    }
+
+    if (tbase == CMFAIL) {    /* Cope with partial failure */
+      if (br != CMFAIL) {    /* Try to use/extend the space we did get */
+        if (ssize < HALF_MAX_SIZE_T &&
+            ssize < nb + SYS_ALLOC_PADDING) {
+          size_t esize = granularity_align(nb + SYS_ALLOC_PADDING - ssize);
+          if (esize < HALF_MAX_SIZE_T) {
+            char* end = (char*)CALL_MORECORE(esize);
+            if (end != CMFAIL)
+              ssize += esize;
+            else {            /* Can't use; try to release */
+              (void) CALL_MORECORE(-ssize);
+              br = CMFAIL;
+            }
+          }
+        }
+      }
+      if (br != CMFAIL) {    /* Use the space we did get */
+        tbase = br;
+        tsize = ssize;
+      }
+      else
+        disable_contiguous(m); /* Don't try contiguous path in the future */
+    }
+
+    RELEASE_MALLOC_GLOBAL_LOCK();
+  }
+
+  if (HAVE_MMAP && tbase == CMFAIL) {  /* Try MMAP */
+    char* mp = (char*)(CALL_MMAP(asize));
+    if (mp != CMFAIL) {
+      tbase = mp;
+      tsize = asize;
+      mmap_flag = USE_MMAP_BIT;
+    }
+  }
+
+  if (HAVE_MORECORE && tbase == CMFAIL) { /* Try noncontiguous MORECORE */
+    if (asize < HALF_MAX_SIZE_T) {
+      char* br = CMFAIL;
+      char* end = CMFAIL;
+      ACQUIRE_MALLOC_GLOBAL_LOCK();
+      br = (char*)(CALL_MORECORE(asize));
+      end = (char*)(CALL_MORECORE(0));
+      RELEASE_MALLOC_GLOBAL_LOCK();
+      if (br != CMFAIL && end != CMFAIL && br < end) {
+        size_t ssize = end - br;
+        if (ssize > nb + TOP_FOOT_SIZE) {
+          tbase = br;
+          tsize = ssize;
+        }
+      }
+    }
+  }
+
+  if (tbase != CMFAIL) {
+
+    if ((m->footprint += tsize) > m->max_footprint)
+      m->max_footprint = m->footprint;
+
+    if (!is_initialized(m)) { /* first-time initialization */
+      if (m->least_addr == 0 || tbase < m->least_addr)
+        m->least_addr = tbase;
+      m->seg.base = tbase;
+      m->seg.size = tsize;
+      m->seg.sflags = mmap_flag;
+      m->magic = mparams.magic;
+      m->release_checks = MAX_RELEASE_CHECK_RATE;
+      init_bins(m);
+#if !ONLY_MSPACES
+      if (is_global(m))
+        init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE);
+      else
+#endif
+      {
+        /* Offset top by embedded malloc_state */
+        mchunkptr mn = next_chunk(mem2chunk(m));
+        init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) -TOP_FOOT_SIZE);
+      }
+    }
+
+    else {
+      /* Try to merge with an existing segment */
+      msegmentptr sp = &m->seg;
+      /* Only consider most recent segment if traversal suppressed */
+      while (sp != 0 && tbase != sp->base + sp->size)
+        sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next;
+      if (sp != 0 &&
+          !is_extern_segment(sp) &&
+          (sp->sflags & USE_MMAP_BIT) == mmap_flag &&
+          segment_holds(sp, m->top)) { /* append */
+        sp->size += tsize;
+        init_top(m, m->top, m->topsize + tsize);
+      }
+      else {
+        if (tbase < m->least_addr)
+          m->least_addr = tbase;
+        sp = &m->seg;
+        while (sp != 0 && sp->base != tbase + tsize)
+          sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next;
+        if (sp != 0 &&
+            !is_extern_segment(sp) &&
+            (sp->sflags & USE_MMAP_BIT) == mmap_flag) {
+          char* oldbase = sp->base;
+          sp->base = tbase;
+          sp->size += tsize;
+          return prepend_alloc(m, tbase, oldbase, nb);
+        }
+        else
+          add_segment(m, tbase, tsize, mmap_flag);
+      }
+    }
+
+    if (nb < m->topsize) { /* Allocate from new or extended top space */
+      size_t rsize = m->topsize -= nb;
+      mchunkptr p = m->top;
+      mchunkptr r = m->top = chunk_plus_offset(p, nb);
+      r->head = rsize | PINUSE_BIT;
+      set_size_and_pinuse_of_inuse_chunk(m, p, nb);
+      check_top_chunk(m, m->top);
+      check_malloced_chunk(m, chunk2mem(p), nb);
+      return chunk2mem(p);
+    }
+  }
+
+  MALLOC_FAILURE_ACTION;
+  return 0;
+}
+
+/* -----------------------  system deallocation -------------------------- */
+
+/* Unmap and unlink any mmapped segments that don't contain used chunks */
+static size_t release_unused_segments(mstate m) {
+  size_t released = 0;
+  int nsegs = 0;
+  msegmentptr pred = &m->seg;
+  msegmentptr sp = pred->next;
+  while (sp != 0) {
+    char* base = sp->base;
+    size_t size = sp->size;
+    msegmentptr next = sp->next;
+    ++nsegs;
+    if (is_mmapped_segment(sp) && !is_extern_segment(sp)) {
+      mchunkptr p = align_as_chunk(base);
+      size_t psize = chunksize(p);
+      /* Can unmap if first chunk holds entire segment and not pinned */
+      if (!is_inuse(p) && (char*)p + psize >= base + size - TOP_FOOT_SIZE) {
+        tchunkptr tp = (tchunkptr)p;
+        assert(segment_holds(sp, (char*)sp));
+        if (p == m->dv) {
+          m->dv = 0;
+          m->dvsize = 0;
+        }
+        else {
+          unlink_large_chunk(m, tp);
+        }
+        if (CALL_MUNMAP(base, size) == 0) {
+          released += size;
+          m->footprint -= size;
+          /* unlink obsoleted record */
+          sp = pred;
+          sp->next = next;
+        }
+        else { /* back out if cannot unmap */
+          insert_large_chunk(m, tp, psize);
+        }
+      }
+    }
+    if (NO_SEGMENT_TRAVERSAL) /* scan only first segment */
+      break;
+    pred = sp;
+    sp = next;
+  }
+  /* Reset check counter */
+  m->release_checks = (((size_t) nsegs > (size_t) MAX_RELEASE_CHECK_RATE)?
+                       (size_t) nsegs : (size_t) MAX_RELEASE_CHECK_RATE);
+  return released;
+}
+
+static int sys_trim(mstate m, size_t pad) {
+  size_t released = 0;
+  ensure_initialization();
+  if (pad < MAX_REQUEST && is_initialized(m)) {
+    pad += TOP_FOOT_SIZE; /* ensure enough room for segment overhead */
+
+    if (m->topsize > pad) {
+      /* Shrink top space in granularity-size units, keeping at least one */
+      size_t unit = mparams.granularity;
+      size_t extra = ((m->topsize - pad + (unit - SIZE_T_ONE)) / unit -
+                      SIZE_T_ONE) * unit;
+      msegmentptr sp = segment_holding(m, (char*)m->top);
+
+      if (!is_extern_segment(sp)) {
+        if (is_mmapped_segment(sp)) {
+          if (HAVE_MMAP &&
+              sp->size >= extra &&
+              !has_segment_link(m, sp)) { /* can't shrink if pinned */
+            size_t newsize = sp->size - extra;
+            (void)newsize; /* placate people compiling -Wunused-variable */
+            /* Prefer mremap, fall back to munmap */
+            if ((CALL_MREMAP(sp->base, sp->size, newsize, 0) != MFAIL) ||
+                (CALL_MUNMAP(sp->base + newsize, extra) == 0)) {
+              released = extra;
+            }
+          }
+        }
+        else if (HAVE_MORECORE) {
+          if (extra >= HALF_MAX_SIZE_T) /* Avoid wrapping negative */
+            extra = (HALF_MAX_SIZE_T) + SIZE_T_ONE - unit;
+          ACQUIRE_MALLOC_GLOBAL_LOCK();
+          {
+            /* Make sure end of memory is where we last set it. */
+            char* old_br = (char*)(CALL_MORECORE(0));
+            if (old_br == sp->base + sp->size) {
+              char* rel_br = (char*)(CALL_MORECORE(-extra));
+              char* new_br = (char*)(CALL_MORECORE(0));
+              if (rel_br != CMFAIL && new_br < old_br)
+                released = old_br - new_br;
+            }
+          }
+          RELEASE_MALLOC_GLOBAL_LOCK();
+        }
+      }
+
+      if (released != 0) {
+        sp->size -= released;
+        m->footprint -= released;
+        init_top(m, m->top, m->topsize - released);
+        check_top_chunk(m, m->top);
+      }
+    }
+
+    /* Unmap any unused mmapped segments */
+    if (HAVE_MMAP)
+      released += release_unused_segments(m);
+
+    /* On failure, disable autotrim to avoid repeated failed future calls */
+    if (released == 0 && m->topsize > m->trim_check)
+      m->trim_check = MAX_SIZE_T;
+  }
+
+  return (released != 0)? 1 : 0;
+}
+
+/* Consolidate and bin a chunk. Differs from exported versions
+   of free mainly in that the chunk need not be marked as inuse.
+*/
+static void dispose_chunk(mstate m, mchunkptr p, size_t psize) {
+  mchunkptr next = chunk_plus_offset(p, psize);
+  if (!pinuse(p)) {
+    mchunkptr prev;
+    size_t prevsize = p->prev_foot;
+    if (is_mmapped(p)) {
+      psize += prevsize + MMAP_FOOT_PAD;
+      if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
+        m->footprint -= psize;
+      return;
+    }
+    prev = chunk_minus_offset(p, prevsize);
+    psize += prevsize;
+    p = prev;
+    if (RTCHECK(ok_address(m, prev))) { /* consolidate backward */
+      if (p != m->dv) {
+        unlink_chunk(m, p, prevsize);
+      }
+      else if ((next->head & INUSE_BITS) == INUSE_BITS) {
+        m->dvsize = psize;
+        set_free_with_pinuse(p, psize, next);
+        return;
+      }
+    }
+    else {
+      CORRUPTION_ERROR_ACTION(m);
+      return;
+    }
+  }
+  if (RTCHECK(ok_address(m, next))) {
+    if (!cinuse(next)) {  /* consolidate forward */
+      if (next == m->top) {
+        size_t tsize = m->topsize += psize;
+        m->top = p;
+        p->head = tsize | PINUSE_BIT;
+        if (p == m->dv) {
+          m->dv = 0;
+          m->dvsize = 0;
+        }
+        return;
+      }
+      else if (next == m->dv) {
+        size_t dsize = m->dvsize += psize;
+        m->dv = p;
+        set_size_and_pinuse_of_free_chunk(p, dsize);
+        return;
+      }
+      else {
+        size_t nsize = chunksize(next);
+        psize += nsize;
+        unlink_chunk(m, next, nsize);
+        set_size_and_pinuse_of_free_chunk(p, psize);
+        if (p == m->dv) {
+          m->dvsize = psize;
+          return;
+        }
+      }
+    }
+    else {
+      set_free_with_pinuse(p, psize, next);
+    }
+    insert_chunk(m, p, psize);
+  }
+  else {
+    CORRUPTION_ERROR_ACTION(m);
+  }
+}
+
+/* ---------------------------- malloc --------------------------- */
+
+/* allocate a large request from the best fitting chunk in a treebin */
+static void* tmalloc_large(mstate m, size_t nb) {
+  tchunkptr v = 0;
+  size_t rsize = -nb; /* Unsigned negation */
+  tchunkptr t;
+  bindex_t idx;
+  compute_tree_index(nb, idx);
+  if ((t = *treebin_at(m, idx)) != 0) {
+    /* Traverse tree for this bin looking for node with size == nb */
+    size_t sizebits = nb << leftshift_for_tree_index(idx);
+    tchunkptr rst = 0;  /* The deepest untaken right subtree */
+    for (;;) {
+      tchunkptr rt;
+      size_t trem = chunksize(t) - nb;
+      if (trem < rsize) {
+        v = t;
+        if ((rsize = trem) == 0)
+          break;
+      }
+      rt = t->child[1];
+      t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1];
+      if (rt != 0 && rt != t)
+        rst = rt;
+      if (t == 0) {
+        t = rst; /* set t to least subtree holding sizes > nb */
+        break;
+      }
+      sizebits <<= 1;
+    }
+  }
+  if (t == 0 && v == 0) { /* set t to root of next non-empty treebin */
+    binmap_t leftbits = left_bits(idx2bit(idx)) & m->treemap;
+    if (leftbits != 0) {
+      bindex_t i;
+      binmap_t leastbit = least_bit(leftbits);
+      compute_bit2idx(leastbit, i);
+      t = *treebin_at(m, i);
+    }
+  }
+
+  while (t != 0) { /* find smallest of tree or subtree */
+    size_t trem = chunksize(t) - nb;
+    if (trem < rsize) {
+      rsize = trem;
+      v = t;
+    }
+    t = leftmost_child(t);
+  }
+
+  /*  If dv is a better fit, return 0 so malloc will use it */
+  if (v != 0 && rsize < (size_t)(m->dvsize - nb)) {
+    if (RTCHECK(ok_address(m, v))) { /* split */
+      mchunkptr r = chunk_plus_offset(v, nb);
+      assert(chunksize(v) == rsize + nb);
+      if (RTCHECK(ok_next(v, r))) {
+        unlink_large_chunk(m, v);
+        if (rsize < MIN_CHUNK_SIZE)
+          set_inuse_and_pinuse(m, v, (rsize + nb));
+        else {
+          set_size_and_pinuse_of_inuse_chunk(m, v, nb);
+          set_size_and_pinuse_of_free_chunk(r, rsize);
+          insert_chunk(m, r, rsize);
+        }
+        return chunk2mem(v);
+      }
+    }
+    CORRUPTION_ERROR_ACTION(m);
+  }
+  return 0;
+}
+
+/* allocate a small request from the best fitting chunk in a treebin */
+static void* tmalloc_small(mstate m, size_t nb) {
+  tchunkptr t, v;
+  size_t rsize;
+  bindex_t i;
+  binmap_t leastbit = least_bit(m->treemap);
+  compute_bit2idx(leastbit, i);
+  v = t = *treebin_at(m, i);
+  rsize = chunksize(t) - nb;
+
+  while ((t = leftmost_child(t)) != 0) {
+    size_t trem = chunksize(t) - nb;
+    if (trem < rsize) {
+      rsize = trem;
+      v = t;
+    }
+  }
+
+  if (RTCHECK(ok_address(m, v))) {
+    mchunkptr r = chunk_plus_offset(v, nb);
+    assert(chunksize(v) == rsize + nb);
+    if (RTCHECK(ok_next(v, r))) {
+      unlink_large_chunk(m, v);
+      if (rsize < MIN_CHUNK_SIZE)
+        set_inuse_and_pinuse(m, v, (rsize + nb));
+      else {
+        set_size_and_pinuse_of_inuse_chunk(m, v, nb);
+        set_size_and_pinuse_of_free_chunk(r, rsize);
+        replace_dv(m, r, rsize);
+      }
+      return chunk2mem(v);
+    }
+  }
+
+  CORRUPTION_ERROR_ACTION(m);
+  return 0;
+}
+
+#if !ONLY_MSPACES
+
+void* dlmalloc(size_t bytes) {
+  /*
+     Basic algorithm:
+     If a small request (< 256 bytes minus per-chunk overhead):
+       1. If one exists, use a remainderless chunk in associated smallbin.
+          (Remainderless means that there are too few excess bytes to
+          represent as a chunk.)
+       2. If it is big enough, use the dv chunk, which is normally the
+          chunk adjacent to the one used for the most recent small request.
+       3. If one exists, split the smallest available chunk in a bin,
+          saving remainder in dv.
+       4. If it is big enough, use the top chunk.
+       5. If available, get memory from system and use it
+     Otherwise, for a large request:
+       1. Find the smallest available binned chunk that fits, and use it
+          if it is better fitting than dv chunk, splitting if necessary.
+       2. If better fitting than any binned chunk, use the dv chunk.
+       3. If it is big enough, use the top chunk.
+       4. If request size >= mmap threshold, try to directly mmap this chunk.
+       5. If available, get memory from system and use it
+
+     The ugly goto's here ensure that postaction occurs along all paths.
+  */
+
+#if USE_LOCKS
+  ensure_initialization(); /* initialize in sys_alloc if not using locks */
+#endif
+
+  if (!PREACTION(gm)) {
+    void* mem;
+    size_t nb;
+    if (bytes <= MAX_SMALL_REQUEST) {
+      bindex_t idx;
+      binmap_t smallbits;
+      nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes);
+      idx = small_index(nb);
+      smallbits = gm->smallmap >> idx;
+
+      if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */
+        mchunkptr b, p;
+        idx += ~smallbits & 1;       /* Uses next bin if idx empty */
+        b = smallbin_at(gm, idx);
+        p = b->fd;
+        assert(chunksize(p) == small_index2size(idx));
+        unlink_first_small_chunk(gm, b, p, idx);
+        set_inuse_and_pinuse(gm, p, small_index2size(idx));
+        mem = chunk2mem(p);
+        check_malloced_chunk(gm, mem, nb);
+        goto postaction;
+      }
+
+      else if (nb > gm->dvsize) {
+        if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
+          mchunkptr b, p, r;
+          size_t rsize;
+          bindex_t i;
+          binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx));
+          binmap_t leastbit = least_bit(leftbits);
+          compute_bit2idx(leastbit, i);
+          b = smallbin_at(gm, i);
+          p = b->fd;
+          assert(chunksize(p) == small_index2size(i));
+          unlink_first_small_chunk(gm, b, p, i);
+          rsize = small_index2size(i) - nb;
+          /* Fit here cannot be remainderless if 4byte sizes */
+          if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE)
+            set_inuse_and_pinuse(gm, p, small_index2size(i));
+          else {
+            set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
+            r = chunk_plus_offset(p, nb);
+            set_size_and_pinuse_of_free_chunk(r, rsize);
+            replace_dv(gm, r, rsize);
+          }
+          mem = chunk2mem(p);
+          check_malloced_chunk(gm, mem, nb);
+          goto postaction;
+        }
+
+        else if (gm->treemap != 0 && (mem = tmalloc_small(gm, nb)) != 0) {
+          check_malloced_chunk(gm, mem, nb);
+          goto postaction;
+        }
+      }
+    }
+    else if (bytes >= MAX_REQUEST)
+      nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */
+    else {
+      nb = pad_request(bytes);
+      if (gm->treemap != 0 && (mem = tmalloc_large(gm, nb)) != 0) {
+        check_malloced_chunk(gm, mem, nb);
+        goto postaction;
+      }
+    }
+
+    if (nb <= gm->dvsize) {
+      size_t rsize = gm->dvsize - nb;
+      mchunkptr p = gm->dv;
+      if (rsize >= MIN_CHUNK_SIZE) { /* split dv */
+        mchunkptr r = gm->dv = chunk_plus_offset(p, nb);
+        gm->dvsize = rsize;
+        set_size_and_pinuse_of_free_chunk(r, rsize);
+        set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
+      }
+      else { /* exhaust dv */
+        size_t dvs = gm->dvsize;
+        gm->dvsize = 0;
+        gm->dv = 0;
+        set_inuse_and_pinuse(gm, p, dvs);
+      }
+      mem = chunk2mem(p);
+      check_malloced_chunk(gm, mem, nb);
+      goto postaction;
+    }
+
+    else if (nb < gm->topsize) { /* Split top */
+      size_t rsize = gm->topsize -= nb;
+      mchunkptr p = gm->top;
+      mchunkptr r = gm->top = chunk_plus_offset(p, nb);
+      r->head = rsize | PINUSE_BIT;
+      set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
+      mem = chunk2mem(p);
+      check_top_chunk(gm, gm->top);
+      check_malloced_chunk(gm, mem, nb);
+      goto postaction;
+    }
+
+    mem = sys_alloc(gm, nb);
+
+  postaction:
+    POSTACTION(gm);
+    return mem;
+  }
+
+  return 0;
+}
+
+/* ---------------------------- free --------------------------- */
+
+void dlfree(void* mem) {
+  /*
+     Consolidate freed chunks with preceeding or succeeding bordering
+     free chunks, if they exist, and then place in a bin.  Intermixed
+     with special cases for top, dv, mmapped chunks, and usage errors.
+  */
+
+  if (mem != 0) {
+    mchunkptr p  = mem2chunk(mem);
+#if FOOTERS
+    mstate fm = get_mstate_for(p);
+    if (!ok_magic(fm)) {
+      USAGE_ERROR_ACTION(fm, p);
+      return;
+    }
+#else /* FOOTERS */
+#define fm gm
+#endif /* FOOTERS */
+    if (!PREACTION(fm)) {
+      check_inuse_chunk(fm, p);
+      if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) {
+        size_t psize = chunksize(p);
+        mchunkptr next = chunk_plus_offset(p, psize);
+        if (!pinuse(p)) {
+          size_t prevsize = p->prev_foot;
+          if (is_mmapped(p)) {
+            psize += prevsize + MMAP_FOOT_PAD;
+            if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
+              fm->footprint -= psize;
+            goto postaction;
+          }
+          else {
+            mchunkptr prev = chunk_minus_offset(p, prevsize);
+            psize += prevsize;
+            p = prev;
+            if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */
+              if (p != fm->dv) {
+                unlink_chunk(fm, p, prevsize);
+              }
+              else if ((next->head & INUSE_BITS) == INUSE_BITS) {
+                fm->dvsize = psize;
+                set_free_with_pinuse(p, psize, next);
+                goto postaction;
+              }
+            }
+            else
+              goto erroraction;
+          }
+        }
+
+        if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) {
+          if (!cinuse(next)) {  /* consolidate forward */
+            if (next == fm->top) {
+              size_t tsize = fm->topsize += psize;
+              fm->top = p;
+              p->head = tsize | PINUSE_BIT;
+              if (p == fm->dv) {
+                fm->dv = 0;
+                fm->dvsize = 0;
+              }
+              if (should_trim(fm, tsize))
+                sys_trim(fm, 0);
+              goto postaction;
+            }
+            else if (next == fm->dv) {
+              size_t dsize = fm->dvsize += psize;
+              fm->dv = p;
+              set_size_and_pinuse_of_free_chunk(p, dsize);
+              goto postaction;
+            }
+            else {
+              size_t nsize = chunksize(next);
+              psize += nsize;
+              unlink_chunk(fm, next, nsize);
+              set_size_and_pinuse_of_free_chunk(p, psize);
+              if (p == fm->dv) {
+                fm->dvsize = psize;
+                goto postaction;
+              }
+            }
+          }
+          else
+            set_free_with_pinuse(p, psize, next);
+
+          if (is_small(psize)) {
+            insert_small_chunk(fm, p, psize);
+            check_free_chunk(fm, p);
+          }
+          else {
+            tchunkptr tp = (tchunkptr)p;
+            insert_large_chunk(fm, tp, psize);
+            check_free_chunk(fm, p);
+            if (--fm->release_checks == 0)
+              release_unused_segments(fm);
+          }
+          goto postaction;
+        }
+      }
+    erroraction:
+      USAGE_ERROR_ACTION(fm, p);
+    postaction:
+      POSTACTION(fm);
+    }
+  }
+#if !FOOTERS
+#undef fm
+#endif /* FOOTERS */
+}
+
+void* dlcalloc(size_t n_elements, size_t elem_size) {
+  void* mem;
+  size_t req = 0;
+  if (n_elements != 0) {
+    req = n_elements * elem_size;
+    if (((n_elements | elem_size) & ~(size_t)0xffff) &&
+        (req / n_elements != elem_size))
+      req = MAX_SIZE_T; /* force downstream failure on overflow */
+  }
+  mem = dlmalloc(req);
+  if (mem != 0 && calloc_must_clear(mem2chunk(mem)))
+    memset(mem, 0, req);
+  return mem;
+}
+
+#endif /* !ONLY_MSPACES */
+
+/* ------------ Internal support for realloc, memalign, etc -------------- */
+
+/* Try to realloc; only in-place unless can_move true */
+static mchunkptr try_realloc_chunk(mstate m, mchunkptr p, size_t nb,
+                                   int can_move) {
+  mchunkptr newp = 0;
+  size_t oldsize = chunksize(p);
+  mchunkptr next = chunk_plus_offset(p, oldsize);
+  if (RTCHECK(ok_address(m, p) && ok_inuse(p) &&
+              ok_next(p, next) && ok_pinuse(next))) {
+    if (is_mmapped(p)) {
+      newp = mmap_resize(m, p, nb, can_move);
+    }
+    else if (oldsize >= nb) {             /* already big enough */
+      size_t rsize = oldsize - nb;
+      if (rsize >= MIN_CHUNK_SIZE) {      /* split off remainder */
+        mchunkptr r = chunk_plus_offset(p, nb);
+        set_inuse(m, p, nb);
+        set_inuse(m, r, rsize);
+        dispose_chunk(m, r, rsize);
+      }
+      newp = p;
+    }
+    else if (next == m->top) {  /* extend into top */
+      if (oldsize + m->topsize > nb) {
+        size_t newsize = oldsize + m->topsize;
+        size_t newtopsize = newsize - nb;
+        mchunkptr newtop = chunk_plus_offset(p, nb);
+        set_inuse(m, p, nb);
+        newtop->head = newtopsize |PINUSE_BIT;
+        m->top = newtop;
+        m->topsize = newtopsize;
+        newp = p;
+      }
+    }
+    else if (next == m->dv) { /* extend into dv */
+      size_t dvs = m->dvsize;
+      if (oldsize + dvs >= nb) {
+        size_t dsize = oldsize + dvs - nb;
+        if (dsize >= MIN_CHUNK_SIZE) {
+          mchunkptr r = chunk_plus_offset(p, nb);
+          mchunkptr n = chunk_plus_offset(r, dsize);
+          set_inuse(m, p, nb);
+          set_size_and_pinuse_of_free_chunk(r, dsize);
+          clear_pinuse(n);
+          m->dvsize = dsize;
+          m->dv = r;
+        }
+        else { /* exhaust dv */
+          size_t newsize = oldsize + dvs;
+          set_inuse(m, p, newsize);
+          m->dvsize = 0;
+          m->dv = 0;
+        }
+        newp = p;
+      }
+    }
+    else if (!cinuse(next)) { /* extend into next free chunk */
+      size_t nextsize = chunksize(next);
+      if (oldsize + nextsize >= nb) {
+        size_t rsize = oldsize + nextsize - nb;
+        unlink_chunk(m, next, nextsize);
+        if (rsize < MIN_CHUNK_SIZE) {
+          size_t newsize = oldsize + nextsize;
+          set_inuse(m, p, newsize);
+        }
+        else {
+          mchunkptr r = chunk_plus_offset(p, nb);
+          set_inuse(m, p, nb);
+          set_inuse(m, r, rsize);
+          dispose_chunk(m, r, rsize);
+        }
+        newp = p;
+      }
+    }
+  }
+  else {
+    USAGE_ERROR_ACTION(m, chunk2mem(p));
+  }
+  return newp;
+}
+
+static void* internal_memalign(mstate m, size_t alignment, size_t bytes) {
+  void* mem = 0;
+  if (alignment <  MIN_CHUNK_SIZE) /* must be at least a minimum chunk size */
+    alignment = MIN_CHUNK_SIZE;
+  if ((alignment & (alignment-SIZE_T_ONE)) != 0) {/* Ensure a power of 2 */
+    size_t a = MALLOC_ALIGNMENT << 1;
+    while (a < alignment) a <<= 1;
+    alignment = a;
+  }
+  if (bytes >= MAX_REQUEST - alignment) {
+    if (m != 0)  { /* Test isn't needed but avoids compiler warning */
+      MALLOC_FAILURE_ACTION;
+    }
+  }
+  else {
+    size_t nb = request2size(bytes);
+    size_t req = nb + alignment + MIN_CHUNK_SIZE - CHUNK_OVERHEAD;
+    mem = internal_malloc(m, req);
+    if (mem != 0) {
+      mchunkptr p = mem2chunk(mem);
+      if (PREACTION(m))
+        return 0;
+      if ((((size_t)(mem)) & (alignment - 1)) != 0) { /* misaligned */
+        /*
+          Find an aligned spot inside chunk.  Since we need to give
+          back leading space in a chunk of at least MIN_CHUNK_SIZE, if
+          the first calculation places us at a spot with less than
+          MIN_CHUNK_SIZE leader, we can move to the next aligned spot.
+          We've allocated enough total room so that this is always
+          possible.
+        */
+        char* br = (char*)mem2chunk((size_t)(((size_t)((char*)mem + alignment -
+                                                       SIZE_T_ONE)) &
+                                             -alignment));
+        char* pos = ((size_t)(br - (char*)(p)) >= MIN_CHUNK_SIZE)?
+          br : br+alignment;
+        mchunkptr newp = (mchunkptr)pos;
+        size_t leadsize = pos - (char*)(p);
+        size_t newsize = chunksize(p) - leadsize;
+
+        if (is_mmapped(p)) { /* For mmapped chunks, just adjust offset */
+          newp->prev_foot = p->prev_foot + leadsize;
+          newp->head = newsize;
+        }
+        else { /* Otherwise, give back leader, use the rest */
+          set_inuse(m, newp, newsize);
+          set_inuse(m, p, leadsize);
+          dispose_chunk(m, p, leadsize);
+        }
+        p = newp;
+      }
+
+      /* Give back spare room at the end */
+      if (!is_mmapped(p)) {
+        size_t size = chunksize(p);
+        if (size > nb + MIN_CHUNK_SIZE) {
+          size_t remainder_size = size - nb;
+          mchunkptr remainder = chunk_plus_offset(p, nb);
+          set_inuse(m, p, nb);
+          set_inuse(m, remainder, remainder_size);
+          dispose_chunk(m, remainder, remainder_size);
+        }
+      }
+
+      mem = chunk2mem(p);
+      assert (chunksize(p) >= nb);
+      assert(((size_t)mem & (alignment - 1)) == 0);
+      check_inuse_chunk(m, p);
+      POSTACTION(m);
+    }
+  }
+  return mem;
+}
+
+/*
+  Common support for independent_X routines, handling
+    all of the combinations that can result.
+  The opts arg has:
+    bit 0 set if all elements are same size (using sizes[0])
+    bit 1 set if elements should be zeroed
+*/
+static void** ialloc(mstate m,
+                     size_t n_elements,
+                     size_t* sizes,
+                     int opts,
+                     void* chunks[]) {
+
+  size_t    element_size;   /* chunksize of each element, if all same */
+  size_t    contents_size;  /* total size of elements */
+  size_t    array_size;     /* request size of pointer array */
+  void*     mem;            /* malloced aggregate space */
+  mchunkptr p;              /* corresponding chunk */
+  size_t    remainder_size; /* remaining bytes while splitting */
+  void**    marray;         /* either "chunks" or malloced ptr array */
+  mchunkptr array_chunk;    /* chunk for malloced ptr array */
+  flag_t    was_enabled;    /* to disable mmap */
+  size_t    size;
+  size_t    i;
+
+  ensure_initialization();
+  /* compute array length, if needed */
+  if (chunks != 0) {
+    if (n_elements == 0)
+      return chunks; /* nothing to do */
+    marray = chunks;
+    array_size = 0;
+  }
+  else {
+    /* if empty req, must still return chunk representing empty array */
+    if (n_elements == 0)
+      return (void**)internal_malloc(m, 0);
+    marray = 0;
+    array_size = request2size(n_elements * (sizeof(void*)));
+  }
+
+  /* compute total element size */
+  if (opts & 0x1) { /* all-same-size */
+    element_size = request2size(*sizes);
+    contents_size = n_elements * element_size;
+  }
+  else { /* add up all the sizes */
+    element_size = 0;
+    contents_size = 0;
+    for (i = 0; i != n_elements; ++i)
+      contents_size += request2size(sizes[i]);
+  }
+
+  size = contents_size + array_size;
+
+  /*
+     Allocate the aggregate chunk.  First disable direct-mmapping so
+     malloc won't use it, since we would not be able to later
+     free/realloc space internal to a segregated mmap region.
+  */
+  was_enabled = use_mmap(m);
+  disable_mmap(m);
+  mem = internal_malloc(m, size - CHUNK_OVERHEAD);
+  if (was_enabled)
+    enable_mmap(m);
+  if (mem == 0)
+    return 0;
+
+  if (PREACTION(m)) return 0;
+  p = mem2chunk(mem);
+  remainder_size = chunksize(p);
+
+  assert(!is_mmapped(p));
+
+  if (opts & 0x2) {       /* optionally clear the elements */
+    memset((size_t*)mem, 0, remainder_size - SIZE_T_SIZE - array_size);
+  }
+
+  /* If not provided, allocate the pointer array as final part of chunk */
+  if (marray == 0) {
+    size_t  array_chunk_size;
+    array_chunk = chunk_plus_offset(p, contents_size);
+    array_chunk_size = remainder_size - contents_size;
+    marray = (void**) (chunk2mem(array_chunk));
+    set_size_and_pinuse_of_inuse_chunk(m, array_chunk, array_chunk_size);
+    remainder_size = contents_size;
+  }
+
+  /* split out elements */
+  for (i = 0; ; ++i) {
+    marray[i] = chunk2mem(p);
+    if (i != n_elements-1) {
+      if (element_size != 0)
+        size = element_size;
+      else
+        size = request2size(sizes[i]);
+      remainder_size -= size;
+      set_size_and_pinuse_of_inuse_chunk(m, p, size);
+      p = chunk_plus_offset(p, size);
+    }
+    else { /* the final element absorbs any overallocation slop */
+      set_size_and_pinuse_of_inuse_chunk(m, p, remainder_size);
+      break;
+    }
+  }
+
+#if DEBUG
+  if (marray != chunks) {
+    /* final element must have exactly exhausted chunk */
+    if (element_size != 0) {
+      assert(remainder_size == element_size);
+    }
+    else {
+      assert(remainder_size == request2size(sizes[i]));
+    }
+    check_inuse_chunk(m, mem2chunk(marray));
+  }
+  for (i = 0; i != n_elements; ++i)
+    check_inuse_chunk(m, mem2chunk(marray[i]));
+
+#endif /* DEBUG */
+
+  POSTACTION(m);
+  return marray;
+}
+
+/* Try to free all pointers in the given array.
+   Note: this could be made faster, by delaying consolidation,
+   at the price of disabling some user integrity checks, We
+   still optimize some consolidations by combining adjacent
+   chunks before freeing, which will occur often if allocated
+   with ialloc or the array is sorted.
+*/
+static size_t internal_bulk_free(mstate m, void* array[], size_t nelem) {
+  size_t unfreed = 0;
+  if (!PREACTION(m)) {
+    void** a;
+    void** fence = &(array[nelem]);
+    for (a = array; a != fence; ++a) {
+      void* mem = *a;
+      if (mem != 0) {
+        mchunkptr p = mem2chunk(mem);
+        size_t psize = chunksize(p);
+#if FOOTERS
+        if (get_mstate_for(p) != m) {
+          ++unfreed;
+          continue;
+        }
+#endif
+        check_inuse_chunk(m, p);
+        *a = 0;
+        if (RTCHECK(ok_address(m, p) && ok_inuse(p))) {
+          void ** b = a + 1; /* try to merge with next chunk */
+          mchunkptr next = next_chunk(p);
+          if (b != fence && *b == chunk2mem(next)) {
+            size_t newsize = chunksize(next) + psize;
+            set_inuse(m, p, newsize);
+            *b = chunk2mem(p);
+          }
+          else
+            dispose_chunk(m, p, psize);
+        }
+        else {
+          CORRUPTION_ERROR_ACTION(m);
+          break;
+        }
+      }
+    }
+    if (should_trim(m, m->topsize))
+      sys_trim(m, 0);
+    POSTACTION(m);
+  }
+  return unfreed;
+}
+
+/* Traversal */
+#if MALLOC_INSPECT_ALL
+static void internal_inspect_all(mstate m,
+                                 void(*handler)(void *start,
+                                                void *end,
+                                                size_t used_bytes,
+                                                void* callback_arg),
+                                 void* arg) {
+  if (is_initialized(m)) {
+    mchunkptr top = m->top;
+    msegmentptr s;
+    for (s = &m->seg; s != 0; s = s->next) {
+      mchunkptr q = align_as_chunk(s->base);
+      while (segment_holds(s, q) && q->head != FENCEPOST_HEAD) {
+        mchunkptr next = next_chunk(q);
+        size_t sz = chunksize(q);
+        size_t used;
+        void* start;
+        if (is_inuse(q)) {
+          used = sz - CHUNK_OVERHEAD; /* must not be mmapped */
+          start = chunk2mem(q);
+        }
+        else {
+          used = 0;
+          if (is_small(sz)) {     /* offset by possible bookkeeping */
+            start = (void*)((char*)q + sizeof(struct malloc_chunk));
+          }
+          else {
+            start = (void*)((char*)q + sizeof(struct malloc_tree_chunk));
+          }
+        }
+        if (start < (void*)next)  /* skip if all space is bookkeeping */
+          handler(start, next, used, arg);
+        if (q == top)
+          break;
+        q = next;
+      }
+    }
+  }
+}
+#endif /* MALLOC_INSPECT_ALL */
+
+/* ------------------ Exported realloc, memalign, etc -------------------- */
+
+#if !ONLY_MSPACES
+
+void* dlrealloc(void* oldmem, size_t bytes) {
+  void* mem = 0;
+  if (oldmem == 0) {
+    mem = dlmalloc(bytes);
+  }
+  else if (bytes >= MAX_REQUEST) {
+    MALLOC_FAILURE_ACTION;
+  }
+#ifdef REALLOC_ZERO_BYTES_FREES
+  else if (bytes == 0) {
+    dlfree(oldmem);
+  }
+#endif /* REALLOC_ZERO_BYTES_FREES */
+  else {
+    size_t nb = request2size(bytes);
+    mchunkptr oldp = mem2chunk(oldmem);
+#if ! FOOTERS
+    mstate m = gm;
+#else /* FOOTERS */
+    mstate m = get_mstate_for(oldp);
+    if (!ok_magic(m)) {
+      USAGE_ERROR_ACTION(m, oldmem);
+      return 0;
+    }
+#endif /* FOOTERS */
+    if (!PREACTION(m)) {
+      mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1);
+      POSTACTION(m);
+      if (newp != 0) {
+        check_inuse_chunk(m, newp);
+        mem = chunk2mem(newp);
+      }
+      else {
+        mem = internal_malloc(m, bytes);
+        if (mem != 0) {
+          size_t oc = chunksize(oldp) - overhead_for(oldp);
+          memcpy(mem, oldmem, (oc < bytes)? oc : bytes);
+          internal_free(m, oldmem);
+        }
+      }
+    }
+  }
+  return mem;
+}
+
+void* dlrealloc_in_place(void* oldmem, size_t bytes) {
+  void* mem = 0;
+  if (oldmem != 0) {
+    if (bytes >= MAX_REQUEST) {
+      MALLOC_FAILURE_ACTION;
+    }
+    else {
+      size_t nb = request2size(bytes);
+      mchunkptr oldp = mem2chunk(oldmem);
+#if ! FOOTERS
+      mstate m = gm;
+#else /* FOOTERS */
+      mstate m = get_mstate_for(oldp);
+      if (!ok_magic(m)) {
+        USAGE_ERROR_ACTION(m, oldmem);
+        return 0;
+      }
+#endif /* FOOTERS */
+      if (!PREACTION(m)) {
+        mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
+        POSTACTION(m);
+        if (newp == oldp) {
+          check_inuse_chunk(m, newp);
+          mem = oldmem;
+        }
+      }
+    }
+  }
+  return mem;
+}
+
+void* dlmemalign(size_t alignment, size_t bytes) {
+  if (alignment <= MALLOC_ALIGNMENT) {
+    return dlmalloc(bytes);
+  }
+  return internal_memalign(gm, alignment, bytes);
+}
+
+int dlposix_memalign(void** pp, size_t alignment, size_t bytes) {
+  void* mem = 0;
+  if (alignment == MALLOC_ALIGNMENT)
+    mem = dlmalloc(bytes);
+  else {
+    size_t d = alignment / sizeof(void*);
+    size_t r = alignment % sizeof(void*);
+    if (r != 0 || d == 0 || (d & (d-SIZE_T_ONE)) != 0)
+      return EINVAL;
+    else if (bytes <= MAX_REQUEST - alignment) {
+      if (alignment <  MIN_CHUNK_SIZE)
+        alignment = MIN_CHUNK_SIZE;
+      mem = internal_memalign(gm, alignment, bytes);
+    }
+  }
+  if (mem == 0)
+    return ENOMEM;
+  else {
+    *pp = mem;
+    return 0;
+  }
+}
+
+void* dlvalloc(size_t bytes) {
+  size_t pagesz;
+  ensure_initialization();
+  pagesz = mparams.page_size;
+  return dlmemalign(pagesz, bytes);
+}
+
+void* dlpvalloc(size_t bytes) {
+  size_t pagesz;
+  ensure_initialization();
+  pagesz = mparams.page_size;
+  return dlmemalign(pagesz, (bytes + pagesz - SIZE_T_ONE) & ~(pagesz - SIZE_T_ONE));
+}
+
+void** dlindependent_calloc(size_t n_elements, size_t elem_size,
+                            void* chunks[]) {
+  size_t sz = elem_size; /* serves as 1-element array */
+  return ialloc(gm, n_elements, &sz, 3, chunks);
+}
+
+void** dlindependent_comalloc(size_t n_elements, size_t sizes[],
+                              void* chunks[]) {
+  return ialloc(gm, n_elements, sizes, 0, chunks);
+}
+
+size_t dlbulk_free(void* array[], size_t nelem) {
+  return internal_bulk_free(gm, array, nelem);
+}
+
+#if MALLOC_INSPECT_ALL
+void dlmalloc_inspect_all(void(*handler)(void *start,
+                                         void *end,
+                                         size_t used_bytes,
+                                         void* callback_arg),
+                          void* arg) {
+  ensure_initialization();
+  if (!PREACTION(gm)) {
+    internal_inspect_all(gm, handler, arg);
+    POSTACTION(gm);
+  }
+}
+#endif /* MALLOC_INSPECT_ALL */
+
+int dlmalloc_trim(size_t pad) {
+  int result = 0;
+  ensure_initialization();
+  if (!PREACTION(gm)) {
+    result = sys_trim(gm, pad);
+    POSTACTION(gm);
+  }
+  return result;
+}
+
+size_t dlmalloc_footprint(void) {
+  return gm->footprint;
+}
+
+size_t dlmalloc_max_footprint(void) {
+  return gm->max_footprint;
+}
+
+size_t dlmalloc_footprint_limit(void) {
+  size_t maf = gm->footprint_limit;
+  return maf == 0 ? MAX_SIZE_T : maf;
+}
+
+size_t dlmalloc_set_footprint_limit(size_t bytes) {
+  ensure_initialization();
+  size_t result;  /* invert sense of 0 */
+  if (bytes == 0)
+    result = granularity_align(1); /* Use minimal size */
+  if (bytes == MAX_SIZE_T)
+    result = 0;                    /* disable */
+  else
+    result = granularity_align(bytes);
+  return gm->footprint_limit = result;
+}
+
+#if !NO_MALLINFO
+struct mallinfo dlmallinfo(void) {
+  return internal_mallinfo(gm);
+}
+#endif /* NO_MALLINFO */
+
+#if !NO_MALLOC_STATS
+void dlmalloc_stats() {
+  internal_malloc_stats(gm);
+}
+#endif /* NO_MALLOC_STATS */
+
+int dlmallopt(int param_number, int value) {
+  return change_mparam(param_number, value);
+}
+
+size_t dlmalloc_usable_size(void* mem) {
+  if (mem != 0) {
+    mchunkptr p = mem2chunk(mem);
+    if (is_inuse(p))
+      return chunksize(p) - overhead_for(p);
+  }
+  return 0;
+}
+
+#endif /* !ONLY_MSPACES */
+
+/* ----------------------------- user mspaces ---------------------------- */
+
+#if MSPACES
+
+static mstate init_user_mstate(char* tbase, size_t tsize) {
+  size_t msize = pad_request(sizeof(struct malloc_state));
+  mchunkptr mn;
+  mchunkptr msp = align_as_chunk(tbase);
+  mstate m = (mstate)(chunk2mem(msp));
+  memset(m, 0, msize);
+  (void)INITIAL_LOCK(&m->mutex);
+  msp->head = (msize|INUSE_BITS);
+  m->seg.base = m->least_addr = tbase;
+  m->seg.size = m->footprint = m->max_footprint = tsize;
+  m->magic = mparams.magic;
+  m->release_checks = MAX_RELEASE_CHECK_RATE;
+  m->mflags = mparams.default_mflags;
+  m->extp = 0;
+  m->exts = 0;
+  disable_contiguous(m);
+  init_bins(m);
+  mn = next_chunk(mem2chunk(m));
+  init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) - TOP_FOOT_SIZE);
+  check_top_chunk(m, m->top);
+  return m;
+}
+
+mspace create_mspace(size_t capacity, int locked) {
+  mstate m = 0;
+  size_t msize;
+  ensure_initialization();
+  msize = pad_request(sizeof(struct malloc_state));
+  if (capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) {
+    size_t rs = ((capacity == 0)? mparams.granularity :
+                 (capacity + TOP_FOOT_SIZE + msize));
+    size_t tsize = granularity_align(rs);
+    char* tbase = (char*)(CALL_MMAP(tsize));
+    if (tbase != CMFAIL) {
+      m = init_user_mstate(tbase, tsize);
+      m->seg.sflags = USE_MMAP_BIT;
+      set_lock(m, locked);
+    }
+  }
+  return (mspace)m;
+}
+
+mspace create_mspace_with_base(void* base, size_t capacity, int locked) {
+  mstate m = 0;
+  size_t msize;
+  ensure_initialization();
+  msize = pad_request(sizeof(struct malloc_state));
+  if (capacity > msize + TOP_FOOT_SIZE &&
+      capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) {
+    m = init_user_mstate((char*)base, capacity);
+    m->seg.sflags = EXTERN_BIT;
+    set_lock(m, locked);
+  }
+  return (mspace)m;
+}
+
+int mspace_track_large_chunks(mspace msp, int enable) {
+  int ret = 0;
+  mstate ms = (mstate)msp;
+  if (!PREACTION(ms)) {
+    if (!use_mmap(ms)) {
+      ret = 1;
+    }
+    if (!enable) {
+      enable_mmap(ms);
+    } else {
+      disable_mmap(ms);
+    }
+    POSTACTION(ms);
+  }
+  return ret;
+}
+
+size_t destroy_mspace(mspace msp) {
+  size_t freed = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    msegmentptr sp = &ms->seg;
+    (void)DESTROY_LOCK(&ms->mutex); /* destroy before unmapped */
+    while (sp != 0) {
+      char* base = sp->base;
+      size_t size = sp->size;
+      flag_t flag = sp->sflags;
+      (void)base; /* placate people compiling -Wunused-variable */
+      sp = sp->next;
+      if ((flag & USE_MMAP_BIT) && !(flag & EXTERN_BIT) &&
+          CALL_MUNMAP(base, size) == 0)
+        freed += size;
+    }
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return freed;
+}
+
+/*
+  mspace versions of routines are near-clones of the global
+  versions. This is not so nice but better than the alternatives.
+*/
+
+void* mspace_malloc(mspace msp, size_t bytes) {
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  if (!PREACTION(ms)) {
+    void* mem;
+    size_t nb;
+    if (bytes <= MAX_SMALL_REQUEST) {
+      bindex_t idx;
+      binmap_t smallbits;
+      nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes);
+      idx = small_index(nb);
+      smallbits = ms->smallmap >> idx;
+
+      if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */
+        mchunkptr b, p;
+        idx += ~smallbits & 1;       /* Uses next bin if idx empty */
+        b = smallbin_at(ms, idx);
+        p = b->fd;
+        assert(chunksize(p) == small_index2size(idx));
+        unlink_first_small_chunk(ms, b, p, idx);
+        set_inuse_and_pinuse(ms, p, small_index2size(idx));
+        mem = chunk2mem(p);
+        check_malloced_chunk(ms, mem, nb);
+        goto postaction;
+      }
+
+      else if (nb > ms->dvsize) {
+        if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
+          mchunkptr b, p, r;
+          size_t rsize;
+          bindex_t i;
+          binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx));
+          binmap_t leastbit = least_bit(leftbits);
+          compute_bit2idx(leastbit, i);
+          b = smallbin_at(ms, i);
+          p = b->fd;
+          assert(chunksize(p) == small_index2size(i));
+          unlink_first_small_chunk(ms, b, p, i);
+          rsize = small_index2size(i) - nb;
+          /* Fit here cannot be remainderless if 4byte sizes */
+          if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE)
+            set_inuse_and_pinuse(ms, p, small_index2size(i));
+          else {
+            set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
+            r = chunk_plus_offset(p, nb);
+            set_size_and_pinuse_of_free_chunk(r, rsize);
+            replace_dv(ms, r, rsize);
+          }
+          mem = chunk2mem(p);
+          check_malloced_chunk(ms, mem, nb);
+          goto postaction;
+        }
+
+        else if (ms->treemap != 0 && (mem = tmalloc_small(ms, nb)) != 0) {
+          check_malloced_chunk(ms, mem, nb);
+          goto postaction;
+        }
+      }
+    }
+    else if (bytes >= MAX_REQUEST)
+      nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */
+    else {
+      nb = pad_request(bytes);
+      if (ms->treemap != 0 && (mem = tmalloc_large(ms, nb)) != 0) {
+        check_malloced_chunk(ms, mem, nb);
+        goto postaction;
+      }
+    }
+
+    if (nb <= ms->dvsize) {
+      size_t rsize = ms->dvsize - nb;
+      mchunkptr p = ms->dv;
+      if (rsize >= MIN_CHUNK_SIZE) { /* split dv */
+        mchunkptr r = ms->dv = chunk_plus_offset(p, nb);
+        ms->dvsize = rsize;
+        set_size_and_pinuse_of_free_chunk(r, rsize);
+        set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
+      }
+      else { /* exhaust dv */
+        size_t dvs = ms->dvsize;
+        ms->dvsize = 0;
+        ms->dv = 0;
+        set_inuse_and_pinuse(ms, p, dvs);
+      }
+      mem = chunk2mem(p);
+      check_malloced_chunk(ms, mem, nb);
+      goto postaction;
+    }
+
+    else if (nb < ms->topsize) { /* Split top */
+      size_t rsize = ms->topsize -= nb;
+      mchunkptr p = ms->top;
+      mchunkptr r = ms->top = chunk_plus_offset(p, nb);
+      r->head = rsize | PINUSE_BIT;
+      set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
+      mem = chunk2mem(p);
+      check_top_chunk(ms, ms->top);
+      check_malloced_chunk(ms, mem, nb);
+      goto postaction;
+    }
+
+    mem = sys_alloc(ms, nb);
+
+  postaction:
+    POSTACTION(ms);
+    return mem;
+  }
+
+  return 0;
+}
+
+void mspace_free(mspace msp, void* mem) {
+  if (mem != 0) {
+    mchunkptr p  = mem2chunk(mem);
+#if FOOTERS
+    mstate fm = get_mstate_for(p);
+    (void)msp; /* placate people compiling -Wunused */
+#else /* FOOTERS */
+    mstate fm = (mstate)msp;
+#endif /* FOOTERS */
+    if (!ok_magic(fm)) {
+      USAGE_ERROR_ACTION(fm, p);
+      return;
+    }
+    if (!PREACTION(fm)) {
+      check_inuse_chunk(fm, p);
+      if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) {
+        size_t psize = chunksize(p);
+        mchunkptr next = chunk_plus_offset(p, psize);
+        if (!pinuse(p)) {
+          size_t prevsize = p->prev_foot;
+          if (is_mmapped(p)) {
+            psize += prevsize + MMAP_FOOT_PAD;
+            if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
+              fm->footprint -= psize;
+            goto postaction;
+          }
+          else {
+            mchunkptr prev = chunk_minus_offset(p, prevsize);
+            psize += prevsize;
+            p = prev;
+            if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */
+              if (p != fm->dv) {
+                unlink_chunk(fm, p, prevsize);
+              }
+              else if ((next->head & INUSE_BITS) == INUSE_BITS) {
+                fm->dvsize = psize;
+                set_free_with_pinuse(p, psize, next);
+                goto postaction;
+              }
+            }
+            else
+              goto erroraction;
+          }
+        }
+
+        if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) {
+          if (!cinuse(next)) {  /* consolidate forward */
+            if (next == fm->top) {
+              size_t tsize = fm->topsize += psize;
+              fm->top = p;
+              p->head = tsize | PINUSE_BIT;
+              if (p == fm->dv) {
+                fm->dv = 0;
+                fm->dvsize = 0;
+              }
+              if (should_trim(fm, tsize))
+                sys_trim(fm, 0);
+              goto postaction;
+            }
+            else if (next == fm->dv) {
+              size_t dsize = fm->dvsize += psize;
+              fm->dv = p;
+              set_size_and_pinuse_of_free_chunk(p, dsize);
+              goto postaction;
+            }
+            else {
+              size_t nsize = chunksize(next);
+              psize += nsize;
+              unlink_chunk(fm, next, nsize);
+              set_size_and_pinuse_of_free_chunk(p, psize);
+              if (p == fm->dv) {
+                fm->dvsize = psize;
+                goto postaction;
+              }
+            }
+          }
+          else
+            set_free_with_pinuse(p, psize, next);
+
+          if (is_small(psize)) {
+            insert_small_chunk(fm, p, psize);
+            check_free_chunk(fm, p);
+          }
+          else {
+            tchunkptr tp = (tchunkptr)p;
+            insert_large_chunk(fm, tp, psize);
+            check_free_chunk(fm, p);
+            if (--fm->release_checks == 0)
+              release_unused_segments(fm);
+          }
+          goto postaction;
+        }
+      }
+    erroraction:
+      USAGE_ERROR_ACTION(fm, p);
+    postaction:
+      POSTACTION(fm);
+    }
+  }
+}
+
+void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size) {
+  void* mem;
+  size_t req = 0;
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  if (n_elements != 0) {
+    req = n_elements * elem_size;
+    if (((n_elements | elem_size) & ~(size_t)0xffff) &&
+        (req / n_elements != elem_size))
+      req = MAX_SIZE_T; /* force downstream failure on overflow */
+  }
+  mem = internal_malloc(ms, req);
+  if (mem != 0 && calloc_must_clear(mem2chunk(mem)))
+    memset(mem, 0, req);
+  return mem;
+}
+
+void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) {
+  void* mem = 0;
+  if (oldmem == 0) {
+    mem = mspace_malloc(msp, bytes);
+  }
+  else if (bytes >= MAX_REQUEST) {
+    MALLOC_FAILURE_ACTION;
+  }
+#ifdef REALLOC_ZERO_BYTES_FREES
+  else if (bytes == 0) {
+    mspace_free(msp, oldmem);
+  }
+#endif /* REALLOC_ZERO_BYTES_FREES */
+  else {
+    size_t nb = request2size(bytes);
+    mchunkptr oldp = mem2chunk(oldmem);
+#if ! FOOTERS
+    mstate m = (mstate)msp;
+#else /* FOOTERS */
+    mstate m = get_mstate_for(oldp);
+    if (!ok_magic(m)) {
+      USAGE_ERROR_ACTION(m, oldmem);
+      return 0;
+    }
+#endif /* FOOTERS */
+    if (!PREACTION(m)) {
+      mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1);
+      POSTACTION(m);
+      if (newp != 0) {
+        check_inuse_chunk(m, newp);
+        mem = chunk2mem(newp);
+      }
+      else {
+        mem = mspace_malloc(m, bytes);
+        if (mem != 0) {
+          size_t oc = chunksize(oldp) - overhead_for(oldp);
+          memcpy(mem, oldmem, (oc < bytes)? oc : bytes);
+          mspace_free(m, oldmem);
+        }
+      }
+    }
+  }
+  return mem;
+}
+
+void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) {
+  void* mem = 0;
+  if (oldmem != 0) {
+    if (bytes >= MAX_REQUEST) {
+      MALLOC_FAILURE_ACTION;
+    }
+    else {
+      size_t nb = request2size(bytes);
+      mchunkptr oldp = mem2chunk(oldmem);
+#if ! FOOTERS
+      mstate m = (mstate)msp;
+#else /* FOOTERS */
+      mstate m = get_mstate_for(oldp);
+      (void)msp; /* placate people compiling -Wunused */
+      if (!ok_magic(m)) {
+        USAGE_ERROR_ACTION(m, oldmem);
+        return 0;
+      }
+#endif /* FOOTERS */
+      if (!PREACTION(m)) {
+        mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
+        POSTACTION(m);
+        if (newp == oldp) {
+          check_inuse_chunk(m, newp);
+          mem = oldmem;
+        }
+      }
+    }
+  }
+  return mem;
+}
+
+void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) {
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  if (alignment <= MALLOC_ALIGNMENT)
+    return mspace_malloc(msp, bytes);
+  return internal_memalign(ms, alignment, bytes);
+}
+
+void** mspace_independent_calloc(mspace msp, size_t n_elements,
+                                 size_t elem_size, void* chunks[]) {
+  size_t sz = elem_size; /* serves as 1-element array */
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  return ialloc(ms, n_elements, &sz, 3, chunks);
+}
+
+void** mspace_independent_comalloc(mspace msp, size_t n_elements,
+                                   size_t sizes[], void* chunks[]) {
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+    return 0;
+  }
+  return ialloc(ms, n_elements, sizes, 0, chunks);
+}
+
+size_t mspace_bulk_free(mspace msp, void* array[], size_t nelem) {
+  return internal_bulk_free((mstate)msp, array, nelem);
+}
+
+#if MALLOC_INSPECT_ALL
+void mspace_inspect_all(mspace msp,
+                        void(*handler)(void *start,
+                                       void *end,
+                                       size_t used_bytes,
+                                       void* callback_arg),
+                        void* arg) {
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    if (!PREACTION(ms)) {
+      internal_inspect_all(ms, handler, arg);
+      POSTACTION(ms);
+    }
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+}
+#endif /* MALLOC_INSPECT_ALL */
+
+int mspace_trim(mspace msp, size_t pad) {
+  int result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    if (!PREACTION(ms)) {
+      result = sys_trim(ms, pad);
+      POSTACTION(ms);
+    }
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+#if !NO_MALLOC_STATS
+void mspace_malloc_stats(mspace msp) {
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    internal_malloc_stats(ms);
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+}
+#endif /* NO_MALLOC_STATS */
+
+size_t mspace_footprint(mspace msp) {
+  size_t result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    result = ms->footprint;
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+size_t mspace_max_footprint(mspace msp) {
+  size_t result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    result = ms->max_footprint;
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+size_t mspace_footprint_limit(mspace msp) {
+  size_t result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    size_t maf = ms->footprint_limit;
+    result = (maf == 0) ? MAX_SIZE_T : maf;
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+size_t mspace_set_footprint_limit(mspace msp, size_t bytes) {
+  size_t result = 0;
+  mstate ms = (mstate)msp;
+  if (ok_magic(ms)) {
+    if (bytes == 0)
+      result = granularity_align(1); /* Use minimal size */
+    if (bytes == MAX_SIZE_T)
+      result = 0;                    /* disable */
+    else
+      result = granularity_align(bytes);
+    ms->footprint_limit = result;
+  }
+  else {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return result;
+}
+
+#if !NO_MALLINFO
+struct mallinfo mspace_mallinfo(mspace msp) {
+  mstate ms = (mstate)msp;
+  if (!ok_magic(ms)) {
+    USAGE_ERROR_ACTION(ms,ms);
+  }
+  return internal_mallinfo(ms);
+}
+#endif /* NO_MALLINFO */
+
+size_t mspace_usable_size(const void* mem) {
+  if (mem != 0) {
+    mchunkptr p = mem2chunk(mem);
+    if (is_inuse(p))
+      return chunksize(p) - overhead_for(p);
+  }
+  return 0;
+}
+
+int mspace_mallopt(int param_number, int value) {
+  return change_mparam(param_number, value);
+}
+
+#endif /* MSPACES */
+
+
+/* -------------------- Alternative MORECORE functions ------------------- */
+
+/*
+  Guidelines for creating a custom version of MORECORE:
+
+  * For best performance, MORECORE should allocate in multiples of pagesize.
+  * MORECORE may allocate more memory than requested. (Or even less,
+      but this will usually result in a malloc failure.)
+  * MORECORE must not allocate memory when given argument zero, but
+      instead return one past the end address of memory from previous
+      nonzero call.
+  * For best performance, consecutive calls to MORECORE with positive
+      arguments should return increasing addresses, indicating that
+      space has been contiguously extended.
+  * Even though consecutive calls to MORECORE need not return contiguous
+      addresses, it must be OK for malloc'ed chunks to span multiple
+      regions in those cases where they do happen to be contiguous.
+  * MORECORE need not handle negative arguments -- it may instead
+      just return MFAIL when given negative arguments.
+      Negative arguments are always multiples of pagesize. MORECORE
+      must not misinterpret negative args as large positive unsigned
+      args. You can suppress all such calls from even occurring by defining
+      MORECORE_CANNOT_TRIM,
+
+  As an example alternative MORECORE, here is a custom allocator
+  kindly contributed for pre-OSX macOS.  It uses virtually but not
+  necessarily physically contiguous non-paged memory (locked in,
+  present and won't get swapped out).  You can use it by uncommenting
+  this section, adding some #includes, and setting up the appropriate
+  defines above:
+
+      #define MORECORE osMoreCore
+
+  There is also a shutdown routine that should somehow be called for
+  cleanup upon program exit.
+
+  #define MAX_POOL_ENTRIES 100
+  #define MINIMUM_MORECORE_SIZE  (64 * 1024U)
+  static int next_os_pool;
+  void *our_os_pools[MAX_POOL_ENTRIES];
+
+  void *osMoreCore(int size)
+  {
+    void *ptr = 0;
+    static void *sbrk_top = 0;
+
+    if (size > 0)
+    {
+      if (size < MINIMUM_MORECORE_SIZE)
+         size = MINIMUM_MORECORE_SIZE;
+      if (CurrentExecutionLevel() == kTaskLevel)
+         ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
+      if (ptr == 0)
+      {
+        return (void *) MFAIL;
+      }
+      // save ptrs so they can be freed during cleanup
+      our_os_pools[next_os_pool] = ptr;
+      next_os_pool++;
+      ptr = (void *) ((((size_t) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
+      sbrk_top = (char *) ptr + size;
+      return ptr;
+    }
+    else if (size < 0)
+    {
+      // we don't currently support shrink behavior
+      return (void *) MFAIL;
+    }
+    else
+    {
+      return sbrk_top;
+    }
+  }
+
+  // cleanup any allocated memory pools
+  // called as last thing before shutting down driver
+
+  void osCleanupMem(void)
+  {
+    void **ptr;
+
+    for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
+      if (*ptr)
+      {
+         PoolDeallocate(*ptr);
+         *ptr = 0;
+      }
+  }
+
+*/
+
+
+/* -----------------------------------------------------------------------
+History:
+    v2.8.6 Wed Aug 29 06:57:58 2012  Doug Lea
+      * fix bad comparison in dlposix_memalign
+      * don't reuse adjusted asize in sys_alloc
+      * add LOCK_AT_FORK -- thanks to Kirill Artamonov for the suggestion
+      * reduce compiler warnings -- thanks to all who reported/suggested these
+
+    v2.8.5 Sun May 22 10:26:02 2011  Doug Lea  (dl at gee)
+      * Always perform unlink checks unless INSECURE
+      * Add posix_memalign.
+      * Improve realloc to expand in more cases; expose realloc_in_place.
+        Thanks to Peter Buhr for the suggestion.
+      * Add footprint_limit, inspect_all, bulk_free. Thanks
+        to Barry Hayes and others for the suggestions.
+      * Internal refactorings to avoid calls while holding locks
+      * Use non-reentrant locks by default. Thanks to Roland McGrath
+        for the suggestion.
+      * Small fixes to mspace_destroy, reset_on_error.
+      * Various configuration extensions/changes. Thanks
+         to all who contributed these.
+
+    V2.8.4a Thu Apr 28 14:39:43 2011 (dl at gee.cs.oswego.edu)
+      * Update Creative Commons URL
+
+    V2.8.4 Wed May 27 09:56:23 2009  Doug Lea  (dl at gee)
+      * Use zeros instead of prev foot for is_mmapped
+      * Add mspace_track_large_chunks; thanks to Jean Brouwers
+      * Fix set_inuse in internal_realloc; thanks to Jean Brouwers
+      * Fix insufficient sys_alloc padding when using 16byte alignment
+      * Fix bad error check in mspace_footprint
+      * Adaptations for ptmalloc; thanks to Wolfram Gloger.
+      * Reentrant spin locks; thanks to Earl Chew and others
+      * Win32 improvements; thanks to Niall Douglas and Earl Chew
+      * Add NO_SEGMENT_TRAVERSAL and MAX_RELEASE_CHECK_RATE options
+      * Extension hook in malloc_state
+      * Various small adjustments to reduce warnings on some compilers
+      * Various configuration extensions/changes for more platforms. Thanks
+         to all who contributed these.
+
+    V2.8.3 Thu Sep 22 11:16:32 2005  Doug Lea  (dl at gee)
+      * Add max_footprint functions
+      * Ensure all appropriate literals are size_t
+      * Fix conditional compilation problem for some #define settings
+      * Avoid concatenating segments with the one provided
+        in create_mspace_with_base
+      * Rename some variables to avoid compiler shadowing warnings
+      * Use explicit lock initialization.
+      * Better handling of sbrk interference.
+      * Simplify and fix segment insertion, trimming and mspace_destroy
+      * Reinstate REALLOC_ZERO_BYTES_FREES option from 2.7.x
+      * Thanks especially to Dennis Flanagan for help on these.
+
+    V2.8.2 Sun Jun 12 16:01:10 2005  Doug Lea  (dl at gee)
+      * Fix memalign brace error.
+
+    V2.8.1 Wed Jun  8 16:11:46 2005  Doug Lea  (dl at gee)
+      * Fix improper #endif nesting in C++
+      * Add explicit casts needed for C++
+
+    V2.8.0 Mon May 30 14:09:02 2005  Doug Lea  (dl at gee)
+      * Use trees for large bins
+      * Support mspaces
+      * Use segments to unify sbrk-based and mmap-based system allocation,
+        removing need for emulation on most platforms without sbrk.
+      * Default safety checks
+      * Optional footer checks. Thanks to William Robertson for the idea.
+      * Internal code refactoring
+      * Incorporate suggestions and platform-specific changes.
+        Thanks to Dennis Flanagan, Colin Plumb, Niall Douglas,
+        Aaron Bachmann,  Emery Berger, and others.
+      * Speed up non-fastbin processing enough to remove fastbins.
+      * Remove useless cfree() to avoid conflicts with other apps.
+      * Remove internal memcpy, memset. Compilers handle builtins better.
+      * Remove some options that no one ever used and rename others.
+
+    V2.7.2 Sat Aug 17 09:07:30 2002  Doug Lea  (dl at gee)
+      * Fix malloc_state bitmap array misdeclaration
+
+    V2.7.1 Thu Jul 25 10:58:03 2002  Doug Lea  (dl at gee)
+      * Allow tuning of FIRST_SORTED_BIN_SIZE
+      * Use PTR_UINT as type for all ptr->int casts. Thanks to John Belmonte.
+      * Better detection and support for non-contiguousness of MORECORE.
+        Thanks to Andreas Mueller, Conal Walsh, and Wolfram Gloger
+      * Bypass most of malloc if no frees. Thanks To Emery Berger.
+      * Fix freeing of old top non-contiguous chunk im sysmalloc.
+      * Raised default trim and map thresholds to 256K.
+      * Fix mmap-related #defines. Thanks to Lubos Lunak.
+      * Fix copy macros; added LACKS_FCNTL_H. Thanks to Neal Walfield.
+      * Branch-free bin calculation
+      * Default trim and mmap thresholds now 256K.
+
+    V2.7.0 Sun Mar 11 14:14:06 2001  Doug Lea  (dl at gee)
+      * Introduce independent_comalloc and independent_calloc.
+        Thanks to Michael Pachos for motivation and help.
+      * Make optional .h file available
+      * Allow > 2GB requests on 32bit systems.
+      * new WIN32 sbrk, mmap, munmap, lock code from <Walter@GeNeSys-e.de>.
+        Thanks also to Andreas Mueller <a.mueller at paradatec.de>,
+        and Anonymous.
+      * Allow override of MALLOC_ALIGNMENT (Thanks to Ruud Waij for
+        helping test this.)
+      * memalign: check alignment arg
+      * realloc: don't try to shift chunks backwards, since this
+        leads to  more fragmentation in some programs and doesn't
+        seem to help in any others.
+      * Collect all cases in malloc requiring system memory into sysmalloc
+      * Use mmap as backup to sbrk
+      * Place all internal state in malloc_state
+      * Introduce fastbins (although similar to 2.5.1)
+      * Many minor tunings and cosmetic improvements
+      * Introduce USE_PUBLIC_MALLOC_WRAPPERS, USE_MALLOC_LOCK
+      * Introduce MALLOC_FAILURE_ACTION, MORECORE_CONTIGUOUS
+        Thanks to Tony E. Bennett <tbennett@nvidia.com> and others.
+      * Include errno.h to support default failure action.
+
+    V2.6.6 Sun Dec  5 07:42:19 1999  Doug Lea  (dl at gee)
+      * return null for negative arguments
+      * Added Several WIN32 cleanups from Martin C. Fong <mcfong at yahoo.com>
+         * Add 'LACKS_SYS_PARAM_H' for those systems without 'sys/param.h'
+          (e.g. WIN32 platforms)
+         * Cleanup header file inclusion for WIN32 platforms
+         * Cleanup code to avoid Microsoft Visual C++ compiler complaints
+         * Add 'USE_DL_PREFIX' to quickly allow co-existence with existing
+           memory allocation routines
+         * Set 'malloc_getpagesize' for WIN32 platforms (needs more work)
+         * Use 'assert' rather than 'ASSERT' in WIN32 code to conform to
+           usage of 'assert' in non-WIN32 code
+         * Improve WIN32 'sbrk()' emulation's 'findRegion()' routine to
+           avoid infinite loop
+      * Always call 'fREe()' rather than 'free()'
+
+    V2.6.5 Wed Jun 17 15:57:31 1998  Doug Lea  (dl at gee)
+      * Fixed ordering problem with boundary-stamping
+
+    V2.6.3 Sun May 19 08:17:58 1996  Doug Lea  (dl at gee)
+      * Added pvalloc, as recommended by H.J. Liu
+      * Added 64bit pointer support mainly from Wolfram Gloger
+      * Added anonymously donated WIN32 sbrk emulation
+      * Malloc, calloc, getpagesize: add optimizations from Raymond Nijssen
+      * malloc_extend_top: fix mask error that caused wastage after
+        foreign sbrks
+      * Add linux mremap support code from HJ Liu
+
+    V2.6.2 Tue Dec  5 06:52:55 1995  Doug Lea  (dl at gee)
+      * Integrated most documentation with the code.
+      * Add support for mmap, with help from
+        Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
+      * Use last_remainder in more cases.
+      * Pack bins using idea from  colin@nyx10.cs.du.edu
+      * Use ordered bins instead of best-fit threshhold
+      * Eliminate block-local decls to simplify tracing and debugging.
+      * Support another case of realloc via move into top
+      * Fix error occuring when initial sbrk_base not word-aligned.
+      * Rely on page size for units instead of SBRK_UNIT to
+        avoid surprises about sbrk alignment conventions.
+      * Add mallinfo, mallopt. Thanks to Raymond Nijssen
+        (raymond@es.ele.tue.nl) for the suggestion.
+      * Add `pad' argument to malloc_trim and top_pad mallopt parameter.
+      * More precautions for cases where other routines call sbrk,
+        courtesy of Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
+      * Added macros etc., allowing use in linux libc from
+        H.J. Lu (hjl@gnu.ai.mit.edu)
+      * Inverted this history list
+
+    V2.6.1 Sat Dec  2 14:10:57 1995  Doug Lea  (dl at gee)
+      * Re-tuned and fixed to behave more nicely with V2.6.0 changes.
+      * Removed all preallocation code since under current scheme
+        the work required to undo bad preallocations exceeds
+        the work saved in good cases for most test programs.
+      * No longer use return list or unconsolidated bins since
+        no scheme using them consistently outperforms those that don't
+        given above changes.
+      * Use best fit for very large chunks to prevent some worst-cases.
+      * Added some support for debugging
+
+    V2.6.0 Sat Nov  4 07:05:23 1995  Doug Lea  (dl at gee)
+      * Removed footers when chunks are in use. Thanks to
+        Paul Wilson (wilson@cs.texas.edu) for the suggestion.
+
+    V2.5.4 Wed Nov  1 07:54:51 1995  Doug Lea  (dl at gee)
+      * Added malloc_trim, with help from Wolfram Gloger
+        (wmglo@Dent.MED.Uni-Muenchen.DE).
+
+    V2.5.3 Tue Apr 26 10:16:01 1994  Doug Lea  (dl at g)
+
+    V2.5.2 Tue Apr  5 16:20:40 1994  Doug Lea  (dl at g)
+      * realloc: try to expand in both directions
+      * malloc: swap order of clean-bin strategy;
+      * realloc: only conditionally expand backwards
+      * Try not to scavenge used bins
+      * Use bin counts as a guide to preallocation
+      * Occasionally bin return list chunks in first scan
+      * Add a few optimizations from colin@nyx10.cs.du.edu
+
+    V2.5.1 Sat Aug 14 15:40:43 1993  Doug Lea  (dl at g)
+      * faster bin computation & slightly different binning
+      * merged all consolidations to one part of malloc proper
+         (eliminating old malloc_find_space & malloc_clean_bin)
+      * Scan 2 returns chunks (not just 1)
+      * Propagate failure in realloc if malloc returns 0
+      * Add stuff to allow compilation on non-ANSI compilers
+          from kpv@research.att.com
+
+    V2.5 Sat Aug  7 07:41:59 1993  Doug Lea  (dl at g.oswego.edu)
+      * removed potential for odd address access in prev_chunk
+      * removed dependency on getpagesize.h
+      * misc cosmetics and a bit more internal documentation
+      * anticosmetics: mangled names in macros to evade debugger strangeness
+      * tested on sparc, hp-700, dec-mips, rs6000
+          with gcc & native cc (hp, dec only) allowing
+          Detlefs & Zorn comparison study (in SIGPLAN Notices.)
+
+    Trial version Fri Aug 28 13:14:29 1992  Doug Lea  (dl at g.oswego.edu)
+      * Based loosely on libg++-1.2X malloc. (It retains some of the overall
+         structure of old version,  but most details differ.)
+
+*/
diff --git a/cpp/src/plasma/thirdparty/xxhash.cc b/cpp/src/plasma/thirdparty/xxhash.cc
new file mode 100644
index 0000000000000..f74880b0de71d
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/xxhash.cc
@@ -0,0 +1,889 @@
+/*
+*  xxHash - Fast Hash algorithm
+*  Copyright (C) 2012-2016, Yann Collet
+*
+*  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+*
+*  Redistribution and use in source and binary forms, with or without
+*  modification, are permitted provided that the following conditions are
+*  met:
+*
+*  * Redistributions of source code must retain the above copyright
+*  notice, this list of conditions and the following disclaimer.
+*  * Redistributions in binary form must reproduce the above
+*  copyright notice, this list of conditions and the following disclaimer
+*  in the documentation and/or other materials provided with the
+*  distribution.
+*
+*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+*  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+*  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+*  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+*  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+*  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+*  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+*  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+*  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+*  You can contact the author at :
+*  - xxHash homepage: http://www.xxhash.com
+*  - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+
+/* *************************************
+*  Tuning parameters
+***************************************/
+/*!XXH_FORCE_MEMORY_ACCESS :
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
+ *            It can generate buggy code on targets which do not support unaligned memory accesses.
+ *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See http://stackoverflow.com/a/32095106/646947 for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
+ */
+#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#    define XXH_FORCE_MEMORY_ACCESS 2
+#  elif defined(__INTEL_COMPILER) || \
+  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+#    define XXH_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+/*!XXH_ACCEPT_NULL_INPUT_POINTER :
+ * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
+ * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
+ * By default, this option is disabled. To enable it, uncomment below define :
+ */
+/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
+
+/*!XXH_FORCE_NATIVE_FORMAT :
+ * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
+ * Results are therefore identical for little-endian and big-endian CPU.
+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+ * Should endian-independence be of no importance for your application, you may set the #define below to 1,
+ * to improve speed for Big-endian CPU.
+ * This option has no impact on Little_Endian CPU.
+ */
+#ifndef XXH_FORCE_NATIVE_FORMAT   /* can be defined externally */
+#  define XXH_FORCE_NATIVE_FORMAT 0
+#endif
+
+/*!XXH_FORCE_ALIGN_CHECK :
+ * This is a minor performance trick, only useful with lots of very small keys.
+ * It means : check for aligned/unaligned input.
+ * The check costs one initial branch per hash; set to 0 when the input data
+ * is guaranteed to be aligned.
+ */
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+#  if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+#    define XXH_FORCE_ALIGN_CHECK 0
+#  else
+#    define XXH_FORCE_ALIGN_CHECK 1
+#  endif
+#endif
+
+
+/* *************************************
+*  Includes & Memory related functions
+***************************************/
+/*! Modify the local functions below should you wish to use some other memory routines
+*   for malloc(), free() */
+#include <stdlib.h>
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void  XXH_free  (void* p)  { free(p); }
+/*! and for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
+
+
+/* *************************************
+*  Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
+#  define FORCE_INLINE static __forceinline
+#else
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
+#endif
+
+
+/* *************************************
+*  Basic Types
+***************************************/
+#ifndef MEM_MODULE
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+    typedef uint8_t  BYTE;
+    typedef uint16_t U16;
+    typedef uint32_t U32;
+    typedef  int32_t S32;
+# else
+    typedef unsigned char      BYTE;
+    typedef unsigned short     U16;
+    typedef unsigned int       U32;
+    typedef   signed int       S32;
+# endif
+#endif
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; } __attribute__((packed)) unalign;
+static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+static U32 XXH_read32(const void* memPtr)
+{
+    U32 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* ****************************************
+*  Compiler-specific Functions and Macros
+******************************************/
+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
+#if defined(_MSC_VER)
+#  define XXH_rotl32(x,r) _rotl(x,r)
+#  define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+#  define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#  define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
+#endif
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap32 _byteswap_ulong
+#elif XXH_GCC_VERSION >= 403
+#  define XXH_swap32 __builtin_bswap32
+#else
+static U32 XXH_swap32 (U32 x)
+{
+    return  ((x << 24) & 0xff000000 ) |
+            ((x <<  8) & 0x00ff0000 ) |
+            ((x >>  8) & 0x0000ff00 ) |
+            ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* *************************************
+*  Architecture Macros
+***************************************/
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+
+/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+    static const int g_one = 1;
+#   define XXH_CPU_LITTLE_ENDIAN   (*(const char*)(&g_one))
+#endif
+
+
+/* ***************************
+*  Memory reads
+*****************************/
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
+}
+
+FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE32_align(ptr, endian, XXH_unaligned);
+}
+
+static U32 XXH_readBE32(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+
+
+/* *************************************
+*  Macros
+***************************************/
+#define XXH_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(int)(!!(c)) }; }    /* use only *after* variable declarations */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+*  32-bits hash functions
+*********************************************************************/
+static const U32 PRIME32_1 = 2654435761U;
+static const U32 PRIME32_2 = 2246822519U;
+static const U32 PRIME32_3 = 3266489917U;
+static const U32 PRIME32_4 =  668265263U;
+static const U32 PRIME32_5 =  374761393U;
+
+static U32 XXH32_round(U32 seed, U32 input)
+{
+    seed += input * PRIME32_2;
+    seed  = XXH_rotl32(seed, 13);
+    seed *= PRIME32_1;
+    return seed;
+}
+
+FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U32 h32;
+#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (p==NULL) {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)16;
+    }
+#endif
+
+    if (len>=16) {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = seed + PRIME32_1 + PRIME32_2;
+        U32 v2 = seed + PRIME32_2;
+        U32 v3 = seed + 0;
+        U32 v4 = seed - PRIME32_1;
+
+        do {
+            v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
+            v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
+            v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
+            v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
+        } while (p<=limit);
+
+        h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+    } else {
+        h32  = seed + PRIME32_5;
+    }
+
+    h32 += (U32) len;
+
+    while (p+4<=bEnd) {
+        h32 += XXH_get32bits(p) * PRIME32_3;
+        h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
+        p+=4;
+    }
+
+    while (p<bEnd) {
+        h32 += (*p) * PRIME32_5;
+        h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+        p++;
+    }
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH32_state_t state;
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, input, len);
+    return XXH32_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
+            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+                return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+            else
+                return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }   }
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+
+
+/*======   Hash streaming   ======*/
+
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
+{
+    XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state)-4);   /* do not write into reserved, for future removal */
+    state.v1 = seed + PRIME32_1 + PRIME32_2;
+    state.v2 = seed + PRIME32_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - PRIME32_1;
+    memcpy(statePtr, &state, sizeof(state));
+    return XXH_OK;
+}
+
+
+FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (input==NULL) return XXH_ERROR;
+#endif
+
+    state->total_len_32 += (unsigned)len;
+    state->large_len |= (len>=16) | (state->total_len_32>=16);
+
+    if (state->memsize + len < 16)  {   /* fill in tmp buffer */
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
+        state->memsize += (unsigned)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize) {   /* some data left from previous update */
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
+        {   const U32* p32 = state->mem32;
+            state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
+            state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
+            state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
+            state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
+        }
+        p += 16-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p <= bEnd-16) {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = state->v1;
+        U32 v2 = state->v2;
+        U32 v3 = state->v3;
+        U32 v4 = state->v4;
+
+        do {
+            v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
+            v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
+            v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
+            v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
+        } while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd) {
+        XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
+{
+    const BYTE * p = (const BYTE*)state->mem32;
+    const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
+    U32 h32;
+
+    if (state->large_len) {
+        h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
+    } else {
+        h32 = state->v3 /* == seed */ + PRIME32_5;
+    }
+
+    h32 += state->total_len_32;
+
+    while (p+4<=bEnd) {
+        h32 += XXH_readLE32(p, endian) * PRIME32_3;
+        h32  = XXH_rotl32(h32, 17) * PRIME32_4;
+        p+=4;
+    }
+
+    while (p<bEnd) {
+        h32 += (*p) * PRIME32_5;
+        h32  = XXH_rotl32(h32, 11) * PRIME32_1;
+        p++;
+    }
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH32_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*======   Canonical representation   ======*/
+
+/*! Default XXH result types are basic unsigned 32 and 64 bits.
+*   The canonical representation follows human-readable write convention, aka big-endian (large digits first).
+*   These functions allow transformation of hash result into and from its canonical format.
+*   This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
+*/
+
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+    return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+*  64-bits hash functions
+*********************************************************************/
+
+/*======   Memory access   ======*/
+
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+    typedef uint64_t U64;
+# else
+    typedef unsigned long long U64;   /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
+# endif
+#endif
+
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
+static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+
+static U64 XXH_read64(const void* memPtr)
+{
+    U64 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap64 _byteswap_uint64
+#elif XXH_GCC_VERSION >= 403
+#  define XXH_swap64 __builtin_bswap64
+#else
+static U64 XXH_swap64 (U64 x)
+{
+    return  ((x << 56) & 0xff00000000000000ULL) |
+            ((x << 40) & 0x00ff000000000000ULL) |
+            ((x << 24) & 0x0000ff0000000000ULL) |
+            ((x << 8)  & 0x000000ff00000000ULL) |
+            ((x >> 8)  & 0x00000000ff000000ULL) |
+            ((x >> 24) & 0x0000000000ff0000ULL) |
+            ((x >> 40) & 0x000000000000ff00ULL) |
+            ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
+}
+
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+}
+
+static U64 XXH_readBE64(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+
+
+/*======   xxh64   ======*/
+
+static const U64 PRIME64_1 = 11400714785074694791ULL;
+static const U64 PRIME64_2 = 14029467366897019727ULL;
+static const U64 PRIME64_3 =  1609587929392839161ULL;
+static const U64 PRIME64_4 =  9650029242287828579ULL;
+static const U64 PRIME64_5 =  2870177450012600261ULL;
+
+static U64 XXH64_round(U64 acc, U64 input)
+{
+    acc += input * PRIME64_2;
+    acc  = XXH_rotl64(acc, 31);
+    acc *= PRIME64_1;
+    return acc;
+}
+
+static U64 XXH64_mergeRound(U64 acc, U64 val)
+{
+    val  = XXH64_round(0, val);
+    acc ^= val;
+    acc  = acc * PRIME64_1 + PRIME64_4;
+    return acc;
+}
+
+FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U64 h64;
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (p==NULL) {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)32;
+    }
+#endif
+
+    if (len>=32) {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = seed + PRIME64_1 + PRIME64_2;
+        U64 v2 = seed + PRIME64_2;
+        U64 v3 = seed + 0;
+        U64 v4 = seed - PRIME64_1;
+
+        do {
+            v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
+            v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
+            v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
+            v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
+        } while (p<=limit);
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+
+    } else {
+        h64  = seed + PRIME64_5;
+    }
+
+    h64 += (U64) len;
+
+    while (p+8<=bEnd) {
+        U64 const k1 = XXH64_round(0, XXH_get64bits(p));
+        h64 ^= k1;
+        h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+        p+=8;
+    }
+
+    if (p+4<=bEnd) {
+        h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
+        h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+        p+=4;
+    }
+
+    while (p<bEnd) {
+        h64 ^= (*p) * PRIME64_5;
+        h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+        p++;
+    }
+
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+
+    return h64;
+}
+
+
+XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH64_state_t state;
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, input, len);
+    return XXH64_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
+            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+                return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+            else
+                return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }   }
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+/*======   Hash Streaming   ======*/
+
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
+{
+    XXH64_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state)-8);   /* do not write into reserved, for future removal */
+    state.v1 = seed + PRIME64_1 + PRIME64_2;
+    state.v2 = seed + PRIME64_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - PRIME64_1;
+    memcpy(statePtr, &state, sizeof(state));
+    return XXH_OK;
+}
+
+FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (input==NULL) return XXH_ERROR;
+#endif
+
+    state->total_len += len;
+
+    if (state->memsize + len < 32) {  /* fill in tmp buffer */
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
+        state->memsize += (U32)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize) {   /* tmp buffer is full */
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
+        state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
+        state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
+        state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
+        state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
+        p += 32-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p+32 <= bEnd) {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = state->v1;
+        U64 v2 = state->v2;
+        U64 v3 = state->v3;
+        U64 v4 = state->v4;
+
+        do {
+            v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
+            v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
+            v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
+            v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
+        } while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd) {
+        XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
+{
+    const BYTE * p = (const BYTE*)state->mem64;
+    const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
+    U64 h64;
+
+    if (state->total_len >= 32) {
+        U64 const v1 = state->v1;
+        U64 const v2 = state->v2;
+        U64 const v3 = state->v3;
+        U64 const v4 = state->v4;
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+    } else {
+        h64  = state->v3 + PRIME64_5;
+    }
+
+    h64 += (U64) state->total_len;
+
+    while (p+8<=bEnd) {
+        U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
+        h64 ^= k1;
+        h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+        p+=8;
+    }
+
+    if (p+4<=bEnd) {
+        h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
+        h64  = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+        p+=4;
+    }
+
+    while (p<bEnd) {
+        h64 ^= (*p) * PRIME64_5;
+        h64  = XXH_rotl64(h64, 11) * PRIME64_1;
+        p++;
+    }
+
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+
+    return h64;
+}
+
+XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH64_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*====== Canonical representation   ======*/
+
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+    return XXH_readBE64(src);
+}
+
+#endif  /* XXH_NO_LONG_LONG */
diff --git a/cpp/src/plasma/thirdparty/xxhash.h b/cpp/src/plasma/thirdparty/xxhash.h
new file mode 100644
index 0000000000000..9d831e03b35f6
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/xxhash.h
@@ -0,0 +1,293 @@
+/*
+   xxHash - Extremely Fast Hash algorithm
+   Header File
+   Copyright (C) 2012-2016, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name            Speed       Q.Score   Author
+xxHash          5.4 GB/s     10
+CrapWow         3.2 GB/s      2       Andrew
+MumurHash 3a    2.7 GB/s     10       Austin Appleby
+SpookyHash      2.0 GB/s     10       Bob Jenkins
+SBox            1.4 GB/s      9       Bret Mulvey
+Lookup3         1.2 GB/s      9       Bob Jenkins
+SuperFastHash   1.2 GB/s      1       Paul Hsieh
+CityHash64      1.05 GB/s    10       Pike & Alakuijala
+FNV             0.55 GB/s     5       Fowler, Noll, Vo
+CRC32           0.43 GB/s     9
+MD5-32          0.33 GB/s    10       Ronald L. Rivest
+SHA1-32         0.28 GB/s    10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+A 64-bits version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bits applications only.
+Name     Speed on 64 bits    Speed on 32 bits
+XXH64       13.8 GB/s            1.9 GB/s
+XXH32        6.8 GB/s            6.0 GB/s
+*/
+
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************
+*  Definitions
+******************************/
+#include <stddef.h>   /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/* ****************************
+*  API modifier
+******************************/
+/** XXH_PRIVATE_API
+*   This is useful to include xxhash functions in `static` mode
+*   in order to inline them, and remove their symbol from the public list.
+*   Methodology :
+*     #define XXH_PRIVATE_API
+*     #include "xxhash.h"
+*   `xxhash.c` is automatically included.
+*   It's not useful to compile and link it as a separate module.
+*/
+#ifdef XXH_PRIVATE_API
+#  ifndef XXH_STATIC_LINKING_ONLY
+#    define XXH_STATIC_LINKING_ONLY
+#  endif
+#  if defined(__GNUC__)
+#    define XXH_PUBLIC_API static __inline __attribute__((unused))
+#  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#    define XXH_PUBLIC_API static inline
+#  elif defined(_MSC_VER)
+#    define XXH_PUBLIC_API static __inline
+#  else
+#    define XXH_PUBLIC_API static   /* this version may generate warnings for unused static functions; disable the relevant warning */
+#  endif
+#else
+#  define XXH_PUBLIC_API   /* do nothing */
+#endif /* XXH_PRIVATE_API */
+
+/*!XXH_NAMESPACE, aka Namespace Emulation :
+
+If you want to include _and expose_ xxHash functions from within your own library,
+but also want to avoid symbol collisions with other libraries which may also include xxHash,
+
+you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
+with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
+
+Note that no change is required within the calling program as long as it includes `xxhash.h` :
+regular symbol name will be automatically translated by this header.
+*/
+#ifdef XXH_NAMESPACE
+#  define XXH_CAT(A,B) A##B
+#  define XXH_NAME2(A,B) XXH_CAT(A,B)
+#  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+#endif
+
+
+/* *************************************
+*  Version
+***************************************/
+#define XXH_VERSION_MAJOR    0
+#define XXH_VERSION_MINOR    6
+#define XXH_VERSION_RELEASE  2
+#define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+XXH_PUBLIC_API unsigned XXH_versionNumber (void);
+
+
+/*-**********************************************************************
+*  32-bits hash
+************************************************************************/
+typedef unsigned int XXH32_hash_t;
+
+/*! XXH32() :
+    Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
+    The memory between input & input+length must be valid (allocated and read-accessible).
+    "seed" can be used to alter the result predictably.
+    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
+
+/*======   Streaming   ======*/
+typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
+XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned int seed);
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH32_hash_t  XXH32_digest (const XXH32_state_t* statePtr);
+
+/*
+These functions generate the xxHash of an input provided in multiple segments.
+Note that, for small input, they are slower than single-call functions, due to state management.
+For small input, prefer `XXH32()` and `XXH64()` .
+
+XXH state must first be allocated, using XXH*_createState() .
+
+Start a new hash by initializing state with a seed, using XXH*_reset().
+
+Then, feed the hash state by calling XXH*_update() as many times as necessary.
+Obviously, input must be allocated and read accessible.
+The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+
+Finally, a hash value can be produced anytime, by using XXH*_digest().
+This function returns the nn-bits hash as an int or long long.
+
+It's still possible to continue inserting input into the hash state after a digest,
+and generate some new hashes later on, by calling again XXH*_digest().
+
+When done, free XXH state space if it was allocated dynamically.
+*/
+
+/*======   Canonical representation   ======*/
+
+typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+*  These functions allow transformation of hash result into and from its canonical format.
+*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+*/
+
+
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+*  64-bits hash
+************************************************************************/
+typedef unsigned long long XXH64_hash_t;
+
+/*! XXH64() :
+    Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
+    "seed" can be used to alter the result predictably.
+    This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
+*/
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
+
+/*======   Streaming   ======*/
+typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
+XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
+
+/*======   Canonical representation   ======*/
+typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+#endif  /* XXH_NO_LONG_LONG */
+
+
+#ifdef XXH_STATIC_LINKING_ONLY
+
+/* ================================================================================================
+   This section contains definitions which are not guaranteed to remain stable.
+   They may change in future versions, becoming incompatible with a different version of the library.
+   They shall only be used with static linking.
+   Never use these definitions in association with dynamic linking !
+=================================================================================================== */
+
+/* These definitions are only meant to make possible
+   static allocation of XXH state, on stack or in a struct for example.
+   Never use members directly. */
+
+struct XXH32_state_s {
+   unsigned total_len_32;
+   unsigned large_len;
+   unsigned v1;
+   unsigned v2;
+   unsigned v3;
+   unsigned v4;
+   unsigned mem32[4];   /* buffer defined as U32 for alignment */
+   unsigned memsize;
+   unsigned reserved;   /* never read nor write, will be removed in a future version */
+};   /* typedef'd to XXH32_state_t */
+
+#ifndef XXH_NO_LONG_LONG   /* remove 64-bits support */
+struct XXH64_state_s {
+   unsigned long long total_len;
+   unsigned long long v1;
+   unsigned long long v2;
+   unsigned long long v3;
+   unsigned long long v4;
+   unsigned long long mem64[4];   /* buffer defined as U64 for alignment */
+   unsigned memsize;
+   unsigned reserved[2];          /* never read nor write, will be removed in a future version */
+};   /* typedef'd to XXH64_state_t */
+#endif
+
+#ifdef XXH_PRIVATE_API
+#  include "xxhash.c"   /* include xxhash function bodies as `static`, for inlining */
+#endif
+
+#endif /* XXH_STATIC_LINKING_ONLY */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* XXHASH_H_5627135585666179 */

From fabf7fb6ae80f7d4dc324bf562e35ad6ee55fe31 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sun, 23 Jul 2017 22:32:47 -0400
Subject: [PATCH 0840/1644] =?UTF-8?q?ARROW-1241:=20[C++]=20Appveyor=20buil?=
 =?UTF-8?q?d=20matrix=20extended=20with=20Visual=20Studio=202=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…017 environment build job

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #870 from MaxRis/ARROW-1241 and squashes the following commits:

a3ba6b0 [Max Risuhin] ARROW-1241: [C++] Appveyor build matrix extended with Visual Studio 2017 environment build job
---
 appveyor.yml                     | 7 +++++++
 cpp/cmake_modules/SnappyConfig.h | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index c58e1dab7d8fd..91e9ee2649073 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -45,6 +45,13 @@ environment:
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Release"
+    - JOB: "Build"
+      GENERATOR: Visual Studio 15 2017 Win64
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Release"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      BOOST_ROOT: C:\Libraries\boost_1_64_0
 
   MSVC_DEFAULT_OPTIONS: ON
   BOOST_ROOT: C:\Libraries\boost_1_63_0
diff --git a/cpp/cmake_modules/SnappyConfig.h b/cpp/cmake_modules/SnappyConfig.h
index 74eb77621626b..c998d1813aa7e 100644
--- a/cpp/cmake_modules/SnappyConfig.h
+++ b/cpp/cmake_modules/SnappyConfig.h
@@ -29,7 +29,7 @@
 #ifndef SNAPPY_CONFIG_H
 #define SNAPPY_CONFIG_H 1
 
-#if defined(_MSC_VER) && (_MSC_VER <= 1900)
+#if defined(_MSC_VER) && (_MSC_VER <= 1910)
 typedef __int64 ssize_t;
 #endif
 

From e1b098e9303b240a07a241bb43d88f787f3875cb Mon Sep 17 00:00:00 2001
From: Matt Darwin <(none)>
Date: Mon, 24 Jul 2017 10:55:33 -0400
Subject: [PATCH 0841/1644] ARROW-1240: [JAVA] security: upgrade slf4j to
 1.7.25 and logback to 1.2.3

This PR supercedes #871 (repo fork for that one was deleted)

Author: Matt Darwin <(none)>

Closes #879 from mattdarwin/ARROW-1240-upgrade-logback and squashes the following commits:

caed163 [Matt Darwin] upgrading slf4j to 1.7.25
---
 java/pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/java/pom.xml b/java/pom.xml
index 2613a44104576..0d4e6128b4738 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -30,7 +30,7 @@
   <properties>
     <target.gen.source.path>${project.basedir}/target/generated-sources</target.gen.source.path>
     <dep.junit.version>4.11</dep.junit.version>
-    <dep.slf4j.version>1.7.6</dep.slf4j.version>
+    <dep.slf4j.version>1.7.25</dep.slf4j.version>
     <dep.guava.version>18.0</dep.guava.version>
     <forkCount>2</forkCount>
     <jackson.version>2.7.1</jackson.version>
@@ -520,7 +520,7 @@
     <dependency>
       <groupId>ch.qos.logback</groupId>
       <artifactId>logback-classic</artifactId>
-      <version>1.0.13</version>
+      <version>1.2.3</version>
       <scope>test</scope>
     </dependency>
     <dependency>

From 457bb071df95e71eef45159619cc6b531fdec08f Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 24 Jul 2017 08:13:24 -0700
Subject: [PATCH 0842/1644] ARROW-1237: [JAVA] expose the ability to set
 lastSet

The changes here expose the ability to set "lastSet" on Nullable<var length>Vector.
I believe this is needed only for NullableVarCharVector and NullableVarBinaryVector. Hence the API is exposed through NullableValueVectors.java

Author: siddharth <siddharth@dremio.com>

Closes #868 from siddharthteotia/ARROW-1237 and squashes the following commits:

786dfea [siddharth] ARROW-1237: addressed review comments and added more tests
73b2fc5 [siddharth] ARROW-1237: added some unit tests
f8c7277 [siddharth] ARROW-1237: expose the ability to set lastSet
---
 .gitignore                                    |   3 +-
 .../templates/NullableValueVectors.java       |  16 ++
 .../arrow/vector/complex/ListVector.java      |   6 +
 .../apache/arrow/vector/TestListVector.java   | 154 ++++++++++++++++++
 .../apache/arrow/vector/TestValueVector.java  | 132 ++++++++++++++-
 5 files changed, 309 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index dd69b6cec9c5f..e6dfe19bb9807 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,4 +27,5 @@ MANIFEST
 
 cpp/.idea/
 python/.eggs/
-.vscode
\ No newline at end of file
+.vscode
+.idea/
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 092097bb2bd6d..1decd0b313802 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -699,6 +699,22 @@ public void reset(){
       setCount = 0;
       <#if type.major = "VarLen">lastSet = -1;</#if>
     }
+
+    public void setLastSet(int value) {
+      <#if type.major = "VarLen">
+        lastSet = value;
+      <#else>
+        throw new UnsupportedOperationException();
+      </#if>
+    }
+
+    public int getLastSet() {
+      <#if type.major != "VarLen">
+        throw new UnsupportedOperationException();
+      <#else>
+        return lastSet;
+      </#if>
+    }
   }
 }
 </#list>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 4ab624f3694cb..6357294566017 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -393,6 +393,12 @@ public void setValueCount(int valueCount) {
       vector.getMutator().setValueCount(childValueCount);
       bits.getMutator().setValueCount(valueCount);
     }
+
+    public void setLastSet(int value) {
+      lastSet = value;
+    }
+
+    public int getLastSet() { return lastSet; }
   }
 
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 11be3298f7533..29ea7628f452b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -17,15 +17,26 @@
  */
 package org.apache.arrow.vector;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.holders.NullableBigIntHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.*;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.List;
+
 public class TestListVector {
 
   private BufferAllocator allocator;
@@ -80,4 +91,147 @@ public void testCopyFrom() throws Exception {
       Assert.assertTrue("shouldn't be null", reader.isSet());
     }
   }
+
+  @Test
+  public void testSetLastSetUsage() throws Exception {
+    try (ListVector listVector = ListVector.empty("input", allocator)) {
+
+      /* Explicitly add the dataVector */
+      MinorType type = MinorType.BIGINT;
+      listVector.addOrGetVector(FieldType.nullable(type.getType()));
+
+      /* allocate memory */
+      listVector.allocateNew();
+
+      /* get inner vectors; bitVector and offsetVector */
+      List<BufferBacked> innerVectors = listVector.getFieldInnerVectors();
+      BitVector bitVector = (BitVector)innerVectors.get(0);
+      UInt4Vector offsetVector = (UInt4Vector)innerVectors.get(1);
+
+      /* get the underlying data vector -- NullableBigIntVector */
+      NullableBigIntVector dataVector = (NullableBigIntVector)listVector.getDataVector();
+
+      /* check current lastSet */
+      assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet()));
+
+      int index = 0;
+      int offset = 0;
+
+      /* write [10, 11, 12] to the list vector at index */
+      bitVector.getMutator().setSafe(index, 1);
+      dataVector.getMutator().setSafe(0, 1, 10);
+      dataVector.getMutator().setSafe(1, 1, 11);
+      dataVector.getMutator().setSafe(2, 1, 12);
+      offsetVector.getMutator().setSafe(index + 1, 3);
+
+      index  += 1;
+
+      /* write [13, 14] to the list vector at index 1 */
+      bitVector.getMutator().setSafe(index, 1);
+      dataVector.getMutator().setSafe(3, 1, 13);
+      dataVector.getMutator().setSafe(4, 1, 14);
+      offsetVector.getMutator().setSafe(index + 1, 5);
+
+      index += 1;
+
+      /* write [15, 16, 17] to the list vector at index 2 */
+      bitVector.getMutator().setSafe(index, 1);
+      dataVector.getMutator().setSafe(5, 1, 15);
+      dataVector.getMutator().setSafe(6, 1, 16);
+      dataVector.getMutator().setSafe(7, 1, 17);
+      offsetVector.getMutator().setSafe(index + 1, 8);
+
+      /* check current lastSet */
+      assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet()));
+
+      /* set lastset and arbitrary valuecount for list vector.
+       *
+       * NOTE: if we don't execute setLastSet() before setLastValueCount(), then
+       * the latter will corrupt the offsetVector and thus the accessor will not
+       * retrieve the correct values from underlying dataVector. Run the test
+       * by commenting out next line and we should see failures from 5th assert
+       * onwards. This is why doing setLastSet() is important before setValueCount()
+       * once the vector has been loaded.
+       *
+       * Another important thing to remember is the value of lastSet itself.
+       * Even though the listVector has elements till index 2 only, the lastSet should
+       * be set as 3. This is because the offsetVector has valid offsets filled till index 3.
+       * If we do setLastSet(2), the offsetVector at index 3 will contain incorrect value
+       * after execution of setValueCount().
+       *
+       * correct state of the listVector
+       * bitvector    {1, 1, 1, 0, 0.... }
+       * offsetvector {0, 3, 5, 8, 8, 8.....}
+       * datavector   { [10, 11, 12],
+       *                [13, 14],
+       *                [15, 16, 17]
+       *              }
+       *
+       * if we don't do setLastSet() before setValueCount --> incorrect state
+       * bitvector    {1, 1, 1, 0, 0.... }
+       * offsetvector {0, 0, 0, 0, 0, 0.....}
+       * datavector   { [10, 11, 12],
+       *                [13, 14],
+       *                [15, 16, 17]
+       *              }
+       *
+       * if we do setLastSet(2) before setValueCount --> incorrect state
+       * bitvector    {1, 1, 1, 0, 0.... }
+       * offsetvector {0, 3, 5, 5, 5, 5.....}
+       * datavector   { [10, 11, 12],
+       *                [13, 14],
+       *                [15, 16, 17]
+       *              }
+       */
+      listVector.getMutator().setLastSet(3);
+      listVector.getMutator().setValueCount(10);
+
+      /* check the vector output */
+      final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
+      final ValueVector.Accessor valueAccessor = dataVector.getAccessor();
+
+      index  = 0;
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(0), Integer.toString(offset));
+
+      Object actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(10), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(11), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(12), (Long)actual);
+
+      index++;
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(3), Integer.toString(offset));
+
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(13), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(14), (Long)actual);
+
+      index++;
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(5), Integer.toString(offset));
+
+      actual = valueAccessor.getObject(offsetAccessor.get(index));
+      assertEquals(new Long(15), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(16), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(17), (Long)actual);
+
+      index++;
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(8), Integer.toString(offset));
+
+      actual = valueAccessor.getObject(offsetAccessor.get(index));
+      assertNull(actual);
+    }
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 63543b0932908..f5508aab2ce1d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -26,11 +26,15 @@
 
 import java.nio.charset.Charset;
 import java.util.List;
+import java.util.ArrayList;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+
+import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.schema.TypeLayout;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
@@ -56,6 +60,9 @@ public void init() {
   private final static byte[] STR1 = "AAAAA1".getBytes(utf8Charset);
   private final static byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset);
   private final static byte[] STR3 = "CCCC3".getBytes(utf8Charset);
+  private final static byte[] STR4 = "DDDDDDDD4".getBytes(utf8Charset);
+  private final static byte[] STR5 = "EEE5".getBytes(utf8Charset);
+  private final static byte[] STR6 = "FFFFF6".getBytes(utf8Charset);
 
   @After
   public void terminate() throws Exception {
@@ -509,11 +516,134 @@ public void testCopyFromWithNulls() {
         } else {
           assertEquals(Integer.toString(i), vector2.getAccessor().getObject(i).toString());
         }
-
       }
+    }
+  }
+
+  @Test
+  public void testSetLastSetUsage() {
+    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
 
+      final NullableVarCharVector.Mutator mutator = vector.getMutator();
+
+      vector.allocateNew(1024 * 10, 1024);
+
+      setBytes(0, STR1, vector);
+      setBytes(1, STR2, vector);
+      setBytes(2, STR3, vector);
+      setBytes(3, STR4, vector);
+      setBytes(4, STR5, vector);
+      setBytes(5, STR6, vector);
+
+      /* Check current lastSet */
+      assertEquals(Integer.toString(-1), Integer.toString(mutator.getLastSet()));
+
+      /* Check the vector output */
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(STR4, accessor.get(3));
+      assertArrayEquals(STR5, accessor.get(4));
+      assertArrayEquals(STR6, accessor.get(5));
+
+      /*
+       * If we don't do setLastSe(5) before setValueCount(), then the latter will corrupt
+       * the value vector by filling in all positions [0,valuecount-1] will empty byte arrays.
+       * Run the test by commenting out next line and we should see incorrect vector output.
+       */
+      mutator.setLastSet(5);
+      mutator.setValueCount(20);
+
+      /* Check the vector output again */
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(STR4, accessor.get(3));
+      assertArrayEquals(STR5, accessor.get(4));
+      assertArrayEquals(STR6, accessor.get(5));
+    }
+  }
 
+  @Test
+  public void testVectorLoadUnload() {
+
+    try (final NullableVarCharVector vector1 = new NullableVarCharVector("myvector", allocator)) {
+
+      final NullableVarCharVector.Mutator mutator1 = vector1.getMutator();
+
+      vector1.allocateNew(1024 * 10, 1024);
+
+      mutator1.set(0, STR1);
+      mutator1.set(1, STR2);
+      mutator1.set(2, STR3);
+      mutator1.set(3, STR4);
+      mutator1.set(4, STR5);
+      mutator1.set(5, STR6);
+      assertEquals(Integer.toString(5), Integer.toString(mutator1.getLastSet()));
+      mutator1.setValueCount(15);
+      assertEquals(Integer.toString(14), Integer.toString(mutator1.getLastSet()));
+
+      /* Check the vector output */
+      final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
+      assertArrayEquals(STR1, accessor1.get(0));
+      assertArrayEquals(STR2, accessor1.get(1));
+      assertArrayEquals(STR3, accessor1.get(2));
+      assertArrayEquals(STR4, accessor1.get(3));
+      assertArrayEquals(STR5, accessor1.get(4));
+      assertArrayEquals(STR6, accessor1.get(5));
+
+      Field field = vector1.getField();
+      String fieldName = field.getName();
+
+      List<Field> fields = new ArrayList<Field>();
+      List<FieldVector> fieldVectors = new ArrayList<FieldVector>();
+
+      fields.add(field);
+      fieldVectors.add(vector1);
+
+      Schema schema = new Schema(fields);
+
+      VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, accessor1.getValueCount());
+      VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1);
+
+      try (
+              ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+              BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("new vector", 0, Long.MAX_VALUE);
+              VectorSchemaRoot schemaRoot2 = VectorSchemaRoot.create(schema, finalVectorsAllocator);
+      ) {
+
+        VectorLoader vectorLoader = new VectorLoader(schemaRoot2);
+        vectorLoader.load(recordBatch);
+
+        NullableVarCharVector vector2 = (NullableVarCharVector)schemaRoot2.getVector(fieldName);
+        NullableVarCharVector.Mutator mutator2 = vector2.getMutator();
+
+        /*
+         * lastSet would have internally been set by VectorLoader.load() when it invokes
+         * loadFieldBuffers.
+         */
+        assertEquals(Integer.toString(14), Integer.toString(mutator2.getLastSet()));
+        mutator2.setValueCount(25);
+        assertEquals(Integer.toString(24), Integer.toString(mutator2.getLastSet()));
+
+        /* Check the vector output */
+        final NullableVarCharVector.Accessor accessor2 = vector2.getAccessor();
+        assertArrayEquals(STR1, accessor2.get(0));
+        assertArrayEquals(STR2, accessor2.get(1));
+        assertArrayEquals(STR3, accessor2.get(2));
+        assertArrayEquals(STR4, accessor2.get(3));
+        assertArrayEquals(STR5, accessor2.get(4));
+        assertArrayEquals(STR6, accessor2.get(5));
+      }
     }
   }
 
+  public static void setBytes(int index, byte[] bytes, NullableVarCharVector vector) {
+    final int currentOffset = vector.values.offsetVector.getAccessor().get(index);
+
+    vector.bits.getMutator().setToOne(index);
+    vector.values.offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
+    vector.values.data.setBytes(currentOffset, bytes, 0, bytes.length);
+  }
 }

From 05f7058ce500278d52118c0f0e8320e93a18cf53 Mon Sep 17 00:00:00 2001
From: Antony Mayi <antonymayi@yahoo.com>
Date: Mon, 24 Jul 2017 08:22:38 -0700
Subject: [PATCH 0843/1644] ARROW-1239: [JAVA] upgrading git-commit-id-plugin

Currently used version of the git-commit-id-plugin maven plugin (2.1.9) doesn't work with recent git structures. This is for majority of the users not manifested since Arrow has the java maven root in the project subdirectory  (`/java`) instead of top level so this plugin normally doesn't kick in if maven is executed from the subdirectory (usual case - ie `cd java; mvn install` - works fine) as the plugin doesn't see the `.git` directory but it does kick in and fail if executed from the main arrow top level dir as `mvn -f java/pom.xml install` (where the `.git` sits):

```
$ mvn -f java/pom.xml package
...
[ERROR] Failed to execute goal pl.project13.maven:git-commit-id-plugin:2.1.9:revision (for-jars) on project arrow-java-root: Execution for-jars of goal pl.project13.maven:git-commit-id-plugin:2.1.9:revision failed: Bare Repository has neither a working tree, nor an index -> [Help 1]
```

Simple fix is upgrading the plugin to recent version (the minimal working version appears to be 2.1.13).

This is required for seamless integration with Jenkins (ARROW-1234).

Author: Antony Mayi <antonymayi@yahoo.com>

Closes #869 from antonymayi/master and squashes the following commits:

6976ee0 [Antony Mayi] ARROW-1239 - upgrading git-commit-id-plugin
---
 java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/pom.xml b/java/pom.xml
index 0d4e6128b4738..81f80b00b563c 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -231,7 +231,7 @@
       <plugin>
         <groupId>pl.project13.maven</groupId>
         <artifactId>git-commit-id-plugin</artifactId>
-        <version>2.1.9</version>
+        <version>2.2.2</version>
         <executions>
           <execution>
             <id>for-jars</id>

From a94f4716be8c33e86222d5a0be5a4d2a9102b93d Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 24 Jul 2017 12:12:42 -0400
Subject: [PATCH 0844/1644] ARROW-1149: [Plasma] Create Cython client library
 for Plasma

This PR introduces a Cython API to Plasma, a FindPlasma.cmake to make it easier to integrate Plasma with CMake projects and sets up packaging with pyarrow.

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #797 from pcmoritz/plasma-cython and squashes the following commits:

d8319fc [Philipp Moritz] get for of PlasmaClient.connect
d14ab87 [Philipp Moritz] get rid of MutableBuffer
08f24a5 [Philipp Moritz] fix typos and move FixedSizeBufferOutputStream
e33443d [Philipp Moritz] fix setup.py develop for plasma
5f7b779 [Philipp Moritz] changes needed to make Ray work with Plasma in Arrow
b9e2dee [Philipp Moritz] fix windows build
3e4a84d [Philipp Moritz] fix segfault
0bea267 [Philipp Moritz] debug
23fe5f5 [Philipp Moritz] make plasma store binary part of the pyarrow package for tests
b863d13 [Philipp Moritz] fix
997de1e [Philipp Moritz] fix
47dc739 [Philipp Moritz] fixes
47033e7 [Philipp Moritz] switch to pytest
ed84c53 [Philipp Moritz] partial fixes
9bc5c15 [Philipp Moritz] implement wait and fetch for the client
45f338f [Philipp Moritz] test plasma on macOS
8b53618 [Philipp Moritz] fix
54f595e [Philipp Moritz] try fixing python 2 tests
2c6d652 [Philipp Moritz] convert docs to numpy format
3270628 [Philipp Moritz] try to get documentation up
44d1a55 [Philipp Moritz] cleanups and release GIL
a9f6502 [Philipp Moritz] more fixes
1ff88e7 [Philipp Moritz] fix travix ci
348f9bf [Philipp Moritz] fixes
4ae1a27 [Philipp Moritz] fix
fd80203 [Philipp Moritz] Plasma Python extension packaging: It compiles!
3b69973 [Robert Nishihara] Fixed minor python linting.
c9f6bcf [Robert Nishihara] Fix indentation and line lengths in plasma.pyx.
67b0951 [Robert Nishihara] Fix long lines in plasma/test/test.py.
e26527c [Robert Nishihara] Convert plasma test.py from 2 space indentation to 4 space indentation.
acc71d2 [Philipp Moritz] add round trip test for dataframes
2b7f949 [Philipp Moritz] implement mutable arrow python buffers
c06f1b5 [Philipp Moritz] fix test
1d7928f [Philipp Moritz] add arrow roundtrip test
6371e2e [Philipp Moritz] fix tests
3021d59 [Philipp Moritz] make ObjectID pickleable
dd5a7d8 [Philipp Moritz] fix tests
777e9c7 [Philipp Moritz] introduce plasma namespace
a4a9628 [Philipp Moritz] fix c++ tests
924888b [Philipp Moritz] update
f970df3 [Philipp Moritz] reduce logging
2ff2480 [Philipp Moritz] workaround for python visibility
d4934a9 [Philipp Moritz] update
cba92c1 [Philipp Moritz] setup.py for plasma
066d0ea [Philipp Moritz] test
1aea320 [Philipp Moritz] run plasma tests
3c4de52 [Philipp Moritz] use cmake to build the cython extension
bf39297 [Philipp Moritz] build and install pyarrow for plasma tests
5bf722a [Philipp Moritz] fix plasma path
1c5434c [Philipp Moritz] fix formatting
187cc24 [Philipp Moritz] add travis tests
c3d462d [Philipp Moritz] remove Python C extension
d9261b4 [Philipp Moritz] add documentation and license
db2d09a [Philipp Moritz] get all python tests in place
78d08ac [Philipp Moritz] make eviction work in Cython
18e0ac4 [Philipp Moritz] get tests
bc681ca [Philipp Moritz] port some python tests
f8e05f2 [Philipp Moritz] implement plasma.get in the cython client
d590c8a [Philipp Moritz] update
5178ee7 [Philipp Moritz] update
9044a01 [Philipp Moritz] initial plasma cython client commit
---
 .travis.yml                                |  21 +
 ci/travis_script_manylinux.sh              |   2 +-
 ci/travis_script_plasma.sh                 |  97 +++
 ci/travis_script_python.sh                 |   4 +-
 cpp/src/arrow/util/logging.h               |   6 +-
 cpp/src/plasma/CMakeLists.txt              |  51 +-
 cpp/src/plasma/client.cc                   |  86 ++-
 cpp/src/plasma/client.h                    |  62 +-
 cpp/src/plasma/common.cc                   |   9 +-
 cpp/src/plasma/common.h                    |  39 +-
 cpp/src/plasma/events.cc                   |   4 +
 cpp/src/plasma/events.h                    |   4 +
 cpp/src/plasma/eviction_policy.cc          |   4 +
 cpp/src/plasma/eviction_policy.h           |   4 +
 cpp/src/plasma/extension.cc                | 456 --------------
 cpp/src/plasma/extension.h                 |  50 --
 cpp/src/plasma/plasma.cc                   |   4 +
 cpp/src/plasma/plasma.h                    |  53 +-
 cpp/src/plasma/plasma.pc.in                |  30 +
 cpp/src/plasma/protocol.cc                 |   6 +-
 cpp/src/plasma/protocol.h                  |   6 +-
 cpp/src/plasma/store.cc                    |  12 +-
 cpp/src/plasma/store.h                     |   4 +
 cpp/src/plasma/test/client_tests.cc        |  10 +-
 cpp/src/plasma/test/serialization_tests.cc |   4 +
 python/CMakeLists.txt                      |  18 +
 python/cmake_modules/FindPlasma.cmake      |  99 +++
 python/doc/source/api.rst                  |  15 +
 python/manylinux1/build_arrow.sh           |   5 +-
 python/pyarrow/__init__.py                 |   2 +-
 python/pyarrow/error.pxi                   |  18 +
 python/pyarrow/includes/common.pxd         |   3 +
 python/pyarrow/includes/libarrow.pxd       |   9 +
 python/pyarrow/io.pxi                      |  23 +-
 python/pyarrow/plasma.pyx                  | 560 +++++++++++++++++
 python/pyarrow/tests/conftest.py           |   8 +-
 python/pyarrow/tests/test_plasma.py        | 683 +++++++++++++++++++++
 python/setup.py                            |  15 +
 38 files changed, 1855 insertions(+), 631 deletions(-)
 create mode 100755 ci/travis_script_plasma.sh
 delete mode 100644 cpp/src/plasma/extension.cc
 delete mode 100644 cpp/src/plasma/extension.h
 create mode 100644 cpp/src/plasma/plasma.pc.in
 create mode 100644 python/cmake_modules/FindPlasma.cmake
 create mode 100644 python/pyarrow/plasma.pyx
 create mode 100644 python/pyarrow/tests/test_plasma.py

diff --git a/.travis.yml b/.travis.yml
index cdf787c831b0f..9cc2b86c05cde 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -120,6 +120,27 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+  - compiler: gcc
+    language: cpp
+    os: linux
+    group: deprecated
+    before_script:
+    - export CC="gcc-4.9"
+    - export CXX="g++-4.9"
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh
+  - compiler: clang
+    osx_image: xcode6.4
+    os: osx
+    cache:
+    addons:
+    before_script:
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh
 
 before_install:
 - ulimit -c unlimited -S
diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh
index 4e6be62bd3e9d..844d5f719f15a 100755
--- a/ci/travis_script_manylinux.sh
+++ b/ci/travis_script_manylinux.sh
@@ -18,4 +18,4 @@ set -ex
 pushd python/manylinux1
 git clone ../../ arrow
 docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
-docker run --rm -e PYARROW_PARALLEL=3 -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
+docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
diff --git a/ci/travis_script_plasma.sh b/ci/travis_script_plasma.sh
new file mode 100755
index 0000000000000..fa384ade89c2f
--- /dev/null
+++ b/ci/travis_script_plasma.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+set -e
+
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
+export ARROW_HOME=$ARROW_CPP_INSTALL
+export PYARROW_WITH_PLASMA=1
+
+pushd $ARROW_PYTHON_DIR
+
+function build_arrow_libraries() {
+  CPP_BUILD_DIR=$1
+  CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+
+  mkdir $CPP_BUILD_DIR
+  pushd $CPP_BUILD_DIR
+
+  cmake -DARROW_BUILD_TESTS=off \
+        -DARROW_PYTHON=on \
+        -DARROW_PLASMA=on \
+        -DCMAKE_INSTALL_PREFIX=$2 \
+        $CPP_DIR
+
+  make -j4
+  make install
+
+  popd
+}
+
+python_version_tests() {
+  PYTHON_VERSION=$1
+  CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
+
+  export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION
+  export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib
+
+  conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
+  source activate $CONDA_ENV_DIR
+
+  python --version
+  which python
+
+  # faster builds, please
+  conda install -y -q nomkl
+
+  # Expensive dependencies install from Continuum package repo
+  conda install -y -q pip numpy pandas cython
+
+  # Build C++ libraries
+  build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME
+
+  # Other stuff pip install
+  pip install -r requirements.txt
+
+  python setup.py build_ext --inplace
+
+  python -m pytest -vv -r sxX pyarrow
+
+  # Build documentation once
+  if [[ "$PYTHON_VERSION" == "3.6" ]]
+  then
+      conda install -y -q --file=doc/requirements.txt
+      python setup.py build_sphinx -s doc/source
+  fi
+
+  # Build and install pyarrow
+  pushd $TRAVIS_BUILD_DIR/python
+    python setup.py install
+  popd
+
+  # Run Plasma tests
+  pushd $TRAVIS_BUILD_DIR/python
+    python -m pytest pyarrow/tests/test_plasma.py
+    if [ $TRAVIS_OS_NAME == "linux" ]; then
+      PLASMA_VALGRIND=1 python -m pytest pyarrow/tests/test_plasma.py
+    fi
+  popd
+}
+
+# run tests for python 2.7 and 3.6
+python_version_tests 2.7
+python_version_tests 3.6
+
+popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index ac64c548d8225..fdb5ad6a62c93 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -17,6 +17,7 @@ set -e
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 export ARROW_HOME=$ARROW_CPP_INSTALL
+export PYARROW_WITH_PLASMA=1
 
 pushd $ARROW_PYTHON_DIR
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
@@ -71,9 +72,8 @@ function build_arrow_libraries() {
   pushd $CPP_BUILD_DIR
 
   cmake -DARROW_BUILD_TESTS=off \
-        -DARROW_PYTHON=on \
-        -DPLASMA_PYTHON=on \
         -DARROW_PLASMA=on \
+        -DARROW_PYTHON=on \
         -DCMAKE_INSTALL_PREFIX=$2 \
         $CPP_DIR
 
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index b6181219dbae6..0edaa9dfc37a3 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -113,8 +113,10 @@ class CerrLog {
 
   template <class T>
   CerrLog& operator<<(const T& t) {
-    has_logged_ = true;
-    std::cerr << t;
+    if (severity_ != ARROW_DEBUG) {
+      has_logged_ = true;
+      std::cerr << t;
+    }
     return *this;
   }
 
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 4ff3beba779c2..8bb7e71fdf11b 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -19,16 +19,13 @@ cmake_minimum_required(VERSION 2.8)
 
 project(plasma)
 
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../python/cmake_modules")
+
 find_package(PythonLibsNew REQUIRED)
 find_package(Threads)
 
-option(PLASMA_PYTHON
-  "Build the Plasma Python extensions"
-  OFF)
-
-if(APPLE)
-  SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
-endif(APPLE)
+set(PLASMA_SO_VERSION "0")
+set(PLASMA_ABI_VERSION "${PLASMA_SO_VERSION}.0.0")
 
 include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
 include_directories("${FLATBUFFERS_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/" "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" "${CMAKE_CURRENT_LIST_DIR}/../")
@@ -40,7 +37,7 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-conversion")
 # Compile flatbuffers
 
 set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
-set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
+set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/)
 
 set(PLASMA_FBS_OUTPUT_FILES
   "${OUTPUT_DIR}/common_generated.h"
@@ -69,8 +66,6 @@ endif()
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
 
-set_source_files_properties(extension.cc PROPERTIES COMPILE_FLAGS -Wno-strict-aliasing)
-
 set(PLASMA_SRCS
   client.cc
   common.cc
@@ -97,17 +92,33 @@ set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error -O3")
 add_executable(plasma_store store.cc)
 target_link_libraries(plasma_store plasma_static)
 
+# Headers: top level
+install(FILES
+  common.h
+  common_generated.h
+  client.h
+  events.h
+  plasma.h
+  plasma_generated.h
+  protocol.h
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/plasma")
+
+# Plasma store
+install(TARGETS plasma_store DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+# pkg-config support
+configure_file(plasma.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/plasma.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/plasma.pc"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
+
+#######################################
+# Unit tests
+#######################################
+
 ADD_ARROW_TEST(test/serialization_tests)
 ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static)
 ADD_ARROW_TEST(test/client_tests)
 ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static)
-
-if(PLASMA_PYTHON)
-  add_library(plasma_extension SHARED extension.cc)
-
-  if(APPLE)
-    target_link_libraries(plasma_extension plasma_static "-undefined dynamic_lookup")
-  else(APPLE)
-    target_link_libraries(plasma_extension plasma_static -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive)
-  endif(APPLE)
-endif()
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index dcb78e7ec52c6..62bfbec21c466 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -51,11 +51,31 @@
 
 #define XXH64_DEFAULT_SEED 0
 
+namespace plasma {
+
 // Number of threads used for memcopy and hash computations.
 constexpr int64_t kThreadPoolSize = 8;
 constexpr int64_t kBytesInMB = 1 << 20;
 static std::vector<std::thread> threadpool_(kThreadPoolSize);
 
+struct ObjectInUseEntry {
+  /// A count of the number of times this client has called PlasmaClient::Create
+  /// or
+  /// PlasmaClient::Get on this object ID minus the number of calls to
+  /// PlasmaClient::Release.
+  /// When this count reaches zero, we remove the entry from the ObjectsInUse
+  /// and decrement a count in the relevant ClientMmapTableEntry.
+  int count;
+  /// Cached information to read the object.
+  PlasmaObject object;
+  /// A flag representing whether the object has been sealed.
+  bool is_sealed;
+};
+
+PlasmaClient::PlasmaClient() {}
+
+PlasmaClient::~PlasmaClient() {}
+
 // If the file descriptor fd has been mmapped in this client process before,
 // return the pointer that was returned by mmap, otherwise mmap it and store the
 // pointer in a hash table.
@@ -300,6 +320,10 @@ Status PlasmaClient::PerformRelease(const ObjectID& object_id) {
 }
 
 Status PlasmaClient::Release(const ObjectID& object_id) {
+  // If the client is already disconnected, ignore release requests.
+  if (store_conn_ < 0) {
+    return Status::OK();
+  }
   // Add the new object to the release history.
   release_history_.push_front(object_id);
   // If there are too many bytes in use by the client or if there are too many
@@ -386,22 +410,6 @@ static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
   return XXH64_digest(&hash_state);
 }
 
-bool plasma_compute_object_hash(
-    PlasmaClient* conn, ObjectID object_id, unsigned char* digest) {
-  // Get the plasma object data. We pass in a timeout of 0 to indicate that
-  // the operation should timeout immediately.
-  ObjectBuffer object_buffer;
-  ARROW_CHECK_OK(conn->Get(&object_id, 1, 0, &object_buffer));
-  // If the object was not retrieved, return false.
-  if (object_buffer.data_size == -1) { return false; }
-  // Compute the hash.
-  uint64_t hash = compute_object_hash(object_buffer);
-  memcpy(digest, &hash, sizeof(hash));
-  // Release the plasma object.
-  ARROW_CHECK_OK(conn->Release(object_id));
-  return true;
-}
-
 Status PlasmaClient::Seal(const ObjectID& object_id) {
   // Make sure this client has a reference to the object before sending the
   // request to Plasma.
@@ -413,7 +421,7 @@ Status PlasmaClient::Seal(const ObjectID& object_id) {
   object_entry->second->is_sealed = true;
   /// Send the seal request to Plasma.
   static unsigned char digest[kDigestSize];
-  ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0]));
+  RETURN_NOT_OK(Hash(object_id, &digest[0]));
   RETURN_NOT_OK(SendSealRequest(store_conn_, object_id, &digest[0]));
   // We call PlasmaClient::Release to decrement the number of instances of this
   // object
@@ -439,6 +447,22 @@ Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) {
   return ReadEvictReply(buffer.data(), num_bytes_evicted);
 }
 
+Status PlasmaClient::Hash(const ObjectID& object_id, uint8_t* digest) {
+  // Get the plasma object data. We pass in a timeout of 0 to indicate that
+  // the operation should timeout immediately.
+  ObjectBuffer object_buffer;
+  RETURN_NOT_OK(Get(&object_id, 1, 0, &object_buffer));
+  // If the object was not retrieved, return false.
+  if (object_buffer.data_size == -1) {
+    return Status::PlasmaObjectNonexistent("Object not found");
+  }
+  // Compute the hash.
+  uint64_t hash = compute_object_hash(object_buffer);
+  memcpy(digest, &hash, sizeof(hash));
+  // Release the plasma object.
+  return Release(object_id);
+}
+
 Status PlasmaClient::Subscribe(int* fd) {
   int sock[2];
   // Create a non-blocking socket pair. This will only be used to send
@@ -459,6 +483,26 @@ Status PlasmaClient::Subscribe(int* fd) {
   return Status::OK();
 }
 
+Status PlasmaClient::GetNotification(
+    int fd, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size) {
+  uint8_t* notification = read_message_async(fd);
+  if (notification == NULL) {
+    return Status::IOError("Failed to read object notification from Plasma socket");
+  }
+  auto object_info = flatbuffers::GetRoot<ObjectInfo>(notification);
+  ARROW_CHECK(object_info->object_id()->size() == sizeof(ObjectID));
+  memcpy(object_id, object_info->object_id()->data(), sizeof(ObjectID));
+  if (object_info->is_deletion()) {
+    *data_size = -1;
+    *metadata_size = -1;
+  } else {
+    *data_size = object_info->data_size();
+    *metadata_size = object_info->metadata_size();
+  }
+  delete[] notification;
+  return Status::OK();
+}
+
 Status PlasmaClient::Connect(const std::string& store_socket_name,
     const std::string& manager_socket_name, int release_delay) {
   store_conn_ = connect_ipc_sock_retry(store_socket_name, -1, -1);
@@ -485,7 +529,11 @@ Status PlasmaClient::Disconnect() {
   // Close the connections to Plasma. The Plasma store will release the objects
   // that were in use by us when handling the SIGPIPE.
   close(store_conn_);
-  if (manager_conn_ >= 0) { close(manager_conn_); }
+  store_conn_ = -1;
+  if (manager_conn_ >= 0) {
+    close(manager_conn_);
+    manager_conn_ = -1;
+  }
   return Status::OK();
 }
 
@@ -555,3 +603,5 @@ Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_req
   }
   return Status::OK();
 }
+
+} // namespace plasma
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index fb3a161795d47..d9ed9f7c26698 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -22,12 +22,18 @@
 #include <time.h>
 
 #include <deque>
+#include <memory>
 #include <string>
+#include <unordered_map>
 
-#include "plasma/plasma.h"
+#include "arrow/status.h"
+#include "arrow/util/visibility.h"
+#include "plasma/common.h"
 
 using arrow::Status;
 
+namespace plasma {
+
 #define PLASMA_DEFAULT_RELEASE_DELAY 64
 
 // Use 100MB as an overestimate of the L3 cache size.
@@ -63,22 +69,16 @@ struct ClientMmapTableEntry {
   int count;
 };
 
-struct ObjectInUseEntry {
-  /// A count of the number of times this client has called PlasmaClient::Create
-  /// or
-  /// PlasmaClient::Get on this object ID minus the number of calls to
-  /// PlasmaClient::Release.
-  /// When this count reaches zero, we remove the entry from the ObjectsInUse
-  /// and decrement a count in the relevant ClientMmapTableEntry.
-  int count;
-  /// Cached information to read the object.
-  PlasmaObject object;
-  /// A flag representing whether the object has been sealed.
-  bool is_sealed;
-};
+struct ObjectInUseEntry;
+struct ObjectRequest;
+struct PlasmaObject;
 
-class PlasmaClient {
+class ARROW_EXPORT PlasmaClient {
  public:
+  PlasmaClient();
+
+  ~PlasmaClient();
+
   /// Connect to the local plasma store and plasma manager. Return
   /// the resulting connection.
   ///
@@ -177,10 +177,18 @@ class PlasmaClient {
   /// @return The return status.
   Status Evict(int64_t num_bytes, int64_t& num_bytes_evicted);
 
+  /// Compute the hash of an object in the object store.
+  ///
+  /// @param conn The object containing the connection state.
+  /// @param object_id The ID of the object we want to hash.
+  /// @param digest A pointer at which to return the hash digest of the object.
+  ///        The pointer must have at least kDigestSize bytes allocated.
+  /// @return The return status.
+  Status Hash(const ObjectID& object_id, uint8_t* digest);
+
   /// Subscribe to notifications when objects are sealed in the object store.
   /// Whenever an object is sealed, a message will be written to the client
-  /// socket
-  /// that is returned by this method.
+  /// socket that is returned by this method.
   ///
   /// @param fd Out parameter for the file descriptor the client should use to
   /// read notifications
@@ -188,6 +196,16 @@ class PlasmaClient {
   /// @return The return status.
   Status Subscribe(int* fd);
 
+  /// Receive next object notification for this client if Subscribe has been called.
+  ///
+  /// @param fd The file descriptor we are reading the notification from.
+  /// @param object_id Out parameter, the object_id of the object that was sealed.
+  /// @param data_size Out parameter, the data size of the object that was sealed.
+  /// @param metadata_size Out parameter, the metadata size of the object that was sealed.
+  /// @return The return status.
+  Status GetNotification(
+      int fd, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size);
+
   /// Disconnect from the local plasma instance, including the local store and
   /// manager.
   ///
@@ -330,14 +348,6 @@ class PlasmaClient {
   int64_t store_capacity_;
 };
 
-/// Compute the hash of an object in the object store.
-///
-/// @param conn The object containing the connection state.
-/// @param object_id The ID of the object we want to hash.
-/// @param digest A pointer at which to return the hash digest of the object.
-///        The pointer must have at least DIGEST_SIZE bytes allocated.
-/// @return A boolean representing whether the hash operation succeeded.
-bool plasma_compute_object_hash(
-    PlasmaClient* conn, ObjectID object_id, unsigned char* digest);
+} // namespace plasma
 
 #endif  // PLASMA_CLIENT_H
diff --git a/cpp/src/plasma/common.cc b/cpp/src/plasma/common.cc
index a09a963fa4769..a5f530e202ff4 100644
--- a/cpp/src/plasma/common.cc
+++ b/cpp/src/plasma/common.cc
@@ -19,7 +19,9 @@
 
 #include <random>
 
-#include "format/plasma_generated.h"
+#include "plasma/plasma_generated.h"
+
+namespace plasma {
 
 using arrow::Status;
 
@@ -81,3 +83,8 @@ Status plasma_error_status(int plasma_error) {
   }
   return Status::OK();
 }
+
+ARROW_EXPORT int ObjectStatusLocal = ObjectStatus_Local;
+ARROW_EXPORT int ObjectStatusRemote = ObjectStatus_Remote;
+
+} // namespace plasma
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
index 85dc74bf86e0d..6f2d4dd841b88 100644
--- a/cpp/src/plasma/common.h
+++ b/cpp/src/plasma/common.h
@@ -29,9 +29,11 @@
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
+namespace plasma {
+
 constexpr int64_t kUniqueIDSize = 20;
 
-class UniqueID {
+class ARROW_EXPORT UniqueID {
  public:
   static UniqueID from_random();
   static UniqueID from_binary(const std::string& binary);
@@ -60,4 +62,39 @@ typedef UniqueID ObjectID;
 
 arrow::Status plasma_error_status(int plasma_error);
 
+/// Size of object hash digests.
+constexpr int64_t kDigestSize = sizeof(uint64_t);
+
+/// Object request data structure. Used for Wait.
+struct ObjectRequest {
+  /// The ID of the requested object. If ID_NIL request any object.
+  ObjectID object_id;
+  /// Request associated to the object. It can take one of the following values:
+  ///  - PLASMA_QUERY_LOCAL: return if or when the object is available in the
+  ///    local Plasma Store.
+  ///  - PLASMA_QUERY_ANYWHERE: return if or when the object is available in
+  ///    the system (i.e., either in the local or a remote Plasma Store).
+  int type;
+  /// Object status. Same as the status returned by plasma_status() function
+  /// call. This is filled in by plasma_wait_for_objects1():
+  ///  - ObjectStatus_Local: object is ready at the local Plasma Store.
+  ///  - ObjectStatus_Remote: object is ready at a remote Plasma Store.
+  ///  - ObjectStatus_Nonexistent: object does not exist in the system.
+  ///  - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
+  ///    for being transferred or it is transferring.
+  int status;
+};
+
+enum ObjectRequestType {
+  /// Query for object in the local plasma store.
+  PLASMA_QUERY_LOCAL = 1,
+  /// Query for object in the local plasma store or in a remote plasma store.
+  PLASMA_QUERY_ANYWHERE
+};
+
+extern int ObjectStatusLocal;
+extern int ObjectStatusRemote;
+
+} // namespace plasma
+
 #endif  // PLASMA_COMMON_H
diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
index a9f7356e1f67e..675424d5c2f1c 100644
--- a/cpp/src/plasma/events.cc
+++ b/cpp/src/plasma/events.cc
@@ -19,6 +19,8 @@
 
 #include <errno.h>
 
+namespace plasma {
+
 void EventLoop::file_event_callback(
     aeEventLoop* loop, int fd, void* context, int events) {
   FileCallback* callback = reinterpret_cast<FileCallback*>(context);
@@ -79,3 +81,5 @@ int EventLoop::remove_timer(int64_t timer_id) {
   timer_callbacks_.erase(timer_id);
   return err;
 }
+
+} // namespace plasma
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
index bd93d6bb2a6fd..b989b7fac2476 100644
--- a/cpp/src/plasma/events.h
+++ b/cpp/src/plasma/events.h
@@ -26,6 +26,8 @@ extern "C" {
 #include "ae/ae.h"
 }
 
+namespace plasma {
+
 /// Constant specifying that the timer is done and it will be removed.
 constexpr int kEventLoopTimerDone = AE_NOMORE;
 
@@ -96,4 +98,6 @@ class EventLoop {
   std::unordered_map<int64_t, std::unique_ptr<TimerCallback>> timer_callbacks_;
 };
 
+} // namespace plasma
+
 #endif  // PLASMA_EVENTS
diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
index 4ae6384d42543..ef18e33372998 100644
--- a/cpp/src/plasma/eviction_policy.cc
+++ b/cpp/src/plasma/eviction_policy.cc
@@ -19,6 +19,8 @@
 
 #include <algorithm>
 
+namespace plasma {
+
 void LRUCache::add(const ObjectID& key, int64_t size) {
   auto it = item_map_.find(key);
   ARROW_CHECK(it == item_map_.end());
@@ -105,3 +107,5 @@ void EvictionPolicy::end_object_access(
   /* Add the object to the LRU cache.*/
   cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
 }
+
+} // namespace plasma
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
index 3815fc6652f0c..c4f218328312d 100644
--- a/cpp/src/plasma/eviction_policy.h
+++ b/cpp/src/plasma/eviction_policy.h
@@ -26,6 +26,8 @@
 #include "plasma/common.h"
 #include "plasma/plasma.h"
 
+namespace plasma {
+
 // ==== The eviction policy ====
 //
 // This file contains declaration for all functions and data structures that
@@ -131,4 +133,6 @@ class EvictionPolicy {
   LRUCache cache_;
 };
 
+} // namespace plasma
+
 #endif  // PLASMA_EVICTION_POLICY_H
diff --git a/cpp/src/plasma/extension.cc b/cpp/src/plasma/extension.cc
deleted file mode 100644
index 5d61e337c108d..0000000000000
--- a/cpp/src/plasma/extension.cc
+++ /dev/null
@@ -1,456 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/extension.h"
-
-#include <algorithm>
-#include <vector>
-
-#include "plasma/client.h"
-#include "plasma/common.h"
-#include "plasma/io.h"
-#include "plasma/protocol.h"
-
-PyObject* PlasmaOutOfMemoryError;
-PyObject* PlasmaObjectExistsError;
-
-PyObject* PyPlasma_connect(PyObject* self, PyObject* args) {
-  const char* store_socket_name;
-  const char* manager_socket_name;
-  int release_delay;
-  if (!PyArg_ParseTuple(
-          args, "ssi", &store_socket_name, &manager_socket_name, &release_delay)) {
-    return NULL;
-  }
-  PlasmaClient* client = new PlasmaClient();
-  ARROW_CHECK_OK(client->Connect(store_socket_name, manager_socket_name, release_delay));
-
-  return PyCapsule_New(client, "plasma", NULL);
-}
-
-PyObject* PyPlasma_disconnect(PyObject* self, PyObject* args) {
-  PyObject* client_capsule;
-  if (!PyArg_ParseTuple(args, "O", &client_capsule)) { return NULL; }
-  PlasmaClient* client;
-  ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client));
-  ARROW_CHECK_OK(client->Disconnect());
-  /* We use the context of the connection capsule to indicate if the connection
-   * is still active (if the context is NULL) or if it is closed (if the context
-   * is (void*) 0x1). This is neccessary because the primary pointer of the
-   * capsule cannot be NULL. */
-  PyCapsule_SetContext(client_capsule, reinterpret_cast<void*>(0x1));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_create(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  Py_ssize_t size;
-  PyObject* metadata;
-  if (!PyArg_ParseTuple(args, "O&O&nO", PyObjectToPlasmaClient, &client,
-          PyStringToUniqueID, &object_id, &size, &metadata)) {
-    return NULL;
-  }
-  if (!PyByteArray_Check(metadata)) {
-    PyErr_SetString(PyExc_TypeError, "metadata must be a bytearray");
-    return NULL;
-  }
-  uint8_t* data;
-  Status s = client->Create(object_id, size,
-      reinterpret_cast<uint8_t*>(PyByteArray_AsString(metadata)),
-      PyByteArray_Size(metadata), &data);
-  if (s.IsPlasmaObjectExists()) {
-    PyErr_SetString(PlasmaObjectExistsError,
-        "An object with this ID already exists in the plasma "
-        "store.");
-    return NULL;
-  }
-  if (s.IsPlasmaStoreFull()) {
-    PyErr_SetString(PlasmaOutOfMemoryError,
-        "The plasma store ran out of memory and could not create "
-        "this object.");
-    return NULL;
-  }
-  ARROW_CHECK(s.ok());
-
-#if PY_MAJOR_VERSION >= 3
-  return PyMemoryView_FromMemory(reinterpret_cast<char*>(data), size, PyBUF_WRITE);
-#else
-  return PyBuffer_FromReadWriteMemory(reinterpret_cast<void*>(data), size);
-#endif
-}
-
-PyObject* PyPlasma_hash(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  unsigned char digest[kDigestSize];
-  bool success = plasma_compute_object_hash(client, object_id, digest);
-  if (success) {
-    PyObject* digest_string =
-        PyBytes_FromStringAndSize(reinterpret_cast<char*>(digest), kDigestSize);
-    return digest_string;
-  } else {
-    Py_RETURN_NONE;
-  }
-}
-
-PyObject* PyPlasma_seal(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Seal(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_release(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Release(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_get(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  PyObject* object_id_list;
-  Py_ssize_t timeout_ms;
-  if (!PyArg_ParseTuple(
-          args, "O&On", PyObjectToPlasmaClient, &client, &object_id_list, &timeout_ms)) {
-    return NULL;
-  }
-
-  Py_ssize_t num_object_ids = PyList_Size(object_id_list);
-  std::vector<ObjectID> object_ids(num_object_ids);
-  std::vector<ObjectBuffer> object_buffers(num_object_ids);
-
-  for (int i = 0; i < num_object_ids; ++i) {
-    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
-  }
-
-  Py_BEGIN_ALLOW_THREADS;
-  ARROW_CHECK_OK(
-      client->Get(object_ids.data(), num_object_ids, timeout_ms, object_buffers.data()));
-  Py_END_ALLOW_THREADS;
-
-  PyObject* returns = PyList_New(num_object_ids);
-  for (int i = 0; i < num_object_ids; ++i) {
-    if (object_buffers[i].data_size != -1) {
-      /* The object was retrieved, so return the object. */
-      PyObject* t = PyTuple_New(2);
-      Py_ssize_t data_size = static_cast<Py_ssize_t>(object_buffers[i].data_size);
-      Py_ssize_t metadata_size = static_cast<Py_ssize_t>(object_buffers[i].metadata_size);
-#if PY_MAJOR_VERSION >= 3
-      char* data = reinterpret_cast<char*>(object_buffers[i].data);
-      char* metadata = reinterpret_cast<char*>(object_buffers[i].metadata);
-      PyTuple_SET_ITEM(t, 0, PyMemoryView_FromMemory(data, data_size, PyBUF_READ));
-      PyTuple_SET_ITEM(
-          t, 1, PyMemoryView_FromMemory(metadata, metadata_size, PyBUF_READ));
-#else
-      void* data = reinterpret_cast<void*>(object_buffers[i].data);
-      void* metadata = reinterpret_cast<void*>(object_buffers[i].metadata);
-      PyTuple_SET_ITEM(t, 0, PyBuffer_FromMemory(data, data_size));
-      PyTuple_SET_ITEM(t, 1, PyBuffer_FromMemory(metadata, metadata_size));
-#endif
-      ARROW_CHECK(PyList_SetItem(returns, i, t) == 0);
-    } else {
-      /* The object was not retrieved, so just add None to the list of return
-       * values. */
-      Py_INCREF(Py_None);
-      ARROW_CHECK(PyList_SetItem(returns, i, Py_None) == 0);
-    }
-  }
-  return returns;
-}
-
-PyObject* PyPlasma_contains(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  bool has_object;
-  ARROW_CHECK_OK(client->Contains(object_id, &has_object));
-
-  if (has_object) {
-    Py_RETURN_TRUE;
-  } else {
-    Py_RETURN_FALSE;
-  }
-}
-
-PyObject* PyPlasma_fetch(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  PyObject* object_id_list;
-  if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client, &object_id_list)) {
-    return NULL;
-  }
-  if (client->get_manager_fd() == -1) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-  Py_ssize_t n = PyList_Size(object_id_list);
-  ObjectID* object_ids = new ObjectID[n];
-  for (int i = 0; i < n; ++i) {
-    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
-  }
-  ARROW_CHECK_OK(client->Fetch(static_cast<int>(n), object_ids));
-  delete[] object_ids;
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_wait(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  PyObject* object_id_list;
-  Py_ssize_t timeout;
-  int num_returns;
-  if (!PyArg_ParseTuple(args, "O&Oni", PyObjectToPlasmaClient, &client, &object_id_list,
-          &timeout, &num_returns)) {
-    return NULL;
-  }
-  Py_ssize_t n = PyList_Size(object_id_list);
-
-  if (client->get_manager_fd() == -1) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-  if (num_returns < 0) {
-    PyErr_SetString(
-        PyExc_RuntimeError, "The argument num_returns cannot be less than zero.");
-    return NULL;
-  }
-  if (num_returns > n) {
-    PyErr_SetString(PyExc_RuntimeError,
-        "The argument num_returns cannot be greater than len(object_ids)");
-    return NULL;
-  }
-  int64_t threshold = 1 << 30;
-  if (timeout > threshold) {
-    PyErr_SetString(
-        PyExc_RuntimeError, "The argument timeout cannot be greater than 2 ** 30.");
-    return NULL;
-  }
-
-  std::vector<ObjectRequest> object_requests(n);
-  for (int i = 0; i < n; ++i) {
-    ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i),
-                    &object_requests[i].object_id) == 1);
-    object_requests[i].type = PLASMA_QUERY_ANYWHERE;
-  }
-  /* Drop the global interpreter lock while we are waiting, so other threads can
-   * run. */
-  int num_return_objects;
-  Py_BEGIN_ALLOW_THREADS;
-  ARROW_CHECK_OK(
-      client->Wait(n, object_requests.data(), num_returns, timeout, &num_return_objects));
-  Py_END_ALLOW_THREADS;
-
-  int num_to_return = std::min(num_return_objects, num_returns);
-  PyObject* ready_ids = PyList_New(num_to_return);
-  PyObject* waiting_ids = PySet_New(object_id_list);
-  int num_returned = 0;
-  for (int i = 0; i < n; ++i) {
-    if (num_returned == num_to_return) { break; }
-    if (object_requests[i].status == ObjectStatus_Local ||
-        object_requests[i].status == ObjectStatus_Remote) {
-      PyObject* ready = PyBytes_FromStringAndSize(
-          reinterpret_cast<char*>(&object_requests[i].object_id),
-          sizeof(object_requests[i].object_id));
-      PyList_SetItem(ready_ids, num_returned, ready);
-      PySet_Discard(waiting_ids, ready);
-      num_returned += 1;
-    } else {
-      ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent);
-    }
-  }
-  ARROW_CHECK(num_returned == num_to_return);
-  /* Return both the ready IDs and the remaining IDs. */
-  PyObject* t = PyTuple_New(2);
-  PyTuple_SetItem(t, 0, ready_ids);
-  PyTuple_SetItem(t, 1, waiting_ids);
-  return t;
-}
-
-PyObject* PyPlasma_evict(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  Py_ssize_t num_bytes;
-  if (!PyArg_ParseTuple(args, "O&n", PyObjectToPlasmaClient, &client, &num_bytes)) {
-    return NULL;
-  }
-  int64_t evicted_bytes;
-  ARROW_CHECK_OK(client->Evict(static_cast<int64_t>(num_bytes), evicted_bytes));
-  return PyLong_FromSsize_t(static_cast<Py_ssize_t>(evicted_bytes));
-}
-
-PyObject* PyPlasma_delete(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
-          &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Delete(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_transfer(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  ObjectID object_id;
-  const char* addr;
-  int port;
-  if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client,
-          PyStringToUniqueID, &object_id, &addr, &port)) {
-    return NULL;
-  }
-
-  if (client->get_manager_fd() == -1) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-
-  ARROW_CHECK_OK(client->Transfer(addr, port, object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_subscribe(PyObject* self, PyObject* args) {
-  PlasmaClient* client;
-  if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) { return NULL; }
-
-  int sock;
-  ARROW_CHECK_OK(client->Subscribe(&sock));
-  return PyLong_FromLong(sock);
-}
-
-PyObject* PyPlasma_receive_notification(PyObject* self, PyObject* args) {
-  int plasma_sock;
-
-  if (!PyArg_ParseTuple(args, "i", &plasma_sock)) { return NULL; }
-  /* Receive object notification from the plasma connection socket. If the
-   * object was added, return a tuple of its fields: ObjectID, data_size,
-   * metadata_size. If the object was deleted, data_size and metadata_size will
-   * be set to -1. */
-  uint8_t* notification = read_message_async(plasma_sock);
-  if (notification == NULL) {
-    PyErr_SetString(
-        PyExc_RuntimeError, "Failed to read object notification from Plasma socket");
-    return NULL;
-  }
-  auto object_info = flatbuffers::GetRoot<ObjectInfo>(notification);
-  /* Construct a tuple from object_info and return. */
-  PyObject* t = PyTuple_New(3);
-  PyTuple_SetItem(t, 0, PyBytes_FromStringAndSize(object_info->object_id()->data(),
-                            object_info->object_id()->size()));
-  if (object_info->is_deletion()) {
-    PyTuple_SetItem(t, 1, PyLong_FromLong(-1));
-    PyTuple_SetItem(t, 2, PyLong_FromLong(-1));
-  } else {
-    PyTuple_SetItem(t, 1, PyLong_FromLong(object_info->data_size()));
-    PyTuple_SetItem(t, 2, PyLong_FromLong(object_info->metadata_size()));
-  }
-
-  delete[] notification;
-  return t;
-}
-
-static PyMethodDef plasma_methods[] = {
-    {"connect", PyPlasma_connect, METH_VARARGS, "Connect to plasma."},
-    {"disconnect", PyPlasma_disconnect, METH_VARARGS, "Disconnect from plasma."},
-    {"create", PyPlasma_create, METH_VARARGS, "Create a new plasma object."},
-    {"hash", PyPlasma_hash, METH_VARARGS, "Compute the hash of a plasma object."},
-    {"seal", PyPlasma_seal, METH_VARARGS, "Seal a plasma object."},
-    {"get", PyPlasma_get, METH_VARARGS, "Get a plasma object."},
-    {"contains", PyPlasma_contains, METH_VARARGS,
-        "Does the plasma store contain this plasma object?"},
-    {"fetch", PyPlasma_fetch, METH_VARARGS,
-        "Fetch the object from another plasma manager instance."},
-    {"wait", PyPlasma_wait, METH_VARARGS,
-        "Wait until num_returns objects in object_ids are ready."},
-    {"evict", PyPlasma_evict, METH_VARARGS,
-        "Evict some objects until we recover some number of bytes."},
-    {"release", PyPlasma_release, METH_VARARGS, "Release the plasma object."},
-    {"delete", PyPlasma_delete, METH_VARARGS, "Delete a plasma object."},
-    {"transfer", PyPlasma_transfer, METH_VARARGS,
-        "Transfer object to another plasma manager."},
-    {"subscribe", PyPlasma_subscribe, METH_VARARGS,
-        "Subscribe to the plasma notification socket."},
-    {"receive_notification", PyPlasma_receive_notification, METH_VARARGS,
-        "Receive next notification from plasma notification socket."},
-    {NULL} /* Sentinel */
-};
-
-#if PY_MAJOR_VERSION >= 3
-static struct PyModuleDef moduledef = {
-    PyModuleDef_HEAD_INIT, "libplasma",    /* m_name */
-    "A Python client library for plasma.", /* m_doc */
-    0,                                     /* m_size */
-    plasma_methods,                        /* m_methods */
-    NULL,                                  /* m_reload */
-    NULL,                                  /* m_traverse */
-    NULL,                                  /* m_clear */
-    NULL,                                  /* m_free */
-};
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define INITERROR return NULL
-#else
-#define INITERROR return
-#endif
-
-#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
-#define PyMODINIT_FUNC void
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
-#else
-#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
-#endif
-
-MOD_INIT(libplasma) {
-#if PY_MAJOR_VERSION >= 3
-  PyObject* m = PyModule_Create(&moduledef);
-#else
-  PyObject* m =
-      Py_InitModule3("libplasma", plasma_methods, "A Python client library for plasma.");
-#endif
-
-  /* Create a custom exception for when an object ID is reused. */
-  char plasma_object_exists_error[] = "plasma_object_exists.error";
-  PlasmaObjectExistsError = PyErr_NewException(plasma_object_exists_error, NULL, NULL);
-  Py_INCREF(PlasmaObjectExistsError);
-  PyModule_AddObject(m, "plasma_object_exists_error", PlasmaObjectExistsError);
-  /* Create a custom exception for when the plasma store is out of memory. */
-  char plasma_out_of_memory_error[] = "plasma_out_of_memory.error";
-  PlasmaOutOfMemoryError = PyErr_NewException(plasma_out_of_memory_error, NULL, NULL);
-  Py_INCREF(PlasmaOutOfMemoryError);
-  PyModule_AddObject(m, "plasma_out_of_memory_error", PlasmaOutOfMemoryError);
-
-#if PY_MAJOR_VERSION >= 3
-  return m;
-#endif
-}
diff --git a/cpp/src/plasma/extension.h b/cpp/src/plasma/extension.h
deleted file mode 100644
index cee30abb3592d..0000000000000
--- a/cpp/src/plasma/extension.h
+++ /dev/null
@@ -1,50 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_EXTENSION_H
-#define PLASMA_EXTENSION_H
-
-#undef _XOPEN_SOURCE
-#undef _POSIX_C_SOURCE
-#include <Python.h>
-
-#include "bytesobject.h"  // NOLINT
-
-#include "plasma/client.h"
-#include "plasma/common.h"
-
-static int PyObjectToPlasmaClient(PyObject* object, PlasmaClient** client) {
-  if (PyCapsule_IsValid(object, "plasma")) {
-    *client = reinterpret_cast<PlasmaClient*>(PyCapsule_GetPointer(object, "plasma"));
-    return 1;
-  } else {
-    PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule");
-    return 0;
-  }
-}
-
-int PyStringToUniqueID(PyObject* object, ObjectID* object_id) {
-  if (PyBytes_Check(object)) {
-    memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID));
-    return 1;
-  } else {
-    PyErr_SetString(PyExc_TypeError, "must be a 20 character string");
-    return 0;
-  }
-}
-
-#endif  // PLASMA_EXTENSION_H
diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc
index 559d8e7f2a65e..bfed5009b6157 100644
--- a/cpp/src/plasma/plasma.cc
+++ b/cpp/src/plasma/plasma.cc
@@ -24,6 +24,8 @@
 #include "plasma/common.h"
 #include "plasma/protocol.h"
 
+namespace plasma {
+
 int warn_if_sigpipe(int status, int client_sock) {
   if (status >= 0) { return 0; }
   if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
@@ -62,3 +64,5 @@ ObjectTableEntry* get_object_table_entry(
   if (it == store_info->objects.end()) { return NULL; }
   return it->second.get();
 }
+
+} // namespace plasma
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
index 275d0c7a41687..db8669ff0ddc2 100644
--- a/cpp/src/plasma/plasma.h
+++ b/cpp/src/plasma/plasma.h
@@ -32,8 +32,10 @@
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
-#include "format/common_generated.h"
 #include "plasma/common.h"
+#include "plasma/common_generated.h"
+
+namespace plasma {
 
 #define HANDLE_SIGPIPE(s, fd_)                                              \
   do {                                                                      \
@@ -54,47 +56,23 @@
 /// Allocation granularity used in plasma for object allocation.
 #define BLOCK_SIZE 64
 
-/// Size of object hash digests.
-constexpr int64_t kDigestSize = sizeof(uint64_t);
-
 struct Client;
 
-/// Object request data structure. Used in the plasma_wait_for_objects()
-/// argument.
-typedef struct {
-  /// The ID of the requested object. If ID_NIL request any object.
-  ObjectID object_id;
-  /// Request associated to the object. It can take one of the following values:
-  ///  - PLASMA_QUERY_LOCAL: return if or when the object is available in the
-  ///    local Plasma Store.
-  ///  - PLASMA_QUERY_ANYWHERE: return if or when the object is available in
-  ///    the system (i.e., either in the local or a remote Plasma Store).
-  int type;
-  /// Object status. Same as the status returned by plasma_status() function
-  /// call. This is filled in by plasma_wait_for_objects1():
-  ///  - ObjectStatus_Local: object is ready at the local Plasma Store.
-  ///  - ObjectStatus_Remote: object is ready at a remote Plasma Store.
-  ///  - ObjectStatus_Nonexistent: object does not exist in the system.
-  ///  - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
-  ///    for being transferred or it is transferring.
-  int status;
-} ObjectRequest;
-
 /// Mapping from object IDs to type and status of the request.
 typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectRequestMap;
 
 /// Handle to access memory mapped file and map it into client address space.
-typedef struct {
+struct object_handle {
   /// The file descriptor of the memory mapped file in the store. It is used as
   /// a unique identifier of the file in the client to look up the corresponding
   /// file descriptor on the client's side.
   int store_fd;
   /// The size in bytes of the memory mapped file.
   int64_t mmap_size;
-} object_handle;
+};
 
 // TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
-typedef struct {
+struct PlasmaObject {
   /// Handle for memory mapped file the object is stored in.
   object_handle handle;
   /// The offset in bytes in the memory mapped file of the data.
@@ -105,28 +83,21 @@ typedef struct {
   int64_t data_size;
   /// The size in bytes of the metadata.
   int64_t metadata_size;
-} PlasmaObject;
+};
 
-typedef enum {
+enum object_state {
   /// Object was created but not sealed in the local Plasma Store.
   PLASMA_CREATED = 1,
   /// Object is sealed and stored in the local Plasma Store.
   PLASMA_SEALED
-} object_state;
+};
 
-typedef enum {
+enum object_status {
   /// The object was not found.
   OBJECT_NOT_FOUND = 0,
   /// The object was found.
   OBJECT_FOUND = 1
-} object_status;
-
-typedef enum {
-  /// Query for object in the local plasma store.
-  PLASMA_QUERY_LOCAL = 1,
-  /// Query for object in the local plasma store or in a remote plasma store.
-  PLASMA_QUERY_ANYWHERE
-} object_request_type;
+};
 
 /// This type is used by the Plasma store. It is here because it is exposed to
 /// the eviction policy.
@@ -188,4 +159,6 @@ int warn_if_sigpipe(int status, int client_sock);
 
 uint8_t* create_object_info_buffer(ObjectInfoT* object_info);
 
+} // namespace plasma
+
 #endif  // PLASMA_PLASMA_H
diff --git a/cpp/src/plasma/plasma.pc.in b/cpp/src/plasma/plasma.pc.in
new file mode 100644
index 0000000000000..d86868939f363
--- /dev/null
+++ b/cpp/src/plasma/plasma.pc.in
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
+includedir=${prefix}/include
+
+so_version=@PLASMA_SO_VERSION@
+abi_version=@PLASMA_ABI_VERSION@
+executable=${prefix}/@CMAKE_INSTALL_BINDIR@/plasma_store
+
+Name: Plasma
+Description: Plasma is an in-memory object store and cache for big data.
+Version: @PLASMA_VERSION@
+Libs: -L${libdir} -lplasma
+Cflags: -I${includedir}
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 246aa29736056..2998c68b82785 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -18,11 +18,13 @@
 #include "plasma/protocol.h"
 
 #include "flatbuffers/flatbuffers.h"
-#include "format/plasma_generated.h"
+#include "plasma/plasma_generated.h"
 
 #include "plasma/common.h"
 #include "plasma/io.h"
 
+namespace plasma {
+
 using flatbuffers::uoffset_t;
 
 flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
@@ -500,3 +502,5 @@ Status ReadDataReply(
   *metadata_size = (int64_t)message->metadata_size();
   return Status::OK();
 }
+
+} // namespace plasma
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
index 5d9d13675144f..835c5a0b58978 100644
--- a/cpp/src/plasma/protocol.h
+++ b/cpp/src/plasma/protocol.h
@@ -21,9 +21,11 @@
 #include <vector>
 
 #include "arrow/status.h"
-#include "format/plasma_generated.h"
+#include "plasma/plasma_generated.h"
 #include "plasma/plasma.h"
 
+namespace plasma {
+
 using arrow::Status;
 
 /* Plasma receive message. */
@@ -167,4 +169,6 @@ Status SendDataReply(
 Status ReadDataReply(
     uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size);
 
+} // namespace plasma
+
 #endif /* PLASMA_PROTOCOL */
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 9394e3de310b2..8d4fb106f5367 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -49,12 +49,14 @@
 #include <unordered_set>
 #include <vector>
 
-#include "format/common_generated.h"
+#include "plasma/common_generated.h"
 #include "plasma/common.h"
 #include "plasma/fling.h"
 #include "plasma/io.h"
 #include "plasma/malloc.h"
 
+namespace plasma {
+
 extern "C" {
 void* dlmalloc(size_t bytes);
 void* dlmemalign(size_t alignment, size_t bytes);
@@ -625,8 +627,10 @@ void start_server(char* socket_name, int64_t system_memory) {
   loop.run();
 }
 
+} // namespace plasma
+
 int main(int argc, char* argv[]) {
-  signal(SIGTERM, signal_handler);
+  signal(SIGTERM, plasma::signal_handler);
   char* socket_name = NULL;
   int64_t system_memory = -1;
   int c;
@@ -677,7 +681,7 @@ int main(int argc, char* argv[]) {
 #endif
   // Make it so dlmalloc fails if we try to request more memory than is
   // available.
-  dlmalloc_set_footprint_limit((size_t)system_memory);
+  plasma::dlmalloc_set_footprint_limit((size_t)system_memory);
   ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
-  start_server(socket_name, system_memory);
+  plasma::start_server(socket_name, system_memory);
 }
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index 8bd94265410f6..27c3813da8e7a 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -27,6 +27,8 @@
 #include "plasma/plasma.h"
 #include "plasma/protocol.h"
 
+namespace plasma {
+
 struct GetRequest;
 
 struct NotificationQueue {
@@ -166,4 +168,6 @@ class PlasmaStore {
   std::unordered_map<int, NotificationQueue> pending_notifications_;
 };
 
+} // namespace plasma
+
 #endif  // PLASMA_STORE_H
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index 29b5b135144c3..6dc558e77078b 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -29,7 +29,9 @@
 #include "plasma/plasma.h"
 #include "plasma/protocol.h"
 
-std::string g_test_executable;  // NOLINT
+namespace plasma {
+
+std::string test_executable;  // NOLINT
 
 class TestPlasmaStore : public ::testing::Test {
  public:
@@ -37,7 +39,7 @@ class TestPlasmaStore : public ::testing::Test {
   // stdout of the object store. Consider changing that.
   void SetUp() {
     std::string plasma_directory =
-        g_test_executable.substr(0, g_test_executable.find_last_of("/"));
+        test_executable.substr(0, test_executable.find_last_of("/"));
     std::string plasma_command =
         plasma_directory +
         "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &";
@@ -125,8 +127,10 @@ TEST_F(TestPlasmaStore, MultipleGetTest) {
   ASSERT_EQ(object_buffer[1].data[0], 2);
 }
 
+} // namespace plasma
+
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
-  g_test_executable = std::string(argv[0]);
+  plasma::test_executable = std::string(argv[0]);
   return RUN_ALL_TESTS();
 }
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
index 325cead06e770..13938cd6fb042 100644
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -25,6 +25,8 @@
 #include "plasma/plasma.h"
 #include "plasma/protocol.h"
 
+namespace plasma {
+
 /**
  * Create a temporary file. Needs to be closed by the caller.
  *
@@ -386,3 +388,5 @@ TEST(PlasmaSerialization, DataReply) {
   ASSERT_EQ(object_size1, object_size2);
   ASSERT_EQ(metadata_size1, metadata_size2);
 }
+
+} // namespace plasma
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 224147d8b5c3b..6ff66462958ef 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -220,6 +220,12 @@ include_directories(SYSTEM
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 
+## Plasma
+find_package(Plasma)
+if (PLASMA_FOUND)
+  include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
+endif()
+
 function(bundle_arrow_lib library_path)
   get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY)
   get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
@@ -252,6 +258,9 @@ if (PYARROW_BUNDLE_ARROW_CPP)
   file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
   bundle_arrow_lib(ARROW_SHARED_LIB)
   bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
+  if (PLASMA_FOUND)
+    bundle_arrow_lib(PLASMA_SHARED_LIB)
+  endif()
 endif()
 
 if (MSVC)
@@ -278,9 +287,14 @@ set(CYTHON_EXTENSIONS
   lib
 )
 
+if (PLASMA_FOUND)
+  set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} plasma)
+endif()
+
 set(LINK_LIBS
   arrow_shared
   arrow_python_shared
+  ${PLASMA_SHARED_LIB}
 )
 
 if (PYARROW_BUILD_PARQUET)
@@ -379,3 +393,7 @@ foreach(module ${CYTHON_EXTENSIONS})
 
     target_link_libraries(${module_name} ${LINK_LIBS})
 endforeach(module)
+
+if (PLASMA_FOUND)
+  file(COPY ${PLASMA_EXECUTABLE} DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY})
+endif()
diff --git a/python/cmake_modules/FindPlasma.cmake b/python/cmake_modules/FindPlasma.cmake
new file mode 100644
index 0000000000000..3acaa348bffa8
--- /dev/null
+++ b/python/cmake_modules/FindPlasma.cmake
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find PLASMA (plasma/client.h, libplasma.a, libplasma.so)
+# This module defines
+#  PLASMA_INCLUDE_DIR, directory containing headers
+#  PLASMA_LIBS, directory containing plasma libraries
+#  PLASMA_STATIC_LIB, path to libplasma.a
+#  PLASMA_SHARED_LIB, path to libplasma's shared library
+#  PLASMA_SHARED_IMP_LIB, path to libplasma's import library (MSVC only)
+#  PLASMA_FOUND, whether plasma has been found
+
+include(FindPkgConfig)
+
+if ("$ENV{ARROW_HOME}" STREQUAL "")
+  pkg_check_modules(PLASMA plasma)
+  if (PLASMA_FOUND)
+    pkg_get_variable(PLASMA_EXECUTABLE plasma executable)
+    pkg_get_variable(PLASMA_ABI_VERSION plasma abi_version)
+    message(STATUS "Plasma ABI version: ${PLASMA_ABI_VERSION}")
+    pkg_get_variable(PLASMA_SO_VERSION plasma so_version)
+    message(STATUS "Plasma SO version: ${PLASMA_SO_VERSION}")
+    set(PLASMA_INCLUDE_DIR ${PLASMA_INCLUDE_DIRS})
+    set(PLASMA_LIBS ${PLASMA_LIBRARY_DIRS})
+    set(PLASMA_SEARCH_LIB_PATH ${PLASMA_LIBRARY_DIRS})
+  endif()
+else()
+  set(PLASMA_HOME "$ENV{ARROW_HOME}")
+
+  set(PLASMA_EXECUTABLE ${PLASMA_HOME}/bin/plasma_store)
+
+  set(PLASMA_SEARCH_HEADER_PATHS
+    ${PLASMA_HOME}/include
+    )
+
+  set(PLASMA_SEARCH_LIB_PATH
+    ${PLASMA_HOME}/lib
+    )
+
+  find_path(PLASMA_INCLUDE_DIR plasma/client.h PATHS
+    ${PLASMA_SEARCH_HEADER_PATHS}
+    # make sure we don't accidentally pick up a different version
+    NO_DEFAULT_PATH
+    )
+endif()
+
+find_library(PLASMA_LIB_PATH NAMES plasma
+  PATHS
+  ${PLASMA_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+get_filename_component(PLASMA_LIBS ${PLASMA_LIB_PATH} DIRECTORY)
+
+if (PLASMA_INCLUDE_DIR AND PLASMA_LIBS)
+  set(PLASMA_FOUND TRUE)
+  set(PLASMA_LIB_NAME plasma)
+
+  set(PLASMA_STATIC_LIB ${PLASMA_LIBS}/lib${PLASMA_LIB_NAME}.a)
+
+  set(PLASMA_SHARED_LIB ${PLASMA_LIBS}/lib${PLASMA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+endif()
+
+if (PLASMA_FOUND)
+  if (NOT Plasma_FIND_QUIETLY)
+    message(STATUS "Found the Plasma core library: ${PLASMA_LIB_PATH}")
+    message(STATUS "Found Plasma executable: ${PLASMA_EXECUTABLE}")
+  endif ()
+else ()
+  if (NOT Plasma_FIND_QUIETLY)
+    set(PLASMA_ERR_MSG "Could not find the Plasma library. Looked for headers")
+    set(PLASMA_ERR_MSG "${PLASMA_ERR_MSG} in ${PLASMA_SEARCH_HEADER_PATHS}, and for libs")
+    set(PLASMA_ERR_MSG "${PLASMA_ERR_MSG} in ${PLASMA_SEARCH_LIB_PATH}")
+    if (Plasma_FIND_REQUIRED)
+      message(FATAL_ERROR "${PLASMA_ERR_MSG}")
+    else (Plasma_FIND_REQUIRED)
+      message(STATUS "${PLASMA_ERR_MSG}")
+    endif (Plasma_FIND_REQUIRED)
+  endif ()
+  set(PLASMA_FOUND FALSE)
+endif ()
+
+mark_as_advanced(
+  PLASMA_INCLUDE_DIR
+  PLASMA_STATIC_LIB
+  PLASMA_SHARED_LIB
+)
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index c52d400cef1c7..780aa4839610f 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -212,6 +212,21 @@ Type Classes
    Field
    Schema
 
+.. currentmodule:: pyarrow.plasma
+
+.. _api.plasma:
+
+In-Memory Object Store
+----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   ObjectID
+   PlasmaClient
+   PlasmaBuffer
+   MutablePlasmaBuffer
+
 .. currentmodule:: pyarrow.parquet
 
 .. _api.parquet:
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 8c6bda9550e87..85c096a5c11d0 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -35,6 +35,7 @@ cd /arrow/python
 # PyArrow build configuration
 export PYARROW_BUILD_TYPE='release'
 export PYARROW_WITH_PARQUET=1
+export PYARROW_WITH_PLASMA=1
 export PYARROW_BUNDLE_ARROW_CPP=1
 # Need as otherwise arrow_io is sometimes not linked
 export LDFLAGS="-Wl,--no-as-needed"
@@ -52,7 +53,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
     mkdir -p "${ARROW_BUILD_DIR}"
     pushd "${ARROW_BUILD_DIR}"
-    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} ..
+    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON ..
     make -j5 install
     popd
 
@@ -65,6 +66,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     echo "=== (${PYTHON}) Test the existence of optional modules ==="
     $PIPI_IO -r requirements.txt
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.plasma"
 
     echo "=== (${PYTHON}) Tag the wheel with manylinux1 ==="
     mkdir -p repaired_wheels/
@@ -78,4 +80,3 @@ for PYTHON in ${PYTHON_VERSIONS}; do
 
     mv repaired_wheels/*.whl /io/dist
 done
-
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index e3d783aee58b4..6d0ce204382e3 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -68,6 +68,7 @@
                          Date32Value, Date64Value, TimestampValue)
 
 from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
+                         FixedSizeBufferOutputStream,
                          Buffer, BufferReader, BufferOutputStream,
                          OSFile, MemoryMappedFile, memory_map,
                          frombuffer,
@@ -99,7 +100,6 @@
                          open_file,
                          serialize_pandas, deserialize_pandas)
 
-
 localfs = LocalFilesystem.get_instance()
 
 
diff --git a/python/pyarrow/error.pxi b/python/pyarrow/error.pxi
index 259aeb074e3c2..8a3f57d209ac0 100644
--- a/python/pyarrow/error.pxi
+++ b/python/pyarrow/error.pxi
@@ -48,6 +48,18 @@ class ArrowNotImplementedError(NotImplementedError, ArrowException):
     pass
 
 
+class PlasmaObjectExists(ArrowException):
+    pass
+
+
+class PlasmaObjectNonexistent(ArrowException):
+    pass
+
+
+class PlasmaStoreFull(ArrowException):
+    pass
+
+
 cdef int check_status(const CStatus& status) nogil except -1:
     if status.ok():
         return 0
@@ -66,5 +78,11 @@ cdef int check_status(const CStatus& status) nogil except -1:
             raise ArrowNotImplementedError(message)
         elif status.IsTypeError():
             raise ArrowTypeError(message)
+        elif status.IsPlasmaObjectExists():
+            raise PlasmaObjectExists(message)
+        elif status.IsPlasmaObjectNonexistent():
+            raise PlasmaObjectNonexistent(message)
+        elif status.IsPlasmaStoreFull():
+            raise PlasmaStoreFull(message)
         else:
             raise ArrowException(message)
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 3487d48ce9b52..637a133afb02b 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -50,6 +50,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsKeyError()
         c_bool IsNotImplemented()
         c_bool IsTypeError()
+        c_bool IsPlasmaObjectExists()
+        c_bool IsPlasmaObjectNonexistent()
+        c_bool IsPlasmaStoreFull()
 
 
 cdef inline object PyObject_to_object(PyObject* o):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index edf50ad54e787..ffe867b0af0f5 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -148,9 +148,15 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CLoggingMemoryPool(CMemoryPool*)
 
     cdef cppclass CBuffer" arrow::Buffer":
+        CBuffer(const uint8_t* data, int64_t size)
         uint8_t* data()
         int64_t size()
         shared_ptr[CBuffer] parent()
+        c_bool is_mutable() const
+
+    cdef cppclass CMutableBuffer" arrow::MutableBuffer"(CBuffer):
+        CMutableBuffer(const uint8_t* data, int64_t size)
+        uint8_t* mutable_data()
 
     cdef cppclass ResizableBuffer(CBuffer):
         CStatus Resize(int64_t nbytes)
@@ -558,6 +564,9 @@ cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
         CMockOutputStream()
         int64_t GetExtentBytesWritten()
 
+    cdef cppclass CFixedSizeBufferWriter" arrow::io::FixedSizeBufferWriter"(WriteableFile):
+        CFixedSizeBufferWriter(const shared_ptr[CBuffer]& buffer)
+
 
 cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
     enum MessageType" arrow::ipc::Message::Type":
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 8b213a33053d4..181b0b18a712f 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -473,6 +473,15 @@ cdef class OSFile(NativeFile):
         self.wr_file = <shared_ptr[OutputStream]> handle
 
 
+cdef class FixedSizeBufferOutputStream(NativeFile):
+
+    def __cinit__(self, Buffer buffer):
+        self.wr_file.reset(new CFixedSizeBufferWriter(buffer.buffer))
+        self.is_readable = 0
+        self.is_writeable = 1
+        self.is_open = True
+
+
 # ----------------------------------------------------------------------
 # Arrow buffers
 
@@ -523,7 +532,10 @@ cdef class Buffer:
         buffer.len = self.size
         buffer.ndim = 1
         buffer.obj = self
-        buffer.readonly = 1
+        if self.buffer.get().is_mutable():
+            buffer.readonly = 0
+        else:
+            buffer.readonly = 1
         buffer.shape = self.shape
         buffer.strides = self.strides
         buffer.suboffsets = NULL
@@ -540,6 +552,15 @@ cdef class Buffer:
             p[0] = <void*> self.buffer.get().data()
         return self.size
 
+    def __getwritebuffer__(self, Py_ssize_t idx, void **p):
+        if not self.buffer.get().is_mutable():
+            raise SystemError("trying to write an immutable buffer")
+        if idx != 0:
+            raise SystemError("accessing non-existent buffer segment")
+        if p != NULL:
+            p[0] = <void*> self.buffer.get().data()
+        return self.size
+
 
 cdef shared_ptr[PoolBuffer] allocate_buffer(CMemoryPool* pool):
     cdef shared_ptr[PoolBuffer] result
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
new file mode 100644
index 0000000000000..bb17685277af1
--- /dev/null
+++ b/python/pyarrow/plasma.pyx
@@ -0,0 +1,560 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from libcpp cimport bool as c_bool, nullptr
+from libcpp.memory cimport shared_ptr, unique_ptr, make_shared
+from libcpp.string cimport string as c_string
+from libcpp.vector cimport vector as c_vector
+from libc.stdint cimport int64_t, uint8_t, uintptr_t
+from cpython.pycapsule cimport *
+
+from pyarrow.lib cimport Buffer, NativeFile, check_status
+from pyarrow.includes.libarrow cimport (CMutableBuffer, CBuffer,
+                                        CFixedSizeBufferWriter, CStatus)
+
+
+PLASMA_WAIT_TIMEOUT = 2 ** 30
+
+
+cdef extern from "plasma/common.h" nogil:
+
+    cdef cppclass CUniqueID" plasma::UniqueID":
+
+        @staticmethod
+        CUniqueID from_binary(const c_string& binary)
+
+        c_bool operator==(const CUniqueID& rhs) const
+
+        c_string hex() const
+
+        c_string binary() const
+
+    cdef struct CObjectRequest" plasma::ObjectRequest":
+        CUniqueID object_id
+        int type
+        int status
+
+
+cdef extern from "plasma/common.h":
+    cdef int64_t kDigestSize" plasma::kDigestSize"
+
+    cdef enum ObjectRequestType:
+        PLASMA_QUERY_LOCAL"plasma::PLASMA_QUERY_LOCAL",
+        PLASMA_QUERY_ANYWHERE"plasma::PLASMA_QUERY_ANYWHERE"
+
+    cdef int ObjectStatusLocal"plasma::ObjectStatusLocal";
+    cdef int ObjectStatusRemote"plasma::ObjectStatusRemote";
+
+cdef extern from "plasma/client.h" nogil:
+
+    cdef cppclass CPlasmaClient" plasma::PlasmaClient":
+
+        CPlasmaClient()
+
+        CStatus Connect(const c_string& store_socket_name,
+                        const c_string& manager_socket_name, int release_delay)
+
+        CStatus Create(const CUniqueID& object_id, int64_t data_size,
+                       const uint8_t* metadata, int64_t metadata_size,
+                       uint8_t** data)
+
+        CStatus Get(const CUniqueID* object_ids, int64_t num_objects,
+                    int64_t timeout_ms, CObjectBuffer* object_buffers)
+
+        CStatus Seal(const CUniqueID& object_id)
+
+        CStatus Evict(int64_t num_bytes, int64_t& num_bytes_evicted)
+
+        CStatus Hash(const CUniqueID& object_id, uint8_t* digest)
+
+        CStatus Release(const CUniqueID& object_id)
+
+        CStatus Contains(const CUniqueID& object_id, c_bool* has_object)
+
+        CStatus Subscribe(int* fd)
+
+        CStatus GetNotification(int fd, CUniqueID* object_id,
+                                int64_t* data_size, int64_t* metadata_size)
+
+        CStatus Disconnect()
+
+        CStatus Fetch(int num_object_ids, const CUniqueID* object_ids)
+
+        CStatus Wait(int64_t num_object_requests, CObjectRequest* object_requests,
+           int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
+
+        CStatus Transfer(const char* addr, int port, const CUniqueID& object_id)
+
+
+cdef extern from "plasma/client.h" nogil:
+
+    cdef struct CObjectBuffer" plasma::ObjectBuffer":
+        int64_t data_size
+        uint8_t* data
+        int64_t metadata_size
+        uint8_t* metadata
+
+
+def make_object_id(object_id):
+    return ObjectID(object_id)
+
+
+cdef class ObjectID:
+    """
+    An ObjectID represents a string of bytes used to identify Plasma objects.
+    """
+
+    cdef:
+        CUniqueID data
+
+    def __cinit__(self, object_id):
+        self.data = CUniqueID.from_binary(object_id)
+
+    def __richcmp__(ObjectID self, ObjectID object_id, operation):
+        if operation != 2:
+            raise ValueError("operation != 2 (only equality is supported)")
+        return self.data == object_id.data
+
+    def __hash__(self):
+        return hash(self.data.binary())
+
+    def __repr__(self):
+        return "ObjectID(" + self.data.hex().decode() + ")"
+
+    def __reduce__(self):
+        return (make_object_id, (self.data.binary(),))
+
+    def binary(self):
+        """
+        Return the binary representation of this ObjectID.
+
+        Returns
+        -------
+        bytes
+            Binary representation of the ObjectID.
+        """
+        return self.data.binary()
+
+
+cdef class PlasmaBuffer(Buffer):
+    """
+    This is the type returned by calls to get with a PlasmaClient.
+
+    We define our own class instead of directly returning a buffer object so
+    that we can add a custom destructor which notifies Plasma that the object
+    is no longer being used, so the memory in the Plasma store backing the
+    object can potentially be freed.
+
+    Attributes
+    ----------
+    object_id : ObjectID
+        The ID of the object in the buffer.
+    client : PlasmaClient
+        The PlasmaClient that we use to communicate with the store and manager.
+    """
+
+    cdef:
+        ObjectID object_id
+        PlasmaClient client
+
+    def __cinit__(self, ObjectID object_id, PlasmaClient client):
+        """
+        Initialize a PlasmaBuffer.
+        """
+        self.object_id = object_id
+        self.client = client
+
+    def __dealloc__(self):
+        """
+        Notify Plasma that the object is no longer needed.
+
+        If the plasma client has been shut down, then don't do anything.
+        """
+        self.client.release(self.object_id)
+
+
+cdef class PlasmaClient:
+    """
+    The PlasmaClient is used to interface with a plasma store and manager.
+
+    The PlasmaClient can ask the PlasmaStore to allocate a new buffer, seal a
+    buffer, and get a buffer. Buffers are referred to by object IDs, which are
+    strings.
+    """
+
+    cdef:
+        shared_ptr[CPlasmaClient] client
+        int notification_fd
+        c_string store_socket_name
+        c_string manager_socket_name
+
+    def __cinit__(self, store_socket_name, manager_socket_name, int release_delay):
+        """
+        Create a new PlasmaClient that is connected to a plasma store
+        and optionally a plasma manager.
+
+        Parameters
+        ----------
+        store_socket_name : str
+            Name of the socket the plasma store is listening at.
+        manager_socket_name : str
+            Name of the socket the plasma manager is listening at.
+        release_delay : int
+            The maximum number of objects that the client will keep and
+            delay releasing (for caching reasons).
+        """
+        self.client.reset(new CPlasmaClient())
+        self.notification_fd = -1
+        self.store_socket_name = store_socket_name.encode()
+        self.manager_socket_name = manager_socket_name.encode()
+        with nogil:
+            check_status(self.client.get().Connect(self.store_socket_name,
+                         self.manager_socket_name, release_delay))
+
+    cdef _get_object_buffers(self, object_ids, int64_t timeout_ms,
+                             c_vector[CObjectBuffer]* result):
+        cdef c_vector[CUniqueID] ids
+        cdef ObjectID object_id
+        for object_id in object_ids:
+            ids.push_back(object_id.data)
+        result[0].resize(ids.size())
+        with nogil:
+            check_status(self.client.get().Get(ids.data(), ids.size(),
+                         timeout_ms, result[0].data()))
+
+    cdef _make_plasma_buffer(self, ObjectID object_id, uint8_t* data,
+                             int64_t size):
+        cdef shared_ptr[CBuffer] buffer
+        buffer.reset(new CBuffer(data, size))
+        result = PlasmaBuffer(object_id, self)
+        result.init(buffer)
+        return result
+
+    cdef _make_mutable_plasma_buffer(self, ObjectID object_id, uint8_t* data,
+                                     int64_t size):
+        cdef shared_ptr[CBuffer] buffer
+        buffer.reset(new CMutableBuffer(data, size))
+        result = PlasmaBuffer(object_id, self)
+        result.init(buffer)
+        return result
+
+    @property
+    def store_socket_name(self):
+        return self.store_socket_name.decode()
+
+    @property
+    def manager_socket_name(self):
+        return self.manager_socket_name.decode()
+
+    def create(self, ObjectID object_id, int64_t data_size, c_string metadata=b""):
+        """
+        Create a new buffer in the PlasmaStore for a particular object ID.
+
+        The returned buffer is mutable until seal is called.
+
+        Parameters
+        ----------
+        object_id : ObjectID
+            The object ID used to identify an object.
+        size : int
+            The size in bytes of the created buffer.
+        metadata : bytes
+            An optional string of bytes encoding whatever metadata the user
+            wishes to encode.
+
+        Raises
+        ------
+        PlasmaObjectExists
+            This exception is raised if the object could not be created because
+            there already is an object with the same ID in the plasma store.
+
+        PlasmaStoreFull: This exception is raised if the object could
+                not be created because the plasma store is unable to evict
+                enough objects to create room for it.
+        """
+        cdef uint8_t* data
+        with nogil:
+            check_status(self.client.get().Create(object_id.data, data_size,
+                                                  <uint8_t*>(metadata.data()),
+                                                  metadata.size(), &data))
+        return self._make_mutable_plasma_buffer(object_id, data, data_size)
+
+    def get(self, object_ids, timeout_ms=-1):
+        """
+        Returns data buffer from the PlasmaStore based on object ID.
+
+        If the object has not been sealed yet, this call will block. The
+        retrieved buffer is immutable.
+
+        Parameters
+        ----------
+        object_ids : list
+            A list of ObjectIDs used to identify some objects.
+        timeout_ms :int
+            The number of milliseconds that the get call should block before
+            timing out and returning. Pass -1 if the call should block and 0
+            if the call should return immediately.
+
+        Returns
+        -------
+        list
+            List of PlasmaBuffers for the data associated with the object_ids
+            and None if the object was not available.
+        """
+        cdef c_vector[CObjectBuffer] object_buffers
+        self._get_object_buffers(object_ids, timeout_ms, &object_buffers)
+        result = []
+        for i in range(object_buffers.size()):
+            if object_buffers[i].data_size != -1:
+                result.append(self._make_plasma_buffer(
+                                  object_ids[i], object_buffers[i].data,
+                                  object_buffers[i].data_size))
+            else:
+                result.append(None)
+        return result
+
+    def get_metadata(self, object_ids, timeout_ms=-1):
+        """
+        Returns metadata buffer from the PlasmaStore based on object ID.
+
+        If the object has not been sealed yet, this call will block. The
+        retrieved buffer is immutable.
+
+        Parameters
+        ----------
+        object_ids : list
+            A list of ObjectIDs used to identify some objects.
+        timeout_ms : int
+            The number of milliseconds that the get call should block before
+            timing out and returning. Pass -1 if the call should block and 0
+            if the call should return immediately.
+
+        Returns
+        -------
+        list
+            List of PlasmaBuffers for the metadata associated with the
+            object_ids and None if the object was not available.
+        """
+        cdef c_vector[CObjectBuffer] object_buffers
+        self._get_object_buffers(object_ids, timeout_ms, &object_buffers)
+        result = []
+        for i in range(object_buffers.size()):
+            result.append(self._make_plasma_buffer(
+                              object_ids[i], object_buffers[i].metadata,
+                              object_buffers[i].metadata_size))
+        return result
+
+    def seal(self, ObjectID object_id):
+        """
+        Seal the buffer in the PlasmaStore for a particular object ID.
+
+        Once a buffer has been sealed, the buffer is immutable and can only be
+        accessed through get.
+
+        Parameters
+        ----------
+        object_id : ObjectID
+            A string used to identify an object.
+        """
+        with nogil:
+            check_status(self.client.get().Seal(object_id.data))
+
+    def release(self, ObjectID object_id):
+        """
+        Notify Plasma that the object is no longer needed.
+
+        Parameters
+        ----------
+        object_id : ObjectID
+            A string used to identify an object.
+        """
+        with nogil:
+            check_status(self.client.get().Release(object_id.data))
+
+    def contains(self, ObjectID object_id):
+        """
+        Check if the object is present and sealed in the PlasmaStore.
+
+        Parameters
+        ----------
+        object_id : ObjectID
+            A string used to identify an object.
+        """
+        cdef c_bool is_contained
+        with nogil:
+            check_status(self.client.get().Contains(object_id.data,
+                                                    &is_contained))
+        return is_contained
+
+    def hash(self, ObjectID object_id):
+        """
+        Compute the checksum of an object in the object store.
+
+        Parameters
+        ----------
+        object_id : ObjectID
+            A string used to identify an object.
+
+        Returns
+        -------
+        bytes
+            A digest string object's hash. If the object isn't in the object
+            store, the string will have length zero.
+        """
+        cdef c_vector[uint8_t] digest = c_vector[uint8_t](kDigestSize)
+        with nogil:
+            check_status(self.client.get().Hash(object_id.data,
+                                                digest.data()))
+        return bytes(digest[:])
+
+    def evict(self, int64_t num_bytes):
+        """
+        Evict some objects until to recover some bytes.
+
+        Recover at least num_bytes bytes if possible.
+
+        Parameters
+        ----------
+        num_bytes : int
+            The number of bytes to attempt to recover.
+        """
+        cdef int64_t num_bytes_evicted = -1
+        with nogil:
+            check_status(self.client.get().Evict(num_bytes, num_bytes_evicted))
+        return num_bytes_evicted
+
+    def transfer(self, address, int port, ObjectID object_id):
+        """
+        Transfer local object with id object_id to another plasma instance
+
+        Parameters
+        ----------
+        addr : str
+            IPv4 address of the plasma instance the object is sent to.
+        port : int
+            Port number of the plasma instance the object is sent to.
+        object_id : str
+            A string used to identify an object.
+        """
+        cdef c_string addr = address.encode()
+        with nogil:
+            check_status(self.client.get().Transfer(addr.c_str(), port, object_id.data))
+
+    def fetch(self, object_ids):
+        """
+        Fetch the objects with the given IDs from other plasma managers.
+
+        Parameters
+        ----------
+        object_ids : list
+            A list of strings used to identify the objects.
+        """
+        cdef c_vector[CUniqueID] ids
+        cdef ObjectID object_id
+        for object_id in object_ids:
+            ids.push_back(object_id.data)
+        with nogil:
+            check_status(self.client.get().Fetch(ids.size(), ids.data()))
+
+    def wait(self, object_ids, int64_t timeout=PLASMA_WAIT_TIMEOUT, int num_returns=1):
+        """
+        Wait until num_returns objects in object_ids are ready.
+        Currently, the object ID arguments to wait must be unique.
+
+        Parameters
+        ----------
+        object_ids : list
+            List of object IDs to wait for.
+        timeout :int
+            Return to the caller after timeout milliseconds.
+        num_returns : int
+            We are waiting for this number of objects to be ready.
+
+        Returns
+        -------
+        list
+            List of object IDs that are ready.
+        list
+            List of object IDs we might still wait on.
+        """
+        # Check that the object ID arguments are unique. The plasma manager
+        # currently crashes if given duplicate object IDs.
+        if len(object_ids) != len(set(object_ids)):
+            raise Exception("Wait requires a list of unique object IDs.")
+        cdef int64_t num_object_requests = len(object_ids)
+        cdef c_vector[CObjectRequest] object_requests = c_vector[CObjectRequest](num_object_requests)
+        cdef int num_objects_ready = 0
+        cdef ObjectID object_id
+        for i, object_id in enumerate(object_ids):
+            object_requests[i].object_id = object_id.data
+            object_requests[i].type = PLASMA_QUERY_ANYWHERE
+        with nogil:
+            check_status(self.client.get().Wait(num_object_requests, object_requests.data(), num_returns, timeout, &num_objects_ready))
+        cdef int num_to_return = min(num_objects_ready, num_returns);
+        ready_ids = []
+        waiting_ids = set(object_ids)
+        cdef int num_returned = 0
+        for i in range(len(object_ids)):
+            if num_returned == num_to_return:
+                break
+            if object_requests[i].status == ObjectStatusLocal or object_requests[i].status == ObjectStatusRemote:
+                ready_ids.append(ObjectID(object_requests[i].object_id.binary()))
+                waiting_ids.discard(ObjectID(object_requests[i].object_id.binary()))
+                num_returned += 1
+        return ready_ids, list(waiting_ids)
+
+    def subscribe(self):
+        """Subscribe to notifications about sealed objects."""
+        with nogil:
+            check_status(self.client.get().Subscribe(&self.notification_fd))
+
+    def get_next_notification(self):
+        """
+        Get the next notification from the notification socket.
+
+        Returns
+        -------
+        ObjectID
+            The object ID of the object that was stored.
+        int
+            The data size of the object that was stored.
+        int
+            The metadata size of the object that was stored.
+        """
+        cdef ObjectID object_id = ObjectID(20 * b"\0")
+        cdef int64_t data_size
+        cdef int64_t metadata_size
+        with nogil:
+            check_status(self.client.get().GetNotification(self.notification_fd,
+                                                           &object_id.data,
+                                                           &data_size,
+                                                           &metadata_size))
+        return object_id, data_size, metadata_size
+
+    def to_capsule(self):
+        return PyCapsule_New(<void *>self.client.get(), "plasma", NULL)
+
+    def disconnect(self):
+        """
+        Disconnect this client from the Plasma store.
+        """
+        with nogil:
+            check_status(self.client.get().Disconnect())
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index 2aeeab7294ccc..21288e4f35e74 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -18,11 +18,12 @@
 from pytest import skip
 
 
-groups = ['hdfs', 'parquet', 'large_memory']
+groups = ['hdfs', 'parquet', 'plasma', 'large_memory']
 
 defaults = {
     'hdfs': False,
     'parquet': False,
+    'plasma': False,
     'large_memory': False
 }
 
@@ -32,6 +33,11 @@
 except ImportError:
     pass
 
+try:
+    import pyarrow.plasma as plasma
+    defaults['plasma'] = True
+except ImportError:
+    pass
 
 def pytest_configure(config):
     pass
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
new file mode 100644
index 0000000000000..ce684e3e41f1b
--- /dev/null
+++ b/python/pyarrow/tests/test_plasma.py
@@ -0,0 +1,683 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import glob
+import numpy as np
+import os
+import pytest
+import random
+import signal
+import subprocess
+import sys
+import time
+import unittest
+
+import pyarrow as pa
+import pandas as pd
+
+DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9
+
+def random_name():
+    return str(random.randint(0, 99999999))
+
+
+def random_object_id():
+    import pyarrow.plasma as plasma
+    return plasma.ObjectID(np.random.bytes(20))
+
+
+def generate_metadata(length):
+    metadata = bytearray(length)
+    if length > 0:
+        metadata[0] = random.randint(0, 255)
+        metadata[-1] = random.randint(0, 255)
+        for _ in range(100):
+            metadata[random.randint(0, length - 1)] = random.randint(0, 255)
+    return metadata
+
+
+def write_to_data_buffer(buff, length):
+    array = np.frombuffer(buff, dtype="uint8")
+    if length > 0:
+        array[0] = random.randint(0, 255)
+        array[-1] = random.randint(0, 255)
+        for _ in range(100):
+            array[random.randint(0, length - 1)] = random.randint(0, 255)
+
+
+def create_object_with_id(client, object_id, data_size, metadata_size,
+                          seal=True):
+    metadata = generate_metadata(metadata_size)
+    memory_buffer = client.create(object_id, data_size, metadata)
+    write_to_data_buffer(memory_buffer, data_size)
+    if seal:
+        client.seal(object_id)
+    return memory_buffer, metadata
+
+
+def create_object(client, data_size, metadata_size, seal=True):
+    object_id = random_object_id()
+    memory_buffer, metadata = create_object_with_id(client, object_id,
+                                                    data_size, metadata_size,
+                                                    seal=seal)
+    return object_id, memory_buffer, metadata
+
+
+def assert_get_object_equal(unit_test, client1, client2, object_id,
+                            memory_buffer=None, metadata=None):
+    import pyarrow.plasma as plasma
+    client1_buff = client1.get([object_id])[0]
+    client2_buff = client2.get([object_id])[0]
+    client1_metadata = client1.get_metadata([object_id])[0]
+    client2_metadata = client2.get_metadata([object_id])[0]
+    assert len(client1_buff) == len(client2_buff)
+    assert len(client1_metadata) == len(client2_metadata)
+    # Check that the buffers from the two clients are the same.
+    assert plasma.buffers_equal(client1_buff, client2_buff)
+    # Check that the metadata buffers from the two clients are the same.
+    assert plasma.buffers_equal(client1_metadata, client2_metadata)
+    # If a reference buffer was provided, check that it is the same as well.
+    if memory_buffer is not None:
+        assert plasma.buffers_equal(memory_buffer, client1_buff)
+    # If reference metadata was provided, check that it is the same as well.
+    if metadata is not None:
+        assert plasma.buffers_equal(metadata, client1_metadata)
+
+
+def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
+                       use_valgrind=False, use_profiler=False,
+                       stdout_file=None, stderr_file=None):
+    """Start a plasma store process.
+    Args:
+        use_valgrind (bool): True if the plasma store should be started inside
+            of valgrind. If this is True, use_profiler must be False.
+        use_profiler (bool): True if the plasma store should be started inside
+            a profiler. If this is True, use_valgrind must be False.
+        stdout_file: A file handle opened for writing to redirect stdout to. If
+            no redirection should happen, then this should be None.
+        stderr_file: A file handle opened for writing to redirect stderr to. If
+            no redirection should happen, then this should be None.
+    Return:
+        A tuple of the name of the plasma store socket and the process ID of
+            the plasma store process.
+    """
+    if use_valgrind and use_profiler:
+        raise Exception("Cannot use valgrind and profiler at the same time.")
+    plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store")
+    plasma_store_name = "/tmp/plasma_store{}".format(random_name())
+    command = [plasma_store_executable,
+               "-s", plasma_store_name,
+               "-m", str(plasma_store_memory)]
+    if use_valgrind:
+        pid = subprocess.Popen(["valgrind",
+                                "--track-origins=yes",
+                                "--leak-check=full",
+                                "--show-leak-kinds=all",
+                                "--leak-check-heuristics=stdstring",
+                                "--error-exitcode=1"] + command,
+                               stdout=stdout_file, stderr=stderr_file)
+        time.sleep(1.0)
+    elif use_profiler:
+        pid = subprocess.Popen(["valgrind", "--tool=callgrind"] + command,
+                               stdout=stdout_file, stderr=stderr_file)
+        time.sleep(1.0)
+    else:
+        pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
+        time.sleep(0.1)
+    return plasma_store_name, pid
+
+
+@pytest.mark.plasma
+class TestPlasmaClient(object):
+
+    def setup_method(self, test_method):
+        import pyarrow.plasma as plasma
+        # Start Plasma store.
+        plasma_store_name, self.p = start_plasma_store(
+            use_valgrind=os.getenv("PLASMA_VALGRIND") == "1")
+        # Connect to Plasma.
+        self.plasma_client = plasma.PlasmaClient(plasma_store_name, "", 64)
+        # For the eviction test
+        self.plasma_client2 = plasma.PlasmaClient(plasma_store_name, "", 0)
+
+    def teardown_method(self, test_method):
+        # Check that the Plasma store is still alive.
+        assert self.p.poll() == None
+        # Kill the plasma store process.
+        if os.getenv("PLASMA_VALGRIND") == "1":
+            self.p.send_signal(signal.SIGTERM)
+            self.p.wait()
+            if self.p.returncode != 0:
+                assert False
+        else:
+            self.p.kill()
+
+    def test_create(self):
+        # Create an object id string.
+        object_id = random_object_id()
+        # Create a new buffer and write to it.
+        length = 50
+        memory_buffer = np.frombuffer(self.plasma_client.create(object_id,
+                                                                length),
+                                      dtype="uint8")
+        for i in range(length):
+            memory_buffer[i] = i % 256
+        # Seal the object.
+        self.plasma_client.seal(object_id)
+        # Get the object.
+        memory_buffer = np.frombuffer(self.plasma_client.get([object_id])[0],
+                                      dtype="uint8")
+        for i in range(length):
+            assert memory_buffer[i] == i % 256
+
+    def test_create_with_metadata(self):
+        for length in range(1000):
+            # Create an object id string.
+            object_id = random_object_id()
+            # Create a random metadata string.
+            metadata = generate_metadata(length)
+            # Create a new buffer and write to it.
+            memory_buffer = np.frombuffer(self.plasma_client.create(object_id,
+                                                                    length,
+                                                                    metadata),
+                                          dtype="uint8")
+            for i in range(length):
+                memory_buffer[i] = i % 256
+            # Seal the object.
+            self.plasma_client.seal(object_id)
+            # Get the object.
+            memory_buffer = np.frombuffer(
+                self.plasma_client.get([object_id])[0], dtype="uint8")
+            for i in range(length):
+                assert memory_buffer[i] == i % 256
+            # Get the metadata.
+            metadata_buffer = np.frombuffer(
+                self.plasma_client.get_metadata([object_id])[0], dtype="uint8")
+            assert len(metadata) == len(metadata_buffer)
+            for i in range(len(metadata)):
+                assert metadata[i] == metadata_buffer[i]
+
+    def test_create_existing(self):
+        # This test is partially used to test the code path in which we create
+        # an object with an ID that already exists
+        length = 100
+        for _ in range(1000):
+            object_id = random_object_id()
+            self.plasma_client.create(object_id, length,
+                                      generate_metadata(length))
+            try:
+                self.plasma_client.create(object_id, length,
+                                          generate_metadata(length))
+            # TODO(pcm): Introduce a more specific error type here.
+            except pa.lib.ArrowException as e:
+                pass
+            else:
+                assert False
+
+    def test_get(self):
+        num_object_ids = 100
+        # Test timing out of get with various timeouts.
+        for timeout in [0, 10, 100, 1000]:
+            object_ids = [random_object_id() for _ in range(num_object_ids)]
+            results = self.plasma_client.get(object_ids, timeout_ms=timeout)
+            assert results == num_object_ids * [None]
+
+        data_buffers = []
+        metadata_buffers = []
+        for i in range(num_object_ids):
+            if i % 2 == 0:
+                data_buffer, metadata_buffer = create_object_with_id(
+                    self.plasma_client, object_ids[i], 2000, 2000)
+                data_buffers.append(data_buffer)
+                metadata_buffers.append(metadata_buffer)
+
+        # Test timing out from some but not all get calls with various
+        # timeouts.
+        for timeout in [0, 10, 100, 1000]:
+            data_results = self.plasma_client.get(object_ids,
+                                                  timeout_ms=timeout)
+            # metadata_results = self.plasma_client.get_metadata(
+            #     object_ids, timeout_ms=timeout)
+            for i in range(num_object_ids):
+                if i % 2 == 0:
+                    array1 = np.frombuffer(data_buffers[i // 2], dtype="uint8")
+                    array2 = np.frombuffer(data_results[i], dtype="uint8")
+                    np.testing.assert_equal(array1, array2)
+                    # TODO(rkn): We should compare the metadata as well. But
+                    # currently the types are different (e.g., memoryview
+                    # versus bytearray).
+                    # assert plasma.buffers_equal(
+                    #     metadata_buffers[i // 2], metadata_results[i])
+                else:
+                    assert results[i] is None
+
+    def test_store_arrow_objects(self):
+        import pyarrow.plasma as plasma
+        data = np.random.randn(10, 4)
+        # Write an arrow object.
+        object_id = random_object_id()
+        tensor = pa.Tensor.from_numpy(data)
+        data_size = pa.get_tensor_size(tensor)
+        buf = self.plasma_client.create(object_id, data_size)
+        stream = pa.FixedSizeBufferOutputStream(buf)
+        pa.write_tensor(tensor, stream)
+        self.plasma_client.seal(object_id)
+        # Read the arrow object.
+        [tensor] = self.plasma_client.get([object_id])
+        reader = pa.BufferReader(tensor)
+        array = pa.read_tensor(reader).to_numpy()
+        # Assert that they are equal.
+        np.testing.assert_equal(data, array)
+
+    def test_store_pandas_dataframe(self):
+        import pyarrow.plasma as plasma
+        d = {'one': pd.Series([1., 2., 3.], index=['a', 'b', 'c']),
+             'two': pd.Series([1., 2., 3., 4.], index=['a', 'b', 'c', 'd'])}
+        df = pd.DataFrame(d)
+
+        # Write the DataFrame.
+        record_batch = pa.RecordBatch.from_pandas(df)
+        # Determine the size.
+        s = pa.MockOutputStream()
+        stream_writer = pa.RecordBatchStreamWriter(s, record_batch.schema)
+        stream_writer.write_batch(record_batch)
+        data_size = s.size()
+        object_id = plasma.ObjectID(np.random.bytes(20))
+
+        buf = self.plasma_client.create(object_id, data_size)
+        stream = pa.FixedSizeBufferOutputStream(buf)
+        stream_writer = pa.RecordBatchStreamWriter(stream, record_batch.schema)
+        stream_writer.write_batch(record_batch)
+
+        self.plasma_client.seal(object_id)
+
+        # Read the DataFrame.
+        [data] = self.plasma_client.get([object_id])
+        reader = pa.RecordBatchStreamReader(pa.BufferReader(data))
+        result = reader.get_next_batch().to_pandas()
+
+        pd.util.testing.assert_frame_equal(df, result)
+
+    def test_pickle_object_ids(self):
+        # This can be used for sharing object IDs between processes.
+        import pickle
+        object_id = random_object_id()
+        data = pickle.dumps(object_id)
+        object_id2 = pickle.loads(data)
+        assert object_id == object_id2
+
+    def test_store_full(self):
+        # The store is started with 1GB, so make sure that create throws an
+        # exception when it is full.
+        def assert_create_raises_plasma_full(unit_test, size):
+            partial_size = np.random.randint(size)
+            try:
+                _, memory_buffer, _ = create_object(unit_test.plasma_client,
+                                                    partial_size,
+                                                    size - partial_size)
+            # TODO(pcm): More specific error here.
+            except pa.lib.ArrowException as e:
+                pass
+            else:
+                # For some reason the above didn't throw an exception, so fail.
+                assert False
+
+        # Create a list to keep some of the buffers in scope.
+        memory_buffers = []
+        _, memory_buffer, _ = create_object(self.plasma_client, 5 * 10 ** 8, 0)
+        memory_buffers.append(memory_buffer)
+        # Remaining space is 5 * 10 ** 8. Make sure that we can't create an
+        # object of size 5 * 10 ** 8 + 1, but we can create one of size
+        # 2 * 10 ** 8.
+        assert_create_raises_plasma_full(self, 5 * 10 ** 8 + 1)
+        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
+        del memory_buffer
+        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
+        del memory_buffer
+        assert_create_raises_plasma_full(self, 5 * 10 ** 8 + 1)
+
+        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
+        memory_buffers.append(memory_buffer)
+        # Remaining space is 3 * 10 ** 8.
+        assert_create_raises_plasma_full(self, 3 * 10 ** 8 + 1)
+
+        _, memory_buffer, _ = create_object(self.plasma_client, 10 ** 8, 0)
+        memory_buffers.append(memory_buffer)
+        # Remaining space is 2 * 10 ** 8.
+        assert_create_raises_plasma_full(self, 2 * 10 ** 8 + 1)
+
+    def test_contains(self):
+        fake_object_ids = [random_object_id() for _ in range(100)]
+        real_object_ids = [random_object_id() for _ in range(100)]
+        for object_id in real_object_ids:
+            assert self.plasma_client.contains(object_id) == False
+            self.plasma_client.create(object_id, 100)
+            self.plasma_client.seal(object_id)
+            assert self.plasma_client.contains(object_id)
+        for object_id in fake_object_ids:
+            assert not self.plasma_client.contains(object_id)
+        for object_id in real_object_ids:
+            assert self.plasma_client.contains(object_id)
+
+    def test_hash(self):
+        # Check the hash of an object that doesn't exist.
+        object_id1 = random_object_id()
+        try:
+            self.plasma_client.hash(object_id1)
+            # TODO(pcm): Introduce a more specific error type here
+        except pa.lib.ArrowException as e:
+            pass
+        else:
+            assert False
+
+        length = 1000
+        # Create a random object, and check that the hash function always
+        # returns the same value.
+        metadata = generate_metadata(length)
+        memory_buffer = np.frombuffer(self.plasma_client.create(object_id1,
+                                                                length,
+                                                                metadata),
+                                      dtype="uint8")
+        for i in range(length):
+            memory_buffer[i] = i % 256
+        self.plasma_client.seal(object_id1)
+        assert (self.plasma_client.hash(object_id1) ==
+                self.plasma_client.hash(object_id1))
+
+        # Create a second object with the same value as the first, and check
+        # that their hashes are equal.
+        object_id2 = random_object_id()
+        memory_buffer = np.frombuffer(self.plasma_client.create(object_id2,
+                                                                length,
+                                                                metadata),
+                                      dtype="uint8")
+        for i in range(length):
+            memory_buffer[i] = i % 256
+        self.plasma_client.seal(object_id2)
+        assert (self.plasma_client.hash(object_id1) ==
+                self.plasma_client.hash(object_id2))
+
+        # Create a third object with a different value from the first two, and
+        # check that its hash is different.
+        object_id3 = random_object_id()
+        metadata = generate_metadata(length)
+        memory_buffer = np.frombuffer(self.plasma_client.create(object_id3,
+                                                                length,
+                                                                metadata),
+                                      dtype="uint8")
+        for i in range(length):
+            memory_buffer[i] = (i + 1) % 256
+        self.plasma_client.seal(object_id3)
+        assert (self.plasma_client.hash(object_id1) !=
+                self.plasma_client.hash(object_id3))
+
+        # Create a fourth object with the same value as the third, but
+        # different metadata. Check that its hash is different from any of the
+        # previous three.
+        object_id4 = random_object_id()
+        metadata4 = generate_metadata(length)
+        memory_buffer = np.frombuffer(self.plasma_client.create(object_id4,
+                                                                length,
+                                                                metadata4),
+                                      dtype="uint8")
+        for i in range(length):
+            memory_buffer[i] = (i + 1) % 256
+        self.plasma_client.seal(object_id4)
+        assert (self.plasma_client.hash(object_id1) !=
+                self.plasma_client.hash(object_id4))
+        assert (self.plasma_client.hash(object_id3) !=
+                self.plasma_client.hash(object_id4))
+
+    def test_many_hashes(self):
+        hashes = []
+        length = 2 ** 10
+
+        for i in range(256):
+            object_id = random_object_id()
+            memory_buffer = np.frombuffer(self.plasma_client.create(object_id,
+                                                                    length),
+                                          dtype="uint8")
+            for j in range(length):
+                memory_buffer[j] = i
+            self.plasma_client.seal(object_id)
+            hashes.append(self.plasma_client.hash(object_id))
+
+        # Create objects of varying length. Each pair has two bits different.
+        for i in range(length):
+            object_id = random_object_id()
+            memory_buffer = np.frombuffer(self.plasma_client.create(object_id,
+                                                                    length),
+                                          dtype="uint8")
+            for j in range(length):
+                memory_buffer[j] = 0
+            memory_buffer[i] = 1
+            self.plasma_client.seal(object_id)
+            hashes.append(self.plasma_client.hash(object_id))
+
+        # Create objects of varying length, all with value 0.
+        for i in range(length):
+            object_id = random_object_id()
+            memory_buffer = np.frombuffer(self.plasma_client.create(object_id,
+                                                                    i),
+                                          dtype="uint8")
+            for j in range(i):
+                memory_buffer[j] = 0
+            self.plasma_client.seal(object_id)
+            hashes.append(self.plasma_client.hash(object_id))
+
+        # Check that all hashes were unique.
+        assert len(set(hashes)) == 256 + length + length
+
+    # def test_individual_delete(self):
+    #     length = 100
+    #     # Create an object id string.
+    #     object_id = random_object_id()
+    #     # Create a random metadata string.
+    #     metadata = generate_metadata(100)
+    #     # Create a new buffer and write to it.
+    #     memory_buffer = self.plasma_client.create(object_id, length,
+    #                                               metadata)
+    #     for i in range(length):
+    #         memory_buffer[i] = chr(i % 256)
+    #     # Seal the object.
+    #     self.plasma_client.seal(object_id)
+    #     # Check that the object is present.
+    #     assert self.plasma_client.contains(object_id)
+    #     # Delete the object.
+    #     self.plasma_client.delete(object_id)
+    #     # Make sure the object is no longer present.
+    #     self.assertFalse(self.plasma_client.contains(object_id))
+    #
+    # def test_delete(self):
+    #     # Create some objects.
+    #     object_ids = [random_object_id() for _ in range(100)]
+    #     for object_id in object_ids:
+    #         length = 100
+    #         # Create a random metadata string.
+    #         metadata = generate_metadata(100)
+    #         # Create a new buffer and write to it.
+    #         memory_buffer = self.plasma_client.create(object_id, length,
+    #                                                   metadata)
+    #         for i in range(length):
+    #             memory_buffer[i] = chr(i % 256)
+    #         # Seal the object.
+    #         self.plasma_client.seal(object_id)
+    #         # Check that the object is present.
+    #         assert self.plasma_client.contains(object_id)
+    #
+    #     # Delete the objects and make sure they are no longer present.
+    #     for object_id in object_ids:
+    #         # Delete the object.
+    #         self.plasma_client.delete(object_id)
+    #         # Make sure the object is no longer present.
+    #         self.assertFalse(self.plasma_client.contains(object_id))
+
+    def test_illegal_functionality(self):
+        # Create an object id string.
+        object_id = random_object_id()
+        # Create a new buffer and write to it.
+        length = 1000
+        memory_buffer = self.plasma_client.create(object_id, length)
+        # Make sure we cannot access memory out of bounds.
+        with pytest.raises(Exception):
+            memory_buffer[length]
+        # Seal the object.
+        self.plasma_client.seal(object_id)
+        # This test is commented out because it currently fails.
+        # # Make sure the object is ready only now.
+        # def illegal_assignment():
+        #     memory_buffer[0] = chr(0)
+        # with pytest.raises(Exception):
+        # illegal_assignment()
+        # Get the object.
+        memory_buffer = self.plasma_client.get([object_id])[0]
+
+        # Make sure the object is read only.
+        def illegal_assignment():
+            memory_buffer[0] = chr(0)
+        with pytest.raises(Exception):
+            illegal_assignment()
+
+    def test_evict(self):
+        client = self.plasma_client2
+        object_id1 = random_object_id()
+        b1 = client.create(object_id1, 1000)
+        client.seal(object_id1)
+        del b1
+        assert client.evict(1) == 1000
+
+        object_id2 = random_object_id()
+        object_id3 = random_object_id()
+        b2 = client.create(object_id2, 999)
+        b3 = client.create(object_id3, 998)
+        client.seal(object_id3)
+        del b3
+        assert client.evict(1000) == 998
+
+        object_id4 = random_object_id()
+        b4 = client.create(object_id4, 997)
+        client.seal(object_id4)
+        del b4
+        client.seal(object_id2)
+        del b2
+        assert client.evict(1) == 997
+        assert client.evict(1) == 999
+
+        object_id5 = random_object_id()
+        object_id6 = random_object_id()
+        object_id7 = random_object_id()
+        b5 = client.create(object_id5, 996)
+        b6 = client.create(object_id6, 995)
+        b7 = client.create(object_id7, 994)
+        client.seal(object_id5)
+        client.seal(object_id6)
+        client.seal(object_id7)
+        del b5
+        del b6
+        del b7
+        assert client.evict(2000) == 996 + 995 + 994
+
+    def test_subscribe(self):
+        # Subscribe to notifications from the Plasma Store.
+        self.plasma_client.subscribe()
+        for i in [1, 10, 100, 1000, 10000, 100000]:
+            object_ids = [random_object_id() for _ in range(i)]
+            metadata_sizes = [np.random.randint(1000) for _ in range(i)]
+            data_sizes = [np.random.randint(1000) for _ in range(i)]
+            for j in range(i):
+                self.plasma_client.create(
+                    object_ids[j], data_sizes[j],
+                    metadata=bytearray(np.random.bytes(metadata_sizes[j])))
+                self.plasma_client.seal(object_ids[j])
+            # Check that we received notifications for all of the objects.
+            for j in range(i):
+                notification_info = self.plasma_client.get_next_notification()
+                recv_objid, recv_dsize, recv_msize = notification_info
+                assert object_ids[j] == recv_objid
+                assert data_sizes[j] == recv_dsize
+                assert metadata_sizes[j] == recv_msize
+
+    def test_subscribe_deletions(self):
+        # Subscribe to notifications from the Plasma Store. We use
+        # plasma_client2 to make sure that all used objects will get evicted
+        # properly.
+        self.plasma_client2.subscribe()
+        for i in [1, 10, 100, 1000, 10000, 100000]:
+            object_ids = [random_object_id() for _ in range(i)]
+            # Add 1 to the sizes to make sure we have nonzero object sizes.
+            metadata_sizes = [np.random.randint(1000) + 1 for _ in range(i)]
+            data_sizes = [np.random.randint(1000) + 1 for _ in range(i)]
+            for j in range(i):
+                x = self.plasma_client2.create(
+                        object_ids[j], data_sizes[j],
+                        metadata=bytearray(np.random.bytes(metadata_sizes[j])))
+                self.plasma_client2.seal(object_ids[j])
+            del x
+            # Check that we received notifications for creating all of the
+            # objects.
+            for j in range(i):
+                notification_info = self.plasma_client2.get_next_notification()
+                recv_objid, recv_dsize, recv_msize = notification_info
+                assert object_ids[j] == recv_objid
+                assert data_sizes[j] == recv_dsize
+                assert metadata_sizes[j] == recv_msize
+
+            # Check that we receive notifications for deleting all objects, as
+            # we evict them.
+            for j in range(i):
+                assert (self.plasma_client2.evict(1) ==
+                        data_sizes[j] + metadata_sizes[j])
+                notification_info = self.plasma_client2.get_next_notification()
+                recv_objid, recv_dsize, recv_msize = notification_info
+                assert object_ids[j] == recv_objid
+                assert -1 == recv_dsize
+                assert -1 == recv_msize
+
+        # Test multiple deletion notifications. The first 9 object IDs have
+        # size 0, and the last has a nonzero size. When Plasma evicts 1 byte,
+        # it will evict all objects, so we should receive deletion
+        # notifications for each.
+        num_object_ids = 10
+        object_ids = [random_object_id() for _ in range(num_object_ids)]
+        metadata_sizes = [0] * (num_object_ids - 1)
+        data_sizes = [0] * (num_object_ids - 1)
+        metadata_sizes.append(np.random.randint(1000))
+        data_sizes.append(np.random.randint(1000))
+        for i in range(num_object_ids):
+            x = self.plasma_client2.create(
+                    object_ids[i], data_sizes[i],
+                    metadata=bytearray(np.random.bytes(metadata_sizes[i])))
+            self.plasma_client2.seal(object_ids[i])
+        del x
+        for i in range(num_object_ids):
+            notification_info = self.plasma_client2.get_next_notification()
+            recv_objid, recv_dsize, recv_msize = notification_info
+            assert object_ids[i] == recv_objid
+            assert data_sizes[i] == recv_dsize
+            assert metadata_sizes[i] == recv_msize
+        assert (self.plasma_client2.evict(1) ==
+                data_sizes[-1] + metadata_sizes[-1])
+        for i in range(num_object_ids):
+            notification_info = self.plasma_client2.get_next_notification()
+            recv_objid, recv_dsize, recv_msize = notification_info
+            assert object_ids[i] == recv_objid
+            assert -1 == recv_dsize
+            assert -1 == recv_msize
diff --git a/python/setup.py b/python/setup.py
index 1ea57ae2d858d..7425b71916001 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -99,6 +99,10 @@ def initialize_options(self):
 
         self.with_parquet = strtobool(
             os.environ.get('PYARROW_WITH_PARQUET', '0'))
+        self.with_plasma = strtobool(
+            os.environ.get('PYARROW_WITH_PLASMA', '0'))
+        if self.with_plasma and "plasma" not in self.CYTHON_MODULE_NAMES:
+            self.CYTHON_MODULE_NAMES.append("plasma")
         self.bundle_arrow_cpp = strtobool(
             os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
 
@@ -242,6 +246,8 @@ def move_lib(lib_name):
             shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
             move_lib("arrow")
             move_lib("arrow_python")
+            if self.with_plasma:
+                move_lib("plasma")
             if self.with_parquet:
                 move_lib("parquet")
 
@@ -270,11 +276,20 @@ def move_lib(lib_name):
                 shutil.move(self.get_ext_built_api_header(name),
                             pjoin(os.path.dirname(ext_path), name + '_api.h'))
 
+        # Move the plasma store
+        if self.with_plasma:
+            build_py = self.get_finalized_command('build_py')
+            source = os.path.join(self.build_type, "plasma_store")
+            target = os.path.join(build_lib, build_py.get_package_dir('pyarrow'), "plasma_store")
+            shutil.move(source, target)
+
         os.chdir(saved_cwd)
 
     def _failure_permitted(self, name):
         if name == '_parquet' and not self.with_parquet:
             return True
+        if name == 'plasma' and not self.with_plasma:
+            return True
         return False
 
     def _get_inplace_dir(self):

From 6042c48952306bbc091055de812ad5b1f1b56818 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Mon, 24 Jul 2017 17:27:09 -0400
Subject: [PATCH 0845/1644] =?UTF-8?q?ARROW-1195:=20[C++]=20CpuInfo=20init?=
 =?UTF-8?q?=20with=20cores=20number,=20frequency=20and=20cache=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… sizes on Windows

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #877 from MaxRis/ARROW-1195 and squashes the following commits:

675d5846 [Max Risuhin] ARROW-1195: [C++] CpuInfo init with cores number, frequency and cache sizes on Windows
---
 cpp/src/arrow/util/cpu-info.cc | 87 +++++++++++++++++++++++++++++-----
 cpp/src/arrow/util/cpu-info.h  |  3 ++
 2 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/cpp/src/arrow/util/cpu-info.cc b/cpp/src/arrow/util/cpu-info.cc
index c0fc8bdddf4bf..dcd6b4027d966 100644
--- a/cpp/src/arrow/util/cpu-info.cc
+++ b/cpp/src/arrow/util/cpu-info.cc
@@ -30,6 +30,10 @@
 #include <unistd.h>
 #endif
 
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
 #include <boost/algorithm/string.hpp>
 
 #include <algorithm>
@@ -79,6 +83,45 @@ int64_t ParseCPUFlags(const string& values) {
   return flags;
 }
 
+#ifdef _WIN32
+bool RetrieveCacheSize(int64_t* cache_sizes) {
+  if (!cache_sizes) { return false; }
+  PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = nullptr;
+  PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer_position = nullptr;
+  DWORD buffer_size = 0;
+  DWORD offset = 0;
+  typedef BOOL(WINAPI * GetLogicalProcessorInformationFuncPointer)(void*, void*);
+  GetLogicalProcessorInformationFuncPointer func_pointer =
+      (GetLogicalProcessorInformationFuncPointer)GetProcAddress(
+          GetModuleHandle("kernel32"), "GetLogicalProcessorInformation");
+
+  if (!func_pointer) { return false; }
+
+  // Get buffer size
+  if (func_pointer(buffer, &buffer_size) && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
+    return false;
+
+  buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(buffer_size);
+
+  if (!buffer || !func_pointer(buffer, &buffer_size)) { return false; }
+
+  buffer_position = buffer;
+  while (offset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= buffer_size) {
+    if (RelationCache == buffer_position->Relationship) {
+      PCACHE_DESCRIPTOR cache = &buffer_position->Cache;
+      if (cache->Level >= 1 && cache->Level <= 3) {
+        cache_sizes[cache->Level - 1] += cache->Size;
+      }
+    }
+    offset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+    buffer_position++;
+  }
+
+  if (buffer) { free(buffer); }
+  return true;
+}
+#endif
+
 void CpuInfo::Init() {
   std::lock_guard<std::mutex> cpuinfo_lock(cpuinfo_mutex);
 
@@ -93,6 +136,16 @@ void CpuInfo::Init() {
 
   memset(&cache_sizes_, 0, sizeof(cache_sizes_));
 
+#ifdef _WIN32
+  SYSTEM_INFO system_info;
+  GetSystemInfo(&system_info);
+  num_cores = system_info.dwNumberOfProcessors;
+
+  LARGE_INTEGER performance_frequency;
+  if (QueryPerformanceFrequency(&performance_frequency)) {
+    max_mhz = static_cast<float>(performance_frequency.QuadPart);
+  }
+#else
   // Read from /proc/cpuinfo
   std::ifstream cpuinfo("/proc/cpuinfo", std::ios::in);
   while (cpuinfo) {
@@ -120,6 +173,7 @@ void CpuInfo::Init() {
     }
   }
   if (cpuinfo.is_open()) cpuinfo.close();
+#endif
 
 #ifdef __APPLE__
   // On Mac OS X use sysctl() to get the cache sizes
@@ -131,22 +185,17 @@ void CpuInfo::Init() {
   for (size_t i = 0; i < 3; ++i) {
     cache_sizes_[i] = data[i];
   }
+#elif _WIN32
+  if (!RetrieveCacheSize(cache_sizes_)) { SetDefaultCacheSize(); }
 #else
-#ifndef _SC_LEVEL1_DCACHE_SIZE
-  // Provide reasonable default values if no info
-  cache_sizes_[0] = 32 * 1024;    // Level 1: 32k
-  cache_sizes_[1] = 256 * 1024;   // Level 2: 256k
-  cache_sizes_[2] = 3072 * 1024;  // Level 3: 3M
-#else
-  // Call sysconf to query for the cache sizes
-  cache_sizes_[0] = sysconf(_SC_LEVEL1_DCACHE_SIZE);
-  cache_sizes_[1] = sysconf(_SC_LEVEL2_CACHE_SIZE);
-  cache_sizes_[2] = sysconf(_SC_LEVEL3_CACHE_SIZE);
-#endif
+  SetDefaultCacheSize();
 #endif
 
   if (max_mhz != 0) {
-    cycles_per_ms_ = static_cast<int64_t>(max_mhz) * 1000;
+    cycles_per_ms_ = static_cast<int64_t>(max_mhz);
+#ifndef _WIN32
+    cycles_per_ms_ *= 1000;
+#endif
   } else {
     cycles_per_ms_ = 1000000;
   }
@@ -203,4 +252,18 @@ std::string CpuInfo::model_name() {
   return model_name_;
 }
 
+void CpuInfo::SetDefaultCacheSize() {
+#ifndef _SC_LEVEL1_DCACHE_SIZE
+  // Provide reasonable default values if no info
+  cache_sizes_[0] = 32 * 1024;    // Level 1: 32k
+  cache_sizes_[1] = 256 * 1024;   // Level 2: 256k
+  cache_sizes_[2] = 3072 * 1024;  // Level 3: 3M
+#else
+  // Call sysconf to query for the cache sizes
+  cache_sizes_[0] = sysconf(_SC_LEVEL1_DCACHE_SIZE);
+  cache_sizes_[1] = sysconf(_SC_LEVEL2_CACHE_SIZE);
+  cache_sizes_[2] = sysconf(_SC_LEVEL3_CACHE_SIZE);
+#endif
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/cpu-info.h b/cpp/src/arrow/util/cpu-info.h
index 06800fc275572..f4bc8c35e3447 100644
--- a/cpp/src/arrow/util/cpu-info.h
+++ b/cpp/src/arrow/util/cpu-info.h
@@ -78,6 +78,9 @@ class ARROW_EXPORT CpuInfo {
   static bool initialized() { return initialized_; }
 
  private:
+  /// Inits CPU cache size variables with default values
+  static void SetDefaultCacheSize();
+
   static bool initialized_;
   static int64_t hardware_flags_;
   static int64_t original_hardware_flags_;

From ecdc86ba45c9dc1739587ec664080e9f2a9b479f Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 24 Jul 2017 21:39:02 -0400
Subject: [PATCH 0846/1644] ARROW-1249: [JAVA] expose fillEmpties from Nullable
 variable length vectors

This will allow us to do some cleanup in Dremio where we have written wrapper routines using Reflection to access the fillEmpties method of mutator.

Unit tests have been added.

Author: siddharth <siddharth@dremio.com>

Closes #880 from siddharthteotia/ARROW-1249 and squashes the following commits:

e0532c5c [siddharth] Merge branch 'ARROW-1249' of https://github.com/siddharthteotia/arrow into ARROW-1249
dc052061 [siddharth] ARROW-1249: Expose fillEmpties() from Nullable Variable Length Vectors
f24d8f11 [siddharth] ARROW-1249: expose fillEmpties from Nullable variable length vectors
---
 .../templates/NullableValueVectors.java       |  2 +-
 .../apache/arrow/vector/TestValueVector.java  | 97 +++++++++++++++++++
 2 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 1decd0b313802..5b993678012b5 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -540,7 +540,7 @@ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.widt
 
     <#if type.major == "VarLen">
 
-    private void fillEmpties(int index){
+    public void fillEmpties(int index){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       for (int i = lastSet + 1; i < index; i++) {
         valuesMutator.setSafe(i, emptyByteArray);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index f5508aab2ce1d..0f41c2dd790e1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -639,6 +639,103 @@ public void testVectorLoadUnload() {
     }
   }
 
+  @Test
+  public void testFillEmptiesUsage() {
+    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
+
+      final NullableVarCharVector.Mutator mutator = vector.getMutator();
+
+      vector.allocateNew(1024 * 10, 1024);
+
+      setBytes(0, STR1, vector);
+      setBytes(1, STR2, vector);
+      setBytes(2, STR3, vector);
+      setBytes(3, STR4, vector);
+      setBytes(4, STR5, vector);
+      setBytes(5, STR6, vector);
+
+      /* Check current lastSet */
+      assertEquals(Integer.toString(-1), Integer.toString(mutator.getLastSet()));
+
+      /* Check the vector output */
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(STR4, accessor.get(3));
+      assertArrayEquals(STR5, accessor.get(4));
+      assertArrayEquals(STR6, accessor.get(5));
+
+      mutator.setLastSet(5);
+      /* fill empty byte arrays from index [6, 9] */
+      mutator.fillEmpties(10);
+
+      /* Check current lastSet */
+      assertEquals(Integer.toString(9), Integer.toString(mutator.getLastSet()));
+
+      /* Check the vector output */
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(STR4, accessor.get(3));
+      assertArrayEquals(STR5, accessor.get(4));
+      assertArrayEquals(STR6, accessor.get(5));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(6)));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(7)));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(8)));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(9)));
+
+      setBytes(10, STR1, vector);
+      setBytes(11, STR2, vector);
+
+      mutator.setLastSet(11);
+      /* fill empty byte arrays from index [12, 14] */
+      mutator.setValueCount(15);
+
+      /* Check current lastSet */
+      assertEquals(Integer.toString(14), Integer.toString(mutator.getLastSet()));
+
+      /* Check the vector output */
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(STR4, accessor.get(3));
+      assertArrayEquals(STR5, accessor.get(4));
+      assertArrayEquals(STR6, accessor.get(5));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(6)));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(7)));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(8)));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(9)));
+      assertArrayEquals(STR1, accessor.get(10));
+      assertArrayEquals(STR2, accessor.get(11));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(12)));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(13)));
+      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(14)));
+
+      /* Check offsets */
+      final UInt4Vector.Accessor offsetAccessor = vector.values.offsetVector.getAccessor();
+      assertEquals(Integer.toString(0), Integer.toString(offsetAccessor.get(0)));
+      assertEquals(Integer.toString(6), Integer.toString(offsetAccessor.get(1)));
+      assertEquals(Integer.toString(16), Integer.toString(offsetAccessor.get(2)));
+      assertEquals(Integer.toString(21), Integer.toString(offsetAccessor.get(3)));
+      assertEquals(Integer.toString(30), Integer.toString(offsetAccessor.get(4)));
+      assertEquals(Integer.toString(34), Integer.toString(offsetAccessor.get(5)));
+
+      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(6)));
+      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(7)));
+      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(8)));
+      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(9)));
+      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(10)));
+
+      assertEquals(Integer.toString(46), Integer.toString(offsetAccessor.get(11)));
+      assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(12)));
+
+      assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(13)));
+      assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(14)));
+      assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(15)));
+    }
+  }
+
   public static void setBytes(int index, byte[] bytes, NullableVarCharVector vector) {
     final int currentOffset = vector.values.offsetVector.getAccessor().get(index);
 

From 886e2af77d29166b5f45829e0d76e33529e00f15 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 24 Jul 2017 21:41:08 -0400
Subject: [PATCH 0847/1644] ARROW-1259: [Plasma] Speed up plasma tests

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #882 from pcmoritz/plasma-test-speedup and squashes the following commits:

09ffbdfd [Philipp Moritz] speed up plasma tests
---
 python/pyarrow/tests/test_plasma.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index ce684e3e41f1b..ab64463e93111 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -598,7 +598,7 @@ def test_evict(self):
     def test_subscribe(self):
         # Subscribe to notifications from the Plasma Store.
         self.plasma_client.subscribe()
-        for i in [1, 10, 100, 1000, 10000, 100000]:
+        for i in [1, 10, 100, 1000, 10000]:
             object_ids = [random_object_id() for _ in range(i)]
             metadata_sizes = [np.random.randint(1000) for _ in range(i)]
             data_sizes = [np.random.randint(1000) for _ in range(i)]
@@ -620,7 +620,7 @@ def test_subscribe_deletions(self):
         # plasma_client2 to make sure that all used objects will get evicted
         # properly.
         self.plasma_client2.subscribe()
-        for i in [1, 10, 100, 1000, 10000, 100000]:
+        for i in [1, 10, 100, 1000, 10000]:
             object_ids = [random_object_id() for _ in range(i)]
             # Add 1 to the sizes to make sure we have nonzero object sizes.
             metadata_sizes = [np.random.randint(1000) + 1 for _ in range(i)]

From 9e692af8b267c9c676e568baa5f45bdb435b7b62 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 24 Jul 2017 21:41:58 -0400
Subject: [PATCH 0848/1644] ARROW-1245: [Integration] Enable JavaTester in
 Integration tests

JavaTester was commented out, probably accidentally from a previous commit, this re-enables it.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #875 from BryanCutler/enable-java-integration-ARROW-1245 and squashes the following commits:

c08c6e23 [Bryan Cutler] enabled JavaTester in integration tests
---
 integration/integration_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 215ba58232a00..b7f1609935e79 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -945,7 +945,7 @@ def get_static_json_files():
 
 
 def run_all_tests(debug=False):
-    testers = [CPPTester(debug=debug)]  # , JavaTester(debug=debug)]
+    testers = [CPPTester(debug=debug), JavaTester(debug=debug)]
     static_json_files = get_static_json_files()
     generated_json_files = get_generated_json_files()
     json_files = static_json_files + generated_json_files

From 11c92bf282de5dad5b40f0e483a241876b807ddd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 24 Jul 2017 21:54:30 -0400
Subject: [PATCH 0849/1644] ARROW-1246: [Format] Draft Flatbuffer metadata
 description for Map

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #876 from wesm/ARROW-1246 and squashes the following commits:

98790dfc [Wes McKinney] Review feedback to clarify nullability of map components
346b48dd [Wes McKinney] Typo
06ae8ebf [Wes McKinney] Draft Flatbuffer metadata for Map
---
 format/Schema.fbs | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/format/Schema.fbs b/format/Schema.fbs
index a7e802b9dcba6..186f8e362bde2 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -44,6 +44,35 @@ table FixedSizeList {
   listSize: int;
 }
 
+/// A Map is a logical nested type that is represented as
+///
+/// List<entry: Struct<key: K, value: V>>
+///
+/// In this layout, the keys and values are each respectively contiguous. We do
+/// not constrain the key and value types, so the application is responsible
+/// for ensuring that the keys are hashable and unique. Whether the keys are sorted
+/// may be set in the metadata for this field
+///
+/// In a Field with Map type, the Field has a child Struct field, which then
+/// has two children: key type and the second the value type. The names of the
+/// child fields may be respectively "entry", "key", and "value", but this is
+/// not enforced
+///
+/// Map
+///   - child[0] entry: Struct
+///     - child[0] key: K
+///     - child[1] value: V
+///
+/// Neither the "entry" field nor the "key" field may be nullable.
+///
+/// The metadata is structured so that Arrow systems without special handling
+/// for Map can make Map an alias for List. The "layout" attribute for the Map
+/// field must have the same contents as a List.
+table Map {
+  /// Set to true if the keys within each value are sorted
+  keysSorted: bool;
+}
+
 enum UnionMode:short { Sparse, Dense }
 
 /// A union is a complex type with children in Field
@@ -170,7 +199,8 @@ union Type {
   Struct_,
   Union,
   FixedSizeBinary,
-  FixedSizeList
+  FixedSizeList,
+  Map
 }
 
 /// ----------------------------------------------------------------------

From 204f148bf560dfb1291f5d7e4d237e3b6c120430 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 24 Jul 2017 22:53:11 -0400
Subject: [PATCH 0850/1644] ARROW-1260: [Plasma] Use factory method to create
 Python PlasmaClient

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #883 from pcmoritz/plasma-client-connect and squashes the following commits:

667629f3 [Philipp Moritz] cleanup
886beabb [Philipp Moritz] create factory method for connecting to the plasma client
09ffbdfd [Philipp Moritz] speed up plasma tests
---
 python/pyarrow/plasma.pyx           | 46 ++++++++++++++++-------------
 python/pyarrow/tests/test_plasma.py |  4 +--
 2 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index bb17685277af1..8aaca9963c131 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -206,28 +206,11 @@ cdef class PlasmaClient:
         c_string store_socket_name
         c_string manager_socket_name
 
-    def __cinit__(self, store_socket_name, manager_socket_name, int release_delay):
-        """
-        Create a new PlasmaClient that is connected to a plasma store
-        and optionally a plasma manager.
-
-        Parameters
-        ----------
-        store_socket_name : str
-            Name of the socket the plasma store is listening at.
-        manager_socket_name : str
-            Name of the socket the plasma manager is listening at.
-        release_delay : int
-            The maximum number of objects that the client will keep and
-            delay releasing (for caching reasons).
-        """
+    def __cinit__(self):
         self.client.reset(new CPlasmaClient())
         self.notification_fd = -1
-        self.store_socket_name = store_socket_name.encode()
-        self.manager_socket_name = manager_socket_name.encode()
-        with nogil:
-            check_status(self.client.get().Connect(self.store_socket_name,
-                         self.manager_socket_name, release_delay))
+        self.store_socket_name = ""
+        self.manager_socket_name = ""
 
     cdef _get_object_buffers(self, object_ids, int64_t timeout_ms,
                              c_vector[CObjectBuffer]* result):
@@ -558,3 +541,26 @@ cdef class PlasmaClient:
         """
         with nogil:
             check_status(self.client.get().Disconnect())
+
+def connect(store_socket_name, manager_socket_name, int release_delay):
+    """
+    Return a new PlasmaClient that is connected a plasma store and
+    optionally a manager.
+
+    Parameters
+    ----------
+    store_socket_name : str
+        Name of the socket the plasma store is listening at.
+    manager_socket_name : str
+        Name of the socket the plasma manager is listening at.
+    release_delay : int
+        The maximum number of objects that the client will keep and
+        delay releasing (for caching reasons).
+    """
+    cdef PlasmaClient result = PlasmaClient()
+    result.store_socket_name = store_socket_name.encode()
+    result.manager_socket_name = manager_socket_name.encode()
+    with nogil:
+        check_status(result.client.get().Connect(result.store_socket_name,
+                     result.manager_socket_name, release_delay))
+    return result
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index ab64463e93111..8f8d5b5ed607b 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -154,9 +154,9 @@ def setup_method(self, test_method):
         plasma_store_name, self.p = start_plasma_store(
             use_valgrind=os.getenv("PLASMA_VALGRIND") == "1")
         # Connect to Plasma.
-        self.plasma_client = plasma.PlasmaClient(plasma_store_name, "", 64)
+        self.plasma_client = plasma.connect(plasma_store_name, "", 64)
         # For the eviction test
-        self.plasma_client2 = plasma.PlasmaClient(plasma_store_name, "", 0)
+        self.plasma_client2 = plasma.connect(plasma_store_name, "", 0)
 
     def teardown_method(self, test_method):
         # Check that the Plasma store is still alive.

From 07b89bf3aaa6639303014bc173e3e371aed363c1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 24 Jul 2017 22:55:51 -0400
Subject: [PATCH 0851/1644] ARROW-1219: [C++] Use Google C++ code formatting

Our coding style guide has a ton of rules. I put up this patch to discuss since we don't have too many patches outstanding right now. This uses the exact Google style used in TensorFlow and other projects, but relaxes the column limit to 90 characters.

The main change is horizontal alignment in function signatures:

```diff
 void ValidateBasicStructArray(const StructArray* result,
-    const vector<uint8_t>& struct_is_valid, const vector<char>& list_values,
-    const vector<uint8_t>& list_is_valid, const vector<int>& list_lengths,
-    const vector<int>& list_offsets, const vector<int32_t>& int_values) {
+                              const vector<uint8_t>& struct_is_valid,
+                              const vector<char>& list_values,
+                              const vector<uint8_t>& list_is_valid,
+                              const vector<int>& list_lengths,
+                              const vector<int>& list_offsets,
+                              const vector<int32_t>& int_values) {
```

I find the paren-aligned version a bit more readable, but it's a matter of taste

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #848 from wesm/google-style and squashes the following commits:

9e8fe4fe [Wes McKinney] Move import to platform.h
8690257d [Wes McKinney] Use Google C++ code formatting
---
 cpp/.clang-format                             |  65 +-----
 cpp/src/arrow/allocator-test.cc               |   2 +-
 cpp/src/arrow/array-decimal-test.cc           |  73 ++++---
 cpp/src/arrow/array-test.cc                   | 108 +++++-----
 cpp/src/arrow/array.cc                        | 141 ++++++++-----
 cpp/src/arrow/array.h                         | 110 +++++-----
 cpp/src/arrow/buffer.cc                       |  41 ++--
 cpp/src/arrow/buffer.h                        |  62 +++---
 cpp/src/arrow/builder-benchmark.cc            |  16 +-
 cpp/src/arrow/builder.cc                      | 196 +++++++++--------
 cpp/src/arrow/builder.h                       |  63 +++---
 cpp/src/arrow/compare.cc                      | 168 +++++++++------
 cpp/src/arrow/compare.h                       |  15 +-
 cpp/src/arrow/io/file.cc                      | 146 ++++++-------
 cpp/src/arrow/io/file.h                       |  12 +-
 cpp/src/arrow/io/hdfs-internal.cc             |  66 +++---
 cpp/src/arrow/io/hdfs-internal.h              |  12 +-
 cpp/src/arrow/io/hdfs.cc                      | 155 +++++++-------
 cpp/src/arrow/io/hdfs.h                       |  18 +-
 cpp/src/arrow/io/interfaces.cc                |  20 +-
 cpp/src/arrow/io/interfaces.h                 |   4 +-
 cpp/src/arrow/io/io-file-test.cc              |  29 ++-
 cpp/src/arrow/io/io-hdfs-test.cc              |  29 ++-
 cpp/src/arrow/io/memory.cc                    |  20 +-
 cpp/src/arrow/io/memory.h                     |   2 +-
 cpp/src/arrow/io/test-common.h                |   4 +-
 cpp/src/arrow/ipc/feather-internal.h          |   8 +-
 cpp/src/arrow/ipc/feather-test.cc             |   7 +-
 cpp/src/arrow/ipc/feather.cc                  | 146 ++++++-------
 cpp/src/arrow/ipc/feather.h                   |   6 +-
 cpp/src/arrow/ipc/file-to-stream.cc           |   2 +-
 cpp/src/arrow/ipc/ipc-json-test.cc            |  42 ++--
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc |   4 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc      |  92 ++++----
 cpp/src/arrow/ipc/json-integration-test.cc    |  35 ++--
 cpp/src/arrow/ipc/json-internal.cc            |  92 ++++----
 cpp/src/arrow/ipc/json-internal.h             |  10 +-
 cpp/src/arrow/ipc/json.cc                     |  32 ++-
 cpp/src/arrow/ipc/json.h                      |  10 +-
 cpp/src/arrow/ipc/metadata.cc                 | 198 +++++++++---------
 cpp/src/arrow/ipc/metadata.h                  |  45 ++--
 cpp/src/arrow/ipc/reader.cc                   |  92 ++++----
 cpp/src/arrow/ipc/reader.h                    |  29 +--
 cpp/src/arrow/ipc/stream-to-file.cc           |   2 +-
 cpp/src/arrow/ipc/test-common.h               |  87 ++++----
 cpp/src/arrow/ipc/writer.cc                   | 159 +++++++-------
 cpp/src/arrow/ipc/writer.h                    |  23 +-
 cpp/src/arrow/memory_pool-test.cc             |  10 +-
 cpp/src/arrow/memory_pool.cc                  |  30 ++-
 cpp/src/arrow/pretty_print-test.cc            |   2 +-
 cpp/src/arrow/pretty_print.cc                 |  62 +++---
 cpp/src/arrow/python/arrow_to_pandas.cc       |  86 ++++----
 cpp/src/arrow/python/arrow_to_pandas.h        |  12 +-
 cpp/src/arrow/python/builtin_convert.cc       |  39 ++--
 cpp/src/arrow/python/builtin_convert.h        |  19 +-
 cpp/src/arrow/python/config.cc                |   2 -
 cpp/src/arrow/python/helpers.cc               |  16 +-
 cpp/src/arrow/python/helpers.h                |  13 +-
 cpp/src/arrow/python/init.cc                  |   4 +-
 cpp/src/arrow/python/io.cc                    |  24 +--
 cpp/src/arrow/python/numpy_convert.cc         |  43 ++--
 cpp/src/arrow/python/numpy_convert.h          |   4 +-
 cpp/src/arrow/python/pandas_to_arrow.cc       |  77 ++++---
 cpp/src/arrow/python/pandas_to_arrow.h        |   5 +-
 cpp/src/arrow/python/platform.h               |   1 +
 cpp/src/arrow/python/pyarrow.cc               |  40 +---
 cpp/src/arrow/python/pyarrow.h                |   4 +-
 cpp/src/arrow/python/python-test.cc           |  11 +-
 cpp/src/arrow/python/util/datetime.h          |   8 +-
 cpp/src/arrow/status.cc                       |   8 +-
 cpp/src/arrow/status.h                        |  24 ++-
 cpp/src/arrow/table-test.cc                   |  10 +-
 cpp/src/arrow/table.cc                        |  99 ++++++---
 cpp/src/arrow/table.h                         |  23 +-
 cpp/src/arrow/tensor.cc                       |  26 +--
 cpp/src/arrow/tensor.h                        |   8 +-
 cpp/src/arrow/test-util.h                     |  74 ++++---
 cpp/src/arrow/type-test.cc                    |   8 +-
 cpp/src/arrow/type.cc                         | 118 +++++------
 cpp/src/arrow/type.h                          |  36 ++--
 cpp/src/arrow/type_traits.h                   |   7 +-
 cpp/src/arrow/util/bit-stream-utils.h         |  25 ++-
 cpp/src/arrow/util/bit-util-test.cc           |  24 ++-
 cpp/src/arrow/util/bit-util.cc                |  30 ++-
 cpp/src/arrow/util/bit-util.h                 | 179 +++++-----------
 cpp/src/arrow/util/bpacking.h                 |  99 +++------
 cpp/src/arrow/util/compression-test.cc        |  38 ++--
 cpp/src/arrow/util/compression.h              |   5 +-
 cpp/src/arrow/util/compression_brotli.cc      |   9 +-
 cpp/src/arrow/util/compression_brotli.h       |   4 +-
 cpp/src/arrow/util/compression_lz4.cc         |  27 ++-
 cpp/src/arrow/util/compression_lz4.h          |   4 +-
 cpp/src/arrow/util/compression_snappy.cc      |  14 +-
 cpp/src/arrow/util/compression_snappy.h       |   4 +-
 cpp/src/arrow/util/compression_zlib.cc        |  35 ++--
 cpp/src/arrow/util/compression_zlib.h         |   4 +-
 cpp/src/arrow/util/compression_zstd.cc        |  14 +-
 cpp/src/arrow/util/compression_zstd.h         |   4 +-
 cpp/src/arrow/util/cpu-info.cc                |  32 ++-
 cpp/src/arrow/util/decimal.cc                 |  60 ++++--
 cpp/src/arrow/util/decimal.h                  |  22 +-
 cpp/src/arrow/util/key_value_metadata.cc      |   4 +-
 cpp/src/arrow/util/key_value_metadata.h       |   4 +-
 cpp/src/arrow/util/logging.h                  |  33 ++-
 cpp/src/arrow/util/memory.h                   |  10 +-
 cpp/src/arrow/util/random.h                   |  14 +-
 cpp/src/arrow/util/rle-encoding-test.cc       |  31 ++-
 cpp/src/arrow/util/rle-encoding.h             |  26 +--
 cpp/src/arrow/util/sse-util.h                 |   4 +-
 cpp/src/arrow/util/stl.h                      |   4 +-
 cpp/src/arrow/util/string.h                   |   4 +-
 cpp/src/plasma/client.cc                      |  70 ++++---
 cpp/src/plasma/client.h                       |  18 +-
 cpp/src/plasma/common.cc                      |  10 +-
 cpp/src/plasma/common.h                       |   2 +-
 cpp/src/plasma/events.cc                      |  22 +-
 cpp/src/plasma/events.h                       |   2 +-
 cpp/src/plasma/eviction_policy.cc             |  22 +-
 cpp/src/plasma/eviction_policy.h              |  18 +-
 cpp/src/plasma/io.cc                          |  40 ++--
 cpp/src/plasma/malloc.cc                      |  20 +-
 cpp/src/plasma/plasma.cc                      |  14 +-
 cpp/src/plasma/plasma.h                       |   6 +-
 cpp/src/plasma/protocol.cc                    | 116 +++++-----
 cpp/src/plasma/protocol.h                     |  53 ++---
 cpp/src/plasma/store.cc                       |  68 +++---
 cpp/src/plasma/store.h                        |   8 +-
 cpp/src/plasma/test/client_tests.cc           |   2 +-
 cpp/src/plasma/test/serialization_tests.cc    |  18 +-
 129 files changed, 2565 insertions(+), 2362 deletions(-)

diff --git a/cpp/.clang-format b/cpp/.clang-format
index 33f282a20de20..06453dfbb25b7 100644
--- a/cpp/.clang-format
+++ b/cpp/.clang-format
@@ -15,67 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-Language:        Cpp
-# BasedOnStyle:  Google
-AccessModifierOffset: -1
-AlignAfterOpenBracket: false
-AlignConsecutiveAssignments: false
-AlignEscapedNewlinesLeft: true
-AlignOperands:   true
-AlignTrailingComments: true
-AllowAllParametersOfDeclarationOnNextLine: true
-AllowShortBlocksOnASingleLine: true
-AllowShortCaseLabelsOnASingleLine: false
-AllowShortFunctionsOnASingleLine: Inline
-AllowShortIfStatementsOnASingleLine: true
-AllowShortLoopsOnASingleLine: false
-AlwaysBreakAfterDefinitionReturnType: None
-AlwaysBreakBeforeMultilineStrings: true
-AlwaysBreakTemplateDeclarations: true
-BinPackArguments: true
-BinPackParameters: true
-BreakBeforeBinaryOperators: None
-BreakBeforeBraces: Attach
-BreakBeforeTernaryOperators: true
-BreakConstructorInitializersBeforeComma: false
-ColumnLimit: 90
-CommentPragmas:  '^ IWYU pragma:'
-ConstructorInitializerAllOnOneLineOrOnePerLine: true
-ConstructorInitializerIndentWidth: 4
-ContinuationIndentWidth: 4
-Cpp11BracedListStyle: true
+BasedOnStyle:  Google
 DerivePointerAlignment: false
-DisableFormat:   false
-ExperimentalAutoDetectBinPacking: false
-ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
-IndentCaseLabels: true
-IndentWidth:     2
-IndentWrappedFunctionNames: false
-KeepEmptyLinesAtTheStartOfBlocks: false
-MacroBlockBegin: ''
-MacroBlockEnd:   ''
-MaxEmptyLinesToKeep: 1
-NamespaceIndentation: None
-ObjCBlockIndentWidth: 2
-ObjCSpaceAfterProperty: false
-ObjCSpaceBeforeProtocolList: false
-PenaltyBreakBeforeFirstCallParameter: 1000
-PenaltyBreakComment: 300
-PenaltyBreakFirstLessLess: 120
-PenaltyBreakString: 1000
-PenaltyExcessCharacter: 1000000
-PenaltyReturnTypeOnItsOwnLine: 200
-PointerAlignment: Left
-SpaceAfterCStyleCast: false
-SpaceBeforeAssignmentOperators: true
-SpaceBeforeParens: ControlStatements
-SpaceInEmptyParentheses: false
-SpacesBeforeTrailingComments: 2
-SpacesInAngles:  false
-SpacesInContainerLiterals: true
-SpacesInCStyleCastParentheses: false
-SpacesInParentheses: false
-SpacesInSquareBrackets: false
-Standard:        Cpp11
-TabWidth:        8
-UseTab:          Never
+ColumnLimit: 90
diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
index 5a4e98d76600f..f3a80cdae818b 100644
--- a/cpp/src/arrow/allocator-test.cc
+++ b/cpp/src/arrow/allocator-test.cc
@@ -48,7 +48,7 @@ TEST(stl_allocator, FreeLargeMemory) {
 
 #ifndef NDEBUG
   EXPECT_EXIT(alloc.deallocate(data, 120), ::testing::ExitedWithCode(1),
-      ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
+              ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
 #endif
 
   alloc.deallocate(data, 100);
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 0959d686498d5..436ce9cf7c312 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -28,12 +28,12 @@ namespace decimal {
 template <typename T>
 class DecimalTestBase {
  public:
-  virtual std::vector<uint8_t> data(
-      const std::vector<T>& input, size_t byte_width) const = 0;
+  virtual std::vector<uint8_t> data(const std::vector<T>& input,
+                                    size_t byte_width) const = 0;
 
   void test(int precision, const std::vector<T>& draw,
-      const std::vector<uint8_t>& valid_bytes,
-      const std::vector<uint8_t>& sign_bitmap = {}, int64_t offset = 0) const {
+            const std::vector<uint8_t>& valid_bytes,
+            const std::vector<uint8_t>& sign_bitmap = {}, int64_t offset = 0) const {
     auto type = std::make_shared<DecimalType>(precision, 4);
     int byte_width = type->byte_width();
     auto pool = default_memory_pool();
@@ -63,8 +63,9 @@ class DecimalTestBase {
     ASSERT_OK(BitUtil::BytesToBits(valid_bytes, &expected_null_bitmap));
 
     int64_t expected_null_count = test::null_count(valid_bytes);
-    auto expected = std::make_shared<DecimalArray>(type, size, expected_data,
-        expected_null_bitmap, expected_null_count, offset, expected_sign_bitmap);
+    auto expected =
+        std::make_shared<DecimalArray>(type, size, expected_data, expected_null_bitmap,
+                                       expected_null_count, offset, expected_sign_bitmap);
 
     std::shared_ptr<Array> out;
     ASSERT_OK(builder->Finish(&out));
@@ -75,8 +76,8 @@ class DecimalTestBase {
 template <typename T>
 class DecimalTest : public DecimalTestBase<T> {
  public:
-  std::vector<uint8_t> data(
-      const std::vector<T>& input, size_t byte_width) const override {
+  std::vector<uint8_t> data(const std::vector<T>& input,
+                            size_t byte_width) const override {
     std::vector<uint8_t> result(input.size() * byte_width);
     // TODO(phillipc): There's probably a better way to do this
     constexpr static const size_t bytes_per_element = sizeof(T);
@@ -90,8 +91,8 @@ class DecimalTest : public DecimalTestBase<T> {
 template <>
 class DecimalTest<Decimal128> : public DecimalTestBase<Decimal128> {
  public:
-  std::vector<uint8_t> data(
-      const std::vector<Decimal128>& input, size_t byte_width) const override {
+  std::vector<uint8_t> data(const std::vector<Decimal128>& input,
+                            size_t byte_width) const override {
     std::vector<uint8_t> result;
     result.reserve(input.size() * byte_width);
     constexpr static const size_t bytes_per_element = 16;
@@ -120,24 +121,24 @@ class Decimal128BuilderTest : public ::testing::TestWithParam<int>,
 
 TEST_P(Decimal32BuilderTest, NoNulls) {
   int precision = GetParam();
-  std::vector<Decimal32> draw = {
-      Decimal32(1), Decimal32(2), Decimal32(2389), Decimal32(4), Decimal32(-12348)};
+  std::vector<Decimal32> draw = {Decimal32(1), Decimal32(2), Decimal32(2389),
+                                 Decimal32(4), Decimal32(-12348)};
   std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
   this->test(precision, draw, valid_bytes);
 }
 
 TEST_P(Decimal64BuilderTest, NoNulls) {
   int precision = GetParam();
-  std::vector<Decimal64> draw = {
-      Decimal64(1), Decimal64(2), Decimal64(2389), Decimal64(4), Decimal64(-12348)};
+  std::vector<Decimal64> draw = {Decimal64(1), Decimal64(2), Decimal64(2389),
+                                 Decimal64(4), Decimal64(-12348)};
   std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
   this->test(precision, draw, valid_bytes);
 }
 
 TEST_P(Decimal128BuilderTest, NoNulls) {
   int precision = GetParam();
-  std::vector<Decimal128> draw = {
-      Decimal128(1), Decimal128(-2), Decimal128(2389), Decimal128(4), Decimal128(-12348)};
+  std::vector<Decimal128> draw = {Decimal128(1), Decimal128(-2), Decimal128(2389),
+                                  Decimal128(4), Decimal128(-12348)};
   std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
   std::vector<uint8_t> sign_bitmap = {false, true, false, false, true};
   this->test(precision, draw, valid_bytes, sign_bitmap);
@@ -145,41 +146,47 @@ TEST_P(Decimal128BuilderTest, NoNulls) {
 
 TEST_P(Decimal32BuilderTest, WithNulls) {
   int precision = GetParam();
-  std::vector<Decimal32> draw = {
-      Decimal32(1), Decimal32(2), Decimal32(-1), Decimal32(4), Decimal32(-1)};
+  std::vector<Decimal32> draw = {Decimal32(1), Decimal32(2), Decimal32(-1), Decimal32(4),
+                                 Decimal32(-1)};
   std::vector<uint8_t> valid_bytes = {true, true, false, true, false};
   this->test(precision, draw, valid_bytes);
 }
 
 TEST_P(Decimal64BuilderTest, WithNulls) {
   int precision = GetParam();
-  std::vector<Decimal64> draw = {
-      Decimal64(-1), Decimal64(2), Decimal64(-1), Decimal64(4), Decimal64(-1)};
+  std::vector<Decimal64> draw = {Decimal64(-1), Decimal64(2), Decimal64(-1), Decimal64(4),
+                                 Decimal64(-1)};
   std::vector<uint8_t> valid_bytes = {true, true, false, true, false};
   this->test(precision, draw, valid_bytes);
 }
 
 TEST_P(Decimal128BuilderTest, WithNulls) {
   int precision = GetParam();
-  std::vector<Decimal128> draw = {Decimal128(1), Decimal128(2), Decimal128(-1),
-      Decimal128(4), Decimal128(-1), Decimal128(1), Decimal128(2),
-      Decimal128("230342903942.234234"), Decimal128("-23049302932.235234")};
-  std::vector<uint8_t> valid_bytes = {
-      true, true, false, true, false, true, true, true, true};
-  std::vector<uint8_t> sign_bitmap = {
-      false, false, false, false, false, false, false, false, true};
+  std::vector<Decimal128> draw = {Decimal128(1),
+                                  Decimal128(2),
+                                  Decimal128(-1),
+                                  Decimal128(4),
+                                  Decimal128(-1),
+                                  Decimal128(1),
+                                  Decimal128(2),
+                                  Decimal128("230342903942.234234"),
+                                  Decimal128("-23049302932.235234")};
+  std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
+                                      true, true, true,  true};
+  std::vector<uint8_t> sign_bitmap = {false, false, false, false, false,
+                                      false, false, false, true};
   this->test(precision, draw, valid_bytes, sign_bitmap);
 }
 
 INSTANTIATE_TEST_CASE_P(Decimal32BuilderTest, Decimal32BuilderTest,
-    ::testing::Range(
-        DecimalPrecision<int32_t>::minimum, DecimalPrecision<int32_t>::maximum));
+                        ::testing::Range(DecimalPrecision<int32_t>::minimum,
+                                         DecimalPrecision<int32_t>::maximum));
 INSTANTIATE_TEST_CASE_P(Decimal64BuilderTest, Decimal64BuilderTest,
-    ::testing::Range(
-        DecimalPrecision<int64_t>::minimum, DecimalPrecision<int64_t>::maximum));
+                        ::testing::Range(DecimalPrecision<int64_t>::minimum,
+                                         DecimalPrecision<int64_t>::maximum));
 INSTANTIATE_TEST_CASE_P(Decimal128BuilderTest, Decimal128BuilderTest,
-    ::testing::Range(
-        DecimalPrecision<int128_t>::minimum, DecimalPrecision<int128_t>::maximum));
+                        ::testing::Range(DecimalPrecision<int128_t>::minimum,
+                                         DecimalPrecision<int128_t>::maximum));
 
 }  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index acb4819dd0949..5d63d921cdd52 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -64,8 +64,8 @@ TEST_F(TestArray, TestLength) {
   ASSERT_EQ(arr->length(), 100);
 }
 
-Status MakeArrayFromValidBytes(
-    const vector<uint8_t>& v, MemoryPool* pool, std::shared_ptr<Array>* out) {
+Status MakeArrayFromValidBytes(const vector<uint8_t>& v, MemoryPool* pool,
+                               std::shared_ptr<Array>* out) {
   int64_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
 
   std::shared_ptr<Buffer> null_buf;
@@ -147,7 +147,9 @@ TEST_F(TestArray, TestIsNull) {
   // clang-format on
   int64_t null_count = 0;
   for (uint8_t x : null_bitmap) {
-    if (x == 0) { ++null_count; }
+    if (x == 0) {
+      ++null_count;
+    }
   }
 
   std::shared_ptr<Buffer> null_buf;
@@ -223,8 +225,8 @@ class TestPrimitiveBuilder : public TestBuilder {
   void Check(const std::unique_ptr<BuilderType>& builder, bool nullable) {
     int64_t size = builder->length();
 
-    auto ex_data = std::make_shared<Buffer>(
-        reinterpret_cast<uint8_t*>(draws_.data()), size * sizeof(T));
+    auto ex_data = std::make_shared<Buffer>(reinterpret_cast<uint8_t*>(draws_.data()),
+                                            size * sizeof(T));
 
     std::shared_ptr<Buffer> ex_null_bitmap;
     int64_t ex_null_count = 0;
@@ -316,8 +318,8 @@ void TestPrimitiveBuilder<PBoolean>::RandomData(int64_t N, double pct_null) {
 }
 
 template <>
-void TestPrimitiveBuilder<PBoolean>::Check(
-    const std::unique_ptr<BooleanBuilder>& builder, bool nullable) {
+void TestPrimitiveBuilder<PBoolean>::Check(const std::unique_ptr<BooleanBuilder>& builder,
+                                           bool nullable) {
   int64_t size = builder->length();
 
   std::shared_ptr<Buffer> ex_data;
@@ -351,7 +353,9 @@ void TestPrimitiveBuilder<PBoolean>::Check(
   ASSERT_EQ(expected->length(), result->length());
 
   for (int64_t i = 0; i < result->length(); ++i) {
-    if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
+    if (nullable) {
+      ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i;
+    }
     bool actual = BitUtil::GetBit(result->values()->data(), i);
     ASSERT_EQ(draws_[i] != 0, actual) << i;
   }
@@ -359,7 +363,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 }
 
 typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
-    PInt32, PInt64, PFloat, PDouble>
+                         PInt32, PInt64, PFloat, PDouble>
     Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
@@ -377,7 +381,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestInit) {
   ASSERT_OK(this->builder_->Reserve(n));
   ASSERT_EQ(BitUtil::NextPower2(n), this->builder_->capacity());
   ASSERT_EQ(BitUtil::NextPower2(TypeTraits<Type>::bytes_required(n)),
-      this->builder_->data()->size());
+            this->builder_->data()->size());
 
   // unsure if this should go in all builder classes
   ASSERT_EQ(0, this->builder_->num_children());
@@ -440,8 +444,8 @@ TYPED_TEST(TestPrimitiveBuilder, Equality) {
   ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &equal_array));
 
   // Make the not equal array by negating the first valid element with itself.
-  const auto first_valid = std::find_if(
-      valid_bytes.begin(), valid_bytes.end(), [](uint8_t valid) { return valid > 0; });
+  const auto first_valid = std::find_if(valid_bytes.begin(), valid_bytes.end(),
+                                        [](uint8_t valid) { return valid > 0; });
   const int64_t first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
   // This should be true with a very high probability, but might introduce flakiness
   ASSERT_LT(first_valid_idx, size - 1);
@@ -679,8 +683,8 @@ class TestStringArray : public ::testing::Test {
     ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_));
     null_count_ = test::null_count(valid_bytes_);
 
-    strings_ = std::make_shared<StringArray>(
-        length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
+    strings_ = std::make_shared<StringArray>(length_, offsets_buf_, value_buf_,
+                                             null_bitmap_, null_count_);
   }
 
  protected:
@@ -723,8 +727,8 @@ TEST_F(TestStringArray, TestListFunctions) {
 }
 
 TEST_F(TestStringArray, TestDestructor) {
-  auto arr = std::make_shared<StringArray>(
-      length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
+  auto arr = std::make_shared<StringArray>(length_, offsets_buf_, value_buf_,
+                                           null_bitmap_, null_count_);
 }
 
 TEST_F(TestStringArray, TestGetString) {
@@ -742,10 +746,10 @@ TEST_F(TestStringArray, TestEmptyStringComparison) {
   offsets_buf_ = test::GetBufferFromVector(offsets_);
   length_ = static_cast<int64_t>(offsets_.size() - 1);
 
-  auto strings_a = std::make_shared<StringArray>(
-      length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
-  auto strings_b = std::make_shared<StringArray>(
-      length_, offsets_buf_, nullptr, null_bitmap_, null_count_);
+  auto strings_a = std::make_shared<StringArray>(length_, offsets_buf_, nullptr,
+                                                 null_bitmap_, null_count_);
+  auto strings_b = std::make_shared<StringArray>(length_, offsets_buf_, nullptr,
+                                                 null_bitmap_, null_count_);
   ASSERT_TRUE(strings_a->Equals(strings_b));
 }
 
@@ -893,8 +897,8 @@ class TestBinaryArray : public ::testing::Test {
     ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_));
     null_count_ = test::null_count(valid_bytes_);
 
-    strings_ = std::make_shared<BinaryArray>(
-        length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
+    strings_ = std::make_shared<BinaryArray>(length_, offsets_buf_, value_buf_,
+                                             null_bitmap_, null_count_);
   }
 
  protected:
@@ -937,8 +941,8 @@ TEST_F(TestBinaryArray, TestListFunctions) {
 }
 
 TEST_F(TestBinaryArray, TestDestructor) {
-  auto arr = std::make_shared<BinaryArray>(
-      length_, offsets_buf_, value_buf_, null_bitmap_, null_count_);
+  auto arr = std::make_shared<BinaryArray>(length_, offsets_buf_, value_buf_,
+                                           null_bitmap_, null_count_);
 }
 
 TEST_F(TestBinaryArray, TestGetValue) {
@@ -965,8 +969,9 @@ TEST_F(TestBinaryArray, TestEqualsEmptyStrings) {
   ASSERT_OK(builder.Finish(&left_arr));
 
   const BinaryArray& left = static_cast<const BinaryArray&>(*left_arr);
-  std::shared_ptr<Array> right = std::make_shared<BinaryArray>(left.length(),
-      left.value_offsets(), nullptr, left.null_bitmap(), left.null_count());
+  std::shared_ptr<Array> right =
+      std::make_shared<BinaryArray>(left.length(), left.value_offsets(), nullptr,
+                                    left.null_bitmap(), left.null_count());
 
   ASSERT_TRUE(left.Equals(right));
   ASSERT_TRUE(left.RangeEquals(0, left.length(), 0, right));
@@ -1082,17 +1087,11 @@ void CheckSliceEquality() {
   ASSERT_TRUE(array->RangeEquals(5, 25, 0, slice));
 }
 
-TEST_F(TestBinaryArray, TestSliceEquality) {
-  CheckSliceEquality<BinaryType>();
-}
+TEST_F(TestBinaryArray, TestSliceEquality) { CheckSliceEquality<BinaryType>(); }
 
-TEST_F(TestStringArray, TestSliceEquality) {
-  CheckSliceEquality<BinaryType>();
-}
+TEST_F(TestStringArray, TestSliceEquality) { CheckSliceEquality<BinaryType>(); }
 
-TEST_F(TestBinaryArray, LengthZeroCtor) {
-  BinaryArray array(0, nullptr, nullptr);
-}
+TEST_F(TestBinaryArray, LengthZeroCtor) { BinaryArray array(0, nullptr, nullptr); }
 
 // ----------------------------------------------------------------------
 // FixedSizeBinary tests
@@ -1126,8 +1125,8 @@ TEST_F(TestFWBinaryArray, Builder) {
 
   std::shared_ptr<Array> result;
 
-  auto CheckResult = [this, &length, &is_valid, &raw_data, &byte_width](
-      const Array& result) {
+  auto CheckResult = [this, &length, &is_valid, &raw_data,
+                      &byte_width](const Array& result) {
     // Verify output
     const auto& fw_result = static_cast<const FixedSizeBinaryArray&>(result);
 
@@ -1135,8 +1134,8 @@ TEST_F(TestFWBinaryArray, Builder) {
 
     for (int64_t i = 0; i < result.length(); ++i) {
       if (is_valid[i]) {
-        ASSERT_EQ(
-            0, memcmp(raw_data + byte_width * i, fw_result.GetValue(i), byte_width));
+        ASSERT_EQ(0,
+                  memcmp(raw_data + byte_width * i, fw_result.GetValue(i), byte_width));
       } else {
         ASSERT_TRUE(fw_result.IsNull(i));
       }
@@ -1323,8 +1322,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt16) {
   SetUp();
   ASSERT_OK(builder_->Append(std::numeric_limits<int16_t>::max()));
   ASSERT_OK(builder_->Append(std::numeric_limits<int16_t>::min()));
-  expected_values = {
-      std::numeric_limits<int16_t>::max(), std::numeric_limits<int16_t>::min()};
+  expected_values = {std::numeric_limits<int16_t>::max(),
+                     std::numeric_limits<int16_t>::min()};
   Done();
 
   ArrayFromVector<Int16Type, int16_t>(expected_values, &expected_);
@@ -1354,8 +1353,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt32) {
   SetUp();
   ASSERT_OK(builder_->Append(std::numeric_limits<int32_t>::max()));
   ASSERT_OK(builder_->Append(std::numeric_limits<int32_t>::min()));
-  expected_values = {
-      std::numeric_limits<int32_t>::max(), std::numeric_limits<int32_t>::min()};
+  expected_values = {std::numeric_limits<int32_t>::max(),
+                     std::numeric_limits<int32_t>::min()};
   Done();
 
   ArrayFromVector<Int32Type, int32_t>(expected_values, &expected_);
@@ -1385,8 +1384,8 @@ TEST_F(TestAdaptiveIntBuilder, TestInt64) {
   SetUp();
   ASSERT_OK(builder_->Append(std::numeric_limits<int64_t>::max()));
   ASSERT_OK(builder_->Append(std::numeric_limits<int64_t>::min()));
-  expected_values = {
-      std::numeric_limits<int64_t>::max(), std::numeric_limits<int64_t>::min()};
+  expected_values = {std::numeric_limits<int64_t>::max(),
+                     std::numeric_limits<int64_t>::min()};
   Done();
 
   ArrayFromVector<Int64Type, int64_t>(expected_values, &expected_);
@@ -1505,7 +1504,7 @@ template <typename Type>
 class TestDictionaryBuilder : public TestBuilder {};
 
 typedef ::testing::Types<Int8Type, UInt8Type, Int16Type, UInt16Type, Int32Type,
-    UInt32Type, Int64Type, UInt64Type, FloatType, DoubleType>
+                         UInt32Type, Int64Type, UInt64Type, FloatType, DoubleType>
     PrimitiveDictionaries;
 
 TYPED_TEST_CASE(TestDictionaryBuilder, PrimitiveDictionaries);
@@ -1784,7 +1783,7 @@ TEST_F(TestListBuilder, TestAppendNull) {
 }
 
 void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& values,
-    const vector<uint8_t>& is_valid) {
+                            const vector<uint8_t>& is_valid) {
   ASSERT_OK(ValidateArray(*result));
   ASSERT_EQ(1, result->null_count());
   ASSERT_EQ(0, result->values()->null_count());
@@ -1997,9 +1996,12 @@ TEST(TestDictionary, Validate) {
 // Struct tests
 
 void ValidateBasicStructArray(const StructArray* result,
-    const vector<uint8_t>& struct_is_valid, const vector<char>& list_values,
-    const vector<uint8_t>& list_is_valid, const vector<int>& list_lengths,
-    const vector<int>& list_offsets, const vector<int32_t>& int_values) {
+                              const vector<uint8_t>& struct_is_valid,
+                              const vector<char>& list_values,
+                              const vector<uint8_t>& list_is_valid,
+                              const vector<int>& list_lengths,
+                              const vector<int>& list_offsets,
+                              const vector<int32_t>& int_values) {
   ASSERT_EQ(4, result->length());
   ASSERT_OK(ValidateArray(*result));
 
@@ -2134,7 +2136,7 @@ TEST_F(TestStructBuilder, TestBasics) {
   Done();
 
   ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
-      list_lengths, list_offsets, int_values);
+                           list_lengths, list_offsets, int_values);
 }
 
 TEST_F(TestStructBuilder, BulkAppend) {
@@ -2166,7 +2168,7 @@ TEST_F(TestStructBuilder, BulkAppend) {
 
   Done();
   ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
-      list_lengths, list_offsets, int_values);
+                           list_lengths, list_offsets, int_values);
 }
 
 TEST_F(TestStructBuilder, BulkAppendInvalid) {
@@ -2280,7 +2282,7 @@ TEST_F(TestStructBuilder, TestEquality) {
   // setup an unequal one with unequal offsets
   ASSERT_OK(builder_->Append(struct_is_valid.size(), struct_is_valid.data()));
   ASSERT_OK(list_vb->Append(unequal_list_offsets.data(), unequal_list_offsets.size(),
-      unequal_list_is_valid.data()));
+                            unequal_list_is_valid.data()));
   for (int8_t value : list_values) {
     char_vb->UnsafeAppend(value);
   }
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 4a405f24342fb..61791c9457756 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -57,45 +57,57 @@ int64_t Array::null_count() const {
 bool Array::Equals(const Array& arr) const {
   bool are_equal = false;
   Status error = ArrayEquals(*this, arr, &are_equal);
-  if (!error.ok()) { DCHECK(false) << "Arrays not comparable: " << error.ToString(); }
+  if (!error.ok()) {
+    DCHECK(false) << "Arrays not comparable: " << error.ToString();
+  }
   return are_equal;
 }
 
 bool Array::Equals(const std::shared_ptr<Array>& arr) const {
-  if (!arr) { return false; }
+  if (!arr) {
+    return false;
+  }
   return Equals(*arr);
 }
 
 bool Array::ApproxEquals(const Array& arr) const {
   bool are_equal = false;
   Status error = ArrayApproxEquals(*this, arr, &are_equal);
-  if (!error.ok()) { DCHECK(false) << "Arrays not comparable: " << error.ToString(); }
+  if (!error.ok()) {
+    DCHECK(false) << "Arrays not comparable: " << error.ToString();
+  }
   return are_equal;
 }
 
 bool Array::ApproxEquals(const std::shared_ptr<Array>& arr) const {
-  if (!arr) { return false; }
+  if (!arr) {
+    return false;
+  }
   return ApproxEquals(*arr);
 }
 
 bool Array::RangeEquals(int64_t start_idx, int64_t end_idx, int64_t other_start_idx,
-    const std::shared_ptr<Array>& other) const {
-  if (!other) { return false; }
+                        const std::shared_ptr<Array>& other) const {
+  if (!other) {
+    return false;
+  }
   return RangeEquals(*other, start_idx, end_idx, other_start_idx);
 }
 
 bool Array::RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
-    int64_t other_start_idx) const {
+                        int64_t other_start_idx) const {
   bool are_equal = false;
   Status error =
       ArrayRangeEquals(*this, other, start_idx, end_idx, other_start_idx, &are_equal);
-  if (!error.ok()) { DCHECK(false) << "Arrays not comparable: " << error.ToString(); }
+  if (!error.ok()) {
+    DCHECK(false) << "Arrays not comparable: " << error.ToString();
+  }
   return are_equal;
 }
 
 // Last two parameters are in-out parameters
-static inline void ConformSliceParams(
-    int64_t array_offset, int64_t array_length, int64_t* offset, int64_t* length) {
+static inline void ConformSliceParams(int64_t array_offset, int64_t array_length,
+                                      int64_t* offset, int64_t* length) {
   DCHECK_LE(*offset, array_length);
   DCHECK_NE(offset, nullptr);
   *length = std::min(array_length - *offset, *length);
@@ -113,8 +125,8 @@ std::string Array::ToString() const {
   return ss.str();
 }
 
-static inline std::shared_ptr<ArrayData> SliceData(
-    const ArrayData& data, int64_t offset, int64_t length) {
+static inline std::shared_ptr<ArrayData> SliceData(const ArrayData& data, int64_t offset,
+                                                   int64_t length) {
   ConformSliceParams(data.offset, data.length, &offset, &length);
 
   auto new_data = data.ShallowCopy();
@@ -139,8 +151,9 @@ std::shared_ptr<Array> NullArray::Slice(int64_t offset, int64_t length) const {
 // Primitive array base
 
 PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset) {
+                               const std::shared_ptr<Buffer>& data,
+                               const std::shared_ptr<Buffer>& null_bitmap,
+                               int64_t null_count, int64_t offset) {
   BufferVector buffers = {null_bitmap, data};
   SetData(
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
@@ -166,7 +179,8 @@ BooleanArray::BooleanArray(const std::shared_ptr<internal::ArrayData>& data)
 }
 
 BooleanArray::BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
+                           const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                           int64_t offset)
     : PrimitiveArray(boolean(), length, data, null_bitmap, null_count, offset) {}
 
 std::shared_ptr<Array> BooleanArray::Slice(int64_t offset, int64_t length) const {
@@ -182,8 +196,10 @@ ListArray::ListArray(const std::shared_ptr<ArrayData>& data) {
 }
 
 ListArray::ListArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Array>& values,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset) {
+                     const std::shared_ptr<Buffer>& value_offsets,
+                     const std::shared_ptr<Array>& values,
+                     const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                     int64_t offset) {
   BufferVector buffers = {null_bitmap, value_offsets};
   auto internal_data =
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
@@ -192,7 +208,7 @@ ListArray::ListArray(const std::shared_ptr<DataType>& type, int64_t length,
 }
 
 Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPool* pool,
-    std::shared_ptr<Array>* out) {
+                             std::shared_ptr<Array>* out) {
   if (ARROW_PREDICT_FALSE(offsets.length() == 0)) {
     return Status::Invalid("List offsets must have non-zero length");
   }
@@ -205,12 +221,13 @@ Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPo
     return Status::Invalid("List offsets must be signed int32");
   }
 
-  BufferVector buffers = {
-      offsets.null_bitmap(), static_cast<const Int32Array&>(offsets).values()};
+  BufferVector buffers = {offsets.null_bitmap(),
+                          static_cast<const Int32Array&>(offsets).values()};
 
   auto list_type = list(values.type());
-  auto internal_data = std::make_shared<internal::ArrayData>(list_type,
-      offsets.length() - 1, std::move(buffers), offsets.null_count(), offsets.offset());
+  auto internal_data = std::make_shared<internal::ArrayData>(
+      list_type, offsets.length() - 1, std::move(buffers), offsets.null_count(),
+      offsets.offset());
   internal_data->child_data.push_back(values.data());
 
   *out = std::make_shared<ListArray>(internal_data);
@@ -230,14 +247,12 @@ std::shared_ptr<DataType> ListArray::value_type() const {
   return static_cast<const ListType&>(*type()).value_type();
 }
 
-std::shared_ptr<Array> ListArray::values() const {
-  return values_;
-}
+std::shared_ptr<Array> ListArray::values() const { return values_; }
 
 std::shared_ptr<Array> ListArray::Slice(int64_t offset, int64_t length) const {
   ConformSliceParams(data_->offset, data_->length, &offset, &length);
   return std::make_shared<ListArray>(type(), length, value_offsets(), values(),
-      null_bitmap(), kUnknownNullCount, offset);
+                                     null_bitmap(), kUnknownNullCount, offset);
 }
 
 // ----------------------------------------------------------------------
@@ -262,14 +277,17 @@ void BinaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
 }
 
 BinaryArray::BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
-    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset)
+                         const std::shared_ptr<Buffer>& data,
+                         const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                         int64_t offset)
     : BinaryArray(kBinary, length, value_offsets, data, null_bitmap, null_count, offset) {
 }
 
 BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset) {
+                         const std::shared_ptr<Buffer>& value_offsets,
+                         const std::shared_ptr<Buffer>& data,
+                         const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                         int64_t offset) {
   BufferVector buffers = {null_bitmap, value_offsets, data};
   SetData(
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
@@ -285,8 +303,9 @@ StringArray::StringArray(const std::shared_ptr<internal::ArrayData>& data) {
 }
 
 StringArray::StringArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
-    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset)
+                         const std::shared_ptr<Buffer>& data,
+                         const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                         int64_t offset)
     : BinaryArray(kString, length, value_offsets, data, null_bitmap, null_count, offset) {
 }
 
@@ -304,8 +323,10 @@ FixedSizeBinaryArray::FixedSizeBinaryArray(
 }
 
 FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<DataType>& type,
-    int64_t length, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset)
+                                           int64_t length,
+                                           const std::shared_ptr<Buffer>& data,
+                                           const std::shared_ptr<Buffer>& null_bitmap,
+                                           int64_t null_count, int64_t offset)
     : PrimitiveArray(type, length, data, null_bitmap, null_count, offset),
       byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {}
 
@@ -335,8 +356,9 @@ void DecimalArray::SetData(const std::shared_ptr<ArrayData>& data) {
 }
 
 DecimalArray::DecimalArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset, const std::shared_ptr<Buffer>& sign_bitmap) {
+                           const std::shared_ptr<Buffer>& data,
+                           const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                           int64_t offset, const std::shared_ptr<Buffer>& sign_bitmap) {
   BufferVector buffers = {null_bitmap, data, sign_bitmap};
   SetData(
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
@@ -392,8 +414,9 @@ StructArray::StructArray(const std::shared_ptr<ArrayData>& data) {
 }
 
 StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::vector<std::shared_ptr<Array>>& children,
-    std::shared_ptr<Buffer> null_bitmap, int64_t null_count, int64_t offset) {
+                         const std::vector<std::shared_ptr<Array>>& children,
+                         std::shared_ptr<Buffer> null_bitmap, int64_t null_count,
+                         int64_t offset) {
   BufferVector buffers = {null_bitmap};
   SetData(
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
@@ -433,9 +456,11 @@ UnionArray::UnionArray(const std::shared_ptr<ArrayData>& data) {
 }
 
 UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::vector<std::shared_ptr<Array>>& children,
-    const std::shared_ptr<Buffer>& type_ids, const std::shared_ptr<Buffer>& value_offsets,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset) {
+                       const std::vector<std::shared_ptr<Array>>& children,
+                       const std::shared_ptr<Buffer>& type_ids,
+                       const std::shared_ptr<Buffer>& value_offsets,
+                       const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                       int64_t offset) {
   BufferVector buffers = {null_bitmap, type_ids, value_offsets};
   auto internal_data =
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
@@ -464,8 +489,8 @@ DictionaryArray::DictionaryArray(const std::shared_ptr<ArrayData>& data)
   SetData(data);
 }
 
-DictionaryArray::DictionaryArray(
-    const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices)
+DictionaryArray::DictionaryArray(const std::shared_ptr<DataType>& type,
+                                 const std::shared_ptr<Array>& indices)
     : dict_type_(static_cast<const DictionaryType*>(type.get())) {
   DCHECK_EQ(type->id(), Type::DICTIONARY);
   DCHECK_EQ(indices->type_id(), dict_type_->index_type()->id());
@@ -482,9 +507,7 @@ void DictionaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
   DCHECK(internal::MakeArray(indices_data, &indices_).ok());
 }
 
-std::shared_ptr<Array> DictionaryArray::indices() const {
-  return indices_;
-}
+std::shared_ptr<Array> DictionaryArray::indices() const { return indices_; }
 
 std::shared_ptr<Array> DictionaryArray::dictionary() const {
   return dict_type_->dictionary();
@@ -517,7 +540,9 @@ struct ValidateVisitor {
   }
 
   Status Visit(const ListArray& array) {
-    if (array.length() < 0) { return Status::Invalid("Length was negative"); }
+    if (array.length() < 0) {
+      return Status::Invalid("Length was negative");
+    }
 
     auto value_offsets = array.value_offsets();
     if (array.length() && !value_offsets) {
@@ -550,7 +575,9 @@ struct ValidateVisitor {
     }
 
     int32_t prev_offset = array.value_offset(0);
-    if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
+    if (prev_offset != 0) {
+      return Status::Invalid("The first offset wasn't zero");
+    }
     for (int64_t i = 1; i <= array.length(); ++i) {
       int32_t current_offset = array.value_offset(i);
       if (array.IsNull(i - 1) && current_offset != prev_offset) {
@@ -573,7 +600,9 @@ struct ValidateVisitor {
   }
 
   Status Visit(const StructArray& array) {
-    if (array.length() < 0) { return Status::Invalid("Length was negative"); }
+    if (array.length() < 0) {
+      return Status::Invalid("Length was negative");
+    }
 
     if (array.null_count() > array.length()) {
       return Status::Invalid("Null count exceeds the length of this struct");
@@ -610,7 +639,9 @@ struct ValidateVisitor {
   }
 
   Status Visit(const UnionArray& array) {
-    if (array.length() < 0) { return Status::Invalid("Length was negative"); }
+    if (array.length() < 0) {
+      return Status::Invalid("Length was negative");
+    }
 
     if (array.null_count() > array.length()) {
       return Status::Invalid("Null count exceeds the length of this struct");
@@ -661,8 +692,9 @@ Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>*
 }  // namespace internal
 
 Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
-    const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap,
-    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
+                          const std::shared_ptr<Buffer>& data,
+                          const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                          int64_t offset, std::shared_ptr<Array>* out) {
   BufferVector buffers = {null_bitmap, data};
   auto internal_data = std::make_shared<internal::ArrayData>(
       type, length, std::move(buffers), null_count, offset);
@@ -670,8 +702,9 @@ Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
 }
 
 Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
-    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) {
+                          const std::vector<std::shared_ptr<Buffer>>& buffers,
+                          int64_t length, int64_t null_count, int64_t offset,
+                          std::shared_ptr<Array>* out) {
   auto internal_data =
       std::make_shared<internal::ArrayData>(type, length, buffers, null_count, offset);
   return internal::MakeArray(internal_data, out);
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index c32d5e1c93ffd..a853f2bb5f93d 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -88,8 +88,8 @@ struct ARROW_EXPORT ArrayData {
   ArrayData() {}
 
   ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::vector<std::shared_ptr<Buffer>>& buffers,
-      int64_t null_count = kUnknownNullCount, int64_t offset = 0)
+            const std::vector<std::shared_ptr<Buffer>>& buffers,
+            int64_t null_count = kUnknownNullCount, int64_t offset = 0)
       : type(type),
         length(length),
         buffers(buffers),
@@ -97,8 +97,8 @@ struct ARROW_EXPORT ArrayData {
         offset(offset) {}
 
   ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
-      std::vector<std::shared_ptr<Buffer>>&& buffers,
-      int64_t null_count = kUnknownNullCount, int64_t offset = 0)
+            std::vector<std::shared_ptr<Buffer>>&& buffers,
+            int64_t null_count = kUnknownNullCount, int64_t offset = 0)
       : type(type),
         length(length),
         buffers(std::move(buffers)),
@@ -145,8 +145,8 @@ struct ARROW_EXPORT ArrayData {
   std::vector<std::shared_ptr<ArrayData>> child_data;
 };
 
-Status ARROW_EXPORT MakeArray(
-    const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out);
+Status ARROW_EXPORT MakeArray(const std::shared_ptr<ArrayData>& data,
+                              std::shared_ptr<Array>* out);
 
 }  // namespace internal
 
@@ -211,10 +211,10 @@ class ARROW_EXPORT Array {
   /// Compare if the range of slots specified are equal for the given array and
   /// this array.  end_idx exclusive.  This methods does not bounds check.
   bool RangeEquals(int64_t start_idx, int64_t end_idx, int64_t other_start_idx,
-      const std::shared_ptr<Array>& other) const;
+                   const std::shared_ptr<Array>& other) const;
 
   bool RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
-      int64_t other_start_idx) const;
+                   int64_t other_start_idx) const;
 
   Status Accept(ArrayVisitor* visitor) const;
 
@@ -285,9 +285,9 @@ class ARROW_EXPORT NullArray : public FlatArray {
 class ARROW_EXPORT PrimitiveArray : public FlatArray {
  public:
   PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+                 const std::shared_ptr<Buffer>& data,
+                 const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+                 int64_t null_count = 0, int64_t offset = 0);
 
   /// Does not account for any slice offset
   std::shared_ptr<Buffer> values() const { return data_->buffers[1]; }
@@ -328,7 +328,7 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
       const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
       int64_t offset = 0)
       : PrimitiveArray(TypeTraits<T1>::type_singleton(), length, data, null_bitmap,
-            null_count, offset) {}
+                       null_count, offset) {}
 
   const value_type* raw_values() const {
     return reinterpret_cast<const value_type*>(raw_values_) + data_->offset;
@@ -349,14 +349,14 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
   explicit BooleanArray(const std::shared_ptr<internal::ArrayData>& data);
 
   BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+               const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+               int64_t null_count = 0, int64_t offset = 0);
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
   bool Value(int64_t i) const {
-    return BitUtil::GetBit(
-        reinterpret_cast<const uint8_t*>(raw_values_), i + data_->offset);
+    return BitUtil::GetBit(reinterpret_cast<const uint8_t*>(raw_values_),
+                           i + data_->offset);
   }
 
  protected:
@@ -373,9 +373,10 @@ class ARROW_EXPORT ListArray : public Array {
   explicit ListArray(const std::shared_ptr<internal::ArrayData>& data);
 
   ListArray(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Array>& values,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+            const std::shared_ptr<Buffer>& value_offsets,
+            const std::shared_ptr<Array>& values,
+            const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+            int64_t offset = 0);
 
   /// \brief Construct ListArray from array of offsets and child value array
   ///
@@ -388,7 +389,7 @@ class ARROW_EXPORT ListArray : public Array {
   /// allocated because of null values
   /// \param[out] out Will have length equal to offsets.length() - 1
   static Status FromArrays(const Array& offsets, const Array& values, MemoryPool* pool,
-      std::shared_ptr<Array>* out);
+                           std::shared_ptr<Array>* out);
 
   /// \brief Return array object containing the list's values
   std::shared_ptr<Array> values() const;
@@ -428,9 +429,9 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
   explicit BinaryArray(const std::shared_ptr<internal::ArrayData>& data);
 
   BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
-      const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+              const std::shared_ptr<Buffer>& data,
+              const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+              int64_t null_count = 0, int64_t offset = 0);
 
   // Return the pointer to the given elements bytes
   // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
@@ -471,9 +472,10 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
   // Constructor that allows sub-classes/builders to propagate there logical type up the
   // class hierarchy.
   BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::shared_ptr<Buffer>& value_offsets, const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+              const std::shared_ptr<Buffer>& value_offsets,
+              const std::shared_ptr<Buffer>& data,
+              const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+              int64_t null_count = 0, int64_t offset = 0);
 
   const int32_t* raw_value_offsets_;
   const uint8_t* raw_data_;
@@ -486,9 +488,9 @@ class ARROW_EXPORT StringArray : public BinaryArray {
   explicit StringArray(const std::shared_ptr<internal::ArrayData>& data);
 
   StringArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
-      const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+              const std::shared_ptr<Buffer>& data,
+              const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+              int64_t null_count = 0, int64_t offset = 0);
 
   // Construct a std::string
   // TODO: std::bad_alloc possibility
@@ -511,9 +513,9 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
   explicit FixedSizeBinaryArray(const std::shared_ptr<internal::ArrayData>& data);
 
   FixedSizeBinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+                       const std::shared_ptr<Buffer>& data,
+                       const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+                       int64_t null_count = 0, int64_t offset = 0);
 
   const uint8_t* GetValue(int64_t i) const;
 
@@ -542,9 +544,10 @@ class ARROW_EXPORT DecimalArray : public FlatArray {
   explicit DecimalArray(const std::shared_ptr<internal::ArrayData>& data);
 
   DecimalArray(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0, const std::shared_ptr<Buffer>& sign_bitmap = nullptr);
+               const std::shared_ptr<Buffer>& data,
+               const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+               int64_t null_count = 0, int64_t offset = 0,
+               const std::shared_ptr<Buffer>& sign_bitmap = nullptr);
 
   bool IsNegative(int64_t i) const;
 
@@ -582,9 +585,9 @@ class ARROW_EXPORT StructArray : public Array {
   explicit StructArray(const std::shared_ptr<internal::ArrayData>& data);
 
   StructArray(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::vector<std::shared_ptr<Array>>& children,
-      std::shared_ptr<Buffer> null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+              const std::vector<std::shared_ptr<Array>>& children,
+              std::shared_ptr<Buffer> null_bitmap = nullptr, int64_t null_count = 0,
+              int64_t offset = 0);
 
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
@@ -604,11 +607,11 @@ class ARROW_EXPORT UnionArray : public Array {
   explicit UnionArray(const std::shared_ptr<internal::ArrayData>& data);
 
   UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
-      const std::vector<std::shared_ptr<Array>>& children,
-      const std::shared_ptr<Buffer>& type_ids,
-      const std::shared_ptr<Buffer>& value_offsets = nullptr,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
-      int64_t offset = 0);
+             const std::vector<std::shared_ptr<Array>>& children,
+             const std::shared_ptr<Buffer>& type_ids,
+             const std::shared_ptr<Buffer>& value_offsets = nullptr,
+             const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+             int64_t offset = 0);
 
   /// Note that this buffer does not account for any slice offset
   std::shared_ptr<Buffer> type_ids() const { return data_->buffers[1]; }
@@ -656,8 +659,8 @@ class ARROW_EXPORT DictionaryArray : public Array {
 
   explicit DictionaryArray(const std::shared_ptr<internal::ArrayData>& data);
 
-  DictionaryArray(
-      const std::shared_ptr<DataType>& type, const std::shared_ptr<Array>& indices);
+  DictionaryArray(const std::shared_ptr<DataType>& type,
+                  const std::shared_ptr<Array>& indices);
 
   std::shared_ptr<Array> indices() const;
   std::shared_ptr<Array> dictionary() const;
@@ -705,13 +708,16 @@ Status ARROW_EXPORT ValidateArray(const Array& array);
 
 /// Create new arrays for logical types that are backed by primitive arrays.
 Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    int64_t length, const std::shared_ptr<Buffer>& data,
-    const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset,
-    std::shared_ptr<Array>* out);
-
-Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
-    int64_t null_count, int64_t offset, std::shared_ptr<Array>* out);
+                                       int64_t length,
+                                       const std::shared_ptr<Buffer>& data,
+                                       const std::shared_ptr<Buffer>& null_bitmap,
+                                       int64_t null_count, int64_t offset,
+                                       std::shared_ptr<Array>* out);
+
+Status ARROW_EXPORT
+MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+                   const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
+                   int64_t null_count, int64_t offset, std::shared_ptr<Array>* out);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index a1d119ecdcae5..b9c5897f8a228 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -27,8 +27,8 @@
 
 namespace arrow {
 
-Status Buffer::Copy(
-    int64_t start, int64_t nbytes, MemoryPool* pool, std::shared_ptr<Buffer>* out) const {
+Status Buffer::Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
+                    std::shared_ptr<Buffer>* out) const {
   // Sanity checks
   DCHECK_LT(start, size_);
   DCHECK_LE(nbytes, size_ - start);
@@ -47,25 +47,28 @@ Status Buffer::Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out)
 }
 
 bool Buffer::Equals(const Buffer& other, int64_t nbytes) const {
-  return this == &other ||
-         (size_ >= nbytes && other.size_ >= nbytes &&
-             (data_ == other.data_ ||
-                 !memcmp(data_, other.data_, static_cast<size_t>(nbytes))));
+  return this == &other || (size_ >= nbytes && other.size_ >= nbytes &&
+                            (data_ == other.data_ ||
+                             !memcmp(data_, other.data_, static_cast<size_t>(nbytes))));
 }
 
 bool Buffer::Equals(const Buffer& other) const {
-  return this == &other || (size_ == other.size_ && (data_ == other.data_ ||
-                                                        !memcmp(data_, other.data_,
-                                                            static_cast<size_t>(size_))));
+  return this == &other || (size_ == other.size_ &&
+                            (data_ == other.data_ ||
+                             !memcmp(data_, other.data_, static_cast<size_t>(size_))));
 }
 
 PoolBuffer::PoolBuffer(MemoryPool* pool) : ResizableBuffer(nullptr, 0) {
-  if (pool == nullptr) { pool = default_memory_pool(); }
+  if (pool == nullptr) {
+    pool = default_memory_pool();
+  }
   pool_ = pool;
 }
 
 PoolBuffer::~PoolBuffer() {
-  if (mutable_data_ != nullptr) { pool_->Free(mutable_data_, capacity_); }
+  if (mutable_data_ != nullptr) {
+    pool_->Free(mutable_data_, capacity_);
+  }
 }
 
 Status PoolBuffer::Reserve(int64_t new_capacity) {
@@ -109,28 +112,28 @@ Status PoolBuffer::Resize(int64_t new_size, bool shrink_to_fit) {
   return Status::OK();
 }
 
-std::shared_ptr<Buffer> SliceMutableBuffer(
-    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
+std::shared_ptr<Buffer> SliceMutableBuffer(const std::shared_ptr<Buffer>& buffer,
+                                           int64_t offset, int64_t length) {
   return std::make_shared<MutableBuffer>(buffer, offset, length);
 }
 
-MutableBuffer::MutableBuffer(
-    const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size)
+MutableBuffer::MutableBuffer(const std::shared_ptr<Buffer>& parent, int64_t offset,
+                             int64_t size)
     : MutableBuffer(parent->mutable_data() + offset, size) {
   DCHECK(parent->is_mutable()) << "Must pass mutable buffer";
   parent_ = parent;
 }
 
-Status AllocateBuffer(
-    MemoryPool* pool, int64_t size, std::shared_ptr<MutableBuffer>* out) {
+Status AllocateBuffer(MemoryPool* pool, int64_t size,
+                      std::shared_ptr<MutableBuffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(buffer->Resize(size));
   *out = buffer;
   return Status::OK();
 }
 
-Status AllocateResizableBuffer(
-    MemoryPool* pool, int64_t size, std::shared_ptr<ResizableBuffer>* out) {
+Status AllocateResizableBuffer(MemoryPool* pool, int64_t size,
+                               std::shared_ptr<ResizableBuffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(buffer->Resize(size));
   *out = buffer;
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 488a4c05334d5..09e539d162fb2 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -72,7 +72,7 @@ class ARROW_EXPORT Buffer {
 
   /// Copy a section of the buffer into a new Buffer.
   Status Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
-      std::shared_ptr<Buffer>* out) const;
+              std::shared_ptr<Buffer>* out) const;
 
   /// Copy a section of the buffer using the default memory pool into a new Buffer.
   Status Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const;
@@ -106,21 +106,21 @@ class ARROW_EXPORT Buffer {
 /// \param str std::string instance
 /// \return std::shared_ptr<Buffer>
 static inline std::shared_ptr<Buffer> GetBufferFromString(const std::string& str) {
-  return std::make_shared<Buffer>(
-      reinterpret_cast<const uint8_t*>(str.c_str()), static_cast<int64_t>(str.size()));
+  return std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(str.c_str()),
+                                  static_cast<int64_t>(str.size()));
 }
 
 /// Construct a view on passed buffer at the indicated offset and length. This
 /// function cannot fail and does not error checking (except in debug builds)
-static inline std::shared_ptr<Buffer> SliceBuffer(
-    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
+static inline std::shared_ptr<Buffer> SliceBuffer(const std::shared_ptr<Buffer>& buffer,
+                                                  int64_t offset, int64_t length) {
   return std::make_shared<Buffer>(buffer, offset, length);
 }
 
 /// Construct a mutable buffer slice. If the parent buffer is not mutable, this
 /// will abort in debug builds
-std::shared_ptr<Buffer> ARROW_EXPORT SliceMutableBuffer(
-    const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
+std::shared_ptr<Buffer> ARROW_EXPORT
+SliceMutableBuffer(const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
 
 /// A Buffer whose contents can be mutated. May or may not own its data.
 class ARROW_EXPORT MutableBuffer : public Buffer {
@@ -186,8 +186,12 @@ class ARROW_EXPORT BufferBuilder {
   /// Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
   Status Resize(int64_t elements) {
     // Resize(0) is a no-op
-    if (elements == 0) { return Status::OK(); }
-    if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
+    if (elements == 0) {
+      return Status::OK();
+    }
+    if (capacity_ == 0) {
+      buffer_ = std::make_shared<PoolBuffer>(pool_);
+    }
     int64_t old_capacity = capacity_;
     RETURN_NOT_OK(buffer_->Resize(elements));
     capacity_ = buffer_->capacity();
@@ -199,14 +203,18 @@ class ARROW_EXPORT BufferBuilder {
   }
 
   Status Append(const uint8_t* data, int64_t length) {
-    if (capacity_ < length + size_) { RETURN_NOT_OK(Resize(length + size_)); }
+    if (capacity_ < length + size_) {
+      RETURN_NOT_OK(Resize(length + size_));
+    }
     UnsafeAppend(data, length);
     return Status::OK();
   }
 
   // Advance pointer and zero out memory
   Status Advance(int64_t length) {
-    if (capacity_ < length + size_) { RETURN_NOT_OK(Resize(length + size_)); }
+    if (capacity_ < length + size_) {
+      RETURN_NOT_OK(Resize(length + size_));
+    }
     memset(data_ + size_, 0, static_cast<size_t>(length));
     size_ += length;
     return Status::OK();
@@ -220,7 +228,9 @@ class ARROW_EXPORT BufferBuilder {
 
   Status Finish(std::shared_ptr<Buffer>* out) {
     // Do not shrink to fit to avoid unneeded realloc
-    if (size_ > 0) { RETURN_NOT_OK(buffer_->Resize(size_, false)); }
+    if (size_ > 0) {
+      RETURN_NOT_OK(buffer_->Resize(size_, false));
+    }
     *out = buffer_;
     Reset();
     return Status::OK();
@@ -250,29 +260,29 @@ class ARROW_EXPORT TypedBufferBuilder : public BufferBuilder {
 
   Status Append(T arithmetic_value) {
     static_assert(std::is_arithmetic<T>::value,
-        "Convenience buffer append only supports arithmetic types");
-    return BufferBuilder::Append(
-        reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
+                  "Convenience buffer append only supports arithmetic types");
+    return BufferBuilder::Append(reinterpret_cast<uint8_t*>(&arithmetic_value),
+                                 sizeof(T));
   }
 
   Status Append(const T* arithmetic_values, int64_t num_elements) {
     static_assert(std::is_arithmetic<T>::value,
-        "Convenience buffer append only supports arithmetic types");
-    return BufferBuilder::Append(
-        reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
+                  "Convenience buffer append only supports arithmetic types");
+    return BufferBuilder::Append(reinterpret_cast<const uint8_t*>(arithmetic_values),
+                                 num_elements * sizeof(T));
   }
 
   void UnsafeAppend(T arithmetic_value) {
     static_assert(std::is_arithmetic<T>::value,
-        "Convenience buffer append only supports arithmetic types");
+                  "Convenience buffer append only supports arithmetic types");
     BufferBuilder::UnsafeAppend(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
   }
 
   void UnsafeAppend(const T* arithmetic_values, int64_t num_elements) {
     static_assert(std::is_arithmetic<T>::value,
-        "Convenience buffer append only supports arithmetic types");
-    BufferBuilder::UnsafeAppend(
-        reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
+                  "Convenience buffer append only supports arithmetic types");
+    BufferBuilder::UnsafeAppend(reinterpret_cast<const uint8_t*>(arithmetic_values),
+                                num_elements * sizeof(T));
   }
 
   const T* data() const { return reinterpret_cast<const T*>(data_); }
@@ -286,11 +296,11 @@ class ARROW_EXPORT TypedBufferBuilder : public BufferBuilder {
 /// \param[out] out the allocated buffer with padding
 ///
 /// \return Status message
-Status ARROW_EXPORT AllocateBuffer(
-    MemoryPool* pool, int64_t size, std::shared_ptr<MutableBuffer>* out);
+Status ARROW_EXPORT AllocateBuffer(MemoryPool* pool, int64_t size,
+                                   std::shared_ptr<MutableBuffer>* out);
 
-Status ARROW_EXPORT AllocateResizableBuffer(
-    MemoryPool* pool, int64_t size, std::shared_ptr<ResizableBuffer>* out);
+Status ARROW_EXPORT AllocateResizableBuffer(MemoryPool* pool, int64_t size,
+                                            std::shared_ptr<ResizableBuffer>* out);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index 7ca7bb4999801..8ba9360e917fc 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -38,8 +38,8 @@ static void BM_BuildPrimitiveArrayNoNulls(
     std::shared_ptr<Array> out;
     ABORT_NOT_OK(builder.Finish(&out));
   }
-  state.SetBytesProcessed(
-      state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
+  state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t) *
+                          kFinalSize);
 }
 
 static void BM_BuildVectorNoNulls(
@@ -53,8 +53,8 @@ static void BM_BuildVectorNoNulls(
       builder.insert(builder.end(), data.cbegin(), data.cend());
     }
   }
-  state.SetBytesProcessed(
-      state.iterations() * data.size() * sizeof(int64_t) * kFinalSize);
+  state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t) *
+                          kFinalSize);
 }
 
 static void BM_BuildAdaptiveIntNoNulls(
@@ -127,8 +127,8 @@ static void BM_BuildDictionary(benchmark::State& state) {  // NOLINT non-const r
     std::shared_ptr<Array> out;
     ABORT_NOT_OK(builder.Finish(&out));
   }
-  state.SetBytesProcessed(
-      state.iterations() * iterations * (iterations + 1) / 2 * sizeof(int64_t));
+  state.SetBytesProcessed(state.iterations() * iterations * (iterations + 1) / 2 *
+                          sizeof(int64_t));
 }
 
 static void BM_BuildStringDictionary(
@@ -152,8 +152,8 @@ static void BM_BuildStringDictionary(
     ABORT_NOT_OK(builder.Finish(&out));
   }
   // Assuming a string here needs on average 2 bytes
-  state.SetBytesProcessed(
-      state.iterations() * iterations * (iterations + 1) / 2 * sizeof(int32_t));
+  state.SetBytesProcessed(state.iterations() * iterations * (iterations + 1) / 2 *
+                          sizeof(int32_t));
 }
 
 BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index ee363b91d8fcc..d3a299e5412fc 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -69,7 +69,9 @@ Status ArrayBuilder::Init(int64_t capacity) {
 }
 
 Status ArrayBuilder::Resize(int64_t new_bits) {
-  if (!null_bitmap_) { return Init(new_bits); }
+  if (!null_bitmap_) {
+    return Init(new_bits);
+  }
   int64_t new_bytes = BitUtil::CeilByte(new_bits) / 8;
   int64_t old_bytes = null_bitmap_->size();
   RETURN_NOT_OK(null_bitmap_->Resize(new_bytes));
@@ -78,8 +80,8 @@ Status ArrayBuilder::Resize(int64_t new_bits) {
   const int64_t byte_capacity = null_bitmap_->capacity();
   capacity_ = new_bits;
   if (old_bytes < new_bytes) {
-    memset(
-        null_bitmap_data_ + old_bytes, 0, static_cast<size_t>(byte_capacity - old_bytes));
+    memset(null_bitmap_data_ + old_bytes, 0,
+           static_cast<size_t>(byte_capacity - old_bytes));
   }
   return Status::OK();
 }
@@ -140,7 +142,9 @@ void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t leng
 
     bit_offset++;
   }
-  if (bit_offset != 0) { null_bitmap_data_[byte_offset] = bitset; }
+  if (bit_offset != 0) {
+    null_bitmap_data_[byte_offset] = bitset;
+  }
   length_ += length;
 }
 
@@ -149,7 +153,9 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
 
   // Fill up the bytes until we have a byte alignment
   int64_t pad_to_byte = std::min<int64_t>(8 - (length_ % 8), length);
-  if (pad_to_byte == 8) { pad_to_byte = 0; }
+  if (pad_to_byte == 8) {
+    pad_to_byte = 0;
+  }
   for (int64_t i = length_; i < length_ + pad_to_byte; ++i) {
     BitUtil::SetBit(null_bitmap_data_, i);
   }
@@ -157,7 +163,7 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
   // Fast bitsetting
   int64_t fast_length = (length - pad_to_byte) / 8;
   memset(null_bitmap_data_ + ((length_ + pad_to_byte) / 8), 0xFF,
-      static_cast<size_t>(fast_length));
+         static_cast<size_t>(fast_length));
 
   // Trailing bytes
   for (int64_t i = length_ + pad_to_byte + (fast_length * 8); i < new_length; ++i) {
@@ -184,7 +190,9 @@ Status PrimitiveBuilder<T>::Init(int64_t capacity) {
 template <typename T>
 Status PrimitiveBuilder<T>::Resize(int64_t capacity) {
   // XXX: Set floor size for now
-  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
 
   if (capacity_ == 0) {
     RETURN_NOT_OK(Init(capacity));
@@ -195,20 +203,20 @@ Status PrimitiveBuilder<T>::Resize(int64_t capacity) {
     RETURN_NOT_OK(data_->Resize(new_bytes));
     raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
     // TODO(emkornfield) valgrind complains without this
-    memset(
-        data_->mutable_data() + old_bytes, 0, static_cast<size_t>(new_bytes - old_bytes));
+    memset(data_->mutable_data() + old_bytes, 0,
+           static_cast<size_t>(new_bytes - old_bytes));
   }
   return Status::OK();
 }
 
 template <typename T>
-Status PrimitiveBuilder<T>::Append(
-    const value_type* values, int64_t length, const uint8_t* valid_bytes) {
+Status PrimitiveBuilder<T>::Append(const value_type* values, int64_t length,
+                                   const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
 
   if (length > 0) {
     std::memcpy(raw_data_ + length_, values,
-        static_cast<std::size_t>(TypeTraits<T>::bytes_required(length)));
+                static_cast<std::size_t>(TypeTraits<T>::bytes_required(length)));
   }
 
   // length_ is update by these
@@ -224,8 +232,8 @@ Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
-  *out = std::make_shared<typename TypeTraits<T>::ArrayType>(
-      type_, length_, data_, null_bitmap_, null_count_);
+  *out = std::make_shared<typename TypeTraits<T>::ArrayType>(type_, length_, data_,
+                                                             null_bitmap_, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -267,7 +275,9 @@ Status AdaptiveIntBuilderBase::Init(int64_t capacity) {
 
 Status AdaptiveIntBuilderBase::Resize(int64_t capacity) {
   // XXX: Set floor size for now
-  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
 
   if (capacity_ == 0) {
     RETURN_NOT_OK(Init(capacity));
@@ -278,8 +288,8 @@ Status AdaptiveIntBuilderBase::Resize(int64_t capacity) {
     RETURN_NOT_OK(data_->Resize(new_bytes));
     raw_data_ = data_->mutable_data();
     // TODO(emkornfield) valgrind complains without this
-    memset(
-        data_->mutable_data() + old_bytes, 0, static_cast<size_t>(new_bytes - old_bytes));
+    memset(data_->mutable_data() + old_bytes, 0,
+           static_cast<size_t>(new_bytes - old_bytes));
   }
   return Status::OK();
 }
@@ -298,16 +308,16 @@ Status AdaptiveIntBuilder::Finish(std::shared_ptr<Array>* out) {
           std::make_shared<Int8Array>(int8(), length_, data_, null_bitmap_, null_count_);
       break;
     case 2:
-      *out = std::make_shared<Int16Array>(
-          int16(), length_, data_, null_bitmap_, null_count_);
+      *out = std::make_shared<Int16Array>(int16(), length_, data_, null_bitmap_,
+                                          null_count_);
       break;
     case 4:
-      *out = std::make_shared<Int32Array>(
-          int32(), length_, data_, null_bitmap_, null_count_);
+      *out = std::make_shared<Int32Array>(int32(), length_, data_, null_bitmap_,
+                                          null_count_);
       break;
     case 8:
-      *out = std::make_shared<Int64Array>(
-          int64(), length_, data_, null_bitmap_, null_count_);
+      *out = std::make_shared<Int64Array>(int64(), length_, data_, null_bitmap_,
+                                          null_count_);
       break;
     default:
       DCHECK(false);
@@ -319,8 +329,8 @@ Status AdaptiveIntBuilder::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
-Status AdaptiveIntBuilder::Append(
-    const int64_t* values, int64_t length, const uint8_t* valid_bytes) {
+Status AdaptiveIntBuilder::Append(const int64_t* values, int64_t length,
+                                  const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
 
   if (length > 0) {
@@ -331,13 +341,15 @@ Status AdaptiveIntBuilder::Append(
           new_int_size = expanded_int_size(values[i], new_int_size);
         }
       }
-      if (new_int_size != int_size_) { RETURN_NOT_OK(ExpandIntSize(new_int_size)); }
+      if (new_int_size != int_size_) {
+        RETURN_NOT_OK(ExpandIntSize(new_int_size));
+      }
     }
   }
 
   if (int_size_ == 8) {
     std::memcpy(reinterpret_cast<int64_t*>(raw_data_) + length_, values,
-        sizeof(int64_t) * length);
+                sizeof(int64_t) * length);
   } else {
 #ifdef _MSC_VER
 #pragma warning(push)
@@ -348,17 +360,17 @@ Status AdaptiveIntBuilder::Append(
       case 1: {
         int8_t* data_ptr = reinterpret_cast<int8_t*>(raw_data_) + length_;
         std::transform(values, values + length, data_ptr,
-            [](int64_t x) { return static_cast<int8_t>(x); });
+                       [](int64_t x) { return static_cast<int8_t>(x); });
       } break;
       case 2: {
         int16_t* data_ptr = reinterpret_cast<int16_t*>(raw_data_) + length_;
         std::transform(values, values + length, data_ptr,
-            [](int64_t x) { return static_cast<int16_t>(x); });
+                       [](int64_t x) { return static_cast<int16_t>(x); });
       } break;
       case 4: {
         int32_t* data_ptr = reinterpret_cast<int32_t*>(raw_data_) + length_;
         std::transform(values, values + length, data_ptr,
-            [](int64_t x) { return static_cast<int32_t>(x); });
+                       [](int64_t x) { return static_cast<int32_t>(x); });
       } break;
       default:
         DCHECK(false);
@@ -449,20 +461,20 @@ Status AdaptiveUIntBuilder::Finish(std::shared_ptr<Array>* out) {
   }
   switch (int_size_) {
     case 1:
-      *out = std::make_shared<UInt8Array>(
-          uint8(), length_, data_, null_bitmap_, null_count_);
+      *out = std::make_shared<UInt8Array>(uint8(), length_, data_, null_bitmap_,
+                                          null_count_);
       break;
     case 2:
-      *out = std::make_shared<UInt16Array>(
-          uint16(), length_, data_, null_bitmap_, null_count_);
+      *out = std::make_shared<UInt16Array>(uint16(), length_, data_, null_bitmap_,
+                                           null_count_);
       break;
     case 4:
-      *out = std::make_shared<UInt32Array>(
-          uint32(), length_, data_, null_bitmap_, null_count_);
+      *out = std::make_shared<UInt32Array>(uint32(), length_, data_, null_bitmap_,
+                                           null_count_);
       break;
     case 8:
-      *out = std::make_shared<UInt64Array>(
-          uint64(), length_, data_, null_bitmap_, null_count_);
+      *out = std::make_shared<UInt64Array>(uint64(), length_, data_, null_bitmap_,
+                                           null_count_);
       break;
     default:
       DCHECK(false);
@@ -474,8 +486,8 @@ Status AdaptiveUIntBuilder::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
-Status AdaptiveUIntBuilder::Append(
-    const uint64_t* values, int64_t length, const uint8_t* valid_bytes) {
+Status AdaptiveUIntBuilder::Append(const uint64_t* values, int64_t length,
+                                   const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
 
   if (length > 0) {
@@ -486,13 +498,15 @@ Status AdaptiveUIntBuilder::Append(
           new_int_size = expanded_uint_size(values[i], new_int_size);
         }
       }
-      if (new_int_size != int_size_) { RETURN_NOT_OK(ExpandIntSize(new_int_size)); }
+      if (new_int_size != int_size_) {
+        RETURN_NOT_OK(ExpandIntSize(new_int_size));
+      }
     }
   }
 
   if (int_size_ == 8) {
     std::memcpy(reinterpret_cast<uint64_t*>(raw_data_) + length_, values,
-        sizeof(uint64_t) * length);
+                sizeof(uint64_t) * length);
   } else {
 #ifdef _MSC_VER
 #pragma warning(push)
@@ -503,17 +517,17 @@ Status AdaptiveUIntBuilder::Append(
       case 1: {
         uint8_t* data_ptr = reinterpret_cast<uint8_t*>(raw_data_) + length_;
         std::transform(values, values + length, data_ptr,
-            [](uint64_t x) { return static_cast<uint8_t>(x); });
+                       [](uint64_t x) { return static_cast<uint8_t>(x); });
       } break;
       case 2: {
         uint16_t* data_ptr = reinterpret_cast<uint16_t*>(raw_data_) + length_;
         std::transform(values, values + length, data_ptr,
-            [](uint64_t x) { return static_cast<uint16_t>(x); });
+                       [](uint64_t x) { return static_cast<uint16_t>(x); });
       } break;
       case 4: {
         uint32_t* data_ptr = reinterpret_cast<uint32_t*>(raw_data_) + length_;
         std::transform(values, values + length, data_ptr,
-            [](uint64_t x) { return static_cast<uint32_t>(x); });
+                       [](uint64_t x) { return static_cast<uint32_t>(x); });
       } break;
       default:
         DCHECK(false);
@@ -616,7 +630,9 @@ Status BooleanBuilder::Init(int64_t capacity) {
 
 Status BooleanBuilder::Resize(int64_t capacity) {
   // XXX: Set floor size for now
-  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
 
   if (capacity_ == 0) {
     RETURN_NOT_OK(Init(capacity));
@@ -627,8 +643,8 @@ Status BooleanBuilder::Resize(int64_t capacity) {
 
     RETURN_NOT_OK(data_->Resize(new_bytes));
     raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
-    memset(
-        data_->mutable_data() + old_bytes, 0, static_cast<size_t>(new_bytes - old_bytes));
+    memset(data_->mutable_data() + old_bytes, 0,
+           static_cast<size_t>(new_bytes - old_bytes));
   }
   return Status::OK();
 }
@@ -647,8 +663,8 @@ Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
-Status BooleanBuilder::Append(
-    const uint8_t* values, int64_t length, const uint8_t* valid_bytes) {
+Status BooleanBuilder::Append(const uint8_t* values, int64_t length,
+                              const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
 
   for (int64_t i = 0; i < length; ++i) {
@@ -673,14 +689,16 @@ Status BooleanBuilder::Append(
 // DictionaryBuilder
 
 template <typename T>
-DictionaryBuilder<T>::DictionaryBuilder(
-    MemoryPool* pool, const std::shared_ptr<DataType>& type)
+DictionaryBuilder<T>::DictionaryBuilder(MemoryPool* pool,
+                                        const std::shared_ptr<DataType>& type)
     : ArrayBuilder(pool, type),
       hash_table_(new PoolBuffer(pool)),
       hash_slots_(nullptr),
       dict_builder_(pool, type),
       values_builder_(pool) {
-  if (!::arrow::CpuInfo::initialized()) { ::arrow::CpuInfo::Init(); }
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
 }
 
 template <typename T>
@@ -699,7 +717,9 @@ Status DictionaryBuilder<T>::Init(int64_t elements) {
 
 template <typename T>
 Status DictionaryBuilder<T>::Resize(int64_t capacity) {
-  if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
 
   if (capacity_ == 0) {
     return Init(capacity);
@@ -732,7 +752,9 @@ Status DictionaryBuilder<T>::Append(const Scalar& value) {
   while (kHashSlotEmpty != index && SlotDifferent(index, value)) {
     // Linear probing
     ++j;
-    if (j == hash_table_size_) { j = 0; }
+    if (j == hash_table_size_) {
+      j = 0;
+    }
     index = hash_slots_[j];
   }
 
@@ -784,7 +806,9 @@ Status DictionaryBuilder<T>::DoubleTableSize() {
   for (int i = 0; i < hash_table_size_; ++i) {
     hash_slot_t index = hash_slots_[i];
 
-    if (index == kHashSlotEmpty) { continue; }
+    if (index == kHashSlotEmpty) {
+      continue;
+    }
 
     // Compute the hash value mod the new table size to start looking for an
     // empty slot
@@ -796,7 +820,9 @@ Status DictionaryBuilder<T>::DoubleTableSize() {
 
     while (kHashSlotEmpty != slot && SlotDifferent(slot, value)) {
       ++j;
-      if (j == new_size) { j = 0; }
+      if (j == new_size) {
+        j = 0;
+      }
       slot = new_hash_slots[j];
     }
 
@@ -870,8 +896,8 @@ Status DictionaryBuilder<T>::AppendDictionary(const Scalar& value) {
   }                                                                                    \
                                                                                        \
   template <>                                                                          \
-  bool DictionaryBuilder<Type>::SlotDifferent(                                         \
-      hash_slot_t index, const internal::WrappedBinary& value) {                       \
+  bool DictionaryBuilder<Type>::SlotDifferent(hash_slot_t index,                       \
+                                              const internal::WrappedBinary& value) {  \
     int32_t other_length;                                                              \
     const uint8_t* other_value =                                                       \
         dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);            \
@@ -951,7 +977,9 @@ Status DecimalBuilder::Init(int64_t capacity) {
 
 Status DecimalBuilder::Resize(int64_t capacity) {
   int64_t old_bytes = null_bitmap_ != nullptr ? null_bitmap_->size() : 0;
-  if (sign_bitmap_ == nullptr) { return Init(capacity); }
+  if (sign_bitmap_ == nullptr) {
+    return Init(capacity);
+  }
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Resize(capacity));
 
   if (byte_width_ == 16) {
@@ -962,7 +990,7 @@ Status DecimalBuilder::Resize(int64_t capacity) {
     // The buffer might be overpadded to deal with padding according to the spec
     if (old_bytes < new_bytes) {
       memset(sign_bitmap_data_ + old_bytes, 0,
-          static_cast<size_t>(sign_bitmap_->capacity() - old_bytes));
+             static_cast<size_t>(sign_bitmap_->capacity() - old_bytes));
     }
   }
   return Status::OK();
@@ -973,8 +1001,8 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
   RETURN_NOT_OK(byte_builder_.Finish(&data));
 
   /// TODO(phillipc): not sure where to get the offset argument here
-  *out = std::make_shared<DecimalArray>(
-      type_, length_, data, null_bitmap_, null_count_, 0, sign_bitmap_);
+  *out = std::make_shared<DecimalArray>(type_, length_, data, null_bitmap_, null_count_,
+                                        0, sign_bitmap_);
   return Status::OK();
 }
 
@@ -982,15 +1010,15 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
 // ListBuilder
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
-    const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(
-          pool, type ? type : std::static_pointer_cast<DataType>(
-                                  std::make_shared<ListType>(value_builder->type()))),
+                         const std::shared_ptr<DataType>& type)
+    : ArrayBuilder(pool,
+                   type ? type : std::static_pointer_cast<DataType>(
+                                     std::make_shared<ListType>(value_builder->type()))),
       offsets_builder_(pool),
       value_builder_(std::move(value_builder)) {}
 
-Status ListBuilder::Append(
-    const int32_t* offsets, int64_t length, const uint8_t* valid_bytes) {
+Status ListBuilder::Append(const int32_t* offsets, int64_t length,
+                           const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
   UnsafeAppendToBitmap(valid_bytes, length);
   offsets_builder_.UnsafeAppend(offsets, length);
@@ -1035,10 +1063,12 @@ Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
   RETURN_NOT_OK(offsets_builder_.Finish(&offsets));
 
   std::shared_ptr<Array> items = values_;
-  if (!items) { RETURN_NOT_OK(value_builder_->Finish(&items)); }
+  if (!items) {
+    RETURN_NOT_OK(value_builder_->Finish(&items));
+  }
 
-  *out = std::make_shared<ListArray>(
-      type_, length_, offsets, items, null_bitmap_, null_count_);
+  *out = std::make_shared<ListArray>(type_, length_, offsets, items, null_bitmap_,
+                                     null_count_);
 
   Reset();
   return Status::OK();
@@ -1111,8 +1141,8 @@ Status BinaryBuilder::FinishInternal(std::shared_ptr<internal::ArrayData>* out)
   RETURN_NOT_OK(value_data_builder_.Finish(&value_data));
 
   BufferVector buffers = {null_bitmap_, offsets, value_data};
-  *out = std::make_shared<internal::ArrayData>(
-      type_, length_, std::move(buffers), null_count_, 0);
+  *out = std::make_shared<internal::ArrayData>(type_, length_, std::move(buffers),
+                                               null_count_, 0);
   return Status::OK();
 }
 
@@ -1154,8 +1184,8 @@ Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
 // ----------------------------------------------------------------------
 // Fixed width binary
 
-FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(
-    MemoryPool* pool, const std::shared_ptr<DataType>& type)
+FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(MemoryPool* pool,
+                                               const std::shared_ptr<DataType>& type)
     : ArrayBuilder(pool, type),
       byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()),
       byte_builder_(pool) {}
@@ -1166,8 +1196,8 @@ Status FixedSizeBinaryBuilder::Append(const uint8_t* value) {
   return byte_builder_.Append(value, byte_width_);
 }
 
-Status FixedSizeBinaryBuilder::Append(
-    const uint8_t* data, int64_t length, const uint8_t* valid_bytes) {
+Status FixedSizeBinaryBuilder::Append(const uint8_t* data, int64_t length,
+                                      const uint8_t* valid_bytes) {
   RETURN_NOT_OK(Reserve(length));
   UnsafeAppendToBitmap(valid_bytes, length);
   return byte_builder_.Append(data, length * byte_width_);
@@ -1196,8 +1226,8 @@ Status FixedSizeBinaryBuilder::Resize(int64_t capacity) {
 Status FixedSizeBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
-  *out = std::make_shared<FixedSizeBinaryArray>(
-      type_, length_, data, null_bitmap_, null_count_);
+  *out = std::make_shared<FixedSizeBinaryArray>(type_, length_, data, null_bitmap_,
+                                                null_count_);
   return Status::OK();
 }
 
@@ -1205,7 +1235,7 @@ Status FixedSizeBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
 // Struct
 
 StructBuilder::StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-    std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders)
+                             std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders)
     : ArrayBuilder(pool, type) {
   field_builders_ = std::move(field_builders);
 }
@@ -1237,7 +1267,7 @@ Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
 //
 // TODO(wesm): come up with a less monolithic strategy
 Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-    std::unique_ptr<ArrayBuilder>* out) {
+                   std::unique_ptr<ArrayBuilder>* out) {
   switch (type->id()) {
     BUILDER_CASE(UINT8, UInt8Builder);
     BUILDER_CASE(INT8, Int8Builder);
@@ -1292,7 +1322,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     return Status::OK();
 
 Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-    std::shared_ptr<ArrayBuilder>* out) {
+                             std::shared_ptr<ArrayBuilder>* out) {
   switch (type->id()) {
     DICTIONARY_BUILDER_CASE(UINT8, DictionaryBuilder<UInt8Type>);
     DICTIONARY_BUILDER_CASE(INT8, DictionaryBuilder<Int8Type>);
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 065e115ac5872..080a32900555c 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -186,8 +186,8 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   ///
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
-  Status Append(
-      const value_type* values, int64_t length, const uint8_t* valid_bytes = nullptr);
+  Status Append(const value_type* values, int64_t length,
+                const uint8_t* valid_bytes = nullptr);
 
   Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int64_t capacity) override;
@@ -298,15 +298,15 @@ class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
 inline uint8_t expanded_uint_size(uint64_t val, uint8_t current_int_size) {
   if (current_int_size == 8 ||
       (current_int_size < 8 &&
-          (val > static_cast<uint64_t>(std::numeric_limits<uint32_t>::max())))) {
+       (val > static_cast<uint64_t>(std::numeric_limits<uint32_t>::max())))) {
     return 8;
   } else if (current_int_size == 4 ||
              (current_int_size < 4 &&
-                 (val > static_cast<uint64_t>(std::numeric_limits<uint16_t>::max())))) {
+              (val > static_cast<uint64_t>(std::numeric_limits<uint16_t>::max())))) {
     return 4;
   } else if (current_int_size == 2 ||
              (current_int_size == 1 &&
-                 (val > static_cast<uint64_t>(std::numeric_limits<uint8_t>::max())))) {
+              (val > static_cast<uint64_t>(std::numeric_limits<uint8_t>::max())))) {
     return 2;
   } else {
     return 1;
@@ -325,7 +325,9 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public AdaptiveIntBuilderBase {
     BitUtil::SetBit(null_bitmap_data_, length_);
 
     uint8_t new_int_size = expanded_uint_size(val, int_size_);
-    if (new_int_size != int_size_) { RETURN_NOT_OK(ExpandIntSize(new_int_size)); }
+    if (new_int_size != int_size_) {
+      RETURN_NOT_OK(ExpandIntSize(new_int_size));
+    }
 
     switch (int_size_) {
       case 1:
@@ -350,8 +352,8 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public AdaptiveIntBuilderBase {
   ///
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
-  Status Append(
-      const uint64_t* values, int64_t length, const uint8_t* valid_bytes = nullptr);
+  Status Append(const uint64_t* values, int64_t length,
+                const uint8_t* valid_bytes = nullptr);
 
   Status ExpandIntSize(uint8_t new_int_size);
   Status Finish(std::shared_ptr<Array>* out) override;
@@ -374,18 +376,18 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public AdaptiveIntBuilderBase {
 inline uint8_t expanded_int_size(int64_t val, uint8_t current_int_size) {
   if (current_int_size == 8 ||
       (current_int_size < 8 &&
-          (val > static_cast<int64_t>(std::numeric_limits<int32_t>::max()) ||
-              val < static_cast<int64_t>(std::numeric_limits<int32_t>::min())))) {
+       (val > static_cast<int64_t>(std::numeric_limits<int32_t>::max()) ||
+        val < static_cast<int64_t>(std::numeric_limits<int32_t>::min())))) {
     return 8;
   } else if (current_int_size == 4 ||
              (current_int_size < 4 &&
-                 (val > static_cast<int64_t>(std::numeric_limits<int16_t>::max()) ||
-                     val < static_cast<int64_t>(std::numeric_limits<int16_t>::min())))) {
+              (val > static_cast<int64_t>(std::numeric_limits<int16_t>::max()) ||
+               val < static_cast<int64_t>(std::numeric_limits<int16_t>::min())))) {
     return 4;
   } else if (current_int_size == 2 ||
              (current_int_size == 1 &&
-                 (val > static_cast<int64_t>(std::numeric_limits<int8_t>::max()) ||
-                     val < static_cast<int64_t>(std::numeric_limits<int8_t>::min())))) {
+              (val > static_cast<int64_t>(std::numeric_limits<int8_t>::max()) ||
+               val < static_cast<int64_t>(std::numeric_limits<int8_t>::min())))) {
     return 2;
   } else {
     return 1;
@@ -404,7 +406,9 @@ class ARROW_EXPORT AdaptiveIntBuilder : public AdaptiveIntBuilderBase {
     BitUtil::SetBit(null_bitmap_data_, length_);
 
     uint8_t new_int_size = expanded_int_size(val, int_size_);
-    if (new_int_size != int_size_) { RETURN_NOT_OK(ExpandIntSize(new_int_size)); }
+    if (new_int_size != int_size_) {
+      RETURN_NOT_OK(ExpandIntSize(new_int_size));
+    }
 
     switch (int_size_) {
       case 1:
@@ -429,8 +433,8 @@ class ARROW_EXPORT AdaptiveIntBuilder : public AdaptiveIntBuilderBase {
   ///
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
-  Status Append(
-      const int64_t* values, int64_t length, const uint8_t* valid_bytes = nullptr);
+  Status Append(const int64_t* values, int64_t length,
+                const uint8_t* valid_bytes = nullptr);
 
   Status ExpandIntSize(uint8_t new_int_size);
   Status Finish(std::shared_ptr<Array>* out) override;
@@ -490,8 +494,8 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   ///
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
-  Status Append(
-      const uint8_t* values, int64_t length, const uint8_t* valid_bytes = nullptr);
+  Status Append(const uint8_t* values, int64_t length,
+                const uint8_t* valid_bytes = nullptr);
 
   Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int64_t capacity) override;
@@ -526,7 +530,7 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   /// Use this constructor to incrementally build the value array along with offsets and
   /// null bitmap.
   ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
-      const std::shared_ptr<DataType>& type = nullptr);
+              const std::shared_ptr<DataType>& type = nullptr);
 
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
@@ -536,8 +540,8 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   ///
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
-  Status Append(
-      const int32_t* offsets, int64_t length, const uint8_t* valid_bytes = nullptr);
+  Status Append(const int32_t* offsets, int64_t length,
+                const uint8_t* valid_bytes = nullptr);
 
   /// \brief Start a new variable-length list slot
   ///
@@ -626,8 +630,8 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
   FixedSizeBinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 
   Status Append(const uint8_t* value);
-  Status Append(
-      const uint8_t* data, int64_t length, const uint8_t* valid_bytes = nullptr);
+  Status Append(const uint8_t* data, int64_t length,
+                const uint8_t* valid_bytes = nullptr);
   Status Append(const std::string& value);
   Status AppendNull();
 
@@ -672,7 +676,7 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
 class ARROW_EXPORT StructBuilder : public ArrayBuilder {
  public:
   StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-      std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders);
+                std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders);
 
   Status Finish(std::shared_ptr<Array>* out) override;
 
@@ -808,7 +812,7 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 
   Status Append(const std::string& value) {
     return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
-        static_cast<int32_t>(value.size())));
+                                          static_cast<int32_t>(value.size())));
   }
 };
 
@@ -829,7 +833,7 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 
   Status Append(const std::string& value) {
     return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
-        static_cast<int32_t>(value.size())));
+                                          static_cast<int32_t>(value.size())));
   }
 };
 
@@ -837,10 +841,11 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 // Helper functions
 
 Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-    std::unique_ptr<ArrayBuilder>* out);
+                                std::unique_ptr<ArrayBuilder>* out);
 
 Status ARROW_EXPORT MakeDictionaryBuilder(MemoryPool* pool,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<ArrayBuilder>* out);
+                                          const std::shared_ptr<DataType>& type,
+                                          std::shared_ptr<ArrayBuilder>* out);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 1465e0b414fe3..da10c2ad90177 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -41,7 +41,7 @@ namespace arrow {
 class RangeEqualsVisitor {
  public:
   RangeEqualsVisitor(const Array& right, int64_t left_start_idx, int64_t left_end_idx,
-      int64_t right_start_idx)
+                     int64_t right_start_idx)
       : right_(right),
         left_start_idx_(left_start_idx),
         left_end_idx_(left_end_idx),
@@ -71,7 +71,9 @@ class RangeEqualsVisitor {
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
       const bool is_null = left.IsNull(i);
-      if (is_null != right.IsNull(o_i)) { return false; }
+      if (is_null != right.IsNull(o_i)) {
+        return false;
+      }
       if (is_null) continue;
       const int32_t begin_offset = left.value_offset(i);
       const int32_t end_offset = left.value_offset(i + 1);
@@ -84,8 +86,8 @@ class RangeEqualsVisitor {
 
       if (end_offset - begin_offset > 0 &&
           std::memcmp(left.value_data()->data() + begin_offset,
-              right.value_data()->data() + right_begin_offset,
-              static_cast<size_t>(end_offset - begin_offset))) {
+                      right.value_data()->data() + right_begin_offset,
+                      static_cast<size_t>(end_offset - begin_offset))) {
         return false;
       }
     }
@@ -101,7 +103,9 @@ class RangeEqualsVisitor {
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
       const bool is_null = left.IsNull(i);
-      if (is_null != right.IsNull(o_i)) { return false; }
+      if (is_null != right.IsNull(o_i)) {
+        return false;
+      }
       if (is_null) continue;
       const int32_t begin_offset = left.value_offset(i);
       const int32_t end_offset = left.value_offset(i + 1);
@@ -111,8 +115,8 @@ class RangeEqualsVisitor {
       if (end_offset - begin_offset != right_end_offset - right_begin_offset) {
         return false;
       }
-      if (!left_values->RangeEquals(
-              begin_offset, end_offset, right_begin_offset, right_values)) {
+      if (!left_values->RangeEquals(begin_offset, end_offset, right_begin_offset,
+                                    right_values)) {
         return false;
       }
     }
@@ -124,7 +128,9 @@ class RangeEqualsVisitor {
     bool equal_fields = true;
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
-      if (left.IsNull(i) != right.IsNull(o_i)) { return false; }
+      if (left.IsNull(i) != right.IsNull(o_i)) {
+        return false;
+      }
       if (left.IsNull(i)) continue;
       for (int j = 0; j < left.num_fields(); ++j) {
         // TODO: really we should be comparing stretches of non-null data rather
@@ -132,9 +138,11 @@ class RangeEqualsVisitor {
         const int64_t left_abs_index = i + left.offset();
         const int64_t right_abs_index = o_i + right.offset();
 
-        equal_fields = left.field(j)->RangeEquals(
-            left_abs_index, left_abs_index + 1, right_abs_index, right.field(j));
-        if (!equal_fields) { return false; }
+        equal_fields = left.field(j)->RangeEquals(left_abs_index, left_abs_index + 1,
+                                                  right_abs_index, right.field(j));
+        if (!equal_fields) {
+          return false;
+        }
       }
     }
     return true;
@@ -144,7 +152,9 @@ class RangeEqualsVisitor {
     const auto& right = static_cast<const UnionArray&>(right_);
 
     const UnionMode union_mode = left.mode();
-    if (union_mode != right.mode()) { return false; }
+    if (union_mode != right.mode()) {
+      return false;
+    }
 
     const auto& left_type = static_cast<const UnionType&>(*left.type());
 
@@ -154,7 +164,9 @@ class RangeEqualsVisitor {
     const std::vector<uint8_t>& type_codes = left_type.type_codes();
     for (size_t i = 0; i < type_codes.size(); ++i) {
       const uint8_t code = type_codes[i];
-      if (code > max_code) { max_code = code; }
+      if (code > max_code) {
+        max_code = code;
+      }
     }
 
     // Store mapping in a vector for constant time lookups
@@ -169,9 +181,13 @@ class RangeEqualsVisitor {
     uint8_t id, child_num;
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
-      if (left.IsNull(i) != right.IsNull(o_i)) { return false; }
+      if (left.IsNull(i) != right.IsNull(o_i)) {
+        return false;
+      }
       if (left.IsNull(i)) continue;
-      if (left_ids[i] != right_ids[o_i]) { return false; }
+      if (left_ids[i] != right_ids[o_i]) {
+        return false;
+      }
 
       id = left_ids[i];
       child_num = type_id_to_child_num[id];
@@ -183,14 +199,15 @@ class RangeEqualsVisitor {
       // rather than looking at one value at a time.
       if (union_mode == UnionMode::SPARSE) {
         if (!left.child(child_num)->RangeEquals(left_abs_index, left_abs_index + 1,
-                right_abs_index, right.child(child_num))) {
+                                                right_abs_index,
+                                                right.child(child_num))) {
           return false;
         }
       } else {
         const int32_t offset = left.raw_value_offsets()[i];
         const int32_t o_offset = right.raw_value_offsets()[o_i];
-        if (!left.child(child_num)->RangeEquals(
-                offset, offset + 1, o_offset, right.child(child_num))) {
+        if (!left.child(child_num)->RangeEquals(offset, offset + 1, o_offset,
+                                                right.child(child_num))) {
           return false;
         }
       }
@@ -211,9 +228,13 @@ class RangeEqualsVisitor {
     const uint8_t* left_data = nullptr;
     const uint8_t* right_data = nullptr;
 
-    if (left.values()) { left_data = left.raw_values() + left.offset() * width; }
+    if (left.values()) {
+      left_data = left.raw_values() + left.offset() * width;
+    }
 
-    if (right.values()) { right_data = right.raw_values() + right.offset() * width; }
+    if (right.values()) {
+      right_data = right.raw_values() + right.offset() * width;
+    }
 
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
@@ -241,9 +262,13 @@ class RangeEqualsVisitor {
     const uint8_t* left_data = nullptr;
     const uint8_t* right_data = nullptr;
 
-    if (left.values()) { left_data = left.raw_values() + left.offset() * width; }
+    if (left.values()) {
+      left_data = left.raw_values() + left.offset() * width;
+    }
 
-    if (right.values()) { right_data = right.raw_values() + right.offset() * width; }
+    if (right.values()) {
+      right_data = right.raw_values() + right.offset() * width;
+    }
 
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
          ++i, ++o_i) {
@@ -301,8 +326,8 @@ class RangeEqualsVisitor {
       result_ = false;
       return Status::OK();
     }
-    result_ = left.indices()->RangeEquals(
-        left_start_idx_, left_end_idx_, right_start_idx_, right.indices());
+    result_ = left.indices()->RangeEquals(left_start_idx_, left_end_idx_,
+                                          right_start_idx_, right.indices());
     return Status::OK();
   }
 
@@ -324,7 +349,9 @@ static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& r
   const uint8_t* left_data = nullptr;
   const uint8_t* right_data = nullptr;
 
-  if (left.values()) { left_data = left.values()->data() + left.offset() * byte_width; }
+  if (left.values()) {
+    left_data = left.values()->data() + left.offset() * byte_width;
+  }
   if (right.values()) {
     right_data = right.values()->data() + right.offset() * byte_width;
   }
@@ -341,13 +368,13 @@ static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& r
     return true;
   } else {
     return memcmp(left_data, right_data,
-               static_cast<size_t>(byte_width * left.length())) == 0;
+                  static_cast<size_t>(byte_width * left.length())) == 0;
   }
 }
 
 template <typename T>
-static inline bool CompareBuiltIn(
-    const Array& left, const Array& right, const T* ldata, const T* rdata) {
+static inline bool CompareBuiltIn(const Array& left, const Array& right, const T* ldata,
+                                  const T* rdata) {
   if (left.null_count() > 0) {
     for (int64_t i = 0; i < left.length(); ++i) {
       if (left.IsNull(i) != right.IsNull(i)) {
@@ -369,17 +396,21 @@ static bool IsEqualDecimal(const DecimalArray& left, const DecimalArray& right)
   const uint8_t* left_data = nullptr;
   const uint8_t* right_data = nullptr;
 
-  if (left.values()) { left_data = left.values()->data(); }
-  if (right.values()) { right_data = right.values()->data(); }
+  if (left.values()) {
+    left_data = left.values()->data();
+  }
+  if (right.values()) {
+    right_data = right.values()->data();
+  }
 
   const int32_t byte_width = left.byte_width();
   if (byte_width == 4) {
-    return CompareBuiltIn<int32_t>(left, right,
-        reinterpret_cast<const int32_t*>(left_data) + loffset,
+    return CompareBuiltIn<int32_t>(
+        left, right, reinterpret_cast<const int32_t*>(left_data) + loffset,
         reinterpret_cast<const int32_t*>(right_data) + roffset);
   } else if (byte_width == 8) {
-    return CompareBuiltIn<int64_t>(left, right,
-        reinterpret_cast<const int64_t*>(left_data) + loffset,
+    return CompareBuiltIn<int64_t>(
+        left, right, reinterpret_cast<const int64_t*>(left_data) + loffset,
         reinterpret_cast<const int64_t*>(right_data) + roffset);
   } else {
     // 128-bit
@@ -387,8 +418,12 @@ static bool IsEqualDecimal(const DecimalArray& left, const DecimalArray& right)
     // Must also compare sign bitmap
     const uint8_t* left_sign = nullptr;
     const uint8_t* right_sign = nullptr;
-    if (left.sign_bitmap()) { left_sign = left.sign_bitmap()->data(); }
-    if (right.sign_bitmap()) { right_sign = right.sign_bitmap()->data(); }
+    if (left.sign_bitmap()) {
+      left_sign = left.sign_bitmap()->data();
+    }
+    if (right.sign_bitmap()) {
+      right_sign = right.sign_bitmap()->data();
+    }
 
     for (int64_t i = 0; i < left.length(); ++i) {
       bool left_null = left.IsNull(i);
@@ -434,7 +469,7 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       result_ = true;
     } else {
       result_ = BitmapEquals(left.values()->data(), left.offset(), right.values()->data(),
-          right.offset(), left.length());
+                             right.offset(), left.length());
     }
     return Status::OK();
   }
@@ -442,7 +477,7 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
   template <typename T>
   typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value &&
                               !std::is_base_of<BooleanArray, T>::value,
-      Status>::type
+                          Status>::type
   Visit(const T& left) {
     result_ = IsEqualPrimitive(left, static_cast<const PrimitiveArray&>(right_));
     return Status::OK();
@@ -458,8 +493,8 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     const auto& right = static_cast<const ArrayType&>(right_);
 
     if (left.offset() == 0 && right.offset() == 0) {
-      return left.value_offsets()->Equals(
-          *right.value_offsets(), (left.length() + 1) * sizeof(int32_t));
+      return left.value_offsets()->Equals(*right.value_offsets(),
+                                          (left.length() + 1) * sizeof(int32_t));
     } else {
       // One of the arrays is sliced; logic is more complicated because the
       // value offsets are not both 0-based
@@ -482,10 +517,16 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     const auto& right = static_cast<const BinaryArray&>(right_);
 
     bool equal_offsets = ValueOffsetsEqual<BinaryArray>(left);
-    if (!equal_offsets) { return false; }
+    if (!equal_offsets) {
+      return false;
+    }
 
-    if (!left.value_data() && !(right.value_data())) { return true; }
-    if (left.value_offset(left.length()) == 0) { return true; }
+    if (!left.value_data() && !(right.value_data())) {
+      return true;
+    }
+    if (left.value_offset(left.length()) == 0) {
+      return true;
+    }
 
     const uint8_t* left_data = left.value_data()->data();
     const uint8_t* right_data = right.value_data()->data();
@@ -493,23 +534,25 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     if (left.null_count() == 0) {
       // Fast path for null count 0, single memcmp
       if (left.offset() == 0 && right.offset() == 0) {
-        return std::memcmp(
-                   left_data, right_data, left.raw_value_offsets()[left.length()]) == 0;
+        return std::memcmp(left_data, right_data,
+                           left.raw_value_offsets()[left.length()]) == 0;
       } else {
         const int64_t total_bytes =
             left.value_offset(left.length()) - left.value_offset(0);
         return std::memcmp(left_data + left.value_offset(0),
-                   right_data + right.value_offset(0),
-                   static_cast<size_t>(total_bytes)) == 0;
+                           right_data + right.value_offset(0),
+                           static_cast<size_t>(total_bytes)) == 0;
       }
     } else {
       // ARROW-537: Only compare data in non-null slots
       const int32_t* left_offsets = left.raw_value_offsets();
       const int32_t* right_offsets = right.raw_value_offsets();
       for (int64_t i = 0; i < left.length(); ++i) {
-        if (left.IsNull(i)) { continue; }
+        if (left.IsNull(i)) {
+          continue;
+        }
         if (std::memcmp(left_data + left_offsets[i], right_data + right_offsets[i],
-                left.value_length(i))) {
+                        left.value_length(i))) {
           return false;
         }
       }
@@ -530,8 +573,9 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       return Status::OK();
     }
 
-    result_ = left.values()->RangeEquals(left.value_offset(0),
-        left.value_offset(left.length()), right.value_offset(0), right.values());
+    result_ =
+        left.values()->RangeEquals(left.value_offset(0), left.value_offset(left.length()),
+                                   right.value_offset(0), right.values());
     return Status::OK();
   }
 
@@ -547,15 +591,15 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
 
   template <typename T>
   typename std::enable_if<std::is_base_of<NestedType, typename T::TypeClass>::value,
-      Status>::type
+                          Status>::type
   Visit(const T& left) {
     return RangeEqualsVisitor::Visit(left);
   }
 };
 
 template <typename TYPE>
-inline bool FloatingApproxEquals(
-    const NumericArray<TYPE>& left, const NumericArray<TYPE>& right) {
+inline bool FloatingApproxEquals(const NumericArray<TYPE>& left,
+                                 const NumericArray<TYPE>& right) {
   using T = typename TYPE::c_type;
 
   const T* left_data = left.raw_values();
@@ -566,11 +610,15 @@ inline bool FloatingApproxEquals(
   if (left.null_count() > 0) {
     for (int64_t i = 0; i < left.length(); ++i) {
       if (left.IsNull(i)) continue;
-      if (fabs(left_data[i] - right_data[i]) > EPSILON) { return false; }
+      if (fabs(left_data[i] - right_data[i]) > EPSILON) {
+        return false;
+      }
     }
   } else {
     for (int64_t i = 0; i < left.length(); ++i) {
-      if (fabs(left_data[i] - right_data[i]) > EPSILON) { return false; }
+      if (fabs(left_data[i] - right_data[i]) > EPSILON) {
+        return false;
+      }
     }
   }
   return true;
@@ -601,7 +649,7 @@ static bool BaseDataEquals(const Array& left, const Array& right) {
   }
   if (left.null_count() > 0 && left.null_count() < left.length()) {
     return BitmapEquals(left.null_bitmap()->data(), left.offset(),
-        right.null_bitmap()->data(), right.offset(), left.length());
+                        right.null_bitmap()->data(), right.offset(), left.length());
   }
   return true;
 }
@@ -634,7 +682,7 @@ Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal)
 }
 
 Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_start_idx,
-    int64_t left_end_idx, int64_t right_start_idx, bool* are_equal) {
+                        int64_t left_end_idx, int64_t right_start_idx, bool* are_equal) {
   if (&left == &right) {
     *are_equal = true;
   } else if (left.type_id() != right.type_id()) {
@@ -705,7 +753,7 @@ class TypeEqualsVisitor {
   template <typename T>
   typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
                               std::is_base_of<PrimitiveCType, T>::value,
-      Status>::type
+                          Status>::type
   Visit(const T& type) {
     result_ = true;
     return Status::OK();
@@ -714,7 +762,7 @@ class TypeEqualsVisitor {
   template <typename T>
   typename std::enable_if<std::is_base_of<TimeType, T>::value ||
                               std::is_base_of<DateType, T>::value,
-      Status>::type
+                          Status>::type
   Visit(const T& left) {
     const auto& right = static_cast<const T&>(right_);
     result_ = left.unit() == right.unit();
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index 96a6435c5df33..a36b55320b5a2 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -34,21 +34,22 @@ class Tensor;
 /// Returns true if the arrays are exactly equal
 Status ARROW_EXPORT ArrayEquals(const Array& left, const Array& right, bool* are_equal);
 
-Status ARROW_EXPORT TensorEquals(
-    const Tensor& left, const Tensor& right, bool* are_equal);
+Status ARROW_EXPORT TensorEquals(const Tensor& left, const Tensor& right,
+                                 bool* are_equal);
 
 /// Returns true if the arrays are approximately equal. For non-floating point
 /// types, this is equivalent to ArrayEquals(left, right)
-Status ARROW_EXPORT ArrayApproxEquals(
-    const Array& left, const Array& right, bool* are_equal);
+Status ARROW_EXPORT ArrayApproxEquals(const Array& left, const Array& right,
+                                      bool* are_equal);
 
 /// Returns true if indicated equal-length segment of arrays is exactly equal
 Status ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right,
-    int64_t start_idx, int64_t end_idx, int64_t other_start_idx, bool* are_equal);
+                                     int64_t start_idx, int64_t end_idx,
+                                     int64_t other_start_idx, bool* are_equal);
 
 /// Returns true if the type metadata are exactly equal
-Status ARROW_EXPORT TypeEquals(
-    const DataType& left, const DataType& right, bool* are_equal);
+Status ARROW_EXPORT TypeEquals(const DataType& left, const DataType& right,
+                               bool* are_equal);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 936655f26dbda..82e3ba8109c23 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -123,8 +123,8 @@ constexpr const char* kRangeExceptionError =
     "Range exception during wide-char string conversion";
 #endif
 
-static inline Status CheckOpenResult(
-    int ret, int errno_actual, const char* filename, size_t filename_length) {
+static inline Status CheckOpenResult(int ret, int errno_actual, const char* filename,
+                                     size_t filename_length) {
   if (ret == -1) {
     // TODO: errno codes to strings
     std::stringstream ss;
@@ -134,12 +134,14 @@ static inline Status CheckOpenResult(
 
     // this requires c++11
     std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
-    std::wstring wide_string(
-        reinterpret_cast<const wchar_t*>(filename), filename_length / sizeof(wchar_t));
+    std::wstring wide_string(reinterpret_cast<const wchar_t*>(filename),
+                             filename_length / sizeof(wchar_t));
     try {
       std::string byte_string = converter.to_bytes(wide_string);
       ss << byte_string;
-    } catch (const std::range_error&) { ss << kRangeExceptionError; }
+    } catch (const std::range_error&) {
+      ss << kRangeExceptionError;
+    }
 #else
     ss << filename;
 #endif
@@ -161,7 +163,9 @@ static inline int64_t lseek64_compat(int fd, int64_t pos, int whence) {
 
 #if defined(_MSC_VER)
 static inline Status ConvertToUtf16(const std::string& input, std::wstring* result) {
-  if (result == nullptr) { return Status::Invalid("Pointer to result is not valid"); }
+  if (result == nullptr) {
+    return Status::Invalid("Pointer to result is not valid");
+  }
 
   if (input.empty()) {
     *result = std::wstring();
@@ -171,7 +175,9 @@ static inline Status ConvertToUtf16(const std::string& input, std::wstring* resu
   std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> utf16_converter;
   try {
     *result = utf16_converter.from_bytes(input);
-  } catch (const std::range_error&) { return Status::Invalid(kRangeExceptionError); }
+  } catch (const std::range_error&) {
+    return Status::Invalid(kRangeExceptionError);
+  }
   return Status::OK();
 }
 #endif
@@ -194,8 +200,8 @@ static inline Status FileOpenReadable(const std::string& filename, int* fd) {
   return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
 }
 
-static inline Status FileOpenWriteable(
-    const std::string& filename, bool write_only, bool truncate, int* fd) {
+static inline Status FileOpenWriteable(const std::string& filename, bool write_only,
+                                       bool truncate, int* fd) {
   int ret;
   errno_t errno_actual = 0;
 
@@ -205,9 +211,13 @@ static inline Status FileOpenWriteable(
 
   int oflag = _O_CREAT | _O_BINARY;
   int pmode = _S_IWRITE;
-  if (!write_only) { pmode |= _S_IREAD; }
+  if (!write_only) {
+    pmode |= _S_IREAD;
+  }
 
-  if (truncate) { oflag |= _O_TRUNC; }
+  if (truncate) {
+    oflag |= _O_TRUNC;
+  }
 
   if (write_only) {
     oflag |= _O_WRONLY;
@@ -221,7 +231,9 @@ static inline Status FileOpenWriteable(
 #else
   int oflag = O_CREAT | O_BINARY;
 
-  if (truncate) { oflag |= O_TRUNC; }
+  if (truncate) {
+    oflag |= O_TRUNC;
+  }
 
   if (write_only) {
     oflag |= O_WRONLY;
@@ -239,7 +251,9 @@ static inline Status FileTell(int fd, int64_t* pos) {
 
 #if defined(_MSC_VER)
   current_pos = _telli64(fd);
-  if (current_pos == -1) { return Status::IOError("_telli64 failed"); }
+  if (current_pos == -1) {
+    return Status::IOError("_telli64 failed");
+  }
 #else
   current_pos = lseek64_compat(fd, 0, SEEK_CUR);
   CHECK_LSEEK(current_pos);
@@ -255,10 +269,12 @@ static inline Status FileSeek(int fd, int64_t pos) {
   return Status::OK();
 }
 
-static inline Status FileRead(
-    int fd, uint8_t* buffer, int64_t nbytes, int64_t* bytes_read) {
+static inline Status FileRead(int fd, uint8_t* buffer, int64_t nbytes,
+                              int64_t* bytes_read) {
 #if defined(_MSC_VER)
-  if (nbytes > INT32_MAX) { return Status::IOError("Unable to read > 2GB blocks yet"); }
+  if (nbytes > INT32_MAX) {
+    return Status::IOError("Unable to read > 2GB blocks yet");
+  }
   *bytes_read = static_cast<int64_t>(_read(fd, buffer, static_cast<uint32_t>(nbytes)));
 #else
   *bytes_read = static_cast<int64_t>(read(fd, buffer, static_cast<size_t>(nbytes)));
@@ -323,7 +339,9 @@ static inline Status FileClose(int fd) {
   ret = static_cast<int>(close(fd));
 #endif
 
-  if (ret == -1) { return Status::IOError("error closing file"); }
+  if (ret == -1) {
+    return Status::IOError("error closing file");
+  }
   return Status::OK();
 }
 
@@ -371,7 +389,9 @@ class OSFile {
   }
 
   Status Seek(int64_t pos) {
-    if (pos < 0) { return Status::Invalid("Invalid position"); }
+    if (pos < 0) {
+      return Status::Invalid("Invalid position");
+    }
     return FileSeek(fd_, pos);
   }
 
@@ -379,7 +399,9 @@ class OSFile {
 
   Status Write(const uint8_t* data, int64_t length) {
     std::lock_guard<std::mutex> guard(lock_);
-    if (length < 0) { return Status::IOError("Length must be non-negative"); }
+    if (length < 0) {
+      return Status::IOError("Length must be non-negative");
+    }
     return FileWrite(fd_, data, length);
   }
 
@@ -421,7 +443,9 @@ class ReadableFile::ReadableFileImpl : public OSFile {
 
     int64_t bytes_read = 0;
     RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
-    if (bytes_read < nbytes) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
+    if (bytes_read < nbytes) {
+      RETURN_NOT_OK(buffer->Resize(bytes_read));
+    }
     *out = buffer;
     return Status::OK();
   }
@@ -430,13 +454,9 @@ class ReadableFile::ReadableFileImpl : public OSFile {
   MemoryPool* pool_;
 };
 
-ReadableFile::ReadableFile(MemoryPool* pool) {
-  impl_.reset(new ReadableFileImpl(pool));
-}
+ReadableFile::ReadableFile(MemoryPool* pool) { impl_.reset(new ReadableFileImpl(pool)); }
 
-ReadableFile::~ReadableFile() {
-  DCHECK(impl_->Close().ok());
-}
+ReadableFile::~ReadableFile() { DCHECK(impl_->Close().ok()); }
 
 Status ReadableFile::Open(const std::string& path, std::shared_ptr<ReadableFile>* file) {
   *file = std::shared_ptr<ReadableFile>(new ReadableFile(default_memory_pool()));
@@ -444,18 +464,14 @@ Status ReadableFile::Open(const std::string& path, std::shared_ptr<ReadableFile>
 }
 
 Status ReadableFile::Open(const std::string& path, MemoryPool* memory_pool,
-    std::shared_ptr<ReadableFile>* file) {
+                          std::shared_ptr<ReadableFile>* file) {
   *file = std::shared_ptr<ReadableFile>(new ReadableFile(memory_pool));
   return (*file)->impl_->Open(path);
 }
 
-Status ReadableFile::Close() {
-  return impl_->Close();
-}
+Status ReadableFile::Close() { return impl_->Close(); }
 
-Status ReadableFile::Tell(int64_t* pos) {
-  return impl_->Tell(pos);
-}
+Status ReadableFile::Tell(int64_t* pos) { return impl_->Tell(pos); }
 
 Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
   return impl_->Read(nbytes, bytes_read, out);
@@ -470,17 +486,11 @@ Status ReadableFile::GetSize(int64_t* size) {
   return Status::OK();
 }
 
-Status ReadableFile::Seek(int64_t pos) {
-  return impl_->Seek(pos);
-}
+Status ReadableFile::Seek(int64_t pos) { return impl_->Seek(pos); }
 
-bool ReadableFile::supports_zero_copy() const {
-  return false;
-}
+bool ReadableFile::supports_zero_copy() const { return false; }
 
-int ReadableFile::file_descriptor() const {
-  return impl_->fd();
-}
+int ReadableFile::file_descriptor() const { return impl_->fd(); }
 
 // ----------------------------------------------------------------------
 // FileOutputStream
@@ -492,42 +502,34 @@ class FileOutputStream::FileOutputStreamImpl : public OSFile {
   }
 };
 
-FileOutputStream::FileOutputStream() {
-  impl_.reset(new FileOutputStreamImpl());
-}
+FileOutputStream::FileOutputStream() { impl_.reset(new FileOutputStreamImpl()); }
 
 FileOutputStream::~FileOutputStream() {
   // This can fail; better to explicitly call close
   DCHECK(impl_->Close().ok());
 }
 
-Status FileOutputStream::Open(
-    const std::string& path, std::shared_ptr<FileOutputStream>* file) {
+Status FileOutputStream::Open(const std::string& path,
+                              std::shared_ptr<FileOutputStream>* file) {
   return Open(path, false, file);
 }
 
-Status FileOutputStream::Open(
-    const std::string& path, bool append, std::shared_ptr<FileOutputStream>* file) {
+Status FileOutputStream::Open(const std::string& path, bool append,
+                              std::shared_ptr<FileOutputStream>* file) {
   // private ctor
   *file = std::shared_ptr<FileOutputStream>(new FileOutputStream());
   return (*file)->impl_->Open(path, append);
 }
 
-Status FileOutputStream::Close() {
-  return impl_->Close();
-}
+Status FileOutputStream::Close() { return impl_->Close(); }
 
-Status FileOutputStream::Tell(int64_t* pos) {
-  return impl_->Tell(pos);
-}
+Status FileOutputStream::Tell(int64_t* pos) { return impl_->Tell(pos); }
 
 Status FileOutputStream::Write(const uint8_t* data, int64_t length) {
   return impl_->Write(data, length);
 }
 
-int FileOutputStream::file_descriptor() const {
-  return impl_->fd();
-}
+int FileOutputStream::file_descriptor() const { return impl_->fd(); }
 
 // ----------------------------------------------------------------------
 // Implement MemoryMappedFile
@@ -567,7 +569,7 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
     }
 
     void* result = mmap(nullptr, static_cast<size_t>(file_->size()), prot_flags, map_mode,
-        file_->fd(), 0);
+                        file_->fd(), 0);
     if (result == MAP_FAILED) {
       std::stringstream ss;
       ss << "Memory mapping file failed, errno: " << errno;
@@ -585,7 +587,9 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
   int64_t size() const { return size_; }
 
   Status Seek(int64_t position) {
-    if (position < 0) { return Status::Invalid("position is out of bounds"); }
+    if (position < 0) {
+      return Status::Invalid("position is out of bounds");
+    }
     position_ = position;
     return Status::OK();
   }
@@ -610,8 +614,8 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
 MemoryMappedFile::MemoryMappedFile() {}
 MemoryMappedFile::~MemoryMappedFile() {}
 
-Status MemoryMappedFile::Create(
-    const std::string& path, int64_t size, std::shared_ptr<MemoryMappedFile>* out) {
+Status MemoryMappedFile::Create(const std::string& path, int64_t size,
+                                std::shared_ptr<MemoryMappedFile>* out) {
   std::shared_ptr<FileOutputStream> file;
   RETURN_NOT_OK(FileOutputStream::Open(path, &file));
 #ifdef _MSC_VER
@@ -624,7 +628,7 @@ Status MemoryMappedFile::Create(
 }
 
 Status MemoryMappedFile::Open(const std::string& path, FileMode::type mode,
-    std::shared_ptr<MemoryMappedFile>* out) {
+                              std::shared_ptr<MemoryMappedFile>* out) {
   std::shared_ptr<MemoryMappedFile> result(new MemoryMappedFile());
 
   result->memory_map_.reset(new MemoryMap());
@@ -644,9 +648,7 @@ Status MemoryMappedFile::Tell(int64_t* position) {
   return Status::OK();
 }
 
-Status MemoryMappedFile::Seek(int64_t position) {
-  return memory_map_->Seek(position);
-}
+Status MemoryMappedFile::Seek(int64_t position) { return memory_map_->Seek(position); }
 
 Status MemoryMappedFile::Close() {
   // munmap handled in pimpl dtor
@@ -656,7 +658,9 @@ Status MemoryMappedFile::Close() {
 Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
   nbytes = std::max<int64_t>(
       0, std::min(nbytes, memory_map_->size() - memory_map_->position()));
-  if (nbytes > 0) { std::memcpy(out, memory_map_->head(), static_cast<size_t>(nbytes)); }
+  if (nbytes > 0) {
+    std::memcpy(out, memory_map_->head(), static_cast<size_t>(nbytes));
+  }
   *bytes_read = nbytes;
   memory_map_->advance(nbytes);
   return Status::OK();
@@ -675,9 +679,7 @@ Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   return Status::OK();
 }
 
-bool MemoryMappedFile::supports_zero_copy() const {
-  return true;
-}
+bool MemoryMappedFile::supports_zero_copy() const { return true; }
 
 Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
   std::lock_guard<std::mutex> guard(lock_);
@@ -708,9 +710,7 @@ Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
   return Status::OK();
 }
 
-int MemoryMappedFile::file_descriptor() const {
-  return memory_map_->fd();
-}
+int MemoryMappedFile::file_descriptor() const { return memory_map_->fd(); }
 
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index f0be3cf980162..ba740f1e8f4a9 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -44,8 +44,8 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
   // truncated to 0 bytes, deleting any existing memory
   static Status Open(const std::string& path, std::shared_ptr<FileOutputStream>* file);
 
-  static Status Open(
-      const std::string& path, bool append, std::shared_ptr<FileOutputStream>* file);
+  static Status Open(const std::string& path, bool append,
+                     std::shared_ptr<FileOutputStream>* file);
 
   // OutputStream interface
   Status Close() override;
@@ -73,7 +73,7 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile {
 
   // Open file with one's own memory pool for memory allocations
   static Status Open(const std::string& path, MemoryPool* memory_pool,
-      std::shared_ptr<ReadableFile>* file);
+                     std::shared_ptr<ReadableFile>* file);
 
   Status Close() override;
   Status Tell(int64_t* position) override;
@@ -107,11 +107,11 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   ~MemoryMappedFile();
 
   /// Create new file with indicated size, return in read/write mode
-  static Status Create(
-      const std::string& path, int64_t size, std::shared_ptr<MemoryMappedFile>* out);
+  static Status Create(const std::string& path, int64_t size,
+                       std::shared_ptr<MemoryMappedFile>* out);
 
   static Status Open(const std::string& path, FileMode::type mode,
-      std::shared_ptr<MemoryMappedFile>* out);
+                     std::shared_ptr<MemoryMappedFile>* out);
 
   Status Close() override;
 
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index 8b4a92b396789..8f42b1c817fe4 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -59,9 +59,9 @@ static std::vector<fs::path> get_potential_libhdfs_paths();
 static std::vector<fs::path> get_potential_libhdfs3_paths();
 static arrow::Status try_dlopen(std::vector<fs::path> potential_paths, const char* name,
 #ifndef _WIN32
-    void*& out_handle);
+                                void*& out_handle);
 #else
-    HINSTANCE& out_handle);
+                                HINSTANCE& out_handle);
 #endif
 
 static std::vector<fs::path> get_potential_libhdfs_paths() {
@@ -88,7 +88,9 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
   }
 
   const char* libhdfs_dir = std::getenv("ARROW_LIBHDFS_DIR");
-  if (libhdfs_dir != nullptr) { search_paths.push_back(fs::path(libhdfs_dir)); }
+  if (libhdfs_dir != nullptr) {
+    search_paths.push_back(fs::path(libhdfs_dir));
+  }
 
   // All paths with file name
   for (auto& path : search_paths) {
@@ -115,7 +117,9 @@ static std::vector<fs::path> get_potential_libhdfs3_paths() {
   std::vector<fs::path> search_paths = {fs::path(""), fs::path(".")};
 
   const char* libhdfs3_dir = std::getenv("ARROW_LIBHDFS3_DIR");
-  if (libhdfs3_dir != nullptr) { search_paths.push_back(fs::path(libhdfs3_dir)); }
+  if (libhdfs3_dir != nullptr) {
+    search_paths.push_back(fs::path(libhdfs3_dir));
+  }
 
   // All paths with file name
   for (auto& path : search_paths) {
@@ -188,8 +192,8 @@ static std::vector<fs::path> get_potential_libjvm_paths() {
 }
 
 #ifndef _WIN32
-static arrow::Status try_dlopen(
-    std::vector<fs::path> potential_paths, const char* name, void*& out_handle) {
+static arrow::Status try_dlopen(std::vector<fs::path> potential_paths, const char* name,
+                                void*& out_handle) {
   std::vector<std::string> error_messages;
 
   for (auto& i : potential_paths) {
@@ -219,8 +223,8 @@ static arrow::Status try_dlopen(
 }
 
 #else
-static arrow::Status try_dlopen(
-    std::vector<fs::path> potential_paths, const char* name, HINSTANCE& out_handle) {
+static arrow::Status try_dlopen(std::vector<fs::path> potential_paths, const char* name,
+                                HINSTANCE& out_handle) {
   std::vector<std::string> error_messages;
 
   for (auto& i : potential_paths) {
@@ -282,9 +286,7 @@ namespace io {
 static LibHdfsShim libhdfs_shim;
 static LibHdfsShim libhdfs3_shim;
 
-hdfsBuilder* LibHdfsShim::NewBuilder(void) {
-  return this->hdfsNewBuilder();
-}
+hdfsBuilder* LibHdfsShim::NewBuilder(void) { return this->hdfsNewBuilder(); }
 
 void LibHdfsShim::BuilderSetNameNode(hdfsBuilder* bld, const char* nn) {
   this->hdfsBuilderSetNameNode(bld, nn);
@@ -298,8 +300,8 @@ void LibHdfsShim::BuilderSetUserName(hdfsBuilder* bld, const char* userName) {
   this->hdfsBuilderSetUserName(bld, userName);
 }
 
-void LibHdfsShim::BuilderSetKerbTicketCachePath(
-    hdfsBuilder* bld, const char* kerbTicketCachePath) {
+void LibHdfsShim::BuilderSetKerbTicketCachePath(hdfsBuilder* bld,
+                                                const char* kerbTicketCachePath) {
   this->hdfsBuilderSetKerbTicketCachePath(bld, kerbTicketCachePath);
 }
 
@@ -307,12 +309,10 @@ hdfsFS LibHdfsShim::BuilderConnect(hdfsBuilder* bld) {
   return this->hdfsBuilderConnect(bld);
 }
 
-int LibHdfsShim::Disconnect(hdfsFS fs) {
-  return this->hdfsDisconnect(fs);
-}
+int LibHdfsShim::Disconnect(hdfsFS fs) { return this->hdfsDisconnect(fs); }
 
 hdfsFile LibHdfsShim::OpenFile(hdfsFS fs, const char* path, int flags, int bufferSize,
-    short replication, tSize blocksize) {  // NOLINT
+                               short replication, tSize blocksize) {  // NOLINT
   return this->hdfsOpenFile(fs, path, flags, bufferSize, replication, blocksize);
 }
 
@@ -328,9 +328,7 @@ int LibHdfsShim::Seek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
   return this->hdfsSeek(fs, file, desiredPos);
 }
 
-tOffset LibHdfsShim::Tell(hdfsFS fs, hdfsFile file) {
-  return this->hdfsTell(fs, file);
-}
+tOffset LibHdfsShim::Tell(hdfsFS fs, hdfsFile file) { return this->hdfsTell(fs, file); }
 
 tSize LibHdfsShim::Read(hdfsFS fs, hdfsFile file, void* buffer, tSize length) {
   return this->hdfsRead(fs, file, buffer, length);
@@ -341,8 +339,8 @@ bool LibHdfsShim::HasPread() {
   return this->hdfsPread != nullptr;
 }
 
-tSize LibHdfsShim::Pread(
-    hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length) {
+tSize LibHdfsShim::Pread(hdfsFS fs, hdfsFile file, tOffset position, void* buffer,
+                         tSize length) {
   GET_SYMBOL(this, hdfsPread);
   return this->hdfsPread(fs, file, position, buffer, length);
 }
@@ -351,9 +349,7 @@ tSize LibHdfsShim::Write(hdfsFS fs, hdfsFile file, const void* buffer, tSize len
   return this->hdfsWrite(fs, file, buffer, length);
 }
 
-int LibHdfsShim::Flush(hdfsFS fs, hdfsFile file) {
-  return this->hdfsFlush(fs, file);
-}
+int LibHdfsShim::Flush(hdfsFS fs, hdfsFile file) { return this->hdfsFlush(fs, file); }
 
 int LibHdfsShim::Available(hdfsFS fs, hdfsFile file) {
   GET_SYMBOL(this, hdfsAvailable);
@@ -434,8 +430,8 @@ void LibHdfsShim::FreeFileInfo(hdfsFileInfo* hdfsFileInfo, int numEntries) {
   this->hdfsFreeFileInfo(hdfsFileInfo, numEntries);
 }
 
-char*** LibHdfsShim::GetHosts(
-    hdfsFS fs, const char* path, tOffset start, tOffset length) {
+char*** LibHdfsShim::GetHosts(hdfsFS fs, const char* path, tOffset start,
+                              tOffset length) {
   GET_SYMBOL(this, hdfsGetHosts);
   if (this->hdfsGetHosts) {
     return this->hdfsGetHosts(fs, path, start, length);
@@ -446,7 +442,9 @@ char*** LibHdfsShim::GetHosts(
 
 void LibHdfsShim::FreeHosts(char*** blockHosts) {
   GET_SYMBOL(this, hdfsFreeHosts);
-  if (this->hdfsFreeHosts) { this->hdfsFreeHosts(blockHosts); }
+  if (this->hdfsFreeHosts) {
+    this->hdfsFreeHosts(blockHosts);
+  }
 }
 
 tOffset LibHdfsShim::GetDefaultBlockSize(hdfsFS fs) {
@@ -458,16 +456,12 @@ tOffset LibHdfsShim::GetDefaultBlockSize(hdfsFS fs) {
   }
 }
 
-tOffset LibHdfsShim::GetCapacity(hdfsFS fs) {
-  return this->hdfsGetCapacity(fs);
-}
+tOffset LibHdfsShim::GetCapacity(hdfsFS fs) { return this->hdfsGetCapacity(fs); }
 
-tOffset LibHdfsShim::GetUsed(hdfsFS fs) {
-  return this->hdfsGetUsed(fs);
-}
+tOffset LibHdfsShim::GetUsed(hdfsFS fs) { return this->hdfsGetUsed(fs); }
 
-int LibHdfsShim::Chown(
-    hdfsFS fs, const char* path, const char* owner, const char* group) {
+int LibHdfsShim::Chown(hdfsFS fs, const char* path, const char* owner,
+                       const char* group) {
   GET_SYMBOL(this, hdfsChown);
   if (this->hdfsChown) {
     return this->hdfsChown(fs, path, owner, group);
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
index c5ea397af0bd5..db6a21c2b36ac 100644
--- a/cpp/src/arrow/io/hdfs-internal.h
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -45,22 +45,22 @@ struct LibHdfsShim {
   void (*hdfsBuilderSetNameNode)(hdfsBuilder* bld, const char* nn);
   void (*hdfsBuilderSetNameNodePort)(hdfsBuilder* bld, tPort port);
   void (*hdfsBuilderSetUserName)(hdfsBuilder* bld, const char* userName);
-  void (*hdfsBuilderSetKerbTicketCachePath)(
-      hdfsBuilder* bld, const char* kerbTicketCachePath);
+  void (*hdfsBuilderSetKerbTicketCachePath)(hdfsBuilder* bld,
+                                            const char* kerbTicketCachePath);
   hdfsFS (*hdfsBuilderConnect)(hdfsBuilder* bld);
 
   int (*hdfsDisconnect)(hdfsFS fs);
 
   hdfsFile (*hdfsOpenFile)(hdfsFS fs, const char* path, int flags, int bufferSize,
-      short replication, tSize blocksize);  // NOLINT
+                           short replication, tSize blocksize);  // NOLINT
 
   int (*hdfsCloseFile)(hdfsFS fs, hdfsFile file);
   int (*hdfsExists)(hdfsFS fs, const char* path);
   int (*hdfsSeek)(hdfsFS fs, hdfsFile file, tOffset desiredPos);
   tOffset (*hdfsTell)(hdfsFS fs, hdfsFile file);
   tSize (*hdfsRead)(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
-  tSize (*hdfsPread)(
-      hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length);
+  tSize (*hdfsPread)(hdfsFS fs, hdfsFile file, tOffset position, void* buffer,
+                     tSize length);
   tSize (*hdfsWrite)(hdfsFS fs, hdfsFile file, const void* buffer, tSize length);
   int (*hdfsFlush)(hdfsFS fs, hdfsFile file);
   int (*hdfsAvailable)(hdfsFS fs, hdfsFile file);
@@ -139,7 +139,7 @@ struct LibHdfsShim {
   int Disconnect(hdfsFS fs);
 
   hdfsFile OpenFile(hdfsFS fs, const char* path, int flags, int bufferSize,
-      short replication, tSize blocksize);  // NOLINT
+                    short replication, tSize blocksize);  // NOLINT
 
   int CloseFile(hdfsFS fs, hdfsFile file);
 
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 9ded9bc3f9902..500f42caf5277 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -61,8 +61,8 @@ static constexpr int kDefaultHdfsBufferSize = 1 << 16;
 
 class HdfsAnyFileImpl {
  public:
-  void set_members(
-      const std::string& path, LibHdfsShim* driver, hdfsFS fs, hdfsFile handle) {
+  void set_members(const std::string& path, LibHdfsShim* driver, hdfsFS fs,
+                   hdfsFile handle) {
     path_ = path;
     driver_ = driver;
     fs_ = fs;
@@ -118,7 +118,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     tSize ret;
     if (driver_->HasPread()) {
       ret = driver_->Pread(fs_, file_, static_cast<tOffset>(position),
-          reinterpret_cast<void*>(buffer), static_cast<tSize>(nbytes));
+                           reinterpret_cast<void*>(buffer), static_cast<tSize>(nbytes));
     } else {
       std::lock_guard<std::mutex> guard(lock_);
       RETURN_NOT_OK(Seek(position));
@@ -136,7 +136,9 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     int64_t bytes_read = 0;
     RETURN_NOT_OK(ReadAt(position, nbytes, &bytes_read, buffer->mutable_data()));
 
-    if (bytes_read < nbytes) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
+    if (bytes_read < nbytes) {
+      RETURN_NOT_OK(buffer->Resize(bytes_read));
+    }
 
     *out = buffer;
     return Status::OK();
@@ -145,11 +147,14 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
     int64_t total_bytes = 0;
     while (total_bytes < nbytes) {
-      tSize ret = driver_->Read(fs_, file_, reinterpret_cast<void*>(buffer + total_bytes),
+      tSize ret = driver_->Read(
+          fs_, file_, reinterpret_cast<void*>(buffer + total_bytes),
           static_cast<tSize>(std::min<int64_t>(buffer_size_, nbytes - total_bytes)));
       RETURN_NOT_OK(CheckReadResult(ret));
       total_bytes += ret;
-      if (ret == 0) { break; }
+      if (ret == 0) {
+        break;
+      }
     }
 
     *bytes_read = total_bytes;
@@ -162,7 +167,9 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
 
     int64_t bytes_read = 0;
     RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
-    if (bytes_read < nbytes) { RETURN_NOT_OK(buffer->Resize(bytes_read)); }
+    if (bytes_read < nbytes) {
+      RETURN_NOT_OK(buffer->Resize(bytes_read));
+    }
 
     *out = buffer;
     return Status::OK();
@@ -170,7 +177,9 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
 
   Status GetSize(int64_t* size) {
     hdfsFileInfo* entry = driver_->GetPathInfo(fs_, path_.c_str());
-    if (entry == nullptr) { return Status::IOError("HDFS: GetPathInfo failed"); }
+    if (entry == nullptr) {
+      return Status::IOError("HDFS: GetPathInfo failed");
+    }
 
     *size = entry->mSize;
     driver_->FreeFileInfo(entry, 1);
@@ -187,31 +196,27 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
 };
 
 HdfsReadableFile::HdfsReadableFile(MemoryPool* pool) {
-  if (pool == nullptr) { pool = default_memory_pool(); }
+  if (pool == nullptr) {
+    pool = default_memory_pool();
+  }
   impl_.reset(new HdfsReadableFileImpl(pool));
 }
 
-HdfsReadableFile::~HdfsReadableFile() {
-  DCHECK(impl_->Close().ok());
-}
+HdfsReadableFile::~HdfsReadableFile() { DCHECK(impl_->Close().ok()); }
 
-Status HdfsReadableFile::Close() {
-  return impl_->Close();
-}
+Status HdfsReadableFile::Close() { return impl_->Close(); }
 
-Status HdfsReadableFile::ReadAt(
-    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+Status HdfsReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                                uint8_t* buffer) {
   return impl_->ReadAt(position, nbytes, bytes_read, buffer);
 }
 
-Status HdfsReadableFile::ReadAt(
-    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+Status HdfsReadableFile::ReadAt(int64_t position, int64_t nbytes,
+                                std::shared_ptr<Buffer>* out) {
   return impl_->ReadAt(position, nbytes, out);
 }
 
-bool HdfsReadableFile::supports_zero_copy() const {
-  return false;
-}
+bool HdfsReadableFile::supports_zero_copy() const { return false; }
 
 Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
   return impl_->Read(nbytes, bytes_read, buffer);
@@ -221,17 +226,11 @@ Status HdfsReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* buffer) {
   return impl_->Read(nbytes, buffer);
 }
 
-Status HdfsReadableFile::GetSize(int64_t* size) {
-  return impl_->GetSize(size);
-}
+Status HdfsReadableFile::GetSize(int64_t* size) { return impl_->GetSize(size); }
 
-Status HdfsReadableFile::Seek(int64_t position) {
-  return impl_->Seek(position);
-}
+Status HdfsReadableFile::Seek(int64_t position) { return impl_->Seek(position); }
 
-Status HdfsReadableFile::Tell(int64_t* position) {
-  return impl_->Tell(position);
-}
+Status HdfsReadableFile::Tell(int64_t* position) { return impl_->Tell(position); }
 
 // ----------------------------------------------------------------------
 // File writing
@@ -259,28 +258,22 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
 
   Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
     std::lock_guard<std::mutex> guard(lock_);
-    tSize ret = driver_->Write(
-        fs_, file_, reinterpret_cast<const void*>(buffer), static_cast<tSize>(nbytes));
+    tSize ret = driver_->Write(fs_, file_, reinterpret_cast<const void*>(buffer),
+                               static_cast<tSize>(nbytes));
     CHECK_FAILURE(ret, "Write");
     *bytes_written = ret;
     return Status::OK();
   }
 };
 
-HdfsOutputStream::HdfsOutputStream() {
-  impl_.reset(new HdfsOutputStreamImpl());
-}
+HdfsOutputStream::HdfsOutputStream() { impl_.reset(new HdfsOutputStreamImpl()); }
 
-HdfsOutputStream::~HdfsOutputStream() {
-  DCHECK(impl_->Close().ok());
-}
+HdfsOutputStream::~HdfsOutputStream() { DCHECK(impl_->Close().ok()); }
 
-Status HdfsOutputStream::Close() {
-  return impl_->Close();
-}
+Status HdfsOutputStream::Close() { return impl_->Close(); }
 
-Status HdfsOutputStream::Write(
-    const uint8_t* buffer, int64_t nbytes, int64_t* bytes_read) {
+Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes,
+                               int64_t* bytes_read) {
   return impl_->Write(buffer, nbytes, bytes_read);
 }
 
@@ -289,13 +282,9 @@ Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes) {
   return Write(buffer, nbytes, &bytes_written_dummy);
 }
 
-Status HdfsOutputStream::Flush() {
-  return impl_->Flush();
-}
+Status HdfsOutputStream::Flush() { return impl_->Flush(); }
 
-Status HdfsOutputStream::Tell(int64_t* position) {
-  return impl_->Tell(position);
-}
+Status HdfsOutputStream::Tell(int64_t* position) { return impl_->Tell(position); }
 
 // ----------------------------------------------------------------------
 // HDFS client
@@ -344,7 +333,9 @@ class HdfsClient::HdfsClientImpl {
     }
     fs_ = driver_->BuilderConnect(builder);
 
-    if (fs_ == nullptr) { return Status::IOError("HDFS connection failed"); }
+    if (fs_ == nullptr) {
+      return Status::IOError("HDFS connection failed");
+    }
     namenode_host_ = config->host;
     port_ = config->port;
     user_ = config->user;
@@ -395,7 +386,9 @@ class HdfsClient::HdfsClientImpl {
   Status GetPathInfo(const std::string& path, HdfsPathInfo* info) {
     hdfsFileInfo* entry = driver_->GetPathInfo(fs_, path.c_str());
 
-    if (entry == nullptr) { return Status::IOError("HDFS: GetPathInfo failed"); }
+    if (entry == nullptr) {
+      return Status::IOError("HDFS: GetPathInfo failed");
+    }
 
     SetPathInfo(entry, info);
     driver_->FreeFileInfo(entry, 1);
@@ -435,7 +428,7 @@ class HdfsClient::HdfsClientImpl {
   }
 
   Status OpenReadable(const std::string& path, int32_t buffer_size,
-      std::shared_ptr<HdfsReadableFile>* file) {
+                      std::shared_ptr<HdfsReadableFile>* file) {
     hdfsFile handle = driver_->OpenFile(fs_, path.c_str(), O_RDONLY, buffer_size, 0, 0);
 
     if (handle == nullptr) {
@@ -454,13 +447,14 @@ class HdfsClient::HdfsClientImpl {
   }
 
   Status OpenWriteable(const std::string& path, bool append, int32_t buffer_size,
-      int16_t replication, int64_t default_block_size,
-      std::shared_ptr<HdfsOutputStream>* file) {
+                       int16_t replication, int64_t default_block_size,
+                       std::shared_ptr<HdfsOutputStream>* file) {
     int flags = O_WRONLY;
     if (append) flags |= O_APPEND;
 
-    hdfsFile handle = driver_->OpenFile(fs_, path.c_str(), flags, buffer_size,
-        replication, static_cast<tSize>(default_block_size));
+    hdfsFile handle =
+        driver_->OpenFile(fs_, path.c_str(), flags, buffer_size, replication,
+                          static_cast<tSize>(default_block_size));
 
     if (handle == nullptr) {
       // TODO(wesm): determine cause of failure
@@ -496,14 +490,12 @@ class HdfsClient::HdfsClientImpl {
 // ----------------------------------------------------------------------
 // Public API for HDFSClient
 
-HdfsClient::HdfsClient() {
-  impl_.reset(new HdfsClientImpl());
-}
+HdfsClient::HdfsClient() { impl_.reset(new HdfsClientImpl()); }
 
 HdfsClient::~HdfsClient() {}
 
-Status HdfsClient::Connect(
-    const HdfsConnectionConfig* config, std::shared_ptr<HdfsClient>* fs) {
+Status HdfsClient::Connect(const HdfsConnectionConfig* config,
+                           std::shared_ptr<HdfsClient>* fs) {
   // ctor is private, make_shared will not work
   *fs = std::shared_ptr<HdfsClient>(new HdfsClient());
 
@@ -519,50 +511,43 @@ Status HdfsClient::Delete(const std::string& path, bool recursive) {
   return impl_->Delete(path, recursive);
 }
 
-Status HdfsClient::Disconnect() {
-  return impl_->Disconnect();
-}
+Status HdfsClient::Disconnect() { return impl_->Disconnect(); }
 
-bool HdfsClient::Exists(const std::string& path) {
-  return impl_->Exists(path);
-}
+bool HdfsClient::Exists(const std::string& path) { return impl_->Exists(path); }
 
 Status HdfsClient::GetPathInfo(const std::string& path, HdfsPathInfo* info) {
   return impl_->GetPathInfo(path, info);
 }
 
-Status HdfsClient::GetCapacity(int64_t* nbytes) {
-  return impl_->GetCapacity(nbytes);
-}
+Status HdfsClient::GetCapacity(int64_t* nbytes) { return impl_->GetCapacity(nbytes); }
 
-Status HdfsClient::GetUsed(int64_t* nbytes) {
-  return impl_->GetUsed(nbytes);
-}
+Status HdfsClient::GetUsed(int64_t* nbytes) { return impl_->GetUsed(nbytes); }
 
-Status HdfsClient::ListDirectory(
-    const std::string& path, std::vector<HdfsPathInfo>* listing) {
+Status HdfsClient::ListDirectory(const std::string& path,
+                                 std::vector<HdfsPathInfo>* listing) {
   return impl_->ListDirectory(path, listing);
 }
 
 Status HdfsClient::OpenReadable(const std::string& path, int32_t buffer_size,
-    std::shared_ptr<HdfsReadableFile>* file) {
+                                std::shared_ptr<HdfsReadableFile>* file) {
   return impl_->OpenReadable(path, buffer_size, file);
 }
 
-Status HdfsClient::OpenReadable(
-    const std::string& path, std::shared_ptr<HdfsReadableFile>* file) {
+Status HdfsClient::OpenReadable(const std::string& path,
+                                std::shared_ptr<HdfsReadableFile>* file) {
   return OpenReadable(path, kDefaultHdfsBufferSize, file);
 }
 
 Status HdfsClient::OpenWriteable(const std::string& path, bool append,
-    int32_t buffer_size, int16_t replication, int64_t default_block_size,
-    std::shared_ptr<HdfsOutputStream>* file) {
-  return impl_->OpenWriteable(
-      path, append, buffer_size, replication, default_block_size, file);
+                                 int32_t buffer_size, int16_t replication,
+                                 int64_t default_block_size,
+                                 std::shared_ptr<HdfsOutputStream>* file) {
+  return impl_->OpenWriteable(path, append, buffer_size, replication, default_block_size,
+                              file);
 }
 
-Status HdfsClient::OpenWriteable(
-    const std::string& path, bool append, std::shared_ptr<HdfsOutputStream>* file) {
+Status HdfsClient::OpenWriteable(const std::string& path, bool append,
+                                 std::shared_ptr<HdfsOutputStream>* file) {
   return OpenWriteable(path, append, 0, 0, 0, file);
 }
 
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index f3de4a2bf174f..63c3ae0d53724 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -75,8 +75,8 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   // @param config (in): configuration for connecting
   // @param fs (out): the created client
   // @returns Status
-  static Status Connect(
-      const HdfsConnectionConfig* config, std::shared_ptr<HdfsClient>* fs);
+  static Status Connect(const HdfsConnectionConfig* config,
+                        std::shared_ptr<HdfsClient>* fs);
 
   // Create directory and all parents
   //
@@ -132,7 +132,7 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   //
   // @param path complete file path
   Status OpenReadable(const std::string& path, int32_t buffer_size,
-      std::shared_ptr<HdfsReadableFile>* file);
+                      std::shared_ptr<HdfsReadableFile>* file);
 
   Status OpenReadable(const std::string& path, std::shared_ptr<HdfsReadableFile>* file);
 
@@ -142,11 +142,11 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   // @param replication, 0 for default
   // @param default_block_size, 0 for default
   Status OpenWriteable(const std::string& path, bool append, int32_t buffer_size,
-      int16_t replication, int64_t default_block_size,
-      std::shared_ptr<HdfsOutputStream>* file);
+                       int16_t replication, int64_t default_block_size,
+                       std::shared_ptr<HdfsOutputStream>* file);
 
-  Status OpenWriteable(
-      const std::string& path, bool append, std::shared_ptr<HdfsOutputStream>* file);
+  Status OpenWriteable(const std::string& path, bool append,
+                       std::shared_ptr<HdfsOutputStream>* file);
 
  private:
   friend class HdfsReadableFile;
@@ -173,8 +173,8 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
 
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
-  Status ReadAt(
-      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                uint8_t* buffer) override;
 
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 06957d4de560d..57dc42d8a9b2a 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -29,32 +29,28 @@ namespace io {
 
 FileInterface::~FileInterface() {}
 
-RandomAccessFile::RandomAccessFile() {
-  set_mode(FileMode::READ);
-}
+RandomAccessFile::RandomAccessFile() { set_mode(FileMode::READ); }
 
-Status RandomAccessFile::ReadAt(
-    int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                                uint8_t* out) {
   std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
 }
 
-Status RandomAccessFile::ReadAt(
-    int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) {
+Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes,
+                                std::shared_ptr<Buffer>* out) {
   std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, out);
 }
 
 Status Writeable::Write(const std::string& data) {
-  return Write(
-      reinterpret_cast<const uint8_t*>(data.c_str()), static_cast<int64_t>(data.size()));
+  return Write(reinterpret_cast<const uint8_t*>(data.c_str()),
+               static_cast<int64_t>(data.size()));
 }
 
-Status Writeable::Flush() {
-  return Status::OK();
-}
+Status Writeable::Flush() { return Status::OK(); }
 
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index b5a0bd85bf27b..e71a5c93baa32 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -107,8 +107,8 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
   /// be overridden
   ///
   /// Default implementation is thread-safe
-  virtual Status ReadAt(
-      int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out);
+  virtual Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                        uint8_t* out);
 
   /// Default implementation is thread-safe
   virtual Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out);
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index a077f8cb921c7..36c35700d6496 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -43,9 +43,10 @@ static bool FileExists(const std::string& path) {
 
 #if defined(_MSC_VER)
 void InvalidParamHandler(const wchar_t* expr, const wchar_t* func,
-    const wchar_t* source_file, unsigned int source_line, uintptr_t reserved) {
+                         const wchar_t* source_file, unsigned int source_line,
+                         uintptr_t reserved) {
   wprintf(L"Invalid parameter in funcion %s. Source: %s line %d expression %s", func,
-      source_file, source_line, expr);
+          source_file, source_line, expr);
 }
 #endif
 
@@ -61,7 +62,9 @@ static bool FileIsClosed(int fd) {
   int ret = static_cast<int>(_close(fd));
   return (ret == -1);
 #else
-  if (-1 != fcntl(fd, F_GETFD)) { return false; }
+  if (-1 != fcntl(fd, F_GETFD)) {
+    return false;
+  }
   return errno == EBADF;
 #endif
 }
@@ -76,7 +79,9 @@ class FileTestFixture : public ::testing::Test {
   void TearDown() { EnsureFileDeleted(); }
 
   void EnsureFileDeleted() {
-    if (FileExists(path_)) { std::remove(path_.c_str()); }
+    if (FileExists(path_)) {
+      std::remove(path_.c_str());
+    }
   }
 
  protected:
@@ -382,7 +387,9 @@ TEST_F(TestReadableFile, ThreadSafety) {
 
     for (int i = 0; i < niter; ++i) {
       ASSERT_OK(file_->ReadAt(0, 3, &buffer));
-      if (0 == memcmp(data.c_str(), buffer->data(), 3)) { correct_count += 1; }
+      if (0 == memcmp(data.c_str(), buffer->data(), 3)) {
+        correct_count += 1;
+      }
     }
   };
 
@@ -547,8 +554,8 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
   std::string non_existent_path = "invalid-file-name-asfd";
 
   std::shared_ptr<MemoryMappedFile> result;
-  ASSERT_RAISES(
-      IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result));
+  ASSERT_RAISES(IOError,
+                MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result));
 }
 
 TEST_F(TestMemoryMappedFile, CastableToFileInterface) {
@@ -563,8 +570,8 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) {
 
   std::shared_ptr<MemoryMappedFile> file;
   ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &file));
-  ASSERT_OK(file->Write(
-      reinterpret_cast<const uint8_t*>(data.c_str()), static_cast<int64_t>(data.size())));
+  ASSERT_OK(file->Write(reinterpret_cast<const uint8_t*>(data.c_str()),
+                        static_cast<int64_t>(data.size())));
 
   std::atomic<int> correct_count(0);
   const int niter = 10000;
@@ -574,7 +581,9 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) {
 
     for (int i = 0; i < niter; ++i) {
       ASSERT_OK(file->ReadAt(0, 3, &buffer));
-      if (0 == memcmp(data.c_str(), buffer->data(), 3)) { correct_count += 1; }
+      if (0 == memcmp(data.c_str(), buffer->data(), 3)) {
+        correct_count += 1;
+      }
     }
   };
 
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 74f80428c4561..c584cf5adeaf2 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -58,11 +58,11 @@ class TestHdfsClient : public ::testing::Test {
   }
 
   Status WriteDummyFile(const std::string& path, const uint8_t* buffer, int64_t size,
-      bool append = false, int buffer_size = 0, int16_t replication = 0,
-      int default_block_size = 0) {
+                        bool append = false, int buffer_size = 0, int16_t replication = 0,
+                        int default_block_size = 0) {
     std::shared_ptr<HdfsOutputStream> file;
-    RETURN_NOT_OK(client_->OpenWriteable(
-        path, append, buffer_size, replication, default_block_size, &file));
+    RETURN_NOT_OK(client_->OpenWriteable(path, append, buffer_size, replication,
+                                         default_block_size, &file));
 
     RETURN_NOT_OK(file->Write(buffer, size));
     RETURN_NOT_OK(file->Close());
@@ -87,9 +87,10 @@ class TestHdfsClient : public ::testing::Test {
     LibHdfsShim* driver_shim;
 
     client_ = nullptr;
-    scratch_dir_ = boost::filesystem::unique_path(
-        boost::filesystem::temp_directory_path() / "arrow-hdfs/scratch-%%%%")
-                       .string();
+    scratch_dir_ =
+        boost::filesystem::unique_path(boost::filesystem::temp_directory_path() /
+                                       "arrow-hdfs/scratch-%%%%")
+            .string();
 
     loaded_driver_ = false;
 
@@ -175,7 +176,9 @@ TYPED_TEST(TestHdfsClient, MakeDirectory) {
 
   std::string path = this->ScratchPath("create-directory");
 
-  if (this->client_->Exists(path)) { ASSERT_OK(this->client_->Delete(path, true)); }
+  if (this->client_->Exists(path)) {
+    ASSERT_OK(this->client_->Delete(path, true));
+  }
 
   ASSERT_OK(this->client_->MakeDirectory(path));
   ASSERT_TRUE(this->client_->Exists(path));
@@ -396,7 +399,7 @@ TYPED_TEST(TestHdfsClient, ThreadSafety) {
 
   std::string data = "foobar";
   ASSERT_OK(this->WriteDummyFile(src_path, reinterpret_cast<const uint8_t*>(data.c_str()),
-      static_cast<int64_t>(data.size())));
+                                 static_cast<int64_t>(data.size())));
 
   std::shared_ptr<HdfsReadableFile> file;
   ASSERT_OK(this->client_->OpenReadable(src_path, &file));
@@ -409,10 +412,14 @@ TYPED_TEST(TestHdfsClient, ThreadSafety) {
       std::shared_ptr<Buffer> buffer;
       if (i % 2 == 0) {
         ASSERT_OK(file->ReadAt(3, 3, &buffer));
-        if (0 == memcmp(data.c_str() + 3, buffer->data(), 3)) { correct_count += 1; }
+        if (0 == memcmp(data.c_str() + 3, buffer->data(), 3)) {
+          correct_count += 1;
+        }
       } else {
         ASSERT_OK(file->ReadAt(0, 4, &buffer));
-        if (0 == memcmp(data.c_str() + 0, buffer->data(), 4)) { correct_count += 1; }
+        if (0 == memcmp(data.c_str() + 0, buffer->data(), 4)) {
+          correct_count += 1;
+        }
       }
     }
   };
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 4d8bf63757d64..b6c48ec39be89 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -46,7 +46,7 @@ BufferOutputStream::BufferOutputStream(const std::shared_ptr<ResizableBuffer>& b
       mutable_data_(buffer->mutable_data()) {}
 
 Status BufferOutputStream::Create(int64_t initial_capacity, MemoryPool* pool,
-    std::shared_ptr<BufferOutputStream>* out) {
+                                  std::shared_ptr<BufferOutputStream>* out) {
   std::shared_ptr<ResizableBuffer> buffer;
   RETURN_NOT_OK(AllocateResizableBuffer(pool, initial_capacity, &buffer));
   *out = std::make_shared<BufferOutputStream>(buffer);
@@ -55,7 +55,9 @@ Status BufferOutputStream::Create(int64_t initial_capacity, MemoryPool* pool,
 
 BufferOutputStream::~BufferOutputStream() {
   // This can fail, better to explicitly call close
-  if (buffer_) { DCHECK(Close().ok()); }
+  if (buffer_) {
+    DCHECK(Close().ok());
+  }
 }
 
 Status BufferOutputStream::Close() {
@@ -102,9 +104,7 @@ Status BufferOutputStream::Reserve(int64_t nbytes) {
 // ----------------------------------------------------------------------
 // OutputStream that doesn't write anything
 
-Status MockOutputStream::Close() {
-  return Status::OK();
-}
+Status MockOutputStream::Close() { return Status::OK(); }
 
 Status MockOutputStream::Tell(int64_t* position) {
   *position = extent_bytes_written_;
@@ -158,7 +158,7 @@ Status FixedSizeBufferWriter::Tell(int64_t* position) {
 Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
   if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) {
     parallel_memcopy(mutable_data_ + position_, data, nbytes, memcopy_blocksize_,
-        memcopy_num_threads_);
+                     memcopy_num_threads_);
   } else {
     memcpy(mutable_data_ + position_, data, nbytes);
   }
@@ -166,8 +166,8 @@ Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
   return Status::OK();
 }
 
-Status FixedSizeBufferWriter::WriteAt(
-    int64_t position, const uint8_t* data, int64_t nbytes) {
+Status FixedSizeBufferWriter::WriteAt(int64_t position, const uint8_t* data,
+                                      int64_t nbytes) {
   std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
   return Write(data, nbytes);
@@ -206,9 +206,7 @@ Status BufferReader::Tell(int64_t* position) {
   return Status::OK();
 }
 
-bool BufferReader::supports_zero_copy() const {
-  return true;
-}
+bool BufferReader::supports_zero_copy() const { return true; }
 
 Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
   memcpy(buffer, data_ + position_, nbytes);
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 06384f0d4c4b7..1f8177436471c 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -45,7 +45,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   explicit BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer);
 
   static Status Create(int64_t initial_capacity, MemoryPool* pool,
-      std::shared_ptr<BufferOutputStream>* out);
+                       std::shared_ptr<BufferOutputStream>* out);
 
   ~BufferOutputStream();
 
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index 438f378085f65..a4974b77528de 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -73,8 +73,8 @@ class MemoryMapFixture {
     tmp_files_.push_back(path);
   }
 
-  Status InitMemoryMap(
-      int64_t size, const std::string& path, std::shared_ptr<MemoryMappedFile>* mmap) {
+  Status InitMemoryMap(int64_t size, const std::string& path,
+                       std::shared_ptr<MemoryMappedFile>* mmap) {
     RETURN_NOT_OK(MemoryMappedFile::Create(path, size, mmap));
     tmp_files_.push_back(path);
     return Status::OK();
diff --git a/cpp/src/arrow/ipc/feather-internal.h b/cpp/src/arrow/ipc/feather-internal.h
index 646c3b2f9f2e3..36cfecc0493f5 100644
--- a/cpp/src/arrow/ipc/feather-internal.h
+++ b/cpp/src/arrow/ipc/feather-internal.h
@@ -49,7 +49,7 @@ struct ARROW_EXPORT ArrayMetadata {
   ArrayMetadata() {}
 
   ArrayMetadata(fbs::Type type, int64_t offset, int64_t length, int64_t null_count,
-      int64_t total_bytes)
+                int64_t total_bytes)
       : type(type),
         offset(offset),
         length(length),
@@ -135,7 +135,9 @@ class ARROW_EXPORT TableMetadata {
   bool HasDescription() const { return table_->description() != 0; }
 
   std::string GetDescription() const {
-    if (!HasDescription()) { return std::string(""); }
+    if (!HasDescription()) {
+      return std::string("");
+    }
     return table_->description()->str();
   }
 
@@ -153,7 +155,7 @@ class ARROW_EXPORT TableMetadata {
 static inline flatbuffers::Offset<fbs::PrimitiveArray> GetPrimitiveArray(
     FBB& fbb, const ArrayMetadata& array) {
   return fbs::CreatePrimitiveArray(fbb, array.type, fbs::Encoding_PLAIN, array.offset,
-      array.length, array.null_count, array.total_bytes);
+                                   array.length, array.null_count, array.total_bytes);
 }
 
 static inline fbs::TimeUnit ToFlatbufferEnum(TimeUnit::type unit) {
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index 029aae31ff52c..b76b518788b91 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -365,8 +365,8 @@ TEST_F(TestTableWriter, TimeTypes) {
   ArrayFromVector<Date32Type, int32_t>(is_valid, date_values_vec, &date_array);
 
   const auto& prim_values = static_cast<const PrimitiveArray&>(*values);
-  std::vector<std::shared_ptr<Buffer>> buffers = {
-      prim_values.null_bitmap(), prim_values.values()};
+  std::vector<std::shared_ptr<Buffer>> buffers = {prim_values.null_bitmap(),
+                                                  prim_values.values()};
 
   std::vector<std::shared_ptr<internal::ArrayData>> arrays;
   arrays.push_back(date_array->data());
@@ -400,7 +400,8 @@ TEST_F(TestTableWriter, PrimitiveNullRoundTrip) {
     ASSERT_OK(reader_->GetColumn(i, &col));
     ASSERT_EQ(batch->column_name(i), col->name());
     StringArray str_values(batch->column(i)->length(), nullptr, nullptr,
-        batch->column(i)->null_bitmap(), batch->column(i)->null_count());
+                           batch->column(i)->null_bitmap(),
+                           batch->column(i)->null_count());
     CheckArrays(str_values, *col->data()->chunk(0));
   }
 }
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 61b96e0c1dc3b..54771d3356b83 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -61,26 +61,30 @@ static int64_t GetOutputLength(int64_t nbytes) {
 }
 
 static Status WritePadded(io::OutputStream* stream, const uint8_t* data, int64_t length,
-    int64_t* bytes_written) {
+                          int64_t* bytes_written) {
   RETURN_NOT_OK(stream->Write(data, length));
 
   int64_t remainder = PaddedLength(length) - length;
-  if (remainder != 0) { RETURN_NOT_OK(stream->Write(kPaddingBytes, remainder)); }
+  if (remainder != 0) {
+    RETURN_NOT_OK(stream->Write(kPaddingBytes, remainder));
+  }
   *bytes_written = length + remainder;
   return Status::OK();
 }
 
 /// For compability, we need to write any data sometimes just to keep producing
 /// files that can be read with an older reader.
-static Status WritePaddedBlank(
-    io::OutputStream* stream, int64_t length, int64_t* bytes_written) {
+static Status WritePaddedBlank(io::OutputStream* stream, int64_t length,
+                               int64_t* bytes_written) {
   const uint8_t null = 0;
   for (int64_t i = 0; i < length; i++) {
     RETURN_NOT_OK(stream->Write(&null, 1));
   }
 
   int64_t remainder = PaddedLength(length) - length;
-  if (remainder != 0) { RETURN_NOT_OK(stream->Write(kPaddingBytes, remainder)); }
+  if (remainder != 0) {
+    RETURN_NOT_OK(stream->Write(kPaddingBytes, remainder));
+  }
   *bytes_written = length + remainder;
   return Status::OK();
 }
@@ -90,20 +94,22 @@ static Status WritePaddedBlank(
 
 TableBuilder::TableBuilder(int64_t num_rows) : finished_(false), num_rows_(num_rows) {}
 
-FBB& TableBuilder::fbb() {
-  return fbb_;
-}
+FBB& TableBuilder::fbb() { return fbb_; }
 
 Status TableBuilder::Finish() {
-  if (finished_) { return Status::Invalid("can only call this once"); }
+  if (finished_) {
+    return Status::Invalid("can only call this once");
+  }
 
   FBString desc = 0;
-  if (!description_.empty()) { desc = fbb_.CreateString(description_); }
+  if (!description_.empty()) {
+    desc = fbb_.CreateString(description_);
+  }
 
   flatbuffers::Offset<flatbuffers::String> metadata = 0;
 
-  auto root = fbs::CreateCTable(
-      fbb_, desc, num_rows_, fbb_.CreateVector(columns_), kFeatherVersion, metadata);
+  auto root = fbs::CreateCTable(fbb_, desc, num_rows_, fbb_.CreateVector(columns_),
+                                kFeatherVersion, metadata);
   fbb_.Finish(root);
   finished_ = true;
 
@@ -111,17 +117,15 @@ Status TableBuilder::Finish() {
 }
 
 std::shared_ptr<Buffer> TableBuilder::GetBuffer() const {
-  return std::make_shared<Buffer>(
-      fbb_.GetBufferPointer(), static_cast<int64_t>(fbb_.GetSize()));
+  return std::make_shared<Buffer>(fbb_.GetBufferPointer(),
+                                  static_cast<int64_t>(fbb_.GetSize()));
 }
 
 void TableBuilder::SetDescription(const std::string& description) {
   description_ = description;
 }
 
-void TableBuilder::SetNumRows(int64_t num_rows) {
-  num_rows_ = num_rows;
-}
+void TableBuilder::SetNumRows(int64_t num_rows) { num_rows_ = num_rows; }
 
 void TableBuilder::add_column(const flatbuffers::Offset<fbs::Column>& col) {
   columns_.push_back(col);
@@ -177,21 +181,17 @@ Status ColumnBuilder::Finish() {
   flatbuffers::Offset<void> metadata = CreateColumnMetadata();
 
   auto column = fbs::CreateColumn(buf, buf.CreateString(name_), values,
-      ToFlatbufferEnum(type_),  // metadata_type
-      metadata, buf.CreateString(user_metadata_));
+                                  ToFlatbufferEnum(type_),  // metadata_type
+                                  metadata, buf.CreateString(user_metadata_));
 
   // bad coupling, but OK for now
   parent_->add_column(column);
   return Status::OK();
 }
 
-void ColumnBuilder::SetValues(const ArrayMetadata& values) {
-  values_ = values;
-}
+void ColumnBuilder::SetValues(const ArrayMetadata& values) { values_ = values; }
 
-void ColumnBuilder::SetUserMetadata(const std::string& data) {
-  user_metadata_ = data;
-}
+void ColumnBuilder::SetUserMetadata(const std::string& data) { user_metadata_ = data; }
 
 void ColumnBuilder::SetCategory(const ArrayMetadata& levels, bool ordered) {
   type_ = ColumnType::CATEGORY;
@@ -209,18 +209,14 @@ void ColumnBuilder::SetTimestamp(TimeUnit::type unit, const std::string& timezon
   meta_timestamp_.timezone = timezone;
 }
 
-void ColumnBuilder::SetDate() {
-  type_ = ColumnType::DATE;
-}
+void ColumnBuilder::SetDate() { type_ = ColumnType::DATE; }
 
 void ColumnBuilder::SetTime(TimeUnit::type unit) {
   type_ = ColumnType::TIME;
   meta_time_.unit = unit;
 }
 
-FBB& ColumnBuilder::fbb() {
-  return *fbb_;
-}
+FBB& ColumnBuilder::fbb() { return *fbb_; }
 
 std::unique_ptr<ColumnBuilder> TableBuilder::AddColumn(const std::string& name) {
   return std::unique_ptr<ColumnBuilder>(new ColumnBuilder(this, name));
@@ -272,7 +268,7 @@ class TableReader::TableReaderImpl {
   }
 
   Status GetDataType(const fbs::PrimitiveArray* values, fbs::TypeMetadata metadata_type,
-      const void* metadata, std::shared_ptr<DataType>* out) {
+                     const void* metadata, std::shared_ptr<DataType>* out) {
 #define PRIMITIVE_CASE(CAP_TYPE, FACTORY_FUNC) \
   case fbs::Type_##CAP_TYPE:                   \
     *out = FACTORY_FUNC();                     \
@@ -342,7 +338,7 @@ class TableReader::TableReaderImpl {
   // @returns: a Buffer instance, the precise type will depend on the kind of
   // input data source (which may or may not have memory-map like semantics)
   Status LoadValues(const fbs::PrimitiveArray* meta, fbs::TypeMetadata metadata_type,
-      const void* metadata, std::shared_ptr<Array>* out) {
+                    const void* metadata, std::shared_ptr<Array>* out) {
     std::shared_ptr<DataType> type;
     RETURN_NOT_OK(GetDataType(meta, metadata_type, metadata, &type));
 
@@ -394,8 +390,8 @@ class TableReader::TableReaderImpl {
     // if (user_meta->size() > 0) { user_metadata_ = user_meta->str(); }
 
     std::shared_ptr<Array> values;
-    RETURN_NOT_OK(LoadValues(
-        col_meta->values(), col_meta->metadata_type(), col_meta->metadata(), &values));
+    RETURN_NOT_OK(LoadValues(col_meta->values(), col_meta->metadata_type(),
+                             col_meta->metadata(), &values));
     out->reset(new Column(col_meta->name()->str(), values));
     return Status::OK();
   }
@@ -410,41 +406,27 @@ class TableReader::TableReaderImpl {
 // ----------------------------------------------------------------------
 // TableReader public API
 
-TableReader::TableReader() {
-  impl_.reset(new TableReaderImpl());
-}
+TableReader::TableReader() { impl_.reset(new TableReaderImpl()); }
 
 TableReader::~TableReader() {}
 
 Status TableReader::Open(const std::shared_ptr<io::RandomAccessFile>& source,
-    std::unique_ptr<TableReader>* out) {
+                         std::unique_ptr<TableReader>* out) {
   out->reset(new TableReader());
   return (*out)->impl_->Open(source);
 }
 
-bool TableReader::HasDescription() const {
-  return impl_->HasDescription();
-}
+bool TableReader::HasDescription() const { return impl_->HasDescription(); }
 
-std::string TableReader::GetDescription() const {
-  return impl_->GetDescription();
-}
+std::string TableReader::GetDescription() const { return impl_->GetDescription(); }
 
-int TableReader::version() const {
-  return impl_->version();
-}
+int TableReader::version() const { return impl_->version(); }
 
-int64_t TableReader::num_rows() const {
-  return impl_->num_rows();
-}
+int64_t TableReader::num_rows() const { return impl_->num_rows(); }
 
-int64_t TableReader::num_columns() const {
-  return impl_->num_columns();
-}
+int64_t TableReader::num_columns() const { return impl_->num_columns(); }
 
-std::string TableReader::GetColumnName(int i) const {
-  return impl_->GetColumnName(i);
-}
+std::string TableReader::GetColumnName(int i) const { return impl_->GetColumnName(i); }
 
 Status TableReader::GetColumn(int i, std::shared_ptr<Column>* out) {
   return impl_->GetColumn(i, out);
@@ -501,8 +483,8 @@ static Status SanitizeUnsupportedTypes(const Array& values, std::shared_ptr<Arra
   if (values.type_id() == Type::NA) {
     // As long as R doesn't support NA, we write this as a StringColumn
     // to ensure stable roundtrips.
-    *out = std::make_shared<StringArray>(
-        values.length(), nullptr, nullptr, values.null_bitmap(), values.null_count());
+    *out = std::make_shared<StringArray>(values.length(), nullptr, nullptr,
+                                         values.null_bitmap(), values.null_count());
     return Status::OK();
   } else {
     return MakeArray(values.data(), out);
@@ -537,8 +519,8 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     // Footer: metadata length, magic bytes
     RETURN_NOT_OK(
         stream_->Write(reinterpret_cast<const uint8_t*>(&buffer_size), sizeof(uint32_t)));
-    return stream_->Write(
-        reinterpret_cast<const uint8_t*>(kFeatherMagicBytes), strlen(kFeatherMagicBytes));
+    return stream_->Write(reinterpret_cast<const uint8_t*>(kFeatherMagicBytes),
+                          strlen(kFeatherMagicBytes));
   }
 
   Status LoadArrayMetadata(const Array& values, ArrayMetadata* meta) {
@@ -571,7 +553,7 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
       // byte boundary, and we write this much data into the stream
       if (values.null_bitmap()) {
         RETURN_NOT_OK(WritePadded(stream_.get(), values.null_bitmap()->data(),
-            values.null_bitmap()->size(), &bytes_written));
+                                  values.null_bitmap()->size(), &bytes_written));
       } else {
         RETURN_NOT_OK(WritePaddedBlank(
             stream_.get(), BitUtil::BytesForBits(values.length()), &bytes_written));
@@ -592,15 +574,17 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
         values_bytes = bin_values.raw_value_offsets()[values.length()];
 
         // Write the variable-length offsets
-        RETURN_NOT_OK(WritePadded(stream_.get(),
-            reinterpret_cast<const uint8_t*>(bin_values.raw_value_offsets()),
-            offset_bytes, &bytes_written));
+        RETURN_NOT_OK(WritePadded(stream_.get(), reinterpret_cast<const uint8_t*>(
+                                                     bin_values.raw_value_offsets()),
+                                  offset_bytes, &bytes_written));
       } else {
         RETURN_NOT_OK(WritePaddedBlank(stream_.get(), offset_bytes, &bytes_written));
       }
       meta->total_bytes += bytes_written;
 
-      if (bin_values.value_data()) { values_buffer = bin_values.value_data()->data(); }
+      if (bin_values.value_data()) {
+        values_buffer = bin_values.value_data()->data();
+      }
     } else {
       const auto& prim_values = static_cast<const PrimitiveArray&>(values);
       const auto& fw_type = static_cast<const FixedWidthType&>(*values.type());
@@ -612,7 +596,9 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
         values_bytes = values.length() * fw_type.bit_width() / 8;
       }
 
-      if (prim_values.values()) { values_buffer = prim_values.values()->data(); }
+      if (prim_values.values()) {
+        values_buffer = prim_values.values()->data();
+      }
     }
     if (values_buffer) {
       RETURN_NOT_OK(
@@ -710,9 +696,9 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
   Status CheckStarted() {
     if (!initialized_stream_) {
       int64_t bytes_written_unused;
-      RETURN_NOT_OK(
-          WritePadded(stream_.get(), reinterpret_cast<const uint8_t*>(kFeatherMagicBytes),
-              strlen(kFeatherMagicBytes), &bytes_written_unused));
+      RETURN_NOT_OK(WritePadded(stream_.get(),
+                                reinterpret_cast<const uint8_t*>(kFeatherMagicBytes),
+                                strlen(kFeatherMagicBytes), &bytes_written_unused));
       initialized_stream_ = true;
     }
     return Status::OK();
@@ -728,33 +714,25 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
   Status AppendPrimitive(const PrimitiveArray& values, ArrayMetadata* out);
 };
 
-TableWriter::TableWriter() {
-  impl_.reset(new TableWriterImpl());
-}
+TableWriter::TableWriter() { impl_.reset(new TableWriterImpl()); }
 
 TableWriter::~TableWriter() {}
 
-Status TableWriter::Open(
-    const std::shared_ptr<io::OutputStream>& stream, std::unique_ptr<TableWriter>* out) {
+Status TableWriter::Open(const std::shared_ptr<io::OutputStream>& stream,
+                         std::unique_ptr<TableWriter>* out) {
   out->reset(new TableWriter());
   return (*out)->impl_->Open(stream);
 }
 
-void TableWriter::SetDescription(const std::string& desc) {
-  impl_->SetDescription(desc);
-}
+void TableWriter::SetDescription(const std::string& desc) { impl_->SetDescription(desc); }
 
-void TableWriter::SetNumRows(int64_t num_rows) {
-  impl_->SetNumRows(num_rows);
-}
+void TableWriter::SetNumRows(int64_t num_rows) { impl_->SetNumRows(num_rows); }
 
 Status TableWriter::Append(const std::string& name, const Array& values) {
   return impl_->Append(name, values);
 }
 
-Status TableWriter::Finalize() {
-  return impl_->Finalize();
-}
+Status TableWriter::Finalize() { return impl_->Finalize(); }
 
 }  // namespace feather
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
index 4d59a8bbd54a9..8abcb5c0f2599 100644
--- a/cpp/src/arrow/ipc/feather.h
+++ b/cpp/src/arrow/ipc/feather.h
@@ -56,7 +56,7 @@ class ARROW_EXPORT TableReader {
   ~TableReader();
 
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& source,
-      std::unique_ptr<TableReader>* out);
+                     std::unique_ptr<TableReader>* out);
 
   // Optional table description
   //
@@ -83,8 +83,8 @@ class ARROW_EXPORT TableWriter {
  public:
   ~TableWriter();
 
-  static Status Open(
-      const std::shared_ptr<io::OutputStream>& stream, std::unique_ptr<TableWriter>* out);
+  static Status Open(const std::shared_ptr<io::OutputStream>& stream,
+                     std::unique_ptr<TableWriter>* out);
 
   void SetDescription(const std::string& desc);
   void SetNumRows(int64_t num_rows);
diff --git a/cpp/src/arrow/ipc/file-to-stream.cc b/cpp/src/arrow/ipc/file-to-stream.cc
index a1feedc212618..4707c4fcdf0f7 100644
--- a/cpp/src/arrow/ipc/file-to-stream.cc
+++ b/cpp/src/arrow/ipc/file-to-stream.cc
@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <iostream>
 #include "arrow/io/file.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
 #include "arrow/status.h"
-#include <iostream>
 
 #include "arrow/util/io-util.h"
 
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 79344df46b243..35264fa02c5ba 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -77,7 +77,9 @@ void TestArrayRoundTrip(const Array& array) {
   rj::Document d;
   d.Parse(array_as_json);
 
-  if (d.HasParseError()) { FAIL() << "JSON parsing failed"; }
+  if (d.HasParseError()) {
+    FAIL() << "JSON parsing failed";
+  }
 
   std::shared_ptr<Array> out;
   ASSERT_OK(internal::ReadArray(default_memory_pool(), d, array.type(), &out));
@@ -88,7 +90,8 @@ void TestArrayRoundTrip(const Array& array) {
 
 template <typename T, typename ValueType>
 void CheckPrimitive(const std::shared_ptr<DataType>& type,
-    const std::vector<bool>& is_valid, const std::vector<ValueType>& values) {
+                    const std::vector<bool>& is_valid,
+                    const std::vector<ValueType>& values) {
   MemoryPool* pool = default_memory_pool();
   typename TypeTraits<T>::BuilderType builder(pool);
 
@@ -108,16 +111,17 @@ void CheckPrimitive(const std::shared_ptr<DataType>& type,
 TEST(TestJsonSchemaWriter, FlatTypes) {
   // TODO
   // field("f14", date32())
-  std::vector<std::shared_ptr<Field>> fields = {field("f0", int8()),
-      field("f1", int16(), false), field("f2", int32()), field("f3", int64(), false),
-      field("f4", uint8()), field("f5", uint16()), field("f6", uint32()),
-      field("f7", uint64()), field("f8", float32()), field("f9", float64()),
-      field("f10", utf8()), field("f11", binary()), field("f12", list(int32())),
+  std::vector<std::shared_ptr<Field>> fields = {
+      field("f0", int8()), field("f1", int16(), false), field("f2", int32()),
+      field("f3", int64(), false), field("f4", uint8()), field("f5", uint16()),
+      field("f6", uint32()), field("f7", uint64()), field("f8", float32()),
+      field("f9", float64()), field("f10", utf8()), field("f11", binary()),
+      field("f12", list(int32())),
       field("f13", struct_({field("s1", int32()), field("s2", utf8())})),
       field("f15", date64()), field("f16", timestamp(TimeUnit::NANO)),
       field("f17", time64(TimeUnit::MICRO)),
       field("f18", union_({field("u1", int8()), field("u2", time32(TimeUnit::MILLI))},
-                       {0, 1}, UnionMode::DENSE))};
+                          {0, 1}, UnionMode::DENSE))};
 
   Schema schema(fields);
   TestSchemaRoundTrip(schema);
@@ -185,8 +189,8 @@ TEST(TestJsonArrayWriter, NestedTypes) {
       struct_({field("f1", int32()), field("f2", int32()), field("f3", int32())});
 
   std::vector<std::shared_ptr<Array>> fields = {values_array, values_array, values_array};
-  StructArray struct_array(
-      struct_type, static_cast<int>(struct_is_valid.size()), fields, struct_bitmap, 2);
+  StructArray struct_array(struct_type, static_cast<int>(struct_is_valid.size()), fields,
+                           struct_bitmap, 2);
   TestArrayRoundTrip(struct_array);
 }
 
@@ -202,7 +206,7 @@ TEST(TestJsonArrayWriter, Unions) {
 
 // Data generation for test case below
 void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
-    std::vector<std::shared_ptr<Array>>* arrays) {
+                     std::vector<std::shared_ptr<Array>>* arrays) {
   std::vector<bool> is_valid;
   test::random_is_valid(num_rows, 0.25, &is_valid);
 
@@ -266,8 +270,8 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
 
   std::unique_ptr<JsonReader> reader;
 
-  auto buffer = std::make_shared<Buffer>(
-      reinterpret_cast<const uint8_t*>(result.c_str()), static_cast<int>(result.size()));
+  auto buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(result.c_str()),
+                                         static_cast<int>(result.size()));
 
   ASSERT_OK(JsonReader::Open(buffer, &reader));
   ASSERT_TRUE(reader->schema()->Equals(*schema));
@@ -332,8 +336,8 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
 }
 )example";
 
-  auto buffer = std::make_shared<Buffer>(
-      reinterpret_cast<const uint8_t*>(example), strlen(example));
+  auto buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(example),
+                                         strlen(example));
 
   std::unique_ptr<JsonReader> reader;
   ASSERT_OK(JsonReader::Open(buffer, &reader));
@@ -361,9 +365,9 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
 
 #define BATCH_CASES()                                                                   \
   ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
-      &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,   \
-      &MakeStruct, &MakeUnion, &MakeDates, &MakeTimestamps, &MakeTimes, &MakeFWBinary,  \
-      &MakeDictionary);
+                    &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,                  \
+                    &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion, &MakeDates,   \
+                    &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDictionary);
 
 class TestJsonRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*> {
  public:
@@ -382,7 +386,7 @@ void CheckRoundtrip(const RecordBatch& batch) {
   ASSERT_OK(writer->Finish(&result));
 
   auto buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(result.c_str()),
-      static_cast<int64_t>(result.size()));
+                                         static_cast<int64_t>(result.size()));
 
   std::unique_ptr<JsonReader> reader;
   ASSERT_OK(JsonReader::Open(buffer, &reader));
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
index c890d829849fd..a88120a248d2d 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -80,7 +80,7 @@ static void BM_WriteRecordBatch(benchmark::State& state) {  // NOLINT non-const
     int32_t metadata_length;
     int64_t body_length;
     if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
-            default_memory_pool())
+                               default_memory_pool())
              .ok()) {
       state.SkipWithError("Failed to write!");
     }
@@ -101,7 +101,7 @@ static void BM_ReadRecordBatch(benchmark::State& state) {  // NOLINT non-const r
   int32_t metadata_length;
   int64_t body_length;
   if (!ipc::WriteRecordBatch(*record_batch, 0, &stream, &metadata_length, &body_length,
-          default_memory_pool())
+                             default_memory_pool())
            .ok()) {
     state.SkipWithError("Failed to write!");
   }
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 2119ff74056f2..6c7051750b7cb 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -126,40 +126,45 @@ TEST_F(TestSchemaMetadata, NestedFields) {
   CheckRoundtrip(schema, &memo);
 }
 
-#define BATCH_CASES()                                                                    \
-  ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch,  \
-      &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch,    \
-      &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes, \
-      &MakeFWBinary, &MakeBooleanBatch);
+#define BATCH_CASES()                                                                   \
+  ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
+                    &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,                  \
+                    &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion,               \
+                    &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes,           \
+                    &MakeFWBinary, &MakeBooleanBatch);
 
 static int g_file_number = 0;
 
 class IpcTestFixture : public io::MemoryMapFixture {
  public:
   Status DoStandardRoundTrip(const RecordBatch& batch, bool zero_data,
-      std::shared_ptr<RecordBatch>* batch_result) {
+                             std::shared_ptr<RecordBatch>* batch_result) {
     int32_t metadata_length;
     int64_t body_length;
 
     const int64_t buffer_offset = 0;
 
-    if (zero_data) { RETURN_NOT_OK(ZeroMemoryMap(mmap_.get())); }
+    if (zero_data) {
+      RETURN_NOT_OK(ZeroMemoryMap(mmap_.get()));
+    }
     RETURN_NOT_OK(mmap_->Seek(0));
 
-    RETURN_NOT_OK(WriteRecordBatch(
-        batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
+    RETURN_NOT_OK(WriteRecordBatch(batch, buffer_offset, mmap_.get(), &metadata_length,
+                                   &body_length, pool_));
 
     std::unique_ptr<Message> message;
     RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
 
     io::BufferReader buffer_reader(message->body());
-    return ReadRecordBatch(
-        *message->metadata(), batch.schema(), &buffer_reader, batch_result);
+    return ReadRecordBatch(*message->metadata(), batch.schema(), &buffer_reader,
+                           batch_result);
   }
 
-  Status DoLargeRoundTrip(
-      const RecordBatch& batch, bool zero_data, std::shared_ptr<RecordBatch>* result) {
-    if (zero_data) { RETURN_NOT_OK(ZeroMemoryMap(mmap_.get())); }
+  Status DoLargeRoundTrip(const RecordBatch& batch, bool zero_data,
+                          std::shared_ptr<RecordBatch>* result) {
+    if (zero_data) {
+      RETURN_NOT_OK(ZeroMemoryMap(mmap_.get()));
+    }
     RETURN_NOT_OK(mmap_->Seek(0));
 
     std::shared_ptr<RecordBatchFileWriter> file_writer;
@@ -244,8 +249,8 @@ TEST_F(TestIpcRoundTrip, MetadataVersion) {
 
   const int64_t buffer_offset = 0;
 
-  ASSERT_OK(WriteRecordBatch(
-      *batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
+  ASSERT_OK(WriteRecordBatch(*batch, buffer_offset, mmap_.get(), &metadata_length,
+                             &body_length, pool_));
 
   std::unique_ptr<Message> message;
   ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
@@ -258,7 +263,9 @@ TEST_P(TestIpcRoundTrip, SliceRoundTrip) {
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
 
   // Skip the zero-length case
-  if (batch->num_rows() < 2) { return; }
+  if (batch->num_rows() < 2) {
+    return;
+  }
 
   auto sliced_batch = batch->Slice(2, 10);
   CheckRoundtrip(*sliced_batch, 1 << 20);
@@ -282,8 +289,9 @@ TEST_P(TestIpcRoundTrip, ZeroLengthArrays) {
   ASSERT_OK(AllocateBuffer(pool_, sizeof(int32_t), &value_offsets));
   *reinterpret_cast<int32_t*>(value_offsets->mutable_data()) = 0;
 
-  std::shared_ptr<Array> bin_array = std::make_shared<BinaryArray>(0, value_offsets,
-      std::make_shared<Buffer>(nullptr, 0), std::make_shared<Buffer>(nullptr, 0));
+  std::shared_ptr<Array> bin_array = std::make_shared<BinaryArray>(
+      0, value_offsets, std::make_shared<Buffer>(nullptr, 0),
+      std::make_shared<Buffer>(nullptr, 0));
 
   // null value_offsets
   std::shared_ptr<Array> bin_array2 = std::make_shared<BinaryArray>(0, nullptr, nullptr);
@@ -357,8 +365,8 @@ TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) {
   std::shared_ptr<Buffer> offsets_buffer;
   ASSERT_OK(
       test::CopyBufferFromVector(type_offsets, default_memory_pool(), &offsets_buffer));
-  a1 = std::make_shared<UnionArray>(
-      dense_union_type, a0->length(), struct_children, ids_buffer, offsets_buffer);
+  a1 = std::make_shared<UnionArray>(dense_union_type, a0->length(), struct_children,
+                                    ids_buffer, offsets_buffer);
   CheckArray(a1);
 }
 
@@ -367,8 +375,8 @@ void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
   int32_t mock_metadata_length = -1;
   int64_t mock_body_length = -1;
   int64_t size = -1;
-  ASSERT_OK(WriteRecordBatch(
-      *batch, 0, &mock, &mock_metadata_length, &mock_body_length, default_memory_pool()));
+  ASSERT_OK(WriteRecordBatch(*batch, 0, &mock, &mock_metadata_length, &mock_body_length,
+                             default_memory_pool()));
   ASSERT_OK(GetRecordBatchSize(*batch, &size));
   ASSERT_EQ(mock.GetExtentBytesWritten(), size);
 }
@@ -398,8 +406,8 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
   void TearDown() { io::MemoryMapFixture::TearDown(); }
 
   Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
-      int64_t* body_length, std::shared_ptr<RecordBatch>* batch,
-      std::shared_ptr<Schema>* schema) {
+                     int64_t* body_length, std::shared_ptr<RecordBatch>* batch,
+                     std::shared_ptr<Schema>* schema) {
     const int batch_length = 5;
     TypePtr type = int32();
     std::shared_ptr<Array> array;
@@ -425,10 +433,10 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
 
     if (override_level) {
       return WriteRecordBatch(**batch, 0, mmap_.get(), metadata_length, body_length,
-          pool_, recursion_level + 1);
+                              pool_, recursion_level + 1);
     } else {
-      return WriteRecordBatch(
-          **batch, 0, mmap_.get(), metadata_length, body_length, pool_);
+      return WriteRecordBatch(**batch, 0, mmap_.get(), metadata_length, body_length,
+                              pool_);
     }
   }
 
@@ -442,8 +450,8 @@ TEST_F(RecursionLimits, WriteLimit) {
   int64_t body_length = -1;
   std::shared_ptr<Schema> schema;
   std::shared_ptr<RecordBatch> batch;
-  ASSERT_RAISES(Invalid,
-      WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &batch, &schema));
+  ASSERT_RAISES(Invalid, WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length,
+                                     &batch, &schema));
 }
 
 TEST_F(RecursionLimits, ReadLimit) {
@@ -454,8 +462,8 @@ TEST_F(RecursionLimits, ReadLimit) {
   const int recursion_depth = 64;
 
   std::shared_ptr<RecordBatch> batch;
-  ASSERT_OK(WriteToMmap(
-      recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
+  ASSERT_OK(WriteToMmap(recursion_depth, true, &metadata_length, &body_length, &batch,
+                        &schema));
 
   std::unique_ptr<Message> message;
   ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
@@ -472,16 +480,16 @@ TEST_F(RecursionLimits, StressLimit) {
     int64_t body_length = -1;
     std::shared_ptr<Schema> schema;
     std::shared_ptr<RecordBatch> batch;
-    ASSERT_OK(WriteToMmap(
-        recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
+    ASSERT_OK(WriteToMmap(recursion_depth, true, &metadata_length, &body_length, &batch,
+                          &schema));
 
     std::unique_ptr<Message> message;
     ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
 
     io::BufferReader reader(message->body());
     std::shared_ptr<RecordBatch> result;
-    ASSERT_OK(ReadRecordBatch(
-        *message->metadata(), schema, recursion_depth + 1, &reader, &result));
+    ASSERT_OK(ReadRecordBatch(*message->metadata(), schema, recursion_depth + 1, &reader,
+                              &result));
     *it_works = result->Equals(*batch);
   };
 
@@ -568,8 +576,8 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
   }
   void TearDown() {}
 
-  Status RoundTripHelper(
-      const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
+  Status RoundTripHelper(const RecordBatch& batch,
+                         std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
     // Write the file
     std::shared_ptr<RecordBatchStreamWriter> writer;
     RETURN_NOT_OK(RecordBatchStreamWriter::Open(sink_.get(), batch.schema(), &writer));
@@ -589,7 +597,9 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
     std::shared_ptr<RecordBatch> chunk;
     while (true) {
       RETURN_NOT_OK(reader->ReadNextRecordBatch(&chunk));
-      if (chunk == nullptr) { break; }
+      if (chunk == nullptr) {
+        break;
+      }
       out_batches->emplace_back(chunk);
     }
     return Status::OK();
@@ -747,8 +757,8 @@ TEST_F(TestTensorRoundTrip, NonContiguous) {
   int32_t metadata_length;
   int64_t body_length;
   ASSERT_OK(mmap_->Seek(0));
-  ASSERT_RAISES(
-      Invalid, WriteTensor(tensor, mmap_.get(), &metadata_length, &body_length));
+  ASSERT_RAISES(Invalid,
+                WriteTensor(tensor, mmap_.get(), &metadata_length, &body_length));
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 18f5dfaf57098..035f7086e7e53 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -40,7 +40,8 @@
 
 DEFINE_string(arrow, "", "Arrow file name");
 DEFINE_string(json, "", "JSON file name");
-DEFINE_string(mode, "VALIDATE",
+DEFINE_string(
+    mode, "VALIDATE",
     "Mode of integration testing tool (ARROW_TO_JSON, JSON_TO_ARROW, VALIDATE)");
 DEFINE_bool(integration, false, "Run in integration test mode");
 DEFINE_bool(verbose, true, "Verbose output");
@@ -55,8 +56,8 @@ bool file_exists(const char* path) {
 }
 
 // Convert JSON file to IPC binary format
-static Status ConvertJsonToArrow(
-    const std::string& json_path, const std::string& arrow_path) {
+static Status ConvertJsonToArrow(const std::string& json_path,
+                                 const std::string& arrow_path) {
   std::shared_ptr<io::ReadableFile> in_file;
   std::shared_ptr<io::FileOutputStream> out_file;
 
@@ -89,8 +90,8 @@ static Status ConvertJsonToArrow(
 }
 
 // Convert IPC binary format to JSON
-static Status ConvertArrowToJson(
-    const std::string& arrow_path, const std::string& json_path) {
+static Status ConvertArrowToJson(const std::string& arrow_path,
+                                 const std::string& json_path) {
   std::shared_ptr<io::ReadableFile> in_file;
   std::shared_ptr<io::FileOutputStream> out_file;
 
@@ -116,11 +117,11 @@ static Status ConvertArrowToJson(
   std::string result;
   RETURN_NOT_OK(writer->Finish(&result));
   return out_file->Write(reinterpret_cast<const uint8_t*>(result.c_str()),
-      static_cast<int64_t>(result.size()));
+                         static_cast<int64_t>(result.size()));
 }
 
-static Status ValidateArrowVsJson(
-    const std::string& arrow_path, const std::string& json_path) {
+static Status ValidateArrowVsJson(const std::string& arrow_path,
+                                  const std::string& json_path) {
   // Construct JSON reader
   std::shared_ptr<io::ReadableFile> json_file;
   RETURN_NOT_OK(io::ReadableFile::Open(json_path, &json_file));
@@ -151,7 +152,9 @@ static Status ValidateArrowVsJson(
        << "Arrow schema: \n"
        << arrow_schema->ToString();
 
-    if (FLAGS_verbose) { std::cout << ss.str() << std::endl; }
+    if (FLAGS_verbose) {
+      std::cout << ss.str() << std::endl;
+    }
     return Status::Invalid("Schemas did not match");
   }
 
@@ -188,10 +191,14 @@ static Status ValidateArrowVsJson(
 }
 
 Status RunCommand(const std::string& json_path, const std::string& arrow_path,
-    const std::string& command) {
-  if (json_path == "") { return Status::Invalid("Must specify json file name"); }
+                  const std::string& command) {
+  if (json_path == "") {
+    return Status::Invalid("Must specify json file name");
+  }
 
-  if (arrow_path == "") { return Status::Invalid("Must specify arrow file name"); }
+  if (arrow_path == "") {
+    return Status::Invalid("Must specify arrow file name");
+  }
 
   if (command == "ARROW_TO_JSON") {
     if (!file_exists(arrow_path.c_str())) {
@@ -240,8 +247,8 @@ class TestJSONIntegration : public ::testing::Test {
     do {
       std::shared_ptr<io::FileOutputStream> out;
       RETURN_NOT_OK(io::FileOutputStream::Open(path, &out));
-      RETURN_NOT_OK(out->Write(
-          reinterpret_cast<const uint8_t*>(data), static_cast<int64_t>(strlen(data))));
+      RETURN_NOT_OK(out->Write(reinterpret_cast<const uint8_t*>(data),
+                               static_cast<int64_t>(strlen(data))));
     } while (0);
     return Status::OK();
   }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 69e4ae8d14a04..175d75b7d1e97 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -199,7 +199,7 @@ class SchemaWriter {
   typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
                               std::is_base_of<ListType, T>::value ||
                               std::is_base_of<StructType, T>::value,
-      void>::type
+                          void>::type
   WriteTypeMetadata(const T& type) {}
 
   void WriteTypeMetadata(const Integer& type) {
@@ -508,7 +508,7 @@ class ArrayWriter {
   }
 
   Status WriteChildren(const std::vector<std::shared_ptr<Field>>& fields,
-      const std::vector<std::shared_ptr<Array>>& arrays) {
+                       const std::vector<std::shared_ptr<Array>>& arrays) {
     writer_->Key("children");
     writer_->StartArray();
     for (size_t i = 0; i < fields.size(); ++i) {
@@ -602,16 +602,16 @@ static Status GetObjectBool(const RjObject& obj, const std::string& key, bool* o
   return Status::OK();
 }
 
-static Status GetObjectString(
-    const RjObject& obj, const std::string& key, std::string* out) {
+static Status GetObjectString(const RjObject& obj, const std::string& key,
+                              std::string* out) {
   const auto& it = obj.FindMember(key);
   RETURN_NOT_STRING(key, it, obj);
   *out = it->value.GetString();
   return Status::OK();
 }
 
-static Status GetInteger(
-    const rj::Value::ConstObject& json_type, std::shared_ptr<DataType>* type) {
+static Status GetInteger(const rj::Value::ConstObject& json_type,
+                         std::shared_ptr<DataType>* type) {
   const auto& it_bit_width = json_type.FindMember("bitWidth");
   RETURN_NOT_INT("bitWidth", it_bit_width, json_type);
 
@@ -642,8 +642,8 @@ static Status GetInteger(
   return Status::OK();
 }
 
-static Status GetFloatingPoint(
-    const RjObject& json_type, std::shared_ptr<DataType>* type) {
+static Status GetFloatingPoint(const RjObject& json_type,
+                               std::shared_ptr<DataType>* type) {
   const auto& it_precision = json_type.FindMember("precision");
   RETURN_NOT_STRING("precision", it_precision, json_type);
 
@@ -663,8 +663,8 @@ static Status GetFloatingPoint(
   return Status::OK();
 }
 
-static Status GetFixedSizeBinary(
-    const RjObject& json_type, std::shared_ptr<DataType>* type) {
+static Status GetFixedSizeBinary(const RjObject& json_type,
+                                 std::shared_ptr<DataType>* type) {
   const auto& it_byte_width = json_type.FindMember("byteWidth");
   RETURN_NOT_INT("byteWidth", it_byte_width, json_type);
 
@@ -756,8 +756,8 @@ static Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>*
 }
 
 static Status GetUnion(const RjObject& json_type,
-    const std::vector<std::shared_ptr<Field>>& children,
-    std::shared_ptr<DataType>* type) {
+                       const std::vector<std::shared_ptr<Field>>& children,
+                       std::shared_ptr<DataType>* type) {
   const auto& it_mode = json_type.FindMember("mode");
   RETURN_NOT_STRING("mode", it_mode, json_type);
 
@@ -790,8 +790,8 @@ static Status GetUnion(const RjObject& json_type,
 }
 
 static Status GetType(const RjObject& json_type,
-    const std::vector<std::shared_ptr<Field>>& children,
-    std::shared_ptr<DataType>* type) {
+                      const std::vector<std::shared_ptr<Field>>& children,
+                      std::shared_ptr<DataType>* type) {
   const auto& it_type_name = json_type.FindMember("name");
   RETURN_NOT_STRING("name", it_type_name, json_type);
 
@@ -831,10 +831,11 @@ static Status GetType(const RjObject& json_type,
 }
 
 static Status GetField(const rj::Value& obj, const DictionaryMemo* dictionary_memo,
-    std::shared_ptr<Field>* field);
+                       std::shared_ptr<Field>* field);
 
 static Status GetFieldsFromArray(const rj::Value& obj,
-    const DictionaryMemo* dictionary_memo, std::vector<std::shared_ptr<Field>>* fields) {
+                                 const DictionaryMemo* dictionary_memo,
+                                 std::vector<std::shared_ptr<Field>>* fields) {
   const auto& values = obj.GetArray();
 
   fields->resize(values.Size());
@@ -845,7 +846,7 @@ static Status GetFieldsFromArray(const rj::Value& obj,
 }
 
 static Status ParseDictionary(const RjObject& obj, int64_t* id, bool* is_ordered,
-    std::shared_ptr<DataType>* index_type) {
+                              std::shared_ptr<DataType>* index_type) {
   int32_t int32_id;
   RETURN_NOT_OK(GetObjectInt(obj, "id", &int32_id));
   *id = int32_id;
@@ -866,8 +867,10 @@ static Status ParseDictionary(const RjObject& obj, int64_t* id, bool* is_ordered
 }
 
 static Status GetField(const rj::Value& obj, const DictionaryMemo* dictionary_memo,
-    std::shared_ptr<Field>* field) {
-  if (!obj.IsObject()) { return Status::Invalid("Field was not a JSON object"); }
+                       std::shared_ptr<Field>* field) {
+  if (!obj.IsObject()) {
+    return Status::Invalid("Field was not a JSON object");
+  }
   const auto& json_field = obj.GetObject();
 
   std::string name;
@@ -884,8 +887,8 @@ static Status GetField(const rj::Value& obj, const DictionaryMemo* dictionary_me
     int64_t dictionary_id;
     bool is_ordered;
     std::shared_ptr<DataType> index_type;
-    RETURN_NOT_OK(ParseDictionary(
-        it_dictionary->value.GetObject(), &dictionary_id, &is_ordered, &index_type));
+    RETURN_NOT_OK(ParseDictionary(it_dictionary->value.GetObject(), &dictionary_id,
+                                  &is_ordered, &index_type));
 
     std::shared_ptr<Array> dictionary;
     RETURN_NOT_OK(dictionary_memo->GetDictionary(dictionary_id, &dictionary));
@@ -941,13 +944,13 @@ UnboxValue(const rj::Value& val) {
 class ArrayReader {
  public:
   explicit ArrayReader(const rj::Value& json_array, const std::shared_ptr<DataType>& type,
-      MemoryPool* pool)
+                       MemoryPool* pool)
       : json_array_(json_array), type_(type), pool_(pool) {}
 
   Status ParseTypeValues(const DataType& type);
 
   Status GetValidityBuffer(const std::vector<bool>& is_valid, int32_t* null_count,
-      std::shared_ptr<Buffer>* validity_buffer) {
+                           std::shared_ptr<Buffer>* validity_buffer) {
     int length = static_cast<int>(is_valid.size());
 
     std::shared_ptr<MutableBuffer> out_buffer;
@@ -1024,7 +1027,9 @@ class ArrayReader {
         DCHECK(hex_string.size() % 2 == 0) << "Expected base16 hex string";
         int32_t length = static_cast<int>(hex_string.size()) / 2;
 
-        if (byte_buffer->size() < length) { RETURN_NOT_OK(byte_buffer->Resize(length)); }
+        if (byte_buffer->size() < length) {
+          RETURN_NOT_OK(byte_buffer->Resize(length));
+        }
 
         const char* hex_data = hex_string.c_str();
         uint8_t* byte_buffer_data = byte_buffer->mutable_data();
@@ -1078,8 +1083,8 @@ class ArrayReader {
   }
 
   template <typename T>
-  Status GetIntArray(
-      const RjArray& json_array, const int32_t length, std::shared_ptr<Buffer>* out) {
+  Status GetIntArray(const RjArray& json_array, const int32_t length,
+                     std::shared_ptr<Buffer>* out) {
     std::shared_ptr<MutableBuffer> buffer;
     RETURN_NOT_OK(AllocateBuffer(pool_, length * sizeof(T), &buffer));
 
@@ -1102,15 +1107,15 @@ class ArrayReader {
     const auto& json_offsets = obj_->FindMember("OFFSET");
     RETURN_NOT_ARRAY("OFFSET", json_offsets, *obj_);
     std::shared_ptr<Buffer> offsets_buffer;
-    RETURN_NOT_OK(GetIntArray<int32_t>(
-        json_offsets->value.GetArray(), length_ + 1, &offsets_buffer));
+    RETURN_NOT_OK(GetIntArray<int32_t>(json_offsets->value.GetArray(), length_ + 1,
+                                       &offsets_buffer));
 
     std::vector<std::shared_ptr<Array>> children;
     RETURN_NOT_OK(GetChildren(*obj_, type, &children));
     DCHECK_EQ(children.size(), 1);
 
-    result_ = std::make_shared<ListArray>(
-        type_, length_, offsets_buffer, children[0], validity_buffer, null_count);
+    result_ = std::make_shared<ListArray>(type_, length_, offsets_buffer, children[0],
+                                          validity_buffer, null_count);
 
     return Status::OK();
   }
@@ -1123,8 +1128,8 @@ class ArrayReader {
     std::vector<std::shared_ptr<Array>> fields;
     RETURN_NOT_OK(GetChildren(*obj_, type, &fields));
 
-    result_ = std::make_shared<StructArray>(
-        type_, length_, fields, validity_buffer, null_count);
+    result_ = std::make_shared<StructArray>(type_, length_, fields, validity_buffer,
+                                            null_count);
 
     return Status::OK();
   }
@@ -1154,7 +1159,7 @@ class ArrayReader {
     RETURN_NOT_OK(GetChildren(*obj_, type, &children));
 
     result_ = std::make_shared<UnionArray>(type_, length_, children, type_id_buffer,
-        offsets_buffer, validity_buffer, null_count);
+                                           offsets_buffer, validity_buffer, null_count);
 
     return Status::OK();
   }
@@ -1177,7 +1182,7 @@ class ArrayReader {
   }
 
   Status GetChildren(const RjObject& obj, const DataType& type,
-      std::vector<std::shared_ptr<Array>>* array) {
+                     std::vector<std::shared_ptr<Array>>* array) {
     const auto& json_children = obj.FindMember("children");
     RETURN_NOT_ARRAY("children", json_children, obj);
     const auto& json_children_arr = json_children->value.GetArray();
@@ -1280,7 +1285,8 @@ static Status GetDictionaryTypes(const RjArray& fields, DictionaryTypeMap* id_to
 }
 
 static Status ReadDictionary(const RjObject& obj, const DictionaryTypeMap& id_to_field,
-    MemoryPool* pool, int64_t* dictionary_id, std::shared_ptr<Array>* out) {
+                             MemoryPool* pool, int64_t* dictionary_id,
+                             std::shared_ptr<Array>* out) {
   int id;
   RETURN_NOT_OK(GetObjectInt(obj, "id", &id));
 
@@ -1312,7 +1318,7 @@ static Status ReadDictionary(const RjObject& obj, const DictionaryTypeMap& id_to
 }
 
 static Status ReadDictionaries(const rj::Value& doc, const DictionaryTypeMap& id_to_field,
-    MemoryPool* pool, DictionaryMemo* dictionary_memo) {
+                               MemoryPool* pool, DictionaryMemo* dictionary_memo) {
   auto it = doc.FindMember("dictionaries");
   if (it == doc.MemberEnd()) {
     // No dictionaries
@@ -1334,8 +1340,8 @@ static Status ReadDictionaries(const rj::Value& doc, const DictionaryTypeMap& id
   return Status::OK();
 }
 
-Status ReadSchema(
-    const rj::Value& json_schema, MemoryPool* pool, std::shared_ptr<Schema>* schema) {
+Status ReadSchema(const rj::Value& json_schema, MemoryPool* pool,
+                  std::shared_ptr<Schema>* schema) {
   auto it = json_schema.FindMember("schema");
   RETURN_NOT_OBJECT("schema", it, json_schema);
   const auto& obj_schema = it->value.GetObject();
@@ -1359,7 +1365,7 @@ Status ReadSchema(
 }
 
 Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr<Schema>& schema,
-    MemoryPool* pool, std::shared_ptr<RecordBatch>* batch) {
+                       MemoryPool* pool, std::shared_ptr<RecordBatch>* batch) {
   DCHECK(json_obj.IsObject());
   const auto& batch_obj = json_obj.GetObject();
 
@@ -1409,14 +1415,16 @@ Status WriteArray(const std::string& name, const Array& array, RjWriter* json_wr
 }
 
 Status ReadArray(MemoryPool* pool, const rj::Value& json_array,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
+                 const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array) {
   ArrayReader converter(json_array, type, pool);
   return converter.GetArray(array);
 }
 
 Status ReadArray(MemoryPool* pool, const rj::Value& json_array, const Schema& schema,
-    std::shared_ptr<Array>* array) {
-  if (!json_array.IsObject()) { return Status::Invalid("Element was not a JSON object"); }
+                 std::shared_ptr<Array>* array) {
+  if (!json_array.IsObject()) {
+    return Status::Invalid("Element was not a JSON object");
+  }
 
   const auto& json_obj = json_array.GetObject();
 
diff --git a/cpp/src/arrow/ipc/json-internal.h b/cpp/src/arrow/ipc/json-internal.h
index 5571d9233969c..9b641cd53329b 100644
--- a/cpp/src/arrow/ipc/json-internal.h
+++ b/cpp/src/arrow/ipc/json-internal.h
@@ -99,17 +99,17 @@ Status WriteSchema(const Schema& schema, RjWriter* writer);
 Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer);
 Status WriteArray(const std::string& name, const Array& array, RjWriter* writer);
 
-Status ReadSchema(
-    const rj::Value& json_obj, MemoryPool* pool, std::shared_ptr<Schema>* schema);
+Status ReadSchema(const rj::Value& json_obj, MemoryPool* pool,
+                  std::shared_ptr<Schema>* schema);
 
 Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr<Schema>& schema,
-    MemoryPool* pool, std::shared_ptr<RecordBatch>* batch);
+                       MemoryPool* pool, std::shared_ptr<RecordBatch>* batch);
 
 Status ReadArray(MemoryPool* pool, const rj::Value& json_obj,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array);
+                 const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array);
 
 Status ReadArray(MemoryPool* pool, const rj::Value& json_obj, const Schema& schema,
-    std::shared_ptr<Array>* array);
+                 std::shared_ptr<Array>* array);
 
 }  // namespace internal
 }  // namespace json
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index 36e343e5fb5bc..f57101a31a97d 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -79,15 +79,13 @@ JsonWriter::JsonWriter(const std::shared_ptr<Schema>& schema) {
 
 JsonWriter::~JsonWriter() {}
 
-Status JsonWriter::Open(
-    const std::shared_ptr<Schema>& schema, std::unique_ptr<JsonWriter>* writer) {
+Status JsonWriter::Open(const std::shared_ptr<Schema>& schema,
+                        std::unique_ptr<JsonWriter>* writer) {
   *writer = std::unique_ptr<JsonWriter>(new JsonWriter(schema));
   return (*writer)->impl_->Start();
 }
 
-Status JsonWriter::Finish(std::string* result) {
-  return impl_->Finish(result);
-}
+Status JsonWriter::Finish(std::string* result) { return impl_->Finish(result); }
 
 Status JsonWriter::WriteRecordBatch(const RecordBatch& batch) {
   return impl_->WriteRecordBatch(batch);
@@ -103,8 +101,10 @@ class JsonReader::JsonReaderImpl {
 
   Status ParseAndReadSchema() {
     doc_.Parse(reinterpret_cast<const rj::Document::Ch*>(data_->data()),
-        static_cast<size_t>(data_->size()));
-    if (doc_.HasParseError()) { return Status::IOError("JSON parsing failed"); }
+               static_cast<size_t>(data_->size()));
+    if (doc_.HasParseError()) {
+      return Status::IOError("JSON parsing failed");
+    }
 
     RETURN_NOT_OK(json::internal::ReadSchema(doc_, pool_, &schema_));
 
@@ -120,8 +120,8 @@ class JsonReader::JsonReaderImpl {
     DCHECK_LT(i, static_cast<int>(record_batches_->GetArray().Size()))
         << "i out of bounds";
 
-    return json::internal::ReadRecordBatch(
-        record_batches_->GetArray()[i], schema_, pool_, batch);
+    return json::internal::ReadRecordBatch(record_batches_->GetArray()[i], schema_, pool_,
+                                           batch);
   }
 
   std::shared_ptr<Schema> schema() const { return schema_; }
@@ -145,24 +145,20 @@ JsonReader::JsonReader(MemoryPool* pool, const std::shared_ptr<Buffer>& data) {
 
 JsonReader::~JsonReader() {}
 
-Status JsonReader::Open(
-    const std::shared_ptr<Buffer>& data, std::unique_ptr<JsonReader>* reader) {
+Status JsonReader::Open(const std::shared_ptr<Buffer>& data,
+                        std::unique_ptr<JsonReader>* reader) {
   return Open(default_memory_pool(), data, reader);
 }
 
 Status JsonReader::Open(MemoryPool* pool, const std::shared_ptr<Buffer>& data,
-    std::unique_ptr<JsonReader>* reader) {
+                        std::unique_ptr<JsonReader>* reader) {
   *reader = std::unique_ptr<JsonReader>(new JsonReader(pool, data));
   return (*reader)->impl_->ParseAndReadSchema();
 }
 
-std::shared_ptr<Schema> JsonReader::schema() const {
-  return impl_->schema();
-}
+std::shared_ptr<Schema> JsonReader::schema() const { return impl_->schema(); }
 
-int JsonReader::num_record_batches() const {
-  return impl_->num_record_batches();
-}
+int JsonReader::num_record_batches() const { return impl_->num_record_batches(); }
 
 Status JsonReader::ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const {
   return impl_->ReadRecordBatch(i, batch);
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
index 2ba27c7f2c37d..be26f0233ebeb 100644
--- a/cpp/src/arrow/ipc/json.h
+++ b/cpp/src/arrow/ipc/json.h
@@ -41,8 +41,8 @@ class ARROW_EXPORT JsonWriter {
  public:
   ~JsonWriter();
 
-  static Status Open(
-      const std::shared_ptr<Schema>& schema, std::unique_ptr<JsonWriter>* out);
+  static Status Open(const std::shared_ptr<Schema>& schema,
+                     std::unique_ptr<JsonWriter>* out);
 
   Status WriteRecordBatch(const RecordBatch& batch);
   Status Finish(std::string* result);
@@ -61,11 +61,11 @@ class ARROW_EXPORT JsonReader {
   ~JsonReader();
 
   static Status Open(MemoryPool* pool, const std::shared_ptr<Buffer>& data,
-      std::unique_ptr<JsonReader>* reader);
+                     std::unique_ptr<JsonReader>* reader);
 
   // Use the default memory pool
-  static Status Open(
-      const std::shared_ptr<Buffer>& data, std::unique_ptr<JsonReader>* reader);
+  static Status Open(const std::shared_ptr<Buffer>& data,
+                     std::unique_ptr<JsonReader>* reader);
 
   std::shared_ptr<Schema> schema() const;
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 49c24c72727b7..20fd280db6de6 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -58,8 +58,8 @@ static constexpr flatbuf::MetadataVersion kCurrentMetadataVersion =
 static constexpr flatbuf::MetadataVersion kMinMetadataVersion =
     flatbuf::MetadataVersion_V3;
 
-static Status IntFromFlatbuffer(
-    const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
+static Status IntFromFlatbuffer(const flatbuf::Int* int_data,
+                                std::shared_ptr<DataType>* out) {
   if (int_data->bitWidth() > 64) {
     return Status::NotImplemented("Integers with more than 64 bits not implemented");
   }
@@ -86,8 +86,8 @@ static Status IntFromFlatbuffer(
   return Status::OK();
 }
 
-static Status FloatFromFlatuffer(
-    const flatbuf::FloatingPoint* float_data, std::shared_ptr<DataType>* out) {
+static Status FloatFromFlatuffer(const flatbuf::FloatingPoint* float_data,
+                                 std::shared_ptr<DataType>* out) {
   if (float_data->precision() == flatbuf::Precision_HALF) {
     *out = float16();
   } else if (float_data->precision() == flatbuf::Precision_SINGLE) {
@@ -100,7 +100,7 @@ static Status FloatFromFlatuffer(
 
 // Forward declaration
 static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
-    DictionaryMemo* dictionary_memo, FieldOffset* offset);
+                                DictionaryMemo* dictionary_memo, FieldOffset* offset);
 
 static Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
   return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
@@ -111,7 +111,8 @@ static Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
 }
 
 static Status AppendChildFields(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo) {
+                                std::vector<FieldOffset>* out_children,
+                                DictionaryMemo* dictionary_memo) {
   FieldOffset field;
   for (int i = 0; i < type->num_children(); ++i) {
     RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), dictionary_memo, &field));
@@ -121,16 +122,16 @@ static Status AppendChildFields(FBB& fbb, const std::shared_ptr<DataType>& type,
 }
 
 static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
-    Offset* offset) {
+                               std::vector<FieldOffset>* out_children,
+                               DictionaryMemo* dictionary_memo, Offset* offset) {
   RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
   *offset = flatbuf::CreateList(fbb).Union();
   return Status::OK();
 }
 
 static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
-    Offset* offset) {
+                                 std::vector<FieldOffset>* out_children,
+                                 DictionaryMemo* dictionary_memo, Offset* offset) {
   RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
   *offset = flatbuf::CreateStruct_(fbb).Union();
   return Status::OK();
@@ -140,7 +141,8 @@ static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type
 // Union implementation
 
 static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
-    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
+                                  const std::vector<std::shared_ptr<Field>>& children,
+                                  std::shared_ptr<DataType>* out) {
   UnionMode mode = union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE
                                                                    : UnionMode::DENSE;
 
@@ -163,8 +165,8 @@ static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
 }
 
 static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
-    Offset* offset) {
+                                std::vector<FieldOffset>* out_children,
+                                DictionaryMemo* dictionary_memo, Offset* offset) {
   RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
 
   const auto& union_type = static_cast<const UnionType&>(*type);
@@ -224,15 +226,16 @@ static inline TimeUnit::type FromFlatbufferUnit(flatbuf::TimeUnit unit) {
 }
 
 static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
-    const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
+                                 const std::vector<std::shared_ptr<Field>>& children,
+                                 std::shared_ptr<DataType>* out) {
   switch (type) {
     case flatbuf::Type_NONE:
       return Status::Invalid("Type metadata cannot be none");
     case flatbuf::Type_Int:
       return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
     case flatbuf::Type_FloatingPoint:
-      return FloatFromFlatuffer(
-          static_cast<const flatbuf::FloatingPoint*>(type_data), out);
+      return FloatFromFlatuffer(static_cast<const flatbuf::FloatingPoint*>(type_data),
+                                out);
     case flatbuf::Type_Binary:
       *out = binary();
       return Status::OK();
@@ -301,8 +304,8 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
       *out = std::make_shared<StructType>(children);
       return Status::OK();
     case flatbuf::Type_Union:
-      return UnionFromFlatbuffer(
-          static_cast<const flatbuf::Union*>(type_data), children, out);
+      return UnionFromFlatbuffer(static_cast<const flatbuf::Union*>(type_data), children,
+                                 out);
     default:
       return Status::Invalid("Unrecognized type");
   }
@@ -310,15 +313,17 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 
 // TODO(wesm): Convert this to visitor pattern
 static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
-    flatbuf::Type* out_type, DictionaryMemo* dictionary_memo, Offset* offset) {
+                               std::vector<FieldOffset>* children,
+                               std::vector<VectorLayoutOffset>* layout,
+                               flatbuf::Type* out_type, DictionaryMemo* dictionary_memo,
+                               Offset* offset) {
   if (type->id() == Type::DICTIONARY) {
     // In this library, the dictionary "type" is a logical construct. Here we
     // pass through to the value type, as we've already captured the index
     // type in the DictionaryEncoding metadata in the parent field
     const auto& dict_type = static_cast<const DictionaryType&>(*type);
     return TypeToFlatbuffer(fbb, dict_type.dictionary()->type(), children, layout,
-        out_type, dictionary_memo, offset);
+                            out_type, dictionary_memo, offset);
   }
 
   std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
@@ -436,7 +441,7 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
 }
 
 static Status TensorTypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    flatbuf::Type* out_type, Offset* offset) {
+                                     flatbuf::Type* out_type, Offset* offset) {
   switch (type->id()) {
     case Type::UINT8:
       INT_TO_FB_CASE(8, false);
@@ -475,8 +480,8 @@ static Status TensorTypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>&
   return Status::OK();
 }
 
-static DictionaryOffset GetDictionaryEncoding(
-    FBB& fbb, const DictionaryType& type, DictionaryMemo* memo) {
+static DictionaryOffset GetDictionaryEncoding(FBB& fbb, const DictionaryType& type,
+                                              DictionaryMemo* memo) {
   int64_t dictionary_id = memo->GetId(type.dictionary());
 
   // We assume that the dictionary index type (as an integer) has already been
@@ -491,7 +496,7 @@ static DictionaryOffset GetDictionaryEncoding(
 }
 
 static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
-    DictionaryMemo* dictionary_memo, FieldOffset* offset) {
+                                DictionaryMemo* dictionary_memo, FieldOffset* offset) {
   auto fb_name = fbb.CreateString(field->name());
 
   flatbuf::Type type_enum;
@@ -500,8 +505,8 @@ static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
   std::vector<FieldOffset> children;
   std::vector<VectorLayoutOffset> layout;
 
-  RETURN_NOT_OK(TypeToFlatbuffer(
-      fbb, field->type(), &children, &layout, &type_enum, dictionary_memo, &type_offset));
+  RETURN_NOT_OK(TypeToFlatbuffer(fbb, field->type(), &children, &layout, &type_enum,
+                                 dictionary_memo, &type_offset));
   auto fb_children = fbb.CreateVector(children);
   auto fb_layout = fbb.CreateVector(layout);
 
@@ -513,13 +518,14 @@ static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
 
   // TODO: produce the list of VectorTypes
   *offset = flatbuf::CreateField(fbb, fb_name, field->nullable(), type_enum, type_offset,
-      dictionary, fb_children, fb_layout);
+                                 dictionary, fb_children, fb_layout);
 
   return Status::OK();
 }
 
 static Status FieldFromFlatbuffer(const flatbuf::Field* field,
-    const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out) {
+                                  const DictionaryMemo& dictionary_memo,
+                                  std::shared_ptr<Field>* out) {
   std::shared_ptr<DataType> type;
 
   const flatbuf::DictionaryEncoding* encoding = field->dictionary();
@@ -551,8 +557,8 @@ static Status FieldFromFlatbuffer(const flatbuf::Field* field,
   return Status::OK();
 }
 
-static Status FieldFromFlatbufferDictionary(
-    const flatbuf::Field* field, std::shared_ptr<Field>* out) {
+static Status FieldFromFlatbufferDictionary(const flatbuf::Field* field,
+                                            std::shared_ptr<Field>* out) {
   // Need an empty memo to pass down for constructing children
   DictionaryMemo dummy_memo;
 
@@ -584,7 +590,8 @@ flatbuf::Endianness endianness() {
 }
 
 static Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
-    DictionaryMemo* dictionary_memo, flatbuffers::Offset<flatbuf::Schema>* out) {
+                                 DictionaryMemo* dictionary_memo,
+                                 flatbuffers::Offset<flatbuf::Schema>* out) {
   /// Fields
   std::vector<FieldOffset> field_offsets;
   for (int i = 0; i < schema.num_fields(); ++i) {
@@ -609,8 +616,8 @@ static Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
       key_value_offsets.push_back(
           flatbuf::CreateKeyValue(fbb, fbb.CreateString(key), fbb.CreateString(value)));
     }
-    *out = flatbuf::CreateSchema(
-        fbb, endianness(), fb_offsets, fbb.CreateVector(key_value_offsets));
+    *out = flatbuf::CreateSchema(fbb, endianness(), fb_offsets,
+                                 fbb.CreateVector(key_value_offsets));
   } else {
     *out = flatbuf::CreateSchema(fbb, endianness(), fb_offsets);
   }
@@ -631,15 +638,16 @@ static Status WriteFlatbufferBuilder(FBB& fbb, std::shared_ptr<Buffer>* out) {
 }
 
 static Status WriteFBMessage(FBB& fbb, flatbuf::MessageHeader header_type,
-    flatbuffers::Offset<void> header, int64_t body_length, std::shared_ptr<Buffer>* out) {
-  auto message = flatbuf::CreateMessage(
-      fbb, kCurrentMetadataVersion, header_type, header, body_length);
+                             flatbuffers::Offset<void> header, int64_t body_length,
+                             std::shared_ptr<Buffer>* out) {
+  auto message = flatbuf::CreateMessage(fbb, kCurrentMetadataVersion, header_type, header,
+                                        body_length);
   fbb.Finish(message);
   return WriteFlatbufferBuilder(fbb, out);
 }
 
-Status WriteSchemaMessage(
-    const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out) {
+Status WriteSchemaMessage(const Schema& schema, DictionaryMemo* dictionary_memo,
+                          std::shared_ptr<Buffer>* out) {
   FBB fbb;
   flatbuffers::Offset<flatbuf::Schema> fb_schema;
   RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
@@ -650,8 +658,8 @@ using FieldNodeVector =
     flatbuffers::Offset<flatbuffers::Vector<const flatbuf::FieldNode*>>;
 using BufferVector = flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Buffer*>>;
 
-static Status WriteFieldNodes(
-    FBB& fbb, const std::vector<FieldMetadata>& nodes, FieldNodeVector* out) {
+static Status WriteFieldNodes(FBB& fbb, const std::vector<FieldMetadata>& nodes,
+                              FieldNodeVector* out) {
   std::vector<flatbuf::FieldNode> fb_nodes;
   fb_nodes.reserve(nodes.size());
 
@@ -666,8 +674,8 @@ static Status WriteFieldNodes(
   return Status::OK();
 }
 
-static Status WriteBuffers(
-    FBB& fbb, const std::vector<BufferMetadata>& buffers, BufferVector* out) {
+static Status WriteBuffers(FBB& fbb, const std::vector<BufferMetadata>& buffers,
+                           BufferVector* out) {
   std::vector<flatbuf::Buffer> fb_buffers;
   fb_buffers.reserve(buffers.size());
 
@@ -680,8 +688,9 @@ static Status WriteBuffers(
 }
 
 static Status MakeRecordBatch(FBB& fbb, int64_t length, int64_t body_length,
-    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
-    RecordBatchOffset* offset) {
+                              const std::vector<FieldMetadata>& nodes,
+                              const std::vector<BufferMetadata>& buffers,
+                              RecordBatchOffset* offset) {
   FieldNodeVector fb_nodes;
   BufferVector fb_buffers;
 
@@ -693,17 +702,18 @@ static Status MakeRecordBatch(FBB& fbb, int64_t length, int64_t body_length,
 }
 
 Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
-    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
-    std::shared_ptr<Buffer>* out) {
+                               const std::vector<FieldMetadata>& nodes,
+                               const std::vector<BufferMetadata>& buffers,
+                               std::shared_ptr<Buffer>* out) {
   FBB fbb;
   RecordBatchOffset record_batch;
   RETURN_NOT_OK(MakeRecordBatch(fbb, length, body_length, nodes, buffers, &record_batch));
-  return WriteFBMessage(
-      fbb, flatbuf::MessageHeader_RecordBatch, record_batch.Union(), body_length, out);
+  return WriteFBMessage(fbb, flatbuf::MessageHeader_RecordBatch, record_batch.Union(),
+                        body_length, out);
 }
 
-Status WriteTensorMessage(
-    const Tensor& tensor, int64_t buffer_start_offset, std::shared_ptr<Buffer>* out) {
+Status WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset,
+                          std::shared_ptr<Buffer>* out) {
   using TensorDimOffset = flatbuffers::Offset<flatbuf::TensorDim>;
   using TensorOffset = flatbuffers::Offset<flatbuf::Tensor>;
 
@@ -727,19 +737,20 @@ Status WriteTensorMessage(
   TensorOffset fb_tensor =
       flatbuf::CreateTensor(fbb, fb_type_type, fb_type, fb_shape, fb_strides, &buffer);
 
-  return WriteFBMessage(
-      fbb, flatbuf::MessageHeader_Tensor, fb_tensor.Union(), body_length, out);
+  return WriteFBMessage(fbb, flatbuf::MessageHeader_Tensor, fb_tensor.Union(),
+                        body_length, out);
 }
 
 Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
-    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
-    std::shared_ptr<Buffer>* out) {
+                              const std::vector<FieldMetadata>& nodes,
+                              const std::vector<BufferMetadata>& buffers,
+                              std::shared_ptr<Buffer>* out) {
   FBB fbb;
   RecordBatchOffset record_batch;
   RETURN_NOT_OK(MakeRecordBatch(fbb, length, body_length, nodes, buffers, &record_batch));
   auto dictionary_batch = flatbuf::CreateDictionaryBatch(fbb, id, record_batch).Union();
-  return WriteFBMessage(
-      fbb, flatbuf::MessageHeader_DictionaryBatch, dictionary_batch, body_length, out);
+  return WriteFBMessage(fbb, flatbuf::MessageHeader_DictionaryBatch, dictionary_batch,
+                        body_length, out);
 }
 
 static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
@@ -754,8 +765,8 @@ FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
 }
 
 Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
-    io::OutputStream* out) {
+                       const std::vector<FileBlock>& record_batches,
+                       DictionaryMemo* dictionary_memo, io::OutputStream* out) {
   FBB fbb;
 
   flatbuffers::Offset<flatbuf::Schema> fb_schema;
@@ -764,8 +775,8 @@ Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dicti
   auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
   auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
 
-  auto footer = flatbuf::CreateFooter(
-      fbb, kCurrentMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
+  auto footer = flatbuf::CreateFooter(fbb, kCurrentMetadataVersion, fb_schema,
+                                      fb_dictionaries, fb_record_batches);
 
   fbb.Finish(footer);
 
@@ -780,8 +791,8 @@ Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dicti
 DictionaryMemo::DictionaryMemo() {}
 
 // Returns KeyError if dictionary not found
-Status DictionaryMemo::GetDictionary(
-    int64_t id, std::shared_ptr<Array>* dictionary) const {
+Status DictionaryMemo::GetDictionary(int64_t id,
+                                     std::shared_ptr<Array>* dictionary) const {
   auto it = id_to_dictionary_.find(id);
   if (it == id_to_dictionary_.end()) {
     std::stringstream ss;
@@ -817,8 +828,8 @@ bool DictionaryMemo::HasDictionaryId(int64_t id) const {
   return it != id_to_dictionary_.end();
 }
 
-Status DictionaryMemo::AddDictionary(
-    int64_t id, const std::shared_ptr<Array>& dictionary) {
+Status DictionaryMemo::AddDictionary(int64_t id,
+                                     const std::shared_ptr<Array>& dictionary) {
   if (HasDictionaryId(id)) {
     std::stringstream ss;
     ss << "Dictionary with id " << id << " already exists";
@@ -835,8 +846,8 @@ Status DictionaryMemo::AddDictionary(
 
 class Message::MessageImpl {
  public:
-  explicit MessageImpl(
-      const std::shared_ptr<Buffer>& metadata, const std::shared_ptr<Buffer>& body)
+  explicit MessageImpl(const std::shared_ptr<Buffer>& metadata,
+                       const std::shared_ptr<Buffer>& body)
       : metadata_(metadata), message_(nullptr), body_(body) {}
 
   Status Open() {
@@ -897,43 +908,35 @@ class Message::MessageImpl {
   std::shared_ptr<Buffer> body_;
 };
 
-Message::Message(
-    const std::shared_ptr<Buffer>& metadata, const std::shared_ptr<Buffer>& body) {
+Message::Message(const std::shared_ptr<Buffer>& metadata,
+                 const std::shared_ptr<Buffer>& body) {
   impl_.reset(new MessageImpl(metadata, body));
 }
 
 Status Message::Open(const std::shared_ptr<Buffer>& metadata,
-    const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out) {
+                     const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out) {
   out->reset(new Message(metadata, body));
   return (*out)->impl_->Open();
 }
 
 Message::~Message() {}
 
-std::shared_ptr<Buffer> Message::body() const {
-  return impl_->body();
-}
+std::shared_ptr<Buffer> Message::body() const { return impl_->body(); }
 
-std::shared_ptr<Buffer> Message::metadata() const {
-  return impl_->metadata();
-}
+std::shared_ptr<Buffer> Message::metadata() const { return impl_->metadata(); }
 
-Message::Type Message::type() const {
-  return impl_->type();
-}
+Message::Type Message::type() const { return impl_->type(); }
 
-MetadataVersion Message::metadata_version() const {
-  return impl_->version();
-}
+MetadataVersion Message::metadata_version() const { return impl_->version(); }
 
-const void* Message::header() const {
-  return impl_->header();
-}
+const void* Message::header() const { return impl_->header(); }
 
 bool Message::Equals(const Message& other) const {
   int64_t metadata_bytes = std::min(metadata()->size(), other.metadata()->size());
 
-  if (!metadata()->Equals(*other.metadata(), metadata_bytes)) { return false; }
+  if (!metadata()->Equals(*other.metadata(), metadata_bytes)) {
+    return false;
+  }
 
   // Compare bodies, if they have them
   auto this_body = body();
@@ -1012,7 +1015,7 @@ Status GetDictionaryTypes(const void* opaque_schema, DictionaryTypeMap* id_to_fi
 }
 
 Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_memo,
-    std::shared_ptr<Schema>* out) {
+                 std::shared_ptr<Schema>* out) {
   auto schema = static_cast<const flatbuf::Schema*>(opaque_schema);
   int num_fields = static_cast<int>(schema->fields()->size());
 
@@ -1036,8 +1039,8 @@ Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_mem
 }
 
 Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>* type,
-    std::vector<int64_t>* shape, std::vector<int64_t>* strides,
-    std::vector<std::string>* dim_names) {
+                         std::vector<int64_t>* shape, std::vector<int64_t>* strides,
+                         std::vector<std::string>* dim_names) {
   auto message = flatbuf::GetMessage(metadata.data());
   auto tensor = reinterpret_cast<const flatbuf::Tensor*>(message->header());
 
@@ -1068,7 +1071,8 @@ Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>* type
 // Read and write messages
 
 static Status ReadFullMessage(const std::shared_ptr<Buffer>& metadata,
-    io::InputStream* stream, std::unique_ptr<Message>* message) {
+                              io::InputStream* stream,
+                              std::unique_ptr<Message>* message) {
   auto fb_message = flatbuf::GetMessage(metadata->data());
 
   int64_t body_length = fb_message->bodyLength();
@@ -1087,7 +1091,7 @@ static Status ReadFullMessage(const std::shared_ptr<Buffer>& metadata,
 }
 
 Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
-    std::unique_ptr<Message>* message) {
+                   std::unique_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
 
@@ -1141,8 +1145,8 @@ InputStreamMessageReader::~InputStreamMessageReader() {}
 // ----------------------------------------------------------------------
 // Implement message writing
 
-Status WriteMessage(
-    const Buffer& message, io::OutputStream* file, int32_t* message_length) {
+Status WriteMessage(const Buffer& message, io::OutputStream* file,
+                    int32_t* message_length) {
   // Need to write 4 bytes (message size), the message, plus padding to
   // end on an 8-byte offset
   int64_t start_offset;
@@ -1151,7 +1155,9 @@ Status WriteMessage(
   int32_t padded_message_length = static_cast<int32_t>(message.size()) + 4;
   const int32_t remainder =
       (padded_message_length + static_cast<int32_t>(start_offset)) % 8;
-  if (remainder != 0) { padded_message_length += 8 - remainder; }
+  if (remainder != 0) {
+    padded_message_length += 8 - remainder;
+  }
 
   // The returned message size includes the length prefix, the flatbuffer,
   // plus padding
@@ -1167,7 +1173,9 @@ Status WriteMessage(
 
   // Write any padding
   int32_t padding = padded_message_length - static_cast<int32_t>(message.size()) - 4;
-  if (padding > 0) { RETURN_NOT_OK(file->Write(kPaddingBytes, padding)); }
+  if (padding > 0) {
+    RETURN_NOT_OK(file->Write(kPaddingBytes, padding));
+  }
 
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 614f7a6a922cc..90e4defd6a300 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -133,11 +133,14 @@ Status GetDictionaryTypes(const void* opaque_schema, DictionaryTypeMap* id_to_fi
 // Construct a complete Schema from the message. May be expensive for very
 // large schemas if you are only interested in a few fields
 Status ARROW_EXPORT GetSchema(const void* opaque_schema,
-    const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out);
+                              const DictionaryMemo& dictionary_memo,
+                              std::shared_ptr<Schema>* out);
 
 Status ARROW_EXPORT GetTensorMetadata(const Buffer& metadata,
-    std::shared_ptr<DataType>* type, std::vector<int64_t>* shape,
-    std::vector<int64_t>* strides, std::vector<std::string>* dim_names);
+                                      std::shared_ptr<DataType>* type,
+                                      std::vector<int64_t>* shape,
+                                      std::vector<int64_t>* strides,
+                                      std::vector<std::string>* dim_names);
 
 /// \brief An IPC message including metadata and body
 class ARROW_EXPORT Message {
@@ -157,7 +160,7 @@ class ARROW_EXPORT Message {
   /// \param[in] body a buffer containing the message body, which may be nullptr
   /// \param[out] out the created message
   static Status Open(const std::shared_ptr<Buffer>& metadata,
-      const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out);
+                     const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out);
 
   /// \brief Write length-prefixed metadata and body to output stream
   ///
@@ -242,22 +245,23 @@ class ARROW_EXPORT InputStreamMessageReader : public MessageReader {
 /// \param[out] message the message read
 /// \return Status success or failure
 Status ARROW_EXPORT ReadMessage(int64_t offset, int32_t metadata_length,
-    io::RandomAccessFile* file, std::unique_ptr<Message>* message);
+                                io::RandomAccessFile* file,
+                                std::unique_ptr<Message>* message);
 
 /// \brief Read encapulated RPC message (metadata and body) from InputStream
 ///
 /// Read length-prefixed message with as-yet unknown length. Returns nullptr if
 /// there are not enough bytes available or the message length is 0 (e.g. EOS
 /// in a stream)
-Status ARROW_EXPORT ReadMessage(
-    io::InputStream* stream, std::unique_ptr<Message>* message);
+Status ARROW_EXPORT ReadMessage(io::InputStream* stream,
+                                std::unique_ptr<Message>* message);
 
 /// Write a serialized message metadata with a length-prefix and padding to an
 /// 8-byte offset
 ///
 /// <message_size: int32><message: const void*><padding>
-Status ARROW_EXPORT WriteMessage(
-    const Buffer& message, io::OutputStream* file, int32_t* message_length);
+Status ARROW_EXPORT WriteMessage(const Buffer& message, io::OutputStream* file,
+                                 int32_t* message_length);
 
 // Serialize arrow::Schema as a Flatbuffer
 //
@@ -266,23 +270,26 @@ Status ARROW_EXPORT WriteMessage(
 // dictionary ids
 // \param[out] out the serialized arrow::Buffer
 // \return Status outcome
-Status ARROW_EXPORT WriteSchemaMessage(
-    const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out);
+Status ARROW_EXPORT WriteSchemaMessage(const Schema& schema,
+                                       DictionaryMemo* dictionary_memo,
+                                       std::shared_ptr<Buffer>* out);
 
 Status ARROW_EXPORT WriteRecordBatchMessage(int64_t length, int64_t body_length,
-    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
-    std::shared_ptr<Buffer>* out);
+                                            const std::vector<FieldMetadata>& nodes,
+                                            const std::vector<BufferMetadata>& buffers,
+                                            std::shared_ptr<Buffer>* out);
 
-Status ARROW_EXPORT WriteTensorMessage(
-    const Tensor& tensor, int64_t buffer_start_offset, std::shared_ptr<Buffer>* out);
+Status ARROW_EXPORT WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset,
+                                       std::shared_ptr<Buffer>* out);
 
 Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
-    const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
-    std::shared_ptr<Buffer>* out);
+                              const std::vector<FieldMetadata>& nodes,
+                              const std::vector<BufferMetadata>& buffers,
+                              std::shared_ptr<Buffer>* out);
 
 Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-    const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
-    io::OutputStream* out);
+                       const std::vector<FileBlock>& record_batches,
+                       DictionaryMemo* dictionary_memo, io::OutputStream* out);
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 88ab33087b637..8ae82804c3164 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -95,12 +95,12 @@ struct ArrayLoaderContext {
 };
 
 static Status LoadArray(const std::shared_ptr<DataType>& type,
-    ArrayLoaderContext* context, internal::ArrayData* out);
+                        ArrayLoaderContext* context, internal::ArrayData* out);
 
 class ArrayLoader {
  public:
   ArrayLoader(const std::shared_ptr<DataType>& type, internal::ArrayData* out,
-      ArrayLoaderContext* context)
+              ArrayLoaderContext* context)
       : type_(type), context_(context), out_(out) {}
 
   Status Load() {
@@ -184,7 +184,7 @@ class ArrayLoader {
   typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
                               !std::is_base_of<FixedSizeBinaryType, T>::value &&
                               !std::is_base_of<DictionaryType, T>::value,
-      Status>::type
+                          Status>::type
   Visit(const T& type) {
     return LoadPrimitive<T>();
   }
@@ -252,18 +252,18 @@ class ArrayLoader {
 };
 
 static Status LoadArray(const std::shared_ptr<DataType>& type,
-    ArrayLoaderContext* context, internal::ArrayData* out) {
+                        ArrayLoaderContext* context, internal::ArrayData* out) {
   ArrayLoader loader(type, out, context);
   return loader.Load();
 }
 
 Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+                       io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
   return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
 }
 
 Status ReadRecordBatch(const Message& message, const std::shared_ptr<Schema>& schema,
-    std::shared_ptr<RecordBatch>* out) {
+                       std::shared_ptr<RecordBatch>* out) {
   io::BufferReader reader(message.body());
   DCHECK_EQ(message.type(), Message::RECORD_BATCH);
   return ReadRecordBatch(*message.metadata(), schema, kMaxNestingDepth, &reader, out);
@@ -273,8 +273,9 @@ Status ReadRecordBatch(const Message& message, const std::shared_ptr<Schema>& sc
 // Array loading
 
 static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
-    int64_t num_rows, int max_recursion_depth, IpcComponentSource* source,
-    std::shared_ptr<RecordBatch>* out) {
+                                        int64_t num_rows, int max_recursion_depth,
+                                        IpcComponentSource* source,
+                                        std::shared_ptr<RecordBatch>* out) {
   ArrayLoaderContext context;
   context.source = source;
   context.field_index = 0;
@@ -294,16 +295,17 @@ static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
 }
 
 static inline Status ReadRecordBatch(const flatbuf::RecordBatch* metadata,
-    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+                                     const std::shared_ptr<Schema>& schema,
+                                     int max_recursion_depth, io::RandomAccessFile* file,
+                                     std::shared_ptr<RecordBatch>* out) {
   IpcComponentSource source(metadata, file);
-  return LoadRecordBatchFromSource(
-      schema, metadata->length(), max_recursion_depth, &source, out);
+  return LoadRecordBatchFromSource(schema, metadata->length(), max_recursion_depth,
+                                   &source, out);
 }
 
 Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
-    int max_recursion_depth, io::RandomAccessFile* file,
-    std::shared_ptr<RecordBatch>* out) {
+                       int max_recursion_depth, io::RandomAccessFile* file,
+                       std::shared_ptr<RecordBatch>* out) {
   auto message = flatbuf::GetMessage(metadata.data());
   if (message->header_type() != flatbuf::MessageHeader_RecordBatch) {
     DCHECK_EQ(message->header_type(), flatbuf::MessageHeader_RecordBatch);
@@ -313,7 +315,8 @@ Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& sc
 }
 
 Status ReadDictionary(const Buffer& metadata, const DictionaryTypeMap& dictionary_types,
-    io::RandomAccessFile* file, int64_t* dictionary_id, std::shared_ptr<Array>* out) {
+                      io::RandomAccessFile* file, int64_t* dictionary_id,
+                      std::shared_ptr<Array>* out) {
   auto message = flatbuf::GetMessage(metadata.data());
   auto dictionary_batch =
       reinterpret_cast<const flatbuf::DictionaryBatch*>(message->header());
@@ -347,7 +350,7 @@ Status ReadDictionary(const Buffer& metadata, const DictionaryTypeMap& dictionar
 }
 
 static Status ReadMessageAndValidate(MessageReader* reader, Message::Type expected_type,
-    bool allow_null, std::unique_ptr<Message>* message) {
+                                     bool allow_null, std::unique_ptr<Message>* message) {
   RETURN_NOT_OK(reader->ReadNextMessage(message));
 
   if (!(*message) && !allow_null) {
@@ -357,7 +360,9 @@ static Status ReadMessageAndValidate(MessageReader* reader, Message::Type expect
     return Status::Invalid(ss.str());
   }
 
-  if ((*message) == nullptr) { return Status::OK(); }
+  if ((*message) == nullptr) {
+    return Status::OK();
+  }
 
   if ((*message)->type() != expected_type) {
     std::stringstream ss;
@@ -389,15 +394,15 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
 
   Status ReadNextDictionary() {
     std::unique_ptr<Message> message;
-    RETURN_NOT_OK(ReadMessageAndValidate(
-        message_reader_.get(), Message::DICTIONARY_BATCH, false, &message));
+    RETURN_NOT_OK(ReadMessageAndValidate(message_reader_.get(), Message::DICTIONARY_BATCH,
+                                         false, &message));
 
     io::BufferReader reader(message->body());
 
     std::shared_ptr<Array> dictionary;
     int64_t id;
-    RETURN_NOT_OK(ReadDictionary(
-        *message->metadata(), dictionary_types_, &reader, &id, &dictionary));
+    RETURN_NOT_OK(ReadDictionary(*message->metadata(), dictionary_types_, &reader, &id,
+                                 &dictionary));
     return dictionary_memo_.AddDictionary(id, dictionary);
   }
 
@@ -420,8 +425,8 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
 
   Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
     std::unique_ptr<Message> message;
-    RETURN_NOT_OK(ReadMessageAndValidate(
-        message_reader_.get(), Message::RECORD_BATCH, true, &message));
+    RETURN_NOT_OK(ReadMessageAndValidate(message_reader_.get(), Message::RECORD_BATCH,
+                                         true, &message));
 
     if (message == nullptr) {
       // End of stream
@@ -451,14 +456,14 @@ RecordBatchStreamReader::RecordBatchStreamReader() {
 RecordBatchStreamReader::~RecordBatchStreamReader() {}
 
 Status RecordBatchStreamReader::Open(std::unique_ptr<MessageReader> message_reader,
-    std::shared_ptr<RecordBatchStreamReader>* reader) {
+                                     std::shared_ptr<RecordBatchStreamReader>* reader) {
   // Private ctor
   *reader = std::shared_ptr<RecordBatchStreamReader>(new RecordBatchStreamReader());
   return (*reader)->impl_->Open(std::move(message_reader));
 }
 
 Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
-    std::shared_ptr<RecordBatchStreamReader>* out) {
+                                     std::shared_ptr<RecordBatchStreamReader>* out) {
   std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
   return Open(std::move(message_reader), out);
 }
@@ -502,8 +507,8 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
     }
 
     // Now read the footer
-    RETURN_NOT_OK(file_->ReadAt(
-        footer_offset_ - footer_length - file_end_size, footer_length, &footer_buffer_));
+    RETURN_NOT_OK(file_->ReadAt(footer_offset_ - footer_length - file_end_size,
+                                footer_length, &footer_buffer_));
 
     // TODO(wesm): Verify the footer
     footer_ = flatbuf::GetFooter(footer_buffer_->data());
@@ -568,7 +573,7 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
       std::shared_ptr<Array> dictionary;
       int64_t dictionary_id;
       RETURN_NOT_OK(ReadDictionary(*message->metadata(), dictionary_fields_, &reader,
-          &dictionary_id, &dictionary));
+                                   &dictionary_id, &dictionary));
       RETURN_NOT_OK(dictionary_memo_->AddDictionary(dictionary_id, dictionary));
     }
 
@@ -610,37 +615,34 @@ RecordBatchFileReader::RecordBatchFileReader() {
 RecordBatchFileReader::~RecordBatchFileReader() {}
 
 Status RecordBatchFileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
-    std::shared_ptr<RecordBatchFileReader>* reader) {
+                                   std::shared_ptr<RecordBatchFileReader>* reader) {
   int64_t footer_offset;
   RETURN_NOT_OK(file->GetSize(&footer_offset));
   return Open(file, footer_offset, reader);
 }
 
 Status RecordBatchFileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
-    int64_t footer_offset, std::shared_ptr<RecordBatchFileReader>* reader) {
+                                   int64_t footer_offset,
+                                   std::shared_ptr<RecordBatchFileReader>* reader) {
   *reader = std::shared_ptr<RecordBatchFileReader>(new RecordBatchFileReader());
   return (*reader)->impl_->Open(file, footer_offset);
 }
 
-std::shared_ptr<Schema> RecordBatchFileReader::schema() const {
-  return impl_->schema();
-}
+std::shared_ptr<Schema> RecordBatchFileReader::schema() const { return impl_->schema(); }
 
 int RecordBatchFileReader::num_record_batches() const {
   return impl_->num_record_batches();
 }
 
-MetadataVersion RecordBatchFileReader::version() const {
-  return impl_->version();
-}
+MetadataVersion RecordBatchFileReader::version() const { return impl_->version(); }
 
-Status RecordBatchFileReader::ReadRecordBatch(
-    int i, std::shared_ptr<RecordBatch>* batch) {
+Status RecordBatchFileReader::ReadRecordBatch(int i,
+                                              std::shared_ptr<RecordBatch>* batch) {
   return impl_->ReadRecordBatch(i, batch);
 }
 
-static Status ReadContiguousPayload(
-    int64_t offset, io::RandomAccessFile* file, std::unique_ptr<Message>* message) {
+static Status ReadContiguousPayload(int64_t offset, io::RandomAccessFile* file,
+                                    std::unique_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->Seek(offset));
   RETURN_NOT_OK(ReadMessage(file, message));
@@ -652,16 +654,16 @@ static Status ReadContiguousPayload(
 }
 
 Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+                       io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
   std::unique_ptr<Message> message;
   RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message));
   io::BufferReader buffer_reader(message->body());
-  return ReadRecordBatch(
-      *message->metadata(), schema, kMaxNestingDepth, &buffer_reader, out);
+  return ReadRecordBatch(*message->metadata(), schema, kMaxNestingDepth, &buffer_reader,
+                         out);
 }
 
-Status ReadTensor(
-    int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out) {
+Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
+                  std::shared_ptr<Tensor>* out) {
   // Respect alignment of Tensor messages (see WriteTensor)
   offset = PaddedLength(offset);
   std::unique_ptr<Message> message;
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index d6c261475014c..c0d3fb1f185f9 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -72,7 +72,7 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
   /// \param(out) out the created RecordBatchStreamReader object
   /// \return Status
   static Status Open(std::unique_ptr<MessageReader> message_reader,
-      std::shared_ptr<RecordBatchStreamReader>* out);
+                     std::shared_ptr<RecordBatchStreamReader>* out);
 
   /// \Create Record batch stream reader from InputStream
   ///
@@ -80,7 +80,7 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
   /// \param(out) out the created RecordBatchStreamReader object
   /// \return Status
   static Status Open(const std::shared_ptr<io::InputStream>& stream,
-      std::shared_ptr<RecordBatchStreamReader>* out);
+                     std::shared_ptr<RecordBatchStreamReader>* out);
 
   std::shared_ptr<Schema> schema() const override;
   Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) override;
@@ -103,7 +103,7 @@ class ARROW_EXPORT RecordBatchFileReader {
   // need only locate the end of the Arrow file stream to discover the metadata
   // and then proceed to read the data into memory.
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
-      std::shared_ptr<RecordBatchFileReader>* reader);
+                     std::shared_ptr<RecordBatchFileReader>* reader);
 
   // If the file is embedded within some larger file or memory region, you can
   // pass the absolute memory offset to the end of the file (which contains the
@@ -113,7 +113,8 @@ class ARROW_EXPORT RecordBatchFileReader {
   // @param file: the data source
   // @param footer_offset: the position of the end of the Arrow "file"
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
-      int64_t footer_offset, std::shared_ptr<RecordBatchFileReader>* reader);
+                     int64_t footer_offset,
+                     std::shared_ptr<RecordBatchFileReader>* reader);
 
   /// The schema includes any dictionaries
   std::shared_ptr<Schema> schema() const;
@@ -148,8 +149,9 @@ class ARROW_EXPORT RecordBatchFileReader {
 /// \param(in) file a random access file
 /// \param(out) out the read record batch
 Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
-    const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
-    std::shared_ptr<RecordBatch>* out);
+                                    const std::shared_ptr<Schema>& schema,
+                                    io::RandomAccessFile* file,
+                                    std::shared_ptr<RecordBatch>* out);
 
 /// \brief Read record batch from fully encapulated Message
 ///
@@ -158,7 +160,8 @@ Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
 /// \param[out] out the resulting RecordBatch
 /// \return Status
 Status ARROW_EXPORT ReadRecordBatch(const Message& message,
-    const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out);
+                                    const std::shared_ptr<Schema>& schema,
+                                    std::shared_ptr<RecordBatch>* out);
 
 /// Read record batch from file given metadata and schema
 ///
@@ -168,8 +171,9 @@ Status ARROW_EXPORT ReadRecordBatch(const Message& message,
 /// \param(in) max_recursion_depth the maximum permitted nesting depth
 /// \param(out) out the read record batch
 Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
-    const std::shared_ptr<Schema>& schema, int max_recursion_depth,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
+                                    const std::shared_ptr<Schema>& schema,
+                                    int max_recursion_depth, io::RandomAccessFile* file,
+                                    std::shared_ptr<RecordBatch>* out);
 
 /// Read record batch as encapsulated IPC message with metadata size prefix and
 /// header
@@ -179,15 +183,16 @@ Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
 /// \param(in) file the file where the batch is located
 /// \param(out) out the read record batch
 Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
-    io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
+                                    io::RandomAccessFile* file,
+                                    std::shared_ptr<RecordBatch>* out);
 
 /// EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
 ///
 /// \param(in) offset the file location of the start of the message
 /// \param(in) file the file where the batch is located
 /// \param(out) out the read tensor
-Status ARROW_EXPORT ReadTensor(
-    int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out);
+Status ARROW_EXPORT ReadTensor(int64_t offset, io::RandomAccessFile* file,
+                               std::shared_ptr<Tensor>* out);
 
 /// Backwards-compatibility for Arrow < 0.4.0
 ///
diff --git a/cpp/src/arrow/ipc/stream-to-file.cc b/cpp/src/arrow/ipc/stream-to-file.cc
index de65883910120..33719b3c89c9e 100644
--- a/cpp/src/arrow/ipc/stream-to-file.cc
+++ b/cpp/src/arrow/ipc/stream-to-file.cc
@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <iostream>
 #include "arrow/io/file.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
 #include "arrow/status.h"
-#include <iostream>
 
 #include "arrow/util/io-util.h"
 
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 67a41ba086b75..a8767926b2a07 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -69,8 +69,8 @@ static inline void CompareBatch(const RecordBatch& left, const RecordBatch& righ
   }
 }
 
-static inline void CompareArraysDetailed(
-    int index, const Array& result, const Array& expected) {
+static inline void CompareArraysDetailed(int index, const Array& result,
+                                         const Array& expected) {
   if (!expected.Equals(result)) {
     std::stringstream pp_result;
     std::stringstream pp_expected;
@@ -83,8 +83,8 @@ static inline void CompareArraysDetailed(
   }
 }
 
-static inline void CompareBatchColumnsDetailed(
-    const RecordBatch& result, const RecordBatch& expected) {
+static inline void CompareBatchColumnsDetailed(const RecordBatch& result,
+                                               const RecordBatch& expected) {
   for (int i = 0; i < expected.num_columns(); ++i) {
     auto left = result.column(i);
     auto right = expected.column(i);
@@ -95,16 +95,16 @@ static inline void CompareBatchColumnsDetailed(
 const auto kListInt32 = list(int32());
 const auto kListListInt32 = list(kListInt32);
 
-Status MakeRandomInt32Array(
-    int64_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
+Status MakeRandomInt32Array(int64_t length, bool include_nulls, MemoryPool* pool,
+                            std::shared_ptr<Array>* out) {
   std::shared_ptr<PoolBuffer> data;
   RETURN_NOT_OK(test::MakeRandomInt32PoolBuffer(length, pool, &data));
   Int32Builder builder(pool, int32());
   if (include_nulls) {
     std::shared_ptr<PoolBuffer> valid_bytes;
     RETURN_NOT_OK(test::MakeRandomBytePoolBuffer(length, pool, &valid_bytes));
-    RETURN_NOT_OK(builder.Append(
-        reinterpret_cast<const int32_t*>(data->data()), length, valid_bytes->data()));
+    RETURN_NOT_OK(builder.Append(reinterpret_cast<const int32_t*>(data->data()), length,
+                                 valid_bytes->data()));
     return builder.Finish(out);
   }
   RETURN_NOT_OK(builder.Append(reinterpret_cast<const int32_t*>(data->data()), length));
@@ -112,7 +112,8 @@ Status MakeRandomInt32Array(
 }
 
 Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_lists,
-    bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
+                           bool include_nulls, MemoryPool* pool,
+                           std::shared_ptr<Array>* out) {
   // Create the null list values
   std::vector<uint8_t> valid_lists(num_lists);
   const double null_percent = include_nulls ? 0.1 : 0;
@@ -129,15 +130,16 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
     test::rand_uniform_int(num_lists, seed, 0, max_list_size, list_sizes.data());
     // make sure sizes are consistent with null
     std::transform(list_sizes.begin(), list_sizes.end(), valid_lists.begin(),
-        list_sizes.begin(),
-        [](int32_t size, int32_t valid) { return valid == 0 ? 0 : size; });
+                   list_sizes.begin(),
+                   [](int32_t size, int32_t valid) { return valid == 0 ? 0 : size; });
     std::partial_sum(list_sizes.begin(), list_sizes.end(), ++offsets.begin());
 
     // Force invariants
     const int32_t child_length = static_cast<int32_t>(child_array->length());
     offsets[0] = 0;
     std::replace_if(offsets.begin(), offsets.end(),
-        [child_length](int32_t offset) { return offset > child_length; }, child_length);
+                    [child_length](int32_t offset) { return offset > child_length; },
+                    child_length);
   }
 
   offsets[num_lists] = static_cast<int32_t>(child_array->length());
@@ -148,14 +150,14 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
   RETURN_NOT_OK(test::CopyBufferFromVector(offsets, pool, &offsets_buffer));
 
   *out = std::make_shared<ListArray>(list(child_array->type()), num_lists, offsets_buffer,
-      child_array, null_bitmap, kUnknownNullCount);
+                                     child_array, null_bitmap, kUnknownNullCount);
   return ValidateArray(**out);
 }
 
 typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
 
-Status MakeRandomBooleanArray(
-    const int length, bool include_nulls, std::shared_ptr<Array>* out) {
+Status MakeRandomBooleanArray(const int length, bool include_nulls,
+                              std::shared_ptr<Array>* out) {
   std::vector<uint8_t> values(length);
   test::random_null_bytes(length, 0.5, values.data());
   std::shared_ptr<Buffer> data;
@@ -210,10 +212,10 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
 }
 
 template <class Builder, class RawType>
-Status MakeRandomBinaryArray(
-    int64_t length, bool include_nulls, MemoryPool* pool, std::shared_ptr<Array>* out) {
-  const std::vector<std::string> values = {
-      "", "", "abc", "123", "efg", "456!@#!@#", "12312"};
+Status MakeRandomBinaryArray(int64_t length, bool include_nulls, MemoryPool* pool,
+                             std::shared_ptr<Array>* out) {
+  const std::vector<std::string> values = {"",    "",          "abc",  "123",
+                                           "efg", "456!@#!@#", "12312"};
   Builder builder(pool);
   const size_t values_len = values.size();
   for (int64_t i = 0; i < length; ++i) {
@@ -223,7 +225,7 @@ Status MakeRandomBinaryArray(
     } else {
       const std::string& value = values[values_index];
       RETURN_NOT_OK(builder.Append(reinterpret_cast<const RawType*>(value.data()),
-          static_cast<int32_t>(value.size())));
+                                   static_cast<int32_t>(value.size())));
     }
   }
   return builder.Finish(out);
@@ -434,11 +436,12 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   // construct individual nullable/non-nullable struct arrays
   auto sparse_no_nulls =
       std::make_shared<UnionArray>(sparse_type, length, sparse_children, type_ids_buffer);
-  auto sparse = std::make_shared<UnionArray>(
-      sparse_type, length, sparse_children, type_ids_buffer, nullptr, null_bitmask, 1);
+  auto sparse = std::make_shared<UnionArray>(sparse_type, length, sparse_children,
+                                             type_ids_buffer, nullptr, null_bitmask, 1);
 
-  auto dense = std::make_shared<UnionArray>(dense_type, length, dense_children,
-      type_ids_buffer, offsets_buffer, null_bitmask, 1);
+  auto dense =
+      std::make_shared<UnionArray>(dense_type, length, dense_children, type_ids_buffer,
+                                   offsets_buffer, null_bitmask, 1);
 
   // construct batch
   std::vector<std::shared_ptr<Array>> arrays = {sparse_no_nulls, sparse, dense};
@@ -480,8 +483,8 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
 
   std::vector<int32_t> list_offsets = {0, 0, 2, 2, 5, 6, 9};
   std::shared_ptr<Array> offsets, indices3;
-  ArrayFromVector<Int32Type, int32_t>(
-      std::vector<bool>(list_offsets.size(), true), list_offsets, &offsets);
+  ArrayFromVector<Int32Type, int32_t>(std::vector<bool>(list_offsets.size(), true),
+                                      list_offsets, &offsets);
 
   std::vector<int8_t> indices3_values = {0, 1, 2, 0, 1, 2, 0, 1, 2};
   std::vector<bool> is_valid3(9, true);
@@ -490,8 +493,8 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Buffer> null_bitmap;
   RETURN_NOT_OK(test::GetBitmapFromVector(is_valid, &null_bitmap));
 
-  std::shared_ptr<Array> a3 = std::make_shared<ListArray>(f3_type, length,
-      std::static_pointer_cast<PrimitiveArray>(offsets)->values(),
+  std::shared_ptr<Array> a3 = std::make_shared<ListArray>(
+      f3_type, length, std::static_pointer_cast<PrimitiveArray>(offsets)->values(),
       std::make_shared<DictionaryArray>(f1_type, indices3), null_bitmap, 1);
 
   // Dictionary-encoded list of integer
@@ -500,14 +503,15 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Array> offsets4, values4, indices4;
 
   std::vector<int32_t> list_offsets4 = {0, 2, 2, 3};
-  ArrayFromVector<Int32Type, int32_t>(
-      std::vector<bool>(4, true), list_offsets4, &offsets4);
+  ArrayFromVector<Int32Type, int32_t>(std::vector<bool>(4, true), list_offsets4,
+                                      &offsets4);
 
   std::vector<int8_t> list_values4 = {0, 1, 2};
   ArrayFromVector<Int8Type, int8_t>(std::vector<bool>(3, true), list_values4, &values4);
 
-  auto dict3 = std::make_shared<ListArray>(f4_value_type, 3,
-      std::static_pointer_cast<PrimitiveArray>(offsets4)->values(), values4);
+  auto dict3 = std::make_shared<ListArray>(
+      f4_value_type, 3, std::static_pointer_cast<PrimitiveArray>(offsets4)->values(),
+      values4);
 
   std::vector<int8_t> indices4_values = {0, 1, 2, 0, 1, 2};
   ArrayFromVector<Int8Type, int8_t>(is_valid, indices4_values, &indices4);
@@ -516,9 +520,9 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   auto a4 = std::make_shared<DictionaryArray>(f4_type, indices4);
 
   // construct batch
-  std::shared_ptr<Schema> schema(new Schema({field("dict1", f0_type),
-      field("sparse", f1_type), field("dense", f2_type),
-      field("list of encoded string", f3_type), field("encoded list<int8>", f4_type)}));
+  std::shared_ptr<Schema> schema(new Schema(
+      {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type),
+       field("list of encoded string", f3_type), field("encoded list<int8>", f4_type)}));
 
   std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2, a3, a4};
 
@@ -575,7 +579,8 @@ Status MakeDates(std::shared_ptr<RecordBatch>* out) {
   ArrayFromVector<Date32Type, int32_t>(is_valid, date32_values, &date32_array);
 
   std::vector<int64_t> date64_values = {1489269000000, 1489270000000, 1489271000000,
-      1489272000000, 1489272000000, 1489273000000, 1489274000000};
+                                        1489272000000, 1489272000000, 1489273000000,
+                                        1489274000000};
   std::shared_ptr<Array> date64_array;
   ArrayFromVector<Date64Type, int64_t>(is_valid, date64_values, &date64_array);
 
@@ -592,7 +597,7 @@ Status MakeTimestamps(std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
 
   std::vector<int64_t> ts_values = {1489269000000, 1489270000000, 1489271000000,
-      1489272000000, 1489272000000, 1489273000000};
+                                    1489272000000, 1489272000000, 1489273000000};
 
   std::shared_ptr<Array> a0, a1, a2;
   ArrayFromVector<TimestampType, int64_t>(f0->type(), is_valid, ts_values, &a0);
@@ -612,10 +617,10 @@ Status MakeTimes(std::shared_ptr<RecordBatch>* out) {
   auto f3 = field("f3", time64(TimeUnit::NANO));
   std::shared_ptr<Schema> schema(new Schema({f0, f1, f2, f3}));
 
-  std::vector<int32_t> t32_values = {
-      1489269000, 1489270000, 1489271000, 1489272000, 1489272000, 1489273000};
+  std::vector<int32_t> t32_values = {1489269000, 1489270000, 1489271000,
+                                     1489272000, 1489272000, 1489273000};
   std::vector<int64_t> t64_values = {1489269000000, 1489270000000, 1489271000000,
-      1489272000000, 1489272000000, 1489273000000};
+                                     1489272000000, 1489272000000, 1489273000000};
 
   std::shared_ptr<Array> a0, a1, a2, a3;
   ArrayFromVector<Time32Type, int32_t>(f0->type(), is_valid, t32_values, &a0);
@@ -630,7 +635,7 @@ Status MakeTimes(std::shared_ptr<RecordBatch>* out) {
 
 template <typename BuilderType, typename T>
 void AppendValues(const std::vector<bool>& is_valid, const std::vector<T>& values,
-    BuilderType* builder) {
+                  BuilderType* builder) {
   for (size_t i = 0; i < values.size(); ++i) {
     if (is_valid[i]) {
       ASSERT_OK(builder->Append(values[i]));
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 14708a1e7a032..163b27b443351 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -45,8 +45,9 @@ namespace ipc {
 // Record batch write path
 
 static inline Status GetTruncatedBitmap(int64_t offset, int64_t length,
-    const std::shared_ptr<Buffer> input, MemoryPool* pool,
-    std::shared_ptr<Buffer>* buffer) {
+                                        const std::shared_ptr<Buffer> input,
+                                        MemoryPool* pool,
+                                        std::shared_ptr<Buffer>* buffer) {
   if (!input) {
     *buffer = input;
     return Status::OK();
@@ -63,8 +64,8 @@ static inline Status GetTruncatedBitmap(int64_t offset, int64_t length,
 
 template <typename T>
 inline Status GetTruncatedBuffer(int64_t offset, int64_t length,
-    const std::shared_ptr<Buffer> input, MemoryPool* pool,
-    std::shared_ptr<Buffer>* buffer) {
+                                 const std::shared_ptr<Buffer> input, MemoryPool* pool,
+                                 std::shared_ptr<Buffer>* buffer) {
   if (!input) {
     *buffer = input;
     return Status::OK();
@@ -80,17 +81,19 @@ inline Status GetTruncatedBuffer(int64_t offset, int64_t length,
   return Status::OK();
 }
 
-static inline bool NeedTruncate(
-    int64_t offset, const Buffer* buffer, int64_t min_length) {
+static inline bool NeedTruncate(int64_t offset, const Buffer* buffer,
+                                int64_t min_length) {
   // buffer can be NULL
-  if (buffer == nullptr) { return false; }
+  if (buffer == nullptr) {
+    return false;
+  }
   return offset != 0 || min_length < buffer->size();
 }
 
 class RecordBatchSerializer : public ArrayVisitor {
  public:
   RecordBatchSerializer(MemoryPool* pool, int64_t buffer_start_offset,
-      int max_recursion_depth, bool allow_64bit)
+                        int max_recursion_depth, bool allow_64bit)
       : pool_(pool),
         max_recursion_depth_(max_recursion_depth),
         buffer_start_offset_(buffer_start_offset),
@@ -114,8 +117,8 @@ class RecordBatchSerializer : public ArrayVisitor {
 
     if (arr.null_count() > 0) {
       std::shared_ptr<Buffer> bitmap;
-      RETURN_NOT_OK(GetTruncatedBitmap(
-          arr.offset(), arr.length(), arr.null_bitmap(), pool_, &bitmap));
+      RETURN_NOT_OK(GetTruncatedBitmap(arr.offset(), arr.length(), arr.null_bitmap(),
+                                       pool_, &bitmap));
       buffers_.push_back(bitmap);
     } else {
       // Push a dummy zero-length buffer, not to be copied
@@ -175,14 +178,14 @@ class RecordBatchSerializer : public ArrayVisitor {
   }
 
   // Override this for writing dictionary metadata
-  virtual Status WriteMetadataMessage(
-      int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) {
-    return WriteRecordBatchMessage(
-        num_rows, body_length, field_nodes_, buffer_meta_, out);
+  virtual Status WriteMetadataMessage(int64_t num_rows, int64_t body_length,
+                                      std::shared_ptr<Buffer>* out) {
+    return WriteRecordBatchMessage(num_rows, body_length, field_nodes_, buffer_meta_,
+                                   out);
   }
 
   Status Write(const RecordBatch& batch, io::OutputStream* dst, int32_t* metadata_length,
-      int64_t* body_length) {
+               int64_t* body_length) {
     RETURN_NOT_OK(Assemble(batch, body_length));
 
 #ifndef NDEBUG
@@ -216,9 +219,13 @@ class RecordBatchSerializer : public ArrayVisitor {
         padding = BitUtil::RoundUpToMultipleOf64(size) - size;
       }
 
-      if (size > 0) { RETURN_NOT_OK(dst->Write(buffer->data(), size)); }
+      if (size > 0) {
+        RETURN_NOT_OK(dst->Write(buffer->data(), size));
+      }
 
-      if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
+      if (padding > 0) {
+        RETURN_NOT_OK(dst->Write(kPaddingBytes, padding));
+      }
     }
 
 #ifndef NDEBUG
@@ -245,7 +252,7 @@ class RecordBatchSerializer : public ArrayVisitor {
       // Send padding if it's available
       const int64_t buffer_length =
           std::min(BitUtil::RoundUpToMultipleOf64(array.length() * type_width),
-              data->size() - byte_offset);
+                   data->size() - byte_offset);
       data = SliceBuffer(data, byte_offset, buffer_length);
     }
     buffers_.push_back(data);
@@ -253,8 +260,8 @@ class RecordBatchSerializer : public ArrayVisitor {
   }
 
   template <typename ArrayType>
-  Status GetZeroBasedValueOffsets(
-      const ArrayType& array, std::shared_ptr<Buffer>* value_offsets) {
+  Status GetZeroBasedValueOffsets(const ArrayType& array,
+                                  std::shared_ptr<Buffer>* value_offsets) {
     // Share slicing logic between ListArray and BinaryArray
 
     auto offsets = array.value_offsets();
@@ -265,8 +272,8 @@ class RecordBatchSerializer : public ArrayVisitor {
       // b) slice the values array accordingly
 
       std::shared_ptr<MutableBuffer> shifted_offsets;
-      RETURN_NOT_OK(AllocateBuffer(
-          pool_, sizeof(int32_t) * (array.length() + 1), &shifted_offsets));
+      RETURN_NOT_OK(AllocateBuffer(pool_, sizeof(int32_t) * (array.length() + 1),
+                                   &shifted_offsets));
 
       int32_t* dest_offsets = reinterpret_cast<int32_t*>(shifted_offsets->mutable_data());
       const int32_t start_offset = array.value_offset(0);
@@ -392,13 +399,15 @@ class RecordBatchSerializer : public ArrayVisitor {
       const auto& type = static_cast<const UnionType&>(*array.type());
 
       std::shared_ptr<Buffer> value_offsets;
-      RETURN_NOT_OK(GetTruncatedBuffer<int32_t>(
-          offset, length, array.value_offsets(), pool_, &value_offsets));
+      RETURN_NOT_OK(GetTruncatedBuffer<int32_t>(offset, length, array.value_offsets(),
+                                                pool_, &value_offsets));
 
       // The Union type codes are not necessary 0-indexed
       uint8_t max_code = 0;
       for (uint8_t code : type.type_codes()) {
-        if (code > max_code) { max_code = code; }
+        if (code > max_code) {
+          max_code = code;
+        }
       }
 
       // Allocate an array of child offsets. Set all to -1 to indicate that we
@@ -424,7 +433,9 @@ class RecordBatchSerializer : public ArrayVisitor {
         for (int64_t i = 0; i < length; ++i) {
           const uint8_t code = type_ids[i];
           int32_t shift = child_offsets[code];
-          if (shift == -1) { child_offsets[code] = shift = unshifted_offsets[i]; }
+          if (shift == -1) {
+            child_offsets[code] = shift = unshifted_offsets[i];
+          }
           shifted_offsets[i] = unshifted_offsets[i] - shift;
 
           // Update the child length to account for observed value
@@ -486,14 +497,14 @@ class DictionaryWriter : public RecordBatchSerializer {
  public:
   using RecordBatchSerializer::RecordBatchSerializer;
 
-  Status WriteMetadataMessage(
-      int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
-    return WriteDictionaryMessage(
-        dictionary_id_, num_rows, body_length, field_nodes_, buffer_meta_, out);
+  Status WriteMetadataMessage(int64_t num_rows, int64_t body_length,
+                              std::shared_ptr<Buffer>* out) override {
+    return WriteDictionaryMessage(dictionary_id_, num_rows, body_length, field_nodes_,
+                                  buffer_meta_, out);
   }
 
   Status Write(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
-      io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
+               io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
     dictionary_id_ = dictionary_id;
 
     // Make a dummy record batch. A bit tedious as we have to make a schema
@@ -516,27 +527,30 @@ Status AlignStreamPosition(io::OutputStream* stream) {
   int64_t position;
   RETURN_NOT_OK(stream->Tell(&position));
   int64_t remainder = PaddedLength(position) - position;
-  if (remainder > 0) { return stream->Write(kPaddingBytes, remainder); }
+  if (remainder > 0) {
+    return stream->Write(kPaddingBytes, remainder);
+  }
   return Status::OK();
 }
 
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
-    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool, int max_recursion_depth, bool allow_64bit) {
-  RecordBatchSerializer writer(
-      pool, buffer_start_offset, max_recursion_depth, allow_64bit);
+                        io::OutputStream* dst, int32_t* metadata_length,
+                        int64_t* body_length, MemoryPool* pool, int max_recursion_depth,
+                        bool allow_64bit) {
+  RecordBatchSerializer writer(pool, buffer_start_offset, max_recursion_depth,
+                               allow_64bit);
   return writer.Write(batch, dst, metadata_length, body_length);
 }
 
 Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
-    io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
-    MemoryPool* pool) {
+                             io::OutputStream* dst, int32_t* metadata_length,
+                             int64_t* body_length, MemoryPool* pool) {
   return WriteRecordBatch(batch, buffer_start_offset, dst, metadata_length, body_length,
-      pool, kMaxNestingDepth, true);
+                          pool, kMaxNestingDepth, true);
 }
 
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length) {
+                   int64_t* body_length) {
   if (!tensor.is_contiguous()) {
     return Status::Invalid("No support yet for writing non-contiguous tensors");
   }
@@ -556,8 +570,8 @@ Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadat
 }
 
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
-    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length, MemoryPool* pool) {
+                       int64_t buffer_start_offset, io::OutputStream* dst,
+                       int32_t* metadata_length, int64_t* body_length, MemoryPool* pool) {
   DictionaryWriter writer(pool, buffer_start_offset, kMaxNestingDepth, false);
   return writer.Write(dictionary_id, dictionary, dst, metadata_length, body_length);
 }
@@ -568,7 +582,7 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
   int64_t body_length = 0;
   io::MockOutputStream dst;
   RETURN_NOT_OK(WriteRecordBatch(batch, 0, &dst, &metadata_length, &body_length,
-      default_memory_pool(), kMaxNestingDepth, true));
+                                 default_memory_pool(), kMaxNestingDepth, true));
   *size = dst.GetExtentBytesWritten();
   return Status::OK();
 }
@@ -632,7 +646,9 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
   }
 
   Status CheckStarted() {
-    if (!started_) { return Start(); }
+    if (!started_) {
+      return Start();
+    }
     return Status::OK();
   }
 
@@ -653,7 +669,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
       // Frame of reference in file format is 0, see ARROW-384
       const int64_t buffer_start_offset = 0;
       RETURN_NOT_OK(WriteDictionary(entry.first, entry.second, buffer_start_offset, sink_,
-          &block->metadata_length, &block->body_length, pool_));
+                                    &block->metadata_length, &block->body_length, pool_));
       RETURN_NOT_OK(UpdatePosition());
       DCHECK(position_ % 8 == 0) << "WriteDictionary did not perform aligned writes";
     }
@@ -668,9 +684,9 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
 
     // Frame of reference in file format is 0, see ARROW-384
     const int64_t buffer_start_offset = 0;
-    RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(batch, buffer_start_offset, sink_,
-        &block->metadata_length, &block->body_length, pool_, kMaxNestingDepth,
-        allow_64bit));
+    RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(
+        batch, buffer_start_offset, sink_, &block->metadata_length, &block->body_length,
+        pool_, kMaxNestingDepth, allow_64bit));
     RETURN_NOT_OK(UpdatePosition());
 
     DCHECK(position_ % 8 == 0) << "WriteRecordBatch did not perform aligned writes";
@@ -681,15 +697,17 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit) {
     // Push an empty FileBlock. Can be written in the footer later
     record_batches_.push_back({0, 0, 0});
-    return WriteRecordBatch(
-        batch, allow_64bit, &record_batches_[record_batches_.size() - 1]);
+    return WriteRecordBatch(batch, allow_64bit,
+                            &record_batches_[record_batches_.size() - 1]);
   }
 
   // Adds padding bytes if necessary to ensure all memory blocks are written on
   // 64-byte (or other alignment) boundaries.
   Status Align(int64_t alignment = kArrowAlignment) {
     int64_t remainder = PaddedLength(position_, alignment) - position_;
-    if (remainder > 0) { return Write(kPaddingBytes, remainder); }
+    if (remainder > 0) {
+      return Write(kPaddingBytes, remainder);
+    }
     return Status::OK();
   }
 
@@ -725,8 +743,8 @@ RecordBatchStreamWriter::RecordBatchStreamWriter() {
 
 RecordBatchStreamWriter::~RecordBatchStreamWriter() {}
 
-Status RecordBatchStreamWriter::WriteRecordBatch(
-    const RecordBatch& batch, bool allow_64bit) {
+Status RecordBatchStreamWriter::WriteRecordBatch(const RecordBatch& batch,
+                                                 bool allow_64bit) {
   return impl_->WriteRecordBatch(batch, allow_64bit);
 }
 
@@ -735,16 +753,14 @@ void RecordBatchStreamWriter::set_memory_pool(MemoryPool* pool) {
 }
 
 Status RecordBatchStreamWriter::Open(io::OutputStream* sink,
-    const std::shared_ptr<Schema>& schema,
-    std::shared_ptr<RecordBatchStreamWriter>* out) {
+                                     const std::shared_ptr<Schema>& schema,
+                                     std::shared_ptr<RecordBatchStreamWriter>* out) {
   // ctor is private
   *out = std::shared_ptr<RecordBatchStreamWriter>(new RecordBatchStreamWriter());
   return (*out)->impl_->Open(sink, schema);
 }
 
-Status RecordBatchStreamWriter::Close() {
-  return impl_->Close();
-}
+Status RecordBatchStreamWriter::Close() { return impl_->Close(); }
 
 // ----------------------------------------------------------------------
 // File writer implementation
@@ -756,8 +772,8 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
 
   Status Start() override {
     // It is only necessary to align to 8-byte boundary at the start of the file
-    RETURN_NOT_OK(Write(
-        reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes)));
+    RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
+                        strlen(kArrowMagicBytes)));
     RETURN_NOT_OK(Align(8));
 
     // We write the schema at the start of the file (and the end). This also
@@ -768,21 +784,23 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
   Status Close() override {
     // Write metadata
     int64_t initial_position = position_;
-    RETURN_NOT_OK(WriteFileFooter(
-        *schema_, dictionaries_, record_batches_, &dictionary_memo_, sink_));
+    RETURN_NOT_OK(WriteFileFooter(*schema_, dictionaries_, record_batches_,
+                                  &dictionary_memo_, sink_));
     RETURN_NOT_OK(UpdatePosition());
 
     // Write footer length
     int32_t footer_length = static_cast<int32_t>(position_ - initial_position);
 
-    if (footer_length <= 0) { return Status::Invalid("Invalid file footer"); }
+    if (footer_length <= 0) {
+      return Status::Invalid("Invalid file footer");
+    }
 
     RETURN_NOT_OK(
         Write(reinterpret_cast<const uint8_t*>(&footer_length), sizeof(int32_t)));
 
     // Write magic bytes to end file
-    return Write(
-        reinterpret_cast<const uint8_t*>(kArrowMagicBytes), strlen(kArrowMagicBytes));
+    return Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
+                 strlen(kArrowMagicBytes));
   }
 };
 
@@ -793,20 +811,19 @@ RecordBatchFileWriter::RecordBatchFileWriter() {
 RecordBatchFileWriter::~RecordBatchFileWriter() {}
 
 Status RecordBatchFileWriter::Open(io::OutputStream* sink,
-    const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatchFileWriter>* out) {
+                                   const std::shared_ptr<Schema>& schema,
+                                   std::shared_ptr<RecordBatchFileWriter>* out) {
   *out = std::shared_ptr<RecordBatchFileWriter>(
       new RecordBatchFileWriter());  // ctor is private
   return (*out)->impl_->Open(sink, schema);
 }
 
-Status RecordBatchFileWriter::WriteRecordBatch(
-    const RecordBatch& batch, bool allow_64bit) {
+Status RecordBatchFileWriter::WriteRecordBatch(const RecordBatch& batch,
+                                               bool allow_64bit) {
   return impl_->WriteRecordBatch(batch, allow_64bit);
 }
 
-Status RecordBatchFileWriter::Close() {
-  return impl_->Close();
-}
+Status RecordBatchFileWriter::Close() { return impl_->Close(); }
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 899a1b2cc1e30..c28dfe0afbb11 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -85,7 +85,7 @@ class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
   /// \param(out) out the created stream writer
   /// \return Status indicating success or failure
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-      std::shared_ptr<RecordBatchStreamWriter>* out);
+                     std::shared_ptr<RecordBatchStreamWriter>* out);
 
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
   Status Close() override;
@@ -113,7 +113,7 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
   /// \param(out) out the created stream writer
   /// \return Status indicating success or failure
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-      std::shared_ptr<RecordBatchFileWriter>* out);
+                     std::shared_ptr<RecordBatchFileWriter>* out);
 
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
   Status Close() override;
@@ -145,14 +145,16 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
 /// \param(out) body_length: the size of the contiguous buffer block plus
 /// padding bytes
 Status ARROW_EXPORT WriteRecordBatch(const RecordBatch& batch,
-    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length, MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth,
-    bool allow_64bit = false);
+                                     int64_t buffer_start_offset, io::OutputStream* dst,
+                                     int32_t* metadata_length, int64_t* body_length,
+                                     MemoryPool* pool,
+                                     int max_recursion_depth = kMaxNestingDepth,
+                                     bool allow_64bit = false);
 
 // Write Array as a DictionaryBatch message
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
-    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length, MemoryPool* pool);
+                       int64_t buffer_start_offset, io::OutputStream* dst,
+                       int32_t* metadata_length, int64_t* body_length, MemoryPool* pool);
 
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the record batch. This involves generating the complete serialized
@@ -166,13 +168,14 @@ Status ARROW_EXPORT GetTensorSize(const Tensor& tensor, int64_t* size);
 /// EXPERIMENTAL: Write RecordBatch allowing lengths over INT32_MAX. This data
 /// may not be readable by all Arrow implementations
 Status ARROW_EXPORT WriteLargeRecordBatch(const RecordBatch& batch,
-    int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
-    int64_t* body_length, MemoryPool* pool);
+                                          int64_t buffer_start_offset,
+                                          io::OutputStream* dst, int32_t* metadata_length,
+                                          int64_t* body_length, MemoryPool* pool);
 
 /// EXPERIMENTAL: Write arrow::Tensor as a contiguous message
 /// <metadata size><metadata><tensor data>
 Status ARROW_EXPORT WriteTensor(const Tensor& tensor, io::OutputStream* dst,
-    int32_t* metadata_length, int64_t* body_length);
+                                int32_t* metadata_length, int64_t* body_length);
 
 /// Backwards-compatibility for Arrow < 0.4.0
 ///
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 8a185abca71cc..52e48dbefab9e 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -27,9 +27,7 @@ class TestDefaultMemoryPool : public ::arrow::test::TestMemoryPoolBase {
   ::arrow::MemoryPool* memory_pool() override { return ::arrow::default_memory_pool(); }
 };
 
-TEST_F(TestDefaultMemoryPool, MemoryTracking) {
-  this->TestMemoryTracking();
-}
+TEST_F(TestDefaultMemoryPool, MemoryTracking) { this->TestMemoryTracking(); }
 
 TEST_F(TestDefaultMemoryPool, OOM) {
 #ifndef ADDRESS_SANITIZER
@@ -37,9 +35,7 @@ TEST_F(TestDefaultMemoryPool, OOM) {
 #endif
 }
 
-TEST_F(TestDefaultMemoryPool, Reallocate) {
-  this->TestReallocate();
-}
+TEST_F(TestDefaultMemoryPool, Reallocate) { this->TestReallocate(); }
 
 // Death tests and valgrind are known to not play well 100% of the time. See
 // googletest documentation
@@ -53,7 +49,7 @@ TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
 
 #ifndef NDEBUG
   EXPECT_EXIT(pool->Free(data, 120), ::testing::ExitedWithCode(1),
-      ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
+              ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
 #endif
 
   pool->Free(data, 100);
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index e7de5c4fc589a..769fc1037ee80 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -17,12 +17,12 @@
 
 #include "arrow/memory_pool.h"
 
+#include <stdlib.h>
 #include <algorithm>
 #include <cstdlib>
 #include <iostream>
 #include <mutex>
 #include <sstream>
-#include <stdlib.h>
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
@@ -60,8 +60,8 @@ Status AllocateAligned(int64_t size, uint8_t** out) {
     return Status::OutOfMemory(ss.str());
   }
 #else
-  const int result = posix_memalign(
-      reinterpret_cast<void**>(out), kAlignment, static_cast<size_t>(size));
+  const int result = posix_memalign(reinterpret_cast<void**>(out), kAlignment,
+                                    static_cast<size_t>(size));
   if (result == ENOMEM) {
     std::stringstream ss;
     ss << "malloc of size " << size << " failed";
@@ -82,13 +82,9 @@ MemoryPool::MemoryPool() {}
 
 MemoryPool::~MemoryPool() {}
 
-int64_t MemoryPool::max_memory() const {
-  return -1;
-}
+int64_t MemoryPool::max_memory() const { return -1; }
 
-DefaultMemoryPool::DefaultMemoryPool() : bytes_allocated_(0) {
-  max_memory_ = 0;
-}
+DefaultMemoryPool::DefaultMemoryPool() : bytes_allocated_(0) { max_memory_ = 0; }
 
 Status DefaultMemoryPool::Allocate(int64_t size, uint8_t** out) {
   RETURN_NOT_OK(AllocateAligned(size, out));
@@ -96,7 +92,9 @@ Status DefaultMemoryPool::Allocate(int64_t size, uint8_t** out) {
 
   {
     std::lock_guard<std::mutex> guard(lock_);
-    if (bytes_allocated_ > max_memory_) { max_memory_ = bytes_allocated_.load(); }
+    if (bytes_allocated_ > max_memory_) {
+      max_memory_ = bytes_allocated_.load();
+    }
   }
   return Status::OK();
 }
@@ -128,15 +126,15 @@ Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t
   bytes_allocated_ += new_size - old_size;
   {
     std::lock_guard<std::mutex> guard(lock_);
-    if (bytes_allocated_ > max_memory_) { max_memory_ = bytes_allocated_.load(); }
+    if (bytes_allocated_ > max_memory_) {
+      max_memory_ = bytes_allocated_.load();
+    }
   }
 
   return Status::OK();
 }
 
-int64_t DefaultMemoryPool::bytes_allocated() const {
-  return bytes_allocated_.load();
-}
+int64_t DefaultMemoryPool::bytes_allocated() const { return bytes_allocated_.load(); }
 
 void DefaultMemoryPool::Free(uint8_t* buffer, int64_t size) {
   DCHECK_GE(bytes_allocated_, size);
@@ -150,9 +148,7 @@ void DefaultMemoryPool::Free(uint8_t* buffer, int64_t size) {
   bytes_allocated_ -= size;
 }
 
-int64_t DefaultMemoryPool::max_memory() const {
-  return max_memory_.load();
-}
+int64_t DefaultMemoryPool::max_memory() const { return max_memory_.load(); }
 
 DefaultMemoryPool::~DefaultMemoryPool() {}
 
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index 10a91f5e4e461..049f5a58a6841 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -57,7 +57,7 @@ void CheckArray(const Array& arr, int indent, const char* expected) {
 
 template <typename TYPE, typename C_TYPE>
 void CheckPrimitive(int indent, const std::vector<bool>& is_valid,
-    const std::vector<C_TYPE>& values, const char* expected) {
+                    const std::vector<C_TYPE>& values, const char* expected) {
   std::shared_ptr<Array> array;
   ArrayFromVector<TYPE, C_TYPE>(is_valid, values, &array);
   CheckArray(*array, indent, expected);
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 93f6ff0f363f0..aedad1228dfb2 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -42,7 +42,9 @@ class ArrayPrinter {
       const T& array) {
     const auto data = array.raw_values();
     for (int i = 0; i < array.length(); ++i) {
-      if (i > 0) { (*sink_) << ", "; }
+      if (i > 0) {
+        (*sink_) << ", ";
+      }
       if (array.IsNull(i)) {
         (*sink_) << "null";
       } else {
@@ -56,7 +58,9 @@ class ArrayPrinter {
       const T& array) {
     const auto data = array.raw_values();
     for (int i = 0; i < array.length(); ++i) {
-      if (i > 0) { (*sink_) << ", "; }
+      if (i > 0) {
+        (*sink_) << ", ";
+      }
       if (array.IsNull(i)) {
         Write("null");
       } else {
@@ -71,7 +75,9 @@ class ArrayPrinter {
   WriteDataValues(const T& array) {
     int32_t length;
     for (int i = 0; i < array.length(); ++i) {
-      if (i > 0) { (*sink_) << ", "; }
+      if (i > 0) {
+        (*sink_) << ", ";
+      }
       if (array.IsNull(i)) {
         Write("null");
       } else {
@@ -87,7 +93,9 @@ class ArrayPrinter {
   WriteDataValues(const T& array) {
     int32_t length;
     for (int i = 0; i < array.length(); ++i) {
-      if (i > 0) { (*sink_) << ", "; }
+      if (i > 0) {
+        (*sink_) << ", ";
+      }
       if (array.IsNull(i)) {
         Write("null");
       } else {
@@ -102,7 +110,9 @@ class ArrayPrinter {
   WriteDataValues(const T& array) {
     int32_t width = array.byte_width();
     for (int i = 0; i < array.length(); ++i) {
-      if (i > 0) { (*sink_) << ", "; }
+      if (i > 0) {
+        (*sink_) << ", ";
+      }
       if (array.IsNull(i)) {
         Write("null");
       } else {
@@ -116,7 +126,9 @@ class ArrayPrinter {
   inline typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
   WriteDataValues(const T& array) {
     for (int i = 0; i < array.length(); ++i) {
-      if (i > 0) { (*sink_) << ", "; }
+      if (i > 0) {
+        (*sink_) << ", ";
+      }
       if (array.IsNull(i)) {
         Write("null");
       } else {
@@ -138,7 +150,7 @@ class ArrayPrinter {
   typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value ||
                               std::is_base_of<FixedSizeBinaryArray, T>::value ||
                               std::is_base_of<BinaryArray, T>::value,
-      Status>::type
+                          Status>::type
   Visit(const T& array) {
     OpenArray();
     WriteDataValues(array);
@@ -157,8 +169,8 @@ class ArrayPrinter {
 
     Newline();
     Write("-- value_offsets: ");
-    Int32Array value_offsets(
-        array.length() + 1, array.value_offsets(), nullptr, 0, array.offset());
+    Int32Array value_offsets(array.length() + 1, array.value_offsets(), nullptr, 0,
+                             array.offset());
     RETURN_NOT_OK(PrettyPrint(value_offsets, indent_ + 2, sink_));
 
     Newline();
@@ -170,8 +182,8 @@ class ArrayPrinter {
     return Status::OK();
   }
 
-  Status PrintChildren(
-      const std::vector<std::shared_ptr<Array>>& fields, int64_t offset, int64_t length) {
+  Status PrintChildren(const std::vector<std::shared_ptr<Array>>& fields, int64_t offset,
+                       int64_t length) {
     for (size_t i = 0; i < fields.size(); ++i) {
       Newline();
       std::stringstream ss;
@@ -179,7 +191,9 @@ class ArrayPrinter {
       Write(ss.str());
 
       std::shared_ptr<Array> field = fields[i];
-      if (offset != 0) { field = field->Slice(offset, length); }
+      if (offset != 0) {
+        field = field->Slice(offset, length);
+      }
 
       RETURN_NOT_OK(PrettyPrint(*field, indent_ + 2, sink_));
     }
@@ -207,8 +221,8 @@ class ArrayPrinter {
     if (array.mode() == UnionMode::DENSE) {
       Newline();
       Write("-- value_offsets: ");
-      Int32Array value_offsets(
-          array.length(), array.value_offsets(), nullptr, 0, array.offset());
+      Int32Array value_offsets(array.length(), array.value_offsets(), nullptr, 0,
+                               array.offset());
       RETURN_NOT_OK(PrettyPrint(value_offsets, indent_ + 2, sink_));
     }
 
@@ -247,8 +261,8 @@ Status ArrayPrinter::WriteValidityBitmap(const Array& array) {
   Write("-- is_valid: ");
 
   if (array.null_count() > 0) {
-    BooleanArray is_valid(
-        array.length(), array.null_bitmap(), nullptr, 0, array.offset());
+    BooleanArray is_valid(array.length(), array.null_bitmap(), nullptr, 0,
+                          array.offset());
     return PrettyPrint(is_valid, indent_ + 2, sink_);
   } else {
     Write("all not null");
@@ -256,20 +270,12 @@ Status ArrayPrinter::WriteValidityBitmap(const Array& array) {
   }
 }
 
-void ArrayPrinter::OpenArray() {
-  (*sink_) << "[";
-}
-void ArrayPrinter::CloseArray() {
-  (*sink_) << "]";
-}
+void ArrayPrinter::OpenArray() { (*sink_) << "["; }
+void ArrayPrinter::CloseArray() { (*sink_) << "]"; }
 
-void ArrayPrinter::Write(const char* data) {
-  (*sink_) << data;
-}
+void ArrayPrinter::Write(const char* data) { (*sink_) << data; }
 
-void ArrayPrinter::Write(const std::string& data) {
-  (*sink_) << data;
-}
+void ArrayPrinter::Write(const std::string& data) { (*sink_) << data; }
 
 void ArrayPrinter::Newline() {
   (*sink_) << "\n";
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index d40609fe3fad2..462bdb7b7d744 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -147,8 +147,8 @@ static inline PyArray_Descr* GetSafeNumPyDtype(int type) {
     return PyArray_DescrFromType(type);
   }
 }
-static inline PyObject* NewArray1DFromType(
-    DataType* arrow_type, int type, int64_t length, void* data) {
+static inline PyObject* NewArray1DFromType(DataType* arrow_type, int type, int64_t length,
+                                           void* data) {
   npy_intp dims[1] = {length};
 
   PyArray_Descr* descr = GetSafeNumPyDtype(type);
@@ -159,7 +159,8 @@ static inline PyObject* NewArray1DFromType(
 
   set_numpy_metadata(type, arrow_type, descr);
   return PyArray_NewFromDescr(&PyArray_Type, descr, 1, dims, nullptr, data,
-      NPY_ARRAY_OWNDATA | NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEABLE, nullptr);
+                              NPY_ARRAY_OWNDATA | NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEABLE,
+                              nullptr);
 }
 
 class PandasBlock {
@@ -188,7 +189,7 @@ class PandasBlock {
 
   virtual Status Allocate() = 0;
   virtual Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) = 0;
+                       int64_t rel_placement) = 0;
 
   PyObject* block_arr() const { return block_arr_.obj(); }
 
@@ -408,7 +409,9 @@ inline Status ConvertFixedSizeBinary(const ChunkedArray& data, PyObject** out_va
 
 inline Status ConvertStruct(const ChunkedArray& data, PyObject** out_values) {
   PyAcquireGIL lock;
-  if (data.num_chunks() <= 0) { return Status::OK(); }
+  if (data.num_chunks() <= 0) {
+    return Status::OK();
+  }
   // ChunkedArray has at least one chunk
   auto arr = static_cast<const StructArray*>(data.chunk(0).get());
   // Use it to cache the struct type and number of fields for all chunks
@@ -467,8 +470,8 @@ inline Status ConvertStruct(const ChunkedArray& data, PyObject** out_values) {
 }
 
 template <typename ArrowType>
-inline Status ConvertListsLike(
-    const std::shared_ptr<Column>& col, PyObject** out_values) {
+inline Status ConvertListsLike(const std::shared_ptr<Column>& col,
+                               PyObject** out_values) {
   const ChunkedArray& data = *col->data().get();
   auto list_type = std::static_pointer_cast<ListType>(col->type());
 
@@ -532,8 +535,8 @@ inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_
 }
 
 template <typename InType, typename OutType>
-inline void ConvertNumericNullableCast(
-    const ChunkedArray& data, OutType na_value, OutType* out_values) {
+inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_value,
+                                       OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
@@ -602,8 +605,8 @@ Status ValidateDecimalPrecision(int precision) {
 }
 
 template <typename T>
-Status RawDecimalToString(
-    const uint8_t* bytes, int precision, int scale, std::string* result) {
+Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
+                          std::string* result) {
   DCHECK_NE(bytes, nullptr);
   DCHECK_NE(result, nullptr);
   RETURN_NOT_OK(ValidateDecimalPrecision<T>(precision));
@@ -613,13 +616,13 @@ Status RawDecimalToString(
   return Status::OK();
 }
 
-template Status RawDecimalToString<int32_t>(
-    const uint8_t*, int, int, std::string* result);
-template Status RawDecimalToString<int64_t>(
-    const uint8_t*, int, int, std::string* result);
+template Status RawDecimalToString<int32_t>(const uint8_t*, int, int,
+                                            std::string* result);
+template Status RawDecimalToString<int64_t>(const uint8_t*, int, int,
+                                            std::string* result);
 
 Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
-    bool is_negative, std::string* result) {
+                          bool is_negative, std::string* result) {
   DCHECK_NE(bytes, nullptr);
   DCHECK_NE(result, nullptr);
   RETURN_NOT_OK(ValidateDecimalPrecision<boost::multiprecision::int128_t>(precision));
@@ -684,7 +687,7 @@ class ObjectBlock : public PandasBlock {
   Status Allocate() override { return AllocateNDArray(NPY_OBJECT); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
+               int64_t rel_placement) override {
     Type::type type = col->type()->id();
 
     PyObject** out_buffer =
@@ -753,7 +756,7 @@ class IntBlock : public PandasBlock {
   }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
+               int64_t rel_placement) override {
     Type::type type = col->type()->id();
 
     C_TYPE* out_buffer =
@@ -789,7 +792,7 @@ class Float32Block : public PandasBlock {
   Status Allocate() override { return AllocateNDArray(NPY_FLOAT32); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
+               int64_t rel_placement) override {
     Type::type type = col->type()->id();
 
     if (type != Type::FLOAT) {
@@ -813,7 +816,7 @@ class Float64Block : public PandasBlock {
   Status Allocate() override { return AllocateNDArray(NPY_FLOAT64); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
+               int64_t rel_placement) override {
     Type::type type = col->type()->id();
 
     double* out_buffer =
@@ -868,7 +871,7 @@ class BoolBlock : public PandasBlock {
   Status Allocate() override { return AllocateNDArray(NPY_BOOL); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
+               int64_t rel_placement) override {
     Type::type type = col->type()->id();
 
     if (type != Type::BOOL) {
@@ -903,7 +906,7 @@ class DatetimeBlock : public PandasBlock {
   Status Allocate() override { return AllocateDatetime(2); }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
+               int64_t rel_placement) override {
     Type::type type = col->type()->id();
 
     int64_t* out_buffer =
@@ -981,14 +984,14 @@ class CategoricalBlock : public PandasBlock {
     constexpr int npy_type = arrow_traits<ARROW_INDEX_TYPE>::npy_type;
 
     if (!(npy_type == NPY_INT8 || npy_type == NPY_INT16 || npy_type == NPY_INT32 ||
-            npy_type == NPY_INT64)) {
+          npy_type == NPY_INT64)) {
       return Status::Invalid("Category indices must be signed integers");
     }
     return AllocateNDArray(npy_type, 1);
   }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-      int64_t rel_placement) override {
+               int64_t rel_placement) override {
     using T = typename arrow_traits<ARROW_INDEX_TYPE>::T;
 
     T* out_values = reinterpret_cast<T*>(block_data_) + rel_placement * num_rows_;
@@ -1036,7 +1039,7 @@ class CategoricalBlock : public PandasBlock {
 };
 
 Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
-    std::shared_ptr<PandasBlock>* block) {
+                 std::shared_ptr<PandasBlock>* block) {
 #define BLOCK_CASE(NAME, TYPE)                              \
   case PandasBlock::NAME:                                   \
     *block = std::make_shared<TYPE>(num_rows, num_columns); \
@@ -1066,7 +1069,8 @@ Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
 }
 
 static inline Status MakeCategoricalBlock(const std::shared_ptr<DataType>& type,
-    int64_t num_rows, std::shared_ptr<PandasBlock>* block) {
+                                          int64_t num_rows,
+                                          std::shared_ptr<PandasBlock>* block) {
   // All categoricals become a block with a single column
   auto dict_type = static_cast<const DictionaryType*>(type.get());
   switch (dict_type->index_type()->id()) {
@@ -1259,7 +1263,9 @@ class DataFrameBlockCreator {
         block = it->second;
       } else {
         auto it = this->blocks_.find(output_type);
-        if (it == this->blocks_.end()) { return Status::KeyError("No block allocated"); }
+        if (it == this->blocks_.end()) {
+          return Status::KeyError("No block allocated");
+        }
         block = it->second;
       }
       return block->Write(col, i, rel_placement);
@@ -1286,7 +1292,9 @@ class DataFrameBlockCreator {
               int column_num;
               while (!error_occurred) {
                 column_num = task_counter.fetch_add(1);
-                if (column_num >= this->table_->num_columns()) { break; }
+                if (column_num >= this->table_->num_columns()) {
+                  break;
+                }
                 Status s = WriteColumn(column_num);
                 if (!s.ok()) {
                   std::lock_guard<std::mutex> lock(error_mtx);
@@ -1301,7 +1309,9 @@ class DataFrameBlockCreator {
         thread.join();
       }
 
-      if (error_occurred) { return error; }
+      if (error_occurred) {
+        return error;
+      }
     }
     return Status::OK();
   }
@@ -1310,7 +1320,9 @@ class DataFrameBlockCreator {
     for (const auto& it : blocks) {
       PyObject* item;
       RETURN_NOT_OK(it.second->GetPyResult(&item));
-      if (PyList_Append(list, item) < 0) { RETURN_IF_PYERROR(); }
+      if (PyList_Append(list, item) < 0) {
+        RETURN_IF_PYERROR();
+      }
 
       // ARROW-1017; PyList_Append increments object refcount
       Py_DECREF(item);
@@ -1432,7 +1444,7 @@ class ArrowDeserializer {
   template <typename Type>
   typename std::enable_if<std::is_base_of<DateType, Type>::value ||
                               std::is_base_of<TimestampType, Type>::value,
-      Status>::type
+                          Status>::type
   Visit(const Type& type) {
     constexpr int TYPE = Type::type_id;
     using traits = arrow_traits<TYPE>;
@@ -1603,22 +1615,22 @@ class ArrowDeserializer {
   PyObject* result_;
 };
 
-Status ConvertArrayToPandas(
-    const std::shared_ptr<Array>& arr, PyObject* py_ref, PyObject** out) {
+Status ConvertArrayToPandas(const std::shared_ptr<Array>& arr, PyObject* py_ref,
+                            PyObject** out) {
   static std::string dummy_name = "dummy";
   auto field = std::make_shared<Field>(dummy_name, arr->type());
   auto col = std::make_shared<Column>(field, arr);
   return ConvertColumnToPandas(col, py_ref, out);
 }
 
-Status ConvertColumnToPandas(
-    const std::shared_ptr<Column>& col, PyObject* py_ref, PyObject** out) {
+Status ConvertColumnToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
+                             PyObject** out) {
   ArrowDeserializer converter(col, py_ref);
   return converter.Convert(out);
 }
 
-Status ConvertTableToPandas(
-    const std::shared_ptr<Table>& table, int nthreads, PyObject** out) {
+Status ConvertTableToPandas(const std::shared_ptr<Table>& table, int nthreads,
+                            PyObject** out) {
   DataFrameBlockCreator helper(table);
   return helper.Convert(nthreads, out);
 }
diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h
index c606dcbbe0aa7..5a99274a33ee0 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.h
+++ b/cpp/src/arrow/python/arrow_to_pandas.h
@@ -40,12 +40,12 @@ class Table;
 namespace py {
 
 ARROW_EXPORT
-Status ConvertArrayToPandas(
-    const std::shared_ptr<Array>& arr, PyObject* py_ref, PyObject** out);
+Status ConvertArrayToPandas(const std::shared_ptr<Array>& arr, PyObject* py_ref,
+                            PyObject** out);
 
 ARROW_EXPORT
-Status ConvertColumnToPandas(
-    const std::shared_ptr<Column>& col, PyObject* py_ref, PyObject** out);
+Status ConvertColumnToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
+                             PyObject** out);
 
 struct PandasOptions {
   bool strings_to_categorical;
@@ -58,8 +58,8 @@ struct PandasOptions {
 //
 // tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2])
 ARROW_EXPORT
-Status ConvertTableToPandas(
-    const std::shared_ptr<Table>& table, int nthreads, PyObject** out);
+Status ConvertTableToPandas(const std::shared_ptr<Table>& table, int nthreads,
+                            PyObject** out);
 
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index a76b6ba25531c..6eaa37fb8ca93 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -44,8 +44,8 @@ static inline bool IsPyInteger(PyObject* obj) {
 #endif
 }
 
-Status InvalidConversion(
-    PyObject* obj, const std::string& expected_types, std::ostream* out) {
+Status InvalidConversion(PyObject* obj, const std::string& expected_types,
+                         std::ostream* out) {
   OwnedRef type(PyObject_Type(obj));
   RETURN_IF_PYERROR();
   DCHECK_NE(type.obj(), nullptr);
@@ -161,7 +161,9 @@ class SeqVisitor {
 
   // co-recursive with VisitElem
   Status Visit(PyObject* obj, int level = 0) {
-    if (level > max_nesting_level_) { max_nesting_level_ = level; }
+    if (level > max_nesting_level_) {
+      max_nesting_level_ = level;
+    }
     // Loop through either a sequence or an iterator.
     if (PySequence_Check(obj)) {
       Py_ssize_t size = PySequence_Size(obj);
@@ -226,7 +228,9 @@ class SeqVisitor {
   int max_observed_level() const {
     int result = 0;
     for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
-      if (nesting_histogram_[i] > 0) { result = i; }
+      if (nesting_histogram_[i] > 0) {
+        result = i;
+      }
     }
     return result;
   }
@@ -235,7 +239,9 @@ class SeqVisitor {
   int num_nesting_levels() const {
     int result = 0;
     for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
-      if (nesting_histogram_[i] > 0) { ++result; }
+      if (nesting_histogram_[i] > 0) {
+        ++result;
+      }
     }
     return result;
   }
@@ -300,13 +306,15 @@ Status InferArrowType(PyObject* obj, std::shared_ptr<DataType>* out_type) {
   RETURN_NOT_OK(seq_visitor.Validate());
 
   *out_type = seq_visitor.GetType();
-  if (*out_type == nullptr) { return Status::TypeError("Unable to determine data type"); }
+  if (*out_type == nullptr) {
+    return Status::TypeError("Unable to determine data type");
+  }
 
   return Status::OK();
 }
 
-Status InferArrowTypeAndSize(
-    PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
+Status InferArrowTypeAndSize(PyObject* obj, int64_t* size,
+                             std::shared_ptr<DataType>* out_type) {
   RETURN_NOT_OK(InferArrowSize(obj, size));
 
   // For 0-length sequences, refuse to guess
@@ -372,7 +380,9 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
         RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
         ++i;
       }
-      if (size != i) { RETURN_NOT_OK(this->typed_builder_->Resize(i)); }
+      if (size != i) {
+        RETURN_NOT_OK(this->typed_builder_->Resize(i));
+      }
     } else {
       return Status::TypeError("Object is not a sequence or iterable");
     }
@@ -487,8 +497,9 @@ class FixedWidthBytesConverter
   inline Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
-    Py_ssize_t expected_length = std::dynamic_pointer_cast<FixedSizeBinaryType>(
-        typed_builder_->type())->byte_width();
+    Py_ssize_t expected_length =
+        std::dynamic_pointer_cast<FixedSizeBinaryType>(typed_builder_->type())
+            ->byte_width();
     if (item.obj() == Py_None) {
       RETURN_NOT_OK(typed_builder_->AppendNull());
       return Status::OK();
@@ -636,7 +647,7 @@ Status ListConverter::Init(ArrayBuilder* builder) {
 }
 
 Status AppendPySequence(PyObject* obj, int64_t size,
-    const std::shared_ptr<DataType>& type, ArrayBuilder* builder) {
+                        const std::shared_ptr<DataType>& type, ArrayBuilder* builder) {
   PyDateTime_IMPORT;
   std::shared_ptr<SeqConverter> converter = GetConverter(type);
   if (converter == nullptr) {
@@ -656,7 +667,7 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
 }
 
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
-    const std::shared_ptr<DataType>& type, int64_t size) {
+                         const std::shared_ptr<DataType>& type, int64_t size) {
   // Handle NA / NullType case
   if (type->id() == Type::NA) {
     out->reset(new NullArray(size));
@@ -671,7 +682,7 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
 }
 
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
-    const std::shared_ptr<DataType>& type) {
+                         const std::shared_ptr<DataType>& type) {
   int64_t size;
   RETURN_NOT_OK(InferArrowSize(obj, &size));
   return ConvertPySequence(obj, pool, out, type, size);
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index 4f84fbb7caca9..cde7a1bd4cfdc 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -39,14 +39,15 @@ class Status;
 
 namespace py {
 
-ARROW_EXPORT arrow::Status InferArrowType(
-    PyObject* obj, std::shared_ptr<arrow::DataType>* out_type);
+ARROW_EXPORT arrow::Status InferArrowType(PyObject* obj,
+                                          std::shared_ptr<arrow::DataType>* out_type);
 ARROW_EXPORT arrow::Status InferArrowTypeAndSize(
     PyObject* obj, int64_t* size, std::shared_ptr<arrow::DataType>* out_type);
 ARROW_EXPORT arrow::Status InferArrowSize(PyObject* obj, int64_t* size);
 
 ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj, int64_t size,
-    const std::shared_ptr<arrow::DataType>& type, arrow::ArrayBuilder* builder);
+                                            const std::shared_ptr<arrow::DataType>& type,
+                                            arrow::ArrayBuilder* builder);
 
 // Type and size inference
 ARROW_EXPORT
@@ -55,19 +56,19 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
 // Size inference
 ARROW_EXPORT
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
-    const std::shared_ptr<DataType>& type);
+                         const std::shared_ptr<DataType>& type);
 
 // No inference
 ARROW_EXPORT
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
-    const std::shared_ptr<DataType>& type, int64_t size);
+                         const std::shared_ptr<DataType>& type, int64_t size);
 
 ARROW_EXPORT
-Status InvalidConversion(
-    PyObject* obj, const std::string& expected_type_name, std::ostream* out);
+Status InvalidConversion(PyObject* obj, const std::string& expected_type_name,
+                         std::ostream* out);
 
-ARROW_EXPORT Status CheckPythonBytesAreFixedLength(
-    PyObject* obj, Py_ssize_t expected_length);
+ARROW_EXPORT Status CheckPythonBytesAreFixedLength(PyObject* obj,
+                                                   Py_ssize_t expected_length);
 
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/config.cc b/cpp/src/arrow/python/config.cc
index 3cec7c41a2f31..92ca9db9cc391 100644
--- a/cpp/src/arrow/python/config.cc
+++ b/cpp/src/arrow/python/config.cc
@@ -16,8 +16,6 @@
 // under the License.
 
 #include "arrow/python/platform.h"
-#include <datetime.h>
-
 #include "arrow/python/config.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 76ec3a1ba8746..164e42e52e48e 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -89,8 +89,8 @@ Status PythonDecimalToString(PyObject* python_decimal, std::string* out) {
   return Status::OK();
 }
 
-Status InferDecimalPrecisionAndScale(
-    PyObject* python_decimal, int* precision, int* scale) {
+Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision,
+                                     int* scale) {
   // Call Python's str(decimal_object)
   OwnedRef str_obj(PyObject_Str(python_decimal));
   RETURN_IF_PYERROR();
@@ -102,12 +102,12 @@ Status InferDecimalPrecisionAndScale(
   auto size = str.size;
 
   std::string c_string(bytes, size);
-  return FromString(
-      c_string, static_cast<decimal::Decimal32*>(nullptr), precision, scale);
+  return FromString(c_string, static_cast<decimal::Decimal32*>(nullptr), precision,
+                    scale);
 }
 
-Status DecimalFromString(
-    PyObject* decimal_constructor, const std::string& decimal_string, PyObject** out) {
+Status DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string,
+                         PyObject** out) {
   DCHECK_NE(decimal_constructor, nullptr);
   DCHECK_NE(out, nullptr);
 
@@ -117,8 +117,8 @@ Status DecimalFromString(
   auto string_bytes = decimal_string.c_str();
   DCHECK_NE(string_bytes, nullptr);
 
-  *out = PyObject_CallFunction(
-      decimal_constructor, const_cast<char*>("s#"), string_bytes, string_size);
+  *out = PyObject_CallFunction(decimal_constructor, const_cast<char*>("s#"), string_bytes,
+                               string_size);
   RETURN_IF_PYERROR();
   return Status::OK();
 }
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index e0656699ce4c2..8b8c6673c8ebb 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -36,16 +36,17 @@ class OwnedRef;
 ARROW_EXPORT std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
 
 Status ARROW_EXPORT ImportModule(const std::string& module_name, OwnedRef* ref);
-Status ARROW_EXPORT ImportFromModule(
-    const OwnedRef& module, const std::string& module_name, OwnedRef* ref);
+Status ARROW_EXPORT ImportFromModule(const OwnedRef& module,
+                                     const std::string& module_name, OwnedRef* ref);
 
 Status ARROW_EXPORT PythonDecimalToString(PyObject* python_decimal, std::string* out);
 
-Status ARROW_EXPORT InferDecimalPrecisionAndScale(
-    PyObject* python_decimal, int* precision = nullptr, int* scale = nullptr);
+Status ARROW_EXPORT InferDecimalPrecisionAndScale(PyObject* python_decimal,
+                                                  int* precision = nullptr,
+                                                  int* scale = nullptr);
 
-Status ARROW_EXPORT DecimalFromString(
-    PyObject* decimal_constructor, const std::string& decimal_string, PyObject** out);
+Status ARROW_EXPORT DecimalFromString(PyObject* decimal_constructor,
+                                      const std::string& decimal_string, PyObject** out);
 
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/init.cc b/cpp/src/arrow/python/init.cc
index db648915465a8..dba293bbe2366 100644
--- a/cpp/src/arrow/python/init.cc
+++ b/cpp/src/arrow/python/init.cc
@@ -21,6 +21,4 @@
 #include "arrow/python/init.h"
 #include "arrow/python/numpy_interop.h"
 
-int arrow_init_numpy() {
-  return arrow::py::import_numpy();
-}
+int arrow_init_numpy() { return arrow::py::import_numpy(); }
diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index a7193854c4d01..4c73fd6401cb6 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -33,23 +33,19 @@ namespace py {
 // ----------------------------------------------------------------------
 // Python file
 
-PythonFile::PythonFile(PyObject* file) : file_(file) {
-  Py_INCREF(file_);
-}
+PythonFile::PythonFile(PyObject* file) : file_(file) { Py_INCREF(file_); }
 
-PythonFile::~PythonFile() {
-  Py_DECREF(file_);
-}
+PythonFile::~PythonFile() { Py_DECREF(file_); }
 
 // This is annoying: because C++11 does not allow implicit conversion of string
 // literals to non-const char*, we need to go through some gymnastics to use
 // PyObject_CallMethod without a lot of pain (its arguments are non-const
 // char*)
 template <typename... ArgTypes>
-static inline PyObject* cpp_PyObject_CallMethod(
-    PyObject* obj, const char* method_name, const char* argspec, ArgTypes... args) {
-  return PyObject_CallMethod(
-      obj, const_cast<char*>(method_name), const_cast<char*>(argspec), args...);
+static inline PyObject* cpp_PyObject_CallMethod(PyObject* obj, const char* method_name,
+                                                const char* argspec, ArgTypes... args) {
+  return PyObject_CallMethod(obj, const_cast<char*>(method_name),
+                             const_cast<char*>(argspec), args...);
 }
 
 Status PythonFile::Close() {
@@ -103,9 +99,7 @@ Status PythonFile::Tell(int64_t* position) {
 // ----------------------------------------------------------------------
 // Seekable input stream
 
-PyReadableFile::PyReadableFile(PyObject* file) {
-  file_.reset(new PythonFile(file));
-}
+PyReadableFile::PyReadableFile(PyObject* file) { file_.reset(new PythonFile(file)); }
 
 PyReadableFile::~PyReadableFile() {}
 
@@ -167,9 +161,7 @@ Status PyReadableFile::GetSize(int64_t* size) {
   return Status::OK();
 }
 
-bool PyReadableFile::supports_zero_copy() const {
-  return false;
-}
+bool PyReadableFile::supports_zero_copy() const { return false; }
 
 // ----------------------------------------------------------------------
 // Output stream
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index c391b5d7a1018..95d63b8fecb5b 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -38,7 +38,7 @@ namespace py {
 bool is_contiguous(PyObject* array) {
   if (PyArray_Check(array)) {
     return (PyArray_FLAGS(reinterpret_cast<PyArrayObject*>(array)) &
-               (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS)) != 0;
+            (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS)) != 0;
   } else {
     return false;
   }
@@ -49,8 +49,12 @@ int cast_npy_type_compat(int type_num) {
 // U/LONGLONG to U/INT64 so things work properly.
 
 #if (NPY_INT64 == NPY_LONGLONG) && (NPY_SIZEOF_LONGLONG == 8)
-  if (type_num == NPY_LONGLONG) { type_num = NPY_INT64; }
-  if (type_num == NPY_ULONGLONG) { type_num = NPY_UINT64; }
+  if (type_num == NPY_LONGLONG) {
+    type_num = NPY_INT64;
+  }
+  if (type_num == NPY_ULONGLONG) {
+    type_num = NPY_UINT64;
+  }
 #endif
 
   return type_num;
@@ -66,13 +70,13 @@ NumPyBuffer::NumPyBuffer(PyObject* ao) : Buffer(nullptr, 0) {
     size_ = PyArray_SIZE(ndarray) * PyArray_DESCR(ndarray)->elsize;
     capacity_ = size_;
 
-    if (PyArray_FLAGS(ndarray) & NPY_ARRAY_WRITEABLE) { is_mutable_ = true; }
+    if (PyArray_FLAGS(ndarray) & NPY_ARRAY_WRITEABLE) {
+      is_mutable_ = true;
+    }
   }
 }
 
-NumPyBuffer::~NumPyBuffer() {
-  Py_XDECREF(arr_);
-}
+NumPyBuffer::~NumPyBuffer() { Py_XDECREF(arr_); }
 
 #define TO_ARROW_TYPE_CASE(NPY_NAME, FACTORY) \
   case NPY_##NPY_NAME:                        \
@@ -198,7 +202,9 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
 #undef TO_ARROW_TYPE_CASE
 
 Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>* out) {
-  if (!PyArray_Check(ao)) { return Status::TypeError("Did not pass ndarray object"); }
+  if (!PyArray_Check(ao)) {
+    return Status::TypeError("Did not pass ndarray object");
+  }
 
   PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(ao);
 
@@ -242,18 +248,27 @@ Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out) {
   }
 
   const void* immutable_data = nullptr;
-  if (tensor.data()) { immutable_data = tensor.data()->data(); }
+  if (tensor.data()) {
+    immutable_data = tensor.data()->data();
+  }
 
   // Remove const =(
   void* mutable_data = const_cast<void*>(immutable_data);
 
   int array_flags = 0;
-  if (tensor.is_row_major()) { array_flags |= NPY_ARRAY_C_CONTIGUOUS; }
-  if (tensor.is_column_major()) { array_flags |= NPY_ARRAY_F_CONTIGUOUS; }
-  if (tensor.is_mutable()) { array_flags |= NPY_ARRAY_WRITEABLE; }
+  if (tensor.is_row_major()) {
+    array_flags |= NPY_ARRAY_C_CONTIGUOUS;
+  }
+  if (tensor.is_column_major()) {
+    array_flags |= NPY_ARRAY_F_CONTIGUOUS;
+  }
+  if (tensor.is_mutable()) {
+    array_flags |= NPY_ARRAY_WRITEABLE;
+  }
 
-  PyObject* result = PyArray_NewFromDescr(&PyArray_Type, dtype, tensor.ndim(),
-      npy_shape.data(), npy_strides.data(), mutable_data, array_flags, nullptr);
+  PyObject* result =
+      PyArray_NewFromDescr(&PyArray_Type, dtype, tensor.ndim(), npy_shape.data(),
+                           npy_strides.data(), mutable_data, array_flags, nullptr);
   RETURN_IF_PYERROR()
 
   if (base != Py_None) {
diff --git a/cpp/src/arrow/python/numpy_convert.h b/cpp/src/arrow/python/numpy_convert.h
index a486646cdec64..7b3b3b7c9a2a0 100644
--- a/cpp/src/arrow/python/numpy_convert.h
+++ b/cpp/src/arrow/python/numpy_convert.h
@@ -63,8 +63,8 @@ Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out);
 ARROW_EXPORT
 Status GetNumPyType(const DataType& type, int* type_num);
 
-ARROW_EXPORT Status NdarrayToTensor(
-    MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>* out);
+ARROW_EXPORT Status NdarrayToTensor(MemoryPool* pool, PyObject* ao,
+                                    std::shared_ptr<Tensor>* out);
 
 ARROW_EXPORT Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out);
 
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 1368c3605a4e3..be5634b53bbfe 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -75,9 +75,7 @@ static inline bool PyObject_is_string(const PyObject* obj) {
 #endif
 }
 
-static inline bool PyObject_is_float(const PyObject* obj) {
-  return PyFloat_Check(obj);
-}
+static inline bool PyObject_is_float(const PyObject* obj) { return PyFloat_Check(obj); }
 
 static inline bool PyObject_is_integer(const PyObject* obj) {
   return (!PyBool_Check(obj)) && PyArray_IsIntegerScalar(obj);
@@ -120,8 +118,8 @@ static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap
 }
 
 template <int TYPE>
-static int64_t ValuesToValidBytes(
-    const void* data, int64_t length, uint8_t* valid_bytes) {
+static int64_t ValuesToValidBytes(const void* data, int64_t length,
+                                  uint8_t* valid_bytes) {
   typedef npy_traits<TYPE> traits;
   typedef typename traits::value_type T;
 
@@ -163,7 +161,8 @@ constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max();
 /// be length of arr if fully consumed
 /// \param[out] have_bytes true if we encountered any PyBytes object
 static Status AppendObjectStrings(PyArrayObject* arr, PyArrayObject* mask, int64_t offset,
-    StringBuilder* builder, int64_t* end_offset, bool* have_bytes) {
+                                  StringBuilder* builder, int64_t* end_offset,
+                                  bool* have_bytes) {
   PyObject* obj;
 
   Ndarray1DIndexer<PyObject*> objects(arr);
@@ -210,8 +209,9 @@ static Status AppendObjectStrings(PyArrayObject* arr, PyArrayObject* mask, int64
 }
 
 static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mask,
-    int byte_width, int64_t offset, FixedSizeBinaryBuilder* builder,
-    int64_t* end_offset) {
+                                          int byte_width, int64_t offset,
+                                          FixedSizeBinaryBuilder* builder,
+                                          int64_t* end_offset) {
   PyObject* obj;
 
   Ndarray1DIndexer<PyObject*> objects(arr);
@@ -245,8 +245,8 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
     }
 
     RETURN_NOT_OK(CheckPythonBytesAreFixedLength(obj, byte_width));
-    if (ARROW_PREDICT_FALSE(
-            builder->value_data_length() + byte_width > kBinaryMemoryLimit)) {
+    if (ARROW_PREDICT_FALSE(builder->value_data_length() + byte_width >
+                            kBinaryMemoryLimit)) {
       break;
     }
     RETURN_NOT_OK(
@@ -263,13 +263,15 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
 
 class PandasConverter {
  public:
-  PandasConverter(
-      MemoryPool* pool, PyObject* ao, PyObject* mo, const std::shared_ptr<DataType>& type)
+  PandasConverter(MemoryPool* pool, PyObject* ao, PyObject* mo,
+                  const std::shared_ptr<DataType>& type)
       : pool_(pool),
         type_(type),
         arr_(reinterpret_cast<PyArrayObject*>(ao)),
         mask_(nullptr) {
-    if (mo != nullptr && mo != Py_None) { mask_ = reinterpret_cast<PyArrayObject*>(mo); }
+    if (mo != nullptr && mo != Py_None) {
+      mask_ = reinterpret_cast<PyArrayObject*>(mo);
+    }
     length_ = static_cast<int64_t>(PyArray_SIZE(arr_));
   }
 
@@ -315,7 +317,9 @@ class PandasConverter {
   Status VisitNative() {
     using traits = arrow_traits<ArrowType::type_id>;
 
-    if (mask_ != nullptr || traits::supports_nulls) { RETURN_NOT_OK(InitNullBitmap()); }
+    if (mask_ != nullptr || traits::supports_nulls) {
+      RETURN_NOT_OK(InitNullBitmap());
+    }
 
     std::shared_ptr<Buffer> data;
     RETURN_NOT_OK(ConvertData<ArrowType>(&data));
@@ -337,7 +341,7 @@ class PandasConverter {
   template <typename T>
   typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
                               std::is_same<BooleanType, T>::value,
-      Status>::type
+                          Status>::type
   Visit(const T& type) {
     return VisitNative<T>();
   }
@@ -373,7 +377,9 @@ class PandasConverter {
       return Status::Invalid("only handle 1-dimensional arrays");
     }
 
-    if (type_ == nullptr) { return Status::Invalid("Must pass data type"); }
+    if (type_ == nullptr) {
+      return Status::Invalid("Must pass data type");
+    }
 
     // Visit the type to perform conversion
     return VisitTypeInline(*type_, this);
@@ -385,8 +391,8 @@ class PandasConverter {
   // Conversion logic for various object dtype arrays
 
   template <int ITEM_TYPE, typename ArrowType>
-  Status ConvertTypedLists(
-      const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list);
+  Status ConvertTypedLists(const std::shared_ptr<DataType>& type, ListBuilder* builder,
+                           PyObject* list);
 
   template <typename ArrowType>
   Status ConvertDates();
@@ -397,8 +403,8 @@ class PandasConverter {
   Status ConvertObjectFixedWidthBytes(const std::shared_ptr<DataType>& type);
   Status ConvertObjectIntegers();
   Status ConvertLists(const std::shared_ptr<DataType>& type);
-  Status ConvertLists(
-      const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list);
+  Status ConvertLists(const std::shared_ptr<DataType>& type, ListBuilder* builder,
+                      PyObject* list);
   Status ConvertObjects();
   Status ConvertDecimals();
   Status ConvertTimes();
@@ -428,12 +434,14 @@ void CopyStrided(T* input_data, int64_t length, int64_t stride, T* output_data)
 }
 
 template <>
-void CopyStrided<PyObject*>(
-    PyObject** input_data, int64_t length, int64_t stride, PyObject** output_data) {
+void CopyStrided<PyObject*>(PyObject** input_data, int64_t length, int64_t stride,
+                            PyObject** output_data) {
   int64_t j = 0;
   for (int64_t i = 0; i < length; ++i) {
     output_data[i] = input_data[j];
-    if (output_data[i] != nullptr) { Py_INCREF(output_data[i]); }
+    if (output_data[i] != nullptr) {
+      Py_INCREF(output_data[i]);
+    }
     j += stride;
   }
 }
@@ -458,7 +466,7 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
     auto new_buffer = std::make_shared<PoolBuffer>(pool_);
     RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length_));
     CopyStrided(reinterpret_cast<T*>(PyArray_DATA(arr_)), length_, stride_elements,
-        reinterpret_cast<T*>(new_buffer->mutable_data()));
+                reinterpret_cast<T*>(new_buffer->mutable_data()));
     *data = new_buffer;
   } else {
     // Can zero-copy
@@ -479,7 +487,9 @@ inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>*
 
   memset(bitmap, 0, nbytes);
   for (int i = 0; i < length_; ++i) {
-    if (values[i] > 0) { BitUtil::SetBit(bitmap, i); }
+    if (values[i] > 0) {
+      BitUtil::SetBit(bitmap, i);
+    }
   }
 
   *data = buffer;
@@ -913,8 +923,8 @@ Status LoopPySequence(PyObject* sequence, T func) {
 }
 
 template <int ITEM_TYPE, typename ArrowType>
-inline Status PandasConverter::ConvertTypedLists(
-    const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
+inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>& type,
+                                                 ListBuilder* builder, PyObject* list) {
   typedef npy_traits<ITEM_TYPE> traits;
   typedef typename traits::value_type T;
   typedef typename traits::BuilderClass BuilderT;
@@ -1002,8 +1012,8 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
       RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
 
       int64_t offset = 0;
-      RETURN_NOT_OK(AppendObjectStrings(
-          numpy_array, nullptr, 0, value_builder, &offset, &have_bytes));
+      RETURN_NOT_OK(AppendObjectStrings(numpy_array, nullptr, 0, value_builder, &offset,
+                                        &have_bytes));
       if (offset < PyArray_SIZE(numpy_array)) {
         return Status::Invalid("Array cell value exceeded 2GB");
       }
@@ -1032,8 +1042,8 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type, builder, list); \
   }
 
-Status PandasConverter::ConvertLists(
-    const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
+Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type,
+                                     ListBuilder* builder, PyObject* list) {
   switch (type->id()) {
     LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
     LIST_CASE(INT8, NPY_INT8, Int8Type)
@@ -1080,7 +1090,7 @@ Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
 }
 
 Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
+                     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
   PandasConverter converter(pool, ao, mo, type);
   RETURN_NOT_OK(converter.Convert());
   *out = converter.result()[0];
@@ -1088,7 +1098,8 @@ Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
 }
 
 Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<ChunkedArray>* out) {
+                            const std::shared_ptr<DataType>& type,
+                            std::shared_ptr<ChunkedArray>* out) {
   PandasConverter converter(pool, ao, mo, type);
   RETURN_NOT_OK(converter.ConvertObjects());
   *out = std::make_shared<ChunkedArray>(converter.result());
diff --git a/cpp/src/arrow/python/pandas_to_arrow.h b/cpp/src/arrow/python/pandas_to_arrow.h
index 8f1862470bc94..3e655ba3feec0 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.h
+++ b/cpp/src/arrow/python/pandas_to_arrow.h
@@ -38,7 +38,7 @@ namespace py {
 
 ARROW_EXPORT
 Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
+                     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
 
 /// Convert dtype=object arrays. If target data type is not known, pass a type
 /// with nullptr
@@ -50,7 +50,8 @@ Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
 /// \param[out] out a ChunkedArray, to accommodate chunked output
 ARROW_EXPORT
 Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-    const std::shared_ptr<DataType>& type, std::shared_ptr<ChunkedArray>* out);
+                            const std::shared_ptr<DataType>& type,
+                            std::shared_ptr<ChunkedArray>* out);
 
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/platform.h b/cpp/src/arrow/python/platform.h
index a354b38f04cea..ae394695fac0d 100644
--- a/cpp/src/arrow/python/platform.h
+++ b/cpp/src/arrow/python/platform.h
@@ -23,6 +23,7 @@
 
 #include <iostream>
 #include <Python.h>
+#include <datetime.h>
 
 // Work around C2528 error
 #if _MSC_VER >= 1900
diff --git a/cpp/src/arrow/python/pyarrow.cc b/cpp/src/arrow/python/pyarrow.cc
index 5d88051117b78..d080cc0a8147f 100644
--- a/cpp/src/arrow/python/pyarrow.cc
+++ b/cpp/src/arrow/python/pyarrow.cc
@@ -31,13 +31,9 @@ namespace {
 namespace arrow {
 namespace py {
 
-int import_pyarrow() {
-  return ::import_pyarrow__lib();
-}
+int import_pyarrow() { return ::import_pyarrow__lib(); }
 
-bool is_buffer(PyObject* buffer) {
-  return ::pyarrow_is_buffer(buffer) != 0;
-}
+bool is_buffer(PyObject* buffer) { return ::pyarrow_is_buffer(buffer) != 0; }
 
 Status unwrap_buffer(PyObject* buffer, std::shared_ptr<Buffer>* out) {
   *out = ::pyarrow_unwrap_buffer(buffer);
@@ -52,9 +48,7 @@ PyObject* wrap_buffer(const std::shared_ptr<Buffer>& buffer) {
   return ::pyarrow_wrap_buffer(buffer);
 }
 
-bool is_data_type(PyObject* data_type) {
-  return ::pyarrow_is_data_type(data_type) != 0;
-}
+bool is_data_type(PyObject* data_type) { return ::pyarrow_is_data_type(data_type) != 0; }
 
 Status unwrap_data_type(PyObject* object, std::shared_ptr<DataType>* out) {
   *out = ::pyarrow_unwrap_data_type(object);
@@ -69,9 +63,7 @@ PyObject* wrap_data_type(const std::shared_ptr<DataType>& type) {
   return ::pyarrow_wrap_data_type(type);
 }
 
-bool is_field(PyObject* field) {
-  return ::pyarrow_is_field(field) != 0;
-}
+bool is_field(PyObject* field) { return ::pyarrow_is_field(field) != 0; }
 
 Status unwrap_field(PyObject* field, std::shared_ptr<Field>* out) {
   *out = ::pyarrow_unwrap_field(field);
@@ -86,9 +78,7 @@ PyObject* wrap_field(const std::shared_ptr<Field>& field) {
   return ::pyarrow_wrap_field(field);
 }
 
-bool is_schema(PyObject* schema) {
-  return ::pyarrow_is_schema(schema) != 0;
-}
+bool is_schema(PyObject* schema) { return ::pyarrow_is_schema(schema) != 0; }
 
 Status unwrap_schema(PyObject* schema, std::shared_ptr<Schema>* out) {
   *out = ::pyarrow_unwrap_schema(schema);
@@ -103,9 +93,7 @@ PyObject* wrap_schema(const std::shared_ptr<Schema>& schema) {
   return ::pyarrow_wrap_schema(schema);
 }
 
-bool is_array(PyObject* array) {
-  return ::pyarrow_is_array(array) != 0;
-}
+bool is_array(PyObject* array) { return ::pyarrow_is_array(array) != 0; }
 
 Status unwrap_array(PyObject* array, std::shared_ptr<Array>* out) {
   *out = ::pyarrow_unwrap_array(array);
@@ -120,9 +108,7 @@ PyObject* wrap_array(const std::shared_ptr<Array>& array) {
   return ::pyarrow_wrap_array(array);
 }
 
-bool is_tensor(PyObject* tensor) {
-  return ::pyarrow_is_tensor(tensor) != 0;
-}
+bool is_tensor(PyObject* tensor) { return ::pyarrow_is_tensor(tensor) != 0; }
 
 Status unwrap_tensor(PyObject* tensor, std::shared_ptr<Tensor>* out) {
   *out = ::pyarrow_unwrap_tensor(tensor);
@@ -137,9 +123,7 @@ PyObject* wrap_tensor(const std::shared_ptr<Tensor>& tensor) {
   return ::pyarrow_wrap_tensor(tensor);
 }
 
-bool is_column(PyObject* column) {
-  return ::pyarrow_is_column(column) != 0;
-}
+bool is_column(PyObject* column) { return ::pyarrow_is_column(column) != 0; }
 
 Status unwrap_column(PyObject* column, std::shared_ptr<Column>* out) {
   *out = ::pyarrow_unwrap_column(column);
@@ -154,9 +138,7 @@ PyObject* wrap_column(const std::shared_ptr<Column>& column) {
   return ::pyarrow_wrap_column(column);
 }
 
-bool is_table(PyObject* table) {
-  return ::pyarrow_is_table(table) != 0;
-}
+bool is_table(PyObject* table) { return ::pyarrow_is_table(table) != 0; }
 
 Status unwrap_table(PyObject* table, std::shared_ptr<Table>* out) {
   *out = ::pyarrow_unwrap_table(table);
@@ -171,9 +153,7 @@ PyObject* wrap_table(const std::shared_ptr<Table>& table) {
   return ::pyarrow_wrap_table(table);
 }
 
-bool is_record_batch(PyObject* batch) {
-  return ::pyarrow_is_batch(batch) != 0;
-}
+bool is_record_batch(PyObject* batch) { return ::pyarrow_is_batch(batch) != 0; }
 
 Status unwrap_record_batch(PyObject* batch, std::shared_ptr<RecordBatch>* out) {
   *out = ::pyarrow_unwrap_batch(batch);
diff --git a/cpp/src/arrow/python/pyarrow.h b/cpp/src/arrow/python/pyarrow.h
index 7278d1c285785..e637627006177 100644
--- a/cpp/src/arrow/python/pyarrow.h
+++ b/cpp/src/arrow/python/pyarrow.h
@@ -74,8 +74,8 @@ ARROW_EXPORT Status unwrap_table(PyObject* table, std::shared_ptr<Table>* out);
 ARROW_EXPORT PyObject* wrap_table(const std::shared_ptr<Table>& table);
 
 ARROW_EXPORT bool is_record_batch(PyObject* batch);
-ARROW_EXPORT Status unwrap_record_batch(
-    PyObject* batch, std::shared_ptr<RecordBatch>* out);
+ARROW_EXPORT Status unwrap_record_batch(PyObject* batch,
+                                        std::shared_ptr<RecordBatch>* out);
 ARROW_EXPORT PyObject* wrap_record_batch(const std::shared_ptr<RecordBatch>& batch);
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index c0e555d4904d5..b50699d1ae9d4 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -36,9 +36,7 @@
 namespace arrow {
 namespace py {
 
-TEST(PyBuffer, InvalidInputObject) {
-  PyBuffer buffer(Py_None);
-}
+TEST(PyBuffer, InvalidInputObject) { PyBuffer buffer(Py_None); }
 
 TEST(DecimalTest, TestPythonDecimalToString) {
   PyAcquireGIL lock;
@@ -58,8 +56,8 @@ TEST(DecimalTest, TestPythonDecimalToString) {
 
   auto c_string_size = decimal_string.size();
   ASSERT_GT(c_string_size, 0);
-  OwnedRef pydecimal(PyObject_CallFunction(
-      Decimal.obj(), const_cast<char*>(format), c_string, c_string_size));
+  OwnedRef pydecimal(PyObject_CallFunction(Decimal.obj(), const_cast<char*>(format),
+                                           c_string, c_string_size));
   ASSERT_NE(pydecimal.obj(), nullptr);
   ASSERT_EQ(PyErr_Occurred(), nullptr);
 
@@ -88,7 +86,8 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   auto f3 = field("f3", utf8());
   std::vector<std::shared_ptr<Field>> fields = {f1, f2, f3};
   std::vector<std::shared_ptr<Column>> cols = {std::make_shared<Column>(f1, arr),
-      std::make_shared<Column>(f2, arr), std::make_shared<Column>(f3, arr)};
+                                               std::make_shared<Column>(f2, arr),
+                                               std::make_shared<Column>(f3, arr)};
 
   auto schema = std::make_shared<Schema>(fields);
   auto table = std::make_shared<Table>(schema, cols);
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index d32421e8e3652..de7515101518a 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -18,8 +18,8 @@
 #ifndef PYARROW_UTIL_DATETIME_H
 #define PYARROW_UTIL_DATETIME_H
 
-#include "arrow/python/platform.h"
 #include <datetime.h>
+#include "arrow/python/platform.h"
 
 namespace arrow {
 namespace py {
@@ -31,8 +31,8 @@ static inline int64_t PyTime_to_us(PyObject* pytime) {
           PyDateTime_TIME_GET_MICROSECOND(pytime));
 }
 
-static inline Status PyTime_from_int(
-    int64_t val, const TimeUnit::type unit, PyObject** out) {
+static inline Status PyTime_from_int(int64_t val, const TimeUnit::type unit,
+                                     PyObject** out) {
   int64_t hour = 0, minute = 0, second = 0, microsecond = 0;
   switch (unit) {
     case TimeUnit::NANO:
@@ -65,7 +65,7 @@ static inline Status PyTime_from_int(
       break;
   }
   *out = PyTime_FromTime(static_cast<int32_t>(hour), static_cast<int32_t>(minute),
-      static_cast<int32_t>(second), static_cast<int32_t>(microsecond));
+                         static_cast<int32_t>(second), static_cast<int32_t>(microsecond));
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/status.cc b/cpp/src/arrow/status.cc
index 99897428eaed3..9b509b4835126 100644
--- a/cpp/src/arrow/status.cc
+++ b/cpp/src/arrow/status.cc
@@ -33,7 +33,9 @@ void Status::CopyFrom(const State* state) {
 }
 
 std::string Status::CodeAsString() const {
-  if (state_ == NULL) { return "OK"; }
+  if (state_ == NULL) {
+    return "OK";
+  }
 
   const char* type;
   switch (code()) {
@@ -70,7 +72,9 @@ std::string Status::CodeAsString() const {
 
 std::string Status::ToString() const {
   std::string result(CodeAsString());
-  if (state_ == NULL) { return result; }
+  if (state_ == NULL) {
+    return result;
+  }
   result += ": ";
   result += state_->msg;
   return result;
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 1bea1fca84ebb..a02752f21e4b9 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -23,10 +23,12 @@
 #include "arrow/util/visibility.h"
 
 // Return the given status if it is not OK.
-#define ARROW_RETURN_NOT_OK(s)                        \
-  do {                                                \
-    ::arrow::Status _s = (s);                         \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; } \
+#define ARROW_RETURN_NOT_OK(s)           \
+  do {                                   \
+    ::arrow::Status _s = (s);            \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { \
+      return _s;                         \
+    }                                    \
   } while (0)
 
 // If 'to_call' returns a bad status, CHECK immediately with a logged message
@@ -43,10 +45,12 @@
 
 namespace arrow {
 
-#define RETURN_NOT_OK(s)                              \
-  do {                                                \
-    Status _s = (s);                                  \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) { return _s; } \
+#define RETURN_NOT_OK(s)                 \
+  do {                                   \
+    Status _s = (s);                     \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { \
+      return _s;                         \
+    }                                    \
   } while (0)
 
 #define RETURN_NOT_OK_ELSE(s, else_) \
@@ -187,7 +191,9 @@ inline Status::Status(const Status& s)
 inline void Status::operator=(const Status& s) {
   // The following condition catches both aliasing (when this == &s),
   // and the common case where both s and *this are ok.
-  if (state_ != s.state_) { CopyFrom(s.state_); }
+  if (state_ != s.state_) {
+    CopyFrom(s.state_);
+  }
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index e46fdc77cf761..8dba8c052e922 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -198,11 +198,11 @@ class TestTable : public TestBase {
     schema_ = std::make_shared<Schema>(fields);
 
     arrays_ = {MakePrimitive<Int32Array>(length), MakePrimitive<UInt8Array>(length),
-        MakePrimitive<Int16Array>(length)};
+               MakePrimitive<Int16Array>(length)};
 
     columns_ = {std::make_shared<Column>(schema_->field(0), arrays_[0]),
-        std::make_shared<Column>(schema_->field(1), arrays_[1]),
-        std::make_shared<Column>(schema_->field(2), arrays_[2])};
+                std::make_shared<Column>(schema_->field(1), arrays_[1]),
+                std::make_shared<Column>(schema_->field(2), arrays_[2])};
   }
 
  protected:
@@ -412,8 +412,8 @@ TEST_F(TestTable, AddColumn) {
   ASSERT_OK(table.AddColumn(0, columns_[0], &result));
   auto ex_schema = std::shared_ptr<Schema>(new Schema(
       {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)}));
-  std::vector<std::shared_ptr<Column>> ex_columns = {
-      table.column(0), table.column(0), table.column(1), table.column(2)};
+  std::vector<std::shared_ptr<Column>> ex_columns = {table.column(0), table.column(0),
+                                                     table.column(1), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.AddColumn(1, columns_[0], &result));
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index c09628ed395c4..a0a25079e6ed7 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -43,8 +43,12 @@ ChunkedArray::ChunkedArray(const ArrayVector& chunks) : chunks_(chunks) {
 }
 
 bool ChunkedArray::Equals(const ChunkedArray& other) const {
-  if (length_ != other.length()) { return false; }
-  if (null_count_ != other.null_count()) { return false; }
+  if (length_ != other.length()) {
+    return false;
+  }
+  if (null_count_ != other.null_count()) {
+    return false;
+  }
 
   // Check contents of the underlying arrays. This checks for equality of
   // the underlying data independently of the chunk size.
@@ -57,10 +61,10 @@ bool ChunkedArray::Equals(const ChunkedArray& other) const {
   while (elements_compared < length_) {
     const std::shared_ptr<Array> this_array = chunks_[this_chunk_idx];
     const std::shared_ptr<Array> other_array = other.chunk(other_chunk_idx);
-    int64_t common_length = std::min(
-        this_array->length() - this_start_idx, other_array->length() - other_start_idx);
+    int64_t common_length = std::min(this_array->length() - this_start_idx,
+                                     other_array->length() - other_start_idx);
     if (!this_array->RangeEquals(this_start_idx, this_start_idx + common_length,
-            other_start_idx, other_array)) {
+                                 other_start_idx, other_array)) {
       return false;
     }
 
@@ -85,8 +89,12 @@ bool ChunkedArray::Equals(const ChunkedArray& other) const {
 }
 
 bool ChunkedArray::Equals(const std::shared_ptr<ChunkedArray>& other) const {
-  if (this == other.get()) { return true; }
-  if (!other) { return false; }
+  if (this == other.get()) {
+    return true;
+  }
+  if (!other) {
+    return false;
+  }
   return Equals(*other.get());
 }
 
@@ -107,18 +115,24 @@ Column::Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>
 Column::Column(const std::string& name, const std::shared_ptr<Array>& data)
     : Column(::arrow::field(name, data->type()), data) {}
 
-Column::Column(
-    const std::shared_ptr<Field>& field, const std::shared_ptr<ChunkedArray>& data)
+Column::Column(const std::shared_ptr<Field>& field,
+               const std::shared_ptr<ChunkedArray>& data)
     : field_(field), data_(data) {}
 
 bool Column::Equals(const Column& other) const {
-  if (!field_->Equals(other.field())) { return false; }
+  if (!field_->Equals(other.field())) {
+    return false;
+  }
   return data_->Equals(other.data());
 }
 
 bool Column::Equals(const std::shared_ptr<Column>& other) const {
-  if (this == other.get()) { return true; }
-  if (!other) { return false; }
+  if (this == other.get()) {
+    return true;
+  }
+  if (!other) {
+    return false;
+  }
 
   return Equals(*other.get());
 }
@@ -141,11 +155,13 @@ Status Column::ValidateData() {
 
 void AssertBatchValid(const RecordBatch& batch) {
   Status s = batch.Validate();
-  if (!s.ok()) { DCHECK(false) << s.ToString(); }
+  if (!s.ok()) {
+    DCHECK(false) << s.ToString();
+  }
 }
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-    const std::vector<std::shared_ptr<Array>>& columns)
+                         const std::vector<std::shared_ptr<Array>>& columns)
     : schema_(schema), num_rows_(num_rows), columns_(columns.size()) {
   for (size_t i = 0; i < columns.size(); ++i) {
     columns_[i] = columns[i]->data();
@@ -153,7 +169,7 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
 }
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-    std::vector<std::shared_ptr<Array>>&& columns)
+                         std::vector<std::shared_ptr<Array>>&& columns)
     : schema_(schema), num_rows_(num_rows), columns_(columns.size()) {
   for (size_t i = 0; i < columns.size(); ++i) {
     columns_[i] = columns[i]->data();
@@ -161,11 +177,11 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
 }
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-    std::vector<std::shared_ptr<internal::ArrayData>>&& columns)
+                         std::vector<std::shared_ptr<internal::ArrayData>>&& columns)
     : schema_(schema), num_rows_(num_rows), columns_(std::move(columns)) {}
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-    const std::vector<std::shared_ptr<internal::ArrayData>>& columns)
+                         const std::vector<std::shared_ptr<internal::ArrayData>>& columns)
     : schema_(schema), num_rows_(num_rows), columns_(columns) {}
 
 std::shared_ptr<Array> RecordBatch::column(int i) const {
@@ -184,7 +200,9 @@ bool RecordBatch::Equals(const RecordBatch& other) const {
   }
 
   for (int i = 0; i < num_columns(); ++i) {
-    if (!column(i)->Equals(other.column(i))) { return false; }
+    if (!column(i)->Equals(other.column(i))) {
+      return false;
+    }
   }
 
   return true;
@@ -196,7 +214,9 @@ bool RecordBatch::ApproxEquals(const RecordBatch& other) const {
   }
 
   for (int i = 0; i < num_columns(); ++i) {
-    if (!column(i)->ApproxEquals(other.column(i))) { return false; }
+    if (!column(i)->ApproxEquals(other.column(i))) {
+      return false;
+    }
   }
 
   return true;
@@ -253,7 +273,7 @@ Status RecordBatch::Validate() const {
 // Table methods
 
 Table::Table(const std::shared_ptr<Schema>& schema,
-    const std::vector<std::shared_ptr<Column>>& columns)
+             const std::vector<std::shared_ptr<Column>>& columns)
     : schema_(schema), columns_(columns) {
   if (columns.size() == 0) {
     num_rows_ = 0;
@@ -263,7 +283,7 @@ Table::Table(const std::shared_ptr<Schema>& schema,
 }
 
 Table::Table(const std::shared_ptr<Schema>& schema,
-    const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
+             const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
     : schema_(schema), columns_(columns), num_rows_(num_rows) {}
 
 std::shared_ptr<Table> Table::ReplaceSchemaMetadata(
@@ -273,7 +293,7 @@ std::shared_ptr<Table> Table::ReplaceSchemaMetadata(
 }
 
 Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>& batches,
-    std::shared_ptr<Table>* table) {
+                                std::shared_ptr<Table>* table) {
   if (batches.size() == 0) {
     return Status::Invalid("Must pass at least one record batch");
   }
@@ -307,9 +327,11 @@ Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>&
   return Status::OK();
 }
 
-Status ConcatenateTables(
-    const std::vector<std::shared_ptr<Table>>& tables, std::shared_ptr<Table>* table) {
-  if (tables.size() == 0) { return Status::Invalid("Must pass at least one table"); }
+Status ConcatenateTables(const std::vector<std::shared_ptr<Table>>& tables,
+                         std::shared_ptr<Table>* table) {
+  if (tables.size() == 0) {
+    return Status::Invalid("Must pass at least one table");
+  }
 
   std::shared_ptr<Schema> schema = tables[0]->schema();
 
@@ -343,12 +365,20 @@ Status ConcatenateTables(
 }
 
 bool Table::Equals(const Table& other) const {
-  if (this == &other) { return true; }
-  if (!schema_->Equals(*other.schema())) { return false; }
-  if (static_cast<int64_t>(columns_.size()) != other.num_columns()) { return false; }
+  if (this == &other) {
+    return true;
+  }
+  if (!schema_->Equals(*other.schema())) {
+    return false;
+  }
+  if (static_cast<int64_t>(columns_.size()) != other.num_columns()) {
+    return false;
+  }
 
   for (int i = 0; i < static_cast<int>(columns_.size()); i++) {
-    if (!columns_[i]->Equals(other.column(i))) { return false; }
+    if (!columns_[i]->Equals(other.column(i))) {
+      return false;
+    }
   }
   return true;
 }
@@ -361,9 +391,11 @@ Status Table::RemoveColumn(int i, std::shared_ptr<Table>* out) const {
   return Status::OK();
 }
 
-Status Table::AddColumn(
-    int i, const std::shared_ptr<Column>& col, std::shared_ptr<Table>* out) const {
-  if (i < 0 || i > num_columns() + 1) { return Status::Invalid("Invalid column index."); }
+Status Table::AddColumn(int i, const std::shared_ptr<Column>& col,
+                        std::shared_ptr<Table>* out) const {
+  if (i < 0 || i > num_columns() + 1) {
+    return Status::Invalid("Invalid column index.");
+  }
   if (col == nullptr) {
     std::stringstream ss;
     ss << "Column " << i << " was null";
@@ -407,7 +439,8 @@ Status Table::ValidateColumns() const {
 }
 
 Status ARROW_EXPORT MakeTable(const std::shared_ptr<Schema>& schema,
-    const std::vector<std::shared_ptr<Array>>& arrays, std::shared_ptr<Table>* table) {
+                              const std::vector<std::shared_ptr<Array>>& arrays,
+                              std::shared_ptr<Table>* table) {
   // Make sure the length of the schema corresponds to the length of the vector
   if (schema->num_fields() != static_cast<int>(arrays.size())) {
     std::stringstream ss;
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 7ada0e9709f05..6afd618da043b 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -121,11 +121,11 @@ class ARROW_EXPORT RecordBatch {
   /// num_rows
 
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-      const std::vector<std::shared_ptr<Array>>& columns);
+              const std::vector<std::shared_ptr<Array>>& columns);
 
   /// \brief Deprecated move constructor for a vector of Array instances
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-      std::vector<std::shared_ptr<Array>>&& columns);
+              std::vector<std::shared_ptr<Array>>&& columns);
 
   /// \brief Construct record batch from vector of internal data structures
   /// \since 0.5.0
@@ -138,12 +138,12 @@ class ARROW_EXPORT RecordBatch {
   /// should be equal to the length of each field
   /// \param columns the data for the batch's columns
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-      std::vector<std::shared_ptr<internal::ArrayData>>&& columns);
+              std::vector<std::shared_ptr<internal::ArrayData>>&& columns);
 
   /// \brief Construct record batch by copying vector of array data
   /// \since 0.5.0
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-      const std::vector<std::shared_ptr<internal::ArrayData>>& columns);
+              const std::vector<std::shared_ptr<internal::ArrayData>>& columns);
 
   bool Equals(const RecordBatch& other) const;
 
@@ -194,14 +194,14 @@ class ARROW_EXPORT Table {
  public:
   // If columns is zero-length, the table's number of rows is zero
   Table(const std::shared_ptr<Schema>& schema,
-      const std::vector<std::shared_ptr<Column>>& columns);
+        const std::vector<std::shared_ptr<Column>>& columns);
 
   // num_rows is a parameter to allow for tables of a particular size not
   // having any materialized columns. Each column should therefore have the
   // same length as num_rows -- you can validate this using
   // Table::ValidateColumns
   Table(const std::shared_ptr<Schema>& schema,
-      const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows);
+        const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows);
 
   // Construct table from RecordBatch, but only if all of the batch schemas are
   // equal. Returns Status::Invalid if there is some problem
@@ -221,8 +221,8 @@ class ARROW_EXPORT Table {
   Status RemoveColumn(int i, std::shared_ptr<Table>* out) const;
 
   /// Add column to the table, producing a new Table
-  Status AddColumn(
-      int i, const std::shared_ptr<Column>& column, std::shared_ptr<Table>* out) const;
+  Status AddColumn(int i, const std::shared_ptr<Column>& column,
+                   std::shared_ptr<Table>* out) const;
 
   /// \brief Replace schema key-value metadata with new metadata (EXPERIMENTAL)
   /// \since 0.5.0
@@ -252,11 +252,12 @@ class ARROW_EXPORT Table {
 
 // Construct table from multiple input tables. Return Status::Invalid if
 // schemas are not equal
-Status ARROW_EXPORT ConcatenateTables(
-    const std::vector<std::shared_ptr<Table>>& tables, std::shared_ptr<Table>* table);
+Status ARROW_EXPORT ConcatenateTables(const std::vector<std::shared_ptr<Table>>& tables,
+                                      std::shared_ptr<Table>* table);
 
 Status ARROW_EXPORT MakeTable(const std::shared_ptr<Schema>& schema,
-    const std::vector<std::shared_ptr<Array>>& arrays, std::shared_ptr<Table>* table);
+                              const std::vector<std::shared_ptr<Array>>& arrays,
+                              std::shared_ptr<Table>* table);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index bcd9d8d94c6b4..31b1a359219a6 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -35,7 +35,8 @@
 namespace arrow {
 
 static void ComputeRowMajorStrides(const FixedWidthType& type,
-    const std::vector<int64_t>& shape, std::vector<int64_t>* strides) {
+                                   const std::vector<int64_t>& shape,
+                                   std::vector<int64_t>* strides) {
   int64_t remaining = type.bit_width() / 8;
   for (int64_t dimsize : shape) {
     remaining *= dimsize;
@@ -53,7 +54,8 @@ static void ComputeRowMajorStrides(const FixedWidthType& type,
 }
 
 static void ComputeColumnMajorStrides(const FixedWidthType& type,
-    const std::vector<int64_t>& shape, std::vector<int64_t>* strides) {
+                                      const std::vector<int64_t>& shape,
+                                      std::vector<int64_t>* strides) {
   int64_t total = type.bit_width() / 8;
   for (int64_t dimsize : shape) {
     if (dimsize == 0) {
@@ -69,8 +71,8 @@ static void ComputeColumnMajorStrides(const FixedWidthType& type,
 
 /// Constructor with strides and dimension names
 Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
-    const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
-    const std::vector<std::string>& dim_names)
+               const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
+               const std::vector<std::string>& dim_names)
     : type_(type), data_(data), shape_(shape), strides_(strides), dim_names_(dim_names) {
   DCHECK(is_tensor_supported(type->id()));
   if (shape.size() > 0 && strides.size() == 0) {
@@ -79,11 +81,11 @@ Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buff
 }
 
 Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
-    const std::vector<int64_t>& shape, const std::vector<int64_t>& strides)
+               const std::vector<int64_t>& shape, const std::vector<int64_t>& strides)
     : Tensor(type, data, shape, strides, {}) {}
 
 Tensor::Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
-    const std::vector<int64_t>& shape)
+               const std::vector<int64_t>& shape)
     : Tensor(type, data, shape, {}, {}) {}
 
 const std::string& Tensor::dim_name(int i) const {
@@ -100,9 +102,7 @@ int64_t Tensor::size() const {
   return std::accumulate(shape_.begin(), shape_.end(), 1LL, std::multiplies<int64_t>());
 }
 
-bool Tensor::is_contiguous() const {
-  return is_row_major() || is_column_major();
-}
+bool Tensor::is_contiguous() const { return is_row_major() || is_column_major(); }
 
 bool Tensor::is_row_major() const {
   std::vector<int64_t> c_strides;
@@ -118,14 +118,14 @@ bool Tensor::is_column_major() const {
   return strides_ == f_strides;
 }
 
-Type::type Tensor::type_id() const {
-  return type_->id();
-}
+Type::type Tensor::type_id() const { return type_->id(); }
 
 bool Tensor::Equals(const Tensor& other) const {
   bool are_equal = false;
   Status error = TensorEquals(*this, other, &are_equal);
-  if (!error.ok()) { DCHECK(false) << "Tensors not comparable: " << error.ToString(); }
+  if (!error.ok()) {
+    DCHECK(false) << "Tensors not comparable: " << error.ToString();
+  }
   return are_equal;
 }
 
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index 371f5911a4396..b074b8c309ba1 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -62,16 +62,16 @@ class ARROW_EXPORT Tensor {
 
   /// Constructor with no dimension names or strides, data assumed to be row-major
   Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
-      const std::vector<int64_t>& shape);
+         const std::vector<int64_t>& shape);
 
   /// Constructor with non-negative strides
   Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
-      const std::vector<int64_t>& shape, const std::vector<int64_t>& strides);
+         const std::vector<int64_t>& shape, const std::vector<int64_t>& strides);
 
   /// Constructor with strides and dimension names
   Tensor(const std::shared_ptr<DataType>& type, const std::shared_ptr<Buffer>& data,
-      const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
-      const std::vector<std::string>& dim_names);
+         const std::vector<int64_t>& shape, const std::vector<int64_t>& strides,
+         const std::vector<std::string>& dim_names);
 
   std::shared_ptr<DataType> type() const { return type_; }
   std::shared_ptr<Buffer> data() const { return data_; }
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 2bc662526713e..1a3376cee6053 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -39,16 +39,20 @@
 #include "arrow/util/logging.h"
 #include "arrow/util/random.h"
 
-#define ASSERT_RAISES(ENUM, expr)                  \
-  do {                                             \
-    ::arrow::Status s = (expr);                    \
-    if (!s.Is##ENUM()) { FAIL() << s.ToString(); } \
+#define ASSERT_RAISES(ENUM, expr) \
+  do {                            \
+    ::arrow::Status s = (expr);   \
+    if (!s.Is##ENUM()) {          \
+      FAIL() << s.ToString();     \
+    }                             \
   } while (0)
 
-#define ASSERT_OK(expr)                      \
-  do {                                       \
-    ::arrow::Status s = (expr);              \
-    if (!s.ok()) { FAIL() << s.ToString(); } \
+#define ASSERT_OK(expr)         \
+  do {                          \
+    ::arrow::Status s = (expr); \
+    if (!s.ok()) {              \
+      FAIL() << s.ToString();   \
+    }                           \
   } while (0)
 
 #define ASSERT_OK_NO_THROW(expr) ASSERT_NO_THROW(ASSERT_OK(expr))
@@ -59,10 +63,12 @@
     EXPECT_TRUE(s.ok());        \
   } while (0)
 
-#define ABORT_NOT_OK(s)                              \
-  do {                                               \
-    ::arrow::Status _s = (s);                        \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) { exit(-1); } \
+#define ABORT_NOT_OK(s)                  \
+  do {                                   \
+    ::arrow::Status _s = (s);            \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { \
+      exit(-1);                          \
+    }                                    \
   } while (0);
 
 namespace arrow {
@@ -85,8 +91,8 @@ void randint(int64_t N, T lower, T upper, std::vector<T>* out) {
 }
 
 template <typename T>
-void random_real(
-    int64_t n, uint32_t seed, T min_value, T max_value, std::vector<T>* out) {
+void random_real(int64_t n, uint32_t seed, T min_value, T max_value,
+                 std::vector<T>* out) {
   std::mt19937 gen(seed);
   std::uniform_real_distribution<T> d(min_value, max_value);
   for (int64_t i = 0; i < n; ++i) {
@@ -96,13 +102,13 @@ void random_real(
 
 template <typename T>
 std::shared_ptr<Buffer> GetBufferFromVector(const std::vector<T>& values) {
-  return std::make_shared<Buffer>(
-      reinterpret_cast<const uint8_t*>(values.data()), values.size() * sizeof(T));
+  return std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(values.data()),
+                                  values.size() * sizeof(T));
 }
 
 template <typename T>
-inline Status CopyBufferFromVector(
-    const std::vector<T>& values, MemoryPool* pool, std::shared_ptr<Buffer>* result) {
+inline Status CopyBufferFromVector(const std::vector<T>& values, MemoryPool* pool,
+                                   std::shared_ptr<Buffer>* result) {
   int64_t nbytes = static_cast<int>(values.size()) * sizeof(T);
 
   auto buffer = std::make_shared<PoolBuffer>(pool);
@@ -114,8 +120,8 @@ inline Status CopyBufferFromVector(
 }
 
 template <typename T>
-static inline Status GetBitmapFromVector(
-    const std::vector<T>& is_valid, std::shared_ptr<Buffer>* result) {
+static inline Status GetBitmapFromVector(const std::vector<T>& is_valid,
+                                         std::shared_ptr<Buffer>* result) {
   size_t length = is_valid.size();
 
   std::shared_ptr<MutableBuffer> buffer;
@@ -123,7 +129,9 @@ static inline Status GetBitmapFromVector(
 
   uint8_t* bitmap = buffer->mutable_data();
   for (size_t i = 0; i < static_cast<size_t>(length); ++i) {
-    if (is_valid[i]) { BitUtil::SetBit(bitmap, i); }
+    if (is_valid[i]) {
+      BitUtil::SetBit(bitmap, i);
+    }
   }
 
   *result = buffer;
@@ -139,8 +147,8 @@ static inline void random_null_bytes(int64_t n, double pct_null, uint8_t* null_b
   }
 }
 
-static inline void random_is_valid(
-    int64_t n, double pct_null, std::vector<bool>* is_valid) {
+static inline void random_is_valid(int64_t n, double pct_null,
+                                   std::vector<bool>* is_valid) {
   Random rng(random_seed());
   for (int64_t i = 0; i < n; ++i) {
     is_valid->push_back(rng.NextDoubleFraction() > pct_null);
@@ -178,24 +186,28 @@ void rand_uniform_int(int64_t n, uint32_t seed, T min_value, T max_value, T* out
 static inline int64_t null_count(const std::vector<uint8_t>& valid_bytes) {
   int64_t result = 0;
   for (size_t i = 0; i < valid_bytes.size(); ++i) {
-    if (valid_bytes[i] == 0) { ++result; }
+    if (valid_bytes[i] == 0) {
+      ++result;
+    }
   }
   return result;
 }
 
 Status MakeRandomInt32PoolBuffer(int64_t length, MemoryPool* pool,
-    std::shared_ptr<PoolBuffer>* pool_buffer, uint32_t seed = 0) {
+                                 std::shared_ptr<PoolBuffer>* pool_buffer,
+                                 uint32_t seed = 0) {
   DCHECK(pool);
   auto data = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(data->Resize(length * sizeof(int32_t)));
   test::rand_uniform_int(length, seed, 0, std::numeric_limits<int32_t>::max(),
-      reinterpret_cast<int32_t*>(data->mutable_data()));
+                         reinterpret_cast<int32_t*>(data->mutable_data()));
   *pool_buffer = data;
   return Status::OK();
 }
 
 Status MakeRandomBytePoolBuffer(int64_t length, MemoryPool* pool,
-    std::shared_ptr<PoolBuffer>* pool_buffer, uint32_t seed = 0) {
+                                std::shared_ptr<PoolBuffer>* pool_buffer,
+                                uint32_t seed = 0) {
   auto bytes = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(bytes->Resize(length));
   test::random_bytes(length, seed, bytes->mutable_data());
@@ -207,8 +219,8 @@ Status MakeRandomBytePoolBuffer(int64_t length, MemoryPool* pool,
 
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::shared_ptr<DataType>& type,
-    const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
-    std::shared_ptr<Array>* out) {
+                     const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
+                     std::shared_ptr<Array>* out) {
   MemoryPool* pool = default_memory_pool();
   typename TypeTraits<TYPE>::BuilderType builder(pool, type);
   for (size_t i = 0; i < values.size(); ++i) {
@@ -223,7 +235,7 @@ void ArrayFromVector(const std::shared_ptr<DataType>& type,
 
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
-    std::shared_ptr<Array>* out) {
+                     std::shared_ptr<Array>* out) {
   MemoryPool* pool = default_memory_pool();
   typename TypeTraits<TYPE>::BuilderType builder(pool);
   for (size_t i = 0; i < values.size(); ++i) {
@@ -248,7 +260,7 @@ void ArrayFromVector(const std::vector<C_TYPE>& values, std::shared_ptr<Array>*
 
 template <class T, class Builder>
 Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
-    int64_t size, Builder* builder, std::shared_ptr<Array>* out) {
+                 int64_t size, Builder* builder, std::shared_ptr<Array>* out) {
   // Append the first 1000
   for (int64_t i = 0; i < size; ++i) {
     if (valid_bytes[i] > 0) {
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 7f3adef633767..6b86b4d2f1024 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -345,16 +345,16 @@ TEST(TestTimestampType, ToString) {
 }
 
 TEST(TestNestedType, Equals) {
-  auto create_struct = [](
-      std::string inner_name, std::string struct_name) -> shared_ptr<Field> {
+  auto create_struct = [](std::string inner_name,
+                          std::string struct_name) -> shared_ptr<Field> {
     auto f_type = field(inner_name, int32());
     vector<shared_ptr<Field>> fields = {f_type};
     auto s_type = std::make_shared<StructType>(fields);
     return field(struct_name, s_type);
   };
 
-  auto create_union = [](
-      std::string inner_name, std::string union_name) -> shared_ptr<Field> {
+  auto create_union = [](std::string inner_name,
+                         std::string union_name) -> shared_ptr<Field> {
     auto f_type = field(inner_name, int32());
     vector<shared_ptr<Field>> fields = {f_type};
     vector<uint8_t> codes = {Type::INT32};
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 623c1934f875e..586da2d86d909 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -37,7 +37,7 @@ std::shared_ptr<Field> Field::AddMetadata(
 }
 
 Status Field::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
-    std::shared_ptr<Field>* out) const {
+                          std::shared_ptr<Field>* out) const {
   *out = AddMetadata(metadata);
   return Status::OK();
 }
@@ -47,7 +47,9 @@ std::shared_ptr<Field> Field::RemoveMetadata() const {
 }
 
 bool Field::Equals(const Field& other) const {
-  if (this == &other) { return true; }
+  if (this == &other) {
+    return true;
+  }
   if (this->name_ == other.name_ && this->nullable_ == other.nullable_ &&
       this->type_->Equals(*other.type_.get())) {
     if (metadata_ == nullptr && other.metadata_ == nullptr) {
@@ -68,7 +70,9 @@ bool Field::Equals(const std::shared_ptr<Field>& other) const {
 std::string Field::ToString() const {
   std::stringstream ss;
   ss << this->name_ << ": " << this->type_->ToString();
-  if (!this->nullable_) { ss << " not null"; }
+  if (!this->nullable_) {
+    ss << " not null";
+  }
   return ss.str();
 }
 
@@ -77,34 +81,28 @@ DataType::~DataType() {}
 bool DataType::Equals(const DataType& other) const {
   bool are_equal = false;
   Status error = TypeEquals(*this, other, &are_equal);
-  if (!error.ok()) { DCHECK(false) << "Types not comparable: " << error.ToString(); }
+  if (!error.ok()) {
+    DCHECK(false) << "Types not comparable: " << error.ToString();
+  }
   return are_equal;
 }
 
 bool DataType::Equals(const std::shared_ptr<DataType>& other) const {
-  if (!other) { return false; }
+  if (!other) {
+    return false;
+  }
   return Equals(*other.get());
 }
 
-std::string BooleanType::ToString() const {
-  return name();
-}
+std::string BooleanType::ToString() const { return name(); }
 
-FloatingPoint::Precision HalfFloatType::precision() const {
-  return FloatingPoint::HALF;
-}
+FloatingPoint::Precision HalfFloatType::precision() const { return FloatingPoint::HALF; }
 
-FloatingPoint::Precision FloatType::precision() const {
-  return FloatingPoint::SINGLE;
-}
+FloatingPoint::Precision FloatType::precision() const { return FloatingPoint::SINGLE; }
 
-FloatingPoint::Precision DoubleType::precision() const {
-  return FloatingPoint::DOUBLE;
-}
+FloatingPoint::Precision DoubleType::precision() const { return FloatingPoint::DOUBLE; }
 
-std::string StringType::ToString() const {
-  return std::string("string");
-}
+std::string StringType::ToString() const { return std::string("string"); }
 
 std::string ListType::ToString() const {
   std::stringstream s;
@@ -112,13 +110,9 @@ std::string ListType::ToString() const {
   return s.str();
 }
 
-std::string BinaryType::ToString() const {
-  return std::string("binary");
-}
+std::string BinaryType::ToString() const { return std::string("binary"); }
 
-int FixedSizeBinaryType::bit_width() const {
-  return CHAR_BIT * byte_width();
-}
+int FixedSizeBinaryType::bit_width() const { return CHAR_BIT * byte_width(); }
 
 std::string FixedSizeBinaryType::ToString() const {
   std::stringstream ss;
@@ -130,7 +124,9 @@ std::string StructType::ToString() const {
   std::stringstream s;
   s << "struct<";
   for (int i = 0; i < this->num_children(); ++i) {
-    if (i > 0) { s << ", "; }
+    if (i > 0) {
+      s << ", ";
+    }
     std::shared_ptr<Field> field = this->child(i);
     s << field->name() << ": " << field->type()->ToString();
   }
@@ -148,13 +144,9 @@ Date32Type::Date32Type() : DateType(Type::DATE32, DateUnit::DAY) {}
 
 Date64Type::Date64Type() : DateType(Type::DATE64, DateUnit::MILLI) {}
 
-std::string Date64Type::ToString() const {
-  return std::string("date64[ms]");
-}
+std::string Date64Type::ToString() const { return std::string("date64[ms]"); }
 
-std::string Date32Type::ToString() const {
-  return std::string("date32[day]");
-}
+std::string Date32Type::ToString() const { return std::string("date32[day]"); }
 
 // ----------------------------------------------------------------------
 // Time types
@@ -190,7 +182,9 @@ std::string Time64Type::ToString() const {
 std::string TimestampType::ToString() const {
   std::stringstream ss;
   ss << "timestamp[" << this->unit_;
-  if (this->timezone_.size() > 0) { ss << ", tz=" << this->timezone_; }
+  if (this->timezone_.size() > 0) {
+    ss << ", tz=" << this->timezone_;
+  }
   ss << "]";
   return ss.str();
 }
@@ -199,7 +193,7 @@ std::string TimestampType::ToString() const {
 // Union type
 
 UnionType::UnionType(const std::vector<std::shared_ptr<Field>>& fields,
-    const std::vector<uint8_t>& type_codes, UnionMode mode)
+                     const std::vector<uint8_t>& type_codes, UnionMode mode)
     : NestedType(Type::UNION), mode_(mode), type_codes_(type_codes) {
   children_ = fields;
 }
@@ -214,7 +208,9 @@ std::string UnionType::ToString() const {
   }
 
   for (size_t i = 0; i < children_.size(); ++i) {
-    if (i) { s << ", "; }
+    if (i) {
+      s << ", ";
+    }
     s << children_[i]->ToString() << "=" << static_cast<int>(type_codes_[i]);
   }
   s << ">";
@@ -225,7 +221,7 @@ std::string UnionType::ToString() const {
 // DictionaryType
 
 DictionaryType::DictionaryType(const std::shared_ptr<DataType>& index_type,
-    const std::shared_ptr<Array>& dictionary, bool ordered)
+                               const std::shared_ptr<Array>& dictionary, bool ordered)
     : FixedWidthType(Type::DICTIONARY),
       index_type_(index_type),
       dictionary_(dictionary),
@@ -235,9 +231,7 @@ int DictionaryType::bit_width() const {
   return static_cast<const FixedWidthType*>(index_type_.get())->bit_width();
 }
 
-std::shared_ptr<Array> DictionaryType::dictionary() const {
-  return dictionary_;
-}
+std::shared_ptr<Array> DictionaryType::dictionary() const { return dictionary_; }
 
 std::string DictionaryType::ToString() const {
   std::stringstream ss;
@@ -249,23 +243,27 @@ std::string DictionaryType::ToString() const {
 // ----------------------------------------------------------------------
 // Null type
 
-std::string NullType::ToString() const {
-  return name();
-}
+std::string NullType::ToString() const { return name(); }
 
 // ----------------------------------------------------------------------
 // Schema implementation
 
 Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields,
-    const std::shared_ptr<const KeyValueMetadata>& metadata)
+               const std::shared_ptr<const KeyValueMetadata>& metadata)
     : fields_(fields), metadata_(metadata) {}
 
 bool Schema::Equals(const Schema& other) const {
-  if (this == &other) { return true; }
+  if (this == &other) {
+    return true;
+  }
 
-  if (num_fields() != other.num_fields()) { return false; }
+  if (num_fields() != other.num_fields()) {
+    return false;
+  }
   for (int i = 0; i < num_fields(); ++i) {
-    if (!field(i)->Equals(*other.field(i).get())) { return false; }
+    if (!field(i)->Equals(*other.field(i).get())) {
+      return false;
+    }
   }
   return true;
 }
@@ -290,8 +288,8 @@ int64_t Schema::GetFieldIndex(const std::string& name) const {
   }
 }
 
-Status Schema::AddField(
-    int i, const std::shared_ptr<Field>& field, std::shared_ptr<Schema>* out) const {
+Status Schema::AddField(int i, const std::shared_ptr<Field>& field,
+                        std::shared_ptr<Schema>* out) const {
   DCHECK_GE(i, 0);
   DCHECK_LE(i, this->num_fields());
 
@@ -305,7 +303,7 @@ std::shared_ptr<Schema> Schema::AddMetadata(
 }
 
 Status Schema::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
-    std::shared_ptr<Schema>* out) const {
+                           std::shared_ptr<Schema>* out) const {
   *out = AddMetadata(metadata);
   return Status::OK();
 }
@@ -327,7 +325,9 @@ std::string Schema::ToString() const {
 
   int i = 0;
   for (auto field : fields_) {
-    if (i > 0) { buffer << std::endl; }
+    if (i > 0) {
+      buffer << std::endl;
+    }
     buffer << field->ToString();
     ++i;
   }
@@ -422,18 +422,18 @@ std::shared_ptr<DataType> struct_(const std::vector<std::shared_ptr<Field>>& fie
 }
 
 std::shared_ptr<DataType> union_(const std::vector<std::shared_ptr<Field>>& child_fields,
-    const std::vector<uint8_t>& type_codes, UnionMode mode) {
+                                 const std::vector<uint8_t>& type_codes, UnionMode mode) {
   return std::make_shared<UnionType>(child_fields, type_codes, mode);
 }
 
 std::shared_ptr<DataType> dictionary(const std::shared_ptr<DataType>& index_type,
-    const std::shared_ptr<Array>& dict_values) {
+                                     const std::shared_ptr<Array>& dict_values) {
   return std::make_shared<DictionaryType>(index_type, dict_values);
 }
 
 std::shared_ptr<Field> field(const std::string& name,
-    const std::shared_ptr<DataType>& type, bool nullable,
-    const std::shared_ptr<const KeyValueMetadata>& metadata) {
+                             const std::shared_ptr<DataType>& type, bool nullable,
+                             const std::shared_ptr<const KeyValueMetadata>& metadata) {
   return std::make_shared<Field>(name, type, nullable, metadata);
 }
 
@@ -454,9 +454,7 @@ std::vector<BufferDescr> FixedWidthType::GetBufferLayout() const {
   return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
 }
 
-std::vector<BufferDescr> NullType::GetBufferLayout() const {
-  return {};
-}
+std::vector<BufferDescr> NullType::GetBufferLayout() const { return {}; }
 
 std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
   return {kValidityBuffer, kOffsetBuffer, kValues8};
@@ -474,9 +472,7 @@ std::vector<BufferDescr> ListType::GetBufferLayout() const {
   return {kValidityBuffer, kOffsetBuffer};
 }
 
-std::vector<BufferDescr> StructType::GetBufferLayout() const {
-  return {kValidityBuffer};
-}
+std::vector<BufferDescr> StructType::GetBufferLayout() const { return {kValidityBuffer}; }
 
 std::vector<BufferDescr> UnionType::GetBufferLayout() const {
   if (mode_ == UnionMode::SPARSE) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index fffb840e3cef7..e0df722e5668a 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -204,15 +204,15 @@ class NoExtraMeta {};
 class ARROW_EXPORT Field {
  public:
   Field(const std::string& name, const std::shared_ptr<DataType>& type,
-      bool nullable = true,
-      const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr)
+        bool nullable = true,
+        const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr)
       : name_(name), type_(type), nullable_(nullable), metadata_(metadata) {}
 
   std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
 
   /// \deprecated
   Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
-      std::shared_ptr<Field>* out) const;
+                     std::shared_ptr<Field>* out) const;
 
   std::shared_ptr<Field> AddMetadata(
       const std::shared_ptr<const KeyValueMetadata>& metadata) const;
@@ -489,7 +489,7 @@ class ARROW_EXPORT UnionType : public NestedType {
   static constexpr Type::type type_id = Type::UNION;
 
   UnionType(const std::vector<std::shared_ptr<Field>>& fields,
-      const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
+            const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
 
   std::string ToString() const override;
   static std::string name() { return "union"; }
@@ -669,7 +669,7 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
   static constexpr Type::type type_id = Type::DICTIONARY;
 
   DictionaryType(const std::shared_ptr<DataType>& index_type,
-      const std::shared_ptr<Array>& dictionary, bool ordered = false);
+                 const std::shared_ptr<Array>& dictionary, bool ordered = false);
 
   int bit_width() const override;
 
@@ -699,7 +699,7 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 class ARROW_EXPORT Schema {
  public:
   explicit Schema(const std::vector<std::shared_ptr<Field>>& fields,
-      const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+                  const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
   virtual ~Schema() = default;
 
   /// Returns true if all of the schema fields are equal
@@ -724,13 +724,13 @@ class ARROW_EXPORT Schema {
   /// \brief Render a string representation of the schema suitable for debugging
   std::string ToString() const;
 
-  Status AddField(
-      int i, const std::shared_ptr<Field>& field, std::shared_ptr<Schema>* out) const;
+  Status AddField(int i, const std::shared_ptr<Field>& field,
+                  std::shared_ptr<Schema>* out) const;
   Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
 
   /// \deprecated
   Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
-      std::shared_ptr<Schema>* out) const;
+                     std::shared_ptr<Schema>* out) const;
 
   /// \brief Replace key-value metadata with new metadata
   ///
@@ -761,8 +761,8 @@ std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_
 std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
 
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit);
-std::shared_ptr<DataType> ARROW_EXPORT timestamp(
-    TimeUnit::type unit, const std::string& timezone);
+std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit,
+                                                 const std::string& timezone);
 
 /// Unit can be either SECOND or MILLI
 std::shared_ptr<DataType> ARROW_EXPORT time32(TimeUnit::type unit);
@@ -770,18 +770,18 @@ std::shared_ptr<DataType> ARROW_EXPORT time32(TimeUnit::type unit);
 /// Unit can be either MICRO or NANO
 std::shared_ptr<DataType> ARROW_EXPORT time64(TimeUnit::type unit);
 
-std::shared_ptr<DataType> ARROW_EXPORT struct_(
-    const std::vector<std::shared_ptr<Field>>& fields);
+std::shared_ptr<DataType> ARROW_EXPORT
+struct_(const std::vector<std::shared_ptr<Field>>& fields);
 
-std::shared_ptr<DataType> ARROW_EXPORT union_(
-    const std::vector<std::shared_ptr<Field>>& child_fields,
-    const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
+std::shared_ptr<DataType> ARROW_EXPORT
+union_(const std::vector<std::shared_ptr<Field>>& child_fields,
+       const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
 
 std::shared_ptr<DataType> ARROW_EXPORT dictionary(
     const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& values);
 
-std::shared_ptr<Field> ARROW_EXPORT field(const std::string& name,
-    const std::shared_ptr<DataType>& type, bool nullable = true,
+std::shared_ptr<Field> ARROW_EXPORT field(
+    const std::string& name, const std::shared_ptr<DataType>& type, bool nullable = true,
     const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 3e8ea23432b98..8be67b2a3829c 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -319,9 +319,10 @@ GET_ATTR(TypeClass, void);
 
 #undef GET_ATTR
 
-#define PRIMITIVE_TRAITS(T)                                                           \
-  using TypeClass = typename std::conditional<std::is_base_of<DataType, T>::value, T, \
-      typename GetAttr_TypeClass<T>::type>::type;                                     \
+#define PRIMITIVE_TRAITS(T)                                                 \
+  using TypeClass =                                                         \
+      typename std::conditional<std::is_base_of<DataType, T>::value, T,     \
+                                typename GetAttr_TypeClass<T>::type>::type; \
   using c_type = typename GetAttr_c_type<TypeClass>::type;
 
 template <typename T>
diff --git a/cpp/src/arrow/util/bit-stream-utils.h b/cpp/src/arrow/util/bit-stream-utils.h
index 537fdc3045ca5..318f5ba8b0e17 100644
--- a/cpp/src/arrow/util/bit-stream-utils.h
+++ b/cpp/src/arrow/util/bit-stream-utils.h
@@ -20,9 +20,9 @@
 #ifndef ARROW_UTIL_BIT_STREAM_UTILS_H
 #define ARROW_UTIL_BIT_STREAM_UTILS_H
 
+#include <string.h>
 #include <algorithm>
 #include <cstdint>
-#include <string.h>
 
 #include "arrow/util/bit-util.h"
 #include "arrow/util/bpacking.h"
@@ -229,13 +229,13 @@ inline bool BitWriter::PutVlqInt(uint32_t v) {
 
 template <typename T>
 inline void GetValue_(int num_bits, T* v, int max_bytes, const uint8_t* buffer,
-    int* bit_offset, int* byte_offset, uint64_t* buffered_values) {
+                      int* bit_offset, int* byte_offset, uint64_t* buffered_values) {
 #ifdef _MSC_VER
 #pragma warning(push)
 #pragma warning(disable : 4800)
 #endif
-  *v = static_cast<T>(
-      BitUtil::TrailingBits(*buffered_values, *bit_offset + num_bits) >> *bit_offset);
+  *v = static_cast<T>(BitUtil::TrailingBits(*buffered_values, *bit_offset + num_bits) >>
+                      *bit_offset);
 #ifdef _MSC_VER
 #pragma warning(pop)
 #endif
@@ -292,13 +292,14 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
   if (UNLIKELY(bit_offset != 0)) {
     for (; i < batch_size && bit_offset != 0; ++i) {
       GetValue_(num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset,
-          &buffered_values);
+                &buffered_values);
     }
   }
 
   if (sizeof(T) == 4) {
-    int num_unpacked = unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
-        reinterpret_cast<uint32_t*>(v + i), batch_size - i, num_bits);
+    int num_unpacked =
+        unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
+                 reinterpret_cast<uint32_t*>(v + i), batch_size - i, num_bits);
     i += num_unpacked;
     byte_offset += num_unpacked * num_bits / 8;
   } else {
@@ -307,8 +308,10 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
     while (i < batch_size) {
       int unpack_size = std::min(buffer_size, batch_size - i);
       int num_unpacked = unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
-          unpack_buffer, unpack_size, num_bits);
-      if (num_unpacked == 0) { break; }
+                                  unpack_buffer, unpack_size, num_bits);
+      if (num_unpacked == 0) {
+        break;
+      }
       for (int k = 0; k < num_unpacked; ++k) {
 #ifdef _MSC_VER
 #pragma warning(push)
@@ -332,8 +335,8 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
   }
 
   for (; i < batch_size; ++i) {
-    GetValue_(
-        num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset, &buffered_values);
+    GetValue_(num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset,
+              &buffered_values);
   }
 
   bit_offset_ = bit_offset;
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index cd945585ba210..231bf54a2a3b6 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -35,7 +35,9 @@
 namespace arrow {
 
 static void EnsureCpuInfoInitialized() {
-  if (!CpuInfo::initialized()) { CpuInfo::Init(); }
+  if (!CpuInfo::initialized()) {
+    CpuInfo::Init();
+  }
 }
 
 TEST(BitUtilTests, TestIsMultipleOf64) {
@@ -68,11 +70,13 @@ TEST(BitUtilTests, TestNextPower2) {
   ASSERT_EQ(1LL << 62, NextPower2((1LL << 62) - 1));
 }
 
-static inline int64_t SlowCountBits(
-    const uint8_t* data, int64_t bit_offset, int64_t length) {
+static inline int64_t SlowCountBits(const uint8_t* data, int64_t bit_offset,
+                                    int64_t length) {
   int64_t count = 0;
   for (int64_t i = bit_offset; i < bit_offset + length; ++i) {
-    if (BitUtil::GetBit(data, i)) { ++count; }
+    if (BitUtil::GetBit(data, i)) {
+      ++count;
+    }
   }
   return count;
 }
@@ -175,9 +179,9 @@ TEST(BitUtil, TrailingBits) {
   EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 0), 0);
   EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 1), 1);
   EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 64),
-      BOOST_BINARY(1 1 1 1 1 1 1 1));
+            BOOST_BINARY(1 1 1 1 1 1 1 1));
   EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 100),
-      BOOST_BINARY(1 1 1 1 1 1 1 1));
+            BOOST_BINARY(1 1 1 1 1 1 1 1));
   EXPECT_EQ(BitUtil::TrailingBits(0, 1), 0);
   EXPECT_EQ(BitUtil::TrailingBits(0, 64), 0);
   EXPECT_EQ(BitUtil::TrailingBits(1LL << 63, 0), 0);
@@ -193,12 +197,12 @@ TEST(BitUtil, ByteSwap) {
   EXPECT_EQ(BitUtil::ByteSwap(static_cast<int32_t>(0x11223344)), 0x44332211);
 
   EXPECT_EQ(BitUtil::ByteSwap(static_cast<uint64_t>(0)), 0);
-  EXPECT_EQ(
-      BitUtil::ByteSwap(static_cast<uint64_t>(0x1122334455667788)), 0x8877665544332211);
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<uint64_t>(0x1122334455667788)),
+            0x8877665544332211);
 
   EXPECT_EQ(BitUtil::ByteSwap(static_cast<int64_t>(0)), 0);
-  EXPECT_EQ(
-      BitUtil::ByteSwap(static_cast<int64_t>(0x1122334455667788)), 0x8877665544332211);
+  EXPECT_EQ(BitUtil::ByteSwap(static_cast<int64_t>(0x1122334455667788)),
+            0x8877665544332211);
 
   EXPECT_EQ(BitUtil::ByteSwap(static_cast<int16_t>(0)), 0);
   EXPECT_EQ(BitUtil::ByteSwap(static_cast<int16_t>(0x1122)), 0x2211);
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 5bbec6f23111e..f255f95f30a76 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -36,12 +36,14 @@ namespace arrow {
 
 void BitUtil::FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits) {
   for (size_t i = 0; i < bytes.size(); ++i) {
-    if (bytes[i] > 0) { SetBit(bits, i); }
+    if (bytes[i] > 0) {
+      SetBit(bits, i);
+    }
   }
 }
 
-Status BitUtil::BytesToBits(
-    const std::vector<uint8_t>& bytes, std::shared_ptr<Buffer>* out) {
+Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes,
+                            std::shared_ptr<Buffer>* out) {
   int64_t bit_length = BitUtil::BytesForBits(bytes.size());
 
   std::shared_ptr<MutableBuffer> buffer;
@@ -65,7 +67,9 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
   // The number of bits until fast_count_start
   const int64_t initial_bits = std::min(length, fast_count_start - bit_offset);
   for (int64_t i = bit_offset; i < bit_offset + initial_bits; ++i) {
-    if (BitUtil::GetBit(data, i)) { ++count; }
+    if (BitUtil::GetBit(data, i)) {
+      ++count;
+    }
   }
 
   const int64_t fast_counts = (length - initial_bits) / pop_len;
@@ -85,21 +89,23 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
   // versions of popcount but the code complexity is likely not worth it)
   const int64_t tail_index = bit_offset + initial_bits + fast_counts * pop_len;
   for (int64_t i = tail_index; i < bit_offset + length; ++i) {
-    if (BitUtil::GetBit(data, i)) { ++count; }
+    if (BitUtil::GetBit(data, i)) {
+      ++count;
+    }
   }
 
   return count;
 }
 
-Status GetEmptyBitmap(
-    MemoryPool* pool, int64_t length, std::shared_ptr<MutableBuffer>* result) {
+Status GetEmptyBitmap(MemoryPool* pool, int64_t length,
+                      std::shared_ptr<MutableBuffer>* result) {
   RETURN_NOT_OK(AllocateBuffer(pool, BitUtil::BytesForBits(length), result));
   memset((*result)->mutable_data(), 0, static_cast<size_t>((*result)->size()));
   return Status::OK();
 }
 
 Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t length,
-    std::shared_ptr<Buffer>* out) {
+                  std::shared_ptr<Buffer>* out) {
   std::shared_ptr<MutableBuffer> buffer;
   RETURN_NOT_OK(GetEmptyBitmap(pool, length, &buffer));
   uint8_t* dest = buffer->mutable_data();
@@ -111,12 +117,14 @@ Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t
 }
 
 bool BitmapEquals(const uint8_t* left, int64_t left_offset, const uint8_t* right,
-    int64_t right_offset, int64_t bit_length) {
+                  int64_t right_offset, int64_t bit_length) {
   if (left_offset % 8 == 0 && right_offset % 8 == 0) {
     // byte aligned, can use memcmp
     bool bytes_equal = std::memcmp(left + left_offset / 8, right + right_offset / 8,
-                           bit_length / 8) == 0;
-    if (!bytes_equal) { return false; }
+                                   bit_length / 8) == 0;
+    if (!bytes_equal) {
+      return false;
+    }
     for (int64_t i = (bit_length / 8) * 8; i < bit_length; ++i) {
       if (BitUtil::GetBit(left, left_offset + i) !=
           BitUtil::GetBit(right, right_offset + i)) {
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index d055c751d16fa..f036763b8106e 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -101,17 +101,11 @@ static constexpr uint8_t kBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128};
 // the ~i byte version of kBitmaks
 static constexpr uint8_t kFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127};
 
-static inline int64_t CeilByte(int64_t size) {
-  return (size + 7) & ~7;
-}
+static inline int64_t CeilByte(int64_t size) { return (size + 7) & ~7; }
 
-static inline int64_t BytesForBits(int64_t size) {
-  return CeilByte(size) / 8;
-}
+static inline int64_t BytesForBits(int64_t size) { return CeilByte(size) / 8; }
 
-static inline int64_t Ceil2Bytes(int64_t size) {
-  return (size + 15) & ~15;
-}
+static inline int64_t Ceil2Bytes(int64_t size) { return (size + 15) & ~15; }
 
 static inline bool GetBit(const uint8_t* bits, int64_t i) {
   return (bits[i / 8] & kBitmask[i % 8]) != 0;
@@ -125,13 +119,13 @@ static inline void ClearBit(uint8_t* bits, int64_t i) {
   bits[i / 8] &= kFlippedBitmask[i % 8];
 }
 
-static inline void SetBit(uint8_t* bits, int64_t i) {
-  bits[i / 8] |= kBitmask[i % 8];
-}
+static inline void SetBit(uint8_t* bits, int64_t i) { bits[i / 8] |= kBitmask[i % 8]; }
 
 /// Set bit if is_set is true, but cannot clear bit
 static inline void SetArrayBit(uint8_t* bits, int i, bool is_set) {
-  if (is_set) { SetBit(bits, i); }
+  if (is_set) {
+    SetBit(bits, i);
+  }
 }
 
 static inline void SetBitTo(uint8_t* bits, int64_t i, bool bit_is_set) {
@@ -168,13 +162,9 @@ static inline int64_t NextPower2(int64_t n) {
   return n;
 }
 
-static inline bool IsMultipleOf64(int64_t n) {
-  return (n & 63) == 0;
-}
+static inline bool IsMultipleOf64(int64_t n) { return (n & 63) == 0; }
 
-static inline bool IsMultipleOf8(int64_t n) {
-  return (n & 7) == 0;
-}
+static inline bool IsMultipleOf8(int64_t n) { return (n & 7) == 0; }
 
 /// Returns the ceil of value/divisor
 static inline int64_t Ceil(int64_t value, int64_t divisor) {
@@ -206,34 +196,22 @@ static inline int RoundDownToPowerOf2(int value, int factor) {
 /// Specialized round up and down functions for frequently used factors,
 /// like 8 (bits->bytes), 32 (bits->i32), and 64 (bits->i64).
 /// Returns the rounded up number of bytes that fit the number of bits.
-static inline uint32_t RoundUpNumBytes(uint32_t bits) {
-  return (bits + 7) >> 3;
-}
+static inline uint32_t RoundUpNumBytes(uint32_t bits) { return (bits + 7) >> 3; }
 
 /// Returns the rounded down number of bytes that fit the number of bits.
-static inline uint32_t RoundDownNumBytes(uint32_t bits) {
-  return bits >> 3;
-}
+static inline uint32_t RoundDownNumBytes(uint32_t bits) { return bits >> 3; }
 
 /// Returns the rounded up to 32 multiple. Used for conversions of bits to i32.
-static inline uint32_t RoundUpNumi32(uint32_t bits) {
-  return (bits + 31) >> 5;
-}
+static inline uint32_t RoundUpNumi32(uint32_t bits) { return (bits + 31) >> 5; }
 
 /// Returns the rounded up 32 multiple.
-static inline uint32_t RoundDownNumi32(uint32_t bits) {
-  return bits >> 5;
-}
+static inline uint32_t RoundDownNumi32(uint32_t bits) { return bits >> 5; }
 
 /// Returns the rounded up to 64 multiple. Used for conversions of bits to i64.
-static inline uint32_t RoundUpNumi64(uint32_t bits) {
-  return (bits + 63) >> 6;
-}
+static inline uint32_t RoundUpNumi64(uint32_t bits) { return (bits + 63) >> 6; }
 
 /// Returns the rounded down to 64 multiple.
-static inline uint32_t RoundDownNumi64(uint32_t bits) {
-  return bits >> 6;
-}
+static inline uint32_t RoundDownNumi64(uint32_t bits) { return bits >> 6; }
 
 static inline int64_t RoundUpToMultipleOf64(int64_t num) {
   // TODO(wesm): is this definitely needed?
@@ -242,7 +220,9 @@ static inline int64_t RoundUpToMultipleOf64(int64_t num) {
   constexpr int64_t force_carry_addend = round_to - 1;
   constexpr int64_t truncate_bitmask = ~(round_to - 1);
   constexpr int64_t max_roundable_num = std::numeric_limits<int64_t>::max() - round_to;
-  if (num <= max_roundable_num) { return (num + force_carry_addend) & truncate_bitmask; }
+  if (num <= max_roundable_num) {
+    return (num + force_carry_addend) & truncate_bitmask;
+  }
   // handle overflow case.  This should result in a malloc error upstream
   return num;
 }
@@ -252,8 +232,7 @@ static inline int64_t RoundUpToMultipleOf64(int64_t num) {
 /// might be a much faster way to implement this.
 static inline int PopcountNoHw(uint64_t x) {
   int count = 0;
-  for (; x != 0; ++count)
-    x &= x - 1;
+  for (; x != 0; ++count) x &= x - 1;
   return count;
 }
 
@@ -297,21 +276,16 @@ static inline int Log2(uint64_t x) {
   // (floor(log2(n)) = MSB(n) (0-indexed))
   --x;
   int result = 1;
-  while (x >>= 1)
-    ++result;
+  while (x >>= 1) ++result;
   return result;
 }
 
 /// Swaps the byte order (i.e. endianess)
-static inline int64_t ByteSwap(int64_t value) {
-  return ARROW_BYTE_SWAP64(value);
-}
+static inline int64_t ByteSwap(int64_t value) { return ARROW_BYTE_SWAP64(value); }
 static inline uint64_t ByteSwap(uint64_t value) {
   return static_cast<uint64_t>(ARROW_BYTE_SWAP64(value));
 }
-static inline int32_t ByteSwap(int32_t value) {
-  return ARROW_BYTE_SWAP32(value);
-}
+static inline int32_t ByteSwap(int32_t value) { return ARROW_BYTE_SWAP32(value); }
 static inline uint32_t ByteSwap(uint32_t value) {
   return static_cast<uint32_t>(ARROW_BYTE_SWAP32(value));
 }
@@ -352,84 +326,36 @@ static inline void ByteSwap(void* dst, const void* src, int len) {
 /// Converts to big endian format (if not already in big endian) from the
 /// machine's native endian format.
 #if __BYTE_ORDER == __LITTLE_ENDIAN
-static inline int64_t ToBigEndian(int64_t value) {
-  return ByteSwap(value);
-}
-static inline uint64_t ToBigEndian(uint64_t value) {
-  return ByteSwap(value);
-}
-static inline int32_t ToBigEndian(int32_t value) {
-  return ByteSwap(value);
-}
-static inline uint32_t ToBigEndian(uint32_t value) {
-  return ByteSwap(value);
-}
-static inline int16_t ToBigEndian(int16_t value) {
-  return ByteSwap(value);
-}
-static inline uint16_t ToBigEndian(uint16_t value) {
-  return ByteSwap(value);
-}
+static inline int64_t ToBigEndian(int64_t value) { return ByteSwap(value); }
+static inline uint64_t ToBigEndian(uint64_t value) { return ByteSwap(value); }
+static inline int32_t ToBigEndian(int32_t value) { return ByteSwap(value); }
+static inline uint32_t ToBigEndian(uint32_t value) { return ByteSwap(value); }
+static inline int16_t ToBigEndian(int16_t value) { return ByteSwap(value); }
+static inline uint16_t ToBigEndian(uint16_t value) { return ByteSwap(value); }
 #else
-static inline int64_t ToBigEndian(int64_t val) {
-  return val;
-}
-static inline uint64_t ToBigEndian(uint64_t val) {
-  return val;
-}
-static inline int32_t ToBigEndian(int32_t val) {
-  return val;
-}
-static inline uint32_t ToBigEndian(uint32_t val) {
-  return val;
-}
-static inline int16_t ToBigEndian(int16_t val) {
-  return val;
-}
-static inline uint16_t ToBigEndian(uint16_t val) {
-  return val;
-}
+static inline int64_t ToBigEndian(int64_t val) { return val; }
+static inline uint64_t ToBigEndian(uint64_t val) { return val; }
+static inline int32_t ToBigEndian(int32_t val) { return val; }
+static inline uint32_t ToBigEndian(uint32_t val) { return val; }
+static inline int16_t ToBigEndian(int16_t val) { return val; }
+static inline uint16_t ToBigEndian(uint16_t val) { return val; }
 #endif
 
 /// Converts from big endian format to the machine's native endian format.
 #if __BYTE_ORDER == __LITTLE_ENDIAN
-static inline int64_t FromBigEndian(int64_t value) {
-  return ByteSwap(value);
-}
-static inline uint64_t FromBigEndian(uint64_t value) {
-  return ByteSwap(value);
-}
-static inline int32_t FromBigEndian(int32_t value) {
-  return ByteSwap(value);
-}
-static inline uint32_t FromBigEndian(uint32_t value) {
-  return ByteSwap(value);
-}
-static inline int16_t FromBigEndian(int16_t value) {
-  return ByteSwap(value);
-}
-static inline uint16_t FromBigEndian(uint16_t value) {
-  return ByteSwap(value);
-}
+static inline int64_t FromBigEndian(int64_t value) { return ByteSwap(value); }
+static inline uint64_t FromBigEndian(uint64_t value) { return ByteSwap(value); }
+static inline int32_t FromBigEndian(int32_t value) { return ByteSwap(value); }
+static inline uint32_t FromBigEndian(uint32_t value) { return ByteSwap(value); }
+static inline int16_t FromBigEndian(int16_t value) { return ByteSwap(value); }
+static inline uint16_t FromBigEndian(uint16_t value) { return ByteSwap(value); }
 #else
-static inline int64_t FromBigEndian(int64_t val) {
-  return val;
-}
-static inline uint64_t FromBigEndian(uint64_t val) {
-  return val;
-}
-static inline int32_t FromBigEndian(int32_t val) {
-  return val;
-}
-static inline uint32_t FromBigEndian(uint32_t val) {
-  return val;
-}
-static inline int16_t FromBigEndian(int16_t val) {
-  return val;
-}
-static inline uint16_t FromBigEndian(uint16_t val) {
-  return val;
-}
+static inline int64_t FromBigEndian(int64_t val) { return val; }
+static inline uint64_t FromBigEndian(uint64_t val) { return val; }
+static inline int32_t FromBigEndian(int32_t val) { return val; }
+static inline uint32_t FromBigEndian(uint32_t val) { return val; }
+static inline int16_t FromBigEndian(int16_t val) { return val; }
+static inline uint16_t FromBigEndian(uint16_t val) { return val; }
 #endif
 
 // Logical right shift for signed integer types
@@ -449,8 +375,8 @@ ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buf
 // ----------------------------------------------------------------------
 // Bitmap utilities
 
-Status ARROW_EXPORT GetEmptyBitmap(
-    MemoryPool* pool, int64_t length, std::shared_ptr<MutableBuffer>* result);
+Status ARROW_EXPORT GetEmptyBitmap(MemoryPool* pool, int64_t length,
+                                   std::shared_ptr<MutableBuffer>* result);
 
 /// Copy a bit range of an existing bitmap
 ///
@@ -462,7 +388,7 @@ Status ARROW_EXPORT GetEmptyBitmap(
 ///
 /// \return Status message
 Status ARROW_EXPORT CopyBitmap(MemoryPool* pool, const uint8_t* bitmap, int64_t offset,
-    int64_t length, std::shared_ptr<Buffer>* out);
+                               int64_t length, std::shared_ptr<Buffer>* out);
 
 /// Compute the number of 1's in the given data array
 ///
@@ -471,11 +397,12 @@ Status ARROW_EXPORT CopyBitmap(MemoryPool* pool, const uint8_t* bitmap, int64_t
 /// \param[in] length the number of bits to inspect in the bitmap relative to the offset
 ///
 /// \return The number of set (1) bits in the range
-int64_t ARROW_EXPORT CountSetBits(
-    const uint8_t* data, int64_t bit_offset, int64_t length);
+int64_t ARROW_EXPORT CountSetBits(const uint8_t* data, int64_t bit_offset,
+                                  int64_t length);
 
 bool ARROW_EXPORT BitmapEquals(const uint8_t* left, int64_t left_offset,
-    const uint8_t* right, int64_t right_offset, int64_t bit_length);
+                               const uint8_t* right, int64_t right_offset,
+                               int64_t bit_length);
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_BIT_UTIL_H
diff --git a/cpp/src/arrow/util/bpacking.h b/cpp/src/arrow/util/bpacking.h
index fce5f55224cd4..4d25de0ab060c 100644
--- a/cpp/src/arrow/util/bpacking.h
+++ b/cpp/src/arrow/util/bpacking.h
@@ -3199,136 +3199,103 @@ inline int unpack32(const uint32_t* in, uint32_t* out, int batch_size, int num_b
 
   switch (num_bits) {
     case 0:
-      for (int i = 0; i < num_loops; ++i)
-        in = nullunpacker32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = nullunpacker32(in, out + i * 32);
       break;
     case 1:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack1_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack1_32(in, out + i * 32);
       break;
     case 2:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack2_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack2_32(in, out + i * 32);
       break;
     case 3:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack3_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack3_32(in, out + i * 32);
       break;
     case 4:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack4_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack4_32(in, out + i * 32);
       break;
     case 5:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack5_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack5_32(in, out + i * 32);
       break;
     case 6:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack6_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack6_32(in, out + i * 32);
       break;
     case 7:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack7_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack7_32(in, out + i * 32);
       break;
     case 8:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack8_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack8_32(in, out + i * 32);
       break;
     case 9:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack9_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack9_32(in, out + i * 32);
       break;
     case 10:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack10_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack10_32(in, out + i * 32);
       break;
     case 11:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack11_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack11_32(in, out + i * 32);
       break;
     case 12:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack12_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack12_32(in, out + i * 32);
       break;
     case 13:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack13_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack13_32(in, out + i * 32);
       break;
     case 14:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack14_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack14_32(in, out + i * 32);
       break;
     case 15:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack15_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack15_32(in, out + i * 32);
       break;
     case 16:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack16_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack16_32(in, out + i * 32);
       break;
     case 17:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack17_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack17_32(in, out + i * 32);
       break;
     case 18:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack18_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack18_32(in, out + i * 32);
       break;
     case 19:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack19_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack19_32(in, out + i * 32);
       break;
     case 20:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack20_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack20_32(in, out + i * 32);
       break;
     case 21:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack21_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack21_32(in, out + i * 32);
       break;
     case 22:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack22_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack22_32(in, out + i * 32);
       break;
     case 23:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack23_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack23_32(in, out + i * 32);
       break;
     case 24:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack24_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack24_32(in, out + i * 32);
       break;
     case 25:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack25_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack25_32(in, out + i * 32);
       break;
     case 26:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack26_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack26_32(in, out + i * 32);
       break;
     case 27:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack27_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack27_32(in, out + i * 32);
       break;
     case 28:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack28_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack28_32(in, out + i * 32);
       break;
     case 29:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack29_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack29_32(in, out + i * 32);
       break;
     case 30:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack30_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack30_32(in, out + i * 32);
       break;
     case 31:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack31_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack31_32(in, out + i * 32);
       break;
     case 32:
-      for (int i = 0; i < num_loops; ++i)
-        in = unpack32_32(in, out + i * 32);
+      for (int i = 0; i < num_loops; ++i) in = unpack32_32(in, out + i * 32);
       break;
     default:
       DCHECK(false) << "Unsupported num_bits";
diff --git a/cpp/src/arrow/util/compression-test.cc b/cpp/src/arrow/util/compression-test.cc
index f7739fc6dd7c3..64896dd6a4a15 100644
--- a/cpp/src/arrow/util/compression-test.cc
+++ b/cpp/src/arrow/util/compression-test.cc
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <cstdint>
 #include <gtest/gtest.h>
+#include <cstdint>
 #include <string>
 #include <vector>
 
@@ -43,25 +43,25 @@ void CheckCodecRoundtrip(const vector<uint8_t>& data) {
 
   // compress with c1
   int64_t actual_size;
-  ASSERT_OK(c1->Compress(
-      data.size(), &data[0], max_compressed_len, &compressed[0], &actual_size));
+  ASSERT_OK(c1->Compress(data.size(), &data[0], max_compressed_len, &compressed[0],
+                         &actual_size));
   compressed.resize(actual_size);
 
   // decompress with c2
-  ASSERT_OK(c2->Decompress(
-      compressed.size(), &compressed[0], decompressed.size(), &decompressed[0]));
+  ASSERT_OK(c2->Decompress(compressed.size(), &compressed[0], decompressed.size(),
+                           &decompressed[0]));
 
   ASSERT_EQ(data, decompressed);
 
   // compress with c2
   int64_t actual_size2;
-  ASSERT_OK(c2->Compress(
-      data.size(), &data[0], max_compressed_len, &compressed[0], &actual_size2));
+  ASSERT_OK(c2->Compress(data.size(), &data[0], max_compressed_len, &compressed[0],
+                         &actual_size2));
   ASSERT_EQ(actual_size2, actual_size);
 
   // decompress with c1
-  ASSERT_OK(c1->Decompress(
-      compressed.size(), &compressed[0], decompressed.size(), &decompressed[0]));
+  ASSERT_OK(c1->Decompress(compressed.size(), &compressed[0], decompressed.size(),
+                           &decompressed[0]));
 
   ASSERT_EQ(data, decompressed);
 }
@@ -76,24 +76,14 @@ void CheckCodec() {
   }
 }
 
-TEST(TestCompressors, Snappy) {
-  CheckCodec<Compression::SNAPPY>();
-}
+TEST(TestCompressors, Snappy) { CheckCodec<Compression::SNAPPY>(); }
 
-TEST(TestCompressors, Brotli) {
-  CheckCodec<Compression::BROTLI>();
-}
+TEST(TestCompressors, Brotli) { CheckCodec<Compression::BROTLI>(); }
 
-TEST(TestCompressors, GZip) {
-  CheckCodec<Compression::GZIP>();
-}
+TEST(TestCompressors, GZip) { CheckCodec<Compression::GZIP>(); }
 
-TEST(TestCompressors, ZSTD) {
-  CheckCodec<Compression::ZSTD>();
-}
+TEST(TestCompressors, ZSTD) { CheckCodec<Compression::ZSTD>(); }
 
-TEST(TestCompressors, Lz4) {
-  CheckCodec<Compression::LZ4>();
-}
+TEST(TestCompressors, Lz4) { CheckCodec<Compression::LZ4>(); }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h
index 19c61179a502a..ae187a7fcdf1c 100644
--- a/cpp/src/arrow/util/compression.h
+++ b/cpp/src/arrow/util/compression.h
@@ -37,10 +37,11 @@ class ARROW_EXPORT Codec {
   static Status Create(Compression::type codec, std::unique_ptr<Codec>* out);
 
   virtual Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) = 0;
+                            uint8_t* output_buffer) = 0;
 
   virtual Status Compress(int64_t input_len, const uint8_t* input,
-      int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) = 0;
+                          int64_t output_buffer_len, uint8_t* output_buffer,
+                          int64_t* output_length) = 0;
 
   virtual int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) = 0;
 
diff --git a/cpp/src/arrow/util/compression_brotli.cc b/cpp/src/arrow/util/compression_brotli.cc
index c03573bc46c1c..e4639083dfadb 100644
--- a/cpp/src/arrow/util/compression_brotli.cc
+++ b/cpp/src/arrow/util/compression_brotli.cc
@@ -33,8 +33,8 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Brotli implementation
 
-Status BrotliCodec::Decompress(
-    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+Status BrotliCodec::Decompress(int64_t input_len, const uint8_t* input,
+                               int64_t output_len, uint8_t* output_buffer) {
   size_t output_size = output_len;
   if (BrotliDecoderDecompress(input_len, input, &output_size, output_buffer) !=
       BROTLI_DECODER_RESULT_SUCCESS) {
@@ -48,12 +48,13 @@ int64_t BrotliCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
 }
 
 Status BrotliCodec::Compress(int64_t input_len, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+                             int64_t output_buffer_len, uint8_t* output_buffer,
+                             int64_t* output_length) {
   size_t output_len = output_buffer_len;
   // TODO: Make quality configurable. We use 8 as a default as it is the best
   //       trade-off for Parquet workload
   if (BrotliEncoderCompress(8, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, input_len,
-          input, &output_len, output_buffer) == BROTLI_FALSE) {
+                            input, &output_len, output_buffer) == BROTLI_FALSE) {
     return Status::IOError("Brotli compression failure.");
   }
   *output_length = output_len;
diff --git a/cpp/src/arrow/util/compression_brotli.h b/cpp/src/arrow/util/compression_brotli.h
index 08bd3379e3489..9e92cb106d422 100644
--- a/cpp/src/arrow/util/compression_brotli.h
+++ b/cpp/src/arrow/util/compression_brotli.h
@@ -30,10 +30,10 @@ namespace arrow {
 class ARROW_EXPORT BrotliCodec : public Codec {
  public:
   Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
+                    uint8_t* output_buffer) override;
 
   Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
+                  uint8_t* output_buffer, int64_t* output_length) override;
 
   int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
 
diff --git a/cpp/src/arrow/util/compression_lz4.cc b/cpp/src/arrow/util/compression_lz4.cc
index 65eaa08946e79..295e9a438f799 100644
--- a/cpp/src/arrow/util/compression_lz4.cc
+++ b/cpp/src/arrow/util/compression_lz4.cc
@@ -32,12 +32,14 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Lz4 implementation
 
-Status Lz4Codec::Decompress(
-    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
-  int64_t decompressed_size = LZ4_decompress_safe(reinterpret_cast<const char*>(input),
-      reinterpret_cast<char*>(output_buffer), static_cast<int>(input_len),
-      static_cast<int>(output_len));
-  if (decompressed_size < 1) { return Status::IOError("Corrupt Lz4 compressed data."); }
+Status Lz4Codec::Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+                            uint8_t* output_buffer) {
+  int64_t decompressed_size = LZ4_decompress_safe(
+      reinterpret_cast<const char*>(input), reinterpret_cast<char*>(output_buffer),
+      static_cast<int>(input_len), static_cast<int>(output_len));
+  if (decompressed_size < 1) {
+    return Status::IOError("Corrupt Lz4 compressed data.");
+  }
   return Status::OK();
 }
 
@@ -46,11 +48,14 @@ int64_t Lz4Codec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
 }
 
 Status Lz4Codec::Compress(int64_t input_len, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
-  *output_length = LZ4_compress_default(reinterpret_cast<const char*>(input),
-      reinterpret_cast<char*>(output_buffer), static_cast<int>(input_len),
-      static_cast<int>(output_buffer_len));
-  if (*output_length < 1) { return Status::IOError("Lz4 compression failure."); }
+                          int64_t output_buffer_len, uint8_t* output_buffer,
+                          int64_t* output_length) {
+  *output_length = LZ4_compress_default(
+      reinterpret_cast<const char*>(input), reinterpret_cast<char*>(output_buffer),
+      static_cast<int>(input_len), static_cast<int>(output_buffer_len));
+  if (*output_length < 1) {
+    return Status::IOError("Lz4 compression failure.");
+  }
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/util/compression_lz4.h b/cpp/src/arrow/util/compression_lz4.h
index 9668fec126b12..0af228963f320 100644
--- a/cpp/src/arrow/util/compression_lz4.h
+++ b/cpp/src/arrow/util/compression_lz4.h
@@ -30,10 +30,10 @@ namespace arrow {
 class ARROW_EXPORT Lz4Codec : public Codec {
  public:
   Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
+                    uint8_t* output_buffer) override;
 
   Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
+                  uint8_t* output_buffer, int64_t* output_length) override;
 
   int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
 
diff --git a/cpp/src/arrow/util/compression_snappy.cc b/cpp/src/arrow/util/compression_snappy.cc
index db2b67355109a..947ffe559bda6 100644
--- a/cpp/src/arrow/util/compression_snappy.cc
+++ b/cpp/src/arrow/util/compression_snappy.cc
@@ -37,10 +37,11 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Snappy implementation
 
-Status SnappyCodec::Decompress(
-    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
+Status SnappyCodec::Decompress(int64_t input_len, const uint8_t* input,
+                               int64_t output_len, uint8_t* output_buffer) {
   if (!snappy::RawUncompress(reinterpret_cast<const char*>(input),
-          static_cast<size_t>(input_len), reinterpret_cast<char*>(output_buffer))) {
+                             static_cast<size_t>(input_len),
+                             reinterpret_cast<char*>(output_buffer))) {
     return Status::IOError("Corrupt snappy compressed data.");
   }
   return Status::OK();
@@ -51,11 +52,12 @@ int64_t SnappyCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
 }
 
 Status SnappyCodec::Compress(int64_t input_len, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+                             int64_t output_buffer_len, uint8_t* output_buffer,
+                             int64_t* output_length) {
   size_t output_len;
   snappy::RawCompress(reinterpret_cast<const char*>(input),
-      static_cast<size_t>(input_len), reinterpret_cast<char*>(output_buffer),
-      &output_len);
+                      static_cast<size_t>(input_len),
+                      reinterpret_cast<char*>(output_buffer), &output_len);
   *output_length = static_cast<int64_t>(output_len);
   return Status::OK();
 }
diff --git a/cpp/src/arrow/util/compression_snappy.h b/cpp/src/arrow/util/compression_snappy.h
index 25281e1a97a16..5cc10c470af45 100644
--- a/cpp/src/arrow/util/compression_snappy.h
+++ b/cpp/src/arrow/util/compression_snappy.h
@@ -29,10 +29,10 @@ namespace arrow {
 class ARROW_EXPORT SnappyCodec : public Codec {
  public:
   Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
+                    uint8_t* output_buffer) override;
 
   Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
+                  uint8_t* output_buffer, int64_t* output_length) override;
 
   int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
 
diff --git a/cpp/src/arrow/util/compression_zlib.cc b/cpp/src/arrow/util/compression_zlib.cc
index 3ff33b82028e8..ae6627ea6442f 100644
--- a/cpp/src/arrow/util/compression_zlib.cc
+++ b/cpp/src/arrow/util/compression_zlib.cc
@@ -69,7 +69,7 @@ class GZipCodec::GZipCodecImpl {
       window_bits += GZIP_CODEC;
     }
     if ((ret = deflateInit2(&stream_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits, 9,
-             Z_DEFAULT_STRATEGY)) != Z_OK) {
+                            Z_DEFAULT_STRATEGY)) != Z_OK) {
       std::stringstream ss;
       ss << "zlib deflateInit failed: " << std::string(stream_.msg);
       return Status::IOError(ss.str());
@@ -79,7 +79,9 @@ class GZipCodec::GZipCodecImpl {
   }
 
   void EndCompressor() {
-    if (compressor_initialized_) { (void)deflateEnd(&stream_); }
+    if (compressor_initialized_) {
+      (void)deflateEnd(&stream_);
+    }
     compressor_initialized_ = false;
   }
 
@@ -100,13 +102,17 @@ class GZipCodec::GZipCodecImpl {
   }
 
   void EndDecompressor() {
-    if (decompressor_initialized_) { (void)inflateEnd(&stream_); }
+    if (decompressor_initialized_) {
+      (void)inflateEnd(&stream_);
+    }
     decompressor_initialized_ = false;
   }
 
   Status Decompress(int64_t input_length, const uint8_t* input, int64_t output_length,
-      uint8_t* output) {
-    if (!decompressor_initialized_) { RETURN_NOT_OK(InitDecompressor()); }
+                    uint8_t* output) {
+    if (!decompressor_initialized_) {
+      RETURN_NOT_OK(InitDecompressor());
+    }
     if (output_length == 0) {
       // The zlib library does not allow *output to be NULL, even when output_length
       // is 0 (inflate() will return Z_STREAM_ERROR). We don't consider this an
@@ -168,8 +174,10 @@ class GZipCodec::GZipCodecImpl {
   }
 
   Status Compress(int64_t input_length, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output, int64_t* output_length) {
-    if (!compressor_initialized_) { RETURN_NOT_OK(InitCompressor()); }
+                  uint8_t* output, int64_t* output_length) {
+    if (!compressor_initialized_) {
+      RETURN_NOT_OK(InitCompressor());
+    }
     stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input));
     stream_.avail_in = static_cast<uInt>(input_length);
     stream_.next_out = reinterpret_cast<Bytef*>(output);
@@ -218,14 +226,12 @@ class GZipCodec::GZipCodecImpl {
   bool decompressor_initialized_;
 };
 
-GZipCodec::GZipCodec(Format format) {
-  impl_.reset(new GZipCodecImpl(format));
-}
+GZipCodec::GZipCodec(Format format) { impl_.reset(new GZipCodecImpl(format)); }
 
 GZipCodec::~GZipCodec() {}
 
 Status GZipCodec::Decompress(int64_t input_length, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output) {
+                             int64_t output_buffer_len, uint8_t* output) {
   return impl_->Decompress(input_length, input, output_buffer_len, output);
 }
 
@@ -234,12 +240,11 @@ int64_t GZipCodec::MaxCompressedLen(int64_t input_length, const uint8_t* input)
 }
 
 Status GZipCodec::Compress(int64_t input_length, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output, int64_t* output_length) {
+                           int64_t output_buffer_len, uint8_t* output,
+                           int64_t* output_length) {
   return impl_->Compress(input_length, input, output_buffer_len, output, output_length);
 }
 
-const char* GZipCodec::name() const {
-  return "gzip";
-}
+const char* GZipCodec::name() const { return "gzip"; }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/compression_zlib.h b/cpp/src/arrow/util/compression_zlib.h
index 517a06175ec8f..f55d6689edfa9 100644
--- a/cpp/src/arrow/util/compression_zlib.h
+++ b/cpp/src/arrow/util/compression_zlib.h
@@ -40,10 +40,10 @@ class ARROW_EXPORT GZipCodec : public Codec {
   virtual ~GZipCodec();
 
   Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
+                    uint8_t* output_buffer) override;
 
   Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
+                  uint8_t* output_buffer, int64_t* output_length) override;
 
   int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
 
diff --git a/cpp/src/arrow/util/compression_zstd.cc b/cpp/src/arrow/util/compression_zstd.cc
index 5511cb9dd8f37..ac6e9065d22dd 100644
--- a/cpp/src/arrow/util/compression_zstd.cc
+++ b/cpp/src/arrow/util/compression_zstd.cc
@@ -32,10 +32,11 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // ZSTD implementation
 
-Status ZSTDCodec::Decompress(
-    int64_t input_len, const uint8_t* input, int64_t output_len, uint8_t* output_buffer) {
-  int64_t decompressed_size = ZSTD_decompress(output_buffer,
-      static_cast<size_t>(output_len), input, static_cast<size_t>(input_len));
+Status ZSTDCodec::Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+                             uint8_t* output_buffer) {
+  int64_t decompressed_size =
+      ZSTD_decompress(output_buffer, static_cast<size_t>(output_len), input,
+                      static_cast<size_t>(input_len));
   if (decompressed_size != output_len) {
     return Status::IOError("Corrupt ZSTD compressed data.");
   }
@@ -47,9 +48,10 @@ int64_t ZSTDCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
 }
 
 Status ZSTDCodec::Compress(int64_t input_len, const uint8_t* input,
-    int64_t output_buffer_len, uint8_t* output_buffer, int64_t* output_length) {
+                           int64_t output_buffer_len, uint8_t* output_buffer,
+                           int64_t* output_length) {
   *output_length = ZSTD_compress(output_buffer, static_cast<size_t>(output_buffer_len),
-      input, static_cast<size_t>(input_len), 1);
+                                 input, static_cast<size_t>(input_len), 1);
   if (ZSTD_isError(*output_length)) {
     return Status::IOError("ZSTD compression failure.");
   }
diff --git a/cpp/src/arrow/util/compression_zstd.h b/cpp/src/arrow/util/compression_zstd.h
index 2356d5862e01a..6e40e19d280d7 100644
--- a/cpp/src/arrow/util/compression_zstd.h
+++ b/cpp/src/arrow/util/compression_zstd.h
@@ -30,10 +30,10 @@ namespace arrow {
 class ARROW_EXPORT ZSTDCodec : public Codec {
  public:
   Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
-      uint8_t* output_buffer) override;
+                    uint8_t* output_buffer) override;
 
   Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
-      uint8_t* output_buffer, int64_t* output_length) override;
+                  uint8_t* output_buffer, int64_t* output_length) override;
 
   int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
 
diff --git a/cpp/src/arrow/util/cpu-info.cc b/cpp/src/arrow/util/cpu-info.cc
index dcd6b4027d966..b0667cb33ada4 100644
--- a/cpp/src/arrow/util/cpu-info.cc
+++ b/cpp/src/arrow/util/cpu-info.cc
@@ -66,7 +66,9 @@ static struct {
   string name;
   int64_t flag;
 } flag_mappings[] = {
-    {"ssse3", CpuInfo::SSSE3}, {"sse4_1", CpuInfo::SSE4_1}, {"sse4_2", CpuInfo::SSE4_2},
+    {"ssse3", CpuInfo::SSSE3},
+    {"sse4_1", CpuInfo::SSE4_1},
+    {"sse4_2", CpuInfo::SSE4_2},
     {"popcnt", CpuInfo::POPCNT},
 };
 static const int64_t num_flags = sizeof(flag_mappings) / sizeof(flag_mappings[0]);
@@ -78,14 +80,18 @@ static const int64_t num_flags = sizeof(flag_mappings) / sizeof(flag_mappings[0]
 int64_t ParseCPUFlags(const string& values) {
   int64_t flags = 0;
   for (int i = 0; i < num_flags; ++i) {
-    if (contains(values, flag_mappings[i].name)) { flags |= flag_mappings[i].flag; }
+    if (contains(values, flag_mappings[i].name)) {
+      flags |= flag_mappings[i].flag;
+    }
   }
   return flags;
 }
 
 #ifdef _WIN32
 bool RetrieveCacheSize(int64_t* cache_sizes) {
-  if (!cache_sizes) { return false; }
+  if (!cache_sizes) {
+    return false;
+  }
   PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = nullptr;
   PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer_position = nullptr;
   DWORD buffer_size = 0;
@@ -95,7 +101,9 @@ bool RetrieveCacheSize(int64_t* cache_sizes) {
       (GetLogicalProcessorInformationFuncPointer)GetProcAddress(
           GetModuleHandle("kernel32"), "GetLogicalProcessorInformation");
 
-  if (!func_pointer) { return false; }
+  if (!func_pointer) {
+    return false;
+  }
 
   // Get buffer size
   if (func_pointer(buffer, &buffer_size) && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
@@ -103,7 +111,9 @@ bool RetrieveCacheSize(int64_t* cache_sizes) {
 
   buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(buffer_size);
 
-  if (!buffer || !func_pointer(buffer, &buffer_size)) { return false; }
+  if (!buffer || !func_pointer(buffer, &buffer_size)) {
+    return false;
+  }
 
   buffer_position = buffer;
   while (offset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= buffer_size) {
@@ -117,7 +127,9 @@ bool RetrieveCacheSize(int64_t* cache_sizes) {
     buffer_position++;
   }
 
-  if (buffer) { free(buffer); }
+  if (buffer) {
+    free(buffer);
+  }
   return true;
 }
 #endif
@@ -125,7 +137,9 @@ bool RetrieveCacheSize(int64_t* cache_sizes) {
 void CpuInfo::Init() {
   std::lock_guard<std::mutex> cpuinfo_lock(cpuinfo_mutex);
 
-  if (initialized()) { return; }
+  if (initialized()) {
+    return;
+  }
 
   string line;
   string name;
@@ -186,7 +200,9 @@ void CpuInfo::Init() {
     cache_sizes_[i] = data[i];
   }
 #elif _WIN32
-  if (!RetrieveCacheSize(cache_sizes_)) { SetDefaultCacheSize(); }
+  if (!RetrieveCacheSize(cache_sizes_)) {
+    SetDefaultCacheSize();
+  }
 #else
   SetDefaultCacheSize();
 #endif
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 72ede35bef9b5..1a12e20f9f93f 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -21,8 +21,8 @@ namespace arrow {
 namespace decimal {
 
 template <typename T>
-ARROW_EXPORT Status FromString(
-    const std::string& s, Decimal<T>* out, int* precision, int* scale) {
+ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out, int* precision,
+                               int* scale) {
   // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?";
   if (s.empty()) {
     return Status::Invalid("Empty string cannot be converted to decimal");
@@ -34,7 +34,9 @@ ARROW_EXPORT Status FromString(
 
   char first_char = *charp;
   if (first_char == '+' || first_char == '-') {
-    if (first_char == '-') { sign = -1; }
+    if (first_char == '-') {
+      sign = -1;
+    }
     ++charp;
   }
 
@@ -55,7 +57,9 @@ ARROW_EXPORT Status FromString(
 
   // all zeros and no decimal point
   if (charp == end) {
-    if (out != nullptr) { out->value = static_cast<T>(0); }
+    if (out != nullptr) {
+      out->value = static_cast<T>(0);
+    }
 
     // Not sure what other libraries assign precision to for this case (this case of
     // a string consisting only of one or more zeros)
@@ -63,7 +67,9 @@ ARROW_EXPORT Status FromString(
       *precision = static_cast<int>(charp - numeric_string_start);
     }
 
-    if (scale != nullptr) { *scale = 0; }
+    if (scale != nullptr) {
+      *scale = 0;
+    }
 
     return Status::OK();
   }
@@ -127,22 +133,26 @@ ARROW_EXPORT Status FromString(
     *precision = static_cast<int>(whole_part.size() + fractional_part.size());
   }
 
-  if (scale != nullptr) { *scale = static_cast<int>(fractional_part.size()); }
+  if (scale != nullptr) {
+    *scale = static_cast<int>(fractional_part.size());
+  }
 
-  if (out != nullptr) { StringToInteger(whole_part, fractional_part, sign, &out->value); }
+  if (out != nullptr) {
+    StringToInteger(whole_part, fractional_part, sign, &out->value);
+  }
 
   return Status::OK();
 }
 
-template ARROW_EXPORT Status FromString(
-    const std::string& s, Decimal32* out, int* precision, int* scale);
-template ARROW_EXPORT Status FromString(
-    const std::string& s, Decimal64* out, int* precision, int* scale);
-template ARROW_EXPORT Status FromString(
-    const std::string& s, Decimal128* out, int* precision, int* scale);
+template ARROW_EXPORT Status FromString(const std::string& s, Decimal32* out,
+                                        int* precision, int* scale);
+template ARROW_EXPORT Status FromString(const std::string& s, Decimal64* out,
+                                        int* precision, int* scale);
+template ARROW_EXPORT Status FromString(const std::string& s, Decimal128* out,
+                                        int* precision, int* scale);
 
-void StringToInteger(
-    const std::string& whole, const std::string& fractional, int8_t sign, int32_t* out) {
+void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
+                     int32_t* out) {
   DCHECK(sign == -1 || sign == 1);
   DCHECK_NE(out, nullptr);
   DCHECK(!whole.empty() || !fractional.empty());
@@ -150,12 +160,14 @@ void StringToInteger(
     *out = std::stoi(whole, nullptr, 10) *
            static_cast<int32_t>(pow(10.0, static_cast<double>(fractional.size())));
   }
-  if (!fractional.empty()) { *out += std::stoi(fractional, nullptr, 10); }
+  if (!fractional.empty()) {
+    *out += std::stoi(fractional, nullptr, 10);
+  }
   *out *= sign;
 }
 
-void StringToInteger(
-    const std::string& whole, const std::string& fractional, int8_t sign, int64_t* out) {
+void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
+                     int64_t* out) {
   DCHECK(sign == -1 || sign == 1);
   DCHECK_NE(out, nullptr);
   DCHECK(!whole.empty() || !fractional.empty());
@@ -163,12 +175,14 @@ void StringToInteger(
     *out = static_cast<int64_t>(std::stoll(whole, nullptr, 10)) *
            static_cast<int64_t>(pow(10.0, static_cast<double>(fractional.size())));
   }
-  if (!fractional.empty()) { *out += std::stoll(fractional, nullptr, 10); }
+  if (!fractional.empty()) {
+    *out += std::stoll(fractional, nullptr, 10);
+  }
   *out *= sign;
 }
 
-void StringToInteger(
-    const std::string& whole, const std::string& fractional, int8_t sign, int128_t* out) {
+void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
+                     int128_t* out) {
   DCHECK(sign == -1 || sign == 1);
   DCHECK_NE(out, nullptr);
   DCHECK(!whole.empty() || !fractional.empty());
@@ -200,7 +214,9 @@ void FromBytes(const uint8_t* bytes, bool is_negative, Decimal128* decimal) {
   int128_t::backend_type& backend(decimal_value.backend());
   backend.resize(LIMBS_IN_INT128, LIMBS_IN_INT128);
   std::memcpy(backend.limbs(), bytes, BYTES_IN_128_BITS);
-  if (is_negative) { decimal->value = -decimal->value; }
+  if (is_negative) {
+    decimal->value = -decimal->value;
+  }
 }
 
 void ToBytes(const Decimal32& value, uint8_t** bytes) {
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 0d84ba89db973..20142faea3ec5 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -37,16 +37,16 @@ using boost::multiprecision::int128_t;
 template <typename T>
 struct ARROW_EXPORT Decimal;
 
-ARROW_EXPORT void StringToInteger(
-    const std::string& whole, const std::string& fractional, int8_t sign, int32_t* out);
-ARROW_EXPORT void StringToInteger(
-    const std::string& whole, const std::string& fractional, int8_t sign, int64_t* out);
-ARROW_EXPORT void StringToInteger(
-    const std::string& whole, const std::string& fractional, int8_t sign, int128_t* out);
+ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
+                                  int8_t sign, int32_t* out);
+ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
+                                  int8_t sign, int64_t* out);
+ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
+                                  int8_t sign, int128_t* out);
 
 template <typename T>
 ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out,
-    int* precision = nullptr, int* scale = nullptr);
+                               int* precision = nullptr, int* scale = nullptr);
 
 template <typename T>
 struct ARROW_EXPORT Decimal {
@@ -85,8 +85,8 @@ struct ARROW_EXPORT DecimalPrecision<int128_t> {
 };
 
 template <typename T>
-ARROW_EXPORT std::string ToString(
-    const Decimal<T>& decimal_value, int precision, int scale) {
+ARROW_EXPORT std::string ToString(const Decimal<T>& decimal_value, int precision,
+                                  int scale) {
   T value = decimal_value.value;
 
   // Decimal values are sent to clients as strings so in the interest of
@@ -108,8 +108,8 @@ ARROW_EXPORT std::string ToString(
   if (scale > 0) {
     int remaining_scale = scale;
     do {
-      str[--last_char_idx] = static_cast<char>(
-          (remaining_value % 10) + static_cast<T>('0'));  // Ascii offset
+      str[--last_char_idx] = static_cast<char>((remaining_value % 10) +
+                                               static_cast<T>('0'));  // Ascii offset
       remaining_value /= 10;
     } while (--remaining_scale > 0);
     str[--last_char_idx] = '.';
diff --git a/cpp/src/arrow/util/key_value_metadata.cc b/cpp/src/arrow/util/key_value_metadata.cc
index 8bddd5d0164c2..6877a6a5382fe 100644
--- a/cpp/src/arrow/util/key_value_metadata.cc
+++ b/cpp/src/arrow/util/key_value_metadata.cc
@@ -48,8 +48,8 @@ KeyValueMetadata::KeyValueMetadata(
     const std::unordered_map<std::string, std::string>& map)
     : keys_(UnorderedMapKeys(map)), values_(UnorderedMapValues(map)) {}
 
-KeyValueMetadata::KeyValueMetadata(
-    const std::vector<std::string>& keys, const std::vector<std::string>& values)
+KeyValueMetadata::KeyValueMetadata(const std::vector<std::string>& keys,
+                                   const std::vector<std::string>& values)
     : keys_(keys), values_(values) {
   DCHECK_EQ(keys.size(), values.size());
 }
diff --git a/cpp/src/arrow/util/key_value_metadata.h b/cpp/src/arrow/util/key_value_metadata.h
index a2a4623aee7cc..3d602131684f6 100644
--- a/cpp/src/arrow/util/key_value_metadata.h
+++ b/cpp/src/arrow/util/key_value_metadata.h
@@ -32,8 +32,8 @@ namespace arrow {
 class ARROW_EXPORT KeyValueMetadata {
  public:
   KeyValueMetadata();
-  KeyValueMetadata(
-      const std::vector<std::string>& keys, const std::vector<std::string>& values);
+  KeyValueMetadata(const std::vector<std::string>& keys,
+                   const std::vector<std::string>& values);
   explicit KeyValueMetadata(const std::unordered_map<std::string, std::string>& map);
   virtual ~KeyValueMetadata() = default;
 
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 0edaa9dfc37a3..89e69f932d52d 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -50,32 +50,25 @@ namespace arrow {
 
 #define DCHECK(condition)      \
   ARROW_IGNORE_EXPR(condition) \
-  while (false)                \
-  ::arrow::internal::NullLog()
+  while (false) ::arrow::internal::NullLog()
 #define DCHECK_EQ(val1, val2) \
   ARROW_IGNORE_EXPR(val1)     \
-  while (false)               \
-  ::arrow::internal::NullLog()
+  while (false) ::arrow::internal::NullLog()
 #define DCHECK_NE(val1, val2) \
   ARROW_IGNORE_EXPR(val1)     \
-  while (false)               \
-  ::arrow::internal::NullLog()
+  while (false) ::arrow::internal::NullLog()
 #define DCHECK_LE(val1, val2) \
   ARROW_IGNORE_EXPR(val1)     \
-  while (false)               \
-  ::arrow::internal::NullLog()
+  while (false) ::arrow::internal::NullLog()
 #define DCHECK_LT(val1, val2) \
   ARROW_IGNORE_EXPR(val1)     \
-  while (false)               \
-  ::arrow::internal::NullLog()
+  while (false) ::arrow::internal::NullLog()
 #define DCHECK_GE(val1, val2) \
   ARROW_IGNORE_EXPR(val1)     \
-  while (false)               \
-  ::arrow::internal::NullLog()
+  while (false) ::arrow::internal::NullLog()
 #define DCHECK_GT(val1, val2) \
   ARROW_IGNORE_EXPR(val1)     \
-  while (false)               \
-  ::arrow::internal::NullLog()
+  while (false) ::arrow::internal::NullLog()
 
 #else
 #define ARROW_DFATAL ARROW_FATAL
@@ -107,8 +100,12 @@ class CerrLog {
         has_logged_(false) {}
 
   virtual ~CerrLog() {
-    if (has_logged_) { std::cerr << std::endl; }
-    if (severity_ == ARROW_FATAL) { std::exit(1); }
+    if (has_logged_) {
+      std::cerr << std::endl;
+    }
+    if (severity_ == ARROW_FATAL) {
+      std::exit(1);
+    }
   }
 
   template <class T>
@@ -133,7 +130,9 @@ class FatalLog : public CerrLog {
       : CerrLog(ARROW_FATAL){}           // NOLINT
 
             [[noreturn]] ~FatalLog() {
-    if (has_logged_) { std::cerr << std::endl; }
+    if (has_logged_) {
+      std::cerr << std::endl;
+    }
     std::exit(1);
   }
 };
diff --git a/cpp/src/arrow/util/memory.h b/cpp/src/arrow/util/memory.h
index c5c17ef907c22..fce9e19293249 100644
--- a/cpp/src/arrow/util/memory.h
+++ b/cpp/src/arrow/util/memory.h
@@ -31,7 +31,7 @@ uint8_t* pointer_logical_and(const uint8_t* address, uintptr_t bits) {
 // A helper function for doing memcpy with multiple threads. This is required
 // to saturate the memory bandwidth of modern cpus.
 void parallel_memcopy(uint8_t* dst, const uint8_t* src, int64_t nbytes,
-    uintptr_t block_size, int num_threads) {
+                      uintptr_t block_size, int num_threads) {
   std::vector<std::thread> threadpool(num_threads);
   uint8_t* left = pointer_logical_and(src + block_size - 1, ~(block_size - 1));
   uint8_t* right = pointer_logical_and(src + nbytes, ~(block_size - 1));
@@ -52,15 +52,17 @@ void parallel_memcopy(uint8_t* dst, const uint8_t* src, int64_t nbytes,
 
   // Start all threads first and handle leftovers while threads run.
   for (int i = 0; i < num_threads; i++) {
-    threadpool[i] = std::thread(
-        memcpy, dst + prefix + i * chunk_size, left + i * chunk_size, chunk_size);
+    threadpool[i] = std::thread(memcpy, dst + prefix + i * chunk_size,
+                                left + i * chunk_size, chunk_size);
   }
 
   memcpy(dst, src, prefix);
   memcpy(dst + prefix + num_threads * chunk_size, right, suffix);
 
   for (auto& t : threadpool) {
-    if (t.joinable()) { t.join(); }
+    if (t.joinable()) {
+      t.join();
+    }
   }
 }
 
diff --git a/cpp/src/arrow/util/random.h b/cpp/src/arrow/util/random.h
index 31f2b0680fe0a..ec48d5d4a529c 100644
--- a/cpp/src/arrow/util/random.h
+++ b/cpp/src/arrow/util/random.h
@@ -27,7 +27,9 @@ class Random {
  public:
   explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) {
     // Avoid bad seeds.
-    if (seed_ == 0 || seed_ == random_internal::M) { seed_ = 1; }
+    if (seed_ == 0 || seed_ == random_internal::M) {
+      seed_ = 1;
+    }
   }
 
   // Next pseudo-random 32-bit unsigned integer.
@@ -48,7 +50,9 @@ class Random {
     // The first reduction may overflow by 1 bit, so we may need to
     // repeat.  mod == M is not possible; using > allows the faster
     // sign-bit-based test.
-    if (seed_ > random_internal::M) { seed_ -= random_internal::M; }
+    if (seed_ > random_internal::M) {
+      seed_ -= random_internal::M;
+    }
     return seed_;
   }
 
@@ -97,9 +101,9 @@ class Random {
   double Normal(double mean, double std_dev) {
     double uniform1 = (Next() + 1.0) / (random_internal::M + 1.0);
     double uniform2 = (Next() + 1.0) / (random_internal::M + 1.0);
-    return (
-        mean +
-        std_dev * sqrt(-2 * ::log(uniform1)) * cos(random_internal::kTwoPi * uniform2));
+    return (mean +
+            std_dev * sqrt(-2 * ::log(uniform1)) *
+                cos(random_internal::kTwoPi * uniform2));
   }
 
   // Return a random number between 0.0 and 1.0 inclusive.
diff --git a/cpp/src/arrow/util/rle-encoding-test.cc b/cpp/src/arrow/util/rle-encoding-test.cc
index 7c9b33c349496..7549b874355df 100644
--- a/cpp/src/arrow/util/rle-encoding-test.cc
+++ b/cpp/src/arrow/util/rle-encoding-test.cc
@@ -178,7 +178,7 @@ TEST(BitArray, TestMixed) {
 // exactly 'expected_encoding'.
 // if expected_len is not -1, it will validate the encoded size is correct.
 void ValidateRle(const vector<int>& values, int bit_width, uint8_t* expected_encoding,
-    int expected_len) {
+                 int expected_len) {
   const int len = 64 * 1024;
   uint8_t buffer[len];
   EXPECT_LE(expected_len, len);
@@ -190,7 +190,9 @@ void ValidateRle(const vector<int>& values, int bit_width, uint8_t* expected_enc
   }
   int encoded_len = encoder.Flush();
 
-  if (expected_len != -1) { EXPECT_EQ(encoded_len, expected_len); }
+  if (expected_len != -1) {
+    EXPECT_EQ(encoded_len, expected_len);
+  }
   if (expected_encoding != NULL) {
     EXPECT_EQ(memcmp(buffer, expected_encoding, expected_len), 0);
   }
@@ -211,7 +213,7 @@ void ValidateRle(const vector<int>& values, int bit_width, uint8_t* expected_enc
     RleDecoder decoder(buffer, len, bit_width);
     vector<int> values_read(values.size());
     ASSERT_EQ(values.size(),
-        decoder.GetBatch(values_read.data(), static_cast<int>(values.size())));
+              decoder.GetBatch(values_read.data(), static_cast<int>(values.size())));
     EXPECT_EQ(values, values_read);
   }
 }
@@ -224,7 +226,9 @@ bool CheckRoundTrip(const vector<int>& values, int bit_width) {
   RleEncoder encoder(buffer, len, bit_width);
   for (size_t i = 0; i < values.size(); ++i) {
     bool result = encoder.Put(values[i]);
-    if (!result) { return false; }
+    if (!result) {
+      return false;
+    }
   }
   int encoded_len = encoder.Flush();
   int out = 0;
@@ -233,7 +237,9 @@ bool CheckRoundTrip(const vector<int>& values, int bit_width) {
     RleDecoder decoder(buffer, encoded_len, bit_width);
     for (size_t i = 0; i < values.size(); ++i) {
       EXPECT_TRUE(decoder.Get(&out));
-      if (values[i] != out) { return false; }
+      if (values[i] != out) {
+        return false;
+      }
     }
   }
 
@@ -245,7 +251,9 @@ bool CheckRoundTrip(const vector<int>& values, int bit_width) {
         decoder.GetBatch(values_read.data(), static_cast<int>(values.size()))) {
       return false;
     }
-    if (values != values_read) { return false; }
+    if (values != values_read) {
+      return false;
+    }
   }
 
   return true;
@@ -294,8 +302,8 @@ TEST(Rle, SpecificSequences) {
   ValidateRle(values, 1, expected_buffer, 1 + num_groups);
   for (int width = 2; width <= MAX_WIDTH; ++width) {
     int num_values = static_cast<int>(BitUtil::Ceil(100, 8)) * 8;
-    ValidateRle(
-        values, width, NULL, 1 + static_cast<int>(BitUtil::Ceil(width * num_values, 8)));
+    ValidateRle(values, width, NULL,
+                1 + static_cast<int>(BitUtil::Ceil(width * num_values, 8)));
   }
 }
 
@@ -352,8 +360,7 @@ TEST(Rle, BitWidthZeroLiteral) {
 // group but flush before finishing.
 TEST(BitRle, Flush) {
   vector<int> values;
-  for (int i = 0; i < 16; ++i)
-    values.push_back(1);
+  for (int i = 0; i < 16; ++i) values.push_back(1);
   values.push_back(0);
   ValidateRle(values, 1, NULL, -1);
   values.push_back(1);
@@ -385,7 +392,9 @@ TEST(BitRle, Random) {
 
     for (int i = 0; i < ngroups; ++i) {
       int group_size = dist(gen);
-      if (group_size > max_group_size) { group_size = 1; }
+      if (group_size > max_group_size) {
+        group_size = 1;
+      }
       for (int i = 0; i < group_size; ++i) {
         values.push_back(parity);
       }
diff --git a/cpp/src/arrow/util/rle-encoding.h b/cpp/src/arrow/util/rle-encoding.h
index 9ec6235144665..e69077807df3a 100644
--- a/cpp/src/arrow/util/rle-encoding.h
+++ b/cpp/src/arrow/util/rle-encoding.h
@@ -21,8 +21,8 @@
 #ifndef ARROW_UTIL_RLE_ENCODING_H
 #define ARROW_UTIL_RLE_ENCODING_H
 
-#include <algorithm>
 #include <math.h>
+#include <algorithm>
 
 #include "arrow/util/bit-stream-utils.h"
 #include "arrow/util/bit-util.h"
@@ -122,7 +122,8 @@ class RleDecoder {
   /// Like GetBatchWithDict but add spacing for null entries
   template <typename T>
   int GetBatchWithDictSpaced(const T* dictionary, T* values, int batch_size,
-      int null_count, const uint8_t* valid_bits, int64_t valid_bits_offset);
+                             int null_count, const uint8_t* valid_bits,
+                             int64_t valid_bits_offset);
 
  protected:
   BitReader bit_reader_;
@@ -289,7 +290,7 @@ inline int RleDecoder::GetBatch(T* values, int batch_size) {
       int repeat_batch =
           std::min(batch_size - values_read, static_cast<int>(repeat_count_));
       std::fill(values + values_read, values + values_read + repeat_batch,
-          static_cast<T>(current_value_));
+                static_cast<T>(current_value_));
       repeat_count_ -= repeat_batch;
       values_read += repeat_batch;
     } else if (literal_count_ > 0) {
@@ -318,7 +319,7 @@ inline int RleDecoder::GetBatchWithDict(const T* dictionary, T* values, int batc
       int repeat_batch =
           std::min(batch_size - values_read, static_cast<int>(repeat_count_));
       std::fill(values + values_read, values + values_read + repeat_batch,
-          dictionary[current_value_]);
+                dictionary[current_value_]);
       repeat_count_ -= repeat_batch;
       values_read += repeat_batch;
     } else if (literal_count_ > 0) {
@@ -345,8 +346,9 @@ inline int RleDecoder::GetBatchWithDict(const T* dictionary, T* values, int batc
 
 template <typename T>
 inline int RleDecoder::GetBatchWithDictSpaced(const T* dictionary, T* values,
-    int batch_size, int null_count, const uint8_t* valid_bits,
-    int64_t valid_bits_offset) {
+                                              int batch_size, int null_count,
+                                              const uint8_t* valid_bits,
+                                              int64_t valid_bits_offset) {
   DCHECK_GE(bit_width_, 0);
   int values_read = 0;
   int remaining_nulls = null_count;
@@ -379,8 +381,8 @@ inline int RleDecoder::GetBatchWithDictSpaced(const T* dictionary, T* values,
         std::fill(values + values_read, values + values_read + repeat_batch, value);
         values_read += repeat_batch;
       } else if (literal_count_ > 0) {
-        int literal_batch = std::min(
-            batch_size - values_read - remaining_nulls, static_cast<int>(literal_count_));
+        int literal_batch = std::min(batch_size - values_read - remaining_nulls,
+                                     static_cast<int>(literal_count_));
 
         // Decode the literals
         constexpr int kBufferSize = 1024;
@@ -434,7 +436,7 @@ bool RleDecoder::NextCounts() {
     repeat_count_ = indicator_value >> 1;
     bool result =
         bit_reader_.GetAligned<T>(static_cast<int>(BitUtil::Ceil(bit_width_, 8)),
-            reinterpret_cast<T*>(&current_value_));
+                                  reinterpret_cast<T*>(&current_value_));
     DCHECK(result);
   }
   return true;
@@ -509,8 +511,8 @@ inline void RleEncoder::FlushRepeatedRun() {
   // The lsb of 0 indicates this is a repeated run
   int32_t indicator_value = repeat_count_ << 1 | 0;
   result &= bit_writer_.PutVlqInt(indicator_value);
-  result &= bit_writer_.PutAligned(
-      current_value_, static_cast<int>(BitUtil::Ceil(bit_width_, 8)));
+  result &= bit_writer_.PutAligned(current_value_,
+                                   static_cast<int>(BitUtil::Ceil(bit_width_, 8)));
   DCHECK(result);
   num_buffered_values_ = 0;
   repeat_count_ = 0;
@@ -552,7 +554,7 @@ inline void RleEncoder::FlushBufferedValues(bool done) {
 inline int RleEncoder::Flush() {
   if (literal_count_ > 0 || repeat_count_ > 0 || num_buffered_values_ > 0) {
     bool all_repeat = literal_count_ == 0 && (repeat_count_ == num_buffered_values_ ||
-                                                 num_buffered_values_ == 0);
+                                              num_buffered_values_ == 0);
     // There is something pending, figure out if it's a repeated or literal run
     if (repeat_count_ > 0 && all_repeat) {
       FlushRepeatedRun();
diff --git a/cpp/src/arrow/util/sse-util.h b/cpp/src/arrow/util/sse-util.h
index 570c4057a7573..a0ec8a2e93911 100644
--- a/cpp/src/arrow/util/sse-util.h
+++ b/cpp/src/arrow/util/sse-util.h
@@ -53,8 +53,8 @@ static const int STRCMP_MODE =
 
 /// Precomputed mask values up to 16 bits.
 static const int SSE_BITMASK[CHARS_PER_128_BIT_REGISTER] = {
-    1 << 0, 1 << 1, 1 << 2, 1 << 3, 1 << 4, 1 << 5, 1 << 6, 1 << 7, 1 << 8, 1 << 9,
-    1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15,
+    1 << 0, 1 << 1, 1 << 2,  1 << 3,  1 << 4,  1 << 5,  1 << 6,  1 << 7,
+    1 << 8, 1 << 9, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15,
 };
 }  // namespace SSEUtil
 
diff --git a/cpp/src/arrow/util/stl.h b/cpp/src/arrow/util/stl.h
index d58689b748896..4b8916f6eaa54 100644
--- a/cpp/src/arrow/util/stl.h
+++ b/cpp/src/arrow/util/stl.h
@@ -40,8 +40,8 @@ inline std::vector<T> DeleteVectorElement(const std::vector<T>& values, size_t i
 }
 
 template <typename T>
-inline std::vector<T> AddVectorElement(
-    const std::vector<T>& values, size_t index, const T& new_element) {
+inline std::vector<T> AddVectorElement(const std::vector<T>& values, size_t index,
+                                       const T& new_element) {
   DCHECK_LE(index, values.size());
   std::vector<T> out;
   out.reserve(values.size() + 1);
diff --git a/cpp/src/arrow/util/string.h b/cpp/src/arrow/util/string.h
index 5d9fdc88ced7e..6e70ddcccefec 100644
--- a/cpp/src/arrow/util/string.h
+++ b/cpp/src/arrow/util/string.h
@@ -46,7 +46,9 @@ static inline Status ParseHexValue(const char* data, uint8_t* out) {
   const char* pos2 = std::lower_bound(kAsciiTable, kAsciiTable + 16, c2);
 
   // Error checking
-  if (*pos1 != c1 || *pos2 != c2) { return Status::Invalid("Encountered non-hex digit"); }
+  if (*pos1 != c1 || *pos2 != c2) {
+    return Status::Invalid("Encountered non-hex digit");
+  }
 
   *out = static_cast<uint8_t>((pos1 - kAsciiTable) << 4 | (pos2 - kAsciiTable));
   return Status::OK();
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 62bfbec21c466..bbbeb55813ccf 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -88,7 +88,9 @@ uint8_t* PlasmaClient::lookup_or_mmap(int fd, int store_fd_val, int64_t map_size
     uint8_t* result = reinterpret_cast<uint8_t*>(
         mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
     // TODO(pcm): Don't fail here, instead return a Status.
-    if (result == MAP_FAILED) { ARROW_LOG(FATAL) << "mmap failed"; }
+    if (result == MAP_FAILED) {
+      ARROW_LOG(FATAL) << "mmap failed";
+    }
     close(fd);
     ClientMmapTableEntry& entry = mmap_table_[store_fd_val];
     entry.pointer = result;
@@ -106,8 +108,8 @@ uint8_t* PlasmaClient::lookup_mmapped_file(int store_fd_val) {
   return entry->second.pointer;
 }
 
-void PlasmaClient::increment_object_count(
-    const ObjectID& object_id, PlasmaObject* object, bool is_sealed) {
+void PlasmaClient::increment_object_count(const ObjectID& object_id, PlasmaObject* object,
+                                          bool is_sealed) {
   // Increment the count of the object to track the fact that it is being used.
   // The corresponding decrement should happen in PlasmaClient::Release.
   auto elem = objects_in_use_.find(object_id);
@@ -142,7 +144,7 @@ void PlasmaClient::increment_object_count(
 }
 
 Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
-    uint8_t* metadata, int64_t metadata_size, uint8_t** data) {
+                            uint8_t* metadata, int64_t metadata_size, uint8_t** data) {
   ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
                    << data_size << " and metadata size " << metadata_size;
   RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
@@ -183,7 +185,7 @@ Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
 }
 
 Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
-    int64_t timeout_ms, ObjectBuffer* object_buffers) {
+                         int64_t timeout_ms, ObjectBuffer* object_buffers) {
   // Fill out the info for the objects that are already in use locally.
   bool all_present = true;
   for (int i = 0; i < num_objects; ++i) {
@@ -213,7 +215,9 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
     }
   }
 
-  if (all_present) { return Status::OK(); }
+  if (all_present) {
+    return Status::OK();
+  }
 
   // If we get here, then the objects aren't all currently in use by this
   // client, so we need to send a request to the plasma store.
@@ -223,8 +227,8 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
   std::vector<ObjectID> received_object_ids(num_objects);
   std::vector<PlasmaObject> object_data(num_objects);
   PlasmaObject* object;
-  RETURN_NOT_OK(ReadGetReply(
-      buffer.data(), received_object_ids.data(), object_data.data(), num_objects));
+  RETURN_NOT_OK(ReadGetReply(buffer.data(), received_object_ids.data(),
+                             object_data.data(), num_objects));
 
   for (int i = 0; i < num_objects; ++i) {
     DCHECK(received_object_ids[i] == object_ids[i]);
@@ -330,7 +334,7 @@ Status PlasmaClient::Release(const ObjectID& object_id) {
   // pending release calls, and there are at least some pending release calls in
   // the release_history list, then release some objects.
   while ((in_use_object_bytes_ > std::min(kL3CacheSizeBytes, store_capacity_ / 100) ||
-             release_history_.size() > config_.release_delay) &&
+          release_history_.size() > config_.release_delay) &&
          release_history_.size() > 0) {
     // Perform a release for the object ID for the first pending release.
     RETURN_NOT_OK(PerformRelease(release_history_.back()));
@@ -364,8 +368,9 @@ static void ComputeBlockHash(const unsigned char* data, int64_t nbytes, uint64_t
   *hash = XXH64_digest(&hash_state);
 }
 
-static inline bool compute_object_hash_parallel(
-    XXH64_state_t* hash_state, const unsigned char* data, int64_t nbytes) {
+static inline bool compute_object_hash_parallel(XXH64_state_t* hash_state,
+                                                const unsigned char* data,
+                                                int64_t nbytes) {
   // Note that this function will likely be faster if the address of data is
   // aligned on a 64-byte boundary.
   const int num_threads = kThreadPoolSize;
@@ -380,16 +385,18 @@ static inline bool compute_object_hash_parallel(
   // Each thread gets a "chunk" of k blocks, except the suffix thread.
 
   for (int i = 0; i < num_threads; i++) {
-    threadpool_[i] = std::thread(ComputeBlockHash,
-        reinterpret_cast<uint8_t*>(data_address) + i * chunk_size, chunk_size,
-        &threadhash[i]);
+    threadpool_[i] = std::thread(
+        ComputeBlockHash, reinterpret_cast<uint8_t*>(data_address) + i * chunk_size,
+        chunk_size, &threadhash[i]);
   }
-  ComputeBlockHash(
-      reinterpret_cast<uint8_t*>(right_address), suffix, &threadhash[num_threads]);
+  ComputeBlockHash(reinterpret_cast<uint8_t*>(right_address), suffix,
+                   &threadhash[num_threads]);
 
   // Join the threads.
   for (auto& t : threadpool_) {
-    if (t.joinable()) { t.join(); }
+    if (t.joinable()) {
+      t.join();
+    }
   }
 
   XXH64_update(hash_state, (unsigned char*)threadhash, sizeof(threadhash));
@@ -400,13 +407,13 @@ static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
   XXH64_state_t hash_state;
   XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
   if (obj_buffer.data_size >= kBytesInMB) {
-    compute_object_hash_parallel(
-        &hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
+    compute_object_hash_parallel(&hash_state, (unsigned char*)obj_buffer.data,
+                                 obj_buffer.data_size);
   } else {
     XXH64_update(&hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
   }
-  XXH64_update(
-      &hash_state, (unsigned char*)obj_buffer.metadata, obj_buffer.metadata_size);
+  XXH64_update(&hash_state, (unsigned char*)obj_buffer.metadata,
+               obj_buffer.metadata_size);
   return XXH64_digest(&hash_state);
 }
 
@@ -483,8 +490,8 @@ Status PlasmaClient::Subscribe(int* fd) {
   return Status::OK();
 }
 
-Status PlasmaClient::GetNotification(
-    int fd, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size) {
+Status PlasmaClient::GetNotification(int fd, ObjectID* object_id, int64_t* data_size,
+                                     int64_t* metadata_size) {
   uint8_t* notification = read_message_async(fd);
   if (notification == NULL) {
     return Status::IOError("Failed to read object notification from Plasma socket");
@@ -504,7 +511,7 @@ Status PlasmaClient::GetNotification(
 }
 
 Status PlasmaClient::Connect(const std::string& store_socket_name,
-    const std::string& manager_socket_name, int release_delay) {
+                             const std::string& manager_socket_name, int release_delay) {
   store_conn_ = connect_ipc_sock_retry(store_socket_name, -1, -1);
   if (manager_socket_name != "") {
     manager_conn_ = connect_ipc_sock_retry(manager_socket_name, -1, -1);
@@ -548,9 +555,7 @@ Status PlasmaClient::Fetch(int num_object_ids, const ObjectID* object_ids) {
   return SendFetchRequest(manager_conn_, object_ids, num_object_ids);
 }
 
-int PlasmaClient::get_manager_fd() {
-  return manager_conn_;
-}
+int PlasmaClient::get_manager_fd() { return manager_conn_; }
 
 Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
   ARROW_CHECK(manager_conn_ >= 0);
@@ -565,7 +570,8 @@ Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
 }
 
 Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_requests,
-    int num_ready_objects, int64_t timeout_ms, int* num_objects_ready) {
+                          int num_ready_objects, int64_t timeout_ms,
+                          int* num_objects_ready) {
   ARROW_CHECK(manager_conn_ >= 0);
   ARROW_CHECK(num_object_requests > 0);
   ARROW_CHECK(num_ready_objects > 0);
@@ -577,7 +583,7 @@ Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_req
   }
 
   RETURN_NOT_OK(SendWaitRequest(manager_conn_, object_requests, num_object_requests,
-      num_ready_objects, timeout_ms));
+                                num_ready_objects, timeout_ms));
   std::vector<uint8_t> buffer;
   RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaWaitReply, &buffer));
   RETURN_NOT_OK(ReadWaitReply(buffer.data(), object_requests, &num_ready_objects));
@@ -588,7 +594,9 @@ Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_req
     int status = object_requests[i].status;
     switch (type) {
       case PLASMA_QUERY_LOCAL:
-        if (status == ObjectStatus_Local) { *num_objects_ready += 1; }
+        if (status == ObjectStatus_Local) {
+          *num_objects_ready += 1;
+        }
         break;
       case PLASMA_QUERY_ANYWHERE:
         if (status == ObjectStatus_Local || status == ObjectStatus_Remote) {
@@ -604,4 +612,4 @@ Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_req
   return Status::OK();
 }
 
-} // namespace plasma
+}  // namespace plasma
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index d9ed9f7c26698..cc05a064511fe 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -91,7 +91,7 @@ class ARROW_EXPORT PlasmaClient {
   ///        and not evicted to avoid too many munmaps.
   /// @return The return status.
   Status Connect(const std::string& store_socket_name,
-      const std::string& manager_socket_name, int release_delay);
+                 const std::string& manager_socket_name, int release_delay);
 
   /// Create an object in the Plasma Store. Any metadata for this object must be
   /// be passed in when the object is created.
@@ -108,7 +108,7 @@ class ARROW_EXPORT PlasmaClient {
   /// @param data The address of the newly created object will be written here.
   /// @return The return status.
   Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
-      int64_t metadata_size, uint8_t** data);
+                int64_t metadata_size, uint8_t** data);
 
   /// Get some objects from the Plasma Store. This function will block until the
   /// objects have all been created and sealed in the Plasma Store or the
@@ -126,7 +126,7 @@ class ARROW_EXPORT PlasmaClient {
   ///        size field is -1, then the object was not retrieved.
   /// @return The return status.
   Status Get(const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms,
-      ObjectBuffer* object_buffers);
+             ObjectBuffer* object_buffers);
 
   /// Tell Plasma that the client no longer needs the object. This should be
   /// called
@@ -203,8 +203,8 @@ class ARROW_EXPORT PlasmaClient {
   /// @param data_size Out parameter, the data size of the object that was sealed.
   /// @param metadata_size Out parameter, the metadata size of the object that was sealed.
   /// @return The return status.
-  Status GetNotification(
-      int fd, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size);
+  Status GetNotification(int fd, ObjectID* object_id, int64_t* data_size,
+                         int64_t* metadata_size);
 
   /// Disconnect from the local plasma instance, including the local store and
   /// manager.
@@ -271,7 +271,7 @@ class ARROW_EXPORT PlasmaClient {
   ///        min_num_ready_objects this means that timeout expired.
   /// @return The return status.
   Status Wait(int64_t num_object_requests, ObjectRequest* object_requests,
-      int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
+              int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
 
   /// Transfer local object to a different plasma manager.
   ///
@@ -315,8 +315,8 @@ class ARROW_EXPORT PlasmaClient {
 
   uint8_t* lookup_mmapped_file(int store_fd_val);
 
-  void increment_object_count(
-      const ObjectID& object_id, PlasmaObject* object, bool is_sealed);
+  void increment_object_count(const ObjectID& object_id, PlasmaObject* object,
+                              bool is_sealed);
 
   /// File descriptor of the Unix domain socket that connects to the store.
   int store_conn_;
@@ -348,6 +348,6 @@ class ARROW_EXPORT PlasmaClient {
   int64_t store_capacity_;
 };
 
-} // namespace plasma
+}  // namespace plasma
 
 #endif  // PLASMA_CLIENT_H
diff --git a/cpp/src/plasma/common.cc b/cpp/src/plasma/common.cc
index a5f530e202ff4..d7a7965078533 100644
--- a/cpp/src/plasma/common.cc
+++ b/cpp/src/plasma/common.cc
@@ -41,13 +41,9 @@ UniqueID UniqueID::from_binary(const std::string& binary) {
   return id;
 }
 
-const uint8_t* UniqueID::data() const {
-  return id_;
-}
+const uint8_t* UniqueID::data() const { return id_; }
 
-uint8_t* UniqueID::mutable_data() {
-  return id_;
-}
+uint8_t* UniqueID::mutable_data() { return id_; }
 
 std::string UniqueID::binary() const {
   return std::string(reinterpret_cast<const char*>(id_), kUniqueIDSize);
@@ -87,4 +83,4 @@ Status plasma_error_status(int plasma_error) {
 ARROW_EXPORT int ObjectStatusLocal = ObjectStatus_Local;
 ARROW_EXPORT int ObjectStatusRemote = ObjectStatus_Remote;
 
-} // namespace plasma
+}  // namespace plasma
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
index 6f2d4dd841b88..2b71da67015cd 100644
--- a/cpp/src/plasma/common.h
+++ b/cpp/src/plasma/common.h
@@ -95,6 +95,6 @@ enum ObjectRequestType {
 extern int ObjectStatusLocal;
 extern int ObjectStatusRemote;
 
-} // namespace plasma
+}  // namespace plasma
 
 #endif  // PLASMA_COMMON_H
diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
index 675424d5c2f1c..f98ced2faf180 100644
--- a/cpp/src/plasma/events.cc
+++ b/cpp/src/plasma/events.cc
@@ -21,8 +21,8 @@
 
 namespace plasma {
 
-void EventLoop::file_event_callback(
-    aeEventLoop* loop, int fd, void* context, int events) {
+void EventLoop::file_event_callback(aeEventLoop* loop, int fd, void* context,
+                                    int events) {
   FileCallback* callback = reinterpret_cast<FileCallback*>(context);
   (*callback)(events);
 }
@@ -34,12 +34,12 @@ int EventLoop::timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* c
 
 constexpr int kInitialEventLoopSize = 1024;
 
-EventLoop::EventLoop() {
-  loop_ = aeCreateEventLoop(kInitialEventLoopSize);
-}
+EventLoop::EventLoop() { loop_ = aeCreateEventLoop(kInitialEventLoopSize); }
 
 bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback) {
-  if (file_callbacks_.find(fd) != file_callbacks_.end()) { return false; }
+  if (file_callbacks_.find(fd) != file_callbacks_.end()) {
+    return false;
+  }
   auto data = std::unique_ptr<FileCallback>(new FileCallback(callback));
   void* context = reinterpret_cast<void*>(data.get());
   // Try to add the file descriptor.
@@ -47,7 +47,9 @@ bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback)
   // If it cannot be added, increase the size of the event loop.
   if (err == AE_ERR && errno == ERANGE) {
     err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2);
-    if (err != AE_OK) { return false; }
+    if (err != AE_OK) {
+      return false;
+    }
     err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
   }
   // In any case, test if there were errors.
@@ -63,9 +65,7 @@ void EventLoop::remove_file_event(int fd) {
   file_callbacks_.erase(fd);
 }
 
-void EventLoop::run() {
-  aeMain(loop_);
-}
+void EventLoop::run() { aeMain(loop_); }
 
 int64_t EventLoop::add_timer(int64_t timeout, const TimerCallback& callback) {
   auto data = std::unique_ptr<TimerCallback>(new TimerCallback(callback));
@@ -82,4 +82,4 @@ int EventLoop::remove_timer(int64_t timer_id) {
   return err;
 }
 
-} // namespace plasma
+}  // namespace plasma
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
index b989b7fac2476..6cb5b73fe94eb 100644
--- a/cpp/src/plasma/events.h
+++ b/cpp/src/plasma/events.h
@@ -98,6 +98,6 @@ class EventLoop {
   std::unordered_map<int64_t, std::unique_ptr<TimerCallback>> timer_callbacks_;
 };
 
-} // namespace plasma
+}  // namespace plasma
 
 #endif  // PLASMA_EVENTS
diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
index ef18e33372998..6c2309f1709d2 100644
--- a/cpp/src/plasma/eviction_policy.cc
+++ b/cpp/src/plasma/eviction_policy.cc
@@ -36,8 +36,8 @@ void LRUCache::remove(const ObjectID& key) {
   item_map_.erase(it);
 }
 
-int64_t LRUCache::choose_objects_to_evict(
-    int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
+int64_t LRUCache::choose_objects_to_evict(int64_t num_bytes_required,
+                                          std::vector<ObjectID>* objects_to_evict) {
   int64_t bytes_evicted = 0;
   auto it = item_list_.end();
   while (bytes_evicted < num_bytes_required && it != item_list_.begin()) {
@@ -51,8 +51,8 @@ int64_t LRUCache::choose_objects_to_evict(
 EvictionPolicy::EvictionPolicy(PlasmaStoreInfo* store_info)
     : memory_used_(0), store_info_(store_info) {}
 
-int64_t EvictionPolicy::choose_objects_to_evict(
-    int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
+int64_t EvictionPolicy::choose_objects_to_evict(int64_t num_bytes_required,
+                                                std::vector<ObjectID>* objects_to_evict) {
   int64_t bytes_evicted =
       cache_.choose_objects_to_evict(num_bytes_required, objects_to_evict);
   /* Update the LRU cache. */
@@ -69,8 +69,8 @@ void EvictionPolicy::object_created(const ObjectID& object_id) {
   cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
 }
 
-bool EvictionPolicy::require_space(
-    int64_t size, std::vector<ObjectID>* objects_to_evict) {
+bool EvictionPolicy::require_space(int64_t size,
+                                   std::vector<ObjectID>* objects_to_evict) {
   /* Check if there is enough space to create the object. */
   int64_t required_space = memory_used_ + size - store_info_->memory_capacity;
   int64_t num_bytes_evicted;
@@ -95,17 +95,17 @@ bool EvictionPolicy::require_space(
   return num_bytes_evicted >= required_space;
 }
 
-void EvictionPolicy::begin_object_access(
-    const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
+void EvictionPolicy::begin_object_access(const ObjectID& object_id,
+                                         std::vector<ObjectID>* objects_to_evict) {
   /* If the object is in the LRU cache, remove it. */
   cache_.remove(object_id);
 }
 
-void EvictionPolicy::end_object_access(
-    const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
+void EvictionPolicy::end_object_access(const ObjectID& object_id,
+                                       std::vector<ObjectID>* objects_to_evict) {
   auto entry = store_info_->objects[object_id].get();
   /* Add the object to the LRU cache.*/
   cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
 }
 
-} // namespace plasma
+}  // namespace plasma
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
index c4f218328312d..dd1c873466ec9 100644
--- a/cpp/src/plasma/eviction_policy.h
+++ b/cpp/src/plasma/eviction_policy.h
@@ -42,8 +42,8 @@ class LRUCache {
 
   void remove(const ObjectID& key);
 
-  int64_t choose_objects_to_evict(
-      int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
+  int64_t choose_objects_to_evict(int64_t num_bytes_required,
+                                  std::vector<ObjectID>* objects_to_evict);
 
  private:
   /// A doubly-linked list containing the items in the cache and
@@ -95,8 +95,8 @@ class EvictionPolicy {
   /// @param objects_to_evict The object IDs that were chosen for eviction will
   ///        be stored into this vector.
   /// @return Void.
-  void begin_object_access(
-      const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
+  void begin_object_access(const ObjectID& object_id,
+                           std::vector<ObjectID>* objects_to_evict);
 
   /// This method will be called whenever an object in the Plasma store that was
   /// being used is no longer being used. When this method is called, the
@@ -107,8 +107,8 @@ class EvictionPolicy {
   /// @param objects_to_evict The object IDs that were chosen for eviction will
   ///        be stored into this vector.
   /// @return Void.
-  void end_object_access(
-      const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
+  void end_object_access(const ObjectID& object_id,
+                         std::vector<ObjectID>* objects_to_evict);
 
   /// Choose some objects to evict from the Plasma store. When this method is
   /// called, the eviction policy will assume that the objects chosen to be
@@ -121,8 +121,8 @@ class EvictionPolicy {
   /// @param objects_to_evict The object IDs that were chosen for eviction will
   ///        be stored into this vector.
   /// @return The total number of bytes of space chosen to be evicted.
-  int64_t choose_objects_to_evict(
-      int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
+  int64_t choose_objects_to_evict(int64_t num_bytes_required,
+                                  std::vector<ObjectID>* objects_to_evict);
 
  private:
   /// The amount of memory (in bytes) currently being used.
@@ -133,6 +133,6 @@ class EvictionPolicy {
   LRUCache cache_;
 };
 
-} // namespace plasma
+}  // namespace plasma
 
 #endif  // PLASMA_EVICTION_POLICY_H
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
index 5875ebb7ae611..e3b6b617fbc06 100644
--- a/cpp/src/plasma/io.cc
+++ b/cpp/src/plasma/io.cc
@@ -38,7 +38,9 @@ Status WriteBytes(int fd, uint8_t* cursor, size_t length) {
      * advance the cursor, and decrease the amount left to write. */
     nbytes = write(fd, cursor + offset, bytesleft);
     if (nbytes < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
+      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
+        continue;
+      }
       return Status::IOError(std::string(strerror(errno)));
     } else if (nbytes == 0) {
       return Status::IOError("Encountered unexpected EOF");
@@ -67,7 +69,9 @@ Status ReadBytes(int fd, uint8_t* cursor, size_t length) {
   while (bytesleft > 0) {
     nbytes = read(fd, cursor + offset, bytesleft);
     if (nbytes < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
+      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
+        continue;
+      }
       return Status::IOError(std::string(strerror(errno)));
     } else if (0 == nbytes) {
       return Status::IOError("Encountered unexpected EOF");
@@ -83,14 +87,16 @@ Status ReadBytes(int fd, uint8_t* cursor, size_t length) {
 Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
   int64_t version;
   RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)),
-      *type = DISCONNECT_CLIENT);
+                     *type = DISCONNECT_CLIENT);
   ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version;
   size_t length;
   RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(type), sizeof(*type)),
-      *type = DISCONNECT_CLIENT);
+                     *type = DISCONNECT_CLIENT);
   RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)),
-      *type = DISCONNECT_CLIENT);
-  if (length > buffer->size()) { buffer->resize(length); }
+                     *type = DISCONNECT_CLIENT);
+  if (length > buffer->size()) {
+    buffer->resize(length);
+  }
   RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer->data(), length), *type = DISCONNECT_CLIENT);
   return Status::OK();
 }
@@ -105,7 +111,7 @@ int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
   /* Tell the system to allow the port to be reused. */
   int on = 1;
   if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&on),
-          sizeof(on)) < 0) {
+                 sizeof(on)) < 0) {
     ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname;
     close(socket_fd);
     return -1;
@@ -134,16 +140,22 @@ int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
   return socket_fd;
 }
 
-int connect_ipc_sock_retry(
-    const std::string& pathname, int num_retries, int64_t timeout) {
+int connect_ipc_sock_retry(const std::string& pathname, int num_retries,
+                           int64_t timeout) {
   /* Pick the default values if the user did not specify. */
-  if (num_retries < 0) { num_retries = NUM_CONNECT_ATTEMPTS; }
-  if (timeout < 0) { timeout = CONNECT_TIMEOUT_MS; }
+  if (num_retries < 0) {
+    num_retries = NUM_CONNECT_ATTEMPTS;
+  }
+  if (timeout < 0) {
+    timeout = CONNECT_TIMEOUT_MS;
+  }
 
   int fd = -1;
   for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
     fd = connect_ipc_sock(pathname);
-    if (fd >= 0) { break; }
+    if (fd >= 0) {
+      break;
+    }
     if (num_attempts == 0) {
       ARROW_LOG(ERROR) << "Connection to socket failed for pathname " << pathname;
     }
@@ -151,7 +163,9 @@ int connect_ipc_sock_retry(
     usleep(static_cast<int>(timeout * 1000));
   }
   /* If we could not connect to the socket, exit. */
-  if (fd == -1) { ARROW_LOG(FATAL) << "Could not connect to socket " << pathname; }
+  if (fd == -1) {
+    ARROW_LOG(FATAL) << "Could not connect to socket " << pathname;
+  }
   return fd;
 }
 
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
index 97c9a16c0c0bd..77a8afea75424 100644
--- a/cpp/src/plasma/malloc.cc
+++ b/cpp/src/plasma/malloc.cc
@@ -69,13 +69,9 @@ std::unordered_map<void*, mmap_record> mmap_records;
 
 constexpr int GRANULARITY_MULTIPLIER = 2;
 
-static void* pointer_advance(void* p, ptrdiff_t n) {
-  return (unsigned char*)p + n;
-}
+static void* pointer_advance(void* p, ptrdiff_t n) { return (unsigned char*)p + n; }
 
-static void* pointer_retreat(void* p, ptrdiff_t n) {
-  return (unsigned char*)p - n;
-}
+static void* pointer_retreat(void* p, ptrdiff_t n) { return (unsigned char*)p - n; }
 
 static ptrdiff_t pointer_distance(void const* pfrom, void const* pto) {
   return (unsigned char const*)pto - (unsigned char const*)pfrom;
@@ -87,8 +83,8 @@ int create_buffer(int64_t size) {
   int fd;
 #ifdef _WIN32
   if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
-          (DWORD)((uint64_t)size >> (CHAR_BIT * sizeof(DWORD))), (DWORD)(uint64_t)size,
-          NULL)) {
+                         (DWORD)((uint64_t)size >> (CHAR_BIT * sizeof(DWORD))),
+                         (DWORD)(uint64_t)size, NULL)) {
     fd = -1;
   }
 #else
@@ -127,7 +123,9 @@ void* fake_mmap(size_t size) {
   int fd = create_buffer(size);
   ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
   void* pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-  if (pointer == MAP_FAILED) { return pointer; }
+  if (pointer == MAP_FAILED) {
+    return pointer;
+  }
 
   /* Increase dlmalloc's allocation granularity directly. */
   mparams.granularity *= GRANULARITY_MULTIPLIER;
@@ -156,7 +154,9 @@ int fake_munmap(void* addr, int64_t size) {
   }
 
   int r = munmap(addr, size);
-  if (r == 0) { close(entry->second.fd); }
+  if (r == 0) {
+    close(entry->second.fd);
+  }
 
   mmap_records.erase(entry);
   return r;
diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc
index bfed5009b6157..87082817f12e9 100644
--- a/cpp/src/plasma/plasma.cc
+++ b/cpp/src/plasma/plasma.cc
@@ -27,7 +27,9 @@
 namespace plasma {
 
 int warn_if_sigpipe(int status, int client_sock) {
-  if (status >= 0) { return 0; }
+  if (status >= 0) {
+    return 0;
+  }
   if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
     ARROW_LOG(WARNING) << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when "
                           "sending a message to client on fd "
@@ -58,11 +60,13 @@ uint8_t* create_object_info_buffer(ObjectInfoT* object_info) {
   return notification;
 }
 
-ObjectTableEntry* get_object_table_entry(
-    PlasmaStoreInfo* store_info, const ObjectID& object_id) {
+ObjectTableEntry* get_object_table_entry(PlasmaStoreInfo* store_info,
+                                         const ObjectID& object_id) {
   auto it = store_info->objects.find(object_id);
-  if (it == store_info->objects.end()) { return NULL; }
+  if (it == store_info->objects.end()) {
+    return NULL;
+  }
   return it->second.get();
 }
 
-} // namespace plasma
+}  // namespace plasma
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
index db8669ff0ddc2..d60e5a8363035 100644
--- a/cpp/src/plasma/plasma.h
+++ b/cpp/src/plasma/plasma.h
@@ -138,8 +138,8 @@ struct PlasmaStoreInfo {
 /// @param object_id The object_id of the entry we are looking for.
 /// @return The entry associated with the object_id or NULL if the object_id
 ///         is not present.
-ObjectTableEntry* get_object_table_entry(
-    PlasmaStoreInfo* store_info, const ObjectID& object_id);
+ObjectTableEntry* get_object_table_entry(PlasmaStoreInfo* store_info,
+                                         const ObjectID& object_id);
 
 /// Print a warning if the status is less than zero. This should be used to check
 /// the success of messages sent to plasma clients. We print a warning instead of
@@ -159,6 +159,6 @@ int warn_if_sigpipe(int status, int client_sock);
 
 uint8_t* create_object_info_buffer(ObjectInfoT* object_info);
 
-} // namespace plasma
+}  // namespace plasma
 
 #endif  // PLASMA_PLASMA_H
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 2998c68b82785..19240bb4b8a1f 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -29,7 +29,7 @@ using flatbuffers::uoffset_t;
 
 flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
 to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
-    int64_t num_objects) {
+              int64_t num_objects) {
   std::vector<flatbuffers::Offset<flatbuffers::String>> results;
   for (int64_t i = 0; i < num_objects; i++) {
     results.push_back(fbb->CreateString(object_ids[i].binary()));
@@ -47,23 +47,23 @@ Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffe
 
 template <typename Message>
 Status PlasmaSend(int sock, int64_t message_type, flatbuffers::FlatBufferBuilder* fbb,
-    const Message& message) {
+                  const Message& message) {
   fbb->Finish(message);
   return WriteMessage(sock, message_type, fbb->GetSize(), fbb->GetBufferPointer());
 }
 
 // Create messages.
 
-Status SendCreateRequest(
-    int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size) {
+Status SendCreateRequest(int sock, ObjectID object_id, int64_t data_size,
+                         int64_t metadata_size) {
   flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaCreateRequest(
-      fbb, fbb.CreateString(object_id.binary()), data_size, metadata_size);
+  auto message = CreatePlasmaCreateRequest(fbb, fbb.CreateString(object_id.binary()),
+                                           data_size, metadata_size);
   return PlasmaSend(sock, MessageType_PlasmaCreateRequest, &fbb, message);
 }
 
-Status ReadCreateRequest(
-    uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size) {
+Status ReadCreateRequest(uint8_t* data, ObjectID* object_id, int64_t* data_size,
+                         int64_t* metadata_size) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
   *data_size = message->data_size();
@@ -72,14 +72,14 @@ Status ReadCreateRequest(
   return Status::OK();
 }
 
-Status SendCreateReply(
-    int sock, ObjectID object_id, PlasmaObject* object, int error_code) {
+Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object,
+                       int error_code) {
   flatbuffers::FlatBufferBuilder fbb;
   PlasmaObjectSpec plasma_object(object->handle.store_fd, object->handle.mmap_size,
-      object->data_offset, object->data_size, object->metadata_offset,
-      object->metadata_size);
-  auto message = CreatePlasmaCreateReply(
-      fbb, fbb.CreateString(object_id.binary()), &plasma_object, (PlasmaError)error_code);
+                                 object->data_offset, object->data_size,
+                                 object->metadata_offset, object->metadata_size);
+  auto message = CreatePlasmaCreateReply(fbb, fbb.CreateString(object_id.binary()),
+                                         &plasma_object, (PlasmaError)error_code);
   return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
 }
 
@@ -117,8 +117,8 @@ Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest
 
 Status SendSealReply(int sock, ObjectID object_id, int error) {
   flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaSealReply(
-      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  auto message = CreatePlasmaSealReply(fbb, fbb.CreateString(object_id.binary()),
+                                       (PlasmaError)error);
   return PlasmaSend(sock, MessageType_PlasmaSealReply, &fbb, message);
 }
 
@@ -146,8 +146,8 @@ Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id) {
 
 Status SendReleaseReply(int sock, ObjectID object_id, int error) {
   flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaReleaseReply(
-      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  auto message = CreatePlasmaReleaseReply(fbb, fbb.CreateString(object_id.binary()),
+                                          (PlasmaError)error);
   return PlasmaSend(sock, MessageType_PlasmaReleaseReply, &fbb, message);
 }
 
@@ -175,8 +175,8 @@ Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id) {
 
 Status SendDeleteReply(int sock, ObjectID object_id, int error) {
   flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaDeleteReply(
-      fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+  auto message = CreatePlasmaDeleteReply(fbb, fbb.CreateString(object_id.binary()),
+                                         (PlasmaError)error);
   return PlasmaSend(sock, MessageType_PlasmaDeleteReply, &fbb, message);
 }
 
@@ -205,12 +205,12 @@ Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objec
   return Status::OK();
 }
 
-Status SendStatusReply(
-    int sock, ObjectID object_ids[], int object_status[], int64_t num_objects) {
+Status SendStatusReply(int sock, ObjectID object_ids[], int object_status[],
+                       int64_t num_objects) {
   flatbuffers::FlatBufferBuilder fbb;
   auto message =
       CreatePlasmaStatusReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
-          fbb.CreateVector(object_status, num_objects));
+                              fbb.CreateVector(object_status, num_objects));
   return PlasmaSend(sock, MessageType_PlasmaStatusReply, &fbb, message);
 }
 
@@ -220,8 +220,8 @@ int64_t ReadStatusReply_num_objects(uint8_t* data) {
   return message->object_ids()->size();
 }
 
-Status ReadStatusReply(
-    uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects) {
+Status ReadStatusReply(uint8_t* data, ObjectID object_ids[], int object_status[],
+                       int64_t num_objects) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
   for (uoffset_t i = 0; i < num_objects; ++i) {
@@ -271,9 +271,7 @@ Status SendConnectRequest(int sock) {
   return PlasmaSend(sock, MessageType_PlasmaConnectRequest, &fbb, message);
 }
 
-Status ReadConnectRequest(uint8_t* data) {
-  return Status::OK();
-}
+Status ReadConnectRequest(uint8_t* data) { return Status::OK(); }
 
 Status SendConnectReply(int sock, int64_t memory_capacity) {
   flatbuffers::FlatBufferBuilder fbb;
@@ -318,16 +316,16 @@ Status ReadEvictReply(uint8_t* data, int64_t& num_bytes) {
 
 // Get messages.
 
-Status SendGetRequest(
-    int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms) {
+Status SendGetRequest(int sock, const ObjectID* object_ids, int64_t num_objects,
+                      int64_t timeout_ms) {
   flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaGetRequest(
-      fbb, to_flatbuffer(&fbb, object_ids, num_objects), timeout_ms);
+  auto message = CreatePlasmaGetRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+                                        timeout_ms);
   return PlasmaSend(sock, MessageType_PlasmaGetRequest, &fbb, message);
 }
 
-Status ReadGetRequest(
-    uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms) {
+Status ReadGetRequest(uint8_t* data, std::vector<ObjectID>& object_ids,
+                      int64_t* timeout_ms) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
   for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
@@ -338,7 +336,8 @@ Status ReadGetRequest(
   return Status::OK();
 }
 
-Status SendGetReply(int sock, ObjectID object_ids[],
+Status SendGetReply(
+    int sock, ObjectID object_ids[],
     std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
     int64_t num_objects) {
   flatbuffers::FlatBufferBuilder fbb;
@@ -347,16 +346,17 @@ Status SendGetReply(int sock, ObjectID object_ids[],
   for (int i = 0; i < num_objects; ++i) {
     const PlasmaObject& object = plasma_objects[object_ids[i]];
     objects.push_back(PlasmaObjectSpec(object.handle.store_fd, object.handle.mmap_size,
-        object.data_offset, object.data_size, object.metadata_offset,
-        object.metadata_size));
+                                       object.data_offset, object.data_size,
+                                       object.metadata_offset, object.metadata_size));
   }
-  auto message = CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
-      fbb.CreateVectorOfStructs(objects.data(), num_objects));
+  auto message =
+      CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+                           fbb.CreateVectorOfStructs(objects.data(), num_objects));
   return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
 }
 
 Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
-    int64_t num_objects) {
+                    int64_t num_objects) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
   for (uoffset_t i = 0; i < num_objects; ++i) {
@@ -395,23 +395,23 @@ Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids) {
 // Wait messages.
 
 Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
-    int num_ready_objects, int64_t timeout_ms) {
+                       int num_ready_objects, int64_t timeout_ms) {
   flatbuffers::FlatBufferBuilder fbb;
 
   std::vector<flatbuffers::Offset<ObjectRequestSpec>> object_request_specs;
   for (int i = 0; i < num_requests; i++) {
-    object_request_specs.push_back(CreateObjectRequestSpec(fbb,
-        fbb.CreateString(object_requests[i].object_id.binary()),
+    object_request_specs.push_back(CreateObjectRequestSpec(
+        fbb, fbb.CreateString(object_requests[i].object_id.binary()),
         object_requests[i].type));
   }
 
-  auto message = CreatePlasmaWaitRequest(
-      fbb, fbb.CreateVector(object_request_specs), num_ready_objects, timeout_ms);
+  auto message = CreatePlasmaWaitRequest(fbb, fbb.CreateVector(object_request_specs),
+                                         num_ready_objects, timeout_ms);
   return PlasmaSend(sock, MessageType_PlasmaWaitRequest, &fbb, message);
 }
 
 Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
-    int64_t* timeout_ms, int* num_ready_objects) {
+                       int64_t* timeout_ms, int* num_ready_objects) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaWaitRequest>(data);
   *num_ready_objects = message->num_ready_objects();
@@ -421,14 +421,14 @@ Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
     ObjectID object_id =
         ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
     ObjectRequest object_request({object_id, message->object_requests()->Get(i)->type(),
-        ObjectStatus_Nonexistent});
+                                  ObjectStatus_Nonexistent});
     object_requests[object_id] = object_request;
   }
   return Status::OK();
 }
 
-Status SendWaitReply(
-    int sock, const ObjectRequestMap& object_requests, int num_ready_objects) {
+Status SendWaitReply(int sock, const ObjectRequestMap& object_requests,
+                     int num_ready_objects) {
   flatbuffers::FlatBufferBuilder fbb;
 
   std::vector<flatbuffers::Offset<ObjectReply>> object_replies;
@@ -443,8 +443,8 @@ Status SendWaitReply(
   return PlasmaSend(sock, MessageType_PlasmaWaitReply, &fbb, message);
 }
 
-Status ReadWaitReply(
-    uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects) {
+Status ReadWaitReply(uint8_t* data, ObjectRequest object_requests[],
+                     int* num_ready_objects) {
   DCHECK(data);
 
   auto message = flatbuffers::GetRoot<PlasmaWaitReply>(data);
@@ -485,16 +485,16 @@ Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int*
   return Status::OK();
 }
 
-Status SendDataReply(
-    int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size) {
+Status SendDataReply(int sock, ObjectID object_id, int64_t object_size,
+                     int64_t metadata_size) {
   flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaDataReply(
-      fbb, fbb.CreateString(object_id.binary()), object_size, metadata_size);
+  auto message = CreatePlasmaDataReply(fbb, fbb.CreateString(object_id.binary()),
+                                       object_size, metadata_size);
   return PlasmaSend(sock, MessageType_PlasmaDataReply, &fbb, message);
 }
 
-Status ReadDataReply(
-    uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size) {
+Status ReadDataReply(uint8_t* data, ObjectID* object_id, int64_t* object_size,
+                     int64_t* metadata_size) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
   *object_id = ObjectID::from_binary(message->object_id()->str());
@@ -503,4 +503,4 @@ Status ReadDataReply(
   return Status::OK();
 }
 
-} // namespace plasma
+}  // namespace plasma
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
index 835c5a0b58978..bab08b6cbd8ef 100644
--- a/cpp/src/plasma/protocol.h
+++ b/cpp/src/plasma/protocol.h
@@ -21,8 +21,8 @@
 #include <vector>
 
 #include "arrow/status.h"
-#include "plasma/plasma_generated.h"
 #include "plasma/plasma.h"
+#include "plasma/plasma_generated.h"
 
 namespace plasma {
 
@@ -34,11 +34,11 @@ Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffe
 
 /* Plasma Create message functions. */
 
-Status SendCreateRequest(
-    int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size);
+Status SendCreateRequest(int sock, ObjectID object_id, int64_t data_size,
+                         int64_t metadata_size);
 
-Status ReadCreateRequest(
-    uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size);
+Status ReadCreateRequest(uint8_t* data, ObjectID* object_id, int64_t* data_size,
+                         int64_t* metadata_size);
 
 Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error);
 
@@ -56,18 +56,19 @@ Status ReadSealReply(uint8_t* data, ObjectID* object_id);
 
 /* Plasma Get message functions. */
 
-Status SendGetRequest(
-    int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms);
+Status SendGetRequest(int sock, const ObjectID* object_ids, int64_t num_objects,
+                      int64_t timeout_ms);
 
-Status ReadGetRequest(
-    uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms);
+Status ReadGetRequest(uint8_t* data, std::vector<ObjectID>& object_ids,
+                      int64_t* timeout_ms);
 
-Status SendGetReply(int sock, ObjectID object_ids[],
+Status SendGetReply(
+    int sock, ObjectID object_ids[],
     std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
     int64_t num_objects);
 
 Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
-    int64_t num_objects);
+                    int64_t num_objects);
 
 /* Plasma Release message functions. */
 
@@ -95,13 +96,13 @@ Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objec
 
 Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects);
 
-Status SendStatusReply(
-    int sock, ObjectID object_ids[], int object_status[], int64_t num_objects);
+Status SendStatusReply(int sock, ObjectID object_ids[], int object_status[],
+                       int64_t num_objects);
 
 int64_t ReadStatusReply_num_objects(uint8_t* data);
 
-Status ReadStatusReply(
-    uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects);
+Status ReadStatusReply(uint8_t* data, ObjectID object_ids[], int object_status[],
+                       int64_t num_objects);
 
 /* Plasma Constains message functions. */
 
@@ -142,16 +143,16 @@ Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids);
 /* Plasma Wait message functions. */
 
 Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
-    int num_ready_objects, int64_t timeout_ms);
+                       int num_ready_objects, int64_t timeout_ms);
 
 Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
-    int64_t* timeout_ms, int* num_ready_objects);
+                       int64_t* timeout_ms, int* num_ready_objects);
 
-Status SendWaitReply(
-    int sock, const ObjectRequestMap& object_requests, int num_ready_objects);
+Status SendWaitReply(int sock, const ObjectRequestMap& object_requests,
+                     int num_ready_objects);
 
-Status ReadWaitReply(
-    uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects);
+Status ReadWaitReply(uint8_t* data, ObjectRequest object_requests[],
+                     int* num_ready_objects);
 
 /* Plasma Subscribe message functions. */
 
@@ -163,12 +164,12 @@ Status SendDataRequest(int sock, ObjectID object_id, const char* address, int po
 
 Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port);
 
-Status SendDataReply(
-    int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size);
+Status SendDataReply(int sock, ObjectID object_id, int64_t object_size,
+                     int64_t metadata_size);
 
-Status ReadDataReply(
-    uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size);
+Status ReadDataReply(uint8_t* data, ObjectID* object_id, int64_t* object_size,
+                     int64_t* metadata_size);
 
-} // namespace plasma
+}  // namespace plasma
 
 #endif /* PLASMA_PROTOCOL */
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 8d4fb106f5367..9ceecdceadc5c 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -49,8 +49,8 @@
 #include <unordered_set>
 #include <vector>
 
-#include "plasma/common_generated.h"
 #include "plasma/common.h"
+#include "plasma/common_generated.h"
 #include "plasma/fling.h"
 #include "plasma/io.h"
 #include "plasma/malloc.h"
@@ -89,8 +89,8 @@ GetRequest::GetRequest(Client* client, const std::vector<ObjectID>& object_ids)
       object_ids(object_ids.begin(), object_ids.end()),
       objects(object_ids.size()),
       num_satisfied(0) {
-  std::unordered_set<ObjectID, UniqueIDHasher> unique_ids(
-      object_ids.begin(), object_ids.end());
+  std::unordered_set<ObjectID, UniqueIDHasher> unique_ids(object_ids.begin(),
+                                                          object_ids.end());
   num_objects_to_wait_for = unique_ids.size();
 }
 
@@ -118,7 +118,9 @@ PlasmaStore::~PlasmaStore() {
 // object's list of clients, otherwise do nothing.
 void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client* client) {
   // Check if this client is already using the object.
-  if (entry->clients.find(client) != entry->clients.end()) { return; }
+  if (entry->clients.find(client) != entry->clients.end()) {
+    return;
+  }
   // If there are no other clients using this object, notify the eviction policy
   // that the object is being used.
   if (entry->clients.size() == 0) {
@@ -133,7 +135,8 @@ void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client*
 
 // Create a new object buffer in the hash table.
 int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
-    int64_t metadata_size, Client* client, PlasmaObject* result) {
+                               int64_t metadata_size, Client* client,
+                               PlasmaObject* result) {
   ARROW_LOG(DEBUG) << "creating object " << object_id.hex();
   if (store_info_.objects.count(object_id) != 0) {
     // There is already an object with the same ID in the Plasma Store, so
@@ -160,7 +163,9 @@ int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
       delete_objects(objects_to_evict);
       // Return an error to the client if not enough space could be freed to
       // create the object.
-      if (!success) { return PlasmaError_OutOfMemory; }
+      if (!success) {
+        return PlasmaError_OutOfMemory;
+      }
     }
   } while (pointer == NULL);
   int fd;
@@ -212,7 +217,7 @@ void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
 void PlasmaStore::return_from_get(GetRequest* get_req) {
   // Send the get reply to the client.
   Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], get_req->objects,
-      get_req->object_ids.size());
+                          get_req->object_ids.size());
   warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd);
   // If we successfully sent the get reply message to the client, then also send
   // the file descriptors.
@@ -249,10 +254,14 @@ void PlasmaStore::return_from_get(GetRequest* get_req) {
     auto& get_requests = object_get_requests_[object_id];
     // Erase get_req from the vector.
     auto it = std::find(get_requests.begin(), get_requests.end(), get_req);
-    if (it != get_requests.end()) { get_requests.erase(it); }
+    if (it != get_requests.end()) {
+      get_requests.erase(it);
+    }
   }
   // Remove the get request.
-  if (get_req->timer != -1) { ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK); }
+  if (get_req->timer != -1) {
+    ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK);
+  }
   delete get_req;
 }
 
@@ -287,8 +296,9 @@ void PlasmaStore::update_object_get_requests(const ObjectID& object_id) {
   object_get_requests_.erase(object_id);
 }
 
-void PlasmaStore::process_get_request(
-    Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms) {
+void PlasmaStore::process_get_request(Client* client,
+                                      const std::vector<ObjectID>& object_ids,
+                                      int64_t timeout_ms) {
   // Create a get request for this object.
   GetRequest* get_req = new GetRequest(client, object_ids);
 
@@ -327,8 +337,8 @@ void PlasmaStore::process_get_request(
   }
 }
 
-int PlasmaStore::remove_client_from_object_clients(
-    ObjectTableEntry* entry, Client* client) {
+int PlasmaStore::remove_client_from_object_clients(ObjectTableEntry* entry,
+                                                   Client* client) {
   auto it = entry->clients.find(client);
   if (it != entry->clients.end()) {
     entry->clients.erase(it);
@@ -408,7 +418,9 @@ void PlasmaStore::connect_client(int listener_sock) {
   // TODO(pcm): Check return value.
   loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) {
     Status s = process_message(client);
-    if (!s.ok()) { ARROW_LOG(FATAL) << "Failed to process file event: " << s; }
+    if (!s.ok()) {
+      ARROW_LOG(FATAL) << "Failed to process file event: " << s;
+    }
   });
   ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
 }
@@ -466,8 +478,9 @@ void PlasmaStore::send_notifications(int client_fd) {
       // at the end of the method.
       // TODO(pcm): Introduce status codes and check in case the file descriptor
       // is added twice.
-      loop_->add_file_event(client_fd, kEventLoopWrite,
-          [this, client_fd](int events) { send_notifications(client_fd); });
+      loop_->add_file_event(client_fd, kEventLoopWrite, [this, client_fd](int events) {
+        send_notifications(client_fd);
+      });
       break;
     } else {
       ARROW_LOG(WARNING) << "Failed to send notification to client on fd " << client_fd;
@@ -482,7 +495,8 @@ void PlasmaStore::send_notifications(int client_fd) {
     delete[] notification;
   }
   // Remove the sent notifications from the array.
-  it->second.object_notifications.erase(it->second.object_notifications.begin(),
+  it->second.object_notifications.erase(
+      it->second.object_notifications.begin(),
       it->second.object_notifications.begin() + num_processed);
 
   // Stop sending notifications if the pipe was broken.
@@ -492,7 +506,9 @@ void PlasmaStore::send_notifications(int client_fd) {
   }
 
   // If we have sent all notifications, remove the fd from the event loop.
-  if (it->second.object_notifications.empty()) { loop_->remove_file_event(client_fd); }
+  if (it->second.object_notifications.empty()) {
+    loop_->remove_file_event(client_fd);
+  }
 }
 
 void PlasmaStore::push_notification(ObjectInfoT* object_info) {
@@ -550,8 +566,8 @@ Status PlasmaStore::process_message(Client* client) {
       RETURN_NOT_OK(ReadCreateRequest(input, &object_id, &data_size, &metadata_size));
       int error_code =
           create_object(object_id, data_size, metadata_size, client, &object);
-      HANDLE_SIGPIPE(
-          SendCreateReply(client->fd, object_id, &object, error_code), client->fd);
+      HANDLE_SIGPIPE(SendCreateReply(client->fd, object_id, &object, error_code),
+                     client->fd);
       if (error_code == PlasmaError_OK) {
         warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
       }
@@ -593,8 +609,8 @@ Status PlasmaStore::process_message(Client* client) {
       subscribe_to_updates(client);
       break;
     case MessageType_PlasmaConnectRequest: {
-      HANDLE_SIGPIPE(
-          SendConnectReply(client->fd, store_info_.memory_capacity), client->fd);
+      HANDLE_SIGPIPE(SendConnectReply(client->fd, store_info_.memory_capacity),
+                     client->fd);
     } break;
     case DISCONNECT_CLIENT:
       ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd;
@@ -609,7 +625,9 @@ Status PlasmaStore::process_message(Client* client) {
 
 // Report "success" to valgrind.
 void signal_handler(int signal) {
-  if (signal == SIGTERM) { exit(0); }
+  if (signal == SIGTERM) {
+    exit(0);
+  }
 }
 
 void start_server(char* socket_name, int64_t system_memory) {
@@ -623,11 +641,11 @@ void start_server(char* socket_name, int64_t system_memory) {
   ARROW_CHECK(socket >= 0);
   // TODO(pcm): Check return value.
   loop.add_file_event(socket, kEventLoopRead,
-      [&store, socket](int events) { store.connect_client(socket); });
+                      [&store, socket](int events) { store.connect_client(socket); });
   loop.run();
 }
 
-} // namespace plasma
+}  // namespace plasma
 
 int main(int argc, char* argv[]) {
   signal(SIGTERM, plasma::signal_handler);
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index 27c3813da8e7a..fec25c133ce17 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -66,7 +66,7 @@ class PlasmaStore {
   ///    cannot create the object. In this case, the client should not call
   ///    plasma_release.
   int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
-      Client* client, PlasmaObject* result);
+                    Client* client, PlasmaObject* result);
 
   /// Delete objects that have been created in the hash table. This should only
   /// be called on objects that are returned by the eviction policy to evict.
@@ -87,8 +87,8 @@ class PlasmaStore {
   /// @param object_ids Object IDs of the objects to be gotten.
   /// @param timeout_ms The timeout for the get request in milliseconds.
   /// @return Void.
-  void process_get_request(
-      Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms);
+  void process_get_request(Client* client, const std::vector<ObjectID>& object_ids,
+                           int64_t timeout_ms);
 
   /// Seal an object. The object is now immutable and can be accessed with get.
   ///
@@ -168,6 +168,6 @@ class PlasmaStore {
   std::unordered_map<int, NotificationQueue> pending_notifications_;
 };
 
-} // namespace plasma
+}  // namespace plasma
 
 #endif  // PLASMA_STORE_H
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index 6dc558e77078b..02b3832145186 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -127,7 +127,7 @@ TEST_F(TestPlasmaStore, MultipleGetTest) {
   ASSERT_EQ(object_buffer[1].data[0], 2);
 }
 
-} // namespace plasma
+}  // namespace plasma
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
index 13938cd6fb042..aca47d3d6f911 100644
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -167,11 +167,11 @@ TEST(PlasmaSerialization, GetReply) {
   ASSERT_EQ(object_ids[0], object_ids_return[0]);
   ASSERT_EQ(object_ids[1], object_ids_return[1]);
   ASSERT_EQ(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
-                sizeof(PlasmaObject)),
-      0);
+                   sizeof(PlasmaObject)),
+            0);
   ASSERT_EQ(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
-                sizeof(PlasmaObject)),
-      0);
+                   sizeof(PlasmaObject)),
+            0);
   close(fd);
 }
 
@@ -303,15 +303,15 @@ TEST(PlasmaSerialization, WaitRequest) {
   const int num_ready_objects_in = 1;
   int64_t timeout_ms = 1000;
 
-  ARROW_CHECK_OK(SendWaitRequest(
-      fd, &object_requests_in[0], num_objects_in, num_ready_objects_in, timeout_ms));
+  ARROW_CHECK_OK(SendWaitRequest(fd, &object_requests_in[0], num_objects_in,
+                                 num_ready_objects_in, timeout_ms));
   /* Read message back. */
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitRequest);
   int num_ready_objects_out;
   int64_t timeout_ms_read;
   ObjectRequestMap object_requests_out;
-  ARROW_CHECK_OK(ReadWaitRequest(
-      data.data(), object_requests_out, &timeout_ms_read, &num_ready_objects_out));
+  ARROW_CHECK_OK(ReadWaitRequest(data.data(), object_requests_out, &timeout_ms_read,
+                                 &num_ready_objects_out));
   ASSERT_EQ(num_objects_in, object_requests_out.size());
   ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
   for (int i = 0; i < num_objects_in; i++) {
@@ -389,4 +389,4 @@ TEST(PlasmaSerialization, DataReply) {
   ASSERT_EQ(metadata_size1, metadata_size2);
 }
 
-} // namespace plasma
+}  // namespace plasma

From 08cec90975a6d4d9e86acd3284675a055d61fd27 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 25 Jul 2017 10:38:46 -0400
Subject: [PATCH 0852/1644] ARROW-1252: [Website] Updates for 0.5.0 and short
 blog post summarizing the release

Also updated the CHANGELOG.md

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #885 from wesm/ARROW-1252 and squashes the following commits:

e603f388 [Wes McKinney] Fix up markdown formatting of underscores
797215b2 [Wes McKinney] Release announcement blog post
3babc7b4 [Wes McKinney] Add release page
3fd41e11 [Wes McKinney] First cut revising install page
b8416ee5 [Wes McKinney] Add changelog to CHANGELOG.md
3f9dec05 [Wes McKinney] Start on 0.5.0 website updates
---
 CHANGELOG.md                            | 274 ++++++++++++++++++------
 dev/make_changelog.py                   |   1 +
 site/_posts/2017-07-24-0.5.0-release.md | 114 ++++++++++
 site/_release/0.5.0.md                  | 203 ++++++++++++++++++
 site/_release/index.md                  |   2 +
 site/index.html                         |  16 +-
 site/install.md                         |  36 ++--
 7 files changed, 553 insertions(+), 93 deletions(-)
 create mode 100644 site/_posts/2017-07-24-0.5.0-release.md
 create mode 100644 site/_release/0.5.0.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 55b02e0f9a1a8..0cc6c6ee62a8a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,13 +12,151 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
+# Apache Arrow 0.5.0 (23 July 2017)
+
+## Bug
+
+* ARROW-1074 - from_pandas doesnt convert ndarray to list
+* ARROW-1079 - [Python] Empty "private" directories should be ignored by Parquet interface
+* ARROW-1081 - C++: arrow::test::TestBase::MakePrimitive doesn't fill null_bitmap
+* ARROW-1096 - [C++] Memory mapping file over 4GB fails on Windows
+* ARROW-1097 - Reading tensor needs file to be opened in writeable mode
+* ARROW-1098 - Document Error?
+* ARROW-1101 - UnionListWriter is not implementing all methods on interface ScalarWriter
+* ARROW-1103 - [Python] Utilize pandas metadata from common `_metadata` Parquet file if it exists
+* ARROW-1107 - [JAVA] NullableMapVector getField() should return nullable type
+* ARROW-1108 - Check if ArrowBuf is empty buffer in getActualConsumedMemory() and getPossibleConsumedMemory()
+* ARROW-1109 - [JAVA] transferOwnership fails when readerIndex is not 0
+* ARROW-1110 - [JAVA] make union vector naming consistent
+* ARROW-1111 - [JAVA] Make aligning buffers optional, and allow -1 for unknown null count
+* ARROW-1112 - [JAVA] Set lastSet for VarLength and List vectors when loading
+* ARROW-1113 - [C++] gflags EP build gets triggered (as a no-op) on subsequent calls to make or ninja build
+* ARROW-1115 - [C++] Use absolute path for ccache
+* ARROW-1117 - [Docs] Minor issues in GLib README
+* ARROW-1124 - [Python] pyarrow needs to depend on numpy>=1.10 (not 1.9)
+* ARROW-1125 - Python: `Table.from_pandas` doesn't work anymore on partial schemas
+* ARROW-1128 - [Docs] command to build a wheel is not properly rendered
+* ARROW-1129 - [C++] Fix Linux toolchain build regression from ARROW-742
+* ARROW-1131 - Python: Parquet unit tests are always skipped
+* ARROW-1132 - [Python] Unable to write pandas DataFrame w/MultiIndex containing duplicate values to parquet
+* ARROW-1136 - [C++/Python] Segfault on empty stream
+* ARROW-1138 - Travis: Use OpenJDK7 instead of OracleJDK7
+* ARROW-1139 - [C++] dlmalloc doesn't allow arrow to be built with clang 4 or gcc 7.1.1
+* ARROW-1141 - on import get libjemalloc.so.2: cannot allocate memory in static TLS block
+* ARROW-1143 - C++: Fix comparison of NullArray
+* ARROW-1144 - [C++] Remove unused variable
+* ARROW-1150 - [C++] AdaptiveIntBuilder compiler warning on MSVC
+* ARROW-1152 - [Cython] `read_tensor` should work with a readable file
+* ARROW-1155 - segmentation fault when run pa.Int16Value()
+* ARROW-1157 - C++/Python: Decimal templates are not correctly exported on OSX
+* ARROW-1159 - [C++] Static data members cannot be accessed from inline functions in Arrow headers by thirdparty users
+* ARROW-1162 - Transfer Between Empty Lists Should Not Invoke Callback
+* ARROW-1166 - Errors in Struct type's example and missing reference in Layout.md
+* ARROW-1167 - [Python] Create chunked BinaryArray in `Table.from_pandas` when a column's data exceeds 2GB
+* ARROW-1168 - [Python] pandas metadata may contain "mixed" data types
+* ARROW-1169 - C++: jemalloc externalproject doesn't build with CMake's ninja generator
+* ARROW-1170 - C++: `ARROW_JEMALLOC=OFF` breaks linking on unittest
+* ARROW-1174 - [GLib] Investigate root cause of ListArray glib test failure
+* ARROW-1177 - [C++] Detect int32 overflow in ListBuilder::Append
+* ARROW-1179 - C++: Add missing virtual destructors
+* ARROW-1180 - [GLib] `garrow_tensor_get_dimension_name()` returns invalid address
+* ARROW-1181 - [Python] Parquet test fail if not enabled
+* ARROW-1182 - C++: Specify `BUILD_BYPRODUCTS` for zlib and zstd
+* ARROW-1186 - [C++] Enable option to build arrow with minimal dependencies needed to build Parquet library
+* ARROW-1188 - Segfault when trying to serialize a DataFrame with Null-only Categorical Column
+* ARROW-1190 - VectorLoader corrupts vectors with duplicate names
+* ARROW-1191 - [JAVA] Implement getField() method for the complex readers
+* ARROW-1194 - Getting record batch size with `pa.get_record_batch_size` returns a size that is too small for pandas DataFrame.
+* ARROW-1197 - [GLib] `record_batch.hpp` Inclusion is missing
+* ARROW-1200 - [C++] DictionaryBuilder should use signed integers for indices
+* ARROW-1201 - [Python] Incomplete Python types cause a core dump when repr-ing
+* ARROW-1203 - [C++] Disallow BinaryBuilder to append byte strings larger than the maximum value of `int32_t`
+* ARROW-1205 - C++: Reference to type objects in ArrayLoader may cause segmentation faults.
+* ARROW-1206 - [C++] Enable MSVC builds to work with some compression library support disabled
+* ARROW-1208 - [C++] Toolchain build with ZSTD library from conda-forge failure
+* ARROW-1215 - [Python] Class methods in API reference
+* ARROW-1216 - Numpy arrays cannot be created from Arrow Buffers on Python 2
+* ARROW-1218 - Arrow doesn't compile if all compression libraries are deactivated
+* ARROW-1222 - [Python] pyarrow.array returns NullArray for array of unsupported Python objects
+* ARROW-1223 - [GLib] Fix function name that returns wrapped object
+* ARROW-1235 - [C++] macOS linker failure with operator<< and std::ostream
+* ARROW-1236 - Library paths in exported pkg-config file are incorrect
+* ARROW-601 - Some logical types not supported when loading Parquet
+* ARROW-784 - Cleaning up thirdparty toolchain support in Arrow on Windows
+* ARROW-992 - [Python] In place development builds do not have a `__version__`
+
+## Improvement
+
+* ARROW-1041 - [Python] Support `read_pandas` on a directory of Parquet files
+* ARROW-1100 - [Python] Add "mode" property to NativeFile instances
+* ARROW-1102 - Make MessageSerializer.serializeMessage() public
+* ARROW-1120 - [Python] Write support for int96
+* ARROW-1137 - Python: Ensure Pandas roundtrip of all-None column
+* ARROW-1148 - [C++] Raise minimum CMake version to 3.2
+* ARROW-1151 - [C++] Add gcc branch prediction to status check macro
+* ARROW-1160 - C++: Implement DictionaryBuilder
+* ARROW-1165 - [C++] Refactor PythonDecimalToArrowDecimal to not use templates
+* ARROW-1185 - [C++] Clean up arrow::Status implementation, add `warn_unused_result` attribute for clang
+* ARROW-1187 - Serialize a DataFrame with None column
+* ARROW-1193 - [C++] Support pkg-config for `arrow_python.so`
+* ARROW-1196 - [C++] Appveyor separate jobs for Debug/Release builds from sources; Build with conda toolchain; Build with NMake Makefiles Generator
+* ARROW-1199 - [C++] Introduce mutable POD struct for generic array data
+* ARROW-1202 - Remove semicolons from status macros
+* ARROW-1217 - [GLib] Add GInputStream based arrow::io::RandomAccessFile
+* ARROW-1220 - [C++] Standartize usage of `*_HOME` cmake script variables for 3rd party libs
+* ARROW-1221 - [C++] Pin clang-format version
+* ARROW-1229 - [GLib] Follow Reader API change (get -> read)
+* ARROW-742 - Handling exceptions during execution of `std::wstring_convert`
+* ARROW-834 - [Python] Support creating Arrow arrays from Python iterables
+* ARROW-915 - Struct Array reads limited support
+* ARROW-935 - [Java] Build Javadoc in Travis CI
+* ARROW-960 - [Python] Add source build guide for macOS + Homebrew
+* ARROW-962 - [Python] Add schema attribute to FileReader
+* ARROW-966 - [Python] `pyarrow.list_` should also accept Field instance
+* ARROW-978 - [Python] Use sphinx-bootstrap-theme for Sphinx documentation
+
+## New Feature
+
+* ARROW-1048 - Allow user `LD_LIBRARY_PATH` to be used with source release script
+* ARROW-1073 - C++: Adapative integer builder
+* ARROW-1095 - [Website] Add Arrow icon asset
+* ARROW-111 - [C++] Add static analyzer to tool chain to verify checking of Status returns
+* ARROW-1122 - [Website] Guest blog post on Arrow + ODBC from turbodbc
+* ARROW-1123 - C++: Make jemalloc the default allocator
+* ARROW-1135 - Upgrade Travis CI clang builds to use LLVM 4.0
+* ARROW-1142 - [C++] Move over compression library toolchain from parquet-cpp
+* ARROW-1145 - [GLib] Add `get_values()`
+* ARROW-1154 - [C++] Migrate more computational utility code from parquet-cpp
+* ARROW-1183 - [Python] Implement time type conversions in `to_pandas`
+* ARROW-1198 - Python: Add public C++ API to unwrap PyArrow object
+* ARROW-1212 - [GLib] Add `garrow_binary_array_get_offsets_buffer()`
+* ARROW-1214 - [Python] Add classes / functions to enable stream message components to be handled outside of the stream reader class
+* ARROW-1227 - [GLib] Support GOutputStream
+* ARROW-460 - [C++] Implement JSON round trip for DictionaryArray
+* ARROW-462 - [C++] Implement in-memory conversions between non-nested primitive types and DictionaryArray equivalent
+* ARROW-575 - Python: Auto-detect nested lists and nested numpy arrays in Pandas
+* ARROW-597 - [Python] Add convenience function to yield DataFrame from any object that a StreamReader or FileReader can read from
+* ARROW-599 - [C++] Add LZ4 codec to 3rd-party toolchain
+* ARROW-600 - [C++] Add ZSTD codec to 3rd-party toolchain
+* ARROW-692 - Java<->C++ Integration tests for dictionary-encoded vectors
+* ARROW-693 - [Java] Add JSON support for dictionary vectors
+
+## Task
+
+* ARROW-1052 - Arrow 0.5.0 release
+
+## Test
+
+* ARROW-1228 - [GLib] Test file name should be the same name as target class
+* ARROW-1233 - [C++] Validate cmake script resolving of 3rd party linked libs from correct location in toolchain build
+
 # Apache Arrow 0.4.1 (9 June 2017)
 
 ## Bug
 
-* ARROW-1039 - Python: pyarrow.Filesystem.read_parquet causing error if nthreads>1
+* ARROW-1039 - Python: `pyarrow.Filesystem.read_parquet` causing error if nthreads>1
 * ARROW-1050 - [C++] Export arrow::ValidateArray
-* ARROW-1051 - [Python] If pyarrow.parquet fails to import due to a shared library ABI conflict, the test_parquet.py tests silently do not run
+* ARROW-1051 - [Python] If pyarrow.parquet fails to import due to a shared library ABI conflict, the `test_parquet.py` tests silently do not run
 * ARROW-1056 - [Python] Parquet+HDFS test failure due to writing pandas index
 * ARROW-1057 - Fix cmake warning and msvc debug asserts
 * ARROW-1062 - [GLib] Examples use old API
@@ -27,8 +165,8 @@
 * ARROW-1075 - [GLib] Build error on macOS
 * ARROW-1085 - [java] Follow up on template cleanup. Missing method for IntervalYear
 * ARROW-1086 - [Python] pyarrow 0.4.0 on pypi is missing pxd files
-* ARROW-1088 - [Python] test_unicode_filename test fails when unicode filenames aren't supported by system
-* ARROW-1090 - [Python] build_ext usability
+* ARROW-1088 - [Python] `test_unicode_filename` test fails when unicode filenames aren't supported by system
+* ARROW-1090 - [Python] `build_ext` usability
 * ARROW-1091 - Decimal scale and precision are flipped
 * ARROW-1092 - More Decimal and scale flipped follow-up
 * ARROW-1094 - [C++] Incomplete buffer reads in arrow::io::ReadableFile should exactly truncate returned buffer
@@ -63,9 +201,9 @@
 
 * ARROW-1003 - [C++] Hdfs and java dlls fail to load when built for Windows with MSVC
 * ARROW-1004 - ArrowInvalid: Invalid: Python object of type float is not None and is not a string, bool, or date object
-* ARROW-1017 - Python: Table.to_pandas leaks memory
+* ARROW-1017 - Python: `Table.to_pandas` leaks memory
 * ARROW-1023 - Python: Fix bundling of arrow-cpp for macOS
-* ARROW-1033 - [Python] pytest discovers scripts/test_leak.py
+* ARROW-1033 - [Python] pytest discovers `scripts/test_leak.py`
 * ARROW-1046 - [Python] Conform DataFrame metadata to pandas spec
 * ARROW-1053 - [Python] Memory leak with RecordBatchFileReader
 * ARROW-1054 - [Python] Test suite fails on pandas 0.19.2
@@ -74,16 +212,16 @@
 * ARROW-813 - [Python] setup.py sdist must also bundle dependent cmake modules
 * ARROW-824 - Date and Time Vectors should reflect timezone-less semantics
 * ARROW-856 - CmakeError by Unknown compiler.
-* ARROW-881 - [Python] Reconstruct Pandas DataFrame indexes using custom_metadata
+* ARROW-881 - [Python] Reconstruct Pandas DataFrame indexes using `custom_metadata`
 * ARROW-909 - libjemalloc.so.2: cannot open shared object file:
 * ARROW-939 - Fix division by zero for zero-dimensional Tensors
 * ARROW-940 - [JS] Generate multiple sets of artifacts
 * ARROW-944 - Python: Compat broken for pandas==0.18.1
 * ARROW-948 - [GLib] Update C++ header file list
 * ARROW-952 - Compilation error on macOS with clang-802.0.42
-* ARROW-958 - [Python] Conda build guide still needs ARROW_HOME, PARQUET_HOME
-* ARROW-979 - [Python] Fix setuptools_scm version when release tag is not in the master timeline
-* ARROW-991 - [Python] PyArray_SimpleNew should not be used with NPY_DATETIME
+* ARROW-958 - [Python] Conda build guide still needs `ARROW_HOME`, `PARQUET_HOME`
+* ARROW-979 - [Python] Fix `setuptools_scm` version when release tag is not in the master timeline
+* ARROW-991 - [Python] `PyArray_SimpleNew` should not be used with `NPY_DATETIME`
 * ARROW-995 - [Website] 0.3 release announce has a typo in reference
 * ARROW-998 - [Doc] File format documents incorrect schema location
 
@@ -138,9 +276,9 @@
 * ARROW-1044 - [GLib] Support Feather
 * ARROW-29 - C++: Add re2 as optional 3rd-party toolchain dependency
 * ARROW-446 - [Python] Document NativeFile interfaces, HDFS client in Sphinx
-* ARROW-482 - [Java] Provide API access to "custom_metadata" Field attribute in IPC setting
+* ARROW-482 - [Java] Provide API access to `custom_metadata` Field attribute in IPC setting
 * ARROW-596 - [Python] Add convenience function to convert pandas.DataFrame to pyarrow.Buffer containing a file or stream representation
-* ARROW-714 - [C++] Add import_pyarrow C API in the style of NumPy for thirdparty C++ users
+* ARROW-714 - [C++] Add `import_pyarrow` C API in the style of NumPy for thirdparty C++ users
 * ARROW-819 - [Python] Define public Cython API
 * ARROW-872 - [JS] Read streaming format
 * ARROW-873 - [JS] Implement fixed width list type
@@ -165,8 +303,8 @@
 * ARROW-208 - Add checkstyle policy to java project
 * ARROW-347 - Add method to pass CallBack when creating a transfer pair
 * ARROW-413 - DATE type is not specified clearly
-* ARROW-431 - [Python] Review GIL release and acquisition in to_pandas conversion
-* ARROW-443 - [Python] Support for converting from strided pandas data in Table.from_pandas
+* ARROW-431 - [Python] Review GIL release and acquisition in `to_pandas` conversion
+* ARROW-443 - [Python] Support for converting from strided pandas data in `Table.from_pandas`
 * ARROW-451 - [C++] Override DataType::Equals for other types with additional metadata
 * ARROW-454 - pojo.Field doesn't implement hashCode()
 * ARROW-526 - [Format] Update IPC.md to account for File format changes and Streaming format
@@ -178,8 +316,8 @@
 * ARROW-604 - Python: boxed Field instances are missing the reference to DataType
 * ARROW-613 - [JS] Implement random-access file format
 * ARROW-617 - Time type is not specified clearly
-* ARROW-619 - Python: Fix typos in setup.py args and LD_LIBRARY_PATH
-* ARROW-623 - segfault with __repr__ of empty Field
+* ARROW-619 - Python: Fix typos in setup.py args and `LD_LIBRARY_PATH`
+* ARROW-623 - segfault with `__repr__` of empty Field
 * ARROW-624 - [C++] Restore MakePrimitiveArray function
 * ARROW-627 - [C++] Compatibility macros for exported extern template class declarations
 * ARROW-628 - [Python] Install nomkl metapackage when building parquet-cpp for faster Travis builds
@@ -201,7 +339,7 @@
 * ARROW-686 - [C++] Account for time metadata changes, add time32 and time64 types
 * ARROW-689 - [GLib] Install header files and documents to wrong directories
 * ARROW-691 - [Java] Encode dictionary Int type in message format
-* ARROW-697 - [Java] Raise appropriate exceptions when encountering large (> INT32_MAX) record batches
+* ARROW-697 - [Java] Raise appropriate exceptions when encountering large (> `INT32_MAX`) record batches
 * ARROW-699 - [C++] Arrow dynamic libraries are missed on run of unit tests on Windows
 * ARROW-702 - Fix BitVector.copyFromSafe to reAllocate instead of returning false
 * ARROW-703 - Fix issue where setValueCount(0) doesn’t work in the case that we’ve shipped vectors across the wire
@@ -211,14 +349,14 @@
 * ARROW-715 - Python: Explicit pandas import makes it a hard requirement
 * ARROW-716 - error building arrow/python
 * ARROW-720 - [java] arrow should not have a dependency on slf4j bridges in compile
-* ARROW-723 - Arrow freezes on write if chunk_size=0
+* ARROW-723 - Arrow freezes on write if `chunk_size=0`
 * ARROW-726 - [C++] PyBuffer dtor may segfault if constructor passed an object not exporting buffer protocol
 * ARROW-732 - Schema comparison bugs in struct and union types
 * ARROW-736 - [Python] Mixed-type object DataFrame columns should not silently coerce to an Arrow type by default
 * ARROW-738 - [Python] Fix manylinux1 packaging
 * ARROW-739 - Parallel build fails non-deterministically.
 * ARROW-740 - FileReader fails for large objects
-* ARROW-747 - [C++] Fix spurious warning caused by passing dl to add_dependencies
+* ARROW-747 - [C++] Fix spurious warning caused by passing dl to `add_dependencies`
 * ARROW-749 - [Python] Delete incomplete binary files when writing fails
 * ARROW-753 - [Python] Unit tests in arrow/python fail to link on some OS X platforms
 * ARROW-756 - [C++] Do not pass -fPIC when compiling with MSVC
@@ -238,13 +376,13 @@
 * ARROW-809 - C++: Writing sliced record batch to IPC writes the entire array
 * ARROW-812 - Pip install pyarrow on mac failed.
 * ARROW-817 - [C++] Fix incorrect code comment from ARROW-722
-* ARROW-821 - [Python] Extra file _table_api.h generated during Python build process
+* ARROW-821 - [Python] Extra file `_table_api.h` generated during Python build process
 * ARROW-822 - [Python] StreamWriter fails to open with socket as sink
-* ARROW-826 - Compilation error on Mac with -DARROW_PYTHON=on
+* ARROW-826 - Compilation error on Mac with `-DARROW_PYTHON=on`
 * ARROW-829 - Python: Parquet: Dictionary encoding is deactivated if column-wise compression was selected
 * ARROW-830 - Python: jemalloc is not anymore publicly exposed
-* ARROW-839 - [C++] Portable alternative to PyDate_to_ms function
-* ARROW-847 - C++: BUILD_BYPRODUCTS not specified anymore for gtest
+* ARROW-839 - [C++] Portable alternative to `PyDate_to_ms` function
+* ARROW-847 - C++: `BUILD_BYPRODUCTS` not specified anymore for gtest
 * ARROW-852 - Python: Also set Arrow Library PATHS when detection was done through pkg-config
 * ARROW-853 - [Python] It is no longer necessary to modify the RPATH of the Cython extensions on many environments
 * ARROW-858 - Remove dependency on boost regex
@@ -262,7 +400,7 @@
 * ARROW-914 - [C++/Python] Fix Decimal ToBytes
 * ARROW-922 - Allow Flatbuffers and RapidJSON to be used locally on Windows
 * ARROW-928 - Update CMAKE script to detect unsupported msvc compilers versions
-* ARROW-933 - [Python] arrow_python bindings have debug print statement
+* ARROW-933 - [Python] `arrow_python` bindings have debug print statement
 * ARROW-934 - [GLib] Glib sources missing from result of 02-source.sh
 * ARROW-936 - Fix release README
 * ARROW-938 - Fix Apache Rat errors from source release build
@@ -275,7 +413,7 @@
 * ARROW-566 - Python: Deterministic position of libarrow in manylinux1 wheels
 * ARROW-569 - [C++] Set version for .pc
 * ARROW-577 - [C++] Refactor StreamWriter and FileWriter to have private implementations
-* ARROW-580 - C++: Also provide jemalloc_X targets if only a static or shared version is found
+* ARROW-580 - C++: Also provide `jemalloc_X` targets if only a static or shared version is found
 * ARROW-582 - [Java] Add Date/Time Support to JSON File
 * ARROW-589 - C++: Use system provided shared jemalloc if static is unavailable
 * ARROW-593 - [C++] Rename ReadableFileInterface to RandomAccessFile
@@ -296,7 +434,7 @@
 * ARROW-679 - [Format] Change RecordBatch and Field length members from int to long
 * ARROW-681 - [C++] Build Arrow on Windows with dynamically linked boost
 * ARROW-684 - Python: More informative message when parquet-cpp but not parquet-arrow is available
-* ARROW-688 - [C++] Use CMAKE_INSTALL_INCLUDEDIR for consistency
+* ARROW-688 - [C++] Use `CMAKE_INSTALL_INCLUDEDIR` for consistency
 * ARROW-690 - Only send JIRA updates to issues@arrow.apache.org
 * ARROW-700 - Add headroom interface for allocator.
 * ARROW-706 - [GLib] Add package install document
@@ -311,13 +449,13 @@
 * ARROW-731 - [C++] Add shared library related versions to .pc
 * ARROW-741 - [Python] Add Python 3.6 to Travis CI
 * ARROW-743 - [C++] Consolidate unit tests for code in array.h
-* ARROW-744 - [GLib] Re-add an assertion to garrow_table_new() test
+* ARROW-744 - [GLib] Re-add an assertion to `garrow_table_new()` test
 * ARROW-745 - [C++] Allow use of system cpplint
-* ARROW-746 - [GLib] Add garrow_array_get_data_type()
+* ARROW-746 - [GLib] Add `garrow_array_get_data_type()`
 * ARROW-751 - [Python] Rename all Cython extensions to "private" status with leading underscore
 * ARROW-752 - [Python] Construct pyarrow.DictionaryArray from boxed pyarrow array objects
-* ARROW-754 - [GLib] Add garrow_array_is_null()
-* ARROW-755 - [GLib] Add garrow_array_get_value_type()
+* ARROW-754 - [GLib] Add `garrow_array_is_null()`
+* ARROW-755 - [GLib] Add `garrow_array_get_value_type()`
 * ARROW-758 - [C++] Fix compiler warnings on MSVC x64
 * ARROW-761 - [Python] Add function to compute the total size of tensor payloads, including metadata and padding
 * ARROW-763 - C++: Use `python-config` to find libpythonX.X.dylib
@@ -329,7 +467,7 @@
 * ARROW-779 - [C++/Python] Raise exception if old metadata encountered
 * ARROW-782 - [C++] Change struct to class for objects that meet the criteria in the Google style guide
 * ARROW-788 - Possible nondeterminism in Tensor serialization code
-* ARROW-795 - [C++] Combine libarrow/libarrow_io/libarrow_ipc
+* ARROW-795 - [C++] Combine `libarrow/libarrow_io/libarrow_ipc`
 * ARROW-802 - [GLib] Add read examples
 * ARROW-803 - [GLib] Update package repository URL
 * ARROW-804 - [GLib] Update build document
@@ -342,7 +480,7 @@
 * ARROW-816 - [C++] Use conda packages for RapidJSON, Flatbuffers to speed up builds
 * ARROW-818 - [Python] Review public pyarrow. API completeness and update docs
 * ARROW-820 - [C++] Build dependencies for Parquet library without arrow support
-* ARROW-825 - [Python] Generalize pyarrow.from_pylist to accept any object implementing the PySequence protocol
+* ARROW-825 - [Python] Generalize `pyarrow.from_pylist` to accept any object implementing the PySequence protocol
 * ARROW-827 - [Python] Variety of Parquet improvements to support Dask integration
 * ARROW-828 - [CPP] Document new requirement (libboost-regex-dev) in README.md
 * ARROW-832 - [C++] Upgrade thirdparty gtest to 1.8.0
@@ -352,7 +490,7 @@
 * ARROW-845 - [Python] Sync FindArrow.cmake changes from parquet-cpp
 * ARROW-846 - [GLib] Add GArrowTensor, GArrowInt8Tensor and GArrowUInt8Tensor
 * ARROW-848 - [Python] Improvements / fixes to conda quickstart guide
-* ARROW-849 - [C++] Add optional $ARROW_BUILD_TOOLCHAIN environment variable option for configuring build environment
+* ARROW-849 - [C++] Add optional `$ARROW_BUILD_TOOLCHAIN` environment variable option for configuring build environment
 * ARROW-857 - [Python] Automate publishing Python documentation to arrow-site
 * ARROW-860 - [C++] Decide if typed Tensor subclasses are worthwhile
 * ARROW-861 - [Python] Move DEVELOPMENT.md to Sphinx docs
@@ -362,8 +500,8 @@
 * ARROW-868 - [GLib] Use GBytes to reduce copy
 * ARROW-871 - [GLib] Unify DataType files
 * ARROW-876 - [GLib] Unify ArrayBuffer files
-* ARROW-877 - [GLib] Add garrow_array_get_null_bitmap()
-* ARROW-878 - [GLib] Add garrow_binary_array_get_buffer()
+* ARROW-877 - [GLib] Add `garrow_array_get_null_bitmap()`
+* ARROW-878 - [GLib] Add `garrow_binary_array_get_buffer()`
 * ARROW-892 - [GLib] Fix GArrowTensor document
 * ARROW-893 - Add GLib document to Web site
 * ARROW-894 - [GLib] Add GArrowPoolBuffer
@@ -389,13 +527,13 @@
 * ARROW-341 - [Python] Making libpyarrow available to third parties
 * ARROW-452 - [C++/Python] Merge "Feather" file format implementation
 * ARROW-459 - [C++] Implement IPC round trip for DictionaryArray, dictionaries shared across record batches
-* ARROW-483 - [C++/Python] Provide access to "custom_metadata" Field attribute in IPC setting
+* ARROW-483 - [C++/Python] Provide access to `custom_metadata` Field attribute in IPC setting
 * ARROW-491 - [C++] Add FixedWidthBinary type
 * ARROW-493 - [C++] Allow in-memory array over 2^31 -1 elements but require splitting at IPC / RPC boundaries
 * ARROW-502 - [C++/Python] Add MemoryPool implementation that logs allocation activity to std::cout
 * ARROW-510 - Add integration tests for date and time types
 * ARROW-520 - [C++] Add STL-compliant allocator that hooks into an arrow::MemoryPool
-* ARROW-528 - [Python] Support _metadata or _common_metadata files when reading Parquet directories
+* ARROW-528 - [Python] Support `_metadata` or `_common_metadata` files when reading Parquet directories
 * ARROW-534 - [C++] Add IPC tests for date/time types
 * ARROW-539 - [Python] Support reading Parquet datasets with standard partition directory schemes
 * ARROW-550 - [Format] Add a TensorMessage type
@@ -444,7 +582,7 @@
 * ARROW-771 - [Python] Add APIs for reading individual Parquet row groups
 * ARROW-773 - [C++] Add function to create arrow::Table with column appended to existing table
 * ARROW-865 - [Python] Verify Parquet roundtrips for new date/time types
-* ARROW-880 - [GLib] Add garrow_primitive_array_get_buffer()
+* ARROW-880 - [GLib] Add `garrow_primitive_array_get_buffer()`
 * ARROW-890 - [GLib] Add GArrowMutableBuffer
 * ARROW-926 - Update KEYS to include wesm
 
@@ -481,7 +619,7 @@
 * ARROW-323 - [Python] Opt-in to PyArrow parquet build rather than skipping silently on failure
 * ARROW-334 - [Python] OS X rpath issues on some configurations
 * ARROW-337 - UnionListWriter.list() is doing more than it should, this can cause data corruption
-* ARROW-339 - Make merge_arrow_pr script work with Python 3
+* ARROW-339 - Make `merge_arrow_pr` script work with Python 3
 * ARROW-340 - [C++] Opening a writeable file on disk that already exists does not truncate to zero
 * ARROW-342 - Set Python version on release
 * ARROW-345 - libhdfs integration doesn't work for Mac
@@ -490,15 +628,15 @@
 * ARROW-349 - Six is missing as a requirement in the python setup.py
 * ARROW-351 - Time type has no unit
 * ARROW-354 - Connot compare an array of empty strings to another
-* ARROW-357 - Default Parquet chunk_size of 64k is too small
+* ARROW-357 - Default Parquet `chunk_size` of 64k is too small
 * ARROW-358 - [C++] libhdfs can be in non-standard locations in some Hadoop distributions
-* ARROW-362 - Python: Calling to_pandas on a table read from Parquet leaks memory
+* ARROW-362 - Python: Calling `to_pandas` on a table read from Parquet leaks memory
 * ARROW-371 - Python: Table with null timestamp becomes float in pandas
-* ARROW-375 - columns parameter in parquet.read_table() raises KeyError for valid column
+* ARROW-375 - columns parameter in `parquet.read_table()` raises KeyError for valid column
 * ARROW-384 - Align Java and C++ RecordBatch data and metadata layout
 * ARROW-386 - [Java] Respect case of struct / map field names
 * ARROW-387 - [C++] arrow::io::BufferReader does not permit shared memory ownership in zero-copy reads
-* ARROW-390 - C++: CMake fails on json-integration-test with ARROW_BUILD_TESTS=OFF
+* ARROW-390 - C++: CMake fails on json-integration-test with `ARROW_BUILD_TESTS=OFF`
 * ARROW-392 - Fix string/binary integration tests
 * ARROW-393 - [JAVA] JSON file reader fails to set the buffer size on String data vector
 * ARROW-395 - Arrow file format writes record batches in reverse order.
@@ -509,19 +647,19 @@
 * ARROW-402 - [Java] "refCnt gone negative" error in integration tests
 * ARROW-403 - [JAVA] UnionVector: Creating a transfer pair doesn't transfer the schema to destination vector
 * ARROW-404 - [Python] Closing an HdfsClient while there are still open file handles results in a crash
-* ARROW-405 - [C++] Be less stringent about finding include/hdfs.h in HADOOP_HOME
+* ARROW-405 - [C++] Be less stringent about finding include/hdfs.h in `HADOOP_HOME`
 * ARROW-406 - [C++] Large HDFS reads must utilize the set file buffer size when making RPCs
 * ARROW-408 - [C++/Python] Remove defunct conda recipes
 * ARROW-414 - [Java] "Buffer too large to resize to ..." error
 * ARROW-420 - Align Date implementation between Java and C++
 * ARROW-421 - [Python] Zero-copy buffers read by pyarrow::PyBytesReader must retain a reference to the parent PyBytes to avoid premature garbage collection issues
-* ARROW-422 - C++: IPC should depend on rapidjson_ep if RapidJSON is vendored
+* ARROW-422 - C++: IPC should depend on `rapidjson_ep` if RapidJSON is vendored
 * ARROW-429 - git-archive SHA-256 checksums are changing
 * ARROW-433 - [Python] Date conversion is locale-dependent
 * ARROW-434 - Segfaults and encoding issues in Python Parquet reads
-* ARROW-435 - C++: Spelling mistake in if(RAPIDJSON_VENDORED)
+* ARROW-435 - C++: Spelling mistake in `if(RAPIDJSON_VENDORED)`
 * ARROW-437 - [C++] clang compiler warnings from overridden virtual functions
-* ARROW-445 - C++: arrow_ipc is built before arrow/ipc/Message_generated.h was generated
+* ARROW-445 - C++: `arrow_ipc` is built before `arrow/ipc/Message_generated.h` was generated
 * ARROW-447 - Python: Align scalar/pylist string encoding with pandas' one.
 * ARROW-455 - [C++] BufferOutputStream dtor does not call Close()
 * ARROW-469 - C++: Add option so that resize doesn't decrease the capacity
@@ -536,13 +674,13 @@
 * ARROW-519 - [C++] Missing vtable in libarrow.dylib on Xcode 6.4
 * ARROW-523 - Python: Account for changes in PARQUET-834
 * ARROW-533 - [C++] arrow::TimestampArray / TimeArray has a broken constructor
-* ARROW-535 - [Python] Add type mapping for NPY_LONGLONG
+* ARROW-535 - [Python] Add type mapping for `NPY_LONGLONG`
 * ARROW-537 - [C++] StringArray/BinaryArray comparisons may be incorrect when values with non-zero length are null
 * ARROW-540 - [C++] Fix build in aftermath of ARROW-33
-* ARROW-543 - C++: Lazily computed null_counts counts number of non-null entries
+* ARROW-543 - C++: Lazily computed `null_counts` counts number of non-null entries
 * ARROW-544 - [C++] ArrayLoader::LoadBinary fails for length-0 arrays
 * ARROW-545 - [Python] Ignore files without .parq or .parquet prefix when reading directory of files
-* ARROW-548 - [Python] Add nthreads option to pyarrow.Filesystem.read_parquet
+* ARROW-548 - [Python] Add nthreads option to `pyarrow.Filesystem.read_parquet`
 * ARROW-551 - C++: Construction of Column with nullptr Array segfaults
 * ARROW-556 - [Integration] Can not run Integration tests if different cpp build path
 * ARROW-561 - Update java & python dependencies to improve downstream packaging experience
@@ -551,7 +689,7 @@
 
 * ARROW-189 - C++: Use ExternalProject to build thirdparty dependencies
 * ARROW-191 - Python: Provide infrastructure for manylinux1 wheels
-* ARROW-328 - [C++] Return shared_ptr by value instead of const-ref?
+* ARROW-328 - [C++] Return `shared_ptr` by value instead of const-ref?
 * ARROW-330 - [C++] CMake functions to simplify shared / static library configuration
 * ARROW-333 - Make writers update their internal schema even when no data is written.
 * ARROW-335 - Improve Type apis and toString() by encapsulating flatbuffers better
@@ -562,20 +700,20 @@
 * ARROW-356 - Add documentation about reading Parquet
 * ARROW-360 - C++: Add method to shrink PoolBuffer using realloc
 * ARROW-361 - Python: Support reading a column-selection from Parquet files
-* ARROW-365 - Python: Provide Array.to_pandas()
+* ARROW-365 - Python: Provide `Array.to_pandas()`
 * ARROW-366 - [java] implement Dictionary vector
 * ARROW-374 - Python: clarify unicode vs. binary in API
-* ARROW-379 - Python: Use setuptools_scm/setuptools_scm_git_archive to provide the version number
+* ARROW-379 - Python: Use `setuptools_scm`/`setuptools_scm_git_archive` to provide the version number
 * ARROW-380 - [Java] optimize null count when serializing vectors.
 * ARROW-382 - Python: Extend API documentation
 * ARROW-396 - Python: Add pyarrow.schema.Schema.equals
-* ARROW-409 - Python: Change pyarrow.Table.dataframe_from_batches API to create Table instead
+* ARROW-409 - Python: Change `pyarrow.Table.dataframe_from_batches` API to create Table instead
 * ARROW-411 - [Java] Move Intergration.compare and Intergration.compareSchemas to a public utils class
-* ARROW-423 - C++: Define BUILD_BYPRODUCTS in external project to support non-make CMake generators
+* ARROW-423 - C++: Define `BUILD_BYPRODUCTS` in external project to support non-make CMake generators
 * ARROW-425 - Python: Expose a C function to convert arrow::Table to pyarrow.Table
 * ARROW-426 - Python: Conversion from pyarrow.Array to a Python list
 * ARROW-430 - Python: Better version handling
-* ARROW-432 - [Python] Avoid unnecessary memory copy in to_pandas conversion by using low-level pandas internals APIs
+* ARROW-432 - [Python] Avoid unnecessary memory copy in `to_pandas` conversion by using low-level pandas internals APIs
 * ARROW-450 - Python: Fixes for PARQUET-818
 * ARROW-457 - Python: Better control over memory pool
 * ARROW-458 - Python: Expose jemalloc MemoryPool
@@ -596,7 +734,7 @@
 
 * ARROW-108 - [C++] Add IPC round trip for union types
 * ARROW-221 - Add switch for writing Parquet 1.0 compatible logical types
-* ARROW-227 - [C++/Python] Hook arrow_io generic reader / writer interface into arrow_parquet
+* ARROW-227 - [C++/Python] Hook `arrow_io` generic reader / writer interface into `arrow_parquet`
 * ARROW-228 - [Python] Create an Arrow-cpp-compatible interface for reading bytes from Python file-like objects
 * ARROW-243 - [C++] Add "driver" option to HdfsClient to choose between libhdfs and libhdfs3 at runtime
 * ARROW-303 - [C++] Also build static libraries for leaf libraries
@@ -624,7 +762,7 @@
 * ARROW-440 - [C++] Support pkg-config
 * ARROW-441 - [Python] Expose Arrow's file and memory map classes as NativeFile subclasses
 * ARROW-442 - [Python] Add public Python API to inspect Parquet file metadata
-* ARROW-444 - [Python] Avoid unnecessary memory copies from use of PyBytes_* C APIs
+* ARROW-444 - [Python] Avoid unnecessary memory copies from use of `PyBytes_*` C APIs
 * ARROW-449 - Python: Conversion from pyarrow.{Table,RecordBatch} to a Python dict
 * ARROW-456 - C++: Add jemalloc based MemoryPool
 * ARROW-461 - [Python] Implement conversion between arrow::DictionaryArray and pandas.Categorical
@@ -657,9 +795,9 @@
 * ARROW-268 - [C++] Flesh out union implementation to have all required methods for IPC
 * ARROW-327 - [Python] Remove conda builds from Travis CI processes
 * ARROW-353 - Arrow release 0.2
-* ARROW-359 - Need to document ARROW_LIBHDFS_DIR
+* ARROW-359 - Need to document `ARROW_LIBHDFS_DIR`
 * ARROW-367 - [java] converter csv/json <=> Arrow file format for Integration tests
-* ARROW-368 - Document use of LD_LIBRARY_PATH when using Python
+* ARROW-368 - Document use of `LD_LIBRARY_PATH` when using Python
 * ARROW-372 - Create JSON arrow file format for integration tests
 * ARROW-506 - Implement Arrow Echo server for integration testing
 * ARROW-527 - clean drill-module.conf file
@@ -687,7 +825,7 @@
 * ARROW-210 - [C++] Tidy up the type system a little bit
 * ARROW-211 - Several typos/errors in Layout.md examples
 * ARROW-217 - Fix Travis w.r.t conda 4.1.0 changes
-* ARROW-219 - [C++] Passed CMAKE_CXX_FLAGS are being dropped, fix compiler warnings
+* ARROW-219 - [C++] Passed `CMAKE_CXX_FLAGS` are being dropped, fix compiler warnings
 * ARROW-223 - Do not link against libpython
 * ARROW-225 - [C++/Python] master Travis CI build is broken
 * ARROW-244 - [C++] Some global APIs of IPC module should be visible to the outside
@@ -699,7 +837,7 @@
 * ARROW-266 - [C++] Fix the broken build
 * ARROW-274 - Make the MapVector nullable
 * ARROW-278 - [Format] Struct type name consistency in implementations and metadata
-* ARROW-283 - [C++] Update arrow_parquet to account for API changes in PARQUET-573
+* ARROW-283 - [C++] Update `arrow_parquet` to account for API changes in PARQUET-573
 * ARROW-284 - [C++] Triage builds by disabling Arrow-Parquet module
 * ARROW-287 - [java] Make nullable vectors use a BitVecor instead of UInt1Vector for bits
 * ARROW-297 - Fix Arrow pom for release
@@ -737,7 +875,7 @@
 * ARROW-212 - [C++] Clarify the fact that PrimitiveArray is now abstract class
 * ARROW-213 - Exposing static arrow build
 * ARROW-218 - Add option to use GitHub API token via environment variable when merging PRs
-* ARROW-234 - [C++] Build with libhdfs support in arrow_io in conda builds
+* ARROW-234 - [C++] Build with libhdfs support in `arrow_io` in conda builds
 * ARROW-238 - C++: InternalMemoryPool::Free() should throw an error when there is insufficient allocated memory
 * ARROW-245 - [Format] Clarify Arrow's relationship with big endian platforms
 * ARROW-252 - Add implementation guidelines to the documentation
@@ -757,7 +895,7 @@
 * ARROW-290 - Specialize alloc() in ArrowBuf
 * ARROW-292 - [Java] Upgrade Netty to 4.041
 * ARROW-299 - Use absolute namespace in macros
-* ARROW-305 - Add compression and use_dictionary options to Parquet interface
+* ARROW-305 - Add compression and `use_dictionary` options to Parquet interface
 * ARROW-306 - Add option to pass cmake arguments via environment variable
 * ARROW-315 - Finalize timestamp type
 * ARROW-319 - Add canonical Arrow Schema json representation
@@ -767,7 +905,7 @@
 * ARROW-54 - Python: rename package to "pyarrow"
 * ARROW-64 - Add zsh support to C++ build scripts
 * ARROW-66 - Maybe some missing steps in installation guide
-* ARROW-68 - Update setup_build_env and third-party script to be more userfriendly
+* ARROW-68 - Update `setup_build_env` and third-party script to be more userfriendly
 * ARROW-71 - C++: Add script to run clang-tidy on codebase
 * ARROW-73 - Support CMake 2.8
 * ARROW-78 - C++: Add constructor for DecimalType
@@ -809,7 +947,7 @@
 * ARROW-267 - [C++] C++ implementation of file-like layout for RPC / IPC
 * ARROW-28 - C++: Add google/benchmark to the 3rd-party build toolchain
 * ARROW-293 - [C++] Implementations of IO interfaces for operating system files
-* ARROW-296 - [C++] Remove arrow_parquet C++ module and related parts of build system
+* ARROW-296 - [C++] Remove `arrow_parquet` C++ module and related parts of build system
 * ARROW-3 - Post Initial Arrow Format Spec
 * ARROW-30 - Python: pandas/NumPy to/from Arrow conversion routines
 * ARROW-301 - [Format] Add some form of user field metadata to IPC schemas
@@ -819,8 +957,8 @@
 * ARROW-37 - C++: Represent boolean array data in bit-packed form
 * ARROW-4 - Initial Arrow CPP Implementation
 * ARROW-42 - Python: Add to Travis CI build
-* ARROW-43 - Python: Add rudimentary console __repr__ for array types
-* ARROW-44 - Python: Implement basic object model for scalar values (i.e. results of arrow_arr[i])
+* ARROW-43 - Python: Add rudimentary console `__repr__` for array types
+* ARROW-44 - Python: Implement basic object model for scalar values (i.e. results of `arrow_arr[i]`)
 * ARROW-48 - Python: Add Schema object wrapper
 * ARROW-49 - Python: Add Column and Table wrapper interface
 * ARROW-53 - Python: Fix RPATH and add source installation instructions
diff --git a/dev/make_changelog.py b/dev/make_changelog.py
index 47127903b7b1f..b4b0070df8e2d 100644
--- a/dev/make_changelog.py
+++ b/dev/make_changelog.py
@@ -74,6 +74,7 @@ def format_changelog_website(issues, out):
     CATEGORIES = {
         'New Feature': NEW_FEATURE,
         'Improvement': NEW_FEATURE,
+        'Wish': NEW_FEATURE,
         'Task': NEW_FEATURE,
         'Test': NEW_FEATURE,
         'Bug': BUGFIX
diff --git a/site/_posts/2017-07-24-0.5.0-release.md b/site/_posts/2017-07-24-0.5.0-release.md
new file mode 100644
index 0000000000000..5c156bfec783d
--- /dev/null
+++ b/site/_posts/2017-07-24-0.5.0-release.md
@@ -0,0 +1,114 @@
+---
+layout: post
+title: "Apache Arrow 0.5.0 Release"
+date: "2017-07-25 00:00:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.5.0 release. It includes
+[**130 resolved JIRAs**][1] with some new features, expanded integration
+testing between implementations, and bug fixes. The Arrow memory format remains
+stable since the 0.3.x and 0.4.x releases.
+
+See the [Install Page][2] to learn how to get the libraries for your
+platform. The [complete changelog][5] is also available.
+
+## Expanded Integration Testing
+
+In this release, we added compatibility tests for dictionary-encoded data
+between Java and C++. This enables the distinct values (the *dictionary*) in a
+vector to be transmitted as part of an Arrow schema while the record batches
+contain integers which correspond to the dictionary.
+
+So we might have:
+
+```
+data (string): ['foo', 'bar', 'foo', 'bar']
+```
+
+In dictionary-encoded form, this could be represented as:
+
+```
+indices (int8): [0, 1, 0, 1]
+dictionary (string): ['foo', 'bar']
+```
+
+In upcoming releases, we plan to complete integration testing for the remaining
+data types (including some more complicated types like unions and decimals) on
+the road to a 1.0.0 release in the future.
+
+## C++ Activity
+
+We completed a number of significant pieces of work in the C++ part of Apache
+Arrow.
+
+### Using jemalloc as default memory allocator
+
+We decided to use [jemalloc][4] as the default memory allocator unless it is
+explicitly disabled. This memory allocator has significant performance
+advantages in Arrow workloads over the default `malloc` implementation. We will
+publish a blog post going into more detail about this and why you might care.
+
+### Sharing more C++ code with Apache Parquet
+
+We imported the compression library interfaces and dictionary encoding
+algorithms from the [Apache Parquet C++ library][3]. The Parquet library now
+depends on this code in Arrow, and we will be able to use it more easily for
+data compression in Arrow use cases.
+
+As part of incorporating Parquet's dictionary encoding utilities, we have
+developed an `arrow::DictionaryBuilder` class to enable building
+dictionary-encoded arrays iteratively. This can help save memory and yield
+better performance when interacting with databases, Parquet files, or other
+sources which may have columns having many duplicates.
+
+### Support for LZ4 and ZSTD compressors
+
+We added LZ4 and ZSTD compression library support. In ARROW-300 and other
+planned work, we intend to add some compression features for data sent via RPC.
+
+## Python Activity
+
+We fixed many bugs which were affecting Parquet and Feather users and fixed
+several other rough edges with normal Arrow use. We also added some additional
+Arrow type conversions: structs, lists embedded in pandas objects, and Arrow
+time types (which deserialize to the `datetime.time` type).
+
+In upcoming releases we plan to continue to improve [Dask][7] support and
+performance for distributed processing of Apache Parquet files with pyarrow.
+
+## The Road Ahead
+
+We have much work ahead of us to build out Arrow integrations in other data
+systems to improve their processing performance and interoperability with other
+systems.
+
+We are discussing the roadmap to a future 1.0.0 release on the [developer
+mailing list][6]. Please join the discussion there.
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.5.0
+[2]: http://arrow.apache.org/install
+[3]: http://github.com/apache/parquet-cpp
+[4]: https://github.com/jemalloc/jemalloc
+[5]: http://arrow.apache.org/release/0.5.0.html
+[6]: http://mail-archives.apache.org/mod_mbox/arrow-dev/
+[7]: http://github.com/dask/dask
\ No newline at end of file
diff --git a/site/_release/0.5.0.md b/site/_release/0.5.0.md
new file mode 100644
index 0000000000000..3073f8c526fca
--- /dev/null
+++ b/site/_release/0.5.0.md
@@ -0,0 +1,203 @@
+---
+layout: default
+title: Apache Arrow 0.5.0 Release
+permalink: /release/0.5.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.5.0 (23 July 2017)
+
+This is a major release, with expanded features in the supported languages and
+additional integration test coverage between Java and C++.
+
+Read more in the [release blog post][8].
+
+## Download
+
+* [**Source Artifacts**][6]
+* [Git tag][2]
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.4.1..apache-arrow-0.5.0
+    42  Wes McKinney
+    22  Uwe L. Korn
+    12  Kouhei Sutou
+     9  Max Risuhin
+     9  Phillip Cloud
+     6  Philipp Moritz
+     5  Steven Phillips
+     3  Julien Le Dem
+     2  Bryan Cutler
+     2  Kengo Seki
+     2  Max Risukhin
+     2  fjetter
+     1  Antony Mayi
+     1  Deepak Majeti
+     1  Fang Zheng
+     1  Hideo Hattori
+     1  Holden Karau
+     1  Itai Incze
+     1  Jeff Knupp
+     1  LynnYuan
+     1  Mark Lavrynenko
+     1  Michael König
+     1  Robert Nishihara
+     1  Sudheesh Katkam
+     1  Zahari
+     1  vkorukanti
+```
+
+# Changelog
+
+## New Features and Improvements
+
+* [ARROW-1041](https://issues.apache.org/jira/browse/ARROW-1041) - [Python] Support read_pandas on a directory of Parquet files
+* [ARROW-1048](https://issues.apache.org/jira/browse/ARROW-1048) - Allow user LD_LIBRARY_PATH to be used with source release script
+* [ARROW-1052](https://issues.apache.org/jira/browse/ARROW-1052) - Arrow 0.5.0 release
+* [ARROW-1073](https://issues.apache.org/jira/browse/ARROW-1073) - C++: Adapative integer builder
+* [ARROW-1095](https://issues.apache.org/jira/browse/ARROW-1095) - [Website] Add Arrow icon asset
+* [ARROW-1100](https://issues.apache.org/jira/browse/ARROW-1100) - [Python] Add "mode" property to NativeFile instances
+* [ARROW-1102](https://issues.apache.org/jira/browse/ARROW-1102) - Make MessageSerializer.serializeMessage() public
+* [ARROW-111](https://issues.apache.org/jira/browse/ARROW-111) - [C++] Add static analyzer to tool chain to verify checking of Status returns
+* [ARROW-1120](https://issues.apache.org/jira/browse/ARROW-1120) - [Python] Write support for int96
+* [ARROW-1122](https://issues.apache.org/jira/browse/ARROW-1122) - [Website] Guest blog post on Arrow + ODBC from turbodbc
+* [ARROW-1123](https://issues.apache.org/jira/browse/ARROW-1123) - C++: Make jemalloc the default allocator
+* [ARROW-1135](https://issues.apache.org/jira/browse/ARROW-1135) - Upgrade Travis CI clang builds to use LLVM 4.0
+* [ARROW-1137](https://issues.apache.org/jira/browse/ARROW-1137) - Python: Ensure Pandas roundtrip of all-None column
+* [ARROW-1142](https://issues.apache.org/jira/browse/ARROW-1142) - [C++] Move over compression library toolchain from parquet-cpp
+* [ARROW-1145](https://issues.apache.org/jira/browse/ARROW-1145) - [GLib] Add get_values()
+* [ARROW-1146](https://issues.apache.org/jira/browse/ARROW-1146) - Add .gitignore for *_generated.h files in src/plasma/format
+* [ARROW-1148](https://issues.apache.org/jira/browse/ARROW-1148) - [C++] Raise minimum CMake version to 3.2
+* [ARROW-1151](https://issues.apache.org/jira/browse/ARROW-1151) - [C++] Add gcc branch prediction to status check macro
+* [ARROW-1154](https://issues.apache.org/jira/browse/ARROW-1154) - [C++] Migrate more computational utility code from parquet-cpp
+* [ARROW-1160](https://issues.apache.org/jira/browse/ARROW-1160) - C++: Implement DictionaryBuilder
+* [ARROW-1165](https://issues.apache.org/jira/browse/ARROW-1165) - [C++] Refactor PythonDecimalToArrowDecimal to not use templates
+* [ARROW-1172](https://issues.apache.org/jira/browse/ARROW-1172) - [C++] Use unique_ptr with array builder classes
+* [ARROW-1183](https://issues.apache.org/jira/browse/ARROW-1183) - [Python] Implement time type conversions in to_pandas
+* [ARROW-1185](https://issues.apache.org/jira/browse/ARROW-1185) - [C++] Clean up arrow::Status implementation, add warn_unused_result attribute for clang
+* [ARROW-1187](https://issues.apache.org/jira/browse/ARROW-1187) - Serialize a DataFrame with None column
+* [ARROW-1193](https://issues.apache.org/jira/browse/ARROW-1193) - [C++] Support pkg-config forarrow_python.so
+* [ARROW-1196](https://issues.apache.org/jira/browse/ARROW-1196) - [C++] Appveyor separate jobs for Debug/Release builds from sources; Build with conda toolchain; Build with NMake Makefiles Generator
+* [ARROW-1198](https://issues.apache.org/jira/browse/ARROW-1198) - Python: Add public C++ API to unwrap PyArrow object
+* [ARROW-1199](https://issues.apache.org/jira/browse/ARROW-1199) - [C++] Introduce mutable POD struct for generic array data
+* [ARROW-1202](https://issues.apache.org/jira/browse/ARROW-1202) - Remove semicolons from status macros
+* [ARROW-1212](https://issues.apache.org/jira/browse/ARROW-1212) - [GLib] Add garrow_binary_array_get_offsets_buffer()
+* [ARROW-1214](https://issues.apache.org/jira/browse/ARROW-1214) - [Python] Add classes / functions to enable stream message components to be handled outside of the stream reader class
+* [ARROW-1217](https://issues.apache.org/jira/browse/ARROW-1217) - [GLib] Add GInputStream based arrow::io::RandomAccessFile
+* [ARROW-1220](https://issues.apache.org/jira/browse/ARROW-1220) - [C++] Standartize usage of *_HOME cmake script variables for 3rd party libs
+* [ARROW-1221](https://issues.apache.org/jira/browse/ARROW-1221) - [C++] Pin clang-format version
+* [ARROW-1227](https://issues.apache.org/jira/browse/ARROW-1227) - [GLib] Support GOutputStream
+* [ARROW-1228](https://issues.apache.org/jira/browse/ARROW-1228) - [GLib] Test file name should be the same name as target class
+* [ARROW-1229](https://issues.apache.org/jira/browse/ARROW-1229) - [GLib] Follow Reader API change (get -> read)
+* [ARROW-1233](https://issues.apache.org/jira/browse/ARROW-1233) - [C++] Validate cmake script resolving of 3rd party linked libs from correct location in toolchain build
+* [ARROW-460](https://issues.apache.org/jira/browse/ARROW-460) - [C++] Implement JSON round trip for DictionaryArray
+* [ARROW-462](https://issues.apache.org/jira/browse/ARROW-462) - [C++] Implement in-memory conversions between non-nested primitive types and DictionaryArray equivalent
+* [ARROW-575](https://issues.apache.org/jira/browse/ARROW-575) - Python: Auto-detect nested lists and nested numpy arrays in Pandas
+* [ARROW-597](https://issues.apache.org/jira/browse/ARROW-597) - [Python] Add convenience function to yield DataFrame from any object that a StreamReader or FileReader can read from
+* [ARROW-599](https://issues.apache.org/jira/browse/ARROW-599) - [C++] Add LZ4 codec to 3rd-party toolchain
+* [ARROW-600](https://issues.apache.org/jira/browse/ARROW-600) - [C++] Add ZSTD codec to 3rd-party toolchain
+* [ARROW-692](https://issues.apache.org/jira/browse/ARROW-692) - Java<->C++ Integration tests for dictionary-encoded vectors
+* [ARROW-693](https://issues.apache.org/jira/browse/ARROW-693) - [Java] Add JSON support for dictionary vectors
+* [ARROW-742](https://issues.apache.org/jira/browse/ARROW-742) - Handling exceptions during execution of std::wstring_convert
+* [ARROW-834](https://issues.apache.org/jira/browse/ARROW-834) - [Python] Support creating Arrow arrays from Python iterables
+* [ARROW-915](https://issues.apache.org/jira/browse/ARROW-915) - Struct Array reads limited support
+* [ARROW-935](https://issues.apache.org/jira/browse/ARROW-935) - [Java] Build Javadoc in Travis CI
+* [ARROW-960](https://issues.apache.org/jira/browse/ARROW-960) - [Python] Add source build guide for macOS + Homebrew
+* [ARROW-962](https://issues.apache.org/jira/browse/ARROW-962) - [Python] Add schema attribute to FileReader
+* [ARROW-966](https://issues.apache.org/jira/browse/ARROW-966) - [Python] pyarrow.list_ should also accept Field instance
+* [ARROW-978](https://issues.apache.org/jira/browse/ARROW-978) - [Python] Use sphinx-bootstrap-theme for Sphinx documentation
+
+## Bug Fixes
+
+* [ARROW-1074](https://issues.apache.org/jira/browse/ARROW-1074) - from_pandas doesnt convert ndarray to list
+* [ARROW-1079](https://issues.apache.org/jira/browse/ARROW-1079) - [Python] Empty "private" directories should be ignored by Parquet interface
+* [ARROW-1081](https://issues.apache.org/jira/browse/ARROW-1081) - C++: arrow::test::TestBase::MakePrimitive doesn't fill null_bitmap
+* [ARROW-1096](https://issues.apache.org/jira/browse/ARROW-1096) - [C++] Memory mapping file over 4GB fails on Windows
+* [ARROW-1097](https://issues.apache.org/jira/browse/ARROW-1097) - Reading tensor needs file to be opened in writeable mode
+* [ARROW-1098](https://issues.apache.org/jira/browse/ARROW-1098) - Document Error?
+* [ARROW-1101](https://issues.apache.org/jira/browse/ARROW-1101) - UnionListWriter is not implementing all methods on interface ScalarWriter
+* [ARROW-1103](https://issues.apache.org/jira/browse/ARROW-1103) - [Python] Utilize pandas metadata from common _metadata Parquet file if it exists
+* [ARROW-1107](https://issues.apache.org/jira/browse/ARROW-1107) - [JAVA] NullableMapVector getField() should return nullable type
+* [ARROW-1108](https://issues.apache.org/jira/browse/ARROW-1108) - Check if ArrowBuf is empty buffer in getActualConsumedMemory() and getPossibleConsumedMemory()
+* [ARROW-1109](https://issues.apache.org/jira/browse/ARROW-1109) - [JAVA] transferOwnership fails when readerIndex is not 0
+* [ARROW-1110](https://issues.apache.org/jira/browse/ARROW-1110) - [JAVA] make union vector naming consistent
+* [ARROW-1111](https://issues.apache.org/jira/browse/ARROW-1111) - [JAVA] Make aligning buffers optional, and allow -1 for unknown null count
+* [ARROW-1112](https://issues.apache.org/jira/browse/ARROW-1112) - [JAVA] Set lastSet for VarLength and List vectors when loading
+* [ARROW-1113](https://issues.apache.org/jira/browse/ARROW-1113) - [C++] gflags EP build gets triggered (as a no-op) on subsequent calls to make or ninja build
+* [ARROW-1115](https://issues.apache.org/jira/browse/ARROW-1115) - [C++] Use absolute path for ccache
+* [ARROW-1117](https://issues.apache.org/jira/browse/ARROW-1117) - [Docs] Minor issues in GLib README
+* [ARROW-1124](https://issues.apache.org/jira/browse/ARROW-1124) - [Python] pyarrow needs to depend on numpy>=1.10 (not 1.9)
+* [ARROW-1125](https://issues.apache.org/jira/browse/ARROW-1125) - Python: Table.from_pandas doesn't work anymore on partial schemas
+* [ARROW-1128](https://issues.apache.org/jira/browse/ARROW-1128) - [Docs] command to build a wheel is not properly rendered
+* [ARROW-1129](https://issues.apache.org/jira/browse/ARROW-1129) - [C++] Fix Linux toolchain build regression from ARROW-742
+* [ARROW-1131](https://issues.apache.org/jira/browse/ARROW-1131) - Python: Parquet unit tests are always skipped
+* [ARROW-1132](https://issues.apache.org/jira/browse/ARROW-1132) - [Python] Unable to write pandas DataFrame w/MultiIndex containing duplicate values to parquet
+* [ARROW-1136](https://issues.apache.org/jira/browse/ARROW-1136) - [C++/Python] Segfault on empty stream
+* [ARROW-1138](https://issues.apache.org/jira/browse/ARROW-1138) - Travis: Use OpenJDK7 instead of OracleJDK7
+* [ARROW-1139](https://issues.apache.org/jira/browse/ARROW-1139) - [C++] dlmalloc doesn't allow arrow to be built with clang 4 or gcc 7.1.1
+* [ARROW-1141](https://issues.apache.org/jira/browse/ARROW-1141) - on import get libjemalloc.so.2: cannot allocate memory in static TLS block
+* [ARROW-1143](https://issues.apache.org/jira/browse/ARROW-1143) - C++: Fix comparison of NullArray
+* [ARROW-1144](https://issues.apache.org/jira/browse/ARROW-1144) - [C++] Remove unused variable
+* [ARROW-1147](https://issues.apache.org/jira/browse/ARROW-1147) - [C++] Allow optional vendoring of flatbuffers in plasma
+* [ARROW-1150](https://issues.apache.org/jira/browse/ARROW-1150) - [C++] AdaptiveIntBuilder compiler warning on MSVC
+* [ARROW-1152](https://issues.apache.org/jira/browse/ARROW-1152) - [Cython] read_tensor should work with a readable file
+* [ARROW-1155](https://issues.apache.org/jira/browse/ARROW-1155) - segmentation fault when run pa.Int16Value()
+* [ARROW-1157](https://issues.apache.org/jira/browse/ARROW-1157) - C++/Python: Decimal templates are not correctly exported on OSX
+* [ARROW-1159](https://issues.apache.org/jira/browse/ARROW-1159) - [C++] Static data members cannot be accessed from inline functions in Arrow headers by thirdparty users
+* [ARROW-1162](https://issues.apache.org/jira/browse/ARROW-1162) - Transfer Between Empty Lists Should Not Invoke Callback
+* [ARROW-1166](https://issues.apache.org/jira/browse/ARROW-1166) - Errors in Struct type's example and missing reference in Layout.md
+* [ARROW-1167](https://issues.apache.org/jira/browse/ARROW-1167) - [Python] Create chunked BinaryArray in Table.from_pandas when a column's data exceeds 2GB
+* [ARROW-1168](https://issues.apache.org/jira/browse/ARROW-1168) - [Python] pandas metadata may contain "mixed" data types
+* [ARROW-1169](https://issues.apache.org/jira/browse/ARROW-1169) - C++: jemalloc externalproject doesn't build with CMake's ninja generator
+* [ARROW-1170](https://issues.apache.org/jira/browse/ARROW-1170) - C++: ARROW_JEMALLOC=OFF breaks linking on unittest
+* [ARROW-1174](https://issues.apache.org/jira/browse/ARROW-1174) - [GLib] Investigate root cause of ListArray glib test failure
+* [ARROW-1177](https://issues.apache.org/jira/browse/ARROW-1177) - [C++] Detect int32 overflow in ListBuilder::Append
+* [ARROW-1179](https://issues.apache.org/jira/browse/ARROW-1179) - C++: Add missing virtual destructors
+* [ARROW-1180](https://issues.apache.org/jira/browse/ARROW-1180) - [GLib] garrow_tensor_get_dimension_name() returns invalid address
+* [ARROW-1181](https://issues.apache.org/jira/browse/ARROW-1181) - [Python] Parquet test fail if not enabled
+* [ARROW-1182](https://issues.apache.org/jira/browse/ARROW-1182) - C++: Specify BUILD_BYPRODUCTS for zlib and zstd
+* [ARROW-1186](https://issues.apache.org/jira/browse/ARROW-1186) - [C++] Enable option to build arrow with minimal dependencies needed to build Parquet library
+* [ARROW-1188](https://issues.apache.org/jira/browse/ARROW-1188) - Segfault when trying to serialize a DataFrame with Null-only Categorical Column
+* [ARROW-1190](https://issues.apache.org/jira/browse/ARROW-1190) - VectorLoader corrupts vectors with duplicate names
+* [ARROW-1191](https://issues.apache.org/jira/browse/ARROW-1191) - [JAVA] Implement getField() method for the complex readers
+* [ARROW-1194](https://issues.apache.org/jira/browse/ARROW-1194) - Getting record batch size with pa.get_record_batch_size returns a size that is too small for pandas DataFrame.
+* [ARROW-1197](https://issues.apache.org/jira/browse/ARROW-1197) - [GLib] record_batch.hpp Inclusion is missing
+* [ARROW-1200](https://issues.apache.org/jira/browse/ARROW-1200) - [C++] DictionaryBuilder should use signed integers for indices
+* [ARROW-1201](https://issues.apache.org/jira/browse/ARROW-1201) - [Python] Incomplete Python types cause a core dump when repr-ing
+* [ARROW-1203](https://issues.apache.org/jira/browse/ARROW-1203) - [C++] Disallow BinaryBuilder to append byte strings larger than the maximum value of int32_t
+* [ARROW-1205](https://issues.apache.org/jira/browse/ARROW-1205) - C++: Reference to type objects in ArrayLoader may cause segmentation faults.
+* [ARROW-1206](https://issues.apache.org/jira/browse/ARROW-1206) - [C++] Enable MSVC builds to work with some compression library support disabled
+* [ARROW-1208](https://issues.apache.org/jira/browse/ARROW-1208) - [C++] Toolchain build with ZSTD library from conda-forge failure
+* [ARROW-1215](https://issues.apache.org/jira/browse/ARROW-1215) - [Python] Class methods in API reference
+* [ARROW-1216](https://issues.apache.org/jira/browse/ARROW-1216) - Numpy arrays cannot be created from Arrow Buffers on Python 2
+* [ARROW-1218](https://issues.apache.org/jira/browse/ARROW-1218) - Arrow doesn't compile if all compression libraries are deactivated
+* [ARROW-1222](https://issues.apache.org/jira/browse/ARROW-1222) - [Python] pyarrow.array returns NullArray for array of unsupported Python objects
+* [ARROW-1223](https://issues.apache.org/jira/browse/ARROW-1223) - [GLib] Fix function name that returns wrapped object
+* [ARROW-1235](https://issues.apache.org/jira/browse/ARROW-1235) - [C++] macOS linker failure with operator<< and std::ostream
+* [ARROW-1236](https://issues.apache.org/jira/browse/ARROW-1236) - Library paths in exported pkg-config file are incorrect
+* [ARROW-601](https://issues.apache.org/jira/browse/ARROW-601) - Some logical types not supported when loading Parquet
+* [ARROW-784](https://issues.apache.org/jira/browse/ARROW-784) - Cleaning up thirdparty toolchain support in Arrow on Windows
+* [ARROW-992](https://issues.apache.org/jira/browse/ARROW-992) - [Python] In place development builds do not have a __version__
+
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.5.0
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/
+[8]: http://arrow.apache.org/blog/2017/07/25/0.5.0-release/
\ No newline at end of file
diff --git a/site/_release/index.md b/site/_release/index.md
index 2dd657976229d..f18cff3b649e9 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -26,6 +26,7 @@ limitations under the License.
 
 Navigate to the release page for downloads and the changelog.
 
+* [0.5.0 (23 July 2017)][6]
 * [0.4.1 (9 June 2017)][5]
 * [0.4.0 (22 May 2017)][4]
 * [0.3.0 (5 May 2017)][1]
@@ -37,3 +38,4 @@ Navigate to the release page for downloads and the changelog.
 [3]: {{ site.baseurl }}/release/0.1.0.html
 [4]: {{ site.baseurl }}/release/0.4.0.html
 [5]: {{ site.baseurl }}/release/0.4.1.html
+[6]: {{ site.baseurl }}/release/0.5.0.html
diff --git a/site/index.html b/site/index.html
index 5b60a5fc3e2ba..8a06c6acec580 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,14 +7,18 @@ <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
         <p>
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
-          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.4.1 Release - June 9, 2017)</a>
+          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.5.0 Release - July 23, 2017)</a>
         </p>
       </div>
-      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.4.1 release</a></h4>
+      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.5.0 release</a></h4>
       <div class="row">
         <div class="col-lg-4">
           <h2>Fast</h2>
-          <p>Apache Arrow&#8482; enables execution engines to take advantage of the latest SIMD (Single input multiple data) operations included in modern processors, for native vectorized optimization of analytical data processing. Columnar layout of data also allows for a better use of CPU caches by placing all data relevant to a column operation in as compact of a format as possible.</p>
+          <p>Apache Arrow&#8482; enables execution engines to take advantage of the latest SIM
+D (Single input multiple data) operations included in modern processors, for native vectorized optimization of analytical data processing. Columnar layout of data also allows for a better use of CPU caches by placing all data relevant to a column operation in as compact of a format
+ as possible.</p>
+          <p>The Arrow memory format supports <strong>zero-copy reads</strong>
+          for lightning-fast data access without serialization overhead.</p>
         </div>
         <div class="col-lg-4">
           <h2>Flexible</h2>
@@ -26,12 +30,6 @@ <h2>Standard</h2>
         </div>
      </div> <!-- close "row" div -->
 
-<h2>Zero-Copy IPC and Streaming Messaging</h2>
-<div align="left">
-  <h4>Apache Arrow supports zero-copy shared memory IPC and a streaming wire
-  format that fully avoids traditional data serialization costs</h4>
-</div>
-
 <h2>Performance Advantage of Columnar In-Memory</h2>
 <div align="center">
   <img src="img/simd.png" alt="SIMD" style="width:60%" />
diff --git a/site/install.md b/site/install.md
index 4252e7f4bf905..6ad8f7ece7e65 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,36 +20,40 @@ limitations under the License.
 {% endcomment %}
 -->
 
-## Current Version: 0.4.1
+## Current Version: 0.5.0
 
-### Released: 9 June 2017
+### Released: 23 July 2017
 
 See the [release notes][10] and [blog post][11] for more about what's new.
 
 ### Source release
 
-* **Source Release**: [apache-arrow-0.4.1.tar.gz][6]
+* **Source Release**: [apache-arrow-0.5.0.tar.gz][6]
 * **Verification**: [md5][3], [asc][7]
-* [Git tag 46315431][2]
+* [Git tag e9f76e1][2]
 
 ### Java Packages
 
 [Java Artifacts on Maven Central][4]
 
+## Binary Installers for C, C++, Python
+
+It may take a little time for the binary packages to get updated
+
 ### C++ and Python Conda Packages (Unofficial)
 
 We have provided binary conda packages on [conda-forge][5] for the following
 platforms:
 
-* Linux and OS X (Python 2.7, 3.5, and 3.6)
+* Linux and macOS (Python 2.7, 3.5, and 3.6)
 * Windows (Python 3.5 and 3.6)
 
 Install them with:
 
 
 ```shell
-conda install arrow-cpp -c conda-forge
-conda install pyarrow -c conda-forge
+conda install arrow-cpp=0.5.0 -c conda-forge
+conda install pyarrow=0.5.0 -c conda-forge
 ```
 
 ### Python Wheels on PyPI (Unofficial)
@@ -57,7 +61,7 @@ conda install pyarrow -c conda-forge
 We have provided binary wheels on PyPI for Linux, macOS, and Windows:
 
 ```shell
-pip install pyarrow
+pip install pyarrow==0.5.0
 ```
 
 These include the Apache Arrow and Apache Parquet C++ binary libraries bundled
@@ -129,14 +133,14 @@ These repositories are managed at
 [red-data-tools/arrow-packages][9]. If you have any feedback, please
 send it to the project instead of Apache Arrow project.
 
-[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.4.1
-[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/apache-arrow-0.4.1.tar.gz.md5
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.4.1%22
+[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.5.0
+[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/apache-arrow-0.5.0.tar.gz.md5
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.5.0%22
 [5]: http://conda-forge.github.io
-[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/apache-arrow-0.4.1.tar.gz
-[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.1/apache-arrow-0.4.1.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/apache-arrow-0.5.0.tar.gz
+[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/apache-arrow-0.5.0.tar.gz.asc
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
-[10]: http://arrow.apache.org/release/0.4.1.html
-[11]: http://arrow.apache.org/blog/2017/06/14/0.4.1-release/
\ No newline at end of file
+[10]: http://arrow.apache.org/release/0.5.0.html
+[11]: http://arrow.apache.org/blog/2017/07/24/0.5.0-release/
\ No newline at end of file

From ed54dce64a4651a1d26aed373b09b9af9ee2123e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 25 Jul 2017 13:13:56 -0400
Subject: [PATCH 0853/1644] ARROW-1253: [C++/Python] Speed up C++ / Python
 builds by using conda-forge toolchain for thirdparty libraries

@pcmoritz I removed the travis_script_plasma.sh and combined the common bits with travis_script_python.sh; it seemed like there was some redundant work and would be easier to test everything in a single build. Let me know if I messed something up

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #884 from wesm/ARROW-1253 and squashes the following commits:

30a13ee3 [Wes McKinney] Formatting
28bc6be8 [Wes McKinney] Run unit tests from installed library
fcf7940d [Wes McKinney] Prevent existing LD_LIBRARY_PATH from being blown away
c67bb68b [Wes McKinney] Speed up C++ / Python builds by using conda-forge toolchain to avoid rebuilding EPs. Use ninja for builds when using toolchain
---
 .travis.yml                    | 27 ++--------
 ci/travis_before_script_cpp.sh | 50 +++++++++++++-----
 ci/travis_env_common.sh        | 13 +++++
 ci/travis_script_cpp.sh        |  4 +-
 ci/travis_script_plasma.sh     | 97 ----------------------------------
 ci/travis_script_python.sh     | 72 ++++++++++++-------------
 cpp/.gitignore                 | 17 ++++++
 cpp/CMakeLists.txt             | 28 +++++-----
 cpp/src/plasma/.gitignore      | 18 +++++++
 python/doc/source/api.rst      |  1 -
 python/setup.py                |  6 +--
 11 files changed, 141 insertions(+), 192 deletions(-)
 delete mode 100755 ci/travis_script_plasma.sh
 create mode 100644 cpp/src/plasma/.gitignore

diff --git a/.travis.yml b/.travis.yml
index 9cc2b86c05cde..b93f1c2519b9b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,7 +42,6 @@ cache:
   ccache: true
   directories:
     - $HOME/.conda_packages
-    - $HOME/.ccache
 
 matrix:
   fast_finish: true
@@ -56,6 +55,9 @@ matrix:
     before_script:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
+    - export ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - export ARROW_TRAVIS_VALGRIND=1
+    - export ARROW_TRAVIS_PLASMA=1
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
@@ -66,6 +68,8 @@ matrix:
     cache:
     addons:
     before_script:
+    - export ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - export ARROW_TRAVIS_PLASMA=1
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
@@ -120,27 +124,6 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
-  - compiler: gcc
-    language: cpp
-    os: linux
-    group: deprecated
-    before_script:
-    - export CC="gcc-4.9"
-    - export CXX="g++-4.9"
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh
-  - compiler: clang
-    osx_image: xcode6.4
-    os: osx
-    cache:
-    addons:
-    before_script:
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh
 
 before_install:
 - ulimit -c unlimited -S
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index e250e705f1fbd..a1697dae59fbf 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -15,22 +15,33 @@
 
 set -ex
 
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
 if [ "$1" == "--only-library" ]; then
   only_library_mode=yes
 else
   only_library_mode=no
+  source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 fi
 
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-
-if [ $only_library_mode == "no" ]; then
-  # C++ toolchain
-  export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
-  export RAPIDJSON_HOME=$CPP_TOOLCHAIN
-
+if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
   # Set up C++ toolchain from conda-forge packages for faster builds
-  source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-  conda create -y -q -p $CPP_TOOLCHAIN python=2.7 rapidjson
+  conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
+        jemalloc=4.4.0 \
+        nomkl \
+        boost-cpp \
+        rapidjson \
+        flatbuffers \
+        gflags \
+        lz4-c \
+        snappy \
+        zstd \
+        brotli \
+        zlib \
+        cmake \
+        curl \
+        thrift-cpp \
+        ninja
 fi
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
@@ -45,7 +56,6 @@ pushd $ARROW_CPP_BUILD_DIR
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
 -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \
--DARROW_PLASMA=ON \
 -DARROW_NO_DEPRECATED_API=ON"
 CMAKE_LINUX_FLAGS=""
 CMAKE_OSX_FLAGS=""
@@ -60,8 +70,20 @@ else
   # also in the manylinux1 image.
   CMAKE_LINUX_FLAGS="\
 $CMAKE_LINUX_FLAGS \
--DARROW_JEMALLOC=ON \
--DARROW_TEST_MEMCHECK=ON"
+-DARROW_JEMALLOC=ON"
+fi
+
+# Use Ninja for faster builds when using toolchain
+if [ $ARROW_TRAVIS_USE_TOOLCHAIN == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja"
+fi
+
+if [ $ARROW_TRAVIS_PLASMA == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON"
+fi
+
+if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON"
 fi
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
@@ -76,7 +98,7 @@ else
           $ARROW_CPP_DIR
 fi
 
-make VERBOSE=1 -j4
-make install
+$TRAVIS_MAKE -j4
+$TRAVIS_MAKE install
 
 popd
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index a2e591014cfd3..4122eacaa2ee3 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -29,6 +29,19 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
 export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
 export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
 
+if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
+  # C++ toolchain
+  export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+  export ARROW_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+  export BOOST_ROOT=$CPP_TOOLCHAIN
+
+  export PATH=$CPP_TOOLCHAIN/bin:$PATH
+  export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
+  export TRAVIS_MAKE=ninja
+else
+  export TRAVIS_MAKE=make
+fi
+
 if [ $TRAVIS_OS_NAME == "osx" ]; then
   export GOPATH=$TRAVIS_BUILD_DIR/gopath
 fi
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index c368a1daeddb9..a4f6b056d9de4 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -14,7 +14,7 @@
 
 set -e
 
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 # Check licenses according to Apache policy
 git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
@@ -22,7 +22,7 @@ git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
 
 pushd $CPP_BUILD_DIR
 
-make lint
+$TRAVIS_MAKE lint
 
 # ARROW-209: checks depending on the LLVM toolchain are disabled temporarily
 # until we are able to install the full LLVM toolchain in Travis CI again
diff --git a/ci/travis_script_plasma.sh b/ci/travis_script_plasma.sh
deleted file mode 100755
index fa384ade89c2f..0000000000000
--- a/ci/travis_script_plasma.sh
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/env bash
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
-set -e
-
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-
-export ARROW_HOME=$ARROW_CPP_INSTALL
-export PYARROW_WITH_PLASMA=1
-
-pushd $ARROW_PYTHON_DIR
-
-function build_arrow_libraries() {
-  CPP_BUILD_DIR=$1
-  CPP_DIR=$TRAVIS_BUILD_DIR/cpp
-
-  mkdir $CPP_BUILD_DIR
-  pushd $CPP_BUILD_DIR
-
-  cmake -DARROW_BUILD_TESTS=off \
-        -DARROW_PYTHON=on \
-        -DARROW_PLASMA=on \
-        -DCMAKE_INSTALL_PREFIX=$2 \
-        $CPP_DIR
-
-  make -j4
-  make install
-
-  popd
-}
-
-python_version_tests() {
-  PYTHON_VERSION=$1
-  CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
-
-  export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION
-  export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib
-
-  conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
-  source activate $CONDA_ENV_DIR
-
-  python --version
-  which python
-
-  # faster builds, please
-  conda install -y -q nomkl
-
-  # Expensive dependencies install from Continuum package repo
-  conda install -y -q pip numpy pandas cython
-
-  # Build C++ libraries
-  build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME
-
-  # Other stuff pip install
-  pip install -r requirements.txt
-
-  python setup.py build_ext --inplace
-
-  python -m pytest -vv -r sxX pyarrow
-
-  # Build documentation once
-  if [[ "$PYTHON_VERSION" == "3.6" ]]
-  then
-      conda install -y -q --file=doc/requirements.txt
-      python setup.py build_sphinx -s doc/source
-  fi
-
-  # Build and install pyarrow
-  pushd $TRAVIS_BUILD_DIR/python
-    python setup.py install
-  popd
-
-  # Run Plasma tests
-  pushd $TRAVIS_BUILD_DIR/python
-    python -m pytest pyarrow/tests/test_plasma.py
-    if [ $TRAVIS_OS_NAME == "linux" ]; then
-      PLASMA_VALGRIND=1 python -m pytest pyarrow/tests/test_plasma.py
-    fi
-  popd
-}
-
-# run tests for python 2.7 and 3.6
-python_version_tests 2.7
-python_version_tests 3.6
-
-popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index fdb5ad6a62c93..dcba78273caa7 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -17,26 +16,14 @@ set -e
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 export ARROW_HOME=$ARROW_CPP_INSTALL
-export PYARROW_WITH_PLASMA=1
-
-pushd $ARROW_PYTHON_DIR
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
+export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH
 
 build_parquet_cpp() {
   export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
-  conda create -y -q -p $PARQUET_HOME python=3.6 cmake curl
-  source activate $PARQUET_HOME
-
-  # In case some package wants to download the MKL
-  conda install -y -q nomkl
-
-  conda install -y -q thrift-cpp snappy zlib brotli boost
 
-  export BOOST_ROOT=$PARQUET_HOME
-  export SNAPPY_HOME=$PARQUET_HOME
-  export THRIFT_HOME=$PARQUET_HOME
-  export ZLIB_HOME=$PARQUET_HOME
-  export BROTLI_HOME=$PARQUET_HOME
+  # $CPP_TOOLCHAIN set up in before_script_cpp
+  export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
 
   PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
   mkdir -p $PARQUET_DIR
@@ -48,37 +35,39 @@ build_parquet_cpp() {
   cd build-dir
 
   cmake \
+      -GNinja \
       -DCMAKE_BUILD_TYPE=debug \
       -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+      -DPARQUET_BOOST_USE_SHARED=off \
       -DPARQUET_BUILD_BENCHMARKS=off \
       -DPARQUET_BUILD_EXECUTABLES=off \
-      -DPARQUET_ZLIB_VENDORED=off \
-      -DPARQUET_BUILD_TESTS=on \
+      -DPARQUET_BUILD_TESTS=off \
       ..
 
-  make -j${CPU_COUNT}
-  make install
+  ninja
+  ninja install
 
   popd
 }
 
 build_parquet_cpp
 
-function build_arrow_libraries() {
-  CPP_BUILD_DIR=$1
-  CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+function rebuild_arrow_libraries() {
+  pushd $ARROW_CPP_BUILD_DIR
 
-  mkdir $CPP_BUILD_DIR
-  pushd $CPP_BUILD_DIR
+  # Clear out prior build files
+  rm -rf *
 
-  cmake -DARROW_BUILD_TESTS=off \
+  cmake -GNinja \
+        -DARROW_BUILD_TESTS=off \
+        -DARROW_BUILD_UTILITIES=off \
         -DARROW_PLASMA=on \
         -DARROW_PYTHON=on \
-        -DCMAKE_INSTALL_PREFIX=$2 \
-        $CPP_DIR
+        -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+        $ARROW_CPP_DIR
 
-  make -j4
-  make install
+  ninja
+  ninja install
 
   popd
 }
@@ -87,9 +76,6 @@ python_version_tests() {
   PYTHON_VERSION=$1
   CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
 
-  export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION
-  export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib
-
   conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
   source activate $CONDA_ENV_DIR
 
@@ -103,27 +89,35 @@ python_version_tests() {
   conda install -y -q pip numpy pandas cython
 
   # Build C++ libraries
-  build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME
+  rebuild_arrow_libraries
 
   # Other stuff pip install
+  pushd $ARROW_PYTHON_DIR
   pip install -r requirements.txt
-
-  python setup.py build_ext --inplace --with-parquet
+  python setup.py build_ext --with-parquet --with-plasma \
+         install --single-version-externally-managed --record=record.text
+  popd
 
   python -c "import pyarrow.parquet"
+  python -c "import pyarrow.plasma"
+
+  if [ $TRAVIS_OS_NAME == "linux" ]; then
+    export PLASMA_VALGRIND=1
+  fi
 
-  python -m pytest -vv -r sxX pyarrow --parquet
+  PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
+  python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
 
+  pushd $ARROW_PYTHON_DIR
   # Build documentation once
   if [[ "$PYTHON_VERSION" == "3.6" ]]
   then
       conda install -y -q --file=doc/requirements.txt
       python setup.py build_sphinx -s doc/source
   fi
+  popd
 }
 
 # run tests for python 2.7 and 3.6
 python_version_tests 2.7
 python_version_tests 3.6
-
-popd
diff --git a/cpp/.gitignore b/cpp/.gitignore
index 4910544ec87cd..ec846b35ba61a 100644
--- a/cpp/.gitignore
+++ b/cpp/.gitignore
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 thirdparty/
 CMakeFiles/
 CMakeCache.txt
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 2891a5d76186c..41af1e16ce28d 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -582,20 +582,6 @@ if (ARROW_STATIC_LINK_LIBS)
   add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
 endif()
 
-set(ARROW_MIN_TEST_LIBS
-  arrow_static
-  ${ARROW_STATIC_LINK_LIBS}
-  gtest
-  gtest_main)
-
-if(NOT MSVC)
-  set(ARROW_MIN_TEST_LIBS
-    ${ARROW_MIN_TEST_LIBS}
-    ${CMAKE_DL_LIBS})
-endif()
-
-set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
-
 set(ARROW_BENCHMARK_LINK_LIBS
   arrow_static
   arrow_benchmark_main
@@ -618,6 +604,20 @@ if (NOT MSVC)
     ${CMAKE_DL_LIBS})
 endif()
 
+set(ARROW_MIN_TEST_LIBS
+  arrow_static
+  ${ARROW_STATIC_LINK_LIBS}
+  gtest
+  gtest_main)
+
+if(NOT MSVC)
+  set(ARROW_MIN_TEST_LIBS
+    ${ARROW_MIN_TEST_LIBS}
+    ${CMAKE_DL_LIBS})
+endif()
+
+set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
+
 if (ARROW_JEMALLOC)
   add_definitions(-DARROW_JEMALLOC)
   # In the case that jemalloc is only available as a shared library also use it to
diff --git a/cpp/src/plasma/.gitignore b/cpp/src/plasma/.gitignore
new file mode 100644
index 0000000000000..163b5c56e91a2
--- /dev/null
+++ b/cpp/src/plasma/.gitignore
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+*_generated.h
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 780aa4839610f..6554465466a55 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -225,7 +225,6 @@ In-Memory Object Store
    ObjectID
    PlasmaClient
    PlasmaBuffer
-   MutablePlasmaBuffer
 
 .. currentmodule:: pyarrow.parquet
 
diff --git a/python/setup.py b/python/setup.py
index 7425b71916001..be0e26b92a876 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -82,6 +82,7 @@ def run(self):
     user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
                      ('build-type=', None, 'build type (debug or release)'),
                      ('with-parquet', None, 'build the Parquet extension'),
+                     ('with-plasma', None, 'build the Plasma extension'),
                      ('bundle-arrow-cpp', None,
                       'bundle the Arrow C++ libraries')] +
                     _build_ext.user_options)
@@ -101,14 +102,13 @@ def initialize_options(self):
             os.environ.get('PYARROW_WITH_PARQUET', '0'))
         self.with_plasma = strtobool(
             os.environ.get('PYARROW_WITH_PLASMA', '0'))
-        if self.with_plasma and "plasma" not in self.CYTHON_MODULE_NAMES:
-            self.CYTHON_MODULE_NAMES.append("plasma")
         self.bundle_arrow_cpp = strtobool(
             os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
 
     CYTHON_MODULE_NAMES = [
         'lib',
-        '_parquet']
+        '_parquet',
+        'plasma']
 
     def _run_cmake(self):
         # The directory containing this setup.py

From f90fa4971813e2d96b9be74d3874da6d7f0508df Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 25 Jul 2017 13:39:22 -0400
Subject: [PATCH 0854/1644] [Website] Fix link to 0.5.0 post on install page

Change-Id: I290f70f8db4b553017e7976092ee5cefdbb14861
---
 site/_release/0.5.0.md | 2 +-
 site/install.md        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/site/_release/0.5.0.md b/site/_release/0.5.0.md
index 3073f8c526fca..f28d86690f38f 100644
--- a/site/_release/0.5.0.md
+++ b/site/_release/0.5.0.md
@@ -200,4 +200,4 @@ $ git shortlog -sn apache-arrow-0.4.1..apache-arrow-0.5.0
 
 [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.5.0
 [6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/
-[8]: http://arrow.apache.org/blog/2017/07/25/0.5.0-release/
\ No newline at end of file
+[8]: http://arrow.apache.org/blog/2017/07/25/0.5.0-release/
diff --git a/site/install.md b/site/install.md
index 6ad8f7ece7e65..bd45642fe201f 100644
--- a/site/install.md
+++ b/site/install.md
@@ -143,4 +143,4 @@ send it to the project instead of Apache Arrow project.
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
 [10]: http://arrow.apache.org/release/0.5.0.html
-[11]: http://arrow.apache.org/blog/2017/07/24/0.5.0-release/
\ No newline at end of file
+[11]: http://arrow.apache.org/blog/2017/07/25/0.5.0-release/

From e9e17b56aede6987aba84a414629b7aeb1ab1403 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 25 Jul 2017 21:46:45 -0400
Subject: [PATCH 0855/1644] ARROW-1258: [C++] Suppress Clang dlmalloc compiler
 warnings

cc @pcmoritz

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #889 from wesm/ARROW-1258 and squashes the following commits:

e9ecf109 [Wes McKinney] Set -Wno-conversion for dlmalloc on gcc builds
3a4ee476 [Wes McKinney] Suppress Clang dlmalloc compiler warnings
---
 cpp/src/plasma/CMakeLists.txt        | 17 ++++++++++++++++-
 cpp/src/plasma/thirdparty/dlmalloc.c | 19 +++++++++++++++++--
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 8bb7e71fdf11b..7e91202623e24 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -87,7 +87,22 @@ ADD_ARROW_LIB(plasma
 
 # The optimization flag -O3 is suggested by dlmalloc.c, which is #included in
 # malloc.cc; we set it here regardless of whether we do a debug or release build.
-set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error -O3")
+set_source_files_properties(malloc.cc PROPERTIES
+  COMPILE_FLAGS "-O3")
+
+if ("${COMPILER_FAMILY}" STREQUAL "clang")
+  set_property(SOURCE malloc.cc
+    APPEND_STRING
+    PROPERTY COMPILE_FLAGS
+    " -Wno-parentheses-equality -Wno-shorten-64-to-32")
+endif()
+
+if ("${COMPILER_FAMILY}" STREQUAL "gcc")
+  set_property(SOURCE malloc.cc
+    APPEND_STRING
+    PROPERTY COMPILE_FLAGS
+    " -Wno-conversion")
+endif()
 
 add_executable(plasma_store store.cc)
 target_link_libraries(plasma_store plasma_static)
diff --git a/cpp/src/plasma/thirdparty/dlmalloc.c b/cpp/src/plasma/thirdparty/dlmalloc.c
index 84ccbd28fc4ec..7f3fd63964989 100644
--- a/cpp/src/plasma/thirdparty/dlmalloc.c
+++ b/cpp/src/plasma/thirdparty/dlmalloc.c
@@ -521,6 +521,7 @@ MAX_RELEASE_CHECK_RATE   default: 4095 unless not HAVE_MMAP
   improvement at the expense of carrying around more memory.
 */
 
+
 /* Version identifier to allow people to support multiple versions */
 #ifndef DLMALLOC_VERSION
 #define DLMALLOC_VERSION 20806
@@ -584,9 +585,21 @@ MAX_RELEASE_CHECK_RATE   default: 4095 unless not HAVE_MMAP
 /* The maximum possible size_t value has all bits set */
 #define MAX_SIZE_T           (~(size_t)0)
 
+#if (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0)
+#define RECURSIVE_LOCKS_ENABLED 1
+#else
+#define RECURSIVE_LOCKS_ENABLED 0
+#endif
+
+#if (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0)
+#define SPIN_LOCKS_ENABLED 1
+#else
+#define SPIN_LOCKS_ENABLED 0
+#endif
+
 #ifndef USE_LOCKS /* ensure true if spin or recursive locks set */
-#define USE_LOCKS  ((defined(USE_SPIN_LOCKS) && USE_SPIN_LOCKS != 0) || \
-                    (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0))
+#define USE_LOCKS  ((SPIN_LOCKS_ENABLED != 0) || \
+                    (RECURSIVE_LOCKS_ENABLED != 0))
 #endif /* USE_LOCKS */
 
 #if USE_LOCKS /* Spin locks for gcc >= 4.1, older gcc on x86, MSC >= 1310 */
@@ -645,7 +658,9 @@ MAX_RELEASE_CHECK_RATE   default: 4095 unless not HAVE_MMAP
 #ifndef HAVE_MREMAP
 #ifdef linux
 #define HAVE_MREMAP 1
+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE /* Turns on mremap() definition */
+#endif  /* _GNU_SOURCE */
 #else   /* linux */
 #define HAVE_MREMAP 0
 #endif  /* linux */

From 2eeaa95d492e2e1f9400a8bf420f4db9fb4f24e0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 25 Jul 2017 21:48:18 -0400
Subject: [PATCH 0856/1644] ARROW-1248: [Python] Suppress return-type-c-linkage
 warning in Cython clang builds

I also removed some other unused CMake cruft from the build system. I think the warning is innocuous, but these symbols in question will only be callable from C++:

```
$ nm -g pyarrow/lib.cpython-35m-x86_64-linux-gnu.so | grep unwrap
0000000000055ef0 T pyarrow_unwrap_array
0000000000058590 T pyarrow_unwrap_batch
0000000000054690 T pyarrow_unwrap_buffer
0000000000057d90 T pyarrow_unwrap_column
0000000000054df0 T pyarrow_unwrap_data_type
0000000000055640 T pyarrow_unwrap_field
0000000000055af0 T pyarrow_unwrap_schema
0000000000058190 T pyarrow_unwrap_table
0000000000057880 T pyarrow_unwrap_tensor
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #888 from wesm/ARROW-1248 and squashes the following commits:

986cf889 [Wes McKinney] Suppress return-type-c-linkage warning in Cython clang builds. Remove other python CMake cruft
---
 python/CMakeLists.txt | 75 ++++---------------------------------------
 1 file changed, 6 insertions(+), 69 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6ff66462958ef..71ce163e4949f 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -90,80 +90,17 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
 
   # Cython warnings in clang
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality -Wno-constant-logical-operand")
-endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-constant-logical-operand")
 
-set(PYARROW_LINK "a")
+  # We have public Cython APIs which return C++ types, which are in an extern
+  # "C" blog (no symbol mangling) and clang doesn't like this
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type-c-linkage")
+endif()
 
 # For any C code, use the same flags.
 set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
 
-# Code coverage
-if ("${PYARROW_GENERATE_COVERAGE}")
-  if("${CMAKE_CXX_COMPILER}" MATCHES ".*clang.*")
-    # There appears to be some bugs in clang 3.3 which cause code coverage
-    # to have link errors, not locating the llvm_gcda_* symbols.
-    # This should be fixed in llvm 3.4 with http://llvm.org/viewvc/llvm-project?view=revision&revision=184666
-    message(SEND_ERROR "Cannot currently generate coverage with clang")
-  endif()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -DCOVERAGE_BUILD")
-
-  # For coverage to work properly, we need to use static linkage. Otherwise,
-  # __gcov_flush() doesn't properly flush coverage from every module.
-  # See http://stackoverflow.com/questions/28164543/using-gcov-flush-within-a-library-doesnt-force-the-other-modules-to-yield-gc
-  if("${PYARROW_LINK}" STREQUAL "a")
-    message("Using static linking for coverage build")
-    set(PYARROW_LINK "s")
-  elseif("${PYARROW_LINK}" STREQUAL "d")
-    message(SEND_ERROR "Cannot use coverage with static linking")
-  endif()
-endif()
-
-# If we still don't know what kind of linking to perform, choose based on
-# build type (developers like fast builds).
-if ("${PYARROW_LINK}" STREQUAL "a")
-  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
-      "${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
-    message("Using dynamic linking for ${CMAKE_BUILD_TYPE} builds")
-    set(PYARROW_LINK "d")
-  else()
-    message("Using static linking for ${CMAKE_BUILD_TYPE} builds")
-    set(PYARROW_LINK "s")
-  endif()
-endif()
-
-# Are we using the gold linker? It doesn't work with dynamic linking as
-# weak symbols aren't properly overridden, causing tcmalloc to be omitted.
-# Let's flag this as an error in RELEASE builds (we shouldn't release a
-# product like this).
-#
-# See https://sourceware.org/bugzilla/show_bug.cgi?id=16979 for details.
-#
-# The gold linker is only for ELF binaries, which OSX doesn't use. We can
-# just skip.
-if (NOT APPLE AND NOT MSVC)
-  execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Wl,--version OUTPUT_VARIABLE LINKER_OUTPUT)
-endif ()
-
-if (LINKER_OUTPUT MATCHES "gold")
-  if ("${PYARROW_LINK}" STREQUAL "d" AND
-      "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
-    message(SEND_ERROR "Cannot use gold with dynamic linking in a RELEASE build "
-      "as it would cause tcmalloc symbols to get dropped")
-  else()
-    message("Using gold linker")
-  endif()
-  set(PYARROW_USING_GOLD 1)
-else()
-  message("Using ld linker")
-endif()
-
-# Having set PYARROW_LINK due to build type and/or sanitizer, it's now safe to
-# act on its value.
-if ("${PYARROW_LINK}" STREQUAL "d")
-  set(BUILD_SHARED_LIBS ON)
-endif()
-
 # set compile output directory
 string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
 

From 676a4a9476b042bd9c7e801c083a49357ff4d4ac Mon Sep 17 00:00:00 2001
From: Yeolar <yeolar@gmail.com>
Date: Tue, 25 Jul 2017 23:40:21 -0400
Subject: [PATCH 0857/1644] ARROW-1255: [Plasma] Fix typo in plasma protocol;
 add DCHECK for ReadXXX in plasma protocol.

Related to https://github.com/apache/arrow/pull/878, add DCHECK for ReadXXX.

Author: Yeolar <yeolar@gmail.com>

Closes #887 from Yeolar/fixtypo_plasma_and_add_DCHECK and squashes the following commits:

4df63bc [Yeolar] clang-format for too long lines.
143d254 [Yeolar] Update, compile passed.
09ff103 [Yeolar] Fix conflicts.
b951d8d [Yeolar] Merge pull request #1 from apache/master
ebae611 [Yeolar] Fix typo in plasma protocol; add DCHECK for ReadXXX in plasma protocol.
---
 cpp/src/plasma/client.cc                   | 16 +++--
 cpp/src/plasma/protocol.cc                 | 84 ++++++++++++++--------
 cpp/src/plasma/protocol.h                  | 67 +++++++++--------
 cpp/src/plasma/store.cc                    | 19 ++---
 cpp/src/plasma/test/serialization_tests.cc | 53 +++++++-------
 5 files changed, 143 insertions(+), 96 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index bbbeb55813ccf..e14b3d9a46c82 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -152,7 +152,7 @@ Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
   RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaCreateReply, &buffer));
   ObjectID id;
   PlasmaObject object;
-  RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object));
+  RETURN_NOT_OK(ReadCreateReply(buffer.data(), buffer.size(), &id, &object));
   // If the CreateReply included an error, then the store will not send a file
   // descriptor.
   int fd = recv_fd(store_conn_);
@@ -227,7 +227,7 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
   std::vector<ObjectID> received_object_ids(num_objects);
   std::vector<PlasmaObject> object_data(num_objects);
   PlasmaObject* object;
-  RETURN_NOT_OK(ReadGetReply(buffer.data(), received_object_ids.data(),
+  RETURN_NOT_OK(ReadGetReply(buffer.data(), buffer.size(), received_object_ids.data(),
                              object_data.data(), num_objects));
 
   for (int i = 0; i < num_objects; ++i) {
@@ -356,7 +356,8 @@ Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) {
     std::vector<uint8_t> buffer;
     RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaContainsReply, &buffer));
     ObjectID object_id2;
-    RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object));
+    RETURN_NOT_OK(
+        ReadContainsReply(buffer.data(), buffer.size(), &object_id2, has_object));
   }
   return Status::OK();
 }
@@ -451,7 +452,7 @@ Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) {
   std::vector<uint8_t> buffer;
   int64_t type;
   RETURN_NOT_OK(ReadMessage(store_conn_, &type, &buffer));
-  return ReadEvictReply(buffer.data(), num_bytes_evicted);
+  return ReadEvictReply(buffer.data(), buffer.size(), num_bytes_evicted);
 }
 
 Status PlasmaClient::Hash(const ObjectID& object_id, uint8_t* digest) {
@@ -524,7 +525,7 @@ Status PlasmaClient::Connect(const std::string& store_socket_name,
   RETURN_NOT_OK(SendConnectRequest(store_conn_));
   std::vector<uint8_t> buffer;
   RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaConnectReply, &buffer));
-  RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity_));
+  RETURN_NOT_OK(ReadConnectReply(buffer.data(), buffer.size(), &store_capacity_));
   return Status::OK();
 }
 
@@ -564,7 +565,7 @@ Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
   std::vector<uint8_t> buffer;
   RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaStatusReply, &buffer));
   ObjectID id;
-  RETURN_NOT_OK(ReadStatusReply(buffer.data(), &id, object_status, 1));
+  RETURN_NOT_OK(ReadStatusReply(buffer.data(), buffer.size(), &id, object_status, 1));
   ARROW_CHECK(object_id == id);
   return Status::OK();
 }
@@ -586,7 +587,8 @@ Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_req
                                 num_ready_objects, timeout_ms));
   std::vector<uint8_t> buffer;
   RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaWaitReply, &buffer));
-  RETURN_NOT_OK(ReadWaitReply(buffer.data(), object_requests, &num_ready_objects));
+  RETURN_NOT_OK(
+      ReadWaitReply(buffer.data(), buffer.size(), object_requests, &num_ready_objects));
 
   *num_objects_ready = 0;
   for (int i = 0; i < num_object_requests; ++i) {
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 19240bb4b8a1f..77bc8b7aae303 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -62,10 +62,11 @@ Status SendCreateRequest(int sock, ObjectID object_id, int64_t data_size,
   return PlasmaSend(sock, MessageType_PlasmaCreateRequest, &fbb, message);
 }
 
-Status ReadCreateRequest(uint8_t* data, ObjectID* object_id, int64_t* data_size,
-                         int64_t* metadata_size) {
+Status ReadCreateRequest(uint8_t* data, size_t size, ObjectID* object_id,
+                         int64_t* data_size, int64_t* metadata_size) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *data_size = message->data_size();
   *metadata_size = message->metadata_size();
   *object_id = ObjectID::from_binary(message->object_id()->str());
@@ -83,9 +84,11 @@ Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object,
   return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
 }
 
-Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object) {
+Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id,
+                       PlasmaObject* object) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   object->handle.store_fd = message->plasma_object()->segment_index();
   object->handle.mmap_size = message->plasma_object()->mmap_size();
@@ -106,9 +109,11 @@ Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest) {
   return PlasmaSend(sock, MessageType_PlasmaSealRequest, &fbb, message);
 }
 
-Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest) {
+Status ReadSealRequest(uint8_t* data, size_t size, ObjectID* object_id,
+                       unsigned char* digest) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaSealRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   ARROW_CHECK(message->digest()->size() == kDigestSize);
   memcpy(digest, message->digest()->data(), kDigestSize);
@@ -122,9 +127,10 @@ Status SendSealReply(int sock, ObjectID object_id, int error) {
   return PlasmaSend(sock, MessageType_PlasmaSealReply, &fbb, message);
 }
 
-Status ReadSealReply(uint8_t* data, ObjectID* object_id) {
+Status ReadSealReply(uint8_t* data, size_t size, ObjectID* object_id) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaSealReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   return plasma_error_status(message->error());
 }
@@ -133,13 +139,14 @@ Status ReadSealReply(uint8_t* data, ObjectID* object_id) {
 
 Status SendReleaseRequest(int sock, ObjectID object_id) {
   flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()));
+  auto message = CreatePlasmaReleaseRequest(fbb, fbb.CreateString(object_id.binary()));
   return PlasmaSend(sock, MessageType_PlasmaReleaseRequest, &fbb, message);
 }
 
-Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id) {
+Status ReadReleaseRequest(uint8_t* data, size_t size, ObjectID* object_id) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaReleaseRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   return Status::OK();
 }
@@ -151,9 +158,10 @@ Status SendReleaseReply(int sock, ObjectID object_id, int error) {
   return PlasmaSend(sock, MessageType_PlasmaReleaseReply, &fbb, message);
 }
 
-Status ReadReleaseReply(uint8_t* data, ObjectID* object_id) {
+Status ReadReleaseReply(uint8_t* data, size_t size, ObjectID* object_id) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   return plasma_error_status(message->error());
 }
@@ -166,9 +174,10 @@ Status SendDeleteRequest(int sock, ObjectID object_id) {
   return PlasmaSend(sock, MessageType_PlasmaDeleteRequest, &fbb, message);
 }
 
-Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id) {
+Status ReadDeleteRequest(uint8_t* data, size_t size, ObjectID* object_id) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   return Status::OK();
 }
@@ -180,9 +189,10 @@ Status SendDeleteReply(int sock, ObjectID object_id, int error) {
   return PlasmaSend(sock, MessageType_PlasmaDeleteReply, &fbb, message);
 }
 
-Status ReadDeleteReply(uint8_t* data, ObjectID* object_id) {
+Status ReadDeleteReply(uint8_t* data, size_t size, ObjectID* object_id) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaDeleteReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   return plasma_error_status(message->error());
 }
@@ -196,9 +206,11 @@ Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objec
   return PlasmaSend(sock, MessageType_PlasmaStatusRequest, &fbb, message);
 }
 
-Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects) {
+Status ReadStatusRequest(uint8_t* data, size_t size, ObjectID object_ids[],
+                         int64_t num_objects) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaStatusRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   for (uoffset_t i = 0; i < num_objects; ++i) {
     object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
   }
@@ -214,16 +226,18 @@ Status SendStatusReply(int sock, ObjectID object_ids[], int object_status[],
   return PlasmaSend(sock, MessageType_PlasmaStatusReply, &fbb, message);
 }
 
-int64_t ReadStatusReply_num_objects(uint8_t* data) {
+int64_t ReadStatusReply_num_objects(uint8_t* data, size_t size) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   return message->object_ids()->size();
 }
 
-Status ReadStatusReply(uint8_t* data, ObjectID object_ids[], int object_status[],
-                       int64_t num_objects) {
+Status ReadStatusReply(uint8_t* data, size_t size, ObjectID object_ids[],
+                       int object_status[], int64_t num_objects) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   for (uoffset_t i = 0; i < num_objects; ++i) {
     object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
   }
@@ -241,9 +255,10 @@ Status SendContainsRequest(int sock, ObjectID object_id) {
   return PlasmaSend(sock, MessageType_PlasmaContainsRequest, &fbb, message);
 }
 
-Status ReadContainsRequest(uint8_t* data, ObjectID* object_id) {
+Status ReadContainsRequest(uint8_t* data, size_t size, ObjectID* object_id) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaContainsRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   return Status::OK();
 }
@@ -255,9 +270,11 @@ Status SendContainsReply(int sock, ObjectID object_id, bool has_object) {
   return PlasmaSend(sock, MessageType_PlasmaContainsReply, &fbb, message);
 }
 
-Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object) {
+Status ReadContainsReply(uint8_t* data, size_t size, ObjectID* object_id,
+                         bool* has_object) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaContainsReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   *has_object = message->has_object();
   return Status::OK();
@@ -279,9 +296,10 @@ Status SendConnectReply(int sock, int64_t memory_capacity) {
   return PlasmaSend(sock, MessageType_PlasmaConnectReply, &fbb, message);
 }
 
-Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity) {
+Status ReadConnectReply(uint8_t* data, size_t size, int64_t* memory_capacity) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaConnectReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *memory_capacity = message->memory_capacity();
   return Status::OK();
 }
@@ -294,9 +312,10 @@ Status SendEvictRequest(int sock, int64_t num_bytes) {
   return PlasmaSend(sock, MessageType_PlasmaEvictRequest, &fbb, message);
 }
 
-Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes) {
+Status ReadEvictRequest(uint8_t* data, size_t size, int64_t* num_bytes) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaEvictRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *num_bytes = message->num_bytes();
   return Status::OK();
 }
@@ -307,9 +326,10 @@ Status SendEvictReply(int sock, int64_t num_bytes) {
   return PlasmaSend(sock, MessageType_PlasmaEvictReply, &fbb, message);
 }
 
-Status ReadEvictReply(uint8_t* data, int64_t& num_bytes) {
+Status ReadEvictReply(uint8_t* data, size_t size, int64_t& num_bytes) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaEvictReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   num_bytes = message->num_bytes();
   return Status::OK();
 }
@@ -324,10 +344,11 @@ Status SendGetRequest(int sock, const ObjectID* object_ids, int64_t num_objects,
   return PlasmaSend(sock, MessageType_PlasmaGetRequest, &fbb, message);
 }
 
-Status ReadGetRequest(uint8_t* data, std::vector<ObjectID>& object_ids,
+Status ReadGetRequest(uint8_t* data, size_t size, std::vector<ObjectID>& object_ids,
                       int64_t* timeout_ms) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
     auto object_id = message->object_ids()->Get(i)->str();
     object_ids.push_back(ObjectID::from_binary(object_id));
@@ -355,10 +376,11 @@ Status SendGetReply(
   return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
 }
 
-Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
-                    int64_t num_objects) {
+Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
+                    PlasmaObject plasma_objects[], int64_t num_objects) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   for (uoffset_t i = 0; i < num_objects; ++i) {
     object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
   }
@@ -383,9 +405,10 @@ Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_object
   return PlasmaSend(sock, MessageType_PlasmaFetchRequest, &fbb, message);
 }
 
-Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids) {
+Status ReadFetchRequest(uint8_t* data, size_t size, std::vector<ObjectID>& object_ids) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaFetchRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
     object_ids.push_back(ObjectID::from_binary(message->object_ids()->Get(i)->str()));
   }
@@ -410,10 +433,11 @@ Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_re
   return PlasmaSend(sock, MessageType_PlasmaWaitRequest, &fbb, message);
 }
 
-Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
+Status ReadWaitRequest(uint8_t* data, size_t size, ObjectRequestMap& object_requests,
                        int64_t* timeout_ms, int* num_ready_objects) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaWaitRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *num_ready_objects = message->num_ready_objects();
   *timeout_ms = message->timeout();
 
@@ -443,11 +467,12 @@ Status SendWaitReply(int sock, const ObjectRequestMap& object_requests,
   return PlasmaSend(sock, MessageType_PlasmaWaitReply, &fbb, message);
 }
 
-Status ReadWaitReply(uint8_t* data, ObjectRequest object_requests[],
+Status ReadWaitReply(uint8_t* data, size_t size, ObjectRequest object_requests[],
                      int* num_ready_objects) {
   DCHECK(data);
 
   auto message = flatbuffers::GetRoot<PlasmaWaitReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *num_ready_objects = message->num_ready_objects();
   for (int i = 0; i < *num_ready_objects; i++) {
     object_requests[i].object_id =
@@ -475,9 +500,11 @@ Status SendDataRequest(int sock, ObjectID object_id, const char* address, int po
   return PlasmaSend(sock, MessageType_PlasmaDataRequest, &fbb, message);
 }
 
-Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port) {
+Status ReadDataRequest(uint8_t* data, size_t size, ObjectID* object_id, char** address,
+                       int* port) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaDataRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   DCHECK(message->object_id()->size() == sizeof(ObjectID));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   *address = strdup(message->address()->c_str());
@@ -493,10 +520,11 @@ Status SendDataReply(int sock, ObjectID object_id, int64_t object_size,
   return PlasmaSend(sock, MessageType_PlasmaDataReply, &fbb, message);
 }
 
-Status ReadDataReply(uint8_t* data, ObjectID* object_id, int64_t* object_size,
-                     int64_t* metadata_size) {
+Status ReadDataReply(uint8_t* data, size_t size, ObjectID* object_id,
+                     int64_t* object_size, int64_t* metadata_size) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
   *object_size = (int64_t)message->object_size();
   *metadata_size = (int64_t)message->metadata_size();
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
index bab08b6cbd8ef..af4b13978c697 100644
--- a/cpp/src/plasma/protocol.h
+++ b/cpp/src/plasma/protocol.h
@@ -28,6 +28,12 @@ namespace plasma {
 
 using arrow::Status;
 
+template <class T>
+bool verify_flatbuffer(T* object, uint8_t* data, size_t size) {
+  flatbuffers::Verifier verifier(data, size);
+  return object->Verify(verifier);
+}
+
 /* Plasma receive message. */
 
 Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer);
@@ -37,29 +43,31 @@ Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffe
 Status SendCreateRequest(int sock, ObjectID object_id, int64_t data_size,
                          int64_t metadata_size);
 
-Status ReadCreateRequest(uint8_t* data, ObjectID* object_id, int64_t* data_size,
-                         int64_t* metadata_size);
+Status ReadCreateRequest(uint8_t* data, size_t size, ObjectID* object_id,
+                         int64_t* data_size, int64_t* metadata_size);
 
 Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error);
 
-Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object);
+Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id,
+                       PlasmaObject* object);
 
 /* Plasma Seal message functions. */
 
 Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest);
 
-Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest);
+Status ReadSealRequest(uint8_t* data, size_t size, ObjectID* object_id,
+                       unsigned char* digest);
 
 Status SendSealReply(int sock, ObjectID object_id, int error);
 
-Status ReadSealReply(uint8_t* data, ObjectID* object_id);
+Status ReadSealReply(uint8_t* data, size_t size, ObjectID* object_id);
 
 /* Plasma Get message functions. */
 
 Status SendGetRequest(int sock, const ObjectID* object_ids, int64_t num_objects,
                       int64_t timeout_ms);
 
-Status ReadGetRequest(uint8_t* data, std::vector<ObjectID>& object_ids,
+Status ReadGetRequest(uint8_t* data, size_t size, std::vector<ObjectID>& object_ids,
                       int64_t* timeout_ms);
 
 Status SendGetReply(
@@ -67,91 +75,93 @@ Status SendGetReply(
     std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
     int64_t num_objects);
 
-Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
-                    int64_t num_objects);
+Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
+                    PlasmaObject plasma_objects[], int64_t num_objects);
 
 /* Plasma Release message functions. */
 
 Status SendReleaseRequest(int sock, ObjectID object_id);
 
-Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id);
+Status ReadReleaseRequest(uint8_t* data, size_t size, ObjectID* object_id);
 
 Status SendReleaseReply(int sock, ObjectID object_id, int error);
 
-Status ReadReleaseReply(uint8_t* data, ObjectID* object_id);
+Status ReadReleaseReply(uint8_t* data, size_t size, ObjectID* object_id);
 
 /* Plasma Delete message functions. */
 
 Status SendDeleteRequest(int sock, ObjectID object_id);
 
-Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id);
+Status ReadDeleteRequest(uint8_t* data, size_t size, ObjectID* object_id);
 
 Status SendDeleteReply(int sock, ObjectID object_id, int error);
 
-Status ReadDeleteReply(uint8_t* data, ObjectID* object_id);
+Status ReadDeleteReply(uint8_t* data, size_t size, ObjectID* object_id);
 
 /* Satus messages. */
 
 Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
 
-Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects);
+Status ReadStatusRequest(uint8_t* data, size_t size, ObjectID object_ids[],
+                         int64_t num_objects);
 
 Status SendStatusReply(int sock, ObjectID object_ids[], int object_status[],
                        int64_t num_objects);
 
-int64_t ReadStatusReply_num_objects(uint8_t* data);
+int64_t ReadStatusReply_num_objects(uint8_t* data, size_t size);
 
-Status ReadStatusReply(uint8_t* data, ObjectID object_ids[], int object_status[],
-                       int64_t num_objects);
+Status ReadStatusReply(uint8_t* data, size_t size, ObjectID object_ids[],
+                       int object_status[], int64_t num_objects);
 
 /* Plasma Constains message functions. */
 
 Status SendContainsRequest(int sock, ObjectID object_id);
 
-Status ReadContainsRequest(uint8_t* data, ObjectID* object_id);
+Status ReadContainsRequest(uint8_t* data, size_t size, ObjectID* object_id);
 
 Status SendContainsReply(int sock, ObjectID object_id, bool has_object);
 
-Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object);
+Status ReadContainsReply(uint8_t* data, size_t size, ObjectID* object_id,
+                         bool* has_object);
 
 /* Plasma Connect message functions. */
 
 Status SendConnectRequest(int sock);
 
-Status ReadConnectRequest(uint8_t* data);
+Status ReadConnectRequest(uint8_t* data, size_t size);
 
 Status SendConnectReply(int sock, int64_t memory_capacity);
 
-Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity);
+Status ReadConnectReply(uint8_t* data, size_t size, int64_t* memory_capacity);
 
 /* Plasma Evict message functions (no reply so far). */
 
 Status SendEvictRequest(int sock, int64_t num_bytes);
 
-Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes);
+Status ReadEvictRequest(uint8_t* data, size_t size, int64_t* num_bytes);
 
 Status SendEvictReply(int sock, int64_t num_bytes);
 
-Status ReadEvictReply(uint8_t* data, int64_t& num_bytes);
+Status ReadEvictReply(uint8_t* data, size_t size, int64_t& num_bytes);
 
 /* Plasma Fetch Remote message functions. */
 
 Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
 
-Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids);
+Status ReadFetchRequest(uint8_t* data, size_t size, std::vector<ObjectID>& object_ids);
 
 /* Plasma Wait message functions. */
 
 Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
                        int num_ready_objects, int64_t timeout_ms);
 
-Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
+Status ReadWaitRequest(uint8_t* data, size_t size, ObjectRequestMap& object_requests,
                        int64_t* timeout_ms, int* num_ready_objects);
 
 Status SendWaitReply(int sock, const ObjectRequestMap& object_requests,
                      int num_ready_objects);
 
-Status ReadWaitReply(uint8_t* data, ObjectRequest object_requests[],
+Status ReadWaitReply(uint8_t* data, size_t size, ObjectRequest object_requests[],
                      int* num_ready_objects);
 
 /* Plasma Subscribe message functions. */
@@ -162,13 +172,14 @@ Status SendSubscribeRequest(int sock);
 
 Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port);
 
-Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port);
+Status ReadDataRequest(uint8_t* data, size_t size, ObjectID* object_id, char** address,
+                       int* port);
 
 Status SendDataReply(int sock, ObjectID object_id, int64_t object_size,
                      int64_t metadata_size);
 
-Status ReadDataReply(uint8_t* data, ObjectID* object_id, int64_t* object_size,
-                     int64_t* metadata_size);
+Status ReadDataReply(uint8_t* data, size_t size, ObjectID* object_id,
+                     int64_t* object_size, int64_t* metadata_size);
 
 }  // namespace plasma
 
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 9ceecdceadc5c..34adc6261ebe0 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -553,6 +553,7 @@ Status PlasmaStore::process_message(Client* client) {
   ARROW_CHECK(s.ok() || s.IsIOError());
 
   uint8_t* input = input_buffer_.data();
+  size_t input_size = input_buffer_.size();
   ObjectID object_id;
   PlasmaObject object;
   // TODO(pcm): Get rid of the following.
@@ -563,7 +564,8 @@ Status PlasmaStore::process_message(Client* client) {
     case MessageType_PlasmaCreateRequest: {
       int64_t data_size;
       int64_t metadata_size;
-      RETURN_NOT_OK(ReadCreateRequest(input, &object_id, &data_size, &metadata_size));
+      RETURN_NOT_OK(
+          ReadCreateRequest(input, input_size, &object_id, &data_size, &metadata_size));
       int error_code =
           create_object(object_id, data_size, metadata_size, client, &object);
       HANDLE_SIGPIPE(SendCreateReply(client->fd, object_id, &object, error_code),
@@ -575,15 +577,15 @@ Status PlasmaStore::process_message(Client* client) {
     case MessageType_PlasmaGetRequest: {
       std::vector<ObjectID> object_ids_to_get;
       int64_t timeout_ms;
-      RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms));
+      RETURN_NOT_OK(ReadGetRequest(input, input_size, object_ids_to_get, &timeout_ms));
       process_get_request(client, object_ids_to_get, timeout_ms);
     } break;
     case MessageType_PlasmaReleaseRequest:
-      RETURN_NOT_OK(ReadReleaseRequest(input, &object_id));
+      RETURN_NOT_OK(ReadReleaseRequest(input, input_size, &object_id));
       release_object(object_id, client);
       break;
     case MessageType_PlasmaContainsRequest:
-      RETURN_NOT_OK(ReadContainsRequest(input, &object_id));
+      RETURN_NOT_OK(ReadContainsRequest(input, input_size, &object_id));
       if (contains_object(object_id) == OBJECT_FOUND) {
         HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd);
       } else {
@@ -592,13 +594,13 @@ Status PlasmaStore::process_message(Client* client) {
       break;
     case MessageType_PlasmaSealRequest: {
       unsigned char digest[kDigestSize];
-      RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0]));
+      RETURN_NOT_OK(ReadSealRequest(input, input_size, &object_id, &digest[0]));
       seal_object(object_id, &digest[0]);
     } break;
     case MessageType_PlasmaEvictRequest: {
       // This code path should only be used for testing.
       int64_t num_bytes;
-      RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes));
+      RETURN_NOT_OK(ReadEvictRequest(input, input_size, &num_bytes));
       std::vector<ObjectID> objects_to_evict;
       int64_t num_bytes_evicted =
           eviction_policy_.choose_objects_to_evict(num_bytes, &objects_to_evict);
@@ -688,9 +690,8 @@ int main(int argc, char* argv[]) {
   close(shm_fd);
   if (system_memory > shm_mem_avail) {
     ARROW_LOG(FATAL) << "System memory request exceeds memory available in /dev/shm. The "
-                        "request is for "
-                     << system_memory << " bytes, and the amount available is "
-                     << shm_mem_avail
+                        "request is for " << system_memory
+                     << " bytes, and the amount available is " << shm_mem_avail
                      << " bytes. You may be able to free up space by deleting files in "
                         "/dev/shm. If you are inside a Docker container, you may need to "
                         "pass "
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
index aca47d3d6f911..c76f5ce109251 100644
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -82,8 +82,8 @@ TEST(PlasmaSerialization, CreateRequest) {
   ObjectID object_id2;
   int64_t data_size2;
   int64_t metadata_size2;
-  ARROW_CHECK_OK(
-      ReadCreateRequest(data.data(), &object_id2, &data_size2, &metadata_size2));
+  ARROW_CHECK_OK(ReadCreateRequest(data.data(), data.size(), &object_id2, &data_size2,
+                                   &metadata_size2));
   ASSERT_EQ(data_size1, data_size2);
   ASSERT_EQ(metadata_size1, metadata_size2);
   ASSERT_EQ(object_id1, object_id2);
@@ -99,7 +99,7 @@ TEST(PlasmaSerialization, CreateReply) {
   ObjectID object_id2;
   PlasmaObject object2;
   memset(&object2, 0, sizeof(object2));
-  ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2));
+  ARROW_CHECK_OK(ReadCreateReply(data.data(), data.size(), &object_id2, &object2));
   ASSERT_EQ(object_id1, object_id2);
   ASSERT_EQ(memcmp(&object1, &object2, sizeof(object1)), 0);
   close(fd);
@@ -114,7 +114,7 @@ TEST(PlasmaSerialization, SealRequest) {
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealRequest);
   ObjectID object_id2;
   unsigned char digest2[kDigestSize];
-  ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0]));
+  ARROW_CHECK_OK(ReadSealRequest(data.data(), data.size(), &object_id2, &digest2[0]));
   ASSERT_EQ(object_id1, object_id2);
   ASSERT_EQ(memcmp(&digest1[0], &digest2[0], kDigestSize), 0);
   close(fd);
@@ -126,7 +126,7 @@ TEST(PlasmaSerialization, SealReply) {
   ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealReply);
   ObjectID object_id2;
-  Status s = ReadSealReply(data.data(), &object_id2);
+  Status s = ReadSealReply(data.data(), data.size(), &object_id2);
   ASSERT_EQ(object_id1, object_id2);
   ASSERT_TRUE(s.IsPlasmaObjectExists());
   close(fd);
@@ -142,7 +142,8 @@ TEST(PlasmaSerialization, GetRequest) {
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetRequest);
   std::vector<ObjectID> object_ids_return;
   int64_t timeout_ms_return;
-  ARROW_CHECK_OK(ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return));
+  ARROW_CHECK_OK(
+      ReadGetRequest(data.data(), data.size(), object_ids_return, &timeout_ms_return));
   ASSERT_EQ(object_ids[0], object_ids_return[0]);
   ASSERT_EQ(object_ids[1], object_ids_return[1]);
   ASSERT_EQ(timeout_ms, timeout_ms_return);
@@ -162,8 +163,8 @@ TEST(PlasmaSerialization, GetReply) {
   ObjectID object_ids_return[2];
   PlasmaObject plasma_objects_return[2];
   memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
-  ARROW_CHECK_OK(
-      ReadGetReply(data.data(), object_ids_return, &plasma_objects_return[0], 2));
+  ARROW_CHECK_OK(ReadGetReply(data.data(), data.size(), object_ids_return,
+                              &plasma_objects_return[0], 2));
   ASSERT_EQ(object_ids[0], object_ids_return[0]);
   ASSERT_EQ(object_ids[1], object_ids_return[1]);
   ASSERT_EQ(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
@@ -182,7 +183,7 @@ TEST(PlasmaSerialization, ReleaseRequest) {
   std::vector<uint8_t> data =
       read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
   ObjectID object_id2;
-  ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2));
+  ARROW_CHECK_OK(ReadReleaseRequest(data.data(), data.size(), &object_id2));
   ASSERT_EQ(object_id1, object_id2);
   close(fd);
 }
@@ -193,7 +194,7 @@ TEST(PlasmaSerialization, ReleaseReply) {
   ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaReleaseReply);
   ObjectID object_id2;
-  Status s = ReadReleaseReply(data.data(), &object_id2);
+  Status s = ReadReleaseReply(data.data(), data.size(), &object_id2);
   ASSERT_EQ(object_id1, object_id2);
   ASSERT_TRUE(s.IsPlasmaObjectExists());
   close(fd);
@@ -205,7 +206,7 @@ TEST(PlasmaSerialization, DeleteRequest) {
   ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
   ObjectID object_id2;
-  ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2));
+  ARROW_CHECK_OK(ReadDeleteRequest(data.data(), data.size(), &object_id2));
   ASSERT_EQ(object_id1, object_id2);
   close(fd);
 }
@@ -217,7 +218,7 @@ TEST(PlasmaSerialization, DeleteReply) {
   ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteReply);
   ObjectID object_id2;
-  Status s = ReadDeleteReply(data.data(), &object_id2);
+  Status s = ReadDeleteReply(data.data(), data.size(), &object_id2);
   ASSERT_EQ(object_id1, object_id2);
   ASSERT_TRUE(s.IsPlasmaObjectExists());
   close(fd);
@@ -232,7 +233,8 @@ TEST(PlasmaSerialization, StatusRequest) {
   ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusRequest);
   ObjectID object_ids_read[num_objects];
-  ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects));
+  ARROW_CHECK_OK(
+      ReadStatusRequest(data.data(), data.size(), object_ids_read, num_objects));
   ASSERT_EQ(object_ids[0], object_ids_read[0]);
   ASSERT_EQ(object_ids[1], object_ids_read[1]);
   close(fd);
@@ -246,11 +248,11 @@ TEST(PlasmaSerialization, StatusReply) {
   int object_statuses[2] = {42, 43};
   ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusReply);
-  int64_t num_objects = ReadStatusReply_num_objects(data.data());
+  int64_t num_objects = ReadStatusReply_num_objects(data.data(), data.size());
   ObjectID object_ids_read[num_objects];
   int object_statuses_read[num_objects];
-  ARROW_CHECK_OK(
-      ReadStatusReply(data.data(), object_ids_read, object_statuses_read, num_objects));
+  ARROW_CHECK_OK(ReadStatusReply(data.data(), data.size(), object_ids_read,
+                                 object_statuses_read, num_objects));
   ASSERT_EQ(object_ids[0], object_ids_read[0]);
   ASSERT_EQ(object_ids[1], object_ids_read[1]);
   ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
@@ -264,7 +266,7 @@ TEST(PlasmaSerialization, EvictRequest) {
   ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictRequest);
   int64_t num_bytes_received;
-  ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received));
+  ARROW_CHECK_OK(ReadEvictRequest(data.data(), data.size(), &num_bytes_received));
   ASSERT_EQ(num_bytes, num_bytes_received);
   close(fd);
 }
@@ -275,7 +277,7 @@ TEST(PlasmaSerialization, EvictReply) {
   ARROW_CHECK_OK(SendEvictReply(fd, num_bytes));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictReply);
   int64_t num_bytes_received;
-  ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received));
+  ARROW_CHECK_OK(ReadEvictReply(data.data(), data.size(), num_bytes_received));
   ASSERT_EQ(num_bytes, num_bytes_received);
   close(fd);
 }
@@ -288,7 +290,7 @@ TEST(PlasmaSerialization, FetchRequest) {
   ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaFetchRequest);
   std::vector<ObjectID> object_ids_read;
-  ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read));
+  ARROW_CHECK_OK(ReadFetchRequest(data.data(), data.size(), object_ids_read));
   ASSERT_EQ(object_ids[0], object_ids_read[0]);
   ASSERT_EQ(object_ids[1], object_ids_read[1]);
   close(fd);
@@ -310,8 +312,8 @@ TEST(PlasmaSerialization, WaitRequest) {
   int num_ready_objects_out;
   int64_t timeout_ms_read;
   ObjectRequestMap object_requests_out;
-  ARROW_CHECK_OK(ReadWaitRequest(data.data(), object_requests_out, &timeout_ms_read,
-                                 &num_ready_objects_out));
+  ARROW_CHECK_OK(ReadWaitRequest(data.data(), data.size(), object_requests_out,
+                                 &timeout_ms_read, &num_ready_objects_out));
   ASSERT_EQ(num_objects_in, object_requests_out.size());
   ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
   for (int i = 0; i < num_objects_in; i++) {
@@ -340,7 +342,8 @@ TEST(PlasmaSerialization, WaitReply) {
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitReply);
   ObjectRequest objects_out[2];
   int num_objects_out;
-  ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out));
+  ARROW_CHECK_OK(
+      ReadWaitReply(data.data(), data.size(), &objects_out[0], &num_objects_out));
   ASSERT_EQ(num_objects_in, num_objects_out);
   for (int i = 0; i < num_objects_out; i++) {
     /* Each object request must appear exactly once. */
@@ -364,7 +367,8 @@ TEST(PlasmaSerialization, DataRequest) {
   ObjectID object_id2;
   char* address2;
   int port2;
-  ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2));
+  ARROW_CHECK_OK(
+      ReadDataRequest(data.data(), data.size(), &object_id2, &address2, &port2));
   ASSERT_EQ(object_id1, object_id2);
   ASSERT_EQ(strcmp(address1, address2), 0);
   ASSERT_EQ(port1, port2);
@@ -383,7 +387,8 @@ TEST(PlasmaSerialization, DataReply) {
   ObjectID object_id2;
   int64_t object_size2;
   int64_t metadata_size2;
-  ARROW_CHECK_OK(ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2));
+  ARROW_CHECK_OK(ReadDataReply(data.data(), data.size(), &object_id2, &object_size2,
+                               &metadata_size2));
   ASSERT_EQ(object_id1, object_id2);
   ASSERT_EQ(object_size1, object_size2);
   ASSERT_EQ(metadata_size1, metadata_size2);

From 5708cd10347ea92163f32d4943104d4207e486f1 Mon Sep 17 00:00:00 2001
From: rendel <renaud@siren.solutions>
Date: Wed, 26 Jul 2017 11:33:41 +0200
Subject: [PATCH 0858/1644] [Java] Fix some typos in code comments and
 exception messages

Closes #893

Change-Id: I4f8279086788986fed3c7a65f2be95dfc4809c28
---
 java/memory/src/main/java/io/netty/buffer/ArrowBuf.java         | 2 +-
 .../src/main/java/org/apache/arrow/memory/BaseAllocator.java    | 2 +-
 .../src/main/java/org/apache/arrow/vector/ValueVector.java      | 2 +-
 .../apache/arrow/vector/complex/AbstractContainerVector.java    | 2 +-
 .../main/java/org/apache/arrow/vector/complex/StateTool.java    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index 0328a16719058..09886a6ffe316 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -208,7 +208,7 @@ public ArrowBuf retain(BufferAllocator target) {
    * that carries an association with the underlying memory of this ArrowBuf. If this ArrowBuf is
    * connected to the
    * owning BufferLedger of this memory, that memory ownership/accounting will be transferred to
-   * the taret allocator. If
+   * the target allocator. If
    * this ArrowBuf does not currently own the memory underlying it (and is only associated with
    * it), this does not
    * transfer any ownership to the newly created ArrowBuf.
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index ddc78f03f0f72..be0ba77f5b2fa 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -640,7 +640,7 @@ private void dumpBuffers(final StringBuilder sb, final Set<BufferLedger> ledgerS
         continue;
       }
       final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
-      sb.append("UnsafeDirectLittleEndian[dentityHashCode == ");
+      sb.append("UnsafeDirectLittleEndian[identityHashCode == ");
       sb.append(Integer.toString(System.identityHashCode(udle)));
       sb.append("] size ");
       sb.append(Integer.toString(udle.capacity()));
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 2e83836b64626..3812c0b2fc319 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -47,7 +47,7 @@
  *   <li>you should never write to a vector once it has been read.</li>
  * </ul>
  *
- * Please note that the current implementation doesn't enfore those rules, hence we may find few places that
+ * Please note that the current implementation doesn't enforce those rules, hence we may find few places that
  * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)
  *
  * This interface "should" strive to guarantee this order of operation:
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 7f8e6796285fd..2aeeca25f0e9e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -79,7 +79,7 @@ protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
     if (clazz.isAssignableFrom(v.getClass())) {
       return clazz.cast(v);
     }
-    throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Arrow doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
+    throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Arrow doesn't yet support heterogenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
   }
 
   protected boolean supportsDirectRead() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
index 852c72c549729..05a79d24295e4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
@@ -28,7 +28,7 @@ public static <T extends Enum<?>> void check(T currentState, T... expectedStates
         return;
       }
     }
-    throw new IllegalArgumentException(String.format("Expected to be in one of these states %s but was actuall in state %s", Arrays.toString(expectedStates), currentState));
+    throw new IllegalArgumentException(String.format("Expected to be in one of these states %s but was actually in state %s", Arrays.toString(expectedStates), currentState));
   }
 
 }

From dca5d96c7a029c079183e2903db425e486e2deb9 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Wed, 26 Jul 2017 12:06:21 -0400
Subject: [PATCH 0859/1644] ARROW-1275: [C++] Deafult Snappy static lib suffix
 updated to "_static"

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #894 from MaxRis/ARROW-1275 and squashes the following commits:

3dcf216 [Max Risuhin] ARROW-1275: [C++] Deafult Snappy static lib suffix updated to "_static"
---
 cpp/CMakeLists.txt                          | 2 +-
 cpp/cmake_modules/SnappyCMakeLists.txt      | 6 +++---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 41af1e16ce28d..1e9aef0682b92 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -165,7 +165,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   if (MSVC)
     set(BROTLI_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Brotli static lib suffix used on Windows with MSVC (default _static)")
-    set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "" CACHE STRING
+    set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Snappy static lib suffix used on Windows with MSVC (default is empty string)")
     set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING
       "Zlib static lib suffix used on Windows with MSVC (default libstatic)")
diff --git a/cpp/cmake_modules/SnappyCMakeLists.txt b/cpp/cmake_modules/SnappyCMakeLists.txt
index 9d0a166064ea6..50083ce405e9c 100644
--- a/cpp/cmake_modules/SnappyCMakeLists.txt
+++ b/cpp/cmake_modules/SnappyCMakeLists.txt
@@ -68,10 +68,10 @@ set(SNAPPY_SRCS snappy.cc
   snappy-stubs-public.h)
 
 add_library(snappy SHARED ${SNAPPY_SRCS})
-add_library(snappystatic STATIC ${SNAPPY_SRCS})
+add_library(snappy_static STATIC ${SNAPPY_SRCS})
 
 TARGET_COMPILE_DEFINITIONS(snappy PRIVATE -DHAVE_CONFIG_H)
-TARGET_COMPILE_DEFINITIONS(snappystatic PRIVATE -DHAVE_CONFIG_H)
+TARGET_COMPILE_DEFINITIONS(snappy_static PRIVATE -DHAVE_CONFIG_H)
 
 install(FILES snappy.h
   snappy-c.h
@@ -79,7 +79,7 @@ install(FILES snappy.h
   ${snappy_BINARY_DIR}/snappy-stubs-public.h
   DESTINATION include)
 
-install(TARGETS snappy snappystatic
+install(TARGETS snappy snappy_static
   RUNTIME DESTINATION bin
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index b9d9823e80ce3..721e866e1d638 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -501,7 +501,7 @@ if (ARROW_WITH_SNAPPY)
     set(SNAPPY_HOME "${SNAPPY_PREFIX}")
     set(SNAPPY_INCLUDE_DIR "${SNAPPY_PREFIX}/include")
     if (MSVC)
-      set(SNAPPY_STATIC_LIB_NAME snappystatic)
+      set(SNAPPY_STATIC_LIB_NAME snappy_static)
     else()
       set(SNAPPY_STATIC_LIB_NAME snappy)
     endif()

From d76e43e7270ab24c0240dcd83e07ba3445235489 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 27 Jul 2017 11:27:54 -0400
Subject: [PATCH 0860/1644] ARROW-1268: [WEBSITE] Added blog post for Spark
 integration toPandas()

Adding blog post to highlight some of the work done in integrating Arrow with Spark for `toPandas()`

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #897 from BryanCutler/spark-blogpost-ARROW-1268 and squashes the following commits:

1f8dffd [Bryan Cutler] fixes and adding collaborators
2fa3587 [Bryan Cutler] fixed spelling and formatting
6a14066 [Bryan Cutler] Added blogpost for Spark integration toPandas()
---
 site/_posts/2017-07-26-spark-arrow.md | 158 ++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)
 create mode 100644 site/_posts/2017-07-26-spark-arrow.md

diff --git a/site/_posts/2017-07-26-spark-arrow.md b/site/_posts/2017-07-26-spark-arrow.md
new file mode 100644
index 0000000000000..c4b16c0738c0c
--- /dev/null
+++ b/site/_posts/2017-07-26-spark-arrow.md
@@ -0,0 +1,158 @@
+---
+layout: post
+title: "Speeding up PySpark with Apache Arrow"
+date: "2017-07-26 08:00:00 -0800"
+author: BryanCutler
+categories: [application]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+*[Bryan Cutler][11] is a software engineer at IBM's Spark Technology Center [STC][12]*
+
+Beginning with [Apache Spark][1] version 2.3, [Apache Arrow][2] will be a supported
+dependency and begin to offer increased performance with columnar data transfer.
+If you are a Spark user that prefers to work in Python and Pandas, this is a cause
+to be excited over! The initial work is limited to collecting a Spark DataFrame
+with `toPandas()`, which I will discuss below, however there are many additional
+improvements that are currently [underway][3].
+
+# Optimizing Spark Conversion to Pandas
+
+The previous way of converting a Spark DataFrame to Pandas with `DataFrame.toPandas()`
+in PySpark was painfully inefficient. Basically, it worked by first collecting all
+rows to the Spark driver. Next, each row would get serialized into Python's pickle
+format and sent to a Python worker process. This child process unpickles each row into
+a huge list of tuples. Finally, a Pandas DataFrame is created from the list using
+`pandas.DataFrame.from_records()`.
+
+This all might seem like standard procedure, but suffers from 2 glaring issues: 1)
+even using CPickle, Python serialization is a slow process and 2) creating
+a `pandas.DataFrame` using `from_records` must slowly iterate over the list of pure
+Python data and convert each value to Pandas format. See [here][4] for a detailed
+analysis.
+
+Here is where Arrow really shines to help optimize these steps: 1) Once the data is
+in Arrow memory format, there is no need to serialize/pickle anymore as Arrow data can
+be sent directly to the Python process, 2) When the Arrow data is received in Python,
+then pyarrow can utilize zero-copy methods to create a `pandas.DataFrame` from entire
+chunks of data at once instead of processing individual scalar values. Additionally,
+the conversion to Arrow data can be done on the JVM and pushed back for the Spark
+executors to perform in parallel, drastically reducing the load on the driver.
+
+As of the merging of [SPARK-13534][5], the use of Arrow when calling `toPandas()`
+needs to be enabled by setting the SQLConf "spark.sql.execution.arrow.enable" to
+"true".  Let's look at a simple usage example.
+
+```
+Welcome to
+      ____              __
+     / __/__  ___ _____/ /__
+    _\ \/ _ \/ _ `/ __/  '_/
+   /__ / .__/\_,_/_/ /_/\_\   version 2.3.0-SNAPSHOT
+      /_/
+
+Using Python version 2.7.13 (default, Dec 20 2016 23:09:15)
+SparkSession available as 'spark'.
+
+In [1]: from pyspark.sql.functions import rand
+   ...: df = spark.range(1 << 22).toDF("id").withColumn("x", rand())
+   ...: df.printSchema()
+   ...: 
+root
+ |-- id: long (nullable = false)
+ |-- x: double (nullable = false)
+
+
+In [2]: %time pdf = df.toPandas()
+CPU times: user 17.4 s, sys: 792 ms, total: 18.1 s
+Wall time: 20.7 s
+
+In [3]: spark.conf.set("spark.sql.execution.arrow.enable", "true")
+
+In [4]: %time pdf = df.toPandas()
+CPU times: user 40 ms, sys: 32 ms, total: 72 ms                                 
+Wall time: 737 ms
+
+In [5]: pdf.describe()
+Out[5]: 
+                 id             x
+count  4.194304e+06  4.194304e+06
+mean   2.097152e+06  4.998996e-01
+std    1.210791e+06  2.887247e-01
+min    0.000000e+00  8.291929e-07
+25%    1.048576e+06  2.498116e-01
+50%    2.097152e+06  4.999210e-01
+75%    3.145727e+06  7.498380e-01
+max    4.194303e+06  9.999996e-01
+```
+
+This example was run locally on my laptop using Spark defaults so the times
+shown should not be taken precisely. Even though, it is clear there is a huge
+performance boost and using Arrow took something that was excruciatingly slow
+and speeds it up to be barely noticeable.
+
+# Notes on Usage
+
+Here are some things to keep in mind before making use of this new feature. At
+the time of writing this, pyarrow will not be installed automatically with
+pyspark and needs to be manually installed, see installation [instructions][6].
+It is planned to add pyarrow as a pyspark dependency so that 
+`> pip install pyspark` will also install pyarrow.
+
+Currently, the controlling SQLConf is disabled by default. This can be enabled
+programmatically as in the example above or by adding the line
+"spark.sql.execution.arrow.enable=true" to `SPARK_HOME/conf/spark-defaults.conf`.
+
+Also, not all Spark data types are currently supported and limited to primitive
+types. Expanded type support is in the works and expected to also be in the Spark
+2.3 release.
+
+# Future Improvements
+
+As mentioned, this was just a first step in using Arrow to make life easier for
+Spark Python users. A few exciting initiatives in the works are to allow for
+vectorized UDF evaluation ([SPARK-21190][7], [SPARK-21404][8]), and the ability
+to apply a function on grouped data using a Pandas DataFrame ([SPARK-20396][9]).
+Just as Arrow helped in converting a Spark to Pandas, it can also work in the
+other direction when creating a Spark DataFrame from an existing Pandas
+DataFrame ([SPARK-20791][10]). Stay tuned for more!
+
+# Collaborators
+
+Reaching this first milestone was a group effort from both the Apache Arrow and
+Spark communities. Thanks to the hard work of [Wes McKinney][13], [Li Jin][14],
+[Holden Karau][15], Reynold Xin, Wenchen Fan, Shane Knapp and many others that
+helped push this effort forwards.
+
+[1]: https://spark.apache.org/
+[2]: https://arrow.apache.org/
+[3]: https://issues.apache.org/jira/issues/?filter=12335725&jql=project%20%3D%20SPARK%20AND%20status%20in%20(Open%2C%20%22In%20Progress%22%2C%20Reopened)%20AND%20text%20~%20%22arrow%22%20ORDER%20BY%20createdDate%20DESC
+[4]: https://gist.github.com/wesm/0cb5531b1c2e346a0007
+[5]: https://issues.apache.org/jira/browse/SPARK-13534
+[6]: https://github.com/apache/arrow/blob/master/site/install.md
+[7]: https://issues.apache.org/jira/browse/SPARK-21190
+[8]: https://issues.apache.org/jira/browse/SPARK-21404
+[9]: https://issues.apache.org/jira/browse/SPARK-20396
+[10]: https://issues.apache.org/jira/browse/SPARK-20791
+[11]: https://github.com/BryanCutler
+[12]: http://www.spark.tc/
+[13]: https://github.com/wesm
+[14]: https://github.com/icexelloss
+[15]: https://github.com/holdenk

From cae3510d28d7a218f524d03ac07e21b4fb2f566b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 27 Jul 2017 12:21:48 -0400
Subject: [PATCH 0861/1644] ARROW-1274: [C++] Fix CMake >= 3.3 warning. Also
 add option to suppress ExternalProject output

The default is the current behavior, but if enabled the build output will log ExternalProject build progress to files instead of dumping everything to the console. It might be nice to change the default to ON but we could do that in a separate patch

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #891 from wesm/ARROW-1274 and squashes the following commits:

a43d4e8 [Wes McKinney] Set verbose externalproject logging to true
10f8e92 [Wes McKinney] Fix CMake >= 3.3 warning. Also add option for verbose ExternalProject logging, otherwise suppress
---
 cpp/CMakeLists.txt                          | 28 +++++++++++++--
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 40 ++++++++++++++++-----
 2 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 1e9aef0682b92..07b8e15b504e4 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -162,6 +162,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build with zstd compression"
     ON)
 
+  option(ARROW_VERBOSE_THIRDPARTY_BUILD
+    "If off, output from ExternalProjects will be logged to files rather than shown"
+    ON)
+
   if (MSVC)
     set(BROTLI_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Brotli static lib suffix used on Windows with MSVC (default _static)")
@@ -303,8 +307,28 @@ include_directories(src)
 # For generate_export_header() and add_compiler_export_flags().
 include(GenerateExportHeader)
 
-# Sets -fvisibility=hidden for gcc
-add_compiler_export_flags()
+# Adapted from Apache Kudu: https://github.com/apache/kudu/commit/bd549e13743a51013585
+# Honor visibility properties for all target types. See
+# "cmake --help-policy CMP0063" for details.
+#
+# This policy was only added to cmake in version 3.3, so until the cmake in
+# thirdparty is updated, we must check if the policy exists before setting it.
+if(POLICY CMP0063)
+  cmake_policy(SET CMP0063 NEW)
+endif()
+
+if (PARQUET_BUILD_SHARED)
+  if (POLICY CMP0063)
+    set_target_properties(arrow_shared
+      PROPERTIES
+      C_VISIBILITY_PRESET hidden
+      CXX_VISIBILITY_PRESET hidden
+      VISIBILITY_INLINES_HIDDEN 1)
+  else()
+    # Sets -fvisibility=hidden for gcc
+    add_compiler_export_flags()
+  endif()
+endif()
 
 ############################################################
 # Benchmarking
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 721e866e1d638..6d473869cbc3d 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -35,6 +35,16 @@ string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
 set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}")
 set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}")
 
+if (NOT ARROW_VERBOSE_THIRDPARTY_BUILD)
+  set(EP_LOG_OPTIONS
+    LOG_CONFIGURE 1
+    LOG_BUILD 1
+    LOG_INSTALL 1
+    LOG_DOWNLOAD 1)
+else()
+  set(EP_LOG_OPTIONS)
+endif()
+
 if (NOT MSVC)
   # Set -fPIC on all external projects
   set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC")
@@ -205,7 +215,8 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
     ExternalProject_Add(googletest_ep
       URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
       BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB}
-      CMAKE_ARGS ${GTEST_CMAKE_ARGS})
+      CMAKE_ARGS ${GTEST_CMAKE_ARGS}
+      ${EP_LOG_OPTIONS})
   else()
     find_package(GTest REQUIRED)
     set(GTEST_VENDORED 0)
@@ -250,6 +261,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
 
     ExternalProject_Add(gflags_ep
       URL ${GFLAGS_URL}
+      ${EP_LOG_OPTIONS}
       BUILD_IN_SOURCE 1
       BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}"
       CMAKE_ARGS ${GFLAGS_CMAKE_ARGS})
@@ -300,7 +312,8 @@ if(ARROW_BUILD_BENCHMARKS)
     ExternalProject_Add(gbenchmark_ep
       URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
       BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}"
-      CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS})
+      CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS}
+      ${EP_LOG_OPTIONS})
   else()
     find_package(GBenchmark REQUIRED)
     set(GBENCHMARK_VENDORED 0)
@@ -327,6 +340,7 @@ if (ARROW_IPC)
       CONFIGURE_COMMAND ""
       BUILD_COMMAND ""
       BUILD_IN_SOURCE 1
+      ${EP_LOG_OPTIONS}
       INSTALL_COMMAND "")
 
     ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR)
@@ -356,7 +370,8 @@ if (ARROW_IPC)
       CMAKE_ARGS
       "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}"
       "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
-      "-DFLATBUFFERS_BUILD_TESTS=OFF")
+      "-DFLATBUFFERS_BUILD_TESTS=OFF"
+      ${EP_LOG_OPTIONS})
 
     set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
     set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
@@ -395,6 +410,7 @@ if (ARROW_JEMALLOC)
     ExternalProject_Add(jemalloc_ep
       URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
       CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
+      ${EP_LOG_OPTIONS}
       BUILD_IN_SOURCE 1
       BUILD_COMMAND ${MAKE}
       BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
@@ -475,6 +491,7 @@ if (ARROW_WITH_ZLIB)
 
     ExternalProject_Add(zlib_ep
       URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz"
+      ${EP_LOG_OPTIONS}
       BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}"
       CMAKE_ARGS ${ZLIB_CMAKE_ARGS})
     set(ZLIB_VENDORED 1)
@@ -529,6 +546,7 @@ if (ARROW_WITH_SNAPPY)
                         ./config.h)
       ExternalProject_Add(snappy_ep
         UPDATE_COMMAND ${SNAPPY_UPDATE_COMMAND}
+        ${EP_LOG_OPTIONS}
         BUILD_IN_SOURCE 1
         BUILD_COMMAND ${MAKE}
         INSTALL_DIR ${SNAPPY_PREFIX}
@@ -538,6 +556,7 @@ if (ARROW_WITH_SNAPPY)
     else()
       ExternalProject_Add(snappy_ep
         CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS}
+        ${EP_LOG_OPTIONS}
         BUILD_IN_SOURCE 1
         BUILD_COMMAND ${MAKE}
         INSTALL_DIR ${SNAPPY_PREFIX}
@@ -586,6 +605,7 @@ if (ARROW_WITH_BROTLI)
       URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz"
       BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}"
       ${BROTLI_BUILD_BYPRODUCTS}
+      ${EP_LOG_OPTIONS}
       CMAKE_ARGS ${BROTLI_CMAKE_ARGS}
       STEP_TARGETS headers_copy)
     if (MSVC)
@@ -624,7 +644,7 @@ if (ARROW_WITH_LZ4)
   if("${LZ4_HOME}" STREQUAL "")
     set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep")
     set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
-  
+
     if (MSVC)
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
       set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
@@ -632,9 +652,10 @@ if (ARROW_WITH_LZ4)
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
       set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh)
     endif()
-  
+
     ExternalProject_Add(lz4_ep
         URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz"
+        ${EP_LOG_OPTIONS}
         UPDATE_COMMAND ""
         PATCH_COMMAND ""
         CONFIGURE_COMMAND ""
@@ -643,22 +664,22 @@ if (ARROW_WITH_LZ4)
         BUILD_BYPRODUCTS ${LZ4_STATIC_LIB}
         ${LZ4_BUILD_COMMAND}
         )
-  
+
     set(LZ4_VENDORED 1)
   else()
     find_package(Lz4 REQUIRED)
     set(LZ4_VENDORED 0)
   endif()
-  
+
   include_directories(SYSTEM ${LZ4_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(lz4_static
     STATIC_LIB ${LZ4_STATIC_LIB})
-  
+
   if (LZ4_VENDORED)
     add_dependencies(lz4_static lz4_ep)
   endif()
 endif()
-  
+
 if (ARROW_WITH_ZSTD)
 # ----------------------------------------------------------------------
 # ZSTD
@@ -677,6 +698,7 @@ if (ARROW_WITH_ZSTD)
 
     ExternalProject_Add(zstd_ep
         URL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz"
+        ${EP_LOG_OPTIONS}
         UPDATE_COMMAND ""
         PATCH_COMMAND ""
         CONFIGURE_COMMAND ""

From 7b3378fa549dbbcdd5fa07e0563e3e5945d333e6 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Thu, 27 Jul 2017 12:23:22 -0400
Subject: [PATCH 0862/1644] =?UTF-8?q?ARROW-1204:=20[C++]=20Remove=20WholeP?=
 =?UTF-8?q?rogramOptimization(/GL)=20compilation=20fl=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ag for lz4 and zstd libs

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #896 from MaxRis/ARROW-1204 and squashes the following commits:

280b5e3 [Max Risuhin] ARROW-1204: [C++] Remove WholeProgramOptimization(/GL) compilation flag for lz4 and zstd libs
---
 appveyor.yml                                  |   2 +-
 ...build_wholeprogramoptimization_param.patch | 225 ++++++++++++++++++
 ...build_wholeprogramoptimization_param.patch | 199 ++++++++++++++++
 cpp/cmake_modules/ThirdpartyToolchain.cmake   |   6 +-
 4 files changed, 429 insertions(+), 3 deletions(-)
 create mode 100644 cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch
 create mode 100644 cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch

diff --git a/appveyor.yml b/appveyor.yml
index 91e9ee2649073..55c58d0bf664d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -60,7 +60,7 @@ environment:
 init:
   - set MINICONDA=C:\Miniconda35-x64
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
-  - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64
+  - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
 
 build_script:
   - git config core.symlinks true
diff --git a/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch b/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch
new file mode 100644
index 0000000000000..ee0f8a12054bc
--- /dev/null
+++ b/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch
@@ -0,0 +1,225 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj
+index aaf81ad..67b716f 100644
+--- a/visual/VS2010/datagen/datagen.vcxproj
++++ b/visual/VS2010/datagen/datagen.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj
+index 76d12c9..723571d 100644
+--- a/visual/VS2010/frametest/frametest.vcxproj
++++ b/visual/VS2010/frametest/frametest.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
+index c10552a..0c8f293 100644
+--- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
++++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj
+index e2d95c9..4cd88d0 100644
+--- a/visual/VS2010/fullbench/fullbench.vcxproj
++++ b/visual/VS2010/fullbench/fullbench.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj
+index 85d6c9b..3ddc77d 100644
+--- a/visual/VS2010/fuzzer/fuzzer.vcxproj
++++ b/visual/VS2010/fuzzer/fuzzer.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
+index 389f13c..038a4d2 100644
+--- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
++++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
+@@ -40,15 +40,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj
+index a0b8000..9aad8c2 100644
+--- a/visual/VS2010/liblz4/liblz4.vcxproj
++++ b/visual/VS2010/liblz4/liblz4.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj
+index 693e121..7e63f1e 100644
+--- a/visual/VS2010/lz4/lz4.vcxproj
++++ b/visual/VS2010/lz4/lz4.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
diff --git a/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch b/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch
new file mode 100644
index 0000000000000..8bfb928947e06
--- /dev/null
+++ b/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch
@@ -0,0 +1,199 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+diff --git a/build/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj
+index bd8a213..8e4dc89 100644
+--- a/build/VS2010/datagen/datagen.vcxproj
++++ b/build/VS2010/datagen/datagen.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
+index e697318..82cd4ab 100644
+--- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
++++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj
+index 2bff4ca..ced4047 100644
+--- a/build/VS2010/fullbench/fullbench.vcxproj
++++ b/build/VS2010/fullbench/fullbench.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
+index 12a4b93..227efd1 100644
+--- a/build/VS2010/fuzzer/fuzzer.vcxproj
++++ b/build/VS2010/fuzzer/fuzzer.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+index 364b3be..b227320 100644
+--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
++++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+@@ -94,15 +94,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
+index 6087d73..51a0572 100644
+--- a/build/VS2010/libzstd/libzstd.vcxproj
++++ b/build/VS2010/libzstd/libzstd.vcxproj
+@@ -91,15 +91,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
+index 438dc61..834ae01 100644
+--- a/build/VS2010/zstd/zstd.vcxproj
++++ b/build/VS2010/zstd/zstd.vcxproj
+@@ -100,15 +100,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 6d473869cbc3d..1271b8a4ab3f4 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -648,6 +648,7 @@ if (ARROW_WITH_LZ4)
     if (MSVC)
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
       set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
+      set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_wholeprogramoptimization_param.patch)
     else()
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
       set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh)
@@ -657,7 +658,7 @@ if (ARROW_WITH_LZ4)
         URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz"
         ${EP_LOG_OPTIONS}
         UPDATE_COMMAND ""
-        PATCH_COMMAND ""
+        ${LZ4_PATCH_COMMAND}
         CONFIGURE_COMMAND ""
         INSTALL_COMMAND ""
         BINARY_DIR ${LZ4_BUILD_DIR}
@@ -691,6 +692,7 @@ if (ARROW_WITH_ZSTD)
     if (MSVC)
       set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib")
       set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
+      set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_wholeprogramoptimization_param.patch)
     else()
       set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a")
       set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh)
@@ -700,7 +702,7 @@ if (ARROW_WITH_ZSTD)
         URL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz"
         ${EP_LOG_OPTIONS}
         UPDATE_COMMAND ""
-        PATCH_COMMAND ""
+        ${ZSTD_PATCH_COMMAND}
         CONFIGURE_COMMAND ""
         INSTALL_COMMAND ""
         BINARY_DIR ${ZSTD_BUILD_DIR}

From f72279b2dbfc663d2217e64075dd731199f12611 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 27 Jul 2017 14:18:45 -0400
Subject: [PATCH 0863/1644] ARROW-1288: Fix many license headers to use proper
 ASF one

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #900 from wesm/fix-license-headers and squashes the following commits:

73941bf2 [Wes McKinney] Fix many license headers to use proper ASF one
---
 CHANGELOG.md                                  | 27 ++++++-----
 README.md                                     | 27 ++++++-----
 c_glib/README.md                              | 27 ++++++-----
 c_glib/doc/reference/arrow-glib-docs.sgml     | 23 +++++----
 c_glib/example/README.md                      | 27 ++++++-----
 c_glib/example/go/Makefile                    | 23 +++++----
 c_glib/example/go/README.md                   | 27 ++++++-----
 c_glib/example/lua/README.md                  | 27 ++++++-----
 ci/travis_before_script_c_glib.sh             | 22 +++++----
 ci/travis_before_script_cpp.sh                | 21 +++++----
 ci/travis_before_script_js.sh                 | 21 +++++----
 ci/travis_env_common.sh                       | 21 +++++----
 ci/travis_install_clang_tools.sh              | 21 +++++----
 ci/travis_install_conda.sh                    | 21 +++++----
 ci/travis_script_c_glib.sh                    | 21 +++++----
 ci/travis_script_cpp.sh                       | 21 +++++----
 ci/travis_script_integration.sh               | 21 +++++----
 ci/travis_script_java.sh                      | 21 +++++----
 ci/travis_script_js.sh                        | 21 +++++----
 ci/travis_script_manylinux.sh                 | 21 +++++----
 ci/travis_script_python.sh                    | 21 +++++----
 cpp/README.md                                 | 27 ++++++-----
 cpp/apidoc/HDFS.md                            | 28 ++++++-----
 cpp/apidoc/Windows.md                         | 47 ++++++++++---------
 cpp/apidoc/index.md                           | 27 ++++++-----
 cpp/apidoc/tutorials/row_wise_conversion.md   | 29 +++++++-----
 cpp/build-support/build-lz4-lib.sh            | 25 ++++++----
 cpp/build-support/build-zstd-lib.sh           | 25 ++++++----
 cpp/build-support/run-clang-tidy.sh           | 27 ++++++-----
 cpp/doc/Parquet.md                            | 23 +++++----
 cpp/src/arrow/symbols.map                     | 21 +++++----
 format/Guidelines.md                          | 23 +++++----
 format/IPC.md                                 | 27 ++++++-----
 format/Layout.md                              | 27 ++++++-----
 format/Metadata.md                            | 27 ++++++-----
 format/README.md                              | 27 ++++++-----
 integration/README.md                         | 27 ++++++-----
 java/README.md                                | 25 ++++++----
 js/README.md                                  | 23 +++++----
 js/flatbuffers.sh                             | 23 +++++----
 js/webpack.config.js                          | 23 +++++----
 python/README.md                              | 27 ++++++-----
 python/asv.conf.json                          | 21 +++++----
 python/doc/Benchmarks.md                      | 23 +++++----
 python/doc/Makefile                           | 24 ++++++----
 python/doc/source/conf.py                     | 21 +++++----
 python/manylinux1/Dockerfile-x86_64           | 21 +++++----
 python/manylinux1/Dockerfile-x86_64_base      | 21 +++++----
 python/manylinux1/README.md                   | 23 +++++----
 python/manylinux1/build_arrow.sh              | 21 +++++----
 python/manylinux1/scripts/build_boost.sh      | 22 +++++----
 python/manylinux1/scripts/build_brotli.sh     | 21 +++++----
 python/manylinux1/scripts/build_ccache.sh     | 21 +++++----
 .../manylinux1/scripts/build_flatbuffers.sh   | 21 +++++----
 python/manylinux1/scripts/build_gtest.sh      | 21 +++++----
 python/manylinux1/scripts/build_jemalloc.sh   | 21 +++++----
 python/manylinux1/scripts/build_lz4.sh        | 21 +++++----
 python/manylinux1/scripts/build_openssl.sh    | 21 +++++----
 python/manylinux1/scripts/build_snappy.sh     | 21 +++++----
 python/manylinux1/scripts/build_thrift.sh     | 21 +++++----
 .../manylinux1/scripts/build_virtualenvs.sh   | 21 +++++----
 python/manylinux1/scripts/build_zstd.sh       | 21 +++++----
 python/pyarrow/_config.pyx                    | 21 +++++----
 python/pyarrow/tests/test_feather.py          | 25 +++++-----
 site/README.md                                | 27 ++++++-----
 65 files changed, 931 insertions(+), 610 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0cc6c6ee62a8a..6cedf32df628f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Apache Arrow 0.5.0 (23 July 2017)
diff --git a/README.md b/README.md
index 2790895878563..9dda25de91178 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 ## Apache Arrow
diff --git a/c_glib/README.md b/c_glib/README.md
index 622938550d830..fec877e236fed 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Arrow GLib
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 4fa1b7c42def5..26fd2f6262ba5 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -1,16 +1,21 @@
 <?xml version="1.0"?>
 <!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
 
-   http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
                "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd"
diff --git a/c_glib/example/README.md b/c_glib/example/README.md
index 99730d59ce1c2..b69145d68fd7f 100644
--- a/c_glib/example/README.md
+++ b/c_glib/example/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Arrow GLib example
diff --git a/c_glib/example/go/Makefile b/c_glib/example/go/Makefile
index d8831122d4dc6..fa2163ca81b3d 100644
--- a/c_glib/example/go/Makefile
+++ b/c_glib/example/go/Makefile
@@ -1,14 +1,19 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#  http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 PROGRAMS =					\
 	read-batch				\
diff --git a/c_glib/example/go/README.md b/c_glib/example/go/README.md
index 2054055e6554e..76eeed78c71d4 100644
--- a/c_glib/example/go/README.md
+++ b/c_glib/example/go/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Arrow Go example
diff --git a/c_glib/example/lua/README.md b/c_glib/example/lua/README.md
index 6145bc74ddd5a..e7e3351fef148 100644
--- a/c_glib/example/lua/README.md
+++ b/c_glib/example/lua/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Arrow Lua example
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 6547ea4e53709..bf2d385d79d4d 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -1,17 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -ex
 
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index a1697dae59fbf..d456d308c53e3 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 
 set -ex
diff --git a/ci/travis_before_script_js.sh b/ci/travis_before_script_js.sh
index 304c48137aa55..b72accc219332 100755
--- a/ci/travis_before_script_js.sh
+++ b/ci/travis_before_script_js.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 
 set -ex
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index 4122eacaa2ee3..d84753125d5c4 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 export MINICONDA=$HOME/miniconda
 export PATH="$MINICONDA/bin:$PATH"
diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
index a4fd0e24619b1..bad1e73d24a4a 100644
--- a/ci/travis_install_clang_tools.sh
+++ b/ci/travis_install_clang_tools.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
 sudo apt-add-repository -y \
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index 369820b37f5c1..c2502a3744c11 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
diff --git a/ci/travis_script_c_glib.sh b/ci/travis_script_c_glib.sh
index 4bfa0c0af49ca..d392abdfbbc36 100755
--- a/ci/travis_script_c_glib.sh
+++ b/ci/travis_script_c_glib.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index a4f6b056d9de4..4e3e7bbea1c08 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 6e93ed79a2266..be025512f0b88 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
diff --git a/ci/travis_script_java.sh b/ci/travis_script_java.sh
index 259b73ec24e11..2f6b685253b10 100755
--- a/ci/travis_script_java.sh
+++ b/ci/travis_script_java.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
diff --git a/ci/travis_script_js.sh b/ci/travis_script_js.sh
index 52ac3b9bdf8a3..cb1e9e194408a 100755
--- a/ci/travis_script_js.sh
+++ b/ci/travis_script_js.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh
index 844d5f719f15a..14e6404d3de08 100755
--- a/ci/travis_script_manylinux.sh
+++ b/ci/travis_script_manylinux.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 
 set -ex
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index dcba78273caa7..907bc60cd719e 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -1,15 +1,20 @@
 #!/usr/bin/env bash
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
diff --git a/cpp/README.md b/cpp/README.md
index 5bb516fc99b5d..05a8ba48475ea 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Arrow C++
diff --git a/cpp/apidoc/HDFS.md b/cpp/apidoc/HDFS.md
index 180d31e54d573..d54ad270c05f4 100644
--- a/cpp/apidoc/HDFS.md
+++ b/cpp/apidoc/HDFS.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 ## Using Arrow's HDFS (Apache Hadoop Distributed File System) interface
@@ -72,4 +77,3 @@ If you get an error about needing to install Java 6, then add *BundledApp* and
 https://oliverdowling.com.au/2015/10/09/oracles-jre-8-on-mac-os-x-el-capitan/
 
 https://derflounder.wordpress.com/2015/08/08/modifying-oracles-java-sdk-to-run-java-applications-on-os-x/
-
diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index 6bfb951548ace..30b7b8f3ce210 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Developing Arrow C++ on Windows
@@ -26,7 +31,7 @@ other development instructions for Windows here.
 [Miniconda][1] is a minimal Python distribution including the conda package
 manager. To get started, download and install a 64-bit distribution.
 
-We recommend using packages from [conda-forge][2].  
+We recommend using packages from [conda-forge][2].
 Launch cmd.exe and run following commands:
 
 ```shell
@@ -46,7 +51,7 @@ previous step:
 activate arrow-dev
 ```
 
-We are using [cmake][4] tool to support Windows builds.  
+We are using [cmake][4] tool to support Windows builds.
 To allow cmake to pick up 3rd party dependencies, you should set
 `ARROW_BUILD_TOOLCHAIN` environment variable to contain `Library` folder
 path of new created on previous step `arrow-dev` conda environment.
@@ -71,16 +76,16 @@ As alternative to `ARROW_BUILD_TOOLCHAIN`, it's possible to configure path
 to each 3rd party dependency separately by setting appropriate environment
 variable:
 
-`FLATBUFFERS_HOME` variable with path to `flatbuffers` installation  
-`RAPIDJSON_HOME` variable with path to `rapidjson` installation  
-`GFLAGS_HOME` variable with path to `gflags` installation  
-`SNAPPY_HOME` variable with path to `snappy` installation  
-`ZLIB_HOME` variable with path to `zlib` installation  
-`BROTLI_HOME` variable with path to `brotli` installation  
-`LZ4_HOME` variable with path to `lz4` installation  
+`FLATBUFFERS_HOME` variable with path to `flatbuffers` installation
+`RAPIDJSON_HOME` variable with path to `rapidjson` installation
+`GFLAGS_HOME` variable with path to `gflags` installation
+`SNAPPY_HOME` variable with path to `snappy` installation
+`ZLIB_HOME` variable with path to `zlib` installation
+`BROTLI_HOME` variable with path to `brotli` installation
+`LZ4_HOME` variable with path to `lz4` installation
 `ZSTD_HOME` variable with path to `zstd` installation
 
-### Customize static libraries names lookup of 3rd party dependencies 
+### Customize static libraries names lookup of 3rd party dependencies
 
 If you decided to use pre-built 3rd party dependencies libs, it's possible to
 configure Arrow's cmake build script to search for customized names of 3rd
diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
index 4004e1ef42ea9..8389d16b4aa1b 100644
--- a/cpp/apidoc/index.md
+++ b/cpp/apidoc/index.md
@@ -2,17 +2,22 @@ Apache Arrow C++ API documentation      {#index}
 ==================================
 
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 Apache Arrow is a columnar in-memory analytics layer designed to accelerate
diff --git a/cpp/apidoc/tutorials/row_wise_conversion.md b/cpp/apidoc/tutorials/row_wise_conversion.md
index 1486fc2a4e041..e91c26e9da120 100644
--- a/cpp/apidoc/tutorials/row_wise_conversion.md
+++ b/cpp/apidoc/tutorials/row_wise_conversion.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 Convert a vector of row-wise data into an Arrow table
@@ -118,7 +123,7 @@ To convert an Arrow table back into the same row-wise representation as in the
 above section, we first will check that the table conforms to our expected
 schema and then will build up the vector of rows incrementally.
 
-For the check if the table is as expected, we can utilise solely its schema. 
+For the check if the table is as expected, we can utilise solely its schema.
 
 ```
 // This is our input that was passed in from the outside.
diff --git a/cpp/build-support/build-lz4-lib.sh b/cpp/build-support/build-lz4-lib.sh
index 62805bae286a2..8cb5c18782a7f 100755
--- a/cpp/build-support/build-lz4-lib.sh
+++ b/cpp/build-support/build-lz4-lib.sh
@@ -1,16 +1,21 @@
 #!/bin/sh
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 export CFLAGS="${CFLAGS} -O3 -fPIC"
-make -j4
\ No newline at end of file
+make -j4
diff --git a/cpp/build-support/build-zstd-lib.sh b/cpp/build-support/build-zstd-lib.sh
index 62805bae286a2..8cb5c18782a7f 100755
--- a/cpp/build-support/build-zstd-lib.sh
+++ b/cpp/build-support/build-zstd-lib.sh
@@ -1,16 +1,21 @@
 #!/bin/sh
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 export CFLAGS="${CFLAGS} -O3 -fPIC"
-make -j4
\ No newline at end of file
+make -j4
diff --git a/cpp/build-support/run-clang-tidy.sh b/cpp/build-support/run-clang-tidy.sh
index 4ba8ab8cd766d..75e9458e257ca 100755
--- a/cpp/build-support/run-clang-tidy.sh
+++ b/cpp/build-support/run-clang-tidy.sh
@@ -1,16 +1,21 @@
 #!/bin/bash
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 #
 # Runs clang format in the given directory
@@ -27,7 +32,7 @@ shift
 APPLY_FIXES=$1
 shift
 
-# clang format will only find its configuration if we are in 
+# clang format will only find its configuration if we are in
 # the source tree or in a path relative to the source tree
 if [ "$APPLY_FIXES" == "1" ]; then
   $CLANG_TIDY -p $COMPILE_COMMANDS -fix  $@
@@ -37,4 +42,4 @@ else
     echo "clang-tidy had suggested fixes.  Please fix these!!!"
     exit 1
   fi
-fi 
+fi
diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md
index ce2961ab26a0d..0ed100731ca1a 100644
--- a/cpp/doc/Parquet.md
+++ b/cpp/doc/Parquet.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
 
-   http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 ## Building Arrow-Parquet integration
diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map
index cc8c9ba3c94bf..49511c6a74934 100644
--- a/cpp/src/arrow/symbols.map
+++ b/cpp/src/arrow/symbols.map
@@ -1,14 +1,19 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 {
   # Symbols marked as 'local' are not exported by the DSO and thus may not
diff --git a/format/Guidelines.md b/format/Guidelines.md
index c75da9f98bebe..ff3a63d9a2fc2 100644
--- a/format/Guidelines.md
+++ b/format/Guidelines.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
 
-   http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 # Implementation guidelines
 
diff --git a/format/IPC.md b/format/IPC.md
index 7d689216d5595..3fd234e4aa160 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Interprocess messaging / communication (IPC)
diff --git a/format/Layout.md b/format/Layout.md
index 1e817ff1375ac..b62b1565a754b 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Arrow: Physical memory layout
diff --git a/format/Metadata.md b/format/Metadata.md
index 18fac527470d5..80ca08ae13fb1 100644
--- a/format/Metadata.md
+++ b/format/Metadata.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Metadata: Logical types, schemas, data headers
diff --git a/format/README.md b/format/README.md
index 3aa8fdd6d4d6e..c87ac2a00d6ea 100644
--- a/format/README.md
+++ b/format/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 ## Arrow specification documents
diff --git a/integration/README.md b/integration/README.md
index 6005b62c41cd5..5b6ea45ff7370 100644
--- a/integration/README.md
+++ b/integration/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Arrow integration testing
diff --git a/java/README.md b/java/README.md
index a57e35afbbd20..dd4f924515606 100644
--- a/java/README.md
+++ b/java/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
 
-   http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 # Arrow Java
@@ -25,4 +30,4 @@ install:
 ```
 cd java
 mvn install
-``` 
+```
diff --git a/js/README.md b/js/README.md
index de9070c59aaa6..38e8fafcea9c5 100644
--- a/js/README.md
+++ b/js/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
 
-   http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 ### Installation
diff --git a/js/flatbuffers.sh b/js/flatbuffers.sh
index 55967f84a9fa9..0f8e3f9fe99a0 100755
--- a/js/flatbuffers.sh
+++ b/js/flatbuffers.sh
@@ -1,16 +1,21 @@
 #!/bin/bash
 
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#  http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 echo "Compiling flatbuffer schemas..."
 mkdir -p lib lib-esm
diff --git a/js/webpack.config.js b/js/webpack.config.js
index b9c3e83a890a2..aa123bd39f9b0 100644
--- a/js/webpack.config.js
+++ b/js/webpack.config.js
@@ -1,14 +1,19 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
 //
-//  http://www.apache.org/licenses/LICENSE-2.0
+//   http://www.apache.org/licenses/LICENSE-2.0
 //
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License. See accompanying LICENSE file.
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 
 var path = require('path');
 var UglifyJSPlugin = require('uglifyjs-webpack-plugin');
diff --git a/python/README.md b/python/README.md
index 816fbf0c85daf..29d213babd93b 100644
--- a/python/README.md
+++ b/python/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 ## Python library for Apache Arrow
diff --git a/python/asv.conf.json b/python/asv.conf.json
index 0c059fd79c1f2..2a1dd42aba136 100644
--- a/python/asv.conf.json
+++ b/python/asv.conf.json
@@ -1,14 +1,19 @@
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
 //
 //   http://www.apache.org/licenses/LICENSE-2.0
 //
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License. See accompanying LICENSE file.
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 
 {
     // The version of the config file format.  Do not change, unless
diff --git a/python/doc/Benchmarks.md b/python/doc/Benchmarks.md
index 1c36801858278..c84bf0dc1eb62 100644
--- a/python/doc/Benchmarks.md
+++ b/python/doc/Benchmarks.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
 
-   http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 ## Benchmark Requirements
 
diff --git a/python/doc/Makefile b/python/doc/Makefile
index 65d6a4df3b20f..1b9f707021a1a 100644
--- a/python/doc/Makefile
+++ b/python/doc/Makefile
@@ -1,15 +1,21 @@
-
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
 # Makefile for Sphinx documentation
 #
 
diff --git a/python/doc/source/conf.py b/python/doc/source/conf.py
index c7f098fc0d5cd..d71b66870729f 100644
--- a/python/doc/source/conf.py
+++ b/python/doc/source/conf.py
@@ -1,16 +1,21 @@
 # -*- coding: utf-8 -*-
 #
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 # This file is execfile()d with the current directory set to its
 # containing dir.
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 13919a2408768..6d72ec7538c0f 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -1,14 +1,19 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
 
 ADD arrow /arrow
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index cdd13e2e93304..0160aa4eea509 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -1,14 +1,19 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 FROM quay.io/pypa/manylinux1_x86_64:latest
 
 # Install dependencies
diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md
index 2e7f56bd6203f..a74f7a27b930a 100644
--- a/python/manylinux1/README.md
+++ b/python/manylinux1/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
 
-   http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/licenses/LICENSE-2.0
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 ## Manylinux1 wheels for Apache Arrow
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 85c096a5c11d0..5725b2a87f45e 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -1,16 +1,21 @@
 #!/bin/bash
 #
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 # Usage:
 #   docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh
index 6a313366494c6..3c11f3aeb6f7e 100755
--- a/python/manylinux1/scripts/build_boost.sh
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -1,16 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
 tar xf boost_1_60_0.tar.gz
diff --git a/python/manylinux1/scripts/build_brotli.sh b/python/manylinux1/scripts/build_brotli.sh
index 4b4cbf17ca9bf..9a1eca7b7809f 100755
--- a/python/manylinux1/scripts/build_brotli.sh
+++ b/python/manylinux1/scripts/build_brotli.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 export BROTLI_VERSION="0.6.0"
 wget "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -O brotli-${BROTLI_VERSION}.tar.gz
diff --git a/python/manylinux1/scripts/build_ccache.sh b/python/manylinux1/scripts/build_ccache.sh
index 6ad5d29f83292..681adecd9a724 100755
--- a/python/manylinux1/scripts/build_ccache.sh
+++ b/python/manylinux1/scripts/build_ccache.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 wget https://www.samba.org/ftp/ccache/ccache-3.3.4.tar.bz2 -O ccache-3.3.4.tar.bz2
 tar xf ccache-3.3.4.tar.bz2
diff --git a/python/manylinux1/scripts/build_flatbuffers.sh b/python/manylinux1/scripts/build_flatbuffers.sh
index 7703855b6efbf..683a89ce5c474 100755
--- a/python/manylinux1/scripts/build_flatbuffers.sh
+++ b/python/manylinux1/scripts/build_flatbuffers.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 wget https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -O flatbuffers-1.6.0.tar.gz
 tar xf flatbuffers-1.6.0.tar.gz
diff --git a/python/manylinux1/scripts/build_gtest.sh b/python/manylinux1/scripts/build_gtest.sh
index 3427bed091ed3..4ce20c1fb44b3 100755
--- a/python/manylinux1/scripts/build_gtest.sh
+++ b/python/manylinux1/scripts/build_gtest.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -O googletest-release-1.7.0.tar.gz
 tar xf googletest-release-1.7.0.tar.gz
diff --git a/python/manylinux1/scripts/build_jemalloc.sh b/python/manylinux1/scripts/build_jemalloc.sh
index 8153baa097e52..1bf1a06b27ed1 100755
--- a/python/manylinux1/scripts/build_jemalloc.sh
+++ b/python/manylinux1/scripts/build_jemalloc.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2
 tar xf jemalloc-4.4.0.tar.bz2
diff --git a/python/manylinux1/scripts/build_lz4.sh b/python/manylinux1/scripts/build_lz4.sh
index 975a3015412c3..8242a5fe27e53 100755
--- a/python/manylinux1/scripts/build_lz4.sh
+++ b/python/manylinux1/scripts/build_lz4.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 export LZ4_VERSION="1.7.5"
 export PREFIX="/usr"
diff --git a/python/manylinux1/scripts/build_openssl.sh b/python/manylinux1/scripts/build_openssl.sh
index 3bcb2b9a053a9..1a54d72f04696 100755
--- a/python/manylinux1/scripts/build_openssl.sh
+++ b/python/manylinux1/scripts/build_openssl.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 wget --no-check-certificate https://www.openssl.org/source/openssl-1.0.2k.tar.gz -O openssl-1.0.2k.tar.gz
 tar xf openssl-1.0.2k.tar.gz
diff --git a/python/manylinux1/scripts/build_snappy.sh b/python/manylinux1/scripts/build_snappy.sh
index 973b4ff7d8089..5392e14a33a45 100755
--- a/python/manylinux1/scripts/build_snappy.sh
+++ b/python/manylinux1/scripts/build_snappy.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 export SNAPPY_VERSION="1.1.3"
 wget "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz" -O snappy-${SNAPPY_VERSION}.tar.gz
diff --git a/python/manylinux1/scripts/build_thrift.sh b/python/manylinux1/scripts/build_thrift.sh
index 1db745855489f..28aa75b7413de 100755
--- a/python/manylinux1/scripts/build_thrift.sh
+++ b/python/manylinux1/scripts/build_thrift.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 export THRIFT_VERSION=0.10.0
 wget http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
index 60d6580de0c21..ddedcf61fde05 100755
--- a/python/manylinux1/scripts/build_virtualenvs.sh
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -e
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
 # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
diff --git a/python/manylinux1/scripts/build_zstd.sh b/python/manylinux1/scripts/build_zstd.sh
index 268e5c8894ccf..ef0e267757eab 100755
--- a/python/manylinux1/scripts/build_zstd.sh
+++ b/python/manylinux1/scripts/build_zstd.sh
@@ -1,15 +1,20 @@
 #!/bin/bash -ex
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 export ZSTD_VERSION="1.2.0"
 export CFLAGS="${CFLAGS} -O3 -fPIC"
diff --git a/python/pyarrow/_config.pyx b/python/pyarrow/_config.pyx
index e5fdbef8af5f6..a2d2d719e68d0 100644
--- a/python/pyarrow/_config.pyx
+++ b/python/pyarrow/_config.pyx
@@ -1,14 +1,19 @@
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License. See accompanying LICENSE file.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 # cython: profile=False
 # distutils: language = c++
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 93d67365c8ac5..a7013ba59986d 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -1,16 +1,19 @@
-# Copyright 2016 Feather Developers
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 import os
 import sys
diff --git a/site/README.md b/site/README.md
index 0e052c84aebd9..1b0a82e03db43 100644
--- a/site/README.md
+++ b/site/README.md
@@ -1,15 +1,20 @@
 <!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
 -->
 
 ## Apache Arrow Website

From b7639c1539af9327483ce17c95dab3b19896b002 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 27 Jul 2017 22:25:34 -0400
Subject: [PATCH 0864/1644] ARROW-1285: [Python] Delete any incomplete file
 when attempt to write single Parquet file fails

cc @jreback

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #902 from wesm/ARROW-1285 and squashes the following commits:

b8f9ef4d [Wes McKinney] Delete any incomplete file when attempt to write single Parquet file fails
---
 python/pyarrow/_parquet.pyx          |  8 ++++----
 python/pyarrow/parquet.py            | 19 ++++++++++++++++---
 python/pyarrow/tests/test_parquet.py | 25 +++++++++++++++++++++++++
 3 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index bbe5203352627..20f189a413ef8 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -542,6 +542,7 @@ cdef ParquetCompression compression_from_name(str name):
 cdef class ParquetWriter:
     cdef:
         unique_ptr[FileWriter] writer
+        shared_ptr[OutputStream] sink
 
     cdef readonly:
         object use_dictionary
@@ -555,14 +556,13 @@ cdef class ParquetWriter:
                   MemoryPool memory_pool=None, use_deprecated_int96_timestamps=False):
         cdef:
             shared_ptr[FileOutputStream] filestream
-            shared_ptr[OutputStream] sink
             shared_ptr[WriterProperties] properties
 
         if isinstance(where, six.string_types):
             check_status(FileOutputStream.Open(tobytes(where), &filestream))
-            sink = <shared_ptr[OutputStream]> filestream
+            self.sink = <shared_ptr[OutputStream]> filestream
         else:
-            get_writer(where, &sink)
+            get_writer(where, &self.sink)
 
         self.use_dictionary = use_dictionary
         self.compression = compression
@@ -582,7 +582,7 @@ cdef class ParquetWriter:
         check_status(
             FileWriter.Open(deref(schema.schema),
                             maybe_unbox_memory_pool(memory_pool),
-                            sink, properties, arrow_properties,
+                            self.sink, properties, arrow_properties,
                             &self.writer))
 
     cdef void _set_int96_support(self, ArrowWriterProperties.Builder* props):
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index fea73978e3ecf..34c1d120ce0d9 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -769,9 +769,22 @@ def write_table(table, where, row_group_size=None, version='1.0',
         compression=compression,
         version=version,
         use_deprecated_int96_timestamps=use_deprecated_int96_timestamps)
-    writer = ParquetWriter(where, table.schema, **options)
-    writer.write_table(table, row_group_size=row_group_size)
-    writer.close()
+
+    writer = None
+    try:
+        writer = ParquetWriter(where, table.schema, **options)
+        writer.write_table(table, row_group_size=row_group_size)
+    except:
+        if writer is not None:
+            writer.close()
+        if isinstance(where, six.string_types):
+            try:
+                os.remove(where)
+            except os.error:
+                pass
+        raise
+    else:
+        writer.close()
 
 
 def write_metadata(schema, where, version='1.0',
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 40e44b352ace3..6763fb3e04f73 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1006,3 +1006,28 @@ def test_multiindex_duplicate_values(tmpdir):
 
     result_df = result_table.to_pandas()
     tm.assert_frame_equal(result_df, df)
+
+
+@parquet
+def test_write_error_deletes_incomplete_file(tmpdir):
+    # ARROW-1285
+    df = pd.DataFrame({'a': list('abc'),
+                       'b': list(range(1, 4)),
+                       'c': np.arange(3, 6).astype('u1'),
+                       'd': np.arange(4.0, 7.0, dtype='float64'),
+                       'e': [True, False, True],
+                       'f': pd.Categorical(list('abc')),
+                       'g': pd.date_range('20130101', periods=3),
+                       'h': pd.date_range('20130101', periods=3,
+                                          tz='US/Eastern'),
+                       'i': pd.date_range('20130101', periods=3, freq='ns')})
+
+    pdf = pa.Table.from_pandas(df)
+
+    filename = tmpdir.join('tmp_file').strpath
+    try:
+        _write_table(pdf, filename)
+    except pa.ArrowException:
+        pass
+
+    assert not os.path.exists(filename)

From ff6c6e0f99a26f0509d1368e3aa8aefd201a5e28 Mon Sep 17 00:00:00 2001
From: Marco Neumann <marco.neumann@blue-yonder.com>
Date: Fri, 28 Jul 2017 10:29:42 -0400
Subject: [PATCH 0865/1644] ARROW-1276: enable parquet serialization of empty
 DataFrames

Fixes ARROW-1276 and fixes Python dev. documentation (encountered during the preparation of this PR).

Author: Marco Neumann <marco.neumann@blue-yonder.com>

Closes #906 from crepererum/ARROW-1276 and squashes the following commits:

1c1c92cd [Marco Neumann] ARROW-1276: enable parquet serialization of empty DataFrames
1d9cc411 [Marco Neumann] add missing conda packages to python dev. doc
---
 python/doc/source/development.rst    |  2 +-
 python/pyarrow/_parquet.pyx          |  5 ++++-
 python/pyarrow/tests/test_parquet.py | 12 ++++++++++++
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index b5aba6c53efb9..55b3efdad17c6 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -84,7 +84,7 @@ from conda-forge:
    conda create -y -q -n pyarrow-dev \
          python=3.6 numpy six setuptools cython pandas pytest \
          cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
-         brotli jemalloc -c conda-forge
+         brotli jemalloc lz4-c zstd -c conda-forge
    source activate pyarrow-dev
 
 
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 20f189a413ef8..aef661818f401 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -629,7 +629,10 @@ cdef class ParquetWriter:
         cdef CTable* ctable = table.table
 
         if row_group_size is None or row_group_size == -1:
-            row_group_size = ctable.num_rows()
+            if ctable.num_rows() > 0:
+                row_group_size = ctable.num_rows()
+            else:
+                row_group_size = 1
         elif row_group_size == 0:
             raise ValueError('Row group size cannot be 0')
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 6763fb3e04f73..7443df87c40c6 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -265,6 +265,18 @@ def test_read_pandas_column_subset(tmpdir):
     tm.assert_frame_equal(df[['strings', 'uint8']], df_read)
 
 
+@parquet
+def test_pandas_parquet_empty_roundtrip(tmpdir):
+    df = _test_dataframe(0)
+    arrow_table = pa.Table.from_pandas(df)
+    imos = pa.BufferOutputStream()
+    _write_table(arrow_table, imos, version="2.0")
+    buf = imos.get_result()
+    reader = pa.BufferReader(buf)
+    df_read = _read_table(reader).to_pandas()
+    tm.assert_frame_equal(df, df_read)
+
+
 @parquet
 def test_pandas_parquet_pyfile_roundtrip(tmpdir):
     filename = tmpdir.join('pandas_pyfile_roundtrip.parquet').strpath

From 8841bc071b1d0a3eff2592af5ca9b5591ed9e5c5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 28 Jul 2017 10:30:57 -0400
Subject: [PATCH 0866/1644] ARROW-1281: [C++/Python] Add Docker setup for
 testing HDFS IO in C++ and Python

We aren't testing this in Travis CI because spinning up an HDFS cluster is a bit heavy weight, but this will at least enable us to do easier ongoing validation that this functionality is working properly.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #895 from wesm/ARROW-1281 and squashes the following commits:

a96e1665 [Wes McKinney] Fix header
4effee78 [Wes McKinney] Fix license header
d12eea48 [Wes McKinney] Fix license headers
591e7c6b [Wes McKinney] Add Python tests
bbbd8c10 [Wes McKinney] Docker HDFS testing scripts, use hdfs-client.xml from Apache HAWQ (incubating)
---
 python/testing/README.md                      |  26 ++
 python/testing/functions.sh                   | 100 ++++++
 python/testing/hdfs/Dockerfile                |  50 +++
 python/testing/hdfs/libhdfs3-hdfs-client.xml  | 332 ++++++++++++++++++
 .../testing/hdfs/restart_docker_container.sh  |  38 ++
 python/testing/hdfs/run_tests.sh              |  41 +++
 python/testing/set_env_common.sh              |  70 ++++
 python/testing/setup_toolchain.sh             |  65 ++++
 python/testing/test_hdfs.sh                   |  25 ++
 9 files changed, 747 insertions(+)
 create mode 100644 python/testing/README.md
 create mode 100644 python/testing/functions.sh
 create mode 100644 python/testing/hdfs/Dockerfile
 create mode 100644 python/testing/hdfs/libhdfs3-hdfs-client.xml
 create mode 100644 python/testing/hdfs/restart_docker_container.sh
 create mode 100755 python/testing/hdfs/run_tests.sh
 create mode 100644 python/testing/set_env_common.sh
 create mode 100644 python/testing/setup_toolchain.sh
 create mode 100755 python/testing/test_hdfs.sh

diff --git a/python/testing/README.md b/python/testing/README.md
new file mode 100644
index 0000000000000..07970a231b54b
--- /dev/null
+++ b/python/testing/README.md
@@ -0,0 +1,26 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Testing tools for odds and ends
+
+## Testing HDFS file interface
+
+```shell
+./test_hdfs.sh
+```
\ No newline at end of file
diff --git a/python/testing/functions.sh b/python/testing/functions.sh
new file mode 100644
index 0000000000000..6bc342bd79437
--- /dev/null
+++ b/python/testing/functions.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+use_gcc() {
+  export CC=gcc-4.9
+  export CXX=g++-4.9
+}
+
+use_clang() {
+  export CC=clang-4.0
+  export CXX=clang++-4.0
+}
+
+bootstrap_python_env() {
+  PYTHON_VERSION=$1
+  CONDA_ENV_DIR=$BUILD_DIR/pyarrow-test-$PYTHON_VERSION
+
+  conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
+  source activate $CONDA_ENV_DIR
+
+  python --version
+  which python
+
+  # faster builds, please
+  conda install -y -q nomkl pip numpy pandas cython
+}
+
+build_pyarrow() {
+  # Other stuff pip install
+  pushd $ARROW_PYTHON_DIR
+  pip install -r requirements.txt
+  python setup.py build_ext --with-parquet --with-plasma \
+         install --single-version-externally-managed --record=record.text
+  popd
+
+  python -c "import pyarrow.parquet"
+  python -c "import pyarrow.plasma"
+
+  export PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
+}
+
+build_arrow() {
+  mkdir -p $ARROW_CPP_BUILD_DIR
+  pushd $ARROW_CPP_BUILD_DIR
+
+  cmake -GNinja \
+        -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+        -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+        -DARROW_NO_DEPRECATED_API=ON \
+        -DARROW_PYTHON=ON \
+        -DARROW_PLASMA=ON \
+        -DARROW_BOOST_USE_SHARED=off \
+        $ARROW_CPP_DIR
+
+  ninja
+  ninja install
+  popd
+}
+
+build_parquet() {
+  PARQUET_DIR=$BUILD_DIR/parquet
+  mkdir -p $PARQUET_DIR
+
+  git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
+
+  pushd $PARQUET_DIR
+  mkdir build-dir
+  cd build-dir
+
+  cmake \
+      -GNinja \
+      -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+      -DPARQUET_BOOST_USE_SHARED=off \
+      -DPARQUET_BUILD_BENCHMARKS=off \
+      -DPARQUET_BUILD_EXECUTABLES=off \
+      -DPARQUET_BUILD_TESTS=off \
+      ..
+
+  ninja
+  ninja install
+
+  popd
+}
diff --git a/python/testing/hdfs/Dockerfile b/python/testing/hdfs/Dockerfile
new file mode 100644
index 0000000000000..97355137ff35b
--- /dev/null
+++ b/python/testing/hdfs/Dockerfile
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# TODO Replace this with a complete clean image build
+FROM cpcloud86/impala:metastore
+
+USER root
+
+RUN apt-add-repository -y ppa:ubuntu-toolchain-r/test && \
+    apt-get update && \
+    apt-get install -y \
+            gcc-4.9 \
+            g++-4.9 \
+            build-essential \
+            autotools-dev \
+            autoconf \
+            gtk-doc-tools \
+            autoconf-archive \
+            libgirepository1.0-dev \
+            libtool \
+            libjemalloc-dev \
+            ccache \
+            valgrind \
+            gdb
+
+RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add - && \
+    apt-add-repository -y \
+     "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-4.0 main" && \
+    apt-get update && \
+    apt-get install -y clang-4.0 clang-format-4.0 clang-tidy-4.0
+
+USER ubuntu
+
+RUN wget -O /tmp/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
+    rm /tmp/miniconda.sh
diff --git a/python/testing/hdfs/libhdfs3-hdfs-client.xml b/python/testing/hdfs/libhdfs3-hdfs-client.xml
new file mode 100644
index 0000000000000..f929929b386da
--- /dev/null
+++ b/python/testing/hdfs/libhdfs3-hdfs-client.xml
@@ -0,0 +1,332 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<!-- From Apache HAWQ (incubating) -->
+
+<configuration>
+
+<!-- KDC
+	<property>
+		<name>hadoop.security.authentication</name>
+		<value>kerberos</value>
+	</property>
+KDC -->
+
+<!-- HA
+	<property>
+		<name>dfs.nameservices</name>
+		<value>phdcluster</value>
+	</property>
+
+	<property>
+		<name>dfs.ha.namenodes.phdcluster</name>
+		<value>nn1,nn2</value>
+	</property>6
+
+	<property>
+		<name>dfs.namenode.rpc-address.phdcluster.nn1</name>
+		<value>mdw:9000</value>
+	</property>
+
+	<property>
+		<name>dfs.namenode.rpc-address.phdcluster.nn2</name>
+		<value>smdw:9000</value>
+	</property>
+
+	<property>
+		<name>dfs.namenode.http-address.phdcluster.nn1</name>
+		<value>mdw:50070</value>
+	</property>
+
+	<property>
+		<name>dfs.namenode.http-address.phdcluster.nn2</name>
+		<value>smdw:50070</value>
+	</property>
+
+HA -->
+
+	<!-- RPC client configuration -->
+	<property>
+		<name>rpc.client.timeout</name>
+		<value>3600000</value>
+		<description>
+		timeout interval of a RPC invocation in millisecond. default is 3600000.
+		</description>
+	</property>
+	<property>
+		<name>rpc.client.connect.tcpnodelay</name>
+		<value>true</value>
+		<description>
+		whether set socket TCP_NODELAY to true when connect to RPC server. default is true.
+		</description>
+	</property>
+
+	<property>
+		<name>rpc.client.max.idle</name>
+		<value>10000</value>
+		<description>
+		the max idle time of a RPC connection in millisecond. default is 10000.
+		</description>
+	</property>
+
+	<property>
+		<name>rpc.client.ping.interval</name>
+		<value>10000</value>
+		<description>
+		the interval which the RPC client send a heart beat to server. 0 means disable, default is 10000.
+		</description>
+	</property>
+
+	<property>
+		<name>rpc.client.connect.timeout</name>
+		<value>600000</value>
+		<description>
+		the timeout interval in millisecond when the RPC client is trying to setup the connection. default is 600000.
+		</description>
+	</property>
+
+	<property>
+		<name>rpc.client.connect.retry</name>
+		<value>10</value>
+		<description>
+		the max retry times if the RPC client fail to setup the connection to server. default is 10.
+		</description>
+	</property>
+
+	<property>
+		<name>rpc.client.read.timeout</name>
+		<value>3600000</value>
+		<description>
+		the timeout interval in millisecond when the RPC client is trying to read from server. default is 3600000.
+		</description>
+	</property>
+
+	<property>
+		<name>rpc.client.write.timeout</name>
+		<value>3600000</value>
+		<description>
+		the timeout interval in millisecond when the RPC client is trying to write to server. default is 3600000.
+		</description>
+	</property>
+
+	<property>
+		<name>rpc.client.socket.linger.timeout</name>
+		<value>-1</value>
+		<description>
+		set value to socket SO_LINGER when connect to RPC server. -1 means default OS value. default is -1.
+		</description>
+	</property>
+
+	<!-- dfs client configuration -->
+	<property>
+		<name>dfs.client.read.shortcircuit</name>
+		<value>false</value>
+		<description>
+		whether reading block file bypass datanode if the block and the client are on the same node. default is true.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.default.replica</name>
+		<value>1</value>
+		<description>
+		the default number of replica. default is 3.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.prefetchsize</name>
+		<value>10</value>
+		<description>
+		the default number of blocks which information will be prefetched. default is 10.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.client.failover.max.attempts</name>
+		<value>15</value>
+		<description>
+		if multiply namenodes are configured, it is the max retry times when the dfs client try to issue a RPC call. default is 15.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.default.blocksize</name>
+		<value>134217728</value>
+		<description>
+		default block size. default is 134217728.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.client.log.severity</name>
+		<value>INFO</value>
+		<description>
+		the minimal log severity level, valid values include FATAL, ERROR, INFO, DEBUG1, DEBUG2, DEBUG3. default is INFO.
+		</description>
+	</property>
+
+	<!-- input client configuration -->
+	<property>
+		<name>input.connect.timeout</name>
+		<value>600000</value>
+		<description>
+		the timeout interval in millisecond when the input stream is trying to setup the connection to datanode. default is 600000.
+		</description>
+	</property>
+
+	<property>
+		<name>input.read.timeout</name>
+		<value>3600000</value>
+		<description>
+		the timeout interval in millisecond when the input stream is trying to read from datanode. default is 3600000.
+		</description>
+	</property>
+
+	<property>
+		<name>input.write.timeout</name>
+		<value>3600000</value>
+		<description>
+		the timeout interval in millisecond when the input stream is trying to write to datanode. default is 3600000.
+		</description>
+	</property>
+
+	<property>
+		<name>input.localread.default.buffersize</name>
+		<value>2097152</value>
+		<description>
+		number of bytes of the buffer which is used to hold the data from block file and verify checksum.
+		it is only used when "dfs.client.read.shortcircuit" is set to true. default is 1048576.
+		</description>
+	</property>
+
+	<property>
+		<name>input.localread.blockinfo.cachesize</name>
+		<value>1000</value>
+		<description>
+		the size of block file path information cache. default is 1000.
+		</description>
+	</property>
+
+	<property>
+		<name>input.read.getblockinfo.retry</name>
+		<value>3</value>
+		<description>
+		the max retry times when the client fail to get block information from namenode. default is 3.
+		</description>
+	</property>
+
+	<!-- output client configuration -->
+	<property>
+		<name>output.replace-datanode-on-failure</name>
+		<value>false</value>
+		<description>
+		whether the client add new datanode into pipeline if the number of nodes in pipeline is less the specified number of replicas. default is false.
+		</description>
+	</property>
+
+	<property>
+		<name>output.default.chunksize</name>
+		<value>512</value>
+		<description>
+		the number of bytes of a chunk in pipeline. default is 512.
+		</description>
+	</property>
+
+	<property>
+		<name>output.default.packetsize</name>
+		<value>65536</value>
+		<description>
+		the number of bytes of a packet in pipeline. default is 65536.
+		</description>
+	</property>
+
+	<property>
+		<name>output.default.write.retry</name>
+		<value>10</value>
+		<description>
+		the max retry times when the client fail to setup the pipeline. default is 10.
+		</description>
+	</property>
+
+	<property>
+		<name>output.connect.timeout</name>
+		<value>600000</value>
+		<description>
+		the timeout interval in millisecond when the output stream is trying to setup the connection to datanode. default is 600000.
+		</description>
+	</property>
+
+	<property>
+		<name>output.read.timeout</name>
+		<value>3600000</value>
+		<description>
+		the timeout interval in millisecond when the output stream is trying to read from datanode. default is 3600000.
+		</description>
+	</property>
+
+	<property>
+		<name>output.write.timeout</name>
+		<value>3600000</value>
+		<description>
+		the timeout interval in millisecond when the output stream is trying to write to datanode. default is 3600000.
+		</description>
+	</property>
+
+	<property>
+		<name>output.packetpool.size</name>
+		<value>1024</value>
+		<description>
+		the max number of packets in a file's packet pool. default is 1024.
+		</description>
+	</property>
+
+	<property>
+		<name>output.close.timeout</name>
+		<value>900000</value>
+		<description>
+		the timeout interval in millisecond when close an output stream. default is 900000.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.domain.socket.path</name>
+		<value>/var/lib/hadoop-hdfs/dn_socket</value>
+		<description>
+		Optional.  This is a path to a UNIX domain socket that will be used for
+		communication between the DataNode and local HDFS clients.
+		If the string "_PORT" is present in this path, it will be replaced by the
+		TCP port of the DataNode.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.client.use.legacy.blockreader.local</name>
+		<value>false</value>
+		<description>
+		Legacy short-circuit reader implementation based on HDFS-2246 is used
+		if this configuration parameter is true.
+		This is for the platforms other than Linux
+		where the new implementation based on HDFS-347 is not available.
+		</description>
+	</property>
+
+</configuration>
diff --git a/python/testing/hdfs/restart_docker_container.sh b/python/testing/hdfs/restart_docker_container.sh
new file mode 100644
index 0000000000000..15076cc287387
--- /dev/null
+++ b/python/testing/hdfs/restart_docker_container.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export ARROW_TEST_NN_HOST=arrow-hdfs
+export ARROW_TEST_IMPALA_HOST=$ARROW_TEST_NN_HOST
+export ARROW_TEST_IMPALA_PORT=21050
+export ARROW_TEST_WEBHDFS_PORT=50070
+export ARROW_TEST_WEBHDFS_USER=ubuntu
+
+docker stop $ARROW_TEST_NN_HOST
+docker rm $ARROW_TEST_NN_HOST
+
+docker run -d -it --name $ARROW_TEST_NN_HOST \
+       -v $PWD:/io \
+       --hostname $ARROW_TEST_NN_HOST \
+       --shm-size=2gb \
+       -p $ARROW_TEST_WEBHDFS_PORT -p $ARROW_TEST_IMPALA_PORT \
+       arrow-hdfs-test
+
+while ! docker exec $ARROW_TEST_NN_HOST impala-shell -q 'SELECT VERSION()'; do
+    sleep 1
+done
diff --git a/python/testing/hdfs/run_tests.sh b/python/testing/hdfs/run_tests.sh
new file mode 100755
index 0000000000000..e0d36df58a300
--- /dev/null
+++ b/python/testing/hdfs/run_tests.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
+
+source $HERE/../set_env_common.sh
+source $HERE/../setup_toolchain.sh
+source $HERE/../functions.sh
+
+git clone https://github.com/apache/arrow.git $ARROW_CHECKOUT
+
+use_clang
+
+bootstrap_python_env 3.6
+
+build_arrow
+build_parquet
+
+build_pyarrow
+
+$ARROW_CPP_BUILD_DIR/debug/io-hdfs-test
+
+python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet --hdfs
diff --git a/python/testing/set_env_common.sh b/python/testing/set_env_common.sh
new file mode 100644
index 0000000000000..00251f92be4b4
--- /dev/null
+++ b/python/testing/set_env_common.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export MINICONDA=$HOME/miniconda
+export CPP_TOOLCHAIN=$HOME/cpp-toolchain
+
+export PATH="$MINICONDA/bin:$PATH"
+export CONDA_PKGS_DIRS=$HOME/.conda_packages
+
+export ARROW_CHECKOUT=$HOME/arrow
+export BUILD_DIR=$ARROW_CHECKOUT
+
+export BUILD_OS_NAME=linux
+export BUILD_TYPE=debug
+
+export ARROW_CPP_DIR=$BUILD_DIR/cpp
+export ARROW_PYTHON_DIR=$BUILD_DIR/python
+export ARROW_C_GLIB_DIR=$BUILD_DIR/c_glib
+export ARROW_JAVA_DIR=${BUILD_DIR}/java
+export ARROW_JS_DIR=${BUILD_DIR}/js
+export ARROW_INTEGRATION_DIR=$BUILD_DIR/integration
+
+export CPP_BUILD_DIR=$BUILD_DIR/cpp-build
+
+export ARROW_CPP_INSTALL=$BUILD_DIR/cpp-install
+export ARROW_CPP_BUILD_DIR=$BUILD_DIR/cpp-build
+export ARROW_C_GLIB_INSTALL=$BUILD_DIR/c-glib-install
+
+export ARROW_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+
+export BOOST_ROOT=$CPP_TOOLCHAIN
+export PATH=$CPP_TOOLCHAIN/bin:$PATH
+export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
+
+export VALGRIND="valgrind --tool=memcheck"
+
+export ARROW_HOME=$CPP_TOOLCHAIN
+export PARQUET_HOME=$CPP_TOOLCHAIN
+
+# Arrow test variables
+
+export JAVA_HOME=/usr/lib/jvm/java-7-oracle
+export HADOOP_HOME=/usr/lib/hadoop
+export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
+export HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$HADOOP_HOME/lib/native"
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native/
+
+export ARROW_HDFS_TEST_HOST=arrow-hdfs
+export ARROW_HDFS_TEST_PORT=9000
+export ARROW_HDFS_TEST_USER=ubuntu
+export ARROW_LIBHDFS_DIR=/usr/lib
+
+export LIBHDFS3_CONF=/io/hdfs/libhdfs3-hdfs-client.xml
diff --git a/python/testing/setup_toolchain.sh b/python/testing/setup_toolchain.sh
new file mode 100644
index 0000000000000..c3837b45cbcd4
--- /dev/null
+++ b/python/testing/setup_toolchain.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+export PATH="$MINICONDA/bin:$PATH"
+conda update -y -q conda
+conda config --set auto_update_conda false
+conda info -a
+
+conda config --set show_channel_urls True
+
+# Help with SSL timeouts to S3
+conda config --set remote_connect_timeout_secs 12
+
+conda config --add channels https://repo.continuum.io/pkgs/free
+conda config --add channels conda-forge
+conda info -a
+
+# faster builds, please
+conda install -y nomkl
+
+conda install --y conda-build jinja2 anaconda-client cmake curl
+
+# Set up C++ toolchain
+conda create -y -q -p $CPP_TOOLCHAIN python=3.6 \
+    jemalloc=4.4.0 \
+    nomkl \
+    boost-cpp \
+    rapidjson \
+    flatbuffers \
+    gflags \
+    lz4-c \
+    snappy \
+    zstd \
+    brotli \
+    zlib \
+    git \
+    cmake \
+    curl \
+    thrift-cpp \
+    libhdfs3 \
+    ninja
+
+if [ $BUILD_OS_NAME == "osx" ]; then
+  brew update > /dev/null
+  brew install jemalloc
+  brew install ccache
+fi
diff --git a/python/testing/test_hdfs.sh b/python/testing/test_hdfs.sh
new file mode 100755
index 0000000000000..016e54a66a671
--- /dev/null
+++ b/python/testing/test_hdfs.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+docker build -t arrow-hdfs-test -f hdfs/Dockerfile .
+bash hdfs/restart_docker_container.sh
+docker exec -it arrow-hdfs /io/hdfs/run_tests.sh
+docker stop arrow-hdfs

From 33c85cd0396bd7a76b81828630b1d35ceb7b970a Mon Sep 17 00:00:00 2001
From: Stepan Kadlec <stepan.kadlec@oracle.com>
Date: Thu, 27 Jul 2017 15:50:11 +0200
Subject: [PATCH 0867/1644] [Java] Fix letter case in rat plugin config

Contributed by Stepan Kadlec <stepan.kadlec@oracle.com>

Closes #899

Change-Id: I3ad6cb9cc30946fda76e2dd454e0e19966276abc
---
 java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/pom.xml b/java/pom.xml
index 81f80b00b563c..de2113e397e15 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -93,7 +93,7 @@
           </execution>
         </executions>
         <configuration>
-          <excludeSubprojects>false</excludeSubprojects>
+          <excludeSubProjects>false</excludeSubProjects>
           <excludes>
             <exclude>**/*.log</exclude>
             <exclude>**/*.css</exclude>

From 4df2a0bfa15ac28f794a18543801a0e79194617b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 28 Jul 2017 11:13:39 -0400
Subject: [PATCH 0868/1644] ARROW-1290: [C++] Double buffer size when exceeding
 capacity in arrow::BufferBuilder as in array builders

Kind of an embarrassing oversight, but it's good that we caught it.

In a test for incrementally building a BinaryArray, this yields about 4x speedup

```
Benchmark                                     Time           CPU Iterations
---------------------------------------------------------------------------
BM_BuildBinaryArray/repeats:3             11892 us      11892 us         59   840.886MB/s
BM_BuildBinaryArray/repeats:3             11903 us      11904 us         59   840.082MB/s
BM_BuildBinaryArray/repeats:3             11909 us      11910 us         59   839.662MB/s
BM_BuildBinaryArray/repeats:3_mean        11902 us      11902 us         59    840.21MB/s
BM_BuildBinaryArray/repeats:3_stddev          7 us          7 us          0   520.137kB/s
```

before:

```
Benchmark                                     Time           CPU Iterations
---------------------------------------------------------------------------
BM_BuildBinaryArray/repeats:3             45678 us      45571 us         15   219.439MB/s
BM_BuildBinaryArray/repeats:3             45416 us      45209 us         15   221.197MB/s
BM_BuildBinaryArray/repeats:3             45227 us      45122 us         15   221.619MB/s
BM_BuildBinaryArray/repeats:3_mean        45440 us      45301 us         15   220.752MB/s
BM_BuildBinaryArray/repeats:3_stddev        185 us        194 us          0   966.716kB/s
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #905 from wesm/ARROW-1290 and squashes the following commits:

59d4d9cd [Wes McKinney] Double buffer size when exceeding capacity in arrow::BufferBuilder, like in other array builder classes
---
 cpp/src/arrow/buffer.h             |  7 +++++--
 cpp/src/arrow/builder-benchmark.cc | 18 ++++++++++++++++++
 cpp/src/arrow/python/config.cc     |  1 +
 cpp/src/plasma/store.cc            |  5 +++--
 4 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 09e539d162fb2..5d050b77f77a8 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -25,6 +25,7 @@
 #include <string>
 
 #include "arrow/status.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -204,7 +205,8 @@ class ARROW_EXPORT BufferBuilder {
 
   Status Append(const uint8_t* data, int64_t length) {
     if (capacity_ < length + size_) {
-      RETURN_NOT_OK(Resize(length + size_));
+      int64_t new_capacity = BitUtil::NextPower2(length + size_);
+      RETURN_NOT_OK(Resize(new_capacity));
     }
     UnsafeAppend(data, length);
     return Status::OK();
@@ -213,7 +215,8 @@ class ARROW_EXPORT BufferBuilder {
   // Advance pointer and zero out memory
   Status Advance(int64_t length) {
     if (capacity_ < length + size_) {
-      RETURN_NOT_OK(Resize(length + size_));
+      int64_t new_capacity = BitUtil::NextPower2(length + size_);
+      RETURN_NOT_OK(Resize(new_capacity));
     }
     memset(data_ + size_, 0, static_cast<size_t>(length));
     size_ += length;
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index 8ba9360e917fc..13d7b20591dad 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -156,6 +156,22 @@ static void BM_BuildStringDictionary(
                           sizeof(int32_t));
 }
 
+static void BM_BuildBinaryArray(benchmark::State& state) {  // NOLINT non-const reference
+  const int64_t iterations = 1 << 20;
+
+  std::string value = "1234567890";
+  while (state.KeepRunning()) {
+    BinaryBuilder builder(default_memory_pool());
+    for (int64_t i = 0; i < iterations; i++) {
+      ABORT_NOT_OK(builder.Append(value));
+    }
+    std::shared_ptr<Array> out;
+    ABORT_NOT_OK(builder.Finish(&out));
+  }
+  // Assuming a string here needs on average 2 bytes
+  state.SetBytesProcessed(state.iterations() * iterations * value.size());
+}
+
 BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildVectorNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildAdaptiveIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
@@ -166,4 +182,6 @@ BENCHMARK(BM_BuildAdaptiveUIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrose
 BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 
+BENCHMARK(BM_BuildBinaryArray)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/config.cc b/cpp/src/arrow/python/config.cc
index 92ca9db9cc391..bda7a7af163d6 100644
--- a/cpp/src/arrow/python/config.cc
+++ b/cpp/src/arrow/python/config.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include "arrow/python/platform.h"
+
 #include "arrow/python/config.h"
 
 namespace arrow {
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 34adc6261ebe0..a9425b6f7d907 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -690,8 +690,9 @@ int main(int argc, char* argv[]) {
   close(shm_fd);
   if (system_memory > shm_mem_avail) {
     ARROW_LOG(FATAL) << "System memory request exceeds memory available in /dev/shm. The "
-                        "request is for " << system_memory
-                     << " bytes, and the amount available is " << shm_mem_avail
+                        "request is for "
+                     << system_memory << " bytes, and the amount available is "
+                     << shm_mem_avail
                      << " bytes. You may be able to free up space by deleting files in "
                         "/dev/shm. If you are inside a Docker container, you may need to "
                         "pass "

From 44855bb16312031a6d4285632d0071c676ef38aa Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 28 Jul 2017 11:14:35 -0400
Subject: [PATCH 0869/1644] ARROW-1273: [Python] Add Parquet read_metadata,
 read_schema convenience functions

cc @xhochy @cpcloud for feedback on API

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #904 from wesm/ARROW-1273 and squashes the following commits:

13725654 [Wes McKinney] Add Parquet read_metadata, read_schema convenience functions
---
 python/doc/source/api.rst            |  3 +++
 python/pyarrow/parquet.py            | 30 +++++++++++++++++++++++++++
 python/pyarrow/tests/test_parquet.py | 31 +++++++++++++++++++++-------
 3 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 6554465466a55..b84163b8f75c1 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -239,5 +239,8 @@ Apache Parquet
    ParquetDataset
    ParquetFile
    read_table
+   read_metadata
+   read_pandas
+   read_schema
    write_metadata
    write_table
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 34c1d120ce0d9..a3af9ae75186e 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -805,3 +805,33 @@ def write_metadata(schema, where, version='1.0',
     )
     writer = ParquetWriter(where, schema, **options)
     writer.close()
+
+
+def read_metadata(where):
+    """
+    Read FileMetadata from footer of a single Parquet file
+
+    Parameters
+    ----------
+    where : string (filepath) or file-like object
+
+    Returns
+    -------
+    metadata : FileMetadata
+    """
+    return ParquetFile(where).metadata
+
+
+def read_schema(where):
+    """
+    Read effective Arrow schema from Parquet file metadata
+
+    Parameters
+    ----------
+    where : string (filepath) or file-like object
+
+    Returns
+    -------
+    schema : pyarrow.Schema
+    """
+    return ParquetFile(where).schema.to_arrow_schema()
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 7443df87c40c6..f840673f61c49 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -124,9 +124,8 @@ def test_pandas_parquet_custom_metadata(tmpdir):
     assert b'pandas' in arrow_table.schema.metadata
 
     _write_table(arrow_table, filename.strpath, version="2.0")
-    pf = pq.ParquetFile(filename.strpath)
 
-    md = pf.metadata.metadata
+    md = pq.read_metadata(filename.strpath).metadata
     assert b'pandas' in md
 
     js = json.loads(md[b'pandas'].decode('utf8'))
@@ -592,7 +591,7 @@ def test_pass_separate_metadata():
     _write_table(a_table, buf, compression='snappy', version='2.0')
 
     buf.seek(0)
-    metadata = pq.ParquetFile(buf).metadata
+    metadata = pq.read_metadata(buf)
 
     buf.seek(0)
 
@@ -788,14 +787,32 @@ def test_read_common_metadata_files(tmpdir):
     dataset = pq.ParquetDataset(base_path)
     assert dataset.metadata_path == metadata_path
 
-    pf = pq.ParquetFile(data_path)
-    assert dataset.schema.equals(pf.schema)
+    common_schema = pq.read_metadata(data_path).schema
+    assert dataset.schema.equals(common_schema)
 
     # handle list of one directory
     dataset2 = pq.ParquetDataset([base_path])
     assert dataset2.schema.equals(dataset.schema)
 
 
+@parquet
+def test_read_schema(tmpdir):
+    import pyarrow.parquet as pq
+
+    N = 100
+    df = pd.DataFrame({
+        'index': np.arange(N),
+        'values': np.random.randn(N)
+    }, columns=['index', 'values'])
+
+    data_path = pjoin(str(tmpdir), 'test.parquet')
+
+    table = pa.Table.from_pandas(df)
+    _write_table(table, data_path)
+
+    assert table.schema.equals(pq.read_schema(data_path))
+
+
 def _filter_partition(df, part_keys):
     predicate = np.ones(len(df), dtype=bool)
 
@@ -847,7 +864,7 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
     assert result.equals(expected)
 
     # Read with provided metadata
-    metadata = pq.ParquetFile(paths[0]).metadata
+    metadata = pq.read_metadata(paths[0])
 
     result2 = read_multiple_files(paths, metadata=metadata)
     assert result2.equals(expected)
@@ -873,7 +890,7 @@ def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
     t = pa.Table.from_pandas(bad_apple)
     _write_table(t, bad_apple_path)
 
-    bad_meta = pq.ParquetFile(bad_apple_path).metadata
+    bad_meta = pq.read_metadata(bad_apple_path)
 
     with pytest.raises(ValueError):
         read_multiple_files(paths + [bad_apple_path])

From 3b14765e89b2ac58c6c5212f6f90859a17f5ea22 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 28 Jul 2017 12:11:51 -0400
Subject: [PATCH 0870/1644] ARROW-1289: [Python] Add PYARROW_BUILD_PLASMA CMake
 option, follow semantics of --with-parquet

Now the test suite does not fail if you build the Plasma libraries but forget to pass `--with-plasma` to the Python build

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #903 from wesm/ARROW-1289 and squashes the following commits:

0e9ce78b [Wes McKinney] Add PYARROW_BUILD_PLASMA CMake option, make plasma build work like parquet build
---
 python/CMakeLists.txt                  | 44 +++++++++++++++-----------
 python/cmake_modules/FindParquet.cmake |  2 ++
 python/setup.py                        | 16 +++++++---
 3 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 71ce163e4949f..846e4dd5f6eaf 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -51,6 +51,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUILD_PARQUET
     "Build the PyArrow Parquet integration"
     OFF)
+  option(PYARROW_BUILD_PLASMA
+    "Build the PyArrow Plasma integration"
+    OFF)
   option(PYARROW_BUNDLE_ARROW_CPP
     "Bundle the Arrow C++ libraries"
     OFF)
@@ -157,12 +160,6 @@ include_directories(SYSTEM
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 
-## Plasma
-find_package(Plasma)
-if (PLASMA_FOUND)
-  include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
-endif()
-
 function(bundle_arrow_lib library_path)
   get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY)
   get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
@@ -195,9 +192,6 @@ if (PYARROW_BUNDLE_ARROW_CPP)
   file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
   bundle_arrow_lib(ARROW_SHARED_LIB)
   bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
-  if (PLASMA_FOUND)
-    bundle_arrow_lib(PLASMA_SHARED_LIB)
-  endif()
 endif()
 
 if (MSVC)
@@ -224,14 +218,9 @@ set(CYTHON_EXTENSIONS
   lib
 )
 
-if (PLASMA_FOUND)
-  set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} plasma)
-endif()
-
 set(LINK_LIBS
   arrow_shared
   arrow_python_shared
-  ${PLASMA_SHARED_LIB}
 )
 
 if (PYARROW_BUILD_PARQUET)
@@ -284,6 +273,29 @@ if (PYARROW_BUILD_PARQUET)
     _parquet)
 endif()
 
+## Plasma
+if (PYARROW_BUILD_PLASMA)
+  find_package(Plasma)
+
+  if(NOT PLASMA_FOUND)
+    message(FATAL_ERROR "Unable to locate Plasma libraries")
+  endif()
+
+  include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(libplasma
+    SHARED_LIB ${PLASMA_SHARED_LIB})
+
+  if (PYARROW_BUNDLE_ARROW_CPP)
+    bundle_arrow_lib(PLASMA_SHARED_LIB)
+  endif()
+  set(LINK_LIBS
+    ${LINK_LIBS}
+    libplasma_shared)
+
+  set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} plasma)
+  file(COPY ${PLASMA_EXECUTABLE} DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY})
+endif()
+
 ############################################################
 # Setup and build Cython modules
 ############################################################
@@ -330,7 +342,3 @@ foreach(module ${CYTHON_EXTENSIONS})
 
     target_link_libraries(${module_name} ${LINK_LIBS})
 endforeach(module)
-
-if (PLASMA_FOUND)
-  file(COPY ${PLASMA_EXECUTABLE} DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY})
-endif()
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index 88dca2ed6464c..0339ec56ae20a 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -60,6 +60,8 @@ if(PARQUET_HOME)
         PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
         PATH_SUFFIXES "lib")
     get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
+    set(PARQUET_ABI_VERSION "1.0.0")
+    set(PARQUET_SO_VERSION "1")
 else()
     pkg_check_modules(PARQUET parquet)
     if (PARQUET_FOUND)
diff --git a/python/setup.py b/python/setup.py
index be0e26b92a876..801cd17f440ec 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -143,14 +143,18 @@ def _run_cmake(self):
         if self.with_parquet:
             cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
 
+        if self.with_plasma:
+            cmake_options.append('-DPYARROW_BUILD_PLASMA=on')
+
         if self.bundle_arrow_cpp:
             cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
             # ARROW-1090: work around CMake rough edges
             if 'ARROW_HOME' in os.environ and sys.platform != 'win32':
-                os.environ['PKG_CONFIG_PATH'] = pjoin(os.environ['ARROW_HOME'], 'lib', 'pkgconfig')
+                pkg_config = pjoin(os.environ['ARROW_HOME'], 'lib',
+                                   'pkgconfig')
+                os.environ['PKG_CONFIG_PATH'] = pkg_config
                 del os.environ['ARROW_HOME']
 
-
         cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                              .format(self.build_type.lower()))
 
@@ -243,7 +247,8 @@ def move_lib(lib_name):
             print(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
             if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
                 shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
-            shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
+            shutil.move(pjoin(build_prefix, 'include'),
+                        pjoin(build_lib, 'pyarrow'))
             move_lib("arrow")
             move_lib("arrow_python")
             if self.with_plasma:
@@ -280,7 +285,9 @@ def move_lib(lib_name):
         if self.with_plasma:
             build_py = self.get_finalized_command('build_py')
             source = os.path.join(self.build_type, "plasma_store")
-            target = os.path.join(build_lib, build_py.get_package_dir('pyarrow'), "plasma_store")
+            target = os.path.join(build_lib,
+                                  build_py.get_package_dir('pyarrow'),
+                                  "plasma_store")
             shutil.move(source, target)
 
         os.chdir(saved_cwd)
@@ -350,6 +357,7 @@ def get_outputs(self):
 language-bindings for structure manipulation. It also provides IPC
 and common algorithm implementations."""
 
+
 class BinaryDistribution(Distribution):
     def has_ext_modules(foo):
         return True

From 1dd0f5f580bd751d6e10879775d9441358c4fe66 Mon Sep 17 00:00:00 2001
From: Steven Phillips <steven@dremio.com>
Date: Fri, 28 Jul 2017 17:09:54 -0700
Subject: [PATCH 0871/1644] ARROW-1267: [Java] Handle zero length case in
 BitVector.splitAndTransfer

@jacques-n
@StevenMPhillips

PR for the change: https://github.com/dremio/arrow/commit/b794dfa5fe209cf8e3c17cb828964a0a0863c1f8

A new unit test has been added on top of original change.

Author: Steven Phillips <steven@dremio.com>
Author: siddharth <siddharth@dremio.com>

Closes #890 from siddharthteotia/ARROW-1267-PR and squashes the following commits:

89a08d9 [siddharth] Handle zero length in BitVector.splitAndTransfer
84946b7 [Steven Phillips] Handle zero length case in BitVector.splitAndTransfer()
b55c146 [Steven Phillips] Handle zero length case in BitVector.splitAndTransfer()
---
 .../org/apache/arrow/vector/BitVector.java    | 52 ++++++------
 .../apache/arrow/vector/TestBitVector.java    | 80 ++++++++++++++++++-
 2 files changed, 106 insertions(+), 26 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 82cbd47d75816..f34ef2c2a2244 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -261,32 +261,34 @@ public void splitAndTransferTo(int startIndex, int length, BitVector target) {
     int firstByte = getByteIndex(startIndex);
     int byteSize = getSizeFromCount(length);
     int offset = startIndex % 8;
-    if (offset == 0) {
-      target.clear();
-      // slice
-      if (target.data != null) {
-        target.data.release();
-      }
-      target.data = data.slice(firstByte, byteSize);
-      target.data.retain(1);
-    } else {
-      // Copy data
-      // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
-      // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
-      // The last byte copied to target is a bit tricky :
-      //   1) if length requires partly byte (length % 8 !=0), copy the remaining bits only.
-      //   2) otherwise, copy the last byte in the same way as to the prior bytes.
-      target.clear();
-      target.allocateNew(length);
-      // TODO maybe do this one word at a time, rather than byte?
-      for(int i = 0; i < byteSize - 1; i++) {
-        target.data.setByte(i, (((this.data.getByte(firstByte + i) & 0xFF) >>> offset) + (this.data.getByte(firstByte + i + 1) <<  (8 - offset))));
-      }
-      if (length % 8 != 0) {
-        target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
+    if (length > 0) {
+      if (offset == 0) {
+        target.clear();
+        // slice
+        if (target.data != null) {
+          target.data.release();
+        }
+        target.data = data.slice(firstByte, byteSize);
+        target.data.retain(1);
       } else {
-        target.data.setByte(byteSize - 1,
-            (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) <<  (8 - offset))));
+        // Copy data
+        // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
+        // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
+        // The last byte copied to target is a bit tricky :
+        //   1) if length requires partly byte (length % 8 !=0), copy the remaining bits only.
+        //   2) otherwise, copy the last byte in the same way as to the prior bytes.
+        target.clear();
+        target.allocateNew(length);
+        // TODO maybe do this one word at a time, rather than byte?
+        for (int i = 0; i < byteSize - 1; i++) {
+          target.data.setByte(i, (((this.data.getByte(firstByte + i) & 0xFF) >>> offset) + (this.data.getByte(firstByte + i + 1) << (8 - offset))));
+        }
+        if (length % 8 != 0) {
+          target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
+        } else {
+          target.data.setByte(byteSize - 1,
+                  (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) << (8 - offset))));
+        }
       }
     }
     target.getMutator().setValueCount(length);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
index f2343c88e70a5..194b78585faaf 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -20,6 +20,8 @@
 import static org.junit.Assert.assertEquals;
 
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.util.TransferPair;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -31,7 +33,7 @@ public class TestBitVector {
 
   @Before
   public void init() {
-    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+    allocator = new RootAllocator(Long.MAX_VALUE);
   }
 
   @After
@@ -63,4 +65,80 @@ public void testBitVectorCopyFromSafe() {
     }
   }
 
+  @Test
+  public void testSplitAndTransfer() throws Exception {
+
+    try (final BitVector sourceVector = new BitVector("bitvector", allocator)) {
+      final BitVector.Mutator sourceMutator = sourceVector.getMutator();
+      final BitVector.Accessor sourceAccessor = sourceVector.getAccessor();
+
+      sourceVector.allocateNew(40);
+
+      /* populate the bitvector -- 010101010101010101010101..... */
+      for(int i = 0; i < 40; i++) {
+        if((i & 1) ==  1) {
+          sourceMutator.set(i, 1);
+        }
+        else {
+          sourceMutator.set(i, 0);
+        }
+      }
+
+      sourceMutator.setValueCount(40);
+
+      /* check the vector output */
+      for(int i = 0; i < 40; i++) {
+        int result = sourceAccessor.get(i);
+        if((i & 1) ==  1) {
+          assertEquals(Integer.toString(1), Integer.toString(result));
+        }
+        else {
+          assertEquals(Integer.toString(0), Integer.toString(result));
+        }
+      }
+
+      final TransferPair transferPair = sourceVector.getTransferPair(allocator);
+      final BitVector toVector = (BitVector)transferPair.getTo();
+      final BitVector.Accessor toAccessor = toVector.getAccessor();
+      final BitVector.Mutator toMutator = toVector.getMutator();
+
+      /*
+       * form test cases such that we cover:
+       *
+       * (1) the start index is exactly where a particular byte starts in the source bit vector
+       * (2) the start index is randomly positioned within a byte in the source bit vector
+       *    (2.1) the length is a multiple of 8
+       *    (2.2) the length is not a multiple of 8
+       */
+      final int[][] transferLengths = {  {0, 8},     /* (1) */
+                                         {8, 10},    /* (1) */
+                                         {18, 0},    /* zero length scenario */
+                                         {18, 8},    /* (2.1) */
+                                         {26, 0},    /* zero length scenario */
+                                         {26, 14}    /* (2.2) */
+                                      };
+
+      for (final int[] transferLength : transferLengths) {
+        final int start = transferLength[0];
+        final int length = transferLength[1];
+
+        transferPair.splitAndTransfer(start, length);
+
+        /* check the toVector output after doing splitAndTransfer */
+        for (int i = 0; i < length; i++) {
+          int result = toAccessor.get(i);
+          if((i & 1) == 1) {
+            assertEquals(Integer.toString(1), Integer.toString(result));
+          }
+          else {
+            assertEquals(Integer.toString(0), Integer.toString(result));
+          }
+        }
+
+        toVector.clear();
+      }
+
+      sourceVector.close();
+    }
+  }
 }

From 05af6400d7815e7733b866884feec102fa801838 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Fri, 28 Jul 2017 17:10:58 -0700
Subject: [PATCH 0872/1644] =?UTF-8?q?ARROW-276:=20[JAVA]=20Nullable=20Vect?=
 =?UTF-8?q?ors=20should=20extend=20BaseValueVector=20and=20not=20Bas?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixing the inheritance tree:

Nullable <Fixed-length | Var-Length>Vectors do not use "ArrowBuf data" field in BaseDataValueVector. Therefore, they should extend BaseValueVector class..

Author: siddharth <siddharth@dremio.com>

Closes #892 from siddharthteotia/ARROW-276 and squashes the following commits:

d919538 [siddharth] ARROW-276: Nullable Vectors should extend BaseValueVector and not BaseDataValueVector
---
 .../main/codegen/templates/NullableValueVectors.java | 12 +++++++-----
 .../org/apache/arrow/vector/BaseDataValueVector.java |  2 --
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 5b993678012b5..624ba9d2cec69 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -44,9 +44,10 @@
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
 @SuppressWarnings("unused")
-public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector, FieldVector {
+public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector, FieldVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
+protected final static byte[] emptyByteArray = new byte[]{};
   private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this);
 
   private final String bitsField = "$bits$";
@@ -217,7 +218,6 @@ public int getBufferSizeFor(final int valueCount) {
         + bits.getBufferSizeFor(valueCount);
   }
 
-  @Override
   public ArrowBuf getBuffer() {
     return values.getBuffer();
   }
@@ -286,7 +286,6 @@ public void reset() {
     bits.zeroVector();
     mutator.reset();
     accessor.reset();
-    super.reset();
   }
 
   @Override
@@ -314,12 +313,10 @@ public void allocateNew(int valueCount) {
     accessor.reset();
   }
 
-  @Override
   public void reset() {
     bits.zeroVector();
     mutator.reset();
     accessor.reset();
-    super.reset();
   }
 
   /**
@@ -332,6 +329,11 @@ public void zeroVector() {
   }
   </#if>
 
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+        return getTransferPair(ref, allocator);
+  }
+
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator){
     return new TransferImpl(name, allocator);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 6d7d3f04a6d04..0fea719da8815 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -30,8 +30,6 @@
 
 public abstract class BaseDataValueVector extends BaseValueVector implements BufferBacked {
 
-  protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
-
   public static void load(ArrowFieldNode fieldNode, List<BufferBacked> vectors, List<ArrowBuf> buffers) {
     int expectedSize = vectors.size();
     if (buffers.size() != expectedSize) {

From ec3261782b864ed82e72971121667b57ab2d2e0b Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Sat, 29 Jul 2017 00:05:55 -0400
Subject: [PATCH 0873/1644] ARROW-1192: [JAVA] Use buffer slice for
 splitAndTransfer in List and Union Vectors.

@jacques-n
@StevenMPhillips

PR for the original change-set https://github.com/dremio/arrow/commit/75396ba07f58eb122110edb33ee2305bcdc122d7
with corresponding unit tests.

Author: siddharth <siddharth@dremio.com>
Author: Steven Phillips <steven@dremio.com>

Closes #901 from siddharthteotia/ARROW-1192-PR and squashes the following commits:

3d89c999 [siddharth] ARROW-1192: splitAndTransfer changes for ListVector,UnionVector and corresponding unit tests
035e886c [Steven Phillips] Use buffer slice for splitAndTransfer in List and Union vectors
---
 .../main/codegen/templates/UnionVector.java   |   6 +-
 .../arrow/vector/complex/ListVector.java      |  30 ++-
 .../apache/arrow/vector/TestListVector.java   | 199 ++++++++++++++++++
 .../apache/arrow/vector/TestUnionVector.java  | 181 ++++++++++++++++
 4 files changed, 405 insertions(+), 11 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index aa9d34d6e2666..eabe42a7c4ce5 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -321,10 +321,8 @@ public void transfer() {
 
     @Override
     public void splitAndTransfer(int startIndex, int length) {
-      to.allocateNew();
-      for (int i = 0; i < length; i++) {
-        to.copyFromSafe(startIndex + i, i, org.apache.arrow.vector.complex.UnionVector.this);
-      }
+      internalMapVectorTransferPair.splitAndTransfer(startIndex, length);
+      typeVectorTransferPair.splitAndTransfer(startIndex, length);
       to.getMutator().setValueCount(length);
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 6357294566017..fdeac3971657d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -38,6 +38,7 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
@@ -179,7 +180,11 @@ public TransferPair makeTransferPair(ValueVector target) {
   private class TransferImpl implements TransferPair {
 
     ListVector to;
-    TransferPair pairs[] = new TransferPair[3];
+    TransferPair bitsTransferPair;
+    TransferPair offsetsTransferPair;
+    TransferPair dataTransferPair;
+
+    TransferPair[] pairs;
 
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
       this(new ListVector(name, allocator, fieldType, callBack));
@@ -188,12 +193,13 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
     public TransferImpl(ListVector to) {
       this.to = to;
       to.addOrGetVector(vector.getField().getFieldType());
-      pairs[0] = offsets.makeTransferPair(to.offsets);
-      pairs[1] = bits.makeTransferPair(to.bits);
+      offsetsTransferPair = offsets.makeTransferPair(to.offsets);
+      bitsTransferPair = bits.makeTransferPair(to.bits);
       if (to.getDataVector() instanceof ZeroVector) {
         to.addOrGetVector(vector.getField().getFieldType());
       }
-      pairs[2] = getDataVector().makeTransferPair(to.getDataVector());
+      dataTransferPair = getDataVector().makeTransferPair(to.getDataVector());
+      pairs = new TransferPair[] { bitsTransferPair, offsetsTransferPair, dataTransferPair };
     }
 
     @Override
@@ -206,10 +212,20 @@ public void transfer() {
 
     @Override
     public void splitAndTransfer(int startIndex, int length) {
-      to.allocateNew();
-      for (int i = 0; i < length; i++) {
-        copyValueSafe(startIndex + i, i);
+      UInt4Vector.Accessor offsetVectorAccessor = ListVector.this.offsets.getAccessor();
+      final int startPoint = offsetVectorAccessor.get(startIndex);
+      final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint;
+      to.clear();
+      to.offsets.allocateNew(length + 1);
+      offsetVectorAccessor = ListVector.this.offsets.getAccessor();
+      final UInt4Vector.Mutator targetOffsetVectorMutator = to.offsets.getMutator();
+      for (int i = 0; i < length + 1; i++) {
+        targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
       }
+      bitsTransferPair.splitAndTransfer(startIndex, length);
+      dataTransferPair.splitAndTransfer(startPoint, sliceLength);
+      to.lastSet = length;
+      to.mutator.setValueCount(length);
     }
 
     @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 29ea7628f452b..a1762c466ce0b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -19,6 +19,8 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.ListVector;
@@ -30,6 +32,7 @@
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.*;
 import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -234,4 +237,200 @@ public void testSetLastSetUsage() throws Exception {
       assertNull(actual);
     }
   }
+
+  @Test
+  public void testSplitAndTransfer() throws Exception {
+    try (ListVector listVector = ListVector.empty("sourceVector", allocator)) {
+
+      /* Explicitly add the dataVector */
+      MinorType type = MinorType.BIGINT;
+      listVector.addOrGetVector(FieldType.nullable(type.getType()));
+
+      UnionListWriter listWriter = listVector.getWriter();
+
+      /* allocate memory */
+      listWriter.allocate();
+
+      /* populate data */
+      listWriter.setPosition(0);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(10);
+      listWriter.bigInt().writeBigInt(11);
+      listWriter.bigInt().writeBigInt(12);
+      listWriter.endList();
+
+      listWriter.setPosition(1);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(13);
+      listWriter.bigInt().writeBigInt(14);
+      listWriter.endList();
+
+      listWriter.setPosition(2);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(15);
+      listWriter.bigInt().writeBigInt(16);
+      listWriter.bigInt().writeBigInt(17);
+      listWriter.bigInt().writeBigInt(18);
+      listWriter.endList();
+
+      listWriter.setPosition(3);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(19);
+      listWriter.endList();
+
+      listWriter.setPosition(4);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(20);
+      listWriter.bigInt().writeBigInt(21);
+      listWriter.bigInt().writeBigInt(22);
+      listWriter.bigInt().writeBigInt(23);
+      listWriter.endList();
+
+      listVector.getMutator().setValueCount(5);
+
+      assertEquals(5, listVector.getMutator().getLastSet());
+
+      /* get offsetVector */
+      UInt4Vector offsetVector = (UInt4Vector)listVector.getOffsetVector();
+
+      /* get dataVector */
+      NullableBigIntVector dataVector = (NullableBigIntVector)listVector.getDataVector();
+
+      /* check the vector output */
+      final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
+      final ValueVector.Accessor valueAccessor = dataVector.getAccessor();
+
+      int index  = 0;
+      int offset = 0;
+      Object actual = null;
+
+      /* index 0 */
+      assertFalse(listVector.getAccessor().isNull(index));
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(0), Integer.toString(offset));
+
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(10), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(11), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(12), (Long)actual);
+
+      /* index 1 */
+      index++;
+      assertFalse(listVector.getAccessor().isNull(index));
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(3), Integer.toString(offset));
+
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(13), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(14), (Long)actual);
+
+      /* index 2 */
+      index++;
+      assertFalse(listVector.getAccessor().isNull(index));
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(5), Integer.toString(offset));
+
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(15), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(16), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(17), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(18), (Long)actual);
+
+      /* index 3 */
+      index++;
+      assertFalse(listVector.getAccessor().isNull(index));
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(9), Integer.toString(offset));
+
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(19), (Long)actual);
+
+      /* index 4 */
+      index++;
+      assertFalse(listVector.getAccessor().isNull(index));
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(10), Integer.toString(offset));
+
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(20), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(21), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(22), (Long)actual);
+      offset++;
+      actual = valueAccessor.getObject(offset);
+      assertEquals(new Long(23), (Long)actual);
+
+      /* index 5 */
+      index++;
+      assertTrue(listVector.getAccessor().isNull(index));
+      offset = offsetAccessor.get(index);
+      assertEquals(Integer.toString(14), Integer.toString(offset));
+
+      /* do split and transfer */
+      try (ListVector toVector = ListVector.empty("toVector", allocator)) {
+
+        TransferPair transferPair = listVector.makeTransferPair(toVector);
+
+        int[][] transferLengths = { {0, 2},
+                                    {3, 1},
+                                    {4, 1}
+                                  };
+
+        for (final int[] transferLength : transferLengths) {
+          int start = transferLength[0];
+          int splitLength = transferLength[1];
+
+          int dataLength1 = 0;
+          int dataLength2 = 0;
+
+          int offset1 = 0;
+          int offset2 = 0;
+
+          transferPair.splitAndTransfer(start, splitLength);
+
+          /* get offsetVector of toVector */
+          UInt4Vector offsetVector1 = (UInt4Vector)toVector.getOffsetVector();
+          UInt4Vector.Accessor offsetAccessor1 = offsetVector1.getAccessor();
+
+          /* get dataVector of toVector */
+          NullableBigIntVector dataVector1 = (NullableBigIntVector)toVector.getDataVector();
+          NullableBigIntVector.Accessor valueAccessor1 = dataVector1.getAccessor();
+
+          for(int i = 0; i < splitLength; i++) {
+            dataLength1 = offsetAccessor.get(start + i + 1) - offsetAccessor.get(start + i);
+            dataLength2 = offsetAccessor1.get(i + 1) - offsetAccessor1.get(i);
+
+            assertEquals("Different data lengths at index: " + i + " and start: " + start,
+                         dataLength1, dataLength2);
+
+            offset1 = offsetAccessor.get(start + i);
+            offset2 = offsetAccessor1.get(i);
+
+            for(int j = 0; j < dataLength1; j++) {
+              assertEquals("Different data at indexes: " + offset1 + " and " + offset2,
+                           valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
+
+              offset1++;
+              offset2++;
+            }
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index a5b90ee90b8f9..a5159242d76f9 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -24,6 +24,7 @@
 import org.apache.arrow.vector.holders.NullableBitHolder;
 import org.apache.arrow.vector.holders.NullableIntHolder;
 import org.apache.arrow.vector.holders.NullableUInt4Holder;
+import org.apache.arrow.vector.holders.NullableFloat4Holder;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.TransferPair;
@@ -117,6 +118,179 @@ public void testTransfer() throws Exception {
     }
   }
 
+  @Test
+  public void testSplitAndTransfer() throws Exception {
+    try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      final UnionVector.Mutator sourceMutator = sourceVector.getMutator();
+      final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor();
+
+      sourceVector.allocateNew();
+
+      /* populate the UnionVector */
+      sourceMutator.setType(0, MinorType.INT);
+      sourceMutator.setSafe(0, newIntHolder(5));
+      sourceMutator.setType(1, MinorType.INT);
+      sourceMutator.setSafe(1, newIntHolder(10));
+      sourceMutator.setType(2, MinorType.INT);
+      sourceMutator.setSafe(2, newIntHolder(15));
+      sourceMutator.setType(3, MinorType.INT);
+      sourceMutator.setSafe(3, newIntHolder(20));
+      sourceMutator.setType(4, MinorType.INT);
+      sourceMutator.setSafe(4, newIntHolder(25));
+      sourceMutator.setType(5, MinorType.INT);
+      sourceMutator.setSafe(5, newIntHolder(30));
+      sourceMutator.setType(6, MinorType.INT);
+      sourceMutator.setSafe(6, newIntHolder(35));
+      sourceMutator.setType(7, MinorType.INT);
+      sourceMutator.setSafe(7, newIntHolder(40));
+      sourceMutator.setType(8, MinorType.INT);
+      sourceMutator.setSafe(8, newIntHolder(45));
+      sourceMutator.setType(9, MinorType.INT);
+      sourceMutator.setSafe(9, newIntHolder(50));
+      sourceMutator.setValueCount(10);
+
+      /* check the vector output */
+      assertEquals(10, sourceAccessor.getValueCount());
+      assertEquals(false, sourceAccessor.isNull(0));
+      assertEquals(5, sourceAccessor.getObject(0));
+      assertEquals(false, sourceAccessor.isNull(1));
+      assertEquals(10, sourceAccessor.getObject(1));
+      assertEquals(false, sourceAccessor.isNull(2));
+      assertEquals(15, sourceAccessor.getObject(2));
+      assertEquals(false, sourceAccessor.isNull(3));
+      assertEquals(20, sourceAccessor.getObject(3));
+      assertEquals(false, sourceAccessor.isNull(4));
+      assertEquals(25, sourceAccessor.getObject(4));
+      assertEquals(false, sourceAccessor.isNull(5));
+      assertEquals(30, sourceAccessor.getObject(5));
+      assertEquals(false, sourceAccessor.isNull(6));
+      assertEquals(35, sourceAccessor.getObject(6));
+      assertEquals(false, sourceAccessor.isNull(7));
+      assertEquals(40, sourceAccessor.getObject(7));
+      assertEquals(false, sourceAccessor.isNull(8));
+      assertEquals(45, sourceAccessor.getObject(8));
+      assertEquals(false, sourceAccessor.isNull(9));
+      assertEquals(50, sourceAccessor.getObject(9));
+
+      try(UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+
+        final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
+        final UnionVector.Accessor toAccessor = toVector.getAccessor();
+
+        final int[][] transferLengths = { {0, 3},
+                                          {3, 1},
+                                          {4, 2},
+                                          {6, 1},
+                                          {7, 1},
+                                          {8, 2}
+                                        };
+
+        for (final int[] transferLength : transferLengths) {
+          final int start = transferLength[0];
+          final int length = transferLength[1];
+
+          transferPair.splitAndTransfer(start, length);
+
+          /* check the toVector output after doing the splitAndTransfer */
+          for (int i = 0; i < length; i++) {
+            assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceAccessor.getObject(start + i),
+                         toAccessor.getObject(i));
+          }
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testSplitAndTransferWithMixedVectors() throws Exception {
+    try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      final UnionVector.Mutator sourceMutator = sourceVector.getMutator();
+      final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor();
+
+      sourceVector.allocateNew();
+
+      /* populate the UnionVector */
+      sourceMutator.setType(0, MinorType.INT);
+      sourceMutator.setSafe(0, newIntHolder(5));
+
+      sourceMutator.setType(1, MinorType.FLOAT4);
+      sourceMutator.setSafe(1, newFloat4Holder(5.5f));
+
+      sourceMutator.setType(2, MinorType.INT);
+      sourceMutator.setSafe(2, newIntHolder(10));
+
+      sourceMutator.setType(3, MinorType.FLOAT4);
+      sourceMutator.setSafe(3, newFloat4Holder(10.5f));
+
+      sourceMutator.setType(4, MinorType.INT);
+      sourceMutator.setSafe(4, newIntHolder(15));
+
+      sourceMutator.setType(5, MinorType.FLOAT4);
+      sourceMutator.setSafe(5, newFloat4Holder(15.5f));
+
+      sourceMutator.setType(6, MinorType.INT);
+      sourceMutator.setSafe(6, newIntHolder(20));
+
+      sourceMutator.setType(7, MinorType.FLOAT4);
+      sourceMutator.setSafe(7, newFloat4Holder(20.5f));
+
+      sourceMutator.setType(8, MinorType.INT);
+      sourceMutator.setSafe(8, newIntHolder(30));
+
+      sourceMutator.setType(9, MinorType.FLOAT4);
+      sourceMutator.setSafe(9, newFloat4Holder(30.5f));
+      sourceMutator.setValueCount(10);
+
+      /* check the vector output */
+      assertEquals(10, sourceAccessor.getValueCount());
+      assertEquals(false, sourceAccessor.isNull(0));
+      assertEquals(5, sourceAccessor.getObject(0));
+      assertEquals(false, sourceAccessor.isNull(1));
+      assertEquals(5.5f, sourceAccessor.getObject(1));
+      assertEquals(false, sourceAccessor.isNull(2));
+      assertEquals(10, sourceAccessor.getObject(2));
+      assertEquals(false, sourceAccessor.isNull(3));
+      assertEquals(10.5f, sourceAccessor.getObject(3));
+      assertEquals(false, sourceAccessor.isNull(4));
+      assertEquals(15, sourceAccessor.getObject(4));
+      assertEquals(false, sourceAccessor.isNull(5));
+      assertEquals(15.5f, sourceAccessor.getObject(5));
+      assertEquals(false, sourceAccessor.isNull(6));
+      assertEquals(20, sourceAccessor.getObject(6));
+      assertEquals(false, sourceAccessor.isNull(7));
+      assertEquals(20.5f, sourceAccessor.getObject(7));
+      assertEquals(false, sourceAccessor.isNull(8));
+      assertEquals(30, sourceAccessor.getObject(8));
+      assertEquals(false, sourceAccessor.isNull(9));
+      assertEquals(30.5f, sourceAccessor.getObject(9));
+
+      try(UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+
+        final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
+        final UnionVector.Accessor toAccessor = toVector.getAccessor();
+
+        final int[][] transferLengths = { {0, 2},
+                                          {2, 1},
+                                          {3, 2},
+                                          {5, 3},
+                                          {8, 2}
+                                        };
+
+        for (final int[] transferLength : transferLengths) {
+          final int start = transferLength[0];
+          final int length = transferLength[1];
+
+          transferPair.splitAndTransfer(start, length);
+
+          /* check the toVector output after doing the splitAndTransfer */
+          for (int i = 0; i < length; i++) {
+            assertEquals("Different values at index: " + i, sourceAccessor.getObject(start + i), toAccessor.getObject(i));
+          }
+        }
+      }
+    }
+  }
+
   private static NullableIntHolder newIntHolder(int value) {
     final NullableIntHolder holder = new NullableIntHolder();
     holder.isSet = 1;
@@ -130,4 +304,11 @@ private static NullableBitHolder newBitHolder(boolean value) {
     holder.value = value ? 1 : 0;
     return holder;
   }
+
+  private static NullableFloat4Holder newFloat4Holder(float value) {
+    final NullableFloat4Holder holder = new NullableFloat4Holder();
+    holder.isSet = 1;
+    holder.value = value;
+    return holder;
+  }
 }

From 5aea3a3d9340928e2d9f19c7fce4dcd4688dbee1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 29 Jul 2017 10:48:44 -0400
Subject: [PATCH 0874/1644] ARROW-1287: [Python] Implement whence argument for
 pyarrow.NativeFile.seek

I still need to validate this against the use case in https://github.com/dask/fastparquet/issues/188

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #907 from wesm/ARROW-1287 and squashes the following commits:

933f3f6d [Wes McKinney] Add testing script for checking thirdparty library against pyarrow.HdfsClient
423ca87b [Wes McKinney] Implement whence argument for pyarrow.NativeFile.seek
---
 python/pyarrow/io.pxi               | 58 ++++++++++++++++++++++++++++-
 python/pyarrow/tests/conftest.py    |  4 +-
 python/pyarrow/tests/test_io.py     |  9 +++++
 python/pyarrow/tests/test_plasma.py | 15 +++-----
 python/testing/parquet_interop.py   | 53 ++++++++++++++++++++++++++
 5 files changed, 127 insertions(+), 12 deletions(-)
 create mode 100644 python/testing/parquet_interop.py

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 181b0b18a712f..9e4e9078ceb35 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -106,6 +106,9 @@ cdef class NativeFile:
             raise IOError("file not open")
 
     def size(self):
+        """
+        Return file size
+        """
         cdef int64_t size
         self._assert_readable()
         with nogil:
@@ -113,6 +116,9 @@ cdef class NativeFile:
         return size
 
     def tell(self):
+        """
+        Return current stream position
+        """
         cdef int64_t position
         with nogil:
             if self.is_readable:
@@ -121,10 +127,46 @@ cdef class NativeFile:
                 check_status(self.wr_file.get().Tell(&position))
         return position
 
-    def seek(self, int64_t position):
+    def seek(self, int64_t position, int whence=0):
+        """
+        Change current file stream position
+
+        Parameters
+        ----------
+        position : int
+            Byte offset, interpreted relative to value of whence argument
+        whence : int, default 0
+            Point of reference for seek offset
+
+        Notes
+        -----
+        Values of whence:
+        * 0 -- start of stream (the default); offset should be zero or positive
+        * 1 -- current stream position; offset may be negative
+        * 2 -- end of stream; offset is usually negative
+
+        Returns
+        -------
+        new_position : the new absolute stream position
+        """
+        cdef int64_t offset
         self._assert_readable()
         with nogil:
-            check_status(self.rd_file.get().Seek(position))
+            if whence == 0:
+                offset = position
+            elif whence == 1:
+                check_status(self.rd_file.get().Tell(&offset))
+                offset = offset + position
+            elif whence == 2:
+                check_status(self.rd_file.get().GetSize(&offset))
+                offset = offset + position
+            else:
+                with gil:
+                    raise ValueError("Invalid value of whence: {0}"
+                                     .format(whence))
+            check_status(self.rd_file.get().Seek(offset))
+
+        return self.tell()
 
     def write(self, data):
         """
@@ -144,6 +186,18 @@ cdef class NativeFile:
             check_status(self.wr_file.get().Write(buf, bufsize))
 
     def read(self, nbytes=None):
+        """
+        Read indicated number of bytes from file, or read all remaining bytes
+        if no argument passed
+
+        Parameters
+        ----------
+        nbytes : int, default None
+
+        Returns
+        -------
+        data : bytes
+        """
         cdef:
             int64_t c_nbytes
             int64_t bytes_read = 0
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index 21288e4f35e74..f2d67f6641bd3 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -33,12 +33,14 @@
 except ImportError:
     pass
 
+
 try:
-    import pyarrow.plasma as plasma
+    import pyarrow.plasma as plasma  # noqa
     defaults['plasma'] = True
 except ImportError:
     pass
 
+
 def pytest_configure(config):
     pass
 
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 835f50874f7fd..c81a0485ce1ee 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -323,6 +323,15 @@ def _check_native_file_reader(FACTORY, sample_data):
     assert f.tell() == len(data) + 1
     assert f.read(5) == b''
 
+    # Test whence argument of seek, ARROW-1287
+    assert f.seek(3) == 3
+    assert f.seek(3, os.SEEK_CUR) == 6
+    assert f.tell() == 6
+
+    ex_length = len(data) - 2
+    assert f.seek(-2, os.SEEK_END) == ex_length
+    assert f.tell() == ex_length
+
 
 def test_memory_map_reader(sample_disk_data):
     _check_native_file_reader(pa.memory_map, sample_disk_data)
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index 8f8d5b5ed607b..e168d9fb1ef43 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -19,22 +19,20 @@
 from __future__ import division
 from __future__ import print_function
 
-import glob
 import numpy as np
 import os
 import pytest
 import random
 import signal
 import subprocess
-import sys
 import time
-import unittest
 
 import pyarrow as pa
 import pandas as pd
 
 DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9
 
+
 def random_name():
     return str(random.randint(0, 99999999))
 
@@ -160,7 +158,7 @@ def setup_method(self, test_method):
 
     def teardown_method(self, test_method):
         # Check that the Plasma store is still alive.
-        assert self.p.poll() == None
+        assert self.p.poll() is None
         # Kill the plasma store process.
         if os.getenv("PLASMA_VALGRIND") == "1":
             self.p.send_signal(signal.SIGTERM)
@@ -227,7 +225,7 @@ def test_create_existing(self):
                 self.plasma_client.create(object_id, length,
                                           generate_metadata(length))
             # TODO(pcm): Introduce a more specific error type here.
-            except pa.lib.ArrowException as e:
+            except pa.lib.ArrowException:
                 pass
             else:
                 assert False
@@ -270,7 +268,6 @@ def test_get(self):
                     assert results[i] is None
 
     def test_store_arrow_objects(self):
-        import pyarrow.plasma as plasma
         data = np.random.randn(10, 4)
         # Write an arrow object.
         object_id = random_object_id()
@@ -334,7 +331,7 @@ def assert_create_raises_plasma_full(unit_test, size):
                                                     partial_size,
                                                     size - partial_size)
             # TODO(pcm): More specific error here.
-            except pa.lib.ArrowException as e:
+            except pa.lib.ArrowException:
                 pass
             else:
                 # For some reason the above didn't throw an exception, so fail.
@@ -368,7 +365,7 @@ def test_contains(self):
         fake_object_ids = [random_object_id() for _ in range(100)]
         real_object_ids = [random_object_id() for _ in range(100)]
         for object_id in real_object_ids:
-            assert self.plasma_client.contains(object_id) == False
+            assert self.plasma_client.contains(object_id) is False
             self.plasma_client.create(object_id, 100)
             self.plasma_client.seal(object_id)
             assert self.plasma_client.contains(object_id)
@@ -383,7 +380,7 @@ def test_hash(self):
         try:
             self.plasma_client.hash(object_id1)
             # TODO(pcm): Introduce a more specific error type here
-        except pa.lib.ArrowException as e:
+        except pa.lib.ArrowException:
             pass
         else:
             assert False
diff --git a/python/testing/parquet_interop.py b/python/testing/parquet_interop.py
new file mode 100644
index 0000000000000..ba2eb6fa416fa
--- /dev/null
+++ b/python/testing/parquet_interop.py
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import pytest
+
+import fastparquet
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pandas.util.testing as tm
+
+
+def hdfs_test_client(driver='libhdfs'):
+    host = os.environ.get('ARROW_HDFS_TEST_HOST', 'localhost')
+    user = os.environ['ARROW_HDFS_TEST_USER']
+    try:
+        port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 20500))
+    except ValueError:
+        raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
+                         'an integer')
+
+    return pa.HdfsClient(host, port, user, driver=driver)
+
+
+def test_fastparquet_read_with_hdfs():
+    fs = hdfs_test_client()
+
+    df = tm.makeDataFrame()
+    table = pa.Table.from_pandas(df)
+
+    path = '/tmp/testing.parquet'
+    with fs.open(path, 'wb') as f:
+        pq.write_table(table, f)
+
+    parquet_file = fastparquet.ParquetFile(path, open_with=fs.open)
+
+    result = parquet_file.to_pandas()
+    tm.assert_frame_equal(result, df)

From b4e9ba1ae68bcc449e4426b7c08d2984ed20c6be Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 29 Jul 2017 11:00:58 -0400
Subject: [PATCH 0875/1644] ARROW-968: [Python] Support slices in
 RecordBatch.__getitem__

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #908 from wesm/ARROW-968 and squashes the following commits:

47b71a5d [Wes McKinney] Support slices in RecordBatch.__getitem__
---
 python/pyarrow/array.pxi           | 34 +++++++++++++++++-------------
 python/pyarrow/table.pxi           |  9 ++++++--
 python/pyarrow/tests/test_table.py | 11 ++++++++--
 3 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index efbe36f80b3ea..67418aa5eac67 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -89,6 +89,23 @@ def array(object sequence, DataType type=None, MemoryPool memory_pool=None,
     return pyarrow_wrap_array(sp_array)
 
 
+def _normalize_slice(object arrow_obj, slice key):
+    cdef Py_ssize_t n = len(arrow_obj)
+
+    start = key.start or 0
+    while start < 0:
+        start += n
+
+    stop = key.stop if key.stop is not None else n
+    while stop < 0:
+        stop += n
+
+    step = key.step or 1
+    if step != 1:
+        raise IndexError('only slices with step 1 supported')
+    else:
+        return arrow_obj.slice(start, stop - start)
+
 
 cdef class Array:
 
@@ -230,23 +247,10 @@ cdef class Array:
         raise NotImplemented
 
     def __getitem__(self, key):
-        cdef:
-            Py_ssize_t n = len(self)
+        cdef Py_ssize_t n = len(self)
 
         if PySlice_Check(key):
-            start = key.start or 0
-            while start < 0:
-                start += n
-
-            stop = key.stop if key.stop is not None else n
-            while stop < 0:
-                stop += n
-
-            step = key.step or 1
-            if step != 1:
-                raise IndexError('only slices with step 1 supported')
-            else:
-                return self.slice(start, stop - start)
+            return _normalize_slice(self, key)
 
         while key < 0:
             key += len(self)
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 6188e90616ba2..a9cb06480cd38 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -475,8 +475,13 @@ cdef class RecordBatch:
             )
         return pyarrow_wrap_array(self.batch.column(i))
 
-    def __getitem__(self, i):
-        return self.column(i)
+    def __getitem__(self, key):
+        cdef:
+            Py_ssize_t start, stop
+        if isinstance(key, slice):
+            return _normalize_slice(self, key)
+        else:
+            return self.column(key)
 
     def slice(self, offset=0, length=None):
         """
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index c2aeda9b2df62..28b98f0952a9a 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -80,7 +80,7 @@ def test_recordbatch_basics():
         batch[2]
 
 
-def test_recordbatch_slice():
+def test_recordbatch_slice_getitem():
     data = [
         pa.array(range(5)),
         pa.array([-10, -5, 0, 5, 10])
@@ -90,7 +90,6 @@ def test_recordbatch_slice():
     batch = pa.RecordBatch.from_arrays(data, names)
 
     sliced = batch.slice(2)
-
     assert sliced.num_rows == 3
 
     expected = pa.RecordBatch.from_arrays(
@@ -111,6 +110,14 @@ def test_recordbatch_slice():
     with pytest.raises(IndexError):
         batch.slice(-1)
 
+    # Check __getitem__-based slicing
+    assert batch.slice(0, 0).equals(batch[:0])
+    assert batch.slice(0, 2).equals(batch[:2])
+    assert batch.slice(2, 2).equals(batch[2:4])
+    assert batch.slice(2, len(batch) - 2).equals(batch[2:])
+    assert batch.slice(len(batch) - 2, 2).equals(batch[-2:])
+    assert batch.slice(len(batch) - 4, 2).equals(batch[-4:-2])
+
 
 def test_recordbatch_from_to_pandas():
     data = pd.DataFrame({

From ea1b67ceb494ce7eb4302e6d09c3cd5101aa5da1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 29 Jul 2017 11:35:36 -0400
Subject: [PATCH 0876/1644] ARROW-1294: [C++] Pin cmake=3.8.0 in MSVC toolchain
 build

See https://github.com/conda-forge/cmake-feedstock/issues/38. Not sure the origin of the build failure but we will pin at 3.8.0 for now

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #910 from wesm/ARROW-1294 and squashes the following commits:

69dfecc4 [Wes McKinney] Pin cmake=3.8.0 in MSVC toolchainbuild
---
 ci/msvc-build.bat | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 22108abdd3b97..2a537769f82fc 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -51,9 +51,16 @@ conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython ^
       thrift-cpp
 
+@rem ARROW-1294 CMake 3.9.0 in conda-forge breaks the build
+set ARROW_CMAKE_VERSION=3.8.0
+
 if "%JOB%" == "Toolchain" (
+
   conda install -n arrow -q -y -c conda-forge ^
-      flatbuffers rapidjson cmake git boost-cpp ^
+      flatbuffers rapidjson ^
+      cmake=%ARROW_CMAKE_VERSION% ^
+      git ^
+      boost-cpp ^
       snappy zlib brotli gflags lz4-c zstd
 )
 

From 4108bda82b6574adc95e371b328cee748cd1fbce Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 29 Jul 2017 13:55:15 -0400
Subject: [PATCH 0877/1644] ARROW-1291: [Python] Cast non-string DataFrame
 columns to strings in RecordBatch/Table.from_pandas

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #911 from wesm/ARROW-1291 and squashes the following commits:

d442f3bb [Wes McKinney] Cast non-string DataFrame columns to strings in RecordBatch/Table.from_pandas
---
 python/pyarrow/pandas_compat.py             | 96 ++++++++++++++-------
 python/pyarrow/table.pxi                    | 49 +----------
 python/pyarrow/tests/test_convert_pandas.py |  5 ++
 python/pyarrow/tests/test_ipc.py            |  2 +-
 4 files changed, 74 insertions(+), 78 deletions(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 9b2a5c4c60dbf..cd7ad47782646 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -155,7 +155,7 @@ def index_level_name(index, i):
         return '__index_level_{:d}__'.format(i)
 
 
-def construct_metadata(df, index_levels, preserve_index, types):
+def construct_metadata(df, column_names, index_levels, preserve_index, types):
     """Returns a dictionary containing enough metadata to reconstruct a pandas
     DataFrame as an Arrow Table, including index columns.
 
@@ -170,41 +170,77 @@ def construct_metadata(df, index_levels, preserve_index, types):
     -------
     dict
     """
-    ncolumns = len(df.columns)
+    ncolumns = len(column_names)
     df_types = types[:ncolumns]
     index_types = types[ncolumns:ncolumns + len(index_levels)]
+
+    column_metadata = [
+        get_column_metadata(df[col_name], name=sanitized_name,
+                            arrow_type=arrow_type)
+        for col_name, sanitized_name, arrow_type in
+        zip(df.columns, column_names, df_types)
+    ]
+
+    if preserve_index:
+        index_column_names = [index_level_name(level, i)
+                              for i, level in enumerate(index_levels)]
+        index_column_metadata = [
+            get_column_metadata(level, name=index_level_name(level, i),
+                                arrow_type=arrow_type)
+            for i, (level, arrow_type) in enumerate(zip(index_levels,
+                                                        index_types))
+        ]
+    else:
+        index_column_names = index_column_metadata = []
+
     return {
-        b'pandas': json.dumps(
-            {
-                'index_columns': [
-                    index_level_name(level, i)
-                    for i, level in enumerate(index_levels)
-                ] if preserve_index else [],
-                'columns': [
-                    get_column_metadata(
-                        df[name],
-                        name=name,
-                        arrow_type=arrow_type
-                    )
-                    for name, arrow_type in zip(df.columns, df_types)
-                ] + (
-                    [
-                        get_column_metadata(
-                            level,
-                            name=index_level_name(level, i),
-                            arrow_type=arrow_type
-                        )
-                        for i, (level, arrow_type) in enumerate(
-                            zip(index_levels, index_types)
-                        )
-                    ] if preserve_index else []
-                ),
-                'pandas_version': pd.__version__,
-            }
-        ).encode('utf8')
+        b'pandas': json.dumps({
+            'index_columns': index_column_names,
+            'columns': column_metadata + index_column_metadata,
+            'pandas_version': pd.__version__
+        }).encode('utf8')
     }
 
 
+def dataframe_to_arrays(df, timestamps_to_ms, schema, preserve_index):
+    names = []
+    arrays = []
+    index_columns = []
+    types = []
+    type = None
+
+    if preserve_index:
+        n = len(getattr(df.index, 'levels', [df.index]))
+        index_columns.extend(df.index.get_level_values(i) for i in range(n))
+
+    for name in df.columns:
+        col = df[name]
+        if not isinstance(name, six.string_types):
+            name = str(name)
+
+        if schema is not None:
+            field = schema.field_by_name(name)
+            type = getattr(field, "type", None)
+
+        array = pa.Array.from_pandas(
+            col, type=type, timestamps_to_ms=timestamps_to_ms
+        )
+        arrays.append(array)
+        names.append(name)
+        types.append(array.type)
+
+    for i, column in enumerate(index_columns):
+        array = pa.Array.from_pandas(column, timestamps_to_ms=timestamps_to_ms)
+        arrays.append(array)
+        names.append(index_level_name(column, i))
+        types.append(array.type)
+
+    metadata = construct_metadata(
+        df, names, index_columns, preserve_index, types
+    )
+    return names, arrays, metadata
+
+
 def table_to_blockmanager(table, nthreads=1):
     import pandas.core.internals as _int
     from pyarrow.compat import DatetimeTZDtype
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index a9cb06480cd38..c1d5a50d487a3 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -317,51 +317,6 @@ cdef int _schema_from_arrays(
     return 0
 
 
-cdef tuple _dataframe_to_arrays(
-    df,
-    bint timestamps_to_ms,
-    Schema schema,
-    bint preserve_index
-):
-    cdef:
-        list names = []
-        list arrays = []
-        list index_columns = []
-        list types = []
-        DataType type = None
-        dict metadata
-        Py_ssize_t i
-        Py_ssize_t n
-
-    if preserve_index:
-        n = len(getattr(df.index, 'levels', [df.index]))
-        index_columns.extend(df.index.get_level_values(i) for i in range(n))
-
-    for name in df.columns:
-        col = df[name]
-        if schema is not None:
-            field = schema.field_by_name(name)
-            type = getattr(field, "type", None)
-
-        array = Array.from_pandas(
-            col, type=type, timestamps_to_ms=timestamps_to_ms
-        )
-        arrays.append(array)
-        names.append(name)
-        types.append(array.type)
-
-    for i, column in enumerate(index_columns):
-        array = Array.from_pandas(column, timestamps_to_ms=timestamps_to_ms)
-        arrays.append(array)
-        names.append(pdcompat.index_level_name(column, i))
-        types.append(array.type)
-
-    metadata = pdcompat.construct_metadata(
-        df, index_columns, preserve_index, types
-    )
-    return names, arrays, metadata
-
-
 cdef class RecordBatch:
     """
     Batch of rows of columns of equal length
@@ -570,7 +525,7 @@ cdef class RecordBatch:
         -------
         pyarrow.RecordBatch
         """
-        names, arrays, metadata = _dataframe_to_arrays(
+        names, arrays, metadata = pdcompat.dataframe_to_arrays(
             df, False, schema, preserve_index
         )
         return cls.from_arrays(arrays, names, metadata)
@@ -748,7 +703,7 @@ cdef class Table:
         >>> pa.Table.from_pandas(df)
         <pyarrow.lib.Table object at 0x7f05d1fb1b40>
         """
-        names, arrays, metadata = _dataframe_to_arrays(
+        names, arrays, metadata = pdcompat.dataframe_to_arrays(
             df,
             timestamps_to_ms=timestamps_to_ms,
             schema=schema,
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 43e0bad5e3d2d..d488658563306 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -109,6 +109,11 @@ def test_all_none_category(self):
         df['a'] = df['a'].astype('category')
         self._check_pandas_roundtrip(df)
 
+    def test_non_string_columns(self):
+        df = pd.DataFrame({0: [1, 2, 3]})
+        table = pa.Table.from_pandas(df)
+        assert table.column(0).name == '0'
+
     def test_float_no_nulls(self):
         data = {}
         fields = []
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index bcaca6df777b8..3ad369c31f4f2 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -360,7 +360,7 @@ def test_pandas_serialize_round_trip_multi_index():
 
 
 @pytest.mark.xfail(
-    raises=TypeError,
+    raises=AssertionError,
     reason='Non string columns are not supported',
 )
 def test_pandas_serialize_round_trip_not_string_columns():

From 2288bfc18fdbd6f50eb6c184d2349bcdd538f469 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 29 Jul 2017 23:14:09 -0400
Subject: [PATCH 0878/1644] ARROW-1264: [Python] Raise exception in Python
 instead of aborting if cannot connect to Plasma store

cc @pcmoritz @robertnishihara

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #912 from wesm/ARROW-1264 and squashes the following commits:

bd134d7e [Wes McKinney] Add flags to disable certain classes of unit tests
1d9de777 [Wes McKinney] Raise exception in Python instead of aborting if cannot connect to Plasma store
---
 cpp/src/plasma/client.cc            |  8 ++--
 cpp/src/plasma/client.h             |  4 +-
 cpp/src/plasma/io.cc                | 30 ++++++++++-----
 cpp/src/plasma/io.h                 | 17 +++++---
 python/manylinux1/build_arrow.sh    |  5 ++-
 python/pyarrow/plasma.pyx           | 60 ++++++++++++++++++++---------
 python/pyarrow/tests/conftest.py    |  9 ++++-
 python/pyarrow/tests/test_plasma.py |  6 +++
 8 files changed, 100 insertions(+), 39 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index e14b3d9a46c82..8ea62c6e553c7 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -512,10 +512,12 @@ Status PlasmaClient::GetNotification(int fd, ObjectID* object_id, int64_t* data_
 }
 
 Status PlasmaClient::Connect(const std::string& store_socket_name,
-                             const std::string& manager_socket_name, int release_delay) {
-  store_conn_ = connect_ipc_sock_retry(store_socket_name, -1, -1);
+                             const std::string& manager_socket_name, int release_delay,
+                             int num_retries) {
+  RETURN_NOT_OK(ConnectIpcSocketRetry(store_socket_name, num_retries, -1, &store_conn_));
   if (manager_socket_name != "") {
-    manager_conn_ = connect_ipc_sock_retry(manager_socket_name, -1, -1);
+    RETURN_NOT_OK(
+        ConnectIpcSocketRetry(manager_socket_name, num_retries, -1, &manager_conn_));
   } else {
     manager_conn_ = -1;
   }
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index cc05a064511fe..50ec55f5ec8f3 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -89,9 +89,11 @@ class ARROW_EXPORT PlasmaClient {
   ///        function will not connect to a manager.
   /// @param release_delay Number of released objects that are kept around
   ///        and not evicted to avoid too many munmaps.
+  /// @param num_retries number of attempts to connect to IPC socket, default 50
   /// @return The return status.
   Status Connect(const std::string& store_socket_name,
-                 const std::string& manager_socket_name, int release_delay);
+                 const std::string& manager_socket_name, int release_delay,
+                 int num_retries = -1);
 
   /// Create an object in the Plasma Store. Any metadata for this object must be
   /// be passed in when the object is created.
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
index e3b6b617fbc06..9bb433990820c 100644
--- a/cpp/src/plasma/io.cc
+++ b/cpp/src/plasma/io.cc
@@ -17,6 +17,11 @@
 
 #include "plasma/io.h"
 
+#include <cstdint>
+#include <sstream>
+
+#include "arrow/status.h"
+
 #include "plasma/common.h"
 
 using arrow::Status;
@@ -29,6 +34,8 @@ using arrow::Status;
 #define NUM_CONNECT_ATTEMPTS 50
 #define CONNECT_TIMEOUT_MS 100
 
+namespace plasma {
+
 Status WriteBytes(int fd, uint8_t* cursor, size_t length) {
   ssize_t nbytes = 0;
   size_t bytesleft = length;
@@ -140,8 +147,8 @@ int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
   return socket_fd;
 }
 
-int connect_ipc_sock_retry(const std::string& pathname, int num_retries,
-                           int64_t timeout) {
+Status ConnectIpcSocketRetry(const std::string& pathname, int num_retries,
+                             int64_t timeout, int* fd) {
   /* Pick the default values if the user did not specify. */
   if (num_retries < 0) {
     num_retries = NUM_CONNECT_ATTEMPTS;
@@ -150,23 +157,26 @@ int connect_ipc_sock_retry(const std::string& pathname, int num_retries,
     timeout = CONNECT_TIMEOUT_MS;
   }
 
-  int fd = -1;
+  *fd = -1;
   for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
-    fd = connect_ipc_sock(pathname);
-    if (fd >= 0) {
+    *fd = connect_ipc_sock(pathname);
+    if (*fd >= 0) {
       break;
     }
     if (num_attempts == 0) {
-      ARROW_LOG(ERROR) << "Connection to socket failed for pathname " << pathname;
+      ARROW_LOG(ERROR) << "Connection to IPC socket failed for pathname " << pathname
+                       << ", retrying " << num_retries << " times";
     }
     /* Sleep for timeout milliseconds. */
     usleep(static_cast<int>(timeout * 1000));
   }
   /* If we could not connect to the socket, exit. */
-  if (fd == -1) {
-    ARROW_LOG(FATAL) << "Could not connect to socket " << pathname;
+  if (*fd == -1) {
+    std::stringstream ss;
+    ss << "Could not connect to socket " << pathname;
+    return Status::IOError(ss.str());
   }
-  return fd;
+  return Status::OK();
 }
 
 int connect_ipc_sock(const std::string& pathname) {
@@ -224,3 +234,5 @@ uint8_t* read_message_async(int sock) {
   }
   return message;
 }
+
+}  // namespace plasma
diff --git a/cpp/src/plasma/io.h b/cpp/src/plasma/io.h
index 43c3fb535497f..ef96c06ccea57 100644
--- a/cpp/src/plasma/io.h
+++ b/cpp/src/plasma/io.h
@@ -34,22 +34,29 @@
 #define PLASMA_PROTOCOL_VERSION 0x0000000000000000
 #define DISCONNECT_CLIENT 0
 
-arrow::Status WriteBytes(int fd, uint8_t* cursor, size_t length);
+namespace plasma {
 
-arrow::Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes);
+using arrow::Status;
 
-arrow::Status ReadBytes(int fd, uint8_t* cursor, size_t length);
+Status WriteBytes(int fd, uint8_t* cursor, size_t length);
 
-arrow::Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer);
+Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes);
+
+Status ReadBytes(int fd, uint8_t* cursor, size_t length);
+
+Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer);
 
 int bind_ipc_sock(const std::string& pathname, bool shall_listen);
 
 int connect_ipc_sock(const std::string& pathname);
 
-int connect_ipc_sock_retry(const std::string& pathname, int num_retries, int64_t timeout);
+Status ConnectIpcSocketRetry(const std::string& pathname, int num_retries,
+                             int64_t timeout, int* fd);
 
 int AcceptClient(int socket_fd);
 
 uint8_t* read_message_async(int sock);
 
+}  // namespace plasma
+
 #endif  // PLASMA_IO_H
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 5725b2a87f45e..5a21e36e4d7d5 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -80,7 +80,10 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     echo "=== (${PYTHON}) Testing manylinux1 wheel ==="
     source /venv-test-${PYTHON}/bin/activate
     pip install repaired_wheels/*.whl
-    py.test --parquet /venv-test-${PYTHON}/lib/*/site-packages/pyarrow
+
+    # ARROW-1264; for some reason the test case added causes a segfault inside
+    # the Docker container when writing and error message to stderr
+    py.test --parquet /venv-test-${PYTHON}/lib/*/site-packages/pyarrow -v -s --disable-plasma
     deactivate
 
     mv repaired_wheels/*.whl /io/dist
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index 8aaca9963c131..dd62d473b001a 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -70,7 +70,8 @@ cdef extern from "plasma/client.h" nogil:
         CPlasmaClient()
 
         CStatus Connect(const c_string& store_socket_name,
-                        const c_string& manager_socket_name, int release_delay)
+                        const c_string& manager_socket_name,
+                        int release_delay, int num_retries)
 
         CStatus Create(const CUniqueID& object_id, int64_t data_size,
                        const uint8_t* metadata, int64_t metadata_size,
@@ -98,10 +99,13 @@ cdef extern from "plasma/client.h" nogil:
 
         CStatus Fetch(int num_object_ids, const CUniqueID* object_ids)
 
-        CStatus Wait(int64_t num_object_requests, CObjectRequest* object_requests,
-           int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
+        CStatus Wait(int64_t num_object_requests,
+                     CObjectRequest* object_requests,
+                     int num_ready_objects, int64_t timeout_ms,
+                     int* num_objects_ready);
 
-        CStatus Transfer(const char* addr, int port, const CUniqueID& object_id)
+        CStatus Transfer(const char* addr, int port,
+                         const CUniqueID& object_id)
 
 
 cdef extern from "plasma/client.h" nogil:
@@ -247,7 +251,8 @@ cdef class PlasmaClient:
     def manager_socket_name(self):
         return self.manager_socket_name.decode()
 
-    def create(self, ObjectID object_id, int64_t data_size, c_string metadata=b""):
+    def create(self, ObjectID object_id, int64_t data_size,
+               c_string metadata=b""):
         """
         Create a new buffer in the PlasmaStore for a particular object ID.
 
@@ -439,7 +444,8 @@ cdef class PlasmaClient:
         """
         cdef c_string addr = address.encode()
         with nogil:
-            check_status(self.client.get().Transfer(addr.c_str(), port, object_id.data))
+            check_status(self.client.get()
+                         .Transfer(addr.c_str(), port, object_id.data))
 
     def fetch(self, object_ids):
         """
@@ -457,7 +463,8 @@ cdef class PlasmaClient:
         with nogil:
             check_status(self.client.get().Fetch(ids.size(), ids.data()))
 
-    def wait(self, object_ids, int64_t timeout=PLASMA_WAIT_TIMEOUT, int num_returns=1):
+    def wait(self, object_ids, int64_t timeout=PLASMA_WAIT_TIMEOUT,
+             int num_returns=1):
         """
         Wait until num_returns objects in object_ids are ready.
         Currently, the object ID arguments to wait must be unique.
@@ -483,14 +490,18 @@ cdef class PlasmaClient:
         if len(object_ids) != len(set(object_ids)):
             raise Exception("Wait requires a list of unique object IDs.")
         cdef int64_t num_object_requests = len(object_ids)
-        cdef c_vector[CObjectRequest] object_requests = c_vector[CObjectRequest](num_object_requests)
+        cdef c_vector[CObjectRequest] object_requests = (
+            c_vector[CObjectRequest](num_object_requests))
         cdef int num_objects_ready = 0
         cdef ObjectID object_id
         for i, object_id in enumerate(object_ids):
             object_requests[i].object_id = object_id.data
             object_requests[i].type = PLASMA_QUERY_ANYWHERE
         with nogil:
-            check_status(self.client.get().Wait(num_object_requests, object_requests.data(), num_returns, timeout, &num_objects_ready))
+            check_status(self.client.get().Wait(num_object_requests,
+                                                object_requests.data(),
+                                                num_returns, timeout,
+                                                &num_objects_ready))
         cdef int num_to_return = min(num_objects_ready, num_returns);
         ready_ids = []
         waiting_ids = set(object_ids)
@@ -498,9 +509,12 @@ cdef class PlasmaClient:
         for i in range(len(object_ids)):
             if num_returned == num_to_return:
                 break
-            if object_requests[i].status == ObjectStatusLocal or object_requests[i].status == ObjectStatusRemote:
-                ready_ids.append(ObjectID(object_requests[i].object_id.binary()))
-                waiting_ids.discard(ObjectID(object_requests[i].object_id.binary()))
+            if (object_requests[i].status == ObjectStatusLocal or
+                object_requests[i].status == ObjectStatusRemote):
+                ready_ids.append(
+                    ObjectID(object_requests[i].object_id.binary()))
+                waiting_ids.discard(
+                    ObjectID(object_requests[i].object_id.binary()))
                 num_returned += 1
         return ready_ids, list(waiting_ids)
 
@@ -526,10 +540,11 @@ cdef class PlasmaClient:
         cdef int64_t data_size
         cdef int64_t metadata_size
         with nogil:
-            check_status(self.client.get().GetNotification(self.notification_fd,
-                                                           &object_id.data,
-                                                           &data_size,
-                                                           &metadata_size))
+            check_status(self.client.get()
+                         .GetNotification(self.notification_fd,
+                                          &object_id.data,
+                                          &data_size,
+                                          &metadata_size))
         return object_id, data_size, metadata_size
 
     def to_capsule(self):
@@ -542,7 +557,9 @@ cdef class PlasmaClient:
         with nogil:
             check_status(self.client.get().Disconnect())
 
-def connect(store_socket_name, manager_socket_name, int release_delay):
+
+def connect(store_socket_name, manager_socket_name, int release_delay,
+            int num_retries=-1):
     """
     Return a new PlasmaClient that is connected a plasma store and
     optionally a manager.
@@ -556,11 +573,16 @@ def connect(store_socket_name, manager_socket_name, int release_delay):
     release_delay : int
         The maximum number of objects that the client will keep and
         delay releasing (for caching reasons).
+    num_retries : int, default -1
+        Number of times tor ty to connect to plasma store. Default value of -1
+        uses the default (50)
     """
     cdef PlasmaClient result = PlasmaClient()
     result.store_socket_name = store_socket_name.encode()
     result.manager_socket_name = manager_socket_name.encode()
     with nogil:
-        check_status(result.client.get().Connect(result.store_socket_name,
-                     result.manager_socket_name, release_delay))
+        check_status(result.client.get()
+                     .Connect(result.store_socket_name,
+                              result.manager_socket_name,
+                              release_delay, num_retries))
     return result
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index f2d67f6641bd3..651438bb53404 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -51,6 +51,11 @@ def pytest_addoption(parser):
                          default=defaults[group],
                          help=('Enable the {0} test group'.format(group)))
 
+    for group in groups:
+        parser.addoption('--disable-{0}'.format(group), action='store_true',
+                         default=False,
+                         help=('Disable the {0} test group'.format(group)))
+
     for group in groups:
         parser.addoption('--only-{0}'.format(group), action='store_true',
                          default=False,
@@ -62,12 +67,14 @@ def pytest_runtest_setup(item):
 
     for group in groups:
         only_flag = '--only-{0}'.format(group)
+        disable_flag = '--disable-{0}'.format(group)
         flag = '--{0}'.format(group)
 
         if item.config.getoption(only_flag):
             only_set = True
         elif getattr(item.obj, group, None):
-            if not item.config.getoption(flag):
+            if (item.config.getoption(disable_flag) or
+                    not item.config.getoption(flag)):
                 skip('{0} NOT enabled'.format(flag))
 
     if only_set:
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index e168d9fb1ef43..04162bbbbade7 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -168,6 +168,12 @@ def teardown_method(self, test_method):
         else:
             self.p.kill()
 
+    def test_connection_failure_raises_exception(self):
+        import pyarrow.plasma as plasma
+        # ARROW-1264
+        with pytest.raises(IOError):
+            plasma.connect('unknown-store-name', '', 0, 1)
+
     def test_create(self):
         # Create an object id string.
         object_id = random_object_id()

From b4eec62030c5bc54ff0161d97d8e85c7a963e592 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 30 Jul 2017 12:58:16 -0400
Subject: [PATCH 0879/1644] ARROW-932: [Python] Fix MSVC compiler warnings,
 build Python with /WX and -Werror in CI

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #913 from wesm/ARROW-932 and squashes the following commits:

9534ae9d [Wes McKinney] Only pass PYARROW_CXXFLAGS when set
dedcbb9a [Wes McKinney] Fix typo
b5a6d9af [Wes McKinney] Supress another clang warning
2e8f1053 [Wes McKinney] typo
5740f001 [Wes McKinney] Add PYARROW_CXXFLAGS option, fix MSVC compiler warnings
c32ee092 [Wes McKinney] Remove print statement. Disable MSVC 4190 warning
---
 ci/msvc-build.bat                    |  3 +++
 ci/travis_script_python.sh           |  1 +
 python/CMakeLists.txt                | 11 +++++++++++
 python/doc/source/development.rst    |  1 -
 python/pyarrow/_parquet.pyx          |  2 +-
 python/pyarrow/includes/libarrow.pxd |  2 +-
 python/pyarrow/io.pxi                |  2 +-
 python/pyarrow/scalar.pxi            |  1 -
 python/pyarrow/table.pxi             | 10 +++++-----
 python/pyarrow/types.pxi             |  8 ++++----
 python/setup.py                      |  6 ++++++
 11 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 2a537769f82fc..04fe2ab62cbd4 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -114,6 +114,9 @@ popd
 @rem see PARQUET-1018
 
 pushd python
+
+set PYARROW_CXXFLAGS=/WX
 python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel  || exit /B
 py.test pyarrow -v -s --parquet || exit /B
+
 popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 907bc60cd719e..9135aaf38e4e7 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -23,6 +23,7 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 export ARROW_HOME=$ARROW_CPP_INSTALL
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH
+export PYARROW_CXXFLAGS="-Werror"
 
 build_parquet_cpp() {
   export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 846e4dd5f6eaf..bfae157ed6b9c 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -57,6 +57,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUNDLE_ARROW_CPP
     "Bundle the Arrow C++ libraries"
     OFF)
+  set(PYARROW_CXXFLAGS "" CACHE STRING
+    "Compiler flags to append when compiling Arrow")
 endif()
 
 find_program(CCACHE_FOUND ccache)
@@ -75,6 +77,7 @@ include(CompilerInfo)
 
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PYARROW_CXXFLAGS}")
 
 if (NOT MSVC)
   # Enable perf and other tools to work properly
@@ -82,6 +85,13 @@ if (NOT MSVC)
 
   # Suppress Cython warnings
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
+else()
+  # MSVC version of -Wno-return-type-c-linkage
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4190")
+
+  # Cython generates some bitshift expressions that MSVC does not like in
+  # __Pyx_PyFloat_DivideObjC
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4293")
 endif()
 
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
@@ -95,6 +105,7 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   # Cython warnings in clang
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-constant-logical-operand")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-declarations")
 
   # We have public Cython APIs which return C++ types, which are in an extern
   # "C" blog (no symbol mangling) and clang doesn't like this
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 55b3efdad17c6..d0a1c544dd091 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -267,7 +267,6 @@ Now, we build and install Arrow C++ libraries
          -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
          -DCMAKE_BUILD_TYPE=Release ^
          -DARROW_BUILD_TESTS=off ^
-         -DARROW_ZLIB_VENDORED=off ^
          -DARROW_PYTHON=on ..
    cmake --build . --target INSTALL --config Release
    cd ..\..
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index aef661818f401..c940122da5dcf 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -636,7 +636,7 @@ cdef class ParquetWriter:
         elif row_group_size == 0:
             raise ValueError('Row group size cannot be 0')
 
-        cdef int c_row_group_size = row_group_size
+        cdef int64_t c_row_group_size = row_group_size
 
         with nogil:
             check_status(self.writer.get()
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index ffe867b0af0f5..db6770f586bd5 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -369,7 +369,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
             shared_ptr[CTable]* table)
 
         int num_columns()
-        int num_rows()
+        int64_t num_rows()
 
         c_bool Equals(const CTable& other)
 
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 9e4e9078ceb35..01c987d286f92 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -923,7 +923,7 @@ cdef class _HdfsClient:
         cdef c_string c_path = tobytes(path)
         with nogil:
             check_status(self.client.get()
-                         .Delete(c_path, recursive))
+                         .Delete(c_path, recursive == 1))
 
     def open(self, path, mode='rb', buffer_size=None, replication=None,
              default_block_size=None):
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index dec5341ca4ad1..1f72070cb7e12 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -169,7 +169,6 @@ cdef class Time64Value(ArrayValue):
             CTime64Type* dtype = <CTime64Type*> ap.type().get()
 
         cdef int64_t val = ap.Value(self.index)
-        print(val)
         if dtype.unit() == TimeUnit_MICRO:
             return (datetime.datetime(1970, 1, 1) +
                     datetime.timedelta(microseconds=val)).time()
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index c1d5a50d487a3..6277761b7d6ec 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -286,7 +286,7 @@ cdef int _schema_from_arrays(
         c_string c_name
         vector[shared_ptr[CField]] fields
         shared_ptr[CDataType] type_
-        int K = len(arrays)
+        Py_ssize_t K = len(arrays)
 
     fields.resize(K)
 
@@ -733,7 +733,7 @@ cdef class Table:
             vector[shared_ptr[CColumn]] columns
             shared_ptr[CSchema] schema
             shared_ptr[CTable] table
-            size_t K = len(arrays)
+            int i, K = <int> len(arrays)
 
         _schema_from_arrays(arrays, names, metadata, &schema)
 
@@ -841,7 +841,7 @@ cdef class Table:
         self._check_nullptr()
         return pyarrow_wrap_schema(self.table.schema())
 
-    def column(self, int64_t i):
+    def column(self, int i):
         """
         Select a column by its numeric index.
 
@@ -855,8 +855,8 @@ cdef class Table:
         """
         cdef:
             Column column = Column()
-            int64_t num_columns = self.num_columns
-            int64_t index
+            int num_columns = self.num_columns
+            int index
 
         self._check_nullptr()
         if not -num_columns <= i < num_columns:
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index a8d7aa0ee8122..fefde55bc2f95 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -281,12 +281,12 @@ cdef class Schema:
     def __len__(self):
         return self.schema.num_fields()
 
-    def __getitem__(self, int64_t i):
+    def __getitem__(self, int i):
 
         cdef:
             Field result = Field()
-            int64_t num_fields = self.schema.num_fields()
-            int64_t index
+            int num_fields = self.schema.num_fields()
+            int index
 
         if not -num_fields <= i < num_fields:
             raise IndexError(
@@ -456,7 +456,7 @@ def field(name, DataType type, bint nullable=True, dict metadata=None):
         convert_metadata(metadata, &c_meta)
 
     result.sp_field.reset(new CField(tobytes(name), type.sp_type,
-                                     nullable, c_meta))
+                                     nullable == 1, c_meta))
     result.field = result.sp_field.get()
     result.type = type
     return result
diff --git a/python/setup.py b/python/setup.py
index 801cd17f440ec..ebf28cc64e990 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -92,6 +92,8 @@ def initialize_options(self):
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
 
+        self.cmake_cxxflags = os.environ.get('PYARROW_CXXFLAGS', '')
+
         if sys.platform == 'win32':
             # Cannot do debug builds in Windows unless Python itself is a debug
             # build
@@ -146,6 +148,10 @@ def _run_cmake(self):
         if self.with_plasma:
             cmake_options.append('-DPYARROW_BUILD_PLASMA=on')
 
+        if len(self.cmake_cxxflags) > 0:
+            cmake_options.append('-DPYARROW_CXXFLAGS="{0}"'
+                                 .format(self.cmake_cxxflags))
+
         if self.bundle_arrow_cpp:
             cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
             # ARROW-1090: work around CMake rough edges

From af2aeafca6b5e7f460fa157d29b571b1ef3c2053 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 31 Jul 2017 18:46:54 -0400
Subject: [PATCH 0880/1644] ARROW-1213: [Python] Support s3fs filesystem for
 Amazon S3 in ParquetDataset

cc @yackoa

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #916 from wesm/ARROW-1213 and squashes the following commits:

f8a0aff1 [Wes McKinney] Add HDFS section to API docs
c54302df [Wes McKinney] Add deprecation warning for HdfsClient
4d3e7222 [Wes McKinney] Auto-wrap s3fs filesystem when using ParquetDataset
0be33bb8 [Wes McKinney] Implement os.walk emulation layer for s3fs
719f806d [Wes McKinney] Progress toward supporting s3fs in Parquet reader
bbd664ed [Wes McKinney] Refactor HdfsClient into pyarrow/hdfs.py. Add connect factory method. Rename to HadoopFilesystem. Add walk implementation for HDFS, base Parquet directory walker on that
4984a9d4 [Wes McKinney] Refactoring slightly
4c0bcf4a [Wes McKinney] Start on Dask filesystem wrapper, S3-Parquet dataset test case
---
 python/.gitignore                    |   6 ++
 python/doc/source/api.rst            |  10 ++
 python/doc/source/filesystems.rst    |  15 +--
 python/pyarrow/__init__.py           |  20 ++--
 python/pyarrow/filesystem.py         | 140 ++++++++++++++++++---------
 python/pyarrow/hdfs.py               | 133 +++++++++++++++++++++++++
 python/pyarrow/parquet.py            |  67 +++++++------
 python/pyarrow/tests/conftest.py     |  13 ++-
 python/pyarrow/tests/test_hdfs.py    |   2 +-
 python/pyarrow/tests/test_parquet.py |  50 ++++++++--
 python/pyarrow/util.py               |  12 +++
 11 files changed, 368 insertions(+), 100 deletions(-)
 create mode 100644 python/pyarrow/hdfs.py

diff --git a/python/.gitignore b/python/.gitignore
index 6c0d5a93cd35c..1bf20c4ca5298 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -34,3 +34,9 @@ coverage.xml
 # benchmark working dir
 .asv
 pyarrow/_table_api.h
+
+# manylinux1 temporary files
+manylinux1/arrow
+
+# plasma store
+pyarrow/plasma_store
\ No newline at end of file
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index b84163b8f75c1..6f26076bf63cb 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -164,6 +164,16 @@ Input / Output and Shared Memory
    create_memory_map
    PythonFile
 
+Filesystems
+-----------
+
+.. autosummary::
+   :toctree: generated/
+
+   hdfs.connect
+   HadoopFilesystem
+   LocalFilesystem
+
 .. _api.ipc:
 
 Interprocess Communication and Messaging
diff --git a/python/doc/source/filesystems.rst b/python/doc/source/filesystems.rst
index 61c03c57dfad9..78f6f2a45eace 100644
--- a/python/doc/source/filesystems.rst
+++ b/python/doc/source/filesystems.rst
@@ -31,12 +31,13 @@ System. You connect like so:
 .. code-block:: python
 
    import pyarrow as pa
-   hdfs = pa.HdfsClient(host, port, user=user, kerb_ticket=ticket_cache_path)
+   hdfs = pa.hdfs.connect(host, port, user=user, kerb_ticket=ticket_cache_path)
+   type(hdfs)
 
-By default, ``pyarrow.HdfsClient`` uses libhdfs, a JNI-based interface to the
-Java Hadoop client. This library is loaded **at runtime** (rather than at link
-/ library load time, since the library may not be in your LD_LIBRARY_PATH), and
-relies on some environment variables.
+By default, ``pyarrow.hdfs.HadoopFilesystem`` uses libhdfs, a JNI-based
+interface to the Java Hadoop client. This library is loaded **at runtime**
+(rather than at link / library load time, since the library may not be in your
+LD_LIBRARY_PATH), and relies on some environment variables.
 
 * ``HADOOP_HOME``: the root of your installed Hadoop distribution. Often has
   `lib/native/libhdfs.so`.
@@ -56,5 +57,5 @@ You can also use libhdfs3, a thirdparty C++ library for HDFS from Pivotal Labs:
 
 .. code-block:: python
 
-   hdfs3 = pa.HdfsClient(host, port, user=user, kerb_ticket=ticket_cache_path,
-                         driver='libhdfs3')
+   hdfs3 = pa.hdfs.connect(host, port, user=user, kerb_ticket=ticket_cache_path,
+                           driver='libhdfs3')
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 6d0ce204382e3..42e58034f3535 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -88,7 +88,10 @@
                          ArrowTypeError)
 
 
-from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
+from pyarrow.filesystem import Filesystem, LocalFilesystem
+
+from pyarrow.hdfs import HadoopFilesystem
+import pyarrow.hdfs as hdfs
 
 from pyarrow.ipc import (Message, MessageReader,
                          RecordBatchFileReader, RecordBatchFileWriter,
@@ -106,16 +109,7 @@
 # ----------------------------------------------------------------------
 # 0.4.0 deprecations
 
-import warnings
-
-def _deprecate_class(old_name, new_name, klass, next_version='0.5.0'):
-    msg = ('pyarrow.{0} has been renamed to '
-           '{1}, will be removed in {2}'
-           .format(old_name, new_name, next_version))
-    def deprecated_factory(*args, **kwargs):
-        warnings.warn(msg, FutureWarning)
-        return klass(*args)
-    return deprecated_factory
+from pyarrow.util import _deprecate_class
 
 FileReader = _deprecate_class('FileReader',
                               'RecordBatchFileReader',
@@ -136,3 +130,7 @@ def deprecated_factory(*args, **kwargs):
 InMemoryOutputStream = _deprecate_class('InMemoryOutputStream',
                                         'BufferOutputStream',
                                         BufferOutputStream, '0.5.0')
+
+# Backwards compatibility with pyarrow < 0.6.0
+HdfsClient = _deprecate_class('HdfsClient', 'pyarrow.hdfs.connect',
+                              hdfs.connect, '0.6.0')
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 9fa4f7627775a..4b8ca3279ce12 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -17,9 +17,9 @@
 
 from os.path import join as pjoin
 import os
+import posixpath
 
 from pyarrow.util import implements
-import pyarrow.lib as lib
 
 
 class Filesystem(object):
@@ -44,6 +44,12 @@ def delete(self, path, recursive=False):
         """
         raise NotImplementedError
 
+    def rm(self, path, recursive=False):
+        """
+        Alias for Filesystem.delete
+        """
+        return self.delete(path, recursive=recursive)
+
     def mkdir(self, path, create_parents=True):
         raise NotImplementedError
 
@@ -96,6 +102,12 @@ def read_parquet(self, path, columns=None, metadata=None, schema=None,
         return dataset.read(columns=columns, nthreads=nthreads,
                             use_pandas_metadata=use_pandas_metadata)
 
+    def open(self, path, mode='rb'):
+        """
+        Open file for reading or writing
+        """
+        raise NotImplementedError
+
     @property
     def pathsep(self):
         return '/'
@@ -134,6 +146,7 @@ def isfile(self, path):
     def exists(self, path):
         return os.path.exists(path)
 
+    @implements(Filesystem.open)
     def open(self, path, mode='rb'):
         """
         Open file for reading or writing
@@ -144,68 +157,103 @@ def open(self, path, mode='rb'):
     def pathsep(self):
         return os.path.sep
 
+    def walk(self, top_dir):
+        """
+        Directory tree generator, see os.walk
+        """
+        return os.walk(top_dir)
 
-class HdfsClient(lib._HdfsClient, Filesystem):
+
+class DaskFilesystem(Filesystem):
     """
-    Connect to an HDFS cluster. All parameters are optional and should
-    only be set if the defaults need to be overridden.
-
-    Authentication should be automatic if the HDFS cluster uses Kerberos.
-    However, if a username is specified, then the ticket cache will likely
-    be required.
-
-    Parameters
-    ----------
-    host : NameNode. Set to "default" for fs.defaultFS from core-site.xml.
-    port : NameNode's port. Set to 0 for default or logical (HA) nodes.
-    user : Username when connecting to HDFS; None implies login user.
-    kerb_ticket : Path to Kerberos ticket cache.
-    driver : {'libhdfs', 'libhdfs3'}, default 'libhdfs'
-      Connect using libhdfs (JNI-based) or libhdfs3 (3rd-party C++
-      library from Pivotal Labs)
-
-    Notes
-    -----
-    The first time you call this method, it will take longer than usual due
-    to JNI spin-up time.
-
-    Returns
-    -------
-    client : HDFSClient
+    Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.
     """
 
-    def __init__(self, host="default", port=0, user=None, kerb_ticket=None,
-                 driver='libhdfs'):
-        self._connect(host, port, user, kerb_ticket, driver)
+    def __init__(self, fs):
+        self.fs = fs
 
     @implements(Filesystem.isdir)
     def isdir(self, path):
-        return lib._HdfsClient.isdir(self, path)
+        raise NotImplementedError("Unsupported file system API")
 
     @implements(Filesystem.isfile)
     def isfile(self, path):
-        return lib._HdfsClient.isfile(self, path)
+        raise NotImplementedError("Unsupported file system API")
 
     @implements(Filesystem.delete)
     def delete(self, path, recursive=False):
-        return lib._HdfsClient.delete(self, path, recursive)
+        return self.fs.rm(path, recursive=recursive)
 
     @implements(Filesystem.mkdir)
-    def mkdir(self, path, create_parents=True):
-        return lib._HdfsClient.mkdir(self, path)
+    def mkdir(self, path):
+        return self.fs.mkdir(path)
 
-    def ls(self, path, full_info=False):
+    @implements(Filesystem.open)
+    def open(self, path, mode='rb'):
+        """
+        Open file for reading or writing
         """
-        Retrieve directory contents and metadata, if requested.
+        return self.fs.open(path, mode=mode)
 
-        Parameters
-        ----------
-        path : HDFS path
-        full_info : boolean, default False
-            If False, only return list of paths
+    def ls(self, path, detail=False):
+        return self.fs.ls(path, detail=detail)
 
-        Returns
-        -------
-        result : list of dicts (full_info=True) or strings (full_info=False)
+    def walk(self, top_path):
+        """
+        Directory tree generator, like os.walk
+        """
+        return self.fs.walk(top_path)
+
+
+class S3FSWrapper(DaskFilesystem):
+
+    @implements(Filesystem.isdir)
+    def isdir(self, path):
+        try:
+            contents = self.fs.ls(path)
+            if len(contents) == 1 and contents[0] == path:
+                return False
+            else:
+                return True
+        except OSError:
+            return False
+
+    @implements(Filesystem.isfile)
+    def isfile(self, path):
+        try:
+            contents = self.fs.ls(path)
+            return len(contents) == 1 and contents[0] == path
+        except OSError:
+            return False
+
+    def walk(self, path, refresh=False):
+        """
+        Directory tree generator, like os.walk
+
+        Generator version of what is in s3fs, which yields a flattened list of
+        files
         """
-        return lib._HdfsClient.ls(self, path, full_info)
+        path = path.replace('s3://', '')
+        directories = set()
+        files = set()
+
+        for key in list(self.fs._ls(path, refresh=refresh)):
+            path = key['Key']
+            if key['StorageClass'] == 'DIRECTORY':
+                directories.add(path)
+            elif key['StorageClass'] == 'BUCKET':
+                pass
+            else:
+                files.add(path)
+
+        # s3fs creates duplicate 'DIRECTORY' entries
+        files = sorted([posixpath.split(f)[1] for f in files
+                        if f not in directories])
+        directories = sorted([posixpath.split(x)[1]
+                              for x in directories])
+
+        yield path, directories, files
+
+        for directory in directories:
+            for tup in self.walk(directory, refresh=refresh):
+                yield tup
diff --git a/python/pyarrow/hdfs.py b/python/pyarrow/hdfs.py
new file mode 100644
index 0000000000000..3240f99bc6bb8
--- /dev/null
+++ b/python/pyarrow/hdfs.py
@@ -0,0 +1,133 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import posixpath
+
+from pyarrow.util import implements
+from pyarrow.filesystem import Filesystem
+import pyarrow.lib as lib
+
+
+class HadoopFilesystem(lib._HdfsClient, Filesystem):
+    """
+    Filesystem interface for HDFS cluster. See pyarrow.hdfs.connect for full
+    connection details
+    """
+
+    def __init__(self, host="default", port=0, user=None, kerb_ticket=None,
+                 driver='libhdfs'):
+        self._connect(host, port, user, kerb_ticket, driver)
+
+    @implements(Filesystem.isdir)
+    def isdir(self, path):
+        return lib._HdfsClient.isdir(self, path)
+
+    @implements(Filesystem.isfile)
+    def isfile(self, path):
+        return lib._HdfsClient.isfile(self, path)
+
+    @implements(Filesystem.delete)
+    def delete(self, path, recursive=False):
+        return lib._HdfsClient.delete(self, path, recursive)
+
+    @implements(Filesystem.mkdir)
+    def mkdir(self, path, create_parents=True):
+        return lib._HdfsClient.mkdir(self, path)
+
+    def ls(self, path, detail=False):
+        """
+        Retrieve directory contents and metadata, if requested.
+
+        Parameters
+        ----------
+        path : HDFS path
+        detail : boolean, default False
+            If False, only return list of paths
+
+        Returns
+        -------
+        result : list of dicts (detail=True) or strings (detail=False)
+        """
+        return lib._HdfsClient.ls(self, path, detail)
+
+    def walk(self, top_path):
+        """
+        Directory tree generator for HDFS, like os.walk
+
+        Parameters
+        ----------
+        top_path : string
+            Root directory for tree traversal
+
+        Returns
+        -------
+        Generator yielding 3-tuple (dirpath, dirnames, filename)
+        """
+        contents = self.ls(top_path, detail=True)
+
+        directories, files = _libhdfs_walk_files_dirs(top_path, contents)
+        yield top_path, directories, files
+        for dirname in directories:
+            for tup in self.walk(dirname):
+                yield tup
+
+
+def _libhdfs_walk_files_dirs(top_path, contents):
+    files = []
+    directories = []
+    for c in contents:
+        scrubbed_name = posixpath.split(c['name'])[1]
+        if c['kind'] == 'file':
+            files.append(scrubbed_name)
+        else:
+            directories.append(scrubbed_name)
+
+    return directories, files
+
+
+def connect(host="default", port=0, user=None, kerb_ticket=None,
+            driver='libhdfs'):
+    """
+    Connect to an HDFS cluster. All parameters are optional and should
+    only be set if the defaults need to be overridden.
+
+    Authentication should be automatic if the HDFS cluster uses Kerberos.
+    However, if a username is specified, then the ticket cache will likely
+    be required.
+
+    Parameters
+    ----------
+    host : NameNode. Set to "default" for fs.defaultFS from core-site.xml.
+    port : NameNode's port. Set to 0 for default or logical (HA) nodes.
+    user : Username when connecting to HDFS; None implies login user.
+    kerb_ticket : Path to Kerberos ticket cache.
+    driver : {'libhdfs', 'libhdfs3'}, default 'libhdfs'
+      Connect using libhdfs (JNI-based) or libhdfs3 (3rd-party C++
+      library from Apache HAWQ (incubating) )
+
+    Notes
+    -----
+    The first time you call this method, it will take longer than usual due
+    to JNI spin-up time.
+
+    Returns
+    -------
+    filesystem : HadoopFilesystem
+    """
+    fs = HadoopFilesystem(host=host, port=port, user=user,
+                          kerb_ticket=kerb_ticket, driver=driver)
+    return fs
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index a3af9ae75186e..c870412138d7d 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -22,7 +22,7 @@
 
 import numpy as np
 
-from pyarrow.filesystem import LocalFilesystem
+from pyarrow.filesystem import Filesystem, LocalFilesystem
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
                               RowGroupMetaData, ParquetSchema,
                               ParquetWriter)
@@ -416,40 +416,41 @@ def __init__(self, dirpath, filesystem=None, pathsep='/',
         self._visit_level(0, self.dirpath, [])
 
     def _visit_level(self, level, base_path, part_keys):
-        directories = []
-        files = []
         fs = self.filesystem
 
-        if not fs.isdir(base_path):
-            raise ValueError('"{0}" is not a directory'.format(base_path))
-
-        for path in sorted(fs.ls(base_path)):
-            if fs.isfile(path):
-                if _is_parquet_file(path):
-                    files.append(path)
-                elif path.endswith('_common_metadata'):
-                    self.common_metadata_path = path
-                elif path.endswith('_metadata'):
-                    self.metadata_path = path
-                elif not self._should_silently_exclude(path):
-                    print('Ignoring path: {0}'.format(path))
-            elif fs.isdir(path):
-                directories.append(path)
+        _, directories, files = next(fs.walk(base_path))
+
+        filtered_files = []
+        for path in files:
+            full_path = self.pathsep.join((base_path, path))
+            if _is_parquet_file(path):
+                filtered_files.append(full_path)
+            elif path.endswith('_common_metadata'):
+                self.common_metadata_path = full_path
+            elif path.endswith('_metadata'):
+                self.metadata_path = full_path
+            elif not self._should_silently_exclude(path):
+                print('Ignoring path: {0}'.format(full_path))
 
         # ARROW-1079: Filter out "private" directories starting with underscore
-        directories = [x for x in directories if not _is_private_directory(x)]
+        filtered_directories = [self.pathsep.join((base_path, x))
+                                for x in directories
+                                if not _is_private_directory(x)]
+
+        filtered_files.sort()
+        filtered_directories.sort()
 
-        if len(files) > 0 and len(directories) > 0:
+        if len(files) > 0 and len(filtered_directories) > 0:
             raise ValueError('Found files in an intermediate '
                              'directory: {0}'.format(base_path))
-        elif len(directories) > 0:
-            self._visit_directories(level, directories, part_keys)
+        elif len(filtered_directories) > 0:
+            self._visit_directories(level, filtered_directories, part_keys)
         else:
-            self._push_pieces(files, part_keys)
+            self._push_pieces(filtered_files, part_keys)
 
-    def _should_silently_exclude(self, path):
-        _, tail = path.rsplit(self.pathsep, 1)
-        return tail.endswith('.crc') or tail in EXCLUDED_PARQUET_PATHS
+    def _should_silently_exclude(self, file_name):
+        return (file_name.endswith('.crc') or
+                file_name in EXCLUDED_PARQUET_PATHS)
 
     def _visit_directories(self, level, directories, part_keys):
         for path in directories:
@@ -523,7 +524,7 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
         if filesystem is None:
             self.fs = LocalFilesystem.get_instance()
         else:
-            self.fs = filesystem
+            self.fs = _ensure_filesystem(filesystem)
 
         self.paths = path_or_paths
 
@@ -642,6 +643,18 @@ def open_file(path, meta=None):
         return open_file
 
 
+def _ensure_filesystem(fs):
+    if not isinstance(fs, Filesystem):
+        if type(fs).__name__ == 'S3FileSystem':
+            from pyarrow.filesystem import S3FSWrapper
+            return S3FSWrapper(fs)
+        else:
+            raise IOError('Unrecognized filesystem: {0}'
+                          .format(type(fs)))
+    else:
+        return fs
+
+
 def _make_manifest(path_or_paths, fs, pathsep='/'):
     partitions = None
     metadata_path = None
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index 651438bb53404..c6bd6c9b3a2d7 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -18,13 +18,22 @@
 from pytest import skip
 
 
-groups = ['hdfs', 'parquet', 'plasma', 'large_memory']
+groups = [
+    'hdfs',
+    'parquet',
+    'plasma',
+    'large_memory',
+    's3',
+]
+
 
 defaults = {
     'hdfs': False,
+    'large_memory': False,
     'parquet': False,
     'plasma': False,
-    'large_memory': False
+    'large_memory': False,
+    's3': False,
 }
 
 try:
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index cea02fbecc75b..10264080d6e2c 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -43,7 +43,7 @@ def hdfs_test_client(driver='libhdfs'):
         raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
                          'an integer')
 
-    return pa.HdfsClient(host, port, user, driver=driver)
+    return pa.hdfs.connect(host, port, user, driver=driver)
 
 
 @pytest.mark.hdfs
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index f840673f61c49..06265ca95cec6 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -700,6 +700,45 @@ def test_partition_set_dictionary_type():
 
 @parquet
 def test_read_partitioned_directory(tmpdir):
+    fs = LocalFilesystem.get_instance()
+    base_path = str(tmpdir)
+
+    _partition_test_for_filesystem(fs, base_path)
+
+
+@pytest.yield_fixture
+def s3_example():
+    access_key = os.environ['PYARROW_TEST_S3_ACCESS_KEY']
+    secret_key = os.environ['PYARROW_TEST_S3_SECRET_KEY']
+    bucket_name = os.environ['PYARROW_TEST_S3_BUCKET']
+
+    import s3fs
+    fs = s3fs.S3FileSystem(key=access_key, secret=secret_key)
+
+    test_dir = guid()
+
+    bucket_uri = 's3://{0}/{1}'.format(bucket_name, test_dir)
+    fs.mkdir(bucket_uri)
+    yield fs, bucket_uri
+    fs.rm(bucket_uri, recursive=True)
+
+
+@pytest.mark.s3
+@parquet
+def test_read_partitioned_directory_s3fs(s3_example):
+    from pyarrow.filesystem import S3FSWrapper
+    import pyarrow.parquet as pq
+
+    fs, bucket_uri = s3_example
+    wrapper = S3FSWrapper(fs)
+    _partition_test_for_filesystem(wrapper, bucket_uri)
+
+    # Check that we can auto-wrap
+    dataset = pq.ParquetDataset(bucket_uri, filesystem=fs)
+    dataset.read()
+
+
+def _partition_test_for_filesystem(fs, base_path):
     import pyarrow.parquet as pq
 
     foo_keys = [0, 1]
@@ -717,10 +756,9 @@ def test_read_partitioned_directory(tmpdir):
         'values': np.random.randn(N)
     }, columns=['index', 'foo', 'bar', 'values'])
 
-    base_path = str(tmpdir)
-    _generate_partition_directories(base_path, partition_spec, df)
+    _generate_partition_directories(fs, base_path, partition_spec, df)
 
-    dataset = pq.ParquetDataset(base_path)
+    dataset = pq.ParquetDataset(base_path, filesystem=fs)
     table = dataset.read()
     result_df = (table.to_pandas()
                  .sort_values(by='index')
@@ -737,12 +775,11 @@ def test_read_partitioned_directory(tmpdir):
     tm.assert_frame_equal(result_df, expected_df)
 
 
-def _generate_partition_directories(base_dir, partition_spec, df):
+def _generate_partition_directories(fs, base_dir, partition_spec, df):
     # partition_spec : list of lists, e.g. [['foo', [0, 1, 2],
     #                                       ['bar', ['a', 'b', 'c']]
     # part_table : a pyarrow.Table to write to each partition
     DEPTH = len(partition_spec)
-    fs = LocalFilesystem.get_instance()
 
     def _visit_level(base_dir, level, part_keys):
         name, values = partition_spec[level]
@@ -758,7 +795,8 @@ def _visit_level(base_dir, level, part_keys):
 
                 filtered_df = _filter_partition(df, this_part_keys)
                 part_table = pa.Table.from_pandas(filtered_df)
-                _write_table(part_table, file_path)
+                with fs.open(file_path, 'wb') as f:
+                    _write_table(part_table, f)
             else:
                 _visit_level(level_dir, level + 1, this_part_keys)
 
diff --git a/python/pyarrow/util.py b/python/pyarrow/util.py
index 4b6a8356330d5..d984e19215b49 100644
--- a/python/pyarrow/util.py
+++ b/python/pyarrow/util.py
@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import warnings
+
 # Miscellaneous utility code
 
 
@@ -23,3 +25,13 @@ def decorator(g):
         g.__doc__ = f.__doc__
         return g
     return decorator
+
+
+def _deprecate_class(old_name, new_name, klass, next_version='0.5.0'):
+    msg = ('pyarrow.{0} is deprecated as of {1}, please use {2} instead'
+           .format(old_name, next_version, new_name))
+
+    def deprecated_factory(*args, **kwargs):
+        warnings.warn(msg, FutureWarning)
+        return klass(*args)
+    return deprecated_factory

From 900105ac2e096c4dc24f1b95142bf40c383bed82 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 31 Jul 2017 18:55:43 -0400
Subject: [PATCH 0881/1644] ARROW-187: [C++] Add development style notes to C++
 README, note about esoteric exceptions in constructors

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #917 from wesm/ARROW-187 and squashes the following commits:

8424e968 [Wes McKinney] Typo
0bee93cd [Wes McKinney] Add development style notes to C++ README, note about esoteric exceptions
---
 cpp/README.md | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 05a8ba48475ea..6b165895633ce 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -107,6 +107,35 @@ directoy:
 
 This requires [Doxygen](http://www.doxygen.org) to be installed.
 
+## Development
+
+This project follows [Google's C++ Style Guide][3] with minor exceptions. We do
+not encourage anonymous namespaces and we relax the line length restriction to
+90 characters.
+
+### Error Handling and Exceptions
+
+For error handling, we use `arrow::Status` values instead of throwing C++
+exceptions. Since the Arrow C++ libraries are intended to be useful as a
+component in larger C++ projects, using `Status` objects can help with good
+code hygiene by making explicit when a function is expected to be able to fail.
+
+For expressing invariants and "cannot fail" errors, we use DCHECK macros
+defined in `arrow/util/logging.h`. These checks are disabled in release builds
+and are intended to catch internal development errors, particularly when
+refactoring. These macros are not to be included in any public header files.
+
+Since we do not use exceptions, we avoid doing expensive work in object
+constructors. Objects that are expensive to construct may often have private
+constructors, with public static factory methods that return `Status`.
+
+There are a number of object constructors, like `arrow::Schema` and
+`arrow::RecordBatch` where larger STL container objects like `std::vector` may
+be created. While it is possible for `std::bad_alloc` to be thrown in these
+constructors, the circumstances where they would are somewhat esoteric, and it
+is likely that an application would have encountered other more serious
+problems prior to having `std::bad_alloc` thrown in a constructor.
+
 ## Continuous Integration
 
 Pull requests are run through travis-ci for continuous integration.  You can avoid
@@ -114,9 +143,8 @@ build failures by running the following checks before submitting your pull reque
 
     make unittest
     make lint
-    # The next two commands may change your code.  It is recommended you commit
-    # before running them.
-    make clang-tidy # requires clang-tidy is installed
+    # The next command may change your code.  It is recommended you commit
+    # before running it.
     make format # requires clang-format is installed
 
 Note that the clang-tidy target may take a while to run.  You might consider
@@ -132,3 +160,4 @@ both of these options would be used rarely.  Current known uses-cases whent hey
 
 [1]: https://brew.sh/
 [2]: https://github.com/apache/arrow/blob/master/cpp/apidoc/Windows.md
+[3]: https://google.github.io/styleguide/cppguide.html
\ No newline at end of file

From b5ff2f6b4ccbcddff8030545506f841ec7f64c8c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 31 Jul 2017 20:04:02 -0400
Subject: [PATCH 0882/1644] ARROW-1251: [C++] Update C++ README to account for
 toolchain evolution

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #919 from wesm/ARROW-1251 and squashes the following commits:

b0698379 [Wes McKinney] Update C++ README to account for toolchain evolution
---
 cpp/README.md | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/cpp/README.md b/cpp/README.md
index 6b165895633ce..2f98b085115f5 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -82,11 +82,24 @@ Benchmark logs will be placed in the build directory under `build/benchmark-logs
 To set up your own specific build toolchain, here are the relevant environment
 variables
 
+* Boost: `BOOST_ROOT`
 * Googletest: `GTEST_HOME` (only required to build the unit tests)
+* gflags: `GFLAGS_HOME` (only required to build the unit tests)
 * Google Benchmark: `GBENCHMARK_HOME` (only required if building benchmarks)
 * Flatbuffers: `FLATBUFFERS_HOME` (only required for the IPC extensions)
 * Hadoop: `HADOOP_HOME` (only required for the HDFS I/O extensions)
-* jemalloc: `JEMALLOC_HOME` (only required for the jemalloc-based memory pool)
+* jemalloc: `JEMALLOC_HOME`
+* brotli: `BROTLI_HOME`, can be disabled with `-DARROW_WITH_BROTLI=off`
+* lz4: `LZ4_HOME`, can be disabled with `-DARROW_WITH_LZ4=off`
+* snappy: `SNAPPY_HOME`, can be disabled with `-DARROW_WITH_SNAPPY=off`
+* zlib: `ZLIB_HOME`, can be disabled with `-DARROW_WITH_ZLIB=off`
+* zstd: `ZSTD_HOME`, can be disabled with `-DARROW_WITH_ZSTD=off`
+
+If you have all of your toolchain libraries installed at the same prefix, you
+can use the environment variable `$ARROW_BUILD_TOOLCHAIN` to automatically set
+all of these variables. Note that `ARROW_BUILD_TOOLCHAIN` will not set
+`BOOST_ROOT`, so if you have custom Boost installation, you must set this
+environment variable separately.
 
 ### Building Python integration library
 

From 3a84653a3aa00f36f6312a11e58d1daf41dedcee Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 1 Aug 2017 01:05:16 -0400
Subject: [PATCH 0883/1644] ARROW-1265: [Plasma] Clean up all resources on
 SIGTERM to keep valgrind output clean

@pcmoritz this might be a little bit OCD since all of the valgrind warnings were for memory that was still reachable, but let me know what you think.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #921 from wesm/ARROW-1265 and squashes the following commits:

ac2e8309 [Wes McKinney] Consistent function naming style for EventLoop methods
22f440f4 [Wes McKinney] clang-format
97dbd16f [Wes McKinney] Clean up all resources on SIGTERM to keep valgrind output clean
---
 cpp/src/plasma/events.cc | 26 +++++++-----
 cpp/src/plasma/events.h  | 19 +++++----
 cpp/src/plasma/store.cc  | 87 +++++++++++++++++++++++++++-------------
 cpp/src/plasma/store.h   |  6 ++-
 4 files changed, 90 insertions(+), 48 deletions(-)

diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
index f98ced2faf180..4e4ecfaaaca31 100644
--- a/cpp/src/plasma/events.cc
+++ b/cpp/src/plasma/events.cc
@@ -21,13 +21,12 @@
 
 namespace plasma {
 
-void EventLoop::file_event_callback(aeEventLoop* loop, int fd, void* context,
-                                    int events) {
+void EventLoop::FileEventCallback(aeEventLoop* loop, int fd, void* context, int events) {
   FileCallback* callback = reinterpret_cast<FileCallback*>(context);
   (*callback)(events);
 }
 
-int EventLoop::timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context) {
+int EventLoop::TimerEventCallback(aeEventLoop* loop, TimerID timer_id, void* context) {
   TimerCallback* callback = reinterpret_cast<TimerCallback*>(context);
   return (*callback)(timer_id);
 }
@@ -36,21 +35,21 @@ constexpr int kInitialEventLoopSize = 1024;
 
 EventLoop::EventLoop() { loop_ = aeCreateEventLoop(kInitialEventLoopSize); }
 
-bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback) {
+bool EventLoop::AddFileEvent(int fd, int events, const FileCallback& callback) {
   if (file_callbacks_.find(fd) != file_callbacks_.end()) {
     return false;
   }
   auto data = std::unique_ptr<FileCallback>(new FileCallback(callback));
   void* context = reinterpret_cast<void*>(data.get());
   // Try to add the file descriptor.
-  int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
+  int err = aeCreateFileEvent(loop_, fd, events, EventLoop::FileEventCallback, context);
   // If it cannot be added, increase the size of the event loop.
   if (err == AE_ERR && errno == ERANGE) {
     err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2);
     if (err != AE_OK) {
       return false;
     }
-    err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
+    err = aeCreateFileEvent(loop_, fd, events, EventLoop::FileEventCallback, context);
   }
   // In any case, test if there were errors.
   if (err == AE_OK) {
@@ -60,23 +59,28 @@ bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback)
   return false;
 }
 
-void EventLoop::remove_file_event(int fd) {
+void EventLoop::RemoveFileEvent(int fd) {
   aeDeleteFileEvent(loop_, fd, AE_READABLE | AE_WRITABLE);
   file_callbacks_.erase(fd);
 }
 
-void EventLoop::run() { aeMain(loop_); }
+void EventLoop::Start() { aeMain(loop_); }
 
-int64_t EventLoop::add_timer(int64_t timeout, const TimerCallback& callback) {
+void EventLoop::Stop() {
+  aeStop(loop_);
+  aeDeleteEventLoop(loop_);
+}
+
+int64_t EventLoop::AddTimer(int64_t timeout, const TimerCallback& callback) {
   auto data = std::unique_ptr<TimerCallback>(new TimerCallback(callback));
   void* context = reinterpret_cast<void*>(data.get());
   int64_t timer_id =
-      aeCreateTimeEvent(loop_, timeout, EventLoop::timer_event_callback, context, NULL);
+      aeCreateTimeEvent(loop_, timeout, EventLoop::TimerEventCallback, context, NULL);
   timer_callbacks_.emplace(timer_id, std::move(data));
   return timer_id;
 }
 
-int EventLoop::remove_timer(int64_t timer_id) {
+int EventLoop::RemoveTimer(int64_t timer_id) {
   int err = aeDeleteTimeEvent(loop_, timer_id);
   timer_callbacks_.erase(timer_id);
   return err;
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
index 6cb5b73fe94eb..42914848645de 100644
--- a/cpp/src/plasma/events.h
+++ b/cpp/src/plasma/events.h
@@ -61,13 +61,13 @@ class EventLoop {
   /// @param events The flags for events we are listening to (read or write).
   /// @param callback The callback that will be called when the event happens.
   /// @return Returns true if the event handler was added successfully.
-  bool add_file_event(int fd, int events, const FileCallback& callback);
+  bool AddFileEvent(int fd, int events, const FileCallback& callback);
 
   /// Remove a file event handler from the event loop.
   ///
   /// @param fd The file descriptor of the event handler.
   /// @return Void.
-  void remove_file_event(int fd);
+  void RemoveFileEvent(int fd);
 
   /// Register a handler that will be called after a time slice of
   ///  "timeout" milliseconds.
@@ -75,23 +75,26 @@ class EventLoop {
   ///  @param timeout The timeout in milliseconds.
   ///  @param callback The callback for the timeout.
   ///  @return The ID of the newly created timer.
-  int64_t add_timer(int64_t timeout, const TimerCallback& callback);
+  int64_t AddTimer(int64_t timeout, const TimerCallback& callback);
 
   /// Remove a timer handler from the event loop.
   ///
   /// @param timer_id The ID of the timer that is to be removed.
   /// @return The ae.c error code. TODO(pcm): needs to be standardized
-  int remove_timer(int64_t timer_id);
+  int RemoveTimer(int64_t timer_id);
 
-  /// Run the event loop.
+  /// \brief Run the event loop.
   ///
   /// @return Void.
-  void run();
+  void Start();
+
+  /// \brief Stop the event loop
+  void Stop();
 
  private:
-  static void file_event_callback(aeEventLoop* loop, int fd, void* context, int events);
+  static void FileEventCallback(aeEventLoop* loop, int fd, void* context, int events);
 
-  static int timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context);
+  static int TimerEventCallback(aeEventLoop* loop, TimerID timer_id, void* context);
 
   aeEventLoop* loop_;
   std::unordered_map<int, std::unique_ptr<FileCallback>> file_callbacks_;
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index a9425b6f7d907..9f4b98c0ee7d0 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -260,7 +260,7 @@ void PlasmaStore::return_from_get(GetRequest* get_req) {
   }
   // Remove the get request.
   if (get_req->timer != -1) {
-    ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK);
+    ARROW_CHECK(loop_->RemoveTimer(get_req->timer) == AE_OK);
   }
   delete get_req;
 }
@@ -330,7 +330,7 @@ void PlasmaStore::process_get_request(Client* client,
   } else if (timeout_ms != -1) {
     // Set a timer that will cause the get request to return to the client. Note
     // that a timeout of -1 is used to indicate that no timer should be set.
-    get_req->timer = loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) {
+    get_req->timer = loop_->AddTimer(timeout_ms, [this, get_req](int64_t timer_id) {
       return_from_get(get_req);
       return kEventLoopTimerDone;
     });
@@ -412,11 +412,13 @@ void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
 
 void PlasmaStore::connect_client(int listener_sock) {
   int client_fd = AcceptClient(listener_sock);
-  // This is freed in disconnect_client.
+
   Client* client = new Client(client_fd);
+  connected_clients_[client_fd] = std::unique_ptr<Client>(client);
+
   // Add a callback to handle events on this socket.
   // TODO(pcm): Check return value.
-  loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) {
+  loop_->AddFileEvent(client_fd, kEventLoopRead, [this, client](int events) {
     Status s = process_message(client);
     if (!s.ok()) {
       ARROW_LOG(FATAL) << "Failed to process file event: " << s;
@@ -425,23 +427,25 @@ void PlasmaStore::connect_client(int listener_sock) {
   ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
 }
 
-void PlasmaStore::disconnect_client(Client* client) {
-  ARROW_CHECK(client != NULL);
-  ARROW_CHECK(client->fd > 0);
-  loop_->remove_file_event(client->fd);
+void PlasmaStore::disconnect_client(int client_fd) {
+  ARROW_CHECK(client_fd > 0);
+  auto it = connected_clients_.find(client_fd);
+  ARROW_CHECK(it != connected_clients_.end());
+  loop_->RemoveFileEvent(client_fd);
   // Close the socket.
-  close(client->fd);
-  ARROW_LOG(INFO) << "Disconnecting client on fd " << client->fd;
+  close(client_fd);
+  ARROW_LOG(INFO) << "Disconnecting client on fd " << client_fd;
   // If this client was using any objects, remove it from the appropriate
   // lists.
   for (const auto& entry : store_info_.objects) {
-    remove_client_from_object_clients(entry.second.get(), client);
+    remove_client_from_object_clients(entry.second.get(), it->second.get());
   }
+
   // Note, the store may still attempt to send a message to the disconnected
   // client (for example, when an object ID that the client was waiting for
   // is ready). In these cases, the attempt to send the message will fail, but
   // the store should just ignore the failure.
-  delete client;
+  connected_clients_.erase(it);
 }
 
 /// Send notifications about sealed objects to the subscribers. This is called
@@ -478,7 +482,7 @@ void PlasmaStore::send_notifications(int client_fd) {
       // at the end of the method.
       // TODO(pcm): Introduce status codes and check in case the file descriptor
       // is added twice.
-      loop_->add_file_event(client_fd, kEventLoopWrite, [this, client_fd](int events) {
+      loop_->AddFileEvent(client_fd, kEventLoopWrite, [this, client_fd](int events) {
         send_notifications(client_fd);
       });
       break;
@@ -507,7 +511,7 @@ void PlasmaStore::send_notifications(int client_fd) {
 
   // If we have sent all notifications, remove the fd from the event loop.
   if (it->second.object_notifications.empty()) {
-    loop_->remove_file_event(client_fd);
+    loop_->RemoveFileEvent(client_fd);
   }
 }
 
@@ -616,7 +620,7 @@ Status PlasmaStore::process_message(Client* client) {
     } break;
     case DISCONNECT_CLIENT:
       ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd;
-      disconnect_client(client);
+      disconnect_client(client->fd);
       break;
     default:
       // This code should be unreachable.
@@ -625,9 +629,43 @@ Status PlasmaStore::process_message(Client* client) {
   return Status::OK();
 }
 
-// Report "success" to valgrind.
-void signal_handler(int signal) {
+class PlasmaStoreRunner {
+ public:
+  PlasmaStoreRunner() {}
+
+  void Start(char* socket_name, int64_t system_memory) {
+    // Create the event loop.
+    loop_.reset(new EventLoop);
+    store_.reset(new PlasmaStore(loop_.get(), system_memory));
+    int socket = bind_ipc_sock(socket_name, true);
+    // TODO(pcm): Check return value.
+    ARROW_CHECK(socket >= 0);
+
+    loop_->AddFileEvent(socket, kEventLoopRead, [this, socket](int events) {
+      this->store_->connect_client(socket);
+    });
+    loop_->Start();
+  }
+
+  void Shutdown() {
+    loop_->Stop();
+    loop_ = nullptr;
+    store_ = nullptr;
+  }
+
+ private:
+  std::unique_ptr<EventLoop> loop_;
+  std::unique_ptr<PlasmaStore> store_;
+};
+
+static PlasmaStoreRunner* g_runner = nullptr;
+
+void HandleSignal(int signal) {
   if (signal == SIGTERM) {
+    if (g_runner != nullptr) {
+      g_runner->Shutdown();
+    }
+    // Report "success" to valgrind.
     exit(0);
   }
 }
@@ -636,21 +674,16 @@ void start_server(char* socket_name, int64_t system_memory) {
   // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
   // to a client that has already died, the store could die.
   signal(SIGPIPE, SIG_IGN);
-  // Create the event loop.
-  EventLoop loop;
-  PlasmaStore store(&loop, system_memory);
-  int socket = bind_ipc_sock(socket_name, true);
-  ARROW_CHECK(socket >= 0);
-  // TODO(pcm): Check return value.
-  loop.add_file_event(socket, kEventLoopRead,
-                      [&store, socket](int events) { store.connect_client(socket); });
-  loop.run();
+
+  PlasmaStoreRunner runner;
+  g_runner = &runner;
+  signal(SIGTERM, HandleSignal);
+  runner.Start(socket_name, system_memory);
 }
 
 }  // namespace plasma
 
 int main(int argc, char* argv[]) {
-  signal(SIGTERM, plasma::signal_handler);
   char* socket_name = NULL;
   int64_t system_memory = -1;
   int c;
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index fec25c133ce17..fb732a1375da6 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -127,9 +127,9 @@ class PlasmaStore {
 
   /// Disconnect a client from the PlasmaStore.
   ///
-  /// @param client The client that is disconnected.
+  /// @param client_fd The client file descriptor that is disconnected.
   /// @return Void.
-  void disconnect_client(Client* client);
+  void disconnect_client(int client_fd);
 
   void send_notifications(int client_fd);
 
@@ -166,6 +166,8 @@ class PlasmaStore {
   /// TODO(pcm): Consider putting this into the Client data structure and
   /// reorganize the code slightly.
   std::unordered_map<int, NotificationQueue> pending_notifications_;
+
+  std::unordered_map<int, std::unique_ptr<Client>> connected_clients_;
 };
 
 }  // namespace plasma

From e1d574c7cf2207ddde64273cd34ba8352bc003bc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 1 Aug 2017 15:51:18 +0200
Subject: [PATCH 0884/1644] ARROW-1301: [C++/Python] More complete filesystem
 API for HDFS

This also includes a fair bit of API normalization and cleaning.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #926 from wesm/ARROW-1301 and squashes the following commits:

bcc9310 [Wes McKinney] Add missing API
8bf51f5 [Wes McKinney] Add more filesystem methods, tests for HDFS
98847b5 [Wes McKinney] Some HDFS refactoring. Implement chmod, chown. Normalize Filesystem->FileSystem
---
 cpp/src/arrow/io/hdfs-internal.cc    |  16 +-
 cpp/src/arrow/io/hdfs.cc             | 108 +++++--
 cpp/src/arrow/io/hdfs.h              |  48 ++-
 cpp/src/arrow/io/interfaces.h        |  25 +-
 cpp/src/arrow/io/io-hdfs-test.cc     |  59 +++-
 python/doc/source/api.rst            |  10 +-
 python/doc/source/conf.py            |   3 +-
 python/doc/source/filesystems.rst    |  43 ++-
 python/doc/source/memory.rst         |   7 -
 python/pyarrow/__init__.py           |   6 +-
 python/pyarrow/filesystem.py         | 105 ++++--
 python/pyarrow/hdfs.py               |  34 +-
 python/pyarrow/includes/libarrow.pxd |  15 +-
 python/pyarrow/io-hdfs.pxi           | 468 +++++++++++++++++++++++++++
 python/pyarrow/io.pxi                | 298 -----------------
 python/pyarrow/lib.pyx               |   1 +
 python/pyarrow/parquet.py            |  14 +-
 python/pyarrow/tests/test_hdfs.py    |  97 +++++-
 python/pyarrow/tests/test_parquet.py |   4 +-
 19 files changed, 915 insertions(+), 446 deletions(-)
 create mode 100644 python/pyarrow/io-hdfs.pxi

diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index 8f42b1c817fe4..fd7417b9fba48 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -462,21 +462,11 @@ tOffset LibHdfsShim::GetUsed(hdfsFS fs) { return this->hdfsGetUsed(fs); }
 
 int LibHdfsShim::Chown(hdfsFS fs, const char* path, const char* owner,
                        const char* group) {
-  GET_SYMBOL(this, hdfsChown);
-  if (this->hdfsChown) {
-    return this->hdfsChown(fs, path, owner, group);
-  } else {
-    return 0;
-  }
+  return this->hdfsChown(fs, path, owner, group);
 }
 
 int LibHdfsShim::Chmod(hdfsFS fs, const char* path, short mode) {  // NOLINT
-  GET_SYMBOL(this, hdfsChmod);
-  if (this->hdfsChmod) {
-    return this->hdfsChmod(fs, path, mode);
-  } else {
-    return 0;
-  }
+  return this->hdfsChmod(fs, path, mode);
 }
 
 int LibHdfsShim::Utime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
@@ -504,6 +494,8 @@ Status LibHdfsShim::GetRequiredSymbols() {
   GET_SYMBOL_REQUIRED(this, hdfsGetUsed);
   GET_SYMBOL_REQUIRED(this, hdfsGetPathInfo);
   GET_SYMBOL_REQUIRED(this, hdfsListDirectory);
+  GET_SYMBOL_REQUIRED(this, hdfsChown);
+  GET_SYMBOL_REQUIRED(this, hdfsChmod);
 
   // File methods
   GET_SYMBOL_REQUIRED(this, hdfsCloseFile);
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 500f42caf5277..254e483bb97e8 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -308,9 +308,9 @@ static void SetPathInfo(const hdfsFileInfo* input, HdfsPathInfo* out) {
 }
 
 // Private implementation
-class HdfsClient::HdfsClientImpl {
+class HadoopFileSystem::HadoopFileSystemImpl {
  public:
-  HdfsClientImpl() {}
+  HadoopFileSystemImpl() {}
 
   Status Connect(const HdfsConnectionConfig* config) {
     if (config->driver == HdfsDriver::LIBHDFS3) {
@@ -396,6 +396,24 @@ class HdfsClient::HdfsClientImpl {
     return Status::OK();
   }
 
+  Status Stat(const std::string& path, FileStatistics* stat) {
+    HdfsPathInfo info;
+    RETURN_NOT_OK(GetPathInfo(path, &info));
+
+    stat->size = info.size;
+    stat->kind = info.kind;
+    return Status::OK();
+  }
+
+  Status GetChildren(const std::string& path, std::vector<std::string>* listing) {
+    std::vector<HdfsPathInfo> detailed_listing;
+    RETURN_NOT_OK(ListDirectory(path, &detailed_listing));
+    for (const auto& info : detailed_listing) {
+      listing->push_back(info.name);
+    }
+    return Status::OK();
+  }
+
   Status ListDirectory(const std::string& path, std::vector<HdfsPathInfo>* listing) {
     int num_entries = 0;
     hdfsFileInfo* entries = driver_->ListDirectory(fs_, path.c_str(), &num_entries);
@@ -476,6 +494,18 @@ class HdfsClient::HdfsClientImpl {
     return Status::OK();
   }
 
+  Status Chmod(const std::string& path, int mode) {
+    int ret = driver_->Chmod(fs_, path.c_str(), static_cast<short>(mode));  // NOLINT
+    CHECK_FAILURE(ret, "Chmod");
+    return Status::OK();
+  }
+
+  Status Chown(const std::string& path, const char* owner, const char* group) {
+    int ret = driver_->Chown(fs_, path.c_str(), owner, group);
+    CHECK_FAILURE(ret, "Chown");
+    return Status::OK();
+  }
+
  private:
   LibHdfsShim* driver_;
 
@@ -490,68 +520,92 @@ class HdfsClient::HdfsClientImpl {
 // ----------------------------------------------------------------------
 // Public API for HDFSClient
 
-HdfsClient::HdfsClient() { impl_.reset(new HdfsClientImpl()); }
+HadoopFileSystem::HadoopFileSystem() { impl_.reset(new HadoopFileSystemImpl()); }
 
-HdfsClient::~HdfsClient() {}
+HadoopFileSystem::~HadoopFileSystem() {}
 
-Status HdfsClient::Connect(const HdfsConnectionConfig* config,
-                           std::shared_ptr<HdfsClient>* fs) {
+Status HadoopFileSystem::Connect(const HdfsConnectionConfig* config,
+                                 std::shared_ptr<HadoopFileSystem>* fs) {
   // ctor is private, make_shared will not work
-  *fs = std::shared_ptr<HdfsClient>(new HdfsClient());
+  *fs = std::shared_ptr<HadoopFileSystem>(new HadoopFileSystem());
 
   RETURN_NOT_OK((*fs)->impl_->Connect(config));
   return Status::OK();
 }
 
-Status HdfsClient::MakeDirectory(const std::string& path) {
+Status HadoopFileSystem::MakeDirectory(const std::string& path) {
   return impl_->MakeDirectory(path);
 }
 
-Status HdfsClient::Delete(const std::string& path, bool recursive) {
+Status HadoopFileSystem::Delete(const std::string& path, bool recursive) {
   return impl_->Delete(path, recursive);
 }
 
-Status HdfsClient::Disconnect() { return impl_->Disconnect(); }
+Status HadoopFileSystem::DeleteDirectory(const std::string& path) {
+  return Delete(path, true);
+}
+
+Status HadoopFileSystem::Disconnect() { return impl_->Disconnect(); }
 
-bool HdfsClient::Exists(const std::string& path) { return impl_->Exists(path); }
+bool HadoopFileSystem::Exists(const std::string& path) { return impl_->Exists(path); }
 
-Status HdfsClient::GetPathInfo(const std::string& path, HdfsPathInfo* info) {
+Status HadoopFileSystem::GetPathInfo(const std::string& path, HdfsPathInfo* info) {
   return impl_->GetPathInfo(path, info);
 }
 
-Status HdfsClient::GetCapacity(int64_t* nbytes) { return impl_->GetCapacity(nbytes); }
+Status HadoopFileSystem::Stat(const std::string& path, FileStatistics* stat) {
+  return impl_->Stat(path, stat);
+}
+
+Status HadoopFileSystem::GetCapacity(int64_t* nbytes) {
+  return impl_->GetCapacity(nbytes);
+}
+
+Status HadoopFileSystem::GetUsed(int64_t* nbytes) { return impl_->GetUsed(nbytes); }
 
-Status HdfsClient::GetUsed(int64_t* nbytes) { return impl_->GetUsed(nbytes); }
+Status HadoopFileSystem::GetChildren(const std::string& path,
+                                     std::vector<std::string>* listing) {
+  return impl_->GetChildren(path, listing);
+}
 
-Status HdfsClient::ListDirectory(const std::string& path,
-                                 std::vector<HdfsPathInfo>* listing) {
+Status HadoopFileSystem::ListDirectory(const std::string& path,
+                                       std::vector<HdfsPathInfo>* listing) {
   return impl_->ListDirectory(path, listing);
 }
 
-Status HdfsClient::OpenReadable(const std::string& path, int32_t buffer_size,
-                                std::shared_ptr<HdfsReadableFile>* file) {
+Status HadoopFileSystem::OpenReadable(const std::string& path, int32_t buffer_size,
+                                      std::shared_ptr<HdfsReadableFile>* file) {
   return impl_->OpenReadable(path, buffer_size, file);
 }
 
-Status HdfsClient::OpenReadable(const std::string& path,
-                                std::shared_ptr<HdfsReadableFile>* file) {
+Status HadoopFileSystem::OpenReadable(const std::string& path,
+                                      std::shared_ptr<HdfsReadableFile>* file) {
   return OpenReadable(path, kDefaultHdfsBufferSize, file);
 }
 
-Status HdfsClient::OpenWriteable(const std::string& path, bool append,
-                                 int32_t buffer_size, int16_t replication,
-                                 int64_t default_block_size,
-                                 std::shared_ptr<HdfsOutputStream>* file) {
+Status HadoopFileSystem::OpenWriteable(const std::string& path, bool append,
+                                       int32_t buffer_size, int16_t replication,
+                                       int64_t default_block_size,
+                                       std::shared_ptr<HdfsOutputStream>* file) {
   return impl_->OpenWriteable(path, append, buffer_size, replication, default_block_size,
                               file);
 }
 
-Status HdfsClient::OpenWriteable(const std::string& path, bool append,
-                                 std::shared_ptr<HdfsOutputStream>* file) {
+Status HadoopFileSystem::OpenWriteable(const std::string& path, bool append,
+                                       std::shared_ptr<HdfsOutputStream>* file) {
   return OpenWriteable(path, append, 0, 0, 0, file);
 }
 
-Status HdfsClient::Rename(const std::string& src, const std::string& dst) {
+Status HadoopFileSystem::Chmod(const std::string& path, int mode) {
+  return impl_->Chmod(path, mode);
+}
+
+Status HadoopFileSystem::Chown(const std::string& path, const char* owner,
+                               const char* group) {
+  return impl_->Chown(path, owner, group);
+}
+
+Status HadoopFileSystem::Rename(const std::string& src, const std::string& dst) {
   return impl_->Rename(src, dst);
 }
 
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 63c3ae0d53724..1507ca969cf5f 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -34,7 +34,7 @@ class Status;
 
 namespace io {
 
-class HdfsClient;
+class HadoopFileSystem;
 class HdfsReadableFile;
 class HdfsOutputStream;
 
@@ -66,9 +66,9 @@ struct HdfsConnectionConfig {
   HdfsDriver driver;
 };
 
-class ARROW_EXPORT HdfsClient : public FileSystemClient {
+class ARROW_EXPORT HadoopFileSystem : public FileSystem {
  public:
-  ~HdfsClient();
+  ~HadoopFileSystem();
 
   // Connect to an HDFS cluster given a configuration
   //
@@ -76,13 +76,13 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   // @param fs (out): the created client
   // @returns Status
   static Status Connect(const HdfsConnectionConfig* config,
-                        std::shared_ptr<HdfsClient>* fs);
+                        std::shared_ptr<HadoopFileSystem>* fs);
 
   // Create directory and all parents
   //
   // @param path (in): absolute HDFS path
   // @returns Status
-  Status MakeDirectory(const std::string& path);
+  Status MakeDirectory(const std::string& path) override;
 
   // Delete file or directory
   // @param path: absolute path to data
@@ -90,6 +90,8 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   // @returns error status on failure
   Status Delete(const std::string& path, bool recursive = false);
 
+  Status DeleteDirectory(const std::string& path) override;
+
   // Disconnect from cluster
   //
   // @returns Status
@@ -112,18 +114,29 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   // @returns Status
   Status GetUsed(int64_t* nbytes);
 
+  Status GetChildren(const std::string& path, std::vector<std::string>* listing) override;
+
   Status ListDirectory(const std::string& path, std::vector<HdfsPathInfo>* listing);
 
-  // @param path file path to change
-  // @param owner pass nullptr for no change
-  // @param group pass nullptr for no change
+  /// Change
+  ///
+  /// @param path file path to change
+  /// @param owner pass nullptr for no change
+  /// @param group pass nullptr for no change
   Status Chown(const std::string& path, const char* owner, const char* group);
 
+  /// Change path permissions
+  ///
+  /// \param path Absolute path in file system
+  /// \param mode Mode bitset
+  /// \return Status
   Status Chmod(const std::string& path, int mode);
 
   // Move file or directory from source path to destination path within the
   // current filesystem
-  Status Rename(const std::string& src, const std::string& dst);
+  Status Rename(const std::string& src, const std::string& dst) override;
+
+  Status Stat(const std::string& path, FileStatistics* stat) override;
 
   // TODO(wesm): GetWorkingDirectory, SetWorkingDirectory
 
@@ -152,13 +165,18 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient {
   friend class HdfsReadableFile;
   friend class HdfsOutputStream;
 
-  class ARROW_NO_EXPORT HdfsClientImpl;
-  std::unique_ptr<HdfsClientImpl> impl_;
+  class ARROW_NO_EXPORT HadoopFileSystemImpl;
+  std::unique_ptr<HadoopFileSystemImpl> impl_;
 
-  HdfsClient();
-  DISALLOW_COPY_AND_ASSIGN(HdfsClient);
+  HadoopFileSystem();
+  DISALLOW_COPY_AND_ASSIGN(HadoopFileSystem);
 };
 
+// 0.6.0
+#ifndef ARROW_NO_DEPRECATED_API
+using HdfsClient = HadoopFileSystem;
+#endif
+
 class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
  public:
   ~HdfsReadableFile();
@@ -191,7 +209,7 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
   class ARROW_NO_EXPORT HdfsReadableFileImpl;
   std::unique_ptr<HdfsReadableFileImpl> impl_;
 
-  friend class HdfsClient::HdfsClientImpl;
+  friend class HadoopFileSystem::HadoopFileSystemImpl;
 
   DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
 };
@@ -216,7 +234,7 @@ class ARROW_EXPORT HdfsOutputStream : public OutputStream {
   class ARROW_NO_EXPORT HdfsOutputStreamImpl;
   std::unique_ptr<HdfsOutputStreamImpl> impl_;
 
-  friend class HdfsClient::HdfsClientImpl;
+  friend class HadoopFileSystem::HadoopFileSystemImpl;
 
   HdfsOutputStream();
 
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index e71a5c93baa32..4bb7ebe2fd9f4 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -22,6 +22,7 @@
 #include <memory>
 #include <mutex>
 #include <string>
+#include <vector>
 
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
@@ -42,9 +43,29 @@ struct ObjectType {
   enum type { FILE, DIRECTORY };
 };
 
-class ARROW_EXPORT FileSystemClient {
+struct ARROW_EXPORT FileStatistics {
+  /// Size of file, -1 if finding length is unsupported
+  int64_t size;
+  ObjectType::type kind;
+
+  FileStatistics() {}
+  FileStatistics(int64_t size, ObjectType::type kind) : size(size), kind(kind) {}
+};
+
+class ARROW_EXPORT FileSystem {
  public:
-  virtual ~FileSystemClient() {}
+  virtual ~FileSystem() {}
+
+  virtual Status MakeDirectory(const std::string& path) = 0;
+
+  virtual Status DeleteDirectory(const std::string& path) = 0;
+
+  virtual Status GetChildren(const std::string& path,
+                             std::vector<std::string>* listing) = 0;
+
+  virtual Status Rename(const std::string& src, const std::string& dst) = 0;
+
+  virtual Status Stat(const std::string& path, FileStatistics* stat) = 0;
 };
 
 class ARROW_EXPORT FileInterface {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index c584cf5adeaf2..92f4291346294 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -48,7 +48,7 @@ struct PivotalDriver {
 };
 
 template <typename DRIVER>
-class TestHdfsClient : public ::testing::Test {
+class TestHadoopFileSystem : public ::testing::Test {
  public:
   Status MakeScratchDir() {
     if (client_->Exists(scratch_dir_)) {
@@ -124,7 +124,7 @@ class TestHdfsClient : public ::testing::Test {
     conf_.port = port == nullptr ? 20500 : atoi(port);
     conf_.driver = DRIVER::type;
 
-    ASSERT_OK(HdfsClient::Connect(&conf_, &client_));
+    ASSERT_OK(HadoopFileSystem::Connect(&conf_, &client_));
   }
 
   void TearDown() {
@@ -141,11 +141,11 @@ class TestHdfsClient : public ::testing::Test {
 
   // Resources shared amongst unit tests
   std::string scratch_dir_;
-  std::shared_ptr<HdfsClient> client_;
+  std::shared_ptr<HadoopFileSystem> client_;
 };
 
 template <>
-std::string TestHdfsClient<PivotalDriver>::HdfsAbsPath(const std::string& relpath) {
+std::string TestHadoopFileSystem<PivotalDriver>::HdfsAbsPath(const std::string& relpath) {
   std::stringstream ss;
   ss << relpath;
   return ss.str();
@@ -161,17 +161,17 @@ HdfsDriver JNIDriver::type = HdfsDriver::LIBHDFS;
 HdfsDriver PivotalDriver::type = HdfsDriver::LIBHDFS3;
 
 typedef ::testing::Types<JNIDriver, PivotalDriver> DriverTypes;
-TYPED_TEST_CASE(TestHdfsClient, DriverTypes);
+TYPED_TEST_CASE(TestHadoopFileSystem, DriverTypes);
 
-TYPED_TEST(TestHdfsClient, ConnectsAgain) {
+TYPED_TEST(TestHadoopFileSystem, ConnectsAgain) {
   SKIP_IF_NO_DRIVER();
 
-  std::shared_ptr<HdfsClient> client;
-  ASSERT_OK(HdfsClient::Connect(&this->conf_, &client));
+  std::shared_ptr<HadoopFileSystem> client;
+  ASSERT_OK(HadoopFileSystem::Connect(&this->conf_, &client));
   ASSERT_OK(client->Disconnect());
 }
 
-TYPED_TEST(TestHdfsClient, MakeDirectory) {
+TYPED_TEST(TestHadoopFileSystem, MakeDirectory) {
   SKIP_IF_NO_DRIVER();
 
   std::string path = this->ScratchPath("create-directory");
@@ -190,7 +190,7 @@ TYPED_TEST(TestHdfsClient, MakeDirectory) {
   ASSERT_RAISES(IOError, this->client_->ListDirectory(path, &listing));
 }
 
-TYPED_TEST(TestHdfsClient, GetCapacityUsed) {
+TYPED_TEST(TestHadoopFileSystem, GetCapacityUsed) {
   SKIP_IF_NO_DRIVER();
 
   // Who knows what is actually in your DFS cluster, but expect it to have
@@ -203,7 +203,7 @@ TYPED_TEST(TestHdfsClient, GetCapacityUsed) {
   ASSERT_LT(0, nbytes);
 }
 
-TYPED_TEST(TestHdfsClient, GetPathInfo) {
+TYPED_TEST(TestHadoopFileSystem, GetPathInfo) {
   SKIP_IF_NO_DRIVER();
 
   HdfsPathInfo info;
@@ -233,7 +233,7 @@ TYPED_TEST(TestHdfsClient, GetPathInfo) {
   ASSERT_EQ(size, info.size);
 }
 
-TYPED_TEST(TestHdfsClient, AppendToFile) {
+TYPED_TEST(TestHadoopFileSystem, AppendToFile) {
   SKIP_IF_NO_DRIVER();
 
   ASSERT_OK(this->MakeScratchDir());
@@ -252,7 +252,7 @@ TYPED_TEST(TestHdfsClient, AppendToFile) {
   ASSERT_EQ(size * 2, info.size);
 }
 
-TYPED_TEST(TestHdfsClient, ListDirectory) {
+TYPED_TEST(TestHadoopFileSystem, ListDirectory) {
   SKIP_IF_NO_DRIVER();
 
   const int size = 100;
@@ -292,7 +292,7 @@ TYPED_TEST(TestHdfsClient, ListDirectory) {
   }
 }
 
-TYPED_TEST(TestHdfsClient, ReadableMethods) {
+TYPED_TEST(TestHadoopFileSystem, ReadableMethods) {
   SKIP_IF_NO_DRIVER();
 
   ASSERT_OK(this->MakeScratchDir());
@@ -339,7 +339,7 @@ TYPED_TEST(TestHdfsClient, ReadableMethods) {
   ASSERT_EQ(60, position);
 }
 
-TYPED_TEST(TestHdfsClient, LargeFile) {
+TYPED_TEST(TestHadoopFileSystem, LargeFile) {
   SKIP_IF_NO_DRIVER();
 
   ASSERT_OK(this->MakeScratchDir());
@@ -374,7 +374,7 @@ TYPED_TEST(TestHdfsClient, LargeFile) {
   ASSERT_EQ(size, bytes_read);
 }
 
-TYPED_TEST(TestHdfsClient, RenameFile) {
+TYPED_TEST(TestHadoopFileSystem, RenameFile) {
   SKIP_IF_NO_DRIVER();
   ASSERT_OK(this->MakeScratchDir());
 
@@ -391,7 +391,32 @@ TYPED_TEST(TestHdfsClient, RenameFile) {
   ASSERT_TRUE(this->client_->Exists(dst_path));
 }
 
-TYPED_TEST(TestHdfsClient, ThreadSafety) {
+TYPED_TEST(TestHadoopFileSystem, ChmodChown) {
+  SKIP_IF_NO_DRIVER();
+  ASSERT_OK(this->MakeScratchDir());
+
+  auto path = this->ScratchPath("path-to-chmod");
+
+  int16_t mode = 0755;
+  const int size = 100;
+
+  std::vector<uint8_t> data = RandomData(size);
+  ASSERT_OK(this->WriteDummyFile(path, data.data(), size));
+
+  HdfsPathInfo info;
+  ASSERT_OK(this->client_->Chmod(path, mode));
+  ASSERT_OK(this->client_->GetPathInfo(path, &info));
+  ASSERT_EQ(mode, info.permissions);
+
+  std::string owner = "hadoop";
+  std::string group = "hadoop";
+  ASSERT_OK(this->client_->Chown(path, owner.c_str(), group.c_str()));
+  ASSERT_OK(this->client_->GetPathInfo(path, &info));
+  ASSERT_EQ("hadoop", info.owner);
+  ASSERT_EQ("hadoop", info.group);
+}
+
+TYPED_TEST(TestHadoopFileSystem, ThreadSafety) {
   SKIP_IF_NO_DRIVER();
   ASSERT_OK(this->MakeScratchDir());
 
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 6f26076bf63cb..fd1cb728d9828 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -164,15 +164,17 @@ Input / Output and Shared Memory
    create_memory_map
    PythonFile
 
-Filesystems
------------
+File Systems
+------------
 
 .. autosummary::
    :toctree: generated/
 
    hdfs.connect
-   HadoopFilesystem
-   LocalFilesystem
+   LocalFileSystem
+
+.. class:: HadoopFileSystem
+   :noindex:
 
 .. _api.ipc:
 
diff --git a/python/doc/source/conf.py b/python/doc/source/conf.py
index d71b66870729f..25e6d5e2d44cf 100644
--- a/python/doc/source/conf.py
+++ b/python/doc/source/conf.py
@@ -62,7 +62,8 @@
 ]
 
 # Show members for classes in .. autosummary
-autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance', 'inherited-members']
+autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance',
+                         'inherited-members']
 
 # numpydoc configuration
 napoleon_use_rtype = False
diff --git a/python/doc/source/filesystems.rst b/python/doc/source/filesystems.rst
index 78f6f2a45eace..c0530f93c2cdd 100644
--- a/python/doc/source/filesystems.rst
+++ b/python/doc/source/filesystems.rst
@@ -15,8 +15,8 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
-Filesystem Interfaces
-=====================
+File System Interfaces
+======================
 
 In this section, we discuss filesystem-like interfaces in PyArrow.
 
@@ -31,10 +31,11 @@ System. You connect like so:
 .. code-block:: python
 
    import pyarrow as pa
-   hdfs = pa.hdfs.connect(host, port, user=user, kerb_ticket=ticket_cache_path)
-   type(hdfs)
+   fs = pa.hdfs.connect(host, port, user=user, kerb_ticket=ticket_cache_path)
+   with fs.open(path, 'rb') as f:
+       # Do something with f
 
-By default, ``pyarrow.hdfs.HadoopFilesystem`` uses libhdfs, a JNI-based
+By default, ``pyarrow.hdfs.HadoopFileSystem`` uses libhdfs, a JNI-based
 interface to the Java Hadoop client. This library is loaded **at runtime**
 (rather than at link / library load time, since the library may not be in your
 LD_LIBRARY_PATH), and relies on some environment variables.
@@ -57,5 +58,33 @@ You can also use libhdfs3, a thirdparty C++ library for HDFS from Pivotal Labs:
 
 .. code-block:: python
 
-   hdfs3 = pa.hdfs.connect(host, port, user=user, kerb_ticket=ticket_cache_path,
-                           driver='libhdfs3')
+   fs = pa.hdfs.connect(host, port, user=user, kerb_ticket=ticket_cache_path,
+                       driver='libhdfs3')
+
+HDFS API
+~~~~~~~~
+
+.. currentmodule:: pyarrow
+
+.. autosummary::
+   :toctree: generated/
+
+   hdfs.connect
+   HadoopFileSystem.cat
+   HadoopFileSystem.chmod
+   HadoopFileSystem.chown
+   HadoopFileSystem.delete
+   HadoopFileSystem.df
+   HadoopFileSystem.disk_usage
+   HadoopFileSystem.download
+   HadoopFileSystem.exists
+   HadoopFileSystem.get_capacity
+   HadoopFileSystem.get_space_used
+   HadoopFileSystem.info
+   HadoopFileSystem.ls
+   HadoopFileSystem.mkdir
+   HadoopFileSystem.open
+   HadoopFileSystem.rename
+   HadoopFileSystem.rm
+   HadoopFileSystem.upload
+   HdfsFile
diff --git a/python/doc/source/memory.rst b/python/doc/source/memory.rst
index ccc6298b66154..f18919999e062 100644
--- a/python/doc/source/memory.rst
+++ b/python/doc/source/memory.rst
@@ -226,10 +226,3 @@ file interfaces that can read and write to Arrow Buffers.
    reader.read(7)
 
 These have similar semantics to Python's built-in ``io.BytesIO``.
-
-Hadoop Filesystem
------------------
-
-:class:`~pyarrow.HdfsFile` is an implementation of :class:`~pyarrow.NativeFile`
-that can read and write to the Hadoop filesytem. Read more in the
-:ref:`Filesystems Section <hdfs>`.
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 42e58034f3535..8d4a214ba263d 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -88,9 +88,9 @@
                          ArrowTypeError)
 
 
-from pyarrow.filesystem import Filesystem, LocalFilesystem
+from pyarrow.filesystem import FileSystem, LocalFileSystem
 
-from pyarrow.hdfs import HadoopFilesystem
+from pyarrow.hdfs import HadoopFileSystem
 import pyarrow.hdfs as hdfs
 
 from pyarrow.ipc import (Message, MessageReader,
@@ -103,7 +103,7 @@
                          open_file,
                          serialize_pandas, deserialize_pandas)
 
-localfs = LocalFilesystem.get_instance()
+localfs = LocalFileSystem.get_instance()
 
 
 # ----------------------------------------------------------------------
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 4b8ca3279ce12..8d2d8fcd342f7 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -22,10 +22,21 @@
 from pyarrow.util import implements
 
 
-class Filesystem(object):
+class FileSystem(object):
     """
     Abstract filesystem interface
     """
+    def cat(self, path):
+        """
+        Return contents of file as a bytes object
+
+        Returns
+        -------
+        contents : bytes
+        """
+        with self.open(path, 'rb') as f:
+            return f.read()
+
     def ls(self, path):
         """
         Return list of file paths
@@ -44,12 +55,68 @@ def delete(self, path, recursive=False):
         """
         raise NotImplementedError
 
+    def disk_usage(self, path):
+        """
+        Compute bytes used by all contents under indicated path in file tree
+
+        Parameters
+        ----------
+        path : string
+            Can be a file path or directory
+
+        Returns
+        -------
+        usage : int
+        """
+        path_info = self.stat(path)
+        if path_info['kind'] == 'file':
+            return path_info['size']
+
+        total = 0
+        for root, directories, files in self.walk(path):
+            for child_path in files:
+                abspath = self._path_join(root, child_path)
+                total += self.stat(abspath)['size']
+
+        return total
+
+    def _path_join(self, *args):
+        return self.pathsep.join(args)
+
+    def stat(self, path):
+        """
+
+        Returns
+        -------
+        stat : dict
+        """
+        raise NotImplementedError('FileSystem.stat')
+
     def rm(self, path, recursive=False):
         """
-        Alias for Filesystem.delete
+        Alias for FileSystem.delete
         """
         return self.delete(path, recursive=recursive)
 
+    def mv(self, path, new_path):
+        """
+        Alias for FileSystem.rename
+        """
+        return self.rename(path, new_path)
+
+    def rename(self, path, new_path):
+        """
+        Rename file, like UNIX mv command
+
+        Parameters
+        ----------
+        path : string
+            Path to alter
+        new_path : string
+            Path to move to
+        """
+        raise NotImplementedError('FileSystem.rename')
+
     def mkdir(self, path, create_parents=True):
         raise NotImplementedError
 
@@ -113,40 +180,40 @@ def pathsep(self):
         return '/'
 
 
-class LocalFilesystem(Filesystem):
+class LocalFileSystem(FileSystem):
 
     _instance = None
 
     @classmethod
     def get_instance(cls):
         if cls._instance is None:
-            cls._instance = LocalFilesystem()
+            cls._instance = LocalFileSystem()
         return cls._instance
 
-    @implements(Filesystem.ls)
+    @implements(FileSystem.ls)
     def ls(self, path):
         return sorted(pjoin(path, x) for x in os.listdir(path))
 
-    @implements(Filesystem.mkdir)
+    @implements(FileSystem.mkdir)
     def mkdir(self, path, create_parents=True):
         if create_parents:
             os.makedirs(path)
         else:
             os.mkdir(path)
 
-    @implements(Filesystem.isdir)
+    @implements(FileSystem.isdir)
     def isdir(self, path):
         return os.path.isdir(path)
 
-    @implements(Filesystem.isfile)
+    @implements(FileSystem.isfile)
     def isfile(self, path):
         return os.path.isfile(path)
 
-    @implements(Filesystem.exists)
+    @implements(FileSystem.exists)
     def exists(self, path):
         return os.path.exists(path)
 
-    @implements(Filesystem.open)
+    @implements(FileSystem.open)
     def open(self, path, mode='rb'):
         """
         Open file for reading or writing
@@ -164,7 +231,7 @@ def walk(self, top_dir):
         return os.walk(top_dir)
 
 
-class DaskFilesystem(Filesystem):
+class DaskFileSystem(FileSystem):
     """
     Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.
     """
@@ -172,23 +239,23 @@ class DaskFilesystem(Filesystem):
     def __init__(self, fs):
         self.fs = fs
 
-    @implements(Filesystem.isdir)
+    @implements(FileSystem.isdir)
     def isdir(self, path):
         raise NotImplementedError("Unsupported file system API")
 
-    @implements(Filesystem.isfile)
+    @implements(FileSystem.isfile)
     def isfile(self, path):
         raise NotImplementedError("Unsupported file system API")
 
-    @implements(Filesystem.delete)
+    @implements(FileSystem.delete)
     def delete(self, path, recursive=False):
         return self.fs.rm(path, recursive=recursive)
 
-    @implements(Filesystem.mkdir)
+    @implements(FileSystem.mkdir)
     def mkdir(self, path):
         return self.fs.mkdir(path)
 
-    @implements(Filesystem.open)
+    @implements(FileSystem.open)
     def open(self, path, mode='rb'):
         """
         Open file for reading or writing
@@ -205,9 +272,9 @@ def walk(self, top_path):
         return self.fs.walk(top_path)
 
 
-class S3FSWrapper(DaskFilesystem):
+class S3FSWrapper(DaskFileSystem):
 
-    @implements(Filesystem.isdir)
+    @implements(FileSystem.isdir)
     def isdir(self, path):
         try:
             contents = self.fs.ls(path)
@@ -218,7 +285,7 @@ def isdir(self, path):
         except OSError:
             return False
 
-    @implements(Filesystem.isfile)
+    @implements(FileSystem.isfile)
     def isfile(self, path):
         try:
             contents = self.fs.ls(path)
diff --git a/python/pyarrow/hdfs.py b/python/pyarrow/hdfs.py
index 3240f99bc6bb8..855cc1e76bd92 100644
--- a/python/pyarrow/hdfs.py
+++ b/python/pyarrow/hdfs.py
@@ -18,13 +18,13 @@
 import posixpath
 
 from pyarrow.util import implements
-from pyarrow.filesystem import Filesystem
+from pyarrow.filesystem import FileSystem
 import pyarrow.lib as lib
 
 
-class HadoopFilesystem(lib._HdfsClient, Filesystem):
+class HadoopFileSystem(lib.HadoopFileSystem, FileSystem):
     """
-    Filesystem interface for HDFS cluster. See pyarrow.hdfs.connect for full
+    FileSystem interface for HDFS cluster. See pyarrow.hdfs.connect for full
     connection details
     """
 
@@ -32,21 +32,25 @@ def __init__(self, host="default", port=0, user=None, kerb_ticket=None,
                  driver='libhdfs'):
         self._connect(host, port, user, kerb_ticket, driver)
 
-    @implements(Filesystem.isdir)
+    @implements(FileSystem.isdir)
     def isdir(self, path):
-        return lib._HdfsClient.isdir(self, path)
+        return super(HadoopFileSystem, self).isdir(path)
 
-    @implements(Filesystem.isfile)
+    @implements(FileSystem.isfile)
     def isfile(self, path):
-        return lib._HdfsClient.isfile(self, path)
+        return super(HadoopFileSystem, self).isfile(path)
 
-    @implements(Filesystem.delete)
+    @implements(FileSystem.delete)
     def delete(self, path, recursive=False):
-        return lib._HdfsClient.delete(self, path, recursive)
+        return super(HadoopFileSystem, self).delete(path, recursive)
 
-    @implements(Filesystem.mkdir)
+    @implements(FileSystem.mkdir)
     def mkdir(self, path, create_parents=True):
-        return lib._HdfsClient.mkdir(self, path)
+        return super(HadoopFileSystem, self).mkdir(path)
+
+    @implements(FileSystem.rename)
+    def rename(self, path, new_path):
+        return super(HadoopFileSystem, self).rename(path, new_path)
 
     def ls(self, path, detail=False):
         """
@@ -62,7 +66,7 @@ def ls(self, path, detail=False):
         -------
         result : list of dicts (detail=True) or strings (detail=False)
         """
-        return lib._HdfsClient.ls(self, path, detail)
+        return super(HadoopFileSystem, self).ls(path, detail)
 
     def walk(self, top_path):
         """
@@ -82,7 +86,7 @@ def walk(self, top_path):
         directories, files = _libhdfs_walk_files_dirs(top_path, contents)
         yield top_path, directories, files
         for dirname in directories:
-            for tup in self.walk(dirname):
+            for tup in self.walk(self._path_join(top_path, dirname)):
                 yield tup
 
 
@@ -126,8 +130,8 @@ def connect(host="default", port=0, user=None, kerb_ticket=None,
 
     Returns
     -------
-    filesystem : HadoopFilesystem
+    filesystem : HadoopFileSystem
     """
-    fs = HadoopFilesystem(host=host, port=port, user=user,
+    fs = HadoopFileSystem(host=host, port=port, user=user,
                           kerb_ticket=kerb_ticket, driver=driver)
     return fs
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index db6770f586bd5..8d7e27915eede 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -413,6 +413,10 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
         ObjectType_FILE" arrow::io::ObjectType::FILE"
         ObjectType_DIRECTORY" arrow::io::ObjectType::DIRECTORY"
 
+    cdef cppclass FileStatistics:
+        int64_t size
+        ObjectType kind
+
     cdef cppclass FileInterface:
         CStatus Close()
         CStatus Tell(int64_t* position)
@@ -450,6 +454,9 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
                                          WriteableFile):
         pass
 
+    cdef cppclass FileSystem:
+        CStatus Stat(const c_string& path, FileStatistics* stat)
+
 
 cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
 
@@ -517,10 +524,10 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
     cdef cppclass HdfsOutputStream(OutputStream):
         pass
 
-    cdef cppclass CHdfsClient" arrow::io::HdfsClient":
+    cdef cppclass CHadoopFileSystem" arrow::io::HadoopFileSystem"(FileSystem):
         @staticmethod
         CStatus Connect(const HdfsConnectionConfig* config,
-                        shared_ptr[CHdfsClient]* client)
+                        shared_ptr[CHadoopFileSystem]* client)
 
         CStatus MakeDirectory(const c_string& path)
 
@@ -530,6 +537,10 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
 
         c_bool Exists(const c_string& path)
 
+        CStatus Chmod(const c_string& path, int mode)
+        CStatus Chown(const c_string& path, const char* owner,
+                      const char* group)
+
         CStatus GetCapacity(int64_t* nbytes)
         CStatus GetUsed(int64_t* nbytes)
 
diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi
new file mode 100644
index 0000000000000..8ac4e8c2319c1
--- /dev/null
+++ b/python/pyarrow/io-hdfs.pxi
@@ -0,0 +1,468 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ----------------------------------------------------------------------
+# HDFS IO implementation
+
+_HDFS_PATH_RE = re.compile('hdfs://(.*):(\d+)(.*)')
+
+try:
+    # Python 3
+    from queue import Queue, Empty as QueueEmpty, Full as QueueFull
+except ImportError:
+    from Queue import Queue, Empty as QueueEmpty, Full as QueueFull
+
+
+def have_libhdfs():
+    try:
+        check_status(HaveLibHdfs())
+        return True
+    except:
+        return False
+
+
+def have_libhdfs3():
+    try:
+        check_status(HaveLibHdfs3())
+        return True
+    except:
+        return False
+
+
+def strip_hdfs_abspath(path):
+    m = _HDFS_PATH_RE.match(path)
+    if m:
+        return m.group(3)
+    else:
+        return path
+
+
+cdef class HadoopFileSystem:
+    cdef:
+        shared_ptr[CHadoopFileSystem] client
+
+    cdef readonly:
+        bint is_open
+
+    def __cinit__(self):
+        pass
+
+    def _connect(self, host, port, user, kerb_ticket, driver):
+        cdef HdfsConnectionConfig conf
+
+        if host is not None:
+            conf.host = tobytes(host)
+        conf.port = port
+        if user is not None:
+            conf.user = tobytes(user)
+        if kerb_ticket is not None:
+            conf.kerb_ticket = tobytes(kerb_ticket)
+
+        if driver == 'libhdfs':
+            check_status(HaveLibHdfs())
+            conf.driver = HdfsDriver_LIBHDFS
+        else:
+            check_status(HaveLibHdfs3())
+            conf.driver = HdfsDriver_LIBHDFS3
+
+        with nogil:
+            check_status(CHadoopFileSystem.Connect(&conf, &self.client))
+        self.is_open = True
+
+    @classmethod
+    def connect(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def __dealloc__(self):
+        if self.is_open:
+            self.close()
+
+    def close(self):
+        """
+        Disconnect from the HDFS cluster
+        """
+        self._ensure_client()
+        with nogil:
+            check_status(self.client.get().Disconnect())
+        self.is_open = False
+
+    cdef _ensure_client(self):
+        if self.client.get() == NULL:
+            raise IOError('HDFS client improperly initialized')
+        elif not self.is_open:
+            raise IOError('HDFS client is closed')
+
+    def exists(self, path):
+        """
+        Returns True if the path is known to the cluster, False if it does not
+        (or there is an RPC error)
+        """
+        self._ensure_client()
+
+        cdef c_string c_path = tobytes(path)
+        cdef c_bool result
+        with nogil:
+            result = self.client.get().Exists(c_path)
+        return result
+
+    def isdir(self, path):
+        cdef HdfsPathInfo info
+        self._path_info(path, &info)
+        return info.kind == ObjectType_DIRECTORY
+
+    def isfile(self, path):
+        cdef HdfsPathInfo info
+        self._path_info(path, &info)
+        return info.kind == ObjectType_FILE
+
+    def get_capacity(self):
+        """
+        Get reported total capacity of file system
+
+        Returns
+        -------
+        capacity : int
+        """
+        cdef int64_t capacity = 0
+        with nogil:
+            check_status(self.client.get().GetCapacity(&capacity))
+        return capacity
+
+    def get_space_used(self):
+        """
+        Get space used on file system
+
+        Returns
+        -------
+        space_used : int
+        """
+        cdef int64_t space_used = 0
+        with nogil:
+            check_status(self.client.get().GetUsed(&space_used))
+        return space_used
+
+    def df(self):
+        """
+        Return free space on disk, like the UNIX df command
+
+        Returns
+        -------
+        space : int
+        """
+        return self.get_capacity() - self.get_space_used()
+
+    def rename(self, path, new_path):
+        cdef c_string c_path = tobytes(path)
+        cdef c_string c_new_path = tobytes(new_path)
+        with nogil:
+            check_status(self.client.get().Rename(c_path, c_new_path))
+
+    def info(self, path):
+        """
+        Return detailed HDFS information for path
+
+        Parameters
+        ----------
+        path : string
+            Path to file or directory
+
+        Returns
+        -------
+        path_info : dict
+        """
+        cdef HdfsPathInfo info
+        self._path_info(path, &info)
+        return {
+            'path': frombytes(info.name),
+            'owner': frombytes(info.owner),
+            'group': frombytes(info.group),
+            'size': info.size,
+            'block_size': info.block_size,
+            'last_modified': info.last_modified_time,
+            'last_accessed': info.last_access_time,
+            'replication': info.replication,
+            'permissions': info.permissions,
+            'kind': ('directory' if info.kind == ObjectType_DIRECTORY
+                     else 'file')
+        }
+
+    def stat(self, path):
+        """
+        Return basic file system statistics about path
+
+        Parameters
+        ----------
+        path : string
+            Path to file or directory
+
+        Returns
+        -------
+        stat : dict
+        """
+        cdef FileStatistics info
+        cdef c_string c_path = tobytes(path)
+        with nogil:
+            check_status(self.client.get()
+                         .Stat(c_path, &info))
+        return {
+            'size': info.size,
+            'kind': ('directory' if info.kind == ObjectType_DIRECTORY
+                     else 'file')
+        }
+
+    cdef _path_info(self, path, HdfsPathInfo* info):
+        cdef c_string c_path = tobytes(path)
+
+        with nogil:
+            check_status(self.client.get()
+                         .GetPathInfo(c_path, info))
+
+
+    def ls(self, path, bint full_info):
+        cdef:
+            c_string c_path = tobytes(path)
+            vector[HdfsPathInfo] listing
+            list results = []
+            int i
+
+        self._ensure_client()
+
+        with nogil:
+            check_status(self.client.get()
+                         .ListDirectory(c_path, &listing))
+
+        cdef const HdfsPathInfo* info
+        for i in range(<int> listing.size()):
+            info = &listing[i]
+
+            # Try to trim off the hdfs://HOST:PORT piece
+            name = strip_hdfs_abspath(frombytes(info.name))
+
+            if full_info:
+                kind = ('file' if info.kind == ObjectType_FILE
+                        else 'directory')
+
+                results.append({
+                    'kind': kind,
+                    'name': name,
+                    'owner': frombytes(info.owner),
+                    'group': frombytes(info.group),
+                    'list_modified_time': info.last_modified_time,
+                    'list_access_time': info.last_access_time,
+                    'size': info.size,
+                    'replication': info.replication,
+                    'block_size': info.block_size,
+                    'permissions': info.permissions
+                })
+            else:
+                results.append(name)
+
+        return results
+
+    def chmod(self, path, mode):
+        """
+        Change file permissions
+
+        Parameters
+        ----------
+        path : string
+            absolute path to file or directory
+        mode : int
+            POSIX-like bitmask
+        """
+        self._ensure_client()
+        cdef c_string c_path = tobytes(path)
+        cdef int c_mode = mode
+        with nogil:
+            check_status(self.client.get()
+                         .Chmod(c_path, c_mode))
+
+    def chown(self, path, owner=None, group=None):
+        """
+        Change file permissions
+
+        Parameters
+        ----------
+        path : string
+            absolute path to file or directory
+        owner : string, default None
+            New owner, None for no change
+        group : string, default None
+            New group, None for no change
+        """
+        cdef:
+            c_string c_path
+            c_string c_owner
+            c_string c_group
+            const char* c_owner_ptr = NULL
+            const char* c_group_ptr = NULL
+
+        self._ensure_client()
+
+        c_path = tobytes(path)
+        if owner is not None:
+            c_owner = tobytes(owner)
+            c_owner_ptr = c_owner.c_str()
+
+        if group is not None:
+            c_group = tobytes(group)
+            c_group_ptr = c_group.c_str()
+
+        with nogil:
+            check_status(self.client.get()
+                         .Chown(c_path, c_owner_ptr, c_group_ptr))
+
+    def mkdir(self, path):
+        """
+        Create indicated directory and any necessary parent directories
+        """
+        self._ensure_client()
+        cdef c_string c_path = tobytes(path)
+        with nogil:
+            check_status(self.client.get()
+                         .MakeDirectory(c_path))
+
+    def delete(self, path, bint recursive=False):
+        """
+        Delete the indicated file or directory
+
+        Parameters
+        ----------
+        path : string
+        recursive : boolean, default False
+            If True, also delete child paths for directories
+        """
+        self._ensure_client()
+
+        cdef c_string c_path = tobytes(path)
+        with nogil:
+            check_status(self.client.get()
+                         .Delete(c_path, recursive == 1))
+
+    def open(self, path, mode='rb', buffer_size=None, replication=None,
+             default_block_size=None):
+        """
+        Open HDFS file for reading or writing
+
+        Parameters
+        ----------
+        mode : string
+            Must be one of 'rb', 'wb', 'ab'
+
+        Returns
+        -------
+        handle : HdfsFile
+        """
+        self._ensure_client()
+
+        cdef HdfsFile out = HdfsFile()
+
+        if mode not in ('rb', 'wb', 'ab'):
+            raise Exception("Mode must be 'rb' (read), "
+                            "'wb' (write, new file), or 'ab' (append)")
+
+        cdef c_string c_path = tobytes(path)
+        cdef c_bool append = False
+
+        # 0 in libhdfs means "use the default"
+        cdef int32_t c_buffer_size = buffer_size or 0
+        cdef int16_t c_replication = replication or 0
+        cdef int64_t c_default_block_size = default_block_size or 0
+
+        cdef shared_ptr[HdfsOutputStream] wr_handle
+        cdef shared_ptr[HdfsReadableFile] rd_handle
+
+        if mode in ('wb', 'ab'):
+            if mode == 'ab':
+                append = True
+
+            with nogil:
+                check_status(
+                    self.client.get()
+                    .OpenWriteable(c_path, append, c_buffer_size,
+                                   c_replication, c_default_block_size,
+                                   &wr_handle))
+
+            out.wr_file = <shared_ptr[OutputStream]> wr_handle
+
+            out.is_readable = False
+            out.is_writeable = 1
+        else:
+            with nogil:
+                check_status(self.client.get()
+                             .OpenReadable(c_path, &rd_handle))
+
+            out.rd_file = <shared_ptr[RandomAccessFile]> rd_handle
+            out.is_readable = True
+            out.is_writeable = 0
+
+        if c_buffer_size == 0:
+            c_buffer_size = 2 ** 16
+
+        out.mode = mode
+        out.buffer_size = c_buffer_size
+        out.parent = _HdfsFileNanny(self, out)
+        out.is_open = True
+        out.own_file = True
+
+        return out
+
+    def download(self, path, stream, buffer_size=None):
+        with self.open(path, 'rb') as f:
+            f.download(stream, buffer_size=buffer_size)
+
+    def upload(self, path, stream, buffer_size=None):
+        """
+        Upload file-like object to HDFS path
+        """
+        with self.open(path, 'wb') as f:
+            f.upload(stream, buffer_size=buffer_size)
+
+
+# ARROW-404: Helper class to ensure that files are closed before the
+# client. During deallocation of the extension class, the attributes are
+# decref'd which can cause the client to get closed first if the file has the
+# last remaining reference
+cdef class _HdfsFileNanny:
+    cdef:
+        object client
+        object file_handle_ref
+
+    def __cinit__(self, client, file_handle):
+        import weakref
+        self.client = client
+        self.file_handle_ref = weakref.ref(file_handle)
+
+    def __dealloc__(self):
+        fh = self.file_handle_ref()
+        if fh:
+            fh.close()
+        # avoid cyclic GC
+        self.file_handle_ref = None
+        self.client = None
+
+
+cdef class HdfsFile(NativeFile):
+    cdef readonly:
+        int32_t buffer_size
+        object mode
+        object parent
+
+    cdef object __weakref__
+
+    def __dealloc__(self):
+        self.parent = None
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 01c987d286f92..211c2a3e6e9cf 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -733,301 +733,3 @@ cdef get_writer(object source, shared_ptr[OutputStream]* writer):
     else:
         raise TypeError('Unable to read from object of type: {0}'
                         .format(type(source)))
-
-# ----------------------------------------------------------------------
-# HDFS IO implementation
-
-_HDFS_PATH_RE = re.compile('hdfs://(.*):(\d+)(.*)')
-
-try:
-    # Python 3
-    from queue import Queue, Empty as QueueEmpty, Full as QueueFull
-except ImportError:
-    from Queue import Queue, Empty as QueueEmpty, Full as QueueFull
-
-
-def have_libhdfs():
-    try:
-        check_status(HaveLibHdfs())
-        return True
-    except:
-        return False
-
-
-def have_libhdfs3():
-    try:
-        check_status(HaveLibHdfs3())
-        return True
-    except:
-        return False
-
-
-def strip_hdfs_abspath(path):
-    m = _HDFS_PATH_RE.match(path)
-    if m:
-        return m.group(3)
-    else:
-        return path
-
-
-cdef class _HdfsClient:
-    cdef:
-        shared_ptr[CHdfsClient] client
-
-    cdef readonly:
-        bint is_open
-
-    def __cinit__(self):
-        pass
-
-    def _connect(self, host, port, user, kerb_ticket, driver):
-        cdef HdfsConnectionConfig conf
-
-        if host is not None:
-            conf.host = tobytes(host)
-        conf.port = port
-        if user is not None:
-            conf.user = tobytes(user)
-        if kerb_ticket is not None:
-            conf.kerb_ticket = tobytes(kerb_ticket)
-
-        if driver == 'libhdfs':
-            check_status(HaveLibHdfs())
-            conf.driver = HdfsDriver_LIBHDFS
-        else:
-            check_status(HaveLibHdfs3())
-            conf.driver = HdfsDriver_LIBHDFS3
-
-        with nogil:
-            check_status(CHdfsClient.Connect(&conf, &self.client))
-        self.is_open = True
-
-    @classmethod
-    def connect(cls, *args, **kwargs):
-        return cls(*args, **kwargs)
-
-    def __dealloc__(self):
-        if self.is_open:
-            self.close()
-
-    def close(self):
-        """
-        Disconnect from the HDFS cluster
-        """
-        self._ensure_client()
-        with nogil:
-            check_status(self.client.get().Disconnect())
-        self.is_open = False
-
-    cdef _ensure_client(self):
-        if self.client.get() == NULL:
-            raise IOError('HDFS client improperly initialized')
-        elif not self.is_open:
-            raise IOError('HDFS client is closed')
-
-    def exists(self, path):
-        """
-        Returns True if the path is known to the cluster, False if it does not
-        (or there is an RPC error)
-        """
-        self._ensure_client()
-
-        cdef c_string c_path = tobytes(path)
-        cdef c_bool result
-        with nogil:
-            result = self.client.get().Exists(c_path)
-        return result
-
-    def isdir(self, path):
-        cdef HdfsPathInfo info
-        self._path_info(path, &info)
-        return info.kind == ObjectType_DIRECTORY
-
-    def isfile(self, path):
-        cdef HdfsPathInfo info
-        self._path_info(path, &info)
-        return info.kind == ObjectType_FILE
-
-    cdef _path_info(self, path, HdfsPathInfo* info):
-        cdef c_string c_path = tobytes(path)
-
-        with nogil:
-            check_status(self.client.get()
-                         .GetPathInfo(c_path, info))
-
-
-    def ls(self, path, bint full_info):
-        cdef:
-            c_string c_path = tobytes(path)
-            vector[HdfsPathInfo] listing
-            list results = []
-            int i
-
-        self._ensure_client()
-
-        with nogil:
-            check_status(self.client.get()
-                         .ListDirectory(c_path, &listing))
-
-        cdef const HdfsPathInfo* info
-        for i in range(<int> listing.size()):
-            info = &listing[i]
-
-            # Try to trim off the hdfs://HOST:PORT piece
-            name = strip_hdfs_abspath(frombytes(info.name))
-
-            if full_info:
-                kind = ('file' if info.kind == ObjectType_FILE
-                        else 'directory')
-
-                results.append({
-                    'kind': kind,
-                    'name': name,
-                    'owner': frombytes(info.owner),
-                    'group': frombytes(info.group),
-                    'list_modified_time': info.last_modified_time,
-                    'list_access_time': info.last_access_time,
-                    'size': info.size,
-                    'replication': info.replication,
-                    'block_size': info.block_size,
-                    'permissions': info.permissions
-                })
-            else:
-                results.append(name)
-
-        return results
-
-    def mkdir(self, path):
-        """
-        Create indicated directory and any necessary parent directories
-        """
-        self._ensure_client()
-
-        cdef c_string c_path = tobytes(path)
-        with nogil:
-            check_status(self.client.get()
-                         .MakeDirectory(c_path))
-
-    def delete(self, path, bint recursive=False):
-        """
-        Delete the indicated file or directory
-
-        Parameters
-        ----------
-        path : string
-        recursive : boolean, default False
-            If True, also delete child paths for directories
-        """
-        self._ensure_client()
-
-        cdef c_string c_path = tobytes(path)
-        with nogil:
-            check_status(self.client.get()
-                         .Delete(c_path, recursive == 1))
-
-    def open(self, path, mode='rb', buffer_size=None, replication=None,
-             default_block_size=None):
-        """
-        Parameters
-        ----------
-        mode : string, 'rb', 'wb', 'ab'
-        """
-        self._ensure_client()
-
-        cdef HdfsFile out = HdfsFile()
-
-        if mode not in ('rb', 'wb', 'ab'):
-            raise Exception("Mode must be 'rb' (read), "
-                            "'wb' (write, new file), or 'ab' (append)")
-
-        cdef c_string c_path = tobytes(path)
-        cdef c_bool append = False
-
-        # 0 in libhdfs means "use the default"
-        cdef int32_t c_buffer_size = buffer_size or 0
-        cdef int16_t c_replication = replication or 0
-        cdef int64_t c_default_block_size = default_block_size or 0
-
-        cdef shared_ptr[HdfsOutputStream] wr_handle
-        cdef shared_ptr[HdfsReadableFile] rd_handle
-
-        if mode in ('wb', 'ab'):
-            if mode == 'ab':
-                append = True
-
-            with nogil:
-                check_status(
-                    self.client.get()
-                    .OpenWriteable(c_path, append, c_buffer_size,
-                                   c_replication, c_default_block_size,
-                                   &wr_handle))
-
-            out.wr_file = <shared_ptr[OutputStream]> wr_handle
-
-            out.is_readable = False
-            out.is_writeable = 1
-        else:
-            with nogil:
-                check_status(self.client.get()
-                             .OpenReadable(c_path, &rd_handle))
-
-            out.rd_file = <shared_ptr[RandomAccessFile]> rd_handle
-            out.is_readable = True
-            out.is_writeable = 0
-
-        if c_buffer_size == 0:
-            c_buffer_size = 2 ** 16
-
-        out.mode = mode
-        out.buffer_size = c_buffer_size
-        out.parent = _HdfsFileNanny(self, out)
-        out.is_open = True
-        out.own_file = True
-
-        return out
-
-    def download(self, path, stream, buffer_size=None):
-        with self.open(path, 'rb') as f:
-            f.download(stream, buffer_size=buffer_size)
-
-    def upload(self, path, stream, buffer_size=None):
-        """
-        Upload file-like object to HDFS path
-        """
-        with self.open(path, 'wb') as f:
-            f.upload(stream, buffer_size=buffer_size)
-
-
-# ARROW-404: Helper class to ensure that files are closed before the
-# client. During deallocation of the extension class, the attributes are
-# decref'd which can cause the client to get closed first if the file has the
-# last remaining reference
-cdef class _HdfsFileNanny:
-    cdef:
-        object client
-        object file_handle_ref
-
-    def __cinit__(self, client, file_handle):
-        import weakref
-        self.client = client
-        self.file_handle_ref = weakref.ref(file_handle)
-
-    def __dealloc__(self):
-        fh = self.file_handle_ref()
-        if fh:
-            fh.close()
-        # avoid cyclic GC
-        self.file_handle_ref = None
-        self.client = None
-
-
-cdef class HdfsFile(NativeFile):
-    cdef readonly:
-        int32_t buffer_size
-        object mode
-        object parent
-
-    cdef object __weakref__
-
-    def __dealloc__(self):
-        self.parent = None
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 599030888178b..4df2fcd64f60f 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -114,6 +114,7 @@ include "table.pxi"
 
 # File IO
 include "io.pxi"
+include "io-hdfs.pxi"
 
 # IPC / Messaging
 include "ipc.pxi"
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index c870412138d7d..6d39a2354f653 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -22,7 +22,7 @@
 
 import numpy as np
 
-from pyarrow.filesystem import Filesystem, LocalFilesystem
+from pyarrow.filesystem import FileSystem, LocalFileSystem
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
                               RowGroupMetaData, ParquetSchema,
                               ParquetWriter)
@@ -403,7 +403,7 @@ class ParquetManifest(object):
     """
     def __init__(self, dirpath, filesystem=None, pathsep='/',
                  partition_scheme='hive'):
-        self.filesystem = filesystem or LocalFilesystem.get_instance()
+        self.filesystem = filesystem or LocalFileSystem.get_instance()
         self.pathsep = pathsep
         self.dirpath = dirpath
         self.partition_scheme = partition_scheme
@@ -506,7 +506,7 @@ class ParquetDataset(object):
     ----------
     path_or_paths : str or List[str]
         A directory name, single file name, or list of file names
-    filesystem : Filesystem, default None
+    filesystem : FileSystem, default None
         If nothing passed, paths assumed to be found in the local on-disk
         filesystem
     metadata : pyarrow.parquet.FileMetaData
@@ -522,7 +522,7 @@ class ParquetDataset(object):
     def __init__(self, path_or_paths, filesystem=None, schema=None,
                  metadata=None, split_row_groups=False, validate_schema=True):
         if filesystem is None:
-            self.fs = LocalFilesystem.get_instance()
+            self.fs = LocalFileSystem.get_instance()
         else:
             self.fs = _ensure_filesystem(filesystem)
 
@@ -631,7 +631,7 @@ def _get_common_pandas_metadata(self):
         return keyvalues.get(b'pandas', None)
 
     def _get_open_file_func(self):
-        if self.fs is None or isinstance(self.fs, LocalFilesystem):
+        if self.fs is None or isinstance(self.fs, LocalFileSystem):
             def open_file(path, meta=None):
                 return ParquetFile(path, metadata=meta,
                                    common_metadata=self.common_metadata)
@@ -644,7 +644,7 @@ def open_file(path, meta=None):
 
 
 def _ensure_filesystem(fs):
-    if not isinstance(fs, Filesystem):
+    if not isinstance(fs, FileSystem):
         if type(fs).__name__ == 'S3FileSystem':
             from pyarrow.filesystem import S3FSWrapper
             return S3FSWrapper(fs)
@@ -716,7 +716,7 @@ def read_table(source, columns=None, nthreads=1, metadata=None,
         Content of the file as a table (of columns)
     """
     if is_string(source):
-        fs = LocalFilesystem.get_instance()
+        fs = LocalFileSystem.get_instance()
         if fs.isdir(source):
             return fs.read_parquet(source, columns=columns,
                                    metadata=metadata)
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 10264080d6e2c..79638f2c64d43 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -72,7 +72,26 @@ def tearDownClass(cls):
         cls.hdfs.delete(cls.tmp_path, recursive=True)
         cls.hdfs.close()
 
-    def test_hdfs_close(self):
+    def test_cat(self):
+        path = pjoin(self.tmp_path, 'cat-test')
+
+        data = b'foobarbaz'
+        with self.hdfs.open(path, 'wb') as f:
+            f.write(data)
+
+        contents = self.hdfs.cat(path)
+        assert contents == data
+
+    def test_capacity_space(self):
+        capacity = self.hdfs.get_capacity()
+        space_used = self.hdfs.get_space_used()
+        disk_free = self.hdfs.df()
+
+        assert capacity > 0
+        assert capacity > space_used
+        assert disk_free == (capacity - space_used)
+
+    def test_close(self):
         client = hdfs_test_client()
         assert client.is_open
         client.close()
@@ -81,7 +100,7 @@ def test_hdfs_close(self):
         with pytest.raises(Exception):
             client.ls('/')
 
-    def test_hdfs_mkdir(self):
+    def test_mkdir(self):
         path = pjoin(self.tmp_path, 'test-dir/test-dir')
         parent_path = pjoin(self.tmp_path, 'test-dir')
 
@@ -91,7 +110,64 @@ def test_hdfs_mkdir(self):
         self.hdfs.delete(parent_path, recursive=True)
         assert not self.hdfs.exists(path)
 
-    def test_hdfs_ls(self):
+    def test_mv_rename(self):
+        path = pjoin(self.tmp_path, 'mv-test')
+        new_path = pjoin(self.tmp_path, 'mv-new-test')
+
+        data = b'foobarbaz'
+        with self.hdfs.open(path, 'wb') as f:
+            f.write(data)
+
+        assert self.hdfs.exists(path)
+        self.hdfs.mv(path, new_path)
+        assert not self.hdfs.exists(path)
+        assert self.hdfs.exists(new_path)
+
+        assert self.hdfs.cat(new_path) == data
+
+        self.hdfs.rename(new_path, path)
+        assert self.hdfs.cat(path) == data
+
+    def test_info(self):
+        path = pjoin(self.tmp_path, 'info-base')
+        file_path = pjoin(path, 'ex')
+        self.hdfs.mkdir(path)
+
+        data = b'foobarbaz'
+        with self.hdfs.open(file_path, 'wb') as f:
+            f.write(data)
+
+        path_info = self.hdfs.info(path)
+        file_path_info = self.hdfs.info(file_path)
+
+        assert path_info['kind'] == 'directory'
+
+        assert file_path_info['kind'] == 'file'
+        assert file_path_info['size'] == len(data)
+
+    def test_disk_usage(self):
+        path = pjoin(self.tmp_path, 'disk-usage-base')
+        p1 = pjoin(path, 'p1')
+        p2 = pjoin(path, 'p2')
+
+        subdir = pjoin(path, 'subdir')
+        p3 = pjoin(subdir, 'p3')
+
+        if self.hdfs.exists(path):
+            self.hdfs.delete(path, True)
+
+        self.hdfs.mkdir(path)
+        self.hdfs.mkdir(subdir)
+
+        data = b'foobarbaz'
+
+        for file_path in [p1, p2, p3]:
+            with self.hdfs.open(file_path, 'wb') as f:
+                f.write(data)
+
+        assert self.hdfs.disk_usage(path) == len(data) * 3
+
+    def test_ls(self):
         base_path = pjoin(self.tmp_path, 'ls-test')
         self.hdfs.mkdir(base_path)
 
@@ -106,7 +182,12 @@ def test_hdfs_ls(self):
         contents = sorted(self.hdfs.ls(base_path, False))
         assert contents == [dir_path, f1_path]
 
-    def test_hdfs_download_upload(self):
+    def test_chmod_chown(self):
+        path = pjoin(self.tmp_path, 'chmod-test')
+        with self.hdfs.open(path, 'wb') as f:
+            f.write(b'a' * 10)
+
+    def test_download_upload(self):
         base_path = pjoin(self.tmp_path, 'upload-test')
 
         data = b'foobarbaz'
@@ -120,7 +201,7 @@ def test_hdfs_download_upload(self):
         out_buf.seek(0)
         assert out_buf.getvalue() == data
 
-    def test_hdfs_file_context_manager(self):
+    def test_file_context_manager(self):
         path = pjoin(self.tmp_path, 'ctx-manager')
 
         data = b'foo'
@@ -132,7 +213,7 @@ def test_hdfs_file_context_manager(self):
             result = f.read(10)
             assert result == data
 
-    def test_hdfs_read_whole_file(self):
+    def test_read_whole_file(self):
         path = pjoin(self.tmp_path, 'read-whole-file')
 
         data = b'foo' * 1000
@@ -145,7 +226,7 @@ def test_hdfs_read_whole_file(self):
         assert result == data
 
     @test_parquet.parquet
-    def test_hdfs_read_multiple_parquet_files(self):
+    def test_read_multiple_parquet_files(self):
         import pyarrow.parquet as pq
 
         nfiles = 10
@@ -191,7 +272,7 @@ def check_driver(cls):
         if not pa.have_libhdfs():
             pytest.fail('No libhdfs available on system')
 
-    def test_hdfs_orphaned_file(self):
+    def test_orphaned_file(self):
         hdfs = hdfs_test_client()
         file_path = self._make_test_file(hdfs, 'orphaned_file_test', 'fname',
                                          'foobarbaz')
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 06265ca95cec6..ab3b26cd4e0f1 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -23,7 +23,7 @@
 import pytest
 
 from pyarrow.compat import guid, u
-from pyarrow.filesystem import LocalFilesystem
+from pyarrow.filesystem import LocalFileSystem
 import pyarrow as pa
 from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
@@ -700,7 +700,7 @@ def test_partition_set_dictionary_type():
 
 @parquet
 def test_read_partitioned_directory(tmpdir):
-    fs = LocalFilesystem.get_instance()
+    fs = LocalFileSystem.get_instance()
     base_path = str(tmpdir)
 
     _partition_test_for_filesystem(fs, base_path)

From b8754eba4683e7300a751f60a2fc1eef152cea1d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 1 Aug 2017 15:52:43 +0200
Subject: [PATCH 0885/1644] ARROW-884: [C++] Exclude internal namespaces from
 generated Doxygen docs

This includes a fair bit of namespace scrubbing. Still lots more to do

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #918 from wesm/ARROW-884 and squashes the following commits:

7606dc5 [Wes McKinney] Typo and cpplint fixes
451eeb1 [Wes McKinney] Restore arrow::TypePtr define
6b7e632 [Wes McKinney] Fix function capitalization
97433bb [Wes McKinney] Exclude internal namespaces from generated Doxygen docs. Various cleanups of current API page to exclude internal details
---
 cpp/apidoc/Doxyfile                      |  53 +++--------
 cpp/build-support/run_clang_format.py    |   8 +-
 cpp/src/arrow/api.h                      |   3 +
 cpp/src/arrow/array-test.cc              |   8 +-
 cpp/src/arrow/array.cc                   |   6 +-
 cpp/src/arrow/builder.cc                 | 102 ++++++++++----------
 cpp/src/arrow/builder.h                  |  58 ++++++------
 cpp/src/arrow/compare.cc                 | 115 +++++++++++------------
 cpp/src/arrow/io/hdfs-internal.cc        |   2 +
 cpp/src/arrow/io/hdfs-internal.h         |   2 +
 cpp/src/arrow/io/hdfs.cc                 |  14 +--
 cpp/src/arrow/io/io-hdfs-test.cc         |   2 +-
 cpp/src/arrow/io/memory.cc               |   4 +-
 cpp/src/arrow/ipc/feather-internal.h     |   4 +-
 cpp/src/arrow/ipc/feather.h              |   4 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |   2 +-
 cpp/src/arrow/ipc/test-common.h          |   2 +-
 cpp/src/arrow/python/arrow_to_pandas.cc  |  19 ++--
 cpp/src/arrow/python/pandas_to_arrow.cc  |  26 +++--
 cpp/src/arrow/python/type_traits.h       |   4 +-
 cpp/src/arrow/table.cc                   |   5 +-
 cpp/src/arrow/test-common.h              |   4 +-
 cpp/src/arrow/type.cc                    |   5 +-
 cpp/src/arrow/type.h                     |  44 +++++----
 cpp/src/arrow/type_traits.h              |  14 ++-
 cpp/src/arrow/util/bit-stream-utils.h    |  21 +++--
 cpp/src/arrow/util/bit-util.h            |   8 +-
 cpp/src/arrow/util/bpacking.h            |  13 ++-
 cpp/src/arrow/util/memory.h              |   2 +
 cpp/src/arrow/util/random.h              |  23 ++---
 cpp/src/arrow/util/stl-util-test.cc      |   2 +
 cpp/src/arrow/util/stl.h                 |   2 +
 cpp/src/arrow/visitor.cc                 |   4 +
 cpp/src/arrow/visitor_inline.h           |   2 +
 34 files changed, 314 insertions(+), 273 deletions(-)

diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile
index 3127662413328..f32ad5425da35 100644
--- a/cpp/apidoc/Doxyfile
+++ b/cpp/apidoc/Doxyfile
@@ -833,50 +833,17 @@ INPUT_ENCODING         = UTF-8
 # *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08,
 # *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf.
 
-FILE_PATTERNS          = *.c \
-                         *.cc \
-                         *.cxx \
-                         *.cpp \
-                         *.c++ \
-                         *.java \
-                         *.ii \
-                         *.ixx \
-                         *.ipp \
-                         *.i++ \
-                         *.inl \
-                         *.idl \
-                         *.ddl \
-                         *.odl \
-                         *.h \
+FILE_PATTERNS          = *.h \
                          *.hh \
                          *.hxx \
                          *.hpp \
-                         *.h++ \
-                         *.cs \
-                         *.d \
-                         *.php \
-                         *.php4 \
-                         *.php5 \
-                         *.phtml \
                          *.inc \
                          *.m \
                          *.markdown \
                          *.md \
                          *.mm \
                          *.dox \
-                         *.py \
-                         *.pyw \
-                         *.f90 \
-                         *.f95 \
-                         *.f03 \
-                         *.f08 \
-                         *.f \
-                         *.for \
-                         *.tcl \
-                         *.vhd \
-                         *.vhdl \
-                         *.ucf \
-                         *.qsf
+                         *.py
 
 # The RECURSIVE tag can be used to specify whether or not subdirectories should
 # be searched for input files as well.
@@ -908,6 +875,7 @@ EXCLUDE_SYMLINKS       = NO
 # exclude all test directories for example use the pattern */test/*
 
 EXCLUDE_PATTERNS       = *-test.cc \
+                         *test* \
                          *_generated.h \
                          *-benchmark.cc
 
@@ -920,7 +888,11 @@ EXCLUDE_PATTERNS       = *-test.cc \
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        =
+EXCLUDE_SYMBOLS = detail
+EXCLUDE_SYMBOLS += internal
+EXCLUDE_SYMBOLS += _*
+EXCLUDE_SYMBOLS += BitUtil
+EXCLUDE_SYMBOLS += SSEUtil
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include
@@ -2060,7 +2032,7 @@ ENABLE_PREPROCESSING   = YES
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-MACRO_EXPANSION        = NO
+MACRO_EXPANSION        = YES
 
 # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
 # the macro expansion is limited to the macros specified with the PREDEFINED and
@@ -2068,7 +2040,7 @@ MACRO_EXPANSION        = NO
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_ONLY_PREDEF     = NO
+EXPAND_ONLY_PREDEF     = YES
 
 # If the SEARCH_INCLUDES tag is set to YES, the include files in the
 # INCLUDE_PATH will be searched if a #include is found.
@@ -2100,7 +2072,10 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             =
+PREDEFINED = __attribute__(x)= \
+             __declspec(x)= \
+             ARROW_EXPORT= \
+             ARROW_EXTERN_TEMPLATE=
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py
index ab800e641b5d2..ac4954ca5708f 100755
--- a/cpp/build-support/run_clang_format.py
+++ b/cpp/build-support/run_clang_format.py
@@ -57,5 +57,9 @@
 #   exit 1
 # fi
 
-subprocess.check_output([CLANG_FORMAT, '-i'] + files_to_format,
-                        stderr=subprocess.STDOUT)
+try:
+    subprocess.check_output([CLANG_FORMAT, '-i'] + files_to_format,
+                            stderr=subprocess.STDOUT)
+except Exception as e:
+    print(e)
+    raise
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 731f23918e4a4..4d731bd32bf1b 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -32,4 +32,7 @@
 #include "arrow/type.h"
 #include "arrow/visitor.h"
 
+/// \brief Top-level namespace for Apache Arrow C++ API
+namespace arrow {}
+
 #endif  // ARROW_API_H
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 5d63d921cdd52..0efb51ccece0c 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -2042,9 +2042,9 @@ class TestStructBuilder : public TestBuilder {
     auto list_type = list(char_type);
 
     vector<std::shared_ptr<DataType>> types = {list_type, int32_type};
-    vector<FieldPtr> fields;
-    fields.push_back(FieldPtr(new Field("list", list_type)));
-    fields.push_back(FieldPtr(new Field("int", int32_type)));
+    vector<std::shared_ptr<Field>> fields;
+    fields.push_back(field("list", list_type));
+    fields.push_back(field("int", int32_type));
 
     type_ = struct_(fields);
     value_fields_ = fields;
@@ -2062,7 +2062,7 @@ class TestStructBuilder : public TestBuilder {
   }
 
  protected:
-  vector<FieldPtr> value_fields_;
+  vector<std::shared_ptr<Field>> value_fields_;
   std::shared_ptr<DataType> type_;
 
   std::shared_ptr<StructBuilder> builder_;
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 61791c9457756..ab0be7a0964c6 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -527,6 +527,8 @@ Status Array::Accept(ArrayVisitor* visitor) const {
 // ----------------------------------------------------------------------
 // Implement Array::Validate as inline visitor
 
+namespace internal {
+
 struct ValidateVisitor {
   Status Visit(const NullArray& array) { return Status::OK(); }
 
@@ -658,8 +660,10 @@ struct ValidateVisitor {
   }
 };
 
+}  // namespace internal
+
 Status ValidateArray(const Array& array) {
-  ValidateVisitor validate_visitor;
+  internal::ValidateVisitor validate_visitor;
   return VisitArrayInline(array, &validate_visitor);
 }
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index d3a299e5412fc..391204f566954 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -37,6 +37,10 @@
 
 namespace arrow {
 
+using internal::AdaptiveIntBuilderBase;
+using internal::ArrayData;
+using internal::WrappedBinary;
+
 Status ArrayBuilder::AppendToBitmap(bool is_valid) {
   if (length_ == capacity_) {
     // If the capacity was not already a multiple of 2, do so here
@@ -338,7 +342,7 @@ Status AdaptiveIntBuilder::Append(const int64_t* values, int64_t length,
       uint8_t new_int_size = int_size_;
       for (int64_t i = 0; i < length; i++) {
         if (valid_bytes == nullptr || valid_bytes[i]) {
-          new_int_size = expanded_int_size(values[i], new_int_size);
+          new_int_size = internal::ExpandedIntSize(values[i], new_int_size);
         }
       }
       if (new_int_size != int_size_) {
@@ -495,7 +499,7 @@ Status AdaptiveUIntBuilder::Append(const uint64_t* values, int64_t length,
       uint8_t new_int_size = int_size_;
       for (int64_t i = 0; i < length; i++) {
         if (valid_bytes == nullptr || valid_bytes[i]) {
-          new_int_size = expanded_uint_size(values[i], new_int_size);
+          new_int_size = internal::ExpandedUIntSize(values[i], new_int_size);
         }
       }
       if (new_int_size != int_size_) {
@@ -861,48 +865,47 @@ Status DictionaryBuilder<T>::AppendDictionary(const Scalar& value) {
   return dict_builder_.Append(value);
 }
 
-#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                        \
-  template <>                                                                          \
-  internal::WrappedBinary DictionaryBuilder<Type>::GetDictionaryValue(int64_t index) { \
-    int32_t v_len;                                                                     \
-    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len);    \
-    return internal::WrappedBinary(v, v_len);                                          \
-  }                                                                                    \
-                                                                                       \
-  template <>                                                                          \
-  Status DictionaryBuilder<Type>::AppendDictionary(                                    \
-      const internal::WrappedBinary& value) {                                          \
-    return dict_builder_.Append(value.ptr_, value.length_);                            \
-  }                                                                                    \
-                                                                                       \
-  template <>                                                                          \
-  Status DictionaryBuilder<Type>::AppendArray(const Array& array) {                    \
-    const BinaryArray& binary_array = static_cast<const BinaryArray&>(array);          \
-    internal::WrappedBinary value(nullptr, 0);                                         \
-    for (int64_t i = 0; i < array.length(); i++) {                                     \
-      if (array.IsNull(i)) {                                                           \
-        RETURN_NOT_OK(AppendNull());                                                   \
-      } else {                                                                         \
-        value.ptr_ = binary_array.GetValue(i, &value.length_);                         \
-        RETURN_NOT_OK(Append(value));                                                  \
-      }                                                                                \
-    }                                                                                  \
-    return Status::OK();                                                               \
-  }                                                                                    \
-                                                                                       \
-  template <>                                                                          \
-  int DictionaryBuilder<Type>::HashValue(const internal::WrappedBinary& value) {       \
-    return HashUtil::Hash(value.ptr_, value.length_, 0);                               \
-  }                                                                                    \
-                                                                                       \
-  template <>                                                                          \
-  bool DictionaryBuilder<Type>::SlotDifferent(hash_slot_t index,                       \
-                                              const internal::WrappedBinary& value) {  \
-    int32_t other_length;                                                              \
-    const uint8_t* other_value =                                                       \
-        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);            \
-    return !(other_length == value.length_ &&                                          \
-             0 == memcmp(other_value, value.ptr_, value.length_));                     \
+#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                     \
+  template <>                                                                       \
+  WrappedBinary DictionaryBuilder<Type>::GetDictionaryValue(int64_t index) {        \
+    int32_t v_len;                                                                  \
+    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len); \
+    return WrappedBinary(v, v_len);                                                 \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  Status DictionaryBuilder<Type>::AppendDictionary(const WrappedBinary& value) {    \
+    return dict_builder_.Append(value.ptr_, value.length_);                         \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  Status DictionaryBuilder<Type>::AppendArray(const Array& array) {                 \
+    const BinaryArray& binary_array = static_cast<const BinaryArray&>(array);       \
+    WrappedBinary value(nullptr, 0);                                                \
+    for (int64_t i = 0; i < array.length(); i++) {                                  \
+      if (array.IsNull(i)) {                                                        \
+        RETURN_NOT_OK(AppendNull());                                                \
+      } else {                                                                      \
+        value.ptr_ = binary_array.GetValue(i, &value.length_);                      \
+        RETURN_NOT_OK(Append(value));                                               \
+      }                                                                             \
+    }                                                                               \
+    return Status::OK();                                                            \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  int DictionaryBuilder<Type>::HashValue(const WrappedBinary& value) {              \
+    return HashUtil::Hash(value.ptr_, value.length_, 0);                            \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  bool DictionaryBuilder<Type>::SlotDifferent(hash_slot_t index,                    \
+                                              const WrappedBinary& value) {         \
+    int32_t other_length;                                                           \
+    const uint8_t* other_value =                                                    \
+        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);         \
+    return !(other_length == value.length_ &&                                       \
+             0 == memcmp(other_value, value.ptr_, value.length_));                  \
   }
 
 BINARY_DICTIONARY_SPECIALIZATIONS(StringType);
@@ -1132,7 +1135,7 @@ Status BinaryBuilder::AppendNull() {
   return Status::OK();
 }
 
-Status BinaryBuilder::FinishInternal(std::shared_ptr<internal::ArrayData>* out) {
+Status BinaryBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   // Write final offset (values length)
   RETURN_NOT_OK(AppendNextOffset());
   std::shared_ptr<Buffer> offsets, value_data;
@@ -1141,13 +1144,12 @@ Status BinaryBuilder::FinishInternal(std::shared_ptr<internal::ArrayData>* out)
   RETURN_NOT_OK(value_data_builder_.Finish(&value_data));
 
   BufferVector buffers = {null_bitmap_, offsets, value_data};
-  *out = std::make_shared<internal::ArrayData>(type_, length_, std::move(buffers),
-                                               null_count_, 0);
+  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_, 0);
   return Status::OK();
 }
 
 Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<internal::ArrayData> data;
+  std::shared_ptr<ArrayData> data;
   RETURN_NOT_OK(FinishInternal(&data));
   *out = std::make_shared<BinaryArray>(data);
   Reset();
@@ -1174,7 +1176,7 @@ const uint8_t* BinaryBuilder::GetValue(int64_t i, int32_t* out_length) const {
 StringBuilder::StringBuilder(MemoryPool* pool) : BinaryBuilder(pool, utf8()) {}
 
 Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<internal::ArrayData> data;
+  std::shared_ptr<ArrayData> data;
   RETURN_NOT_OK(FinishInternal(&data));
   *out = std::make_shared<StringArray>(data);
   Reset();
@@ -1299,7 +1301,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     }
 
     case Type::STRUCT: {
-      const std::vector<FieldPtr>& fields = type->children();
+      const std::vector<std::shared_ptr<Field>>& fields = type->children();
       std::vector<std::unique_ptr<ArrayBuilder>> values_builder;
 
       for (auto it : fields) {
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 080a32900555c..009fd7ae47d19 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -262,6 +262,8 @@ using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
 using FloatBuilder = NumericBuilder<FloatType>;
 using DoubleBuilder = NumericBuilder<DoubleType>;
 
+namespace internal {
+
 class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
  public:
   explicit AdaptiveIntBuilderBase(MemoryPool* pool);
@@ -295,7 +297,29 @@ class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
 };
 
 // Check if we would need to expand the underlying storage type
-inline uint8_t expanded_uint_size(uint64_t val, uint8_t current_int_size) {
+inline uint8_t ExpandedIntSize(int64_t val, uint8_t current_int_size) {
+  if (current_int_size == 8 ||
+      (current_int_size < 8 &&
+       (val > static_cast<int64_t>(std::numeric_limits<int32_t>::max()) ||
+        val < static_cast<int64_t>(std::numeric_limits<int32_t>::min())))) {
+    return 8;
+  } else if (current_int_size == 4 ||
+             (current_int_size < 4 &&
+              (val > static_cast<int64_t>(std::numeric_limits<int16_t>::max()) ||
+               val < static_cast<int64_t>(std::numeric_limits<int16_t>::min())))) {
+    return 4;
+  } else if (current_int_size == 2 ||
+             (current_int_size == 1 &&
+              (val > static_cast<int64_t>(std::numeric_limits<int8_t>::max()) ||
+               val < static_cast<int64_t>(std::numeric_limits<int8_t>::min())))) {
+    return 2;
+  } else {
+    return 1;
+  }
+}
+
+// Check if we would need to expand the underlying storage type
+inline uint8_t ExpandedUIntSize(uint64_t val, uint8_t current_int_size) {
   if (current_int_size == 8 ||
       (current_int_size < 8 &&
        (val > static_cast<uint64_t>(std::numeric_limits<uint32_t>::max())))) {
@@ -313,7 +337,9 @@ inline uint8_t expanded_uint_size(uint64_t val, uint8_t current_int_size) {
   }
 }
 
-class ARROW_EXPORT AdaptiveUIntBuilder : public AdaptiveIntBuilderBase {
+}  // namespace internal
+
+class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
   explicit AdaptiveUIntBuilder(MemoryPool* pool);
 
@@ -324,7 +350,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public AdaptiveIntBuilderBase {
     RETURN_NOT_OK(Reserve(1));
     BitUtil::SetBit(null_bitmap_data_, length_);
 
-    uint8_t new_int_size = expanded_uint_size(val, int_size_);
+    uint8_t new_int_size = internal::ExpandedUIntSize(val, int_size_);
     if (new_int_size != int_size_) {
       RETURN_NOT_OK(ExpandIntSize(new_int_size));
     }
@@ -372,29 +398,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public AdaptiveIntBuilderBase {
   Status ExpandIntSizeN();
 };
 
-// Check if we would need to expand the underlying storage type
-inline uint8_t expanded_int_size(int64_t val, uint8_t current_int_size) {
-  if (current_int_size == 8 ||
-      (current_int_size < 8 &&
-       (val > static_cast<int64_t>(std::numeric_limits<int32_t>::max()) ||
-        val < static_cast<int64_t>(std::numeric_limits<int32_t>::min())))) {
-    return 8;
-  } else if (current_int_size == 4 ||
-             (current_int_size < 4 &&
-              (val > static_cast<int64_t>(std::numeric_limits<int16_t>::max()) ||
-               val < static_cast<int64_t>(std::numeric_limits<int16_t>::min())))) {
-    return 4;
-  } else if (current_int_size == 2 ||
-             (current_int_size == 1 &&
-              (val > static_cast<int64_t>(std::numeric_limits<int8_t>::max()) ||
-               val < static_cast<int64_t>(std::numeric_limits<int8_t>::min())))) {
-    return 2;
-  } else {
-    return 1;
-  }
-}
-
-class ARROW_EXPORT AdaptiveIntBuilder : public AdaptiveIntBuilderBase {
+class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
   explicit AdaptiveIntBuilder(MemoryPool* pool);
 
@@ -405,7 +409,7 @@ class ARROW_EXPORT AdaptiveIntBuilder : public AdaptiveIntBuilderBase {
     RETURN_NOT_OK(Reserve(1));
     BitUtil::SetBit(null_bitmap_data_, length_);
 
-    uint8_t new_int_size = expanded_int_size(val, int_size_);
+    uint8_t new_int_size = internal::ExpandedIntSize(val, int_size_);
     if (new_int_size != int_size_) {
       RETURN_NOT_OK(ExpandIntSize(new_int_size));
     }
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index da10c2ad90177..dda5fdd95d0c3 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -38,6 +38,8 @@ namespace arrow {
 // ----------------------------------------------------------------------
 // Public method implementations
 
+namespace internal {
+
 class RangeEqualsVisitor {
  public:
   RangeEqualsVisitor(const Array& right, int64_t left_start_idx, int64_t left_end_idx,
@@ -673,63 +675,6 @@ inline Status ArrayEqualsImpl(const Array& left, const Array& right, bool* are_e
   return Status::OK();
 }
 
-Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
-  return ArrayEqualsImpl<ArrayEqualsVisitor>(left, right, are_equal);
-}
-
-Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal) {
-  return ArrayEqualsImpl<ApproxEqualsVisitor>(left, right, are_equal);
-}
-
-Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_start_idx,
-                        int64_t left_end_idx, int64_t right_start_idx, bool* are_equal) {
-  if (&left == &right) {
-    *are_equal = true;
-  } else if (left.type_id() != right.type_id()) {
-    *are_equal = false;
-  } else if (left.length() == 0) {
-    *are_equal = true;
-  } else {
-    RangeEqualsVisitor visitor(right, left_start_idx, left_end_idx, right_start_idx);
-    RETURN_NOT_OK(VisitArrayInline(left, &visitor));
-    *are_equal = visitor.result();
-  }
-  return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// Implement TensorEquals
-
-Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
-  // The arrays are the same object
-  if (&left == &right) {
-    *are_equal = true;
-  } else if (left.type_id() != right.type_id()) {
-    *are_equal = false;
-  } else if (left.size() == 0) {
-    *are_equal = true;
-  } else {
-    if (!left.is_contiguous() || !right.is_contiguous()) {
-      return Status::NotImplemented(
-          "Comparison not implemented for non-contiguous tensors");
-    }
-
-    const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
-    const int byte_width = size_meta.bit_width() / 8;
-    DCHECK_GT(byte_width, 0);
-
-    const uint8_t* left_data = left.data()->data();
-    const uint8_t* right_data = right.data()->data();
-
-    *are_equal =
-        memcmp(left_data, right_data, static_cast<size_t>(byte_width * left.size())) == 0;
-  }
-  return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// Implement TypeEquals
-
 class TypeEqualsVisitor {
  public:
   explicit TypeEqualsVisitor(const DataType& right) : right_(right), result_(false) {}
@@ -835,6 +780,60 @@ class TypeEqualsVisitor {
   bool result_;
 };
 
+}  // namespace internal
+
+Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
+  return internal::ArrayEqualsImpl<internal::ArrayEqualsVisitor>(left, right, are_equal);
+}
+
+Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal) {
+  return internal::ArrayEqualsImpl<internal::ApproxEqualsVisitor>(left, right, are_equal);
+}
+
+Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_start_idx,
+                        int64_t left_end_idx, int64_t right_start_idx, bool* are_equal) {
+  if (&left == &right) {
+    *are_equal = true;
+  } else if (left.type_id() != right.type_id()) {
+    *are_equal = false;
+  } else if (left.length() == 0) {
+    *are_equal = true;
+  } else {
+    internal::RangeEqualsVisitor visitor(right, left_start_idx, left_end_idx,
+                                         right_start_idx);
+    RETURN_NOT_OK(VisitArrayInline(left, &visitor));
+    *are_equal = visitor.result();
+  }
+  return Status::OK();
+}
+
+Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
+  // The arrays are the same object
+  if (&left == &right) {
+    *are_equal = true;
+  } else if (left.type_id() != right.type_id()) {
+    *are_equal = false;
+  } else if (left.size() == 0) {
+    *are_equal = true;
+  } else {
+    if (!left.is_contiguous() || !right.is_contiguous()) {
+      return Status::NotImplemented(
+          "Comparison not implemented for non-contiguous tensors");
+    }
+
+    const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
+    const int byte_width = size_meta.bit_width() / 8;
+    DCHECK_GT(byte_width, 0);
+
+    const uint8_t* left_data = left.data()->data();
+    const uint8_t* right_data = right.data()->data();
+
+    *are_equal =
+        memcmp(left_data, right_data, static_cast<size_t>(byte_width * left.size())) == 0;
+  }
+  return Status::OK();
+}
+
 Status TypeEquals(const DataType& left, const DataType& right, bool* are_equal) {
   // The arrays are the same object
   if (&left == &right) {
@@ -842,7 +841,7 @@ Status TypeEquals(const DataType& left, const DataType& right, bool* are_equal)
   } else if (left.id() != right.id()) {
     *are_equal = false;
   } else {
-    TypeEqualsVisitor visitor(right);
+    internal::TypeEqualsVisitor visitor(right);
     RETURN_NOT_OK(VisitTypeInline(left, &visitor));
     *are_equal = visitor.result();
   }
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index fd7417b9fba48..35657df4620f5 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -282,6 +282,7 @@ static inline void* GetLibrarySymbol(void* handle, const char* symbol) {
 
 namespace arrow {
 namespace io {
+namespace internal {
 
 static LibHdfsShim libhdfs_shim;
 static LibHdfsShim libhdfs3_shim;
@@ -556,5 +557,6 @@ Status ConnectLibHdfs3(LibHdfsShim** driver) {
   return shim->GetRequiredSymbols();
 }
 
+}  // namespace internal
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
index db6a21c2b36ac..f2de00de8b934 100644
--- a/cpp/src/arrow/io/hdfs-internal.h
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -32,6 +32,7 @@ namespace arrow {
 class Status;
 
 namespace io {
+namespace internal {
 
 // NOTE(wesm): cpplint does not like use of short and other imprecise C types
 struct LibHdfsShim {
@@ -205,6 +206,7 @@ struct LibHdfsShim {
 Status ARROW_EXPORT ConnectLibHdfs(LibHdfsShim** driver);
 Status ARROW_EXPORT ConnectLibHdfs3(LibHdfsShim** driver);
 
+}  // namespace internal
 }  // namespace io
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 254e483bb97e8..ba446b56e00bb 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -61,7 +61,7 @@ static constexpr int kDefaultHdfsBufferSize = 1 << 16;
 
 class HdfsAnyFileImpl {
  public:
-  void set_members(const std::string& path, LibHdfsShim* driver, hdfsFS fs,
+  void set_members(const std::string& path, internal::LibHdfsShim* driver, hdfsFS fs,
                    hdfsFile handle) {
     path_ = path;
     driver_ = driver;
@@ -88,7 +88,7 @@ class HdfsAnyFileImpl {
  protected:
   std::string path_;
 
-  LibHdfsShim* driver_;
+  internal::LibHdfsShim* driver_;
 
   // For threadsafety
   std::mutex lock_;
@@ -507,7 +507,7 @@ class HadoopFileSystem::HadoopFileSystemImpl {
   }
 
  private:
-  LibHdfsShim* driver_;
+  internal::LibHdfsShim* driver_;
 
   std::string namenode_host_;
   std::string user_;
@@ -613,13 +613,13 @@ Status HadoopFileSystem::Rename(const std::string& src, const std::string& dst)
 // Allow public API users to check whether we are set up correctly
 
 Status HaveLibHdfs() {
-  LibHdfsShim* driver;
-  return ConnectLibHdfs(&driver);
+  internal::LibHdfsShim* driver;
+  return internal::ConnectLibHdfs(&driver);
 }
 
 Status HaveLibHdfs3() {
-  LibHdfsShim* driver;
-  return ConnectLibHdfs3(&driver);
+  internal::LibHdfsShim* driver;
+  return internal::ConnectLibHdfs3(&driver);
 }
 
 }  // namespace io
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 92f4291346294..b6a40e094c9cf 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -84,7 +84,7 @@ class TestHadoopFileSystem : public ::testing::Test {
 
   // Set up shared state between unit tests
   void SetUp() {
-    LibHdfsShim* driver_shim;
+    internal::LibHdfsShim* driver_shim;
 
     client_ = nullptr;
     scratch_dir_ =
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index b6c48ec39be89..50f3ddfaf6512 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -157,8 +157,8 @@ Status FixedSizeBufferWriter::Tell(int64_t* position) {
 
 Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
   if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) {
-    parallel_memcopy(mutable_data_ + position_, data, nbytes, memcopy_blocksize_,
-                     memcopy_num_threads_);
+    internal::parallel_memcopy(mutable_data_ + position_, data, nbytes,
+                               memcopy_blocksize_, memcopy_num_threads_);
   } else {
     memcpy(mutable_data_ + position_, data, nbytes);
   }
diff --git a/cpp/src/arrow/ipc/feather-internal.h b/cpp/src/arrow/ipc/feather-internal.h
index 36cfecc0493f5..1b5924e3030ab 100644
--- a/cpp/src/arrow/ipc/feather-internal.h
+++ b/cpp/src/arrow/ipc/feather-internal.h
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-/// Public API for the "Feather" file format, originally created at
-/// http://github.com/wesm/feather
+// Public API for the "Feather" file format, originally created at
+// http://github.com/wesm/feather
 
 #ifndef ARROW_IPC_FEATHER_INTERNAL_H
 #define ARROW_IPC_FEATHER_INTERNAL_H
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
index 8abcb5c0f2599..2ab35a9556d73 100644
--- a/cpp/src/arrow/ipc/feather.h
+++ b/cpp/src/arrow/ipc/feather.h
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-/// Public API for the "Feather" file format, originally created at
-/// http://github.com/wesm/feather
+// Public API for the "Feather" file format, originally created at
+// http://github.com/wesm/feather
 
 #ifndef ARROW_IPC_FEATHER_H
 #define ARROW_IPC_FEATHER_H
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 6c7051750b7cb..a6246c96f2d9a 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -409,7 +409,7 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
                      int64_t* body_length, std::shared_ptr<RecordBatch>* batch,
                      std::shared_ptr<Schema>* schema) {
     const int batch_length = 5;
-    TypePtr type = int32();
+    auto type = int32();
     std::shared_ptr<Array> array;
     const bool include_nulls = true;
     RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index a8767926b2a07..cb827372d21c4 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -333,7 +333,7 @@ Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
 
 Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
   const int batch_length = 5;
-  TypePtr type = int32();
+  auto type = int32();
 
   MemoryPool* pool = default_memory_pool();
   std::shared_ptr<Array> array;
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 462bdb7b7d744..86f82fdbd8de5 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -56,6 +56,9 @@
 namespace arrow {
 namespace py {
 
+using internal::kPandasTimestampNull;
+using internal::kNanosecondsInDay;
+
 // ----------------------------------------------------------------------
 // Utility code
 
@@ -752,7 +755,7 @@ class IntBlock : public PandasBlock {
  public:
   using PandasBlock::PandasBlock;
   Status Allocate() override {
-    return AllocateNDArray(arrow_traits<ARROW_TYPE>::npy_type);
+    return AllocateNDArray(internal::arrow_traits<ARROW_TYPE>::npy_type);
   }
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
@@ -981,7 +984,7 @@ class CategoricalBlock : public PandasBlock {
  public:
   explicit CategoricalBlock(int64_t num_rows) : PandasBlock(num_rows, 1) {}
   Status Allocate() override {
-    constexpr int npy_type = arrow_traits<ARROW_INDEX_TYPE>::npy_type;
+    constexpr int npy_type = internal::arrow_traits<ARROW_INDEX_TYPE>::npy_type;
 
     if (!(npy_type == NPY_INT8 || npy_type == NPY_INT16 || npy_type == NPY_INT32 ||
           npy_type == NPY_INT64)) {
@@ -992,7 +995,7 @@ class CategoricalBlock : public PandasBlock {
 
   Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
                int64_t rel_placement) override {
-    using T = typename arrow_traits<ARROW_INDEX_TYPE>::T;
+    using T = typename internal::arrow_traits<ARROW_INDEX_TYPE>::T;
 
     T* out_values = reinterpret_cast<T*>(block_data_) + rel_placement * num_rows_;
 
@@ -1381,7 +1384,7 @@ class ArrowDeserializer {
 
   template <int TYPE>
   Status ConvertValuesZeroCopy(int npy_type, std::shared_ptr<Array> arr) {
-    typedef typename arrow_traits<TYPE>::T T;
+    typedef typename internal::arrow_traits<TYPE>::T T;
 
     auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
     auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
@@ -1425,7 +1428,7 @@ class ArrowDeserializer {
   typename std::enable_if<std::is_base_of<FloatingPoint, Type>::value, Status>::type
   Visit(const Type& type) {
     constexpr int TYPE = Type::type_id;
-    using traits = arrow_traits<TYPE>;
+    using traits = internal::arrow_traits<TYPE>;
 
     typedef typename traits::T T;
     int npy_type = traits::npy_type;
@@ -1447,7 +1450,7 @@ class ArrowDeserializer {
                           Status>::type
   Visit(const Type& type) {
     constexpr int TYPE = Type::type_id;
-    using traits = arrow_traits<TYPE>;
+    using traits = internal::arrow_traits<TYPE>;
 
     typedef typename traits::T T;
 
@@ -1480,7 +1483,7 @@ class ArrowDeserializer {
   typename std::enable_if<std::is_base_of<Integer, Type>::value, Status>::type Visit(
       const Type& type) {
     constexpr int TYPE = Type::type_id;
-    using traits = arrow_traits<TYPE>;
+    using traits = internal::arrow_traits<TYPE>;
 
     typedef typename traits::T T;
 
@@ -1535,7 +1538,7 @@ class ArrowDeserializer {
     if (data_.null_count() > 0) {
       return VisitObjects(ConvertBooleanWithNulls);
     } else {
-      RETURN_NOT_OK(AllocateOutput(arrow_traits<Type::BOOL>::npy_type));
+      RETURN_NOT_OK(AllocateOutput(internal::arrow_traits<Type::BOOL>::npy_type));
       auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(arr_));
       ConvertBooleanNoNulls(data_, out_values);
     }
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index be5634b53bbfe..2fbed1b8fdf08 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -50,8 +50,14 @@
 #include "arrow/python/util/datetime.h"
 
 namespace arrow {
+
+using internal::ArrayData;
+using internal::MakeArray;
+
 namespace py {
 
+using internal::NumPyTypeSize;
+
 // ----------------------------------------------------------------------
 // Conversion utilities
 
@@ -83,7 +89,7 @@ static inline bool PyObject_is_integer(const PyObject* obj) {
 
 template <int TYPE>
 static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
-  typedef npy_traits<TYPE> traits;
+  typedef internal::npy_traits<TYPE> traits;
   typedef typename traits::value_type T;
 
   int64_t null_count = 0;
@@ -120,7 +126,7 @@ static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap
 template <int TYPE>
 static int64_t ValuesToValidBytes(const void* data, int64_t length,
                                   uint8_t* valid_bytes) {
-  typedef npy_traits<TYPE> traits;
+  typedef internal::npy_traits<TYPE> traits;
   typedef typename traits::value_type T;
 
   int64_t null_count = 0;
@@ -306,16 +312,16 @@ class PandasConverter {
     return Status::OK();
   }
 
-  Status PushArray(const std::shared_ptr<internal::ArrayData>& data) {
+  Status PushArray(const std::shared_ptr<ArrayData>& data) {
     std::shared_ptr<Array> result;
-    RETURN_NOT_OK(internal::MakeArray(data, &result));
+    RETURN_NOT_OK(MakeArray(data, &result));
     out_arrays_.emplace_back(std::move(result));
     return Status::OK();
   }
 
   template <typename ArrowType>
   Status VisitNative() {
-    using traits = arrow_traits<ArrowType::type_id>;
+    using traits = internal::arrow_traits<ArrowType::type_id>;
 
     if (mask_ != nullptr || traits::supports_nulls) {
       RETURN_NOT_OK(InitNullBitmap());
@@ -334,8 +340,8 @@ class PandasConverter {
     }
 
     BufferVector buffers = {null_bitmap_, data};
-    return PushArray(std::make_shared<internal::ArrayData>(
-        type_, length_, std::move(buffers), null_count, 0));
+    return PushArray(
+        std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count, 0));
   }
 
   template <typename T>
@@ -448,13 +454,13 @@ void CopyStrided<PyObject*>(PyObject** input_data, int64_t length, int64_t strid
 
 template <typename ArrowType>
 inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
-  using traits = arrow_traits<ArrowType::type_id>;
+  using traits = internal::arrow_traits<ArrowType::type_id>;
   using T = typename traits::T;
 
   // Handle LONGLONG->INT64 and other fun things
   int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
 
-  if (numpy_type_size(traits::npy_type) != numpy_type_size(type_num_compat)) {
+  if (NumPyTypeSize(traits::npy_type) != NumPyTypeSize(type_num_compat)) {
     return Status::NotImplemented("NumPy type casts not yet implemented");
   }
 
@@ -925,7 +931,7 @@ Status LoopPySequence(PyObject* sequence, T func) {
 template <int ITEM_TYPE, typename ArrowType>
 inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>& type,
                                                  ListBuilder* builder, PyObject* list) {
-  typedef npy_traits<ITEM_TYPE> traits;
+  typedef internal::npy_traits<ITEM_TYPE> traits;
   typedef typename traits::value_type T;
   typedef typename traits::BuilderClass BuilderT;
 
diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h
index b6761ae0d2611..2cbbdf4cf1519 100644
--- a/cpp/src/arrow/python/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -30,6 +30,7 @@
 
 namespace arrow {
 namespace py {
+namespace internal {
 
 template <int TYPE>
 struct npy_traits {};
@@ -227,7 +228,7 @@ struct arrow_traits<Type::BINARY> {
   static constexpr bool supports_nulls = true;
 };
 
-static inline int numpy_type_size(int npy_type) {
+static inline int NumPyTypeSize(int npy_type) {
   switch (npy_type) {
     case NPY_BOOL:
       return 1;
@@ -272,5 +273,6 @@ static inline int numpy_type_size(int npy_type) {
   return -1;
 }
 
+}  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index a0a25079e6ed7..665ce2d84dea4 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -387,7 +387,7 @@ Status Table::RemoveColumn(int i, std::shared_ptr<Table>* out) const {
   std::shared_ptr<Schema> new_schema;
   RETURN_NOT_OK(schema_->RemoveField(i, &new_schema));
 
-  *out = std::make_shared<Table>(new_schema, DeleteVectorElement(columns_, i));
+  *out = std::make_shared<Table>(new_schema, internal::DeleteVectorElement(columns_, i));
   return Status::OK();
 }
 
@@ -411,7 +411,8 @@ Status Table::AddColumn(int i, const std::shared_ptr<Column>& col,
   std::shared_ptr<Schema> new_schema;
   RETURN_NOT_OK(schema_->AddField(i, col->field(), &new_schema));
 
-  *out = std::make_shared<Table>(new_schema, AddVectorElement(columns_, i, col));
+  *out =
+      std::make_shared<Table>(new_schema, internal::AddVectorElement(columns_, i, col));
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h
index b3e5af86d4b53..4ce06408d17d0 100644
--- a/cpp/src/arrow/test-common.h
+++ b/cpp/src/arrow/test-common.h
@@ -70,7 +70,7 @@ class TestBuilder : public ::testing::Test {
  public:
   void SetUp() {
     pool_ = default_memory_pool();
-    type_ = TypePtr(new UInt8Type());
+    type_ = uint8();
     builder_.reset(new UInt8Builder(pool_));
     builder_nn_.reset(new UInt8Builder(pool_));
   }
@@ -78,7 +78,7 @@ class TestBuilder : public ::testing::Test {
  protected:
   MemoryPool* pool_;
 
-  TypePtr type_;
+  std::shared_ptr<DataType> type_;
   std::unique_ptr<ArrayBuilder> builder_;
   std::unique_ptr<ArrayBuilder> builder_nn_;
 };
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 586da2d86d909..b8489d44cdb00 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -293,7 +293,8 @@ Status Schema::AddField(int i, const std::shared_ptr<Field>& field,
   DCHECK_GE(i, 0);
   DCHECK_LE(i, this->num_fields());
 
-  *out = std::make_shared<Schema>(AddVectorElement(fields_, i, field), metadata_);
+  *out =
+      std::make_shared<Schema>(internal::AddVectorElement(fields_, i, field), metadata_);
   return Status::OK();
 }
 
@@ -316,7 +317,7 @@ Status Schema::RemoveField(int i, std::shared_ptr<Schema>* out) const {
   DCHECK_GE(i, 0);
   DCHECK_LT(i, this->num_fields());
 
-  *out = std::make_shared<Schema>(DeleteVectorElement(fields_, i), metadata_);
+  *out = std::make_shared<Schema>(internal::DeleteVectorElement(fields_, i), metadata_);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index e0df722e5668a..45d97fdb32bbc 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -162,7 +162,8 @@ class ARROW_EXPORT DataType {
   DISALLOW_COPY_AND_ASSIGN(DataType);
 };
 
-typedef std::shared_ptr<DataType> TypePtr;
+// TODO(wesm): Remove this from parquet-cpp
+using TypePtr = std::shared_ptr<DataType>;
 
 class ARROW_EXPORT FixedWidthType : public DataType {
  public:
@@ -241,7 +242,7 @@ class ARROW_EXPORT Field {
   std::shared_ptr<const KeyValueMetadata> metadata_;
 };
 
-typedef std::shared_ptr<Field> FieldPtr;
+namespace detail {
 
 template <typename DERIVED, typename BASE, Type::type TYPE_ID, typename C_TYPE>
 class ARROW_EXPORT CTypeImpl : public BASE {
@@ -260,6 +261,13 @@ class ARROW_EXPORT CTypeImpl : public BASE {
   std::string ToString() const override { return std::string(DERIVED::name()); }
 };
 
+template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
+class IntegerTypeImpl : public detail::CTypeImpl<DERIVED, Integer, TYPE_ID, C_TYPE> {
+  bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
+};
+
+}  // namespace detail
+
 class ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
  public:
   static constexpr Type::type type_id = Type::NA;
@@ -274,11 +282,6 @@ class ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
-class IntegerTypeImpl : public CTypeImpl<DERIVED, Integer, TYPE_ID, C_TYPE> {
-  bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
-};
-
 class ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
  public:
   static constexpr Type::type type_id = Type::BOOL;
@@ -292,65 +295,70 @@ class ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
   static std::string name() { return "bool"; }
 };
 
-class ARROW_EXPORT UInt8Type : public IntegerTypeImpl<UInt8Type, Type::UINT8, uint8_t> {
+class ARROW_EXPORT UInt8Type
+    : public detail::IntegerTypeImpl<UInt8Type, Type::UINT8, uint8_t> {
  public:
   static std::string name() { return "uint8"; }
 };
 
-class ARROW_EXPORT Int8Type : public IntegerTypeImpl<Int8Type, Type::INT8, int8_t> {
+class ARROW_EXPORT Int8Type
+    : public detail::IntegerTypeImpl<Int8Type, Type::INT8, int8_t> {
  public:
   static std::string name() { return "int8"; }
 };
 
 class ARROW_EXPORT UInt16Type
-    : public IntegerTypeImpl<UInt16Type, Type::UINT16, uint16_t> {
+    : public detail::IntegerTypeImpl<UInt16Type, Type::UINT16, uint16_t> {
  public:
   static std::string name() { return "uint16"; }
 };
 
-class ARROW_EXPORT Int16Type : public IntegerTypeImpl<Int16Type, Type::INT16, int16_t> {
+class ARROW_EXPORT Int16Type
+    : public detail::IntegerTypeImpl<Int16Type, Type::INT16, int16_t> {
  public:
   static std::string name() { return "int16"; }
 };
 
 class ARROW_EXPORT UInt32Type
-    : public IntegerTypeImpl<UInt32Type, Type::UINT32, uint32_t> {
+    : public detail::IntegerTypeImpl<UInt32Type, Type::UINT32, uint32_t> {
  public:
   static std::string name() { return "uint32"; }
 };
 
-class ARROW_EXPORT Int32Type : public IntegerTypeImpl<Int32Type, Type::INT32, int32_t> {
+class ARROW_EXPORT Int32Type
+    : public detail::IntegerTypeImpl<Int32Type, Type::INT32, int32_t> {
  public:
   static std::string name() { return "int32"; }
 };
 
 class ARROW_EXPORT UInt64Type
-    : public IntegerTypeImpl<UInt64Type, Type::UINT64, uint64_t> {
+    : public detail::IntegerTypeImpl<UInt64Type, Type::UINT64, uint64_t> {
  public:
   static std::string name() { return "uint64"; }
 };
 
-class ARROW_EXPORT Int64Type : public IntegerTypeImpl<Int64Type, Type::INT64, int64_t> {
+class ARROW_EXPORT Int64Type
+    : public detail::IntegerTypeImpl<Int64Type, Type::INT64, int64_t> {
  public:
   static std::string name() { return "int64"; }
 };
 
 class ARROW_EXPORT HalfFloatType
-    : public CTypeImpl<HalfFloatType, FloatingPoint, Type::HALF_FLOAT, uint16_t> {
+    : public detail::CTypeImpl<HalfFloatType, FloatingPoint, Type::HALF_FLOAT, uint16_t> {
  public:
   Precision precision() const override;
   static std::string name() { return "halffloat"; }
 };
 
 class ARROW_EXPORT FloatType
-    : public CTypeImpl<FloatType, FloatingPoint, Type::FLOAT, float> {
+    : public detail::CTypeImpl<FloatType, FloatingPoint, Type::FLOAT, float> {
  public:
   Precision precision() const override;
   static std::string name() { return "float"; }
 };
 
 class ARROW_EXPORT DoubleType
-    : public CTypeImpl<DoubleType, FloatingPoint, Type::DOUBLE, double> {
+    : public detail::CTypeImpl<DoubleType, FloatingPoint, Type::DOUBLE, double> {
  public:
   Precision precision() const override;
   static std::string name() { return "double"; }
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 8be67b2a3829c..973b0e15c5434 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -296,6 +296,8 @@ struct TypeTraits<DictionaryType> {
   constexpr static bool is_parameter_free = false;
 };
 
+namespace detail {
+
 // Not all type classes have a c_type
 template <typename T>
 struct as_void {
@@ -319,11 +321,13 @@ GET_ATTR(TypeClass, void);
 
 #undef GET_ATTR
 
-#define PRIMITIVE_TRAITS(T)                                                 \
-  using TypeClass =                                                         \
-      typename std::conditional<std::is_base_of<DataType, T>::value, T,     \
-                                typename GetAttr_TypeClass<T>::type>::type; \
-  using c_type = typename GetAttr_c_type<TypeClass>::type;
+}  // namespace detail
+
+#define PRIMITIVE_TRAITS(T)                                                         \
+  using TypeClass =                                                                 \
+      typename std::conditional<std::is_base_of<DataType, T>::value, T,             \
+                                typename detail::GetAttr_TypeClass<T>::type>::type; \
+  using c_type = typename detail::GetAttr_c_type<TypeClass>::type;
 
 template <typename T>
 struct IsUnsignedInt {
diff --git a/cpp/src/arrow/util/bit-stream-utils.h b/cpp/src/arrow/util/bit-stream-utils.h
index 318f5ba8b0e17..d312fef4d7d07 100644
--- a/cpp/src/arrow/util/bit-stream-utils.h
+++ b/cpp/src/arrow/util/bit-stream-utils.h
@@ -227,6 +227,8 @@ inline bool BitWriter::PutVlqInt(uint32_t v) {
   return result;
 }
 
+namespace detail {
+
 template <typename T>
 inline void GetValue_(int num_bits, T* v, int max_bytes, const uint8_t* buffer,
                       int* bit_offset, int* byte_offset, uint64_t* buffered_values) {
@@ -264,6 +266,8 @@ inline void GetValue_(int num_bits, T* v, int max_bytes, const uint8_t* buffer,
   }
 }
 
+}  // namespace detail
+
 template <typename T>
 inline bool BitReader::GetValue(int num_bits, T* v) {
   return GetBatch(num_bits, v, 1) == 1;
@@ -291,15 +295,15 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
   int i = 0;
   if (UNLIKELY(bit_offset != 0)) {
     for (; i < batch_size && bit_offset != 0; ++i) {
-      GetValue_(num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset,
-                &buffered_values);
+      detail::GetValue_(num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset,
+                        &buffered_values);
     }
   }
 
   if (sizeof(T) == 4) {
     int num_unpacked =
-        unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
-                 reinterpret_cast<uint32_t*>(v + i), batch_size - i, num_bits);
+        internal::unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
+                           reinterpret_cast<uint32_t*>(v + i), batch_size - i, num_bits);
     i += num_unpacked;
     byte_offset += num_unpacked * num_bits / 8;
   } else {
@@ -307,8 +311,9 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
     uint32_t unpack_buffer[buffer_size];
     while (i < batch_size) {
       int unpack_size = std::min(buffer_size, batch_size - i);
-      int num_unpacked = unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
-                                  unpack_buffer, unpack_size, num_bits);
+      int num_unpacked =
+          internal::unpack32(reinterpret_cast<const uint32_t*>(buffer + byte_offset),
+                             unpack_buffer, unpack_size, num_bits);
       if (num_unpacked == 0) {
         break;
       }
@@ -335,8 +340,8 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
   }
 
   for (; i < batch_size; ++i) {
-    GetValue_(num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset,
-              &buffered_values);
+    detail::GetValue_(num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset,
+                      &buffered_values);
   }
 
   bit_offset_ = bit_offset;
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index f036763b8106e..fc360bae4e451 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -66,6 +66,8 @@ namespace arrow {
 //
 // We add a partial stub implementation here
 
+namespace detail {
+
 template <typename T>
 struct make_unsigned {};
 
@@ -89,6 +91,8 @@ struct make_unsigned<int64_t> {
   typedef uint64_t type;
 };
 
+}  // namespace detail
+
 class Buffer;
 class MemoryPool;
 class MutableBuffer;
@@ -253,7 +257,7 @@ static inline int Popcount(uint64_t x) {
 template <typename T>
 static inline int PopcountSigned(T v) {
   // Converting to same-width unsigned then extending preserves the bit pattern.
-  return BitUtil::Popcount(static_cast<typename make_unsigned<T>::type>(v));
+  return BitUtil::Popcount(static_cast<typename detail::make_unsigned<T>::type>(v));
 }
 
 /// Returns the 'num_bits' least-significant bits of 'v'.
@@ -364,7 +368,7 @@ static inline uint16_t FromBigEndian(uint16_t val) { return val; }
 template <typename T>
 static T ShiftRightLogical(T v, int shift) {
   // Conversion to unsigned ensures most significant bits always filled with 0's
-  return static_cast<typename make_unsigned<T>::type>(v) >> shift;
+  return static_cast<typename detail::make_unsigned<T>::type>(v) >> shift;
 }
 
 void FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits);
diff --git a/cpp/src/arrow/util/bpacking.h b/cpp/src/arrow/util/bpacking.h
index 4d25de0ab060c..14258cff6e410 100644
--- a/cpp/src/arrow/util/bpacking.h
+++ b/cpp/src/arrow/util/bpacking.h
@@ -20,12 +20,9 @@
 // https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
 // The original copyright notice follows.
 
-/**
-*
-* This code is released under the
-* Apache License Version 2.0 http://www.apache.org/licenses/.
-* (c) Daniel Lemire 2013
-*/
+// This code is released under the
+// Apache License Version 2.0 http://www.apache.org/licenses/.
+// (c) Daniel Lemire 2013
 
 #ifndef ARROW_UTIL_BPACKING_H
 #define ARROW_UTIL_BPACKING_H
@@ -33,6 +30,7 @@
 #include "arrow/util/logging.h"
 
 namespace arrow {
+namespace internal {
 
 inline const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) {
   *out = ((*in) >> 0) & 1;
@@ -3304,6 +3302,7 @@ inline int unpack32(const uint32_t* in, uint32_t* out, int batch_size, int num_b
   return batch_size;
 }
 
-};  // namespace arrow
+}  // namespace internal
+}  // namespace arrow
 
 #endif  // ARROW_UTIL_BPACKING_H
diff --git a/cpp/src/arrow/util/memory.h b/cpp/src/arrow/util/memory.h
index fce9e19293249..fef6b315779a3 100644
--- a/cpp/src/arrow/util/memory.h
+++ b/cpp/src/arrow/util/memory.h
@@ -22,6 +22,7 @@
 #include <vector>
 
 namespace arrow {
+namespace internal {
 
 uint8_t* pointer_logical_and(const uint8_t* address, uintptr_t bits) {
   uintptr_t value = reinterpret_cast<uintptr_t>(address);
@@ -66,6 +67,7 @@ void parallel_memcopy(uint8_t* dst, const uint8_t* src, int64_t nbytes,
   }
 }
 
+}  // namespace internal
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_MEMORY_H
diff --git a/cpp/src/arrow/util/random.h b/cpp/src/arrow/util/random.h
index ec48d5d4a529c..2e05a73033d0f 100644
--- a/cpp/src/arrow/util/random.h
+++ b/cpp/src/arrow/util/random.h
@@ -12,13 +12,14 @@
 #include <cmath>
 
 namespace arrow {
-
-namespace random_internal {
+namespace internal {
+namespace random {
 
 static const uint32_t M = 2147483647L;  // 2^31-1
 const double kTwoPi = 6.283185307179586476925286;
 
-}  // namespace random_internal
+}  // namespace random
+}  // namespace internal
 
 // A very simple random number generator.  Not especially good at
 // generating truly random bits, but good enough for our needs in this
@@ -27,7 +28,7 @@ class Random {
  public:
   explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) {
     // Avoid bad seeds.
-    if (seed_ == 0 || seed_ == random_internal::M) {
+    if (seed_ == 0 || seed_ == internal::random::M) {
       seed_ = 1;
     }
   }
@@ -46,12 +47,12 @@ class Random {
     uint64_t product = seed_ * A;
 
     // Compute (product % M) using the fact that ((x << 31) % M) == x.
-    seed_ = static_cast<uint32_t>((product >> 31) + (product & random_internal::M));
+    seed_ = static_cast<uint32_t>((product >> 31) + (product & internal::random::M));
     // The first reduction may overflow by 1 bit, so we may need to
     // repeat.  mod == M is not possible; using > allows the faster
     // sign-bit-based test.
-    if (seed_ > random_internal::M) {
-      seed_ -= random_internal::M;
+    if (seed_ > internal::random::M) {
+      seed_ -= internal::random::M;
     }
     return seed_;
   }
@@ -99,16 +100,16 @@ class Random {
   // Adapted from WebRTC source code at:
   // webrtc/trunk/modules/video_coding/main/test/test_util.cc
   double Normal(double mean, double std_dev) {
-    double uniform1 = (Next() + 1.0) / (random_internal::M + 1.0);
-    double uniform2 = (Next() + 1.0) / (random_internal::M + 1.0);
+    double uniform1 = (Next() + 1.0) / (internal::random::M + 1.0);
+    double uniform2 = (Next() + 1.0) / (internal::random::M + 1.0);
     return (mean +
             std_dev * sqrt(-2 * ::log(uniform1)) *
-                cos(random_internal::kTwoPi * uniform2));
+                cos(internal::random::kTwoPi * uniform2));
   }
 
   // Return a random number between 0.0 and 1.0 inclusive.
   double NextDoubleFraction() {
-    return Next() / static_cast<double>(random_internal::M + 1.0);
+    return Next() / static_cast<double>(internal::random::M + 1.0);
   }
 
  private:
diff --git a/cpp/src/arrow/util/stl-util-test.cc b/cpp/src/arrow/util/stl-util-test.cc
index 526520e7a2dec..629eb24c3d97e 100644
--- a/cpp/src/arrow/util/stl-util-test.cc
+++ b/cpp/src/arrow/util/stl-util-test.cc
@@ -25,6 +25,7 @@
 #include "arrow/test-util.h"
 
 namespace arrow {
+namespace internal {
 
 TEST(StlUtilTest, VectorAddRemoveTest) {
   std::vector<int> values;
@@ -57,4 +58,5 @@ TEST(StlUtilTest, VectorAddRemoveTest) {
   EXPECT_TRUE(result3.empty());
 }
 
+}  // namespace internal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/stl.h b/cpp/src/arrow/util/stl.h
index 4b8916f6eaa54..27c1778680c37 100644
--- a/cpp/src/arrow/util/stl.h
+++ b/cpp/src/arrow/util/stl.h
@@ -23,6 +23,7 @@
 #include "arrow/util/logging.h"
 
 namespace arrow {
+namespace internal {
 
 template <typename T>
 inline std::vector<T> DeleteVectorElement(const std::vector<T>& values, size_t index) {
@@ -55,6 +56,7 @@ inline std::vector<T> AddVectorElement(const std::vector<T>& values, size_t inde
   return out;
 }
 
+}  // namespace internal
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_STL_H
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index 117578965ccc4..203ed6d4af91a 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -59,6 +59,8 @@ Status ArrayVisitor::Visit(const DecimalArray& array) {
   return Status::NotImplemented("decimal");
 }
 
+#undef ARRAY_VISITOR_DEFAULT
+
 // ----------------------------------------------------------------------
 // Default implementations of TypeVisitor methods
 
@@ -95,4 +97,6 @@ TYPE_VISITOR_DEFAULT(StructType);
 TYPE_VISITOR_DEFAULT(UnionType);
 TYPE_VISITOR_DEFAULT(DictionaryType);
 
+#undef TYPE_VISITOR_DEFAULT
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index 7478950b894c5..54f9e880b834d 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -65,6 +65,8 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
   return Status::NotImplemented("Type not implemented");
 }
 
+#undef TYPE_VISIT_INLINE
+
 #define ARRAY_VISIT_INLINE(TYPE_CLASS) \
   case TYPE_CLASS::type_id:            \
     return visitor->Visit(             \

From aa1d753a74b5517c0b20db8e5540786520b9956f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 1 Aug 2017 11:57:43 -0400
Subject: [PATCH 0886/1644] ARROW-573: [C++/Python] Implement IPC metadata
 handling for ordered dictionaries, pandas conversions

This was an oversight in the IPC implementation and pandas conversion path, and has been fixed.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #922 from wesm/ARROW-573 and squashes the following commits:

458820e5 [Wes McKinney] Suppress C4800 in MSVC
46361f3f [Wes McKinney] Implement IPC metadata handling for ordered dictionaries, faithful conversion to/from pandas.Categorical
---
 cpp/src/arrow/array-test.cc                 | 11 +++++++----
 cpp/src/arrow/compare.cc                    |  3 ++-
 cpp/src/arrow/ipc/metadata.cc               |  5 +++--
 cpp/src/arrow/ipc/test-common.h             |  2 +-
 cpp/src/arrow/type.cc                       |  7 ++++---
 cpp/src/arrow/type.h                        |  5 +++--
 python/CMakeLists.txt                       |  4 ++++
 python/pyarrow/array.pxi                    | 11 ++++++++---
 python/pyarrow/includes/libarrow.pxd        |  4 +++-
 python/pyarrow/pandas_compat.py             |  3 ++-
 python/pyarrow/tests/test_convert_pandas.py |  3 +++
 python/pyarrow/tests/test_ipc.py            | 22 +++++++++++++++++----
 python/pyarrow/types.pxi                    | 11 +++++++++--
 13 files changed, 67 insertions(+), 24 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 0efb51ccece0c..57d2c8b8493a5 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1881,15 +1881,18 @@ TEST(TestDictionary, Basics) {
 
   std::shared_ptr<DictionaryType> type1 =
       std::dynamic_pointer_cast<DictionaryType>(dictionary(int16(), dict));
-  DictionaryType type2(int16(), dict);
+
+  auto type2 =
+      std::dynamic_pointer_cast<DictionaryType>(::arrow::dictionary(int16(), dict, true));
 
   ASSERT_TRUE(int16()->Equals(type1->index_type()));
   ASSERT_TRUE(type1->dictionary()->Equals(dict));
 
-  ASSERT_TRUE(int16()->Equals(type2.index_type()));
-  ASSERT_TRUE(type2.dictionary()->Equals(dict));
+  ASSERT_TRUE(int16()->Equals(type2->index_type()));
+  ASSERT_TRUE(type2->dictionary()->Equals(dict));
 
-  ASSERT_EQ("dictionary<values=int32, indices=int16>", type1->ToString());
+  ASSERT_EQ("dictionary<values=int32, indices=int16, ordered=0>", type1->ToString());
+  ASSERT_EQ("dictionary<values=int32, indices=int16, ordered=1>", type2->ToString());
 }
 
 TEST(TestDictionary, Equals) {
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index dda5fdd95d0c3..3a4a4009c6b16 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -769,7 +769,8 @@ class TypeEqualsVisitor {
   Status Visit(const DictionaryType& left) {
     const auto& right = static_cast<const DictionaryType&>(right_);
     result_ = left.index_type()->Equals(right.index_type()) &&
-              left.dictionary()->Equals(right.dictionary());
+              left.dictionary()->Equals(right.dictionary()) &&
+              (left.ordered() == right.ordered());
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 20fd280db6de6..d764e203e7552 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -492,7 +492,8 @@ static DictionaryOffset GetDictionaryEncoding(FBB& fbb, const DictionaryType& ty
   auto index_type_offset = flatbuf::CreateInt(fbb, fw_index_type.bit_width(), true);
 
   // TODO(wesm): ordered dictionaries
-  return flatbuf::CreateDictionaryEncoding(fbb, dictionary_id, index_type_offset);
+  return flatbuf::CreateDictionaryEncoding(fbb, dictionary_id, index_type_offset,
+                                           type.ordered());
 }
 
 static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
@@ -551,7 +552,7 @@ static Status FieldFromFlatbuffer(const flatbuf::Field* field,
 
     std::shared_ptr<DataType> index_type;
     RETURN_NOT_OK(IntFromFlatbuffer(encoding->indexType(), &index_type));
-    type = std::make_shared<DictionaryType>(index_type, dictionary);
+    type = ::arrow::dictionary(index_type, dictionary, encoding->isOrdered());
   }
   *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
   return Status::OK();
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index cb827372d21c4..76cc8430636f8 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -462,7 +462,7 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   ArrayFromVector<StringType, std::string>(dict2_values, &dict2);
 
   auto f0_type = arrow::dictionary(arrow::int32(), dict1);
-  auto f1_type = arrow::dictionary(arrow::int8(), dict1);
+  auto f1_type = arrow::dictionary(arrow::int8(), dict1, true);
   auto f2_type = arrow::dictionary(arrow::int32(), dict2);
 
   std::shared_ptr<Array> indices0, indices1, indices2;
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index b8489d44cdb00..edf4d33b23f39 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -236,7 +236,7 @@ std::shared_ptr<Array> DictionaryType::dictionary() const { return dictionary_;
 std::string DictionaryType::ToString() const {
   std::stringstream ss;
   ss << "dictionary<values=" << dictionary_->type()->ToString()
-     << ", indices=" << index_type_->ToString() << ">";
+     << ", indices=" << index_type_->ToString() << ", ordered=" << ordered_ << ">";
   return ss.str();
 }
 
@@ -428,8 +428,9 @@ std::shared_ptr<DataType> union_(const std::vector<std::shared_ptr<Field>>& chil
 }
 
 std::shared_ptr<DataType> dictionary(const std::shared_ptr<DataType>& index_type,
-                                     const std::shared_ptr<Array>& dict_values) {
-  return std::make_shared<DictionaryType>(index_type, dict_values);
+                                     const std::shared_ptr<Array>& dict_values,
+                                     bool ordered) {
+  return std::make_shared<DictionaryType>(index_type, dict_values, ordered);
 }
 
 std::shared_ptr<Field> field(const std::string& name,
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 45d97fdb32bbc..b28fe9229b2ae 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -785,8 +785,9 @@ std::shared_ptr<DataType> ARROW_EXPORT
 union_(const std::vector<std::shared_ptr<Field>>& child_fields,
        const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
 
-std::shared_ptr<DataType> ARROW_EXPORT dictionary(
-    const std::shared_ptr<DataType>& index_type, const std::shared_ptr<Array>& values);
+std::shared_ptr<DataType> ARROW_EXPORT
+dictionary(const std::shared_ptr<DataType>& index_type,
+           const std::shared_ptr<Array>& values, bool ordered = false);
 
 std::shared_ptr<Field> ARROW_EXPORT field(
     const std::string& name, const std::shared_ptr<DataType>& type, bool nullable = true,
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index bfae157ed6b9c..af95073f5da35 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -92,6 +92,10 @@ else()
   # Cython generates some bitshift expressions that MSVC does not like in
   # __Pyx_PyFloat_DivideObjC
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4293")
+
+  # Converting to/from C++ bool is pretty wonky in Cython. The C4800 warning
+  # seem harmless, and probably not worth the effort of working around it
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800")
 endif()
 
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 67418aa5eac67..f320cbedc8d3a 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -189,7 +189,8 @@ cdef class Array:
         if isinstance(values, Categorical):
             return DictionaryArray.from_arrays(
                 values.codes, values.categories.values,
-                mask=mask, memory_pool=memory_pool)
+                mask=mask, ordered=values.ordered,
+                memory_pool=memory_pool)
         elif values.dtype == object:
             # Object dtype undergoes a different conversion path as more type
             # inference may be needed
@@ -564,7 +565,7 @@ cdef class DictionaryArray(Array):
             return self._indices
 
     @staticmethod
-    def from_arrays(indices, dictionary, mask=None,
+    def from_arrays(indices, dictionary, mask=None, ordered=False,
                     MemoryPool memory_pool=None):
         """
         Construct Arrow DictionaryArray from array of indices (must be
@@ -576,6 +577,8 @@ cdef class DictionaryArray(Array):
         dictionary : ndarray or pandas.Series
         mask : ndarray or pandas.Series, boolean type
             True values indicate that indices are actually null
+        ordered : boolean, default False
+            Set to True if the category values are ordered
 
         Returns
         -------
@@ -609,8 +612,10 @@ cdef class DictionaryArray(Array):
         if not isinstance(arrow_indices, IntegerArray):
             raise ValueError('Indices must be integer type')
 
+        cdef c_bool c_ordered = ordered
+
         c_type.reset(new CDictionaryType(arrow_indices.type.sp_type,
-                                         arrow_dictionary.sp_array))
+                                         arrow_dictionary.sp_array, c_ordered))
         c_result.reset(new CDictionaryArray(c_type, arrow_indices.sp_array))
 
         result = DictionaryArray()
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 8d7e27915eede..a25d7a2f5b7a5 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -132,10 +132,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CDictionaryType" arrow::DictionaryType"(CFixedWidthType):
         CDictionaryType(const shared_ptr[CDataType]& index_type,
-                        const shared_ptr[CArray]& dictionary)
+                        const shared_ptr[CArray]& dictionary,
+                        c_bool ordered)
 
         shared_ptr[CDataType] index_type()
         shared_ptr[CArray] dictionary()
+        c_bool ordered()
 
     shared_ptr[CDataType] ctimestamp" arrow::timestamp"(TimeUnit unit)
     shared_ptr[CDataType] ctimestamp" arrow::timestamp"(
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index cd7ad47782646..62547a42f7359 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -284,9 +284,10 @@ def table_to_blockmanager(table, nthreads=1):
         block_arr = item['block']
         placement = item['placement']
         if 'dictionary' in item:
+            ordered = block_table.schema[placement[0]].type.ordered
             cat = pd.Categorical(block_arr,
                                  categories=item['dictionary'],
-                                 ordered=False, fastpath=True)
+                                 ordered=ordered, fastpath=True)
             block = _int.make_block(cat, placement=placement,
                                     klass=_int.CategoricalBlock,
                                     fastpath=True)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index d488658563306..f6ea1636a3d62 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -536,6 +536,9 @@ def test_category(self):
         df = pd.DataFrame({'cat_strings': pd.Categorical(v1 * repeats),
                            'cat_ints': pd.Categorical(v2 * repeats),
                            'cat_binary': pd.Categorical(v3 * repeats),
+                           'cat_strings_ordered': pd.Categorical(
+                               v1 * repeats, categories=['bar', 'qux', 'foo'],
+                               ordered=True),
                            'ints': v2 * repeats,
                            'ints2': v2 * repeats,
                            'strings': v1 * repeats,
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 3ad369c31f4f2..120a9825a7b56 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -40,22 +40,20 @@ def _get_sink(self):
     def _get_source(self):
         return self.sink.getvalue()
 
-    def write_batches(self):
+    def write_batches(self, num_batches=5):
         nrows = 5
         df = pd.DataFrame({
             'one': np.random.randn(nrows),
             'two': ['foo', np.nan, 'bar', 'bazbaz', 'qux']})
-
         batch = pa.RecordBatch.from_pandas(df)
 
         writer = self._get_writer(self.sink, batch.schema)
 
-        num_batches = 5
         frames = []
         batches = []
         for i in range(num_batches):
             unique_df = df.copy()
-            unique_df['one'] = np.random.randn(nrows)
+            unique_df['one'] = np.random.randn(len(df))
 
             batch = pa.RecordBatch.from_pandas(unique_df)
             writer.write_batch(batch)
@@ -122,6 +120,22 @@ def test_empty_stream(self):
         with pytest.raises(pa.ArrowInvalid):
             pa.open_stream(buf)
 
+    def test_categorical_roundtrip(self):
+        df = pd.DataFrame({
+            'one': np.random.randn(5),
+            'two': pd.Categorical(['foo', np.nan, 'bar', 'foo', 'foo'],
+                                  categories=['foo', 'bar'],
+                                  ordered=True)
+        })
+        batch = pa.RecordBatch.from_pandas(df)
+        writer = self._get_writer(self.sink, batch.schema)
+        writer.write_batch(pa.RecordBatch.from_pandas(df))
+        writer.close()
+
+        table = (pa.open_stream(pa.BufferReader(self._get_source()))
+                 .read_all())
+        assert_frame_equal(table.to_pandas(), df)
+
     def test_simple_roundtrip(self):
         _, batches = self.write_batches()
         file_contents = pa.BufferReader(self._get_source())
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index fefde55bc2f95..ad2f336061580 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -97,6 +97,11 @@ cdef class DictionaryType(DataType):
         DataType.init(self, type)
         self.dict_type = <const CDictionaryType*> type.get()
 
+    property ordered:
+
+        def __get__(self):
+            return self.dict_type.ordered()
+
 
 cdef class ListType(DataType):
 
@@ -798,7 +803,8 @@ cpdef ListType list_(value_type):
     return out
 
 
-cpdef DictionaryType dictionary(DataType index_type, Array dictionary):
+cpdef DictionaryType dictionary(DataType index_type, Array dictionary,
+                                bint ordered=False):
     """
     Dictionary (categorical, or simply encoded) type
 
@@ -814,7 +820,8 @@ cpdef DictionaryType dictionary(DataType index_type, Array dictionary):
     cdef DictionaryType out = DictionaryType()
     cdef shared_ptr[CDataType] dict_type
     dict_type.reset(new CDictionaryType(index_type.sp_type,
-                                        dictionary.sp_array))
+                                        dictionary.sp_array,
+                                        ordered == 1))
     out.init(dict_type)
     return out
 

From e5ed31fc5e903ea0a2102623413ab85577cba123 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 1 Aug 2017 22:50:21 -0400
Subject: [PATCH 0887/1644] ARROW-1093: [Python] Run flake8 in Travis CI. Add
 note about development to README

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #924 from wesm/ARROW-1093 and squashes the following commits:

75969c37 [Wes McKinney] Add spaces before continuation backslash
fa24ec92 [Wes McKinney] Specify file suffixes completely
c0af17c3 [Wes McKinney] Add flake8 file for Cython, fix Cython style errors
7ffa6135 [Wes McKinney] Add Cython flake8 file
f10e8d1f [Wes McKinney] Run flake8 in Travis CI. Add note to README
---
 ci/travis_script_python.sh           |  8 +++-
 python/.flake8.cython                | 20 ++++++++++
 python/README.md                     | 19 ++++++++-
 python/pyarrow/_config.pyx           |  9 +++--
 python/pyarrow/_parquet.pxd          | 45 ++++++++++++---------
 python/pyarrow/_parquet.pyx          |  9 +++--
 python/pyarrow/array.pxi             |  9 ++---
 python/pyarrow/feather.pxi           |  2 +-
 python/pyarrow/includes/libarrow.pxd | 60 ++++++++++++++--------------
 python/pyarrow/io-hdfs.pxi           |  1 -
 python/pyarrow/io.pxi                | 10 ++++-
 python/pyarrow/ipc.pxi               |  2 +-
 python/pyarrow/lib.pxd               |  4 +-
 python/pyarrow/lib.pyx               | 23 +++++------
 python/pyarrow/plasma.pyx            | 24 ++++++-----
 python/pyarrow/public-api.pxi        | 11 ++---
 python/pyarrow/scalar.pxi            |  8 ++--
 python/pyarrow/table.pxi             |  5 +--
 python/pyarrow/types.pxi             |  2 +-
 19 files changed, 160 insertions(+), 111 deletions(-)
 create mode 100644 python/.flake8.cython

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 9135aaf38e4e7..4a50d2faaf551 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -92,7 +92,13 @@ python_version_tests() {
   conda install -y -q nomkl
 
   # Expensive dependencies install from Continuum package repo
-  conda install -y -q pip numpy pandas cython
+  conda install -y -q pip numpy pandas cython flake8
+
+  # Fail fast on style checks
+  flake8 pyarrow
+
+  # Check Cython files with some checks turned off
+  flake8 --config=.flake8.cython pyarrow
 
   # Build C++ libraries
   rebuild_arrow_libraries
diff --git a/python/.flake8.cython b/python/.flake8.cython
new file mode 100644
index 0000000000000..53e41323051f9
--- /dev/null
+++ b/python/.flake8.cython
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[flake8]
+filename = *.pyx,*.pxd,*.pxi
+ignore = E211,E901,E225,E226,E227
diff --git a/python/README.md b/python/README.md
index 29d213babd93b..3c48d5d30b595 100644
--- a/python/README.md
+++ b/python/README.md
@@ -38,7 +38,21 @@ On Linux, you can also install binary wheels from PyPI with pip:
 pip install pyarrow
 ```
 
-### Development details
+## Development
+
+### Coding Style
+
+We follow a similar PEP8-like coding style to the [pandas project][3].
+
+The code must pass `flake8` (available from pip or conda) or it will fail the
+build. Check for style errors before submitting your pull request with:
+
+```
+flake8 pyarrow
+flake8 --config=.flake8.cython pyarrow
+```
+
+### Building from Source
 
 See the [Development][2] page in the documentation.
 
@@ -50,4 +64,5 @@ python setup.py build_sphinx -s doc/source
 ```
 
 [1]: https://github.com/apache/parquet-cpp
-[2]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst
\ No newline at end of file
+[2]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst
+[3]: https://github.com/pandas-dev/pandas
\ No newline at end of file
diff --git a/python/pyarrow/_config.pyx b/python/pyarrow/_config.pyx
index a2d2d719e68d0..bc9f36d8e50cb 100644
--- a/python/pyarrow/_config.pyx
+++ b/python/pyarrow/_config.pyx
@@ -19,6 +19,10 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+import numpy as np
+import multiprocessing
+import os
+
 cdef extern from 'arrow/python/init.h':
     int arrow_init_numpy() except -1
 
@@ -27,15 +31,13 @@ cdef extern from 'arrow/python/config.h' namespace 'arrow::py':
 
 arrow_init_numpy()
 
-import numpy as np
 set_numpy_nan(np.nan)
 
-import multiprocessing
-import os
 cdef int CPU_COUNT = int(
     os.environ.get('OMP_NUM_THREADS',
                    max(multiprocessing.cpu_count() // 2, 1)))
 
+
 def cpu_count():
     """
     Returns
@@ -49,6 +51,7 @@ def cpu_count():
     """
     return CPU_COUNT
 
+
 def set_cpu_count(count):
     global CPU_COUNT
     CPU_COUNT = max(int(count), 1)
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index b1cd5eb2c2be0..7299e19b81906 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -25,17 +25,18 @@ from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
-  cdef cppclass Node:
-    pass
+    cdef cppclass Node:
+        pass
 
-  cdef cppclass GroupNode(Node):
-    pass
+    cdef cppclass GroupNode(Node):
+        pass
 
-  cdef cppclass PrimitiveNode(Node):
-    pass
+    cdef cppclass PrimitiveNode(Node):
+        pass
+
+    cdef cppclass ColumnPath:
+        c_string ToDotString()
 
-  cdef cppclass ColumnPath:
-    c_string ToDotString()
 
 cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
     enum ParquetType" parquet::Type::type":
@@ -59,8 +60,10 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
         ParquetLogicalType_DATE" parquet::LogicalType::DATE"
         ParquetLogicalType_TIME_MILLIS" parquet::LogicalType::TIME_MILLIS"
         ParquetLogicalType_TIME_MICROS" parquet::LogicalType::TIME_MICROS"
-        ParquetLogicalType_TIMESTAMP_MILLIS" parquet::LogicalType::TIMESTAMP_MILLIS"
-        ParquetLogicalType_TIMESTAMP_MICROS" parquet::LogicalType::TIMESTAMP_MICROS"
+        ParquetLogicalType_TIMESTAMP_MILLIS \
+            " parquet::LogicalType::TIMESTAMP_MILLIS"
+        ParquetLogicalType_TIMESTAMP_MICROS \
+            " parquet::LogicalType::TIMESTAMP_MICROS"
         ParquetLogicalType_UINT_8" parquet::LogicalType::UINT_8"
         ParquetLogicalType_UINT_16" parquet::LogicalType::UINT_16"
         ParquetLogicalType_UINT_32" parquet::LogicalType::UINT_32"
@@ -83,8 +86,10 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
         ParquetEncoding_PLAIN_DICTIONARY" parquet::Encoding::PLAIN_DICTIONARY"
         ParquetEncoding_RLE" parquet::Encoding::RLE"
         ParquetEncoding_BIT_PACKED" parquet::Encoding::BIT_PACKED"
-        ParquetEncoding_DELTA_BINARY_PACKED" parquet::Encoding::DELTA_BINARY_PACKED"
-        ParquetEncoding_DELTA_LENGTH_BYTE_ARRAY" parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY"
+        ParquetEncoding_DELTA_BINARY_PACKED \
+            " parquet::Encoding::DELTA_BINARY_PACKED"
+        ParquetEncoding_DELTA_LENGTH_BYTE_ARRAY \
+            " parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY"
         ParquetEncoding_DELTA_BYTE_ARRAY" parquet::Encoding::DELTA_BYTE_ARRAY"
         ParquetEncoding_RLE_DICTIONARY" parquet::Encoding::RLE_DICTIONARY"
 
@@ -231,13 +236,15 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 
 
 cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
-    CStatus FromParquetSchema(const SchemaDescriptor* parquet_schema,
-                              const shared_ptr[const CKeyValueMetadata]& key_value_metadata,
-                              shared_ptr[CSchema]* out)
-
-    CStatus ToParquetSchema(const CSchema* arrow_schema,
-                            const shared_ptr[const CKeyValueMetadata]& key_value_metadata,
-                            shared_ptr[SchemaDescriptor]* out)
+    CStatus FromParquetSchema(
+        const SchemaDescriptor* parquet_schema,
+        const shared_ptr[const CKeyValueMetadata]& key_value_metadata,
+        shared_ptr[CSchema]* out)
+
+    CStatus ToParquetSchema(
+        const CSchema* arrow_schema,
+        const shared_ptr[const CKeyValueMetadata]& key_value_metadata,
+        shared_ptr[SchemaDescriptor]* out)
 
 
 cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index c940122da5dcf..919e82c109451 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -279,8 +279,8 @@ cdef class ColumnSchema:
   max_repetition_level: {3}
   physical_type: {4}
   logical_type: {5}""".format(self.name, self.path, self.max_definition_level,
-                       self.max_repetition_level, physical_type,
-                       logical_type)
+                              self.max_repetition_level, physical_type,
+                              logical_type)
 
     property name:
 
@@ -514,7 +514,7 @@ cdef class ParquetReader:
 
         with nogil:
             check_status(self.reader.get()
-                         .ReadSchemaField(field_index, &carray));
+                         .ReadSchemaField(field_index, &carray))
 
         array.init(carray)
         return array
@@ -553,7 +553,8 @@ cdef class ParquetWriter:
 
     def __cinit__(self, where, Schema schema, use_dictionary=None,
                   compression=None, version=None,
-                  MemoryPool memory_pool=None, use_deprecated_int96_timestamps=False):
+                  MemoryPool memory_pool=None,
+                  use_deprecated_int96_timestamps=False):
         cdef:
             shared_ptr[FileOutputStream] filestream
             shared_ptr[WriterProperties] properties
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index f320cbedc8d3a..cbd036c08431f 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -66,8 +66,8 @@ def array(object sequence, DataType type=None, MemoryPool memory_pool=None,
     array : pyarrow.Array
     """
     cdef:
-       shared_ptr[CArray] sp_array
-       CMemoryPool* pool
+        shared_ptr[CArray] sp_array
+        CMemoryPool* pool
 
     pool = maybe_unbox_memory_pool(memory_pool)
     if type is None:
@@ -78,13 +78,13 @@ def array(object sequence, DataType type=None, MemoryPool memory_pool=None,
                 ConvertPySequence(
                     sequence, pool, &sp_array, type.sp_type
                 )
-             )
+            )
         else:
             check_status(
                 ConvertPySequence(
                     sequence, pool, &sp_array, type.sp_type, size
                 )
-             )
+            )
 
     return pyarrow_wrap_array(sp_array)
 
@@ -401,7 +401,6 @@ strides: {2}""".format(self.type, self.shape, self.strides)
             return py_strides
 
 
-
 cdef wrap_array_output(PyObject* output):
     cdef object obj = PyObject_to_object(output)
 
diff --git a/python/pyarrow/feather.pxi b/python/pyarrow/feather.pxi
index 2e7cf6c9bd1b8..6faf2f9c69c7b 100644
--- a/python/pyarrow/feather.pxi
+++ b/python/pyarrow/feather.pxi
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-#----------------------------------------------------------------------
+# ---------------------------------------------------------------------
 # Implement legacy Feather file format
 
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index a25d7a2f5b7a5..3ea487385de76 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -178,7 +178,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
 
-    cdef cppclass CFixedSizeBinaryType" arrow::FixedSizeBinaryType"(CFixedWidthType):
+    cdef cppclass CFixedSizeBinaryType \
+            " arrow::FixedSizeBinaryType"(CFixedWidthType):
         CFixedSizeBinaryType(int byte_width)
         int byte_width()
         int bit_width()
@@ -209,7 +210,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
             const shared_ptr[CKeyValueMetadata]& metadata)
         shared_ptr[CField] RemoveMetadata()
 
-
     cdef cppclass CStructType" arrow::StructType"(CDataType):
         CStructType(const vector[shared_ptr[CField]]& fields)
 
@@ -309,9 +309,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CStructArray" arrow::StructArray"(CArray):
         CStructArray(shared_ptr[CDataType] type, int64_t length,
-            vector[shared_ptr[CArray]] children,
-            shared_ptr[CBuffer] null_bitmap = nullptr, int64_t null_count = 0,
-            int64_t offset = 0)
+                     vector[shared_ptr[CArray]] children,
+                     shared_ptr[CBuffer] null_bitmap=nullptr,
+                     int64_t null_count=0,
+                     int64_t offset=0)
 
         shared_ptr[CArray] field(int pos)
         const vector[shared_ptr[CArray]] fields()
@@ -462,7 +463,6 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
 
 cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
 
-
     cdef cppclass FileOutputStream(OutputStream):
         @staticmethod
         CStatus Open(const c_string& path, shared_ptr[FileOutputStream]* file)
@@ -479,12 +479,12 @@ cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
 
         int file_descriptor()
 
-    cdef cppclass CMemoryMappedFile" arrow::io::MemoryMappedFile"\
-        (ReadWriteFileInterface):
+    cdef cppclass CMemoryMappedFile \
+            " arrow::io::MemoryMappedFile"(ReadWriteFileInterface):
 
         @staticmethod
         CStatus Create(const c_string& path, int64_t size,
-                     shared_ptr[CMemoryMappedFile]* file)
+                       shared_ptr[CMemoryMappedFile]* file)
 
         @staticmethod
         CStatus Open(const c_string& path, FileMode mode,
@@ -509,7 +509,7 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
         HdfsDriver driver
 
     cdef cppclass HdfsPathInfo:
-        ObjectType kind;
+        ObjectType kind
         c_string name
         c_string owner
         c_string group
@@ -563,21 +563,22 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
 
 
 cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
-    cdef cppclass CBufferReader" arrow::io::BufferReader"\
-        (RandomAccessFile):
+    cdef cppclass CBufferReader \
+            " arrow::io::BufferReader"(RandomAccessFile):
         CBufferReader(const shared_ptr[CBuffer]& buffer)
         CBufferReader(const uint8_t* data, int64_t nbytes)
 
-    cdef cppclass CBufferOutputStream" arrow::io::BufferOutputStream"\
-        (OutputStream):
+    cdef cppclass CBufferOutputStream \
+            " arrow::io::BufferOutputStream"(OutputStream):
         CBufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
 
-    cdef cppclass CMockOutputStream" arrow::io::MockOutputStream"\
-        (OutputStream):
+    cdef cppclass CMockOutputStream \
+            " arrow::io::MockOutputStream"(OutputStream):
         CMockOutputStream()
         int64_t GetExtentBytesWritten()
 
-    cdef cppclass CFixedSizeBufferWriter" arrow::io::FixedSizeBufferWriter"(WriteableFile):
+    cdef cppclass CFixedSizeBufferWriter \
+            " arrow::io::FixedSizeBufferWriter"(WriteableFile):
         CFixedSizeBufferWriter(const shared_ptr[CBuffer]& buffer)
 
 
@@ -609,48 +610,45 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
 
     c_string FormatMessageType(MessageType type)
 
-    cdef cppclass CMessageReader \
-        " arrow::ipc::MessageReader":
+    cdef cppclass CMessageReader" arrow::ipc::MessageReader":
         CStatus ReadNextMessage(unique_ptr[CMessage]* out)
 
     cdef cppclass CInputStreamMessageReader \
-        " arrow::ipc::InputStreamMessageReader":
+            " arrow::ipc::InputStreamMessageReader":
         CInputStreamMessageReader(const shared_ptr[InputStream]& stream)
 
-    cdef cppclass CRecordBatchWriter \
-        " arrow::ipc::RecordBatchWriter":
+    cdef cppclass CRecordBatchWriter" arrow::ipc::RecordBatchWriter":
         CStatus Close()
         CStatus WriteRecordBatch(const CRecordBatch& batch)
 
-    cdef cppclass CRecordBatchReader \
-        " arrow::ipc::RecordBatchReader":
+    cdef cppclass CRecordBatchReader" arrow::ipc::RecordBatchReader":
         shared_ptr[CSchema] schema()
         CStatus ReadNextRecordBatch(shared_ptr[CRecordBatch]* batch)
 
     cdef cppclass CRecordBatchStreamReader \
-        " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
+            " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
         @staticmethod
         CStatus Open(const shared_ptr[InputStream]& stream,
                      shared_ptr[CRecordBatchStreamReader]* out)
 
         @staticmethod
         CStatus Open2" Open"(unique_ptr[CMessageReader] message_reader,
-                     shared_ptr[CRecordBatchStreamReader]* out)
+                             shared_ptr[CRecordBatchStreamReader]* out)
 
     cdef cppclass CRecordBatchStreamWriter \
-        " arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter):
+            " arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter):
         @staticmethod
         CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
                      shared_ptr[CRecordBatchStreamWriter]* out)
 
     cdef cppclass CRecordBatchFileWriter \
-        " arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter):
+            " arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter):
         @staticmethod
         CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
                      shared_ptr[CRecordBatchFileWriter]* out)
 
     cdef cppclass CRecordBatchFileReader \
-        " arrow::ipc::RecordBatchFileReader":
+            " arrow::ipc::RecordBatchFileReader":
         @staticmethod
         CStatus Open(const shared_ptr[RandomAccessFile]& file,
                      shared_ptr[CRecordBatchFileReader]* out)
@@ -724,7 +722,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     CStatus ConvertPySequence(object obj, CMemoryPool* pool,
                               shared_ptr[CArray]* out,
                               const shared_ptr[CDataType]& type,
-			      int64_t size)
+                              int64_t size)
 
     CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
 
@@ -737,7 +735,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
                                  shared_ptr[CChunkedArray]* out)
 
     CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
-                            shared_ptr[CTensor]* out);
+                            shared_ptr[CTensor]* out)
 
     CStatus TensorToNdarray(const CTensor& tensor, object base,
                             PyObject** out)
diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi
index 8ac4e8c2319c1..27e9948b3a4ea 100644
--- a/python/pyarrow/io-hdfs.pxi
+++ b/python/pyarrow/io-hdfs.pxi
@@ -231,7 +231,6 @@ cdef class HadoopFileSystem:
             check_status(self.client.get()
                          .GetPathInfo(c_path, info))
 
-
     def ls(self, path, bint full_info):
         cdef:
             c_string c_path = tobytes(path)
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 211c2a3e6e9cf..cccb1736be6fa 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -255,13 +255,18 @@ cdef class NativeFile:
 
         if not hasattr(stream_or_path, 'read'):
             stream = open(stream_or_path, 'wb')
-            cleanup = lambda: stream.close()
+
+            def cleanup():
+                stream.close()
         else:
             stream = stream_or_path
-            cleanup = lambda: None
+
+            def cleanup():
+                pass
 
         done = False
         exc_info = None
+
         def bg_write():
             try:
                 while not done or write_queue.qsize() > 0:
@@ -326,6 +331,7 @@ cdef class NativeFile:
 
         done = False
         exc_info = None
+
         def bg_write():
             try:
                 while not done or write_queue.qsize() > 0:
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index 31ee578920eae..d6f62aa95c5fb 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -279,7 +279,7 @@ cdef class _RecordBatchFileWriter(_RecordBatchWriter):
         with nogil:
             check_status(
                 CRecordBatchFileWriter.Open(self.sink.get(), schema.sp_schema,
-                                      &writer))
+                                            &writer))
 
         # Cast to base class, because has same interface
         self.writer = <shared_ptr[CRecordBatchWriter]> writer
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 3e1419bdfc072..48a58f7b82660 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -17,6 +17,7 @@
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.libarrow cimport CStatus
 from cpython cimport PyObject
 from libcpp cimport nullptr
 
@@ -24,9 +25,6 @@ cdef extern from "Python.h":
     int PySlice_Check(object)
 
 
-from pyarrow.includes.libarrow cimport CStatus
-
-
 cdef int check_status(const CStatus& status) nogil except -1
 
 
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 4df2fcd64f60f..789801b9f06a9 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -19,31 +19,27 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from cython.operator cimport dereference as deref
-from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.common cimport PyObject_to_object
-cimport pyarrow.includes.libarrow as libarrow
-cimport cpython as cp
-
-
 import datetime
 import decimal as _pydecimal
+import multiprocessing
 import numpy as np
+import os
 import six
 from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
 
+from cython.operator cimport dereference as deref
+from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.common cimport PyObject_to_object
+cimport pyarrow.includes.libarrow as libarrow
+cimport cpython as cp
+
 cdef _pandas():
     import pandas as pd
     return pd
 
-
 arrow_init_numpy()
-
-import numpy as np
 set_numpy_nan(np.nan)
 
-import multiprocessing
-import os
 cdef int CPU_COUNT = int(
     os.environ.get('OMP_NUM_THREADS',
                    max(multiprocessing.cpu_count() // 2, 1)))
@@ -62,6 +58,7 @@ def cpu_count():
     """
     return CPU_COUNT
 
+
 def set_cpu_count(count):
     global CPU_COUNT
     CPU_COUNT = max(int(count), 1)
@@ -122,7 +119,5 @@ include "ipc.pxi"
 # Feather format
 include "feather.pxi"
 
-#----------------------------------------------------------------------
 # Public API
-
 include "public-api.pxi"
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index dd62d473b001a..befa283d85b54 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -60,8 +60,8 @@ cdef extern from "plasma/common.h":
         PLASMA_QUERY_LOCAL"plasma::PLASMA_QUERY_LOCAL",
         PLASMA_QUERY_ANYWHERE"plasma::PLASMA_QUERY_ANYWHERE"
 
-    cdef int ObjectStatusLocal"plasma::ObjectStatusLocal";
-    cdef int ObjectStatusRemote"plasma::ObjectStatusRemote";
+    cdef int ObjectStatusLocal"plasma::ObjectStatusLocal"
+    cdef int ObjectStatusRemote"plasma::ObjectStatusRemote"
 
 cdef extern from "plasma/client.h" nogil:
 
@@ -102,7 +102,7 @@ cdef extern from "plasma/client.h" nogil:
         CStatus Wait(int64_t num_object_requests,
                      CObjectRequest* object_requests,
                      int num_ready_objects, int64_t timeout_ms,
-                     int* num_objects_ready);
+                     int* num_objects_ready)
 
         CStatus Transfer(const char* addr, int port,
                          const CUniqueID& object_id)
@@ -312,9 +312,10 @@ cdef class PlasmaClient:
         result = []
         for i in range(object_buffers.size()):
             if object_buffers[i].data_size != -1:
-                result.append(self._make_plasma_buffer(
-                                  object_ids[i], object_buffers[i].data,
-                                  object_buffers[i].data_size))
+                result.append(
+                    self._make_plasma_buffer(object_ids[i],
+                                             object_buffers[i].data,
+                                             object_buffers[i].data_size))
             else:
                 result.append(None)
         return result
@@ -345,9 +346,10 @@ cdef class PlasmaClient:
         self._get_object_buffers(object_ids, timeout_ms, &object_buffers)
         result = []
         for i in range(object_buffers.size()):
-            result.append(self._make_plasma_buffer(
-                              object_ids[i], object_buffers[i].metadata,
-                              object_buffers[i].metadata_size))
+            result.append(
+                self._make_plasma_buffer(object_ids[i],
+                                         object_buffers[i].metadata,
+                                         object_buffers[i].metadata_size))
         return result
 
     def seal(self, ObjectID object_id):
@@ -502,7 +504,7 @@ cdef class PlasmaClient:
                                                 object_requests.data(),
                                                 num_returns, timeout,
                                                 &num_objects_ready))
-        cdef int num_to_return = min(num_objects_ready, num_returns);
+        cdef int num_to_return = min(num_objects_ready, num_returns)
         ready_ids = []
         waiting_ids = set(object_ids)
         cdef int num_returned = 0
@@ -510,7 +512,7 @@ cdef class PlasmaClient:
             if num_returned == num_to_return:
                 break
             if (object_requests[i].status == ObjectStatusLocal or
-                object_requests[i].status == ObjectStatusRemote):
+                    object_requests[i].status == ObjectStatusRemote):
                 ready_ids.append(
                     ObjectID(object_requests[i].object_id.binary()))
                 waiting_ids.discard(
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 28e07ffc37dc3..7e08f632e872e 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -47,7 +47,8 @@ cdef public api bint pyarrow_is_data_type(object type_):
     return isinstance(type_, DataType)
 
 
-cdef public api shared_ptr[CDataType] pyarrow_unwrap_data_type(object data_type):
+cdef public api shared_ptr[CDataType] pyarrow_unwrap_data_type(
+        object data_type):
     cdef DataType type_
     if pyarrow_is_data_type(data_type):
         type_ = <DataType>(data_type)
@@ -57,7 +58,7 @@ cdef public api shared_ptr[CDataType] pyarrow_unwrap_data_type(object data_type)
 
 
 cdef public api object pyarrow_wrap_data_type(
-    const shared_ptr[CDataType]& type):
+        const shared_ptr[CDataType]& type):
     cdef:
         DataType out
 
@@ -149,7 +150,7 @@ cdef public api object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array):
 
 
 cdef public api object pyarrow_wrap_chunked_array(
-    const shared_ptr[CChunkedArray]& sp_array):
+        const shared_ptr[CChunkedArray]& sp_array):
     if sp_array.get() == NULL:
         raise ValueError('ChunkedArray was NULL')
 
@@ -177,7 +178,7 @@ cdef public api shared_ptr[CTensor] pyarrow_unwrap_tensor(object tensor):
 
 
 cdef public api object pyarrow_wrap_tensor(
-    const shared_ptr[CTensor]& sp_tensor):
+        const shared_ptr[CTensor]& sp_tensor):
     if sp_tensor.get() == NULL:
         raise ValueError('Tensor was NULL')
 
@@ -238,7 +239,7 @@ cdef public api shared_ptr[CRecordBatch] pyarrow_unwrap_batch(object batch):
 
 
 cdef public api object pyarrow_wrap_batch(
-    const shared_ptr[CRecordBatch]& cbatch):
+        const shared_ptr[CRecordBatch]& cbatch):
     cdef RecordBatch batch = RecordBatch()
     batch.init(cbatch)
     return batch
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index 1f72070cb7e12..16d2bad0d2d8d 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -154,11 +154,11 @@ cdef class Time32Value(ArrayValue):
             CTime32Type* dtype = <CTime32Type*> ap.type().get()
 
         if dtype.unit() == TimeUnit_SECOND:
-            return (datetime.datetime(1970, 1, 1) +
-                    datetime.timedelta(seconds=ap.Value(self.index))).time()
+            delta = datetime.timedelta(seconds=ap.Value(self.index))
+            return (datetime.datetime(1970, 1, 1) + delta).time()
         else:
-            return (datetime.datetime(1970, 1, 1) +
-                    datetime.timedelta(milliseconds=ap.Value(self.index))).time()
+            delta = datetime.timedelta(milliseconds=ap.Value(self.index))
+            return (datetime.datetime(1970, 1, 1) + delta).time()
 
 
 cdef class Time64Value(ArrayValue):
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 6277761b7d6ec..997b28579f847 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -495,7 +495,6 @@ cdef class RecordBatch:
             entries.append((name, column))
         return OrderedDict(entries)
 
-
     def to_pandas(self, nthreads=None):
         """
         Convert the arrow::RecordBatch to a pandas DataFrame
@@ -585,7 +584,6 @@ def table_to_blocks(Table table, int nthreads):
     return PyObject_to_object(result_obj)
 
 
-
 cdef class Table:
     """
     A collection of top-level named, equal length Arrow arrays.
@@ -897,7 +895,8 @@ cdef class Table:
         """
         Number of rows in this table.
 
-        Due to the definition of a table, all columns have the same number of rows.
+        Due to the definition of a table, all columns have the same number of
+        rows.
 
         Returns
         -------
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index ad2f336061580..592db4f90dac1 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -419,7 +419,7 @@ cdef DataType primitive_type(Type type):
     _type_cache[type] = out
     return out
 
-#------------------------------------------------------------
+# -----------------------------------------------------------
 # Type factory functions
 
 cdef int convert_metadata(dict metadata,

From 7e7861c55108dd257ed45512ec15323e9e7fb583 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Tue, 1 Aug 2017 22:54:57 -0400
Subject: [PATCH 0888/1644] ARROW-1257: Plasma documentation

Thanks a lot to @crystalzyan who did all the heavy lifting for this PR!

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Crystal Yan <crystalzyan@yahoo.com>
Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #881 from pcmoritz/plasma-docs and squashes the following commits:

c4ab47e0 [Robert Nishihara] Remove unsupported shell keyword from plasma.md.
4b987e83 [Robert Nishihara] Fix typo.
21bdc014 [Robert Nishihara] Small changes to python plasma documentation.
4163ccfa [Robert Nishihara] Some changes to plasma.md and add syntax highlighting.
791e5b0b [Philipp Moritz] API changes
80aaf89d [Philipp Moritz] cleanup
c8847204 [Philipp Moritz] more fixes
ba8b0dfa [Philipp Moritz] fix docs
193e00b0 [Philipp Moritz] unify installation instructions
84141b6c [Philipp Moritz] update C++ documentation
9a8437c9 [Philipp Moritz] edit the C++ tutorial (work in progress)
caac4791 [Crystal Yan] Plasma C++ tutorial documentation - minor formatting fixes
5a8433e9 [Crystal Yan] Plasma C++ tutorial documentation - created a tutorial on C++ Plasma for Starting the Object Store, Creating Clients, Creating Objects, Getting Objects, Transferring to Remote Stores, Querying Status, Releasing Objects, and Shutting Down Clients and Stores. Basically all of the PlasmaClient API. Warning- I could not get C++ running on my machine to verify that any of the code runs properly/works. Please verify all code and tutorial content
bc078ff8 [Philipp Moritz] complete installation instructions on macOS
3f3f373b [Philipp Moritz] fix plasma documentation
f51f41e0 [Philipp Moritz] remove old test.py
2be9eab6 [Crystal Yan] Plasma documentation- Added using Pandas with Plasma sections.
a49e1222 [Crystal Yan] Plasma documentation- Added parts on using Arrow with Plasma
25abf830 [Crystal Yan] Plasma documentation- tweaked contents headings hierarchy, added a bit to 'Getting an Object' subsection in Plasma API.
5cf63e92 [Crystal Yan] Plasma documentation- Copied and edited Plasma API section, added a contents header at top, minor tweaks to Linux Installation section. Still need to do Installation on Mac OS and storing Arrow/Panda in Plasma
c02955bf [Crystal Yan] Plasma documentation- initial writeup of installation for linux. Installation for mac incomplete
---
 cpp/apidoc/index.md               |   1 +
 cpp/apidoc/tutorials/plasma.md    | 442 ++++++++++++++++++++++++++++++
 python/doc/source/development.rst |  14 +-
 python/doc/source/index.rst       |   1 +
 python/doc/source/plasma.rst      | 337 +++++++++++++++++++++++
 5 files changed, 791 insertions(+), 4 deletions(-)
 create mode 100644 cpp/apidoc/tutorials/plasma.md
 create mode 100644 python/doc/source/plasma.rst

diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
index 8389d16b4aa1b..ab9bbaa405abc 100644
--- a/cpp/apidoc/index.md
+++ b/cpp/apidoc/index.md
@@ -39,6 +39,7 @@ Table of Contents
  * How to access [HDFS](HDFS.md)
  * Tutorials
    * [Convert a vector of row-wise data into an Arrow table](tutorials/row_wise_conversion.md)
+   * [Using the Plasma In-Memory Object Store](tutorials/plasma.md)
 
 Getting Started
 ---------------
diff --git a/cpp/apidoc/tutorials/plasma.md b/cpp/apidoc/tutorials/plasma.md
new file mode 100644
index 0000000000000..9911546ed5c69
--- /dev/null
+++ b/cpp/apidoc/tutorials/plasma.md
@@ -0,0 +1,442 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+Using the Plasma In-Memory Object Store from C++
+================================================
+
+Apache Arrow offers the ability to share your data structures among multiple
+processes simultaneously through Plasma, an in-memory object store.
+
+Note that **the Plasma API is not stable**.
+
+Plasma clients are processes that run on the same machine as the object store.
+They communicate with the object store over Unix domain sockets, and they read
+and write data in the object store through shared memory.
+
+Plasma objects are immutable once they have been created.
+
+The following goes over the basics so you can begin using Plasma in your big
+data applications.
+
+Starting the Plasma store
+-------------------------
+
+To start running the Plasma object store so that clients may
+connect and access the data, run the following command:
+
+```
+plasma_store -m 1000000000 -s /tmp/plasma
+```
+
+The `-m` flag specifies the size of the object store in bytes. The `-s` flag
+specifies the path of the Unix domain socket that the store will listen at.
+
+Therefore, the above command initializes a Plasma store up to 1 GB of memory
+and sets the socket to `/tmp/plasma.`
+
+The Plasma store will remain available as long as the `plasma_store` process is
+running in a terminal window. Messages, such as alerts for disconnecting
+clients, may occasionally be output. To stop running the Plasma store, you
+can press `Ctrl-C` in the terminal window.
+
+Alternatively, you can run the Plasma store in the background and ignore all
+message output with the following terminal command:
+
+```
+plasma_store -m 1000000000 -s /tmp/plasma 1> /dev/null 2> /dev/null &
+```
+
+The Plasma store will instead run silently in the background. To stop running
+the Plasma store in this case, issue the command below:
+
+```
+killall plasma_store
+```
+
+Creating a Plasma client
+------------------------
+
+Now that the Plasma object store is up and running, it is time to make a client
+process connect to it. To use the Plasma object store as a client, your
+application should initialize a `plasma::PlasmaClient` object and tell it to
+connect to the socket specified when starting up the Plasma object store.
+
+```cpp
+#include <plasma/client.h>
+
+using namespace plasma;
+
+int main(int argc, char** argv) {
+  // Start up and connect a Plasma client.
+  PlasmaClient client;
+  ARROW_CHECK_OK(client.Connect("/tmp/plasma", "", PLASMA_DEFAULT_RELEASE_DELAY));
+  // Disconnect the Plasma client.
+  ARROW_CHECK_OK(client.Disconnect());
+}
+```
+
+Save this program in a file `test.cc` and compile it with
+
+```
+g++ test.cc `pkg-config --cflags --libs plasma` --std=c++11
+```
+
+Note that multiple clients can be created within the same process.
+
+Note that a `PlasmaClient` object is **not thread safe**.
+
+If the Plasma store is still running, you can now execute the `a.out` executable
+and the store will print something like
+
+```
+Disconnecting client on fd 5
+```
+
+which shows that the client was successfully disconnected.
+
+Object IDs
+----------
+
+The Plasma object store uses twenty-byte identifiers for accessing objects
+stored in shared memory. Each object in the Plasma store should be associated
+with a unique ID. The Object ID is then a key that can be used by **any** client
+to fetch that object from the Plasma store.
+
+Random generation of Object IDs is often good enough to ensure unique IDs:
+
+```cpp
+// Randomly generate an Object ID.
+ObjectID object_id = ObjectID::from_random();
+```
+
+Now, any connected client that knows the object's Object ID can access the
+same object from the Plasma object store. For easy transportation of Object IDs,
+you can convert/serialize an Object ID into a binary string and back as
+follows:
+
+```cpp
+// From ObjectID to binary string
+std:string id_string = object_id.binary();
+
+// From binary string to ObjectID
+ObjectID id_object = ObjectID::from_binary(&id_string);
+```
+
+You can also get a human readable representation of ObjectIDs in the same
+format that git uses for commit hashes by running `ObjectID::hex`.
+
+Here is a test program you can run:
+
+```cpp
+#include <iostream>
+#include <string>
+#include <plasma/client.h>
+
+using namespace plasma;
+
+int main(int argc, char** argv) {
+  ObjectID object_id1 = ObjectID::from_random();
+  std::cout << "object_id1 is " << object_id1.hex() << std::endl;
+
+  std::string id_string = object_id1.binary();
+  ObjectID object_id2 = ObjectID::from_binary(id_string);
+  std::cout << "object_id2 is " << object_id2.hex() << std::endl;
+}
+```
+
+Creating an Object
+------------------
+
+Now that you learned about Object IDs that are used to refer to objects,
+let's look at how objects can be stored in Plasma.
+
+Storing objects is a two-stage process. First a buffer is allocated with a call
+to `Create`. Then it can be constructed in place by the client. Then it is made
+immutable and shared with other clients via a call to `Seal`.
+
+The `Create` call blocks while the Plasma store allocates a buffer of the
+appropriate size. The client will then map the buffer into its own address
+space. At this point the object can be constructed in place using a pointer that
+was written by the `Create` command.
+
+```cpp
+int64_t data_size = 100;
+// The address of the buffer allocated by the Plasma store will be written at
+// this address.
+uint8_t* data;
+// Create a Plasma object by specifying its ID and size.
+ARROW_CHECK_OK(client.Create(object_id, data_size, NULL, 0, &data));
+```
+
+You can also specify metadata for the object; the third argument is the
+metadata (as raw bytes) and the fourth argument is the size of the metadata.
+
+```cpp
+// Create a Plasma object without metadata.
+int64_t data_size = 100;
+std::string metadata = "{'author': 'john'}";
+uint8_t* data;
+client.Create(object_id, data_size, (uint8_t*) metadata.data(), metadata.size(), &data);
+```
+
+Now that we've obtained a pointer to our object's data, we can
+write our data to it:
+
+```cpp
+// Write some data for the Plasma object.
+for (int64_t i = 0; i < data_size; i++) {
+    data[i] = static_cast<uint8_t>(i % 4);
+}
+```
+
+When the client is done, the client **seals** the buffer, making the object
+immutable, and making it available to other Plasma clients:
+
+```cpp
+// Seal the object. This makes it available for all clients.
+client.Seal(object_id);
+```
+
+Here is an example that combines all these features:
+
+```cpp
+#include <plasma/client.h>
+
+using namespace plasma;
+
+int main(int argc, char** argv) {
+  // Start up and connect a Plasma client.
+  PlasmaClient client;
+  ARROW_CHECK_OK(client.Connect("/tmp/plasma", "", PLASMA_DEFAULT_RELEASE_DELAY));
+  // Create an object with a random ObjectID.
+  ObjectID object_id = ObjectID::from_binary("00000000000000000000");
+  int64_t data_size = 1000;
+  uint8_t *data;
+  std::string metadata = "{'author': 'john'}";
+  ARROW_CHECK_OK(client.Create(object_id, data_size, (uint8_t*) metadata.data(), metadata.size(), &data));
+  // Write some data into the object.
+  for (int64_t i = 0; i < data_size; i++) {
+    data[i] = static_cast<uint8_t>(i % 4);
+  }
+  // Seal the object.
+  ARROW_CHECK_OK(client.Seal(object_id));
+  // Disconnect the client.
+  ARROW_CHECK_OK(client.Disconnect());
+}
+```
+
+This example can be compiled with
+
+```
+g++ create.cc `pkg-config --cflags --libs plasma` --std=c++11 -o create
+```
+
+To verify that an object exists in the Plasma object store, you can
+call `PlasmaClient::Contains()` to check if an object has
+been created and sealed for a given Object ID. Note that this function
+will still return False if the object has been created, but not yet
+sealed:
+
+```cpp
+// Check if an object has been created and sealed.
+bool has_object;
+client.Contains(object_id, &has_object);
+if (has_object) {
+    // Object has been created and sealed, proceed
+}
+```
+
+Getting an Object
+-----------------
+
+After an object has been sealed, any client who knows the Object ID can get
+the object. To store the retrieved object contents, you should create an
+`ObjectBuffer`, then call `PlasmaClient::Get()` as follows:
+
+```cpp
+// Get from the Plasma store by Object ID.
+ObjectBuffer object_buffer;
+client.Get(&object_id, 1, -1, &object_buffer);
+```
+
+`PlasmaClient::Get()` isn't limited to fetching a single object
+from the Plasma store at once. You can specify an array of Object IDs and
+`ObjectBuffers` to fetch at once, so long as you also specify the
+number of objects being fetched:
+
+```cpp
+// Get two objects at once from the Plasma store. This function
+// call will block until both objects have been fetched.
+ObjectBuffer multiple_buffers[2];
+ObjectID multiple_ids[2] = {object_id1, object_id2};
+client.Get(multiple_ids, 2, -1, multiple_buffers);
+```
+
+Since `PlasmaClient::Get()` is a blocking function call, it may be
+necessary to limit the amount of time the function is allowed to take
+when trying to fetch from the Plasma store. You can pass in a timeout
+in milliseconds when calling `PlasmaClient::Get().` To use `PlasmaClient::Get()`
+without a timeout, just pass in -1 like in the previous example calls:
+
+```cpp
+// Make the function call give up fetching the object if it takes
+// more than 100 milliseconds.
+int64_t timeout = 100;
+client.Get(&object_id, 1, timeout, &object_buffer);
+```
+
+Finally, to access the object, you can access the `data` and
+`metadata` attributes of the `ObjectBuffer`. The `data` can be indexed
+like any array:
+
+```cpp
+// Access object data.
+uint8_t* data = object_buffer.data;
+int64_t data_size = object_buffer.data_size;
+
+// Access object metadata.
+uint8_t* metadata = object_buffer.metadata;
+uint8_t metadata_size = object_buffer.metadata_size;
+
+// Index into data array.
+uint8_t first_data_byte = data[0];
+```
+
+Here is a longer example that shows these capabilities:
+
+```cpp
+#include <plasma/client.h>
+
+using namespace plasma;
+
+int main(int argc, char** argv) {
+  // Start up and connect a Plasma client.
+  PlasmaClient client;
+  ARROW_CHECK_OK(client.Connect("/tmp/plasma", "", PLASMA_DEFAULT_RELEASE_DELAY));
+  ObjectID object_id = ObjectID::from_binary("00000000000000000000");
+  ObjectBuffer object_buffer;
+  ARROW_CHECK_OK(client.Get(&object_id, 1, -1, &object_buffer));
+
+  // Retrieve object data.
+  uint8_t* data = object_buffer.data;
+  int64_t data_size = object_buffer.data_size;
+
+  // Check that the data agrees with what was written in the other process.
+  for (int64_t i = 0; i < data_size; i++) {
+    ARROW_CHECK(data[i] == static_cast<uint8_t>(i % 4));
+  }
+
+  // Disconnect the client.
+  ARROW_CHECK_OK(client.Disconnect());
+}
+```
+
+If you compile it with
+
+```
+g++ get.cc `pkg-config --cflags --libs plasma` --std=c++11 -o get
+```
+
+and run it with `./get`, all the assertions will pass if you run the `create`
+example from above on the same Plasma store.
+
+
+Object Lifetime Management
+--------------------------
+
+The Plasma store internally does reference counting to make sure objects that
+are mapped into the address space of one of the clients with `PlasmaClient::Get`
+are accessible. To unmap objects from a client, call `PlasmaClient::Release`.
+All objects that are mapped into a clients address space will automatically
+be released when the client is disconnected from the store (this happens even
+if the client process crashes or otherwise fails to call `Disconnect`).
+
+If a new object is created and there is not enough space in the Plasma store,
+the store will evict the least recently used object (an object is in use if at
+least one client has gotten it but not released it).
+
+Object notifications
+--------------------
+
+Additionally, you can arrange to have Plasma notify you when objects are
+sealed in the object store. This may especially be handy when your
+program is collaborating with other Plasma clients, and needs to know
+when they make objects available.
+
+First, you can subscribe your current Plasma client to such notifications
+by getting a file descriptor:
+
+```cpp
+// Start receiving notifications into file_descriptor.
+int fd;
+ARROW_CHECK_OK(client.Subscribe(&fd));
+```
+
+Once you have the file descriptor, you can have your current Plasma client
+wait to receive the next object notification. Object notifications
+include information such as Object ID, data size, and metadata size of
+the next newly available object:
+
+```cpp
+// Receive notification of the next newly available object.
+// Notification information is stored in object_id, data_size, and metadata_size
+ObjectID new_object_id;
+int64_t data_size;
+int64_t metadata_size;
+ARROW_CHECK_OK(client.GetNotification(fd, &object_id, &data_size, &metadata_size));
+
+// Get the newly available object.
+ObjectBuffer object_buffer;
+ARROW_CHECK_OK(client.Get(&object_id, 1, -1, &object_buffer));
+```
+
+Here is a full program that shows this capability:
+
+```cpp
+#include <plasma/client.h>
+
+using namespace plasma;
+
+int main(int argc, char** argv) {
+  // Start up and connect a Plasma client.
+  PlasmaClient client;
+  ARROW_CHECK_OK(client.Connect("/tmp/plasma", "", PLASMA_DEFAULT_RELEASE_DELAY));
+
+  int fd;
+  ARROW_CHECK_OK(client.Subscribe(&fd));
+
+  ObjectID object_id;
+  int64_t data_size;
+  int64_t metadata_size;
+  while (true) {
+    ARROW_CHECK_OK(client.GetNotification(fd, &object_id, &data_size, &metadata_size));
+
+    std::cout << "Received object notification for object_id = "
+              << object_id.hex() << ", with data_size = " << data_size
+              << ", and metadata_size = " << metadata_size << std::endl;
+  }
+
+  // Disconnect the client.
+  ARROW_CHECK_OK(client.Disconnect());
+}
+```
+
+If you compile it with
+
+```
+g++ subscribe.cc `pkg-config --cflags --libs plasma` --std=c++11 -o subscribe
+```
+
+and invoke `./create` and `./subscribe` while the Plasma store is running,
+you can observe the new object arriving.
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index d0a1c544dd091..53544ba7a6ac3 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -159,12 +159,16 @@ Now build and install the Arrow C++ libraries:
    cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
          -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
          -DARROW_PYTHON=on \
+         -DARROW_PLASMA=on \
          -DARROW_BUILD_TESTS=OFF \
          ..
    make -j4
    make install
    popd
 
+If you don't want to build and install the Plasma in-memory object store,
+you can omit the ``-DARROW_PLASMA=on`` flag.
+
 Now, optionally build and install the Apache Parquet libraries in your
 toolchain:
 
@@ -190,9 +194,10 @@ Now, build pyarrow:
 
    cd arrow/python
    python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
-          --with-parquet --inplace
+          --with-parquet --with-plasma --inplace
 
-If you did not build parquet-cpp, you can omit ``--with-parquet``.
+If you did not build parquet-cpp, you can omit ``--with-parquet`` and if
+you did not build with plasma, you can omit ``--with-plasma``.
 
 You should be able to run the unit tests with:
 
@@ -224,9 +229,10 @@ You can build a wheel by running:
 .. code-block:: shell
 
    python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
-          --with-parquet --bundle-arrow-cpp bdist_wheel
+          --with-parquet --with-plasma --bundle-arrow-cpp bdist_wheel
 
-Again, if you did not build parquet-cpp, you should omit ``--with-parquet``.
+Again, if you did not build parquet-cpp, you should omit ``--with-parquet`` and
+if you did not build with plasma, you should omit ``--with-plasma``.
 
 Developing on Windows
 =====================
diff --git a/python/doc/source/index.rst b/python/doc/source/index.rst
index a12853c448209..c2ae769b23e83 100644
--- a/python/doc/source/index.rst
+++ b/python/doc/source/index.rst
@@ -40,6 +40,7 @@ structures.
    data
    ipc
    filesystems
+   plasma
    pandas
    parquet
    api
diff --git a/python/doc/source/plasma.rst b/python/doc/source/plasma.rst
new file mode 100644
index 0000000000000..98dd62f97e951
--- /dev/null
+++ b/python/doc/source/plasma.rst
@@ -0,0 +1,337 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+.. _io:
+
+The Plasma In-Memory Object Store
+=================================
+
+.. contents:: Contents
+  :depth: 3
+
+
+The Plasma API
+--------------
+
+Starting the Plasma store
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can start the Plasma store by issuing a terminal command similar to the
+following:
+
+.. code-block:: bash
+
+  plasma_store -m 1000000000 -s /tmp/plasma
+
+The ``-m`` flag specifies the size of the store in bytes, and the ``-s`` flag
+specifies the socket that the store will listen at. Thus, the above command
+allows the Plasma store to use up to 1GB of memory, and sets the socket to
+``/tmp/plasma``.
+
+Leaving the current terminal window open as long as Plasma store should keep
+running. Messages, concerning such as disconnecting clients, may occasionally be
+printed to the screen. To stop running the Plasma store, you can press
+``Ctrl-C`` in the terminal.
+
+Creating a Plasma client
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+To start a Plasma client from Python, call ``plasma.connect`` using the same
+socket name:
+
+.. code-block:: python
+
+  import pyarrow.plasma as plasma
+  client = plasma.connect("/tmp/plasma", "", 0)
+
+If the following error occurs from running the above Python code, that
+means that either the socket given is incorrect, or the ``./plasma_store`` is
+not currently running. Check to see if the Plasma store is still running.
+
+.. code-block:: shell
+
+  >>> client = plasma.connect("/tmp/plasma", "", 0)
+  Connection to socket failed for pathname /tmp/plasma
+  Could not connect to socket /tmp/plasma
+
+
+Object IDs
+^^^^^^^^^^
+
+Each object in the Plasma store should be associated with a unique ID. The
+Object ID then serves as a key that any client can use to retrieve that object
+from the Plasma store. You can form an ``ObjectID`` object from a byte string of
+length 20.
+
+.. code-block:: shell
+
+  # Create an ObjectID.
+  >>> id = plasma.ObjectID(20 * b"a")
+
+  # The character "a" is encoded as 61 in hex.
+  >>> id
+  ObjectID(6161616161616161616161616161616161616161)
+
+The random generation of Object IDs is often good enough to ensure unique IDs.
+You can easily create a helper function that randomly generates object IDs as
+follows:
+
+.. code-block:: python
+
+  import numpy as np
+
+  def random_object_id():
+    return plasma.ObjectID(np.random.bytes(20))
+
+
+Creating an Object
+^^^^^^^^^^^^^^^^^^
+
+Objects are created in Plasma in two stages. First, they are **created**, which
+allocates a buffer for the object. At this point, the client can write to the
+buffer and construct the object within the allocated buffer.
+
+To create an object for Plasma, you need to create an object ID, as well as
+give the object's maximum size in bytes.
+
+.. code-block:: python
+
+  # Create an object.
+  object_id = plasma.ObjectID(20 * b"a")
+  object_size = 1000
+  buffer = memoryview(client.create(object_id, object_size))
+
+  # Write to the buffer.
+  for i in range(1000):
+    buffer[i] = i % 128
+
+When the client is done, the client **seals** the buffer, making the object
+immutable, and making it available to other Plasma clients.
+
+.. code-block:: python
+
+  # Seal the object. This makes the object immutable and available to other clients.
+  client.seal(object_id)
+
+
+Getting an Object
+^^^^^^^^^^^^^^^^^
+
+After an object has been sealed, any client who knows the object ID can get
+the object.
+
+.. code-block:: python
+
+  # Create a different client. Note that this second client could be
+  # created in the same or in a separate, concurrent Python session.
+  client2 = plasma.connect("/tmp/plasma", "", 0)
+
+  # Get the object in the second client. This blocks until the object has been sealed.
+  object_id2 = plasma.ObjectID(20 * b"a")
+  [buffer2] = client2.get([object_id])
+
+If the object has not been sealed yet, then the call to client.get will block
+until the object has been sealed by the client constructing the object. Using
+the ``timeout_ms`` argument to get, you can specify a timeout for this (in
+milliseconds). After the timeout, the interpreter will yield control back.
+
+.. code-block:: shell
+
+  >>> buffer
+  <memory at 0x7fdbdc96e708>
+  >>> buffer[1]
+  1
+  >>> buffer2
+  <plasma.plasma.PlasmaBuffer object at 0x7fdbf2770e88>
+  >>> view2 = memoryview(buffer2)
+  >>> view2[1]
+  1
+  >>> view2[129]
+  1
+  >>> bytes(buffer[1:4])
+  b'\x01\x02\x03'
+  >>> bytes(view2[1:4])
+  b'\x01\x02\x03'
+
+
+Using Arrow and Pandas with Plasma
+----------------------------------
+
+Storing Arrow Objects in Plasma
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To store an Arrow object in Plasma, we must first **create** the object and then
+**seal** it. However, Arrow objects such as ``Tensors`` may be more complicated
+to write than simple binary data.
+
+To create the object in Plasma, you still need an ``ObjectID`` and a size to
+pass in. To find out the size of your Arrow object, you can use pyarrow
+API such as ``pyarrow.get_tensor_size``.
+
+.. code-block:: python
+
+  import numpy as np
+  import pyarrow as pa
+
+  # Create a pyarrow.Tensor object from a numpy random 2-dimensional array
+  data = np.random.randn(10, 4)
+  tensor = pa.Tensor.from_numpy(data)
+
+  # Create the object in Plasma
+  object_id = plasma.ObjectID(np.random.bytes(20))
+  data_size = pa.get_tensor_size(tensor)
+  buf = client.create(object_id, data_size)
+
+To write the Arrow ``Tensor`` object into the buffer, you can use Plasma to
+convert the ``memoryview`` buffer into a ``pyarrow.FixedSizeBufferOutputStream``
+object. A ``pyarrow.FixedSizeBufferOutputStream`` is a format suitable for Arrow's
+``pyarrow.write_tensor``:
+
+.. code-block:: python
+
+  # Write the tensor into the Plasma-allocated buffer
+  stream = pa.FixedSizeBufferOutputStream(buf)
+  pa.write_tensor(tensor, stream)  # Writes tensor's 552 bytes to Plasma stream
+
+To finish storing the Arrow object in Plasma, call ``seal``:
+
+.. code-block:: python
+
+  # Seal the Plasma object
+  client.seal(object_id)
+
+Getting Arrow Objects from Plasma
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To read the object, first retrieve it as a ``PlasmaBuffer`` using its object ID.
+
+.. code-block:: python
+
+  # Get the arrow object by ObjectID.
+  [buf2] = client.get([object_id])
+
+To convert the ``PlasmaBuffer`` back into an Arrow ``Tensor``, first create a
+pyarrow ``BufferReader`` object from it. You can then pass the ``BufferReader``
+into ``pyarrow.read_tensor`` to reconstruct the Arrow ``Tensor`` object:
+
+.. code-block:: python
+
+  # Reconstruct the Arrow tensor object.
+  reader = pa.BufferReader(buf2)
+  tensor2 = pa.read_tensor(reader)
+
+Finally, you can use ``pyarrow.read_tensor`` to convert the Arrow object
+back into numpy data:
+
+.. code-block:: python
+
+  # Convert back to numpy
+  array = tensor2.to_numpy()
+
+Storing Pandas DataFrames in Plasma
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Storing a Pandas ``DataFrame`` still follows the **create** then **seal**
+process of storing an object in the Plasma store, however one cannot directly
+write the ``DataFrame`` to Plasma with Pandas alone. Plasma also needs to know
+the size of the ``DataFrame`` to allocate a buffer for.
+
+See :ref:`pandas` for more information on using Arrow with Pandas.
+
+You can create the pyarrow equivalent of a Pandas ``DataFrame`` by using
+``pyarrow.from_pandas`` to convert it to a ``RecordBatch``.
+
+.. code-block:: python
+
+  import pyarrow as pa
+  import pandas as pd
+
+  # Create a Pandas DataFrame
+  d = {'one' : pd.Series([1., 2., 3.], index=['a', 'b', 'c']),
+       'two' : pd.Series([1., 2., 3., 4.], index=['a', 'b', 'c', 'd'])}
+  df = pd.DataFrame(d)
+
+  # Convert the Pandas DataFrame into a PyArrow RecordBatch
+  record_batch = pa.RecordBatch.from_pandas(df)
+
+Creating the Plasma object requires an ``ObjectID`` and the size of the
+data. Now that we have converted the Pandas ``DataFrame`` into a PyArrow
+``RecordBatch``, use the ``MockOutputStream`` to determine the
+size of the Plasma object.
+
+.. code-block:: python
+
+  # Create the Plasma object from the PyArrow RecordBatch. Most of the work here
+  # is done to determine the size of buffer to request from the object store.
+  object_id = plasma.ObjectID(np.random.bytes(20))
+  mock_sink = pa.MockOutputStream()
+  stream_writer = pa.RecordBatchStreamWriter(mock_sink, record_batch.schema)
+  stream_writer.write_batch(record_batch)
+  stream_writer.close()
+  data_size = mock_sink.size()
+  buf = client.create(object_id, data_size)
+
+The DataFrame can now be written to the buffer as follows.
+
+.. code-block:: python
+
+  # Write the PyArrow RecordBatch to Plasma
+  stream = pa.FixedSizeBufferOutputStream(buf)
+  stream_writer = pa.RecordBatchStreamWriter(stream, record_batch.schema)
+  stream_writer.write_batch(record_batch)
+  stream_writer.close()
+
+Finally, seal the finished object for use by all clients:
+
+.. code-block:: python
+
+  # Seal the Plasma object
+  client.seal(object_id)
+
+Getting Pandas DataFrames from Plasma
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Since we store the Pandas DataFrame as a PyArrow ``RecordBatch`` object,
+to get the object back from the Plasma store, we follow similar steps
+to those specified in `Getting Arrow Objects from Plasma`_.
+
+We first have to convert the ``PlasmaBuffer`` returned from ``client.get``
+into an Arrow ``BufferReader`` object.
+
+.. code-block:: python
+
+  # Fetch the Plasma object
+  [data] = client.get([object_id])  # Get PlasmaBuffer from ObjectID
+  buffer = pa.BufferReader(data)
+
+From the ``BufferReader``, we can create a specific ``RecordBatchStreamReader``
+in Arrow to reconstruct the stored PyArrow ``RecordBatch`` object.
+
+.. code-block:: python
+
+  # Convert object back into an Arrow RecordBatch
+  reader = pa.RecordBatchStreamReader(buffer)
+  record_batch = reader.read_next_batch()
+
+The last step is to convert the PyArrow ``RecordBatch`` object back into
+the original Pandas ``DataFrame`` structure.
+
+.. code-block:: python
+
+  # Convert back into Pandas
+  result = record_batch.to_pandas()

From e50b6ae5c8044ac425db8609fb837fe7c95bd393 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 1 Aug 2017 23:00:37 -0400
Subject: [PATCH 0889/1644] ARROW-1308: [C++] Link utility executables to Arrow
 shared library if ARROW_BUILD_STATIC=off

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #931 from wesm/ARROW-1308 and squashes the following commits:

88391fe0 [Wes McKinney] Link utility executables to Arrow shared library if ARROW_BUILD_STATIC=off
---
 cpp/src/arrow/ipc/CMakeLists.txt | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 9cc61bced0619..76e52a0f78b9a 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -90,18 +90,22 @@ install(FILES
   writer.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/ipc")
 
-if(MSVC)
-  set(UTIL_LINK_LIBS
-    arrow_static
-    ${BOOST_FILESYSTEM_LIBRARY}
-    ${BOOST_SYSTEM_LIBRARY})
+if (ARROW_BUILD_STATIC)
+  set(ARROW_UTIL_LIB arrow_static)
 else()
+  set(ARROW_UTIL_LIB arrow_shared)
+endif()
+
+set(UTIL_LINK_LIBS
+  ${ARROW_UTIL_LIB}
+  ${BOOST_FILESYSTEM_LIBRARY}
+  ${BOOST_SYSTEM_LIBRARY})
+
+if(NOT MSVC)
   set(UTIL_LINK_LIBS
-    arrow_static
+    ${UTIL_LINK_LIBS}
     pthread
-    ${BOOST_FILESYSTEM_LIBRARY}
-    ${BOOST_SYSTEM_LIBRARY}
-    dl)
+    ${CMAKE_DL_LIBS})
 endif()
 
 if (ARROW_BUILD_UTILITIES)

From b95bed050c6d5d0943fd7866cfdade7fd9cc4904 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 2 Aug 2017 11:44:40 -0400
Subject: [PATCH 0890/1644] ARROW-1303: [C++] Support downloading Boost

CentOS 6 ships old Boost. If we support downloading Boost, we'll build
RPM for CentOS 6 easily.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #927 from kou/cpp-boost-download and squashes the following commits:

5bf6818 [Kouhei Sutou] [C++] Always use static link for vendored Boost
832673d [Kouhei Sutou] [C++] Add ARROW_BOOST_VENDORED option
290e6e1 [Kouhei Sutou] [C++] Support downloading Boost
---
 cpp/CMakeLists.txt                          |   4 +
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 118 ++++++++++++++------
 2 files changed, 88 insertions(+), 34 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 07b8e15b504e4..eeff9124ad983 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -114,6 +114,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Rely on boost shared libraries where relevant"
     ON)
 
+  option(ARROW_BOOST_VENDORED
+    "Use vendored Boost instead of existing Boost"
+    OFF)
+
   option(ARROW_PYTHON
     "Build the Arrow CPython extensions"
     OFF)
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 1271b8a4ab3f4..ae48e8d2fb979 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -124,50 +124,100 @@ set(Boost_ADDITIONAL_VERSIONS
   "1.62.0" "1.61"
   "1.61.0" "1.62"
   "1.60.0" "1.60")
-
-if (ARROW_BOOST_USE_SHARED)
-  # Find shared Boost libraries.
-  set(Boost_USE_STATIC_LIBS OFF)
-
-  if(MSVC)
-    # disable autolinking in boost
-    add_definitions(-DBOOST_ALL_NO_LIB)
-
-    # force all boost libraries to dynamic link
-    add_definitions(-DBOOST_ALL_DYN_LINK)
-  endif()
-
+list(GET Boost_ADDITIONAL_VERSIONS 0 BOOST_LATEST_VERSION)
+string(REPLACE "." "_" BOOST_LATEST_VERSION_IN_PATH ${BOOST_LATEST_VERSION})
+set(BOOST_LATEST_URL
+  "https://dl.bintray.com/boostorg/release/${BOOST_LATEST_VERSION}/source/boost_${BOOST_LATEST_VERSION_IN_PATH}.tar.gz")
+
+if (ARROW_BOOST_VENDORED)
+  set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep")
+  set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib")
+  set(BOOST_BUILD_LINK "static")
+  set(BOOST_STATIC_SYSTEM_LIBRARY
+    "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(BOOST_STATIC_FILESYSTEM_LIBRARY
+    "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
+  set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
   if (ARROW_BOOST_HEADER_ONLY)
-    find_package(Boost)
+    set(BOOST_BUILD_PRODUCTS)
+    set(BOOST_CONFIGURE_COMMAND "")
+    set(BOOST_BUILD_COMMAND "")
   else()
-    find_package(Boost COMPONENTS system filesystem REQUIRED)
+    set(BOOST_BUILD_PRODUCTS
+      ${BOOST_SYSTEM_LIBRARY}
+      ${BOOST_FILESYSTEM_LIBRARY})
+    set(BOOST_CONFIGURE_COMMAND
+      "./bootstrap.sh"
+      "--prefix=${BOOST_PREFIX}"
+      "--with-libraries=filesystem,system")
     if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-      set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-      set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+      set(BOOST_BUILD_VARIANT "debug")
     else()
-      set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-      set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+      set(BOOST_BUILD_VARIANT "release")
     endif()
-    set(BOOST_SYSTEM_LIBRARY boost_system_shared)
-    set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
+    set(BOOST_BUILD_COMMAND
+      "./b2"
+      "link=${BOOST_BUILD_LINK}"
+      "variant=${BOOST_BUILD_VARIANT}"
+      "cxxflags=-fPIC")
   endif()
+  ExternalProject_Add(boost_ep
+    URL ${BOOST_LATEST_URL}
+    BUILD_BYPRODUCTS ${BOOST_BUILD_PRODUCTS}
+    BUILD_IN_SOURCE 1
+    CONFIGURE_COMMAND ${BOOST_CONFIGURE_COMMAND}
+    BUILD_COMMAND ${BOOST_BUILD_COMMAND}
+    INSTALL_COMMAND ""
+    ${EP_LOG_OPTIONS})
+  set(Boost_INCLUDE_DIR "${BOOST_PREFIX}")
+  set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}")
+  add_dependencies(arrow_dependencies boost_ep)
 else()
-  # Find static boost headers and libs
-  # TODO Differentiate here between release and debug builds
-  set(Boost_USE_STATIC_LIBS ON)
-  if (ARROW_BOOST_HEADER_ONLY)
-    find_package(Boost)
+  if (ARROW_BOOST_USE_SHARED)
+    # Find shared Boost libraries.
+    set(Boost_USE_STATIC_LIBS OFF)
+
+    if(MSVC)
+      # disable autolinking in boost
+      add_definitions(-DBOOST_ALL_NO_LIB)
+
+      # force all boost libraries to dynamic link
+      add_definitions(-DBOOST_ALL_DYN_LINK)
+    endif()
+
+    if (ARROW_BOOST_HEADER_ONLY)
+      find_package(Boost REQUIRED)
+    else()
+      find_package(Boost COMPONENTS system filesystem REQUIRED)
+      if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+	set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+	set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+      else()
+	set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+	set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+      endif()
+      set(BOOST_SYSTEM_LIBRARY boost_system_shared)
+      set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
+    endif()
   else()
-    find_package(Boost COMPONENTS system filesystem REQUIRED)
-    if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-      set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-      set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+    # Find static boost headers and libs
+    # TODO Differentiate here between release and debug builds
+    set(Boost_USE_STATIC_LIBS ON)
+    if (ARROW_BOOST_HEADER_ONLY)
+      find_package(Boost REQUIRED)
     else()
-      set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-      set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+      find_package(Boost COMPONENTS system filesystem REQUIRED)
+      if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+	set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+	set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+      else()
+	set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+	set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+      endif()
+      set(BOOST_SYSTEM_LIBRARY boost_system_static)
+      set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
     endif()
-    set(BOOST_SYSTEM_LIBRARY boost_system_static)
-    set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
   endif()
 endif()
 

From 5917e07c730ed8ac70fdc5b4872795de539ae9ba Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 2 Aug 2017 11:47:00 -0400
Subject: [PATCH 0891/1644] ARROW-1305: [GLib] Add GArrowIntArrayBuilder

Author: Kouhei Sutou <kou@clear-code.com>

Closes #928 from kou/glib-add-int-array-builder and squashes the following commits:

3419edf [Kouhei Sutou] [GLib] Make buildable with Clang
c5112d5 [Kouhei Sutou] [GLib] Add GArrowIntArrayBuilder
---
 c_glib/arrow-glib/array-builder.cpp   | 486 ++++++++++++++------------
 c_glib/arrow-glib/array-builder.h     |  51 +++
 c_glib/arrow-glib/array-builder.hpp   |   3 +-
 c_glib/test/helper/buildable.rb       |   4 +
 c_glib/test/test-int-array-builder.rb |  59 ++++
 5 files changed, 383 insertions(+), 220 deletions(-)
 create mode 100644 c_glib/test/test-int-array-builder.rb

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 17b2ec8b55069..23bc842756e65 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -25,6 +25,31 @@
 #include <arrow-glib/data-type.hpp>
 #include <arrow-glib/error.hpp>
 
+template <typename BUILDER, typename VALUE>
+gboolean
+garrow_array_builder_append(GArrowArrayBuilder *builder,
+                            VALUE value,
+                            GError **error,
+                            const gchar *context)
+{
+  auto arrow_builder =
+    static_cast<BUILDER>(garrow_array_builder_get_raw(builder));
+  auto status = arrow_builder->Append(value);
+  return garrow_error_check(error, status, context);
+}
+
+template <typename BUILDER>
+gboolean
+garrow_array_builder_append_null(GArrowArrayBuilder *builder,
+                                 GError **error,
+                                 const gchar *context)
+{
+  auto arrow_builder =
+    static_cast<BUILDER>(garrow_array_builder_get_raw(builder));
+  auto status = arrow_builder->AppendNull();
+  return garrow_error_check(error, status, context);
+}
+
 G_BEGIN_DECLS
 
 /**
@@ -41,6 +66,11 @@ G_BEGIN_DECLS
  * #GArrowBooleanArrayBuilder is the class to create a new
  * #GArrowBooleanArray.
  *
+ * #GArrowIntArrayBuilder is the class to create a new integer
+ * array. Integer size is automatically chosen. It's recommend that
+ * you use this builder instead of specific integer size builder such
+ * as #GArrowInt8ArrayBuilder.
+ *
  * #GArrowInt8ArrayBuilder is the class to create a new
  * #GArrowInt8Array.
  *
@@ -244,12 +274,11 @@ garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
                                     gboolean value,
                                     GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::BooleanBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(static_cast<bool>(value));
-  return garrow_error_check(error, status, "[boolean-array-builder][append]");
+  return garrow_array_builder_append<arrow::BooleanBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     static_cast<bool>(value),
+     error,
+     "[boolean-array-builder][append]");
 }
 
 /**
@@ -263,14 +292,83 @@ gboolean
 garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
                                          GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::BooleanBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
+  return garrow_array_builder_append_null<arrow::BooleanBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[boolean-array-builder][append-null]");
+}
 
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error,
-                            status,
-                            "[boolean-array-builder][append-null]");
+
+G_DEFINE_TYPE(GArrowIntArrayBuilder,
+              garrow_int_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int_array_builder_init(GArrowIntArrayBuilder *builder)
+{
+}
+
+static void
+garrow_int_array_builder_class_init(GArrowIntArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_int_array_builder_new:
+ *
+ * Returns: A newly created #GArrowIntArrayBuilder.
+ *
+ * Since: 0.6.0
+ */
+GArrowIntArrayBuilder *
+garrow_int_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder = new arrow::AdaptiveIntBuilder(memory_pool);
+  auto builder = garrow_array_builder_new_raw(arrow_builder,
+                                              GARROW_TYPE_INT_ARRAY_BUILDER);
+  return GARROW_INT_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_int_array_builder_append:
+ * @builder: A #GArrowIntArrayBuilder.
+ * @value: A int value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.6.0
+ */
+gboolean
+garrow_int_array_builder_append(GArrowIntArrayBuilder *builder,
+                                gint64 value,
+                                GError **error)
+{
+  return garrow_array_builder_append<arrow::AdaptiveIntBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[int-array-builder][append]");
+}
+
+/**
+ * garrow_int_array_builder_append_null:
+ * @builder: A #GArrowIntArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.6.0
+ */
+gboolean
+garrow_int_array_builder_append_null(GArrowIntArrayBuilder *builder,
+                                     GError **error)
+{
+  return garrow_array_builder_append_null<arrow::AdaptiveIntBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[int-array-builder][append-null]");
 }
 
 
@@ -315,12 +413,11 @@ garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
                                  gint8 value,
                                  GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::Int8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[int8-array-builder][append]");
+  return garrow_array_builder_append<arrow::Int8Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[int8-array-builder][append]");
 }
 
 /**
@@ -334,12 +431,10 @@ gboolean
 garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
                                       GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::Int8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error, status, "[int8-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::Int8Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[int8-array-builder][append-null]");
 }
 
 
@@ -384,12 +479,11 @@ garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
                                   guint8 value,
                                   GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::UInt8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[uint8-array-builder][append]");
+  return garrow_array_builder_append<arrow::UInt8Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[uint8-array-builder][append]");
 }
 
 /**
@@ -403,12 +497,10 @@ gboolean
 garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
                                        GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::UInt8Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error, status, "[uint8-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::UInt8Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[uint8-array-builder][append-null]");
 }
 
 
@@ -453,12 +545,11 @@ garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
                                   gint16 value,
                                   GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::Int16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[int16-array-builder][append]");
+  return garrow_array_builder_append<arrow::Int16Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[int16-array-builder][append]");
 }
 
 /**
@@ -472,12 +563,10 @@ gboolean
 garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
                                        GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::Int16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error, status, "[int16-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::Int16Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[int16-array-builder][append-null]");
 }
 
 
@@ -522,12 +611,11 @@ garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
                                    guint16 value,
                                    GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::UInt16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[uint16-array-builder][append]");
+  return garrow_array_builder_append<arrow::UInt16Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[uint16-array-builder][append]");
 }
 
 /**
@@ -541,14 +629,10 @@ gboolean
 garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
                                         GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::UInt16Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error,
-                            status,
-                            "[uint16-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::UInt16Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[uint16-array-builder][append-null]");
 }
 
 
@@ -593,12 +677,11 @@ garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
                                   gint32 value,
                                   GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::Int32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[int32-array-builder][append]");
+  return garrow_array_builder_append<arrow::Int32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[int32-array-builder][append]");
 }
 
 /**
@@ -612,12 +695,10 @@ gboolean
 garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
                                       GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::Int32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error, status, "[int32-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::Int32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[int32-array-builder][append-null]");
 }
 
 
@@ -662,12 +743,11 @@ garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
                                    guint32 value,
                                    GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::UInt32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[uint32-array-builder][append]");
+  return garrow_array_builder_append<arrow::UInt32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[uint32-array-builder][append]");
 }
 
 /**
@@ -681,14 +761,10 @@ gboolean
 garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
                                         GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::UInt32Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error,
-                            status,
-                            "[uint32-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::UInt32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[uint32-array-builder][append-null]");
 }
 
 
@@ -733,12 +809,11 @@ garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
                                   gint64 value,
                                   GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::Int64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[int64-array-builder][append]");
+  return garrow_array_builder_append<arrow::Int64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[int64-array-builder][append]");
 }
 
 /**
@@ -752,12 +827,10 @@ gboolean
 garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
                                        GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::Int64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error, status, "[int64-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::Int64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[int64-array-builder][append-null]");
 }
 
 
@@ -802,12 +875,11 @@ garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
                                   guint64 value,
                                   GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::UInt64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[uint64-array-builder][append]");
+  return garrow_array_builder_append<arrow::UInt64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[uint64-array-builder][append]");
 }
 
 /**
@@ -821,17 +893,10 @@ gboolean
 garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
                                        GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::UInt64Builder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  if (status.ok()) {
-    return TRUE;
-  } else {
-    garrow_error_check(error, status, "[uint64-array-builder][append-null]");
-    return FALSE;
-  }
+  return garrow_array_builder_append_null<arrow::UInt64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[uint64-array-builder][append-null]");
 }
 
 
@@ -876,12 +941,11 @@ garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
                                   gfloat value,
                                   GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::FloatBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[float-array-builder][append]");
+  return garrow_array_builder_append<arrow::FloatBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[float-array-builder][append]");
 }
 
 /**
@@ -895,12 +959,10 @@ gboolean
 garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
                                        GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::FloatBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error, status, "[float-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::FloatBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[float-array-builder][append-null]");
 }
 
 
@@ -945,12 +1007,11 @@ garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
                                    gdouble value,
                                    GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::DoubleBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->Append(value);
-  return garrow_error_check(error, status, "[double-array-builder][append]");
+  return garrow_array_builder_append<arrow::DoubleBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[double-array-builder][append]");
 }
 
 /**
@@ -964,14 +1025,10 @@ gboolean
 garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
                                         GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::DoubleBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error,
-                            status,
-                            "[double-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::DoubleBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[double-array-builder][append-null]");
 }
 
 
@@ -1037,14 +1094,10 @@ gboolean
 garrow_binary_array_builder_append_null(GArrowBinaryArrayBuilder *builder,
                                         GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::BinaryBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error,
-                            status,
-                            "[binary-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::BinaryBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[binary-array-builder][append-null]");
 }
 
 
@@ -1240,12 +1293,10 @@ gboolean
 garrow_list_array_builder_append_null(GArrowListArrayBuilder *builder,
                                       GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::ListBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error, status, "[list-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::ListBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[list-array-builder][append-null]");
 }
 
 /**
@@ -1390,14 +1441,10 @@ gboolean
 garrow_struct_array_builder_append_null(GArrowStructArrayBuilder *builder,
                                         GError **error)
 {
-  auto arrow_builder =
-    static_cast<arrow::StructBuilder *>(
-      garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-
-  auto status = arrow_builder->AppendNull();
-  return garrow_error_check(error,
-                            status,
-                            "[struct-array-builder][append-null]");
+  return garrow_array_builder_append_null<arrow::StructBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[struct-array-builder][append-null]");
 }
 
 /**
@@ -1450,59 +1497,60 @@ garrow_struct_array_builder_get_field_builders(GArrowStructArrayBuilder *builder
 G_END_DECLS
 
 GArrowArrayBuilder *
-garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder)
-{
-  GType type;
-
-  switch (arrow_builder->type()->id()) {
-  case arrow::Type::type::BOOL:
-    type = GARROW_TYPE_BOOLEAN_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::UINT8:
-    type = GARROW_TYPE_UINT8_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::INT8:
-    type = GARROW_TYPE_INT8_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::UINT16:
-    type = GARROW_TYPE_UINT16_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::INT16:
-    type = GARROW_TYPE_INT16_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::UINT32:
-    type = GARROW_TYPE_UINT32_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::INT32:
-    type = GARROW_TYPE_INT32_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::UINT64:
-    type = GARROW_TYPE_UINT64_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::INT64:
-    type = GARROW_TYPE_INT64_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::FLOAT:
-    type = GARROW_TYPE_FLOAT_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::DOUBLE:
-    type = GARROW_TYPE_DOUBLE_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::BINARY:
-    type = GARROW_TYPE_BINARY_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::STRING:
-    type = GARROW_TYPE_STRING_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::LIST:
-    type = GARROW_TYPE_LIST_ARRAY_BUILDER;
-    break;
-  case arrow::Type::type::STRUCT:
-    type = GARROW_TYPE_STRUCT_ARRAY_BUILDER;
-    break;
-  default:
-    type = GARROW_TYPE_ARRAY_BUILDER;
-    break;
+garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder,
+                             GType type)
+{
+  if (type == G_TYPE_INVALID) {
+    switch (arrow_builder->type()->id()) {
+    case arrow::Type::type::BOOL:
+      type = GARROW_TYPE_BOOLEAN_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::UINT8:
+      type = GARROW_TYPE_UINT8_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::INT8:
+      type = GARROW_TYPE_INT8_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::UINT16:
+      type = GARROW_TYPE_UINT16_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::INT16:
+      type = GARROW_TYPE_INT16_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::UINT32:
+      type = GARROW_TYPE_UINT32_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::INT32:
+      type = GARROW_TYPE_INT32_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::UINT64:
+      type = GARROW_TYPE_UINT64_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::INT64:
+      type = GARROW_TYPE_INT64_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::FLOAT:
+      type = GARROW_TYPE_FLOAT_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::DOUBLE:
+      type = GARROW_TYPE_DOUBLE_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::BINARY:
+      type = GARROW_TYPE_BINARY_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::STRING:
+      type = GARROW_TYPE_STRING_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::LIST:
+      type = GARROW_TYPE_LIST_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::STRUCT:
+      type = GARROW_TYPE_STRUCT_ARRAY_BUILDER;
+      break;
+    default:
+      type = GARROW_TYPE_ARRAY_BUILDER;
+      break;
+    }
   }
 
   auto builder =
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index f5a8ac73d630a..97cea63078b12 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -119,6 +119,57 @@ gboolean garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *bui
                                                   GError **error);
 
 
+#define GARROW_TYPE_INT_ARRAY_BUILDER           \
+  (garrow_int_array_builder_get_type())
+#define GARROW_INT_ARRAY_BUILDER(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT_ARRAY_BUILDER,    \
+                              GArrowIntArrayBuilder))
+#define GARROW_INT_ARRAY_BUILDER_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_INT_ARRAY_BUILDER,       \
+                           GArrowIntArrayBuilderClass))
+#define GARROW_IS_INT_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT_ARRAY_BUILDER))
+#define GARROW_IS_INT_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT_ARRAY_BUILDER))
+#define GARROW_INT_ARRAY_BUILDER_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT_ARRAY_BUILDER,     \
+                             GArrowIntArrayBuilderClass))
+
+typedef struct _GArrowIntArrayBuilder         GArrowIntArrayBuilder;
+typedef struct _GArrowIntArrayBuilderClass    GArrowIntArrayBuilderClass;
+
+/**
+ * GArrowIntArrayBuilder:
+ *
+ * It wraps `arrow::AdaptiveIntBuilder`.
+ */
+struct _GArrowIntArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowIntArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_int_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowIntArrayBuilder *garrow_int_array_builder_new(void);
+
+gboolean garrow_int_array_builder_append(GArrowIntArrayBuilder *builder,
+                                         gint64 value,
+                                         GError **error);
+gboolean garrow_int_array_builder_append_null(GArrowIntArrayBuilder *builder,
+                                              GError **error);
+
+
 #define GARROW_TYPE_INT8_ARRAY_BUILDER          \
   (garrow_int8_array_builder_get_type())
 #define GARROW_INT8_ARRAY_BUILDER(obj)                          \
diff --git a/c_glib/arrow-glib/array-builder.hpp b/c_glib/arrow-glib/array-builder.hpp
index e65ad005c12fd..bcdc58fd8844b 100644
--- a/c_glib/arrow-glib/array-builder.hpp
+++ b/c_glib/arrow-glib/array-builder.hpp
@@ -22,5 +22,6 @@
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/array-builder.h>
 
-GArrowArrayBuilder *garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder);
+GArrowArrayBuilder *garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder,
+                                                 GType type=G_TYPE_INVALID);
 arrow::ArrayBuilder *garrow_array_builder_get_raw(GArrowArrayBuilder *builder);
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index f1bac47d6c7b9..3181c098c002b 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -21,6 +21,10 @@ def build_boolean_array(values)
       build_array(Arrow::BooleanArrayBuilder, values)
     end
 
+    def build_int_array(values)
+      build_array(Arrow::IntArrayBuilder, values)
+    end
+
     def build_int8_array(values)
       build_array(Arrow::Int8ArrayBuilder, values)
     end
diff --git a/c_glib/test/test-int-array-builder.rb b/c_glib/test/test-int-array-builder.rb
new file mode 100644
index 0000000000000..e1a6c3b216597
--- /dev/null
+++ b/c_glib/test/test-int-array-builder.rb
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestIntArrayBuilder < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_int8
+    values = [-1, 2]
+    assert_equal(build_int_array([*values, nil]),
+                 Arrow::Int8Array.new(3,
+                                      Arrow::Buffer.new(values.pack("c*")),
+                                      Arrow::Buffer.new([0b011].pack("C*")),
+                                      -1))
+  end
+
+  def test_int16
+    border_value = (2 ** (8 - 1))
+    values = [-1, border_value]
+    assert_equal(build_int_array([*values, nil]),
+                 Arrow::Int16Array.new(3,
+                                       Arrow::Buffer.new(values.pack("s*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
+  def test_int32
+    border_value = (2 ** (16 - 1))
+    values = [-1, border_value]
+    assert_equal(build_int_array([*values, nil]),
+                 Arrow::Int32Array.new(3,
+                                       Arrow::Buffer.new(values.pack("l*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
+  def test_int64
+    border_value = (2 ** (32 - 1))
+    values = [-1, border_value]
+    assert_equal(build_int_array([*values, nil]),
+                 Arrow::Int64Array.new(3,
+                                       Arrow::Buffer.new(values.pack("q*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+end

From ee928d2233da89ebd1f567ffda4833f4f07e795c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 2 Aug 2017 11:48:55 -0400
Subject: [PATCH 0892/1644] ARROW-1211: [C++] Enable builder classes to
 automatically use the default memory pool

I added the `ARROW_NO_DEFAULT_MEMORY_POOL` define option to disable this in third party use.

I also flipped the order of arguments to the builder constructors to be a bit more natural. I don't feel strongly about this, but it does make the code a bit nicer:

```c++
FixedSizeBinaryBuilder builder(type);
FixedSizeBinaryBuilder builder(type, pool);
```

versus

```c++
FixedSizeBinaryBuilder builder(type);
FixedSizeBinaryBuilder builder(pool, type);
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #923 from wesm/ARROW-1211 and squashes the following commits:

e8a129d [Wes McKinney] Use CONFIGURE_OPTIONS as advised
7835e67 [Wes McKinney] Build c_glib in Travis CI without deprecated Arrow APIs
d86a6a3 [Wes McKinney] Remove copy-paste error, unneeded using statement
a712445 [Wes McKinney] Benchmark fixes
1488bb4 [Wes McKinney] Some MSVC fixes, cannot get DictionaryBuilder default ctor working there
33cd7d5 [Wes McKinney] Use macro for memory pool argument to avoid code duplication. Add macro expansion to Doxyfile
085ca86 [Wes McKinney] Add option builder constructors to use default memory pool without passing explicitly. Add ARROW_NO_DEFAULT_MEMORY_POOL define. Flip builder constructor order, deprecate old constructors. README documentation
---
 c_glib/configure.ac                           |  1 +
 ci/msvc-build.bat                             |  1 -
 ci/travis_before_script_c_glib.sh             |  4 +
 cpp/README.md                                 | 14 ++++
 cpp/apidoc/Doxyfile                           |  2 +-
 cpp/src/arrow/array-decimal-test.cc           |  2 +-
 cpp/src/arrow/array-test.cc                   | 48 +++++------
 cpp/src/arrow/builder-benchmark.cc            | 10 +--
 cpp/src/arrow/builder.cc                      | 81 +++++++++++++------
 cpp/src/arrow/builder.h                       | 79 ++++++++++++++----
 cpp/src/arrow/ipc/ipc-json-test.cc            |  2 +-
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc |  3 +-
 cpp/src/arrow/ipc/json-internal.cc            |  4 +-
 cpp/src/arrow/ipc/test-common.h               |  6 +-
 cpp/src/arrow/pretty_print-test.cc            |  2 +-
 cpp/src/arrow/python/pandas_to_arrow.cc       |  8 +-
 cpp/src/arrow/python/python-test.cc           |  2 +-
 cpp/src/arrow/test-util.h                     |  9 +--
 18 files changed, 189 insertions(+), 89 deletions(-)

diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index d4e828ba55c1b..375f76efcdd51 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -79,6 +79,7 @@ else
   ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}"
 
   ARROW_CFLAGS="-I${ARROW_INCLUDE_DIR}"
+
   ARROW_LIBS="-L${ARROW_LIB_DIR} -larrow"
 
   AC_SUBST(ARROW_LIB_DIR)
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 04fe2ab62cbd4..6ebd22fc3e354 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -104,7 +104,6 @@ cmake -G "%GENERATOR%" ^
      -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
      -DPARQUET_BOOST_USE_SHARED=OFF ^
-     -DPARQUET_ZLIB_VENDORED=off ^
      -DPARQUET_BUILD_TESTS=off .. || exit /B
 cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
 popd
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index bf2d385d79d4d..7ab8e2eaa03ee 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -74,6 +74,10 @@ CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"
 if [ $TRAVIS_OS_NAME != "osx" ]; then
   CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"
 fi
+
+CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CFLAGS=-DARROW_NO_DEPRECATED_API"
+CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API"
+
 ./configure $CONFIGURE_OPTIONS
 
 make -j4
diff --git a/cpp/README.md b/cpp/README.md
index 2f98b085115f5..0228faf7349c5 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -126,6 +126,14 @@ This project follows [Google's C++ Style Guide][3] with minor exceptions. We do
 not encourage anonymous namespaces and we relax the line length restriction to
 90 characters.
 
+### Memory Pools
+
+We provide a default memory pool with `arrow::default_memory_pool()`. As a
+matter of convenience, some of the array builder classes have constructors
+which use the default pool without explicitly passing it. You can disable these
+constructors in your application (so that you are accounting properly for all
+memory allocations) by defining `ARROW_NO_DEFAULT_MEMORY_POOL`.
+
 ### Error Handling and Exceptions
 
 For error handling, we use `arrow::Status` values instead of throwing C++
@@ -149,6 +157,12 @@ constructors, the circumstances where they would are somewhat esoteric, and it
 is likely that an application would have encountered other more serious
 problems prior to having `std::bad_alloc` thrown in a constructor.
 
+### Deprecations and API Changes
+
+We use the compiler definition `ARROW_NO_DEPRECATED_API` to disable APIs that
+have been deprecated. It is a good practice to compile third party applications
+with this flag to proactively catch and account for API changes.
+
 ## Continuous Integration
 
 Pull requests are run through travis-ci for continuous integration.  You can avoid
diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile
index f32ad5425da35..94156d55801f1 100644
--- a/cpp/apidoc/Doxyfile
+++ b/cpp/apidoc/Doxyfile
@@ -2084,7 +2084,7 @@ PREDEFINED = __attribute__(x)= \
 # definition found in the source code.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_AS_DEFINED      =
+EXPAND_AS_DEFINED = ARROW_MEMORY_POOL_ARG
 
 # If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
 # remove all references to function-like macros that are alone on a line, have
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 436ce9cf7c312..e94ba48d60840 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -37,7 +37,7 @@ class DecimalTestBase {
     auto type = std::make_shared<DecimalType>(precision, 4);
     int byte_width = type->byte_width();
     auto pool = default_memory_pool();
-    auto builder = std::make_shared<DecimalBuilder>(pool, type);
+    auto builder = std::make_shared<DecimalBuilder>(type, pool);
     size_t null_count = 0;
 
     size_t size = draw.size();
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 57d2c8b8493a5..38aceb2d185bb 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -171,7 +171,7 @@ TEST_F(TestArray, TestIsNull) {
 TEST_F(TestArray, BuildLargeInMemoryArray) {
   const int64_t length = static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1;
 
-  BooleanBuilder builder(default_memory_pool());
+  BooleanBuilder builder;
   ASSERT_OK(builder.Reserve(length));
   ASSERT_OK(builder.Advance(length));
 
@@ -754,9 +754,9 @@ TEST_F(TestStringArray, TestEmptyStringComparison) {
 }
 
 TEST_F(TestStringArray, CompareNullByteSlots) {
-  StringBuilder builder(default_memory_pool());
-  StringBuilder builder2(default_memory_pool());
-  StringBuilder builder3(default_memory_pool());
+  StringBuilder builder;
+  StringBuilder builder2;
+  StringBuilder builder3;
 
   ASSERT_OK(builder.Append("foo"));
   ASSERT_OK(builder2.Append("foo"));
@@ -795,7 +795,7 @@ TEST_F(TestStringArray, CompareNullByteSlots) {
 }
 
 TEST_F(TestStringArray, TestSliceGetString) {
-  StringBuilder builder(default_memory_pool());
+  StringBuilder builder;
 
   ASSERT_OK(builder.Append("a"));
   ASSERT_OK(builder.Append("b"));
@@ -958,7 +958,7 @@ TEST_F(TestBinaryArray, TestGetValue) {
 }
 
 TEST_F(TestBinaryArray, TestEqualsEmptyStrings) {
-  BinaryBuilder builder(default_memory_pool(), arrow::binary());
+  BinaryBuilder builder;
 
   string empty_string("");
   for (int i = 0; i < 5; ++i) {
@@ -1045,7 +1045,7 @@ void CheckSliceEquality() {
   using Traits = TypeTraits<TYPE>;
   using BuilderType = typename Traits::BuilderType;
 
-  BuilderType builder(default_memory_pool());
+  BuilderType builder;
 
   vector<string> strings = {"foo", "", "bar", "baz", "qux", ""};
   vector<uint8_t> is_null = {0, 1, 0, 1, 0, 0};
@@ -1102,7 +1102,7 @@ class TestFWBinaryArray : public ::testing::Test {
 
   void InitBuilder(int byte_width) {
     auto type = fixed_size_binary(byte_width);
-    builder_.reset(new FixedSizeBinaryBuilder(default_memory_pool(), type));
+    builder_.reset(new FixedSizeBinaryBuilder(type, default_memory_pool()));
   }
 
  protected:
@@ -1184,8 +1184,8 @@ TEST_F(TestFWBinaryArray, EqualsRangeEquals) {
   // Check that we don't compare data in null slots
 
   auto type = fixed_size_binary(4);
-  FixedSizeBinaryBuilder builder1(default_memory_pool(), type);
-  FixedSizeBinaryBuilder builder2(default_memory_pool(), type);
+  FixedSizeBinaryBuilder builder1(type);
+  FixedSizeBinaryBuilder builder2(type);
 
   ASSERT_OK(builder1.Append("foo1"));
   ASSERT_OK(builder1.AppendNull());
@@ -1209,7 +1209,7 @@ TEST_F(TestFWBinaryArray, EqualsRangeEquals) {
 
 TEST_F(TestFWBinaryArray, ZeroSize) {
   auto type = fixed_size_binary(0);
-  FixedSizeBinaryBuilder builder(default_memory_pool(), type);
+  FixedSizeBinaryBuilder builder(type);
 
   ASSERT_OK(builder.Append(nullptr));
   ASSERT_OK(builder.Append(nullptr));
@@ -1233,7 +1233,7 @@ TEST_F(TestFWBinaryArray, ZeroSize) {
 
 TEST_F(TestFWBinaryArray, Slice) {
   auto type = fixed_size_binary(4);
-  FixedSizeBinaryBuilder builder(default_memory_pool(), type);
+  FixedSizeBinaryBuilder builder(type);
 
   vector<string> strings = {"foo1", "foo2", "foo3", "foo4", "foo5"};
   vector<uint8_t> is_null = {0, 1, 0, 0, 0};
@@ -1519,14 +1519,14 @@ TYPED_TEST(TestDictionaryBuilder, Basic) {
   ASSERT_OK(builder.Finish(&result));
 
   // Build expected data
-  NumericBuilder<TypeParam> dict_builder(default_memory_pool());
+  NumericBuilder<TypeParam> dict_builder;
   ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
   ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
   std::shared_ptr<Array> dict_array;
   ASSERT_OK(dict_builder.Finish(&dict_array));
   auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
 
-  Int8Builder int_builder(default_memory_pool());
+  Int8Builder int_builder;
   ASSERT_OK(int_builder.Append(0));
   ASSERT_OK(int_builder.Append(1));
   ASSERT_OK(int_builder.Append(0));
@@ -1538,8 +1538,8 @@ TYPED_TEST(TestDictionaryBuilder, Basic) {
 }
 
 TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
-  NumericBuilder<TypeParam> builder(default_memory_pool());
-  // DictionaryBuilder<TypeParam> builder(default_memory_pool());
+  NumericBuilder<TypeParam> builder;
+  // DictionaryBuilder<TypeParam> builder;
   ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
   ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
   ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
@@ -1552,14 +1552,14 @@ TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
   ASSERT_OK(dictionary_builder.Finish(&result));
 
   // Build expected data
-  NumericBuilder<TypeParam> dict_builder(default_memory_pool());
+  NumericBuilder<TypeParam> dict_builder;
   ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
   ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
   std::shared_ptr<Array> dict_array;
   ASSERT_OK(dict_builder.Finish(&dict_array));
   auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
 
-  Int8Builder int_builder(default_memory_pool());
+  Int8Builder int_builder;
   ASSERT_OK(int_builder.Append(0));
   ASSERT_OK(int_builder.Append(1));
   ASSERT_OK(int_builder.Append(0));
@@ -1577,8 +1577,8 @@ TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
     // Build the dictionary Array
     DictionaryBuilder<TypeParam> builder(default_memory_pool());
     // Build expected data
-    NumericBuilder<TypeParam> dict_builder(default_memory_pool());
-    Int16Builder int_builder(default_memory_pool());
+    NumericBuilder<TypeParam> dict_builder;
+    Int16Builder int_builder;
 
     // Fill with 1024 different values
     for (int64_t i = 0; i < 1024; i++) {
@@ -1619,14 +1619,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
   ASSERT_OK(builder.Finish(&result));
 
   // Build expected data
-  StringBuilder str_builder(default_memory_pool());
+  StringBuilder str_builder;
   ASSERT_OK(str_builder.Append("test"));
   ASSERT_OK(str_builder.Append("test2"));
   std::shared_ptr<Array> str_array;
   ASSERT_OK(str_builder.Finish(&str_array));
   auto dtype = std::make_shared<DictionaryType>(int8(), str_array);
 
-  Int8Builder int_builder(default_memory_pool());
+  Int8Builder int_builder;
   ASSERT_OK(int_builder.Append(0));
   ASSERT_OK(int_builder.Append(1));
   ASSERT_OK(int_builder.Append(0));
@@ -1641,8 +1641,8 @@ TEST(TestStringDictionaryBuilder, DoubleTableSize) {
   // Build the dictionary Array
   StringDictionaryBuilder builder(default_memory_pool());
   // Build expected data
-  StringBuilder str_builder(default_memory_pool());
-  Int16Builder int_builder(default_memory_pool());
+  StringBuilder str_builder;
+  Int16Builder int_builder;
 
   // Fill with 1024 different values
   for (int64_t i = 0; i < 1024; i++) {
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index 13d7b20591dad..7ac7fe3bed533 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -30,7 +30,7 @@ static void BM_BuildPrimitiveArrayNoNulls(
   // 2 MiB block
   std::vector<int64_t> data(256 * 1024, 100);
   while (state.KeepRunning()) {
-    Int64Builder builder(default_memory_pool());
+    Int64Builder builder;
     for (int i = 0; i < kFinalSize; i++) {
       // Build up an array of 512 MiB in size
       ABORT_NOT_OK(builder.Append(data.data(), data.size(), nullptr));
@@ -66,7 +66,7 @@ static void BM_BuildAdaptiveIntNoNulls(
     data.push_back(i);
   }
   while (state.KeepRunning()) {
-    AdaptiveIntBuilder builder(default_memory_pool());
+    AdaptiveIntBuilder builder;
     for (int64_t i = 0; i < size; i += chunk_size) {
       // Build up an array of 512 MiB in size
       ABORT_NOT_OK(builder.Append(data.data() + i, chunk_size, nullptr));
@@ -85,7 +85,7 @@ static void BM_BuildAdaptiveIntNoNullsScalarAppend(
     data.push_back(i);
   }
   while (state.KeepRunning()) {
-    AdaptiveIntBuilder builder(default_memory_pool());
+    AdaptiveIntBuilder builder;
     for (int64_t i = 0; i < size; i++) {
       ABORT_NOT_OK(builder.Append(data[i]));
     }
@@ -104,7 +104,7 @@ static void BM_BuildAdaptiveUIntNoNulls(
     data.push_back(i);
   }
   while (state.KeepRunning()) {
-    AdaptiveUIntBuilder builder(default_memory_pool());
+    AdaptiveUIntBuilder builder;
     for (int64_t i = 0; i < size; i += chunk_size) {
       // Build up an array of 512 MiB in size
       ABORT_NOT_OK(builder.Append(data.data() + i, chunk_size, nullptr));
@@ -161,7 +161,7 @@ static void BM_BuildBinaryArray(benchmark::State& state) {  // NOLINT non-const
 
   std::string value = "1234567890";
   while (state.KeepRunning()) {
-    BinaryBuilder builder(default_memory_pool());
+    BinaryBuilder builder;
     for (int64_t i = 0; i < iterations; i++) {
       ABORT_NOT_OK(builder.Append(value));
     }
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 391204f566954..e3eda2401a02b 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -262,7 +262,7 @@ template class PrimitiveBuilder<FloatType>;
 template class PrimitiveBuilder<DoubleType>;
 
 AdaptiveIntBuilderBase::AdaptiveIntBuilderBase(MemoryPool* pool)
-    : ArrayBuilder(pool, int64()), data_(nullptr), raw_data_(nullptr), int_size_(1) {}
+    : ArrayBuilder(int64(), pool), data_(nullptr), raw_data_(nullptr), int_size_(1) {}
 
 Status AdaptiveIntBuilderBase::Init(int64_t capacity) {
   RETURN_NOT_OK(ArrayBuilder::Init(capacity));
@@ -612,13 +612,18 @@ Status AdaptiveUIntBuilder::ExpandIntSize(uint8_t new_int_size) {
 }
 
 BooleanBuilder::BooleanBuilder(MemoryPool* pool)
-    : ArrayBuilder(pool, boolean()), data_(nullptr), raw_data_(nullptr) {}
+    : ArrayBuilder(boolean(), pool), data_(nullptr), raw_data_(nullptr) {}
 
-BooleanBuilder::BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
+BooleanBuilder::BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
     : BooleanBuilder(pool) {
   DCHECK_EQ(Type::BOOL, type->id());
 }
 
+#ifndef ARROW_NO_DEPRECATED_API
+BooleanBuilder::BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
+    : BooleanBuilder(type, pool) {}
+#endif
+
 Status BooleanBuilder::Init(int64_t capacity) {
   RETURN_NOT_OK(ArrayBuilder::Init(capacity));
   data_ = std::make_shared<PoolBuffer>(pool_);
@@ -693,18 +698,25 @@ Status BooleanBuilder::Append(const uint8_t* values, int64_t length,
 // DictionaryBuilder
 
 template <typename T>
-DictionaryBuilder<T>::DictionaryBuilder(MemoryPool* pool,
-                                        const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool, type),
+DictionaryBuilder<T>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
+                                        MemoryPool* pool)
+    : ArrayBuilder(type, pool),
       hash_table_(new PoolBuffer(pool)),
       hash_slots_(nullptr),
-      dict_builder_(pool, type),
+      dict_builder_(type, pool),
       values_builder_(pool) {
   if (!::arrow::CpuInfo::initialized()) {
     ::arrow::CpuInfo::Init();
   }
 }
 
+#ifndef ARROW_NO_DEPRECATED_API
+template <typename T>
+DictionaryBuilder<T>::DictionaryBuilder(MemoryPool* pool,
+                                        const std::shared_ptr<DataType>& type)
+    : DictionaryBuilder(type, pool) {}
+#endif
+
 template <typename T>
 Status DictionaryBuilder<T>::Init(int64_t elements) {
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
@@ -931,11 +943,17 @@ template class DictionaryBuilder<StringType>;
 
 // ----------------------------------------------------------------------
 // DecimalBuilder
-DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-    : FixedSizeBinaryBuilder(pool, type),
+
+DecimalBuilder::DecimalBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+    : FixedSizeBinaryBuilder(type, pool),
       sign_bitmap_(nullptr),
       sign_bitmap_data_(nullptr) {}
 
+#ifndef ARROW_NO_DEPRECATED_API
+DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
+    : DecimalBuilder(type, pool) {}
+#endif
+
 template <typename T>
 ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal<T>& val) {
   DCHECK_EQ(sign_bitmap_, nullptr) << "sign_bitmap_ is not null";
@@ -1014,9 +1032,9 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
                          const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool,
-                   type ? type : std::static_pointer_cast<DataType>(
-                                     std::make_shared<ListType>(value_builder->type()))),
+    : ArrayBuilder(type ? type : std::static_pointer_cast<DataType>(
+                                     std::make_shared<ListType>(value_builder->type())),
+                   pool),
       offsets_builder_(pool),
       value_builder_(std::move(value_builder)) {}
 
@@ -1090,10 +1108,15 @@ ArrayBuilder* ListBuilder::value_builder() const {
 // ----------------------------------------------------------------------
 // String and binary
 
+BinaryBuilder::BinaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+    : ArrayBuilder(type, pool), offsets_builder_(pool), value_data_builder_(pool) {}
+
+#ifndef ARROW_NO_DEPRECATED_API
 BinaryBuilder::BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool, type), offsets_builder_(pool), value_data_builder_(pool) {}
+    : BinaryBuilder(type, pool) {}
+#endif
 
-BinaryBuilder::BinaryBuilder(MemoryPool* pool) : BinaryBuilder(pool, binary()) {}
+BinaryBuilder::BinaryBuilder(MemoryPool* pool) : BinaryBuilder(binary(), pool) {}
 
 Status BinaryBuilder::Init(int64_t elements) {
   DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
@@ -1173,7 +1196,7 @@ const uint8_t* BinaryBuilder::GetValue(int64_t i, int32_t* out_length) const {
   return value_data_builder_.data() + offset;
 }
 
-StringBuilder::StringBuilder(MemoryPool* pool) : BinaryBuilder(pool, utf8()) {}
+StringBuilder::StringBuilder(MemoryPool* pool) : BinaryBuilder(utf8(), pool) {}
 
 Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<ArrayData> data;
@@ -1186,12 +1209,18 @@ Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
 // ----------------------------------------------------------------------
 // Fixed width binary
 
-FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(MemoryPool* pool,
-                                               const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(pool, type),
+FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
+                                               MemoryPool* pool)
+    : ArrayBuilder(type, pool),
       byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()),
       byte_builder_(pool) {}
 
+#ifndef ARROW_NO_DEPRECATED_API
+FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(MemoryPool* pool,
+                                               const std::shared_ptr<DataType>& type)
+    : FixedSizeBinaryBuilder(type, pool) {}
+#endif
+
 Status FixedSizeBinaryBuilder::Append(const uint8_t* value) {
   RETURN_NOT_OK(Reserve(1));
   UnsafeAppendToBitmap(true);
@@ -1236,12 +1265,18 @@ Status FixedSizeBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
 // ----------------------------------------------------------------------
 // Struct
 
-StructBuilder::StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+StructBuilder::StructBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool,
                              std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders)
-    : ArrayBuilder(pool, type) {
+    : ArrayBuilder(type, pool) {
   field_builders_ = std::move(field_builders);
 }
 
+#ifndef ARROW_NO_DEPRECATED_API
+StructBuilder::StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+                             std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders)
+    : StructBuilder(type, pool, std::move(field_builders)) {}
+#endif
+
 Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
   std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
   for (size_t i = 0; i < field_builders_.size(); ++i) {
@@ -1261,7 +1296,7 @@ Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
 
 #define BUILDER_CASE(ENUM, BuilderType)      \
   case Type::ENUM:                           \
-    out->reset(new BuilderType(pool, type)); \
+    out->reset(new BuilderType(type, pool)); \
     return Status::OK();
 
 // Initially looked at doing this with vtables, but shared pointers makes it
@@ -1309,7 +1344,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
         RETURN_NOT_OK(MakeBuilder(pool, it->type(), &builder));
         values_builder.emplace_back(std::move(builder));
       }
-      out->reset(new StructBuilder(pool, type, std::move(values_builder)));
+      out->reset(new StructBuilder(type, pool, std::move(values_builder)));
       return Status::OK();
     }
 
@@ -1320,7 +1355,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 
 #define DICTIONARY_BUILDER_CASE(ENUM, BuilderType) \
   case Type::ENUM:                                 \
-    out->reset(new BuilderType(pool, type));       \
+    out->reset(new BuilderType(type, pool));       \
     return Status::OK();
 
 Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 009fd7ae47d19..e441179ae7864 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -53,6 +53,12 @@ struct Decimal;
 
 static constexpr int64_t kMinBuilderCapacity = 1 << 5;
 
+#ifdef ARROW_NO_DEFAULT_MEMORY_POOL
+#define ARROW_MEMORY_POOL_ARG pool
+#else
+#define ARROW_MEMORY_POOL_ARG pool = default_memory_pool()
+#endif
+
 /// Base class for all data array builders.
 //
 /// This class provides a facilities for incrementally building the null bitmap
@@ -60,9 +66,9 @@ static constexpr int64_t kMinBuilderCapacity = 1 << 5;
 /// the null count.
 class ARROW_EXPORT ArrayBuilder {
  public:
-  explicit ArrayBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-      : pool_(pool),
-        type_(type),
+  explicit ArrayBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : type_(type),
+        pool_(pool),
         null_bitmap_(nullptr),
         null_count_(0),
         null_bitmap_data_(nullptr),
@@ -117,9 +123,8 @@ class ARROW_EXPORT ArrayBuilder {
   std::shared_ptr<DataType> type() const { return type_; }
 
  protected:
-  MemoryPool* pool_;
-
   std::shared_ptr<DataType> type_;
+  MemoryPool* pool_;
 
   // When null_bitmap are first appended to the builder, the null bitmap is allocated
   std::shared_ptr<PoolBuffer> null_bitmap_;
@@ -162,8 +167,13 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
  public:
   using value_type = typename Type::c_type;
 
+  explicit PrimitiveBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : ArrayBuilder(type, pool), data_(nullptr), raw_data_(nullptr) {}
+
+#ifndef ARROW_NO_DEPRECATED_API
   explicit PrimitiveBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-      : ArrayBuilder(pool, type), data_(nullptr), raw_data_(nullptr) {}
+      : PrimitiveBuilder(type, pool) {}
+#endif
 
   using ArrayBuilder::Advance;
 
@@ -210,8 +220,9 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
 
   template <typename T1 = T>
   explicit NumericBuilder(
-      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool)
-      : PrimitiveBuilder<T1>(pool, TypeTraits<T1>::type_singleton()) {}
+      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type
+          ARROW_MEMORY_POOL_ARG)
+      : PrimitiveBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 
   using PrimitiveBuilder<T>::Append;
   using PrimitiveBuilder<T>::Init;
@@ -341,7 +352,7 @@ inline uint8_t ExpandedUIntSize(uint64_t val, uint8_t current_int_size) {
 
 class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
-  explicit AdaptiveUIntBuilder(MemoryPool* pool);
+  explicit AdaptiveUIntBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
 
   using ArrayBuilder::Advance;
 
@@ -400,7 +411,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
 
 class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
-  explicit AdaptiveIntBuilder(MemoryPool* pool);
+  explicit AdaptiveIntBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
 
   using ArrayBuilder::Advance;
 
@@ -459,8 +470,14 @@ class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase
 
 class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
  public:
-  explicit BooleanBuilder(MemoryPool* pool);
+  explicit BooleanBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
+
+  explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
+
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.6.0
   explicit BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+#endif
 
   using ArrayBuilder::Advance;
 
@@ -574,8 +591,14 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
 /// \brief Builder class for variable-length binary data
 class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
  public:
-  explicit BinaryBuilder(MemoryPool* pool);
-  explicit BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+  explicit BinaryBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
+
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.6.0
+  BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+#endif
+
+  BinaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
 
   Status Append(const uint8_t* value, int32_t length);
 
@@ -617,7 +640,7 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
 class ARROW_EXPORT StringBuilder : public BinaryBuilder {
  public:
   using BinaryBuilder::BinaryBuilder;
-  explicit StringBuilder(MemoryPool* pool);
+  explicit StringBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
 
   using BinaryBuilder::Append;
 
@@ -631,7 +654,13 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
 
 class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
  public:
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.6.0
   FixedSizeBinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+#endif
+
+  FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
+                         MemoryPool* ARROW_MEMORY_POOL_ARG);
 
   Status Append(const uint8_t* value);
   Status Append(const uint8_t* data, int64_t length,
@@ -653,7 +682,13 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 
 class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
  public:
+  explicit DecimalBuilder(const std::shared_ptr<DataType>& type,
+                          MemoryPool* ARROW_MEMORY_POOL_ARG);
+
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.6.0
   explicit DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+#endif
 
   template <typename T>
   ARROW_EXPORT Status Append(const decimal::Decimal<T>& val);
@@ -679,8 +714,14 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
 /// called to maintain data-structure consistency.
 class ARROW_EXPORT StructBuilder : public ArrayBuilder {
  public:
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.6.0
   StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
                 std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders);
+#endif
+
+  StructBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool,
+                std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders);
 
   Status Finish(std::shared_ptr<Array>* out) override;
 
@@ -759,12 +800,20 @@ template <typename T>
 class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
  public:
   using Scalar = typename internal::DictionaryScalar<T>::type;
+
+  ~DictionaryBuilder() {}
+
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.6.0
   explicit DictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+#endif
+
+  DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
 
   template <typename T1 = T>
   explicit DictionaryBuilder(
       typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool)
-      : DictionaryBuilder<T1>(pool, TypeTraits<T1>::type_singleton()) {}
+      : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 
   /// \brief Append a scalar value
   Status Append(const Scalar& value);
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 35264fa02c5ba..1d5a6997ae920 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -225,7 +225,7 @@ void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
   static const int kBufferSize = 10;
   static uint8_t buffer[kBufferSize];
   static uint32_t seed = 0;
-  StringBuilder string_builder(default_memory_pool());
+  StringBuilder string_builder;
   for (int i = 0; i < num_rows; ++i) {
     if (!is_valid[i]) {
       ASSERT_OK(string_builder.AppendNull());
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
index a88120a248d2d..a6da6377c0531 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -42,8 +42,7 @@ std::shared_ptr<RecordBatch> MakeRecordBatch(int64_t total_size, int64_t num_fie
   std::vector<T> values;
   test::randint<T>(length, 0, 100, &values);
 
-  MemoryPool* pool = default_memory_pool();
-  typename TypeTraits<TYPE>::BuilderType builder(pool, type);
+  typename TypeTraits<TYPE>::BuilderType builder(type, default_memory_pool());
   for (size_t i = 0; i < values.size(); ++i) {
     if (is_valid[i]) {
       ABORT_NOT_OK(builder.Append(values[i]));
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 175d75b7d1e97..49fb6ac7ce30f 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -977,7 +977,7 @@ class ArrayReader {
           std::is_base_of<TimeType, T>::value || std::is_base_of<BooleanType, T>::value,
       Status>::type
   Visit(const T& type) {
-    typename TypeTraits<T>::BuilderType builder(pool_, type_);
+    typename TypeTraits<T>::BuilderType builder(type_, pool_);
 
     const auto& json_data = obj_->FindMember("DATA");
     RETURN_NOT_ARRAY("DATA", json_data, *obj_);
@@ -1046,7 +1046,7 @@ class ArrayReader {
   template <typename T>
   typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value, Status>::type
   Visit(const T& type) {
-    FixedSizeBinaryBuilder builder(pool_, type_);
+    FixedSizeBinaryBuilder builder(type_, pool_);
 
     const auto& json_data = obj_->FindMember("DATA");
     RETURN_NOT_ARRAY("DATA", json_data, *obj_);
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 76cc8430636f8..ed33e6e95b13a 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -99,7 +99,7 @@ Status MakeRandomInt32Array(int64_t length, bool include_nulls, MemoryPool* pool
                             std::shared_ptr<Array>* out) {
   std::shared_ptr<PoolBuffer> data;
   RETURN_NOT_OK(test::MakeRandomInt32PoolBuffer(length, pool, &data));
-  Int32Builder builder(pool, int32());
+  Int32Builder builder(int32(), pool);
   if (include_nulls) {
     std::shared_ptr<PoolBuffer> valid_bytes;
     RETURN_NOT_OK(test::MakeRandomBytePoolBuffer(length, pool, &valid_bytes));
@@ -653,8 +653,8 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
 
   std::shared_ptr<Array> a1, a2;
 
-  FixedSizeBinaryBuilder b1(default_memory_pool(), f0->type());
-  FixedSizeBinaryBuilder b2(default_memory_pool(), f1->type());
+  FixedSizeBinaryBuilder b1(f0->type());
+  FixedSizeBinaryBuilder b2(f1->type());
 
   std::vector<std::string> values1 = {"foo1", "foo2", "foo3", "foo4"};
   AppendValues(is_valid, values1, &b1);
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index 049f5a58a6841..a687a8fc0e703 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -89,7 +89,7 @@ TEST_F(TestPrettyPrint, FixedSizeBinaryType) {
 
   std::shared_ptr<Array> array;
   auto type = fixed_size_binary(3);
-  FixedSizeBinaryBuilder builder(default_memory_pool(), type);
+  FixedSizeBinaryBuilder builder(type);
 
   ASSERT_OK(builder.Append(values[0]));
   ASSERT_OK(builder.Append(values[1]));
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 2fbed1b8fdf08..590be223d3f07 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -17,6 +17,8 @@
 
 // Functions for pandas conversion via NumPy
 
+#define ARROW_NO_DEFAULT_MEMORY_POOL
+
 #include "arrow/python/numpy_interop.h"
 
 #include "arrow/python/pandas_to_arrow.h"
@@ -586,7 +588,7 @@ Status PandasConverter::ConvertDecimals() {
   type_ = std::make_shared<DecimalType>(precision, scale);
 
   const int bit_width = std::dynamic_pointer_cast<DecimalType>(type_)->bit_width();
-  DecimalBuilder builder(pool_, type_);
+  DecimalBuilder builder(type_, pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
   for (int64_t i = 0; i < length_; ++i) {
@@ -619,7 +621,7 @@ Status PandasConverter::ConvertTimes() {
   PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
 
   // datetime.time stores microsecond resolution
-  Time64Builder builder(pool_, ::arrow::time64(TimeUnit::MICRO));
+  Time64Builder builder(::arrow::time64(TimeUnit::MICRO), pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
   PyObject* obj;
@@ -751,7 +753,7 @@ Status PandasConverter::ConvertObjectFixedWidthBytes(
 
   // The output type at this point is inconclusive because there may be bytes
   // and unicode mixed in the object array
-  FixedSizeBinaryBuilder builder(pool_, type);
+  FixedSizeBinaryBuilder builder(type, pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
   int64_t offset = 0;
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index b50699d1ae9d4..433ce9b37a80a 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -71,7 +71,7 @@ TEST(DecimalTest, TestPythonDecimalToString) {
 }
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
-  StringBuilder builder(default_memory_pool());
+  StringBuilder builder;
   const char value[] = {'\xf1', '\0'};
 
   for (int i = 0; i < 1000; ++i) {
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 1a3376cee6053..711d2b04025c1 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -221,8 +221,7 @@ template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::shared_ptr<DataType>& type,
                      const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
                      std::shared_ptr<Array>* out) {
-  MemoryPool* pool = default_memory_pool();
-  typename TypeTraits<TYPE>::BuilderType builder(pool, type);
+  typename TypeTraits<TYPE>::BuilderType builder(type, default_memory_pool());
   for (size_t i = 0; i < values.size(); ++i) {
     if (is_valid[i]) {
       ASSERT_OK(builder.Append(values[i]));
@@ -236,8 +235,7 @@ void ArrayFromVector(const std::shared_ptr<DataType>& type,
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
                      std::shared_ptr<Array>* out) {
-  MemoryPool* pool = default_memory_pool();
-  typename TypeTraits<TYPE>::BuilderType builder(pool);
+  typename TypeTraits<TYPE>::BuilderType builder;
   for (size_t i = 0; i < values.size(); ++i) {
     if (is_valid[i]) {
       ASSERT_OK(builder.Append(values[i]));
@@ -250,8 +248,7 @@ void ArrayFromVector(const std::vector<bool>& is_valid, const std::vector<C_TYPE
 
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
-  MemoryPool* pool = default_memory_pool();
-  typename TypeTraits<TYPE>::BuilderType builder(pool);
+  typename TypeTraits<TYPE>::BuilderType builder;
   for (size_t i = 0; i < values.size(); ++i) {
     ASSERT_OK(builder.Append(values[i]));
   }

From 93b51a039e8901671e2fee4289e492eaadaa4def Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 2 Aug 2017 18:27:18 -0400
Subject: [PATCH 0893/1644] ARROW-1315: [GLib] Add missing status check for
 arrow::ArrayBuilder::Finish()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #933 from kou/glib-add-missing-status-check-for-array-builder-finish and squashes the following commits:

ba98ff54 [Kouhei Sutou] [GLib] Update Go examples to follow Finish() API change
b951b8f6 [Kouhei Sutou] [GLib] Add missing status check for arrow::ArrayBuilder::Finish()
---
 c_glib/arrow-glib/array-builder.cpp | 14 +++++--
 c_glib/arrow-glib/array-builder.h   |  3 +-
 c_glib/example/build.c              |  8 +++-
 c_glib/example/go/write-batch.go    | 60 ++++++++++++++++++++++++-----
 c_glib/example/go/write-stream.go   | 60 ++++++++++++++++++++++++-----
 5 files changed, 119 insertions(+), 26 deletions(-)

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 23bc842756e65..1ef4e77e8fc50 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -220,16 +220,22 @@ garrow_array_builder_new(const std::shared_ptr<arrow::DataType> &type,
 /**
  * garrow_array_builder_finish:
  * @builder: A #GArrowArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: (transfer full): The built #GArrowArray.
+ * Returns: (transfer full): The built #GArrowArray on success,
+ *   %NULL on error.
  */
 GArrowArray *
-garrow_array_builder_finish(GArrowArrayBuilder *builder)
+garrow_array_builder_finish(GArrowArrayBuilder *builder, GError **error)
 {
   auto arrow_builder = garrow_array_builder_get_raw(builder);
   std::shared_ptr<arrow::Array> arrow_array;
-  arrow_builder->Finish(&arrow_array);
-  return garrow_array_new_raw(&arrow_array);
+  auto status = arrow_builder->Finish(&arrow_array);
+  if (garrow_error_check(error, status, "[array-builder][finish]")) {
+    return garrow_array_new_raw(&arrow_array);
+  } else {
+    return NULL;
+  }
 }
 
 
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index 97cea63078b12..613a5bad494d1 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -65,7 +65,8 @@ struct _GArrowArrayBuilderClass
 
 GType               garrow_array_builder_get_type (void) G_GNUC_CONST;
 
-GArrowArray        *garrow_array_builder_finish   (GArrowArrayBuilder *builder);
+GArrowArray        *garrow_array_builder_finish   (GArrowArrayBuilder *builder,
+                                                   GError **error);
 
 
 #define GARROW_TYPE_BOOLEAN_ARRAY_BUILDER       \
diff --git a/c_glib/example/build.c b/c_glib/example/build.c
index 2722458acd5c4..8c6cf74d74815 100644
--- a/c_glib/example/build.c
+++ b/c_glib/example/build.c
@@ -47,7 +47,13 @@ main(int argc, char **argv)
       g_object_unref(builder);
       return EXIT_FAILURE;
     }
-    array = garrow_array_builder_finish(GARROW_ARRAY_BUILDER(builder));
+    array = garrow_array_builder_finish(GARROW_ARRAY_BUILDER(builder), &error);
+    if (!array) {
+      g_print("failed to finish: %s\n", error->message);
+      g_error_free(error);
+      g_object_unref(builder);
+      return EXIT_FAILURE;
+    }
     g_object_unref(builder);
   }
 
diff --git a/c_glib/example/go/write-batch.go b/c_glib/example/go/write-batch.go
index cda09a9b4e8f7..9dbc3c00acc50 100644
--- a/c_glib/example/go/write-batch.go
+++ b/c_glib/example/go/write-batch.go
@@ -29,7 +29,11 @@ func BuildUInt8Array() *arrow.Array {
 	for _, value := range []uint8{1, 2, 4, 8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildUInt16Array() *arrow.Array {
@@ -37,7 +41,11 @@ func BuildUInt16Array() *arrow.Array {
 	for _, value := range []uint16{1, 2, 4, 8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildUInt32Array() *arrow.Array {
@@ -45,7 +53,11 @@ func BuildUInt32Array() *arrow.Array {
 	for _, value := range []uint32{1, 2, 4, 8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildUInt64Array() *arrow.Array {
@@ -53,7 +65,11 @@ func BuildUInt64Array() *arrow.Array {
 	for _, value := range []uint64{1, 2, 4, 8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildInt8Array() *arrow.Array {
@@ -61,7 +77,11 @@ func BuildInt8Array() *arrow.Array {
 	for _, value := range []int8{1, -2, 4, -8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildInt16Array() *arrow.Array {
@@ -69,7 +89,11 @@ func BuildInt16Array() *arrow.Array {
 	for _, value := range []int16{1, -2, 4, -8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildInt32Array() *arrow.Array {
@@ -77,7 +101,11 @@ func BuildInt32Array() *arrow.Array {
 	for _, value := range []int32{1, -2, 4, -8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildInt64Array() *arrow.Array {
@@ -85,7 +113,11 @@ func BuildInt64Array() *arrow.Array {
 	for _, value := range []int64{1, -2, 4, -8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildFloatArray() *arrow.Array {
@@ -93,7 +125,11 @@ func BuildFloatArray() *arrow.Array {
 	for _, value := range []float32{1.1, -2.2, 4.4, -8.8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildDoubleArray() *arrow.Array {
@@ -101,7 +137,11 @@ func BuildDoubleArray() *arrow.Array {
 	for _, value := range []float64{1.1, -2.2, 4.4, -8.8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func main() {
diff --git a/c_glib/example/go/write-stream.go b/c_glib/example/go/write-stream.go
index 20cb03ef2e324..244741e8cfeb0 100644
--- a/c_glib/example/go/write-stream.go
+++ b/c_glib/example/go/write-stream.go
@@ -29,7 +29,11 @@ func BuildUInt8Array() *arrow.Array {
 	for _, value := range []uint8{1, 2, 4, 8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildUInt16Array() *arrow.Array {
@@ -37,7 +41,11 @@ func BuildUInt16Array() *arrow.Array {
 	for _, value := range []uint16{1, 2, 4, 8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildUInt32Array() *arrow.Array {
@@ -45,7 +53,11 @@ func BuildUInt32Array() *arrow.Array {
 	for _, value := range []uint32{1, 2, 4, 8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildUInt64Array() *arrow.Array {
@@ -53,7 +65,11 @@ func BuildUInt64Array() *arrow.Array {
 	for _, value := range []uint64{1, 2, 4, 8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildInt8Array() *arrow.Array {
@@ -61,7 +77,11 @@ func BuildInt8Array() *arrow.Array {
 	for _, value := range []int8{1, -2, 4, -8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildInt16Array() *arrow.Array {
@@ -69,7 +89,11 @@ func BuildInt16Array() *arrow.Array {
 	for _, value := range []int16{1, -2, 4, -8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildInt32Array() *arrow.Array {
@@ -77,7 +101,11 @@ func BuildInt32Array() *arrow.Array {
 	for _, value := range []int32{1, -2, 4, -8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildInt64Array() *arrow.Array {
@@ -85,7 +113,11 @@ func BuildInt64Array() *arrow.Array {
 	for _, value := range []int64{1, -2, 4, -8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildFloatArray() *arrow.Array {
@@ -93,7 +125,11 @@ func BuildFloatArray() *arrow.Array {
 	for _, value := range []float32{1.1, -2.2, 4.4, -8.8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func BuildDoubleArray() *arrow.Array {
@@ -101,7 +137,11 @@ func BuildDoubleArray() *arrow.Array {
 	for _, value := range []float64{1.1, -2.2, 4.4, -8.8} {
 		builder.Append(value)
 	}
-	return builder.Finish()
+	array, err := builder.Finish()
+		if err != nil {
+		log.Fatalf("Failed to build array: %v", err)
+	}
+	return array
 }
 
 func main() {

From 21a0191579e1065212d2a435a51d9721dd33c176 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 2 Aug 2017 18:28:18 -0400
Subject: [PATCH 0894/1644] ARROW-1323: [GLib] Add
 garrow_boolean_array_get_values()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #934 from kou/glib-add-boolean-get-values and squashes the following commits:

d6502ba2 [Kouhei Sutou] [GLib] Add garrow_boolean_array_get_values()
---
 c_glib/arrow-glib/array.cpp       | 24 ++++++++++++++++++++++++
 c_glib/arrow-glib/array.h         |  2 ++
 c_glib/test/test-boolean-array.rb | 11 +++++++++++
 3 files changed, 37 insertions(+)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 30e51fb309f97..a3c45a890321c 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -557,6 +557,30 @@ garrow_boolean_array_get_value(GArrowBooleanArray *array,
   return static_cast<arrow::BooleanArray *>(arrow_array.get())->Value(i);
 }
 
+/**
+ * garrow_boolean_array_get_values:
+ * @array: A #GArrowBooleanArray.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw boolean values.
+ *
+ *   It should be freed with g_free() when no longer needed.
+ */
+gboolean *
+garrow_boolean_array_get_values(GArrowBooleanArray *array,
+                                gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_boolean_array =
+    std::static_pointer_cast<arrow::BooleanArray>(arrow_array);
+  *length = arrow_boolean_array->length();
+  auto values = static_cast<gboolean *>(g_new(gboolean, *length));
+  for (gint64 i = 0; i < *length; ++i) {
+    values[i] = arrow_boolean_array->Value(i);
+  }
+  return values;
+}
+
 
 G_DEFINE_TYPE(GArrowInt8Array,               \
               garrow_int8_array,             \
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 1b2ba9fc1cc39..10b2279be4bbf 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -221,6 +221,8 @@ GArrowBooleanArray *garrow_boolean_array_new(gint64 length,
 
 gboolean       garrow_boolean_array_get_value (GArrowBooleanArray *array,
                                                gint64 i);
+gboolean      *garrow_boolean_array_get_values(GArrowBooleanArray *array,
+                                               gint64 *length);
 
 
 #define GARROW_TYPE_INT8_ARRAY                  \
diff --git a/c_glib/test/test-boolean-array.rb b/c_glib/test/test-boolean-array.rb
index 43b83655638e3..622e4e90c482b 100644
--- a/c_glib/test/test-boolean-array.rb
+++ b/c_glib/test/test-boolean-array.rb
@@ -17,6 +17,7 @@
 
 class TestBooleanArray < Test::Unit::TestCase
   include Helper::Buildable
+  include Helper::Omittable
 
   def test_new
     assert_equal(build_boolean_array([true, false, nil]),
@@ -41,4 +42,14 @@ def test_value
     array = builder.finish
     assert_equal(true, array.get_value(0))
   end
+
+  def test_values
+    require_gi(3, 1, 9)
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append(true)
+    builder.append(false)
+    builder.append(true)
+    array = builder.finish
+    assert_equal([true, false, true], array.values)
+  end
 end

From 84b7a0d176270911a9d911b3fd2522ead67ee9ee Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 2 Aug 2017 19:32:57 -0400
Subject: [PATCH 0895/1644] ARROW-1312: [C++] Make ARROW_JEMALLOC OFF by
 default until ARROW-1282 is resolved

See ARROW-1282, ARROW-1311 for related discussions.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #935 from wesm/ARROW-1312 and squashes the following commits:

2a06763c [Wes McKinney] Make ARROW_JEMALLOC OFF by default until ARROW-1282 is resolved
---
 ci/travis_before_script_cpp.sh | 6 ------
 cpp/CMakeLists.txt             | 2 +-
 cpp/apidoc/Windows.md          | 8 ++++++--
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index d456d308c53e3..7418b7614ae34 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -70,12 +70,6 @@ if [ $only_library_mode == "yes" ]; then
 $CMAKE_COMMON_FLAGS \
 -DARROW_BUILD_TESTS=OFF \
 -DARROW_BUILD_UTILITIES=OFF"
-else
-  # Deactivate jemalloc on Linux builds. We check the jemalloc+Linux build
-  # also in the manylinux1 image.
-  CMAKE_LINUX_FLAGS="\
-$CMAKE_LINUX_FLAGS \
--DARROW_JEMALLOC=ON"
 fi
 
 # Use Ninja for faster builds when using toolchain
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index eeff9124ad983..7d73d1ffff089 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -100,7 +100,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
 
   option(ARROW_JEMALLOC
     "Build the Arrow jemalloc-based allocator"
-    ON)
+    OFF)
 
   option(ARROW_JEMALLOC_USE_SHARED
     "Rely on jemalloc shared libraries where relevant"
diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index 30b7b8f3ce210..774482ea1c4f3 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -187,14 +187,18 @@ Command line to build Arrow in Debug might look as following:
 cd cpp
 mkdir build
 cd build
-cmake -G "Visual Studio 14 2015 Win64" -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=OFF -DCMAKE_BUILD_TYPE=Debug -DBOOST_ROOT=C:/local/boost_1_63_0  -DBOOST_LIBRARYDIR=C:/local/boost_1_63_0/lib64-msvc-14.0 ..
+cmake -G "Visual Studio 14 2015 Win64" ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=Debug ^
+      -DBOOST_ROOT=C:/local/boost_1_63_0  ^
+      -DBOOST_LIBRARYDIR=C:/local/boost_1_63_0/lib64-msvc-14.0 ^
+      ..
 cmake --build . --config Debug
 ```
 
 To get the latest build instructions, you can reference [msvc-build.bat][5], which is used by automated Appveyor builds.
 
 
-
 [1]: https://conda.io/miniconda.html
 [2]: https://conda-forge.github.io/
 [3]: http://cmder.net/

From 1874a8b7ccbd5a531eb2a8af479ef154694efd9d Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Thu, 3 Aug 2017 15:06:20 -0400
Subject: [PATCH 0896/1644] ARROW-1310: [JAVA] revert changes made in ARROW-886

@elahrvivaz , @StevenMPhillips

Reverting the changes made for ARROW-886 -- https://github.com/apache/arrow/pull/591

(1) Don't explicitly reallocate the offsetVector in realloc() function of Variable Length Vectors.   If we call setSafe() on variable length vector, it will internally invoke setSafe() on the corresponding offsetVector and the latter function can decide whether to reallocate the offsetVector or not.

(2) Doing (1) will break the unit test added as part of PR 591 so we need to remove that as well.

Author: siddharth <siddharth@dremio.com>

Closes #937 from siddharthteotia/ARROW-1310 and squashes the following commits:

c5a2707c [siddharth] ARROW-1310: revert changes made in ARROW-886
---
 .../templates/VariableLengthVectors.java      |  1 -
 .../arrow/vector/TestVectorReAlloc.java       | 27 +------------------
 2 files changed, 1 insertion(+), 27 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index 2ad7d20de2651..70c7209947460 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -352,7 +352,6 @@ public void reset() {
   }
 
   public void reAlloc() {
-    offsetVector.reAlloc();
     final long newAllocationSize = allocationSizeInBytes*2L;
     if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
       throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index da9cb00361c0b..a5d5527539322 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -72,31 +72,6 @@ public void testFixedType() {
     }
   }
 
-  @Test
-  public void testVariableLengthType() {
-    try (final VarCharVector vector = new VarCharVector("", allocator)) {
-      final VarCharVector.Mutator m = vector.getMutator();
-      // note: capacity ends up being - 1 due to offsets vector
-      vector.setInitialCapacity(511);
-      vector.allocateNew();
-
-      assertEquals(511, vector.getValueCapacity());
-
-      try {
-        m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
-        Assert.fail("Expected out of bounds exception");
-      } catch (Exception e) {
-        // ok
-      }
-
-      vector.reAlloc();
-      assertEquals(1023, vector.getValueCapacity());
-
-      m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
-      assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
-    }
-  }
-
   @Test
   public void testNullableType() {
     try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
@@ -114,7 +89,7 @@ public void testNullableType() {
       }
 
       vector.reAlloc();
-      assertEquals(1024, vector.getValueCapacity());
+      assertEquals(1023, vector.getValueCapacity());
 
       m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
       assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));

From 37323242ec378134f3dc6991c5d9fab44017d4cd Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Thu, 3 Aug 2017 15:42:37 -0400
Subject: [PATCH 0897/1644] =?UTF-8?q?ARROW-1224:=20[Format]=20Clarify=20la?=
 =?UTF-8?q?nguage=20around=20buffer=20padding=20and=20align=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Added some clarifications on memory layout w.r.t padding and alignment.

Author: siddharth <siddharth@dremio.com>

Closes #932 from siddharthteotia/ARROW-1224 and squashes the following commits:

dd5f2214 [siddharth] ARROW-1224: [Format] Clarify language around buffer padding and alignment in IPC
---
 format/Layout.md | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index b62b1565a754b..334251103d732 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -62,7 +62,11 @@ Base requirements
   linearly in the nesting level
 * Capable of representing fully-materialized and decoded / decompressed [Parquet][5]
   data
-* All contiguous memory buffers are aligned at 64-byte boundaries and padded to a multiple of 64 bytes.
+* It is required to have all the contiguous memory buffers in an IPC payload
+  aligned at 8-byte boundaries. In other words, each buffer must start at
+  an aligned 8-byte offset.
+* The general recommendation is to align the buffers at 64-byte boundary, but
+  this is not absolutely necessary.
 * Any relative type can have null slots
 * Arrays are immutable once created. Implementations can provide APIs to mutate
   an array, but applying mutations will require a new array data structure to
@@ -108,21 +112,23 @@ via byte swapping.
 
 ## Alignment and Padding
 
-As noted above, all buffers are intended to be aligned in memory at 64 byte
-boundaries and padded to a length that is a multiple of 64 bytes.  The alignment
-requirement follows best practices for optimized memory access:
+As noted above, all buffers must be aligned in memory at 8-byte boundaries and padded
+to a length that is a multiple of 8 bytes.  The alignment requirement follows best
+practices for optimized memory access:
 
 * Elements in numeric arrays will be guaranteed to be retrieved via aligned access.
 * On some architectures alignment can help limit partially used cache lines.
 * 64 byte alignment is recommended by the [Intel performance guide][2] for
-data-structures over 64 bytes (which will be a common case for Arrow Arrays).
+  data-structures over 64 bytes (which will be a common case for Arrow Arrays).
 
-Requiring padding to a multiple of 64 bytes allows for using [SIMD][4] instructions
+Recommending padding to a multiple of 64 bytes allows for using [SIMD][4] instructions
 consistently in loops without additional conditional checks.
-This should allow for simpler and more efficient code.
+This should allow for simpler, efficient and CPU cache-friendly code.
 The specific padding length was chosen because it matches the largest known
-SIMD instruction registers available as of April 2016 (Intel AVX-512).
-Guaranteed padding can also allow certain compilers
+SIMD instruction registers available as of April 2016 (Intel AVX-512). In other
+words, we can load the entire 64-byte buffer into a 512-bit wide SIMD register
+and get data-level parallelism on all the columnar values packed into the 64-byte
+buffer. Guaranteed padding can also allow certain compilers
 to generate more optimized code directly (e.g. One can safely use Intel's
 `-qopt-assume-safe-padding`).
 

From f775af7ef00504924bad3a1d68b99dae75aeff6c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 3 Aug 2017 17:26:07 -0400
Subject: [PATCH 0898/1644] ARROW-1312: [Python] Follow-up: do not use jemalloc
 in manylinux1 builds

Missed this usage in the original patch

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #938 from wesm/ARROW-1312-followup and squashes the following commits:

008f2105 [Wes McKinney] Do not use jemalloc in manylinux1 builds
---
 python/manylinux1/build_arrow.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 5a21e36e4d7d5..ab7416c3275b7 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -58,7 +58,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
     mkdir -p "${ARROW_BUILD_DIR}"
     pushd "${ARROW_BUILD_DIR}"
-    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON ..
+    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON ..
     make -j5 install
     popd
 

From a388ddf8865dc7502779d3adbecaed0b42983ca6 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 4 Aug 2017 11:27:15 -0400
Subject: [PATCH 0899/1644] ARROW-1330: [Plasma] Turn on plasma tests on
 manylinux1

With ARROW-1312 merged, these tests should pass now.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #939 from pcmoritz/reactivate-plasma-tests and squashes the following commits:

6ec77495 [Philipp Moritz] reactivate plasma tests in manylinux1
---
 python/manylinux1/build_arrow.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index ab7416c3275b7..074bd0056a948 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -81,9 +81,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     source /venv-test-${PYTHON}/bin/activate
     pip install repaired_wheels/*.whl
 
-    # ARROW-1264; for some reason the test case added causes a segfault inside
-    # the Docker container when writing and error message to stderr
-    py.test --parquet /venv-test-${PYTHON}/lib/*/site-packages/pyarrow -v -s --disable-plasma
+    py.test --parquet /venv-test-${PYTHON}/lib/*/site-packages/pyarrow -v
     deactivate
 
     mv repaired_wheels/*.whl /io/dist

From aa5d417bbd32fcdf96462181791f6c044ab3215d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 4 Aug 2017 11:36:08 -0400
Subject: [PATCH 0900/1644] ARROW-1326: [Python] Fix Sphinx Build in Travis CI,
 treat Sphinx warnings as errors

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Wes McKinney <wesmckinn@gmail.com>

Closes #936 from wesm/ARROW-1326 and squashes the following commits:

8af31d72 [Wes McKinney] Fix bash syntax
1d4dc699 [Wes McKinney] Only build Sphinx documentation on Linux because of case sensitivity issue
b88adc95 [Wes McKinney] Travis OS X is case insensitive file system
7eb12b02 [Wes McKinney] Nudge build
b33ced76 [Wes McKinney] Use sphinx-build directly, error on Sphinx warnings
11418fbb [Wes McKinney] Build Python docs inside docs directory to avoid import errors
---
 ci/travis_script_python.sh   | 13 ++++++-------
 python/doc/source/api.rst    | 11 ++++++++---
 python/doc/source/plasma.rst |  2 +-
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 4a50d2faaf551..66cd17d5ff212 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -120,14 +120,13 @@ python_version_tests() {
   PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
   python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
 
-  pushd $ARROW_PYTHON_DIR
-  # Build documentation once
-  if [[ "$PYTHON_VERSION" == "3.6" ]]
-  then
-      conda install -y -q --file=doc/requirements.txt
-      python setup.py build_sphinx -s doc/source
+  if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
+      # Build documentation once
+      pushd $ARROW_PYTHON_DIR/doc
+      conda install -y -q --file=requirements.txt
+      sphinx-build -b html -d _build/doctrees -W source _build/html
+      popd
   fi
-  popd
 }
 
 # run tests for python 2.7 and 3.6
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index fd1cb728d9828..1aaf89ce9a1f0 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -91,13 +91,14 @@ Scalar Value Types
 
 .. _api.array:
 
-Array Types and Constructors
-----------------------------
+.. currentmodule:: pyarrow.lib
+
+Array Types
+-----------
 
 .. autosummary::
    :toctree: generated/
 
-   array
    Array
    BooleanArray
    DictionaryArray
@@ -126,6 +127,8 @@ Array Types and Constructors
 
 .. _api.table:
 
+.. currentmodule:: pyarrow
+
 Tables and Record Batches
 -------------------------
 
@@ -214,6 +217,8 @@ Memory Pools
 
 .. _api.type_classes:
 
+.. currentmodule:: pyarrow.lib
+
 Type Classes
 ------------
 
diff --git a/python/doc/source/plasma.rst b/python/doc/source/plasma.rst
index 98dd62f97e951..832d9960cb539 100644
--- a/python/doc/source/plasma.rst
+++ b/python/doc/source/plasma.rst
@@ -16,7 +16,7 @@
 .. under the License.
 
 .. currentmodule:: pyarrow
-.. _io:
+.. _plasma:
 
 The Plasma In-Memory Object Store
 =================================

From 717bed0d0a6a038cca6d44b46f60833aed6b09fc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 4 Aug 2017 14:25:53 -0400
Subject: [PATCH 0901/1644] ARROW-1328: [Python] Set correct Arrow type when
 coercing to milliseconds and passing explicit type

cc @fjetter

I'm planning to deprecate the `timestamps_to_ms` argument but it needs a patch in parquet-cpp first (so that nanoseconds/microseconds can be casted on the fly to milliseconds), so let's handle this fix first

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #941 from wesm/ARROW-1328 and squashes the following commits:

4632dc36 [Wes McKinney] Set correct Arrow type when coercing to milliseconds and passing explicit type (which may not be milliseconds)
---
 python/pyarrow/array.pxi                    | 26 +-------------
 python/pyarrow/pandas_compat.py             | 24 +++++++++++++
 python/pyarrow/table.pxi                    | 19 ++++++----
 python/pyarrow/tests/test_convert_pandas.py | 39 +++++++++++++--------
 4 files changed, 62 insertions(+), 46 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index cbd036c08431f..4e0c21c79e1c0 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -16,30 +16,6 @@
 # under the License.
 
 
-cdef maybe_coerce_datetime64(values, dtype, DataType type,
-                             timestamps_to_ms=False):
-
-    from pyarrow.compat import DatetimeTZDtype
-
-    if values.dtype.type != np.datetime64:
-        return values, type
-
-    coerce_ms = timestamps_to_ms and values.dtype != 'datetime64[ms]'
-
-    if coerce_ms:
-        values = values.astype('datetime64[ms]')
-
-    if isinstance(dtype, DatetimeTZDtype):
-        tz = dtype.tz
-        unit = 'ms' if coerce_ms else dtype.unit
-        type = timestamp(unit, tz)
-    elif type is None:
-        # Trust the NumPy dtype
-        type = from_numpy_dtype(values.dtype)
-
-    return values, type
-
-
 def array(object sequence, DataType type=None, MemoryPool memory_pool=None,
           size=None):
     """
@@ -205,7 +181,7 @@ cdef class Array:
             else:
                 out = chunked_out.get().chunk(0)
         else:
-            values, type = maybe_coerce_datetime64(
+            values, type = pdcompat.maybe_coerce_datetime64(
                 values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
 
             if type is None:
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 62547a42f7359..2881588208eb1 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -17,6 +17,7 @@
 
 import re
 import json
+import numpy as np
 import pandas as pd
 
 import six
@@ -241,6 +242,29 @@ def dataframe_to_arrays(df, timestamps_to_ms, schema, preserve_index):
     return names, arrays, metadata
 
 
+def maybe_coerce_datetime64(values, dtype, type_, timestamps_to_ms=False):
+    from pyarrow.compat import DatetimeTZDtype
+
+    if values.dtype.type != np.datetime64:
+        return values, type_
+
+    coerce_ms = timestamps_to_ms and values.dtype != 'datetime64[ms]'
+
+    if coerce_ms:
+        values = values.astype('datetime64[ms]')
+        type_ = pa.timestamp('ms')
+
+    if isinstance(dtype, DatetimeTZDtype):
+        tz = dtype.tz
+        unit = 'ms' if coerce_ms else dtype.unit
+        type_ = pa.timestamp(unit, tz)
+    elif type_ is None:
+        # Trust the NumPy dtype
+        type_ = pa.from_numpy_dtype(values.dtype)
+
+    return values, type_
+
+
 def table_to_blockmanager(table, nthreads=1):
     import pandas.core.internals as _int
     from pyarrow.compat import DatetimeTZDtype
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 997b28579f847..e33c9ba2f20e9 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -134,6 +134,16 @@ cdef class Column:
         self.sp_column = column
         self.column = column.get()
 
+    def __repr__(self):
+        from pyarrow.compat import StringIO
+        result = StringIO()
+        result.write(object.__repr__(self))
+        data = self.data
+        for i in range(len(data)):
+            result.write('\nchunk {0}: {1}'.format(i, repr(data.chunk(0))))
+
+        return result.getvalue()
+
     @staticmethod
     def from_array(object field_or_name, Array arr):
         cdef Field boxed_field
@@ -661,13 +671,8 @@ cdef class Table:
         return result
 
     @classmethod
-    def from_pandas(
-        cls,
-        df,
-        bint timestamps_to_ms=False,
-        Schema schema=None,
-        bint preserve_index=True
-    ):
+    def from_pandas(cls, df, bint timestamps_to_ms=False,
+                    Schema schema=None, bint preserve_index=True):
         """
         Convert pandas.DataFrame to an Arrow Table
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index f6ea1636a3d62..2a51d3283203f 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -18,7 +18,7 @@
 
 from collections import OrderedDict
 
-import datetime
+from datetime import datetime, date, time
 import unittest
 import decimal
 import json
@@ -351,6 +351,17 @@ def test_timestamps_notimezone_no_nulls(self):
             expected_schema=schema,
         )
 
+    def test_timestamps_to_ms_explicit_schema(self):
+        # ARROW-1328
+        df = pd.DataFrame({'datetime': [datetime(2017, 1, 1)]})
+        pa_type = pa.from_numpy_dtype(df['datetime'].dtype)
+
+        arr = pa.Array.from_pandas(df['datetime'], type=pa_type,
+                                   timestamps_to_ms=True)
+
+        tm.assert_almost_equal(df['datetime'].values.astype('M8[ms]'),
+                               arr.to_pandas())
+
     def test_timestamps_notimezone_nulls(self):
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -409,10 +420,10 @@ def test_timestamps_with_timezone(self):
 
     def test_date_infer(self):
         df = pd.DataFrame({
-            'date': [datetime.date(2000, 1, 1),
+            'date': [date(2000, 1, 1),
                      None,
-                     datetime.date(1970, 1, 1),
-                     datetime.date(2040, 2, 26)]})
+                     date(1970, 1, 1),
+                     date(2040, 2, 26)]})
         table = pa.Table.from_pandas(df, preserve_index=False)
         field = pa.field('date', pa.date32())
         schema = pa.schema([field])
@@ -424,10 +435,10 @@ def test_date_infer(self):
 
     def test_date_objects_typed(self):
         arr = np.array([
-            datetime.date(2017, 4, 3),
+            date(2017, 4, 3),
             None,
-            datetime.date(2017, 4, 4),
-            datetime.date(2017, 4, 5)], dtype=object)
+            date(2017, 4, 4),
+            date(2017, 4, 5)], dtype=object)
 
         arr_i4 = np.array([17259, -1, 17260, 17261], dtype='int32')
         arr_i8 = arr_i4.astype('int64') * 86400000
@@ -470,7 +481,7 @@ def test_dates_from_integers(self):
         a1 = pa.Array.from_pandas(arr, type=t1)
         a2 = pa.Array.from_pandas(arr2, type=t2)
 
-        expected = datetime.date(2017, 4, 3)
+        expected = date(2017, 4, 3)
         assert a1[0].as_py() == expected
         assert a2[0].as_py() == expected
 
@@ -669,8 +680,8 @@ def test_decimal_128_to_pandas(self):
         tm.assert_frame_equal(df, expected)
 
     def test_pytime_from_pandas(self):
-        pytimes = [datetime.time(1, 2, 3, 1356),
-                   datetime.time(4, 5, 6, 1356)]
+        pytimes = [time(1, 2, 3, 1356),
+                   time(4, 5, 6, 1356)]
 
         # microseconds
         t1 = pa.time64('us')
@@ -706,9 +717,9 @@ def test_pytime_from_pandas(self):
         assert a4[0].as_py() == pytimes[0].replace(microsecond=0)
 
     def test_arrow_time_to_pandas(self):
-        pytimes = [datetime.time(1, 2, 3, 1356),
-                   datetime.time(4, 5, 6, 1356),
-                   datetime.time(0, 0, 0)]
+        pytimes = [time(1, 2, 3, 1356),
+                   time(4, 5, 6, 1356),
+                   time(0, 0, 0)]
 
         expected = np.array(pytimes[:2] + [None])
         expected_ms = np.array([x.replace(microsecond=1000)
@@ -902,7 +913,7 @@ def _pytime_from_micros(val):
     val //= 60
     minutes = val % 60
     hours = val // 60
-    return datetime.time(hours, minutes, seconds, microseconds)
+    return time(hours, minutes, seconds, microseconds)
 
 
 def _pytime_to_micros(pytime):

From 3bc7d4604c68930cbaac0262e0c9281adf31f8c0 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Fri, 4 Aug 2017 17:34:46 -0400
Subject: [PATCH 0902/1644] =?UTF-8?q?ARROW-1296:=20[Java]=20Fix=20allocati?=
 =?UTF-8?q?onSizeInBytes=20in=20FixedValueVectors.res=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fix allocationSizeInBytes in FixedValueVectors.reset

Author: Li Jin <ice.xelloss@gmail.com>

Closes #915 from icexelloss/vector-reset-ARROW-1296 and squashes the following commits:

f9c8607f [Li Jin] Add VectorReset test; Add package private method getAllocationSize to FixedValueVectors
d0117a83 [Li Jin] ARROW-1296: [Java] Fix allocationSizeInBytes in FixedValueVectors.reset()
---
 .../codegen/templates/FixedValueVectors.java  |  6 +-
 .../apache/arrow/vector/TestVectorReset.java  | 55 +++++++++++++++++++
 2 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java

diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 5d92cd232efb3..61164ab6c9b3d 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -111,6 +111,10 @@ public Mutator getMutator(){
     return mutator;
   }
 
+  int getAllocationSize() {
+    return allocationSizeInBytes;
+  }
+
   @Override
   public void setInitialCapacity(final int valueCount) {
     final long size = 1L * valueCount * ${type.width};
@@ -162,7 +166,7 @@ public void allocateNew(final int valueCount) {
 
   @Override
   public void reset() {
-    allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+    allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
     allocationMonitor = 0;
     zeroVector();
     super.reset();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
new file mode 100644
index 0000000000000..d53f69489d4da
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestVectorReset {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testFixedTypeReset() {
+    try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+      final UInt4Vector.Mutator m = vector.getMutator();
+      vector.allocateNew();
+      final int sizeBefore = vector.getAllocationSize();
+      vector.reAlloc();
+      vector.reset();
+      final int sizeAfter = vector.getAllocationSize();
+      assertEquals(sizeBefore, sizeAfter);
+    }
+  }
+}

From 25439e7fb2d1047d8af5d77b36b6ffbfe4721beb Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Fri, 4 Aug 2017 17:40:40 -0400
Subject: [PATCH 0903/1644] ARROW-1300: [JAVA] Fix Tests for ListVector

@StevenMPhillips

Fixed the following:

(1) TestListVector.java doesn't include tests for nested lists where the underlying dataVector for a listVector is also a listVector.

(2) The copy test in TestListVector.java only checks the bit vector contents and doesn't verify the actual contents of list vector

Author: siddharth <siddharth@dremio.com>

Closes #925 from siddharthteotia/ARROW-1300 and squashes the following commits:

584c79ef [siddharth] ARROW-1300: Fix tests for ListVector
ff842533 [siddharth] ARROW-1300: Fix Tests for ListVector
9978199d [siddharth] ARROW-1300: Fix tests for ListVector
777e0ded [siddharth] ARROW-1300: Fix Tests for ListVector
---
 .../apache/arrow/vector/TestListVector.java   | 151 +++++++++++++++++-
 1 file changed, 143 insertions(+), 8 deletions(-)

diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index a1762c466ce0b..eb30fdd46bf60 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -38,8 +38,10 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.List;
 
+
 public class TestListVector {
 
   private BufferAllocator allocator;
@@ -92,6 +94,26 @@ public void testCopyFrom() throws Exception {
       Assert.assertFalse("should be null", reader.isSet());
       reader.setPosition(2);
       Assert.assertTrue("shouldn't be null", reader.isSet());
+
+      /* check the exact contents of vector */
+      final ListVector.Accessor accessor = outVector.getAccessor();
+
+      /* index 0 */
+      Object result = accessor.getObject(0);
+      ArrayList<Long> resultSet = (ArrayList<Long>)result;
+      assertEquals(3, resultSet.size());
+      assertEquals(new Long(1), (Long)resultSet.get(0));
+      assertEquals(new Long(2), (Long)resultSet.get(1));
+      assertEquals(new Long(3), (Long)resultSet.get(2));
+
+      /* index 1 */
+      result = accessor.getObject(1);
+      assertNull(result);
+
+      /* index 2 */
+      result = accessor.getObject(2);
+      resultSet = (ArrayList<Long>)result;
+      assertEquals(0, resultSet.size());
     }
   }
 
@@ -127,7 +149,7 @@ public void testSetLastSetUsage() throws Exception {
       dataVector.getMutator().setSafe(2, 1, 12);
       offsetVector.getMutator().setSafe(index + 1, 3);
 
-      index  += 1;
+      index += 1;
 
       /* write [13, 14] to the list vector at index 1 */
       bitVector.getMutator().setSafe(index, 1);
@@ -193,7 +215,7 @@ public void testSetLastSetUsage() throws Exception {
       final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
       final ValueVector.Accessor valueAccessor = dataVector.getAccessor();
 
-      index  = 0;
+      index = 0;
       offset = offsetAccessor.get(index);
       assertEquals(Integer.toString(0), Integer.toString(offset));
 
@@ -300,7 +322,7 @@ public void testSplitAndTransfer() throws Exception {
       final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
       final ValueVector.Accessor valueAccessor = dataVector.getAccessor();
 
-      int index  = 0;
+      int index = 0;
       int offset = 0;
       Object actual = null;
 
@@ -386,10 +408,7 @@ public void testSplitAndTransfer() throws Exception {
 
         TransferPair transferPair = listVector.makeTransferPair(toVector);
 
-        int[][] transferLengths = { {0, 2},
-                                    {3, 1},
-                                    {4, 1}
-                                  };
+        int[][] transferLengths = {{0, 2}, {3, 1}, {4, 1}};
 
         for (final int[] transferLength : transferLengths) {
           int start = transferLength[0];
@@ -423,7 +442,7 @@ public void testSplitAndTransfer() throws Exception {
 
             for(int j = 0; j < dataLength1; j++) {
               assertEquals("Different data at indexes: " + offset1 + " and " + offset2,
-                           valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
+                            valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
 
               offset1++;
               offset2++;
@@ -433,4 +452,120 @@ public void testSplitAndTransfer() throws Exception {
       }
     }
   }
+
+  @Test
+  public void testNestedListVector() throws Exception {
+    try (ListVector listVector = ListVector.empty("sourceVector", allocator)) {
+
+      UnionListWriter listWriter = listVector.getWriter();
+
+      /* allocate memory */
+      listWriter.allocate();
+
+      /* the dataVector that backs a listVector will also be a
+       * listVector for this test.
+       */
+
+      /* write one or more inner lists at index 0 */
+      listWriter.setPosition(0);
+      listWriter.startList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(50);
+      listWriter.list().bigInt().writeBigInt(100);
+      listWriter.list().bigInt().writeBigInt(200);
+      listWriter.list().endList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(75);
+      listWriter.list().bigInt().writeBigInt(125);
+      listWriter.list().bigInt().writeBigInt(150);
+      listWriter.list().bigInt().writeBigInt(175);
+      listWriter.list().endList();
+
+      listWriter.endList();
+
+      /* write one or more inner lists at index 1 */
+      listWriter.setPosition(1);
+      listWriter.startList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(10);
+      listWriter.list().endList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(15);
+      listWriter.list().bigInt().writeBigInt(20);
+      listWriter.list().endList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(25);
+      listWriter.list().bigInt().writeBigInt(30);
+      listWriter.list().bigInt().writeBigInt(35);
+      listWriter.list().endList();
+
+      listWriter.endList();
+
+      assertEquals(2, listVector.getMutator().getLastSet());
+
+      listVector.getMutator().setValueCount(2);
+
+      final ListVector.Accessor accessor = listVector.getAccessor();
+      assertEquals(2, accessor.getValueCount());
+
+      /* get listVector value at index 0 -- the value itself is a listvector */
+      Object result = accessor.getObject(0);
+      ArrayList<ArrayList<Long>> resultSet = (ArrayList<ArrayList<Long>>)result;
+      ArrayList<Long> list;
+
+      assertEquals(2, resultSet.size());              /* 2 inner lists at index 0 */
+      assertEquals(3, resultSet.get(0).size());       /* size of first inner list */
+      assertEquals(4, resultSet.get(1).size());      /* size of second inner list */
+
+      list = resultSet.get(0);
+      assertEquals(new Long(50), list.get(0));
+      assertEquals(new Long(100), list.get(1));
+      assertEquals(new Long(200), list.get(2));
+
+      list = resultSet.get(1);
+      assertEquals(new Long(75), list.get(0));
+      assertEquals(new Long(125), list.get(1));
+      assertEquals(new Long(150), list.get(2));
+      assertEquals(new Long(175), list.get(3));
+
+       /* get listVector value at index 1 -- the value itself is a listvector */
+      result = accessor.getObject(1);
+      resultSet = (ArrayList<ArrayList<Long>>)result;
+
+      assertEquals(3, resultSet.size());              /* 3 inner lists at index 1 */
+      assertEquals(1, resultSet.get(0).size());       /* size of first inner list */
+      assertEquals(2, resultSet.get(1).size());      /* size of second inner list */
+      assertEquals(3, resultSet.get(2).size());       /* size of third inner list */
+
+      list = resultSet.get(0);
+      assertEquals(new Long(10), list.get(0));
+
+      list = resultSet.get(1);
+      assertEquals(new Long(15), list.get(0));
+      assertEquals(new Long(20), list.get(1));
+
+      list = resultSet.get(2);
+      assertEquals(new Long(25), list.get(0));
+      assertEquals(new Long(30), list.get(1));
+      assertEquals(new Long(35), list.get(2));
+
+      /* check underlying bitVector */
+      assertFalse(accessor.isNull(0));
+      assertFalse(accessor.isNull(1));
+
+      /* check underlying offsetVector */
+      UInt4Vector offsetVector = listVector.getOffsetVector();
+      final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
+
+      /* listVector has 2 lists at index 0 and 3 lists at index 1 */
+      assertEquals(0, offsetAccessor.get(0));
+      assertEquals(2, offsetAccessor.get(1));
+      assertEquals(5, offsetAccessor.get(2));
+    }
+  }
 }

From 3200e914d78773bf4a59c3c0a1e1e7164d77fa64 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 7 Aug 2017 10:44:50 -0400
Subject: [PATCH 0904/1644] ARROW-1327: [Python] Always release GIL before
 calling check_status in Cython

This should prevent deadlock in some multithreaded or subinterpreter contexts. We can be more mindful of this in the future

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #945 from wesm/ARROW-1327 and squashes the following commits:

d690c5b3 [Wes McKinney] Fix some GIL acquisitions
870e5222 [Wes McKinney] Always release GIL before calling check_status in Cython
---
 cpp/src/arrow/python/builtin_convert.cc |  7 ++++-
 cpp/src/arrow/python/numpy_convert.cc   |  4 +++
 python/pyarrow/_parquet.pyx             | 18 ++++++++----
 python/pyarrow/array.pxi                | 37 ++++++++++++++++---------
 python/pyarrow/feather.pxi              |  3 +-
 python/pyarrow/io-hdfs.pxi              | 12 +++++---
 python/pyarrow/io.pxi                   |  6 ++--
 python/pyarrow/table.pxi                |  5 ++--
 8 files changed, 63 insertions(+), 29 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 6eaa37fb8ca93..d3bf76dbed2cc 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -660,6 +660,7 @@ Status AppendPySequence(PyObject* obj, int64_t size,
 }
 
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out) {
+  PyAcquireGIL lock;
   std::shared_ptr<DataType> type;
   int64_t size;
   RETURN_NOT_OK(InferArrowTypeAndSize(obj, &size, &type));
@@ -668,6 +669,7 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
 
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
                          const std::shared_ptr<DataType>& type, int64_t size) {
+  PyAcquireGIL lock;
   // Handle NA / NullType case
   if (type->id() == Type::NA) {
     out->reset(new NullArray(size));
@@ -684,7 +686,10 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
                          const std::shared_ptr<DataType>& type) {
   int64_t size;
-  RETURN_NOT_OK(InferArrowSize(obj, &size));
+  {
+    PyAcquireGIL lock;
+    RETURN_NOT_OK(InferArrowSize(obj, &size));
+  }
   return ConvertPySequence(obj, pool, out, type, size);
 }
 
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 95d63b8fecb5b..61192f313d29d 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -202,6 +202,8 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
 #undef TO_ARROW_TYPE_CASE
 
 Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>* out) {
+  PyAcquireGIL lock;
+
   if (!PyArray_Check(ao)) {
     return Status::TypeError("Did not pass ndarray object");
   }
@@ -234,6 +236,8 @@ Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>*
 }
 
 Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out) {
+  PyAcquireGIL lock;
+
   int type_num;
   RETURN_NOT_OK(GetNumPyType(*tensor.type(), &type_num));
   PyArray_Descr* dtype = PyArray_DescrNewFromType(type_num);
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 919e82c109451..65ca468772710 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -558,9 +558,14 @@ cdef class ParquetWriter:
         cdef:
             shared_ptr[FileOutputStream] filestream
             shared_ptr[WriterProperties] properties
+            c_string c_where
+            CMemoryPool* pool
 
         if isinstance(where, six.string_types):
-            check_status(FileOutputStream.Open(tobytes(where), &filestream))
+            c_where = tobytes(where)
+            with nogil:
+                check_status(FileOutputStream.Open(c_where,
+                                                   &filestream))
             self.sink = <shared_ptr[OutputStream]> filestream
         else:
             get_writer(where, &self.sink)
@@ -580,11 +585,12 @@ cdef class ParquetWriter:
         self._set_int96_support(&arrow_properties_builder)
         arrow_properties = arrow_properties_builder.build()
 
-        check_status(
-            FileWriter.Open(deref(schema.schema),
-                            maybe_unbox_memory_pool(memory_pool),
-                            self.sink, properties, arrow_properties,
-                            &self.writer))
+        pool = maybe_unbox_memory_pool(memory_pool)
+        with nogil:
+            check_status(
+                FileWriter.Open(deref(schema.schema), pool,
+                                self.sink, properties, arrow_properties,
+                                &self.writer))
 
     cdef void _set_int96_support(self, ArrowWriterProperties.Builder* props):
         if self.use_deprecated_int96_timestamps:
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 4e0c21c79e1c0..c0c7ac6da09ed 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -44,23 +44,28 @@ def array(object sequence, DataType type=None, MemoryPool memory_pool=None,
     cdef:
         shared_ptr[CArray] sp_array
         CMemoryPool* pool
+        int64_t c_size
 
     pool = maybe_unbox_memory_pool(memory_pool)
     if type is None:
-        check_status(ConvertPySequence(sequence, pool, &sp_array))
+        with nogil:
+            check_status(ConvertPySequence(sequence, pool, &sp_array))
     else:
         if size is None:
-            check_status(
-                ConvertPySequence(
-                    sequence, pool, &sp_array, type.sp_type
+            with nogil:
+                check_status(
+                    ConvertPySequence(
+                        sequence, pool, &sp_array, type.sp_type
+                    )
                 )
-            )
         else:
-            check_status(
-                ConvertPySequence(
-                    sequence, pool, &sp_array, type.sp_type, size
+            c_size = size
+            with nogil:
+                check_status(
+                    ConvertPySequence(
+                        sequence, pool, &sp_array, type.sp_type, c_size
+                    )
                 )
-            )
 
     return pyarrow_wrap_array(sp_array)
 
@@ -91,7 +96,8 @@ cdef class Array:
         self.type = pyarrow_wrap_data_type(self.sp_array.get().type())
 
     def _debug_print(self):
-        check_status(DebugPrint(deref(self.ap), 0))
+        with nogil:
+            check_status(DebugPrint(deref(self.ap), 0))
 
     @staticmethod
     def from_pandas(obj, mask=None, DataType type=None,
@@ -185,7 +191,9 @@ cdef class Array:
                 values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
 
             if type is None:
-                check_status(NumPyDtypeToArrow(values.dtype, &c_type))
+                dtype = values.dtype
+                with nogil:
+                    check_status(NumPyDtypeToArrow(dtype, &c_type))
             else:
                 c_type = type.sp_type
 
@@ -319,7 +327,9 @@ strides: {2}""".format(self.type, self.shape, self.strides)
     @staticmethod
     def from_numpy(obj):
         cdef shared_ptr[CTensor] ctensor
-        check_status(NdarrayToTensor(c_default_memory_pool(), obj, &ctensor))
+        with nogil:
+            check_status(NdarrayToTensor(c_default_memory_pool(), obj,
+                                         &ctensor))
         return pyarrow_wrap_tensor(ctensor)
 
     def to_numpy(self):
@@ -329,7 +339,8 @@ strides: {2}""".format(self.type, self.shape, self.strides)
         cdef:
             PyObject* out
 
-        check_status(TensorToNdarray(deref(self.tp), self, &out))
+        with nogil:
+            check_status(TensorToNdarray(deref(self.tp), self, &out))
         return PyObject_to_object(out)
 
     def equals(self, Tensor other):
diff --git a/python/pyarrow/feather.pxi b/python/pyarrow/feather.pxi
index 6faf2f9c69c7b..6a1fa30ba63e7 100644
--- a/python/pyarrow/feather.pxi
+++ b/python/pyarrow/feather.pxi
@@ -44,7 +44,8 @@ cdef class FeatherWriter:
         if self.num_rows < 0:
             self.num_rows = 0
         self.writer.get().SetNumRows(self.num_rows)
-        check_status(self.writer.get().Finalize())
+        with nogil:
+            check_status(self.writer.get().Finalize())
 
     def write_array(self, object name, object col, object mask=None):
         cdef Array arr
diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi
index 27e9948b3a4ea..e6285e465d2be 100644
--- a/python/pyarrow/io-hdfs.pxi
+++ b/python/pyarrow/io-hdfs.pxi
@@ -29,7 +29,8 @@ except ImportError:
 
 def have_libhdfs():
     try:
-        check_status(HaveLibHdfs())
+        with nogil:
+            check_status(HaveLibHdfs())
         return True
     except:
         return False
@@ -37,7 +38,8 @@ def have_libhdfs():
 
 def have_libhdfs3():
     try:
-        check_status(HaveLibHdfs3())
+        with nogil:
+            check_status(HaveLibHdfs3())
         return True
     except:
         return False
@@ -73,10 +75,12 @@ cdef class HadoopFileSystem:
             conf.kerb_ticket = tobytes(kerb_ticket)
 
         if driver == 'libhdfs':
-            check_status(HaveLibHdfs())
+            with nogil:
+                check_status(HaveLibHdfs())
             conf.driver = HdfsDriver_LIBHDFS
         else:
-            check_status(HaveLibHdfs3())
+            with nogil:
+                check_status(HaveLibHdfs3())
             conf.driver = HdfsDriver_LIBHDFS3
 
         with nogil:
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index cccb1736be6fa..eda8de730281d 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -447,7 +447,8 @@ cdef class MemoryMappedFile(NativeFile):
         else:
             raise ValueError('Invalid file mode: {0}'.format(mode))
 
-        check_status(CMemoryMappedFile.Open(c_path, c_mode, &handle))
+        with nogil:
+            check_status(CMemoryMappedFile.Open(c_path, c_mode, &handle))
 
         self.wr_file = <shared_ptr[OutputStream]> handle
         self.rd_file = <shared_ptr[RandomAccessFile]> handle
@@ -642,7 +643,8 @@ cdef class BufferOutputStream(NativeFile):
         self.is_open = True
 
     def get_result(self):
-        check_status(self.wr_file.get().Close())
+        with nogil:
+            check_status(self.wr_file.get().Close())
         self.is_open = False
         return pyarrow_wrap_buffer(<shared_ptr[CBuffer]> self.buffer)
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index e33c9ba2f20e9..b9b08998b3372 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -168,8 +168,9 @@ cdef class Column:
         cdef:
             PyObject* out
 
-        check_status(libarrow.ConvertColumnToPandas(self.sp_column,
-                                                    self, &out))
+        with nogil:
+            check_status(libarrow.ConvertColumnToPandas(self.sp_column,
+                                                        self, &out))
 
         return pd.Series(wrap_array_output(out), name=self.name)
 

From 619472ec0a6256fc6ead491fb12881b97f3acec3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 7 Aug 2017 10:45:40 -0400
Subject: [PATCH 0905/1644] ARROW-1225: [Python] Decode bytes to utf8 unicode
 if possible when passing explicit utf8 type to pyarrow.array

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #946 from wesm/ARROW-1225 and squashes the following commits:

aa737b11 [Wes McKinney] Clearer error message
2f439285 [Wes McKinney] Decode bytes to utf8 unicode if possible when passing explicit utf8 type in pyarrow.array
---
 cpp/src/arrow/python/builtin_convert.cc      | 22 ++++++++++++++------
 python/pyarrow/tests/test_convert_builtin.py | 13 ++++++++++++
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index d3bf76dbed2cc..218fe2925fd86 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -530,14 +530,24 @@ class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter>
     const char* bytes;
     Py_ssize_t length;
 
-    if (item.obj() == Py_None) {
+    PyObject* obj = item.obj();
+    if (obj == Py_None) {
       return typed_builder_->AppendNull();
-    } else if (!PyUnicode_Check(item.obj())) {
-      return Status::Invalid("Non-unicode value encountered");
+    } else if (PyBytes_Check(obj)) {
+      tmp.reset(PyUnicode_FromStringAndSize(PyBytes_AS_STRING(obj),
+                                            PyBytes_GET_SIZE(obj)));
+      RETURN_IF_PYERROR();
+      bytes_obj = obj;
+    } else if (!PyUnicode_Check(obj)) {
+      PyObjectStringify stringified(obj);
+      std::stringstream ss;
+      ss << "Non bytes/unicode value encountered: " << stringified.bytes;
+      return Status::Invalid(ss.str());
+    } else {
+      tmp.reset(PyUnicode_AsUTF8String(obj));
+      RETURN_IF_PYERROR();
+      bytes_obj = tmp.obj();
     }
-    tmp.reset(PyUnicode_AsUTF8String(item.obj()));
-    RETURN_IF_PYERROR();
-    bytes_obj = tmp.obj();
 
     // No error checking
     length = PyBytes_GET_SIZE(bytes_obj);
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index ec261595585c2..d18ed9506bbbb 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -16,6 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import pytest
+
 from pyarrow.compat import unittest, u  # noqa
 import pyarrow as pa
 
@@ -140,6 +142,17 @@ def test_bytes(self):
         assert arr.type == pa.binary()
         assert arr.to_pylist() == [b'foo', u1, None]
 
+    def test_utf8_to_unicode(self):
+        # ARROW-1225
+        data = [b'foo', None, b'bar']
+        arr = pa.array(data, type=pa.string())
+        assert arr[0].as_py() == u'foo'
+
+        # test a non-utf8 unicode string
+        val = (u'mañana').encode('utf-16-le')
+        with pytest.raises(pa.ArrowException):
+            pa.array([val], type=pa.string())
+
     def test_fixed_size_bytes(self):
         data = [b'foof', None, b'barb', b'2346']
         arr = pa.array(data, type=pa.binary(4))

From c0acb86540ec01b62fb4d787ac50438b0ecf39e3 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Mon, 7 Aug 2017 10:49:03 -0400
Subject: [PATCH 0906/1644] ARROW-1333: [Plasma] Example code for using Plasma
 to sort a DataFrame

This provides the example code used to generate the speedup mentioned in #940. It is also an end-to-end example using the Plasma object store.

Author: Robert Nishihara <robertnishihara@gmail.com>
Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #942 from robertnishihara/sortdf and squashes the following commits:

a94ce907 [Philipp Moritz] add licenses
221b3362 [Robert Nishihara] Small cleanup.
0cba9904 [Robert Nishihara] Small naming changes.
94e85ee8 [Philipp Moritz] move files
07a53b36 [Philipp Moritz] add experimental setup
b39caacc [Philipp Moritz] remove original multimerge function and document multimerge2d
4df2accf [Philipp Moritz] batch calls to object store
0f7c44c6 [Philipp Moritz] small cleanups
8d997786 [Robert Nishihara] Fix indentation in cython.
f516ab7b [Robert Nishihara] Small fixes.
1f74047d [Robert Nishihara] Speed up Cython
6ef8a776 [Robert Nishihara] remove some printing
e830587a [Robert Nishihara] Fix bug.
a92996a2 [Robert Nishihara] Update plasma executable path.
fc6707f4 [Robert Nishihara] Fix sorting example (fortran versus C arrays).
5d3c74aa [Robert Nishihara] Almost working sorting dataframe.
---
 python/examples/plasma/sorting/multimerge.pyx | 102 +++++++++
 python/examples/plasma/sorting/setup.py       |  27 +++
 python/examples/plasma/sorting/sort_df.py     | 204 ++++++++++++++++++
 3 files changed, 333 insertions(+)
 create mode 100644 python/examples/plasma/sorting/multimerge.pyx
 create mode 100644 python/examples/plasma/sorting/setup.py
 create mode 100644 python/examples/plasma/sorting/sort_df.py

diff --git a/python/examples/plasma/sorting/multimerge.pyx b/python/examples/plasma/sorting/multimerge.pyx
new file mode 100644
index 0000000000000..6dd5aaef95cb9
--- /dev/null
+++ b/python/examples/plasma/sorting/multimerge.pyx
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from libc.stdint cimport uintptr_t
+from libcpp.vector cimport vector
+from libcpp.pair cimport pair
+
+cimport numpy as np
+import numpy as np
+
+
+cdef extern from "<queue>" namespace "std" nogil:
+    cdef cppclass priority_queue[T]:
+        priority_queue() except +
+        priority_queue(priority_queue&) except +
+        bint empty()
+        void pop()
+        void push(T&)
+        size_t size()
+        T& top()
+
+
+def multimerge2d(*arrays):
+    """Merge a list of sorted 2d arrays into a sorted 2d array.
+
+    This assumes C style ordering for both input and output arrays. For
+    each input array we have array[i,0] <= array[i+1,0] and for the output
+    array the same will hold.
+    
+    Ideally this code would be simpler and also support both C style
+    and Fortran style ordering.
+    """
+    cdef int num_arrays = len(arrays)
+    assert num_arrays > 0
+
+    cdef int num_cols = arrays[0].shape[1]
+
+    for i in range(num_arrays):
+        assert arrays[i].ndim == 2
+        assert arrays[i].dtype == np.float64
+        assert arrays[i].shape[1] == num_cols
+        assert not np.isfortran(arrays[i])
+
+    cdef vector[double*] data
+
+    # The indices vector keeps track of the index of the next row to process in
+    # each array.
+    cdef vector[int] indices = num_arrays * [0]
+
+    # The sizes vector stores the total number of elements that each array has.
+    cdef vector[int] sizes
+
+    cdef priority_queue[pair[double, int]] queue
+    cdef pair[double, int] top
+    cdef int num_rows = sum([array.shape[0] for array in arrays])
+    cdef np.ndarray[np.float64_t, ndim=2] result = np.zeros(
+        (num_rows, num_cols), dtype=np.float64)
+    cdef double* result_ptr = <double*> np.PyArray_DATA(result)
+    for i in range(num_arrays):
+        if arrays[i].size > 0:
+            sizes.push_back(arrays[i].size)
+            data.push_back(<double*> np.PyArray_DATA(arrays[i]))
+            queue.push(pair[double, int](-data[i][0], i))
+
+    cdef int curr_idx = 0
+    cdef int j
+    cdef int col = 0
+
+    for j in range(num_rows):
+        top = queue.top()
+        for col in range(num_cols):
+            result_ptr[curr_idx + col] = (
+                data[top.second][indices[top.second] + col])
+
+        indices[top.second] += num_cols
+        curr_idx += num_cols
+
+        queue.pop()
+        if indices[top.second] < sizes[top.second]:
+            queue.push(
+                pair[double, int](-data[top.second][indices[top.second]],
+                                  top.second))
+
+    return result
diff --git a/python/examples/plasma/sorting/setup.py b/python/examples/plasma/sorting/setup.py
new file mode 100644
index 0000000000000..a578085a8e4cc
--- /dev/null
+++ b/python/examples/plasma/sorting/setup.py
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(
+    name="multimerge",
+    extra_compile_args=["-O3", "-mtune=native", "-march=native"],
+    ext_modules=cythonize("multimerge.pyx"),
+    include_dirs=[np.get_include()],
+)
diff --git a/python/examples/plasma/sorting/sort_df.py b/python/examples/plasma/sorting/sort_df.py
new file mode 100644
index 0000000000000..03cfd13c6d76f
--- /dev/null
+++ b/python/examples/plasma/sorting/sort_df.py
@@ -0,0 +1,204 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from multiprocessing import Pool
+import numpy as np
+import os
+import pandas as pd
+import pyarrow as pa
+import pyarrow.plasma as plasma
+import subprocess
+import time
+
+import multimerge
+
+# To run this example, you will first need to run "python setup.py install" in
+# this directory to build the Cython module.
+#
+# You will only see speedups if you run this code on more data, this is just a
+# small example that can run on a laptop.
+#
+# The values we used to get a speedup (on a m4.10xlarge instance on EC2) were
+#     object_store_size = 84 * 10 ** 9
+#     num_cores = 20
+#     num_rows = 10 ** 9
+#     num_cols = 1
+
+client = None
+object_store_size = 2 * 10 ** 9 # 2 GB
+num_cores = 8
+num_rows = 200000
+num_cols = 2
+column_names = [str(i) for i in range(num_cols)]
+column_to_sort = column_names[0]
+
+
+# Connect to clients
+def connect():
+    global client
+    client = plasma.connect('/tmp/store', '', 0)
+    np.random.seed(int(time.time() * 10e7) % 10000000)
+
+
+def put_df(df):
+    record_batch = pa.RecordBatch.from_pandas(df)
+
+    # Get size of record batch and schema
+    mock_sink = pa.MockOutputStream()
+    stream_writer = pa.RecordBatchStreamWriter(mock_sink, record_batch.schema)
+    stream_writer.write_batch(record_batch)
+    data_size = mock_sink.size()
+
+    # Generate an ID and allocate a buffer in the object store for the
+    # serialized DataFrame
+    object_id = plasma.ObjectID(np.random.bytes(20))
+    buf = client.create(object_id, data_size)
+
+    # Write the serialized DataFrame to the object store
+    sink = pa.FixedSizeBufferOutputStream(buf)
+    stream_writer = pa.RecordBatchStreamWriter(sink, record_batch.schema)
+    stream_writer.write_batch(record_batch)
+
+    # Seal the object
+    client.seal(object_id)
+
+    return object_id
+
+
+def get_dfs(object_ids):
+    """Retrieve dataframes from the object store given their object IDs."""
+    buffers = client.get(object_ids)
+    return [pa.RecordBatchStreamReader(buf).read_next_batch().to_pandas()
+            for buf in buffers]
+
+
+def local_sort(object_id):
+    """Sort a partition of a dataframe."""
+    # Get the dataframe from the object store.
+    [df] = get_dfs([object_id])
+    # Sort the dataframe.
+    sorted_df = df.sort_values(by=column_to_sort)
+    # Get evenly spaced values from the dataframe.
+    indices = np.linspace(0, len(df) - 1, num=num_cores, dtype=np.int64)
+    # Put the sorted dataframe in the object store and return the corresponding
+    # object ID as well as the sampled values.
+    return put_df(sorted_df), sorted_df.as_matrix().take(indices)
+
+
+def local_partitions(object_id_and_pivots):
+    """Take a sorted partition of a dataframe and split it into more pieces."""
+    object_id, pivots = object_id_and_pivots
+    [df] = get_dfs([object_id])
+    split_at = df[column_to_sort].searchsorted(pivots)
+    split_at = [0] + list(split_at) + [len(df)]
+    # Partition the sorted dataframe and put each partition into the object
+    # store.
+    return [put_df(df[i:j]) for i, j in zip(split_at[:-1], split_at[1:])]
+
+
+def merge(object_ids):
+    """Merge a number of sorted dataframes into a single sorted dataframe."""
+    dfs = get_dfs(object_ids)
+
+    # In order to use our multimerge code, we have to convert the arrays from
+    # the Fortran format to the C format.
+    arrays = [np.ascontiguousarray(df.as_matrix()) for df in dfs]
+    for a in arrays:
+        assert a.dtype == np.float64
+        assert not np.isfortran(a)
+
+    # Filter out empty arrays.
+    arrays = [a for a in arrays if a.shape[0] > 0]
+
+    if len(arrays) == 0:
+        return None
+
+    resulting_array = multimerge.multimerge2d(*arrays)
+    merged_df2 = pd.DataFrame(resulting_array, columns=column_names)
+
+    return put_df(merged_df2)
+
+
+if __name__ == '__main__':
+    # Start the plasma store.
+    p = subprocess.Popen(['plasma_store',
+                          '-s', '/tmp/store',
+                          '-m', str(object_store_size)])
+
+    # Connect to the plasma store.
+    connect()
+
+    # Connect the processes in the pool.
+    pool = Pool(initializer=connect, initargs=(), processes=num_cores)
+
+    # Create a DataFrame from a numpy array.
+    df = pd.DataFrame(np.random.randn(num_rows, num_cols),
+                      columns=column_names)
+
+    partition_ids = [put_df(partition) for partition
+                     in np.split(df, num_cores)]
+
+    # Begin timing the parallel sort example.
+    parallel_sort_start = time.time()
+
+    # Sort each partition and subsample them. The subsampled values will be
+    # used to create buckets.
+    sorted_df_ids, pivot_groups = list(zip(*pool.map(local_sort,
+                                                     partition_ids)))
+
+    # Choose the pivots.
+    all_pivots = np.concatenate(pivot_groups)
+    indices = np.linspace(0, len(all_pivots) - 1, num=num_cores,
+                          dtype=np.int64)
+    pivots = np.take(np.sort(all_pivots), indices)
+
+    # Break all of the sorted partitions into even smaller partitions. Group
+    # the object IDs from each bucket together.
+    results = list(zip(*pool.map(local_partitions,
+                                 zip(sorted_df_ids,
+                                     len(sorted_df_ids) * [pivots]))))
+
+    # Merge each of the buckets and store the results in the object store.
+    object_ids = pool.map(merge, results)
+
+    resulting_ids = [object_id for object_id in object_ids
+                     if object_id is not None]
+
+    # Stop timing the paralle sort example.
+    parallel_sort_end = time.time()
+
+    print('Parallel sort took {} seconds.'
+          .format(parallel_sort_end - parallel_sort_start))
+
+    serial_sort_start = time.time()
+
+    original_sorted_df = df.sort_values(by=column_to_sort)
+
+    serial_sort_end = time.time()
+
+    # Check that we sorted the DataFrame properly.
+
+    sorted_dfs = get_dfs(resulting_ids)
+    sorted_df = pd.concat(sorted_dfs)
+
+    print('Serial sort took {} seconds.'
+          .format(serial_sort_end - serial_sort_start))
+
+    assert np.allclose(sorted_df.values, original_sorted_df.values)
+
+    # Kill the object store.
+    p.kill()

From f9d983391695a22ca400da9135ddfbeb041859ca Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 7 Aug 2017 15:04:29 -0400
Subject: [PATCH 0907/1644] ARROW-1283: [JAVA] Allow VectorSchemaRoot to close
 more than once

This change allows the VectorSchemaRoot/FieldVectors to close more than once, even if the allocator has already been closed.  Before, an empty ArrowBuf was created during closing which required the allocator to not be closed, however this empty buffer is not needed once the FieldVector has been closed.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #898 from BryanCutler/java-vectorSchemaRoot-close-twice-ARROW-1283 and squashes the following commits:

2921d848 [Bryan Cutler] removed resolved comment
3b3718b8 [Bryan Cutler] Merge remote-tracking branch 'upstream/master' into java-vectorSchemaRoot-close-twice-ARROW-1283
e992fc79 [Bryan Cutler] BaseDataValueVector.close will now just clear, which releases previous and assigns an empty buffer
8ecfce2a [Bryan Cutler] Merge remote-tracking branch 'upstream/master' into java-vectorSchemaRoot-close-twice-ARROW-1283
ca38d3d8 [Bryan Cutler] use clear to release data, ensure that an empty buffer is never allocated again after closing
10ff7c36 [Bryan Cutler] Added regression test
---
 .../org/apache/arrow/memory/BaseAllocator.java   |  3 ---
 .../apache/arrow/vector/BaseDataValueVector.java | 16 +---------------
 .../org/apache/arrow/vector/TestValueVector.java | 10 ++++++++++
 3 files changed, 11 insertions(+), 18 deletions(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index be0ba77f5b2fa..b38cf679e2a12 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -171,7 +171,6 @@ public String getName() {
 
   @Override
   public ArrowBuf getEmpty() {
-    assertOpen();
     return empty;
   }
 
@@ -236,8 +235,6 @@ public ArrowBuf buffer(final int initialRequestSize) {
   }
 
   private ArrowBuf createEmpty() {
-    assertOpen();
-
     return new ArrowBuf(new AtomicInteger(), null, AllocationManager.EMPTY, null, null, 0, 0, true);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 0fea719da8815..88e02495bfc99 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -59,8 +59,6 @@ public static List<ArrowBuf> unload(List<BufferBacked> vectors) {
     return result;
   }
 
-  // TODO: Nullable vectors extend BaseDataValueVector but do not use the data field
-  // We should fix the inheritance tree
   protected ArrowBuf data;
 
   public BaseDataValueVector(String name, BufferAllocator allocator) {
@@ -70,23 +68,11 @@ public BaseDataValueVector(String name, BufferAllocator allocator) {
 
   @Override
   public void clear() {
-    if (data != null) {
-      data.release();
-    }
+    data.release();
     data = allocator.getEmpty();
     super.clear();
   }
 
-  @Override
-  public void close() {
-    clear();
-    if (data != null) {
-      data.release();
-      data = null;
-    }
-    super.close();
-  }
-
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
     return getTransferPair(ref, allocator);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 0f41c2dd790e1..159d534a31072 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -736,6 +736,16 @@ public void testFillEmptiesUsage() {
     }
   }
 
+  @Test
+  public void testMultipleClose() {
+    BufferAllocator vectorAllocator = allocator.newChildAllocator("vector_allocator", 0, Long.MAX_VALUE);
+    NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, vectorAllocator);
+    vector.close();
+    vectorAllocator.close();
+    vector.close();
+    vectorAllocator.close();
+  }
+
   public static void setBytes(int index, byte[] bytes, NullableVarCharVector vector) {
     final int currentOffset = vector.values.offsetVector.getAccessor().get(index);
 

From 7a4026a2b43b164ac28c96e1e796c02ca9efe496 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Mon, 7 Aug 2017 15:05:48 -0400
Subject: [PATCH 0908/1644] ARROW-1304: [Java] Fix Indentation,
 WhitespaceAround and EmptyLineSeparator checkstyle warnings in Java

Author: Li Jin <ice.xelloss@gmail.com>

Closes #930 from icexelloss/checkstyle-fix-ARROW-1304 and squashes the following commits:

0ba9e03f [Li Jin] ARROW-1304: [Java] Reformat java code with google_checks.xml to improve checkstyle
---
 .../main/java/io/netty/buffer/ArrowBuf.java   |    4 +-
 .../org/apache/arrow/memory/Accountant.java   |    4 +-
 .../arrow/memory/AllocationManager.java       |    1 +
 .../apache/arrow/memory/BaseAllocator.java    |    3 +-
 .../apache/arrow/memory/BufferAllocator.java  |    6 +-
 .../arrow/memory/util/HistoricalLog.java      |    7 +-
 .../apache/arrow/memory/TestAccountant.java   |    1 +
 .../arrow/memory/TestBaseAllocator.java       |   91 +-
 .../apache/arrow/memory/TestEndianess.java    |    2 +
 .../org/apache/arrow/tools/EchoServer.java    |    4 +-
 .../org/apache/arrow/tools/FileRoundtrip.java |    1 +
 .../org/apache/arrow/tools/Integration.java   |    4 +-
 .../apache/arrow/tools/EchoServerTest.java    |   20 +-
 .../apache/arrow/vector/AddOrGetResult.java   |    1 +
 .../apache/arrow/vector/AllocationHelper.java |   18 +-
 .../arrow/vector/BaseDataValueVector.java     |    6 +-
 .../apache/arrow/vector/BaseValueVector.java  |   17 +-
 .../org/apache/arrow/vector/BitVector.java    |   45 +-
 .../org/apache/arrow/vector/BufferBacked.java |    1 +
 .../org/apache/arrow/vector/FieldVector.java  |    7 +-
 .../apache/arrow/vector/FixedWidthVector.java |   11 +-
 .../apache/arrow/vector/NullableVector.java   |    1 +
 .../NullableVectorDefinitionSetter.java       |    1 +
 .../arrow/vector/SchemaChangeCallBack.java    |    1 +
 .../org/apache/arrow/vector/ValueVector.java  |   29 +-
 .../arrow/vector/VariableWidthVector.java     |    8 +-
 .../org/apache/arrow/vector/VectorLoader.java |    7 +-
 .../apache/arrow/vector/VectorSchemaRoot.java |    5 +-
 .../apache/arrow/vector/VectorTrimmer.java    |    1 +
 .../apache/arrow/vector/VectorUnloader.java   |    1 +
 .../org/apache/arrow/vector/ZeroVector.java   |   34 +-
 .../complex/AbstractContainerVector.java      |    6 +-
 .../vector/complex/AbstractMapVector.java     |   54 +-
 .../complex/BaseRepeatedValueVector.java      |   19 +-
 .../vector/complex/EmptyValuePopulator.java   |    7 +-
 .../vector/complex/FixedSizeListVector.java   |    7 +-
 .../arrow/vector/complex/ListVector.java      |   15 +-
 .../arrow/vector/complex/MapVector.java       |   21 +-
 .../vector/complex/NullableMapVector.java     |   17 +-
 .../arrow/vector/complex/Positionable.java    |    2 +
 .../vector/complex/PromotableVector.java      |    1 +
 .../complex/RepeatedFixedWidthVectorLike.java |    5 +-
 .../vector/complex/RepeatedValueVector.java   |    7 +-
 .../RepeatedVariableWidthVectorLike.java      |    8 +-
 .../arrow/vector/complex/StateTool.java       |    1 +
 .../vector/complex/VectorWithOrdinal.java     |    1 +
 .../complex/impl/AbstractBaseReader.java      |    9 +-
 .../complex/impl/AbstractBaseWriter.java      |    1 +
 .../complex/impl/ComplexWriterImpl.java       |  145 +-
 .../complex/impl/MapOrListWriterImpl.java     |    1 +
 .../complex/impl/NullableMapReaderImpl.java   |   11 +-
 .../impl/NullableMapWriterFactory.java        |   31 +-
 .../vector/complex/impl/PromotableWriter.java |    7 +-
 .../complex/impl/SingleListReaderImpl.java    |    3 +-
 .../complex/impl/SingleMapReaderImpl.java     |   25 +-
 .../impl/UnionFixedSizeListReader.java        |    1 +
 .../vector/complex/impl/UnionListReader.java  |    1 +
 .../vector/complex/reader/FieldReader.java    |    2 +-
 .../vector/complex/writer/FieldWriter.java    |    2 +
 .../arrow/vector/dictionary/Dictionary.java   |    9 +-
 .../vector/dictionary/DictionaryEncoder.java  |   11 +-
 .../vector/dictionary/DictionaryProvider.java |    3 +-
 .../apache/arrow/vector/file/ArrowBlock.java  |   19 +-
 .../arrow/vector/file/ArrowFileReader.java    |    3 +-
 .../arrow/vector/file/ArrowFileWriter.java    |    3 +-
 .../apache/arrow/vector/file/ArrowFooter.java |   30 +-
 .../apache/arrow/vector/file/ArrowMagic.java  |    1 +
 .../apache/arrow/vector/file/ArrowReader.java |   22 +-
 .../apache/arrow/vector/file/ArrowWriter.java |   17 +-
 .../file/InvalidArrowFileException.java       |    1 +
 .../apache/arrow/vector/file/ReadChannel.java |   17 +-
 .../vector/file/SeekableReadChannel.java      |   23 +-
 .../arrow/vector/file/WriteChannel.java       |   11 +-
 .../vector/file/json/JsonFileReader.java      |  175 +--
 .../vector/file/json/JsonFileWriter.java      |   32 +-
 .../arrow/vector/holders/ComplexHolder.java   |    1 +
 .../vector/holders/RepeatedListHolder.java    |    3 +-
 .../vector/holders/RepeatedMapHolder.java     |    3 +-
 .../arrow/vector/holders/UnionHolder.java     |    1 +
 .../arrow/vector/holders/ValueHolder.java     |    1 +
 .../arrow/vector/schema/ArrowBuffer.java      |   19 +-
 .../vector/schema/ArrowDictionaryBatch.java   |   80 +-
 .../arrow/vector/schema/ArrowFieldNode.java   |    3 +-
 .../arrow/vector/schema/ArrowMessage.java     |   14 +-
 .../arrow/vector/schema/ArrowRecordBatch.java |   17 +-
 .../arrow/vector/schema/ArrowVectorType.java  |    6 +-
 .../arrow/vector/schema/FBSerializable.java   |    1 +
 .../arrow/vector/schema/FBSerializables.java  |    1 +
 .../arrow/vector/schema/TypeLayout.java       |   87 +-
 .../arrow/vector/schema/VectorLayout.java     |   35 +-
 .../vector/stream/ArrowStreamReader.java      |   57 +-
 .../vector/stream/ArrowStreamWriter.java      |   32 +-
 .../vector/stream/MessageSerializer.java      |  101 +-
 .../apache/arrow/vector/types/DateUnit.java   |    2 +
 .../vector/types/FloatingPointPrecision.java  |    2 +
 .../arrow/vector/types/IntervalUnit.java      |    2 +
 .../apache/arrow/vector/types/TimeUnit.java   |    2 +
 .../org/apache/arrow/vector/types/Types.java  |   85 +-
 .../apache/arrow/vector/types/UnionMode.java  |    2 +
 .../vector/types/pojo/DictionaryEncoding.java |    1 +
 .../apache/arrow/vector/types/pojo/Field.java |   13 +-
 .../arrow/vector/types/pojo/FieldType.java    |    4 +
 .../arrow/vector/types/pojo/Schema.java       |    7 +-
 .../ByteArrayReadableSeekableByteChannel.java |    3 +-
 .../vector/util/ByteFunctionHelpers.java      |   31 +-
 .../apache/arrow/vector/util/CallBack.java    |    1 +
 .../apache/arrow/vector/util/DateUtility.java | 1270 ++++++++---------
 .../arrow/vector/util/DecimalUtility.java     |   59 +-
 .../arrow/vector/util/DictionaryUtility.java  |    6 +-
 .../vector/util/JsonStringArrayList.java      |    3 +-
 .../arrow/vector/util/JsonStringHashMap.java  |    7 +-
 .../arrow/vector/util/MapWithOrdinal.java     |    9 +-
 .../util/OversizedAllocationException.java    |    2 +-
 .../util/SchemaChangeRuntimeException.java    |    1 +
 .../org/apache/arrow/vector/util/Text.java    |  258 ++--
 .../arrow/vector/util/TransferPair.java       |    4 +
 .../apache/arrow/vector/util/Validator.java   |    7 +-
 .../java/org/joda/time/LocalDateTimes.java    |    2 +-
 .../arrow/vector/DirtyRootAllocator.java      |    1 +
 .../apache/arrow/vector/TestBitVector.java    |   36 +-
 .../vector/TestBufferOwnershipTransfer.java   |   11 +-
 .../arrow/vector/TestDecimalVector.java       |    2 +
 .../arrow/vector/TestDictionaryVector.java    |    7 +-
 .../arrow/vector/TestFixedSizeListVector.java |    1 +
 .../apache/arrow/vector/TestListVector.java   |   83 +-
 ...TestOversizedAllocationForValueVector.java |    2 +-
 .../apache/arrow/vector/TestUnionVector.java  |   37 +-
 .../org/apache/arrow/vector/TestUtils.java    |    3 +-
 .../apache/arrow/vector/TestValueVector.java  |   13 +-
 .../arrow/vector/TestVectorReAlloc.java       |    1 +
 .../arrow/vector/TestVectorUnloadLoad.java    |   26 +-
 .../complex/impl/TestPromotableWriter.java    |    3 +-
 .../complex/writer/TestComplexWriter.java     |   17 +-
 .../arrow/vector/file/BaseFileTest.java       |   79 +-
 .../arrow/vector/file/TestArrowFile.java      |   51 +-
 .../arrow/vector/file/TestArrowFooter.java    |    3 +-
 .../vector/file/TestArrowReaderWriter.java    |    9 +-
 .../arrow/vector/file/TestArrowStream.java    |    3 +-
 .../vector/file/TestArrowStreamPipe.java      |   21 +-
 .../arrow/vector/file/json/TestJSONFile.java  |   11 +-
 .../apache/arrow/vector/pojo/TestConvert.java |    9 +-
 .../vector/stream/MessageSerializerTest.java  |    7 +-
 .../arrow/vector/types/pojo/TestSchema.java   |   27 +-
 .../arrow/vector/util/TestValidator.java      |   57 +-
 144 files changed, 2102 insertions(+), 1792 deletions(-)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index 09886a6ffe316..6d17430c66966 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -406,12 +406,12 @@ public ByteBuffer internalNioBuffer(int index, int length) {
 
   @Override
   public ByteBuffer[] nioBuffers() {
-    return new ByteBuffer[]{nioBuffer()};
+    return new ByteBuffer[] {nioBuffer()};
   }
 
   @Override
   public ByteBuffer[] nioBuffers(int index, int length) {
-    return new ByteBuffer[]{nioBuffer(index, length)};
+    return new ByteBuffer[] {nioBuffer(index, length)};
   }
 
   @Override
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
index 89329b2766357..5bd6b9fe37956 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -249,9 +249,9 @@ public long getPeakMemoryAllocation() {
     return peakAllocation.get();
   }
 
-  public long getHeadroom(){
+  public long getHeadroom() {
     long localHeadroom = allocationLimit.get() - locallyHeldMemory.get();
-    if(parent == null){
+    if (parent == null) {
       return localHeadroom;
     }
 
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index 70ca1dc32a1b3..c528937bfdcaa 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -328,6 +328,7 @@ private void inc() {
      * Decrement the ledger's reference count. If the ledger is decremented to zero, this ledger
      * should release its
      * ownership back to the AllocationManager
+     *
      * @param decrement amout to decrease the reference count by
      * @return the new reference count
      */
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index b38cf679e2a12..2749b6fe030f4 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -274,8 +274,7 @@ public ArrowBuf buffer(final int initialRequestSize, BufferManager manager) {
         throw new OutOfMemoryException(e);
       }
       throw e;
-    }
-    finally {
+    } finally {
       if (!success) {
         releaseBytes(actualRequestSize);
       }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
index 8a40441863889..b23a6e4bd8507 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -117,9 +117,8 @@ public interface BufferAllocator extends AutoCloseable {
    * Create an allocation reservation. A reservation is a way of building up
    * a request for a buffer whose size is not known in advance. See
    *
-   * @see AllocationReservation
-   *
    * @return the newly created reservation
+   * @see AllocationReservation
    */
   public AllocationReservation newReservation();
 
@@ -128,6 +127,7 @@ public interface BufferAllocator extends AutoCloseable {
    * special because we don't
    * worry about them leaking or managing reference counts on them since they don't actually
    * point to any memory.
+   *
    * @return the empty buffer
    */
   public ArrowBuf getEmpty();
@@ -136,6 +136,7 @@ public interface BufferAllocator extends AutoCloseable {
    * Return the name of this allocator. This is a human readable name that can help debugging.
    * Typically provides
    * coordinates about where this allocator was created
+   *
    * @return the name of the allocator
    */
   public String getName();
@@ -145,6 +146,7 @@ public interface BufferAllocator extends AutoCloseable {
    * that an allocator is
    * over its limit, all consumers of that allocator should aggressively try to addrss the
    * overlimit situation.
+   *
    * @return whether or not this allocator (or one if its parents) is over its limits
    */
   public boolean isOverLimit();
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
index 0452dc9adf256..10a64cd984ea0 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
@@ -97,7 +97,7 @@ public synchronized void recordEvent(final String noteFormat, Object... args) {
    * includes the identifying string provided at construction time, and all the recorded
    * events with their stack traces.
    *
-   * @param sb {@link StringBuilder} to write to
+   * @param sb                {@link StringBuilder} to write to
    * @param includeStackTrace whether to include the stacktrace of each event in the history
    */
   public void buildHistory(final StringBuilder sb, boolean includeStackTrace) {
@@ -106,8 +106,9 @@ public void buildHistory(final StringBuilder sb, boolean includeStackTrace) {
 
   /**
    * build the history and write it to sb
-   * @param sb output
-   * @param indent starting indent (usually "")
+   *
+   * @param sb                output
+   * @param indent            starting indent (usually "")
    * @param includeStackTrace whether to include the stacktrace of each event.
    */
   public synchronized void buildHistory(
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java b/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
index 2624a4a047e7e..100be069fe6d4 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 import static org.junit.Assert.assertEquals;
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index 59b7be87e17be..76f2c501cf4c7 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 import static org.junit.Assert.assertEquals;
@@ -59,13 +60,13 @@ public void checkBuffers() {
 
   @Test
   public void test_privateMax() throws Exception {
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
       final ArrowBuf arrowBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2);
       assertNotNull("allocation failed", arrowBuf1);
 
-      try(final BufferAllocator childAllocator =
-          rootAllocator.newChildAllocator("noLimits", 0, MAX_ALLOCATION)) {
+      try (final BufferAllocator childAllocator =
+               rootAllocator.newChildAllocator("noLimits", 0, MAX_ALLOCATION)) {
         final ArrowBuf arrowBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2);
         assertNotNull("allocation failed", arrowBuf2);
         arrowBuf2.release();
@@ -75,11 +76,11 @@ public void test_privateMax() throws Exception {
     }
   }
 
-  @Test(expected=IllegalStateException.class)
+  @Test(expected = IllegalStateException.class)
   public void testRootAllocator_closeWithOutstanding() throws Exception {
     try {
-      try(final RootAllocator rootAllocator =
-          new RootAllocator(MAX_ALLOCATION)) {
+      try (final RootAllocator rootAllocator =
+               new RootAllocator(MAX_ALLOCATION)) {
         final ArrowBuf arrowBuf = rootAllocator.buffer(512);
         assertNotNull("allocation failed", arrowBuf);
       }
@@ -100,8 +101,8 @@ public void testRootAllocator_closeWithOutstanding() throws Exception {
 
   @Test
   public void testRootAllocator_getEmpty() throws Exception {
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
       final ArrowBuf arrowBuf = rootAllocator.buffer(0);
       assertNotNull("allocation failed", arrowBuf);
       assertEquals("capacity was non-zero", 0, arrowBuf.capacity());
@@ -112,8 +113,8 @@ public void testRootAllocator_getEmpty() throws Exception {
   @Ignore // TODO(DRILL-2740)
   @Test(expected = IllegalStateException.class)
   public void testAllocator_unreleasedEmpty() throws Exception {
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
       @SuppressWarnings("unused")
       final ArrowBuf arrowBuf = rootAllocator.buffer(0);
     }
@@ -121,8 +122,8 @@ public void testAllocator_unreleasedEmpty() throws Exception {
 
   @Test
   public void testAllocator_transferOwnership() throws Exception {
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
       final BufferAllocator childAllocator1 =
           rootAllocator.newChildAllocator("changeOwnership1", 0, MAX_ALLOCATION);
       final BufferAllocator childAllocator2 =
@@ -197,7 +198,7 @@ public void testRootAllocator_createChildAndUse() throws Exception {
     }
   }
 
-  @Test(expected=IllegalStateException.class)
+  @Test(expected = IllegalStateException.class)
   public void testRootAllocator_createChildDontClose() throws Exception {
     try {
       try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
@@ -232,22 +233,22 @@ private static void allocateAndFree(final BufferAllocator allocator) {
 
     final int nBufs = 8;
     final ArrowBuf[] arrowBufs = new ArrowBuf[nBufs];
-    for(int i = 0; i < arrowBufs.length; ++i) {
+    for (int i = 0; i < arrowBufs.length; ++i) {
       ArrowBuf arrowBufi = allocator.buffer(MAX_ALLOCATION / nBufs);
       assertNotNull("allocation failed", arrowBufi);
       arrowBufs[i] = arrowBufi;
     }
-    for(ArrowBuf arrowBufi : arrowBufs) {
+    for (ArrowBuf arrowBufi : arrowBufs) {
       arrowBufi.release();
     }
   }
 
   @Test
   public void testAllocator_manyAllocations() throws Exception {
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
-      try(final BufferAllocator childAllocator =
-          rootAllocator.newChildAllocator("manyAllocations", 0, MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
+      try (final BufferAllocator childAllocator =
+               rootAllocator.newChildAllocator("manyAllocations", 0, MAX_ALLOCATION)) {
         allocateAndFree(childAllocator);
       }
     }
@@ -255,10 +256,10 @@ public void testAllocator_manyAllocations() throws Exception {
 
   @Test
   public void testAllocator_overAllocate() throws Exception {
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
-      try(final BufferAllocator childAllocator =
-          rootAllocator.newChildAllocator("overAllocate", 0, MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
+      try (final BufferAllocator childAllocator =
+               rootAllocator.newChildAllocator("overAllocate", 0, MAX_ALLOCATION)) {
         allocateAndFree(childAllocator);
 
         try {
@@ -273,10 +274,10 @@ public void testAllocator_overAllocate() throws Exception {
 
   @Test
   public void testAllocator_overAllocateParent() throws Exception {
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
-      try(final BufferAllocator childAllocator =
-          rootAllocator.newChildAllocator("overAllocateParent", 0, MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
+      try (final BufferAllocator childAllocator =
+               rootAllocator.newChildAllocator("overAllocateParent", 0, MAX_ALLOCATION)) {
         final ArrowBuf arrowBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2);
         assertNotNull("allocation failed", arrowBuf1);
         final ArrowBuf arrowBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2);
@@ -326,7 +327,7 @@ public void testAllocator_createSlices() throws Exception {
 
       try (final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) {
         try (final BufferAllocator childAllocator2 =
-            childAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) {
+                 childAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) {
           final ArrowBuf arrowBuf1 = childAllocator2.buffer(MAX_ALLOCATION / 8);
           @SuppressWarnings("unused")
           final ArrowBuf arrowBuf2 = arrowBuf1.slice(MAX_ALLOCATION / 16, MAX_ALLOCATION / 16);
@@ -345,8 +346,8 @@ public void testAllocator_createSlices() throws Exception {
   @Test
   public void testAllocator_sliceRanges() throws Exception {
 //    final AllocatorOwner allocatorOwner = new NamedOwner("sliceRanges");
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
       // Populate a buffer with byte values corresponding to their indices.
       final ArrowBuf arrowBuf = rootAllocator.buffer(256);
       assertEquals(256, arrowBuf.capacity());
@@ -362,7 +363,7 @@ public void testAllocator_sliceRanges() throws Exception {
 //      assertEquals(256, slice3.capacity());
 //      assertEquals(256, slice3.writableBytes());
 
-      for(int i = 0; i < 256; ++i) {
+      for (int i = 0; i < 256; ++i) {
         arrowBuf.writeByte(i);
       }
       assertEquals(0, arrowBuf.readerIndex());
@@ -373,18 +374,18 @@ public void testAllocator_sliceRanges() throws Exception {
       final ArrowBuf slice1 = (ArrowBuf) arrowBuf.slice();
       assertEquals(0, slice1.readerIndex());
       assertEquals(256, slice1.readableBytes());
-      for(int i = 0; i < 10; ++i) {
+      for (int i = 0; i < 10; ++i) {
         assertEquals(i, slice1.readByte());
       }
       assertEquals(256 - 10, slice1.readableBytes());
-      for(int i = 0; i < 256; ++i) {
+      for (int i = 0; i < 256; ++i) {
         assertEquals((byte) i, slice1.getByte(i));
       }
 
       final ArrowBuf slice2 = arrowBuf.slice(25, 25);
       assertEquals(0, slice2.readerIndex());
       assertEquals(25, slice2.readableBytes());
-      for(int i = 25; i < 50; ++i) {
+      for (int i = 25; i < 50; ++i) {
         assertEquals(i, slice2.readByte());
       }
 
@@ -404,32 +405,32 @@ public void testAllocator_sliceRanges() throws Exception {
   @Test
   public void testAllocator_slicesOfSlices() throws Exception {
 //    final AllocatorOwner allocatorOwner = new NamedOwner("slicesOfSlices");
-    try(final RootAllocator rootAllocator =
-        new RootAllocator(MAX_ALLOCATION)) {
+    try (final RootAllocator rootAllocator =
+             new RootAllocator(MAX_ALLOCATION)) {
       // Populate a buffer with byte values corresponding to their indices.
       final ArrowBuf arrowBuf = rootAllocator.buffer(256);
-      for(int i = 0; i < 256; ++i) {
+      for (int i = 0; i < 256; ++i) {
         arrowBuf.writeByte(i);
       }
 
       // Slice it up.
       final ArrowBuf slice0 = arrowBuf.slice(0, arrowBuf.capacity());
-      for(int i = 0; i < 256; ++i) {
+      for (int i = 0; i < 256; ++i) {
         assertEquals((byte) i, arrowBuf.getByte(i));
       }
 
       final ArrowBuf slice10 = slice0.slice(10, arrowBuf.capacity() - 10);
-      for(int i = 10; i < 256; ++i) {
+      for (int i = 10; i < 256; ++i) {
         assertEquals((byte) i, slice10.getByte(i - 10));
       }
 
       final ArrowBuf slice20 = slice10.slice(10, arrowBuf.capacity() - 20);
-      for(int i = 20; i < 256; ++i) {
+      for (int i = 20; i < 256; ++i) {
         assertEquals((byte) i, slice20.getByte(i - 20));
       }
 
-      final ArrowBuf slice30 = slice20.slice(10,  arrowBuf.capacity() - 30);
-      for(int i = 30; i < 256; ++i) {
+      final ArrowBuf slice30 = slice20.slice(10, arrowBuf.capacity() - 30);
+      for (int i = 30; i < 256; ++i) {
         assertEquals((byte) i, slice30.getByte(i - 30));
       }
 
@@ -556,8 +557,8 @@ public void testAllocator_transferShared() throws Exception {
   public void testAllocator_unclaimedReservation() throws Exception {
     try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
       try (final BufferAllocator childAllocator1 =
-          rootAllocator.newChildAllocator("unclaimedReservation", 0, MAX_ALLOCATION)) {
-        try(final AllocationReservation reservation = childAllocator1.newReservation()) {
+               rootAllocator.newChildAllocator("unclaimedReservation", 0, MAX_ALLOCATION)) {
+        try (final AllocationReservation reservation = childAllocator1.newReservation()) {
           assertTrue(reservation.add(64));
         }
         rootAllocator.verify();
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestEndianess.java b/java/memory/src/test/java/org/apache/arrow/memory/TestEndianess.java
index 25357dc7b07ef..accd15eb71705 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestEndianess.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestEndianess.java
@@ -15,9 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.memory;
 
 import static org.junit.Assert.assertEquals;
+
 import io.netty.buffer.ByteBuf;
 
 import org.apache.arrow.memory.BufferAllocator;
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
index 24079b62da919..c53f0ea86935e 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
@@ -73,7 +73,9 @@ public void run() throws IOException {
         LOGGER.info("Closed connection with client");
       }
     } catch (java.net.SocketException ex) {
-      if (!closed) throw ex;
+      if (!closed) {
+        throw ex;
+      }
     } finally {
       serverSocket.close();
       LOGGER.info("Server closed.");
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
index 135d4921ed128..7d71b0b8f9d3f 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
@@ -44,6 +44,7 @@ public class FileRoundtrip {
   private final Options options;
   private final PrintStream out;
   private final PrintStream err;
+
   FileRoundtrip(PrintStream out, PrintStream err) {
     this.out = out;
     this.err = err;
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index 7d4c86f81670f..d2b35e65a8172 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -94,7 +94,7 @@ private File validateFile(String type, String fileName, boolean shouldExist) {
   }
 
   static void extractDictionaryEncodings(List<Field> fields, List<DictionaryEncoding> encodings) {
-    for (Field field: fields) {
+    for (Field field : fields) {
       DictionaryEncoding encoding = field.getDictionary();
       if (encoding != null) {
         encodings.add(encoding);
@@ -214,7 +214,7 @@ public void execute(File arrowFile, File jsonFile) throws IOException {
           boolean hasMoreArrow = iterator.hasNext();
           if (hasMoreJSON || hasMoreArrow) {
             throw new IllegalArgumentException("Unexpected RecordBatches. Total: " + totalBatches
-                                               + " J:" + hasMoreJSON + " "
+                + " J:" + hasMoreJSON + " "
                 + "A:" + hasMoreArrow);
           }
         }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index d7f6388721ebb..467965aff95d9 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -158,16 +158,16 @@ public void basicTest() throws InterruptedException, IOException {
   public void testFlatDictionary() throws IOException {
     DictionaryEncoding writeEncoding = new DictionaryEncoding(1L, false, null);
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
-        NullableIntVector writeVector =
-            new NullableIntVector(
-                "varchar",
-                new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
-                allocator);
-        NullableVarCharVector writeDictionaryVector =
-            new NullableVarCharVector(
-                "dict",
-                FieldType.nullable(VARCHAR.getType()),
-                allocator)) {
+         NullableIntVector writeVector =
+             new NullableIntVector(
+                 "varchar",
+                 new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
+                 allocator);
+         NullableVarCharVector writeDictionaryVector =
+             new NullableVarCharVector(
+                 "dict",
+                 FieldType.nullable(VARCHAR.getType()),
+                 allocator)) {
       writeVector.allocateNewSafe();
       NullableIntVector.Mutator mutator = writeVector.getMutator();
       mutator.set(0, 0);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
index 388eb9c447977..a5b5c9d1d6ad2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import com.google.common.base.Preconditions;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
index 15c3a0227c656..2a0f39d0cb59b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import org.apache.arrow.vector.complex.RepeatedFixedWidthVectorLike;
@@ -27,32 +28,33 @@ public static void allocate(ValueVector v, int valueCount, int bytesPerValue) {
     allocate(v, valueCount, bytesPerValue, 5);
   }
 
-  public static void allocatePrecomputedChildCount(ValueVector v, int valueCount, int bytesPerValue, int childValCount){
-    if(v instanceof FixedWidthVector) {
+  public static void allocatePrecomputedChildCount(ValueVector v, int valueCount, int bytesPerValue, int childValCount) {
+    if (v instanceof FixedWidthVector) {
       ((FixedWidthVector) v).allocateNew(valueCount);
     } else if (v instanceof VariableWidthVector) {
       ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount);
-    } else if(v instanceof RepeatedFixedWidthVectorLike) {
+    } else if (v instanceof RepeatedFixedWidthVectorLike) {
       ((RepeatedFixedWidthVectorLike) v).allocateNew(valueCount, childValCount);
-    } else if(v instanceof RepeatedVariableWidthVectorLike) {
+    } else if (v instanceof RepeatedVariableWidthVectorLike) {
       ((RepeatedVariableWidthVectorLike) v).allocateNew(childValCount * bytesPerValue, valueCount, childValCount);
     } else {
       v.allocateNew();
     }
   }
 
-  public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){
+  public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop) {
     allocatePrecomputedChildCount(v, valueCount, bytesPerValue, repeatedPerTop * valueCount);
   }
 
   /**
    * Allocates the exact amount if v is fixed width, otherwise falls back to dynamic allocation
-   * @param v value vector we are trying to allocate
-   * @param valueCount  size we are trying to allocate
+   *
+   * @param v          value vector we are trying to allocate
+   * @param valueCount size we are trying to allocate
    * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the memory
    */
   public static void allocateNew(ValueVector v, int valueCount) {
-    if (v instanceof  FixedWidthVector) {
+    if (v instanceof FixedWidthVector) {
       ((FixedWidthVector) v).allocateNew(valueCount);
     } else {
       v.allocateNew();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 88e02495bfc99..01340f66c4095 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import java.util.ArrayList;
@@ -84,7 +85,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
     if (getBufferSize() == 0) {
       out = new ArrowBuf[0];
     } else {
-      out = new ArrowBuf[]{data};
+      out = new ArrowBuf[] {data};
       data.readerIndex(0);
       if (clear) {
         data.retain(1);
@@ -123,5 +124,6 @@ public ArrowBuf unLoad() {
    * This method has a similar effect of allocateNew() without actually clearing and reallocating
    * the value vector. The purpose is to move the value vector to a "mutate" state
    */
-  public void reset() {}
+  public void reset() {
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 2a61403c0dcbe..05d2aa933cbef 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import java.util.Iterator;
@@ -66,7 +67,8 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
   }
 
   public abstract static class BaseAccessor implements ValueVector.Accessor {
-    protected BaseAccessor() { }
+    protected BaseAccessor() {
+    }
 
     @Override
     public boolean isNull(int index) {
@@ -79,7 +81,7 @@ public int getNullCount() {
       int nullCount = 0;
       for (int i = 0; i < getValueCount(); i++) {
         if (isNull(i)) {
-          nullCount ++;
+          nullCount++;
         }
       }
       return nullCount;
@@ -87,14 +89,17 @@ public int getNullCount() {
   }
 
   public abstract static class BaseMutator implements ValueVector.Mutator {
-    protected BaseMutator() { }
+    protected BaseMutator() {
+    }
 
     @Override
-    public void generateTestData(int values) {}
+    public void generateTestData(int values) {
+    }
 
     //TODO: consider making mutator stateless(if possible) on another issue.
     @Override
-    public void reset() {}
+    public void reset() {
+    }
   }
 
   @Override
@@ -103,7 +108,7 @@ public Iterator<ValueVector> iterator() {
   }
 
   public static boolean checkBufRefs(final ValueVector vv) {
-    for(final ArrowBuf buffer : vv.getBuffers(false)) {
+    for (final ArrowBuf buffer : vv.getBuffers(false)) {
       if (buffer.refCnt() <= 0) {
         throw new IllegalStateException("zero refcount");
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index f34ef2c2a2244..e80ca829c667e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -115,7 +116,7 @@ int getSizeFromCount(int valueCount) {
 
   @Override
   public int getValueCapacity() {
-    return (int)Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L);
+    return (int) Math.min((long) Integer.MAX_VALUE, data.capacity() * 8L);
   }
 
   private int getByteIndex(int index) {
@@ -165,8 +166,7 @@ public void reset() {
   /**
    * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
    *
-   * @param valueCount
-   *          The number of values which can be contained within this vector.
+   * @param valueCount The number of values which can be contained within this vector.
    */
   @Override
   public void allocateNew(int valueCount) {
@@ -195,7 +195,7 @@ public void reAlloc() {
       throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
     }
 
-    final int curSize = (int)newAllocationSize;
+    final int curSize = (int) newAllocationSize;
     final ArrowBuf newBuf = allocator.buffer(curSize);
     newBuf.setZero(0, newBuf.capacity());
     newBuf.setBytes(0, data, 0, data.capacity());
@@ -287,7 +287,7 @@ public void splitAndTransferTo(int startIndex, int length, BitVector target) {
           target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
         } else {
           target.data.setByte(byteSize - 1,
-                  (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) << (8 - offset))));
+              (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) << (8 - offset))));
         }
       }
     }
@@ -342,15 +342,14 @@ public class Accessor extends BaseAccessor {
     /**
      * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the bit was not set.
      *
-     * @param index
-     *          position of the bit in the vector
+     * @param index position of the bit in the vector
      * @return 1 if set, otherwise 0
      */
     public final int get(int index) {
       int byteIndex = index >> 3;
       byte b = data.getByte(byteIndex);
       int bitIndex = index & 7;
-      return Long.bitCount(b &  (1L << bitIndex));
+      return Long.bitCount(b & (1L << bitIndex));
     }
 
     @Override
@@ -379,6 +378,7 @@ public final void get(int index, NullableBitHolder holder) {
 
     /**
      * Get the number nulls, this correspond to the number of bits set to 0 in the vector
+     *
      * @return the number of bits set to 0
      */
     @Override
@@ -414,10 +414,8 @@ private Mutator() {
     /**
      * Set the bit at the given index to the specified value.
      *
-     * @param index
-     *          position of the bit to set
-     * @param value
-     *          value to set (either 1 or 0)
+     * @param index position of the bit to set
+     * @param value value to set (either 1 or 0)
      */
     public final void set(int index, int value) {
       int byteIndex = byteIndex(index);
@@ -448,8 +446,9 @@ public final void setToOne(int index) {
 
     /**
      * set count bits to 1 in data starting at firstBitIndex
+     *
      * @param firstBitIndex the index of the first bit to set
-     * @param count the number of bits to set
+     * @param count         the number of bits to set
      */
     public void setRangeToOne(int firstBitIndex, int count) {
       int starByteIndex = byteIndex(firstBitIndex);
@@ -473,7 +472,7 @@ public void setRangeToOne(int firstBitIndex, int count) {
           final byte bitMask = (byte) (0xFFL << startByteBitIndex);
           currentByte |= bitMask;
           data.setByte(starByteIndex, currentByte);
-          ++ starByteIndex;
+          ++starByteIndex;
         }
 
         // fill in one full byte at a time
@@ -518,28 +517,28 @@ final void set(int index, NullableBitHolder holder) {
     }
 
     public void setSafe(int index, int value) {
-      while(index >= getValueCapacity()) {
+      while (index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, value);
     }
 
     public void setSafeToOne(int index) {
-      while(index >= getValueCapacity()) {
+      while (index >= getValueCapacity()) {
         reAlloc();
       }
       setToOne(index);
     }
 
     public void setSafe(int index, BitHolder holder) {
-      while(index >= getValueCapacity()) {
+      while (index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, holder.value);
     }
 
     public void setSafe(int index, NullableBitHolder holder) {
-      while(index >= getValueCapacity()) {
+      while (index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, holder.value);
@@ -550,7 +549,7 @@ public final void setValueCount(int valueCount) {
       int currentValueCapacity = getValueCapacity();
       BitVector.this.valueCount = valueCount;
       int idx = getSizeFromCount(valueCount);
-      while(valueCount > getValueCapacity()) {
+      while (valueCount > getValueCapacity()) {
         reAlloc();
       }
       if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
@@ -564,7 +563,7 @@ public final void setValueCount(int valueCount) {
     @Override
     public final void generateTestData(int values) {
       boolean even = true;
-      for(int i = 0; i < values; i++, even = !even) {
+      for (int i = 0; i < values; i++, even = !even) {
         if (even) {
           set(i, 1);
         }
@@ -576,10 +575,10 @@ public void generateTestDataAlt(int size) {
       setValueCount(size);
       boolean even = true;
       final int valueCount = getAccessor().getValueCount();
-      for(int i = 0; i < valueCount; i++, even = !even) {
-        if(even){
+      for (int i = 0; i < valueCount; i++, even = !even) {
+        if (even) {
           set(i, (byte) 1);
-        }else{
+        } else {
           set(i, (byte) 0);
         }
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
index 3c8b3210d77ff..a0dbf2bdcf101 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import org.apache.arrow.vector.schema.ArrowFieldNode;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
index 6c2c8302a7b8b..af7a7912c72cc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import java.util.List;
@@ -33,12 +34,14 @@ public interface FieldVector extends ValueVector {
   /**
    * Initializes the child vectors
    * to be later loaded with loadBuffers
+   *
    * @param children the schema
    */
   void initializeChildrenFromFields(List<Field> children);
 
   /**
    * the returned list is the same size as the list passed to initializeChildrenFromFields
+   *
    * @return the children according to schema (empty for primitive types)
    */
   List<FieldVector> getChildrenFromFields();
@@ -46,13 +49,15 @@ public interface FieldVector extends ValueVector {
   /**
    * loads data in the vectors
    * (ownBuffers must be the same size as getFieldVectors())
-   * @param fieldNode the fieldNode
+   *
+   * @param fieldNode  the fieldNode
    * @param ownBuffers the buffers for this Field (own buffers only, children not included)
    */
   void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers);
 
   /**
    * (same size as getFieldVectors() since it is their content)
+   *
    * @return the buffers containing the data for this vector (ready for reading)
    */
   List<ArrowBuf> getFieldBuffers();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
index 59057000bbca9..ec410fc089c9c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
@@ -15,21 +15,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 
-public interface FixedWidthVector extends ValueVector{
+public interface FixedWidthVector extends ValueVector {
 
   /**
    * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
    *
-   * @param valueCount   Number of values in the vector.
+   * @param valueCount Number of values in the vector.
    */
   void allocateNew(int valueCount);
 
-/**
- * Zero out the underlying buffer backing this vector.
- */
+  /**
+   * Zero out the underlying buffer backing this vector.
+   */
   void zeroVector();
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
index b49e9167c2589..b2455e9e42b4b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 public interface NullableVector extends ValueVector {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
index b819c5d39e99c..1e0746aabaa61 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 public interface NullableVectorDefinitionSetter {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java b/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
index 6fdcda20480f8..54c0c591e2b92 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
@@ -42,6 +42,7 @@ public void doWork() {
   /**
    * Returns the value of schema-changed state, <strong>resetting</strong> the
    * schema-changed state to {@code false}.
+   *
    * @return the previous schema-changed state
    */
   public boolean getSchemaChangedAndReset() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 3812c0b2fc319..0c95bcfcbd6b5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import java.io.Closeable;
@@ -40,11 +41,11 @@
  * There are a few "rules" around vectors:
  *
  * <ul>
- *   <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
- *   <li>null vectors start with all values as null before writing anything</li>
- *   <li>for variable width types, the offset vector should be all zeros before writing</li>
- *   <li>you must call setValueCount before a vector can be read</li>
- *   <li>you should never write to a vector once it has been read.</li>
+ * <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
+ * <li>null vectors start with all values as null before writing anything</li>
+ * <li>for variable width types, the offset vector should be all zeros before writing</li>
+ * <li>you must call setValueCount before a vector can be read</li>
+ * <li>you should never write to a vector once it has been read.</li>
  * </ul>
  *
  * Please note that the current implementation doesn't enforce those rules, hence we may find few places that
@@ -58,12 +59,14 @@
 public interface ValueVector extends Closeable, Iterable<ValueVector> {
   /**
    * Allocate new buffers. ValueVector implements logic to determine how much to allocate.
+   *
    * @throws OutOfMemoryException Thrown if no memory can be allocated.
    */
   void allocateNew() throws OutOfMemoryException;
 
   /**
    * Allocates new buffers. ValueVector implements logic to determine how much to allocate.
+   *
    * @return Returns true if allocation was successful.
    */
   boolean allocateNewSafe();
@@ -78,12 +81,14 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * Set the initial record capacity
+   *
    * @param numRecords the initial record capacity.
    */
   void setInitialCapacity(int numRecords);
 
   /**
    * Returns the maximum number of values that can be stored in this vector instance.
+   *
    * @return the maximum number of values that can be stored in this vector instance.
    */
   int getValueCapacity();
@@ -101,6 +106,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * Get information about how this field is materialized.
+   *
    * @return the field corresponding to this vector
    */
   Field getField();
@@ -109,6 +115,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * to transfer quota responsibility
+   *
    * @param allocator the target allocator
    * @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
    * the same type.
@@ -121,6 +128,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * makes a new transfer pair used to transfer underlying buffers
+   *
    * @param target the target for the transfer
    * @return a new {@link org.apache.arrow.vector.util.TransferPair transfer pair} that is used to transfer underlying
    * buffers into the target vector.
@@ -167,9 +175,9 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
    * Return the underlying buffers associated with this vector. Note that this doesn't impact the reference counts for
    * this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus
    * external classes shouldn't hold a reference to it (unless they change it).
-   * @param clear Whether to clear vector before returning; the buffers will still be refcounted;
-   *   but the returned array will be the only reference to them
    *
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted;
+   *              but the returned array will be the only reference to them
    * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this vector instance.
    */
   ArrowBuf[] getBuffers(boolean clear);
@@ -181,8 +189,7 @@ interface Accessor {
     /**
      * Get the Java Object representation of the element at the specified position. Useful for testing.
      *
-     * @param index
-     *          Index of the value to get
+     * @param index Index of the value to get
      * @return the friendly java type
      */
     Object getObject(int index);
@@ -211,7 +218,7 @@ interface Mutator {
     /**
      * Sets the number of values that is stored in this vector to the given value count.
      *
-     * @param valueCount  value count to set.
+     * @param valueCount value count to set.
      */
     void setValueCount(int valueCount);
 
@@ -221,8 +228,8 @@ interface Mutator {
     void reset();
 
     /**
-     * @deprecated  this has nothing to do with value vector abstraction and should be removed.
      * @param values the number of values to generate
+     * @deprecated this has nothing to do with value vector abstraction and should be removed.
      */
     @Deprecated
     void generateTestData(int values);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
index ed164b548b5bd..04c00b7c8349c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
@@ -15,20 +15,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
-public interface VariableWidthVector extends ValueVector{
+public interface VariableWidthVector extends ValueVector {
 
   /**
    * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
    *
-   * @param totalBytes   Desired size of the underlying data buffer.
-   * @param valueCount   Number of values in the vector.
+   * @param totalBytes Desired size of the underlying data buffer.
+   * @param valueCount Number of values in the vector.
    */
   void allocateNew(int totalBytes, int valueCount);
 
   /**
    * Provide the maximum amount of variable width bytes that can be stored in this vector.
+   *
    * @return the byte capacity of this vector
    */
   int getByteCapacity();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index e640c7cb78418..58fc80bbba17c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static com.google.common.base.Preconditions.checkArgument;
@@ -41,6 +42,7 @@ public class VectorLoader {
 
   /**
    * will create children in root based on schema
+   *
    * @param root the root to add vectors to based on schema
    */
   public VectorLoader(VectorSchemaRoot root) {
@@ -50,12 +52,13 @@ public VectorLoader(VectorSchemaRoot root) {
   /**
    * Loads the record batch in the vectors
    * will not close the record batch
+   *
    * @param recordBatch the batch to load
    */
   public void load(ArrowRecordBatch recordBatch) {
     Iterator<ArrowBuf> buffers = recordBatch.getBuffers().iterator();
     Iterator<ArrowFieldNode> nodes = recordBatch.getNodes().iterator();
-    for (FieldVector fieldVector: root.getFieldVectors()) {
+    for (FieldVector fieldVector : root.getFieldVectors()) {
       loadBuffers(fieldVector, fieldVector.getField(), buffers, nodes);
     }
     root.setRowCount(recordBatch.getLength());
@@ -77,7 +80,7 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
       vector.loadFieldBuffers(fieldNode, ownBuffers);
     } catch (RuntimeException e) {
       throw new IllegalArgumentException("Could not load buffers for field " +
-            field + ". error message: " + e.getMessage(), e);
+          field + ". error message: " + e.getMessage(), e);
     }
     List<Field> children = field.getChildren();
     if (children.size() > 0) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
index 73deb0b3a426e..0c8868cad55b5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import java.util.ArrayList;
@@ -67,7 +68,7 @@ public static VectorSchemaRoot create(Schema schema, BufferAllocator allocator)
     }
     if (fieldVectors.size() != schema.getFields().size()) {
       throw new IllegalArgumentException("The root vector did not create the right number of children. found " +
-        fieldVectors.size() + " expected " + schema.getFields().size());
+          fieldVectors.size() + " expected " + schema.getFields().size());
     }
     return new VectorSchemaRoot(schema, fieldVectors, 0);
   }
@@ -102,7 +103,7 @@ public void close() {
         ex = chain(ex, e);
       }
     }
-    if (ex!= null) {
+    if (ex != null) {
       throw ex;
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
index 055857e956084..ada471e63e710 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import io.netty.buffer.ByteBuf;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 996524521cb68..f8385a7262a21 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import java.util.ArrayList;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 01e22f2574346..cce73897718a3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import java.util.Collections;
@@ -42,10 +43,12 @@ public class ZeroVector implements FieldVector {
 
   private final TransferPair defaultPair = new TransferPair() {
     @Override
-    public void transfer() { }
+    public void transfer() {
+    }
 
     @Override
-    public void splitAndTransfer(int startIndex, int length) { }
+    public void splitAndTransfer(int startIndex, int length) {
+    }
 
     @Override
     public ValueVector getTo() {
@@ -53,7 +56,8 @@ public ValueVector getTo() {
     }
 
     @Override
-    public void copyValueSafe(int from, int to) { }
+    public void copyValueSafe(int from, int to) {
+    }
   };
 
   private final Accessor defaultAccessor = new Accessor() {
@@ -80,22 +84,28 @@ public int getNullCount() {
 
   private final Mutator defaultMutator = new Mutator() {
     @Override
-    public void setValueCount(int valueCount) { }
+    public void setValueCount(int valueCount) {
+    }
 
     @Override
-    public void reset() { }
+    public void reset() {
+    }
 
     @Override
-    public void generateTestData(int values) { }
+    public void generateTestData(int values) {
+    }
   };
 
-  public ZeroVector() { }
+  public ZeroVector() {
+  }
 
   @Override
-  public void close() { }
+  public void close() {
+  }
 
   @Override
-  public void clear() { }
+  public void clear() {
+  }
 
   @Override
   public Field getField() {
@@ -144,7 +154,8 @@ public boolean allocateNewSafe() {
   }
 
   @Override
-  public void reAlloc() {}
+  public void reAlloc() {
+  }
 
   @Override
   public BufferAllocator getAllocator() {
@@ -152,7 +163,8 @@ public BufferAllocator getAllocator() {
   }
 
   @Override
-  public void setInitialCapacity(int numRecords) { }
+  public void setInitialCapacity(int numRecords) {
+  }
 
   @Override
   public int getValueCapacity() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 2aeeca25f0e9e..db0ff86df47a9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -58,6 +59,7 @@ public BufferAllocator getAllocator() {
 
   /**
    * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
+   *
    * @param name the name of the child to return
    * @return the corresponding FieldVector
    */
@@ -68,9 +70,9 @@ public FieldVector getChild(String name) {
   /**
    * Clears out all underlying child vectors.
    */
- @Override
+  @Override
   public void close() {
-    for (ValueVector vector:(Iterable<ValueVector>)this) {
+    for (ValueVector vector : (Iterable<ValueVector>) this) {
       vector.close();
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index 4b6d82cc8b291..26b0f90581ffc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import java.util.ArrayList;
@@ -42,7 +43,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
 
   // Maintains a map with key as field name and value is the vector itself
-  private final MapWithOrdinal<String, FieldVector> vectors =  new MapWithOrdinal<>();
+  private final MapWithOrdinal<String, FieldVector> vectors = new MapWithOrdinal<>();
 
   protected AbstractMapVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator, callBack);
@@ -50,7 +51,7 @@ protected AbstractMapVector(String name, BufferAllocator allocator, CallBack cal
 
   @Override
   public void close() {
-    for(final ValueVector valueVector : vectors.values()) {
+    for (final ValueVector valueVector : vectors.values()) {
       valueVector.close();
     }
     vectors.clear();
@@ -83,7 +84,7 @@ public boolean allocateNewSafe() {
 
   @Override
   public void reAlloc() {
-    for (final ValueVector v: vectors.values()) {
+    for (final ValueVector v : vectors.values()) {
       v.reAlloc();
     }
   }
@@ -94,27 +95,26 @@ public void reAlloc() {
    *
    * Execution takes place in the following order:
    * <ul>
-   *   <li>
-   *     if field is new, create and insert a new vector of desired type.
-   *   </li>
-   *   <li>
-   *     if field exists and existing vector is of desired vector type, return the vector.
-   *   </li>
-   *   <li>
-   *     if field exists and null filled, clear the existing vector; create and insert a new vector of desired type.
-   *   </li>
-   *   <li>
-   *     otherwise, throw an {@link java.lang.IllegalStateException}
-   *   </li>
+   * <li>
+   * if field is new, create and insert a new vector of desired type.
+   * </li>
+   * <li>
+   * if field exists and existing vector is of desired vector type, return the vector.
+   * </li>
+   * <li>
+   * if field exists and null filled, clear the existing vector; create and insert a new vector of desired type.
+   * </li>
+   * <li>
+   * otherwise, throw an {@link java.lang.IllegalStateException}
+   * </li>
    * </ul>
    *
    * @param childName the name of the field
    * @param fieldType the type for the vector
-   * @param clazz class of expected vector type
-   * @param <T> class type of expected vector type
-   * @throws java.lang.IllegalStateException raised if there is a hard schema change
-   *
+   * @param clazz     class of expected vector type
+   * @param <T>       class type of expected vector type
    * @return resultant {@link org.apache.arrow.vector.ValueVector}
+   * @throws java.lang.IllegalStateException raised if there is a hard schema change
    */
   @Override
   public <T extends FieldVector> T addOrGet(String childName, FieldType fieldType, Class<T> clazz) {
@@ -151,6 +151,7 @@ private boolean nullFilled(ValueVector vector) {
 
   /**
    * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given ordinal identifier.
+   *
    * @param id the ordinal of the child to return
    * @return the corresponding child
    */
@@ -161,7 +162,8 @@ public ValueVector getChildByOrdinal(int id) {
   /**
    * Returns a {@link org.apache.arrow.vector.ValueVector} instance of subtype of T corresponding to the given
    * field name if exists or null.
-   * @param name the name of the child to return
+   *
+   * @param name  the name of the child to return
    * @param clazz the expected type of the child
    * @return the child corresponding to this name
    */
@@ -191,7 +193,8 @@ protected ValueVector add(String childName, FieldType fieldType) {
    * Inserts the vector with the given name if it does not exist else replaces it with the new value.
    *
    * Note that this method does not enforce any vector type check nor throws a schema change exception.
-   * @param name the name of the child to add
+   *
+   * @param name   the name of the child to add
    * @param vector the vector to add as a child
    */
   protected void putChild(String name, FieldVector vector) {
@@ -200,8 +203,9 @@ protected void putChild(String name, FieldVector vector) {
 
   /**
    * Inserts the input vector into the map if it does not exist, replaces if it exists already
-   * @param name  field name
-   * @param vector  vector to be inserted
+   *
+   * @param name   field name
+   * @param vector vector to be inserted
    */
   protected void putVector(String name, FieldVector vector) {
     final ValueVector old = vectors.put(
@@ -210,7 +214,7 @@ protected void putVector(String name, FieldVector vector) {
     );
     if (old != null && old != vector) {
       logger.debug("Field [{}] mutated from [{}] to [{}]", name, old.getClass().getSimpleName(),
-                   vector.getClass().getSimpleName());
+          vector.getClass().getSimpleName());
     }
   }
 
@@ -298,7 +302,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
 
   @Override
   public int getBufferSize() {
-    int actualBufSize = 0 ;
+    int actualBufSize = 0;
 
     for (final ValueVector v : vectors.values()) {
       for (final ArrowBuf buf : v.getBuffers(false)) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 3bfa8e7f7ce67..8e2877f892a64 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import java.util.Collections;
@@ -144,7 +145,7 @@ public void clear() {
   public ArrowBuf[] getBuffers(boolean clear) {
     final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
     if (clear) {
-      for (ArrowBuf buffer:buffers) {
+      for (ArrowBuf buffer : buffers) {
         buffer.retain();
       }
       clear();
@@ -156,7 +157,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
    * @return 1 if inner vector is explicitly set via #addOrGetVector else 0
    */
   public int size() {
-    return vector == DEFAULT_DATA_VECTOR ? 0:1;
+    return vector == DEFAULT_DATA_VECTOR ? 0 : 1;
   }
 
   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) {
@@ -166,8 +167,8 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldT
       // returned vector must have the same field
       created = true;
       if (callBack != null &&
-        // not a schema change if changing from ZeroVector to ZeroVector
-        (fieldType.getType().getTypeID() != ArrowTypeID.Null)) {
+          // not a schema change if changing from ZeroVector to ZeroVector
+          (fieldType.getType().getTypeID() != ArrowTypeID.Null)) {
         callBack.doWork();
       }
     }
@@ -178,7 +179,7 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldT
       throw new SchemaChangeRuntimeException(msg);
     }
 
-    return new AddOrGetResult<>((T)vector, created);
+    return new AddOrGetResult<>((T) vector, created);
   }
 
   protected void replaceDataVector(FieldVector v) {
@@ -200,7 +201,7 @@ public int getInnerValueCount() {
 
     @Override
     public int getInnerValueCountAt(int index) {
-      return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+      return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index);
     }
 
     @Override
@@ -222,15 +223,15 @@ public int startNewValue(int index) {
         offsets.reAlloc();
       }
       int offset = offsets.getAccessor().get(index);
-      offsets.getMutator().setSafe(index+1, offset);
-      setValueCount(index+1);
+      offsets.getMutator().setSafe(index + 1, offset);
+      setValueCount(index + 1);
       return offset;
     }
 
     @Override
     public void setValueCount(int valueCount) {
       // TODO: populate offset end points
-      offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+      offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1);
       final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
       vector.getMutator().setValueCount(childValueCount);
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
index df699755770a5..a76fbbe11a1fb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import org.apache.arrow.vector.UInt4Vector;
@@ -34,8 +35,8 @@ public EmptyValuePopulator(UInt4Vector offsets) {
   /**
    * Marks all values since the last set as empty. The last set value is obtained from underlying offsets vector.
    *
-   * @param lastIndex  the last index (inclusive) in the offsets vector until which empty population takes place
-   * @throws java.lang.IndexOutOfBoundsException  if lastIndex is negative or greater than offsets capacity.
+   * @param lastIndex the last index (inclusive) in the offsets vector until which empty population takes place
+   * @throws java.lang.IndexOutOfBoundsException if lastIndex is negative or greater than offsets capacity.
    */
   public void populate(int lastIndex) {
     if (lastIndex < 0) {
@@ -48,7 +49,7 @@ public void populate(int lastIndex) {
     for (int i = lastSet; i < lastIndex; i++) {
       mutator.setSafe(i + 1, previousEnd);
     }
-    mutator.setValueCount(lastIndex+1);
+    mutator.setValueCount(lastIndex + 1);
   }
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index b6d938f3fd863..3f0f1b05b6733 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.complex;
 
 import static java.util.Collections.singletonList;
@@ -240,7 +241,7 @@ public void clear() {
   public ArrowBuf[] getBuffers(boolean clear) {
     final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
     if (clear) {
-      for (ArrowBuf buffer: buffers) {
+      for (ArrowBuf buffer : buffers) {
         buffer.retain();
       }
       clear();
@@ -267,7 +268,7 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType type)
     // returned vector must have the same field
     if (!Objects.equals(vector.getField().getType(), type.getType())) {
       final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
-        type.getType(), vector.getField().getType());
+          type.getType(), vector.getField().getType());
       throw new SchemaChangeRuntimeException(msg);
     }
 
@@ -301,7 +302,7 @@ public Object getObject(int index) {
       }
       final List<Object> vals = new JsonStringArrayList<>(listSize);
       final ValueVector.Accessor valuesAccessor = vector.getAccessor();
-      for(int i = 0; i < listSize; i++) {
+      for (int i = 0; i < listSize; i++) {
         vals.add(valuesAccessor.getObject(index * listSize + i));
       }
       return vals;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index fdeac3971657d..7b6b97a8ed997 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.complex;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -199,7 +200,7 @@ public TransferImpl(ListVector to) {
         to.addOrGetVector(vector.getField().getFieldType());
       }
       dataTransferPair = getDataVector().makeTransferPair(to.getDataVector());
-      pairs = new TransferPair[] { bitsTransferPair, offsetsTransferPair, dataTransferPair };
+      pairs = new TransferPair[] {bitsTransferPair, offsetsTransferPair, dataTransferPair};
     }
 
     @Override
@@ -316,9 +317,9 @@ public void clear() {
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
     final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false),
-            vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
+        vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
     if (clear) {
-      for (ArrowBuf buffer:buffers) {
+      for (ArrowBuf buffer : buffers) {
         buffer.retain();
       }
       clear();
@@ -351,7 +352,7 @@ public Object getObject(int index) {
       final int start = offsetsAccessor.get(index);
       final int end = offsetsAccessor.get(index + 1);
       final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor();
-      for(int i = start; i < end; i++) {
+      for (int i = start; i < end; i++) {
         vals.add(valuesAccessor.getObject(i));
       }
       return vals;
@@ -388,7 +389,7 @@ public int startNewValue(int index) {
      * End the current value
      *
      * @param index index of the value to end
-     * @param size number of elements in the list that was written
+     * @param size  number of elements in the list that was written
      */
     public void endValue(int index, int size) {
       offsets.getMutator().set(index + 1, offsets.getAccessor().get(index + 1) + size);
@@ -414,7 +415,9 @@ public void setLastSet(int value) {
       lastSet = value;
     }
 
-    public int getLastSet() { return lastSet; }
+    public int getLastSet() {
+      return lastSet;
+    }
   }
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index bdd30f88f2cc2..d8d0964ab3792 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -79,7 +80,7 @@ public FieldReader getReader() {
   transient private MapTransferPair ephPair;
 
   public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
-    if(ephPair == null || ephPair.from != from) {
+    if (ephPair == null || ephPair.from != from) {
       ephPair = (MapTransferPair) from.makeTransferPair(this);
     }
     ephPair.copyValueSafe(fromIndex, thisIndex);
@@ -107,7 +108,7 @@ public int getBufferSize() {
       return 0;
     }
     long buffer = 0;
-    for (final ValueVector v : (Iterable<ValueVector>)this) {
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
       buffer += v.getBufferSize();
     }
 
@@ -148,7 +149,7 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
     return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false);
   }
 
-  protected static class MapTransferPair implements TransferPair{
+  protected static class MapTransferPair implements TransferPair {
     private final TransferPair[] pairs;
     private final MapVector from;
     private final MapVector to;
@@ -165,7 +166,7 @@ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
 
       int i = 0;
       FieldVector vector;
-      for (String child:from.getChildFieldNames()) {
+      for (String child : from.getChildFieldNames()) {
         int preSize = to.size();
         vector = from.getChild(child);
         if (vector == null) {
@@ -252,7 +253,7 @@ public class Accessor extends BaseValueVector.BaseAccessor {
     @Override
     public Object getObject(int index) {
       Map<String, Object> vv = new JsonStringHashMap<>();
-      for (String child:getChildFieldNames()) {
+      for (String child : getChildFieldNames()) {
         ValueVector v = getChild(child);
         if (v != null && index < v.getAccessor().getValueCount()) {
           Object value = v.getAccessor().getObject(index);
@@ -290,10 +291,12 @@ public void setValueCount(int valueCount) {
     }
 
     @Override
-    public void reset() { }
+    public void reset() {
+    }
 
     @Override
-    public void generateTestData(int values) { }
+    public void generateTestData(int values) {
+    }
   }
 
   @Override
@@ -329,11 +332,11 @@ public void close() {
     valueCount = 0;
 
     super.close();
- }
+  }
 
   public void initializeChildrenFromFields(List<Field> children) {
     for (Field field : children) {
-      FieldVector vector = (FieldVector)this.add(field.getName(), field.getFieldType());
+      FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType());
       vector.initializeChildrenFromFields(field.getChildren());
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index ee95fdef59401..e70a915561f8b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -185,7 +186,7 @@ public void clear() {
 
 
   @Override
-  public int getBufferSize(){
+  public int getBufferSize() {
     return super.getBufferSize() + bits.getBufferSize();
   }
 
@@ -229,7 +230,7 @@ public void reAlloc() {
     super.reAlloc();
   }
 
-  public final class Accessor extends MapVector.Accessor  {
+  public final class Accessor extends MapVector.Accessor {
     final BitVector.Accessor bAccessor = bits.getAccessor();
 
     @Override
@@ -257,7 +258,7 @@ public boolean isNull(int index) {
       return isSet(index) == 0;
     }
 
-    public int isSet(int index){
+    public int isSet(int index) {
       return bAccessor.get(index);
     }
 
@@ -265,15 +266,15 @@ public int isSet(int index){
 
   public final class Mutator extends MapVector.Mutator implements NullableVectorDefinitionSetter {
 
-    private Mutator(){
+    private Mutator() {
     }
 
     @Override
-    public void setIndexDefined(int index){
+    public void setIndexDefined(int index) {
       bits.getMutator().setSafe(index, 1);
     }
 
-    public void setNull(int index){
+    public void setNull(int index) {
       bits.getMutator().setSafe(index, 0);
     }
 
@@ -285,13 +286,13 @@ public void setValueCount(int valueCount) {
     }
 
     @Override
-    public void generateTestData(int valueCount){
+    public void generateTestData(int valueCount) {
       super.generateTestData(valueCount);
       bits.getMutator().generateTestDataAlt(valueCount);
     }
 
     @Override
-    public void reset(){
+    public void reset() {
       bits.getMutator().setValueCount(0);
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
index e1a4f36296987..f0a5174b2dce3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
@@ -15,9 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 public interface Positionable {
   public int getPosition();
+
   public void setPosition(int index);
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/PromotableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/PromotableVector.java
index 8b528b4ccab9b..4b19b9ffa0f9d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/PromotableVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/PromotableVector.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.complex;
 
 import org.apache.arrow.vector.AddOrGetResult;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
index 23850bc9034df..866883f532665 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 /**
@@ -25,8 +26,8 @@ public interface RepeatedFixedWidthVectorLike {
   /**
    * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
    *
-   * @param valueCount   Number of separate repeating groupings.
-   * @param innerValueCount   Number of supported values in the vector.
+   * @param valueCount      Number of separate repeating groupings.
+   * @param innerValueCount Number of supported values in the vector.
    */
   void allocateNew(int valueCount, int innerValueCount);
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index de58eda0b11a2..91147c663f248 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import org.apache.arrow.vector.UInt4Vector;
@@ -26,7 +27,6 @@
  * A repeated vector contains values that may either be flat or nested. A value consists of zero or more cells(inner values).
  * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector
  * uses the offset vector to determine the sequence of cells pertaining to an individual value.
- *
  */
 public interface RepeatedValueVector extends ValueVector {
 
@@ -51,6 +51,7 @@ public interface RepeatedValueVector extends ValueVector {
   interface RepeatedAccessor extends ValueVector.Accessor {
     /**
      * The result includes empty, null valued cells.
+     *
      * @return total number of cells that vector contains.
      */
     int getInnerValueCount();
@@ -63,7 +64,7 @@ interface RepeatedAccessor extends ValueVector.Accessor {
     int getInnerValueCountAt(int index);
 
     /**
-     * @param index  value index
+     * @param index value index
      * @return true if the value at the given index is empty, false otherwise.
      */
     boolean isEmpty(int index);
@@ -74,7 +75,7 @@ interface RepeatedMutator extends ValueVector.Mutator {
     /**
      * Starts a new value that is a container of cells.
      *
-     * @param index  index of new value to start
+     * @param index index of new value to start
      * @return index into the child vector
      */
     int startNewValue(int index);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
index 29f9d75c74671..1e4f54ea37209 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedVariableWidthVectorLike.java
@@ -15,20 +15,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 public interface RepeatedVariableWidthVectorLike {
   /**
    * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
    *
-   * @param totalBytes   Desired size of the underlying data buffer.
-   * @param parentValueCount   Number of separate repeating groupings.
-   * @param childValueCount   Number of supported values in the vector.
+   * @param totalBytes       Desired size of the underlying data buffer.
+   * @param parentValueCount Number of separate repeating groupings.
+   * @param childValueCount  Number of supported values in the vector.
    */
   void allocateNew(int totalBytes, int parentValueCount, int childValueCount);
 
   /**
    * Provide the maximum amount of variable width bytes that can be stored int his vector.
+   *
    * @return the byte capacity
    */
   int getByteCapacity();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
index 05a79d24295e4..627998045c93c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StateTool.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import java.util.Arrays;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/VectorWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/VectorWithOrdinal.java
index d04fc1c022c05..1633b3ad09892 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/VectorWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/VectorWithOrdinal.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex;
 
 import org.apache.arrow.vector.ValueVector;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
index 7c73c27ecff41..1eeced4598a55 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.impl;
 
 import java.util.Iterator;
@@ -25,7 +26,7 @@
 import org.apache.arrow.vector.holders.UnionHolder;
 
 
-abstract class AbstractBaseReader implements FieldReader{
+abstract class AbstractBaseReader implements FieldReader {
 
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseReader.class);
 
@@ -40,11 +41,11 @@ public int getPosition() {
     return index;
   }
 
-  public void setPosition(int index){
+  public void setPosition(int index) {
     this.index = index;
   }
 
-  protected int idx(){
+  protected int idx() {
     return index;
   }
 
@@ -86,6 +87,6 @@ public void copyAsValue(UnionWriter writer) {
 
   @Override
   public void copyAsValue(ListWriter writer) {
-    ComplexCopier.copy(this, (FieldWriter)writer);
+    ComplexCopier.copy(this, (FieldWriter) writer);
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
index 13a0a6bd9e28f..2f224fe3a5b7a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.writer.FieldWriter;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 6851d6d45d562..a6960238b9165 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.ListVector;
@@ -38,13 +39,15 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
   private final boolean unionEnabled;
   private final NullableMapWriterFactory nullableMapWriterFactory;
 
-  private enum Mode { INIT, MAP, LIST };
+  private enum Mode {INIT, MAP, LIST}
+
+  ;
 
-  public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled, boolean caseSensitive){
+  public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled, boolean caseSensitive) {
     this.name = name;
     this.container = container;
     this.unionEnabled = unionEnabled;
-    nullableMapWriterFactory = caseSensitive? NullableMapWriterFactory.getNullableCaseSensitiveMapWriterFactoryInstance() :
+    nullableMapWriterFactory = caseSensitive ? NullableMapWriterFactory.getNullableCaseSensitiveMapWriterFactoryInstance() :
         NullableMapWriterFactory.getNullableMapWriterFactoryInstance();
   }
 
@@ -52,7 +55,7 @@ public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled)
     this(name, container, unionEnabled, false);
   }
 
-  public ComplexWriterImpl(String name, MapVector container){
+  public ComplexWriterImpl(String name, MapVector container) {
     this(name, container, false);
   }
 
@@ -66,12 +69,12 @@ public int getValueCapacity() {
     return container.getValueCapacity();
   }
 
-  private void check(Mode... modes){
+  private void check(Mode... modes) {
     StateTool.check(mode, modes);
   }
 
   @Override
-  public void reset(){
+  public void reset() {
     setPosition(0);
   }
 
@@ -85,58 +88,58 @@ public void close() throws Exception {
   }
 
   @Override
-  public void clear(){
-    switch(mode){
-    case MAP:
-      mapRoot.clear();
-      break;
-    case LIST:
-      listRoot.clear();
-      break;
+  public void clear() {
+    switch (mode) {
+      case MAP:
+        mapRoot.clear();
+        break;
+      case LIST:
+        listRoot.clear();
+        break;
     }
   }
 
   @Override
-  public void setValueCount(int count){
-    switch(mode){
-    case MAP:
-      mapRoot.setValueCount(count);
-      break;
-    case LIST:
-      listRoot.setValueCount(count);
-      break;
+  public void setValueCount(int count) {
+    switch (mode) {
+      case MAP:
+        mapRoot.setValueCount(count);
+        break;
+      case LIST:
+        listRoot.setValueCount(count);
+        break;
     }
   }
 
   @Override
-  public void setPosition(int index){
+  public void setPosition(int index) {
     super.setPosition(index);
-    switch(mode){
-    case MAP:
-      mapRoot.setPosition(index);
-      break;
-    case LIST:
-      listRoot.setPosition(index);
-      break;
+    switch (mode) {
+      case MAP:
+        mapRoot.setPosition(index);
+        break;
+      case LIST:
+        listRoot.setPosition(index);
+        break;
     }
   }
 
 
-  public MapWriter directMap(){
+  public MapWriter directMap() {
     Preconditions.checkArgument(name == null);
 
-    switch(mode){
+    switch (mode) {
 
-    case INIT:
-      mapRoot = nullableMapWriterFactory.build((NullableMapVector) container);
-      mapRoot.setPosition(idx());
-      mode = Mode.MAP;
-      break;
+      case INIT:
+        mapRoot = nullableMapWriterFactory.build((NullableMapVector) container);
+        mapRoot.setPosition(idx());
+        mode = Mode.MAP;
+        break;
 
-    case MAP:
-      break;
+      case MAP:
+        break;
 
-    default:
+      default:
         check(Mode.INIT, Mode.MAP);
     }
 
@@ -145,20 +148,20 @@ public MapWriter directMap(){
 
   @Override
   public MapWriter rootAsMap() {
-    switch(mode){
+    switch (mode) {
 
-    case INIT:
-      // TODO allow dictionaries in complex types
-      NullableMapVector map = container.addOrGetMap(name);
-      mapRoot = nullableMapWriterFactory.build(map);
-      mapRoot.setPosition(idx());
-      mode = Mode.MAP;
-      break;
+      case INIT:
+        // TODO allow dictionaries in complex types
+        NullableMapVector map = container.addOrGetMap(name);
+        mapRoot = nullableMapWriterFactory.build(map);
+        mapRoot.setPosition(idx());
+        mode = Mode.MAP;
+        break;
 
-    case MAP:
-      break;
+      case MAP:
+        break;
 
-    default:
+      default:
         check(Mode.INIT, Mode.MAP);
     }
 
@@ -167,33 +170,33 @@ public MapWriter rootAsMap() {
 
   @Override
   public void allocate() {
-    if(mapRoot != null) {
+    if (mapRoot != null) {
       mapRoot.allocate();
-    } else if(listRoot != null) {
+    } else if (listRoot != null) {
       listRoot.allocate();
     }
   }
 
   @Override
   public ListWriter rootAsList() {
-    switch(mode){
-
-    case INIT:
-      int vectorCount = container.size();
-      // TODO allow dictionaries in complex types
-      ListVector listVector = container.addOrGetList(name);
-      if (container.size() > vectorCount) {
-        listVector.allocateNew();
-      }
-      listRoot = new UnionListWriter(listVector, nullableMapWriterFactory);
-      listRoot.setPosition(idx());
-      mode = Mode.LIST;
-      break;
-
-    case LIST:
-      break;
-
-    default:
+    switch (mode) {
+
+      case INIT:
+        int vectorCount = container.size();
+        // TODO allow dictionaries in complex types
+        ListVector listVector = container.addOrGetList(name);
+        if (container.size() > vectorCount) {
+          listVector.allocateNew();
+        }
+        listRoot = new UnionListWriter(listVector, nullableMapWriterFactory);
+        listRoot.setPosition(idx());
+        mode = Mode.LIST;
+        break;
+
+      case LIST:
+        break;
+
+      default:
         check(Mode.INIT, Mode.MAP);
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java
index f8a9d4232aadc..0d860b6a04115 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.writer.BaseWriter;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
index 067716e8ea290..614c266acf147 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.MapVector;
@@ -28,8 +29,8 @@ public class NullableMapReaderImpl extends SingleMapReaderImpl {
   private NullableMapVector nullableMapVector;
 
   public NullableMapReaderImpl(MapVector vector) {
-    super((NullableMapVector)vector);
-    this.nullableMapVector = (NullableMapVector)vector;
+    super((NullableMapVector) vector);
+    this.nullableMapVector = (NullableMapVector) vector;
   }
 
   @Override
@@ -38,19 +39,19 @@ public Field getField() {
   }
 
   @Override
-  public void copyAsValue(MapWriter writer){
+  public void copyAsValue(MapWriter writer) {
     NullableMapWriter impl = (NullableMapWriter) writer;
     impl.container.copyFromSafe(idx(), impl.idx(), nullableMapVector);
   }
 
   @Override
-  public void copyAsField(String name, MapWriter writer){
+  public void copyAsField(String name, MapWriter writer) {
     NullableMapWriter impl = (NullableMapWriter) writer.map(name);
     impl.container.copyFromSafe(idx(), impl.idx(), nullableMapVector);
   }
 
   @Override
-  public boolean isSet(){
+  public boolean isSet() {
     return !nullableMapVector.getAccessor().isNull(idx());
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
index d932cfb3e1287..d2dcb2374d0e7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
@@ -15,28 +15,29 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.NullableMapVector;
 
 public class NullableMapWriterFactory {
-    private final boolean caseSensitive;
-    private static final NullableMapWriterFactory nullableMapWriterFactory = new NullableMapWriterFactory(false);
-    private static final NullableMapWriterFactory nullableCaseSensitiveWriterFactory = new NullableMapWriterFactory(true);
+  private final boolean caseSensitive;
+  private static final NullableMapWriterFactory nullableMapWriterFactory = new NullableMapWriterFactory(false);
+  private static final NullableMapWriterFactory nullableCaseSensitiveWriterFactory = new NullableMapWriterFactory(true);
 
-    public NullableMapWriterFactory(boolean caseSensitive) {
-        this.caseSensitive = caseSensitive;
-    }
+  public NullableMapWriterFactory(boolean caseSensitive) {
+    this.caseSensitive = caseSensitive;
+  }
 
-    public NullableMapWriter build(NullableMapVector container) {
-        return this.caseSensitive? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container);
-    }
+  public NullableMapWriter build(NullableMapVector container) {
+    return this.caseSensitive ? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container);
+  }
 
-    public static NullableMapWriterFactory getNullableMapWriterFactoryInstance() {
-        return nullableMapWriterFactory;
-    }
+  public static NullableMapWriterFactory getNullableMapWriterFactoryInstance() {
+    return nullableMapWriterFactory;
+  }
 
-    public static NullableMapWriterFactory getNullableCaseSensitiveMapWriterFactoryInstance() {
-        return nullableCaseSensitiveWriterFactory;
-    }
+  public static NullableMapWriterFactory getNullableCaseSensitiveMapWriterFactoryInstance() {
+    return nullableCaseSensitiveWriterFactory;
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index d16718e75a701..9722196ed7cd2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.FieldVector;
@@ -120,7 +121,7 @@ public void setPosition(int index) {
 
   protected FieldWriter getWriter(MinorType type) {
     if (state == State.UNION) {
-      ((UnionWriter)writer).getWriter(type);
+      ((UnionWriter) writer).getWriter(type);
     } else if (state == State.UNTYPED) {
       if (type == null) {
         // ???
@@ -132,7 +133,7 @@ protected FieldWriter getWriter(MinorType type) {
       writer.setPosition(position);
     } else if (type != this.type) {
       promoteToUnion();
-      ((UnionWriter)writer).getWriter(type);
+      ((UnionWriter) writer).getWriter(type);
     }
     return writer;
   }
@@ -157,7 +158,7 @@ private FieldWriter promoteToUnion() {
     } else if (listVector != null) {
       unionVector = listVector.promoteToUnion();
     }
-    unionVector.addVector((FieldVector)tp.getTo());
+    unionVector.addVector((FieldVector) tp.getTo());
     writer = new UnionWriter(unionVector, nullableMapWriterFactory);
     writer.setPosition(idx());
     for (int i = 0; i <= idx(); i++) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
index b8f58658eae15..f2b46ab98db7f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
@@ -17,6 +17,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.complex.impl;
 
 
@@ -27,7 +28,7 @@
 import org.apache.arrow.vector.types.Types.MinorType;
 
 @SuppressWarnings("unused")
-public class SingleListReaderImpl extends AbstractFieldReader{
+public class SingleListReaderImpl extends AbstractFieldReader {
 
   private final String name;
   private final AbstractContainerVector container;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index 48019093e387f..3ebd0cd7dd959 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.complex.impl;
 
 
@@ -31,7 +32,7 @@
 import com.google.common.collect.Maps;
 
 @SuppressWarnings("unused")
-public class SingleMapReaderImpl extends AbstractFieldReader{
+public class SingleMapReaderImpl extends AbstractFieldReader {
 
   private final MapVector vector;
   private final Map<String, FieldReader> fields = Maps.newHashMap();
@@ -40,8 +41,8 @@ public SingleMapReaderImpl(MapVector vector) {
     this.vector = vector;
   }
 
-  private void setChildrenPosition(int index){
-    for(FieldReader r : fields.values()){
+  private void setChildrenPosition(int index) {
+    for (FieldReader r : fields.values()) {
       r.setPosition(index);
     }
   }
@@ -52,13 +53,13 @@ public Field getField() {
   }
 
   @Override
-  public FieldReader reader(String name){
+  public FieldReader reader(String name) {
     FieldReader reader = fields.get(name);
-    if(reader == null){
+    if (reader == null) {
       ValueVector child = vector.getChild(name);
-      if(child == null){
+      if (child == null) {
         reader = NullReader.INSTANCE;
-      }else{
+      } else {
         reader = child.getReader();
       }
       fields.put(name, reader);
@@ -68,9 +69,9 @@ public FieldReader reader(String name){
   }
 
   @Override
-  public void setPosition(int index){
+  public void setPosition(int index) {
     super.setPosition(index);
-    for(FieldReader r : fields.values()){
+    for (FieldReader r : fields.values()) {
       r.setPosition(index);
     }
   }
@@ -91,18 +92,18 @@ public boolean isSet() {
   }
 
   @Override
-  public java.util.Iterator<String> iterator(){
+  public java.util.Iterator<String> iterator() {
     return vector.fieldNameIterator();
   }
 
   @Override
-  public void copyAsValue(MapWriter writer){
+  public void copyAsValue(MapWriter writer) {
     SingleMapWriter impl = (SingleMapWriter) writer;
     impl.container.copyFromSafe(idx(), impl.idx(), vector);
   }
 
   @Override
-  public void copyAsField(String name, MapWriter writer){
+  public void copyAsField(String name, MapWriter writer) {
     SingleMapWriter impl = (SingleMapWriter) writer.map(name);
     impl.container.copyFromSafe(idx(), impl.idx(), vector);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
index 515d4ab8ce907..f3e9b8773f25e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.ValueVector;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
index 2bd0ca87cd074..b98c36d2bf721 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.UInt4Vector;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
index c4eb3dc739a49..df142c7819d7c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.reader;
 
 import org.apache.arrow.vector.complex.reader.BaseReader.ListReader;
@@ -24,6 +25,5 @@
 import org.apache.arrow.vector.complex.reader.BaseReader.ScalarReader;
 
 
-
 public interface FieldReader extends MapReader, ListReader, ScalarReader, RepeatedMapReader, RepeatedListReader {
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
index ecffe0bec0e84..a2a1f5d000a8a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.writer;
 
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
@@ -23,5 +24,6 @@
 
 public interface FieldWriter extends MapWriter, ListWriter, ScalarWriter {
   void allocate();
+
   void clear();
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java
index 0c1cadfdafdbf..c2f692035946f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.dictionary;
 
 import java.util.Objects;
@@ -53,8 +54,12 @@ public String toString() {
 
   @Override
   public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
     Dictionary that = (Dictionary) o;
     return Objects.equals(encoding, that.encoding) && Objects.equals(dictionary, that.dictionary);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
index 0f49ce61f1cdf..7e20794cbbed2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.dictionary;
 
 import java.lang.reflect.InvocationTargetException;
@@ -39,7 +40,7 @@ public class DictionaryEncoder {
   /**
    * Dictionary encodes a vector with a provided dictionary. The dictionary must contain all values in the vector.
    *
-   * @param vector vector to encode
+   * @param vector     vector to encode
    * @param dictionary dictionary used for encoding
    * @return dictionary encoded vector
    */
@@ -55,7 +56,7 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
 
     Field valueField = vector.getField();
     FieldType indexFieldType = new FieldType(valueField.isNullable(), dictionary.getEncoding().getIndexType(),
-      dictionary.getEncoding(), valueField.getMetadata());
+        dictionary.getEncoding(), valueField.getMetadata());
     Field indexField = new Field(valueField.getName(), indexFieldType, null);
 
     // vector to hold our indices (dictionary encoded values)
@@ -65,11 +66,11 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
     // use reflection to pull out the set method
     // TODO implement a common interface for int vectors
     Method setter = null;
-    for (Class<?> c: ImmutableList.of(int.class, long.class)) {
+    for (Class<?> c : ImmutableList.of(int.class, long.class)) {
       try {
         setter = mutator.getClass().getMethod("set", int.class, c);
         break;
-      } catch(NoSuchMethodException e) {
+      } catch (NoSuchMethodException e) {
         // ignore
       }
     }
@@ -108,7 +109,7 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
   /**
    * Decodes a dictionary encoded array using the provided dictionary.
    *
-   * @param indices dictionary encoded values, must be int type
+   * @param indices    dictionary encoded values, must be int type
    * @param dictionary dictionary used to decode the values
    * @return vector with values restored from dictionary
    */
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
index 87516c9a8fc5b..a170cea21d273 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryProvider.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.dictionary;
 
 import java.util.HashMap;
@@ -31,7 +32,7 @@ public static class MapDictionaryProvider implements DictionaryProvider {
 
     public MapDictionaryProvider(Dictionary... dictionaries) {
       this.map = new HashMap<>();
-      for (Dictionary dictionary: dictionaries) {
+      for (Dictionary dictionary : dictionaries) {
         put(dictionary);
       }
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java
index 90fb02b059707..e1b4d6a8b215e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import org.apache.arrow.flatbuf.Block;
@@ -64,19 +65,25 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (this == obj)
+    if (this == obj) {
       return true;
-    if (obj == null)
+    }
+    if (obj == null) {
       return false;
-    if (getClass() != obj.getClass())
+    }
+    if (getClass() != obj.getClass()) {
       return false;
+    }
     ArrowBlock other = (ArrowBlock) obj;
-    if (bodyLength != other.bodyLength)
+    if (bodyLength != other.bodyLength) {
       return false;
-    if (metadataLength != other.metadataLength)
+    }
+    if (metadataLength != other.metadataLength) {
       return false;
-    if (offset != other.offset)
+    }
+    if (offset != other.offset) {
       return false;
+    }
     return true;
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
index f4d6ada932494..d711b9c6c1e26 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
@@ -117,7 +118,7 @@ private ArrowDictionaryBatch readDictionaryBatch(SeekableReadChannel in,
                                                    ArrowBlock block,
                                                    BufferAllocator allocator) throws IOException {
     LOGGER.debug(String.format("DictionaryRecordBatch at %d, metadata: %d, body: %d",
-       block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
+        block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
     in.setPosition(block.getOffset());
     ArrowDictionaryBatch batch = MessageSerializer.deserializeDictionaryBatch(in, block, allocator);
     if (batch == null) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
index 23d210a3ee73b..06519bc49fd1c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
@@ -47,7 +48,7 @@ protected void endInternal(WriteChannel out,
                              List<ArrowBlock> records) throws IOException {
     long footerStart = out.getCurrentPosition();
     out.write(new ArrowFooter(schema, dictionaries, records), false);
-    int footerLength = (int)(out.getCurrentPosition() - footerStart);
+    int footerLength = (int) (out.getCurrentPosition() - footerStart);
     if (footerLength <= 0) {
       throw new InvalidArrowFileException("invalid footer");
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
index 1c0008a9184a0..1e95321fdec5b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector;
@@ -48,7 +49,7 @@ public ArrowFooter(Footer footer) {
         Schema.convertSchema(footer.schema()),
         dictionaries(footer),
         recordBatches(footer)
-        );
+    );
   }
 
   private static List<ArrowBlock> recordBatches(Footer footer) {
@@ -112,28 +113,37 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (this == obj)
+    if (this == obj) {
       return true;
-    if (obj == null)
+    }
+    if (obj == null) {
       return false;
-    if (getClass() != obj.getClass())
+    }
+    if (getClass() != obj.getClass()) {
       return false;
+    }
     ArrowFooter other = (ArrowFooter) obj;
     if (dictionaries == null) {
-      if (other.dictionaries != null)
+      if (other.dictionaries != null) {
         return false;
-    } else if (!dictionaries.equals(other.dictionaries))
+      }
+    } else if (!dictionaries.equals(other.dictionaries)) {
       return false;
+    }
     if (recordBatches == null) {
-      if (other.recordBatches != null)
+      if (other.recordBatches != null) {
         return false;
-    } else if (!recordBatches.equals(other.recordBatches))
+      }
+    } else if (!recordBatches.equals(other.recordBatches)) {
       return false;
+    }
     if (schema == null) {
-      if (other.schema != null)
+      if (other.schema != null) {
         return false;
-    } else if (!schema.equals(other.schema))
+      }
+    } else if (!schema.equals(other.schema)) {
       return false;
+    }
     return true;
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
index 99ea96b3856d5..0d2da375295fe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index f6b104145527c..646d6feeef086 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
@@ -94,12 +95,21 @@ public boolean loadNextBatch() throws IOException {
     ArrowMessageVisitor<Boolean> visitor = new ArrowMessageVisitor<Boolean>() {
       @Override
       public Boolean visit(ArrowDictionaryBatch message) {
-        try { load(message); } finally { message.close(); }
+        try {
+          load(message);
+        } finally {
+          message.close();
+        }
         return true;
       }
+
       @Override
       public Boolean visit(ArrowRecordBatch message) {
-        try { loader.load(message); } finally { message.close(); }
+        try {
+          loader.load(message);
+        } finally {
+          message.close();
+        }
         return false;
       }
     };
@@ -119,13 +129,15 @@ public Boolean visit(ArrowRecordBatch message) {
     return readBatch;
   }
 
-  public long bytesRead() { return in.bytesRead(); }
+  public long bytesRead() {
+    return in.bytesRead();
+  }
 
   @Override
   public void close() throws IOException {
     if (initialized) {
       root.close();
-      for (Dictionary dictionary: dictionaries.values()) {
+      for (Dictionary dictionary : dictionaries.values()) {
         dictionary.getVector().close();
       }
     }
@@ -153,7 +165,7 @@ private void initialize() throws IOException {
     Map<Long, Dictionary> dictionaries = new HashMap<>();
 
     // Convert fields with dictionaries to have the index type
-    for (Field field: originalSchema.getFields()) {
+    for (Field field : originalSchema.getFields()) {
       Field updated = DictionaryUtility.toMemoryFormat(field, allocator, dictionaries);
       fields.add(updated);
       vectors.add(updated.createVector(allocator));
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 3b37071382ff6..b35aba5426e4a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
@@ -60,9 +61,9 @@ public abstract class ArrowWriter implements AutoCloseable {
   /**
    * Note: fields are not closed when the writer is closed
    *
-   * @param root the vectors to write to the output
+   * @param root     the vectors to write to the output
    * @param provider where to find the dictionaries
-   * @param out the output where to write
+   * @param out      the output where to write
    */
   protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
     this.unloader = new VectorUnloader(root);
@@ -72,13 +73,13 @@ protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, Writab
     Set<Long> dictionaryIdsUsed = new HashSet<>();
 
     // Convert fields with dictionaries to have dictionary type
-    for (Field field: root.getSchema().getFields()) {
+    for (Field field : root.getSchema().getFields()) {
       fields.add(DictionaryUtility.toMessageFormat(field, provider, dictionaryIdsUsed));
     }
 
     // Create a record batch for each dictionary
     this.dictionaries = new ArrayList<>(dictionaryIdsUsed.size());
-    for (long id: dictionaryIdsUsed) {
+    for (long id : dictionaryIdsUsed) {
       Dictionary dictionary = provider.lookup(id);
       FieldVector vector = dictionary.getVector();
       int count = vector.getAccessor().getValueCount();
@@ -105,7 +106,7 @@ public void writeBatch() throws IOException {
   protected void writeRecordBatch(ArrowRecordBatch batch) throws IOException {
     ArrowBlock block = MessageSerializer.serialize(out, batch);
     LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d",
-      block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
+        block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
     recordBlocks.add(block);
   }
 
@@ -114,7 +115,9 @@ public void end() throws IOException {
     ensureEnded();
   }
 
-  public long bytesWritten() { return out.getCurrentPosition(); }
+  public long bytesWritten() {
+    return out.getCurrentPosition();
+  }
 
   private void ensureStarted() throws IOException {
     if (!started) {
@@ -128,7 +131,7 @@ private void ensureStarted() throws IOException {
         try {
           ArrowBlock block = MessageSerializer.serialize(out, batch);
           LOGGER.debug(String.format("DictionaryRecordBatch at %d, metadata: %d, body: %d",
-            block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
+              block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
           dictionaryBlocks.add(block);
         } finally {
           batch.close();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java b/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java
index 3ec75dcb12a2b..607207f41b06c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 public class InvalidArrowFileException extends RuntimeException {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
index 87450e38f6852..b0eb8f3d84d9a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
@@ -37,11 +38,14 @@ public ReadChannel(ReadableByteChannel in) {
     this.in = in;
   }
 
-  public long bytesRead() { return bytesRead; }
+  public long bytesRead() {
+    return bytesRead;
+  }
 
   /**
    * Reads bytes into buffer until it is full (buffer.remaining() == 0). Returns the
    * number of bytes read which can be less than full if there are no more.
+   *
    * @param buffer The buffer to read to
    * @return the number of byte read
    * @throws IOException if nit enough bytes left to read
@@ -51,9 +55,13 @@ public int readFully(ByteBuffer buffer) throws IOException {
     int totalRead = 0;
     while (buffer.remaining() != 0) {
       int read = in.read(buffer);
-      if (read < 0) return totalRead;
+      if (read < 0) {
+        return totalRead;
+      }
       totalRead += read;
-      if (read == 0) break;
+      if (read == 0) {
+        break;
+      }
     }
     this.bytesRead += totalRead;
     return totalRead;
@@ -61,8 +69,9 @@ public int readFully(ByteBuffer buffer) throws IOException {
 
   /**
    * Reads up to len into buffer. Returns bytes read.
+   *
    * @param buffer the buffer to read to
-   * @param l the amount of bytes to read
+   * @param l      the amount of bytes to read
    * @return the number of bytes read
    * @throws IOException if nit enough bytes left to read
    */
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java
index 914c3cb4b33a9..46bea1314da63 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
@@ -22,18 +23,18 @@
 
 public class SeekableReadChannel extends ReadChannel {
 
-    private final SeekableByteChannel in;
+  private final SeekableByteChannel in;
 
-    public SeekableReadChannel(SeekableByteChannel in) {
-        super(in);
-        this.in = in;
-    }
+  public SeekableReadChannel(SeekableByteChannel in) {
+    super(in);
+    this.in = in;
+  }
 
-    public void setPosition(long position) throws IOException {
-        in.position(position);
-    }
+  public void setPosition(long position) throws IOException {
+    in.position(position);
+  }
 
-    public long size() throws IOException {
-        return in.size();
-    }
+  public long size() throws IOException {
+    return in.size();
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
index 42104d181a2d0..89c9d1f9b7a44 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.IOException;
@@ -62,7 +63,7 @@ public long writeZeros(int zeroCount) throws IOException {
 
   public long align() throws IOException {
     if (currentPosition % 8 != 0) { // align on 8 byte boundaries
-      return writeZeros(8 - (int)(currentPosition % 8));
+      return writeZeros(8 - (int) (currentPosition % 8));
     }
     return 0;
   }
@@ -77,10 +78,10 @@ public long write(ByteBuffer buffer) throws IOException {
 
   public static byte[] intToBytes(int value) {
     byte[] outBuffer = new byte[4];
-    outBuffer[3] = (byte)(value >>> 24);
-    outBuffer[2] = (byte)(value >>> 16);
-    outBuffer[1] = (byte)(value >>>  8);
-    outBuffer[0] = (byte)(value >>>  0);
+    outBuffer[3] = (byte) (value >>> 24);
+    outBuffer[2] = (byte) (value >>> 16);
+    outBuffer[1] = (byte) (value >>> 8);
+    outBuffer[0] = (byte) (value >>> 0);
     return outBuffer;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 364d273fadae4..484a82fdaab67 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.file.json;
 
 import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
@@ -114,7 +115,7 @@ public Schema start() throws JsonParseException, IOException {
       dictionaries = new HashMap<>();
 
       // Convert fields with dictionaries to have the index type
-      for (Field field: originalSchema.getFields()) {
+      for (Field field : originalSchema.getFields()) {
         fields.add(DictionaryUtility.toMemoryFormat(field, allocator, dictionaries));
       }
       this.schema = new Schema(fields, originalSchema.getCustomMetadata());
@@ -233,7 +234,7 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         BufferBacked innerVector = fieldInnerVectors.get(v);
         nextFieldIs(vectorType.getName());
         readToken(START_ARRAY);
-        ValueVector valueVector = (ValueVector)innerVector;
+        ValueVector valueVector = (ValueVector) innerVector;
         valueVector.allocateNew();
         Mutator mutator = valueVector.getMutator();
 
@@ -262,7 +263,7 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         readToken(END_ARRAY);
       }
       if (vector instanceof NullableMapVector) {
-        ((NullableMapVector)vector).valueCount = count;
+        ((NullableMapVector) vector).valueCount = count;
       }
     }
     readToken(END_OBJECT);
@@ -278,96 +279,96 @@ private byte[] decodeHexSafe(String hexString) throws IOException {
 
   private void setValueFromParser(ValueVector valueVector, int i) throws IOException {
     switch (valueVector.getMinorType()) {
-    case BIT:
-      ((BitVector)valueVector).getMutator().set(i, parser.readValueAs(Boolean.class) ? 1 : 0);
-      break;
-    case TINYINT:
-      ((TinyIntVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case SMALLINT:
-      ((SmallIntVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case INT:
-      ((IntVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case BIGINT:
-      ((BigIntVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case UINT1:
-      ((UInt1Vector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case UINT2:
-      ((UInt2Vector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case UINT4:
-      ((UInt4Vector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case UINT8:
-      ((UInt8Vector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case FLOAT4:
-      ((Float4Vector)valueVector).getMutator().set(i, parser.readValueAs(Float.class));
-      break;
-    case FLOAT8:
-      ((Float8Vector)valueVector).getMutator().set(i, parser.readValueAs(Double.class));
-      break;
-    case VARBINARY:
-      ((VarBinaryVector)valueVector).getMutator().setSafe(i, decodeHexSafe(parser.readValueAs(String.class)));
-      break;
-    case VARCHAR:
-      ((VarCharVector)valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8));
-      break;
-    case DATEDAY:
-      ((DateDayVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case DATEMILLI:
-      ((DateMilliVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESEC:
-      ((TimeSecVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case TIMEMILLI:
-      ((TimeMilliVector)valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-      break;
-    case TIMEMICRO:
-      ((TimeMicroVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMENANO:
-      ((TimeNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESTAMPSEC:
-      ((TimeStampSecVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESTAMPMILLI:
-      ((TimeStampMilliVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESTAMPMICRO:
-      ((TimeStampMicroVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESTAMPNANO:
-      ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESTAMPSECTZ:
-      ((TimeStampSecTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESTAMPMILLITZ:
-      ((TimeStampMilliTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESTAMPMICROTZ:
-      ((TimeStampMicroTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    case TIMESTAMPNANOTZ:
-      ((TimeStampNanoTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-      break;
-    default:
-      throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
+      case BIT:
+        ((BitVector) valueVector).getMutator().set(i, parser.readValueAs(Boolean.class) ? 1 : 0);
+        break;
+      case TINYINT:
+        ((TinyIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case SMALLINT:
+        ((SmallIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case INT:
+        ((IntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case BIGINT:
+        ((BigIntVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case UINT1:
+        ((UInt1Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case UINT2:
+        ((UInt2Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case UINT4:
+        ((UInt4Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case UINT8:
+        ((UInt8Vector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case FLOAT4:
+        ((Float4Vector) valueVector).getMutator().set(i, parser.readValueAs(Float.class));
+        break;
+      case FLOAT8:
+        ((Float8Vector) valueVector).getMutator().set(i, parser.readValueAs(Double.class));
+        break;
+      case VARBINARY:
+        ((VarBinaryVector) valueVector).getMutator().setSafe(i, decodeHexSafe(parser.readValueAs(String.class)));
+        break;
+      case VARCHAR:
+        ((VarCharVector) valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8));
+        break;
+      case DATEDAY:
+        ((DateDayVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case DATEMILLI:
+        ((DateMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESEC:
+        ((TimeSecVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case TIMEMILLI:
+        ((TimeMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
+        break;
+      case TIMEMICRO:
+        ((TimeMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMENANO:
+        ((TimeNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESTAMPSEC:
+        ((TimeStampSecVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESTAMPMILLI:
+        ((TimeStampMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESTAMPMICRO:
+        ((TimeStampMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESTAMPNANO:
+        ((TimeStampNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESTAMPSECTZ:
+        ((TimeStampSecTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESTAMPMILLITZ:
+        ((TimeStampMilliTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESTAMPMICROTZ:
+        ((TimeStampMicroTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      case TIMESTAMPNANOTZ:
+        ((TimeStampNanoTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+        break;
+      default:
+        throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
     }
   }
 
   @Override
   public void close() throws IOException {
     parser.close();
-    for (Dictionary dictionary: dictionaries.values()) {
+    for (Dictionary dictionary : dictionaries.values()) {
       dictionary.getVector().close();
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index befa92251f0f9..a2229cef23150 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.file.json;
 
 import java.io.File;
@@ -60,12 +61,15 @@ public class JsonFileWriter implements AutoCloseable {
 
   public static final class JSONWriteConfig {
     private final boolean pretty;
+
     private JSONWriteConfig(boolean pretty) {
       this.pretty = pretty;
     }
+
     private JSONWriteConfig() {
       this.pretty = false;
     }
+
     public JSONWriteConfig pretty(boolean pretty) {
       return new JSONWriteConfig(pretty);
     }
@@ -98,7 +102,7 @@ public void start(Schema schema, DictionaryProvider provider) throws IOException
     this.schema = schema;  // Store original Schema to ensure batches written match
 
     // Convert fields with dictionaries to have dictionary type
-    for (Field field: schema.getFields()) {
+    for (Field field : schema.getFields()) {
       fields.add(DictionaryUtility.toMessageFormat(field, provider, dictionaryIdsUsed));
     }
     Schema updatedSchema = new Schema(fields, schema.getCustomMetadata());
@@ -117,7 +121,7 @@ public void start(Schema schema, DictionaryProvider provider) throws IOException
 
   private void writeDictionaryBatches(JsonGenerator generator, Set<Long> dictionaryIdsUsed, DictionaryProvider provider) throws IOException {
     generator.writeArrayFieldStart("dictionaries");
-    for (Long id: dictionaryIdsUsed) {
+    for (Long id : dictionaryIdsUsed) {
       generator.writeStartObject();
       generator.writeObjectField("id", id);
 
@@ -170,7 +174,7 @@ private void writeVector(Field field, FieldVector vector) throws IOException {
         ArrowVectorType vectorType = vectorTypes.get(v);
         BufferBacked innerVector = fieldInnerVectors.get(v);
         generator.writeArrayFieldStart(vectorType.getName());
-        ValueVector valueVector = (ValueVector)innerVector;
+        ValueVector valueVector = (ValueVector) innerVector;
         for (int i = 0; i < valueVector.getAccessor().getValueCount(); i++) {
           writeValueToGenerator(valueVector, i);
         }
@@ -197,37 +201,37 @@ private void writeVector(Field field, FieldVector vector) throws IOException {
   private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException {
     switch (valueVector.getMinorType()) {
       case DATEDAY:
-        generator.writeNumber(((DateDayVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((DateDayVector) valueVector).getAccessor().get(i));
         break;
       case DATEMILLI:
-        generator.writeNumber(((DateMilliVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((DateMilliVector) valueVector).getAccessor().get(i));
         break;
       case TIMESEC:
-        generator.writeNumber(((TimeSecVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((TimeSecVector) valueVector).getAccessor().get(i));
         break;
       case TIMEMILLI:
-        generator.writeNumber(((TimeMilliVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((TimeMilliVector) valueVector).getAccessor().get(i));
         break;
       case TIMEMICRO:
-        generator.writeNumber(((TimeMicroVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((TimeMicroVector) valueVector).getAccessor().get(i));
         break;
       case TIMENANO:
-        generator.writeNumber(((TimeNanoVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((TimeNanoVector) valueVector).getAccessor().get(i));
         break;
       case TIMESTAMPSEC:
-        generator.writeNumber(((TimeStampSecVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((TimeStampSecVector) valueVector).getAccessor().get(i));
         break;
       case TIMESTAMPMILLI:
-        generator.writeNumber(((TimeStampMilliVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((TimeStampMilliVector) valueVector).getAccessor().get(i));
         break;
       case TIMESTAMPMICRO:
-        generator.writeNumber(((TimeStampMicroVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((TimeStampMicroVector) valueVector).getAccessor().get(i));
         break;
       case TIMESTAMPNANO:
-        generator.writeNumber(((TimeStampNanoVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((TimeStampNanoVector) valueVector).getAccessor().get(i));
         break;
       case BIT:
-        generator.writeNumber(((BitVector)valueVector).getAccessor().get(i));
+        generator.writeNumber(((BitVector) valueVector).getAccessor().get(i));
         break;
       case VARBINARY:
         String hexString = Hex.encodeHexString(((VarBinaryVector) valueVector).getAccessor().get(i));
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ComplexHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ComplexHolder.java
index 0f9310da55b79..9f923ee343cdc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/ComplexHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/ComplexHolder.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.holders;
 
 import org.apache.arrow.vector.complex.reader.FieldReader;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedListHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedListHolder.java
index 83506cdc17549..d8b2317f8ff4f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedListHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedListHolder.java
@@ -15,9 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.holders;
 
-public final class RepeatedListHolder implements ValueHolder{
+public final class RepeatedListHolder implements ValueHolder {
   public int start;
   public int end;
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java
index 85d782b381835..c400b4dc49a80 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java
@@ -15,9 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.holders;
 
-public final class RepeatedMapHolder implements ValueHolder{
+public final class RepeatedMapHolder implements ValueHolder {
   public int start;
   public int end;
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
index b1b695e58a954..46cf4c8e8c712 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.holders;
 
 import org.apache.arrow.vector.complex.reader.FieldReader;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
index 16777c806ec2d..4d012635e548a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.holders;
 
 /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
index 4e2e200d67645..d8c9e3001d0a5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import org.apache.arrow.flatbuf.Buffer;
@@ -58,19 +59,25 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (this == obj)
+    if (this == obj) {
       return true;
-    if (obj == null)
+    }
+    if (obj == null) {
       return false;
-    if (getClass() != obj.getClass())
+    }
+    if (getClass() != obj.getClass()) {
       return false;
+    }
     ArrowBuffer other = (ArrowBuffer) obj;
-    if (offset != other.offset)
+    if (offset != other.offset) {
       return false;
-    if (page != other.page)
+    }
+    if (page != other.page) {
       return false;
-    if (size != other.size)
+    }
+    if (size != other.size) {
       return false;
+    }
     return true;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java
index 901877b7058cd..635fa3fb42307 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import com.google.flatbuffers.FlatBufferBuilder;
@@ -22,39 +23,48 @@
 
 public class ArrowDictionaryBatch implements ArrowMessage {
 
-    private final long dictionaryId;
-    private final ArrowRecordBatch dictionary;
-
-    public ArrowDictionaryBatch(long dictionaryId, ArrowRecordBatch dictionary) {
-        this.dictionaryId = dictionaryId;
-        this.dictionary = dictionary;
-    }
-
-    public long getDictionaryId() { return dictionaryId; }
-    public ArrowRecordBatch getDictionary() { return dictionary; }
-
-    @Override
-    public int writeTo(FlatBufferBuilder builder) {
-        int dataOffset = dictionary.writeTo(builder);
-        DictionaryBatch.startDictionaryBatch(builder);
-        DictionaryBatch.addId(builder, dictionaryId);
-        DictionaryBatch.addData(builder, dataOffset);
-        return DictionaryBatch.endDictionaryBatch(builder);
-    }
-
-    @Override
-    public int computeBodyLength() { return dictionary.computeBodyLength(); }
-
-    @Override
-    public <T> T accepts(ArrowMessageVisitor<T> visitor) { return visitor.visit(this); }
-
-    @Override
-    public String toString() {
-       return "ArrowDictionaryBatch [dictionaryId=" + dictionaryId + ", dictionary=" + dictionary + "]";
-    }
-
-    @Override
-    public void close() {
-      dictionary.close();
-    }
+  private final long dictionaryId;
+  private final ArrowRecordBatch dictionary;
+
+  public ArrowDictionaryBatch(long dictionaryId, ArrowRecordBatch dictionary) {
+    this.dictionaryId = dictionaryId;
+    this.dictionary = dictionary;
+  }
+
+  public long getDictionaryId() {
+    return dictionaryId;
+  }
+
+  public ArrowRecordBatch getDictionary() {
+    return dictionary;
+  }
+
+  @Override
+  public int writeTo(FlatBufferBuilder builder) {
+    int dataOffset = dictionary.writeTo(builder);
+    DictionaryBatch.startDictionaryBatch(builder);
+    DictionaryBatch.addId(builder, dictionaryId);
+    DictionaryBatch.addData(builder, dataOffset);
+    return DictionaryBatch.endDictionaryBatch(builder);
+  }
+
+  @Override
+  public int computeBodyLength() {
+    return dictionary.computeBodyLength();
+  }
+
+  @Override
+  public <T> T accepts(ArrowMessageVisitor<T> visitor) {
+    return visitor.visit(this);
+  }
+
+  @Override
+  public String toString() {
+    return "ArrowDictionaryBatch [dictionaryId=" + dictionaryId + ", dictionary=" + dictionary + "]";
+  }
+
+  @Override
+  public void close() {
+    dictionary.close();
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
index 72ce982f2e7ee..3ed384ed7e280 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import org.apache.arrow.flatbuf.FieldNode;
@@ -34,7 +35,7 @@ public ArrowFieldNode(int length, int nullCount) {
 
   @Override
   public int writeTo(FlatBufferBuilder builder) {
-    return FieldNode.createFieldNode(builder, (long)length, (long)nullCount);
+    return FieldNode.createFieldNode(builder, (long) length, (long) nullCount);
   }
 
   public int getNullCount() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java
index d307428889b0f..f59b4b6c1721e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java
@@ -15,16 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 public interface ArrowMessage extends FBSerializable, AutoCloseable {
 
-    public int computeBodyLength();
+  public int computeBodyLength();
+
+  public <T> T accepts(ArrowMessageVisitor<T> visitor);
 
-    public <T> T accepts(ArrowMessageVisitor<T> visitor);
+  public static interface ArrowMessageVisitor<T> {
+    public T visit(ArrowDictionaryBatch message);
 
-    public static interface ArrowMessageVisitor<T> {
-        public T visit(ArrowDictionaryBatch message);
-        public T visit(ArrowRecordBatch message);
-    }
+    public T visit(ArrowRecordBatch message);
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index 6a716fa138a7a..d2f3782469597 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector;
@@ -36,10 +37,14 @@ public class ArrowRecordBatch implements ArrowMessage {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowRecordBatch.class);
 
-  /** number of records */
+  /**
+   * number of records
+   */
   private final int length;
 
-  /** Nodes correspond to the pre-ordered flattened logical schema */
+  /**
+   * Nodes correspond to the pre-ordered flattened logical schema
+   */
   private final List<ArrowFieldNode> nodes;
 
   private final List<ArrowBuf> buffers;
@@ -53,8 +58,8 @@ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> b
   }
 
   /**
-   * @param length how many rows in this batch
-   * @param nodes field level info
+   * @param length  how many rows in this batch
+   * @param nodes   field level info
    * @param buffers will be retained until this recordBatch is closed
    */
   public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers, boolean alignBuffers) {
@@ -119,7 +124,9 @@ public int writeTo(FlatBufferBuilder builder) {
   }
 
   @Override
-  public <T> T accepts(ArrowMessageVisitor<T> visitor) { return visitor.visit(this); }
+  public <T> T accepts(ArrowMessageVisitor<T> visitor) {
+    return visitor.visit(this);
+  }
 
   /**
    * releases the buffers
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
index 68da7052f2b8b..9d2fdfaafe4aa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import java.util.Map;
@@ -34,10 +35,11 @@ public class ArrowVectorType {
   public static final ArrowVectorType TYPE = new ArrowVectorType(VectorType.TYPE);
 
   private static final Map<String, ArrowVectorType> typeByName;
+
   static {
-    ArrowVectorType[] types = { DATA, OFFSET, VALIDITY, TYPE };
+    ArrowVectorType[] types = {DATA, OFFSET, VALIDITY, TYPE};
     Builder<String, ArrowVectorType> builder = ImmutableMap.builder();
-    for (ArrowVectorType type: types) {
+    for (ArrowVectorType type : types) {
       builder.put(type.getName(), type);
     }
     typeByName = builder.build();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java
index d23ed91948e5d..91d60ea995b89 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import com.google.flatbuffers.FlatBufferBuilder;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java
index 31c17ad6df02b..ae5aa555e745e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import java.util.ArrayList;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
index 24840ec988ac3..29407bf1ab4e1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import static java.util.Arrays.asList;
@@ -61,11 +62,13 @@ public class TypeLayout {
   public static TypeLayout getTypeLayout(final ArrowType arrowType) {
     TypeLayout layout = arrowType.accept(new ArrowTypeVisitor<TypeLayout>() {
 
-      @Override public TypeLayout visit(Int type) {
+      @Override
+      public TypeLayout visit(Int type) {
         return newFixedWidthTypeLayout(dataVector(type.getBitWidth()));
       }
 
-      @Override public TypeLayout visit(Union type) {
+      @Override
+      public TypeLayout visit(Union type) {
         List<VectorLayout> vectors;
         switch (type.getMode()) {
           case Dense:
@@ -74,12 +77,12 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
                 validityVector(),
                 typeVector(),
                 offsetVector() // offset to find the vector
-                );
+            );
             break;
           case Sparse:
             vectors = asList(
                 typeVector() // type of the value at the index or 0 if null
-                );
+            );
             break;
           default:
             throw new UnsupportedOperationException("Unsupported Union Mode: " + type.getMode());
@@ -87,64 +90,73 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
         return new TypeLayout(vectors);
       }
 
-      @Override public TypeLayout visit(Struct type) {
+      @Override
+      public TypeLayout visit(Struct type) {
         List<VectorLayout> vectors = asList(
             validityVector()
-            );
+        );
         return new TypeLayout(vectors);
       }
 
-      @Override public TypeLayout visit(Timestamp type) {
+      @Override
+      public TypeLayout visit(Timestamp type) {
         return newFixedWidthTypeLayout(dataVector(64));
       }
 
-      @Override public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) {
+      @Override
+      public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) {
         List<VectorLayout> vectors = asList(
             validityVector(),
             offsetVector()
-            );
+        );
         return new TypeLayout(vectors);
       }
 
-      @Override public TypeLayout visit(FixedSizeList type) {
+      @Override
+      public TypeLayout visit(FixedSizeList type) {
         List<VectorLayout> vectors = asList(
             validityVector()
-            );
+        );
         return new TypeLayout(vectors);
       }
 
-      @Override public TypeLayout visit(FloatingPoint type) {
+      @Override
+      public TypeLayout visit(FloatingPoint type) {
         int bitWidth;
         switch (type.getPrecision()) {
-        case HALF:
-          bitWidth = 16;
-          break;
-        case SINGLE:
-          bitWidth = 32;
-          break;
-        case DOUBLE:
-          bitWidth = 64;
-          break;
-        default:
-          throw new UnsupportedOperationException("Unsupported Precision: " + type.getPrecision());
+          case HALF:
+            bitWidth = 16;
+            break;
+          case SINGLE:
+            bitWidth = 32;
+            break;
+          case DOUBLE:
+            bitWidth = 64;
+            break;
+          default:
+            throw new UnsupportedOperationException("Unsupported Precision: " + type.getPrecision());
         }
         return newFixedWidthTypeLayout(dataVector(bitWidth));
       }
 
-      @Override public TypeLayout visit(Decimal type) {
+      @Override
+      public TypeLayout visit(Decimal type) {
         // TODO: check size
         return newFixedWidthTypeLayout(dataVector(64)); // actually depends on the type fields
       }
 
-      @Override public TypeLayout visit(Bool type) {
+      @Override
+      public TypeLayout visit(Bool type) {
         return newFixedWidthTypeLayout(booleanVector());
       }
 
-      @Override public TypeLayout visit(Binary type) {
+      @Override
+      public TypeLayout visit(Binary type) {
         return newVariableWidthTypeLayout();
       }
 
-      @Override public TypeLayout visit(Utf8 type) {
+      @Override
+      public TypeLayout visit(Utf8 type) {
         return newVariableWidthTypeLayout();
       }
 
@@ -178,12 +190,12 @@ public TypeLayout visit(Time type) {
       @Override
       public TypeLayout visit(Interval type) { // TODO: check size
         switch (type.getUnit()) {
-        case DAY_TIME:
-          return newFixedWidthTypeLayout(dataVector(64));
-        case YEAR_MONTH:
-          return newFixedWidthTypeLayout(dataVector(64));
-        default:
-          throw new UnsupportedOperationException("Unknown unit " + type.getUnit());
+          case DAY_TIME:
+            return newFixedWidthTypeLayout(dataVector(64));
+          case YEAR_MONTH:
+            return newFixedWidthTypeLayout(dataVector(64));
+          default:
+            throw new UnsupportedOperationException("Unknown unit " + type.getUnit());
         }
       }
 
@@ -228,12 +240,15 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (this == obj)
+    if (this == obj) {
       return true;
-    if (obj == null)
+    }
+    if (obj == null) {
       return false;
-    if (getClass() != obj.getClass())
+    }
+    if (getClass() != obj.getClass()) {
       return false;
+    }
     TypeLayout other = (TypeLayout) obj;
     return vectors.equals(other.vectors);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
index 2073795b2a199..0871baf38edaa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.schema;
 
 import static org.apache.arrow.vector.schema.ArrowVectorType.DATA;
@@ -48,16 +49,16 @@ public static VectorLayout offsetVector() {
 
   public static VectorLayout dataVector(int typeBitWidth) {
     switch (typeBitWidth) {
-    case 8:
-      return VALUES_8;
-    case 16:
-      return VALUES_16;
-    case 32:
-      return VALUES_32;
-    case 64:
-      return VALUES_64;
-    default:
-      throw new IllegalArgumentException("only 8, 16, 32, or 64 bits supported");
+      case 8:
+        return VALUES_8;
+      case 16:
+        return VALUES_16;
+      case 32:
+        return VALUES_32;
+      case 64:
+        return VALUES_64;
+      default:
+        throw new IllegalArgumentException("only 8, 16, 32, or 64 bits supported");
     }
   }
 
@@ -81,7 +82,7 @@ public static VectorLayout byteVector() {
   private VectorLayout(@JsonProperty("type") ArrowVectorType type, @JsonProperty("typeBitWidth") int typeBitWidth) {
     super();
     this.type = Preconditions.checkNotNull(type);
-    this.typeBitWidth = (short)typeBitWidth;
+    this.typeBitWidth = (short) typeBitWidth;
     if (typeBitWidth <= 0) {
       throw new IllegalArgumentException("bitWidth invalid: " + typeBitWidth);
     }
@@ -111,18 +112,22 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (this == obj)
+    if (this == obj) {
       return true;
-    if (obj == null)
+    }
+    if (obj == null) {
       return false;
-    if (getClass() != obj.getClass())
+    }
+    if (getClass() != obj.getClass()) {
       return false;
+    }
     VectorLayout other = (VectorLayout) obj;
     return type.equals(other.type) && (typeBitWidth == other.typeBitWidth);
   }
 
   @Override
-  public int writeTo(FlatBufferBuilder builder) {;
+  public int writeTo(FlatBufferBuilder builder) {
+    ;
     return org.apache.arrow.flatbuf.VectorLayout.createVectorLayout(builder, typeBitWidth, type.getType());
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
index 641978a516ae4..5b6300076b6c2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.stream;
 
 import java.io.IOException;
@@ -33,31 +34,33 @@
  */
 public class ArrowStreamReader extends ArrowReader<ReadChannel> {
 
-    /**
-    * Constructs a streaming read, reading bytes from 'in'. Non-blocking.
-    * @param in the stream to read from
-    * @param allocator to allocate new buffers
-    */
-    public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) {
-        super(new ReadChannel(in), allocator);
-    }
-
-    public ArrowStreamReader(InputStream in, BufferAllocator allocator) {
-        this(Channels.newChannel(in), allocator);
-    }
-
-    /**
-     * Reads the schema message from the beginning of the stream.
-     * @param in to allocate new buffers
-     * @return the deserialized arrow schema
-     */
-    @Override
-    protected Schema readSchema(ReadChannel in) throws IOException {
-        return MessageSerializer.deserializeSchema(in);
-    }
-
-    @Override
-    protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
-        return MessageSerializer.deserializeMessageBatch(in, allocator);
-    }
+  /**
+   * Constructs a streaming read, reading bytes from 'in'. Non-blocking.
+   *
+   * @param in        the stream to read from
+   * @param allocator to allocate new buffers
+   */
+  public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) {
+    super(new ReadChannel(in), allocator);
+  }
+
+  public ArrowStreamReader(InputStream in, BufferAllocator allocator) {
+    this(Channels.newChannel(in), allocator);
+  }
+
+  /**
+   * Reads the schema message from the beginning of the stream.
+   *
+   * @param in to allocate new buffers
+   * @return the deserialized arrow schema
+   */
+  @Override
+  protected Schema readSchema(ReadChannel in) throws IOException {
+    return MessageSerializer.deserializeSchema(in);
+  }
+
+  @Override
+  protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
+    return MessageSerializer.deserializeMessageBatch(in, allocator);
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
index ea29cd99804c8..b854cd2bb6e74 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.stream;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -35,22 +36,23 @@
 
 public class ArrowStreamWriter extends ArrowWriter {
 
-    public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, OutputStream out) {
-       this(root, provider, Channels.newChannel(out));
-    }
+  public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, OutputStream out) {
+    this(root, provider, Channels.newChannel(out));
+  }
 
-    public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
-       super(root, provider, out);
-    }
+  public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
+    super(root, provider, out);
+  }
 
-    @Override
-    protected void startInternal(WriteChannel out) throws IOException {}
+  @Override
+  protected void startInternal(WriteChannel out) throws IOException {
+  }
 
-    @Override
-    protected void endInternal(WriteChannel out,
-                               Schema schema,
-                               List<ArrowBlock> dictionaries,
-                               List<ArrowBlock> records) throws IOException {
-       out.writeIntLittleEndian(0);
-    }
+  @Override
+  protected void endInternal(WriteChannel out,
+                             Schema schema,
+                             List<ArrowBlock> dictionaries,
+                             List<ArrowBlock> records) throws IOException {
+    out.writeIntLittleEndian(0);
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 05ad92ded1d52..a70d029389427 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.stream;
 
 import java.io.IOException;
@@ -48,29 +49,30 @@
  * Utility class for serializing Messages. Messages are all serialized a similar way.
  * 1. 4 byte little endian message header prefix
  * 2. FB serialized Message: This includes it the body length, which is the serialized
- *    body and the type of the message.
+ * body and the type of the message.
  * 3. Serialized message.
  *
  * For schema messages, the serialization is simply the FB serialized Schema.
  *
  * For RecordBatch messages the serialization is:
- *   1. 4 byte little endian batch metadata header
- *   2. FB serialized RowBatch
- *   3. Padding to align to 8 byte boundary.
- *   4. serialized RowBatch buffers.
+ * 1. 4 byte little endian batch metadata header
+ * 2. FB serialized RowBatch
+ * 3. Padding to align to 8 byte boundary.
+ * 4. serialized RowBatch buffers.
  */
 public class MessageSerializer {
 
   public static int bytesToInt(byte[] bytes) {
     return ((bytes[3] & 255) << 24) +
-           ((bytes[2] & 255) << 16) +
-           ((bytes[1] & 255) <<  8) +
-           ((bytes[0] & 255) <<  0);
+        ((bytes[2] & 255) << 16) +
+        ((bytes[1] & 255) << 8) +
+        ((bytes[0] & 255) << 0);
   }
 
   /**
    * Serialize a schema object.
-   * @param out where to write the schema
+   *
+   * @param out    where to write the schema
    * @param schema the object to serialize to out
    * @return the resulting size of the serialized schema
    * @throws IOException if something went wrong
@@ -86,6 +88,7 @@ public static long serialize(WriteChannel out, Schema schema) throws IOException
 
   /**
    * Deserializes a schema object. Format is from serialize().
+   *
    * @param in the channel to deserialize from
    * @return the deserialized object
    * @throws IOException if something went wrong
@@ -106,13 +109,14 @@ public static Schema deserializeSchema(ReadChannel in) throws IOException {
 
   /**
    * Serializes an ArrowRecordBatch. Returns the offset and length of the written batch.
-   * @param out where to write the batch
+   *
+   * @param out   where to write the batch
    * @param batch the object to serialize to out
    * @return the serialized block metadata
    * @throws IOException if something went wrong
    */
   public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
-          throws IOException {
+      throws IOException {
 
     long start = out.getCurrentPosition();
     int bodyLength = batch.computeBodyLength();
@@ -125,9 +129,9 @@ public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
     int metadataLength = serializedMessage.remaining();
 
     // calculate alignment bytes so that metadata length points to the correct location after alignment
-    int padding = (int)((start + metadataLength + 4) % 8);
+    int padding = (int) ((start + metadataLength + 4) % 8);
     if (padding != 0) {
-        metadataLength += (8 - padding);
+      metadataLength += (8 - padding);
     }
 
     out.writeIntLittleEndian(metadataLength);
@@ -152,7 +156,7 @@ public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) t
       ArrowBuffer layout = buffersLayout.get(i);
       long startPosition = bufferStart + layout.getOffset();
       if (startPosition != out.getCurrentPosition()) {
-        out.writeZeros((int)(startPosition - out.getCurrentPosition()));
+        out.writeZeros((int) (startPosition - out.getCurrentPosition()));
       }
       out.write(buffer);
       if (out.getCurrentPosition() != startPosition + layout.getSize()) {
@@ -165,9 +169,10 @@ public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) t
 
   /**
    * Deserializes a RecordBatch
-   * @param in the channel to deserialize from
+   *
+   * @param in      the channel to deserialize from
    * @param message the object to derialize to
-   * @param alloc to allocate buffers
+   * @param alloc   to allocate buffers
    * @return the deserialized object
    * @throws IOException if something went wrong
    */
@@ -188,14 +193,15 @@ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message me
   /**
    * Deserializes a RecordBatch knowing the size of the entire message up front. This
    * minimizes the number of reads to the underlying stream.
-   * @param in the channel to deserialize from
+   *
+   * @param in    the channel to deserialize from
    * @param block the object to derialize to
    * @param alloc to allocate buffers
    * @return the deserialized object
    * @throws IOException if something went wrong
    */
   public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock block,
-      BufferAllocator alloc) throws IOException {
+                                                        BufferAllocator alloc) throws IOException {
     // Metadata length contains integer prefix plus byte padding
     long totalLen = block.getMetadataLength() + block.getBodyLength();
 
@@ -223,37 +229,38 @@ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock
 
   // Deserializes a record batch given the Flatbuffer metadata and in-memory body
   public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
-      ArrowBuf body) throws IOException {
+                                                        ArrowBuf body) throws IOException {
     // Now read the body
     int nodesLength = recordBatchFB.nodesLength();
     List<ArrowFieldNode> nodes = new ArrayList<>();
     for (int i = 0; i < nodesLength; ++i) {
       FieldNode node = recordBatchFB.nodes(i);
-      if ((int)node.length() != node.length() ||
-          (int)node.nullCount() != node.nullCount()) {
+      if ((int) node.length() != node.length() ||
+          (int) node.nullCount() != node.nullCount()) {
         throw new IOException("Cannot currently deserialize record batches with " +
-                              "node length larger than Int.MAX_VALUE");
+            "node length larger than Int.MAX_VALUE");
       }
-      nodes.add(new ArrowFieldNode((int)node.length(), (int)node.nullCount()));
+      nodes.add(new ArrowFieldNode((int) node.length(), (int) node.nullCount()));
     }
     List<ArrowBuf> buffers = new ArrayList<>();
     for (int i = 0; i < recordBatchFB.buffersLength(); ++i) {
       Buffer bufferFB = recordBatchFB.buffers(i);
-      ArrowBuf vectorBuffer = body.slice((int)bufferFB.offset(), (int)bufferFB.length());
+      ArrowBuf vectorBuffer = body.slice((int) bufferFB.offset(), (int) bufferFB.length());
       buffers.add(vectorBuffer);
     }
-    if ((int)recordBatchFB.length() != recordBatchFB.length()) {
+    if ((int) recordBatchFB.length() != recordBatchFB.length()) {
       throw new IOException("Cannot currently deserialize record batches over 2GB");
     }
     ArrowRecordBatch arrowRecordBatch =
-        new ArrowRecordBatch((int)recordBatchFB.length(), nodes, buffers);
+        new ArrowRecordBatch((int) recordBatchFB.length(), nodes, buffers);
     body.release();
     return arrowRecordBatch;
   }
 
   /**
    * Serializes a dictionary ArrowRecordBatch. Returns the offset and length of the written batch.
-   * @param out where to serialize
+   *
+   * @param out   where to serialize
    * @param batch the batch to serialize
    * @return the metadata of the serialized block
    * @throws IOException if something went wrong
@@ -290,15 +297,16 @@ public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch)
 
   /**
    * Deserializes a DictionaryBatch
-   * @param in where to read from
+   *
+   * @param in      where to read from
    * @param message the message message metadata to deserialize
-   * @param alloc the allocator for new buffers
+   * @param alloc   the allocator for new buffers
    * @return the corresponding dictionary batch
    * @throws IOException if something went wrong
    */
   public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
-                                                                 Message message,
-                                                                 BufferAllocator alloc) throws IOException {
+                                                                Message message,
+                                                                BufferAllocator alloc) throws IOException {
     DictionaryBatch dictionaryBatchFB = (DictionaryBatch) message.header(new DictionaryBatch());
 
     int bodyLength = (int) message.bodyLength();
@@ -315,7 +323,8 @@ public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
   /**
    * Deserializes a DictionaryBatch knowing the size of the entire message up front. This
    * minimizes the number of reads to the underlying stream.
-   * @param in where to read from
+   *
+   * @param in    where to read from
    * @param block block metadata for deserializing
    * @param alloc to allocate new buffers
    * @return the corresponding dictionary
@@ -345,7 +354,7 @@ public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
 
     // Now read the body
     final ArrowBuf body = buffer.slice(block.getMetadataLength(),
-                                       (int) totalLen - block.getMetadataLength());
+        (int) totalLen - block.getMetadataLength());
     ArrowRecordBatch recordBatch = deserializeRecordBatch(dictionaryBatchFB.data(), body);
     return new ArrowDictionaryBatch(dictionaryBatchFB.id(), recordBatch);
   }
@@ -359,22 +368,26 @@ public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocat
     }
 
     switch (message.headerType()) {
-      case MessageHeader.RecordBatch: return deserializeRecordBatch(in, message, alloc);
-      case MessageHeader.DictionaryBatch: return deserializeDictionaryBatch(in, message, alloc);
-      default: throw new IOException("Unexpected message header type " + message.headerType());
+      case MessageHeader.RecordBatch:
+        return deserializeRecordBatch(in, message, alloc);
+      case MessageHeader.DictionaryBatch:
+        return deserializeDictionaryBatch(in, message, alloc);
+      default:
+        throw new IOException("Unexpected message header type " + message.headerType());
     }
   }
 
   /**
    * Serializes a message header.
-   * @param builder to write the flatbuf to
-   * @param headerType headerType field
+   *
+   * @param builder      to write the flatbuf to
+   * @param headerType   headerType field
    * @param headerOffset header offset field
-   * @param bodyLength body length field
+   * @param bodyLength   body length field
    * @return the corresponding ByteBuffer
    */
   public static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte headerType,
-      int headerOffset, int bodyLength) {
+                                            int headerOffset, int bodyLength) {
     Message.startMessage(builder);
     Message.addHeaderType(builder, headerType);
     Message.addHeader(builder, headerOffset);
@@ -387,9 +400,13 @@ public static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte header
   private static Message deserializeMessage(ReadChannel in) throws IOException {
     // Read the message size. There is an i32 little endian prefix.
     ByteBuffer buffer = ByteBuffer.allocate(4);
-    if (in.readFully(buffer) != 4) return null;
+    if (in.readFully(buffer) != 4) {
+      return null;
+    }
     int messageLength = bytesToInt(buffer.array());
-    if (messageLength == 0) return null;
+    if (messageLength == 0) {
+      return null;
+    }
 
     buffer = ByteBuffer.allocate(messageLength);
     if (in.readFully(buffer) != messageLength) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/DateUnit.java b/java/vector/src/main/java/org/apache/arrow/vector/types/DateUnit.java
index e5beebffde9e4..003d3cdbceb3b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/DateUnit.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/DateUnit.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.types;
 
 public enum DateUnit {
@@ -22,6 +23,7 @@ public enum DateUnit {
   MILLISECOND(org.apache.arrow.flatbuf.DateUnit.MILLISECOND);
 
   private static final DateUnit[] valuesByFlatbufId = new DateUnit[DateUnit.values().length];
+
   static {
     for (DateUnit v : DateUnit.values()) {
       valuesByFlatbufId[v.flatbufID] = v;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java b/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java
index 3206969fb7ead..ec253287b261d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.types;
 
 import org.apache.arrow.flatbuf.Precision;
@@ -25,6 +26,7 @@ public enum FloatingPointPrecision {
   DOUBLE(Precision.DOUBLE);
 
   private static final FloatingPointPrecision[] valuesByFlatbufId = new FloatingPointPrecision[FloatingPointPrecision.values().length];
+
   static {
     for (FloatingPointPrecision v : FloatingPointPrecision.values()) {
       valuesByFlatbufId[v.flatbufID] = v;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java b/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java
index b3ddf1fe497de..a8157f19e1a69 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.types;
 
 public enum IntervalUnit {
@@ -22,6 +23,7 @@ public enum IntervalUnit {
   DAY_TIME(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME);
 
   private static final IntervalUnit[] valuesByFlatbufId = new IntervalUnit[IntervalUnit.values().length];
+
   static {
     for (IntervalUnit v : IntervalUnit.values()) {
       valuesByFlatbufId[v.flatbufID] = v;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/TimeUnit.java b/java/vector/src/main/java/org/apache/arrow/vector/types/TimeUnit.java
index cea9866965854..1da9321fcc4ee 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/TimeUnit.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/TimeUnit.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.types;
 
 public enum TimeUnit {
@@ -24,6 +25,7 @@ public enum TimeUnit {
   NANOSECOND(org.apache.arrow.flatbuf.TimeUnit.NANOSECOND);
 
   private static final TimeUnit[] valuesByFlatbufId = new TimeUnit[TimeUnit.values().length];
+
   static {
     for (TimeUnit v : TimeUnit.values()) {
       valuesByFlatbufId[v.flatbufID] = v;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 6591a4b16da16..c57dd6dafe9e6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.types;
 
 import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE;
@@ -531,73 +532,83 @@ public final ArrowType getType() {
 
   public static MinorType getMinorTypeForArrowType(ArrowType arrowType) {
     return arrowType.accept(new ArrowTypeVisitor<MinorType>() {
-      @Override public MinorType visit(Null type) {
+      @Override
+      public MinorType visit(Null type) {
         return MinorType.NULL;
       }
 
-      @Override public MinorType visit(Struct type) {
+      @Override
+      public MinorType visit(Struct type) {
         return MinorType.MAP;
       }
 
-      @Override public MinorType visit(List type) {
+      @Override
+      public MinorType visit(List type) {
         return MinorType.LIST;
       }
 
-      @Override public MinorType visit(FixedSizeList type) {
+      @Override
+      public MinorType visit(FixedSizeList type) {
         return MinorType.FIXED_SIZE_LIST;
       }
 
-      @Override public MinorType visit(Union type) {
+      @Override
+      public MinorType visit(Union type) {
         return MinorType.UNION;
       }
 
       @Override
       public MinorType visit(Int type) {
         switch (type.getBitWidth()) {
-        case 8:
-          return type.getIsSigned() ? MinorType.TINYINT : MinorType.UINT1;
-        case 16:
-          return type.getIsSigned() ? MinorType.SMALLINT : MinorType.UINT2;
-        case 32:
-          return type.getIsSigned() ? MinorType.INT : MinorType.UINT4;
-        case 64:
-          return type.getIsSigned() ? MinorType.BIGINT : MinorType.UINT8;
-        default:
-          throw new IllegalArgumentException("only 8, 16, 32, 64 supported: " + type);
+          case 8:
+            return type.getIsSigned() ? MinorType.TINYINT : MinorType.UINT1;
+          case 16:
+            return type.getIsSigned() ? MinorType.SMALLINT : MinorType.UINT2;
+          case 32:
+            return type.getIsSigned() ? MinorType.INT : MinorType.UINT4;
+          case 64:
+            return type.getIsSigned() ? MinorType.BIGINT : MinorType.UINT8;
+          default:
+            throw new IllegalArgumentException("only 8, 16, 32, 64 supported: " + type);
         }
       }
 
       @Override
       public MinorType visit(FloatingPoint type) {
         switch (type.getPrecision()) {
-        case HALF:
-          throw new UnsupportedOperationException("NYI: " + type);
-        case SINGLE:
-          return MinorType.FLOAT4;
-        case DOUBLE:
-          return MinorType.FLOAT8;
-        default:
-          throw new IllegalArgumentException("unknown precision: " + type);
+          case HALF:
+            throw new UnsupportedOperationException("NYI: " + type);
+          case SINGLE:
+            return MinorType.FLOAT4;
+          case DOUBLE:
+            return MinorType.FLOAT8;
+          default:
+            throw new IllegalArgumentException("unknown precision: " + type);
         }
       }
 
-      @Override public MinorType visit(Utf8 type) {
+      @Override
+      public MinorType visit(Utf8 type) {
         return MinorType.VARCHAR;
       }
 
-      @Override public MinorType visit(Binary type) {
+      @Override
+      public MinorType visit(Binary type) {
         return MinorType.VARBINARY;
       }
 
-      @Override public MinorType visit(Bool type) {
+      @Override
+      public MinorType visit(Bool type) {
         return MinorType.BIT;
       }
 
-      @Override public MinorType visit(Decimal type) {
+      @Override
+      public MinorType visit(Decimal type) {
         return MinorType.DECIMAL;
       }
 
-      @Override public MinorType visit(Date type) {
+      @Override
+      public MinorType visit(Date type) {
         switch (type.getUnit()) {
           case DAY:
             return MinorType.DATEDAY;
@@ -608,7 +619,8 @@ public MinorType visit(FloatingPoint type) {
         }
       }
 
-      @Override public MinorType visit(Time type) {
+      @Override
+      public MinorType visit(Time type) {
         switch (type.getUnit()) {
           case SECOND:
             return MinorType.TIMESEC;
@@ -623,7 +635,8 @@ public MinorType visit(FloatingPoint type) {
         }
       }
 
-      @Override public MinorType visit(Timestamp type) {
+      @Override
+      public MinorType visit(Timestamp type) {
         String tz = type.getTimezone();
         switch (type.getUnit()) {
           case SECOND:
@@ -642,12 +655,12 @@ public MinorType visit(FloatingPoint type) {
       @Override
       public MinorType visit(Interval type) {
         switch (type.getUnit()) {
-        case DAY_TIME:
-          return MinorType.INTERVALDAY;
-        case YEAR_MONTH:
-          return MinorType.INTERVALYEAR;
-        default:
-          throw new IllegalArgumentException("unknown unit: " + type);
+          case DAY_TIME:
+            return MinorType.INTERVALDAY;
+          case YEAR_MONTH:
+            return MinorType.INTERVALYEAR;
+          default:
+            throw new IllegalArgumentException("unknown unit: " + type);
         }
       }
     });
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/UnionMode.java b/java/vector/src/main/java/org/apache/arrow/vector/types/UnionMode.java
index 8e957bc0b6e34..231e85fb2c726 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/UnionMode.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/UnionMode.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.types;
 
 public enum UnionMode {
@@ -22,6 +23,7 @@ public enum UnionMode {
   Dense(org.apache.arrow.flatbuf.UnionMode.Dense);
 
   private static final UnionMode[] valuesByFlatbufId = new UnionMode[UnionMode.values().length];
+
   static {
     for (UnionMode v : UnionMode.values()) {
       valuesByFlatbufId[v.flatbufID] = v;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
index 8a0224d5564b0..2657532fa9be6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.types.pojo;
 
 import java.util.Objects;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index a8f2ae5fbab8c..48e71a976c0e8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.types.pojo;
 
 
@@ -166,7 +167,7 @@ public int getField(FlatBufferBuilder builder) {
     int layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData);
     int[] metadataOffsets = new int[getMetadata().size()];
     Iterator<Entry<String, String>> metadataIterator = getMetadata().entrySet().iterator();
-    for (int i = 0; i < metadataOffsets.length; i ++) {
+    for (int i = 0; i < metadataOffsets.length; i++) {
       Entry<String, String> kv = metadataIterator.next();
       int keyOffset = builder.createString(kv.getKey());
       int valueOffset = builder.createString(kv.getValue());
@@ -239,11 +240,11 @@ public boolean equals(Object obj) {
     }
     Field that = (Field) obj;
     return Objects.equals(this.name, that.name) &&
-             Objects.equals(this.isNullable(), that.isNullable()) &&
-             Objects.equals(this.getType(), that.getType()) &&
-             Objects.equals(this.getDictionary(), that.getDictionary()) &&
-             Objects.equals(this.getMetadata(), that.getMetadata()) &&
-             Objects.equals(this.children, that.children);
+        Objects.equals(this.isNullable(), that.isNullable()) &&
+        Objects.equals(this.getType(), that.getType()) &&
+        Objects.equals(this.getDictionary(), that.getDictionary()) &&
+        Objects.equals(this.getMetadata(), that.getMetadata()) &&
+        Objects.equals(this.children, that.children);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
index c8fc689cd2c9c..f0856198a4c79 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.types.pojo;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -55,12 +56,15 @@ public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary
   public boolean isNullable() {
     return nullable;
   }
+
   public ArrowType getType() {
     return type;
   }
+
   public DictionaryEncoding getDictionary() {
     return dictionary;
   }
+
   public Map<String, String> getMetadata() {
     return metadata;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
index 82e2ef55c20c6..a87d4490060df 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.types.pojo;
 
 
@@ -53,7 +54,7 @@ public class Schema {
 
   /**
    * @param fields the list of the fields
-   * @param name the name of the field to return
+   * @param name   the name of the field to return
    * @return the corresponding field
    * @throws IllegalArgumentException if the field was not found
    */
@@ -146,7 +147,7 @@ public int getSchema(FlatBufferBuilder builder) {
     int fieldsOffset = org.apache.arrow.flatbuf.Schema.createFieldsVector(builder, fieldOffsets);
     int[] metadataOffsets = new int[metadata.size()];
     Iterator<Entry<String, String>> metadataIterator = metadata.entrySet().iterator();
-    for (int i = 0; i < metadataOffsets.length; i ++) {
+    for (int i = 0; i < metadataOffsets.length; i++) {
       Entry<String, String> kv = metadataIterator.next();
       int keyOffset = builder.createString(kv.getKey());
       int valueOffset = builder.createString(kv.getValue());
@@ -174,7 +175,7 @@ public boolean equals(Object obj) {
       return false;
     }
     return Objects.equals(this.fields, ((Schema) obj).fields) &&
-           Objects.equals(this.metadata, ((Schema) obj).metadata);
+        Objects.equals(this.metadata, ((Schema) obj).metadata);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteArrayReadableSeekableByteChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteArrayReadableSeekableByteChannel.java
index 69840fefa968b..80d4a4684c512 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteArrayReadableSeekableByteChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteArrayReadableSeekableByteChannel.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import java.io.IOException;
@@ -58,7 +59,7 @@ public long position() throws IOException {
 
   @Override
   public SeekableByteChannel position(final long newPosition) throws IOException {
-    this.position = (int)newPosition;
+    this.position = (int) newPosition;
     return this;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
index 68b9fb25f2112..5fe556a552714 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  ******************************************************************************/
+
 package org.apache.arrow.vector.util;
 
 import io.netty.buffer.ArrowBuf;
@@ -31,15 +32,15 @@ public class ByteFunctionHelpers {
   /**
    * Helper function to check for equality of bytes in two ArrowBufs
    *
-   * @param left Left ArrowBuf for comparison
+   * @param left   Left ArrowBuf for comparison
    * @param lStart start offset in the buffer
-   * @param lEnd end offset in the buffer
-   * @param right Right ArrowBuf for comparison
+   * @param lEnd   end offset in the buffer
+   * @param right  Right ArrowBuf for comparison
    * @param rStart start offset in the buffer
-   * @param rEnd end offset in the buffer
+   * @param rEnd   end offset in the buffer
    * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
    */
-  public static final int equal(final ArrowBuf left, int lStart, int lEnd, final ArrowBuf right, int rStart, int rEnd){
+  public static final int equal(final ArrowBuf left, int lStart, int lEnd, final ArrowBuf right, int rStart, int rEnd) {
     if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
       left.checkBytes(lStart, lEnd);
       right.checkBytes(rStart, rEnd);
@@ -48,7 +49,7 @@ public static final int equal(final ArrowBuf left, int lStart, int lEnd, final A
   }
 
   private static final int memEqual(final long laddr, int lStart, int lEnd, final long raddr, int rStart,
-      final int rEnd) {
+                                    final int rEnd) {
 
     int n = lEnd - lStart;
     if (n == rEnd - rStart) {
@@ -85,15 +86,15 @@ private static final int memEqual(final long laddr, int lStart, int lEnd, final
    *
    * Function will check data before completing in the case that
    *
-   * @param left Left ArrowBuf to compare
+   * @param left   Left ArrowBuf to compare
    * @param lStart start offset in the buffer
-   * @param lEnd end offset in the buffer
-   * @param right Right ArrowBuf to compare
+   * @param lEnd   end offset in the buffer
+   * @param right  Right ArrowBuf to compare
    * @param rStart start offset in the buffer
-   * @param rEnd end offset in the buffer
+   * @param rEnd   end offset in the buffer
    * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
    */
-  public static final int compare(final ArrowBuf left, int lStart, int lEnd, final ArrowBuf right, int rStart, int rEnd){
+  public static final int compare(final ArrowBuf left, int lStart, int lEnd, final ArrowBuf right, int rStart, int rEnd) {
     if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
       left.checkBytes(lStart, lEnd);
       right.checkBytes(rStart, rEnd);
@@ -140,12 +141,12 @@ private static final int memcmp(final long laddr, int lStart, int lEnd, final lo
   /**
    * Helper function to compare a set of bytes in ArrowBuf to a ByteArray.
    *
-   * @param left Left ArrowBuf for comparison purposes
+   * @param left   Left ArrowBuf for comparison purposes
    * @param lStart start offset in the buffer
-   * @param lEnd end offset in the buffer
-   * @param right second input to be compared
+   * @param lEnd   end offset in the buffer
+   * @param right  second input to be compared
    * @param rStart start offset in the byte array
-   * @param rEnd end offset in the byte array
+   * @param rEnd   end offset in the byte array
    * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
    */
   public static final int compare(final ArrowBuf left, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/CallBack.java b/java/vector/src/main/java/org/apache/arrow/vector/util/CallBack.java
index 249834270b3fe..38e3b78c778ea 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/CallBack.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/CallBack.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
index 8aad41744f673..3dd169b82357b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
@@ -33,660 +33,660 @@
 public class DateUtility {
 
 
-    /* We have a hashmap that stores the timezone as the key and an index as the value
-     * While storing the timezone in value vectors, holders we only use this index. As we
-     * reconstruct the timestamp, we use this index to index through the array timezoneList
-     * and get the corresponding timezone and pass it to joda-time
-     */
+  /* We have a hashmap that stores the timezone as the key and an index as the value
+   * While storing the timezone in value vectors, holders we only use this index. As we
+   * reconstruct the timestamp, we use this index to index through the array timezoneList
+   * and get the corresponding timezone and pass it to joda-time
+   */
   public static ObjectIntHashMap<String> timezoneMap = new ObjectIntHashMap<String>();
 
-    public static String[] timezoneList =  {"Africa/Abidjan",
-                                            "Africa/Accra",
-                                            "Africa/Addis_Ababa",
-                                            "Africa/Algiers",
-                                            "Africa/Asmara",
-                                            "Africa/Asmera",
-                                            "Africa/Bamako",
-                                            "Africa/Bangui",
-                                            "Africa/Banjul",
-                                            "Africa/Bissau",
-                                            "Africa/Blantyre",
-                                            "Africa/Brazzaville",
-                                            "Africa/Bujumbura",
-                                            "Africa/Cairo",
-                                            "Africa/Casablanca",
-                                            "Africa/Ceuta",
-                                            "Africa/Conakry",
-                                            "Africa/Dakar",
-                                            "Africa/Dar_es_Salaam",
-                                            "Africa/Djibouti",
-                                            "Africa/Douala",
-                                            "Africa/El_Aaiun",
-                                            "Africa/Freetown",
-                                            "Africa/Gaborone",
-                                            "Africa/Harare",
-                                            "Africa/Johannesburg",
-                                            "Africa/Juba",
-                                            "Africa/Kampala",
-                                            "Africa/Khartoum",
-                                            "Africa/Kigali",
-                                            "Africa/Kinshasa",
-                                            "Africa/Lagos",
-                                            "Africa/Libreville",
-                                            "Africa/Lome",
-                                            "Africa/Luanda",
-                                            "Africa/Lubumbashi",
-                                            "Africa/Lusaka",
-                                            "Africa/Malabo",
-                                            "Africa/Maputo",
-                                            "Africa/Maseru",
-                                            "Africa/Mbabane",
-                                            "Africa/Mogadishu",
-                                            "Africa/Monrovia",
-                                            "Africa/Nairobi",
-                                            "Africa/Ndjamena",
-                                            "Africa/Niamey",
-                                            "Africa/Nouakchott",
-                                            "Africa/Ouagadougou",
-                                            "Africa/Porto-Novo",
-                                            "Africa/Sao_Tome",
-                                            "Africa/Timbuktu",
-                                            "Africa/Tripoli",
-                                            "Africa/Tunis",
-                                            "Africa/Windhoek",
-                                            "America/Adak",
-                                            "America/Anchorage",
-                                            "America/Anguilla",
-                                            "America/Antigua",
-                                            "America/Araguaina",
-                                            "America/Argentina/Buenos_Aires",
-                                            "America/Argentina/Catamarca",
-                                            "America/Argentina/ComodRivadavia",
-                                            "America/Argentina/Cordoba",
-                                            "America/Argentina/Jujuy",
-                                            "America/Argentina/La_Rioja",
-                                            "America/Argentina/Mendoza",
-                                            "America/Argentina/Rio_Gallegos",
-                                            "America/Argentina/Salta",
-                                            "America/Argentina/San_Juan",
-                                            "America/Argentina/San_Luis",
-                                            "America/Argentina/Tucuman",
-                                            "America/Argentina/Ushuaia",
-                                            "America/Aruba",
-                                            "America/Asuncion",
-                                            "America/Atikokan",
-                                            "America/Atka",
-                                            "America/Bahia",
-                                            "America/Bahia_Banderas",
-                                            "America/Barbados",
-                                            "America/Belem",
-                                            "America/Belize",
-                                            "America/Blanc-Sablon",
-                                            "America/Boa_Vista",
-                                            "America/Bogota",
-                                            "America/Boise",
-                                            "America/Buenos_Aires",
-                                            "America/Cambridge_Bay",
-                                            "America/Campo_Grande",
-                                            "America/Cancun",
-                                            "America/Caracas",
-                                            "America/Catamarca",
-                                            "America/Cayenne",
-                                            "America/Cayman",
-                                            "America/Chicago",
-                                            "America/Chihuahua",
-                                            "America/Coral_Harbour",
-                                            "America/Cordoba",
-                                            "America/Costa_Rica",
-                                            "America/Cuiaba",
-                                            "America/Curacao",
-                                            "America/Danmarkshavn",
-                                            "America/Dawson",
-                                            "America/Dawson_Creek",
-                                            "America/Denver",
-                                            "America/Detroit",
-                                            "America/Dominica",
-                                            "America/Edmonton",
-                                            "America/Eirunepe",
-                                            "America/El_Salvador",
-                                            "America/Ensenada",
-                                            "America/Fort_Wayne",
-                                            "America/Fortaleza",
-                                            "America/Glace_Bay",
-                                            "America/Godthab",
-                                            "America/Goose_Bay",
-                                            "America/Grand_Turk",
-                                            "America/Grenada",
-                                            "America/Guadeloupe",
-                                            "America/Guatemala",
-                                            "America/Guayaquil",
-                                            "America/Guyana",
-                                            "America/Halifax",
-                                            "America/Havana",
-                                            "America/Hermosillo",
-                                            "America/Indiana/Indianapolis",
-                                            "America/Indiana/Knox",
-                                            "America/Indiana/Marengo",
-                                            "America/Indiana/Petersburg",
-                                            "America/Indiana/Tell_City",
-                                            "America/Indiana/Vevay",
-                                            "America/Indiana/Vincennes",
-                                            "America/Indiana/Winamac",
-                                            "America/Indianapolis",
-                                            "America/Inuvik",
-                                            "America/Iqaluit",
-                                            "America/Jamaica",
-                                            "America/Jujuy",
-                                            "America/Juneau",
-                                            "America/Kentucky/Louisville",
-                                            "America/Kentucky/Monticello",
-                                            "America/Knox_IN",
-                                            "America/Kralendijk",
-                                            "America/La_Paz",
-                                            "America/Lima",
-                                            "America/Los_Angeles",
-                                            "America/Louisville",
-                                            "America/Lower_Princes",
-                                            "America/Maceio",
-                                            "America/Managua",
-                                            "America/Manaus",
-                                            "America/Marigot",
-                                            "America/Martinique",
-                                            "America/Matamoros",
-                                            "America/Mazatlan",
-                                            "America/Mendoza",
-                                            "America/Menominee",
-                                            "America/Merida",
-                                            "America/Metlakatla",
-                                            "America/Mexico_City",
-                                            "America/Miquelon",
-                                            "America/Moncton",
-                                            "America/Monterrey",
-                                            "America/Montevideo",
-                                            "America/Montreal",
-                                            "America/Montserrat",
-                                            "America/Nassau",
-                                            "America/New_York",
-                                            "America/Nipigon",
-                                            "America/Nome",
-                                            "America/Noronha",
-                                            "America/North_Dakota/Beulah",
-                                            "America/North_Dakota/Center",
-                                            "America/North_Dakota/New_Salem",
-                                            "America/Ojinaga",
-                                            "America/Panama",
-                                            "America/Pangnirtung",
-                                            "America/Paramaribo",
-                                            "America/Phoenix",
-                                            "America/Port-au-Prince",
-                                            "America/Port_of_Spain",
-                                            "America/Porto_Acre",
-                                            "America/Porto_Velho",
-                                            "America/Puerto_Rico",
-                                            "America/Rainy_River",
-                                            "America/Rankin_Inlet",
-                                            "America/Recife",
-                                            "America/Regina",
-                                            "America/Resolute",
-                                            "America/Rio_Branco",
-                                            "America/Rosario",
-                                            "America/Santa_Isabel",
-                                            "America/Santarem",
-                                            "America/Santiago",
-                                            "America/Santo_Domingo",
-                                            "America/Sao_Paulo",
-                                            "America/Scoresbysund",
-                                            "America/Shiprock",
-                                            "America/Sitka",
-                                            "America/St_Barthelemy",
-                                            "America/St_Johns",
-                                            "America/St_Kitts",
-                                            "America/St_Lucia",
-                                            "America/St_Thomas",
-                                            "America/St_Vincent",
-                                            "America/Swift_Current",
-                                            "America/Tegucigalpa",
-                                            "America/Thule",
-                                            "America/Thunder_Bay",
-                                            "America/Tijuana",
-                                            "America/Toronto",
-                                            "America/Tortola",
-                                            "America/Vancouver",
-                                            "America/Virgin",
-                                            "America/Whitehorse",
-                                            "America/Winnipeg",
-                                            "America/Yakutat",
-                                            "America/Yellowknife",
-                                            "Antarctica/Casey",
-                                            "Antarctica/Davis",
-                                            "Antarctica/DumontDUrville",
-                                            "Antarctica/Macquarie",
-                                            "Antarctica/Mawson",
-                                            "Antarctica/McMurdo",
-                                            "Antarctica/Palmer",
-                                            "Antarctica/Rothera",
-                                            "Antarctica/South_Pole",
-                                            "Antarctica/Syowa",
-                                            "Antarctica/Vostok",
-                                            "Arctic/Longyearbyen",
-                                            "Asia/Aden",
-                                            "Asia/Almaty",
-                                            "Asia/Amman",
-                                            "Asia/Anadyr",
-                                            "Asia/Aqtau",
-                                            "Asia/Aqtobe",
-                                            "Asia/Ashgabat",
-                                            "Asia/Ashkhabad",
-                                            "Asia/Baghdad",
-                                            "Asia/Bahrain",
-                                            "Asia/Baku",
-                                            "Asia/Bangkok",
-                                            "Asia/Beirut",
-                                            "Asia/Bishkek",
-                                            "Asia/Brunei",
-                                            "Asia/Calcutta",
-                                            "Asia/Choibalsan",
-                                            "Asia/Chongqing",
-                                            "Asia/Chungking",
-                                            "Asia/Colombo",
-                                            "Asia/Dacca",
-                                            "Asia/Damascus",
-                                            "Asia/Dhaka",
-                                            "Asia/Dili",
-                                            "Asia/Dubai",
-                                            "Asia/Dushanbe",
-                                            "Asia/Gaza",
-                                            "Asia/Harbin",
-                                            "Asia/Hebron",
-                                            "Asia/Ho_Chi_Minh",
-                                            "Asia/Hong_Kong",
-                                            "Asia/Hovd",
-                                            "Asia/Irkutsk",
-                                            "Asia/Istanbul",
-                                            "Asia/Jakarta",
-                                            "Asia/Jayapura",
-                                            "Asia/Jerusalem",
-                                            "Asia/Kabul",
-                                            "Asia/Kamchatka",
-                                            "Asia/Karachi",
-                                            "Asia/Kashgar",
-                                            "Asia/Kathmandu",
-                                            "Asia/Katmandu",
-                                            "Asia/Kolkata",
-                                            "Asia/Krasnoyarsk",
-                                            "Asia/Kuala_Lumpur",
-                                            "Asia/Kuching",
-                                            "Asia/Kuwait",
-                                            "Asia/Macao",
-                                            "Asia/Macau",
-                                            "Asia/Magadan",
-                                            "Asia/Makassar",
-                                            "Asia/Manila",
-                                            "Asia/Muscat",
-                                            "Asia/Nicosia",
-                                            "Asia/Novokuznetsk",
-                                            "Asia/Novosibirsk",
-                                            "Asia/Omsk",
-                                            "Asia/Oral",
-                                            "Asia/Phnom_Penh",
-                                            "Asia/Pontianak",
-                                            "Asia/Pyongyang",
-                                            "Asia/Qatar",
-                                            "Asia/Qyzylorda",
-                                            "Asia/Rangoon",
-                                            "Asia/Riyadh",
-                                            "Asia/Saigon",
-                                            "Asia/Sakhalin",
-                                            "Asia/Samarkand",
-                                            "Asia/Seoul",
-                                            "Asia/Shanghai",
-                                            "Asia/Singapore",
-                                            "Asia/Taipei",
-                                            "Asia/Tashkent",
-                                            "Asia/Tbilisi",
-                                            "Asia/Tehran",
-                                            "Asia/Tel_Aviv",
-                                            "Asia/Thimbu",
-                                            "Asia/Thimphu",
-                                            "Asia/Tokyo",
-                                            "Asia/Ujung_Pandang",
-                                            "Asia/Ulaanbaatar",
-                                            "Asia/Ulan_Bator",
-                                            "Asia/Urumqi",
-                                            "Asia/Vientiane",
-                                            "Asia/Vladivostok",
-                                            "Asia/Yakutsk",
-                                            "Asia/Yekaterinburg",
-                                            "Asia/Yerevan",
-                                            "Atlantic/Azores",
-                                            "Atlantic/Bermuda",
-                                            "Atlantic/Canary",
-                                            "Atlantic/Cape_Verde",
-                                            "Atlantic/Faeroe",
-                                            "Atlantic/Faroe",
-                                            "Atlantic/Jan_Mayen",
-                                            "Atlantic/Madeira",
-                                            "Atlantic/Reykjavik",
-                                            "Atlantic/South_Georgia",
-                                            "Atlantic/St_Helena",
-                                            "Atlantic/Stanley",
-                                            "Australia/ACT",
-                                            "Australia/Adelaide",
-                                            "Australia/Brisbane",
-                                            "Australia/Broken_Hill",
-                                            "Australia/Canberra",
-                                            "Australia/Currie",
-                                            "Australia/Darwin",
-                                            "Australia/Eucla",
-                                            "Australia/Hobart",
-                                            "Australia/LHI",
-                                            "Australia/Lindeman",
-                                            "Australia/Lord_Howe",
-                                            "Australia/Melbourne",
-                                            "Australia/NSW",
-                                            "Australia/North",
-                                            "Australia/Perth",
-                                            "Australia/Queensland",
-                                            "Australia/South",
-                                            "Australia/Sydney",
-                                            "Australia/Tasmania",
-                                            "Australia/Victoria",
-                                            "Australia/West",
-                                            "Australia/Yancowinna",
-                                            "Brazil/Acre",
-                                            "Brazil/DeNoronha",
-                                            "Brazil/East",
-                                            "Brazil/West",
-                                            "CET",
-                                            "CST6CDT",
-                                            "Canada/Atlantic",
-                                            "Canada/Central",
-                                            "Canada/East-Saskatchewan",
-                                            "Canada/Eastern",
-                                            "Canada/Mountain",
-                                            "Canada/Newfoundland",
-                                            "Canada/Pacific",
-                                            "Canada/Saskatchewan",
-                                            "Canada/Yukon",
-                                            "Chile/Continental",
-                                            "Chile/EasterIsland",
-                                            "Cuba",
-                                            "EET",
-                                            "EST",
-                                            "EST5EDT",
-                                            "Egypt",
-                                            "Eire",
-                                            "Etc/GMT",
-                                            "Etc/GMT+0",
-                                            "Etc/GMT+1",
-                                            "Etc/GMT+10",
-                                            "Etc/GMT+11",
-                                            "Etc/GMT+12",
-                                            "Etc/GMT+2",
-                                            "Etc/GMT+3",
-                                            "Etc/GMT+4",
-                                            "Etc/GMT+5",
-                                            "Etc/GMT+6",
-                                            "Etc/GMT+7",
-                                            "Etc/GMT+8",
-                                            "Etc/GMT+9",
-                                            "Etc/GMT-0",
-                                            "Etc/GMT-1",
-                                            "Etc/GMT-10",
-                                            "Etc/GMT-11",
-                                            "Etc/GMT-12",
-                                            "Etc/GMT-13",
-                                            "Etc/GMT-14",
-                                            "Etc/GMT-2",
-                                            "Etc/GMT-3",
-                                            "Etc/GMT-4",
-                                            "Etc/GMT-5",
-                                            "Etc/GMT-6",
-                                            "Etc/GMT-7",
-                                            "Etc/GMT-8",
-                                            "Etc/GMT-9",
-                                            "Etc/GMT0",
-                                            "Etc/Greenwich",
-                                            "Etc/UCT",
-                                            "Etc/UTC",
-                                            "Etc/Universal",
-                                            "Etc/Zulu",
-                                            "Europe/Amsterdam",
-                                            "Europe/Andorra",
-                                            "Europe/Athens",
-                                            "Europe/Belfast",
-                                            "Europe/Belgrade",
-                                            "Europe/Berlin",
-                                            "Europe/Bratislava",
-                                            "Europe/Brussels",
-                                            "Europe/Bucharest",
-                                            "Europe/Budapest",
-                                            "Europe/Chisinau",
-                                            "Europe/Copenhagen",
-                                            "Europe/Dublin",
-                                            "Europe/Gibraltar",
-                                            "Europe/Guernsey",
-                                            "Europe/Helsinki",
-                                            "Europe/Isle_of_Man",
-                                            "Europe/Istanbul",
-                                            "Europe/Jersey",
-                                            "Europe/Kaliningrad",
-                                            "Europe/Kiev",
-                                            "Europe/Lisbon",
-                                            "Europe/Ljubljana",
-                                            "Europe/London",
-                                            "Europe/Luxembourg",
-                                            "Europe/Madrid",
-                                            "Europe/Malta",
-                                            "Europe/Mariehamn",
-                                            "Europe/Minsk",
-                                            "Europe/Monaco",
-                                            "Europe/Moscow",
-                                            "Europe/Nicosia",
-                                            "Europe/Oslo",
-                                            "Europe/Paris",
-                                            "Europe/Podgorica",
-                                            "Europe/Prague",
-                                            "Europe/Riga",
-                                            "Europe/Rome",
-                                            "Europe/Samara",
-                                            "Europe/San_Marino",
-                                            "Europe/Sarajevo",
-                                            "Europe/Simferopol",
-                                            "Europe/Skopje",
-                                            "Europe/Sofia",
-                                            "Europe/Stockholm",
-                                            "Europe/Tallinn",
-                                            "Europe/Tirane",
-                                            "Europe/Tiraspol",
-                                            "Europe/Uzhgorod",
-                                            "Europe/Vaduz",
-                                            "Europe/Vatican",
-                                            "Europe/Vienna",
-                                            "Europe/Vilnius",
-                                            "Europe/Volgograd",
-                                            "Europe/Warsaw",
-                                            "Europe/Zagreb",
-                                            "Europe/Zaporozhye",
-                                            "Europe/Zurich",
-                                            "GB",
-                                            "GB-Eire",
-                                            "GMT",
-                                            "GMT+0",
-                                            "GMT-0",
-                                            "GMT0",
-                                            "Greenwich",
-                                            "HST",
-                                            "Hongkong",
-                                            "Iceland",
-                                            "Indian/Antananarivo",
-                                            "Indian/Chagos",
-                                            "Indian/Christmas",
-                                            "Indian/Cocos",
-                                            "Indian/Comoro",
-                                            "Indian/Kerguelen",
-                                            "Indian/Mahe",
-                                            "Indian/Maldives",
-                                            "Indian/Mauritius",
-                                            "Indian/Mayotte",
-                                            "Indian/Reunion",
-                                            "Iran",
-                                            "Israel",
-                                            "Jamaica",
-                                            "Japan",
-                                            "Kwajalein",
-                                            "Libya",
-                                            "MET",
-                                            "MST",
-                                            "MST7MDT",
-                                            "Mexico/BajaNorte",
-                                            "Mexico/BajaSur",
-                                            "Mexico/General",
-                                            "NZ",
-                                            "NZ-CHAT",
-                                            "Navajo",
-                                            "PRC",
-                                            "PST8PDT",
-                                            "Pacific/Apia",
-                                            "Pacific/Auckland",
-                                            "Pacific/Chatham",
-                                            "Pacific/Chuuk",
-                                            "Pacific/Easter",
-                                            "Pacific/Efate",
-                                            "Pacific/Enderbury",
-                                            "Pacific/Fakaofo",
-                                            "Pacific/Fiji",
-                                            "Pacific/Funafuti",
-                                            "Pacific/Galapagos",
-                                            "Pacific/Gambier",
-                                            "Pacific/Guadalcanal",
-                                            "Pacific/Guam",
-                                            "Pacific/Honolulu",
-                                            "Pacific/Johnston",
-                                            "Pacific/Kiritimati",
-                                            "Pacific/Kosrae",
-                                            "Pacific/Kwajalein",
-                                            "Pacific/Majuro",
-                                            "Pacific/Marquesas",
-                                            "Pacific/Midway",
-                                            "Pacific/Nauru",
-                                            "Pacific/Niue",
-                                            "Pacific/Norfolk",
-                                            "Pacific/Noumea",
-                                            "Pacific/Pago_Pago",
-                                            "Pacific/Palau",
-                                            "Pacific/Pitcairn",
-                                            "Pacific/Pohnpei",
-                                            "Pacific/Ponape",
-                                            "Pacific/Port_Moresby",
-                                            "Pacific/Rarotonga",
-                                            "Pacific/Saipan",
-                                            "Pacific/Samoa",
-                                            "Pacific/Tahiti",
-                                            "Pacific/Tarawa",
-                                            "Pacific/Tongatapu",
-                                            "Pacific/Truk",
-                                            "Pacific/Wake",
-                                            "Pacific/Wallis",
-                                            "Pacific/Yap",
-                                            "Poland",
-                                            "Portugal",
-                                            "ROC",
-                                            "ROK",
-                                            "Singapore",
-                                            "Turkey",
-                                            "UCT",
-                                            "US/Alaska",
-                                            "US/Aleutian",
-                                            "US/Arizona",
-                                            "US/Central",
-                                            "US/East-Indiana",
-                                            "US/Eastern",
-                                            "US/Hawaii",
-                                            "US/Indiana-Starke",
-                                            "US/Michigan",
-                                            "US/Mountain",
-                                            "US/Pacific",
-                                            "US/Pacific-New",
-                                            "US/Samoa",
-                                            "UTC",
-                                            "Universal",
-                                            "W-SU",
-                                            "WET",
-                                            "Zulu"};
+  public static String[] timezoneList = {"Africa/Abidjan",
+      "Africa/Accra",
+      "Africa/Addis_Ababa",
+      "Africa/Algiers",
+      "Africa/Asmara",
+      "Africa/Asmera",
+      "Africa/Bamako",
+      "Africa/Bangui",
+      "Africa/Banjul",
+      "Africa/Bissau",
+      "Africa/Blantyre",
+      "Africa/Brazzaville",
+      "Africa/Bujumbura",
+      "Africa/Cairo",
+      "Africa/Casablanca",
+      "Africa/Ceuta",
+      "Africa/Conakry",
+      "Africa/Dakar",
+      "Africa/Dar_es_Salaam",
+      "Africa/Djibouti",
+      "Africa/Douala",
+      "Africa/El_Aaiun",
+      "Africa/Freetown",
+      "Africa/Gaborone",
+      "Africa/Harare",
+      "Africa/Johannesburg",
+      "Africa/Juba",
+      "Africa/Kampala",
+      "Africa/Khartoum",
+      "Africa/Kigali",
+      "Africa/Kinshasa",
+      "Africa/Lagos",
+      "Africa/Libreville",
+      "Africa/Lome",
+      "Africa/Luanda",
+      "Africa/Lubumbashi",
+      "Africa/Lusaka",
+      "Africa/Malabo",
+      "Africa/Maputo",
+      "Africa/Maseru",
+      "Africa/Mbabane",
+      "Africa/Mogadishu",
+      "Africa/Monrovia",
+      "Africa/Nairobi",
+      "Africa/Ndjamena",
+      "Africa/Niamey",
+      "Africa/Nouakchott",
+      "Africa/Ouagadougou",
+      "Africa/Porto-Novo",
+      "Africa/Sao_Tome",
+      "Africa/Timbuktu",
+      "Africa/Tripoli",
+      "Africa/Tunis",
+      "Africa/Windhoek",
+      "America/Adak",
+      "America/Anchorage",
+      "America/Anguilla",
+      "America/Antigua",
+      "America/Araguaina",
+      "America/Argentina/Buenos_Aires",
+      "America/Argentina/Catamarca",
+      "America/Argentina/ComodRivadavia",
+      "America/Argentina/Cordoba",
+      "America/Argentina/Jujuy",
+      "America/Argentina/La_Rioja",
+      "America/Argentina/Mendoza",
+      "America/Argentina/Rio_Gallegos",
+      "America/Argentina/Salta",
+      "America/Argentina/San_Juan",
+      "America/Argentina/San_Luis",
+      "America/Argentina/Tucuman",
+      "America/Argentina/Ushuaia",
+      "America/Aruba",
+      "America/Asuncion",
+      "America/Atikokan",
+      "America/Atka",
+      "America/Bahia",
+      "America/Bahia_Banderas",
+      "America/Barbados",
+      "America/Belem",
+      "America/Belize",
+      "America/Blanc-Sablon",
+      "America/Boa_Vista",
+      "America/Bogota",
+      "America/Boise",
+      "America/Buenos_Aires",
+      "America/Cambridge_Bay",
+      "America/Campo_Grande",
+      "America/Cancun",
+      "America/Caracas",
+      "America/Catamarca",
+      "America/Cayenne",
+      "America/Cayman",
+      "America/Chicago",
+      "America/Chihuahua",
+      "America/Coral_Harbour",
+      "America/Cordoba",
+      "America/Costa_Rica",
+      "America/Cuiaba",
+      "America/Curacao",
+      "America/Danmarkshavn",
+      "America/Dawson",
+      "America/Dawson_Creek",
+      "America/Denver",
+      "America/Detroit",
+      "America/Dominica",
+      "America/Edmonton",
+      "America/Eirunepe",
+      "America/El_Salvador",
+      "America/Ensenada",
+      "America/Fort_Wayne",
+      "America/Fortaleza",
+      "America/Glace_Bay",
+      "America/Godthab",
+      "America/Goose_Bay",
+      "America/Grand_Turk",
+      "America/Grenada",
+      "America/Guadeloupe",
+      "America/Guatemala",
+      "America/Guayaquil",
+      "America/Guyana",
+      "America/Halifax",
+      "America/Havana",
+      "America/Hermosillo",
+      "America/Indiana/Indianapolis",
+      "America/Indiana/Knox",
+      "America/Indiana/Marengo",
+      "America/Indiana/Petersburg",
+      "America/Indiana/Tell_City",
+      "America/Indiana/Vevay",
+      "America/Indiana/Vincennes",
+      "America/Indiana/Winamac",
+      "America/Indianapolis",
+      "America/Inuvik",
+      "America/Iqaluit",
+      "America/Jamaica",
+      "America/Jujuy",
+      "America/Juneau",
+      "America/Kentucky/Louisville",
+      "America/Kentucky/Monticello",
+      "America/Knox_IN",
+      "America/Kralendijk",
+      "America/La_Paz",
+      "America/Lima",
+      "America/Los_Angeles",
+      "America/Louisville",
+      "America/Lower_Princes",
+      "America/Maceio",
+      "America/Managua",
+      "America/Manaus",
+      "America/Marigot",
+      "America/Martinique",
+      "America/Matamoros",
+      "America/Mazatlan",
+      "America/Mendoza",
+      "America/Menominee",
+      "America/Merida",
+      "America/Metlakatla",
+      "America/Mexico_City",
+      "America/Miquelon",
+      "America/Moncton",
+      "America/Monterrey",
+      "America/Montevideo",
+      "America/Montreal",
+      "America/Montserrat",
+      "America/Nassau",
+      "America/New_York",
+      "America/Nipigon",
+      "America/Nome",
+      "America/Noronha",
+      "America/North_Dakota/Beulah",
+      "America/North_Dakota/Center",
+      "America/North_Dakota/New_Salem",
+      "America/Ojinaga",
+      "America/Panama",
+      "America/Pangnirtung",
+      "America/Paramaribo",
+      "America/Phoenix",
+      "America/Port-au-Prince",
+      "America/Port_of_Spain",
+      "America/Porto_Acre",
+      "America/Porto_Velho",
+      "America/Puerto_Rico",
+      "America/Rainy_River",
+      "America/Rankin_Inlet",
+      "America/Recife",
+      "America/Regina",
+      "America/Resolute",
+      "America/Rio_Branco",
+      "America/Rosario",
+      "America/Santa_Isabel",
+      "America/Santarem",
+      "America/Santiago",
+      "America/Santo_Domingo",
+      "America/Sao_Paulo",
+      "America/Scoresbysund",
+      "America/Shiprock",
+      "America/Sitka",
+      "America/St_Barthelemy",
+      "America/St_Johns",
+      "America/St_Kitts",
+      "America/St_Lucia",
+      "America/St_Thomas",
+      "America/St_Vincent",
+      "America/Swift_Current",
+      "America/Tegucigalpa",
+      "America/Thule",
+      "America/Thunder_Bay",
+      "America/Tijuana",
+      "America/Toronto",
+      "America/Tortola",
+      "America/Vancouver",
+      "America/Virgin",
+      "America/Whitehorse",
+      "America/Winnipeg",
+      "America/Yakutat",
+      "America/Yellowknife",
+      "Antarctica/Casey",
+      "Antarctica/Davis",
+      "Antarctica/DumontDUrville",
+      "Antarctica/Macquarie",
+      "Antarctica/Mawson",
+      "Antarctica/McMurdo",
+      "Antarctica/Palmer",
+      "Antarctica/Rothera",
+      "Antarctica/South_Pole",
+      "Antarctica/Syowa",
+      "Antarctica/Vostok",
+      "Arctic/Longyearbyen",
+      "Asia/Aden",
+      "Asia/Almaty",
+      "Asia/Amman",
+      "Asia/Anadyr",
+      "Asia/Aqtau",
+      "Asia/Aqtobe",
+      "Asia/Ashgabat",
+      "Asia/Ashkhabad",
+      "Asia/Baghdad",
+      "Asia/Bahrain",
+      "Asia/Baku",
+      "Asia/Bangkok",
+      "Asia/Beirut",
+      "Asia/Bishkek",
+      "Asia/Brunei",
+      "Asia/Calcutta",
+      "Asia/Choibalsan",
+      "Asia/Chongqing",
+      "Asia/Chungking",
+      "Asia/Colombo",
+      "Asia/Dacca",
+      "Asia/Damascus",
+      "Asia/Dhaka",
+      "Asia/Dili",
+      "Asia/Dubai",
+      "Asia/Dushanbe",
+      "Asia/Gaza",
+      "Asia/Harbin",
+      "Asia/Hebron",
+      "Asia/Ho_Chi_Minh",
+      "Asia/Hong_Kong",
+      "Asia/Hovd",
+      "Asia/Irkutsk",
+      "Asia/Istanbul",
+      "Asia/Jakarta",
+      "Asia/Jayapura",
+      "Asia/Jerusalem",
+      "Asia/Kabul",
+      "Asia/Kamchatka",
+      "Asia/Karachi",
+      "Asia/Kashgar",
+      "Asia/Kathmandu",
+      "Asia/Katmandu",
+      "Asia/Kolkata",
+      "Asia/Krasnoyarsk",
+      "Asia/Kuala_Lumpur",
+      "Asia/Kuching",
+      "Asia/Kuwait",
+      "Asia/Macao",
+      "Asia/Macau",
+      "Asia/Magadan",
+      "Asia/Makassar",
+      "Asia/Manila",
+      "Asia/Muscat",
+      "Asia/Nicosia",
+      "Asia/Novokuznetsk",
+      "Asia/Novosibirsk",
+      "Asia/Omsk",
+      "Asia/Oral",
+      "Asia/Phnom_Penh",
+      "Asia/Pontianak",
+      "Asia/Pyongyang",
+      "Asia/Qatar",
+      "Asia/Qyzylorda",
+      "Asia/Rangoon",
+      "Asia/Riyadh",
+      "Asia/Saigon",
+      "Asia/Sakhalin",
+      "Asia/Samarkand",
+      "Asia/Seoul",
+      "Asia/Shanghai",
+      "Asia/Singapore",
+      "Asia/Taipei",
+      "Asia/Tashkent",
+      "Asia/Tbilisi",
+      "Asia/Tehran",
+      "Asia/Tel_Aviv",
+      "Asia/Thimbu",
+      "Asia/Thimphu",
+      "Asia/Tokyo",
+      "Asia/Ujung_Pandang",
+      "Asia/Ulaanbaatar",
+      "Asia/Ulan_Bator",
+      "Asia/Urumqi",
+      "Asia/Vientiane",
+      "Asia/Vladivostok",
+      "Asia/Yakutsk",
+      "Asia/Yekaterinburg",
+      "Asia/Yerevan",
+      "Atlantic/Azores",
+      "Atlantic/Bermuda",
+      "Atlantic/Canary",
+      "Atlantic/Cape_Verde",
+      "Atlantic/Faeroe",
+      "Atlantic/Faroe",
+      "Atlantic/Jan_Mayen",
+      "Atlantic/Madeira",
+      "Atlantic/Reykjavik",
+      "Atlantic/South_Georgia",
+      "Atlantic/St_Helena",
+      "Atlantic/Stanley",
+      "Australia/ACT",
+      "Australia/Adelaide",
+      "Australia/Brisbane",
+      "Australia/Broken_Hill",
+      "Australia/Canberra",
+      "Australia/Currie",
+      "Australia/Darwin",
+      "Australia/Eucla",
+      "Australia/Hobart",
+      "Australia/LHI",
+      "Australia/Lindeman",
+      "Australia/Lord_Howe",
+      "Australia/Melbourne",
+      "Australia/NSW",
+      "Australia/North",
+      "Australia/Perth",
+      "Australia/Queensland",
+      "Australia/South",
+      "Australia/Sydney",
+      "Australia/Tasmania",
+      "Australia/Victoria",
+      "Australia/West",
+      "Australia/Yancowinna",
+      "Brazil/Acre",
+      "Brazil/DeNoronha",
+      "Brazil/East",
+      "Brazil/West",
+      "CET",
+      "CST6CDT",
+      "Canada/Atlantic",
+      "Canada/Central",
+      "Canada/East-Saskatchewan",
+      "Canada/Eastern",
+      "Canada/Mountain",
+      "Canada/Newfoundland",
+      "Canada/Pacific",
+      "Canada/Saskatchewan",
+      "Canada/Yukon",
+      "Chile/Continental",
+      "Chile/EasterIsland",
+      "Cuba",
+      "EET",
+      "EST",
+      "EST5EDT",
+      "Egypt",
+      "Eire",
+      "Etc/GMT",
+      "Etc/GMT+0",
+      "Etc/GMT+1",
+      "Etc/GMT+10",
+      "Etc/GMT+11",
+      "Etc/GMT+12",
+      "Etc/GMT+2",
+      "Etc/GMT+3",
+      "Etc/GMT+4",
+      "Etc/GMT+5",
+      "Etc/GMT+6",
+      "Etc/GMT+7",
+      "Etc/GMT+8",
+      "Etc/GMT+9",
+      "Etc/GMT-0",
+      "Etc/GMT-1",
+      "Etc/GMT-10",
+      "Etc/GMT-11",
+      "Etc/GMT-12",
+      "Etc/GMT-13",
+      "Etc/GMT-14",
+      "Etc/GMT-2",
+      "Etc/GMT-3",
+      "Etc/GMT-4",
+      "Etc/GMT-5",
+      "Etc/GMT-6",
+      "Etc/GMT-7",
+      "Etc/GMT-8",
+      "Etc/GMT-9",
+      "Etc/GMT0",
+      "Etc/Greenwich",
+      "Etc/UCT",
+      "Etc/UTC",
+      "Etc/Universal",
+      "Etc/Zulu",
+      "Europe/Amsterdam",
+      "Europe/Andorra",
+      "Europe/Athens",
+      "Europe/Belfast",
+      "Europe/Belgrade",
+      "Europe/Berlin",
+      "Europe/Bratislava",
+      "Europe/Brussels",
+      "Europe/Bucharest",
+      "Europe/Budapest",
+      "Europe/Chisinau",
+      "Europe/Copenhagen",
+      "Europe/Dublin",
+      "Europe/Gibraltar",
+      "Europe/Guernsey",
+      "Europe/Helsinki",
+      "Europe/Isle_of_Man",
+      "Europe/Istanbul",
+      "Europe/Jersey",
+      "Europe/Kaliningrad",
+      "Europe/Kiev",
+      "Europe/Lisbon",
+      "Europe/Ljubljana",
+      "Europe/London",
+      "Europe/Luxembourg",
+      "Europe/Madrid",
+      "Europe/Malta",
+      "Europe/Mariehamn",
+      "Europe/Minsk",
+      "Europe/Monaco",
+      "Europe/Moscow",
+      "Europe/Nicosia",
+      "Europe/Oslo",
+      "Europe/Paris",
+      "Europe/Podgorica",
+      "Europe/Prague",
+      "Europe/Riga",
+      "Europe/Rome",
+      "Europe/Samara",
+      "Europe/San_Marino",
+      "Europe/Sarajevo",
+      "Europe/Simferopol",
+      "Europe/Skopje",
+      "Europe/Sofia",
+      "Europe/Stockholm",
+      "Europe/Tallinn",
+      "Europe/Tirane",
+      "Europe/Tiraspol",
+      "Europe/Uzhgorod",
+      "Europe/Vaduz",
+      "Europe/Vatican",
+      "Europe/Vienna",
+      "Europe/Vilnius",
+      "Europe/Volgograd",
+      "Europe/Warsaw",
+      "Europe/Zagreb",
+      "Europe/Zaporozhye",
+      "Europe/Zurich",
+      "GB",
+      "GB-Eire",
+      "GMT",
+      "GMT+0",
+      "GMT-0",
+      "GMT0",
+      "Greenwich",
+      "HST",
+      "Hongkong",
+      "Iceland",
+      "Indian/Antananarivo",
+      "Indian/Chagos",
+      "Indian/Christmas",
+      "Indian/Cocos",
+      "Indian/Comoro",
+      "Indian/Kerguelen",
+      "Indian/Mahe",
+      "Indian/Maldives",
+      "Indian/Mauritius",
+      "Indian/Mayotte",
+      "Indian/Reunion",
+      "Iran",
+      "Israel",
+      "Jamaica",
+      "Japan",
+      "Kwajalein",
+      "Libya",
+      "MET",
+      "MST",
+      "MST7MDT",
+      "Mexico/BajaNorte",
+      "Mexico/BajaSur",
+      "Mexico/General",
+      "NZ",
+      "NZ-CHAT",
+      "Navajo",
+      "PRC",
+      "PST8PDT",
+      "Pacific/Apia",
+      "Pacific/Auckland",
+      "Pacific/Chatham",
+      "Pacific/Chuuk",
+      "Pacific/Easter",
+      "Pacific/Efate",
+      "Pacific/Enderbury",
+      "Pacific/Fakaofo",
+      "Pacific/Fiji",
+      "Pacific/Funafuti",
+      "Pacific/Galapagos",
+      "Pacific/Gambier",
+      "Pacific/Guadalcanal",
+      "Pacific/Guam",
+      "Pacific/Honolulu",
+      "Pacific/Johnston",
+      "Pacific/Kiritimati",
+      "Pacific/Kosrae",
+      "Pacific/Kwajalein",
+      "Pacific/Majuro",
+      "Pacific/Marquesas",
+      "Pacific/Midway",
+      "Pacific/Nauru",
+      "Pacific/Niue",
+      "Pacific/Norfolk",
+      "Pacific/Noumea",
+      "Pacific/Pago_Pago",
+      "Pacific/Palau",
+      "Pacific/Pitcairn",
+      "Pacific/Pohnpei",
+      "Pacific/Ponape",
+      "Pacific/Port_Moresby",
+      "Pacific/Rarotonga",
+      "Pacific/Saipan",
+      "Pacific/Samoa",
+      "Pacific/Tahiti",
+      "Pacific/Tarawa",
+      "Pacific/Tongatapu",
+      "Pacific/Truk",
+      "Pacific/Wake",
+      "Pacific/Wallis",
+      "Pacific/Yap",
+      "Poland",
+      "Portugal",
+      "ROC",
+      "ROK",
+      "Singapore",
+      "Turkey",
+      "UCT",
+      "US/Alaska",
+      "US/Aleutian",
+      "US/Arizona",
+      "US/Central",
+      "US/East-Indiana",
+      "US/Eastern",
+      "US/Hawaii",
+      "US/Indiana-Starke",
+      "US/Michigan",
+      "US/Mountain",
+      "US/Pacific",
+      "US/Pacific-New",
+      "US/Samoa",
+      "UTC",
+      "Universal",
+      "W-SU",
+      "WET",
+      "Zulu"};
 
-    static {
-      for (int i = 0; i < timezoneList.length; i++) {
-        timezoneMap.put(timezoneList[i], i);
-      }
+  static {
+    for (int i = 0; i < timezoneList.length; i++) {
+      timezoneMap.put(timezoneList[i], i);
     }
+  }
 
-    public static final DateTimeFormatter formatDate           = DateTimeFormat.forPattern("yyyy-MM-dd");
-    public static final DateTimeFormatter formatTimeStampMilli = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
-    public static final DateTimeFormatter formatTimeStampTZ    = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ");
-    public static final DateTimeFormatter formatTime           = DateTimeFormat.forPattern("HH:mm:ss.SSS");
+  public static final DateTimeFormatter formatDate = DateTimeFormat.forPattern("yyyy-MM-dd");
+  public static final DateTimeFormatter formatTimeStampMilli = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+  public static final DateTimeFormatter formatTimeStampTZ = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ");
+  public static final DateTimeFormatter formatTime = DateTimeFormat.forPattern("HH:mm:ss.SSS");
 
-    public static DateTimeFormatter dateTimeTZFormat = null;
-    public static DateTimeFormatter timeFormat = null;
+  public static DateTimeFormatter dateTimeTZFormat = null;
+  public static DateTimeFormatter timeFormat = null;
 
-    public static final int yearsToMonths = 12;
-    public static final int hoursToMillis = 60 * 60 * 1000;
-    public static final int minutesToMillis = 60 * 1000;
-    public static final int secondsToMillis = 1000;
-    public static final int monthToStandardDays = 30;
-    public static final long monthsToMillis = 2592000000L; // 30 * 24 * 60 * 60 * 1000
-    public static final int daysToStandardMillis = 24 * 60 * 60 * 1000;
+  public static final int yearsToMonths = 12;
+  public static final int hoursToMillis = 60 * 60 * 1000;
+  public static final int minutesToMillis = 60 * 1000;
+  public static final int secondsToMillis = 1000;
+  public static final int monthToStandardDays = 30;
+  public static final long monthsToMillis = 2592000000L; // 30 * 24 * 60 * 60 * 1000
+  public static final int daysToStandardMillis = 24 * 60 * 60 * 1000;
 
 
   public static int getIndex(String timezone) {
-        return timezoneMap.get(timezone);
-    }
-
-    public static String getTimeZone(int index) {
-        return timezoneList[index];
-    }
+    return timezoneMap.get(timezone);
+  }
 
-    // Function returns the date time formatter used to parse date strings
-    public static DateTimeFormatter getDateTimeFormatter() {
+  public static String getTimeZone(int index) {
+    return timezoneList[index];
+  }
 
-        if (dateTimeTZFormat == null) {
-            DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");
-            DateTimeParser optionalTime = DateTimeFormat.forPattern(" HH:mm:ss").getParser();
-            DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
-            DateTimeParser optionalZone = DateTimeFormat.forPattern(" ZZZ").getParser();
+  // Function returns the date time formatter used to parse date strings
+  public static DateTimeFormatter getDateTimeFormatter() {
 
-            dateTimeTZFormat = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).appendOptional(optionalZone).toFormatter();
-        }
+    if (dateTimeTZFormat == null) {
+      DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");
+      DateTimeParser optionalTime = DateTimeFormat.forPattern(" HH:mm:ss").getParser();
+      DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
+      DateTimeParser optionalZone = DateTimeFormat.forPattern(" ZZZ").getParser();
 
-        return dateTimeTZFormat;
+      dateTimeTZFormat = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).appendOptional(optionalZone).toFormatter();
     }
 
-    // Function returns time formatter used to parse time strings
-    public static DateTimeFormatter getTimeFormatter() {
-        if (timeFormat == null) {
-            DateTimeFormatter timeFormatter = DateTimeFormat.forPattern("HH:mm:ss");
-            DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
-            timeFormat = new DateTimeFormatterBuilder().append(timeFormatter).appendOptional(optionalSec).toFormatter();
-        }
-        return timeFormat;
-    }
+    return dateTimeTZFormat;
+  }
 
-    public static int monthsFromPeriod(Period period){
-      return (period.getYears() * yearsToMonths) + period.getMonths();
+  // Function returns time formatter used to parse time strings
+  public static DateTimeFormatter getTimeFormatter() {
+    if (timeFormat == null) {
+      DateTimeFormatter timeFormatter = DateTimeFormat.forPattern("HH:mm:ss");
+      DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
+      timeFormat = new DateTimeFormatterBuilder().append(timeFormatter).appendOptional(optionalSec).toFormatter();
     }
+    return timeFormat;
+  }
 
-    public static int millisFromPeriod(final Period period){
-      return (period.getHours() * hoursToMillis) +
-      (period.getMinutes() * minutesToMillis) +
-      (period.getSeconds() * secondsToMillis) +
-      (period.getMillis());
-    }
+  public static int monthsFromPeriod(Period period) {
+    return (period.getYears() * yearsToMonths) + period.getMonths();
+  }
 
-    public static long toMillis(LocalDateTime localDateTime) {
-      return LocalDateTimes.getLocalMillis(localDateTime);
-    }
+  public static int millisFromPeriod(final Period period) {
+    return (period.getHours() * hoursToMillis) +
+        (period.getMinutes() * minutesToMillis) +
+        (period.getSeconds() * secondsToMillis) +
+        (period.getMillis());
+  }
 
-    public static int toMillisOfDay(final LocalDateTime localDateTime) {
-      return localDateTime.toDateTime(DateTimeZone.UTC).millisOfDay().get();
-    }
+  public static long toMillis(LocalDateTime localDateTime) {
+    return LocalDateTimes.getLocalMillis(localDateTime);
+  }
+
+  public static int toMillisOfDay(final LocalDateTime localDateTime) {
+    return localDateTime.toDateTime(DateTimeZone.UTC).millisOfDay().get();
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index 4c439b2cc1066..4b11b368dff1e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import io.netty.buffer.ArrowBuf;
@@ -32,39 +33,39 @@ public class DecimalUtility {
   public final static int MAX_DIGITS = 9;
   public final static int DIGITS_BASE = 1000000000;
   public final static int DIGITS_MAX = 999999999;
-  public final static int INTEGER_SIZE = (Integer.SIZE/8);
+  public final static int INTEGER_SIZE = (Integer.SIZE / 8);
 
   public final static String[] decimalToString = {"",
-          "0",
-          "00",
-          "000",
-          "0000",
-          "00000",
-          "000000",
-          "0000000",
-          "00000000",
-          "000000000"};
+      "0",
+      "00",
+      "000",
+      "0000",
+      "00000",
+      "000000",
+      "0000000",
+      "00000000",
+      "000000000"};
 
   public final static long[] scale_long_constants = {
-          1,
-          10,
-          100,
-          1000,
-          10000,
-          100000,
-          1000000,
-          10000000,
-          100000000,
-          1000000000,
-          10000000000l,
-          100000000000l,
-          1000000000000l,
-          10000000000000l,
-          100000000000000l,
-          1000000000000000l,
-          10000000000000000l,
-          100000000000000000l,
-          1000000000000000000l};
+      1,
+      10,
+      100,
+      1000,
+      10000,
+      100000,
+      1000000,
+      10000000,
+      100000000,
+      1000000000,
+      10000000000l,
+      100000000000l,
+      1000000000000l,
+      10000000000000l,
+      100000000000000l,
+      1000000000000000l,
+      10000000000000000l,
+      100000000000000000l,
+      1000000000000000000l};
 
   public static final int DECIMAL_BYTE_LENGTH = 16;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
index 4108dc4610838..cf0596c8c1fb4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
@@ -40,7 +40,7 @@ public class DictionaryUtility {
    * have the dictionary type
    *
    * NOTE: in the message format, fields have the dictionary type
-   *       in the memory format, they have the index type
+   * in the memory format, they have the index type
    */
   public static Field toMessageFormat(Field field, DictionaryProvider provider, Set<Long> dictionaryIdsUsed) {
     DictionaryEncoding encoding = field.getDictionary();
@@ -51,7 +51,7 @@ public static Field toMessageFormat(Field field, DictionaryProvider provider, Se
     }
 
     List<Field> updatedChildren = new ArrayList<>(children.size());
-    for (Field child: children) {
+    for (Field child : children) {
       updatedChildren.add(toMessageFormat(child, provider, dictionaryIdsUsed));
     }
 
@@ -85,7 +85,7 @@ public static Field toMemoryFormat(Field field, BufferAllocator allocator, Map<L
     }
 
     List<Field> updatedChildren = new ArrayList<>(children.size());
-    for (Field child: children) {
+    for (Field child : children) {
       updatedChildren.add(toMemoryFormat(child, allocator, dictionaries));
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
index c598069c2c309..480bd76d445b0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import java.util.ArrayList;
@@ -58,7 +59,7 @@ public boolean equals(Object obj) {
   public final String toString() {
     try {
       return mapper.writeValueAsString(this);
-    } catch(JsonProcessingException e) {
+    } catch (JsonProcessingException e) {
       throw new IllegalStateException("Cannot serialize array list to JSON string", e);
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
index e8ce5221eebd9..6455389d582b9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import java.util.LinkedHashMap;
@@ -51,14 +52,14 @@ public boolean equals(Object obj) {
       return false;
     }
     for (K key : this.keySet()) {
-      if (this.get(key) == null ) {
+      if (this.get(key) == null) {
         if (other.get(key) == null) {
           continue;
         } else {
           return false;
         }
       }
-      if ( ! this.get(key).equals(other.get(key))) {
+      if (!this.get(key).equals(other.get(key))) {
         return false;
       }
     }
@@ -69,7 +70,7 @@ public boolean equals(Object obj) {
   public final String toString() {
     try {
       return mapper.writeValueAsString(this);
-    } catch(JsonProcessingException e) {
+    } catch (JsonProcessingException e) {
       throw new IllegalStateException("Cannot serialize hash map to JSON string", e);
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
index b35aaa401bae4..6d3b390379a56 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import java.util.AbstractMap;
@@ -95,16 +96,16 @@ public V get(Object key) {
     public V put(K key, V value) {
       final Entry<Integer, V> oldPair = primary.get(key);
       // if key exists try replacing otherwise, assign a new ordinal identifier
-      final int ordinal = oldPair == null ? primary.size():oldPair.getKey();
+      final int ordinal = oldPair == null ? primary.size() : oldPair.getKey();
       primary.put(key, new AbstractMap.SimpleImmutableEntry<>(ordinal, value));
       secondary.put(ordinal, value);
-      return oldPair==null ? null:oldPair.getValue();
+      return oldPair == null ? null : oldPair.getValue();
     }
 
     @Override
     public V remove(Object key) {
       final Entry<Integer, V> oldPair = primary.remove(key);
-      if (oldPair!=null) {
+      if (oldPair != null) {
         final int lastOrdinal = secondary.size();
         final V last = secondary.get(lastOrdinal);
         // normalize mappings so that all numbers until primary.size() is assigned
@@ -112,7 +113,7 @@ public V remove(Object key) {
         secondary.put(oldPair.getKey(), last);
         primary.put((K) key, new AbstractMap.SimpleImmutableEntry<>(oldPair.getKey(), last));
       }
-      return oldPair==null ? null:oldPair.getValue();
+      return oldPair == null ? null : oldPair.getValue();
     }
 
     @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java b/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
index bd7396249a72c..b4ff2522daf33 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/OversizedAllocationException.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 
@@ -23,7 +24,6 @@
  * {@link org.apache.arrow.memory.BufferAllocator#buffer(int) allocator}.
  *
  * <p>Operators should handle this exception to split the batch and later resume the execution on the next iteration.</p>
- *
  */
 public class OversizedAllocationException extends RuntimeException {
   public OversizedAllocationException() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/SchemaChangeRuntimeException.java b/java/vector/src/main/java/org/apache/arrow/vector/util/SchemaChangeRuntimeException.java
index c281561430707..ddfea948a8f74 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/SchemaChangeRuntimeException.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/SchemaChangeRuntimeException.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
index ce82f445ad883..15ce132fc801c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import java.io.DataInput;
@@ -74,13 +75,16 @@ public Text() {
 
   /**
    * Construct from a string.
+   *
    * @param string initialize from that string
    */
   public Text(String string) {
     set(string);
   }
 
-  /** Construct from another text.
+  /**
+   * Construct from another text.
+   *
    * @param utf8 initialize from that Text
    */
   public Text(Text utf8) {
@@ -89,6 +93,7 @@ public Text(Text utf8) {
 
   /**
    * Construct from a byte array.
+   *
    * @param utf8 initialize from that byte array
    */
   public Text(byte[] utf8) {
@@ -98,6 +103,7 @@ public Text(byte[] utf8) {
   /**
    * Get a copy of the bytes that is exactly the length of the data. See {@link #getBytes()} for faster access to the
    * underlying array.
+   *
    * @return a copy of the underlying array
    */
   public byte[] copyBytes() {
@@ -109,13 +115,16 @@ public byte[] copyBytes() {
   /**
    * Returns the raw bytes; however, only data up to {@link #getLength()} is valid. Please use {@link #copyBytes()} if
    * you need the returned array to be precisely the length of the data.
+   *
    * @return the underlying array
    */
   public byte[] getBytes() {
     return bytes;
   }
 
-  /** @return the number of bytes in the byte array */
+  /**
+   * @return the number of bytes in the byte array
+   */
   public int getLength() {
     return length;
   }
@@ -128,12 +137,10 @@ public int getLength() {
    * @return the Unicode scalar value at position or -1 if the position is invalid or points to a trailing byte
    */
   public int charAt(int position) {
-    if (position > this.length)
-    {
+    if (position > this.length) {
       return -1; // too long
     }
-    if (position < 0)
-    {
+    if (position < 0) {
       return -1; // duh.
     }
 
@@ -150,7 +157,7 @@ public int find(String what) {
    * starting position is measured in bytes and the return value is in terms of byte position in the buffer. The backing
    * buffer is not converted to a string for this operation.
    *
-   * @param what the string to search for
+   * @param what  the string to search for
    * @param start where to start from
    * @return byte position of the first occurence of the search string in the UTF-8 buffer or -1 if not found
    */
@@ -196,6 +203,7 @@ public int find(String what, int start) {
 
   /**
    * Set to contain the contents of a string.
+   *
    * @param string the string to initialize from
    */
   public void set(String string) {
@@ -210,14 +218,18 @@ public void set(String string) {
 
   /**
    * Set to a utf8 byte array
+   *
    * @param utf8 the byte array to initialize from
    */
   public void set(byte[] utf8) {
     set(utf8, 0, utf8.length);
   }
 
-  /** copy a text.
-   * @param other the text to initialize from */
+  /**
+   * copy a text.
+   *
+   * @param other the text to initialize from
+   */
   public void set(Text other) {
     set(other.getBytes(), 0, other.getLength());
   }
@@ -225,12 +237,9 @@ public void set(Text other) {
   /**
    * Set the Text to range of bytes
    *
-   * @param utf8
-   *          the data to copy from
-   * @param start
-   *          the first position of the new string
-   * @param len
-   *          the number of bytes of the new string
+   * @param utf8  the data to copy from
+   * @param start the first position of the new string
+   * @param len   the number of bytes of the new string
    */
   public void set(byte[] utf8, int start, int len) {
     setCapacity(len, false);
@@ -241,12 +250,9 @@ public void set(byte[] utf8, int start, int len) {
   /**
    * Append a range of bytes to the end of the given text
    *
-   * @param utf8
-   *          the data to copy from
-   * @param start
-   *          the first position to append from utf8
-   * @param len
-   *          the number of bytes to append
+   * @param utf8  the data to copy from
+   * @param start the first position to append from utf8
+   * @param len   the number of bytes to append
    */
   public void append(byte[] utf8, int start, int len) {
     setCapacity(length + len, true);
@@ -270,7 +276,7 @@ public void clear() {
    * then the capacity and existing content of the buffer are unchanged. If <code>len</code> is larger than the current
    * capacity, the Text object's capacity is increased to match.
    *
-   * @param len the number of bytes we need
+   * @param len      the number of bytes we need
    * @param keepData should the old data be kept
    */
   private void setCapacity(int len, boolean keepData) {
@@ -295,7 +301,8 @@ public String toString() {
   /**
    * Read a Text object whose length is already known. This allows creating Text from a stream which uses a different
    * serialization format.
-   * @param in the input to initialize from
+   *
+   * @param in  the input to initialize from
    * @param len how many bytes to read from in
    * @throws IOException if something bad happens
    */
@@ -351,9 +358,11 @@ public int hashCode() {
   }
 
   // / STATIC UTILITIES FROM HERE DOWN
+
   /**
    * Converts the provided byte array to a String using the UTF-8 encoding. If the input is malformed, replace by a
    * default value.
+   *
    * @param utf8 bytes to decode
    * @return the decoded string
    * @throws CharacterCodingException if this is not valid UTF-8
@@ -371,9 +380,10 @@ public static String decode(byte[] utf8, int start, int length)
    * Converts the provided byte array to a String using the UTF-8 encoding. If <code>replace</code> is true, then
    * malformed input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a
    * MalformedInputException.
-   * @param utf8 the bytes to decode
-   * @param start where to start from
-   * @param length length of the bytes to decode
+   *
+   * @param utf8    the bytes to decode
+   * @param start   where to start from
+   * @param length  length of the bytes to decode
    * @param replace whether to replace malformed characters with U+FFFD
    * @return the decoded string
    * @throws CharacterCodingException if the input could not be decoded
@@ -418,8 +428,7 @@ public static ByteBuffer encode(String string)
    * input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a
    * MalformedInputException.
    *
-
-   * @param string the string to encode
+   * @param string  the string to encode
    * @param replace whether to replace malformed characters with U+FFFD
    * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit()
    * @throws CharacterCodingException if the string could not be encoded
@@ -453,10 +462,8 @@ public static ByteBuffer encode(String string, boolean replace)
   /**
    * Check if a byte array contains valid utf-8
    *
-   * @param utf8
-   *          byte array
-   * @throws MalformedInputException
-   *           if the byte array contains invalid utf-8
+   * @param utf8 byte array
+   * @throws MalformedInputException if the byte array contains invalid utf-8
    */
   public static void validateUTF8(byte[] utf8) throws MalformedInputException {
     validateUTF8(utf8, 0, utf8.length);
@@ -465,14 +472,10 @@ public static void validateUTF8(byte[] utf8) throws MalformedInputException {
   /**
    * Check to see if a byte array is valid utf-8
    *
-   * @param utf8
-   *          the array of bytes
-   * @param start
-   *          the offset of the first byte in the array
-   * @param len
-   *          the length of the byte sequence
-   * @throws MalformedInputException
-   *           if the byte array contains invalid bytes
+   * @param utf8  the array of bytes
+   * @param start the offset of the first byte in the array
+   * @param len   the length of the byte sequence
+   * @throws MalformedInputException if the byte array contains invalid bytes
    */
   public static void validateUTF8(byte[] utf8, int start, int len)
       throws MalformedInputException {
@@ -484,67 +487,67 @@ public static void validateUTF8(byte[] utf8, int start, int len)
       int aByte = utf8[count] & 0xFF;
 
       switch (state) {
-      case LEAD_BYTE:
-        leadByte = aByte;
-        length = bytesFromUTF8[aByte];
+        case LEAD_BYTE:
+          leadByte = aByte;
+          length = bytesFromUTF8[aByte];
+
+          switch (length) {
+            case 0: // check for ASCII
+              if (leadByte > 0x7F) {
+                throw new MalformedInputException(count);
+              }
+              break;
+            case 1:
+              if (leadByte < 0xC2 || leadByte > 0xDF) {
+                throw new MalformedInputException(count);
+              }
+              state = TRAIL_BYTE_1;
+              break;
+            case 2:
+              if (leadByte < 0xE0 || leadByte > 0xEF) {
+                throw new MalformedInputException(count);
+              }
+              state = TRAIL_BYTE_1;
+              break;
+            case 3:
+              if (leadByte < 0xF0 || leadByte > 0xF4) {
+                throw new MalformedInputException(count);
+              }
+              state = TRAIL_BYTE_1;
+              break;
+            default:
+              // too long! Longest valid UTF-8 is 4 bytes (lead + three)
+              // or if < 0 we got a trail byte in the lead byte position
+              throw new MalformedInputException(count);
+          } // switch (length)
+          break;
 
-        switch (length) {
-        case 0: // check for ASCII
-          if (leadByte > 0x7F) {
+        case TRAIL_BYTE_1:
+          if (leadByte == 0xF0 && aByte < 0x90) {
             throw new MalformedInputException(count);
           }
-          break;
-        case 1:
-          if (leadByte < 0xC2 || leadByte > 0xDF) {
+          if (leadByte == 0xF4 && aByte > 0x8F) {
             throw new MalformedInputException(count);
           }
-          state = TRAIL_BYTE_1;
-          break;
-        case 2:
-          if (leadByte < 0xE0 || leadByte > 0xEF) {
+          if (leadByte == 0xE0 && aByte < 0xA0) {
             throw new MalformedInputException(count);
           }
-          state = TRAIL_BYTE_1;
-          break;
-        case 3:
-          if (leadByte < 0xF0 || leadByte > 0xF4) {
+          if (leadByte == 0xED && aByte > 0x9F) {
+            throw new MalformedInputException(count);
+          }
+          // falls through to regular trail-byte test!!
+        case TRAIL_BYTE:
+          if (aByte < 0x80 || aByte > 0xBF) {
             throw new MalformedInputException(count);
           }
-          state = TRAIL_BYTE_1;
+          if (--length == 0) {
+            state = LEAD_BYTE;
+          } else {
+            state = TRAIL_BYTE;
+          }
           break;
         default:
-          // too long! Longest valid UTF-8 is 4 bytes (lead + three)
-          // or if < 0 we got a trail byte in the lead byte position
-          throw new MalformedInputException(count);
-        } // switch (length)
-        break;
-
-      case TRAIL_BYTE_1:
-        if (leadByte == 0xF0 && aByte < 0x90) {
-          throw new MalformedInputException(count);
-        }
-        if (leadByte == 0xF4 && aByte > 0x8F) {
-          throw new MalformedInputException(count);
-        }
-        if (leadByte == 0xE0 && aByte < 0xA0) {
-          throw new MalformedInputException(count);
-        }
-        if (leadByte == 0xED && aByte > 0x9F) {
-          throw new MalformedInputException(count);
-        }
-        // falls through to regular trail-byte test!!
-      case TRAIL_BYTE:
-        if (aByte < 0x80 || aByte > 0xBF) {
-          throw new MalformedInputException(count);
-        }
-        if (--length == 0) {
-          state = LEAD_BYTE;
-        } else {
-          state = TRAIL_BYTE;
-        }
-        break;
-      default:
-        break;
+          break;
       } // switch (state)
       count++;
     }
@@ -556,25 +559,26 @@ public static void validateUTF8(byte[] utf8, int start, int len)
    * six byte sequences.
    */
   static final int[] bytesFromUTF8 =
-  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0,
-      // trail bytes
-      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1,
-      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-      1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
-      3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5 };
+      {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+          0, 0, 0, 0, 0, 0, 0,
+          // trail bytes
+          -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+          -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+          -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+          -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
+          3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5};
 
   /**
    * Returns the next code point at the current position in the buffer. The buffer's position will be incremented. Any
    * mark set on this buffer will be changed by this method!
+   *
    * @param bytes the incoming bytes
    * @return the corresponding unicode codepoint
    */
@@ -583,30 +587,29 @@ public static int bytesToCodePoint(ByteBuffer bytes) {
     byte b = bytes.get();
     bytes.reset();
     int extraBytesToRead = bytesFromUTF8[(b & 0xFF)];
-    if (extraBytesToRead < 0)
-    {
+    if (extraBytesToRead < 0) {
       return -1; // trailing byte!
     }
     int ch = 0;
 
     switch (extraBytesToRead) {
-    case 5:
-      ch += (bytes.get() & 0xFF);
-      ch <<= 6; /* remember, illegal UTF-8 */
-    case 4:
-      ch += (bytes.get() & 0xFF);
-      ch <<= 6; /* remember, illegal UTF-8 */
-    case 3:
-      ch += (bytes.get() & 0xFF);
-      ch <<= 6;
-    case 2:
-      ch += (bytes.get() & 0xFF);
-      ch <<= 6;
-    case 1:
-      ch += (bytes.get() & 0xFF);
-      ch <<= 6;
-    case 0:
-      ch += (bytes.get() & 0xFF);
+      case 5:
+        ch += (bytes.get() & 0xFF);
+        ch <<= 6; /* remember, illegal UTF-8 */
+      case 4:
+        ch += (bytes.get() & 0xFF);
+        ch <<= 6; /* remember, illegal UTF-8 */
+      case 3:
+        ch += (bytes.get() & 0xFF);
+        ch <<= 6;
+      case 2:
+        ch += (bytes.get() & 0xFF);
+        ch <<= 6;
+      case 1:
+        ch += (bytes.get() & 0xFF);
+        ch <<= 6;
+      case 0:
+        ch += (bytes.get() & 0xFF);
     }
     ch -= offsetsFromUTF8[extraBytesToRead];
 
@@ -614,14 +617,13 @@ public static int bytesToCodePoint(ByteBuffer bytes) {
   }
 
   static final int offsetsFromUTF8[] =
-  { 0x00000000, 0x00003080,
-      0x000E2080, 0x03C82080, 0xFA082080, 0x82082080 };
+      {0x00000000, 0x00003080,
+          0x000E2080, 0x03C82080, 0xFA082080, 0x82082080};
 
   /**
    * For the given string, returns the number of UTF-8 bytes required to encode the string.
    *
-   * @param string
-   *          text to encode
+   * @param string text to encode
    * @return number of UTF-8 bytes required to encode
    */
   public static int utf8Length(String string) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/TransferPair.java b/java/vector/src/main/java/org/apache/arrow/vector/util/TransferPair.java
index 6e68d55226266..314ffdcb6637c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/TransferPair.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/TransferPair.java
@@ -15,13 +15,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import org.apache.arrow.vector.ValueVector;
 
 public interface TransferPair {
   public void transfer();
+
   public void splitAndTransfer(int startIndex, int length);
+
   public ValueVector getTo();
+
   public void copyValueSafe(int from, int to);
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
index b70a63fe7dd02..5851bd5fa5d97 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import java.util.Arrays;
@@ -68,7 +69,7 @@ public static void compareDictionaries(List<DictionaryEncoding> encodings1, List
       Dictionary dict2 = provider2.lookup(id);
 
       if (dict1 == null || dict2 == null) {
-        throw new IllegalArgumentException("The DictionaryProvider did not contain the required dictionary with id: " + id +"\n" + dict1 + "\n" + dict2);
+        throw new IllegalArgumentException("The DictionaryProvider did not contain the required dictionary with id: " + id + "\n" + dict1 + "\n" + dict2);
       }
 
       try {
@@ -132,9 +133,9 @@ static boolean equals(ArrowType type, final Object o1, final Object o2) {
       ArrowType.FloatingPoint fpType = (ArrowType.FloatingPoint) type;
       switch (fpType.getPrecision()) {
         case DOUBLE:
-          return equalEnough((Double)o1, (Double)o2);
+          return equalEnough((Double) o1, (Double) o2);
         case SINGLE:
-          return equalEnough((Float)o1, (Float)o2);
+          return equalEnough((Float) o1, (Float) o2);
         case HALF:
         default:
           throw new UnsupportedOperationException("unsupported precision: " + fpType);
diff --git a/java/vector/src/main/java/org/joda/time/LocalDateTimes.java b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java
index e4f999e1d828e..a1c18fe9a5f41 100644
--- a/java/vector/src/main/java/org/joda/time/LocalDateTimes.java
+++ b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java
@@ -15,11 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.joda.time;
 
 /**
  * Workaround to access package protected fields in JODA
- *
  */
 public class LocalDateTimes {
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/DirtyRootAllocator.java b/java/vector/src/test/java/org/apache/arrow/vector/DirtyRootAllocator.java
index f775f1d2d67af..febd59fba7408 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/DirtyRootAllocator.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/DirtyRootAllocator.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import org.apache.arrow.memory.BufferManager;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
index 194b78585faaf..495bed389e568 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
@@ -75,11 +76,10 @@ public void testSplitAndTransfer() throws Exception {
       sourceVector.allocateNew(40);
 
       /* populate the bitvector -- 010101010101010101010101..... */
-      for(int i = 0; i < 40; i++) {
-        if((i & 1) ==  1) {
+      for (int i = 0; i < 40; i++) {
+        if ((i & 1) == 1) {
           sourceMutator.set(i, 1);
-        }
-        else {
+        } else {
           sourceMutator.set(i, 0);
         }
       }
@@ -87,18 +87,17 @@ public void testSplitAndTransfer() throws Exception {
       sourceMutator.setValueCount(40);
 
       /* check the vector output */
-      for(int i = 0; i < 40; i++) {
+      for (int i = 0; i < 40; i++) {
         int result = sourceAccessor.get(i);
-        if((i & 1) ==  1) {
+        if ((i & 1) == 1) {
           assertEquals(Integer.toString(1), Integer.toString(result));
-        }
-        else {
+        } else {
           assertEquals(Integer.toString(0), Integer.toString(result));
         }
       }
 
       final TransferPair transferPair = sourceVector.getTransferPair(allocator);
-      final BitVector toVector = (BitVector)transferPair.getTo();
+      final BitVector toVector = (BitVector) transferPair.getTo();
       final BitVector.Accessor toAccessor = toVector.getAccessor();
       final BitVector.Mutator toMutator = toVector.getMutator();
 
@@ -110,13 +109,13 @@ public void testSplitAndTransfer() throws Exception {
        *    (2.1) the length is a multiple of 8
        *    (2.2) the length is not a multiple of 8
        */
-      final int[][] transferLengths = {  {0, 8},     /* (1) */
-                                         {8, 10},    /* (1) */
-                                         {18, 0},    /* zero length scenario */
-                                         {18, 8},    /* (2.1) */
-                                         {26, 0},    /* zero length scenario */
-                                         {26, 14}    /* (2.2) */
-                                      };
+      final int[][] transferLengths = {{0, 8},     /* (1) */
+          {8, 10},    /* (1) */
+          {18, 0},    /* zero length scenario */
+          {18, 8},    /* (2.1) */
+          {26, 0},    /* zero length scenario */
+          {26, 14}    /* (2.2) */
+      };
 
       for (final int[] transferLength : transferLengths) {
         final int start = transferLength[0];
@@ -127,10 +126,9 @@ public void testSplitAndTransfer() throws Exception {
         /* check the toVector output after doing splitAndTransfer */
         for (int i = 0; i < length; i++) {
           int result = toAccessor.get(i);
-          if((i & 1) == 1) {
+          if ((i & 1) == 1) {
             assertEquals(Integer.toString(1), Integer.toString(result));
-          }
-          else {
+          } else {
             assertEquals(Integer.toString(0), Integer.toString(result));
           }
         }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
index 08e3700daeebf..54fc306717088 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
@@ -65,7 +66,7 @@ public void testTransferVariableidth() {
     v1.makeTransferPair(v2).transfer();
 
     assertEquals(0, childAllocator1.getAllocatedMemory());
-    int expected = 8*4096 + 4*4096 + 4096;
+    int expected = 8 * 4096 + 4 * 4096 + 4096;
     assertEquals(expected, childAllocator2.getAllocatedMemory());
   }
 
@@ -90,11 +91,11 @@ public void emptyListTransferShouldNotTriggerSchemaChange() {
     final Pointer<Boolean> trigger1 = new Pointer<>();
     final Pointer<Boolean> trigger2 = new Pointer<>();
     final ListVector v1 = new ListVector("v1", allocator,
-            FieldType.nullable(ArrowType.Null.INSTANCE),
-            newTriggerCallback(trigger1));
+        FieldType.nullable(ArrowType.Null.INSTANCE),
+        newTriggerCallback(trigger1));
     final ListVector v2 = new ListVector("v2", allocator,
-            FieldType.nullable(ArrowType.Null.INSTANCE),
-            newTriggerCallback(trigger2));
+        FieldType.nullable(ArrowType.Null.INSTANCE),
+        newTriggerCallback(trigger2));
 
     v1.makeTransferPair(v2).transfer();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index b98c24d189528..774fbe084f1c2 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
@@ -39,6 +40,7 @@ public class TestDecimalVector {
       intValues[2 * i] = -1 * (1 << i + 1);
     }
   }
+
   private int scale = 3;
 
   @Test
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
index 3bf3b1cedff38..f2db9baac04ca 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
@@ -35,8 +36,8 @@ public class TestDictionaryVector {
   private BufferAllocator allocator;
 
   byte[] zero = "foo".getBytes(StandardCharsets.UTF_8);
-  byte[] one  = "bar".getBytes(StandardCharsets.UTF_8);
-  byte[] two  = "baz".getBytes(StandardCharsets.UTF_8);
+  byte[] one = "bar".getBytes(StandardCharsets.UTF_8);
+  byte[] two = "baz".getBytes(StandardCharsets.UTF_8);
 
   @Before
   public void init() {
@@ -74,7 +75,7 @@ public void testEncodeStrings() {
 
       Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null));
 
-      try(final ValueVector encoded = (FieldVector) DictionaryEncoder.encode(vector, dictionary)) {
+      try (final ValueVector encoded = (FieldVector) DictionaryEncoder.encode(vector, dictionary)) {
         // verify indices
         assertEquals(NullableIntVector.class, encoded.getClass());
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index 304db9dcc00bd..5677f2566797a 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import com.google.common.collect.Lists;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index eb30fdd46bf60..fdb576ef75cc4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -15,12 +15,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.ListVector;
@@ -100,11 +101,11 @@ public void testCopyFrom() throws Exception {
 
       /* index 0 */
       Object result = accessor.getObject(0);
-      ArrayList<Long> resultSet = (ArrayList<Long>)result;
+      ArrayList<Long> resultSet = (ArrayList<Long>) result;
       assertEquals(3, resultSet.size());
-      assertEquals(new Long(1), (Long)resultSet.get(0));
-      assertEquals(new Long(2), (Long)resultSet.get(1));
-      assertEquals(new Long(3), (Long)resultSet.get(2));
+      assertEquals(new Long(1), (Long) resultSet.get(0));
+      assertEquals(new Long(2), (Long) resultSet.get(1));
+      assertEquals(new Long(3), (Long) resultSet.get(2));
 
       /* index 1 */
       result = accessor.getObject(1);
@@ -112,7 +113,7 @@ public void testCopyFrom() throws Exception {
 
       /* index 2 */
       result = accessor.getObject(2);
-      resultSet = (ArrayList<Long>)result;
+      resultSet = (ArrayList<Long>) result;
       assertEquals(0, resultSet.size());
     }
   }
@@ -130,11 +131,11 @@ public void testSetLastSetUsage() throws Exception {
 
       /* get inner vectors; bitVector and offsetVector */
       List<BufferBacked> innerVectors = listVector.getFieldInnerVectors();
-      BitVector bitVector = (BitVector)innerVectors.get(0);
-      UInt4Vector offsetVector = (UInt4Vector)innerVectors.get(1);
+      BitVector bitVector = (BitVector) innerVectors.get(0);
+      UInt4Vector offsetVector = (UInt4Vector) innerVectors.get(1);
 
       /* get the underlying data vector -- NullableBigIntVector */
-      NullableBigIntVector dataVector = (NullableBigIntVector)listVector.getDataVector();
+      NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector();
 
       /* check current lastSet */
       assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet()));
@@ -220,36 +221,36 @@ public void testSetLastSetUsage() throws Exception {
       assertEquals(Integer.toString(0), Integer.toString(offset));
 
       Object actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(10), (Long)actual);
+      assertEquals(new Long(10), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(11), (Long)actual);
+      assertEquals(new Long(11), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(12), (Long)actual);
+      assertEquals(new Long(12), (Long) actual);
 
       index++;
       offset = offsetAccessor.get(index);
       assertEquals(Integer.toString(3), Integer.toString(offset));
 
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(13), (Long)actual);
+      assertEquals(new Long(13), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(14), (Long)actual);
+      assertEquals(new Long(14), (Long) actual);
 
       index++;
       offset = offsetAccessor.get(index);
       assertEquals(Integer.toString(5), Integer.toString(offset));
 
       actual = valueAccessor.getObject(offsetAccessor.get(index));
-      assertEquals(new Long(15), (Long)actual);
+      assertEquals(new Long(15), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(16), (Long)actual);
+      assertEquals(new Long(16), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(17), (Long)actual);
+      assertEquals(new Long(17), (Long) actual);
 
       index++;
       offset = offsetAccessor.get(index);
@@ -313,10 +314,10 @@ public void testSplitAndTransfer() throws Exception {
       assertEquals(5, listVector.getMutator().getLastSet());
 
       /* get offsetVector */
-      UInt4Vector offsetVector = (UInt4Vector)listVector.getOffsetVector();
+      UInt4Vector offsetVector = (UInt4Vector) listVector.getOffsetVector();
 
       /* get dataVector */
-      NullableBigIntVector dataVector = (NullableBigIntVector)listVector.getDataVector();
+      NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector();
 
       /* check the vector output */
       final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
@@ -332,13 +333,13 @@ public void testSplitAndTransfer() throws Exception {
       assertEquals(Integer.toString(0), Integer.toString(offset));
 
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(10), (Long)actual);
+      assertEquals(new Long(10), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(11), (Long)actual);
+      assertEquals(new Long(11), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(12), (Long)actual);
+      assertEquals(new Long(12), (Long) actual);
 
       /* index 1 */
       index++;
@@ -347,10 +348,10 @@ public void testSplitAndTransfer() throws Exception {
       assertEquals(Integer.toString(3), Integer.toString(offset));
 
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(13), (Long)actual);
+      assertEquals(new Long(13), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(14), (Long)actual);
+      assertEquals(new Long(14), (Long) actual);
 
       /* index 2 */
       index++;
@@ -359,16 +360,16 @@ public void testSplitAndTransfer() throws Exception {
       assertEquals(Integer.toString(5), Integer.toString(offset));
 
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(15), (Long)actual);
+      assertEquals(new Long(15), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(16), (Long)actual);
+      assertEquals(new Long(16), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(17), (Long)actual);
+      assertEquals(new Long(17), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(18), (Long)actual);
+      assertEquals(new Long(18), (Long) actual);
 
       /* index 3 */
       index++;
@@ -377,7 +378,7 @@ public void testSplitAndTransfer() throws Exception {
       assertEquals(Integer.toString(9), Integer.toString(offset));
 
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(19), (Long)actual);
+      assertEquals(new Long(19), (Long) actual);
 
       /* index 4 */
       index++;
@@ -386,16 +387,16 @@ public void testSplitAndTransfer() throws Exception {
       assertEquals(Integer.toString(10), Integer.toString(offset));
 
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(20), (Long)actual);
+      assertEquals(new Long(20), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(21), (Long)actual);
+      assertEquals(new Long(21), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(22), (Long)actual);
+      assertEquals(new Long(22), (Long) actual);
       offset++;
       actual = valueAccessor.getObject(offset);
-      assertEquals(new Long(23), (Long)actual);
+      assertEquals(new Long(23), (Long) actual);
 
       /* index 5 */
       index++;
@@ -423,26 +424,26 @@ public void testSplitAndTransfer() throws Exception {
           transferPair.splitAndTransfer(start, splitLength);
 
           /* get offsetVector of toVector */
-          UInt4Vector offsetVector1 = (UInt4Vector)toVector.getOffsetVector();
+          UInt4Vector offsetVector1 = (UInt4Vector) toVector.getOffsetVector();
           UInt4Vector.Accessor offsetAccessor1 = offsetVector1.getAccessor();
 
           /* get dataVector of toVector */
-          NullableBigIntVector dataVector1 = (NullableBigIntVector)toVector.getDataVector();
+          NullableBigIntVector dataVector1 = (NullableBigIntVector) toVector.getDataVector();
           NullableBigIntVector.Accessor valueAccessor1 = dataVector1.getAccessor();
 
-          for(int i = 0; i < splitLength; i++) {
+          for (int i = 0; i < splitLength; i++) {
             dataLength1 = offsetAccessor.get(start + i + 1) - offsetAccessor.get(start + i);
             dataLength2 = offsetAccessor1.get(i + 1) - offsetAccessor1.get(i);
 
             assertEquals("Different data lengths at index: " + i + " and start: " + start,
-                         dataLength1, dataLength2);
+                dataLength1, dataLength2);
 
             offset1 = offsetAccessor.get(start + i);
             offset2 = offsetAccessor1.get(i);
 
-            for(int j = 0; j < dataLength1; j++) {
+            for (int j = 0; j < dataLength1; j++) {
               assertEquals("Different data at indexes: " + offset1 + " and " + offset2,
-                            valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
+                  valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
 
               offset1++;
               offset2++;
@@ -515,7 +516,7 @@ public void testNestedListVector() throws Exception {
 
       /* get listVector value at index 0 -- the value itself is a listvector */
       Object result = accessor.getObject(0);
-      ArrayList<ArrayList<Long>> resultSet = (ArrayList<ArrayList<Long>>)result;
+      ArrayList<ArrayList<Long>> resultSet = (ArrayList<ArrayList<Long>>) result;
       ArrayList<Long> list;
 
       assertEquals(2, resultSet.size());              /* 2 inner lists at index 0 */
@@ -535,7 +536,7 @@ public void testNestedListVector() throws Exception {
 
        /* get listVector value at index 1 -- the value itself is a listvector */
       result = accessor.getObject(1);
-      resultSet = (ArrayList<ArrayList<Long>>)result;
+      resultSet = (ArrayList<ArrayList<Long>>) result;
 
       assertEquals(3, resultSet.size());              /* 3 inner lists at index 1 */
       assertEquals(1, resultSet.get(0).size());       /* size of first inner list */
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
index 9baebc5a2992c..ba2ebbf05ad0d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
@@ -90,7 +90,7 @@ public void testBitVectorReallocation() {
     // common: value count < MAX_VALUE_ALLOCATION
     try {
       vector.allocateNew(expectedValueCapacity);
-      for (int i=0; i<3;i++) {
+      for (int i = 0; i < 3; i++) {
         vector.reAlloc(); // expand buffer size
       }
       assertEquals(Integer.MAX_VALUE, vector.getValueCapacity());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index a5159242d76f9..a75b196fbcc30 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
@@ -98,7 +99,7 @@ public void testTransfer() throws Exception {
       mutator.setSafe(5, newBitHolder(false));
       mutator.setValueCount(6);
 
-      try(UnionVector destVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      try (UnionVector destVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
         TransferPair pair = srcVector.makeTransferPair(destVector);
 
         // Creating the transfer should transfer the type of the field at least.
@@ -111,7 +112,7 @@ public void testTransfer() throws Exception {
 
         // now check the values are transferred
         assertEquals(srcVector.getAccessor().getValueCount(), destVector.getAccessor().getValueCount());
-        for(int i=0; i<srcVector.getAccessor().getValueCount(); i++) {
+        for (int i = 0; i < srcVector.getAccessor().getValueCount(); i++) {
           assertEquals("Different values at index " + i, srcVector.getAccessor().get(i), destVector.getAccessor().get(i));
         }
       }
@@ -172,18 +173,18 @@ public void testSplitAndTransfer() throws Exception {
       assertEquals(false, sourceAccessor.isNull(9));
       assertEquals(50, sourceAccessor.getObject(9));
 
-      try(UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
 
         final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
         final UnionVector.Accessor toAccessor = toVector.getAccessor();
 
-        final int[][] transferLengths = { {0, 3},
-                                          {3, 1},
-                                          {4, 2},
-                                          {6, 1},
-                                          {7, 1},
-                                          {8, 2}
-                                        };
+        final int[][] transferLengths = {{0, 3},
+            {3, 1},
+            {4, 2},
+            {6, 1},
+            {7, 1},
+            {8, 2}
+        };
 
         for (final int[] transferLength : transferLengths) {
           final int start = transferLength[0];
@@ -194,7 +195,7 @@ public void testSplitAndTransfer() throws Exception {
           /* check the toVector output after doing the splitAndTransfer */
           for (int i = 0; i < length; i++) {
             assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceAccessor.getObject(start + i),
-                         toAccessor.getObject(i));
+                toAccessor.getObject(i));
           }
         }
       }
@@ -264,17 +265,17 @@ public void testSplitAndTransferWithMixedVectors() throws Exception {
       assertEquals(false, sourceAccessor.isNull(9));
       assertEquals(30.5f, sourceAccessor.getObject(9));
 
-      try(UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
 
         final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
         final UnionVector.Accessor toAccessor = toVector.getAccessor();
 
-        final int[][] transferLengths = { {0, 2},
-                                          {2, 1},
-                                          {3, 2},
-                                          {5, 3},
-                                          {8, 2}
-                                        };
+        final int[][] transferLengths = {{0, 2},
+            {2, 1},
+            {3, 2},
+            {5, 3},
+            {8, 2}
+        };
 
         for (final int[] transferLength : transferLengths) {
           final int start = transferLength[0];
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
index b79f2da9210ab..15f511707d628 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -25,7 +26,7 @@
 public class TestUtils {
 
   public static NullableVarCharVector newNullableVarCharVector(String name, BufferAllocator allocator) {
-    return (NullableVarCharVector)FieldType.nullable(new ArrowType.Utf8()).createNewSingleVector(name, allocator, null);
+    return (NullableVarCharVector) FieldType.nullable(new ArrowType.Utf8()).createNewSingleVector(name, allocator, null);
   }
 
   public static <T> T newVector(Class<T> c, String name, ArrowType type, BufferAllocator allocator) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 159d534a31072..72214ed2ed6fb 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
@@ -394,7 +395,7 @@ public void testBitVectorRangeSetAllOnes() {
   }
 
   private void validateRange(int length, int start, int count) {
-    String desc = "[" + start + ", "  + (start + count) + ") ";
+    String desc = "[" + start + ", " + (start + count) + ") ";
     try (BitVector bitVector = new BitVector("bits", allocator)) {
       bitVector.reset();
       bitVector.allocateNew(length);
@@ -489,7 +490,7 @@ public void testFillEmptiesNotOverfill() {
   @Test
   public void testCopyFromWithNulls() {
     try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
-          final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+         final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
       vector.allocateNew();
 
       for (int i = 0; i < 4095; i++) {
@@ -608,15 +609,15 @@ public void testVectorLoadUnload() {
       VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1);
 
       try (
-              ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
-              BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("new vector", 0, Long.MAX_VALUE);
-              VectorSchemaRoot schemaRoot2 = VectorSchemaRoot.create(schema, finalVectorsAllocator);
+          ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
+          BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("new vector", 0, Long.MAX_VALUE);
+          VectorSchemaRoot schemaRoot2 = VectorSchemaRoot.create(schema, finalVectorsAllocator);
       ) {
 
         VectorLoader vectorLoader = new VectorLoader(schemaRoot2);
         vectorLoader.load(recordBatch);
 
-        NullableVarCharVector vector2 = (NullableVarCharVector)schemaRoot2.getVector(fieldName);
+        NullableVarCharVector vector2 = (NullableVarCharVector) schemaRoot2.getVector(fieldName);
         NullableVarCharVector.Mutator mutator2 = vector2.getMutator();
 
         /*
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index a5d5527539322..4ac7536c017db 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index f3694659a8f51..7facf73f511da 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector;
 
 import static java.util.Arrays.asList;
@@ -176,8 +177,9 @@ public void testUnloadLoadAddPadding() throws IOException {
 
   /**
    * The validity buffer can be empty if:
-   *  - all values are defined
-   *  - all values are null
+   * - all values are defined
+   * - all values are null
+   *
    * @throws IOException
    */
   @Test
@@ -185,7 +187,7 @@ public void testLoadEmptyValidityBuffer() throws IOException {
     Schema schema = new Schema(asList(
         new Field("intDefined", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList()),
         new Field("intNull", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList())
-                                     ));
+    ));
     int count = 10;
     ArrowBuf validity = allocator.buffer(10).slice(0, 0);
     ArrowBuf[] values = new ArrowBuf[2];
@@ -208,8 +210,8 @@ public void testLoadEmptyValidityBuffer() throws IOException {
 
       vectorLoader.load(recordBatch);
 
-      NullableIntVector intDefinedVector = (NullableIntVector)newRoot.getVector("intDefined");
-      NullableIntVector intNullVector = (NullableIntVector)newRoot.getVector("intNull");
+      NullableIntVector intDefinedVector = (NullableIntVector) newRoot.getVector("intDefined");
+      NullableIntVector intNullVector = (NullableIntVector) newRoot.getVector("intNull");
       for (int i = 0; i < count; i++) {
         assertFalse("#" + i, intDefinedVector.getAccessor().isNull(i));
         assertEquals("#" + i, i, intDefinedVector.getAccessor().get(i));
@@ -244,20 +246,20 @@ public void testLoadEmptyValidityBuffer() throws IOException {
   public void testUnloadLoadDuplicates() throws IOException {
     int count = 10;
     Schema schema = new Schema(asList(
-      new Field("duplicate", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList()),
-      new Field("duplicate", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList())
+        new Field("duplicate", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList()),
+        new Field("duplicate", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList())
     ));
 
     try (
-      BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
     ) {
       List<FieldVector> sources = new ArrayList<>();
-      for (Field field: schema.getFields()) {
+      for (Field field : schema.getFields()) {
         FieldVector vector = field.createVector(originalVectorsAllocator);
         vector.allocateNew();
         sources.add(vector);
         NullableIntVector.Mutator mutator = (NullableIntVector.Mutator) vector.getMutator();
-        for (int i =  0; i < count; i++) {
+        for (int i = 0; i < count; i++) {
           mutator.set(i, i);
         }
         mutator.setValueCount(count);
@@ -266,8 +268,8 @@ public void testUnloadLoadDuplicates() throws IOException {
       try (VectorSchemaRoot root = new VectorSchemaRoot(schema.getFields(), sources, count)) {
         VectorUnloader vectorUnloader = new VectorUnloader(root);
         try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();
-                BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-                VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator);) {
+             BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
+             VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator);) {
           // load it
           VectorLoader vectorLoader = new VectorLoader(newRoot);
           vectorLoader.load(recordBatch);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index e826fa53b0977..97efb7d5a6d30 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.impl;
 
 import static org.junit.Assert.assertEquals;
@@ -67,7 +68,7 @@ public void testPromoteToUnion() throws Exception {
       writer.setPosition(1);
       writer.bit("A").writeBit(1);
 
-      writer.decimal("dec", 10,10);
+      writer.decimal("dec", 10, 10);
 
       writer.setPosition(2);
       writer.integer("A").writeInt(10);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 5a9c80dc124a2..f81cd557a9d8f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.complex.writer;
 
 import static org.junit.Assert.*;
@@ -217,7 +218,7 @@ public void listScalarType() {
     for (int i = 0; i < COUNT; i++) {
       listWriter.startList();
       for (int j = 0; j < i % 7; j++) {
-        if (j%2 == 0) {
+        if (j % 2 == 0) {
           listWriter.writeInt(j);
         } else {
           IntHolder holder = new IntHolder();
@@ -259,7 +260,7 @@ public void listScalarTypeNullable() {
       listReader.setPosition(i);
       if (i % 2 == 0) {
         assertTrue("index is set: " + i, listReader.isSet());
-        assertEquals("correct length at: " + i, i % 7, ((List<?>)listReader.readObject()).size());
+        assertEquals("correct length at: " + i, i % 7, ((List<?>) listReader.readObject()).size());
       } else {
         assertFalse("index is not set: " + i, listReader.isSet());
         assertNull("index is not set: " + i, listReader.readObject());
@@ -529,10 +530,10 @@ public void promotableWriterSchema() {
 
   private Set<String> getFieldNames(List<Field> fields) {
     Set<String> fieldNames = new HashSet<>();
-    for (Field field: fields) {
+    for (Field field : fields) {
       fieldNames.add(field.getName());
       if (!field.getChildren().isEmpty()) {
-        for (String name: getFieldNames(field.getChildren())) {
+        for (String name : getFieldNames(field.getChildren())) {
           fieldNames.add(field.getName() + "::" + name);
         }
       }
@@ -698,7 +699,7 @@ private void checkTimestampField(Field field, String name) {
 
   private void checkTimestampTZField(Field field, String name, String tz) {
     checkTimestampField(field, name);
-    Assert.assertEquals(tz, ((Timestamp)field.getType()).getTimezone());
+    Assert.assertEquals(tz, ((Timestamp) field.getType()).getTimezone());
   }
 
   @Test
@@ -824,13 +825,13 @@ public void complexCopierWithList() {
     TransferPair tp = mapVector.getTransferPair(allocator);
     tp.splitAndTransfer(0, 1);
     MapVector toMapVector = (MapVector) tp.getTo();
-    JsonStringHashMap<?,?> toMapValue = (JsonStringHashMap<?,?>) toMapVector.getAccessor().getObject(0);
+    JsonStringHashMap<?, ?> toMapValue = (JsonStringHashMap<?, ?>) toMapVector.getAccessor().getObject(0);
     JsonStringArrayList<?> object = (JsonStringArrayList<?>) toMapValue.get("list");
     assertEquals(1, object.get(0));
     assertEquals(2, object.get(1));
-    JsonStringHashMap<?,?> innerMap = (JsonStringHashMap<?,?>) object.get(2);
+    JsonStringHashMap<?, ?> innerMap = (JsonStringHashMap<?, ?>) object.get(2);
     assertEquals(1, innerMap.get("a"));
-    innerMap = (JsonStringHashMap<?,?>) object.get(3);
+    innerMap = (JsonStringHashMap<?, ?>) object.get(3);
     assertEquals(2, innerMap.get("a"));
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 3f717cbc18b6e..732bd98b7c61c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.nio.charset.StandardCharsets;
@@ -155,7 +156,7 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) {
         Assert.assertNull(intVal);
       }
       Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
-      Assert.assertEquals(i % 3, ((List<?>)root.getVector("list").getAccessor().getObject(i)).size());
+      Assert.assertEquals(i % 3, ((List<?>) root.getVector("list").getAccessor().getObject(i)).size());
       NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
       FieldReader mapReader = root.getVector("map").getReader();
       mapReader.setPosition(i);
@@ -198,11 +199,11 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
     Assert.assertEquals(count, root.getRowCount());
     printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
-      long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i);
+      long dateVal = ((NullableDateMilliVector) root.getVector("date")).getAccessor().get(i);
       LocalDateTime dt = makeDateTimeFromCount(i);
       LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
       Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal);
-      long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i);
+      long timeVal = ((NullableTimeMilliVector) root.getVector("time")).getAccessor().get(i);
       Assert.assertEquals(dt.getMillisOfDay(), timeVal);
       Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
       Assert.assertEquals(dt, timestampMilliVal);
@@ -450,20 +451,20 @@ public void validateUnionData(int count, VectorSchemaRoot root) {
     for (int i = 0; i < count; i++) {
       unionReader.setPosition(i);
       switch (i % 4) {
-      case 0:
-        Assert.assertEquals(i, unionReader.readInteger().intValue());
-        break;
-      case 1:
-        Assert.assertEquals(i, unionReader.readLong().longValue());
-        break;
-      case 2:
-        Assert.assertEquals(i % 3, unionReader.size());
-        break;
-      case 3:
-        NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
-        unionReader.reader("timestamp").read(h);
-        Assert.assertEquals(i, h.value);
-        break;
+        case 0:
+          Assert.assertEquals(i, unionReader.readInteger().intValue());
+          break;
+        case 1:
+          Assert.assertEquals(i, unionReader.readLong().longValue());
+          break;
+        case 2:
+          Assert.assertEquals(i % 3, unionReader.size());
+          break;
+        case 3:
+          NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
+          unionReader.reader("timestamp").read(h);
+          Assert.assertEquals(i, h.value);
+          break;
       }
     }
   }
@@ -483,28 +484,28 @@ public void writeUnionData(int count, NullableMapVector parent) {
     MapWriter mapWriter = rootWriter.map("union");
     for (int i = 0; i < count; i++) {
       switch (i % 4) {
-      case 0:
-        intWriter.setPosition(i);
-        intWriter.writeInt(i);
-        break;
-      case 1:
-        bigIntWriter.setPosition(i);
-        bigIntWriter.writeBigInt(i);
-        break;
-      case 2:
-        listWriter.setPosition(i);
-        listWriter.startList();
-        for (int j = 0; j < i % 3; j++) {
-          listWriter.varChar().writeVarChar(0, 3, varchar);
-        }
-        listWriter.endList();
-        break;
-      case 3:
-        mapWriter.setPosition(i);
-        mapWriter.start();
-        mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
-        mapWriter.end();
-        break;
+        case 0:
+          intWriter.setPosition(i);
+          intWriter.writeInt(i);
+          break;
+        case 1:
+          bigIntWriter.setPosition(i);
+          bigIntWriter.writeBigInt(i);
+          break;
+        case 2:
+          listWriter.setPosition(i);
+          listWriter.startList();
+          for (int j = 0; j < i % 3; j++) {
+            listWriter.varChar().writeVarChar(0, 3, varchar);
+          }
+          listWriter.endList();
+          break;
+        case 3:
+          mapWriter.setPosition(i);
+          mapWriter.start();
+          mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
+          mapWriter.end();
+          break;
       }
     }
     writer.setValueCount(count);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 90fb5d252d68d..c483ba7de91c6 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import java.io.ByteArrayInputStream;
@@ -107,19 +108,19 @@ public void testWriteRead() throws IOException {
     // read
     try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
          FileInputStream fileInputStream = new FileInputStream(file);
-         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator){
-            @Override
-            protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator allocator) throws IOException {
-              ArrowMessage message = super.readMessage(in, allocator);
-              if (message != null) {
-                ArrowRecordBatch batch = (ArrowRecordBatch) message;
-                List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
-                for (ArrowBuffer arrowBuffer : buffersLayout) {
-                  Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
-                }
-              }
-              return message;
-            }
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator) {
+           @Override
+           protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator allocator) throws IOException {
+             ArrowMessage message = super.readMessage(in, allocator);
+             if (message != null) {
+               ArrowRecordBatch batch = (ArrowRecordBatch) message;
+               List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
+               for (ArrowBuffer arrowBuffer : buffersLayout) {
+                 Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+               }
+             }
+             return message;
+           }
          }) {
       Schema schema = arrowReader.getVectorSchemaRoot().getSchema();
       LOGGER.debug("reading schema: " + schema);
@@ -134,7 +135,7 @@ protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator alloc
     // Read from stream.
     try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
          ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
-         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator){
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator) {
            @Override
            protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
              ArrowMessage message = super.readMessage(in, allocator);
@@ -203,17 +204,17 @@ public void testWriteReadComplex() throws IOException {
   public void testWriteReadMultipleRBs() throws IOException {
     File file = new File("target/mytest_multiple.arrow");
     ByteArrayOutputStream stream = new ByteArrayOutputStream();
-    int[] counts = { 10, 5 };
+    int[] counts = {10, 5};
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
          MapVector parent = MapVector.empty("parent", originalVectorAllocator);
-         FileOutputStream fileOutputStream = new FileOutputStream(file)){
+         FileOutputStream fileOutputStream = new FileOutputStream(file)) {
       writeData(counts[0], parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
 
-      try(ArrowFileWriter fileWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel());
-          ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, null, stream)) {
+      try (ArrowFileWriter fileWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel());
+           ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, null, stream)) {
         fileWriter.start();
         streamWriter.start();
 
@@ -318,7 +319,7 @@ public void testWriteReadTiny() throws IOException {
       root.getFieldVectors().get(0).allocateNew();
       NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator();
       for (int i = 0; i < 16; i++) {
-        mutator.set(i, i < 8 ? 1 : 0, (byte)(i + 1));
+        mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
       }
       mutator.setValueCount(16);
       root.setRowCount(16);
@@ -367,7 +368,7 @@ private void validateTinyData(VectorSchemaRoot root) {
     NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
     for (int i = 0; i < 16; i++) {
       if (i < 8) {
-        Assert.assertEquals((byte)(i + 1), vector.getAccessor().get(i));
+        Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i));
       } else {
         Assert.assertTrue(vector.getAccessor().isNull(i));
       }
@@ -384,7 +385,7 @@ public void testWriteReadMetadata() throws IOException {
     childFields.add(new Field("float-child", new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null, metadata(2)), null));
     childFields.add(new Field("int-child", new FieldType(false, new ArrowType.Int(32, true), null, metadata(3)), null));
     childFields.add(new Field("list-child", new FieldType(true, ArrowType.List.INSTANCE, null, metadata(4)),
-                              ImmutableList.of(new Field("l1", FieldType.nullable(new ArrowType.Int(16 ,true)), null))));
+        ImmutableList.of(new Field("l1", FieldType.nullable(new ArrowType.Int(16, true)), null))));
     Field field = new Field("meta", new FieldType(true, ArrowType.Struct.INSTANCE, null, metadata(0)), childFields);
     Map<String, String> metadata = new HashMap<>();
     metadata.put("s1", "v1");
@@ -425,7 +426,7 @@ public void testWriteReadMetadata() throws IOException {
       Assert.assertEquals(originalSchema.getCustomMetadata(), schema.getCustomMetadata());
       Field top = schema.getFields().get(0);
       Assert.assertEquals(metadata(0), top.getMetadata());
-      for (int i = 0; i < 4; i ++) {
+      for (int i = 0; i < 4; i++) {
         Assert.assertEquals(metadata(i + 1), top.getChildren().get(i).getMetadata());
       }
     }
@@ -441,7 +442,7 @@ public void testWriteReadMetadata() throws IOException {
       Assert.assertEquals(originalSchema.getCustomMetadata(), schema.getCustomMetadata());
       Field top = schema.getFields().get(0);
       Assert.assertEquals(metadata(0), top.getMetadata());
-      for (int i = 0; i < 4; i ++) {
+      for (int i = 0; i < 4; i++) {
         Assert.assertEquals(metadata(i + 1), top.getChildren().get(i).getMetadata());
       }
     }
@@ -475,7 +476,7 @@ public void testWriteReadDictionary() throws IOException {
       }
 
       // Need to close dictionary vectors
-      for (long id: provider.getDictionaryIds()) {
+      for (long id : provider.getDictionaryIds()) {
         provider.lookup(id).getVector().close();
       }
     }
@@ -534,7 +535,7 @@ public void testWriteReadNestedDictionary() throws IOException {
       }
 
       // Need to close dictionary vectors
-      for (long id: provider.getDictionaryIds()) {
+      for (long id : provider.getDictionaryIds()) {
         provider.lookup(id).getVector().close();
       }
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
index 3014e64b4eea4..4612465323130 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import static java.util.Arrays.asList;
@@ -40,7 +41,7 @@ public class TestArrowFooter {
   public void test() {
     Schema schema = new Schema(asList(
         new Field("a", FieldType.nullable(new ArrowType.Int(8, true)), Collections.<Field>emptyList())
-        ));
+    ));
     ArrowFooter footer = new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), Collections.<ArrowBlock>emptyList());
     ArrowFooter newFooter = roundTrip(footer);
     assertEquals(footer, newFooter);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index 55629d5107c86..65332aa2c7de2 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import static java.nio.channels.Channels.newChannel;
@@ -77,15 +78,15 @@ public void test() throws IOException {
     FieldVector vector = TestUtils.newVector(FieldVector.class, "testField", type, allocator);
     vector.initializeChildrenFromFields(schema.getFields().get(0).getChildren());
 
-    byte[] validity = new byte[] { (byte) 255, 0};
+    byte[] validity = new byte[] {(byte) 255, 0};
     // second half is "undefined"
-    byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+    byte[] values = new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
 
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     try (VectorSchemaRoot root = new VectorSchemaRoot(schema.getFields(), asList(vector), 16);
          ArrowFileWriter writer = new ArrowFileWriter(root, null, newChannel(out))) {
       ArrowBuf validityb = buf(validity);
-      ArrowBuf valuesb =  buf(values);
+      ArrowBuf valuesb = buf(values);
       writer.writeRecordBatch(new ArrowRecordBatch(16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)));
     }
 
@@ -113,7 +114,7 @@ public void test() throws IOException {
       // Read just the header. This demonstrates being able to read without need to
       // deserialize the buffer.
       ByteBuffer headerBuffer = ByteBuffer.allocate(recordBatches.get(0).getMetadataLength());
-      headerBuffer.put(byteArray, (int)recordBatches.get(0).getOffset(), headerBuffer.capacity());
+      headerBuffer.put(byteArray, (int) recordBatches.get(0).getOffset(), headerBuffer.capacity());
       headerBuffer.position(4);
       Message messageFB = Message.getRootAsMessage(headerBuffer);
       RecordBatch recordBatchFB = (RecordBatch) messageFB.header(new RecordBatch());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
index 7e9afd381c181..e2efabef0095b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import static java.util.Arrays.asList;
@@ -71,7 +72,7 @@ public void testReadWrite() throws IOException {
       root.getFieldVectors().get(0).allocateNew();
       NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator();
       for (int i = 0; i < 16; i++) {
-        mutator.set(i, i < 8 ? 1 : 0, (byte)(i + 1));
+        mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
       }
       mutator.setValueCount(16);
       root.setRowCount(16);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
index 20d4482da7c98..a19c3795fd5bb 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file;
 
 import static org.junit.Assert.assertEquals;
@@ -65,7 +66,7 @@ public void run() {
           // Send a changing batch id first
           mutator.set(0, j);
           for (int i = 1; i < 16; i++) {
-            mutator.set(i, i < 8 ? 1 : 0, (byte)(i + 1));
+            mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
           }
           mutator.setValueCount(16);
           root.setRowCount(16);
@@ -80,7 +81,9 @@ public void run() {
       }
     }
 
-    public long bytesWritten() { return writer.bytesWritten(); }
+    public long bytesWritten() {
+      return writer.bytesWritten();
+    }
   }
 
   private final class ReaderThread extends Thread {
@@ -104,6 +107,7 @@ protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) th
           }
           return message;
         }
+
         @Override
         public boolean loadNextBatch() throws IOException {
           if (!super.loadNextBatch()) {
@@ -113,10 +117,10 @@ public boolean loadNextBatch() throws IOException {
             VectorSchemaRoot root = getVectorSchemaRoot();
             Assert.assertEquals(16, root.getRowCount());
             NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
-            Assert.assertEquals((byte)(batchesRead - 1), vector.getAccessor().get(0));
+            Assert.assertEquals((byte) (batchesRead - 1), vector.getAccessor().get(0));
             for (int i = 1; i < 16; i++) {
               if (i < 8) {
-                Assert.assertEquals((byte)(i + 1), vector.getAccessor().get(i));
+                Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i));
               } else {
                 Assert.assertTrue(vector.getAccessor().isNull(i));
               }
@@ -143,8 +147,13 @@ public void run() {
       }
     }
 
-    public int getBatchesRead() { return batchesRead; }
-    public long bytesRead() { return reader.bytesRead(); }
+    public int getBatchesRead() {
+      return batchesRead;
+    }
+
+    public long bytesRead() {
+      return reader.bytesRead();
+    }
   }
 
   // Starts up a producer and consumer thread to read/write batches.
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 311cada194eaf..24b2138386da1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.file.json;
 
 import java.io.File;
@@ -53,7 +54,7 @@ public void testWriteReadComplexJSON() throws IOException {
     // read
     try (
         BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        ) {
+    ) {
       JsonFileReader reader = new JsonFileReader(file, readerAllocator);
       Schema schema = reader.start();
       LOGGER.debug("reading schema: " + schema);
@@ -109,7 +110,7 @@ public void testWriteReadUnionJSON() throws IOException {
     try (
         BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
         BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-        ) {
+    ) {
       JsonFileReader reader = new JsonFileReader(file, readerAllocator);
       Schema schema = reader.start();
       LOGGER.debug("reading schema: " + schema);
@@ -174,7 +175,7 @@ public void testWriteReadDictionaryJSON() throws IOException {
       }
 
       // Need to close dictionary vectors
-      for (long id: provider.getDictionaryIds()) {
+      for (long id : provider.getDictionaryIds()) {
         provider.lookup(id).getVector().close();
       }
     }
@@ -215,7 +216,7 @@ public void testWriteReadNestedDictionaryJSON() throws IOException {
       }
 
       // Need to close dictionary vectors
-      for (long id: provider.getDictionaryIds()) {
+      for (long id : provider.getDictionaryIds()) {
         provider.lookup(id).getVector().close();
       }
     }
@@ -240,7 +241,7 @@ public void testWriteReadNestedDictionaryJSON() throws IOException {
   public void testSetStructLength() throws IOException {
     File file = new File("../../integration/data/struct_example.json");
     try (
-            BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
     ) {
       JsonFileReader reader = new JsonFileReader(file, readerAllocator);
       Schema schema = reader.start();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 62c21f7da0db6..f98aeac8c8196 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.pojo;
 
 import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE;
@@ -92,15 +93,15 @@ public void nestedSchema() {
     childrenBuilder.add(new Field("child3", FieldType.nullable(new Struct()), ImmutableList.<Field>of(
         new Field("child3.1", FieldType.nullable(Utf8.INSTANCE), null),
         new Field("child3.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of())
-        )));
+    )));
     childrenBuilder.add(new Field("child4", FieldType.nullable(new List()), ImmutableList.<Field>of(
         new Field("child4.1", FieldType.nullable(Utf8.INSTANCE), null)
-        )));
-    childrenBuilder.add(new Field("child5", FieldType.nullable(new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } )), ImmutableList.<Field>of(
+    )));
+    childrenBuilder.add(new Field("child5", FieldType.nullable(new Union(UnionMode.Sparse, new int[] {MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal()})), ImmutableList.<Field>of(
         new Field("child5.1", FieldType.nullable(new Timestamp(TimeUnit.MILLISECOND, null)), null),
         new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of()),
         new Field("child5.3", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null)
-        )));
+    )));
     Schema initialSchema = new Schema(childrenBuilder.build());
     run(initialSchema);
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
index 9678423c0fbbe..f968768f5e67d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.stream;
 
 import static java.util.Arrays.asList;
@@ -107,13 +108,13 @@ public void testdeSerializeRecordBatchLongMetaData() throws IOException {
 
   @Test
   public void testSerializeRecordBatch() throws IOException {
-    byte[] validity = new byte[] { (byte)255, 0};
+    byte[] validity = new byte[] {(byte) 255, 0};
     // second half is "undefined"
-    byte[] values = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+    byte[] values = new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
 
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
     ArrowBuf validityb = buf(alloc, validity);
-    ArrowBuf valuesb =  buf(alloc, values);
+    ArrowBuf valuesb = buf(alloc, values);
 
     ArrowRecordBatch batch = new ArrowRecordBatch(
         16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb));
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 84cc10787f7b0..43b0907720f83 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.arrow.vector.types.pojo;
 
 import static java.util.Arrays.asList;
@@ -66,7 +67,7 @@ public void testComplex() throws IOException {
         field("g", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
         field("h", new Timestamp(TimeUnit.MICROSECOND, null)),
         field("i", new Interval(IntervalUnit.DAY_TIME))
-        ));
+    ));
     roundTrip(schema);
     assertEquals(
         "Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>, e: List<Date(MILLISECOND)>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND, UTC), h: Timestamp(MICROSECOND, null), i: Interval(DAY_TIME)>",
@@ -95,7 +96,7 @@ public void testAll() throws IOException {
         field("q", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
         field("r", new Timestamp(TimeUnit.MICROSECOND, null)),
         field("s", new Interval(IntervalUnit.DAY_TIME))
-        ));
+    ));
     roundTrip(schema);
   }
 
@@ -103,7 +104,7 @@ public void testAll() throws IOException {
   public void testUnion() throws IOException {
     Schema schema = new Schema(asList(
         field("d", new Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new Null()))
-        ));
+    ));
     roundTrip(schema);
     contains(schema, "Sparse");
   }
@@ -113,7 +114,7 @@ public void testDate() throws IOException {
     Schema schema = new Schema(asList(
         field("a", new Date(DateUnit.DAY)),
         field("b", new Date(DateUnit.MILLISECOND))
-        ));
+    ));
     roundTrip(schema);
     assertEquals(
         "Schema<a: Date(DAY), b: Date(MILLISECOND)>",
@@ -123,15 +124,15 @@ public void testDate() throws IOException {
   @Test
   public void testTime() throws IOException {
     Schema schema = new Schema(asList(
-            field("a", new Time(TimeUnit.SECOND, 32)),
-            field("b", new Time(TimeUnit.MILLISECOND, 32)),
-            field("c", new Time(TimeUnit.MICROSECOND, 64)),
-            field("d", new Time(TimeUnit.NANOSECOND, 64))
+        field("a", new Time(TimeUnit.SECOND, 32)),
+        field("b", new Time(TimeUnit.MILLISECOND, 32)),
+        field("c", new Time(TimeUnit.MICROSECOND, 64)),
+        field("d", new Time(TimeUnit.NANOSECOND, 64))
     ));
     roundTrip(schema);
     assertEquals(
-            "Schema<a: Time(SECOND, 32), b: Time(MILLISECOND, 32), c: Time(MICROSECOND, 64), d: Time(NANOSECOND, 64)>",
-            schema.toString());
+        "Schema<a: Time(SECOND, 32), b: Time(MILLISECOND, 32), c: Time(MICROSECOND, 64), d: Time(NANOSECOND, 64)>",
+        schema.toString());
   }
 
   @Test
@@ -145,7 +146,7 @@ public void testTS() throws IOException {
         field("f", new Timestamp(TimeUnit.MILLISECOND, null)),
         field("g", new Timestamp(TimeUnit.MICROSECOND, null)),
         field("h", new Timestamp(TimeUnit.NANOSECOND, null))
-        ));
+    ));
     roundTrip(schema);
     assertEquals(
         "Schema<a: Timestamp(SECOND, UTC), b: Timestamp(MILLISECOND, UTC), c: Timestamp(MICROSECOND, UTC), d: Timestamp(NANOSECOND, UTC), e: Timestamp(SECOND, null), f: Timestamp(MILLISECOND, null), g: Timestamp(MICROSECOND, null), h: Timestamp(NANOSECOND, null)>",
@@ -157,7 +158,7 @@ public void testInterval() throws IOException {
     Schema schema = new Schema(asList(
         field("a", new Interval(IntervalUnit.YEAR_MONTH)),
         field("b", new Interval(IntervalUnit.DAY_TIME))
-        ));
+    ));
     roundTrip(schema);
     contains(schema, "YEAR_MONTH", "DAY_TIME");
   }
@@ -168,7 +169,7 @@ public void testFP() throws IOException {
         field("a", new FloatingPoint(FloatingPointPrecision.HALF)),
         field("b", new FloatingPoint(FloatingPointPrecision.SINGLE)),
         field("c", new FloatingPoint(FloatingPointPrecision.DOUBLE))
-        ));
+    ));
     roundTrip(schema);
     contains(schema, "HALF", "SINGLE", "DOUBLE");
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java
index 7cf638e57d849..95b08099c204d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.apache.arrow.vector.util;
 
 import static org.apache.arrow.vector.util.Validator.equalEnough;
@@ -26,32 +27,32 @@
 
 public class TestValidator {
 
-    @Test
-    public void testFloatComp() {
-        assertTrue(equalEnough(912.4140000000002F, 912.414F));
-        assertTrue(equalEnough(912.4140000000002D, 912.414D));
-        assertTrue(equalEnough(912.414F, 912.4140000000002F));
-        assertTrue(equalEnough(912.414D, 912.4140000000002D));
-        assertFalse(equalEnough(912.414D, 912.4140001D));
-        assertFalse(equalEnough(null, 912.414D));
-        assertTrue(equalEnough((Float)null, null));
-        assertTrue(equalEnough((Double)null, null));
-        assertFalse(equalEnough(912.414D, null));
-        assertFalse(equalEnough(Double.MAX_VALUE, Double.MIN_VALUE));
-        assertFalse(equalEnough(Double.MIN_VALUE, Double.MAX_VALUE));
-        assertTrue(equalEnough(Double.MAX_VALUE, Double.MAX_VALUE));
-        assertTrue(equalEnough(Double.MIN_VALUE, Double.MIN_VALUE));
-        assertTrue(equalEnough(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY));
-        assertFalse(equalEnough(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY));
-        assertTrue(equalEnough(Double.NaN, Double.NaN));
-        assertFalse(equalEnough(1.0, Double.NaN));
-        assertFalse(equalEnough(Float.MAX_VALUE, Float.MIN_VALUE));
-        assertFalse(equalEnough(Float.MIN_VALUE, Float.MAX_VALUE));
-        assertTrue(equalEnough(Float.MAX_VALUE, Float.MAX_VALUE));
-        assertTrue(equalEnough(Float.MIN_VALUE, Float.MIN_VALUE));
-        assertTrue(equalEnough(Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY));
-        assertFalse(equalEnough(Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY));
-        assertTrue(equalEnough(Float.NaN, Float.NaN));
-        assertFalse(equalEnough(1.0F, Float.NaN));
-    }
+  @Test
+  public void testFloatComp() {
+    assertTrue(equalEnough(912.4140000000002F, 912.414F));
+    assertTrue(equalEnough(912.4140000000002D, 912.414D));
+    assertTrue(equalEnough(912.414F, 912.4140000000002F));
+    assertTrue(equalEnough(912.414D, 912.4140000000002D));
+    assertFalse(equalEnough(912.414D, 912.4140001D));
+    assertFalse(equalEnough(null, 912.414D));
+    assertTrue(equalEnough((Float) null, null));
+    assertTrue(equalEnough((Double) null, null));
+    assertFalse(equalEnough(912.414D, null));
+    assertFalse(equalEnough(Double.MAX_VALUE, Double.MIN_VALUE));
+    assertFalse(equalEnough(Double.MIN_VALUE, Double.MAX_VALUE));
+    assertTrue(equalEnough(Double.MAX_VALUE, Double.MAX_VALUE));
+    assertTrue(equalEnough(Double.MIN_VALUE, Double.MIN_VALUE));
+    assertTrue(equalEnough(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY));
+    assertFalse(equalEnough(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY));
+    assertTrue(equalEnough(Double.NaN, Double.NaN));
+    assertFalse(equalEnough(1.0, Double.NaN));
+    assertFalse(equalEnough(Float.MAX_VALUE, Float.MIN_VALUE));
+    assertFalse(equalEnough(Float.MIN_VALUE, Float.MAX_VALUE));
+    assertTrue(equalEnough(Float.MAX_VALUE, Float.MAX_VALUE));
+    assertTrue(equalEnough(Float.MIN_VALUE, Float.MIN_VALUE));
+    assertTrue(equalEnough(Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY));
+    assertFalse(equalEnough(Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY));
+    assertTrue(equalEnough(Float.NaN, Float.NaN));
+    assertFalse(equalEnough(1.0F, Float.NaN));
+  }
 }

From 0b91cadfe1cdf5964f275c7d8324332e9e49b0d9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 7 Aug 2017 19:10:04 -0400
Subject: [PATCH 0909/1644] ARROW-622: [Python] Add coerce_timestamps option to
 parquet.write_table, deprecate timestamps_to_ms argument

Requires PARQUET-1078 https://github.com/apache/parquet-cpp/pull/380

cc @xhochy @fjetter @cpcloud, could you have a look. This needs to go into 0.6.0

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #944 from wesm/ARROW-622 and squashes the following commits:

3a21dfe5 [Wes McKinney] Add test to exhaust more paths of coerce_timestamps, error handling
45bbf5b2 [Wes McKinney] Add coerce_timestamps to write_metadata
172a9e1a [Wes McKinney] Implement coerce_timestamps option
---
 python/pyarrow/_parquet.pxd          |  4 +-
 python/pyarrow/_parquet.pyx          | 16 +++++-
 python/pyarrow/pandas_compat.py      |  4 ++
 python/pyarrow/parquet.py            | 22 +++++++--
 python/pyarrow/tests/test_parquet.py | 73 +++++++++++++++++++---------
 5 files changed, 91 insertions(+), 28 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 7299e19b81906..ced654915e57e 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -21,7 +21,8 @@ from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
                                         CTable, CMemoryPool,
                                         CKeyValueMetadata,
-                                        RandomAccessFile, OutputStream)
+                                        RandomAccessFile, OutputStream,
+                                        TimeUnit)
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
@@ -266,5 +267,6 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
             Builder()
             Builder* disable_deprecated_int96_timestamps()
             Builder* enable_deprecated_int96_timestamps()
+            Builder* coerce_timestamps(TimeUnit unit)
             shared_ptr[ArrowWriterProperties] build()
         c_bool support_deprecated_int96_timestamps()
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 65ca468772710..f3b7875045904 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -547,6 +547,7 @@ cdef class ParquetWriter:
     cdef readonly:
         object use_dictionary
         object use_deprecated_int96_timestamps
+        object coerce_timestamps
         object compression
         object version
         int row_group_size
@@ -554,7 +555,8 @@ cdef class ParquetWriter:
     def __cinit__(self, where, Schema schema, use_dictionary=None,
                   compression=None, version=None,
                   MemoryPool memory_pool=None,
-                  use_deprecated_int96_timestamps=False):
+                  use_deprecated_int96_timestamps=False,
+                  coerce_timestamps=None):
         cdef:
             shared_ptr[FileOutputStream] filestream
             shared_ptr[WriterProperties] properties
@@ -574,6 +576,7 @@ cdef class ParquetWriter:
         self.compression = compression
         self.version = version
         self.use_deprecated_int96_timestamps = use_deprecated_int96_timestamps
+        self.coerce_timestamps = coerce_timestamps
 
         cdef WriterProperties.Builder properties_builder
         self._set_version(&properties_builder)
@@ -583,6 +586,7 @@ cdef class ParquetWriter:
 
         cdef ArrowWriterProperties.Builder arrow_properties_builder
         self._set_int96_support(&arrow_properties_builder)
+        self._set_coerce_timestamps(&arrow_properties_builder)
         arrow_properties = arrow_properties_builder.build()
 
         pool = maybe_unbox_memory_pool(memory_pool)
@@ -598,6 +602,16 @@ cdef class ParquetWriter:
         else:
             props.disable_deprecated_int96_timestamps()
 
+    cdef int _set_coerce_timestamps(
+            self, ArrowWriterProperties.Builder* props) except -1:
+        if self.coerce_timestamps == 'ms':
+            props.coerce_timestamps(TimeUnit_MILLI)
+        elif self.coerce_timestamps == 'us':
+            props.coerce_timestamps(TimeUnit_MICRO)
+        elif self.coerce_timestamps is not None:
+            raise ValueError('Invalid value for coerce_timestamps: {0}'
+                             .format(self.coerce_timestamps))
+
     cdef void _set_version(self, WriterProperties.Builder* props):
         if self.version is not None:
             if self.version == "1.0":
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 2881588208eb1..ddd562238e38a 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -243,6 +243,10 @@ def dataframe_to_arrays(df, timestamps_to_ms, schema, preserve_index):
 
 
 def maybe_coerce_datetime64(values, dtype, type_, timestamps_to_ms=False):
+    if timestamps_to_ms:
+        import warnings
+        warnings.warn('timestamps_to_ms=True is deprecated', FutureWarning)
+
     from pyarrow.compat import DatetimeTZDtype
 
     if values.dtype.type != np.datetime64:
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 6d39a2354f653..89dbf83ee3523 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -757,7 +757,8 @@ def read_pandas(source, columns=None, nthreads=1, metadata=None):
 
 def write_table(table, where, row_group_size=None, version='1.0',
                 use_dictionary=True, compression='snappy',
-                use_deprecated_int96_timestamps=False, **kwargs):
+                use_deprecated_int96_timestamps=False,
+                coerce_timestamps=None, **kwargs):
     """
     Write a Table to Parquet format
 
@@ -773,6 +774,11 @@ def write_table(table, where, row_group_size=None, version='1.0',
     use_dictionary : bool or list
         Specify if we should use dictionary encoding in general or only for
         some columns.
+    use_deprecated_int96_timestamps : boolean, default False
+        Write nanosecond resolution timestamps to INT96 Parquet format
+    coerce_timestamps : string, default None
+        Cast timestamps a particular resolution.
+        Valid values: {None, 'ms', 'us'}
     compression : str or dict
         Specify the compression codec, either on a general basis or per-column.
     """
@@ -781,7 +787,8 @@ def write_table(table, where, row_group_size=None, version='1.0',
         use_dictionary=use_dictionary,
         compression=compression,
         version=version,
-        use_deprecated_int96_timestamps=use_deprecated_int96_timestamps)
+        use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
+        coerce_timestamps=coerce_timestamps)
 
     writer = None
     try:
@@ -801,7 +808,8 @@ def write_table(table, where, row_group_size=None, version='1.0',
 
 
 def write_metadata(schema, where, version='1.0',
-                   use_deprecated_int96_timestamps=False):
+                   use_deprecated_int96_timestamps=False,
+                   coerce_timestamps=None):
     """
     Write metadata-only Parquet file from schema
 
@@ -811,10 +819,16 @@ def write_metadata(schema, where, version='1.0',
     where: string or pyarrow.io.NativeFile
     version : {"1.0", "2.0"}, default "1.0"
         The Parquet format version, defaults to 1.0
+    use_deprecated_int96_timestamps : boolean, default False
+        Write nanosecond resolution timestamps to INT96 Parquet format
+    coerce_timestamps : string, default None
+        Cast timestamps a particular resolution.
+        Valid values: {None, 'ms', 'us'}
     """
     options = dict(
         version=version,
-        use_deprecated_int96_timestamps=use_deprecated_int96_timestamps
+        use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
+        coerce_timestamps=coerce_timestamps
     )
     writer = ParquetWriter(where, schema, **options)
     writer.close()
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index ab3b26cd4e0f1..9a570b9d5dab4 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -100,10 +100,11 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
     df = alltypes_sample(size=10000)
 
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    arrow_table = pa.Table.from_pandas(df)
     assert b'pandas' in arrow_table.schema.metadata
 
-    _write_table(arrow_table, filename.strpath, version="2.0")
+    _write_table(arrow_table, filename.strpath, version="2.0",
+                 coerce_timestamps='ms')
     table_read = pq.read_pandas(filename.strpath)
     assert b'pandas' in table_read.schema.metadata
 
@@ -120,10 +121,11 @@ def test_pandas_parquet_custom_metadata(tmpdir):
     df = alltypes_sample(size=10000)
 
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    arrow_table = pa.Table.from_pandas(df)
     assert b'pandas' in arrow_table.schema.metadata
 
-    _write_table(arrow_table, filename.strpath, version="2.0")
+    _write_table(arrow_table, filename.strpath, version="2.0",
+                 coerce_timestamps='ms')
 
     md = pq.read_metadata(filename.strpath).metadata
     assert b'pandas' in md
@@ -139,13 +141,12 @@ def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tmpdir):
     df = alltypes_sample(size=10000)
 
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = pa.Table.from_pandas(
-        df, timestamps_to_ms=True, preserve_index=False
-    )
+    arrow_table = pa.Table.from_pandas(df, preserve_index=False)
     js = json.loads(arrow_table.schema.metadata[b'pandas'].decode('utf8'))
     assert not js['index_columns']
 
-    _write_table(arrow_table, filename.strpath, version="2.0")
+    _write_table(arrow_table, filename.strpath, version="2.0",
+                 coerce_timestamps='ms')
     table_read = pq.read_pandas(filename.strpath)
 
     js = json.loads(table_read.schema.metadata[b'pandas'].decode('utf8'))
@@ -340,10 +341,11 @@ def test_pandas_parquet_configuration_options(tmpdir):
 def make_sample_file(df):
     import pyarrow.parquet as pq
 
-    a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    a_table = pa.Table.from_pandas(df)
 
     buf = io.BytesIO()
-    _write_table(a_table, buf, compression='SNAPPY', version='2.0')
+    _write_table(a_table, buf, compression='SNAPPY', version='2.0',
+                 coerce_timestamps='ms')
 
     buf.seek(0)
     return pq.ParquetFile(buf)
@@ -418,22 +420,47 @@ def test_column_of_arrays(tmpdir):
     df, schema = dataframe_with_arrays()
 
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True,
-                                       schema=schema)
-    _write_table(arrow_table, filename.strpath, version="2.0")
+    arrow_table = pa.Table.from_pandas(df, schema=schema)
+    _write_table(arrow_table, filename.strpath, version="2.0",
+                 coerce_timestamps='ms')
     table_read = _read_table(filename.strpath)
     df_read = table_read.to_pandas()
     tm.assert_frame_equal(df, df_read)
 
 
+@parquet
+def test_coerce_timestamps(tmpdir):
+    # ARROW-622
+    df, schema = dataframe_with_arrays()
+
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = pa.Table.from_pandas(df, schema=schema)
+
+    _write_table(arrow_table, filename.strpath, version="2.0",
+                 coerce_timestamps='us')
+    table_read = _read_table(filename.strpath)
+    df_read = table_read.to_pandas()
+
+    df_expected = df.copy()
+    for i, x in enumerate(df_expected['datetime64']):
+        if isinstance(x, np.ndarray):
+            df_expected['datetime64'][i] = x.astype('M8[us]')
+
+    tm.assert_frame_equal(df_expected, df_read)
+
+    with pytest.raises(ValueError):
+        _write_table(arrow_table, filename.strpath, version="2.0",
+                     coerce_timestamps='unknown')
+
+
 @parquet
 def test_column_of_lists(tmpdir):
     df, schema = dataframe_with_lists()
 
     filename = tmpdir.join('pandas_rountrip.parquet')
-    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True,
-                                       schema=schema)
-    _write_table(arrow_table, filename.strpath, version="2.0")
+    arrow_table = pa.Table.from_pandas(df, schema=schema)
+    _write_table(arrow_table, filename.strpath, version="2.0",
+                 coerce_timestamps='ms')
     table_read = _read_table(filename.strpath)
     df_read = table_read.to_pandas()
     tm.assert_frame_equal(df, df_read)
@@ -469,12 +496,14 @@ def test_date_time_types():
 
     t7 = pa.timestamp('ns')
     start = pd.Timestamp('2001-01-01').value
-    data7 = np.array([start, start + 1, start + 2], dtype='int64')
+    data7 = np.array([start, start + 1000, start + 2000],
+                     dtype='int64')
     a7 = pa.Array.from_pandas(data7, type=t7)
 
     t7_us = pa.timestamp('us')
     start = pd.Timestamp('2001-01-01').value
-    data7_us = np.array([start, start + 1, start + 2], dtype='int64') // 1000
+    data7_us = np.array([start, start + 1000, start + 2000],
+                        dtype='int64') // 1000
     a7_us = pa.Array.from_pandas(data7_us, type=t7_us)
 
     table = pa.Table.from_arrays([a1, a2, a3, a4, a5, a6, a7],
@@ -547,7 +576,7 @@ def _check_roundtrip(table, expected=None, **params):
 def test_multithreaded_read():
     df = alltypes_sample(size=10000)
 
-    table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    table = pa.Table.from_pandas(df)
 
     buf = io.BytesIO()
     _write_table(table, buf, compression='SNAPPY', version='2.0')
@@ -585,7 +614,7 @@ def test_pass_separate_metadata():
     # ARROW-471
     df = alltypes_sample(size=10000)
 
-    a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    a_table = pa.Table.from_pandas(df)
 
     buf = io.BytesIO()
     _write_table(a_table, buf, compression='snappy', version='2.0')
@@ -608,7 +637,7 @@ def test_read_single_row_group():
     N, K = 10000, 4
     df = alltypes_sample(size=N)
 
-    a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    a_table = pa.Table.from_pandas(df)
 
     buf = io.BytesIO()
     _write_table(a_table, buf, row_group_size=N / K,
@@ -631,7 +660,7 @@ def test_read_single_row_group_with_column_subset():
 
     N, K = 10000, 4
     df = alltypes_sample(size=N)
-    a_table = pa.Table.from_pandas(df, timestamps_to_ms=True)
+    a_table = pa.Table.from_pandas(df)
 
     buf = io.BytesIO()
     _write_table(a_table, buf, row_group_size=N / K,

From 2015198f1b6fcd8a0219d81bccc5a2f34fa66d34 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Mon, 7 Aug 2017 19:13:21 -0400
Subject: [PATCH 0910/1644] =?UTF-8?q?ARROW-1263:=20[C++]=20Get=20CPU=20inf?=
 =?UTF-8?q?o=20on=20Windows;=20Resolve=20patching=20whitespac=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…es issue on cmake build.

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #943 from MaxRis/ARROW-1263 and squashes the following commits:

14ed7c05 [Max Risuhin] ARROW-1263: [C++] Get CPU info on Windows; Resolve patching whitespaces issue on cmake build.
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake |  4 +-
 cpp/src/arrow/util/cpu-info.cc              | 45 +++++++++++++++++++++
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index ae48e8d2fb979..a888e92392db6 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -698,7 +698,7 @@ if (ARROW_WITH_LZ4)
     if (MSVC)
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
       set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
-      set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_wholeprogramoptimization_param.patch)
+      set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_wholeprogramoptimization_param.patch)
     else()
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
       set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh)
@@ -742,7 +742,7 @@ if (ARROW_WITH_ZSTD)
     if (MSVC)
       set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib")
       set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
-      set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_wholeprogramoptimization_param.patch)
+      set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_wholeprogramoptimization_param.patch)
     else()
       set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a")
       set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh)
diff --git a/cpp/src/arrow/util/cpu-info.cc b/cpp/src/arrow/util/cpu-info.cc
index b0667cb33ada4..d0a9a14fc60c0 100644
--- a/cpp/src/arrow/util/cpu-info.cc
+++ b/cpp/src/arrow/util/cpu-info.cc
@@ -31,7 +31,11 @@
 #endif
 
 #ifdef _WIN32
+#include <intrin.h>
 #include <windows.h>
+#include <array>
+#include <bitset>
+
 #endif
 
 #include <boost/algorithm/string.hpp>
@@ -132,6 +136,46 @@ bool RetrieveCacheSize(int64_t* cache_sizes) {
   }
   return true;
 }
+
+bool RetrieveCPUInfo(int64_t* hardware_flags, std::string* model_name) {
+  if (!hardware_flags || !model_name) {
+    return false;
+  }
+  const int register_ECX_id = 1;
+  int highest_valid_id = 0;
+  int highest_extended_valid_id = 0;
+  std::bitset<32> features_ECX;
+  std::array<int, 4> cpu_info;
+
+  // Get highest valid id
+  __cpuid(cpu_info.data(), 0);
+  highest_valid_id = cpu_info[0];
+
+  if (highest_valid_id <= register_ECX_id) return false;
+
+  __cpuidex(cpu_info.data(), register_ECX_id, 0);
+  features_ECX = cpu_info[2];
+
+  // Get highest extended id
+  __cpuid(cpu_info.data(), 0x80000000);
+  highest_extended_valid_id = cpu_info[0];
+
+  // Retrieve CPU model name
+  if (highest_extended_valid_id >= 0x80000004) {
+    model_name->clear();
+    for (int i = 0x80000002; i <= 0x80000004; ++i) {
+      __cpuidex(cpu_info.data(), i, 0);
+      *model_name +=
+          std::string(reinterpret_cast<char*>(cpu_info.data()), sizeof(cpu_info));
+    }
+  }
+
+  if (features_ECX[9]) *hardware_flags |= CpuInfo::SSSE3;
+  if (features_ECX[19]) *hardware_flags |= CpuInfo::SSE4_1;
+  if (features_ECX[20]) *hardware_flags |= CpuInfo::SSE4_2;
+  if (features_ECX[23]) *hardware_flags |= CpuInfo::POPCNT;
+  return true;
+}
 #endif
 
 void CpuInfo::Init() {
@@ -203,6 +247,7 @@ void CpuInfo::Init() {
   if (!RetrieveCacheSize(cache_sizes_)) {
     SetDefaultCacheSize();
   }
+  RetrieveCPUInfo(&hardware_flags_, &model_name_);
 #else
   SetDefaultCacheSize();
 #endif

From 02ab74841d1a2f3f15aeab20fc4acbc5d737047d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 7 Aug 2017 22:06:28 -0400
Subject: [PATCH 0911/1644] ARROW-1336: [C++] Add arrow::schema factory
 function, simply some awkward constructors

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #948 from wesm/ARROW-1336 and squashes the following commits:

a901b375 [Wes McKinney] Add arrow::schema factory function, simply awkward constructors with initializer list
---
 cpp/src/arrow/ipc/feather-test.cc        |  2 +-
 cpp/src/arrow/ipc/ipc-json-test.cc       |  4 +--
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  4 +--
 cpp/src/arrow/ipc/json-internal.cc       |  3 +-
 cpp/src/arrow/ipc/metadata.cc            |  2 +-
 cpp/src/arrow/ipc/test-common.h          | 36 ++++++++++++------------
 cpp/src/arrow/python/builtin_convert.cc  |  4 +--
 cpp/src/arrow/table-test.cc              | 27 +++++++++---------
 cpp/src/arrow/type-test.cc               | 25 +++++++---------
 cpp/src/arrow/type.cc                    | 14 +++++++++
 cpp/src/arrow/type.h                     | 33 ++++++++++++++++++++++
 11 files changed, 97 insertions(+), 57 deletions(-)

diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index b76b518788b91..e74a60dd48925 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -354,7 +354,7 @@ TEST_F(TestTableWriter, TimeTypes) {
   auto f1 = field("f1", time32(TimeUnit::MILLI));
   auto f2 = field("f2", timestamp(TimeUnit::NANO));
   auto f3 = field("f3", timestamp(TimeUnit::SECOND, "US/Los_Angeles"));
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2, f3}));
+  auto schema = ::arrow::schema({f0, f1, f2, f3});
 
   std::vector<int64_t> values_vec = {0, 1, 2, 3, 4, 5, 6};
   std::shared_ptr<Array> values;
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 1d5a6997ae920..ddb2e37288e46 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -247,8 +247,8 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
   auto v2_type = int32();
   auto v3_type = utf8();
 
-  std::shared_ptr<Schema> schema(
-      new Schema({field("f1", v1_type), field("f2", v2_type), field("f3", v3_type)}));
+  auto schema =
+      ::arrow::schema({field("f1", v1_type), field("f2", v2_type), field("f3", v3_type)});
 
   std::unique_ptr<JsonWriter> writer;
   ASSERT_OK(JsonWriter::Open(schema, &writer));
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index a6246c96f2d9a..045296163ea11 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -303,7 +303,7 @@ TEST_P(TestIpcRoundTrip, ZeroLengthArrays) {
 TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) {
   auto CheckArray = [this](const std::shared_ptr<Array>& array) {
     auto f0 = field("f0", array->type());
-    auto schema = std::shared_ptr<Schema>(new Schema({f0}));
+    auto schema = ::arrow::schema({f0});
     RecordBatch batch(schema, array->length(), {array});
     auto sliced_batch = batch.Slice(0, 5);
 
@@ -421,7 +421,7 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
 
     auto f0 = field("f0", type);
 
-    *schema = std::shared_ptr<Schema>(new Schema({f0}));
+    *schema = ::arrow::schema({f0});
 
     std::vector<std::shared_ptr<Array>> arrays = {array};
     *batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 49fb6ac7ce30f..bc2b0d18e72c7 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -129,8 +129,7 @@ class SchemaWriter {
     writer_->Key("data");
 
     // Make a dummy record batch. A bit tedious as we have to make a schema
-    auto schema = std::shared_ptr<Schema>(
-        new Schema({arrow::field("dictionary", dictionary->type())}));
+    auto schema = ::arrow::schema({arrow::field("dictionary", dictionary->type())});
     RecordBatch batch(schema, dictionary->length(), {dictionary});
     RETURN_NOT_OK(WriteRecordBatch(batch, writer_));
     writer_->EndObject();
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index d764e203e7552..faf01a568483a 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -1035,7 +1035,7 @@ Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_mem
     }
   }
 
-  *out = std::make_shared<Schema>(fields, metadata);
+  *out = ::arrow::schema(std::move(fields), metadata);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index ed33e6e95b13a..c1e79d43cc61d 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -179,7 +179,7 @@ Status MakeBooleanBatchSized(const int length, std::shared_ptr<RecordBatch>* out
   // Make the schema
   auto f0 = field("f0", boolean());
   auto f1 = field("f1", boolean());
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+  auto schema = ::arrow::schema({f0, f1});
 
   std::shared_ptr<Array> a0, a1;
   RETURN_NOT_OK(MakeRandomBooleanArray(length, true, &a0));
@@ -196,7 +196,7 @@ Status MakeIntBatchSized(int length, std::shared_ptr<RecordBatch>* out) {
   // Make the schema
   auto f0 = field("f0", int32());
   auto f1 = field("f1", int32());
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+  auto schema = ::arrow::schema({f0, f1});
 
   // Example data
   std::shared_ptr<Array> a0, a1;
@@ -237,7 +237,7 @@ Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
   auto binary_type = binary();
   auto f0 = field("f0", string_type);
   auto f1 = field("f1", binary_type);
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+  auto schema = ::arrow::schema({f0, f1});
 
   std::shared_ptr<Array> a0, a1;
   MemoryPool* pool = default_memory_pool();
@@ -259,7 +259,7 @@ Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
 Status MakeNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
   const int64_t length = 500;
   auto f0 = field("f0", null());
-  std::shared_ptr<Schema> schema(new Schema({f0}));
+  auto schema = ::arrow::schema({f0});
   std::shared_ptr<Array> a0 = std::make_shared<NullArray>(length);
   out->reset(new RecordBatch(schema, length, {a0}));
   return Status::OK();
@@ -270,7 +270,7 @@ Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out) {
   auto f0 = field("f0", kListInt32);
   auto f1 = field("f1", kListListInt32);
   auto f2 = field("f2", int32());
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+  auto schema = ::arrow::schema({f0, f1, f2});
 
   // Example data
 
@@ -293,7 +293,7 @@ Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out) {
   auto f0 = field("f0", kListInt32);
   auto f1 = field("f1", kListListInt32);
   auto f2 = field("f2", int32());
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+  auto schema = ::arrow::schema({f0, f1, f2});
 
   // Example data
   MemoryPool* pool = default_memory_pool();
@@ -313,7 +313,7 @@ Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
   auto f0 = field("f0", kListInt32);
   auto f1 = field("f1", kListListInt32);
   auto f2 = field("f2", int32());
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+  auto schema = ::arrow::schema({f0, f1, f2});
 
   // Example data
   MemoryPool* pool = default_memory_pool();
@@ -345,7 +345,7 @@ Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
   }
 
   auto f0 = field("f0", type);
-  std::shared_ptr<Schema> schema(new Schema({f0}));
+  auto schema = ::arrow::schema({f0});
   std::vector<std::shared_ptr<Array>> arrays = {array};
   out->reset(new RecordBatch(schema, batch_length, arrays));
   return Status::OK();
@@ -364,7 +364,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
       {list_schema->field(0), list_schema->field(1), list_schema->field(2)}));
   auto f0 = field("non_null_struct", type);
   auto f1 = field("null_struct", type);
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+  auto schema = ::arrow::schema({f0, f1});
 
   // construct individual nullable/non-nullable struct arrays
   std::shared_ptr<Array> no_nulls(new StructArray(type, list_batch->num_rows(), columns));
@@ -397,7 +397,7 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   auto f1 = field("sparse", sparse_type);
   auto f2 = field("dense", dense_type);
 
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+  auto schema = ::arrow::schema({f0, f1, f2});
 
   // Create data
   std::vector<std::shared_ptr<Array>> sparse_children(2);
@@ -520,9 +520,9 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
   auto a4 = std::make_shared<DictionaryArray>(f4_type, indices4);
 
   // construct batch
-  std::shared_ptr<Schema> schema(new Schema(
+  auto schema = ::arrow::schema(
       {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type),
-       field("list of encoded string", f3_type), field("encoded list<int8>", f4_type)}));
+       field("list of encoded string", f3_type), field("encoded list<int8>", f4_type)});
 
   std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2, a3, a4};
 
@@ -560,8 +560,8 @@ Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out) {
   auto a2 = std::make_shared<DictionaryArray>(f2_type, indices2);
 
   // construct batch
-  std::shared_ptr<Schema> schema(new Schema(
-      {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type)}));
+  auto schema = ::arrow::schema(
+      {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type)});
 
   std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2};
   out->reset(new RecordBatch(schema, length, arrays));
@@ -572,7 +572,7 @@ Status MakeDates(std::shared_ptr<RecordBatch>* out) {
   std::vector<bool> is_valid = {true, true, true, false, true, true, true};
   auto f0 = field("f0", date32());
   auto f1 = field("f1", date64());
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+  auto schema = ::arrow::schema({f0, f1});
 
   std::vector<int32_t> date32_values = {0, 1, 2, 3, 4, 5, 6};
   std::shared_ptr<Array> date32_array;
@@ -594,7 +594,7 @@ Status MakeTimestamps(std::shared_ptr<RecordBatch>* out) {
   auto f0 = field("f0", timestamp(TimeUnit::MILLI));
   auto f1 = field("f1", timestamp(TimeUnit::NANO, "America/New_York"));
   auto f2 = field("f2", timestamp(TimeUnit::SECOND));
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
+  auto schema = ::arrow::schema({f0, f1, f2});
 
   std::vector<int64_t> ts_values = {1489269000000, 1489270000000, 1489271000000,
                                     1489272000000, 1489272000000, 1489273000000};
@@ -615,7 +615,7 @@ Status MakeTimes(std::shared_ptr<RecordBatch>* out) {
   auto f1 = field("f1", time64(TimeUnit::NANO));
   auto f2 = field("f2", time32(TimeUnit::SECOND));
   auto f3 = field("f3", time64(TimeUnit::NANO));
-  std::shared_ptr<Schema> schema(new Schema({f0, f1, f2, f3}));
+  auto schema = ::arrow::schema({f0, f1, f2, f3});
 
   std::vector<int32_t> t32_values = {1489269000, 1489270000, 1489271000,
                                      1489272000, 1489272000, 1489273000};
@@ -649,7 +649,7 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
   std::vector<bool> is_valid = {true, true, true, false};
   auto f0 = field("f0", fixed_size_binary(4));
   auto f1 = field("f1", fixed_size_binary(0));
-  std::shared_ptr<Schema> schema(new Schema({f0, f1}));
+  auto schema = ::arrow::schema({f0, f1});
 
   std::shared_ptr<Array> a1, a2;
 
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 218fe2925fd86..b693b3e6b4d76 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -534,8 +534,8 @@ class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter>
     if (obj == Py_None) {
       return typed_builder_->AppendNull();
     } else if (PyBytes_Check(obj)) {
-      tmp.reset(PyUnicode_FromStringAndSize(PyBytes_AS_STRING(obj),
-                                            PyBytes_GET_SIZE(obj)));
+      tmp.reset(
+          PyUnicode_FromStringAndSize(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)));
       RETURN_IF_PYERROR();
       bytes_obj = obj;
     } else if (!PyUnicode_Check(obj)) {
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 8dba8c052e922..1ba7a2f95798c 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -214,7 +214,7 @@ class TestTable : public TestBase {
 };
 
 TEST_F(TestTable, EmptySchema) {
-  auto empty_schema = shared_ptr<Schema>(new Schema({}));
+  auto empty_schema = ::arrow::schema({});
   table_.reset(new Table(empty_schema, columns_));
   ASSERT_OK(table_->ValidateColumns());
   ASSERT_EQ(0, table_->num_rows());
@@ -373,18 +373,17 @@ TEST_F(TestTable, RemoveColumn) {
   std::shared_ptr<Table> result;
   ASSERT_OK(table.RemoveColumn(0, &result));
 
-  auto ex_schema =
-      std::shared_ptr<Schema>(new Schema({schema_->field(1), schema_->field(2)}));
+  auto ex_schema = ::arrow::schema({schema_->field(1), schema_->field(2)});
   std::vector<std::shared_ptr<Column>> ex_columns = {table.column(1), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.RemoveColumn(1, &result));
-  ex_schema = std::shared_ptr<Schema>(new Schema({schema_->field(0), schema_->field(2)}));
+  ex_schema = ::arrow::schema({schema_->field(0), schema_->field(2)});
   ex_columns = {table.column(0), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.RemoveColumn(2, &result));
-  ex_schema = std::shared_ptr<Schema>(new Schema({schema_->field(0), schema_->field(1)}));
+  ex_schema = ::arrow::schema({schema_->field(0), schema_->field(1)});
   ex_columns = {table.column(0), table.column(1)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 }
@@ -410,27 +409,27 @@ TEST_F(TestTable, AddColumn) {
 
   // Add column 0 in different places
   ASSERT_OK(table.AddColumn(0, columns_[0], &result));
-  auto ex_schema = std::shared_ptr<Schema>(new Schema(
-      {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)}));
+  auto ex_schema = ::arrow::schema(
+      {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)});
   std::vector<std::shared_ptr<Column>> ex_columns = {table.column(0), table.column(0),
                                                      table.column(1), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.AddColumn(1, columns_[0], &result));
-  ex_schema = std::shared_ptr<Schema>(new Schema(
-      {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)}));
+  ex_schema = ::arrow::schema(
+      {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)});
   ex_columns = {table.column(0), table.column(0), table.column(1), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.AddColumn(2, columns_[0], &result));
-  ex_schema = std::shared_ptr<Schema>(new Schema(
-      {schema_->field(0), schema_->field(1), schema_->field(0), schema_->field(2)}));
+  ex_schema = ::arrow::schema(
+      {schema_->field(0), schema_->field(1), schema_->field(0), schema_->field(2)});
   ex_columns = {table.column(0), table.column(1), table.column(0), table.column(2)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 
   ASSERT_OK(table.AddColumn(3, columns_[0], &result));
-  ex_schema = std::shared_ptr<Schema>(new Schema(
-      {schema_->field(0), schema_->field(1), schema_->field(2), schema_->field(0)}));
+  ex_schema = ::arrow::schema(
+      {schema_->field(0), schema_->field(1), schema_->field(2), schema_->field(0)});
   ex_columns = {table.column(0), table.column(1), table.column(2), table.column(0)};
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 }
@@ -470,7 +469,7 @@ TEST_F(TestRecordBatch, Validate) {
   auto f1 = field("f1", uint8());
   auto f2 = field("f2", int16());
 
-  auto schema = std::shared_ptr<Schema>(new Schema({f0, f1, f2}));
+  auto schema = ::arrow::schema({f0, f1, f2});
 
   auto a0 = MakePrimitive<Int32Array>(length);
   auto a1 = MakePrimitive<UInt8Array>(length);
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 6b86b4d2f1024..4ac5c85d480ed 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -97,15 +97,14 @@ TEST_F(TestSchema, Basics) {
 
   auto f2 = field("f2", utf8());
 
-  vector<shared_ptr<Field>> fields = {f0, f1, f2};
-  auto schema = std::make_shared<Schema>(fields);
+  auto schema = ::arrow::schema({f0, f1, f2});
 
   ASSERT_EQ(3, schema->num_fields());
   ASSERT_TRUE(f0->Equals(schema->field(0)));
   ASSERT_TRUE(f1->Equals(schema->field(1)));
   ASSERT_TRUE(f2->Equals(schema->field(2)));
 
-  auto schema2 = std::make_shared<Schema>(fields);
+  auto schema2 = ::arrow::schema({f0, f1, f2});
 
   vector<shared_ptr<Field>> fields3 = {f0, f1_optional, f2};
   auto schema3 = std::make_shared<Schema>(fields3);
@@ -119,8 +118,7 @@ TEST_F(TestSchema, ToString) {
   auto f2 = field("f2", utf8());
   auto f3 = field("f3", list(int16()));
 
-  vector<shared_ptr<Field>> fields = {f0, f1, f2, f3};
-  auto schema = std::make_shared<Schema>(fields);
+  auto schema = ::arrow::schema({f0, f1, f2, f3});
 
   std::string result = schema->ToString();
   std::string expected = R"(f0: int32
@@ -137,8 +135,7 @@ TEST_F(TestSchema, GetFieldByName) {
   auto f2 = field("f2", utf8());
   auto f3 = field("f3", list(int16()));
 
-  vector<shared_ptr<Field>> fields = {f0, f1, f2, f3};
-  auto schema = std::make_shared<Schema>(fields);
+  auto schema = ::arrow::schema({f0, f1, f2, f3});
 
   std::shared_ptr<Field> result;
 
@@ -158,13 +155,12 @@ TEST_F(TestSchema, GetFieldIndex) {
   auto f2 = field("f2", utf8());
   auto f3 = field("f3", list(int16()));
 
-  vector<shared_ptr<Field>> fields = {f0, f1, f2, f3};
-  auto schema = std::make_shared<Schema>(fields);
+  auto schema = ::arrow::schema({f0, f1, f2, f3});
 
-  ASSERT_EQ(0, schema->GetFieldIndex(fields[0]->name()));
-  ASSERT_EQ(1, schema->GetFieldIndex(fields[1]->name()));
-  ASSERT_EQ(2, schema->GetFieldIndex(fields[2]->name()));
-  ASSERT_EQ(3, schema->GetFieldIndex(fields[3]->name()));
+  ASSERT_EQ(0, schema->GetFieldIndex(f0->name()));
+  ASSERT_EQ(1, schema->GetFieldIndex(f1->name()));
+  ASSERT_EQ(2, schema->GetFieldIndex(f2->name()));
+  ASSERT_EQ(3, schema->GetFieldIndex(f3->name()));
   ASSERT_EQ(-1, schema->GetFieldIndex("not-found"));
 }
 
@@ -172,10 +168,9 @@ TEST_F(TestSchema, TestMetadataConstruction) {
   auto f0 = field("f0", int32());
   auto f1 = field("f1", uint8(), false);
   auto f2 = field("f2", utf8());
-  vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto metadata = std::shared_ptr<KeyValueMetadata>(
       new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
-  auto schema = std::make_shared<Schema>(fields, metadata);
+  auto schema = ::arrow::schema({f0, f1, f2}, metadata);
   ASSERT_TRUE(metadata->Equals(*schema->metadata()));
 }
 
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index edf4d33b23f39..4443e8d8a455b 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -252,6 +252,10 @@ Schema::Schema(const std::vector<std::shared_ptr<Field>>& fields,
                const std::shared_ptr<const KeyValueMetadata>& metadata)
     : fields_(fields), metadata_(metadata) {}
 
+Schema::Schema(std::vector<std::shared_ptr<Field>>&& fields,
+               const std::shared_ptr<const KeyValueMetadata>& metadata)
+    : fields_(std::move(fields)), metadata_(metadata) {}
+
 bool Schema::Equals(const Schema& other) const {
   if (this == &other) {
     return true;
@@ -343,6 +347,16 @@ std::string Schema::ToString() const {
   return buffer.str();
 }
 
+std::shared_ptr<Schema> schema(const std::vector<std::shared_ptr<Field>>& fields,
+                               const std::shared_ptr<const KeyValueMetadata>& metadata) {
+  return std::make_shared<Schema>(fields, metadata);
+}
+
+std::shared_ptr<Schema> schema(std::vector<std::shared_ptr<Field>>&& fields,
+                               const std::shared_ptr<const KeyValueMetadata>& metadata) {
+  return std::make_shared<Schema>(std::move(fields), metadata);
+}
+
 // ----------------------------------------------------------------------
 // Visitors and factory functions
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index b28fe9229b2ae..4917ebb481368 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -708,6 +708,10 @@ class ARROW_EXPORT Schema {
  public:
   explicit Schema(const std::vector<std::shared_ptr<Field>>& fields,
                   const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+
+  explicit Schema(std::vector<std::shared_ptr<Field>>&& fields,
+                  const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+
   virtual ~Schema() = default;
 
   /// Returns true if all of the schema fields are equal
@@ -772,27 +776,56 @@ std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit);
 std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit,
                                                  const std::string& timezone);
 
+/// \brief Create an instance of 32-bit time type
 /// Unit can be either SECOND or MILLI
 std::shared_ptr<DataType> ARROW_EXPORT time32(TimeUnit::type unit);
 
+/// \brief Create an instance of 64-bit time type
 /// Unit can be either MICRO or NANO
 std::shared_ptr<DataType> ARROW_EXPORT time64(TimeUnit::type unit);
 
+/// \brief Create an instance of Struct type
 std::shared_ptr<DataType> ARROW_EXPORT
 struct_(const std::vector<std::shared_ptr<Field>>& fields);
 
+/// \brief Create an instance of Union type
 std::shared_ptr<DataType> ARROW_EXPORT
 union_(const std::vector<std::shared_ptr<Field>>& child_fields,
        const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
 
+/// \brief Create an instance of Dictionary type
 std::shared_ptr<DataType> ARROW_EXPORT
 dictionary(const std::shared_ptr<DataType>& index_type,
            const std::shared_ptr<Array>& values, bool ordered = false);
 
+/// \brief Create a Field instance
+///
+/// \param name the field name
+/// \param type the field value type
+/// \param nullable whether the values are nullable, default true
+/// \param metadata any custom key-value metadata, default nullptr
 std::shared_ptr<Field> ARROW_EXPORT field(
     const std::string& name, const std::shared_ptr<DataType>& type, bool nullable = true,
     const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
 
+/// \brief Create a Schema instance
+///
+/// \param fields the schema's fields
+/// \param metadata any custom key-value metadata, default nullptr
+/// \return schema shared_ptr to Schema
+std::shared_ptr<Schema> ARROW_EXPORT
+schema(const std::vector<std::shared_ptr<Field>>& fields,
+       const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+
+/// \brief Create a Schema instance
+///
+/// \param fields the schema's fields (rvalue reference)
+/// \param metadata any custom key-value metadata, default nullptr
+/// \return schema shared_ptr to Schema
+std::shared_ptr<Schema> ARROW_EXPORT
+schema(std::vector<std::shared_ptr<Field>>&& fields,
+       const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+
 // ----------------------------------------------------------------------
 //
 

From 66ab6b2616260977ab9a29bdd59872fb98133d13 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 7 Aug 2017 22:08:12 -0400
Subject: [PATCH 0912/1644] ARROW-1309: [Python] Handle nested lists with all
 None values in Array.from_pandas

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #947 from wesm/ARROW-1309 and squashes the following commits:

dc464922 [Wes McKinney] Expand test case to include ndarray
86039ec2 [Wes McKinney] Bugfix, add multiple nulls at start of array
08687cac [Wes McKinney] NullBuilder, scaffolding
---
 cpp/src/arrow/builder.cc                    | 62 +++++++++++++--------
 cpp/src/arrow/builder.h                     | 13 +++++
 cpp/src/arrow/python/builtin_convert.cc     | 42 ++++++++------
 cpp/src/arrow/python/pandas_to_arrow.cc     | 50 ++++++++++++++---
 cpp/src/arrow/type_fwd.h                    |  1 +
 cpp/src/arrow/type_traits.h                 |  1 +
 python/pyarrow/tests/test_convert_pandas.py | 15 +++++
 7 files changed, 136 insertions(+), 48 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index e3eda2401a02b..889c64db9fdb8 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -177,6 +177,17 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
   length_ = new_length;
 }
 
+// ----------------------------------------------------------------------
+// Null builder
+
+Status NullBuilder::Finish(std::shared_ptr<Array>* out) {
+  *out = std::make_shared<NullArray>(length_);
+  length_ = null_count_ = 0;
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+
 template <typename T>
 Status PrimitiveBuilder<T>::Init(int64_t capacity) {
   RETURN_NOT_OK(ArrayBuilder::Init(capacity));
@@ -1306,26 +1317,30 @@ Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
 Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
                    std::unique_ptr<ArrayBuilder>* out) {
   switch (type->id()) {
-    BUILDER_CASE(UINT8, UInt8Builder);
-    BUILDER_CASE(INT8, Int8Builder);
-    BUILDER_CASE(UINT16, UInt16Builder);
-    BUILDER_CASE(INT16, Int16Builder);
-    BUILDER_CASE(UINT32, UInt32Builder);
-    BUILDER_CASE(INT32, Int32Builder);
-    BUILDER_CASE(UINT64, UInt64Builder);
-    BUILDER_CASE(INT64, Int64Builder);
-    BUILDER_CASE(DATE32, Date32Builder);
-    BUILDER_CASE(DATE64, Date64Builder);
-    BUILDER_CASE(TIME32, Time32Builder);
-    BUILDER_CASE(TIME64, Time64Builder);
-    BUILDER_CASE(TIMESTAMP, TimestampBuilder);
-    BUILDER_CASE(BOOL, BooleanBuilder);
-    BUILDER_CASE(FLOAT, FloatBuilder);
-    BUILDER_CASE(DOUBLE, DoubleBuilder);
-    BUILDER_CASE(STRING, StringBuilder);
-    BUILDER_CASE(BINARY, BinaryBuilder);
-    BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder);
-    BUILDER_CASE(DECIMAL, DecimalBuilder);
+    case Type::NA: {
+      out->reset(new NullBuilder(pool));
+      return Status::OK();
+    }
+      BUILDER_CASE(UINT8, UInt8Builder);
+      BUILDER_CASE(INT8, Int8Builder);
+      BUILDER_CASE(UINT16, UInt16Builder);
+      BUILDER_CASE(INT16, Int16Builder);
+      BUILDER_CASE(UINT32, UInt32Builder);
+      BUILDER_CASE(INT32, Int32Builder);
+      BUILDER_CASE(UINT64, UInt64Builder);
+      BUILDER_CASE(INT64, Int64Builder);
+      BUILDER_CASE(DATE32, Date32Builder);
+      BUILDER_CASE(DATE64, Date64Builder);
+      BUILDER_CASE(TIME32, Time32Builder);
+      BUILDER_CASE(TIME64, Time64Builder);
+      BUILDER_CASE(TIMESTAMP, TimestampBuilder);
+      BUILDER_CASE(BOOL, BooleanBuilder);
+      BUILDER_CASE(FLOAT, FloatBuilder);
+      BUILDER_CASE(DOUBLE, DoubleBuilder);
+      BUILDER_CASE(STRING, StringBuilder);
+      BUILDER_CASE(BINARY, BinaryBuilder);
+      BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder);
+      BUILDER_CASE(DECIMAL, DecimalBuilder);
     case Type::LIST: {
       std::unique_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
@@ -1348,8 +1363,11 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
       return Status::OK();
     }
 
-    default:
-      return Status::NotImplemented(type->ToString());
+    default: {
+      std::stringstream ss;
+      ss << "MakeBuilder: cannot construct builder for type " << type->ToString();
+      return Status::NotImplemented(ss.str());
+    }
   }
 }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index e441179ae7864..b15005f62bc7e 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -162,6 +162,19 @@ class ARROW_EXPORT ArrayBuilder {
   DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
 };
 
+class ARROW_EXPORT NullBuilder : public ArrayBuilder {
+ public:
+  explicit NullBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG) : ArrayBuilder(null(), pool) {}
+
+  Status AppendNull() {
+    ++null_count_;
+    ++length_;
+    return Status::OK();
+  }
+
+  Status Finish(std::shared_ptr<Array>* out) override;
+};
+
 template <typename Type>
 class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
  public:
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index b693b3e6b4d76..ccaf280b0a383 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -155,7 +155,7 @@ static constexpr int MAX_NESTING_LEVELS = 32;
 // SeqVisitor is used to infer the type.
 class SeqVisitor {
  public:
-  SeqVisitor() : max_nesting_level_(0) {
+  SeqVisitor() : max_nesting_level_(0), max_observed_level_(0) {
     memset(nesting_histogram_, 0, MAX_NESTING_LEVELS * sizeof(int));
   }
 
@@ -217,24 +217,13 @@ class SeqVisitor {
       if (num_nesting_levels() > 1) {
         return Status::Invalid("Mixed nesting levels not supported");
         // If the nesting goes deeper than the deepest scalar
-      } else if (max_observed_level() < max_nesting_level_) {
+      } else if (max_observed_level_ < max_nesting_level_) {
         return Status::Invalid("Mixed nesting levels not supported");
       }
     }
     return Status::OK();
   }
 
-  // Returns the deepest level which has scalar elements.
-  int max_observed_level() const {
-    int result = 0;
-    for (int i = 0; i < MAX_NESTING_LEVELS; ++i) {
-      if (nesting_histogram_[i] > 0) {
-        result = i;
-      }
-    }
-    return result;
-  }
-
   // Returns the number of nesting levels which have scalar elements.
   int num_nesting_levels() const {
     int result = 0;
@@ -252,6 +241,8 @@ class SeqVisitor {
   // Track observed
   // Deapest nesting level (irregardless of scalars)
   int max_nesting_level_;
+  int max_observed_level_;
+
   // Number of scalar elements at each nesting level.
   // (TOOD: We really only need to know if a scalar is present, not the count).
   int nesting_histogram_[MAX_NESTING_LEVELS];
@@ -263,13 +254,15 @@ class SeqVisitor {
     } else if (PyDict_Check(item_ref.obj())) {
       return Status::NotImplemented("No type inference for dicts");
     } else {
-      // We permit nulls at any level of nesting
-      if (item_ref.obj() == Py_None) {
-        // TODO
-      } else {
+      // We permit nulls at any level of nesting, but they aren't treated like
+      // other scalar values as far as the checking for mixed nesting structure
+      if (item_ref.obj() != Py_None) {
         ++nesting_histogram_[level];
-        return scalars_.Visit(item_ref.obj());
       }
+      if (level > max_observed_level_) {
+        max_observed_level_ = level;
+      }
+      return scalars_.Visit(item_ref.obj());
     }
     return Status::OK();
   }
@@ -392,6 +385,17 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
   virtual Status AppendItem(const OwnedRef& item) = 0;
 };
 
+class NullConverter : public TypedConverterVisitor<NullBuilder, NullConverter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    if (item.obj() == Py_None) {
+      return typed_builder_->AppendNull();
+    } else {
+      return Status::Invalid("NullConverter: passed non-None value");
+    }
+  }
+};
+
 class BoolConverter : public TypedConverterVisitor<BooleanBuilder, BoolConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
@@ -616,6 +620,8 @@ class DecimalConverter
 // Dynamic constructor for sequence converters
 std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type) {
   switch (type->id()) {
+    case Type::NA:
+      return std::make_shared<NullConverter>();
     case Type::BOOL:
       return std::make_shared<BoolConverter>();
     case Type::INT64:
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 590be223d3f07..060fcb2453800 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -944,10 +944,6 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
     return Status::NotImplemented("mask not supported in object conversions yet");
   }
 
-  if (is_strided()) {
-    return Status::NotImplemented("strided arrays not implemented for lists");
-  }
-
   BuilderT* value_builder = static_cast<BuilderT*>(builder->value_builder());
 
   auto foreach_item = [&](PyObject* object) {
@@ -991,6 +987,47 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
   return LoopPySequence(list, foreach_item);
 }
 
+template <>
+inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
+    const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
+  PyAcquireGIL lock;
+
+  // TODO: mask not supported here
+  if (mask_ != nullptr) {
+    return Status::NotImplemented("mask not supported in object conversions yet");
+  }
+
+  auto value_builder = static_cast<NullBuilder*>(builder->value_builder());
+
+  auto foreach_item = [&](PyObject* object) {
+    if (PandasObjectIsNull(object)) {
+      return builder->AppendNull();
+    } else if (PyArray_Check(object)) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
+      RETURN_NOT_OK(builder->Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+
+      for (int64_t i = 0; i < static_cast<int64_t>(PyArray_SIZE(numpy_array)); ++i) {
+        RETURN_NOT_OK(value_builder->AppendNull());
+      }
+      return Status::OK();
+    } else if (PyList_Check(object)) {
+      RETURN_NOT_OK(builder->Append(true));
+      const Py_ssize_t size = PySequence_Size(object);
+      for (Py_ssize_t i = 0; i < size; ++i) {
+        RETURN_NOT_OK(value_builder->AppendNull());
+      }
+      return Status::OK();
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
+  };
+
+  return LoopPySequence(list, foreach_item);
+}
+
 template <>
 inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
@@ -1003,10 +1040,6 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     return Status::NotImplemented("mask not supported in object conversions yet");
   }
 
-  if (is_strided()) {
-    return Status::NotImplemented("strided arrays not implemented for lists");
-  }
-
   auto value_builder = static_cast<StringBuilder*>(builder->value_builder());
 
   auto foreach_item = [&](PyObject* object) {
@@ -1053,6 +1086,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
 Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type,
                                      ListBuilder* builder, PyObject* list) {
   switch (type->id()) {
+    LIST_CASE(NA, NPY_OBJECT, NullType)
     LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
     LIST_CASE(INT8, NPY_INT8, Int8Type)
     LIST_CASE(UINT16, NPY_UINT16, UInt16Type)
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 99c09bd6b7dca..0d06b6f6cb86e 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -42,6 +42,7 @@ class DictionaryArray;
 
 class NullType;
 class NullArray;
+class NullBuilder;
 
 class BooleanType;
 class BooleanArray;
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 973b0e15c5434..f05eb56718f5f 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -31,6 +31,7 @@ struct TypeTraits {};
 template <>
 struct TypeTraits<NullType> {
   using ArrayType = NullArray;
+  using BuilderType = NullBuilder;
   constexpr static bool is_parameter_free = false;
 };
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 2a51d3283203f..93058fb0a47b4 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -534,6 +534,21 @@ def test_column_of_lists(self):
             field = schema.field_by_name(column)
             self._check_array_roundtrip(df[column], type=field.type)
 
+    def test_nested_lists_all_none(self):
+        data = np.array([[None, None], None], dtype=object)
+
+        arr = pa.Array.from_pandas(data)
+        expected = pa.array(list(data))
+        assert arr.equals(expected)
+        assert arr.type == pa.list_(pa.null())
+
+        data2 = np.array([None, None, [None, None],
+                          np.array([None, None], dtype=object)],
+                         dtype=object)
+        arr = pa.Array.from_pandas(data2)
+        expected = pa.array([None, None, [None, None], [None, None]])
+        assert arr.equals(expected)
+
     def test_threaded_conversion(self):
         df = _alltypes_example()
         self._check_pandas_roundtrip(df, nthreads=2,

From 03dcce44671f355b3d259b913fcabace609a9cd2 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Mon, 7 Aug 2017 22:50:38 -0400
Subject: [PATCH 0913/1644] ARROW-1173: [Plasma] Add blog post describing
 Plasma object store

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #940 from robertnishihara/plasmablogpost and squashes the following commits:

d7230930 [Robert Nishihara] Update blog post date.
48c9c7b9 [Robert Nishihara] Change speedup after improving baseline.
2ae1d66e [Robert Nishihara] Add blog post describing Plasma object store.
---
 ...017-08-08-plasma-in-memory-object-store.md | 150 ++++++++++++++++++
 1 file changed, 150 insertions(+)
 create mode 100644 site/_posts/2017-08-08-plasma-in-memory-object-store.md

diff --git a/site/_posts/2017-08-08-plasma-in-memory-object-store.md b/site/_posts/2017-08-08-plasma-in-memory-object-store.md
new file mode 100644
index 0000000000000..48cfb6613cf73
--- /dev/null
+++ b/site/_posts/2017-08-08-plasma-in-memory-object-store.md
@@ -0,0 +1,150 @@
+---
+layout: post
+title: "Plasma In-Memory Object Store"
+date: "2017-08-08 00:00:00 -0400"
+author: Philipp Moritz and Robert Nishihara
+categories: [application]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+*[Philipp Moritz][1] and [Robert Nishihara][2] are graduate students at UC
+ Berkeley.*
+
+## Plasma: A High-Performance Shared-Memory Object Store
+
+### Motivating Plasma
+
+This blog post presents Plasma, an in-memory object store that is being
+developed as part of Apache Arrow. **Plasma holds immutable objects in shared
+memory so that they can be accessed efficiently by many clients across process
+boundaries.** In light of the trend toward larger and larger multicore machines,
+Plasma enables critical performance optimizations in the big data regime.
+
+Plasma was initially developed as part of [Ray][3], and has recently been moved
+to Apache Arrow in the hopes that it will be broadly useful.
+
+One of the goals of Apache Arrow is to serve as a common data layer enabling
+zero-copy data exchange between multiple frameworks. A key component of this
+vision is the use of off-heap memory management (via Plasma) for storing and
+sharing Arrow-serialized objects between applications.
+
+**Expensive serialization and deserialization as well as data copying are a
+common performance bottleneck in distributed computing.** For example, a
+Python-based execution framework that wishes to distribute computation across
+multiple Python “worker” processes and then aggregate the results in a single
+“driver” process may choose to serialize data using the built-in `pickle`
+library. Assuming one Python process per core, each worker process would have to
+copy and deserialize the data, resulting in excessive memory usage. The driver
+process would then have to deserialize results from each of the workers,
+resulting in a bottleneck.
+
+Using Plasma plus Arrow, the data being operated on would be placed in the
+Plasma store once, and all of the workers would read the data without copying or
+deserializing it (the workers would map the relevant region of memory into their
+own address spaces). The workers would then put the results of their computation
+back into the Plasma store, which the driver could then read and aggregate
+without copying or deserializing the data.
+
+### The Plasma API:
+
+Below we illustrate a subset of the API. The C++ API is documented more fully
+[here][6], and the Python API is documented [here][7].
+
+**Object IDs:** Each object is associated with a string of bytes.
+
+**Creating an object:** Objects are stored in Plasma in two stages. First, the
+object store *creates* the object by allocating a buffer for it. At this point,
+the client can write to the buffer and construct the object within the allocated
+buffer. When the client is done, the client *seals* the buffer making the object
+immutable and making it available to other Plasma clients.
+
+```python
+# Create an object.
+object_id = pyarrow.plasma.ObjectID(20 * b'a')
+object_size = 1000
+buffer = memoryview(client.create(object_id, object_size))
+
+# Write to the buffer.
+for i in range(1000):
+    buffer[i] = 0
+
+# Seal the object making it immutable and available to other clients.
+client.seal(object_id)
+```
+
+**Getting an object:** After an object has been sealed, any client who knows the
+object ID can get the object.
+
+```python
+# Get the object from the store. This blocks until the object has been sealed.
+object_id = pyarrow.plasma.ObjectID(20 * b'a')
+[buff] = client.get([object_id])
+buffer = memoryview(buff)
+```
+
+If the object has not been sealed yet, then the call to `client.get` will block
+until the object has been sealed.
+
+### A sorting application
+
+To illustrate the benefits of Plasma, we demonstrate an **11x speedup** (on a
+machine with 20 physical cores) for sorting a large pandas DataFrame (one
+billion entries). The baseline is the built-in pandas sort function, which sorts
+the DataFrame in 477 seconds. To leverage multiple cores, we implement the
+following standard distributed sorting scheme.
+
+* We assume that the data is partitioned across K pandas DataFrames and that
+  each one already lives in the Plasma store.
+* We subsample the data, sort the subsampled data, and use the result to define
+  L non-overlapping buckets.
+* For each of the K data partitions and each of the L buckets, we find the
+  subset of the data partition that falls in the bucket, and we sort that
+  subset.
+* For each of the L buckets, we gather all of the K sorted subsets that fall in
+  that bucket.
+* For each of the L buckets, we merge the corresponding K sorted subsets.
+* We turn each bucket into a pandas DataFrame and place it in the Plasma store.
+
+Using this scheme, we can sort the DataFrame (the data starts and ends in the
+Plasma store), in 44 seconds, giving an 11x speedup over the baseline.
+
+### Design
+
+The Plasma store runs as a separate process. It is written in C++ and is
+designed as a single-threaded event loop based on the [Redis][4] event loop library.
+The plasma client library can be linked into applications. Clients communicate
+with the Plasma store via messages serialized using [Google Flatbuffers][5].
+
+### Call for contributions
+
+Plasma is a work in progress, and the API is currently unstable. Today Plasma is
+primarily used in [Ray][3] as an in-memory cache for Arrow serialized objects.
+We are looking for a broader set of use cases to help refine Plasma’s API. In
+addition, we are looking for contributions in a variety of areas including
+improving performance and building other language bindings. Please let us know
+if you are interested in getting involved with the project.
+
+[1]: https://people.eecs.berkeley.edu/~pcmoritz/
+[2]: http://www.robertnishihara.com
+[3]: https://github.com/ray-project/ray
+[4]: https://redis.io/
+[5]: https://google.github.io/flatbuffers/
+[6]: https://github.com/apache/arrow/blob/master/cpp/apidoc/tutorials/plasma.md
+[7]: https://github.com/apache/arrow/blob/master/python/doc/source/plasma.rst

From 939957f33ed0dd02013917b366ff85eb857c3947 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 7 Aug 2017 23:38:42 -0400
Subject: [PATCH 0914/1644] ARROW-1335: [C++] Add offset to
 PrimitiveArray::raw_values to make consistent with other raw_values

This is an API change, but fixes an existing inconsistency that was the source of several bugs that had gone unnoticed because they were only being tested with code having 0 offset (i.e. unsliced). We'll need a corresponding patch in parquet-cpp

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #949 from wesm/ARROW-1335 and squashes the following commits:

10431ebf [Wes McKinney] Use raw_values in more places
3c96eb4a [Wes McKinney] Add offset to PrimitiveArray::raw_values to make consistent with other raw_values functions
---
 cpp/src/arrow/array.cc                  |  5 ++
 cpp/src/arrow/array.h                   | 10 ++--
 cpp/src/arrow/compare.cc                | 16 +++---
 cpp/src/arrow/python/arrow_to_pandas.cc | 74 ++++++++++++-------------
 4 files changed, 53 insertions(+), 52 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index ab0be7a0964c6..637eb2417fcfd 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -159,6 +159,11 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t le
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
 }
 
+const uint8_t* PrimitiveArray::raw_values() const {
+  return raw_values_ +
+         offset() * static_cast<const FixedWidthType&>(*type()).bit_width() / 8;
+}
+
 template <typename T>
 NumericArray<T>::NumericArray(const std::shared_ptr<internal::ArrayData>& data)
     : PrimitiveArray(data) {
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index a853f2bb5f93d..777fbe0b006b3 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -292,8 +292,8 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray {
   /// Does not account for any slice offset
   std::shared_ptr<Buffer> values() const { return data_->buffers[1]; }
 
-  /// Does not account for any slice offset
-  const uint8_t* raw_values() const { return raw_values_; }
+  /// \brief Return pointer to start of raw data
+  const uint8_t* raw_values() const;
 
  protected:
   PrimitiveArray() {}
@@ -521,7 +521,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 
   int32_t byte_width() const { return byte_width_; }
 
-  const uint8_t* raw_values() const { return raw_values_; }
+  const uint8_t* raw_values() const { return raw_values_ + byte_width_ * data_->offset; }
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
@@ -567,7 +567,9 @@ class ARROW_EXPORT DecimalArray : public FlatArray {
   int32_t byte_width() const {
     return static_cast<const DecimalType&>(*type()).byte_width();
   }
-  const uint8_t* raw_values() const { return raw_values_; }
+
+  /// \brief Return pointer to value data, accounting for any offset
+  const uint8_t* raw_values() const { return raw_values_ + byte_width() * data_->offset; }
 
  private:
   void SetData(const std::shared_ptr<internal::ArrayData>& data);
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 3a4a4009c6b16..c01f190351044 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -231,11 +231,11 @@ class RangeEqualsVisitor {
     const uint8_t* right_data = nullptr;
 
     if (left.values()) {
-      left_data = left.raw_values() + left.offset() * width;
+      left_data = left.raw_values();
     }
 
     if (right.values()) {
-      right_data = right.raw_values() + right.offset() * width;
+      right_data = right.raw_values();
     }
 
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
@@ -265,11 +265,11 @@ class RangeEqualsVisitor {
     const uint8_t* right_data = nullptr;
 
     if (left.values()) {
-      left_data = left.raw_values() + left.offset() * width;
+      left_data = left.raw_values();
     }
 
     if (right.values()) {
-      right_data = right.raw_values() + right.offset() * width;
+      right_data = right.raw_values();
     }
 
     for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
@@ -352,10 +352,10 @@ static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& r
   const uint8_t* right_data = nullptr;
 
   if (left.values()) {
-    left_data = left.values()->data() + left.offset() * byte_width;
+    left_data = left.raw_values();
   }
   if (right.values()) {
-    right_data = right.values()->data() + right.offset() * byte_width;
+    right_data = right.raw_values();
   }
 
   if (left.null_count() > 0) {
@@ -399,10 +399,10 @@ static bool IsEqualDecimal(const DecimalArray& left, const DecimalArray& right)
   const uint8_t* right_data = nullptr;
 
   if (left.values()) {
-    left_data = left.values()->data();
+    left_data = left.raw_values();
   }
   if (right.values()) {
-    right_data = right.values()->data();
+    right_data = right.raw_values();
   }
 
   const int32_t byte_width = left.byte_width();
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 86f82fdbd8de5..8c769ee5eeaf8 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -266,13 +266,12 @@ class PandasBlock {
 template <typename T>
 inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
+    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
+    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
     // Upcast to double, set NaN as appropriate
 
-    for (int i = 0; i < arr->length(); ++i) {
-      *out_values++ = prim_arr->IsNull(i) ? NAN : static_cast<double>(in_values[i]);
+    for (int i = 0; i < arr.length(); ++i) {
+      *out_values++ = arr.IsNull(i) ? NAN : static_cast<double>(in_values[i]);
     }
   }
 }
@@ -280,21 +279,19 @@ inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values
 template <typename T>
 inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
-    memcpy(out_values, in_values, sizeof(T) * arr->length());
-    out_values += arr->length();
+    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
+    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
+    memcpy(out_values, in_values, sizeof(T) * arr.length());
+    out_values += arr.length();
   }
 }
 
 template <typename InType, typename OutType>
 inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->raw_values());
-    for (int64_t i = 0; i < arr->length(); ++i) {
+    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
+    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
+    for (int64_t i = 0; i < arr.length(); ++i) {
       *out_values = in_values[i];
     }
   }
@@ -520,19 +517,18 @@ inline Status ConvertListsLike(const std::shared_ptr<Column>& col,
 template <typename T>
 inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
+    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
+    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
 
-    const uint8_t* valid_bits = arr->null_bitmap_data();
+    const uint8_t* valid_bits = arr.null_bitmap_data();
 
-    if (arr->null_count() > 0) {
-      for (int64_t i = 0; i < arr->length(); ++i) {
+    if (arr.null_count() > 0) {
+      for (int64_t i = 0; i < arr.length(); ++i) {
         *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i];
       }
     } else {
-      memcpy(out_values, in_values, sizeof(T) * arr->length());
-      out_values += arr->length();
+      memcpy(out_values, in_values, sizeof(T) * arr.length());
+      out_values += arr.length();
     }
   }
 }
@@ -541,12 +537,11 @@ template <typename InType, typename OutType>
 inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_value,
                                        OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->raw_values());
+    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
+    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = arr->IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      *out_values++ = arr.IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
     }
   }
 }
@@ -554,13 +549,12 @@ inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_valu
 template <typename InType, int64_t SHIFT>
 inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const InType*>(prim_arr->raw_values());
+    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
+    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = arr->IsNull(i) ? kPandasTimestampNull
-                                     : (static_cast<int64_t>(in_values[i]) * SHIFT);
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      *out_values++ = arr.IsNull(i) ? kPandasTimestampNull
+                                    : (static_cast<int64_t>(in_values[i]) * SHIFT);
     }
   }
 }
@@ -1004,6 +998,7 @@ class CategoricalBlock : public PandasBlock {
     for (int c = 0; c < data.num_chunks(); c++) {
       const std::shared_ptr<Array> arr = data.chunk(c);
       const auto& dict_arr = static_cast<const DictionaryArray&>(*arr);
+
       const auto& indices = static_cast<const PrimitiveArray&>(*dict_arr.indices());
       auto in_values = reinterpret_cast<const T*>(indices.raw_values());
 
@@ -1386,8 +1381,8 @@ class ArrowDeserializer {
   Status ConvertValuesZeroCopy(int npy_type, std::shared_ptr<Array> arr) {
     typedef typename internal::arrow_traits<TYPE>::T T;
 
-    auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-    auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
+    const auto& prim_arr = static_cast<const PrimitiveArray&>(*arr);
+    auto in_values = reinterpret_cast<const T*>(prim_arr.raw_values());
 
     // Zero-Copy. We can pass the data pointer directly to NumPy.
     void* data = const_cast<T*>(in_values);
@@ -1461,12 +1456,11 @@ class ArrowDeserializer {
     constexpr int64_t kShift = traits::npy_shift;
 
     for (int c = 0; c < data_.num_chunks(); c++) {
-      const std::shared_ptr<Array> arr = data_.chunk(c);
-      auto prim_arr = static_cast<PrimitiveArray*>(arr.get());
-      auto in_values = reinterpret_cast<const T*>(prim_arr->raw_values());
+      const auto& arr = static_cast<const PrimitiveArray&>(*data_.chunk(c));
+      auto in_values = reinterpret_cast<const T*>(arr.raw_values());
 
-      for (int64_t i = 0; i < arr->length(); ++i) {
-        *out_values++ = arr->IsNull(i) ? na_value : in_values[i] / kShift;
+      for (int64_t i = 0; i < arr.length(); ++i) {
+        *out_values++ = arr.IsNull(i) ? na_value : in_values[i] / kShift;
       }
     }
     return Status::OK();

From 5281a8264e0af807043b4fd70a7213ee4b176742 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 8 Aug 2017 09:29:09 -0400
Subject: [PATCH 0915/1644] ARROW-1334: [C++] Add alternate Table constructor
 that takes vector of Array

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #950 from wesm/ARROW-1334 and squashes the following commits:

f0655408 [Wes McKinney] Fix compiler warning
d9559682 [Wes McKinney] Add Table constructor that takes vector of Array instead of Column
---
 cpp/src/arrow/python/python-test.cc |  4 +---
 cpp/src/arrow/table-test.cc         |  3 +++
 cpp/src/arrow/table.cc              | 32 +++++++++++++++++++++++------
 cpp/src/arrow/table.h               | 18 +++++++++-------
 4 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 433ce9b37a80a..dd956463fec76 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -85,9 +85,7 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   auto f2 = field("f2", utf8());
   auto f3 = field("f3", utf8());
   std::vector<std::shared_ptr<Field>> fields = {f1, f2, f3};
-  std::vector<std::shared_ptr<Column>> cols = {std::make_shared<Column>(f1, arr),
-                                               std::make_shared<Column>(f2, arr),
-                                               std::make_shared<Column>(f3, arr)};
+  std::vector<std::shared_ptr<Array>> cols = {arr, arr, arr};
 
   auto schema = std::make_shared<Schema>(fields);
   auto table = std::make_shared<Table>(schema, cols);
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 1ba7a2f95798c..4b67492b7ed4a 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -230,6 +230,9 @@ TEST_F(TestTable, Ctors) {
   ASSERT_EQ(length, table_->num_rows());
   ASSERT_EQ(3, table_->num_columns());
 
+  auto array_ctor = std::make_shared<Table>(schema_, arrays_);
+  ASSERT_TRUE(table_->Equals(*array_ctor));
+
   table_.reset(new Table(schema_, columns_, length));
   ASSERT_OK(table_->ValidateColumns());
   ASSERT_EQ(length, table_->num_rows());
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 665ce2d84dea4..1f0c6d785448d 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -273,18 +273,38 @@ Status RecordBatch::Validate() const {
 // Table methods
 
 Table::Table(const std::shared_ptr<Schema>& schema,
-             const std::vector<std::shared_ptr<Column>>& columns)
+             const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
     : schema_(schema), columns_(columns) {
-  if (columns.size() == 0) {
-    num_rows_ = 0;
+  if (num_rows < 0) {
+    if (columns.size() == 0) {
+      num_rows_ = 0;
+    } else {
+      num_rows_ = columns[0]->length();
+    }
   } else {
-    num_rows_ = columns[0]->length();
+    num_rows_ = num_rows;
   }
 }
 
 Table::Table(const std::shared_ptr<Schema>& schema,
-             const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
-    : schema_(schema), columns_(columns), num_rows_(num_rows) {}
+             const std::vector<std::shared_ptr<Array>>& columns, int64_t num_rows)
+    : schema_(schema) {
+  if (num_rows < 0) {
+    if (columns.size() == 0) {
+      num_rows_ = 0;
+    } else {
+      num_rows_ = columns[0]->length();
+    }
+  } else {
+    num_rows_ = num_rows;
+  }
+
+  columns_.resize(columns.size());
+  for (size_t i = 0; i < columns.size(); ++i) {
+    columns_[i] = std::make_shared<Column>(schema->field(static_cast<int>(i)),
+                                           columns[i]);
+  }
+}
 
 std::shared_ptr<Table> Table::ReplaceSchemaMetadata(
     const std::shared_ptr<const KeyValueMetadata>& metadata) const {
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 6afd618da043b..31ca97a37078c 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -192,16 +192,20 @@ class ARROW_EXPORT RecordBatch {
 // Immutable container of fixed-length columns conforming to a particular schema
 class ARROW_EXPORT Table {
  public:
-  // If columns is zero-length, the table's number of rows is zero
+  /// \brief Construct Table from schema and columns
+  /// If columns is zero-length, the table's number of rows is zero
+  /// \param schema
+  /// \param columns
+  /// \param num_rows number of rows in table, -1 (default) to infer from columns
   Table(const std::shared_ptr<Schema>& schema,
-        const std::vector<std::shared_ptr<Column>>& columns);
+        const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows = -1);
 
-  // num_rows is a parameter to allow for tables of a particular size not
-  // having any materialized columns. Each column should therefore have the
-  // same length as num_rows -- you can validate this using
-  // Table::ValidateColumns
+  /// \brief Construct Table from schema and arrays
+  /// \param schema
+  /// \param arrays
+  /// \param num_rows number of rows in table, -1 (default) to infer from columns
   Table(const std::shared_ptr<Schema>& schema,
-        const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows);
+        const std::vector<std::shared_ptr<Array>>& arrays, int64_t num_rows = -1);
 
   // Construct table from RecordBatch, but only if all of the batch schemas are
   // equal. Returns Status::Invalid if there is some problem

From 20cee707cbfdaa5dc4f2b7dea09619f34a1f9f71 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 8 Aug 2017 09:30:16 -0400
Subject: [PATCH 0916/1644] ARROW-1338: [Python] Do not close RecordBatchWriter
 on dealloc in case sink is no longer valid

Also add missing close() statements to test_mock_output_stream to fix invalid writes causing core dump on OS X.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #952 from wesm/ARROW-1338 and squashes the following commits:

88e8cefe [Wes McKinney] Do not close RecordBatchWriter on dealloc in case sink is no longer valid. Add missing close() statements to test_mock_output_stream
---
 python/pyarrow/ipc.pxi          | 16 ++++++++++++++--
 python/pyarrow/tests/test_io.py |  2 ++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index d6f62aa95c5fb..ceed4b0e85248 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -163,8 +163,7 @@ cdef class _RecordBatchWriter:
         self.closed = True
 
     def __dealloc__(self):
-        if not self.closed:
-            self.close()
+        pass
 
     def _open(self, sink, Schema schema):
         cdef:
@@ -182,11 +181,24 @@ cdef class _RecordBatchWriter:
         self.closed = False
 
     def write_batch(self, RecordBatch batch):
+        """
+        Write RecordBatch to stream
+
+        Parameters
+        ----------
+        batch : RecordBatch
+        """
         with nogil:
             check_status(self.writer.get()
                          .WriteRecordBatch(deref(batch.batch)))
 
     def close(self):
+        """
+        Close stream and write end-of-stream 0 marker
+        """
+        if self.closed:
+            return
+
         with nogil:
             check_status(self.writer.get().Close())
         self.closed = True
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index c81a0485ce1ee..d503ea22464d5 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -277,6 +277,8 @@ def test_mock_output_stream():
 
     stream_writer1.write_batch(record_batch)
     stream_writer2.write_batch(record_batch)
+    stream_writer1.close()
+    stream_writer2.close()
 
     assert f1.size() == len(f2.get_result())
 

From 2615b47032d58284e0606b21cb216aa4b303a72c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 8 Aug 2017 11:56:32 -0400
Subject: [PATCH 0917/1644] ARROW-1306: [C++] Use UTF8 filenames in local file
 error messages

Encoded utf16-le bytes were being written to error messages (which are output to UTF-8 consoles), resulting in unintelligible displays.

This also improves the error message when opening the file fails per ARROW-1121.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #951 from wesm/ARROW-1306 and squashes the following commits:

fd0d93f7 [Wes McKinney] Restore utf8_data method
a4aae504 [Wes McKinney] MSVC fixes
b847b66c [Wes McKinney] Change PlatformFilename to be allocated with OSFile
d445fcad [Wes McKinney] Add Python unit test for ARROW-1306
0dc220c2 [Wes McKinney] MSVC fixes
9d80e491 [Wes McKinney] Add PlatformFilename abstraction, write error messages with UTF8 filenames
---
 cpp/src/arrow/io/file.cc             | 128 ++++++++++++++-------------
 cpp/src/arrow/io/file.h              |  23 ++++-
 cpp/src/arrow/io/io-file-test.cc     |   9 +-
 python/pyarrow/compat.py             |   3 +-
 python/pyarrow/tests/test_parquet.py |  11 +++
 5 files changed, 106 insertions(+), 68 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 82e3ba8109c23..57d30f73baa29 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -118,33 +118,64 @@ namespace io {
 
 // ----------------------------------------------------------------------
 // Cross-platform file compatability layer
+
 #if defined(_MSC_VER)
+
 constexpr const char* kRangeExceptionError =
     "Range exception during wide-char string conversion";
+
+struct PlatformFilename {
+  static Status Init(const std::string& utf8_path, PlatformFilename* out) {
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> utf16_converter;
+
+    if (!utf8_path.empty()) {
+      try {
+        out->utf16_path = utf16_converter.from_bytes(utf8_path);
+      } catch (const std::range_error&) {
+        return Status::Invalid(kRangeExceptionError);
+      }
+    } else {
+      out->utf16_path = std::wstring();
+    }
+    out->utf8_path = utf8_path;
+    return Status::OK();
+  }
+
+  const char* data() const { return reinterpret_cast<const char*>(utf16_path.c_str()); }
+
+  const char* utf8_data() const { return utf8_path.c_str(); }
+
+  size_t length() const { return utf16_path.size(); }
+
+  std::string utf8_path;
+  std::wstring utf16_path;
+};
+
+#else
+
+struct PlatformFilename {
+  static Status Init(const std::string& utf8_path, PlatformFilename* out) {
+    out->utf8_path = utf8_path;
+    return Status::OK();
+  }
+
+  const char* data() const { return utf8_path.c_str(); }
+
+  const char* utf8_data() const { return data(); }
+
+  size_t length() const { return utf8_path.size(); }
+
+  std::string utf8_path;
+};
+
 #endif
 
-static inline Status CheckOpenResult(int ret, int errno_actual, const char* filename,
-                                     size_t filename_length) {
+static inline Status CheckOpenResult(int ret, int errno_actual,
+                                     const PlatformFilename& filename) {
   if (ret == -1) {
     // TODO: errno codes to strings
     std::stringstream ss;
-    ss << "Failed to open file: ";
-#if defined(_MSC_VER)
-    // using wchar_t
-
-    // this requires c++11
-    std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
-    std::wstring wide_string(reinterpret_cast<const wchar_t*>(filename),
-                             filename_length / sizeof(wchar_t));
-    try {
-      std::string byte_string = converter.to_bytes(wide_string);
-      ss << byte_string;
-    } catch (const std::range_error&) {
-      ss << kRangeExceptionError;
-    }
-#else
-    ss << filename;
-#endif
+    ss << "Failed to open local file: " << filename.utf8_data();
     return Status::IOError(ss.str());
   }
   return Status::OK();
@@ -161,54 +192,27 @@ static inline int64_t lseek64_compat(int fd, int64_t pos, int whence) {
 #endif
 }
 
-#if defined(_MSC_VER)
-static inline Status ConvertToUtf16(const std::string& input, std::wstring* result) {
-  if (result == nullptr) {
-    return Status::Invalid("Pointer to result is not valid");
-  }
-
-  if (input.empty()) {
-    *result = std::wstring();
-    return Status::OK();
-  }
-
-  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> utf16_converter;
-  try {
-    *result = utf16_converter.from_bytes(input);
-  } catch (const std::range_error&) {
-    return Status::Invalid(kRangeExceptionError);
-  }
-  return Status::OK();
-}
-#endif
-
-static inline Status FileOpenReadable(const std::string& filename, int* fd) {
+static inline Status FileOpenReadable(const PlatformFilename& filename, int* fd) {
   int ret;
   errno_t errno_actual = 0;
 #if defined(_MSC_VER)
-  std::wstring wide_filename;
-  RETURN_NOT_OK(ConvertToUtf16(filename, &wide_filename));
-
-  errno_actual =
-      _wsopen_s(fd, wide_filename.c_str(), _O_RDONLY | _O_BINARY, _SH_DENYNO, _S_IREAD);
+  errno_actual = _wsopen_s(fd, reinterpret_cast<const wchar_t*>(filename.data()),
+                           _O_RDONLY | _O_BINARY, _SH_DENYNO, _S_IREAD);
   ret = *fd;
 #else
-  ret = *fd = open(filename.c_str(), O_RDONLY | O_BINARY);
+  ret = *fd = open(filename.data(), O_RDONLY | O_BINARY);
   errno_actual = errno;
 #endif
 
-  return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
+  return CheckOpenResult(ret, errno_actual, filename);
 }
 
-static inline Status FileOpenWriteable(const std::string& filename, bool write_only,
+static inline Status FileOpenWriteable(const PlatformFilename& filename, bool write_only,
                                        bool truncate, int* fd) {
   int ret;
   errno_t errno_actual = 0;
 
 #if defined(_MSC_VER)
-  std::wstring wide_filename;
-  RETURN_NOT_OK(ConvertToUtf16(filename, &wide_filename));
-
   int oflag = _O_CREAT | _O_BINARY;
   int pmode = _S_IWRITE;
   if (!write_only) {
@@ -225,7 +229,8 @@ static inline Status FileOpenWriteable(const std::string& filename, bool write_o
     oflag |= _O_RDWR;
   }
 
-  errno_actual = _wsopen_s(fd, wide_filename.c_str(), oflag, _SH_DENYNO, pmode);
+  errno_actual = _wsopen_s(fd, reinterpret_cast<const wchar_t*>(filename.data()), oflag,
+                           _SH_DENYNO, pmode);
   ret = *fd;
 
 #else
@@ -241,9 +246,9 @@ static inline Status FileOpenWriteable(const std::string& filename, bool write_o
     oflag |= O_RDWR;
   }
 
-  ret = *fd = open(filename.c_str(), oflag, ARROW_WRITE_SHMODE);
+  ret = *fd = open(filename.data(), oflag, ARROW_WRITE_SHMODE);
 #endif
-  return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
+  return CheckOpenResult(ret, errno_actual, filename);
 }
 
 static inline Status FileTell(int fd, int64_t* pos) {
@@ -352,8 +357,9 @@ class OSFile {
   ~OSFile() {}
 
   Status OpenWriteable(const std::string& path, bool append, bool write_only) {
-    RETURN_NOT_OK(FileOpenWriteable(path, write_only, !append, &fd_));
-    path_ = path;
+    RETURN_NOT_OK(PlatformFilename::Init(path, &path_));
+
+    RETURN_NOT_OK(FileOpenWriteable(path_, write_only, !append, &fd_));
     is_open_ = true;
     mode_ = write_only ? FileMode::WRITE : FileMode::READWRITE;
 
@@ -366,10 +372,11 @@ class OSFile {
   }
 
   Status OpenReadable(const std::string& path) {
-    RETURN_NOT_OK(FileOpenReadable(path, &fd_));
+    RETURN_NOT_OK(PlatformFilename::Init(path, &path_));
+
+    RETURN_NOT_OK(FileOpenReadable(path_, &fd_));
     RETURN_NOT_OK(FileGetSize(fd_, &size_));
 
-    path_ = path;
     is_open_ = true;
     mode_ = FileMode::READ;
     return Status::OK();
@@ -408,14 +415,13 @@ class OSFile {
   int fd() const { return fd_; }
 
   bool is_open() const { return is_open_; }
-  const std::string& path() const { return path_; }
 
   int64_t size() const { return size_; }
 
   FileMode::type mode() const { return mode_; }
 
  protected:
-  std::string path_;
+  PlatformFilename path_;
 
   std::mutex lock_;
 
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index ba740f1e8f4a9..2a0e89ca325fa 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -40,10 +40,18 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
  public:
   ~FileOutputStream();
 
-  // When opening a new file, any existing file with the indicated path is
-  // truncated to 0 bytes, deleting any existing memory
+  /// \brief Open a local file for writing, truncating any existing file
+  /// \param[in] path with UTF8 encoding
+  /// \param[out] file a FileOutputStream instance
+  ///
+  /// When opening a new file, any existing file with the indicated path is
+  /// truncated to 0 bytes, deleting any existing memory
   static Status Open(const std::string& path, std::shared_ptr<FileOutputStream>* file);
 
+  /// \brief Open a local file for writing
+  /// \param[in] path with UTF8 encoding
+  /// \param[in] append append to existing file, otherwise truncate to 0 bytes
+  /// \param[out] file a FileOutputStream instance
   static Status Open(const std::string& path, bool append,
                      std::shared_ptr<FileOutputStream>* file);
 
@@ -68,10 +76,17 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile {
  public:
   ~ReadableFile();
 
-  // Open file, allocate memory (if needed) from default memory pool
+  /// \brief Open a local file for reading
+  /// \param[in] path with UTF8 encoding
+  /// \param[out] file ReadableFile instance
+  /// Open file, allocate memory (if needed) from default memory pool
   static Status Open(const std::string& path, std::shared_ptr<ReadableFile>* file);
 
-  // Open file with one's own memory pool for memory allocations
+  /// \brief Open a local file for reading
+  /// \param[in] path with UTF8 encoding
+  /// \param[in] pool a MemoryPool for memory allocations
+  /// \param[out] file ReadableFile instance
+  /// Open file with one's own memory pool for memory allocations
   static Status Open(const std::string& path, MemoryPool* memory_pool,
                      std::shared_ptr<ReadableFile>* file);
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 36c35700d6496..630356fa2af38 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -45,7 +45,7 @@ static bool FileExists(const std::string& path) {
 void InvalidParamHandler(const wchar_t* expr, const wchar_t* func,
                          const wchar_t* source_file, unsigned int source_line,
                          uintptr_t reserved) {
-  wprintf(L"Invalid parameter in funcion %s. Source: %s line %d expression %s", func,
+  wprintf(L"Invalid parameter in function %s. Source: %s line %d expression %s", func,
           source_file, source_line, expr);
 }
 #endif
@@ -320,7 +320,12 @@ TEST_F(TestReadableFile, ReadAt) {
 }
 
 TEST_F(TestReadableFile, NonExistentFile) {
-  ASSERT_RAISES(IOError, ReadableFile::Open("0xDEADBEEF.txt", &file_));
+  std::string path = "0xDEADBEEF.txt";
+  Status s = ReadableFile::Open(path, &file_);
+  ASSERT_TRUE(s.IsIOError());
+
+  std::string message = s.message();
+  ASSERT_NE(std::string::npos, message.find(path));
 }
 
 class MyMemoryPool : public MemoryPool {
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 7be35dfc2c81f..2252e85e6ef77 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -132,7 +132,6 @@ def frombytes(o):
 
 def encode_file_path(path):
     import os
-    # Windows requires utf-16le encoding for unicode file names
     if isinstance(path, unicode_type):
         # POSIX systems can handle utf-8. UTF8 is converted to utf16-le in
         # libarrow
@@ -140,6 +139,8 @@ def encode_file_path(path):
     else:
         encoded_path = path
 
+    # Windows file system requires utf-16le for file names; Arrow C++ libraries
+    # will convert utf8 to utf16
     return encoded_path
 
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 9a570b9d5dab4..8a20f4c469200 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1127,3 +1127,14 @@ def test_write_error_deletes_incomplete_file(tmpdir):
         pass
 
     assert not os.path.exists(filename)
+
+
+@parquet
+def test_read_non_existent_file(tmpdir):
+    import pyarrow.parquet as pq
+
+    path = 'non-existent-file.parquet'
+    try:
+        pq.read_table(path)
+    except Exception as e:
+        assert path in e.args[0]

From 6e26701257be160fa95ce174d80b046adb493e57 Mon Sep 17 00:00:00 2001
From: fjetter <florian.jetter@blue-yonder.com>
Date: Tue, 8 Aug 2017 13:51:37 -0400
Subject: [PATCH 0918/1644] ARROW-439: [Python] Add option in "to_pandas"
 conversions to yield Categorical from String/Binary arrays

I added support to cast Arrays and Columns to dictionaries with the possibility to extend the casting to different types.

I intend to add more types to the casting, at least for trivial cases, but first I wanted to get some feedback on the current state.

Author: fjetter <florian.jetter@blue-yonder.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #909 from fjetter/feature/make_dictionary_array and squashes the following commits:

d1189395 [Wes McKinney] Fix deprecated API usage
606724df [Wes McKinney] Handle ordered categories in arrow_to_pandas.cc. flake8 Cython fixes
d2bb8d8e [Wes McKinney] Move dictionary index type dispatch and memory allocation into CategoricalBlock::Write
6ab28730 [fjetter] Remove dead code
bea4cb9e [fjetter] Merge master
bb3209ba [fjetter] Add pool to ConvertTableToPandas in python-test
24fbf424 [fjetter] Format arrow_to_pandas
39b22ff6 [fjetter] Allocate categorical blocks in write path
b7f389f3 [fjetter] Pass memory pool from the outside
c496cb5f [fjetter] Pass pandas options through to pandas write before conversion
4b12aa13 [fjetter] Push pandas options down
b6fca35c [fjetter] Rename and add docs for EncodeDictionary
6479d292 [fjetter] add MakeDictionaryArray
---
 cpp/src/arrow/builder.cc                    |  81 +++-
 cpp/src/arrow/builder.h                     |  16 +
 cpp/src/arrow/python/arrow_to_pandas.cc     | 504 ++++++++++----------
 cpp/src/arrow/python/arrow_to_pandas.h      |  20 +-
 cpp/src/arrow/python/python-test.cc         |   4 +-
 cpp/src/arrow/util/parallel.h               |  70 +++
 python/pyarrow/array.pxi                    |  12 +-
 python/pyarrow/includes/libarrow.pxd        |  15 +-
 python/pyarrow/pandas_compat.py             |   7 +-
 python/pyarrow/table.pxi                    |  29 +-
 python/pyarrow/tests/test_convert_pandas.py |  29 +-
 11 files changed, 502 insertions(+), 285 deletions(-)
 create mode 100644 cpp/src/arrow/util/parallel.h

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 889c64db9fdb8..e2054dbfde688 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -27,6 +27,7 @@
 #include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/status.h"
+#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -1396,8 +1397,84 @@ Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>&
     DICTIONARY_BUILDER_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
     DICTIONARY_BUILDER_CASE(STRING, StringDictionaryBuilder);
     DICTIONARY_BUILDER_CASE(BINARY, BinaryDictionaryBuilder);
-    // DICTIONARY_BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder);
-    // DICTIONARY_BUILDER_CASE(DECIMAL, DecimalBuilder);
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+}
+
+#define DICTIONARY_ARRAY_CASE(ENUM, BuilderType)                           \
+  case Type::ENUM:                                                         \
+    builder = std::make_shared<BuilderType>(type, pool);                   \
+    RETURN_NOT_OK(static_cast<BuilderType&>(*builder).AppendArray(input)); \
+    RETURN_NOT_OK(builder->Finish(out));                                   \
+    return Status::OK();
+
+Status EncodeArrayToDictionary(const Array& input, MemoryPool* pool,
+                               std::shared_ptr<Array>* out) {
+  const std::shared_ptr<DataType>& type = input.data()->type;
+  std::shared_ptr<ArrayBuilder> builder;
+  switch (type->id()) {
+    DICTIONARY_ARRAY_CASE(UINT8, DictionaryBuilder<UInt8Type>);
+    DICTIONARY_ARRAY_CASE(INT8, DictionaryBuilder<Int8Type>);
+    DICTIONARY_ARRAY_CASE(UINT16, DictionaryBuilder<UInt16Type>);
+    DICTIONARY_ARRAY_CASE(INT16, DictionaryBuilder<Int16Type>);
+    DICTIONARY_ARRAY_CASE(UINT32, DictionaryBuilder<UInt32Type>);
+    DICTIONARY_ARRAY_CASE(INT32, DictionaryBuilder<Int32Type>);
+    DICTIONARY_ARRAY_CASE(UINT64, DictionaryBuilder<UInt64Type>);
+    DICTIONARY_ARRAY_CASE(INT64, DictionaryBuilder<Int64Type>);
+    DICTIONARY_ARRAY_CASE(DATE32, DictionaryBuilder<Date32Type>);
+    DICTIONARY_ARRAY_CASE(DATE64, DictionaryBuilder<Date64Type>);
+    DICTIONARY_ARRAY_CASE(TIME32, DictionaryBuilder<Time32Type>);
+    DICTIONARY_ARRAY_CASE(TIME64, DictionaryBuilder<Time64Type>);
+    DICTIONARY_ARRAY_CASE(TIMESTAMP, DictionaryBuilder<TimestampType>);
+    DICTIONARY_ARRAY_CASE(FLOAT, DictionaryBuilder<FloatType>);
+    DICTIONARY_ARRAY_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
+    DICTIONARY_ARRAY_CASE(STRING, StringDictionaryBuilder);
+    DICTIONARY_ARRAY_CASE(BINARY, BinaryDictionaryBuilder);
+    default:
+      return Status::NotImplemented(type->ToString());
+  }
+}
+#define DICTIONARY_COLUMN_CASE(ENUM, BuilderType)                             \
+  case Type::ENUM:                                                            \
+    builder = std::make_shared<BuilderType>(type, pool);                      \
+    chunks = input.data();                                                    \
+    for (auto chunk : chunks->chunks()) {                                     \
+      RETURN_NOT_OK(static_cast<BuilderType&>(*builder).AppendArray(*chunk)); \
+    }                                                                         \
+    RETURN_NOT_OK(builder->Finish(&arr));                                     \
+    *out = std::make_shared<Column>(input.name(), arr);                       \
+    return Status::OK();
+
+/// \brief Encodes a column to a suitable dictionary type
+/// \param input Column to be encoded
+/// \param pool MemoryPool to allocate the dictionary
+/// \param out The new column
+/// \return Status
+Status EncodeColumnToDictionary(const Column& input, MemoryPool* pool,
+                                std::shared_ptr<Column>* out) {
+  const std::shared_ptr<DataType>& type = input.type();
+  std::shared_ptr<ArrayBuilder> builder;
+  std::shared_ptr<Array> arr;
+  std::shared_ptr<ChunkedArray> chunks;
+  switch (type->id()) {
+    DICTIONARY_COLUMN_CASE(UINT8, DictionaryBuilder<UInt8Type>);
+    DICTIONARY_COLUMN_CASE(INT8, DictionaryBuilder<Int8Type>);
+    DICTIONARY_COLUMN_CASE(UINT16, DictionaryBuilder<UInt16Type>);
+    DICTIONARY_COLUMN_CASE(INT16, DictionaryBuilder<Int16Type>);
+    DICTIONARY_COLUMN_CASE(UINT32, DictionaryBuilder<UInt32Type>);
+    DICTIONARY_COLUMN_CASE(INT32, DictionaryBuilder<Int32Type>);
+    DICTIONARY_COLUMN_CASE(UINT64, DictionaryBuilder<UInt64Type>);
+    DICTIONARY_COLUMN_CASE(INT64, DictionaryBuilder<Int64Type>);
+    DICTIONARY_COLUMN_CASE(DATE32, DictionaryBuilder<Date32Type>);
+    DICTIONARY_COLUMN_CASE(DATE64, DictionaryBuilder<Date64Type>);
+    DICTIONARY_COLUMN_CASE(TIME32, DictionaryBuilder<Time32Type>);
+    DICTIONARY_COLUMN_CASE(TIME64, DictionaryBuilder<Time64Type>);
+    DICTIONARY_COLUMN_CASE(TIMESTAMP, DictionaryBuilder<TimestampType>);
+    DICTIONARY_COLUMN_CASE(FLOAT, DictionaryBuilder<FloatType>);
+    DICTIONARY_COLUMN_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
+    DICTIONARY_COLUMN_CASE(STRING, StringDictionaryBuilder);
+    DICTIONARY_COLUMN_CASE(BINARY, BinaryDictionaryBuilder);
     default:
       return Status::NotImplemented(type->ToString());
   }
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index b15005f62bc7e..46900fc7129c1 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -28,6 +28,7 @@
 #include "arrow/buffer.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
+#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -913,6 +914,21 @@ Status ARROW_EXPORT MakeDictionaryBuilder(MemoryPool* pool,
                                           const std::shared_ptr<DataType>& type,
                                           std::shared_ptr<ArrayBuilder>* out);
 
+/// \brief Convert Array to encoded DictionaryArray form
+///
+/// \param[in] input The Array to be encoded
+/// \param[in] pool MemoryPool to allocate memory for the hash table
+/// \param[out] out Array encoded to DictionaryArray
+Status ARROW_EXPORT EncodeArrayToDictionary(const Array& input, MemoryPool* pool,
+                                            std::shared_ptr<Array>* out);
+
+/// \brief Convert a Column's data internally to DictionaryArray
+///
+/// \param[in] input The ChunkedArray to be encoded
+/// \param[in] pool MemoryPool to allocate memory for the hash table
+/// \param[out] out Column with data converted to DictionaryArray
+Status ARROW_EXPORT EncodeColumnToDictionary(const Column& input, MemoryPool* pool,
+                                             std::shared_ptr<Column>* out);
 }  // namespace arrow
 
 #endif  // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 8c769ee5eeaf8..23bef7bcae65d 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -22,14 +22,11 @@
 #include "arrow/python/arrow_to_pandas.h"
 
 #include <algorithm>
-#include <atomic>
 #include <cmath>
 #include <cstdint>
 #include <memory>
-#include <mutex>
 #include <sstream>
 #include <string>
-#include <thread>
 #include <unordered_map>
 #include <vector>
 
@@ -42,6 +39,7 @@
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
+#include "arrow/util/parallel.h"
 #include "arrow/visitor_inline.h"
 
 #include "arrow/python/builtin_convert.h"
@@ -186,8 +184,8 @@ class PandasBlock {
     CATEGORICAL
   };
 
-  PandasBlock(int64_t num_rows, int num_columns)
-      : num_rows_(num_rows), num_columns_(num_columns) {}
+  PandasBlock(PandasOptions options, int64_t num_rows, int num_columns)
+      : num_rows_(num_rows), num_columns_(num_columns), options_(options) {}
   virtual ~PandasBlock() {}
 
   virtual Status Allocate() = 0;
@@ -255,6 +253,8 @@ class PandasBlock {
   OwnedRef block_arr_;
   uint8_t* block_data_;
 
+  PandasOptions options_;
+
   // ndarray<int32>
   OwnedRef placement_arr_;
   int64_t* placement_data_;
@@ -264,7 +264,8 @@ class PandasBlock {
 };
 
 template <typename T>
-inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values) {
+inline void ConvertIntegerWithNulls(PandasOptions options, const ChunkedArray& data,
+                                    double* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
     auto in_values = reinterpret_cast<const T*>(arr.raw_values());
@@ -277,7 +278,8 @@ inline void ConvertIntegerWithNulls(const ChunkedArray& data, double* out_values
 }
 
 template <typename T>
-inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_values) {
+inline void ConvertIntegerNoNullsSameType(PandasOptions options, const ChunkedArray& data,
+                                          T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
     auto in_values = reinterpret_cast<const T*>(arr.raw_values());
@@ -287,7 +289,8 @@ inline void ConvertIntegerNoNullsSameType(const ChunkedArray& data, T* out_value
 }
 
 template <typename InType, typename OutType>
-inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_values) {
+inline void ConvertIntegerNoNullsCast(PandasOptions options, const ChunkedArray& data,
+                                      OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
     auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
@@ -297,7 +300,8 @@ inline void ConvertIntegerNoNullsCast(const ChunkedArray& data, OutType* out_val
   }
 }
 
-static Status ConvertBooleanWithNulls(const ChunkedArray& data, PyObject** out_values) {
+static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray& data,
+                                      PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
@@ -321,7 +325,8 @@ static Status ConvertBooleanWithNulls(const ChunkedArray& data, PyObject** out_v
   return Status::OK();
 }
 
-static void ConvertBooleanNoNulls(const ChunkedArray& data, uint8_t* out_values) {
+static void ConvertBooleanNoNulls(PandasOptions options, const ChunkedArray& data,
+                                  uint8_t* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const std::shared_ptr<Array> arr = data.chunk(c);
     auto bool_arr = static_cast<BooleanArray*>(arr.get());
@@ -332,7 +337,8 @@ static void ConvertBooleanNoNulls(const ChunkedArray& data, uint8_t* out_values)
 }
 
 template <typename Type>
-inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values) {
+inline Status ConvertBinaryLike(PandasOptions options, const ChunkedArray& data,
+                                PyObject** out_values) {
   using ArrayType = typename TypeTraits<Type>::ArrayType;
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
@@ -362,7 +368,8 @@ inline Status ConvertBinaryLike(const ChunkedArray& data, PyObject** out_values)
   return Status::OK();
 }
 
-inline Status ConvertNulls(const ChunkedArray& data, PyObject** out_values) {
+inline Status ConvertNulls(PandasOptions options, const ChunkedArray& data,
+                           PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
     std::shared_ptr<Array> arr = data.chunk(c);
@@ -377,7 +384,8 @@ inline Status ConvertNulls(const ChunkedArray& data, PyObject** out_values) {
   return Status::OK();
 }
 
-inline Status ConvertFixedSizeBinary(const ChunkedArray& data, PyObject** out_values) {
+inline Status ConvertFixedSizeBinary(PandasOptions options, const ChunkedArray& data,
+                                     PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
     auto arr = static_cast<FixedSizeBinaryArray*>(data.chunk(c).get());
@@ -407,7 +415,8 @@ inline Status ConvertFixedSizeBinary(const ChunkedArray& data, PyObject** out_va
   return Status::OK();
 }
 
-inline Status ConvertStruct(const ChunkedArray& data, PyObject** out_values) {
+inline Status ConvertStruct(PandasOptions options, const ChunkedArray& data,
+                            PyObject** out_values) {
   PyAcquireGIL lock;
   if (data.num_chunks() <= 0) {
     return Status::OK();
@@ -424,8 +433,8 @@ inline Status ConvertStruct(const ChunkedArray& data, PyObject** out_values) {
     // Convert the struct arrays first
     for (int32_t i = 0; i < num_fields; i++) {
       PyObject* numpy_array;
-      RETURN_NOT_OK(
-          ConvertArrayToPandas(arr->field(static_cast<int>(i)), nullptr, &numpy_array));
+      RETURN_NOT_OK(ConvertArrayToPandas(options, arr->field(static_cast<int>(i)),
+                                         nullptr, &numpy_array));
       fields_data[i].reset(numpy_array);
     }
 
@@ -470,7 +479,7 @@ inline Status ConvertStruct(const ChunkedArray& data, PyObject** out_values) {
 }
 
 template <typename ArrowType>
-inline Status ConvertListsLike(const std::shared_ptr<Column>& col,
+inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Column>& col,
                                PyObject** out_values) {
   const ChunkedArray& data = *col->data().get();
   auto list_type = std::static_pointer_cast<ListType>(col->type());
@@ -485,7 +494,7 @@ inline Status ConvertListsLike(const std::shared_ptr<Column>& col,
   // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
   //    Storing a reference to the whole Array would be to expensive.
   PyObject* numpy_array;
-  RETURN_NOT_OK(ConvertColumnToPandas(flat_column, nullptr, &numpy_array));
+  RETURN_NOT_OK(ConvertColumnToPandas(options, flat_column, nullptr, &numpy_array));
 
   PyAcquireGIL lock;
 
@@ -560,7 +569,8 @@ inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values)
 }
 
 template <typename TYPE>
-static Status ConvertTimes(const ChunkedArray& data, PyObject** out_values) {
+static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
+                           PyObject** out_values) {
   using ArrayType = typename TypeTraits<TYPE>::ArrayType;
 
   PyAcquireGIL lock;
@@ -629,7 +639,8 @@ Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
   return Status::OK();
 }
 
-static Status ConvertDecimals(const ChunkedArray& data, PyObject** out_values) {
+static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
+                              PyObject** out_values) {
   PyAcquireGIL lock;
   OwnedRef decimal_ref;
   OwnedRef Decimal_ref;
@@ -673,9 +684,9 @@ static Status ConvertDecimals(const ChunkedArray& data, PyObject** out_values) {
   return Status::OK();
 }
 
-#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum)                \
-  case Type::ArrowEnum:                                            \
-    RETURN_NOT_OK((ConvertListsLike<ArrowType>(col, out_buffer))); \
+#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum)                          \
+  case Type::ArrowEnum:                                                      \
+    RETURN_NOT_OK((ConvertListsLike<ArrowType>(options_, col, out_buffer))); \
     break;
 
 class ObjectBlock : public PandasBlock {
@@ -693,21 +704,21 @@ class ObjectBlock : public PandasBlock {
     const ChunkedArray& data = *col->data().get();
 
     if (type == Type::BOOL) {
-      RETURN_NOT_OK(ConvertBooleanWithNulls(data, out_buffer));
+      RETURN_NOT_OK(ConvertBooleanWithNulls(options_, data, out_buffer));
     } else if (type == Type::BINARY) {
-      RETURN_NOT_OK(ConvertBinaryLike<BinaryType>(data, out_buffer));
+      RETURN_NOT_OK(ConvertBinaryLike<BinaryType>(options_, data, out_buffer));
     } else if (type == Type::STRING) {
-      RETURN_NOT_OK(ConvertBinaryLike<StringType>(data, out_buffer));
+      RETURN_NOT_OK(ConvertBinaryLike<StringType>(options_, data, out_buffer));
     } else if (type == Type::FIXED_SIZE_BINARY) {
-      RETURN_NOT_OK(ConvertFixedSizeBinary(data, out_buffer));
+      RETURN_NOT_OK(ConvertFixedSizeBinary(options_, data, out_buffer));
     } else if (type == Type::TIME32) {
-      RETURN_NOT_OK(ConvertTimes<Time32Type>(data, out_buffer));
+      RETURN_NOT_OK(ConvertTimes<Time32Type>(options_, data, out_buffer));
     } else if (type == Type::TIME64) {
-      RETURN_NOT_OK(ConvertTimes<Time64Type>(data, out_buffer));
+      RETURN_NOT_OK(ConvertTimes<Time64Type>(options_, data, out_buffer));
     } else if (type == Type::DECIMAL) {
-      RETURN_NOT_OK(ConvertDecimals(data, out_buffer));
+      RETURN_NOT_OK(ConvertDecimals(options_, data, out_buffer));
     } else if (type == Type::NA) {
-      RETURN_NOT_OK(ConvertNulls(data, out_buffer));
+      RETURN_NOT_OK(ConvertNulls(options_, data, out_buffer));
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
       switch (list_type->value_type()->id()) {
@@ -732,7 +743,7 @@ class ObjectBlock : public PandasBlock {
         }
       }
     } else if (type == Type::STRUCT) {
-      RETURN_NOT_OK(ConvertStruct(data, out_buffer));
+      RETURN_NOT_OK(ConvertStruct(options_, data, out_buffer));
     } else {
       std::stringstream ss;
       ss << "Unsupported type for object array output: " << col->type()->ToString();
@@ -768,7 +779,7 @@ class IntBlock : public PandasBlock {
       return Status::NotImplemented(ss.str());
     }
 
-    ConvertIntegerNoNullsSameType<C_TYPE>(data, out_buffer);
+    ConvertIntegerNoNullsSameType<C_TYPE>(options_, data, out_buffer);
     placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
@@ -821,8 +832,8 @@ class Float64Block : public PandasBlock {
 
     const ChunkedArray& data = *col->data().get();
 
-#define INTEGER_CASE(IN_TYPE)                         \
-  ConvertIntegerWithNulls<IN_TYPE>(data, out_buffer); \
+#define INTEGER_CASE(IN_TYPE)                                   \
+  ConvertIntegerWithNulls<IN_TYPE>(options_, data, out_buffer); \
   break;
 
     switch (type) {
@@ -881,7 +892,7 @@ class BoolBlock : public PandasBlock {
     uint8_t* out_buffer =
         reinterpret_cast<uint8_t*>(block_data_) + rel_placement * num_rows_;
 
-    ConvertBooleanNoNulls(*col->data().get(), out_buffer);
+    ConvertBooleanNoNulls(options_, *col->data().get(), out_buffer);
     placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
@@ -946,8 +957,8 @@ class DatetimeBlock : public PandasBlock {
 
 class DatetimeTZBlock : public DatetimeBlock {
  public:
-  DatetimeTZBlock(const std::string& timezone, int64_t num_rows)
-      : DatetimeBlock(num_rows, 1), timezone_(timezone) {}
+  DatetimeTZBlock(PandasOptions options, const std::string& timezone, int64_t num_rows)
+      : DatetimeBlock(options, num_rows, 1), timezone_(timezone) {}
 
   // Like Categorical, the internal ndarray is 1-dimensional
   Status Allocate() override { return AllocateDatetime(1); }
@@ -973,25 +984,25 @@ class DatetimeTZBlock : public DatetimeBlock {
   std::string timezone_;
 };
 
-template <int ARROW_INDEX_TYPE>
 class CategoricalBlock : public PandasBlock {
  public:
-  explicit CategoricalBlock(int64_t num_rows) : PandasBlock(num_rows, 1) {}
-  Status Allocate() override {
-    constexpr int npy_type = internal::arrow_traits<ARROW_INDEX_TYPE>::npy_type;
+  explicit CategoricalBlock(PandasOptions options, MemoryPool* pool, int64_t num_rows)
+      : PandasBlock(options, num_rows, 1), pool_(pool) {}
 
-    if (!(npy_type == NPY_INT8 || npy_type == NPY_INT16 || npy_type == NPY_INT32 ||
-          npy_type == NPY_INT64)) {
-      return Status::Invalid("Category indices must be signed integers");
-    }
-    return AllocateNDArray(npy_type, 1);
+  Status Allocate() override {
+    return Status::NotImplemented(
+        "CategoricalBlock allocation happens when calling Write");
   }
 
-  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
-               int64_t rel_placement) override {
-    using T = typename internal::arrow_traits<ARROW_INDEX_TYPE>::T;
+  template <int ARROW_INDEX_TYPE>
+  Status WriteIndices(const std::shared_ptr<Column>& col) {
+    using TRAITS = internal::arrow_traits<ARROW_INDEX_TYPE>;
+    using T = typename TRAITS::T;
+    constexpr int npy_type = TRAITS::npy_type;
+    RETURN_NOT_OK(AllocateNDArray(npy_type, 1));
 
-    T* out_values = reinterpret_cast<T*>(block_data_) + rel_placement * num_rows_;
+    // No relative placement offset because a single column
+    T* out_values = reinterpret_cast<T*>(block_data_);
 
     const ChunkedArray& data = *col->data().get();
 
@@ -1008,13 +1019,48 @@ class CategoricalBlock : public PandasBlock {
       }
     }
 
-    placement_data_[rel_placement] = abs_placement;
+    return Status::OK();
+  }
+
+  Status Write(const std::shared_ptr<Column>& col, int64_t abs_placement,
+               int64_t rel_placement) override {
+    std::shared_ptr<Column> converted_col;
+    if (options_.strings_to_categorical &&
+        (col->type()->id() == Type::STRING || col->type()->id() == Type::BINARY)) {
+      RETURN_NOT_OK(EncodeColumnToDictionary(static_cast<const Column&>(*col), pool_,
+                                             &converted_col));
+    } else {
+      converted_col = col;
+    }
 
-    auto dict_type = static_cast<const DictionaryType*>(col->type().get());
+    const auto& dict_type = static_cast<const DictionaryType&>(*converted_col->type());
+
+    switch (dict_type.index_type()->id()) {
+      case Type::INT8:
+        RETURN_NOT_OK(WriteIndices<Type::INT8>(converted_col));
+        break;
+      case Type::INT16:
+        RETURN_NOT_OK(WriteIndices<Type::INT16>(converted_col));
+        break;
+      case Type::INT32:
+        RETURN_NOT_OK(WriteIndices<Type::INT32>(converted_col));
+        break;
+      case Type::INT64:
+        RETURN_NOT_OK(WriteIndices<Type::INT64>(converted_col));
+        break;
+      default: {
+        std::stringstream ss;
+        ss << "Categorical index type not supported: "
+           << dict_type.index_type()->ToString();
+        return Status::NotImplemented(ss.str());
+      }
+    }
 
+    placement_data_[rel_placement] = abs_placement;
     PyObject* dict;
-    RETURN_NOT_OK(ConvertArrayToPandas(dict_type->dictionary(), nullptr, &dict));
+    RETURN_NOT_OK(ConvertArrayToPandas(options_, dict_type.dictionary(), nullptr, &dict));
     dictionary_.reset(dict);
+    ordered_ = dict_type.ordered();
 
     return Status::OK();
   }
@@ -1027,20 +1073,26 @@ class CategoricalBlock : public PandasBlock {
     PyDict_SetItemString(result, "dictionary", dictionary_.obj());
     PyDict_SetItemString(result, "placement", placement_arr_.obj());
 
+    PyObject* py_ordered = ordered_ ? Py_True : Py_False;
+    Py_INCREF(py_ordered);
+    PyDict_SetItemString(result, "ordered", py_ordered);
+
     *output = result;
 
     return Status::OK();
   }
 
  protected:
+  MemoryPool* pool_;
   OwnedRef dictionary_;
+  bool ordered_;
 };
 
-Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
-                 std::shared_ptr<PandasBlock>* block) {
-#define BLOCK_CASE(NAME, TYPE)                              \
-  case PandasBlock::NAME:                                   \
-    *block = std::make_shared<TYPE>(num_rows, num_columns); \
+Status MakeBlock(PandasOptions options, PandasBlock::type type, int64_t num_rows,
+                 int num_columns, std::shared_ptr<PandasBlock>* block) {
+#define BLOCK_CASE(NAME, TYPE)                                       \
+  case PandasBlock::NAME:                                            \
+    *block = std::make_shared<TYPE>(options, num_rows, num_columns); \
     break;
 
   switch (type) {
@@ -1066,36 +1118,94 @@ Status MakeBlock(PandasBlock::type type, int64_t num_rows, int num_columns,
   return (*block)->Allocate();
 }
 
-static inline Status MakeCategoricalBlock(const std::shared_ptr<DataType>& type,
-                                          int64_t num_rows,
-                                          std::shared_ptr<PandasBlock>* block) {
-  // All categoricals become a block with a single column
-  auto dict_type = static_cast<const DictionaryType*>(type.get());
-  switch (dict_type->index_type()->id()) {
+using BlockMap = std::unordered_map<int, std::shared_ptr<PandasBlock>>;
+
+static Status GetPandasBlockType(const Column& col, const PandasOptions& options,
+                                 PandasBlock::type* output_type) {
+  switch (col.type()->id()) {
+    case Type::BOOL:
+      *output_type = col.null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL;
+      break;
+    case Type::UINT8:
+      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT8;
+      break;
     case Type::INT8:
-      *block = std::make_shared<CategoricalBlock<Type::INT8>>(num_rows);
+      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT8;
+      break;
+    case Type::UINT16:
+      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT16;
       break;
     case Type::INT16:
-      *block = std::make_shared<CategoricalBlock<Type::INT16>>(num_rows);
+      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT16;
+      break;
+    case Type::UINT32:
+      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT32;
       break;
     case Type::INT32:
-      *block = std::make_shared<CategoricalBlock<Type::INT32>>(num_rows);
+      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT32;
       break;
     case Type::INT64:
-      *block = std::make_shared<CategoricalBlock<Type::INT64>>(num_rows);
+      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT64;
+      break;
+    case Type::UINT64:
+      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT64;
+      break;
+    case Type::FLOAT:
+      *output_type = PandasBlock::FLOAT;
+      break;
+    case Type::DOUBLE:
+      *output_type = PandasBlock::DOUBLE;
+      break;
+    case Type::STRING:
+    case Type::BINARY:
+      if (options.strings_to_categorical) {
+        *output_type = PandasBlock::CATEGORICAL;
+        break;
+      }
+    case Type::NA:
+    case Type::FIXED_SIZE_BINARY:
+    case Type::STRUCT:
+    case Type::TIME32:
+    case Type::TIME64:
+    case Type::DECIMAL:
+      *output_type = PandasBlock::OBJECT;
+      break;
+    case Type::DATE32:
+      *output_type = PandasBlock::DATETIME;
+      break;
+    case Type::DATE64:
+      *output_type = PandasBlock::DATETIME;
+      break;
+    case Type::TIMESTAMP: {
+      const auto& ts_type = static_cast<const TimestampType&>(*col.type());
+      if (ts_type.timezone() != "") {
+        *output_type = PandasBlock::DATETIME_WITH_TZ;
+      } else {
+        *output_type = PandasBlock::DATETIME;
+      }
+    } break;
+    case Type::LIST: {
+      auto list_type = std::static_pointer_cast<ListType>(col.type());
+      if (!ListTypeSupported(*list_type->value_type())) {
+        std::stringstream ss;
+        ss << "Not implemented type for list in DataFrameBlock: "
+           << list_type->value_type()->ToString();
+        return Status::NotImplemented(ss.str());
+      }
+      *output_type = PandasBlock::OBJECT;
+    } break;
+    case Type::DICTIONARY:
+      *output_type = PandasBlock::CATEGORICAL;
       break;
-    default: {
+    default:
       std::stringstream ss;
-      ss << "Categorical index type not implemented: "
-         << dict_type->index_type()->ToString();
+      ss << "No known equivalent Pandas block for Arrow data of type ";
+      ss << col.type()->ToString() << " is known.";
       return Status::NotImplemented(ss.str());
-    }
   }
-  return (*block)->Allocate();
+  return Status::OK();
 }
 
-using BlockMap = std::unordered_map<int, std::shared_ptr<PandasBlock>>;
-
 // Construct the exact pandas 0.x "BlockManager" memory layout
 //
 // * For each column determine the correct output pandas type
@@ -1105,7 +1215,9 @@ using BlockMap = std::unordered_map<int, std::shared_ptr<PandasBlock>>;
 // * placement arrays as we go
 class DataFrameBlockCreator {
  public:
-  explicit DataFrameBlockCreator(const std::shared_ptr<Table>& table) : table_(table) {}
+  explicit DataFrameBlockCreator(const PandasOptions& options,
+                                 const std::shared_ptr<Table>& table, MemoryPool* pool)
+      : table_(table), options_(options), pool_(pool) {}
 
   Status Convert(int nthreads, PyObject** output) {
     column_types_.resize(table_->num_columns());
@@ -1123,94 +1235,17 @@ class DataFrameBlockCreator {
     for (int i = 0; i < table_->num_columns(); ++i) {
       std::shared_ptr<Column> col = table_->column(i);
       PandasBlock::type output_type;
-
-      Type::type column_type = col->type()->id();
-      switch (column_type) {
-        case Type::BOOL:
-          output_type = col->null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL;
-          break;
-        case Type::UINT8:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT8;
-          break;
-        case Type::INT8:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT8;
-          break;
-        case Type::UINT16:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT16;
-          break;
-        case Type::INT16:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT16;
-          break;
-        case Type::UINT32:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT32;
-          break;
-        case Type::INT32:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT32;
-          break;
-        case Type::INT64:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT64;
-          break;
-        case Type::UINT64:
-          output_type = col->null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT64;
-          break;
-        case Type::FLOAT:
-          output_type = PandasBlock::FLOAT;
-          break;
-        case Type::DOUBLE:
-          output_type = PandasBlock::DOUBLE;
-          break;
-        case Type::NA:
-        case Type::STRING:
-        case Type::BINARY:
-        case Type::FIXED_SIZE_BINARY:
-        case Type::STRUCT:
-        case Type::TIME32:
-        case Type::TIME64:
-        case Type::DECIMAL:
-          output_type = PandasBlock::OBJECT;
-          break;
-        case Type::DATE32:
-          output_type = PandasBlock::DATETIME;
-          break;
-        case Type::DATE64:
-          output_type = PandasBlock::DATETIME;
-          break;
-        case Type::TIMESTAMP: {
-          const auto& ts_type = static_cast<const TimestampType&>(*col->type());
-          if (ts_type.timezone() != "") {
-            output_type = PandasBlock::DATETIME_WITH_TZ;
-          } else {
-            output_type = PandasBlock::DATETIME;
-          }
-        } break;
-        case Type::LIST: {
-          auto list_type = std::static_pointer_cast<ListType>(col->type());
-          if (!ListTypeSupported(*list_type->value_type())) {
-            std::stringstream ss;
-            ss << "Not implemented type for list in DataFrameBlock: "
-               << list_type->value_type()->ToString();
-            return Status::NotImplemented(ss.str());
-          }
-          output_type = PandasBlock::OBJECT;
-        } break;
-        case Type::DICTIONARY:
-          output_type = PandasBlock::CATEGORICAL;
-          break;
-        default:
-          std::stringstream ss;
-          ss << "No known equivalent Pandas block for Arrow data of type ";
-          ss << col->type()->ToString() << " is known.";
-          return Status::NotImplemented(ss.str());
-      }
+      RETURN_NOT_OK(GetPandasBlockType(*col, options_, &output_type));
 
       int block_placement = 0;
       std::shared_ptr<PandasBlock> block;
       if (output_type == PandasBlock::CATEGORICAL) {
-        RETURN_NOT_OK(MakeCategoricalBlock(col->type(), table_->num_rows(), &block));
+        block = std::make_shared<CategoricalBlock>(options_, pool_, table_->num_rows());
         categorical_blocks_[i] = block;
       } else if (output_type == PandasBlock::DATETIME_WITH_TZ) {
         const auto& ts_type = static_cast<const TimestampType&>(*col->type());
-        block = std::make_shared<DatetimeTZBlock>(ts_type.timezone(), table_->num_rows());
+        block = std::make_shared<DatetimeTZBlock>(options_, ts_type.timezone(),
+                                                  table_->num_rows());
         RETURN_NOT_OK(block->Allocate());
         datetimetz_blocks_[i] = block;
       } else {
@@ -1224,92 +1259,61 @@ class DataFrameBlockCreator {
           type_counts_[output_type] = 1;
         }
       }
-
       column_types_[i] = output_type;
       column_block_placement_[i] = block_placement;
     }
 
     // Create normal non-categorical blocks
-    for (const auto& it : type_counts_) {
+    for (const auto& it : this->type_counts_) {
       PandasBlock::type type = static_cast<PandasBlock::type>(it.first);
       std::shared_ptr<PandasBlock> block;
-      RETURN_NOT_OK(MakeBlock(type, table_->num_rows(), it.second, &block));
-      blocks_[type] = block;
+      RETURN_NOT_OK(
+          MakeBlock(this->options_, type, this->table_->num_rows(), it.second, &block));
+      this->blocks_[type] = block;
     }
     return Status::OK();
   }
 
-  Status WriteTableToBlocks(int nthreads) {
-    auto WriteColumn = [this](int i) {
-      std::shared_ptr<Column> col = this->table_->column(i);
-      PandasBlock::type output_type = this->column_types_[i];
+  Status GetBlock(int i, std::shared_ptr<PandasBlock>* block) {
+    PandasBlock::type output_type = this->column_types_[i];
 
-      int rel_placement = this->column_block_placement_[i];
+    if (output_type == PandasBlock::CATEGORICAL) {
+      auto it = this->categorical_blocks_.find(i);
+      if (it == this->blocks_.end()) {
+        return Status::KeyError("No categorical block allocated");
+      }
+      *block = it->second;
+    } else if (output_type == PandasBlock::DATETIME_WITH_TZ) {
+      auto it = this->datetimetz_blocks_.find(i);
+      if (it == this->datetimetz_blocks_.end()) {
+        return Status::KeyError("No datetimetz block allocated");
+      }
+      *block = it->second;
+    } else {
+      auto it = this->blocks_.find(output_type);
+      if (it == this->blocks_.end()) {
+        return Status::KeyError("No block allocated");
+      }
+      *block = it->second;
+    }
+    return Status::OK();
+  }
 
+  Status WriteTableToBlocks(int nthreads) {
+    auto WriteColumn = [this](int i) {
       std::shared_ptr<PandasBlock> block;
-      if (output_type == PandasBlock::CATEGORICAL) {
-        auto it = this->categorical_blocks_.find(i);
-        if (it == this->blocks_.end()) {
-          return Status::KeyError("No categorical block allocated");
-        }
-        block = it->second;
-      } else if (output_type == PandasBlock::DATETIME_WITH_TZ) {
-        auto it = this->datetimetz_blocks_.find(i);
-        if (it == this->datetimetz_blocks_.end()) {
-          return Status::KeyError("No datetimetz block allocated");
-        }
-        block = it->second;
-      } else {
-        auto it = this->blocks_.find(output_type);
-        if (it == this->blocks_.end()) {
-          return Status::KeyError("No block allocated");
-        }
-        block = it->second;
-      }
-      return block->Write(col, i, rel_placement);
+      RETURN_NOT_OK(this->GetBlock(i, &block));
+      return block->Write(this->table_->column(i), i, this->column_block_placement_[i]);
     };
 
-    nthreads = std::min<int>(nthreads, table_->num_columns());
-
+    int num_tasks = table_->num_columns();
+    nthreads = std::min<int>(nthreads, num_tasks);
     if (nthreads == 1) {
-      for (int i = 0; i < table_->num_columns(); ++i) {
+      for (int i = 0; i < num_tasks; ++i) {
         RETURN_NOT_OK(WriteColumn(i));
       }
     } else {
-      std::vector<std::thread> thread_pool;
-      thread_pool.reserve(nthreads);
-      std::atomic<int> task_counter(0);
-
-      std::mutex error_mtx;
-      bool error_occurred = false;
-      Status error;
-
-      for (int thread_id = 0; thread_id < nthreads; ++thread_id) {
-        thread_pool.emplace_back(
-            [this, &error, &error_occurred, &error_mtx, &task_counter, &WriteColumn]() {
-              int column_num;
-              while (!error_occurred) {
-                column_num = task_counter.fetch_add(1);
-                if (column_num >= this->table_->num_columns()) {
-                  break;
-                }
-                Status s = WriteColumn(column_num);
-                if (!s.ok()) {
-                  std::lock_guard<std::mutex> lock(error_mtx);
-                  error_occurred = true;
-                  error = s;
-                  break;
-                }
-              }
-            });
-      }
-      for (auto&& thread : thread_pool) {
-        thread.join();
-      }
-
-      if (error_occurred) {
-        return error;
-      }
+      RETURN_NOT_OK(ParallelFor(nthreads, num_tasks, WriteColumn));
     }
     return Status::OK();
   }
@@ -1354,6 +1358,11 @@ class DataFrameBlockCreator {
   // block type -> type count
   std::unordered_map<int, int> type_counts_;
 
+  PandasOptions options_;
+
+  // Memory pool for dictionary encoding
+  MemoryPool* pool_;
+
   // block type -> block
   BlockMap blocks_;
 
@@ -1366,8 +1375,9 @@ class DataFrameBlockCreator {
 
 class ArrowDeserializer {
  public:
-  ArrowDeserializer(const std::shared_ptr<Column>& col, PyObject* py_ref)
-      : col_(col), data_(*col->data().get()), py_ref_(py_ref) {}
+  ArrowDeserializer(PandasOptions options, const std::shared_ptr<Column>& col,
+                    PyObject* py_ref)
+      : col_(col), data_(*col->data().get()), options_(options), py_ref_(py_ref) {}
 
   Status AllocateOutput(int type) {
     PyAcquireGIL lock;
@@ -1378,7 +1388,8 @@ class ArrowDeserializer {
   }
 
   template <int TYPE>
-  Status ConvertValuesZeroCopy(int npy_type, std::shared_ptr<Array> arr) {
+  Status ConvertValuesZeroCopy(PandasOptions options, int npy_type,
+                               std::shared_ptr<Array> arr) {
     typedef typename internal::arrow_traits<TYPE>::T T;
 
     const auto& prim_arr = static_cast<const PrimitiveArray&>(*arr);
@@ -1429,7 +1440,7 @@ class ArrowDeserializer {
     int npy_type = traits::npy_type;
 
     if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
-      return ConvertValuesZeroCopy<TYPE>(npy_type, data_.chunk(0));
+      return ConvertValuesZeroCopy<TYPE>(options_, npy_type, data_.chunk(0));
     }
 
     RETURN_NOT_OK(AllocateOutput(npy_type));
@@ -1482,17 +1493,17 @@ class ArrowDeserializer {
     typedef typename traits::T T;
 
     if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
-      return ConvertValuesZeroCopy<TYPE>(traits::npy_type, data_.chunk(0));
+      return ConvertValuesZeroCopy<TYPE>(options_, traits::npy_type, data_.chunk(0));
     }
 
     if (data_.null_count() > 0) {
       RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
       auto out_values = reinterpret_cast<double*>(PyArray_DATA(arr_));
-      ConvertIntegerWithNulls<T>(data_, out_values);
+      ConvertIntegerWithNulls<T>(options_, data_, out_values);
     } else {
       RETURN_NOT_OK(AllocateOutput(traits::npy_type));
       auto out_values = reinterpret_cast<T*>(PyArray_DATA(arr_));
-      ConvertIntegerNoNullsSameType<T>(data_, out_values);
+      ConvertIntegerNoNullsSameType<T>(options_, data_, out_values);
     }
 
     return Status::OK();
@@ -1502,7 +1513,7 @@ class ArrowDeserializer {
   inline Status VisitObjects(FUNCTOR func) {
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
-    return func(data_, out_values);
+    return func(options_, data_, out_values);
   }
 
   // UTF8 strings
@@ -1534,7 +1545,7 @@ class ArrowDeserializer {
     } else {
       RETURN_NOT_OK(AllocateOutput(internal::arrow_traits<Type::BOOL>::npy_type));
       auto out_values = reinterpret_cast<uint8_t*>(PyArray_DATA(arr_));
-      ConvertBooleanNoNulls(data_, out_values);
+      ConvertBooleanNoNulls(options_, data_, out_values);
     }
     return Status::OK();
   }
@@ -1542,7 +1553,7 @@ class ArrowDeserializer {
   Status Visit(const ListType& type) {
 #define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \
   case Type::ArrowEnum:                                    \
-    return ConvertListsLike<ArrowType>(col_, out_values);
+    return ConvertListsLike<ArrowType>(options_, col_, out_values);
 
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
@@ -1572,8 +1583,7 @@ class ArrowDeserializer {
   }
 
   Status Visit(const DictionaryType& type) {
-    std::shared_ptr<PandasBlock> block;
-    RETURN_NOT_OK(MakeCategoricalBlock(col_->type(), col_->length(), &block));
+    auto block = std::make_shared<CategoricalBlock>(options_, nullptr, col_->length());
     RETURN_NOT_OK(block->Write(col_, 0, 0));
 
     auto dict_type = static_cast<const DictionaryType*>(col_->type().get());
@@ -1587,7 +1597,8 @@ class ArrowDeserializer {
     // Release GIL before calling ConvertArrayToPandas, will be reacquired
     // there if needed
     lock.release();
-    RETURN_NOT_OK(ConvertArrayToPandas(dict_type->dictionary(), nullptr, &dictionary));
+    RETURN_NOT_OK(
+        ConvertArrayToPandas(options_, dict_type->dictionary(), nullptr, &dictionary));
     lock.acquire();
 
     PyDict_SetItemString(result_, "indices", block->block_arr());
@@ -1607,28 +1618,29 @@ class ArrowDeserializer {
  private:
   std::shared_ptr<Column> col_;
   const ChunkedArray& data_;
+  PandasOptions options_;
   PyObject* py_ref_;
   PyArrayObject* arr_;
   PyObject* result_;
 };
 
-Status ConvertArrayToPandas(const std::shared_ptr<Array>& arr, PyObject* py_ref,
-                            PyObject** out) {
+Status ConvertArrayToPandas(PandasOptions options, const std::shared_ptr<Array>& arr,
+                            PyObject* py_ref, PyObject** out) {
   static std::string dummy_name = "dummy";
   auto field = std::make_shared<Field>(dummy_name, arr->type());
   auto col = std::make_shared<Column>(field, arr);
-  return ConvertColumnToPandas(col, py_ref, out);
+  return ConvertColumnToPandas(options, col, py_ref, out);
 }
 
-Status ConvertColumnToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
-                             PyObject** out) {
-  ArrowDeserializer converter(col, py_ref);
+Status ConvertColumnToPandas(PandasOptions options, const std::shared_ptr<Column>& col,
+                             PyObject* py_ref, PyObject** out) {
+  ArrowDeserializer converter(options, col, py_ref);
   return converter.Convert(out);
 }
 
-Status ConvertTableToPandas(const std::shared_ptr<Table>& table, int nthreads,
-                            PyObject** out) {
-  DataFrameBlockCreator helper(table);
+Status ConvertTableToPandas(PandasOptions options, const std::shared_ptr<Table>& table,
+                            int nthreads, MemoryPool* pool, PyObject** out) {
+  DataFrameBlockCreator helper(options, table, pool);
   return helper.Convert(nthreads, out);
 }
 
diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h
index 5a99274a33ee0..1d716a5c94fa6 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.h
+++ b/cpp/src/arrow/python/arrow_to_pandas.h
@@ -39,18 +39,18 @@ class Table;
 
 namespace py {
 
-ARROW_EXPORT
-Status ConvertArrayToPandas(const std::shared_ptr<Array>& arr, PyObject* py_ref,
-                            PyObject** out);
-
-ARROW_EXPORT
-Status ConvertColumnToPandas(const std::shared_ptr<Column>& col, PyObject* py_ref,
-                             PyObject** out);
-
 struct PandasOptions {
   bool strings_to_categorical;
 };
 
+ARROW_EXPORT
+Status ConvertArrayToPandas(PandasOptions options, const std::shared_ptr<Array>& arr,
+                            PyObject* py_ref, PyObject** out);
+
+ARROW_EXPORT
+Status ConvertColumnToPandas(PandasOptions options, const std::shared_ptr<Column>& col,
+                             PyObject* py_ref, PyObject** out);
+
 // Convert a whole table as efficiently as possible to a pandas.DataFrame.
 //
 // The returned Python object is a list of tuples consisting of the exact 2D
@@ -58,8 +58,8 @@ struct PandasOptions {
 //
 // tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2])
 ARROW_EXPORT
-Status ConvertTableToPandas(const std::shared_ptr<Table>& table, int nthreads,
-                            PyObject** out);
+Status ConvertTableToPandas(PandasOptions options, const std::shared_ptr<Table>& table,
+                            int nthreads, MemoryPool* pool, PyObject** out);
 
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index dd956463fec76..0d830127ee9b0 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -92,7 +92,9 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) {
 
   PyObject* out;
   Py_BEGIN_ALLOW_THREADS;
-  ASSERT_RAISES(UnknownError, ConvertTableToPandas(table, 2, &out));
+  PandasOptions options;
+  MemoryPool* pool = default_memory_pool();
+  ASSERT_RAISES(UnknownError, ConvertTableToPandas(options, table, 2, pool, &out));
   Py_END_ALLOW_THREADS;
 }
 
diff --git a/cpp/src/arrow/util/parallel.h b/cpp/src/arrow/util/parallel.h
new file mode 100644
index 0000000000000..9fec000c3ed35
--- /dev/null
+++ b/cpp/src/arrow/util/parallel.h
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_PARALLEL_H
+#define ARROW_UTIL_PARALLEL_H
+
+#include <atomic>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+#include "arrow/status.h"
+
+namespace arrow {
+
+template <class FUNCTION>
+Status ParallelFor(int nthreads, int num_tasks, FUNCTION&& func) {
+  std::vector<std::thread> thread_pool;
+  thread_pool.reserve(nthreads);
+  std::atomic<int> task_counter(0);
+
+  std::mutex error_mtx;
+  bool error_occurred = false;
+  Status error;
+
+  for (int thread_id = 0; thread_id < nthreads; ++thread_id) {
+    thread_pool.emplace_back(
+        [&num_tasks, &task_counter, &error, &error_occurred, &error_mtx, &func]() {
+          int task_id;
+          while (!error_occurred) {
+            task_id = task_counter.fetch_add(1);
+            if (task_id >= num_tasks) {
+              break;
+            }
+            Status s = func(task_id);
+            if (!s.ok()) {
+              std::lock_guard<std::mutex> lock(error_mtx);
+              error_occurred = true;
+              error = s;
+              break;
+            }
+          }
+        });
+  }
+  for (auto&& thread : thread_pool) {
+    thread.join();
+  }
+  if (error_occurred) {
+    return error;
+  }
+  return Status::OK();
+}
+
+}  // namespace arrow
+
+#endif
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index c0c7ac6da09ed..20e778d068ff8 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -274,10 +274,15 @@ cdef class Array:
 
         return pyarrow_wrap_array(result)
 
-    def to_pandas(self):
+    def to_pandas(self, c_bool strings_to_categorical=False):
         """
         Convert to an array object suitable for use in pandas
 
+        Parameters
+        ----------
+        strings_to_categorical : boolean, default False
+            Encode string (UTF8) and binary types to pandas.Categorical
+
         See also
         --------
         Column.to_pandas
@@ -286,9 +291,12 @@ cdef class Array:
         """
         cdef:
             PyObject* out
+            PandasOptions options
 
+        options = PandasOptions(strings_to_categorical=strings_to_categorical)
         with nogil:
-            check_status(ConvertArrayToPandas(self.sp_array, self, &out))
+            check_status(ConvertArrayToPandas(options, self.sp_array,
+                                              self, &out))
         return wrap_array_output(out)
 
     def to_pylist(self):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 3ea487385de76..eed9640861fac 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -740,14 +740,18 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     CStatus TensorToNdarray(const CTensor& tensor, object base,
                             PyObject** out)
 
-    CStatus ConvertArrayToPandas(const shared_ptr[CArray]& arr,
+    CStatus ConvertArrayToPandas(PandasOptions options,
+                                 const shared_ptr[CArray]& arr,
                                  object py_ref, PyObject** out)
 
-    CStatus ConvertColumnToPandas(const shared_ptr[CColumn]& arr,
+    CStatus ConvertColumnToPandas(PandasOptions options,
+                                  const shared_ptr[CColumn]& arr,
                                   object py_ref, PyObject** out)
 
-    CStatus ConvertTableToPandas(const shared_ptr[CTable]& table,
-                                 int nthreads, PyObject** out)
+    CStatus ConvertTableToPandas(PandasOptions options,
+                                 const shared_ptr[CTable]& table,
+                                 int nthreads, CMemoryPool* pool,
+                                 PyObject** out)
 
     void c_set_default_memory_pool \
         " arrow::py::set_default_memory_pool"(CMemoryPool* pool)\
@@ -767,6 +771,9 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     cdef cppclass PyBytesReader(CBufferReader):
         PyBytesReader(object fo)
 
+    cdef struct PandasOptions:
+        c_bool strings_to_categorical
+
 
 cdef extern from 'arrow/python/init.h':
     int arrow_init_numpy() except -1
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index ddd562238e38a..434b1c9eab90e 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -269,7 +269,7 @@ def maybe_coerce_datetime64(values, dtype, type_, timestamps_to_ms=False):
     return values, type_
 
 
-def table_to_blockmanager(table, nthreads=1):
+def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     import pandas.core.internals as _int
     from pyarrow.compat import DatetimeTZDtype
     import pyarrow.lib as lib
@@ -305,17 +305,16 @@ def table_to_blockmanager(table, nthreads=1):
                 block_table.schema.get_field_index(name)
             )
 
-    result = lib.table_to_blocks(block_table, nthreads)
+    result = lib.table_to_blocks(options, block_table, nthreads, memory_pool)
 
     blocks = []
     for item in result:
         block_arr = item['block']
         placement = item['placement']
         if 'dictionary' in item:
-            ordered = block_table.schema[placement[0]].type.ordered
             cat = pd.Categorical(block_arr,
                                  categories=item['dictionary'],
-                                 ordered=ordered, fastpath=True)
+                                 ordered=item['ordered'], fastpath=True)
             block = _int.make_block(cat, placement=placement,
                                     klass=_int.CategoricalBlock,
                                     fastpath=True)
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index b9b08998b3372..976f4297d5228 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -157,7 +157,7 @@ cdef class Column:
         sp_column.reset(new CColumn(boxed_field.sp_field, arr.sp_array))
         return pyarrow_wrap_column(sp_column)
 
-    def to_pandas(self):
+    def to_pandas(self, strings_to_categorical=False):
         """
         Convert the arrow::Column to a pandas.Series
 
@@ -167,9 +167,13 @@ cdef class Column:
         """
         cdef:
             PyObject* out
+            PandasOptions options
+
+        options = PandasOptions(strings_to_categorical=strings_to_categorical)
 
         with nogil:
-            check_status(libarrow.ConvertColumnToPandas(self.sp_column,
+            check_status(libarrow.ConvertColumnToPandas(options,
+                                                        self.sp_column,
                                                         self, &out))
 
         return pd.Series(wrap_array_output(out), name=self.name)
@@ -580,15 +584,18 @@ cdef class RecordBatch:
         return pyarrow_wrap_batch(batch)
 
 
-def table_to_blocks(Table table, int nthreads):
+def table_to_blocks(PandasOptions options, Table table, int nthreads,
+                    MemoryPool memory_pool):
     cdef:
         PyObject* result_obj
         shared_ptr[CTable] c_table = table.sp_table
+        CMemoryPool* pool
 
+    pool = maybe_unbox_memory_pool(memory_pool)
     with nogil:
         check_status(
             libarrow.ConvertTableToPandas(
-                c_table, nthreads, &result_obj
+                options, c_table, nthreads, pool, &result_obj
             )
         )
 
@@ -790,7 +797,8 @@ cdef class Table:
 
         return pyarrow_wrap_table(c_table)
 
-    def to_pandas(self, nthreads=None):
+    def to_pandas(self, nthreads=None, strings_to_categorical=False,
+                  memory_pool=None):
         """
         Convert the arrow::Table to a pandas DataFrame
 
@@ -800,16 +808,23 @@ cdef class Table:
             For the default, we divide the CPU count by 2 because most modern
             computers have hyperthreading turned on, so doubling the CPU count
             beyond the number of physical cores does not help
+        strings_to_categorical : boolean, default False
+            Encode string (UTF8) and binary types to pandas.Categorical
+        memory_pool: MemoryPool, optional
+            Specific memory pool to use to allocate casted columns
 
         Returns
         -------
         pandas.DataFrame
         """
+        cdef:
+            PandasOptions options
+        options = PandasOptions(strings_to_categorical=strings_to_categorical)
         self._check_nullptr()
         if nthreads is None:
             nthreads = cpu_count()
-
-        mgr = pdcompat.table_to_blockmanager(self, nthreads)
+        mgr = pdcompat.table_to_blockmanager(options, self, memory_pool,
+                                             nthreads)
         return pd.DataFrame(mgr)
 
     def to_pydict(self):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 93058fb0a47b4..8969777b526c0 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -327,7 +327,7 @@ def test_timestamps_notimezone_no_nulls(self):
                 '2006-01-13T12:34:56.432',
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
-            })
+        })
         field = pa.field('datetime64', pa.timestamp('ms'))
         schema = pa.schema([field])
         self._check_pandas_roundtrip(
@@ -342,7 +342,7 @@ def test_timestamps_notimezone_no_nulls(self):
                 '2006-01-13T12:34:56.432539784',
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
-            })
+        })
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
         self._check_pandas_roundtrip(
@@ -369,7 +369,7 @@ def test_timestamps_notimezone_nulls(self):
                 None,
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
-            })
+        })
         field = pa.field('datetime64', pa.timestamp('ms'))
         schema = pa.schema([field])
         self._check_pandas_roundtrip(
@@ -384,7 +384,7 @@ def test_timestamps_notimezone_nulls(self):
                 None,
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
-            })
+        })
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
         self._check_pandas_roundtrip(
@@ -400,7 +400,7 @@ def test_timestamps_with_timezone(self):
                 '2006-01-13T12:34:56.432',
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
-            })
+        })
         df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
                             .to_frame())
         self._check_pandas_roundtrip(df, timestamps_to_ms=True)
@@ -413,7 +413,7 @@ def test_timestamps_with_timezone(self):
                 '2006-01-13T12:34:56.432539784',
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
-            })
+        })
         df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
                             .to_frame())
         self._check_pandas_roundtrip(df, timestamps_to_ms=False)
@@ -462,7 +462,7 @@ def test_date_objects_typed(self):
         table_pandas = table.to_pandas()
 
         ex_values = (np.array(['2017-04-03', '2017-04-04', '2017-04-04',
-                              '2017-04-05'],
+                               '2017-04-05'],
                               dtype='datetime64[D]')
                      .astype('datetime64[ns]'))
         ex_values[1] = pd.NaT.value
@@ -491,10 +491,10 @@ def test_timedelta(self):
         # TODO(jreback): Pandas only support ns resolution
         # Arrow supports ??? for resolution
         df = pd.DataFrame({
-            'timedelta': np.arange(start=0, stop=3*86400000,
+            'timedelta': np.arange(start=0, stop=3 * 86400000,
                                    step=86400000,
                                    dtype='timedelta64[ms]')
-            })
+        })
         pa.Table.from_pandas(df)
 
     def test_column_of_arrays(self):
@@ -920,6 +920,17 @@ def test_decimal_metadata(self):
         assert data_column['numpy_type'] == 'object'
         assert data_column['metadata'] == {'precision': 26, 'scale': 11}
 
+    def test_table_str_to_categorical(self):
+        values = [None, 'a', 'b', np.nan]
+        df = pd.DataFrame({'strings': values})
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
+        table = pa.Table.from_pandas(df, schema=schema)
+
+        result = table.to_pandas(strings_to_categorical=True)
+        expected = pd.DataFrame({'strings': pd.Categorical(values)})
+        tm.assert_frame_equal(result, expected, check_dtype=True)
+
 
 def _pytime_from_micros(val):
     microseconds = val % 1000000

From a9c2f196e244e4073c750cbf07872a5ffccf48d2 Mon Sep 17 00:00:00 2001
From: Matt Darwin <(none)>
Date: Tue, 8 Aug 2017 13:52:21 -0400
Subject: [PATCH 0919/1644] ARROW-1242: [JAVA] - upgrade jackson to mitigate
 security vulnerabilities

As per #872 I am upgrading Jackson to the latest version on the current train (2.7.1 --> 2.7.9)

Author: Matt Darwin <(none)>
Author: Matt <mattdarwin@yahoo.co.uk>

Closes #929 from mattdarwin/ARROW-1242-upgrade-jackson and squashes the following commits:

d0595176 [Matt Darwin] 1242 upgraing jackson to 2.7.9
bc3b6a07 [Matt] Merge pull request #1 from apache/master
---
 java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/pom.xml b/java/pom.xml
index de2113e397e15..5702e960498fa 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -33,7 +33,7 @@
     <dep.slf4j.version>1.7.25</dep.slf4j.version>
     <dep.guava.version>18.0</dep.guava.version>
     <forkCount>2</forkCount>
-    <jackson.version>2.7.1</jackson.version>
+    <jackson.version>2.7.9</jackson.version>
     <hadoop.version>2.7.1</hadoop.version>
     <checkstyle.failOnViolation>false</checkstyle.failOnViolation>
   </properties>

From 7fdbcc60693884f701b40ee74ff86f5da71976af Mon Sep 17 00:00:00 2001
From: Matt Darwin <(none)>
Date: Wed, 9 Aug 2017 09:10:08 -0400
Subject: [PATCH 0920/1644] ARROW-1243: [JAVA] update all libs to latest
 versions

NB this commit excludes Jackson and logback upgrades, since they are dealt with in 871 and 872

Author: Matt Darwin <(none)>
Author: Matt Darwin <matt.darwin@oracle.com>
Author: Matt <mattdarwin@yahoo.co.uk>

Closes #873 from mattdarwin/upgrade-libs and squashes the following commits:

9b51f46e [Matt Darwin] Merge branch 'master' into upgrade-libs
284a4ce3 [Matt Darwin] Merge branch 'master' of https://github.com/apache/arrow
79550b15 [Matt Darwin] rolling back lilith to 0.9.44 since 8 doesn't support java 7
c63eef64 [Matt Darwin] Merge branch 'master' into upgrade-libs
bc3b6a07 [Matt] Merge pull request #1 from apache/master
8599ba06 [Matt Darwin] backing out guava upgrade
80d81e64 [Matt Darwin] downgrading guava to 20 for java 7 compatibility
806f3489 [Matt Darwin] Merge branch 'master' into upgrade-libs
8aafb7e9 [Matt Darwin] correcting indentation in BaseValueVector
94c14698 [Matt Darwin] upgrading netty to 4.0.49
cff5596a [Matt Darwin] reverting to netty 4.0.41.Final
568737d3 [Matt Darwin] switching to Collections from Guava for empty iterator
c194e48d [Matt Darwin] upgraded hppc to 0.7.2
38be468c [Matt Darwin] upgrading libs except jackson and logback
---
 java/memory/pom.xml                                         | 4 ++--
 java/pom.xml                                                | 6 +++---
 java/tools/pom.xml                                          | 4 ++--
 java/vector/pom.xml                                         | 6 +++---
 .../main/java/org/apache/arrow/vector/BaseValueVector.java  | 3 ++-
 5 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 9a8d2d7c9b56f..1070747691cfc 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -23,13 +23,13 @@
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
-      <version>3.0.1</version>
+      <version>3.0.2</version>
     </dependency>
 
     <dependency>
       <groupId>com.carrotsearch</groupId>
       <artifactId>hppc</artifactId>
-      <version>0.7.1</version>
+      <version>0.7.2</version>
     </dependency>
   </dependencies>
 
diff --git a/java/pom.xml b/java/pom.xml
index 5702e960498fa..b0621c5a41ea6 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -454,7 +454,7 @@
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-handler</artifactId>
-      <version>4.0.41.Final</version>
+      <version>4.0.49.Final</version>
     </dependency>
 
     <dependency>
@@ -495,7 +495,7 @@
       <!-- JMockit needs to be on class path before JUnit. -->
       <groupId>com.googlecode.jmockit</groupId>
       <artifactId>jmockit</artifactId>
-      <version>1.3</version>
+      <version>1.7</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -507,7 +507,7 @@
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
-      <version>1.9.5</version>
+      <version>2.7.22</version>
       <scope>test</scope>
       <!-- Mockito  contains old Hamcrest classes, causes issues with JUNIT -->
       <exclusions>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 9d067ef1e9bc2..8aadb44f3f69f 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -38,12 +38,12 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.4</version>
+            <version>3.6</version>
         </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
             <artifactId>commons-cli</artifactId>
-            <version>1.2</version>
+            <version>1.4</version>
         </dependency>
         <dependency>
           <groupId>ch.qos.logback</groupId>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index e15ab9a2497fc..a81bbd0f12cca 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -34,7 +34,7 @@
     <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
-      <version>2.9</version>
+      <version>2.9.9</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
@@ -49,12 +49,12 @@
     <dependency>
       <groupId>com.carrotsearch</groupId>
       <artifactId>hppc</artifactId>
-      <version>0.7.1</version>
+      <version>0.7.2</version>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
-      <version>3.4</version>
+      <version>3.6</version>
     </dependency>
     <dependency>
       <groupId>commons-codec</groupId>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 05d2aa933cbef..598e578e55a6d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -18,6 +18,7 @@
 
 package org.apache.arrow.vector;
 
+import java.util.Collections;
 import java.util.Iterator;
 
 import com.google.flatbuffers.FlatBufferBuilder;
@@ -104,7 +105,7 @@ public void reset() {
 
   @Override
   public Iterator<ValueVector> iterator() {
-    return Iterators.emptyIterator();
+    return Collections.emptyIterator();
   }
 
   public static boolean checkBufRefs(final ValueVector vv) {

From 86154f0be3fbafcd27716f3b3f7058c31242a52f Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Wed, 9 Aug 2017 09:11:52 -0400
Subject: [PATCH 0921/1644] ARROW-1340: [Java] Fix NullableMapVector field
 metadata

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #953 from elahrvivaz/ARROW-1340 and squashes the following commits:

a307779e [Emilio Lahr-Vivaz] ARROW-1340: [Java] Fix NullableMapVector field metadata
---
 .../vector/complex/NullableMapVector.java     |  3 +-
 .../apache/arrow/vector/TestMapVector.java    | 57 +++++++++++++++++++
 2 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index e70a915561f8b..fda9c1471e589 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -86,7 +86,8 @@ public NullableMapVector(String name, BufferAllocator allocator, FieldType field
   @Override
   public Field getField() {
     Field f = super.getField();
-    return new Field(f.getName(), true, f.getType(), f.getChildren());
+    FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata());
+    return new Field(f.getName(), type, f.getChildren());
   }
 
   @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
new file mode 100644
index 0000000000000..357df96aa2efc
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestMapVector {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testFieldMetadata() throws Exception {
+    Map<String, String> metadata = new HashMap<>();
+    metadata.put("k1", "v1");
+    FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata);
+    try (NullableMapVector vector = new NullableMapVector("map", allocator, type, null)) {
+      Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata());
+    }
+  }
+}

From e44ede87c069087e11b4f57682090e01ae06a746 Mon Sep 17 00:00:00 2001
From: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Date: Wed, 9 Aug 2017 09:36:18 -0400
Subject: [PATCH 0922/1644] ARROW-1343: [Java] Aligning serialized schema, end
 of buffers in RecordBatches

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #954 from elahrvivaz/align_end and squashes the following commits:

79ac1204 [Wes McKinney] Revert to NDEBUG because it's a standard define in release builds in MSVC
ae6bc9f2 [Wes McKinney] Use __declspec(noreturn) in MSVC. Not sure why this suddenly showed up
74b29ccf [Wes McKinney] Add notes to IPC.md to make alignment contract more clear
e2f0114b [Wes McKinney] Add C++ DCHECKs on read path for aligned buffers, aligned file block offset, lengths
3d64c9f5 [Wes McKinney] Align stream schema message in C++, DCHECKs for FileBlocks
4778ee1f [Emilio Lahr-Vivaz] adding padding to magic bytes in file format
53429159 [Emilio Lahr-Vivaz] using asserts instead of padding checks, adding padding to ArrowRecordBatch.calculateBodySize, moving align to writeBufferBatches
a12b4ff8 [Emilio Lahr-Vivaz] comments
0b32265b [Emilio Lahr-Vivaz] aligning schema write
26bbc255 [Emilio Lahr-Vivaz] Merge branch 'ARROW-1340' into align_end
a307779e [Emilio Lahr-Vivaz] ARROW-1340: [Java] Fix NullableMapVector field metadata
b2bf86d4 [Emilio Lahr-Vivaz] WIP for aligning end of buffers
---
 cpp/src/arrow/ipc/metadata.cc                 | 15 ++++++++
 cpp/src/arrow/ipc/reader.cc                   | 13 +++++++
 cpp/src/arrow/ipc/util.h                      |  4 +-
 cpp/src/arrow/ipc/writer.cc                   | 37 ++++++++++---------
 cpp/src/arrow/util/bit-util.h                 | 18 ++++++---
 cpp/src/arrow/util/logging.h                  |  6 ++-
 cpp/src/arrow/util/macros.h                   |  6 +++
 format/IPC.md                                 | 10 ++++-
 .../arrow/vector/file/ArrowFileWriter.java    |  4 +-
 .../apache/arrow/vector/file/ArrowMagic.java  |  5 ++-
 .../arrow/vector/schema/ArrowRecordBatch.java |  3 ++
 .../vector/stream/MessageSerializer.java      | 34 +++++++++++++----
 12 files changed, 118 insertions(+), 37 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index faf01a568483a..c9534217e927c 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -36,6 +36,7 @@
 #include "arrow/status.h"
 #include "arrow/tensor.h"
 #include "arrow/type.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 
@@ -773,6 +774,20 @@ Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dicti
   flatbuffers::Offset<flatbuf::Schema> fb_schema;
   RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
 
+#ifndef NDEBUG
+  for (size_t i = 0; i < dictionaries.size(); ++i) {
+    DCHECK(BitUtil::IsMultipleOf8(dictionaries[i].offset)) << i;
+    DCHECK(BitUtil::IsMultipleOf8(dictionaries[i].metadata_length)) << i;
+    DCHECK(BitUtil::IsMultipleOf8(dictionaries[i].body_length)) << i;
+  }
+
+  for (size_t i = 0; i < record_batches.size(); ++i) {
+    DCHECK(BitUtil::IsMultipleOf8(record_batches[i].offset)) << i;
+    DCHECK(BitUtil::IsMultipleOf8(record_batches[i].metadata_length)) << i;
+    DCHECK(BitUtil::IsMultipleOf8(record_batches[i].body_length)) << i;
+  }
+#endif
+
   auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
   auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
 
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 8ae82804c3164..6ea907e0ed09f 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -34,6 +34,7 @@
 #include "arrow/table.h"
 #include "arrow/tensor.h"
 #include "arrow/type.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
 #include "arrow/visitor_inline.h"
 
@@ -59,6 +60,9 @@ class IpcComponentSource {
       *out = nullptr;
       return Status::OK();
     } else {
+      DCHECK(BitUtil::IsMultipleOf8(buffer->offset()))
+          << "Buffer " << buffer_index
+          << " did not start on 8-byte aligned offset: " << buffer->offset();
       return file_->ReadAt(buffer->offset(), buffer->length(), out);
     }
   }
@@ -550,6 +554,10 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
     DCHECK_LT(i, num_record_batches());
     FileBlock block = record_batch(i);
 
+    DCHECK(BitUtil::IsMultipleOf8(block.offset));
+    DCHECK(BitUtil::IsMultipleOf8(block.metadata_length));
+    DCHECK(BitUtil::IsMultipleOf8(block.body_length));
+
     std::unique_ptr<Message> message;
     RETURN_NOT_OK(
         ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
@@ -564,6 +572,11 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
     // Read all the dictionaries
     for (int i = 0; i < num_dictionaries(); ++i) {
       FileBlock block = dictionary(i);
+
+      DCHECK(BitUtil::IsMultipleOf8(block.offset));
+      DCHECK(BitUtil::IsMultipleOf8(block.metadata_length));
+      DCHECK(BitUtil::IsMultipleOf8(block.body_length));
+
       std::unique_ptr<Message> message;
       RETURN_NOT_OK(
           ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
diff --git a/cpp/src/arrow/ipc/util.h b/cpp/src/arrow/ipc/util.h
index 49a7d01ac6f35..412f31215ed65 100644
--- a/cpp/src/arrow/ipc/util.h
+++ b/cpp/src/arrow/ipc/util.h
@@ -27,10 +27,12 @@
 namespace arrow {
 namespace ipc {
 
-// Align on 8-byte boundaries
 // Buffers are padded to 64-byte boundaries (for SIMD)
 static constexpr int kArrowAlignment = 64;
 
+// Align on 8-byte boundaries in IPC
+static constexpr int kArrowIpcAlignment = 8;
+
 static constexpr uint8_t kPaddingBytes[kArrowAlignment] = {0};
 
 static inline int64_t PaddedLength(int64_t nbytes, int64_t alignment = kArrowAlignment) {
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 163b27b443351..bc07dc659f601 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -156,7 +156,7 @@ class RecordBatchSerializer : public ArrayVisitor {
       // The buffer might be null if we are handling zero row lengths.
       if (buffer) {
         size = buffer->size();
-        padding = BitUtil::RoundUpToMultipleOf64(size) - size;
+        padding = BitUtil::RoundUpToMultipleOf8(size) - size;
       }
 
       // TODO(wesm): We currently have no notion of shared memory page id's,
@@ -172,7 +172,7 @@ class RecordBatchSerializer : public ArrayVisitor {
     }
 
     *body_length = offset - buffer_start_offset_;
-    DCHECK(BitUtil::IsMultipleOf64(*body_length));
+    DCHECK(BitUtil::IsMultipleOf8(*body_length));
 
     return Status::OK();
   }
@@ -216,7 +216,7 @@ class RecordBatchSerializer : public ArrayVisitor {
       // The buffer might be null if we are handling zero row lengths.
       if (buffer) {
         size = buffer->size();
-        padding = BitUtil::RoundUpToMultipleOf64(size) - size;
+        padding = BitUtil::RoundUpToMultipleOf8(size) - size;
       }
 
       if (size > 0) {
@@ -251,7 +251,7 @@ class RecordBatchSerializer : public ArrayVisitor {
 
       // Send padding if it's available
       const int64_t buffer_length =
-          std::min(BitUtil::RoundUpToMultipleOf64(array.length() * type_width),
+          std::min(BitUtil::RoundUpToMultipleOf8(array.length() * type_width),
                    data->size() - byte_offset);
       data = SliceBuffer(data, byte_offset, buffer_length);
     }
@@ -618,15 +618,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
   }
 
   virtual Status Start() {
-    std::shared_ptr<Buffer> schema_fb;
-    RETURN_NOT_OK(WriteSchemaMessage(*schema_, &dictionary_memo_, &schema_fb));
-
-    int32_t flatbuffer_size = static_cast<int32_t>(schema_fb->size());
-    RETURN_NOT_OK(
-        Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
-
-    // Write the flatbuffer
-    RETURN_NOT_OK(Write(schema_fb->data(), flatbuffer_size));
+    RETURN_NOT_OK(WriteSchema());
 
     // If there are any dictionaries, write them as the next messages
     RETURN_NOT_OK(WriteDictionaries());
@@ -635,6 +627,17 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
     return Status::OK();
   }
 
+  Status WriteSchema() {
+    std::shared_ptr<Buffer> schema_fb;
+    RETURN_NOT_OK(WriteSchemaMessage(*schema_, &dictionary_memo_, &schema_fb));
+
+    int32_t metadata_length = 0;
+    RETURN_NOT_OK(WriteMessage(*schema_fb, sink_, &metadata_length));
+    RETURN_NOT_OK(UpdatePosition());
+    DCHECK_EQ(0, position_ % 8) << "WriteSchema did not perform an aligned write";
+    return Status::OK();
+  }
+
   virtual Status Close() {
     // Write the schema if not already written
     // User is responsible for closing the OutputStream
@@ -701,9 +704,9 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
                             &record_batches_[record_batches_.size() - 1]);
   }
 
-  // Adds padding bytes if necessary to ensure all memory blocks are written on
-  // 64-byte (or other alignment) boundaries.
-  Status Align(int64_t alignment = kArrowAlignment) {
+  Status Align(int64_t alignment = kArrowIpcAlignment) {
+    // Adds padding bytes if necessary to ensure all memory blocks are written on
+    // 8-byte (or other alignment) boundaries.
     int64_t remainder = PaddedLength(position_, alignment) - position_;
     if (remainder > 0) {
       return Write(kPaddingBytes, remainder);
@@ -774,7 +777,7 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
     // It is only necessary to align to 8-byte boundary at the start of the file
     RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
                         strlen(kArrowMagicBytes)));
-    RETURN_NOT_OK(Align(8));
+    RETURN_NOT_OK(Align());
 
     // We write the schema at the start of the file (and the end). This also
     // writes all the dictionaries at the beginning of the file
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index fc360bae4e451..5c3938aadbc66 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -217,13 +217,13 @@ static inline uint32_t RoundUpNumi64(uint32_t bits) { return (bits + 63) >> 6; }
 /// Returns the rounded down to 64 multiple.
 static inline uint32_t RoundDownNumi64(uint32_t bits) { return bits >> 6; }
 
-static inline int64_t RoundUpToMultipleOf64(int64_t num) {
+template <int64_t ROUND_TO>
+static inline int64_t RoundToPowerOfTwo(int64_t num) {
   // TODO(wesm): is this definitely needed?
   // DCHECK_GE(num, 0);
-  constexpr int64_t round_to = 64;
-  constexpr int64_t force_carry_addend = round_to - 1;
-  constexpr int64_t truncate_bitmask = ~(round_to - 1);
-  constexpr int64_t max_roundable_num = std::numeric_limits<int64_t>::max() - round_to;
+  constexpr int64_t force_carry_addend = ROUND_TO - 1;
+  constexpr int64_t truncate_bitmask = ~(ROUND_TO - 1);
+  constexpr int64_t max_roundable_num = std::numeric_limits<int64_t>::max() - ROUND_TO;
   if (num <= max_roundable_num) {
     return (num + force_carry_addend) & truncate_bitmask;
   }
@@ -231,6 +231,14 @@ static inline int64_t RoundUpToMultipleOf64(int64_t num) {
   return num;
 }
 
+static inline int64_t RoundUpToMultipleOf64(int64_t num) {
+  return RoundToPowerOfTwo<64>(num);
+}
+
+static inline int64_t RoundUpToMultipleOf8(int64_t num) {
+  return RoundToPowerOfTwo<8>(num);
+}
+
 /// Non hw accelerated pop count.
 /// TODO: we don't use this in any perf sensitive code paths currently.  There
 /// might be a much faster way to implement this.
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 89e69f932d52d..998f7ed7bfaaa 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -21,6 +21,8 @@
 #include <cstdlib>
 #include <iostream>
 
+#include "arrow/util/macros.h"
+
 namespace arrow {
 
 // Stubbed versions of macros defined in glog/logging.h, intended for
@@ -127,9 +129,9 @@ class CerrLog {
 class FatalLog : public CerrLog {
  public:
   explicit FatalLog(int /* severity */)  // NOLINT
-      : CerrLog(ARROW_FATAL){}           // NOLINT
+      : CerrLog(ARROW_FATAL) {}          // NOLINT
 
-            [[noreturn]] ~FatalLog() {
+  ARROW_NORETURN ~FatalLog() {
     if (has_logged_) {
       std::cerr << std::endl;
     }
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index a2f704f0c68bc..fe2d7689bf641 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -36,7 +36,13 @@
 #if defined(__GNUC__)
 #define ARROW_PREDICT_FALSE(x) (__builtin_expect(x, 0))
 #define ARROW_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
+#define ARROW_NORETURN __attribute__((noreturn))
+#elif defined(_MSC_VER)
+#define ARROW_NORETURN __declspec(noreturn)
+#define ARROW_PREDICT_FALSE(x) x
+#define ARROW_PREDICT_TRUE(x) x
 #else
+#define ARROW_NORETURN
 #define ARROW_PREDICT_FALSE(x) x
 #define ARROW_PREDICT_TRUE(x) x
 #endif
diff --git a/format/IPC.md b/format/IPC.md
index 3fd234e4aa160..2f79031443b17 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -27,7 +27,7 @@ Data components in the stream and file formats are represented as encapsulated
 * A length prefix indicating the metadata size
 * The message metadata as a [Flatbuffer][3]
 * Padding bytes to an 8-byte boundary
-* The message body
+* The message body, which must be a multiple of 8 bytes
 
 Schematically, we have:
 
@@ -38,6 +38,10 @@ Schematically, we have:
 <message body>
 ```
 
+The complete serialized message must be a multiple of 8 bytes so that messages
+can be relocated between streams. Otherwise the amount of padding between the
+metadata and the message body could be non-deterministic.
+
 The `metadata_size` includes the size of the flatbuffer plus padding. The
 `Message` flatbuffer includes a version number, the particular message (as a
 flatbuffer union), and the size of the message body:
@@ -154,6 +158,10 @@ struct Block {
 }
 ```
 
+The `metaDataLength` here includes the metadata length prefix, serialized
+metadata, and any additional padding bytes, and by construction must be a
+multiple of 8 bytes.
+
 Some notes about this
 
 * The `Block` offset indicates the starting byte of the record batch.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
index 06519bc49fd1c..1d92d2bde1c6f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
@@ -38,7 +38,7 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
 
   @Override
   protected void startInternal(WriteChannel out) throws IOException {
-    ArrowMagic.writeMagic(out);
+    ArrowMagic.writeMagic(out, true);
   }
 
   @Override
@@ -54,7 +54,7 @@ protected void endInternal(WriteChannel out,
     }
     out.writeIntLittleEndian(footerLength);
     LOGGER.debug(String.format("Footer starts at %d, length: %d", footerStart, footerLength));
-    ArrowMagic.writeMagic(out);
+    ArrowMagic.writeMagic(out, false);
     LOGGER.debug(String.format("magic written, now at %d", out.getCurrentPosition()));
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
index 0d2da375295fe..68313e7878b71 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
@@ -28,8 +28,11 @@ public class ArrowMagic {
 
   public static final int MAGIC_LENGTH = MAGIC.length;
 
-  public static void writeMagic(WriteChannel out) throws IOException {
+  public static void writeMagic(WriteChannel out, boolean align) throws IOException {
     out.write(MAGIC);
+    if (align) {
+      out.align();
+    }
   }
 
   public static boolean validateMagic(byte[] array) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index d2f3782469597..c842d4c3f9a74 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -168,6 +168,9 @@ public int computeBodyLength() {
       ByteBuffer nioBuffer =
           buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes());
       size += nioBuffer.remaining();
+      if (size % 8 != 0) {
+        size += 8 - (size % 8);
+      }
     }
     return size;
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index a70d029389427..f69aa41e7f6bd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -78,12 +78,25 @@ public static int bytesToInt(byte[] bytes) {
    * @throws IOException if something went wrong
    */
   public static long serialize(WriteChannel out, Schema schema) throws IOException {
+    long start = out.getCurrentPosition();
+    assert start % 8 == 0;
+
     FlatBufferBuilder builder = new FlatBufferBuilder();
     int schemaOffset = schema.getSchema(builder);
     ByteBuffer serializedMessage = serializeMessage(builder, MessageHeader.Schema, schemaOffset, 0);
-    long size = out.writeIntLittleEndian(serializedMessage.remaining());
-    size += out.write(serializedMessage);
-    return size;
+
+    int size = serializedMessage.remaining();
+    // ensure that message aligns to 8 byte padding - 4 bytes for size, then message body
+    if ((size + 4) % 8 != 0) {
+      size += 8 - (size + 4) % 8;
+    }
+
+    out.writeIntLittleEndian(size);
+    out.write(serializedMessage);
+    out.align(); // any bytes written are already captured by our size modification above
+
+    assert (size + 4) % 8 == 0;
+    return size + 4;
   }
 
   /**
@@ -120,6 +133,7 @@ public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
 
     long start = out.getCurrentPosition();
     int bodyLength = batch.computeBodyLength();
+    assert bodyLength % 8 == 0;
 
     FlatBufferBuilder builder = new FlatBufferBuilder();
     int batchOffset = batch.writeTo(builder);
@@ -141,6 +155,7 @@ public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch)
     out.align();
 
     long bufferLength = writeBatchBuffers(out, batch);
+    assert bufferLength % 8 == 0;
 
     // Metadata size in the Block account for the size prefix
     return new ArrowBlock(start, metadataLength + 4, bufferLength);
@@ -164,6 +179,7 @@ public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) t
             " != " + startPosition + layout.getSize());
       }
     }
+    out.align();
     return out.getCurrentPosition() - bufferStart;
   }
 
@@ -268,6 +284,7 @@ public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
   public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch) throws IOException {
     long start = out.getCurrentPosition();
     int bodyLength = batch.computeBodyLength();
+    assert bodyLength % 8 == 0;
 
     FlatBufferBuilder builder = new FlatBufferBuilder();
     int batchOffset = batch.writeTo(builder);
@@ -276,10 +293,10 @@ public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch)
 
     int metadataLength = serializedMessage.remaining();
 
-    // Add extra padding bytes so that length prefix + metadata is a multiple
-    // of 8 after alignment
-    if ((start + metadataLength + 4) % 8 != 0) {
-      metadataLength += 8 - (start + metadataLength + 4) % 8;
+    // calculate alignment bytes so that metadata length points to the correct location after alignment
+    int padding = (int) ((start + metadataLength + 4) % 8);
+    if (padding != 0) {
+      metadataLength += (8 - padding);
     }
 
     out.writeIntLittleEndian(metadataLength);
@@ -290,9 +307,10 @@ public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch)
 
     // write the embedded record batch
     long bufferLength = writeBatchBuffers(out, batch.getDictionary());
+    assert bufferLength % 8 == 0;
 
     // Metadata size in the Block account for the size prefix
-    return new ArrowBlock(start, metadataLength + 4, bufferLength + 8);
+    return new ArrowBlock(start, metadataLength + 4, bufferLength);
   }
 
   /**

From 2972c9d3a0d371dbdcf69c68a0109b83aa6fd944 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 9 Aug 2017 17:47:30 -0400
Subject: [PATCH 0923/1644] ARROW-1342: [Python] Support strided ndarrays in
 pandas conversion from nested lists

This does drop the vector append to the builder that was there before. I'm going to do some local benchmarking to make sure this doesn't degrade performance unacceptably, will report back here

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #956 from wesm/ARROW-1342 and squashes the following commits:

f2ebeba8 [Wes McKinney] Fix cpplint issue
f403f9dd [Wes McKinney] Fix test case to be platform independent, note ARROW-1345. Improve quality of error message
f4f44c18 [Wes McKinney] Fix test case where inferred list type is null
ae5c8312 [Wes McKinney] Drop striding check
b4aecd3a [Wes McKinney] Support strided ndarrays in pandas conversion from nested lists
---
 cpp/src/arrow/python/numpy-internal.h       | 41 +++++++++++++
 cpp/src/arrow/python/pandas_to_arrow.cc     | 67 +++++++++------------
 cpp/src/arrow/table.cc                      |  4 +-
 python/pyarrow/error.pxi                    |  3 +-
 python/pyarrow/includes/common.pxd          |  1 +
 python/pyarrow/tests/pandas_examples.py     | 10 ++-
 python/pyarrow/tests/test_convert_pandas.py |  9 +++
 7 files changed, 91 insertions(+), 44 deletions(-)

diff --git a/cpp/src/arrow/python/numpy-internal.h b/cpp/src/arrow/python/numpy-internal.h
index f1ef7dadde084..db34d24d99da5 100644
--- a/cpp/src/arrow/python/numpy-internal.h
+++ b/cpp/src/arrow/python/numpy-internal.h
@@ -25,6 +25,7 @@
 #include "arrow/python/platform.h"
 
 #include <cstdint>
+#include <string>
 
 namespace arrow {
 namespace py {
@@ -51,7 +52,12 @@ class Ndarray1DIndexer {
 
   int64_t size() const { return PyArray_SIZE(arr_); }
 
+  T* data() const { return data_; }
+
+  bool is_strided() const { return stride_ == 1; }
+
   T& operator[](size_type index) { return *(data_ + index * stride_); }
+  T& operator[](size_type index) const { return *(data_ + index * stride_); }
 
  private:
   PyArrayObject* arr_;
@@ -59,6 +65,41 @@ class Ndarray1DIndexer {
   int64_t stride_;
 };
 
+static inline std::string GetNumPyTypeName(int npy_type) {
+#define TYPE_CASE(TYPE, NAME) \
+  case NPY_##TYPE:            \
+    return NAME;
+
+  switch (npy_type) {
+    TYPE_CASE(BOOL, "bool")
+    TYPE_CASE(INT8, "int8")
+    TYPE_CASE(INT16, "int16")
+    TYPE_CASE(INT32, "int32")
+    TYPE_CASE(INT64, "int64")
+#if (NPY_INT64 != NPY_LONGLONG)
+    TYPE_CASE(LONGLONG, "longlong")
+#endif
+    TYPE_CASE(UINT8, "uint8")
+    TYPE_CASE(UINT16, "uint16")
+    TYPE_CASE(UINT32, "uint32")
+    TYPE_CASE(UINT64, "uint64")
+#if (NPY_UINT64 != NPY_ULONGLONG)
+    TYPE_CASE(ULONGLONG, "ulonglong")
+#endif
+    TYPE_CASE(FLOAT16, "float16")
+    TYPE_CASE(FLOAT32, "float32")
+    TYPE_CASE(FLOAT64, "float64")
+    TYPE_CASE(DATETIME, "datetime64")
+    TYPE_CASE(OBJECT, "object")
+    TYPE_CASE(VOID, "void")
+    default:
+      break;
+  }
+
+#undef TYPE_CASE
+  return "unrecognized type in GetNumPyTypeName";
+}
+
 }  // namespace py
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 060fcb2453800..b6cc16b4179b3 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -97,8 +97,6 @@ static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
   int64_t null_count = 0;
 
   Ndarray1DIndexer<T> values(arr);
-
-  // TODO(wesm): striding
   for (int i = 0; i < values.size(); ++i) {
     if (traits::isnull(values[i])) {
       ++null_count;
@@ -125,22 +123,27 @@ static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap
   return null_count;
 }
 
-template <int TYPE>
-static int64_t ValuesToValidBytes(const void* data, int64_t length,
-                                  uint8_t* valid_bytes) {
+template <int TYPE, typename BuilderType>
+static Status AppendNdarrayToBuilder(PyArrayObject* array, BuilderType* builder) {
   typedef internal::npy_traits<TYPE> traits;
   typedef typename traits::value_type T;
 
-  int64_t null_count = 0;
-  const T* values = reinterpret_cast<const T*>(data);
-
-  // TODO(wesm): striding
-  for (int i = 0; i < length; ++i) {
-    valid_bytes[i] = !traits::isnull(values[i]);
-    if (traits::isnull(values[i])) null_count++;
+  // TODO(wesm): Vector append when not strided
+  Ndarray1DIndexer<T> values(array);
+  if (traits::supports_nulls) {
+    for (int64_t i = 0; i < values.size(); ++i) {
+      if (traits::isnull(values[i])) {
+        RETURN_NOT_OK(builder->AppendNull());
+      } else {
+        RETURN_NOT_OK(builder->Append(values[i]));
+      }
+    }
+  } else {
+    for (int64_t i = 0; i < values.size(); ++i) {
+      RETURN_NOT_OK(builder->Append(values[i]));
+    }
   }
-
-  return null_count;
+  return Status::OK();
 }
 
 Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
@@ -148,14 +151,14 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
     return Status::Invalid("only handle 1-dimensional arrays");
   }
 
-  if (PyArray_DESCR(numpy_array)->type_num != np_type) {
-    return Status::Invalid("can only handle exact conversions");
+  const int received_type = PyArray_DESCR(numpy_array)->type_num;
+  if (received_type != np_type) {
+    std::stringstream ss;
+    ss << "trying to convert NumPy type " << GetNumPyTypeName(np_type) << " but got "
+       << GetNumPyTypeName(received_type);
+    return Status::Invalid(ss.str());
   }
 
-  npy_intp* astrides = PyArray_STRIDES(numpy_array);
-  if (astrides[0] != PyArray_DESCR(numpy_array)->elsize) {
-    return Status::Invalid("No support for strided arrays in lists yet");
-  }
   return Status::OK();
 }
 
@@ -577,7 +580,7 @@ Status PandasConverter::ConvertDecimals() {
   RETURN_NOT_OK(ImportModule("decimal", &decimal));
   RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
 
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  Ndarray1DIndexer<PyObject*> objects(arr_);
   PyObject* object = objects[0];
 
   int precision;
@@ -618,7 +621,7 @@ Status PandasConverter::ConvertTimes() {
   PyAcquireGIL lock;
   PyDateTime_IMPORT;
 
-  PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
+  Ndarray1DIndexer<PyObject*> objects(arr_);
 
   // datetime.time stores microsecond resolution
   Time64Builder builder(::arrow::time64(TimeUnit::MICRO), pool_);
@@ -906,7 +909,7 @@ Status LoopPySequence(PyObject* sequence, T func) {
     Py_ssize_t size = PySequence_Size(sequence);
     if (PyArray_Check(sequence)) {
       auto array = reinterpret_cast<PyArrayObject*>(sequence);
-      PyObject** objects = reinterpret_cast<PyObject**>(PyArray_DATA(array));
+      Ndarray1DIndexer<PyObject*> objects(array);
       for (int64_t i = 0; i < size; ++i) {
         RETURN_NOT_OK(func(objects[i]));
       }
@@ -934,7 +937,6 @@ template <int ITEM_TYPE, typename ArrowType>
 inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>& type,
                                                  ListBuilder* builder, PyObject* list) {
   typedef internal::npy_traits<ITEM_TYPE> traits;
-  typedef typename traits::value_type T;
   typedef typename traits::BuilderClass BuilderT;
 
   PyAcquireGIL lock;
@@ -956,24 +958,13 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
       // TODO(uwe): Support more complex numpy array structures
       RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, ITEM_TYPE));
 
-      int64_t size = PyArray_DIM(numpy_array, 0);
-      auto data = reinterpret_cast<const T*>(PyArray_DATA(numpy_array));
-      if (traits::supports_nulls) {
-        RETURN_NOT_OK(null_bitmap_->Resize(size, false));
-        // TODO(uwe): A bitmap would be more space-efficient but the Builder API doesn't
-        // currently support this.
-        // ValuesToBitmap<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        ValuesToValidBytes<ITEM_TYPE>(data, size, null_bitmap_->mutable_data());
-        return value_builder->Append(data, size, null_bitmap_->data());
-      } else {
-        return value_builder->Append(data, size);
-      }
+      return AppendNdarrayToBuilder<ITEM_TYPE, BuilderT>(numpy_array, value_builder);
     } else if (PyList_Check(object)) {
       int64_t size;
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(builder->Append(true));
       RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
-      if (inferred_type->id() != type->id()) {
+      if (inferred_type->id() != Type::NA && inferred_type->id() != type->id()) {
         std::stringstream ss;
         ss << inferred_type->ToString() << " cannot be converted to " << type->ToString();
         return Status::TypeError(ss.str());
@@ -1064,7 +1055,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(builder->Append(true));
       RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
-      if (inferred_type->id() != Type::STRING) {
+      if (inferred_type->id() != Type::NA && inferred_type->id() != Type::STRING) {
         std::stringstream ss;
         ss << inferred_type->ToString() << " cannot be converted to STRING.";
         return Status::TypeError(ss.str());
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 1f0c6d785448d..ae486987601bf 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -301,8 +301,8 @@ Table::Table(const std::shared_ptr<Schema>& schema,
 
   columns_.resize(columns.size());
   for (size_t i = 0; i < columns.size(); ++i) {
-    columns_[i] = std::make_shared<Column>(schema->field(static_cast<int>(i)),
-                                           columns[i]);
+    columns_[i] =
+        std::make_shared<Column>(schema->field(static_cast<int>(i)), columns[i]);
   }
 }
 
diff --git a/python/pyarrow/error.pxi b/python/pyarrow/error.pxi
index 8a3f57d209ac0..8793c4e90f6b4 100644
--- a/python/pyarrow/error.pxi
+++ b/python/pyarrow/error.pxi
@@ -65,7 +65,7 @@ cdef int check_status(const CStatus& status) nogil except -1:
         return 0
 
     with gil:
-        message = frombytes(status.ToString())
+        message = frombytes(status.message())
         if status.IsInvalid():
             raise ArrowInvalid(message)
         elif status.IsIOError():
@@ -85,4 +85,5 @@ cdef int check_status(const CStatus& status) nogil except -1:
         elif status.IsPlasmaStoreFull():
             raise PlasmaStoreFull(message)
         else:
+            message = frombytes(status.ToString())
             raise ArrowException(message)
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 637a133afb02b..6be08b0e59256 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -42,6 +42,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CStatus()
 
         c_string ToString()
+        c_string message()
 
         c_bool ok()
         c_bool IsIOError()
diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py
index 17ad4b22b9cb2..c145e96342668 100644
--- a/python/pyarrow/tests/pandas_examples.py
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -98,21 +98,25 @@ def dataframe_with_lists(include_index=False):
         [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
         [0, 1, 2, 3, 4],
         None,
-        [0]
+        [0],
+        np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9] * 2,
+                 dtype=np.int64)[::2]
     ]
     fields.append(pa.field('double', pa.list_(pa.float64())))
     arrays['double'] = [
         [0., 1., 2., 3., 4., 5., 6., 7., 8., 9.],
         [0., 1., 2., 3., 4.],
         None,
-        [0.]
+        [0.],
+        np.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.] * 2)[::2],
     ]
     fields.append(pa.field('str_list', pa.list_(pa.string())))
     arrays['str_list'] = [
         [u"1", u"ä"],
         None,
         [u"1"],
-        [u"1", u"2", u"3"]
+        [u"1", u"2", u"3"],
+        [],
     ]
 
     if include_index:
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 8969777b526c0..61bd072f6bae9 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -534,6 +534,15 @@ def test_column_of_lists(self):
             field = schema.field_by_name(column)
             self._check_array_roundtrip(df[column], type=field.type)
 
+    def test_column_of_lists_strided(self):
+        df, schema = dataframe_with_lists()
+        df = pd.concat([df] * 6, ignore_index=True)
+
+        arr = df['int64'].values[::3]
+        assert arr.strides[0] != 8
+
+        self._check_array_roundtrip(arr)
+
     def test_nested_lists_all_none(self):
         data = np.array([[None, None], None], dtype=object)
 

From b795e5cff92c840155911f5de7a8323f0c8d3fb6 Mon Sep 17 00:00:00 2001
From: Matt Darwin <(none)>
Date: Fri, 11 Aug 2017 17:13:30 -0400
Subject: [PATCH 0924/1644] ARROW-1240: [JAVA] security: upgrade logback to
 address CVE-2017-5929 (take 2)

sorry, this was still not fixed properly.  logback version is separately specified in 2 places.

Fixed properly this time.

Author: Matt Darwin <(none)>
Author: Matt <mattdarwin@yahoo.co.uk>

Closes #960 from mattdarwin/ARROW-1240-upgrade-logback and squashes the following commits:

3492f66 [Matt Darwin] upgrading logback in tools/pom.xml
206b48d [Matt Darwin] Merge branch 'master' into ARROW-1240-upgrade-logback
284a4ce [Matt Darwin] Merge branch 'master' of https://github.com/apache/arrow
bc3b6a0 [Matt] Merge pull request #1 from apache/master
3e2f676 [Matt Darwin] Merge branch 'master' into ARROW-1240-upgrade-logback
caed163 [Matt Darwin] upgrading slf4j to 1.7.25
---
 java/tools/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 8aadb44f3f69f..f01f3e3931bca 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -48,7 +48,7 @@
         <dependency>
           <groupId>ch.qos.logback</groupId>
           <artifactId>logback-classic</artifactId>
-          <version>1.0.13</version>
+          <version>1.2.3</version>
           <scope>run</scope>
         </dependency>
     </dependencies>

From 214334977823f9cbe495e920eb02fe306517abd3 Mon Sep 17 00:00:00 2001
From: Matt Darwin <(none)>
Date: Fri, 11 Aug 2017 17:16:25 -0400
Subject: [PATCH 0925/1644] ARROW-1242: [JAVA] - upgrade jackson to mitigate
 security vulnerabilities (take 2)

sorry, PR #929 failed to actually change the Jackson version, since the `jackson.version` variable defined in java/pom.xml is not used in java/vector/pom.xml

That's now fixed in this PR.

Author: Matt Darwin <(none)>
Author: Matt <mattdarwin@yahoo.co.uk>

Closes #957 from mattdarwin/ARROW-1242-upgrade-jackson and squashes the following commits:

ad15e5f [Matt Darwin] Merge branch 'master' into ARROW-1242-upgrade-jackson
ee29d65 [Matt Darwin] Merge branch 'master' of https://github.com/apache/arrow into ARROW-1242-upgrade-jackson
06d7745 [Matt Darwin] upgrading jackson to 2.7.9 PROPERLY this time...
284a4ce [Matt Darwin] Merge branch 'master' of https://github.com/apache/arrow
d059517 [Matt Darwin] 1242 upgraing jackson to 2.7.9
bc3b6a0 [Matt] Merge pull request #1 from apache/master
---
 java/vector/pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index a81bbd0f12cca..09f3d743a33de 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -39,12 +39,12 @@
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
-      <version>2.7.1</version>
+      <version>${jackson.version}</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
-      <version>2.7.1</version>
+      <version>${jackson.version}</version>
     </dependency>
     <dependency>
       <groupId>com.carrotsearch</groupId>

From 63954c02fa2c6d8f468fa6fe93529852dacf522c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 11 Aug 2017 17:17:32 -0400
Subject: [PATCH 0926/1644] ARROW-1350: [C++] Do not exclude Plasma source tree
 from source release

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #962 from wesm/ARROW-1350 and squashes the following commits:

5d7f87e [Wes McKinney] Do not exclude Plasma source tree from source release
---
 dev/release/02-source.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index 6e8e7e4f4e3ca..6ab160ca5169b 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -89,10 +89,6 @@ git archive $release_hash --prefix ${tag}/ | tar xf -
 rm -rf ${tag}/c_glib
 mv tmp-c_glib ${tag}/c_glib
 
-# ARROW-1244 Remove cpp/src/plasma from source release until after IP clearance
-# is resolved
-rm -rf ${tag}/cpp/src/plasma
-
 # Create new tarball from modified source directory
 tar czf ${tarball} ${tag}
 rm -rf ${tag}

From b17333482ea1da3728538bc912b1053ba70ed2e7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 11 Aug 2017 17:27:51 -0400
Subject: [PATCH 0927/1644] [maven-release-plugin] prepare release
 apache-arrow-0.6.0

Change-Id: Iee1f58a35beffecb0fdf94232bc235bf0eb46a8f
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index af8ff609dbc9c..f0998147e2cfa 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.6.0-SNAPSHOT</version>
+  <version>0.6.0</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 1070747691cfc..c46a96e14c472 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.6.0-SNAPSHOT</version>
+    <version>0.6.0</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index b0621c5a41ea6..67698393e02fb 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.6.0-SNAPSHOT</version>
+  <version>0.6.0</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>HEAD</tag>
+    <tag>apache-arrow-0.6.0</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index f01f3e3931bca..79c9305581cff 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.6.0-SNAPSHOT</version>
+        <version>0.6.0</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 09f3d743a33de..b221f3d50d635 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.6.0-SNAPSHOT</version>
+    <version>0.6.0</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 4db732c320491992bd99e8e534792359ccd86db5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 11 Aug 2017 17:28:02 -0400
Subject: [PATCH 0928/1644] [maven-release-plugin] prepare for next development
 iteration

Change-Id: I34272bbc580af1fec9beb2a71151b95036860dbc
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index f0998147e2cfa..0e43d5e946cd7 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.6.0</version>
+  <version>0.7.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index c46a96e14c472..765013bee1f66 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.6.0</version>
+    <version>0.7.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 67698393e02fb..f205f6464f7aa 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.6.0</version>
+  <version>0.7.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.6.0</tag>
+    <tag>HEAD</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 79c9305581cff..f1a7dcd68e4fe 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.6.0</version>
+        <version>0.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index b221f3d50d635..a89742ec142bd 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.6.0</version>
+    <version>0.7.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 6135958f5b46f974c4e7ed5f1faaff9c90824050 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 14 Aug 2017 19:39:03 -0400
Subject: [PATCH 0929/1644] ARROW-1348: [C++/Python] Release verification
 script for Windows

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #961 from wesm/ARROW-1348 and squashes the following commits:

1a8cafb [Wes McKinney] Add release verification script for Windows
---
 dev/release/VERIFY.md                    |  25 ++++++
 dev/release/verify-release-candidate.bat | 101 +++++++++++++++++++++++
 2 files changed, 126 insertions(+)
 create mode 100644 dev/release/VERIFY.md
 create mode 100644 dev/release/verify-release-candidate.bat

diff --git a/dev/release/VERIFY.md b/dev/release/VERIFY.md
new file mode 100644
index 0000000000000..3f073e4086538
--- /dev/null
+++ b/dev/release/VERIFY.md
@@ -0,0 +1,25 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Verifying Arrow releases
+
+## Windows
+
+We've provided a convenience script for verifying the C++ and Python builds on
+Windows. Read the comments in `verify-release-candidate.bat` for instructions
\ No newline at end of file
diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat
new file mode 100644
index 0000000000000..8dcaa3d966023
--- /dev/null
+++ b/dev/release/verify-release-candidate.bat
@@ -0,0 +1,101 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@rem To use this script, first create the following conda environment. Change
+@rem the Python version if so desired. You can also omit one or more of the
+@rem libray build rem dependencies if you want to build them from source as well
+@rem
+
+@rem ARROW-1294 CMake 3.9.0 in conda-forge breaks the build
+@rem set ARROW_CMAKE_VERSION=3.8.0
+@rem set PYTHON=3.6
+@rem conda create -n arrow-verify-release -f -q -y python=%PYTHON%
+@rem conda install -y ^
+@rem       six pytest setuptools numpy pandas cython ^
+@rem       thrift-cpp flatbuffers rapidjson ^
+@rem       cmake=%ARROW_CMAKE_VERSION% ^
+@rem       git ^
+@rem       boost-cpp ^
+@rem       snappy zlib brotli gflags lz4-c zstd || exit /B
+
+@rem Then run from the directory containing the RC tarball
+@rem
+@rem verify-release-candidate.bat apache-arrow-%VERSION%
+
+@echo on
+
+if not exist "C:\tmp\" mkdir C:\tmp
+if exist "C:\tmp\arrow-verify-release" rd C:\tmp\arrow-verify-release /s /q
+if not exist "C:\tmp\arrow-verify-release" mkdir C:\tmp\arrow-verify-release
+
+tar xvf %1.tar.gz -C "C:/tmp/"
+
+set GENERATOR=Visual Studio 14 2015 Win64
+set CONFIGURATION=release
+set ARROW_SOURCE=C:\tmp\%1
+
+pushd %ARROW_SOURCE%
+
+call activate arrow-verify-release
+
+set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+set ARROW_HOME=%CONDA_PREFIX%\Library
+set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+set PARQUET_HOME=%CONDA_PREFIX%\Library
+
+@rem Build and test Arrow C++ libraries
+mkdir cpp\build
+pushd cpp\build
+
+cmake -G "%GENERATOR%" ^
+      -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_CXXFLAGS="/WX /MP" ^
+      -DARROW_PYTHON=ON ^
+      ..  || exit /B
+cmake --build . --target INSTALL --config %CONFIGURATION%  || exit /B
+
+@rem Needed so python-test.exe works
+set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
+
+ctest -VV  || exit /B
+popd
+
+@rem Build parquet-cpp
+git clone https://github.com/apache/parquet-cpp.git || exit /B
+mkdir parquet-cpp\build
+pushd parquet-cpp\build
+
+cmake -G "%GENERATOR%" ^
+     -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
+     -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+     -DPARQUET_BOOST_USE_SHARED=OFF ^
+     -DPARQUET_BUILD_TESTS=off .. || exit /B
+cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
+popd
+
+@rem Build and import pyarrow
+@rem parquet-cpp has some additional runtime dependencies that we need to figure out
+@rem see PARQUET-1018
+pushd python
+
+set PYARROW_CXXFLAGS=/WX
+python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel  || exit /B
+py.test pyarrow -v -s --parquet || exit /B
+
+popd

From 142f74e78340bd9cd4598f7eafacb8fa8c31fd8d Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 14 Aug 2017 19:40:53 -0400
Subject: [PATCH 0930/1644] ARROW-1331: [JAVA] Refactor unit tests

[cc @jacques-n , @StevenMPhillips ]

Refactored the java-vector unit tests in TestValueVector.java

(1) Moved all the BitVector related tests to TestBitVector.java
(2) Enhanced and added the tests for different vector types and grouped them into different categories in TestValueVector.java
(3) Moved https://github.com/dremio/dremio-oss/blob/master/sabot/kernel/src/test/java/com/dremio/exec/vector/TestSplitAndTransfer.java to arrow. This currently has splitAndTransfer() test for NullableVarCharVector.
(4) No tests have been removed -- they have either been enhanced or moved around for grouping.

Recently we added splitAndTransfer tests for BitVector, UnionVector and ListVector in their respective test files.  We should ideally move all the splitAndTransfer() related tests to TestSplitAndTransfer.java and should try to cover other vector types as well. A follow-up will address this (I want to keep the patch size small for easy review).

Author: siddharth <siddharth@dremio.com>

Closes #955 from siddharthteotia/ARROW-1331 and squashes the following commits:

ead5cef [siddharth] ARROW-1331: Fixed unit tests
2174e65 [siddharth] ARROW-1331: Refactor unit tests
---
 .../apache/arrow/vector/TestBitVector.java    | 120 +++
 .../arrow/vector/TestSplitAndTransfer.java    |  88 ++
 .../apache/arrow/vector/TestValueVector.java  | 903 +++++++++++++-----
 3 files changed, 873 insertions(+), 238 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java

diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
index 495bed389e568..16316608c0f7c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -19,11 +19,14 @@
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.util.TransferPair;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -139,4 +142,121 @@ public void testSplitAndTransfer() throws Exception {
       sourceVector.close();
     }
   }
+
+  @Test
+  public void testBitVector() {
+    // Create a new value vector for 1024 integers
+    try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
+      final BitVector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024);
+      m.setValueCount(1024);
+
+      // Put and set a few values
+      m.set(0, 1);
+      m.set(1, 0);
+      m.set(100, 0);
+      m.set(1022, 1);
+
+      m.setValueCount(1024);
+
+      final BitVector.Accessor accessor = vector.getAccessor();
+      assertEquals(1, accessor.get(0));
+      assertEquals(0, accessor.get(1));
+      assertEquals(0, accessor.get(100));
+      assertEquals(1, accessor.get(1022));
+
+      assertEquals(1022, accessor.getNullCount());
+
+      // test setting the same value twice
+      m.set(0, 1);
+      m.set(0, 1);
+      m.set(1, 0);
+      m.set(1, 0);
+      assertEquals(1, accessor.get(0));
+      assertEquals(0, accessor.get(1));
+
+      // test toggling the values
+      m.set(0, 0);
+      m.set(1, 1);
+      assertEquals(0, accessor.get(0));
+      assertEquals(1, accessor.get(1));
+
+      // should not change
+      assertEquals(1022, accessor.getNullCount());
+
+      // Ensure unallocated space returns 0
+      assertEquals(0, accessor.get(3));
+
+      // unset the previously set bits
+      m.set(1, 0);
+      m.set(1022, 0);
+      // this should set all the array to 0
+      assertEquals(1024, accessor.getNullCount());
+
+      // set all the array to 1
+      for (int i = 0; i < 1024; ++i) {
+        assertEquals(1024 - i, accessor.getNullCount());
+        m.set(i, 1);
+      }
+
+      assertEquals(0, accessor.getNullCount());
+
+      vector.allocateNew(1015);
+      m.setValueCount(1015);
+
+      // ensure it has been zeroed
+      assertEquals(1015, accessor.getNullCount());
+
+      m.set(0, 1);
+      m.set(1014, 1); // ensure that the last item of the last byte is allocated
+
+      assertEquals(1013, accessor.getNullCount());
+
+      vector.zeroVector();
+      assertEquals(1015, accessor.getNullCount());
+
+      // set all the array to 1
+      for (int i = 0; i < 1015; ++i) {
+        assertEquals(1015 - i, accessor.getNullCount());
+        m.set(i, 1);
+      }
+
+      assertEquals(0, accessor.getNullCount());
+    }
+  }
+
+  @Test
+  public void testBitVectorRangeSetAllOnes() {
+    validateRange(1000, 0, 1000);
+    validateRange(1000, 0, 1);
+    validateRange(1000, 1, 2);
+    validateRange(1000, 5, 6);
+    validateRange(1000, 5, 10);
+    validateRange(1000, 5, 150);
+    validateRange(1000, 5, 27);
+    for (int i = 0; i < 8; i++) {
+      for (int j = 0; j < 8; j++) {
+        validateRange(1000, 10 + i, 27 + j);
+        validateRange(1000, i, j);
+      }
+    }
+  }
+
+  private void validateRange(int length, int start, int count) {
+    String desc = "[" + start + ", " + (start + count) + ") ";
+    try (BitVector bitVector = new BitVector("bits", allocator)) {
+      bitVector.reset();
+      bitVector.allocateNew(length);
+      bitVector.getMutator().setRangeToOne(start, count);
+      for (int i = 0; i < start; i++) {
+        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+      }
+      for (int i = start; i < start + count; i++) {
+        Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i));
+      }
+      for (int i = start + count; i < length; i++) {
+        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+      }
+    }
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
new file mode 100644
index 0000000000000..4ff42b5b0e0aa
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+
+import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.NullableVarCharVector.Accessor;
+import org.apache.arrow.vector.util.TransferPair;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestSplitAndTransfer {
+
+    private BufferAllocator allocator;
+
+    @Before
+    public void init() {
+        allocator = new RootAllocator(Long.MAX_VALUE);
+    }
+
+    @After
+    public void terminate() throws Exception {
+        allocator.close();
+    }
+
+    @Test /* NullableVarCharVector */
+    public void test() throws Exception {
+        try(final NullableVarCharVector varCharVector = new NullableVarCharVector("myvector", allocator)) {
+            varCharVector.allocateNew(10000, 1000);
+
+            final int valueCount = 500;
+            final String[] compareArray = new String[valueCount];
+
+            final NullableVarCharVector.Mutator mutator = varCharVector.getMutator();
+            for (int i = 0; i < valueCount; i += 3) {
+                final String s = String.format("%010d", i);
+                mutator.set(i, s.getBytes());
+                compareArray[i] = s;
+            }
+            mutator.setValueCount(valueCount);
+
+            final TransferPair tp = varCharVector.getTransferPair(allocator);
+            final NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo();
+            final Accessor accessor = newVarCharVector.getAccessor();
+            final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}};
+
+            for (final int[] startLength : startLengths) {
+                final int start = startLength[0];
+                final int length = startLength[1];
+                tp.splitAndTransfer(start, length);
+                newVarCharVector.getMutator().setValueCount(length);
+                for (int i = 0; i < length; i++) {
+                    final boolean expectedSet = ((start + i) % 3) == 0;
+                    if (expectedSet) {
+                        final byte[] expectedValue = compareArray[start + i].getBytes();
+                        assertFalse(accessor.isNull(i));
+                        assertArrayEquals(expectedValue, accessor.get(i));
+                    } else {
+                        assertTrue(accessor.isNull(i));
+                    }
+                }
+                newVarCharVector.clear();
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 72214ed2ed6fb..4e3021b7e719f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -17,6 +17,7 @@
  */
 
 package org.apache.arrow.vector;
+import org.apache.arrow.vector.util.OversizedAllocationException;
 
 import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
 import static org.apache.arrow.vector.TestUtils.newVector;
@@ -24,6 +25,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import java.nio.charset.Charset;
 import java.util.List;
@@ -64,210 +66,661 @@ public void init() {
   private final static byte[] STR4 = "DDDDDDDD4".getBytes(utf8Charset);
   private final static byte[] STR5 = "EEE5".getBytes(utf8Charset);
   private final static byte[] STR6 = "FFFFF6".getBytes(utf8Charset);
+  private final static int MAX_VALUE_COUNT =
+            Integer.getInteger("arrow.vector.max_allocation_bytes", Integer.MAX_VALUE)/4;
+  private final static int MAX_VALUE_COUNT_8BYTE = MAX_VALUE_COUNT/2;
 
   @After
   public void terminate() throws Exception {
     allocator.close();
   }
 
-  @Test
-  public void testFixedType() {
+  /*
+   * Tests for Fixed-Width vectors
+   *
+   * Covered types as of now
+   *
+   *  -- UInt4Vector
+   *  -- IntVector
+   *  -- Float4Vector
+   *  -- Float8Vector
+   *
+   *  -- NullableUInt4Vector
+   *  -- NullableIntVector
+   *  -- NullableFloat4Vector
+   *
+   * TODO:
+   *
+   *  -- SmallIntVector
+   *  -- BigIntVector
+   *  -- TinyIntVector
+   */
+
+  @Test /* UInt4Vector */
+  public void testFixedType1() {
 
     // Create a new value vector for 1024 integers.
     try (final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator)) {
-      final UInt4Vector.Mutator m = vector.getMutator();
+
+      boolean error = false;
+      int initialCapacity = 0;
+      final UInt4Vector.Mutator mutator = vector.getMutator();
+      final UInt4Vector.Accessor accessor = vector.getAccessor();
+
       vector.allocateNew(1024);
+      initialCapacity = vector.getValueCapacity();
+      assertEquals(1024, initialCapacity);
 
       // Put and set a few values
-      m.setSafe(0, 100);
-      m.setSafe(1, 101);
-      m.setSafe(100, 102);
-      m.setSafe(1022, 103);
-      m.setSafe(1023, 104);
+      mutator.setSafe(0, 100);
+      mutator.setSafe(1, 101);
+      mutator.setSafe(100, 102);
+      mutator.setSafe(1022, 103);
+      mutator.setSafe(1023, 104);
 
-      final UInt4Vector.Accessor accessor = vector.getAccessor();
       assertEquals(100, accessor.get(0));
       assertEquals(101, accessor.get(1));
       assertEquals(102, accessor.get(100));
       assertEquals(103, accessor.get(1022));
       assertEquals(104, accessor.get(1023));
+
+      try {
+        mutator.set(1024, 10000);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      try {
+        accessor.get(1024);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* this should trigger a realloc() */
+      mutator.setSafe(1024, 10000);
+
+      /* underlying buffer should now be able to store double the number of values */
+      assertEquals(initialCapacity * 2, vector.getValueCapacity());
+
+      /* check vector data after realloc */
+      assertEquals(100, accessor.get(0));
+      assertEquals(101, accessor.get(1));
+      assertEquals(102, accessor.get(100));
+      assertEquals(103, accessor.get(1022));
+      assertEquals(104, accessor.get(1023));
+      assertEquals(10000, accessor.get(1024));
+
+      /* reset the vector */
+      vector.reset();
+
+      /* capacity shouldn't change after reset */
+      assertEquals(initialCapacity * 2, vector.getValueCapacity());
+
+      /* vector data should have been zeroed out */
+      for(int i = 0; i < (initialCapacity * 2); i++) {
+        assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i));
+      }
     }
   }
 
-  @Test
-  public void testNullableVarLen2() {
+  @Test /* IntVector */
+  public void testFixedType2() {
+    try (final IntVector intVector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) {
+      final IntVector.Mutator mutator = intVector.getMutator();
+      final IntVector.Accessor accessor = intVector.getAccessor();
+      boolean error = false;
+      int initialCapacity = 16;
 
-    // Create a new value vector for 1024 integers.
-    try (final NullableVarCharVector vector = newNullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
-      final NullableVarCharVector.Mutator m = vector.getMutator();
-      vector.allocateNew(1024 * 10, 1024);
+      /* we should not throw exception for these values of capacity */
+      intVector.setInitialCapacity(MAX_VALUE_COUNT - 1);
+      intVector.setInitialCapacity(MAX_VALUE_COUNT);
 
-      m.set(0, STR1);
-      m.set(1, STR2);
-      m.set(2, STR3);
+      try {
+        intVector.setInitialCapacity(MAX_VALUE_COUNT + 1);
+      }
+      catch (OversizedAllocationException oe) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
 
-      // Check the sample strings.
-      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
+      intVector.setInitialCapacity(initialCapacity);
+      /* no memory allocation has happened yet so capacity of underlying buffer should be 0 */
+      assertEquals(0, intVector.getValueCapacity());
+
+      /* allocate 64 bytes (16 * 4) */
+      intVector.allocateNew();
+      /* underlying buffer should be able to store 16 values */
+      assertEquals(initialCapacity, intVector.getValueCapacity());
+
+      /* populate the vector */
+      int j = 1;
+      for(int i = 0; i < 16; i += 2) {
+        mutator.set(i, j);
+        j++;
+      }
 
-      // Ensure null value throws.
-      boolean b = false;
       try {
-        vector.getAccessor().get(3);
-      } catch (IllegalStateException e) {
-        b = true;
-      } finally {
-        assertTrue(b);
+        mutator.set(16, 9);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* check vector contents */
+      j = 1;
+      for(int i = 0; i < 16; i += 2) {
+        assertEquals("unexpected value at index: " + i, j, accessor.get(i));
+        j++;
+      }
+
+      try {
+        accessor.get(16);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* this should trigger a realloc() */
+      mutator.setSafe(16, 9);
+
+      /* underlying buffer should now be able to store double the number of values */
+      assertEquals(initialCapacity * 2, intVector.getValueCapacity());
+
+      /* vector data should still be intact after realloc */
+      j = 1;
+      for(int i = 0; i <= 16; i += 2) {
+        assertEquals("unexpected value at index: " + i, j, accessor.get(i));
+        j++;
+      }
+
+      /* reset the vector */
+      intVector.reset();
+
+      /* capacity shouldn't change after reset */
+      assertEquals(initialCapacity * 2, intVector.getValueCapacity());
+
+      /* vector data should have been zeroed out */
+      for(int i = 0; i < (initialCapacity * 2); i++) {
+        assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i));
       }
     }
   }
 
-  @Test
-  public void testNullableFixedType() {
+  @Test /* Float4Vector */
+  public void testFixedType3() {
+    try (final Float4Vector floatVector = new Float4Vector(EMPTY_SCHEMA_PATH, allocator)) {
+      final Float4Vector.Mutator mutator = floatVector.getMutator();
+      final Float4Vector.Accessor accessor = floatVector.getAccessor();
+      boolean error = false;
+      int initialCapacity = 16;
+
+      /* we should not throw exception for these values of capacity */
+      floatVector.setInitialCapacity(MAX_VALUE_COUNT - 1);
+      floatVector.setInitialCapacity(MAX_VALUE_COUNT);
+
+      try {
+        floatVector.setInitialCapacity(MAX_VALUE_COUNT + 1);
+      }
+      catch (OversizedAllocationException oe) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      floatVector.setInitialCapacity(initialCapacity);
+      /* no memory allocation has happened yet so capacity of underlying buffer should be 0 */
+      assertEquals(0, floatVector.getValueCapacity());
+
+      /* allocate 64 bytes (16 * 4) */
+      floatVector.allocateNew();
+      /* underlying buffer should be able to store 16 values */
+      assertEquals(initialCapacity, floatVector.getValueCapacity());
+
+      floatVector.zeroVector();
+
+      /* populate the vector */
+      mutator.set(0, 1.5f);
+      mutator.set(2, 2.5f);
+      mutator.set(4, 3.3f);
+      mutator.set(6, 4.8f);
+      mutator.set(8, 5.6f);
+      mutator.set(10, 6.6f);
+      mutator.set(12, 7.8f);
+      mutator.set(14, 8.5f);
+
+      try {
+        mutator.set(16, 9.5f);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* check vector contents */
+      assertEquals(1.5f, accessor.get(0), 0);
+      assertEquals(2.5f, accessor.get(2), 0);
+      assertEquals(3.3f, accessor.get(4), 0);
+      assertEquals(4.8f, accessor.get(6), 0);
+      assertEquals(5.6f, accessor.get(8), 0);
+      assertEquals(6.6f, accessor.get(10), 0);
+      assertEquals(7.8f, accessor.get(12), 0);
+      assertEquals(8.5f, accessor.get(14), 0);
+
+      try {
+        accessor.get(16);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* this should trigger a realloc() */
+      mutator.setSafe(16, 9.5f);
+
+      /* underlying buffer should now be able to store double the number of values */
+      assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
+
+      /* vector data should still be intact after realloc */
+      assertEquals(1.5f, accessor.get(0), 0);
+      assertEquals(2.5f, accessor.get(2), 0);
+      assertEquals(3.3f, accessor.get(4), 0);
+      assertEquals(4.8f, accessor.get(6), 0);
+      assertEquals(5.6f, accessor.get(8), 0);
+      assertEquals(6.6f, accessor.get(10), 0);
+      assertEquals(7.8f, accessor.get(12), 0);
+      assertEquals(8.5f, accessor.get(14), 0);
+      assertEquals(9.5f, accessor.get(16), 0);
+
+      /* reset the vector */
+      floatVector.reset();
+
+      /* capacity shouldn't change after reset */
+      assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
+
+      /* vector data should be zeroed out */
+      for(int i = 0; i < (initialCapacity * 2); i++) {
+        assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i), 0);
+      }
+    }
+  }
+
+  @Test /* Float8Vector */
+  public void testFixedType4() {
+    try (final Float8Vector floatVector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
+      final Float8Vector.Mutator mutator = floatVector.getMutator();
+      final Float8Vector.Accessor accessor = floatVector.getAccessor();
+      boolean error = false;
+      int initialCapacity = 16;
+
+      /* we should not throw exception for these values of capacity */
+      floatVector.setInitialCapacity(MAX_VALUE_COUNT_8BYTE - 1);
+      floatVector.setInitialCapacity(MAX_VALUE_COUNT_8BYTE);
+
+      try {
+        floatVector.setInitialCapacity(MAX_VALUE_COUNT_8BYTE + 1);
+      }
+      catch (OversizedAllocationException oe) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      floatVector.setInitialCapacity(initialCapacity);
+      /* no memory allocation has happened yet so capacity of underlying buffer should be 0 */
+      assertEquals(0, floatVector.getValueCapacity());
+
+      /* allocate 128 bytes (16 * 8) */
+      floatVector.allocateNew();
+      /* underlying buffer should be able to store 16 values */
+      assertEquals(initialCapacity, floatVector.getValueCapacity());
+
+      /* populate the vector */
+      mutator.set(0, 1.55);
+      mutator.set(2, 2.53);
+      mutator.set(4, 3.36);
+      mutator.set(6, 4.82);
+      mutator.set(8, 5.67);
+      mutator.set(10, 6.67);
+      mutator.set(12, 7.87);
+      mutator.set(14, 8.56);
+
+      try {
+        mutator.set(16, 9.53);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* check vector contents */
+      assertEquals(1.55, accessor.get(0), 0);
+      assertEquals(2.53, accessor.get(2), 0);
+      assertEquals(3.36, accessor.get(4), 0);
+      assertEquals(4.82, accessor.get(6), 0);
+      assertEquals(5.67, accessor.get(8), 0);
+      assertEquals(6.67, accessor.get(10), 0);
+      assertEquals(7.87, accessor.get(12), 0);
+      assertEquals(8.56, accessor.get(14), 0);
+
+      try {
+        accessor.get(16);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* this should trigger a realloc() */
+      mutator.setSafe(16, 9.53);
+
+      /* underlying buffer should now be able to store double the number of values */
+      assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
+
+      /* vector data should still be intact after realloc */
+      assertEquals(1.55, accessor.get(0), 0);
+      assertEquals(2.53, accessor.get(2), 0);
+      assertEquals(3.36, accessor.get(4), 0);
+      assertEquals(4.82, accessor.get(6), 0);
+      assertEquals(5.67, accessor.get(8), 0);
+      assertEquals(6.67, accessor.get(10), 0);
+      assertEquals(7.87, accessor.get(12), 0);
+      assertEquals(8.56, accessor.get(14), 0);
+      assertEquals(9.53, accessor.get(16), 0);
+
+      /* reset the vector */
+      floatVector.reset();
+
+      /* capacity shouldn't change after reset */
+      assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
+
+      /* vector data should be zeroed out */
+      for(int i = 0; i < (initialCapacity * 2); i++) {
+        assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i), 0);
+      }
+    }
+  }
+
+  @Test /* NullableUInt4Vector */
+  public void testNullableFixedType1() {
 
     // Create a new value vector for 1024 integers.
     try (final NullableUInt4Vector vector = newVector(NullableUInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) {
-      final NullableUInt4Vector.Mutator m = vector.getMutator();
-      vector.allocateNew(1024);
+      final NullableUInt4Vector.Mutator mutator = vector.getMutator();
+      final NullableUInt4Vector.Accessor accessor = vector.getAccessor();
+      boolean error = false;
+      int initialCapacity = 1024;
+
+      vector.setInitialCapacity(initialCapacity);
+      /* no memory allocation has happened yet */
+      assertEquals(0, vector.getValueCapacity());
+
+      vector.allocateNew();
+      assertEquals(initialCapacity, vector.getValueCapacity());
 
       // Put and set a few values
-      m.set(0, 100);
-      m.set(1, 101);
-      m.set(100, 102);
-      m.set(1022, 103);
-      m.set(1023, 104);
+      mutator.set(0, 100);
+      mutator.set(1, 101);
+      mutator.set(100, 102);
+      mutator.set(1022, 103);
+      mutator.set(1023, 104);
 
-      final NullableUInt4Vector.Accessor accessor = vector.getAccessor();
+      /* check vector contents */
       assertEquals(100, accessor.get(0));
       assertEquals(101, accessor.get(1));
       assertEquals(102, accessor.get(100));
       assertEquals(103, accessor.get(1022));
       assertEquals(104, accessor.get(1023));
 
-      // Ensure null values throw
-      {
-        boolean b = false;
-        try {
-          accessor.get(3);
-        } catch (IllegalStateException e) {
-          b = true;
-        } finally {
-          assertTrue(b);
+      int val = 0;
+
+      /* check unset bits/null values */
+      for (int i = 2, j = 101; i <= 99 || j <= 1021; i++, j++) {
+        if (i <= 99) {
+          assertTrue(accessor.isNull(i));
+        }
+        if(j <= 1021) {
+          assertTrue(accessor.isNull(j));
         }
       }
 
-      vector.allocateNew(2048);
-      {
-        boolean b = false;
-        try {
-          accessor.get(0);
-        } catch (IllegalStateException e) {
-          b = true;
-        } finally {
-          assertTrue(b);
-        }
+      try {
+        mutator.set(1024, 10000);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
       }
 
-      m.set(0, 100);
-      m.set(1, 101);
-      m.set(100, 102);
-      m.set(1022, 103);
-      m.set(1023, 104);
+      try {
+        accessor.get(1024);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* should trigger a realloc of the underlying bitvector and valuevector */
+      mutator.setSafe(1024, 10000);
+
+      /* check new capacity */
+      assertEquals(initialCapacity * 2, vector.getValueCapacity());
+
+      /* vector contents should still be intact after realloc */
       assertEquals(100, accessor.get(0));
       assertEquals(101, accessor.get(1));
       assertEquals(102, accessor.get(100));
       assertEquals(103, accessor.get(1022));
       assertEquals(104, accessor.get(1023));
+      assertEquals(10000, accessor.get(1024));
 
-      // Ensure null values throw.
-      {
-        boolean b = false;
-        try {
-          vector.getAccessor().get(3);
-        } catch (IllegalStateException e) {
-          b = true;
-        } finally {
-          assertTrue(b);
+      val = 0;
+
+      /* check unset bits/null values */
+      for (int i = 2, j = 101; i < 99 || j < 1021; i++, j++) {
+        if (i <= 99) {
+          assertTrue(accessor.isNull(i));
+        }
+        if(j <= 1021) {
+          assertTrue(accessor.isNull(j));
         }
       }
+
+      /* reset the vector */
+      vector.reset();
+
+       /* capacity shouldn't change after reset */
+      assertEquals(initialCapacity * 2, vector.getValueCapacity());
+
+      /* vector data should be zeroed out */
+      for(int i = 0; i < (initialCapacity * 2); i++) {
+        assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+      }
     }
   }
 
-  @Test
-  public void testNullableFloat() {
+  @Test /* NullableFloat4Vector */
+  public void testNullableFixedType2() {
     // Create a new value vector for 1024 integers
     try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) {
-      final NullableFloat4Vector.Mutator m = vector.getMutator();
-      vector.allocateNew(1024);
+      final NullableFloat4Vector.Mutator mutator = vector.getMutator();
+      final NullableFloat4Vector.Accessor accessor = vector.getAccessor();
+      boolean error = false;
+      int initialCapacity = 16;
 
-      // Put and set a few values.
-      m.set(0, 100.1f);
-      m.set(1, 101.2f);
-      m.set(100, 102.3f);
-      m.set(1022, 103.4f);
-      m.set(1023, 104.5f);
+      vector.setInitialCapacity(initialCapacity);
+      /* no memory allocation has happened yet */
+      assertEquals(0, vector.getValueCapacity());
 
-      final NullableFloat4Vector.Accessor accessor = vector.getAccessor();
-      assertEquals(100.1f, accessor.get(0), 0);
-      assertEquals(101.2f, accessor.get(1), 0);
-      assertEquals(102.3f, accessor.get(100), 0);
-      assertEquals(103.4f, accessor.get(1022), 0);
-      assertEquals(104.5f, accessor.get(1023), 0);
+      vector.allocateNew();
+      assertEquals(initialCapacity, vector.getValueCapacity());
+
+      /* populate the vector */
+      mutator.set(0, 100.5f);
+      mutator.set(2, 201.5f);
+      mutator.set(4, 300.3f);
+      mutator.set(6, 423.8f);
+      mutator.set(8, 555.6f);
+      mutator.set(10, 66.6f);
+      mutator.set(12, 78.8f);
+      mutator.set(14, 89.5f);
 
-      // Ensure null values throw.
-      {
-        boolean b = false;
-        try {
-          vector.getAccessor().get(3);
-        } catch (IllegalStateException e) {
-          b = true;
-        } finally {
-          assertTrue(b);
-        }
+      try {
+        mutator.set(16, 90.5f);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
       }
 
-      vector.allocateNew(2048);
-      {
-        boolean b = false;
-        try {
-          accessor.get(0);
-        } catch (IllegalStateException e) {
-          b = true;
-        } finally {
-          assertTrue(b);
-        }
+      /* check vector contents */
+      assertEquals(100.5f, accessor.get(0), 0);
+      assertTrue(accessor.isNull(1));
+      assertEquals(201.5f, accessor.get(2), 0);
+      assertTrue(accessor.isNull(3));
+      assertEquals(300.3f, accessor.get(4), 0);
+      assertTrue(accessor.isNull(5));
+      assertEquals(423.8f, accessor.get(6), 0);
+      assertTrue(accessor.isNull(7));
+      assertEquals(555.6f, accessor.get(8), 0);
+      assertTrue(accessor.isNull(9));
+      assertEquals(66.6f, accessor.get(10), 0);
+      assertTrue(accessor.isNull(11));
+      assertEquals(78.8f, accessor.get(12), 0);
+      assertTrue(accessor.isNull(13));
+      assertEquals(89.5f, accessor.get(14), 0);
+      assertTrue(accessor.isNull(15));
+
+      try {
+        accessor.get(16);
+      }
+      catch (IndexOutOfBoundsException ie) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      /* this should trigger a realloc() */
+      mutator.setSafe(16, 90.5f);
+
+      /* underlying buffer should now be able to store double the number of values */
+      assertEquals(initialCapacity * 2, vector.getValueCapacity());
+
+      /* vector data should still be intact after realloc */
+      assertEquals(100.5f, accessor.get(0), 0);
+      assertTrue(accessor.isNull(1));
+      assertEquals(201.5f, accessor.get(2), 0);
+      assertTrue(accessor.isNull(3));
+      assertEquals(300.3f, accessor.get(4), 0);
+      assertTrue(accessor.isNull(5));
+      assertEquals(423.8f, accessor.get(6), 0);
+      assertTrue(accessor.isNull(7));
+      assertEquals(555.6f, accessor.get(8), 0);
+      assertTrue(accessor.isNull(9));
+      assertEquals(66.6f, accessor.get(10), 0);
+      assertTrue(accessor.isNull(11));
+      assertEquals(78.8f, accessor.get(12), 0);
+      assertTrue(accessor.isNull(13));
+      assertEquals(89.5f, accessor.get(14), 0);
+      assertTrue(accessor.isNull(15));
+      assertEquals(90.5f, accessor.get(16), 0);
+
+      /* reset the vector */
+      vector.reset();
+
+      /* capacity shouldn't change after reset */
+      assertEquals(initialCapacity * 2, vector.getValueCapacity());
+
+      /* vector data should be zeroed out */
+      for(int i = 0; i < (initialCapacity * 2); i++) {
+        assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
       }
     }
   }
 
-  @Test
-  public void testNullableInt() {
+  @Test /* NullableIntVector */
+  public void testNullableFixedType3() {
     // Create a new value vector for 1024 integers
     try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) {
-      final NullableIntVector.Mutator m = vector.getMutator();
-      vector.allocateNew(1024);
-
-      // Put and set a few values.
-      m.set(0, 1);
-      m.set(1, 2);
-      m.set(100, 3);
-      m.set(1022, 4);
-      m.set(1023, 5);
-
-      m.setValueCount(1024);
-
+      final NullableIntVector.Mutator mutator = vector.getMutator();
       final NullableIntVector.Accessor accessor = vector.getAccessor();
-      assertEquals(1, accessor.get(0));
-      assertEquals(2, accessor.get(1));
-      assertEquals(3, accessor.get(100));
-      assertEquals(4, accessor.get(1022));
-      assertEquals(5, accessor.get(1023));
-
-      // Ensure null values.
-      assertTrue(vector.getAccessor().isNull(3));
+      boolean error = false;
+      int initialCapacity = 1024;
+
+      /* no memory allocation has happened yet so capacity of underlying buffer should be 0 */
+      assertEquals(0, vector.getValueCapacity());
+      /* allocate space for 4KB data (1024 * 4) */
+      vector.allocateNew(initialCapacity);
+      /* underlying buffer should be able to store 16 values */
+      assertEquals(initialCapacity, vector.getValueCapacity());
+
+      mutator.set(0, 1);
+      mutator.set(1, 2);
+      mutator.set(100, 3);
+      mutator.set(1022, 4);
+      mutator.set(1023, 5);
+
+      /* check vector contents */
+      int j = 1;
+      for(int i = 0; i <= 1023; i++) {
+        if((i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) {
+          assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+        }
+        else {
+          assertFalse("null data not expected at index: " + i, accessor.isNull(i));
+          assertEquals("unexpected value at index: " + i, j, accessor.get(i));
+          j++;
+        }
+      }
 
+      mutator.setValueCount(1024);
       Field field = vector.getField();
       TypeLayout typeLayout = field.getTypeLayout();
 
@@ -277,6 +730,8 @@ public void testNullableInt() {
       assertEquals(2, buffers.size());
 
       ArrowBuf validityVectorBuf = buffers.get(0);
+
+      /* bitvector tracks 1024 integers --> 1024 bits --> 128 bytes */
       assertEquals(128, validityVectorBuf.readableBytes());
       assertEquals(3, validityVectorBuf.getByte(0)); // 1st and second bit defined
       for (int i = 1; i < 12; i++) {
@@ -288,129 +743,101 @@ public void testNullableInt() {
       }
       assertEquals(-64, validityVectorBuf.getByte(127)); // 1022nd and 1023rd bit defined
 
-      vector.allocateNew(2048);
-      // vector has been erased
-      assertTrue(vector.getAccessor().isNull(0));
-    }
-  }
-
-
-  @Test
-  public void testBitVector() {
-    // Create a new value vector for 1024 integers
-    try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
-      final BitVector.Mutator m = vector.getMutator();
-      vector.allocateNew(1024);
-      m.setValueCount(1024);
-
-      // Put and set a few values
-      m.set(0, 1);
-      m.set(1, 0);
-      m.set(100, 0);
-      m.set(1022, 1);
-
-      m.setValueCount(1024);
-
-      final BitVector.Accessor accessor = vector.getAccessor();
-      assertEquals(1, accessor.get(0));
-      assertEquals(0, accessor.get(1));
-      assertEquals(0, accessor.get(100));
-      assertEquals(1, accessor.get(1022));
-
-      assertEquals(1022, accessor.getNullCount());
-
-      // test setting the same value twice
-      m.set(0, 1);
-      m.set(0, 1);
-      m.set(1, 0);
-      m.set(1, 0);
-      assertEquals(1, accessor.get(0));
-      assertEquals(0, accessor.get(1));
-
-      // test toggling the values
-      m.set(0, 0);
-      m.set(1, 1);
-      assertEquals(0, accessor.get(0));
-      assertEquals(1, accessor.get(1));
-
-      // should not change
-      assertEquals(1022, accessor.getNullCount());
-
-      // Ensure unallocated space returns 0
-      assertEquals(0, accessor.get(3));
+      /* this should trigger a realloc() */
+      mutator.setSafe(1024, 6);
 
-      // unset the previously set bits
-      m.set(1, 0);
-      m.set(1022, 0);
-      // this should set all the array to 0
-      assertEquals(1024, accessor.getNullCount());
+      /* underlying buffer should now be able to store double the number of values */
+      assertEquals(initialCapacity * 2, vector.getValueCapacity());
 
-      // set all the array to 1
-      for (int i = 0; i < 1024; ++i) {
-        assertEquals(1024 - i, accessor.getNullCount());
-        m.set(i, 1);
+      /* vector data should still be intact after realloc */
+      j = 1;
+      for(int i = 0; i < (initialCapacity * 2); i++) {
+        if((i > 1024) || (i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) {
+          assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+        }
+        else {
+          assertFalse("null data not expected at index: " + i, accessor.isNull(i));
+          assertEquals("unexpected value at index: " + i, j, accessor.get(i));
+          j++;
+        }
       }
 
-      assertEquals(0, accessor.getNullCount());
+      /* reset the vector */
+      vector.reset();
 
-      vector.allocateNew(1015);
-      m.setValueCount(1015);
+      /* capacity shouldn't change after reset */
+      assertEquals(initialCapacity * 2, vector.getValueCapacity());
 
-      // ensure it has been zeroed
-      assertEquals(1015, accessor.getNullCount());
+      /* vector data should have been zeroed out */
+      for(int i = 0; i < (initialCapacity * 2); i++) {
+        assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+      }
 
-      m.set(0, 1);
-      m.set(1014, 1); // ensure that the last item of the last byte is allocated
+      vector.allocateNew(4096);
+      // vector has been erased
+      for(int i = 0; i < 4096; i++) {
+        assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+      }
+    }
+  }
 
-      assertEquals(1013, accessor.getNullCount());
+  /*
+   * Tests for Variable Width Vectors
+   *
+   * Covered types as of now
+   *
+   *  -- NullableVarCharVector
+   *
+   * TODO:
+   *
+   *  -- VarCharVector
+   *  -- VarBinaryVector
+   *  -- NullableVarBinaryVector
+   */
+
+  @Test /* NullableVarCharVector */
+  public void testNullableVarType1() {
 
-      vector.zeroVector();
-      assertEquals(1015, accessor.getNullCount());
+    // Create a new value vector for 1024 integers.
+    try (final NullableVarCharVector vector = newNullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024 * 10, 1024);
 
-      // set all the array to 1
-      for (int i = 0; i < 1015; ++i) {
-        assertEquals(1015 - i, accessor.getNullCount());
-        m.set(i, 1);
-      }
+      m.set(0, STR1);
+      m.set(1, STR2);
+      m.set(2, STR3);
 
-      assertEquals(0, accessor.getNullCount());
-    }
-  }
+      // Check the sample strings.
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
 
-  @Test
-  public void testBitVectorRangeSetAllOnes() {
-    validateRange(1000, 0, 1000);
-    validateRange(1000, 0, 1);
-    validateRange(1000, 1, 2);
-    validateRange(1000, 5, 6);
-    validateRange(1000, 5, 10);
-    validateRange(1000, 5, 150);
-    validateRange(1000, 5, 27);
-    for (int i = 0; i < 8; i++) {
-      for (int j = 0; j < 8; j++) {
-        validateRange(1000, 10 + i, 27 + j);
-        validateRange(1000, i, j);
+      // Ensure null value throws.
+      boolean b = false;
+      try {
+        vector.getAccessor().get(3);
+      } catch (IllegalStateException e) {
+        b = true;
+      } finally {
+        assertTrue(b);
       }
     }
   }
 
-  private void validateRange(int length, int start, int count) {
-    String desc = "[" + start + ", " + (start + count) + ") ";
-    try (BitVector bitVector = new BitVector("bits", allocator)) {
-      bitVector.reset();
-      bitVector.allocateNew(length);
-      bitVector.getMutator().setRangeToOne(start, count);
-      for (int i = 0; i < start; i++) {
-        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
-      }
-      for (int i = start; i < start + count; i++) {
-        Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i));
-      }
-      for (int i = start + count; i < length; i++) {
-        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
-      }
-    }
-  }
+  /*
+   * generic tests
+   *
+   *  -- lastSet() and setValueCount()
+   *  -- fillEmpties()
+   *  -- VectorLoader and VectorUnloader
+   *  -- some realloc tests
+   *
+   *  TODO:
+   *
+   *  The realloc() related tests below should be moved up and we need to
+   *  realloc related tests (edge cases) for more vector types.
+   */
 
   @Test
   public void testReAllocNullableFixedWidthVector() {

From a2f432392fa57dfba46ace4b33eb6eb270618ce8 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 14 Aug 2017 19:42:09 -0400
Subject: [PATCH 0931/1644] ARROW-1352: [Integration] Added specific formatting
 for producer consumer output

Changed output of integration tests to give a specific format when indicating producer/consumer.  The new output will look like this:

```
-- Creating binary inputs
-- Validating file
-- Validating stream
##########################################################
Java producing, C++ consuming
##########################################################
==========================================================
Testing file /home/bryan/git/arrow/integration/data/struct_example.json
==========================================================
-- Creating binary inputs
-- Validating file
-- Validating stream
```

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #963 from BryanCutler/integration-producer-format-ARROW-1352 and squashes the following commits:

7e2b8ba [Bryan Cutler] added header formatting for producer consumer output
---
 integration/integration_test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index b7f1609935e79..465394844882f 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -789,8 +789,10 @@ def run(self):
             self._compare_implementations(producer, consumer)
 
     def _compare_implementations(self, producer, consumer):
-        print('-- {0} producing, {1} consuming'.format(producer.name,
+        print('##########################################################')
+        print('{0} producing, {1} consuming'.format(producer.name,
                                                        consumer.name))
+        print('##########################################################')
 
         for json_path in self.json_files:
             print('==========================================================')

From 94b7cfafae0bda8f68ee3e5e9702c954b0116203 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Mon, 14 Aug 2017 19:44:24 -0400
Subject: [PATCH 0932/1644] ARROW-1339: [C++] Use of boost::filesystem::path to
 handle file paths

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #964 from MaxRis/ARROW-1339 and squashes the following commits:

5cdc3de [Max Risuhin] ARROW-1339: [C++] Use of boost::filesystem::path to handle file paths
---
 cpp/src/arrow/io/file.cc | 105 +++++++++++++--------------------------
 1 file changed, 34 insertions(+), 71 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 57d30f73baa29..85caee1fd829f 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -113,69 +113,19 @@
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
-namespace arrow {
-namespace io {
-
-// ----------------------------------------------------------------------
-// Cross-platform file compatability layer
-
-#if defined(_MSC_VER)
-
-constexpr const char* kRangeExceptionError =
-    "Range exception during wide-char string conversion";
-
-struct PlatformFilename {
-  static Status Init(const std::string& utf8_path, PlatformFilename* out) {
-    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> utf16_converter;
+#include <boost/filesystem.hpp>  // NOLINT
 
-    if (!utf8_path.empty()) {
-      try {
-        out->utf16_path = utf16_converter.from_bytes(utf8_path);
-      } catch (const std::range_error&) {
-        return Status::Invalid(kRangeExceptionError);
-      }
-    } else {
-      out->utf16_path = std::wstring();
-    }
-    out->utf8_path = utf8_path;
-    return Status::OK();
-  }
-
-  const char* data() const { return reinterpret_cast<const char*>(utf16_path.c_str()); }
-
-  const char* utf8_data() const { return utf8_path.c_str(); }
-
-  size_t length() const { return utf16_path.size(); }
-
-  std::string utf8_path;
-  std::wstring utf16_path;
-};
-
-#else
+namespace fs = boost::filesystem;
 
-struct PlatformFilename {
-  static Status Init(const std::string& utf8_path, PlatformFilename* out) {
-    out->utf8_path = utf8_path;
-    return Status::OK();
-  }
-
-  const char* data() const { return utf8_path.c_str(); }
-
-  const char* utf8_data() const { return data(); }
-
-  size_t length() const { return utf8_path.size(); }
-
-  std::string utf8_path;
-};
-
-#endif
+namespace arrow {
+namespace io {
 
 static inline Status CheckOpenResult(int ret, int errno_actual,
-                                     const PlatformFilename& filename) {
+                                     const fs::path& file_name) {
   if (ret == -1) {
     // TODO: errno codes to strings
     std::stringstream ss;
-    ss << "Failed to open local file: " << filename.utf8_data();
+    ss << "Failed to open local file: " << file_name.string();
     return Status::IOError(ss.str());
   }
   return Status::OK();
@@ -192,22 +142,22 @@ static inline int64_t lseek64_compat(int fd, int64_t pos, int whence) {
 #endif
 }
 
-static inline Status FileOpenReadable(const PlatformFilename& filename, int* fd) {
+static inline Status FileOpenReadable(const fs::path& file_name, int* fd) {
   int ret;
   errno_t errno_actual = 0;
 #if defined(_MSC_VER)
-  errno_actual = _wsopen_s(fd, reinterpret_cast<const wchar_t*>(filename.data()),
-                           _O_RDONLY | _O_BINARY, _SH_DENYNO, _S_IREAD);
+  errno_actual = _wsopen_s(fd, file_name.wstring().c_str(), _O_RDONLY | _O_BINARY,
+                           _SH_DENYNO, _S_IREAD);
   ret = *fd;
 #else
-  ret = *fd = open(filename.data(), O_RDONLY | O_BINARY);
+  ret = *fd = open(file_name.c_str(), O_RDONLY | O_BINARY);
   errno_actual = errno;
 #endif
 
-  return CheckOpenResult(ret, errno_actual, filename);
+  return CheckOpenResult(ret, errno_actual, file_name);
 }
 
-static inline Status FileOpenWriteable(const PlatformFilename& filename, bool write_only,
+static inline Status FileOpenWriteable(const fs::path& file_name, bool write_only,
                                        bool truncate, int* fd) {
   int ret;
   errno_t errno_actual = 0;
@@ -229,8 +179,7 @@ static inline Status FileOpenWriteable(const PlatformFilename& filename, bool wr
     oflag |= _O_RDWR;
   }
 
-  errno_actual = _wsopen_s(fd, reinterpret_cast<const wchar_t*>(filename.data()), oflag,
-                           _SH_DENYNO, pmode);
+  errno_actual = _wsopen_s(fd, file_name.wstring().c_str(), oflag, _SH_DENYNO, pmode);
   ret = *fd;
 
 #else
@@ -246,9 +195,9 @@ static inline Status FileOpenWriteable(const PlatformFilename& filename, bool wr
     oflag |= O_RDWR;
   }
 
-  ret = *fd = open(filename.data(), oflag, ARROW_WRITE_SHMODE);
+  ret = *fd = open(file_name.c_str(), oflag, ARROW_WRITE_SHMODE);
 #endif
-  return CheckOpenResult(ret, errno_actual, filename);
+  return CheckOpenResult(ret, errno_actual, file_name);
 }
 
 static inline Status FileTell(int fd, int64_t* pos) {
@@ -357,9 +306,9 @@ class OSFile {
   ~OSFile() {}
 
   Status OpenWriteable(const std::string& path, bool append, bool write_only) {
-    RETURN_NOT_OK(PlatformFilename::Init(path, &path_));
+    RETURN_NOT_OK(SetFileName(path));
 
-    RETURN_NOT_OK(FileOpenWriteable(path_, write_only, !append, &fd_));
+    RETURN_NOT_OK(FileOpenWriteable(file_name_, write_only, !append, &fd_));
     is_open_ = true;
     mode_ = write_only ? FileMode::WRITE : FileMode::READWRITE;
 
@@ -372,9 +321,9 @@ class OSFile {
   }
 
   Status OpenReadable(const std::string& path) {
-    RETURN_NOT_OK(PlatformFilename::Init(path, &path_));
+    RETURN_NOT_OK(SetFileName(path));
 
-    RETURN_NOT_OK(FileOpenReadable(path_, &fd_));
+    RETURN_NOT_OK(FileOpenReadable(file_name_, &fd_));
     RETURN_NOT_OK(FileGetSize(fd_, &size_));
 
     is_open_ = true;
@@ -421,7 +370,21 @@ class OSFile {
   FileMode::type mode() const { return mode_; }
 
  protected:
-  PlatformFilename path_;
+  Status SetFileName(const std::string& file_name) {
+    try {
+#if defined(_MSC_VER)
+      std::codecvt_utf8_utf16<wchar_t> utf16_converter;
+      file_name_.assign(file_name, utf16_converter);
+#else
+      file_name_ = file_name;
+#endif
+    } catch (boost::system::system_error& e) {
+      return Status::Invalid(e.what());
+    }
+    return Status::OK();
+  }
+
+  fs::path file_name_;
 
   std::mutex lock_;
 

From 31457ae826b7710db4982ae6680172c1068e4d88 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Tue, 15 Aug 2017 20:16:27 -0400
Subject: [PATCH 0933/1644] ARROW-801: Provide direct access to underlying
 buffer memory addresses

Added following methods at FieldVector interface

public long getValidityBufferAddress();
public long getDataBufferAddress();
public long getOffsetBufferAddress();

Couple of points:

For the UnionVector, we don't have an explicit BitVector and the typeVector also acts as a validity vector during get(), isNull(). So getValidityBufferAddress() returns the address of buffer associated with typeVector.

Both ListVector and FixedSizeListVector are backed by a FieldVector which acts as the dataVector. Right now the getDataBufferAddress() is unsupported for these two vectors but I think that we could just simply delegate this call to the dataVector instead of throwing exception

Author: siddharth <siddharth@dremio.com>

Closes #958 from siddharthteotia/ARROW-801 and squashes the following commits:

cb6dd25 [siddharth] ARROW-801: Provide direct access to underlying buffer memory addresses
---
 .../templates/NullableValueVectors.java       | 19 +++++
 .../main/codegen/templates/UnionVector.java   | 15 ++++
 .../org/apache/arrow/vector/FieldVector.java  | 21 +++++
 .../org/apache/arrow/vector/ZeroVector.java   | 15 ++++
 .../vector/complex/FixedSizeListVector.java   | 15 ++++
 .../arrow/vector/complex/ListVector.java      | 15 ++++
 .../vector/complex/NullableMapVector.java     | 15 ++++
 .../apache/arrow/vector/TestListVector.java   | 63 ++++++++++++++-
 .../apache/arrow/vector/TestUnionVector.java  | 70 ++++++++++++++++
 .../apache/arrow/vector/TestValueVector.java  | 81 +++++++++++++++++++
 10 files changed, 327 insertions(+), 2 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 624ba9d2cec69..bd322ea302866 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -435,6 +435,25 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
     <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return (bits.getBuffer().memoryAddress());
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    return (values.getBuffer().memoryAddress());
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    <#if type.major != "VarLen">
+        throw new UnsupportedOperationException();
+    <#else>
+        return (values.getOffsetAddr());
+    </#if>
+  }
+
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
     final BitVector.Accessor bAccessor = bits.getAccessor();
     final ${valuesName}.Accessor vAccessor = values.getAccessor();
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index eabe42a7c4ce5..8c01c4a3c7213 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -132,6 +132,21 @@ private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
     return internalMap.addOrGet(fieldName(minorType), fieldType(minorType), c);
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return typeVector.getBuffer().memoryAddress();
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
   public NullableMapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
index af7a7912c72cc..c2ed17eb4dd31 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
@@ -66,4 +66,25 @@ public interface FieldVector extends ValueVector {
    * @return the inner vectors for this field as defined by the TypeLayout
    */
   List<BufferBacked> getFieldInnerVectors();
+
+  /**
+   * Gets the starting address of the underlying buffer associated with validity vector
+   *
+   * @return buffer address
+   */
+  public long getValidityBufferAddress();
+
+  /**
+   * Gets the starting address of the underlying buffer associated with data vector
+   *
+   * @return buffer address
+   */
+  public long getDataBufferAddress();
+
+  /**
+   * Gets the starting address of the underlying buffer associated with offset vector
+   *
+   * @return buffer address
+   */
+  public long getOffsetBufferAddress();
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index cce73897718a3..fe224ba8a0589 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -229,4 +229,19 @@ public List<ArrowBuf> getFieldBuffers() {
   public List<BufferBacked> getFieldInnerVectors() {
     return Collections.emptyList();
   }
+
+  @Override
+  public long getValidityBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 3f0f1b05b6733..aab6e5715b18d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -293,6 +293,21 @@ public UnionVector promoteToUnion() {
     return vector;
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return (bits.getBuffer().memoryAddress());
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
   public class Accessor extends BaseValueVector.BaseAccessor {
 
     @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 7b6b97a8ed997..4b2808dabe7b7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -178,6 +178,21 @@ public TransferPair makeTransferPair(ValueVector target) {
     return new TransferImpl((ListVector) target);
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return (bits.getBuffer().memoryAddress());
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    return (offsets.getBuffer().memoryAddress());
+  }
+
   private class TransferImpl implements TransferPair {
 
     ListVector to;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index fda9c1471e589..09560c238c1fd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -231,6 +231,21 @@ public void reAlloc() {
     super.reAlloc();
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return bits.getBuffer().memoryAddress();
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
   public final class Accessor extends MapVector.Accessor {
     final BitVector.Accessor bAccessor = bits.getAccessor();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index fdb576ef75cc4..276121c0d1f3b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -42,6 +42,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import io.netty.buffer.ArrowBuf;
+
 
 public class TestListVector {
 
@@ -436,14 +438,14 @@ public void testSplitAndTransfer() throws Exception {
             dataLength2 = offsetAccessor1.get(i + 1) - offsetAccessor1.get(i);
 
             assertEquals("Different data lengths at index: " + i + " and start: " + start,
-                dataLength1, dataLength2);
+                    dataLength1, dataLength2);
 
             offset1 = offsetAccessor.get(start + i);
             offset2 = offsetAccessor1.get(i);
 
             for (int j = 0; j < dataLength1; j++) {
               assertEquals("Different data at indexes: " + offset1 + " and " + offset2,
-                  valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
+                      valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
 
               offset1++;
               offset2++;
@@ -569,4 +571,61 @@ public void testNestedListVector() throws Exception {
       assertEquals(5, offsetAccessor.get(2));
     }
   }
+
+  @Test
+  public void testGetBufferAddress() throws Exception {
+    try (ListVector listVector = ListVector.empty("vector", allocator)) {
+
+      UnionListWriter listWriter = listVector.getWriter();
+      boolean error = false;
+
+      listWriter.allocate();
+
+      listWriter.setPosition(0);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(50);
+      listWriter.bigInt().writeBigInt(100);
+      listWriter.bigInt().writeBigInt(200);
+      listWriter.endList();
+
+      listWriter.setPosition(1);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(250);
+      listWriter.bigInt().writeBigInt(300);
+      listWriter.endList();
+
+      final ListVector.Accessor accessor = listVector.getAccessor();
+
+      /* check listVector contents */
+      Object result = accessor.getObject(0);
+      ArrayList<Long> resultSet = (ArrayList<Long>) result;
+      assertEquals(3, resultSet.size());
+      assertEquals(new Long(50), resultSet.get(0));
+      assertEquals(new Long(100), resultSet.get(1));
+      assertEquals(new Long(200), resultSet.get(2));
+
+      result = accessor.getObject(1);
+      resultSet = (ArrayList<Long>) result;
+      assertEquals(2, resultSet.size());
+      assertEquals(new Long(250), resultSet.get(0));
+      assertEquals(new Long(300), resultSet.get(1));
+
+      List<ArrowBuf> buffers = listVector.getFieldBuffers();
+
+      long bitAddress = listVector.getValidityBufferAddress();
+      long offsetAddress = listVector.getOffsetBufferAddress();
+
+      try {
+        long dataAddress = listVector.getDataBufferAddress();
+      } catch (UnsupportedOperationException ue) {
+        error = true;
+      } finally {
+        assertTrue(error);
+      }
+
+      assertEquals(2, buffers.size());
+      assertEquals(bitAddress, buffers.get(0).memoryAddress());
+      assertEquals(offsetAddress, buffers.get(1).memoryAddress());
+    }
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index a75b196fbcc30..86f0bf337f9d4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -19,6 +19,9 @@
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.UnionVector;
@@ -33,6 +36,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import io.netty.buffer.ArrowBuf;
+
 public class TestUnionVector {
   private final static String EMPTY_SCHEMA_PATH = "";
 
@@ -292,6 +297,71 @@ public void testSplitAndTransferWithMixedVectors() throws Exception {
     }
   }
 
+  @Test
+  public void testGetBufferAddress() throws Exception {
+    try (UnionVector vector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      final UnionVector.Mutator mutator = vector.getMutator();
+      final UnionVector.Accessor accessor = vector.getAccessor();
+      boolean error = false;
+
+      vector.allocateNew();
+
+      /* populate the UnionVector */
+      mutator.setType(0, MinorType.INT);
+      mutator.setSafe(0, newIntHolder(5));
+
+      mutator.setType(1, MinorType.FLOAT4);
+      mutator.setSafe(1, newFloat4Holder(5.5f));
+
+      mutator.setType(2, MinorType.INT);
+      mutator.setSafe(2, newIntHolder(10));
+
+      mutator.setType(3, MinorType.FLOAT4);
+      mutator.setSafe(3, newFloat4Holder(10.5f));
+
+      mutator.setValueCount(10);
+
+      /* check the vector output */
+      assertEquals(10, accessor.getValueCount());
+      assertEquals(false, accessor.isNull(0));
+      assertEquals(5, accessor.getObject(0));
+      assertEquals(false, accessor.isNull(1));
+      assertEquals(5.5f, accessor.getObject(1));
+      assertEquals(false, accessor.isNull(2));
+      assertEquals(10, accessor.getObject(2));
+      assertEquals(false, accessor.isNull(3));
+      assertEquals(10.5f, accessor.getObject(3));
+
+      List<ArrowBuf> buffers = vector.getFieldBuffers();
+
+      long bitAddress = vector.getValidityBufferAddress();
+
+      try {
+        long offsetAddress = vector.getOffsetBufferAddress();
+      }
+      catch (UnsupportedOperationException ue) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      try {
+        long dataAddress = vector.getDataBufferAddress();
+      }
+      catch (UnsupportedOperationException ue) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+      }
+
+      assertEquals(1, buffers.size());
+      assertEquals(bitAddress, buffers.get(0).memoryAddress());
+    }
+  }
+
   private static NullableIntHolder newIntHolder(int value) {
     final NullableIntHolder holder = new NullableIntHolder();
     holder.isSet = 1;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 4e3021b7e719f..cb1fa89b272da 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -1164,6 +1164,87 @@ public void testFillEmptiesUsage() {
     }
   }
 
+  @Test /* NullableVarCharVector */
+  public void testGetBufferAddress1() {
+
+    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
+
+      final NullableVarCharVector.Mutator mutator = vector.getMutator();
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+
+      vector.allocateNew(1024 * 10, 1024);
+
+      /* populate the vector */
+      mutator.set(0, STR1);
+      mutator.set(1, STR2);
+      mutator.set(2, STR3);
+      mutator.set(3, STR4);
+      mutator.set(4, STR5);
+      mutator.set(5, STR6);
+
+      mutator.setValueCount(15);
+
+      /* check the vector output */
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(STR4, accessor.get(3));
+      assertArrayEquals(STR5, accessor.get(4));
+      assertArrayEquals(STR6, accessor.get(5));
+
+      List<ArrowBuf> buffers = vector.getFieldBuffers();
+      long bitAddress = vector.getValidityBufferAddress();
+      long offsetAddress = vector.getOffsetBufferAddress();
+      long dataAddress = vector.getDataBufferAddress();
+
+      assertEquals(3, buffers.size());
+      assertEquals(bitAddress, buffers.get(0).memoryAddress());
+      assertEquals(offsetAddress, buffers.get(1).memoryAddress());
+      assertEquals(dataAddress, buffers.get(2).memoryAddress());
+    }
+  }
+
+  @Test /* NullableIntVector */
+  public void testGetBufferAddress2() {
+
+    try (final NullableIntVector vector = new NullableIntVector("myvector", allocator)) {
+
+      final NullableIntVector.Mutator mutator = vector.getMutator();
+      final NullableIntVector.Accessor accessor = vector.getAccessor();
+      boolean error = false;
+
+      vector.allocateNew(16);
+
+      /* populate the vector */
+      for(int i = 0; i < 16; i += 2) {
+        mutator.set(i, i+10);
+      }
+
+      /* check the vector output */
+      for(int i = 0; i < 16; i += 2) {
+        assertEquals(i+10, accessor.get(i));
+      }
+
+      List<ArrowBuf> buffers = vector.getFieldBuffers();
+      long bitAddress = vector.getValidityBufferAddress();
+      long dataAddress = vector.getDataBufferAddress();
+
+      try {
+        long offsetAddress = vector.getOffsetBufferAddress();
+      }
+      catch (UnsupportedOperationException ue) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+      }
+
+      assertEquals(2, buffers.size());
+      assertEquals(bitAddress, buffers.get(0).memoryAddress());
+      assertEquals(dataAddress, buffers.get(1).memoryAddress());
+    }
+  }
+
   @Test
   public void testMultipleClose() {
     BufferAllocator vectorAllocator = allocator.newChildAllocator("vector_allocator", 0, Long.MAX_VALUE);

From c2fb9cb737be8231273dc68334fc3b16d75db85a Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 16 Aug 2017 09:17:19 -0400
Subject: [PATCH 0934/1644] ARROW-1356: [Website] Add new committers

Author: Kouhei Sutou <kou@clear-code.com>

Closes #968 from kou/add-new-committers and squashes the following commits:

710558b [Kouhei Sutou] [Website] Add new committers
---
 site/committers.html | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/site/committers.html b/site/committers.html
index 91afbabe0d4e6..3909922c744fd 100644
--- a/site/committers.html
+++ b/site/committers.html
@@ -93,6 +93,14 @@ <h2>Committers</h2>
 <td>Uwe L. Korn</td>
 <td>uwe</td>
 </tr>
+<tr>
+<td>Kouhei Sutou</td>
+<td>kou</td>
+</tr>
+<tr>
+<td>Philipp Moritz</td>
+<td>pcmoritz</td>
+</tr>
 </tbody></table>
 
     </div> <!-- /container -->

From b78e2efa84636b72e0432023b1b3b849cb1cd0b7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 16 Aug 2017 09:51:19 -0400
Subject: [PATCH 0935/1644] ARROW-1353: [Website] Update website for 0.6.0
 release and add short release blog post

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #967 from wesm/ARROW-1353 and squashes the following commits:

804fe35 [Wes McKinney] Escape underscores in CHANGELOG.md
1b7c4b6 [Wes McKinney] Finish 0.6.0 blog post
a78cb94 [Wes McKinney] Some updates for 0.6.0 site update
---
 CHANGELOG.md                            | 110 ++++++++++++++++-
 site/_posts/2017-08-16-0.6.0-release.md | 112 +++++++++++++++++
 site/_release/0.6.0.md                  | 156 ++++++++++++++++++++++++
 site/_release/index.md                  |   2 +
 site/index.html                         |   4 +-
 site/install.md                         |  36 +++---
 6 files changed, 399 insertions(+), 21 deletions(-)
 create mode 100644 site/_posts/2017-08-16-0.6.0-release.md
 create mode 100644 site/_release/0.6.0.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6cedf32df628f..8fa4e4242e598 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,13 +17,119 @@
   under the License.
 -->
 
+# Apache Arrow 0.6.0 (14 August 2017)
+
+## Bug
+
+* ARROW-1192 - [JAVA] Improve splitAndTransfer performance for List and Union vectors
+* ARROW-1195 - [C++] CpuInfo doesn't get cache size on Windows
+* ARROW-1204 - [C++] lz4 ExternalProject fails in Visual Studio 2015
+* ARROW-1225 - [Python] pyarrow.array does not attempt to convert bytes to UTF8 when passed a StringType
+* ARROW-1237 - [JAVA] Expose the ability to set lastSet
+* ARROW-1239 - issue with current version of git-commit-id-plugin
+* ARROW-1240 - security: upgrade logback to address CVE-2017-5929
+* ARROW-1242 - [Java] security - upgrade Jackson to mitigate 3 CVE vulnerabilities
+* ARROW-1245 - [Integration] Java Integration Tests Disabled
+* ARROW-1248 - [Python] C linkage warnings in Clang with public Cython API
+* ARROW-1249 - [JAVA] Expose the fillEmpties function from Nullable<Varlength>Vector.mutator
+* ARROW-1263 - [C++] CpuInfo should be able to get CPU features on Windows
+* ARROW-1265 - [Plasma] Plasma store memory leak warnings in Python test suite
+* ARROW-1267 - [Java] Handle zero length case in BitVector.splitAndTransfer
+* ARROW-1269 - [Packaging] Add Windows wheel build scripts from ARROW-1068 to arrow-dist
+* ARROW-1275 - [C++] Default static library prefix for Snappy should be "\_static"
+* ARROW-1276 - Cannot serializer empty DataFrame to parquet
+* ARROW-1283 - [Java] VectorSchemaRoot should be able to be closed() more than once
+* ARROW-1285 - PYTHON: NotImplemented exception creates empty parquet file
+* ARROW-1287 - [Python] Emulate "whence" argument of seek in NativeFile
+* ARROW-1290 - [C++] Use array capacity doubling in arrow::BufferBuilder
+* ARROW-1291 - [Python] `pa.RecordBatch.from_pandas` doesn't accept DataFrame with numeric column names
+* ARROW-1294 - [C++] New Appveyor build failures
+* ARROW-1296 - [Java] templates/FixValueVectors reset() method doesn't set allocationSizeInBytes correctly
+* ARROW-1300 - [JAVA] Fix ListVector Tests
+* ARROW-1306 - [Python] Encoding? issue with error reporting for `parquet.read_table`
+* ARROW-1308 - [C++] ld tries to link `arrow_static` even when -DARROW_BUILD_STATIC=off
+* ARROW-1309 - [Python] Error inferring List type in `Array.from_pandas` when inner values are all None
+* ARROW-1310 - [JAVA] Revert ARROW-886
+* ARROW-1312 - [C++] Set default value to `ARROW_JEMALLOC` to OFF until ARROW-1282 is resolved
+* ARROW-1326 - [Python] Fix Sphinx build in Travis CI
+* ARROW-1327 - [Python] Failing to release GIL in `MemoryMappedFile._open` causes deadlock
+* ARROW-1328 - [Python] `pyarrow.Table.from_pandas` option `timestamps_to_ms` changes column values
+* ARROW-1330 - [Plasma] Turn on plasma tests on manylinux1
+* ARROW-1335 - [C++] `PrimitiveArray::raw_values` has inconsistent semantics re: offsets compared with subclasses
+* ARROW-1338 - [Python] Investigate non-deterministic core dump on Python 2.7, Travis CI builds
+* ARROW-1340 - [Java] NullableMapVector field doesn't maintain metadata
+* ARROW-1342 - [Python] Support strided array of lists
+* ARROW-1343 - [Format/Java/C++] Ensuring encapsulated stream / IPC message sizes are always a multiple of 8
+* ARROW-1350 - [C++] Include Plasma source tree in source distribution
+* ARROW-187 - [C++] Decide on how pedantic we want to be about exceptions
+* ARROW-276 - [JAVA] Nullable Value Vectors should extend BaseValueVector instead of BaseDataValueVector
+* ARROW-573 - [Python/C++] Support ordered dictionaries data, pandas Categorical
+* ARROW-884 - [C++] Exclude internal classes from documentation
+* ARROW-932 - [Python] Fix compiler warnings on MSVC
+* ARROW-968 - [Python] RecordBatch [i:j] syntax is incomplete
+
+## Improvement
+
+* ARROW-1093 - [Python] Fail Python builds if flake8 yields warnings
+* ARROW-1121 - [C++] Improve error message when opening OS file fails
+* ARROW-1140 - [C++] Allow optional build of plasma
+* ARROW-1149 - [Plasma] Create Cython client library for Plasma
+* ARROW-1173 - [Plasma] Blog post for Plasma
+* ARROW-1211 - [C++] Consider making `default_memory_pool()` the default for builder classes
+* ARROW-1213 - [Python] Enable s3fs to be used with ParquetDataset and reader/writer functions
+* ARROW-1219 - [C++] Use more vanilla Google C++ formatting
+* ARROW-1224 - [Format] Clarify language around buffer padding and alignment in IPC
+* ARROW-1230 - [Plasma] Install libraries and headers
+* ARROW-1243 - [Java] security: upgrade all libraries to latest stable versions
+* ARROW-1251 - [Python/C++] Revise build documentation to account for latest build toolchain
+* ARROW-1253 - [C++] Use pre-built toolchain libraries where prudent to speed up CI builds
+* ARROW-1255 - [Plasma] Check plasma flatbuffer messages with the flatbuffer verifier
+* ARROW-1257 - [Plasma] Plasma documentation
+* ARROW-1258 - [C++] Suppress dlmalloc warnings on Clang
+* ARROW-1259 - [Plasma] Speed up Plasma tests
+* ARROW-1260 - [Plasma] Use factory method to create Python PlasmaClient
+* ARROW-1264 - [Plasma] Don't exit the Python interpreter if the plasma client can't connect to the store
+* ARROW-1274 - [C++] `add_compiler_export_flags()` throws warning with CMake >= 3.3
+* ARROW-1288 - Clean up many ASF license headers
+* ARROW-1289 - [Python] Add `PYARROW_BUILD_PLASMA` option like Parquet
+* ARROW-1301 - [C++/Python] Add remaining supported libhdfs UNIX-like filesystem APIs
+* ARROW-1303 - [C++] Support downloading Boost
+* ARROW-1315 - [GLib] Status check of arrow::ArrayBuilder::Finish() is missing
+* ARROW-1323 - [GLib] Add `garrow_boolean_array_get_values()`
+* ARROW-1333 - [Plasma] Sorting example for DataFrames in plasma
+* ARROW-1334 - [C++] Instantiate arrow::Table from vector of Array objects (instead of Columns)
+
+## New Feature
+
+* ARROW-1076 - [Python] Handle nanosecond timestamps more gracefully when writing to Parquet format
+* ARROW-1104 - Integrate in-memory object store from Ray
+* ARROW-1246 - [Format] Add Map logical type to metadata
+* ARROW-1268 - [Website] Blog post on Arrow integration with Spark
+* ARROW-1281 - [C++/Python] Add Docker setup for running HDFS tests and other tests we may not run in Travis CI
+* ARROW-1305 - [GLib] Add GArrowIntArrayBuilder
+* ARROW-1336 - [C++] Add arrow::schema factory function
+* ARROW-439 - [Python] Add option in `to_pandas` conversions to yield Categorical from String/Binary arrays
+* ARROW-622 - [Python] Investigate alternatives to `timestamps_to_ms` argument in pandas conversion
+
+## Task
+
+* ARROW-1270 - [Packaging] Add Python wheel build scripts for macOS to arrow-dist
+* ARROW-1272 - [Python] Add script to arrow-dist to generate and upload manylinux1 Python wheels
+* ARROW-1273 - [Python] Add convenience functions for reading only Parquet metadata or effective Arrow schema from a particular Parquet file
+* ARROW-1297 - 0.6.0 Release
+* ARROW-1304 - [Java] Fix checkstyle checks warning
+
+## Test
+
+* ARROW-1241 - [C++] Visual Studio 2017 Appveyor build job
+
 # Apache Arrow 0.5.0 (23 July 2017)
 
 ## Bug
 
-* ARROW-1074 - from_pandas doesnt convert ndarray to list
+* ARROW-1074 - `from_pandas` doesnt convert ndarray to list
 * ARROW-1079 - [Python] Empty "private" directories should be ignored by Parquet interface
-* ARROW-1081 - C++: arrow::test::TestBase::MakePrimitive doesn't fill null_bitmap
+* ARROW-1081 - C++: arrow::test::TestBase::MakePrimitive doesn't fill `null_bitmap`
 * ARROW-1096 - [C++] Memory mapping file over 4GB fails on Windows
 * ARROW-1097 - Reading tensor needs file to be opened in writeable mode
 * ARROW-1098 - Document Error?
diff --git a/site/_posts/2017-08-16-0.6.0-release.md b/site/_posts/2017-08-16-0.6.0-release.md
new file mode 100644
index 0000000000000..2796c4b821c69
--- /dev/null
+++ b/site/_posts/2017-08-16-0.6.0-release.md
@@ -0,0 +1,112 @@
+---
+layout: post
+title: "Apache Arrow 0.6.0 Release"
+date: "2017-08-16 00:00:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.6.0 release. It includes
+[**90 resolved JIRAs**][1] with the new Plasma shared memory object store, and
+improvements and bug fixes to the various language implementations. The Arrow
+memory format remains stable since the 0.3.x release.
+
+See the [Install Page][2] to learn how to get the libraries for your
+platform. The [complete changelog][5] is also available.
+
+## Plasma Shared Memory Object Store
+
+This release includes the [Plasma Store][7], which you can read more about in
+the linked blog post. This system was originally developed as part of the [Ray
+Project][8] at the [UC Berkeley RISELab][9]. We recognized that Plasma would be
+highly valuable to the Arrow community as a tool for shared memory management
+and zero-copy deserialization. Additionally, we believe we will be able to
+develop a stronger software stack through sharing of IO and buffer management
+code.
+
+The Plasma store is a server application which runs as a separate process. A
+reference C++ client, with Python bindings, is made available in this
+release. Clients can be developed in Java or other languages in the future to
+enable simple sharing of complex datasets through shared memory.
+
+## Arrow Format Addition: Map type
+
+We added a Map logical type to represent ordered and unordered maps
+in-memory. This corresponds to the `MAP` logical type annotation in the Parquet
+format (where maps are represented as repeated structs).
+
+Map is represented as a list of structs. It is the first example of a logical
+type whose physical representation is a nested type. We have not yet created
+implementations of Map containers in any of the implementations, but this can
+be done in a future release.
+
+As an example, the Python data:
+
+```
+data = [{'a': 1, 'bb': 2, 'cc': 3}, {'dddd': 4}]
+```
+
+Could be represented in an Arrow `Map<String, Int32>` as:
+
+```
+Map<String, Int32> = List<Struct<keys: String, values: Int32>>
+  is_valid: [true, true]
+  offsets: [0, 3, 4]
+  values: Struct<keys: String, values: Int32>
+    children:
+      - keys: String
+          is_valid: [true, true, true, true]
+          offsets: [0, 1, 3, 5, 9]
+          data: abbccdddd
+      - values: Int32
+          is_valid: [true, true, true, true]
+          data: [1, 2, 3, 4]
+```
+## Python Changes
+
+Some highlights of Python development outside of bug fixes and general API
+improvements include:
+
+* New `strings_to_categorical=True` option when calling `Table.to_pandas` will
+  yield pandas `Categorical` types from Arrow binary and string columns
+* Expanded Hadoop Filesystem (HDFS) functionality to improve compatibility with
+  Dask and other HDFS-aware Python libraries.
+* s3fs and other Dask-oriented filesystems can now be used with
+  `pyarrow.parquet.ParquetDataset`
+* More graceful handling of pandas's nanosecond timestamps when writing to
+  Parquet format. You can now pass `coerce_timestamps='ms'` to cast to
+  milliseconds, or `'us'` for microseconds.
+
+## Toward Arrow 1.0.0 and Beyond
+
+We are still discussing the roadmap to 1.0.0 release on the [developer mailing
+list][6]. The focus of the 1.0.0 release will likely be memory format stability
+and hardening integration tests across the remaining data types implemented in
+Java and C++. Please join the discussion there.
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.6.0
+[2]: http://arrow.apache.org/install
+[3]: http://github.com/apache/parquet-cpp
+[5]: http://arrow.apache.org/release/0.6.0.html
+[6]: http://mail-archives.apache.org/mod_mbox/arrow-dev/
+[7]: http://arrow.apache.org/blog/2017/08/08/plasma-in-memory-object-store/
+[8]: https://ray-project.github.io/ray/
+[9]: https://rise.cs.berkeley.edu/
\ No newline at end of file
diff --git a/site/_release/0.6.0.md b/site/_release/0.6.0.md
new file mode 100644
index 0000000000000..061393c383cd6
--- /dev/null
+++ b/site/_release/0.6.0.md
@@ -0,0 +1,156 @@
+---
+layout: default
+title: Apache Arrow 0.6.0 Release
+permalink: /release/0.6.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.6.0 (14 August 2017)
+
+This is a major release. Read more in the [release blog post][3].
+
+## Download
+
+* [**Source Artifacts**][2]
+* [Git tag][1]
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.5.0..apache-arrow-0.6.0
+    48  Wes McKinney
+     7  siddharth
+     5  Matt Darwin
+     5  Max Risuhin
+     5  Philipp Moritz
+     4  Kouhei Sutou
+     3  Bryan Cutler
+     2  Emilio Lahr-Vivaz
+     2  Li Jin
+     2  Robert Nishihara
+     1  Antony Mayi
+     1  Marco Neumann
+     1  Stepan Kadlec
+     1  Steven Phillips
+     1  Yeolar
+     1  fjetter
+     1  rendel
+```
+
+# Changelog
+
+## New Features and Improvements
+
+* [ARROW-1076](https://issues.apache.org/jira/browse/ARROW-1076) - [Python] Handle nanosecond timestamps more gracefully when writing to Parquet format
+* [ARROW-1093](https://issues.apache.org/jira/browse/ARROW-1093) - [Python] Fail Python builds if flake8 yields warnings
+* [ARROW-1104](https://issues.apache.org/jira/browse/ARROW-1104) - Integrate in-memory object store from Ray
+* [ARROW-1121](https://issues.apache.org/jira/browse/ARROW-1121) - [C++] Improve error message when opening OS file fails
+* [ARROW-1140](https://issues.apache.org/jira/browse/ARROW-1140) - [C++] Allow optional build of plasma
+* [ARROW-1149](https://issues.apache.org/jira/browse/ARROW-1149) - [Plasma] Create Cython client library for Plasma
+* [ARROW-1173](https://issues.apache.org/jira/browse/ARROW-1173) - [Plasma] Blog post for Plasma
+* [ARROW-1211](https://issues.apache.org/jira/browse/ARROW-1211) - [C++] Consider making default_memory_pool() the default for builder classes
+* [ARROW-1213](https://issues.apache.org/jira/browse/ARROW-1213) - [Python] Enable s3fs to be used with ParquetDataset and reader/writer functions
+* [ARROW-1219](https://issues.apache.org/jira/browse/ARROW-1219) - [C++] Use more vanilla Google C++ formatting
+* [ARROW-1224](https://issues.apache.org/jira/browse/ARROW-1224) - [Format] Clarify language around buffer padding and alignment in IPC
+* [ARROW-1230](https://issues.apache.org/jira/browse/ARROW-1230) - [Plasma] Install libraries and headers
+* [ARROW-1241](https://issues.apache.org/jira/browse/ARROW-1241) - [C++] Visual Studio 2017 Appveyor build job
+* [ARROW-1243](https://issues.apache.org/jira/browse/ARROW-1243) - [Java] security: upgrade all libraries to latest stable versions
+* [ARROW-1246](https://issues.apache.org/jira/browse/ARROW-1246) - [Format] Add Map logical type to metadata
+* [ARROW-1251](https://issues.apache.org/jira/browse/ARROW-1251) - [Python/C++] Revise build documentation to account for latest build toolchain
+* [ARROW-1253](https://issues.apache.org/jira/browse/ARROW-1253) - [C++] Use pre-built toolchain libraries where prudent to speed up CI builds
+* [ARROW-1255](https://issues.apache.org/jira/browse/ARROW-1255) - [Plasma] Check plasma flatbuffer messages with the flatbuffer verifier
+* [ARROW-1257](https://issues.apache.org/jira/browse/ARROW-1257) - [Plasma] Plasma documentation
+* [ARROW-1258](https://issues.apache.org/jira/browse/ARROW-1258) - [C++] Suppress dlmalloc warnings on Clang
+* [ARROW-1259](https://issues.apache.org/jira/browse/ARROW-1259) - [Plasma] Speed up Plasma tests
+* [ARROW-1260](https://issues.apache.org/jira/browse/ARROW-1260) - [Plasma] Use factory method to create Python PlasmaClient
+* [ARROW-1264](https://issues.apache.org/jira/browse/ARROW-1264) - [Plasma] Don't exit the Python interpreter if the plasma client can't connect to the store
+* [ARROW-1268](https://issues.apache.org/jira/browse/ARROW-1268) - [Website] Blog post on Arrow integration with Spark
+* [ARROW-1270](https://issues.apache.org/jira/browse/ARROW-1270) - [Packaging] Add Python wheel build scripts for macOS to arrow-dist
+* [ARROW-1272](https://issues.apache.org/jira/browse/ARROW-1272) - [Python] Add script to arrow-dist to generate and upload manylinux1 Python wheels
+* [ARROW-1273](https://issues.apache.org/jira/browse/ARROW-1273) - [Python] Add convenience functions for reading only Parquet metadata or effective Arrow schema from a particular Parquet file
+* [ARROW-1274](https://issues.apache.org/jira/browse/ARROW-1274) - [C++] add_compiler_export_flags() throws warning with CMake >= 3.3
+* [ARROW-1281](https://issues.apache.org/jira/browse/ARROW-1281) - [C++/Python] Add Docker setup for running HDFS tests and other tests we may not run in Travis CI
+* [ARROW-1288](https://issues.apache.org/jira/browse/ARROW-1288) - Clean up many ASF license headers
+* [ARROW-1289](https://issues.apache.org/jira/browse/ARROW-1289) - [Python] Add PYARROW_BUILD_PLASMA option like Parquet
+* [ARROW-1297](https://issues.apache.org/jira/browse/ARROW-1297) - 0.6.0 Release
+* [ARROW-1301](https://issues.apache.org/jira/browse/ARROW-1301) - [C++/Python] Add remaining supported libhdfs UNIX-like filesystem APIs
+* [ARROW-1303](https://issues.apache.org/jira/browse/ARROW-1303) - [C++] Support downloading Boost
+* [ARROW-1304](https://issues.apache.org/jira/browse/ARROW-1304) - [Java] Fix checkstyle checks warning
+* [ARROW-1305](https://issues.apache.org/jira/browse/ARROW-1305) - [GLib] Add GArrowIntArrayBuilder
+* [ARROW-1315](https://issues.apache.org/jira/browse/ARROW-1315) - [GLib] Status check of arrow::ArrayBuilder::Finish() is missing
+* [ARROW-1323](https://issues.apache.org/jira/browse/ARROW-1323) - [GLib] Add garrow_boolean_array_get_values()
+* [ARROW-1333](https://issues.apache.org/jira/browse/ARROW-1333) - [Plasma] Sorting example for DataFrames in plasma
+* [ARROW-1334](https://issues.apache.org/jira/browse/ARROW-1334) - [C++] Instantiate arrow::Table from vector of Array objects (instead of Columns)
+* [ARROW-1336](https://issues.apache.org/jira/browse/ARROW-1336) - [C++] Add arrow::schema factory function
+* [ARROW-439](https://issues.apache.org/jira/browse/ARROW-439) - [Python] Add option in "to_pandas" conversions to yield Categorical from String/Binary arrays
+* [ARROW-622](https://issues.apache.org/jira/browse/ARROW-622) - [Python] Investigate alternatives to timestamps_to_ms argument in pandas conversion
+
+## Bug Fixes
+
+* [ARROW-1192](https://issues.apache.org/jira/browse/ARROW-1192) - [JAVA] Improve splitAndTransfer performance for List and Union vectors
+* [ARROW-1195](https://issues.apache.org/jira/browse/ARROW-1195) - [C++] CpuInfo doesn't get cache size on Windows
+* [ARROW-1204](https://issues.apache.org/jira/browse/ARROW-1204) - [C++] lz4 ExternalProject fails in Visual Studio 2015
+* [ARROW-1225](https://issues.apache.org/jira/browse/ARROW-1225) - [Python] pyarrow.array does not attempt to convert bytes to UTF8 when passed a StringType
+* [ARROW-1237](https://issues.apache.org/jira/browse/ARROW-1237) - [JAVA] Expose the ability to set lastSet
+* [ARROW-1239](https://issues.apache.org/jira/browse/ARROW-1239) - issue with current version of git-commit-id-plugin
+* [ARROW-1240](https://issues.apache.org/jira/browse/ARROW-1240) - security: upgrade logback to address CVE-2017-5929
+* [ARROW-1242](https://issues.apache.org/jira/browse/ARROW-1242) - [Java] security - upgrade Jackson to mitigate 3 CVE vulnerabilities
+* [ARROW-1245](https://issues.apache.org/jira/browse/ARROW-1245) - [Integration] Java Integration Tests Disabled
+* [ARROW-1248](https://issues.apache.org/jira/browse/ARROW-1248) - [Python] C linkage warnings in Clang with public Cython API
+* [ARROW-1249](https://issues.apache.org/jira/browse/ARROW-1249) - [JAVA] Expose the fillEmpties function from Nullable<Varlength>Vector.mutator
+* [ARROW-1263](https://issues.apache.org/jira/browse/ARROW-1263) - [C++] CpuInfo should be able to get CPU features on Windows
+* [ARROW-1265](https://issues.apache.org/jira/browse/ARROW-1265) - [Plasma] Plasma store memory leak warnings in Python test suite
+* [ARROW-1267](https://issues.apache.org/jira/browse/ARROW-1267) - [Java] Handle zero length case in BitVector.splitAndTransfer
+* [ARROW-1269](https://issues.apache.org/jira/browse/ARROW-1269) - [Packaging] Add Windows wheel build scripts from ARROW-1068 to arrow-dist
+* [ARROW-1275](https://issues.apache.org/jira/browse/ARROW-1275) - [C++] Default static library prefix for Snappy should be "_static"
+* [ARROW-1276](https://issues.apache.org/jira/browse/ARROW-1276) - Cannot serializer empty DataFrame to parquet
+* [ARROW-1283](https://issues.apache.org/jira/browse/ARROW-1283) - [Java] VectorSchemaRoot should be able to be closed() more than once
+* [ARROW-1285](https://issues.apache.org/jira/browse/ARROW-1285) - PYTHON: NotImplemented exception creates empty parquet file
+* [ARROW-1287](https://issues.apache.org/jira/browse/ARROW-1287) - [Python] Emulate "whence" argument of seek in NativeFile
+* [ARROW-1290](https://issues.apache.org/jira/browse/ARROW-1290) - [C++] Use array capacity doubling in arrow::BufferBuilder
+* [ARROW-1291](https://issues.apache.org/jira/browse/ARROW-1291) - [Python] pa.RecordBatch.from_pandas doesn't accept DataFrame with numeric column names
+* [ARROW-1294](https://issues.apache.org/jira/browse/ARROW-1294) - [C++] New Appveyor build failures
+* [ARROW-1296](https://issues.apache.org/jira/browse/ARROW-1296) - [Java] templates/FixValueVectors reset() method doesn't set allocationSizeInBytes correctly
+* [ARROW-1300](https://issues.apache.org/jira/browse/ARROW-1300) - [JAVA] Fix ListVector Tests
+* [ARROW-1306](https://issues.apache.org/jira/browse/ARROW-1306) - [Python] Encoding? issue with error reporting for parquet.read_table
+* [ARROW-1308](https://issues.apache.org/jira/browse/ARROW-1308) - [C++] ld tries to link 'arrow_static' even when -DARROW_BUILD_STATIC=off
+* [ARROW-1309](https://issues.apache.org/jira/browse/ARROW-1309) - [Python] Error inferring List type in Array.from_pandas when inner values are all None
+* [ARROW-1310](https://issues.apache.org/jira/browse/ARROW-1310) - [JAVA] Revert ARROW-886
+* [ARROW-1312](https://issues.apache.org/jira/browse/ARROW-1312) - [C++] Set default value to ARROW_JEMALLOC to OFF until ARROW-1282 is resolved
+* [ARROW-1326](https://issues.apache.org/jira/browse/ARROW-1326) - [Python] Fix Sphinx build in Travis CI
+* [ARROW-1327](https://issues.apache.org/jira/browse/ARROW-1327) - [Python] Failing to release GIL in MemoryMappedFile._open causes deadlock
+* [ARROW-1328](https://issues.apache.org/jira/browse/ARROW-1328) - [Python] pyarrow.Table.from_pandas option timestamps_to_ms changes column values
+* [ARROW-1330](https://issues.apache.org/jira/browse/ARROW-1330) - [Plasma] Turn on plasma tests on manylinux1
+* [ARROW-1335](https://issues.apache.org/jira/browse/ARROW-1335) - [C++] PrimitiveArray::raw_values has inconsistent semantics re: offsets compared with subclasses
+* [ARROW-1338](https://issues.apache.org/jira/browse/ARROW-1338) - [Python] Investigate non-deterministic core dump on Python 2.7, Travis CI builds
+* [ARROW-1340](https://issues.apache.org/jira/browse/ARROW-1340) - [Java] NullableMapVector field doesn't maintain metadata
+* [ARROW-1342](https://issues.apache.org/jira/browse/ARROW-1342) - [Python] Support strided array of lists
+* [ARROW-1343](https://issues.apache.org/jira/browse/ARROW-1343) - [Format/Java/C++] Ensuring encapsulated stream / IPC message sizes are always a multiple of 8
+* [ARROW-1350](https://issues.apache.org/jira/browse/ARROW-1350) - [C++] Include Plasma source tree in source distribution
+* [ARROW-187](https://issues.apache.org/jira/browse/ARROW-187) - [C++] Decide on how pedantic we want to be about exceptions
+* [ARROW-276](https://issues.apache.org/jira/browse/ARROW-276) - [JAVA] Nullable Value Vectors should extend BaseValueVector instead of BaseDataValueVector
+* [ARROW-573](https://issues.apache.org/jira/browse/ARROW-573) - [Python/C++] Support ordered dictionaries data, pandas Categorical
+* [ARROW-884](https://issues.apache.org/jira/browse/ARROW-884) - [C++] Exclude internal classes from documentation
+* [ARROW-932](https://issues.apache.org/jira/browse/ARROW-932) - [Python] Fix compiler warnings on MSVC
+* [ARROW-968](https://issues.apache.org/jira/browse/ARROW-968) - [Python] RecordBatch [i:j] syntax is incomplete
+
+[1]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.6.0
+[2]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/
+[3]: http://arrow.apache.org/blog/2017/08/16/0.6.0-release/
\ No newline at end of file
diff --git a/site/_release/index.md b/site/_release/index.md
index f18cff3b649e9..b373d8bfe1993 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -26,6 +26,7 @@ limitations under the License.
 
 Navigate to the release page for downloads and the changelog.
 
+* [0.6.0 (14 August 2017)][7]
 * [0.5.0 (23 July 2017)][6]
 * [0.4.1 (9 June 2017)][5]
 * [0.4.0 (22 May 2017)][4]
@@ -39,3 +40,4 @@ Navigate to the release page for downloads and the changelog.
 [4]: {{ site.baseurl }}/release/0.4.0.html
 [5]: {{ site.baseurl }}/release/0.4.1.html
 [6]: {{ site.baseurl }}/release/0.5.0.html
+[7]: {{ site.baseurl }}/release/0.6.0.html
diff --git a/site/index.html b/site/index.html
index 8a06c6acec580..224e5da358333 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,10 +7,10 @@ <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
         <p>
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
-          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.5.0 Release - July 23, 2017)</a>
+          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.6.0 Release - August 14, 2017)</a>
         </p>
       </div>
-      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.5.0 release</a></h4>
+      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.6.0 release</a></h4>
       <div class="row">
         <div class="col-lg-4">
           <h2>Fast</h2>
diff --git a/site/install.md b/site/install.md
index bd45642fe201f..bfea0b179d00d 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,17 +20,17 @@ limitations under the License.
 {% endcomment %}
 -->
 
-## Current Version: 0.5.0
+## Current Version: 0.6.0
 
-### Released: 23 July 2017
+### Released: 14 August 2017
 
-See the [release notes][10] and [blog post][11] for more about what's new.
+See the [release notes][10] for more about what's new.
 
 ### Source release
 
-* **Source Release**: [apache-arrow-0.5.0.tar.gz][6]
+* **Source Release**: [apache-arrow-0.6.0.tar.gz][6]
 * **Verification**: [md5][3], [asc][7]
-* [Git tag e9f76e1][2]
+* [Git tag b173334][2]
 
 ### Java Packages
 
@@ -38,7 +38,7 @@ See the [release notes][10] and [blog post][11] for more about what's new.
 
 ## Binary Installers for C, C++, Python
 
-It may take a little time for the binary packages to get updated
+Binary packages may not be updated immediately after the source release is posted.
 
 ### C++ and Python Conda Packages (Unofficial)
 
@@ -52,8 +52,8 @@ Install them with:
 
 
 ```shell
-conda install arrow-cpp=0.5.0 -c conda-forge
-conda install pyarrow=0.5.0 -c conda-forge
+conda install arrow-cpp=0.6.* -c conda-forge
+conda install pyarrow==0.6.* -c conda-forge
 ```
 
 ### Python Wheels on PyPI (Unofficial)
@@ -61,9 +61,12 @@ conda install pyarrow=0.5.0 -c conda-forge
 We have provided binary wheels on PyPI for Linux, macOS, and Windows:
 
 ```shell
-pip install pyarrow==0.5.0
+pip install pyarrow==0.6.*
 ```
 
+We recommend pinning `0.6.*` in `requirements.txt` to install the latest patch
+release.
+
 These include the Apache Arrow and Apache Parquet C++ binary libraries bundled
 with the wheel.
 
@@ -133,14 +136,13 @@ These repositories are managed at
 [red-data-tools/arrow-packages][9]. If you have any feedback, please
 send it to the project instead of Apache Arrow project.
 
-[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.5.0
-[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/apache-arrow-0.5.0.tar.gz.md5
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.5.0%22
+[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.6.0
+[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/apache-arrow-0.6.0.tar.gz.md5
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.6.0%22
 [5]: http://conda-forge.github.io
-[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/apache-arrow-0.5.0.tar.gz
-[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.5.0/apache-arrow-0.5.0.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/apache-arrow-0.6.0.tar.gz
+[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/apache-arrow-0.6.0.tar.gz.asc
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
-[10]: http://arrow.apache.org/release/0.5.0.html
-[11]: http://arrow.apache.org/blog/2017/07/25/0.5.0-release/
+[10]: http://arrow.apache.org/release/0.6.0.html

From 4471dc948bc3a666722c5bd82a85ed8017ad5f8f Mon Sep 17 00:00:00 2001
From: HorimotoYasuhiro <spasgate870@movie.ocn.ne.jp>
Date: Wed, 16 Aug 2017 22:55:55 +0900
Subject: [PATCH 0936/1644] [C++] DOC: Fix a typo in plasma.md

Closes #970

Change-Id: I49ea3f7f99d080c517fb21b86b7a27e17b04e20b
---
 cpp/apidoc/tutorials/plasma.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/apidoc/tutorials/plasma.md b/cpp/apidoc/tutorials/plasma.md
index 9911546ed5c69..9fc4175c64be5 100644
--- a/cpp/apidoc/tutorials/plasma.md
+++ b/cpp/apidoc/tutorials/plasma.md
@@ -183,7 +183,7 @@ You can also specify metadata for the object; the third argument is the
 metadata (as raw bytes) and the fourth argument is the size of the metadata.
 
 ```cpp
-// Create a Plasma object without metadata.
+// Create a Plasma object with metadata.
 int64_t data_size = 100;
 std::string metadata = "{'author': 'john'}";
 uint8_t* data;

From c0fa8e03a6d36b85ae6babb7f33e0653b3f7a7ba Mon Sep 17 00:00:00 2001
From: Kyle Kelley <rgbkrk@gmail.com>
Date: Wed, 16 Aug 2017 09:54:22 -0700
Subject: [PATCH 0937/1644] [Python] DOC: Fix Parquet docs to use
 pyarrow.parquet namespace for write_table

Closes #971

Change-Id: I7c689b200a4f04af51928f6765362fef52c613e8
---
 python/doc/source/parquet.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/doc/source/parquet.rst b/python/doc/source/parquet.rst
index 3317b99c0b685..7626c15048157 100644
--- a/python/doc/source/parquet.rst
+++ b/python/doc/source/parquet.rst
@@ -175,7 +175,7 @@ These settings can also be set on a per-column basis:
 
 .. code-block:: python
 
-   pa.write_table(table, where, compression={'foo': 'snappy', 'bar': 'gzip'},
+   pq.write_table(table, where, compression={'foo': 'snappy', 'bar': 'gzip'},
                   use_dictionary=['foo', 'bar'])
 
 Reading Multiples Files and Partitioned Datasets

From 3c5290a7bc5b0689df6dbba1ef0b3f7c385d9c2a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 16 Aug 2017 23:13:49 -0400
Subject: [PATCH 0938/1644] ARROW-1365: [Python] Remove outdated
 pyarrow.jemalloc_memory_pool example. Update API doc site build instructions

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #973 from wesm/site-doc-updates and squashes the following commits:

8884b4a4 [Wes McKinney] Remove outdated pyarrow.jemalloc_memory_pool example. Add --with-plasma to Python doc build
---
 python/doc/source/memory.rst | 12 ------------
 site/README.md               |  2 +-
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/python/doc/source/memory.rst b/python/doc/source/memory.rst
index f18919999e062..4806bbb85722e 100644
--- a/python/doc/source/memory.rst
+++ b/python/doc/source/memory.rst
@@ -142,18 +142,6 @@ the stream is garbaged-collected, all of the memory is freed:
    stream = None
    pa.total_allocated_bytes()
 
-Classes and functions that may allocate memory will often have an option to
-pass in a custom memory pool:
-
-.. ipython:: python
-
-   my_pool = pa.jemalloc_memory_pool()
-   my_pool
-   my_pool.bytes_allocated()
-   stream = pa.BufferOutputStream(my_pool)
-   stream.write(b'foo')
-   my_pool.bytes_allocated()
-
 On-Disk and Memory Mapped Files
 -------------------------------
 
diff --git a/site/README.md b/site/README.md
index 1b0a82e03db43..2c5f979197067 100644
--- a/site/README.md
+++ b/site/README.md
@@ -82,7 +82,7 @@ First, build PyArrow with all optional extensions (Apache Parquet).
 
 ```
 cd ../python
-python setup.py build_ext --inplace --with-parquet
+python setup.py build_ext --inplace --with-parquet --with-plasma
 python setup.py build_sphinx -s doc/source
 rsync -r doc/_build/html/ ../site/asf-site/docs/python/
 ```

From 4ef7c898bb82cd3513e0ad3d80730e29ebaeb60e Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent@dremio.com>
Date: Thu, 17 Aug 2017 16:38:03 -0400
Subject: [PATCH 0939/1644] ARROW-1355: [Java] Make Arrow buildable with jdk9

Make Arrow buildable with jdk9:
- upgrade checkstyle plugin to 6.19
- upgrade assembly plugin to 3.0.0
- update jmockit version to 1.33

Also add travis entry to build using Oracle JDK9 EA

Author: Laurent Goujon <laurent@dremio.com>

Closes #966 from laurentgo/laurent/jdk-9 and squashes the following commits:

d009d012 [Laurent Goujon] Make mvn site optional since not working yet with jdk9
b3e58225 [Laurent Goujon] Update plugin version according to Maven team recommendations
d62d4096 [Laurent Goujon] Fix travis id for jdk9
92fe6d4f [Laurent Goujon] Make Arrow buildable with jdk9
---
 .travis.yml              |  6 ++++++
 ci/travis_script_java.sh |  3 +--
 java/pom.xml             | 14 +++++++-------
 java/tools/pom.xml       |  2 +-
 4 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b93f1c2519b9b..016d1110720f4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -84,6 +84,12 @@ matrix:
     jdk: openjdk8
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+  - language: java
+    os: linux
+    env: ARROW_TRAVIS_SKIP_SITE=yes
+    jdk: oraclejdk9
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
   - language: java
     os: linux
     env: ARROW_TEST_GROUP=integration
diff --git a/ci/travis_script_java.sh b/ci/travis_script_java.sh
index 2f6b685253b10..58e5d42708b30 100755
--- a/ci/travis_script_java.sh
+++ b/ci/travis_script_java.sh
@@ -24,8 +24,7 @@ JAVA_DIR=${TRAVIS_BUILD_DIR}/java
 pushd $JAVA_DIR
 
 export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
-mvn -B test
 mvn -B install
-mvn -B site
+[ "${ARROW_TRAVIS_SKIP_SITE}" = "yes" ] || mvn -B site
 
 popd
diff --git a/java/pom.xml b/java/pom.xml
index f205f6464f7aa..cfcfcd448f0da 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -279,7 +279,7 @@
             <dependency>
               <groupId>com.puppycrawl.tools</groupId>
               <artifactId>checkstyle</artifactId>
-              <version>6.15</version>
+              <version>6.19</version>
             </dependency>
             <dependency>
               <groupId>com.google.guava</groupId>
@@ -333,15 +333,15 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
-          <version>3.2</version>
+          <version>3.6.2</version>
         </plugin>
         <plugin>
           <artifactId>maven-enforcer-plugin</artifactId>
-          <version>1.3.1</version>
+          <version>3.0.0-M1</version>
         </plugin>
         <plugin>
           <artifactId>maven-surefire-plugin</artifactId>
-          <version>2.17</version>
+          <version>2.20</version>
           <configuration>
             <enableAssertions>true</enableAssertions>
             <forkCount>${forkCount}</forkCount>
@@ -493,9 +493,9 @@
 
     <dependency>
       <!-- JMockit needs to be on class path before JUnit. -->
-      <groupId>com.googlecode.jmockit</groupId>
+      <groupId>org.jmockit</groupId>
       <artifactId>jmockit</artifactId>
-      <version>1.7</version>
+      <version>1.33</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -537,7 +537,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-javadoc-plugin</artifactId>
-        <version>2.9</version>
+        <version>3.0.0-M1</version>
         <reportSets>
           <reportSet><!-- by default, id = "default" -->
             <reports><!-- select non-aggregate reports -->
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index f1a7dcd68e4fe..e2e3f8129c707 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -57,7 +57,7 @@
       <plugins>
         <plugin>
           <artifactId>maven-assembly-plugin</artifactId>
-          <version>2.6</version>
+          <version>3.0.0</version>
           <configuration>
             <descriptorRefs>
               <descriptorRef>jar-with-dependencies</descriptorRef>

From c9805d6fd111071b9da6533f8ddd477492ed44f6 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Fri, 18 Aug 2017 20:10:27 -0400
Subject: [PATCH 0940/1644] ARROW-1373: Implement getBuffer() methods for
 ValueVector

cc @jacques-n , @StevenMPhillips

Patch Summary:

As part of ARROW-801, we recently added getValidityBufferAddress(), getOffsetBufferAddress(), getDataBufferAddress() interfaces to get the virtual address of the ArrowBuf.

We now have the following new interfaces to get the corresponding ArrowBuf:

getValidityBuffer()
getDataBuffer()
getOffsetBuffer()

Background:

Currently we have getBuffer() method implemented as part of BaseDataValueVector abstract class. As part of patch for ARROW-276, NullableValueVectors no longer extends BaseDataValueVector -- they don't have to since they don't need the underlying data buffer  (ArrowBuf data field) of BaseDataValueVector.

The call to getBuffer() on NullableValueVectors simply delegates the operation to getBuffer() of underlying data/value vector.

Problem:

If a piece of code is working with ValueVector abstraction and the expected runtime type is Nullable<something>Vector, the compiler obviously complains about doing
(v of type ValueVector).getBuffer().

Until now this worked as we kept the compiler happy by casting the ValueVector to BaseDataValueVector and then do ((BaseDataValueVector)(v of type ValueVector)).getBuffer(). This code broke since NullableValueVectors are no longer a subtype of BaseDataValueVector -- the inheritance hierarchy was changed as part of ARROW-276.

Solution:

Similar to what was done in ARROW-801, we have new methods at ValueVector interface to get the underlying buffer. ValueVector has always had the methods getBuffers(), getBufferSizeFor(), getBufferSize(), so it makes sense to augment the ValueVector interface with new APIs.

It looks like new unit tests are not needed since the unit tests added for ARROW-801 test the new APIs as well --> getDataBufferAddress() underneath invokes getDataBuffer() to get the memory address of ArrowBuf so we are good.

Author: siddharth <siddharth@dremio.com>

Closes #976 from siddharthteotia/ARROW-1373 and squashes the following commits:

1ef2022e [siddharth] Fixed failures and added javadocs
e5ff0233 [siddharth] ARROW-1373: Implement getBuffer() methods for ValueVector
---
 .../codegen/templates/FixedValueVectors.java  | 18 ++++++++++
 .../templates/NullableValueVectors.java       | 33 +++++++++++++++++--
 .../main/codegen/templates/UnionVector.java   | 11 ++++++-
 .../templates/VariableLengthVectors.java      | 20 ++++++++++-
 .../org/apache/arrow/vector/BitVector.java    | 18 ++++++++++
 .../org/apache/arrow/vector/ValueVector.java  | 21 ++++++++++++
 .../org/apache/arrow/vector/ZeroVector.java   | 15 +++++++++
 .../vector/complex/FixedSizeListVector.java   | 17 +++++++++-
 .../arrow/vector/complex/ListVector.java      | 15 +++++++--
 .../arrow/vector/complex/MapVector.java       | 17 ++++++++++
 .../vector/complex/NullableMapVector.java     | 15 +++++++++
 11 files changed, 192 insertions(+), 8 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 61164ab6c9b3d..9747d421c41fc 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -96,6 +96,24 @@ public int getBufferSizeFor(final int valueCount) {
     return valueCount * ${type.width};
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    /* this operation is not supported for non-nullable vectors */
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
+    return getBuffer();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    /* this operation is not supported for fixed-width vectors */
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public int getValueCapacity(){
     return (int) (data.capacity() *1.0 / ${type.width});
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index bd322ea302866..a431333256394 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -219,7 +219,7 @@ public int getBufferSizeFor(final int valueCount) {
   }
 
   public ArrowBuf getBuffer() {
-    return values.getBuffer();
+    return values.getDataBuffer();
   }
 
   @Override
@@ -437,16 +437,21 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
 
   @Override
   public long getValidityBufferAddress() {
-    return (bits.getBuffer().memoryAddress());
+    /* address of the databuffer associated with the bitVector */
+    return (bits.getDataBuffer().memoryAddress());
   }
 
   @Override
   public long getDataBufferAddress() {
-    return (values.getBuffer().memoryAddress());
+    /* address of the dataBuffer associated with the valueVector */
+    return (values.getDataBuffer().memoryAddress());
   }
 
   @Override
   public long getOffsetBufferAddress() {
+    /* address of the dataBuffer associated with the offsetVector
+     * this operation is not supported for fixed-width vector types.
+     */
     <#if type.major != "VarLen">
         throw new UnsupportedOperationException();
     <#else>
@@ -454,6 +459,28 @@ public long getOffsetBufferAddress() {
     </#if>
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    /* dataBuffer associated with the bitVector */
+    return (bits.getDataBuffer());
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    /* dataBuffer associated with the valueVector */
+    return (values.getDataBuffer());
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    /* dataBuffer associated with the offsetVector of the valueVector */
+    <#if type.major != "VarLen">
+        throw new UnsupportedOperationException();
+    <#else>
+        return (values.getOffsetBuffer());
+    </#if>
+  }
+
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
     final BitVector.Accessor bAccessor = bits.getAccessor();
     final ${valuesName}.Accessor vAccessor = values.getAccessor();
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 8c01c4a3c7213..fe24a8674bdc5 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -134,7 +134,7 @@ private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
 
   @Override
   public long getValidityBufferAddress() {
-    return typeVector.getBuffer().memoryAddress();
+    return typeVector.getDataBuffer().memoryAddress();
   }
 
   @Override
@@ -147,6 +147,15 @@ public long getOffsetBufferAddress() {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() { return typeVector.getDataBuffer(); }
+
+  @Override
+  public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); }
+
   public NullableMapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index 70c7209947460..c276f11d79645 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -123,6 +123,24 @@ public int getBufferSizeFor(final int valueCount) {
     return offsetVector.getBufferSizeFor(valueCount + 1) + idx;
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    /* this operation is not supported for non-nullable vectors */
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
+    return getBuffer();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    /* dataBuffer associated with the underlying offsetVector */
+    return offsetVector.getDataBuffer();
+  }
+
   @Override
   public int getValueCapacity(){
     return Math.max(offsetVector.getValueCapacity() - 1, 0);
@@ -170,7 +188,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
   }
 
   public long getOffsetAddr(){
-    return offsetVector.getBuffer().memoryAddress();
+    return offsetVector.getDataBuffer().memoryAddress();
   }
 
   public UInt${type.width}Vector getOffsetVector(){
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index e80ca829c667e..44001d4190098 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -110,6 +110,24 @@ public int getBufferSizeFor(final int valueCount) {
     return getSizeFromCount(valueCount);
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    /* this operation is not supported for non-nullable vectors */
+    throw new  UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
+    return getBuffer();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    /* this operation is not supported for fixed-width vectors */
+    throw new UnsupportedOperationException();
+  }
+
   int getSizeFromCount(int valueCount) {
     return (int) Math.ceil(valueCount / 8.0);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 0c95bcfcbd6b5..fb7286f852c06 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -234,4 +234,25 @@ interface Mutator {
     @Deprecated
     void generateTestData(int values);
   }
+
+  /**
+   * Gets the underlying buffer associated with validity vector
+   *
+   * @return buffer
+   */
+  public ArrowBuf getValidityBuffer();
+
+  /**
+   * Gets the underlying buffer associated with data vector
+   *
+   * @return buffer
+   */
+  public ArrowBuf getDataBuffer();
+
+  /**
+   * Gets the underlying buffer associated with offset vector
+   *
+   * @return buffer
+   */
+  public ArrowBuf getOffsetBuffer();
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index fe224ba8a0589..b267b2087d05c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -244,4 +244,19 @@ public long getDataBufferAddress() {
   public long getOffsetBufferAddress() {
     throw new UnsupportedOperationException();
   }
+
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index aab6e5715b18d..b3be37541c172 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -295,7 +295,7 @@ public UnionVector promoteToUnion() {
 
   @Override
   public long getValidityBufferAddress() {
-    return (bits.getBuffer().memoryAddress());
+    return (bits.getDataBuffer().memoryAddress());
   }
 
   @Override
@@ -308,6 +308,21 @@ public long getOffsetBufferAddress() {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    return (bits.getDataBuffer());
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
   public class Accessor extends BaseValueVector.BaseAccessor {
 
     @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 4b2808dabe7b7..79aca0c7b307c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -180,7 +180,7 @@ public TransferPair makeTransferPair(ValueVector target) {
 
   @Override
   public long getValidityBufferAddress() {
-    return (bits.getBuffer().memoryAddress());
+    return (bits.getDataBuffer().memoryAddress());
   }
 
   @Override
@@ -190,9 +190,20 @@ public long getDataBufferAddress() {
 
   @Override
   public long getOffsetBufferAddress() {
-    return (offsets.getBuffer().memoryAddress());
+    return (offsets.getDataBuffer().memoryAddress());
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() { return bits.getDataBuffer(); }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() { return offsets.getDataBuffer(); }
+
   private class TransferImpl implements TransferPair {
 
     ListVector to;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index d8d0964ab3792..f46635ad76021 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -32,6 +32,8 @@
 import com.google.common.collect.Ordering;
 import com.google.common.primitives.Ints;
 
+import io.netty.buffer.ArrowBuf;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.FieldVector;
@@ -129,6 +131,21 @@ public int getBufferSizeFor(final int valueCount) {
     return (int) bufferSize;
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
     return getTransferPair(name, allocator, null);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 09560c238c1fd..1c9d5aa265963 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -246,6 +246,21 @@ public long getOffsetBufferAddress() {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    return bits.getDataBuffer();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
   public final class Accessor extends MapVector.Accessor {
     final BitVector.Accessor bAccessor = bits.getAccessor();
 

From e1bad9fd21f83b4793ccd243ec5ea96618aa52fa Mon Sep 17 00:00:00 2001
From: HorimotoYasuhiro <spasgate870@movie.ocn.ne.jp>
Date: Sat, 19 Aug 2017 16:45:12 +0900
Subject: [PATCH 0941/1644] [C++] Fix a typo in in plasma.md

Closes #977

Change-Id: I494db4952036a8e52078f1d698d003904f91a34f
---
 cpp/apidoc/tutorials/plasma.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/apidoc/tutorials/plasma.md b/cpp/apidoc/tutorials/plasma.md
index 9fc4175c64be5..aed902413bb36 100644
--- a/cpp/apidoc/tutorials/plasma.md
+++ b/cpp/apidoc/tutorials/plasma.md
@@ -391,7 +391,7 @@ the next newly available object:
 ```cpp
 // Receive notification of the next newly available object.
 // Notification information is stored in object_id, data_size, and metadata_size
-ObjectID new_object_id;
+ObjectID object_id;
 int64_t data_size;
 int64_t metadata_size;
 ARROW_CHECK_OK(client.GetNotification(fd, &object_id, &data_size, &metadata_size));

From 652fd36a7012b5bf484abee9ba1ce22542af6ac0 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 19 Aug 2017 14:13:45 -0400
Subject: [PATCH 0942/1644] ARROW-1366: [Plasma] Define entry point for the
 plasma store

The method for starting the Plasma store is already documented in https://arrow.apache.org/docs/python/plasma.html. So far it only worked if the store was installed with "make install" from the C++ sources. This makes it also possible to start it if the pyarrow wheels are installed.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #975 from pcmoritz/plasma-store-ep and squashes the following commits:

eddc487a [Philipp Moritz] make plasma store entry point private
4c051409 [Philipp Moritz] define entry point for the plasma store
---
 python/pyarrow/__init__.py | 16 ++++++++++++++++
 python/setup.py            |  5 +++++
 2 files changed, 21 insertions(+)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 8d4a214ba263d..d0348b43c503c 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -105,6 +105,22 @@
 
 localfs = LocalFileSystem.get_instance()
 
+# Entry point for starting the plasma store
+
+def _plasma_store_entry_point():
+    """Entry point for starting the plasma store.
+
+    This can be used by invoking e. g. ``plasma_store -s /tmp/plasma -m 1000000000``
+    from the command line and will start the plasma_store executable with the
+    given arguments.
+    """
+    import os
+    import pyarrow
+    import subprocess
+    import sys
+    plasma_store_executable = os.path.join(pyarrow.__path__[0], "plasma_store")
+    process = subprocess.Popen([plasma_store_executable] + sys.argv[1:])
+    process.wait()
 
 # ----------------------------------------------------------------------
 # 0.4.0 deprecations
diff --git a/python/setup.py b/python/setup.py
index ebf28cc64e990..4657da0bf86f2 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -382,6 +382,11 @@ def has_ext_modules(foo):
         'clean': clean,
         'build_ext': build_ext
     },
+    entry_points = {
+        'console_scripts': [
+            'plasma_store = pyarrow:_plasma_store_entry_point'
+        ]
+    },
     use_scm_version={"root": "..", "relative_to": __file__},
     setup_requires=['setuptools_scm', 'cython >= 0.23'],
     install_requires=['numpy >= 1.10', 'six >= 1.0.0'],

From 10f7158df46d838d32ef214b0573b6d719756516 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 19 Aug 2017 18:43:06 -0700
Subject: [PATCH 0943/1644] ARROW-1372: [Plasma] enable HUGETLB support on
 Linux to improve plasma put performance

This PR makes it possible to use Plasma object store backed by a pre-mounted hugetlbfs.

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Alexey Tumanov <atumanov@gmail.com>

Closes #974 from atumanov/putperf and squashes the following commits:

077b78f [Philipp Moritz] add more comments
5aa4b0d [Philipp Moritz] preflight script formatting changes
22188a6 [Philipp Moritz] formatting
ffb9916 [Philipp Moritz] address comments
225429b [Philipp Moritz] update documentation with Alexey's fix
713a0c4 [Philipp Moritz] add missing includes
4c976bb [Philipp Moritz] make format
fb8e1b4 [Philipp Moritz] add helpful error message
7260d59 [Philipp Moritz] expose number of threads to python and try out cleanups
98b603e [Alexey Tumanov] map_populate on linux; fall back to mlock/memset otherwise
ce90ef4 [Alexey Tumanov] documenting new plasma store info fields
c52f211 [Philipp Moritz] cleanups (TODO: See if memory locking helps)
4702703 [Philipp Moritz] preliminary documentation
3073a99 [Alexey Tumanov] reenable hashing
a20ca56 [Alexey Tumanov] fix bug
dd04b87 [Alexey Tumanov] [arrow][putperf] enable HUGETLBFS support on linux
---
 cpp/src/plasma/client.cc             |  3 +-
 cpp/src/plasma/common.cc             |  2 +
 cpp/src/plasma/common.h              |  5 ++
 cpp/src/plasma/malloc.cc             | 82 +++++++++++++++++---------
 cpp/src/plasma/malloc.h              |  2 +
 cpp/src/plasma/plasma.h              |  8 +++
 cpp/src/plasma/store.cc              | 86 ++++++++++++++++++++--------
 cpp/src/plasma/store.h               |  7 ++-
 python/doc/source/plasma.rst         | 47 +++++++++++++++
 python/pyarrow/includes/libarrow.pxd |  4 ++
 python/pyarrow/io.pxi                | 15 +++++
 11 files changed, 206 insertions(+), 55 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 8ea62c6e553c7..5e28d4f2af7be 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -91,7 +91,8 @@ uint8_t* PlasmaClient::lookup_or_mmap(int fd, int store_fd_val, int64_t map_size
     if (result == MAP_FAILED) {
       ARROW_LOG(FATAL) << "mmap failed";
     }
-    close(fd);
+    close(fd);  // Closing this fd has an effect on performance.
+
     ClientMmapTableEntry& entry = mmap_table_[store_fd_val];
     entry.pointer = result;
     entry.length = map_size;
diff --git a/cpp/src/plasma/common.cc b/cpp/src/plasma/common.cc
index d7a7965078533..2de06d5f8cf02 100644
--- a/cpp/src/plasma/common.cc
+++ b/cpp/src/plasma/common.cc
@@ -83,4 +83,6 @@ Status plasma_error_status(int plasma_error) {
 ARROW_EXPORT int ObjectStatusLocal = ObjectStatus_Local;
 ARROW_EXPORT int ObjectStatusRemote = ObjectStatus_Remote;
 
+const PlasmaStoreInfo* plasma_config;
+
 }  // namespace plasma
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
index 2b71da67015cd..66d5f3069d0dd 100644
--- a/cpp/src/plasma/common.h
+++ b/cpp/src/plasma/common.h
@@ -95,6 +95,11 @@ enum ObjectRequestType {
 extern int ObjectStatusLocal;
 extern int ObjectStatusRemote;
 
+/// Globally accessible reference to plasma store configuration.
+/// TODO(pcm): This can be avoided with some refactoring of existing code
+/// by making it possible to pass a context object through dlmalloc.
+struct PlasmaStoreInfo;
+extern const PlasmaStoreInfo* plasma_config;
 }  // namespace plasma
 
 #endif  // PLASMA_COMMON_H
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
index 77a8afea75424..6b9bc62ab5a57 100644
--- a/cpp/src/plasma/malloc.cc
+++ b/cpp/src/plasma/malloc.cc
@@ -25,9 +25,13 @@
 #include <sys/mman.h>
 #include <unistd.h>
 
+#include <cerrno>
+#include <string>
 #include <unordered_map>
+#include <vector>
 
 #include "plasma/common.h"
+#include "plasma/plasma.h"
 
 extern "C" {
 void* fake_mmap(size_t);
@@ -60,12 +64,12 @@ struct mmap_record {
 
 namespace {
 
-/** Hashtable that contains one entry per segment that we got from the OS
- *  via mmap. Associates the address of that segment with its file descriptor
- *  and size. */
+/// Hashtable that contains one entry per segment that we got from the OS
+/// via mmap. Associates the address of that segment with its file descriptor
+/// and size.
 std::unordered_map<void*, mmap_record> mmap_records;
 
-} /* namespace */
+}  // namespace
 
 constexpr int GRANULARITY_MULTIPLIER = 2;
 
@@ -77,10 +81,11 @@ static ptrdiff_t pointer_distance(void const* pfrom, void const* pto) {
   return (unsigned char const*)pto - (unsigned char const*)pfrom;
 }
 
-/* Create a buffer. This is creating a temporary file and then
- * immediately unlinking it so we do not leave traces in the system. */
+// Create a buffer. This is creating a temporary file and then
+// immediately unlinking it so we do not leave traces in the system.
 int create_buffer(int64_t size) {
   int fd;
+  std::string file_template = plasma::plasma_config->directory;
 #ifdef _WIN32
   if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
                          (DWORD)((uint64_t)size >> (CHAR_BIT * sizeof(DWORD))),
@@ -88,53 +93,72 @@ int create_buffer(int64_t size) {
     fd = -1;
   }
 #else
-#ifdef __linux__
-  constexpr char file_template[] = "/dev/shm/plasmaXXXXXX";
-#else
-  constexpr char file_template[] = "/tmp/plasmaXXXXXX";
-#endif
-  char file_name[32];
-  strncpy(file_name, file_template, 32);
-  fd = mkstemp(file_name);
-  if (fd < 0) return -1;
+  file_template += "/plasmaXXXXXX";
+  std::vector<char> file_name(file_template.begin(), file_template.end());
+  file_name.push_back('\0');
+  fd = mkstemp(&file_name[0]);
+  if (fd < 0) {
+    ARROW_LOG(FATAL) << "create_buffer failed to open file " << &file_name[0];
+    return -1;
+  }
+
   FILE* file = fdopen(fd, "a+");
   if (!file) {
     close(fd);
+    ARROW_LOG(FATAL) << "create_buffer: fdopen failed for " << &file_name[0];
     return -1;
   }
-  if (unlink(file_name) != 0) {
-    ARROW_LOG(FATAL) << "unlink error";
+  // Immediately unlink the file so we do not leave traces in the system.
+  if (unlink(&file_name[0]) != 0) {
+    ARROW_LOG(FATAL) << "failed to unlink file " << &file_name[0];
     return -1;
   }
-  if (ftruncate(fd, (off_t)size) != 0) {
-    ARROW_LOG(FATAL) << "ftruncate error";
-    return -1;
+  if (!plasma::plasma_config->hugepages_enabled) {
+    // Increase the size of the file to the desired size. This seems not to be
+    // needed for files that are backed by the huge page fs, see also
+    // http://www.mail-archive.com/kvm-devel@lists.sourceforge.net/msg14737.html
+    if (ftruncate(fd, (off_t)size) != 0) {
+      ARROW_LOG(FATAL) << "failed to ftruncate file " << &file_name[0];
+      return -1;
+    }
   }
 #endif
   return fd;
 }
 
 void* fake_mmap(size_t size) {
-  /* Add sizeof(size_t) so that the returned pointer is deliberately not
-   * page-aligned. This ensures that the segments of memory returned by
-   * fake_mmap are never contiguous. */
+  // Add sizeof(size_t) so that the returned pointer is deliberately not
+  // page-aligned. This ensures that the segments of memory returned by
+  // fake_mmap are never contiguous.
   size += sizeof(size_t);
 
   int fd = create_buffer(size);
   ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
+#ifdef __linux__
+  // MAP_POPULATE will pre-populate the page tables for this memory region
+  // which avoids work when accessing the pages later. Only supported on Linux.
+  void* pointer =
+      mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0);
+#else
   void* pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+#endif
   if (pointer == MAP_FAILED) {
+    ARROW_LOG(ERROR) << "mmap failed with error: " << std::strerror(errno);
+    if (errno == ENOMEM && plasma::plasma_config->hugepages_enabled) {
+      ARROW_LOG(ERROR)
+          << "  (this probably means you have to increase /proc/sys/vm/nr_hugepages)";
+    }
     return pointer;
   }
 
-  /* Increase dlmalloc's allocation granularity directly. */
+  // Increase dlmalloc's allocation granularity directly.
   mparams.granularity *= GRANULARITY_MULTIPLIER;
 
   mmap_record& record = mmap_records[pointer];
   record.fd = fd;
   record.size = size;
 
-  /* We lie to dlmalloc about where mapped memory actually lives. */
+  // We lie to dlmalloc about where mapped memory actually lives.
   pointer = pointer_advance(pointer, sizeof(size_t));
   ARROW_LOG(DEBUG) << pointer << " = fake_mmap(" << size << ")";
   return pointer;
@@ -148,8 +172,8 @@ int fake_munmap(void* addr, int64_t size) {
   auto entry = mmap_records.find(addr);
 
   if (entry == mmap_records.end() || entry->second.size != size) {
-    /* Reject requests to munmap that don't directly match previous
-     * calls to mmap, to prevent dlmalloc from trimming. */
+    // Reject requests to munmap that don't directly match previous
+    // calls to mmap, to prevent dlmalloc from trimming.
     return -1;
   }
 
@@ -163,7 +187,7 @@ int fake_munmap(void* addr, int64_t size) {
 }
 
 void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_size, ptrdiff_t* offset) {
-  /* TODO(rshin): Implement a more efficient search through mmap_records. */
+  // TODO(rshin): Implement a more efficient search through mmap_records.
   for (const auto& entry : mmap_records) {
     if (addr >= entry.first && addr < pointer_advance(entry.first, entry.second.size)) {
       *fd = entry.second.fd;
@@ -176,3 +200,5 @@ void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_size, ptrdiff_t* offse
   *map_size = 0;
   *offset = 0;
 }
+
+void set_malloc_granularity(int value) { change_mparam(M_GRANULARITY, value); }
diff --git a/cpp/src/plasma/malloc.h b/cpp/src/plasma/malloc.h
index b4af2c826b5c9..0df720db59817 100644
--- a/cpp/src/plasma/malloc.h
+++ b/cpp/src/plasma/malloc.h
@@ -23,4 +23,6 @@
 
 void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_length, ptrdiff_t* offset);
 
+void set_malloc_granularity(int value);
+
 #endif  // MALLOC_H
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
index d60e5a8363035..476002f68c05f 100644
--- a/cpp/src/plasma/plasma.h
+++ b/cpp/src/plasma/plasma.h
@@ -27,6 +27,7 @@
 #include <string.h>
 #include <unistd.h>  // pid_t
 
+#include <string>
 #include <unordered_map>
 #include <unordered_set>
 
@@ -129,6 +130,13 @@ struct PlasmaStoreInfo {
   /// The amount of memory (in bytes) that we allow to be allocated in the
   /// store.
   int64_t memory_capacity;
+  /// Boolean flag indicating whether to start the object store with hugepages
+  /// support enabled. Huge pages are substantially larger than normal memory
+  /// pages (e.g. 2MB or 1GB instead of 4KB) and using them can reduce
+  /// bookkeeping overhead from the OS.
+  bool hugepages_enabled;
+  /// A (platform-dependent) directory where to create the memory-backed file.
+  std::string directory;
 };
 
 /// Get an entry from the object table and return NULL if the object_id
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 9f4b98c0ee7d0..aaa2bad67c38b 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -96,9 +96,12 @@ GetRequest::GetRequest(Client* client, const std::vector<ObjectID>& object_ids)
 
 Client::Client(int fd) : fd(fd) {}
 
-PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory)
+PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory, std::string directory,
+                         bool hugepages_enabled)
     : loop_(loop), eviction_policy_(&store_info_) {
   store_info_.memory_capacity = system_memory;
+  store_info_.directory = directory;
+  store_info_.hugepages_enabled = hugepages_enabled;
 }
 
 // TODO(pcm): Get rid of this destructor by using RAII to clean up data.
@@ -114,6 +117,8 @@ PlasmaStore::~PlasmaStore() {
   }
 }
 
+const PlasmaStoreInfo* PlasmaStore::get_plasma_store_info() { return &store_info_; }
+
 // If this client is not already using the object, add the client to the
 // object's list of clients, otherwise do nothing.
 void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client* client) {
@@ -633,10 +638,13 @@ class PlasmaStoreRunner {
  public:
   PlasmaStoreRunner() {}
 
-  void Start(char* socket_name, int64_t system_memory) {
+  void Start(char* socket_name, int64_t system_memory, std::string directory,
+             bool hugepages_enabled) {
     // Create the event loop.
     loop_.reset(new EventLoop);
-    store_.reset(new PlasmaStore(loop_.get(), system_memory));
+    store_.reset(
+        new PlasmaStore(loop_.get(), system_memory, directory, hugepages_enabled));
+    plasma_config = store_->get_plasma_store_info();
     int socket = bind_ipc_sock(socket_name, true);
     // TODO(pcm): Check return value.
     ARROW_CHECK(socket >= 0);
@@ -670,7 +678,8 @@ void HandleSignal(int signal) {
   }
 }
 
-void start_server(char* socket_name, int64_t system_memory) {
+void start_server(char* socket_name, int64_t system_memory, std::string plasma_directory,
+                  bool hugepages_enabled) {
   // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
   // to a client that has already died, the store could die.
   signal(SIGPIPE, SIG_IGN);
@@ -678,17 +687,26 @@ void start_server(char* socket_name, int64_t system_memory) {
   PlasmaStoreRunner runner;
   g_runner = &runner;
   signal(SIGTERM, HandleSignal);
-  runner.Start(socket_name, system_memory);
+  runner.Start(socket_name, system_memory, plasma_directory, hugepages_enabled);
 }
 
 }  // namespace plasma
 
 int main(int argc, char* argv[]) {
   char* socket_name = NULL;
+  // Directory where plasma memory mapped files are stored.
+  std::string plasma_directory;
+  bool hugepages_enabled = false;
   int64_t system_memory = -1;
   int c;
-  while ((c = getopt(argc, argv, "s:m:")) != -1) {
+  while ((c = getopt(argc, argv, "s:m:d:h")) != -1) {
     switch (c) {
+      case 'd':
+        plasma_directory = std::string(optarg);
+        break;
+      case 'h':
+        hugepages_enabled = true;
+        break;
       case 's':
         socket_name = optarg;
         break;
@@ -705,36 +723,54 @@ int main(int argc, char* argv[]) {
         exit(-1);
     }
   }
+  // Sanity check command line options.
   if (!socket_name) {
     ARROW_LOG(FATAL) << "please specify socket for incoming connections with -s switch";
   }
   if (system_memory == -1) {
     ARROW_LOG(FATAL) << "please specify the amount of system memory with -m switch";
   }
+  if (hugepages_enabled && plasma_directory.empty()) {
+    ARROW_LOG(FATAL) << "if you want to use hugepages, please specify path to huge pages "
+                        "filesystem with -d";
+  }
+  if (plasma_directory.empty()) {
 #ifdef __linux__
-  // On Linux, check that the amount of memory available in /dev/shm is large
-  // enough to accommodate the request. If it isn't, then fail.
-  int shm_fd = open("/dev/shm", O_RDONLY);
-  struct statvfs shm_vfs_stats;
-  fstatvfs(shm_fd, &shm_vfs_stats);
-  // The value shm_vfs_stats.f_bsize is the block size, and the value
-  // shm_vfs_stats.f_bavail is the number of available blocks.
-  int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail;
-  close(shm_fd);
-  if (system_memory > shm_mem_avail) {
-    ARROW_LOG(FATAL) << "System memory request exceeds memory available in /dev/shm. The "
-                        "request is for "
-                     << system_memory << " bytes, and the amount available is "
-                     << shm_mem_avail
-                     << " bytes. You may be able to free up space by deleting files in "
-                        "/dev/shm. If you are inside a Docker container, you may need to "
-                        "pass "
-                        "an argument with the flag '--shm-size' to 'docker run'.";
+    plasma_directory = "/dev/shm";
+#else
+    plasma_directory = "/tmp";
+#endif
+  }
+  ARROW_LOG(INFO) << "Starting object store with directory " << plasma_directory
+                  << " and huge page support "
+                  << (hugepages_enabled ? "enabled" : "disabled");
+#ifdef __linux__
+  if (!hugepages_enabled) {
+    // On Linux, check that the amount of memory available in /dev/shm is large
+    // enough to accommodate the request. If it isn't, then fail.
+    int shm_fd = open(plasma_directory.c_str(), O_RDONLY);
+    struct statvfs shm_vfs_stats;
+    fstatvfs(shm_fd, &shm_vfs_stats);
+    // The value shm_vfs_stats.f_bsize is the block size, and the value
+    // shm_vfs_stats.f_bavail is the number of available blocks.
+    int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail;
+    close(shm_fd);
+    if (system_memory > shm_mem_avail) {
+      ARROW_LOG(FATAL)
+          << "System memory request exceeds memory available in " << plasma_directory
+          << ". The request is for " << system_memory
+          << " bytes, and the amount available is " << shm_mem_avail
+          << " bytes. You may be able to free up space by deleting files in "
+             "/dev/shm. If you are inside a Docker container, you may need to "
+             "pass an argument with the flag '--shm-size' to 'docker run'.";
+    }
+  } else {
+    set_malloc_granularity(1024 * 1024 * 1024);  // 1 GB
   }
 #endif
   // Make it so dlmalloc fails if we try to request more memory than is
   // available.
   plasma::dlmalloc_set_footprint_limit((size_t)system_memory);
   ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
-  plasma::start_server(socket_name, system_memory);
+  plasma::start_server(socket_name, system_memory, plasma_directory, hugepages_enabled);
 }
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index fb732a1375da6..61a3a245610c3 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -19,6 +19,7 @@
 #define PLASMA_STORE_H
 
 #include <deque>
+#include <string>
 #include <vector>
 
 #include "plasma/common.h"
@@ -47,10 +48,14 @@ struct Client {
 
 class PlasmaStore {
  public:
-  PlasmaStore(EventLoop* loop, int64_t system_memory);
+  PlasmaStore(EventLoop* loop, int64_t system_memory, std::string directory,
+              bool hugetlbfs_enabled);
 
   ~PlasmaStore();
 
+  /// Get a const pointer to the internal PlasmaStoreInfo object.
+  const PlasmaStoreInfo* get_plasma_store_info();
+
   /// Create a new object. The client must do a call to release_object to tell
   /// the store when it is done with the object.
   ///
diff --git a/python/doc/source/plasma.rst b/python/doc/source/plasma.rst
index 832d9960cb539..e4665d187e1f5 100644
--- a/python/doc/source/plasma.rst
+++ b/python/doc/source/plasma.rst
@@ -335,3 +335,50 @@ the original Pandas ``DataFrame`` structure.
 
   # Convert back into Pandas
   result = record_batch.to_pandas()
+
+Using Plasma with Huge Pages
+----------------------------
+
+On Linux it is possible to use the Plasma store with huge pages for increased
+throughput. You first need to create a file system and activate huge pages with
+
+.. code-block:: shell
+
+  sudo mkdir -p /mnt/hugepages
+  gid=`id -g`
+  uid=`id -u`
+  sudo mount -t hugetlbfs -o uid=$uid -o gid=$gid none /mnt/hugepages
+  sudo bash -c "echo $gid > /proc/sys/vm/hugetlb_shm_group"
+  sudo bash -c "echo 20000 > /proc/sys/vm/nr_hugepages"
+
+Note that you only need root access to create the file system, not for
+running the object store. You can then start the Plasma store with the ``-d``
+flag for the mount point of the huge page file system and the ``-h`` flag
+which indicates that huge pages are activated:
+
+.. code-block:: shell
+
+  plasma_store -s /tmp/plasma -m 10000000000 -d /mnt/hugepages -h
+
+You can test this with the following script:
+
+.. code-block:: python
+
+  import numpy as np
+  import pyarrow as pa
+  import pyarrow.plasma as plasma
+  import time
+
+  client = plasma.connect("/tmp/plasma", "", 0)
+
+  data = np.random.randn(100000000)
+  tensor = pa.Tensor.from_numpy(data)
+
+  object_id = plasma.ObjectID(np.random.bytes(20))
+  buf = client.create(object_id, pa.get_tensor_size(tensor))
+
+  stream = pa.FixedSizeBufferOutputStream(buf)
+  stream.set_memcopy_threads(4)
+  a = time.time()
+  pa.write_tensor(tensor, stream)
+  print("Writing took ", time.time() - a)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index eed9640861fac..c6a9d9d0b7b18 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -581,6 +581,10 @@ cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
             " arrow::io::FixedSizeBufferWriter"(WriteableFile):
         CFixedSizeBufferWriter(const shared_ptr[CBuffer]& buffer)
 
+        void set_memcopy_threads(int num_threads)
+        void set_memcopy_blocksize(int64_t blocksize)
+        void set_memcopy_threshold(int64_t threshold)
+
 
 cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
     enum MessageType" arrow::ipc::Message::Type":
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index eda8de730281d..061a7a9a4d6e6 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -542,6 +542,21 @@ cdef class FixedSizeBufferOutputStream(NativeFile):
         self.is_writeable = 1
         self.is_open = True
 
+    def set_memcopy_threads(self, int num_threads):
+        cdef CFixedSizeBufferWriter* writer = \
+            <CFixedSizeBufferWriter*> self.wr_file.get()
+        writer.set_memcopy_threads(num_threads)
+
+    def set_memcopy_blocksize(self, int64_t blocksize):
+        cdef CFixedSizeBufferWriter* writer = \
+            <CFixedSizeBufferWriter*> self.wr_file.get()
+        writer.set_memcopy_blocksize(blocksize)
+
+    def set_memcopy_threshold(self, int64_t threshold):
+        cdef CFixedSizeBufferWriter* writer = \
+            <CFixedSizeBufferWriter*> self.wr_file.get()
+        writer.set_memcopy_threshold(threshold)
+
 
 # ----------------------------------------------------------------------
 # Arrow buffers

From b50f2351e4972adad4d1bac4765cc5fedfa5836c Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sun, 20 Aug 2017 00:15:09 -0400
Subject: [PATCH 0944/1644] ARROW-759: [Python] Serializing large class of
 Python objects in Apache Arrow

This PR adds the capability to serialize a large class of (nested) Python objects in Apache Arrow. The eventual goal is to evolve this into a more modern version of pickle that will make it possible to read the data from other languages supported by Apache Arrow (and might also be faster).

Currently we support lists, tuples, dicts, strings, numpy objects, Python classes and namedtuples. A fallback to (cloud-)pickle can be provided for objects that cannot be natively represented in Arrow (for example lambdas).

Numpy data within objects is efficiently represented using Arrow's Tensor facilities and for the nested Python sequences we use Arrow's UnionArray.

There are many loose ends that will need to be addressed in follow up PRs.

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #965 from pcmoritz/python-serialization and squashes the following commits:

31486edd [Wes McKinney] Fix typo
2164db72 [Wes McKinney] Add SerializedPyObject to public API
b70235cd [Wes McKinney] Add pyarrow.deserialize convenience method
a6a402ee [Wes McKinney] Memory map fixture robustness on Windows
114a5fbf [Wes McKinney] Add a Python container for the SerializedPyObject data, total_bytes method
8e596172 [Wes McKinney] Use pytest tmpdir for large memory map fixture so works on Windows
8a42f30f [Wes McKinney] Add doxygen comment to set_serialization_callbacks
a9522c51 [Wes McKinney] Refactoring, address code review comments. fix flake8 issues
ce5784d5 [Wes McKinney] Do not use ARROW_CHECK in production code. Consolidate python_to_arrow code
c8efef94 [Wes McKinney] Fix various Clang compiler warnings due to integer conversions. clang-format
831e2f21 [Philipp Moritz] remove sequence.h
54af39bc [Philipp Moritz] more fixes
a6fdb76a [Philipp Moritz] make tests work
fe56c735 [Philipp Moritz] fixes
84d62f64 [Philipp Moritz] more fixes
49aba8a1 [Philipp Moritz] make it compile on windows
aa1f3009 [Philipp Moritz] linting
95cb9da6 [Philipp Moritz] fix GIL
adcc8f7a [Philipp Moritz] shuffle stuff around
bcebdfef [Philipp Moritz] fix longlong vs int64 and unsigned variant
4cc45cd7 [Philipp Moritz] cleanup
f25f3f3b [Philipp Moritz] cleanups
a88d4107 [Philipp Moritz] convert DESERIALIZE_SEQUENCE back to a macro
c4259785 [Philipp Moritz] prevent possible memory leaks
aeafd827 [Philipp Moritz] fix callbacks
389bfc6e [Philipp Moritz] documentation
2f0760c2 [Philipp Moritz] fix api
faf9a3e6 [Philipp Moritz] make exported API more consistent
e1fc0c59 [Philipp Moritz] restructure
c1f377b7 [Philipp Moritz] more fixes
3e94e6da [Philipp Moritz] clang-format
99e2d1af [Philipp Moritz] cleanups
32983297 [Philipp Moritz] mutable refs and small fixes
e73c1ea8 [Philipp Moritz] make DictBuilder private
39292735 [Philipp Moritz] increase Py_True refcount and hide helper methods
aaf6f095 [Philipp Moritz] remove code duplication
c38c58db [Philipp Moritz] get rid of leaks and clarify reference counting for dicts
74b9e469 [Philipp Moritz] convert DESERIALIZE_SEQUENCE to a template
080db030 [Philipp Moritz] fix first few comments
a6105d2e [Philipp Moritz] lint fix
802e739c [Philipp Moritz] clang-format
2e08de4c [Philipp Moritz] fix namespaces
91b57d57 [Philipp Moritz] fix linting
c4782ac0 [Philipp Moritz] fix
7069e208 [Philipp Moritz] fix imports
2171761b [Philipp Moritz] fix python unicode string
30bb960f [Philipp Moritz] rebase
f229d8d2 [Philipp Moritz] serialization of custom objects
8b2ffe60 [Philipp Moritz] working version
bd36c83e [Philipp Moritz] handle very long longs with custom serialization callback
49a4acb2 [Philipp Moritz] roundtrip working for the first time
44fb98bf [Philipp Moritz] work in progress
3af1c67c [Philipp Moritz] deserialization path (need to figure out if base object and refcounting is handled correctly)
deb3b461 [Philipp Moritz] rename serialization entry point
5766b8ca [Philipp Moritz] python to arrow serialization
---
 cpp/src/arrow/builder.h                    |  12 +-
 cpp/src/arrow/python/CMakeLists.txt        |   4 +
 cpp/src/arrow/python/api.h                 |   2 +
 cpp/src/arrow/python/arrow_to_python.cc    | 221 +++++++
 cpp/src/arrow/python/arrow_to_python.h     |  66 +++
 cpp/src/arrow/python/common.h              |  30 +
 cpp/src/arrow/python/python_to_arrow.cc    | 654 +++++++++++++++++++++
 cpp/src/arrow/python/python_to_arrow.h     |  78 +++
 python/doc/source/api.rst                  |  10 +-
 python/pyarrow/__init__.py                 |   4 +
 python/pyarrow/compat.py                   |   7 +
 python/pyarrow/includes/libarrow.pxd       |  20 +
 python/pyarrow/io.pxi                      |   2 +-
 python/pyarrow/lib.pyx                     |   3 +
 python/pyarrow/serialization.pxi           | 279 +++++++++
 python/pyarrow/tests/test_serialization.py | 236 ++++++++
 16 files changed, 1619 insertions(+), 9 deletions(-)
 create mode 100644 cpp/src/arrow/python/arrow_to_python.cc
 create mode 100644 cpp/src/arrow/python/arrow_to_python.h
 create mode 100644 cpp/src/arrow/python/python_to_arrow.cc
 create mode 100644 cpp/src/arrow/python/python_to_arrow.h
 create mode 100644 python/pyarrow/serialization.pxi
 create mode 100644 python/pyarrow/tests/test_serialization.py

diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 46900fc7129c1..3b851f92c1726 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -244,7 +244,7 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
   using PrimitiveBuilder<T>::Reserve;
 
   /// Append a single scalar and increase the size if necessary.
-  Status Append(value_type val) {
+  Status Append(const value_type val) {
     RETURN_NOT_OK(ArrayBuilder::Reserve(1));
     UnsafeAppend(val);
     return Status::OK();
@@ -255,7 +255,7 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
   ///
   /// This method does not capacity-check; make sure to call Reserve
   /// beforehand.
-  void UnsafeAppend(value_type val) {
+  void UnsafeAppend(const value_type val) {
     BitUtil::SetBit(null_bitmap_data_, length_);
     raw_data_[length_++] = val;
   }
@@ -371,7 +371,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
   using ArrayBuilder::Advance;
 
   /// Scalar append
-  Status Append(uint64_t val) {
+  Status Append(const uint64_t val) {
     RETURN_NOT_OK(Reserve(1));
     BitUtil::SetBit(null_bitmap_data_, length_);
 
@@ -430,7 +430,7 @@ class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase
   using ArrayBuilder::Advance;
 
   /// Scalar append
-  Status Append(int64_t val) {
+  Status Append(const int64_t val) {
     RETURN_NOT_OK(Reserve(1));
     BitUtil::SetBit(null_bitmap_data_, length_);
 
@@ -511,7 +511,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   std::shared_ptr<Buffer> data() const { return data_; }
 
   /// Scalar append
-  Status Append(bool val) {
+  Status Append(const bool val) {
     RETURN_NOT_OK(Reserve(1));
     BitUtil::SetBit(null_bitmap_data_, length_);
     if (val) {
@@ -523,7 +523,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  Status Append(uint8_t val) { return Append(val != 0); }
+  Status Append(const uint8_t val) { return Append(val != 0); }
 
   /// Vector append
   ///
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 0fdf81e7aa9d7..f2807b930a33c 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -43,6 +43,7 @@ set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
 
 set(ARROW_PYTHON_SRCS
   arrow_to_pandas.cc
+  arrow_to_python.cc
   builtin_convert.cc
   common.cc
   config.cc
@@ -51,6 +52,7 @@ set(ARROW_PYTHON_SRCS
   io.cc
   numpy_convert.cc
   pandas_to_arrow.cc
+  python_to_arrow.cc
   pyarrow.cc
 )
 
@@ -83,6 +85,7 @@ endif()
 install(FILES
   api.h
   arrow_to_pandas.h
+  arrow_to_python.h
   builtin_convert.h
   common.h
   config.h
@@ -92,6 +95,7 @@ install(FILES
   numpy_convert.h
   numpy_interop.h
   pandas_to_arrow.h
+  python_to_arrow.h
   platform.h
   pyarrow.h
   type_traits.h
diff --git a/cpp/src/arrow/python/api.h b/cpp/src/arrow/python/api.h
index 7cb36ad636fc9..4ceb3f1a45dc1 100644
--- a/cpp/src/arrow/python/api.h
+++ b/cpp/src/arrow/python/api.h
@@ -19,11 +19,13 @@
 #define ARROW_PYTHON_API_H
 
 #include "arrow/python/arrow_to_pandas.h"
+#include "arrow/python/arrow_to_python.h"
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/io.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/pandas_to_arrow.h"
+#include "arrow/python/python_to_arrow.h"
 
 #endif  // ARROW_PYTHON_API_H
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
new file mode 100644
index 0000000000000..622ef8299374a
--- /dev/null
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -0,0 +1,221 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/arrow_to_python.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/ipc/reader.h"
+#include "arrow/python/common.h"
+#include "arrow/python/helpers.h"
+#include "arrow/python/numpy_convert.h"
+#include "arrow/table.h"
+#include "arrow/util/logging.h"
+
+extern "C" {
+extern PyObject* pyarrow_serialize_callback;
+extern PyObject* pyarrow_deserialize_callback;
+}
+
+namespace arrow {
+namespace py {
+
+Status CallCustomCallback(PyObject* callback, PyObject* elem, PyObject** result);
+
+Status DeserializeTuple(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
+                        PyObject* base,
+                        const std::vector<std::shared_ptr<Tensor>>& tensors,
+                        PyObject** out);
+
+Status DeserializeList(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
+                       PyObject* base,
+                       const std::vector<std::shared_ptr<Tensor>>& tensors,
+                       PyObject** out);
+
+Status DeserializeDict(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
+                       PyObject* base,
+                       const std::vector<std::shared_ptr<Tensor>>& tensors,
+                       PyObject** out) {
+  auto data = std::dynamic_pointer_cast<StructArray>(array);
+  ScopedRef keys, vals;
+  ScopedRef result(PyDict_New());
+  RETURN_NOT_OK(
+      DeserializeList(data->field(0), start_idx, stop_idx, base, tensors, keys.ref()));
+  RETURN_NOT_OK(
+      DeserializeList(data->field(1), start_idx, stop_idx, base, tensors, vals.ref()));
+  for (int64_t i = start_idx; i < stop_idx; ++i) {
+    // PyDict_SetItem behaves differently from PyList_SetItem and PyTuple_SetItem.
+    // The latter two steal references whereas PyDict_SetItem does not. So we need
+    // to make sure the reference count is decremented by letting the ScopedRef
+    // go out of scope at the end.
+    PyDict_SetItem(result.get(), PyList_GET_ITEM(keys.get(), i - start_idx),
+                   PyList_GET_ITEM(vals.get(), i - start_idx));
+  }
+  static PyObject* py_type = PyUnicode_FromString("_pytype_");
+  if (PyDict_Contains(result.get(), py_type)) {
+    RETURN_NOT_OK(CallCustomCallback(pyarrow_deserialize_callback, result.get(), out));
+  } else {
+    *out = result.release();
+  }
+  return Status::OK();
+}
+
+Status DeserializeArray(std::shared_ptr<Array> array, int64_t offset, PyObject* base,
+                        const std::vector<std::shared_ptr<arrow::Tensor>>& tensors,
+                        PyObject** out) {
+  DCHECK(array);
+  int32_t index = std::static_pointer_cast<Int32Array>(array)->Value(offset);
+  RETURN_NOT_OK(py::TensorToNdarray(*tensors[index], base, out));
+  // Mark the array as immutable
+  ScopedRef flags(PyObject_GetAttrString(*out, "flags"));
+  DCHECK(flags.get() != NULL) << "Could not mark Numpy array immutable";
+  Py_INCREF(Py_False);
+  int flag_set = PyObject_SetAttrString(flags.get(), "writeable", Py_False);
+  DCHECK(flag_set == 0) << "Could not mark Numpy array immutable";
+  return Status::OK();
+}
+
+Status GetValue(std::shared_ptr<Array> arr, int64_t index, int32_t type, PyObject* base,
+                const std::vector<std::shared_ptr<Tensor>>& tensors, PyObject** result) {
+  switch (arr->type()->id()) {
+    case Type::BOOL:
+      *result =
+          PyBool_FromLong(std::static_pointer_cast<BooleanArray>(arr)->Value(index));
+      return Status::OK();
+    case Type::INT64:
+      *result =
+          PyLong_FromSsize_t(std::static_pointer_cast<Int64Array>(arr)->Value(index));
+      return Status::OK();
+    case Type::BINARY: {
+      int32_t nchars;
+      const uint8_t* str =
+          std::static_pointer_cast<BinaryArray>(arr)->GetValue(index, &nchars);
+      *result = PyBytes_FromStringAndSize(reinterpret_cast<const char*>(str), nchars);
+      return CheckPyError();
+    }
+    case Type::STRING: {
+      int32_t nchars;
+      const uint8_t* str =
+          std::static_pointer_cast<StringArray>(arr)->GetValue(index, &nchars);
+      *result = PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(str), nchars);
+      return CheckPyError();
+    }
+    case Type::FLOAT:
+      *result =
+          PyFloat_FromDouble(std::static_pointer_cast<FloatArray>(arr)->Value(index));
+      return Status::OK();
+    case Type::DOUBLE:
+      *result =
+          PyFloat_FromDouble(std::static_pointer_cast<DoubleArray>(arr)->Value(index));
+      return Status::OK();
+    case Type::STRUCT: {
+      auto s = std::static_pointer_cast<StructArray>(arr);
+      auto l = std::static_pointer_cast<ListArray>(s->field(0));
+      if (s->type()->child(0)->name() == "list") {
+        return DeserializeList(l->values(), l->value_offset(index),
+                               l->value_offset(index + 1), base, tensors, result);
+      } else if (s->type()->child(0)->name() == "tuple") {
+        return DeserializeTuple(l->values(), l->value_offset(index),
+                                l->value_offset(index + 1), base, tensors, result);
+      } else if (s->type()->child(0)->name() == "dict") {
+        return DeserializeDict(l->values(), l->value_offset(index),
+                               l->value_offset(index + 1), base, tensors, result);
+      } else {
+        DCHECK(false) << "unexpected StructArray type " << s->type()->child(0)->name();
+      }
+    }
+    // We use an Int32Builder here to distinguish the tensor indices from
+    // the Type::INT64 above (see tensor_indices_ in SequenceBuilder).
+    case Type::INT32: {
+      return DeserializeArray(arr, index, base, tensors, result);
+    }
+    default:
+      DCHECK(false) << "union tag " << type << " not recognized";
+  }
+  return Status::OK();
+}
+
+#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                        \
+  auto data = std::dynamic_pointer_cast<UnionArray>(array);                 \
+  int64_t size = array->length();                                           \
+  ScopedRef result(CREATE_FN(stop_idx - start_idx));                        \
+  auto types = std::make_shared<Int8Array>(size, data->type_ids());         \
+  auto offsets = std::make_shared<Int32Array>(size, data->value_offsets()); \
+  for (int64_t i = start_idx; i < stop_idx; ++i) {                          \
+    if (data->IsNull(i)) {                                                  \
+      Py_INCREF(Py_None);                                                   \
+      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                    \
+    } else {                                                                \
+      int64_t offset = offsets->Value(i);                                   \
+      int8_t type = types->Value(i);                                        \
+      std::shared_ptr<Array> arr = data->child(type);                       \
+      PyObject* value;                                                      \
+      RETURN_NOT_OK(GetValue(arr, offset, type, base, tensors, &value));    \
+      SET_ITEM_FN(result.get(), i - start_idx, value);                      \
+    }                                                                       \
+  }                                                                         \
+  *out = result.release();                                                  \
+  return Status::OK();
+
+Status DeserializeList(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
+                       PyObject* base,
+                       const std::vector<std::shared_ptr<Tensor>>& tensors,
+                       PyObject** out) {
+  DESERIALIZE_SEQUENCE(PyList_New, PyList_SET_ITEM)
+}
+
+Status DeserializeTuple(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
+                        PyObject* base,
+                        const std::vector<std::shared_ptr<Tensor>>& tensors,
+                        PyObject** out) {
+  DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SET_ITEM)
+}
+
+Status ReadSerializedObject(std::shared_ptr<io::RandomAccessFile> src,
+                            SerializedPyObject* out) {
+  std::shared_ptr<ipc::RecordBatchStreamReader> reader;
+  int64_t offset;
+  int64_t bytes_read;
+  int32_t num_tensors;
+  // Read number of tensors
+  RETURN_NOT_OK(
+      src->Read(sizeof(int32_t), &bytes_read, reinterpret_cast<uint8_t*>(&num_tensors)));
+  RETURN_NOT_OK(ipc::RecordBatchStreamReader::Open(src, &reader));
+  RETURN_NOT_OK(reader->ReadNextRecordBatch(&out->batch));
+  RETURN_NOT_OK(src->Tell(&offset));
+  offset += 4;  // Skip the end-of-stream message
+  for (int i = 0; i < num_tensors; ++i) {
+    std::shared_ptr<Tensor> tensor;
+    RETURN_NOT_OK(ipc::ReadTensor(offset, src.get(), &tensor));
+    out->tensors.push_back(tensor);
+    RETURN_NOT_OK(src->Tell(&offset));
+  }
+  return Status::OK();
+}
+
+Status DeserializeObject(const SerializedPyObject& obj, PyObject* base, PyObject** out) {
+  PyAcquireGIL lock;
+  return DeserializeList(obj.batch->column(0), 0, obj.batch->num_rows(), base,
+                         obj.tensors, out);
+}
+
+}  // namespace py
+}  // namespace arrow
diff --git a/cpp/src/arrow/python/arrow_to_python.h b/cpp/src/arrow/python/arrow_to_python.h
new file mode 100644
index 0000000000000..559ce18c50709
--- /dev/null
+++ b/cpp/src/arrow/python/arrow_to_python.h
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PYTHON_ARROW_TO_PYTHON_H
+#define ARROW_PYTHON_ARROW_TO_PYTHON_H
+
+#include "arrow/python/platform.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "arrow/python/python_to_arrow.h"
+#include "arrow/status.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class RecordBatch;
+class Tensor;
+
+namespace io {
+
+class RandomAccessFile;
+
+}  // namespace io
+
+namespace py {
+
+/// \brief Read serialized Python sequence from file interface using Arrow IPC
+/// \param[in] src a RandomAccessFile
+/// \param[out] out the reconstructed data
+/// \return Status
+ARROW_EXPORT
+Status ReadSerializedObject(std::shared_ptr<io::RandomAccessFile> src,
+                            SerializedPyObject* out);
+
+/// \brief Reconstruct Python object from Arrow-serialized representation
+/// \param[in] object
+/// \param[in] base a Python object holding the underlying data that any NumPy
+/// arrays will reference, to avoid premature deallocation
+/// \param[out] out the returned object
+/// \return Status
+/// This acquires the GIL
+ARROW_EXPORT
+Status DeserializeObject(const SerializedPyObject& object, PyObject* base,
+                         PyObject** out);
+
+}  // namespace py
+}  // namespace arrow
+
+#endif  // ARROW_PYTHON_ARROW_TO_PYTHON_H
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index ec40d0eafa3b5..7f94f9554f1e8 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -91,6 +91,36 @@ class ARROW_EXPORT OwnedRef {
   PyObject* obj_;
 };
 
+// This is different from OwnedRef in that it assumes that
+// the GIL is held by the caller and doesn't decrement the
+// reference count when release is called.
+class ARROW_EXPORT ScopedRef {
+ public:
+  ScopedRef() : obj_(nullptr) {}
+
+  explicit ScopedRef(PyObject* obj) : obj_(obj) {}
+
+  ~ScopedRef() { Py_XDECREF(obj_); }
+
+  void reset(PyObject* obj) {
+    Py_XDECREF(obj_);
+    obj_ = obj;
+  }
+
+  PyObject* release() {
+    PyObject* result = obj_;
+    obj_ = nullptr;
+    return result;
+  }
+
+  PyObject* get() const { return obj_; }
+
+  PyObject** ref() { return &obj_; }
+
+ private:
+  PyObject* obj_;
+};
+
 struct ARROW_EXPORT PyObjectStringify {
   OwnedRef tmp_obj;
   const char* bytes;
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
new file mode 100644
index 0000000000000..47d8ef60c4b48
--- /dev/null
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -0,0 +1,654 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/python_to_arrow.h"
+#include "arrow/python/numpy_interop.h"
+
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <numpy/arrayobject.h>
+#include <numpy/arrayscalars.h>
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/ipc/writer.h"
+#include "arrow/python/common.h"
+#include "arrow/python/helpers.h"
+#include "arrow/python/numpy_convert.h"
+#include "arrow/python/platform.h"
+#include "arrow/tensor.h"
+#include "arrow/util/logging.h"
+
+constexpr int32_t kMaxRecursionDepth = 100;
+
+extern "C" {
+PyObject* pyarrow_serialize_callback = NULL;
+PyObject* pyarrow_deserialize_callback = NULL;
+}
+
+namespace arrow {
+namespace py {
+
+/// A Sequence is a heterogeneous collections of elements. It can contain
+/// scalar Python types, lists, tuples, dictionaries and tensors.
+class SequenceBuilder {
+ public:
+  explicit SequenceBuilder(MemoryPool* pool = nullptr)
+      : pool_(pool),
+        types_(pool, ::arrow::int8()),
+        offsets_(pool, ::arrow::int32()),
+        nones_(pool),
+        bools_(pool, ::arrow::boolean()),
+        ints_(pool, ::arrow::int64()),
+        bytes_(pool, ::arrow::binary()),
+        strings_(pool),
+        floats_(pool, ::arrow::float32()),
+        doubles_(pool, ::arrow::float64()),
+        tensor_indices_(pool, ::arrow::int32()),
+        list_offsets_({0}),
+        tuple_offsets_({0}),
+        dict_offsets_({0}) {}
+
+  /// Appending a none to the sequence
+  Status AppendNone() {
+    RETURN_NOT_OK(offsets_.Append(0));
+    RETURN_NOT_OK(types_.Append(0));
+    return nones_.AppendToBitmap(false);
+  }
+
+  Status Update(int64_t offset, int8_t* tag) {
+    if (*tag == -1) {
+      *tag = num_tags_++;
+    }
+    RETURN_NOT_OK(offsets_.Append(static_cast<int32_t>(offset)));
+    RETURN_NOT_OK(types_.Append(*tag));
+    return nones_.AppendToBitmap(true);
+  }
+
+  template <typename BuilderType, typename T>
+  Status AppendPrimitive(const T val, int8_t* tag, BuilderType* out) {
+    RETURN_NOT_OK(Update(out->length(), tag));
+    return out->Append(val);
+  }
+
+  /// Appending a boolean to the sequence
+  Status AppendBool(const bool data) {
+    return AppendPrimitive(data, &bool_tag_, &bools_);
+  }
+
+  /// Appending an int64_t to the sequence
+  Status AppendInt64(const int64_t data) {
+    return AppendPrimitive(data, &int_tag_, &ints_);
+  }
+
+  /// Appending an uint64_t to the sequence
+  Status AppendUInt64(const uint64_t data) {
+    // TODO(wesm): Bounds check
+    return AppendPrimitive(static_cast<int64_t>(data), &int_tag_, &ints_);
+  }
+
+  /// Append a list of bytes to the sequence
+  Status AppendBytes(const uint8_t* data, int32_t length) {
+    RETURN_NOT_OK(Update(bytes_.length(), &bytes_tag_));
+    return bytes_.Append(data, length);
+  }
+
+  /// Appending a string to the sequence
+  Status AppendString(const char* data, int32_t length) {
+    RETURN_NOT_OK(Update(strings_.length(), &string_tag_));
+    return strings_.Append(data, length);
+  }
+
+  /// Appending a float to the sequence
+  Status AppendFloat(const float data) {
+    return AppendPrimitive(data, &float_tag_, &floats_);
+  }
+
+  /// Appending a double to the sequence
+  Status AppendDouble(const double data) {
+    return AppendPrimitive(data, &double_tag_, &doubles_);
+  }
+
+  /// Appending a tensor to the sequence
+  ///
+  /// \param tensor_index Index of the tensor in the object.
+  Status AppendTensor(const int32_t tensor_index) {
+    RETURN_NOT_OK(Update(tensor_indices_.length(), &tensor_tag_));
+    return tensor_indices_.Append(tensor_index);
+  }
+
+  /// Add a sublist to the sequence. The data contained in the sublist will be
+  /// specified in the "Finish" method.
+  ///
+  /// To construct l = [[11, 22], 33, [44, 55]] you would for example run
+  /// list = ListBuilder();
+  /// list.AppendList(2);
+  /// list.Append(33);
+  /// list.AppendList(2);
+  /// list.Finish([11, 22, 44, 55]);
+  /// list.Finish();
+
+  /// \param size
+  /// The size of the sublist
+  Status AppendList(Py_ssize_t size) {
+    RETURN_NOT_OK(Update(list_offsets_.size() - 1, &list_tag_));
+    list_offsets_.push_back(list_offsets_.back() + static_cast<int32_t>(size));
+    return Status::OK();
+  }
+
+  Status AppendTuple(Py_ssize_t size) {
+    RETURN_NOT_OK(Update(tuple_offsets_.size() - 1, &tuple_tag_));
+    tuple_offsets_.push_back(tuple_offsets_.back() + static_cast<int32_t>(size));
+    return Status::OK();
+  }
+
+  Status AppendDict(Py_ssize_t size) {
+    RETURN_NOT_OK(Update(dict_offsets_.size() - 1, &dict_tag_));
+    dict_offsets_.push_back(dict_offsets_.back() + static_cast<int32_t>(size));
+    return Status::OK();
+  }
+
+  template <typename BuilderType>
+  Status AddElement(const int8_t tag, BuilderType* out) {
+    if (tag != -1) {
+      fields_[tag] = ::arrow::field("", out->type());
+      RETURN_NOT_OK(out->Finish(&children_[tag]));
+      RETURN_NOT_OK(nones_.AppendToBitmap(true));
+      type_ids_.push_back(tag);
+    }
+    return Status::OK();
+  }
+
+  Status AddSubsequence(int8_t tag, const Array* data,
+                        const std::vector<int32_t>& offsets, const std::string& name) {
+    if (data != nullptr) {
+      DCHECK(data->length() == offsets.back());
+      std::shared_ptr<Array> offset_array;
+      Int32Builder builder(pool_, std::make_shared<Int32Type>());
+      RETURN_NOT_OK(builder.Append(offsets.data(), offsets.size()));
+      RETURN_NOT_OK(builder.Finish(&offset_array));
+      std::shared_ptr<Array> list_array;
+      RETURN_NOT_OK(ListArray::FromArrays(*offset_array, *data, pool_, &list_array));
+      auto field = ::arrow::field(name, list_array->type());
+      auto type = ::arrow::struct_({field});
+      fields_[tag] = ::arrow::field("", type);
+      children_[tag] = std::shared_ptr<StructArray>(
+          new StructArray(type, list_array->length(), {list_array}));
+      RETURN_NOT_OK(nones_.AppendToBitmap(true));
+      type_ids_.push_back(tag);
+    } else {
+      DCHECK_EQ(offsets.size(), 1);
+    }
+    return Status::OK();
+  }
+
+  /// Finish building the sequence and return the result.
+  /// Input arrays may be nullptr
+  Status Finish(const Array* list_data, const Array* tuple_data, const Array* dict_data,
+                std::shared_ptr<Array>* out) {
+    fields_.resize(num_tags_);
+    children_.resize(num_tags_);
+
+    RETURN_NOT_OK(AddElement(bool_tag_, &bools_));
+    RETURN_NOT_OK(AddElement(int_tag_, &ints_));
+    RETURN_NOT_OK(AddElement(string_tag_, &strings_));
+    RETURN_NOT_OK(AddElement(bytes_tag_, &bytes_));
+    RETURN_NOT_OK(AddElement(float_tag_, &floats_));
+    RETURN_NOT_OK(AddElement(double_tag_, &doubles_));
+    RETURN_NOT_OK(AddElement(tensor_tag_, &tensor_indices_));
+
+    RETURN_NOT_OK(AddSubsequence(list_tag_, list_data, list_offsets_, "list"));
+    RETURN_NOT_OK(AddSubsequence(tuple_tag_, tuple_data, tuple_offsets_, "tuple"));
+    RETURN_NOT_OK(AddSubsequence(dict_tag_, dict_data, dict_offsets_, "dict"));
+
+    auto type = ::arrow::union_(fields_, type_ids_, UnionMode::DENSE);
+    out->reset(new UnionArray(type, types_.length(), children_, types_.data(),
+                              offsets_.data(), nones_.null_bitmap(),
+                              nones_.null_count()));
+    return Status::OK();
+  }
+
+ private:
+  MemoryPool* pool_;
+
+  Int8Builder types_;
+  Int32Builder offsets_;
+
+  NullBuilder nones_;
+  BooleanBuilder bools_;
+  Int64Builder ints_;
+  BinaryBuilder bytes_;
+  StringBuilder strings_;
+  FloatBuilder floats_;
+  DoubleBuilder doubles_;
+
+  // We use an Int32Builder here to distinguish the tensor indices from
+  // the ints_ above (see the case Type::INT32 in get_value in python.cc).
+  // TODO(pcm): Replace this by using the union tags to distinguish between
+  // these two cases.
+  Int32Builder tensor_indices_;
+
+  std::vector<int32_t> list_offsets_;
+  std::vector<int32_t> tuple_offsets_;
+  std::vector<int32_t> dict_offsets_;
+
+  // Tags for members of the sequence. If they are set to -1 it means
+  // they are not used and will not part be of the metadata when we call
+  // SequenceBuilder::Finish. If a member with one of the tags is added,
+  // the associated variable gets a unique index starting from 0. This
+  // happens in the UPDATE macro in sequence.cc.
+  int8_t bool_tag_ = -1;
+  int8_t int_tag_ = -1;
+  int8_t string_tag_ = -1;
+  int8_t bytes_tag_ = -1;
+  int8_t float_tag_ = -1;
+  int8_t double_tag_ = -1;
+
+  int8_t tensor_tag_ = -1;
+  int8_t list_tag_ = -1;
+  int8_t tuple_tag_ = -1;
+  int8_t dict_tag_ = -1;
+
+  int8_t num_tags_ = 0;
+
+  // Members for the output union constructed in Finish
+  std::vector<std::shared_ptr<Field>> fields_;
+  std::vector<std::shared_ptr<Array>> children_;
+  std::vector<uint8_t> type_ids_;
+};
+
+/// Constructing dictionaries of key/value pairs. Sequences of
+/// keys and values are built separately using a pair of
+/// SequenceBuilders. The resulting Arrow representation
+/// can be obtained via the Finish method.
+class DictBuilder {
+ public:
+  explicit DictBuilder(MemoryPool* pool = nullptr) : keys_(pool), vals_(pool) {}
+
+  /// Builder for the keys of the dictionary
+  SequenceBuilder& keys() { return keys_; }
+  /// Builder for the values of the dictionary
+  SequenceBuilder& vals() { return vals_; }
+
+  /// Construct an Arrow StructArray representing the dictionary.
+  /// Contains a field "keys" for the keys and "vals" for the values.
+
+  /// \param list_data
+  ///    List containing the data from nested lists in the value
+  ///   list of the dictionary
+  ///
+  /// \param dict_data
+  ///   List containing the data from nested dictionaries in the
+  ///   value list of the dictionary
+  Status Finish(const Array* key_tuple_data, const Array* key_dict_data,
+                const Array* val_list_data, const Array* val_tuple_data,
+                const Array* val_dict_data, std::shared_ptr<Array>* out) {
+    // lists and dicts can't be keys of dicts in Python, that is why for
+    // the keys we do not need to collect sublists
+    std::shared_ptr<Array> keys, vals;
+    RETURN_NOT_OK(keys_.Finish(nullptr, key_tuple_data, key_dict_data, &keys));
+    RETURN_NOT_OK(vals_.Finish(val_list_data, val_tuple_data, val_dict_data, &vals));
+    auto keys_field = std::make_shared<Field>("keys", keys->type());
+    auto vals_field = std::make_shared<Field>("vals", vals->type());
+    auto type = std::make_shared<StructType>(
+        std::vector<std::shared_ptr<Field>>({keys_field, vals_field}));
+    std::vector<std::shared_ptr<Array>> field_arrays({keys, vals});
+    DCHECK(keys->length() == vals->length());
+    out->reset(new StructArray(type, keys->length(), field_arrays));
+    return Status::OK();
+  }
+
+ private:
+  SequenceBuilder keys_;
+  SequenceBuilder vals_;
+};
+
+Status CallCustomCallback(PyObject* callback, PyObject* elem, PyObject** result) {
+  *result = NULL;
+  if (!callback) {
+    std::stringstream ss;
+    ScopedRef repr(PyObject_Repr(elem));
+    RETURN_IF_PYERROR();
+    ScopedRef ascii(PyUnicode_AsASCIIString(repr.get()));
+    ss << "error while calling callback on " << PyBytes_AsString(ascii.get())
+       << ": handler not registered";
+    return Status::NotImplemented(ss.str());
+  } else {
+    ScopedRef arglist(Py_BuildValue("(O)", elem));
+    *result = PyObject_CallObject(callback, arglist.get());
+    RETURN_IF_PYERROR();
+  }
+  return Status::OK();
+}
+
+void set_serialization_callbacks(PyObject* serialize_callback,
+                                 PyObject* deserialize_callback) {
+  pyarrow_serialize_callback = serialize_callback;
+  pyarrow_deserialize_callback = deserialize_callback;
+}
+
+Status CallCustomSerializationCallback(PyObject* elem, PyObject** serialized_object) {
+  RETURN_NOT_OK(CallCustomCallback(pyarrow_serialize_callback, elem, serialized_object));
+  if (!PyDict_Check(*serialized_object)) {
+    return Status::TypeError("serialization callback must return a valid dictionary");
+  }
+  return Status::OK();
+}
+
+Status SerializeDict(std::vector<PyObject*> dicts, int32_t recursion_depth,
+                     std::shared_ptr<Array>* out, std::vector<PyObject*>* tensors_out);
+
+Status SerializeArray(PyArrayObject* array, SequenceBuilder* builder,
+                      std::vector<PyObject*>* subdicts,
+                      std::vector<PyObject*>* tensors_out);
+
+Status SerializeSequences(std::vector<PyObject*> sequences, int32_t recursion_depth,
+                          std::shared_ptr<Array>* out,
+                          std::vector<PyObject*>* tensors_out);
+
+Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
+  if (PyArray_IsScalar(obj, Bool)) {
+    return builder->AppendBool(reinterpret_cast<PyBoolScalarObject*>(obj)->obval != 0);
+  } else if (PyArray_IsScalar(obj, Float)) {
+    return builder->AppendFloat(reinterpret_cast<PyFloatScalarObject*>(obj)->obval);
+  } else if (PyArray_IsScalar(obj, Double)) {
+    return builder->AppendDouble(reinterpret_cast<PyDoubleScalarObject*>(obj)->obval);
+  }
+  int64_t value = 0;
+  if (PyArray_IsScalar(obj, Byte)) {
+    value = reinterpret_cast<PyByteScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, UByte)) {
+    value = reinterpret_cast<PyUByteScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, Short)) {
+    value = reinterpret_cast<PyShortScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, UShort)) {
+    value = reinterpret_cast<PyUShortScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, Int)) {
+    value = reinterpret_cast<PyIntScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, UInt)) {
+    value = reinterpret_cast<PyUIntScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, Long)) {
+    value = reinterpret_cast<PyLongScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, ULong)) {
+    value = reinterpret_cast<PyULongScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, LongLong)) {
+    value = reinterpret_cast<PyLongLongScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, Int64)) {
+    value = reinterpret_cast<PyInt64ScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, ULongLong)) {
+    value = reinterpret_cast<PyULongLongScalarObject*>(obj)->obval;
+  } else if (PyArray_IsScalar(obj, UInt64)) {
+    value = reinterpret_cast<PyUInt64ScalarObject*>(obj)->obval;
+  } else {
+    DCHECK(false) << "scalar type not recognized";
+  }
+  return builder->AppendInt64(value);
+}
+
+Status Append(PyObject* elem, SequenceBuilder* builder, std::vector<PyObject*>* sublists,
+              std::vector<PyObject*>* subtuples, std::vector<PyObject*>* subdicts,
+              std::vector<PyObject*>* tensors_out) {
+  // The bool case must precede the int case (PyInt_Check passes for bools)
+  if (PyBool_Check(elem)) {
+    RETURN_NOT_OK(builder->AppendBool(elem == Py_True));
+  } else if (PyFloat_Check(elem)) {
+    RETURN_NOT_OK(builder->AppendDouble(PyFloat_AS_DOUBLE(elem)));
+  } else if (PyLong_Check(elem)) {
+    int overflow = 0;
+    int64_t data = PyLong_AsLongLongAndOverflow(elem, &overflow);
+    if (!overflow) {
+      RETURN_NOT_OK(builder->AppendInt64(data));
+    } else {
+      // Attempt to serialize the object using the custom callback.
+      PyObject* serialized_object;
+      // The reference count of serialized_object will be decremented in SerializeDict
+      RETURN_NOT_OK(CallCustomSerializationCallback(elem, &serialized_object));
+      RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object)));
+      subdicts->push_back(serialized_object);
+    }
+#if PY_MAJOR_VERSION < 3
+  } else if (PyInt_Check(elem)) {
+    RETURN_NOT_OK(builder->AppendInt64(static_cast<int64_t>(PyInt_AS_LONG(elem))));
+#endif
+  } else if (PyBytes_Check(elem)) {
+    auto data = reinterpret_cast<uint8_t*>(PyBytes_AS_STRING(elem));
+    const int64_t size = static_cast<int64_t>(PyBytes_GET_SIZE(elem));
+    if (size > std::numeric_limits<int32_t>::max()) {
+      return Status::Invalid("Cannot writes bytes over 2GB");
+    }
+    RETURN_NOT_OK(builder->AppendBytes(data, static_cast<int32_t>(size)));
+  } else if (PyUnicode_Check(elem)) {
+    Py_ssize_t size;
+#if PY_MAJOR_VERSION >= 3
+    char* data = PyUnicode_AsUTF8AndSize(elem, &size);
+#else
+    ScopedRef str(PyUnicode_AsUTF8String(elem));
+    char* data = PyString_AS_STRING(str.get());
+    size = PyString_GET_SIZE(str.get());
+#endif
+    if (size > std::numeric_limits<int32_t>::max()) {
+      return Status::Invalid("Cannot writes bytes over 2GB");
+    }
+    RETURN_NOT_OK(builder->AppendString(data, static_cast<int32_t>(size)));
+  } else if (PyList_Check(elem)) {
+    RETURN_NOT_OK(builder->AppendList(PyList_Size(elem)));
+    sublists->push_back(elem);
+  } else if (PyDict_Check(elem)) {
+    RETURN_NOT_OK(builder->AppendDict(PyDict_Size(elem)));
+    subdicts->push_back(elem);
+  } else if (PyTuple_CheckExact(elem)) {
+    RETURN_NOT_OK(builder->AppendTuple(PyTuple_Size(elem)));
+    subtuples->push_back(elem);
+  } else if (PyArray_IsScalar(elem, Generic)) {
+    RETURN_NOT_OK(AppendScalar(elem, builder));
+  } else if (PyArray_Check(elem)) {
+    RETURN_NOT_OK(SerializeArray(reinterpret_cast<PyArrayObject*>(elem), builder,
+                                 subdicts, tensors_out));
+  } else if (elem == Py_None) {
+    RETURN_NOT_OK(builder->AppendNone());
+  } else {
+    // Attempt to serialize the object using the custom callback.
+    PyObject* serialized_object;
+    // The reference count of serialized_object will be decremented in SerializeDict
+    RETURN_NOT_OK(CallCustomSerializationCallback(elem, &serialized_object));
+    RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object)));
+    subdicts->push_back(serialized_object);
+  }
+  return Status::OK();
+}
+
+Status SerializeArray(PyArrayObject* array, SequenceBuilder* builder,
+                      std::vector<PyObject*>* subdicts,
+                      std::vector<PyObject*>* tensors_out) {
+  int dtype = PyArray_TYPE(array);
+  switch (dtype) {
+    case NPY_BOOL:
+    case NPY_UINT8:
+    case NPY_INT8:
+    case NPY_UINT16:
+    case NPY_INT16:
+    case NPY_UINT32:
+    case NPY_INT32:
+    case NPY_UINT64:
+    case NPY_INT64:
+    case NPY_FLOAT:
+    case NPY_DOUBLE: {
+      RETURN_NOT_OK(builder->AppendTensor(static_cast<int32_t>(tensors_out->size())));
+      tensors_out->push_back(reinterpret_cast<PyObject*>(array));
+    } break;
+    default: {
+      PyObject* serialized_object;
+      // The reference count of serialized_object will be decremented in SerializeDict
+      RETURN_NOT_OK(CallCustomSerializationCallback(reinterpret_cast<PyObject*>(array),
+                                                    &serialized_object));
+      RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object)));
+      subdicts->push_back(serialized_object);
+    }
+  }
+  return Status::OK();
+}
+
+Status SerializeSequences(std::vector<PyObject*> sequences, int32_t recursion_depth,
+                          std::shared_ptr<Array>* out,
+                          std::vector<PyObject*>* tensors_out) {
+  DCHECK(out);
+  if (recursion_depth >= kMaxRecursionDepth) {
+    return Status::NotImplemented(
+        "This object exceeds the maximum recursion depth. It may contain itself "
+        "recursively.");
+  }
+  SequenceBuilder builder(nullptr);
+  std::vector<PyObject *> sublists, subtuples, subdicts;
+  for (const auto& sequence : sequences) {
+    ScopedRef iterator(PyObject_GetIter(sequence));
+    RETURN_IF_PYERROR();
+    ScopedRef item;
+    while (item.reset(PyIter_Next(iterator.get())), item.get()) {
+      RETURN_NOT_OK(
+          Append(item.get(), &builder, &sublists, &subtuples, &subdicts, tensors_out));
+    }
+  }
+  std::shared_ptr<Array> list;
+  if (sublists.size() > 0) {
+    RETURN_NOT_OK(SerializeSequences(sublists, recursion_depth + 1, &list, tensors_out));
+  }
+  std::shared_ptr<Array> tuple;
+  if (subtuples.size() > 0) {
+    RETURN_NOT_OK(
+        SerializeSequences(subtuples, recursion_depth + 1, &tuple, tensors_out));
+  }
+  std::shared_ptr<Array> dict;
+  if (subdicts.size() > 0) {
+    RETURN_NOT_OK(SerializeDict(subdicts, recursion_depth + 1, &dict, tensors_out));
+  }
+  return builder.Finish(list.get(), tuple.get(), dict.get(), out);
+}
+
+Status SerializeDict(std::vector<PyObject*> dicts, int32_t recursion_depth,
+                     std::shared_ptr<Array>* out, std::vector<PyObject*>* tensors_out) {
+  DictBuilder result;
+  if (recursion_depth >= kMaxRecursionDepth) {
+    return Status::NotImplemented(
+        "This object exceeds the maximum recursion depth. It may contain itself "
+        "recursively.");
+  }
+  std::vector<PyObject *> key_tuples, key_dicts, val_lists, val_tuples, val_dicts, dummy;
+  for (const auto& dict : dicts) {
+    PyObject *key, *value;
+    Py_ssize_t pos = 0;
+    while (PyDict_Next(dict, &pos, &key, &value)) {
+      RETURN_NOT_OK(
+          Append(key, &result.keys(), &dummy, &key_tuples, &key_dicts, tensors_out));
+      DCHECK_EQ(dummy.size(), 0);
+      RETURN_NOT_OK(Append(value, &result.vals(), &val_lists, &val_tuples, &val_dicts,
+                           tensors_out));
+    }
+  }
+  std::shared_ptr<Array> key_tuples_arr;
+  if (key_tuples.size() > 0) {
+    RETURN_NOT_OK(SerializeSequences(key_tuples, recursion_depth + 1, &key_tuples_arr,
+                                     tensors_out));
+  }
+  std::shared_ptr<Array> key_dicts_arr;
+  if (key_dicts.size() > 0) {
+    RETURN_NOT_OK(
+        SerializeDict(key_dicts, recursion_depth + 1, &key_dicts_arr, tensors_out));
+  }
+  std::shared_ptr<Array> val_list_arr;
+  if (val_lists.size() > 0) {
+    RETURN_NOT_OK(
+        SerializeSequences(val_lists, recursion_depth + 1, &val_list_arr, tensors_out));
+  }
+  std::shared_ptr<Array> val_tuples_arr;
+  if (val_tuples.size() > 0) {
+    RETURN_NOT_OK(SerializeSequences(val_tuples, recursion_depth + 1, &val_tuples_arr,
+                                     tensors_out));
+  }
+  std::shared_ptr<Array> val_dict_arr;
+  if (val_dicts.size() > 0) {
+    RETURN_NOT_OK(
+        SerializeDict(val_dicts, recursion_depth + 1, &val_dict_arr, tensors_out));
+  }
+  RETURN_NOT_OK(result.Finish(key_tuples_arr.get(), key_dicts_arr.get(),
+                              val_list_arr.get(), val_tuples_arr.get(),
+                              val_dict_arr.get(), out));
+
+  // This block is used to decrement the reference counts of the results
+  // returned by the serialization callback, which is called in SerializeArray,
+  // in DeserializeDict and in Append
+  static PyObject* py_type = PyUnicode_FromString("_pytype_");
+  for (const auto& dict : dicts) {
+    if (PyDict_Contains(dict, py_type)) {
+      // If the dictionary contains the key "_pytype_", then the user has to
+      // have registered a callback.
+      if (pyarrow_serialize_callback == nullptr) {
+        return Status::Invalid("No serialization callback set");
+      }
+      Py_XDECREF(dict);
+    }
+  }
+
+  return Status::OK();
+}
+
+std::shared_ptr<RecordBatch> MakeBatch(std::shared_ptr<Array> data) {
+  auto field = std::make_shared<Field>("list", data->type());
+  auto schema = ::arrow::schema({field});
+  return std::shared_ptr<RecordBatch>(new RecordBatch(schema, data->length(), {data}));
+}
+
+Status SerializeObject(PyObject* sequence, SerializedPyObject* out) {
+  PyAcquireGIL lock;
+  std::vector<PyObject*> sequences = {sequence};
+  std::shared_ptr<Array> array;
+  std::vector<PyObject*> py_tensors;
+  RETURN_NOT_OK(SerializeSequences(sequences, 0, &array, &py_tensors));
+  out->batch = MakeBatch(array);
+  for (const auto& py_tensor : py_tensors) {
+    std::shared_ptr<Tensor> arrow_tensor;
+    RETURN_NOT_OK(NdarrayToTensor(default_memory_pool(), py_tensor, &arrow_tensor));
+    out->tensors.push_back(arrow_tensor);
+  }
+  return Status::OK();
+}
+
+Status WriteSerializedObject(const SerializedPyObject& obj, io::OutputStream* dst) {
+  int32_t num_tensors = static_cast<int32_t>(obj.tensors.size());
+  std::shared_ptr<ipc::RecordBatchStreamWriter> writer;
+  int32_t metadata_length;
+  int64_t body_length;
+
+  RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&num_tensors), sizeof(int32_t)));
+  RETURN_NOT_OK(ipc::RecordBatchStreamWriter::Open(dst, obj.batch->schema(), &writer));
+  RETURN_NOT_OK(writer->WriteRecordBatch(*obj.batch));
+  RETURN_NOT_OK(writer->Close());
+
+  for (const auto& tensor : obj.tensors) {
+    RETURN_NOT_OK(ipc::WriteTensor(*tensor, dst, &metadata_length, &body_length));
+  }
+
+  return Status::OK();
+}
+
+}  // namespace py
+}  // namespace arrow
diff --git a/cpp/src/arrow/python/python_to_arrow.h b/cpp/src/arrow/python/python_to_arrow.h
new file mode 100644
index 0000000000000..8ac03965a1c44
--- /dev/null
+++ b/cpp/src/arrow/python/python_to_arrow.h
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PYTHON_PYTHON_TO_ARROW_H
+#define ARROW_PYTHON_PYTHON_TO_ARROW_H
+
+#include "arrow/python/platform.h"
+
+#include <memory>
+#include <vector>
+
+#include "arrow/status.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class RecordBatch;
+class Tensor;
+
+namespace io {
+
+class OutputStream;
+
+}  // namespace io
+
+namespace py {
+
+struct ARROW_EXPORT SerializedPyObject {
+  std::shared_ptr<RecordBatch> batch;
+  std::vector<std::shared_ptr<Tensor>> tensors;
+};
+
+/// \brief Register callback functions to perform conversions to or from other
+/// Python representations en route to/from deserialization
+///
+/// \param[in] serialize_callback a Python callable
+/// \param[in] deserialize_callback a Python callable
+///
+/// Analogous to Python custom picklers / unpicklers
+ARROW_EXPORT
+void set_serialization_callbacks(PyObject* serialize_callback,
+                                 PyObject* deserialize_callback);
+
+/// \brief Serialize Python sequence as a RecordBatch plus
+/// \param[in] sequence a Python sequence object to serialize to Arrow data
+/// structures
+/// \param[out] out the serialized representation
+/// \return Status
+///
+/// Release GIL before calling
+ARROW_EXPORT
+Status SerializeObject(PyObject* sequence, SerializedPyObject* out);
+
+/// \brief Write serialized Python object to OutputStream
+/// \param[in] object a serialized Python object to write out
+/// \param[out] dst an OutputStream
+/// \return Status
+ARROW_EXPORT
+Status WriteSerializedObject(const SerializedPyObject& object, io::OutputStream* dst);
+
+}  // namespace py
+}  // namespace arrow
+
+#endif  // ARROW_PYTHON_PYTHON_TO_ARROW_H
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 1aaf89ce9a1f0..846af4c7f2ef8 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -181,8 +181,8 @@ File Systems
 
 .. _api.ipc:
 
-Interprocess Communication and Messaging
-----------------------------------------
+Interprocess Communication and Serialization
+--------------------------------------------
 
 .. autosummary::
    :toctree: generated/
@@ -201,6 +201,12 @@ Interprocess Communication and Messaging
    read_tensor
    write_tensor
    get_tensor_size
+   serialize
+   serialize_to
+   deserialize
+   deserialize_from
+   read_serialized
+   SerializedPyObject
 
 .. _api.memory_pool:
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index d0348b43c503c..4abf29edd2612 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -87,6 +87,10 @@
                          ArrowNotImplementedError,
                          ArrowTypeError)
 
+# Serialization
+from pyarrow.lib import (deserialize_from, deserialize,
+                         serialize, serialize_to, read_serialized,
+                         SerializedPyObject)
 
 from pyarrow.filesystem import FileSystem, LocalFileSystem
 
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 2252e85e6ef77..df5e4faadd443 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -129,6 +129,13 @@ def tobytes(o):
     def frombytes(o):
         return o.decode('utf8')
 
+try:
+    import cloudpickle as pickle
+except ImportError:
+    try:
+        import cPickle as pickle
+    except ImportError:
+        import pickle
 
 def encode_file_path(path):
     import os
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index c6a9d9d0b7b18..082fb61429825 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -778,6 +778,26 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     cdef struct PandasOptions:
         c_bool strings_to_categorical
 
+cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:
+
+    cdef cppclass CSerializedPyObject" arrow::py::SerializedPyObject":
+        shared_ptr[CRecordBatch] batch
+        vector[shared_ptr[CTensor]] tensors
+
+    CStatus SerializeObject(object sequence, CSerializedPyObject* out)
+
+    CStatus WriteSerializedObject(const CSerializedPyObject& obj,
+                                  OutputStream* dst)
+
+    CStatus DeserializeObject(const CSerializedPyObject& obj,
+                              PyObject* base, PyObject** out)
+
+    CStatus ReadSerializedObject(shared_ptr[RandomAccessFile] src,
+                                 CSerializedPyObject* out)
+
+    void set_serialization_callbacks(object serialize_callback,
+                                     object deserialize_callback)
+
 
 cdef extern from 'arrow/python/init.h':
     int arrow_init_numpy() except -1
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 061a7a9a4d6e6..b5858ab20ba7f 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -749,7 +749,7 @@ cdef get_writer(object source, shared_ptr[OutputStream]* writer):
     if isinstance(source, NativeFile):
         nf = source
 
-        if nf.is_readable:
+        if not nf.is_writeable:
             raise IOError('Native file is not writeable')
 
         nf.write_handle(writer)
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 789801b9f06a9..4ea327ef9264c 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -119,5 +119,8 @@ include "ipc.pxi"
 # Feather format
 include "feather.pxi"
 
+# Python serialization
+include "serialization.pxi"
+
 # Public API
 include "public-api.pxi"
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
new file mode 100644
index 0000000000000..a6c955bef99ba
--- /dev/null
+++ b/python/pyarrow/serialization.pxi
@@ -0,0 +1,279 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from cpython.ref cimport PyObject
+
+from pyarrow.compat import pickle
+
+
+def is_named_tuple(cls):
+    """Return True if cls is a namedtuple and False otherwise."""
+    b = cls.__bases__
+    if len(b) != 1 or b[0] != tuple:
+        return False
+    f = getattr(cls, "_fields", None)
+    if not isinstance(f, tuple):
+        return False
+    return all(type(n) == str for n in f)
+
+
+class SerializationException(Exception):
+    def __init__(self, message, example_object):
+        Exception.__init__(self, message)
+        self.example_object = example_object
+
+
+class DeserializationException(Exception):
+    def __init__(self, message, type_id):
+        Exception.__init__(self, message)
+        self.type_id = type_id
+
+
+# Types with special serialization handlers
+type_to_type_id = dict()
+whitelisted_types = dict()
+types_to_pickle = set()
+custom_serializers = dict()
+custom_deserializers = dict()
+
+
+def register_type(type, type_id, pickle=False,
+                  custom_serializer=None, custom_deserializer=None):
+    """Add type to the list of types we can serialize.
+
+    Parameters
+    ----------
+    type :type
+        The type that we can serialize.
+    type_id : bytes
+        A string of bytes used to identify the type.
+    pickle : bool
+        True if the serialization should be done with pickle.
+        False if it should be done efficiently with Arrow.
+    custom_serializer : callable
+        This argument is optional, but can be provided to
+        serialize objects of the class in a particular way.
+    custom_deserializer : callable
+        This argument is optional, but can be provided to
+        deserialize objects of the class in a particular way.
+    """
+    type_to_type_id[type] = type_id
+    whitelisted_types[type_id] = type
+    if pickle:
+        types_to_pickle.add(type_id)
+    if custom_serializer is not None:
+        custom_serializers[type_id] = custom_serializer
+        custom_deserializers[type_id] = custom_deserializer
+
+
+def _serialization_callback(obj):
+    if type(obj) not in type_to_type_id:
+        raise SerializationException("pyarrow does not know how to "
+                                     "serialize objects of type {}."
+                                     .format(type(obj)),
+                                     obj)
+    type_id = type_to_type_id[type(obj)]
+    if type_id in types_to_pickle:
+        serialized_obj = {"data": pickle.dumps(obj), "pickle": True}
+    elif type_id in custom_serializers:
+        serialized_obj = {"data": custom_serializers[type_id](obj)}
+    else:
+        if is_named_tuple(type(obj)):
+            serialized_obj = {}
+            serialized_obj["_pa_getnewargs_"] = obj.__getnewargs__()
+        elif hasattr(obj, "__dict__"):
+            serialized_obj = obj.__dict__
+        else:
+            raise SerializationException("We do not know how to serialize "
+                                         "the object '{}'".format(obj), obj)
+    return dict(serialized_obj, **{"_pytype_": type_id})
+
+
+def _deserialization_callback(serialized_obj):
+    type_id = serialized_obj["_pytype_"]
+
+    if "pickle" in serialized_obj:
+        # The object was pickled, so unpickle it.
+        obj = pickle.loads(serialized_obj["data"])
+    else:
+        assert type_id not in types_to_pickle
+        if type_id not in whitelisted_types:
+            raise "error"
+        type = whitelisted_types[type_id]
+        if type_id in custom_deserializers:
+            obj = custom_deserializers[type_id](serialized_obj["data"])
+        else:
+            # In this case, serialized_obj should just be the __dict__ field.
+            if "_pa_getnewargs_" in serialized_obj:
+                obj = type.__new__(type, *serialized_obj["_pa_getnewargs_"])
+            else:
+                obj = type.__new__(type)
+                serialized_obj.pop("_pytype_")
+                obj.__dict__.update(serialized_obj)
+    return obj
+
+
+set_serialization_callbacks(_serialization_callback,
+                            _deserialization_callback)
+
+
+cdef class SerializedPyObject:
+    """
+    Arrow-serialized representation of Python object
+    """
+    cdef:
+        CSerializedPyObject data
+
+    cdef readonly:
+        object base
+
+    property total_bytes:
+
+        def __get__(self):
+            cdef CMockOutputStream mock_stream
+            with nogil:
+                check_status(WriteSerializedObject(self.data, &mock_stream))
+
+            return mock_stream.GetExtentBytesWritten()
+
+    def write_to(self, sink):
+        """
+        Write serialized object to a sink
+        """
+        cdef shared_ptr[OutputStream] stream
+        get_writer(sink, &stream)
+        self._write_to(stream.get())
+
+    cdef _write_to(self, OutputStream* stream):
+        with nogil:
+            check_status(WriteSerializedObject(self.data, stream))
+
+    def deserialize(self):
+        """
+        Convert back to Python object
+        """
+        cdef PyObject* result
+
+        with nogil:
+            check_status(DeserializeObject(self.data, <PyObject*> self.base,
+                                           &result))
+
+        # This is necessary to avoid a memory leak
+        return PyObject_to_object(result)
+
+    def to_buffer(self):
+        """
+        Write serialized data as Buffer
+        """
+        sink = BufferOutputStream()
+        self.write_to(sink)
+        return sink.get_result()
+
+
+def serialize(object value):
+    """EXPERIMENTAL: Serialize a Python sequence
+
+    Parameters
+    ----------
+    value: object
+        Python object for the sequence that is to be serialized.
+
+    Returns
+    -------
+    serialized : SerializedPyObject
+    """
+    cdef SerializedPyObject serialized = SerializedPyObject()
+    with nogil:
+        check_status(SerializeObject(value, &serialized.data))
+    return serialized
+
+
+def serialize_to(object value, sink):
+    """EXPERIMENTAL: Serialize a Python sequence to a file.
+
+    Parameters
+    ----------
+    value: object
+        Python object for the sequence that is to be serialized.
+    sink: NativeFile or file-like
+        File the sequence will be written to.
+    """
+    serialized = serialize(value)
+    serialized.write_to(sink)
+
+
+def read_serialized(source, base=None):
+    """EXPERIMENTAL: Read serialized Python sequence from file-like object
+
+    Parameters
+    ----------
+    source: NativeFile
+        File to read the sequence from.
+    base: object
+        This object will be the base object of all the numpy arrays
+        contained in the sequence.
+
+    Returns
+    -------
+    serialized : the serialized data
+    """
+    cdef shared_ptr[RandomAccessFile] stream
+    get_reader(source, &stream)
+
+    cdef SerializedPyObject serialized = SerializedPyObject()
+    serialized.base = base
+    with nogil:
+        check_status(ReadSerializedObject(stream, &serialized.data))
+
+    return serialized
+
+
+def deserialize_from(source, object base):
+    """EXPERIMENTAL: Deserialize a Python sequence from a file.
+
+    Parameters
+    ----------
+    source: NativeFile
+        File to read the sequence from.
+    base: object
+        This object will be the base object of all the numpy arrays
+        contained in the sequence.
+
+    Returns
+    -------
+    object
+        Python object for the deserialized sequence.
+    """
+    serialized = read_serialized(source, base=base)
+    return serialized.deserialize()
+
+
+def deserialize(obj):
+    """
+    EXPERIMENTAL: Deserialize Python object from Buffer or other Python object
+    supporting the buffer protocol
+
+    Parameters
+    ----------
+    obj : pyarrow.Buffer or Python object supporting buffer protocol
+
+    Returns
+    -------
+    deserialized : object
+    """
+    source = BufferReader(obj)
+    return deserialize_from(source, obj)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
new file mode 100644
index 0000000000000..f6f98402ac056
--- /dev/null
+++ b/python/pyarrow/tests/test_serialization.py
@@ -0,0 +1,236 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import division
+
+import pytest
+
+from collections import namedtuple
+import os
+import string
+import sys
+
+import pyarrow as pa
+import numpy as np
+
+
+def assert_equal(obj1, obj2):
+    module_numpy = (type(obj1).__module__ == np.__name__ or
+                    type(obj2).__module__ == np.__name__)
+    if module_numpy:
+        empty_shape = ((hasattr(obj1, "shape") and obj1.shape == ()) or
+                       (hasattr(obj2, "shape") and obj2.shape == ()))
+        if empty_shape:
+            # This is a special case because currently np.testing.assert_equal
+            # fails because we do not properly handle different numerical
+            # types.
+            assert obj1 == obj2, ("Objects {} and {} are "
+                                  "different.".format(obj1, obj2))
+        else:
+            np.testing.assert_equal(obj1, obj2)
+    elif hasattr(obj1, "__dict__") and hasattr(obj2, "__dict__"):
+        special_keys = ["_pytype_"]
+        assert (set(list(obj1.__dict__.keys()) + special_keys) ==
+                set(list(obj2.__dict__.keys()) + special_keys)), ("Objects {} "
+                                                                  "and {} are "
+                                                                  "different."
+                                                                  .format(
+                                                                      obj1,
+                                                                      obj2))
+        for key in obj1.__dict__.keys():
+            if key not in special_keys:
+                assert_equal(obj1.__dict__[key], obj2.__dict__[key])
+    elif type(obj1) is dict or type(obj2) is dict:
+        assert_equal(obj1.keys(), obj2.keys())
+        for key in obj1.keys():
+            assert_equal(obj1[key], obj2[key])
+    elif type(obj1) is list or type(obj2) is list:
+        assert len(obj1) == len(obj2), ("Objects {} and {} are lists with "
+                                        "different lengths."
+                                        .format(obj1, obj2))
+        for i in range(len(obj1)):
+            assert_equal(obj1[i], obj2[i])
+    elif type(obj1) is tuple or type(obj2) is tuple:
+        assert len(obj1) == len(obj2), ("Objects {} and {} are tuples with "
+                                        "different lengths."
+                                        .format(obj1, obj2))
+        for i in range(len(obj1)):
+            assert_equal(obj1[i], obj2[i])
+    elif (pa.lib.is_named_tuple(type(obj1)) or
+          pa.lib.is_named_tuple(type(obj2))):
+        assert len(obj1) == len(obj2), ("Objects {} and {} are named tuples "
+                                        "with different lengths."
+                                        .format(obj1, obj2))
+        for i in range(len(obj1)):
+            assert_equal(obj1[i], obj2[i])
+    else:
+        assert obj1 == obj2, "Objects {} and {} are different.".format(obj1,
+                                                                       obj2)
+
+
+def array_custom_serializer(obj):
+    return obj.tolist(), obj.dtype.str
+
+
+def array_custom_deserializer(serialized_obj):
+    return np.array(serialized_obj[0], dtype=np.dtype(serialized_obj[1]))
+
+pa.lib.register_type(np.ndarray, 20 * b"\x00", pickle=False,
+                     custom_serializer=array_custom_serializer,
+                     custom_deserializer=array_custom_deserializer)
+
+if sys.version_info >= (3, 0):
+    long_extras = [0, np.array([["hi", u"hi"], [1.3, 1]])]
+else:
+    _LONG_ZERO, _LONG_ONE = long(0), long(1)  # noqa: E501,F821
+    long_extras = [_LONG_ZERO, np.array([["hi", u"hi"],
+                                         [1.3, _LONG_ONE]])]
+
+PRIMITIVE_OBJECTS = [
+    0, 0.0, 0.9, 1 << 62, 1 << 100, 1 << 999,
+    [1 << 100, [1 << 100]], "a", string.printable, "\u262F",
+    u"hello world", u"\xff\xfe\x9c\x001\x000\x00", None, True,
+    False, [], (), {}, np.int8(3), np.int32(4), np.int64(5),
+    np.uint8(3), np.uint32(4), np.uint64(5), np.float32(1.9),
+    np.float64(1.9), np.zeros([100, 100]),
+    np.random.normal(size=[100, 100]), np.array(["hi", 3]),
+    np.array(["hi", 3], dtype=object)] + long_extras
+
+COMPLEX_OBJECTS = [
+    [[[[[[[[[[[[]]]]]]]]]]]],
+    {"obj{}".format(i): np.random.normal(size=[100, 100]) for i in range(10)},
+    # {(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {
+    #       (): {(): {}}}}}}}}}}}}},
+    ((((((((((),),),),),),),),),),
+    {"a": {"b": {"c": {"d": {}}}}}]
+
+
+class Foo(object):
+    def __init__(self, value=0):
+        self.value = value
+
+    def __hash__(self):
+        return hash(self.value)
+
+    def __eq__(self, other):
+        return other.value == self.value
+
+
+class Bar(object):
+    def __init__(self):
+        for i, val in enumerate(PRIMITIVE_OBJECTS + COMPLEX_OBJECTS):
+            setattr(self, "field{}".format(i), val)
+
+
+class Baz(object):
+    def __init__(self):
+        self.foo = Foo()
+        self.bar = Bar()
+
+    def method(self, arg):
+        pass
+
+
+class Qux(object):
+    def __init__(self):
+        self.objs = [Foo(), Bar(), Baz()]
+
+
+class SubQux(Qux):
+    def __init__(self):
+        Qux.__init__(self)
+
+
+class CustomError(Exception):
+    pass
+
+
+Point = namedtuple("Point", ["x", "y"])
+NamedTupleExample = namedtuple("Example",
+                               "field1, field2, field3, field4, field5")
+
+
+CUSTOM_OBJECTS = [Exception("Test object."), CustomError(), Point(11, y=22),
+                  Foo(), Bar(), Baz(), Qux(), SubQux(),
+                  NamedTupleExample(1, 1.0, "hi", np.zeros([3, 5]), [1, 2, 3])]
+
+pa.lib.register_type(Foo, 20 * b"\x01")
+pa.lib.register_type(Bar, 20 * b"\x02")
+pa.lib.register_type(Baz, 20 * b"\x03")
+pa.lib.register_type(Qux, 20 * b"\x04")
+pa.lib.register_type(SubQux, 20 * b"\x05")
+pa.lib.register_type(Exception, 20 * b"\x06")
+pa.lib.register_type(CustomError, 20 * b"\x07")
+pa.lib.register_type(Point, 20 * b"\x08")
+pa.lib.register_type(NamedTupleExample, 20 * b"\x09")
+
+# TODO(pcm): This is currently a workaround until arrow supports
+# arbitrary precision integers. This is only called on long integers,
+# see the associated case in the append method in python_to_arrow.cc
+pa.lib.register_type(int, 20 * b"\x10", pickle=False,
+                     custom_serializer=lambda obj: str(obj),
+                     custom_deserializer=(
+                         lambda serialized_obj: int(serialized_obj)))
+
+
+if (sys.version_info < (3, 0)):
+    deserializer = (
+        lambda serialized_obj: long(serialized_obj))  # noqa: E501,F821
+    pa.lib.register_type(long, 20 * b"\x11", pickle=False,  # noqa: E501,F821
+                         custom_serializer=lambda obj: str(obj),
+                         custom_deserializer=deserializer)
+
+
+def serialization_roundtrip(value, f):
+    f.seek(0)
+    pa.serialize_to(value, f)
+    f.seek(0)
+    result = pa.deserialize_from(f, None)
+    assert_equal(value, result)
+
+
+@pytest.yield_fixture(scope='session')
+def large_memory_map(tmpdir_factory):
+    path = (tmpdir_factory.mktemp('data')
+            .join('pyarrow-serialization-tmp-file').strpath)
+
+    # Create a large memory mapped file
+    SIZE = 100 * 1024 * 1024  # 100 MB
+    with open(path, 'wb') as f:
+        f.write(np.random.randint(0, 256, size=SIZE)
+                .astype('u1')
+                .tobytes()
+                [:SIZE])
+    return path
+
+
+def test_primitive_serialization(large_memory_map):
+    with pa.memory_map(large_memory_map, mode="r+") as mmap:
+        for obj in PRIMITIVE_OBJECTS:
+            serialization_roundtrip([obj], mmap)
+
+
+def test_complex_serialization(large_memory_map):
+    with pa.memory_map(large_memory_map, mode="r+") as mmap:
+        for obj in COMPLEX_OBJECTS:
+            serialization_roundtrip([obj], mmap)
+
+
+def test_custom_serialization(large_memory_map):
+    with pa.memory_map(large_memory_map, mode="r+") as mmap:
+        for obj in CUSTOM_OBJECTS:
+            serialization_roundtrip([obj], mmap)

From de7c6715ba244e119913bfa31b8de46dbbd450bf Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 20 Aug 2017 13:49:46 -0400
Subject: [PATCH 0945/1644] ARROW-1357: [Python] Account for chunked arrays
 when converting lists back to pandas form

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #979 from wesm/ARROW-1357 and squashes the following commits:

8318a121 [Wes McKinney] Use PyLong_FromLongLong so Windows is happy
18acdd91 [Wes McKinney] Account for chunked arrays when converting lists back to pandas form
---
 cpp/src/arrow/python/arrow_to_pandas.cc     |  7 ++++--
 python/pyarrow/tests/test_convert_pandas.py | 25 +++++++++++++++++++++
 python/pyarrow/tests/test_serialization.py  |  1 -
 3 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 23bef7bcae65d..d1fca70412c42 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -498,6 +498,7 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
 
   PyAcquireGIL lock;
 
+  int64_t chunk_offset = 0;
   for (int c = 0; c < data.num_chunks(); c++) {
     auto arr = std::static_pointer_cast<ListArray>(data.chunk(c));
 
@@ -507,8 +508,8 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
         Py_INCREF(Py_None);
         *out_values = Py_None;
       } else {
-        PyObject* start = PyLong_FromLong(arr->value_offset(i));
-        PyObject* end = PyLong_FromLong(arr->value_offset(i + 1));
+        PyObject* start = PyLong_FromLongLong(arr->value_offset(i) + chunk_offset);
+        PyObject* end = PyLong_FromLongLong(arr->value_offset(i + 1) + chunk_offset);
         PyObject* slice = PySlice_New(start, end, NULL);
         *out_values = PyObject_GetItem(numpy_array, slice);
         Py_DECREF(start);
@@ -517,6 +518,8 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
       }
       ++out_values;
     }
+
+    chunk_offset += arr->length();
   }
 
   Py_XDECREF(numpy_array);
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 61bd072f6bae9..c2631842278e0 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -534,6 +534,31 @@ def test_column_of_lists(self):
             field = schema.field_by_name(column)
             self._check_array_roundtrip(df[column], type=field.type)
 
+    def test_column_of_lists_chunked(self):
+        # ARROW-1357
+        df = pd.DataFrame({
+            'lists': np.array([
+                [1, 2],
+                None,
+                [2, 3],
+                [4, 5],
+                [6, 7],
+                [8, 9]
+            ], dtype=object)
+        })
+
+        schema = pa.schema([
+            pa.field('lists', pa.list_(pa.int64()))
+        ])
+
+        t1 = pa.Table.from_pandas(df[:2], schema=schema)
+        t2 = pa.Table.from_pandas(df[2:], schema=schema)
+
+        table = pa.concat_tables([t1, t2])
+        result = table.to_pandas()
+
+        tm.assert_frame_equal(result, df)
+
     def test_column_of_lists_strided(self):
         df, schema = dataframe_with_lists()
         df = pd.concat([df] * 6, ignore_index=True)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index f6f98402ac056..013d86ebf9cb9 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -20,7 +20,6 @@
 import pytest
 
 from collections import namedtuple
-import os
 import string
 import sys
 

From 6ad976e2492f35d1ee25dbd69ed7f7e8b3b980af Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sun, 20 Aug 2017 19:49:32 -0400
Subject: [PATCH 0946/1644] ARROW-1375: [C++] Remove dependency on msvc version
 for Snappy build

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #980 from MaxRis/ARROW-1375 and squashes the following commits:

f5e4156e [Max Risuhin] ARROW-1375: [C++] Remove dependency on msvc version for Snappy build
---
 cpp/cmake_modules/SnappyConfig.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/cmake_modules/SnappyConfig.h b/cpp/cmake_modules/SnappyConfig.h
index c998d1813aa7e..bd73a4c3cb521 100644
--- a/cpp/cmake_modules/SnappyConfig.h
+++ b/cpp/cmake_modules/SnappyConfig.h
@@ -29,7 +29,7 @@
 #ifndef SNAPPY_CONFIG_H
 #define SNAPPY_CONFIG_H 1
 
-#if defined(_MSC_VER) && (_MSC_VER <= 1910)
+#if defined(_MSC_VER)
 typedef __int64 ssize_t;
 #endif
 

From 4e0aa3cdd18b72aba9100bd88b87a755ebd95def Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 21 Aug 2017 11:35:29 -0400
Subject: [PATCH 0947/1644] ARROW-1387: [C++] Set up GPU leaf library, add unit
 test module for CUDA tests

This is an optional leaf library for users who want to use Arrow data on graphics cards. See parent JIRA ARROW-1055 for a roadmap for some basic GPU extensions

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #982 from wesm/arrow-gpu-lib and squashes the following commits:

f8c00eb3 [Wes McKinney] Remove cruft from CMakeLists.txt
e8f04a86 [Wes McKinney] Set up libarrow_gpu, add simple unit test that allocates memory on device

Change-Id: Ia1851ea6f30cb7cf3de422779d2d029e4ded672f
---
 cpp/CMakeLists.txt                  |   8 +++
 cpp/src/arrow/builder.h             |   2 +
 cpp/src/arrow/gpu/CMakeLists.txt    | 106 ++++++++++++++++++++++++++++
 cpp/src/arrow/gpu/arrow-gpu.pc.in   |  26 +++++++
 cpp/src/arrow/gpu/cuda-test.cc      |  45 ++++++++++++
 cpp/src/arrow/gpu/cuda_common.h     |  46 ++++++++++++
 cpp/src/arrow/gpu/cuda_memory.cc    |  65 +++++++++++++++++
 cpp/src/arrow/gpu/cuda_memory.h     |  78 ++++++++++++++++++++
 cpp/src/arrow/python/CMakeLists.txt |   6 ++
 cpp/src/arrow/type_traits.h         |   2 +-
 10 files changed, 383 insertions(+), 1 deletion(-)
 create mode 100644 cpp/src/arrow/gpu/CMakeLists.txt
 create mode 100644 cpp/src/arrow/gpu/arrow-gpu.pc.in
 create mode 100644 cpp/src/arrow/gpu/cuda-test.cc
 create mode 100644 cpp/src/arrow/gpu/cuda_common.h
 create mode 100644 cpp/src/arrow/gpu/cuda_memory.cc
 create mode 100644 cpp/src/arrow/gpu/cuda_memory.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 7d73d1ffff089..b55a9bb1d0256 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -98,6 +98,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow IPC extensions"
     ON)
 
+  option(ARROW_GPU
+    "Build the Arrow GPU extensions (requires CUDA installation)"
+    OFF)
+
   option(ARROW_JEMALLOC
     "Build the Arrow jemalloc-based allocator"
     OFF)
@@ -713,6 +717,10 @@ if (ARROW_IPC)
   add_dependencies(arrow_dependencies metadata_fbs)
 endif()
 
+if (ARROW_GPU)
+  add_subdirectory(src/arrow/gpu)
+endif()
+
 set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/buffer.cc
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 3b851f92c1726..687286d4eb88a 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -124,6 +124,8 @@ class ARROW_EXPORT ArrayBuilder {
   std::shared_ptr<DataType> type() const { return type_; }
 
  protected:
+  ArrayBuilder() {}
+
   std::shared_ptr<DataType> type_;
   MemoryPool* pool_;
 
diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt
new file mode 100644
index 0000000000000..97de4923f0180
--- /dev/null
+++ b/cpp/src/arrow/gpu/CMakeLists.txt
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function(ADD_ARROW_CUDA_TEST REL_TEST_NAME)
+  set(options)
+  set(single_value_args)
+  set(multi_value_args STATIC_LINK_LIBS)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
+    # This test has a corresponding .cc file, set it up as an executable.
+    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+    cuda_add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
+
+    if (ARG_STATIC_LINK_LIBS)
+      # Customize link libraries
+      target_link_libraries(${TEST_NAME} ${ARG_STATIC_LINK_LIBS})
+    else()
+      target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+    endif()
+    add_dependencies(unittest ${TEST_NAME})
+  else()
+    # No executable, just invoke the test (probably a script) directly.
+    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
+  endif()
+
+  if (ARROW_TEST_MEMCHECK)
+    SET_PROPERTY(TARGET ${TEST_NAME}
+      APPEND_STRING PROPERTY
+      COMPILE_FLAGS " -DARROW_VALGRIND")
+    add_test(${TEST_NAME}
+      bash -c "cd ${EXECUTABLE_OUTPUT_PATH}; valgrind --tool=memcheck --leak-check=full --leak-check-heuristics=stdstring --error-exitcode=1 ${TEST_PATH}")
+  elseif(MSVC)
+    add_test(${TEST_NAME} ${TEST_PATH})
+  else()
+    add_test(${TEST_NAME}
+      ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
+  endif()
+  set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
+endfunction()
+
+#######################################
+# arrow_gpu
+#######################################
+
+if (DEFINED ENV{CUDA_HOME})
+  set(CUDA_TOOLKIT_ROOT_DIR "$ENV{CUDA_HOME}")
+endif()
+
+find_package(CUDA REQUIRED)
+include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
+
+set(ARROW_GPU_SRCS
+  cuda_memory.cc
+)
+
+set(ARROW_GPU_SHARED_LINK_LIBS
+  arrow_shared
+)
+
+cuda_add_library(arrow_gpu SHARED
+  ${ARROW_GPU_SRCS}
+)
+
+install(FILES
+  cuda_common.h
+  cuda_memory.h
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
+
+# pkg-config support
+configure_file(arrow-gpu.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/arrow-gpu.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-gpu.pc"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
+
+if (ARROW_BUILD_TESTS)
+  set(ARROW_GPU_TEST_LINK_LIBS
+    ${ARROW_TEST_LINK_LIBS}
+    arrow_gpu)
+  ADD_ARROW_CUDA_TEST(cuda-test
+    STATIC_LINK_LIBS ${ARROW_GPU_TEST_LINK_LIBS})
+endif()
diff --git a/cpp/src/arrow/gpu/arrow-gpu.pc.in b/cpp/src/arrow/gpu/arrow-gpu.pc.in
new file mode 100644
index 0000000000000..3889d03b204ca
--- /dev/null
+++ b/cpp/src/arrow/gpu/arrow-gpu.pc.in
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+libdir=@CMAKE_INSTALL_FULL_LIBDIR@
+includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
+
+Name: Apache Arrow GPU
+Description: GPU integration library for Apache Arrow
+Version: @ARROW_VERSION@
+Requires: arrow
+Libs: -L${libdir} -larrow_gpu
+Cflags: -I${includedir}
diff --git a/cpp/src/arrow/gpu/cuda-test.cc b/cpp/src/arrow/gpu/cuda-test.cc
new file mode 100644
index 0000000000000..9dc8e1aff946a
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda-test.cc
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <limits>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+
+#include "arrow/gpu/cuda_memory.h"
+
+namespace arrow {
+namespace gpu {
+
+class TestCudaBuffer : public ::testing::Test {};
+
+TEST_F(TestCudaBuffer, Allocate) {
+  const int device = 0;
+
+  const int64_t kSize = 100;
+  std::shared_ptr<CudaBuffer> buffer;
+
+  ASSERT_OK(AllocateCudaBuffer(device, kSize, &buffer));
+  ASSERT_EQ(kSize, buffer->size());
+}
+
+}  // namespace gpu
+}  // namespace arrow
diff --git a/cpp/src/arrow/gpu/cuda_common.h b/cpp/src/arrow/gpu/cuda_common.h
new file mode 100644
index 0000000000000..75f41c9a41eab
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_common.h
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Non-public header
+
+#ifndef ARROW_GPU_CUDA_COMMON_H
+#define ARROW_GPU_CUDA_COMMON_H
+
+#include <cuda_runtime_api.h>
+
+namespace arrow {
+namespace gpu {
+
+#define CUDA_DCHECK(STMT) \
+  do {                    \
+    int ret = (STMT);     \
+    DCHECK_EQ(0, ret);    \
+    (void)ret;            \
+  } while (0)
+
+#define CUDA_RETURN_NOT_OK(STMT)                              \
+  do {                                                        \
+    cudaError_t ret = (STMT);                                 \
+    if (ret != cudaSuccess) {                                 \
+      return Status::IOError("Cuda API call failed: " #STMT); \
+    }                                                         \
+  } while (0)
+
+}  // namespace gpu
+}  // namespace arrow
+
+#endif  // ARROW_GPU_CUDA_COMMON_H
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
new file mode 100644
index 0000000000000..cf37bdae2f3ef
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/gpu/cuda_memory.h"
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+#include "arrow/gpu/cuda_common.h"
+
+namespace arrow {
+namespace gpu {
+
+CudaBuffer::~CudaBuffer() {
+  if (own_data_) {
+    CUDA_DCHECK(cudaFree(mutable_data_));
+  }
+}
+
+Status CudaBuffer::CopyHost(uint8_t* out) {
+  CUDA_RETURN_NOT_OK(cudaMemcpy(out, data_, size_, cudaMemcpyDeviceToHost));
+  return Status::OK();
+}
+
+Status AllocateCudaBuffer(int gpu_number, const int64_t size,
+                          std::shared_ptr<CudaBuffer>* out) {
+  CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number));
+  uint8_t* data = nullptr;
+  CUDA_RETURN_NOT_OK(
+      cudaMalloc(reinterpret_cast<void**>(&data), static_cast<size_t>(size)));
+  *out = std::make_shared<CudaBuffer>(data, size, gpu_number, true);
+  return Status::OK();
+}
+
+CudaHostBuffer::~CudaHostBuffer() { CUDA_DCHECK(cudaFreeHost(mutable_data_)); }
+
+Status AllocateCudaHostBuffer(const int gpu_number, const int64_t size,
+                              std::shared_ptr<CudaHostBuffer>* out) {
+  uint8_t* data = nullptr;
+  CUDA_RETURN_NOT_OK(
+      cudaMallocHost(reinterpret_cast<void**>(&data), static_cast<size_t>(size)));
+  *out = std::make_shared<CudaHostBuffer>(data, size);
+  return Status::OK();
+}
+
+}  // namespace gpu
+}  // namespace arrow
diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h
new file mode 100644
index 0000000000000..885373a26c943
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_memory.h
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_GPU_CUDA_MEMORY_H
+#define ARROW_GPU_CUDA_MEMORY_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+
+namespace arrow {
+namespace gpu {
+
+/// \brief An Arrow buffer located on a GPU device
+///
+/// Be careful using this in any Arrow code which may not be GPU-aware
+class ARROW_EXPORT CudaBuffer : public MutableBuffer {
+ public:
+  CudaBuffer(uint8_t* data, int64_t size, const int gpu_number, bool own_data = false)
+      : MutableBuffer(data, size), gpu_number_(gpu_number), own_data_(own_data) {}
+
+  ~CudaBuffer();
+
+  /// \brief Copy memory from GPU device to CPU host
+  /// \param[out] out a pre-allocated output buffer
+  /// \return Status
+  Status CopyHost(uint8_t* out);
+
+  int gpu_number() const { return gpu_number_; }
+
+ private:
+  const int gpu_number_;
+  bool own_data_;
+};
+
+/// \brief Device-accessible CPU memory created using cudaHostAlloc
+class ARROW_EXPORT CudaHostBuffer : public MutableBuffer {
+ public:
+  using MutableBuffer::MutableBuffer;
+  ~CudaHostBuffer();
+};
+
+/// \brief Allocate CUDA memory on a GPU device
+/// \param[in] gpu_number Device number to allocate
+/// \param[in] size number of bytes
+/// \param[out] out the allocated buffer
+/// \return Status
+ARROW_EXPORT
+Status AllocateCudaBuffer(const int gpu_number, const int64_t size,
+                          std::shared_ptr<CudaBuffer>* out);
+
+/// \brief Allocate CUDA-accessible memory on CPU host
+/// \param[in] size number of bytes
+/// \param[out] out the allocated buffer
+/// \return Status
+ARROW_EXPORT
+Status AllocateCudaHostBuffer(const int64_t size, std::shared_ptr<CudaHostBuffer>* out);
+
+}  // namespace gpu
+}  // namespace arrow
+
+#endif  // ARROW_GPU_CUDA_MEMORY_H
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index f2807b930a33c..84aad82e2a90e 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -8,6 +8,12 @@
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 #######################################
 # arrow_python
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index f05eb56718f5f..d424cc81ff92a 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -328,7 +328,7 @@ GET_ATTR(TypeClass, void);
   using TypeClass =                                                                 \
       typename std::conditional<std::is_base_of<DataType, T>::value, T,             \
                                 typename detail::GetAttr_TypeClass<T>::type>::type; \
-  using c_type = typename detail::GetAttr_c_type<TypeClass>::type;
+  using c_type = typename detail::GetAttr_c_type<TypeClass>::type
 
 template <typename T>
 struct IsUnsignedInt {

From 5303594bf3bfddfee14ca12e45ef32bcabb085c2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 21 Aug 2017 22:30:20 -0400
Subject: [PATCH 0948/1644] ARROW-1395: [C++/Python] Remove APIs deprecated
 from 0.5.0 onward

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #983 from wesm/ARROW-1395 and squashes the following commits:

c105a215 [Wes McKinney] Remove deprecated APIs from <= 0.4.0
---
 cpp/src/arrow/io/hdfs.h                   |  2 +-
 cpp/src/arrow/ipc/reader.h                |  7 ----
 cpp/src/arrow/ipc/writer.h                |  7 ----
 python/pyarrow/__init__.py                | 25 ++-----------
 python/pyarrow/ipc.py                     |  2 +-
 python/pyarrow/tests/test_deprecations.py | 45 +----------------------
 6 files changed, 7 insertions(+), 81 deletions(-)

diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 1507ca969cf5f..ca8895304b4d6 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -172,8 +172,8 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
   DISALLOW_COPY_AND_ASSIGN(HadoopFileSystem);
 };
 
-// 0.6.0
 #ifndef ARROW_NO_DEPRECATED_API
+/// \deprecated Since 0.6.0
 using HdfsClient = HadoopFileSystem;
 #endif
 
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index c0d3fb1f185f9..22d236a6e2ca6 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -194,13 +194,6 @@ Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64
 Status ARROW_EXPORT ReadTensor(int64_t offset, io::RandomAccessFile* file,
                                std::shared_ptr<Tensor>* out);
 
-/// Backwards-compatibility for Arrow < 0.4.0
-///
-#ifndef ARROW_NO_DEPRECATED_API
-using StreamReader = RecordBatchReader;
-using FileReader = RecordBatchFileReader;
-#endif
-
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index c28dfe0afbb11..2a31d2cbec100 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -177,13 +177,6 @@ Status ARROW_EXPORT WriteLargeRecordBatch(const RecordBatch& batch,
 Status ARROW_EXPORT WriteTensor(const Tensor& tensor, io::OutputStream* dst,
                                 int32_t* metadata_length, int64_t* body_length);
 
-/// Backwards-compatibility for Arrow < 0.4.0
-///
-#ifndef ARROW_NO_DEPRECATED_API
-using FileWriter = RecordBatchFileWriter;
-using StreamWriter = RecordBatchStreamWriter;
-#endif
-
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 4abf29edd2612..d00286de409b3 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -114,7 +114,8 @@
 def _plasma_store_entry_point():
     """Entry point for starting the plasma store.
 
-    This can be used by invoking e. g. ``plasma_store -s /tmp/plasma -m 1000000000``
+    This can be used by invoking e.g.
+    ``plasma_store -s /tmp/plasma -m 1000000000``
     from the command line and will start the plasma_store executable with the
     given arguments.
     """
@@ -127,30 +128,10 @@ def _plasma_store_entry_point():
     process.wait()
 
 # ----------------------------------------------------------------------
-# 0.4.0 deprecations
+# Deprecations
 
 from pyarrow.util import _deprecate_class
 
-FileReader = _deprecate_class('FileReader',
-                              'RecordBatchFileReader',
-                              RecordBatchFileReader, '0.5.0')
-
-FileWriter = _deprecate_class('FileWriter',
-                              'RecordBatchFileWriter',
-                              RecordBatchFileWriter, '0.5.0')
-
-StreamReader = _deprecate_class('StreamReader',
-                                'RecordBatchStreamReader',
-                                RecordBatchStreamReader, '0.5.0')
-
-StreamWriter = _deprecate_class('StreamWriter',
-                                'RecordBatchStreamWriter',
-                                RecordBatchStreamWriter, '0.5.0')
-
-InMemoryOutputStream = _deprecate_class('InMemoryOutputStream',
-                                        'BufferOutputStream',
-                                        BufferOutputStream, '0.5.0')
-
 # Backwards compatibility with pyarrow < 0.6.0
 HdfsClient = _deprecate_class('HdfsClient', 'pyarrow.hdfs.connect',
                               hdfs.connect, '0.6.0')
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index f86312867a9f8..d527722295d1a 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -155,7 +155,7 @@ def serialize_pandas(df):
         An object compatible with the buffer protocol
     """
     batch = pa.RecordBatch.from_pandas(df)
-    sink = pa.InMemoryOutputStream()
+    sink = pa.BufferOutputStream()
     writer = pa.RecordBatchStreamWriter(sink, batch.schema)
     writer.write_batch(batch)
     writer.close()
diff --git a/python/pyarrow/tests/test_deprecations.py b/python/pyarrow/tests/test_deprecations.py
index 62b96663833a7..b165289377aab 100644
--- a/python/pyarrow/tests/test_deprecations.py
+++ b/python/pyarrow/tests/test_deprecations.py
@@ -17,48 +17,7 @@
 
 # Check that various deprecation warnings are raised
 
+# flake8: noqa
+
 import pyarrow as pa
 import pytest
-
-
-def test_inmemory_output_stream():
-    with pytest.warns(FutureWarning):
-        stream = pa.InMemoryOutputStream()
-        assert isinstance(stream, pa.BufferOutputStream)
-
-
-def test_file_reader_writer():
-    data = [
-        pa.array([1, 2, 3, 4]),
-        pa.array(['foo', 'bar', 'baz', None]),
-        pa.array([True, None, False, True])
-    ]
-    batch = pa.RecordBatch.from_arrays(data, ['f0', 'f1', 'f2'])
-
-    sink = pa.BufferOutputStream()
-
-    with pytest.warns(FutureWarning):
-        stream_writer = pa.StreamWriter(sink, batch.schema)
-        assert isinstance(stream_writer, pa.RecordBatchStreamWriter)
-
-    sink2 = pa.BufferOutputStream()
-    with pytest.warns(FutureWarning):
-        file_writer = pa.FileWriter(sink2, batch.schema)
-        assert isinstance(file_writer, pa.RecordBatchFileWriter)
-
-    file_writer.write_batch(batch)
-    stream_writer.write_batch(batch)
-
-    file_writer.close()
-    stream_writer.close()
-
-    buf = sink.get_result()
-    buf2 = sink2.get_result()
-
-    with pytest.warns(FutureWarning):
-        stream_reader = pa.StreamReader(buf)
-        assert isinstance(stream_reader, pa.RecordBatchStreamReader)
-
-    with pytest.warns(FutureWarning):
-        file_reader = pa.FileReader(buf2)
-        assert isinstance(file_reader, pa.RecordBatchFileReader)

From 3c70ff1d0a229c8be0e824abf1964ad9648abc84 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 22 Aug 2017 09:17:49 -0400
Subject: [PATCH 0949/1644] ARROW-1384: [C++] Add SerializeRecordBatch API for
 writing a record batch as an IPC message to a new buffer

There's also a little bit of API scrubbing as I went through this code.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #984 from wesm/ARROW-1384 and squashes the following commits:

a3996fed [Wes McKinney] Add DCHECK to catch unequal schemas
2952cfb4 [Wes McKinney] Add SerializeRecordBatch API, various API scrubbing, make some integer arguments const
---
 cpp/src/arrow/buffer.cc                  | 25 ++++-----
 cpp/src/arrow/buffer.h                   | 53 +++++++++++-------
 cpp/src/arrow/ipc/ipc-read-write-test.cc | 29 +++-------
 cpp/src/arrow/ipc/metadata.cc            | 11 ++--
 cpp/src/arrow/ipc/metadata.h             | 23 ++++----
 cpp/src/arrow/ipc/reader.h               | 32 +++++------
 cpp/src/arrow/ipc/writer.cc              | 29 ++++++++++
 cpp/src/arrow/ipc/writer.h               | 68 +++++++++++++++++-------
 cpp/src/arrow/python/python_to_arrow.cc  | 10 ++--
 9 files changed, 168 insertions(+), 112 deletions(-)

diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index b9c5897f8a228..9e8eb8bd4b6bc 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -27,7 +27,7 @@
 
 namespace arrow {
 
-Status Buffer::Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
+Status Buffer::Copy(const int64_t start, const int64_t nbytes, MemoryPool* pool,
                     std::shared_ptr<Buffer>* out) const {
   // Sanity checks
   DCHECK_LT(start, size_);
@@ -42,11 +42,12 @@ Status Buffer::Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
   return Status::OK();
 }
 
-Status Buffer::Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const {
+Status Buffer::Copy(const int64_t start, const int64_t nbytes,
+                    std::shared_ptr<Buffer>* out) const {
   return Copy(start, nbytes, default_memory_pool(), out);
 }
 
-bool Buffer::Equals(const Buffer& other, int64_t nbytes) const {
+bool Buffer::Equals(const Buffer& other, const int64_t nbytes) const {
   return this == &other || (size_ >= nbytes && other.size_ >= nbytes &&
                             (data_ == other.data_ ||
                              !memcmp(data_, other.data_, static_cast<size_t>(nbytes))));
@@ -71,10 +72,10 @@ PoolBuffer::~PoolBuffer() {
   }
 }
 
-Status PoolBuffer::Reserve(int64_t new_capacity) {
-  if (!mutable_data_ || new_capacity > capacity_) {
+Status PoolBuffer::Reserve(const int64_t capacity) {
+  if (!mutable_data_ || capacity > capacity_) {
     uint8_t* new_data;
-    new_capacity = BitUtil::RoundUpToMultipleOf64(new_capacity);
+    int64_t new_capacity = BitUtil::RoundUpToMultipleOf64(capacity);
     if (mutable_data_) {
       RETURN_NOT_OK(pool_->Reallocate(capacity_, new_capacity, &mutable_data_));
     } else {
@@ -87,7 +88,7 @@ Status PoolBuffer::Reserve(int64_t new_capacity) {
   return Status::OK();
 }
 
-Status PoolBuffer::Resize(int64_t new_size, bool shrink_to_fit) {
+Status PoolBuffer::Resize(const int64_t new_size, bool shrink_to_fit) {
   if (!shrink_to_fit || (new_size > size_)) {
     RETURN_NOT_OK(Reserve(new_size));
   } else {
@@ -113,18 +114,18 @@ Status PoolBuffer::Resize(int64_t new_size, bool shrink_to_fit) {
 }
 
 std::shared_ptr<Buffer> SliceMutableBuffer(const std::shared_ptr<Buffer>& buffer,
-                                           int64_t offset, int64_t length) {
+                                           const int64_t offset, const int64_t length) {
   return std::make_shared<MutableBuffer>(buffer, offset, length);
 }
 
-MutableBuffer::MutableBuffer(const std::shared_ptr<Buffer>& parent, int64_t offset,
-                             int64_t size)
+MutableBuffer::MutableBuffer(const std::shared_ptr<Buffer>& parent, const int64_t offset,
+                             const int64_t size)
     : MutableBuffer(parent->mutable_data() + offset, size) {
   DCHECK(parent->is_mutable()) << "Must pass mutable buffer";
   parent_ = parent;
 }
 
-Status AllocateBuffer(MemoryPool* pool, int64_t size,
+Status AllocateBuffer(MemoryPool* pool, const int64_t size,
                       std::shared_ptr<MutableBuffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(buffer->Resize(size));
@@ -132,7 +133,7 @@ Status AllocateBuffer(MemoryPool* pool, int64_t size,
   return Status::OK();
 }
 
-Status AllocateResizableBuffer(MemoryPool* pool, int64_t size,
+Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
                                std::shared_ptr<ResizableBuffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(buffer->Resize(size));
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 5d050b77f77a8..f8f0b3df47501 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -59,7 +59,7 @@ class ARROW_EXPORT Buffer {
   /// This method makes no assertions about alignment or padding of the buffer but
   /// in general we expected buffers to be aligned and padded to 64 bytes.  In the future
   /// we might add utility methods to help determine if a buffer satisfies this contract.
-  Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size)
+  Buffer(const std::shared_ptr<Buffer>& parent, const int64_t offset, const int64_t size)
       : Buffer(parent->data() + offset, size) {
     parent_ = parent;
   }
@@ -72,11 +72,12 @@ class ARROW_EXPORT Buffer {
   bool Equals(const Buffer& other) const;
 
   /// Copy a section of the buffer into a new Buffer.
-  Status Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
+  Status Copy(const int64_t start, const int64_t nbytes, MemoryPool* pool,
               std::shared_ptr<Buffer>* out) const;
 
   /// Copy a section of the buffer using the default memory pool into a new Buffer.
-  Status Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const;
+  Status Copy(const int64_t start, const int64_t nbytes,
+              std::shared_ptr<Buffer>* out) const;
 
   int64_t capacity() const { return capacity_; }
   const uint8_t* data() const { return data_; }
@@ -114,24 +115,27 @@ static inline std::shared_ptr<Buffer> GetBufferFromString(const std::string& str
 /// Construct a view on passed buffer at the indicated offset and length. This
 /// function cannot fail and does not error checking (except in debug builds)
 static inline std::shared_ptr<Buffer> SliceBuffer(const std::shared_ptr<Buffer>& buffer,
-                                                  int64_t offset, int64_t length) {
+                                                  const int64_t offset,
+                                                  const int64_t length) {
   return std::make_shared<Buffer>(buffer, offset, length);
 }
 
 /// Construct a mutable buffer slice. If the parent buffer is not mutable, this
 /// will abort in debug builds
-std::shared_ptr<Buffer> ARROW_EXPORT
-SliceMutableBuffer(const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
+ARROW_EXPORT
+std::shared_ptr<Buffer> SliceMutableBuffer(const std::shared_ptr<Buffer>& buffer,
+                                           const int64_t offset, const int64_t length);
 
 /// A Buffer whose contents can be mutated. May or may not own its data.
 class ARROW_EXPORT MutableBuffer : public Buffer {
  public:
-  MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
+  MutableBuffer(uint8_t* data, const int64_t size) : Buffer(data, size) {
     mutable_data_ = data;
     is_mutable_ = true;
   }
 
-  MutableBuffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
+  MutableBuffer(const std::shared_ptr<Buffer>& parent, const int64_t offset,
+                const int64_t size);
 
  protected:
   MutableBuffer() : Buffer(nullptr, 0) {}
@@ -145,20 +149,20 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
   ///
   /// @param shrink_to_fit On deactivating this option, the capacity of the Buffer won't
   /// decrease.
-  virtual Status Resize(int64_t new_size, bool shrink_to_fit = true) = 0;
+  virtual Status Resize(const int64_t new_size, bool shrink_to_fit = true) = 0;
 
   /// Ensure that buffer has enough memory allocated to fit the indicated
   /// capacity (and meets the 64 byte padding requirement in Layout.md).
   /// It does not change buffer's reported size.
-  virtual Status Reserve(int64_t new_capacity) = 0;
+  virtual Status Reserve(const int64_t new_capacity) = 0;
 
   template <class T>
-  Status TypedResize(int64_t new_nb_elements, bool shrink_to_fit = true) {
+  Status TypedResize(const int64_t new_nb_elements, bool shrink_to_fit = true) {
     return Resize(sizeof(T) * new_nb_elements, shrink_to_fit);
   }
 
   template <class T>
-  Status TypedReserve(int64_t new_nb_elements) {
+  Status TypedReserve(const int64_t new_nb_elements) {
     return Reserve(sizeof(T) * new_nb_elements);
   }
 
@@ -172,8 +176,8 @@ class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
   explicit PoolBuffer(MemoryPool* pool = nullptr);
   virtual ~PoolBuffer();
 
-  Status Resize(int64_t new_size, bool shrink_to_fit = true) override;
-  Status Reserve(int64_t new_capacity) override;
+  Status Resize(const int64_t new_size, bool shrink_to_fit = true) override;
+  Status Reserve(const int64_t new_capacity) override;
 
  private:
   MemoryPool* pool_;
@@ -185,7 +189,7 @@ class ARROW_EXPORT BufferBuilder {
       : pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
 
   /// Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
-  Status Resize(int64_t elements) {
+  Status Resize(const int64_t elements) {
     // Resize(0) is a no-op
     if (elements == 0) {
       return Status::OK();
@@ -213,7 +217,7 @@ class ARROW_EXPORT BufferBuilder {
   }
 
   // Advance pointer and zero out memory
-  Status Advance(int64_t length) {
+  Status Advance(const int64_t length) {
     if (capacity_ < length + size_) {
       int64_t new_capacity = BitUtil::NextPower2(length + size_);
       RETURN_NOT_OK(Resize(new_capacity));
@@ -299,11 +303,20 @@ class ARROW_EXPORT TypedBufferBuilder : public BufferBuilder {
 /// \param[out] out the allocated buffer with padding
 ///
 /// \return Status message
-Status ARROW_EXPORT AllocateBuffer(MemoryPool* pool, int64_t size,
-                                   std::shared_ptr<MutableBuffer>* out);
+ARROW_EXPORT
+Status AllocateBuffer(MemoryPool* pool, const int64_t size,
+                      std::shared_ptr<MutableBuffer>* out);
 
-Status ARROW_EXPORT AllocateResizableBuffer(MemoryPool* pool, int64_t size,
-                                            std::shared_ptr<ResizableBuffer>* out);
+/// Allocate resizeable buffer from a memory pool
+///
+/// \param[in] pool a memory pool
+/// \param[in] size size of buffer to allocate
+/// \param[out] out the allocated buffer
+///
+/// \return Status message
+ARROW_EXPORT
+Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
+                               std::shared_ptr<ResizableBuffer>* out);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 045296163ea11..5aaff5f569154 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -137,27 +137,13 @@ static int g_file_number = 0;
 
 class IpcTestFixture : public io::MemoryMapFixture {
  public:
-  Status DoStandardRoundTrip(const RecordBatch& batch, bool zero_data,
+  Status DoStandardRoundTrip(const RecordBatch& batch,
                              std::shared_ptr<RecordBatch>* batch_result) {
-    int32_t metadata_length;
-    int64_t body_length;
-
-    const int64_t buffer_offset = 0;
-
-    if (zero_data) {
-      RETURN_NOT_OK(ZeroMemoryMap(mmap_.get()));
-    }
-    RETURN_NOT_OK(mmap_->Seek(0));
-
-    RETURN_NOT_OK(WriteRecordBatch(batch, buffer_offset, mmap_.get(), &metadata_length,
-                                   &body_length, pool_));
+    std::shared_ptr<Buffer> serialized_batch;
+    RETURN_NOT_OK(SerializeRecordBatch(batch, pool_, &serialized_batch));
 
-    std::unique_ptr<Message> message;
-    RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
-
-    io::BufferReader buffer_reader(message->body());
-    return ReadRecordBatch(*message->metadata(), batch.schema(), &buffer_reader,
-                           batch_result);
+    io::BufferReader buf_reader(serialized_batch);
+    return ReadRecordBatch(batch.schema(), 0, &buf_reader, batch_result);
   }
 
   Status DoLargeRoundTrip(const RecordBatch& batch, bool zero_data,
@@ -197,7 +183,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
     ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(buffer_size, ss.str(), &mmap_));
 
     std::shared_ptr<RecordBatch> result;
-    ASSERT_OK(DoStandardRoundTrip(batch, true, &result));
+    ASSERT_OK(DoStandardRoundTrip(batch, &result));
     CheckReadResult(*result, batch);
 
     ASSERT_OK(DoLargeRoundTrip(batch, true, &result));
@@ -657,9 +643,6 @@ TEST_F(TestIpcRoundTrip, LargeRecordBatch) {
   CheckReadResult(*result, batch);
 
   ASSERT_EQ(length, result->num_rows());
-
-  // Fails if we try to write this with the normal code path
-  ASSERT_RAISES(Invalid, DoStandardRoundTrip(batch, false, &result));
 }
 
 void CheckBatchDictionaries(const RecordBatch& batch) {
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index c9534217e927c..a31b9668da3c2 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -703,7 +703,7 @@ static Status MakeRecordBatch(FBB& fbb, int64_t length, int64_t body_length,
   return Status::OK();
 }
 
-Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
+Status WriteRecordBatchMessage(const int64_t length, const int64_t body_length,
                                const std::vector<FieldMetadata>& nodes,
                                const std::vector<BufferMetadata>& buffers,
                                std::shared_ptr<Buffer>* out) {
@@ -714,7 +714,7 @@ Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
                         body_length, out);
 }
 
-Status WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset,
+Status WriteTensorMessage(const Tensor& tensor, const int64_t buffer_start_offset,
                           std::shared_ptr<Buffer>* out) {
   using TensorDimOffset = flatbuffers::Offset<flatbuf::TensorDim>;
   using TensorOffset = flatbuffers::Offset<flatbuf::Tensor>;
@@ -743,7 +743,8 @@ Status WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset,
                         body_length, out);
 }
 
-Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
+Status WriteDictionaryMessage(const int64_t id, const int64_t length,
+                              const int64_t body_length,
                               const std::vector<FieldMetadata>& nodes,
                               const std::vector<BufferMetadata>& buffers,
                               std::shared_ptr<Buffer>* out) {
@@ -1106,8 +1107,8 @@ static Status ReadFullMessage(const std::shared_ptr<Buffer>& metadata,
   return Message::Open(metadata, body, message);
 }
 
-Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
-                   std::unique_ptr<Message>* message) {
+Status ReadMessage(const int64_t offset, const int32_t metadata_length,
+                   io::RandomAccessFile* file, std::unique_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
 
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 90e4defd6a300..81716ae467baf 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -244,9 +244,9 @@ class ARROW_EXPORT InputStreamMessageReader : public MessageReader {
 /// \param[in] file the seekable file interface to read from
 /// \param[out] message the message read
 /// \return Status success or failure
-Status ARROW_EXPORT ReadMessage(int64_t offset, int32_t metadata_length,
-                                io::RandomAccessFile* file,
-                                std::unique_ptr<Message>* message);
+ARROW_EXPORT
+Status ReadMessage(const int64_t offset, const int32_t metadata_length,
+                   io::RandomAccessFile* file, std::unique_ptr<Message>* message);
 
 /// \brief Read encapulated RPC message (metadata and body) from InputStream
 ///
@@ -274,15 +274,18 @@ Status ARROW_EXPORT WriteSchemaMessage(const Schema& schema,
                                        DictionaryMemo* dictionary_memo,
                                        std::shared_ptr<Buffer>* out);
 
-Status ARROW_EXPORT WriteRecordBatchMessage(int64_t length, int64_t body_length,
-                                            const std::vector<FieldMetadata>& nodes,
-                                            const std::vector<BufferMetadata>& buffers,
-                                            std::shared_ptr<Buffer>* out);
+ARROW_EXPORT
+Status WriteRecordBatchMessage(const int64_t length, const int64_t body_length,
+                               const std::vector<FieldMetadata>& nodes,
+                               const std::vector<BufferMetadata>& buffers,
+                               std::shared_ptr<Buffer>* out);
 
-Status ARROW_EXPORT WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset,
-                                       std::shared_ptr<Buffer>* out);
+ARROW_EXPORT
+Status WriteTensorMessage(const Tensor& tensor, const int64_t buffer_start_offset,
+                          std::shared_ptr<Buffer>* out);
 
-Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
+Status WriteDictionaryMessage(const int64_t id, const int64_t length,
+                              const int64_t body_length,
                               const std::vector<FieldMetadata>& nodes,
                               const std::vector<BufferMetadata>& buffers,
                               std::shared_ptr<Buffer>* out);
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 22d236a6e2ca6..4cffb18488047 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -148,10 +148,9 @@ class ARROW_EXPORT RecordBatchFileReader {
 /// \param(in) schema the record batch schema
 /// \param(in) file a random access file
 /// \param(out) out the read record batch
-Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
-                                    const std::shared_ptr<Schema>& schema,
-                                    io::RandomAccessFile* file,
-                                    std::shared_ptr<RecordBatch>* out);
+ARROW_EXPORT
+Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
+                       io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
 /// \brief Read record batch from fully encapulated Message
 ///
@@ -159,9 +158,9 @@ Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
 /// \param[in] schema
 /// \param[out] out the resulting RecordBatch
 /// \return Status
-Status ARROW_EXPORT ReadRecordBatch(const Message& message,
-                                    const std::shared_ptr<Schema>& schema,
-                                    std::shared_ptr<RecordBatch>* out);
+ARROW_EXPORT
+Status ReadRecordBatch(const Message& message, const std::shared_ptr<Schema>& schema,
+                       std::shared_ptr<RecordBatch>* out);
 
 /// Read record batch from file given metadata and schema
 ///
@@ -170,10 +169,10 @@ Status ARROW_EXPORT ReadRecordBatch(const Message& message,
 /// \param(in) file a random access file
 /// \param(in) max_recursion_depth the maximum permitted nesting depth
 /// \param(out) out the read record batch
-Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
-                                    const std::shared_ptr<Schema>& schema,
-                                    int max_recursion_depth, io::RandomAccessFile* file,
-                                    std::shared_ptr<RecordBatch>* out);
+ARROW_EXPORT
+Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
+                       int max_recursion_depth, io::RandomAccessFile* file,
+                       std::shared_ptr<RecordBatch>* out);
 
 /// Read record batch as encapsulated IPC message with metadata size prefix and
 /// header
@@ -182,17 +181,18 @@ Status ARROW_EXPORT ReadRecordBatch(const Buffer& metadata,
 /// \param(in) offset the file location of the start of the message
 /// \param(in) file the file where the batch is located
 /// \param(out) out the read record batch
-Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
-                                    io::RandomAccessFile* file,
-                                    std::shared_ptr<RecordBatch>* out);
+ARROW_EXPORT
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
+                       io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
 /// EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
 ///
 /// \param(in) offset the file location of the start of the message
 /// \param(in) file the file where the batch is located
 /// \param(out) out the read tensor
-Status ARROW_EXPORT ReadTensor(int64_t offset, io::RandomAccessFile* file,
-                               std::shared_ptr<Tensor>* out);
+ARROW_EXPORT
+Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
+                  std::shared_ptr<Tensor>* out);
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index bc07dc659f601..9492364413e05 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -542,6 +542,35 @@ Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
   return writer.Write(batch, dst, metadata_length, body_length);
 }
 
+Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
+                            std::shared_ptr<Buffer>* out) {
+  int64_t size = 0;
+  RETURN_NOT_OK(GetRecordBatchSize(batch, &size));
+  std::shared_ptr<MutableBuffer> buffer;
+  RETURN_NOT_OK(AllocateBuffer(pool, size, &buffer));
+
+  io::FixedSizeBufferWriter stream(buffer);
+  int32_t metadata_length = 0;
+  int64_t body_length = 0;
+  RETURN_NOT_OK(WriteRecordBatch(batch, 0, &stream, &metadata_length, &body_length, pool,
+                                 kMaxNestingDepth, true));
+  *out = buffer;
+  return Status::OK();
+}
+
+Status WriteRecordBatchStream(const std::vector<std::shared_ptr<RecordBatch>>& batches,
+                              io::OutputStream* dst) {
+  std::shared_ptr<RecordBatchStreamWriter> writer;
+  RETURN_NOT_OK(RecordBatchStreamWriter::Open(dst, batches[0]->schema(), &writer));
+  for (const auto& batch : batches) {
+    // allow sizes > INT32_MAX
+    DCHECK(batch->schema()->Equals(*batches[0]->schema())) << "Schemas unequal";
+    RETURN_NOT_OK(writer->WriteRecordBatch(*batch, true));
+  }
+  RETURN_NOT_OK(writer->Close());
+  return Status::OK();
+}
+
 Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
                              io::OutputStream* dst, int32_t* metadata_length,
                              int64_t* body_length, MemoryPool* pool) {
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 2a31d2cbec100..79ea7c6664bbd 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -21,6 +21,7 @@
 #define ARROW_IPC_WRITER_H
 
 #include <cstdint>
+#include <functional>
 #include <memory>
 #include <vector>
 
@@ -144,38 +145,67 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
 /// including padding to a 64-byte boundary
 /// \param(out) body_length: the size of the contiguous buffer block plus
 /// padding bytes
-Status ARROW_EXPORT WriteRecordBatch(const RecordBatch& batch,
-                                     int64_t buffer_start_offset, io::OutputStream* dst,
-                                     int32_t* metadata_length, int64_t* body_length,
-                                     MemoryPool* pool,
-                                     int max_recursion_depth = kMaxNestingDepth,
-                                     bool allow_64bit = false);
-
-// Write Array as a DictionaryBatch message
+/// \return Status
+///
+/// Low-level API
+ARROW_EXPORT
+Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+                        io::OutputStream* dst, int32_t* metadata_length,
+                        int64_t* body_length, MemoryPool* pool,
+                        int max_recursion_depth = kMaxNestingDepth,
+                        bool allow_64bit = false);
+
+/// \brief Write dictionary message to output stream
+///
+/// Low-level API
+ARROW_EXPORT
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
                        int64_t buffer_start_offset, io::OutputStream* dst,
                        int32_t* metadata_length, int64_t* body_length, MemoryPool* pool);
 
+/// \brief Serialize record batch as encapsulated IPC message in a new buffer
+///
+/// \param[in] batch the record batch
+/// \param[in] pool a MemoryPool to allocate memory from
+/// \param[out] out the serialized message
+/// \return Status
+ARROW_EXPORT
+Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
+                            std::shared_ptr<Buffer>* out);
+
+/// \brief Write multiple record batches to OutputStream
+/// \param[in] batches a vector of batches. Must all have same schema
+/// \param[out] dst an OutputStream
+ARROW_EXPORT
+Status WriteRecordBatchStream(const std::vector<std::shared_ptr<RecordBatch>>& batches,
+                              io::OutputStream* dst);
+
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the record batch. This involves generating the complete serialized
 // Flatbuffers metadata.
-Status ARROW_EXPORT GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
+ARROW_EXPORT
+Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 
 // Compute the precise number of bytes needed in a contiguous memory segment to
 // write the tensor including metadata, padding, and data
-Status ARROW_EXPORT GetTensorSize(const Tensor& tensor, int64_t* size);
-
-/// EXPERIMENTAL: Write RecordBatch allowing lengths over INT32_MAX. This data
-/// may not be readable by all Arrow implementations
-Status ARROW_EXPORT WriteLargeRecordBatch(const RecordBatch& batch,
-                                          int64_t buffer_start_offset,
-                                          io::OutputStream* dst, int32_t* metadata_length,
-                                          int64_t* body_length, MemoryPool* pool);
+ARROW_EXPORT
+Status GetTensorSize(const Tensor& tensor, int64_t* size);
 
 /// EXPERIMENTAL: Write arrow::Tensor as a contiguous message
 /// <metadata size><metadata><tensor data>
-Status ARROW_EXPORT WriteTensor(const Tensor& tensor, io::OutputStream* dst,
-                                int32_t* metadata_length, int64_t* body_length);
+ARROW_EXPORT
+Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
+                   int64_t* body_length);
+
+#ifndef ARROW_NO_DEPRECATED_API
+/// EXPERIMENTAL: Write RecordBatch allowing lengths over INT32_MAX. This data
+/// may not be readable by all Arrow implementations
+/// \deprecated Since 0.7.0
+ARROW_EXPORT
+Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+                             io::OutputStream* dst, int32_t* metadata_length,
+                             int64_t* body_length, MemoryPool* pool);
+#endif
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 47d8ef60c4b48..a1ccd994e6caa 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -634,15 +634,11 @@ Status SerializeObject(PyObject* sequence, SerializedPyObject* out) {
 
 Status WriteSerializedObject(const SerializedPyObject& obj, io::OutputStream* dst) {
   int32_t num_tensors = static_cast<int32_t>(obj.tensors.size());
-  std::shared_ptr<ipc::RecordBatchStreamWriter> writer;
-  int32_t metadata_length;
-  int64_t body_length;
-
   RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&num_tensors), sizeof(int32_t)));
-  RETURN_NOT_OK(ipc::RecordBatchStreamWriter::Open(dst, obj.batch->schema(), &writer));
-  RETURN_NOT_OK(writer->WriteRecordBatch(*obj.batch));
-  RETURN_NOT_OK(writer->Close());
+  RETURN_NOT_OK(ipc::WriteRecordBatchStream({obj.batch}, dst));
 
+  int32_t metadata_length;
+  int64_t body_length;
   for (const auto& tensor : obj.tensors) {
     RETURN_NOT_OK(ipc::WriteTensor(*tensor, dst, &metadata_length, &body_length));
   }

From 2c3a5f4e9dbe9d89545e6033a43b273ac1499870 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 23 Aug 2017 09:41:58 -0400
Subject: [PATCH 0950/1644] ARROW-1392: [C++] Add GPU IO interfaces for CUDA

This makes it easy to write from host to device and read from device to host. We also need a zero-copy device reader for IPC purposes (where we don't want to move any data to the host), can do that in a subsequent patch.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #985 from wesm/ARROW-1392 and squashes the following commits:

ae24cb53 [Wes McKinney] Add section to C++ README about building libarrow_gpu
229a2687 [Wes McKinney] Refactor CudaBufferReader to return zero-copy device pointers. Add unit tests
415157ac [Wes McKinney] Make Tell overrides in arrow-glib const
5daa59eb [Wes McKinney] Add cuda-benchmark module
1cf1196c [Wes McKinney] Test CudaBuffer::CopyFromHost
a2708f2c [Wes McKinney] Implement IO interfaces for CUDA buffers
---
 c_glib/arrow-glib/input-stream.cpp  |   2 +-
 c_glib/arrow-glib/output-stream.cpp |   2 +-
 cpp/README.md                       |  19 +++-
 cpp/apidoc/Doxyfile                 |   2 +-
 cpp/src/arrow/builder.h             |  27 ++---
 cpp/src/arrow/gpu/CMakeLists.txt    |  34 +++++-
 cpp/src/arrow/gpu/cuda-benchmark.cc |  93 +++++++++++++++
 cpp/src/arrow/gpu/cuda-test.cc      | 169 +++++++++++++++++++++++++++-
 cpp/src/arrow/gpu/cuda_common.h     |  17 ++-
 cpp/src/arrow/gpu/cuda_memory.cc    | 116 ++++++++++++++++++-
 cpp/src/arrow/gpu/cuda_memory.h     |  89 ++++++++++++++-
 cpp/src/arrow/io/file.cc            |   6 +-
 cpp/src/arrow/io/file.h             |   6 +-
 cpp/src/arrow/io/hdfs.cc            |   4 +-
 cpp/src/arrow/io/hdfs.h             |   4 +-
 cpp/src/arrow/io/interfaces.h       |   4 +-
 cpp/src/arrow/io/memory.cc          |  16 ++-
 cpp/src/arrow/io/memory.h           |  13 +--
 cpp/src/arrow/memory_pool.h         |   6 +
 cpp/src/arrow/python/io.cc          |   4 +-
 cpp/src/arrow/python/io.h           |   4 +-
 cpp/src/arrow/util/io-util.h        |  16 +--
 22 files changed, 574 insertions(+), 79 deletions(-)
 create mode 100644 cpp/src/arrow/gpu/cuda-benchmark.cc

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 645ce8aa2295c..db3efd4b0b24b 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -405,7 +405,7 @@ namespace garrow {
       }
     }
 
-    arrow::Status Tell(int64_t *position) override {
+    arrow::Status Tell(int64_t *position) const override {
       if (!G_IS_SEEKABLE(input_stream_)) {
         std::string message("[gio-input-stream][tell] "
                             "not seekable input stream: <");
diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index 7f37864028651..739992fb62b0e 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -312,7 +312,7 @@ namespace garrow {
       }
     }
 
-    arrow::Status Tell(int64_t *position) override {
+    arrow::Status Tell(int64_t *position) const override {
       if (!G_IS_SEEKABLE(output_stream_)) {
         std::string message("[gio-output-stream][tell] "
                             "not seekable output stream: <");
diff --git a/cpp/README.md b/cpp/README.md
index 0228faf7349c5..c05e03fece374 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -101,16 +101,27 @@ all of these variables. Note that `ARROW_BUILD_TOOLCHAIN` will not set
 `BOOST_ROOT`, so if you have custom Boost installation, you must set this
 environment variable separately.
 
-### Building Python integration library
+### Building Python integration library (optional)
 
-The `arrow_python` shared library can be built by passing `-DARROW_PYTHON=on`
-to CMake. This must be installed or in your library load path to be able to
-build pyarrow, the Arrow Python bindings.
+The optional `arrow_python` shared library can be built by passing
+`-DARROW_PYTHON=on` to CMake. This must be installed or in your library load
+path to be able to build pyarrow, the Arrow Python bindings.
 
 The Python library must be built against the same Python version for which you
 are building pyarrow, e.g. Python 2.7 or Python 3.6. NumPy must also be
 installed.
 
+### Building GPU extension library (optional)
+
+The optional `arrow_gpu` shared library can be built by passing
+`-DARROW_GPU=on`. This requires a CUDA installation to build, and to use many
+of the functions you must have a functioning GPU. Currently only CUDA
+functionality is supported, though if there is demand we can also add OpenCL
+interfaces in this library as needed.
+
+The CUDA toolchain used to build the library can be customized by using the
+`$CUDA_HOME` environment variable.
+
 ### API documentation
 
 To generate the (html) API documentation, run the following command in the apidoc
diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile
index 94156d55801f1..fadb6d5be4af2 100644
--- a/cpp/apidoc/Doxyfile
+++ b/cpp/apidoc/Doxyfile
@@ -2084,7 +2084,7 @@ PREDEFINED = __attribute__(x)= \
 # definition found in the source code.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_AS_DEFINED = ARROW_MEMORY_POOL_ARG
+EXPAND_AS_DEFINED = ARROW_MEMORY_POOL_DEFAULT
 
 # If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
 # remove all references to function-like macros that are alone on a line, have
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 687286d4eb88a..4e3cd92da9beb 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -54,12 +54,6 @@ struct Decimal;
 
 static constexpr int64_t kMinBuilderCapacity = 1 << 5;
 
-#ifdef ARROW_NO_DEFAULT_MEMORY_POOL
-#define ARROW_MEMORY_POOL_ARG pool
-#else
-#define ARROW_MEMORY_POOL_ARG pool = default_memory_pool()
-#endif
-
 /// Base class for all data array builders.
 //
 /// This class provides a facilities for incrementally building the null bitmap
@@ -167,7 +161,8 @@ class ARROW_EXPORT ArrayBuilder {
 
 class ARROW_EXPORT NullBuilder : public ArrayBuilder {
  public:
-  explicit NullBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG) : ArrayBuilder(null(), pool) {}
+  explicit NullBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
+      : ArrayBuilder(null(), pool) {}
 
   Status AppendNull() {
     ++null_count_;
@@ -236,8 +231,8 @@ class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
 
   template <typename T1 = T>
   explicit NumericBuilder(
-      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type
-          ARROW_MEMORY_POOL_ARG)
+      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool
+          ARROW_MEMORY_POOL_DEFAULT)
       : PrimitiveBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 
   using PrimitiveBuilder<T>::Append;
@@ -368,7 +363,7 @@ inline uint8_t ExpandedUIntSize(uint64_t val, uint8_t current_int_size) {
 
 class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
-  explicit AdaptiveUIntBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
+  explicit AdaptiveUIntBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
   using ArrayBuilder::Advance;
 
@@ -427,7 +422,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
 
 class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
-  explicit AdaptiveIntBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
+  explicit AdaptiveIntBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
   using ArrayBuilder::Advance;
 
@@ -486,7 +481,7 @@ class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase
 
 class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
  public:
-  explicit BooleanBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
+  explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
   explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
 
@@ -607,7 +602,7 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
 /// \brief Builder class for variable-length binary data
 class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
  public:
-  explicit BinaryBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
+  explicit BinaryBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
 #ifndef ARROW_NO_DEPRECATED_API
   /// \deprecated Since 0.6.0
@@ -656,7 +651,7 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
 class ARROW_EXPORT StringBuilder : public BinaryBuilder {
  public:
   using BinaryBuilder::BinaryBuilder;
-  explicit StringBuilder(MemoryPool* ARROW_MEMORY_POOL_ARG);
+  explicit StringBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
   using BinaryBuilder::Append;
 
@@ -676,7 +671,7 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 #endif
 
   FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
-                         MemoryPool* ARROW_MEMORY_POOL_ARG);
+                         MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
   Status Append(const uint8_t* value);
   Status Append(const uint8_t* data, int64_t length,
@@ -699,7 +694,7 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
  public:
   explicit DecimalBuilder(const std::shared_ptr<DataType>& type,
-                          MemoryPool* ARROW_MEMORY_POOL_ARG);
+                          MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
 #ifndef ARROW_NO_DEPRECATED_API
   /// \deprecated Since 0.6.0
diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt
index 97de4923f0180..9fc9efa594712 100644
--- a/cpp/src/arrow/gpu/CMakeLists.txt
+++ b/cpp/src/arrow/gpu/CMakeLists.txt
@@ -80,9 +80,26 @@ set(ARROW_GPU_SHARED_LINK_LIBS
   arrow_shared
 )
 
-cuda_add_library(arrow_gpu SHARED
+add_library(arrow_gpu_objlib OBJECT
   ${ARROW_GPU_SRCS}
 )
+set_property(TARGET arrow_gpu_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
+
+if (ARROW_BUILD_SHARED)
+  cuda_add_library(arrow_gpu_shared SHARED $<TARGET_OBJECTS:arrow_gpu_objlib>)
+  install(TARGETS arrow_gpu_shared
+    RUNTIME DESTINATION bin
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endif()
+
+if (ARROW_BUILD_STATIC)
+  add_library(arrow_gpu_static STATIC $<TARGET_OBJECTS:arrow_gpu_objlib>)
+  install(TARGETS arrow_gpu_static
+    RUNTIME DESTINATION bin
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endif()
 
 install(FILES
   cuda_common.h
@@ -97,10 +114,19 @@ install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-gpu.pc"
   DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
 
+set(ARROW_GPU_TEST_LINK_LIBS
+  arrow_gpu_shared
+  ${ARROW_TEST_LINK_LIBS})
+
 if (ARROW_BUILD_TESTS)
-  set(ARROW_GPU_TEST_LINK_LIBS
-    ${ARROW_TEST_LINK_LIBS}
-    arrow_gpu)
   ADD_ARROW_CUDA_TEST(cuda-test
     STATIC_LINK_LIBS ${ARROW_GPU_TEST_LINK_LIBS})
 endif()
+
+if (ARROW_BUILD_BENCHMARKS)
+  cuda_add_executable(cuda-benchmark cuda-benchmark.cc)
+  target_link_libraries(cuda-benchmark
+    arrow_gpu_shared
+    gtest
+    ${ARROW_BENCHMARK_LINK_LIBS})
+endif()
diff --git a/cpp/src/arrow/gpu/cuda-benchmark.cc b/cpp/src/arrow/gpu/cuda-benchmark.cc
new file mode 100644
index 0000000000000..82caacc05e478
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda-benchmark.cc
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/memory_pool.h"
+#include "arrow/test-util.h"
+
+#include "arrow/gpu/cuda_memory.h"
+
+namespace arrow {
+namespace gpu {
+
+constexpr int64_t kGpuNumber = 0;
+
+static void CudaBufferWriterBenchmark(benchmark::State& state, const int64_t total_bytes,
+                                      const int64_t chunksize,
+                                      const int64_t buffer_size) {
+  std::shared_ptr<CudaBuffer> device_buffer;
+  ABORT_NOT_OK(AllocateCudaBuffer(kGpuNumber, total_bytes, &device_buffer));
+  CudaBufferWriter writer(device_buffer);
+
+  if (buffer_size > 0) {
+    ABORT_NOT_OK(writer.SetBufferSize(buffer_size));
+  }
+
+  std::shared_ptr<PoolBuffer> buffer;
+  ASSERT_OK(test::MakeRandomBytePoolBuffer(total_bytes, default_memory_pool(), &buffer));
+
+  const uint8_t* host_data = buffer->data();
+  while (state.KeepRunning()) {
+    int64_t bytes_written = 0;
+    ABORT_NOT_OK(writer.Seek(0));
+    while (bytes_written < total_bytes) {
+      int64_t bytes_to_write = std::min(chunksize, total_bytes - bytes_written);
+      ABORT_NOT_OK(writer.Write(host_data + bytes_written, bytes_to_write));
+      bytes_written += bytes_to_write;
+    }
+  }
+  state.SetBytesProcessed(int64_t(state.iterations()) * total_bytes);
+}
+
+static void BM_Writer_Buffered(benchmark::State& state) {
+  // 128MB
+  const int64_t kTotalBytes = 1 << 27;
+
+  // 8MB
+  const int64_t kBufferSize = 1 << 23;
+
+  CudaBufferWriterBenchmark(state, kTotalBytes, state.range(0), kBufferSize);
+}
+
+static void BM_Writer_Unbuffered(benchmark::State& state) {
+  // 128MB
+  const int64_t kTotalBytes = 1 << 27;
+  CudaBufferWriterBenchmark(state, kTotalBytes, state.range(0), 0);
+}
+
+// Vary chunk write size from 256 bytes to 64K
+BENCHMARK(BM_Writer_Buffered)
+    ->RangeMultiplier(16)
+    ->Range(1 << 8, 1 << 16)
+    ->MinTime(1.0)
+    ->UseRealTime();
+
+BENCHMARK(BM_Writer_Unbuffered)
+    ->RangeMultiplier(4)
+    ->RangeMultiplier(16)
+    ->Range(1 << 8, 1 << 16)
+    ->MinTime(1.0)
+    ->UseRealTime();
+
+}  // namespace gpu
+}  // namespace arrow
diff --git a/cpp/src/arrow/gpu/cuda-test.cc b/cpp/src/arrow/gpu/cuda-test.cc
index 9dc8e1aff946a..f479701eaeb31 100644
--- a/cpp/src/arrow/gpu/cuda-test.cc
+++ b/cpp/src/arrow/gpu/cuda-test.cc
@@ -29,17 +29,180 @@
 namespace arrow {
 namespace gpu {
 
+constexpr int kGpuNumber = 0;
+
 class TestCudaBuffer : public ::testing::Test {};
 
 TEST_F(TestCudaBuffer, Allocate) {
-  const int device = 0;
-
   const int64_t kSize = 100;
   std::shared_ptr<CudaBuffer> buffer;
 
-  ASSERT_OK(AllocateCudaBuffer(device, kSize, &buffer));
+  ASSERT_OK(AllocateCudaBuffer(kGpuNumber, kSize, &buffer));
   ASSERT_EQ(kSize, buffer->size());
 }
 
+void AssertCudaBufferEquals(const CudaBuffer& buffer, const uint8_t* host_data,
+                            const int64_t nbytes) {
+  std::shared_ptr<MutableBuffer> result;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), nbytes, &result));
+  ASSERT_OK(buffer.CopyToHost(0, buffer.size(), result->mutable_data()));
+  ASSERT_EQ(0, std::memcmp(result->data(), host_data, nbytes));
+}
+
+TEST_F(TestCudaBuffer, CopyFromHost) {
+  const int64_t kSize = 1000;
+  std::shared_ptr<CudaBuffer> device_buffer;
+  ASSERT_OK(AllocateCudaBuffer(kGpuNumber, kSize, &device_buffer));
+
+  std::shared_ptr<PoolBuffer> host_buffer;
+  ASSERT_OK(test::MakeRandomBytePoolBuffer(kSize, default_memory_pool(), &host_buffer));
+
+  ASSERT_OK(device_buffer->CopyFromHost(0, host_buffer->data(), 500));
+  ASSERT_OK(device_buffer->CopyFromHost(500, host_buffer->data() + 500, kSize - 500));
+
+  AssertCudaBufferEquals(*device_buffer, host_buffer->data(), kSize);
+}
+
+class TestCudaBufferWriter : public ::testing::Test {
+ public:
+  void Allocate(const int64_t size) {
+    ASSERT_OK(AllocateCudaBuffer(kGpuNumber, size, &device_buffer_));
+    writer_.reset(new CudaBufferWriter(device_buffer_));
+  }
+
+  void TestWrites(const int64_t total_bytes, const int64_t chunksize,
+                  const int64_t buffer_size = 0) {
+    std::shared_ptr<PoolBuffer> buffer;
+    ASSERT_OK(
+        test::MakeRandomBytePoolBuffer(total_bytes, default_memory_pool(), &buffer));
+
+    if (buffer_size > 0) {
+      ASSERT_OK(writer_->SetBufferSize(buffer_size));
+    }
+
+    int64_t position = 0;
+    ASSERT_OK(writer_->Tell(&position));
+    ASSERT_EQ(0, position);
+
+    const uint8_t* host_data = buffer->data();
+    ASSERT_OK(writer_->Write(host_data, chunksize));
+    ASSERT_OK(writer_->Tell(&position));
+    ASSERT_EQ(chunksize, position);
+
+    ASSERT_OK(writer_->Seek(0));
+    ASSERT_OK(writer_->Tell(&position));
+    ASSERT_EQ(0, position);
+
+    while (position < total_bytes) {
+      int64_t bytes_to_write = std::min(chunksize, total_bytes - position);
+      ASSERT_OK(writer_->Write(host_data + position, bytes_to_write));
+      position += bytes_to_write;
+    }
+
+    ASSERT_OK(writer_->Flush());
+
+    AssertCudaBufferEquals(*device_buffer_, buffer->data(), total_bytes);
+  }
+
+ protected:
+  std::shared_ptr<CudaBuffer> device_buffer_;
+  std::unique_ptr<CudaBufferWriter> writer_;
+};
+
+TEST_F(TestCudaBufferWriter, UnbufferedWrites) {
+  const int64_t kTotalSize = 1 << 16;
+  Allocate(kTotalSize);
+  TestWrites(kTotalSize, 1000);
+}
+
+TEST_F(TestCudaBufferWriter, BufferedWrites) {
+  const int64_t kTotalSize = 1 << 16;
+  Allocate(kTotalSize);
+  TestWrites(kTotalSize, 1000, 1 << 12);
+}
+
+TEST_F(TestCudaBufferWriter, EdgeCases) {
+  Allocate(1000);
+
+  std::shared_ptr<PoolBuffer> buffer;
+  ASSERT_OK(test::MakeRandomBytePoolBuffer(1000, default_memory_pool(), &buffer));
+  const uint8_t* host_data = buffer->data();
+
+  ASSERT_EQ(0, writer_->buffer_size());
+  ASSERT_OK(writer_->SetBufferSize(100));
+  ASSERT_EQ(100, writer_->buffer_size());
+
+  // Write 0 bytes
+  int64_t position = 0;
+  ASSERT_OK(writer_->Write(host_data, 0));
+  ASSERT_OK(writer_->Tell(&position));
+  ASSERT_EQ(0, position);
+
+  // Write some data, then change buffer size
+  ASSERT_OK(writer_->Write(host_data, 10));
+  ASSERT_OK(writer_->SetBufferSize(200));
+  ASSERT_EQ(200, writer_->buffer_size());
+
+  ASSERT_EQ(0, writer_->num_bytes_buffered());
+
+  // Write more than buffer size
+  ASSERT_OK(writer_->Write(host_data + 10, 300));
+  ASSERT_EQ(0, writer_->num_bytes_buffered());
+
+  // Write exactly buffer size
+  ASSERT_OK(writer_->Write(host_data + 310, 200));
+  ASSERT_EQ(0, writer_->num_bytes_buffered());
+
+  // Write rest of bytes
+  ASSERT_OK(writer_->Write(host_data + 510, 390));
+  ASSERT_OK(writer_->Write(host_data + 900, 100));
+
+  // Close flushes
+  ASSERT_OK(writer_->Close());
+
+  // Check that everything was written
+  AssertCudaBufferEquals(*device_buffer_, host_data, 1000);
+}
+
+TEST(TestCudaBufferReader, Basics) {
+  std::shared_ptr<CudaBuffer> device_buffer;
+
+  const int64_t size = 1000;
+  ASSERT_OK(AllocateCudaBuffer(kGpuNumber, size, &device_buffer));
+
+  std::shared_ptr<PoolBuffer> buffer;
+  ASSERT_OK(test::MakeRandomBytePoolBuffer(1000, default_memory_pool(), &buffer));
+  const uint8_t* host_data = buffer->data();
+
+  ASSERT_OK(device_buffer->CopyFromHost(0, host_data, 1000));
+
+  CudaBufferReader reader(device_buffer);
+
+  // Read to host memory
+  uint8_t stack_buffer[100] = {0};
+  int64_t bytes_read = 0;
+  ASSERT_OK(reader.Seek(950));
+
+  int64_t position = 0;
+  ASSERT_OK(reader.Tell(&position));
+  ASSERT_EQ(950, position);
+
+  ASSERT_OK(reader.Read(100, &bytes_read, stack_buffer));
+  ASSERT_EQ(50, bytes_read);
+  ASSERT_EQ(0, std::memcmp(stack_buffer, host_data + 950, 50));
+  ASSERT_OK(reader.Tell(&position));
+  ASSERT_EQ(1000, position);
+
+  ASSERT_OK(reader.Seek(925));
+  std::shared_ptr<Buffer> tmp;
+  ASSERT_OK(reader.Read(100, &tmp));
+  ASSERT_EQ(75, tmp->size());
+  ASSERT_OK(reader.Tell(&position));
+  ASSERT_EQ(1000, position);
+
+  ASSERT_OK(std::dynamic_pointer_cast<CudaBuffer>(tmp)->CopyToHost(0, 75, stack_buffer));
+  ASSERT_EQ(0, std::memcmp(stack_buffer, host_data + 925, 75));
+}
+
 }  // namespace gpu
 }  // namespace arrow
diff --git a/cpp/src/arrow/gpu/cuda_common.h b/cpp/src/arrow/gpu/cuda_common.h
index 75f41c9a41eab..fae2f6edf8084 100644
--- a/cpp/src/arrow/gpu/cuda_common.h
+++ b/cpp/src/arrow/gpu/cuda_common.h
@@ -20,6 +20,8 @@
 #ifndef ARROW_GPU_CUDA_COMMON_H
 #define ARROW_GPU_CUDA_COMMON_H
 
+#include <sstream>
+
 #include <cuda_runtime_api.h>
 
 namespace arrow {
@@ -32,12 +34,15 @@ namespace gpu {
     (void)ret;            \
   } while (0)
 
-#define CUDA_RETURN_NOT_OK(STMT)                              \
-  do {                                                        \
-    cudaError_t ret = (STMT);                                 \
-    if (ret != cudaSuccess) {                                 \
-      return Status::IOError("Cuda API call failed: " #STMT); \
-    }                                                         \
+#define CUDA_RETURN_NOT_OK(STMT)                                       \
+  do {                                                                 \
+    cudaError_t ret = (STMT);                                          \
+    if (ret != cudaSuccess) {                                          \
+      std::stringstream ss;                                            \
+      ss << "Cuda API call in " << __FILE__ << " at line " << __LINE__ \
+         << " failed: " << #STMT;                                      \
+      return Status::IOError(ss.str());                                \
+    }                                                                  \
   } while (0)
 
 }  // namespace gpu
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
index cf37bdae2f3ef..d669619791a8d 100644
--- a/cpp/src/arrow/gpu/cuda_memory.cc
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -17,10 +17,12 @@
 
 #include "arrow/gpu/cuda_memory.h"
 
+#include <algorithm>
 #include <cstdint>
 #include <memory>
 
 #include "arrow/buffer.h"
+#include "arrow/io/memory.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
@@ -35,8 +37,21 @@ CudaBuffer::~CudaBuffer() {
   }
 }
 
-Status CudaBuffer::CopyHost(uint8_t* out) {
-  CUDA_RETURN_NOT_OK(cudaMemcpy(out, data_, size_, cudaMemcpyDeviceToHost));
+CudaBuffer::CudaBuffer(const std::shared_ptr<CudaBuffer>& parent, const int64_t offset,
+                       const int64_t size)
+    : Buffer(parent, offset, size), gpu_number_(parent->gpu_number()) {}
+
+Status CudaBuffer::CopyToHost(const int64_t position, const int64_t nbytes,
+                              uint8_t* out) const {
+  CUDA_RETURN_NOT_OK(cudaMemcpy(out, data_ + position, nbytes, cudaMemcpyDeviceToHost));
+  return Status::OK();
+}
+
+Status CudaBuffer::CopyFromHost(const int64_t position, const uint8_t* data,
+                                int64_t nbytes) {
+  DCHECK_LE(nbytes, size_ - position) << "Copy would overflow buffer";
+  CUDA_RETURN_NOT_OK(
+      cudaMemcpy(mutable_data_ + position, data, nbytes, cudaMemcpyHostToDevice));
   return Status::OK();
 }
 
@@ -52,8 +67,101 @@ Status AllocateCudaBuffer(int gpu_number, const int64_t size,
 
 CudaHostBuffer::~CudaHostBuffer() { CUDA_DCHECK(cudaFreeHost(mutable_data_)); }
 
-Status AllocateCudaHostBuffer(const int gpu_number, const int64_t size,
-                              std::shared_ptr<CudaHostBuffer>* out) {
+// ----------------------------------------------------------------------
+// CudaBufferReader
+
+CudaBufferReader::CudaBufferReader(const std::shared_ptr<CudaBuffer>& buffer)
+    : io::BufferReader(buffer), cuda_buffer_(buffer) {}
+
+CudaBufferReader::~CudaBufferReader() {}
+
+Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+  nbytes = std::min(nbytes, size_ - position_);
+  CUDA_RETURN_NOT_OK(
+      cudaMemcpy(buffer, data_ + position_, nbytes, cudaMemcpyDeviceToHost));
+  *bytes_read = nbytes;
+  position_ += nbytes;
+  return Status::OK();
+}
+
+Status CudaBufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  int64_t size = std::min(nbytes, size_ - position_);
+  *out = std::make_shared<CudaBuffer>(cuda_buffer_, position_, size);
+  position_ += size;
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// CudaBufferWriter
+
+CudaBufferWriter::CudaBufferWriter(const std::shared_ptr<CudaBuffer>& buffer)
+    : io::FixedSizeBufferWriter(buffer), buffer_size_(0), buffer_position_(0) {}
+
+CudaBufferWriter::~CudaBufferWriter() {}
+
+Status CudaBufferWriter::Close() { return Flush(); }
+
+Status CudaBufferWriter::Flush() {
+  if (buffer_size_ > 0 && buffer_position_ > 0) {
+    // Only need to flush when the write has been buffered
+    CUDA_RETURN_NOT_OK(cudaMemcpy(mutable_data_ + position_ - buffer_position_,
+                                  host_buffer_data_, buffer_position_,
+                                  cudaMemcpyHostToDevice));
+    buffer_position_ = 0;
+  }
+  return Status::OK();
+}
+
+Status CudaBufferWriter::Seek(int64_t position) {
+  if (buffer_position_ > 0) {
+    RETURN_NOT_OK(Flush());
+  }
+  return io::FixedSizeBufferWriter::Seek(position);
+}
+
+Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
+  if (memcopy_num_threads_ > 1) {
+    return Status::Invalid("parallel CUDA memcpy not supported");
+  }
+
+  if (nbytes == 0) {
+    return Status::OK();
+  }
+
+  if (buffer_size_ > 0) {
+    if (nbytes + buffer_position_ >= buffer_size_) {
+      // Reach end of buffer, write everything
+      RETURN_NOT_OK(Flush());
+      CUDA_RETURN_NOT_OK(
+          cudaMemcpy(mutable_data_ + position_, data, nbytes, cudaMemcpyHostToDevice));
+    } else {
+      // Write bytes to buffer
+      std::memcpy(host_buffer_data_ + buffer_position_, data, nbytes);
+      buffer_position_ += nbytes;
+    }
+  } else {
+    // Unbuffered write
+    CUDA_RETURN_NOT_OK(
+        cudaMemcpy(mutable_data_ + position_, data, nbytes, cudaMemcpyHostToDevice));
+  }
+  position_ += nbytes;
+  return Status::OK();
+}
+
+Status CudaBufferWriter::SetBufferSize(const int64_t buffer_size) {
+  if (buffer_position_ > 0) {
+    // Flush any buffered data
+    RETURN_NOT_OK(Flush());
+  }
+  RETURN_NOT_OK(AllocateCudaHostBuffer(buffer_size, &host_buffer_));
+  host_buffer_data_ = host_buffer_->mutable_data();
+  buffer_size_ = buffer_size;
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+
+Status AllocateCudaHostBuffer(const int64_t size, std::shared_ptr<CudaHostBuffer>* out) {
   uint8_t* data = nullptr;
   CUDA_RETURN_NOT_OK(
       cudaMallocHost(reinterpret_cast<void**>(&data), static_cast<size_t>(size)));
diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h
index 885373a26c943..acf65e6574557 100644
--- a/cpp/src/arrow/gpu/cuda_memory.h
+++ b/cpp/src/arrow/gpu/cuda_memory.h
@@ -22,25 +22,41 @@
 #include <memory>
 
 #include "arrow/buffer.h"
+#include "arrow/io/memory.h"
+#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 
 namespace arrow {
 namespace gpu {
 
+/// \class CudaBuffer
 /// \brief An Arrow buffer located on a GPU device
 ///
 /// Be careful using this in any Arrow code which may not be GPU-aware
-class ARROW_EXPORT CudaBuffer : public MutableBuffer {
+class ARROW_EXPORT CudaBuffer : public Buffer {
  public:
   CudaBuffer(uint8_t* data, int64_t size, const int gpu_number, bool own_data = false)
-      : MutableBuffer(data, size), gpu_number_(gpu_number), own_data_(own_data) {}
+      : Buffer(data, size), gpu_number_(gpu_number), own_data_(own_data) {
+    is_mutable_ = true;
+    mutable_data_ = data;
+  }
+
+  CudaBuffer(const std::shared_ptr<CudaBuffer>& parent, const int64_t offset,
+             const int64_t size);
 
   ~CudaBuffer();
 
   /// \brief Copy memory from GPU device to CPU host
   /// \param[out] out a pre-allocated output buffer
   /// \return Status
-  Status CopyHost(uint8_t* out);
+  Status CopyToHost(const int64_t position, const int64_t nbytes, uint8_t* out) const;
+
+  /// \brief Copy memory to device at position
+  /// \param[in] position start position to copy bytes
+  /// \param[in] data the host data to copy
+  /// \param[in] nbytes number of bytes to copy
+  /// \return Status
+  Status CopyFromHost(const int64_t position, const uint8_t* data, int64_t nbytes);
 
   int gpu_number() const { return gpu_number_; }
 
@@ -49,6 +65,7 @@ class ARROW_EXPORT CudaBuffer : public MutableBuffer {
   bool own_data_;
 };
 
+/// \class CudaHostBuffer
 /// \brief Device-accessible CPU memory created using cudaHostAlloc
 class ARROW_EXPORT CudaHostBuffer : public MutableBuffer {
  public:
@@ -56,6 +73,72 @@ class ARROW_EXPORT CudaHostBuffer : public MutableBuffer {
   ~CudaHostBuffer();
 };
 
+/// \class CudaBufferReader
+/// \brief File interface for zero-copy read from CUDA buffers
+///
+/// Note: Reads return pointers to device memory. This means you must be
+/// careful using this interface with any Arrow code which may expect to be
+/// able to do anything other than pointer arithmetic on the returned buffers
+class ARROW_EXPORT CudaBufferReader : public io::BufferReader {
+ public:
+  explicit CudaBufferReader(const std::shared_ptr<CudaBuffer>& buffer);
+  ~CudaBufferReader();
+
+  /// \brief Read bytes into pre-allocated host memory
+  /// \param[in] nbytes number of bytes to read
+  /// \param[out] bytes_read actual number of bytes read
+  /// \param[out] buffer pre-allocated memory to write into
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+
+  /// \brief Zero-copy read from device memory
+  /// \param[in] nbytes number of bytes to read
+  /// \param[out] out a Buffer referencing device memory
+  /// \return Status
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
+ private:
+  // In case we need to access anything GPU-specific, like device number
+  std::shared_ptr<CudaBuffer> cuda_buffer_;
+};
+
+/// \class CudaBufferWriter
+/// \brief File interface for writing to CUDA buffers, with optional buffering
+class ARROW_EXPORT CudaBufferWriter : public io::FixedSizeBufferWriter {
+ public:
+  explicit CudaBufferWriter(const std::shared_ptr<CudaBuffer>& buffer);
+  ~CudaBufferWriter();
+
+  /// \brief Close writer and flush buffered bytes to GPU
+  Status Close() override;
+
+  /// \brief Flush buffered bytes to GPU
+  Status Flush() override;
+
+  // Seek requires flushing if any bytes are buffered
+  Status Seek(int64_t position) override;
+  Status Write(const uint8_t* data, int64_t nbytes) override;
+
+  /// \brief Set CPU buffer size to limit calls to cudaMemcpy
+  /// \param[in] buffer_size the size of CPU buffer to allocate
+  /// \return Status
+  ///
+  /// By default writes are unbuffered
+  Status SetBufferSize(const int64_t buffer_size);
+
+  /// \brief Returns size of host (CPU) buffer, 0 for unbuffered
+  int64_t buffer_size() const { return buffer_size_; }
+
+  /// \brief Returns number of bytes buffered on host
+  int64_t num_bytes_buffered() const { return buffer_position_; }
+
+ private:
+  // Pinned host buffer for buffering writes on CPU before calling cudaMalloc
+  int64_t buffer_size_;
+  int64_t buffer_position_;
+  std::shared_ptr<CudaHostBuffer> host_buffer_;
+  uint8_t* host_buffer_data_;
+};
+
 /// \brief Allocate CUDA memory on a GPU device
 /// \param[in] gpu_number Device number to allocate
 /// \param[in] size number of bytes
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 85caee1fd829f..395785299e135 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -440,7 +440,7 @@ Status ReadableFile::Open(const std::string& path, MemoryPool* memory_pool,
 
 Status ReadableFile::Close() { return impl_->Close(); }
 
-Status ReadableFile::Tell(int64_t* pos) { return impl_->Tell(pos); }
+Status ReadableFile::Tell(int64_t* pos) const { return impl_->Tell(pos); }
 
 Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
   return impl_->Read(nbytes, bytes_read, out);
@@ -492,7 +492,7 @@ Status FileOutputStream::Open(const std::string& path, bool append,
 
 Status FileOutputStream::Close() { return impl_->Close(); }
 
-Status FileOutputStream::Tell(int64_t* pos) { return impl_->Tell(pos); }
+Status FileOutputStream::Tell(int64_t* pos) const { return impl_->Tell(pos); }
 
 Status FileOutputStream::Write(const uint8_t* data, int64_t length) {
   return impl_->Write(data, length);
@@ -612,7 +612,7 @@ Status MemoryMappedFile::GetSize(int64_t* size) {
   return Status::OK();
 }
 
-Status MemoryMappedFile::Tell(int64_t* position) {
+Status MemoryMappedFile::Tell(int64_t* position) const {
   *position = memory_map_->position();
   return Status::OK();
 }
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 2a0e89ca325fa..f26d905141ad4 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -57,7 +57,7 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
 
   // OutputStream interface
   Status Close() override;
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
 
   // Write bytes to the stream. Thread-safe
   Status Write(const uint8_t* data, int64_t nbytes) override;
@@ -91,7 +91,7 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile {
                      std::shared_ptr<ReadableFile>* file);
 
   Status Close() override;
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
 
   // Read bytes from the file. Thread-safe
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
@@ -130,7 +130,7 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
 
   Status Close() override;
 
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
 
   Status Seek(int64_t position) override;
 
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index ba446b56e00bb..8b143c3ed833d 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -230,7 +230,7 @@ Status HdfsReadableFile::GetSize(int64_t* size) { return impl_->GetSize(size); }
 
 Status HdfsReadableFile::Seek(int64_t position) { return impl_->Seek(position); }
 
-Status HdfsReadableFile::Tell(int64_t* position) { return impl_->Tell(position); }
+Status HdfsReadableFile::Tell(int64_t* position) const { return impl_->Tell(position); }
 
 // ----------------------------------------------------------------------
 // File writing
@@ -284,7 +284,7 @@ Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes) {
 
 Status HdfsOutputStream::Flush() { return impl_->Flush(); }
 
-Status HdfsOutputStream::Tell(int64_t* position) { return impl_->Tell(position); }
+Status HdfsOutputStream::Tell(int64_t* position) const { return impl_->Tell(position); }
 
 // ----------------------------------------------------------------------
 // HDFS client
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index ca8895304b4d6..82e3f7108bc87 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -199,7 +199,7 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
   bool supports_zero_copy() const override;
 
   Status Seek(int64_t position) override;
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
 
   void set_memory_pool(MemoryPool* pool);
 
@@ -228,7 +228,7 @@ class ARROW_EXPORT HdfsOutputStream : public OutputStream {
 
   Status Flush() override;
 
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
 
  private:
   class ARROW_NO_EXPORT HdfsOutputStreamImpl;
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 4bb7ebe2fd9f4..3388acb0ed752 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -72,7 +72,7 @@ class ARROW_EXPORT FileInterface {
  public:
   virtual ~FileInterface() = 0;
   virtual Status Close() = 0;
-  virtual Status Tell(int64_t* position) = 0;
+  virtual Status Tell(int64_t* position) const = 0;
 
   FileMode::type mode() const { return mode_; }
 
@@ -94,7 +94,7 @@ class ARROW_EXPORT Writeable {
  public:
   virtual Status Write(const uint8_t* data, int64_t nbytes) = 0;
 
-  // Default implementation is a no-op
+  /// \brief Flush buffered bytes, if any
   virtual Status Flush();
 
   Status Write(const std::string& data);
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 50f3ddfaf6512..532c2c64a4588 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -75,7 +75,7 @@ Status BufferOutputStream::Finish(std::shared_ptr<Buffer>* result) {
   return Status::OK();
 }
 
-Status BufferOutputStream::Tell(int64_t* position) {
+Status BufferOutputStream::Tell(int64_t* position) const {
   *position = position_;
   return Status::OK();
 }
@@ -104,9 +104,12 @@ Status BufferOutputStream::Reserve(int64_t nbytes) {
 // ----------------------------------------------------------------------
 // OutputStream that doesn't write anything
 
-Status MockOutputStream::Close() { return Status::OK(); }
+Status MockOutputStream::Close() {
+  // no-op
+  return Status::OK();
+}
 
-Status MockOutputStream::Tell(int64_t* position) {
+Status MockOutputStream::Tell(int64_t* position) const {
   *position = extent_bytes_written_;
   return Status::OK();
 }
@@ -128,6 +131,7 @@ FixedSizeBufferWriter::FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buff
     : memcopy_num_threads_(kMemcopyDefaultNumThreads),
       memcopy_blocksize_(kMemcopyDefaultBlocksize),
       memcopy_threshold_(kMemcopyDefaultThreshold) {
+  DCHECK(buffer) << "Buffer was nullptr";
   buffer_ = buffer;
   DCHECK(buffer->is_mutable()) << "Must pass mutable buffer";
   mutable_data_ = buffer->mutable_data();
@@ -138,7 +142,7 @@ FixedSizeBufferWriter::FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buff
 FixedSizeBufferWriter::~FixedSizeBufferWriter() {}
 
 Status FixedSizeBufferWriter::Close() {
-  // No-op
+  // no-op
   return Status::OK();
 }
 
@@ -150,7 +154,7 @@ Status FixedSizeBufferWriter::Seek(int64_t position) {
   return Status::OK();
 }
 
-Status FixedSizeBufferWriter::Tell(int64_t* position) {
+Status FixedSizeBufferWriter::Tell(int64_t* position) const {
   *position = position_;
   return Status::OK();
 }
@@ -201,7 +205,7 @@ Status BufferReader::Close() {
   return Status::OK();
 }
 
-Status BufferReader::Tell(int64_t* position) {
+Status BufferReader::Tell(int64_t* position) const {
   *position = position_;
   return Status::OK();
 }
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 1f8177436471c..667a7f087f1db 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -51,7 +51,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
 
   // Implement the OutputStream interface
   Status Close() override;
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
   Status Write(const uint8_t* data, int64_t nbytes) override;
 
   /// Close the stream and return the buffer
@@ -74,7 +74,7 @@ class ARROW_EXPORT MockOutputStream : public OutputStream {
 
   // Implement the OutputStream interface
   Status Close() override;
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
   Status Write(const uint8_t* data, int64_t nbytes) override;
 
   int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
@@ -93,7 +93,7 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
 
   Status Close() override;
   Status Seek(int64_t position) override;
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
   Status Write(const uint8_t* data, int64_t nbytes) override;
   Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
 
@@ -101,7 +101,7 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
   void set_memcopy_blocksize(int64_t blocksize);
   void set_memcopy_threshold(int64_t threshold);
 
- private:
+ protected:
   std::mutex lock_;
   std::shared_ptr<Buffer> buffer_;
   uint8_t* mutable_data_;
@@ -120,8 +120,7 @@ class ARROW_EXPORT BufferReader : public RandomAccessFile {
   virtual ~BufferReader();
 
   Status Close() override;
-  Status Tell(int64_t* position) override;
-
+  Status Tell(int64_t* position) const override;
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
 
   // Zero copy read
@@ -134,7 +133,7 @@ class ARROW_EXPORT BufferReader : public RandomAccessFile {
 
   std::shared_ptr<Buffer> buffer() const { return buffer_; }
 
- private:
+ protected:
   std::shared_ptr<Buffer> buffer_;
   const uint8_t* data_;
   int64_t size_;
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 90bc593ab71fe..5bb2b5669ed31 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -109,6 +109,12 @@ class ARROW_EXPORT LoggingMemoryPool : public MemoryPool {
 
 ARROW_EXPORT MemoryPool* default_memory_pool();
 
+#ifdef ARROW_NO_DEFAULT_MEMORY_POOL
+#define ARROW_MEMORY_POOL_DEFAULT
+#else
+#define ARROW_MEMORY_POOL_DEFAULT = default_memory_pool()
+#endif
+
 }  // namespace arrow
 
 #endif  // ARROW_MEMORY_POOL_H
diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index 4c73fd6401cb6..99b99f7469f9e 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -113,7 +113,7 @@ Status PyReadableFile::Seek(int64_t position) {
   return file_->Seek(position, 0);
 }
 
-Status PyReadableFile::Tell(int64_t* position) {
+Status PyReadableFile::Tell(int64_t* position) const {
   PyAcquireGIL lock;
   return file_->Tell(position);
 }
@@ -177,7 +177,7 @@ Status PyOutputStream::Close() {
   return file_->Close();
 }
 
-Status PyOutputStream::Tell(int64_t* position) {
+Status PyOutputStream::Tell(int64_t* position) const {
   *position = position_;
   return Status::OK();
 }
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index ebd4c5a61e938..eda7346741d0b 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -63,7 +63,7 @@ class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile {
 
   Status Seek(int64_t position) override;
 
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
 
   bool supports_zero_copy() const override;
 
@@ -77,7 +77,7 @@ class ARROW_EXPORT PyOutputStream : public io::OutputStream {
   virtual ~PyOutputStream();
 
   Status Close() override;
-  Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) const override;
   Status Write(const uint8_t* data, int64_t nbytes) override;
 
  private:
diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
index 34bee18df5229..6fe3a5c1739d7 100644
--- a/cpp/src/arrow/util/io-util.h
+++ b/cpp/src/arrow/util/io-util.h
@@ -33,13 +33,14 @@ class StdoutStream : public OutputStream {
   StdoutStream() : pos_(0) { set_mode(FileMode::WRITE); }
   virtual ~StdoutStream() {}
 
-  Status Close() { return Status::OK(); }
-  Status Tell(int64_t* position) {
+  Status Close() override { return Status::OK(); }
+
+  Status Tell(int64_t* position) const override {
     *position = pos_;
     return Status::OK();
   }
 
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const uint8_t* data, int64_t nbytes) override {
     pos_ += nbytes;
     std::cout.write(reinterpret_cast<const char*>(data), nbytes);
     return Status::OK();
@@ -55,13 +56,14 @@ class StdinStream : public InputStream {
   StdinStream() : pos_(0) { set_mode(FileMode::READ); }
   virtual ~StdinStream() {}
 
-  Status Close() { return Status::OK(); }
-  Status Tell(int64_t* position) {
+  Status Close() override { return Status::OK(); }
+
+  Status Tell(int64_t* position) const override {
     *position = pos_;
     return Status::OK();
   }
 
-  virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override {
     std::cin.read(reinterpret_cast<char*>(out), nbytes);
     if (std::cin) {
       *bytes_read = nbytes;
@@ -72,7 +74,7 @@ class StdinStream : public InputStream {
     return Status::OK();
   }
 
-  virtual Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override {
     auto buffer = std::make_shared<PoolBuffer>(nullptr);
     RETURN_NOT_OK(buffer->Resize(nbytes));
     int64_t bytes_read;

From b31269726694dad0b5adaf297258949f04f61a27 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 23 Aug 2017 23:39:16 -0400
Subject: [PATCH 0951/1644] ARROW-1386: [C++] Unpin CMake version in MSVC
 toolchain builds

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #989 from wesm/ARROW-1386 and squashes the following commits:

be3b53a1 [Wes McKinney] Unpin CMake version in MSVC toolchain builds now that 3.9.1 is in conda-forge
---
 ci/msvc-build.bat                        | 5 +----
 dev/release/verify-release-candidate.bat | 4 +---
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 6ebd22fc3e354..e8eb0945e516e 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -51,14 +51,11 @@ conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython ^
       thrift-cpp
 
-@rem ARROW-1294 CMake 3.9.0 in conda-forge breaks the build
-set ARROW_CMAKE_VERSION=3.8.0
-
 if "%JOB%" == "Toolchain" (
 
   conda install -n arrow -q -y -c conda-forge ^
       flatbuffers rapidjson ^
-      cmake=%ARROW_CMAKE_VERSION% ^
+      cmake ^
       git ^
       boost-cpp ^
       snappy zlib brotli gflags lz4-c zstd
diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat
index 8dcaa3d966023..503f1fd40b668 100644
--- a/dev/release/verify-release-candidate.bat
+++ b/dev/release/verify-release-candidate.bat
@@ -20,14 +20,12 @@
 @rem libray build rem dependencies if you want to build them from source as well
 @rem
 
-@rem ARROW-1294 CMake 3.9.0 in conda-forge breaks the build
-@rem set ARROW_CMAKE_VERSION=3.8.0
 @rem set PYTHON=3.6
 @rem conda create -n arrow-verify-release -f -q -y python=%PYTHON%
 @rem conda install -y ^
 @rem       six pytest setuptools numpy pandas cython ^
 @rem       thrift-cpp flatbuffers rapidjson ^
-@rem       cmake=%ARROW_CMAKE_VERSION% ^
+@rem       cmake ^
 @rem       git ^
 @rem       boost-cpp ^
 @rem       snappy zlib brotli gflags lz4-c zstd || exit /B

From 750b77dc6360464aa3621722b5ee6530b761391c Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 24 Aug 2017 13:57:42 -0400
Subject: [PATCH 0952/1644] ARROW-786: [Format] In-memory format for 128-bit
 Decimals, handling of sign bit

* Reimplement Decimal128 types to use the Int128 type as the underlying integer
representation, adapted from the Apache ORC project's C++ in memory format.
This enables us to write integration tests and results in an in-memory
Decimal128 format that is compatible with the Java implementation
* Additionaly, this PR also fixes Decimal slice comparison and adds related
regression tests
* Follow-ups include ARROW-695 (C++ Decimal integration tests), ARROW-696 (JSON
read/write support for decimals) and ARROW-1238 (Java Decimal integration
tests).

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #981 from cpcloud/decimal-rewrite and squashes the following commits:

53ce04b6 [Phillip Cloud] Formatting
fe13ef36 [Phillip Cloud] Remove redundant constructor
86db1841 [Phillip Cloud] Subclass from FixedSizeBinaryArray for code reuse
535f9ff4 [Phillip Cloud] Use a macro for cases
1cc43cee [Phillip Cloud] Use CHAR_BIT
355fb24c [Phillip Cloud] Include the correct header for _BitScanReverse
b53d7cdd [Phillip Cloud] Share comparison code
162eeeb7 [Phillip Cloud] BUG: Double export
b98c8942 [Phillip Cloud] BUG: Export symbols
be220c83 [Phillip Cloud] Cast so we have enough space to contain the integer
57160108 [Phillip Cloud] Cast 18 to matching type size_t for msvc
88339041 [Phillip Cloud] Remove unnecessary args to sto* function calls
628ce85c [Phillip Cloud] Fix more docs
e4a17926 [Phillip Cloud] More const
8ecb3157 [Phillip Cloud] Formatting
178d3f20 [Phillip Cloud] NOLINT for MSVC specific and necessary types
38c9b506 [Phillip Cloud] Fix doc style in int128.h and add const where possible
2930d7bc [Phillip Cloud] Fix naming convention in decimal-test.cc
1eab5c4c [Phillip Cloud] Remove unnecessary header from CMakeLists.txt
22eda4b0 [Phillip Cloud] kMaximumPrecision
9af97d88 [Phillip Cloud] MSVC fix
349dc58b [Phillip Cloud] ARROW-786: [Format] In-memory format for 128-bit Decimals, handling of sign bit
---
 cpp/CMakeLists.txt                      |   1 +
 cpp/src/arrow/array-decimal-test.cc     |  92 +++--
 cpp/src/arrow/array.cc                  |  64 +--
 cpp/src/arrow/array.h                   |  37 +-
 cpp/src/arrow/builder.cc                |  57 +--
 cpp/src/arrow/builder.h                 |   8 -
 cpp/src/arrow/compare.cc                | 127 ++----
 cpp/src/arrow/python/arrow_to_pandas.cc |  64 ++-
 cpp/src/arrow/python/python-test.cc     |   4 -
 cpp/src/arrow/type.cc                   |   4 -
 cpp/src/arrow/type.h                    |   1 -
 cpp/src/arrow/util/CMakeLists.txt       |   7 +-
 cpp/src/arrow/util/decimal-test.cc      |  48 ++-
 cpp/src/arrow/util/decimal.cc           |  41 +-
 cpp/src/arrow/util/decimal.h            |  26 +-
 cpp/src/arrow/util/int128.cc            | 527 ++++++++++++++++++++++++
 cpp/src/arrow/util/int128.h             | 128 ++++++
 17 files changed, 846 insertions(+), 390 deletions(-)
 create mode 100644 cpp/src/arrow/util/int128.cc
 create mode 100644 cpp/src/arrow/util/int128.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index b55a9bb1d0256..6c56103717b60 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -742,6 +742,7 @@ set(ARROW_SRCS
   src/arrow/util/compression.cc
   src/arrow/util/cpu-info.cc
   src/arrow/util/decimal.cc
+  src/arrow/util/int128.cc
   src/arrow/util/key_value_metadata.cc
 )
 
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index e94ba48d60840..37852a3d1c798 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -28,56 +28,66 @@ namespace decimal {
 template <typename T>
 class DecimalTestBase {
  public:
-  virtual std::vector<uint8_t> data(const std::vector<T>& input,
-                                    size_t byte_width) const = 0;
+  DecimalTestBase() : pool_(default_memory_pool()) {}
 
-  void test(int precision, const std::vector<T>& draw,
-            const std::vector<uint8_t>& valid_bytes,
-            const std::vector<uint8_t>& sign_bitmap = {}, int64_t offset = 0) const {
-    auto type = std::make_shared<DecimalType>(precision, 4);
-    int byte_width = type->byte_width();
-    auto pool = default_memory_pool();
-    auto builder = std::make_shared<DecimalBuilder>(type, pool);
-    size_t null_count = 0;
+  virtual std::vector<uint8_t> MakeData(const std::vector<T>& input,
+                                        size_t byte_width) const = 0;
+
+  void InitBuilder(const std::shared_ptr<DecimalType>& type, const std::vector<T>& draw,
+                   const std::vector<uint8_t>& valid_bytes, int byte_width,
+                   std::shared_ptr<DecimalBuilder>* builder, size_t* null_count) const {
+    *builder = std::make_shared<DecimalBuilder>(type, pool_);
 
     size_t size = draw.size();
-    ASSERT_OK(builder->Reserve(size));
+    ASSERT_OK((*builder)->Reserve(size));
 
     for (size_t i = 0; i < size; ++i) {
       if (valid_bytes[i]) {
-        ASSERT_OK(builder->Append(draw[i]));
+        ASSERT_OK((*builder)->Append(draw[i]));
       } else {
-        ASSERT_OK(builder->AppendNull());
-        ++null_count;
+        ASSERT_OK((*builder)->AppendNull());
+        ++*null_count;
       }
     }
+  }
 
-    std::shared_ptr<Buffer> expected_sign_bitmap;
-    if (!sign_bitmap.empty()) {
-      ASSERT_OK(BitUtil::BytesToBits(sign_bitmap, &expected_sign_bitmap));
-    }
+  void TestCreate(int precision, const std::vector<T>& draw,
+                  const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
+    auto type = std::make_shared<DecimalType>(precision, 4);
 
-    auto raw_bytes = data(draw, byte_width);
+    std::shared_ptr<DecimalBuilder> builder;
+
+    size_t null_count = 0;
+
+    const size_t size = draw.size();
+    const int byte_width = type->byte_width();
+
+    InitBuilder(type, draw, valid_bytes, byte_width, &builder, &null_count);
+
+    auto raw_bytes = MakeData(draw, static_cast<size_t>(byte_width));
     auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), size * byte_width);
     std::shared_ptr<Buffer> expected_null_bitmap;
     ASSERT_OK(BitUtil::BytesToBits(valid_bytes, &expected_null_bitmap));
 
     int64_t expected_null_count = test::null_count(valid_bytes);
-    auto expected =
-        std::make_shared<DecimalArray>(type, size, expected_data, expected_null_bitmap,
-                                       expected_null_count, offset, expected_sign_bitmap);
+    auto expected = std::make_shared<DecimalArray>(
+        type, size, expected_data, expected_null_bitmap, expected_null_count, 0);
 
     std::shared_ptr<Array> out;
     ASSERT_OK(builder->Finish(&out));
-    ASSERT_TRUE(out->Equals(*expected));
+    ASSERT_TRUE(out->Slice(offset)->Equals(
+        *expected->Slice(offset, expected->length() - offset)));
   }
+
+ private:
+  MemoryPool* pool_;
 };
 
 template <typename T>
 class DecimalTest : public DecimalTestBase<T> {
  public:
-  std::vector<uint8_t> data(const std::vector<T>& input,
-                            size_t byte_width) const override {
+  std::vector<uint8_t> MakeData(const std::vector<T>& input,
+                                size_t byte_width) const override {
     std::vector<uint8_t> result(input.size() * byte_width);
     // TODO(phillipc): There's probably a better way to do this
     constexpr static const size_t bytes_per_element = sizeof(T);
@@ -91,16 +101,15 @@ class DecimalTest : public DecimalTestBase<T> {
 template <>
 class DecimalTest<Decimal128> : public DecimalTestBase<Decimal128> {
  public:
-  std::vector<uint8_t> data(const std::vector<Decimal128>& input,
-                            size_t byte_width) const override {
+  std::vector<uint8_t> MakeData(const std::vector<Decimal128>& input,
+                                size_t byte_width) const override {
     std::vector<uint8_t> result;
     result.reserve(input.size() * byte_width);
     constexpr static const size_t bytes_per_element = 16;
     for (size_t i = 0; i < input.size(); ++i) {
       uint8_t stack_bytes[bytes_per_element] = {0};
       uint8_t* bytes = stack_bytes;
-      bool is_negative;
-      ToBytes(input[i], &bytes, &is_negative);
+      ToBytes(input[i], &bytes);
 
       for (size_t i = 0; i < bytes_per_element; ++i) {
         result.push_back(bytes[i]);
@@ -124,7 +133,8 @@ TEST_P(Decimal32BuilderTest, NoNulls) {
   std::vector<Decimal32> draw = {Decimal32(1), Decimal32(2), Decimal32(2389),
                                  Decimal32(4), Decimal32(-12348)};
   std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
-  this->test(precision, draw, valid_bytes);
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
 TEST_P(Decimal64BuilderTest, NoNulls) {
@@ -132,7 +142,8 @@ TEST_P(Decimal64BuilderTest, NoNulls) {
   std::vector<Decimal64> draw = {Decimal64(1), Decimal64(2), Decimal64(2389),
                                  Decimal64(4), Decimal64(-12348)};
   std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
-  this->test(precision, draw, valid_bytes);
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
 TEST_P(Decimal128BuilderTest, NoNulls) {
@@ -140,8 +151,8 @@ TEST_P(Decimal128BuilderTest, NoNulls) {
   std::vector<Decimal128> draw = {Decimal128(1), Decimal128(-2), Decimal128(2389),
                                   Decimal128(4), Decimal128(-12348)};
   std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
-  std::vector<uint8_t> sign_bitmap = {false, true, false, false, true};
-  this->test(precision, draw, valid_bytes, sign_bitmap);
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
 TEST_P(Decimal32BuilderTest, WithNulls) {
@@ -149,7 +160,8 @@ TEST_P(Decimal32BuilderTest, WithNulls) {
   std::vector<Decimal32> draw = {Decimal32(1), Decimal32(2), Decimal32(-1), Decimal32(4),
                                  Decimal32(-1)};
   std::vector<uint8_t> valid_bytes = {true, true, false, true, false};
-  this->test(precision, draw, valid_bytes);
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
 TEST_P(Decimal64BuilderTest, WithNulls) {
@@ -157,7 +169,8 @@ TEST_P(Decimal64BuilderTest, WithNulls) {
   std::vector<Decimal64> draw = {Decimal64(-1), Decimal64(2), Decimal64(-1), Decimal64(4),
                                  Decimal64(-1)};
   std::vector<uint8_t> valid_bytes = {true, true, false, true, false};
-  this->test(precision, draw, valid_bytes);
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
 TEST_P(Decimal128BuilderTest, WithNulls) {
@@ -173,9 +186,8 @@ TEST_P(Decimal128BuilderTest, WithNulls) {
                                   Decimal128("-23049302932.235234")};
   std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
                                       true, true, true,  true};
-  std::vector<uint8_t> sign_bitmap = {false, false, false, false, false,
-                                      false, false, false, true};
-  this->test(precision, draw, valid_bytes, sign_bitmap);
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
 INSTANTIATE_TEST_CASE_P(Decimal32BuilderTest, Decimal32BuilderTest,
@@ -185,8 +197,8 @@ INSTANTIATE_TEST_CASE_P(Decimal64BuilderTest, Decimal64BuilderTest,
                         ::testing::Range(DecimalPrecision<int64_t>::minimum,
                                          DecimalPrecision<int64_t>::maximum));
 INSTANTIATE_TEST_CASE_P(Decimal128BuilderTest, Decimal128BuilderTest,
-                        ::testing::Range(DecimalPrecision<int128_t>::minimum,
-                                         DecimalPrecision<int128_t>::maximum));
+                        ::testing::Range(DecimalPrecision<Int128>::minimum,
+                                         DecimalPrecision<Int128>::maximum));
 
 }  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 637eb2417fcfd..9fbadfe95bad6 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -161,7 +161,7 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t le
 
 const uint8_t* PrimitiveArray::raw_values() const {
   return raw_values_ +
-         offset() * static_cast<const FixedWidthType&>(*type()).bit_width() / 8;
+         offset() * static_cast<const FixedWidthType&>(*type()).bit_width() / CHAR_BIT;
 }
 
 template <typename T>
@@ -323,7 +323,6 @@ std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const
 
 FixedSizeBinaryArray::FixedSizeBinaryArray(
     const std::shared_ptr<internal::ArrayData>& data) {
-  DCHECK_EQ(data->type->id(), Type::FIXED_SIZE_BINARY);
   SetData(data);
 }
 
@@ -346,61 +345,30 @@ const uint8_t* FixedSizeBinaryArray::GetValue(int64_t i) const {
 // ----------------------------------------------------------------------
 // Decimal
 
-DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data) {
+DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data)
+    : FixedSizeBinaryArray(data) {
   DCHECK_EQ(data->type->id(), Type::DECIMAL);
-  SetData(data);
-}
-
-void DecimalArray::SetData(const std::shared_ptr<ArrayData>& data) {
-  auto fixed_size_data = data->buffers[1];
-  auto sign_bitmap = data->buffers[2];
-  this->Array::SetData(data);
-
-  raw_values_ = fixed_size_data != nullptr ? fixed_size_data->data() : nullptr;
-  sign_bitmap_data_ = sign_bitmap != nullptr ? sign_bitmap->data() : nullptr;
 }
 
-DecimalArray::DecimalArray(const std::shared_ptr<DataType>& type, int64_t length,
-                           const std::shared_ptr<Buffer>& data,
-                           const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
-                           int64_t offset, const std::shared_ptr<Buffer>& sign_bitmap) {
-  BufferVector buffers = {null_bitmap, data, sign_bitmap};
-  SetData(
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
-}
-
-bool DecimalArray::IsNegative(int64_t i) const {
-  return sign_bitmap_data_ != nullptr ? BitUtil::GetBit(sign_bitmap_data_, i) : false;
-}
-
-const uint8_t* DecimalArray::GetValue(int64_t i) const {
-  return raw_values_ + (i + data_->offset) * byte_width();
-}
+#define DECIMAL_TO_STRING_CASE(bits, bytes, precision, scale) \
+  case bits: {                                                \
+    decimal::Decimal##bits value;                             \
+    decimal::FromBytes((bytes), &value);                      \
+    return decimal::ToString(value, (precision), (scale));    \
+  }
 
 std::string DecimalArray::FormatValue(int64_t i) const {
   const auto& type_ = static_cast<const DecimalType&>(*type());
   const int precision = type_.precision();
   const int scale = type_.scale();
-  const int byte_width = type_.byte_width();
-  const uint8_t* bytes = raw_values_ + (i + data_->offset) * byte_width;
-  switch (byte_width) {
-    case 4: {
-      decimal::Decimal32 value;
-      decimal::FromBytes(bytes, &value);
-      return decimal::ToString(value, precision, scale);
-    }
-    case 8: {
-      decimal::Decimal64 value;
-      decimal::FromBytes(bytes, &value);
-      return decimal::ToString(value, precision, scale);
-    }
-    case 16: {
-      decimal::Decimal128 value;
-      decimal::FromBytes(bytes, IsNegative(i), &value);
-      return decimal::ToString(value, precision, scale);
-    }
+  const int bit_width = type_.bit_width();
+  const uint8_t* bytes = GetValue(i);
+  switch (bit_width) {
+    DECIMAL_TO_STRING_CASE(32, bytes, precision, scale)
+    DECIMAL_TO_STRING_CASE(64, bytes, precision, scale)
+    DECIMAL_TO_STRING_CASE(128, bytes, precision, scale)
     default: {
-      DCHECK(false) << "Invalid byte width: " << byte_width;
+      DCHECK(false) << "Invalid bit width: " << bit_width;
       return "";
     }
   }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 777fbe0b006b3..f9f1e31be8ce7 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -521,8 +521,6 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 
   int32_t byte_width() const { return byte_width_; }
 
-  const uint8_t* raw_values() const { return raw_values_ + byte_width_ * data_->offset; }
-
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 
  protected:
@@ -536,45 +534,18 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 
 // ----------------------------------------------------------------------
 // DecimalArray
-class ARROW_EXPORT DecimalArray : public FlatArray {
+class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray {
  public:
-  using TypeClass = Type;
+  using TypeClass = DecimalType;
+
+  using FixedSizeBinaryArray::FixedSizeBinaryArray;
 
   /// \brief Construct DecimalArray from internal::ArrayData instance
   explicit DecimalArray(const std::shared_ptr<internal::ArrayData>& data);
 
-  DecimalArray(const std::shared_ptr<DataType>& type, int64_t length,
-               const std::shared_ptr<Buffer>& data,
-               const std::shared_ptr<Buffer>& null_bitmap = nullptr,
-               int64_t null_count = 0, int64_t offset = 0,
-               const std::shared_ptr<Buffer>& sign_bitmap = nullptr);
-
-  bool IsNegative(int64_t i) const;
-
-  const uint8_t* GetValue(int64_t i) const;
-
   std::string FormatValue(int64_t i) const;
 
   std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
-  /// \brief The main decimal data
-  /// For 32/64-bit decimal this is everything
-  std::shared_ptr<Buffer> values() const { return data_->buffers[1]; }
-
-  /// Only needed for 128 bit Decimals
-  std::shared_ptr<Buffer> sign_bitmap() const { return data_->buffers[2]; }
-
-  int32_t byte_width() const {
-    return static_cast<const DecimalType&>(*type()).byte_width();
-  }
-
-  /// \brief Return pointer to value data, accounting for any offset
-  const uint8_t* raw_values() const { return raw_values_ + byte_width() * data_->offset; }
-
- private:
-  void SetData(const std::shared_ptr<internal::ArrayData>& data);
-  const uint8_t* raw_values_;
-  const uint8_t* sign_bitmap_data_;
 };
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index e2054dbfde688..5dba0e1c4efb6 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -957,9 +957,7 @@ template class DictionaryBuilder<StringType>;
 // DecimalBuilder
 
 DecimalBuilder::DecimalBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
-    : FixedSizeBinaryBuilder(type, pool),
-      sign_bitmap_(nullptr),
-      sign_bitmap_data_(nullptr) {}
+    : FixedSizeBinaryBuilder(type, pool) {}
 
 #ifndef ARROW_NO_DEPRECATED_API
 DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
@@ -968,9 +966,6 @@ DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>
 
 template <typename T>
 ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal<T>& val) {
-  DCHECK_EQ(sign_bitmap_, nullptr) << "sign_bitmap_ is not null";
-  DCHECK_EQ(sign_bitmap_data_, nullptr) << "sign_bitmap_data_ is not null";
-
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
   return FixedSizeBinaryBuilder::Append(reinterpret_cast<const uint8_t*>(&val.value));
 }
@@ -980,53 +975,11 @@ template ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal64& va
 
 template <>
 ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal128& value) {
-  DCHECK_NE(sign_bitmap_, nullptr) << "sign_bitmap_ is null";
-  DCHECK_NE(sign_bitmap_data_, nullptr) << "sign_bitmap_data_ is null";
-
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
   uint8_t stack_bytes[16] = {0};
   uint8_t* bytes = stack_bytes;
-  bool is_negative;
-  decimal::ToBytes(value, &bytes, &is_negative);
-  RETURN_NOT_OK(FixedSizeBinaryBuilder::Append(bytes));
-
-  // TODO(phillipc): calculate the proper storage size here (do we have a function to do
-  // this)?
-  // TODO(phillipc): Reserve number of elements
-  RETURN_NOT_OK(sign_bitmap_->Reserve(1));
-  BitUtil::SetBitTo(sign_bitmap_data_, length_ - 1, is_negative);
-  return Status::OK();
-}
-
-Status DecimalBuilder::Init(int64_t capacity) {
-  RETURN_NOT_OK(FixedSizeBinaryBuilder::Init(capacity));
-  if (byte_width_ == 16) {
-    RETURN_NOT_OK(AllocateResizableBuffer(pool_, null_bitmap_->size(), &sign_bitmap_));
-    sign_bitmap_data_ = sign_bitmap_->mutable_data();
-    memset(sign_bitmap_data_, 0, static_cast<size_t>(sign_bitmap_->capacity()));
-  }
-  return Status::OK();
-}
-
-Status DecimalBuilder::Resize(int64_t capacity) {
-  int64_t old_bytes = null_bitmap_ != nullptr ? null_bitmap_->size() : 0;
-  if (sign_bitmap_ == nullptr) {
-    return Init(capacity);
-  }
-  RETURN_NOT_OK(FixedSizeBinaryBuilder::Resize(capacity));
-
-  if (byte_width_ == 16) {
-    RETURN_NOT_OK(sign_bitmap_->Resize(null_bitmap_->size()));
-    int64_t new_bytes = sign_bitmap_->size();
-    sign_bitmap_data_ = sign_bitmap_->mutable_data();
-
-    // The buffer might be overpadded to deal with padding according to the spec
-    if (old_bytes < new_bytes) {
-      memset(sign_bitmap_data_ + old_bytes, 0,
-             static_cast<size_t>(sign_bitmap_->capacity() - old_bytes));
-    }
-  }
-  return Status::OK();
+  decimal::ToBytes(value, &bytes);
+  return FixedSizeBinaryBuilder::Append(bytes);
 }
 
 Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
@@ -1034,8 +987,8 @@ Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
   RETURN_NOT_OK(byte_builder_.Finish(&data));
 
   /// TODO(phillipc): not sure where to get the offset argument here
-  *out = std::make_shared<DecimalArray>(type_, length_, data, null_bitmap_, null_count_,
-                                        0, sign_bitmap_);
+  *out =
+      std::make_shared<DecimalArray>(type_, length_, data, null_bitmap_, null_count_, 0);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 4e3cd92da9beb..8e412667dab73 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -704,15 +704,7 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
   template <typename T>
   ARROW_EXPORT Status Append(const decimal::Decimal<T>& val);
 
-  Status Init(int64_t capacity) override;
-  Status Resize(int64_t capacity) override;
   Status Finish(std::shared_ptr<Array>* out) override;
-
- private:
-  /// We only need these for 128 bit decimals, because boost stores the sign
-  /// separate from the underlying bytes.
-  std::shared_ptr<ResizableBuffer> sign_bitmap_;
-  uint8_t* sign_bitmap_data_;
 };
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index c01f190351044..beb22e7126ba9 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -257,42 +257,7 @@ class RangeEqualsVisitor {
   }
 
   Status Visit(const DecimalArray& left) {
-    const auto& right = static_cast<const DecimalArray&>(right_);
-
-    int32_t width = left.byte_width();
-
-    const uint8_t* left_data = nullptr;
-    const uint8_t* right_data = nullptr;
-
-    if (left.values()) {
-      left_data = left.raw_values();
-    }
-
-    if (right.values()) {
-      right_data = right.raw_values();
-    }
-
-    for (int64_t i = left_start_idx_, o_i = right_start_idx_; i < left_end_idx_;
-         ++i, ++o_i) {
-      if (left.IsNegative(i) != right.IsNegative(o_i)) {
-        result_ = false;
-        return Status::OK();
-      }
-
-      const bool is_null = left.IsNull(i);
-      if (is_null != right.IsNull(o_i)) {
-        result_ = false;
-        return Status::OK();
-      }
-      if (is_null) continue;
-
-      if (std::memcmp(left_data + width * i, right_data + width * o_i, width)) {
-        result_ = false;
-        return Status::OK();
-      }
-    }
-    result_ = true;
-    return Status::OK();
+    return Visit(static_cast<const FixedSizeBinaryArray&>(left));
   }
 
   Status Visit(const NullArray& left) {
@@ -346,7 +311,7 @@ class RangeEqualsVisitor {
 
 static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& right) {
   const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
-  const int byte_width = size_meta.bit_width() / 8;
+  const int byte_width = size_meta.bit_width() / CHAR_BIT;
 
   const uint8_t* left_data = nullptr;
   const uint8_t* right_data = nullptr;
@@ -377,70 +342,26 @@ static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& r
 template <typename T>
 static inline bool CompareBuiltIn(const Array& left, const Array& right, const T* ldata,
                                   const T* rdata) {
-  if (left.null_count() > 0) {
-    for (int64_t i = 0; i < left.length(); ++i) {
-      if (left.IsNull(i) != right.IsNull(i)) {
-        return false;
-      } else if (!left.IsNull(i) && (ldata[i] != rdata[i])) {
-        return false;
-      }
-    }
+  if (ldata == nullptr && rdata == nullptr) {
     return true;
-  } else {
-    return memcmp(ldata, rdata, sizeof(T) * left.length()) == 0;
   }
-}
 
-static bool IsEqualDecimal(const DecimalArray& left, const DecimalArray& right) {
-  const int64_t loffset = left.offset();
-  const int64_t roffset = right.offset();
-
-  const uint8_t* left_data = nullptr;
-  const uint8_t* right_data = nullptr;
-
-  if (left.values()) {
-    left_data = left.raw_values();
-  }
-  if (right.values()) {
-    right_data = right.raw_values();
+  if (ldata == nullptr || rdata == nullptr) {
+    return false;
   }
 
-  const int32_t byte_width = left.byte_width();
-  if (byte_width == 4) {
-    return CompareBuiltIn<int32_t>(
-        left, right, reinterpret_cast<const int32_t*>(left_data) + loffset,
-        reinterpret_cast<const int32_t*>(right_data) + roffset);
-  } else if (byte_width == 8) {
-    return CompareBuiltIn<int64_t>(
-        left, right, reinterpret_cast<const int64_t*>(left_data) + loffset,
-        reinterpret_cast<const int64_t*>(right_data) + roffset);
-  } else {
-    // 128-bit
-
-    // Must also compare sign bitmap
-    const uint8_t* left_sign = nullptr;
-    const uint8_t* right_sign = nullptr;
-    if (left.sign_bitmap()) {
-      left_sign = left.sign_bitmap()->data();
-    }
-    if (right.sign_bitmap()) {
-      right_sign = right.sign_bitmap()->data();
-    }
-
+  if (left.null_count() > 0) {
     for (int64_t i = 0; i < left.length(); ++i) {
-      bool left_null = left.IsNull(i);
-      if (!left_null && (memcmp(left_data, right_data, byte_width) || right.IsNull(i))) {
+      if (left.IsNull(i) != right.IsNull(i)) {
         return false;
-      }
-      if (BitUtil::GetBit(left_sign, i + loffset) !=
-          BitUtil::GetBit(right_sign, i + roffset)) {
+      } else if (!left.IsNull(i) && (ldata[i] != rdata[i])) {
         return false;
       }
-      left_data += byte_width;
-      right_data += byte_width;
     }
     return true;
   }
+
+  return memcmp(ldata, rdata, sizeof(T) * left.length()) == 0;
 }
 
 class ArrayEqualsVisitor : public RangeEqualsVisitor {
@@ -485,11 +406,6 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DecimalArray& left) {
-    result_ = IsEqualDecimal(left, static_cast<const DecimalArray&>(right_));
-    return Status::OK();
-  }
-
   template <typename ArrayType>
   bool ValueOffsetsEqual(const ArrayType& left) {
     const auto& right = static_cast<const ArrayType&>(right_);
@@ -591,6 +507,27 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     return Status::OK();
   }
 
+  Status Visit(const DecimalArray& left) {
+    const int byte_width = left.byte_width();
+    if (byte_width == 4) {
+      result_ = CompareBuiltIn<int32_t>(
+          left, right_, reinterpret_cast<const int32_t*>(left.raw_values()),
+          reinterpret_cast<const int32_t*>(
+              static_cast<const DecimalArray&>(right_).raw_values()));
+      return Status::OK();
+    }
+
+    if (byte_width == 8) {
+      result_ = CompareBuiltIn<int64_t>(
+          left, right_, reinterpret_cast<const int64_t*>(left.raw_values()),
+          reinterpret_cast<const int64_t*>(
+              static_cast<const DecimalArray&>(right_).raw_values()));
+      return Status::OK();
+    }
+
+    return RangeEqualsVisitor::Visit(left);
+  }
+
   template <typename T>
   typename std::enable_if<std::is_base_of<NestedType, typename T::TypeClass>::value,
                           Status>::type
@@ -823,7 +760,7 @@ Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
     }
 
     const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
-    const int byte_width = size_meta.bit_width() / 8;
+    const int byte_width = size_meta.bit_width() / CHAR_BIT;
     DCHECK_GT(byte_width, 0);
 
     const uint8_t* left_data = left.data()->data();
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index d1fca70412c42..1f62ef86c0957 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -604,11 +604,12 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
 
 template <typename T>
 Status ValidateDecimalPrecision(int precision) {
-  constexpr static const int maximum_precision = decimal::DecimalPrecision<T>::maximum;
-  if (!(precision > 0 && precision <= maximum_precision)) {
+  constexpr static const int kMaximumPrecision =
+      decimal::DecimalPrecision<typename T::value_type>::maximum;
+  if (!(precision > 0 && precision <= kMaximumPrecision)) {
     std::stringstream ss;
     ss << "Invalid precision: " << precision << ". Minimum is 1, maximum is "
-       << maximum_precision;
+       << kMaximumPrecision;
     return Status::Invalid(ss.str());
   }
   return Status::OK();
@@ -620,27 +621,24 @@ Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
   DCHECK_NE(bytes, nullptr);
   DCHECK_NE(result, nullptr);
   RETURN_NOT_OK(ValidateDecimalPrecision<T>(precision));
-  decimal::Decimal<T> decimal;
-  FromBytes(bytes, &decimal);
-  *result = ToString(decimal, precision, scale);
+  T decimal;
+  decimal::FromBytes(bytes, &decimal);
+  *result = decimal::ToString(decimal, precision, scale);
   return Status::OK();
 }
 
-template Status RawDecimalToString<int32_t>(const uint8_t*, int, int,
-                                            std::string* result);
-template Status RawDecimalToString<int64_t>(const uint8_t*, int, int,
-                                            std::string* result);
-
-Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
-                          bool is_negative, std::string* result) {
-  DCHECK_NE(bytes, nullptr);
-  DCHECK_NE(result, nullptr);
-  RETURN_NOT_OK(ValidateDecimalPrecision<boost::multiprecision::int128_t>(precision));
-  decimal::Decimal128 decimal;
-  FromBytes(bytes, is_negative, &decimal);
-  *result = ToString(decimal, precision, scale);
-  return Status::OK();
-}
+template Status RawDecimalToString<decimal::Decimal32>(const uint8_t*, int, int,
+                                                       std::string*);
+template Status RawDecimalToString<decimal::Decimal64>(const uint8_t*, int, int,
+                                                       std::string*);
+template Status RawDecimalToString<decimal::Decimal128>(const uint8_t*, int, int,
+                                                        std::string*);
+
+#define RAW_DECIMAL_TO_STRING_CASE(bits, value, precision, scale, output)          \
+  case bits:                                                                       \
+    RETURN_NOT_OK(RawDecimalToString<decimal::Decimal##bits>((value), (precision), \
+                                                             (scale), (output)));  \
+    break;
 
 static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
                               PyObject** out_values) {
@@ -664,22 +662,18 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
         *out_values++ = Py_None;
       } else {
         const uint8_t* raw_value = arr->GetValue(i);
-        std::string s;
+        std::string decimal_string;
         switch (bit_width) {
-          case 32:
-            RETURN_NOT_OK(RawDecimalToString<int32_t>(raw_value, precision, scale, &s));
-            break;
-          case 64:
-            RETURN_NOT_OK(RawDecimalToString<int64_t>(raw_value, precision, scale, &s));
-            break;
-          case 128:
-            RETURN_NOT_OK(
-                RawDecimalToString(raw_value, precision, scale, arr->IsNegative(i), &s));
-            break;
-          default:
-            break;
+          RAW_DECIMAL_TO_STRING_CASE(32, raw_value, precision, scale, &decimal_string)
+          RAW_DECIMAL_TO_STRING_CASE(64, raw_value, precision, scale, &decimal_string)
+          RAW_DECIMAL_TO_STRING_CASE(128, raw_value, precision, scale, &decimal_string)
+          default: {
+            std::stringstream buf;
+            buf << "Invalid bit_width " << bit_width << " for decimal value";
+            return Status::Invalid(buf.str());
+          }
         }
-        RETURN_NOT_OK(DecimalFromString(Decimal, s, out_values++));
+        RETURN_NOT_OK(DecimalFromString(Decimal, decimal_string, out_values++));
       }
     }
   }
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 0d830127ee9b0..0241ff0579563 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -23,12 +23,10 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
-#include "arrow/table.h"
 #include "arrow/test-util.h"
 
 #include "arrow/python/arrow_to_pandas.h"
 #include "arrow/python/builtin_convert.h"
-#include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 
 #include "arrow/util/decimal.h"
@@ -61,13 +59,11 @@ TEST(DecimalTest, TestPythonDecimalToString) {
   ASSERT_NE(pydecimal.obj(), nullptr);
   ASSERT_EQ(PyErr_Occurred(), nullptr);
 
-  boost::multiprecision::int128_t boost_decimal(decimal_string);
   PyObject* python_object = pydecimal.obj();
   ASSERT_NE(python_object, nullptr);
 
   std::string string_result;
   ASSERT_OK(PythonDecimalToString(python_object, &string_result));
-  ASSERT_EQ(boost_decimal.str(), string_result);
 }
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 4443e8d8a455b..82cd13713763d 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -480,10 +480,6 @@ std::vector<BufferDescr> FixedSizeBinaryType::GetBufferLayout() const {
   return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
 }
 
-std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
-  return {kValidityBuffer, kBooleanBuffer, BufferDescr(BufferType::DATA, bit_width())};
-}
-
 std::vector<BufferDescr> ListType::GetBufferLayout() const {
   return {kValidityBuffer, kOffsetBuffer};
 }
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 4917ebb481368..30cd71e6c5ca0 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -477,7 +477,6 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
         precision_(precision),
         scale_(scale) {}
 
-  std::vector<BufferDescr> GetBufferLayout() const override;
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "decimal"; }
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 8c1a23d62f0cd..070582008aec0 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -21,19 +21,20 @@
 
 # Headers: top level
 install(FILES
-  bit-util.h
   bit-stream-utils.h
+  bit-util.h
   bpacking.h
   compiler-util.h
-  compression.h
   compression_brotli.h
+  compression.h
   compression_lz4.h
   compression_snappy.h
   compression_zlib.h
   compression_zstd.h
   cpu-info.h
-  key_value_metadata.h
   hash-util.h
+  int128.h
+  key_value_metadata.h
   logging.h
   macros.h
   random.h
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index e3fd4802627ae..fb8833c9af2cf 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -28,27 +28,27 @@ namespace decimal {
 template <typename T>
 class DecimalTest : public ::testing::Test {
  public:
-  DecimalTest() : string_value("234.23445") { integer_value.value = 23423445; }
-  Decimal<T> integer_value;
-  std::string string_value;
+  DecimalTest() : decimal_value_(23423445), string_value_("234.23445") {}
+  Decimal<T> decimal_value_;
+  std::string string_value_;
 };
 
-typedef ::testing::Types<int32_t, int64_t, int128_t> DecimalTypes;
+typedef ::testing::Types<int32_t, int64_t, Int128> DecimalTypes;
 TYPED_TEST_CASE(DecimalTest, DecimalTypes);
 
 TYPED_TEST(DecimalTest, TestToString) {
-  Decimal<TypeParam> decimal(this->integer_value);
+  Decimal<TypeParam> decimal(this->decimal_value_);
   int precision = 8;
   int scale = 5;
   std::string result = ToString(decimal, precision, scale);
-  ASSERT_EQ(result, this->string_value);
+  ASSERT_EQ(result, this->string_value_);
 }
 
 TYPED_TEST(DecimalTest, TestFromString) {
-  Decimal<TypeParam> expected(this->integer_value);
+  Decimal<TypeParam> expected(this->decimal_value_);
   Decimal<TypeParam> result;
   int precision, scale;
-  ASSERT_OK(FromString(this->string_value, &result, &precision, &scale));
+  ASSERT_OK(FromString(this->string_value_, &result, &precision, &scale));
   ASSERT_EQ(result.value, expected.value);
   ASSERT_EQ(precision, 8);
   ASSERT_EQ(scale, 5);
@@ -67,7 +67,7 @@ TEST(DecimalTest, TestStringStartingWithPlus) {
 
 TEST(DecimalTest, TestStringStartingWithPlus128) {
   std::string plus_value("+2342394230592.232349023094");
-  decimal::int128_t expected_value("2342394230592232349023094");
+  Int128 expected_value("2342394230592232349023094");
   Decimal128 out;
   int scale;
   int precision;
@@ -90,29 +90,30 @@ TEST(DecimalTest, TestStringToInt64) {
 }
 
 TEST(DecimalTest, TestStringToInt128) {
-  int128_t value = 0;
+  Int128 value = 0;
   StringToInteger("123456789", "456789123", 1, &value);
-  ASSERT_EQ(value, 123456789456789123);
+  ASSERT_EQ(value.high_bits(), 0);
+  ASSERT_EQ(value.low_bits(), 123456789456789123);
 }
 
 TEST(DecimalTest, TestFromString128) {
   static const std::string string_value("-23049223942343532412");
-  Decimal<int128_t> result(string_value);
-  int128_t expected = -230492239423435324;
+  Decimal128 result(string_value);
+  Int128 expected(static_cast<int64_t>(-230492239423435324));
   ASSERT_EQ(result.value, expected * 100 - 12);
 
   // Sanity check that our number is actually using more than 64 bits
-  ASSERT_NE(result.value, static_cast<int64_t>(result.value));
+  ASSERT_NE(result.value.high_bits(), 0);
 }
 
 TEST(DecimalTest, TestFromDecimalString128) {
   static const std::string string_value("-23049223942343.532412");
-  Decimal<int128_t> result(string_value);
-  int128_t expected = -230492239423435324;
+  Decimal128 result(string_value);
+  Int128 expected(static_cast<int64_t>(-230492239423435324));
   ASSERT_EQ(result.value, expected * 100 - 12);
 
   // Sanity check that our number is actually using more than 64 bits
-  ASSERT_NE(result.value, static_cast<int64_t>(result.value));
+  ASSERT_NE(result.value.high_bits(), 0);
 }
 
 TEST(DecimalTest, TestDecimal32Precision) {
@@ -130,8 +131,8 @@ TEST(DecimalTest, TestDecimal64Precision) {
 }
 
 TEST(DecimalTest, TestDecimal128Precision) {
-  auto min_precision = DecimalPrecision<int128_t>::minimum;
-  auto max_precision = DecimalPrecision<int128_t>::maximum;
+  auto min_precision = DecimalPrecision<Int128>::minimum;
+  auto max_precision = DecimalPrecision<Int128>::maximum;
   ASSERT_EQ(min_precision, 19);
   ASSERT_EQ(max_precision, 38);
 }
@@ -166,19 +167,16 @@ TEST(DecimalTest, TestDecimal128StringAndBytesRoundTrip) {
   Decimal128 expected(string_value);
 
   std::string expected_string_value("-340282366920938463463374607431711455");
-  int128_t expected_underlying_value(expected_string_value);
+  Int128 expected_underlying_value(expected_string_value);
 
   ASSERT_EQ(expected.value, expected_underlying_value);
 
   uint8_t stack_bytes[16] = {0};
   uint8_t* bytes = stack_bytes;
-  bool is_negative;
-  ToBytes(expected, &bytes, &is_negative);
-
-  ASSERT_TRUE(is_negative);
+  ToBytes(expected, &bytes);
 
   Decimal128 result;
-  FromBytes(bytes, is_negative, &result);
+  FromBytes(bytes, &result);
 
   ASSERT_EQ(expected.value, result.value);
 }
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 1a12e20f9f93f..c195cf3e2c5e3 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <sstream>
+
 #include "arrow/util/decimal.h"
 
 namespace arrow {
@@ -156,8 +158,9 @@ void StringToInteger(const std::string& whole, const std::string& fractional, in
   DCHECK(sign == -1 || sign == 1);
   DCHECK_NE(out, nullptr);
   DCHECK(!whole.empty() || !fractional.empty());
+
   if (!whole.empty()) {
-    *out = std::stoi(whole, nullptr, 10) *
+    *out = std::stoi(whole) *
            static_cast<int32_t>(pow(10.0, static_cast<double>(fractional.size())));
   }
   if (!fractional.empty()) {
@@ -172,7 +175,7 @@ void StringToInteger(const std::string& whole, const std::string& fractional, in
   DCHECK_NE(out, nullptr);
   DCHECK(!whole.empty() || !fractional.empty());
   if (!whole.empty()) {
-    *out = static_cast<int64_t>(std::stoll(whole, nullptr, 10)) *
+    *out = static_cast<int64_t>(std::stoll(whole)) *
            static_cast<int64_t>(pow(10.0, static_cast<double>(fractional.size())));
   }
   if (!fractional.empty()) {
@@ -182,11 +185,11 @@ void StringToInteger(const std::string& whole, const std::string& fractional, in
 }
 
 void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
-                     int128_t* out) {
+                     Int128* out) {
   DCHECK(sign == -1 || sign == 1);
   DCHECK_NE(out, nullptr);
   DCHECK(!whole.empty() || !fractional.empty());
-  *out = int128_t(whole + fractional) * sign;
+  *out = Int128(whole + fractional) * sign;
 }
 
 void FromBytes(const uint8_t* bytes, Decimal32* decimal) {
@@ -201,22 +204,8 @@ void FromBytes(const uint8_t* bytes, Decimal64* decimal) {
   decimal->value = *reinterpret_cast<const int64_t*>(bytes);
 }
 
-constexpr static const size_t BYTES_IN_128_BITS = 128 / CHAR_BIT;
-constexpr static const size_t LIMB_SIZE =
-    sizeof(std::remove_pointer<int128_t::backend_type::limb_pointer>::type);
-constexpr static const size_t LIMBS_IN_INT128 = BYTES_IN_128_BITS / LIMB_SIZE;
-
-void FromBytes(const uint8_t* bytes, bool is_negative, Decimal128* decimal) {
-  DCHECK_NE(bytes, nullptr);
-  DCHECK_NE(decimal, nullptr);
-
-  auto& decimal_value(decimal->value);
-  int128_t::backend_type& backend(decimal_value.backend());
-  backend.resize(LIMBS_IN_INT128, LIMBS_IN_INT128);
-  std::memcpy(backend.limbs(), bytes, BYTES_IN_128_BITS);
-  if (is_negative) {
-    decimal->value = -decimal->value;
-  }
+void FromBytes(const uint8_t* bytes, Decimal128* decimal) {
+  decimal->value = Int128(bytes);
 }
 
 void ToBytes(const Decimal32& value, uint8_t** bytes) {
@@ -229,16 +218,10 @@ void ToBytes(const Decimal64& value, uint8_t** bytes) {
   *reinterpret_cast<int64_t*>(*bytes) = value.value;
 }
 
-void ToBytes(const Decimal128& decimal, uint8_t** bytes, bool* is_negative) {
+void ToBytes(const Decimal128& decimal, uint8_t** bytes) {
+  DCHECK_NE(bytes, nullptr);
   DCHECK_NE(*bytes, nullptr);
-  DCHECK_NE(is_negative, nullptr);
-
-  /// TODO(phillipc): boost multiprecision is unreliable here, int128_t can't be
-  /// roundtripped
-  const auto& backend(decimal.value.backend());
-  const size_t bytes_in_use = LIMB_SIZE * backend.size();
-  std::memcpy(*bytes, backend.limbs(), bytes_in_use);
-  *is_negative = backend.isneg();
+  decimal.value.ToBytes(bytes);
 }
 
 }  // namespace decimal
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 20142faea3ec5..88ac2f53fa57a 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -25,15 +25,12 @@
 
 #include "arrow/status.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 
-#include <boost/multiprecision/cpp_int.hpp>
-
 namespace arrow {
 namespace decimal {
 
-using boost::multiprecision::int128_t;
-
 template <typename T>
 struct ARROW_EXPORT Decimal;
 
@@ -42,7 +39,7 @@ ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& f
 ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
                                   int8_t sign, int64_t* out);
 ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
-                                  int8_t sign, int128_t* out);
+                                  int8_t sign, Int128* out);
 
 template <typename T>
 ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out,
@@ -61,7 +58,7 @@ struct ARROW_EXPORT Decimal {
 
 using Decimal32 = Decimal<int32_t>;
 using Decimal64 = Decimal<int64_t>;
-using Decimal128 = Decimal<int128_t>;
+using Decimal128 = Decimal<Int128>;
 
 template <typename T>
 struct ARROW_EXPORT DecimalPrecision {};
@@ -79,7 +76,7 @@ struct ARROW_EXPORT DecimalPrecision<int64_t> {
 };
 
 template <>
-struct ARROW_EXPORT DecimalPrecision<int128_t> {
+struct ARROW_EXPORT DecimalPrecision<Int128> {
   constexpr static const int minimum = 19;
   constexpr static const int maximum = 38;
 };
@@ -108,8 +105,8 @@ ARROW_EXPORT std::string ToString(const Decimal<T>& decimal_value, int precision
   if (scale > 0) {
     int remaining_scale = scale;
     do {
-      str[--last_char_idx] = static_cast<char>((remaining_value % 10) +
-                                               static_cast<T>('0'));  // Ascii offset
+      str[--last_char_idx] =
+          static_cast<char>(remaining_value % 10 + static_cast<T>('0'));  // Ascii offset
       remaining_value /= 10;
     } while (--remaining_scale > 0);
     str[--last_char_idx] = '.';
@@ -117,11 +114,14 @@ ARROW_EXPORT std::string ToString(const Decimal<T>& decimal_value, int precision
   }
   do {
     str[--last_char_idx] =
-        static_cast<char>((remaining_value % 10) + static_cast<T>('0'));  // Ascii offset
+        static_cast<char>(remaining_value % 10 + static_cast<T>('0'));  // Ascii offset
     remaining_value /= 10;
     if (remaining_value == 0) {
       // Trim any extra leading 0's.
-      if (last_char_idx > first_digit_idx) str.erase(0, last_char_idx - first_digit_idx);
+      if (last_char_idx > first_digit_idx) {
+        str.erase(0, last_char_idx - first_digit_idx);
+      }
+
       break;
     }
     // For safety, enforce string length independent of remaining_value.
@@ -133,12 +133,12 @@ ARROW_EXPORT std::string ToString(const Decimal<T>& decimal_value, int precision
 /// Conversion from raw bytes to a Decimal value
 ARROW_EXPORT void FromBytes(const uint8_t* bytes, Decimal32* value);
 ARROW_EXPORT void FromBytes(const uint8_t* bytes, Decimal64* value);
-ARROW_EXPORT void FromBytes(const uint8_t* bytes, bool is_negative, Decimal128* decimal);
+ARROW_EXPORT void FromBytes(const uint8_t* bytes, Decimal128* decimal);
 
 /// Conversion from a Decimal value to raw bytes
 ARROW_EXPORT void ToBytes(const Decimal32& value, uint8_t** bytes);
 ARROW_EXPORT void ToBytes(const Decimal64& value, uint8_t** bytes);
-ARROW_EXPORT void ToBytes(const Decimal128& decimal, uint8_t** bytes, bool* is_negative);
+ARROW_EXPORT void ToBytes(const Decimal128& decimal, uint8_t** bytes);
 
 }  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/int128.cc b/cpp/src/arrow/util/int128.cc
new file mode 100644
index 0000000000000..9d1d06276e05c
--- /dev/null
+++ b/cpp/src/arrow/util/int128.cc
@@ -0,0 +1,527 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <cmath>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#pragma intrinsic(_BitScanReverse)
+#endif
+
+#include "arrow/util/int128.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+namespace decimal {
+
+static constexpr uint64_t kIntMask = 0xFFFFFFFF;
+static constexpr auto kCarryBit = static_cast<uint64_t>(1) << static_cast<uint64_t>(32);
+
+Int128::Int128(const std::string& str) : Int128() {
+  const size_t length = str.length();
+
+  if (length > 0) {
+    bool is_negative = str[0] == '-';
+    auto posn = static_cast<size_t>(is_negative);
+
+    while (posn < length) {
+      const size_t group = std::min(static_cast<size_t>(18), length - posn);
+      const auto chunk = static_cast<int64_t>(std::stoll(str.substr(posn, group)));
+      const auto multiple =
+          static_cast<int64_t>(std::pow(10.0, static_cast<double>(group)));
+
+      *this *= multiple;
+      *this += chunk;
+
+      posn += group;
+    }
+
+    if (is_negative) {
+      Negate();
+    }
+  }
+}
+
+Int128::Int128(const uint8_t* bytes)
+    : Int128(reinterpret_cast<const int64_t*>(bytes)[0],
+             reinterpret_cast<const uint64_t*>(bytes)[1]) {}
+
+void Int128::ToBytes(uint8_t** out) const {
+  DCHECK_NE(out, nullptr) << "Cannot fill nullptr of bytes from Int128";
+  DCHECK_NE(*out, nullptr) << "Cannot fill nullptr of bytes from Int128";
+  const uint64_t raw[] = {static_cast<uint64_t>(high_bits_), low_bits_};
+  std::memcpy(*out, raw, 16);
+}
+
+Int128& Int128::Negate() {
+  low_bits_ = ~low_bits_ + 1;
+  high_bits_ = ~high_bits_;
+  if (low_bits_ == 0) {
+    ++high_bits_;
+  }
+  return *this;
+}
+
+Int128& Int128::operator+=(const Int128& right) {
+  const uint64_t sum = low_bits_ + right.low_bits_;
+  high_bits_ += right.high_bits_;
+  if (sum < low_bits_) {
+    ++high_bits_;
+  }
+  low_bits_ = sum;
+  return *this;
+}
+
+Int128& Int128::operator-=(const Int128& right) {
+  const uint64_t diff = low_bits_ - right.low_bits_;
+  high_bits_ -= right.high_bits_;
+  if (diff > low_bits_) {
+    --high_bits_;
+  }
+  low_bits_ = diff;
+  return *this;
+}
+
+Int128& Int128::operator/=(const Int128& right) {
+  Int128 remainder;
+  DCHECK(Divide(right, this, &remainder).ok());
+  return *this;
+}
+
+Int128::operator char() const {
+  DCHECK(high_bits_ == 0 || high_bits_ == -1)
+      << "Trying to cast an Int128 greater than the value range of a "
+         "char. high_bits_ must be equal to 0 or -1, got: "
+      << high_bits_;
+  DCHECK_LE(low_bits_, std::numeric_limits<char>::max())
+      << "low_bits_ too large for C type char, got: " << low_bits_;
+  return static_cast<char>(low_bits_);
+}
+
+Int128& Int128::operator|=(const Int128& right) {
+  low_bits_ |= right.low_bits_;
+  high_bits_ |= right.high_bits_;
+  return *this;
+}
+
+Int128& Int128::operator&=(const Int128& right) {
+  low_bits_ &= right.low_bits_;
+  high_bits_ &= right.high_bits_;
+  return *this;
+}
+
+Int128& Int128::operator<<=(uint32_t bits) {
+  if (bits != 0) {
+    if (bits < 64) {
+      high_bits_ <<= bits;
+      high_bits_ |= (low_bits_ >> (64 - bits));
+      low_bits_ <<= bits;
+    } else if (bits < 128) {
+      high_bits_ = static_cast<int64_t>(low_bits_) << (bits - 64);
+      low_bits_ = 0;
+    } else {
+      high_bits_ = 0;
+      low_bits_ = 0;
+    }
+  }
+  return *this;
+}
+
+Int128& Int128::operator>>=(uint32_t bits) {
+  if (bits != 0) {
+    if (bits < 64) {
+      low_bits_ >>= bits;
+      low_bits_ |= static_cast<uint64_t>(high_bits_ << (64 - bits));
+      high_bits_ = static_cast<int64_t>(static_cast<uint64_t>(high_bits_) >> bits);
+    } else if (bits < 128) {
+      low_bits_ = static_cast<uint64_t>(high_bits_ >> (bits - 64));
+      high_bits_ = static_cast<int64_t>(high_bits_ >= 0L ? 0L : -1L);
+    } else {
+      high_bits_ = static_cast<int64_t>(high_bits_ >= 0L ? 0L : -1L);
+      low_bits_ = static_cast<uint64_t>(high_bits_);
+    }
+  }
+  return *this;
+}
+
+Int128& Int128::operator*=(const Int128& right) {
+  // Break the left and right numbers into 32 bit chunks
+  // so that we can multiply them without overflow.
+  const uint64_t L0 = static_cast<uint64_t>(high_bits_) >> 32;
+  const uint64_t L1 = static_cast<uint64_t>(high_bits_) & kIntMask;
+  const uint64_t L2 = low_bits_ >> 32;
+  const uint64_t L3 = low_bits_ & kIntMask;
+
+  const uint64_t R0 = static_cast<uint64_t>(right.high_bits_) >> 32;
+  const uint64_t R1 = static_cast<uint64_t>(right.high_bits_) & kIntMask;
+  const uint64_t R2 = right.low_bits_ >> 32;
+  const uint64_t R3 = right.low_bits_ & kIntMask;
+
+  uint64_t product = L3 * R3;
+  low_bits_ = product & kIntMask;
+
+  uint64_t sum = product >> 32;
+
+  product = L2 * R3;
+  sum += product;
+
+  product = L3 * R2;
+  sum += product;
+
+  low_bits_ += sum << 32;
+
+  high_bits_ = static_cast<int64_t>(sum < product ? kCarryBit : 0);
+  if (sum < product) {
+    high_bits_ += kCarryBit;
+  }
+
+  high_bits_ += static_cast<int64_t>(sum >> 32);
+  high_bits_ += L1 * R3 + L2 * R2 + L3 * R1;
+  high_bits_ += (L0 * R3 + L1 * R2 + L2 * R1 + L3 * R0) << 32;
+  return *this;
+}
+
+/// Expands the given value into an array of ints so that we can work on
+/// it. The array will be converted to an absolute value and the wasNegative
+/// flag will be set appropriately. The array will remove leading zeros from
+/// the value.
+/// \param array an array of length 4 to set with the value
+/// \param was_negative a flag for whether the value was original negative
+/// \result the output length of the array
+static int64_t FillInArray(const Int128& value, uint32_t* array, bool& was_negative) {
+  uint64_t high;
+  uint64_t low;
+  const int64_t highbits = value.high_bits();
+  const uint64_t lowbits = value.low_bits();
+
+  if (highbits < 0) {
+    low = ~lowbits + 1;
+    high = static_cast<uint64_t>(~highbits);
+    if (low == 0) {
+      ++high;
+    }
+    was_negative = true;
+  } else {
+    low = lowbits;
+    high = static_cast<uint64_t>(highbits);
+    was_negative = false;
+  }
+
+  if (high != 0) {
+    if (high > std::numeric_limits<uint32_t>::max()) {
+      array[0] = static_cast<uint32_t>(high >> 32);
+      array[1] = static_cast<uint32_t>(high);
+      array[2] = static_cast<uint32_t>(low >> 32);
+      array[3] = static_cast<uint32_t>(low);
+      return 4;
+    }
+
+    array[0] = static_cast<uint32_t>(high);
+    array[1] = static_cast<uint32_t>(low >> 32);
+    array[2] = static_cast<uint32_t>(low);
+    return 3;
+  }
+
+  if (low >= std::numeric_limits<uint32_t>::max()) {
+    array[0] = static_cast<uint32_t>(low >> 32);
+    array[1] = static_cast<uint32_t>(low);
+    return 2;
+  }
+
+  if (low == 0) {
+    return 0;
+  }
+
+  array[0] = static_cast<uint32_t>(low);
+  return 1;
+}
+
+/// \brief Find last set bit in a 32 bit integer. Bit 1 is the LSB and bit 32 is the MSB.
+static int64_t FindLastSetBit(uint32_t value) {
+#if defined(__clang__) || defined(__GNUC__)
+  // Count leading zeros
+  return __builtin_clz(value) + 1;
+#elif defined(_MSC_VER)
+  unsigned long index;                                         // NOLINT
+  _BitScanReverse(&index, static_cast<unsigned long>(value));  // NOLINT
+  return static_cast<int64_t>(index + 1UL);
+#endif
+}
+
+/// Shift the number in the array left by bits positions.
+/// \param array the number to shift, must have length elements
+/// \param length the number of entries in the array
+/// \param bits the number of bits to shift (0 <= bits < 32)
+static void ShiftArrayLeft(uint32_t* array, int64_t length, int64_t bits) {
+  if (length > 0 && bits != 0) {
+    for (int64_t i = 0; i < length - 1; ++i) {
+      array[i] = (array[i] << bits) | (array[i + 1] >> (32 - bits));
+    }
+    array[length - 1] <<= bits;
+  }
+}
+
+/// Shift the number in the array right by bits positions.
+/// \param array the number to shift, must have length elements
+/// \param length the number of entries in the array
+/// \param bits the number of bits to shift (0 <= bits < 32)
+static void ShiftArrayRight(uint32_t* array, int64_t length, int64_t bits) {
+  if (length > 0 && bits != 0) {
+    for (int64_t i = length - 1; i > 0; --i) {
+      array[i] = (array[i] >> bits) | (array[i - 1] << (32 - bits));
+    }
+    array[0] >>= bits;
+  }
+}
+
+/// \brief Fix the signs of the result and remainder at the end of the division based on
+/// the signs of the dividend and divisor.
+static void FixDivisionSigns(Int128* result, Int128* remainder,
+                             bool dividend_was_negative, bool divisor_was_negative) {
+  if (dividend_was_negative != divisor_was_negative) {
+    result->Negate();
+  }
+
+  if (dividend_was_negative) {
+    remainder->Negate();
+  }
+}
+
+/// \brief Build a Int128 from a list of ints.
+static Status BuildFromArray(Int128* value, uint32_t* array, int64_t length) {
+  switch (length) {
+    case 0:
+      *value = {static_cast<int64_t>(0)};
+      break;
+    case 1:
+      *value = {static_cast<int64_t>(array[0])};
+      break;
+    case 2:
+      *value = {static_cast<int64_t>(0),
+                (static_cast<uint64_t>(array[0]) << 32) + array[1]};
+      break;
+    case 3:
+      *value = {static_cast<int64_t>(array[0]),
+                (static_cast<uint64_t>(array[1]) << 32) + array[2]};
+      break;
+    case 4:
+      *value = {(static_cast<int64_t>(array[0]) << 32) + array[1],
+                (static_cast<uint64_t>(array[2]) << 32) + array[3]};
+      break;
+    case 5:
+      if (array[0] != 0) {
+        return Status::Invalid("Can't build Int128 with 5 ints.");
+      }
+      *value = {(static_cast<int64_t>(array[1]) << 32) + array[2],
+                (static_cast<uint64_t>(array[3]) << 32) + array[4]};
+      break;
+    default:
+      return Status::Invalid("Unsupported length for building Int128");
+  }
+
+  return Status::OK();
+}
+
+/// \brief Do a division where the divisor fits into a single 32 bit value.
+static Status SingleDivide(const uint32_t* dividend, int64_t dividend_length,
+                           uint32_t divisor, Int128* remainder,
+                           bool dividend_was_negative, bool divisor_was_negative,
+                           Int128* result) {
+  uint64_t r = 0;
+  uint32_t result_array[5];
+  for (int64_t j = 0; j < dividend_length; j++) {
+    r <<= 32;
+    r += dividend[j];
+    result_array[j] = static_cast<uint32_t>(r / divisor);
+    r %= divisor;
+  }
+  RETURN_NOT_OK(BuildFromArray(result, result_array, dividend_length));
+  *remainder = static_cast<int64_t>(r);
+  FixDivisionSigns(result, remainder, dividend_was_negative, divisor_was_negative);
+  return Status::OK();
+}
+
+Status Int128::Divide(const Int128& divisor, Int128* result, Int128* remainder) const {
+  // Split the dividend and divisor into integer pieces so that we can
+  // work on them.
+  uint32_t dividend_array[5];
+  uint32_t divisor_array[4];
+  bool dividend_was_negative;
+  bool divisor_was_negative;
+  // leave an extra zero before the dividend
+  dividend_array[0] = 0;
+  int64_t dividend_length =
+      FillInArray(*this, dividend_array + 1, dividend_was_negative) + 1;
+  int64_t divisor_length = FillInArray(divisor, divisor_array, divisor_was_negative);
+
+  // Handle some of the easy cases.
+  if (dividend_length <= divisor_length) {
+    *remainder = *this;
+    *result = 0;
+    return Status::OK();
+  }
+
+  if (divisor_length == 0) {
+    return Status::Invalid("Division by 0 in Int128");
+  }
+
+  if (divisor_length == 1) {
+    return SingleDivide(dividend_array, dividend_length, divisor_array[0], remainder,
+                        dividend_was_negative, divisor_was_negative, result);
+  }
+
+  int64_t result_length = dividend_length - divisor_length;
+  uint32_t result_array[4];
+
+  // Normalize by shifting both by a multiple of 2 so that
+  // the digit guessing is better. The requirement is that
+  // divisor_array[0] is greater than 2**31.
+  int64_t normalize_bits = 32 - FindLastSetBit(divisor_array[0]);
+  ShiftArrayLeft(divisor_array, divisor_length, normalize_bits);
+  ShiftArrayLeft(dividend_array, dividend_length, normalize_bits);
+
+  // compute each digit in the result
+  for (int64_t j = 0; j < result_length; ++j) {
+    // Guess the next digit. At worst it is two too large
+    uint32_t guess = std::numeric_limits<uint32_t>::max();
+    auto high_dividend =
+        static_cast<uint64_t>(dividend_array[j]) << 32 | dividend_array[j + 1];
+    if (dividend_array[j] != divisor_array[0]) {
+      guess = static_cast<uint32_t>(high_dividend / divisor_array[0]);
+    }
+
+    // catch all of the cases where guess is two too large and most of the
+    // cases where it is one too large
+    auto rhat = static_cast<uint32_t>(high_dividend -
+                                      guess * static_cast<uint64_t>(divisor_array[0]));
+    while (static_cast<uint64_t>(divisor_array[1]) * guess >
+           (static_cast<uint64_t>(rhat) << 32) + dividend_array[j + 2]) {
+      --guess;
+      rhat += divisor_array[0];
+      if (static_cast<uint64_t>(rhat) < divisor_array[0]) {
+        break;
+      }
+    }
+
+    // subtract off the guess * divisor from the dividend
+    uint64_t mult = 0;
+    for (int64_t i = divisor_length - 1; i >= 0; --i) {
+      mult += static_cast<uint64_t>(guess) * divisor_array[i];
+      uint32_t prev = dividend_array[j + i + 1];
+      dividend_array[j + i + 1] -= static_cast<uint32_t>(mult);
+      mult >>= 32;
+      if (dividend_array[j + i + 1] > prev) {
+        ++mult;
+      }
+    }
+    uint32_t prev = dividend_array[j];
+    dividend_array[j] -= static_cast<uint32_t>(mult);
+
+    // if guess was too big, we add back divisor
+    if (dividend_array[j] > prev) {
+      --guess;
+
+      uint32_t carry = 0;
+      for (int64_t i = divisor_length - 1; i >= 0; --i) {
+        uint64_t sum =
+            static_cast<uint64_t>(divisor_array[i]) + dividend_array[j + i + 1] + carry;
+        dividend_array[j + i + 1] = static_cast<uint32_t>(sum);
+        carry = static_cast<uint32_t>(sum >> 32);
+      }
+      dividend_array[j] += carry;
+    }
+
+    result_array[j] = guess;
+  }
+
+  // denormalize the remainder
+  ShiftArrayRight(dividend_array, dividend_length, normalize_bits);
+
+  // return result and remainder
+  RETURN_NOT_OK(BuildFromArray(result, result_array, result_length));
+  RETURN_NOT_OK(BuildFromArray(remainder, dividend_array, dividend_length));
+  FixDivisionSigns(result, remainder, dividend_was_negative, divisor_was_negative);
+  return Status::OK();
+}
+
+bool operator==(const Int128& left, const Int128& right) {
+  return left.high_bits() == right.high_bits() && left.low_bits() == right.low_bits();
+}
+
+bool operator!=(const Int128& left, const Int128& right) {
+  return !operator==(left, right);
+}
+
+bool operator<(const Int128& left, const Int128& right) {
+  return left.high_bits() < right.high_bits() ||
+         (left.high_bits() == right.high_bits() && left.low_bits() < right.low_bits());
+}
+
+bool operator<=(const Int128& left, const Int128& right) {
+  return !operator>(left, right);
+}
+
+bool operator>(const Int128& left, const Int128& right) { return operator<(right, left); }
+
+bool operator>=(const Int128& left, const Int128& right) {
+  return !operator<(left, right);
+}
+
+Int128 operator-(const Int128& operand) {
+  Int128 result(operand.high_bits(), operand.low_bits());
+  return result.Negate();
+}
+
+Int128 operator+(const Int128& left, const Int128& right) {
+  Int128 result(left.high_bits(), left.low_bits());
+  result += right;
+  return result;
+}
+
+Int128 operator-(const Int128& left, const Int128& right) {
+  Int128 result(left.high_bits(), left.low_bits());
+  result -= right;
+  return result;
+}
+
+Int128 operator*(const Int128& left, const Int128& right) {
+  Int128 result(left.high_bits(), left.low_bits());
+  result *= right;
+  return result;
+}
+
+Int128 operator/(const Int128& left, const Int128& right) {
+  Int128 remainder;
+  Int128 result;
+  DCHECK(left.Divide(right, &result, &remainder).ok());
+  return result;
+}
+
+Int128 operator%(const Int128& left, const Int128& right) {
+  Int128 remainder;
+  Int128 result;
+  DCHECK(left.Divide(right, &result, &remainder).ok());
+  return remainder;
+}
+
+}  // namespace decimal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/int128.h b/cpp/src/arrow/util/int128.h
new file mode 100644
index 0000000000000..1d4bd4007a52e
--- /dev/null
+++ b/cpp/src/arrow/util/int128.h
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ARROW_INT128_H
+#define ARROW_INT128_H
+
+#include <string>
+
+#include "arrow/status.h"
+
+namespace arrow {
+namespace decimal {
+
+/// Represents a signed 128-bit integer in two's complement.
+/// Calculations wrap around and overflow is ignored.
+///
+/// For a discussion of the algorithms, look at Knuth's volume 2,
+/// Semi-numerical Algorithms section 4.3.1.
+///
+/// Adapted from the Apache ORC C++ implementation
+class ARROW_EXPORT Int128 {
+ public:
+  constexpr Int128() : Int128(0, 0) {}
+
+  /// \brief Convert a signed 64 bit value into an Int128.
+  constexpr Int128(int64_t value)
+      : Int128(value >= 0 ? 0 : -1, static_cast<uint64_t>(value)) {}
+
+  /// \brief Convert a signed 32 bit value into an Int128.
+  constexpr Int128(int32_t value) : Int128(static_cast<int64_t>(value)) {}
+
+  /// \brief Create an Int128 from the two's complement representation.
+  constexpr Int128(int64_t high, uint64_t low) : high_bits_(high), low_bits_(low) {}
+
+  /// \brief Parse the number from a base 10 string representation.
+  explicit Int128(const std::string& value);
+
+  /// \brief Create an Int128 from an array of bytes
+  explicit Int128(const uint8_t* bytes);
+
+  /// \brief Negate the current value
+  Int128& Negate();
+
+  /// \brief Add a number to this one. The result is truncated to 128 bits.
+  Int128& operator+=(const Int128& right);
+
+  /// \brief Subtract a number from this one. The result is truncated to 128 bits.
+  Int128& operator-=(const Int128& right);
+
+  /// \brief Multiply this number by another number. The result is truncated to 128 bits.
+  Int128& operator*=(const Int128& right);
+
+  /// Divide this number by right and return the result. This operation is
+  /// not destructive.
+  /// The answer rounds to zero. Signs work like:
+  ///   21 /  5 ->  4,  1
+  ///  -21 /  5 -> -4, -1
+  ///   21 / -5 -> -4,  1
+  ///  -21 / -5 ->  4, -1
+  /// @param right the number to divide by
+  /// @param remainder the remainder after the division
+  Status Divide(const Int128& divisor, Int128* result, Int128* remainder) const;
+
+  /// \brief In-place division.
+  Int128& operator/=(const Int128& right);
+
+  /// \brief Cast the value to char. This is used when converting the value a string.
+  explicit operator char() const;
+
+  /// \brief Bitwise or between two Int128.
+  Int128& operator|=(const Int128& right);
+
+  /// \brief Bitwise and between two Int128.
+  Int128& operator&=(const Int128& right);
+
+  /// \brief Shift left by the given number of bits.
+  Int128& operator<<=(uint32_t bits);
+
+  /// \brief Shift right by the given number of bits. Negative values will
+  Int128& operator>>=(uint32_t bits);
+
+  /// \brief Get the high bits of the two's complement representation of the number.
+  int64_t high_bits() const { return high_bits_; }
+
+  /// \brief Get the low bits of the two's complement representation of the number.
+  uint64_t low_bits() const { return low_bits_; }
+
+  /// \brief Put the raw bytes of the value into a pointer to uint8_t.
+  void ToBytes(uint8_t** out) const;
+
+ private:
+  int64_t high_bits_;
+  uint64_t low_bits_;
+};
+
+ARROW_EXPORT bool operator==(const Int128& left, const Int128& right);
+ARROW_EXPORT bool operator!=(const Int128& left, const Int128& right);
+ARROW_EXPORT bool operator<(const Int128& left, const Int128& right);
+ARROW_EXPORT bool operator<=(const Int128& left, const Int128& right);
+ARROW_EXPORT bool operator>(const Int128& left, const Int128& right);
+ARROW_EXPORT bool operator>=(const Int128& left, const Int128& right);
+
+ARROW_EXPORT Int128 operator-(const Int128& operand);
+ARROW_EXPORT Int128 operator+(const Int128& left, const Int128& right);
+ARROW_EXPORT Int128 operator-(const Int128& left, const Int128& right);
+ARROW_EXPORT Int128 operator*(const Int128& left, const Int128& right);
+ARROW_EXPORT Int128 operator/(const Int128& left, const Int128& right);
+ARROW_EXPORT Int128 operator%(const Int128& left, const Int128& right);
+
+}  // namespace decimal
+}  // namespace arrow
+
+#endif  //  ARROW_INT128_H

From f50f2eacb499906012da377009921feb21445bb8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 24 Aug 2017 23:04:58 -0400
Subject: [PATCH 0953/1644] ARROW-1408: [C++] IPC public API cleanup,
 refactoring. Add SerializeSchema, ReadSchema public APIs

This is mostly moving code around. In reviewing I recommend focusing on the public headers. There were a number of places where it is more consistent to use naked pointers versus shared_ptr. Also some constructors were returning shared_ptr to subclass, where it would be simpler for clients to return a pointer to base.

This includes ARROW-1376 and ARROW-1406

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #988 from wesm/ARROW-1408 and squashes the following commits:

b156767c [Wes McKinney] Fix up glib bindings, undeprecate some APIs
4bdebfab [Wes McKinney] Add serialize methods to RecordBatch, Schema. Test round trip
ef12e0f2 [Wes McKinney] Fix a valgrind warning
73d30c98 [Wes McKinney] Better comments
8597b96f [Wes McKinney] Remove API that was never intended to be public, unlikely to be used anywhere
122a7591 [Wes McKinney] Refactoring sweep and cleanup of public IPC API. Move non-public APIs from metadata.h to metadata-internal.h and create message.h, dictionary.h
b646f965 [Wes McKinney] Set device in more places
---
 c_glib/arrow-glib/reader.cpp                  |  12 +-
 c_glib/arrow-glib/writer.cpp                  |  30 +-
 cpp/CMakeLists.txt                            |   8 +-
 cpp/src/arrow/gpu/cuda_common.h               |  11 +
 cpp/src/arrow/gpu/cuda_memory.cc              |  11 +-
 cpp/src/arrow/gpu/cuda_memory.h               |   4 +-
 cpp/src/arrow/ipc/CMakeLists.txt              |   3 +-
 cpp/src/arrow/ipc/api.h                       |   3 +-
 cpp/src/arrow/ipc/dictionary.cc               |  85 +++++
 cpp/src/arrow/ipc/dictionary.h                |  88 ++++++
 cpp/src/arrow/ipc/file-to-stream.cc           |   4 +-
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc |   2 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc      |  58 ++--
 cpp/src/arrow/ipc/json-integration-test.cc    |   6 +-
 cpp/src/arrow/ipc/json-internal.cc            |   2 +-
 cpp/src/arrow/ipc/message.cc                  | 257 +++++++++++++++
 cpp/src/arrow/ipc/{metadata.h => message.h}   | 142 +--------
 .../ipc/{metadata.cc => metadata-internal.cc} | 293 +-----------------
 cpp/src/arrow/ipc/metadata-internal.h         | 146 +++++++++
 cpp/src/arrow/ipc/reader.cc                   |  84 ++++-
 cpp/src/arrow/ipc/reader.h                    |  91 ++++--
 cpp/src/arrow/ipc/stream-to-file.cc           |   8 +-
 cpp/src/arrow/ipc/writer.cc                   | 247 +++++++++------
 cpp/src/arrow/ipc/writer.h                    |  33 +-
 cpp/src/arrow/python/arrow_to_python.cc       |   9 +-
 cpp/src/arrow/python/arrow_to_python.h        |   3 +-
 cpp/src/arrow/python/python_to_arrow.cc       |  20 +-
 cpp/src/arrow/type.cc                         |   2 +
 cpp/src/arrow/type.h                          |   2 +-
 cpp/src/arrow/util/key_value_metadata.cc      |   4 +-
 python/pyarrow/__init__.py                    |   4 +-
 python/pyarrow/includes/libarrow.pxd          |  27 +-
 python/pyarrow/ipc.pxi                        |  81 +++--
 python/pyarrow/ipc.py                         |   2 +-
 python/pyarrow/serialization.pxi              |   2 +-
 python/pyarrow/table.pxi                      |  22 ++
 python/pyarrow/tests/test_ipc.py              |  15 +
 python/pyarrow/tests/test_table.py            |  12 +
 python/pyarrow/types.pxi                      |  36 ++-
 39 files changed, 1191 insertions(+), 678 deletions(-)
 create mode 100644 cpp/src/arrow/ipc/dictionary.cc
 create mode 100644 cpp/src/arrow/ipc/dictionary.h
 create mode 100644 cpp/src/arrow/ipc/message.cc
 rename cpp/src/arrow/ipc/{metadata.h => message.h} (52%)
 rename cpp/src/arrow/ipc/{metadata.cc => metadata-internal.cc} (78%)
 create mode 100644 cpp/src/arrow/ipc/metadata-internal.h

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index 25c8d67d95f60..a21b375616752 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -237,12 +237,15 @@ GArrowRecordBatchStreamReader *
 garrow_record_batch_stream_reader_new(GArrowInputStream *stream,
                                       GError **error)
 {
+  using BaseType = arrow::ipc::RecordBatchReader;
+  using ReaderType = arrow::ipc::RecordBatchStreamReader;
+
   auto arrow_input_stream = garrow_input_stream_get_raw(stream);
-  std::shared_ptr<arrow::ipc::RecordBatchStreamReader> arrow_reader;
-  auto status =
-    arrow::ipc::RecordBatchStreamReader::Open(arrow_input_stream, &arrow_reader);
+  std::shared_ptr<BaseType> arrow_reader;
+  auto status = ReaderType::Open(arrow_input_stream, &arrow_reader);
+  auto subtype = std::dynamic_pointer_cast<ReaderType>(arrow_reader);
   if (garrow_error_check(error, status, "[record-batch-stream-reader][open]")) {
-    return garrow_record_batch_stream_reader_new_raw(&arrow_reader);
+    return garrow_record_batch_stream_reader_new_raw(&subtype);
   } else {
     return NULL;
   }
@@ -354,6 +357,7 @@ garrow_record_batch_file_reader_new(GArrowSeekableInputStream *file,
                                     GError **error)
 {
   auto arrow_random_access_file = garrow_seekable_input_stream_get_raw(file);
+
   std::shared_ptr<arrow::ipc::RecordBatchFileReader> arrow_reader;
   auto status =
     arrow::ipc::RecordBatchFileReader::Open(arrow_random_access_file,
diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp
index d4c36c2a8ddea..c08db19987ea1 100644
--- a/c_glib/arrow-glib/writer.cpp
+++ b/c_glib/arrow-glib/writer.cpp
@@ -215,14 +215,17 @@ garrow_record_batch_stream_writer_new(GArrowOutputStream *sink,
                                       GArrowSchema *schema,
                                       GError **error)
 {
+  using BaseType = arrow::ipc::RecordBatchWriter;
+  using WriterType = arrow::ipc::RecordBatchStreamWriter;
+
   auto arrow_sink = garrow_output_stream_get_raw(sink).get();
-  std::shared_ptr<arrow::ipc::RecordBatchStreamWriter> arrow_writer;
-  auto status =
-    arrow::ipc::RecordBatchStreamWriter::Open(arrow_sink,
-                                              garrow_schema_get_raw(schema),
-                                              &arrow_writer);
+  std::shared_ptr<BaseType> arrow_writer;
+  auto status = WriterType::Open(arrow_sink,
+                                 garrow_schema_get_raw(schema),
+                                 &arrow_writer);
+  auto subtype = std::dynamic_pointer_cast<WriterType>(arrow_writer);
   if (garrow_error_check(error, status, "[record-batch-stream-writer][open]")) {
-    return garrow_record_batch_stream_writer_new_raw(&arrow_writer);
+    return garrow_record_batch_stream_writer_new_raw(&subtype);
   } else {
     return NULL;
   }
@@ -259,14 +262,17 @@ garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
                        GArrowSchema *schema,
                        GError **error)
 {
+  using BaseType = arrow::ipc::RecordBatchWriter;
+  using WriterType = arrow::ipc::RecordBatchFileWriter;
+
   auto arrow_sink = garrow_output_stream_get_raw(sink);
-  std::shared_ptr<arrow::ipc::RecordBatchFileWriter> arrow_writer;
-  auto status =
-    arrow::ipc::RecordBatchFileWriter::Open(arrow_sink.get(),
-                                            garrow_schema_get_raw(schema),
-                                            &arrow_writer);
+  std::shared_ptr<BaseType> arrow_writer;
+  auto status = WriterType::Open(arrow_sink.get(),
+                                 garrow_schema_get_raw(schema),
+                                 &arrow_writer);
+  auto subtype = std::dynamic_pointer_cast<WriterType>(arrow_writer);
   if (garrow_error_check(error, status, "[record-batch-file-writer][open]")) {
-    return garrow_record_batch_file_writer_new_raw(&arrow_writer);
+    return garrow_record_batch_file_writer_new_raw(&subtype);
   } else {
     return NULL;
   }
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6c56103717b60..5d643336c05b0 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -217,12 +217,12 @@ include(BuildUtils)
 # Compiler flags
 ############################################################
 
-include(SetupCxxFlags)
-
 if (ARROW_NO_DEPRECATED_API)
   add_definitions(-DARROW_NO_DEPRECATED_API)
 endif()
 
+include(SetupCxxFlags)
+
 ############################################################
 # Dependencies
 ############################################################
@@ -780,10 +780,12 @@ endif()
 
 if (ARROW_IPC)
   set(ARROW_SRCS ${ARROW_SRCS}
+    src/arrow/ipc/dictionary.cc
     src/arrow/ipc/feather.cc
     src/arrow/ipc/json.cc
     src/arrow/ipc/json-internal.cc
-    src/arrow/ipc/metadata.cc
+    src/arrow/ipc/message.cc
+    src/arrow/ipc/metadata-internal.cc
     src/arrow/ipc/reader.cc
     src/arrow/ipc/writer.cc
   )
diff --git a/cpp/src/arrow/gpu/cuda_common.h b/cpp/src/arrow/gpu/cuda_common.h
index fae2f6edf8084..1d65f96adbcdc 100644
--- a/cpp/src/arrow/gpu/cuda_common.h
+++ b/cpp/src/arrow/gpu/cuda_common.h
@@ -45,6 +45,17 @@ namespace gpu {
     }                                                                  \
   } while (0)
 
+#define CUDADRV_RETURN_NOT_OK(STMT)                                           \
+  do {                                                                        \
+    CUresult ret = (STMT);                                                    \
+    if (ret != CUDA_SUCCESS) {                                                \
+      std::stringstream ss;                                                   \
+      ss << "Cuda Driver API call in " << __FILE__ << " at line " << __LINE__ \
+         << " failed: " << #STMT;                                             \
+      return Status::IOError(ss.str());                                       \
+    }                                                                         \
+  } while (0)
+
 }  // namespace gpu
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
index d669619791a8d..4870813b6619b 100644
--- a/cpp/src/arrow/gpu/cuda_memory.cc
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -71,12 +71,13 @@ CudaHostBuffer::~CudaHostBuffer() { CUDA_DCHECK(cudaFreeHost(mutable_data_)); }
 // CudaBufferReader
 
 CudaBufferReader::CudaBufferReader(const std::shared_ptr<CudaBuffer>& buffer)
-    : io::BufferReader(buffer), cuda_buffer_(buffer) {}
+    : io::BufferReader(buffer), cuda_buffer_(buffer), gpu_number_(buffer->gpu_number()) {}
 
 CudaBufferReader::~CudaBufferReader() {}
 
 Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
   nbytes = std::min(nbytes, size_ - position_);
+  CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number_));
   CUDA_RETURN_NOT_OK(
       cudaMemcpy(buffer, data_ + position_, nbytes, cudaMemcpyDeviceToHost));
   *bytes_read = nbytes;
@@ -95,7 +96,10 @@ Status CudaBufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 // CudaBufferWriter
 
 CudaBufferWriter::CudaBufferWriter(const std::shared_ptr<CudaBuffer>& buffer)
-    : io::FixedSizeBufferWriter(buffer), buffer_size_(0), buffer_position_(0) {}
+    : io::FixedSizeBufferWriter(buffer),
+      gpu_number_(buffer->gpu_number()),
+      buffer_size_(0),
+      buffer_position_(0) {}
 
 CudaBufferWriter::~CudaBufferWriter() {}
 
@@ -104,6 +108,7 @@ Status CudaBufferWriter::Close() { return Flush(); }
 Status CudaBufferWriter::Flush() {
   if (buffer_size_ > 0 && buffer_position_ > 0) {
     // Only need to flush when the write has been buffered
+    CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number_));
     CUDA_RETURN_NOT_OK(cudaMemcpy(mutable_data_ + position_ - buffer_position_,
                                   host_buffer_data_, buffer_position_,
                                   cudaMemcpyHostToDevice));
@@ -132,6 +137,7 @@ Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
     if (nbytes + buffer_position_ >= buffer_size_) {
       // Reach end of buffer, write everything
       RETURN_NOT_OK(Flush());
+      CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number_));
       CUDA_RETURN_NOT_OK(
           cudaMemcpy(mutable_data_ + position_, data, nbytes, cudaMemcpyHostToDevice));
     } else {
@@ -141,6 +147,7 @@ Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
     }
   } else {
     // Unbuffered write
+    CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number_));
     CUDA_RETURN_NOT_OK(
         cudaMemcpy(mutable_data_ + position_, data, nbytes, cudaMemcpyHostToDevice));
   }
diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h
index acf65e6574557..bd8b89a75edde 100644
--- a/cpp/src/arrow/gpu/cuda_memory.h
+++ b/cpp/src/arrow/gpu/cuda_memory.h
@@ -97,8 +97,8 @@ class ARROW_EXPORT CudaBufferReader : public io::BufferReader {
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
  private:
-  // In case we need to access anything GPU-specific, like device number
   std::shared_ptr<CudaBuffer> cuda_buffer_;
+  int gpu_number_;
 };
 
 /// \class CudaBufferWriter
@@ -132,6 +132,8 @@ class ARROW_EXPORT CudaBufferWriter : public io::FixedSizeBufferWriter {
   int64_t num_bytes_buffered() const { return buffer_position_; }
 
  private:
+  int gpu_number_;
+
   // Pinned host buffer for buffering writes on CPU before calling cudaMalloc
   int64_t buffer_size_;
   int64_t buffer_position_;
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 76e52a0f78b9a..9689970a183fe 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -83,9 +83,10 @@ add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES})
 # Headers: top level
 install(FILES
   api.h
+  dictionary.h
   feather.h
   json.h
-  metadata.h
+  message.h
   reader.h
   writer.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/ipc")
diff --git a/cpp/src/arrow/ipc/api.h b/cpp/src/arrow/ipc/api.h
index 3f05e69d5843d..5cc64071317f8 100644
--- a/cpp/src/arrow/ipc/api.h
+++ b/cpp/src/arrow/ipc/api.h
@@ -18,9 +18,10 @@
 #ifndef ARROW_IPC_API_H
 #define ARROW_IPC_API_H
 
+#include "arrow/ipc/dictionary.h"
 #include "arrow/ipc/feather.h"
 #include "arrow/ipc/json.h"
-#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/message.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
 
diff --git a/cpp/src/arrow/ipc/dictionary.cc b/cpp/src/arrow/ipc/dictionary.cc
new file mode 100644
index 0000000000000..fcfb3e2c8d230
--- /dev/null
+++ b/cpp/src/arrow/ipc/dictionary.cc
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/dictionary.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+
+#include "arrow/array.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+
+namespace arrow {
+namespace ipc {
+
+DictionaryMemo::DictionaryMemo() {}
+
+// Returns KeyError if dictionary not found
+Status DictionaryMemo::GetDictionary(int64_t id,
+                                     std::shared_ptr<Array>* dictionary) const {
+  auto it = id_to_dictionary_.find(id);
+  if (it == id_to_dictionary_.end()) {
+    std::stringstream ss;
+    ss << "Dictionary with id " << id << " not found";
+    return Status::KeyError(ss.str());
+  }
+  *dictionary = it->second;
+  return Status::OK();
+}
+
+int64_t DictionaryMemo::GetId(const std::shared_ptr<Array>& dictionary) {
+  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
+  auto it = dictionary_to_id_.find(address);
+  if (it != dictionary_to_id_.end()) {
+    // Dictionary already observed, return the id
+    return it->second;
+  } else {
+    int64_t new_id = static_cast<int64_t>(dictionary_to_id_.size());
+    dictionary_to_id_[address] = new_id;
+    id_to_dictionary_[new_id] = dictionary;
+    return new_id;
+  }
+}
+
+bool DictionaryMemo::HasDictionary(const std::shared_ptr<Array>& dictionary) const {
+  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
+  auto it = dictionary_to_id_.find(address);
+  return it != dictionary_to_id_.end();
+}
+
+bool DictionaryMemo::HasDictionaryId(int64_t id) const {
+  auto it = id_to_dictionary_.find(id);
+  return it != id_to_dictionary_.end();
+}
+
+Status DictionaryMemo::AddDictionary(int64_t id,
+                                     const std::shared_ptr<Array>& dictionary) {
+  if (HasDictionaryId(id)) {
+    std::stringstream ss;
+    ss << "Dictionary with id " << id << " already exists";
+    return Status::KeyError(ss.str());
+  }
+  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
+  id_to_dictionary_[id] = dictionary;
+  dictionary_to_id_[address] = id;
+  return Status::OK();
+}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/dictionary.h b/cpp/src/arrow/ipc/dictionary.h
new file mode 100644
index 0000000000000..744ab64e11c82
--- /dev/null
+++ b/cpp/src/arrow/ipc/dictionary.h
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Tools for dictionaries in IPC context
+
+#ifndef ARROW_IPC_DICTIONARY_H
+#define ARROW_IPC_DICTIONARY_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "arrow/status.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+class Buffer;
+class Field;
+
+namespace io {
+
+class InputStream;
+class OutputStream;
+class RandomAccessFile;
+
+}  // namespace io
+
+namespace ipc {
+
+using DictionaryMap = std::unordered_map<int64_t, std::shared_ptr<Array>>;
+using DictionaryTypeMap = std::unordered_map<int64_t, std::shared_ptr<Field>>;
+
+// Memoization data structure for handling shared dictionaries
+class ARROW_EXPORT DictionaryMemo {
+ public:
+  DictionaryMemo();
+
+  // Returns KeyError if dictionary not found
+  Status GetDictionary(int64_t id, std::shared_ptr<Array>* dictionary) const;
+
+  /// Return id for dictionary, computing new id if necessary
+  int64_t GetId(const std::shared_ptr<Array>& dictionary);
+
+  bool HasDictionary(const std::shared_ptr<Array>& dictionary) const;
+  bool HasDictionaryId(int64_t id) const;
+
+  // Add a dictionary to the memo with a particular id. Returns KeyError if
+  // that dictionary already exists
+  Status AddDictionary(int64_t id, const std::shared_ptr<Array>& dictionary);
+
+  const DictionaryMap& id_to_dictionary() const { return id_to_dictionary_; }
+
+  int size() const { return static_cast<int>(id_to_dictionary_.size()); }
+
+ private:
+  // Dictionary memory addresses, to track whether a dictionary has been seen
+  // before
+  std::unordered_map<intptr_t, int64_t> dictionary_to_id_;
+
+  // Map of dictionary id to dictionary array
+  DictionaryMap id_to_dictionary_;
+
+  DISALLOW_COPY_AND_ASSIGN(DictionaryMemo);
+};
+
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_DICTIONARY_H
diff --git a/cpp/src/arrow/ipc/file-to-stream.cc b/cpp/src/arrow/ipc/file-to-stream.cc
index 4707c4fcdf0f7..e69444a560c11 100644
--- a/cpp/src/arrow/ipc/file-to-stream.cc
+++ b/cpp/src/arrow/ipc/file-to-stream.cc
@@ -32,10 +32,10 @@ Status ConvertToStream(const char* path) {
   std::shared_ptr<RecordBatchFileReader> reader;
 
   RETURN_NOT_OK(io::ReadableFile::Open(path, &in_file));
-  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file, &reader));
+  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file.get(), &reader));
 
   io::StdoutStream sink;
-  std::shared_ptr<RecordBatchStreamWriter> writer;
+  std::shared_ptr<RecordBatchWriter> writer;
   RETURN_NOT_OK(RecordBatchStreamWriter::Open(&sink, reader->schema(), &writer));
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> chunk;
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
index a6da6377c0531..9ed0abde651f6 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -109,7 +109,7 @@ static void BM_ReadRecordBatch(benchmark::State& state) {  // NOLINT non-const r
     std::shared_ptr<RecordBatch> result;
     io::BufferReader reader(buffer);
 
-    if (!ipc::ReadRecordBatch(record_batch->schema(), 0, &reader, &result).ok()) {
+    if (!ipc::ReadRecordBatch(record_batch->schema(), &reader, &result).ok()) {
       state.SkipWithError("Failed to read!");
     }
   }
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 5aaff5f569154..e7b0a34547754 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -29,6 +29,7 @@
 #include "arrow/io/memory.h"
 #include "arrow/io/test-common.h"
 #include "arrow/ipc/api.h"
+#include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/ipc/util.h"
 #include "arrow/memory_pool.h"
@@ -47,21 +48,14 @@ class TestSchemaMetadata : public ::testing::Test {
  public:
   void SetUp() {}
 
-  void CheckRoundtrip(const Schema& schema, DictionaryMemo* memo) {
+  void CheckRoundtrip(const Schema& schema) {
     std::shared_ptr<Buffer> buffer;
-    ASSERT_OK(WriteSchemaMessage(schema, memo, &buffer));
+    ASSERT_OK(SerializeSchema(schema, default_memory_pool(), &buffer));
 
-    std::unique_ptr<Message> message;
-    ASSERT_OK(Message::Open(buffer, nullptr, &message));
-
-    ASSERT_EQ(Message::SCHEMA, message->type());
-
-    DictionaryMemo empty_memo;
-
-    std::shared_ptr<Schema> schema2;
-    ASSERT_OK(GetSchema(message->header(), empty_memo, &schema2));
-
-    AssertSchemaEqual(schema, *schema2);
+    std::shared_ptr<Schema> result;
+    io::BufferReader reader(buffer);
+    ASSERT_OK(ReadSchema(&reader, &result));
+    AssertSchemaEqual(schema, *result);
   }
 };
 
@@ -107,9 +101,7 @@ TEST_F(TestSchemaMetadata, PrimitiveFields) {
   auto f10 = field("f10", std::make_shared<BooleanType>());
 
   Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
-  DictionaryMemo memo;
-
-  CheckRoundtrip(schema, &memo);
+  CheckRoundtrip(schema);
 }
 
 TEST_F(TestSchemaMetadata, NestedFields) {
@@ -121,9 +113,7 @@ TEST_F(TestSchemaMetadata, NestedFields) {
   auto f1 = field("f1", type2);
 
   Schema schema({f0, f1});
-  DictionaryMemo memo;
-
-  CheckRoundtrip(schema, &memo);
+  CheckRoundtrip(schema);
 }
 
 #define BATCH_CASES()                                                                   \
@@ -137,13 +127,21 @@ static int g_file_number = 0;
 
 class IpcTestFixture : public io::MemoryMapFixture {
  public:
+  Status DoSchemaRoundTrip(const Schema& schema, std::shared_ptr<Schema>* result) {
+    std::shared_ptr<Buffer> serialized_schema;
+    RETURN_NOT_OK(SerializeSchema(schema, pool_, &serialized_schema));
+
+    io::BufferReader buf_reader(serialized_schema);
+    return ReadSchema(&buf_reader, result);
+  }
+
   Status DoStandardRoundTrip(const RecordBatch& batch,
                              std::shared_ptr<RecordBatch>* batch_result) {
     std::shared_ptr<Buffer> serialized_batch;
     RETURN_NOT_OK(SerializeRecordBatch(batch, pool_, &serialized_batch));
 
     io::BufferReader buf_reader(serialized_batch);
-    return ReadRecordBatch(batch.schema(), 0, &buf_reader, batch_result);
+    return ReadRecordBatch(batch.schema(), &buf_reader, batch_result);
   }
 
   Status DoLargeRoundTrip(const RecordBatch& batch, bool zero_data,
@@ -153,7 +151,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
     }
     RETURN_NOT_OK(mmap_->Seek(0));
 
-    std::shared_ptr<RecordBatchFileWriter> file_writer;
+    std::shared_ptr<RecordBatchWriter> file_writer;
     RETURN_NOT_OK(RecordBatchFileWriter::Open(mmap_.get(), batch.schema(), &file_writer));
     RETURN_NOT_OK(file_writer->WriteRecordBatch(batch, true));
     RETURN_NOT_OK(file_writer->Close());
@@ -162,7 +160,7 @@ class IpcTestFixture : public io::MemoryMapFixture {
     RETURN_NOT_OK(mmap_->Tell(&offset));
 
     std::shared_ptr<RecordBatchFileReader> file_reader;
-    RETURN_NOT_OK(RecordBatchFileReader::Open(mmap_, offset, &file_reader));
+    RETURN_NOT_OK(RecordBatchFileReader::Open(mmap_.get(), offset, &file_reader));
 
     return file_reader->ReadRecordBatch(0, result);
   }
@@ -182,6 +180,10 @@ class IpcTestFixture : public io::MemoryMapFixture {
     ss << "test-write-row-batch-" << g_file_number++;
     ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(buffer_size, ss.str(), &mmap_));
 
+    std::shared_ptr<Schema> schema_result;
+    ASSERT_OK(DoSchemaRoundTrip(*batch.schema(), &schema_result));
+    ASSERT_TRUE(batch.schema()->Equals(*schema_result));
+
     std::shared_ptr<RecordBatch> result;
     ASSERT_OK(DoStandardRoundTrip(batch, &result));
     CheckReadResult(*result, batch);
@@ -498,7 +500,7 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
 
   Status RoundTripHelper(const BatchVector& in_batches, BatchVector* out_batches) {
     // Write the file
-    std::shared_ptr<RecordBatchFileWriter> writer;
+    std::shared_ptr<RecordBatchWriter> writer;
     RETURN_NOT_OK(
         RecordBatchFileWriter::Open(sink_.get(), in_batches[0]->schema(), &writer));
 
@@ -517,7 +519,7 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
     // Open the file
     auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
     std::shared_ptr<RecordBatchFileReader> reader;
-    RETURN_NOT_OK(RecordBatchFileReader::Open(buf_reader, footer_offset, &reader));
+    RETURN_NOT_OK(RecordBatchFileReader::Open(buf_reader.get(), footer_offset, &reader));
 
     EXPECT_EQ(num_batches, reader->num_record_batches());
     for (int i = 0; i < num_batches; ++i) {
@@ -565,7 +567,7 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
   Status RoundTripHelper(const RecordBatch& batch,
                          std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
     // Write the file
-    std::shared_ptr<RecordBatchStreamWriter> writer;
+    std::shared_ptr<RecordBatchWriter> writer;
     RETURN_NOT_OK(RecordBatchStreamWriter::Open(sink_.get(), batch.schema(), &writer));
     int num_batches = 5;
     for (int i = 0; i < num_batches; ++i) {
@@ -575,10 +577,10 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
     RETURN_NOT_OK(sink_->Close());
 
     // Open the file
-    auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+    io::BufferReader buf_reader(buffer_);
 
-    std::shared_ptr<RecordBatchStreamReader> reader;
-    RETURN_NOT_OK(RecordBatchStreamReader::Open(buf_reader, &reader));
+    std::shared_ptr<RecordBatchReader> reader;
+    RETURN_NOT_OK(RecordBatchStreamReader::Open(&buf_reader, &reader));
 
     std::shared_ptr<RecordBatch> chunk;
     while (true) {
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 035f7086e7e53..e0f01f931c1d7 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -77,7 +77,7 @@ static Status ConvertJsonToArrow(const std::string& json_path,
     std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
   }
 
-  std::shared_ptr<ipc::RecordBatchFileWriter> writer;
+  std::shared_ptr<ipc::RecordBatchWriter> writer;
   RETURN_NOT_OK(
       ipc::RecordBatchFileWriter::Open(out_file.get(), reader->schema(), &writer));
 
@@ -99,7 +99,7 @@ static Status ConvertArrowToJson(const std::string& arrow_path,
   RETURN_NOT_OK(io::FileOutputStream::Open(json_path, &out_file));
 
   std::shared_ptr<ipc::RecordBatchFileReader> reader;
-  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file, &reader));
+  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file.get(), &reader));
 
   if (FLAGS_verbose) {
     std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
@@ -140,7 +140,7 @@ static Status ValidateArrowVsJson(const std::string& arrow_path,
   RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &arrow_file));
 
   std::shared_ptr<ipc::RecordBatchFileReader> arrow_reader;
-  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(arrow_file, &arrow_reader));
+  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(arrow_file.get(), &arrow_reader));
 
   auto json_schema = json_reader->schema();
   auto arrow_schema = arrow_reader->schema();
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index bc2b0d18e72c7..13572aefa753c 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -32,7 +32,7 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
-#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/metadata-internal.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc
new file mode 100644
index 0000000000000..eb06aaf2fc651
--- /dev/null
+++ b/cpp/src/arrow/ipc/message.cc
@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/ipc/message.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/ipc/metadata-internal.h"
+#include "arrow/ipc/util.h"
+#include "arrow/status.h"
+#include "arrow/tensor.h"
+#include "arrow/type.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+namespace ipc {
+
+class Message::MessageImpl {
+ public:
+  explicit MessageImpl(const std::shared_ptr<Buffer>& metadata,
+                       const std::shared_ptr<Buffer>& body)
+      : metadata_(metadata), message_(nullptr), body_(body) {}
+
+  Status Open() {
+    message_ = flatbuf::GetMessage(metadata_->data());
+
+    // Check that the metadata version is supported
+    if (message_->version() < kMinMetadataVersion) {
+      return Status::Invalid("Old metadata version not supported");
+    }
+
+    return Status::OK();
+  }
+
+  Message::Type type() const {
+    switch (message_->header_type()) {
+      case flatbuf::MessageHeader_Schema:
+        return Message::SCHEMA;
+      case flatbuf::MessageHeader_DictionaryBatch:
+        return Message::DICTIONARY_BATCH;
+      case flatbuf::MessageHeader_RecordBatch:
+        return Message::RECORD_BATCH;
+      case flatbuf::MessageHeader_Tensor:
+        return Message::TENSOR;
+      default:
+        return Message::NONE;
+    }
+  }
+
+  MetadataVersion version() const {
+    switch (message_->version()) {
+      case flatbuf::MetadataVersion_V1:
+        // Arrow 0.1
+        return MetadataVersion::V1;
+      case flatbuf::MetadataVersion_V2:
+        // Arrow 0.2
+        return MetadataVersion::V2;
+      case flatbuf::MetadataVersion_V3:
+        // Arrow >= 0.3
+        return MetadataVersion::V3;
+      // Add cases as other versions become available
+      default:
+        return MetadataVersion::V3;
+    }
+  }
+
+  const void* header() const { return message_->header(); }
+
+  std::shared_ptr<Buffer> body() const { return body_; }
+
+  std::shared_ptr<Buffer> metadata() const { return metadata_; }
+
+ private:
+  // The Flatbuffer metadata
+  std::shared_ptr<Buffer> metadata_;
+  const flatbuf::Message* message_;
+
+  // The message body, if any
+  std::shared_ptr<Buffer> body_;
+};
+
+Message::Message(const std::shared_ptr<Buffer>& metadata,
+                 const std::shared_ptr<Buffer>& body) {
+  impl_.reset(new MessageImpl(metadata, body));
+}
+
+Status Message::Open(const std::shared_ptr<Buffer>& metadata,
+                     const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out) {
+  out->reset(new Message(metadata, body));
+  return (*out)->impl_->Open();
+}
+
+Message::~Message() {}
+
+std::shared_ptr<Buffer> Message::body() const { return impl_->body(); }
+
+std::shared_ptr<Buffer> Message::metadata() const { return impl_->metadata(); }
+
+Message::Type Message::type() const { return impl_->type(); }
+
+MetadataVersion Message::metadata_version() const { return impl_->version(); }
+
+const void* Message::header() const { return impl_->header(); }
+
+bool Message::Equals(const Message& other) const {
+  int64_t metadata_bytes = std::min(metadata()->size(), other.metadata()->size());
+
+  if (!metadata()->Equals(*other.metadata(), metadata_bytes)) {
+    return false;
+  }
+
+  // Compare bodies, if they have them
+  auto this_body = body();
+  auto other_body = other.body();
+
+  const bool this_has_body = (this_body != nullptr) && (this_body->size() > 0);
+  const bool other_has_body = (other_body != nullptr) && (other_body->size() > 0);
+
+  if (this_has_body && other_has_body) {
+    return this_body->Equals(*other_body);
+  } else if (this_has_body ^ other_has_body) {
+    // One has a body but not the other
+    return false;
+  } else {
+    // Neither has a body
+    return true;
+  }
+}
+
+Status Message::SerializeTo(io::OutputStream* file, int64_t* output_length) const {
+  int32_t metadata_length = 0;
+  RETURN_NOT_OK(WriteMessage(*metadata(), file, &metadata_length));
+
+  *output_length = metadata_length;
+
+  auto body_buffer = body();
+  if (body_buffer) {
+    RETURN_NOT_OK(file->Write(body_buffer->data(), body_buffer->size()));
+    *output_length += body_buffer->size();
+  }
+
+  return Status::OK();
+}
+
+std::string FormatMessageType(Message::Type type) {
+  switch (type) {
+    case Message::SCHEMA:
+      return "schema";
+    case Message::RECORD_BATCH:
+      return "record batch";
+    case Message::DICTIONARY_BATCH:
+      return "dictionary";
+    default:
+      break;
+  }
+  return "unknown";
+}
+
+// ----------------------------------------------------------------------
+// Read and write messages
+
+static Status ReadFullMessage(const std::shared_ptr<Buffer>& metadata,
+                              io::InputStream* stream,
+                              std::unique_ptr<Message>* message) {
+  auto fb_message = flatbuf::GetMessage(metadata->data());
+
+  int64_t body_length = fb_message->bodyLength();
+
+  std::shared_ptr<Buffer> body;
+  RETURN_NOT_OK(stream->Read(body_length, &body));
+
+  if (body->size() < body_length) {
+    std::stringstream ss;
+    ss << "Expected to be able to read " << body_length << " bytes for message body, got "
+       << body->size();
+    return Status::IOError(ss.str());
+  }
+
+  return Message::Open(metadata, body, message);
+}
+
+Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
+                   std::unique_ptr<Message>* message) {
+  std::shared_ptr<Buffer> buffer;
+  RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
+
+  int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
+
+  if (flatbuffer_size + static_cast<int>(sizeof(int32_t)) > metadata_length) {
+    std::stringstream ss;
+    ss << "flatbuffer size " << metadata_length << " invalid. File offset: " << offset
+       << ", metadata length: " << metadata_length;
+    return Status::Invalid(ss.str());
+  }
+
+  auto metadata = SliceBuffer(buffer, 4, buffer->size() - 4);
+  return ReadFullMessage(metadata, file, message);
+}
+
+Status ReadMessage(io::InputStream* file, std::unique_ptr<Message>* message) {
+  std::shared_ptr<Buffer> buffer;
+
+  RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
+  if (buffer->size() != sizeof(int32_t)) {
+    *message = nullptr;
+    return Status::OK();
+  }
+
+  int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
+
+  if (message_length == 0) {
+    // Optional 0 EOS control message
+    *message = nullptr;
+    return Status::OK();
+  }
+
+  RETURN_NOT_OK(file->Read(message_length, &buffer));
+  if (buffer->size() != message_length) {
+    return Status::IOError("Unexpected end of stream trying to read message");
+  }
+
+  return ReadFullMessage(buffer, file, message);
+}
+
+// ----------------------------------------------------------------------
+// Implement InputStream message reader
+
+Status InputStreamMessageReader::ReadNextMessage(std::unique_ptr<Message>* message) {
+  return ReadMessage(stream_, message);
+}
+
+InputStreamMessageReader::~InputStreamMessageReader() {}
+
+}  // namespace ipc
+}  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/message.h
similarity index 52%
rename from cpp/src/arrow/ipc/metadata.h
rename to cpp/src/arrow/ipc/message.h
index 81716ae467baf..dce4e27fb2d39 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -17,8 +17,8 @@
 
 // C++ object model and user API for interprocess schema messaging
 
-#ifndef ARROW_IPC_METADATA_H
-#define ARROW_IPC_METADATA_H
+#ifndef ARROW_IPC_MESSAGE_H
+#define ARROW_IPC_MESSAGE_H
 
 #include <cstdint>
 #include <memory>
@@ -26,6 +26,7 @@
 #include <unordered_map>
 #include <vector>
 
+#include "arrow/status.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -33,11 +34,7 @@ namespace arrow {
 
 class Array;
 class Buffer;
-class DataType;
 class Field;
-class Schema;
-class Status;
-class Tensor;
 
 namespace io {
 
@@ -51,97 +48,17 @@ namespace ipc {
 
 enum class MetadataVersion : char { V1, V2, V3 };
 
-static constexpr const char* kArrowMagicBytes = "ARROW1";
-
 // ARROW-109: We set this number arbitrarily to help catch user mistakes. For
 // deeply nested schemas, it is expected the user will indicate explicitly the
 // maximum allowed recursion depth
 constexpr int kMaxNestingDepth = 64;
 
-struct ARROW_EXPORT FieldMetadata {
-  int64_t length;
-  int64_t null_count;
-  int64_t offset;
-};
-
-struct ARROW_EXPORT BufferMetadata {
-  /// The shared memory page id where to find this. Set to -1 if unused
-  int32_t page;
-
-  /// The relative offset into the memory page to the starting byte of the buffer
-  int64_t offset;
-
-  /// Absolute length in bytes of the buffer
-  int64_t length;
-};
-
-struct FileBlock {
-  int64_t offset;
-  int32_t metadata_length;
-  int64_t body_length;
-};
-
-//----------------------------------------------------------------------
-
-using DictionaryMap = std::unordered_map<int64_t, std::shared_ptr<Array>>;
-using DictionaryTypeMap = std::unordered_map<int64_t, std::shared_ptr<Field>>;
-
-// Memoization data structure for handling shared dictionaries
-class ARROW_EXPORT DictionaryMemo {
- public:
-  DictionaryMemo();
-
-  // Returns KeyError if dictionary not found
-  Status GetDictionary(int64_t id, std::shared_ptr<Array>* dictionary) const;
-
-  /// Return id for dictionary, computing new id if necessary
-  int64_t GetId(const std::shared_ptr<Array>& dictionary);
-
-  bool HasDictionary(const std::shared_ptr<Array>& dictionary) const;
-  bool HasDictionaryId(int64_t id) const;
-
-  // Add a dictionary to the memo with a particular id. Returns KeyError if
-  // that dictionary already exists
-  Status AddDictionary(int64_t id, const std::shared_ptr<Array>& dictionary);
-
-  const DictionaryMap& id_to_dictionary() const { return id_to_dictionary_; }
-
-  int size() const { return static_cast<int>(id_to_dictionary_.size()); }
-
- private:
-  // Dictionary memory addresses, to track whether a dictionary has been seen
-  // before
-  std::unordered_map<intptr_t, int64_t> dictionary_to_id_;
-
-  // Map of dictionary id to dictionary array
-  DictionaryMap id_to_dictionary_;
-
-  DISALLOW_COPY_AND_ASSIGN(DictionaryMemo);
-};
-
 // Read interface classes. We do not fully deserialize the flatbuffers so that
 // individual fields metadata can be retrieved from very large schema without
 //
 
 class Message;
 
-// Retrieve a list of all the dictionary ids and types required by the schema for
-// reconstruction. The presumption is that these will be loaded either from
-// the stream or file (or they may already be somewhere else in memory)
-Status GetDictionaryTypes(const void* opaque_schema, DictionaryTypeMap* id_to_field);
-
-// Construct a complete Schema from the message. May be expensive for very
-// large schemas if you are only interested in a few fields
-Status ARROW_EXPORT GetSchema(const void* opaque_schema,
-                              const DictionaryMemo& dictionary_memo,
-                              std::shared_ptr<Schema>* out);
-
-Status ARROW_EXPORT GetTensorMetadata(const Buffer& metadata,
-                                      std::shared_ptr<DataType>* type,
-                                      std::vector<int64_t>* shape,
-                                      std::vector<int64_t>* strides,
-                                      std::vector<std::string>* dim_names);
-
 /// \brief An IPC message including metadata and body
 class ARROW_EXPORT Message {
  public:
@@ -219,15 +136,20 @@ class ARROW_EXPORT MessageReader {
 /// \since 0.5.0
 class ARROW_EXPORT InputStreamMessageReader : public MessageReader {
  public:
-  explicit InputStreamMessageReader(const std::shared_ptr<io::InputStream>& stream)
-      : stream_(stream) {}
+  explicit InputStreamMessageReader(io::InputStream* stream) : stream_(stream) {}
+
+  explicit InputStreamMessageReader(const std::shared_ptr<io::InputStream>& owned_stream)
+      : InputStreamMessageReader(owned_stream.get()) {
+    owned_stream_ = owned_stream;
+  }
 
   ~InputStreamMessageReader();
 
   Status ReadNextMessage(std::unique_ptr<Message>* message) override;
 
  private:
-  std::shared_ptr<io::InputStream> stream_;
+  io::InputStream* stream_;
+  std::shared_ptr<io::InputStream> owned_stream_;
 };
 
 /// \brief Read encapulated RPC message from position in file
@@ -253,48 +175,10 @@ Status ReadMessage(const int64_t offset, const int32_t metadata_length,
 /// Read length-prefixed message with as-yet unknown length. Returns nullptr if
 /// there are not enough bytes available or the message length is 0 (e.g. EOS
 /// in a stream)
-Status ARROW_EXPORT ReadMessage(io::InputStream* stream,
-                                std::unique_ptr<Message>* message);
-
-/// Write a serialized message metadata with a length-prefix and padding to an
-/// 8-byte offset
-///
-/// <message_size: int32><message: const void*><padding>
-Status ARROW_EXPORT WriteMessage(const Buffer& message, io::OutputStream* file,
-                                 int32_t* message_length);
-
-// Serialize arrow::Schema as a Flatbuffer
-//
-// \param[in] schema a Schema instance
-// \param[inout] dictionary_memo class for tracking dictionaries and assigning
-// dictionary ids
-// \param[out] out the serialized arrow::Buffer
-// \return Status outcome
-Status ARROW_EXPORT WriteSchemaMessage(const Schema& schema,
-                                       DictionaryMemo* dictionary_memo,
-                                       std::shared_ptr<Buffer>* out);
-
 ARROW_EXPORT
-Status WriteRecordBatchMessage(const int64_t length, const int64_t body_length,
-                               const std::vector<FieldMetadata>& nodes,
-                               const std::vector<BufferMetadata>& buffers,
-                               std::shared_ptr<Buffer>* out);
-
-ARROW_EXPORT
-Status WriteTensorMessage(const Tensor& tensor, const int64_t buffer_start_offset,
-                          std::shared_ptr<Buffer>* out);
-
-Status WriteDictionaryMessage(const int64_t id, const int64_t length,
-                              const int64_t body_length,
-                              const std::vector<FieldMetadata>& nodes,
-                              const std::vector<BufferMetadata>& buffers,
-                              std::shared_ptr<Buffer>* out);
-
-Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
-                       const std::vector<FileBlock>& record_batches,
-                       DictionaryMemo* dictionary_memo, io::OutputStream* out);
+Status ReadMessage(io::InputStream* stream, std::unique_ptr<Message>* message);
 
 }  // namespace ipc
 }  // namespace arrow
 
-#endif  // ARROW_IPC_METADATA_H
+#endif  // ARROW_IPC_MESSAGE_H
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata-internal.cc
similarity index 78%
rename from cpp/src/arrow/ipc/metadata.cc
rename to cpp/src/arrow/ipc/metadata-internal.cc
index a31b9668da3c2..fdaab2176c1fa 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/metadata-internal.h"
 
 #include <algorithm>
 #include <cstdint>
@@ -29,9 +29,6 @@
 #include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
-#include "arrow/ipc/File_generated.h"
-#include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/Tensor_generated.h"
 #include "arrow/ipc/util.h"
 #include "arrow/status.h"
 #include "arrow/tensor.h"
@@ -53,12 +50,6 @@ using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
 using FBString = flatbuffers::Offset<flatbuffers::String>;
 
-static constexpr flatbuf::MetadataVersion kCurrentMetadataVersion =
-    flatbuf::MetadataVersion_V3;
-
-static constexpr flatbuf::MetadataVersion kMinMetadataVersion =
-    flatbuf::MetadataVersion_V3;
-
 static Status IntFromFlatbuffer(const flatbuf::Int* int_data,
                                 std::shared_ptr<DataType>* out) {
   if (int_data->bitWidth() > 64) {
@@ -703,7 +694,7 @@ static Status MakeRecordBatch(FBB& fbb, int64_t length, int64_t body_length,
   return Status::OK();
 }
 
-Status WriteRecordBatchMessage(const int64_t length, const int64_t body_length,
+Status WriteRecordBatchMessage(int64_t length, int64_t body_length,
                                const std::vector<FieldMetadata>& nodes,
                                const std::vector<BufferMetadata>& buffers,
                                std::shared_ptr<Buffer>* out) {
@@ -714,7 +705,7 @@ Status WriteRecordBatchMessage(const int64_t length, const int64_t body_length,
                         body_length, out);
 }
 
-Status WriteTensorMessage(const Tensor& tensor, const int64_t buffer_start_offset,
+Status WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset,
                           std::shared_ptr<Buffer>* out) {
   using TensorDimOffset = flatbuffers::Offset<flatbuf::TensorDim>;
   using TensorOffset = flatbuffers::Offset<flatbuf::Tensor>;
@@ -743,8 +734,7 @@ Status WriteTensorMessage(const Tensor& tensor, const int64_t buffer_start_offse
                         body_length, out);
 }
 
-Status WriteDictionaryMessage(const int64_t id, const int64_t length,
-                              const int64_t body_length,
+Status WriteDictionaryMessage(int64_t id, int64_t length, int64_t body_length,
                               const std::vector<FieldMetadata>& nodes,
                               const std::vector<BufferMetadata>& buffers,
                               std::shared_ptr<Buffer>* out) {
@@ -802,206 +792,6 @@ Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dicti
   return out->Write(fbb.GetBufferPointer(), size);
 }
 
-// ----------------------------------------------------------------------
-// Memoization data structure for handling shared dictionaries
-
-DictionaryMemo::DictionaryMemo() {}
-
-// Returns KeyError if dictionary not found
-Status DictionaryMemo::GetDictionary(int64_t id,
-                                     std::shared_ptr<Array>* dictionary) const {
-  auto it = id_to_dictionary_.find(id);
-  if (it == id_to_dictionary_.end()) {
-    std::stringstream ss;
-    ss << "Dictionary with id " << id << " not found";
-    return Status::KeyError(ss.str());
-  }
-  *dictionary = it->second;
-  return Status::OK();
-}
-
-int64_t DictionaryMemo::GetId(const std::shared_ptr<Array>& dictionary) {
-  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
-  auto it = dictionary_to_id_.find(address);
-  if (it != dictionary_to_id_.end()) {
-    // Dictionary already observed, return the id
-    return it->second;
-  } else {
-    int64_t new_id = static_cast<int64_t>(dictionary_to_id_.size());
-    dictionary_to_id_[address] = new_id;
-    id_to_dictionary_[new_id] = dictionary;
-    return new_id;
-  }
-}
-
-bool DictionaryMemo::HasDictionary(const std::shared_ptr<Array>& dictionary) const {
-  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
-  auto it = dictionary_to_id_.find(address);
-  return it != dictionary_to_id_.end();
-}
-
-bool DictionaryMemo::HasDictionaryId(int64_t id) const {
-  auto it = id_to_dictionary_.find(id);
-  return it != id_to_dictionary_.end();
-}
-
-Status DictionaryMemo::AddDictionary(int64_t id,
-                                     const std::shared_ptr<Array>& dictionary) {
-  if (HasDictionaryId(id)) {
-    std::stringstream ss;
-    ss << "Dictionary with id " << id << " already exists";
-    return Status::KeyError(ss.str());
-  }
-  intptr_t address = reinterpret_cast<intptr_t>(dictionary.get());
-  id_to_dictionary_[id] = dictionary;
-  dictionary_to_id_[address] = id;
-  return Status::OK();
-}
-
-//----------------------------------------------------------------------
-// Message reader
-
-class Message::MessageImpl {
- public:
-  explicit MessageImpl(const std::shared_ptr<Buffer>& metadata,
-                       const std::shared_ptr<Buffer>& body)
-      : metadata_(metadata), message_(nullptr), body_(body) {}
-
-  Status Open() {
-    message_ = flatbuf::GetMessage(metadata_->data());
-
-    // Check that the metadata version is supported
-    if (message_->version() < kMinMetadataVersion) {
-      return Status::Invalid("Old metadata version not supported");
-    }
-
-    return Status::OK();
-  }
-
-  Message::Type type() const {
-    switch (message_->header_type()) {
-      case flatbuf::MessageHeader_Schema:
-        return Message::SCHEMA;
-      case flatbuf::MessageHeader_DictionaryBatch:
-        return Message::DICTIONARY_BATCH;
-      case flatbuf::MessageHeader_RecordBatch:
-        return Message::RECORD_BATCH;
-      case flatbuf::MessageHeader_Tensor:
-        return Message::TENSOR;
-      default:
-        return Message::NONE;
-    }
-  }
-
-  MetadataVersion version() const {
-    switch (message_->version()) {
-      case flatbuf::MetadataVersion_V1:
-        // Arrow 0.1
-        return MetadataVersion::V1;
-      case flatbuf::MetadataVersion_V2:
-        // Arrow 0.2
-        return MetadataVersion::V2;
-      case flatbuf::MetadataVersion_V3:
-        // Arrow >= 0.3
-        return MetadataVersion::V3;
-      // Add cases as other versions become available
-      default:
-        return MetadataVersion::V3;
-    }
-  }
-
-  const void* header() const { return message_->header(); }
-
-  std::shared_ptr<Buffer> body() const { return body_; }
-
-  std::shared_ptr<Buffer> metadata() const { return metadata_; }
-
- private:
-  // The Flatbuffer metadata
-  std::shared_ptr<Buffer> metadata_;
-  const flatbuf::Message* message_;
-
-  // The message body, if any
-  std::shared_ptr<Buffer> body_;
-};
-
-Message::Message(const std::shared_ptr<Buffer>& metadata,
-                 const std::shared_ptr<Buffer>& body) {
-  impl_.reset(new MessageImpl(metadata, body));
-}
-
-Status Message::Open(const std::shared_ptr<Buffer>& metadata,
-                     const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out) {
-  out->reset(new Message(metadata, body));
-  return (*out)->impl_->Open();
-}
-
-Message::~Message() {}
-
-std::shared_ptr<Buffer> Message::body() const { return impl_->body(); }
-
-std::shared_ptr<Buffer> Message::metadata() const { return impl_->metadata(); }
-
-Message::Type Message::type() const { return impl_->type(); }
-
-MetadataVersion Message::metadata_version() const { return impl_->version(); }
-
-const void* Message::header() const { return impl_->header(); }
-
-bool Message::Equals(const Message& other) const {
-  int64_t metadata_bytes = std::min(metadata()->size(), other.metadata()->size());
-
-  if (!metadata()->Equals(*other.metadata(), metadata_bytes)) {
-    return false;
-  }
-
-  // Compare bodies, if they have them
-  auto this_body = body();
-  auto other_body = other.body();
-
-  const bool this_has_body = (this_body != nullptr) && (this_body->size() > 0);
-  const bool other_has_body = (other_body != nullptr) && (other_body->size() > 0);
-
-  if (this_has_body && other_has_body) {
-    return this_body->Equals(*other_body);
-  } else if (this_has_body ^ other_has_body) {
-    // One has a body but not the other
-    return false;
-  } else {
-    // Neither has a body
-    return true;
-  }
-}
-
-Status Message::SerializeTo(io::OutputStream* file, int64_t* output_length) const {
-  int32_t metadata_length = 0;
-  RETURN_NOT_OK(WriteMessage(*metadata(), file, &metadata_length));
-
-  *output_length = metadata_length;
-
-  auto body_buffer = body();
-  if (body_buffer) {
-    RETURN_NOT_OK(file->Write(body_buffer->data(), body_buffer->size()));
-    *output_length += body_buffer->size();
-  }
-
-  return Status::OK();
-}
-
-std::string FormatMessageType(Message::Type type) {
-  switch (type) {
-    case Message::SCHEMA:
-      return "schema";
-    case Message::RECORD_BATCH:
-      return "record batch";
-    case Message::DICTIONARY_BATCH:
-      return "dictionary";
-    default:
-      break;
-  }
-  return "unknown";
-}
-
 // ----------------------------------------------------------------------
 
 static Status VisitField(const flatbuf::Field* field, DictionaryTypeMap* id_to_field) {
@@ -1084,81 +874,6 @@ Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>* type
   return TypeFromFlatbuffer(tensor->type_type(), tensor->type(), {}, type);
 }
 
-// ----------------------------------------------------------------------
-// Read and write messages
-
-static Status ReadFullMessage(const std::shared_ptr<Buffer>& metadata,
-                              io::InputStream* stream,
-                              std::unique_ptr<Message>* message) {
-  auto fb_message = flatbuf::GetMessage(metadata->data());
-
-  int64_t body_length = fb_message->bodyLength();
-
-  std::shared_ptr<Buffer> body;
-  RETURN_NOT_OK(stream->Read(body_length, &body));
-
-  if (body->size() < body_length) {
-    std::stringstream ss;
-    ss << "Expected to be able to read " << body_length << " bytes for message body, got "
-       << body->size();
-    return Status::IOError(ss.str());
-  }
-
-  return Message::Open(metadata, body, message);
-}
-
-Status ReadMessage(const int64_t offset, const int32_t metadata_length,
-                   io::RandomAccessFile* file, std::unique_ptr<Message>* message) {
-  std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
-
-  int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
-
-  if (flatbuffer_size + static_cast<int>(sizeof(int32_t)) > metadata_length) {
-    std::stringstream ss;
-    ss << "flatbuffer size " << metadata_length << " invalid. File offset: " << offset
-       << ", metadata length: " << metadata_length;
-    return Status::Invalid(ss.str());
-  }
-
-  auto metadata = SliceBuffer(buffer, 4, buffer->size() - 4);
-  return ReadFullMessage(metadata, file, message);
-}
-
-Status ReadMessage(io::InputStream* file, std::unique_ptr<Message>* message) {
-  std::shared_ptr<Buffer> buffer;
-
-  RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
-  if (buffer->size() != sizeof(int32_t)) {
-    *message = nullptr;
-    return Status::OK();
-  }
-
-  int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
-
-  if (message_length == 0) {
-    // Optional 0 EOS control message
-    *message = nullptr;
-    return Status::OK();
-  }
-
-  RETURN_NOT_OK(file->Read(message_length, &buffer));
-  if (buffer->size() != message_length) {
-    return Status::IOError("Unexpected end of stream trying to read message");
-  }
-
-  return ReadFullMessage(buffer, file, message);
-}
-
-// ----------------------------------------------------------------------
-// Implement InputStream message reader
-
-Status InputStreamMessageReader::ReadNextMessage(std::unique_ptr<Message>* message) {
-  return ReadMessage(stream_.get(), message);
-}
-
-InputStreamMessageReader::~InputStreamMessageReader() {}
-
 // ----------------------------------------------------------------------
 // Implement message writing
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
new file mode 100644
index 0000000000000..738ed3c5c7a04
--- /dev/null
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Internal metadata serialization matters
+
+#ifndef ARROW_IPC_METADATA_INTERNAL_H
+#define ARROW_IPC_METADATA_INTERNAL_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "arrow/ipc/File_generated.h"
+#include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/Tensor_generated.h"
+#include "arrow/ipc/dictionary.h"
+#include "arrow/ipc/message.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+class Buffer;
+class DataType;
+class Field;
+class Schema;
+class Status;
+class Tensor;
+
+namespace flatbuf = org::apache::arrow::flatbuf;
+
+namespace io {
+
+class InputStream;
+class OutputStream;
+class RandomAccessFile;
+
+}  // namespace io
+
+namespace ipc {
+
+static constexpr flatbuf::MetadataVersion kCurrentMetadataVersion =
+    flatbuf::MetadataVersion_V3;
+
+static constexpr flatbuf::MetadataVersion kMinMetadataVersion =
+    flatbuf::MetadataVersion_V3;
+
+static constexpr const char* kArrowMagicBytes = "ARROW1";
+
+struct FieldMetadata {
+  int64_t length;
+  int64_t null_count;
+  int64_t offset;
+};
+
+struct BufferMetadata {
+  /// The shared memory page id where to find this. Set to -1 if unused
+  int32_t page;
+
+  /// The relative offset into the memory page to the starting byte of the buffer
+  int64_t offset;
+
+  /// Absolute length in bytes of the buffer
+  int64_t length;
+};
+
+struct FileBlock {
+  int64_t offset;
+  int32_t metadata_length;
+  int64_t body_length;
+};
+
+// Read interface classes. We do not fully deserialize the flatbuffers so that
+// individual fields metadata can be retrieved from very large schema without
+//
+
+// Retrieve a list of all the dictionary ids and types required by the schema for
+// reconstruction. The presumption is that these will be loaded either from
+// the stream or file (or they may already be somewhere else in memory)
+Status GetDictionaryTypes(const void* opaque_schema, DictionaryTypeMap* id_to_field);
+
+// Construct a complete Schema from the message. May be expensive for very
+// large schemas if you are only interested in a few fields
+Status GetSchema(const void* opaque_schema, const DictionaryMemo& dictionary_memo,
+                 std::shared_ptr<Schema>* out);
+
+Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>* type,
+                         std::vector<int64_t>* shape, std::vector<int64_t>* strides,
+                         std::vector<std::string>* dim_names);
+
+/// Write a serialized message metadata with a length-prefix and padding to an
+/// 8-byte offset
+///
+/// <message_size: int32><message: const void*><padding>
+Status WriteMessage(const Buffer& message, io::OutputStream* file,
+                    int32_t* message_length);
+
+// Serialize arrow::Schema as a Flatbuffer
+//
+// \param[in] schema a Schema instance
+// \param[inout] dictionary_memo class for tracking dictionaries and assigning
+// dictionary ids
+// \param[out] out the serialized arrow::Buffer
+// \return Status outcome
+Status WriteSchemaMessage(const Schema& schema, DictionaryMemo* dictionary_memo,
+                          std::shared_ptr<Buffer>* out);
+
+Status WriteRecordBatchMessage(const int64_t length, const int64_t body_length,
+                               const std::vector<FieldMetadata>& nodes,
+                               const std::vector<BufferMetadata>& buffers,
+                               std::shared_ptr<Buffer>* out);
+
+Status WriteTensorMessage(const Tensor& tensor, const int64_t buffer_start_offset,
+                          std::shared_ptr<Buffer>* out);
+
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+                       const std::vector<FileBlock>& record_batches,
+                       DictionaryMemo* dictionary_memo, io::OutputStream* out);
+
+Status WriteDictionaryMessage(const int64_t id, const int64_t length,
+                              const int64_t body_length,
+                              const std::vector<FieldMetadata>& nodes,
+                              const std::vector<BufferMetadata>& buffers,
+                              std::shared_ptr<Buffer>* out);
+
+}  // namespace ipc
+}  // namespace arrow
+
+#endif  // ARROW_IPC_METADATA_H
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 6ea907e0ed09f..7d7acad518bb7 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -28,7 +28,8 @@
 #include "arrow/io/memory.h"
 #include "arrow/ipc/File_generated.h"
 #include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/message.h"
+#include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/util.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
@@ -459,6 +460,28 @@ RecordBatchStreamReader::RecordBatchStreamReader() {
 
 RecordBatchStreamReader::~RecordBatchStreamReader() {}
 
+Status RecordBatchStreamReader::Open(std::unique_ptr<MessageReader> message_reader,
+                                     std::shared_ptr<RecordBatchReader>* reader) {
+  // Private ctor
+  auto result = std::shared_ptr<RecordBatchStreamReader>(new RecordBatchStreamReader());
+  RETURN_NOT_OK(result->impl_->Open(std::move(message_reader)));
+  *reader = result;
+  return Status::OK();
+}
+
+Status RecordBatchStreamReader::Open(io::InputStream* stream,
+                                     std::shared_ptr<RecordBatchReader>* out) {
+  std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
+  return Open(std::move(message_reader), out);
+}
+
+Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
+                                     std::shared_ptr<RecordBatchReader>* out) {
+  std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
+  return Open(std::move(message_reader), out);
+}
+
+#ifndef ARROW_NO_DEPRECATED_API
 Status RecordBatchStreamReader::Open(std::unique_ptr<MessageReader> message_reader,
                                      std::shared_ptr<RecordBatchStreamReader>* reader) {
   // Private ctor
@@ -471,6 +494,7 @@ Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& str
   std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
   return Open(std::move(message_reader), out);
 }
+#endif
 
 std::shared_ptr<Schema> RecordBatchStreamReader::schema() const {
   return impl_->schema();
@@ -559,8 +583,7 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
     DCHECK(BitUtil::IsMultipleOf8(block.body_length));
 
     std::unique_ptr<Message> message;
-    RETURN_NOT_OK(
-        ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
+    RETURN_NOT_OK(ReadMessage(block.offset, block.metadata_length, file_, &message));
 
     io::BufferReader reader(message->body());
     return ::arrow::ipc::ReadRecordBatch(*message->metadata(), schema_, &reader, batch);
@@ -578,8 +601,7 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
       DCHECK(BitUtil::IsMultipleOf8(block.body_length));
 
       std::unique_ptr<Message> message;
-      RETURN_NOT_OK(
-          ReadMessage(block.offset, block.metadata_length, file_.get(), &message));
+      RETURN_NOT_OK(ReadMessage(block.offset, block.metadata_length, file_, &message));
 
       io::BufferReader reader(message->body());
 
@@ -595,6 +617,11 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
   }
 
   Status Open(const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset) {
+    owned_file_ = file;
+    return Open(file.get(), footer_offset);
+  }
+
+  Status Open(io::RandomAccessFile* file, int64_t footer_offset) {
     file_ = file;
     footer_offset_ = footer_offset;
     RETURN_NOT_OK(ReadFooter());
@@ -604,7 +631,10 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
   std::shared_ptr<Schema> schema() const { return schema_; }
 
  private:
-  std::shared_ptr<io::RandomAccessFile> file_;
+  io::RandomAccessFile* file_;
+
+  // Deprecated as of 0.7.0
+  std::shared_ptr<io::RandomAccessFile> owned_file_;
 
   // The location where the Arrow file layout ends. May be the end of the file
   // or some other location if embedded in a larger file.
@@ -627,6 +657,19 @@ RecordBatchFileReader::RecordBatchFileReader() {
 
 RecordBatchFileReader::~RecordBatchFileReader() {}
 
+Status RecordBatchFileReader::Open(io::RandomAccessFile* file,
+                                   std::shared_ptr<RecordBatchFileReader>* reader) {
+  int64_t footer_offset;
+  RETURN_NOT_OK(file->GetSize(&footer_offset));
+  return Open(file, footer_offset, reader);
+}
+
+Status RecordBatchFileReader::Open(io::RandomAccessFile* file, int64_t footer_offset,
+                                   std::shared_ptr<RecordBatchFileReader>* reader) {
+  *reader = std::shared_ptr<RecordBatchFileReader>(new RecordBatchFileReader());
+  return (*reader)->impl_->Open(file, footer_offset);
+}
+
 Status RecordBatchFileReader::Open(const std::shared_ptr<io::RandomAccessFile>& file,
                                    std::shared_ptr<RecordBatchFileReader>* reader) {
   int64_t footer_offset;
@@ -654,33 +697,46 @@ Status RecordBatchFileReader::ReadRecordBatch(int i,
   return impl_->ReadRecordBatch(i, batch);
 }
 
-static Status ReadContiguousPayload(int64_t offset, io::RandomAccessFile* file,
+static Status ReadContiguousPayload(io::InputStream* file,
                                     std::unique_ptr<Message>* message) {
-  std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(file->Seek(offset));
   RETURN_NOT_OK(ReadMessage(file, message));
-
   if (*message == nullptr) {
     return Status::Invalid("Unable to read metadata at offset");
   }
   return Status::OK();
 }
 
-Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
-                       io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+Status ReadSchema(io::InputStream* stream, std::shared_ptr<Schema>* out) {
+  std::shared_ptr<RecordBatchReader> reader;
+  RETURN_NOT_OK(RecordBatchStreamReader::Open(stream, &reader));
+  *out = reader->schema();
+  return Status::OK();
+}
+
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, io::InputStream* file,
+                       std::shared_ptr<RecordBatch>* out) {
   std::unique_ptr<Message> message;
-  RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message));
+  RETURN_NOT_OK(ReadContiguousPayload(file, &message));
   io::BufferReader buffer_reader(message->body());
   return ReadRecordBatch(*message->metadata(), schema, kMaxNestingDepth, &buffer_reader,
                          out);
 }
 
+// Deprecated
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
+                       io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+  RETURN_NOT_OK(file->Seek(offset));
+  return ReadRecordBatch(schema, file, out);
+}
+
 Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
                   std::shared_ptr<Tensor>* out) {
   // Respect alignment of Tensor messages (see WriteTensor)
   offset = PaddedLength(offset);
+  RETURN_NOT_OK(file->Seek(offset));
+
   std::unique_ptr<Message> message;
-  RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message));
+  RETURN_NOT_OK(ReadContiguousPayload(file, &message));
 
   std::shared_ptr<DataType> type;
   std::vector<int64_t> shape;
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 4cffb18488047..f822a32c1cc38 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// Implement Arrow file layout for IPC/RPC purposes and short-lived storage
+// Read Arrow files and streams
 
 #ifndef ARROW_IPC_READER_H
 #define ARROW_IPC_READER_H
@@ -24,7 +24,7 @@
 #include <memory>
 #include <vector>
 
-#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/message.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -69,18 +69,32 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
   /// Create batch reader from generic MessageReader
   ///
   /// \param(in) message_reader a MessageReader implementation
-  /// \param(out) out the created RecordBatchStreamReader object
+  /// \param(out) out the created RecordBatchReader object
   /// \return Status
   static Status Open(std::unique_ptr<MessageReader> message_reader,
+                     std::shared_ptr<RecordBatchReader>* out);
+
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.7.0
+  static Status Open(std::unique_ptr<MessageReader> message_reader,
+                     std::shared_ptr<RecordBatchStreamReader>* out);
+
+  /// \deprecated Since 0.7.0
+  static Status Open(const std::shared_ptr<io::InputStream>& stream,
                      std::shared_ptr<RecordBatchStreamReader>* out);
+#endif
 
-  /// \Create Record batch stream reader from InputStream
+  /// \brief Record batch stream reader from InputStream
   ///
-  /// \param(in) stream an input stream instance
+  /// \param(in) stream an input stream instance. Must stay alive throughout
+  /// lifetime of stream reader
   /// \param(out) out the created RecordBatchStreamReader object
   /// \return Status
+  static Status Open(io::InputStream* stream, std::shared_ptr<RecordBatchReader>* out);
+
+  /// \brief Version of Open that retains ownership of stream
   static Status Open(const std::shared_ptr<io::InputStream>& stream,
-                     std::shared_ptr<RecordBatchStreamReader>* out);
+                     std::shared_ptr<RecordBatchReader>* out);
 
   std::shared_ptr<Schema> schema() const override;
   Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) override;
@@ -97,21 +111,31 @@ class ARROW_EXPORT RecordBatchFileReader {
  public:
   ~RecordBatchFileReader();
 
+  /// \brief Open a RecordBatchFileReader
   // Open a file-like object that is assumed to be self-contained; i.e., the
   // end of the file interface is the end of the Arrow file. Note that there
   // can be any amount of data preceding the Arrow-formatted data, because we
   // need only locate the end of the Arrow file stream to discover the metadata
   // and then proceed to read the data into memory.
+  static Status Open(io::RandomAccessFile* file,
+                     std::shared_ptr<RecordBatchFileReader>* reader);
+
+  /// \brief Open a RecordBatchFileReader
+  /// If the file is embedded within some larger file or memory region, you can
+  /// pass the absolute memory offset to the end of the file (which contains the
+  /// metadata footer). The metadata must have been written with memory offsets
+  /// relative to the start of the containing file
+  ///
+  /// @param file: the data source
+  /// @param footer_offset: the position of the end of the Arrow "file"
+  static Status Open(io::RandomAccessFile* file, int64_t footer_offset,
+                     std::shared_ptr<RecordBatchFileReader>* reader);
+
+  /// \brief Version of Open that retains ownership of file
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
                      std::shared_ptr<RecordBatchFileReader>* reader);
 
-  // If the file is embedded within some larger file or memory region, you can
-  // pass the absolute memory offset to the end of the file (which contains the
-  // metadata footer). The metadata must have been written with memory offsets
-  // relative to the start of the containing file
-  //
-  // @param file: the data source
-  // @param footer_offset: the position of the end of the Arrow "file"
+  /// \brief Version of Open that retains ownership of file
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
                      int64_t footer_offset,
                      std::shared_ptr<RecordBatchFileReader>* reader);
@@ -142,6 +166,27 @@ class ARROW_EXPORT RecordBatchFileReader {
 
 // Generic read functions; does not copy data if the input supports zero copy reads
 
+/// \brief Read Schema from stream serialized as a sequence of IPC messages
+///
+/// \param[in] stream an InputStream
+/// \param[out] out the output Schema
+///
+/// If record batches follow the schema, it is better to use
+/// RecordBatchStreamReader
+ARROW_EXPORT
+Status ReadSchema(io::InputStream* stream, std::shared_ptr<Schema>* out);
+
+/// Read record batch as encapsulated IPC message with metadata size prefix and
+/// header
+///
+/// \param(in) schema the record batch schema
+/// \param(in) offset the file location of the start of the message
+/// \param(in) file the file where the batch is located
+/// \param(out) out the read record batch
+ARROW_EXPORT
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, io::InputStream* stream,
+                       std::shared_ptr<RecordBatch>* out);
+
 /// \brief Read record batch from file given metadata and schema
 ///
 /// \param(in) metadata a Message containing the record batch metadata
@@ -174,17 +219,6 @@ Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& sc
                        int max_recursion_depth, io::RandomAccessFile* file,
                        std::shared_ptr<RecordBatch>* out);
 
-/// Read record batch as encapsulated IPC message with metadata size prefix and
-/// header
-///
-/// \param(in) schema the record batch schema
-/// \param(in) offset the file location of the start of the message
-/// \param(in) file the file where the batch is located
-/// \param(out) out the read record batch
-ARROW_EXPORT
-Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
-                       io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
-
 /// EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
 ///
 /// \param(in) offset the file location of the start of the message
@@ -194,6 +228,15 @@ ARROW_EXPORT
 Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
                   std::shared_ptr<Tensor>* out);
 
+#ifndef ARROW_NO_DEPRECATED_API
+/// \deprecated Since 0.7.0
+///
+/// Deprecated in favor of more general InputStream-based API
+ARROW_EXPORT
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
+                       io::RandomAccessFile* stream, std::shared_ptr<RecordBatch>* out);
+#endif
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/stream-to-file.cc b/cpp/src/arrow/ipc/stream-to-file.cc
index 33719b3c89c9e..96339c83f27e3 100644
--- a/cpp/src/arrow/ipc/stream-to-file.cc
+++ b/cpp/src/arrow/ipc/stream-to-file.cc
@@ -30,12 +30,12 @@ namespace ipc {
 // A typical usage would be:
 // $ <program that produces streaming output> | stream-to-file > file.arrow
 Status ConvertToFile() {
-  std::shared_ptr<io::InputStream> input(new io::StdinStream);
-  std::shared_ptr<RecordBatchStreamReader> reader;
-  RETURN_NOT_OK(RecordBatchStreamReader::Open(input, &reader));
+  io::StdinStream input;
+  std::shared_ptr<RecordBatchReader> reader;
+  RETURN_NOT_OK(RecordBatchStreamReader::Open(&input, &reader));
 
   io::StdoutStream sink;
-  std::shared_ptr<RecordBatchFileWriter> writer;
+  std::shared_ptr<RecordBatchWriter> writer;
   RETURN_NOT_OK(RecordBatchFileWriter::Open(&sink, reader->schema(), &writer));
 
   std::shared_ptr<RecordBatch> batch;
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 9492364413e05..9c05cba918d83 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -28,7 +28,8 @@
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
-#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/message.h"
+#include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/util.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
@@ -542,25 +543,9 @@ Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
   return writer.Write(batch, dst, metadata_length, body_length);
 }
 
-Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
-                            std::shared_ptr<Buffer>* out) {
-  int64_t size = 0;
-  RETURN_NOT_OK(GetRecordBatchSize(batch, &size));
-  std::shared_ptr<MutableBuffer> buffer;
-  RETURN_NOT_OK(AllocateBuffer(pool, size, &buffer));
-
-  io::FixedSizeBufferWriter stream(buffer);
-  int32_t metadata_length = 0;
-  int64_t body_length = 0;
-  RETURN_NOT_OK(WriteRecordBatch(batch, 0, &stream, &metadata_length, &body_length, pool,
-                                 kMaxNestingDepth, true));
-  *out = buffer;
-  return Status::OK();
-}
-
 Status WriteRecordBatchStream(const std::vector<std::shared_ptr<RecordBatch>>& batches,
                               io::OutputStream* dst) {
-  std::shared_ptr<RecordBatchStreamWriter> writer;
+  std::shared_ptr<RecordBatchWriter> writer;
   RETURN_NOT_OK(RecordBatchStreamWriter::Open(dst, batches[0]->schema(), &writer));
   for (const auto& batch : batches) {
     // allow sizes > INT32_MAX
@@ -633,32 +618,44 @@ RecordBatchWriter::~RecordBatchWriter() {}
 // ----------------------------------------------------------------------
 // Stream writer implementation
 
-class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
+class StreamBookKeeper {
  public:
-  RecordBatchStreamWriterImpl()
-      : pool_(default_memory_pool()), position_(-1), started_(false) {}
+  StreamBookKeeper() : sink_(nullptr), position_(-1) {}
+  explicit StreamBookKeeper(io::OutputStream* sink) : sink_(sink), position_(-1) {}
 
-  virtual ~RecordBatchStreamWriterImpl() = default;
+  Status UpdatePosition() { return sink_->Tell(&position_); }
 
-  Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema) {
-    sink_ = sink;
-    schema_ = schema;
-    return UpdatePosition();
+  Status Align(int64_t alignment = kArrowIpcAlignment) {
+    // Adds padding bytes if necessary to ensure all memory blocks are written on
+    // 8-byte (or other alignment) boundaries.
+    int64_t remainder = PaddedLength(position_, alignment) - position_;
+    if (remainder > 0) {
+      return Write(kPaddingBytes, remainder);
+    }
+    return Status::OK();
   }
 
-  virtual Status Start() {
-    RETURN_NOT_OK(WriteSchema());
-
-    // If there are any dictionaries, write them as the next messages
-    RETURN_NOT_OK(WriteDictionaries());
-
-    started_ = true;
+  // Write data and update position
+  Status Write(const uint8_t* data, int64_t nbytes) {
+    RETURN_NOT_OK(sink_->Write(data, nbytes));
+    position_ += nbytes;
     return Status::OK();
   }
 
+ protected:
+  io::OutputStream* sink_;
+  int64_t position_;
+};
+
+class SchemaWriter : public StreamBookKeeper {
+ public:
+  SchemaWriter(const Schema& schema, DictionaryMemo* dictionary_memo, MemoryPool* pool,
+               io::OutputStream* sink)
+      : StreamBookKeeper(sink), schema_(schema), dictionary_memo_(dictionary_memo) {}
+
   Status WriteSchema() {
     std::shared_ptr<Buffer> schema_fb;
-    RETURN_NOT_OK(WriteSchemaMessage(*schema_, &dictionary_memo_, &schema_fb));
+    RETURN_NOT_OK(WriteSchemaMessage(schema_, dictionary_memo_, &schema_fb));
 
     int32_t metadata_length = 0;
     RETURN_NOT_OK(WriteMessage(*schema_fb, sink_, &metadata_length));
@@ -667,34 +664,15 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
     return Status::OK();
   }
 
-  virtual Status Close() {
-    // Write the schema if not already written
-    // User is responsible for closing the OutputStream
-    RETURN_NOT_OK(CheckStarted());
+  Status WriteDictionaries(std::vector<FileBlock>* dictionaries) {
+    const DictionaryMap& id_to_dictionary = dictionary_memo_->id_to_dictionary();
 
-    // Write 0 EOS message
-    const int32_t kEos = 0;
-    return Write(reinterpret_cast<const uint8_t*>(&kEos), sizeof(int32_t));
-  }
-
-  Status CheckStarted() {
-    if (!started_) {
-      return Start();
-    }
-    return Status::OK();
-  }
-
-  Status UpdatePosition() { return sink_->Tell(&position_); }
-
-  Status WriteDictionaries() {
-    const DictionaryMap& id_to_dictionary = dictionary_memo_.id_to_dictionary();
-
-    dictionaries_.resize(id_to_dictionary.size());
+    dictionaries->resize(id_to_dictionary.size());
 
     // TODO(wesm): does sorting by id yield any benefit?
     int dict_index = 0;
     for (const auto& entry : id_to_dictionary) {
-      FileBlock* block = &dictionaries_[dict_index++];
+      FileBlock* block = &(*dictionaries)[dict_index++];
 
       block->offset = position_;
 
@@ -709,8 +687,57 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
     return Status::OK();
   }
 
+  Status Write(std::vector<FileBlock>* dictionaries) {
+    RETURN_NOT_OK(WriteSchema());
+
+    // If there are any dictionaries, write them as the next messages
+    return WriteDictionaries(dictionaries);
+  }
+
+ private:
+  MemoryPool* pool_;
+  const Schema& schema_;
+  DictionaryMemo* dictionary_memo_;
+};
+
+class RecordBatchStreamWriter::RecordBatchStreamWriterImpl : public StreamBookKeeper {
+ public:
+  RecordBatchStreamWriterImpl(io::OutputStream* sink,
+                              const std::shared_ptr<Schema>& schema)
+      : StreamBookKeeper(sink),
+        schema_(schema),
+        pool_(default_memory_pool()),
+        started_(false) {}
+
+  virtual ~RecordBatchStreamWriterImpl() = default;
+
+  virtual Status Start() {
+    SchemaWriter schema_writer(*schema_, &dictionary_memo_, pool_, sink_);
+    RETURN_NOT_OK(schema_writer.Write(&dictionaries_));
+    started_ = true;
+    return Status::OK();
+  }
+
+  virtual Status Close() {
+    // Write the schema if not already written
+    // User is responsible for closing the OutputStream
+    RETURN_NOT_OK(CheckStarted());
+
+    // Write 0 EOS message
+    const int32_t kEos = 0;
+    return Write(reinterpret_cast<const uint8_t*>(&kEos), sizeof(int32_t));
+  }
+
+  Status CheckStarted() {
+    if (!started_) {
+      return Start();
+    }
+    return Status::OK();
+  }
+
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit, FileBlock* block) {
     RETURN_NOT_OK(CheckStarted());
+    RETURN_NOT_OK(UpdatePosition());
 
     block->offset = position_;
 
@@ -733,45 +760,22 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl {
                             &record_batches_[record_batches_.size() - 1]);
   }
 
-  Status Align(int64_t alignment = kArrowIpcAlignment) {
-    // Adds padding bytes if necessary to ensure all memory blocks are written on
-    // 8-byte (or other alignment) boundaries.
-    int64_t remainder = PaddedLength(position_, alignment) - position_;
-    if (remainder > 0) {
-      return Write(kPaddingBytes, remainder);
-    }
-    return Status::OK();
-  }
-
-  // Write data and update position
-  Status Write(const uint8_t* data, int64_t nbytes) {
-    RETURN_NOT_OK(sink_->Write(data, nbytes));
-    position_ += nbytes;
-    return Status::OK();
-  }
-
   void set_memory_pool(MemoryPool* pool) { pool_ = pool; }
 
  protected:
-  io::OutputStream* sink_;
   std::shared_ptr<Schema> schema_;
+  MemoryPool* pool_;
+  bool started_;
 
   // When writing out the schema, we keep track of all the dictionaries we
   // encounter, as they must be written out first in the stream
   DictionaryMemo dictionary_memo_;
 
-  MemoryPool* pool_;
-
-  int64_t position_;
-  bool started_;
-
   std::vector<FileBlock> dictionaries_;
   std::vector<FileBlock> record_batches_;
 };
 
-RecordBatchStreamWriter::RecordBatchStreamWriter() {
-  impl_.reset(new RecordBatchStreamWriterImpl());
-}
+RecordBatchStreamWriter::RecordBatchStreamWriter() {}
 
 RecordBatchStreamWriter::~RecordBatchStreamWriter() {}
 
@@ -784,13 +788,26 @@ void RecordBatchStreamWriter::set_memory_pool(MemoryPool* pool) {
   impl_->set_memory_pool(pool);
 }
 
+Status RecordBatchStreamWriter::Open(io::OutputStream* sink,
+                                     const std::shared_ptr<Schema>& schema,
+                                     std::shared_ptr<RecordBatchWriter>* out) {
+  // ctor is private
+  auto result = std::shared_ptr<RecordBatchStreamWriter>(new RecordBatchStreamWriter());
+  result->impl_.reset(new RecordBatchStreamWriterImpl(sink, schema));
+  *out = result;
+  return Status::OK();
+}
+
+#ifndef ARROW_NO_DEPRECATED_API
 Status RecordBatchStreamWriter::Open(io::OutputStream* sink,
                                      const std::shared_ptr<Schema>& schema,
                                      std::shared_ptr<RecordBatchStreamWriter>* out) {
   // ctor is private
   *out = std::shared_ptr<RecordBatchStreamWriter>(new RecordBatchStreamWriter());
-  return (*out)->impl_->Open(sink, schema);
+  (*out)->impl_.reset(new RecordBatchStreamWriterImpl(sink, schema));
+  return Status::OK();
 }
+#endif
 
 Status RecordBatchStreamWriter::Close() { return impl_->Close(); }
 
@@ -802,6 +819,9 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
  public:
   using BASE = RecordBatchStreamWriter::RecordBatchStreamWriterImpl;
 
+  RecordBatchFileWriterImpl(io::OutputStream* sink, const std::shared_ptr<Schema>& schema)
+      : BASE(sink, schema) {}
+
   Status Start() override {
     // It is only necessary to align to 8-byte boundary at the start of the file
     RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
@@ -815,6 +835,8 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
 
   Status Close() override {
     // Write metadata
+    RETURN_NOT_OK(UpdatePosition());
+
     int64_t initial_position = position_;
     RETURN_NOT_OK(WriteFileFooter(*schema_, dictionaries_, record_batches_,
                                   &dictionary_memo_, sink_));
@@ -836,19 +858,30 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
   }
 };
 
-RecordBatchFileWriter::RecordBatchFileWriter() {
-  impl_.reset(new RecordBatchFileWriterImpl());
-}
+RecordBatchFileWriter::RecordBatchFileWriter() {}
 
 RecordBatchFileWriter::~RecordBatchFileWriter() {}
 
+Status RecordBatchFileWriter::Open(io::OutputStream* sink,
+                                   const std::shared_ptr<Schema>& schema,
+                                   std::shared_ptr<RecordBatchWriter>* out) {
+  // ctor is private
+  auto result = std::shared_ptr<RecordBatchFileWriter>(new RecordBatchFileWriter());
+  result->impl_.reset(new RecordBatchFileWriterImpl(sink, schema));
+  *out = result;
+  return Status::OK();
+}
+
+#ifndef ARROW_NO_DEPRECATED_API
 Status RecordBatchFileWriter::Open(io::OutputStream* sink,
                                    const std::shared_ptr<Schema>& schema,
                                    std::shared_ptr<RecordBatchFileWriter>* out) {
-  *out = std::shared_ptr<RecordBatchFileWriter>(
-      new RecordBatchFileWriter());  // ctor is private
-  return (*out)->impl_->Open(sink, schema);
+  // ctor is private
+  *out = std::shared_ptr<RecordBatchFileWriter>(new RecordBatchFileWriter());
+  (*out)->impl_.reset(new RecordBatchFileWriterImpl(sink, schema));
+  return Status::OK();
 }
+#endif
 
 Status RecordBatchFileWriter::WriteRecordBatch(const RecordBatch& batch,
                                                bool allow_64bit) {
@@ -857,5 +890,39 @@ Status RecordBatchFileWriter::WriteRecordBatch(const RecordBatch& batch,
 
 Status RecordBatchFileWriter::Close() { return impl_->Close(); }
 
+// ----------------------------------------------------------------------
+// Serialization public APIs
+
+Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
+                            std::shared_ptr<Buffer>* out) {
+  int64_t size = 0;
+  RETURN_NOT_OK(GetRecordBatchSize(batch, &size));
+  std::shared_ptr<MutableBuffer> buffer;
+  RETURN_NOT_OK(AllocateBuffer(pool, size, &buffer));
+
+  io::FixedSizeBufferWriter stream(buffer);
+  int32_t metadata_length = 0;
+  int64_t body_length = 0;
+  RETURN_NOT_OK(WriteRecordBatch(batch, 0, &stream, &metadata_length, &body_length, pool,
+                                 kMaxNestingDepth, true));
+  *out = buffer;
+  return Status::OK();
+}
+
+Status SerializeSchema(const Schema& schema, MemoryPool* pool,
+                       std::shared_ptr<Buffer>* out) {
+  std::shared_ptr<io::BufferOutputStream> stream;
+  RETURN_NOT_OK(io::BufferOutputStream::Create(1024, pool, &stream));
+
+  DictionaryMemo memo;
+  SchemaWriter schema_writer(schema, &memo, pool, stream.get());
+
+  // Unused
+  std::vector<FileBlock> dictionary_blocks;
+
+  RETURN_NOT_OK(schema_writer.Write(&dictionary_blocks));
+  return stream->Finish(out);
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 79ea7c6664bbd..d867982d2be02 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -25,7 +25,7 @@
 #include <memory>
 #include <vector>
 
-#include "arrow/ipc/metadata.h"
+#include "arrow/ipc/message.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -85,8 +85,14 @@ class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
   /// \param(in) schema the schema of the record batches to be written
   /// \param(out) out the created stream writer
   /// \return Status indicating success or failure
+  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+                     std::shared_ptr<RecordBatchWriter>* out);
+
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.7.0
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchStreamWriter>* out);
+#endif
 
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
   Status Close() override;
@@ -113,8 +119,14 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
   /// \param(in) schema the schema of the record batches to be written
   /// \param(out) out the created stream writer
   /// \return Status indicating success or failure
+  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
+                     std::shared_ptr<RecordBatchWriter>* out);
+
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.7.0
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchFileWriter>* out);
+#endif
 
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
   Status Close() override;
@@ -155,14 +167,6 @@ Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
                         int max_recursion_depth = kMaxNestingDepth,
                         bool allow_64bit = false);
 
-/// \brief Write dictionary message to output stream
-///
-/// Low-level API
-ARROW_EXPORT
-Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
-                       int64_t buffer_start_offset, io::OutputStream* dst,
-                       int32_t* metadata_length, int64_t* body_length, MemoryPool* pool);
-
 /// \brief Serialize record batch as encapsulated IPC message in a new buffer
 ///
 /// \param[in] batch the record batch
@@ -173,6 +177,17 @@ ARROW_EXPORT
 Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
                             std::shared_ptr<Buffer>* out);
 
+/// \brief Serialize schema using stream writer as a sequence of one or more
+/// IPC messages
+///
+/// \param[in] scheam the schema to write
+/// \param[in] pool a MemoryPool to allocate memory from
+/// \param[out] out the serialized schema
+/// \return Status
+ARROW_EXPORT
+Status SerializeSchema(const Schema& schema, MemoryPool* pool,
+                       std::shared_ptr<Buffer>* out);
+
 /// \brief Write multiple record batches to OutputStream
 /// \param[in] batches a vector of batches. Must all have same schema
 /// \param[out] dst an OutputStream
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index 622ef8299374a..dcb06f80acc9e 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -189,22 +189,23 @@ Status DeserializeTuple(std::shared_ptr<Array> array, int64_t start_idx, int64_t
   DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SET_ITEM)
 }
 
-Status ReadSerializedObject(std::shared_ptr<io::RandomAccessFile> src,
-                            SerializedPyObject* out) {
-  std::shared_ptr<ipc::RecordBatchStreamReader> reader;
+Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out) {
   int64_t offset;
   int64_t bytes_read;
   int32_t num_tensors;
   // Read number of tensors
   RETURN_NOT_OK(
       src->Read(sizeof(int32_t), &bytes_read, reinterpret_cast<uint8_t*>(&num_tensors)));
+
+  std::shared_ptr<ipc::RecordBatchReader> reader;
   RETURN_NOT_OK(ipc::RecordBatchStreamReader::Open(src, &reader));
   RETURN_NOT_OK(reader->ReadNextRecordBatch(&out->batch));
+
   RETURN_NOT_OK(src->Tell(&offset));
   offset += 4;  // Skip the end-of-stream message
   for (int i = 0; i < num_tensors; ++i) {
     std::shared_ptr<Tensor> tensor;
-    RETURN_NOT_OK(ipc::ReadTensor(offset, src.get(), &tensor));
+    RETURN_NOT_OK(ipc::ReadTensor(offset, src, &tensor));
     out->tensors.push_back(tensor);
     RETURN_NOT_OK(src->Tell(&offset));
   }
diff --git a/cpp/src/arrow/python/arrow_to_python.h b/cpp/src/arrow/python/arrow_to_python.h
index 559ce18c50709..e187d59696906 100644
--- a/cpp/src/arrow/python/arrow_to_python.h
+++ b/cpp/src/arrow/python/arrow_to_python.h
@@ -46,8 +46,7 @@ namespace py {
 /// \param[out] out the reconstructed data
 /// \return Status
 ARROW_EXPORT
-Status ReadSerializedObject(std::shared_ptr<io::RandomAccessFile> src,
-                            SerializedPyObject* out);
+Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out);
 
 /// \brief Reconstruct Python object from Arrow-serialized representation
 /// \param[in] object
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index a1ccd994e6caa..e00f1943f723c 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -53,18 +53,18 @@ namespace py {
 /// scalar Python types, lists, tuples, dictionaries and tensors.
 class SequenceBuilder {
  public:
-  explicit SequenceBuilder(MemoryPool* pool = nullptr)
+  explicit SequenceBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
       : pool_(pool),
-        types_(pool, ::arrow::int8()),
-        offsets_(pool, ::arrow::int32()),
+        types_(::arrow::int8(), pool),
+        offsets_(::arrow::int32(), pool),
         nones_(pool),
-        bools_(pool, ::arrow::boolean()),
-        ints_(pool, ::arrow::int64()),
-        bytes_(pool, ::arrow::binary()),
+        bools_(::arrow::boolean(), pool),
+        ints_(::arrow::int64(), pool),
+        bytes_(::arrow::binary(), pool),
         strings_(pool),
-        floats_(pool, ::arrow::float32()),
-        doubles_(pool, ::arrow::float64()),
-        tensor_indices_(pool, ::arrow::int32()),
+        floats_(::arrow::float32(), pool),
+        doubles_(::arrow::float64(), pool),
+        tensor_indices_(::arrow::int32(), pool),
         list_offsets_({0}),
         tuple_offsets_({0}),
         dict_offsets_({0}) {}
@@ -184,7 +184,7 @@ class SequenceBuilder {
     if (data != nullptr) {
       DCHECK(data->length() == offsets.back());
       std::shared_ptr<Array> offset_array;
-      Int32Builder builder(pool_, std::make_shared<Int32Type>());
+      Int32Builder builder(::arrow::int32(), pool_);
       RETURN_NOT_OK(builder.Append(offsets.data(), offsets.size()));
       RETURN_NOT_OK(builder.Finish(&offset_array));
       std::shared_ptr<Array> list_array;
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 82cd13713763d..b7963b877d6af 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -313,6 +313,8 @@ Status Schema::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metada
   return Status::OK();
 }
 
+std::shared_ptr<const KeyValueMetadata> Schema::metadata() const { return metadata_; }
+
 std::shared_ptr<Schema> Schema::RemoveMetadata() const {
   return std::make_shared<Schema>(fields_);
 }
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 30cd71e6c5ca0..4cd17bc37f47e 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -730,7 +730,7 @@ class ARROW_EXPORT Schema {
   /// \brief The custom key-value metadata, if any
   ///
   /// \return metadata may be nullptr
-  std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
+  std::shared_ptr<const KeyValueMetadata> metadata() const;
 
   /// \brief Render a string representation of the schema suitable for debugging
   std::string ToString() const;
diff --git a/cpp/src/arrow/util/key_value_metadata.cc b/cpp/src/arrow/util/key_value_metadata.cc
index 6877a6a5382fe..0497f65c30888 100644
--- a/cpp/src/arrow/util/key_value_metadata.cc
+++ b/cpp/src/arrow/util/key_value_metadata.cc
@@ -46,7 +46,9 @@ KeyValueMetadata::KeyValueMetadata() : keys_(), values_() {}
 
 KeyValueMetadata::KeyValueMetadata(
     const std::unordered_map<std::string, std::string>& map)
-    : keys_(UnorderedMapKeys(map)), values_(UnorderedMapValues(map)) {}
+    : keys_(UnorderedMapKeys(map)), values_(UnorderedMapValues(map)) {
+  DCHECK_EQ(keys_.size(), values_.size());
+}
 
 KeyValueMetadata::KeyValueMetadata(const std::vector<std::string>& keys,
                                    const std::vector<std::string>& values)
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index d00286de409b3..68ae017ce71ad 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -100,8 +100,8 @@
 from pyarrow.ipc import (Message, MessageReader,
                          RecordBatchFileReader, RecordBatchFileWriter,
                          RecordBatchStreamReader, RecordBatchStreamWriter,
-                         read_message, read_record_batch, read_tensor,
-                         write_tensor,
+                         read_message, read_record_batch, read_schema,
+                         read_tensor, write_tensor,
                          get_record_batch_size, get_tensor_size,
                          open_stream,
                          open_file,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 082fb61429825..fcf27da3c10c8 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -632,8 +632,8 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
     cdef cppclass CRecordBatchStreamReader \
             " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
         @staticmethod
-        CStatus Open(const shared_ptr[InputStream]& stream,
-                     shared_ptr[CRecordBatchStreamReader]* out)
+        CStatus Open(const InputStream* stream,
+                     shared_ptr[CRecordBatchReader]* out)
 
         @staticmethod
         CStatus Open2" Open"(unique_ptr[CMessageReader] message_reader,
@@ -643,22 +643,22 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
             " arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter):
         @staticmethod
         CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
-                     shared_ptr[CRecordBatchStreamWriter]* out)
+                     shared_ptr[CRecordBatchWriter]* out)
 
     cdef cppclass CRecordBatchFileWriter \
             " arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter):
         @staticmethod
         CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
-                     shared_ptr[CRecordBatchFileWriter]* out)
+                     shared_ptr[CRecordBatchWriter]* out)
 
     cdef cppclass CRecordBatchFileReader \
             " arrow::ipc::RecordBatchFileReader":
         @staticmethod
-        CStatus Open(const shared_ptr[RandomAccessFile]& file,
+        CStatus Open(RandomAccessFile* file,
                      shared_ptr[CRecordBatchFileReader]* out)
 
         @staticmethod
-        CStatus Open2" Open"(const shared_ptr[RandomAccessFile]& file,
+        CStatus Open2" Open"(RandomAccessFile* file,
                              int64_t footer_offset,
                              shared_ptr[CRecordBatchFileReader]* out)
 
@@ -684,6 +684,19 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
                             const shared_ptr[CSchema]& schema,
                             shared_ptr[CRecordBatch]* out)
 
+    CStatus SerializeSchema(const CSchema& schema, CMemoryPool* pool,
+                            shared_ptr[CBuffer]* out)
+
+    CStatus SerializeRecordBatch(const CRecordBatch& schema,
+                                 CMemoryPool* pool,
+                                 shared_ptr[CBuffer]* out)
+
+    CStatus ReadSchema(InputStream* stream, shared_ptr[CSchema]* out)
+
+    CStatus ReadRecordBatch(const shared_ptr[CSchema]& schema,
+                            InputStream* stream,
+                            shared_ptr[CRecordBatch]* out)
+
 
 cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
 
@@ -792,7 +805,7 @@ cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:
     CStatus DeserializeObject(const CSerializedPyObject& obj,
                               PyObject* base, PyObject** out)
 
-    CStatus ReadSerializedObject(shared_ptr[RandomAccessFile] src,
+    CStatus ReadSerializedObject(RandomAccessFile* src,
                                  CSerializedPyObject* out)
 
     void set_serialization_callbacks(object serialize_callback,
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index ceed4b0e85248..027a00d8d4992 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -70,7 +70,7 @@ cdef class Message:
 
     def serialize(self, memory_pool=None):
         """
-        Write message to Buffer with length-prefixed metadata, then body
+        Write message as encapsulated IPC message
 
         Parameters
         ----------
@@ -166,18 +166,13 @@ cdef class _RecordBatchWriter:
         pass
 
     def _open(self, sink, Schema schema):
-        cdef:
-            shared_ptr[CRecordBatchStreamWriter] writer
-
         get_writer(sink, &self.sink)
 
         with nogil:
             check_status(
                 CRecordBatchStreamWriter.Open(self.sink.get(),
                                               schema.sp_schema,
-                                              &writer))
-
-        self.writer = <shared_ptr[CRecordBatchWriter]> writer
+                                              &self.writer))
         self.closed = False
 
     def write_batch(self, RecordBatch batch):
@@ -215,6 +210,7 @@ cdef get_input_stream(object source, shared_ptr[InputStream]* out):
 cdef class _RecordBatchReader:
     cdef:
         shared_ptr[CRecordBatchReader] reader
+        shared_ptr[InputStream] in_stream
 
     cdef readonly:
         Schema schema
@@ -223,16 +219,11 @@ cdef class _RecordBatchReader:
         pass
 
     def _open(self, source):
-        cdef:
-            shared_ptr[InputStream] in_stream
-            shared_ptr[CRecordBatchStreamReader] reader
-
-        get_input_stream(source, &in_stream)
-
+        get_input_stream(source, &self.in_stream)
         with nogil:
-            check_status(CRecordBatchStreamReader.Open(in_stream, &reader))
+            check_status(CRecordBatchStreamReader.Open(
+                self.in_stream.get(), &self.reader))
 
-        self.reader = <shared_ptr[CRecordBatchReader]> reader
         self.schema = Schema()
         self.schema.init_schema(self.reader.get().schema())
 
@@ -285,22 +276,20 @@ cdef class _RecordBatchReader:
 cdef class _RecordBatchFileWriter(_RecordBatchWriter):
 
     def _open(self, sink, Schema schema):
-        cdef shared_ptr[CRecordBatchFileWriter] writer
         get_writer(sink, &self.sink)
 
         with nogil:
             check_status(
                 CRecordBatchFileWriter.Open(self.sink.get(), schema.sp_schema,
-                                            &writer))
+                                            &self.writer))
 
-        # Cast to base class, because has same interface
-        self.writer = <shared_ptr[CRecordBatchWriter]> writer
         self.closed = False
 
 
 cdef class _RecordBatchFileReader:
     cdef:
         shared_ptr[CRecordBatchFileReader] reader
+        shared_ptr[RandomAccessFile] file
 
     cdef readonly:
         Schema schema
@@ -309,8 +298,7 @@ cdef class _RecordBatchFileReader:
         pass
 
     def _open(self, source, footer_offset=None):
-        cdef shared_ptr[RandomAccessFile] reader
-        get_reader(source, &reader)
+        get_reader(source, &self.file)
 
         cdef int64_t offset = 0
         if footer_offset is not None:
@@ -318,10 +306,12 @@ cdef class _RecordBatchFileReader:
 
         with nogil:
             if offset != 0:
-                check_status(CRecordBatchFileReader.Open2(
-                    reader, offset, &self.reader))
+                check_status(
+                    CRecordBatchFileReader.Open2(self.file.get(), offset,
+                                                 &self.reader))
             else:
-                check_status(CRecordBatchFileReader.Open(reader, &self.reader))
+                check_status(
+                    CRecordBatchFileReader.Open(self.file.get(), &self.reader))
 
         self.schema = pyarrow_wrap_schema(self.reader.get().schema())
 
@@ -476,24 +466,57 @@ def read_message(source):
     return result
 
 
-def read_record_batch(Message batch_message, Schema schema):
+def read_schema(obj):
+    """
+    Read Schema from message or buffer
+
+    Parameters
+    ----------
+    obj : buffer or Message
+
+    Returns
+    -------
+    schema : Schema
+    """
+    cdef:
+        shared_ptr[CSchema] result
+        shared_ptr[RandomAccessFile] cpp_file
+
+    if isinstance(obj, Message):
+        raise NotImplementedError(type(obj))
+
+    get_reader(obj, &cpp_file)
+
+    with nogil:
+        check_status(ReadSchema(cpp_file.get(), &result))
+
+    return pyarrow_wrap_schema(result)
+
+
+def read_record_batch(obj, Schema schema):
     """
     Read RecordBatch from message, given a known schema
 
     Parameters
     ----------
-    batch_message : Message
-        Such as that obtained from read_message
+    obj : Message or Buffer-like
     schema : Schema
 
     Returns
     -------
     batch : RecordBatch
     """
-    cdef shared_ptr[CRecordBatch] result
+    cdef:
+        shared_ptr[CRecordBatch] result
+        Message message
+
+    if isinstance(obj, Message):
+        message = obj
+    else:
+        message = read_message(obj)
 
     with nogil:
-        check_status(ReadRecordBatch(deref(batch_message.message.get()),
+        check_status(ReadRecordBatch(deref(message.message.get()),
                                      schema.sp_schema, &result))
 
     return pyarrow_wrap_batch(result)
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index d527722295d1a..6eb4979f1a2b7 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -20,7 +20,7 @@
 import pyarrow as pa
 
 from pyarrow.lib import (Message, MessageReader,  # noqa
-                         read_message, read_record_batch,
+                         read_message, read_record_batch, read_schema,
                          read_tensor, write_tensor,
                          get_record_batch_size, get_tensor_size)
 import pyarrow.lib as lib
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index a6c955bef99ba..3ee34eed5ed95 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -237,7 +237,7 @@ def read_serialized(source, base=None):
     cdef SerializedPyObject serialized = SerializedPyObject()
     serialized.base = base
     with nogil:
-        check_status(ReadSerializedObject(stream, &serialized.data))
+        check_status(ReadSerializedObject(stream.get(), &serialized.data))
 
     return serialized
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 976f4297d5228..dd3359ef13968 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -453,6 +453,28 @@ cdef class RecordBatch:
         else:
             return self.column(key)
 
+    def serialize(self, memory_pool=None):
+        """
+        Write RecordBatch to Buffer as encapsulated IPC message
+
+        Parameters
+        ----------
+        memory_pool : MemoryPool, default None
+            Uses default memory pool if not specified
+
+        Returns
+        -------
+        serialized : Buffer
+        """
+        cdef:
+            shared_ptr[CBuffer] buffer
+            CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)
+
+        with nogil:
+            check_status(SerializeRecordBatch(deref(self.batch),
+                                              pool, &buffer))
+        return pyarrow_wrap_buffer(buffer)
+
     def slice(self, offset=0, length=None):
         """
         Compute zero-copy slice of this RecordBatch
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 120a9825a7b56..ecdbe62113392 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -384,6 +384,21 @@ def test_pandas_serialize_round_trip_not_string_columns():
     assert_frame_equal(result, df)
 
 
+def test_schema_batch_serialize_methods():
+    nrows = 5
+    df = pd.DataFrame({
+        'one': np.random.randn(nrows),
+        'two': ['foo', np.nan, 'bar', 'bazbaz', 'qux']})
+    batch = pa.RecordBatch.from_pandas(df)
+
+    s_schema = batch.schema.serialize()
+    s_batch = batch.serialize()
+
+    recons_schema = pa.read_schema(s_schema)
+    recons_batch = pa.read_record_batch(s_batch, recons_schema)
+    assert recons_batch.equals(batch)
+
+
 def write_file(batch, sink):
     writer = pa.RecordBatchFileWriter(sink, batch.schema)
     writer.write_batch(batch)
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 28b98f0952a9a..424e5184c0d3d 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -67,6 +67,8 @@ def test_recordbatch_basics():
 
     batch = pa.RecordBatch.from_arrays(data, ['c0', 'c1'])
 
+    batch.schema.metadata
+
     assert len(batch) == 5
     assert batch.num_rows == 5
     assert batch.num_columns == len(data)
@@ -80,6 +82,16 @@ def test_recordbatch_basics():
         batch[2]
 
 
+def test_recordbatch_empty_metadata():
+    data = [
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10])
+    ]
+
+    batch = pa.RecordBatch.from_arrays(data, ['c0', 'c1'])
+    assert batch.schema.metadata is None
+
+
 def test_recordbatch_slice_getitem():
     data = [
         pa.array(range(5)),
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 592db4f90dac1..30c3aa6bc7d4f 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -235,7 +235,9 @@ cdef class Field:
 
         def __get__(self):
             self._check_null()
-            return box_metadata(self.field.metadata().get())
+            cdef shared_ptr[const CKeyValueMetadata] metadata = (
+                self.field.metadata())
+            return box_metadata(metadata.get())
 
     def _check_null(self):
         if self.field == NULL:
@@ -306,6 +308,11 @@ cdef class Schema:
 
         return result
 
+    def _check_null(self):
+        if self.schema == NULL:
+            raise ReferenceError(
+                'Schema not initialized (references NULL pointer)')
+
     cdef void init(self, const vector[shared_ptr[CField]]& fields):
         self.schema = new CSchema(fields)
         self.sp_schema.reset(self.schema)
@@ -327,7 +334,10 @@ cdef class Schema:
     property metadata:
 
         def __get__(self):
-            return box_metadata(self.schema.metadata().get())
+            self._check_null()
+            cdef shared_ptr[const CKeyValueMetadata] metadata = (
+                self.schema.metadata())
+            return box_metadata(metadata.get())
 
     def equals(self, other):
         """
@@ -377,6 +387,28 @@ cdef class Schema:
 
         return pyarrow_wrap_schema(new_schema)
 
+    def serialize(self, memory_pool=None):
+        """
+        Write Schema to Buffer as encapsulated IPC message
+
+        Parameters
+        ----------
+        memory_pool : MemoryPool, default None
+            Uses default memory pool if not specified
+
+        Returns
+        -------
+        serialized : Buffer
+        """
+        cdef:
+            shared_ptr[CBuffer] buffer
+            CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)
+
+        with nogil:
+            check_status(SerializeSchema(deref(self.schema),
+                                         pool, &buffer))
+        return pyarrow_wrap_buffer(buffer)
+
     def remove_metadata(self):
         """
         Create new schema without metadata, if any

From a3607d2a4683dd364f36d29cdaf4761a6000ea7d Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Fri, 25 Aug 2017 11:45:19 -0700
Subject: [PATCH 0954/1644] ARROW-1410: Remove MAP_POPULATE flag when mmapping
 files in Plasma store.

cc @pcmoritz @atumanov

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #992 from robertnishihara/removemappopulate and squashes the following commits:

8ed9612 [Robert Nishihara] Remove unnecessary ifdef.
7b75bd9 [Robert Nishihara] Remove MAP_POPULATE flag when mmapping files in Plasma store.
---
 cpp/src/plasma/malloc.cc | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
index 6b9bc62ab5a57..52d362013f1ae 100644
--- a/cpp/src/plasma/malloc.cc
+++ b/cpp/src/plasma/malloc.cc
@@ -134,14 +134,10 @@ void* fake_mmap(size_t size) {
 
   int fd = create_buffer(size);
   ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
-#ifdef __linux__
-  // MAP_POPULATE will pre-populate the page tables for this memory region
-  // which avoids work when accessing the pages later. Only supported on Linux.
-  void* pointer =
-      mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0);
-#else
+  // MAP_POPULATE can be used to pre-populate the page tables for this memory region
+  // which avoids work when accessing the pages later. However it causes long pauses
+  // when mmapping the files. Only supported on Linux.
   void* pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-#endif
   if (pointer == MAP_FAILED) {
     ARROW_LOG(ERROR) << "mmap failed with error: " << std::strerror(errno);
     if (errno == ENOMEM && plasma::plasma_config->hugepages_enabled) {

From ea503a5f020ae48528348bb8cd0eb0635d40c52b Mon Sep 17 00:00:00 2001
From: Antony Mayi <antonymayi@yahoo.com>
Date: Sat, 26 Aug 2017 16:01:15 -0400
Subject: [PATCH 0955/1644] ARROW-1379: [Java] adding maven-dependency-plugin
 and fixing all reported dependency errors

There is couple of dependency issues in the current maven config. This is then leaking into the integrating project which then needs to specify foreign dependencies just because arrow doesn't list them properly or is pulling unnecessary dependencies just because arrow lists them improperly.

* ```arrow-format```
```
[WARNING] Unused declared dependencies found:
[WARNING]    org.slf4j:slf4j-api:jar:1.7.25:compile
[WARNING]    com.vlkan:flatbuffers:jar:1.2.0-3f79e055:compile
[WARNING]    io.netty:netty-handler:jar:4.0.49.Final:compile
[WARNING]    com.google.guava:guava:jar:18.0:compile
```
* ```arrow-memory```
```
[WARNING] Used undeclared dependencies found:
[WARNING]    io.netty:netty-buffer:jar:4.0.49.Final:compile
[WARNING]    io.netty:netty-common:jar:4.0.49.Final:compile
[WARNING] Unused declared dependencies found:
[WARNING]    com.carrotsearch:hppc:jar:0.7.2:compile
[WARNING]    io.netty:netty-handler:jar:4.0.49.Final:compile
```
* ```arrow-tools```
```
[WARNING] Used undeclared dependencies found:
[WARNING]    com.fasterxml.jackson.core:jackson-databind:jar:2.7.9:compile
[WARNING]    com.fasterxml.jackson.core:jackson-core:jar:2.7.9:compile
[WARNING] Unused declared dependencies found:
[WARNING]    org.apache.commons:commons-lang3:jar:3.6:compile
[WARNING]    org.apache.arrow:arrow-format:jar:0.7.0-SNAPSHOT:compile
[WARNING]    io.netty:netty-handler:jar:4.0.49.Final:compile
```
* ```arrow-vector```
```
[WARNING] Used undeclared dependencies found:
[WARNING]    com.google.code.findbugs:jsr305:jar:3.0.2:compile
[WARNING]    com.vlkan:flatbuffers:jar:1.2.0-3f79e055:compile
[WARNING]    io.netty:netty-common:jar:4.0.49.Final:compile
[WARNING]    io.netty:netty-buffer:jar:4.0.49.Final:compile
[WARNING]    com.fasterxml.jackson.core:jackson-core:jar:2.7.9:compile
[WARNING] Unused declared dependencies found:
[WARNING]    org.apache.commons:commons-lang3:jar:3.6:compile
[WARNING]    io.netty:netty-handler:jar:4.0.49.Final:compile
```

I am proposing this PR to:
1. Add maven-dependency-plugin to enforce all dependencies are always listed corrctly
2. Fixing all the current dependency issues

Author: Antony Mayi <antonymayi@yahoo.com>
Author: Stepan Kadlec <stepan.kadlec@oracle.com>

Closes #978 from antonymayi/master and squashes the following commits:

d7f081ea [Antony Mayi] moving `copy-flatc` to initialize phase and `analyze` execution to parent pom
ec727175 [Antony Mayi] removing unused apache.commons.lang3, fixing pom
8cbfe5f4 [Antony Mayi] maven-dependency-plugin: ignoring dependencies of generated sources in arrow-vector
dc833bb3 [Stepan Kadlec] adding maven-dependency-plugin and fixing all reported dependency errors
---
 java/format/pom.xml                           |   7 +-
 java/memory/pom.xml                           |  22 +-
 java/pom.xml                                  | 191 ++++++++++++------
 java/tools/pom.xml                            |  24 ++-
 java/vector/pom.xml                           |  35 +++-
 .../src/main/codegen/includes/vv_imports.ftl  |   2 -
 6 files changed, 183 insertions(+), 98 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 0e43d5e946cd7..647b716039e7b 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -23,12 +23,9 @@
 <name>Arrow Format</name>
 
   <properties>
-    <fbs.version>1.2.0-3f79e055</fbs.version>
     <flatc.download.skip>false</flatc.download.skip>
     <flatc.executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</flatc.executable>
     <flatc.generated.files>${project.build.directory}/generated-sources/flatc</flatc.generated.files>
-    <maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
-    <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
     <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
   </properties>
 
@@ -36,7 +33,6 @@
     <dependency>
       <groupId>com.vlkan</groupId>
       <artifactId>flatbuffers</artifactId>
-      <version>${fbs.version}</version>
     </dependency>
   </dependencies>
 
@@ -54,11 +50,10 @@
     <plugin> <!-- download the flatbuffer compiler -->
       <groupId>org.apache.maven.plugins</groupId>
       <artifactId>maven-dependency-plugin</artifactId>
-      <version>${maven-dependency-plugin.version}</version>
       <executions>
         <execution>
           <id>copy-flatc</id>
-          <phase>generate-sources</phase>
+          <phase>initialize</phase>
           <goals>
             <goal>copy</goal>
           </goals>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 765013bee1f66..1c4eef71fc009 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -23,20 +23,26 @@
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
-      <version>3.0.2</version>
     </dependency>
-
     <dependency>
-      <groupId>com.carrotsearch</groupId>
-      <artifactId>hppc</artifactId>
-      <version>0.7.2</version>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-buffer</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
     </dependency>
   </dependencies>
 
-
   <build>
   </build>
 
-
-
 </project>
diff --git a/java/pom.xml b/java/pom.xml
index cfcfcd448f0da..c4c301d1a419e 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -32,9 +32,11 @@
     <dep.junit.version>4.11</dep.junit.version>
     <dep.slf4j.version>1.7.25</dep.slf4j.version>
     <dep.guava.version>18.0</dep.guava.version>
+    <dep.netty.version>4.0.49.Final</dep.netty.version>
+    <dep.jackson.version>2.7.9</dep.jackson.version>
+    <dep.hadoop.version>2.7.1</dep.hadoop.version>
+    <fbs.version>1.2.0-3f79e055</fbs.version>
     <forkCount>2</forkCount>
-    <jackson.version>2.7.9</jackson.version>
-    <hadoop.version>2.7.1</hadoop.version>
     <checkstyle.failOnViolation>false</checkstyle.failOnViolation>
   </properties>
 
@@ -271,55 +273,75 @@
         </configuration>
       </plugin>
 
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-checkstyle-plugin</artifactId>
-          <version>2.17</version>
-          <dependencies>
-            <dependency>
-              <groupId>com.puppycrawl.tools</groupId>
-              <artifactId>checkstyle</artifactId>
-              <version>6.19</version>
-            </dependency>
-            <dependency>
-              <groupId>com.google.guava</groupId>
-              <artifactId>guava</artifactId>
-              <version>${dep.guava.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.slf4j</groupId>
-                <artifactId>jcl-over-slf4j</artifactId>
-                <version>1.7.5</version>
-            </dependency>
-          </dependencies>
-          <executions>
-            <execution>
-              <id>validate</id>
-              <phase>validate</phase>
-              <goals>
-                <goal>check</goal>
-              </goals>
-            </execution>
-          </executions>
-          <configuration>
-            <configLocation>google_checks.xml</configLocation>
-            <encoding>UTF-8</encoding>
-            <consoleOutput>true</consoleOutput>
-            <failsOnError>${checkstyle.failOnViolation}</failsOnError>
-            <failOnViolation>${checkstyle.failOnViolation}</failOnViolation>
-            <violationSeverity>warning</violationSeverity>
-            <format>xml</format>
-            <format>html</format>
-            <outputFile>${project.build.directory}/test/checkstyle-errors.xml</outputFile>
-            <linkXRef>false</linkXRef>
-          </configuration>
-        </plugin>
-
-
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <version>2.17</version>
+        <dependencies>
+          <dependency>
+            <groupId>com.puppycrawl.tools</groupId>
+            <artifactId>checkstyle</artifactId>
+            <version>6.19</version>
+          </dependency>
+          <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${dep.guava.version}</version>
+          </dependency>
+          <dependency>
+              <groupId>org.slf4j</groupId>
+              <artifactId>jcl-over-slf4j</artifactId>
+              <version>1.7.5</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <id>validate</id>
+            <phase>validate</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <configLocation>google_checks.xml</configLocation>
+          <encoding>UTF-8</encoding>
+          <consoleOutput>true</consoleOutput>
+          <failsOnError>${checkstyle.failOnViolation}</failsOnError>
+          <failOnViolation>${checkstyle.failOnViolation}</failOnViolation>
+          <violationSeverity>warning</violationSeverity>
+          <format>xml</format>
+          <format>html</format>
+          <outputFile>${project.build.directory}/test/checkstyle-errors.xml</outputFile>
+          <linkXRef>false</linkXRef>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>analyze</id>
+            <phase>verify</phase>
+            <goals>
+              <goal>analyze-only</goal>
+            </goals>
+            <configuration>
+              <ignoreNonCompile>true</ignoreNonCompile>
+              <failOnWarning>true</failOnWarning>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
-    <pluginManagement>
 
+    <pluginManagement>
       <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-dependency-plugin</artifactId>
+          <version>3.0.1</version>
+        </plugin>
         <plugin>
           <groupId>org.apache.rat</groupId>
           <artifactId>apache-rat-plugin</artifactId>
@@ -449,26 +471,63 @@
       </plugins>
     </pluginManagement>
   </build>
-  <dependencies>
-
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-handler</artifactId>
-      <version>4.0.49.Final</version>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <version>${dep.guava.version}</version>
-    </dependency>
 
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-      <version>${dep.slf4j.version}</version>
-    </dependency>
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>com.vlkan</groupId>
+        <artifactId>flatbuffers</artifactId>
+        <version>${fbs.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava</artifactId>
+        <version>${dep.guava.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-handler</artifactId>
+        <version>${dep.netty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-buffer</artifactId>
+        <version>${dep.netty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-common</artifactId>
+        <version>${dep.netty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-core</artifactId>
+        <version>${dep.jackson.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-annotations</artifactId>
+        <version>${dep.jackson.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-databind</artifactId>
+        <version>${dep.jackson.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>com.google.code.findbugs</groupId>
+        <artifactId>jsr305</artifactId>
+        <version>3.0.2</version>
+      </dependency>
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-api</artifactId>
+        <version>${dep.slf4j.version}</version>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
 
+  <dependencies>
     <!-- Test Dependencies -->
     <dependency>
       <groupId>org.slf4j</groupId>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index e2e3f8129c707..48b507afdd1ca 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -20,11 +20,6 @@
     <name>Arrow Tools</name>
 
     <dependencies>
-        <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>arrow-format</artifactId>
-            <version>${project.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-memory</artifactId>
@@ -36,9 +31,8 @@
             <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-            <version>3.6</version>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
         </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
@@ -49,7 +43,19 @@
           <groupId>ch.qos.logback</groupId>
           <artifactId>logback-classic</artifactId>
           <version>1.2.3</version>
-          <scope>run</scope>
+          <scope>runtime</scope>
+        </dependency>
+        <dependency>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
         </dependency>
     </dependencies>
 
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index a89742ec142bd..bf33823fef52f 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -36,31 +36,52 @@
       <artifactId>joda-time</artifactId>
       <version>2.9.9</version>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
-      <version>${jackson.version}</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
-      <version>${jackson.version}</version>
     </dependency>
     <dependency>
       <groupId>com.carrotsearch</groupId>
       <artifactId>hppc</artifactId>
       <version>0.7.2</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-      <version>3.6</version>
-    </dependency>
     <dependency>
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
       <version>1.10</version>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-buffer</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.vlkan</groupId>
+      <artifactId>flatbuffers</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
   </dependencies>
 
     <pluginRepositories>
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
index e723e7d7ea3d0..a55304d733539 100644
--- a/java/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -22,8 +22,6 @@ import com.google.flatbuffers.FlatBufferBuilder;
 import com.google.common.base.Preconditions;
 import io.netty.buffer.*;
 
-import org.apache.commons.lang3.ArrayUtils;
-
 import org.apache.arrow.memory.*;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.*;

From b36aab5abfa7b398c9b307930a9f1ba4c9dc08c8 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sat, 26 Aug 2017 16:26:17 -0400
Subject: [PATCH 0956/1644] ARROW-1411: [Python] Booleans in Float Columns
 cause Segfault

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #993 from cpcloud/ARROW-1411 and squashes the following commits:

741269f0 [Phillip Cloud] ARROW-1411: [Python] Booleans in Float Columns cause Segfault
---
 cpp/src/arrow/python/pandas_to_arrow.cc     | 12 ++++++------
 python/pyarrow/tests/test_convert_pandas.py |  4 ++++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index b6cc16b4179b3..408aef3adc40c 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -552,7 +552,7 @@ Status PandasConverter::ConvertDates() {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to Date: ";
       RETURN_NOT_OK(InvalidConversion(obj, "datetime.date", &ss));
       return Status::Invalid(ss.str());
     }
@@ -608,7 +608,7 @@ Status PandasConverter::ConvertDecimals() {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to Decimal: ";
       RETURN_NOT_OK(InvalidConversion(object, "decimal.Decimal", &ss));
       return Status::Invalid(ss.str());
     }
@@ -636,7 +636,7 @@ Status PandasConverter::ConvertTimes() {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to Time: ";
       RETURN_NOT_OK(InvalidConversion(obj, "datetime.time", &ss));
       return Status::Invalid(ss.str());
     }
@@ -704,7 +704,7 @@ Status PandasConverter::ConvertObjectFloats() {
       RETURN_NOT_OK(builder.Append(val));
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to Double: ";
       RETURN_NOT_OK(InvalidConversion(obj, "float", &ss));
       return Status::Invalid(ss.str());
     }
@@ -739,7 +739,7 @@ Status PandasConverter::ConvertObjectIntegers() {
       RETURN_NOT_OK(builder.Append(val));
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to Int64: ";
       RETURN_NOT_OK(InvalidConversion(obj, "integer", &ss));
       return Status::Invalid(ss.str());
     }
@@ -802,7 +802,7 @@ Status PandasConverter::ConvertBooleans() {
       BitUtil::SetBit(null_bitmap_data_, i);
     } else {
       std::stringstream ss;
-      ss << "Error converting from Python objects to " << type_->ToString() << ": ";
+      ss << "Error converting from Python objects to Boolean: ";
       RETURN_NOT_OK(InvalidConversion(obj, "bool", &ss));
       return Status::Invalid(ss.str());
     }
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index c2631842278e0..6e5e8e88ced3d 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -619,6 +619,10 @@ def test_mixed_types_fails(self):
         with self.assertRaises(pa.ArrowException):
             pa.Table.from_pandas(data)
 
+        data = pd.DataFrame({'a': [1, True]})
+        with self.assertRaises(pa.ArrowException):
+            pa.Table.from_pandas(data)
+
     def test_strided_data_import(self):
         cases = []
 

From 26d4e3f593d2007a9cc462ff13375e85b49c34f4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 27 Aug 2017 12:08:03 +0200
Subject: [PATCH 0957/1644] ARROW-1399: [C++] Add CUDA build version defines in
 public headers

When configured, this looks like:

```
#define ARROW_CUDA_ABI_VERSION_MAJOR 8
#define ARROW_CUDA_ABI_VERSION_MINOR 0
```

I'm not sure how to use this yet. It would be nice if we could work out how to enable thirdparty users to detect incompatibility with their nvcc at compiler time

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #990 from wesm/ARROW-1399 and squashes the following commits:

1ad6966 [Wes McKinney] Add CUDA build version defines in public headers
---
 cpp/src/arrow/gpu/CMakeLists.txt    |  9 +++++++++
 cpp/src/arrow/gpu/cuda_api.h        | 24 ++++++++++++++++++++++++
 cpp/src/arrow/gpu/cuda_version.h.in | 25 +++++++++++++++++++++++++
 3 files changed, 58 insertions(+)
 create mode 100644 cpp/src/arrow/gpu/cuda_api.h
 create mode 100644 cpp/src/arrow/gpu/cuda_version.h.in

diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt
index 9fc9efa594712..cab085302c6c4 100644
--- a/cpp/src/arrow/gpu/CMakeLists.txt
+++ b/cpp/src/arrow/gpu/CMakeLists.txt
@@ -114,6 +114,15 @@ install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-gpu.pc"
   DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
 
+# CUDA build version
+configure_file(cuda_version.h.in
+  "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h"
+  @ONLY)
+
+install(FILES
+  "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h"
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
+
 set(ARROW_GPU_TEST_LINK_LIBS
   arrow_gpu_shared
   ${ARROW_TEST_LINK_LIBS})
diff --git a/cpp/src/arrow/gpu/cuda_api.h b/cpp/src/arrow/gpu/cuda_api.h
new file mode 100644
index 0000000000000..a70e0af92aff4
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_api.h
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_GPU_CUDA_API_H
+#define ARROW_GPU_CUDA_API_H
+
+#include "arrow/gpu/cuda_memory.h"
+#include "arrow/gpu/cuda_version.h"
+
+#endif // ARROW_GPU_CUDA_API_H
diff --git a/cpp/src/arrow/gpu/cuda_version.h.in b/cpp/src/arrow/gpu/cuda_version.h.in
new file mode 100644
index 0000000000000..bc687685d0a7d
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_version.h.in
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_GPU_CUDA_VERSION_H
+#define ARROW_GPU_CUDA_VERSION_H
+
+// Set the CUDA version used to build the library
+#define ARROW_CUDA_ABI_VERSION_MAJOR @CUDA_VERSION_MAJOR@
+#define ARROW_CUDA_ABI_VERSION_MINOR @CUDA_VERSION_MINOR@
+
+#endif // ARROW_GPU_CUDA_VERSION_H

From 6cd2a17160411537a26997271f83321dae32845c Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 28 Aug 2017 09:32:25 -0400
Subject: [PATCH 0958/1644] ARROW-1419: [GLib] Suppress sign-conversion
 warnings

Apache Arrow C++ uses int as result type for expression that uses
size_t. It causes sign-conversion warning but the coding style is
expected.

Example:

    .../arrow/buffer.h:296:41: warning:
          implicit conversion changes signedness: 'unsigned long' to 'int64_t'
          (aka 'long') [-Wsign-conversion]
      int64_t length() const { return size_ / sizeof(T); }
                               ~~~~~~ ~~~~~~^~~~~~~~~~~

Author: Kouhei Sutou <kou@clear-code.com>

Closes #999 from kou/glib-suppress-warning-on-clang and squashes the following commits:

397490e9 [Kouhei Sutou] [GLib] Suppress sign-conversion warnings
---
 c_glib/configure.ac | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index 375f76efcdd51..103f11b4bbedd 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -36,8 +36,8 @@ AC_PROG_CXX
 AX_CXX_COMPILE_STDCXX_11([ext], [mandatory])
 LT_INIT
 
-GARROW_CFLAGS="-Wall -Wconversion"
-GARROW_CXXFLAGS="-Wall -Wconversion"
+GARROW_CFLAGS="-Wall"
+GARROW_CXXFLAGS="-Wall"
 AC_ARG_ENABLE(debug,
   [AS_HELP_STRING([--enable-debug],
                   [Use debug flags (default=no)])],

From f250d46578fa02fbfe432f5a4e44b66eed3b4b2b Mon Sep 17 00:00:00 2001
From: Fritz Obermeyer <fritz.obermeyer@gmail.com>
Date: Mon, 28 Aug 2017 09:36:40 -0400
Subject: [PATCH 0959/1644] ARROW-1416: Clarify memory layout documentation

This PR slightly reduces ambiguity in the array example for null bitmaps. The original example was left/right symmetric; this PR changes the example to break that symmetry. Asymmetry is important since readers who skip the byte endianness section could have interpreted the bitmap buffer in two distinct ways: left-to-right with an offset of 3 (wrong), or right-to-left with zero offset (correct).

Author: Fritz Obermeyer <fritz.obermeyer@gmail.com>

Closes #998 from fritzo/patch-1 and squashes the following commits:

af3dcbd0 [Fritz Obermeyer] Clarify memory layout documentation
---
 format/Layout.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/format/Layout.md b/format/Layout.md
index 334251103d732..3c21dbc0dc2b1 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -206,7 +206,7 @@ does not need to be adjacent in memory to the values buffer.
 ### Example Layout: Int32 Array
 For example a primitive array of int32s:
 
-[1, 2, null, 4, 8]
+[1, null, 2, 4, 8]
 
 Would look like:
 
@@ -216,13 +216,13 @@ Would look like:
 
   |Byte 0 (validity bitmap) | Bytes 1-63            |
   |-------------------------|-----------------------|
-  | 00011011                | 0 (padding)           |
+  | 00011101                | 0 (padding)           |
 
 * Value Buffer:
 
   |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-63 |
   |------------|-------------|-------------|-------------|-------------|-------------|
-  | 1          | 2           | unspecified | 4           | 8           | unspecified |
+  | 1          | unspecified | 2           | 4           | 8           | unspecified |
 ```
 
 ### Example Layout: Non-null int32 Array

From 32e266814288f0de889b3e2e5d3b4a337161f47f Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 28 Aug 2017 09:43:23 -0400
Subject: [PATCH 0960/1644] ARROW-1414: [GLib] Cast after status check

Author: Kouhei Sutou <kou@clear-code.com>

Closes #996 from kou/glib-cast-after-status-check and squashes the following commits:

02b59db3 [Kouhei Sutou] [GLib] Cast after status check
---
 c_glib/arrow-glib/reader.cpp | 2 +-
 c_glib/arrow-glib/writer.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index a21b375616752..ba835fd33a6bd 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -243,8 +243,8 @@ garrow_record_batch_stream_reader_new(GArrowInputStream *stream,
   auto arrow_input_stream = garrow_input_stream_get_raw(stream);
   std::shared_ptr<BaseType> arrow_reader;
   auto status = ReaderType::Open(arrow_input_stream, &arrow_reader);
-  auto subtype = std::dynamic_pointer_cast<ReaderType>(arrow_reader);
   if (garrow_error_check(error, status, "[record-batch-stream-reader][open]")) {
+    auto subtype = std::dynamic_pointer_cast<ReaderType>(arrow_reader);
     return garrow_record_batch_stream_reader_new_raw(&subtype);
   } else {
     return NULL;
diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp
index c08db19987ea1..7d3b59457fd33 100644
--- a/c_glib/arrow-glib/writer.cpp
+++ b/c_glib/arrow-glib/writer.cpp
@@ -223,8 +223,8 @@ garrow_record_batch_stream_writer_new(GArrowOutputStream *sink,
   auto status = WriterType::Open(arrow_sink,
                                  garrow_schema_get_raw(schema),
                                  &arrow_writer);
-  auto subtype = std::dynamic_pointer_cast<WriterType>(arrow_writer);
   if (garrow_error_check(error, status, "[record-batch-stream-writer][open]")) {
+    auto subtype = std::dynamic_pointer_cast<WriterType>(arrow_writer);
     return garrow_record_batch_stream_writer_new_raw(&subtype);
   } else {
     return NULL;
@@ -270,8 +270,8 @@ garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
   auto status = WriterType::Open(arrow_sink.get(),
                                  garrow_schema_get_raw(schema),
                                  &arrow_writer);
-  auto subtype = std::dynamic_pointer_cast<WriterType>(arrow_writer);
   if (garrow_error_check(error, status, "[record-batch-file-writer][open]")) {
+    auto subtype = std::dynamic_pointer_cast<WriterType>(arrow_writer);
     return garrow_record_batch_file_writer_new_raw(&subtype);
   } else {
     return NULL;

From c027a03b31085be6811714e16e48b8c59cf75aa5 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 28 Aug 2017 09:51:49 -0400
Subject: [PATCH 0961/1644] ARROW-1412: [Plasma] Add higher level API for
 putting and getting Python objects

Note that this PR breaks the PlasmaClient API (which is still unstable at this point, so this is acceptable). It renames PlasmaClient.get to PlasmaClient.get_buffers and introduces two new functions, PlasmaClient.put and PlasmaClient.get which can put Python objects into the object store and provide access to their content. The old get was renamed to get_buffers because most users will want to use the new get method and therefore it should have the more concise name.

There is some freedom in designing the API; I tried to make it so there is a unified API between getting one and multiple objects (the latter is supported to limit the number of IPC roundtrips with the plasma store when we get many small objects). I also introduced a special object that is returned if one of the objects was not available within the timeout. We could use "None" here, but then it would be hard to distinguish between getting a "None" object and a timeout.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #995 from pcmoritz/plasma-putget and squashes the following commits:

bd24e014 [Philipp Moritz] add documentation
e60ea73e [Philipp Moritz] get_buffer -> get_buffers and update example
8c369037 [Philipp Moritz] support full API
5921148e [Philipp Moritz] move put and get into PlasmaClient
cf4bf24f [Philipp Moritz] add type information
0049c674 [Philipp Moritz] fix flake8 linting
44c3b3dc [Philipp Moritz] fixes
20b119e3 [Philipp Moritz] make it possible to get single objects
36f67d6d [Philipp Moritz] implement ObjectID.from_random
c0449543 [Philipp Moritz] add documentation
eb9694a0 [Philipp Moritz] implement timeouts
3518c71a [Philipp Moritz] fix
e1924a45 [Philipp Moritz] add put and get
44ada470 [Philipp Moritz] export symbols
---
 python/doc/source/plasma.rst               | 65 ++++++++++++----
 python/examples/plasma/sorting/sort_df.py  |  2 +-
 python/pyarrow/__init__.py                 |  6 +-
 python/pyarrow/plasma.pyx                  | 86 +++++++++++++++++++++-
 python/pyarrow/tests/test_plasma.py        | 33 ++++++---
 python/pyarrow/tests/test_serialization.py |  1 +
 6 files changed, 163 insertions(+), 30 deletions(-)

diff --git a/python/doc/source/plasma.rst b/python/doc/source/plasma.rst
index e4665d187e1f5..9a5a74bfaf14f 100644
--- a/python/doc/source/plasma.rst
+++ b/python/doc/source/plasma.rst
@@ -98,9 +98,46 @@ follows:
   def random_object_id():
     return plasma.ObjectID(np.random.bytes(20))
 
+Putting and Getting Python Objects
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Creating an Object
-^^^^^^^^^^^^^^^^^^
+Plasma supports two APIs for creating and accessing objects: A high level
+API that allows storing and retrieving Python objects and a low level
+API that allows creating, writing and sealing buffers and operating on
+the binary data directly. In this section we describe the high level API.
+
+This is how you can put and get a Python object:
+
+.. code-block:: python
+
+    # Create a python object.
+    object_id = client.put("hello, world")
+
+    # Get the object.
+    client.get(object_id)
+
+This works with all Python objects supported by the Arrow Python object
+serialization.
+
+You can also get multiple objects at the same time (which can be more
+efficient since it avoids IPC round trips):
+
+.. code-block:: python
+
+    # Create multiple python objects.
+    object_id1 = client.put(1)
+    object_id2 = client.put(2)
+    object_id3 = client.put(3)
+
+    # Get the objects.
+    client.get([object_id1, object_id2, object_id3])
+
+Furthermore, it is possible to provide a timeout for the get call. If the
+object is not available within the timeout, the special object
+`pyarrow.ObjectNotAvailable` will be returned.
+
+Creating an Object Buffer
+^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Objects are created in Plasma in two stages. First, they are **created**, which
 allocates a buffer for the object. At this point, the client can write to the
@@ -111,7 +148,7 @@ give the object's maximum size in bytes.
 
 .. code-block:: python
 
-  # Create an object.
+  # Create an object buffer.
   object_id = plasma.ObjectID(20 * b"a")
   object_size = 1000
   buffer = memoryview(client.create(object_id, object_size))
@@ -129,11 +166,11 @@ immutable, and making it available to other Plasma clients.
   client.seal(object_id)
 
 
-Getting an Object
-^^^^^^^^^^^^^^^^^
+Getting an Object Buffer
+^^^^^^^^^^^^^^^^^^^^^^^^
 
 After an object has been sealed, any client who knows the object ID can get
-the object.
+the object buffer.
 
 .. code-block:: python
 
@@ -143,11 +180,11 @@ the object.
 
   # Get the object in the second client. This blocks until the object has been sealed.
   object_id2 = plasma.ObjectID(20 * b"a")
-  [buffer2] = client2.get([object_id])
+  [buffer2] = client2.get_buffers([object_id])
 
-If the object has not been sealed yet, then the call to client.get will block
-until the object has been sealed by the client constructing the object. Using
-the ``timeout_ms`` argument to get, you can specify a timeout for this (in
+If the object has not been sealed yet, then the call to client.get_buffers will
+block until the object has been sealed by the client constructing the object.
+Using the ``timeout_ms`` argument to get, you can specify a timeout for this (in
 milliseconds). After the timeout, the interpreter will yield control back.
 
 .. code-block:: shell
@@ -223,7 +260,7 @@ To read the object, first retrieve it as a ``PlasmaBuffer`` using its object ID.
 .. code-block:: python
 
   # Get the arrow object by ObjectID.
-  [buf2] = client.get([object_id])
+  [buf2] = client.get_buffers([object_id])
 
 To convert the ``PlasmaBuffer`` back into an Arrow ``Tensor``, first create a
 pyarrow ``BufferReader`` object from it. You can then pass the ``BufferReader``
@@ -310,13 +347,13 @@ Since we store the Pandas DataFrame as a PyArrow ``RecordBatch`` object,
 to get the object back from the Plasma store, we follow similar steps
 to those specified in `Getting Arrow Objects from Plasma`_.
 
-We first have to convert the ``PlasmaBuffer`` returned from ``client.get``
-into an Arrow ``BufferReader`` object.
+We first have to convert the ``PlasmaBuffer`` returned from
+``client.get_buffers`` into an Arrow ``BufferReader`` object.
 
 .. code-block:: python
 
   # Fetch the Plasma object
-  [data] = client.get([object_id])  # Get PlasmaBuffer from ObjectID
+  [data] = client.get_buffers([object_id])  # Get PlasmaBuffer from ObjectID
   buffer = pa.BufferReader(data)
 
 From the ``BufferReader``, we can create a specific ``RecordBatchStreamReader``
diff --git a/python/examples/plasma/sorting/sort_df.py b/python/examples/plasma/sorting/sort_df.py
index 03cfd13c6d76f..0181ed729be29 100644
--- a/python/examples/plasma/sorting/sort_df.py
+++ b/python/examples/plasma/sorting/sort_df.py
@@ -81,7 +81,7 @@ def put_df(df):
 
 def get_dfs(object_ids):
     """Retrieve dataframes from the object store given their object IDs."""
-    buffers = client.get(object_ids)
+    buffers = client.get_buffers(object_ids)
     return [pa.RecordBatchStreamReader(buf).read_next_batch().to_pandas()
             for buf in buffers]
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 68ae017ce71ad..985e262bced8f 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -85,12 +85,14 @@
                          ArrowIOError,
                          ArrowMemoryError,
                          ArrowNotImplementedError,
-                         ArrowTypeError)
+                         ArrowTypeError,
+                         PlasmaObjectExists)
 
 # Serialization
 from pyarrow.lib import (deserialize_from, deserialize,
                          serialize, serialize_to, read_serialized,
-                         SerializedPyObject)
+                         SerializedPyObject,
+                         SerializationException, DeserializationException)
 
 from pyarrow.filesystem import FileSystem, LocalFileSystem
 
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index befa283d85b54..aebef1b881297 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -26,6 +26,9 @@ from libcpp.vector cimport vector as c_vector
 from libc.stdint cimport int64_t, uint8_t, uintptr_t
 from cpython.pycapsule cimport *
 
+import collections
+import pyarrow
+
 from pyarrow.lib cimport Buffer, NativeFile, check_status
 from pyarrow.includes.libarrow cimport (CMutableBuffer, CBuffer,
                                         CFixedSizeBufferWriter, CStatus)
@@ -41,6 +44,9 @@ cdef extern from "plasma/common.h" nogil:
         @staticmethod
         CUniqueID from_binary(const c_string& binary)
 
+        @staticmethod
+        CUniqueID from_random()
+
         c_bool operator==(const CUniqueID& rhs) const
 
         c_string hex() const
@@ -157,6 +163,18 @@ cdef class ObjectID:
         """
         return self.data.binary()
 
+    @staticmethod
+    def from_random():
+        cdef CUniqueID data = CUniqueID.from_random()
+        return ObjectID(data.binary())
+
+
+cdef class ObjectNotAvailable:
+    """
+    Placeholder for an object that was not available within the given timeout.
+    """
+    pass
+
 
 cdef class PlasmaBuffer(Buffer):
     """
@@ -285,7 +303,7 @@ cdef class PlasmaClient:
                                                   metadata.size(), &data))
         return self._make_mutable_plasma_buffer(object_id, data, data_size)
 
-    def get(self, object_ids, timeout_ms=-1):
+    def get_buffers(self, object_ids, timeout_ms=-1):
         """
         Returns data buffer from the PlasmaStore based on object ID.
 
@@ -296,7 +314,7 @@ cdef class PlasmaClient:
         ----------
         object_ids : list
             A list of ObjectIDs used to identify some objects.
-        timeout_ms :int
+        timeout_ms : int
             The number of milliseconds that the get call should block before
             timing out and returning. Pass -1 if the call should block and 0
             if the call should return immediately.
@@ -352,6 +370,68 @@ cdef class PlasmaClient:
                                          object_buffers[i].metadata_size))
         return result
 
+    def put(self, object value, ObjectID object_id=None):
+        """
+        Store a Python value into the object store.
+
+        Parameters
+        ----------
+        value : object
+            A Python object to store.
+        object_id : ObjectID, default None
+            If this is provided, the specified object ID will be used to refer
+            to the object.
+
+        Returns
+        -------
+        The object ID associated to the Python object.
+        """
+        cdef ObjectID target_id = object_id if object_id else ObjectID.from_random()
+        # TODO(pcm): Make serialization code support non-sequences and
+        # get rid of packing the value into a list here (and unpacking in get)
+        serialized = pyarrow.serialize([value])
+        buffer = self.create(target_id, serialized.total_bytes)
+        stream = pyarrow.FixedSizeBufferOutputStream(buffer)
+        stream.set_memcopy_threads(4)
+        serialized.write_to(stream)
+        self.seal(target_id)
+        return target_id
+
+    def get(self, object_ids, int timeout_ms=-1):
+        """
+        Get one or more Python values from the object store.
+
+        Parameters
+        ----------
+        object_ids : list or ObjectID
+            Object ID or list of object IDs associated to the values we get from
+            the store.
+        timeout_ms : int, default -1
+            The number of milliseconds that the get call should block before
+            timing out and returning. Pass -1 if the call should block and 0
+            if the call should return immediately.
+
+        Returns
+        -------
+        list or object
+            Python value or list of Python values for the data associated with
+            the object_ids and ObjectNotAvailable if the object was not available.
+        """
+        if isinstance(object_ids, collections.Sequence):
+            results = []
+            buffers = self.get_buffers(object_ids, timeout_ms)
+            for i in range(len(object_ids)):
+                # buffers[i] is None if this object was not available within the
+                # timeout
+                if buffers[i]:
+                    value, = pyarrow.deserialize(buffers[i])
+                    results.append(value)
+                else:
+                    results.append(ObjectNotAvailable)
+            return results
+        else:
+            return self.get([object_ids], timeout_ms)[0]
+
     def seal(self, ObjectID object_id):
         """
         Seal the buffer in the PlasmaStore for a particular object ID.
@@ -576,7 +656,7 @@ def connect(store_socket_name, manager_socket_name, int release_delay,
         The maximum number of objects that the client will keep and
         delay releasing (for caching reasons).
     num_retries : int, default -1
-        Number of times tor ty to connect to plasma store. Default value of -1
+        Number of times to try to connect to plasma store. Default value of -1
         uses the default (50)
     """
     cdef PlasmaClient result = PlasmaClient()
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index 04162bbbbade7..d729c1ef2d205 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -82,8 +82,8 @@ def create_object(client, data_size, metadata_size, seal=True):
 def assert_get_object_equal(unit_test, client1, client2, object_id,
                             memory_buffer=None, metadata=None):
     import pyarrow.plasma as plasma
-    client1_buff = client1.get([object_id])[0]
-    client2_buff = client2.get([object_id])[0]
+    client1_buff = client1.get_buffers([object_id])[0]
+    client2_buff = client2.get_buffers([object_id])[0]
     client1_metadata = client1.get_metadata([object_id])[0]
     client2_metadata = client2.get_metadata([object_id])[0]
     assert len(client1_buff) == len(client2_buff)
@@ -187,7 +187,7 @@ def test_create(self):
         # Seal the object.
         self.plasma_client.seal(object_id)
         # Get the object.
-        memory_buffer = np.frombuffer(self.plasma_client.get([object_id])[0],
+        memory_buffer = np.frombuffer(self.plasma_client.get_buffers([object_id])[0],
                                       dtype="uint8")
         for i in range(length):
             assert memory_buffer[i] == i % 256
@@ -209,7 +209,7 @@ def test_create_with_metadata(self):
             self.plasma_client.seal(object_id)
             # Get the object.
             memory_buffer = np.frombuffer(
-                self.plasma_client.get([object_id])[0], dtype="uint8")
+                self.plasma_client.get_buffers([object_id])[0], dtype="uint8")
             for i in range(length):
                 assert memory_buffer[i] == i % 256
             # Get the metadata.
@@ -241,7 +241,7 @@ def test_get(self):
         # Test timing out of get with various timeouts.
         for timeout in [0, 10, 100, 1000]:
             object_ids = [random_object_id() for _ in range(num_object_ids)]
-            results = self.plasma_client.get(object_ids, timeout_ms=timeout)
+            results = self.plasma_client.get_buffers(object_ids, timeout_ms=timeout)
             assert results == num_object_ids * [None]
 
         data_buffers = []
@@ -256,8 +256,8 @@ def test_get(self):
         # Test timing out from some but not all get calls with various
         # timeouts.
         for timeout in [0, 10, 100, 1000]:
-            data_results = self.plasma_client.get(object_ids,
-                                                  timeout_ms=timeout)
+            data_results = self.plasma_client.get_buffers(object_ids,
+                                                         timeout_ms=timeout)
             # metadata_results = self.plasma_client.get_metadata(
             #     object_ids, timeout_ms=timeout)
             for i in range(num_object_ids):
@@ -273,6 +273,19 @@ def test_get(self):
                 else:
                     assert results[i] is None
 
+    def test_put_and_get(self):
+        for value in [["hello", "world", 3, 1.0], None, "hello"]:
+             object_id = self.plasma_client.put(value)
+             [result] = self.plasma_client.get([object_id])
+             assert result == value
+
+             result = self.plasma_client.get(object_id)
+             assert result == value
+
+             object_id = pa.plasma.ObjectID.from_random()
+             [result] = self.plasma_client.get([object_id], timeout_ms=0)
+             assert result == pa.plasma.ObjectNotAvailable
+
     def test_store_arrow_objects(self):
         data = np.random.randn(10, 4)
         # Write an arrow object.
@@ -284,7 +297,7 @@ def test_store_arrow_objects(self):
         pa.write_tensor(tensor, stream)
         self.plasma_client.seal(object_id)
         # Read the arrow object.
-        [tensor] = self.plasma_client.get([object_id])
+        [tensor] = self.plasma_client.get_buffers([object_id])
         reader = pa.BufferReader(tensor)
         array = pa.read_tensor(reader).to_numpy()
         # Assert that they are equal.
@@ -313,7 +326,7 @@ def test_store_pandas_dataframe(self):
         self.plasma_client.seal(object_id)
 
         # Read the DataFrame.
-        [data] = self.plasma_client.get([object_id])
+        [data] = self.plasma_client.get_buffers([object_id])
         reader = pa.RecordBatchStreamReader(pa.BufferReader(data))
         result = reader.get_next_batch().to_pandas()
 
@@ -551,7 +564,7 @@ def test_illegal_functionality(self):
         # with pytest.raises(Exception):
         # illegal_assignment()
         # Get the object.
-        memory_buffer = self.plasma_client.get([object_id])[0]
+        memory_buffer = self.plasma_client.get_buffers([object_id])[0]
 
         # Make sure the object is read only.
         def illegal_assignment():
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 013d86ebf9cb9..12bf65be69c6e 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -88,6 +88,7 @@ def array_custom_serializer(obj):
 def array_custom_deserializer(serialized_obj):
     return np.array(serialized_obj[0], dtype=np.dtype(serialized_obj[1]))
 
+
 pa.lib.register_type(np.ndarray, 20 * b"\x00", pickle=False,
                      custom_serializer=array_custom_serializer,
                      custom_deserializer=array_custom_deserializer)

From 0728148efef4f7d9e783f72db06b1d81e587c3d5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 28 Aug 2017 14:46:43 -0400
Subject: [PATCH 0962/1644] ARROW-1364: [C++] IPC support machinery for record
 batch roundtrips to GPU device memory

This additionally does a few things:

* Change libarrow_gpu to use CUDA driver API instead of runtime API
* Adds code for exporting buffers using CUDA IPC on Linux, but this is not yet tested

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1000 from wesm/ARROW-1364 and squashes the following commits:

e436755b [Wes McKinney] Add newline at end of file
a8812afc [Wes McKinney] Complete basic IPC message and record batch reads on GPU device memory
16d628f7 [Wes McKinney] More Arrow IPC scaffolding
591aceb0 [Wes McKinney] Draft SerializeRecordBatch for CUDA
84e45252 [Wes McKinney] Add classes and methods for simplifying use of CUDA IPC machinery. No tests yet
508febb5 [Wes McKinney] Test suite passing again
f3c724e0 [Wes McKinney] Get things compiling / linking using driver API
5d686fe9 [Wes McKinney] More progress
2840c60b [Wes McKinney] Progress
3a37fdfe [Wes McKinney] Start cuda context class
03d0baf7 [Wes McKinney] Start cuda_ipc file
---
 cpp/CMakeLists.txt                  |  10 +-
 cpp/src/arrow/gpu/.gitignore        |  18 ++
 cpp/src/arrow/gpu/CMakeLists.txt    |  95 +++-------
 cpp/src/arrow/gpu/cuda-benchmark.cc |   9 +-
 cpp/src/arrow/gpu/cuda-test.cc      | 114 +++++++++++-
 cpp/src/arrow/gpu/cuda_api.h        |   4 +-
 cpp/src/arrow/gpu/cuda_arrow_ipc.cc | 110 +++++++++++
 cpp/src/arrow/gpu/cuda_arrow_ipc.h  |  77 ++++++++
 cpp/src/arrow/gpu/cuda_common.h     |  17 +-
 cpp/src/arrow/gpu/cuda_context.cc   | 277 ++++++++++++++++++++++++++++
 cpp/src/arrow/gpu/cuda_context.h    | 108 +++++++++++
 cpp/src/arrow/gpu/cuda_memory.cc    | 120 ++++++++----
 cpp/src/arrow/gpu/cuda_memory.h     |  71 +++++--
 cpp/src/arrow/ipc/message.cc        |  60 +++---
 cpp/src/arrow/ipc/message.h         |   9 +
 cpp/src/arrow/ipc/writer.cc         |  13 +-
 cpp/src/arrow/ipc/writer.h          |  12 ++
 python/pyarrow/plasma.pyx           |  14 +-
 python/pyarrow/tests/test_plasma.py |  25 +--
 19 files changed, 950 insertions(+), 213 deletions(-)
 create mode 100644 cpp/src/arrow/gpu/.gitignore
 create mode 100644 cpp/src/arrow/gpu/cuda_arrow_ipc.cc
 create mode 100644 cpp/src/arrow/gpu/cuda_arrow_ipc.h
 create mode 100644 cpp/src/arrow/gpu/cuda_context.cc
 create mode 100644 cpp/src/arrow/gpu/cuda_context.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5d643336c05b0..cb7aa3a57b981 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -712,15 +712,17 @@ endif()
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
 
+if (ARROW_GPU)
+  # IPC extensions required to build the GPU library
+  set(ARROW_IPC ON)
+  add_subdirectory(src/arrow/gpu)
+endif()
+
 if (ARROW_IPC)
   add_subdirectory(src/arrow/ipc)
   add_dependencies(arrow_dependencies metadata_fbs)
 endif()
 
-if (ARROW_GPU)
-  add_subdirectory(src/arrow/gpu)
-endif()
-
 set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/buffer.cc
diff --git a/cpp/src/arrow/gpu/.gitignore b/cpp/src/arrow/gpu/.gitignore
new file mode 100644
index 0000000000000..0ef3f98c58c07
--- /dev/null
+++ b/cpp/src/arrow/gpu/.gitignore
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cuda_version.h
diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt
index cab085302c6c4..176916e282292 100644
--- a/cpp/src/arrow/gpu/CMakeLists.txt
+++ b/cpp/src/arrow/gpu/CMakeLists.txt
@@ -15,52 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-function(ADD_ARROW_CUDA_TEST REL_TEST_NAME)
-  set(options)
-  set(single_value_args)
-  set(multi_value_args STATIC_LINK_LIBS)
-  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
-  if(ARG_UNPARSED_ARGUMENTS)
-    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
-  endif()
-
-  if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
-    # This test has a corresponding .cc file, set it up as an executable.
-    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-    cuda_add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
-
-    if (ARG_STATIC_LINK_LIBS)
-      # Customize link libraries
-      target_link_libraries(${TEST_NAME} ${ARG_STATIC_LINK_LIBS})
-    else()
-      target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
-    endif()
-    add_dependencies(unittest ${TEST_NAME})
-  else()
-    # No executable, just invoke the test (probably a script) directly.
-    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
-  endif()
-
-  if (ARROW_TEST_MEMCHECK)
-    SET_PROPERTY(TARGET ${TEST_NAME}
-      APPEND_STRING PROPERTY
-      COMPILE_FLAGS " -DARROW_VALGRIND")
-    add_test(${TEST_NAME}
-      bash -c "cd ${EXECUTABLE_OUTPUT_PATH}; valgrind --tool=memcheck --leak-check=full --leak-check-heuristics=stdstring --error-exitcode=1 ${TEST_PATH}")
-  elseif(MSVC)
-    add_test(${TEST_NAME} ${TEST_PATH})
-  else()
-    add_test(${TEST_NAME}
-      ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
-  endif()
-  set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
-endfunction()
-
 #######################################
 # arrow_gpu
 #######################################
@@ -73,36 +27,37 @@ find_package(CUDA REQUIRED)
 include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
 
 set(ARROW_GPU_SRCS
+  cuda_arrow_ipc.cc
+  cuda_context.cc
   cuda_memory.cc
 )
 
 set(ARROW_GPU_SHARED_LINK_LIBS
   arrow_shared
+  ${CUDA_LIBRARIES}
+  ${CUDA_CUDA_LIBRARY}
 )
 
-add_library(arrow_gpu_objlib OBJECT
-  ${ARROW_GPU_SRCS}
+ADD_ARROW_LIB(arrow_gpu
+  SOURCES ${ARROW_GPU_SRCS}
+  SHARED_LINK_FLAGS ""
+  SHARED_LINK_LIBS ${ARROW_GPU_SHARED_LINK_LIBS}
+  STATIC_LINK_LIBS ""
 )
-set_property(TARGET arrow_gpu_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
 
-if (ARROW_BUILD_SHARED)
-  cuda_add_library(arrow_gpu_shared SHARED $<TARGET_OBJECTS:arrow_gpu_objlib>)
-  install(TARGETS arrow_gpu_shared
-    RUNTIME DESTINATION bin
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
+# CUDA build version
+configure_file(cuda_version.h.in
+  "${CMAKE_CURRENT_SOURCE_DIR}/cuda_version.h"
+  @ONLY)
 
-if (ARROW_BUILD_STATIC)
-  add_library(arrow_gpu_static STATIC $<TARGET_OBJECTS:arrow_gpu_objlib>)
-  install(TARGETS arrow_gpu_static
-    RUNTIME DESTINATION bin
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
+install(FILES
+  "${CMAKE_CURRENT_SOURCE_DIR}/cuda_version.h"
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
 
 install(FILES
-  cuda_common.h
+  cuda_api.h
+  cuda_arrow_ipc.h
+  cuda_context.h
   cuda_memory.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
 
@@ -110,25 +65,17 @@ install(FILES
 configure_file(arrow-gpu.pc.in
   "${CMAKE_CURRENT_BINARY_DIR}/arrow-gpu.pc"
   @ONLY)
+
 install(
   FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-gpu.pc"
   DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
 
-# CUDA build version
-configure_file(cuda_version.h.in
-  "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h"
-  @ONLY)
-
-install(FILES
-  "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h"
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
-
 set(ARROW_GPU_TEST_LINK_LIBS
   arrow_gpu_shared
   ${ARROW_TEST_LINK_LIBS})
 
 if (ARROW_BUILD_TESTS)
-  ADD_ARROW_CUDA_TEST(cuda-test
+  ADD_ARROW_TEST(cuda-test
     STATIC_LINK_LIBS ${ARROW_GPU_TEST_LINK_LIBS})
 endif()
 
diff --git a/cpp/src/arrow/gpu/cuda-benchmark.cc b/cpp/src/arrow/gpu/cuda-benchmark.cc
index 82caacc05e478..805a044403c52 100644
--- a/cpp/src/arrow/gpu/cuda-benchmark.cc
+++ b/cpp/src/arrow/gpu/cuda-benchmark.cc
@@ -25,7 +25,7 @@
 #include "arrow/memory_pool.h"
 #include "arrow/test-util.h"
 
-#include "arrow/gpu/cuda_memory.h"
+#include "arrow/gpu/cuda_api.h"
 
 namespace arrow {
 namespace gpu {
@@ -35,8 +35,13 @@ constexpr int64_t kGpuNumber = 0;
 static void CudaBufferWriterBenchmark(benchmark::State& state, const int64_t total_bytes,
                                       const int64_t chunksize,
                                       const int64_t buffer_size) {
+  CudaDeviceManager* manager;
+  ABORT_NOT_OK(CudaDeviceManager::GetInstance(&manager));
+  std::shared_ptr<CudaContext> context;
+  ABORT_NOT_OK(manager->GetContext(kGpuNumber, &context));
+
   std::shared_ptr<CudaBuffer> device_buffer;
-  ABORT_NOT_OK(AllocateCudaBuffer(kGpuNumber, total_bytes, &device_buffer));
+  ABORT_NOT_OK(context->Allocate(total_bytes, &device_buffer));
   CudaBufferWriter writer(device_buffer);
 
   if (buffer_size > 0) {
diff --git a/cpp/src/arrow/gpu/cuda-test.cc b/cpp/src/arrow/gpu/cuda-test.cc
index f479701eaeb31..aa9d3efd2ac9b 100644
--- a/cpp/src/arrow/gpu/cuda-test.cc
+++ b/cpp/src/arrow/gpu/cuda-test.cc
@@ -21,23 +21,39 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/ipc/api.h"
+#include "arrow/ipc/test-common.h"
 #include "arrow/status.h"
 #include "arrow/test-util.h"
 
-#include "arrow/gpu/cuda_memory.h"
+#include "arrow/gpu/cuda_api.h"
 
 namespace arrow {
 namespace gpu {
 
 constexpr int kGpuNumber = 0;
 
-class TestCudaBuffer : public ::testing::Test {};
+class TestCudaBufferBase : public ::testing::Test {
+ public:
+  void SetUp() {
+    ASSERT_OK(CudaDeviceManager::GetInstance(&manager_));
+    ASSERT_OK(manager_->GetContext(kGpuNumber, &context_));
+  }
+
+ protected:
+  CudaDeviceManager* manager_;
+  std::shared_ptr<CudaContext> context_;
+};
+
+class TestCudaBuffer : public TestCudaBufferBase {
+ public:
+  void SetUp() { TestCudaBufferBase::SetUp(); }
+};
 
 TEST_F(TestCudaBuffer, Allocate) {
   const int64_t kSize = 100;
   std::shared_ptr<CudaBuffer> buffer;
-
-  ASSERT_OK(AllocateCudaBuffer(kGpuNumber, kSize, &buffer));
+  ASSERT_OK(context_->Allocate(kSize, &buffer));
   ASSERT_EQ(kSize, buffer->size());
 }
 
@@ -52,7 +68,7 @@ void AssertCudaBufferEquals(const CudaBuffer& buffer, const uint8_t* host_data,
 TEST_F(TestCudaBuffer, CopyFromHost) {
   const int64_t kSize = 1000;
   std::shared_ptr<CudaBuffer> device_buffer;
-  ASSERT_OK(AllocateCudaBuffer(kGpuNumber, kSize, &device_buffer));
+  ASSERT_OK(context_->Allocate(kSize, &device_buffer));
 
   std::shared_ptr<PoolBuffer> host_buffer;
   ASSERT_OK(test::MakeRandomBytePoolBuffer(kSize, default_memory_pool(), &host_buffer));
@@ -63,10 +79,49 @@ TEST_F(TestCudaBuffer, CopyFromHost) {
   AssertCudaBufferEquals(*device_buffer, host_buffer->data(), kSize);
 }
 
-class TestCudaBufferWriter : public ::testing::Test {
+// IPC only supported on Linux
+#if defined(__linux)
+
+TEST_F(TestCudaBuffer, DISABLED_ExportForIpc) {
+  // For this test to work, a second process needs to be spawned
+  const int64_t kSize = 1000;
+  std::shared_ptr<CudaBuffer> device_buffer;
+  ASSERT_OK(context_->Allocate(kSize, &device_buffer));
+
+  std::shared_ptr<PoolBuffer> host_buffer;
+  ASSERT_OK(test::MakeRandomBytePoolBuffer(kSize, default_memory_pool(), &host_buffer));
+  ASSERT_OK(device_buffer->CopyFromHost(0, host_buffer->data(), kSize));
+
+  // Export for IPC and serialize
+  std::unique_ptr<CudaIpcMemHandle> ipc_handle;
+  ASSERT_OK(device_buffer->ExportForIpc(&ipc_handle));
+
+  std::shared_ptr<Buffer> serialized_handle;
+  ASSERT_OK(ipc_handle->Serialize(default_memory_pool(), &serialized_handle));
+
+  // Deserialize IPC handle and open
+  std::unique_ptr<CudaIpcMemHandle> ipc_handle2;
+  ASSERT_OK(CudaIpcMemHandle::FromBuffer(serialized_handle->data(), &ipc_handle2));
+
+  std::shared_ptr<CudaBuffer> ipc_buffer;
+  ASSERT_OK(context_->OpenIpcBuffer(*ipc_handle2, &ipc_buffer));
+
+  ASSERT_EQ(kSize, ipc_buffer->size());
+
+  std::shared_ptr<MutableBuffer> ipc_data;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), kSize, &ipc_data));
+  ASSERT_OK(ipc_buffer->CopyToHost(0, kSize, ipc_data->mutable_data()));
+  ASSERT_EQ(0, std::memcmp(ipc_buffer->data(), host_buffer->data(), kSize));
+}
+
+#endif
+
+class TestCudaBufferWriter : public TestCudaBufferBase {
  public:
+  void SetUp() { TestCudaBufferBase::SetUp(); }
+
   void Allocate(const int64_t size) {
-    ASSERT_OK(AllocateCudaBuffer(kGpuNumber, size, &device_buffer_));
+    ASSERT_OK(context_->Allocate(size, &device_buffer_));
     writer_.reset(new CudaBufferWriter(device_buffer_));
   }
 
@@ -164,11 +219,16 @@ TEST_F(TestCudaBufferWriter, EdgeCases) {
   AssertCudaBufferEquals(*device_buffer_, host_data, 1000);
 }
 
-TEST(TestCudaBufferReader, Basics) {
+class TestCudaBufferReader : public TestCudaBufferBase {
+ public:
+  void SetUp() { TestCudaBufferBase::SetUp(); }
+};
+
+TEST_F(TestCudaBufferReader, Basics) {
   std::shared_ptr<CudaBuffer> device_buffer;
 
   const int64_t size = 1000;
-  ASSERT_OK(AllocateCudaBuffer(kGpuNumber, size, &device_buffer));
+  ASSERT_OK(context_->Allocate(size, &device_buffer));
 
   std::shared_ptr<PoolBuffer> buffer;
   ASSERT_OK(test::MakeRandomBytePoolBuffer(1000, default_memory_pool(), &buffer));
@@ -204,5 +264,41 @@ TEST(TestCudaBufferReader, Basics) {
   ASSERT_EQ(0, std::memcmp(stack_buffer, host_data + 925, 75));
 }
 
+class TestCudaArrowIpc : public TestCudaBufferBase {
+ public:
+  void SetUp() {
+    TestCudaBufferBase::SetUp();
+    pool_ = default_memory_pool();
+  }
+
+ protected:
+  MemoryPool* pool_;
+};
+
+TEST_F(TestCudaArrowIpc, BasicWriteRead) {
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(ipc::MakeIntRecordBatch(&batch));
+
+  std::shared_ptr<CudaBuffer> device_serialized;
+  ASSERT_OK(arrow::gpu::SerializeRecordBatch(*batch, context_.get(), &device_serialized));
+
+  // Test that ReadRecordBatch works properly
+  std::shared_ptr<RecordBatch> device_batch;
+  ASSERT_OK(ReadRecordBatch(batch->schema(), device_serialized, default_memory_pool(),
+                            &device_batch));
+
+  // Copy data from device, read batch, and compare
+  std::shared_ptr<MutableBuffer> host_buffer;
+  int64_t size = device_serialized->size();
+  ASSERT_OK(AllocateBuffer(pool_, size, &host_buffer));
+  ASSERT_OK(device_serialized->CopyToHost(0, size, host_buffer->mutable_data()));
+
+  std::shared_ptr<RecordBatch> cpu_batch;
+  io::BufferReader cpu_reader(host_buffer);
+  ASSERT_OK(ipc::ReadRecordBatch(batch->schema(), &cpu_reader, &cpu_batch));
+
+  ipc::CompareBatch(*batch, *cpu_batch);
+}
+
 }  // namespace gpu
 }  // namespace arrow
diff --git a/cpp/src/arrow/gpu/cuda_api.h b/cpp/src/arrow/gpu/cuda_api.h
index a70e0af92aff4..c63b77e872165 100644
--- a/cpp/src/arrow/gpu/cuda_api.h
+++ b/cpp/src/arrow/gpu/cuda_api.h
@@ -18,7 +18,9 @@
 #ifndef ARROW_GPU_CUDA_API_H
 #define ARROW_GPU_CUDA_API_H
 
+#include "arrow/gpu/cuda_arrow_ipc.h"
+#include "arrow/gpu/cuda_context.h"
 #include "arrow/gpu/cuda_memory.h"
 #include "arrow/gpu/cuda_version.h"
 
-#endif // ARROW_GPU_CUDA_API_H
+#endif  // ARROW_GPU_CUDA_API_H
diff --git a/cpp/src/arrow/gpu/cuda_arrow_ipc.cc b/cpp/src/arrow/gpu/cuda_arrow_ipc.cc
new file mode 100644
index 0000000000000..669857d920336
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_arrow_ipc.cc
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/gpu/cuda_arrow_ipc.h"
+
+#include <cstdint>
+#include <memory>
+#include <sstream>
+#include <vector>
+
+#include "arrow/buffer.h"
+#include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/message.h"
+#include "arrow/ipc/reader.h"
+#include "arrow/ipc/writer.h"
+#include "arrow/status.h"
+#include "arrow/table.h"
+#include "arrow/util/visibility.h"
+
+#include "arrow/gpu/cuda_context.h"
+#include "arrow/gpu/cuda_memory.h"
+
+namespace arrow {
+
+namespace flatbuf = org::apache::arrow::flatbuf;
+
+namespace gpu {
+
+Status SerializeRecordBatch(const RecordBatch& batch, CudaContext* ctx,
+                            std::shared_ptr<CudaBuffer>* out) {
+  int64_t size = 0;
+  RETURN_NOT_OK(ipc::GetRecordBatchSize(batch, &size));
+
+  std::shared_ptr<CudaBuffer> buffer;
+  RETURN_NOT_OK(ctx->Allocate(size, &buffer));
+
+  CudaBufferWriter stream(buffer);
+
+  // Use 8MB buffering, which yields generally good performance
+  RETURN_NOT_OK(stream.SetBufferSize(1 << 23));
+
+  // We use the default memory pool here since any allocations are ephemeral
+  RETURN_NOT_OK(ipc::SerializeRecordBatch(batch, default_memory_pool(), &stream));
+  RETURN_NOT_OK(stream.Close());
+  *out = buffer;
+  return Status::OK();
+}
+
+Status ReadMessage(CudaBufferReader* reader, MemoryPool* pool,
+                   std::unique_ptr<ipc::Message>* out) {
+  int32_t message_length = 0;
+  int64_t bytes_read = 0;
+
+  RETURN_NOT_OK(reader->Read(sizeof(int32_t), &bytes_read,
+                             reinterpret_cast<uint8_t*>(&message_length)));
+  if (bytes_read != sizeof(int32_t)) {
+    *out = nullptr;
+    return Status::OK();
+  }
+
+  if (message_length == 0) {
+    // Optional 0 EOS control message
+    *out = nullptr;
+    return Status::OK();
+  }
+
+  std::shared_ptr<MutableBuffer> metadata;
+  RETURN_NOT_OK(AllocateBuffer(pool, message_length, &metadata));
+  RETURN_NOT_OK(reader->Read(message_length, &bytes_read, metadata->mutable_data()));
+  if (bytes_read != message_length) {
+    std::stringstream ss;
+    ss << "Expected " << message_length << " metadata bytes, but only got " << bytes_read;
+    return Status::IOError(ss.str());
+  }
+
+  return ipc::Message::ReadFrom(metadata, reader, out);
+}
+
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema,
+                       const std::shared_ptr<CudaBuffer>& buffer, MemoryPool* pool,
+                       std::shared_ptr<RecordBatch>* out) {
+  CudaBufferReader cuda_reader(buffer);
+
+  std::unique_ptr<ipc::Message> message;
+  RETURN_NOT_OK(ReadMessage(&cuda_reader, pool, &message));
+
+  if (!message) {
+    return Status::Invalid("Message is length 0");
+  }
+
+  // Zero-copy read on device memory
+  return ipc::ReadRecordBatch(*message, schema, out);
+}
+
+}  // namespace gpu
+}  // namespace arrow
diff --git a/cpp/src/arrow/gpu/cuda_arrow_ipc.h b/cpp/src/arrow/gpu/cuda_arrow_ipc.h
new file mode 100644
index 0000000000000..52dd92473eaec
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_arrow_ipc.h
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_GPU_CUDA_ARROW_IPC_H
+#define ARROW_GPU_CUDA_ARROW_IPC_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+#include "arrow/util/visibility.h"
+
+#include "arrow/gpu/cuda_memory.h"
+
+namespace arrow {
+
+class MemoryPool;
+class RecordBatch;
+class Schema;
+
+namespace ipc {
+
+class Message;
+
+}  // namespace ipc
+
+namespace gpu {
+
+/// \brief Write record batch message to GPU device memory
+/// \param[in] batch record batch to write
+/// \param[in] ctx CudaContext to allocate device memory from
+/// \param[out] out the returned device buffer which contains the record batch message
+/// \return Status
+ARROW_EXPORT
+Status SerializeRecordBatch(const RecordBatch& batch, CudaContext* ctx,
+                            std::shared_ptr<CudaBuffer>* out);
+
+/// \brief Read Arrow IPC message located on GPU device
+/// \param[in] reader a CudaBufferReader
+/// \param[in] pool a MemoryPool to allocate CPU memory for the metadata
+/// \param[out] message the deserialized message, body still on device
+///
+/// This function reads the message metadata into host memory, but leaves the
+/// message body on the device
+ARROW_EXPORT
+Status ReadMessage(CudaBufferReader* reader, MemoryPool* pool,
+                   std::unique_ptr<ipc::Message>* message);
+
+/// \brief ReadRecordBatch specialized to handle metadata on CUDA device
+/// \param[in] schema the Schema for the record batch
+/// \param[in] buffer a CudaBuffer containing the complete IPC message
+/// \param[in] pool a MemoryPool to use for allocating space for the metadata
+/// \param[out] out the reconstructed RecordBatch, with device pointers
+ARROW_EXPORT
+Status ReadRecordBatch(const std::shared_ptr<Schema>& schema,
+                       const std::shared_ptr<CudaBuffer>& buffer, MemoryPool* pool,
+                       std::shared_ptr<RecordBatch>* out);
+
+}  // namespace gpu
+}  // namespace arrow
+
+#endif  // ARROW_GPU_CUDA_ARROW_IPC_H
diff --git a/cpp/src/arrow/gpu/cuda_common.h b/cpp/src/arrow/gpu/cuda_common.h
index 1d65f96adbcdc..c06c1a21ff481 100644
--- a/cpp/src/arrow/gpu/cuda_common.h
+++ b/cpp/src/arrow/gpu/cuda_common.h
@@ -22,7 +22,7 @@
 
 #include <sstream>
 
-#include <cuda_runtime_api.h>
+#include <cuda.h>
 
 namespace arrow {
 namespace gpu {
@@ -34,24 +34,13 @@ namespace gpu {
     (void)ret;            \
   } while (0)
 
-#define CUDA_RETURN_NOT_OK(STMT)                                       \
-  do {                                                                 \
-    cudaError_t ret = (STMT);                                          \
-    if (ret != cudaSuccess) {                                          \
-      std::stringstream ss;                                            \
-      ss << "Cuda API call in " << __FILE__ << " at line " << __LINE__ \
-         << " failed: " << #STMT;                                      \
-      return Status::IOError(ss.str());                                \
-    }                                                                  \
-  } while (0)
-
-#define CUDADRV_RETURN_NOT_OK(STMT)                                           \
+#define CU_RETURN_NOT_OK(STMT)                                                \
   do {                                                                        \
     CUresult ret = (STMT);                                                    \
     if (ret != CUDA_SUCCESS) {                                                \
       std::stringstream ss;                                                   \
       ss << "Cuda Driver API call in " << __FILE__ << " at line " << __LINE__ \
-         << " failed: " << #STMT;                                             \
+         << " failed with code " << ret << ": " << #STMT;                     \
       return Status::IOError(ss.str());                                       \
     }                                                                         \
   } while (0)
diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc
new file mode 100644
index 0000000000000..430ecab6fbe07
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_context.cc
@@ -0,0 +1,277 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/gpu/cuda_context.h"
+
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include <cuda.h>
+
+#include "arrow/gpu/cuda_common.h"
+#include "arrow/gpu/cuda_memory.h"
+
+namespace arrow {
+namespace gpu {
+
+struct CudaDevice {
+  int device_num;
+  CUdevice handle;
+  int64_t total_memory;
+};
+
+class CudaContext::CudaContextImpl {
+ public:
+  CudaContextImpl() {}
+
+  Status Init(const CudaDevice& device) {
+    device_ = device;
+    CU_RETURN_NOT_OK(cuCtxCreate(&context_, 0, device_.handle));
+    is_open_ = true;
+    return Status::OK();
+  }
+
+  Status Close() {
+    if (is_open_ && own_context_) {
+      CU_RETURN_NOT_OK(cuCtxDestroy(context_));
+    }
+    is_open_ = false;
+    return Status::OK();
+  }
+
+  int64_t bytes_allocated() const { return bytes_allocated_.load(); }
+
+  Status Allocate(int64_t nbytes, uint8_t** out) {
+    CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
+
+    CUdeviceptr data;
+    CU_RETURN_NOT_OK(cuMemAlloc(&data, static_cast<size_t>(nbytes)));
+    *out = reinterpret_cast<uint8_t*>(data);
+    return Status::OK();
+  }
+
+  Status CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+    CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
+    CU_RETURN_NOT_OK(cuMemcpyHtoD(reinterpret_cast<CUdeviceptr>(dst),
+                                  reinterpret_cast<const void*>(src),
+                                  static_cast<size_t>(nbytes)));
+    return Status::OK();
+  }
+
+  Status CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+    CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
+    CU_RETURN_NOT_OK(cuMemcpyDtoH(dst, reinterpret_cast<const CUdeviceptr>(src),
+                                  static_cast<size_t>(nbytes)));
+    return Status::OK();
+  }
+
+  Status Free(uint8_t* device_ptr, int64_t nbytes) {
+    CU_RETURN_NOT_OK(cuMemFree(reinterpret_cast<CUdeviceptr>(device_ptr)));
+    return Status::OK();
+  }
+
+  Status ExportIpcBuffer(uint8_t* data, std::unique_ptr<CudaIpcMemHandle>* handle) {
+    CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
+    CUipcMemHandle cu_handle;
+    CU_RETURN_NOT_OK(cuIpcGetMemHandle(&cu_handle, reinterpret_cast<CUdeviceptr>(data)));
+    *handle = std::unique_ptr<CudaIpcMemHandle>(new CudaIpcMemHandle(&cu_handle));
+    return Status::OK();
+  }
+
+  Status OpenIpcBuffer(const CudaIpcMemHandle& ipc_handle, uint8_t** out) {
+    CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
+    auto handle = reinterpret_cast<const CUipcMemHandle*>(ipc_handle.handle());
+
+    CUdeviceptr data;
+    CU_RETURN_NOT_OK(
+        cuIpcOpenMemHandle(&data, *handle, CU_IPC_MEM_LAZY_ENABLE_PEER_ACCESS));
+    *out = reinterpret_cast<uint8_t*>(data);
+    return Status::OK();
+  }
+
+  const CudaDevice device() const { return device_; }
+
+ private:
+  CudaDevice device_;
+  CUcontext context_;
+  bool is_open_;
+
+  // So that we can utilize a CUcontext that was created outside this library
+  bool own_context_;
+
+  std::atomic<int64_t> bytes_allocated_;
+};
+
+class CudaDeviceManager::CudaDeviceManagerImpl {
+ public:
+  CudaDeviceManagerImpl() : host_bytes_allocated_(0) {}
+
+  Status Init() {
+    CU_RETURN_NOT_OK(cuInit(0));
+    CU_RETURN_NOT_OK(cuDeviceGetCount(&num_devices_));
+
+    devices_.resize(num_devices_);
+    for (int i = 0; i < num_devices_; ++i) {
+      RETURN_NOT_OK(GetDeviceProperties(i, &devices_[i]));
+    }
+    return Status::OK();
+  }
+
+  Status AllocateHost(int64_t nbytes, uint8_t** out) {
+    CU_RETURN_NOT_OK(cuMemHostAlloc(reinterpret_cast<void**>(out),
+                                    static_cast<size_t>(nbytes),
+                                    CU_MEMHOSTALLOC_PORTABLE));
+    host_bytes_allocated_ += nbytes;
+    return Status::OK();
+  }
+
+  Status FreeHost(uint8_t* data, int64_t nbytes) {
+    CU_RETURN_NOT_OK(cuMemFreeHost(data));
+    host_bytes_allocated_ -= nbytes;
+    return Status::OK();
+  }
+
+  Status GetDeviceProperties(int device_number, CudaDevice* device) {
+    device->device_num = device_number;
+    CU_RETURN_NOT_OK(cuDeviceGet(&device->handle, device_number));
+
+    size_t total_memory = 0;
+    CU_RETURN_NOT_OK(cuDeviceTotalMem(&total_memory, device->handle));
+    device->total_memory = total_memory;
+    return Status::OK();
+  }
+
+  Status CreateNewContext(int device_number, std::shared_ptr<CudaContext>* out) {
+    *out = std::shared_ptr<CudaContext>(new CudaContext());
+    return (*out)->impl_->Init(devices_[device_number]);
+  }
+
+  Status GetContext(int device_number, std::shared_ptr<CudaContext>* out) {
+    auto it = contexts_.find(device_number);
+    if (it == contexts_.end()) {
+      std::shared_ptr<CudaContext> new_context;
+      RETURN_NOT_OK(CreateNewContext(device_number, &new_context));
+      contexts_[device_number] = *out = new_context;
+    } else {
+      *out = it->second;
+    }
+    return Status::OK();
+  }
+
+  int num_devices() const { return num_devices_; }
+
+ private:
+  int num_devices_;
+  std::vector<CudaDevice> devices_;
+
+  // device_number -> CudaContext
+  std::unordered_map<int, std::shared_ptr<CudaContext>> contexts_;
+
+  int host_bytes_allocated_;
+};
+
+CudaDeviceManager::CudaDeviceManager() { impl_.reset(new CudaDeviceManagerImpl()); }
+
+std::unique_ptr<CudaDeviceManager> CudaDeviceManager::instance_ = nullptr;
+
+Status CudaDeviceManager::GetInstance(CudaDeviceManager** manager) {
+  if (!instance_) {
+    instance_.reset(new CudaDeviceManager());
+    RETURN_NOT_OK(instance_->impl_->Init());
+  }
+  *manager = instance_.get();
+  return Status::OK();
+}
+
+Status CudaDeviceManager::GetContext(int device_number,
+                                     std::shared_ptr<CudaContext>* out) {
+  return impl_->GetContext(device_number, out);
+}
+
+Status CudaDeviceManager::CreateNewContext(int device_number,
+                                           std::shared_ptr<CudaContext>* out) {
+  return impl_->CreateNewContext(device_number, out);
+}
+
+Status CudaDeviceManager::AllocateHost(int64_t nbytes,
+                                       std::shared_ptr<CudaHostBuffer>* out) {
+  uint8_t* data = nullptr;
+  RETURN_NOT_OK(impl_->AllocateHost(nbytes, &data));
+  *out = std::make_shared<CudaHostBuffer>(data, nbytes);
+  return Status::OK();
+}
+
+Status CudaDeviceManager::FreeHost(uint8_t* data, int64_t nbytes) {
+  return impl_->FreeHost(data, nbytes);
+}
+
+int CudaDeviceManager::num_devices() const { return impl_->num_devices(); }
+
+// ----------------------------------------------------------------------
+// CudaContext public API
+
+CudaContext::CudaContext() { impl_.reset(new CudaContextImpl()); }
+
+CudaContext::~CudaContext() {}
+
+Status CudaContext::Allocate(int64_t nbytes, std::shared_ptr<CudaBuffer>* out) {
+  uint8_t* data = nullptr;
+  RETURN_NOT_OK(impl_->Allocate(nbytes, &data));
+  *out = std::make_shared<CudaBuffer>(data, nbytes, this->shared_from_this(), true);
+  return Status::OK();
+}
+
+Status CudaContext::ExportIpcBuffer(uint8_t* data,
+                                    std::unique_ptr<CudaIpcMemHandle>* handle) {
+  return impl_->ExportIpcBuffer(data, handle);
+}
+
+Status CudaContext::CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+  return impl_->CopyHostToDevice(dst, src, nbytes);
+}
+
+Status CudaContext::CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+  return impl_->CopyDeviceToHost(dst, src, nbytes);
+}
+
+Status CudaContext::Close() { return impl_->Close(); }
+
+Status CudaContext::Free(uint8_t* device_ptr, int64_t nbytes) {
+  return impl_->Free(device_ptr, nbytes);
+}
+
+Status CudaContext::OpenIpcBuffer(const CudaIpcMemHandle& ipc_handle,
+                                  std::shared_ptr<CudaBuffer>* out) {
+  uint8_t* data = nullptr;
+  RETURN_NOT_OK(impl_->OpenIpcBuffer(ipc_handle, &data));
+
+  // Need to ask the device how big the buffer is
+  size_t allocation_size = 0;
+  CU_RETURN_NOT_OK(cuMemGetAddressRange(nullptr, &allocation_size,
+                                        reinterpret_cast<CUdeviceptr>(data)));
+
+  *out = std::make_shared<CudaBuffer>(data, allocation_size, this->shared_from_this(),
+                                      true, true);
+  return Status::OK();
+}
+
+}  // namespace gpu
+}  // namespace arrow
diff --git a/cpp/src/arrow/gpu/cuda_context.h b/cpp/src/arrow/gpu/cuda_context.h
new file mode 100644
index 0000000000000..6471059612349
--- /dev/null
+++ b/cpp/src/arrow/gpu/cuda_context.h
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_GPU_CUDA_CONTEXT_H
+#define ARROW_GPU_CUDA_CONTEXT_H
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/status.h"
+#include "arrow/util/visibility.h"
+
+#include "arrow/gpu/cuda_memory.h"
+
+namespace arrow {
+namespace gpu {
+
+// Forward declaration
+class CudaContext;
+
+class ARROW_EXPORT CudaDeviceManager {
+ public:
+  static Status GetInstance(CudaDeviceManager** manager);
+
+  /// \brief Get the shared CUDA driver context for a particular device
+  Status GetContext(int gpu_number, std::shared_ptr<CudaContext>* ctx);
+
+  /// \brief Create a new context for a given device number
+  ///
+  /// In general code will use GetContext
+  Status CreateNewContext(int gpu_number, std::shared_ptr<CudaContext>* ctx);
+
+  Status AllocateHost(int64_t nbytes, std::shared_ptr<CudaHostBuffer>* buffer);
+
+  Status FreeHost(uint8_t* data, int64_t nbytes);
+
+  int num_devices() const;
+
+ private:
+  CudaDeviceManager();
+  static std::unique_ptr<CudaDeviceManager> instance_;
+
+  class CudaDeviceManagerImpl;
+  std::unique_ptr<CudaDeviceManagerImpl> impl_;
+
+  friend CudaContext;
+};
+
+struct ARROW_EXPORT CudaDeviceInfo {};
+
+/// \class CudaContext
+/// \brief Friendlier interface to the CUDA driver API
+class ARROW_EXPORT CudaContext : public std::enable_shared_from_this<CudaContext> {
+ public:
+  ~CudaContext();
+
+  Status Close();
+
+  /// \brief Allocate CUDA memory on GPU device for this context
+  /// \param[in] nbytes number of bytes
+  /// \param[out] out the allocated buffer
+  /// \return Status
+  Status Allocate(int64_t nbytes, std::shared_ptr<CudaBuffer>* out);
+
+  /// \brief Open existing CUDA IPC memory handle
+  /// \param[in] ipc_handle opaque pointer to CUipcMemHandle (driver API)
+  /// \param[out] buffer a CudaBuffer referencing
+  /// \return Status
+  Status OpenIpcBuffer(const CudaIpcMemHandle& ipc_handle,
+                       std::shared_ptr<CudaBuffer>* buffer);
+
+  int64_t bytes_allocated() const;
+
+ private:
+  CudaContext();
+
+  Status ExportIpcBuffer(uint8_t* data, std::unique_ptr<CudaIpcMemHandle>* handle);
+  Status CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes);
+  Status CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes);
+  Status Free(uint8_t* device_ptr, int64_t nbytes);
+
+  class CudaContextImpl;
+  std::unique_ptr<CudaContextImpl> impl_;
+
+  friend CudaBuffer;
+  friend CudaBufferReader;
+  friend CudaBufferWriter;
+  friend CudaDeviceManager::CudaDeviceManagerImpl;
+};
+
+}  // namespace gpu
+}  // namespace arrow
+
+#endif  // ARROW_GPU_CUDA_CONTEXT_H
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
index 4870813b6619b..3c88fe2d59fbc 100644
--- a/cpp/src/arrow/gpu/cuda_memory.cc
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -19,68 +19,124 @@
 
 #include <algorithm>
 #include <cstdint>
+#include <cstdlib>
 #include <memory>
 
+#include <cuda.h>
+
 #include "arrow/buffer.h"
 #include "arrow/io/memory.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
 #include "arrow/gpu/cuda_common.h"
+#include "arrow/gpu/cuda_context.h"
 
 namespace arrow {
 namespace gpu {
 
-CudaBuffer::~CudaBuffer() {
+// ----------------------------------------------------------------------
+// CUDA IPC memory handle
+
+struct CudaIpcMemHandle::CudaIpcMemHandleImpl {
+  explicit CudaIpcMemHandleImpl(const void* handle) {
+    memcpy(&ipc_handle, handle, sizeof(CUipcMemHandle));
+  }
+
+  CUipcMemHandle ipc_handle;
+};
+
+CudaIpcMemHandle::CudaIpcMemHandle(const void* handle) {
+  impl_.reset(new CudaIpcMemHandleImpl(handle));
+}
+
+CudaIpcMemHandle::~CudaIpcMemHandle() {}
+
+Status CudaIpcMemHandle::FromBuffer(const void* opaque_handle,
+                                    std::unique_ptr<CudaIpcMemHandle>* handle) {
+  *handle = std::unique_ptr<CudaIpcMemHandle>(new CudaIpcMemHandle(opaque_handle));
+  return Status::OK();
+}
+
+Status CudaIpcMemHandle::Serialize(MemoryPool* pool, std::shared_ptr<Buffer>* out) const {
+  std::shared_ptr<MutableBuffer> buffer;
+  constexpr size_t kHandleSize = sizeof(CUipcMemHandle);
+  RETURN_NOT_OK(AllocateBuffer(pool, static_cast<int64_t>(kHandleSize), &buffer));
+  memcpy(buffer->mutable_data(), &impl_->ipc_handle, kHandleSize);
+  *out = buffer;
+  return Status::OK();
+}
+
+const void* CudaIpcMemHandle::handle() const { return &impl_->ipc_handle; }
+
+// ----------------------------------------------------------------------
+
+CudaBuffer::CudaBuffer(uint8_t* data, int64_t size,
+                       const std::shared_ptr<CudaContext>& context, bool own_data,
+                       bool is_ipc)
+    : Buffer(data, size), context_(context), own_data_(own_data), is_ipc_(is_ipc) {
+  is_mutable_ = true;
+  mutable_data_ = data;
+}
+
+CudaBuffer::~CudaBuffer() { DCHECK(Close().ok()); }
+
+Status CudaBuffer::Close() {
   if (own_data_) {
-    CUDA_DCHECK(cudaFree(mutable_data_));
+    if (is_ipc_) {
+      CU_RETURN_NOT_OK(cuIpcCloseMemHandle(reinterpret_cast<CUdeviceptr>(mutable_data_)));
+    } else {
+      return context_->Free(mutable_data_, size_);
+    }
   }
+  return Status::OK();
 }
 
 CudaBuffer::CudaBuffer(const std::shared_ptr<CudaBuffer>& parent, const int64_t offset,
                        const int64_t size)
-    : Buffer(parent, offset, size), gpu_number_(parent->gpu_number()) {}
+    : Buffer(parent, offset, size),
+      context_(parent->context()),
+      own_data_(false),
+      is_ipc_(false) {}
 
 Status CudaBuffer::CopyToHost(const int64_t position, const int64_t nbytes,
                               uint8_t* out) const {
-  CUDA_RETURN_NOT_OK(cudaMemcpy(out, data_ + position, nbytes, cudaMemcpyDeviceToHost));
-  return Status::OK();
+  return context_->CopyDeviceToHost(out, data_ + position, nbytes);
 }
 
 Status CudaBuffer::CopyFromHost(const int64_t position, const uint8_t* data,
                                 int64_t nbytes) {
   DCHECK_LE(nbytes, size_ - position) << "Copy would overflow buffer";
-  CUDA_RETURN_NOT_OK(
-      cudaMemcpy(mutable_data_ + position, data, nbytes, cudaMemcpyHostToDevice));
-  return Status::OK();
+  return context_->CopyHostToDevice(mutable_data_ + position, data, nbytes);
 }
 
-Status AllocateCudaBuffer(int gpu_number, const int64_t size,
-                          std::shared_ptr<CudaBuffer>* out) {
-  CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number));
-  uint8_t* data = nullptr;
-  CUDA_RETURN_NOT_OK(
-      cudaMalloc(reinterpret_cast<void**>(&data), static_cast<size_t>(size)));
-  *out = std::make_shared<CudaBuffer>(data, size, gpu_number, true);
+Status CudaBuffer::ExportForIpc(std::unique_ptr<CudaIpcMemHandle>* handle) {
+  if (is_ipc_) {
+    return Status::Invalid("Buffer has already been exported for IPC");
+  }
+  RETURN_NOT_OK(context_->ExportIpcBuffer(mutable_data_, handle));
+  own_data_ = false;
   return Status::OK();
 }
 
-CudaHostBuffer::~CudaHostBuffer() { CUDA_DCHECK(cudaFreeHost(mutable_data_)); }
+CudaHostBuffer::~CudaHostBuffer() {
+  CudaDeviceManager* manager = nullptr;
+  DCHECK(CudaDeviceManager::GetInstance(&manager).ok());
+  DCHECK(manager->FreeHost(mutable_data_, size_).ok());
+}
 
 // ----------------------------------------------------------------------
 // CudaBufferReader
 
 CudaBufferReader::CudaBufferReader(const std::shared_ptr<CudaBuffer>& buffer)
-    : io::BufferReader(buffer), cuda_buffer_(buffer), gpu_number_(buffer->gpu_number()) {}
+    : io::BufferReader(buffer), cuda_buffer_(buffer), context_(buffer->context()) {}
 
 CudaBufferReader::~CudaBufferReader() {}
 
 Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
   nbytes = std::min(nbytes, size_ - position_);
-  CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number_));
-  CUDA_RETURN_NOT_OK(
-      cudaMemcpy(buffer, data_ + position_, nbytes, cudaMemcpyDeviceToHost));
   *bytes_read = nbytes;
+  RETURN_NOT_OK(context_->CopyDeviceToHost(buffer, data_ + position_, nbytes));
   position_ += nbytes;
   return Status::OK();
 }
@@ -97,7 +153,7 @@ Status CudaBufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 
 CudaBufferWriter::CudaBufferWriter(const std::shared_ptr<CudaBuffer>& buffer)
     : io::FixedSizeBufferWriter(buffer),
-      gpu_number_(buffer->gpu_number()),
+      context_(buffer->context()),
       buffer_size_(0),
       buffer_position_(0) {}
 
@@ -108,10 +164,8 @@ Status CudaBufferWriter::Close() { return Flush(); }
 Status CudaBufferWriter::Flush() {
   if (buffer_size_ > 0 && buffer_position_ > 0) {
     // Only need to flush when the write has been buffered
-    CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number_));
-    CUDA_RETURN_NOT_OK(cudaMemcpy(mutable_data_ + position_ - buffer_position_,
-                                  host_buffer_data_, buffer_position_,
-                                  cudaMemcpyHostToDevice));
+    RETURN_NOT_OK(context_->CopyHostToDevice(mutable_data_ + position_ - buffer_position_,
+                                             host_buffer_data_, buffer_position_));
     buffer_position_ = 0;
   }
   return Status::OK();
@@ -137,9 +191,7 @@ Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
     if (nbytes + buffer_position_ >= buffer_size_) {
       // Reach end of buffer, write everything
       RETURN_NOT_OK(Flush());
-      CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number_));
-      CUDA_RETURN_NOT_OK(
-          cudaMemcpy(mutable_data_ + position_, data, nbytes, cudaMemcpyHostToDevice));
+      RETURN_NOT_OK(context_->CopyHostToDevice(mutable_data_ + position_, data, nbytes));
     } else {
       // Write bytes to buffer
       std::memcpy(host_buffer_data_ + buffer_position_, data, nbytes);
@@ -147,9 +199,7 @@ Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
     }
   } else {
     // Unbuffered write
-    CUDA_RETURN_NOT_OK(cudaSetDevice(gpu_number_));
-    CUDA_RETURN_NOT_OK(
-        cudaMemcpy(mutable_data_ + position_, data, nbytes, cudaMemcpyHostToDevice));
+    RETURN_NOT_OK(context_->CopyHostToDevice(mutable_data_ + position_, data, nbytes));
   }
   position_ += nbytes;
   return Status::OK();
@@ -169,11 +219,9 @@ Status CudaBufferWriter::SetBufferSize(const int64_t buffer_size) {
 // ----------------------------------------------------------------------
 
 Status AllocateCudaHostBuffer(const int64_t size, std::shared_ptr<CudaHostBuffer>* out) {
-  uint8_t* data = nullptr;
-  CUDA_RETURN_NOT_OK(
-      cudaMallocHost(reinterpret_cast<void**>(&data), static_cast<size_t>(size)));
-  *out = std::make_shared<CudaHostBuffer>(data, size);
-  return Status::OK();
+  CudaDeviceManager* manager = nullptr;
+  RETURN_NOT_OK(CudaDeviceManager::GetInstance(&manager));
+  return manager->AllocateHost(size, out);
 }
 
 }  // namespace gpu
diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h
index bd8b89a75edde..d5407371f35f5 100644
--- a/cpp/src/arrow/gpu/cuda_memory.h
+++ b/cpp/src/arrow/gpu/cuda_memory.h
@@ -29,17 +29,17 @@
 namespace arrow {
 namespace gpu {
 
+class CudaContext;
+class CudaIpcMemHandle;
+
 /// \class CudaBuffer
 /// \brief An Arrow buffer located on a GPU device
 ///
 /// Be careful using this in any Arrow code which may not be GPU-aware
 class ARROW_EXPORT CudaBuffer : public Buffer {
  public:
-  CudaBuffer(uint8_t* data, int64_t size, const int gpu_number, bool own_data = false)
-      : Buffer(data, size), gpu_number_(gpu_number), own_data_(own_data) {
-    is_mutable_ = true;
-    mutable_data_ = data;
-  }
+  CudaBuffer(uint8_t* data, int64_t size, const std::shared_ptr<CudaContext>& context,
+             bool own_data = false, bool is_ipc = false);
 
   CudaBuffer(const std::shared_ptr<CudaBuffer>& parent, const int64_t offset,
              const int64_t size);
@@ -58,11 +58,22 @@ class ARROW_EXPORT CudaBuffer : public Buffer {
   /// \return Status
   Status CopyFromHost(const int64_t position, const uint8_t* data, int64_t nbytes);
 
-  int gpu_number() const { return gpu_number_; }
+  /// \brief Expose this device buffer as IPC memory which can be used in other processes
+  /// \param[out] handle the exported IPC handle
+  /// \return Status
+  ///
+  /// \note After calling this function, this device memory will not be freed
+  /// when the CudaBuffer is destructed
+  virtual Status ExportForIpc(std::unique_ptr<CudaIpcMemHandle>* handle);
+
+  std::shared_ptr<CudaContext> context() const { return context_; }
 
- private:
-  const int gpu_number_;
+ protected:
+  std::shared_ptr<CudaContext> context_;
   bool own_data_;
+  bool is_ipc_;
+
+  virtual Status Close();
 };
 
 /// \class CudaHostBuffer
@@ -73,6 +84,37 @@ class ARROW_EXPORT CudaHostBuffer : public MutableBuffer {
   ~CudaHostBuffer();
 };
 
+/// \class CudaIpcHandle
+/// \brief A container for a CUDA IPC handle
+class ARROW_EXPORT CudaIpcMemHandle {
+ public:
+  ~CudaIpcMemHandle();
+
+  /// \brief Create CudaIpcMemHandle from opaque buffer (e.g. from another process)
+  /// \param[in] opaque_handle a CUipcMemHandle as a const void*
+  /// \param[out] handle the CudaIpcMemHandle instance
+  /// \return Status
+  static Status FromBuffer(const void* opaque_handle,
+                           std::unique_ptr<CudaIpcMemHandle>* handle);
+
+  /// \brief Write CudaIpcMemHandle to a Buffer
+  /// \param[in] pool a MemoryPool to allocate memory from
+  /// \param[out] out the serialized buffer
+  /// \return Status
+  Status Serialize(MemoryPool* pool, std::shared_ptr<Buffer>* out) const;
+
+ private:
+  explicit CudaIpcMemHandle(const void* handle);
+
+  struct CudaIpcMemHandleImpl;
+  std::unique_ptr<CudaIpcMemHandleImpl> impl_;
+
+  const void* handle() const;
+
+  friend CudaBuffer;
+  friend CudaContext;
+};
+
 /// \class CudaBufferReader
 /// \brief File interface for zero-copy read from CUDA buffers
 ///
@@ -98,7 +140,7 @@ class ARROW_EXPORT CudaBufferReader : public io::BufferReader {
 
  private:
   std::shared_ptr<CudaBuffer> cuda_buffer_;
-  int gpu_number_;
+  std::shared_ptr<CudaContext> context_;
 };
 
 /// \class CudaBufferWriter
@@ -132,7 +174,7 @@ class ARROW_EXPORT CudaBufferWriter : public io::FixedSizeBufferWriter {
   int64_t num_bytes_buffered() const { return buffer_position_; }
 
  private:
-  int gpu_number_;
+  std::shared_ptr<CudaContext> context_;
 
   // Pinned host buffer for buffering writes on CPU before calling cudaMalloc
   int64_t buffer_size_;
@@ -141,15 +183,6 @@ class ARROW_EXPORT CudaBufferWriter : public io::FixedSizeBufferWriter {
   uint8_t* host_buffer_data_;
 };
 
-/// \brief Allocate CUDA memory on a GPU device
-/// \param[in] gpu_number Device number to allocate
-/// \param[in] size number of bytes
-/// \param[out] out the allocated buffer
-/// \return Status
-ARROW_EXPORT
-Status AllocateCudaBuffer(const int gpu_number, const int64_t size,
-                          std::shared_ptr<CudaBuffer>* out);
-
 /// \brief Allocate CUDA-accessible memory on CPU host
 /// \param[in] size number of bytes
 /// \param[out] out the allocated buffer
diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc
index eb06aaf2fc651..53f0203f0807a 100644
--- a/cpp/src/arrow/ipc/message.cc
+++ b/cpp/src/arrow/ipc/message.cc
@@ -149,6 +149,24 @@ bool Message::Equals(const Message& other) const {
   }
 }
 
+Status Message::ReadFrom(const std::shared_ptr<Buffer>& metadata, io::InputStream* stream,
+                         std::unique_ptr<Message>* out) {
+  auto fb_message = flatbuf::GetMessage(metadata->data());
+
+  int64_t body_length = fb_message->bodyLength();
+
+  std::shared_ptr<Buffer> body;
+  RETURN_NOT_OK(stream->Read(body_length, &body));
+  if (body->size() < body_length) {
+    std::stringstream ss;
+    ss << "Expected to be able to read " << body_length << " bytes for message body, got "
+       << body->size();
+    return Status::IOError(ss.str());
+  }
+
+  return Message::Open(metadata, body, out);
+}
+
 Status Message::SerializeTo(io::OutputStream* file, int64_t* output_length) const {
   int32_t metadata_length = 0;
   RETURN_NOT_OK(WriteMessage(*metadata(), file, &metadata_length));
@@ -178,29 +196,6 @@ std::string FormatMessageType(Message::Type type) {
   return "unknown";
 }
 
-// ----------------------------------------------------------------------
-// Read and write messages
-
-static Status ReadFullMessage(const std::shared_ptr<Buffer>& metadata,
-                              io::InputStream* stream,
-                              std::unique_ptr<Message>* message) {
-  auto fb_message = flatbuf::GetMessage(metadata->data());
-
-  int64_t body_length = fb_message->bodyLength();
-
-  std::shared_ptr<Buffer> body;
-  RETURN_NOT_OK(stream->Read(body_length, &body));
-
-  if (body->size() < body_length) {
-    std::stringstream ss;
-    ss << "Expected to be able to read " << body_length << " bytes for message body, got "
-       << body->size();
-    return Status::IOError(ss.str());
-  }
-
-  return Message::Open(metadata, body, message);
-}
-
 Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
                    std::unique_ptr<Message>* message) {
   std::shared_ptr<Buffer> buffer;
@@ -216,32 +211,33 @@ Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile
   }
 
   auto metadata = SliceBuffer(buffer, 4, buffer->size() - 4);
-  return ReadFullMessage(metadata, file, message);
+  return Message::ReadFrom(metadata, file, message);
 }
 
 Status ReadMessage(io::InputStream* file, std::unique_ptr<Message>* message) {
-  std::shared_ptr<Buffer> buffer;
+  int32_t message_length = 0;
+  int64_t bytes_read = 0;
+  RETURN_NOT_OK(file->Read(sizeof(int32_t), &bytes_read,
+                           reinterpret_cast<uint8_t*>(&message_length)));
 
-  RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
-  if (buffer->size() != sizeof(int32_t)) {
+  if (bytes_read != sizeof(int32_t)) {
     *message = nullptr;
     return Status::OK();
   }
 
-  int32_t message_length = *reinterpret_cast<const int32_t*>(buffer->data());
-
   if (message_length == 0) {
     // Optional 0 EOS control message
     *message = nullptr;
     return Status::OK();
   }
 
-  RETURN_NOT_OK(file->Read(message_length, &buffer));
-  if (buffer->size() != message_length) {
+  std::shared_ptr<Buffer> metadata;
+  RETURN_NOT_OK(file->Read(message_length, &metadata));
+  if (metadata->size() != message_length) {
     return Status::IOError("Unexpected end of stream trying to read message");
   }
 
-  return ReadFullMessage(buffer, file, message);
+  return Message::ReadFrom(metadata, file, message);
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index dce4e27fb2d39..dbc50d8449051 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -79,6 +79,15 @@ class ARROW_EXPORT Message {
   static Status Open(const std::shared_ptr<Buffer>& metadata,
                      const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out);
 
+  /// \brief Read message body and create Message given Flatbuffer metadata
+  /// \param[in] metadata containing a serialized Message flatbuffer
+  /// \param[in] stream an InputStream
+  /// \param[out] out the created Message
+  ///
+  /// \note If stream supports zero-copy, this is zero-copy
+  static Status ReadFrom(const std::shared_ptr<Buffer>& metadata, io::InputStream* stream,
+                         std::unique_ptr<Message>* out);
+
   /// \brief Write length-prefixed metadata and body to output stream
   ///
   /// \param[in] file output stream to write to
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 9c05cba918d83..e17b974adfcad 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -901,14 +901,19 @@ Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
   RETURN_NOT_OK(AllocateBuffer(pool, size, &buffer));
 
   io::FixedSizeBufferWriter stream(buffer);
-  int32_t metadata_length = 0;
-  int64_t body_length = 0;
-  RETURN_NOT_OK(WriteRecordBatch(batch, 0, &stream, &metadata_length, &body_length, pool,
-                                 kMaxNestingDepth, true));
+  RETURN_NOT_OK(SerializeRecordBatch(batch, pool, &stream));
   *out = buffer;
   return Status::OK();
 }
 
+Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
+                            io::OutputStream* out) {
+  int32_t metadata_length = 0;
+  int64_t body_length = 0;
+  return WriteRecordBatch(batch, 0, out, &metadata_length, &body_length, pool,
+                          kMaxNestingDepth, true);
+}
+
 Status SerializeSchema(const Schema& schema, MemoryPool* pool,
                        std::shared_ptr<Buffer>* out) {
   std::shared_ptr<io::BufferOutputStream> stream;
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index d867982d2be02..3f110fe26fbdc 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -177,6 +177,18 @@ ARROW_EXPORT
 Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
                             std::shared_ptr<Buffer>* out);
 
+/// \brief Write record batch to OutputStream
+///
+/// \param[in] batch the record batch to write
+/// \param[in] out the OutputStream to write the output to
+/// \return Status
+///
+/// If writing to pre-allocated memory, you can use
+/// arrow::ipc::GetRecordBatchSize to compute how much space is required
+ARROW_EXPORT
+Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
+                            io::OutputStream* out);
+
 /// \brief Serialize schema using stream writer as a sequence of one or more
 /// IPC messages
 ///
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index aebef1b881297..515b600feec7f 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -386,7 +386,8 @@ cdef class PlasmaClient:
         -------
         The object ID associated to the Python object.
         """
-        cdef ObjectID target_id = object_id if object_id else ObjectID.from_random()
+        cdef ObjectID target_id = (object_id if object_id
+                                   else ObjectID.from_random())
         # TODO(pcm): Make serialization code support non-sequences and
         # get rid of packing the value into a list here (and unpacking in get)
         serialized = pyarrow.serialize([value])
@@ -404,8 +405,8 @@ cdef class PlasmaClient:
         Parameters
         ----------
         object_ids : list or ObjectID
-            Object ID or list of object IDs associated to the values we get from
-            the store.
+            Object ID or list of object IDs associated to the values we get
+            from the store.
         timeout_ms : int, default -1
             The number of milliseconds that the get call should block before
             timing out and returning. Pass -1 if the call should block and 0
@@ -415,14 +416,15 @@ cdef class PlasmaClient:
         -------
         list or object
             Python value or list of Python values for the data associated with
-            the object_ids and ObjectNotAvailable if the object was not available.
+            the object_ids and ObjectNotAvailable if the object was not
+            available.
         """
         if isinstance(object_ids, collections.Sequence):
             results = []
             buffers = self.get_buffers(object_ids, timeout_ms)
             for i in range(len(object_ids)):
-                # buffers[i] is None if this object was not available within the
-                # timeout
+                # buffers[i] is None if this object was not available within
+                # the timeout
                 if buffers[i]:
                     value, = pyarrow.deserialize(buffers[i])
                     results.append(value)
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index d729c1ef2d205..a831ef29a5e3c 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -187,8 +187,8 @@ def test_create(self):
         # Seal the object.
         self.plasma_client.seal(object_id)
         # Get the object.
-        memory_buffer = np.frombuffer(self.plasma_client.get_buffers([object_id])[0],
-                                      dtype="uint8")
+        memory_buffer = np.frombuffer(
+            self.plasma_client.get_buffers([object_id])[0], dtype="uint8")
         for i in range(length):
             assert memory_buffer[i] == i % 256
 
@@ -241,7 +241,8 @@ def test_get(self):
         # Test timing out of get with various timeouts.
         for timeout in [0, 10, 100, 1000]:
             object_ids = [random_object_id() for _ in range(num_object_ids)]
-            results = self.plasma_client.get_buffers(object_ids, timeout_ms=timeout)
+            results = self.plasma_client.get_buffers(object_ids,
+                                                     timeout_ms=timeout)
             assert results == num_object_ids * [None]
 
         data_buffers = []
@@ -257,7 +258,7 @@ def test_get(self):
         # timeouts.
         for timeout in [0, 10, 100, 1000]:
             data_results = self.plasma_client.get_buffers(object_ids,
-                                                         timeout_ms=timeout)
+                                                          timeout_ms=timeout)
             # metadata_results = self.plasma_client.get_metadata(
             #     object_ids, timeout_ms=timeout)
             for i in range(num_object_ids):
@@ -275,16 +276,16 @@ def test_get(self):
 
     def test_put_and_get(self):
         for value in [["hello", "world", 3, 1.0], None, "hello"]:
-             object_id = self.plasma_client.put(value)
-             [result] = self.plasma_client.get([object_id])
-             assert result == value
+            object_id = self.plasma_client.put(value)
+            [result] = self.plasma_client.get([object_id])
+            assert result == value
 
-             result = self.plasma_client.get(object_id)
-             assert result == value
+            result = self.plasma_client.get(object_id)
+            assert result == value
 
-             object_id = pa.plasma.ObjectID.from_random()
-             [result] = self.plasma_client.get([object_id], timeout_ms=0)
-             assert result == pa.plasma.ObjectNotAvailable
+            object_id = pa.plasma.ObjectID.from_random()
+            [result] = self.plasma_client.get([object_id], timeout_ms=0)
+            assert result == pa.plasma.ObjectNotAvailable
 
     def test_store_arrow_objects(self):
         data = np.random.randn(10, 4)

From c0beb879e3c4863a10b0d09aa981e8f53776458a Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 28 Aug 2017 20:15:38 -0400
Subject: [PATCH 0963/1644] ARROW-1415: [GLib] Support date32 and date64

Author: Kouhei Sutou <kou@clear-code.com>

Closes #997 from kou/glib-support-date32-and-date64 and squashes the following commits:

e3a7ca4f [Kouhei Sutou] [GLib] Fix a typo
b88345e1 [Kouhei Sutou] [GLib] Fix build error on macOS
c1186c37 [Kouhei Sutou] [GLib] Extract common code
8a337000 [Kouhei Sutou] [GLib] Support date64 type
a1c7b0dd [Kouhei Sutou] [GLib] Support date32 type
---
 c_glib/arrow-glib/array-builder.cpp        | 156 +++++++++
 c_glib/arrow-glib/array-builder.h          | 102 ++++++
 c_glib/arrow-glib/array.cpp                | 356 ++++++++++++++-------
 c_glib/arrow-glib/array.h                  | 106 ++++++
 c_glib/arrow-glib/data-type.cpp            |  80 +++++
 c_glib/arrow-glib/data-type.h              |  88 +++++
 c_glib/test/helper/buildable.rb            |   8 +
 c_glib/test/test-date32-array.rb           |  65 ++++
 c_glib/test/test-date32-data-type.rb       |  28 ++
 c_glib/test/test-date64-array-data-type.rb |  28 ++
 c_glib/test/test-date64-array.rb           |  65 ++++
 11 files changed, 972 insertions(+), 110 deletions(-)
 create mode 100644 c_glib/test/test-date32-array.rb
 create mode 100644 c_glib/test/test-date32-data-type.rb
 create mode 100644 c_glib/test/test-date64-array-data-type.rb
 create mode 100644 c_glib/test/test-date64-array.rb

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 1ef4e77e8fc50..12c50466c5562 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -107,6 +107,12 @@ G_BEGIN_DECLS
  * #GArrowStringArrayBuilder is the class to create a new
  * #GArrowStringArray.
  *
+ * #GArrowDate32ArrayBuilder is the class to create a new
+ * #GArrowDate32Array.
+ *
+ * #GArrowDate64ArrayBuilder is the class to create a new
+ * #GArrowDate64Array.
+ *
  * #GArrowListArrayBuilder is the class to create a new
  * #GArrowListArray.
  *
@@ -1158,6 +1164,150 @@ garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
 }
 
 
+G_DEFINE_TYPE(GArrowDate32ArrayBuilder,
+              garrow_date32_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_date32_array_builder_init(GArrowDate32ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_date32_array_builder_class_init(GArrowDate32ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_date32_array_builder_new:
+ *
+ * Returns: A newly created #GArrowDate32ArrayBuilder.
+ *
+ * Since: 0.7.0
+ */
+GArrowDate32ArrayBuilder *
+garrow_date32_array_builder_new(void)
+{
+  auto builder = garrow_array_builder_new(arrow::date32(),
+                                          NULL,
+                                          "[date32-array-builder][new]");
+  return GARROW_DATE32_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_date32_array_builder_append:
+ * @builder: A #GArrowDate32ArrayBuilder.
+ * @value: The number of days since UNIX epoch in signed 32bit integer.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_date32_array_builder_append(GArrowDate32ArrayBuilder *builder,
+                                   gint32 value,
+                                   GError **error)
+{
+  return garrow_array_builder_append<arrow::Date32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[date32-array-builder][append]");
+}
+
+/**
+ * garrow_date32_array_builder_append_null:
+ * @builder: A #GArrowDate32ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_date32_array_builder_append_null(GArrowDate32ArrayBuilder *builder,
+                                        GError **error)
+{
+  return garrow_array_builder_append_null<arrow::Date32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[date32-array-builder][append-null]");
+}
+
+
+G_DEFINE_TYPE(GArrowDate64ArrayBuilder,
+              garrow_date64_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_date64_array_builder_init(GArrowDate64ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_date64_array_builder_class_init(GArrowDate64ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_date64_array_builder_new:
+ *
+ * Returns: A newly created #GArrowDate64ArrayBuilder.
+ *
+ * Since: 0.7.0
+ */
+GArrowDate64ArrayBuilder *
+garrow_date64_array_builder_new(void)
+{
+  auto builder = garrow_array_builder_new(arrow::date64(),
+                                          NULL,
+                                          "[date64-array-builder][new]");
+  return GARROW_DATE64_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_date64_array_builder_append:
+ * @builder: A #GArrowDate64ArrayBuilder.
+ * @value: The number of milliseconds since UNIX epoch in signed 64bit integer.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_date64_array_builder_append(GArrowDate64ArrayBuilder *builder,
+                                   gint64 value,
+                                   GError **error)
+{
+  return garrow_array_builder_append<arrow::Date64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[date64-array-builder][append]");
+}
+
+/**
+ * garrow_date64_array_builder_append_null:
+ * @builder: A #GArrowDate64ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_date64_array_builder_append_null(GArrowDate64ArrayBuilder *builder,
+                                        GError **error)
+{
+  return garrow_array_builder_append_null<arrow::Date64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[date64-array-builder][append-null]");
+}
+
+
 typedef struct GArrowListArrayBuilderPrivate_ {
   GArrowArrayBuilder *value_builder;
 } GArrowListArrayBuilderPrivate;
@@ -1547,6 +1697,12 @@ garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder,
     case arrow::Type::type::STRING:
       type = GARROW_TYPE_STRING_ARRAY_BUILDER;
       break;
+    case arrow::Type::type::DATE32:
+      type = GARROW_TYPE_DATE32_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::DATE64:
+      type = GARROW_TYPE_DATE64_ARRAY_BUILDER;
+      break;
     case arrow::Type::type::LIST:
       type = GARROW_TYPE_LIST_ARRAY_BUILDER;
       break;
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index 613a5bad494d1..b6d79a8b4dd97 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -782,6 +782,108 @@ gboolean garrow_string_array_builder_append(GArrowStringArrayBuilder *builder,
                                             GError **error);
 
 
+#define GARROW_TYPE_DATE32_ARRAY_BUILDER        \
+  (garrow_date32_array_builder_get_type())
+#define GARROW_DATE32_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DATE32_ARRAY_BUILDER, \
+                              GArrowDate32ArrayBuilder))
+#define GARROW_DATE32_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DATE32_ARRAY_BUILDER,    \
+                           GArrowDate32ArrayBuilderClass))
+#define GARROW_IS_DATE32_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_DATE32_ARRAY_BUILDER))
+#define GARROW_IS_DATE32_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DATE32_ARRAY_BUILDER))
+#define GARROW_DATE32_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DATE32_ARRAY_BUILDER,  \
+                             GArrowDate32ArrayBuilderClass))
+
+typedef struct _GArrowDate32ArrayBuilder         GArrowDate32ArrayBuilder;
+typedef struct _GArrowDate32ArrayBuilderClass    GArrowDate32ArrayBuilderClass;
+
+/**
+ * GArrowDate32ArrayBuilder:
+ *
+ * It wraps `arrow::Date32Builder`.
+ */
+struct _GArrowDate32ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowDate32ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_date32_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowDate32ArrayBuilder *garrow_date32_array_builder_new(void);
+
+gboolean garrow_date32_array_builder_append(GArrowDate32ArrayBuilder *builder,
+                                            gint32 value,
+                                            GError **error);
+gboolean garrow_date32_array_builder_append_null(GArrowDate32ArrayBuilder *builder,
+                                                 GError **error);
+
+
+#define GARROW_TYPE_DATE64_ARRAY_BUILDER        \
+  (garrow_date64_array_builder_get_type())
+#define GARROW_DATE64_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DATE64_ARRAY_BUILDER, \
+                              GArrowDate64ArrayBuilder))
+#define GARROW_DATE64_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DATE64_ARRAY_BUILDER,    \
+                           GArrowDate64ArrayBuilderClass))
+#define GARROW_IS_DATE64_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_DATE64_ARRAY_BUILDER))
+#define GARROW_IS_DATE64_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DATE64_ARRAY_BUILDER))
+#define GARROW_DATE64_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DATE64_ARRAY_BUILDER,  \
+                             GArrowDate64ArrayBuilderClass))
+
+typedef struct _GArrowDate64ArrayBuilder         GArrowDate64ArrayBuilder;
+typedef struct _GArrowDate64ArrayBuilderClass    GArrowDate64ArrayBuilderClass;
+
+/**
+ * GArrowDate64ArrayBuilder:
+ *
+ * It wraps `arrow::Date64Builder`.
+ */
+struct _GArrowDate64ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowDate64ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_date64_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowDate64ArrayBuilder *garrow_date64_array_builder_new(void);
+
+gboolean garrow_date64_array_builder_append(GArrowDate64ArrayBuilder *builder,
+                                            gint64 value,
+                                            GError **error);
+gboolean garrow_date64_array_builder_append_null(GArrowDate64ArrayBuilder *builder,
+                                                 GError **error);
+
+
 #define GARROW_TYPE_LIST_ARRAY_BUILDER          \
   (garrow_list_array_builder_get_type())
 #define GARROW_LIST_ARRAY_BUILDER(obj)                          \
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index a3c45a890321c..d61790f44110e 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -41,6 +41,25 @@ garrow_array_get_values_raw(std::shared_ptr<arrow::Array> arrow_array,
   return arrow_specific_array->raw_values();
 };
 
+template <typename T>
+GArrowArray *
+garrow_primitive_array_new(gint64 length,
+                           GArrowBuffer *data,
+                           GArrowBuffer *null_bitmap,
+                           gint64 n_nulls)
+{
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_specific_array =
+    std::make_shared<typename arrow::TypeTraits<T>::ArrayType>(length,
+                                                               arrow_data,
+                                                               arrow_bitmap,
+                                                               n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_specific_array);
+  return garrow_array_new_raw(&arrow_array);
+};
+
 G_BEGIN_DECLS
 
 /**
@@ -127,6 +146,16 @@ G_BEGIN_DECLS
  * Arrow format data, you need to use #GArrowStringArrayBuilder to
  * create a new array.
  *
+ * #GArrowDate32Array is a class for the number of days since UNIX
+ * epoch in 32-bit signed integer array. It can store zero or more
+ * date data. If you don't have Arrow format data, you need to use
+ * #GArrowDate32ArrayBuilder to create a new array.
+ *
+ * #GArrowDate64Array is a class for the number of milliseconds since
+ * UNIX epoch in 64-bit signed integer array. It can store zero or
+ * more date data. If you don't have Arrow format data, you need to
+ * use #GArrowDate64ArrayBuilder to create a new array.
+ *
  * #GArrowListArray is a class for list array. It can store zero or
  * more list data. If you don't have Arrow format data, you need to
  * use #GArrowListArrayBuilder to create a new array.
@@ -530,16 +559,11 @@ garrow_boolean_array_new(gint64 length,
                          GArrowBuffer *null_bitmap,
                          gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_boolean_array =
-    std::make_shared<arrow::BooleanArray>(length,
-                                          arrow_data,
-                                          arrow_bitmap,
-                                          n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_boolean_array);
-  return GARROW_BOOLEAN_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::BooleanType>(length,
+                                                              data,
+                                                              null_bitmap,
+                                                              n_nulls);
+  return GARROW_BOOLEAN_ARRAY(array);
 }
 
 /**
@@ -617,16 +641,11 @@ garrow_int8_array_new(gint64 length,
                       GArrowBuffer *null_bitmap,
                       gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_int8_array =
-    std::make_shared<arrow::Int8Array>(length,
-                                       arrow_data,
-                                       arrow_bitmap,
-                                       n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_int8_array);
-  return GARROW_INT8_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::Int8Type>(length,
+                                                           data,
+                                                           null_bitmap,
+                                                           n_nulls);
+  return GARROW_INT8_ARRAY(array);
 }
 
 /**
@@ -695,16 +714,11 @@ garrow_uint8_array_new(gint64 length,
                        GArrowBuffer *null_bitmap,
                        gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_uint8_array =
-    std::make_shared<arrow::UInt8Array>(length,
-                                        arrow_data,
-                                        arrow_bitmap,
-                                        n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_uint8_array);
-  return GARROW_UINT8_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::UInt8Type>(length,
+                                                            data,
+                                                            null_bitmap,
+                                                            n_nulls);
+  return GARROW_UINT8_ARRAY(array);
 }
 
 /**
@@ -773,16 +787,11 @@ garrow_int16_array_new(gint64 length,
                        GArrowBuffer *null_bitmap,
                        gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_int16_array =
-    std::make_shared<arrow::Int16Array>(length,
-                                        arrow_data,
-                                        arrow_bitmap,
-                                        n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_int16_array);
-  return GARROW_INT16_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::Int16Type>(length,
+                                                            data,
+                                                            null_bitmap,
+                                                            n_nulls);
+  return GARROW_INT16_ARRAY(array);
 }
 
 /**
@@ -851,16 +860,11 @@ garrow_uint16_array_new(gint64 length,
                         GArrowBuffer *null_bitmap,
                         gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_uint16_array =
-    std::make_shared<arrow::UInt16Array>(length,
-                                         arrow_data,
-                                         arrow_bitmap,
-                                         n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_uint16_array);
-  return GARROW_UINT16_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::UInt16Type>(length,
+                                                             data,
+                                                             null_bitmap,
+                                                             n_nulls);
+  return GARROW_UINT16_ARRAY(array);
 }
 
 /**
@@ -929,16 +933,11 @@ garrow_int32_array_new(gint64 length,
                        GArrowBuffer *null_bitmap,
                        gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_int32_array =
-    std::make_shared<arrow::Int32Array>(length,
-                                        arrow_data,
-                                        arrow_bitmap,
-                                        n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_int32_array);
-  return GARROW_INT32_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::Int32Type>(length,
+                                                            data,
+                                                            null_bitmap,
+                                                            n_nulls);
+  return GARROW_INT32_ARRAY(array);
 }
 
 /**
@@ -1007,16 +1006,11 @@ garrow_uint32_array_new(gint64 length,
                         GArrowBuffer *null_bitmap,
                         gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_uint32_array =
-    std::make_shared<arrow::UInt32Array>(length,
-                                         arrow_data,
-                                         arrow_bitmap,
-                                         n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_uint32_array);
-  return GARROW_UINT32_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::UInt32Type>(length,
+                                                             data,
+                                                             null_bitmap,
+                                                             n_nulls);
+  return GARROW_UINT32_ARRAY(array);
 }
 
 /**
@@ -1085,16 +1079,11 @@ garrow_int64_array_new(gint64 length,
                        GArrowBuffer *null_bitmap,
                        gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_int64_array =
-    std::make_shared<arrow::Int64Array>(length,
-                                        arrow_data,
-                                        arrow_bitmap,
-                                        n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_int64_array);
-  return GARROW_INT64_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::Int64Type>(length,
+                                                            data,
+                                                            null_bitmap,
+                                                            n_nulls);
+  return GARROW_INT64_ARRAY(array);
 }
 
 /**
@@ -1165,16 +1154,11 @@ garrow_uint64_array_new(gint64 length,
                         GArrowBuffer *null_bitmap,
                         gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_uint64_array =
-    std::make_shared<arrow::UInt64Array>(length,
-                                         arrow_data,
-                                         arrow_bitmap,
-                                         n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_uint64_array);
-  return GARROW_UINT64_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::UInt64Type>(length,
+                                                             data,
+                                                             null_bitmap,
+                                                             n_nulls);
+  return GARROW_UINT64_ARRAY(array);
 }
 
 /**
@@ -1245,16 +1229,11 @@ garrow_float_array_new(gint64 length,
                        GArrowBuffer *null_bitmap,
                        gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_float_array =
-    std::make_shared<arrow::FloatArray>(length,
-                                        arrow_data,
-                                        arrow_bitmap,
-                                        n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_float_array);
-  return GARROW_FLOAT_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::FloatType>(length,
+                                                            data,
+                                                            null_bitmap,
+                                                            n_nulls);
+  return GARROW_FLOAT_ARRAY(array);
 }
 
 /**
@@ -1323,16 +1302,11 @@ garrow_double_array_new(gint64 length,
                         GArrowBuffer *null_bitmap,
                         gint64 n_nulls)
 {
-  const auto arrow_data = garrow_buffer_get_raw(data);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_double_array =
-    std::make_shared<arrow::DoubleArray>(length,
-                                         arrow_data,
-                                         arrow_bitmap,
-                                         n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_double_array);
-  return GARROW_DOUBLE_ARRAY(garrow_array_new_raw(&arrow_array));
+  auto array = garrow_primitive_array_new<arrow::DoubleType>(length,
+                                                             data,
+                                                             null_bitmap,
+                                                             n_nulls);
+  return GARROW_DOUBLE_ARRAY(array);
 }
 
 /**
@@ -1542,6 +1516,162 @@ garrow_string_array_get_string(GArrowStringArray *array,
 }
 
 
+G_DEFINE_TYPE(GArrowDate32Array,               \
+              garrow_date32_array,             \
+              GARROW_TYPE_PRIMITIVE_ARRAY)
+
+static void
+garrow_date32_array_init(GArrowDate32Array *object)
+{
+}
+
+static void
+garrow_date32_array_class_init(GArrowDate32ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_date32_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowDate32Array.
+ *
+ * Since: 0.7.0
+ */
+GArrowDate32Array *
+garrow_date32_array_new(gint64 length,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  auto array = garrow_primitive_array_new<arrow::Date32Type>(length,
+                                                             data,
+                                                             null_bitmap,
+                                                             n_nulls);
+  return GARROW_DATE32_ARRAY(array);
+}
+
+/**
+ * garrow_date32_array_get_value:
+ * @array: A #GArrowDate32Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ *
+ * Since: 0.7.0
+ */
+gint32
+garrow_date32_array_get_value(GArrowDate32Array *array,
+                              gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Date32Array *>(arrow_array.get())->Value(i);
+}
+
+/**
+ * garrow_date32_array_get_values:
+ * @array: A #GArrowDate32Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ *
+ * Since: 0.7.0
+ */
+const gint32 *
+garrow_date32_array_get_values(GArrowDate32Array *array,
+                               gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::Date32Type>(arrow_array, length);
+}
+
+
+G_DEFINE_TYPE(GArrowDate64Array,               \
+              garrow_date64_array,             \
+              GARROW_TYPE_PRIMITIVE_ARRAY)
+
+static void
+garrow_date64_array_init(GArrowDate64Array *object)
+{
+}
+
+static void
+garrow_date64_array_class_init(GArrowDate64ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_date64_array_new:
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowDate64Array.
+ *
+ * Since: 0.7.0
+ */
+GArrowDate64Array *
+garrow_date64_array_new(gint64 length,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  auto array = garrow_primitive_array_new<arrow::Date64Type>(length,
+                                                             data,
+                                                             null_bitmap,
+                                                             n_nulls);
+  return GARROW_DATE64_ARRAY(array);
+}
+
+/**
+ * garrow_date64_array_get_value:
+ * @array: A #GArrowDate64Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ *
+ * Since: 0.7.0
+ */
+gint64
+garrow_date64_array_get_value(GArrowDate64Array *array,
+                              gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Date64Array *>(arrow_array.get())->Value(i);
+}
+
+/**
+ * garrow_date64_array_get_values:
+ * @array: A #GArrowDate64Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ *
+ * Since: 0.7.0
+ */
+const gint64 *
+garrow_date64_array_get_values(GArrowDate64Array *array,
+                               gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto values =
+    garrow_array_get_values_raw<arrow::Date64Type>(arrow_array, length);
+  return reinterpret_cast<const gint64 *>(values);
+}
+
+
 G_DEFINE_TYPE(GArrowListArray,               \
               garrow_list_array,             \
               GARROW_TYPE_ARRAY)
@@ -1781,6 +1911,12 @@ garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array)
   case arrow::Type::type::STRING:
     type = GARROW_TYPE_STRING_ARRAY;
     break;
+  case arrow::Type::type::DATE32:
+    type = GARROW_TYPE_DATE32_ARRAY;
+    break;
+  case arrow::Type::type::DATE64:
+    type = GARROW_TYPE_DATE64_ARRAY;
+    break;
   case arrow::Type::type::LIST:
     type = GARROW_TYPE_LIST_ARRAY;
     break;
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 10b2279be4bbf..851c8eabf7dee 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -860,6 +860,112 @@ gchar *garrow_string_array_get_string(GArrowStringArray *array,
                                       gint64 i);
 
 
+#define GARROW_TYPE_DATE32_ARRAY                \
+  (garrow_date32_array_get_type())
+#define GARROW_DATE32_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DATE32_ARRAY, \
+                              GArrowDate32Array))
+#define GARROW_DATE32_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DATE32_ARRAY,    \
+                           GArrowDate32ArrayClass))
+#define GARROW_IS_DATE32_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DATE32_ARRAY))
+#define GARROW_IS_DATE32_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DATE32_ARRAY))
+#define GARROW_DATE32_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DATE32_ARRAY,  \
+                             GArrowDate32ArrayClass))
+
+typedef struct _GArrowDate32Array         GArrowDate32Array;
+typedef struct _GArrowDate32ArrayClass    GArrowDate32ArrayClass;
+
+/**
+ * GArrowDate32Array:
+ *
+ * It wraps `arrow::Date32Array`.
+ */
+struct _GArrowDate32Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowDate32ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_date32_array_get_type(void) G_GNUC_CONST;
+
+GArrowDate32Array *garrow_date32_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+gint32 garrow_date32_array_get_value(GArrowDate32Array *array,
+                                     gint64 i);
+const gint32 *garrow_date32_array_get_values(GArrowDate32Array *array,
+                                             gint64 *length);
+
+
+#define GARROW_TYPE_DATE64_ARRAY                \
+  (garrow_date64_array_get_type())
+#define GARROW_DATE64_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DATE64_ARRAY, \
+                              GArrowDate64Array))
+#define GARROW_DATE64_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DATE64_ARRAY,    \
+                           GArrowDate64ArrayClass))
+#define GARROW_IS_DATE64_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DATE64_ARRAY))
+#define GARROW_IS_DATE64_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DATE64_ARRAY))
+#define GARROW_DATE64_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DATE64_ARRAY,  \
+                             GArrowDate64ArrayClass))
+
+typedef struct _GArrowDate64Array         GArrowDate64Array;
+typedef struct _GArrowDate64ArrayClass    GArrowDate64ArrayClass;
+
+/**
+ * GArrowDate64Array:
+ *
+ * It wraps `arrow::Date64Array`.
+ */
+struct _GArrowDate64Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowDate64ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_date64_array_get_type(void) G_GNUC_CONST;
+
+GArrowDate64Array *garrow_date64_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+gint64 garrow_date64_array_get_value(GArrowDate64Array *array,
+                                     gint64 i);
+const gint64 *garrow_date64_array_get_values(GArrowDate64Array *array,
+                                             gint64 *length);
+
+
 #define GARROW_TYPE_LIST_ARRAY                  \
   (garrow_list_array_get_type())
 #define GARROW_LIST_ARRAY(obj)                          \
diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/data-type.cpp
index 9ce8c16e914e3..7cacf6a40492e 100644
--- a/c_glib/arrow-glib/data-type.cpp
+++ b/c_glib/arrow-glib/data-type.cpp
@@ -67,6 +67,12 @@ G_BEGIN_DECLS
  * #GArrowStringDataType is a class for UTF-8 encoded string data
  * type.
  *
+ * #GArrowData32DataType is a class for the number of days since UNIX
+ * epoch in 32-bit signed integer data type.
+ *
+ * #GArrowData64DataType is a class for the number of milliseconds
+ * since UNIX epoch in 64-bit signed integer data type.
+ *
  * #GArrowListDataType is a class for list data type.
  *
  * #GArrowStructDataType is a class for struct data type.
@@ -654,6 +660,74 @@ garrow_string_data_type_new(void)
 }
 
 
+G_DEFINE_TYPE(GArrowDate32DataType,                \
+              garrow_date32_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_date32_data_type_init(GArrowDate32DataType *object)
+{
+}
+
+static void
+garrow_date32_data_type_class_init(GArrowDate32DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_date32_data_type_new:
+ *
+ * Returns: The newly created 64-bit floating point data type.
+ *
+ * Since: 0.7.0
+ */
+GArrowDate32DataType *
+garrow_date32_data_type_new(void)
+{
+  auto arrow_data_type = arrow::date32();
+
+  GArrowDate32DataType *data_type =
+    GARROW_DATE32_DATA_TYPE(g_object_new(GARROW_TYPE_DATE32_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowDate64DataType,                \
+              garrow_date64_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_date64_data_type_init(GArrowDate64DataType *object)
+{
+}
+
+static void
+garrow_date64_data_type_class_init(GArrowDate64DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_date64_data_type_new:
+ *
+ * Returns: The newly created 64-bit floating point data type.
+ *
+ * Since: 0.7.0
+ */
+GArrowDate64DataType *
+garrow_date64_data_type_new(void)
+{
+  auto arrow_data_type = arrow::date64();
+
+  GArrowDate64DataType *data_type =
+    GARROW_DATE64_DATA_TYPE(g_object_new(GARROW_TYPE_DATE64_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
 G_DEFINE_TYPE(GArrowListDataType,                \
               garrow_list_data_type,             \
               GARROW_TYPE_DATA_TYPE)
@@ -798,6 +872,12 @@ garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type)
   case arrow::Type::type::STRING:
     type = GARROW_TYPE_STRING_DATA_TYPE;
     break;
+  case arrow::Type::type::DATE32:
+    type = GARROW_TYPE_DATE32_DATA_TYPE;
+    break;
+  case arrow::Type::type::DATE64:
+    type = GARROW_TYPE_DATE64_DATA_TYPE;
+    break;
   case arrow::Type::type::LIST:
     type = GARROW_TYPE_LIST_DATA_TYPE;
     break;
diff --git a/c_glib/arrow-glib/data-type.h b/c_glib/arrow-glib/data-type.h
index 933fcfc4b2ccb..5a0d2bc96026d 100644
--- a/c_glib/arrow-glib/data-type.h
+++ b/c_glib/arrow-glib/data-type.h
@@ -692,6 +692,94 @@ GType                 garrow_string_data_type_get_type (void) G_GNUC_CONST;
 GArrowStringDataType *garrow_string_data_type_new      (void);
 
 
+#define GARROW_TYPE_DATE32_DATA_TYPE           \
+  (garrow_date32_data_type_get_type())
+#define GARROW_DATE32_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DATE32_DATA_TYPE,     \
+                              GArrowDate32DataType))
+#define GARROW_DATE32_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DATE32_DATA_TYPE,        \
+                           GArrowDate32DataTypeClass))
+#define GARROW_IS_DATE32_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DATE32_DATA_TYPE))
+#define GARROW_IS_DATE32_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DATE32_DATA_TYPE))
+#define GARROW_DATE32_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DATE32_DATA_TYPE,      \
+                             GArrowDate32DataTypeClass))
+
+typedef struct _GArrowDate32DataType         GArrowDate32DataType;
+typedef struct _GArrowDate32DataTypeClass    GArrowDate32DataTypeClass;
+
+/**
+ * GArrowDate32DataType:
+ *
+ * It wraps `arrow::Date32Type`.
+ */
+struct _GArrowDate32DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowDate32DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_date32_data_type_get_type (void) G_GNUC_CONST;
+GArrowDate32DataType *garrow_date32_data_type_new      (void);
+
+
+#define GARROW_TYPE_DATE64_DATA_TYPE           \
+  (garrow_date64_data_type_get_type())
+#define GARROW_DATE64_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DATE64_DATA_TYPE,     \
+                              GArrowDate64DataType))
+#define GARROW_DATE64_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DATE64_DATA_TYPE,        \
+                           GArrowDate64DataTypeClass))
+#define GARROW_IS_DATE64_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DATE64_DATA_TYPE))
+#define GARROW_IS_DATE64_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DATE64_DATA_TYPE))
+#define GARROW_DATE64_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DATE64_DATA_TYPE,      \
+                             GArrowDate64DataTypeClass))
+
+typedef struct _GArrowDate64DataType         GArrowDate64DataType;
+typedef struct _GArrowDate64DataTypeClass    GArrowDate64DataTypeClass;
+
+/**
+ * GArrowDate64DataType:
+ *
+ * It wraps `arrow::Date64Type`.
+ */
+struct _GArrowDate64DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowDate64DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_date64_data_type_get_type (void) G_GNUC_CONST;
+GArrowDate64DataType *garrow_date64_data_type_new      (void);
+
+
 #define GARROW_TYPE_LIST_DATA_TYPE              \
   (garrow_list_data_type_get_type())
 #define GARROW_LIST_DATA_TYPE(obj)                              \
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index 3181c098c002b..d51503ec34679 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -65,6 +65,14 @@ def build_double_array(values)
       build_array(Arrow::DoubleArrayBuilder, values)
     end
 
+    def build_date32_array(values)
+      build_array(Arrow::Date32ArrayBuilder, values)
+    end
+
+    def build_date64_array(values)
+      build_array(Arrow::Date64ArrayBuilder, values)
+    end
+
     def build_binary_array(values)
       build_array(Arrow::BinaryArrayBuilder, values)
     end
diff --git a/c_glib/test/test-date32-array.rb b/c_glib/test/test-date32-array.rb
new file mode 100644
index 0000000000000..f1425693f381e
--- /dev/null
+++ b/c_glib/test/test-date32-array.rb
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDate32Array < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_new
+    after_epoch = 17406 # 2017-08-28
+    raw_data = [0, after_epoch]
+    assert_equal(build_date32_array([*raw_data, nil]),
+                 Arrow::Date32Array.new(3,
+                                        Arrow::Buffer.new(raw_data.pack("l*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
+  def test_buffer
+    before_epoch = -3653 # 1960-01-01
+    after_epoch = 17406 # 2017-08-28
+
+    builder = Arrow::Date32ArrayBuilder.new
+    builder.append(0)
+    builder.append(after_epoch)
+    builder.append(before_epoch)
+    array = builder.finish
+    assert_equal([0, after_epoch, before_epoch].pack("l*"),
+                 array.buffer.data.to_s)
+  end
+
+  def test_value
+    after_epoch = 17406 # 2017-08-28
+
+    builder = Arrow::Date32ArrayBuilder.new
+    builder.append(after_epoch)
+    array = builder.finish
+    assert_equal(after_epoch, array.get_value(0))
+  end
+
+  def test_values
+    before_epoch = -3653 # 1960-01-01
+    after_epoch = 17406 # 2017-08-28
+
+    builder = Arrow::Date32ArrayBuilder.new
+    builder.append(0)
+    builder.append(after_epoch)
+    builder.append(before_epoch)
+    array = builder.finish
+    assert_equal([0, after_epoch, before_epoch], array.values)
+  end
+end
diff --git a/c_glib/test/test-date32-data-type.rb b/c_glib/test/test-date32-data-type.rb
new file mode 100644
index 0000000000000..ebc8483cdef6b
--- /dev/null
+++ b/c_glib/test/test-date32-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDate32DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::Date32DataType.new
+    assert_equal(Arrow::Type::DATE32, data_type.id)
+  end
+
+  def test_to_s
+    data_type = Arrow::Date32DataType.new
+    assert_equal("date32[day]", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-date64-array-data-type.rb b/c_glib/test/test-date64-array-data-type.rb
new file mode 100644
index 0000000000000..7999cd82597b4
--- /dev/null
+++ b/c_glib/test/test-date64-array-data-type.rb
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDate64DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::Date64DataType.new
+    assert_equal(Arrow::Type::DATE64, data_type.id)
+  end
+
+  def test_to_s
+    data_type = Arrow::Date64DataType.new
+    assert_equal("date64[ms]", data_type.to_s)
+  end
+end
diff --git a/c_glib/test/test-date64-array.rb b/c_glib/test/test-date64-array.rb
new file mode 100644
index 0000000000000..1ea9f5a6a0545
--- /dev/null
+++ b/c_glib/test/test-date64-array.rb
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDate64Array < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_new
+    after_epoch = 1503878400000 # 2017-08-28T00:00:00Z
+    raw_data = [0, after_epoch]
+    assert_equal(build_date64_array([*raw_data, nil]),
+                 Arrow::Date64Array.new(3,
+                                        Arrow::Buffer.new(raw_data.pack("q*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
+  def test_buffer
+    before_epoch = -315619200 # 1960-01-01T00:00:00Z
+    after_epoch = 1503878400000 # 2017-08-28T00:00:00Z
+
+    builder = Arrow::Date64ArrayBuilder.new
+    builder.append(0)
+    builder.append(after_epoch)
+    builder.append(before_epoch)
+    array = builder.finish
+    assert_equal([0, after_epoch, before_epoch].pack("q*"),
+                 array.buffer.data.to_s)
+  end
+
+  def test_value
+    after_epoch = 1503878400000 # 2017-08-28T00:00:00Z
+
+    builder = Arrow::Date64ArrayBuilder.new
+    builder.append(after_epoch)
+    array = builder.finish
+    assert_equal(after_epoch, array.get_value(0))
+  end
+
+  def test_values
+    before_epoch = -315619200 # 1960-01-01T00:00:00Z
+    after_epoch = 1503878400000 # 2017-08-28T00:00:00Z
+
+    builder = Arrow::Date64ArrayBuilder.new
+    builder.append(0)
+    builder.append(after_epoch)
+    builder.append(before_epoch)
+    array = builder.finish
+    assert_equal([0, after_epoch, before_epoch], array.values)
+  end
+end

From 0ced74e1e39587c0ee10ac5979fefbaac97446f5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 29 Aug 2017 10:16:56 +0200
Subject: [PATCH 0964/1644] ARROW-1363: [C++] Use buffer layout from dictionary
 index type in IPC messages

This makes the vector layout consistent with what's being emitted by Java.

This layout isn't used in the C++ IPC reads, as the layout should be deterministic based on the schema types. My understanding is that it's there to enable logical type-agnostic manipulation of the physical memory layout.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1003 from wesm/ARROW-1363 and squashes the following commits:

6eef664 [Wes McKinney] Use buffer layout from index type for dictionary-encoded data
---
 cpp/src/arrow/ipc/metadata-internal.cc | 79 +++++++++++++-------------
 1 file changed, 39 insertions(+), 40 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index fdaab2176c1fa..928e889b79374 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -91,7 +91,7 @@ static Status FloatFromFlatuffer(const flatbuf::FloatingPoint* float_data,
 }
 
 // Forward declaration
-static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+static Status FieldToFlatbuffer(FBB& fbb, const Field& field,
                                 DictionaryMemo* dictionary_memo, FieldOffset* offset);
 
 static Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
@@ -102,18 +102,18 @@ static Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
   return flatbuf::CreateFloatingPoint(fbb, precision).Union();
 }
 
-static Status AppendChildFields(FBB& fbb, const std::shared_ptr<DataType>& type,
+static Status AppendChildFields(FBB& fbb, const DataType& type,
                                 std::vector<FieldOffset>* out_children,
                                 DictionaryMemo* dictionary_memo) {
   FieldOffset field;
-  for (int i = 0; i < type->num_children(); ++i) {
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), dictionary_memo, &field));
+  for (int i = 0; i < type.num_children(); ++i) {
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb, *type.child(i), dictionary_memo, &field));
     out_children->push_back(field);
   }
   return Status::OK();
 }
 
-static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+static Status ListToFlatbuffer(FBB& fbb, const DataType& type,
                                std::vector<FieldOffset>* out_children,
                                DictionaryMemo* dictionary_memo, Offset* offset) {
   RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
@@ -121,7 +121,7 @@ static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
   return Status::OK();
 }
 
-static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+static Status StructToFlatbuffer(FBB& fbb, const DataType& type,
                                  std::vector<FieldOffset>* out_children,
                                  DictionaryMemo* dictionary_memo, Offset* offset) {
   RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
@@ -156,12 +156,12 @@ static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
   return Status::OK();
 }
 
-static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+static Status UnionToFlatBuffer(FBB& fbb, const DataType& type,
                                 std::vector<FieldOffset>* out_children,
                                 DictionaryMemo* dictionary_memo, Offset* offset) {
   RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
 
-  const auto& union_type = static_cast<const UnionType&>(*type);
+  const auto& union_type = static_cast<const UnionType&>(type);
 
   flatbuf::UnionMode mode = union_type.mode() == UnionMode::SPARSE
                                 ? flatbuf::UnionMode_Sparse
@@ -304,21 +304,12 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 }
 
 // TODO(wesm): Convert this to visitor pattern
-static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+static Status TypeToFlatbuffer(FBB& fbb, const DataType& type,
                                std::vector<FieldOffset>* children,
                                std::vector<VectorLayoutOffset>* layout,
                                flatbuf::Type* out_type, DictionaryMemo* dictionary_memo,
                                Offset* offset) {
-  if (type->id() == Type::DICTIONARY) {
-    // In this library, the dictionary "type" is a logical construct. Here we
-    // pass through to the value type, as we've already captured the index
-    // type in the DictionaryEncoding metadata in the parent field
-    const auto& dict_type = static_cast<const DictionaryType&>(*type);
-    return TypeToFlatbuffer(fbb, dict_type.dictionary()->type(), children, layout,
-                            out_type, dictionary_memo, offset);
-  }
-
-  std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
+  std::vector<BufferDescr> buffer_layout = type.GetBufferLayout();
   for (const BufferDescr& descr : buffer_layout) {
     flatbuf::VectorType vector_type;
     switch (descr.type()) {
@@ -343,7 +334,16 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     layout->push_back(offset);
   }
 
-  switch (type->id()) {
+  const DataType* value_type = &type;
+
+  if (type.id() == Type::DICTIONARY) {
+    // In this library, the dictionary "type" is a logical construct. Here we
+    // pass through to the value type, as we've already captured the index
+    // type in the DictionaryEncoding metadata in the parent field
+    value_type = static_cast<const DictionaryType&>(type).dictionary()->type().get();
+  }
+
+  switch (value_type->id()) {
     case Type::BOOL:
       *out_type = flatbuf::Type_Bool;
       *offset = flatbuf::CreateBool(fbb).Union();
@@ -373,7 +373,7 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
       break;
     case Type::FIXED_SIZE_BINARY: {
-      const auto& fw_type = static_cast<const FixedSizeBinaryType&>(*type);
+      const auto& fw_type = static_cast<const FixedSizeBinaryType&>(*value_type);
       *out_type = flatbuf::Type_FixedSizeBinary;
       *offset = flatbuf::CreateFixedSizeBinary(fbb, fw_type.byte_width()).Union();
     } break;
@@ -394,17 +394,17 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
       *offset = flatbuf::CreateDate(fbb, flatbuf::DateUnit_MILLISECOND).Union();
       break;
     case Type::TIME32: {
-      const auto& time_type = static_cast<const Time32Type&>(*type);
+      const auto& time_type = static_cast<const Time32Type&>(*value_type);
       *out_type = flatbuf::Type_Time;
       *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit()), 32).Union();
     } break;
     case Type::TIME64: {
-      const auto& time_type = static_cast<const Time64Type&>(*type);
+      const auto& time_type = static_cast<const Time64Type&>(*value_type);
       *out_type = flatbuf::Type_Time;
       *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit()), 64).Union();
     } break;
     case Type::TIMESTAMP: {
-      const auto& ts_type = static_cast<const TimestampType&>(*type);
+      const auto& ts_type = static_cast<const TimestampType&>(*value_type);
       *out_type = flatbuf::Type_Timestamp;
 
       flatbuf::TimeUnit fb_unit = ToFlatbufferUnit(ts_type.unit());
@@ -416,25 +416,25 @@ static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
     } break;
     case Type::LIST:
       *out_type = flatbuf::Type_List;
-      return ListToFlatbuffer(fbb, type, children, dictionary_memo, offset);
+      return ListToFlatbuffer(fbb, *value_type, children, dictionary_memo, offset);
     case Type::STRUCT:
       *out_type = flatbuf::Type_Struct_;
-      return StructToFlatbuffer(fbb, type, children, dictionary_memo, offset);
+      return StructToFlatbuffer(fbb, *value_type, children, dictionary_memo, offset);
     case Type::UNION:
       *out_type = flatbuf::Type_Union;
-      return UnionToFlatBuffer(fbb, type, children, dictionary_memo, offset);
+      return UnionToFlatBuffer(fbb, *value_type, children, dictionary_memo, offset);
     default:
       *out_type = flatbuf::Type_NONE;  // Make clang-tidy happy
       std::stringstream ss;
-      ss << "Unable to convert type: " << type->ToString() << std::endl;
+      ss << "Unable to convert type: " << type.ToString() << std::endl;
       return Status::NotImplemented(ss.str());
   }
   return Status::OK();
 }
 
-static Status TensorTypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+static Status TensorTypeToFlatbuffer(FBB& fbb, const DataType& type,
                                      flatbuf::Type* out_type, Offset* offset) {
-  switch (type->id()) {
+  switch (type.id()) {
     case Type::UINT8:
       INT_TO_FB_CASE(8, false);
     case Type::INT8:
@@ -466,7 +466,7 @@ static Status TensorTypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>&
     default:
       *out_type = flatbuf::Type_NONE;  // Make clang-tidy happy
       std::stringstream ss;
-      ss << "Unable to convert type: " << type->ToString() << std::endl;
+      ss << "Unable to convert type: " << type.ToString() << std::endl;
       return Status::NotImplemented(ss.str());
   }
   return Status::OK();
@@ -488,9 +488,9 @@ static DictionaryOffset GetDictionaryEncoding(FBB& fbb, const DictionaryType& ty
                                            type.ordered());
 }
 
-static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+static Status FieldToFlatbuffer(FBB& fbb, const Field& field,
                                 DictionaryMemo* dictionary_memo, FieldOffset* offset) {
-  auto fb_name = fbb.CreateString(field->name());
+  auto fb_name = fbb.CreateString(field.name());
 
   flatbuf::Type type_enum;
   Offset type_offset;
@@ -498,19 +498,19 @@ static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
   std::vector<FieldOffset> children;
   std::vector<VectorLayoutOffset> layout;
 
-  RETURN_NOT_OK(TypeToFlatbuffer(fbb, field->type(), &children, &layout, &type_enum,
+  RETURN_NOT_OK(TypeToFlatbuffer(fbb, *field.type(), &children, &layout, &type_enum,
                                  dictionary_memo, &type_offset));
   auto fb_children = fbb.CreateVector(children);
   auto fb_layout = fbb.CreateVector(layout);
 
   DictionaryOffset dictionary = 0;
-  if (field->type()->id() == Type::DICTIONARY) {
+  if (field.type()->id() == Type::DICTIONARY) {
     dictionary = GetDictionaryEncoding(
-        fbb, static_cast<const DictionaryType&>(*field->type()), dictionary_memo);
+        fbb, static_cast<const DictionaryType&>(*field.type()), dictionary_memo);
   }
 
   // TODO: produce the list of VectorTypes
-  *offset = flatbuf::CreateField(fbb, fb_name, field->nullable(), type_enum, type_offset,
+  *offset = flatbuf::CreateField(fbb, fb_name, field.nullable(), type_enum, type_offset,
                                  dictionary, fb_children, fb_layout);
 
   return Status::OK();
@@ -588,9 +588,8 @@ static Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
   /// Fields
   std::vector<FieldOffset> field_offsets;
   for (int i = 0; i < schema.num_fields(); ++i) {
-    std::shared_ptr<Field> field = schema.field(i);
     FieldOffset offset;
-    RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, dictionary_memo, &offset));
+    RETURN_NOT_OK(FieldToFlatbuffer(fbb, *schema.field(i), dictionary_memo, &offset));
     field_offsets.push_back(offset);
   }
 
@@ -714,7 +713,7 @@ Status WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset,
 
   flatbuf::Type fb_type_type;
   Offset fb_type;
-  RETURN_NOT_OK(TensorTypeToFlatbuffer(fbb, tensor.type(), &fb_type_type, &fb_type));
+  RETURN_NOT_OK(TensorTypeToFlatbuffer(fbb, *tensor.type(), &fb_type_type, &fb_type));
 
   std::vector<TensorDimOffset> dims;
   for (int i = 0; i < tensor.ndim(); ++i) {

From e6456b6b35d03a647cb69b15c043efeded29318d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 29 Aug 2017 10:18:13 +0200
Subject: [PATCH 0965/1644] ARROW-1344: [C++] Do not permit writing to closed
 BufferOutputStream

Since this API code path in Python, it would be better to not segfault when writing to a BufferOutputStream whose internal buffer has been closed.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1004 from wesm/ARROW-1344 and squashes the following commits:

2ee0454 [Wes McKinney] Do not shrink to fit in BufferOutputStream
02c7c28 [Wes McKinney] Do not permit writing to closed BufferOutputStream
---
 cpp/src/arrow/io/io-memory-test.cc | 12 ++++++++++++
 cpp/src/arrow/io/memory.cc         |  7 ++++++-
 cpp/src/arrow/io/memory.h          |  1 +
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 75365df6b189d..0d45c63b82bed 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -67,6 +67,18 @@ TEST_F(TestBufferOutputStream, CloseResizes) {
   ASSERT_EQ(static_cast<int64_t>(K * data.size()), buffer_->size());
 }
 
+TEST_F(TestBufferOutputStream, WriteAfterFinish) {
+  std::string data = "data123456";
+  ASSERT_OK(stream_->Write(data));
+
+  auto buffer_stream = static_cast<BufferOutputStream*>(stream_.get());
+
+  std::shared_ptr<Buffer> buffer;
+  ASSERT_OK(buffer_stream->Finish(&buffer));
+
+  ASSERT_RAISES(IOError, stream_->Write(data));
+}
+
 TEST(TestFixedSizeBufferWriter, Basics) {
   std::shared_ptr<MutableBuffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), 1024, &buffer));
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 532c2c64a4588..568d7b2294e20 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -41,6 +41,7 @@ static constexpr int64_t kBufferMinimumSize = 256;
 
 BufferOutputStream::BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer)
     : buffer_(buffer),
+      is_open_(true),
       capacity_(buffer->size()),
       position_(0),
       mutable_data_(buffer->mutable_data()) {}
@@ -62,7 +63,7 @@ BufferOutputStream::~BufferOutputStream() {
 
 Status BufferOutputStream::Close() {
   if (position_ < capacity_) {
-    return buffer_->Resize(position_);
+    return buffer_->Resize(position_, false);
   } else {
     return Status::OK();
   }
@@ -72,6 +73,7 @@ Status BufferOutputStream::Finish(std::shared_ptr<Buffer>* result) {
   RETURN_NOT_OK(Close());
   *result = buffer_;
   buffer_ = nullptr;
+  is_open_ = false;
   return Status::OK();
 }
 
@@ -81,6 +83,9 @@ Status BufferOutputStream::Tell(int64_t* position) const {
 }
 
 Status BufferOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+  if (ARROW_PREDICT_FALSE(!is_open_)) {
+    return Status::IOError("OutputStream is closed");
+  }
   DCHECK(buffer_);
   RETURN_NOT_OK(Reserve(nbytes));
   memcpy(mutable_data_ + position_, data, nbytes);
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 667a7f087f1db..75bc3d0820ba4 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -62,6 +62,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   Status Reserve(int64_t nbytes);
 
   std::shared_ptr<ResizableBuffer> buffer_;
+  bool is_open_;
   int64_t capacity_;
   int64_t position_;
   uint8_t* mutable_data_;

From a009aab0c956c8b6e1e451e2bc62857cb3912a1a Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Tue, 29 Aug 2017 10:22:48 +0200
Subject: [PATCH 0966/1644] ARROW-1421: [Python] Extend Python serialization
 API to accept non-list types

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1005 from pcmoritz/serialize-objects and squashes the following commits:

b194df3 [Philipp Moritz] fix comments
5d34215 [Philipp Moritz] do the wrapping in cython
91d5d38 [Philipp Moritz] allow serialization of arbitrary python objects
---
 python/pyarrow/plasma.pyx                  | 7 ++-----
 python/pyarrow/serialization.pxi           | 8 +++++---
 python/pyarrow/tests/test_serialization.py | 6 +++---
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index 515b600feec7f..b72e49b76eef8 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -388,9 +388,7 @@ cdef class PlasmaClient:
         """
         cdef ObjectID target_id = (object_id if object_id
                                    else ObjectID.from_random())
-        # TODO(pcm): Make serialization code support non-sequences and
-        # get rid of packing the value into a list here (and unpacking in get)
-        serialized = pyarrow.serialize([value])
+        serialized = pyarrow.serialize(value)
         buffer = self.create(target_id, serialized.total_bytes)
         stream = pyarrow.FixedSizeBufferOutputStream(buffer)
         stream.set_memcopy_threads(4)
@@ -426,8 +424,7 @@ cdef class PlasmaClient:
                 # buffers[i] is None if this object was not available within
                 # the timeout
                 if buffers[i]:
-                    value, = pyarrow.deserialize(buffers[i])
-                    results.append(value)
+                    results.append(pyarrow.deserialize(buffers[i]))
                 else:
                     results.append(ObjectNotAvailable)
             return results
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index 3ee34eed5ed95..062121ff39842 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -172,8 +172,9 @@ cdef class SerializedPyObject:
             check_status(DeserializeObject(self.data, <PyObject*> self.base,
                                            &result))
 
-        # This is necessary to avoid a memory leak
-        return PyObject_to_object(result)
+        # PyObject_to_object is necessary to avoid a memory leak;
+        # also unpack the list the object was wrapped in in serialize
+        return PyObject_to_object(result)[0]
 
     def to_buffer(self):
         """
@@ -197,8 +198,9 @@ def serialize(object value):
     serialized : SerializedPyObject
     """
     cdef SerializedPyObject serialized = SerializedPyObject()
+    wrapped_value = [value]
     with nogil:
-        check_status(SerializeObject(value, &serialized.data))
+        check_status(SerializeObject(wrapped_value, &serialized.data))
     return serialized
 
 
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 12bf65be69c6e..b2aa4af3616a0 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -221,16 +221,16 @@ def large_memory_map(tmpdir_factory):
 def test_primitive_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for obj in PRIMITIVE_OBJECTS:
-            serialization_roundtrip([obj], mmap)
+            serialization_roundtrip(obj, mmap)
 
 
 def test_complex_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for obj in COMPLEX_OBJECTS:
-            serialization_roundtrip([obj], mmap)
+            serialization_roundtrip(obj, mmap)
 
 
 def test_custom_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for obj in CUSTOM_OBJECTS:
-            serialization_roundtrip([obj], mmap)
+            serialization_roundtrip(obj, mmap)

From 5cda6934999f9f79368f3fc3f68895fc0f4e0b24 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 29 Aug 2017 10:26:30 +0200
Subject: [PATCH 0967/1644] ARROW-1402: [C++] Deprecate APIs which return
 std::shared_ptr<MutableBuffer> in favor of std::shared_ptr<Buffer>

My view is that exposing this detail serves only to complicate the user API, since instances of the base class are permitted to be mutable (when `buffer->mutable_data()` is not null).

These APIs are used in parquet-cpp, but the build here should pass if the deprecations are handled properly. I will submit a patch there to fix these -- that patch can go in after parquet-cpp 1.3.0 to avoid any disruptions (so parquet-cpp will work fine with either Arrow 0.6.0 or 0.7.0).

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1001 from wesm/ARROW-1402 and squashes the following commits:

c8caae7 [Wes McKinney] Deprecate APIs which return std::shared_ptr<MutableBuffer> in favor of std::shared_ptr<Buffer>
---
 cpp/src/arrow/array-test.cc              |  2 +-
 cpp/src/arrow/buffer-test.cc             |  2 +-
 cpp/src/arrow/buffer.cc                  | 10 ++++++++-
 cpp/src/arrow/buffer.h                   | 14 ++++++++----
 cpp/src/arrow/gpu/cuda-test.cc           |  6 +++---
 cpp/src/arrow/gpu/cuda_arrow_ipc.cc      |  2 +-
 cpp/src/arrow/gpu/cuda_memory.cc         |  2 +-
 cpp/src/arrow/io/io-hdfs-test.cc         |  4 ++--
 cpp/src/arrow/io/io-memory-test.cc       |  4 ++--
 cpp/src/arrow/io/memory.h                |  8 +++----
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  2 +-
 cpp/src/arrow/ipc/json-internal.cc       |  6 +++---
 cpp/src/arrow/ipc/writer.cc              |  6 +++---
 cpp/src/arrow/tensor-test.cc             |  8 +++----
 cpp/src/arrow/tensor.h                   |  1 -
 cpp/src/arrow/test-util.h                |  2 +-
 cpp/src/arrow/util/bit-util-test.cc      |  2 +-
 cpp/src/arrow/util/bit-util.cc           | 15 +++++++++----
 cpp/src/arrow/util/bit-util.h            | 27 ++++++++++++++++--------
 19 files changed, 76 insertions(+), 47 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 38aceb2d185bb..abfd82656a36c 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -128,7 +128,7 @@ TEST_F(TestArray, SliceRecomputeNullCount) {
   ASSERT_EQ(5, slice->null_count());
 
   // No bitmap, compute 0
-  std::shared_ptr<MutableBuffer> data;
+  std::shared_ptr<Buffer> data;
   const int kBufferSize = 64;
   ASSERT_OK(AllocateBuffer(pool_, kBufferSize, &data));
   memset(data->mutable_data(), 0, kBufferSize);
diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index 5815ed17af50e..30c46725c7723 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -172,7 +172,7 @@ TEST_F(TestBuffer, SliceMutableBuffer) {
   std::string data_str = "some data to slice";
   auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), 50, &buffer));
 
   memcpy(buffer->mutable_data(), data, data_str.size());
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 9e8eb8bd4b6bc..d60c8295cf9a9 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -126,13 +126,21 @@ MutableBuffer::MutableBuffer(const std::shared_ptr<Buffer>& parent, const int64_
 }
 
 Status AllocateBuffer(MemoryPool* pool, const int64_t size,
-                      std::shared_ptr<MutableBuffer>* out) {
+                      std::shared_ptr<Buffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(buffer->Resize(size));
   *out = buffer;
   return Status::OK();
 }
 
+Status AllocateBuffer(MemoryPool* pool, const int64_t size,
+                      std::shared_ptr<MutableBuffer>* out) {
+  std::shared_ptr<Buffer> buffer;
+  RETURN_NOT_OK(AllocateBuffer(pool, size, &buffer));
+  *out = std::dynamic_pointer_cast<MutableBuffer>(buffer);
+  return Status::OK();
+}
+
 Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
                                std::shared_ptr<ResizableBuffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index f8f0b3df47501..859fa80072582 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -296,16 +296,15 @@ class ARROW_EXPORT TypedBufferBuilder : public BufferBuilder {
   int64_t length() const { return size_ / sizeof(T); }
 };
 
-/// Allocate a new mutable buffer from a memory pool
+/// \brief Allocate a fixed size mutable buffer from a memory pool
 ///
 /// \param[in] pool a memory pool
 /// \param[in] size size of buffer to allocate
-/// \param[out] out the allocated buffer with padding
+/// \param[out] out the allocated buffer (contains padding)
 ///
 /// \return Status message
 ARROW_EXPORT
-Status AllocateBuffer(MemoryPool* pool, const int64_t size,
-                      std::shared_ptr<MutableBuffer>* out);
+Status AllocateBuffer(MemoryPool* pool, const int64_t size, std::shared_ptr<Buffer>* out);
 
 /// Allocate resizeable buffer from a memory pool
 ///
@@ -318,6 +317,13 @@ ARROW_EXPORT
 Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
                                std::shared_ptr<ResizableBuffer>* out);
 
+#ifndef ARROW_NO_DEPRECATED_API
+/// \deprecated Since 0.7.0
+ARROW_EXPORT
+Status AllocateBuffer(MemoryPool* pool, const int64_t size,
+                      std::shared_ptr<MutableBuffer>* out);
+#endif
+
 }  // namespace arrow
 
 #endif  // ARROW_BUFFER_H
diff --git a/cpp/src/arrow/gpu/cuda-test.cc b/cpp/src/arrow/gpu/cuda-test.cc
index aa9d3efd2ac9b..afdc3020e8581 100644
--- a/cpp/src/arrow/gpu/cuda-test.cc
+++ b/cpp/src/arrow/gpu/cuda-test.cc
@@ -59,7 +59,7 @@ TEST_F(TestCudaBuffer, Allocate) {
 
 void AssertCudaBufferEquals(const CudaBuffer& buffer, const uint8_t* host_data,
                             const int64_t nbytes) {
-  std::shared_ptr<MutableBuffer> result;
+  std::shared_ptr<Buffer> result;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), nbytes, &result));
   ASSERT_OK(buffer.CopyToHost(0, buffer.size(), result->mutable_data()));
   ASSERT_EQ(0, std::memcmp(result->data(), host_data, nbytes));
@@ -108,7 +108,7 @@ TEST_F(TestCudaBuffer, DISABLED_ExportForIpc) {
 
   ASSERT_EQ(kSize, ipc_buffer->size());
 
-  std::shared_ptr<MutableBuffer> ipc_data;
+  std::shared_ptr<Buffer> ipc_data;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), kSize, &ipc_data));
   ASSERT_OK(ipc_buffer->CopyToHost(0, kSize, ipc_data->mutable_data()));
   ASSERT_EQ(0, std::memcmp(ipc_buffer->data(), host_buffer->data(), kSize));
@@ -288,7 +288,7 @@ TEST_F(TestCudaArrowIpc, BasicWriteRead) {
                             &device_batch));
 
   // Copy data from device, read batch, and compare
-  std::shared_ptr<MutableBuffer> host_buffer;
+  std::shared_ptr<Buffer> host_buffer;
   int64_t size = device_serialized->size();
   ASSERT_OK(AllocateBuffer(pool_, size, &host_buffer));
   ASSERT_OK(device_serialized->CopyToHost(0, size, host_buffer->mutable_data()));
diff --git a/cpp/src/arrow/gpu/cuda_arrow_ipc.cc b/cpp/src/arrow/gpu/cuda_arrow_ipc.cc
index 669857d920336..022268e034758 100644
--- a/cpp/src/arrow/gpu/cuda_arrow_ipc.cc
+++ b/cpp/src/arrow/gpu/cuda_arrow_ipc.cc
@@ -78,7 +78,7 @@ Status ReadMessage(CudaBufferReader* reader, MemoryPool* pool,
     return Status::OK();
   }
 
-  std::shared_ptr<MutableBuffer> metadata;
+  std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(AllocateBuffer(pool, message_length, &metadata));
   RETURN_NOT_OK(reader->Read(message_length, &bytes_read, metadata->mutable_data()));
   if (bytes_read != message_length) {
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
index 3c88fe2d59fbc..d1026ca51abae 100644
--- a/cpp/src/arrow/gpu/cuda_memory.cc
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -59,7 +59,7 @@ Status CudaIpcMemHandle::FromBuffer(const void* opaque_handle,
 }
 
 Status CudaIpcMemHandle::Serialize(MemoryPool* pool, std::shared_ptr<Buffer>* out) const {
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   constexpr size_t kHandleSize = sizeof(CUipcMemHandle);
   RETURN_NOT_OK(AllocateBuffer(pool, static_cast<int64_t>(kHandleSize), &buffer));
   memcpy(buffer->mutable_data(), &impl_->ipc_handle, kHandleSize);
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index b6a40e094c9cf..b88ca8f26d42c 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -353,7 +353,7 @@ TYPED_TEST(TestHadoopFileSystem, LargeFile) {
   std::shared_ptr<HdfsReadableFile> file;
   ASSERT_OK(this->client_->OpenReadable(path, &file));
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   ASSERT_OK(AllocateBuffer(nullptr, size, &buffer));
 
   int64_t bytes_read = 0;
@@ -366,7 +366,7 @@ TYPED_TEST(TestHadoopFileSystem, LargeFile) {
   std::shared_ptr<HdfsReadableFile> file2;
   ASSERT_OK(this->client_->OpenReadable(path, 1 << 18, &file2));
 
-  std::shared_ptr<MutableBuffer> buffer2;
+  std::shared_ptr<Buffer> buffer2;
   ASSERT_OK(AllocateBuffer(nullptr, size, &buffer2));
 
   ASSERT_OK(file2->Read(size, &bytes_read, buffer2->mutable_data()));
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 0d45c63b82bed..fadcfc3468829 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -80,7 +80,7 @@ TEST_F(TestBufferOutputStream, WriteAfterFinish) {
 }
 
 TEST(TestFixedSizeBufferWriter, Basics) {
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), 1024, &buffer));
 
   FixedSizeBufferWriter writer(buffer);
@@ -113,7 +113,7 @@ TEST(TestBufferReader, RetainParentReference) {
   std::shared_ptr<Buffer> slice1;
   std::shared_ptr<Buffer> slice2;
   {
-    std::shared_ptr<MutableBuffer> buffer;
+    std::shared_ptr<Buffer> buffer;
     ASSERT_OK(AllocateBuffer(nullptr, static_cast<int64_t>(data.size()), &buffer));
     std::memcpy(buffer->mutable_data(), data.c_str(), data.size());
     BufferReader reader(buffer);
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 75bc3d0820ba4..ab8d103b58a39 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -38,8 +38,7 @@ class Status;
 
 namespace io {
 
-// An output stream that writes to a MutableBuffer, such as one obtained from a
-// memory map
+// \brief An output stream that writes to a resizable buffer
 class ARROW_EXPORT BufferOutputStream : public OutputStream {
  public:
   explicit BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer);
@@ -68,7 +67,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   uint8_t* mutable_data_;
 };
 
-// A helper class to tracks the size of allocations
+// \brief A helper class to tracks the size of allocations
 class ARROW_EXPORT MockOutputStream : public OutputStream {
  public:
   MockOutputStream() : extent_bytes_written_(0) {}
@@ -85,7 +84,6 @@ class ARROW_EXPORT MockOutputStream : public OutputStream {
 };
 
 /// \brief Enables random writes into a fixed-size mutable buffer
-///
 class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
  public:
   /// Input buffer must be mutable, will abort if not
@@ -114,6 +112,8 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
   int64_t memcopy_threshold_;
 };
 
+/// \class BufferReader
+/// \brief Random access zero-copy reads on an arrow::Buffer
 class ARROW_EXPORT BufferReader : public RandomAccessFile {
  public:
   explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index e7b0a34547754..a0005b591b596 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -273,7 +273,7 @@ TEST_P(TestIpcRoundTrip, ZeroLengthArrays) {
   CheckRoundtrip(*zero_length_batch, 1 << 20);
 
   // ARROW-544: check binary array
-  std::shared_ptr<MutableBuffer> value_offsets;
+  std::shared_ptr<Buffer> value_offsets;
   ASSERT_OK(AllocateBuffer(pool_, sizeof(int32_t), &value_offsets));
   *reinterpret_cast<int32_t*>(value_offsets->mutable_data()) = 0;
 
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 13572aefa753c..80244f6c8df9c 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -952,7 +952,7 @@ class ArrayReader {
                            std::shared_ptr<Buffer>* validity_buffer) {
     int length = static_cast<int>(is_valid.size());
 
-    std::shared_ptr<MutableBuffer> out_buffer;
+    std::shared_ptr<Buffer> out_buffer;
     RETURN_NOT_OK(GetEmptyBitmap(pool_, length, &out_buffer));
     uint8_t* bitmap = out_buffer->mutable_data();
 
@@ -1056,7 +1056,7 @@ class ArrayReader {
     int32_t byte_width = type.byte_width();
 
     // Allocate space for parsed values
-    std::shared_ptr<MutableBuffer> byte_buffer;
+    std::shared_ptr<Buffer> byte_buffer;
     RETURN_NOT_OK(AllocateBuffer(pool_, byte_width, &byte_buffer));
     uint8_t* byte_buffer_data = byte_buffer->mutable_data();
 
@@ -1084,7 +1084,7 @@ class ArrayReader {
   template <typename T>
   Status GetIntArray(const RjArray& json_array, const int32_t length,
                      std::shared_ptr<Buffer>* out) {
-    std::shared_ptr<MutableBuffer> buffer;
+    std::shared_ptr<Buffer> buffer;
     RETURN_NOT_OK(AllocateBuffer(pool_, length * sizeof(T), &buffer));
 
     T* values = reinterpret_cast<T*>(buffer->mutable_data());
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index e17b974adfcad..6e05b94ffba18 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -272,7 +272,7 @@ class RecordBatchSerializer : public ArrayVisitor {
       // zero. We must a) create a new offsets array with shifted offsets and
       // b) slice the values array accordingly
 
-      std::shared_ptr<MutableBuffer> shifted_offsets;
+      std::shared_ptr<Buffer> shifted_offsets;
       RETURN_NOT_OK(AllocateBuffer(pool_, sizeof(int32_t) * (array.length() + 1),
                                    &shifted_offsets));
 
@@ -425,7 +425,7 @@ class RecordBatchSerializer : public ArrayVisitor {
         const uint8_t* type_ids = array.raw_type_ids();
 
         // Allocate the shifted offsets
-        std::shared_ptr<MutableBuffer> shifted_offsets_buffer;
+        std::shared_ptr<Buffer> shifted_offsets_buffer;
         RETURN_NOT_OK(
             AllocateBuffer(pool_, length * sizeof(int32_t), &shifted_offsets_buffer));
         int32_t* shifted_offsets =
@@ -897,7 +897,7 @@ Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
                             std::shared_ptr<Buffer>* out) {
   int64_t size = 0;
   RETURN_NOT_OK(GetRecordBatchSize(batch, &size));
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(AllocateBuffer(pool, size, &buffer));
 
   io::FixedSizeBufferWriter stream(buffer);
diff --git a/cpp/src/arrow/tensor-test.cc b/cpp/src/arrow/tensor-test.cc
index 0a11422b75d13..3df0e6ccb4a55 100644
--- a/cpp/src/arrow/tensor-test.cc
+++ b/cpp/src/arrow/tensor-test.cc
@@ -36,7 +36,7 @@ TEST(TestTensor, ZeroDim) {
 
   using T = int64_t;
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer));
 
   Tensor t0(int64(), buffer, shape);
@@ -52,7 +52,7 @@ TEST(TestTensor, BasicCtors) {
 
   using T = int64_t;
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer));
 
   Tensor t1(int64(), buffer, shape);
@@ -78,7 +78,7 @@ TEST(TestTensor, IsContiguous) {
 
   using T = int64_t;
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer));
 
   std::vector<int64_t> c_strides = {48, 8};
@@ -96,7 +96,7 @@ TEST(TestTensor, IsContiguous) {
 TEST(TestTensor, ZeroDimensionalTensor) {
   std::vector<int64_t> shape = {0};
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), 0, &buffer));
 
   Tensor t(int64(), buffer, shape);
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index b074b8c309ba1..bbfd8ae27679f 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -33,7 +33,6 @@ namespace arrow {
 
 class Buffer;
 class MemoryPool;
-class MutableBuffer;
 class Status;
 
 static inline bool is_tensor_supported(Type::type type_id) {
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 711d2b04025c1..91f2bc9ccc6c1 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -124,7 +124,7 @@ static inline Status GetBitmapFromVector(const std::vector<T>& is_valid,
                                          std::shared_ptr<Buffer>* result) {
   size_t length = is_valid.size();
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(GetEmptyBitmap(default_memory_pool(), length, &buffer));
 
   uint8_t* bitmap = buffer->mutable_data();
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index 231bf54a2a3b6..bbe95cf4b09f7 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -102,7 +102,7 @@ TEST(BitUtilTests, TestCountSetBits) {
 TEST(BitUtilTests, TestCopyBitmap) {
   const int kBufferSize = 1000;
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   ASSERT_OK(AllocateBuffer(default_memory_pool(), kBufferSize, &buffer));
   memset(buffer->mutable_data(), 0, kBufferSize);
   test::random_bytes(kBufferSize, 0, buffer->mutable_data());
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index f255f95f30a76..0984055167978 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -46,7 +46,7 @@ Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes,
                             std::shared_ptr<Buffer>* out) {
   int64_t bit_length = BitUtil::BytesForBits(bytes.size());
 
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), bit_length, &buffer));
 
   memset(buffer->mutable_data(), 0, static_cast<size_t>(bit_length));
@@ -97,16 +97,23 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
   return count;
 }
 
-Status GetEmptyBitmap(MemoryPool* pool, int64_t length,
-                      std::shared_ptr<MutableBuffer>* result) {
+Status GetEmptyBitmap(MemoryPool* pool, int64_t length, std::shared_ptr<Buffer>* result) {
   RETURN_NOT_OK(AllocateBuffer(pool, BitUtil::BytesForBits(length), result));
   memset((*result)->mutable_data(), 0, static_cast<size_t>((*result)->size()));
   return Status::OK();
 }
 
+Status GetEmptyBitmap(MemoryPool* pool, int64_t length,
+                      std::shared_ptr<MutableBuffer>* result) {
+  std::shared_ptr<Buffer> buffer;
+  RETURN_NOT_OK(GetEmptyBitmap(pool, length, &buffer));
+  *result = std::dynamic_pointer_cast<MutableBuffer>(buffer);
+  return Status::OK();
+}
+
 Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t length,
                   std::shared_ptr<Buffer>* out) {
-  std::shared_ptr<MutableBuffer> buffer;
+  std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(GetEmptyBitmap(pool, length, &buffer));
   uint8_t* dest = buffer->mutable_data();
   for (int64_t i = 0; i < length; ++i) {
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 5c3938aadbc66..52e78bf767c8c 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -387,8 +387,8 @@ ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buf
 // ----------------------------------------------------------------------
 // Bitmap utilities
 
-Status ARROW_EXPORT GetEmptyBitmap(MemoryPool* pool, int64_t length,
-                                   std::shared_ptr<MutableBuffer>* result);
+ARROW_EXPORT
+Status GetEmptyBitmap(MemoryPool* pool, int64_t length, std::shared_ptr<Buffer>* result);
 
 /// Copy a bit range of an existing bitmap
 ///
@@ -399,8 +399,9 @@ Status ARROW_EXPORT GetEmptyBitmap(MemoryPool* pool, int64_t length,
 /// \param[out] out the resulting copy
 ///
 /// \return Status message
-Status ARROW_EXPORT CopyBitmap(MemoryPool* pool, const uint8_t* bitmap, int64_t offset,
-                               int64_t length, std::shared_ptr<Buffer>* out);
+ARROW_EXPORT
+Status CopyBitmap(MemoryPool* pool, const uint8_t* bitmap, int64_t offset, int64_t length,
+                  std::shared_ptr<Buffer>* out);
 
 /// Compute the number of 1's in the given data array
 ///
@@ -409,12 +410,20 @@ Status ARROW_EXPORT CopyBitmap(MemoryPool* pool, const uint8_t* bitmap, int64_t
 /// \param[in] length the number of bits to inspect in the bitmap relative to the offset
 ///
 /// \return The number of set (1) bits in the range
-int64_t ARROW_EXPORT CountSetBits(const uint8_t* data, int64_t bit_offset,
-                                  int64_t length);
+ARROW_EXPORT
+int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length);
+
+ARROW_EXPORT
+bool BitmapEquals(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+                  int64_t right_offset, int64_t bit_length);
+
+#ifndef ARROW_NO_DEPRECATED_API
+/// \deprecated Since 0.7.0
+ARROW_EXPORT
+Status GetEmptyBitmap(MemoryPool* pool, int64_t length,
+                      std::shared_ptr<MutableBuffer>* result);
+#endif
 
-bool ARROW_EXPORT BitmapEquals(const uint8_t* left, int64_t left_offset,
-                               const uint8_t* right, int64_t right_offset,
-                               int64_t bit_length);
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_BIT_UTIL_H

From 254078ef62ef876210379440f16197e2f0edb946 Mon Sep 17 00:00:00 2001
From: Wataru Shimizu <waruzilla@gmail.com>
Date: Tue, 29 Aug 2017 14:21:44 +0200
Subject: [PATCH 0968/1644] ARROW-1426: [Site] Fix the title of the top page.

The <title> of the top page is empty because top.html has 2 <title> elements
and the 1st one is converted to empty element.

Author: Wataru Shimizu <waruzilla@gmail.com>

Closes #1006 from wagavulin/fix-top-page-title and squashes the following commits:

d3c30c7 [Wataru Shimizu] Fix the title of the top page.
---
 site/_includes/top.html | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/site/_includes/top.html b/site/_includes/top.html
index cfc4cde137ae7..988ae6c742b1f 100644
--- a/site/_includes/top.html
+++ b/site/_includes/top.html
@@ -2,14 +2,13 @@
 <html lang="en-US">
   <head>
     <meta charset="UTF-8">
-    <title>{{ page.title }}</title>
+    <title>Apache Arrow Homepage</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta name="generator" content="Jekyll v{{ jekyll.version }}">
     <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
     <link rel="icon" type="image/x-icon" href="{{ site.baseurl }}/favicon.ico">
 
-    <title>Apache Arrow Homepage</title>
     <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
 
     <link href="{{ site.baseurl }}/css/main.css" rel="stylesheet">

From 441f96594156a798c59a2e925ec6d5aec4541ddc Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Tue, 29 Aug 2017 14:32:38 -0400
Subject: [PATCH 0969/1644] ARROW-1418: [Python] Introduce SerializationContext
 to register custom serialization callbacks

This gets rid of the global serialize and deserialize callbacks and exposes the functionality to provide custom callbacks to the user.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1002 from pcmoritz/serialization-callbacks and squashes the following commits:

81f77796 [Philipp Moritz] Add SerializationContext to eliminate global type registry
---
 ci/travis_script_python.sh                 |   4 +-
 cpp/src/arrow/python/arrow_to_python.cc    |  93 +++++-----
 cpp/src/arrow/python/arrow_to_python.h     |   9 +-
 cpp/src/arrow/python/python_to_arrow.cc    | 115 ++++++------
 cpp/src/arrow/python/python_to_arrow.h     |  18 +-
 python/doc/source/api.rst                  |   1 +
 python/pyarrow/__init__.py                 |   2 +-
 python/pyarrow/includes/libarrow.pxd       |   9 +-
 python/pyarrow/serialization.pxi           | 201 +++++++++++----------
 python/pyarrow/tests/test_serialization.py |  93 ++++++----
 10 files changed, 285 insertions(+), 260 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 66cd17d5ff212..b5ba136b16f65 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -95,10 +95,10 @@ python_version_tests() {
   conda install -y -q pip numpy pandas cython flake8
 
   # Fail fast on style checks
-  flake8 pyarrow
+  flake8 --count pyarrow
 
   # Check Cython files with some checks turned off
-  flake8 --config=.flake8.cython pyarrow
+  flake8 --count --config=.flake8.cython pyarrow
 
   # Build C++ libraries
   rebuild_arrow_libraries
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index dcb06f80acc9e..3ff1f11018d10 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -27,40 +27,37 @@
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
+#include "arrow/python/python_to_arrow.h"
 #include "arrow/table.h"
 #include "arrow/util/logging.h"
 
-extern "C" {
-extern PyObject* pyarrow_serialize_callback;
-extern PyObject* pyarrow_deserialize_callback;
-}
-
 namespace arrow {
 namespace py {
 
-Status CallCustomCallback(PyObject* callback, PyObject* elem, PyObject** result);
+Status CallDeserializeCallback(PyObject* context, PyObject* value,
+                               PyObject** deserialized_object);
 
-Status DeserializeTuple(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
-                        PyObject* base,
+Status DeserializeTuple(PyObject* context, std::shared_ptr<Array> array,
+                        int64_t start_idx, int64_t stop_idx, PyObject* base,
                         const std::vector<std::shared_ptr<Tensor>>& tensors,
                         PyObject** out);
 
-Status DeserializeList(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
-                       PyObject* base,
+Status DeserializeList(PyObject* context, std::shared_ptr<Array> array, int64_t start_idx,
+                       int64_t stop_idx, PyObject* base,
                        const std::vector<std::shared_ptr<Tensor>>& tensors,
                        PyObject** out);
 
-Status DeserializeDict(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
-                       PyObject* base,
+Status DeserializeDict(PyObject* context, std::shared_ptr<Array> array, int64_t start_idx,
+                       int64_t stop_idx, PyObject* base,
                        const std::vector<std::shared_ptr<Tensor>>& tensors,
                        PyObject** out) {
   auto data = std::dynamic_pointer_cast<StructArray>(array);
   ScopedRef keys, vals;
   ScopedRef result(PyDict_New());
-  RETURN_NOT_OK(
-      DeserializeList(data->field(0), start_idx, stop_idx, base, tensors, keys.ref()));
-  RETURN_NOT_OK(
-      DeserializeList(data->field(1), start_idx, stop_idx, base, tensors, vals.ref()));
+  RETURN_NOT_OK(DeserializeList(context, data->field(0), start_idx, stop_idx, base,
+                                tensors, keys.ref()));
+  RETURN_NOT_OK(DeserializeList(context, data->field(1), start_idx, stop_idx, base,
+                                tensors, vals.ref()));
   for (int64_t i = start_idx; i < stop_idx; ++i) {
     // PyDict_SetItem behaves differently from PyList_SetItem and PyTuple_SetItem.
     // The latter two steal references whereas PyDict_SetItem does not. So we need
@@ -71,7 +68,7 @@ Status DeserializeDict(std::shared_ptr<Array> array, int64_t start_idx, int64_t
   }
   static PyObject* py_type = PyUnicode_FromString("_pytype_");
   if (PyDict_Contains(result.get(), py_type)) {
-    RETURN_NOT_OK(CallCustomCallback(pyarrow_deserialize_callback, result.get(), out));
+    RETURN_NOT_OK(CallDeserializeCallback(context, result.get(), out));
   } else {
     *out = result.release();
   }
@@ -93,7 +90,8 @@ Status DeserializeArray(std::shared_ptr<Array> array, int64_t offset, PyObject*
   return Status::OK();
 }
 
-Status GetValue(std::shared_ptr<Array> arr, int64_t index, int32_t type, PyObject* base,
+Status GetValue(PyObject* context, std::shared_ptr<Array> arr, int64_t index,
+                int32_t type, PyObject* base,
                 const std::vector<std::shared_ptr<Tensor>>& tensors, PyObject** result) {
   switch (arr->type()->id()) {
     case Type::BOOL:
@@ -130,13 +128,13 @@ Status GetValue(std::shared_ptr<Array> arr, int64_t index, int32_t type, PyObjec
       auto s = std::static_pointer_cast<StructArray>(arr);
       auto l = std::static_pointer_cast<ListArray>(s->field(0));
       if (s->type()->child(0)->name() == "list") {
-        return DeserializeList(l->values(), l->value_offset(index),
+        return DeserializeList(context, l->values(), l->value_offset(index),
                                l->value_offset(index + 1), base, tensors, result);
       } else if (s->type()->child(0)->name() == "tuple") {
-        return DeserializeTuple(l->values(), l->value_offset(index),
+        return DeserializeTuple(context, l->values(), l->value_offset(index),
                                 l->value_offset(index + 1), base, tensors, result);
       } else if (s->type()->child(0)->name() == "dict") {
-        return DeserializeDict(l->values(), l->value_offset(index),
+        return DeserializeDict(context, l->values(), l->value_offset(index),
                                l->value_offset(index + 1), base, tensors, result);
       } else {
         DCHECK(false) << "unexpected StructArray type " << s->type()->child(0)->name();
@@ -153,37 +151,37 @@ Status GetValue(std::shared_ptr<Array> arr, int64_t index, int32_t type, PyObjec
   return Status::OK();
 }
 
-#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                        \
-  auto data = std::dynamic_pointer_cast<UnionArray>(array);                 \
-  int64_t size = array->length();                                           \
-  ScopedRef result(CREATE_FN(stop_idx - start_idx));                        \
-  auto types = std::make_shared<Int8Array>(size, data->type_ids());         \
-  auto offsets = std::make_shared<Int32Array>(size, data->value_offsets()); \
-  for (int64_t i = start_idx; i < stop_idx; ++i) {                          \
-    if (data->IsNull(i)) {                                                  \
-      Py_INCREF(Py_None);                                                   \
-      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                    \
-    } else {                                                                \
-      int64_t offset = offsets->Value(i);                                   \
-      int8_t type = types->Value(i);                                        \
-      std::shared_ptr<Array> arr = data->child(type);                       \
-      PyObject* value;                                                      \
-      RETURN_NOT_OK(GetValue(arr, offset, type, base, tensors, &value));    \
-      SET_ITEM_FN(result.get(), i - start_idx, value);                      \
-    }                                                                       \
-  }                                                                         \
-  *out = result.release();                                                  \
+#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                              \
+  auto data = std::dynamic_pointer_cast<UnionArray>(array);                       \
+  int64_t size = array->length();                                                 \
+  ScopedRef result(CREATE_FN(stop_idx - start_idx));                              \
+  auto types = std::make_shared<Int8Array>(size, data->type_ids());               \
+  auto offsets = std::make_shared<Int32Array>(size, data->value_offsets());       \
+  for (int64_t i = start_idx; i < stop_idx; ++i) {                                \
+    if (data->IsNull(i)) {                                                        \
+      Py_INCREF(Py_None);                                                         \
+      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                          \
+    } else {                                                                      \
+      int64_t offset = offsets->Value(i);                                         \
+      int8_t type = types->Value(i);                                              \
+      std::shared_ptr<Array> arr = data->child(type);                             \
+      PyObject* value;                                                            \
+      RETURN_NOT_OK(GetValue(context, arr, offset, type, base, tensors, &value)); \
+      SET_ITEM_FN(result.get(), i - start_idx, value);                            \
+    }                                                                             \
+  }                                                                               \
+  *out = result.release();                                                        \
   return Status::OK();
 
-Status DeserializeList(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
-                       PyObject* base,
+Status DeserializeList(PyObject* context, std::shared_ptr<Array> array, int64_t start_idx,
+                       int64_t stop_idx, PyObject* base,
                        const std::vector<std::shared_ptr<Tensor>>& tensors,
                        PyObject** out) {
   DESERIALIZE_SEQUENCE(PyList_New, PyList_SET_ITEM)
 }
 
-Status DeserializeTuple(std::shared_ptr<Array> array, int64_t start_idx, int64_t stop_idx,
-                        PyObject* base,
+Status DeserializeTuple(PyObject* context, std::shared_ptr<Array> array,
+                        int64_t start_idx, int64_t stop_idx, PyObject* base,
                         const std::vector<std::shared_ptr<Tensor>>& tensors,
                         PyObject** out) {
   DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SET_ITEM)
@@ -212,9 +210,10 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out)
   return Status::OK();
 }
 
-Status DeserializeObject(const SerializedPyObject& obj, PyObject* base, PyObject** out) {
+Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObject* base,
+                         PyObject** out) {
   PyAcquireGIL lock;
-  return DeserializeList(obj.batch->column(0), 0, obj.batch->num_rows(), base,
+  return DeserializeList(context, obj.batch->column(0), 0, obj.batch->num_rows(), base,
                          obj.tensors, out);
 }
 
diff --git a/cpp/src/arrow/python/arrow_to_python.h b/cpp/src/arrow/python/arrow_to_python.h
index e187d59696906..3650c058efd97 100644
--- a/cpp/src/arrow/python/arrow_to_python.h
+++ b/cpp/src/arrow/python/arrow_to_python.h
@@ -49,6 +49,11 @@ ARROW_EXPORT
 Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out);
 
 /// \brief Reconstruct Python object from Arrow-serialized representation
+/// \param[in] context Serialization context which contains custom serialization
+/// and deserialization callbacks. Can be any Python object with a
+/// _serialize_callback method for serialization and a _deserialize_callback
+/// method for deserialization. If context is None, no custom serialization
+/// will be attempted.
 /// \param[in] object
 /// \param[in] base a Python object holding the underlying data that any NumPy
 /// arrays will reference, to avoid premature deallocation
@@ -56,8 +61,8 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out);
 /// \return Status
 /// This acquires the GIL
 ARROW_EXPORT
-Status DeserializeObject(const SerializedPyObject& object, PyObject* base,
-                         PyObject** out);
+Status DeserializeObject(PyObject* context, const SerializedPyObject& object,
+                         PyObject* base, PyObject** out);
 
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index e00f1943f723c..9ae66dc35013a 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -41,11 +41,6 @@
 
 constexpr int32_t kMaxRecursionDepth = 100;
 
-extern "C" {
-PyObject* pyarrow_serialize_callback = NULL;
-PyObject* pyarrow_deserialize_callback = NULL;
-}
-
 namespace arrow {
 namespace py {
 
@@ -323,9 +318,10 @@ class DictBuilder {
   SequenceBuilder vals_;
 };
 
-Status CallCustomCallback(PyObject* callback, PyObject* elem, PyObject** result) {
+Status CallCustomCallback(PyObject* context, PyObject* method_name, PyObject* elem,
+                          PyObject** result) {
   *result = NULL;
-  if (!callback) {
+  if (context == Py_None) {
     std::stringstream ss;
     ScopedRef repr(PyObject_Repr(elem));
     RETURN_IF_PYERROR();
@@ -334,36 +330,38 @@ Status CallCustomCallback(PyObject* callback, PyObject* elem, PyObject** result)
        << ": handler not registered";
     return Status::NotImplemented(ss.str());
   } else {
-    ScopedRef arglist(Py_BuildValue("(O)", elem));
-    *result = PyObject_CallObject(callback, arglist.get());
+    *result = PyObject_CallMethodObjArgs(context, method_name, elem, NULL);
     RETURN_IF_PYERROR();
   }
   return Status::OK();
 }
 
-void set_serialization_callbacks(PyObject* serialize_callback,
-                                 PyObject* deserialize_callback) {
-  pyarrow_serialize_callback = serialize_callback;
-  pyarrow_deserialize_callback = deserialize_callback;
-}
-
-Status CallCustomSerializationCallback(PyObject* elem, PyObject** serialized_object) {
-  RETURN_NOT_OK(CallCustomCallback(pyarrow_serialize_callback, elem, serialized_object));
+Status CallSerializeCallback(PyObject* context, PyObject* value,
+                             PyObject** serialized_object) {
+  ScopedRef method_name(PyUnicode_FromString("_serialize_callback"));
+  RETURN_NOT_OK(CallCustomCallback(context, method_name.get(), value, serialized_object));
   if (!PyDict_Check(*serialized_object)) {
     return Status::TypeError("serialization callback must return a valid dictionary");
   }
   return Status::OK();
 }
 
-Status SerializeDict(std::vector<PyObject*> dicts, int32_t recursion_depth,
-                     std::shared_ptr<Array>* out, std::vector<PyObject*>* tensors_out);
+Status CallDeserializeCallback(PyObject* context, PyObject* value,
+                               PyObject** deserialized_object) {
+  ScopedRef method_name(PyUnicode_FromString("_deserialize_callback"));
+  return CallCustomCallback(context, method_name.get(), value, deserialized_object);
+}
+
+Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
+                     int32_t recursion_depth, std::shared_ptr<Array>* out,
+                     std::vector<PyObject*>* tensors_out);
 
-Status SerializeArray(PyArrayObject* array, SequenceBuilder* builder,
+Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder,
                       std::vector<PyObject*>* subdicts,
                       std::vector<PyObject*>* tensors_out);
 
-Status SerializeSequences(std::vector<PyObject*> sequences, int32_t recursion_depth,
-                          std::shared_ptr<Array>* out,
+Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
+                          int32_t recursion_depth, std::shared_ptr<Array>* out,
                           std::vector<PyObject*>* tensors_out);
 
 Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
@@ -405,9 +403,9 @@ Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
   return builder->AppendInt64(value);
 }
 
-Status Append(PyObject* elem, SequenceBuilder* builder, std::vector<PyObject*>* sublists,
-              std::vector<PyObject*>* subtuples, std::vector<PyObject*>* subdicts,
-              std::vector<PyObject*>* tensors_out) {
+Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
+              std::vector<PyObject*>* sublists, std::vector<PyObject*>* subtuples,
+              std::vector<PyObject*>* subdicts, std::vector<PyObject*>* tensors_out) {
   // The bool case must precede the int case (PyInt_Check passes for bools)
   if (PyBool_Check(elem)) {
     RETURN_NOT_OK(builder->AppendBool(elem == Py_True));
@@ -422,7 +420,7 @@ Status Append(PyObject* elem, SequenceBuilder* builder, std::vector<PyObject*>*
       // Attempt to serialize the object using the custom callback.
       PyObject* serialized_object;
       // The reference count of serialized_object will be decremented in SerializeDict
-      RETURN_NOT_OK(CallCustomSerializationCallback(elem, &serialized_object));
+      RETURN_NOT_OK(CallSerializeCallback(context, elem, &serialized_object));
       RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object)));
       subdicts->push_back(serialized_object);
     }
@@ -462,7 +460,7 @@ Status Append(PyObject* elem, SequenceBuilder* builder, std::vector<PyObject*>*
   } else if (PyArray_IsScalar(elem, Generic)) {
     RETURN_NOT_OK(AppendScalar(elem, builder));
   } else if (PyArray_Check(elem)) {
-    RETURN_NOT_OK(SerializeArray(reinterpret_cast<PyArrayObject*>(elem), builder,
+    RETURN_NOT_OK(SerializeArray(context, reinterpret_cast<PyArrayObject*>(elem), builder,
                                  subdicts, tensors_out));
   } else if (elem == Py_None) {
     RETURN_NOT_OK(builder->AppendNone());
@@ -470,14 +468,14 @@ Status Append(PyObject* elem, SequenceBuilder* builder, std::vector<PyObject*>*
     // Attempt to serialize the object using the custom callback.
     PyObject* serialized_object;
     // The reference count of serialized_object will be decremented in SerializeDict
-    RETURN_NOT_OK(CallCustomSerializationCallback(elem, &serialized_object));
+    RETURN_NOT_OK(CallSerializeCallback(context, elem, &serialized_object));
     RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object)));
     subdicts->push_back(serialized_object);
   }
   return Status::OK();
 }
 
-Status SerializeArray(PyArrayObject* array, SequenceBuilder* builder,
+Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder,
                       std::vector<PyObject*>* subdicts,
                       std::vector<PyObject*>* tensors_out) {
   int dtype = PyArray_TYPE(array);
@@ -499,8 +497,8 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder* builder,
     default: {
       PyObject* serialized_object;
       // The reference count of serialized_object will be decremented in SerializeDict
-      RETURN_NOT_OK(CallCustomSerializationCallback(reinterpret_cast<PyObject*>(array),
-                                                    &serialized_object));
+      RETURN_NOT_OK(CallSerializeCallback(context, reinterpret_cast<PyObject*>(array),
+                                          &serialized_object));
       RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object)));
       subdicts->push_back(serialized_object);
     }
@@ -508,8 +506,8 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder* builder,
   return Status::OK();
 }
 
-Status SerializeSequences(std::vector<PyObject*> sequences, int32_t recursion_depth,
-                          std::shared_ptr<Array>* out,
+Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
+                          int32_t recursion_depth, std::shared_ptr<Array>* out,
                           std::vector<PyObject*>* tensors_out) {
   DCHECK(out);
   if (recursion_depth >= kMaxRecursionDepth) {
@@ -524,28 +522,31 @@ Status SerializeSequences(std::vector<PyObject*> sequences, int32_t recursion_de
     RETURN_IF_PYERROR();
     ScopedRef item;
     while (item.reset(PyIter_Next(iterator.get())), item.get()) {
-      RETURN_NOT_OK(
-          Append(item.get(), &builder, &sublists, &subtuples, &subdicts, tensors_out));
+      RETURN_NOT_OK(Append(context, item.get(), &builder, &sublists, &subtuples,
+                           &subdicts, tensors_out));
     }
   }
   std::shared_ptr<Array> list;
   if (sublists.size() > 0) {
-    RETURN_NOT_OK(SerializeSequences(sublists, recursion_depth + 1, &list, tensors_out));
+    RETURN_NOT_OK(
+        SerializeSequences(context, sublists, recursion_depth + 1, &list, tensors_out));
   }
   std::shared_ptr<Array> tuple;
   if (subtuples.size() > 0) {
     RETURN_NOT_OK(
-        SerializeSequences(subtuples, recursion_depth + 1, &tuple, tensors_out));
+        SerializeSequences(context, subtuples, recursion_depth + 1, &tuple, tensors_out));
   }
   std::shared_ptr<Array> dict;
   if (subdicts.size() > 0) {
-    RETURN_NOT_OK(SerializeDict(subdicts, recursion_depth + 1, &dict, tensors_out));
+    RETURN_NOT_OK(
+        SerializeDict(context, subdicts, recursion_depth + 1, &dict, tensors_out));
   }
   return builder.Finish(list.get(), tuple.get(), dict.get(), out);
 }
 
-Status SerializeDict(std::vector<PyObject*> dicts, int32_t recursion_depth,
-                     std::shared_ptr<Array>* out, std::vector<PyObject*>* tensors_out) {
+Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
+                     int32_t recursion_depth, std::shared_ptr<Array>* out,
+                     std::vector<PyObject*>* tensors_out) {
   DictBuilder result;
   if (recursion_depth >= kMaxRecursionDepth) {
     return Status::NotImplemented(
@@ -557,37 +558,37 @@ Status SerializeDict(std::vector<PyObject*> dicts, int32_t recursion_depth,
     PyObject *key, *value;
     Py_ssize_t pos = 0;
     while (PyDict_Next(dict, &pos, &key, &value)) {
-      RETURN_NOT_OK(
-          Append(key, &result.keys(), &dummy, &key_tuples, &key_dicts, tensors_out));
-      DCHECK_EQ(dummy.size(), 0);
-      RETURN_NOT_OK(Append(value, &result.vals(), &val_lists, &val_tuples, &val_dicts,
+      RETURN_NOT_OK(Append(context, key, &result.keys(), &dummy, &key_tuples, &key_dicts,
                            tensors_out));
+      DCHECK_EQ(dummy.size(), 0);
+      RETURN_NOT_OK(Append(context, value, &result.vals(), &val_lists, &val_tuples,
+                           &val_dicts, tensors_out));
     }
   }
   std::shared_ptr<Array> key_tuples_arr;
   if (key_tuples.size() > 0) {
-    RETURN_NOT_OK(SerializeSequences(key_tuples, recursion_depth + 1, &key_tuples_arr,
-                                     tensors_out));
+    RETURN_NOT_OK(SerializeSequences(context, key_tuples, recursion_depth + 1,
+                                     &key_tuples_arr, tensors_out));
   }
   std::shared_ptr<Array> key_dicts_arr;
   if (key_dicts.size() > 0) {
-    RETURN_NOT_OK(
-        SerializeDict(key_dicts, recursion_depth + 1, &key_dicts_arr, tensors_out));
+    RETURN_NOT_OK(SerializeDict(context, key_dicts, recursion_depth + 1, &key_dicts_arr,
+                                tensors_out));
   }
   std::shared_ptr<Array> val_list_arr;
   if (val_lists.size() > 0) {
-    RETURN_NOT_OK(
-        SerializeSequences(val_lists, recursion_depth + 1, &val_list_arr, tensors_out));
+    RETURN_NOT_OK(SerializeSequences(context, val_lists, recursion_depth + 1,
+                                     &val_list_arr, tensors_out));
   }
   std::shared_ptr<Array> val_tuples_arr;
   if (val_tuples.size() > 0) {
-    RETURN_NOT_OK(SerializeSequences(val_tuples, recursion_depth + 1, &val_tuples_arr,
-                                     tensors_out));
+    RETURN_NOT_OK(SerializeSequences(context, val_tuples, recursion_depth + 1,
+                                     &val_tuples_arr, tensors_out));
   }
   std::shared_ptr<Array> val_dict_arr;
   if (val_dicts.size() > 0) {
-    RETURN_NOT_OK(
-        SerializeDict(val_dicts, recursion_depth + 1, &val_dict_arr, tensors_out));
+    RETURN_NOT_OK(SerializeDict(context, val_dicts, recursion_depth + 1, &val_dict_arr,
+                                tensors_out));
   }
   RETURN_NOT_OK(result.Finish(key_tuples_arr.get(), key_dicts_arr.get(),
                               val_list_arr.get(), val_tuples_arr.get(),
@@ -601,7 +602,7 @@ Status SerializeDict(std::vector<PyObject*> dicts, int32_t recursion_depth,
     if (PyDict_Contains(dict, py_type)) {
       // If the dictionary contains the key "_pytype_", then the user has to
       // have registered a callback.
-      if (pyarrow_serialize_callback == nullptr) {
+      if (context == Py_None) {
         return Status::Invalid("No serialization callback set");
       }
       Py_XDECREF(dict);
@@ -617,12 +618,12 @@ std::shared_ptr<RecordBatch> MakeBatch(std::shared_ptr<Array> data) {
   return std::shared_ptr<RecordBatch>(new RecordBatch(schema, data->length(), {data}));
 }
 
-Status SerializeObject(PyObject* sequence, SerializedPyObject* out) {
+Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out) {
   PyAcquireGIL lock;
   std::vector<PyObject*> sequences = {sequence};
   std::shared_ptr<Array> array;
   std::vector<PyObject*> py_tensors;
-  RETURN_NOT_OK(SerializeSequences(sequences, 0, &array, &py_tensors));
+  RETURN_NOT_OK(SerializeSequences(context, sequences, 0, &array, &py_tensors));
   out->batch = MakeBatch(array);
   for (const auto& py_tensor : py_tensors) {
     std::shared_ptr<Tensor> arrow_tensor;
diff --git a/cpp/src/arrow/python/python_to_arrow.h b/cpp/src/arrow/python/python_to_arrow.h
index 8ac03965a1c44..2c80e5d6bf891 100644
--- a/cpp/src/arrow/python/python_to_arrow.h
+++ b/cpp/src/arrow/python/python_to_arrow.h
@@ -44,18 +44,12 @@ struct ARROW_EXPORT SerializedPyObject {
   std::vector<std::shared_ptr<Tensor>> tensors;
 };
 
-/// \brief Register callback functions to perform conversions to or from other
-/// Python representations en route to/from deserialization
-///
-/// \param[in] serialize_callback a Python callable
-/// \param[in] deserialize_callback a Python callable
-///
-/// Analogous to Python custom picklers / unpicklers
-ARROW_EXPORT
-void set_serialization_callbacks(PyObject* serialize_callback,
-                                 PyObject* deserialize_callback);
-
 /// \brief Serialize Python sequence as a RecordBatch plus
+/// \param[in] context Serialization context which contains custom serialization
+/// and deserialization callbacks. Can be any Python object with a
+/// _serialize_callback method for serialization and a _deserialize_callback
+/// method for deserialization. If context is None, no custom serialization
+/// will be attempted.
 /// \param[in] sequence a Python sequence object to serialize to Arrow data
 /// structures
 /// \param[out] out the serialized representation
@@ -63,7 +57,7 @@ void set_serialization_callbacks(PyObject* serialize_callback,
 ///
 /// Release GIL before calling
 ARROW_EXPORT
-Status SerializeObject(PyObject* sequence, SerializedPyObject* out);
+Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out);
 
 /// \brief Write serialized Python object to OutputStream
 /// \param[in] object a serialized Python object to write out
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 846af4c7f2ef8..4761c7fe5f934 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -207,6 +207,7 @@ Interprocess Communication and Serialization
    deserialize_from
    read_serialized
    SerializedPyObject
+   SerializationContext
 
 .. _api.memory_pool:
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 985e262bced8f..ecdcfcdea3dc1 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -91,7 +91,7 @@
 # Serialization
 from pyarrow.lib import (deserialize_from, deserialize,
                          serialize, serialize_to, read_serialized,
-                         SerializedPyObject,
+                         SerializedPyObject, SerializationContext,
                          SerializationException, DeserializationException)
 
 from pyarrow.filesystem import FileSystem, LocalFileSystem
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index fcf27da3c10c8..98eda8bf5c2c7 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -797,20 +797,19 @@ cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:
         shared_ptr[CRecordBatch] batch
         vector[shared_ptr[CTensor]] tensors
 
-    CStatus SerializeObject(object sequence, CSerializedPyObject* out)
+    CStatus SerializeObject(object context, object sequence,
+                            CSerializedPyObject* out)
 
     CStatus WriteSerializedObject(const CSerializedPyObject& obj,
                                   OutputStream* dst)
 
-    CStatus DeserializeObject(const CSerializedPyObject& obj,
+    CStatus DeserializeObject(object context,
+                              const CSerializedPyObject& obj,
                               PyObject* base, PyObject** out)
 
     CStatus ReadSerializedObject(RandomAccessFile* src,
                                  CSerializedPyObject* out)
 
-    void set_serialization_callbacks(object serialize_callback,
-                                     object deserialize_callback)
-
 
 cdef extern from 'arrow/python/init.h':
     int arrow_init_numpy() except -1
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index 062121ff39842..dc9bdaffa3bb5 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -43,93 +43,98 @@ class DeserializationException(Exception):
         self.type_id = type_id
 
 
-# Types with special serialization handlers
-type_to_type_id = dict()
-whitelisted_types = dict()
-types_to_pickle = set()
-custom_serializers = dict()
-custom_deserializers = dict()
-
-
-def register_type(type, type_id, pickle=False,
-                  custom_serializer=None, custom_deserializer=None):
-    """Add type to the list of types we can serialize.
-
-    Parameters
-    ----------
-    type :type
-        The type that we can serialize.
-    type_id : bytes
-        A string of bytes used to identify the type.
-    pickle : bool
-        True if the serialization should be done with pickle.
-        False if it should be done efficiently with Arrow.
-    custom_serializer : callable
-        This argument is optional, but can be provided to
-        serialize objects of the class in a particular way.
-    custom_deserializer : callable
-        This argument is optional, but can be provided to
-        deserialize objects of the class in a particular way.
-    """
-    type_to_type_id[type] = type_id
-    whitelisted_types[type_id] = type
-    if pickle:
-        types_to_pickle.add(type_id)
-    if custom_serializer is not None:
-        custom_serializers[type_id] = custom_serializer
-        custom_deserializers[type_id] = custom_deserializer
-
-
-def _serialization_callback(obj):
-    if type(obj) not in type_to_type_id:
-        raise SerializationException("pyarrow does not know how to "
-                                     "serialize objects of type {}."
-                                     .format(type(obj)),
-                                     obj)
-    type_id = type_to_type_id[type(obj)]
-    if type_id in types_to_pickle:
-        serialized_obj = {"data": pickle.dumps(obj), "pickle": True}
-    elif type_id in custom_serializers:
-        serialized_obj = {"data": custom_serializers[type_id](obj)}
-    else:
-        if is_named_tuple(type(obj)):
-            serialized_obj = {}
-            serialized_obj["_pa_getnewargs_"] = obj.__getnewargs__()
-        elif hasattr(obj, "__dict__"):
-            serialized_obj = obj.__dict__
-        else:
-            raise SerializationException("We do not know how to serialize "
-                                         "the object '{}'".format(obj), obj)
-    return dict(serialized_obj, **{"_pytype_": type_id})
-
-
-def _deserialization_callback(serialized_obj):
-    type_id = serialized_obj["_pytype_"]
-
-    if "pickle" in serialized_obj:
-        # The object was pickled, so unpickle it.
-        obj = pickle.loads(serialized_obj["data"])
-    else:
-        assert type_id not in types_to_pickle
-        if type_id not in whitelisted_types:
-            raise "error"
-        type = whitelisted_types[type_id]
-        if type_id in custom_deserializers:
-            obj = custom_deserializers[type_id](serialized_obj["data"])
+cdef class SerializationContext:
+    cdef:
+        object type_to_type_id
+        object whitelisted_types
+        object types_to_pickle
+        object custom_serializers
+        object custom_deserializers
+
+    def __init__(self):
+        # Types with special serialization handlers
+        self.type_to_type_id = dict()
+        self.whitelisted_types = dict()
+        self.types_to_pickle = set()
+        self.custom_serializers = dict()
+        self.custom_deserializers = dict()
+
+    def register_type(self, type_, type_id, pickle=False,
+                      custom_serializer=None, custom_deserializer=None):
+        """EXPERIMENTAL: Add type to the list of types we can serialize.
+
+        Parameters
+        ----------
+        type_ : TypeType
+            The type that we can serialize.
+        type_id : bytes
+            A string of bytes used to identify the type.
+        pickle : bool
+            True if the serialization should be done with pickle.
+            False if it should be done efficiently with Arrow.
+        custom_serializer : callable
+            This argument is optional, but can be provided to
+            serialize objects of the class in a particular way.
+        custom_deserializer : callable
+            This argument is optional, but can be provided to
+            deserialize objects of the class in a particular way.
+        """
+        self.type_to_type_id[type_] = type_id
+        self.whitelisted_types[type_id] = type_
+        if pickle:
+            self.types_to_pickle.add(type_id)
+        if custom_serializer is not None:
+            self.custom_serializers[type_id] = custom_serializer
+            self.custom_deserializers[type_id] = custom_deserializer
+
+    def _serialize_callback(self, obj):
+        if type(obj) not in self.type_to_type_id:
+            raise SerializationException("pyarrow does not know how to "
+                                         "serialize objects of type {}."
+                                         .format(type(obj)),
+                                         obj)
+        type_id = self.type_to_type_id[type(obj)]
+        if type_id in self.types_to_pickle:
+            serialized_obj = {"data": pickle.dumps(obj), "pickle": True}
+        elif type_id in self.custom_serializers:
+            serialized_obj = {"data": self.custom_serializers[type_id](obj)}
         else:
-            # In this case, serialized_obj should just be the __dict__ field.
-            if "_pa_getnewargs_" in serialized_obj:
-                obj = type.__new__(type, *serialized_obj["_pa_getnewargs_"])
+            if is_named_tuple(type(obj)):
+                serialized_obj = {}
+                serialized_obj["_pa_getnewargs_"] = obj.__getnewargs__()
+            elif hasattr(obj, "__dict__"):
+                serialized_obj = obj.__dict__
             else:
-                obj = type.__new__(type)
-                serialized_obj.pop("_pytype_")
-                obj.__dict__.update(serialized_obj)
-    return obj
+                msg = "We do not know how to serialize " \
+                      "the object '{}'".format(obj)
+                raise SerializationException(msg, obj)
+        return dict(serialized_obj, **{"_pytype_": type_id})
 
+    def _deserialize_callback(self, serialized_obj):
+        type_id = serialized_obj["_pytype_"]
 
-set_serialization_callbacks(_serialization_callback,
-                            _deserialization_callback)
-
+        if "pickle" in serialized_obj:
+            # The object was pickled, so unpickle it.
+            obj = pickle.loads(serialized_obj["data"])
+        else:
+            assert type_id not in self.types_to_pickle
+            if type_id not in self.whitelisted_types:
+                raise "error"
+            type_ = self.whitelisted_types[type_id]
+            if type_id in self.custom_deserializers:
+                obj = self.custom_deserializers[type_id](
+                    serialized_obj["data"])
+            else:
+                # In this case, serialized_obj should just be
+                # the __dict__ field.
+                if "_pa_getnewargs_" in serialized_obj:
+                    obj = type_.__new__(
+                        type_, *serialized_obj["_pa_getnewargs_"])
+                else:
+                    obj = type_.__new__(type_)
+                    serialized_obj.pop("_pytype_")
+                    obj.__dict__.update(serialized_obj)
+        return obj
 
 cdef class SerializedPyObject:
     """
@@ -162,15 +167,15 @@ cdef class SerializedPyObject:
         with nogil:
             check_status(WriteSerializedObject(self.data, stream))
 
-    def deserialize(self):
+    def deserialize(self, SerializationContext context=None):
         """
         Convert back to Python object
         """
         cdef PyObject* result
 
         with nogil:
-            check_status(DeserializeObject(self.data, <PyObject*> self.base,
-                                           &result))
+            check_status(DeserializeObject(context, self.data,
+                                           <PyObject*> self.base, &result))
 
         # PyObject_to_object is necessary to avoid a memory leak;
         # also unpack the list the object was wrapped in in serialize
@@ -185,13 +190,15 @@ cdef class SerializedPyObject:
         return sink.get_result()
 
 
-def serialize(object value):
+def serialize(object value, SerializationContext context=None):
     """EXPERIMENTAL: Serialize a Python sequence
 
     Parameters
     ----------
     value: object
         Python object for the sequence that is to be serialized.
+    context : SerializationContext
+        Custom serialization and deserialization context
 
     Returns
     -------
@@ -200,11 +207,11 @@ def serialize(object value):
     cdef SerializedPyObject serialized = SerializedPyObject()
     wrapped_value = [value]
     with nogil:
-        check_status(SerializeObject(wrapped_value, &serialized.data))
+        check_status(SerializeObject(context, wrapped_value, &serialized.data))
     return serialized
 
 
-def serialize_to(object value, sink):
+def serialize_to(object value, sink, SerializationContext context=None):
     """EXPERIMENTAL: Serialize a Python sequence to a file.
 
     Parameters
@@ -213,8 +220,10 @@ def serialize_to(object value, sink):
         Python object for the sequence that is to be serialized.
     sink: NativeFile or file-like
         File the sequence will be written to.
+    context : SerializationContext
+        Custom serialization and deserialization context
     """
-    serialized = serialize(value)
+    serialized = serialize(value, context)
     serialized.write_to(sink)
 
 
@@ -244,7 +253,7 @@ def read_serialized(source, base=None):
     return serialized
 
 
-def deserialize_from(source, object base):
+def deserialize_from(source, object base, SerializationContext context=None):
     """EXPERIMENTAL: Deserialize a Python sequence from a file.
 
     Parameters
@@ -254,6 +263,8 @@ def deserialize_from(source, object base):
     base: object
         This object will be the base object of all the numpy arrays
         contained in the sequence.
+    context : SerializationContext
+        Custom serialization and deserialization context
 
     Returns
     -------
@@ -261,10 +272,10 @@ def deserialize_from(source, object base):
         Python object for the deserialized sequence.
     """
     serialized = read_serialized(source, base=base)
-    return serialized.deserialize()
+    return serialized.deserialize(context)
 
 
-def deserialize(obj):
+def deserialize(obj, SerializationContext context=None):
     """
     EXPERIMENTAL: Deserialize Python object from Buffer or other Python object
     supporting the buffer protocol
@@ -272,10 +283,12 @@ def deserialize(obj):
     Parameters
     ----------
     obj : pyarrow.Buffer or Python object supporting buffer protocol
+    context : SerializationContext
+        Custom serialization and deserialization context
 
     Returns
     -------
     deserialized : object
     """
     source = BufferReader(obj)
-    return deserialize_from(source, obj)
+    return deserialize_from(source, obj, context)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index b2aa4af3616a0..d922576d36fa3 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -81,18 +81,6 @@ def assert_equal(obj1, obj2):
                                                                        obj2)
 
 
-def array_custom_serializer(obj):
-    return obj.tolist(), obj.dtype.str
-
-
-def array_custom_deserializer(serialized_obj):
-    return np.array(serialized_obj[0], dtype=np.dtype(serialized_obj[1]))
-
-
-pa.lib.register_type(np.ndarray, 20 * b"\x00", pickle=False,
-                     custom_serializer=array_custom_serializer,
-                     custom_deserializer=array_custom_deserializer)
-
 if sys.version_info >= (3, 0):
     long_extras = [0, np.array([["hi", u"hi"], [1.3, 1]])]
 else:
@@ -155,6 +143,11 @@ def __init__(self):
         Qux.__init__(self)
 
 
+class SubQuxPickle(Qux):
+    def __init__(self):
+        Qux.__init__(self)
+
+
 class CustomError(Exception):
     pass
 
@@ -165,41 +158,61 @@ class CustomError(Exception):
 
 
 CUSTOM_OBJECTS = [Exception("Test object."), CustomError(), Point(11, y=22),
-                  Foo(), Bar(), Baz(), Qux(), SubQux(),
+                  Foo(), Bar(), Baz(), Qux(), SubQux(), SubQuxPickle(),
                   NamedTupleExample(1, 1.0, "hi", np.zeros([3, 5]), [1, 2, 3])]
 
-pa.lib.register_type(Foo, 20 * b"\x01")
-pa.lib.register_type(Bar, 20 * b"\x02")
-pa.lib.register_type(Baz, 20 * b"\x03")
-pa.lib.register_type(Qux, 20 * b"\x04")
-pa.lib.register_type(SubQux, 20 * b"\x05")
-pa.lib.register_type(Exception, 20 * b"\x06")
-pa.lib.register_type(CustomError, 20 * b"\x07")
-pa.lib.register_type(Point, 20 * b"\x08")
-pa.lib.register_type(NamedTupleExample, 20 * b"\x09")
-
-# TODO(pcm): This is currently a workaround until arrow supports
-# arbitrary precision integers. This is only called on long integers,
-# see the associated case in the append method in python_to_arrow.cc
-pa.lib.register_type(int, 20 * b"\x10", pickle=False,
-                     custom_serializer=lambda obj: str(obj),
-                     custom_deserializer=(
-                         lambda serialized_obj: int(serialized_obj)))
-
-
-if (sys.version_info < (3, 0)):
-    deserializer = (
-        lambda serialized_obj: long(serialized_obj))  # noqa: E501,F821
-    pa.lib.register_type(long, 20 * b"\x11", pickle=False,  # noqa: E501,F821
-                         custom_serializer=lambda obj: str(obj),
-                         custom_deserializer=deserializer)
+
+def make_serialization_context():
+
+    def array_custom_serializer(obj):
+        return obj.tolist(), obj.dtype.str
+
+    def array_custom_deserializer(serialized_obj):
+        return np.array(serialized_obj[0], dtype=np.dtype(serialized_obj[1]))
+
+    context = pa.SerializationContext()
+
+    context.register_type(np.ndarray, 20 * b"\x00",
+                          custom_serializer=array_custom_serializer,
+                          custom_deserializer=array_custom_deserializer)
+
+    context.register_type(Foo, 20 * b"\x01")
+    context.register_type(Bar, 20 * b"\x02")
+    context.register_type(Baz, 20 * b"\x03")
+    context.register_type(Qux, 20 * b"\x04")
+    context.register_type(SubQux, 20 * b"\x05")
+    context.register_type(SubQuxPickle, 20 * b"\x05", pickle=True)
+    context.register_type(Exception, 20 * b"\x06")
+    context.register_type(CustomError, 20 * b"\x07")
+    context.register_type(Point, 20 * b"\x08")
+    context.register_type(NamedTupleExample, 20 * b"\x09")
+
+    # TODO(pcm): This is currently a workaround until arrow supports
+    # arbitrary precision integers. This is only called on long integers,
+    # see the associated case in the append method in python_to_arrow.cc
+    context.register_type(int, 20 * b"\x10", pickle=False,
+                          custom_serializer=lambda obj: str(obj),
+                          custom_deserializer=(
+                              lambda serialized_obj: int(serialized_obj)))
+
+    if (sys.version_info < (3, 0)):
+        deserializer = (
+            lambda serialized_obj: long(serialized_obj))  # noqa: E501,F821
+        context.register_type(long, 20 * b"\x11", pickle=False,  # noqa: E501,F821
+                              custom_serializer=lambda obj: str(obj),
+                              custom_deserializer=deserializer)
+
+    return context
+
+
+serialization_context = make_serialization_context()
 
 
 def serialization_roundtrip(value, f):
     f.seek(0)
-    pa.serialize_to(value, f)
+    pa.serialize_to(value, f, serialization_context)
     f.seek(0)
-    result = pa.deserialize_from(f, None)
+    result = pa.deserialize_from(f, None, serialization_context)
     assert_equal(value, result)
 
 

From 59fb8d0d2a669f9c48058d74909e81a1398afd01 Mon Sep 17 00:00:00 2001
From: tetsutaroendo <tetsutaro.endo@gmail.com>
Date: Tue, 29 Aug 2017 14:33:36 -0400
Subject: [PATCH 0970/1644] ARROW-1427: [GLib] Add arrow cpp link to readme

I added a link to "Arrow C++" to readme of Arrow GLib.
It is a little hard to find to install Arrow C++ before using Arrow GLib for the bigginer.

Author: tetsutaroendo <tetsutaro.endo@gmail.com>

Closes #1008 from tetsutaroendo/add_link_to_readme_of_c_glib and squashes the following commits:

c08966c1 [tetsutaroendo] Add arrow cpp link to readme of c_lib
---
 c_glib/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index fec877e236fed..5062ceab7a76f 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -19,7 +19,7 @@
 
 # Arrow GLib
 
-Arrow GLib is a wrapper library for Arrow C++. Arrow GLib provides C
+Arrow GLib is a wrapper library for [Arrow C++](https://github.com/apache/arrow/tree/master/cpp). Arrow GLib provides C
 API.
 
 Arrow GLib supports

From 19bbf12a21a3b5370496c43eecb2e9972a6f2b80 Mon Sep 17 00:00:00 2001
From: mnc <nobuya.sasaki@speee.jp>
Date: Tue, 29 Aug 2017 14:36:06 -0400
Subject: [PATCH 0971/1644] ARROW-1428: [C++] Append steps to clone source code
 to README.mb

Author: mnc <nobuya.sasaki@speee.jp>

Closes #1007 from mnc/append_how_to_build_cpp and squashes the following commits:

0ee8f6ce [mnc] append steps to clone source code to README.mb
---
 cpp/README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cpp/README.md b/cpp/README.md
index c05e03fece374..a50d0d82e5bca 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -51,6 +51,8 @@ If you are developing on Windows, see the [Windows developer guide][2].
 
 Simple debug build:
 
+    git clone https://github.com/apache/arrow.git
+    cd arrow/cpp
     mkdir debug
     cd debug
     cmake ..
@@ -58,6 +60,8 @@ Simple debug build:
 
 Simple release build:
 
+    git clone https://github.com/apache/arrow.git
+    cd arrow/cpp
     mkdir release
     cd release
     cmake .. -DCMAKE_BUILD_TYPE=Release

From 6af29317b22b31639e4796824266384f9d53abf7 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 29 Aug 2017 18:07:09 -0400
Subject: [PATCH 0972/1644] ARROW-1302: C++: Set MAKE to make if not defined

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1010 from xhochy/ARROW-1302 and squashes the following commits:

72fe9e5f [Uwe L. Korn] Only search for make on unix systems
95d3a310 [Uwe L. Korn] Use find_program to find make
b7b169e5 [Uwe L. Korn] ARROW-1302: C++: Set MAKE to make if not defined
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index a888e92392db6..e5d38b5f1f2a2 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -105,6 +105,13 @@ if (DEFINED ENV{ZSTD_HOME})
   set(ZSTD_HOME "$ENV{ZSTD_HOME}")
 endif()
 
+# Ensure that a default make is set
+if ("${MAKE}" STREQUAL "")
+    if (NOT MSVC)
+        find_program(MAKE make)
+    endif()
+endif()
+
 # ----------------------------------------------------------------------
 # Find pthreads
 

From af7829309c0a0ffb6799c6a31f44ed03589f21c3 Mon Sep 17 00:00:00 2001
From: Brecht Machiels <brecht@mos6581.org>
Date: Wed, 30 Aug 2017 01:08:08 -0400
Subject: [PATCH 0973/1644] ARROW-1429: [Python] Open common Parquet metadata
 file using passed file system

Fixes reported HDFS bug. Close #1009. Includes @brechtm bugfix and unit test to validate

Author: Brecht Machiels <brecht@mos6581.org>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1013 from wesm/ARROW-1429 and squashes the following commits:

c46fee7d [Wes McKinney] Refactor common metadata Parquet test so it can be run with either HDFS or local filesystem
31e5e26d [Brecht Machiels] Fix ARROW-1429
---
 python/pyarrow/parquet.py            |  3 ++-
 python/pyarrow/tests/test_hdfs.py    |  6 ++++++
 python/pyarrow/tests/test_parquet.py | 25 +++++++++++++++++--------
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 89dbf83ee3523..ac488aedb6336 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -532,7 +532,8 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
          self.metadata_path) = _make_manifest(path_or_paths, self.fs)
 
         if self.metadata_path is not None:
-            self.common_metadata = ParquetFile(self.metadata_path).metadata
+            with self.fs.open(self.metadata_path) as f:
+                self.common_metadata = ParquetFile(f).metadata
         else:
             self.common_metadata = None
 
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 79638f2c64d43..cf63008e69e68 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -262,6 +262,12 @@ def test_read_multiple_parquet_files(self):
                                .sort_values(by='index').reset_index(drop=True),
                                expected.to_pandas())
 
+    @test_parquet.parquet
+    def test_read_common_metadata_files(self):
+        tmpdir = pjoin(self.tmp_path, 'common-metadata-' + guid())
+        self.hdfs.mkdir(tmpdir)
+        test_parquet._test_read_common_metadata_files(self.hdfs, tmpdir)
+
 
 class TestLibHdfs(HdfsTestCases, unittest.TestCase):
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 8a20f4c469200..67944f98fdf18 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -832,8 +832,7 @@ def _visit_level(base_dir, level, part_keys):
     _visit_level(base_dir, 0, [])
 
 
-@parquet
-def test_read_common_metadata_files(tmpdir):
+def _test_read_common_metadata_files(fs, base_path):
     import pyarrow.parquet as pq
 
     N = 100
@@ -842,26 +841,36 @@ def test_read_common_metadata_files(tmpdir):
         'values': np.random.randn(N)
     }, columns=['index', 'values'])
 
-    base_path = str(tmpdir)
     data_path = pjoin(base_path, 'data.parquet')
 
     table = pa.Table.from_pandas(df)
-    _write_table(table, data_path)
+
+    with fs.open(data_path, 'wb') as f:
+        _write_table(table, f)
 
     metadata_path = pjoin(base_path, '_metadata')
-    pq.write_metadata(table.schema, metadata_path)
+    with fs.open(metadata_path, 'wb') as f:
+        pq.write_metadata(table.schema, f)
 
-    dataset = pq.ParquetDataset(base_path)
+    dataset = pq.ParquetDataset(base_path, filesystem=fs)
     assert dataset.metadata_path == metadata_path
 
-    common_schema = pq.read_metadata(data_path).schema
+    with fs.open(data_path) as f:
+        common_schema = pq.read_metadata(f).schema
     assert dataset.schema.equals(common_schema)
 
     # handle list of one directory
-    dataset2 = pq.ParquetDataset([base_path])
+    dataset2 = pq.ParquetDataset([base_path], filesystem=fs)
     assert dataset2.schema.equals(dataset.schema)
 
 
+@parquet
+def test_read_common_metadata_files(tmpdir):
+    base_path = str(tmpdir)
+    fs = LocalFileSystem.get_instance()
+    _test_read_common_metadata_files(fs, base_path)
+
+
 @parquet
 def test_read_schema(tmpdir):
     import pyarrow.parquet as pq

From 7510ae6ef51b0b27688028b9719e612971375e46 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 30 Aug 2017 09:38:29 +0200
Subject: [PATCH 0974/1644] ARROW-837: [Python] Add public
 pyarrow.allocate_buffer API. Rename FixedSizeBufferOutputStream

We may want to refactor to support random writes in `NativeFile`, so I've renamed accordingly

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1014 from wesm/ARROW-837 and squashes the following commits:

0fe22b1 [Wes McKinney] Restore readable assertion check
0d5ebe4 [Wes McKinney] Add public pyarrow.allocate_buffer API. Rename FixedSizeBufferOutputStream to FixedSizeBufferWriter for when we can support seeking at some point
---
 python/doc/source/api.rst                 |  1 +
 python/doc/source/plasma.rst              | 10 +++----
 python/examples/plasma/sorting/sort_df.py |  4 +--
 python/pyarrow/__init__.py                |  9 +++++--
 python/pyarrow/includes/libarrow.pxd      |  6 +++++
 python/pyarrow/io.pxi                     | 33 ++++++++++++++++++++---
 python/pyarrow/plasma.pyx                 |  2 +-
 python/pyarrow/tests/test_io.py           | 12 +++++++++
 python/pyarrow/tests/test_plasma.py       |  4 +--
 9 files changed, 66 insertions(+), 15 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 4761c7fe5f934..473b16d21ec49 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -158,6 +158,7 @@ Input / Output and Shared Memory
 .. autosummary::
    :toctree: generated/
 
+   allocate_buffer
    Buffer
    BufferReader
    BufferOutputStream
diff --git a/python/doc/source/plasma.rst b/python/doc/source/plasma.rst
index 9a5a74bfaf14f..74837b96c4831 100644
--- a/python/doc/source/plasma.rst
+++ b/python/doc/source/plasma.rst
@@ -235,14 +235,14 @@ API such as ``pyarrow.get_tensor_size``.
   buf = client.create(object_id, data_size)
 
 To write the Arrow ``Tensor`` object into the buffer, you can use Plasma to
-convert the ``memoryview`` buffer into a ``pyarrow.FixedSizeBufferOutputStream``
-object. A ``pyarrow.FixedSizeBufferOutputStream`` is a format suitable for Arrow's
+convert the ``memoryview`` buffer into a ``pyarrow.FixedSizeBufferWriter``
+object. A ``pyarrow.FixedSizeBufferWriter`` is a format suitable for Arrow's
 ``pyarrow.write_tensor``:
 
 .. code-block:: python
 
   # Write the tensor into the Plasma-allocated buffer
-  stream = pa.FixedSizeBufferOutputStream(buf)
+  stream = pa.FixedSizeBufferWriter(buf)
   pa.write_tensor(tensor, stream)  # Writes tensor's 552 bytes to Plasma stream
 
 To finish storing the Arrow object in Plasma, call ``seal``:
@@ -328,7 +328,7 @@ The DataFrame can now be written to the buffer as follows.
 .. code-block:: python
 
   # Write the PyArrow RecordBatch to Plasma
-  stream = pa.FixedSizeBufferOutputStream(buf)
+  stream = pa.FixedSizeBufferWriter(buf)
   stream_writer = pa.RecordBatchStreamWriter(stream, record_batch.schema)
   stream_writer.write_batch(record_batch)
   stream_writer.close()
@@ -414,7 +414,7 @@ You can test this with the following script:
   object_id = plasma.ObjectID(np.random.bytes(20))
   buf = client.create(object_id, pa.get_tensor_size(tensor))
 
-  stream = pa.FixedSizeBufferOutputStream(buf)
+  stream = pa.FixedSizeBufferWriter(buf)
   stream.set_memcopy_threads(4)
   a = time.time()
   pa.write_tensor(tensor, stream)
diff --git a/python/examples/plasma/sorting/sort_df.py b/python/examples/plasma/sorting/sort_df.py
index 0181ed729be29..0805592b9f967 100644
--- a/python/examples/plasma/sorting/sort_df.py
+++ b/python/examples/plasma/sorting/sort_df.py
@@ -39,7 +39,7 @@
 #     num_cols = 1
 
 client = None
-object_store_size = 2 * 10 ** 9 # 2 GB
+object_store_size = 2 * 10 ** 9  # 2 GB
 num_cores = 8
 num_rows = 200000
 num_cols = 2
@@ -69,7 +69,7 @@ def put_df(df):
     buf = client.create(object_id, data_size)
 
     # Write the serialized DataFrame to the object store
-    sink = pa.FixedSizeBufferOutputStream(buf)
+    sink = pa.FixedSizeBufferWriter(buf)
     stream_writer = pa.RecordBatchStreamWriter(sink, record_batch.schema)
     stream_writer.write_batch(record_batch)
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index ecdcfcdea3dc1..f97d356e7728f 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -68,10 +68,10 @@
                          Date32Value, Date64Value, TimestampValue)
 
 from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
-                         FixedSizeBufferOutputStream,
+                         FixedSizeBufferWriter,
                          Buffer, BufferReader, BufferOutputStream,
                          OSFile, MemoryMappedFile, memory_map,
-                         frombuffer,
+                         allocate_buffer, frombuffer,
                          memory_map, create_memory_map,
                          have_libhdfs, have_libhdfs3, MockOutputStream)
 
@@ -134,6 +134,11 @@ def _plasma_store_entry_point():
 
 from pyarrow.util import _deprecate_class
 
+FixedSizeBufferOutputStream = (
+    _deprecate_class('FixedSizeBufferOutputStream',
+                     'FixedSizeBufferWriter',
+                     FixedSizeBufferWriter, '0.7.0'))
+
 # Backwards compatibility with pyarrow < 0.6.0
 HdfsClient = _deprecate_class('HdfsClient', 'pyarrow.hdfs.connect',
                               hdfs.connect, '0.6.0')
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 98eda8bf5c2c7..e032448af9774 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -160,6 +160,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CMutableBuffer(const uint8_t* data, int64_t size)
         uint8_t* mutable_data()
 
+    CStatus AllocateBuffer(CMemoryPool* pool, const int64_t size,
+                           shared_ptr[CBuffer]* out)
+
+    CStatus AllocateResizableBuffer(CMemoryPool* pool, const int64_t size,
+                                    shared_ptr[ResizableBuffer]* out)
+
     cdef cppclass ResizableBuffer(CBuffer):
         CStatus Resize(int64_t nbytes)
         CStatus Reserve(int64_t nbytes)
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index b5858ab20ba7f..495e31b5a2176 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -151,6 +151,7 @@ cdef class NativeFile:
         """
         cdef int64_t offset
         self._assert_readable()
+
         with nogil:
             if whence == 0:
                 offset = position
@@ -534,7 +535,7 @@ cdef class OSFile(NativeFile):
         self.wr_file = <shared_ptr[OutputStream]> handle
 
 
-cdef class FixedSizeBufferOutputStream(NativeFile):
+cdef class FixedSizeBufferWriter(NativeFile):
 
     def __cinit__(self, Buffer buffer):
         self.wr_file.reset(new CFixedSizeBufferWriter(buffer.buffer))
@@ -580,6 +581,11 @@ cdef class Buffer:
         def __get__(self):
             return self.buffer.get().size()
 
+    property is_mutable:
+
+        def __get__(self):
+            return self.buffer.get().is_mutable()
+
     property parent:
 
         def __get__(self):
@@ -638,19 +644,40 @@ cdef class Buffer:
         return self.size
 
 
-cdef shared_ptr[PoolBuffer] allocate_buffer(CMemoryPool* pool):
+cdef shared_ptr[PoolBuffer] _allocate_buffer(CMemoryPool* pool):
     cdef shared_ptr[PoolBuffer] result
     result.reset(new PoolBuffer(pool))
     return result
 
 
+def allocate_buffer(int64_t size, MemoryPool pool=None):
+    """
+    Allocate mutable fixed-size buffer
+
+    Parameters
+    ----------
+    size : int
+        Number of bytes to allocate (plus internal padding)
+    pool : MemoryPool, optional
+        Uses default memory pool if not provided
+    """
+    cdef:
+        shared_ptr[CBuffer] buffer
+        CMemoryPool* cpool = maybe_unbox_memory_pool(pool)
+
+    with nogil:
+        check_status(AllocateBuffer(cpool, size, &buffer))
+
+    return pyarrow_wrap_buffer(buffer)
+
+
 cdef class BufferOutputStream(NativeFile):
 
     cdef:
         shared_ptr[PoolBuffer] buffer
 
     def __cinit__(self, MemoryPool memory_pool=None):
-        self.buffer = allocate_buffer(maybe_unbox_memory_pool(memory_pool))
+        self.buffer = _allocate_buffer(maybe_unbox_memory_pool(memory_pool))
         self.wr_file.reset(new CBufferOutputStream(
             <shared_ptr[ResizableBuffer]> self.buffer))
         self.is_readable = 0
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index b72e49b76eef8..cb19bea9966fe 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -390,7 +390,7 @@ cdef class PlasmaClient:
                                    else ObjectID.from_random())
         serialized = pyarrow.serialize(value)
         buffer = self.create(target_id, serialized.total_bytes)
-        stream = pyarrow.FixedSizeBufferOutputStream(buffer)
+        stream = pyarrow.FixedSizeBufferWriter(buffer)
         stream.set_memcopy_threads(4)
         serialized.write_to(stream)
         self.seal(target_id)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index d503ea22464d5..98c465adcdb3d 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -170,6 +170,18 @@ def test_buffer_numpy():
     assert array.base == buf
 
 
+def test_allocate_buffer():
+    buf = pa.allocate_buffer(100)
+    assert buf.size == 100
+    assert buf.is_mutable
+
+    bit = b'abcde'
+    writer = pa.FixedSizeBufferWriter(buf)
+    writer.write(bit)
+
+    assert buf.to_pybytes()[:5] == bit
+
+
 def test_buffer_memoryview_is_immutable():
     val = b'some data'
 
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index a831ef29a5e3c..9af97dfc97d9f 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -294,7 +294,7 @@ def test_store_arrow_objects(self):
         tensor = pa.Tensor.from_numpy(data)
         data_size = pa.get_tensor_size(tensor)
         buf = self.plasma_client.create(object_id, data_size)
-        stream = pa.FixedSizeBufferOutputStream(buf)
+        stream = pa.FixedSizeBufferWriter(buf)
         pa.write_tensor(tensor, stream)
         self.plasma_client.seal(object_id)
         # Read the arrow object.
@@ -320,7 +320,7 @@ def test_store_pandas_dataframe(self):
         object_id = plasma.ObjectID(np.random.bytes(20))
 
         buf = self.plasma_client.create(object_id, data_size)
-        stream = pa.FixedSizeBufferOutputStream(buf)
+        stream = pa.FixedSizeBufferWriter(buf)
         stream_writer = pa.RecordBatchStreamWriter(stream, record_batch.schema)
         stream_writer.write_batch(record_batch)
 

From d8c651ce2ffe17e7450b3f9f3e3642a9a875c751 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 30 Aug 2017 10:20:35 -0400
Subject: [PATCH 0975/1644] ARROW-1433: [C++] Simplify Array::Slice to be
 non-virtual

The virtual implementations are no longer needed now that we have the `ArrayData` data structure

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1011 from wesm/ARROW-1433 and squashes the following commits:

f3d30461 [Wes McKinney] Add unit test for slicing NullArray
357616f0 [Wes McKinney] Always set NullArray null count to array's length
4d6dfec4 [Wes McKinney] Simplify Array::Slice to be non-virtual
---
 cpp/src/arrow/array-test.cc |  4 ++
 cpp/src/arrow/array.cc      | 84 ++++++++-----------------------------
 cpp/src/arrow/array.h       | 42 ++++++-------------
 3 files changed, 34 insertions(+), 96 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index abfd82656a36c..0eb19d3bc90b1 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -135,6 +135,10 @@ TEST_F(TestArray, SliceRecomputeNullCount) {
 
   auto arr = std::make_shared<Int32Array>(16, data, nullptr, -1);
   ASSERT_EQ(0, arr->null_count());
+
+  auto null_arr = std::make_shared<NullArray>(10);
+  auto null_arr_sliced = null_arr->Slice(3, 6);
+  ASSERT_EQ(6, null_arr_sliced->null_count());
 }
 
 TEST_F(TestArray, TestIsNull) {
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 9fbadfe95bad6..720d5ece421e4 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -105,13 +105,23 @@ bool Array::RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
   return are_equal;
 }
 
-// Last two parameters are in-out parameters
-static inline void ConformSliceParams(int64_t array_offset, int64_t array_length,
-                                      int64_t* offset, int64_t* length) {
-  DCHECK_LE(*offset, array_length);
-  DCHECK_NE(offset, nullptr);
-  *length = std::min(array_length - *offset, *length);
-  *offset = array_offset + *offset;
+static inline std::shared_ptr<ArrayData> SliceData(const ArrayData& data, int64_t offset,
+                                                   int64_t length) {
+  DCHECK_LE(offset, data.length);
+  length = std::min(data.length - offset, length);
+  offset += data.offset;
+
+  auto new_data = data.ShallowCopy();
+  new_data->length = length;
+  new_data->offset = offset;
+  new_data->null_count = kUnknownNullCount;
+  return new_data;
+}
+
+std::shared_ptr<Array> Array::Slice(int64_t offset, int64_t length) const {
+  std::shared_ptr<Array> result;
+  DCHECK(MakeArray(SliceData(*data_, offset, length), &result).ok());
+  return result;
 }
 
 std::shared_ptr<Array> Array::Slice(int64_t offset) const {
@@ -125,28 +135,11 @@ std::string Array::ToString() const {
   return ss.str();
 }
 
-static inline std::shared_ptr<ArrayData> SliceData(const ArrayData& data, int64_t offset,
-                                                   int64_t length) {
-  ConformSliceParams(data.offset, data.length, &offset, &length);
-
-  auto new_data = data.ShallowCopy();
-  new_data->length = length;
-  new_data->offset = offset;
-  new_data->null_count = kUnknownNullCount;
-  return new_data;
-}
-
 NullArray::NullArray(int64_t length) {
   BufferVector buffers = {nullptr};
   SetData(std::make_shared<ArrayData>(null(), length, std::move(buffers), length));
 }
 
-std::shared_ptr<Array> NullArray::Slice(int64_t offset, int64_t length) const {
-  DCHECK_LE(offset, data_->length);
-  length = std::min(data_->length - offset, length);
-  return std::make_shared<NullArray>(length);
-}
-
 // ----------------------------------------------------------------------
 // Primitive array base
 
@@ -170,11 +163,6 @@ NumericArray<T>::NumericArray(const std::shared_ptr<internal::ArrayData>& data)
   DCHECK_EQ(data->type->id(), T::type_id);
 }
 
-template <typename T>
-std::shared_ptr<Array> NumericArray<T>::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<NumericArray<T>>(SliceData(*data_, offset, length));
-}
-
 // ----------------------------------------------------------------------
 // BooleanArray
 
@@ -188,10 +176,6 @@ BooleanArray::BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
                            int64_t offset)
     : PrimitiveArray(boolean(), length, data, null_bitmap, null_count, offset) {}
 
-std::shared_ptr<Array> BooleanArray::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<BooleanArray>(SliceData(*data_, offset, length));
-}
-
 // ----------------------------------------------------------------------
 // ListArray
 
@@ -254,12 +238,6 @@ std::shared_ptr<DataType> ListArray::value_type() const {
 
 std::shared_ptr<Array> ListArray::values() const { return values_; }
 
-std::shared_ptr<Array> ListArray::Slice(int64_t offset, int64_t length) const {
-  ConformSliceParams(data_->offset, data_->length, &offset, &length);
-  return std::make_shared<ListArray>(type(), length, value_offsets(), values(),
-                                     null_bitmap(), kUnknownNullCount, offset);
-}
-
 // ----------------------------------------------------------------------
 // String and binary
 
@@ -298,10 +276,6 @@ BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
 }
 
-std::shared_ptr<Array> BinaryArray::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<BinaryArray>(SliceData(*data_, offset, length));
-}
-
 StringArray::StringArray(const std::shared_ptr<internal::ArrayData>& data) {
   DCHECK_EQ(data->type->id(), Type::STRING);
   SetData(data);
@@ -314,10 +288,6 @@ StringArray::StringArray(int64_t length, const std::shared_ptr<Buffer>& value_of
     : BinaryArray(kString, length, value_offsets, data, null_bitmap, null_count, offset) {
 }
 
-std::shared_ptr<Array> StringArray::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<StringArray>(SliceData(*data_, offset, length));
-}
-
 // ----------------------------------------------------------------------
 // Fixed width binary
 
@@ -334,10 +304,6 @@ FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<DataType>& type
     : PrimitiveArray(type, length, data, null_bitmap, null_count, offset),
       byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {}
 
-std::shared_ptr<Array> FixedSizeBinaryArray::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<FixedSizeBinaryArray>(SliceData(*data_, offset, length));
-}
-
 const uint8_t* FixedSizeBinaryArray::GetValue(int64_t i) const {
   return raw_values_ + (i + data_->offset) * byte_width_;
 }
@@ -374,10 +340,6 @@ std::string DecimalArray::FormatValue(int64_t i) const {
   }
 }
 
-std::shared_ptr<Array> DecimalArray::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<DecimalArray>(SliceData(*data_, offset, length));
-}
-
 // ----------------------------------------------------------------------
 // Struct
 
@@ -404,10 +366,6 @@ std::shared_ptr<Array> StructArray::field(int pos) const {
   return result;
 }
 
-std::shared_ptr<Array> StructArray::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<StructArray>(SliceData(*data_, offset, length));
-}
-
 // ----------------------------------------------------------------------
 // UnionArray
 
@@ -449,10 +407,6 @@ std::shared_ptr<Array> UnionArray::child(int pos) const {
   return result;
 }
 
-std::shared_ptr<Array> UnionArray::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<UnionArray>(SliceData(*data_, offset, length));
-}
-
 // ----------------------------------------------------------------------
 // DictionaryArray
 
@@ -486,10 +440,6 @@ std::shared_ptr<Array> DictionaryArray::dictionary() const {
   return dict_type_->dictionary();
 }
 
-std::shared_ptr<Array> DictionaryArray::Slice(int64_t offset, int64_t length) const {
-  return std::make_shared<DictionaryArray>(SliceData(*data_, offset, length));
-}
-
 // ----------------------------------------------------------------------
 // Implement Array::Accept as inline visitor
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index f9f1e31be8ce7..57c1f271db815 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -145,8 +145,8 @@ struct ARROW_EXPORT ArrayData {
   std::vector<std::shared_ptr<ArrayData>> child_data;
 };
 
-Status ARROW_EXPORT MakeArray(const std::shared_ptr<ArrayData>& data,
-                              std::shared_ptr<Array>* out);
+ARROW_EXPORT
+Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out);
 
 }  // namespace internal
 
@@ -221,13 +221,13 @@ class ARROW_EXPORT Array {
   /// Construct a zero-copy slice of the array with the indicated offset and
   /// length
   ///
-  /// \param[in] offset the position of the first element in the constructed slice
-  /// \param[in] length the length of the slice. If there are not enough elements in the
-  /// array,
-  ///     the length will be adjusted accordingly
+  /// \param[in] offset the position of the first element in the constructed
+  /// slice
+  /// \param[in] length the length of the slice. If there are not enough
+  /// elements in the array, the length will be adjusted accordingly
   ///
   /// \return a new object wrapped in std::shared_ptr<Array>
-  virtual std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const = 0;
+  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const;
 
   /// Slice from offset until end of the array
   std::shared_ptr<Array> Slice(int64_t offset) const;
@@ -275,10 +275,14 @@ class ARROW_EXPORT NullArray : public FlatArray {
   using TypeClass = NullType;
 
   explicit NullArray(const std::shared_ptr<internal::ArrayData>& data) { SetData(data); }
-
   explicit NullArray(int64_t length);
 
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
+ private:
+  inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
+    null_bitmap_data_ = nullptr;
+    data->null_count = data->length;
+    data_ = data;
+  }
 };
 
 /// Base class for fixed-size logical types
@@ -334,8 +338,6 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
     return reinterpret_cast<const value_type*>(raw_values_) + data_->offset;
   }
 
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
   value_type Value(int64_t i) const { return raw_values()[i]; }
 
  protected:
@@ -352,8 +354,6 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
                const std::shared_ptr<Buffer>& null_bitmap = nullptr,
                int64_t null_count = 0, int64_t offset = 0);
 
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
   bool Value(int64_t i) const {
     return BitUtil::GetBit(reinterpret_cast<const uint8_t*>(raw_values_),
                            i + data_->offset);
@@ -409,8 +409,6 @@ class ARROW_EXPORT ListArray : public Array {
     return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
   }
 
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
  protected:
   void SetData(const std::shared_ptr<internal::ArrayData>& data);
   const int32_t* raw_value_offsets_;
@@ -460,8 +458,6 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
     return raw_value_offsets_[i + 1] - raw_value_offsets_[i];
   }
 
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
  protected:
   // For subclasses
   BinaryArray() {}
@@ -499,8 +495,6 @@ class ARROW_EXPORT StringArray : public BinaryArray {
     const uint8_t* str = GetValue(i, &nchars);
     return std::string(reinterpret_cast<const char*>(str), nchars);
   }
-
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
 // ----------------------------------------------------------------------
@@ -521,8 +515,6 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 
   int32_t byte_width() const { return byte_width_; }
 
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
  protected:
   inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
     this->PrimitiveArray::SetData(data);
@@ -544,8 +536,6 @@ class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray {
   explicit DecimalArray(const std::shared_ptr<internal::ArrayData>& data);
 
   std::string FormatValue(int64_t i) const;
-
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
 // ----------------------------------------------------------------------
@@ -565,8 +555,6 @@ class ARROW_EXPORT StructArray : public Array {
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   std::shared_ptr<Array> field(int pos) const;
-
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
 };
 
 // ----------------------------------------------------------------------
@@ -599,8 +587,6 @@ class ARROW_EXPORT UnionArray : public Array {
 
   std::shared_ptr<Array> child(int pos) const;
 
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
  protected:
   void SetData(const std::shared_ptr<internal::ArrayData>& data);
 
@@ -640,8 +626,6 @@ class ARROW_EXPORT DictionaryArray : public Array {
 
   const DictionaryType* dict_type() const { return dict_type_; }
 
-  std::shared_ptr<Array> Slice(int64_t offset, int64_t length) const override;
-
  private:
   void SetData(const std::shared_ptr<internal::ArrayData>& data);
 

From f4500255220fd1a4d68579e8b088fe3f315897de Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 30 Aug 2017 14:58:09 -0700
Subject: [PATCH 0976/1644] ARROW-1381: [Python] Use FixedSizeBufferWriter in
 SerializedPyObject.to_buffer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

With this setup:

```
import numpy as np
import pyarrow as pa

objects = [np.random.randn(500, 500) for i in range(400)]
serialized = pa.serialize(objects)
```

I have before:

```
In [3]: %timeit buf = serialized.to_buffer()
201 ms ± 1.87 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
```

and after:

```
In [4]: %timeit buf = serialized.to_buffer()
81.1 ms ± 233 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
```

I added an `nthreads` option but note that when the objects are small, multithreading makes things slower due to the overhead of launching threads. I think the 1MB threshold in `arrow/io/memory.cc` may be too small, we might do some benchmarking to find a better default crossover point for switching between parallel and serial memcpy:

```
In [2]: %timeit buf = serialized.to_buffer(nthreads=4)
134 ms ± 1.38 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
```

cc @pcmoritz @robertnishihara

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1017 from wesm/ARROW-1381 and squashes the following commits:

fbd0028 [Wes McKinney] Add unit test for SerializedPyObject.to_buffer
ab85230 [Wes McKinney] Add nthreads option for turning on multithreaded memcpy
db12072 [Wes McKinney] Use FixedSizeBufferWriter in SerializedPyObject.to_buffer
---
 python/pyarrow/serialization.pxi           | 9 ++++++---
 python/pyarrow/tests/test_serialization.py | 8 ++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index dc9bdaffa3bb5..5c0fbc6d8052c 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -181,13 +181,16 @@ cdef class SerializedPyObject:
         # also unpack the list the object was wrapped in in serialize
         return PyObject_to_object(result)[0]
 
-    def to_buffer(self):
+    def to_buffer(self, nthreads=1):
         """
         Write serialized data as Buffer
         """
-        sink = BufferOutputStream()
+        cdef Buffer output = allocate_buffer(self.total_bytes)
+        sink = FixedSizeBufferWriter(output)
+        if nthreads > 1:
+            sink.set_memcopy_threads(nthreads)
         self.write_to(sink)
-        return sink.get_result()
+        return output
 
 
 def serialize(object value, SerializationContext context=None):
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index d922576d36fa3..4e98bd5706aff 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -237,6 +237,14 @@ def test_primitive_serialization(large_memory_map):
             serialization_roundtrip(obj, mmap)
 
 
+def test_serialize_to_buffer():
+    for nthreads in [1, 4]:
+        for value in COMPLEX_OBJECTS:
+            buf = pa.serialize(value).to_buffer(nthreads=nthreads)
+            result = pa.deserialize(buf)
+            assert_equal(value, result)
+
+
 def test_complex_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for obj in COMPLEX_OBJECTS:

From 6b391f0d7f905da808bf443ac4b30e26ce7e0adc Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 30 Aug 2017 18:14:04 -0400
Subject: [PATCH 0977/1644] =?UTF-8?q?ARROW-1432:=20[C++]=C2=A0Build=20bund?=
 =?UTF-8?q?led=20jemalloc=20functions=20with=20private=20prefix?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1015 from xhochy/ARROW-1432 and squashes the following commits:

8ecebccc [Uwe L. Korn] ARROW-1432: [C++] Build bundled jemalloc functions with private prefix
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +-
 python/manylinux1/scripts/build_jemalloc.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index e5d38b5f1f2a2..194156cc65643 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -466,7 +466,7 @@ if (ARROW_JEMALLOC)
     set(JEMALLOC_VENDORED 1)
     ExternalProject_Add(jemalloc_ep
       URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
-      CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
+      CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
       ${EP_LOG_OPTIONS}
       BUILD_IN_SOURCE 1
       BUILD_COMMAND ${MAKE}
diff --git a/python/manylinux1/scripts/build_jemalloc.sh b/python/manylinux1/scripts/build_jemalloc.sh
index 1bf1a06b27ed1..5bd4aeb1529fc 100755
--- a/python/manylinux1/scripts/build_jemalloc.sh
+++ b/python/manylinux1/scripts/build_jemalloc.sh
@@ -19,7 +19,7 @@
 wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2
 tar xf jemalloc-4.4.0.tar.bz2
 pushd /jemalloc-4.4.0
-./configure
+./configure "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
 make -j5
 make install
 popd

From 78c952d2ec97862efe28c9666a4ff37a70ab78c0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 30 Aug 2017 20:45:25 -0400
Subject: [PATCH 0978/1644] ARROW-1351: Update CHANGELOG.md in 00-prepare.sh
 when creating release candidate

This will ensure the updated changelog is part of source releases

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1018 from wesm/ARROW-1351 and squashes the following commits:

9d60e367 [Wes McKinney] Add today's date after version with US / English month names
5b4b0764 [Wes McKinney] Update CHANGELOG.md in 00-prepare.sh when creating release candidate
---
 dev/release/00-prepare.sh                     |  9 ++-
 .../changelog.py}                             | 58 ++++++++++++++++---
 dev/release/update-changelog.sh               | 31 ++++++++++
 3 files changed, 87 insertions(+), 11 deletions(-)
 rename dev/{make_changelog.py => release/changelog.py} (73%)
 create mode 100755 dev/release/update-changelog.sh

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
index c8d7909ba06f9..6dc3be1a5f31c 100644
--- a/dev/release/00-prepare.sh
+++ b/dev/release/00-prepare.sh
@@ -27,7 +27,11 @@ if [ "$#" -eq 2 ]; then
   nextVersionSNAPSHOT=${nextVersion}-SNAPSHOT
   tag=apache-arrow-${version}
 
-  echo "prepare release ${version} rc ${rcnum} on tag ${tag} then reset to version ${nextVersionSNAPSHOT}" 
+  echo "Updating changelog for $version"
+  # Update changelog
+  $SOURCE_DIR/update-changelog.sh $version
+
+  echo "prepare release ${version} rc ${rcnum} on tag ${tag} then reset to version ${nextVersionSNAPSHOT}"
 
   cd "${SOURCE_DIR}/../../java"
 
@@ -38,8 +42,7 @@ if [ "$#" -eq 2 ]; then
 
   echo "Finish staging binary artifacts by running: sh dev/release/01-perform.sh"
 
-else  
+else
   echo "Usage: $0 <version> <nextVersion>"
   exit
 fi
-
diff --git a/dev/make_changelog.py b/dev/release/changelog.py
similarity index 73%
rename from dev/make_changelog.py
rename to dev/release/changelog.py
index b4b0070df8e2d..65881503abe27 100644
--- a/dev/make_changelog.py
+++ b/dev/release/changelog.py
@@ -21,8 +21,12 @@
 # requirements: pip install jira
 # Set $JIRA_USERNAME, $JIRA_PASSWORD environment variables
 
+from __future__ import print_function
+
 from collections import defaultdict
+from datetime import datetime
 from io import StringIO
+import locale
 import os
 import sys
 
@@ -39,6 +43,9 @@
                             basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
 
 
+locale.setlocale(locale.LC_ALL, 'en_US.utf8')
+
+
 def get_issues_for_version(version):
     jql = ("project=ARROW "
            "AND fixVersion='{0}' "
@@ -99,19 +106,54 @@ def format_changelog_website(issues, out):
         out.write('\n')
 
 
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print('Usage: make_changelog.py $FIX_VERSION [$IS_WEBSITE]')
+def get_changelog(version, for_website=False):
+    issues_for_version = get_issues_for_version(version)
 
     buf = StringIO()
 
-    for_website = len(sys.argv) > 2 and sys.argv[2] == '1'
-
-    issues_for_version = get_issues_for_version(sys.argv[1])
-
     if for_website:
         format_changelog_website(issues_for_version, buf)
     else:
         format_changelog_markdown(issues_for_version, buf)
 
-    print(buf.getvalue())
+    return buf.getvalue()
+
+
+def append_changelog(version, changelog_path):
+    new_changelog = get_changelog(version)
+
+    with open(changelog_path, 'r') as f:
+        old_changelog = f.readlines()
+
+    result = StringIO()
+    # Header
+    print(''.join(old_changelog[:18]), file=result)
+
+    # New version
+    today = datetime.today().strftime('%d %B %Y')
+    print('# Apache Arrow {0} ({1})'.format(version, today),
+          end='', file=result)
+    print('\n', file=result)
+    print(new_changelog.replace('_', '\_'),
+          end='', file=result)
+
+    # Prior versions
+    print(''.join(old_changelog[19:]), file=result)
+
+    with open(changelog_path, 'w') as f:
+        f.write(result.getvalue())
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('Usage: changelog.py $FIX_VERSION [$IS_WEBSITE] '
+              '[$CHANGELOG_TO_UPDATE]')
+
+    for_website = len(sys.argv) > 2 and sys.argv[2] == '1'
+
+    version = sys.argv[1]
+    if len(sys.argv) > 3:
+        changelog_path = sys.argv[3]
+        append_changelog(version, changelog_path)
+    else:
+        print(get_changelog(version, for_website=for_website))
diff --git a/dev/release/update-changelog.sh b/dev/release/update-changelog.sh
new file mode 100755
index 0000000000000..d47c90606c275
--- /dev/null
+++ b/dev/release/update-changelog.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+version=$1
+
+CHANGELOG=$SOURCE_DIR/../../CHANGELOG.md
+
+python $SOURCE_DIR/changelog.py $version 0 $CHANGELOG
+
+git add $CHANGELOG
+git commit -m "[Release] Update CHANGELOG.md for $version"

From 9ab0c955202f94fa930c9a3ed6247c7de2df87d4 Mon Sep 17 00:00:00 2001
From: hatappi <hatappi@hatappi.me>
Date: Thu, 31 Aug 2017 11:38:29 -0400
Subject: [PATCH 0979/1644] ARROW-1441:  [Site] Add Ruby to Flexible section

2017-05-08 C (GLib) Bindings, with support for Ruby, Lua, and more release.
There is a Ruby binding called red-arrow.
How about adding Ruby to the top page Flexible section?
https://arrow.apache.org/

issue link is [here](https://issues.apache.org/jira/browse/ARROW-1441)

Author: hatappi <hatappi@hatappi.me>

Closes #1021 from hatappi/ARROW-1441 and squashes the following commits:

3669f3a6 [hatappi] reflect review content
a4a32342 [hatappi] Add Ruby to top page Flexible section
---
 site/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/index.html b/site/index.html
index 224e5da358333..95d9f4df1e507 100644
--- a/site/index.html
+++ b/site/index.html
@@ -22,7 +22,7 @@ <h2>Fast</h2>
         </div>
         <div class="col-lg-4">
           <h2>Flexible</h2>
-          <p>Arrow acts as a new high-performance interface between various systems. It is also focused on supporting a wide variety of industry-standard programming languages. Java, C, C++, Python are underway and more languages are expected soon.</p>
+          <p>Arrow acts as a new high-performance interface between various systems. It is also focused on supporting a wide variety of industry-standard programming languages. Java, C, C++, Python, Ruby, and JavaScript implementations are in progress and more languages are welcome.</p>
         </div>
         <div class="col-lg-4">
           <h2>Standard</h2>

From c6295f3b74bcc2fa9ea1b9442f922bf564669b8e Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 1 Sep 2017 08:42:38 -0400
Subject: [PATCH 0980/1644] ARROW-1438: [Python] Pull serialization context
 through PlasmaClient put and get

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1019 from pcmoritz/plasma-serialization-context and squashes the following commits:

caf85165 [Philipp Moritz] fix
8e8e9011 [Philipp Moritz] add error handling
1a4fade0 [Philipp Moritz] add tests and fixes
2389659e [Philipp Moritz] Pull serialization context through PlasmaClient put and get
---
 cpp/src/arrow/python/python_to_arrow.cc |  8 +++++++-
 cpp/src/arrow/status.h                  |  7 +++++++
 python/pyarrow/__init__.py              |  4 +++-
 python/pyarrow/error.pxi                |  6 ++++++
 python/pyarrow/includes/common.pxd      |  1 +
 python/pyarrow/plasma.pyx               | 17 ++++++++++++-----
 python/pyarrow/serialization.pxi        | 17 ++++++++---------
 python/pyarrow/tests/test_plasma.py     | 22 ++++++++++++++++++++++
 8 files changed, 66 insertions(+), 16 deletions(-)

diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 9ae66dc35013a..c5cfd6e6cff1a 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -325,10 +325,16 @@ Status CallCustomCallback(PyObject* context, PyObject* method_name, PyObject* el
     std::stringstream ss;
     ScopedRef repr(PyObject_Repr(elem));
     RETURN_IF_PYERROR();
+#if PY_MAJOR_VERSION >= 3
     ScopedRef ascii(PyUnicode_AsASCIIString(repr.get()));
+    RETURN_IF_PYERROR();
     ss << "error while calling callback on " << PyBytes_AsString(ascii.get())
        << ": handler not registered";
-    return Status::NotImplemented(ss.str());
+#else
+    ss << "error while calling callback on " << PyString_AsString(repr.get())
+       << ": handler not registered";
+#endif
+    return Status::SerializationError(ss.str());
   } else {
     *result = PyObject_CallMethodObjArgs(context, method_name, elem, NULL);
     RETURN_IF_PYERROR();
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index a02752f21e4b9..d52addfc90700 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -71,6 +71,7 @@ enum class StatusCode : char {
   IOError = 5,
   UnknownError = 9,
   NotImplemented = 10,
+  SerializationError = 11,
   PlasmaObjectExists = 20,
   PlasmaObjectNonexistent = 21,
   PlasmaStoreFull = 22
@@ -125,6 +126,10 @@ class ARROW_EXPORT Status {
     return Status(StatusCode::IOError, msg);
   }
 
+  static Status SerializationError(const std::string& msg) {
+    return Status(StatusCode::SerializationError, msg);
+  }
+
   static Status PlasmaObjectExists(const std::string& msg) {
     return Status(StatusCode::PlasmaObjectExists, msg);
   }
@@ -147,6 +152,8 @@ class ARROW_EXPORT Status {
   bool IsTypeError() const { return code() == StatusCode::TypeError; }
   bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
   bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
+  // An object could not be serialized or deserialized.
+  bool IsSerializationError() const { return code() == StatusCode::SerializationError; }
   // An object with this object ID already exists in the plasma store.
   bool IsPlasmaObjectExists() const { return code() == StatusCode::PlasmaObjectExists; }
   // An object was requested that doesn't exist in the plasma store.
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index f97d356e7728f..a4a6ed449410a 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -86,13 +86,15 @@
                          ArrowMemoryError,
                          ArrowNotImplementedError,
                          ArrowTypeError,
+                         ArrowSerializationError,
                          PlasmaObjectExists)
 
 # Serialization
 from pyarrow.lib import (deserialize_from, deserialize,
                          serialize, serialize_to, read_serialized,
                          SerializedPyObject, SerializationContext,
-                         SerializationException, DeserializationException)
+                         SerializationCallbackError,
+                         DeserializationCallbackError)
 
 from pyarrow.filesystem import FileSystem, LocalFileSystem
 
diff --git a/python/pyarrow/error.pxi b/python/pyarrow/error.pxi
index 8793c4e90f6b4..2a213026c6550 100644
--- a/python/pyarrow/error.pxi
+++ b/python/pyarrow/error.pxi
@@ -60,6 +60,10 @@ class PlasmaStoreFull(ArrowException):
     pass
 
 
+class ArrowSerializationError(ArrowException):
+    pass
+
+
 cdef int check_status(const CStatus& status) nogil except -1:
     if status.ok():
         return 0
@@ -84,6 +88,8 @@ cdef int check_status(const CStatus& status) nogil except -1:
             raise PlasmaObjectNonexistent(message)
         elif status.IsPlasmaStoreFull():
             raise PlasmaStoreFull(message)
+        elif status.IsSerializationError():
+            raise ArrowSerializationError(message)
         else:
             message = frombytes(status.ToString())
             raise ArrowException(message)
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 6be08b0e59256..1bd840cc53ff2 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -51,6 +51,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsKeyError()
         c_bool IsNotImplemented()
         c_bool IsTypeError()
+        c_bool IsSerializationError()
         c_bool IsPlasmaObjectExists()
         c_bool IsPlasmaObjectNonexistent()
         c_bool IsPlasmaStoreFull()
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index cb19bea9966fe..bc0e94e64906e 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -370,7 +370,8 @@ cdef class PlasmaClient:
                                          object_buffers[i].metadata_size))
         return result
 
-    def put(self, object value, ObjectID object_id=None):
+    def put(self, object value, ObjectID object_id=None,
+            serialization_context=None):
         """
         Store a Python value into the object store.
 
@@ -381,6 +382,8 @@ cdef class PlasmaClient:
         object_id : ObjectID, default None
             If this is provided, the specified object ID will be used to refer
             to the object.
+        serialization_context : pyarrow.SerializationContext, default None
+            Custom serialization and deserialization context.
 
         Returns
         -------
@@ -388,7 +391,7 @@ cdef class PlasmaClient:
         """
         cdef ObjectID target_id = (object_id if object_id
                                    else ObjectID.from_random())
-        serialized = pyarrow.serialize(value)
+        serialized = pyarrow.serialize(value, serialization_context)
         buffer = self.create(target_id, serialized.total_bytes)
         stream = pyarrow.FixedSizeBufferWriter(buffer)
         stream.set_memcopy_threads(4)
@@ -396,7 +399,7 @@ cdef class PlasmaClient:
         self.seal(target_id)
         return target_id
 
-    def get(self, object_ids, int timeout_ms=-1):
+    def get(self, object_ids, int timeout_ms=-1, serialization_context=None):
         """
         Get one or more Python values from the object store.
 
@@ -409,6 +412,8 @@ cdef class PlasmaClient:
             The number of milliseconds that the get call should block before
             timing out and returning. Pass -1 if the call should block and 0
             if the call should return immediately.
+        serialization_context : pyarrow.SerializationContext, default None
+            Custom serialization and deserialization context.
 
         Returns
         -------
@@ -424,12 +429,14 @@ cdef class PlasmaClient:
                 # buffers[i] is None if this object was not available within
                 # the timeout
                 if buffers[i]:
-                    results.append(pyarrow.deserialize(buffers[i]))
+                    val = pyarrow.deserialize(buffers[i],
+                                              serialization_context)
+                    results.append(val)
                 else:
                     results.append(ObjectNotAvailable)
             return results
         else:
-            return self.get([object_ids], timeout_ms)[0]
+            return self.get([object_ids], timeout_ms, serialization_context)[0]
 
     def seal(self, ObjectID object_id):
         """
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index 5c0fbc6d8052c..f38845eea69e0 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -31,15 +31,15 @@ def is_named_tuple(cls):
     return all(type(n) == str for n in f)
 
 
-class SerializationException(Exception):
+class SerializationCallbackError(ArrowSerializationError):
     def __init__(self, message, example_object):
-        Exception.__init__(self, message)
+        ArrowSerializationError.__init__(self, message)
         self.example_object = example_object
 
 
-class DeserializationException(Exception):
+class DeserializationCallbackError(ArrowSerializationError):
     def __init__(self, message, type_id):
-        Exception.__init__(self, message)
+        ArrowSerializationError.__init__(self, message)
         self.type_id = type_id
 
 
@@ -89,10 +89,9 @@ cdef class SerializationContext:
 
     def _serialize_callback(self, obj):
         if type(obj) not in self.type_to_type_id:
-            raise SerializationException("pyarrow does not know how to "
-                                         "serialize objects of type {}."
-                                         .format(type(obj)),
-                                         obj)
+            raise SerializationCallbackError(
+                "pyarrow does not know how to "
+                "serialize objects of type {}.".format(type(obj)), obj)
         type_id = self.type_to_type_id[type(obj)]
         if type_id in self.types_to_pickle:
             serialized_obj = {"data": pickle.dumps(obj), "pickle": True}
@@ -107,7 +106,7 @@ cdef class SerializationContext:
             else:
                 msg = "We do not know how to serialize " \
                       "the object '{}'".format(obj)
-                raise SerializationException(msg, obj)
+                raise SerializationCallbackError(msg, obj)
         return dict(serialized_obj, **{"_pytype_": type_id})
 
     def _deserialize_callback(self, serialized_obj):
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index 9af97dfc97d9f..b73d92d14fd43 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -287,6 +287,28 @@ def test_put_and_get(self):
             [result] = self.plasma_client.get([object_id], timeout_ms=0)
             assert result == pa.plasma.ObjectNotAvailable
 
+    def test_put_and_get_serialization_context(self):
+
+        class CustomType(object):
+            def __init__(self, val):
+                self.val = val
+
+        val = CustomType(42)
+
+        with pytest.raises(pa.ArrowSerializationError):
+            self.plasma_client.put(val)
+
+        serialization_context = pa.SerializationContext()
+        serialization_context.register_type(CustomType, 20*b"\x00")
+
+        object_id = self.plasma_client.put(val, None, serialization_context)
+
+        with pytest.raises(pa.ArrowSerializationError):
+            result = self.plasma_client.get(object_id)
+
+        result = self.plasma_client.get(object_id, -1, serialization_context)
+        assert result.val == val.val
+
     def test_store_arrow_objects(self):
         data = np.random.randn(10, 4)
         # Write an arrow object.

From 8344f28f1db2648c89c688b030ed7f616b086d88 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 1 Sep 2017 13:56:30 -0400
Subject: [PATCH 0981/1644] ARROW-1434: [Python] Support datetime64[D] numpy
 arrays
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1020 from xhochy/ARROW-1434 and squashes the following commits:

e8ff8355 [Wes McKinney] Explicitly static cast in loop to avoid MSVC warning in std::copy
03d95c6f [Uwe L. Korn] Ignore MSVC warning
83168f1f [Uwe L. Korn] ninja lint
cb36ca88 [Uwe L. Korn] ninja format
c55e92e2 [Uwe L. Korn] ARROW-1434: [Python] Support datetime64[D] numpy arrays
---
 cpp/src/arrow/python/arrow_to_pandas.cc     |  5 +-
 cpp/src/arrow/python/numpy_convert.cc       | 13 ++---
 cpp/src/arrow/python/pandas_to_arrow.cc     | 58 ++++++++++++++++++---
 python/pyarrow/tests/test_convert_pandas.py | 47 +++++++++++++++++
 4 files changed, 108 insertions(+), 15 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 1f62ef86c0957..fe4d63b5c95a4 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -1454,6 +1454,7 @@ class ArrowDeserializer {
   Visit(const Type& type) {
     constexpr int TYPE = Type::type_id;
     using traits = internal::arrow_traits<TYPE>;
+    using c_type = typename Type::c_type;
 
     typedef typename traits::T T;
 
@@ -1465,10 +1466,10 @@ class ArrowDeserializer {
 
     for (int c = 0; c < data_.num_chunks(); c++) {
       const auto& arr = static_cast<const PrimitiveArray&>(*data_.chunk(c));
-      auto in_values = reinterpret_cast<const T*>(arr.raw_values());
+      auto in_values = reinterpret_cast<const c_type*>(arr.raw_values());
 
       for (int64_t i = 0; i < arr.length(); ++i) {
-        *out_values++ = arr.IsNull(i) ? na_value : in_values[i] / kShift;
+        *out_values++ = arr.IsNull(i) ? na_value : static_cast<T>(in_values[i]) / kShift;
       }
     }
     return Status::OK();
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 61192f313d29d..4c64cc4c32b4f 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -170,24 +170,25 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
     case NPY_DATETIME: {
       auto date_dtype =
           reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
-      TimeUnit::type unit;
       switch (date_dtype->meta.base) {
         case NPY_FR_s:
-          unit = TimeUnit::SECOND;
+          *out = timestamp(TimeUnit::SECOND);
           break;
         case NPY_FR_ms:
-          unit = TimeUnit::MILLI;
+          *out = timestamp(TimeUnit::MILLI);
           break;
         case NPY_FR_us:
-          unit = TimeUnit::MICRO;
+          *out = timestamp(TimeUnit::MICRO);
           break;
         case NPY_FR_ns:
-          unit = TimeUnit::NANO;
+          *out = timestamp(TimeUnit::NANO);
+          break;
+        case NPY_FR_D:
+          *out = date32();
           break;
         default:
           return Status::NotImplemented("Unsupported datetime64 time unit");
       }
-      *out = timestamp(unit);
     } break;
     default: {
       std::stringstream ss;
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 408aef3adc40c..8f6746af34b14 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -23,6 +23,7 @@
 
 #include "arrow/python/pandas_to_arrow.h"
 
+#include <algorithm>
 #include <cmath>
 #include <cstdint>
 #include <limits>
@@ -357,7 +358,7 @@ class PandasConverter {
     return VisitNative<T>();
   }
 
-  Status Visit(const Date32Type& type) { return VisitNative<Int32Type>(); }
+  Status Visit(const Date32Type& type) { return VisitNative<Date32Type>(); }
   Status Visit(const Date64Type& type) { return VisitNative<Int64Type>(); }
   Status Visit(const TimestampType& type) { return VisitNative<TimestampType>(); }
   Status Visit(const Time32Type& type) { return VisitNative<Int32Type>(); }
@@ -434,19 +435,19 @@ class PandasConverter {
   uint8_t* null_bitmap_data_;
 };
 
-template <typename T>
-void CopyStrided(T* input_data, int64_t length, int64_t stride, T* output_data) {
+template <typename T, typename T2>
+void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) {
   // Passing input_data as non-const is a concession to PyObject*
   int64_t j = 0;
   for (int64_t i = 0; i < length; ++i) {
-    output_data[i] = input_data[j];
+    output_data[i] = static_cast<T2>(input_data[j]);
     j += stride;
   }
 }
 
 template <>
-void CopyStrided<PyObject*>(PyObject** input_data, int64_t length, int64_t stride,
-                            PyObject** output_data) {
+void CopyStrided<PyObject*, PyObject*>(PyObject** input_data, int64_t length,
+                                       int64_t stride, PyObject** output_data) {
   int64_t j = 0;
   for (int64_t i = 0; i < length; ++i) {
     output_data[i] = input_data[j];
@@ -466,7 +467,11 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
 
   if (NumPyTypeSize(traits::npy_type) != NumPyTypeSize(type_num_compat)) {
-    return Status::NotImplemented("NumPy type casts not yet implemented");
+    std::stringstream ss;
+    ss << "NumPy type casts not yet implemented, type sizes differ: ";
+    ss << NumPyTypeSize(traits::npy_type) << " compared to "
+       << NumPyTypeSize(type_num_compat);
+    return Status::NotImplemented(ss.str());
   }
 
   if (is_strided()) {
@@ -486,6 +491,45 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   return Status::OK();
 }
 
+template <>
+inline Status PandasConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* data) {
+  // Handle LONGLONG->INT64 and other fun things
+  int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
+  int type_size = NumPyTypeSize(type_num_compat);
+
+  if (type_size == 4) {
+    // Source and target are INT32, so can refer to the main implementation.
+    return ConvertData<Int32Type>(data);
+  } else if (type_size == 8) {
+    // We need to scale down from int64 to int32
+    auto new_buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(new_buffer->Resize(sizeof(int32_t) * length_));
+
+    auto input = reinterpret_cast<const int64_t*>(PyArray_DATA(arr_));
+    auto output = reinterpret_cast<int32_t*>(new_buffer->mutable_data());
+
+    if (is_strided()) {
+      // Strided, must copy into new contiguous memory
+      const int64_t stride = PyArray_STRIDES(arr_)[0];
+      const int64_t stride_elements = stride / sizeof(int64_t);
+      CopyStrided(input, length_, stride_elements, output);
+    } else {
+      // TODO(wesm): int32 overflow checks
+      for (int64_t i= 0; i < length_; ++i) {
+        *output++ = static_cast<int32_t>(*input++);
+      }
+    }
+    *data = new_buffer;
+  } else {
+    std::stringstream ss;
+    ss << "Cannot convert NumPy array of element size ";
+    ss << type_size << " to a Date32 array";
+    return Status::NotImplemented(ss.str());
+  }
+
+  return Status::OK();
+}
+
 template <>
 inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>* data) {
   int64_t nbytes = BitUtil::BytesForBits(length_);
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 6e5e8e88ced3d..f5107c2dfe974 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -26,6 +26,7 @@
 import pytest
 
 import numpy as np
+import numpy.testing as npt
 
 import pandas as pd
 import pandas.util.testing as tm
@@ -821,6 +822,52 @@ def test_arrow_time_to_pandas(self):
 
         tm.assert_frame_equal(df, expected_df)
 
+    def _check_numpy_array_roundtrip(self, np_array):
+        arr = pa.Array.from_pandas(np_array)
+        result = arr.to_pandas()
+        npt.assert_array_equal(result, np_array)
+
+    def test_numpy_datetime64_columns(self):
+        datetime64_ns = np.array([
+                '2007-07-13T01:23:34.123456789',
+                None,
+                '2006-01-13T12:34:56.432539784',
+                '2010-08-13T05:46:57.437699912'],
+                dtype='datetime64[ns]')
+        self._check_numpy_array_roundtrip(datetime64_ns)
+
+        datetime64_us = np.array([
+                '2007-07-13T01:23:34.123456',
+                None,
+                '2006-01-13T12:34:56.432539',
+                '2010-08-13T05:46:57.437699'],
+                dtype='datetime64[us]')
+        self._check_numpy_array_roundtrip(datetime64_us)
+
+        datetime64_ms = np.array([
+                '2007-07-13T01:23:34.123',
+                None,
+                '2006-01-13T12:34:56.432',
+                '2010-08-13T05:46:57.437'],
+                dtype='datetime64[ms]')
+        self._check_numpy_array_roundtrip(datetime64_ms)
+
+        datetime64_s = np.array([
+                '2007-07-13T01:23:34',
+                None,
+                '2006-01-13T12:34:56',
+                '2010-08-13T05:46:57'],
+                dtype='datetime64[s]')
+        self._check_numpy_array_roundtrip(datetime64_s)
+
+        datetime64_d = np.array([
+                '2007-07-13',
+                None,
+                '2006-01-15',
+                '2010-08-19'],
+                dtype='datetime64[D]')
+        self._check_numpy_array_roundtrip(datetime64_d)
+
     def test_all_nones(self):
         def _check_series(s):
             converted = pa.Array.from_pandas(s)

From 28553b4f1e06cdcf8f88cc2ac0d9ae18913b59ab Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <golthiryus@gmail.com>
Date: Fri, 1 Sep 2017 13:57:07 -0400
Subject: [PATCH 0982/1644] ARROW-1443: [Java] Fixed a small bug on
 ArrowBuf.setBytes with unsliced ByteBuffers

There is a small bug on ArrowBuf at line 750. It says:
`udle.setBytes(index + offset, buf);`

But it should say:
`udle.setBytes(index + offset, newBuf);`

There is a wraparound: You can call the method with the already sliced buffer.

Author: Gonzalo Ortiz <golthiryus@gmail.com>
Author: Li Jin <ice.xelloss@gmail.com>

Closes #1022 from gortiz/master and squashes the following commits:

cdffbcc7 [Gonzalo Ortiz] Removed the old ArrowBufTest (which was renamed as TestArrowBuf)
d8aa00f5 [Gonzalo Ortiz] Renamed to ArrowBufTest to TestArrowBuf
1968a98e [Gonzalo Ortiz] Added license header
aa6c4579 [Li Jin] Add more tests for NullableVarChar and NullableVarBinary
e65e3804 [Gonzalo Ortiz] ARROW-1443 Reapplying the bug fix
41bcacc3 [Gonzalo Ortiz] ARROW-1443 Added a test to verify the bug
f70ceff9 [Gonzalo Ortiz] ARROW-1443 Reverted the previous commit to reproduce the bug
cef014f3 [Gonzalo Ortiz] [Java] Fixed a small bug on ArrowBuf.setBytes with unsliced ByteBuffers
---
 .../main/java/io/netty/buffer/ArrowBuf.java   |  2 +-
 .../java/io/netty/buffer/TestArrowBuf.java    | 85 +++++++++++++++++++
 .../org/apache/arrow/vector/TestUtils.java    |  8 +-
 .../apache/arrow/vector/TestValueVector.java  | 57 ++++++++++++-
 4 files changed, 148 insertions(+), 4 deletions(-)
 create mode 100644 java/memory/src/test/java/io/netty/buffer/TestArrowBuf.java

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index 6d17430c66966..e2bbe35480b66 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -747,7 +747,7 @@ public ArrowBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
         ByteBuffer newBuf = src.duplicate();
         newBuf.position(srcIndex);
         newBuf.limit(srcIndex + length);
-        udle.setBytes(index + offset, src);
+        udle.setBytes(index + offset, newBuf);
       }
     }
 
diff --git a/java/memory/src/test/java/io/netty/buffer/TestArrowBuf.java b/java/memory/src/test/java/io/netty/buffer/TestArrowBuf.java
new file mode 100644
index 0000000000000..88532203fc94c
--- /dev/null
+++ b/java/memory/src/test/java/io/netty/buffer/TestArrowBuf.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.netty.buffer;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.apache.arrow.memory.RootAllocator;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestArrowBuf {
+
+  private final static int MAX_ALLOCATION = 8 * 1024;
+  private static RootAllocator allocator;
+  
+  @BeforeClass
+  public static void beforeClass() {
+    allocator = new RootAllocator(MAX_ALLOCATION);
+  }
+  
+  @AfterClass
+  public static void afterClass() {
+    if (allocator != null) {
+      allocator.close();
+    }
+  }
+  
+  @Test
+  public void testSetBytesSliced() {
+    int arrLength = 64;
+    byte[] expecteds = new byte[arrLength];
+    for (int i = 0; i < expecteds.length; i++) {
+      expecteds[i] = (byte) i;
+    }
+    ByteBuffer data = ByteBuffer.wrap(expecteds);
+    try (ArrowBuf buf = allocator.buffer(expecteds.length)) {
+      buf.setBytes(0, data, 0, data.capacity());
+      
+      byte[] actuals = new byte[expecteds.length];
+      buf.getBytes(0, actuals);
+      assertArrayEquals(expecteds, actuals);
+    }
+  }
+  
+  @Test
+  public void testSetBytesUnsliced() {
+    int arrLength = 64;
+    byte[] arr = new byte[arrLength];
+    for (int i = 0; i < arr.length; i++) {
+      arr[i] = (byte) i;
+    }
+    ByteBuffer data = ByteBuffer.wrap(arr);
+    
+    int from = 10;
+    int to = arrLength;
+    byte[] expecteds = Arrays.copyOfRange(arr, from, to);
+    try (ArrowBuf buf = allocator.buffer(expecteds.length)) {
+      buf.setBytes(0, data, from, to - from);
+      
+      byte[] actuals = new byte[expecteds.length];
+      buf.getBytes(0, actuals);
+      assertArrayEquals(expecteds, actuals);
+    }
+  }
+  
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
index 15f511707d628..a148813090900 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
@@ -26,7 +26,13 @@
 public class TestUtils {
 
   public static NullableVarCharVector newNullableVarCharVector(String name, BufferAllocator allocator) {
-    return (NullableVarCharVector) FieldType.nullable(new ArrowType.Utf8()).createNewSingleVector(name, allocator, null);
+    return (NullableVarCharVector)
+        FieldType.nullable(new ArrowType.Utf8()).createNewSingleVector(name, allocator, null);
+  }
+
+  public static NullableVarBinaryVector newNullableVarBinaryVector(String name, BufferAllocator allocator) {
+    return (NullableVarBinaryVector)
+        FieldType.nullable(new ArrowType.Binary()).createNewSingleVector(name, allocator, null);
   }
 
   public static <T> T newVector(Class<T> c, String name, ArrowType type, BufferAllocator allocator) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index cb1fa89b272da..57119bfdae24e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -17,8 +17,10 @@
  */
 
 package org.apache.arrow.vector;
+import org.apache.arrow.vector.holders.VarCharHolder;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 
+import static org.apache.arrow.vector.TestUtils.newNullableVarBinaryVector;
 import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
 import static org.apache.arrow.vector.TestUtils.newVector;
 import static org.junit.Assert.assertArrayEquals;
@@ -27,7 +29,9 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
 
+import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
+import java.util.Arrays;
 import java.util.List;
 import java.util.ArrayList;
 
@@ -787,12 +791,12 @@ public void testNullableFixedType3() {
    * Covered types as of now
    *
    *  -- NullableVarCharVector
+   *  -- NullableVarBinaryVector
    *
    * TODO:
    *
    *  -- VarCharVector
    *  -- VarBinaryVector
-   *  -- NullableVarBinaryVector
    */
 
   @Test /* NullableVarCharVector */
@@ -806,17 +810,26 @@ public void testNullableVarType1() {
       m.set(0, STR1);
       m.set(1, STR2);
       m.set(2, STR3);
+      m.setSafe(3, STR3, 1, STR3.length - 1);
+      m.setSafe(4, STR3, 2, STR3.length - 2);
+      ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3);
+      m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1);
+      m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2);
 
       // Check the sample strings.
       final NullableVarCharVector.Accessor accessor = vector.getAccessor();
       assertArrayEquals(STR1, accessor.get(0));
       assertArrayEquals(STR2, accessor.get(1));
       assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6));
 
       // Ensure null value throws.
       boolean b = false;
       try {
-        vector.getAccessor().get(3);
+        vector.getAccessor().get(7);
       } catch (IllegalStateException e) {
         b = true;
       } finally {
@@ -825,6 +838,46 @@ public void testNullableVarType1() {
     }
   }
 
+  @Test /* NullableVarBinaryVector */
+  public void testNullableVarType2() {
+
+    // Create a new value vector for 1024 integers.
+    try (final NullableVarBinaryVector vector = newNullableVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) {
+      final NullableVarBinaryVector.Mutator m = vector.getMutator();
+      vector.allocateNew(1024 * 10, 1024);
+
+      m.set(0, STR1);
+      m.set(1, STR2);
+      m.set(2, STR3);
+      m.setSafe(3, STR3, 1, STR3.length - 1);
+      m.setSafe(4, STR3, 2, STR3.length - 2);
+      ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3);
+      m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1);
+      m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2);
+
+      // Check the sample strings.
+      final NullableVarBinaryVector.Accessor accessor = vector.getAccessor();
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6));
+
+      // Ensure null value throws.
+      boolean b = false;
+      try {
+        vector.getAccessor().get(7);
+      } catch (IllegalStateException e) {
+        b = true;
+      } finally {
+        assertTrue(b);
+      }
+    }
+  }
+
+
   /*
    * generic tests
    *

From 75d1f613cf99a822fa61859a7d081c8527c95500 Mon Sep 17 00:00:00 2001
From: fjetter <florian.jetter@blue-yonder.com>
Date: Fri, 1 Sep 2017 13:58:50 -0400
Subject: [PATCH 0983/1644] ARROW-1413: [C++] Add include-what-you-use
 configuration

This adds `iwyu` make target to run `include-what-you-use` on all changed files of the current branch. As suggested in the ticket https://issues.apache.org/jira/browse/ARROW-1413 I took the code of `apache-kudu` as a starting point and added a few modifications.

While it works fine locally, I'm struggling to get it to work (i.e. build is failing) on travis. Somehow the check isn't performed without raising an error. I expect that clang doesn't have the iwyu properly installed but I couldn't verify it, yet.

Also, the tool requires a `compile_commands.json` to be present including all the files to be checked which is why I added the `export CMAKE_EXPORT_COMPILE_COMMANDS=1` to the `travis_env_common.sh`
This may be an issue since we are partially compiling multiple times but never the full codebase which is why I included the make/ninja iwyu twice.

Author: fjetter <florian.jetter@blue-yonder.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1016 from fjetter/ARROW-1413 and squashes the following commits:

1a28703e [Wes McKinney] Add note to README about IWYU
c0dcccb1 [Wes McKinney] Add 'all' option to iwyu.sh for invoking iwyu on whole codebase
126f9f29 [Wes McKinney] Remove extra .clang-format file
e5bfa4ab [Wes McKinney] Remove iwyu from Travis CI files
59f4b3df [fjetter] set -x at beginning of file
31baf2e6 [fjetter] Set -e in iwyu.sh just before python script
f4f747d9 [fjetter] Add compilation database to env_common
970d0e64 [fjetter] Update comment in get-upstream-commit
57f29270 [fjetter] Add proper commit identifier
7fbc6ea5 [fjetter] Add iwyu build support dir to rat exclusion
41edfc28 [fjetter] Migrate kudu iwyu code to arrow
---
 ci/travis_env_common.sh                       |    2 +
 cpp/CMakeLists.txt                            |    7 +
 cpp/README.md                                 |   27 +-
 cpp/build-support/get-upstream-commit.sh      |   25 +
 cpp/build-support/iwyu/iwyu-filter.awk        |   91 +
 cpp/build-support/iwyu/iwyu.sh                |   72 +
 cpp/build-support/iwyu/iwyu_tool.py           |  278 +
 .../iwyu/mappings/boost-all-private.imp       | 4166 ++++++++++++
 cpp/build-support/iwyu/mappings/boost-all.imp | 5679 +++++++++++++++++
 .../iwyu/mappings/boost-extra.imp             |   23 +
 cpp/build-support/iwyu/mappings/gflags.imp    |   20 +
 cpp/build-support/iwyu/mappings/glog.imp      |   27 +
 cpp/build-support/iwyu/mappings/gtest.imp     |   26 +
 cpp/src/arrow/builder.cc                      |    1 +
 dev/release/rat_exclude_files.txt             |    1 +
 15 files changed, 10444 insertions(+), 1 deletion(-)
 create mode 100755 cpp/build-support/get-upstream-commit.sh
 create mode 100644 cpp/build-support/iwyu/iwyu-filter.awk
 create mode 100755 cpp/build-support/iwyu/iwyu.sh
 create mode 100755 cpp/build-support/iwyu/iwyu_tool.py
 create mode 100644 cpp/build-support/iwyu/mappings/boost-all-private.imp
 create mode 100644 cpp/build-support/iwyu/mappings/boost-all.imp
 create mode 100644 cpp/build-support/iwyu/mappings/boost-extra.imp
 create mode 100644 cpp/build-support/iwyu/mappings/gflags.imp
 create mode 100644 cpp/build-support/iwyu/mappings/glog.imp
 create mode 100644 cpp/build-support/iwyu/mappings/gtest.imp

diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index d84753125d5c4..ff49cdfc2bba0 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -34,6 +34,8 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
 export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
 export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
 
+export CMAKE_EXPORT_COMPILE_COMMANDS=1
+
 if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
   # C++ toolchain
   export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index cb7aa3a57b981..12a580374a2aa 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -580,6 +580,13 @@ if (${CLANG_TIDY_FOUND})
 
 endif()
 
+############################################################
+# "make iwyu" target
+############################################################
+if(UNIX)
+  add_custom_target(iwyu ${BUILD_SUPPORT_DIR}/iwyu/iwyu.sh)
+endif(UNIX)
+
 ############################################################
 # Linker and Dependencies
 ############################################################
diff --git a/cpp/README.md b/cpp/README.md
index a50d0d82e5bca..f37ec0368f410 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -178,6 +178,30 @@ We use the compiler definition `ARROW_NO_DEPRECATED_API` to disable APIs that
 have been deprecated. It is a good practice to compile third party applications
 with this flag to proactively catch and account for API changes.
 
+### Keeping includes clean with include-what-you-use
+
+We have provided a `build-support/iwyu/iwyu.sh` convenience script for invoking
+Google's [include-what-you-use][4] tool, also known as IWYU. This includes
+various suppressions for more informative output. After building IWYU
+(following instructions in the README), you can run it on all files by running:
+
+```shell
+CC="clang-4.0" CXX="clang++-4.0" cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
+../build-support/iwyu/iwyu.sh all
+```
+
+This presumes that `include-what-you-use` and `iwyu_tool.py` are in your
+`$PATH`. If you compiled IWYU using a different version of clang, then
+substitute the version number above accordingly. The results of this script are
+logged to a temporary file, whose location can be found by examining the shell
+output:
+
+```
+...
+Logging IWYU to /tmp/arrow-cpp-iwyu.gT7XXV
+...
+```
+
 ## Continuous Integration
 
 Pull requests are run through travis-ci for continuous integration.  You can avoid
@@ -202,4 +226,5 @@ both of these options would be used rarely.  Current known uses-cases whent hey
 
 [1]: https://brew.sh/
 [2]: https://github.com/apache/arrow/blob/master/cpp/apidoc/Windows.md
-[3]: https://google.github.io/styleguide/cppguide.html
\ No newline at end of file
+[3]: https://google.github.io/styleguide/cppguide.html
+[4]: https://github.com/include-what-you-use/include-what-you-use
\ No newline at end of file
diff --git a/cpp/build-support/get-upstream-commit.sh b/cpp/build-support/get-upstream-commit.sh
new file mode 100755
index 0000000000000..6584934cace1a
--- /dev/null
+++ b/cpp/build-support/get-upstream-commit.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script which tries to determine the most recent git hash in the current
+# branch which originates from master by checking for the 
+# 'ARROW-1234: Description` commit message
+set -e
+
+git log --grep='^ARROW-[0-9]*:.*' -n1 --pretty=format:%H
diff --git a/cpp/build-support/iwyu/iwyu-filter.awk b/cpp/build-support/iwyu/iwyu-filter.awk
new file mode 100644
index 0000000000000..a325d0adc8c63
--- /dev/null
+++ b/cpp/build-support/iwyu/iwyu-filter.awk
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# This is an awk script to process output from the include-what-you-use (IWYU)
+# tool. As of now, IWYU is of alpha quality and it gives many incorrect
+# recommendations -- obviously invalid or leading to compilation breakage.
+# Most of those can be silenced using appropriate IWYU pragmas, but it's not
+# the case for the auto-generated files.
+#
+# Also, it's possible to address invalid recommendation using mappings:
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/docs/IWYUMappings.md
+#
+# Usage:
+#  1. Run the CMake with -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=<iwyu_cmd_line>
+#
+#     The path to the IWYU binary should be absolute. The path to the binary
+#     and the command-line options should be separated by semicolon
+#     (that's for feeding it into CMake list variables).
+#
+#     E.g., from the build directory (line breaks are just for readability):
+#
+#     CC=../../thirdparty/clang-toolchain/bin/clang
+#     CXX=../../thirdparty/clang-toolchain/bin/clang++
+#     IWYU="`pwd`../../thirdparty/clang-toolchain/bin/include-what-you-use;\
+#       -Xiwyu;--mapping_file=`pwd`../../build-support/iwyu/mappings/map.imp"
+#
+#     ../../build-support/enable_devtoolset.sh \
+#       env CC=$CC CXX=$CXX \
+#       ../../thirdparty/installed/common/bin/cmake \
+#       -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=\"$IWYU\" \
+#       ../..
+#
+#     NOTE:
+#       Since the arrow code has some 'ifdef NDEBUG' directives, it's possible
+#       that IWYU would produce different results if run against release, not
+#       debug build. However, we plan to use the tool only with debug builds.
+#
+#  2. Run make, separating the output from the IWYU tool into a separate file
+#     (it's possible to use piping the output from the tool to the script
+#      but having a file is good for future reference, if necessary):
+#
+#     make -j$(nproc) 2>/tmp/iwyu.log
+#
+#  3. Process the output from the IWYU tool using the script:
+#
+#     awk -f ../../build-support/iwyu/iwyu-filter.awk /tmp/iwyu.log
+#
+
+BEGIN {
+  # This is the list of the files for which the suggestions from IWYU are
+  # ignored. Eventually, this list should become empty as soon as all the valid
+  # suggestions are addressed and invalid ones are taken care either by proper
+  # IWYU pragmas or adding special mappings (e.g. like boost mappings).
+  # muted["relative/path/to/file"]
+}
+
+# mute all suggestions for the auto-generated files
+/.*\.(pb|proxy|service)\.(cc|h) should (add|remove) these lines:/, /^$/ {
+  next
+}
+
+# mute suggestions for the explicitly specified files
+/.* should (add|remove) these lines:/ {
+  do_print = 1
+  for (path in muted) {
+    if (index($0, path)) {
+      do_print = 0
+      break
+    }
+  }
+}
+/^$/ {
+  if (do_print) print
+  do_print = 0
+}
+{ if (do_print) print }
diff --git a/cpp/build-support/iwyu/iwyu.sh b/cpp/build-support/iwyu/iwyu.sh
new file mode 100755
index 0000000000000..3ee5a120dce29
--- /dev/null
+++ b/cpp/build-support/iwyu/iwyu.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -x
+
+ROOT=$(cd $(dirname $BASH_SOURCE)/../../..; pwd)
+
+IWYU_LOG=$(mktemp -t arrow-cpp-iwyu.XXXXXX)
+trap "rm -f $IWYU_LOG" EXIT
+
+echo "Logging IWYU to $IWYU_LOG"
+
+# Build the list of updated files which are of IWYU interest.
+file_list_tmp=$(git diff --name-only \
+    $($ROOT/cpp/build-support/get-upstream-commit.sh) | grep -E '\.(c|cc|h)$')
+if [ -z "$file_list_tmp" ]; then
+  echo "IWYU verification: no updates on related files, declaring success"
+  exit 0
+fi
+
+# Adjust the path for every element in the list. The iwyu_tool.py normalizes
+# paths (via realpath) to match the records from the compilation database.
+IWYU_FILE_LIST=
+for p in $file_list_tmp; do
+  IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p"
+done
+
+IWYU_MAPPINGS_PATH="$ROOT/cpp/build-support/iwyu/mappings"
+IWYU_ARGS="\
+    --mapping_file=$IWYU_MAPPINGS_PATH/boost-all.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/boost-all-private.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/boost-extra.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/gflags.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/glog.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp"
+
+set -e
+
+if [ "$1" == "all" ]; then
+  python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p . -- \
+       $IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk | \
+       tee $IWYU_LOG
+else
+  python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST  -- \
+       $IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk | \
+       tee $IWYU_LOG
+fi
+
+if [ -s "$IWYU_LOG" ]; then
+  # The output is not empty: the changelist needs correction.
+  exit 1
+fi
+
+# The output is empty: the changelist looks good.
+echo "IWYU verification: the changes look good"
+exit 0
diff --git a/cpp/build-support/iwyu/iwyu_tool.py b/cpp/build-support/iwyu/iwyu_tool.py
new file mode 100755
index 0000000000000..fd52b2f0a1cba
--- /dev/null
+++ b/cpp/build-support/iwyu/iwyu_tool.py
@@ -0,0 +1,278 @@
+#!/usr/bin/env python
+
+# This file has been imported into the apache source tree from
+# the IWYU source tree as of version 0.8
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/iwyu_tool.py
+# and corresponding license has been added:
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/LICENSE.TXT
+#
+# ==============================================================================
+# LLVM Release License
+# ==============================================================================
+# University of Illinois/NCSA
+# Open Source License
+#
+# Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign.
+# All rights reserved.
+#
+# Developed by:
+#
+#     LLVM Team
+#
+#     University of Illinois at Urbana-Champaign
+#
+#     http://llvm.org
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal with
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+#     * Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimers.
+#
+#     * Redistributions in binary form must reproduce the above copyright notice,
+#       this list of conditions and the following disclaimers in the
+#       documentation and/or other materials provided with the distribution.
+#
+#     * Neither the names of the LLVM Team, University of Illinois at
+#       Urbana-Champaign, nor the names of its contributors may be used to
+#       endorse or promote products derived from this Software without specific
+#       prior written permission.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+# SOFTWARE.
+
+""" Driver to consume a Clang compilation database and invoke IWYU.
+
+Example usage with CMake:
+
+  # Unix systems
+  $ mkdir build && cd build
+  $ CC="clang" CXX="clang++" cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ...
+  $ iwyu_tool.py -p .
+
+  # Windows systems
+  $ mkdir build && cd build
+  $ cmake -DCMAKE_CXX_COMPILER="%VCINSTALLDIR%/bin/cl.exe" \
+    -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \
+    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+    -G Ninja ...
+  $ python iwyu_tool.py -p .
+
+See iwyu_tool.py -h for more details on command-line arguments.
+"""
+
+import os
+import sys
+import json
+import argparse
+import subprocess
+import re
+
+import logging
+
+logging.basicConfig(filename='iwyu.log')
+LOGGER = logging.getLogger("iwyu")
+
+
+def iwyu_formatter(output):
+    """ Process iwyu's output, basically a no-op. """
+    print('\n'.join(output))
+
+
+CORRECT_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$')
+SHOULD_ADD_RE = re.compile(r'^(.*?) should add these lines:$')
+SHOULD_REMOVE_RE = re.compile(r'^(.*?) should remove these lines:$')
+FULL_LIST_RE = re.compile(r'The full include-list for (.*?):$')
+END_RE = re.compile(r'^---$')
+LINES_RE = re.compile(r'^- (.*?)  // lines ([0-9]+)-[0-9]+$')
+
+
+GENERAL, ADD, REMOVE, LIST = range(4)
+
+
+def clang_formatter(output):
+    """ Process iwyu's output into something clang-like. """
+    state = (GENERAL, None)
+    for line in output:
+        match = CORRECT_RE.match(line)
+        if match:
+            print('%s:1:1: note: #includes/fwd-decls are correct', match.groups(1))
+            continue
+        match = SHOULD_ADD_RE.match(line)
+        if match:
+            state = (ADD, match.group(1))
+            continue
+        match = SHOULD_REMOVE_RE.match(line)
+        if match:
+            state = (REMOVE, match.group(1))
+            continue
+        match = FULL_LIST_RE.match(line)
+        if match:
+            state = (LIST, match.group(1))
+        elif END_RE.match(line):
+            state = (GENERAL, None)
+        elif not line.strip():
+            continue
+        elif state[0] == GENERAL:
+            print(line)
+        elif state[0] == ADD:
+            print('%s:1:1: error: add the following line', state[1])
+            print(line)
+        elif state[0] == REMOVE:
+            match = LINES_RE.match(line)
+            line_no = match.group(2) if match else '1'
+            print('%s:%s:1: error: remove the following line', state[1], line_no)
+            print(match.group(1))
+
+
+DEFAULT_FORMAT = 'iwyu'
+FORMATTERS = {
+    'iwyu': iwyu_formatter,
+    'clang': clang_formatter
+}
+
+def get_output(cwd, command):
+    """ Run the given command and return its output as a string. """
+    process = subprocess.Popen(command,
+                               cwd=cwd,
+                               shell=True,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.STDOUT)
+    return process.communicate()[0].decode("utf-8").splitlines()
+
+
+def run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter):
+    """ Rewrite compile_command to an IWYU command, and run it. """
+    compiler, _, args = compile_command.partition(' ')
+    if compiler.endswith('cl.exe'):
+        # If the compiler name is cl.exe, let IWYU be cl-compatible
+        clang_args = ['--driver-mode=cl']
+    else:
+        clang_args = []
+
+    iwyu_args = ['-Xiwyu ' + a for a in iwyu_args]
+    command = ['include-what-you-use'] + clang_args + iwyu_args
+    command = '%s %s' % (' '.join(command), args.strip())
+
+    if verbose:
+        print('%s:', command)
+
+    formatter(get_output(cwd, command))
+
+
+def main(compilation_db_path, source_files, verbose, formatter, iwyu_args):
+    """ Entry point. """
+    # Canonicalize compilation database path
+    if os.path.isdir(compilation_db_path):
+        compilation_db_path = os.path.join(compilation_db_path,
+                                           'compile_commands.json')
+
+    compilation_db_path = os.path.realpath(compilation_db_path)
+    if not os.path.isfile(compilation_db_path):
+        print('ERROR: No such file or directory: \'%s\'', compilation_db_path)
+        return 1
+
+    # Read compilation db from disk
+    with open(compilation_db_path, 'r') as fileobj:
+        compilation_db = json.load(fileobj)
+
+    # expand symlinks
+    for entry in compilation_db:
+        entry['file'] = os.path.realpath(entry['file'])
+
+    # Cross-reference source files with compilation database
+    source_files = [os.path.realpath(s) for s in source_files]
+    if not source_files:
+        # No source files specified, analyze entire compilation database
+        entries = compilation_db
+    else:
+        # Source files specified, analyze the ones appearing in compilation db,
+        # warn for the rest.
+        entries = []
+        for source in source_files:
+            matches = [e for e in compilation_db if e['file'] == source]
+            if matches:
+                entries.extend(matches)
+            else:
+                # TODO: As long as there is no complete compilation database available this check cannot be performed
+                pass
+                #print('WARNING: \'%s\' not found in compilation database.', source)
+
+    # Run analysis
+    try:
+        for entry in entries:
+            cwd, compile_command = entry['directory'], entry['command']
+            run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter)
+    except OSError as why:
+        print('ERROR: Failed to launch include-what-you-use: %s', why)
+        return 1
+
+    return 0
+
+
+def _bootstrap():
+    """ Parse arguments and dispatch to main(). """
+    # This hackery is necessary to add the forwarded IWYU args to the
+    # usage and help strings.
+    def customize_usage(parser):
+        """ Rewrite the parser's format_usage. """
+        original_format_usage = parser.format_usage
+        parser.format_usage = lambda: original_format_usage().rstrip() + \
+                              ' -- [<IWYU args>]' + os.linesep
+
+    def customize_help(parser):
+        """ Rewrite the parser's format_help. """
+        original_format_help = parser.format_help
+
+        def custom_help():
+            """ Customized help string, calls the adjusted format_usage. """
+            helpmsg = original_format_help()
+            helplines = helpmsg.splitlines()
+            helplines[0] = parser.format_usage().rstrip()
+            return os.linesep.join(helplines) + os.linesep
+
+        parser.format_help = custom_help
+
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description='Include-what-you-use compilation database driver.',
+        epilog='Assumes include-what-you-use is available on the PATH.')
+    customize_usage(parser)
+    customize_help(parser)
+
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Print IWYU commands')
+    parser.add_argument('-o', '--output-format', type=str,
+                        choices=FORMATTERS.keys(), default=DEFAULT_FORMAT,
+                        help='Output format (default: %s)' % DEFAULT_FORMAT)
+    parser.add_argument('-p', metavar='<build-path>', required=True,
+                        help='Compilation database path', dest='dbpath')
+    parser.add_argument('source', nargs='*',
+                        help='Zero or more source files to run IWYU on. '
+                        'Defaults to all in compilation database.')
+
+    def partition_args(argv):
+        """ Split around '--' into driver args and IWYU args. """
+        try:
+            double_dash = argv.index('--')
+            return argv[:double_dash], argv[double_dash+1:]
+        except ValueError:
+            return argv, []
+    argv, iwyu_args = partition_args(sys.argv[1:])
+    args = parser.parse_args(argv)
+
+    sys.exit(main(args.dbpath, args.source, args.verbose,
+                  FORMATTERS[args.output_format], iwyu_args))
+
+
+if __name__ == '__main__':
+    _bootstrap()
diff --git a/cpp/build-support/iwyu/mappings/boost-all-private.imp b/cpp/build-support/iwyu/mappings/boost-all-private.imp
new file mode 100644
index 0000000000000..133eef1137512
--- /dev/null
+++ b/cpp/build-support/iwyu/mappings/boost-all-private.imp
@@ -0,0 +1,4166 @@
+# This file has been imported into the arrow source tree from
+# the IWYU source tree as of version 0.8
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/boost-all-private.imp
+# and corresponding license has been added:
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/LICENSE.TXT
+#
+# ==============================================================================
+# LLVM Release License
+# ==============================================================================
+# University of Illinois/NCSA
+# Open Source License
+# 
+# Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign.
+# All rights reserved.
+# 
+# Developed by:
+# 
+#     LLVM Team
+# 
+#     University of Illinois at Urbana-Champaign
+# 
+#     http://llvm.org
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal with
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+# 
+#     * Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimers.
+# 
+#     * Redistributions in binary form must reproduce the above copyright notice,
+#       this list of conditions and the following disclaimers in the
+#       documentation and/or other materials provided with the distribution.
+# 
+#     * Neither the names of the LLVM Team, University of Illinois at
+#       Urbana-Champaign, nor the names of its contributors may be used to
+#       endorse or promote products derived from this Software without specific
+#       prior written permission.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+# SOFTWARE.
+
+[
+#grep -r '^ *# *include' boost/ | grep -e "boost/[^:]*/detail/.*hp*:" -e "boost/[^:]*/impl/.*hp*:" | grep -e "\:.*/detail/" -e "\:.*/impl/" | perl -nle 'm/^([^:]+).*["<]([^>]+)[">]/ && print qq@    { include: ["<$2>", private, "<$1>", private ] },@' | grep -e \\[\"\<boost/ | sort -u 
+#remove circular dependencies
+#                 boost/fusion/container/set/detail/value_of_data_impl.hpp with itself...
+#
+#    { include: ["<boost/numeric/odeint/integrate/detail/integrate_adaptive.hpp>", private, "<boost/numeric/odeint/integrate/detail/integrate_n_steps.hpp>", private ] },
+#    { include: ["<boost/numeric/odeint/integrate/detail/integrate_n_steps.hpp>", private, "<boost/numeric/odeint/integrate/detail/integrate_adaptive.hpp>", private ] },
+#
+#    { include: ["<boost/python/detail/type_list.hpp>", private, "<boost/python/detail/type_list_impl.hpp>", private ] },
+#    { include: ["<boost/python/detail/type_list.hpp>", private, "<boost/python/detail/type_list_impl_no_pts.hpp>", private ] },
+#    { include: ["<boost/python/detail/type_list_impl.hpp>", private, "<boost/python/detail/type_list.hpp>", private ] },
+#    { include: ["<boost/python/detail/type_list_impl_no_pts.hpp>", private, "<boost/python/detail/type_list.hpp>", private ] },
+
+    { include: ["<boost/accumulators/numeric/detail/function_n.hpp>", private, "<boost/accumulators/numeric/detail/function2.hpp>", private ] },
+    { include: ["<boost/accumulators/numeric/detail/function_n.hpp>", private, "<boost/accumulators/numeric/detail/function3.hpp>", private ] },
+    { include: ["<boost/accumulators/numeric/detail/function_n.hpp>", private, "<boost/accumulators/numeric/detail/function4.hpp>", private ] },
+    { include: ["<boost/algorithm/searching/detail/debugging.hpp>", private, "<boost/algorithm/searching/detail/bm_traits.hpp>", private ] },
+    { include: ["<boost/algorithm/string/detail/finder_regex.hpp>", private, "<boost/algorithm/string/detail/formatter_regex.hpp>", private ] },
+    { include: ["<boost/algorithm/string/detail/find_format_store.hpp>", private, "<boost/algorithm/string/detail/find_format_all.hpp>", private ] },
+    { include: ["<boost/algorithm/string/detail/find_format_store.hpp>", private, "<boost/algorithm/string/detail/find_format.hpp>", private ] },
+    { include: ["<boost/algorithm/string/detail/replace_storage.hpp>", private, "<boost/algorithm/string/detail/find_format_all.hpp>", private ] },
+    { include: ["<boost/algorithm/string/detail/replace_storage.hpp>", private, "<boost/algorithm/string/detail/find_format.hpp>", private ] },
+    { include: ["<boost/algorithm/string/detail/sequence.hpp>", private, "<boost/algorithm/string/detail/replace_storage.hpp>", private ] },
+    { include: ["<boost/algorithm/string/detail/util.hpp>", private, "<boost/algorithm/string/detail/formatter.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/archive_serializer_map.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/basic_archive_impl.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/basic_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/basic_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/basic_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/basic_oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/basic_pointer_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/basic_pointer_oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/basic_serializer_map.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/interface_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/interface_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/polymorphic_iarchive_route.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/detail/polymorphic_oarchive_route.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/archive_serializer_map.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/basic_archive_impl.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/basic_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/basic_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/basic_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/basic_oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/basic_pointer_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/basic_pointer_oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/basic_serializer_map.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/interface_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/interface_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/polymorphic_iarchive_route.hpp>", private ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/detail/polymorphic_oarchive_route.hpp>", private ] },
+    { include: ["<boost/archive/detail/archive_serializer_map.hpp>", private, "<boost/archive/detail/iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/archive_serializer_map.hpp>", private, "<boost/archive/detail/oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/detail/archive_serializer_map.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/detail/basic_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/detail/basic_oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/detail/basic_pointer_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/detail/basic_pointer_oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/detail/basic_serializer_map.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/detail/interface_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/detail/interface_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/detail/content_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/detail/forward_skeleton_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/detail/forward_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/detail/ignore_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/detail/mpi_datatype_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/detail/text_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_iarchive.hpp>", private, "<boost/archive/detail/common_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_iarchive.hpp>", private, "<boost/archive/detail/iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_iserializer.hpp>", private, "<boost/archive/detail/iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_oarchive.hpp>", private, "<boost/archive/detail/common_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_oarchive.hpp>", private, "<boost/archive/detail/oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_oserializer.hpp>", private, "<boost/archive/detail/oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_pointer_iserializer.hpp>", private, "<boost/archive/detail/common_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_pointer_iserializer.hpp>", private, "<boost/archive/detail/iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_pointer_oserializer.hpp>", private, "<boost/archive/detail/oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_serializer.hpp>", private, "<boost/archive/detail/basic_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_serializer.hpp>", private, "<boost/archive/detail/basic_oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_serializer.hpp>", private, "<boost/archive/detail/basic_pointer_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/basic_serializer.hpp>", private, "<boost/archive/detail/basic_pointer_oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/check.hpp>", private, "<boost/archive/detail/iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/check.hpp>", private, "<boost/archive/detail/oserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/common_iarchive.hpp>", private, "<boost/mpi/detail/forward_skeleton_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/common_oarchive.hpp>", private, "<boost/mpi/detail/forward_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/common_oarchive.hpp>", private, "<boost/mpi/detail/ignore_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/detail/auto_link_archive.hpp>", private ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/detail/auto_link_warchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/detail/basic_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/detail/basic_iserializer.hpp>", private ] },
+    { include: ["<boost/archive/detail/interface_iarchive.hpp>", private, "<boost/archive/detail/common_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/interface_iarchive.hpp>", private, "<boost/mpi/detail/forward_skeleton_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/interface_oarchive.hpp>", private, "<boost/archive/detail/common_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/interface_oarchive.hpp>", private, "<boost/mpi/detail/forward_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/iserializer.hpp>", private, "<boost/archive/detail/interface_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/iserializer.hpp>", private, "<boost/mpi/detail/forward_skeleton_iarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/oserializer.hpp>", private, "<boost/archive/detail/interface_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/oserializer.hpp>", private, "<boost/mpi/detail/forward_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/oserializer.hpp>", private, "<boost/mpi/detail/ignore_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/oserializer.hpp>", private, "<boost/mpi/detail/mpi_datatype_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/mpi/detail/content_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/mpi/detail/mpi_datatype_oarchive.hpp>", private ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/mpi/detail/text_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/completion_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/handler_alloc_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/handler_cont_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/handler_invoke_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/impl/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/impl/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/impl/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/signal_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/wait_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/win_object_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/winrt_ssocket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/addressof.hpp>", private, "<boost/asio/detail/winrt_utils.hpp>", private ] },
+    { include: ["<boost/asio/detail/array_fwd.hpp>", private, "<boost/asio/detail/buffer_sequence_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/array_fwd.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/array_fwd.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/array_fwd.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/array_fwd.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/assert.hpp>", private, "<boost/asio/detail/buffered_stream_storage.hpp>", private ] },
+    { include: ["<boost/asio/detail/assert.hpp>", private, "<boost/asio/detail/hash_map.hpp>", private ] },
+    { include: ["<boost/asio/detail/assert.hpp>", private, "<boost/asio/detail/posix_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/assert.hpp>", private, "<boost/asio/detail/std_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/assert.hpp>", private, "<boost/asio/detail/win_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/assert.hpp>", private, "<boost/asio/ssl/old/detail/openssl_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/atomic_count.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/atomic_count.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/atomic_count.hpp>", private, "<boost/asio/detail/winrt_async_manager.hpp>", private ] },
+    { include: ["<boost/asio/detail/base_from_completion_cond.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/base_from_completion_cond.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/base_from_completion_cond.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/base_from_completion_cond.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/null_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/detail/wrapped_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/ssl/detail/read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/ssl/detail/write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/ssl/old/detail/openssl_stream_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/call_stack.hpp>", private, "<boost/asio/detail/impl/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/call_stack.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/call_stack.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/completion_handler.hpp>", private, "<boost/asio/detail/impl/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/completion_handler.hpp>", private, "<boost/asio/detail/impl/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/completion_handler.hpp>", private, "<boost/asio/detail/impl/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/addressof.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/array_fwd.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/array.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/assert.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/atomic_count.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/base_from_completion_cond.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/bind_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/buffered_stream_storage.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/buffer_resize_guard.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/buffer_sequence_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/call_stack.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/completion_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/consuming_buffers.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/cstdint.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/date_time_fwd.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/dependent_type.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/descriptor_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/eventfd_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/event.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/function.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/gcc_arm_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/gcc_hppa_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/gcc_sync_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/gcc_x86_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/handler_alloc_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/handler_cont_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/handler_invoke_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/handler_tracking.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/handler_type_requirements.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/hash_map.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/impl/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/impl/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/impl/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/impl/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/impl/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/io_control.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/keyword_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/limits.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/local_free_on_block_exit.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/macos_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/noncopyable.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/null_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/null_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/null_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/null_signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/null_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/null_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/null_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/null_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/old_win_sdk_compat.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/pipe_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/posix_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/posix_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/posix_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/posix_signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/posix_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/posix_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/posix_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactor_fwd.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactor_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/reactor_op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/scoped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/shared_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/signal_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/signal_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/signal_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/socket_holder.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/socket_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/socket_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/socket_types.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/solaris_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/std_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/std_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/std_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/std_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/throw_error.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/throw_exception.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/timer_queue_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/timer_queue_set.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/timer_scheduler_fwd.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/type_traits.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/variadic_templates.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/wait_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/wait_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/weak_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/wince_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_object_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_async_manager.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_async_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_ssocket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winrt_utils.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/winsock_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/detail/win_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/generic/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/impl/spawn.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/impl/use_future.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/local/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/buffered_handshake_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/engine.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/handshake_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/io.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/openssl_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/openssl_types.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/password_callback.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/shutdown_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/stream_core.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/verify_callback.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/detail/write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/impl/context.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/old/detail/openssl_context_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/old/detail/openssl_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/old/detail/openssl_stream_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/consuming_buffers.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/consuming_buffers.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/consuming_buffers.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/consuming_buffers.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/consuming_buffers.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/cstdint.hpp>", private, "<boost/asio/detail/chrono_time_traits.hpp>", private ] },
+    { include: ["<boost/asio/detail/cstdint.hpp>", private, "<boost/asio/detail/handler_tracking.hpp>", private ] },
+    { include: ["<boost/asio/detail/cstdint.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/cstdint.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/date_time_fwd.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/dependent_type.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/dependent_type.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/dependent_type.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/dependent_type.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/descriptor_ops.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/descriptor_ops.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/descriptor_ops.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/descriptor_ops.hpp>", private, "<boost/asio/detail/reactive_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/descriptor_read_op.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/descriptor_write_op.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/dev_poll_reactor.hpp>", private, "<boost/asio/detail/reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/dev_poll_reactor.hpp>", private, "<boost/asio/detail/timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/epoll_reactor.hpp>", private, "<boost/asio/detail/reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/epoll_reactor.hpp>", private, "<boost/asio/detail/timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/eventfd_select_interrupter.hpp>", private, "<boost/asio/detail/select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/event.hpp>", private, "<boost/asio/detail/task_io_service_thread_info.hpp>", private ] },
+    { include: ["<boost/asio/detail/event.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/fd_set_adapter.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/completion_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/impl/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/impl/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/impl/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/signal_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/wait_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/fenced_block.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/gcc_arm_fenced_block.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/gcc_hppa_fenced_block.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/gcc_sync_fenced_block.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/gcc_x86_fenced_block.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/bind_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/completion_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/impl/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/impl/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/impl/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/reactive_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/signal_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/wait_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/win_object_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/detail/wrapped_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/buffered_read_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/buffered_write_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/spawn.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_alloc_helpers.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/detail/bind_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/detail/impl/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/detail/wrapped_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/buffered_read_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/buffered_write_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/spawn.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_cont_helpers.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/bind_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/completion_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/impl/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/impl/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/impl/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/reactive_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/signal_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/wait_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/detail/wrapped_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/buffered_read_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/buffered_write_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/spawn.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_invoke_helpers.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_tracking.hpp>", private, "<boost/asio/detail/task_io_service_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_tracking.hpp>", private, "<boost/asio/detail/win_iocp_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/buffered_read_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/buffered_write_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/hash_map.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/hash_map.hpp>", private, "<boost/asio/detail/reactor_op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/buffer_sequence_adapter.ipp>", private, "<boost/asio/detail/buffer_sequence_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/buffer_sequence_adapter.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/descriptor_ops.ipp>", private, "<boost/asio/detail/descriptor_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/descriptor_ops.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/dev_poll_reactor.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/dev_poll_reactor.ipp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/dev_poll_reactor.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/epoll_reactor.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/epoll_reactor.ipp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/epoll_reactor.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/eventfd_select_interrupter.ipp>", private, "<boost/asio/detail/eventfd_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/eventfd_select_interrupter.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/handler_tracking.ipp>", private, "<boost/asio/detail/handler_tracking.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/handler_tracking.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/kqueue_reactor.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/kqueue_reactor.ipp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/kqueue_reactor.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/pipe_select_interrupter.ipp>", private, "<boost/asio/detail/pipe_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/pipe_select_interrupter.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/posix_event.ipp>", private, "<boost/asio/detail/posix_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/posix_event.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/posix_mutex.ipp>", private, "<boost/asio/detail/posix_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/posix_mutex.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/posix_thread.ipp>", private, "<boost/asio/detail/posix_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/posix_thread.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/posix_tss_ptr.ipp>", private, "<boost/asio/detail/posix_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/posix_tss_ptr.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/reactive_descriptor_service.ipp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/reactive_descriptor_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/reactive_serial_port_service.ipp>", private, "<boost/asio/detail/reactive_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/reactive_serial_port_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/reactive_socket_service_base.ipp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/reactive_socket_service_base.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/resolver_service_base.ipp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/resolver_service_base.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/select_reactor.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/select_reactor.ipp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/select_reactor.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/service_registry.hpp>", private, "<boost/asio/detail/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/service_registry.ipp>", private, "<boost/asio/detail/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/service_registry.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/signal_set_service.ipp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/signal_set_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/socket_ops.ipp>", private, "<boost/asio/detail/socket_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/socket_ops.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/socket_select_interrupter.ipp>", private, "<boost/asio/detail/socket_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/socket_select_interrupter.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/strand_service.hpp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/strand_service.ipp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/strand_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/task_io_service.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/task_io_service.ipp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/task_io_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/throw_error.ipp>", private, "<boost/asio/detail/throw_error.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/throw_error.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/timer_queue_ptime.ipp>", private, "<boost/asio/detail/timer_queue_ptime.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/timer_queue_ptime.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/timer_queue_set.ipp>", private, "<boost/asio/detail/timer_queue_set.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/timer_queue_set.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_event.ipp>", private, "<boost/asio/detail/win_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_event.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_handle_service.ipp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_handle_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_io_service.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_io_service.ipp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_io_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_serial_port_service.ipp>", private, "<boost/asio/detail/win_iocp_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_serial_port_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_socket_service_base.ipp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_iocp_socket_service_base.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_mutex.ipp>", private, "<boost/asio/detail/win_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_mutex.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_object_handle_service.ipp>", private, "<boost/asio/detail/win_object_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_object_handle_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/winrt_ssocket_service_base.ipp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/winrt_ssocket_service_base.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/winrt_timer_scheduler.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/winrt_timer_scheduler.ipp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/winrt_timer_scheduler.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/winsock_init.ipp>", private, "<boost/asio/detail/winsock_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/winsock_init.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_static_mutex.ipp>", private, "<boost/asio/detail/win_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_static_mutex.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_thread.ipp>", private, "<boost/asio/detail/win_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_thread.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_tss_ptr.ipp>", private, "<boost/asio/detail/win_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/impl/win_tss_ptr.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/detail/keyword_tss_ptr.hpp>", private, "<boost/asio/detail/tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/kqueue_reactor.hpp>", private, "<boost/asio/detail/reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/kqueue_reactor.hpp>", private, "<boost/asio/detail/timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/buffer_resize_guard.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/consuming_buffers.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/macos_fenced_block.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/mutex.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/call_stack.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/handler_alloc_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/hash_map.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/keyword_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/local_free_on_block_exit.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/null_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/null_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/null_signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/null_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/null_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/object_pool.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/posix_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/posix_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/posix_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/posix_signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/posix_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/posix_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/reactor_op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/scoped_lock.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/socket_holder.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/std_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/std_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/std_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/std_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/thread_info_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/timer_queue_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/wince_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/win_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/win_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/win_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/win_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/detail/win_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/impl/spawn.hpp>", private ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/ssl/detail/openssl_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/null_event.hpp>", private, "<boost/asio/detail/event.hpp>", private ] },
+    { include: ["<boost/asio/detail/null_fenced_block.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/null_mutex.hpp>", private, "<boost/asio/detail/mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/null_reactor.hpp>", private, "<boost/asio/detail/reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/null_signal_blocker.hpp>", private, "<boost/asio/detail/signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/null_static_mutex.hpp>", private, "<boost/asio/detail/static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/null_thread.hpp>", private, "<boost/asio/detail/thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/null_tss_ptr.hpp>", private, "<boost/asio/detail/tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/object_pool.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/object_pool.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/old_win_sdk_compat.hpp>", private, "<boost/asio/detail/socket_types.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/completion_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/reactor_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/signal_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/timer_queue_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/wait_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/operation.hpp>", private, "<boost/asio/detail/winrt_async_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/reactor_op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/task_io_service_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/task_io_service_thread_info.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/timer_queue_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/win_iocp_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/op_queue.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/pipe_select_interrupter.hpp>", private, "<boost/asio/detail/select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/base_from_completion_cond.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/bind_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/buffered_stream_storage.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/buffer_resize_guard.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/buffer_sequence_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/call_stack.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/chrono_time_traits.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/completion_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/consuming_buffers.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/dependent_type.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/descriptor_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/eventfd_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/gcc_arm_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/gcc_hppa_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/gcc_sync_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/gcc_x86_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/handler_alloc_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/handler_cont_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/handler_invoke_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/handler_tracking.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/hash_map.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/impl/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/io_control.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/keyword_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/local_free_on_block_exit.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/macos_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/noncopyable.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/null_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/object_pool.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/old_win_sdk_compat.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/pipe_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/posix_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/posix_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/posix_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/posix_signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/posix_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/posix_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/posix_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactor_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/reactor_op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/scoped_lock.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/scoped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/signal_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/signal_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/signal_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/socket_holder.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/socket_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/socket_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/socket_types.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/solaris_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/std_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/std_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/std_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/std_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/task_io_service_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/task_io_service_thread_info.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/thread_info_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/throw_error.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/timer_queue_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/timer_queue_ptime.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/timer_queue_set.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/wait_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/wait_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/wince_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_iocp_thread_info.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_object_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_async_manager.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_async_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_ssocket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winrt_utils.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/winsock_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/win_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/detail/wrapped_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/generic/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/buffered_read_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/buffered_write_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/serial_port_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/spawn.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/use_future.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/impl/address.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/impl/address_v4.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/impl/address_v6.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/impl/basic_endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/local/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/buffered_handshake_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/engine.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/handshake_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/io.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/openssl_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/password_callback.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/shutdown_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/stream_core.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/verify_callback.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/detail/write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/impl/context.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/old/detail/openssl_context_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/old/detail/openssl_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/old/detail/openssl_stream_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/posix_event.hpp>", private, "<boost/asio/detail/event.hpp>", private ] },
+    { include: ["<boost/asio/detail/posix_fd_set_adapter.hpp>", private, "<boost/asio/detail/fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/posix_mutex.hpp>", private, "<boost/asio/detail/mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/posix_signal_blocker.hpp>", private, "<boost/asio/detail/signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/posix_static_mutex.hpp>", private, "<boost/asio/detail/static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/posix_thread.hpp>", private, "<boost/asio/detail/thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/posix_tss_ptr.hpp>", private, "<boost/asio/detail/tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/base_from_completion_cond.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/bind_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/buffered_stream_storage.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/buffer_resize_guard.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/buffer_sequence_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/call_stack.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/chrono_time_traits.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/completion_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/consuming_buffers.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/dependent_type.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/descriptor_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/eventfd_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/gcc_arm_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/gcc_hppa_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/gcc_sync_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/gcc_x86_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/handler_alloc_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/handler_cont_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/handler_invoke_helpers.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/handler_tracking.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/hash_map.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/impl/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/io_control.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/keyword_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/local_free_on_block_exit.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/macos_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/noncopyable.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/null_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/object_pool.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/old_win_sdk_compat.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/pipe_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/posix_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/posix_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/posix_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/posix_signal_blocker.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/posix_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/posix_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/posix_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactor_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/reactor_op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/scoped_lock.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/scoped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/service_registry.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/signal_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/signal_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/signal_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/socket_holder.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/socket_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/socket_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/socket_types.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/solaris_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/std_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/std_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/std_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/std_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/task_io_service_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/task_io_service_thread_info.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/thread_info_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/throw_error.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/timer_queue_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/timer_queue_ptime.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/timer_queue_set.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/wait_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/wait_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/wince_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_handle_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_handle_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_iocp_thread_info.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_object_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_async_manager.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_async_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_ssocket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winrt_utils.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/winsock_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/win_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/detail/wrapped_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/generic/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/buffered_read_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/buffered_write_stream.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/serial_port_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/spawn.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/use_future.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/impl/address.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/impl/address_v4.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/impl/address_v6.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/impl/basic_endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/local/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/buffered_handshake_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/engine.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/handshake_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/io.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/openssl_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/password_callback.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/shutdown_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/stream_core.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/verify_callback.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/detail/write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/impl/context.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/old/detail/openssl_context_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/old/detail/openssl_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/old/detail/openssl_stream_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_descriptor_service.hpp>", private, "<boost/asio/detail/reactive_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_null_buffers_op.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_null_buffers_op.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_null_buffers_op.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_accept_op.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_connect_op.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_connect_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_recv_op.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_send_op.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_sendto_op.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactive_socket_service_base.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_fwd.hpp>", private, "<boost/asio/detail/reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_fwd.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor.hpp>", private, "<boost/asio/detail/reactive_descriptor_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/descriptor_read_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/descriptor_write_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/reactor_op_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op_queue.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/reactor_op_queue.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/resolve_endpoint_op.hpp>", private, "<boost/asio/detail/resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/resolve_op.hpp>", private, "<boost/asio/detail/resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/resolver_service_base.hpp>", private, "<boost/asio/detail/resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_lock.hpp>", private, "<boost/asio/detail/null_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_lock.hpp>", private, "<boost/asio/detail/null_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_lock.hpp>", private, "<boost/asio/detail/posix_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_lock.hpp>", private, "<boost/asio/detail/posix_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_lock.hpp>", private, "<boost/asio/detail/std_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_lock.hpp>", private, "<boost/asio/detail/std_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_lock.hpp>", private, "<boost/asio/detail/win_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_lock.hpp>", private, "<boost/asio/detail/win_static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_ptr.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_ptr.hpp>", private, "<boost/asio/detail/strand_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/scoped_ptr.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/select_interrupter.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/select_interrupter.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/select_interrupter.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/select_interrupter.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/select_reactor.hpp>", private, "<boost/asio/detail/reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/select_reactor.hpp>", private, "<boost/asio/detail/timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/service_registry.hpp>", private, "<boost/asio/impl/io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/shared_ptr.hpp>", private, "<boost/asio/detail/socket_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/shared_ptr.hpp>", private, "<boost/asio/impl/spawn.hpp>", private ] },
+    { include: ["<boost/asio/detail/shared_ptr.hpp>", private, "<boost/asio/ssl/detail/openssl_init.hpp>", private ] },
+    { include: ["<boost/asio/detail/signal_handler.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/signal_op.hpp>", private, "<boost/asio/detail/signal_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/signal_op.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_holder.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_holder.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_holder.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_holder.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_holder.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_sendto_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/resolve_endpoint_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/socket_holder.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/win_iocp_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/detail/winrt_utils.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/ip/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/ssl/old/detail/openssl_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_select_interrupter.hpp>", private, "<boost/asio/detail/select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/buffer_sequence_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/descriptor_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/hash_map.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/io_control.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/local_free_on_block_exit.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/posix_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/reactive_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/reactive_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/signal_set_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/socket_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/socket_select_interrupter.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/wince_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_event.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_iocp_operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/detail/win_tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/generic/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/local/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ssl/detail/openssl_types.hpp>", private ] },
+    { include: ["<boost/asio/detail/solaris_fenced_block.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/static_mutex.hpp>", private, "<boost/asio/detail/handler_tracking.hpp>", private ] },
+    { include: ["<boost/asio/detail/static_mutex.hpp>", private, "<boost/asio/ssl/detail/engine.hpp>", private ] },
+    { include: ["<boost/asio/detail/std_event.hpp>", private, "<boost/asio/detail/event.hpp>", private ] },
+    { include: ["<boost/asio/detail/std_mutex.hpp>", private, "<boost/asio/detail/mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/std_static_mutex.hpp>", private, "<boost/asio/detail/static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/std_thread.hpp>", private, "<boost/asio/detail/thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/task_io_service.hpp>", private, "<boost/asio/impl/io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/task_io_service_operation.hpp>", private, "<boost/asio/detail/operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/task_io_service_operation.hpp>", private, "<boost/asio/detail/task_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/thread.hpp>", private, "<boost/asio/detail/resolver_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/thread.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/thread.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/thread.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/thread_info_base.hpp>", private, "<boost/asio/detail/task_io_service_thread_info.hpp>", private ] },
+    { include: ["<boost/asio/detail/thread_info_base.hpp>", private, "<boost/asio/detail/win_iocp_thread_info.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/detail/null_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/detail/wince_thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/impl/connect.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/impl/read_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/impl/read.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/impl/read_until.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/impl/write_at.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/impl/write.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ip/impl/address.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ip/impl/address_v4.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ip/impl/address_v6.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ip/impl/basic_endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ssl/impl/context.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ssl/old/detail/openssl_context_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_exception.hpp>", private, "<boost/asio/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/throw_exception.hpp>", private, "<boost/asio/ip/detail/socket_option.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_base.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_base.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_base.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_base.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_base.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_base.hpp>", private, "<boost/asio/detail/timer_queue_set.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_base.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_base.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue.hpp>", private, "<boost/asio/detail/timer_queue_ptime.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_set.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_set.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_set.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_set.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_set.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_queue_set.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_scheduler_fwd.hpp>", private, "<boost/asio/detail/timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/timer_scheduler.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/tss_ptr.hpp>", private, "<boost/asio/detail/call_stack.hpp>", private ] },
+    { include: ["<boost/asio/detail/tss_ptr.hpp>", private, "<boost/asio/detail/handler_tracking.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_handler.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_handler.hpp>", private, "<boost/asio/detail/win_object_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/deadline_timer_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/dev_poll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/epoll_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/kqueue_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/select_reactor.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/timer_queue.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/wait_handler.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/wait_op.hpp>", private, "<boost/asio/detail/winrt_timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/weak_ptr.hpp>", private, "<boost/asio/detail/socket_ops.hpp>", private ] },
+    { include: ["<boost/asio/detail/wince_thread.hpp>", private, "<boost/asio/detail/thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_event.hpp>", private, "<boost/asio/detail/event.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_fd_set_adapter.hpp>", private, "<boost/asio/detail/fd_set_adapter.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_fenced_block.hpp>", private, "<boost/asio/detail/fenced_block.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_handle_read_op.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_handle_service.hpp>", private, "<boost/asio/detail/win_iocp_serial_port_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_handle_write_op.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_io_service.hpp>", private, "<boost/asio/detail/timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_io_service.hpp>", private, "<boost/asio/detail/win_iocp_handle_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_io_service.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_io_service.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_io_service.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_io_service.hpp>", private, "<boost/asio/impl/io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_null_buffers_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_operation.hpp>", private, "<boost/asio/detail/operation.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_operation.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_overlapped_op.hpp>", private, "<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_recvfrom_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_recvmsg_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_recv_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_send_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_send_op.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_service_base.hpp>", private, "<boost/asio/detail/win_iocp_socket_accept_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_service_base.hpp>", private, "<boost/asio/detail/win_iocp_socket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_iocp_thread_info.hpp>", private, "<boost/asio/detail/win_iocp_io_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_mutex.hpp>", private, "<boost/asio/detail/mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_async_manager.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_async_manager.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_async_op.hpp>", private, "<boost/asio/detail/winrt_async_manager.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_async_op.hpp>", private, "<boost/asio/detail/winrt_resolve_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_async_op.hpp>", private, "<boost/asio/detail/winrt_socket_connect_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_async_op.hpp>", private, "<boost/asio/detail/winrt_socket_recv_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_async_op.hpp>", private, "<boost/asio/detail/winrt_socket_send_op.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_resolve_op.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_socket_connect_op.hpp>", private, "<boost/asio/detail/winrt_ssocket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_socket_recv_op.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_socket_send_op.hpp>", private, "<boost/asio/detail/winrt_ssocket_service_base.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_ssocket_service_base.hpp>", private, "<boost/asio/detail/winrt_ssocket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_timer_scheduler.hpp>", private, "<boost/asio/detail/timer_scheduler.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_utils.hpp>", private, "<boost/asio/detail/winrt_resolver_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/winrt_utils.hpp>", private, "<boost/asio/detail/winrt_ssocket_service.hpp>", private ] },
+    { include: ["<boost/asio/detail/winsock_init.hpp>", private, "<boost/asio/ip/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_static_mutex.hpp>", private, "<boost/asio/detail/static_mutex.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_thread.hpp>", private, "<boost/asio/detail/thread.hpp>", private ] },
+    { include: ["<boost/asio/detail/win_tss_ptr.hpp>", private, "<boost/asio/detail/tss_ptr.hpp>", private ] },
+    { include: ["<boost/asio/generic/detail/impl/endpoint.ipp>", private, "<boost/asio/generic/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/generic/detail/impl/endpoint.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/impl/error.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/impl/handler_alloc_hook.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/impl/io_service.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/impl/serial_port_base.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ip/detail/impl/endpoint.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ip/detail/impl/endpoint.ipp>", private, "<boost/asio/ip/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/ip/impl/address.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ip/impl/address_v4.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ip/impl/address_v6.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ip/impl/host_name.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/local/detail/impl/endpoint.ipp>", private, "<boost/asio/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/local/detail/impl/endpoint.ipp>", private, "<boost/asio/local/detail/endpoint.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/engine.hpp>", private, "<boost/asio/ssl/detail/buffered_handshake_op.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/engine.hpp>", private, "<boost/asio/ssl/detail/handshake_op.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/engine.hpp>", private, "<boost/asio/ssl/detail/io.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/engine.hpp>", private, "<boost/asio/ssl/detail/read_op.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/engine.hpp>", private, "<boost/asio/ssl/detail/shutdown_op.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/engine.hpp>", private, "<boost/asio/ssl/detail/stream_core.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/engine.hpp>", private, "<boost/asio/ssl/detail/write_op.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/impl/engine.ipp>", private, "<boost/asio/ssl/detail/engine.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/impl/engine.ipp>", private, "<boost/asio/ssl/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/impl/openssl_init.ipp>", private, "<boost/asio/ssl/detail/openssl_init.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/impl/openssl_init.ipp>", private, "<boost/asio/ssl/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/openssl_init.hpp>", private, "<boost/asio/ssl/old/detail/openssl_context_service.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/detail/engine.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/detail/openssl_init.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/old/detail/openssl_context_service.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/old/detail/openssl_operation.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/old/detail/openssl_stream_service.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/stream_core.hpp>", private, "<boost/asio/ssl/detail/io.hpp>", private ] },
+    { include: ["<boost/asio/ssl/detail/verify_callback.hpp>", private, "<boost/asio/ssl/detail/engine.hpp>", private ] },
+    { include: ["<boost/asio/ssl/impl/context.ipp>", private, "<boost/asio/ssl/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ssl/impl/error.ipp>", private, "<boost/asio/ssl/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ssl/impl/rfc2818_verification.ipp>", private, "<boost/asio/ssl/impl/src.hpp>", private ] },
+    { include: ["<boost/asio/ssl/old/detail/openssl_operation.hpp>", private, "<boost/asio/ssl/old/detail/openssl_stream_service.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/cas128strong.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/cas32strong.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/cas32weak.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/cas64strong.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/cas64strong-ptr.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/gcc-alpha.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/gcc-armv6plus.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/gcc-atomic.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/gcc-cas.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/gcc-ppc.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/gcc-sparcv9.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/gcc-x86.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/generic-cas.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/linux-arm.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/base.hpp>", private, "<boost/atomic/detail/windows.hpp>", private ] },
+    { include: ["<boost/atomic/detail/builder.hpp>", private, "<boost/atomic/detail/gcc-alpha.hpp>", private ] },
+    { include: ["<boost/atomic/detail/builder.hpp>", private, "<boost/atomic/detail/generic-cas.hpp>", private ] },
+    { include: ["<boost/atomic/detail/cas128strong.hpp>", private, "<boost/atomic/detail/gcc-x86.hpp>", private ] },
+    { include: ["<boost/atomic/detail/cas32strong.hpp>", private, "<boost/atomic/detail/gcc-cas.hpp>", private ] },
+    { include: ["<boost/atomic/detail/cas32weak.hpp>", private, "<boost/atomic/detail/gcc-armv6plus.hpp>", private ] },
+    { include: ["<boost/atomic/detail/cas32weak.hpp>", private, "<boost/atomic/detail/linux-arm.hpp>", private ] },
+    { include: ["<boost/atomic/detail/cas64strong.hpp>", private, "<boost/atomic/detail/gcc-x86.hpp>", private ] },
+    { include: ["<boost/atomic/detail/cas64strong.hpp>", private, "<boost/atomic/detail/windows.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/base.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/cas128strong.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/cas32strong.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/cas32weak.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/cas64strong.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/cas64strong-ptr.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/gcc-alpha.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/gcc-armv6plus.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/gcc-atomic.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/gcc-cas.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/gcc-ppc.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/gcc-sparcv9.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/gcc-x86.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/generic-cas.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/interlocked.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/link.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/linux-arm.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/lockpool.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/type-classification.hpp>", private ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/detail/windows.hpp>", private ] },
+    { include: ["<boost/atomic/detail/gcc-alpha.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/gcc-armv6plus.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/gcc-atomic.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/gcc-cas.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/gcc-ppc.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/gcc-sparcv9.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/gcc-x86.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/interlocked.hpp>", private, "<boost/atomic/detail/windows.hpp>", private ] },
+    { include: ["<boost/atomic/detail/link.hpp>", private, "<boost/atomic/detail/lockpool.hpp>", private ] },
+    { include: ["<boost/atomic/detail/linux-arm.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/atomic/detail/lockpool.hpp>", private, "<boost/atomic/detail/base.hpp>", private ] },
+    { include: ["<boost/atomic/detail/windows.hpp>", private, "<boost/atomic/detail/platform.hpp>", private ] },
+    { include: ["<boost/bimap/detail/concept_tags.hpp>", private, "<boost/bimap/detail/is_set_type_of.hpp>", private ] },
+    { include: ["<boost/bimap/detail/debug/static_error.hpp>", private, "<boost/bimap/detail/map_view_base.hpp>", private ] },
+    { include: ["<boost/bimap/detail/debug/static_error.hpp>", private, "<boost/bimap/relation/detail/metadata_access_builder.hpp>", private ] },
+    { include: ["<boost/bimap/detail/debug/static_error.hpp>", private, "<boost/bimap/relation/detail/mutant.hpp>", private ] },
+    { include: ["<boost/bimap/detail/debug/static_error.hpp>", private, "<boost/bimap/relation/detail/static_access_builder.hpp>", private ] },
+    { include: ["<boost/bimap/detail/is_set_type_of.hpp>", private, "<boost/bimap/detail/manage_additional_parameters.hpp>", private ] },
+    { include: ["<boost/bimap/detail/is_set_type_of.hpp>", private, "<boost/bimap/detail/manage_bimap_key.hpp>", private ] },
+    { include: ["<boost/bimap/detail/manage_additional_parameters.hpp>", private, "<boost/bimap/detail/bimap_core.hpp>", private ] },
+    { include: ["<boost/bimap/detail/manage_bimap_key.hpp>", private, "<boost/bimap/detail/bimap_core.hpp>", private ] },
+    { include: ["<boost/bimap/detail/map_view_iterator.hpp>", private, "<boost/bimap/detail/bimap_core.hpp>", private ] },
+    { include: ["<boost/bimap/detail/map_view_iterator.hpp>", private, "<boost/bimap/detail/map_view_base.hpp>", private ] },
+    { include: ["<boost/bimap/detail/modifier_adaptor.hpp>", private, "<boost/bimap/detail/map_view_base.hpp>", private ] },
+    { include: ["<boost/bimap/detail/modifier_adaptor.hpp>", private, "<boost/bimap/detail/set_view_base.hpp>", private ] },
+    { include: ["<boost/bimap/detail/set_view_iterator.hpp>", private, "<boost/bimap/detail/bimap_core.hpp>", private ] },
+    { include: ["<boost/bimap/detail/set_view_iterator.hpp>", private, "<boost/bimap/detail/set_view_base.hpp>", private ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/detail/map_view_iterator.hpp>", private ] },
+    { include: ["<boost/bimap/relation/detail/mutant.hpp>", private, "<boost/bimap/relation/detail/to_mutable_relation_functor.hpp>", private ] },
+    { include: ["<boost/bimap/relation/detail/static_access_builder.hpp>", private, "<boost/bimap/detail/map_view_iterator.hpp>", private ] },
+    { include: ["<boost/bimap/relation/detail/to_mutable_relation_functor.hpp>", private, "<boost/bimap/detail/map_view_base.hpp>", private ] },
+    { include: ["<boost/bimap/relation/detail/to_mutable_relation_functor.hpp>", private, "<boost/bimap/detail/set_view_base.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/mac/chrono.hpp>", private, "<boost/chrono/detail/inlined/chrono.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/mac/process_cpu_clocks.hpp>", private, "<boost/chrono/detail/inlined/process_cpu_clocks.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/mac/thread_clock.hpp>", private, "<boost/chrono/detail/inlined/thread_clock.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/posix/chrono.hpp>", private, "<boost/chrono/detail/inlined/chrono.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/posix/process_cpu_clocks.hpp>", private, "<boost/chrono/detail/inlined/process_cpu_clocks.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/posix/thread_clock.hpp>", private, "<boost/chrono/detail/inlined/mac/thread_clock.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/posix/thread_clock.hpp>", private, "<boost/chrono/detail/inlined/thread_clock.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/win/chrono.hpp>", private, "<boost/chrono/detail/inlined/chrono.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/win/process_cpu_clocks.hpp>", private, "<boost/chrono/detail/inlined/process_cpu_clocks.hpp>", private ] },
+    { include: ["<boost/chrono/detail/inlined/win/thread_clock.hpp>", private, "<boost/chrono/detail/inlined/thread_clock.hpp>", private ] },
+    { include: ["<boost/chrono/detail/system.hpp>", private, "<boost/chrono/detail/inlined/chrono.hpp>", private ] },
+    { include: ["<boost/chrono/detail/system.hpp>", private, "<boost/chrono/detail/inlined/thread_clock.hpp>", private ] },
+    { include: ["<boost/concept/detail/backward_compatibility.hpp>", private, "<boost/concept/detail/borland.hpp>", private ] },
+    { include: ["<boost/concept/detail/backward_compatibility.hpp>", private, "<boost/concept/detail/general.hpp>", private ] },
+    { include: ["<boost/concept/detail/backward_compatibility.hpp>", private, "<boost/concept/detail/has_constraints.hpp>", private ] },
+    { include: ["<boost/concept/detail/backward_compatibility.hpp>", private, "<boost/concept/detail/msvc.hpp>", private ] },
+    { include: ["<boost/concept/detail/concept_def.hpp>", private, "<boost/icl/detail/concept_check.hpp>", private ] },
+    { include: ["<boost/concept/detail/has_constraints.hpp>", private, "<boost/concept/detail/general.hpp>", private ] },
+    { include: ["<boost/concept/detail/has_constraints.hpp>", private, "<boost/concept/detail/msvc.hpp>", private ] },
+    { include: ["<boost/container/detail/adaptive_node_pool_impl.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/container/detail/algorithms.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/algorithms.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/allocation_type.hpp>", private, "<boost/container/detail/allocator_version_traits.hpp>", private ] },
+    { include: ["<boost/container/detail/allocator_version_traits.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/detail/allocator_version_traits.hpp>", private ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/detail/function_detector.hpp>", private ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/detail/preprocessor.hpp>", private ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/detail/workaround.hpp>", private ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/geometry/index/detail/varray.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/adaptive_node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/advanced_insert_int.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/algorithms.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/allocation_type.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/allocator_version_traits.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/destroyers.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/flat_tree.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/function_detector.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/iterators.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/math_functions.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/multiallocation_chain.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/pair.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/pool_common.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/preprocessor.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/type_traits.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/utilities.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/value_init.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/variadic_templates_tools.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/detail/workaround.hpp>", private ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/geometry/index/detail/varray.hpp>", private ] },
+    { include: ["<boost/container/detail/destroyers.hpp>", private, "<boost/container/detail/advanced_insert_int.hpp>", private ] },
+    { include: ["<boost/container/detail/destroyers.hpp>", private, "<boost/container/detail/flat_tree.hpp>", private ] },
+    { include: ["<boost/container/detail/destroyers.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/destroyers.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/iterators.hpp>", private, "<boost/container/detail/algorithms.hpp>", private ] },
+    { include: ["<boost/container/detail/iterators.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/math_functions.hpp>", private, "<boost/container/detail/adaptive_node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/math_functions.hpp>", private, "<boost/container/detail/node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/memory_util.hpp>", private, "<boost/container/detail/utilities.hpp>", private ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/detail/adaptive_node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/detail/algorithms.hpp>", private ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/detail/allocator_version_traits.hpp>", private ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/detail/node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/detail/pair.hpp>", private ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/detail/utilities.hpp>", private ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/detail/version_type.hpp>", private ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/container/detail/allocator_version_traits.hpp>", private ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/allocators/detail/allocator_common.hpp>", private ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/node_alloc_holder.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/node_pool_impl.hpp>", private, "<boost/interprocess/allocators/detail/node_pool.hpp>", private ] },
+    { include: ["<boost/container/detail/pair.hpp>", private, "<boost/container/detail/flat_tree.hpp>", private ] },
+    { include: ["<boost/container/detail/pair.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/pool_common.hpp>", private, "<boost/container/detail/adaptive_node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/pool_common.hpp>", private, "<boost/container/detail/node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/detail/advanced_insert_int.hpp>", private ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/detail/iterators.hpp>", private ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/detail/pair.hpp>", private ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/geometry/index/detail/varray.hpp>", private ] },
+    { include: ["<boost/container/detail/transform_iterator.hpp>", private, "<boost/container/detail/multiallocation_chain.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/adaptive_node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/algorithms.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/iterators.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/multiallocation_chain.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/pair.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/utilities.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/variadic_templates_tools.hpp>", private ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/detail/version_type.hpp>", private ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/detail/adaptive_node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/detail/destroyers.hpp>", private ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/detail/flat_tree.hpp>", private ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/detail/multiallocation_chain.hpp>", private ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/detail/node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/value_init.hpp>", private, "<boost/container/detail/advanced_insert_int.hpp>", private ] },
+    { include: ["<boost/container/detail/value_init.hpp>", private, "<boost/container/detail/flat_tree.hpp>", private ] },
+    { include: ["<boost/container/detail/variadic_templates_tools.hpp>", private, "<boost/container/detail/advanced_insert_int.hpp>", private ] },
+    { include: ["<boost/container/detail/variadic_templates_tools.hpp>", private, "<boost/container/detail/iterators.hpp>", private ] },
+    { include: ["<boost/container/detail/version_type.hpp>", private, "<boost/container/detail/allocator_version_traits.hpp>", private ] },
+    { include: ["<boost/container/detail/version_type.hpp>", private, "<boost/container/detail/destroyers.hpp>", private ] },
+    { include: ["<boost/container/detail/version_type.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/adaptive_node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/advanced_insert_int.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/algorithms.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/allocation_type.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/allocator_version_traits.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/destroyers.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/flat_tree.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/iterators.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/node_pool_impl.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/pair.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/preprocessor.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/value_init.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/container/detail/variadic_templates_tools.hpp>", private ] },
+    { include: ["<boost/container/detail/workaround.hpp>", private, "<boost/geometry/index/detail/varray.hpp>", private ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/detail/fcontext_arm.hpp>", private ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/detail/fcontext_i386.hpp>", private ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/detail/fcontext_i386_win.hpp>", private ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/detail/fcontext_mips.hpp>", private ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/detail/fcontext_ppc.hpp>", private ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/detail/fcontext_sparc.hpp>", private ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/detail/fcontext_x86_64.hpp>", private ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/detail/fcontext_x86_64_win.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/detail/coroutine_context.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/detail/segmented_stack_allocator.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/detail/stack_tuple.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/detail/standard_stack_allocator.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v1/detail/arg.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v1/detail/coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v1/detail/coroutine_base_resume.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v1/detail/coroutine_caller.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v1/detail/coroutine_get.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v1/detail/coroutine_op.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_caller.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_caller.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/coroutine_context.hpp>", private, "<boost/coroutine/detail/holder.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/coroutine_context.hpp>", private, "<boost/coroutine/v1/detail/coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/coroutine_context.hpp>", private, "<boost/coroutine/v1/detail/coroutine_base_resume.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/coroutine_context.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/coroutine_context.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/flags.hpp>", private, "<boost/coroutine/v1/detail/coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/flags.hpp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/flags.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/flags.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/flags.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/flags.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/holder.hpp>", private, "<boost/coroutine/v1/detail/coroutine_base_resume.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/holder.hpp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/holder.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/holder.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/holder.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/param.hpp>", private, "<boost/coroutine/v1/detail/coroutine_get.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/param.hpp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/param.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/param.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/param.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/stack_tuple.hpp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/stack_tuple.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/stack_tuple.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/trampoline.hpp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/trampoline.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/detail/trampoline.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/arg.hpp>", private, "<boost/coroutine/v1/detail/coroutine_base_resume.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/arg.hpp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/arg.hpp>", private, "<boost/coroutine/v1/detail/coroutine_op.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_base.hpp>", private, "<boost/coroutine/v1/detail/coroutine_caller.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_base.hpp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_base_resume.hpp>", private, "<boost/coroutine/v1/detail/coroutine_base.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_object_result_0.ipp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_object_result_1.ipp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_object_result_arity.ipp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_object_void_0.ipp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_object_void_1.ipp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_object_void_arity.ipp>", private, "<boost/coroutine/v1/detail/coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v2/detail/pull_coroutine_base.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_caller.hpp>", private ] },
+    { include: ["<boost/coroutine/v2/detail/pull_coroutine_base.hpp>", private, "<boost/coroutine/v2/detail/pull_coroutine_object.hpp>", private ] },
+    { include: ["<boost/coroutine/v2/detail/push_coroutine_base.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_caller.hpp>", private ] },
+    { include: ["<boost/coroutine/v2/detail/push_coroutine_base.hpp>", private, "<boost/coroutine/v2/detail/push_coroutine_object.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/detail/auto_space.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/detail/hash_index_node.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/detail/index_base.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/detail/ord_index_node.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/detail/rnd_index_loader.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/detail/rnd_index_node.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/detail/rnd_index_ptr_array.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/detail/seq_index_node.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/statechart/detail/memory.hpp>", private ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/statechart/detail/state_base.hpp>", private ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/asio/detail/atomic_count.hpp>", private ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/serialization/detail/shared_ptr_nmt_132.hpp>", private ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/spirit/home/support/iterators/detail/ref_counted_policy.hpp>", private ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/statechart/detail/counted_base.hpp>", private ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/xpressive/detail/utility/counted_base.hpp>", private ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/xpressive/detail/utility/tracking_ptr.hpp>", private ] },
+    { include: ["<boost/detail/binary_search.hpp>", private, "<boost/python/suite/indexing/detail/indexing_suite_detail.hpp>", private ] },
+    { include: ["<boost/detail/container_fwd.hpp>", private, "<boost/lambda/detail/operator_return_type_traits.hpp>", private ] },
+    { include: ["<boost/detail/container_fwd.hpp>", private, "<boost/phoenix/stl/algorithm/detail/is_std_list.hpp>", private ] },
+    { include: ["<boost/detail/container_fwd.hpp>", private, "<boost/phoenix/stl/algorithm/detail/is_std_map.hpp>", private ] },
+    { include: ["<boost/detail/container_fwd.hpp>", private, "<boost/phoenix/stl/algorithm/detail/is_std_set.hpp>", private ] },
+    { include: ["<boost/detail/container_fwd.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/detail/is_std_list.hpp>", private ] },
+    { include: ["<boost/detail/container_fwd.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/detail/is_std_map.hpp>", private ] },
+    { include: ["<boost/detail/container_fwd.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/detail/is_std_set.hpp>", private ] },
+    { include: ["<boost/detail/endian.hpp>", private, "<boost/math/special_functions/detail/fp_traits.hpp>", private ] },
+    { include: ["<boost/detail/endian.hpp>", private, "<boost/spirit/home/support/detail/endian/endian.hpp>", private ] },
+    { include: ["<boost/detail/endian.hpp>", private, "<boost/spirit/home/support/detail/math/detail/fp_traits.hpp>", private ] },
+    { include: ["<boost/detail/fenv.hpp>", private, "<boost/numeric/interval/detail/c99sub_rounding_control.hpp>", private ] },
+    { include: ["<boost/detail/indirect_traits.hpp>", private, "<boost/iterator/detail/facade_iterator_category.hpp>", private ] },
+    { include: ["<boost/detail/indirect_traits.hpp>", private, "<boost/python/detail/caller.hpp>", private ] },
+    { include: ["<boost/detail/indirect_traits.hpp>", private, "<boost/python/detail/indirect_traits.hpp>", private ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/atomic/detail/interlocked.hpp>", private ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/interprocess/detail/win32_api.hpp>", private ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/log/detail/spin_mutex.hpp>", private ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/smart_ptr/detail/atomic_count_win32.hpp>", private ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_w32.hpp>", private ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/smart_ptr/detail/spinlock_w32.hpp>", private ] },
+    { include: ["<boost/detail/is_incrementable.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/detail/is_xxx.hpp>", private, "<boost/python/detail/is_xxx.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/algorithm/string/detail/finder.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/algorithm/string/detail/trim.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iostreams/detail/adapter/range_adapter.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/multi_index/detail/safe_mode.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/range/detail/collection_traits_detail.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/qi/numeric/detail/numeric_utils.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/qi/stream/detail/iterator_source.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/qi/string/detail/tst.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/support/detail/lexer/generate_cpp.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/support/detail/lexer/input.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/support/iterators/detail/buffering_input_iterator_policy.hpp>", private ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/support/iterators/detail/input_iterator_policy.hpp>", private ] },
+    { include: ["<boost/detail/lightweight_mutex.hpp>", private, "<boost/flyweight/detail/recursive_lw_mutex.hpp>", private ] },
+    { include: ["<boost/detail/lightweight_mutex.hpp>", private, "<boost/multi_index/detail/safe_mode.hpp>", private ] },
+    { include: ["<boost/detail/lightweight_mutex.hpp>", private, "<boost/serialization/detail/shared_count_132.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/archive/detail/iserializer.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/detail/adaptive_node_pool_impl.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/detail/advanced_insert_int.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/detail/algorithms.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/detail/allocator_version_traits.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/detail/node_alloc_holder.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/detail/node_pool_impl.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/detail/tree.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/detail/utilities.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/flyweight/detail/flyweight_core.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/geometry/index/detail/varray_detail.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/detail/managed_memory_impl.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/detail/managed_multi_shared_memory.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/detail/segment_manager_helper.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/intrusive/detail/utilities.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index/detail/archive_constructed.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index/detail/copy_map.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index/detail/scope_guard.hpp>", private ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index/detail/seq_index_ops.hpp>", private ] },
+    { include: ["<boost/detail/quick_allocator.hpp>", private, "<boost/serialization/detail/shared_count_132.hpp>", private ] },
+    { include: ["<boost/detail/reference_content.hpp>", private, "<boost/variant/detail/initializer.hpp>", private ] },
+    { include: ["<boost/detail/scoped_enum_emulation.hpp>", private, "<boost/spirit/home/support/detail/scoped_enum_emulation.hpp>", private ] },
+    { include: ["<boost/detail/select_type.hpp>", private, "<boost/unordered/detail/allocate.hpp>", private ] },
+    { include: ["<boost/detail/select_type.hpp>", private, "<boost/unordered/detail/util.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/exception/detail/type_info.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_acc_ia64.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_aix.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_cw_ppc.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_cw_x86.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_gcc_ia64.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_gcc_mips.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_gcc_ppc.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_gcc_sparc.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_nt.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_pt.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_snc_ps3.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_solaris.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_spin.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_sync.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_vacpp_ppc.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_w32.hpp>", private ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/spirit/home/support/detail/hold_any.hpp>", private ] },
+    { include: ["<boost/detail/templated_streams.hpp>", private, "<boost/variant/detail/variant_io.hpp>", private ] },
+    { include: ["<boost/detail/utf8_codecvt_facet.hpp>", private, "<boost/archive/detail/utf8_codecvt_facet.hpp>", private ] },
+    { include: ["<boost/detail/utf8_codecvt_facet.hpp>", private, "<boost/filesystem/detail/utf8_codecvt_facet.hpp>", private ] },
+    { include: ["<boost/detail/utf8_codecvt_facet.hpp>", private, "<boost/program_options/detail/utf8_codecvt_facet.hpp>", private ] },
+    { include: ["<boost/detail/winapi/GetCurrentProcess.hpp>", private, "<boost/chrono/detail/inlined/win/process_cpu_clocks.hpp>", private ] },
+    { include: ["<boost/detail/winapi/GetCurrentThread.hpp>", private, "<boost/chrono/detail/inlined/win/thread_clock.hpp>", private ] },
+    { include: ["<boost/detail/winapi/GetLastError.hpp>", private, "<boost/chrono/detail/inlined/win/chrono.hpp>", private ] },
+    { include: ["<boost/detail/winapi/GetLastError.hpp>", private, "<boost/chrono/detail/inlined/win/process_cpu_clocks.hpp>", private ] },
+    { include: ["<boost/detail/winapi/GetLastError.hpp>", private, "<boost/chrono/detail/inlined/win/thread_clock.hpp>", private ] },
+    { include: ["<boost/detail/winapi/GetProcessTimes.hpp>", private, "<boost/chrono/detail/inlined/win/process_cpu_clocks.hpp>", private ] },
+    { include: ["<boost/detail/winapi/GetThreadTimes.hpp>", private, "<boost/chrono/detail/inlined/win/thread_clock.hpp>", private ] },
+    { include: ["<boost/detail/winapi/time.hpp>", private, "<boost/chrono/detail/inlined/win/chrono.hpp>", private ] },
+    { include: ["<boost/detail/winapi/timers.hpp>", private, "<boost/chrono/detail/inlined/win/chrono.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/archive/detail/iserializer.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/archive/detail/oserializer.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/archive/impl/basic_xml_grammar.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/concept/detail/has_constraints.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/context/detail/config.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/coroutine/detail/config.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/flyweight/detail/flyweight_core.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/flyweight/detail/not_placeholder_expr.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/format/detail/config_macros.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/function_types/detail/cv_traits.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/graph/detail/adjacency_list.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/graph/detail/adj_list_edge_iterator.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/graph/detail/read_graphviz_new.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/graph/detail/read_graphviz_spirit.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_device.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_file_wrapper.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/adapter/direct_adapter.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/adapter/mode_adapter.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/add_facet.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/codecvt_helper.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/config/codecvt.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/config/disable_warnings.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/config/dyn_link.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/config/overload_resolution.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/config/wide_streams.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/default_arg.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/double_object.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/execute.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/forward.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/ios.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/is_dereferenceable.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/is_iterator_range.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/push.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/streambuf/chainbuf.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iostreams/detail/wrap_unwrap.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iterator/detail/config_def.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/iterator/detail/enable_if.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/lambda/detail/lambda_functors.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/multi_index/detail/access_specifier.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/multi_index/detail/base_type.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/multi_index/detail/index_base.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/multi_index/detail/msvc_index_specifier.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/multi_index/detail/node_type.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/multi_index/detail/prevent_eti.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/multi_index/detail/safe_ctr_proxy.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/multi_index/detail/unbounded.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/program_options/detail/cmdline.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/program_options/detail/config_file.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/program_options/detail/convert.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/proto/detail/as_expr.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/proto/detail/decltype.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/python/detail/config.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/python/detail/destroy.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/python/detail/enable_if.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/python/detail/string_literal.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/random/detail/operators.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/range/detail/begin.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/range/detail/end.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/range/detail/size.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/serialization/detail/shared_ptr_132.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/signals2/detail/auto_buffer.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_w32.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/spirit/home/karma/detail/alternative_function.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/spirit/home/karma/numeric/detail/bool_utils.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/spirit/home/karma/numeric/detail/real_utils.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/spirit/home/support/detail/lexer/parser/tokeniser/num_token.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/spirit/home/support/detail/what_function.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/statechart/detail/rtti_policy.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/statechart/detail/state_base.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/test/detail/config.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/thread/detail/config.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/tti/detail/dmem_data.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/tuple/detail/tuple_basic.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/type_traits/detail/cv_traits_impl.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/variant/detail/apply_visitor_binary.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/variant/detail/apply_visitor_unary.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/variant/detail/config.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/variant/detail/move.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/xpressive/detail/core/results_cache.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/xpressive/detail/utility/literals.hpp>", private ] },
+    { include: ["<boost/detail/workaround.hpp>", private, "<boost/xpressive/detail/utility/tracking_ptr.hpp>", private ] },
+    { include: ["<boost/exception/detail/clone_current_exception.hpp>", private, "<boost/exception/detail/exception_ptr.hpp>", private ] },
+    { include: ["<boost/exception/detail/type_info.hpp>", private, "<boost/exception/detail/exception_ptr.hpp>", private ] },
+    { include: ["<boost/exception/detail/type_info.hpp>", private, "<boost/exception/detail/object_hex_dump.hpp>", private ] },
+    { include: ["<boost/flyweight/detail/dyn_perfect_fwd.hpp>", private, "<boost/flyweight/detail/perfect_fwd.hpp>", private ] },
+    { include: ["<boost/flyweight/detail/is_placeholder_expr.hpp>", private, "<boost/flyweight/detail/nested_xxx_if_not_ph.hpp>", private ] },
+    { include: ["<boost/flyweight/detail/perfect_fwd.hpp>", private, "<boost/flyweight/detail/default_value_policy.hpp>", private ] },
+    { include: ["<boost/flyweight/detail/perfect_fwd.hpp>", private, "<boost/flyweight/detail/flyweight_core.hpp>", private ] },
+    { include: ["<boost/flyweight/detail/pp_perfect_fwd.hpp>", private, "<boost/flyweight/detail/perfect_fwd.hpp>", private ] },
+    { include: ["<boost/flyweight/detail/value_tag.hpp>", private, "<boost/flyweight/detail/default_value_policy.hpp>", private ] },
+    { include: ["<boost/format/detail/config_macros.hpp>", private, "<boost/format/detail/compat_workarounds.hpp>", private ] },
+    { include: ["<boost/format/detail/workarounds_gcc-2_95.hpp>", private, "<boost/format/detail/config_macros.hpp>", private ] },
+    { include: ["<boost/format/detail/workarounds_stlport.hpp>", private, "<boost/format/detail/config_macros.hpp>", private ] },
+    { include: ["<boost/functional/hash/detail/float_functions.hpp>", private, "<boost/functional/hash/detail/hash_float.hpp>", private ] },
+    { include: ["<boost/functional/hash/detail/limits.hpp>", private, "<boost/functional/hash/detail/hash_float.hpp>", private ] },
+    { include: ["<boost/function/detail/maybe_include.hpp>", private, "<boost/function/detail/function_iterate.hpp>", private ] },
+    { include: ["<boost/function_types/detail/classifier_impl/arity10_0.hpp>", private, "<boost/function_types/detail/classifier_impl/arity20_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/classifier_impl/arity10_1.hpp>", private, "<boost/function_types/detail/classifier_impl/arity20_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/classifier_impl/arity20_0.hpp>", private, "<boost/function_types/detail/classifier_impl/arity30_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/classifier_impl/arity20_1.hpp>", private, "<boost/function_types/detail/classifier_impl/arity30_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/classifier_impl/arity30_0.hpp>", private, "<boost/function_types/detail/classifier_impl/arity40_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/classifier_impl/arity30_1.hpp>", private, "<boost/function_types/detail/classifier_impl/arity40_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/classifier_impl/arity40_0.hpp>", private, "<boost/function_types/detail/classifier_impl/arity50_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/classifier_impl/arity40_1.hpp>", private, "<boost/function_types/detail/classifier_impl/arity50_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/components_impl/arity10_0.hpp>", private, "<boost/function_types/detail/components_impl/arity20_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/components_impl/arity10_1.hpp>", private, "<boost/function_types/detail/components_impl/arity20_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/components_impl/arity20_0.hpp>", private, "<boost/function_types/detail/components_impl/arity30_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/components_impl/arity20_1.hpp>", private, "<boost/function_types/detail/components_impl/arity30_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/components_impl/arity30_0.hpp>", private, "<boost/function_types/detail/components_impl/arity40_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/components_impl/arity30_1.hpp>", private, "<boost/function_types/detail/components_impl/arity40_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/components_impl/arity40_0.hpp>", private, "<boost/function_types/detail/components_impl/arity50_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/components_impl/arity40_1.hpp>", private, "<boost/function_types/detail/components_impl/arity50_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/cv_traits.hpp>", private, "<boost/function_types/detail/synthesize.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/aliases_def.hpp>", private, "<boost/function_types/detail/pp_cc_loop/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/aliases_def.hpp>", private, "<boost/function_types/detail/pp_loop.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/aliases_def.hpp>", private, "<boost/function_types/detail/pp_retag_default_cc/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/aliases_undef.hpp>", private, "<boost/function_types/detail/pp_cc_loop/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/aliases_undef.hpp>", private, "<boost/function_types/detail/pp_loop.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/aliases_undef.hpp>", private, "<boost/function_types/detail/pp_retag_default_cc/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/def.hpp>", private, "<boost/function_types/detail/pp_cc_loop/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/def.hpp>", private, "<boost/function_types/detail/pp_loop.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/def.hpp>", private, "<boost/function_types/detail/pp_retag_default_cc/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/def.hpp>", private, "<boost/function_types/detail/pp_tags/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/def.hpp>", private, "<boost/function_types/detail/pp_variate_loop/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/undef.hpp>", private, "<boost/function_types/detail/pp_cc_loop/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/undef.hpp>", private, "<boost/function_types/detail/pp_loop.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/undef.hpp>", private, "<boost/function_types/detail/pp_retag_default_cc/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/encoding/undef.hpp>", private, "<boost/function_types/detail/pp_variate_loop/master.hpp>", private ] },
+    { include: ["<boost/function_types/detail/pp_loop.hpp>", private, "<boost/function_types/detail/classifier.hpp>", private ] },
+    { include: ["<boost/function_types/detail/pp_loop.hpp>", private, "<boost/function_types/detail/synthesize.hpp>", private ] },
+    { include: ["<boost/function_types/detail/pp_retag_default_cc/master.hpp>", private, "<boost/function_types/detail/retag_default_cc.hpp>", private ] },
+    { include: ["<boost/function_types/detail/pp_retag_default_cc/preprocessed.hpp>", private, "<boost/function_types/detail/retag_default_cc.hpp>", private ] },
+    { include: ["<boost/function_types/detail/retag_default_cc.hpp>", private, "<boost/function_types/detail/synthesize.hpp>", private ] },
+    { include: ["<boost/function_types/detail/synthesize_impl/arity10_0.hpp>", private, "<boost/function_types/detail/synthesize_impl/arity20_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/synthesize_impl/arity10_1.hpp>", private, "<boost/function_types/detail/synthesize_impl/arity20_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/synthesize_impl/arity20_0.hpp>", private, "<boost/function_types/detail/synthesize_impl/arity30_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/synthesize_impl/arity20_1.hpp>", private, "<boost/function_types/detail/synthesize_impl/arity30_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/synthesize_impl/arity30_0.hpp>", private, "<boost/function_types/detail/synthesize_impl/arity40_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/synthesize_impl/arity30_1.hpp>", private, "<boost/function_types/detail/synthesize_impl/arity40_1.hpp>", private ] },
+    { include: ["<boost/function_types/detail/synthesize_impl/arity40_0.hpp>", private, "<boost/function_types/detail/synthesize_impl/arity50_0.hpp>", private ] },
+    { include: ["<boost/function_types/detail/synthesize_impl/arity40_1.hpp>", private, "<boost/function_types/detail/synthesize_impl/arity50_1.hpp>", private ] },
+    { include: ["<boost/fusion/adapted/struct/detail/adapt_base.hpp>", private, "<boost/fusion/adapted/struct/detail/define_struct.hpp>", private ] },
+    { include: ["<boost/fusion/adapted/struct/detail/define_struct.hpp>", private, "<boost/fusion/adapted/struct/detail/define_struct_inline.hpp>", private ] },
+    { include: ["<boost/fusion/adapted/struct/detail/extension.hpp>", private, "<boost/fusion/adapted/adt/detail/extension.hpp>", private ] },
+    { include: ["<boost/fusion/adapted/struct/detail/namespace.hpp>", private, "<boost/fusion/adapted/struct/detail/define_struct.hpp>", private ] },
+    { include: ["<boost/fusion/adapted/struct/detail/namespace.hpp>", private, "<boost/fusion/adapted/struct/detail/proxy_type.hpp>", private ] },
+    { include: ["<boost/fusion/algorithm/query/detail/find_if.hpp>", private, "<boost/fusion/view/filter_view/detail/next_impl.hpp>", private ] },
+    { include: ["<boost/fusion/algorithm/transformation/detail/preprocessed/zip10.hpp>", private, "<boost/fusion/algorithm/transformation/detail/preprocessed/zip.hpp>", private ] },
+    { include: ["<boost/fusion/algorithm/transformation/detail/preprocessed/zip20.hpp>", private, "<boost/fusion/algorithm/transformation/detail/preprocessed/zip.hpp>", private ] },
+    { include: ["<boost/fusion/algorithm/transformation/detail/preprocessed/zip30.hpp>", private, "<boost/fusion/algorithm/transformation/detail/preprocessed/zip.hpp>", private ] },
+    { include: ["<boost/fusion/algorithm/transformation/detail/preprocessed/zip40.hpp>", private, "<boost/fusion/algorithm/transformation/detail/preprocessed/zip.hpp>", private ] },
+    { include: ["<boost/fusion/algorithm/transformation/detail/preprocessed/zip50.hpp>", private, "<boost/fusion/algorithm/transformation/detail/preprocessed/zip.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/at_impl.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/begin_impl.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/as_deque.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/build_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/deque_forward_ctor.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/deque_initial_size.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/deque_keyed_values_call.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/deque_keyed_values.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/limits.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/limits.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/limits.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque10.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque20.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque30.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque40.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque50.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/as_deque.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/as_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque10_fwd.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque10.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque20_fwd.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque20.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque30_fwd.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque30.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque40_fwd.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque40.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque50_fwd.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque50.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_fwd.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size10.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size20.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size30.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size40.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size50.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_initial_size.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque_initial_size.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values10.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values20.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values30.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values40.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values50.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/preprocessed/deque_keyed_values.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/end_impl.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/is_sequence_impl.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/detail/at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/detail/deque_keyed_values.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/detail/value_at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/deque/detail/value_at_impl.hpp>", private, "<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/deque_tie10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/deque_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/deque_tie20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/deque_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/deque_tie30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/deque_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/deque_tie40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/deque_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/deque_tie50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/deque_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/deque_tie.hpp>", private, "<boost/fusion/container/generation/detail/pp_deque_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/list_tie10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/list_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/list_tie20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/list_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/list_tie30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/list_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/list_tie40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/list_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/list_tie50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/list_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_deque10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_deque20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_deque30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_deque40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_deque50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_deque.hpp>", private, "<boost/fusion/container/generation/detail/pp_make_deque.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_list10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_list.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_list20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_list.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_list30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_list.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_list40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_list.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_list50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_list.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_map10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_map20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_map30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_map40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_map50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_map.hpp>", private, "<boost/fusion/container/generation/detail/pp_make_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_set10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_set20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_set30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_set40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_set50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_vector10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_vector20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_vector30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_vector40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_vector50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/make_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/map_tie10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/map_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/map_tie20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/map_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/map_tie30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/map_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/map_tie40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/map_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/map_tie50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/map_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/map_tie.hpp>", private, "<boost/fusion/container/generation/detail/pp_map_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/vector_tie10.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/vector_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/vector_tie20.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/vector_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/vector_tie30.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/vector_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/vector_tie40.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/vector_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/vector_tie50.hpp>", private, "<boost/fusion/container/generation/detail/preprocessed/vector_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/build_cons.hpp>", private, "<boost/fusion/container/list/detail/convert_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/list_to_cons_call.hpp>", private, "<boost/fusion/container/list/detail/list_to_cons.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list10_fwd.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list10.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list20_fwd.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list20.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list30_fwd.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list30.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list40_fwd.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list40.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list50_fwd.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list50.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list_to_cons10.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_to_cons.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list_to_cons20.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_to_cons.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list_to_cons30.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_to_cons.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list_to_cons40.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_to_cons.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list_to_cons50.hpp>", private, "<boost/fusion/container/list/detail/preprocessed/list_to_cons.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list_to_cons.hpp>", private, "<boost/fusion/container/list/detail/list_to_cons.hpp>", private ] },
+    { include: ["<boost/fusion/container/list/detail/reverse_cons.hpp>", private, "<boost/fusion/view/iterator_range/detail/segmented_iterator_range.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/as_map.hpp>", private, "<boost/fusion/container/map/detail/cpp03/convert.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/as_map.hpp>", private, "<boost/fusion/container/map/detail/cpp03/convert_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/at_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/begin_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/convert_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/convert.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/deref_data_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/deref_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/end_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/key_of_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/limits.hpp>", private, "<boost/fusion/container/generation/detail/pp_map_tie.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/limits.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/map_forward_ctor.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/map_fwd.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/map.hpp>", private, "<boost/fusion/container/map/detail/cpp03/as_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/map.hpp>", private, "<boost/fusion/container/map/detail/cpp03/convert.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/map.hpp>", private, "<boost/fusion/container/map/detail/cpp03/convert_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/as_map10.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/as_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/as_map20.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/as_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/as_map30.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/as_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/as_map40.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/as_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/as_map50.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/as_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/as_map.hpp>", private, "<boost/fusion/container/map/detail/cpp03/as_map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map10_fwd.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map10.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map20_fwd.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map20.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map30_fwd.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map30.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map40_fwd.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map40.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map50_fwd.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map50.hpp>", private, "<boost/fusion/container/map/detail/cpp03/preprocessed/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map_fwd.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/preprocessed/map.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/value_at_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/value_of_data_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/value_of_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/key_of_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/value_of_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/value_of_impl.hpp>", private, "<boost/fusion/container/map/detail/cpp03/value_of_data_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/as_set.hpp>", private, "<boost/fusion/container/set/detail/convert_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/deref_impl.hpp>", private, "<boost/fusion/container/set/detail/deref_data_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/as_set10.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/as_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/as_set20.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/as_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/as_set30.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/as_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/as_set40.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/as_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/as_set50.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/as_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/as_set.hpp>", private, "<boost/fusion/container/set/detail/as_set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set10_fwd.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set10.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set20_fwd.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set20.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set30_fwd.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set30.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set40_fwd.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set40.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set50_fwd.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set50.hpp>", private, "<boost/fusion/container/set/detail/preprocessed/set.hpp>", private ] },
+    { include: ["<boost/fusion/container/set/detail/value_of_data_impl.hpp>", private, "<boost/fusion/container/set/detail/key_of_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/as_vector.hpp>", private, "<boost/fusion/container/vector/detail/convert_impl.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/as_vector10.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/as_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/as_vector20.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/as_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/as_vector30.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/as_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/as_vector40.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/as_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/as_vector50.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/as_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/as_vector.hpp>", private, "<boost/fusion/container/vector/detail/as_vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector_chooser10.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_chooser.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector_chooser20.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_chooser.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector_chooser30.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_chooser.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector_chooser40.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_chooser.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector_chooser50.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_chooser.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector_chooser.hpp>", private, "<boost/fusion/container/vector/detail/vector_n_chooser.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector10_fwd.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector10.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector20_fwd.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector20.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector30_fwd.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector30.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector40_fwd.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector40.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector50_fwd.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vvector50.hpp>", private, "<boost/fusion/container/vector/detail/preprocessed/vector.hpp>", private ] },
+    { include: ["<boost/fusion/iterator/detail/adapt_deref_traits.hpp>", private, "<boost/fusion/view/filter_view/detail/deref_impl.hpp>", private ] },
+    { include: ["<boost/fusion/iterator/detail/adapt_deref_traits.hpp>", private, "<boost/fusion/view/joint_view/detail/deref_impl.hpp>", private ] },
+    { include: ["<boost/fusion/iterator/detail/adapt_value_traits.hpp>", private, "<boost/fusion/view/filter_view/detail/value_of_impl.hpp>", private ] },
+    { include: ["<boost/fusion/iterator/detail/adapt_value_traits.hpp>", private, "<boost/fusion/view/joint_view/detail/value_of_impl.hpp>", private ] },
+    { include: ["<boost/fusion/iterator/detail/segmented_equal_to.hpp>", private, "<boost/fusion/iterator/detail/segmented_iterator.hpp>", private ] },
+    { include: ["<boost/fusion/iterator/detail/segment_sequence.hpp>", private, "<boost/fusion/view/iterator_range/detail/segmented_iterator_range.hpp>", private ] },
+    { include: ["<boost/fusion/sequence/intrinsic/detail/segmented_begin_impl.hpp>", private, "<boost/fusion/sequence/intrinsic/detail/segmented_begin.hpp>", private ] },
+    { include: ["<boost/fusion/sequence/intrinsic/detail/segmented_end_impl.hpp>", private, "<boost/fusion/sequence/intrinsic/detail/segmented_begin_impl.hpp>", private ] },
+    { include: ["<boost/fusion/sequence/intrinsic/detail/segmented_end_impl.hpp>", private, "<boost/fusion/sequence/intrinsic/detail/segmented_end.hpp>", private ] },
+    { include: ["<boost/fusion/sequence/io/detail/manip.hpp>", private, "<boost/fusion/sequence/io/detail/in.hpp>", private ] },
+    { include: ["<boost/fusion/sequence/io/detail/manip.hpp>", private, "<boost/fusion/sequence/io/detail/out.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/adapted/std_tuple/detail/at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/algorithm/query/detail/count.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/deque/detail/keyed_element.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/list/detail/at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/list/detail/value_at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/map/detail/at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/map/detail/at_key_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/map/detail/cpp03/at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/map/detail/cpp03/deref_data_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/map/detail/cpp03/map.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/map/detail/map_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/map/detail/value_at_key_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/vector/detail/at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/vector/detail/deref_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/detail/pp_make_deque.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/detail/pp_make_map.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/functional/generation/detail/gen_make_adapter.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/spirit/home/support/detail/make_vector.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/mpl_iterator_category.hpp>", private, "<boost/fusion/adapted/mpl/detail/category_of_impl.hpp>", private ] },
+    { include: ["<boost/fusion/support/detail/segmented_fold_until_impl.hpp>", private, "<boost/fusion/sequence/intrinsic/detail/segmented_begin_impl.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/make_tuple10.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/make_tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/make_tuple20.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/make_tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/make_tuple30.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/make_tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/make_tuple40.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/make_tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/make_tuple50.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/make_tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple10_fwd.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple10.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple20_fwd.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple20.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple30_fwd.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple30.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple40_fwd.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple40.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple50_fwd.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_fwd.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple50.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple_tie10.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_tie.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple_tie20.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_tie.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple_tie30.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_tie.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple_tie40.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_tie.hpp>", private ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple_tie50.hpp>", private, "<boost/fusion/tuple/detail/preprocessed/tuple_tie.hpp>", private ] },
+    { include: ["<boost/fusion/view/iterator_range/detail/segmented_iterator_range.hpp>", private, "<boost/fusion/view/iterator_range/detail/segments_impl.hpp>", private ] },
+    { include: ["<boost/fusion/view/transform_view/detail/apply_transform_result.hpp>", private, "<boost/fusion/view/transform_view/detail/at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/view/transform_view/detail/apply_transform_result.hpp>", private, "<boost/fusion/view/transform_view/detail/deref_impl.hpp>", private ] },
+    { include: ["<boost/fusion/view/transform_view/detail/apply_transform_result.hpp>", private, "<boost/fusion/view/transform_view/detail/value_at_impl.hpp>", private ] },
+    { include: ["<boost/fusion/view/transform_view/detail/apply_transform_result.hpp>", private, "<boost/fusion/view/transform_view/detail/value_of_impl.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_values.hpp>", private, "<boost/geometry/algorithms/detail/assign_box_corners.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_values.hpp>", private, "<boost/geometry/algorithms/detail/assign_indexed_point.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/convert_point_to_point.hpp>", private, "<boost/geometry/algorithms/detail/point_is_spike_or_equal.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/convert_point_to_point.hpp>", private, "<boost/geometry/algorithms/detail/point_on_border.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/disjoint.hpp>", private, "<boost/geometry/algorithms/detail/overlay/append_no_duplicates.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/disjoint.hpp>", private, "<boost/geometry/algorithms/detail/overlay/append_no_dups_or_spikes.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/disjoint.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/disjoint.hpp>", private, "<boost/geometry/algorithms/detail/overlay/self_turn_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/disjoint.hpp>", private, "<boost/geometry/algorithms/detail/point_on_border.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/for_each_range.hpp>", private, "<boost/geometry/multi/algorithms/detail/for_each_range.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/get_left_turns.hpp>", private, "<boost/geometry/algorithms/detail/occupation_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/has_self_intersections.hpp>", private, "<boost/geometry/algorithms/detail/overlay/backtrack_check_si.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/add_rings.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/append_no_duplicates.hpp>", private, "<boost/geometry/algorithms/detail/overlay/clip_linestring.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/append_no_duplicates.hpp>", private, "<boost/geometry/algorithms/detail/overlay/follow.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/append_no_dups_or_spikes.hpp>", private, "<boost/geometry/algorithms/detail/overlay/copy_segments.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/append_no_dups_or_spikes.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traverse.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/assign_parents.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/backtrack_check_si.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traverse.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/calculate_distance_policy.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/check_enrich.hpp>", private, "<boost/geometry/algorithms/detail/overlay/enrich_intersection_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/clip_linestring.hpp>", private, "<boost/geometry/algorithms/detail/overlay/intersection_insert.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/convert_ring.hpp>", private, "<boost/geometry/algorithms/detail/overlay/add_rings.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/copy_segment_point.hpp>", private, "<boost/geometry/algorithms/detail/overlay/enrich_intersection_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/copy_segment_point.hpp>", private, "<boost/geometry/algorithms/detail/overlay/handle_tangencies.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/copy_segment_point.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/copy_segment_point.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/copy_segments.hpp>", private, "<boost/geometry/algorithms/detail/overlay/follow.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/copy_segments.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traverse.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/copy_segments.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/copy_segments.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/debug_turn_info.hpp>", private, "<boost/geometry/algorithms/detail/has_self_intersections.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/debug_turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/backtrack_check_si.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/debug_turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/enrich_intersection_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/debug_turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traverse.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/enrich_intersection_points.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/enrichment_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/enrichment_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traversal_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/follow.hpp>", private, "<boost/geometry/algorithms/detail/overlay/intersection_insert.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_intersection_points.hpp>", private, "<boost/geometry/algorithms/detail/overlay/intersection_insert.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_relative_order.hpp>", private, "<boost/geometry/algorithms/detail/overlay/enrich_intersection_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_ring.hpp>", private, "<boost/geometry/algorithms/detail/overlay/add_rings.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_ring.hpp>", private, "<boost/geometry/algorithms/detail/overlay/assign_parents.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_ring.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/get_ring.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/algorithms/detail/has_self_intersections.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_intersection_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/algorithms/detail/overlay/self_turn_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/handle_tangencies.hpp>", private, "<boost/geometry/algorithms/detail/overlay/enrich_intersection_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/msm_state.hpp>", private, "<boost/geometry/algorithms/detail/overlay/visit_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private, "<boost/geometry/algorithms/detail/overlay/intersection_insert.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/overlay_type.hpp>", private, "<boost/geometry/algorithms/detail/overlay/intersection_insert.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/overlay_type.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/overlay_type.hpp>", private, "<boost/geometry/algorithms/detail/overlay/select_rings.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/ring_properties.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/ring_properties.hpp>", private, "<boost/geometry/algorithms/detail/overlay/select_rings.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/segment_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/segment_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traversal_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/segment_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/select_rings.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/select_rings.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/select_rings.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/self_turn_points.hpp>", private, "<boost/geometry/algorithms/detail/has_self_intersections.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/self_turn_points.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/self_turn_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/traversal_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/traverse.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/has_self_intersections.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/backtrack_check_si.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/debug_turn_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/follow.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_turn_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/handle_tangencies.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traversal_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/turn_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traverse.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/visit_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/debug_turn_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/visit_info.hpp>", private, "<boost/geometry/algorithms/detail/overlay/traversal_info.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/partition.hpp>", private, "<boost/geometry/algorithms/detail/overlay/assign_parents.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/partition.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/partition.hpp>", private, "<boost/geometry/algorithms/detail/overlay/self_turn_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/point_is_spike_or_equal.hpp>", private, "<boost/geometry/algorithms/detail/overlay/append_no_dups_or_spikes.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/point_on_border.hpp>", private, "<boost/geometry/algorithms/detail/overlay/follow.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/point_on_border.hpp>", private, "<boost/geometry/algorithms/detail/overlay/intersection_insert.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/point_on_border.hpp>", private, "<boost/geometry/algorithms/detail/overlay/ring_properties.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/point_on_border.hpp>", private, "<boost/geometry/algorithms/detail/overlay/select_rings.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/point_on_border.hpp>", private, "<boost/geometry/multi/algorithms/detail/point_on_border.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/ring_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/check_enrich.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/ring_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/convert_ring.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/ring_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/enrich_intersection_points.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/ring_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_ring.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/ring_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/handle_tangencies.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/ring_identifier.hpp>", private, "<boost/geometry/algorithms/detail/overlay/select_rings.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/ring_identifier.hpp>", private, "<boost/geometry/algorithms/detail/sections/sectionalize.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/sections/range_by_section.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/sections/range_by_section.hpp>", private, "<boost/geometry/multi/algorithms/detail/sections/range_by_section.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/sections/sectionalize.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/algorithms/detail/sections/sectionalize.hpp>", private, "<boost/geometry/multi/algorithms/detail/sections/sectionalize.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/bounds.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/comparable_distance_centroid.hpp>", private, "<boost/geometry/index/detail/distance_predicates.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/comparable_distance_far.hpp>", private, "<boost/geometry/index/detail/distance_predicates.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/comparable_distance_near.hpp>", private, "<boost/geometry/index/detail/distance_predicates.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/content.hpp>", private, "<boost/geometry/index/detail/algorithms/intersection_content.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/content.hpp>", private, "<boost/geometry/index/detail/algorithms/union_content.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/content.hpp>", private, "<boost/geometry/index/detail/rtree/linear/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/content.hpp>", private, "<boost/geometry/index/detail/rtree/quadratic/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/content.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/choose_next_node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/content.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/insert.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/content.hpp>", private, "<boost/geometry/index/detail/rtree/visitors/insert.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/diff_abs.hpp>", private, "<boost/geometry/index/detail/algorithms/comparable_distance_centroid.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/diff_abs.hpp>", private, "<boost/geometry/index/detail/algorithms/comparable_distance_far.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/diff_abs.hpp>", private, "<boost/geometry/index/detail/algorithms/minmaxdist.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/intersection_content.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/choose_next_node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/intersection_content.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/margin.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/path_intersection.hpp>", private, "<boost/geometry/index/detail/distance_predicates.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/segment_intersection.hpp>", private, "<boost/geometry/index/detail/algorithms/path_intersection.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/smallest_for_indexable.hpp>", private, "<boost/geometry/index/detail/algorithms/minmaxdist.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/sum_for_indexable.hpp>", private, "<boost/geometry/index/detail/algorithms/comparable_distance_centroid.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/sum_for_indexable.hpp>", private, "<boost/geometry/index/detail/algorithms/comparable_distance_far.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/sum_for_indexable.hpp>", private, "<boost/geometry/index/detail/algorithms/comparable_distance_near.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/sum_for_indexable.hpp>", private, "<boost/geometry/index/detail/algorithms/minmaxdist.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/union_content.hpp>", private, "<boost/geometry/index/detail/rtree/quadratic/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/union_content.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/choose_next_node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/algorithms/union_content.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/assert.hpp>", private, "<boost/geometry/index/detail/pushable_array.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/assert.hpp>", private, "<boost/geometry/index/detail/varray.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/linear/redistribute_elements.hpp>", private, "<boost/geometry/index/detail/rtree/linear/linear.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/auto_deallocator.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/concept.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/dynamic_visitor.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node_auto_ptr.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node_d_mem_dynamic.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node_d_mem_static.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node.hpp>", private, "<boost/geometry/index/detail/rtree/linear/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node.hpp>", private, "<boost/geometry/index/detail/rtree/quadratic/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/choose_next_node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node.hpp>", private, "<boost/geometry/index/detail/rtree/utilities/are_boxes_ok.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node.hpp>", private, "<boost/geometry/index/detail/rtree/utilities/are_levels_ok.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node_s_mem_dynamic.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node_s_mem_static.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/pairs.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/static_visitor.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/quadratic/redistribute_elements.hpp>", private, "<boost/geometry/index/detail/rtree/quadratic/quadratic.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/rstar/choose_next_node.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/rstar.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/rstar/insert.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/rstar.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/rstar/redistribute_elements.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/rstar.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/destroy.hpp>", private, "<boost/geometry/index/detail/rtree/node/node_auto_ptr.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/insert.hpp>", private, "<boost/geometry/index/detail/rtree/linear/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/insert.hpp>", private, "<boost/geometry/index/detail/rtree/quadratic/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/insert.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/is_leaf.hpp>", private, "<boost/geometry/index/detail/rtree/linear/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/is_leaf.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/is_leaf.hpp>", private, "<boost/geometry/index/detail/rtree/quadratic/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/is_leaf.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/choose_next_node.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/is_leaf.hpp>", private, "<boost/geometry/index/detail/rtree/rstar/redistribute_elements.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/is_leaf.hpp>", private, "<boost/geometry/index/detail/rtree/visitors/remove.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/tags.hpp>", private, "<boost/geometry/index/detail/distance_predicates.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/tags.hpp>", private, "<boost/geometry/index/detail/predicates.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/varray_detail.hpp>", private, "<boost/geometry/index/detail/varray.hpp>", private ] },
+    { include: ["<boost/geometry/index/detail/varray.hpp>", private, "<boost/geometry/index/detail/rtree/node/node.hpp>", private ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/self_turn_points.hpp>", private, "<boost/geometry/algorithms/detail/has_self_intersections.hpp>", private ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/sections/range_by_section.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/sections/sectionalize.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/multi/views/detail/range_type.hpp>", private, "<boost/geometry/multi/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/views/detail/range_type.hpp>", private, "<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private ] },
+    { include: ["<boost/geometry/views/detail/range_type.hpp>", private, "<boost/geometry/multi/views/detail/range_type.hpp>", private ] },
+    { include: ["<boost/graph/detail/adj_list_edge_iterator.hpp>", private, "<boost/graph/detail/adjacency_list.hpp>", private ] },
+    { include: ["<boost/graph/detail/histogram_sort.hpp>", private, "<boost/graph/detail/compressed_sparse_row_struct.hpp>", private ] },
+    { include: ["<boost/graph/detail/indexed_properties.hpp>", private, "<boost/graph/detail/compressed_sparse_row_struct.hpp>", private ] },
+    { include: ["<boost/graph/detail/shadow_iterator.hpp>", private, "<boost/graph/detail/permutation.hpp>", private ] },
+    { include: ["<boost/graph/parallel/detail/untracked_pair.hpp>", private, "<boost/graph/parallel/detail/property_holders.hpp>", private ] },
+    { include: ["<boost/heap/detail/ordered_adaptor_iterator.hpp>", private, "<boost/heap/detail/mutable_heap.hpp>", private ] },
+    { include: ["<boost/heap/detail/tree_iterator.hpp>", private, "<boost/heap/detail/ordered_adaptor_iterator.hpp>", private ] },
+    { include: ["<boost/icl/detail/associated_value.hpp>", private, "<boost/icl/detail/interval_set_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/design_config.hpp>", private, "<boost/icl/detail/associated_value.hpp>", private ] },
+    { include: ["<boost/icl/detail/element_comparer.hpp>", private, "<boost/icl/detail/interval_map_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/element_comparer.hpp>", private, "<boost/icl/detail/interval_set_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/interval_subset_comparer.hpp>", private, "<boost/icl/detail/interval_map_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/interval_subset_comparer.hpp>", private, "<boost/icl/detail/interval_set_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/mapped_reference.hpp>", private, "<boost/icl/detail/element_iterator.hpp>", private ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/detail/element_comparer.hpp>", private ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/detail/interval_map_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/detail/interval_morphism.hpp>", private ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/detail/interval_set_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/detail/interval_subset_comparer.hpp>", private ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/detail/map_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/detail/set_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/detail/subset_comparer.hpp>", private ] },
+    { include: ["<boost/icl/detail/relation_state.hpp>", private, "<boost/icl/detail/interval_map_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/relation_state.hpp>", private, "<boost/icl/detail/interval_set_algo.hpp>", private ] },
+    { include: ["<boost/icl/detail/relation_state.hpp>", private, "<boost/icl/detail/interval_subset_comparer.hpp>", private ] },
+    { include: ["<boost/icl/detail/relation_state.hpp>", private, "<boost/icl/detail/subset_comparer.hpp>", private ] },
+    { include: ["<boost/icl/detail/set_algo.hpp>", private, "<boost/icl/detail/map_algo.hpp>", private ] },
+    { include: ["<boost/interprocess/allocators/detail/allocator_common.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/allocators/detail/allocator_common.hpp>", private, "<boost/interprocess/allocators/detail/node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/allocators/detail/node_tools.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/detail/intermodule_singleton_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/detail/robust_emulation.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_base_atomic.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/cast_tags.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/detail/allocator_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/detail/node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/detail/node_tools.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/atomic.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/cast_tags.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/file_locking_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/in_place_interface.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/intermodule_singleton_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/managed_global_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/managed_memory_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/managed_multi_shared_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/min_max.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/multi_segment_services.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/named_proxy.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/os_file_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/os_thread_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/pointer_type.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/preprocessor.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/robust_emulation.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/segment_manager_helper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/tmp_dir_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/type_traits.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/utilities.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/variadic_templates_tools.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/win32_api.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/workaround.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_device.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/detail/bad_weak_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_base_atomic.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/detail/condition_algorithm_8a.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/detail/condition_any_algorithm.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/detail/locks.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/detail/allocator_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/detail/node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/detail/node_tools.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/atomic.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/cast_tags.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/file_locking_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/in_place_interface.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/intermodule_singleton_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/managed_global_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/managed_memory_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/managed_multi_shared_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/min_max.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/multi_segment_services.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/named_proxy.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/os_file_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/os_thread_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/pointer_type.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/preprocessor.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/robust_emulation.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/segment_manager_helper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/tmp_dir_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/type_traits.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/utilities.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/variadic_templates_tools.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/win32_api.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/workaround.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_device.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/detail/bad_weak_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_base_atomic.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/detail/condition_algorithm_8a.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/detail/condition_any_algorithm.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/detail/locks.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/file_locking_helpers.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/in_place_interface.hpp>", private, "<boost/interprocess/detail/named_proxy.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/in_place_interface.hpp>", private, "<boost/interprocess/detail/segment_manager_helper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/intermodule_singleton_common.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/intermodule_singleton_common.hpp>", private, "<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/intermodule_singleton.hpp>", private, "<boost/interprocess/detail/robust_emulation.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/managed_global_memory.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/managed_memory_impl.hpp>", private, "<boost/interprocess/detail/managed_global_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/managed_memory_impl.hpp>", private, "<boost/interprocess/detail/managed_multi_shared_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/detail/managed_global_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/detail/managed_multi_shared_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/math_functions.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/math_functions.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/math_functions.hpp>", private, "<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/min_max.hpp>", private, "<boost/interprocess/detail/utilities.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/min_max.hpp>", private, "<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/min_max.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/min_max.hpp>", private, "<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/detail/intermodule_singleton_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/detail/named_proxy.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/detail/utilities.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/multi_segment_services.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/multi_segment_services.hpp>", private, "<boost/interprocess/detail/managed_multi_shared_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/multi_segment_services.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/file_locking_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/managed_memory_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/robust_emulation.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/tmp_dir_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_device.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/detail/intermodule_singleton_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private, "<boost/interprocess/detail/intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/detail/os_thread_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/preprocessor.hpp>", private, "<boost/interprocess/detail/named_proxy.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/segment_manager_helper.hpp>", private, "<boost/interprocess/allocators/detail/allocator_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/tmp_dir_helpers.hpp>", private, "<boost/interprocess/detail/file_locking_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/tmp_dir_helpers.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/tmp_dir_helpers.hpp>", private, "<boost/interprocess/detail/robust_emulation.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/tmp_dir_helpers.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_device.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/tmp_dir_helpers.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/transform_iterator.hpp>", private, "<boost/interprocess/detail/utilities.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/allocators/detail/allocator_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/detail/in_place_interface.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/detail/pointer_type.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/detail/segment_manager_helper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/detail/utilities.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/detail/variadic_templates_tools.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/detail/allocator_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/detail/node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/detail/managed_memory_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/detail/managed_multi_shared_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/detail/segment_manager_helper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_device.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/variadic_templates_tools.hpp>", private, "<boost/interprocess/detail/named_proxy.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/detail/atomic.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/detail/os_file_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/detail/os_thread_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private, "<boost/interprocess/detail/intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private, "<boost/interprocess/detail/tmp_dir_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/allocators/detail/allocator_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/allocators/detail/node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/allocators/detail/node_tools.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/atomic.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/cast_tags.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/file_locking_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/in_place_interface.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/intermodule_singleton_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/intersegment_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/managed_global_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/managed_memory_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/managed_multi_shared_memory.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/min_max.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/multi_segment_services.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/named_proxy.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/os_file_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/os_thread_functions.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/pointer_type.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/portable_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/robust_emulation.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/segment_manager_helper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/tmp_dir_helpers.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/utilities.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/variadic_templates_tools.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/win32_api.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_device.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/detail/xsi_shared_memory_file_wrapper.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/smart_ptr/detail/bad_weak_ptr.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_base_atomic.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/sync/detail/condition_algorithm_8a.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/sync/detail/condition_any_algorithm.hpp>", private ] },
+    { include: ["<boost/interprocess/detail/workaround.hpp>", private, "<boost/interprocess/sync/detail/locks.hpp>", private ] },
+    { include: ["<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private, "<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private ] },
+    { include: ["<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private, "<boost/interprocess/allocators/detail/allocator_common.hpp>", private ] },
+    { include: ["<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private, "<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private, "<boost/interprocess/mem_algo/detail/multi_simple_seq_fit.hpp>", private ] },
+    { include: ["<boost/interprocess/smart_ptr/detail/bad_weak_ptr.hpp>", private, "<boost/interprocess/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/interprocess/smart_ptr/detail/sp_counted_base_atomic.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/interprocess/smart_ptr/detail/sp_counted_base.hpp>", private, "<boost/interprocess/smart_ptr/detail/sp_counted_impl.hpp>", private ] },
+    { include: ["<boost/interprocess/smart_ptr/detail/sp_counted_impl.hpp>", private, "<boost/interprocess/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/interprocess/sync/detail/locks.hpp>", private, "<boost/interprocess/sync/detail/condition_algorithm_8a.hpp>", private ] },
+    { include: ["<boost/interprocess/sync/detail/locks.hpp>", private, "<boost/interprocess/sync/detail/condition_any_algorithm.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/detail/any_node_and_algorithms.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/detail/common_slist_algorithms.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/detail/hashtable_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/detail/list_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/detail/slist_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/detail/utilities.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/any_node_and_algorithms.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/avltree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/clear_on_destructor_base.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/common_slist_algorithms.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/ebo_functor_holder.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/function_detector.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/generic_hook.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/hashtable_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/has_member_function_callable_with.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/is_stateful_value_traits.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/list_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/mpl.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/parent_from_member.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/preprocessor.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/rbtree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/slist_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/tree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/utilities.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/detail/workaround.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/any_node_and_algorithms.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/avltree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/clear_on_destructor_base.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/common_slist_algorithms.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/ebo_functor_holder.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/function_detector.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/generic_hook.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/has_member_function_callable_with.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/is_stateful_value_traits.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/list_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/mpl.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/parent_from_member.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/preprocessor.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/rbtree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/slist_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/tree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/utilities.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/detail/workaround.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/ebo_functor_holder.hpp>", private, "<boost/intrusive/detail/utilities.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/function_detector.hpp>", private, "<boost/intrusive/detail/is_stateful_value_traits.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/has_member_function_callable_with.hpp>", private, "<boost/container/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/has_member_function_callable_with.hpp>", private, "<boost/intrusive/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/is_stateful_value_traits.hpp>", private, "<boost/intrusive/detail/utilities.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/any_node_and_algorithms.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/avltree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/ebo_functor_holder.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/generic_hook.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/hashtable_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/has_member_function_callable_with.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/is_stateful_value_traits.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/rbtree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/transform_iterator.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/tree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/detail/utilities.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/parent_from_member.hpp>", private, "<boost/intrusive/detail/utilities.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/preprocessor.hpp>", private, "<boost/intrusive/detail/has_member_function_callable_with.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/preprocessor.hpp>", private, "<boost/intrusive/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/tree_node.hpp>", private, "<boost/intrusive/detail/rbtree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/detail/generic_hook.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/detail/hashtable_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/detail/rbtree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/detail/slist_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/detail/tree_node.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/workaround.hpp>", private, "<boost/intrusive/detail/has_member_function_callable_with.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/workaround.hpp>", private, "<boost/intrusive/detail/memory_util.hpp>", private ] },
+    { include: ["<boost/intrusive/detail/workaround.hpp>", private, "<boost/intrusive/detail/preprocessor.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/access_control.hpp>", private, "<boost/iostreams/detail/streambuf/chainbuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/adapter/concept_adapter.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/adapter/device_adapter.hpp>", private, "<boost/iostreams/detail/restrict_impl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/adapter/filter_adapter.hpp>", private, "<boost/iostreams/detail/restrict_impl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/adapter/mode_adapter.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/adapter/non_blocking_adapter.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/adapter/output_iterator_adapter.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/adapter/range_adapter.hpp>", private, "<boost/iostreams/detail/push.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/adapter/range_adapter.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/bool_trait_def.hpp>", private, "<boost/iostreams/detail/is_iterator_range.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/broken_overload_resolution/forward.hpp>", private, "<boost/iostreams/detail/broken_overload_resolution/stream_buffer.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/broken_overload_resolution/forward.hpp>", private, "<boost/iostreams/detail/broken_overload_resolution/stream.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/buffer.hpp>", private, "<boost/iostreams/detail/current_directory.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/buffer.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/call_traits.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/call_traits.hpp>", private, "<boost/iostreams/detail/adapter/device_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/call_traits.hpp>", private, "<boost/iostreams/detail/adapter/filter_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/call_traits.hpp>", private, "<boost/iostreams/detail/restrict_impl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/detail/counted_array.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/detail/streambuf/linked_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/codecvt.hpp>", private, "<boost/iostreams/detail/codecvt_helper.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/codecvt.hpp>", private, "<boost/iostreams/detail/codecvt_holder.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/adapter/direct_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/adapter/range_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/current_directory.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/is_iterator_range.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/restrict_impl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/detail/streambuf/linked_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/adapter/direct_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/adapter/range_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/current_directory.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/is_iterator_range.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/detail/streambuf/linked_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/gcc.hpp>", private, "<boost/iostreams/detail/config/overload_resolution.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/gcc.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/limits.hpp>", private, "<boost/iostreams/detail/adapter/direct_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/limits.hpp>", private, "<boost/iostreams/detail/execute.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/limits.hpp>", private, "<boost/iostreams/detail/forward.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/overload_resolution.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/unreachable_return.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/adapter/direct_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/char_traits.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/config/codecvt.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/fstream.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/ios.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/iostream.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/path.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/push.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/streambuf/chainbuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/windows_posix.hpp>", private, "<boost/iostreams/detail/absolute_path.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/windows_posix.hpp>", private, "<boost/iostreams/detail/config/rtl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/windows_posix.hpp>", private, "<boost/iostreams/detail/current_directory.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/windows_posix.hpp>", private, "<boost/iostreams/detail/file_handle.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/config/windows_posix.hpp>", private, "<boost/iostreams/detail/system_failure.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/current_directory.hpp>", private, "<boost/iostreams/detail/absolute_path.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/dispatch.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/double_object.hpp>", private, "<boost/iostreams/detail/adapter/direct_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/double_object.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/enable_if_stream.hpp>", private, "<boost/iostreams/detail/push.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/enable_if_stream.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/enable_if_stream.hpp>", private, "<boost/iostreams/detail/restrict_impl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/enable_if_stream.hpp>", private, "<boost/iostreams/detail/wrap_unwrap.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/error.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/error.hpp>", private, "<boost/iostreams/detail/adapter/direct_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/error.hpp>", private, "<boost/iostreams/detail/adapter/range_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/error.hpp>", private, "<boost/iostreams/detail/restrict_impl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/error.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/execute.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/execute.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/functional.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/functional.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/adapter/device_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/adapter/direct_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/adapter/filter_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/adapter/mode_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/adapter/non_blocking_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/buffer.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/counted_array.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/error.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/functional.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/restrict_impl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/streambuf/linked_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/detail/system_failure.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/is_dereferenceable.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/is_iterator_range.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/optional.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/optional.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/push.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/push_params.hpp>", private, "<boost/iostreams/detail/forward.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/push_params.hpp>", private, "<boost/iostreams/detail/push.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/resolve.hpp>", private, "<boost/iostreams/detail/push.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/detail/access_control.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/detail/dispatch.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/detail/restrict_impl.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/detail/adapter/concept_adapter.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/detail/streambuf/chainbuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/detail/streambuf/linked_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/streambuf/linked_streambuf.hpp>", private, "<boost/iostreams/detail/streambuf/chainbuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/streambuf/linked_streambuf.hpp>", private, "<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/streambuf/linked_streambuf.hpp>", private, "<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/system_failure.hpp>", private, "<boost/iostreams/detail/current_directory.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/template_params.hpp>", private, "<boost/iostreams/detail/bool_trait_def.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/translate_int_type.hpp>", private, "<boost/iostreams/detail/streambuf/chainbuf.hpp>", private ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/detail/resolve.hpp>", private ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/detail/enable_if.hpp>", private ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/detail/facade_iterator_category.hpp>", private ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/detail/enable_if.hpp>", private ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/detail/facade_iterator_category.hpp>", private ] },
+    { include: ["<boost/iterator/detail/enable_if.hpp>", private, "<boost/bimap/detail/map_view_iterator.hpp>", private ] },
+    { include: ["<boost/iterator/detail/enable_if.hpp>", private, "<boost/bimap/detail/set_view_iterator.hpp>", private ] },
+    { include: ["<boost/lambda/detail/is_instance_of.hpp>", private, "<boost/lambda/detail/operator_return_type_traits.hpp>", private ] },
+    { include: ["<boost/lambda/detail/is_instance_of.hpp>", private, "<boost/lambda/detail/operators.hpp>", private ] },
+    { include: ["<boost/lambda/detail/lambda_fwd.hpp>", private, "<boost/lambda/detail/lambda_functor_base.hpp>", private ] },
+    { include: ["<boost/lambda/detail/lambda_traits.hpp>", private, "<boost/lambda/detail/lambda_functor_base.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/bind.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/const_bind.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/const.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/const_bind.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/auto.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/bind.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/const_bind.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/const.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/default.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/inline.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/recursive.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/register.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/return.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/this.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/thisunderscore.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/add.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/void.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/auto.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/bind.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/const.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/default.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/inline.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/recursive.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/register.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/return.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/this.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/thisunderscore.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/void.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/auto.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/bind.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/const.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/default.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/inline.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/recursive.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/register.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/return.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/this.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/thisunderscore.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/facility/remove.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/void.hpp>", private ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/void.hpp>", private, "<boost/local_function/detail/preprocessor/void_list.hpp>", private ] },
+    { include: ["<boost/lockfree/detail/atomic.hpp>", private, "<boost/lockfree/detail/freelist.hpp>", private ] },
+    { include: ["<boost/lockfree/detail/branch_hints.hpp>", private, "<boost/lockfree/detail/tagged_ptr_dcas.hpp>", private ] },
+    { include: ["<boost/lockfree/detail/branch_hints.hpp>", private, "<boost/lockfree/detail/tagged_ptr_ptrcompression.hpp>", private ] },
+    { include: ["<boost/lockfree/detail/parameter.hpp>", private, "<boost/lockfree/detail/freelist.hpp>", private ] },
+    { include: ["<boost/lockfree/detail/prefix.hpp>", private, "<boost/lockfree/detail/tagged_ptr.hpp>", private ] },
+    { include: ["<boost/lockfree/detail/tagged_ptr_dcas.hpp>", private, "<boost/lockfree/detail/tagged_ptr.hpp>", private ] },
+    { include: ["<boost/lockfree/detail/tagged_ptr.hpp>", private, "<boost/lockfree/detail/freelist.hpp>", private ] },
+    { include: ["<boost/lockfree/detail/tagged_ptr_ptrcompression.hpp>", private, "<boost/lockfree/detail/tagged_ptr.hpp>", private ] },
+    { include: ["<boost/log/detail/attr_output_terminal.hpp>", private, "<boost/log/detail/attr_output_impl.hpp>", private ] },
+    { include: ["<boost/log/detail/cleanup_scope_guard.hpp>", private, "<boost/log/detail/format.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/asio_fwd.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/attachable_sstream_buf.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/attribute_get_value_impl.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/attribute_predicate.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/attr_output_impl.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/attr_output_terminal.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/cleanup_scope_guard.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/code_conversion.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/custom_terminal_spec.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/date_time_fmt_gen_traits_fwd.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/date_time_format_parser.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/decomposed_time.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/deduce_char_type.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/default_attribute_names.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/embedded_string_type.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/event.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/fake_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/format.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/function_traits.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/id.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/light_function.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/light_rw_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/locking_ptr.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/locks.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/malloc_aligned.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/native_typeof.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/parameter_tools.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/pp_identity.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/process_id.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/setup_config.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/singleton.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/sink_init_helpers.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/snprintf.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/spin_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/tagged_integer.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/thread_id.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/threadsafe_queue.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/thread_specific.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/timestamp.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/trivial_keyword.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/unary_function_terminal.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/unhandled_exception_count.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/value_ref_visitation.hpp>", private ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/detail/visible_type.hpp>", private ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/detail/attr_output_terminal.hpp>", private ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/detail/unary_function_terminal.hpp>", private ] },
+    { include: ["<boost/log/detail/date_time_format_parser.hpp>", private, "<boost/log/detail/decomposed_time.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/attachable_sstream_buf.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/attribute_get_value_impl.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/attribute_predicate.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/attr_output_impl.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/attr_output_terminal.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/cleanup_scope_guard.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/code_conversion.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/custom_terminal_spec.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/date_time_format_parser.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/decomposed_time.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/deduce_char_type.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/default_attribute_names.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/embedded_string_type.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/event.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/fake_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/format.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/function_traits.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/id.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/light_function.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/light_rw_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/locking_ptr.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/locks.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/malloc_aligned.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/parameter_tools.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/process_id.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/singleton.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/sink_init_helpers.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/snprintf.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/spin_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/tagged_integer.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/thread_id.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/threadsafe_queue.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/thread_specific.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/timestamp.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/trivial_keyword.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/unary_function_terminal.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/value_ref_visitation.hpp>", private ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/detail/visible_type.hpp>", private ] },
+    { include: ["<boost/log/detail/generate_overloads.hpp>", private, "<boost/log/detail/attr_output_impl.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/attachable_sstream_buf.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/attribute_get_value_impl.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/attribute_predicate.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/attr_output_impl.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/attr_output_terminal.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/cleanup_scope_guard.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/code_conversion.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/custom_terminal_spec.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/date_time_format_parser.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/decomposed_time.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/deduce_char_type.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/default_attribute_names.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/embedded_string_type.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/event.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/fake_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/format.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/function_traits.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/id.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/light_function.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/light_rw_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/locking_ptr.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/locks.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/malloc_aligned.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/parameter_tools.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/process_id.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/singleton.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/sink_init_helpers.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/snprintf.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/spin_mutex.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/tagged_integer.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/thread_id.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/threadsafe_queue.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/thread_specific.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/timestamp.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/trivial_keyword.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/unary_function_terminal.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/value_ref_visitation.hpp>", private ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/detail/visible_type.hpp>", private ] },
+    { include: ["<boost/log/detail/id.hpp>", private, "<boost/log/detail/process_id.hpp>", private ] },
+    { include: ["<boost/log/detail/id.hpp>", private, "<boost/log/detail/thread_id.hpp>", private ] },
+    { include: ["<boost/log/detail/unhandled_exception_count.hpp>", private, "<boost/log/detail/format.hpp>", private ] },
+    { include: ["<boost/math/bindings/detail/big_lanczos.hpp>", private, "<boost/multiprecision/detail/big_lanczos.hpp>", private ] },
+    { include: ["<boost/math/distributions/detail/hypergeometric_pdf.hpp>", private, "<boost/math/distributions/detail/hypergeometric_cdf.hpp>", private ] },
+    { include: ["<boost/math/distributions/detail/hypergeometric_pdf.hpp>", private, "<boost/math/distributions/detail/hypergeometric_quantile.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/airy_ai_bi_zero.hpp>", private, "<boost/math/special_functions/detail/bessel_jy_zero.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_j0.hpp>", private, "<boost/math/special_functions/detail/bessel_jn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_j0.hpp>", private, "<boost/math/special_functions/detail/bessel_y0.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_j1.hpp>", private, "<boost/math/special_functions/detail/bessel_jn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_j1.hpp>", private, "<boost/math/special_functions/detail/bessel_y1.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jy_asym.hpp>", private, "<boost/math/special_functions/detail/bessel_jn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jy_asym.hpp>", private, "<boost/math/special_functions/detail/bessel_jy.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jy.hpp>", private, "<boost/math/special_functions/detail/bessel_jn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jy_series.hpp>", private, "<boost/math/special_functions/detail/bessel_jn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jy_series.hpp>", private, "<boost/math/special_functions/detail/bessel_jy.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jy_series.hpp>", private, "<boost/math/special_functions/detail/bessel_yn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_k0.hpp>", private, "<boost/math/special_functions/detail/bessel_kn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_k1.hpp>", private, "<boost/math/special_functions/detail/bessel_kn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_y0.hpp>", private, "<boost/math/special_functions/detail/bessel_yn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/bessel_y1.hpp>", private, "<boost/math/special_functions/detail/bessel_yn.hpp>", private ] },
+    { include: ["<boost/math/special_functions/detail/t_distribution_inv.hpp>", private, "<boost/math/special_functions/detail/ibeta_inverse.hpp>", private ] },
+    { include: ["<boost/move/detail/config_begin.hpp>", private, "<boost/move/detail/meta_utils.hpp>", private ] },
+    { include: ["<boost/move/detail/config_end.hpp>", private, "<boost/move/detail/meta_utils.hpp>", private ] },
+    { include: ["<boost/mpi/detail/forward_skeleton_oarchive.hpp>", private, "<boost/mpi/detail/text_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/mpi/detail/ignore_oprimitive.hpp>", private, "<boost/mpi/detail/text_skeleton_oarchive.hpp>", private ] },
+    { include: ["<boost/mpi/detail/ignore_skeleton_oarchive.hpp>", private, "<boost/mpi/detail/content_oarchive.hpp>", private ] },
+    { include: ["<boost/mpi/detail/ignore_skeleton_oarchive.hpp>", private, "<boost/mpi/detail/mpi_datatype_oarchive.hpp>", private ] },
+    { include: ["<boost/mpi/detail/mpi_datatype_oarchive.hpp>", private, "<boost/mpi/detail/mpi_datatype_cache.hpp>", private ] },
+    { include: ["<boost/mpi/detail/mpi_datatype_primitive.hpp>", private, "<boost/mpi/detail/content_oarchive.hpp>", private ] },
+    { include: ["<boost/mpi/detail/mpi_datatype_primitive.hpp>", private, "<boost/mpi/detail/mpi_datatype_oarchive.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/access_specifier.hpp>", private, "<boost/multi_index/detail/safe_mode.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/adl_swap.hpp>", private, "<boost/multi_index/detail/auto_space.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/auto_space.hpp>", private, "<boost/multi_index/detail/bucket_array.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/auto_space.hpp>", private, "<boost/multi_index/detail/copy_map.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/auto_space.hpp>", private, "<boost/multi_index/detail/index_loader.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/auto_space.hpp>", private, "<boost/multi_index/detail/index_matcher.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/auto_space.hpp>", private, "<boost/multi_index/detail/rnd_index_loader.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/auto_space.hpp>", private, "<boost/multi_index/detail/rnd_index_ptr_array.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/copy_map.hpp>", private, "<boost/multi_index/detail/index_base.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/do_not_copy_elements_tag.hpp>", private, "<boost/multi_index/detail/index_base.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/hash_index_node.hpp>", private, "<boost/multi_index/detail/bucket_array.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/header_holder.hpp>", private, "<boost/multi_index/detail/node_type.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/index_base.hpp>", private, "<boost/multi_index/detail/base_type.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/index_loader.hpp>", private, "<boost/multi_index/detail/index_base.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/index_matcher.hpp>", private, "<boost/multi_index/detail/index_saver.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/index_node_base.hpp>", private, "<boost/multi_index/detail/node_type.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/index_saver.hpp>", private, "<boost/multi_index/detail/index_base.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/is_index_list.hpp>", private, "<boost/multi_index/detail/base_type.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/is_index_list.hpp>", private, "<boost/multi_index/detail/node_type.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/iter_adaptor.hpp>", private, "<boost/multi_index/detail/safe_mode.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/msvc_index_specifier.hpp>", private, "<boost/multi_index/detail/base_type.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/msvc_index_specifier.hpp>", private, "<boost/multi_index/detail/node_type.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/node_type.hpp>", private, "<boost/multi_index/detail/index_base.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/auto_space.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/bucket_array.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/copy_map.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/hash_index_node.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/iter_adaptor.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/ord_index_node.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/rnd_index_loader.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/rnd_index_node.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/rnd_index_ptr_array.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/detail/seq_index_node.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/rnd_index_node.hpp>", private, "<boost/multi_index/detail/rnd_index_ptr_array.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/rnd_index_ptr_array.hpp>", private, "<boost/multi_index/detail/rnd_index_loader.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/rnd_index_ptr_array.hpp>", private, "<boost/multi_index/detail/rnd_index_ops.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/safe_mode.hpp>", private, "<boost/multi_index/detail/safe_ctr_proxy.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/scope_guard.hpp>", private, "<boost/signals2/detail/auto_buffer.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/seq_index_node.hpp>", private, "<boost/multi_index/detail/seq_index_ops.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/uintptr_type.hpp>", private, "<boost/multi_index/detail/ord_index_node.hpp>", private ] },
+    { include: ["<boost/multi_index/detail/vartempl_support.hpp>", private, "<boost/multi_index/detail/index_base.hpp>", private ] },
+    { include: ["<boost/multiprecision/detail/default_ops.hpp>", private, "<boost/multiprecision/detail/generic_interconvert.hpp>", private ] },
+    { include: ["<boost/multiprecision/detail/et_ops.hpp>", private, "<boost/multiprecision/detail/default_ops.hpp>", private ] },
+    { include: ["<boost/multiprecision/detail/functions/constants.hpp>", private, "<boost/multiprecision/detail/default_ops.hpp>", private ] },
+    { include: ["<boost/multiprecision/detail/functions/pow.hpp>", private, "<boost/multiprecision/detail/default_ops.hpp>", private ] },
+    { include: ["<boost/multiprecision/detail/functions/trig.hpp>", private, "<boost/multiprecision/detail/default_ops.hpp>", private ] },
+    { include: ["<boost/multiprecision/detail/no_et_ops.hpp>", private, "<boost/multiprecision/detail/default_ops.hpp>", private ] },
+    { include: ["<boost/multiprecision/detail/number_base.hpp>", private, "<boost/multiprecision/detail/default_ops.hpp>", private ] },
+    { include: ["<boost/multiprecision/detail/rebind.hpp>", private, "<boost/multiprecision/detail/dynamic_array.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/conversion_traits.hpp>", private, "<boost/numeric/conversion/detail/converter.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/int_float_mixture.hpp>", private, "<boost/numeric/conversion/detail/conversion_traits.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/int_float_mixture.hpp>", private, "<boost/numeric/conversion/detail/is_subranged.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/is_subranged.hpp>", private, "<boost/numeric/conversion/detail/conversion_traits.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/meta.hpp>", private, "<boost/numeric/conversion/detail/conversion_traits.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/meta.hpp>", private, "<boost/numeric/conversion/detail/converter.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/meta.hpp>", private, "<boost/numeric/conversion/detail/int_float_mixture.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/meta.hpp>", private, "<boost/numeric/conversion/detail/is_subranged.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/meta.hpp>", private, "<boost/numeric/conversion/detail/sign_mixture.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/meta.hpp>", private, "<boost/numeric/conversion/detail/udt_builtin_mixture.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/preprocessed/numeric_cast_traits_common.hpp>", private, "<boost/numeric/conversion/detail/numeric_cast_traits.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/preprocessed/numeric_cast_traits_long_long.hpp>", private, "<boost/numeric/conversion/detail/numeric_cast_traits.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/sign_mixture.hpp>", private, "<boost/numeric/conversion/detail/conversion_traits.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/sign_mixture.hpp>", private, "<boost/numeric/conversion/detail/is_subranged.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/udt_builtin_mixture.hpp>", private, "<boost/numeric/conversion/detail/conversion_traits.hpp>", private ] },
+    { include: ["<boost/numeric/conversion/detail/udt_builtin_mixture.hpp>", private, "<boost/numeric/conversion/detail/is_subranged.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/bcc_rounding_control.hpp>", private, "<boost/numeric/interval/detail/x86_rounding_control.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/bugs.hpp>", private, "<boost/numeric/interval/detail/division.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/c99sub_rounding_control.hpp>", private, "<boost/numeric/interval/detail/c99_rounding_control.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/c99sub_rounding_control.hpp>", private, "<boost/numeric/interval/detail/x86_rounding_control.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/detail/division.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/detail/test_input.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/msvc_rounding_control.hpp>", private, "<boost/numeric/interval/detail/x86_rounding_control.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/detail/division.hpp>", private ] },
+    { include: ["<boost/numeric/interval/detail/x86gcc_rounding_control.hpp>", private, "<boost/numeric/interval/detail/x86_rounding_control.hpp>", private ] },
+    { include: ["<boost/numeric/odeint/integrate/detail/integrate_adaptive.hpp>", private, "<boost/numeric/odeint/integrate/detail/integrate_const.hpp>", private ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/generic_rk_call_algebra.hpp>", private, "<boost/numeric/odeint/stepper/detail/generic_rk_algorithm.hpp>", private ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/generic_rk_operations.hpp>", private, "<boost/numeric/odeint/stepper/detail/generic_rk_algorithm.hpp>", private ] },
+    { include: ["<boost/numeric/odeint/util/detail/less_with_sign.hpp>", private, "<boost/numeric/odeint/integrate/detail/integrate_adaptive.hpp>", private ] },
+    { include: ["<boost/numeric/odeint/util/detail/less_with_sign.hpp>", private, "<boost/numeric/odeint/integrate/detail/integrate_const.hpp>", private ] },
+    { include: ["<boost/numeric/odeint/util/detail/less_with_sign.hpp>", private, "<boost/numeric/odeint/integrate/detail/integrate_n_steps.hpp>", private ] },
+    { include: ["<boost/numeric/odeint/util/detail/less_with_sign.hpp>", private, "<boost/numeric/odeint/integrate/detail/integrate_times.hpp>", private ] },
+    { include: ["<boost/numeric/ublas/detail/definitions.hpp>", private, "<boost/numeric/ublas/detail/config.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/function_ptr_10.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/function_ptr_20.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/function_ptr_30.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/function_ptr_40.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/function_ptr_50.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/function_ptr.hpp>", private, "<boost/phoenix/bind/detail/function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/member_function_ptr_10.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/member_function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/member_function_ptr_20.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/member_function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/member_function_ptr_30.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/member_function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/member_function_ptr_40.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/member_function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/member_function_ptr_50.hpp>", private, "<boost/phoenix/bind/detail/preprocessed/member_function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/bind/detail/preprocessed/member_function_ptr.hpp>", private, "<boost/phoenix/bind/detail/member_function_ptr.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/phx2_result.hpp>", private, "<boost/phoenix/core/detail/function_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_operator_10.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_operator_20.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_operator_30.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_operator_40.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_operator_50.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_operator.hpp>", private, "<boost/phoenix/core/detail/actor_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_result_of_10.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_result_of_20.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_result_of_30.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_result_of_40.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_result_of_50.hpp>", private, "<boost/phoenix/core/detail/preprocessed/actor_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/actor_result_of.hpp>", private, "<boost/phoenix/core/detail/actor_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/call_10.hpp>", private, "<boost/phoenix/core/detail/preprocessed/call.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/call_20.hpp>", private, "<boost/phoenix/core/detail/preprocessed/call.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/call_30.hpp>", private, "<boost/phoenix/core/detail/preprocessed/call.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/call_40.hpp>", private, "<boost/phoenix/core/detail/preprocessed/call.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/call_50.hpp>", private, "<boost/phoenix/core/detail/preprocessed/call.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/call.hpp>", private, "<boost/phoenix/core/detail/call.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/function_eval_10.hpp>", private, "<boost/phoenix/core/detail/preprocessed/function_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/function_eval_20.hpp>", private, "<boost/phoenix/core/detail/preprocessed/function_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/function_eval_30.hpp>", private, "<boost/phoenix/core/detail/preprocessed/function_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/function_eval_40.hpp>", private, "<boost/phoenix/core/detail/preprocessed/function_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/function_eval_50.hpp>", private, "<boost/phoenix/core/detail/preprocessed/function_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/function_eval.hpp>", private, "<boost/phoenix/core/detail/function_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/phx2_result_10.hpp>", private, "<boost/phoenix/core/detail/preprocessed/phx2_result.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/phx2_result_20.hpp>", private, "<boost/phoenix/core/detail/preprocessed/phx2_result.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/phx2_result_30.hpp>", private, "<boost/phoenix/core/detail/preprocessed/phx2_result.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/phx2_result_40.hpp>", private, "<boost/phoenix/core/detail/preprocessed/phx2_result.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/phx2_result_50.hpp>", private, "<boost/phoenix/core/detail/preprocessed/phx2_result.hpp>", private ] },
+    { include: ["<boost/phoenix/core/detail/preprocessed/phx2_result.hpp>", private, "<boost/phoenix/core/detail/phx2_result.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_operator_10.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_operator_20.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_operator_30.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_operator_40.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_operator_50.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_operator.hpp>", private, "<boost/phoenix/function/detail/function_operator.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_result_of_10.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_result_of_20.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_result_of_30.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_result_of_40.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_result_of_50.hpp>", private, "<boost/phoenix/function/detail/preprocessed/function_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/function/detail/preprocessed/function_result_of.hpp>", private, "<boost/phoenix/function/detail/function_result_of.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_10.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_20.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_30.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_40.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_50.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_eval_10.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_eval_20.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_eval_30.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_eval_40.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_eval_50.hpp>", private, "<boost/phoenix/object/detail/preprocessed/construct_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct_eval.hpp>", private, "<boost/phoenix/object/detail/construct_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/construct.hpp>", private, "<boost/phoenix/object/detail/construct.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_10.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_20.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_30.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_40.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_50.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_eval_10.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_eval_20.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_eval_30.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_eval_40.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_eval_50.hpp>", private, "<boost/phoenix/object/detail/preprocessed/new_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new_eval.hpp>", private, "<boost/phoenix/object/detail/new_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/object/detail/preprocessed/new.hpp>", private, "<boost/phoenix/object/detail/new.hpp>", private ] },
+    { include: ["<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen_10.hpp>", private, "<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen.hpp>", private ] },
+    { include: ["<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen_20.hpp>", private, "<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen.hpp>", private ] },
+    { include: ["<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen_30.hpp>", private, "<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen.hpp>", private ] },
+    { include: ["<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen_40.hpp>", private, "<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen.hpp>", private ] },
+    { include: ["<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen_50.hpp>", private, "<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen.hpp>", private ] },
+    { include: ["<boost/phoenix/operator/detail/preprocessed/mem_fun_ptr_gen.hpp>", private, "<boost/phoenix/operator/detail/mem_fun_ptr_gen.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/dynamic_10.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/dynamic.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/dynamic_20.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/dynamic.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/dynamic_30.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/dynamic.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/dynamic_40.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/dynamic.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/dynamic_50.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/dynamic.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/dynamic.hpp>", private, "<boost/phoenix/scope/detail/dynamic.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/make_locals_10.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/make_locals.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/make_locals_20.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/make_locals.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/make_locals_30.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/make_locals.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/make_locals_40.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/make_locals.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/make_locals_50.hpp>", private, "<boost/phoenix/scope/detail/preprocessed/make_locals.hpp>", private ] },
+    { include: ["<boost/phoenix/scope/detail/preprocessed/make_locals.hpp>", private, "<boost/phoenix/scope/detail/make_locals.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/catch_push_back_10.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/catch_push_back.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/catch_push_back_20.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/catch_push_back.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/catch_push_back_30.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/catch_push_back.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/catch_push_back_40.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/catch_push_back.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/catch_push_back_50.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/catch_push_back.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/catch_push_back.hpp>", private, "<boost/phoenix/statement/detail/catch_push_back.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/switch_10.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/switch.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/switch_20.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/switch.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/switch_30.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/switch.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/switch_40.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/switch.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/switch_50.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/switch.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/switch.hpp>", private, "<boost/phoenix/statement/detail/switch.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_eval_10.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_eval_20.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_eval_30.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_eval_40.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_eval_50.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_eval.hpp>", private, "<boost/phoenix/statement/detail/try_catch_eval.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_expression_10.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_expression.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_expression_20.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_expression.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_expression_30.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_expression.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_expression_40.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_expression.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_expression_50.hpp>", private, "<boost/phoenix/statement/detail/preprocessed/try_catch_expression.hpp>", private ] },
+    { include: ["<boost/phoenix/statement/detail/preprocessed/try_catch_expression.hpp>", private, "<boost/phoenix/statement/detail/try_catch_expression.hpp>", private ] },
+    { include: ["<boost/phoenix/support/detail/iterate_define.hpp>", private, "<boost/phoenix/support/detail/iterate.hpp>", private ] },
+    { include: ["<boost/phoenix/support/detail/iterate_undef.hpp>", private, "<boost/phoenix/support/detail/iterate.hpp>", private ] },
+    { include: ["<boost/preprocessor/detail/check.hpp>", private, "<boost/preprocessor/detail/is_binary.hpp>", private ] },
+    { include: ["<boost/preprocessor/detail/check.hpp>", private, "<boost/preprocessor/detail/is_nullary.hpp>", private ] },
+    { include: ["<boost/preprocessor/detail/check.hpp>", private, "<boost/preprocessor/detail/is_unary.hpp>", private ] },
+    { include: ["<boost/preprocessor/detail/dmc/auto_rec.hpp>", private, "<boost/preprocessor/detail/auto_rec.hpp>", private ] },
+    { include: ["<boost/preprocessor/detail/is_binary.hpp>", private, "<boost/tti/detail/dvm_template_params.hpp>", private ] },
+    { include: ["<boost/preprocessor/detail/is_unary.hpp>", private, "<boost/local_function/detail/preprocessor/keyword/facility/is.hpp>", private ] },
+    { include: ["<boost/preprocessor/detail/is_unary.hpp>", private, "<boost/proto/detail/remove_typename.hpp>", private ] },
+    { include: ["<boost/preprocessor/detail/is_unary.hpp>", private, "<boost/range/detail/microsoft.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/lower1.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward1.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/lower2.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward2.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/lower3.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward3.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/lower4.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward4.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/lower5.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward5.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/upper1.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward1.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/upper2.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward2.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/upper3.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward3.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/upper4.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward4.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/bounds/upper5.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward5.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/finish.hpp>", private, "<boost/preprocessor/iteration/detail/local.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/iter/reverse1.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward1.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/iter/reverse2.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward2.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/iter/reverse3.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward3.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/iter/reverse4.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward4.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/iter/reverse5.hpp>", private, "<boost/preprocessor/iteration/detail/iter/forward5.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/rlocal.hpp>", private, "<boost/preprocessor/iteration/detail/local.hpp>", private ] },
+    { include: ["<boost/preprocessor/iteration/detail/start.hpp>", private, "<boost/preprocessor/iteration/detail/local.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/lower1.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/lower2.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/lower3.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/lower4.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/lower5.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/upper1.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/upper2.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/upper3.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/upper4.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/bounds/upper5.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/finish.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/iteration/detail/start.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/slot/detail/counter.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/slot/detail/slot1.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/slot/detail/slot2.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/slot/detail/slot3.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/slot/detail/slot4.hpp>", private ] },
+    { include: ["<boost/preprocessor/slot/detail/shared.hpp>", private, "<boost/preprocessor/slot/detail/slot5.hpp>", private ] },
+    { include: ["<boost/program_options/detail/convert.hpp>", private, "<boost/program_options/detail/config_file.hpp>", private ] },
+    { include: ["<boost/program_options/detail/convert.hpp>", private, "<boost/program_options/detail/parsers.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/file_parser_error.hpp>", private, "<boost/property_tree/detail/info_parser_error.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/file_parser_error.hpp>", private, "<boost/property_tree/detail/json_parser_error.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/file_parser_error.hpp>", private, "<boost/property_tree/detail/xml_parser_error.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/info_parser_error.hpp>", private, "<boost/property_tree/detail/info_parser_read.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/info_parser_utils.hpp>", private, "<boost/property_tree/detail/info_parser_read.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/info_parser_utils.hpp>", private, "<boost/property_tree/detail/info_parser_write.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/json_parser_error.hpp>", private, "<boost/property_tree/detail/json_parser_read.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/ptree_utils.hpp>", private, "<boost/property_tree/detail/json_parser_read.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/ptree_utils.hpp>", private, "<boost/property_tree/detail/xml_parser_utils.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/ptree_utils.hpp>", private, "<boost/property_tree/detail/xml_parser_writer_settings.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/rapidxml.hpp>", private, "<boost/property_tree/detail/xml_parser_read_rapidxml.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/xml_parser_error.hpp>", private, "<boost/property_tree/detail/xml_parser_read_rapidxml.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/xml_parser_error.hpp>", private, "<boost/property_tree/detail/xml_parser_utils.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/xml_parser_flags.hpp>", private, "<boost/property_tree/detail/xml_parser_read_rapidxml.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/xml_parser_utils.hpp>", private, "<boost/property_tree/detail/xml_parser_read_rapidxml.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/xml_parser_utils.hpp>", private, "<boost/property_tree/detail/xml_parser_write.hpp>", private ] },
+    { include: ["<boost/property_tree/detail/xml_parser_writer_settings.hpp>", private, "<boost/property_tree/detail/xml_parser_utils.hpp>", private ] },
+    { include: ["<boost/proto/context/detail/preprocessed/callable_eval.hpp>", private, "<boost/proto/context/detail/callable_eval.hpp>", private ] },
+    { include: ["<boost/proto/context/detail/preprocessed/default_eval.hpp>", private, "<boost/proto/context/detail/default_eval.hpp>", private ] },
+    { include: ["<boost/proto/context/detail/preprocessed/null_eval.hpp>", private, "<boost/proto/context/detail/null_eval.hpp>", private ] },
+    { include: ["<boost/proto/detail/any.hpp>", private, "<boost/proto/detail/decltype.hpp>", private ] },
+    { include: ["<boost/proto/detail/class_member_traits.hpp>", private, "<boost/proto/detail/decltype.hpp>", private ] },
+    { include: ["<boost/proto/detail/decltype.hpp>", private, "<boost/phoenix/bind/detail/member_variable.hpp>", private ] },
+    { include: ["<boost/proto/detail/deduce_domain_n.hpp>", private, "<boost/proto/detail/deduce_domain.hpp>", private ] },
+    { include: ["<boost/proto/detail/ignore_unused.hpp>", private, "<boost/xpressive/detail/utility/ignore_unused.hpp>", private ] },
+    { include: ["<boost/proto/detail/is_noncopyable.hpp>", private, "<boost/proto/detail/poly_function.hpp>", private ] },
+    { include: ["<boost/proto/detail/memfun_funop.hpp>", private, "<boost/proto/detail/decltype.hpp>", private ] },
+    { include: ["<boost/proto/detail/poly_function_funop.hpp>", private, "<boost/proto/detail/poly_function.hpp>", private ] },
+    { include: ["<boost/proto/detail/poly_function_traits.hpp>", private, "<boost/proto/detail/poly_function.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/and_n.hpp>", private, "<boost/proto/detail/and_n.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/args.hpp>", private, "<boost/proto/detail/args.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/basic_expr.hpp>", private, "<boost/proto/detail/basic_expr.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/class_member_traits.hpp>", private, "<boost/proto/detail/class_member_traits.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/deduce_domain_n.hpp>", private, "<boost/proto/detail/deduce_domain_n.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/deep_copy.hpp>", private, "<boost/proto/detail/deep_copy.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/expr.hpp>", private, "<boost/proto/detail/expr.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/expr_variadic.hpp>", private, "<boost/proto/detail/expr.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/extends_funop_const.hpp>", private, "<boost/proto/detail/extends_funop_const.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/extends_funop.hpp>", private, "<boost/proto/detail/extends_funop.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/funop.hpp>", private, "<boost/proto/detail/funop.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/generate_by_value.hpp>", private, "<boost/proto/detail/generate_by_value.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/lambda_matches.hpp>", private, "<boost/proto/detail/lambda_matches.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/make_expr_funop.hpp>", private, "<boost/proto/detail/make_expr_funop.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/make_expr_.hpp>", private, "<boost/proto/detail/make_expr_.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/make_expr.hpp>", private, "<boost/proto/detail/make_expr.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/matches_.hpp>", private, "<boost/proto/detail/matches_.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/memfun_funop.hpp>", private, "<boost/proto/detail/memfun_funop.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/or_n.hpp>", private, "<boost/proto/detail/or_n.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/poly_function_funop.hpp>", private, "<boost/proto/detail/poly_function_funop.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/poly_function_traits.hpp>", private, "<boost/proto/detail/poly_function_traits.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/template_arity_helper.hpp>", private, "<boost/proto/detail/template_arity_helper.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/traits.hpp>", private, "<boost/proto/detail/traits.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/unpack_expr_.hpp>", private, "<boost/proto/detail/unpack_expr_.hpp>", private ] },
+    { include: ["<boost/proto/detail/preprocessed/vararg_matches_impl.hpp>", private, "<boost/proto/detail/vararg_matches_impl.hpp>", private ] },
+    { include: ["<boost/proto/detail/template_arity_helper.hpp>", private, "<boost/proto/detail/template_arity.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/expand_pack.hpp>", private, "<boost/proto/transform/detail/pack.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/pack_impl.hpp>", private, "<boost/proto/transform/detail/pack.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/call.hpp>", private, "<boost/proto/transform/detail/call.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/construct_funop.hpp>", private, "<boost/proto/transform/detail/construct_funop.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/construct_pod_funop.hpp>", private, "<boost/proto/transform/detail/construct_pod_funop.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/default_function_impl.hpp>", private, "<boost/proto/transform/detail/default_function_impl.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/expand_pack.hpp>", private, "<boost/proto/transform/detail/expand_pack.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/fold_impl.hpp>", private, "<boost/proto/transform/detail/fold_impl.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/lazy.hpp>", private, "<boost/proto/transform/detail/lazy.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/make_gcc_workaround.hpp>", private, "<boost/proto/transform/detail/make_gcc_workaround.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/make.hpp>", private, "<boost/proto/transform/detail/make.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/pack_impl.hpp>", private, "<boost/proto/transform/detail/pack_impl.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/pass_through_impl.hpp>", private, "<boost/proto/transform/detail/pass_through_impl.hpp>", private ] },
+    { include: ["<boost/proto/transform/detail/preprocessed/when.hpp>", private, "<boost/proto/transform/detail/when.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/default_deleter.hpp>", private, "<boost/ptr_container/detail/static_move_ptr.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/is_convertible.hpp>", private, "<boost/ptr_container/detail/static_move_ptr.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/move.hpp>", private, "<boost/ptr_container/detail/static_move_ptr.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/reversible_ptr_container.hpp>", private, "<boost/ptr_container/detail/associative_ptr_container.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/reversible_ptr_container.hpp>", private, "<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/scoped_deleter.hpp>", private, "<boost/ptr_container/detail/reversible_ptr_container.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/serialize_xml_names.hpp>", private, "<boost/ptr_container/detail/serialize_ptr_map_adapter.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/serialize_xml_names.hpp>", private, "<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/static_move_ptr.hpp>", private, "<boost/ptr_container/detail/reversible_ptr_container.hpp>", private ] },
+    { include: ["<boost/ptr_container/detail/throw_exception.hpp>", private, "<boost/ptr_container/detail/reversible_ptr_container.hpp>", private ] },
+    { include: ["<boost/python/detail/config.hpp>", private, "<boost/python/detail/exception_handler.hpp>", private ] },
+    { include: ["<boost/python/detail/config.hpp>", private, "<boost/python/detail/prefix.hpp>", private ] },
+    { include: ["<boost/python/detail/config.hpp>", private, "<boost/python/detail/scope.hpp>", private ] },
+    { include: ["<boost/python/detail/copy_ctor_mutates_rhs.hpp>", private, "<boost/python/detail/value_arg.hpp>", private ] },
+    { include: ["<boost/python/detail/cv_category.hpp>", private, "<boost/python/detail/unwind_type.hpp>", private ] },
+    { include: ["<boost/python/detail/defaults_gen.hpp>", private, "<boost/python/detail/defaults_def.hpp>", private ] },
+    { include: ["<boost/python/detail/def_helper_fwd.hpp>", private, "<boost/python/detail/def_helper.hpp>", private ] },
+    { include: ["<boost/python/detail/exception_handler.hpp>", private, "<boost/python/detail/translate_exception.hpp>", private ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/detail/decorated_type_id.hpp>", private ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/detail/def_helper.hpp>", private ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/detail/signature.hpp>", private ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/detail/unwind_type.hpp>", private ] },
+    { include: ["<boost/python/detail/invoke.hpp>", private, "<boost/python/detail/caller.hpp>", private ] },
+    { include: ["<boost/python/detail/is_auto_ptr.hpp>", private, "<boost/python/detail/copy_ctor_mutates_rhs.hpp>", private ] },
+    { include: ["<boost/python/detail/is_wrapper.hpp>", private, "<boost/python/detail/unwrap_wrapper.hpp>", private ] },
+    { include: ["<boost/python/detail/is_xxx.hpp>", private, "<boost/python/detail/is_auto_ptr.hpp>", private ] },
+    { include: ["<boost/python/detail/is_xxx.hpp>", private, "<boost/python/detail/is_shared_ptr.hpp>", private ] },
+    { include: ["<boost/python/detail/is_xxx.hpp>", private, "<boost/python/detail/value_is_xxx.hpp>", private ] },
+    { include: ["<boost/python/detail/make_keyword_range_fn.hpp>", private, "<boost/python/detail/defaults_def.hpp>", private ] },
+    { include: ["<boost/python/detail/none.hpp>", private, "<boost/python/detail/invoke.hpp>", private ] },
+    { include: ["<boost/python/detail/not_specified.hpp>", private, "<boost/python/detail/def_helper_fwd.hpp>", private ] },
+    { include: ["<boost/python/detail/not_specified.hpp>", private, "<boost/python/detail/def_helper.hpp>", private ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/detail/aix_init_module.hpp>", private ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/detail/invoke.hpp>", private ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/detail/is_wrapper.hpp>", private ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/detail/none.hpp>", private ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/detail/nullary_function_adaptor.hpp>", private ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/detail/sfinae.hpp>", private ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/detail/unwrap_wrapper.hpp>", private ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/detail/wrapper_base.hpp>", private ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/detail/caller.hpp>", private ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/detail/defaults_gen.hpp>", private ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/detail/invoke.hpp>", private ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/detail/result.hpp>", private ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/detail/signature.hpp>", private ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/detail/target.hpp>", private ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/detail/type_list.hpp>", private ] },
+    { include: ["<boost/python/detail/python22_fixed.h>", private, "<boost/python/detail/wrap_python.hpp>", private ] },
+    { include: ["<boost/python/detail/scope.hpp>", private, "<boost/python/detail/defaults_def.hpp>", private ] },
+    { include: ["<boost/python/detail/sfinae.hpp>", private, "<boost/python/detail/enable_if.hpp>", private ] },
+    { include: ["<boost/python/detail/signature.hpp>", private, "<boost/python/detail/caller.hpp>", private ] },
+    { include: ["<boost/python/detail/value_is_xxx.hpp>", private, "<boost/python/detail/value_is_shared_ptr.hpp>", private ] },
+    { include: ["<boost/python/detail/wrap_python.hpp>", private, "<boost/python/detail/prefix.hpp>", private ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/detail/operators.hpp>", private ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/detail/uniform_int_float.hpp>", private ] },
+    { include: ["<boost/random/detail/const_mod.hpp>", private, "<boost/random/detail/seed_impl.hpp>", private ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/detail/const_mod.hpp>", private ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/detail/large_arithmetic.hpp>", private ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/detail/seed_impl.hpp>", private ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/detail/uniform_int_float.hpp>", private ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/detail/const_mod.hpp>", private ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/detail/large_arithmetic.hpp>", private ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/detail/seed_impl.hpp>", private ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/detail/uniform_int_float.hpp>", private ] },
+    { include: ["<boost/random/detail/generator_bits.hpp>", private, "<boost/random/detail/seed_impl.hpp>", private ] },
+    { include: ["<boost/random/detail/generator_bits.hpp>", private, "<boost/random/detail/uniform_int_float.hpp>", private ] },
+    { include: ["<boost/random/detail/integer_log2.hpp>", private, "<boost/random/detail/large_arithmetic.hpp>", private ] },
+    { include: ["<boost/random/detail/integer_log2.hpp>", private, "<boost/random/detail/seed_impl.hpp>", private ] },
+    { include: ["<boost/random/detail/large_arithmetic.hpp>", private, "<boost/random/detail/const_mod.hpp>", private ] },
+    { include: ["<boost/random/detail/signed_unsigned_tools.hpp>", private, "<boost/random/detail/seed_impl.hpp>", private ] },
+    { include: ["<boost/range/detail/any_iterator_buffer.hpp>", private, "<boost/range/detail/any_iterator.hpp>", private ] },
+    { include: ["<boost/range/detail/any_iterator_buffer.hpp>", private, "<boost/range/detail/any_iterator_interface.hpp>", private ] },
+    { include: ["<boost/range/detail/any_iterator_interface.hpp>", private, "<boost/range/detail/any_iterator.hpp>", private ] },
+    { include: ["<boost/range/detail/any_iterator_interface.hpp>", private, "<boost/range/detail/any_iterator_wrapper.hpp>", private ] },
+    { include: ["<boost/range/detail/any_iterator_wrapper.hpp>", private, "<boost/range/detail/any_iterator.hpp>", private ] },
+    { include: ["<boost/range/detail/begin.hpp>", private, "<boost/range/detail/detail_str.hpp>", private ] },
+    { include: ["<boost/range/detail/collection_traits_detail.hpp>", private, "<boost/range/detail/collection_traits.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/begin.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/const_iterator.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/detail_str.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/difference_type.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/empty.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/end.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/implementation_help.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/iterator.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/size.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/size_type.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/value_type.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/vc6/end.hpp>", private ] },
+    { include: ["<boost/range/detail/common.hpp>", private, "<boost/range/detail/vc6/size.hpp>", private ] },
+    { include: ["<boost/range/detail/demote_iterator_traversal_tag.hpp>", private, "<boost/range/detail/join_iterator.hpp>", private ] },
+    { include: ["<boost/range/detail/detail_str.hpp>", private, "<boost/range/detail/as_literal.hpp>", private ] },
+    { include: ["<boost/range/detail/end.hpp>", private, "<boost/range/detail/detail_str.hpp>", private ] },
+    { include: ["<boost/range/detail/implementation_help.hpp>", private, "<boost/range/detail/end.hpp>", private ] },
+    { include: ["<boost/range/detail/implementation_help.hpp>", private, "<boost/range/detail/size.hpp>", private ] },
+    { include: ["<boost/range/detail/implementation_help.hpp>", private, "<boost/range/detail/vc6/end.hpp>", private ] },
+    { include: ["<boost/range/detail/implementation_help.hpp>", private, "<boost/range/detail/vc6/size.hpp>", private ] },
+    { include: ["<boost/range/detail/remove_extent.hpp>", private, "<boost/range/detail/const_iterator.hpp>", private ] },
+    { include: ["<boost/range/detail/remove_extent.hpp>", private, "<boost/range/detail/end.hpp>", private ] },
+    { include: ["<boost/range/detail/remove_extent.hpp>", private, "<boost/range/detail/iterator.hpp>", private ] },
+    { include: ["<boost/range/detail/remove_extent.hpp>", private, "<boost/range/detail/size.hpp>", private ] },
+    { include: ["<boost/range/detail/remove_extent.hpp>", private, "<boost/range/detail/value_type.hpp>", private ] },
+    { include: ["<boost/range/detail/remove_extent.hpp>", private, "<boost/range/detail/vc6/end.hpp>", private ] },
+    { include: ["<boost/range/detail/remove_extent.hpp>", private, "<boost/range/detail/vc6/size.hpp>", private ] },
+    { include: ["<boost/range/detail/sfinae.hpp>", private, "<boost/range/detail/common.hpp>", private ] },
+    { include: ["<boost/range/detail/size_type.hpp>", private, "<boost/range/detail/detail_str.hpp>", private ] },
+    { include: ["<boost/range/detail/size_type.hpp>", private, "<boost/range/detail/size.hpp>", private ] },
+    { include: ["<boost/range/detail/size_type.hpp>", private, "<boost/range/detail/vc6/size.hpp>", private ] },
+    { include: ["<boost/range/detail/value_type.hpp>", private, "<boost/range/detail/detail_str.hpp>", private ] },
+    { include: ["<boost/range/detail/vc6/end.hpp>", private, "<boost/range/detail/end.hpp>", private ] },
+    { include: ["<boost/range/detail/vc6/size.hpp>", private, "<boost/range/detail/size.hpp>", private ] },
+    { include: ["<boost/ratio/detail/mpl/abs.hpp>", private, "<boost/ratio/detail/mpl/gcd.hpp>", private ] },
+    { include: ["<boost/ratio/detail/mpl/abs.hpp>", private, "<boost/ratio/detail/mpl/lcm.hpp>", private ] },
+    { include: ["<boost/ratio/detail/mpl/abs.hpp>", private, "<boost/ratio/detail/overflow_helpers.hpp>", private ] },
+    { include: ["<boost/ratio/detail/mpl/sign.hpp>", private, "<boost/ratio/detail/overflow_helpers.hpp>", private ] },
+    { include: ["<boost/ratio/detail/overflow_helpers.hpp>", private, "<boost/chrono/detail/is_evenly_divisible_by.hpp>", private ] },
+    { include: ["<boost/serialization/detail/get_data.hpp>", private, "<boost/mpi/detail/mpi_datatype_primitive.hpp>", private ] },
+    { include: ["<boost/serialization/detail/get_data.hpp>", private, "<boost/mpi/detail/packed_iprimitive.hpp>", private ] },
+    { include: ["<boost/serialization/detail/get_data.hpp>", private, "<boost/mpi/detail/packed_oprimitive.hpp>", private ] },
+    { include: ["<boost/serialization/detail/shared_count_132.hpp>", private, "<boost/serialization/detail/shared_ptr_132.hpp>", private ] },
+    { include: ["<boost/serialization/detail/shared_ptr_nmt_132.hpp>", private, "<boost/serialization/detail/shared_ptr_132.hpp>", private ] },
+    { include: ["<boost/signals2/detail/auto_buffer.hpp>", private, "<boost/signals2/detail/slot_call_iterator.hpp>", private ] },
+    { include: ["<boost/signals2/detail/signals_common.hpp>", private, "<boost/signals2/detail/tracked_objects_visitor.hpp>", private ] },
+    { include: ["<boost/signals2/detail/signals_common_macros.hpp>", private, "<boost/signals2/detail/preprocessed_arg_type.hpp>", private ] },
+    { include: ["<boost/signals2/detail/unique_lock.hpp>", private, "<boost/signals2/detail/slot_call_iterator.hpp>", private ] },
+    { include: ["<boost/signals2/detail/variadic_arg_type.hpp>", private, "<boost/signals2/detail/variadic_slot_invoker.hpp>", private ] },
+    { include: ["<boost/signals/detail/config.hpp>", private, "<boost/signals/detail/named_slot_map.hpp>", private ] },
+    { include: ["<boost/signals/detail/config.hpp>", private, "<boost/signals/detail/signal_base.hpp>", private ] },
+    { include: ["<boost/signals/detail/config.hpp>", private, "<boost/signals/detail/signals_common.hpp>", private ] },
+    { include: ["<boost/signals/detail/config.hpp>", private, "<boost/signals/detail/slot_call_iterator.hpp>", private ] },
+    { include: ["<boost/signals/detail/named_slot_map.hpp>", private, "<boost/signals/detail/signal_base.hpp>", private ] },
+    { include: ["<boost/signals/detail/signals_common.hpp>", private, "<boost/signals/detail/named_slot_map.hpp>", private ] },
+    { include: ["<boost/signals/detail/signals_common.hpp>", private, "<boost/signals/detail/signal_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/array_utility.hpp>", private, "<boost/smart_ptr/detail/array_deleter.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/atomic_count_gcc.hpp>", private, "<boost/smart_ptr/detail/atomic_count.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/atomic_count_gcc_x86.hpp>", private, "<boost/smart_ptr/detail/atomic_count.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/atomic_count.hpp>", private, "<boost/smart_ptr/detail/shared_array_nmt.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/atomic_count.hpp>", private, "<boost/smart_ptr/detail/shared_ptr_nmt.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/atomic_count_pthreads.hpp>", private, "<boost/smart_ptr/detail/atomic_count.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/atomic_count_sync.hpp>", private, "<boost/smart_ptr/detail/atomic_count.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/atomic_count_win32.hpp>", private, "<boost/smart_ptr/detail/atomic_count.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/lightweight_mutex.hpp>", private, "<boost/atomic/detail/lockpool.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/lightweight_mutex.hpp>", private, "<boost/smart_ptr/detail/quick_allocator.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/lwm_nop.hpp>", private, "<boost/smart_ptr/detail/lightweight_mutex.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/lwm_pthreads.hpp>", private, "<boost/smart_ptr/detail/lightweight_mutex.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/lwm_win32_cs.hpp>", private, "<boost/smart_ptr/detail/lightweight_mutex.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/quick_allocator.hpp>", private, "<boost/smart_ptr/detail/sp_counted_impl.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_acc_ia64.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_aix.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_cw_ppc.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_gcc_ia64.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_gcc_mips.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_gcc_ppc.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_gcc_sparc.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base.hpp>", private, "<boost/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base.hpp>", private, "<boost/smart_ptr/detail/sp_counted_impl.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_nt.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_pt.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_snc_ps3.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_spin.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_sync.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_vacpp_ppc.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_base_w32.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_counted_impl.hpp>", private, "<boost/smart_ptr/detail/shared_count.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_forward.hpp>", private, "<boost/smart_ptr/detail/array_deleter.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_has_sync.hpp>", private, "<boost/smart_ptr/detail/atomic_count.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_has_sync.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/sp_has_sync.hpp>", private, "<boost/smart_ptr/detail/spinlock.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/spinlock_gcc_arm.hpp>", private, "<boost/smart_ptr/detail/spinlock.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/spinlock.hpp>", private, "<boost/smart_ptr/detail/spinlock_pool.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/spinlock_nt.hpp>", private, "<boost/smart_ptr/detail/spinlock.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/spinlock_pool.hpp>", private, "<boost/smart_ptr/detail/sp_counted_base_spin.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/spinlock_pt.hpp>", private, "<boost/smart_ptr/detail/spinlock.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/spinlock_sync.hpp>", private, "<boost/smart_ptr/detail/spinlock.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/spinlock_w32.hpp>", private, "<boost/smart_ptr/detail/spinlock.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/yield_k.hpp>", private, "<boost/smart_ptr/detail/spinlock_gcc_arm.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/yield_k.hpp>", private, "<boost/smart_ptr/detail/spinlock_sync.hpp>", private ] },
+    { include: ["<boost/smart_ptr/detail/yield_k.hpp>", private, "<boost/smart_ptr/detail/spinlock_w32.hpp>", private ] },
+    { include: ["<boost/spirit/fusion/detail/access.hpp>", private, "<boost/xpressive/detail/utility/cons.hpp>", private ] },
+    { include: ["<boost/spirit/fusion/detail/config.hpp>", private, "<boost/xpressive/detail/utility/cons.hpp>", private ] },
+    { include: ["<boost/spirit/fusion/iterator/detail/iterator_base.hpp>", private, "<boost/xpressive/detail/utility/cons.hpp>", private ] },
+    { include: ["<boost/spirit/fusion/sequence/detail/sequence_base.hpp>", private, "<boost/xpressive/detail/utility/cons.hpp>", private ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/chset/basic_chset.ipp>", private, "<boost/spirit/home/classic/utility/impl/chset/basic_chset.hpp>", private ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/chset/range_run.hpp>", private, "<boost/spirit/home/classic/utility/impl/chset/basic_chset.hpp>", private ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/chset/range_run.ipp>", private, "<boost/spirit/home/classic/utility/impl/chset/range_run.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/detail/alternative_function.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/detail/extract_from.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/detail/pass_container.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/detail/string_compare.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/detail/string_generate.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/numeric/detail/bool_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/numeric/detail/real_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/stream/detail/iterator_sink.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/detail/alternative_function.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/detail/generate.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/detail/generate_to.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/string_generate.hpp>", private, "<boost/spirit/home/karma/numeric/detail/bool_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/string_generate.hpp>", private, "<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/detail/string_generate.hpp>", private, "<boost/spirit/home/karma/numeric/detail/real_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private, "<boost/spirit/home/karma/numeric/detail/bool_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private, "<boost/spirit/home/karma/numeric/detail/real_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/karma/stream/detail/format_manip.hpp>", private, "<boost/spirit/home/karma/stream/detail/format_manip_auto.hpp>", private ] },
+    { include: ["<boost/spirit/home/phoenix/detail/local_reference.hpp>", private, "<boost/spirit/home/phoenix/detail/type_deduction.hpp>", private ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/mem_fun_ptr_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/detail/mem_fun_ptr_gen.hpp>", private ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/mem_fun_ptr_return.hpp>", private, "<boost/spirit/home/phoenix/operator/detail/mem_fun_ptr_eval.hpp>", private ] },
+    { include: ["<boost/spirit/home/phoenix/statement/detail/switch_eval.ipp>", private, "<boost/spirit/home/phoenix/statement/detail/switch_eval.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/detail/alternative_function.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/detail/string_parse.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/detail/alternative_function.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/detail/assign_to.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/detail/pass_container.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/numeric/detail/numeric_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/numeric/detail/real_impl.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/detail/construct.hpp>", private, "<boost/spirit/home/qi/detail/assign_to.hpp>", private ] },
+    { include: ["<boost/spirit/home/qi/stream/detail/match_manip.hpp>", private, "<boost/spirit/home/qi/stream/detail/match_manip_auto.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/endian/cover_operators.hpp>", private, "<boost/spirit/home/support/detail/endian/endian.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/endian/endian.hpp>", private, "<boost/spirit/home/support/detail/endian.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/hold_any.hpp>", private, "<boost/spirit/home/karma/detail/alternative_function.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/hold_any.hpp>", private, "<boost/spirit/home/karma/detail/pass_container.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/math/detail/fp_traits.hpp>", private, "<boost/spirit/home/support/detail/math/fpclassify.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/math/detail/fp_traits.hpp>", private, "<boost/spirit/home/support/detail/math/signbit.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/math/fpclassify.hpp>", private, "<boost/spirit/home/support/detail/sign.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/math/signbit.hpp>", private, "<boost/spirit/home/support/detail/sign.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/pow10.hpp>", private, "<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/pow10.hpp>", private, "<boost/spirit/home/karma/numeric/detail/real_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/pow10.hpp>", private, "<boost/spirit/home/qi/numeric/detail/real_impl.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/scoped_enum_emulation.hpp>", private, "<boost/spirit/home/support/detail/endian/endian.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/sign.hpp>", private, "<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/sign.hpp>", private, "<boost/spirit/home/karma/numeric/detail/real_utils.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/detail/sign.hpp>", private, "<boost/spirit/home/qi/numeric/detail/real_impl.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/fixed_size_queue.hpp>", private, "<boost/spirit/home/support/iterators/detail/fixed_size_queue_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/input_iterator_policy.hpp>", private, "<boost/spirit/home/support/iterators/detail/buffering_input_iterator_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/buffering_input_iterator_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/buf_id_check_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/first_owner_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/fixed_size_queue_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/functor_input_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/input_iterator_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/istream_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/lex_input_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/no_check_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/ref_counted_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/split_functor_input_policy.hpp>", private ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/detail/split_std_deque_policy.hpp>", private ] },
+    { include: ["<boost/statechart/detail/avoid_unused_warning.hpp>", private, "<boost/statechart/detail/memory.hpp>", private ] },
+    { include: ["<boost/statechart/detail/counted_base.hpp>", private, "<boost/statechart/detail/state_base.hpp>", private ] },
+    { include: ["<boost/statechart/detail/state_base.hpp>", private, "<boost/statechart/detail/leaf_state.hpp>", private ] },
+    { include: ["<boost/statechart/detail/state_base.hpp>", private, "<boost/statechart/detail/node_state.hpp>", private ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/detail/global_typedef.hpp>", private ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/detail/unit_test_parameters.hpp>", private ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/detail/workaround.hpp>", private ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/detail/unit_test_parameters.hpp>", private ] },
+    { include: ["<boost/test/detail/log_level.hpp>", private, "<boost/test/detail/unit_test_parameters.hpp>", private ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/detail/global_typedef.hpp>", private ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/detail/unit_test_parameters.hpp>", private ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/detail/workaround.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/log/detail/config.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/counter.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/force_cast.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/lockable_wrapper.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/log.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/move.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/singleton.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/thread.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/thread_interruption.hpp>", private ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/detail/tss_hooks.hpp>", private ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/detail/counter.hpp>", private ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/detail/move.hpp>", private ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/detail/thread_interruption.hpp>", private ] },
+    { include: ["<boost/thread/detail/invoke.hpp>", private, "<boost/thread/detail/async_func.hpp>", private ] },
+    { include: ["<boost/thread/detail/invoke.hpp>", private, "<boost/thread/detail/thread.hpp>", private ] },
+    { include: ["<boost/thread/detail/is_convertible.hpp>", private, "<boost/thread/detail/thread.hpp>", private ] },
+    { include: ["<boost/thread/detail/make_tuple_indices.hpp>", private, "<boost/thread/detail/async_func.hpp>", private ] },
+    { include: ["<boost/thread/detail/make_tuple_indices.hpp>", private, "<boost/thread/detail/thread.hpp>", private ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/detail/async_func.hpp>", private ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/detail/invoke.hpp>", private ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/detail/is_convertible.hpp>", private ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/detail/thread.hpp>", private ] },
+    { include: ["<boost/thread/detail/platform.hpp>", private, "<boost/thread/detail/config.hpp>", private ] },
+    { include: ["<boost/thread/detail/platform.hpp>", private, "<boost/thread/detail/thread_heap_alloc.hpp>", private ] },
+    { include: ["<boost/thread/detail/thread_heap_alloc.hpp>", private, "<boost/thread/detail/thread.hpp>", private ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/detail/config_all.hpp>", private ] },
+    { include: ["<boost/tti/detail/dcomp_mem_fun.hpp>", private, "<boost/tti/detail/dmem_fun.hpp>", private ] },
+    { include: ["<boost/tti/detail/ddeftype.hpp>", private, "<boost/tti/detail/dmem_data.hpp>", private ] },
+    { include: ["<boost/tti/detail/ddeftype.hpp>", private, "<boost/tti/detail/dmem_fun.hpp>", private ] },
+    { include: ["<boost/tti/detail/ddeftype.hpp>", private, "<boost/tti/detail/dtype.hpp>", private ] },
+    { include: ["<boost/tti/detail/dftclass.hpp>", private, "<boost/tti/detail/dcomp_mem_fun.hpp>", private ] },
+    { include: ["<boost/tti/detail/dftclass.hpp>", private, "<boost/tti/detail/dmem_data.hpp>", private ] },
+    { include: ["<boost/tti/detail/dlambda.hpp>", private, "<boost/tti/detail/dtype.hpp>", private ] },
+    { include: ["<boost/tti/detail/dmem_data.hpp>", private, "<boost/tti/detail/ddata.hpp>", private ] },
+    { include: ["<boost/tti/detail/dmem_fun.hpp>", private, "<boost/tti/detail/dfunction.hpp>", private ] },
+    { include: ["<boost/tti/detail/dmem_fun.hpp>", private, "<boost/tti/detail/dmem_data.hpp>", private ] },
+    { include: ["<boost/tti/detail/dmetafunc.hpp>", private, "<boost/tti/detail/dlambda.hpp>", private ] },
+    { include: ["<boost/tti/detail/dnullptr.hpp>", private, "<boost/tti/detail/dcomp_mem_fun.hpp>", private ] },
+    { include: ["<boost/tti/detail/dnullptr.hpp>", private, "<boost/tti/detail/dcomp_static_mem_fun.hpp>", private ] },
+    { include: ["<boost/tti/detail/dnullptr.hpp>", private, "<boost/tti/detail/dmem_fun.hpp>", private ] },
+    { include: ["<boost/tti/detail/dnullptr.hpp>", private, "<boost/tti/detail/dstatic_mem_data.hpp>", private ] },
+    { include: ["<boost/tti/detail/dnullptr.hpp>", private, "<boost/tti/detail/dstatic_mem_fun.hpp>", private ] },
+    { include: ["<boost/tti/detail/dplaceholder.hpp>", private, "<boost/tti/detail/dlambda.hpp>", private ] },
+    { include: ["<boost/tti/detail/dptmf.hpp>", private, "<boost/tti/detail/dmem_fun.hpp>", private ] },
+    { include: ["<boost/tti/detail/dstatic_mem_data.hpp>", private, "<boost/tti/detail/ddata.hpp>", private ] },
+    { include: ["<boost/tti/detail/dstatic_mem_fun.hpp>", private, "<boost/tti/detail/dfunction.hpp>", private ] },
+    { include: ["<boost/tti/detail/dtemplate.hpp>", private, "<boost/tti/detail/dvm_template_params.hpp>", private ] },
+    { include: ["<boost/tti/detail/dtemplate_params.hpp>", private, "<boost/tti/detail/dvm_template_params.hpp>", private ] },
+    { include: ["<boost/tti/detail/dtfunction.hpp>", private, "<boost/tti/detail/dstatic_mem_fun.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/any_base.hpp>", private, "<boost/type_erasure/detail/access.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/get_placeholders.hpp>", private, "<boost/type_erasure/detail/check_map.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/get_placeholders.hpp>", private, "<boost/type_erasure/detail/normalize.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/get_signature.hpp>", private, "<boost/type_erasure/detail/adapt_to_vtable.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/normalize_deduced.hpp>", private, "<boost/type_erasure/detail/normalize.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/normalize.hpp>", private, "<boost/type_erasure/detail/instantiate.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/rebind_placeholders.hpp>", private, "<boost/type_erasure/detail/instantiate.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/rebind_placeholders.hpp>", private, "<boost/type_erasure/detail/normalize.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/rebind_placeholders.hpp>", private, "<boost/type_erasure/detail/vtable.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/storage.hpp>", private, "<boost/type_erasure/detail/access.hpp>", private ] },
+    { include: ["<boost/type_erasure/detail/storage.hpp>", private, "<boost/type_erasure/detail/adapt_to_vtable.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/iostreams/detail/is_dereferenceable.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/detail/has_binary_operator.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/detail/has_postfix_operator.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/detail/has_prefix_operator.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/variant/detail/bool_trait_def.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/detail/has_binary_operator.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/detail/has_postfix_operator.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/detail/has_prefix_operator.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/variant/detail/bool_trait_undef.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/range/detail/common.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/iostreams/detail/is_dereferenceable.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/type_traits/detail/bool_trait_def.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/type_traits/detail/size_t_trait_def.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/type_traits/detail/type_trait_def.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/iostreams/detail/bool_trait_def.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/range/detail/sfinae.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/tti/detail/dcomp_mem_fun.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/tti/detail/dcomp_static_mem_fun.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/tti/detail/dmem_data.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/tti/detail/dmem_fun.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/tti/detail/dstatic_mem_data.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/tti/detail/dstatic_mem_fun.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/detail/cv_traits_impl.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/detail/is_function_ptr_tester.hpp>", private ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/detail/is_mem_fun_pointer_tester.hpp>", private ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/detail/conversion_impl.hpp>", private ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/detail/dimension_impl.hpp>", private ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/detail/linear_algebra.hpp>", private ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/detail/sort.hpp>", private ] },
+    { include: ["<boost/units/detail/heterogeneous_conversion.hpp>", private, "<boost/units/detail/conversion_impl.hpp>", private ] },
+    { include: ["<boost/units/detail/linear_algebra.hpp>", private, "<boost/units/detail/heterogeneous_conversion.hpp>", private ] },
+    { include: ["<boost/units/detail/one.hpp>", private, "<boost/units/detail/conversion_impl.hpp>", private ] },
+    { include: ["<boost/units/detail/one.hpp>", private, "<boost/units/detail/static_rational_power.hpp>", private ] },
+    { include: ["<boost/units/detail/one.hpp>", private, "<boost/units/detail/unscale.hpp>", private ] },
+    { include: ["<boost/units/detail/one.hpp>", private, "<boost/units/systems/detail/constants.hpp>", private ] },
+    { include: ["<boost/units/detail/push_front_if.hpp>", private, "<boost/units/detail/dimension_impl.hpp>", private ] },
+    { include: ["<boost/units/detail/push_front_if.hpp>", private, "<boost/units/detail/push_front_or_add.hpp>", private ] },
+    { include: ["<boost/units/detail/push_front_or_add.hpp>", private, "<boost/units/detail/dimension_impl.hpp>", private ] },
+    { include: ["<boost/units/detail/sort.hpp>", private, "<boost/units/detail/linear_algebra.hpp>", private ] },
+    { include: ["<boost/units/detail/static_rational_power.hpp>", private, "<boost/units/detail/conversion_impl.hpp>", private ] },
+    { include: ["<boost/units/detail/unscale.hpp>", private, "<boost/units/detail/conversion_impl.hpp>", private ] },
+    { include: ["<boost/unordered/detail/allocate.hpp>", private, "<boost/unordered/detail/buckets.hpp>", private ] },
+    { include: ["<boost/unordered/detail/buckets.hpp>", private, "<boost/unordered/detail/table.hpp>", private ] },
+    { include: ["<boost/unordered/detail/extract_key.hpp>", private, "<boost/unordered/detail/equivalent.hpp>", private ] },
+    { include: ["<boost/unordered/detail/extract_key.hpp>", private, "<boost/unordered/detail/unique.hpp>", private ] },
+    { include: ["<boost/unordered/detail/fwd.hpp>", private, "<boost/unordered/detail/allocate.hpp>", private ] },
+    { include: ["<boost/unordered/detail/table.hpp>", private, "<boost/unordered/detail/equivalent.hpp>", private ] },
+    { include: ["<boost/unordered/detail/table.hpp>", private, "<boost/unordered/detail/extract_key.hpp>", private ] },
+    { include: ["<boost/unordered/detail/table.hpp>", private, "<boost/unordered/detail/unique.hpp>", private ] },
+    { include: ["<boost/unordered/detail/util.hpp>", private, "<boost/unordered/detail/buckets.hpp>", private ] },
+    { include: ["<boost/unordered/detail/util.hpp>", private, "<boost/unordered/detail/table.hpp>", private ] },
+    { include: ["<boost/variant/detail/apply_visitor_binary.hpp>", private, "<boost/variant/detail/apply_visitor_delayed.hpp>", private ] },
+    { include: ["<boost/variant/detail/apply_visitor_unary.hpp>", private, "<boost/variant/detail/apply_visitor_binary.hpp>", private ] },
+    { include: ["<boost/variant/detail/apply_visitor_unary.hpp>", private, "<boost/variant/detail/apply_visitor_delayed.hpp>", private ] },
+    { include: ["<boost/variant/detail/backup_holder.hpp>", private, "<boost/variant/detail/visitation_impl.hpp>", private ] },
+    { include: ["<boost/variant/detail/cast_storage.hpp>", private, "<boost/variant/detail/visitation_impl.hpp>", private ] },
+    { include: ["<boost/variant/detail/enable_recursive_fwd.hpp>", private, "<boost/variant/detail/enable_recursive.hpp>", private ] },
+    { include: ["<boost/variant/detail/forced_return.hpp>", private, "<boost/variant/detail/visitation_impl.hpp>", private ] },
+    { include: ["<boost/variant/detail/generic_result_type.hpp>", private, "<boost/variant/detail/apply_visitor_binary.hpp>", private ] },
+    { include: ["<boost/variant/detail/generic_result_type.hpp>", private, "<boost/variant/detail/apply_visitor_delayed.hpp>", private ] },
+    { include: ["<boost/variant/detail/generic_result_type.hpp>", private, "<boost/variant/detail/apply_visitor_unary.hpp>", private ] },
+    { include: ["<boost/variant/detail/generic_result_type.hpp>", private, "<boost/variant/detail/forced_return.hpp>", private ] },
+    { include: ["<boost/variant/detail/generic_result_type.hpp>", private, "<boost/variant/detail/visitation_impl.hpp>", private ] },
+    { include: ["<boost/variant/detail/move.hpp>", private, "<boost/variant/detail/initializer.hpp>", private ] },
+    { include: ["<boost/variant/detail/substitute_fwd.hpp>", private, "<boost/variant/detail/substitute.hpp>", private ] },
+    { include: ["<boost/variant/detail/substitute.hpp>", private, "<boost/variant/detail/enable_recursive.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/access.hpp>", private, "<boost/xpressive/detail/core/results_cache.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/access.hpp>", private, "<boost/xpressive/detail/core/state.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/action.hpp>", private, "<boost/xpressive/detail/core/matcher/action_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/action.hpp>", private, "<boost/xpressive/detail/core/state.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/adaptor.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_byref_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/adaptor.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/adaptor.hpp>", private, "<boost/xpressive/detail/static/compile.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/finder.hpp>", private, "<boost/xpressive/detail/core/optimize.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/flow_control.hpp>", private, "<boost/xpressive/detail/core/matcher/end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/icase.hpp>", private, "<boost/xpressive/detail/dynamic/dynamic.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/linker.hpp>", private, "<boost/xpressive/detail/core/icase.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/linker.hpp>", private, "<boost/xpressive/detail/core/optimize.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/linker.hpp>", private, "<boost/xpressive/detail/static/compile.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/linker.hpp>", private, "<boost/xpressive/detail/static/static.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/list.hpp>", private, "<boost/xpressive/detail/core/results_cache.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/action_matcher.hpp>", private, "<boost/xpressive/detail/core/matcher/predicate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/alternate_end_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private, "<boost/xpressive/detail/static/transforms/as_alternate.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/any_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/assert_bol_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/assert_bos_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/assert_eol_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/assert_eos_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/assert_line_base.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_bol_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/assert_line_base.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_eol_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/assert_word_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/attr_end_matcher.hpp>", private, "<boost/xpressive/detail/static/transforms/as_action.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/attr_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/charset_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/end_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/end_matcher.hpp>", private, "<boost/xpressive/detail/static/compile.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/epsilon_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/keeper_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/literal_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/logical_newline_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/lookahead_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/lookbehind_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/mark_begin_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/mark_begin_matcher.hpp>", private, "<boost/xpressive/detail/static/visitor.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/mark_end_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/mark_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/optional_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/posix_charset_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/range_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/regex_byref_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/regex_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/repeat_begin_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/repeat_end_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/set_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matchers.hpp>", private, "<boost/xpressive/detail/core/linker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matchers.hpp>", private, "<boost/xpressive/detail/core/peeker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matchers.hpp>", private, "<boost/xpressive/detail/dynamic/parser.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matchers.hpp>", private, "<boost/xpressive/detail/static/transmogrify.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/simple_repeat_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/string_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/matcher/true_matcher.hpp>", private, "<boost/xpressive/detail/core/matchers.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/optimize.hpp>", private, "<boost/xpressive/detail/static/compile.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/peeker.hpp>", private, "<boost/xpressive/detail/core/linker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/peeker.hpp>", private, "<boost/xpressive/detail/core/optimize.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/peeker.hpp>", private, "<boost/xpressive/detail/static/static.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/action_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/any_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_bol_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_bos_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_eol_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_eos_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_line_base.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_word_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/charset_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/epsilon_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/keeper_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/literal_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/logical_newline_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/lookahead_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/lookbehind_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/optional_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/posix_charset_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/predicate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/range_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_byref_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/repeat_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/repeat_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/set_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/simple_repeat_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/string_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/core/matcher/true_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/dynamic/dynamic.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/dynamic/matchable.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/quant_style.hpp>", private, "<boost/xpressive/detail/static/placeholders.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/core/finder.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/core/flow_control.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_byref_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/core/optimize.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/core/state.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/static/compile.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/static/placeholders.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/detail/static/visitor.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/flow_control.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/action_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/any_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_bol_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_bos_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_eol_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_eos_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_line_base.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_word_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/charset_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/epsilon_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/keeper_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/literal_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/logical_newline_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/lookahead_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/lookbehind_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/optional_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/posix_charset_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/predicate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/range_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_byref_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/repeat_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/repeat_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/set_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/simple_repeat_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/string_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/core/matcher/true_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/detail/static/static.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/sub_match_impl.hpp>", private, "<boost/xpressive/detail/core/matcher/end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/sub_match_impl.hpp>", private, "<boost/xpressive/detail/core/sub_match_vector.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/core/sub_match_vector.hpp>", private, "<boost/xpressive/detail/core/state.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/access.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/action.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/adaptor.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/finder.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/flow_control.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/icase.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/linker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/action_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/any_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_bol_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_bos_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_eol_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_eos_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_line_base.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_word_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/charset_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/epsilon_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/keeper_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/literal_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/logical_newline_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/lookahead_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/lookbehind_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/optional_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/posix_charset_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/predicate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/range_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_byref_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/regex_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/repeat_begin_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/repeat_end_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/set_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/simple_repeat_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/string_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/matcher/true_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/peeker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/quant_style.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/regex_impl.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/results_cache.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/state.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/core/sub_match_vector.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/dynamic/dynamic.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/dynamic/matchable.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/dynamic/parse_charset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/dynamic/parser.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/dynamic/parser_traits.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/dynamic/sequence.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/is_pure.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/modifier.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/static.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_action.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_alternate.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_independent.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_inverse.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_marker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_modifier.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_quantifier.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_sequence.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transforms/as_set.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/transmogrify.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/type_traits.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/visitor.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/static/width_of.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/utility/boyer_moore.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/detail/utility/chset/chset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/traits/detail/c_ctype.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/dynamic.hpp>", private, "<boost/xpressive/detail/dynamic/parser.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/matchable.hpp>", private, "<boost/xpressive/detail/core/access.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/matchable.hpp>", private, "<boost/xpressive/detail/core/adaptor.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/matchable.hpp>", private, "<boost/xpressive/detail/core/linker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/matchable.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/matchable.hpp>", private, "<boost/xpressive/detail/core/regex_impl.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/matchable.hpp>", private, "<boost/xpressive/detail/dynamic/dynamic.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/matchable.hpp>", private, "<boost/xpressive/detail/dynamic/parser_traits.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/parser_enum.hpp>", private, "<boost/xpressive/detail/dynamic/parse_charset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/parser_enum.hpp>", private, "<boost/xpressive/detail/dynamic/parser_traits.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/sequence.hpp>", private, "<boost/xpressive/detail/dynamic/dynamic.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/dynamic/sequence.hpp>", private, "<boost/xpressive/detail/dynamic/matchable.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/grammar.hpp>", private, "<boost/xpressive/detail/static/compile.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/is_pure.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/modifier.hpp>", private, "<boost/xpressive/detail/core/icase.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/placeholders.hpp>", private, "<boost/xpressive/detail/static/static.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/placeholders.hpp>", private, "<boost/xpressive/detail/static/transmogrify.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/compile.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_action.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_alternate.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_independent.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_inverse.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_marker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_modifier.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_quantifier.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_sequence.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/static.hpp>", private, "<boost/xpressive/detail/static/transforms/as_set.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_action.hpp>", private, "<boost/xpressive/detail/core/matcher/action_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_action.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_alternate.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_independent.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_inverse.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_marker.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_matcher.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_modifier.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_quantifier.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_quantifier.hpp>", private, "<boost/xpressive/detail/static/transforms/as_action.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_sequence.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transforms/as_set.hpp>", private, "<boost/xpressive/detail/static/grammar.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/transmogrify.hpp>", private, "<boost/xpressive/detail/static/visitor.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/type_traits.hpp>", private, "<boost/xpressive/detail/core/matcher/simple_repeat_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/type_traits.hpp>", private, "<boost/xpressive/detail/static/width_of.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/visitor.hpp>", private, "<boost/xpressive/detail/static/compile.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/static/width_of.hpp>", private, "<boost/xpressive/detail/static/is_pure.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/algorithm.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/algorithm.hpp>", private, "<boost/xpressive/detail/core/matcher/lookbehind_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/algorithm.hpp>", private, "<boost/xpressive/detail/core/matcher/string_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/algorithm.hpp>", private, "<boost/xpressive/detail/core/peeker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/algorithm.hpp>", private, "<boost/xpressive/detail/dynamic/parser_traits.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/algorithm.hpp>", private, "<boost/xpressive/detail/utility/chset/chset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/algorithm.hpp>", private, "<boost/xpressive/detail/utility/traits_utils.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/any.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/boyer_moore.hpp>", private, "<boost/xpressive/detail/core/finder.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/chset/basic_chset.ipp>", private, "<boost/xpressive/detail/utility/chset/chset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/chset/basic_chset.ipp>", private, "<boost/xpressive/detail/utility/hash_peek_bitset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/chset/chset.hpp>", private, "<boost/xpressive/detail/dynamic/parse_charset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/chset/chset.hpp>", private, "<boost/xpressive/detail/static/transforms/as_set.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/chset/range_run.ipp>", private, "<boost/xpressive/detail/utility/chset/basic_chset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/cons.hpp>", private, "<boost/xpressive/detail/static/transforms/as_alternate.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/counted_base.hpp>", private, "<boost/xpressive/detail/core/regex_impl.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/counted_base.hpp>", private, "<boost/xpressive/detail/dynamic/matchable.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/dont_care.hpp>", private, "<boost/xpressive/detail/static/transmogrify.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/hash_peek_bitset.hpp>", private, "<boost/xpressive/detail/core/finder.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/hash_peek_bitset.hpp>", private, "<boost/xpressive/detail/core/matcher/alternate_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/hash_peek_bitset.hpp>", private, "<boost/xpressive/detail/core/peeker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/detail/core/flow_control.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/detail/core/icase.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/detail/core/matcher/assert_word_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/detail/core/matcher/lookahead_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/detail/core/matcher/lookbehind_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/detail/dynamic/parser.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/detail/utility/algorithm.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/literals.hpp>", private, "<boost/xpressive/detail/dynamic/parse_charset.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/literals.hpp>", private, "<boost/xpressive/detail/dynamic/parser_traits.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/never_true.hpp>", private, "<boost/xpressive/detail/core/linker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/never_true.hpp>", private, "<boost/xpressive/detail/core/peeker.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/save_restore.hpp>", private, "<boost/xpressive/detail/core/matcher/lookahead_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/save_restore.hpp>", private, "<boost/xpressive/detail/core/matcher/lookbehind_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/sequence_stack.hpp>", private, "<boost/xpressive/detail/core/state.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/symbols.hpp>", private, "<boost/xpressive/detail/core/matcher/attr_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/tracking_ptr.hpp>", private, "<boost/xpressive/detail/core/regex_impl.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/traits_utils.hpp>", private, "<boost/xpressive/detail/core/matcher/literal_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/traits_utils.hpp>", private, "<boost/xpressive/detail/core/matcher/mark_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/traits_utils.hpp>", private, "<boost/xpressive/detail/core/matcher/posix_charset_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/traits_utils.hpp>", private, "<boost/xpressive/detail/core/matcher/string_matcher.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/traits_utils.hpp>", private, "<boost/xpressive/detail/static/transforms/as_set.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/traits_utils.hpp>", private, "<boost/xpressive/detail/static/transmogrify.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/width.hpp>", private, "<boost/xpressive/detail/core/quant_style.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/width.hpp>", private, "<boost/xpressive/detail/dynamic/sequence.hpp>", private ] },
+    { include: ["<boost/xpressive/detail/utility/width.hpp>", private, "<boost/xpressive/detail/static/static.hpp>", private ] }
+]
diff --git a/cpp/build-support/iwyu/mappings/boost-all.imp b/cpp/build-support/iwyu/mappings/boost-all.imp
new file mode 100644
index 0000000000000..4b134b0fffd2f
--- /dev/null
+++ b/cpp/build-support/iwyu/mappings/boost-all.imp
@@ -0,0 +1,5679 @@
+# This file has been imported into the apache source tree from
+# the IWYU source tree as of version 0.8
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/boost-all.imp
+# and corresponding license has been added:
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/LICENSE.TXT
+#
+# ==============================================================================
+# LLVM Release License
+# ==============================================================================
+# University of Illinois/NCSA
+# Open Source License
+# 
+# Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign.
+# All rights reserved.
+# 
+# Developed by:
+# 
+#     LLVM Team
+# 
+#     University of Illinois at Urbana-Champaign
+# 
+#     http://llvm.org
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal with
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+# 
+#     * Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimers.
+# 
+#     * Redistributions in binary form must reproduce the above copyright notice,
+#       this list of conditions and the following disclaimers in the
+#       documentation and/or other materials provided with the distribution.
+# 
+#     * Neither the names of the LLVM Team, University of Illinois at
+#       Urbana-Champaign, nor the names of its contributors may be used to
+#       endorse or promote products derived from this Software without specific
+#       prior written permission.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+# SOFTWARE.
+
+[
+#  cd /usr/include && grep -r --exclude-dir={detail,impl} '^ *# *include' boost/ | perl -nle 'm/^([^:]+).*["<]([^>]+)[">]/ && print qq@    { include: ["<$2>", private, "<$1>", public ] },@' | grep -e \/detail\/ -e \/impl\/ | grep -e \\[\"\<boost/ | sort -u
+#manually include:
+{ include: ["@<boost/bind/.*>", private, "<boost/bind.hpp>", public ] },
+{ include: ["@<boost/format/.*>", private, "<boost/format.hpp>", public ] },
+{ include: ["@<boost/filesystem/.*>", private, "<boost/filesystem.hpp>", public ] },
+{ include: ["@<boost/function/.*>", private, "<boost/function.hpp>", public ] },
+#manually delete $ sed '/workarounds*\.hpp/d' -i boost-all.imp
+#also good idea to remove all lines refering to folders above (e.g., sed '/\/format\//d' -i boost-all.imp)
+#programatically include:
+    { include: ["<boost/accumulators/numeric/detail/function1.hpp>", private, "<boost/accumulators/numeric/functional.hpp>", public ] },
+    { include: ["<boost/accumulators/numeric/detail/function2.hpp>", private, "<boost/accumulators/numeric/functional.hpp>", public ] },
+    { include: ["<boost/accumulators/numeric/detail/pod_singleton.hpp>", private, "<boost/accumulators/numeric/functional.hpp>", public ] },
+    { include: ["<boost/algorithm/searching/detail/bm_traits.hpp>", private, "<boost/algorithm/searching/boyer_moore_horspool.hpp>", public ] },
+    { include: ["<boost/algorithm/searching/detail/bm_traits.hpp>", private, "<boost/algorithm/searching/boyer_moore.hpp>", public ] },
+    { include: ["<boost/algorithm/searching/detail/debugging.hpp>", private, "<boost/algorithm/searching/boyer_moore_horspool.hpp>", public ] },
+    { include: ["<boost/algorithm/searching/detail/debugging.hpp>", private, "<boost/algorithm/searching/boyer_moore.hpp>", public ] },
+    { include: ["<boost/algorithm/searching/detail/debugging.hpp>", private, "<boost/algorithm/searching/knuth_morris_pratt.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/case_conv.hpp>", private, "<boost/algorithm/string/case_conv.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/classification.hpp>", private, "<boost/algorithm/string/classification.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/finder.hpp>", private, "<boost/algorithm/string/finder.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/finder_regex.hpp>", private, "<boost/algorithm/string/regex_find_format.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/find_format_all.hpp>", private, "<boost/algorithm/string/find_format.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/find_format.hpp>", private, "<boost/algorithm/string/find_format.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/find_iterator.hpp>", private, "<boost/algorithm/string/find_iterator.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/formatter.hpp>", private, "<boost/algorithm/string/formatter.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/formatter_regex.hpp>", private, "<boost/algorithm/string/regex_find_format.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/predicate.hpp>", private, "<boost/algorithm/string/predicate.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/sequence.hpp>", private, "<boost/algorithm/string/join.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/trim.hpp>", private, "<boost/algorithm/string/trim.hpp>", public ] },
+    { include: ["<boost/algorithm/string/detail/util.hpp>", private, "<boost/algorithm/string/iter_find.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/archive_exception.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_archive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_binary_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_binary_iprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_binary_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_binary_oprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_text_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_text_iprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_text_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_text_oprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_xml_archive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_xml_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/basic_xml_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/codecvt_null.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/polymorphic_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/polymorphic_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/shared_ptr_helper.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/text_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/text_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/text_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/text_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/xml_archive_exception.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/xml_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/xml_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/xml_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_prefix.hpp>", private, "<boost/archive/xml_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/archive_exception.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_archive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_binary_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_binary_iprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_binary_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_binary_oprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_text_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_text_iprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_text_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_text_oprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_xml_archive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_xml_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/basic_xml_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/codecvt_null.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/polymorphic_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/polymorphic_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/shared_ptr_helper.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/text_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/text_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/text_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/text_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/xml_archive_exception.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/xml_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/xml_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/xml_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/abi_suffix.hpp>", private, "<boost/archive/xml_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/basic_archive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/basic_binary_iprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/basic_binary_oprimitive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/basic_xml_archive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/codecvt_null.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/text_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/text_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/xml_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/archive/xml_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/packed_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/packed_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_archive.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_warchive.hpp>", private, "<boost/archive/text_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_warchive.hpp>", private, "<boost/archive/text_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_warchive.hpp>", private, "<boost/archive/xml_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/auto_link_warchive.hpp>", private, "<boost/archive/xml_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/basic_iarchive.hpp>", private, "<boost/serialization/collections_load_imp.hpp>", public ] },
+    { include: ["<boost/archive/detail/basic_iarchive.hpp>", private, "<boost/serialization/hash_collections_load_imp.hpp>", public ] },
+    { include: ["<boost/archive/detail/basic_iarchive.hpp>", private, "<boost/serialization/optional.hpp>", public ] },
+    { include: ["<boost/archive/detail/common_iarchive.hpp>", private, "<boost/archive/basic_binary_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/common_iarchive.hpp>", private, "<boost/archive/basic_text_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/common_iarchive.hpp>", private, "<boost/archive/basic_xml_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/common_iarchive.hpp>", private, "<boost/mpi/packed_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/common_oarchive.hpp>", private, "<boost/archive/basic_binary_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/common_oarchive.hpp>", private, "<boost/archive/basic_text_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/common_oarchive.hpp>", private, "<boost/archive/basic_xml_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/common_oarchive.hpp>", private, "<boost/mpi/packed_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/archive_exception.hpp>", public ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/polymorphic_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/polymorphic_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/shared_ptr_helper.hpp>", public ] },
+    { include: ["<boost/archive/detail/decl.hpp>", private, "<boost/archive/xml_archive_exception.hpp>", public ] },
+    { include: ["<boost/archive/detail/interface_iarchive.hpp>", private, "<boost/archive/polymorphic_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/interface_oarchive.hpp>", private, "<boost/archive/polymorphic_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/iserializer.hpp>", private, "<boost/archive/polymorphic_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/oserializer.hpp>", private, "<boost/archive/polymorphic_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_iarchive_route.hpp>", private, "<boost/archive/polymorphic_binary_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_iarchive_route.hpp>", private, "<boost/archive/polymorphic_text_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_iarchive_route.hpp>", private, "<boost/archive/polymorphic_text_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_iarchive_route.hpp>", private, "<boost/archive/polymorphic_xml_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_iarchive_route.hpp>", private, "<boost/archive/polymorphic_xml_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_oarchive_route.hpp>", private, "<boost/archive/polymorphic_binary_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_oarchive_route.hpp>", private, "<boost/archive/polymorphic_text_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_oarchive_route.hpp>", private, "<boost/archive/polymorphic_text_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_oarchive_route.hpp>", private, "<boost/archive/polymorphic_xml_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/polymorphic_oarchive_route.hpp>", private, "<boost/archive/polymorphic_xml_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/binary_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/binary_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/binary_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/binary_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/polymorphic_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/polymorphic_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/text_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/text_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/text_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/text_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/xml_iarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/xml_oarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/xml_wiarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/archive/xml_woarchive.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/archive/detail/register_archive.hpp>", private, "<boost/serialization/export.hpp>", public ] },
+    { include: ["<boost/asio/detail/array_fwd.hpp>", private, "<boost/asio/buffer.hpp>", public ] },
+    { include: ["<boost/asio/detail/array.hpp>", private, "<boost/asio/basic_socket_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/array.hpp>", private, "<boost/asio/ip/address_v4.hpp>", public ] },
+    { include: ["<boost/asio/detail/array.hpp>", private, "<boost/asio/ip/address_v6.hpp>", public ] },
+    { include: ["<boost/asio/detail/assert.hpp>", private, "<boost/asio/buffers_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/bind_handler.hpp>", private, "<boost/asio/buffered_write_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/buffered_stream_storage.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/buffered_stream_storage.hpp>", private, "<boost/asio/buffered_write_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/buffer_resize_guard.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/buffer_sequence_adapter.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/chrono_time_traits.hpp>", private, "<boost/asio/waitable_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/async_result.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_datagram_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_io_object.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_raw_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_seq_packet_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_serial_port.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_signal_set.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_socket_acceptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_socket_iostream.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_socket_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_streambuf_fwd.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_stream_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/basic_waitable_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/buffered_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/buffered_write_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/buffer.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/buffers_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/completion_condition.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/connect.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/datagram_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/deadline_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/error.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/generic/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/generic/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/generic/raw_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/generic/seq_packet_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/generic/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/handler_alloc_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/handler_continuation_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/handler_invoke_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/handler_type.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/high_resolution_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/address.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/address_v4.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/address_v6.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/basic_resolver_entry.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/basic_resolver.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/basic_resolver_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/basic_resolver_query.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/host_name.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/icmp.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/multicast.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/resolver_query_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/resolver_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/tcp.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/udp.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/unicast.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ip/v6_only.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/is_read_buffered.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/is_write_buffered.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/local/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/local/connect_pair.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/local/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/local/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/placeholders.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/posix/basic_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/posix/basic_stream_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/posix/descriptor_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/posix/stream_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/posix/stream_descriptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/raw_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/read_at.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/read.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/read_until.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/seq_packet_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/serial_port_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/serial_port.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/serial_port_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/signal_set.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/signal_set_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/socket_acceptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/socket_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/spawn.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/basic_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/context_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/context_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/error.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/old/basic_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/old/context_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/old/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/old/stream_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/rfc2818_verification.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/stream_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/stream_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/verify_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/ssl/verify_mode.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/steady_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/strand.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/stream_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/system_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/use_future.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/waitable_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/basic_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/basic_object_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/basic_random_access_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/basic_stream_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/object_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/object_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/overlapped_ptr.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/random_access_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/random_access_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/stream_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/windows/stream_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/write_at.hpp>", public ] },
+    { include: ["<boost/asio/detail/config.hpp>", private, "<boost/asio/write.hpp>", public ] },
+    { include: ["<boost/asio/detail/cstdint.hpp>", private, "<boost/asio/read_at.hpp>", public ] },
+    { include: ["<boost/asio/detail/cstdint.hpp>", private, "<boost/asio/windows/random_access_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/cstdint.hpp>", private, "<boost/asio/write_at.hpp>", public ] },
+    { include: ["<boost/asio/detail/deadline_timer_service.hpp>", private, "<boost/asio/deadline_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/deadline_timer_service.hpp>", private, "<boost/asio/waitable_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/function.hpp>", private, "<boost/asio/buffer.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_datagram_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_raw_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_seq_packet_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_serial_port.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_signal_set.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_socket_acceptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_stream_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/basic_waitable_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/ip/basic_resolver.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/posix/basic_stream_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/strand.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/windows/basic_random_access_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/handler_type_requirements.hpp>", private, "<boost/asio/windows/basic_stream_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/io_control.hpp>", private, "<boost/asio/posix/descriptor_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/io_control.hpp>", private, "<boost/asio/socket_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/limits.hpp>", private, "<boost/asio/basic_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/basic_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/buffered_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/buffered_write_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/ssl/verify_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/noncopyable.hpp>", private, "<boost/asio/windows/overlapped_ptr.hpp>", public ] },
+    { include: ["<boost/asio/detail/null_socket_service.hpp>", private, "<boost/asio/datagram_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/null_socket_service.hpp>", private, "<boost/asio/raw_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/null_socket_service.hpp>", private, "<boost/asio/seq_packet_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/null_socket_service.hpp>", private, "<boost/asio/socket_acceptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/async_result.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_datagram_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_io_object.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_raw_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_seq_packet_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_serial_port.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_signal_set.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_socket_acceptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_socket_iostream.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_socket_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_stream_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/basic_waitable_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/buffered_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/buffered_write_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/buffer.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/buffers_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/completion_condition.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/connect.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/datagram_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/deadline_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/error.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/generic/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/generic/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/generic/raw_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/generic/seq_packet_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/generic/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/handler_alloc_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/handler_continuation_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/handler_invoke_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/handler_type.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/address.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/address_v4.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/address_v6.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/basic_resolver_entry.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/basic_resolver.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/basic_resolver_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/basic_resolver_query.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/host_name.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/icmp.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/multicast.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/resolver_query_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/resolver_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/tcp.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/udp.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/unicast.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ip/v6_only.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/is_read_buffered.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/is_write_buffered.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/local/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/local/connect_pair.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/local/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/local/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/placeholders.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/posix/basic_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/posix/basic_stream_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/posix/descriptor_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/posix/stream_descriptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/raw_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/read_at.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/read.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/read_until.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/seq_packet_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/serial_port_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/serial_port_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/signal_set_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/socket_acceptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/socket_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/spawn.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/basic_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/context_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/context_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/error.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/old/basic_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/old/context_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/old/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/old/stream_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/rfc2818_verification.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/stream_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/stream_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/verify_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/ssl/verify_mode.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/strand.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/stream_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/time_traits.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/use_future.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/waitable_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/wait_traits.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/windows/basic_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/windows/basic_object_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/windows/basic_random_access_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/windows/basic_stream_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/windows/object_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/windows/overlapped_ptr.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/windows/random_access_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/windows/stream_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/write_at.hpp>", public ] },
+    { include: ["<boost/asio/detail/pop_options.hpp>", private, "<boost/asio/write.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/async_result.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_datagram_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_io_object.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_raw_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_seq_packet_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_serial_port.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_signal_set.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_socket_acceptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_socket_iostream.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_socket_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_stream_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/basic_waitable_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/buffered_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/buffered_write_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/buffer.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/buffers_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/completion_condition.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/connect.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/datagram_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/deadline_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/error.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/generic/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/generic/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/generic/raw_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/generic/seq_packet_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/generic/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/handler_alloc_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/handler_continuation_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/handler_invoke_hook.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/handler_type.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/address.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/address_v4.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/address_v6.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/basic_resolver_entry.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/basic_resolver.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/basic_resolver_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/basic_resolver_query.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/host_name.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/icmp.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/multicast.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/resolver_query_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/resolver_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/tcp.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/udp.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/unicast.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ip/v6_only.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/is_read_buffered.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/is_write_buffered.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/local/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/local/connect_pair.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/local/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/local/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/placeholders.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/posix/basic_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/posix/basic_stream_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/posix/descriptor_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/posix/stream_descriptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/raw_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/read_at.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/read.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/read_until.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/seq_packet_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/serial_port_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/serial_port_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/signal_set_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/socket_acceptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/socket_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/spawn.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/basic_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/context_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/context_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/error.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/old/basic_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/old/context_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/old/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/old/stream_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/rfc2818_verification.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/stream_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/stream_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/verify_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/ssl/verify_mode.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/strand.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/stream_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/time_traits.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/use_future.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/waitable_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/wait_traits.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/windows/basic_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/windows/basic_object_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/windows/basic_random_access_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/windows/basic_stream_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/windows/object_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/windows/overlapped_ptr.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/windows/random_access_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/windows/stream_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/write_at.hpp>", public ] },
+    { include: ["<boost/asio/detail/push_options.hpp>", private, "<boost/asio/write.hpp>", public ] },
+    { include: ["<boost/asio/detail/reactive_descriptor_service.hpp>", private, "<boost/asio/posix/stream_descriptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/reactive_serial_port_service.hpp>", private, "<boost/asio/serial_port_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/reactive_socket_service.hpp>", private, "<boost/asio/datagram_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/reactive_socket_service.hpp>", private, "<boost/asio/raw_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/reactive_socket_service.hpp>", private, "<boost/asio/seq_packet_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/reactive_socket_service.hpp>", private, "<boost/asio/socket_acceptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/reactive_socket_service.hpp>", private, "<boost/asio/stream_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/regex_fwd.hpp>", private, "<boost/asio/read_until.hpp>", public ] },
+    { include: ["<boost/asio/detail/resolver_service.hpp>", private, "<boost/asio/ip/resolver_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/shared_ptr.hpp>", private, "<boost/asio/ip/basic_resolver_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/signal_init.hpp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/signal_set_service.hpp>", private, "<boost/asio/signal_set_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/ip/basic_resolver_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/ip/basic_resolver_query.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_ops.hpp>", private, "<boost/asio/local/connect_pair.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_option.hpp>", private, "<boost/asio/ip/tcp.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_option.hpp>", private, "<boost/asio/ip/v6_only.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_option.hpp>", private, "<boost/asio/posix/descriptor_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_option.hpp>", private, "<boost/asio/socket_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/generic/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/generic/raw_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/generic/seq_packet_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/generic/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/address_v4.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/address_v6.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/basic_resolver_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/icmp.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/resolver_query_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/tcp.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/ip/udp.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/local/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/local/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/serial_port_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/socket_base.hpp>", public ] },
+    { include: ["<boost/asio/detail/socket_types.hpp>", private, "<boost/asio/time_traits.hpp>", public ] },
+    { include: ["<boost/asio/detail/strand_service.hpp>", private, "<boost/asio/strand.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_datagram_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_deadline_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_raw_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_seq_packet_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_serial_port.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_signal_set.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_socket_acceptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_socket_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_stream_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/basic_waitable_timer.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ip/basic_resolver.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/local/connect_pair.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/posix/basic_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/posix/basic_stream_descriptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ssl/old/basic_context.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/ssl/old/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/windows/basic_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/windows/basic_object_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/windows/basic_random_access_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_error.hpp>", private, "<boost/asio/windows/basic_stream_handle.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_exception.hpp>", private, "<boost/asio/basic_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_exception.hpp>", private, "<boost/asio/generic/datagram_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_exception.hpp>", private, "<boost/asio/generic/raw_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_exception.hpp>", private, "<boost/asio/generic/seq_packet_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/throw_exception.hpp>", private, "<boost/asio/generic/stream_protocol.hpp>", public ] },
+    { include: ["<boost/asio/detail/timer_queue_ptime.hpp>", private, "<boost/asio/deadline_timer_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/basic_datagram_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/basic_raw_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/basic_socket_acceptor.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/basic_socket.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/buffered_write_stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/buffer.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/buffers_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/datagram_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/raw_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/read_until.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/seq_packet_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/socket_acceptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/ssl/old/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/detail/type_traits.hpp>", private, "<boost/asio/stream_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/variadic_templates.hpp>", private, "<boost/asio/basic_socket_iostream.hpp>", public ] },
+    { include: ["<boost/asio/detail/variadic_templates.hpp>", private, "<boost/asio/basic_socket_streambuf.hpp>", public ] },
+    { include: ["<boost/asio/detail/weak_ptr.hpp>", private, "<boost/asio/spawn.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_handle_service.hpp>", private, "<boost/asio/windows/random_access_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_handle_service.hpp>", private, "<boost/asio/windows/stream_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_overlapped_ptr.hpp>", private, "<boost/asio/windows/overlapped_ptr.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_serial_port_service.hpp>", private, "<boost/asio/serial_port_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_service.hpp>", private, "<boost/asio/datagram_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_service.hpp>", private, "<boost/asio/raw_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_service.hpp>", private, "<boost/asio/seq_packet_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_service.hpp>", private, "<boost/asio/socket_acceptor_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_iocp_socket_service.hpp>", private, "<boost/asio/stream_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/win_object_handle_service.hpp>", private, "<boost/asio/windows/object_handle_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/winrt_resolver_service.hpp>", private, "<boost/asio/ip/resolver_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/winrt_ssocket_service.hpp>", private, "<boost/asio/stream_socket_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/winrt_utils.hpp>", private, "<boost/asio/ip/basic_resolver_iterator.hpp>", public ] },
+    { include: ["<boost/asio/detail/winsock_init.hpp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/winsock_init.hpp>", private, "<boost/asio/ip/address_v4.hpp>", public ] },
+    { include: ["<boost/asio/detail/winsock_init.hpp>", private, "<boost/asio/ip/address_v6.hpp>", public ] },
+    { include: ["<boost/asio/detail/wrapped_handler.hpp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/detail/wrapped_handler.hpp>", private, "<boost/asio/spawn.hpp>", public ] },
+    { include: ["<boost/asio/detail/wrapped_handler.hpp>", private, "<boost/asio/strand.hpp>", public ] },
+    { include: ["<boost/asio/generic/detail/endpoint.hpp>", private, "<boost/asio/generic/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/impl/buffered_read_stream.hpp>", private, "<boost/asio/buffered_read_stream.hpp>", public ] },
+    { include: ["<boost/asio/impl/buffered_write_stream.hpp>", private, "<boost/asio/buffered_write_stream.hpp>", public ] },
+    { include: ["<boost/asio/impl/connect.hpp>", private, "<boost/asio/connect.hpp>", public ] },
+    { include: ["<boost/asio/impl/error.ipp>", private, "<boost/asio/error.hpp>", public ] },
+    { include: ["<boost/asio/impl/handler_alloc_hook.ipp>", private, "<boost/asio/handler_alloc_hook.hpp>", public ] },
+    { include: ["<boost/asio/impl/io_service.hpp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/impl/io_service.ipp>", private, "<boost/asio/io_service.hpp>", public ] },
+    { include: ["<boost/asio/impl/read_at.hpp>", private, "<boost/asio/read_at.hpp>", public ] },
+    { include: ["<boost/asio/impl/read.hpp>", private, "<boost/asio/read.hpp>", public ] },
+    { include: ["<boost/asio/impl/read_until.hpp>", private, "<boost/asio/read_until.hpp>", public ] },
+    { include: ["<boost/asio/impl/serial_port_base.hpp>", private, "<boost/asio/serial_port_base.hpp>", public ] },
+    { include: ["<boost/asio/impl/serial_port_base.ipp>", private, "<boost/asio/serial_port_base.hpp>", public ] },
+    { include: ["<boost/asio/impl/spawn.hpp>", private, "<boost/asio/spawn.hpp>", public ] },
+    { include: ["<boost/asio/impl/use_future.hpp>", private, "<boost/asio/use_future.hpp>", public ] },
+    { include: ["<boost/asio/impl/write_at.hpp>", private, "<boost/asio/write_at.hpp>", public ] },
+    { include: ["<boost/asio/impl/write.hpp>", private, "<boost/asio/write.hpp>", public ] },
+    { include: ["<boost/asio/ip/detail/endpoint.hpp>", private, "<boost/asio/ip/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/ip/detail/socket_option.hpp>", private, "<boost/asio/ip/multicast.hpp>", public ] },
+    { include: ["<boost/asio/ip/detail/socket_option.hpp>", private, "<boost/asio/ip/unicast.hpp>", public ] },
+    { include: ["<boost/asio/ip/impl/address.hpp>", private, "<boost/asio/ip/address.hpp>", public ] },
+    { include: ["<boost/asio/ip/impl/address.ipp>", private, "<boost/asio/ip/address.hpp>", public ] },
+    { include: ["<boost/asio/ip/impl/address_v4.hpp>", private, "<boost/asio/ip/address_v4.hpp>", public ] },
+    { include: ["<boost/asio/ip/impl/address_v4.ipp>", private, "<boost/asio/ip/address_v4.hpp>", public ] },
+    { include: ["<boost/asio/ip/impl/address_v6.hpp>", private, "<boost/asio/ip/address_v6.hpp>", public ] },
+    { include: ["<boost/asio/ip/impl/address_v6.ipp>", private, "<boost/asio/ip/address_v6.hpp>", public ] },
+    { include: ["<boost/asio/ip/impl/basic_endpoint.hpp>", private, "<boost/asio/ip/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/ip/impl/host_name.ipp>", private, "<boost/asio/ip/host_name.hpp>", public ] },
+    { include: ["<boost/asio/local/detail/endpoint.hpp>", private, "<boost/asio/local/basic_endpoint.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/buffered_handshake_op.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/handshake_op.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/io.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/openssl_init.hpp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/context_base.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/rfc2818_verification.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/verify_context.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/openssl_types.hpp>", private, "<boost/asio/ssl/verify_mode.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/password_callback.hpp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/read_op.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/shutdown_op.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/stream_core.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/verify_callback.hpp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/ssl/detail/write_op.hpp>", private, "<boost/asio/ssl/stream.hpp>", public ] },
+    { include: ["<boost/asio/ssl/impl/context.hpp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/ssl/impl/context.ipp>", private, "<boost/asio/ssl/context.hpp>", public ] },
+    { include: ["<boost/asio/ssl/impl/error.ipp>", private, "<boost/asio/ssl/error.hpp>", public ] },
+    { include: ["<boost/asio/ssl/impl/rfc2818_verification.ipp>", private, "<boost/asio/ssl/rfc2818_verification.hpp>", public ] },
+    { include: ["<boost/asio/ssl/old/detail/openssl_context_service.hpp>", private, "<boost/asio/ssl/old/context_service.hpp>", public ] },
+    { include: ["<boost/asio/ssl/old/detail/openssl_stream_service.hpp>", private, "<boost/asio/ssl/old/stream_service.hpp>", public ] },
+    { include: ["<boost/atomic/detail/config.hpp>", private, "<boost/atomic/atomic.hpp>", public ] },
+    { include: ["<boost/atomic/detail/platform.hpp>", private, "<boost/atomic/atomic.hpp>", public ] },
+    { include: ["<boost/atomic/detail/type-classification.hpp>", private, "<boost/atomic/atomic.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/comparison_adaptor.hpp>", private, "<boost/bimap/container_adaptor/list_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/comparison_adaptor.hpp>", private, "<boost/bimap/container_adaptor/list_map_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/comparison_adaptor.hpp>", private, "<boost/bimap/container_adaptor/ordered_associative_container_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/comparison_adaptor.hpp>", private, "<boost/bimap/views/multiset_view.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/comparison_adaptor.hpp>", private, "<boost/bimap/views/vector_map_view.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/comparison_adaptor.hpp>", private, "<boost/bimap/views/vector_set_view.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/functor_bag.hpp>", private, "<boost/bimap/container_adaptor/container_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/identity_converters.hpp>", private, "<boost/bimap/container_adaptor/associative_container_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/identity_converters.hpp>", private, "<boost/bimap/container_adaptor/container_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/identity_converters.hpp>", private, "<boost/bimap/container_adaptor/list_map_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/identity_converters.hpp>", private, "<boost/bimap/container_adaptor/sequence_container_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/identity_converters.hpp>", private, "<boost/bimap/container_adaptor/vector_map_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/key_extractor.hpp>", private, "<boost/bimap/container_adaptor/list_map_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/non_unique_container_helper.hpp>", private, "<boost/bimap/container_adaptor/multimap_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/non_unique_container_helper.hpp>", private, "<boost/bimap/container_adaptor/multiset_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/non_unique_container_helper.hpp>", private, "<boost/bimap/container_adaptor/unordered_multimap_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/container_adaptor/detail/non_unique_container_helper.hpp>", private, "<boost/bimap/container_adaptor/unordered_multiset_adaptor.hpp>", public ] },
+    { include: ["<boost/bimap/detail/bimap_core.hpp>", private, "<boost/bimap/bimap.hpp>", public ] },
+    { include: ["<boost/bimap/detail/concept_tags.hpp>", private, "<boost/bimap/list_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/concept_tags.hpp>", private, "<boost/bimap/multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/concept_tags.hpp>", private, "<boost/bimap/set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/concept_tags.hpp>", private, "<boost/bimap/unconstrained_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/concept_tags.hpp>", private, "<boost/bimap/unordered_multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/concept_tags.hpp>", private, "<boost/bimap/unordered_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/concept_tags.hpp>", private, "<boost/bimap/vector_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/debug/static_error.hpp>", private, "<boost/bimap/relation/structured_pair.hpp>", public ] },
+    { include: ["<boost/bimap/detail/debug/static_error.hpp>", private, "<boost/bimap/relation/support/member_with_tag.hpp>", public ] },
+    { include: ["<boost/bimap/detail/debug/static_error.hpp>", private, "<boost/bimap/tags/support/tag_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_index_binder.hpp>", private, "<boost/bimap/list_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_index_binder.hpp>", private, "<boost/bimap/multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_index_binder.hpp>", private, "<boost/bimap/set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_index_binder.hpp>", private, "<boost/bimap/unconstrained_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_index_binder.hpp>", private, "<boost/bimap/unordered_multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_index_binder.hpp>", private, "<boost/bimap/unordered_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_index_binder.hpp>", private, "<boost/bimap/vector_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_relation_binder.hpp>", private, "<boost/bimap/list_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_relation_binder.hpp>", private, "<boost/bimap/multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_relation_binder.hpp>", private, "<boost/bimap/set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_relation_binder.hpp>", private, "<boost/bimap/unconstrained_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_relation_binder.hpp>", private, "<boost/bimap/unordered_multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_relation_binder.hpp>", private, "<boost/bimap/unordered_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_relation_binder.hpp>", private, "<boost/bimap/vector_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_view_binder.hpp>", private, "<boost/bimap/list_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_view_binder.hpp>", private, "<boost/bimap/multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_view_binder.hpp>", private, "<boost/bimap/set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_view_binder.hpp>", private, "<boost/bimap/unconstrained_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_view_binder.hpp>", private, "<boost/bimap/unordered_multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_view_binder.hpp>", private, "<boost/bimap/unordered_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/generate_view_binder.hpp>", private, "<boost/bimap/vector_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/bimap.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/views/list_map_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/views/list_set_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/views/map_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/views/multimap_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/views/unordered_map_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/views/unordered_multimap_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/views/vector_map_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/map_view_base.hpp>", private, "<boost/bimap/views/vector_set_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/modifier_adaptor.hpp>", private, "<boost/bimap/bimap.hpp>", public ] },
+    { include: ["<boost/bimap/detail/non_unique_views_helper.hpp>", private, "<boost/bimap/views/multimap_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/non_unique_views_helper.hpp>", private, "<boost/bimap/views/multiset_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/non_unique_views_helper.hpp>", private, "<boost/bimap/views/unordered_multimap_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/non_unique_views_helper.hpp>", private, "<boost/bimap/views/unordered_multiset_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/set_view_base.hpp>", private, "<boost/bimap/views/list_set_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/set_view_base.hpp>", private, "<boost/bimap/views/multiset_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/set_view_base.hpp>", private, "<boost/bimap/views/set_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/set_view_base.hpp>", private, "<boost/bimap/views/unordered_multiset_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/set_view_base.hpp>", private, "<boost/bimap/views/unordered_set_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/set_view_base.hpp>", private, "<boost/bimap/views/vector_set_view.hpp>", public ] },
+    { include: ["<boost/bimap/detail/user_interface_config.hpp>", private, "<boost/bimap/bimap.hpp>", public ] },
+    { include: ["<boost/bimap/detail/user_interface_config.hpp>", private, "<boost/bimap/list_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/user_interface_config.hpp>", private, "<boost/bimap/multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/user_interface_config.hpp>", private, "<boost/bimap/set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/user_interface_config.hpp>", private, "<boost/bimap/unconstrained_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/user_interface_config.hpp>", private, "<boost/bimap/unordered_multiset_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/user_interface_config.hpp>", private, "<boost/bimap/unordered_set_of.hpp>", public ] },
+    { include: ["<boost/bimap/detail/user_interface_config.hpp>", private, "<boost/bimap/vector_of.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/access_builder.hpp>", private, "<boost/bimap/relation/support/get.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/access_builder.hpp>", private, "<boost/bimap/relation/support/pair_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/access_builder.hpp>", private, "<boost/bimap/support/map_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/relation/support/data_extractor.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/relation/support/opposite_tag.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/relation/support/pair_type_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/relation/support/value_type_of.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/support/data_type_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/support/iterator_type_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/support/key_type_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/support/map_type_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/metadata_access_builder.hpp>", private, "<boost/bimap/support/value_type_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/mutant.hpp>", private, "<boost/bimap/relation/mutant_relation.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/static_access_builder.hpp>", private, "<boost/bimap/support/iterator_type_by.hpp>", public ] },
+    { include: ["<boost/bimap/relation/detail/to_mutable_relation_functor.hpp>", private, "<boost/bimap/views/list_map_view.hpp>", public ] },
+    { include: ["<boost/chrono/detail/inlined/chrono.hpp>", private, "<boost/chrono/system_clocks.hpp>", public ] },
+    { include: ["<boost/chrono/detail/inlined/process_cpu_clocks.hpp>", private, "<boost/chrono/process_cpu_clocks.hpp>", public ] },
+    { include: ["<boost/chrono/detail/inlined/thread_clock.hpp>", private, "<boost/chrono/thread_clock.hpp>", public ] },
+    { include: ["<boost/chrono/detail/is_evenly_divisible_by.hpp>", private, "<boost/chrono/duration.hpp>", public ] },
+    { include: ["<boost/chrono/detail/no_warning/signed_unsigned_cmp.hpp>", private, "<boost/chrono/io/duration_get.hpp>", public ] },
+    { include: ["<boost/chrono/detail/no_warning/signed_unsigned_cmp.hpp>", private, "<boost/chrono/io_v1/chrono_io.hpp>", public ] },
+    { include: ["<boost/chrono/detail/scan_keyword.hpp>", private, "<boost/chrono/io/duration_get.hpp>", public ] },
+    { include: ["<boost/chrono/detail/scan_keyword.hpp>", private, "<boost/chrono/io/time_point_get.hpp>", public ] },
+    { include: ["<boost/chrono/detail/scan_keyword.hpp>", private, "<boost/chrono/io/time_point_io.hpp>", public ] },
+    { include: ["<boost/chrono/detail/scan_keyword.hpp>", private, "<boost/chrono/io_v1/chrono_io.hpp>", public ] },
+    { include: ["<boost/chrono/detail/static_assert.hpp>", private, "<boost/chrono/duration.hpp>", public ] },
+    { include: ["<boost/chrono/detail/system.hpp>", private, "<boost/chrono/process_cpu_clocks.hpp>", public ] },
+    { include: ["<boost/chrono/detail/system.hpp>", private, "<boost/chrono/system_clocks.hpp>", public ] },
+    { include: ["<boost/chrono/detail/system.hpp>", private, "<boost/chrono/thread_clock.hpp>", public ] },
+    { include: ["<boost/concept/detail/backward_compatibility.hpp>", private, "<boost/concept/usage.hpp>", public ] },
+    { include: ["<boost/concept/detail/borland.hpp>", private, "<boost/concept/assert.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_def.hpp>", private, "<boost/concept_check.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_def.hpp>", private, "<boost/graph/bron_kerbosch_all_cliques.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_def.hpp>", private, "<boost/graph/buffer_concepts.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_def.hpp>", private, "<boost/graph/distributed/concepts.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_def.hpp>", private, "<boost/graph/graph_concepts.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_def.hpp>", private, "<boost/graph/tiernan_all_cycles.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_def.hpp>", private, "<boost/iterator/iterator_concepts.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_undef.hpp>", private, "<boost/concept_check.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_undef.hpp>", private, "<boost/graph/bron_kerbosch_all_cliques.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_undef.hpp>", private, "<boost/graph/distributed/concepts.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_undef.hpp>", private, "<boost/graph/graph_concepts.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_undef.hpp>", private, "<boost/graph/tiernan_all_cycles.hpp>", public ] },
+    { include: ["<boost/concept/detail/concept_undef.hpp>", private, "<boost/iterator/iterator_concepts.hpp>", public ] },
+    { include: ["<boost/concept/detail/general.hpp>", private, "<boost/concept/assert.hpp>", public ] },
+    { include: ["<boost/concept/detail/msvc.hpp>", private, "<boost/concept/assert.hpp>", public ] },
+    { include: ["<boost/container/detail/advanced_insert_int.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/container/detail/advanced_insert_int.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/algorithms.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/container/detail/algorithms.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/algorithms.hpp>", private, "<boost/container/stable_vector.hpp>", public ] },
+    { include: ["<boost/container/detail/algorithms.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/algorithms.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/allocation_type.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/allocation_type.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/allocation_type.hpp>", private, "<boost/interprocess/containers/allocation_type.hpp>", public ] },
+    { include: ["<boost/container/detail/allocator_version_traits.hpp>", private, "<boost/container/stable_vector.hpp>", public ] },
+    { include: ["<boost/container/detail/allocator_version_traits.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/allocator_version_traits.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/allocator_traits.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/flat_map.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/flat_set.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/scoped_allocator_fwd.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/scoped_allocator.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/set.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/stable_vector.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/static_vector.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/throw_exception.hpp>", public ] },
+    { include: ["<boost/container/detail/config_begin.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/allocator_traits.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/flat_map.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/flat_set.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/scoped_allocator_fwd.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/scoped_allocator.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/set.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/stable_vector.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/static_vector.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/throw_exception.hpp>", public ] },
+    { include: ["<boost/container/detail/config_end.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/destroyers.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/flat_tree.hpp>", private, "<boost/container/flat_map.hpp>", public ] },
+    { include: ["<boost/container/detail/flat_tree.hpp>", private, "<boost/container/flat_set.hpp>", public ] },
+    { include: ["<boost/container/detail/iterators.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/container/detail/iterators.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/iterators.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/container/detail/iterators.hpp>", private, "<boost/container/stable_vector.hpp>", public ] },
+    { include: ["<boost/container/detail/iterators.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/iterators.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/memory_util.hpp>", private, "<boost/container/allocator_traits.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/allocator_traits.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/flat_map.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/flat_set.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/set.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/mpl.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/allocators/adaptive_pool.hpp>", public ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/allocators/allocator.hpp>", public ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/allocators/node_allocator.hpp>", public ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/allocators/private_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/allocators/private_node_allocator.hpp>", public ] },
+    { include: ["<boost/container/detail/multiallocation_chain.hpp>", private, "<boost/interprocess/mem_algo/rbtree_best_fit.hpp>", public ] },
+    { include: ["<boost/container/detail/node_alloc_holder.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/node_alloc_holder.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/container/detail/pair.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/container/detail/pair.hpp>", private, "<boost/container/scoped_allocator.hpp>", public ] },
+    { include: ["<boost/container/detail/pair.hpp>", private, "<boost/interprocess/containers/pair.hpp>", public ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/allocator_traits.hpp>", public ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/scoped_allocator_fwd.hpp>", public ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/set.hpp>", public ] },
+    { include: ["<boost/container/detail/preprocessor.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/container/detail/tree.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/container/detail/tree.hpp>", private, "<boost/container/set.hpp>", public ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/scoped_allocator_fwd.hpp>", public ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/scoped_allocator.hpp>", public ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/type_traits.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/scoped_allocator.hpp>", public ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/stable_vector.hpp>", public ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/utilities.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/value_init.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/container/detail/version_type.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/container/detail/version_type.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/container/detail/version_type.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/container/detail/version_type.hpp>", private, "<boost/interprocess/containers/version_type.hpp>", public ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/context/detail/config.hpp>", private, "<boost/coroutine/stack_allocator.hpp>", public ] },
+    { include: ["<boost/context/detail/fcontext_arm.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/context/detail/fcontext_i386.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/context/detail/fcontext_i386_win.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/context/detail/fcontext_mips.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/context/detail/fcontext_ppc.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/context/detail/fcontext_sparc.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/context/detail/fcontext_x86_64.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/context/detail/fcontext_x86_64_win.hpp>", private, "<boost/context/fcontext.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/exceptions.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/stack_context.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v1/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/config.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/coroutine_context.hpp>", private, "<boost/coroutine/v1/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/coroutine_context.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/param.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/segmented_stack_allocator.hpp>", private, "<boost/coroutine/stack_allocator.hpp>", public ] },
+    { include: ["<boost/coroutine/detail/standard_stack_allocator.hpp>", private, "<boost/coroutine/stack_allocator.hpp>", public ] },
+    { include: ["<boost/coroutine/v1/detail/arg.hpp>", private, "<boost/coroutine/v1/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_base.hpp>", private, "<boost/coroutine/v1/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_caller.hpp>", private, "<boost/coroutine/v1/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_get.hpp>", private, "<boost/coroutine/v1/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_object.hpp>", private, "<boost/coroutine/v1/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v1/detail/coroutine_op.hpp>", private, "<boost/coroutine/v1/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v2/detail/pull_coroutine_base.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v2/detail/pull_coroutine_caller.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v2/detail/pull_coroutine_object.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v2/detail/push_coroutine_base.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v2/detail/push_coroutine_caller.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/coroutine/v2/detail/push_coroutine_object.hpp>", private, "<boost/coroutine/v2/coroutine.hpp>", public ] },
+    { include: ["<boost/detail/algorithm.hpp>", private, "<boost/graph/graph_utility.hpp>", public ] },
+    { include: ["<boost/detail/algorithm.hpp>", private, "<boost/graph/isomorphism.hpp>", public ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/flyweight/set_factory.hpp>", public ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/statechart/fifo_worker.hpp>", public ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/statechart/processor_container.hpp>", public ] },
+    { include: ["<boost/detail/allocator_utilities.hpp>", private, "<boost/statechart/state_machine.hpp>", public ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/flyweight/refcounted.hpp>", public ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/log/attributes/counter.hpp>", public ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/log/core/record_view.hpp>", public ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/wave/cpplexer/cpp_lex_token.hpp>", public ] },
+    { include: ["<boost/detail/atomic_count.hpp>", private, "<boost/wave/util/macro_definition.hpp>", public ] },
+    { include: ["<boost/detail/atomic_redef_macros.hpp>", private, "<boost/thread/future.hpp>", public ] },
+    { include: ["<boost/detail/atomic_undef_macros.hpp>", private, "<boost/thread/future.hpp>", public ] },
+    { include: ["<boost/detail/binary_search.hpp>", private, "<boost/test/utils/fixed_mapping.hpp>", public ] },
+    { include: ["<boost/detail/call_traits.hpp>", private, "<boost/call_traits.hpp>", public ] },
+    { include: ["<boost/detail/compressed_pair.hpp>", private, "<boost/compressed_pair.hpp>", public ] },
+    { include: ["<boost/detail/container_fwd.hpp>", private, "<boost/functional/hash/extensions.hpp>", public ] },
+    { include: ["<boost/detail/dynamic_bitset.hpp>", private, "<boost/dynamic_bitset/dynamic_bitset.hpp>", public ] },
+    { include: ["<boost/detail/endian.hpp>", private, "<boost/mpl/string.hpp>", public ] },
+    { include: ["<boost/detail/endian.hpp>", private, "<boost/multiprecision/cpp_int.hpp>", public ] },
+    { include: ["<boost/detail/fenv.hpp>", private, "<boost/math/tools/config.hpp>", public ] },
+    { include: ["<boost/detail/indirect_traits.hpp>", private, "<boost/iterator/indirect_iterator.hpp>", public ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/thread/win32/basic_timed_mutex.hpp>", public ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/thread/win32/interlocked_read.hpp>", public ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/thread/win32/once.hpp>", public ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/thread/win32/shared_mutex.hpp>", public ] },
+    { include: ["<boost/detail/interlocked.hpp>", private, "<boost/thread/win32/thread_primitives.hpp>", public ] },
+    { include: ["<boost/detail/is_incrementable.hpp>", private, "<boost/icl/type_traits/is_discrete.hpp>", public ] },
+    { include: ["<boost/detail/is_incrementable.hpp>", private, "<boost/indirect_reference.hpp>", public ] },
+    { include: ["<boost/detail/is_incrementable.hpp>", private, "<boost/iterator/new_iterator_tests.hpp>", public ] },
+    { include: ["<boost/detail/is_incrementable.hpp>", private, "<boost/pointee.hpp>", public ] },
+    { include: ["<boost/detail/is_sorted.hpp>", private, "<boost/graph/distributed/connected_components.hpp>", public ] },
+    { include: ["<boost/detail/is_sorted.hpp>", private, "<boost/range/algorithm_ext/is_sorted.hpp>", public ] },
+    { include: ["<boost/detail/is_xxx.hpp>", private, "<boost/parameter/parameters.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/algorithm/string/find_format.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/algorithm/string/formatter.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/array.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/circular_buffer.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/concept_check.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/dynamic_bitset/dynamic_bitset.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/graph/incremental_components.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/indirect_iterator.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/is_lvalue_iterator.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/is_readable_iterator.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/iterator_adaptor.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/iterator_categories.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/iterator_concepts.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/iterator_traits.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/new_iterator_tests.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/iterator/zip_iterator.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/property_map/property_map.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/python/object/iterator.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/classic/core/scanner/scanner.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/classic/iterator/multi_pass.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/classic/iterator/position_iterator_fwd.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/classic/tree/common.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/classic/utility/regex.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/lex/lexer/lexer.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/functor.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/position_token.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/token.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/lex/lexer/token_def.hpp>", public ] },
+    { include: ["<boost/detail/iterator.hpp>", private, "<boost/spirit/home/support/container.hpp>", public ] },
+    { include: ["<boost/detail/lcast_precision.hpp>", private, "<boost/lexical_cast.hpp>", public ] },
+    { include: ["<boost/detail/lightweight_test.hpp>", private, "<boost/iterator/new_iterator_tests.hpp>", public ] },
+    { include: ["<boost/detail/lightweight_test.hpp>", private, "<boost/mpl/aux_/test.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/chrono/io/duration_io.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/chrono/io/time_point_io.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/circular_buffer/details.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/flat_map.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/scoped_allocator.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/stable_vector.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/string.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/ipc/message_queue.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/managed_heap_memory.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/interprocess/smart_ptr/weak_ptr.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/move/algorithm.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/msm/back/state_machine.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/serialization/state_saver.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/thread/pthread/once_atomic.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/thread/pthread/once.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/thread/win32/once.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/thread/win32/thread_heap_alloc.hpp>", public ] },
+    { include: ["<boost/detail/no_exceptions_support.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/detail/numeric_traits.hpp>", private, "<boost/graph/bandwidth.hpp>", public ] },
+    { include: ["<boost/detail/numeric_traits.hpp>", private, "<boost/graph/minimum_degree_ordering.hpp>", public ] },
+    { include: ["<boost/detail/numeric_traits.hpp>", private, "<boost/graph/profile.hpp>", public ] },
+    { include: ["<boost/detail/numeric_traits.hpp>", private, "<boost/graph/wavefront.hpp>", public ] },
+    { include: ["<boost/detail/numeric_traits.hpp>", private, "<boost/iterator/counting_iterator.hpp>", public ] },
+    { include: ["<boost/detail/ob_call_traits.hpp>", private, "<boost/call_traits.hpp>", public ] },
+    { include: ["<boost/detail/ob_compressed_pair.hpp>", private, "<boost/compressed_pair.hpp>", public ] },
+    { include: ["<boost/detail/reference_content.hpp>", private, "<boost/optional/optional.hpp>", public ] },
+    { include: ["<boost/detail/reference_content.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/detail/scoped_enum_emulation.hpp>", private, "<boost/chrono/io/duration_style.hpp>", public ] },
+    { include: ["<boost/detail/scoped_enum_emulation.hpp>", private, "<boost/chrono/io/timezone.hpp>", public ] },
+    { include: ["<boost/detail/scoped_enum_emulation.hpp>", private, "<boost/coroutine/exceptions.hpp>", public ] },
+    { include: ["<boost/detail/scoped_enum_emulation.hpp>", private, "<boost/thread/cv_status.hpp>", public ] },
+    { include: ["<boost/detail/scoped_enum_emulation.hpp>", private, "<boost/thread/future_error_code.hpp>", public ] },
+    { include: ["<boost/detail/scoped_enum_emulation.hpp>", private, "<boost/thread/future.hpp>", public ] },
+    { include: ["<boost/detail/select_type.hpp>", private, "<boost/cast.hpp>", public ] },
+    { include: ["<boost/detail/sp_typeinfo.hpp>", private, "<boost/proto/debug.hpp>", public ] },
+    { include: ["<boost/detail/templated_streams.hpp>", private, "<boost/blank.hpp>", public ] },
+    { include: ["<boost/detail/templated_streams.hpp>", private, "<boost/flyweight/flyweight_fwd.hpp>", public ] },
+    { include: ["<boost/exception/detail/attribute_noreturn.hpp>", private, "<boost/throw_exception.hpp>", public ] },
+    { include: ["<boost/exception/detail/error_info_impl.hpp>", private, "<boost/exception/get_error_info.hpp>", public ] },
+    { include: ["<boost/exception/detail/error_info_impl.hpp>", private, "<boost/exception/info.hpp>", public ] },
+    { include: ["<boost/exception/detail/exception_ptr.hpp>", private, "<boost/exception_ptr.hpp>", public ] },
+    { include: ["<boost/exception/detail/is_output_streamable.hpp>", private, "<boost/exception/to_string.hpp>", public ] },
+    { include: ["<boost/exception/detail/object_hex_dump.hpp>", private, "<boost/exception/to_string_stub.hpp>", public ] },
+    { include: ["<boost/exception/detail/type_info.hpp>", private, "<boost/exception/get_error_info.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/default_value_policy.hpp>", private, "<boost/flyweight/flyweight.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/flyweight_core.hpp>", private, "<boost/flyweight/flyweight.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/is_placeholder_expr.hpp>", private, "<boost/flyweight/assoc_container_factory.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/nested_xxx_if_not_ph.hpp>", private, "<boost/flyweight/assoc_container_factory.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/not_placeholder_expr.hpp>", private, "<boost/flyweight/assoc_container_factory_fwd.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/not_placeholder_expr.hpp>", private, "<boost/flyweight/hashed_factory_fwd.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/not_placeholder_expr.hpp>", private, "<boost/flyweight/set_factory_fwd.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/perfect_fwd.hpp>", private, "<boost/flyweight/flyweight.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/perfect_fwd.hpp>", private, "<boost/flyweight/key_value.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/recursive_lw_mutex.hpp>", private, "<boost/flyweight/simple_locking.hpp>", public ] },
+    { include: ["<boost/flyweight/detail/value_tag.hpp>", private, "<boost/flyweight/key_value.hpp>", public ] },
+    { include: ["<boost/functional/hash/detail/hash_float.hpp>", private, "<boost/functional/hash/hash.hpp>", public ] },
+    { include: ["<boost/functional/overloaded_function/detail/base.hpp>", private, "<boost/functional/overloaded_function.hpp>", public ] },
+    { include: ["<boost/functional/overloaded_function/detail/function_type.hpp>", private, "<boost/functional/overloaded_function.hpp>", public ] },
+    { include: ["<boost/function_types/detail/classifier.hpp>", private, "<boost/function_types/components.hpp>", public ] },
+    { include: ["<boost/function_types/detail/class_transform.hpp>", private, "<boost/function_types/components.hpp>", public ] },
+    { include: ["<boost/function_types/detail/components_as_mpl_sequence.hpp>", private, "<boost/function_types/components.hpp>", public ] },
+    { include: ["<boost/function_types/detail/pp_loop.hpp>", private, "<boost/function_types/components.hpp>", public ] },
+    { include: ["<boost/function_types/detail/pp_loop.hpp>", private, "<boost/function_types/property_tags.hpp>", public ] },
+    { include: ["<boost/function_types/detail/pp_tags/preprocessed.hpp>", private, "<boost/function_types/property_tags.hpp>", public ] },
+    { include: ["<boost/function_types/detail/retag_default_cc.hpp>", private, "<boost/function_types/components.hpp>", public ] },
+    { include: ["<boost/function_types/detail/synthesize.hpp>", private, "<boost/function_types/function_type.hpp>", public ] },
+    { include: ["<boost/function_types/detail/synthesize.hpp>", private, "<boost/function_types/member_function_pointer.hpp>", public ] },
+    { include: ["<boost/function_types/detail/synthesize.hpp>", private, "<boost/function_types/member_object_pointer.hpp>", public ] },
+    { include: ["<boost/function_types/detail/to_sequence.hpp>", private, "<boost/function_types/function_type.hpp>", public ] },
+    { include: ["<boost/function_types/detail/to_sequence.hpp>", private, "<boost/function_types/member_function_pointer.hpp>", public ] },
+    { include: ["<boost/function_types/detail/to_sequence.hpp>", private, "<boost/function_types/member_object_pointer.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/adt/detail/adapt_base.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/adt/detail/adapt_base.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/adt/detail/extension.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/adt/detail/extension.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_array/detail/at_impl.hpp>", private, "<boost/fusion/adapted/boost_array.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_array/detail/begin_impl.hpp>", private, "<boost/fusion/adapted/boost_array.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_array/detail/category_of_impl.hpp>", private, "<boost/fusion/adapted/boost_array.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_array/detail/end_impl.hpp>", private, "<boost/fusion/adapted/boost_array.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_array/detail/is_sequence_impl.hpp>", private, "<boost/fusion/adapted/boost_array.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_array/detail/is_view_impl.hpp>", private, "<boost/fusion/adapted/boost_array.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_array/detail/size_impl.hpp>", private, "<boost/fusion/adapted/boost_array.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_array/detail/value_at_impl.hpp>", private, "<boost/fusion/adapted/boost_array.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_tuple/detail/at_impl.hpp>", private, "<boost/fusion/adapted/boost_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_tuple/detail/begin_impl.hpp>", private, "<boost/fusion/adapted/boost_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_tuple/detail/category_of_impl.hpp>", private, "<boost/fusion/adapted/boost_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_tuple/detail/end_impl.hpp>", private, "<boost/fusion/adapted/boost_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_tuple/detail/is_sequence_impl.hpp>", private, "<boost/fusion/adapted/boost_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_tuple/detail/is_view_impl.hpp>", private, "<boost/fusion/adapted/boost_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_tuple/detail/size_impl.hpp>", private, "<boost/fusion/adapted/boost_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/boost_tuple/detail/value_at_impl.hpp>", private, "<boost/fusion/adapted/boost_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/at_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/begin_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/begin_impl.hpp>", private, "<boost/fusion/mpl/begin.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/category_of_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/empty_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/end_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/end_impl.hpp>", private, "<boost/fusion/mpl/end.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/has_key_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/is_sequence_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/is_view_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/size_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/mpl/detail/value_at_impl.hpp>", private, "<boost/fusion/adapted/mpl.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/std_tuple/detail/at_impl.hpp>", private, "<boost/fusion/adapted/std_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/std_tuple/detail/begin_impl.hpp>", private, "<boost/fusion/adapted/std_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/std_tuple/detail/category_of_impl.hpp>", private, "<boost/fusion/adapted/std_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/std_tuple/detail/end_impl.hpp>", private, "<boost/fusion/adapted/std_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/std_tuple/detail/is_sequence_impl.hpp>", private, "<boost/fusion/adapted/std_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/std_tuple/detail/is_view_impl.hpp>", private, "<boost/fusion/adapted/std_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/std_tuple/detail/size_impl.hpp>", private, "<boost/fusion/adapted/std_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/std_tuple/detail/value_at_impl.hpp>", private, "<boost/fusion/adapted/std_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/adapt_base.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/adapt_base.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/adapt_base.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/adapt_base.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/at_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/at_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/at_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/at_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/begin_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/begin_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/begin_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/begin_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/category_of_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/category_of_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/category_of_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/category_of_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/define_struct.hpp>", private, "<boost/fusion/adapted/struct/define_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/define_struct.hpp>", private, "<boost/fusion/adapted/struct/define_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/define_struct_inline.hpp>", private, "<boost/fusion/adapted/struct/define_struct_inline.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/deref_data_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/deref_data_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/deref_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/deref_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/deref_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/deref_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/end_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/end_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/end_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/end_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/extension.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/extension.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/extension.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/extension.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/is_sequence_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/is_sequence_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/is_sequence_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/is_sequence_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/is_view_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/is_view_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/is_view_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/is_view_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/key_of_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/key_of_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/proxy_type.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt_named.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/proxy_type.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt_named.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/proxy_type.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct_named.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/proxy_type.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct_named.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/proxy_type.hpp>", private, "<boost/fusion/include/proxy_type.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/size_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/size_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/size_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/size_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_at_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_at_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_at_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_at_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_of_data_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_of_data_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_of_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_of_impl.hpp>", private, "<boost/fusion/adapted/adt/adapt_assoc_adt.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_of_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_assoc_struct.hpp>", public ] },
+    { include: ["<boost/fusion/adapted/struct/detail/value_of_impl.hpp>", private, "<boost/fusion/adapted/struct/adapt_struct.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/fold.hpp>", private, "<boost/fusion/algorithm/iteration/fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/fold.hpp>", private, "<boost/fusion/algorithm/iteration/iter_fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/fold.hpp>", private, "<boost/fusion/algorithm/iteration/reverse_fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/fold.hpp>", private, "<boost/fusion/algorithm/iteration/reverse_iter_fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/for_each.hpp>", private, "<boost/fusion/algorithm/iteration/for_each.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/preprocessed/fold.hpp>", private, "<boost/fusion/algorithm/iteration/fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/preprocessed/iter_fold.hpp>", private, "<boost/fusion/algorithm/iteration/iter_fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/preprocessed/reverse_fold.hpp>", private, "<boost/fusion/algorithm/iteration/reverse_fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/preprocessed/reverse_iter_fold.hpp>", private, "<boost/fusion/algorithm/iteration/reverse_iter_fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/segmented_fold.hpp>", private, "<boost/fusion/algorithm/iteration/fold.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/iteration/detail/segmented_for_each.hpp>", private, "<boost/fusion/algorithm/iteration/for_each.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/all.hpp>", private, "<boost/fusion/algorithm/query/all.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/any.hpp>", private, "<boost/fusion/algorithm/query/any.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/count.hpp>", private, "<boost/fusion/algorithm/query/count.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/count_if.hpp>", private, "<boost/fusion/algorithm/query/count_if.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/find_if.hpp>", private, "<boost/fusion/algorithm/query/find.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/find_if.hpp>", private, "<boost/fusion/algorithm/query/find_if.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/find_if.hpp>", private, "<boost/fusion/view/filter_view/filter_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/segmented_find.hpp>", private, "<boost/fusion/algorithm/query/find.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/query/detail/segmented_find_if.hpp>", private, "<boost/fusion/algorithm/query/find_if.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/transformation/detail/preprocessed/zip.hpp>", private, "<boost/fusion/algorithm/transformation/zip.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/transformation/detail/replace.hpp>", private, "<boost/fusion/algorithm/transformation/replace.hpp>", public ] },
+    { include: ["<boost/fusion/algorithm/transformation/detail/replace_if.hpp>", private, "<boost/fusion/algorithm/transformation/replace_if.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/at_impl.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/begin_impl.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/build_deque.hpp>", private, "<boost/fusion/container/deque/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/convert_impl.hpp>", private, "<boost/fusion/container/deque/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/build_deque.hpp>", private, "<boost/fusion/container/deque/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/deque_fwd.hpp>", private, "<boost/fusion/container/deque/deque_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/cpp03/deque.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/deque_keyed_values.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/end_impl.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/is_sequence_impl.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/back_extended_deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/deque_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/keyed_element.hpp>", private, "<boost/fusion/container/deque/front_extended_deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/deque/detail/value_at_impl.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/pp_deque_tie.hpp>", private, "<boost/fusion/container/generation/deque_tie.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/pp_make_deque.hpp>", private, "<boost/fusion/container/generation/make_deque.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/pp_make_map.hpp>", private, "<boost/fusion/container/generation/make_map.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/pp_map_tie.hpp>", private, "<boost/fusion/container/generation/map_tie.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/list_tie.hpp>", private, "<boost/fusion/container/generation/list_tie.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_list.hpp>", private, "<boost/fusion/container/generation/make_list.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_set.hpp>", private, "<boost/fusion/container/generation/make_set.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/make_vector.hpp>", private, "<boost/fusion/container/generation/make_vector.hpp>", public ] },
+    { include: ["<boost/fusion/container/generation/detail/preprocessed/vector_tie.hpp>", private, "<boost/fusion/container/generation/vector_tie.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/at_impl.hpp>", private, "<boost/fusion/container/list/cons.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/begin_impl.hpp>", private, "<boost/fusion/container/list/cons.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/build_cons.hpp>", private, "<boost/fusion/container/list/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/convert_impl.hpp>", private, "<boost/fusion/container/list/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/deref_impl.hpp>", private, "<boost/fusion/container/list/cons_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/empty_impl.hpp>", private, "<boost/fusion/container/list/cons.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/end_impl.hpp>", private, "<boost/fusion/container/list/cons.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/equal_to_impl.hpp>", private, "<boost/fusion/container/list/cons_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/list_forward_ctor.hpp>", private, "<boost/fusion/container/list/list.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/list_to_cons.hpp>", private, "<boost/fusion/container/list/list.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/next_impl.hpp>", private, "<boost/fusion/container/list/cons_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list_fwd.hpp>", private, "<boost/fusion/container/list/list_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/preprocessed/list.hpp>", private, "<boost/fusion/container/list/list.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/value_at_impl.hpp>", private, "<boost/fusion/container/list/cons.hpp>", public ] },
+    { include: ["<boost/fusion/container/list/detail/value_of_impl.hpp>", private, "<boost/fusion/container/list/cons_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/at_impl.hpp>", private, "<boost/fusion/container/map/map.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/at_key_impl.hpp>", private, "<boost/fusion/container/map/map.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/begin_impl.hpp>", private, "<boost/fusion/container/map/map.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/build_map.hpp>", private, "<boost/fusion/container/map/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/convert.hpp>", private, "<boost/fusion/container/map/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/map_fwd.hpp>", private, "<boost/fusion/container/map/map_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/cpp03/map.hpp>", private, "<boost/fusion/container/map/map.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/end_impl.hpp>", private, "<boost/fusion/container/map/map.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/map_impl.hpp>", private, "<boost/fusion/container/map/map_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/map_impl.hpp>", private, "<boost/fusion/container/map/map.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/value_at_impl.hpp>", private, "<boost/fusion/container/map/map.hpp>", public ] },
+    { include: ["<boost/fusion/container/map/detail/value_at_key_impl.hpp>", private, "<boost/fusion/container/map/map.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/as_set.hpp>", private, "<boost/fusion/container/set/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/begin_impl.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/convert_impl.hpp>", private, "<boost/fusion/container/set/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/deref_data_impl.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/deref_impl.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/end_impl.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/key_of_impl.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set_fwd.hpp>", private, "<boost/fusion/container/set/set_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/preprocessed/set.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/set_forward_ctor.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/value_of_data_impl.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/set/detail/value_of_impl.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/advance_impl.hpp>", private, "<boost/fusion/container/vector/vector_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/as_vector.hpp>", private, "<boost/fusion/container/vector/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/at_impl.hpp>", private, "<boost/fusion/container/vector/vector10.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/at_impl.hpp>", private, "<boost/fusion/container/vector/vector20.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/at_impl.hpp>", private, "<boost/fusion/container/vector/vector30.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/at_impl.hpp>", private, "<boost/fusion/container/vector/vector40.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/at_impl.hpp>", private, "<boost/fusion/container/vector/vector50.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/begin_impl.hpp>", private, "<boost/fusion/container/vector/vector10.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/begin_impl.hpp>", private, "<boost/fusion/container/vector/vector20.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/begin_impl.hpp>", private, "<boost/fusion/container/vector/vector30.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/begin_impl.hpp>", private, "<boost/fusion/container/vector/vector40.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/begin_impl.hpp>", private, "<boost/fusion/container/vector/vector50.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/convert_impl.hpp>", private, "<boost/fusion/container/vector/convert.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/deref_impl.hpp>", private, "<boost/fusion/container/vector/vector_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/distance_impl.hpp>", private, "<boost/fusion/container/vector/vector_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/end_impl.hpp>", private, "<boost/fusion/container/vector/vector10.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/end_impl.hpp>", private, "<boost/fusion/container/vector/vector20.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/end_impl.hpp>", private, "<boost/fusion/container/vector/vector30.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/end_impl.hpp>", private, "<boost/fusion/container/vector/vector40.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/end_impl.hpp>", private, "<boost/fusion/container/vector/vector50.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/equal_to_impl.hpp>", private, "<boost/fusion/container/vector/vector_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/next_impl.hpp>", private, "<boost/fusion/container/vector/vector_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector10_fwd.hpp>", private, "<boost/fusion/container/vector/vector10_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector10.hpp>", private, "<boost/fusion/container/vector/vector10.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector20_fwd.hpp>", private, "<boost/fusion/container/vector/vector20_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector20.hpp>", private, "<boost/fusion/container/vector/vector20.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector30_fwd.hpp>", private, "<boost/fusion/container/vector/vector30_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector30.hpp>", private, "<boost/fusion/container/vector/vector30.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector40_fwd.hpp>", private, "<boost/fusion/container/vector/vector40_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector40.hpp>", private, "<boost/fusion/container/vector/vector40.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector50_fwd.hpp>", private, "<boost/fusion/container/vector/vector50_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector50.hpp>", private, "<boost/fusion/container/vector/vector50.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector_fwd.hpp>", private, "<boost/fusion/container/vector/vector_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/preprocessed/vector.hpp>", private, "<boost/fusion/container/vector/vector.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/prior_impl.hpp>", private, "<boost/fusion/container/vector/vector_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/value_at_impl.hpp>", private, "<boost/fusion/container/vector/vector10.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/value_at_impl.hpp>", private, "<boost/fusion/container/vector/vector20.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/value_at_impl.hpp>", private, "<boost/fusion/container/vector/vector30.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/value_at_impl.hpp>", private, "<boost/fusion/container/vector/vector40.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/value_at_impl.hpp>", private, "<boost/fusion/container/vector/vector50.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/value_of_impl.hpp>", private, "<boost/fusion/container/vector/vector_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/vector_forward_ctor.hpp>", private, "<boost/fusion/container/vector/vector.hpp>", public ] },
+    { include: ["<boost/fusion/container/vector/detail/vector_n_chooser.hpp>", private, "<boost/fusion/container/vector/vector.hpp>", public ] },
+    { include: ["<boost/fusion/functional/adapter/detail/access.hpp>", private, "<boost/fusion/functional/adapter/fused_function_object.hpp>", public ] },
+    { include: ["<boost/fusion/functional/adapter/detail/access.hpp>", private, "<boost/fusion/functional/adapter/fused.hpp>", public ] },
+    { include: ["<boost/fusion/functional/adapter/detail/access.hpp>", private, "<boost/fusion/functional/adapter/fused_procedure.hpp>", public ] },
+    { include: ["<boost/fusion/functional/adapter/detail/access.hpp>", private, "<boost/fusion/functional/adapter/unfused.hpp>", public ] },
+    { include: ["<boost/fusion/functional/adapter/detail/access.hpp>", private, "<boost/fusion/functional/adapter/unfused_typed.hpp>", public ] },
+    { include: ["<boost/fusion/functional/generation/detail/gen_make_adapter.hpp>", private, "<boost/fusion/functional/generation/make_fused_function_object.hpp>", public ] },
+    { include: ["<boost/fusion/functional/generation/detail/gen_make_adapter.hpp>", private, "<boost/fusion/functional/generation/make_fused.hpp>", public ] },
+    { include: ["<boost/fusion/functional/generation/detail/gen_make_adapter.hpp>", private, "<boost/fusion/functional/generation/make_fused_procedure.hpp>", public ] },
+    { include: ["<boost/fusion/functional/generation/detail/gen_make_adapter.hpp>", private, "<boost/fusion/functional/generation/make_unfused.hpp>", public ] },
+    { include: ["<boost/fusion/functional/invocation/detail/that_ptr.hpp>", private, "<boost/fusion/functional/invocation/invoke.hpp>", public ] },
+    { include: ["<boost/fusion/functional/invocation/detail/that_ptr.hpp>", private, "<boost/fusion/functional/invocation/invoke_procedure.hpp>", public ] },
+    { include: ["<boost/fusion/iterator/detail/advance.hpp>", private, "<boost/fusion/iterator/advance.hpp>", public ] },
+    { include: ["<boost/fusion/iterator/detail/advance.hpp>", private, "<boost/fusion/iterator/iterator_adapter.hpp>", public ] },
+    { include: ["<boost/fusion/iterator/detail/advance.hpp>", private, "<boost/fusion/iterator/iterator_facade.hpp>", public ] },
+    { include: ["<boost/fusion/iterator/detail/distance.hpp>", private, "<boost/fusion/iterator/distance.hpp>", public ] },
+    { include: ["<boost/fusion/iterator/detail/distance.hpp>", private, "<boost/fusion/iterator/iterator_facade.hpp>", public ] },
+    { include: ["<boost/fusion/iterator/detail/segmented_iterator.hpp>", private, "<boost/fusion/iterator/segmented_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/iterator/detail/segmented_next_impl.hpp>", private, "<boost/fusion/iterator/segmented_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/mpl/detail/clear.hpp>", private, "<boost/fusion/mpl/clear.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/comparison/detail/equal_to.hpp>", private, "<boost/fusion/algorithm/auxiliary/copy.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/comparison/detail/equal_to.hpp>", private, "<boost/fusion/algorithm/auxiliary/move.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/comparison/detail/equal_to.hpp>", private, "<boost/fusion/sequence/comparison/equal_to.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/comparison/detail/greater_equal.hpp>", private, "<boost/fusion/sequence/comparison/greater_equal.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/comparison/detail/greater.hpp>", private, "<boost/fusion/sequence/comparison/greater.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/comparison/detail/less_equal.hpp>", private, "<boost/fusion/sequence/comparison/less_equal.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/comparison/detail/less.hpp>", private, "<boost/fusion/sequence/comparison/less.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/comparison/detail/not_equal_to.hpp>", private, "<boost/fusion/sequence/comparison/not_equal_to.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/intrinsic/detail/segmented_begin.hpp>", private, "<boost/fusion/sequence/intrinsic/begin.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/intrinsic/detail/segmented_end.hpp>", private, "<boost/fusion/sequence/intrinsic/end.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/intrinsic/detail/segmented_size.hpp>", private, "<boost/fusion/sequence/intrinsic/size.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/io/detail/in.hpp>", private, "<boost/fusion/sequence/io/in.hpp>", public ] },
+    { include: ["<boost/fusion/sequence/io/detail/out.hpp>", private, "<boost/fusion/sequence/io/out.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/adapted/std_tuple/std_tuple_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/deque/deque.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/list/cons.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/set/set.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/vector/vector10.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/vector/vector20.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/vector/vector30.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/vector/vector40.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/container/vector/vector50.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/functional/adapter/unfused_typed.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/sequence/intrinsic/at.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/sequence/intrinsic/at_key.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/support/pair.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/view/filter_view/filter_view.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/view/iterator_range/iterator_range.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/view/joint_view/joint_view.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/view/single_view/single_view.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/view/single_view/single_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/access.hpp>", private, "<boost/fusion/view/transform_view/transform_view.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/algorithm/transformation/erase.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/algorithm/transformation/insert.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/algorithm/transformation/insert_range.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/algorithm/transformation/push_back.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/algorithm/transformation/push_front.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/deque_tie.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/make_cons.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/make_deque.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/make_list.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/make_map.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/make_set.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/container/generation/make_vector.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/support/pair.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/tuple/make_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/as_fusion_element.hpp>", private, "<boost/fusion/view/single_view/single_view.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/category_of.hpp>", private, "<boost/fusion/support/category_of.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/is_mpl_sequence.hpp>", private, "<boost/fusion/support/tag_of.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/is_view.hpp>", private, "<boost/fusion/support/is_view.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/mpl_iterator_category.hpp>", private, "<boost/fusion/adapted/mpl/mpl_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/pp_round.hpp>", private, "<boost/fusion/algorithm/transformation/zip.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/pp_round.hpp>", private, "<boost/fusion/container/vector/limits.hpp>", public ] },
+    { include: ["<boost/fusion/support/detail/segmented_fold_until_impl.hpp>", private, "<boost/fusion/support/segmented_fold_until.hpp>", public ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/make_tuple.hpp>", private, "<boost/fusion/tuple/make_tuple.hpp>", public ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple_fwd.hpp>", private, "<boost/fusion/tuple/tuple_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple.hpp>", private, "<boost/fusion/tuple/tuple.hpp>", public ] },
+    { include: ["<boost/fusion/tuple/detail/preprocessed/tuple_tie.hpp>", private, "<boost/fusion/tuple/tuple_tie.hpp>", public ] },
+    { include: ["<boost/fusion/tuple/detail/tuple_expand.hpp>", private, "<boost/fusion/tuple/tuple.hpp>", public ] },
+    { include: ["<boost/fusion/view/detail/strictest_traversal.hpp>", private, "<boost/fusion/view/transform_view/transform_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/detail/strictest_traversal.hpp>", private, "<boost/fusion/view/zip_view/zip_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/detail/strictest_traversal.hpp>", private, "<boost/fusion/view/zip_view/zip_view_iterator_fwd.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/begin_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/deref_data_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/deref_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/end_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/equal_to_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/key_of_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/next_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/size_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/value_of_data_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/filter_view/detail/value_of_impl.hpp>", private, "<boost/fusion/view/filter_view/filter_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/iterator_range/detail/at_impl.hpp>", private, "<boost/fusion/view/iterator_range/iterator_range.hpp>", public ] },
+    { include: ["<boost/fusion/view/iterator_range/detail/begin_impl.hpp>", private, "<boost/fusion/view/iterator_range/iterator_range.hpp>", public ] },
+    { include: ["<boost/fusion/view/iterator_range/detail/end_impl.hpp>", private, "<boost/fusion/view/iterator_range/iterator_range.hpp>", public ] },
+    { include: ["<boost/fusion/view/iterator_range/detail/is_segmented_impl.hpp>", private, "<boost/fusion/view/iterator_range/iterator_range.hpp>", public ] },
+    { include: ["<boost/fusion/view/iterator_range/detail/segments_impl.hpp>", private, "<boost/fusion/view/iterator_range/iterator_range.hpp>", public ] },
+    { include: ["<boost/fusion/view/iterator_range/detail/size_impl.hpp>", private, "<boost/fusion/view/iterator_range/iterator_range.hpp>", public ] },
+    { include: ["<boost/fusion/view/iterator_range/detail/value_at_impl.hpp>", private, "<boost/fusion/view/iterator_range/iterator_range.hpp>", public ] },
+    { include: ["<boost/fusion/view/joint_view/detail/begin_impl.hpp>", private, "<boost/fusion/view/joint_view/joint_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/joint_view/detail/deref_data_impl.hpp>", private, "<boost/fusion/view/joint_view/joint_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/joint_view/detail/deref_impl.hpp>", private, "<boost/fusion/view/joint_view/joint_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/joint_view/detail/end_impl.hpp>", private, "<boost/fusion/view/joint_view/joint_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/joint_view/detail/key_of_impl.hpp>", private, "<boost/fusion/view/joint_view/joint_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/joint_view/detail/next_impl.hpp>", private, "<boost/fusion/view/joint_view/joint_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/joint_view/detail/value_of_data_impl.hpp>", private, "<boost/fusion/view/joint_view/joint_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/joint_view/detail/value_of_impl.hpp>", private, "<boost/fusion/view/joint_view/joint_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/advance_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/at_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/begin_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/deref_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/distance_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/end_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/equal_to_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/next_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/nview_impl.hpp>", private, "<boost/fusion/view/nview/nview.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/prior_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/size_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/value_at_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/nview/detail/value_of_impl.hpp>", private, "<boost/fusion/view/nview/nview_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/repetitive_view/detail/begin_impl.hpp>", private, "<boost/fusion/view/repetitive_view/repetitive_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/repetitive_view/detail/deref_impl.hpp>", private, "<boost/fusion/view/repetitive_view/repetitive_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/repetitive_view/detail/end_impl.hpp>", private, "<boost/fusion/view/repetitive_view/repetitive_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/repetitive_view/detail/next_impl.hpp>", private, "<boost/fusion/view/repetitive_view/repetitive_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/repetitive_view/detail/value_of_impl.hpp>", private, "<boost/fusion/view/repetitive_view/repetitive_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/advance_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/at_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/begin_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/deref_data_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/deref_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/distance_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/end_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/key_of_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/next_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/prior_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/value_at_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/value_of_data_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/reverse_view/detail/value_of_impl.hpp>", private, "<boost/fusion/view/reverse_view/reverse_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/advance_impl.hpp>", private, "<boost/fusion/view/single_view/single_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/at_impl.hpp>", private, "<boost/fusion/view/single_view/single_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/begin_impl.hpp>", private, "<boost/fusion/view/single_view/single_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/deref_impl.hpp>", private, "<boost/fusion/view/single_view/single_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/distance_impl.hpp>", private, "<boost/fusion/view/single_view/single_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/end_impl.hpp>", private, "<boost/fusion/view/single_view/single_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/equal_to_impl.hpp>", private, "<boost/fusion/view/single_view/single_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/next_impl.hpp>", private, "<boost/fusion/view/single_view/single_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/prior_impl.hpp>", private, "<boost/fusion/view/single_view/single_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/size_impl.hpp>", private, "<boost/fusion/view/single_view/single_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/value_at_impl.hpp>", private, "<boost/fusion/view/single_view/single_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/single_view/detail/value_of_impl.hpp>", private, "<boost/fusion/view/single_view/single_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/advance_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/at_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/begin_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/deref_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/distance_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/end_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/equal_to_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/next_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/prior_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/value_at_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/transform_view/detail/value_of_impl.hpp>", private, "<boost/fusion/view/transform_view/transform_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/advance_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/at_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/begin_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/deref_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/distance_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/end_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/equal_to_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/next_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/prior_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view_iterator.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/size_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/value_at_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view.hpp>", public ] },
+    { include: ["<boost/fusion/view/zip_view/detail/value_of_impl.hpp>", private, "<boost/fusion/view/zip_view/zip_view_iterator.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/as_range.hpp>", private, "<boost/geometry/algorithms/convex_hull.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_box_corners.hpp>", private, "<boost/geometry/algorithms/assign.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_box_corners.hpp>", private, "<boost/geometry/algorithms/convert.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_box_corners.hpp>", private, "<boost/geometry/algorithms/convex_hull.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_indexed_point.hpp>", private, "<boost/geometry/algorithms/assign.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_indexed_point.hpp>", private, "<boost/geometry/algorithms/convert.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_values.hpp>", private, "<boost/geometry/algorithms/assign.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_values.hpp>", private, "<boost/geometry/algorithms/convert.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/assign_values.hpp>", private, "<boost/geometry/strategies/cartesian/cart_intersect.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/calculate_null.hpp>", private, "<boost/geometry/algorithms/area.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/calculate_null.hpp>", private, "<boost/geometry/algorithms/length.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/calculate_null.hpp>", private, "<boost/geometry/algorithms/perimeter.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/calculate_sum.hpp>", private, "<boost/geometry/algorithms/area.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/calculate_sum.hpp>", private, "<boost/geometry/algorithms/perimeter.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/convert_indexed_to_indexed.hpp>", private, "<boost/geometry/algorithms/convert.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/convert_point_to_point.hpp>", private, "<boost/geometry/algorithms/append.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/convert_point_to_point.hpp>", private, "<boost/geometry/algorithms/convert.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/disjoint.hpp>", private, "<boost/geometry/algorithms/buffer.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/disjoint.hpp>", private, "<boost/geometry/algorithms/disjoint.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/disjoint.hpp>", private, "<boost/geometry/algorithms/equals.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/equals/collect_vectors.hpp>", private, "<boost/geometry/algorithms/equals.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/for_each_range.hpp>", private, "<boost/geometry/algorithms/disjoint.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/for_each_range.hpp>", private, "<boost/geometry/strategies/agnostic/hull_graham_andrew.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/not.hpp>", private, "<boost/geometry/algorithms/equals.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/algorithms/disjoint.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/algorithms/touches.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/intersection_insert.hpp>", private, "<boost/geometry/algorithms/difference.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/intersection_insert.hpp>", private, "<boost/geometry/algorithms/intersection.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/overlay.hpp>", private, "<boost/geometry/algorithms/union.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/self_turn_points.hpp>", private, "<boost/geometry/algorithms/intersects.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/overlay/self_turn_points.hpp>", private, "<boost/geometry/algorithms/touches.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/point_on_border.hpp>", private, "<boost/geometry/algorithms/disjoint.hpp>", public ] },
+    { include: ["<boost/geometry/algorithms/detail/throw_on_empty_input.hpp>", private, "<boost/geometry/algorithms/distance.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/algorithms/is_valid.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/assert.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/config_begin.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/config_end.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/distance_predicates.hpp>", private, "<boost/geometry/index/distance_predicates.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/exception.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/meta.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/predicates.hpp>", private, "<boost/geometry/index/predicates.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/adaptors.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/linear/linear.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/node/node.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/options.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/pack_create.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/quadratic/quadratic.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/query_iterators.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/rstar/rstar.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/utilities/view.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/children_box.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/copy.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/count.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/destroy.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/distance_query.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/insert.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/remove.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/rtree/visitors/spatial_query.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/serialization.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/translator.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/tuples.hpp>", private, "<boost/geometry/index/predicates.hpp>", public ] },
+    { include: ["<boost/geometry/index/detail/utilities.hpp>", private, "<boost/geometry/index/rtree.hpp>", public ] },
+    { include: ["<boost/geometry/io/wkt/detail/prefix.hpp>", private, "<boost/geometry/io/wkt/read.hpp>", public ] },
+    { include: ["<boost/geometry/io/wkt/detail/prefix.hpp>", private, "<boost/geometry/io/wkt/write.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/for_each_range.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/modify.hpp>", private, "<boost/geometry/multi/algorithms/correct.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/modify.hpp>", private, "<boost/geometry/multi/algorithms/reverse.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/modify_with_predicate.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/multi_sum.hpp>", private, "<boost/geometry/multi/algorithms/area.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/multi_sum.hpp>", private, "<boost/geometry/multi/algorithms/length.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/multi_sum.hpp>", private, "<boost/geometry/multi/algorithms/perimeter.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/multi_sum.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/copy_segment_point.hpp>", private, "<boost/geometry/multi/algorithms/intersection.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/copy_segment_point.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/copy_segments.hpp>", private, "<boost/geometry/multi/algorithms/intersection.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/copy_segments.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/get_ring.hpp>", private, "<boost/geometry/multi/algorithms/intersection.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/get_ring.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/multi/algorithms/intersection.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/get_turns.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/select_rings.hpp>", private, "<boost/geometry/multi/algorithms/intersection.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/select_rings.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/overlay/self_turn_points.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/sections/range_by_section.hpp>", private, "<boost/geometry/multi/algorithms/intersection.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/sections/range_by_section.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/sections/sectionalize.hpp>", private, "<boost/geometry/multi/algorithms/intersection.hpp>", public ] },
+    { include: ["<boost/geometry/multi/algorithms/detail/sections/sectionalize.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/multi/io/wkt/detail/prefix.hpp>", private, "<boost/geometry/multi/io/wkt/read.hpp>", public ] },
+    { include: ["<boost/geometry/multi/io/wkt/detail/prefix.hpp>", private, "<boost/geometry/multi/io/wkt/write.hpp>", public ] },
+    { include: ["<boost/geometry/multi/views/detail/range_type.hpp>", private, "<boost/geometry/multi/multi.hpp>", public ] },
+    { include: ["<boost/geometry/views/detail/points_view.hpp>", private, "<boost/geometry/views/box_view.hpp>", public ] },
+    { include: ["<boost/geometry/views/detail/points_view.hpp>", private, "<boost/geometry/views/segment_view.hpp>", public ] },
+    { include: ["<boost/geometry/views/detail/range_type.hpp>", private, "<boost/geometry/algorithms/convex_hull.hpp>", public ] },
+    { include: ["<boost/geometry/views/detail/range_type.hpp>", private, "<boost/geometry/strategies/agnostic/hull_graham_andrew.hpp>", public ] },
+    { include: ["<boost/graph/detail/adjacency_list.hpp>", private, "<boost/graph/adjacency_list.hpp>", public ] },
+    { include: ["<boost/graph/detail/array_binary_tree.hpp>", private, "<boost/pending/mutable_queue.hpp>", public ] },
+    { include: ["<boost/graph/detail/augment.hpp>", private, "<boost/graph/cycle_canceling.hpp>", public ] },
+    { include: ["<boost/graph/detail/augment.hpp>", private, "<boost/graph/successive_shortest_path_nonnegative_weights.hpp>", public ] },
+    { include: ["<boost/graph/detail/compressed_sparse_row_struct.hpp>", private, "<boost/graph/compressed_sparse_row_graph.hpp>", public ] },
+    { include: ["<boost/graph/detail/d_ary_heap.hpp>", private, "<boost/graph/astar_search.hpp>", public ] },
+    { include: ["<boost/graph/detail/d_ary_heap.hpp>", private, "<boost/graph/core_numbers.hpp>", public ] },
+    { include: ["<boost/graph/detail/d_ary_heap.hpp>", private, "<boost/graph/dijkstra_shortest_paths.hpp>", public ] },
+    { include: ["<boost/graph/detail/d_ary_heap.hpp>", private, "<boost/graph/dijkstra_shortest_paths_no_color_map.hpp>", public ] },
+    { include: ["<boost/graph/detail/d_ary_heap.hpp>", private, "<boost/graph/named_function_params.hpp>", public ] },
+    { include: ["<boost/graph/detail/d_ary_heap.hpp>", private, "<boost/graph/stoer_wagner_min_cut.hpp>", public ] },
+    { include: ["<boost/graph/detail/edge.hpp>", private, "<boost/graph/adjacency_list.hpp>", public ] },
+    { include: ["<boost/graph/detail/edge.hpp>", private, "<boost/graph/adjacency_matrix.hpp>", public ] },
+    { include: ["<boost/graph/detail/edge.hpp>", private, "<boost/property_map/parallel/distributed_property_map.hpp>", public ] },
+    { include: ["<boost/graph/detail/geodesic.hpp>", private, "<boost/graph/closeness_centrality.hpp>", public ] },
+    { include: ["<boost/graph/detail/geodesic.hpp>", private, "<boost/graph/eccentricity.hpp>", public ] },
+    { include: ["<boost/graph/detail/geodesic.hpp>", private, "<boost/graph/geodesic_distance.hpp>", public ] },
+    { include: ["<boost/graph/detail/incremental_components.hpp>", private, "<boost/graph/incremental_components.hpp>", public ] },
+    { include: ["<boost/graph/detail/indexed_properties.hpp>", private, "<boost/graph/compressed_sparse_row_graph.hpp>", public ] },
+    { include: ["<boost/graph/detail/index.hpp>", private, "<boost/graph/property_maps/container_property_map.hpp>", public ] },
+    { include: ["<boost/graph/detail/is_distributed_selector.hpp>", private, "<boost/graph/compressed_sparse_row_graph.hpp>", public ] },
+    { include: ["<boost/graph/detail/is_distributed_selector.hpp>", private, "<boost/graph/distributed/selector.hpp>", public ] },
+    { include: ["<boost/graph/detail/labeled_graph_traits.hpp>", private, "<boost/graph/labeled_graph.hpp>", public ] },
+    { include: ["<boost/graph/detail/read_graphviz_new.hpp>", private, "<boost/graph/graphviz.hpp>", public ] },
+    { include: ["<boost/graph/detail/read_graphviz_spirit.hpp>", private, "<boost/graph/graphviz.hpp>", public ] },
+    { include: ["<boost/graph/detail/set_adaptor.hpp>", private, "<boost/graph/filtered_graph.hpp>", public ] },
+    { include: ["<boost/graph/detail/sparse_ordering.hpp>", private, "<boost/graph/cuthill_mckee_ordering.hpp>", public ] },
+    { include: ["<boost/graph/detail/sparse_ordering.hpp>", private, "<boost/graph/king_ordering.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/dijkstra_shortest_paths.hpp>", private, "<boost/graph/distributed/betweenness_centrality.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/dijkstra_shortest_paths.hpp>", private, "<boost/graph/distributed/crauser_et_al_shortest_paths.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/dijkstra_shortest_paths.hpp>", private, "<boost/graph/distributed/delta_stepping_shortest_paths.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/dijkstra_shortest_paths.hpp>", private, "<boost/graph/distributed/eager_dijkstra_shortest_paths.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/filtered_queue.hpp>", private, "<boost/graph/distributed/breadth_first_search.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/filtered_queue.hpp>", private, "<boost/graph/distributed/strong_components.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/mpi_process_group.ipp>", private, "<boost/graph/distributed/mpi_process_group.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/queue.ipp>", private, "<boost/graph/distributed/queue.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/remote_update_set.hpp>", private, "<boost/graph/distributed/crauser_et_al_shortest_paths.hpp>", public ] },
+    { include: ["<boost/graph/distributed/detail/remote_update_set.hpp>", private, "<boost/graph/distributed/eager_dijkstra_shortest_paths.hpp>", public ] },
+    { include: ["<boost/graph/parallel/detail/inplace_all_to_all.hpp>", private, "<boost/graph/parallel/algorithm.hpp>", public ] },
+    { include: ["<boost/graph/parallel/detail/property_holders.hpp>", private, "<boost/graph/distributed/adjacency_list.hpp>", public ] },
+    { include: ["<boost/graph/parallel/detail/property_holders.hpp>", private, "<boost/graph/distributed/named_graph.hpp>", public ] },
+    { include: ["<boost/graph/parallel/detail/untracked_pair.hpp>", private, "<boost/graph/distributed/adjlist/handlers.hpp>", public ] },
+    { include: ["<boost/graph/parallel/detail/untracked_pair.hpp>", private, "<boost/graph/distributed/dehne_gotz_min_spanning_tree.hpp>", public ] },
+    { include: ["<boost/graph/parallel/detail/untracked_pair.hpp>", private, "<boost/property_map/parallel/distributed_property_map.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_comparison.hpp>", private, "<boost/heap/binomial_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_comparison.hpp>", private, "<boost/heap/d_ary_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_comparison.hpp>", private, "<boost/heap/fibonacci_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_comparison.hpp>", private, "<boost/heap/pairing_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_comparison.hpp>", private, "<boost/heap/priority_queue.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_comparison.hpp>", private, "<boost/heap/skew_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_node.hpp>", private, "<boost/heap/binomial_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_node.hpp>", private, "<boost/heap/fibonacci_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_node.hpp>", private, "<boost/heap/pairing_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/heap_node.hpp>", private, "<boost/heap/skew_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/mutable_heap.hpp>", private, "<boost/heap/d_ary_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/ordered_adaptor_iterator.hpp>", private, "<boost/heap/d_ary_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/stable_heap.hpp>", private, "<boost/heap/binomial_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/stable_heap.hpp>", private, "<boost/heap/d_ary_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/stable_heap.hpp>", private, "<boost/heap/fibonacci_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/stable_heap.hpp>", private, "<boost/heap/pairing_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/stable_heap.hpp>", private, "<boost/heap/priority_queue.hpp>", public ] },
+    { include: ["<boost/heap/detail/stable_heap.hpp>", private, "<boost/heap/skew_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/tree_iterator.hpp>", private, "<boost/heap/binomial_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/tree_iterator.hpp>", private, "<boost/heap/fibonacci_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/tree_iterator.hpp>", private, "<boost/heap/pairing_heap.hpp>", public ] },
+    { include: ["<boost/heap/detail/tree_iterator.hpp>", private, "<boost/heap/skew_heap.hpp>", public ] },
+    { include: ["<boost/icl/detail/boost_config.hpp>", private, "<boost/icl/gregorian.hpp>", public ] },
+    { include: ["<boost/icl/detail/boost_config.hpp>", private, "<boost/icl/impl_config.hpp>", public ] },
+    { include: ["<boost/icl/detail/boost_config.hpp>", private, "<boost/icl/ptime.hpp>", public ] },
+    { include: ["<boost/icl/detail/concept_check.hpp>", private, "<boost/icl/continuous_interval.hpp>", public ] },
+    { include: ["<boost/icl/detail/concept_check.hpp>", private, "<boost/icl/discrete_interval.hpp>", public ] },
+    { include: ["<boost/icl/detail/concept_check.hpp>", private, "<boost/icl/map.hpp>", public ] },
+    { include: ["<boost/icl/detail/design_config.hpp>", private, "<boost/icl/concept/interval.hpp>", public ] },
+    { include: ["<boost/icl/detail/design_config.hpp>", private, "<boost/icl/interval_base_map.hpp>", public ] },
+    { include: ["<boost/icl/detail/design_config.hpp>", private, "<boost/icl/interval_bounds.hpp>", public ] },
+    { include: ["<boost/icl/detail/design_config.hpp>", private, "<boost/icl/map.hpp>", public ] },
+    { include: ["<boost/icl/detail/design_config.hpp>", private, "<boost/icl/type_traits/interval_type_default.hpp>", public ] },
+    { include: ["<boost/icl/detail/element_iterator.hpp>", private, "<boost/icl/interval_base_set.hpp>", public ] },
+    { include: ["<boost/icl/detail/exclusive_less_than.hpp>", private, "<boost/icl/interval_base_set.hpp>", public ] },
+    { include: ["<boost/icl/detail/interval_map_algo.hpp>", private, "<boost/icl/concept/interval_associator.hpp>", public ] },
+    { include: ["<boost/icl/detail/interval_map_algo.hpp>", private, "<boost/icl/concept/interval_map.hpp>", public ] },
+    { include: ["<boost/icl/detail/interval_map_algo.hpp>", private, "<boost/icl/interval_base_map.hpp>", public ] },
+    { include: ["<boost/icl/detail/interval_set_algo.hpp>", private, "<boost/icl/concept/interval_associator.hpp>", public ] },
+    { include: ["<boost/icl/detail/interval_set_algo.hpp>", private, "<boost/icl/concept/interval_set.hpp>", public ] },
+    { include: ["<boost/icl/detail/interval_set_algo.hpp>", private, "<boost/icl/interval_base_set.hpp>", public ] },
+    { include: ["<boost/icl/detail/map_algo.hpp>", private, "<boost/icl/associative_element_container.hpp>", public ] },
+    { include: ["<boost/icl/detail/map_algo.hpp>", private, "<boost/icl/concept/element_map.hpp>", public ] },
+    { include: ["<boost/icl/detail/map_algo.hpp>", private, "<boost/icl/concept/interval_associator.hpp>", public ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/interval_base_map.hpp>", public ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/interval_base_set.hpp>", public ] },
+    { include: ["<boost/icl/detail/notate.hpp>", private, "<boost/icl/map.hpp>", public ] },
+    { include: ["<boost/icl/detail/on_absorbtion.hpp>", private, "<boost/icl/concept/element_map.hpp>", public ] },
+    { include: ["<boost/icl/detail/on_absorbtion.hpp>", private, "<boost/icl/interval_base_map.hpp>", public ] },
+    { include: ["<boost/icl/detail/on_absorbtion.hpp>", private, "<boost/icl/map.hpp>", public ] },
+    { include: ["<boost/icl/detail/set_algo.hpp>", private, "<boost/icl/concept/element_set.hpp>", public ] },
+    { include: ["<boost/icl/detail/set_algo.hpp>", private, "<boost/icl/concept/interval_associator.hpp>", public ] },
+    { include: ["<boost/icl/detail/set_algo.hpp>", private, "<boost/icl/concept/interval_map.hpp>", public ] },
+    { include: ["<boost/icl/detail/set_algo.hpp>", private, "<boost/icl/concept/interval_set.hpp>", public ] },
+    { include: ["<boost/icl/detail/std_set.hpp>", private, "<boost/icl/concept/element_set.hpp>", public ] },
+    { include: ["<boost/icl/detail/subset_comparer.hpp>", private, "<boost/icl/concept/element_associator.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private, "<boost/interprocess/allocators/adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private, "<boost/interprocess/allocators/cached_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/adaptive_node_pool.hpp>", private, "<boost/interprocess/allocators/private_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/allocator_common.hpp>", private, "<boost/interprocess/allocators/adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/allocator_common.hpp>", private, "<boost/interprocess/allocators/allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/allocator_common.hpp>", private, "<boost/interprocess/allocators/cached_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/allocator_common.hpp>", private, "<boost/interprocess/allocators/cached_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/allocator_common.hpp>", private, "<boost/interprocess/allocators/node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/node_pool.hpp>", private, "<boost/interprocess/allocators/cached_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/node_pool.hpp>", private, "<boost/interprocess/allocators/node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/node_pool.hpp>", private, "<boost/interprocess/allocators/private_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/node_tools.hpp>", private, "<boost/interprocess/allocators/cached_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/allocators/detail/node_tools.hpp>", private, "<boost/interprocess/allocators/cached_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/sync/spin/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/sync/spin/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/sync/spin/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/atomic.hpp>", private, "<boost/interprocess/sync/spin/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/cast_tags.hpp>", private, "<boost/interprocess/offset_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/cast_tags.hpp>", private, "<boost/interprocess/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/cached_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/cached_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/private_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/allocators/private_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/anonymous_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/allocation_type.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/containers_fwd.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/deque.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/flat_map.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/flat_set.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/list.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/map.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/pair.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/set.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/slist.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/stable_vector.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/string.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/vector.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/containers/version_type.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/creation_tags.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/errors.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/exceptions.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/file_mapping.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/indexes/flat_map_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/indexes/iset_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/indexes/iunordered_set_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/indexes/map_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/indexes/null_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/indexes/unordered_map_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/interprocess_fwd.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/ipc/message_queue.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/managed_external_buffer.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/managed_heap_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/managed_mapped_file.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/managed_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/managed_windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/managed_xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/mapped_region.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/mem_algo/rbtree_best_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/mem_algo/simple_seq_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/offset_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/permissions.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/shared_memory_object.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/deleter.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/enable_shared_from_this.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/intrusive_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/scoped_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/unique_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/smart_ptr/weak_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/streams/bufferstream.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/streams/vectorstream.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/file_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/interprocess_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/interprocess_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/interprocess_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/interprocess_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/interprocess_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/interprocess_sharable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/interprocess_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/lock_options.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/mutex_family.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/named_sharable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/named_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/null_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/posix/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/posix/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/posix/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/posix/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/posix/pthread_helpers.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/posix/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/posix/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/scoped_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/sharable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/shm/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/shm/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/shm/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/shm/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/shm/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/shm/named_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/spin/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/spin/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/spin/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/spin/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/spin/wait.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/upgradable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/named_sync.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/sync_utils.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/winapi_mutex_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/windows/winapi_semaphore_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/sync/xsi/xsi_named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/xsi_key.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_begin.hpp>", private, "<boost/interprocess/xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/cached_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/cached_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/private_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/allocators/private_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/anonymous_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/allocation_type.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/containers_fwd.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/deque.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/flat_map.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/flat_set.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/list.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/map.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/pair.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/set.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/slist.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/stable_vector.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/string.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/vector.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/containers/version_type.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/creation_tags.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/errors.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/exceptions.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/file_mapping.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/indexes/flat_map_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/indexes/iset_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/indexes/iunordered_set_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/indexes/map_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/indexes/null_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/indexes/unordered_map_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/interprocess_fwd.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/ipc/message_queue.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/managed_external_buffer.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/managed_heap_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/managed_mapped_file.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/managed_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/managed_windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/managed_xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/mapped_region.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/mem_algo/rbtree_best_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/mem_algo/simple_seq_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/offset_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/permissions.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/shared_memory_object.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/deleter.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/enable_shared_from_this.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/intrusive_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/scoped_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/unique_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/smart_ptr/weak_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/streams/bufferstream.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/streams/vectorstream.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/file_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/interprocess_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/interprocess_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/interprocess_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/interprocess_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/interprocess_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/interprocess_sharable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/interprocess_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/lock_options.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/mutex_family.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/named_sharable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/named_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/null_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/posix/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/posix/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/posix/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/posix/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/posix/pthread_helpers.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/posix/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/posix/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/scoped_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/sharable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/shm/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/shm/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/shm/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/shm/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/shm/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/shm/named_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/spin/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/spin/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/spin/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/spin/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/spin/wait.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/upgradable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/named_sync.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/sync_utils.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/winapi_mutex_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/windows/winapi_semaphore_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/sync/xsi/xsi_named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/xsi_key.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_end.hpp>", private, "<boost/interprocess/xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_external_begin.hpp>", private, "<boost/interprocess/sync/windows/sync_utils.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/config_external_end.hpp>", private, "<boost/interprocess/sync/windows/sync_utils.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/file_wrapper.hpp>", private, "<boost/interprocess/managed_mapped_file.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/intermodule_singleton.hpp>", private, "<boost/flyweight/intermodule_holder.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/posix/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/shm/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/shm/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/windows/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/windows/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/interprocess_tester.hpp>", private, "<boost/interprocess/sync/windows/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_memory_impl.hpp>", private, "<boost/interprocess/managed_external_buffer.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_memory_impl.hpp>", private, "<boost/interprocess/managed_heap_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_memory_impl.hpp>", private, "<boost/interprocess/managed_mapped_file.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_memory_impl.hpp>", private, "<boost/interprocess/managed_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_memory_impl.hpp>", private, "<boost/interprocess/managed_windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_memory_impl.hpp>", private, "<boost/interprocess/managed_xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/ipc/message_queue.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/managed_mapped_file.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/managed_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/managed_windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/managed_xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/sync/named_sharable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/sync/named_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/sync/shm/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/sync/shm/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/sync/shm/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/sync/shm/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/sync/shm/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/managed_open_or_create_impl.hpp>", private, "<boost/interprocess/sync/shm/named_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/math_functions.hpp>", private, "<boost/interprocess/mem_algo/rbtree_best_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/min_max.hpp>", private, "<boost/interprocess/mem_algo/rbtree_best_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/allocators/adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/offset_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/smart_ptr/unique_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/sync/scoped_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/sync/sharable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/sync/shm/named_creation_functor.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/mpl.hpp>", private, "<boost/interprocess/sync/upgradable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/named_proxy.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/file_mapping.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/mapped_region.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/shared_memory_object.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/sync/file_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/sync/posix/semaphore_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/sync/xsi/xsi_named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/xsi_key.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_file_functions.hpp>", private, "<boost/interprocess/xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/file_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/posix/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/posix/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/posix/semaphore_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/spin/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/spin/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/spin/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/spin/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/os_thread_functions.hpp>", private, "<boost/interprocess/sync/spin/wait.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/pointer_type.hpp>", private, "<boost/interprocess/smart_ptr/scoped_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/pointer_type.hpp>", private, "<boost/interprocess/smart_ptr/unique_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/file_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/interprocess_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/interprocess_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/interprocess_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/interprocess_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/interprocess_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/interprocess_sharable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/interprocess_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/named_sharable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/named_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/posix/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/posix/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/posix/ptime_to_timespec.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/posix/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/posix/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/posix/semaphore_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/scoped_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/sharable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/shm/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/shm/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/shm/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/shm/named_recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/shm/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/shm/named_upgradable_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/spin/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/spin/interprocess_barrier.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/spin/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/spin/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/spin/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/upgradable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/windows/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/windows/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/windows/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/windows/named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/windows/named_semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/windows/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/windows/winapi_mutex_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/posix_time_types_wrk.hpp>", private, "<boost/interprocess/sync/windows/winapi_semaphore_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/segment_manager_helper.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/tmp_dir_helpers.hpp>", private, "<boost/interprocess/shared_memory_object.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/tmp_dir_helpers.hpp>", private, "<boost/interprocess/sync/posix/semaphore_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/tmp_dir_helpers.hpp>", private, "<boost/interprocess/sync/windows/named_sync.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/transform_iterator.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/allocators/adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/allocators/allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/allocators/node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/ipc/message_queue.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/mem_algo/rbtree_best_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/smart_ptr/unique_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/sync/scoped_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/sync/sharable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/sync/shm/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/sync/shm/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/sync/shm/named_creation_functor.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/type_traits.hpp>", private, "<boost/interprocess/sync/upgradable_lock.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/cached_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/cached_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/private_adaptive_pool.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/allocators/private_node_allocator.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/file_mapping.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/indexes/iset_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/indexes/iunordered_set_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/indexes/unordered_map_index.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/ipc/message_queue.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/mapped_region.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/mem_algo/rbtree_best_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/offset_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/segment_manager.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/smart_ptr/deleter.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/smart_ptr/intrusive_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/smart_ptr/scoped_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/smart_ptr/unique_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/sync/xsi/xsi_named_mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/xsi_key.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/utilities.hpp>", private, "<boost/interprocess/xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/errors.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/mapped_region.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/permissions.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/sync/windows/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/sync/windows/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/sync/windows/sync_utils.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/sync/windows/winapi_mutex_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/sync/windows/winapi_semaphore_wrapper.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/win32_api.hpp>", private, "<boost/interprocess/windows_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private, "<boost/interprocess/mapped_region.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private, "<boost/interprocess/sync/windows/mutex.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/windows_intermodule_singleton.hpp>", private, "<boost/interprocess/sync/windows/semaphore.hpp>", public ] },
+    { include: ["<boost/interprocess/detail/xsi_shared_memory_file_wrapper.hpp>", private, "<boost/interprocess/managed_xsi_shared_memory.hpp>", public ] },
+    { include: ["<boost/interprocess/mem_algo/detail/mem_algo_common.hpp>", private, "<boost/interprocess/mem_algo/rbtree_best_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/mem_algo/detail/simple_seq_fit_impl.hpp>", private, "<boost/interprocess/mem_algo/simple_seq_fit.hpp>", public ] },
+    { include: ["<boost/interprocess/smart_ptr/detail/shared_count.hpp>", private, "<boost/interprocess/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/condition_algorithm_8a.hpp>", private, "<boost/interprocess/sync/windows/condition.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/condition_algorithm_8a.hpp>", private, "<boost/interprocess/sync/windows/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/condition_any_algorithm.hpp>", private, "<boost/interprocess/sync/interprocess_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/condition_any_algorithm.hpp>", private, "<boost/interprocess/sync/shm/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/condition_any_algorithm.hpp>", private, "<boost/interprocess/sync/shm/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/locks.hpp>", private, "<boost/interprocess/sync/interprocess_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/locks.hpp>", private, "<boost/interprocess/sync/named_condition_any.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/locks.hpp>", private, "<boost/interprocess/sync/named_condition.hpp>", public ] },
+    { include: ["<boost/interprocess/sync/detail/locks.hpp>", private, "<boost/interprocess/sync/shm/named_condition.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/any_node_and_algorithms.hpp>", private, "<boost/intrusive/any_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/avltree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/avltree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/bstree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/circular_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/list.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/pointer_plus_bits.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/rbtree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/sgtree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/sgtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/slist.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/splaytree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/splaytree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/treap_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/assert.hpp>", private, "<boost/intrusive/treap.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/avltree_node.hpp>", private, "<boost/intrusive/avl_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/avltree_node.hpp>", private, "<boost/intrusive/avltree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/clear_on_destructor_base.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/clear_on_destructor_base.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/clear_on_destructor_base.hpp>", private, "<boost/intrusive/list.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/clear_on_destructor_base.hpp>", private, "<boost/intrusive/slist.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/common_slist_algorithms.hpp>", private, "<boost/intrusive/circular_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/common_slist_algorithms.hpp>", private, "<boost/intrusive/linear_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/any_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/avl_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/avl_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/avltree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/avltree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/bs_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/bs_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/bstree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/circular_list_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/circular_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/linear_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/list_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/list.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/options.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/parent_from_member.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/pointer_traits.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/priority_compare.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/rbtree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/sg_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/sgtree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/sgtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/slist_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/slist.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/splay_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/splay_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/splaytree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/splaytree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/treap_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/treap.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/treap_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/unordered_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_begin.hpp>", private, "<boost/intrusive/unordered_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/any_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/avl_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/avl_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/avltree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/avltree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/bs_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/bs_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/bstree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/circular_list_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/circular_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/linear_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/list_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/list.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/options.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/parent_from_member.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/pointer_traits.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/priority_compare.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/rbtree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/sg_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/sgtree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/sgtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/slist_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/slist.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/splay_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/splay_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/splaytree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/splaytree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/treap_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/treap.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/treap_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/unordered_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/config_end.hpp>", private, "<boost/intrusive/unordered_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/ebo_functor_holder.hpp>", private, "<boost/intrusive/avltree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/ebo_functor_holder.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/ebo_functor_holder.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/ebo_functor_holder.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/ebo_functor_holder.hpp>", private, "<boost/intrusive/sgtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/ebo_functor_holder.hpp>", private, "<boost/intrusive/splaytree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/ebo_functor_holder.hpp>", private, "<boost/intrusive/treap.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/function_detector.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/function_detector.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/function_detector.hpp>", private, "<boost/intrusive/splaytree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/generic_hook.hpp>", private, "<boost/intrusive/any_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/generic_hook.hpp>", private, "<boost/intrusive/avl_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/generic_hook.hpp>", private, "<boost/intrusive/bs_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/generic_hook.hpp>", private, "<boost/intrusive/list_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/generic_hook.hpp>", private, "<boost/intrusive/set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/generic_hook.hpp>", private, "<boost/intrusive/slist_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/generic_hook.hpp>", private, "<boost/intrusive/unordered_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/hashtable_node.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/list_node.hpp>", private, "<boost/intrusive/list_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/memory_util.hpp>", private, "<boost/container/allocator_traits.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/memory_util.hpp>", private, "<boost/intrusive/pointer_traits.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/avl_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/avltree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/bs_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/list.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/options.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/pointer_plus_bits.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/sg_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/sgtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/splay_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/splaytree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/treap.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/mpl.hpp>", private, "<boost/intrusive/treap_set.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/parent_from_member.hpp>", private, "<boost/intrusive/member_value_traits.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/parent_from_member.hpp>", private, "<boost/intrusive/parent_from_member.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/rbtree_node.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/rbtree_node.hpp>", private, "<boost/intrusive/set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/slist_node.hpp>", private, "<boost/intrusive/slist_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/transform_iterator.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/tree_node.hpp>", private, "<boost/intrusive/avltree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/tree_node.hpp>", private, "<boost/intrusive/bs_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/tree_node.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/tree_node.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/tree_node.hpp>", private, "<boost/intrusive/sgtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/tree_node.hpp>", private, "<boost/intrusive/splaytree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/tree_node.hpp>", private, "<boost/intrusive/treap.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/any_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/avl_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/avltree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/avltree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/bs_set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/bstree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/bstree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/circular_list_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/circular_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/hashtable.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/linear_slist_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/list_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/list.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/options.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/rbtree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/rbtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/set_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/sgtree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/sgtree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/slist_hook.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/slist.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/splaytree_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/splaytree.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/treap_algorithms.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/treap.hpp>", public ] },
+    { include: ["<boost/intrusive/detail/utilities.hpp>", private, "<boost/intrusive/unordered_set_hook.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/access_control.hpp>", private, "<boost/iostreams/chain.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/access_control.hpp>", private, "<boost/iostreams/filtering_streambuf.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/access_control.hpp>", private, "<boost/iostreams/filtering_stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/concept_adapter.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/device_adapter.hpp>", private, "<boost/iostreams/tee.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/direct_adapter.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/direct_adapter.hpp>", private, "<boost/iostreams/compose.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/filter_adapter.hpp>", private, "<boost/iostreams/tee.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/non_blocking_adapter.hpp>", private, "<boost/iostreams/close.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/non_blocking_adapter.hpp>", private, "<boost/iostreams/copy.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/non_blocking_adapter.hpp>", private, "<boost/iostreams/filter/gzip.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/adapter/range_adapter.hpp>", private, "<boost/iostreams/filter/gzip.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/bool_trait_def.hpp>", private, "<boost/iostreams/filter/test.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/bool_trait_def.hpp>", private, "<boost/iostreams/traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/broken_overload_resolution/stream_buffer.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/broken_overload_resolution/stream.hpp>", private, "<boost/iostreams/stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/buffer.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/buffer.hpp>", private, "<boost/iostreams/copy.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/buffer.hpp>", private, "<boost/iostreams/filter/symmetric.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/buffer.hpp>", private, "<boost/iostreams/invert.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/call_traits.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/call_traits.hpp>", private, "<boost/iostreams/compose.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/call_traits.hpp>", private, "<boost/iostreams/tee.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/chain.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/char_traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/filter/aggregate.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/filter/gzip.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/filtering_streambuf.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/filtering_stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/filter/newline.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/filter/stdio.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/filter/symmetric.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/read.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/char_traits.hpp>", private, "<boost/iostreams/write.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/codecvt_helper.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/codecvt_holder.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/auto_link.hpp>", private, "<boost/iostreams/device/file_descriptor.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/auto_link.hpp>", private, "<boost/iostreams/device/mapped_file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/auto_link.hpp>", private, "<boost/iostreams/filter/bzip2.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/auto_link.hpp>", private, "<boost/iostreams/filter/zlib.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/bzip2.hpp>", private, "<boost/iostreams/filter/bzip2.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/codecvt.hpp>", private, "<boost/iostreams/positioning.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/checked_operations.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/close.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/combine.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/compose.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/device/file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/filter/aggregate.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/filter/counter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/filtering_stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/filter/line.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/filter/newline.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/filter/symmetric.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/flush.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/imbue.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/input_sequence.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/invert.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/optimal_buffer_size.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/output_sequence.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/positioning.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/read.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/seek.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/disable_warnings.hpp>", private, "<boost/iostreams/write.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/dyn_link.hpp>", private, "<boost/iostreams/device/file_descriptor.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/dyn_link.hpp>", private, "<boost/iostreams/device/mapped_file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/dyn_link.hpp>", private, "<boost/iostreams/filter/bzip2.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/dyn_link.hpp>", private, "<boost/iostreams/filter/zlib.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/checked_operations.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/close.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/combine.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/compose.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/device/file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/filter/aggregate.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/filter/counter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/filtering_stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/filter/line.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/filter/newline.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/filter/symmetric.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/flush.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/imbue.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/input_sequence.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/invert.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/optimal_buffer_size.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/output_sequence.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/positioning.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/read.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/seek.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/enable_warnings.hpp>", private, "<boost/iostreams/write.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/fpos.hpp>", private, "<boost/iostreams/positioning.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/limits.hpp>", private, "<boost/iostreams/filter/symmetric.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/overload_resolution.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/overload_resolution.hpp>", private, "<boost/iostreams/stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/unreachable_return.hpp>", private, "<boost/iostreams/checked_operations.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/char_traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/device/file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/device/mapped_file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/filter/bzip2.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/filter/stdio.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/filter/zlib.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/wide_streams.hpp>", private, "<boost/iostreams/traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/windows_posix.hpp>", private, "<boost/iostreams/device/file_descriptor.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/config/zlib.hpp>", private, "<boost/iostreams/filter/zlib.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/counted_array.hpp>", private, "<boost/iostreams/invert.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/default_arg.hpp>", private, "<boost/iostreams/concepts.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/dispatch.hpp>", private, "<boost/iostreams/checked_operations.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/dispatch.hpp>", private, "<boost/iostreams/flush.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/dispatch.hpp>", private, "<boost/iostreams/imbue.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/dispatch.hpp>", private, "<boost/iostreams/optimal_buffer_size.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/dispatch.hpp>", private, "<boost/iostreams/read.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/dispatch.hpp>", private, "<boost/iostreams/seek.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/dispatch.hpp>", private, "<boost/iostreams/write.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/double_object.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/enable_if_stream.hpp>", private, "<boost/iostreams/compose.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/enable_if_stream.hpp>", private, "<boost/iostreams/copy.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/error.hpp>", private, "<boost/iostreams/checked_operations.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/error.hpp>", private, "<boost/iostreams/filter/gzip.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/execute.hpp>", private, "<boost/iostreams/chain.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/execute.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/execute.hpp>", private, "<boost/iostreams/compose.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/execute.hpp>", private, "<boost/iostreams/copy.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/execute.hpp>", private, "<boost/iostreams/invert.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/execute.hpp>", private, "<boost/iostreams/tee.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/file_handle.hpp>", private, "<boost/iostreams/device/file_descriptor.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/forward.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/forward.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/forward.hpp>", private, "<boost/iostreams/stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/fstream.hpp>", private, "<boost/iostreams/device/file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/functional.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/functional.hpp>", private, "<boost/iostreams/compose.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/functional.hpp>", private, "<boost/iostreams/copy.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/functional.hpp>", private, "<boost/iostreams/invert.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/functional.hpp>", private, "<boost/iostreams/tee.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/close.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/combine.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/concepts.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/constants.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/copy.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/device/back_inserter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/device/file_descriptor.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/device/file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/device/mapped_file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/device/null.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/filter/aggregate.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/filter/bzip2.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/filter/gzip.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/filter/line.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/filter/newline.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/filter/stdio.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/filter/test.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/filter/zlib.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/positioning.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/read.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/seek.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/skip.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/ios.hpp>", private, "<boost/iostreams/write.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/iostream.hpp>", private, "<boost/iostreams/filtering_stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/iostream.hpp>", private, "<boost/iostreams/stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/is_iterator_range.hpp>", private, "<boost/iostreams/traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/optional.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/path.hpp>", private, "<boost/iostreams/device/file_descriptor.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/path.hpp>", private, "<boost/iostreams/device/mapped_file.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/push.hpp>", private, "<boost/iostreams/chain.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/push.hpp>", private, "<boost/iostreams/filtering_streambuf.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/push.hpp>", private, "<boost/iostreams/filtering_stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/resolve.hpp>", private, "<boost/iostreams/copy.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/restrict_impl.hpp>", private, "<boost/iostreams/restrict.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/restrict_impl.hpp>", private, "<boost/iostreams/slice.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/select_by_size.hpp>", private, "<boost/iostreams/traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/close.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/code_converter.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/filtering_stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/select.hpp>", private, "<boost/iostreams/traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf/chainbuf.hpp>", private, "<boost/iostreams/filtering_streambuf.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf/direct_streambuf.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/chain.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/filtering_streambuf.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/filtering_stream.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/flush.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/imbue.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/read.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/seek.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf.hpp>", private, "<boost/iostreams/write.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/streambuf/indirect_streambuf.hpp>", private, "<boost/iostreams/stream_buffer.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/template_params.hpp>", private, "<boost/iostreams/filter/symmetric.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/template_params.hpp>", private, "<boost/iostreams/pipeline.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/vc6/close.hpp>", private, "<boost/iostreams/close.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/vc6/read.hpp>", private, "<boost/iostreams/read.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/vc6/write.hpp>", private, "<boost/iostreams/write.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/chain.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/close.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/combine.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/copy.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/flush.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/imbue.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/input_sequence.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/optimal_buffer_size.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/output_sequence.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/read.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/seek.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/traits.hpp>", public ] },
+    { include: ["<boost/iostreams/detail/wrap_unwrap.hpp>", private, "<boost/iostreams/write.hpp>", public ] },
+    { include: ["<boost/iterator/detail/any_conversion_eater.hpp>", private, "<boost/iterator/is_lvalue_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/any_conversion_eater.hpp>", private, "<boost/iterator/is_readable_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/indirect_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/interoperable.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/is_lvalue_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/is_readable_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/iterator_adaptor.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/iterator_categories.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/iterator_facade.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/new_iterator_tests.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/iterator/transform_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_def.hpp>", private, "<boost/python/object_operators.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/indirect_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/interoperable.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/is_lvalue_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/is_readable_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/iterator_adaptor.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/iterator_categories.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/iterator_facade.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/new_iterator_tests.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/iterator/transform_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/config_undef.hpp>", private, "<boost/python/object_operators.hpp>", public ] },
+    { include: ["<boost/iterator/detail/enable_if.hpp>", private, "<boost/iterator/iterator_adaptor.hpp>", public ] },
+    { include: ["<boost/iterator/detail/enable_if.hpp>", private, "<boost/iterator/iterator_facade.hpp>", public ] },
+    { include: ["<boost/iterator/detail/enable_if.hpp>", private, "<boost/iterator/transform_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/enable_if.hpp>", private, "<boost/python/object_operators.hpp>", public ] },
+    { include: ["<boost/iterator/detail/facade_iterator_category.hpp>", private, "<boost/iterator/iterator_archetypes.hpp>", public ] },
+    { include: ["<boost/iterator/detail/facade_iterator_category.hpp>", private, "<boost/iterator/iterator_facade.hpp>", public ] },
+    { include: ["<boost/iterator/detail/minimum_category.hpp>", private, "<boost/iterator/zip_iterator.hpp>", public ] },
+    { include: ["<boost/iterator/detail/minimum_category.hpp>", private, "<boost/token_iterator.hpp>", public ] },
+    { include: ["<boost/lambda/detail/actions.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/arity_code.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/bind_functions.hpp>", private, "<boost/lambda/bind.hpp>", public ] },
+    { include: ["<boost/lambda/detail/control_constructs_common.hpp>", private, "<boost/lambda/exceptions.hpp>", public ] },
+    { include: ["<boost/lambda/detail/control_constructs_common.hpp>", private, "<boost/lambda/switch.hpp>", public ] },
+    { include: ["<boost/lambda/detail/function_adaptors.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/lambda_config.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/lambda_functor_base.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/lambda_functors.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/lambda_fwd.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/lambda_traits.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/member_ptr.hpp>", private, "<boost/lambda/lambda.hpp>", public ] },
+    { include: ["<boost/lambda/detail/operator_actions.hpp>", private, "<boost/lambda/control_structures.hpp>", public ] },
+    { include: ["<boost/lambda/detail/operator_actions.hpp>", private, "<boost/lambda/if.hpp>", public ] },
+    { include: ["<boost/lambda/detail/operator_actions.hpp>", private, "<boost/lambda/lambda.hpp>", public ] },
+    { include: ["<boost/lambda/detail/operator_lambda_func_base.hpp>", private, "<boost/lambda/lambda.hpp>", public ] },
+    { include: ["<boost/lambda/detail/operator_return_type_traits.hpp>", private, "<boost/lambda/control_structures.hpp>", public ] },
+    { include: ["<boost/lambda/detail/operator_return_type_traits.hpp>", private, "<boost/lambda/if.hpp>", public ] },
+    { include: ["<boost/lambda/detail/operator_return_type_traits.hpp>", private, "<boost/lambda/lambda.hpp>", public ] },
+    { include: ["<boost/lambda/detail/operators.hpp>", private, "<boost/lambda/lambda.hpp>", public ] },
+    { include: ["<boost/lambda/detail/ret.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/return_type_traits.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/select_functions.hpp>", private, "<boost/lambda/core.hpp>", public ] },
+    { include: ["<boost/lambda/detail/suppress_unused.hpp>", private, "<boost/lambda/casts.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/auto.hpp>", private, "<boost/local_function/aux_/macro/code_/functor.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/bind.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/sign.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/bind.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/validate_/defaults.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/bind.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/validate_/this.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/const_bind.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/any_bind_type.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/const_bind.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/sign.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/const_bind.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/validate_/defaults.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/const_bind.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/validate_/this.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/const.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/validate_/this.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/default.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_params.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/default.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/sign.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/default.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/validate_/defaults.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/inline.hpp>", private, "<boost/local_function/aux_/macro/name.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/recursive.hpp>", private, "<boost/local_function/aux_/macro/name.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/register.hpp>", private, "<boost/local_function/aux_/macro/code_/functor.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/return.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_/append.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/return.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/sign.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/this.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/any_bind_type.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/this.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/validate_/this.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/thisunderscore.hpp>", private, "<boost/local_function/aux_/macro/code_/functor.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/thisunderscore.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/sign.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/keyword/thisunderscore.hpp>", private, "<boost/scope_exit.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/line_counter.hpp>", private, "<boost/local_function.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/line_counter.hpp>", private, "<boost/scope_exit.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/void_list.hpp>", private, "<boost/local_function.hpp>", public ] },
+    { include: ["<boost/local_function/detail/preprocessor/void_list.hpp>", private, "<boost/scope_exit.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/atomic.hpp>", private, "<boost/lockfree/queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/atomic.hpp>", private, "<boost/lockfree/spsc_queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/atomic.hpp>", private, "<boost/lockfree/stack.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/branch_hints.hpp>", private, "<boost/lockfree/spsc_queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/copy_payload.hpp>", private, "<boost/lockfree/queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/copy_payload.hpp>", private, "<boost/lockfree/stack.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/freelist.hpp>", private, "<boost/lockfree/queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/freelist.hpp>", private, "<boost/lockfree/stack.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/parameter.hpp>", private, "<boost/lockfree/queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/parameter.hpp>", private, "<boost/lockfree/spsc_queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/parameter.hpp>", private, "<boost/lockfree/stack.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/prefix.hpp>", private, "<boost/lockfree/spsc_queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/tagged_ptr.hpp>", private, "<boost/lockfree/queue.hpp>", public ] },
+    { include: ["<boost/lockfree/detail/tagged_ptr.hpp>", private, "<boost/lockfree/stack.hpp>", public ] },
+    { include: ["<boost/log/detail/asio_fwd.hpp>", private, "<boost/log/sinks/syslog_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/attachable_sstream_buf.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/attachable_sstream_buf.hpp>", private, "<boost/log/utility/formatting_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/attribute_get_value_impl.hpp>", private, "<boost/log/attributes/attribute.hpp>", public ] },
+    { include: ["<boost/log/detail/attribute_get_value_impl.hpp>", private, "<boost/log/attributes/attribute_value.hpp>", public ] },
+    { include: ["<boost/log/detail/attribute_predicate.hpp>", private, "<boost/log/expressions/predicates/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/attribute_predicate.hpp>", private, "<boost/log/expressions/predicates/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/attribute_predicate.hpp>", private, "<boost/log/expressions/predicates/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/attribute_predicate.hpp>", private, "<boost/log/expressions/predicates/is_in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/attribute_predicate.hpp>", private, "<boost/log/expressions/predicates/matches.hpp>", public ] },
+    { include: ["<boost/log/detail/attr_output_impl.hpp>", private, "<boost/log/expressions/attr.hpp>", public ] },
+    { include: ["<boost/log/detail/attr_output_impl.hpp>", private, "<boost/log/expressions/formatters/stream.hpp>", public ] },
+    { include: ["<boost/log/detail/attr_output_terminal.hpp>", private, "<boost/log/expressions/formatters/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/attr_output_terminal.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/cleanup_scope_guard.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/cleanup_scope_guard.hpp>", private, "<boost/log/sinks/text_multifile_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/code_conversion.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/code_conversion.hpp>", private, "<boost/log/utility/formatting_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/code_conversion.hpp>", private, "<boost/log/utility/setup/filter_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/attribute_cast.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/attribute.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/attribute_name.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/attribute_set.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/attribute_value.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/attribute_value_impl.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/attribute_value_set.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/clock.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/constant.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/counter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/current_process_id.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/current_process_name.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/current_thread_id.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/fallback_policy_fwd.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/fallback_policy.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/function.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/mutable_constant.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/scoped_attribute.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/timer.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/time_traits.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/value_extraction_fwd.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/value_extraction.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/value_visitation_fwd.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/attributes/value_visitation.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/common.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/core/core.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/core.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/core/record.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/core/record_view.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/exceptions.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/attr_fwd.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/attr.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/filter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/c_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/char_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/csv_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/format.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/if.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/stream.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/wrap_formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/formatters/xml_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/is_keyword_descriptor.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/keyword_fwd.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/keyword.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/message.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates/channel_severity_filter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates/has_attr.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates/is_debugger_present.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates/is_in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/predicates/matches.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/expressions/record.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/auto_flush.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/channel.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/delimiter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/depth.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/facility.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/file_name.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/filter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/format.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/ident.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/ip_version.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/iteration.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/log_name.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/log_source.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/max_size.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/message_file.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/min_free_space.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/open_mode.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/order.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/ordering_window.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/registration.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/rotation_size.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/scan_method.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/severity.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/start_thread.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/target.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/time_based_rotation.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/keywords/use_impl.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/async_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/attribute_mapping.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/basic_sink_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/block_on_overflow.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/bounded_fifo_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/bounded_ordering_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/debug_output_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/drop_on_overflow.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/event_log_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/event_log_constants.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/frontend_requirements.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/sink.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/sync_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/syslog_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/syslog_constants.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/text_file_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/text_multifile_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/text_ostream_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/unbounded_fifo_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/unbounded_ordering_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sinks/unlocked_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/basic_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/channel_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/channel_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/exception_handler_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/features.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/global_logger_storage.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/logger.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/record_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/severity_channel_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/severity_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/severity_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/sources/threading_models.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/support/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/support/exception.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/support/regex.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/support/spirit_classic.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/support/spirit_qi.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/support/xpressive.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/trivial.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/empty_deleter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/exception_handler.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/explicit_operator_bool.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/formatting_ostream_fwd.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/formatting_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/as_action.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/bind_assign.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/bind.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/bind_output.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/bind_to_log.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/fun_ref.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/logical.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/matches.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/nop.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/functional/save_result.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/intrusive_ref_counter.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/manipulators/add_value.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/manipulators/dump.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/manipulators.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/manipulators/to_log.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/once_block.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/record_ordering.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/setup/common_attributes.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/setup/console.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/setup/file.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/strictest_lock.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/string_literal_fwd.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/string_literal.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/type_dispatch/date_time_types.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/type_dispatch/dynamic_type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/type_dispatch/standard_types.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/type_dispatch/static_type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/type_dispatch/type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/type_info_wrapper.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/unique_identifier_name.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/unused_variable.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/value_ref_fwd.hpp>", public ] },
+    { include: ["<boost/log/detail/config.hpp>", private, "<boost/log/utility/value_ref.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/attr.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/formatters/char_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/formatters/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/formatters/format.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/formatters/if.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/formatters/wrap_formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/keyword.hpp>", public ] },
+    { include: ["<boost/log/detail/custom_terminal_spec.hpp>", private, "<boost/log/expressions/predicates/channel_severity_filter.hpp>", public ] },
+    { include: ["<boost/log/detail/date_time_fmt_gen_traits_fwd.hpp>", private, "<boost/log/expressions/formatters/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/date_time_fmt_gen_traits_fwd.hpp>", private, "<boost/log/support/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/date_time_format_parser.hpp>", private, "<boost/log/support/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/decomposed_time.hpp>", private, "<boost/log/support/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/deduce_char_type.hpp>", private, "<boost/log/expressions/formatters/char_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/deduce_char_type.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/default_attribute_names.hpp>", private, "<boost/log/expressions/message.hpp>", public ] },
+    { include: ["<boost/log/detail/default_attribute_names.hpp>", private, "<boost/log/sources/channel_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/default_attribute_names.hpp>", private, "<boost/log/sources/severity_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/default_attribute_names.hpp>", private, "<boost/log/utility/setup/common_attributes.hpp>", public ] },
+    { include: ["<boost/log/detail/embedded_string_type.hpp>", private, "<boost/log/attributes/constant.hpp>", public ] },
+    { include: ["<boost/log/detail/embedded_string_type.hpp>", private, "<boost/log/expressions/predicates/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/embedded_string_type.hpp>", private, "<boost/log/expressions/predicates/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/embedded_string_type.hpp>", private, "<boost/log/expressions/predicates/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/embedded_string_type.hpp>", private, "<boost/log/expressions/predicates/is_in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/embedded_string_type.hpp>", private, "<boost/log/utility/manipulators/add_value.hpp>", public ] },
+    { include: ["<boost/log/detail/event.hpp>", private, "<boost/log/sinks/unbounded_fifo_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/fake_mutex.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/fake_mutex.hpp>", private, "<boost/log/sinks/unlocked_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/attribute_cast.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/attribute.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/attribute_name.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/attribute_set.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/attribute_value.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/attribute_value_impl.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/attribute_value_set.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/clock.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/constant.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/counter.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/current_process_id.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/current_process_name.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/current_thread_id.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/fallback_policy.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/function.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/mutable_constant.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/scoped_attribute.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/timer.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/time_traits.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/value_extraction.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/attributes/value_visitation.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/core/core.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/core/record.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/core/record_view.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/exceptions.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/attr.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/filter.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/c_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/char_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/csv_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/format.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/if.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/stream.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/wrap_formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/formatters/xml_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/is_keyword_descriptor.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/keyword.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/message.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/predicates/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/predicates/channel_severity_filter.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/predicates/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/predicates/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/predicates/has_attr.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/predicates/is_debugger_present.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/predicates/is_in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/predicates/matches.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/expressions/record.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/async_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/attribute_mapping.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/basic_sink_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/block_on_overflow.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/bounded_fifo_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/bounded_ordering_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/debug_output_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/drop_on_overflow.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/event_log_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/event_log_constants.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/frontend_requirements.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/sink.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/sync_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/syslog_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/syslog_constants.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/text_file_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/text_multifile_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/text_ostream_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/unbounded_fifo_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/unbounded_ordering_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sinks/unlocked_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/basic_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/channel_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/channel_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/exception_handler_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/features.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/global_logger_storage.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/logger.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/record_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/severity_channel_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/severity_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/severity_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/sources/threading_models.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/support/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/support/exception.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/support/regex.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/support/spirit_classic.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/support/spirit_qi.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/support/xpressive.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/trivial.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/exception_handler.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/formatting_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/as_action.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/bind_assign.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/bind.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/bind_output.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/bind_to_log.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/fun_ref.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/logical.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/matches.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/nop.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/functional/save_result.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/intrusive_ref_counter.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/manipulators/add_value.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/manipulators/dump.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/manipulators/to_log.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/once_block.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/record_ordering.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/common_attributes.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/console.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/file.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/filter_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/formatter_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/from_settings.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/from_stream.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/settings.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/setup/settings_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/strictest_lock.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/string_literal.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/type_dispatch/date_time_types.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/type_dispatch/dynamic_type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/type_dispatch/standard_types.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/type_dispatch/static_type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/type_dispatch/type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/type_info_wrapper.hpp>", public ] },
+    { include: ["<boost/log/detail/footer.hpp>", private, "<boost/log/utility/value_ref.hpp>", public ] },
+    { include: ["<boost/log/detail/format.hpp>", private, "<boost/log/expressions/formatters/format.hpp>", public ] },
+    { include: ["<boost/log/detail/function_traits.hpp>", private, "<boost/log/expressions/formatters/wrap_formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/function_traits.hpp>", private, "<boost/log/utility/record_ordering.hpp>", public ] },
+    { include: ["<boost/log/detail/generate_overloads.hpp>", private, "<boost/log/expressions/formatters/char_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/generate_overloads.hpp>", private, "<boost/log/expressions/formatters/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/generate_overloads.hpp>", private, "<boost/log/expressions/formatters/if.hpp>", public ] },
+    { include: ["<boost/log/detail/generate_overloads.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/generate_overloads.hpp>", private, "<boost/log/expressions/formatters/wrap_formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/attribute_cast.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/attribute.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/attribute_name.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/attribute_set.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/attribute_value.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/attribute_value_impl.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/attribute_value_set.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/clock.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/constant.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/counter.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/current_process_id.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/current_process_name.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/current_thread_id.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/fallback_policy.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/function.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/mutable_constant.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/scoped_attribute.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/timer.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/time_traits.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/value_extraction.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/attributes/value_visitation.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/core/core.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/core/record.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/core/record_view.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/exceptions.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/attr.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/filter.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/c_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/char_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/csv_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/format.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/if.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/stream.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/wrap_formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/formatters/xml_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/is_keyword_descriptor.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/keyword.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/message.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/predicates/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/predicates/channel_severity_filter.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/predicates/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/predicates/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/predicates/has_attr.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/predicates/is_debugger_present.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/predicates/is_in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/predicates/matches.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/expressions/record.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/async_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/attribute_mapping.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/basic_sink_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/block_on_overflow.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/bounded_fifo_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/bounded_ordering_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/debug_output_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/drop_on_overflow.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/event_log_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/event_log_constants.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/frontend_requirements.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/sink.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/sync_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/syslog_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/syslog_constants.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/text_file_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/text_multifile_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/text_ostream_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/unbounded_fifo_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/unbounded_ordering_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sinks/unlocked_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/basic_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/channel_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/channel_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/exception_handler_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/features.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/global_logger_storage.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/logger.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/record_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/severity_channel_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/severity_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/severity_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/sources/threading_models.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/support/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/support/exception.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/support/regex.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/support/spirit_classic.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/support/spirit_qi.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/support/xpressive.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/trivial.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/exception_handler.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/formatting_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/as_action.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/bind_assign.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/bind.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/bind_output.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/bind_to_log.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/fun_ref.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/logical.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/matches.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/nop.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/functional/save_result.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/intrusive_ref_counter.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/manipulators/add_value.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/manipulators/dump.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/manipulators/to_log.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/once_block.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/record_ordering.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/common_attributes.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/console.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/file.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/filter_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/formatter_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/from_settings.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/from_stream.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/settings.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/setup/settings_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/strictest_lock.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/string_literal.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/type_dispatch/date_time_types.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/type_dispatch/dynamic_type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/type_dispatch/standard_types.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/type_dispatch/static_type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/type_dispatch/type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/type_info_wrapper.hpp>", public ] },
+    { include: ["<boost/log/detail/header.hpp>", private, "<boost/log/utility/value_ref.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/core/core.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/expressions/filter.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/expressions/formatter.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/expressions/formatters/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/sinks/event_log_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/sinks/sink.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/sinks/syslog_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/sinks/text_file_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/sinks/text_multifile_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/sources/exception_handler_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/light_function.hpp>", private, "<boost/log/support/date_time.hpp>", public ] },
+    { include: ["<boost/log/detail/light_rw_mutex.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/light_rw_mutex.hpp>", private, "<boost/log/sources/channel_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/light_rw_mutex.hpp>", private, "<boost/log/sources/logger.hpp>", public ] },
+    { include: ["<boost/log/detail/light_rw_mutex.hpp>", private, "<boost/log/sources/severity_channel_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/light_rw_mutex.hpp>", private, "<boost/log/sources/severity_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/locking_ptr.hpp>", private, "<boost/log/sinks/async_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/locking_ptr.hpp>", private, "<boost/log/sinks/sync_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/locks.hpp>", private, "<boost/log/attributes/mutable_constant.hpp>", public ] },
+    { include: ["<boost/log/detail/locks.hpp>", private, "<boost/log/sinks/basic_sink_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/locks.hpp>", private, "<boost/log/sources/channel_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/locks.hpp>", private, "<boost/log/sources/exception_handler_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/locks.hpp>", private, "<boost/log/sources/severity_feature.hpp>", public ] },
+    { include: ["<boost/log/detail/locks.hpp>", private, "<boost/log/sources/threading_models.hpp>", public ] },
+    { include: ["<boost/log/detail/locks.hpp>", private, "<boost/log/utility/strictest_lock.hpp>", public ] },
+    { include: ["<boost/log/detail/native_typeof.hpp>", private, "<boost/log/sources/record_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/native_typeof.hpp>", private, "<boost/log/utility/setup/settings.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/expressions/formatters/named_scope.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/sinks/async_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/sinks/event_log_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/sinks/sync_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/sinks/syslog_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/sinks/text_file_backend.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/sinks/unlocked_frontend.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/sources/basic_logger.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/utility/setup/file.hpp>", public ] },
+    { include: ["<boost/log/detail/parameter_tools.hpp>", private, "<boost/log/utility/value_ref.hpp>", public ] },
+    { include: ["<boost/log/detail/pp_identity.hpp>", private, "<boost/log/utility/strictest_lock.hpp>", public ] },
+    { include: ["<boost/log/detail/process_id.hpp>", private, "<boost/log/attributes/current_process_id.hpp>", public ] },
+    { include: ["<boost/log/detail/setup_config.hpp>", private, "<boost/log/utility/setup/filter_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/setup_config.hpp>", private, "<boost/log/utility/setup/formatter_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/setup_config.hpp>", private, "<boost/log/utility/setup/from_settings.hpp>", public ] },
+    { include: ["<boost/log/detail/setup_config.hpp>", private, "<boost/log/utility/setup/from_stream.hpp>", public ] },
+    { include: ["<boost/log/detail/setup_config.hpp>", private, "<boost/log/utility/setup.hpp>", public ] },
+    { include: ["<boost/log/detail/setup_config.hpp>", private, "<boost/log/utility/setup/settings.hpp>", public ] },
+    { include: ["<boost/log/detail/setup_config.hpp>", private, "<boost/log/utility/setup/settings_parser.hpp>", public ] },
+    { include: ["<boost/log/detail/singleton.hpp>", private, "<boost/log/sources/global_logger_storage.hpp>", public ] },
+    { include: ["<boost/log/detail/sink_init_helpers.hpp>", private, "<boost/log/utility/setup/console.hpp>", public ] },
+    { include: ["<boost/log/detail/sink_init_helpers.hpp>", private, "<boost/log/utility/setup/file.hpp>", public ] },
+    { include: ["<boost/log/detail/snprintf.hpp>", private, "<boost/log/expressions/formatters/c_decorator.hpp>", public ] },
+    { include: ["<boost/log/detail/tagged_integer.hpp>", private, "<boost/log/sinks/attribute_mapping.hpp>", public ] },
+    { include: ["<boost/log/detail/tagged_integer.hpp>", private, "<boost/log/sinks/event_log_constants.hpp>", public ] },
+    { include: ["<boost/log/detail/thread_id.hpp>", private, "<boost/log/attributes/current_thread_id.hpp>", public ] },
+    { include: ["<boost/log/detail/threadsafe_queue.hpp>", private, "<boost/log/sinks/unbounded_fifo_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/timestamp.hpp>", private, "<boost/log/sinks/bounded_ordering_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/timestamp.hpp>", private, "<boost/log/sinks/unbounded_ordering_queue.hpp>", public ] },
+    { include: ["<boost/log/detail/trivial_keyword.hpp>", private, "<boost/log/expressions/keyword.hpp>", public ] },
+    { include: ["<boost/log/detail/trivial_keyword.hpp>", private, "<boost/log/trivial.hpp>", public ] },
+    { include: ["<boost/log/detail/unary_function_terminal.hpp>", private, "<boost/log/expressions/predicates/begins_with.hpp>", public ] },
+    { include: ["<boost/log/detail/unary_function_terminal.hpp>", private, "<boost/log/expressions/predicates/contains.hpp>", public ] },
+    { include: ["<boost/log/detail/unary_function_terminal.hpp>", private, "<boost/log/expressions/predicates/ends_with.hpp>", public ] },
+    { include: ["<boost/log/detail/unary_function_terminal.hpp>", private, "<boost/log/expressions/predicates/has_attr.hpp>", public ] },
+    { include: ["<boost/log/detail/unary_function_terminal.hpp>", private, "<boost/log/expressions/predicates/is_in_range.hpp>", public ] },
+    { include: ["<boost/log/detail/unary_function_terminal.hpp>", private, "<boost/log/expressions/predicates/matches.hpp>", public ] },
+    { include: ["<boost/log/detail/unhandled_exception_count.hpp>", private, "<boost/log/sources/record_ostream.hpp>", public ] },
+    { include: ["<boost/log/detail/value_ref_visitation.hpp>", private, "<boost/log/utility/value_ref.hpp>", public ] },
+    { include: ["<boost/log/detail/visible_type.hpp>", private, "<boost/log/sources/global_logger_storage.hpp>", public ] },
+    { include: ["<boost/log/detail/visible_type.hpp>", private, "<boost/log/utility/type_dispatch/dynamic_type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/visible_type.hpp>", private, "<boost/log/utility/type_dispatch/static_type_dispatcher.hpp>", public ] },
+    { include: ["<boost/log/detail/visible_type.hpp>", private, "<boost/log/utility/type_dispatch/type_dispatcher.hpp>", public ] },
+    { include: ["<boost/math/bindings/detail/big_digamma.hpp>", private, "<boost/math/bindings/e_float.hpp>", public ] },
+    { include: ["<boost/math/bindings/detail/big_digamma.hpp>", private, "<boost/math/bindings/mpfr.hpp>", public ] },
+    { include: ["<boost/math/bindings/detail/big_digamma.hpp>", private, "<boost/math/bindings/mpreal.hpp>", public ] },
+    { include: ["<boost/math/bindings/detail/big_digamma.hpp>", private, "<boost/math/bindings/rr.hpp>", public ] },
+    { include: ["<boost/math/bindings/detail/big_lanczos.hpp>", private, "<boost/math/bindings/e_float.hpp>", public ] },
+    { include: ["<boost/math/bindings/detail/big_lanczos.hpp>", private, "<boost/math/bindings/mpfr.hpp>", public ] },
+    { include: ["<boost/math/bindings/detail/big_lanczos.hpp>", private, "<boost/math/bindings/mpreal.hpp>", public ] },
+    { include: ["<boost/math/bindings/detail/big_lanczos.hpp>", private, "<boost/math/bindings/rr.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/bernoulli.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/beta.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/binomial.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/cauchy.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/chi_squared.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/exponential.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/extreme_value.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/fisher_f.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/gamma.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/geometric.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/hypergeometric.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/inverse_chi_squared.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/inverse_gamma.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/inverse_gaussian.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/laplace.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/logistic.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/lognormal.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/negative_binomial.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/non_central_beta.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/non_central_chi_squared.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/normal.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/pareto.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/poisson.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/rayleigh.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/skew_normal.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/students_t.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/triangular.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/uniform.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/common_error_handling.hpp>", private, "<boost/math/distributions/weibull.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/bernoulli.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/beta.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/binomial.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/cauchy.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/chi_squared.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/exponential.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/extreme_value.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/fisher_f.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/gamma.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/geometric.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/hypergeometric.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/inverse_chi_squared.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/inverse_gamma.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/inverse_gaussian.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/laplace.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/logistic.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/lognormal.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/negative_binomial.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/non_central_beta.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/non_central_chi_squared.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/non_central_f.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/non_central_t.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/normal.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/pareto.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/poisson.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/rayleigh.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/skew_normal.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/students_t.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/triangular.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/uniform.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/derived_accessors.hpp>", private, "<boost/math/distributions/weibull.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/generic_mode.hpp>", private, "<boost/math/distributions/non_central_beta.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/generic_mode.hpp>", private, "<boost/math/distributions/non_central_chi_squared.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/generic_mode.hpp>", private, "<boost/math/distributions/non_central_f.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/generic_mode.hpp>", private, "<boost/math/distributions/skew_normal.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/generic_quantile.hpp>", private, "<boost/math/distributions/non_central_chi_squared.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/generic_quantile.hpp>", private, "<boost/math/distributions/non_central_t.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/hypergeometric_cdf.hpp>", private, "<boost/math/distributions/hypergeometric.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/hypergeometric_pdf.hpp>", private, "<boost/math/distributions/hypergeometric.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/hypergeometric_quantile.hpp>", private, "<boost/math/distributions/hypergeometric.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/inv_discrete_quantile.hpp>", private, "<boost/math/distributions/binomial.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/inv_discrete_quantile.hpp>", private, "<boost/math/distributions/geometric.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/inv_discrete_quantile.hpp>", private, "<boost/math/distributions/negative_binomial.hpp>", public ] },
+    { include: ["<boost/math/distributions/detail/inv_discrete_quantile.hpp>", private, "<boost/math/distributions/poisson.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/airy_ai_bi_zero.hpp>", private, "<boost/math/special_functions/airy.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/bessel_i0.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/bessel_i1.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/bessel_ik.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jn.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jy.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/bessel_jy_zero.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/bessel_kn.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/bessel_yn.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/erf_inv.hpp>", private, "<boost/math/special_functions/erf.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/fp_traits.hpp>", private, "<boost/math/special_functions/fpclassify.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/fp_traits.hpp>", private, "<boost/math/special_functions/sign.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/gamma_inva.hpp>", private, "<boost/math/special_functions/gamma.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/ibeta_inv_ab.hpp>", private, "<boost/math/special_functions/beta.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/ibeta_inverse.hpp>", private, "<boost/math/special_functions/beta.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/iconv.hpp>", private, "<boost/math/special_functions/bessel.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/igamma_inverse.hpp>", private, "<boost/math/special_functions/gamma.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/igamma_large.hpp>", private, "<boost/math/special_functions/gamma.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/lanczos_sse2.hpp>", private, "<boost/math/special_functions/lanczos.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/lgamma_small.hpp>", private, "<boost/math/special_functions/gamma.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/round_fwd.hpp>", private, "<boost/math/special_functions/math_fwd.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/unchecked_factorial.hpp>", private, "<boost/math/special_functions/factorials.hpp>", public ] },
+    { include: ["<boost/math/special_functions/detail/unchecked_factorial.hpp>", private, "<boost/math/special_functions/gamma.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_10.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_11.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_12.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_13.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_14.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_15.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_16.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_17.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_18.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_19.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_20.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_2.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_3.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_4.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_5.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_6.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_7.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_8.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner1_9.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_10.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_11.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_12.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_13.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_14.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_15.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_16.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_17.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_18.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_19.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_20.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_2.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_3.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_4.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_5.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_6.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_7.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_8.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner2_9.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_10.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_11.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_12.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_13.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_14.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_15.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_16.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_17.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_18.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_19.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_20.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_2.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_3.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_4.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_5.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_6.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_7.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_8.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/polynomial_horner3_9.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_10.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_11.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_12.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_13.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_14.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_15.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_16.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_17.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_18.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_19.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_20.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_2.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_3.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_4.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_5.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_6.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_7.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_8.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner1_9.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_10.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_11.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_12.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_13.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_14.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_15.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_16.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_17.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_18.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_19.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_20.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_2.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_3.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_4.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_5.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_6.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_7.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_8.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner2_9.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_10.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_11.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_12.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_13.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_14.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_15.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_16.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_17.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_18.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_19.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_20.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_2.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_3.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_4.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_5.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_6.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_7.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_8.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/math/tools/detail/rational_horner3_9.hpp>", private, "<boost/math/tools/rational.hpp>", public ] },
+    { include: ["<boost/move/detail/config_begin.hpp>", private, "<boost/move/algorithm.hpp>", public ] },
+    { include: ["<boost/move/detail/config_begin.hpp>", private, "<boost/move/core.hpp>", public ] },
+    { include: ["<boost/move/detail/config_begin.hpp>", private, "<boost/move/iterator.hpp>", public ] },
+    { include: ["<boost/move/detail/config_begin.hpp>", private, "<boost/move/move.hpp>", public ] },
+    { include: ["<boost/move/detail/config_begin.hpp>", private, "<boost/move/traits.hpp>", public ] },
+    { include: ["<boost/move/detail/config_begin.hpp>", private, "<boost/move/utility.hpp>", public ] },
+    { include: ["<boost/move/detail/config_end.hpp>", private, "<boost/move/algorithm.hpp>", public ] },
+    { include: ["<boost/move/detail/config_end.hpp>", private, "<boost/move/core.hpp>", public ] },
+    { include: ["<boost/move/detail/config_end.hpp>", private, "<boost/move/iterator.hpp>", public ] },
+    { include: ["<boost/move/detail/config_end.hpp>", private, "<boost/move/move.hpp>", public ] },
+    { include: ["<boost/move/detail/config_end.hpp>", private, "<boost/move/traits.hpp>", public ] },
+    { include: ["<boost/move/detail/config_end.hpp>", private, "<boost/move/utility.hpp>", public ] },
+    { include: ["<boost/move/detail/meta_utils.hpp>", private, "<boost/move/core.hpp>", public ] },
+    { include: ["<boost/move/detail/meta_utils.hpp>", private, "<boost/move/traits.hpp>", public ] },
+    { include: ["<boost/move/detail/meta_utils.hpp>", private, "<boost/move/utility.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/deque.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/flat_map.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/flat_set.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/list.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/map.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/set.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/slist.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/stable_vector.hpp>", public ] },
+    { include: ["<boost/move/detail/move_helpers.hpp>", private, "<boost/container/vector.hpp>", public ] },
+    { include: ["<boost/mpi/detail/binary_buffer_iprimitive.hpp>", private, "<boost/mpi/packed_iarchive.hpp>", public ] },
+    { include: ["<boost/mpi/detail/binary_buffer_oprimitive.hpp>", private, "<boost/mpi/packed_oarchive.hpp>", public ] },
+    { include: ["<boost/mpi/detail/broadcast_sc.hpp>", private, "<boost/mpi/collectives/broadcast.hpp>", public ] },
+    { include: ["<boost/mpi/detail/broadcast_sc.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/mpi/detail/communicator_sc.hpp>", private, "<boost/mpi/communicator.hpp>", public ] },
+    { include: ["<boost/mpi/detail/communicator_sc.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/mpi/detail/computation_tree.hpp>", private, "<boost/mpi/collectives/reduce.hpp>", public ] },
+    { include: ["<boost/mpi/detail/computation_tree.hpp>", private, "<boost/mpi/collectives/scan.hpp>", public ] },
+    { include: ["<boost/mpi/detail/content_oarchive.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/mpi/detail/forward_skeleton_iarchive.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/mpi/detail/forward_skeleton_oarchive.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/mpi/detail/ignore_iprimitive.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/mpi/detail/ignore_oprimitive.hpp>", private, "<boost/mpi/skeleton_and_content.hpp>", public ] },
+    { include: ["<boost/mpi/detail/mpi_datatype_cache.hpp>", private, "<boost/mpi/datatype.hpp>", public ] },
+    { include: ["<boost/mpi/detail/packed_iprimitive.hpp>", private, "<boost/mpi/packed_iarchive.hpp>", public ] },
+    { include: ["<boost/mpi/detail/packed_oprimitive.hpp>", private, "<boost/mpi/packed_oarchive.hpp>", public ] },
+    { include: ["<boost/mpi/detail/point_to_point.hpp>", private, "<boost/mpi/collectives/gather.hpp>", public ] },
+    { include: ["<boost/mpi/detail/point_to_point.hpp>", private, "<boost/mpi/collectives/reduce.hpp>", public ] },
+    { include: ["<boost/mpi/detail/point_to_point.hpp>", private, "<boost/mpi/collectives/scan.hpp>", public ] },
+    { include: ["<boost/mpi/detail/point_to_point.hpp>", private, "<boost/mpi/collectives/scatter.hpp>", public ] },
+    { include: ["<boost/mpi/detail/point_to_point.hpp>", private, "<boost/mpi/communicator.hpp>", public ] },
+    { include: ["<boost/msm/front/detail/common_states.hpp>", private, "<boost/msm/front/common_states.hpp>", public ] },
+    { include: ["<boost/msm/front/detail/row2_helper.hpp>", private, "<boost/msm/front/internal_row.hpp>", public ] },
+    { include: ["<boost/msm/front/detail/row2_helper.hpp>", private, "<boost/msm/front/row2.hpp>", public ] },
+    { include: ["<boost/msm/mpl_graph/detail/adjacency_list_graph.ipp>", private, "<boost/msm/mpl_graph/adjacency_list_graph.hpp>", public ] },
+    { include: ["<boost/msm/mpl_graph/detail/graph_implementation_interface.ipp>", private, "<boost/msm/mpl_graph/mpl_graph.hpp>", public ] },
+    { include: ["<boost/msm/mpl_graph/detail/incidence_list_graph.ipp>", private, "<boost/msm/mpl_graph/incidence_list_graph.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/access_specifier.hpp>", private, "<boost/multi_index/composite_key.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/access_specifier.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/access_specifier.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/access_specifier.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/access_specifier.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/access_specifier.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/adl_swap.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/archive_constructed.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/auto_space.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/base_type.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/bidir_node_iterator.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/bidir_node_iterator.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/bucket_array.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/converter.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/do_not_copy_elements_tag.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/do_not_copy_elements_tag.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/do_not_copy_elements_tag.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/do_not_copy_elements_tag.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/do_not_copy_elements_tag.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/duplicates_iterator.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/hash_index_args.hpp>", private, "<boost/multi_index/hashed_index_fwd.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/hash_index_iterator.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/has_tag.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/header_holder.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/index_node_base.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/index_node_base.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/index_node_base.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/index_node_base.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/invariant_assert.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/modify_key_adaptor.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/modify_key_adaptor.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/no_duplicate_tags.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/no_duplicate_tags.hpp>", private, "<boost/multi_index/tag.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/ord_index_args.hpp>", private, "<boost/multi_index/ordered_index_fwd.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/ord_index_node.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/ord_index_ops.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index/composite_key.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/prevent_eti.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/rnd_index_loader.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/rnd_index_node.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/rnd_index_ops.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/rnd_index_ptr_array.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/rnd_node_iterator.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_ctr_proxy.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_ctr_proxy.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_ctr_proxy.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_ctr_proxy.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_mode.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_mode.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_mode.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_mode.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/safe_mode.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/scope_guard.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/scope_guard.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/scope_guard.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/scope_guard.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/scope_guard.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/seq_index_node.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/seq_index_ops.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/serialization_version.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/unbounded.hpp>", private, "<boost/bimap/bimap.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/unbounded.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/value_compare.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/vartempl_support.hpp>", private, "<boost/multi_index_container.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/vartempl_support.hpp>", private, "<boost/multi_index/hashed_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/vartempl_support.hpp>", private, "<boost/multi_index/ordered_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/vartempl_support.hpp>", private, "<boost/multi_index/random_access_index.hpp>", public ] },
+    { include: ["<boost/multi_index/detail/vartempl_support.hpp>", private, "<boost/multi_index/sequenced_index.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/big_lanczos.hpp>", private, "<boost/multiprecision/cpp_dec_float.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/big_lanczos.hpp>", private, "<boost/multiprecision/gmp.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/big_lanczos.hpp>", private, "<boost/multiprecision/mpfi.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/big_lanczos.hpp>", private, "<boost/multiprecision/mpfr.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/bitscan.hpp>", private, "<boost/multiprecision/cpp_int/misc.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/bitscan.hpp>", private, "<boost/multiprecision/integer.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/digits.hpp>", private, "<boost/multiprecision/gmp.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/digits.hpp>", private, "<boost/multiprecision/mpfi.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/digits.hpp>", private, "<boost/multiprecision/mpfr.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/dynamic_array.hpp>", private, "<boost/multiprecision/cpp_dec_float.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/float_string_cvt.hpp>", private, "<boost/multiprecision/float128.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/generic_interconvert.hpp>", private, "<boost/multiprecision/number.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/integer_ops.hpp>", private, "<boost/multiprecision/cpp_int.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/integer_ops.hpp>", private, "<boost/multiprecision/debug_adaptor.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/integer_ops.hpp>", private, "<boost/multiprecision/gmp.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/integer_ops.hpp>", private, "<boost/multiprecision/logged_adaptor.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/integer_ops.hpp>", private, "<boost/multiprecision/tommath.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/number_base.hpp>", private, "<boost/multiprecision/traits/is_restricted_conversion.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/number_compare.hpp>", private, "<boost/multiprecision/number.hpp>", public ] },
+    { include: ["<boost/multiprecision/detail/ublas_interop.hpp>", private, "<boost/multiprecision/number.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/bounds.hpp>", private, "<boost/numeric/conversion/bounds.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/conversion_traits.hpp>", private, "<boost/numeric/conversion/conversion_traits.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/converter.hpp>", private, "<boost/numeric/conversion/converter.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/int_float_mixture.hpp>", private, "<boost/numeric/conversion/int_float_mixture.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/is_subranged.hpp>", private, "<boost/numeric/conversion/is_subranged.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/numeric_cast_traits.hpp>", private, "<boost/numeric/conversion/numeric_cast_traits.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/old_numeric_cast.hpp>", private, "<boost/numeric/conversion/cast.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/sign_mixture.hpp>", private, "<boost/numeric/conversion/sign_mixture.hpp>", public ] },
+    { include: ["<boost/numeric/conversion/detail/udt_builtin_mixture.hpp>", private, "<boost/numeric/conversion/udt_builtin_mixture.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/alpha_rounding_control.hpp>", private, "<boost/numeric/interval/hw_rounding.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/bugs.hpp>", private, "<boost/numeric/interval/arith2.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/bugs.hpp>", private, "<boost/numeric/interval/arith.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/bugs.hpp>", private, "<boost/numeric/interval/rounded_arith.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/bugs.hpp>", private, "<boost/numeric/interval/rounded_transc.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/bugs.hpp>", private, "<boost/numeric/interval/transc.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/bugs.hpp>", private, "<boost/numeric/interval/utility.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/c99_rounding_control.hpp>", private, "<boost/numeric/interval/hw_rounding.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/division.hpp>", private, "<boost/numeric/interval/arith2.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/division.hpp>", private, "<boost/numeric/interval/arith.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/ia64_rounding_control.hpp>", private, "<boost/numeric/interval/hw_rounding.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/arith2.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/arith3.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/compare/certain.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/compare/explicit.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/compare/lexicographic.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/compare/possible.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/compare/set.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/compare/tribool.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/ext/integer.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/interval.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/limits.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/transc.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/interval_prototype.hpp>", private, "<boost/numeric/interval/utility.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/ppc_rounding_control.hpp>", private, "<boost/numeric/interval/hw_rounding.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/sparc_rounding_control.hpp>", private, "<boost/numeric/interval/hw_rounding.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/arith2.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/arith3.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/arith.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/compare/certain.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/compare/lexicographic.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/compare/possible.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/compare/set.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/compare/tribool.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/ext/integer.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/transc.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/test_input.hpp>", private, "<boost/numeric/interval/utility.hpp>", public ] },
+    { include: ["<boost/numeric/interval/detail/x86_rounding_control.hpp>", private, "<boost/numeric/interval/hw_rounding.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/algebra/detail/for_each.hpp>", private, "<boost/numeric/odeint/algebra/range_algebra.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/algebra/detail/macros.hpp>", private, "<boost/numeric/odeint/algebra/range_algebra.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/algebra/detail/reduce.hpp>", private, "<boost/numeric/odeint/algebra/range_algebra.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/integrate/detail/integrate_adaptive.hpp>", private, "<boost/numeric/odeint/integrate/integrate_adaptive.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/integrate/detail/integrate_adaptive.hpp>", private, "<boost/numeric/odeint/integrate/integrate_const.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/integrate/detail/integrate_const.hpp>", private, "<boost/numeric/odeint/integrate/integrate_const.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/integrate/detail/integrate_n_steps.hpp>", private, "<boost/numeric/odeint/integrate/integrate_n_steps.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/integrate/detail/integrate_times.hpp>", private, "<boost/numeric/odeint/integrate/integrate_times.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/adams_bashforth_call_algebra.hpp>", private, "<boost/numeric/odeint/stepper/adams_bashforth.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/adams_bashforth_coefficients.hpp>", private, "<boost/numeric/odeint/stepper/adams_bashforth.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/adams_moulton_call_algebra.hpp>", private, "<boost/numeric/odeint/stepper/adams_moulton.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/adams_moulton_coefficients.hpp>", private, "<boost/numeric/odeint/stepper/adams_moulton.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/generic_rk_algorithm.hpp>", private, "<boost/numeric/odeint/stepper/explicit_error_generic_rk.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/generic_rk_algorithm.hpp>", private, "<boost/numeric/odeint/stepper/explicit_generic_rk.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/generic_rk_call_algebra.hpp>", private, "<boost/numeric/odeint/stepper/explicit_error_generic_rk.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/generic_rk_operations.hpp>", private, "<boost/numeric/odeint/stepper/explicit_error_generic_rk.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/rotating_buffer.hpp>", private, "<boost/numeric/odeint/stepper/adams_bashforth.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/stepper/detail/rotating_buffer.hpp>", private, "<boost/numeric/odeint/stepper/adams_moulton.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/util/detail/is_range.hpp>", private, "<boost/numeric/odeint/util/copy.hpp>", public ] },
+    { include: ["<boost/numeric/odeint/util/detail/less_with_sign.hpp>", private, "<boost/numeric/odeint/stepper/bulirsch_stoer.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/config.hpp>", private, "<boost/numeric/ublas/exception.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/config.hpp>", private, "<boost/numeric/ublas/operation/num_columns.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/config.hpp>", private, "<boost/numeric/ublas/operation/num_rows.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/config.hpp>", private, "<boost/numeric/ublas/operation/size.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/config.hpp>", private, "<boost/numeric/ublas/traits.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/definitions.hpp>", private, "<boost/numeric/ublas/functional.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/duff.hpp>", private, "<boost/numeric/ublas/functional.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/iterator.hpp>", private, "<boost/numeric/ublas/storage.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/iterator.hpp>", private, "<boost/numeric/ublas/traits.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/matrix_assign.hpp>", private, "<boost/numeric/ublas/experimental/sparse_view.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/matrix_assign.hpp>", private, "<boost/numeric/ublas/matrix.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/matrix_assign.hpp>", private, "<boost/numeric/ublas/matrix_proxy.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/matrix_assign.hpp>", private, "<boost/numeric/ublas/matrix_sparse.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/matrix_assign.hpp>", private, "<boost/numeric/ublas/operation_blocked.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/raw.hpp>", private, "<boost/numeric/ublas/functional.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/returntype_deduction.hpp>", private, "<boost/numeric/ublas/traits.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/temporary.hpp>", private, "<boost/numeric/ublas/banded.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/temporary.hpp>", private, "<boost/numeric/ublas/hermitian.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/temporary.hpp>", private, "<boost/numeric/ublas/matrix_proxy.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/temporary.hpp>", private, "<boost/numeric/ublas/symmetric.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/temporary.hpp>", private, "<boost/numeric/ublas/triangular.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/temporary.hpp>", private, "<boost/numeric/ublas/vector_proxy.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/vector_assign.hpp>", private, "<boost/numeric/ublas/matrix_proxy.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/vector_assign.hpp>", private, "<boost/numeric/ublas/operation_blocked.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/vector_assign.hpp>", private, "<boost/numeric/ublas/vector.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/vector_assign.hpp>", private, "<boost/numeric/ublas/vector_proxy.hpp>", public ] },
+    { include: ["<boost/numeric/ublas/detail/vector_assign.hpp>", private, "<boost/numeric/ublas/vector_sparse.hpp>", public ] },
+    { include: ["<boost/pending/detail/disjoint_sets.hpp>", private, "<boost/pending/disjoint_sets.hpp>", public ] },
+    { include: ["<boost/pending/detail/int_iterator.hpp>", private, "<boost/graph/matrix_as_graph.hpp>", public ] },
+    { include: ["<boost/pending/detail/property.hpp>", private, "<boost/pending/property.hpp>", public ] },
+    { include: ["<boost/phoenix/core/detail/actor_operator.hpp>", private, "<boost/phoenix/core/actor.hpp>", public ] },
+    { include: ["<boost/phoenix/core/detail/actor_result_of.hpp>", private, "<boost/phoenix/core/actor.hpp>", public ] },
+    { include: ["<boost/phoenix/core/detail/argument.hpp>", private, "<boost/phoenix/core/argument.hpp>", public ] },
+    { include: ["<boost/phoenix/core/detail/call.hpp>", private, "<boost/phoenix/core/call.hpp>", public ] },
+    { include: ["<boost/phoenix/core/detail/expression.hpp>", private, "<boost/phoenix/core/expression.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/construct_eval.hpp>", private, "<boost/phoenix/object/construct.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/construct.hpp>", private, "<boost/phoenix/object/construct.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/new_eval.hpp>", private, "<boost/phoenix/object/new.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/new.hpp>", private, "<boost/phoenix/object/new.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/target.hpp>", private, "<boost/phoenix/object/const_cast.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/target.hpp>", private, "<boost/phoenix/object/construct.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/target.hpp>", private, "<boost/phoenix/object/dynamic_cast.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/target.hpp>", private, "<boost/phoenix/object/new.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/target.hpp>", private, "<boost/phoenix/object/reinterpret_cast.hpp>", public ] },
+    { include: ["<boost/phoenix/object/detail/target.hpp>", private, "<boost/phoenix/object/static_cast.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/define_operator.hpp>", private, "<boost/phoenix/operator/arithmetic.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/define_operator.hpp>", private, "<boost/phoenix/operator/bitwise.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/define_operator.hpp>", private, "<boost/phoenix/operator/comparison.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/define_operator.hpp>", private, "<boost/phoenix/operator/logical.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/define_operator.hpp>", private, "<boost/phoenix/operator/member.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/define_operator.hpp>", private, "<boost/phoenix/operator/self.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/mem_fun_ptr_eval_result_of.hpp>", private, "<boost/phoenix/operator/member.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/mem_fun_ptr_gen.hpp>", private, "<boost/phoenix/operator/member.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/undef_operator.hpp>", private, "<boost/phoenix/operator/arithmetic.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/undef_operator.hpp>", private, "<boost/phoenix/operator/bitwise.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/undef_operator.hpp>", private, "<boost/phoenix/operator/comparison.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/undef_operator.hpp>", private, "<boost/phoenix/operator/logical.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/undef_operator.hpp>", private, "<boost/phoenix/operator/member.hpp>", public ] },
+    { include: ["<boost/phoenix/operator/detail/undef_operator.hpp>", private, "<boost/phoenix/operator/self.hpp>", public ] },
+    { include: ["<boost/phoenix/scope/detail/dynamic.hpp>", private, "<boost/phoenix/scope/dynamic.hpp>", public ] },
+    { include: ["<boost/phoenix/scope/detail/local_gen.hpp>", private, "<boost/phoenix/scope/lambda.hpp>", public ] },
+    { include: ["<boost/phoenix/scope/detail/local_gen.hpp>", private, "<boost/phoenix/scope/let.hpp>", public ] },
+    { include: ["<boost/phoenix/scope/detail/local_variable.hpp>", private, "<boost/phoenix/scope/local_variable.hpp>", public ] },
+    { include: ["<boost/phoenix/statement/detail/catch_push_back.hpp>", private, "<boost/phoenix/statement/try_catch.hpp>", public ] },
+    { include: ["<boost/phoenix/statement/detail/switch.hpp>", private, "<boost/phoenix/statement/switch.hpp>", public ] },
+    { include: ["<boost/phoenix/statement/detail/try_catch_eval.hpp>", private, "<boost/phoenix/statement/try_catch.hpp>", public ] },
+    { include: ["<boost/phoenix/statement/detail/try_catch_expression.hpp>", private, "<boost/phoenix/statement/try_catch.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/begin.hpp>", private, "<boost/phoenix/stl/algorithm/iteration.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/begin.hpp>", private, "<boost/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/begin.hpp>", private, "<boost/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/decay_array.hpp>", private, "<boost/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/decay_array.hpp>", private, "<boost/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/end.hpp>", private, "<boost/phoenix/stl/algorithm/iteration.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/end.hpp>", private, "<boost/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/end.hpp>", private, "<boost/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_equal_range.hpp>", private, "<boost/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_find.hpp>", private, "<boost/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_lower_bound.hpp>", private, "<boost/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_remove.hpp>", private, "<boost/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_remove_if.hpp>", private, "<boost/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_reverse.hpp>", private, "<boost/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_sort.hpp>", private, "<boost/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_unique.hpp>", private, "<boost/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/algorithm/detail/has_upper_bound.hpp>", private, "<boost/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/phoenix/stl/container/detail/container.hpp>", private, "<boost/phoenix/stl/container/container.hpp>", public ] },
+    { include: ["<boost/phoenix/support/detail/iterate_define.hpp>", private, "<boost/phoenix/support/iterate.hpp>", public ] },
+    { include: ["<boost/pool/detail/guard.hpp>", private, "<boost/pool/singleton_pool.hpp>", public ] },
+    { include: ["<boost/pool/detail/mutex.hpp>", private, "<boost/pool/poolfwd.hpp>", public ] },
+    { include: ["<boost/pool/detail/pool_construct.ipp>", private, "<boost/pool/object_pool.hpp>", public ] },
+    { include: ["<boost/pool/detail/pool_construct_simple.ipp>", private, "<boost/pool/object_pool.hpp>", public ] },
+    { include: ["<boost/predef/detail/_cassert.h>", private, "<boost/predef/library/c/_prefix.h>", public ] },
+    { include: ["<boost/predef/detail/_exception.h>", private, "<boost/predef/library/std/_prefix.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/aix.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/amigaos.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/android.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/beos.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/bsd/bsdi.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/bsd/dragonfly.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/bsd/free.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/bsd.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/bsd/net.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/bsd/open.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/cygwin.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/hpux.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/irix.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/linux.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/macos.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/os400.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/qnxnto.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/solaris.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/unix.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/vms.h>", public ] },
+    { include: ["<boost/predef/detail/os_detected.h>", private, "<boost/predef/os/windows.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/alpha.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/arm.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/blackfin.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/convex.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/ia64.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/m68k.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/mips.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/parisc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/ppc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/pyramid.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/rs6k.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/sparc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/superh.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/sys370.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/sys390.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/x86/32.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/x86/64.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/x86.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/architecture/z.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/borland.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/clang.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/comeau.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/compaq.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/diab.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/digitalmars.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/dignus.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/edg.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/ekopath.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/gcc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/gcc_xml.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/greenhills.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/hp_acc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/iar.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/ibm.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/intel.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/kai.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/llvm.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/metaware.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/metrowerks.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/microtec.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/mpw.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/palm.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/pgi.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/sgi_mipspro.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/sunpro.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/tendra.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/visualc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/compiler/watcom.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/language/objc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/language/stdc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/language/stdcpp.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/c/gnu.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/c/uc.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/c/vms.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/c/zos.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/cxx.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/dinkumware.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/libcomo.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/modena.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/msl.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/roguewave.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/sgi.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/stdcpp3.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/stlport.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/library/std/vacpp.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/make.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/aix.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/amigaos.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/android.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/beos.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/bsd/bsdi.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/bsd/dragonfly.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/bsd/free.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/bsd.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/bsd/net.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/bsd/open.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/cygwin.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/hpux.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/irix.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/linux.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/macos.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/os400.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/qnxnto.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/solaris.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/unix.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/vms.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/os/windows.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/other/endian.h>", public ] },
+    { include: ["<boost/predef/detail/test.h>", private, "<boost/predef/platform/mingw.h>", public ] },
+    { include: ["<boost/preprocessor/arithmetic/detail/div_base.hpp>", private, "<boost/preprocessor/arithmetic/div.hpp>", public ] },
+    { include: ["<boost/preprocessor/arithmetic/detail/div_base.hpp>", private, "<boost/preprocessor/arithmetic/mod.hpp>", public ] },
+    { include: ["<boost/preprocessor/control/detail/dmc/while.hpp>", private, "<boost/preprocessor/control/while.hpp>", public ] },
+    { include: ["<boost/preprocessor/control/detail/edg/while.hpp>", private, "<boost/preprocessor/control/while.hpp>", public ] },
+    { include: ["<boost/preprocessor/control/detail/msvc/while.hpp>", private, "<boost/preprocessor/control/while.hpp>", public ] },
+    { include: ["<boost/preprocessor/control/detail/while.hpp>", private, "<boost/preprocessor/control/while.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/control/deduce_d.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/control/while.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/list/fold_left.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/list/fold_right.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/repetition/deduce_r.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/repetition/deduce_z.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/repetition/enum.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/repetition/enum_shifted.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/repetition/enum_trailing.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/repetition/for.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/repetition/repeat_from_to.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/repetition/repeat.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/seq/fold_left.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/auto_rec.hpp>", private, "<boost/preprocessor/seq/fold_right.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_binary.hpp>", private, "<boost/parameter/name.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_binary.hpp>", private, "<boost/preprocessor/list/adt.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_binary.hpp>", private, "<boost/spirit/home/classic/utility/rule_parser.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_binary.hpp>", private, "<boost/tti/has_template.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_nullary.hpp>", private, "<boost/parameter/preprocessor.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_unary.hpp>", private, "<boost/local_function/aux_/preprocessor/traits/decl_sign_/any_bind_type.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_unary.hpp>", private, "<boost/preprocessor/facilities/apply.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_unary.hpp>", private, "<boost/spirit/home/classic/utility/rule_parser.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/is_unary.hpp>", private, "<boost/typeof/template_encoding.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/split.hpp>", private, "<boost/parameter/aux_/preprocessor/for_each.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/split.hpp>", private, "<boost/parameter/name.hpp>", public ] },
+    { include: ["<boost/preprocessor/detail/split.hpp>", private, "<boost/preprocessor/facilities/is_empty.hpp>", public ] },
+    { include: ["<boost/preprocessor/list/detail/dmc/fold_left.hpp>", private, "<boost/preprocessor/list/fold_left.hpp>", public ] },
+    { include: ["<boost/preprocessor/list/detail/edg/fold_left.hpp>", private, "<boost/preprocessor/list/fold_left.hpp>", public ] },
+    { include: ["<boost/preprocessor/list/detail/edg/fold_right.hpp>", private, "<boost/preprocessor/list/fold_right.hpp>", public ] },
+    { include: ["<boost/preprocessor/list/detail/fold_left.hpp>", private, "<boost/preprocessor/list/fold_left.hpp>", public ] },
+    { include: ["<boost/preprocessor/list/detail/fold_right.hpp>", private, "<boost/preprocessor/list/fold_right.hpp>", public ] },
+    { include: ["<boost/preprocessor/repetition/detail/dmc/for.hpp>", private, "<boost/preprocessor/repetition/for.hpp>", public ] },
+    { include: ["<boost/preprocessor/repetition/detail/edg/for.hpp>", private, "<boost/preprocessor/repetition/for.hpp>", public ] },
+    { include: ["<boost/preprocessor/repetition/detail/for.hpp>", private, "<boost/preprocessor/repetition/for.hpp>", public ] },
+    { include: ["<boost/preprocessor/repetition/detail/msvc/for.hpp>", private, "<boost/preprocessor/repetition/for.hpp>", public ] },
+    { include: ["<boost/preprocessor/seq/detail/binary_transform.hpp>", private, "<boost/preprocessor/seq/to_list.hpp>", public ] },
+    { include: ["<boost/preprocessor/seq/detail/split.hpp>", private, "<boost/preprocessor/seq/first_n.hpp>", public ] },
+    { include: ["<boost/preprocessor/seq/detail/split.hpp>", private, "<boost/preprocessor/seq/rest_n.hpp>", public ] },
+    { include: ["<boost/preprocessor/slot/detail/def.hpp>", private, "<boost/preprocessor/slot/counter.hpp>", public ] },
+    { include: ["<boost/preprocessor/slot/detail/def.hpp>", private, "<boost/preprocessor/slot/slot.hpp>", public ] },
+    { include: ["<boost/program_options/detail/cmdline.hpp>", private, "<boost/program_options/parsers.hpp>", public ] },
+    { include: ["<boost/program_options/detail/parsers.hpp>", private, "<boost/program_options/parsers.hpp>", public ] },
+    { include: ["<boost/program_options/detail/value_semantic.hpp>", private, "<boost/program_options/value_semantic.hpp>", public ] },
+    { include: ["<boost/property_map/parallel/impl/distributed_property_map.ipp>", private, "<boost/property_map/parallel/distributed_property_map.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/exception_implementation.hpp>", private, "<boost/property_tree/exceptions.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/file_parser_error.hpp>", private, "<boost/property_tree/ini_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/info_parser_error.hpp>", private, "<boost/property_tree/info_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/info_parser_read.hpp>", private, "<boost/property_tree/info_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/info_parser_write.hpp>", private, "<boost/property_tree/info_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/info_parser_writer_settings.hpp>", private, "<boost/property_tree/info_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/json_parser_error.hpp>", private, "<boost/property_tree/json_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/json_parser_read.hpp>", private, "<boost/property_tree/json_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/json_parser_write.hpp>", private, "<boost/property_tree/json_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/ptree_implementation.hpp>", private, "<boost/property_tree/ptree.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/ptree_utils.hpp>", private, "<boost/property_tree/ini_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/ptree_utils.hpp>", private, "<boost/property_tree/ptree.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/ptree_utils.hpp>", private, "<boost/property_tree/string_path.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/xml_parser_error.hpp>", private, "<boost/property_tree/xml_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/xml_parser_flags.hpp>", private, "<boost/property_tree/xml_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/xml_parser_read_rapidxml.hpp>", private, "<boost/property_tree/xml_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/xml_parser_utils.hpp>", private, "<boost/graph/graphml.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/xml_parser_write.hpp>", private, "<boost/property_tree/xml_parser.hpp>", public ] },
+    { include: ["<boost/property_tree/detail/xml_parser_writer_settings.hpp>", private, "<boost/property_tree/xml_parser.hpp>", public ] },
+    { include: ["<boost/proto/context/detail/callable_eval.hpp>", private, "<boost/proto/context/callable.hpp>", public ] },
+    { include: ["<boost/proto/context/detail/default_eval.hpp>", private, "<boost/proto/context/default.hpp>", public ] },
+    { include: ["<boost/proto/context/detail/null_eval.hpp>", private, "<boost/proto/context/null.hpp>", public ] },
+    { include: ["<boost/proto/detail/and_n.hpp>", private, "<boost/proto/matches.hpp>", public ] },
+    { include: ["<boost/proto/detail/any.hpp>", private, "<boost/proto/transform/impl.hpp>", public ] },
+    { include: ["<boost/proto/detail/args.hpp>", private, "<boost/proto/args.hpp>", public ] },
+    { include: ["<boost/proto/detail/as_expr.hpp>", private, "<boost/proto/domain.hpp>", public ] },
+    { include: ["<boost/proto/detail/as_lvalue.hpp>", private, "<boost/proto/transform/call.hpp>", public ] },
+    { include: ["<boost/proto/detail/as_lvalue.hpp>", private, "<boost/proto/transform/make.hpp>", public ] },
+    { include: ["<boost/proto/detail/basic_expr.hpp>", private, "<boost/proto/expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/decltype.hpp>", private, "<boost/proto/context/default.hpp>", public ] },
+    { include: ["<boost/proto/detail/decltype.hpp>", private, "<boost/proto/transform/default.hpp>", public ] },
+    { include: ["<boost/proto/detail/deduce_domain.hpp>", private, "<boost/proto/domain.hpp>", public ] },
+    { include: ["<boost/proto/detail/deep_copy.hpp>", private, "<boost/proto/deep_copy.hpp>", public ] },
+    { include: ["<boost/proto/detail/deprecated.hpp>", private, "<boost/proto/make_expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/expr.hpp>", private, "<boost/proto/expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/extends_funop_const.hpp>", private, "<boost/proto/extends.hpp>", public ] },
+    { include: ["<boost/proto/detail/extends_funop.hpp>", private, "<boost/proto/extends.hpp>", public ] },
+    { include: ["<boost/proto/detail/funop.hpp>", private, "<boost/proto/expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/generate_by_value.hpp>", private, "<boost/proto/generate.hpp>", public ] },
+    { include: ["<boost/proto/detail/ignore_unused.hpp>", private, "<boost/proto/transform/make.hpp>", public ] },
+    { include: ["<boost/proto/detail/ignore_unused.hpp>", private, "<boost/proto/transform/pass_through.hpp>", public ] },
+    { include: ["<boost/proto/detail/is_noncopyable.hpp>", private, "<boost/proto/args.hpp>", public ] },
+    { include: ["<boost/proto/detail/is_noncopyable.hpp>", private, "<boost/proto/transform/env.hpp>", public ] },
+    { include: ["<boost/proto/detail/lambda_matches.hpp>", private, "<boost/proto/matches.hpp>", public ] },
+    { include: ["<boost/proto/detail/make_expr_funop.hpp>", private, "<boost/proto/make_expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/make_expr_.hpp>", private, "<boost/proto/make_expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/make_expr.hpp>", private, "<boost/proto/make_expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/matches_.hpp>", private, "<boost/proto/matches.hpp>", public ] },
+    { include: ["<boost/proto/detail/or_n.hpp>", private, "<boost/proto/matches.hpp>", public ] },
+    { include: ["<boost/proto/detail/poly_function.hpp>", private, "<boost/proto/make_expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/poly_function.hpp>", private, "<boost/proto/transform/call.hpp>", public ] },
+    { include: ["<boost/proto/detail/poly_function.hpp>", private, "<boost/proto/transform/env.hpp>", public ] },
+    { include: ["<boost/proto/detail/remove_typename.hpp>", private, "<boost/proto/extends.hpp>", public ] },
+    { include: ["<boost/proto/detail/static_const.hpp>", private, "<boost/proto/transform/impl.hpp>", public ] },
+    { include: ["<boost/proto/detail/template_arity.hpp>", private, "<boost/proto/matches.hpp>", public ] },
+    { include: ["<boost/proto/detail/template_arity.hpp>", private, "<boost/proto/traits.hpp>", public ] },
+    { include: ["<boost/proto/detail/template_arity.hpp>", private, "<boost/proto/transform/make.hpp>", public ] },
+    { include: ["<boost/proto/detail/traits.hpp>", private, "<boost/proto/traits.hpp>", public ] },
+    { include: ["<boost/proto/detail/unpack_expr_.hpp>", private, "<boost/proto/make_expr.hpp>", public ] },
+    { include: ["<boost/proto/detail/vararg_matches_impl.hpp>", private, "<boost/proto/matches.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/call.hpp>", private, "<boost/proto/transform/call.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/construct_funop.hpp>", private, "<boost/proto/transform/make.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/construct_pod_funop.hpp>", private, "<boost/proto/transform/make.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/default_function_impl.hpp>", private, "<boost/proto/transform/default.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/fold_impl.hpp>", private, "<boost/proto/transform/fold.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/lazy.hpp>", private, "<boost/proto/transform/lazy.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/make.hpp>", private, "<boost/proto/transform/make.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/pack.hpp>", private, "<boost/proto/transform/call.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/pack.hpp>", private, "<boost/proto/transform/lazy.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/pack.hpp>", private, "<boost/proto/transform/make.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/pass_through_impl.hpp>", private, "<boost/proto/transform/pass_through.hpp>", public ] },
+    { include: ["<boost/proto/transform/detail/when.hpp>", private, "<boost/proto/transform/when.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/associative_ptr_container.hpp>", private, "<boost/ptr_container/ptr_map_adapter.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/associative_ptr_container.hpp>", private, "<boost/ptr_container/ptr_set_adapter.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/map_iterator.hpp>", private, "<boost/ptr_container/ptr_map_adapter.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/meta_functions.hpp>", private, "<boost/ptr_container/ptr_map_adapter.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/meta_functions.hpp>", private, "<boost/ptr_container/ptr_set_adapter.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/reversible_ptr_container.hpp>", private, "<boost/ptr_container/ptr_sequence_adapter.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_ptr_map_adapter.hpp>", private, "<boost/ptr_container/serialize_ptr_map.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_ptr_map_adapter.hpp>", private, "<boost/ptr_container/serialize_ptr_unordered_map.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private, "<boost/ptr_container/serialize_ptr_array.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private, "<boost/ptr_container/serialize_ptr_circular_buffer.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private, "<boost/ptr_container/serialize_ptr_deque.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private, "<boost/ptr_container/serialize_ptr_list.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private, "<boost/ptr_container/serialize_ptr_set.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private, "<boost/ptr_container/serialize_ptr_unordered_set.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/serialize_reversible_cont.hpp>", private, "<boost/ptr_container/serialize_ptr_vector.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/void_ptr_iterator.hpp>", private, "<boost/ptr_container/ptr_sequence_adapter.hpp>", public ] },
+    { include: ["<boost/ptr_container/detail/void_ptr_iterator.hpp>", private, "<boost/ptr_container/ptr_set_adapter.hpp>", public ] },
+    { include: ["<boost/python/detail/borrowed_ptr.hpp>", private, "<boost/python/borrowed.hpp>", public ] },
+    { include: ["<boost/python/detail/caller.hpp>", private, "<boost/python/make_constructor.hpp>", public ] },
+    { include: ["<boost/python/detail/caller.hpp>", private, "<boost/python/make_function.hpp>", public ] },
+    { include: ["<boost/python/detail/caller.hpp>", private, "<boost/python/object/function_handle.hpp>", public ] },
+    { include: ["<boost/python/detail/construct.hpp>", private, "<boost/python/converter/obj_mgr_arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/convertible.hpp>", private, "<boost/python/cast.hpp>", public ] },
+    { include: ["<boost/python/detail/convertible.hpp>", private, "<boost/python/converter/arg_to_python.hpp>", public ] },
+    { include: ["<boost/python/detail/copy_ctor_mutates_rhs.hpp>", private, "<boost/python/extract.hpp>", public ] },
+    { include: ["<boost/python/detail/copy_ctor_mutates_rhs.hpp>", private, "<boost/python/object/forward.hpp>", public ] },
+    { include: ["<boost/python/detail/dealloc.hpp>", private, "<boost/python/opaque_pointer_converter.hpp>", public ] },
+    { include: ["<boost/python/detail/decref_guard.hpp>", private, "<boost/python/object/make_instance.hpp>", public ] },
+    { include: ["<boost/python/detail/defaults_def.hpp>", private, "<boost/python/overloads.hpp>", public ] },
+    { include: ["<boost/python/detail/def_helper_fwd.hpp>", private, "<boost/python/object_core.hpp>", public ] },
+    { include: ["<boost/python/detail/def_helper.hpp>", private, "<boost/python/class.hpp>", public ] },
+    { include: ["<boost/python/detail/def_helper.hpp>", private, "<boost/python/def.hpp>", public ] },
+    { include: ["<boost/python/detail/dependent.hpp>", private, "<boost/python/back_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/dependent.hpp>", private, "<boost/python/object_core.hpp>", public ] },
+    { include: ["<boost/python/detail/destroy.hpp>", private, "<boost/python/converter/obj_mgr_arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/destroy.hpp>", private, "<boost/python/converter/rvalue_from_python_data.hpp>", public ] },
+    { include: ["<boost/python/detail/exception_handler.hpp>", private, "<boost/python/exception_translator.hpp>", public ] },
+    { include: ["<boost/python/detail/force_instantiate.hpp>", private, "<boost/python/class.hpp>", public ] },
+    { include: ["<boost/python/detail/force_instantiate.hpp>", private, "<boost/python/object/class_metadata.hpp>", public ] },
+    { include: ["<boost/python/detail/force_instantiate.hpp>", private, "<boost/python/object/pointer_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/force_instantiate.hpp>", private, "<boost/python/object/value_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/force_instantiate.hpp>", private, "<boost/python/return_opaque_pointer.hpp>", public ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/converter/arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/converter/arg_to_python.hpp>", public ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/converter/object_manager.hpp>", public ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/copy_const_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/copy_non_const_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/data_members.hpp>", public ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/manage_new_object.hpp>", public ] },
+    { include: ["<boost/python/detail/indirect_traits.hpp>", private, "<boost/python/reference_existing_object.hpp>", public ] },
+    { include: ["<boost/python/detail/is_xxx.hpp>", private, "<boost/python/object_core.hpp>", public ] },
+    { include: ["<boost/python/detail/make_keyword_range_fn.hpp>", private, "<boost/python/init.hpp>", public ] },
+    { include: ["<boost/python/detail/mpl_lambda.hpp>", private, "<boost/python/args.hpp>", public ] },
+    { include: ["<boost/python/detail/msvc_typeinfo.hpp>", private, "<boost/python/type_id.hpp>", public ] },
+    { include: ["<boost/python/detail/none.hpp>", private, "<boost/python/converter/builtin_converters.hpp>", public ] },
+    { include: ["<boost/python/detail/none.hpp>", private, "<boost/python/make_constructor.hpp>", public ] },
+    { include: ["<boost/python/detail/none.hpp>", private, "<boost/python/object/make_instance.hpp>", public ] },
+    { include: ["<boost/python/detail/none.hpp>", private, "<boost/python/opaque_pointer_converter.hpp>", public ] },
+    { include: ["<boost/python/detail/none.hpp>", private, "<boost/python/return_arg.hpp>", public ] },
+    { include: ["<boost/python/detail/none.hpp>", private, "<boost/python/to_python_indirect.hpp>", public ] },
+    { include: ["<boost/python/detail/not_specified.hpp>", private, "<boost/python/class_fwd.hpp>", public ] },
+    { include: ["<boost/python/detail/not_specified.hpp>", private, "<boost/python/data_members.hpp>", public ] },
+    { include: ["<boost/python/detail/not_specified.hpp>", private, "<boost/python/object/class_metadata.hpp>", public ] },
+    { include: ["<boost/python/detail/not_specified.hpp>", private, "<boost/python/operators.hpp>", public ] },
+    { include: ["<boost/python/detail/nullary_function_adaptor.hpp>", private, "<boost/python/pure_virtual.hpp>", public ] },
+    { include: ["<boost/python/detail/operator_id.hpp>", private, "<boost/python/class.hpp>", public ] },
+    { include: ["<boost/python/detail/operator_id.hpp>", private, "<boost/python/operators.hpp>", public ] },
+    { include: ["<boost/python/detail/overloads_fwd.hpp>", private, "<boost/python/class.hpp>", public ] },
+    { include: ["<boost/python/detail/overloads_fwd.hpp>", private, "<boost/python/def.hpp>", public ] },
+    { include: ["<boost/python/detail/overloads_fwd.hpp>", private, "<boost/python/overloads.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/args_fwd.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/args.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/back_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/bases.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/base_type_traits.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/borrowed.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/call.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/call_method.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/cast.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/class_fwd.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/class.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/builtin_converters.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/obj_mgr_arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/pyobject_traits.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/pytype_function.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/pytype_object_mgr_traits.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/registrations.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/converter/to_python_function_type.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/copy_const_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/copy_non_const_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/data_members.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/default_call_policies.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/def.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/def_visitor.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/dict.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/enum.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/errors.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/exception_translator.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/extract.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/handle_fwd.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/handle.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/has_back_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/implicit.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/init.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/instance_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/iterator.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/list.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/long.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/lvalue_from_pytype.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/make_constructor.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/make_function.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/manage_new_object.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/module.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/module_init.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/numeric.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object_attributes.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/class.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object_core.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/function.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/function_object.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object_fwd.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/instance.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object_items.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/iterator.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/life_support.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/make_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/make_instance.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object_operators.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object/pickle_support.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object_protocol_core.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object_protocol.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/object_slices.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/opaque_pointer_converter.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/operators.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/other.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/overloads.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/override.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/pointee.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/proxy.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/ptr.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/raw_function.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/refcount.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/reference_existing_object.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/return_by_value.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/return_internal_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/return_opaque_pointer.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/return_value_policy.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/scope.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/self.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/signature.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/slice.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/slice_nil.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/ssize_t.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/stl_iterator.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/str.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/tag.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/to_python_converter.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/to_python_indirect.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/to_python_value.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/tuple.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/type_id.hpp>", public ] },
+    { include: ["<boost/python/detail/prefix.hpp>", private, "<boost/python/with_custodian_and_ward.hpp>", public ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/args.hpp>", public ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/call.hpp>", public ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/call_method.hpp>", public ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/object_core.hpp>", public ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/object/make_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/object/pointer_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/object/value_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/preprocessor.hpp>", private, "<boost/python/signature.hpp>", public ] },
+    { include: ["<boost/python/detail/python_type.hpp>", private, "<boost/python/object/make_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/raw_pyobject.hpp>", private, "<boost/python/back_reference.hpp>", public ] },
+    { include: ["<boost/python/detail/raw_pyobject.hpp>", private, "<boost/python/converter/obj_mgr_arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/raw_pyobject.hpp>", private, "<boost/python/converter/pytype_object_mgr_traits.hpp>", public ] },
+    { include: ["<boost/python/detail/raw_pyobject.hpp>", private, "<boost/python/handle.hpp>", public ] },
+    { include: ["<boost/python/detail/raw_pyobject.hpp>", private, "<boost/python/object_core.hpp>", public ] },
+    { include: ["<boost/python/detail/raw_pyobject.hpp>", private, "<boost/python/object/iterator.hpp>", public ] },
+    { include: ["<boost/python/detail/referent_storage.hpp>", private, "<boost/python/converter/arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/referent_storage.hpp>", private, "<boost/python/converter/obj_mgr_arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/referent_storage.hpp>", private, "<boost/python/converter/rvalue_from_python_data.hpp>", public ] },
+    { include: ["<boost/python/detail/scope.hpp>", private, "<boost/python/def.hpp>", public ] },
+    { include: ["<boost/python/detail/sfinae.hpp>", private, "<boost/python/wrapper.hpp>", public ] },
+    { include: ["<boost/python/detail/signature.hpp>", private, "<boost/python/object/function_doc_signature.hpp>", public ] },
+    { include: ["<boost/python/detail/signature.hpp>", private, "<boost/python/object/py_function.hpp>", public ] },
+    { include: ["<boost/python/detail/string_literal.hpp>", private, "<boost/python/converter/arg_to_python.hpp>", public ] },
+    { include: ["<boost/python/detail/string_literal.hpp>", private, "<boost/python/object_core.hpp>", public ] },
+    { include: ["<boost/python/detail/target.hpp>", private, "<boost/python/iterator.hpp>", public ] },
+    { include: ["<boost/python/detail/translate_exception.hpp>", private, "<boost/python/exception_translator.hpp>", public ] },
+    { include: ["<boost/python/detail/type_list.hpp>", private, "<boost/python/args.hpp>", public ] },
+    { include: ["<boost/python/detail/type_list.hpp>", private, "<boost/python/bases.hpp>", public ] },
+    { include: ["<boost/python/detail/type_list.hpp>", private, "<boost/python/init.hpp>", public ] },
+    { include: ["<boost/python/detail/type_list.hpp>", private, "<boost/python/signature.hpp>", public ] },
+    { include: ["<boost/python/detail/unwind_type.hpp>", private, "<boost/python/converter/pytype_function.hpp>", public ] },
+    { include: ["<boost/python/detail/unwrap_type_id.hpp>", private, "<boost/python/class.hpp>", public ] },
+    { include: ["<boost/python/detail/unwrap_wrapper.hpp>", private, "<boost/python/class.hpp>", public ] },
+    { include: ["<boost/python/detail/unwrap_wrapper.hpp>", private, "<boost/python/operators.hpp>", public ] },
+    { include: ["<boost/python/detail/value_arg.hpp>", private, "<boost/python/data_members.hpp>", public ] },
+    { include: ["<boost/python/detail/value_arg.hpp>", private, "<boost/python/default_call_policies.hpp>", public ] },
+    { include: ["<boost/python/detail/value_arg.hpp>", private, "<boost/python/object/forward.hpp>", public ] },
+    { include: ["<boost/python/detail/value_arg.hpp>", private, "<boost/python/return_arg.hpp>", public ] },
+    { include: ["<boost/python/detail/value_arg.hpp>", private, "<boost/python/return_by_value.hpp>", public ] },
+    { include: ["<boost/python/detail/value_arg.hpp>", private, "<boost/python/return_opaque_pointer.hpp>", public ] },
+    { include: ["<boost/python/detail/value_arg.hpp>", private, "<boost/python/to_python_value.hpp>", public ] },
+    { include: ["<boost/python/detail/value_is_shared_ptr.hpp>", private, "<boost/python/converter/arg_to_python.hpp>", public ] },
+    { include: ["<boost/python/detail/value_is_shared_ptr.hpp>", private, "<boost/python/to_python_value.hpp>", public ] },
+    { include: ["<boost/python/detail/void_ptr.hpp>", private, "<boost/python/converter/arg_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/void_ptr.hpp>", private, "<boost/python/converter/return_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/void_ptr.hpp>", private, "<boost/python/extract.hpp>", public ] },
+    { include: ["<boost/python/detail/void_ptr.hpp>", private, "<boost/python/lvalue_from_pytype.hpp>", public ] },
+    { include: ["<boost/python/detail/void_return.hpp>", private, "<boost/python/call.hpp>", public ] },
+    { include: ["<boost/python/detail/void_return.hpp>", private, "<boost/python/call_method.hpp>", public ] },
+    { include: ["<boost/python/detail/void_return.hpp>", private, "<boost/python/converter/return_from_python.hpp>", public ] },
+    { include: ["<boost/python/detail/void_return.hpp>", private, "<boost/python/extract.hpp>", public ] },
+    { include: ["<boost/python/detail/wrapper_base.hpp>", private, "<boost/python/object/pointer_holder.hpp>", public ] },
+    { include: ["<boost/python/detail/wrapper_base.hpp>", private, "<boost/python/wrapper.hpp>", public ] },
+    { include: ["<boost/python/suite/indexing/detail/indexing_suite_detail.hpp>", private, "<boost/python/suite/indexing/indexing_suite.hpp>", public ] },
+    { include: ["<boost/random/detail/auto_link.hpp>", private, "<boost/random/random_device.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/additive_combine.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/bernoulli_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/binomial_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/cauchy_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/chi_squared_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/discard_block.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/discrete_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/exponential_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/gamma_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/geometric_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/independent_bits.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/inversive_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/lagged_fibonacci.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/linear_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/linear_feedback_shift.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/lognormal_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/mersenne_twister.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/negative_binomial_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/normal_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/piecewise_constant_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/piecewise_linear_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/poisson_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/subtract_with_carry.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/triangle_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/uniform_01.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/uniform_int_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/uniform_on_sphere.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/uniform_real_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/uniform_smallint.hpp>", public ] },
+    { include: ["<boost/random/detail/config.hpp>", private, "<boost/random/xor_combine.hpp>", public ] },
+    { include: ["<boost/random/detail/const_mod.hpp>", private, "<boost/random/inversive_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/const_mod.hpp>", private, "<boost/random/linear_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/binomial_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/discrete_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/inversive_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/linear_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/poisson_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/random_number_generator.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/shuffle_order.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/uniform_01.hpp>", public ] },
+    { include: ["<boost/random/detail/disable_warnings.hpp>", private, "<boost/random/variate_generator.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/binomial_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/discrete_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/inversive_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/linear_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/poisson_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/random_number_generator.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/shuffle_order.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/uniform_01.hpp>", public ] },
+    { include: ["<boost/random/detail/enable_warnings.hpp>", private, "<boost/random/variate_generator.hpp>", public ] },
+    { include: ["<boost/random/detail/generator_bits.hpp>", private, "<boost/random/generate_canonical.hpp>", public ] },
+    { include: ["<boost/random/detail/generator_seed_seq.hpp>", private, "<boost/random/lagged_fibonacci.hpp>", public ] },
+    { include: ["<boost/random/detail/generator_seed_seq.hpp>", private, "<boost/random/mersenne_twister.hpp>", public ] },
+    { include: ["<boost/random/detail/generator_seed_seq.hpp>", private, "<boost/random/subtract_with_carry.hpp>", public ] },
+    { include: ["<boost/random/detail/integer_log2.hpp>", private, "<boost/random/independent_bits.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/additive_combine.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/bernoulli_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/cauchy_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/discrete_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/exponential_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/extreme_value_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/fisher_f_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/geometric_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/independent_bits.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/inversive_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/lagged_fibonacci.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/linear_feedback_shift.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/lognormal_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/normal_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/piecewise_constant_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/piecewise_linear_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/shuffle_order.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/student_t_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/subtract_with_carry.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/triangle_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/uniform_int_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/uniform_on_sphere.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/uniform_real_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/uniform_smallint.hpp>", public ] },
+    { include: ["<boost/random/detail/operators.hpp>", private, "<boost/random/weibull_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/ptr_helper.hpp>", private, "<boost/random/mersenne_twister.hpp>", public ] },
+    { include: ["<boost/random/detail/ptr_helper.hpp>", private, "<boost/random/uniform_01.hpp>", public ] },
+    { include: ["<boost/random/detail/ptr_helper.hpp>", private, "<boost/random/variate_generator.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/additive_combine.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/discard_block.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/independent_bits.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/inversive_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/lagged_fibonacci.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/linear_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/linear_feedback_shift.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/mersenne_twister.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/shuffle_order.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/subtract_with_carry.hpp>", public ] },
+    { include: ["<boost/random/detail/seed.hpp>", private, "<boost/random/xor_combine.hpp>", public ] },
+    { include: ["<boost/random/detail/seed_impl.hpp>", private, "<boost/random/independent_bits.hpp>", public ] },
+    { include: ["<boost/random/detail/seed_impl.hpp>", private, "<boost/random/inversive_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/seed_impl.hpp>", private, "<boost/random/linear_congruential.hpp>", public ] },
+    { include: ["<boost/random/detail/seed_impl.hpp>", private, "<boost/random/linear_feedback_shift.hpp>", public ] },
+    { include: ["<boost/random/detail/seed_impl.hpp>", private, "<boost/random/mersenne_twister.hpp>", public ] },
+    { include: ["<boost/random/detail/seed_impl.hpp>", private, "<boost/random/subtract_with_carry.hpp>", public ] },
+    { include: ["<boost/random/detail/seed_impl.hpp>", private, "<boost/random/xor_combine.hpp>", public ] },
+    { include: ["<boost/random/detail/signed_unsigned_tools.hpp>", private, "<boost/random/generate_canonical.hpp>", public ] },
+    { include: ["<boost/random/detail/signed_unsigned_tools.hpp>", private, "<boost/random/independent_bits.hpp>", public ] },
+    { include: ["<boost/random/detail/signed_unsigned_tools.hpp>", private, "<boost/random/shuffle_order.hpp>", public ] },
+    { include: ["<boost/random/detail/signed_unsigned_tools.hpp>", private, "<boost/random/uniform_int_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/signed_unsigned_tools.hpp>", private, "<boost/random/uniform_real_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/signed_unsigned_tools.hpp>", private, "<boost/random/uniform_smallint.hpp>", public ] },
+    { include: ["<boost/random/detail/uniform_int_float.hpp>", private, "<boost/random/uniform_int_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/vector_io.hpp>", private, "<boost/random/discrete_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/vector_io.hpp>", private, "<boost/random/piecewise_constant_distribution.hpp>", public ] },
+    { include: ["<boost/random/detail/vector_io.hpp>", private, "<boost/random/piecewise_linear_distribution.hpp>", public ] },
+    { include: ["<boost/range/detail/any_iterator.hpp>", private, "<boost/range/any_range.hpp>", public ] },
+    { include: ["<boost/range/detail/as_literal.hpp>", private, "<boost/range/as_literal.hpp>", public ] },
+    { include: ["<boost/range/detail/begin.hpp>", private, "<boost/range/begin.hpp>", public ] },
+    { include: ["<boost/range/detail/collection_traits.hpp>", private, "<boost/range.hpp>", public ] },
+    { include: ["<boost/range/detail/const_iterator.hpp>", private, "<boost/range/const_iterator.hpp>", public ] },
+    { include: ["<boost/range/detail/end.hpp>", private, "<boost/range/end.hpp>", public ] },
+    { include: ["<boost/range/detail/extract_optional_type.hpp>", private, "<boost/range/const_iterator.hpp>", public ] },
+    { include: ["<boost/range/detail/extract_optional_type.hpp>", private, "<boost/range/mutable_iterator.hpp>", public ] },
+    { include: ["<boost/range/detail/implementation_help.hpp>", private, "<boost/range/end.hpp>", public ] },
+    { include: ["<boost/range/detail/iterator.hpp>", private, "<boost/range/mutable_iterator.hpp>", public ] },
+    { include: ["<boost/range/detail/join_iterator.hpp>", private, "<boost/range/join.hpp>", public ] },
+    { include: ["<boost/range/detail/microsoft.hpp>", private, "<boost/range/atl.hpp>", public ] },
+    { include: ["<boost/range/detail/microsoft.hpp>", private, "<boost/range/mfc.hpp>", public ] },
+    { include: ["<boost/range/detail/misc_concept.hpp>", private, "<boost/range/concepts.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/adjacent_find.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/find_end.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/find_first_of.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/find.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/find_if.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/lower_bound.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/max_element.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/min_element.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/partition.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/remove.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/remove_if.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/reverse.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/search.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/search_n.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/stable_partition.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/unique.hpp>", public ] },
+    { include: ["<boost/range/detail/range_return.hpp>", private, "<boost/range/algorithm/upper_bound.hpp>", public ] },
+    { include: ["<boost/range/detail/safe_bool.hpp>", private, "<boost/range/iterator_range_core.hpp>", public ] },
+    { include: ["<boost/range/detail/size_type.hpp>", private, "<boost/range/size_type.hpp>", public ] },
+    { include: ["<boost/range/detail/str_types.hpp>", private, "<boost/range/as_array.hpp>", public ] },
+    { include: ["<boost/range/detail/str_types.hpp>", private, "<boost/range/as_literal.hpp>", public ] },
+    { include: ["<boost/ratio/detail/mpl/abs.hpp>", private, "<boost/ratio/mpl/abs.hpp>", public ] },
+    { include: ["<boost/ratio/detail/mpl/abs.hpp>", private, "<boost/ratio/ratio.hpp>", public ] },
+    { include: ["<boost/ratio/detail/mpl/gcd.hpp>", private, "<boost/ratio/mpl/gcd.hpp>", public ] },
+    { include: ["<boost/ratio/detail/mpl/gcd.hpp>", private, "<boost/ratio/ratio.hpp>", public ] },
+    { include: ["<boost/ratio/detail/mpl/lcm.hpp>", private, "<boost/ratio/mpl/lcm.hpp>", public ] },
+    { include: ["<boost/ratio/detail/mpl/lcm.hpp>", private, "<boost/ratio/ratio.hpp>", public ] },
+    { include: ["<boost/ratio/detail/mpl/sign.hpp>", private, "<boost/ratio/mpl/sign.hpp>", public ] },
+    { include: ["<boost/ratio/detail/mpl/sign.hpp>", private, "<boost/ratio/ratio.hpp>", public ] },
+    { include: ["<boost/ratio/detail/overflow_helpers.hpp>", private, "<boost/ratio/ratio.hpp>", public ] },
+    { include: ["<boost/ratio/detail/ratio_io.hpp>", private, "<boost/ratio/ratio_io.hpp>", public ] },
+    { include: ["<boost/serialization/detail/get_data.hpp>", private, "<boost/serialization/valarray.hpp>", public ] },
+    { include: ["<boost/serialization/detail/get_data.hpp>", private, "<boost/serialization/vector.hpp>", public ] },
+    { include: ["<boost/serialization/detail/shared_ptr_132.hpp>", private, "<boost/serialization/shared_ptr_132.hpp>", public ] },
+    { include: ["<boost/serialization/detail/stack_constructor.hpp>", private, "<boost/serialization/collections_load_imp.hpp>", public ] },
+    { include: ["<boost/serialization/detail/stack_constructor.hpp>", private, "<boost/serialization/optional.hpp>", public ] },
+    { include: ["<boost/signals2/detail/foreign_ptr.hpp>", private, "<boost/signals2/slot_base.hpp>", public ] },
+    { include: ["<boost/signals2/detail/lwm_nop.hpp>", private, "<boost/signals2/mutex.hpp>", public ] },
+    { include: ["<boost/signals2/detail/lwm_pthreads.hpp>", private, "<boost/signals2/mutex.hpp>", public ] },
+    { include: ["<boost/signals2/detail/lwm_win32_cs.hpp>", private, "<boost/signals2/mutex.hpp>", public ] },
+    { include: ["<boost/signals2/detail/null_output_iterator.hpp>", private, "<boost/signals2/connection.hpp>", public ] },
+    { include: ["<boost/signals2/detail/preprocessed_arg_type.hpp>", private, "<boost/signals2/preprocessed_signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/preprocessed_arg_type.hpp>", private, "<boost/signals2/preprocessed_slot.hpp>", public ] },
+    { include: ["<boost/signals2/detail/replace_slot_function.hpp>", private, "<boost/signals2/signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/result_type_wrapper.hpp>", private, "<boost/signals2/signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/signals_common.hpp>", private, "<boost/signals2/signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/signals_common.hpp>", private, "<boost/signals2/slot.hpp>", public ] },
+    { include: ["<boost/signals2/detail/signals_common_macros.hpp>", private, "<boost/signals2/signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/signals_common_macros.hpp>", private, "<boost/signals2/slot.hpp>", public ] },
+    { include: ["<boost/signals2/detail/signal_template.hpp>", private, "<boost/signals2/variadic_signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/slot_call_iterator.hpp>", private, "<boost/signals2/signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/slot_groups.hpp>", private, "<boost/signals2/signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/slot_template.hpp>", private, "<boost/signals2/variadic_slot.hpp>", public ] },
+    { include: ["<boost/signals2/detail/tracked_objects_visitor.hpp>", private, "<boost/signals2/slot.hpp>", public ] },
+    { include: ["<boost/signals2/detail/unique_lock.hpp>", private, "<boost/signals2/connection.hpp>", public ] },
+    { include: ["<boost/signals2/detail/unique_lock.hpp>", private, "<boost/signals2/signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/variadic_arg_type.hpp>", private, "<boost/signals2/variadic_signal.hpp>", public ] },
+    { include: ["<boost/signals2/detail/variadic_arg_type.hpp>", private, "<boost/signals2/variadic_slot.hpp>", public ] },
+    { include: ["<boost/signals2/detail/variadic_slot_invoker.hpp>", private, "<boost/signals2/variadic_signal.hpp>", public ] },
+    { include: ["<boost/signals/detail/signal_base.hpp>", private, "<boost/signals/signal_template.hpp>", public ] },
+    { include: ["<boost/signals/detail/signals_common.hpp>", private, "<boost/signals/connection.hpp>", public ] },
+    { include: ["<boost/signals/detail/signals_common.hpp>", private, "<boost/signals/slot.hpp>", public ] },
+    { include: ["<boost/signals/detail/slot_call_iterator.hpp>", private, "<boost/signals/signal_template.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/allocate_array_helper.hpp>", private, "<boost/smart_ptr/allocate_shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/array_deleter.hpp>", private, "<boost/smart_ptr/allocate_shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/array_deleter.hpp>", private, "<boost/smart_ptr/make_shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/array_traits.hpp>", private, "<boost/smart_ptr/allocate_shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/array_traits.hpp>", private, "<boost/smart_ptr/make_shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/atomic_count.hpp>", private, "<boost/smart_ptr/intrusive_ref_counter.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/make_array_helper.hpp>", private, "<boost/smart_ptr/make_shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/operator_bool.hpp>", private, "<boost/smart_ptr/intrusive_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/operator_bool.hpp>", private, "<boost/smart_ptr/scoped_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/operator_bool.hpp>", private, "<boost/smart_ptr/scoped_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/operator_bool.hpp>", private, "<boost/smart_ptr/shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/operator_bool.hpp>", private, "<boost/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/shared_array_nmt.hpp>", private, "<boost/smart_ptr/shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/shared_count.hpp>", private, "<boost/smart_ptr/shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/shared_count.hpp>", private, "<boost/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/shared_count.hpp>", private, "<boost/smart_ptr/weak_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/shared_ptr_nmt.hpp>", private, "<boost/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_convertible.hpp>", private, "<boost/smart_ptr/intrusive_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_convertible.hpp>", private, "<boost/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_forward.hpp>", private, "<boost/smart_ptr/make_shared_object.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_if_array.hpp>", private, "<boost/smart_ptr/allocate_shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_if_array.hpp>", private, "<boost/smart_ptr/make_shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/spinlock_pool.hpp>", private, "<boost/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_nullptr_t.hpp>", private, "<boost/smart_ptr/intrusive_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_nullptr_t.hpp>", private, "<boost/smart_ptr/scoped_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_nullptr_t.hpp>", private, "<boost/smart_ptr/scoped_ptr.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_nullptr_t.hpp>", private, "<boost/smart_ptr/shared_array.hpp>", public ] },
+    { include: ["<boost/smart_ptr/detail/sp_nullptr_t.hpp>", private, "<boost/smart_ptr/shared_ptr.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/alternative.ipp>", private, "<boost/spirit/home/classic/core/composite/alternative.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/difference.ipp>", private, "<boost/spirit/home/classic/core/composite/difference.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/directives.ipp>", private, "<boost/spirit/home/classic/core/composite/directives.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/directives.ipp>", private, "<boost/spirit/home/classic/core/primitives/primitives.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/exclusive_or.ipp>", private, "<boost/spirit/home/classic/core/composite/exclusive_or.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/intersection.ipp>", private, "<boost/spirit/home/classic/core/composite/intersection.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/kleene_star.ipp>", private, "<boost/spirit/home/classic/core/composite/kleene_star.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/list.ipp>", private, "<boost/spirit/home/classic/core/composite/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/optional.ipp>", private, "<boost/spirit/home/classic/core/composite/optional.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/positive.ipp>", private, "<boost/spirit/home/classic/core/composite/positive.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/sequence.ipp>", private, "<boost/spirit/home/classic/core/composite/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/sequential_and.ipp>", private, "<boost/spirit/home/classic/core/composite/sequential_and.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/composite/impl/sequential_or.ipp>", private, "<boost/spirit/home/classic/core/composite/sequential_or.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/impl/match_attr_traits.ipp>", private, "<boost/spirit/home/classic/core/match.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/impl/match.ipp>", private, "<boost/spirit/home/classic/core/match.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/impl/parser.ipp>", private, "<boost/spirit/home/classic/core/parser.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/non_terminal/impl/grammar.ipp>", private, "<boost/spirit/home/classic/core/non_terminal/grammar.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/non_terminal/impl/rule.ipp>", private, "<boost/spirit/home/classic/core/non_terminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/non_terminal/impl/rule.ipp>", private, "<boost/spirit/home/classic/dynamic/stored_rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/non_terminal/impl/static.hpp>", private, "<boost/spirit/include/classic_static.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/non_terminal/impl/subrule.ipp>", private, "<boost/spirit/home/classic/core/non_terminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/primitives/impl/numerics.ipp>", private, "<boost/spirit/home/classic/core/primitives/numerics.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/primitives/impl/primitives.ipp>", private, "<boost/spirit/home/classic/core/primitives/primitives.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/primitives/impl/primitives.ipp>", private, "<boost/spirit/home/classic/core/scanner/skipper.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/core/scanner/impl/skipper.ipp>", private, "<boost/spirit/home/classic/core/scanner/skipper.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/debug/impl/parser_names.ipp>", private, "<boost/spirit/home/classic/debug/parser_names.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/dynamic/impl/conditions.ipp>", private, "<boost/spirit/home/classic/dynamic/for.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/dynamic/impl/conditions.ipp>", private, "<boost/spirit/home/classic/dynamic/if.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/dynamic/impl/conditions.ipp>", private, "<boost/spirit/home/classic/dynamic/while.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/dynamic/impl/select.ipp>", private, "<boost/spirit/home/classic/dynamic/select.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/dynamic/impl/switch.ipp>", private, "<boost/spirit/home/classic/dynamic/switch.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/error_handling/impl/exceptions.ipp>", private, "<boost/spirit/home/classic/error_handling/exceptions.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/iterator/impl/file_iterator.ipp>", private, "<boost/spirit/home/classic/iterator/file_iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/iterator/impl/position_iterator.ipp>", private, "<boost/spirit/home/classic/iterator/position_iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/meta/impl/fundamental.ipp>", private, "<boost/spirit/home/classic/meta/fundamental.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/meta/impl/parser_traits.ipp>", private, "<boost/spirit/home/classic/meta/parser_traits.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/meta/impl/refactoring.ipp>", private, "<boost/spirit/home/classic/meta/refactoring.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/meta/impl/traverse.ipp>", private, "<boost/spirit/home/classic/meta/traverse.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/symbols/impl/symbols.ipp>", private, "<boost/spirit/home/classic/symbols/symbols.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/tree/impl/parse_tree_utils.ipp>", private, "<boost/spirit/home/classic/tree/parse_tree_utils.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/tree/impl/tree_to_xml.ipp>", private, "<boost/spirit/home/classic/tree/tree_to_xml.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/chset/basic_chset.hpp>", private, "<boost/spirit/home/classic/utility/chset.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/chset/basic_chset.hpp>", private, "<boost/spirit/include/classic_basic_chset.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/chset.ipp>", private, "<boost/spirit/home/classic/utility/chset.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/chset_operators.ipp>", private, "<boost/spirit/home/classic/utility/chset_operators.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/chset/range_run.hpp>", private, "<boost/spirit/include/classic_range_run.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/confix.ipp>", private, "<boost/spirit/home/classic/utility/confix.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/escape_char.ipp>", private, "<boost/spirit/home/classic/utility/escape_char.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/lists.ipp>", private, "<boost/spirit/home/classic/utility/lists.hpp>", public ] },
+    { include: ["<boost/spirit/home/classic/utility/impl/regex.ipp>", private, "<boost/spirit/home/classic/utility/regex.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/alternative_function.hpp>", private, "<boost/spirit/home/karma/operator/alternative.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/as.hpp>", private, "<boost/spirit/home/karma/directive/as.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/action/action.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/auxiliary/attr_cast.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/auxiliary/eol.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/auxiliary/lazy.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/as.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/buffer.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/center_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/columns.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/delimit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/duplicate.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/left_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/maxwidth.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/no_delimit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/omit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/repeat.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/right_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/directive/verbatim.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/operator/and_predicate.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/operator/not_predicate.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/operator/optional.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/operator/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/phoenix_attributes.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/home/karma/string/symbols.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/repository/home/karma/directive/confix.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/attributes.hpp>", private, "<boost/spirit/repository/home/karma/nonterminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/default_width.hpp>", private, "<boost/spirit/home/karma/directive/center_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/default_width.hpp>", private, "<boost/spirit/home/karma/directive/columns.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/default_width.hpp>", private, "<boost/spirit/home/karma/directive/left_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/default_width.hpp>", private, "<boost/spirit/home/karma/directive/maxwidth.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/default_width.hpp>", private, "<boost/spirit/home/karma/directive/right_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/enable_lit.hpp>", private, "<boost/spirit/home/karma/char/char.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/enable_lit.hpp>", private, "<boost/spirit/home/karma/numeric/bool.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/enable_lit.hpp>", private, "<boost/spirit/home/karma/numeric/int.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/enable_lit.hpp>", private, "<boost/spirit/home/karma/numeric/real.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/enable_lit.hpp>", private, "<boost/spirit/home/karma/numeric/uint.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/enable_lit.hpp>", private, "<boost/spirit/home/karma/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/binary/binary.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/char/char_generator.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/numeric/bool.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/numeric/int.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/numeric/real.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/numeric/uint.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/extract_from.hpp>", private, "<boost/spirit/home/karma/string/symbols.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/fail_function.hpp>", private, "<boost/spirit/home/karma/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/fail_function.hpp>", private, "<boost/spirit/home/karma/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/fail_function.hpp>", private, "<boost/spirit/home/karma/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/fail_function.hpp>", private, "<boost/spirit/home/karma/operator/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_auto.hpp>", private, "<boost/spirit/home/karma/auto.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_auto.hpp>", private, "<boost/spirit/include/karma_generate_auto.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate.hpp>", private, "<boost/spirit/home/karma/generate.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/auxiliary/eol.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/binary/binary.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/binary/padding.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/char/char_class.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/char/char_generator.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/generate_to.hpp>", private, "<boost/spirit/home/karma/char/char.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/char/char_class.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/char/char.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/numeric/bool.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/numeric/int.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/numeric/real.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/numeric/uint.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_casetag.hpp>", private, "<boost/spirit/home/karma/string/symbols.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_stricttag.hpp>", private, "<boost/spirit/home/karma/directive/repeat.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_stricttag.hpp>", private, "<boost/spirit/home/karma/operator/alternative.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_stricttag.hpp>", private, "<boost/spirit/home/karma/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_stricttag.hpp>", private, "<boost/spirit/home/karma/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_stricttag.hpp>", private, "<boost/spirit/home/karma/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/get_stricttag.hpp>", private, "<boost/spirit/home/karma/operator/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/indirect_iterator.hpp>", private, "<boost/spirit/home/karma/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/indirect_iterator.hpp>", private, "<boost/spirit/home/karma/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/indirect_iterator.hpp>", private, "<boost/spirit/home/karma/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/indirect_iterator.hpp>", private, "<boost/spirit/home/karma/operator/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/indirect_iterator.hpp>", private, "<boost/spirit/home/karma/phoenix_attributes.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/directive/as.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/directive/buffer.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/directive/center_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/directive/left_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/directive/maxwidth.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/directive/repeat.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/directive/right_alignment.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/operator/and_predicate.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/operator/not_predicate.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/output_iterator.hpp>", private, "<boost/spirit/home/karma/stream/format_manip.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/pass_container.hpp>", private, "<boost/spirit/home/karma/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/pass_container.hpp>", private, "<boost/spirit/home/karma/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/pass_container.hpp>", private, "<boost/spirit/home/karma/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/pass_container.hpp>", private, "<boost/spirit/home/karma/operator/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/string_compare.hpp>", private, "<boost/spirit/home/karma/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/string_generate.hpp>", private, "<boost/spirit/home/karma/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/string_generate.hpp>", private, "<boost/spirit/home/karma/string/symbols.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/unused_delimiter.hpp>", private, "<boost/spirit/home/karma/delimit_out.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/unused_delimiter.hpp>", private, "<boost/spirit/home/karma/directive/delimit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/unused_delimiter.hpp>", private, "<boost/spirit/home/karma/directive/no_delimit.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/detail/unused_delimiter.hpp>", private, "<boost/spirit/home/karma/directive/verbatim.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/nonterminal/detail/fcall.hpp>", private, "<boost/spirit/home/karma/nonterminal/grammar.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/nonterminal/detail/fcall.hpp>", private, "<boost/spirit/home/karma/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/nonterminal/detail/fcall.hpp>", private, "<boost/spirit/repository/home/karma/nonterminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/nonterminal/detail/generator_binder.hpp>", private, "<boost/spirit/home/karma/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/nonterminal/detail/generator_binder.hpp>", private, "<boost/spirit/repository/home/karma/nonterminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/nonterminal/detail/parameterized.hpp>", private, "<boost/spirit/home/karma/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/nonterminal/detail/parameterized.hpp>", private, "<boost/spirit/repository/home/karma/nonterminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/numeric/detail/bool_utils.hpp>", private, "<boost/spirit/home/karma/numeric/bool.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private, "<boost/spirit/home/karma/numeric/bool_policies.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private, "<boost/spirit/home/karma/numeric/int.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/numeric/detail/numeric_utils.hpp>", private, "<boost/spirit/home/karma/numeric/uint.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/numeric/detail/real_utils.hpp>", private, "<boost/spirit/home/karma/numeric/real.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/numeric/detail/real_utils.hpp>", private, "<boost/spirit/home/karma/numeric/real_policies.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/stream/detail/format_manip_auto.hpp>", private, "<boost/spirit/home/karma/format_auto.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/stream/detail/format_manip.hpp>", private, "<boost/spirit/home/karma/stream/format_manip.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/stream/detail/format_manip.hpp>", private, "<boost/spirit/home/karma/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/karma/stream/detail/iterator_sink.hpp>", private, "<boost/spirit/home/karma/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/lex/detail/sequence_function.hpp>", private, "<boost/spirit/home/lex/lexer/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/core/detail/actor.hpp>", private, "<boost/spirit/home/phoenix/core/actor.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/core/detail/basic_environment.hpp>", private, "<boost/spirit/home/phoenix/core/basic_environment.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/core/detail/compose.hpp>", private, "<boost/spirit/home/phoenix/core/compose.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/core/detail/composite_eval.hpp>", private, "<boost/spirit/home/phoenix/core/composite.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/core/detail/composite.hpp>", private, "<boost/spirit/home/phoenix/core/composite.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/local_reference.hpp>", private, "<boost/spirit/home/phoenix/scope/lambda.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/local_reference.hpp>", private, "<boost/spirit/home/phoenix/scope/let.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/local_reference.hpp>", private, "<boost/spirit/home/phoenix/scope/local_variable.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/type_deduction.hpp>", private, "<boost/spirit/home/phoenix/operator/arithmetic.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/type_deduction.hpp>", private, "<boost/spirit/home/phoenix/operator/bitwise.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/type_deduction.hpp>", private, "<boost/spirit/home/phoenix/operator/comparison.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/type_deduction.hpp>", private, "<boost/spirit/home/phoenix/operator/if_else.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/type_deduction.hpp>", private, "<boost/spirit/home/phoenix/operator/logical.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/detail/type_deduction.hpp>", private, "<boost/spirit/home/phoenix/operator/self.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/object/detail/construct_eval.hpp>", private, "<boost/spirit/home/phoenix/object/construct.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/object/detail/construct.hpp>", private, "<boost/spirit/home/phoenix/object/construct.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/object/detail/new_eval.hpp>", private, "<boost/spirit/home/phoenix/object/new.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/object/detail/new.hpp>", private, "<boost/spirit/home/phoenix/object/new.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/arithmetic.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/bitwise.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/comparison.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/logical.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/self.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/arithmetic.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/bitwise.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/comparison.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/logical.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/binary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/self.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/io.hpp>", private, "<boost/spirit/home/phoenix/operator/io.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/mem_fun_ptr_gen.hpp>", private, "<boost/spirit/home/phoenix/operator/member.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/arithmetic.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/bitwise.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/comparison.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/logical.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_compose.hpp>", private, "<boost/spirit/home/phoenix/operator/self.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/arithmetic.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/bitwise.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/comparison.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/logical.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/operator/detail/unary_eval.hpp>", private, "<boost/spirit/home/phoenix/operator/self.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/scope/detail/local_gen.hpp>", private, "<boost/spirit/home/phoenix/scope/lambda.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/scope/detail/local_gen.hpp>", private, "<boost/spirit/home/phoenix/scope/let.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/scope/detail/local_variable.hpp>", private, "<boost/spirit/home/phoenix/scope/lambda.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/scope/detail/local_variable.hpp>", private, "<boost/spirit/home/phoenix/scope/let.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/scope/detail/local_variable.hpp>", private, "<boost/spirit/home/phoenix/scope/local_variable.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/statement/detail/catch_all_eval.hpp>", private, "<boost/spirit/home/phoenix/statement/try_catch.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/statement/detail/catch_composite.hpp>", private, "<boost/spirit/home/phoenix/statement/try_catch.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/statement/detail/catch_eval.hpp>", private, "<boost/spirit/home/phoenix/statement/try_catch.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/statement/detail/switch_eval.hpp>", private, "<boost/spirit/home/phoenix/statement/switch.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/statement/detail/switch.hpp>", private, "<boost/spirit/home/phoenix/statement/switch.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/begin.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/iteration.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/begin.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/begin.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/decay_array.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/decay_array.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/end.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/iteration.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/end.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/end.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_equal_range.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_find.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_lower_bound.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_remove.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_remove_if.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_reverse.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_sort.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_unique.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/transformation.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/algorithm/detail/has_upper_bound.hpp>", private, "<boost/spirit/home/phoenix/stl/algorithm/querying.hpp>", public ] },
+    { include: ["<boost/spirit/home/phoenix/stl/container/detail/container.hpp>", private, "<boost/spirit/home/phoenix/stl/container/container.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/alternative_function.hpp>", private, "<boost/spirit/home/qi/operator/alternative.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/lexer/lexer.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/functor_data.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/position_token.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/token.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/lexer/token_def.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/qi/plain_raw_token.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/qi/plain_token.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/qi/plain_tokenid.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/lex/qi/plain_tokenid_mask.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/auxiliary/attr.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/binary/binary.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/char/char_parser.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/directive/as.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/directive/matches.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/directive/raw.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/numeric/bool_policies.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/numeric/numeric_utils.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/operator/optional.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/assign_to.hpp>", private, "<boost/spirit/home/qi/string/symbols.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/lex/qi/plain_raw_token.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/lex/qi/plain_token.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/lex/qi/plain_tokenid.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/lex/qi/plain_tokenid_mask.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/lex/qi/state_switcher.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/action/action.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/auxiliary/attr_cast.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/auxiliary/lazy.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/binary/binary.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/directive/lexeme.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/directive/repeat.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/directive/skip.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/alternative.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/and_predicate.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/difference.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/not_predicate.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/optional.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/permutation.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/sequence_base.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/home/qi/operator/sequential_or.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/repository/home/qi/directive/confix.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/repository/home/qi/directive/distinct.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/repository/home/qi/directive/kwd.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/repository/home/qi/directive/seek.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/repository/home/qi/nonterminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/attributes.hpp>", private, "<boost/spirit/repository/home/qi/operator/keywords.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/construct.hpp>", private, "<boost/spirit/home/lex/lexer/token_def.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/enable_lit.hpp>", private, "<boost/spirit/home/qi/char/char.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/enable_lit.hpp>", private, "<boost/spirit/home/qi/numeric/bool.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/enable_lit.hpp>", private, "<boost/spirit/home/qi/numeric/int.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/enable_lit.hpp>", private, "<boost/spirit/home/qi/numeric/real.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/enable_lit.hpp>", private, "<boost/spirit/home/qi/numeric/uint.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/enable_lit.hpp>", private, "<boost/spirit/home/qi/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/expect_function.hpp>", private, "<boost/spirit/home/qi/operator/expect.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/fail_function.hpp>", private, "<boost/spirit/home/qi/directive/repeat.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/fail_function.hpp>", private, "<boost/spirit/home/qi/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/fail_function.hpp>", private, "<boost/spirit/home/qi/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/fail_function.hpp>", private, "<boost/spirit/home/qi/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/fail_function.hpp>", private, "<boost/spirit/home/qi/operator/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/fail_function.hpp>", private, "<boost/spirit/repository/home/qi/directive/kwd.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/parse_auto.hpp>", private, "<boost/spirit/home/qi/auto.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/parse_auto.hpp>", private, "<boost/spirit/include/qi_parse_auto.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/parse.hpp>", private, "<boost/spirit/home/qi/parse.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/pass_container.hpp>", private, "<boost/spirit/home/qi/directive/repeat.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/pass_container.hpp>", private, "<boost/spirit/home/qi/operator/kleene.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/pass_container.hpp>", private, "<boost/spirit/home/qi/operator/list.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/pass_container.hpp>", private, "<boost/spirit/home/qi/operator/plus.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/pass_container.hpp>", private, "<boost/spirit/home/qi/operator/sequence_base.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/pass_function.hpp>", private, "<boost/spirit/home/qi/operator/sequential_or.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/permute_function.hpp>", private, "<boost/spirit/home/qi/operator/permutation.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/permute_function.hpp>", private, "<boost/spirit/repository/home/qi/operator/keywords.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/string_parse.hpp>", private, "<boost/spirit/home/qi/numeric/bool_policies.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/string_parse.hpp>", private, "<boost/spirit/home/qi/numeric/real_policies.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/string_parse.hpp>", private, "<boost/spirit/home/qi/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/string_parse.hpp>", private, "<boost/spirit/home/qi/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/unused_skipper.hpp>", private, "<boost/spirit/home/qi/directive/lexeme.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/unused_skipper.hpp>", private, "<boost/spirit/home/qi/directive/no_skip.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/unused_skipper.hpp>", private, "<boost/spirit/home/qi/directive/skip.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/unused_skipper.hpp>", private, "<boost/spirit/home/qi/skip_over.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/detail/unused_skipper.hpp>", private, "<boost/spirit/repository/home/qi/directive/distinct.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/nonterminal/detail/fcall.hpp>", private, "<boost/spirit/home/qi/nonterminal/grammar.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/nonterminal/detail/fcall.hpp>", private, "<boost/spirit/home/qi/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/nonterminal/detail/fcall.hpp>", private, "<boost/spirit/repository/home/qi/nonterminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/nonterminal/detail/parameterized.hpp>", private, "<boost/spirit/home/qi/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/nonterminal/detail/parameterized.hpp>", private, "<boost/spirit/repository/home/qi/nonterminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/nonterminal/detail/parser_binder.hpp>", private, "<boost/spirit/home/qi/nonterminal/rule.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/nonterminal/detail/parser_binder.hpp>", private, "<boost/spirit/repository/home/qi/nonterminal/subrule.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/numeric/detail/numeric_utils.hpp>", private, "<boost/spirit/home/qi/numeric/numeric_utils.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/numeric/detail/real_impl.hpp>", private, "<boost/spirit/home/qi/numeric/real.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/stream/detail/iterator_source.hpp>", private, "<boost/spirit/home/qi/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/stream/detail/match_manip_auto.hpp>", private, "<boost/spirit/home/qi/match_auto.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/stream/detail/match_manip.hpp>", private, "<boost/spirit/home/qi/stream/match_manip.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/stream/detail/match_manip.hpp>", private, "<boost/spirit/home/qi/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/string/detail/tst.hpp>", private, "<boost/spirit/home/qi/string/tst.hpp>", public ] },
+    { include: ["<boost/spirit/home/qi/string/detail/tst.hpp>", private, "<boost/spirit/home/qi/string/tst_map.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/as_variant.hpp>", private, "<boost/spirit/home/support/attributes.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/endian.hpp>", private, "<boost/spirit/home/karma/binary/binary.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/endian.hpp>", private, "<boost/spirit/home/qi/binary/binary.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/karma/char/char_class.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/karma/char/char.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/karma/numeric/int.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/karma/numeric/real.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/karma/numeric/uint.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/karma/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/karma/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/karma/string/symbols.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/qi/char/char_class.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/qi/char/char.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/qi/string/lit.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/get_encoding.hpp>", private, "<boost/spirit/home/qi/string/symbols.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/hold_any.hpp>", private, "<boost/spirit/home/karma/auto/auto.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/hold_any.hpp>", private, "<boost/spirit/home/karma/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/hold_any.hpp>", private, "<boost/spirit/home/qi/auto/auto.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/hold_any.hpp>", private, "<boost/spirit/home/qi/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/hold_any.hpp>", private, "<boost/spirit/home/support/attributes.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/is_spirit_tag.hpp>", private, "<boost/spirit/home/karma/numeric/int.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/is_spirit_tag.hpp>", private, "<boost/spirit/home/karma/numeric/uint.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/is_spirit_tag.hpp>", private, "<boost/spirit/home/karma/stream/stream.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/is_spirit_tag.hpp>", private, "<boost/spirit/home/qi/numeric/uint.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/is_spirit_tag.hpp>", private, "<boost/spirit/home/support/char_class.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/is_spirit_tag.hpp>", private, "<boost/spirit/home/support/lazy.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/is_spirit_tag.hpp>", private, "<boost/spirit/home/support/terminal.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/char_traits.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/generate_static.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/char_traits.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/consts.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/generate_static.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/consts.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/consts.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/lexer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/consts.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/position_token.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/consts.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/token.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/debug.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/generate_static.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/debug.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/lexer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/debug.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/generator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/functor_data.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/generator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/lexer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/generator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/position_token.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/generator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/generator.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/token.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/rules.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/functor_data.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/rules.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/generate_static.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/rules.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/lexer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/rules.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/position_token.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/rules.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/rules.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/token.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/size_t.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/generate_static.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/size_t.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/state_machine.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/functor_data.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/state_machine.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/generate_static.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/state_machine.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/lexer/state_machine.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/make_cons.hpp>", private, "<boost/spirit/home/support/make_component.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/make_cons.hpp>", private, "<boost/spirit/home/support/meta_compiler.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/make_vector.hpp>", private, "<boost/spirit/home/support/terminal.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/scoped_enum_emulation.hpp>", private, "<boost/spirit/home/karma/delimit_flag.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/scoped_enum_emulation.hpp>", private, "<boost/spirit/home/lex/lexer/pass_flags.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/scoped_enum_emulation.hpp>", private, "<boost/spirit/home/lex/lexer/support_functions.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/scoped_enum_emulation.hpp>", private, "<boost/spirit/home/qi/skip_flag.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/scoped_enum_emulation.hpp>", private, "<boost/spirit/home/support/multi_pass_wrapper.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/what_function.hpp>", private, "<boost/spirit/home/karma/operator/alternative.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/what_function.hpp>", private, "<boost/spirit/home/karma/operator/sequence.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/what_function.hpp>", private, "<boost/spirit/home/qi/operator/alternative.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/what_function.hpp>", private, "<boost/spirit/home/qi/operator/permutation.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/what_function.hpp>", private, "<boost/spirit/home/qi/operator/sequence_base.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/what_function.hpp>", private, "<boost/spirit/home/qi/operator/sequential_or.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/detail/what_function.hpp>", private, "<boost/spirit/repository/home/qi/operator/keywords.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/buffering_input_iterator_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/buf_id_check_policy.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/buf_id_check_policy.hpp>", private, "<boost/spirit/home/support/iterators/istream_iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/buf_id_check_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/combine_policies.hpp>", private, "<boost/spirit/home/support/iterators/istream_iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/combine_policies.hpp>", private, "<boost/spirit/home/support/iterators/look_ahead.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/combine_policies.hpp>", private, "<boost/spirit/home/support/iterators/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/combine_policies.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/first_owner_policy.hpp>", private, "<boost/spirit/home/support/iterators/look_ahead.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/first_owner_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/fixed_size_queue_policy.hpp>", private, "<boost/spirit/home/support/iterators/look_ahead.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/fixed_size_queue_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/functor_input_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/input_iterator_policy.hpp>", private, "<boost/spirit/home/support/iterators/look_ahead.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/input_iterator_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/istream_policy.hpp>", private, "<boost/spirit/home/support/iterators/istream_iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/istream_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/lex_input_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/iterators/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/multi_pass.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/no_check_policy.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/no_check_policy.hpp>", private, "<boost/spirit/home/support/iterators/istream_iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/no_check_policy.hpp>", private, "<boost/spirit/home/support/iterators/look_ahead.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/no_check_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/ref_counted_policy.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/ref_counted_policy.hpp>", private, "<boost/spirit/home/support/iterators/istream_iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/ref_counted_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/split_functor_input_policy.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/split_functor_input_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/split_std_deque_policy.hpp>", private, "<boost/spirit/home/lex/lexer/lexertl/iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/split_std_deque_policy.hpp>", private, "<boost/spirit/home/support/iterators/istream_iterator.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/iterators/detail/split_std_deque_policy.hpp>", private, "<boost/spirit/home/support/multi_pass.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/utree/detail/utree_detail1.hpp>", private, "<boost/spirit/home/support/utree/utree.hpp>", public ] },
+    { include: ["<boost/spirit/home/support/utree/detail/utree_detail2.hpp>", private, "<boost/spirit/home/support/utree.hpp>", public ] },
+    { include: ["<boost/spirit/repository/home/qi/operator/detail/keywords.hpp>", private, "<boost/spirit/repository/home/qi/operator/keywords.hpp>", public ] },
+    { include: ["<boost/statechart/detail/avoid_unused_warning.hpp>", private, "<boost/statechart/state_machine.hpp>", public ] },
+    { include: ["<boost/statechart/detail/constructor.hpp>", private, "<boost/statechart/simple_state.hpp>", public ] },
+    { include: ["<boost/statechart/detail/constructor.hpp>", private, "<boost/statechart/state_machine.hpp>", public ] },
+    { include: ["<boost/statechart/detail/counted_base.hpp>", private, "<boost/statechart/event_base.hpp>", public ] },
+    { include: ["<boost/statechart/detail/leaf_state.hpp>", private, "<boost/statechart/simple_state.hpp>", public ] },
+    { include: ["<boost/statechart/detail/leaf_state.hpp>", private, "<boost/statechart/state_machine.hpp>", public ] },
+    { include: ["<boost/statechart/detail/memory.hpp>", private, "<boost/statechart/event.hpp>", public ] },
+    { include: ["<boost/statechart/detail/memory.hpp>", private, "<boost/statechart/simple_state.hpp>", public ] },
+    { include: ["<boost/statechart/detail/node_state.hpp>", private, "<boost/statechart/simple_state.hpp>", public ] },
+    { include: ["<boost/statechart/detail/node_state.hpp>", private, "<boost/statechart/state_machine.hpp>", public ] },
+    { include: ["<boost/statechart/detail/reaction_dispatcher.hpp>", private, "<boost/statechart/in_state_reaction.hpp>", public ] },
+    { include: ["<boost/statechart/detail/reaction_dispatcher.hpp>", private, "<boost/statechart/transition.hpp>", public ] },
+    { include: ["<boost/statechart/detail/rtti_policy.hpp>", private, "<boost/statechart/event_base.hpp>", public ] },
+    { include: ["<boost/statechart/detail/rtti_policy.hpp>", private, "<boost/statechart/event.hpp>", public ] },
+    { include: ["<boost/statechart/detail/rtti_policy.hpp>", private, "<boost/statechart/state_machine.hpp>", public ] },
+    { include: ["<boost/statechart/detail/state_base.hpp>", private, "<boost/statechart/state_machine.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/debug.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/exception_safety.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/interaction_based.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/logged_expectations.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/mock_object.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/prg_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/test_observer.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/test_tools.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/unit_test_suite_impl.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/utils/basic_cstring/bcs_char_traits.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/utils/class_properties.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/utils/foreach.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/utils/lazy_ostream.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/utils/runtime/config.hpp>", public ] },
+    { include: ["<boost/test/detail/config.hpp>", private, "<boost/test/utils/wrap_stringstream.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/debug.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/exception_safety.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/execution_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/floating_point_comparison.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/framework.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/interaction_based.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/logged_expectations.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/minimal.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/mock_object.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/output/compiler_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/output/plain_report_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/output_test_stream.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/output/xml_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/output/xml_report_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/parameterized_test.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/predicate_result.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/progress_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/results_collector.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/results_reporter.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/test_observer.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/test_tools.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/unit_test_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/unit_test_log.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/unit_test_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/unit_test_suite_impl.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/algorithm.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/basic_cstring/basic_cstring.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/basic_cstring/bcs_char_traits.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/basic_cstring/compare.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/basic_cstring/io.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/callback.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/class_properties.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/custom_manip.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/fixed_mapping.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/foreach.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/iterator/ifstream_line_iterator.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/iterator/input_iterator_facade.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/iterator/istream_line_iterator.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/iterator/token_iterator.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/lazy_ostream.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/named_params.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/nullstream.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/trivial_singleton.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/wrap_stringstream.hpp>", public ] },
+    { include: ["<boost/test/detail/enable_warnings.hpp>", private, "<boost/test/utils/xml_printer.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/execution_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/framework.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/results_collector.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/results_reporter.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/test_observer.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/unit_test_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/unit_test_log.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/unit_test_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/fwd_decl.hpp>", private, "<boost/test/unit_test_suite_impl.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/execution_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/floating_point_comparison.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/framework.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/interaction_based.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/minimal.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/output/compiler_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/output/plain_report_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/output_test_stream.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/output/xml_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/output/xml_report_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/results_collector.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/results_reporter.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/test_observer.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/test_tools.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/unit_test_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/unit_test_log.hpp>", public ] },
+    { include: ["<boost/test/detail/global_typedef.hpp>", private, "<boost/test/unit_test_suite_impl.hpp>", public ] },
+    { include: ["<boost/test/detail/log_level.hpp>", private, "<boost/test/unit_test_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/log_level.hpp>", private, "<boost/test/unit_test_log.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/debug.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/exception_safety.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/execution_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/floating_point_comparison.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/framework.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/interaction_based.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/logged_expectations.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/minimal.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/mock_object.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/output/compiler_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/output/plain_report_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/output_test_stream.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/output/xml_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/output/xml_report_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/parameterized_test.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/predicate_result.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/progress_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/results_collector.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/results_reporter.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/test_observer.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/test_tools.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/unit_test_log_formatter.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/unit_test_log.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/unit_test_monitor.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/unit_test_suite_impl.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/algorithm.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/basic_cstring/basic_cstring.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/basic_cstring/bcs_char_traits.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/basic_cstring/compare.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/basic_cstring/io.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/callback.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/class_properties.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/custom_manip.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/fixed_mapping.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/foreach.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/iterator/ifstream_line_iterator.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/iterator/input_iterator_facade.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/iterator/istream_line_iterator.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/iterator/token_iterator.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/lazy_ostream.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/named_params.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/nullstream.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/trivial_singleton.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/wrap_stringstream.hpp>", public ] },
+    { include: ["<boost/test/detail/suppress_warnings.hpp>", private, "<boost/test/utils/xml_printer.hpp>", public ] },
+    { include: ["<boost/test/detail/unit_test_parameters.hpp>", private, "<boost/test/logged_expectations.hpp>", public ] },
+    { include: ["<boost/test/impl/compiler_log_formatter.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/compiler_log_formatter.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/cpp_main.ipp>", private, "<boost/test/included/prg_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/debug.ipp>", private, "<boost/test/included/prg_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/debug.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/debug.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/debug.ipp>", private, "<boost/test/minimal.hpp>", public ] },
+    { include: ["<boost/test/impl/exception_safety.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/execution_monitor.ipp>", private, "<boost/test/included/prg_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/execution_monitor.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/execution_monitor.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/execution_monitor.ipp>", private, "<boost/test/minimal.hpp>", public ] },
+    { include: ["<boost/test/impl/framework.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/framework.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/interaction_based.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/logged_expectations.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/plain_report_formatter.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/plain_report_formatter.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/progress_monitor.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/progress_monitor.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/results_collector.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/results_collector.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/results_reporter.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/results_reporter.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/test_main.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/test_tools.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/test_tools.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_log.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_log.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_main.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_main.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_monitor.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_monitor.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_parameters.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_parameters.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_suite.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/unit_test_suite.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/xml_log_formatter.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/xml_log_formatter.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/impl/xml_report_formatter.ipp>", private, "<boost/test/included/test_exec_monitor.hpp>", public ] },
+    { include: ["<boost/test/impl/xml_report_formatter.ipp>", private, "<boost/test/included/unit_test.hpp>", public ] },
+    { include: ["<boost/test/utils/runtime/cla/detail/argument_value_usage.hpp>", private, "<boost/test/utils/runtime/cla/argument_factory.hpp>", public ] },
+    { include: ["<boost/thread/detail/async_func.hpp>", private, "<boost/thread/future.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/barrier.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/completion_latch.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/condition.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/exceptions.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/externally_locked.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/externally_locked_stream.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/future_error_code.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/future.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/is_locked_by_this_thread.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/latch.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/lockable_traits.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/lock_algorithms.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/lock_guard.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/lock_traits.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/lock_types.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/null_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/once.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/pthread/mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/pthread/once_atomic.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/pthread/once.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/pthread/thread_data.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/pthread/timespec.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/reverse_lock.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/scoped_thread.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/shared_lock_guard.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/shared_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/strict_lock.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/sync_bounded_queue.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/synchronized_value.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/sync_queue.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/testable_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/thread_functors.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/tss.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/v2/shared_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/v2/thread.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/win32/interlocked_read.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/win32/thread_data.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/win32/thread_heap_alloc.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/win32/thread_primitives.hpp>", public ] },
+    { include: ["<boost/thread/detail/config.hpp>", private, "<boost/thread/xtime.hpp>", public ] },
+    { include: ["<boost/thread/detail/counter.hpp>", private, "<boost/thread/completion_latch.hpp>", public ] },
+    { include: ["<boost/thread/detail/counter.hpp>", private, "<boost/thread/latch.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/barrier.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/completion_latch.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/externally_locked_stream.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/latch.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/lockable_adapter.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/lock_guard.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/null_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/poly_lockable.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/pthread/condition_variable_fwd.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/pthread/condition_variable.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/pthread/mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/pthread/once.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/pthread/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/pthread/shared_mutex_assert.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/pthread/shared_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/reverse_lock.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/scoped_thread.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/shared_lock_guard.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/strict_lock.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/thread_functors.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/thread_guard.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/win32/mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/win32/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/delete.hpp>", private, "<boost/thread/win32/shared_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/invoke.hpp>", private, "<boost/thread/pthread/once_atomic.hpp>", public ] },
+    { include: ["<boost/thread/detail/invoke.hpp>", private, "<boost/thread/pthread/once.hpp>", public ] },
+    { include: ["<boost/thread/detail/invoke.hpp>", private, "<boost/thread/win32/once.hpp>", public ] },
+    { include: ["<boost/thread/detail/is_convertible.hpp>", private, "<boost/thread/future.hpp>", public ] },
+    { include: ["<boost/thread/detail/lockable_wrapper.hpp>", private, "<boost/thread/lock_guard.hpp>", public ] },
+    { include: ["<boost/thread/detail/lockable_wrapper.hpp>", private, "<boost/thread/strict_lock.hpp>", public ] },
+    { include: ["<boost/thread/detail/memory.hpp>", private, "<boost/thread/future.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/externally_locked_stream.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/future.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/lock_concepts.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/lock_guard.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/lock_types.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/pthread/once_atomic.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/pthread/once.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/reverse_lock.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/scoped_thread.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/sync_bounded_queue.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/synchronized_value.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/sync_queue.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/thread_functors.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/thread_guard.hpp>", public ] },
+    { include: ["<boost/thread/detail/move.hpp>", private, "<boost/thread/win32/once.hpp>", public ] },
+    { include: ["<boost/thread/detail/platform.hpp>", private, "<boost/thread/condition_variable.hpp>", public ] },
+    { include: ["<boost/thread/detail/platform.hpp>", private, "<boost/thread/mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/platform.hpp>", private, "<boost/thread/once.hpp>", public ] },
+    { include: ["<boost/thread/detail/platform.hpp>", private, "<boost/thread/recursive_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/platform.hpp>", private, "<boost/thread/thread_only.hpp>", public ] },
+    { include: ["<boost/thread/detail/thread_group.hpp>", private, "<boost/thread/thread.hpp>", public ] },
+    { include: ["<boost/thread/detail/thread_heap_alloc.hpp>", private, "<boost/thread/tss.hpp>", public ] },
+    { include: ["<boost/thread/detail/thread.hpp>", private, "<boost/thread/thread_only.hpp>", public ] },
+    { include: ["<boost/thread/detail/thread_interruption.hpp>", private, "<boost/thread/pthread/shared_mutex_assert.hpp>", public ] },
+    { include: ["<boost/thread/detail/thread_interruption.hpp>", private, "<boost/thread/pthread/shared_mutex.hpp>", public ] },
+    { include: ["<boost/thread/detail/thread_interruption.hpp>", private, "<boost/thread/thread_only.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/array.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/cmath.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/complex.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/functional.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/memory.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/random.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/regex.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/algorithm>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/array>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/bitset>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/cmath>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/complex>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/deque>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/exception>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/fstream>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/functional>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/iomanip>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/ios>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/iostream>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/istream>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/iterator>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/limits>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/list>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/locale>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/map>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/memory>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/new>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/numeric>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/ostream>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/queue>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/random>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/regex>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/set>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/sstream>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/stack>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/stdexcept>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/streambuf>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/string>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/strstream>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/tuple>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/typeinfo>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/type_traits>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/unordered_map>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/unordered_set>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/utility>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/valarray>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tr1/vector>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/tuple.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/type_traits.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/unordered_map.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/unordered_set.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config_all.hpp>", private, "<boost/tr1/utility.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/math/tools/tuple.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/array.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/cmath.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/complex.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/functional.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/memory.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/random.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/regex.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/tuple.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/type_traits.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/unordered_map.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/unordered_set.hpp>", public ] },
+    { include: ["<boost/tr1/detail/config.hpp>", private, "<boost/tr1/utility.hpp>", public ] },
+    { include: ["<boost/tr1/detail/functor2iterator.hpp>", private, "<boost/tr1/random.hpp>", public ] },
+    { include: ["<boost/tr1/detail/math_overloads.hpp>", private, "<boost/tr1/complex.hpp>", public ] },
+    { include: ["<boost/tti/detail/ddata.hpp>", private, "<boost/tti/has_data.hpp>", public ] },
+    { include: ["<boost/tti/detail/ddeftype.hpp>", private, "<boost/tti/has_member_data.hpp>", public ] },
+    { include: ["<boost/tti/detail/ddeftype.hpp>", private, "<boost/tti/has_member_function.hpp>", public ] },
+    { include: ["<boost/tti/detail/ddeftype.hpp>", private, "<boost/tti/has_type.hpp>", public ] },
+    { include: ["<boost/tti/detail/dfunction.hpp>", private, "<boost/tti/has_function.hpp>", public ] },
+    { include: ["<boost/tti/detail/dmem_data.hpp>", private, "<boost/tti/has_member_data.hpp>", public ] },
+    { include: ["<boost/tti/detail/dmem_fun.hpp>", private, "<boost/tti/has_member_function.hpp>", public ] },
+    { include: ["<boost/tti/detail/dmem_type.hpp>", private, "<boost/tti/member_type.hpp>", public ] },
+    { include: ["<boost/tti/detail/dnotype.hpp>", private, "<boost/tti/member_type.hpp>", public ] },
+    { include: ["<boost/tti/detail/dstatic_mem_data.hpp>", private, "<boost/tti/has_static_member_data.hpp>", public ] },
+    { include: ["<boost/tti/detail/dstatic_mem_fun.hpp>", private, "<boost/tti/has_static_member_function.hpp>", public ] },
+    { include: ["<boost/tti/detail/dtemplate.hpp>", private, "<boost/tti/has_template.hpp>", public ] },
+    { include: ["<boost/tti/detail/dtemplate_params.hpp>", private, "<boost/tti/has_template.hpp>", public ] },
+    { include: ["<boost/tti/detail/dtype.hpp>", private, "<boost/tti/has_type.hpp>", public ] },
+    { include: ["<boost/tti/detail/dvm_template_params.hpp>", private, "<boost/tti/has_template.hpp>", public ] },
+    { include: ["<boost/tuple/detail/tuple_basic.hpp>", private, "<boost/tuple/tuple.hpp>", public ] },
+    { include: ["<boost/tuple/detail/tuple_basic_no_partial_spec.hpp>", private, "<boost/tuple/tuple.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/access.hpp>", private, "<boost/type_erasure/any_cast.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/access.hpp>", private, "<boost/type_erasure/any.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/access.hpp>", private, "<boost/type_erasure/binding_of.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/access.hpp>", private, "<boost/type_erasure/call.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/access.hpp>", private, "<boost/type_erasure/check_match.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/access.hpp>", private, "<boost/type_erasure/is_empty.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/access.hpp>", private, "<boost/type_erasure/param.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/access.hpp>", private, "<boost/type_erasure/typeid_of.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/adapt_to_vtable.hpp>", private, "<boost/type_erasure/binding.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/adapt_to_vtable.hpp>", private, "<boost/type_erasure/call.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/any_base.hpp>", private, "<boost/type_erasure/any.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/check_call.hpp>", private, "<boost/type_erasure/call.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/check_map.hpp>", private, "<boost/type_erasure/binding.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/const.hpp>", private, "<boost/type_erasure/free.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/const.hpp>", private, "<boost/type_erasure/member.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/const.hpp>", private, "<boost/type_erasure/operators.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/construct.hpp>", private, "<boost/type_erasure/any.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/extract_concept.hpp>", private, "<boost/type_erasure/call.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/extract_concept.hpp>", private, "<boost/type_erasure/check_match.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/extract_concept.hpp>", private, "<boost/type_erasure/require_match.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/get_placeholders.hpp>", private, "<boost/type_erasure/deduced.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/get_signature.hpp>", private, "<boost/type_erasure/call.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/instantiate.hpp>", private, "<boost/type_erasure/any.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/instantiate.hpp>", private, "<boost/type_erasure/binding.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/macro.hpp>", private, "<boost/type_erasure/free.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/macro.hpp>", private, "<boost/type_erasure/member.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/normalize.hpp>", private, "<boost/type_erasure/any.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/normalize.hpp>", private, "<boost/type_erasure/binding.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/normalize.hpp>", private, "<boost/type_erasure/is_subconcept.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/null.hpp>", private, "<boost/type_erasure/binding.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/rebind_placeholders.hpp>", private, "<boost/type_erasure/binding.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/rebind_placeholders.hpp>", private, "<boost/type_erasure/is_subconcept.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/storage.hpp>", private, "<boost/type_erasure/any.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/storage.hpp>", private, "<boost/type_erasure/builtin.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/storage.hpp>", private, "<boost/type_erasure/constructible.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/storage.hpp>", private, "<boost/type_erasure/param.hpp>", public ] },
+    { include: ["<boost/type_erasure/detail/vtable.hpp>", private, "<boost/type_erasure/binding.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/aligned_storage.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/iterator/is_lvalue_iterator.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/iterator/is_readable_iterator.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/math/tools/traits.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/mpl/empty_base.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_new_operator.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_nothrow_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_nothrow_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_nothrow_copy.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_nothrow_destructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_trivial_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_trivial_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_trivial_copy.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_trivial_destructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_trivial_move_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_trivial_move_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/has_virtual_destructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_abstract.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_arithmetic.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_array.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_base_and_derived.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_base_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_base_of_tr1.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_class.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_complex.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_compound.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_convertible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_copy_constructible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_empty.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_enum.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_float.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_floating_point.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_function.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_fundamental.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_integral.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_lvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_member_object_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_member_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_nothrow_move_assignable.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_nothrow_move_constructible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_object.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_pod.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_polymorphic.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_rvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_same.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_scalar.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_signed.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_stateless.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_union.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_unsigned.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_virtual_base_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_void.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/is_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/type_traits/type_with_alignment.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/variant/recursive_wrapper_fwd.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_def.hpp>", private, "<boost/variant/static_visitor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/aligned_storage.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/iterator/is_lvalue_iterator.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/mpl/empty_base.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_new_operator.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_nothrow_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_nothrow_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_nothrow_copy.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_nothrow_destructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_trivial_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_trivial_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_trivial_copy.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_trivial_destructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_trivial_move_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_trivial_move_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/has_virtual_destructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_abstract.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_arithmetic.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_array.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_base_and_derived.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_base_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_base_of_tr1.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_class.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_complex.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_compound.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_convertible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_copy_constructible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_empty.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_enum.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_float.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_floating_point.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_function.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_fundamental.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_integral.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_lvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_member_object_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_member_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_nothrow_move_assignable.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_nothrow_move_constructible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_object.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_pod.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_polymorphic.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_rvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_same.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_scalar.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_signed.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_stateless.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_union.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_unsigned.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_virtual_base_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_void.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/is_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/type_traits/type_with_alignment.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/variant/recursive_wrapper_fwd.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/bool_trait_undef.hpp>", private, "<boost/variant/static_visitor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/common_type_imp.hpp>", private, "<boost/type_traits/common_type.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/cv_traits_impl.hpp>", private, "<boost/type_traits/is_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/cv_traits_impl.hpp>", private, "<boost/type_traits/is_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/cv_traits_impl.hpp>", private, "<boost/type_traits/remove_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/cv_traits_impl.hpp>", private, "<boost/type_traits/remove_cv.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/cv_traits_impl.hpp>", private, "<boost/type_traits/remove_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/false_result.hpp>", private, "<boost/type_traits/is_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/false_result.hpp>", private, "<boost/type_traits/is_function.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/false_result.hpp>", private, "<boost/type_traits/is_member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/false_result.hpp>", private, "<boost/type_traits/is_member_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/false_result.hpp>", private, "<boost/type_traits/is_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/false_result.hpp>", private, "<boost/type_traits/is_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_bit_and_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_bit_and.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_bit_or_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_bit_or.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_bit_xor_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_bit_xor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_divides_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_divides.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_equal_to.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_greater_equal.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_greater.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_left_shift_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_left_shift.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_less_equal.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_less.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_logical_and.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_logical_or.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_minus_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_minus.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_modulus_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_modulus.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_multiplies_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_multiplies.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_not_equal_to.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_plus_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_plus.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_right_shift_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_binary_operator.hpp>", private, "<boost/type_traits/has_right_shift.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_postfix_operator.hpp>", private, "<boost/type_traits/has_post_decrement.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_postfix_operator.hpp>", private, "<boost/type_traits/has_post_increment.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_prefix_operator.hpp>", private, "<boost/type_traits/has_complement.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_prefix_operator.hpp>", private, "<boost/type_traits/has_dereference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_prefix_operator.hpp>", private, "<boost/type_traits/has_logical_not.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_prefix_operator.hpp>", private, "<boost/type_traits/has_negate.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_prefix_operator.hpp>", private, "<boost/type_traits/has_pre_decrement.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_prefix_operator.hpp>", private, "<boost/type_traits/has_pre_increment.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_prefix_operator.hpp>", private, "<boost/type_traits/has_unary_minus.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/has_prefix_operator.hpp>", private, "<boost/type_traits/has_unary_plus.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/has_trivial_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/has_trivial_copy.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/has_trivial_move_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/has_trivial_move_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/ice.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_abstract.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_base_and_derived.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_base_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_class.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_empty.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_member_object_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_nothrow_move_assignable.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_nothrow_move_constructible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_object.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_same.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/is_stateless.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/make_signed.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/type_traits/make_unsigned.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_and.hpp>", private, "<boost/units/scaled_base_unit.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_eq.hpp>", private, "<boost/type_traits/ice.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/has_trivial_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/has_trivial_copy.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/has_trivial_move_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/has_trivial_move_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/ice.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/is_class.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/is_compound.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/is_empty.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/is_member_object_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/is_nothrow_move_assignable.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/is_object.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/is_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/make_signed.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_not.hpp>", private, "<boost/type_traits/make_unsigned.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/has_new_operator.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/has_trivial_assign.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/has_trivial_constructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/has_trivial_copy.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/has_trivial_destructor.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/ice.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_arithmetic.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_base_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_base_of_tr1.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_empty.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_fundamental.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_member_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_nothrow_move_assignable.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_nothrow_move_constructible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_pod.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_scalar.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_signed.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/is_unsigned.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/make_signed.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/type_traits/make_unsigned.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/ice_or.hpp>", private, "<boost/units/scaled_base_unit.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/is_function_ptr_helper.hpp>", private, "<boost/type_traits/is_function.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/is_function_ptr_tester.hpp>", private, "<boost/type_traits/is_function.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/is_function_ptr_tester.hpp>", private, "<boost/type_traits/is_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/is_mem_fun_pointer_impl.hpp>", private, "<boost/type_traits/is_member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/is_mem_fun_pointer_tester.hpp>", private, "<boost/type_traits/is_member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/is_mem_fun_pointer_tester.hpp>", private, "<boost/type_traits/is_member_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/size_t_trait_def.hpp>", private, "<boost/type_traits/alignment_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/size_t_trait_def.hpp>", private, "<boost/type_traits/extent.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/size_t_trait_def.hpp>", private, "<boost/type_traits/rank.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/size_t_trait_undef.hpp>", private, "<boost/type_traits/alignment_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/size_t_trait_undef.hpp>", private, "<boost/type_traits/extent.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/size_t_trait_undef.hpp>", private, "<boost/type_traits/rank.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/components.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/function_arity.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/function_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/is_callable_builtin.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/is_function.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/is_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/is_function_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/is_member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/is_member_object_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/is_member_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/is_nonmember_callable_builtin.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/member_object_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/parameter_types.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/template_arity_spec.hpp>", private, "<boost/function_types/result_type.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/add_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/add_cv.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/add_lvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/add_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/add_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/add_rvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/add_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/floating_point_promotion.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/integral_promotion.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/make_signed.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/make_unsigned.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/promote.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/remove_all_extents.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/remove_bounds.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/remove_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/remove_cv.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/remove_extent.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/remove_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/remove_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_def.hpp>", private, "<boost/type_traits/remove_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/add_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/add_cv.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/add_lvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/add_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/add_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/add_rvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/add_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/broken_compiler_spec.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/floating_point_promotion.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/integral_promotion.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/make_signed.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/make_unsigned.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/promote.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/remove_all_extents.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/remove_bounds.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/remove_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/remove_cv.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/remove_extent.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/remove_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/remove_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/type_trait_undef.hpp>", private, "<boost/type_traits/remove_volatile.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/wrap.hpp>", private, "<boost/type_traits/is_array.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/wrap.hpp>", private, "<boost/type_traits/is_lvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/assign/list_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/assign/ptr_list_of.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/ptr_container/nullable.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/has_new_operator.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/ice.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_abstract.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_array.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_class.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_const.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_convertible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_copy_constructible.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_function.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_lvalue_reference.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_member_function_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_member_pointer.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_same.hpp>", public ] },
+    { include: ["<boost/type_traits/detail/yes_no_type.hpp>", private, "<boost/type_traits/is_volatile.hpp>", public ] },
+    { include: ["<boost/units/detail/absolute_impl.hpp>", private, "<boost/units/absolute.hpp>", public ] },
+    { include: ["<boost/units/detail/cmath_impl.hpp>", private, "<boost/units/cmath.hpp>", public ] },
+    { include: ["<boost/units/detail/conversion_impl.hpp>", private, "<boost/units/conversion.hpp>", public ] },
+    { include: ["<boost/units/detail/dimension_impl.hpp>", private, "<boost/units/dimension.hpp>", public ] },
+    { include: ["<boost/units/detail/dimensionless_unit.hpp>", private, "<boost/units/cmath.hpp>", public ] },
+    { include: ["<boost/units/detail/dimensionless_unit.hpp>", private, "<boost/units/lambda.hpp>", public ] },
+    { include: ["<boost/units/detail/dimensionless_unit.hpp>", private, "<boost/units/quantity.hpp>", public ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/base_dimension.hpp>", public ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/base_unit.hpp>", public ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/derived_dimension.hpp>", public ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/dimension.hpp>", public ] },
+    { include: ["<boost/units/detail/dimension_list.hpp>", private, "<boost/units/make_system.hpp>", public ] },
+    { include: ["<boost/units/detail/dim_impl.hpp>", private, "<boost/units/dim.hpp>", public ] },
+    { include: ["<boost/units/detail/linear_algebra.hpp>", private, "<boost/units/heterogeneous_system.hpp>", public ] },
+    { include: ["<boost/units/detail/linear_algebra.hpp>", private, "<boost/units/homogeneous_system.hpp>", public ] },
+    { include: ["<boost/units/detail/one.hpp>", private, "<boost/units/scale.hpp>", public ] },
+    { include: ["<boost/units/detail/ordinal.hpp>", private, "<boost/units/base_dimension.hpp>", public ] },
+    { include: ["<boost/units/detail/ordinal.hpp>", private, "<boost/units/base_unit.hpp>", public ] },
+    { include: ["<boost/units/detail/prevent_redefinition.hpp>", private, "<boost/units/base_dimension.hpp>", public ] },
+    { include: ["<boost/units/detail/prevent_redefinition.hpp>", private, "<boost/units/base_unit.hpp>", public ] },
+    { include: ["<boost/units/detail/push_front_if.hpp>", private, "<boost/units/heterogeneous_system.hpp>", public ] },
+    { include: ["<boost/units/detail/push_front_or_add.hpp>", private, "<boost/units/heterogeneous_system.hpp>", public ] },
+    { include: ["<boost/units/detail/sort.hpp>", private, "<boost/units/make_system.hpp>", public ] },
+    { include: ["<boost/units/detail/static_rational_power.hpp>", private, "<boost/units/pow.hpp>", public ] },
+    { include: ["<boost/units/detail/static_rational_power.hpp>", private, "<boost/units/scale.hpp>", public ] },
+    { include: ["<boost/units/detail/unscale.hpp>", private, "<boost/units/heterogeneous_system.hpp>", public ] },
+    { include: ["<boost/units/detail/utility.hpp>", private, "<boost/units/io.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/alpha_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/deuteron_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/electromagnetic_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/electron_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/helion_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/muon_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/neutron_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/physico-chemical_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/proton_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/tau_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/triton_constants.hpp>", public ] },
+    { include: ["<boost/units/systems/detail/constants.hpp>", private, "<boost/units/systems/si/codata/universal_constants.hpp>", public ] },
+    { include: ["<boost/unordered/detail/equivalent.hpp>", private, "<boost/unordered/unordered_map.hpp>", public ] },
+    { include: ["<boost/unordered/detail/equivalent.hpp>", private, "<boost/unordered/unordered_set.hpp>", public ] },
+    { include: ["<boost/unordered/detail/fwd.hpp>", private, "<boost/unordered/unordered_map_fwd.hpp>", public ] },
+    { include: ["<boost/unordered/detail/fwd.hpp>", private, "<boost/unordered/unordered_set_fwd.hpp>", public ] },
+    { include: ["<boost/unordered/detail/unique.hpp>", private, "<boost/unordered/unordered_map.hpp>", public ] },
+    { include: ["<boost/unordered/detail/unique.hpp>", private, "<boost/unordered/unordered_set.hpp>", public ] },
+    { include: ["<boost/unordered/detail/util.hpp>", private, "<boost/unordered/unordered_map.hpp>", public ] },
+    { include: ["<boost/unordered/detail/util.hpp>", private, "<boost/unordered/unordered_set.hpp>", public ] },
+    { include: ["<boost/utility/detail/in_place_factory_prefix.hpp>", private, "<boost/utility/in_place_factory.hpp>", public ] },
+    { include: ["<boost/utility/detail/in_place_factory_prefix.hpp>", private, "<boost/utility/typed_in_place_factory.hpp>", public ] },
+    { include: ["<boost/utility/detail/in_place_factory_suffix.hpp>", private, "<boost/utility/in_place_factory.hpp>", public ] },
+    { include: ["<boost/utility/detail/in_place_factory_suffix.hpp>", private, "<boost/utility/typed_in_place_factory.hpp>", public ] },
+    { include: ["<boost/variant/detail/apply_visitor_binary.hpp>", private, "<boost/variant/apply_visitor.hpp>", public ] },
+    { include: ["<boost/variant/detail/apply_visitor_delayed.hpp>", private, "<boost/variant/apply_visitor.hpp>", public ] },
+    { include: ["<boost/variant/detail/apply_visitor_unary.hpp>", private, "<boost/variant/apply_visitor.hpp>", public ] },
+    { include: ["<boost/variant/detail/backup_holder.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/config.hpp>", private, "<boost/variant/variant_fwd.hpp>", public ] },
+    { include: ["<boost/variant/detail/config.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/enable_recursive_fwd.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/enable_recursive.hpp>", private, "<boost/variant/recursive_variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/forced_return.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/generic_result_type.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/hash_variant.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/initializer.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/make_variant_list.hpp>", private, "<boost/variant/recursive_variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/make_variant_list.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/move.hpp>", private, "<boost/variant/recursive_wrapper.hpp>", public ] },
+    { include: ["<boost/variant/detail/move.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/over_sequence.hpp>", private, "<boost/variant/recursive_variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/over_sequence.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/substitute_fwd.hpp>", private, "<boost/variant/recursive_variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/substitute_fwd.hpp>", private, "<boost/variant/variant_fwd.hpp>", public ] },
+    { include: ["<boost/variant/detail/variant_io.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/variant/detail/visitation_impl.hpp>", private, "<boost/variant/variant.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/access.hpp>", private, "<boost/xpressive/regex_iterator.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/icase.hpp>", private, "<boost/xpressive/regex_primitives.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/linker.hpp>", private, "<boost/xpressive/regex_compiler.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/matcher/action_matcher.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/matcher/attr_begin_matcher.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/matcher/attr_end_matcher.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/matcher/attr_matcher.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/matcher/predicate_matcher.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/matchers.hpp>", private, "<boost/xpressive/regex_primitives.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/optimize.hpp>", private, "<boost/xpressive/regex_compiler.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/regex_domain.hpp>", private, "<boost/xpressive/basic_regex.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/regex_domain.hpp>", private, "<boost/xpressive/regex_primitives.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/basic_regex.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/regex_impl.hpp>", private, "<boost/xpressive/match_results.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/results_cache.hpp>", private, "<boost/xpressive/match_results.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/state.hpp>", private, "<boost/xpressive/regex_algorithms.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/core/sub_match_vector.hpp>", private, "<boost/xpressive/match_results.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/basic_regex.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/match_results.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/regex_algorithms.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/regex_iterator.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/regex_primitives.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/regex_traits.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/sub_match.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/traits/cpp_regex_traits.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/traits/null_regex_traits.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/detail_fwd.hpp>", private, "<boost/xpressive/xpressive_typeof.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/dynamic/parse_charset.hpp>", private, "<boost/xpressive/regex_compiler.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/dynamic/parser_enum.hpp>", private, "<boost/xpressive/regex_compiler.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/dynamic/parser.hpp>", private, "<boost/xpressive/regex_compiler.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/dynamic/parser_traits.hpp>", private, "<boost/xpressive/regex_compiler.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/static/compile.hpp>", private, "<boost/xpressive/regex_primitives.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/static/grammar.hpp>", private, "<boost/xpressive/basic_regex.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/static/modifier.hpp>", private, "<boost/xpressive/regex_primitives.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/static/type_traits.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/algorithm.hpp>", private, "<boost/xpressive/match_results.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/counted_base.hpp>", private, "<boost/xpressive/match_results.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/counted_base.hpp>", private, "<boost/xpressive/regex_iterator.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/regex_actions.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/regex_primitives.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/ignore_unused.hpp>", private, "<boost/xpressive/traits/null_regex_traits.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/literals.hpp>", private, "<boost/xpressive/match_results.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/literals.hpp>", private, "<boost/xpressive/traits/cpp_regex_traits.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/never_true.hpp>", private, "<boost/xpressive/traits/null_regex_traits.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/save_restore.hpp>", private, "<boost/xpressive/regex_algorithms.hpp>", public ] },
+    { include: ["<boost/xpressive/detail/utility/sequence_stack.hpp>", private, "<boost/xpressive/match_results.hpp>", public ] },
+    { include: ["<boost/xpressive/traits/detail/c_ctype.hpp>", private, "<boost/xpressive/traits/c_regex_traits.hpp>", public ] }
+]
diff --git a/cpp/build-support/iwyu/mappings/boost-extra.imp b/cpp/build-support/iwyu/mappings/boost-extra.imp
new file mode 100644
index 0000000000000..aba1e4191680b
--- /dev/null
+++ b/cpp/build-support/iwyu/mappings/boost-extra.imp
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+[
+    { include: ["<boost/core/explicit_operator_bool.hpp>", private, "<boost/optional/optional.hpp>", public ] },
+    { include: ["<boost/cstdint.hpp>", private, "<cstdint>", public ] },
+    { include: ["<boost/none.hpp>", private, "<boost/optional/optional.hpp>", public ] },
+    { include: ["<boost/optional/detail/optional_relops.hpp>", private, "<boost/optional/optional.hpp>", public ] },
+    { include: ["<boost/optional/detail/optional_reference_spec.hpp>", private, "<boost/optional/optional.hpp>", public ] }
+]
diff --git a/cpp/build-support/iwyu/mappings/gflags.imp b/cpp/build-support/iwyu/mappings/gflags.imp
new file mode 100644
index 0000000000000..46ce63d1e719b
--- /dev/null
+++ b/cpp/build-support/iwyu/mappings/gflags.imp
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+[
+  # <gflags/gflags_declare.h> confuses the IWYU tool because of the 'using '
+  { symbol: [ "fLS::clstring", private, "<string>", public ] }
+]
diff --git a/cpp/build-support/iwyu/mappings/glog.imp b/cpp/build-support/iwyu/mappings/glog.imp
new file mode 100644
index 0000000000000..08c5e3529bc8c
--- /dev/null
+++ b/cpp/build-support/iwyu/mappings/glog.imp
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+[
+  { symbol: [ "LOG", private, "<glog/logging.h>", public ] },
+  { symbol: [ "VLOG", private, "<glog/logging.h>", public ] },
+  { symbol: [ "CHECK_EQ", private, "<glog/logging.h>", public ] },
+  { symbol: [ "CHECK_NE", private, "<glog/logging.h>", public ] },
+  { symbol: [ "CHECK_LT", private, "<glog/logging.h>", public ] },
+  { symbol: [ "CHECK_GE", private, "<glog/logging.h>", public ] },
+  { symbol: [ "CHECK_GT", private, "<glog/logging.h>", public ] },
+  { symbol: [ "ErrnoLogMessage", private, "<glog/logging.h>", public ] },
+  { symbol: [ "COMPACT_GOOGLE_LOG_0", private, "<glog/logging.h>", public ] }
+]
diff --git a/cpp/build-support/iwyu/mappings/gtest.imp b/cpp/build-support/iwyu/mappings/gtest.imp
new file mode 100644
index 0000000000000..a54165027e7da
--- /dev/null
+++ b/cpp/build-support/iwyu/mappings/gtest.imp
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+[
+  { include: [ "<gtest/internal/gtest-internal.h>", private, "<gtest/gtest.h>", public ] },
+  { include: [ "<gtest/internal/gtest-string.h>", private, "<gtest/gtest.h>", public ] },
+  { include: [ "<gtest/gtest-death-test.h>", private, "<gtest/gtest.h>", public ] },
+  { include: [ "<gtest/gtest-message.h>", private, "<gtest/gtest.h>", public ] },
+  { include: [ "<gtest/gtest-param-test.h>", private, "<gtest/gtest.h>", public ] },
+  { include: [ "<gtest/gtest-printers.h>", private, "<gtest/gtest.h>", public ] },
+  { include: [ "<gtest/gtest-test-part.h>", private, "<gtest/gtest.h>", public ] },
+  { include: [ "<gtest/gtest-typed-test.h>", private, "<gtest/gtest.h>", public ] }
+]
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 5dba0e1c4efb6..daaf49c473d8f 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -158,6 +158,7 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
 
   // Fill up the bytes until we have a byte alignment
   int64_t pad_to_byte = std::min<int64_t>(8 - (length_ % 8), length);
+
   if (pad_to_byte == 8) {
     pad_to_byte = 0;
   }
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 1a3461383da1b..15a38934e3f5e 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -8,6 +8,7 @@ cpp/src/arrow/status.h
 cpp/build-support/asan_symbolize.py
 cpp/build-support/cpplint.py
 cpp/build-support/clang_format_exclusions.txt
+cpp/build-support/iwyu/*
 cpp/cmake_modules/BuildUtils.cmake
 cpp/cmake_modules/FindPythonLibsNew.cmake
 cpp/cmake_modules/FindNumPy.cmake

From 4956e90a7c08fdf5b40b5a71253fafa4aacde434 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Fri, 1 Sep 2017 16:50:30 -0400
Subject: [PATCH 0984/1644] =?UTF-8?q?ARROW-1407:=20Fix=20bug=20where=20Dic?=
 =?UTF-8?q?tionaryEncoder=20can=20only=20encode=20vector=20le=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ss than 4096 elements

Author: Li Jin <ice.xelloss@gmail.com>

Closes #1024 from icexelloss/dict-bug-ARROW-1407 and squashes the following commits:

b64258ce [Li Jin] Minor style change
e73ae599 [Li Jin] ARROW-1407: Fix bug where DictionaryEncoder can only encode vector less than 4096 elements
---
 .../vector/dictionary/DictionaryEncoder.java  |  2 +-
 .../arrow/vector/TestDictionaryVector.java    | 48 +++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
index 7e20794cbbed2..3b7dc4a56e1ac 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -68,7 +68,7 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
     Method setter = null;
     for (Class<?> c : ImmutableList.of(int.class, long.class)) {
       try {
-        setter = mutator.getClass().getMethod("set", int.class, c);
+        setter = mutator.getClass().getMethod("setSafe", int.class, c);
         break;
       } catch (NoSuchMethodException e) {
         // ignore
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
index f2db9baac04ca..f8c16e7fc8176 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -39,6 +39,8 @@ public class TestDictionaryVector {
   byte[] one = "bar".getBytes(StandardCharsets.UTF_8);
   byte[] two = "baz".getBytes(StandardCharsets.UTF_8);
 
+  byte[][] data = new byte[][] {zero, one, two};
+
   @Before
   public void init() {
     allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
@@ -98,4 +100,50 @@ public void testEncodeStrings() {
       }
     }
   }
+
+  @Test
+  public void testEncodeLargeVector() {
+    // Create a new value vector
+    try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator);
+         final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
+      vector.allocateNew();
+
+      int count = 10000;
+
+      for (int i = 0; i < 10000; ++i) {
+        vector.getMutator().setSafe(i, data[i % 3], 0, data[i % 3].length);
+      }
+      vector.getMutator().setValueCount(count);
+
+      dictionaryVector.allocateNew(512, 3);
+      dictionaryVector.getMutator().setSafe(0, zero, 0, zero.length);
+      dictionaryVector.getMutator().setSafe(1, one, 0, one.length);
+      dictionaryVector.getMutator().setSafe(2, two, 0, two.length);
+      dictionaryVector.getMutator().setValueCount(3);
+
+      Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null));
+
+
+      try (final ValueVector encoded = (FieldVector) DictionaryEncoder.encode(vector, dictionary)) {
+        // verify indices
+        assertEquals(NullableIntVector.class, encoded.getClass());
+
+        NullableIntVector.Accessor indexAccessor = ((NullableIntVector) encoded).getAccessor();
+        assertEquals(count, indexAccessor.getValueCount());
+        for (int i = 0; i < count; ++i) {
+          assertEquals(i % 3, indexAccessor.get(i));
+        }
+
+        // now run through the decoder and verify we get the original back
+        try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) {
+          assertEquals(vector.getClass(), decoded.getClass());
+          assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount());
+          for (int i = 0; i < count; ++i) {
+            assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i));
+          }
+        }
+      }
+    }
+  }
 }

From 848a0f782dddac788d01eb8ea230957b8372e2dd Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Sat, 2 Sep 2017 10:57:11 -0400
Subject: [PATCH 0985/1644] ARROW-1444: [JAVA] fix last byte copy in BitVector
 splitAndTransfer

cc @jacques-n

Author: siddharth <siddharth@dremio.com>

Closes #1023 from siddharthteotia/ARROW-1444 and squashes the following commits:

eb5f6578 [siddharth] ARROW-1444: fix last byte copy in BitVector splitAndTransfer
---
 .../org/apache/arrow/vector/BitVector.java    |  59 +++++--
 .../apache/arrow/vector/TestBitVector.java    | 163 ++++++++++++++----
 2 files changed, 171 insertions(+), 51 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 44001d4190098..8a60273e179ef 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -276,9 +276,11 @@ public void transferTo(BitVector target) {
 
   public void splitAndTransferTo(int startIndex, int length, BitVector target) {
     assert startIndex + length <= valueCount;
-    int firstByte = getByteIndex(startIndex);
-    int byteSize = getSizeFromCount(length);
+    int firstByteSource = getByteIndex(startIndex);
+    int lastByteSource = getByteIndex(valueCount - 1);
+    int byteSizeTarget = getSizeFromCount(length);
     int offset = startIndex % 8;
+
     if (length > 0) {
       if (offset == 0) {
         target.clear();
@@ -286,32 +288,59 @@ public void splitAndTransferTo(int startIndex, int length, BitVector target) {
         if (target.data != null) {
           target.data.release();
         }
-        target.data = data.slice(firstByte, byteSize);
+        target.data = data.slice(firstByteSource, byteSizeTarget);
         target.data.retain(1);
-      } else {
+      }
+      else {
         // Copy data
         // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
         // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
-        // The last byte copied to target is a bit tricky :
-        //   1) if length requires partly byte (length % 8 !=0), copy the remaining bits only.
-        //   2) otherwise, copy the last byte in the same way as to the prior bytes.
+
         target.clear();
-        target.allocateNew(length);
+        target.allocateNew(byteSizeTarget * 8);
+
         // TODO maybe do this one word at a time, rather than byte?
-        for (int i = 0; i < byteSize - 1; i++) {
-          target.data.setByte(i, (((this.data.getByte(firstByte + i) & 0xFF) >>> offset) + (this.data.getByte(firstByte + i + 1) << (8 - offset))));
+
+        for (int i = 0; i < byteSizeTarget - 1; i++) {
+          byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + i, offset);
+          byte b2 = getBitsFromNextByte(this.data, firstByteSource + i + 1, offset);
+
+          target.data.setByte(i, (b1 + b2));
         }
-        if (length % 8 != 0) {
-          target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
-        } else {
-          target.data.setByte(byteSize - 1,
-              (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) << (8 - offset))));
+
+        /* Copying the last piece is done in the following manner:
+         * if the source vector has 1 or more bytes remaining, we copy
+         * the last piece as a byte formed by shifting data
+         * from the current byte and the next byte.
+         *
+         * if the source vector has no more bytes remaining
+         * (we are at the last byte), we copy the last piece as a byte
+         * by shifting data from the current byte.
+         */
+        if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + byteSizeTarget - 1, offset);
+          byte b2 = getBitsFromNextByte(this.data, firstByteSource + byteSizeTarget, offset);
+
+          target.data.setByte(byteSizeTarget - 1, b1 + b2);
+        }
+        else {
+          byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + byteSizeTarget - 1, offset);
+
+          target.data.setByte(byteSizeTarget - 1, b1);
         }
       }
     }
     target.getMutator().setValueCount(length);
   }
 
+  private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) {
+    return (byte)((data.getByte(index) & 0xFF) >>> offset);
+  }
+
+  private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
+    return (byte)((data.getByte(index) << (8 - offset)));
+  }
+
   private class TransferImpl implements TransferPair {
     BitVector to;
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
index 16316608c0f7c..82e61be45c67b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -99,47 +99,138 @@ public void testSplitAndTransfer() throws Exception {
         }
       }
 
-      final TransferPair transferPair = sourceVector.getTransferPair(allocator);
-      final BitVector toVector = (BitVector) transferPair.getTo();
-      final BitVector.Accessor toAccessor = toVector.getAccessor();
-      final BitVector.Mutator toMutator = toVector.getMutator();
-
-      /*
-       * form test cases such that we cover:
-       *
-       * (1) the start index is exactly where a particular byte starts in the source bit vector
-       * (2) the start index is randomly positioned within a byte in the source bit vector
-       *    (2.1) the length is a multiple of 8
-       *    (2.2) the length is not a multiple of 8
-       */
-      final int[][] transferLengths = {{0, 8},     /* (1) */
-          {8, 10},    /* (1) */
-          {18, 0},    /* zero length scenario */
-          {18, 8},    /* (2.1) */
-          {26, 0},    /* zero length scenario */
-          {26, 14}    /* (2.2) */
-      };
-
-      for (final int[] transferLength : transferLengths) {
-        final int start = transferLength[0];
-        final int length = transferLength[1];
-
-        transferPair.splitAndTransfer(start, length);
-
-        /* check the toVector output after doing splitAndTransfer */
-        for (int i = 0; i < length; i++) {
-          int result = toAccessor.get(i);
-          if ((i & 1) == 1) {
-            assertEquals(Integer.toString(1), Integer.toString(result));
-          } else {
-            assertEquals(Integer.toString(0), Integer.toString(result));
+      try (final BitVector toVector = new BitVector("toVector", allocator)) {
+        final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
+        final BitVector.Accessor toAccessor = toVector.getAccessor();
+        final BitVector.Mutator toMutator = toVector.getMutator();
+
+        /*
+         * form test cases such that we cover:
+         *
+         * (1) the start index is exactly where a particular byte starts in the source bit vector
+         * (2) the start index is randomly positioned within a byte in the source bit vector
+         *    (2.1) the length is a multiple of 8
+         *    (2.2) the length is not a multiple of 8
+         */
+        final int[][] transferLengths = {{0, 8}, {8, 10}, {18, 0}, {18, 8}, {26, 0}, {26, 14}};
+
+        for (final int[] transferLength : transferLengths) {
+          final int start = transferLength[0];
+          final int length = transferLength[1];
+
+          transferPair.splitAndTransfer(start, length);
+
+          /* check the toVector output after doing splitAndTransfer */
+          for (int i = 0; i < length; i++) {
+            int actual = toAccessor.get(i);
+            int expected = sourceAccessor.get(start + i);
+            assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i,
+                    expected, actual);
           }
         }
+      }
+    }
+  }
+
+  @Test
+  public void testSplitAndTransfer1() throws Exception {
+
+    try (final BitVector sourceVector = new BitVector("bitvector", allocator)) {
+      final BitVector.Mutator sourceMutator = sourceVector.getMutator();
+      final BitVector.Accessor sourceAccessor = sourceVector.getAccessor();
+
+      sourceVector.allocateNew(8190);
+
+      /* populate the bitvector */
+      for (int i = 0; i < 8190; i++) {
+        sourceMutator.set(i, 1);
+      }
+
+      sourceMutator.setValueCount(8190);
+
+      /* check the vector output */
+      for (int i = 0; i < 8190; i++) {
+        int result = sourceAccessor.get(i);
+        assertEquals(Integer.toString(1), Integer.toString(result));
+      }
+
+      try (final BitVector toVector = new BitVector("toVector", allocator)) {
+        final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
+        final BitVector.Accessor toAccessor = toVector.getAccessor();
+        final BitVector.Mutator toMutator = toVector.getMutator();
+
+        final int[][] transferLengths = {{0, 4095}, {4095, 4095}};
+
+        for (final int[] transferLength : transferLengths) {
+          final int start = transferLength[0];
+          final int length = transferLength[1];
+
+          transferPair.splitAndTransfer(start, length);
+
+          /* check the toVector output after doing splitAndTransfer */
+          for (int i = 0; i < length; i++) {
+            int actual = toAccessor.get(i);
+            int expected = sourceAccessor.get(start + i);
+            assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i,
+                    expected, actual);
+          }
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testSplitAndTransfer2() throws Exception {
+
+    try (final BitVector sourceVector = new BitVector("bitvector", allocator)) {
+      final BitVector.Mutator sourceMutator = sourceVector.getMutator();
+      final BitVector.Accessor sourceAccessor = sourceVector.getAccessor();
+
+      sourceVector.allocateNew(32);
+
+      /* populate the bitvector */
+      for (int i = 0; i < 32; i++) {
+        if ((i & 1) == 1) {
+          sourceMutator.set(i, 1);
+        } else {
+          sourceMutator.set(i, 0);
+        }
+      }
+
+      sourceMutator.setValueCount(32);
 
-        toVector.clear();
+      /* check the vector output */
+      for (int i = 0; i < 32; i++) {
+        int result = sourceAccessor.get(i);
+        if ((i & 1) == 1) {
+          assertEquals(Integer.toString(1), Integer.toString(result));
+        } else {
+          assertEquals(Integer.toString(0), Integer.toString(result));
+        }
       }
 
-      sourceVector.close();
+      try (final BitVector toVector = new BitVector("toVector", allocator)) {
+        final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
+        final BitVector.Accessor toAccessor = toVector.getAccessor();
+        final BitVector.Mutator toMutator = toVector.getMutator();
+
+        final int[][] transferLengths = {{5,22}, {5,24}, {5,25}, {5,27}, {0,31}, {5,7}, {2,3}};
+
+        for (final int[] transferLength : transferLengths) {
+          final int start = transferLength[0];
+          final int length = transferLength[1];
+
+          transferPair.splitAndTransfer(start, length);
+
+          /* check the toVector output after doing splitAndTransfer */
+          for (int i = 0; i < length; i++) {
+            int actual = toAccessor.get(i);
+            int expected = sourceAccessor.get(start + i);
+            assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i,
+                    expected, actual);
+          }
+        }
+      }
     }
   }
 

From 1a8dafb2b7b5d4eff56cc4eb3de25b87fc41aead Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 2 Sep 2017 16:33:48 -0400
Subject: [PATCH 0986/1644] ARROW-1447: [C++] Fix many include-what-you-use
 warnings

There's still some work to deal with some things that IWYU is reporting incorrectly, but we can do that later

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1025 from wesm/ARROW-1447 and squashes the following commits:

67a20026 [Wes McKinney] Fix another include for Windows
f6ec09a7 [Wes McKinney] Fix include error
5d77388a [Wes McKinney] Fix buglet
09b38b07 [Wes McKinney] Add arrow-specific mappings files, fix more things
49688450 [Wes McKinney] Fix a ton of include-what-you-use warnings
---
 cpp/build-support/iwyu/iwyu-filter.awk        |  4 +++
 cpp/build-support/iwyu/iwyu.sh                | 33 ++++++++---------
 .../iwyu/mappings/arrow-misc.imp              | 32 +++++++++++++++++
 cpp/src/arrow/allocator-test.cc               |  8 +++--
 cpp/src/arrow/array-decimal-test.cc           | 18 ++++++++--
 cpp/src/arrow/buffer-test.cc                  |  5 ++-
 cpp/src/arrow/buffer.cc                       |  1 -
 cpp/src/arrow/buffer.h                        |  3 +-
 cpp/src/arrow/io/file.cc                      | 35 ++++++++-----------
 cpp/src/arrow/io/file.h                       |  1 -
 cpp/src/arrow/io/hdfs-internal.cc             | 11 +++---
 cpp/src/arrow/io/hdfs-internal.h              | 11 +++---
 cpp/src/arrow/io/hdfs.cc                      |  5 +++
 cpp/src/arrow/io/hdfs.h                       |  2 +-
 cpp/src/arrow/io/interfaces.cc                |  1 -
 cpp/src/arrow/io/interfaces.h                 |  1 -
 cpp/src/arrow/io/io-file-test.cc              | 20 +++++++----
 cpp/src/arrow/io/io-hdfs-test.cc              | 11 ++++--
 cpp/src/arrow/io/io-memory-test.cc            | 10 +++---
 cpp/src/arrow/io/memory.cc                    |  6 +---
 cpp/src/arrow/io/memory.h                     |  4 +--
 cpp/src/arrow/ipc/dictionary.cc               |  3 +-
 cpp/src/arrow/ipc/dictionary.h                | 11 ------
 cpp/src/arrow/ipc/feather-internal.h          |  1 +
 cpp/src/arrow/ipc/feather.cc                  |  9 ++---
 cpp/src/arrow/ipc/feather.h                   |  4 +--
 cpp/src/arrow/ipc/file-to-stream.cc           |  6 ++++
 cpp/src/arrow/ipc/json-integration-test.cc    | 11 +++---
 cpp/src/arrow/ipc/json-internal.cc            |  8 +----
 cpp/src/arrow/ipc/json-internal.h             |  2 +-
 cpp/src/arrow/ipc/json.cc                     |  6 ++--
 cpp/src/arrow/ipc/json.h                      | 12 +++----
 cpp/src/arrow/ipc/message.cc                  |  8 ++---
 cpp/src/arrow/ipc/message.h                   |  6 ----
 cpp/src/arrow/ipc/metadata-internal.cc        | 12 +++++--
 cpp/src/arrow/ipc/metadata-internal.h         | 12 +------
 cpp/src/arrow/ipc/reader.cc                   |  6 ++++
 cpp/src/arrow/ipc/reader.h                    |  1 -
 cpp/src/arrow/ipc/stream-to-file.cc           |  7 +++-
 cpp/src/arrow/memory_pool-test.cc             |  9 +++--
 cpp/src/arrow/memory_pool-test.h              |  6 ++--
 cpp/src/arrow/memory_pool.cc                  |  5 +--
 cpp/src/arrow/python/pandas_to_arrow.cc       |  2 +-
 cpp/src/arrow/status-test.cc                  |  3 +-
 cpp/src/arrow/status.h                        |  2 +-
 cpp/src/arrow/table.h                         | 10 +-----
 cpp/src/arrow/tensor-test.cc                  |  4 ++-
 cpp/src/arrow/tensor.cc                       |  5 +--
 cpp/src/arrow/tensor.h                        |  5 ---
 cpp/src/arrow/type-test.cc                    |  4 ++-
 cpp/src/arrow/type.h                          |  2 +-
 cpp/src/arrow/util/bit-util-test.cc           | 12 ++++---
 cpp/src/arrow/util/compression-test.cc        |  7 ++--
 cpp/src/arrow/util/compression.cc             |  4 ---
 cpp/src/arrow/util/compression_brotli.cc      | 10 +++---
 cpp/src/arrow/util/compression_brotli.h       |  2 +-
 cpp/src/arrow/util/compression_lz4.cc         |  4 ---
 cpp/src/arrow/util/compression_lz4.h          |  2 +-
 cpp/src/arrow/util/compression_snappy.cc      |  8 ++---
 cpp/src/arrow/util/compression_snappy.h       |  2 +-
 cpp/src/arrow/util/compression_zlib.cc        |  2 ++
 cpp/src/arrow/util/compression_zlib.h         |  1 +
 cpp/src/arrow/util/compression_zstd.cc        |  7 ++--
 cpp/src/arrow/util/compression_zstd.h         |  2 +-
 cpp/src/arrow/util/cpu-info.cc                |  5 ++-
 cpp/src/arrow/util/decimal-test.cc            |  7 ++--
 cpp/src/arrow/util/decimal.cc                 |  3 ++
 cpp/src/arrow/util/decimal.h                  |  7 ++--
 cpp/src/arrow/util/int128.cc                  |  3 +-
 cpp/src/arrow/util/int128.h                   |  2 ++
 cpp/src/arrow/util/key-value-metadata-test.cc |  9 +++--
 cpp/src/arrow/util/key_value_metadata.cc      |  4 +++
 cpp/src/arrow/util/rle-encoding-test.cc       | 16 ++++-----
 cpp/src/arrow/util/stl-util-test.cc           |  7 ++--
 cpp/src/arrow/visitor.cc                      |  2 ++
 75 files changed, 292 insertions(+), 240 deletions(-)
 create mode 100644 cpp/build-support/iwyu/mappings/arrow-misc.imp

diff --git a/cpp/build-support/iwyu/iwyu-filter.awk b/cpp/build-support/iwyu/iwyu-filter.awk
index a325d0adc8c63..a1c96097f7665 100644
--- a/cpp/build-support/iwyu/iwyu-filter.awk
+++ b/cpp/build-support/iwyu/iwyu-filter.awk
@@ -67,6 +67,10 @@ BEGIN {
   # suggestions are addressed and invalid ones are taken care either by proper
   # IWYU pragmas or adding special mappings (e.g. like boost mappings).
   # muted["relative/path/to/file"]
+  muted["arrow/util/bit-util-test.cc"]
+  muted["arrow/util/rle-encoding-test.cc"]
+  muted["include/hdfs.h"]
+  muted["arrow/visitor.h"]
 }
 
 # mute all suggestions for the auto-generated files
diff --git a/cpp/build-support/iwyu/iwyu.sh b/cpp/build-support/iwyu/iwyu.sh
index 3ee5a120dce29..f3fadd628dc12 100755
--- a/cpp/build-support/iwyu/iwyu.sh
+++ b/cpp/build-support/iwyu/iwyu.sh
@@ -26,21 +26,6 @@ trap "rm -f $IWYU_LOG" EXIT
 
 echo "Logging IWYU to $IWYU_LOG"
 
-# Build the list of updated files which are of IWYU interest.
-file_list_tmp=$(git diff --name-only \
-    $($ROOT/cpp/build-support/get-upstream-commit.sh) | grep -E '\.(c|cc|h)$')
-if [ -z "$file_list_tmp" ]; then
-  echo "IWYU verification: no updates on related files, declaring success"
-  exit 0
-fi
-
-# Adjust the path for every element in the list. The iwyu_tool.py normalizes
-# paths (via realpath) to match the records from the compilation database.
-IWYU_FILE_LIST=
-for p in $file_list_tmp; do
-  IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p"
-done
-
 IWYU_MAPPINGS_PATH="$ROOT/cpp/build-support/iwyu/mappings"
 IWYU_ARGS="\
     --mapping_file=$IWYU_MAPPINGS_PATH/boost-all.imp \
@@ -48,7 +33,8 @@ IWYU_ARGS="\
     --mapping_file=$IWYU_MAPPINGS_PATH/boost-extra.imp \
     --mapping_file=$IWYU_MAPPINGS_PATH/gflags.imp \
     --mapping_file=$IWYU_MAPPINGS_PATH/glog.imp \
-    --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp"
+    --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/arrow-misc.imp"
 
 set -e
 
@@ -57,6 +43,21 @@ if [ "$1" == "all" ]; then
        $IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk | \
        tee $IWYU_LOG
 else
+  # Build the list of updated files which are of IWYU interest.
+  file_list_tmp=$(git diff --name-only \
+      $($ROOT/cpp/build-support/get-upstream-commit.sh) | grep -E '\.(c|cc|h)$')
+  if [ -z "$file_list_tmp" ]; then
+    echo "IWYU verification: no updates on related files, declaring success"
+    exit 0
+  fi
+
+  # Adjust the path for every element in the list. The iwyu_tool.py normalizes
+  # paths (via realpath) to match the records from the compilation database.
+  IWYU_FILE_LIST=
+  for p in $file_list_tmp; do
+    IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p"
+  done
+
   python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST  -- \
        $IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk | \
        tee $IWYU_LOG
diff --git a/cpp/build-support/iwyu/mappings/arrow-misc.imp b/cpp/build-support/iwyu/mappings/arrow-misc.imp
new file mode 100644
index 0000000000000..7d9f09c0cef71
--- /dev/null
+++ b/cpp/build-support/iwyu/mappings/arrow-misc.imp
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+[
+  { symbol: ["bool", private, "<cstdint>", public ] },
+  { symbol: ["false", private, "<cstdint>", public ] },
+  { symbol: ["true", private, "<cstdint>", public ] },
+  { symbol: ["int64_t", private, "<cstdint>", public ] },
+  { symbol: ["int16_t", private, "<cstdint>", public ] },
+  { symbol: ["int32_t", private, "<cstdint>", public ] },
+  { symbol: ["uint8_t", private, "<cstdint>", public ] },
+  { symbol: ["_Node_const_iterator", private, "<flatbuffers/flatbuffers.h>", public ] },
+  { symbol: ["unordered_map<>::mapped_type", private, "<flatbuffers/flatbuffers.h>", public ] },
+  { symbol: ["__alloc_traits<>::value_type", private, "<vector>", public ] },
+  { symbol: ["move", private, "<utility>", public ] },
+  { symbol: ["pair", private, "<utility>", public ] },
+  { symbol: ["errno", private, "<cerrno>", public ] },
+  { symbol: ["posix_memalign", private, "<cstdlib>", public ] }
+]
diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
index f3a80cdae818b..e02741ec6aaf3 100644
--- a/cpp/src/arrow/allocator-test.cc
+++ b/cpp/src/arrow/allocator-test.cc
@@ -15,10 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "gtest/gtest.h"
+#include <cstdint>
+#include <limits>
+#include <new>
+
+#include <gtest/gtest.h>
 
 #include "arrow/allocator.h"
-#include "arrow/test-util.h"
+#include "arrow/memory_pool.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 37852a3d1c798..7bbf2f00fc0f3 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -15,14 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/type.h"
-#include "gtest/gtest.h"
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include <gtest/gtest.h>
 
+#include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
 
+using std::size_t;
+
 namespace arrow {
+
+class Buffer;
+
 namespace decimal {
 
 template <typename T>
diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index 30c46725c7723..334ad7bf7143d 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -16,12 +16,15 @@
 // under the License.
 
 #include <cstdint>
+#include <cstring>
 #include <limits>
+#include <memory>
 #include <string>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/test-util.h"
 
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index d60c8295cf9a9..cf533eb3be439 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -18,7 +18,6 @@
 #include "arrow/buffer.h"
 
 #include <cstdint>
-#include <limits>
 
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 859fa80072582..ed128fdf6b6f6 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -18,11 +18,11 @@
 #ifndef ARROW_BUFFER_H
 #define ARROW_BUFFER_H
 
-#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <memory>
 #include <string>
+#include <type_traits>
 
 #include "arrow/status.h"
 #include "arrow/util/bit-util.h"
@@ -32,7 +32,6 @@
 namespace arrow {
 
 class MemoryPool;
-class Status;
 
 // ----------------------------------------------------------------------
 // Buffer classes
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 395785299e135..0abadbb16af41 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -45,9 +45,21 @@
 #include <sys/mman.h>
 #endif
 
+#include <algorithm>
+#include <cerrno>
+#include <cstdint>
+#include <cstring>
+#include <mutex>
+#include <sstream>  // IWYU pragma: keep
+
+#if defined(_MSC_VER)
+#include <codecvt>
+#include <locale>
+#endif
+
 #include <fcntl.h>
 #include <sys/stat.h>
-#include <sys/types.h>
+#include <sys/types.h>  // IWYU pragma: keep
 
 #ifndef _MSC_VER  // POSIX-like platforms
 
@@ -69,22 +81,6 @@
 #define ARROW_WRITE_SHMODE S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH
 #endif
 
-// ----------------------------------------------------------------------
-// C++ standard library
-
-#include <algorithm>
-#include <cstring>
-#include <iostream>
-#include <limits>
-#include <mutex>
-#include <sstream>
-#include <vector>
-
-#if defined(_MSC_VER)
-#include <codecvt>
-#include <locale>
-#endif
-
 // ----------------------------------------------------------------------
 // file compatibility stuff
 
@@ -96,8 +92,6 @@
 // nothing
 #endif
 
-#include <cstdio>
-
 // POSIX systems do not have this
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -113,7 +107,8 @@
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
-#include <boost/filesystem.hpp>  // NOLINT
+#include <boost/filesystem.hpp>           // NOLINT
+#include <boost/system/system_error.hpp>  // NOLINT
 
 namespace fs = boost::filesystem;
 
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index f26d905141ad4..1b1bbe0e4955d 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -25,7 +25,6 @@
 #include <string>
 
 #include "arrow/io/interfaces.h"
-#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index 35657df4620f5..e6d0487a9db5c 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -30,17 +30,20 @@
 
 #include "arrow/io/hdfs-internal.h"
 
-#include <iostream>
+#include <cstdint>
+#include <cstdlib>
 #include <mutex>
-#include <sstream>
+#include <sstream>  // IWYU pragma: keep
 #include <string>
-#include <type_traits>
 #include <vector>
 
+#ifndef _WIN32
+#include <dlfcn.h>
+#endif
+
 #include <boost/filesystem.hpp>  // NOLINT
 
 #include "arrow/status.h"
-#include "arrow/util/visibility.h"
 
 namespace fs = boost::filesystem;
 
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
index f2de00de8b934..df925cf62823a 100644
--- a/cpp/src/arrow/io/hdfs-internal.h
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -18,15 +18,18 @@
 #ifndef ARROW_IO_HDFS_INTERNAL
 #define ARROW_IO_HDFS_INTERNAL
 
-#ifndef _WIN32
-#include <dlfcn.h>
-#endif
+#include <cstddef>
+#include <cstdint>
 
 #include <hdfs.h>
 
-#include "arrow/io/windows_compatibility.h"
+#include "arrow/io/windows_compatibility.h"  // IWYU pragma: keep
 #include "arrow/util/visibility.h"
 
+using std::size_t;
+
+struct hdfsBuilder;
+
 namespace arrow {
 
 class Status;
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 8b143c3ed833d..d9d420622dad9 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -18,7 +18,10 @@
 #include <hdfs.h>
 
 #include <algorithm>
+#include <cerrno>
+#include <cstddef>
 #include <cstdint>
+#include <mutex>
 #include <sstream>
 #include <string>
 
@@ -29,6 +32,8 @@
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
+using std::size_t;
+
 namespace arrow {
 namespace io {
 
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 82e3f7108bc87..f5ed08281bce5 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -30,11 +30,11 @@
 namespace arrow {
 
 class Buffer;
+class MemoryPool;
 class Status;
 
 namespace io {
 
-class HadoopFileSystem;
 class HdfsReadableFile;
 class HdfsOutputStream;
 
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 57dc42d8a9b2a..694575b5f06ab 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -21,7 +21,6 @@
 #include <memory>
 #include <mutex>
 
-#include "arrow/buffer.h"
 #include "arrow/status.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 3388acb0ed752..dd3f32d3c3b00 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -30,7 +30,6 @@
 namespace arrow {
 
 class Buffer;
-class MemoryPool;
 class Status;
 
 namespace io {
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 630356fa2af38..636fbd842d049 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -16,23 +16,31 @@
 // under the License.
 
 #include <atomic>
+#include <cerrno>
 #include <cstdint>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
-#ifndef _MSC_VER
-#include <fcntl.h>
-#endif
-#include <fstream>
+#include <fstream>  // IWYU pragma: keep
 #include <memory>
-#include <sstream>
+#include <sstream>  // IWYU pragma: keep
 #include <string>
 #include <thread>
+#include <vector>
+
+#ifndef _MSC_VER
+#include <fcntl.h>
+#endif
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
+#include "arrow/buffer.h"
 #include "arrow/io/file.h"
+#include "arrow/io/interfaces.h"
 #include "arrow/io/test-common.h"
 #include "arrow/memory_pool.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index b88ca8f26d42c..eaf638f200251 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -15,18 +15,25 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <atomic>
+#include <cstdint>
 #include <cstdlib>
+#include <cstring>
 #include <iostream>
-#include <sstream>
+#include <memory>
+#include <sstream>  // IWYU pragma: keep
 #include <string>
 #include <thread>
+#include <vector>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include <boost/filesystem.hpp>  // NOLINT
 
+#include "arrow/buffer.h"
 #include "arrow/io/hdfs-internal.h"
 #include "arrow/io/hdfs.h"
+#include "arrow/io/interfaces.h"
 #include "arrow/status.h"
 #include "arrow/test-util.h"
 
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index fadcfc3468829..117972f1cf06e 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -16,17 +16,19 @@
 // under the License.
 
 #include <cstdint>
-#include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <memory>
 #include <string>
-#include <vector>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
+#include "arrow/buffer.h"
+#include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
-#include "arrow/io/test-common.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 568d7b2294e20..0b91ab5186fef 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -18,17 +18,13 @@
 #include "arrow/io/memory.h"
 
 #include <algorithm>
-#include <cerrno>
 #include <cstdint>
-#include <cstdio>
 #include <cstring>
-#include <sstream>
-#include <string>
 
 #include "arrow/buffer.h"
-#include "arrow/io/interfaces.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
 #include "arrow/util/memory.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index ab8d103b58a39..563000f77b4fb 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -23,16 +23,14 @@
 #include <cstdint>
 #include <memory>
 #include <mutex>
-#include <string>
 
 #include "arrow/io/interfaces.h"
-
-#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class Buffer;
+class MemoryPool;
 class ResizableBuffer;
 class Status;
 
diff --git a/cpp/src/arrow/ipc/dictionary.cc b/cpp/src/arrow/ipc/dictionary.cc
index fcfb3e2c8d230..488bb75b9d75f 100644
--- a/cpp/src/arrow/ipc/dictionary.cc
+++ b/cpp/src/arrow/ipc/dictionary.cc
@@ -20,10 +20,9 @@
 #include <cstdint>
 #include <memory>
 #include <sstream>
+#include <utility>
 
-#include "arrow/array.h"
 #include "arrow/status.h"
-#include "arrow/type.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/dictionary.h b/cpp/src/arrow/ipc/dictionary.h
index 744ab64e11c82..13cf7a0b3d15c 100644
--- a/cpp/src/arrow/ipc/dictionary.h
+++ b/cpp/src/arrow/ipc/dictionary.h
@@ -22,9 +22,7 @@
 
 #include <cstdint>
 #include <memory>
-#include <string>
 #include <unordered_map>
-#include <vector>
 
 #include "arrow/status.h"
 #include "arrow/util/macros.h"
@@ -33,17 +31,8 @@
 namespace arrow {
 
 class Array;
-class Buffer;
 class Field;
 
-namespace io {
-
-class InputStream;
-class OutputStream;
-class RandomAccessFile;
-
-}  // namespace io
-
 namespace ipc {
 
 using DictionaryMap = std::unordered_map<int64_t, std::shared_ptr<Array>>;
diff --git a/cpp/src/arrow/ipc/feather-internal.h b/cpp/src/arrow/ipc/feather-internal.h
index 1b5924e3030ab..90512dd117238 100644
--- a/cpp/src/arrow/ipc/feather-internal.h
+++ b/cpp/src/arrow/ipc/feather-internal.h
@@ -22,6 +22,7 @@
 #define ARROW_IPC_FEATHER_INTERNAL_H
 
 #include <cstdint>
+#include <iostream>
 #include <memory>
 #include <string>
 #include <vector>
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 54771d3356b83..5c6e2520766d5 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -17,12 +17,10 @@
 
 #include "arrow/ipc/feather.h"
 
-#include <algorithm>
 #include <cstdint>
 #include <cstring>
-#include <iostream>
 #include <memory>
-#include <sstream>
+#include <sstream>  // IWYU pragma: keep
 #include <string>
 #include <vector>
 
@@ -30,13 +28,16 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
-#include "arrow/io/file.h"
+#include "arrow/io/interfaces.h"
 #include "arrow/ipc/feather-internal.h"
 #include "arrow/ipc/feather_generated.h"
+#include "arrow/ipc/util.h"  // IWYU pragma: keep
 #include "arrow/status.h"
 #include "arrow/table.h"
+#include "arrow/type.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
+#include "arrow/visitor.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
index 2ab35a9556d73..83cd151172c5c 100644
--- a/cpp/src/arrow/ipc/feather.h
+++ b/cpp/src/arrow/ipc/feather.h
@@ -24,14 +24,12 @@
 #include <cstdint>
 #include <memory>
 #include <string>
-#include <vector>
 
-#include "arrow/type.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
-class Buffer;
+class Array;
 class Column;
 class Status;
 
diff --git a/cpp/src/arrow/ipc/file-to-stream.cc b/cpp/src/arrow/ipc/file-to-stream.cc
index e69444a560c11..c5f0eb3a75f1a 100644
--- a/cpp/src/arrow/ipc/file-to-stream.cc
+++ b/cpp/src/arrow/ipc/file-to-stream.cc
@@ -16,6 +16,9 @@
 // under the License.
 
 #include <iostream>
+#include <memory>
+#include <string>
+
 #include "arrow/io/file.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
@@ -24,6 +27,9 @@
 #include "arrow/util/io-util.h"
 
 namespace arrow {
+
+class RecordBatch;
+
 namespace ipc {
 
 // Reads a file on the file system and prints to stdout the stream version of it.
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index e0f01f931c1d7..4a3b2b399741e 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -15,16 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <cstdint>
 #include <cstdio>
-#include <cstdlib>
 #include <cstring>
-#include <fstream>
+#include <fstream>  // IWYU pragma: keep
 #include <iostream>
 #include <memory>
 #include <string>
+#include <vector>
 
-#include "gflags/gflags.h"
-#include "gtest/gtest.h"
+#include <gflags/gflags.h>
+#include <gtest/gtest.h>
 
 #include <boost/filesystem.hpp>  // NOLINT
 
@@ -50,6 +51,8 @@ namespace fs = boost::filesystem;
 
 namespace arrow {
 
+class Buffer;
+
 bool file_exists(const char* path) {
   std::ifstream handle(path);
   return handle.good();
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 80244f6c8df9c..f0fb96369f676 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -17,23 +17,17 @@
 
 #include "arrow/ipc/json-internal.h"
 
-#include <algorithm>
 #include <cstdint>
 #include <cstdlib>
-#include <iostream>
 #include <memory>
 #include <sstream>
 #include <string>
 #include <type_traits>
 #include <vector>
 
-#include "rapidjson/stringbuffer.h"
-#include "rapidjson/writer.h"
-
 #include "arrow/array.h"
 #include "arrow/builder.h"
-#include "arrow/ipc/metadata-internal.h"
-#include "arrow/memory_pool.h"
+#include "arrow/ipc/dictionary.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/ipc/json-internal.h b/cpp/src/arrow/ipc/json-internal.h
index 9b641cd53329b..506fe682949e6 100644
--- a/cpp/src/arrow/ipc/json-internal.h
+++ b/cpp/src/arrow/ipc/json-internal.h
@@ -30,7 +30,7 @@
 #include "rapidjson/stringbuffer.h"
 #include "rapidjson/writer.h"
 
-#include "arrow/type_fwd.h"
+#include "arrow/type_fwd.h"  // IWYU pragma: export
 #include "arrow/util/visibility.h"
 
 namespace rj = rapidjson;
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index f57101a31a97d..30a1bb81e1a1e 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -17,12 +17,10 @@
 
 #include "arrow/ipc/json.h"
 
-#include <cstdint>
+#include <cstddef>
 #include <memory>
 #include <string>
-#include <vector>
 
-#include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/ipc/json-internal.h"
 #include "arrow/memory_pool.h"
@@ -31,6 +29,8 @@
 #include "arrow/type.h"
 #include "arrow/util/logging.h"
 
+using std::size_t;
+
 namespace arrow {
 namespace ipc {
 
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
index be26f0233ebeb..0391172075837 100644
--- a/cpp/src/arrow/ipc/json.h
+++ b/cpp/src/arrow/ipc/json.h
@@ -22,18 +22,16 @@
 
 #include <memory>
 #include <string>
-#include <vector>
 
-#include "arrow/type_fwd.h"
+#include "arrow/status.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
-namespace io {
 
-class OutputStream;
-class RandomAccessFile;
-
-}  // namespace io
+class Buffer;
+class MemoryPool;
+class RecordBatch;
+class Schema;
 
 namespace ipc {
 
diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc
index 53f0203f0807a..082c92556b755 100644
--- a/cpp/src/arrow/ipc/message.cc
+++ b/cpp/src/arrow/ipc/message.cc
@@ -22,17 +22,13 @@
 #include <memory>
 #include <sstream>
 #include <string>
-#include <vector>
 
-#include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
+#include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/Schema_generated.h"
 #include "arrow/ipc/metadata-internal.h"
-#include "arrow/ipc/util.h"
 #include "arrow/status.h"
-#include "arrow/tensor.h"
-#include "arrow/type.h"
-#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace ipc {
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index dbc50d8449051..6fb8de7d26154 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -23,8 +23,6 @@
 #include <cstdint>
 #include <memory>
 #include <string>
-#include <unordered_map>
-#include <vector>
 
 #include "arrow/status.h"
 #include "arrow/util/macros.h"
@@ -32,9 +30,7 @@
 
 namespace arrow {
 
-class Array;
 class Buffer;
-class Field;
 
 namespace io {
 
@@ -57,8 +53,6 @@ constexpr int kMaxNestingDepth = 64;
 // individual fields metadata can be retrieved from very large schema without
 //
 
-class Message;
-
 /// \brief An IPC message including metadata and body
 class ARROW_EXPORT Message {
  public:
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 928e889b79374..1f2340dfbe6f0 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -17,22 +17,28 @@
 
 #include "arrow/ipc/metadata-internal.h"
 
-#include <algorithm>
 #include <cstdint>
+#include <cstring>
 #include <memory>
 #include <sstream>
 #include <string>
-#include <vector>
+#include <utility>
 
-#include "flatbuffers/flatbuffers.h"
+#include <flatbuffers/flatbuffers.h>
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
+#include "arrow/ipc/File_generated.h"
+#include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/Tensor_generated.h"
+#include "arrow/ipc/dictionary.h"
 #include "arrow/ipc/util.h"
 #include "arrow/status.h"
 #include "arrow/tensor.h"
 #include "arrow/type.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/key_value_metadata.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index 738ed3c5c7a04..83d1f13012714 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -23,23 +23,15 @@
 #include <cstdint>
 #include <memory>
 #include <string>
-#include <unordered_map>
 #include <vector>
 
-#include "arrow/ipc/File_generated.h"
-#include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/Tensor_generated.h"
+#include "arrow/ipc/Schema_generated.h"
 #include "arrow/ipc/dictionary.h"
-#include "arrow/ipc/message.h"
-#include "arrow/util/macros.h"
-#include "arrow/util/visibility.h"
 
 namespace arrow {
 
-class Array;
 class Buffer;
 class DataType;
-class Field;
 class Schema;
 class Status;
 class Tensor;
@@ -48,9 +40,7 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace io {
 
-class InputStream;
 class OutputStream;
-class RandomAccessFile;
 
 }  // namespace io
 
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 7d7acad518bb7..a4cac0b802d2c 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -21,13 +21,19 @@
 #include <cstring>
 #include <sstream>
 #include <string>
+#include <type_traits>
 #include <vector>
 
+#include <flatbuffers/flatbuffers.h>  // IWYU pragma: export
+
+#include "arrow/array.h"
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
 #include "arrow/ipc/File_generated.h"
 #include "arrow/ipc/Message_generated.h"
+#include "arrow/ipc/Schema_generated.h"
+#include "arrow/ipc/dictionary.h"
 #include "arrow/ipc/message.h"
 #include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/util.h"
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index f822a32c1cc38..52b04abe2256b 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -22,7 +22,6 @@
 
 #include <cstdint>
 #include <memory>
-#include <vector>
 
 #include "arrow/ipc/message.h"
 #include "arrow/util/visibility.h"
diff --git a/cpp/src/arrow/ipc/stream-to-file.cc b/cpp/src/arrow/ipc/stream-to-file.cc
index 96339c83f27e3..4f8cd80a12f76 100644
--- a/cpp/src/arrow/ipc/stream-to-file.cc
+++ b/cpp/src/arrow/ipc/stream-to-file.cc
@@ -16,7 +16,9 @@
 // under the License.
 
 #include <iostream>
-#include "arrow/io/file.h"
+#include <memory>
+#include <string>
+
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
 #include "arrow/status.h"
@@ -24,6 +26,9 @@
 #include "arrow/util/io-util.h"
 
 namespace arrow {
+
+class RecordBatch;
+
 namespace ipc {
 
 // Converts a stream from stdin to a file written to standard out.
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 52e48dbefab9e..552c79b5ae78d 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -15,10 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/memory_pool-test.h"
-
 #include <cstdint>
-#include <limits>
+
+#include <gtest/gtest.h>
+
+#include "arrow/memory_pool-test.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/memory_pool-test.h b/cpp/src/arrow/memory_pool-test.h
index b9f0337dfac8e..ced59bf701f4d 100644
--- a/cpp/src/arrow/memory_pool-test.h
+++ b/cpp/src/arrow/memory_pool-test.h
@@ -15,11 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "gtest/gtest.h"
-
+#include <cstdint>
 #include <limits>
 
+#include <gtest/gtest.h>
+
 #include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 769fc1037ee80..b5b4461cfe7b5 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -17,12 +17,13 @@
 
 #include "arrow/memory_pool.h"
 
-#include <stdlib.h>
 #include <algorithm>
+#include <cerrno>
 #include <cstdlib>
+#include <cstring>
 #include <iostream>
 #include <mutex>
-#include <sstream>
+#include <sstream>  // IWYU pragma: keep
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 8f6746af34b14..435682cf7a705 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -515,7 +515,7 @@ inline Status PandasConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>*
       CopyStrided(input, length_, stride_elements, output);
     } else {
       // TODO(wesm): int32 overflow checks
-      for (int64_t i= 0; i < length_; ++i) {
+      for (int64_t i = 0; i < length_; ++i) {
         *output++ = static_cast<int32_t>(*input++);
       }
     }
diff --git a/cpp/src/arrow/status-test.cc b/cpp/src/arrow/status-test.cc
index f32eb152d13ec..d4f84e48ec07b 100644
--- a/cpp/src/arrow/status-test.cc
+++ b/cpp/src/arrow/status-test.cc
@@ -17,10 +17,9 @@
 
 #include <sstream>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include "arrow/status.h"
-#include "arrow/test-util.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index d52addfc90700..7a39d0f3ab5b8 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -15,8 +15,8 @@
 #ifndef ARROW_STATUS_H_
 #define ARROW_STATUS_H_
 
-#include <cstdint>
 #include <cstring>
+#include <iosfwd>
 #include <string>
 
 #include "arrow/util/macros.h"
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 31ca97a37078c..ceb564d45c9b2 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -29,15 +29,7 @@
 
 namespace arrow {
 
-namespace internal {
-
-struct ArrayData;
-
-}  // namespace internal
-
-class Array;
-class Column;
-class Schema;
+class KeyValueMetadata;
 class Status;
 
 using ArrayVector = std::vector<std::shared_ptr<Array>>;
diff --git a/cpp/src/arrow/tensor-test.cc b/cpp/src/arrow/tensor-test.cc
index 3df0e6ccb4a55..06815761b5049 100644
--- a/cpp/src/arrow/tensor-test.cc
+++ b/cpp/src/arrow/tensor-test.cc
@@ -17,13 +17,15 @@
 
 // Unit tests for DataType (and subclasses), Field, and Schema
 
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
 #include "arrow/tensor.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index 31b1a359219a6..efadd7cea224d 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -17,7 +17,6 @@
 
 #include "arrow/tensor.h"
 
-#include <algorithm>
 #include <cstdint>
 #include <functional>
 #include <memory>
@@ -25,11 +24,9 @@
 #include <string>
 #include <vector>
 
-#include "arrow/array.h"
-#include "arrow/buffer.h"
 #include "arrow/compare.h"
+#include "arrow/status.h"
 #include "arrow/type.h"
-#include "arrow/type_traits.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index bbfd8ae27679f..d1cd3cf13c00b 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -25,16 +25,11 @@
 
 #include "arrow/buffer.h"
 #include "arrow/type.h"
-#include "arrow/type_traits.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
-class Buffer;
-class MemoryPool;
-class Status;
-
 static inline bool is_tensor_supported(Type::type type_id) {
   switch (type_id) {
     case Type::UINT8:
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 4ac5c85d480ed..138f8dd040fc7 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -17,14 +17,16 @@
 
 // Unit tests for DataType (and subclasses), Field, and Schema
 
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include "arrow/test-util.h"
 #include "arrow/type.h"
+#include "arrow/util/key_value_metadata.h"
 
 using std::shared_ptr;
 using std::vector;
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 4cd17bc37f47e..7026aee0bdf32 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -27,7 +27,7 @@
 #include <vector>
 
 #include "arrow/status.h"
-#include "arrow/type_fwd.h"
+#include "arrow/type_fwd.h"  // IWYU pragma: export
 #include "arrow/util/key_value_metadata.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index bbe95cf4b09f7..d838ab9d7a7b1 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -15,18 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <climits>
 #include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <iostream>
+#include <cstring>
 #include <limits>
+#include <memory>
 #include <vector>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
-#include <boost/utility.hpp>
+#include <boost/utility.hpp>  // IWYU pragma: export
 
 #include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/util/bit-stream-utils.h"
 #include "arrow/util/bit-util.h"
diff --git a/cpp/src/arrow/util/compression-test.cc b/cpp/src/arrow/util/compression-test.cc
index 64896dd6a4a15..6dd518a182826 100644
--- a/cpp/src/arrow/util/compression-test.cc
+++ b/cpp/src/arrow/util/compression-test.cc
@@ -15,13 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
 #include <cstdint>
+#include <memory>
 #include <string>
 #include <vector>
 
-#include "arrow/status.h"
-#include "arrow/test-common.h"
+#include <gtest/gtest.h>
+
+#include "arrow/test-util.h"
 #include "arrow/util/compression.h"
 
 using std::string;
diff --git a/cpp/src/arrow/util/compression.cc b/cpp/src/arrow/util/compression.cc
index b4e5373c5b019..bead6ffc320ad 100644
--- a/cpp/src/arrow/util/compression.cc
+++ b/cpp/src/arrow/util/compression.cc
@@ -17,10 +17,7 @@
 
 #include "arrow/util/compression.h"
 
-#include <cstdint>
 #include <memory>
-#include <sstream>
-#include <string>
 
 #ifdef ARROW_WITH_BROTLI
 #include "arrow/util/compression_brotli.h"
@@ -43,7 +40,6 @@
 #endif
 
 #include "arrow/status.h"
-#include "arrow/util/logging.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/compression_brotli.cc b/cpp/src/arrow/util/compression_brotli.cc
index e4639083dfadb..1aaec11d9caa1 100644
--- a/cpp/src/arrow/util/compression_brotli.cc
+++ b/cpp/src/arrow/util/compression_brotli.cc
@@ -17,16 +17,14 @@
 
 #include "arrow/util/compression_brotli.h"
 
+#include <cstddef>
 #include <cstdint>
-#include <memory>
-#include <sstream>
-#include <string>
 
 #include <brotli/decode.h>
 #include <brotli/encode.h>
+#include <brotli/types.h>
 
 #include "arrow/status.h"
-#include "arrow/util/logging.h"
 
 namespace arrow {
 
@@ -35,7 +33,7 @@ namespace arrow {
 
 Status BrotliCodec::Decompress(int64_t input_len, const uint8_t* input,
                                int64_t output_len, uint8_t* output_buffer) {
-  size_t output_size = output_len;
+  std::size_t output_size = output_len;
   if (BrotliDecoderDecompress(input_len, input, &output_size, output_buffer) !=
       BROTLI_DECODER_RESULT_SUCCESS) {
     return Status::IOError("Corrupt brotli compressed data.");
@@ -50,7 +48,7 @@ int64_t BrotliCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
 Status BrotliCodec::Compress(int64_t input_len, const uint8_t* input,
                              int64_t output_buffer_len, uint8_t* output_buffer,
                              int64_t* output_length) {
-  size_t output_len = output_buffer_len;
+  std::size_t output_len = output_buffer_len;
   // TODO: Make quality configurable. We use 8 as a default as it is the best
   //       trade-off for Parquet workload
   if (BrotliEncoderCompress(8, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, input_len,
diff --git a/cpp/src/arrow/util/compression_brotli.h b/cpp/src/arrow/util/compression_brotli.h
index 9e92cb106d422..89e979d957954 100644
--- a/cpp/src/arrow/util/compression_brotli.h
+++ b/cpp/src/arrow/util/compression_brotli.h
@@ -19,10 +19,10 @@
 #define ARROW_UTIL_COMPRESSION_BROTLI_H
 
 #include <cstdint>
-#include <memory>
 
 #include "arrow/status.h"
 #include "arrow/util/compression.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/compression_lz4.cc b/cpp/src/arrow/util/compression_lz4.cc
index 295e9a438f799..cda40ad8c3a53 100644
--- a/cpp/src/arrow/util/compression_lz4.cc
+++ b/cpp/src/arrow/util/compression_lz4.cc
@@ -18,14 +18,10 @@
 #include "arrow/util/compression_lz4.h"
 
 #include <cstdint>
-#include <memory>
-#include <sstream>
-#include <string>
 
 #include <lz4.h>
 
 #include "arrow/status.h"
-#include "arrow/util/logging.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/compression_lz4.h b/cpp/src/arrow/util/compression_lz4.h
index 0af228963f320..4ef9c58dc6295 100644
--- a/cpp/src/arrow/util/compression_lz4.h
+++ b/cpp/src/arrow/util/compression_lz4.h
@@ -19,10 +19,10 @@
 #define ARROW_UTIL_COMPRESSION_LZ4_H
 
 #include <cstdint>
-#include <memory>
 
 #include "arrow/status.h"
 #include "arrow/util/compression.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/compression_snappy.cc b/cpp/src/arrow/util/compression_snappy.cc
index 947ffe559bda6..542831f405403 100644
--- a/cpp/src/arrow/util/compression_snappy.cc
+++ b/cpp/src/arrow/util/compression_snappy.cc
@@ -22,15 +22,15 @@
 #undef DISALLOW_COPY_AND_ASSIGN
 #endif
 
+#include <cstddef>
 #include <cstdint>
-#include <memory>
-#include <sstream>
-#include <string>
 
 #include <snappy.h>
 
 #include "arrow/status.h"
-#include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
+
+using std::size_t;
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/compression_snappy.h b/cpp/src/arrow/util/compression_snappy.h
index 5cc10c470af45..1db21656b8340 100644
--- a/cpp/src/arrow/util/compression_snappy.h
+++ b/cpp/src/arrow/util/compression_snappy.h
@@ -19,10 +19,10 @@
 #define ARROW_UTIL_COMPRESSION_SNAPPY_H
 
 #include <cstdint>
-#include <memory>
 
 #include "arrow/status.h"
 #include "arrow/util/compression.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/compression_zlib.cc b/cpp/src/arrow/util/compression_zlib.cc
index ae6627ea6442f..0656fd64b5cbd 100644
--- a/cpp/src/arrow/util/compression_zlib.cc
+++ b/cpp/src/arrow/util/compression_zlib.cc
@@ -18,10 +18,12 @@
 #include "arrow/util/compression_zlib.h"
 
 #include <cstdint>
+#include <cstring>
 #include <memory>
 #include <sstream>
 #include <string>
 
+#include <zconf.h>
 #include <zlib.h>
 
 #include "arrow/status.h"
diff --git a/cpp/src/arrow/util/compression_zlib.h b/cpp/src/arrow/util/compression_zlib.h
index f55d6689edfa9..a18af6eaa8079 100644
--- a/cpp/src/arrow/util/compression_zlib.h
+++ b/cpp/src/arrow/util/compression_zlib.h
@@ -23,6 +23,7 @@
 
 #include "arrow/status.h"
 #include "arrow/util/compression.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/compression_zstd.cc b/cpp/src/arrow/util/compression_zstd.cc
index ac6e9065d22dd..d19ac43449b66 100644
--- a/cpp/src/arrow/util/compression_zstd.cc
+++ b/cpp/src/arrow/util/compression_zstd.cc
@@ -17,15 +17,14 @@
 
 #include "arrow/util/compression_zstd.h"
 
+#include <cstddef>
 #include <cstdint>
-#include <memory>
-#include <sstream>
-#include <string>
 
 #include <zstd.h>
 
 #include "arrow/status.h"
-#include "arrow/util/logging.h"
+
+using std::size_t;
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/compression_zstd.h b/cpp/src/arrow/util/compression_zstd.h
index 6e40e19d280d7..946393df5fdab 100644
--- a/cpp/src/arrow/util/compression_zstd.h
+++ b/cpp/src/arrow/util/compression_zstd.h
@@ -19,10 +19,10 @@
 #define ARROW_UTIL_COMPRESSION_ZSTD_H
 
 #include <cstdint>
-#include <memory>
 
 #include "arrow/status.h"
 #include "arrow/util/compression.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/cpu-info.cc b/cpp/src/arrow/util/cpu-info.cc
index d0a9a14fc60c0..639f02e482be4 100644
--- a/cpp/src/arrow/util/cpu-info.cc
+++ b/cpp/src/arrow/util/cpu-info.cc
@@ -38,14 +38,13 @@
 
 #endif
 
-#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string/trim.hpp>
 
 #include <algorithm>
 #include <cstdint>
 #include <fstream>
-#include <iostream>
 #include <mutex>
-#include <sstream>
 #include <string>
 
 #include "arrow/util/logging.h"
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index fb8833c9af2cf..ea8962ba64e8b 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -16,11 +16,14 @@
 // under the License.
 //
 
-#include "arrow/util/decimal.h"
+#include <cstdint>
+#include <string>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
+#include "arrow/status.h"
 #include "arrow/test-util.h"
+#include "arrow/util/decimal.h"
 
 namespace arrow {
 namespace decimal {
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index c195cf3e2c5e3..98768cfeec7c7 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -15,9 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <cctype>
+#include <cmath>
 #include <sstream>
 
 #include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 
 namespace arrow {
 namespace decimal {
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 88ac2f53fa57a..26595ec330bdb 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -18,15 +18,14 @@
 #ifndef ARROW_DECIMAL_H
 #define ARROW_DECIMAL_H
 
-#include <cmath>
+#include <cstdint>
 #include <cstdlib>
-#include <iterator>
 #include <string>
 
 #include "arrow/status.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/int128.h"
+#include "arrow/util/int128.h"  // IWYU pragma: export
 #include "arrow/util/logging.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 namespace decimal {
diff --git a/cpp/src/arrow/util/int128.cc b/cpp/src/arrow/util/int128.cc
index 9d1d06276e05c..7102097b08a34 100644
--- a/cpp/src/arrow/util/int128.cc
+++ b/cpp/src/arrow/util/int128.cc
@@ -18,9 +18,8 @@
 
 #include <algorithm>
 #include <cmath>
-#include <iomanip>
+#include <cstring>
 #include <limits>
-#include <sstream>
 
 #ifdef _MSC_VER
 #include <intrin.h>
diff --git a/cpp/src/arrow/util/int128.h b/cpp/src/arrow/util/int128.h
index 1d4bd4007a52e..19888ae157a2d 100644
--- a/cpp/src/arrow/util/int128.h
+++ b/cpp/src/arrow/util/int128.h
@@ -19,9 +19,11 @@
 #ifndef ARROW_INT128_H
 #define ARROW_INT128_H
 
+#include <cstdint>
 #include <string>
 
 #include "arrow/status.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 namespace decimal {
diff --git a/cpp/src/arrow/util/key-value-metadata-test.cc b/cpp/src/arrow/util/key-value-metadata-test.cc
index 59cfdf597308c..4524b337cf2a2 100644
--- a/cpp/src/arrow/util/key-value-metadata-test.cc
+++ b/cpp/src/arrow/util/key-value-metadata-test.cc
@@ -15,11 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "gtest/gtest.h"
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
 
-#include "arrow/util/key_value_metadata.h"
+#include <gtest/gtest.h>
 
-#include "arrow/test-util.h"
+#include "arrow/util/key_value_metadata.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/util/key_value_metadata.cc b/cpp/src/arrow/util/key_value_metadata.cc
index 0497f65c30888..cf74ddf37b4df 100644
--- a/cpp/src/arrow/util/key_value_metadata.cc
+++ b/cpp/src/arrow/util/key_value_metadata.cc
@@ -16,10 +16,14 @@
 // under the License.
 
 #include <algorithm>
+#include <cstddef>
+#include <utility>
 
 #include "arrow/util/key_value_metadata.h"
 #include "arrow/util/logging.h"
 
+using std::size_t;
+
 namespace arrow {
 
 static std::vector<std::string> UnorderedMapKeys(
diff --git a/cpp/src/arrow/util/rle-encoding-test.cc b/cpp/src/arrow/util/rle-encoding-test.cc
index 7549b874355df..09bcb1a21c13f 100644
--- a/cpp/src/arrow/util/rle-encoding-test.cc
+++ b/cpp/src/arrow/util/rle-encoding-test.cc
@@ -15,21 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// From Apache Impala as of 2016-01-29
-
-#include <gtest/gtest.h>
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <boost/utility.hpp>
+// From Apache Impala (incubating) as of 2016-01-29
 
 #include <cstdint>
-#include <iostream>
+#include <cstring>
 #include <random>
 #include <vector>
 
+#include <gtest/gtest.h>
+
+#include <boost/utility.hpp>  // IWYU pragma: export
+
 #include "arrow/util/bit-stream-utils.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/rle-encoding.h"
 
 using std::vector;
diff --git a/cpp/src/arrow/util/stl-util-test.cc b/cpp/src/arrow/util/stl-util-test.cc
index 629eb24c3d97e..c492647b8215d 100644
--- a/cpp/src/arrow/util/stl-util-test.cc
+++ b/cpp/src/arrow/util/stl-util-test.cc
@@ -15,14 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/stl.h"
-
-#include <cstdint>
 #include <vector>
 
-#include "gtest/gtest.h"
+#include <gtest/gtest.h>
 
-#include "arrow/test-util.h"
+#include "arrow/util/stl.h"
 
 namespace arrow {
 namespace internal {
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index 203ed6d4af91a..53a0fee0aae2d 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -17,6 +17,8 @@
 
 #include "arrow/visitor.h"
 
+#include <memory>
+
 #include "arrow/array.h"
 #include "arrow/status.h"
 #include "arrow/type.h"

From ebca1af3b40f16d8f00cfbfbbe28881e2d267bf4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 3 Sep 2017 15:07:59 -0400
Subject: [PATCH 0987/1644] ARROW-1452: [C++] Make macros in
 arrow/util/macros.h more unique

This fixes a bug encounted in third party use: https://github.com/mapd/mapd-core/blob/master/QueryEngine/ResultSet.h#L37

cc @asuhan

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1030 from wesm/ARROW-1452 and squashes the following commits:

ab7611cf [Wes McKinney] Disallow copy and assign in ArrayBuilder
2a4db73f [Wes McKinney] Remove DISALLOW_COPY_AND_ASSIGN from some abstract classes. Revert MANUALLY_ALIGNED_STRUCT
39f10d64 [Wes McKinney] Remove UNLIKELY, LIKELY macros as they may conflict with thirdparty projects, consolidate compiler-util.h with macros.h
994f5bfe [Wes McKinney] Add unique prefix to DISALLOW_COPY_AND_ASSIGN
c3121df0 [Wes McKinney] Rename UNUSED to ARROW_UNUSED
---
 cpp/src/arrow/array.h                    |  3 --
 cpp/src/arrow/buffer.h                   |  2 +-
 cpp/src/arrow/builder.cc                 |  4 +--
 cpp/src/arrow/builder.h                  |  2 +-
 cpp/src/arrow/compare.cc                 |  2 +-
 cpp/src/arrow/io/hdfs.h                  |  6 ++--
 cpp/src/arrow/io/interfaces.h            |  3 --
 cpp/src/arrow/ipc/dictionary.h           |  2 +-
 cpp/src/arrow/ipc/message.h              |  2 +-
 cpp/src/arrow/python/arrow_to_pandas.cc  |  2 +-
 cpp/src/arrow/python/common.h            |  2 +-
 cpp/src/arrow/tensor.h                   |  2 +-
 cpp/src/arrow/type.h                     |  3 --
 cpp/src/arrow/util/bit-stream-utils.h    | 16 +++++-----
 cpp/src/arrow/util/bit-util.h            |  8 ++---
 cpp/src/arrow/util/compiler-util.h       | 40 ++----------------------
 cpp/src/arrow/util/compression_snappy.cc |  5 ---
 cpp/src/arrow/util/hash-util.h           |  2 +-
 cpp/src/arrow/util/key_value_metadata.h  |  2 +-
 cpp/src/arrow/util/macros.h              | 32 ++++++++++++++++---
 cpp/src/arrow/util/rle-encoding.h        |  6 ++--
 21 files changed, 62 insertions(+), 84 deletions(-)

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 57c1f271db815..8052071d5d0ee 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -254,9 +254,6 @@ class ARROW_EXPORT Array {
     }
     data_ = data;
   }
-
- private:
-  DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
 static inline std::ostream& operator<<(std::ostream& os, const Array& x) {
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index ed128fdf6b6f6..d5b2a26f6bc30 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -97,7 +97,7 @@ class ARROW_EXPORT Buffer {
   std::shared_ptr<Buffer> parent_;
 
  private:
-  DISALLOW_COPY_AND_ASSIGN(Buffer);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(Buffer);
 };
 
 /// \brief Create Buffer referencing std::string memory
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index daaf49c473d8f..71f0bab96f10e 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -793,8 +793,8 @@ Status DictionaryBuilder<T>::Append(const Scalar& value) {
     hash_slots_[j] = index;
     RETURN_NOT_OK(AppendDictionary(value));
 
-    if (UNLIKELY(static_cast<int32_t>(dict_builder_.length()) >
-                 hash_table_size_ * kMaxHashTableLoad)) {
+    if (ARROW_PREDICT_FALSE(static_cast<int32_t>(dict_builder_.length()) >
+                            hash_table_size_ * kMaxHashTableLoad)) {
       RETURN_NOT_OK(DoubleTableSize());
     }
   }
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 8e412667dab73..cb1e4ba69fe2a 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -156,7 +156,7 @@ class ARROW_EXPORT ArrayBuilder {
   void UnsafeSetNotNull(int64_t length);
 
  private:
-  DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
 };
 
 class ARROW_EXPORT NullBuilder : public ArrayBuilder {
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index beb22e7126ba9..1b84603ab90e5 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -261,7 +261,7 @@ class RangeEqualsVisitor {
   }
 
   Status Visit(const NullArray& left) {
-    UNUSED(left);
+    ARROW_UNUSED(left);
     result_ = true;
     return Status::OK();
   }
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index f5ed08281bce5..431ff5e7fe463 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -169,7 +169,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
   std::unique_ptr<HadoopFileSystemImpl> impl_;
 
   HadoopFileSystem();
-  DISALLOW_COPY_AND_ASSIGN(HadoopFileSystem);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(HadoopFileSystem);
 };
 
 #ifndef ARROW_NO_DEPRECATED_API
@@ -211,7 +211,7 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
 
   friend class HadoopFileSystem::HadoopFileSystemImpl;
 
-  DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
 };
 
 // Naming this file OutputStream because it does not support seeking (like the
@@ -238,7 +238,7 @@ class ARROW_EXPORT HdfsOutputStream : public OutputStream {
 
   HdfsOutputStream();
 
-  DISALLOW_COPY_AND_ASSIGN(HdfsOutputStream);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(HdfsOutputStream);
 };
 
 Status ARROW_EXPORT HaveLibHdfs();
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index dd3f32d3c3b00..59aed75a7630e 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -79,9 +79,6 @@ class ARROW_EXPORT FileInterface {
   FileInterface() {}
   FileMode::type mode_;
   void set_mode(FileMode::type mode) { mode_ = mode; }
-
- private:
-  DISALLOW_COPY_AND_ASSIGN(FileInterface);
 };
 
 class ARROW_EXPORT Seekable {
diff --git a/cpp/src/arrow/ipc/dictionary.h b/cpp/src/arrow/ipc/dictionary.h
index 13cf7a0b3d15c..4130e9ba8a0b9 100644
--- a/cpp/src/arrow/ipc/dictionary.h
+++ b/cpp/src/arrow/ipc/dictionary.h
@@ -68,7 +68,7 @@ class ARROW_EXPORT DictionaryMemo {
   // Map of dictionary id to dictionary array
   DictionaryMap id_to_dictionary_;
 
-  DISALLOW_COPY_AND_ASSIGN(DictionaryMemo);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(DictionaryMemo);
 };
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index 6fb8de7d26154..4bc4384f64eab 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -117,7 +117,7 @@ class ARROW_EXPORT Message {
   class MessageImpl;
   std::unique_ptr<MessageImpl> impl_;
 
-  DISALLOW_COPY_AND_ASSIGN(Message);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(Message);
 };
 
 ARROW_EXPORT std::string FormatMessageType(Message::Type type);
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index fe4d63b5c95a4..8ba3f4645fda8 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -260,7 +260,7 @@ class PandasBlock {
   int64_t* placement_data_;
 
  private:
-  DISALLOW_COPY_AND_ASSIGN(PandasBlock);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(PandasBlock);
 };
 
 template <typename T>
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index 7f94f9554f1e8..e315771b7a5fe 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -56,7 +56,7 @@ class ARROW_EXPORT PyAcquireGIL {
  private:
   bool acquired_gil_;
   PyGILState_STATE state_;
-  DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL);
 };
 
 #define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index d1cd3cf13c00b..4e4c6b8d57c33 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -111,7 +111,7 @@ class ARROW_EXPORT Tensor {
   std::vector<std::string> dim_names_;
 
  private:
-  DISALLOW_COPY_AND_ASSIGN(Tensor);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(Tensor);
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 7026aee0bdf32..aeb613084545b 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -157,9 +157,6 @@ class ARROW_EXPORT DataType {
  protected:
   Type::type id_;
   std::vector<std::shared_ptr<Field>> children_;
-
- private:
-  DISALLOW_COPY_AND_ASSIGN(DataType);
 };
 
 // TODO(wesm): Remove this from parquet-cpp
diff --git a/cpp/src/arrow/util/bit-stream-utils.h b/cpp/src/arrow/util/bit-stream-utils.h
index d312fef4d7d07..8c1f0678749b7 100644
--- a/cpp/src/arrow/util/bit-stream-utils.h
+++ b/cpp/src/arrow/util/bit-stream-utils.h
@@ -26,8 +26,8 @@
 
 #include "arrow/util/bit-util.h"
 #include "arrow/util/bpacking.h"
-#include "arrow/util/compiler-util.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
 
 namespace arrow {
 
@@ -171,12 +171,13 @@ inline bool BitWriter::PutValue(uint64_t v, int num_bits) {
   DCHECK_LE(num_bits, 32);
   DCHECK_EQ(v >> num_bits, 0) << "v = " << v << ", num_bits = " << num_bits;
 
-  if (UNLIKELY(byte_offset_ * 8 + bit_offset_ + num_bits > max_bytes_ * 8)) return false;
+  if (ARROW_PREDICT_FALSE(byte_offset_ * 8 + bit_offset_ + num_bits > max_bytes_ * 8))
+    return false;
 
   buffered_values_ |= v << bit_offset_;
   bit_offset_ += num_bits;
 
-  if (UNLIKELY(bit_offset_ >= 64)) {
+  if (ARROW_PREDICT_FALSE(bit_offset_ >= 64)) {
     // Flush buffered_values_ and write out bits of v that did not fit
     memcpy(buffer_ + byte_offset_, &buffered_values_, 8);
     buffered_values_ = 0;
@@ -247,7 +248,7 @@ inline void GetValue_(int num_bits, T* v, int max_bytes, const uint8_t* buffer,
     *bit_offset -= 64;
 
     int bytes_remaining = max_bytes - *byte_offset;
-    if (LIKELY(bytes_remaining >= 8)) {
+    if (ARROW_PREDICT_TRUE(bytes_remaining >= 8)) {
       memcpy(buffered_values, buffer + *byte_offset, 8);
     } else {
       memcpy(buffered_values, buffer + *byte_offset, bytes_remaining);
@@ -293,7 +294,7 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) {
   }
 
   int i = 0;
-  if (UNLIKELY(bit_offset != 0)) {
+  if (ARROW_PREDICT_FALSE(bit_offset != 0)) {
     for (; i < batch_size && bit_offset != 0; ++i) {
       detail::GetValue_(num_bits, &v[i], max_bytes, buffer, &bit_offset, &byte_offset,
                         &buffered_values);
@@ -355,7 +356,8 @@ template <typename T>
 inline bool BitReader::GetAligned(int num_bytes, T* v) {
   DCHECK_LE(num_bytes, static_cast<int>(sizeof(T)));
   int bytes_read = static_cast<int>(BitUtil::Ceil(bit_offset_, 8));
-  if (UNLIKELY(byte_offset_ + bytes_read + num_bytes > max_bytes_)) return false;
+  if (ARROW_PREDICT_FALSE(byte_offset_ + bytes_read + num_bytes > max_bytes_))
+    return false;
 
   // Advance byte_offset to next unread byte and read num_bytes
   byte_offset_ += bytes_read;
@@ -365,7 +367,7 @@ inline bool BitReader::GetAligned(int num_bytes, T* v) {
   // Reset buffered_values_
   bit_offset_ = 0;
   int bytes_remaining = max_bytes_ - byte_offset_;
-  if (LIKELY(bytes_remaining >= 8)) {
+  if (ARROW_PREDICT_TRUE(bytes_remaining >= 8)) {
     memcpy(&buffered_values_, buffer_ + byte_offset_, 8);
   } else {
     memcpy(&buffered_values_, buffer_ + byte_offset_, bytes_remaining);
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 52e78bf767c8c..da2826f405fac 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -39,7 +39,7 @@
 #include <memory>
 #include <vector>
 
-#include "arrow/util/compiler-util.h"
+#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 #ifdef ARROW_USE_SSE
@@ -251,7 +251,7 @@ static inline int PopcountNoHw(uint64_t x) {
 /// Returns the number of set bits in x
 static inline int Popcount(uint64_t x) {
 #ifdef ARROW_USE_SSE
-  if (LIKELY(CpuInfo::IsSupported(CpuInfo::POPCNT))) {
+  if (ARROW_PREDICT_TRUE(CpuInfo::IsSupported(CpuInfo::POPCNT))) {
     return POPCNT_popcnt_u64(x);
   } else {
     return PopcountNoHw(x);
@@ -270,8 +270,8 @@ static inline int PopcountSigned(T v) {
 
 /// Returns the 'num_bits' least-significant bits of 'v'.
 static inline uint64_t TrailingBits(uint64_t v, int num_bits) {
-  if (UNLIKELY(num_bits == 0)) return 0;
-  if (UNLIKELY(num_bits >= 64)) return v;
+  if (ARROW_PREDICT_FALSE(num_bits == 0)) return 0;
+  if (ARROW_PREDICT_FALSE(num_bits >= 64)) return v;
   int n = 64 - num_bits;
   return (v << n) >> n;
 }
diff --git a/cpp/src/arrow/util/compiler-util.h b/cpp/src/arrow/util/compiler-util.h
index ccbe54584b40c..820a9b0c11bee 100644
--- a/cpp/src/arrow/util/compiler-util.h
+++ b/cpp/src/arrow/util/compiler-util.h
@@ -15,45 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
+// Deprecated header, here for backwards compatibility in parquet-cpp
+
 #ifndef ARROW_UTIL_COMPILER_UTIL_H
 #define ARROW_UTIL_COMPILER_UTIL_H
 
-// Branch prediction macro hints for GCC
-#ifdef LIKELY
-#undef LIKELY
-#endif
-
-#ifdef UNLIKELY
-#undef UNLIKELY
-#endif
-
-#ifdef _MSC_VER
-#define LIKELY(expr) expr
-#define UNLIKELY(expr) expr
-#else
-#define LIKELY(expr) __builtin_expect(!!(expr), 1)
-#define UNLIKELY(expr) __builtin_expect(!!(expr), 0)
-#endif
-
-#define PREFETCH(addr) __builtin_prefetch(addr)
-
-// macros to disable padding
-// these macros are portable across different compilers and platforms
-//[https://github.com/google/flatbuffers/blob/master/include/flatbuffers/flatbuffers.h#L1355]
-#if defined(_MSC_VER)
-#define MANUALLY_ALIGNED_STRUCT(alignment) \
-  __pragma(pack(1));                       \
-  struct __declspec(align(alignment))
-#define STRUCT_END(name, size) \
-  __pragma(pack());            \
-  static_assert(sizeof(name) == size, "compiler breaks packing rules")
-#elif defined(__GNUC__) || defined(__clang__)
-#define MANUALLY_ALIGNED_STRUCT(alignment) \
-  _Pragma("pack(1)") struct __attribute__((aligned(alignment)))
-#define STRUCT_END(name, size) \
-  _Pragma("pack()") static_assert(sizeof(name) == size, "compiler breaks packing rules")
-#else
-#error Unknown compiler, please define structure alignment macros
-#endif
+#include "arrow/util/macros.h"
 
 #endif  // ARROW_UTIL_COMPILER_UTIL_H
diff --git a/cpp/src/arrow/util/compression_snappy.cc b/cpp/src/arrow/util/compression_snappy.cc
index 542831f405403..e284bd4358dd6 100644
--- a/cpp/src/arrow/util/compression_snappy.cc
+++ b/cpp/src/arrow/util/compression_snappy.cc
@@ -17,11 +17,6 @@
 
 #include "arrow/util/compression_snappy.h"
 
-// Work around warning caused by Snappy include
-#ifdef DISALLOW_COPY_AND_ASSIGN
-#undef DISALLOW_COPY_AND_ASSIGN
-#endif
-
 #include <cstddef>
 #include <cstdint>
 
diff --git a/cpp/src/arrow/util/hash-util.h b/cpp/src/arrow/util/hash-util.h
index 4c049c1697b72..d5fb212f33a88 100644
--- a/cpp/src/arrow/util/hash-util.h
+++ b/cpp/src/arrow/util/hash-util.h
@@ -22,9 +22,9 @@
 
 #include <cstdint>
 
-#include "arrow/util/compiler-util.h"
 #include "arrow/util/cpu-info.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
 #include "arrow/util/sse-util.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/key_value_metadata.h b/cpp/src/arrow/util/key_value_metadata.h
index 3d602131684f6..adea2bceabb73 100644
--- a/cpp/src/arrow/util/key_value_metadata.h
+++ b/cpp/src/arrow/util/key_value_metadata.h
@@ -55,7 +55,7 @@ class ARROW_EXPORT KeyValueMetadata {
   std::vector<std::string> keys_;
   std::vector<std::string> values_;
 
-  DISALLOW_COPY_AND_ASSIGN(KeyValueMetadata);
+  ARROW_DISALLOW_COPY_AND_ASSIGN(KeyValueMetadata);
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index fe2d7689bf641..93521498903bd 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -19,13 +19,13 @@
 #define ARROW_UTIL_MACROS_H
 
 // From Google gutil
-#ifndef DISALLOW_COPY_AND_ASSIGN
-#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
-  TypeName(const TypeName&) = delete;      \
+#ifndef ARROW_DISALLOW_COPY_AND_ASSIGN
+#define ARROW_DISALLOW_COPY_AND_ASSIGN(TypeName) \
+  TypeName(const TypeName&) = delete;            \
   TypeName& operator=(const TypeName&) = delete
 #endif
 
-#define UNUSED(x) (void)x
+#define ARROW_UNUSED(x) (void)x
 
 //
 // GCC can be told that a certain branch is not likely to be taken (for
@@ -37,14 +37,17 @@
 #define ARROW_PREDICT_FALSE(x) (__builtin_expect(x, 0))
 #define ARROW_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
 #define ARROW_NORETURN __attribute__((noreturn))
+#define ARROW_PREFETCH(addr) __builtin_prefetch(addr)
 #elif defined(_MSC_VER)
 #define ARROW_NORETURN __declspec(noreturn)
 #define ARROW_PREDICT_FALSE(x) x
 #define ARROW_PREDICT_TRUE(x) x
+#define ARROW_PREFETCH(addr)
 #else
 #define ARROW_NORETURN
 #define ARROW_PREDICT_FALSE(x) x
 #define ARROW_PREDICT_TRUE(x) x
+#define ARROW_PREFETCH(addr)
 #endif
 
 #if (defined(__GNUC__) || defined(__APPLE__))
@@ -55,4 +58,25 @@
 #define ARROW_MUST_USE_RESULT
 #endif
 
+// macros to disable padding
+// these macros are portable across different compilers and platforms
+//[https://github.com/google/flatbuffers/blob/master/include/flatbuffers/flatbuffers.h#L1355]
+#if !defined(MANUALLY_ALIGNED_STRUCT)
+#if defined(_MSC_VER)
+#define MANUALLY_ALIGNED_STRUCT(alignment) \
+  __pragma(pack(1));                             \
+  struct __declspec(align(alignment))
+#define STRUCT_END(name, size) \
+  __pragma(pack());            \
+  static_assert(sizeof(name) == size, "compiler breaks packing rules")
+#elif defined(__GNUC__) || defined(__clang__)
+#define MANUALLY_ALIGNED_STRUCT(alignment) \
+  _Pragma("pack(1)") struct __attribute__((aligned(alignment)))
+#define STRUCT_END(name, size) \
+  _Pragma("pack()") static_assert(sizeof(name) == size, "compiler breaks packing rules")
+#else
+#error Unknown compiler, please define structure alignment macros
+#endif
+#endif // !defined(MANUALLY_ALIGNED_STRUCT)
+
 #endif  // ARROW_UTIL_MACROS_H
diff --git a/cpp/src/arrow/util/rle-encoding.h b/cpp/src/arrow/util/rle-encoding.h
index e69077807df3a..f4c8a772efc8c 100644
--- a/cpp/src/arrow/util/rle-encoding.h
+++ b/cpp/src/arrow/util/rle-encoding.h
@@ -26,7 +26,7 @@
 
 #include "arrow/util/bit-stream-utils.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/compiler-util.h"
+#include "arrow/util/macros.h"
 
 namespace arrow {
 
@@ -446,9 +446,9 @@ bool RleDecoder::NextCounts() {
 /// it decides whether they should be encoded as a literal or repeated run.
 inline bool RleEncoder::Put(uint64_t value) {
   DCHECK(bit_width_ == 64 || value < (1ULL << bit_width_));
-  if (UNLIKELY(buffer_full_)) return false;
+  if (ARROW_PREDICT_FALSE(buffer_full_)) return false;
 
-  if (LIKELY(current_value_ == value)) {
+  if (ARROW_PREDICT_TRUE(current_value_ == value)) {
     ++repeat_count_;
     if (repeat_count_ > 8) {
       // This is just a continuation of the current run, no need to buffer the

From 016c45aa2031eebe118763b8465cf19a45781313 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sun, 3 Sep 2017 15:11:00 -0400
Subject: [PATCH 0988/1644] ARROW-1450: [Python] Raise proper error if custom
 serialization handler fails

This fixes the serialize and deserialize methods to throw the right exceptions.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1029 from pcmoritz/pyarrow-serialization-error and squashes the following commits:

249b38ed [Philipp Moritz] formatting
2d872a4c [Philipp Moritz] fixes
caad76d1 [Philipp Moritz] add test for deserialization (needs fixing)
57772124 [Philipp Moritz] fix python serialization error propagation
---
 cpp/src/arrow/python/common.cc             |  9 +++++++++
 cpp/src/arrow/python/common.h              |  2 ++
 cpp/src/arrow/python/python_to_arrow.cc    |  2 +-
 cpp/src/arrow/status.h                     |  3 +++
 python/pyarrow/error.pxi                   |  3 +++
 python/pyarrow/includes/common.pxd         |  1 +
 python/pyarrow/serialization.pxi           |  4 +++-
 python/pyarrow/tests/test_serialization.py | 23 ++++++++++++++++++++++
 8 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
index a248db3abca48..53bd57bf6da94 100644
--- a/cpp/src/arrow/python/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -81,5 +81,14 @@ Status CheckPyError(StatusCode code) {
   return Status::OK();
 }
 
+Status PassPyError() {
+  if (PyErr_Occurred()) {
+    // Do not call PyErr_Clear, the assumption is that someone further
+    // up the call stack will want to deal with the Python error.
+    return Status(StatusCode::PythonError, "");
+  }
+  return Status::OK();
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index e315771b7a5fe..e3fe2ef429a19 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -145,6 +145,8 @@ struct ARROW_EXPORT PyObjectStringify {
 
 Status CheckPyError(StatusCode code = StatusCode::UnknownError);
 
+Status PassPyError();
+
 // TODO(wesm): We can just let errors pass through. To be explored later
 #define RETURN_IF_PYERROR() RETURN_NOT_OK(CheckPyError());
 
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index c5cfd6e6cff1a..d823bad7373c7 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -337,7 +337,7 @@ Status CallCustomCallback(PyObject* context, PyObject* method_name, PyObject* el
     return Status::SerializationError(ss.str());
   } else {
     *result = PyObject_CallMethodObjArgs(context, method_name, elem, NULL);
-    RETURN_IF_PYERROR();
+    return PassPyError();
   }
   return Status::OK();
 }
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 7a39d0f3ab5b8..ece83ac932576 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -72,6 +72,7 @@ enum class StatusCode : char {
   UnknownError = 9,
   NotImplemented = 10,
   SerializationError = 11,
+  PythonError = 12,
   PlasmaObjectExists = 20,
   PlasmaObjectNonexistent = 21,
   PlasmaStoreFull = 22
@@ -154,6 +155,8 @@ class ARROW_EXPORT Status {
   bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
   // An object could not be serialized or deserialized.
   bool IsSerializationError() const { return code() == StatusCode::SerializationError; }
+  // An error is propagated from a nested Python function.
+  bool IsPythonError() const { return code() == StatusCode::PythonError; }
   // An object with this object ID already exists in the plasma store.
   bool IsPlasmaObjectExists() const { return code() == StatusCode::PlasmaObjectExists; }
   // An object was requested that doesn't exist in the plasma store.
diff --git a/python/pyarrow/error.pxi b/python/pyarrow/error.pxi
index 2a213026c6550..dfdfcd73af6ab 100644
--- a/python/pyarrow/error.pxi
+++ b/python/pyarrow/error.pxi
@@ -68,6 +68,9 @@ cdef int check_status(const CStatus& status) nogil except -1:
     if status.ok():
         return 0
 
+    if status.IsPythonError():
+        return -1
+
     with gil:
         message = frombytes(status.message())
         if status.IsInvalid():
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 1bd840cc53ff2..f323feaff9ff0 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -52,6 +52,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsNotImplemented()
         c_bool IsTypeError()
         c_bool IsSerializationError()
+        c_bool IsPythonError()
         c_bool IsPlasmaObjectExists()
         c_bool IsPlasmaObjectNonexistent()
         c_bool IsPlasmaStoreFull()
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index f38845eea69e0..aa1a6a4bf5289 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -118,7 +118,9 @@ cdef class SerializationContext:
         else:
             assert type_id not in self.types_to_pickle
             if type_id not in self.whitelisted_types:
-                raise "error"
+                msg = "Type ID " + str(type_id) + " not registered in " \
+                      "deserialization callback"
+                raise DeserializationCallbackError(msg, type_id)
             type_ = self.whitelisted_types[type_id]
             if type_id in self.custom_deserializers:
                 obj = self.custom_deserializers[type_id](
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 4e98bd5706aff..5526ac68cab76 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -255,3 +255,26 @@ def test_custom_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for obj in CUSTOM_OBJECTS:
             serialization_roundtrip(obj, mmap)
+
+def test_serialization_callback_error():
+
+    class TempClass(object):
+            pass
+
+    # Pass a SerializationContext into serialize, but TempClass
+    # is not registered
+    serialization_context = pa.SerializationContext()
+    val = TempClass()
+    with pytest.raises(pa.SerializationCallbackError) as err:
+        serialized_object = pa.serialize(val, serialization_context)
+    assert err.value.example_object == val
+
+    serialization_context.register_type(TempClass, 20*b"\x00")
+    serialized_object = pa.serialize(TempClass(), serialization_context)
+    deserialization_context = pa.SerializationContext()
+    
+    # Pass a Serialization Context into deserialize, but TempClass
+    # is not registered
+    with pytest.raises(pa.DeserializationCallbackError) as err:
+        serialized_object.deserialize(deserialization_context)
+    assert err.value.type_id == 20*b"\x00"

From 08b41f93462eb4e55aa036178286288774859cb8 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Sun, 3 Sep 2017 15:54:50 -0400
Subject: [PATCH 0989/1644] ARROW-1238: [Java] Adding Decimal type JSON read
 and write support

This change adds support to reading and writing Decimal type vectors from JSON files.  Data values are written as encoded hex and padded with 0's up to 16 bytes.

Added roundtrip unit tests.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #994 from BryanCutler/java-json-decimal-support-ARROW-1238 and squashes the following commits:

28c1e3e6 [Bryan Cutler] added test for BigDecimal precision and scale mismatch
31b7ec18 [Bryan Cutler] Added check that BigDecimal precision and scale matches that of the vector
10cac9cc [Bryan Cutler] added vector API for set and setSafe with BigDecimal
c5e8fba1 [Bryan Cutler] minor tweaks to JsonFileWriter
da11b4f6 [Bryan Cutler] removed debug line
f4560d99 [Bryan Cutler] added Decimal JSON support, Java roundtrip unit tests
---
 .../templates/AbstractFieldWriter.java        |  7 ++
 .../codegen/templates/ComplexWriters.java     | 11 +++
 .../codegen/templates/FixedValueVectors.java  | 18 ++++-
 .../templates/NullableValueVectors.java       | 13 +++
 .../vector/file/json/JsonFileReader.java      | 14 +++-
 .../vector/file/json/JsonFileWriter.java      | 16 +++-
 .../arrow/vector/util/DecimalUtility.java     | 75 +++++++++++++++---
 .../arrow/vector/TestDecimalVector.java       | 79 +++++++++++++++----
 .../arrow/vector/file/BaseFileTest.java       | 72 ++++++++++++-----
 .../arrow/vector/file/json/TestJSONFile.java  | 32 ++++++++
 10 files changed, 280 insertions(+), 57 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index da8e4f54ec228..853f67fd0dd56 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -53,6 +53,7 @@ public void endList() {
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
+  <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
   @Override
   public void write(${name}Holder holder) {
     fail("${name}");
@@ -62,6 +63,12 @@ public void write(${name}Holder holder) {
     fail("${name}");
   }
 
+  <#if minor.class == "Decimal">
+  public void write${minor.class}(${friendlyType} value) {
+    fail("${name}");
+  }
+  </#if>
+
   </#list></#list>
 
   public void writeNull() {
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 8ebecf3e1de69..fe099bede3568 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -24,6 +24,7 @@
 <#assign eName = name />
 <#assign javaType = (minor.javaType!type.javaType) />
 <#assign fields = minor.fields!type.fields />
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
 
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${eName}WriterImpl.java" />
 <#include "/@includes/license.ftl" />
@@ -115,7 +116,13 @@ public void write(Nullable${minor.class}Holder h) {
     mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
     vector.getMutator().setValueCount(idx()+1);
   }
+  <#if minor.class == "Decimal">
 
+  public void write${minor.class}(${friendlyType} value) {
+    mutator.setSafe(idx(), value);
+    vector.getMutator().setValueCount(idx()+1);
+  }
+  </#if>
   <#if mode == "Nullable">
 
   public void writeNull() {
@@ -140,6 +147,10 @@ public interface ${eName}Writer extends BaseWriter {
   public void write(${minor.class}Holder h);
 
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
+<#if minor.class == "Decimal">
+
+  public void write${minor.class}(${friendlyType} value);
+</#if>
 }
 
 </#list>
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 9747d421c41fc..ffd8cad02e2c3 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -403,7 +403,7 @@ public void get(int index, Nullable${minor.class}Holder holder) {
 
     @Override
     public ${friendlyType} getObject(int index) {
-      return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(data, ${type.width} * index, scale);
+      return DecimalUtility.getBigDecimalFromArrowBuf(data, index, scale);
     }
 
       <#else>
@@ -596,10 +596,10 @@ void set(int index, Nullable${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
    }
 
-   public void setSafe(int index,  Nullable${minor.class}Holder holder){
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
      setSafe(index, holder.start, holder.buffer);
    }
-   public void setSafe(int index,  ${minor.class}Holder holder){
+   public void setSafe(int index, ${minor.class}Holder holder){
      setSafe(index, holder.start, holder.buffer);
    }
 
@@ -614,6 +614,18 @@ public void set(int index, int start, ArrowBuf buffer){
      data.setBytes(index * ${type.width}, buffer, start, ${type.width});
    }
 
+   public void set(int index, ${friendlyType} value){
+     DecimalUtility.checkPrecisionAndScale(value, precision, scale);
+     DecimalUtility.writeBigDecimalToArrowBuf(value, data, index);
+   }
+
+   public void setSafe(int index, ${friendlyType} value){
+     while(index >= getValueCapacity()) {
+       reAlloc();
+     }
+     set(index, value);
+   }
+
        <#else>
    protected void set(int index, ${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index a431333256394..319c61c86244b 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -723,6 +723,19 @@ public void setSafe(int index, ${minor.javaType!type.javaType} value) {
       setCount++;
     }
 
+    </#if>
+    <#if minor.class == "Decimal">
+    public void set(int index, ${friendlyType} value) {
+      bits.getMutator().setToOne(index);
+      values.getMutator().set(index, value);
+    }
+
+    public void setSafe(int index, ${friendlyType} value) {
+      bits.getMutator().setSafeToOne(index);
+      values.getMutator().setSafe(index, value);
+      setCount++;
+    }
+
     </#if>
     @Override
     public void setValueCount(int valueCount) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 484a82fdaab67..71685d1358923 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -39,6 +39,7 @@
 import org.apache.arrow.vector.BufferBacked;
 import org.apache.arrow.vector.DateDayVector;
 import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.Float4Vector;
 import org.apache.arrow.vector.Float8Vector;
@@ -72,6 +73,7 @@
 import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.DecimalUtility;
 import org.apache.arrow.vector.util.DictionaryUtility;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
@@ -235,14 +237,16 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         nextFieldIs(vectorType.getName());
         readToken(START_ARRAY);
         ValueVector valueVector = (ValueVector) innerVector;
-        valueVector.allocateNew();
-        Mutator mutator = valueVector.getMutator();
 
         int innerVectorCount = vectorType.equals(OFFSET) ? count + 1 : count;
+        valueVector.setInitialCapacity(innerVectorCount);
+        valueVector.allocateNew();
+
         for (int i = 0; i < innerVectorCount; i++) {
           parser.nextToken();
           setValueFromParser(valueVector, i);
         }
+        Mutator mutator = valueVector.getMutator();
         mutator.setValueCount(innerVectorCount);
         readToken(END_ARRAY);
       }
@@ -312,6 +316,12 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
       case FLOAT8:
         ((Float8Vector) valueVector).getMutator().set(i, parser.readValueAs(Double.class));
         break;
+      case DECIMAL: {
+          DecimalVector decimalVector = ((DecimalVector) valueVector);
+          byte[] value = decodeHexSafe(parser.readValueAs(String.class));
+          DecimalUtility.writeByteArrayToArrowBuf(value, decimalVector.getBuffer(), i);
+        }
+        break;
       case VARBINARY:
         ((VarBinaryVector) valueVector).getMutator().setSafe(i, decodeHexSafe(parser.readValueAs(String.class)));
         break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index a2229cef23150..04e44379e5dfa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -26,10 +26,12 @@
 import java.util.Set;
 
 import com.google.common.collect.ImmutableList;
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.BufferBacked;
 import org.apache.arrow.vector.DateDayVector;
 import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.TimeMicroVector;
 import org.apache.arrow.vector.TimeMilliVector;
@@ -54,6 +56,7 @@
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
 import com.fasterxml.jackson.databind.MappingJsonFactory;
+import org.apache.arrow.vector.util.DecimalUtility;
 import org.apache.arrow.vector.util.DictionaryUtility;
 import org.apache.commons.codec.binary.Hex;
 
@@ -233,9 +236,16 @@ private void writeValueToGenerator(ValueVector valueVector, int i) throws IOExce
       case BIT:
         generator.writeNumber(((BitVector) valueVector).getAccessor().get(i));
         break;
-      case VARBINARY:
-        String hexString = Hex.encodeHexString(((VarBinaryVector) valueVector).getAccessor().get(i));
-        generator.writeObject(hexString);
+      case VARBINARY: {
+          String hexString = Hex.encodeHexString(((VarBinaryVector) valueVector).getAccessor().get(i));
+          generator.writeString(hexString);
+        }
+        break;
+      case DECIMAL: {
+          ArrowBuf bytebuf = valueVector.getDataBuffer();
+          String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(bytebuf, i));
+          generator.writeString(hexString);
+        }
         break;
       default:
         // TODO: each type
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index 4b11b368dff1e..033ae6c09914d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -19,13 +19,12 @@
 package org.apache.arrow.vector.util;
 
 import io.netty.buffer.ArrowBuf;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.UnpooledByteBufAllocator;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
-import java.util.Arrays;
 
 
 public class DecimalUtility {
@@ -69,7 +68,7 @@ public class DecimalUtility {
 
   public static final int DECIMAL_BYTE_LENGTH = 16;
 
-  /*
+  /**
    * Simple function that returns the static precomputed
    * power of ten, instead of using Math.pow
    */
@@ -78,7 +77,7 @@ public static long getPowerOfTen(int power) {
     return scale_long_constants[(power)];
   }
 
-  /*
+  /**
    * Math.pow returns a double and while multiplying with large digits
    * in the decimal data type we encounter noise. So instead of multiplying
    * with Math.pow we use the static constants to perform the multiplication
@@ -103,7 +102,8 @@ public static long adjustScaleDivide(long input, int factor) {
     }
   }
 
-  /* Returns a string representation of the given integer
+  /**
+   * Returns a string representation of the given integer
    * If the length of the given integer is less than the
    * passed length, this function will prepend zeroes to the string
    */
@@ -136,33 +136,86 @@ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
     return str;
   }
 
-  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int startIndex, int scale) {
+  /**
+   * Read an ArrowType.Decimal at the given value index in the ArrowBuf and convert to a BigDecimal
+   * with the given scale.
+   */
+  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale) {
     byte[] value = new byte[DECIMAL_BYTE_LENGTH];
+    final int startIndex = index * DECIMAL_BYTE_LENGTH;
     bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH);
     BigInteger unscaledValue = new BigInteger(value);
     return new BigDecimal(unscaledValue, scale);
   }
 
-  public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int scale) {
+  /**
+   * Read an ArrowType.Decimal from the ByteBuffer and convert to a BigDecimal with the given
+   * scale.
+   */
+  public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int scale) {
     byte[] value = new byte[DECIMAL_BYTE_LENGTH];
     bytebuf.get(value);
     BigInteger unscaledValue = new BigInteger(value);
     return new BigDecimal(unscaledValue, scale);
   }
 
+  /**
+   * Read an ArrowType.Decimal from the ArrowBuf at the given value index and return it as a byte
+   * array.
+   */
+  public static byte[] getByteArrayFromArrowBuf(ArrowBuf bytebuf, int index) {
+    final byte[] value = new byte[DECIMAL_BYTE_LENGTH];
+    final int startIndex = index * DECIMAL_BYTE_LENGTH;
+    bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH);
+    return value;
+  }
+
+  /**
+   * Check that the BigDecimal scale equals the vectorScale and that the BigDecimal precision is
+   * less than or equal to the vectorPrecision. If not, then an UnsupportedOperationException is
+   * thrown, otherwise returns true.
+   */
+  public static boolean checkPrecisionAndScale(BigDecimal value, int vectorPrecision, int vectorScale) {
+    if (value.scale() != vectorScale) {
+      throw new UnsupportedOperationException("BigDecimal scale must equal that in the Arrow vector: " +
+          value.scale() + " != " + vectorScale);
+    }
+    if (value.precision() > vectorPrecision) {
+      throw new UnsupportedOperationException("BigDecimal precision can not be greater than that in the Arrow vector: " +
+          value.precision() + " > " + vectorPrecision);
+    }
+    return true;
+  }
+
+  /**
+   * Write the given BigDecimal to the ArrowBuf at the given value index. Will throw an
+   * UnsupportedOperationException if the decimal size is greater than the Decimal vector byte
+   * width.
+   */
   public static void writeBigDecimalToArrowBuf(BigDecimal value, ArrowBuf bytebuf, int index) {
     final byte[] bytes = value.unscaledValue().toByteArray();
+    final int padValue = value.signum() == -1 ? 0xFF : 0;
+    writeByteArrayToArrowBuf(bytes, bytebuf, index, padValue);
+  }
+
+  /**
+   * Write the given byte array to the ArrowBuf at the given value index. Will throw an
+   * UnsupportedOperationException if the decimal size is greater than the Decimal vector byte
+   * width.
+   */
+  public static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int index) {
+    writeByteArrayToArrowBuf(bytes, bytebuf, index, 0);
+  }
+
+  private static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int index, int padValue) {
     final int startIndex = index * DECIMAL_BYTE_LENGTH;
     if (bytes.length > DECIMAL_BYTE_LENGTH) {
       throw new UnsupportedOperationException("Decimal size greater than 16 bytes");
     }
     final int padLength = DECIMAL_BYTE_LENGTH - bytes.length;
-    final int padValue = value.signum() == -1 ? 0xFF : 0;
     for (int i = 0; i < padLength; i++) {
       bytebuf.setByte(startIndex + i, padValue);
     }
     bytebuf.setBytes(startIndex + padLength, bytes, 0, bytes.length);
   }
 }
-
-
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index 774fbe084f1c2..56d2293276404 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -19,6 +19,7 @@
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -27,6 +28,8 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.util.DecimalUtility;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 public class TestDecimalVector {
@@ -43,27 +46,69 @@ public class TestDecimalVector {
 
   private int scale = 3;
 
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
   @Test
-  public void test() {
-    BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
-    NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(10, scale), allocator);
-    try (NullableDecimalVector oldConstructor = new NullableDecimalVector("decimal", allocator, 10, scale);) {
-      assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType());
-    }
-    decimalVector.allocateNew();
-    BigDecimal[] values = new BigDecimal[intValues.length];
-    for (int i = 0; i < intValues.length; i++) {
-      BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale);
-      values[i] = decimal;
-      decimalVector.getMutator().setIndexDefined(i);
-      DecimalUtility.writeBigDecimalToArrowBuf(decimal, decimalVector.getBuffer(), i);
+  public void testValuesWriteRead() {
+    try (NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(10, scale), allocator);) {
+
+      try (NullableDecimalVector oldConstructor = new NullableDecimalVector("decimal", allocator, 10, scale);) {
+        assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType());
+      }
+
+      decimalVector.allocateNew();
+      BigDecimal[] values = new BigDecimal[intValues.length];
+      for (int i = 0; i < intValues.length; i++) {
+        BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale);
+        values[i] = decimal;
+        decimalVector.getMutator().setSafe(i, decimal);
+      }
+
+      decimalVector.getMutator().setValueCount(intValues.length);
+
+      for (int i = 0; i < intValues.length; i++) {
+        BigDecimal value = decimalVector.getAccessor().getObject(i);
+        assertEquals(values[i], value);
+      }
     }
+  }
+
+  @Test
+  public void testBigDecimalDifferentScaleAndPrecision() {
+    try (NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(4, 2), allocator);) {
+      decimalVector.allocateNew();
 
-    decimalVector.getMutator().setValueCount(intValues.length);
+      // test BigDecimal with different scale
+      boolean hasError = false;
+      try {
+        BigDecimal decimal = new BigDecimal(BigInteger.valueOf(0), 3);
+        decimalVector.getMutator().setSafe(0, decimal);
+      } catch (UnsupportedOperationException ue) {
+        hasError = true;
+      } finally {
+        assertTrue(hasError);
+      }
 
-    for (int i = 0; i < intValues.length; i++) {
-      BigDecimal value = decimalVector.getAccessor().getObject(i);
-      assertEquals(values[i], value);
+      // test BigDecimal with larger precision than initialized
+      hasError = false;
+      try {
+        BigDecimal decimal = new BigDecimal(BigInteger.valueOf(12345), 2);
+        decimalVector.getMutator().setSafe(0, decimal);
+      } catch (UnsupportedOperationException ue) {
+        hasError = true;
+      } finally {
+        assertTrue(hasError);
+      }
     }
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 732bd98b7c61c..c05d59049776f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -18,6 +18,8 @@
 
 package org.apache.arrow.vector.file;
 
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
@@ -27,6 +29,7 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.NullableDateMilliVector;
+import org.apache.arrow.vector.NullableDecimalVector;
 import org.apache.arrow.vector.NullableIntVector;
 import org.apache.arrow.vector.NullableTimeMilliVector;
 import org.apache.arrow.vector.NullableVarCharVector;
@@ -56,7 +59,6 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.DateUtility;
-import org.apache.arrow.vector.util.DictionaryUtility;
 import org.apache.arrow.vector.util.Text;
 import org.joda.time.DateTimeZone;
 import org.joda.time.LocalDateTime;
@@ -314,7 +316,6 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider
     Assert.assertEquals(1, accessor.getObject(4));
     Assert.assertEquals(0, accessor.getObject(5));
 
-
     FieldVector vector2 = root.getVector("sizes");
     Assert.assertNotNull(vector2);
 
@@ -408,28 +409,57 @@ protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvide
     Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
   }
 
-  protected void validateNestedDictionary(ListVector vector, DictionaryProvider provider) {
-    Assert.assertNotNull(vector);
-    Assert.assertNull(vector.getField().getDictionary());
-    Field nestedField = vector.getField().getChildren().get(0);
+  protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) {
+    NullableDecimalVector decimalVector1 = new NullableDecimalVector("decimal1", bufferAllocator, 10, 3);
+    NullableDecimalVector decimalVector2 = new NullableDecimalVector("decimal2", bufferAllocator, 4, 2);
+    NullableDecimalVector decimalVector3 = new NullableDecimalVector("decimal3", bufferAllocator, 16, 8);
 
-    DictionaryEncoding encoding = nestedField.getDictionary();
-    Assert.assertNotNull(encoding);
-    Assert.assertEquals(2L, encoding.getId());
-    Assert.assertEquals(new ArrowType.Int(32, true), encoding.getIndexType());
+    int count = 10;
+    decimalVector1.allocateNew(count);
+    decimalVector2.allocateNew(count);
+    decimalVector3.allocateNew(count);
 
-    ListVector.Accessor accessor = vector.getAccessor();
-    Assert.assertEquals(3, accessor.getValueCount());
-    Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
-    Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
-    Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
+    for (int i = 0; i < count; i++) {
+      decimalVector1.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3));
+      decimalVector2.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2));
+      decimalVector3.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8));
+    }
 
-    Dictionary dictionary = provider.lookup(2L);
-    Assert.assertNotNull(dictionary);
-    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
-    Assert.assertEquals(2, dictionaryAccessor.getValueCount());
-    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
-    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+    decimalVector1.getMutator().setValueCount(count);
+    decimalVector2.getMutator().setValueCount(count);
+    decimalVector3.getMutator().setValueCount(count);
+
+    List<Field> fields = ImmutableList.of(decimalVector1.getField(), decimalVector2.getField(), decimalVector3.getField());
+    List<FieldVector> vectors = ImmutableList.<FieldVector>of(decimalVector1, decimalVector2, decimalVector3);
+    return new VectorSchemaRoot(fields, vectors, count);
+  }
+
+  protected void validateDecimalData(VectorSchemaRoot root) {
+    NullableDecimalVector decimalVector1 = (NullableDecimalVector) root.getVector("decimal1");
+    NullableDecimalVector decimalVector2 = (NullableDecimalVector) root.getVector("decimal2");
+    NullableDecimalVector decimalVector3 = (NullableDecimalVector) root.getVector("decimal3");
+    int count = 10;
+    Assert.assertEquals(count, root.getRowCount());
+
+    for (int i = 0; i < count; i++) {
+      // Verify decimal 1 vector
+      BigDecimal readValue = decimalVector1.getAccessor().getObject(i);
+      ArrowType.Decimal type = (ArrowType.Decimal) decimalVector1.getField().getType();
+      BigDecimal genValue = new BigDecimal(BigInteger.valueOf(i), type.getScale());
+      Assert.assertEquals(genValue, readValue);
+
+      // Verify decimal 2 vector
+      readValue = decimalVector2.getAccessor().getObject(i);
+      type = (ArrowType.Decimal) decimalVector2.getField().getType();
+      genValue = new BigDecimal(BigInteger.valueOf(i * (1 << 10)), type.getScale());
+      Assert.assertEquals(genValue, readValue);
+
+      // Verify decimal 3 vector
+      readValue = decimalVector3.getAccessor().getObject(i);
+      type = (ArrowType.Decimal) decimalVector3.getField().getType();
+      genValue = new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), type.getScale());
+      Assert.assertEquals(genValue, readValue);
+    }
   }
 
   protected void writeData(int count, MapVector parent) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 24b2138386da1..b7c06327291e8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -237,6 +237,38 @@ public void testWriteReadNestedDictionaryJSON() throws IOException {
     }
   }
 
+  @Test
+  public void testWriteReadDecimalJSON() throws IOException {
+    File file = new File("target/mytest_decimal.json");
+
+    // write
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE)
+    ) {
+
+      try (VectorSchemaRoot root = writeDecimalData(vectorAllocator)) {
+        printVectors(root.getFieldVectors());
+        validateDecimalData(root);
+        writeJSON(file, root, null);
+      }
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+    ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateDecimalData(root);
+      }
+      reader.close();
+    }
+  }
+
   @Test
   public void testSetStructLength() throws IOException {
     File file = new File("../../integration/data/struct_example.json");

From 6f2fd8dbf727a6155406e507ee378c86698f976c Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 3 Sep 2017 18:28:50 -0400
Subject: [PATCH 0990/1644] ARROW-1449: Implement Decimal using only Int128

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1028 from cpcloud/decimal-single-representation and squashes the following commits:

2ee85a1e [Phillip Cloud] Formatting
0dfceff5 [Phillip Cloud] Don't assign a new string if not necessary
25853e94 [Phillip Cloud] Use STL instead of while loop
bd5e6838 [Phillip Cloud] Fix appveyor warning from stati_cast
04368e56 [Phillip Cloud] Move template implementation to header
18c59489 [Phillip Cloud] Static cast result of std::isdigit to bool
6e967757 [Phillip Cloud] Fix macros
f72f5466 [Phillip Cloud] Formatting
759247e4 [Phillip Cloud] Remove unused header
41fc9a0f [Phillip Cloud] IWYU
ccac0bda [Phillip Cloud] Fix complaint about bool -> Int128
9908d9c3 [Phillip Cloud] Formatting
3d8e399e [Phillip Cloud] Ignore short and long usage linting in decimal-test.cc
8cfa1785 [Phillip Cloud] Make sure we can construct from any integral type
cd7189fa [Phillip Cloud] ARROW-1449: Implement Decimal using only Int128
---
 cpp/src/arrow/array-decimal-test.cc     | 190 +++++++-----------------
 cpp/src/arrow/array.cc                  |  23 +--
 cpp/src/arrow/buffer.h                  |  14 ++
 cpp/src/arrow/builder.cc                |  23 +--
 cpp/src/arrow/builder.h                 |  14 +-
 cpp/src/arrow/compare.cc                |  67 +--------
 cpp/src/arrow/python/arrow_to_pandas.cc |  49 +-----
 cpp/src/arrow/python/builtin_convert.cc |  37 ++---
 cpp/src/arrow/python/helpers.cc         |   4 +-
 cpp/src/arrow/python/pandas_to_arrow.cc |  27 +---
 cpp/src/arrow/type-test.cc              |  14 +-
 cpp/src/arrow/type.h                    |  15 +-
 cpp/src/arrow/util/decimal-test.cc      | 190 ++++++++++++------------
 cpp/src/arrow/util/decimal.cc           | 149 ++++++++-----------
 cpp/src/arrow/util/decimal.h            | 110 +-------------
 cpp/src/arrow/util/int128.cc            |  45 +++---
 cpp/src/arrow/util/int128.h             |  55 +++----
 cpp/src/arrow/util/macros.h             |   4 +-
 18 files changed, 340 insertions(+), 690 deletions(-)

diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 7bbf2f00fc0f3..5f3a4f15dfb46 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -24,12 +24,9 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
-#include "arrow/memory_pool.h"
-#include "arrow/status.h"
 #include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 
 using std::size_t;
 
@@ -39,180 +36,97 @@ class Buffer;
 
 namespace decimal {
 
-template <typename T>
-class DecimalTestBase {
- public:
-  DecimalTestBase() : pool_(default_memory_pool()) {}
+using DecimalVector = std::vector<Int128>;
 
-  virtual std::vector<uint8_t> MakeData(const std::vector<T>& input,
-                                        size_t byte_width) const = 0;
+class DecimalTest : public ::testing::TestWithParam<int> {
+ public:
+  DecimalTest() {}
 
-  void InitBuilder(const std::shared_ptr<DecimalType>& type, const std::vector<T>& draw,
-                   const std::vector<uint8_t>& valid_bytes, int byte_width,
-                   std::shared_ptr<DecimalBuilder>* builder, size_t* null_count) const {
-    *builder = std::make_shared<DecimalBuilder>(type, pool_);
+  template <size_t BYTE_WIDTH = 16>
+  void MakeData(const DecimalVector& input, std::vector<uint8_t>* out) const {
+    out->reserve(input.size() * BYTE_WIDTH);
 
-    size_t size = draw.size();
-    ASSERT_OK((*builder)->Reserve(size));
+    std::array<uint8_t, BYTE_WIDTH> bytes{{0}};
 
-    for (size_t i = 0; i < size; ++i) {
-      if (valid_bytes[i]) {
-        ASSERT_OK((*builder)->Append(draw[i]));
-      } else {
-        ASSERT_OK((*builder)->AppendNull());
-        ++*null_count;
-      }
+    for (const auto& value : input) {
+      ASSERT_OK(value.ToBytes(&bytes));
+      out->insert(out->end(), bytes.cbegin(), bytes.cend());
     }
   }
 
-  void TestCreate(int precision, const std::vector<T>& draw,
+  template <size_t BYTE_WIDTH = 16>
+  void TestCreate(int precision, const DecimalVector& draw,
                   const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
     auto type = std::make_shared<DecimalType>(precision, 4);
 
-    std::shared_ptr<DecimalBuilder> builder;
+    auto builder = std::make_shared<DecimalBuilder>(type);
 
     size_t null_count = 0;
 
     const size_t size = draw.size();
-    const int byte_width = type->byte_width();
-
-    InitBuilder(type, draw, valid_bytes, byte_width, &builder, &null_count);
 
-    auto raw_bytes = MakeData(draw, static_cast<size_t>(byte_width));
-    auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), size * byte_width);
-    std::shared_ptr<Buffer> expected_null_bitmap;
-    ASSERT_OK(BitUtil::BytesToBits(valid_bytes, &expected_null_bitmap));
+    ASSERT_OK(builder->Reserve(size));
 
-    int64_t expected_null_count = test::null_count(valid_bytes);
-    auto expected = std::make_shared<DecimalArray>(
-        type, size, expected_data, expected_null_bitmap, expected_null_count, 0);
+    for (size_t i = 0; i < size; ++i) {
+      if (valid_bytes[i]) {
+        ASSERT_OK(builder->Append(draw[i]));
+      } else {
+        ASSERT_OK(builder->AppendNull());
+        ++null_count;
+      }
+    }
 
     std::shared_ptr<Array> out;
     ASSERT_OK(builder->Finish(&out));
-    ASSERT_TRUE(out->Slice(offset)->Equals(
-        *expected->Slice(offset, expected->length() - offset)));
-  }
-
- private:
-  MemoryPool* pool_;
-};
-
-template <typename T>
-class DecimalTest : public DecimalTestBase<T> {
- public:
-  std::vector<uint8_t> MakeData(const std::vector<T>& input,
-                                size_t byte_width) const override {
-    std::vector<uint8_t> result(input.size() * byte_width);
-    // TODO(phillipc): There's probably a better way to do this
-    constexpr static const size_t bytes_per_element = sizeof(T);
-    for (size_t i = 0, j = 0; i < input.size(); ++i, j += bytes_per_element) {
-      *reinterpret_cast<typename T::value_type*>(&result[j]) = input[i].value;
-    }
-    return result;
-  }
-};
-
-template <>
-class DecimalTest<Decimal128> : public DecimalTestBase<Decimal128> {
- public:
-  std::vector<uint8_t> MakeData(const std::vector<Decimal128>& input,
-                                size_t byte_width) const override {
-    std::vector<uint8_t> result;
-    result.reserve(input.size() * byte_width);
-    constexpr static const size_t bytes_per_element = 16;
-    for (size_t i = 0; i < input.size(); ++i) {
-      uint8_t stack_bytes[bytes_per_element] = {0};
-      uint8_t* bytes = stack_bytes;
-      ToBytes(input[i], &bytes);
-
-      for (size_t i = 0; i < bytes_per_element; ++i) {
-        result.push_back(bytes[i]);
-      }
-    }
-    return result;
-  }
-};
 
-class Decimal32BuilderTest : public ::testing::TestWithParam<int>,
-                             public DecimalTest<Decimal32> {};
+    std::vector<uint8_t> raw_bytes;
 
-class Decimal64BuilderTest : public ::testing::TestWithParam<int>,
-                             public DecimalTest<Decimal64> {};
+    raw_bytes.reserve(size * BYTE_WIDTH);
+    MakeData<BYTE_WIDTH>(draw, &raw_bytes);
 
-class Decimal128BuilderTest : public ::testing::TestWithParam<int>,
-                              public DecimalTest<Decimal128> {};
+    auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), BYTE_WIDTH);
+    std::shared_ptr<Buffer> expected_null_bitmap;
+    ASSERT_OK(BitUtil::BytesToBits(valid_bytes, &expected_null_bitmap));
 
-TEST_P(Decimal32BuilderTest, NoNulls) {
-  int precision = GetParam();
-  std::vector<Decimal32> draw = {Decimal32(1), Decimal32(2), Decimal32(2389),
-                                 Decimal32(4), Decimal32(-12348)};
-  std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
-  this->TestCreate(precision, draw, valid_bytes, 0);
-  this->TestCreate(precision, draw, valid_bytes, 2);
-}
+    int64_t expected_null_count = test::null_count(valid_bytes);
+    auto expected = std::make_shared<DecimalArray>(
+        type, size, expected_data, expected_null_bitmap, expected_null_count);
 
-TEST_P(Decimal64BuilderTest, NoNulls) {
-  int precision = GetParam();
-  std::vector<Decimal64> draw = {Decimal64(1), Decimal64(2), Decimal64(2389),
-                                 Decimal64(4), Decimal64(-12348)};
-  std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
-  this->TestCreate(precision, draw, valid_bytes, 0);
-  this->TestCreate(precision, draw, valid_bytes, 2);
-}
+    std::shared_ptr<Array> lhs = out->Slice(offset);
+    std::shared_ptr<Array> rhs = expected->Slice(offset);
+    bool result = lhs->Equals(rhs);
+    ASSERT_TRUE(result);
+  }
+};
 
-TEST_P(Decimal128BuilderTest, NoNulls) {
+TEST_P(DecimalTest, NoNulls) {
   int precision = GetParam();
-  std::vector<Decimal128> draw = {Decimal128(1), Decimal128(-2), Decimal128(2389),
-                                  Decimal128(4), Decimal128(-12348)};
+  std::vector<Int128> draw = {Int128(1), Int128(-2), Int128(2389), Int128(4),
+                              Int128(-12348)};
   std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
   this->TestCreate(precision, draw, valid_bytes, 0);
   this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
-TEST_P(Decimal32BuilderTest, WithNulls) {
+TEST_P(DecimalTest, WithNulls) {
   int precision = GetParam();
-  std::vector<Decimal32> draw = {Decimal32(1), Decimal32(2), Decimal32(-1), Decimal32(4),
-                                 Decimal32(-1)};
-  std::vector<uint8_t> valid_bytes = {true, true, false, true, false};
-  this->TestCreate(precision, draw, valid_bytes, 0);
-  this->TestCreate(precision, draw, valid_bytes, 2);
-}
+  std::vector<Int128> draw = {Int128(1),  Int128(2), Int128(-1), Int128(4),
+                              Int128(-1), Int128(1), Int128(2)};
+  Int128 big;
+  ASSERT_OK(FromString("230342903942.234234", &big));
+  draw.push_back(big);
 
-TEST_P(Decimal64BuilderTest, WithNulls) {
-  int precision = GetParam();
-  std::vector<Decimal64> draw = {Decimal64(-1), Decimal64(2), Decimal64(-1), Decimal64(4),
-                                 Decimal64(-1)};
-  std::vector<uint8_t> valid_bytes = {true, true, false, true, false};
-  this->TestCreate(precision, draw, valid_bytes, 0);
-  this->TestCreate(precision, draw, valid_bytes, 2);
-}
+  Int128 big_negative;
+  ASSERT_OK(FromString("-23049302932.235234", &big_negative));
+  draw.push_back(big_negative);
 
-TEST_P(Decimal128BuilderTest, WithNulls) {
-  int precision = GetParam();
-  std::vector<Decimal128> draw = {Decimal128(1),
-                                  Decimal128(2),
-                                  Decimal128(-1),
-                                  Decimal128(4),
-                                  Decimal128(-1),
-                                  Decimal128(1),
-                                  Decimal128(2),
-                                  Decimal128("230342903942.234234"),
-                                  Decimal128("-23049302932.235234")};
   std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
                                       true, true, true,  true};
   this->TestCreate(precision, draw, valid_bytes, 0);
   this->TestCreate(precision, draw, valid_bytes, 2);
 }
 
-INSTANTIATE_TEST_CASE_P(Decimal32BuilderTest, Decimal32BuilderTest,
-                        ::testing::Range(DecimalPrecision<int32_t>::minimum,
-                                         DecimalPrecision<int32_t>::maximum));
-INSTANTIATE_TEST_CASE_P(Decimal64BuilderTest, Decimal64BuilderTest,
-                        ::testing::Range(DecimalPrecision<int64_t>::minimum,
-                                         DecimalPrecision<int64_t>::maximum));
-INSTANTIATE_TEST_CASE_P(Decimal128BuilderTest, Decimal128BuilderTest,
-                        ::testing::Range(DecimalPrecision<Int128>::minimum,
-                                         DecimalPrecision<Int128>::maximum));
+INSTANTIATE_TEST_CASE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38));
 
 }  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 720d5ece421e4..babf326057e63 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -29,6 +29,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/visitor.h"
@@ -316,28 +317,10 @@ DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data)
   DCHECK_EQ(data->type->id(), Type::DECIMAL);
 }
 
-#define DECIMAL_TO_STRING_CASE(bits, bytes, precision, scale) \
-  case bits: {                                                \
-    decimal::Decimal##bits value;                             \
-    decimal::FromBytes((bytes), &value);                      \
-    return decimal::ToString(value, (precision), (scale));    \
-  }
-
 std::string DecimalArray::FormatValue(int64_t i) const {
   const auto& type_ = static_cast<const DecimalType&>(*type());
-  const int precision = type_.precision();
-  const int scale = type_.scale();
-  const int bit_width = type_.bit_width();
-  const uint8_t* bytes = GetValue(i);
-  switch (bit_width) {
-    DECIMAL_TO_STRING_CASE(32, bytes, precision, scale)
-    DECIMAL_TO_STRING_CASE(64, bytes, precision, scale)
-    DECIMAL_TO_STRING_CASE(128, bytes, precision, scale)
-    default: {
-      DCHECK(false) << "Invalid bit width: " << bit_width;
-      return "";
-    }
-  }
+  decimal::Int128 value(GetValue(i));
+  return decimal::ToString(value, type_.precision(), type_.scale());
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index d5b2a26f6bc30..4c3bd79599241 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_BUFFER_H
 #define ARROW_BUFFER_H
 
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <memory>
@@ -215,6 +216,19 @@ class ARROW_EXPORT BufferBuilder {
     return Status::OK();
   }
 
+  template <size_t NBYTES>
+  Status Append(const std::array<uint8_t, NBYTES>& data) {
+    constexpr auto nbytes = static_cast<int64_t>(NBYTES);
+    if (capacity_ < static_cast<int64_t>(nbytes) + size_) {
+      int64_t new_capacity = BitUtil::NextPower2(nbytes + size_);
+      RETURN_NOT_OK(Resize(new_capacity));
+    }
+
+    std::copy(data.cbegin(), data.cend(), data_ + size_);
+    size_ += nbytes;
+    return Status::OK();
+  }
+
   // Advance pointer and zero out memory
   Status Advance(const int64_t length) {
     if (capacity_ < length + size_) {
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 71f0bab96f10e..a1d2366f69176 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -34,6 +34,7 @@
 #include "arrow/util/cpu-info.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/hash-util.h"
+#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -965,31 +966,17 @@ DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>
     : DecimalBuilder(type, pool) {}
 #endif
 
-template <typename T>
-ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal<T>& val) {
-  RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
-  return FixedSizeBinaryBuilder::Append(reinterpret_cast<const uint8_t*>(&val.value));
-}
-
-template ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal32& val);
-template ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal64& val);
-
-template <>
-ARROW_EXPORT Status DecimalBuilder::Append(const decimal::Decimal128& value) {
+Status DecimalBuilder::Append(const decimal::Int128& value) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
-  uint8_t stack_bytes[16] = {0};
-  uint8_t* bytes = stack_bytes;
-  decimal::ToBytes(value, &bytes);
+  std::array<uint8_t, 16> bytes;
+  RETURN_NOT_OK(value.ToBytes(&bytes));
   return FixedSizeBinaryBuilder::Append(bytes);
 }
 
 Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
-
-  /// TODO(phillipc): not sure where to get the offset argument here
-  *out =
-      std::make_shared<DecimalArray>(type_, length_, data, null_bitmap_, null_count_, 0);
+  *out = std::make_shared<DecimalArray>(type_, length_, data, null_bitmap_, null_count_);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index cb1e4ba69fe2a..d1b51d6431da1 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -47,8 +47,7 @@ struct ArrayData;
 
 namespace decimal {
 
-template <typename T>
-struct Decimal;
+class Int128;
 
 }  // namespace decimal
 
@@ -674,6 +673,14 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
                          MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
   Status Append(const uint8_t* value);
+
+  template <size_t NBYTES>
+  Status Append(const std::array<uint8_t, NBYTES>& value) {
+    RETURN_NOT_OK(Reserve(1));
+    UnsafeAppendToBitmap(true);
+    return byte_builder_.Append(value);
+  }
+
   Status Append(const uint8_t* data, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
   Status Append(const std::string& value);
@@ -701,8 +708,7 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
   explicit DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 #endif
 
-  template <typename T>
-  ARROW_EXPORT Status Append(const decimal::Decimal<T>& val);
+  Status Append(const decimal::Int128& val);
 
   Status Finish(std::shared_ptr<Array>* out) override;
 };
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 1b84603ab90e5..b78d5d5fb4d2e 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -313,20 +313,14 @@ static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& r
   const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
   const int byte_width = size_meta.bit_width() / CHAR_BIT;
 
-  const uint8_t* left_data = nullptr;
-  const uint8_t* right_data = nullptr;
-
-  if (left.values()) {
-    left_data = left.raw_values();
-  }
-  if (right.values()) {
-    right_data = right.raw_values();
-  }
+  const uint8_t* left_data = left.values() ? left.raw_values() : nullptr;
+  const uint8_t* right_data = right.values() ? right.raw_values() : nullptr;
 
   if (left.null_count() > 0) {
     for (int64_t i = 0; i < left.length(); ++i) {
-      bool left_null = left.IsNull(i);
-      if (!left_null && (memcmp(left_data, right_data, byte_width) || right.IsNull(i))) {
+      const bool left_null = left.IsNull(i);
+      const bool right_null = right.IsNull(i);
+      if (!left_null && (memcmp(left_data, right_data, byte_width) != 0 || right_null)) {
         return false;
       }
       left_data += byte_width;
@@ -334,42 +328,18 @@ static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& r
     }
     return true;
   } else {
-    return memcmp(left_data, right_data,
-                  static_cast<size_t>(byte_width * left.length())) == 0;
+    auto number_of_bytes_to_compare = static_cast<size_t>(byte_width * left.length());
+    return memcmp(left_data, right_data, number_of_bytes_to_compare) == 0;
   }
 }
 
-template <typename T>
-static inline bool CompareBuiltIn(const Array& left, const Array& right, const T* ldata,
-                                  const T* rdata) {
-  if (ldata == nullptr && rdata == nullptr) {
-    return true;
-  }
-
-  if (ldata == nullptr || rdata == nullptr) {
-    return false;
-  }
-
-  if (left.null_count() > 0) {
-    for (int64_t i = 0; i < left.length(); ++i) {
-      if (left.IsNull(i) != right.IsNull(i)) {
-        return false;
-      } else if (!left.IsNull(i) && (ldata[i] != rdata[i])) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  return memcmp(ldata, rdata, sizeof(T) * left.length()) == 0;
-}
-
 class ArrayEqualsVisitor : public RangeEqualsVisitor {
  public:
   explicit ArrayEqualsVisitor(const Array& right)
       : RangeEqualsVisitor(right, 0, right.length(), 0) {}
 
   Status Visit(const NullArray& left) {
+    ARROW_UNUSED(left);
     result_ = true;
     return Status::OK();
   }
@@ -507,27 +477,6 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DecimalArray& left) {
-    const int byte_width = left.byte_width();
-    if (byte_width == 4) {
-      result_ = CompareBuiltIn<int32_t>(
-          left, right_, reinterpret_cast<const int32_t*>(left.raw_values()),
-          reinterpret_cast<const int32_t*>(
-              static_cast<const DecimalArray&>(right_).raw_values()));
-      return Status::OK();
-    }
-
-    if (byte_width == 8) {
-      result_ = CompareBuiltIn<int64_t>(
-          left, right_, reinterpret_cast<const int64_t*>(left.raw_values()),
-          reinterpret_cast<const int64_t*>(
-              static_cast<const DecimalArray&>(right_).raw_values()));
-      return Status::OK();
-    }
-
-    return RangeEqualsVisitor::Visit(left);
-  }
-
   template <typename T>
   typename std::enable_if<std::is_base_of<NestedType, typename T::TypeClass>::value,
                           Status>::type
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 8ba3f4645fda8..d2d6fb4694de5 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -37,6 +37,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/parallel.h"
@@ -602,44 +603,14 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
   return Status::OK();
 }
 
-template <typename T>
-Status ValidateDecimalPrecision(int precision) {
-  constexpr static const int kMaximumPrecision =
-      decimal::DecimalPrecision<typename T::value_type>::maximum;
-  if (!(precision > 0 && precision <= kMaximumPrecision)) {
-    std::stringstream ss;
-    ss << "Invalid precision: " << precision << ". Minimum is 1, maximum is "
-       << kMaximumPrecision;
-    return Status::Invalid(ss.str());
-  }
-  return Status::OK();
-}
-
-template <typename T>
-Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
-                          std::string* result) {
-  DCHECK_NE(bytes, nullptr);
+static Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
+                                 std::string* result) {
   DCHECK_NE(result, nullptr);
-  RETURN_NOT_OK(ValidateDecimalPrecision<T>(precision));
-  T decimal;
-  decimal::FromBytes(bytes, &decimal);
+  decimal::Int128 decimal(bytes);
   *result = decimal::ToString(decimal, precision, scale);
   return Status::OK();
 }
 
-template Status RawDecimalToString<decimal::Decimal32>(const uint8_t*, int, int,
-                                                       std::string*);
-template Status RawDecimalToString<decimal::Decimal64>(const uint8_t*, int, int,
-                                                       std::string*);
-template Status RawDecimalToString<decimal::Decimal128>(const uint8_t*, int, int,
-                                                        std::string*);
-
-#define RAW_DECIMAL_TO_STRING_CASE(bits, value, precision, scale, output)          \
-  case bits:                                                                       \
-    RETURN_NOT_OK(RawDecimalToString<decimal::Decimal##bits>((value), (precision), \
-                                                             (scale), (output)));  \
-    break;
-
 static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
                               PyObject** out_values) {
   PyAcquireGIL lock;
@@ -654,7 +625,6 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
     auto type(std::dynamic_pointer_cast<arrow::DecimalType>(arr->type()));
     const int precision = type->precision();
     const int scale = type->scale();
-    const int bit_width = type->bit_width();
 
     for (int64_t i = 0; i < arr->length(); ++i) {
       if (arr->IsNull(i)) {
@@ -663,16 +633,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
       } else {
         const uint8_t* raw_value = arr->GetValue(i);
         std::string decimal_string;
-        switch (bit_width) {
-          RAW_DECIMAL_TO_STRING_CASE(32, raw_value, precision, scale, &decimal_string)
-          RAW_DECIMAL_TO_STRING_CASE(64, raw_value, precision, scale, &decimal_string)
-          RAW_DECIMAL_TO_STRING_CASE(128, raw_value, precision, scale, &decimal_string)
-          default: {
-            std::stringstream buf;
-            buf << "Invalid bit_width " << bit_width << " for decimal value";
-            return Status::Invalid(buf.str());
-          }
-        }
+        RETURN_NOT_OK(RawDecimalToString(raw_value, precision, scale, &decimal_string));
         RETURN_NOT_OK(DecimalFromString(Decimal, decimal_string, out_values++));
       }
     }
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index ccaf280b0a383..dca3f457c0f8a 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -28,6 +28,7 @@
 #include "arrow/api.h"
 #include "arrow/status.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 
 #include "arrow/python/helpers.h"
@@ -579,44 +580,24 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
   std::shared_ptr<SeqConverter> value_converter_;
 };
 
-#define DECIMAL_CONVERT_CASE(bit_width, item, builder)         \
-  case bit_width: {                                            \
-    arrow::decimal::Decimal##bit_width out;                    \
-    std::string string_out;                                    \
-    RETURN_NOT_OK(PythonDecimalToString((item), &string_out)); \
-    RETURN_NOT_OK(FromString(string_out, &out));               \
-    return ((builder)->Append(out));                           \
-    break;                                                     \
-  }
-
 class DecimalConverter
     : public TypedConverterVisitor<arrow::DecimalBuilder, DecimalConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
-    /// Can the compiler figure out that the case statement below isn't necessary
-    /// once we're running?
-    const int bit_width =
-        std::dynamic_pointer_cast<arrow::DecimalType>(typed_builder_->type())
-            ->bit_width();
-
     /// TODO(phillipc): Check for nan?
     if (item.obj() != Py_None) {
-      switch (bit_width) {
-        DECIMAL_CONVERT_CASE(32, item.obj(), typed_builder_)
-        DECIMAL_CONVERT_CASE(64, item.obj(), typed_builder_)
-        DECIMAL_CONVERT_CASE(128, item.obj(), typed_builder_)
-        default:
-          return Status::OK();
-      }
-      RETURN_IF_PYERROR();
-    } else {
-      return typed_builder_->AppendNull();
+      std::string string;
+      RETURN_NOT_OK(PythonDecimalToString(item.obj(), &string));
+
+      decimal::Int128 value;
+      RETURN_NOT_OK(decimal::FromString(string, &value));
+      return typed_builder_->Append(value);
     }
+
+    return typed_builder_->AppendNull();
   }
 };
 
-#undef DECIMAL_CONVERT_CASE
-
 // Dynamic constructor for sequence converters
 std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type) {
   switch (type->id()) {
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 164e42e52e48e..67f0655b41fc6 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -18,6 +18,7 @@
 #include "arrow/python/helpers.h"
 #include "arrow/python/common.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/logging.h"
 
 #include <arrow/api.h>
 
@@ -102,8 +103,7 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision,
   auto size = str.size;
 
   std::string c_string(bytes, size);
-  return FromString(c_string, static_cast<decimal::Decimal32*>(nullptr), precision,
-                    scale);
+  return decimal::FromString(c_string, nullptr, precision, scale);
 }
 
 Status DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string,
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 435682cf7a705..fe8ae80d0215b 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -39,6 +39,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/visitor_inline.h"
@@ -605,16 +606,6 @@ Status PandasConverter::ConvertDates() {
   return PushBuilderResult(&builder);
 }
 
-#define CONVERT_DECIMAL_CASE(bit_width, builder, object)         \
-  case bit_width: {                                              \
-    decimal::Decimal##bit_width d;                               \
-    std::string string_out;                                      \
-    RETURN_NOT_OK(PythonDecimalToString((object), &string_out)); \
-    RETURN_NOT_OK(FromString(string_out, &d));                   \
-    RETURN_NOT_OK((builder).Append(d));                          \
-    break;                                                       \
-  }
-
 Status PandasConverter::ConvertDecimals() {
   PyAcquireGIL lock;
 
@@ -634,20 +625,18 @@ Status PandasConverter::ConvertDecimals() {
 
   type_ = std::make_shared<DecimalType>(precision, scale);
 
-  const int bit_width = std::dynamic_pointer_cast<DecimalType>(type_)->bit_width();
   DecimalBuilder builder(type_, pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
   for (int64_t i = 0; i < length_; ++i) {
     object = objects[i];
     if (PyObject_IsInstance(object, Decimal.obj())) {
-      switch (bit_width) {
-        CONVERT_DECIMAL_CASE(32, builder, object)
-        CONVERT_DECIMAL_CASE(64, builder, object)
-        CONVERT_DECIMAL_CASE(128, builder, object)
-        default:
-          break;
-      }
+      std::string string;
+      RETURN_NOT_OK(PythonDecimalToString(object, &string));
+
+      decimal::Int128 value;
+      RETURN_NOT_OK(decimal::FromString(string, &value));
+      RETURN_NOT_OK(builder.Append(value));
     } else if (PandasObjectIsNull(object)) {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
@@ -688,8 +677,6 @@ Status PandasConverter::ConvertTimes() {
   return PushBuilderResult(&builder);
 }
 
-#undef CONVERT_DECIMAL_CASE
-
 Status PandasConverter::ConvertObjectStrings() {
   PyAcquireGIL lock;
 
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 138f8dd040fc7..710344ab0d21e 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -401,7 +401,7 @@ TEST(TestStructType, Basics) {
   // TODO(wesm): out of bounds for field(...)
 }
 
-TEST(TypesTest, TestDecimal32Type) {
+TEST(TypesTest, TestDecimal128Small) {
   DecimalType t1(8, 4);
 
   ASSERT_EQ(t1.id(), Type::DECIMAL);
@@ -411,11 +411,11 @@ TEST(TypesTest, TestDecimal32Type) {
   ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
 
   // Test properties
-  ASSERT_EQ(t1.byte_width(), 4);
-  ASSERT_EQ(t1.bit_width(), 32);
+  ASSERT_EQ(t1.byte_width(), 16);
+  ASSERT_EQ(t1.bit_width(), 128);
 }
 
-TEST(TypesTest, TestDecimal64Type) {
+TEST(TypesTest, TestDecimal128Medium) {
   DecimalType t1(12, 5);
 
   ASSERT_EQ(t1.id(), Type::DECIMAL);
@@ -425,11 +425,11 @@ TEST(TypesTest, TestDecimal64Type) {
   ASSERT_EQ(t1.ToString(), std::string("decimal(12, 5)"));
 
   // Test properties
-  ASSERT_EQ(t1.byte_width(), 8);
-  ASSERT_EQ(t1.bit_width(), 64);
+  ASSERT_EQ(t1.byte_width(), 16);
+  ASSERT_EQ(t1.bit_width(), 128);
 }
 
-TEST(TypesTest, TestDecimal128Type) {
+TEST(TypesTest, TestDecimal128Large) {
   DecimalType t1(27, 7);
 
   ASSERT_EQ(t1.id(), Type::DECIMAL);
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index aeb613084545b..3b7ef19b87547 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -454,25 +454,12 @@ class ARROW_EXPORT StructType : public NestedType {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-static inline int decimal_byte_width(int precision) {
-  if (precision >= 0 && precision < 10) {
-    return 4;
-  } else if (precision >= 10 && precision < 19) {
-    return 8;
-  } else {
-    // TODO(phillipc): validate that we can't construct > 128 bit types
-    return 16;
-  }
-}
-
 class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
  public:
   static constexpr Type::type type_id = Type::DECIMAL;
 
   explicit DecimalType(int precision, int scale)
-      : FixedSizeBinaryType(decimal_byte_width(precision), Type::DECIMAL),
-        precision_(precision),
-        scale_(scale) {}
+      : FixedSizeBinaryType(16, Type::DECIMAL), precision_(precision), scale_(scale) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index ea8962ba64e8b..516297147f18a 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -24,76 +24,73 @@
 #include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 
 namespace arrow {
 namespace decimal {
 
-template <typename T>
-class DecimalTest : public ::testing::Test {
+class DecimalTestFixture : public ::testing::Test {
  public:
-  DecimalTest() : decimal_value_(23423445), string_value_("234.23445") {}
-  Decimal<T> decimal_value_;
+  DecimalTestFixture() : integer_value_(23423445), string_value_("234.23445") {}
+  Int128 integer_value_;
   std::string string_value_;
 };
 
-typedef ::testing::Types<int32_t, int64_t, Int128> DecimalTypes;
-TYPED_TEST_CASE(DecimalTest, DecimalTypes);
-
-TYPED_TEST(DecimalTest, TestToString) {
-  Decimal<TypeParam> decimal(this->decimal_value_);
+TEST_F(DecimalTestFixture, TestToString) {
+  Int128 decimal(this->integer_value_);
   int precision = 8;
   int scale = 5;
   std::string result = ToString(decimal, precision, scale);
   ASSERT_EQ(result, this->string_value_);
 }
 
-TYPED_TEST(DecimalTest, TestFromString) {
-  Decimal<TypeParam> expected(this->decimal_value_);
-  Decimal<TypeParam> result;
+TEST_F(DecimalTestFixture, TestFromString) {
+  Int128 expected(this->integer_value_);
+  Int128 result;
   int precision, scale;
   ASSERT_OK(FromString(this->string_value_, &result, &precision, &scale));
-  ASSERT_EQ(result.value, expected.value);
+  ASSERT_EQ(result, expected);
   ASSERT_EQ(precision, 8);
   ASSERT_EQ(scale, 5);
 }
 
-TEST(DecimalTest, TestStringStartingWithPlus) {
+TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
   std::string plus_value("+234.234");
-  Decimal32 out;
+  Int128 out;
   int scale;
   int precision;
   ASSERT_OK(FromString(plus_value, &out, &precision, &scale));
-  ASSERT_EQ(234234, out.value);
+  ASSERT_EQ(234234, out);
   ASSERT_EQ(6, precision);
   ASSERT_EQ(3, scale);
 }
 
-TEST(DecimalTest, TestStringStartingWithPlus128) {
+TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
   std::string plus_value("+2342394230592.232349023094");
   Int128 expected_value("2342394230592232349023094");
-  Decimal128 out;
+  Int128 out;
   int scale;
   int precision;
   ASSERT_OK(FromString(plus_value, &out, &precision, &scale));
-  ASSERT_EQ(expected_value, out.value);
+  ASSERT_EQ(expected_value, out);
   ASSERT_EQ(25, precision);
   ASSERT_EQ(12, scale);
 }
 
 TEST(DecimalTest, TestStringToInt32) {
-  int32_t value = 0;
+  Int128 value;
   StringToInteger("123", "456", 1, &value);
   ASSERT_EQ(value, 123456);
 }
 
 TEST(DecimalTest, TestStringToInt64) {
-  int64_t value = 0;
+  Int128 value;
   StringToInteger("123456789", "456", -1, &value);
   ASSERT_EQ(value, -123456789456);
 }
 
 TEST(DecimalTest, TestStringToInt128) {
-  Int128 value = 0;
+  Int128 value;
   StringToInteger("123456789", "456789123", 1, &value);
   ASSERT_EQ(value.high_bits(), 0);
   ASSERT_EQ(value.low_bits(), 123456789456789123);
@@ -101,120 +98,99 @@ TEST(DecimalTest, TestStringToInt128) {
 
 TEST(DecimalTest, TestFromString128) {
   static const std::string string_value("-23049223942343532412");
-  Decimal128 result(string_value);
+  Int128 result(string_value);
   Int128 expected(static_cast<int64_t>(-230492239423435324));
-  ASSERT_EQ(result.value, expected * 100 - 12);
+  ASSERT_EQ(result, expected * 100 - 12);
 
   // Sanity check that our number is actually using more than 64 bits
-  ASSERT_NE(result.value.high_bits(), 0);
+  ASSERT_NE(result.high_bits(), 0);
 }
 
 TEST(DecimalTest, TestFromDecimalString128) {
-  static const std::string string_value("-23049223942343.532412");
-  Decimal128 result(string_value);
+  std::string string_value("-23049223942343.532412");
+  Int128 result;
+  ASSERT_OK(FromString(string_value, &result));
   Int128 expected(static_cast<int64_t>(-230492239423435324));
-  ASSERT_EQ(result.value, expected * 100 - 12);
+  expected *= 100;
+  expected -= 12;
+  ASSERT_EQ(result, expected);
 
   // Sanity check that our number is actually using more than 64 bits
-  ASSERT_NE(result.value.high_bits(), 0);
-}
-
-TEST(DecimalTest, TestDecimal32Precision) {
-  auto min_precision = DecimalPrecision<int32_t>::minimum;
-  auto max_precision = DecimalPrecision<int32_t>::maximum;
-  ASSERT_EQ(min_precision, 1);
-  ASSERT_EQ(max_precision, 9);
-}
-
-TEST(DecimalTest, TestDecimal64Precision) {
-  auto min_precision = DecimalPrecision<int64_t>::minimum;
-  auto max_precision = DecimalPrecision<int64_t>::maximum;
-  ASSERT_EQ(min_precision, 10);
-  ASSERT_EQ(max_precision, 18);
-}
-
-TEST(DecimalTest, TestDecimal128Precision) {
-  auto min_precision = DecimalPrecision<Int128>::minimum;
-  auto max_precision = DecimalPrecision<Int128>::maximum;
-  ASSERT_EQ(min_precision, 19);
-  ASSERT_EQ(max_precision, 38);
+  ASSERT_NE(result.high_bits(), 0);
 }
 
 TEST(DecimalTest, TestDecimal32SignedRoundTrip) {
-  Decimal32 expected(std::string("-3402692"));
+  Int128 expected("-3402692");
 
-  uint8_t stack_bytes[4] = {0};
-  uint8_t* bytes = stack_bytes;
-  ToBytes(expected, &bytes);
+  std::array<uint8_t, 16> bytes;
+  ASSERT_OK(expected.ToBytes(&bytes));
 
-  Decimal32 result;
-  FromBytes(bytes, &result);
-  ASSERT_EQ(expected.value, result.value);
+  Int128 result(bytes.data());
+  ASSERT_EQ(expected, result);
 }
 
 TEST(DecimalTest, TestDecimal64SignedRoundTrip) {
-  Decimal64 expected(std::string("-34034293045.921"));
+  Int128 expected;
+  std::string string_value("-34034293045.921");
+  ASSERT_OK(FromString(string_value, &expected));
 
-  uint8_t stack_bytes[8] = {0};
-  uint8_t* bytes = stack_bytes;
-  ToBytes(expected, &bytes);
+  std::array<uint8_t, 16> bytes;
+  ASSERT_OK(expected.ToBytes(&bytes));
 
-  Decimal64 result;
-  FromBytes(bytes, &result);
+  Int128 result(bytes.data());
 
-  ASSERT_EQ(expected.value, result.value);
+  ASSERT_EQ(expected, result);
 }
 
-TEST(DecimalTest, TestDecimal128StringAndBytesRoundTrip) {
+TEST(DecimalTest, TestDecimalStringAndBytesRoundTrip) {
+  Int128 expected;
   std::string string_value("-340282366920938463463374607431.711455");
-  Decimal128 expected(string_value);
+  ASSERT_OK(FromString(string_value, &expected));
 
   std::string expected_string_value("-340282366920938463463374607431711455");
   Int128 expected_underlying_value(expected_string_value);
 
-  ASSERT_EQ(expected.value, expected_underlying_value);
+  ASSERT_EQ(expected, expected_underlying_value);
 
-  uint8_t stack_bytes[16] = {0};
-  uint8_t* bytes = stack_bytes;
-  ToBytes(expected, &bytes);
+  std::array<uint8_t, 16> bytes;
+  ASSERT_OK(expected.ToBytes(&bytes));
 
-  Decimal128 result;
-  FromBytes(bytes, &result);
+  Int128 result(bytes.data());
 
-  ASSERT_EQ(expected.value, result.value);
+  ASSERT_EQ(expected, result);
 }
 
 TEST(DecimalTest, TestInvalidInputMinus) {
   std::string invalid_value("-");
-  Decimal32 out;
+  Int128 out;
   Status status = decimal::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputDot) {
   std::string invalid_value("0.0.0");
-  Decimal32 out;
+  Int128 out;
   Status status = decimal::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputEmbeddedMinus) {
   std::string invalid_value("0-13-32");
-  Decimal32 out;
+  Int128 out;
   Status status = decimal::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputSingleChar) {
   std::string invalid_value("a");
-  Decimal32 out;
+  Int128 out;
   Status status = decimal::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
   std::string invalid_value("-23092.235-");
-  Decimal32 out;
+  Int128 out;
   Status status = decimal::FromString(invalid_value, &out);
   auto msg = status.message();
   ASSERT_RAISES(Invalid, status);
@@ -222,43 +198,39 @@ TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
 
 TEST(DecimalTest, TestInvalidInputWithMinusPlus) {
   std::string invalid_value("-+23092.235");
-  Decimal32 out;
+  Int128 out;
   Status status = decimal::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithPlusMinus) {
   std::string invalid_value("+-23092.235");
-  Decimal32 out;
+  Int128 out;
   Status status = decimal::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
   std::string invalid_value("00a");
-  Decimal32 out;
+  Int128 out;
   Status status = decimal::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
-template <typename T>
-class DecimalZerosTest : public ::testing::Test {};
-TYPED_TEST_CASE(DecimalZerosTest, DecimalTypes);
-
-TYPED_TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
+TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
   std::string string_value("0000000");
-  Decimal<TypeParam> d;
+  Int128 d;
   int precision;
   int scale;
   ASSERT_OK(FromString(string_value, &d, &precision, &scale));
   ASSERT_EQ(precision, 7);
   ASSERT_EQ(scale, 0);
-  ASSERT_EQ(d.value, 0);
+  ASSERT_EQ(d, 0);
 }
 
-TYPED_TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
+TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
   std::string string_value("000.0000");
-  Decimal<TypeParam> d;
+  Int128 d;
   int precision;
   int scale;
   ASSERT_OK(FromString(string_value, &d, &precision, &scale));
@@ -266,18 +238,48 @@ TYPED_TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
   ASSERT_NE(precision, 7);
 
   ASSERT_EQ(scale, 4);
-  ASSERT_EQ(d.value, 0);
+  ASSERT_EQ(d, 0);
 }
 
-TYPED_TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
+TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
   std::string string_value(".00000");
-  Decimal<TypeParam> d;
+  Int128 d;
   int precision;
   int scale;
   ASSERT_OK(FromString(string_value, &d, &precision, &scale));
   ASSERT_EQ(precision, 5);
   ASSERT_EQ(scale, 5);
-  ASSERT_EQ(d.value, 0);
+  ASSERT_EQ(d, 0);
+}
+
+template <typename T>
+class Int128Test : public ::testing::Test {
+ public:
+  Int128Test() : value_(42) {}
+  const T value_;
+};
+
+using Int128Types =
+    ::testing::Types<char, unsigned char, short, unsigned short,  // NOLINT
+                     int, unsigned int, long, unsigned long,      // NOLINT
+                     long long, unsigned long long                // NOLINT
+                     >;
+
+TYPED_TEST_CASE(Int128Test, Int128Types);
+
+TYPED_TEST(Int128Test, ConstructibleFromAnyIntegerType) {
+  Int128 value(this->value_);
+  ASSERT_EQ(42, value.low_bits());
+}
+
+TEST(Int128TestTrue, ConstructibleFromBool) {
+  Int128 value(true);
+  ASSERT_EQ(1, value.low_bits());
+}
+
+TEST(Int128TestFalse, ConstructibleFromBool) {
+  Int128 value(false);
+  ASSERT_EQ(0, value.low_bits());
 }
 
 }  // namespace decimal
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 98768cfeec7c7..bfd87a54673c7 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -15,19 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
 #include <cctype>
-#include <cmath>
+#include <cstdlib>
 #include <sstream>
 
+#include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/int128.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace decimal {
 
-template <typename T>
-ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out, int* precision,
-                               int* scale) {
+void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
+                     Int128* out) {
+  DCHECK(sign == -1 || sign == 1);
+  DCHECK_NE(out, nullptr);
+  DCHECK(!whole.empty() || !fractional.empty());
+  *out = Int128(whole + fractional) * sign;
+}
+
+Status FromString(const std::string& s, Int128* out, int* precision, int* scale) {
   // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?";
   if (s.empty()) {
     return Status::Invalid("Empty string cannot be converted to decimal");
@@ -56,14 +65,12 @@ ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out, int* preci
   DCHECK_LT(charp, end);
 
   // skip leading zeros
-  while (charp != end && *charp == '0') {
-    ++charp;
-  }
+  charp = std::find_if_not(charp, end, [](char c) { return c == '0'; });
 
   // all zeros and no decimal point
   if (charp == end) {
     if (out != nullptr) {
-      out->value = static_cast<T>(0);
+      *out = Int128(0);
     }
 
     // Not sure what other libraries assign precision to for this case (this case of
@@ -81,9 +88,7 @@ ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out, int* preci
 
   std::string::const_iterator whole_part_start = charp;
 
-  while (charp != end && isdigit(*charp)) {
-    ++charp;
-  }
+  charp = std::find_if_not(charp, end, [](char c) { return std::isdigit(c) != 0; });
 
   std::string::const_iterator whole_part_end = charp;
   std::string whole_part(whole_part_start, whole_part_end);
@@ -97,7 +102,7 @@ ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out, int* preci
           "end of the string.");
     }
 
-    if (!isdigit(*charp)) {
+    if (std::isdigit(*charp) == 0) {
       std::stringstream ss;
       ss << "Decimal point must be followed by a base ten digit. Found '" << *charp
          << "'";
@@ -117,9 +122,7 @@ ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out, int* preci
   // The rest must be digits, because if we have a decimal point it must be followed by
   // digits
   if (charp != end) {
-    while (charp != end && isdigit(*charp)) {
-      ++charp;
-    }
+    charp = std::find_if_not(charp, end, [](char c) { return std::isdigit(c) != 0; });
 
     // The while loop has ended before the end of the string which means we've hit a
     // character that isn't a base ten digit
@@ -143,88 +146,64 @@ ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out, int* preci
   }
 
   if (out != nullptr) {
-    StringToInteger(whole_part, fractional_part, sign, &out->value);
+    StringToInteger(whole_part, fractional_part, sign, out);
   }
 
   return Status::OK();
 }
 
-template ARROW_EXPORT Status FromString(const std::string& s, Decimal32* out,
-                                        int* precision, int* scale);
-template ARROW_EXPORT Status FromString(const std::string& s, Decimal64* out,
-                                        int* precision, int* scale);
-template ARROW_EXPORT Status FromString(const std::string& s, Decimal128* out,
-                                        int* precision, int* scale);
-
-void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
-                     int32_t* out) {
-  DCHECK(sign == -1 || sign == 1);
-  DCHECK_NE(out, nullptr);
-  DCHECK(!whole.empty() || !fractional.empty());
-
-  if (!whole.empty()) {
-    *out = std::stoi(whole) *
-           static_cast<int32_t>(pow(10.0, static_cast<double>(fractional.size())));
-  }
-  if (!fractional.empty()) {
-    *out += std::stoi(fractional, nullptr, 10);
+std::string ToString(const Int128& decimal_value, int precision, int scale) {
+  Int128 value(decimal_value);
+
+  // Decimal values are sent to clients as strings so in the interest of
+  // speed the string will be created without the using stringstream with the
+  // whole/fractional_part().
+  size_t last_char_idx = precision + (scale > 0)  // Add a space for decimal place
+                         + (scale == precision)   // Add a space for leading 0
+                         + (value < 0);           // Add a space for negative sign
+  std::string str(last_char_idx, '0');
+
+  // Start filling in the values in reverse order by taking the last digit
+  // of the value. Use a positive value and worry about the sign later. At this
+  // point the last_char_idx points to the string terminator.
+  Int128 remaining_value(value);
+
+  size_t first_digit_idx = 0;
+  if (value < 0) {
+    remaining_value = -value;
+    first_digit_idx = 1;
   }
-  *out *= sign;
-}
 
-void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
-                     int64_t* out) {
-  DCHECK(sign == -1 || sign == 1);
-  DCHECK_NE(out, nullptr);
-  DCHECK(!whole.empty() || !fractional.empty());
-  if (!whole.empty()) {
-    *out = static_cast<int64_t>(std::stoll(whole)) *
-           static_cast<int64_t>(pow(10.0, static_cast<double>(fractional.size())));
+  if (scale > 0) {
+    int remaining_scale = scale;
+    do {
+      str[--last_char_idx] =
+          static_cast<char>(remaining_value % 10 + '0');  // Ascii offset
+      remaining_value /= 10;
+    } while (--remaining_scale > 0);
+    str[--last_char_idx] = '.';
+    DCHECK_GT(last_char_idx, first_digit_idx) << "Not enough space remaining";
   }
-  if (!fractional.empty()) {
-    *out += std::stoll(fractional, nullptr, 10);
-  }
-  *out *= sign;
-}
-
-void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
-                     Int128* out) {
-  DCHECK(sign == -1 || sign == 1);
-  DCHECK_NE(out, nullptr);
-  DCHECK(!whole.empty() || !fractional.empty());
-  *out = Int128(whole + fractional) * sign;
-}
-
-void FromBytes(const uint8_t* bytes, Decimal32* decimal) {
-  DCHECK_NE(bytes, nullptr);
-  DCHECK_NE(decimal, nullptr);
-  decimal->value = *reinterpret_cast<const int32_t*>(bytes);
-}
 
-void FromBytes(const uint8_t* bytes, Decimal64* decimal) {
-  DCHECK_NE(bytes, nullptr);
-  DCHECK_NE(decimal, nullptr);
-  decimal->value = *reinterpret_cast<const int64_t*>(bytes);
-}
+  do {
+    str[--last_char_idx] = static_cast<char>(remaining_value % 10 + '0');  // Ascii offset
+    remaining_value /= 10;
+    if (remaining_value == 0) {
+      // Trim any extra leading 0's.
+      if (last_char_idx > first_digit_idx) {
+        str.erase(0, last_char_idx - first_digit_idx);
+      }
 
-void FromBytes(const uint8_t* bytes, Decimal128* decimal) {
-  decimal->value = Int128(bytes);
-}
-
-void ToBytes(const Decimal32& value, uint8_t** bytes) {
-  DCHECK_NE(*bytes, nullptr);
-  *reinterpret_cast<int32_t*>(*bytes) = value.value;
-}
+      break;
+    }
+    // For safety, enforce string length independent of remaining_value.
+  } while (last_char_idx > first_digit_idx);
 
-void ToBytes(const Decimal64& value, uint8_t** bytes) {
-  DCHECK_NE(*bytes, nullptr);
-  *reinterpret_cast<int64_t*>(*bytes) = value.value;
-}
+  if (value < 0) {
+    str[0] = '-';
+  }
 
-void ToBytes(const Decimal128& decimal, uint8_t** bytes) {
-  DCHECK_NE(bytes, nullptr);
-  DCHECK_NE(*bytes, nullptr);
-  decimal.value.ToBytes(bytes);
+  return str;
 }
 
 }  // namespace decimal
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 26595ec330bdb..4b7cac894e985 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -18,126 +18,22 @@
 #ifndef ARROW_DECIMAL_H
 #define ARROW_DECIMAL_H
 
-#include <cstdint>
-#include <cstdlib>
 #include <string>
 
 #include "arrow/status.h"
-#include "arrow/util/int128.h"  // IWYU pragma: export
-#include "arrow/util/logging.h"
-#include "arrow/util/visibility.h"
 
 namespace arrow {
 namespace decimal {
 
-template <typename T>
-struct ARROW_EXPORT Decimal;
+class Int128;
 
-ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
-                                  int8_t sign, int32_t* out);
-ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
-                                  int8_t sign, int64_t* out);
 ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
                                   int8_t sign, Int128* out);
 
-template <typename T>
-ARROW_EXPORT Status FromString(const std::string& s, Decimal<T>* out,
+ARROW_EXPORT Status FromString(const std::string& string, Int128* out,
                                int* precision = nullptr, int* scale = nullptr);
 
-template <typename T>
-struct ARROW_EXPORT Decimal {
-  Decimal() : value() {}
-  explicit Decimal(const std::string& s) : value() { DCHECK(FromString(s, this).ok()); }
-  explicit Decimal(const char* s) : Decimal(std::string(s)) {}
-  explicit Decimal(const T& value) : value(value) {}
-
-  using value_type = T;
-  value_type value;
-};
-
-using Decimal32 = Decimal<int32_t>;
-using Decimal64 = Decimal<int64_t>;
-using Decimal128 = Decimal<Int128>;
-
-template <typename T>
-struct ARROW_EXPORT DecimalPrecision {};
-
-template <>
-struct ARROW_EXPORT DecimalPrecision<int32_t> {
-  constexpr static const int minimum = 1;
-  constexpr static const int maximum = 9;
-};
-
-template <>
-struct ARROW_EXPORT DecimalPrecision<int64_t> {
-  constexpr static const int minimum = 10;
-  constexpr static const int maximum = 18;
-};
-
-template <>
-struct ARROW_EXPORT DecimalPrecision<Int128> {
-  constexpr static const int minimum = 19;
-  constexpr static const int maximum = 38;
-};
-
-template <typename T>
-ARROW_EXPORT std::string ToString(const Decimal<T>& decimal_value, int precision,
-                                  int scale) {
-  T value = decimal_value.value;
-
-  // Decimal values are sent to clients as strings so in the interest of
-  // speed the string will be created without the using stringstream with the
-  // whole/fractional_part().
-  size_t last_char_idx = precision + (scale > 0)  // Add a space for decimal place
-                         + (scale == precision)   // Add a space for leading 0
-                         + (value < 0);           // Add a space for negative sign
-  std::string str = std::string(last_char_idx, '0');
-  // Start filling in the values in reverse order by taking the last digit
-  // of the value. Use a positive value and worry about the sign later. At this
-  // point the last_char_idx points to the string terminator.
-  T remaining_value = value;
-  size_t first_digit_idx = 0;
-  if (value < 0) {
-    remaining_value = -value;
-    first_digit_idx = 1;
-  }
-  if (scale > 0) {
-    int remaining_scale = scale;
-    do {
-      str[--last_char_idx] =
-          static_cast<char>(remaining_value % 10 + static_cast<T>('0'));  // Ascii offset
-      remaining_value /= 10;
-    } while (--remaining_scale > 0);
-    str[--last_char_idx] = '.';
-    DCHECK_GT(last_char_idx, first_digit_idx) << "Not enough space remaining";
-  }
-  do {
-    str[--last_char_idx] =
-        static_cast<char>(remaining_value % 10 + static_cast<T>('0'));  // Ascii offset
-    remaining_value /= 10;
-    if (remaining_value == 0) {
-      // Trim any extra leading 0's.
-      if (last_char_idx > first_digit_idx) {
-        str.erase(0, last_char_idx - first_digit_idx);
-      }
-
-      break;
-    }
-    // For safety, enforce string length independent of remaining_value.
-  } while (last_char_idx > first_digit_idx);
-  if (value < 0) str[0] = '-';
-  return str;
-}
-
-/// Conversion from raw bytes to a Decimal value
-ARROW_EXPORT void FromBytes(const uint8_t* bytes, Decimal32* value);
-ARROW_EXPORT void FromBytes(const uint8_t* bytes, Decimal64* value);
-ARROW_EXPORT void FromBytes(const uint8_t* bytes, Decimal128* decimal);
-
-/// Conversion from a Decimal value to raw bytes
-ARROW_EXPORT void ToBytes(const Decimal32& value, uint8_t** bytes);
-ARROW_EXPORT void ToBytes(const Decimal64& value, uint8_t** bytes);
-ARROW_EXPORT void ToBytes(const Decimal128& decimal, uint8_t** bytes);
+ARROW_EXPORT std::string ToString(const Int128& decimal_value, int precision, int scale);
 
 }  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/int128.cc b/cpp/src/arrow/util/int128.cc
index 7102097b08a34..552c6cce990ec 100644
--- a/cpp/src/arrow/util/int128.cc
+++ b/cpp/src/arrow/util/int128.cc
@@ -1,20 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 
 #include <algorithm>
 #include <cmath>
@@ -64,11 +63,15 @@ Int128::Int128(const uint8_t* bytes)
     : Int128(reinterpret_cast<const int64_t*>(bytes)[0],
              reinterpret_cast<const uint64_t*>(bytes)[1]) {}
 
-void Int128::ToBytes(uint8_t** out) const {
-  DCHECK_NE(out, nullptr) << "Cannot fill nullptr of bytes from Int128";
-  DCHECK_NE(*out, nullptr) << "Cannot fill nullptr of bytes from Int128";
+Status Int128::ToBytes(std::array<uint8_t, 16>* out) const {
+  if (out == nullptr) {
+    return Status::Invalid("Cannot fill nullptr of bytes from Int128");
+  }
+
   const uint64_t raw[] = {static_cast<uint64_t>(high_bits_), low_bits_};
-  std::memcpy(*out, raw, 16);
+  const auto* raw_data = reinterpret_cast<const uint8_t*>(raw);
+  std::copy(raw_data, raw_data + out->size(), out->begin());
+  return Status::OK();
 }
 
 Int128& Int128::Negate() {
diff --git a/cpp/src/arrow/util/int128.h b/cpp/src/arrow/util/int128.h
index 19888ae157a2d..f6fafd7a3eae3 100644
--- a/cpp/src/arrow/util/int128.h
+++ b/cpp/src/arrow/util/int128.h
@@ -1,26 +1,27 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 
 #ifndef ARROW_INT128_H
 #define ARROW_INT128_H
 
+#include <array>
 #include <cstdint>
 #include <string>
+#include <type_traits>
 
 #include "arrow/status.h"
 #include "arrow/util/visibility.h"
@@ -37,18 +38,18 @@ namespace decimal {
 /// Adapted from the Apache ORC C++ implementation
 class ARROW_EXPORT Int128 {
  public:
-  constexpr Int128() : Int128(0, 0) {}
-
-  /// \brief Convert a signed 64 bit value into an Int128.
-  constexpr Int128(int64_t value)
-      : Int128(value >= 0 ? 0 : -1, static_cast<uint64_t>(value)) {}
-
-  /// \brief Convert a signed 32 bit value into an Int128.
-  constexpr Int128(int32_t value) : Int128(static_cast<int64_t>(value)) {}
-
   /// \brief Create an Int128 from the two's complement representation.
   constexpr Int128(int64_t high, uint64_t low) : high_bits_(high), low_bits_(low) {}
 
+  /// \brief Empty constructor creates an Int128 with a value of 0.
+  constexpr Int128() : Int128(0, 0) {}
+
+  /// \brief Convert any integer value into an Int128.
+  template <typename T,
+            typename = typename std::enable_if<std::is_integral<T>::value, T>::type>
+  constexpr Int128(T value)
+      : Int128(static_cast<int64_t>(value) >= 0 ? 0 : -1, static_cast<uint64_t>(value)) {}
+
   /// \brief Parse the number from a base 10 string representation.
   explicit Int128(const std::string& value);
 
@@ -103,7 +104,7 @@ class ARROW_EXPORT Int128 {
   uint64_t low_bits() const { return low_bits_; }
 
   /// \brief Put the raw bytes of the value into a pointer to uint8_t.
-  void ToBytes(uint8_t** out) const;
+  Status ToBytes(std::array<uint8_t, 16>* out) const;
 
  private:
   int64_t high_bits_;
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index 93521498903bd..ec8d3bd1fc4a1 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -64,7 +64,7 @@
 #if !defined(MANUALLY_ALIGNED_STRUCT)
 #if defined(_MSC_VER)
 #define MANUALLY_ALIGNED_STRUCT(alignment) \
-  __pragma(pack(1));                             \
+  __pragma(pack(1));                       \
   struct __declspec(align(alignment))
 #define STRUCT_END(name, size) \
   __pragma(pack());            \
@@ -77,6 +77,6 @@
 #else
 #error Unknown compiler, please define structure alignment macros
 #endif
-#endif // !defined(MANUALLY_ALIGNED_STRUCT)
+#endif  // !defined(MANUALLY_ALIGNED_STRUCT)
 
 #endif  // ARROW_UTIL_MACROS_H

From 9968d95dff5cd20091db8adb9b7a7707793b9d95 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 3 Sep 2017 22:37:24 -0400
Subject: [PATCH 0991/1644] ARROW-1400: [Python] Adding
 parquet.write_to_dataset() method for writing partitioned .parquet files

Add write_to_dataset in pyarrow.parquet that writes tables to parquet given partitioning columns.

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Safyre <think2wice.saf@gmail.com>

Closes #991 from saffrydaffry/1400-hotfix and squashes the following commits:

af911766 [Wes McKinney] Test parquet.write_to_dataset with HDFS also, small tweaks to implementation
725394bc [Safyre] Consolidate commits from ARROW-1400 so far
---
 python/pyarrow/hdfs.py               |  4 ++
 python/pyarrow/parquet.py            | 78 ++++++++++++++++++++++++++++
 python/pyarrow/tests/test_hdfs.py    | 14 +++++
 python/pyarrow/tests/test_parquet.py | 73 ++++++++++++++++++++++++++
 4 files changed, 169 insertions(+)

diff --git a/python/pyarrow/hdfs.py b/python/pyarrow/hdfs.py
index 855cc1e76bd92..1591de5cb3d28 100644
--- a/python/pyarrow/hdfs.py
+++ b/python/pyarrow/hdfs.py
@@ -52,6 +52,10 @@ def mkdir(self, path, create_parents=True):
     def rename(self, path, new_path):
         return super(HadoopFileSystem, self).rename(path, new_path)
 
+    @implements(FileSystem.exists)
+    def exists(self, path):
+        return super(HadoopFileSystem, self).exists(path)
+
     def ls(self, path, detail=False):
         """
         Retrieve directory contents and metadata, if requested.
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index ac488aedb6336..4bc56ebc37404 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -808,6 +808,84 @@ def write_table(table, where, row_group_size=None, version='1.0',
         writer.close()
 
 
+def write_to_dataset(table, root_path, partition_cols=None,
+                     filesystem=None, preserve_index=True, **kwargs):
+    """
+    Wrapper around parquet.write_table for writing a Table to
+    Parquet format by partitions.
+    For each combination of partition columns and values,
+    a subdirectories are created in the following
+    manner:
+
+    root_dir/
+      group1=value1
+        group2=value1
+          <uuid>.parquet
+        group2=value2
+          <uuid>.parquet
+      group1=valueN
+        group2=value1
+          <uuid>.parquet
+        group2=valueN
+          <uuid>.parquet
+
+    Parameters
+    ----------
+    table : pyarrow.Table
+    root_path : string,
+        The root directory of the dataset
+    filesystem : FileSystem, default None
+        If nothing passed, paths assumed to be found in the local on-disk
+        filesystem
+    partition_cols : list,
+        Column names by which to partition the dataset
+        Columns are partitioned in the order they are given
+    preserve_index : bool,
+        Parameter for instantiating Table; preserve pandas index or not.
+    **kwargs : dict, kwargs for write_table function.
+    """
+    from pyarrow import (
+        Table,
+        compat
+    )
+
+    if filesystem is None:
+        fs = LocalFileSystem.get_instance()
+    else:
+        fs = _ensure_filesystem(filesystem)
+
+    if not fs.exists(root_path):
+        fs.mkdir(root_path)
+
+    if partition_cols is not None and len(partition_cols) > 0:
+        df = table.to_pandas()
+        partition_keys = [df[col] for col in partition_cols]
+        data_df = df.drop(partition_cols, axis='columns')
+        data_cols = df.columns.drop(partition_cols)
+        if len(data_cols) == 0:
+            raise ValueError("No data left to save outside partition columns")
+        for keys, subgroup in data_df.groupby(partition_keys):
+            if not isinstance(keys, tuple):
+                keys = (keys,)
+            subdir = "/".join(
+                ["{colname}={value}".format(colname=name, value=val)
+                 for name, val in zip(partition_cols, keys)])
+            subtable = Table.from_pandas(subgroup,
+                                         preserve_index=preserve_index)
+            prefix = "/".join([root_path, subdir])
+            if not fs.exists(prefix):
+                fs.mkdir(prefix)
+            outfile = compat.guid() + ".parquet"
+            full_path = "/".join([prefix, outfile])
+            with fs.open(full_path, 'wb') as f:
+                write_table(subtable, f, **kwargs)
+    else:
+        outfile = compat.guid() + ".parquet"
+        full_path = "/".join([root_path, outfile])
+        with fs.open(full_path, 'wb') as f:
+            write_table(table, f, **kwargs)
+
+
 def write_metadata(schema, where, version='1.0',
                    use_deprecated_int96_timestamps=False,
                    coerce_timestamps=None):
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index cf63008e69e68..cbc277373c707 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -268,6 +268,20 @@ def test_read_common_metadata_files(self):
         self.hdfs.mkdir(tmpdir)
         test_parquet._test_read_common_metadata_files(self.hdfs, tmpdir)
 
+    @test_parquet.parquet
+    def test_write_to_dataset_with_partitions(self):
+        tmpdir = pjoin(self.tmp_path, 'write-partitions-' + guid())
+        self.hdfs.mkdir(tmpdir)
+        test_parquet._test_write_to_dataset_with_partitions(
+            tmpdir, filesystem=self.hdfs)
+
+    @test_parquet.parquet
+    def test_write_to_dataset_no_partitions(self):
+        tmpdir = pjoin(self.tmp_path, 'write-no_partitions-' + guid())
+        self.hdfs.mkdir(tmpdir)
+        test_parquet._test_write_to_dataset_no_partitions(
+            tmpdir, filesystem=self.hdfs)
+
 
 class TestLibHdfs(HdfsTestCases, unittest.TestCase):
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 67944f98fdf18..de6f431d44dc1 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1147,3 +1147,76 @@ def test_read_non_existent_file(tmpdir):
         pq.read_table(path)
     except Exception as e:
         assert path in e.args[0]
+
+
+def _test_write_to_dataset_with_partitions(base_path, filesystem=None):
+    # ARROW-1400
+    import pyarrow.parquet as pq
+
+    output_df = pd.DataFrame({'group1': list('aaabbbbccc'),
+                              'group2': list('eefeffgeee'),
+                              'num': list(range(10)),
+                              'date': np.arange('2017-01-01', '2017-01-11',
+                                                dtype='datetime64[D]')})
+    cols = output_df.columns.tolist()
+    partition_by = ['group1', 'group2']
+    output_table = pa.Table.from_pandas(output_df)
+    pq.write_to_dataset(output_table, base_path, partition_by,
+                        filesystem=filesystem)
+    input_table = pq.ParquetDataset(base_path, filesystem=filesystem).read()
+    input_df = input_table.to_pandas()
+
+    # Read data back in and compare with original DataFrame
+    # Partitioned columns added to the end of the DataFrame when read
+    input_df_cols = input_df.columns.tolist()
+    assert partition_by == input_df_cols[-1 * len(partition_by):]
+
+    # Partitioned columns become 'categorical' dtypes
+    input_df = input_df[cols]
+    for col in partition_by:
+        output_df[col] = output_df[col].astype('category')
+    assert output_df.equals(input_df)
+
+
+def _test_write_to_dataset_no_partitions(base_path, filesystem=None):
+    # ARROW-1400
+    import pyarrow.parquet as pq
+
+    output_df = pd.DataFrame({'group1': list('aaabbbbccc'),
+                              'group2': list('eefeffgeee'),
+                              'num': list(range(10)),
+                              'date': np.arange('2017-01-01', '2017-01-11',
+                                                dtype='datetime64[D]')})
+    cols = output_df.columns.tolist()
+    output_table = pa.Table.from_pandas(output_df)
+
+    if filesystem is None:
+        filesystem = LocalFileSystem.get_instance()
+
+    # Without partitions, append files to root_path
+    n = 5
+    for i in range(n):
+        pq.write_to_dataset(output_table, base_path,
+                            filesystem=filesystem)
+    output_files = [file for file in filesystem.ls(base_path)
+                    if file.endswith(".parquet")]
+    assert len(output_files) == n
+
+    # Deduplicated incoming DataFrame should match
+    # original outgoing Dataframe
+    input_table = pq.ParquetDataset(base_path,
+                                    filesystem=filesystem).read()
+    input_df = input_table.to_pandas()
+    input_df = input_df.drop_duplicates()
+    input_df = input_df[cols]
+    assert output_df.equals(input_df)
+
+
+@parquet
+def test_write_to_dataset_with_partitions(tmpdir):
+    _test_write_to_dataset_with_partitions(str(tmpdir))
+
+
+@parquet
+def test_write_to_dataset_no_partitions(tmpdir):
+    _test_write_to_dataset_no_partitions(str(tmpdir))

From e5aeb900161a068e665792a678729fee4e69f13c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 4 Sep 2017 09:02:41 -0400
Subject: [PATCH 0992/1644] ARROW-1383: [C++] Add vector append variant to
 primitive array builders that accepts std::vector<bool>

Other libraries may have null indicators in the form of bits or bytes. `std::vector<bool>` is a bit-packed container like Arrow's internal representation.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1033 from wesm/ARROW-1383 and squashes the following commits:

0f05f2d0 [Wes McKinney] Fix another MSVC compiler warning
cba48fc6 [Wes McKinney] More precise bool checks in BooleanBuilder::Append
6f97b063 [Wes McKinney] Add variant of vector Append in primitive array builders that accepts std::vector<bool>
---
 cpp/src/arrow/array-test.cc | 69 +++++++++++++++++++++++++++++
 cpp/src/arrow/builder.cc    | 86 ++++++++++++++++++++++++++++++++-----
 cpp/src/arrow/builder.h     | 75 ++++++++++++++++++++++++--------
 3 files changed, 202 insertions(+), 28 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 0eb19d3bc90b1..3a3d28c777a69 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -593,6 +593,42 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
   this->Check(this->builder_nn_, false);
 }
 
+TYPED_TEST(TestPrimitiveBuilder, TestAppendVectorStdBool) {
+  // ARROW-1383
+  DECL_T();
+
+  int64_t size = 10000;
+  this->RandomData(size);
+
+  vector<T>& draws = this->draws_;
+
+  std::vector<bool> is_valid;
+
+  // first slug
+  int64_t K = 1000;
+
+  for (int64_t i = 0; i < K; ++i) {
+    is_valid.push_back(this->valid_bytes_[i] != 0);
+  }
+  ASSERT_OK(this->builder_->Append(draws.data(), K, is_valid));
+
+  ASSERT_EQ(1000, this->builder_->length());
+  ASSERT_EQ(1024, this->builder_->capacity());
+
+  // Append the next 9000
+  is_valid.clear();
+  for (int64_t i = K; i < size; ++i) {
+    is_valid.push_back(this->valid_bytes_[i] != 0);
+  }
+
+  ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, is_valid));
+
+  ASSERT_EQ(size, this->builder_->length());
+  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
+
+  this->Check(this->builder_, true);
+}
+
 TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
   int64_t n = 1000;
   ASSERT_OK(this->builder_->Reserve(n));
@@ -630,6 +666,39 @@ TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
   ASSERT_EQ(BitUtil::NextPower2(kMinBuilderCapacity + 100), this->builder_->capacity());
 }
 
+TEST(TestBooleanBuilder, TestStdBoolVectorAppend) {
+  BooleanBuilder builder;
+
+  std::vector<bool> values, is_valid;
+
+  const int length = 10000;
+  test::random_is_valid(length, 0.5, &values);
+  test::random_is_valid(length, 0.1, &is_valid);
+
+  const int chunksize = 1000;
+  for (int chunk = 0; chunk < length / chunksize; ++chunk) {
+    std::vector<bool> chunk_values, chunk_is_valid;
+    for (int i = chunk * chunksize; i < (chunk + 1) * chunksize; ++i) {
+      chunk_values.push_back(values[i]);
+      chunk_is_valid.push_back(is_valid[i]);
+    }
+    ASSERT_OK(builder.Append(chunk_values, chunk_is_valid));
+  }
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  const auto& arr = static_cast<const BooleanArray&>(*result);
+  for (int i = 0; i < length; ++i) {
+    if (is_valid[i]) {
+      ASSERT_FALSE(arr.IsNull(i));
+      ASSERT_EQ(values[i], arr.Value(i));
+    } else {
+      ASSERT_TRUE(arr.IsNull(i));
+    }
+  }
+}
+
 template <typename TYPE>
 void CheckSliceApproxEquals() {
   using T = typename TYPE::c_type;
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index a1d2366f69176..24d26d4386406 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -154,6 +154,37 @@ void ArrayBuilder::UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t leng
   length_ += length;
 }
 
+void ArrayBuilder::UnsafeAppendToBitmap(const std::vector<bool>& is_valid) {
+  int64_t byte_offset = length_ / 8;
+  int64_t bit_offset = length_ % 8;
+  uint8_t bitset = null_bitmap_data_[byte_offset];
+
+  const int64_t length = static_cast<int64_t>(is_valid.size());
+
+  for (int64_t i = 0; i < length; ++i) {
+    if (bit_offset == 8) {
+      bit_offset = 0;
+      null_bitmap_data_[byte_offset] = bitset;
+      byte_offset++;
+      // TODO: Except for the last byte, this shouldn't be needed
+      bitset = null_bitmap_data_[byte_offset];
+    }
+
+    if (is_valid[i]) {
+      bitset |= BitUtil::kBitmask[bit_offset];
+    } else {
+      bitset &= BitUtil::kFlippedBitmask[bit_offset];
+      ++null_count_;
+    }
+
+    bit_offset++;
+  }
+  if (bit_offset != 0) {
+    null_bitmap_data_[byte_offset] = bitset;
+  }
+  length_ += length;
+}
+
 void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
   const int64_t new_length = length + length_;
 
@@ -243,6 +274,23 @@ Status PrimitiveBuilder<T>::Append(const value_type* values, int64_t length,
   return Status::OK();
 }
 
+template <typename T>
+Status PrimitiveBuilder<T>::Append(const value_type* values, int64_t length,
+                                   const std::vector<bool>& is_valid) {
+  RETURN_NOT_OK(Reserve(length));
+  DCHECK_EQ(length, static_cast<int64_t>(is_valid.size()));
+
+  if (length > 0) {
+    std::memcpy(raw_data_ + length_, values,
+                static_cast<std::size_t>(TypeTraits<T>::bytes_required(length)));
+  }
+
+  // length_ is update by these
+  ArrayBuilder::UnsafeAppendToBitmap(is_valid);
+
+  return Status::OK();
+}
+
 template <typename T>
 Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
   const int64_t bytes_required = TypeTraits<T>::bytes_required(length_);
@@ -691,16 +739,7 @@ Status BooleanBuilder::Append(const uint8_t* values, int64_t length,
   RETURN_NOT_OK(Reserve(length));
 
   for (int64_t i = 0; i < length; ++i) {
-    // Skip reading from unitialised memory
-    // TODO: This actually is only to keep valgrind happy but may or may not
-    // have a performance impact.
-    if ((valid_bytes != nullptr) && !valid_bytes[i]) continue;
-
-    if (values[i] > 0) {
-      BitUtil::SetBit(raw_data_, length_ + i);
-    } else {
-      BitUtil::ClearBit(raw_data_, length_ + i);
-    }
+    BitUtil::SetBitTo(raw_data_, length_ + i, values[i] != 0);
   }
 
   // this updates length_
@@ -708,6 +747,33 @@ Status BooleanBuilder::Append(const uint8_t* values, int64_t length,
   return Status::OK();
 }
 
+Status BooleanBuilder::Append(const uint8_t* values, int64_t length,
+                              const std::vector<bool>& is_valid) {
+  RETURN_NOT_OK(Reserve(length));
+
+  for (int64_t i = 0; i < length; ++i) {
+    BitUtil::SetBitTo(raw_data_, length_ + i, values[i] != 0);
+  }
+
+  // this updates length_
+  ArrayBuilder::UnsafeAppendToBitmap(is_valid);
+  return Status::OK();
+}
+
+Status BooleanBuilder::Append(const std::vector<bool>& values,
+                              const std::vector<bool>& is_valid) {
+  const int64_t length = static_cast<int64_t>(values.size());
+  RETURN_NOT_OK(Reserve(length));
+
+  for (int64_t i = 0; i < length; ++i) {
+    BitUtil::SetBitTo(raw_data_, length_ + i, values[i]);
+  }
+
+  // this updates length_
+  ArrayBuilder::UnsafeAppendToBitmap(is_valid);
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // DictionaryBuilder
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index d1b51d6431da1..a649d628ad130 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -83,9 +83,11 @@ class ARROW_EXPORT ArrayBuilder {
 
   /// Append to null bitmap
   Status AppendToBitmap(bool is_valid);
+
   /// Vector append. Treat each zero byte as a null.   If valid_bytes is null
   /// assume all of length bits are valid.
   Status AppendToBitmap(const uint8_t* valid_bytes, int64_t length);
+
   /// Set the next length bits to not null (i.e. valid).
   Status SetNotNull(int64_t length);
 
@@ -151,6 +153,9 @@ class ARROW_EXPORT ArrayBuilder {
   // Vector append. Treat each zero byte as a nullzero. If valid_bytes is null
   // assume all of length bits are valid.
   void UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t length);
+
+  void UnsafeAppendToBitmap(const std::vector<bool>& is_valid);
+
   // Set the next length bits to not null (i.e. valid).
   void UnsafeSetNotNull(int64_t length);
 
@@ -202,13 +207,24 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
 
   std::shared_ptr<Buffer> data() const { return data_; }
 
-  /// Vector append
-  ///
-  /// If passed, valid_bytes is of equal length to values, and any zero byte
-  /// will be considered as a null for that slot
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a contiguous C array of values
+  /// \param[in] length the number of values to append
+  /// \param[in] valid_bytes an optional sequence of bytes where non-zero
+  /// indicates a valid (non-null) value)
+  /// \return Status
   Status Append(const value_type* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
 
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a contiguous C array of values
+  /// \param[in] length the number of values to append
+  /// \param[in] is_valid an std::vector<bool> indicating valid (1) or null
+  /// (0). Equal in length to values
+  /// \return Status
+  Status Append(const value_type* values, int64_t length,
+                const std::vector<bool>& is_valid);
+
   Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int64_t capacity) override;
 
@@ -395,17 +411,20 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
     return Status::OK();
   }
 
-  /// Vector append
-  ///
-  /// If passed, valid_bytes is of equal length to values, and any zero byte
-  /// will be considered as a null for that slot
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a contiguous C array of values
+  /// \param[in] length the number of values to append
+  /// \param[in] valid_bytes an optional sequence of bytes where non-zero
+  /// indicates a valid (non-null) value)
+  /// \return Status
   Status Append(const uint64_t* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
 
-  Status ExpandIntSize(uint8_t new_int_size);
   Status Finish(std::shared_ptr<Array>* out) override;
 
  protected:
+  Status ExpandIntSize(uint8_t new_int_size);
+
   template <typename new_type, typename old_type>
   typename std::enable_if<sizeof(old_type) >= sizeof(new_type), Status>::type
   ExpandIntSizeInternal();
@@ -454,17 +473,20 @@ class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase
     return Status::OK();
   }
 
-  /// Vector append
-  ///
-  /// If passed, valid_bytes is of equal length to values, and any zero byte
-  /// will be considered as a null for that slot
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a contiguous C array of values
+  /// \param[in] length the number of values to append
+  /// \param[in] valid_bytes an optional sequence of bytes where non-zero
+  /// indicates a valid (non-null) value)
+  /// \return Status
   Status Append(const int64_t* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
 
-  Status ExpandIntSize(uint8_t new_int_size);
   Status Finish(std::shared_ptr<Array>* out) override;
 
  protected:
+  Status ExpandIntSize(uint8_t new_int_size);
+
   template <typename new_type, typename old_type>
   typename std::enable_if<sizeof(old_type) >= sizeof(new_type), Status>::type
   ExpandIntSizeInternal();
@@ -521,13 +543,30 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 
   Status Append(const uint8_t val) { return Append(val != 0); }
 
-  /// Vector append
-  ///
-  /// If passed, valid_bytes is of equal length to values, and any zero byte
-  /// will be considered as a null for that slot
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a contiguous array of bytes (non-zero is 1)
+  /// \param[in] length the number of values to append
+  /// \param[in] valid_bytes an optional sequence of bytes where non-zero
+  /// indicates a valid (non-null) value)
+  /// \return Status
   Status Append(const uint8_t* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
 
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a contiguous C array of values
+  /// \param[in] length the number of values to append
+  /// \param[in] is_valid an std::vector<bool> indicating valid (1) or null
+  /// (0). Equal in length to values
+  /// \return Status
+  Status Append(const uint8_t* values, int64_t length, const std::vector<bool>& is_valid);
+
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values an std::vector<bool> indicating true (1) or false
+  /// \param[in] is_valid an std::vector<bool> indicating valid (1) or null
+  /// (0). Equal in length to values
+  /// \return Status
+  Status Append(const std::vector<bool>& values, const std::vector<bool>& is_valid);
+
   Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int64_t capacity) override;
 

From cc3051262271ac4a25e54af3e81765ccbbd1fbd0 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 4 Sep 2017 09:53:58 -0400
Subject: [PATCH 0993/1644] ARROW-1390: [Python] Add more serialization tests

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1035 from pcmoritz/serialization-tests and squashes the following commits:

636a3557 [Philipp Moritz] add more serialization tests
---
 python/pyarrow/tests/test_serialization.py | 78 +++++++++++++++++-----
 1 file changed, 63 insertions(+), 15 deletions(-)

diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 5526ac68cab76..aa6301d7bc86f 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -80,23 +80,27 @@ def assert_equal(obj1, obj2):
         assert obj1 == obj2, "Objects {} and {} are different.".format(obj1,
                                                                        obj2)
 
-
-if sys.version_info >= (3, 0):
-    long_extras = [0, np.array([["hi", u"hi"], [1.3, 1]])]
-else:
-    _LONG_ZERO, _LONG_ONE = long(0), long(1)  # noqa: E501,F821
-    long_extras = [_LONG_ZERO, np.array([["hi", u"hi"],
-                                         [1.3, _LONG_ONE]])]
-
 PRIMITIVE_OBJECTS = [
     0, 0.0, 0.9, 1 << 62, 1 << 100, 1 << 999,
     [1 << 100, [1 << 100]], "a", string.printable, "\u262F",
-    u"hello world", u"\xff\xfe\x9c\x001\x000\x00", None, True,
-    False, [], (), {}, np.int8(3), np.int32(4), np.int64(5),
+    "hello world", u"hello world", u"\xff\xfe\x9c\x001\x000\x00",
+    None, True, False, [], (), {}, {(1, 2): 1}, {(): 2},
+    [1, "hello", 3.0], u"\u262F", 42.0, (1.0, "hi"),
+    [1, 2, 3, None], [(None,), 3, 1.0], ["h", "e", "l", "l", "o", None],
+    (None, None), ("hello", None), (True, False),
+    {True: "hello", False: "world"}, {"hello": "world", 1: 42, 2.5: 45},
+    np.int8(3), np.int32(4), np.int64(5),
     np.uint8(3), np.uint32(4), np.uint64(5), np.float32(1.9),
     np.float64(1.9), np.zeros([100, 100]),
     np.random.normal(size=[100, 100]), np.array(["hi", 3]),
-    np.array(["hi", 3], dtype=object)] + long_extras
+    np.array(["hi", 3], dtype=object)]
+
+if sys.version_info >= (3, 0):
+    PRIMITIVE_OBJECTS += [0, np.array([["hi", u"hi"], [1.3, 1]])]
+else:
+    PRIMITIVE_OBJECTS += [long(42), long(1 << 62), long(0), \
+                          np.array([["hi", u"hi"], \
+                          [1.3, long(1)]])] # noqa: E501,F821
 
 COMPLEX_OBJECTS = [
     [[[[[[[[[[[[]]]]]]]]]]]],
@@ -172,6 +176,8 @@ def array_custom_deserializer(serialized_obj):
 
     context = pa.SerializationContext()
 
+    # This is for numpy arrays of "object" only; primitive types are handled
+    # efficiently with Arrow's Tensor facilities (see python_to_arrow.cc)
     context.register_type(np.ndarray, 20 * b"\x00",
                           custom_serializer=array_custom_serializer,
                           custom_deserializer=array_custom_deserializer)
@@ -217,17 +223,16 @@ def serialization_roundtrip(value, f):
 
 
 @pytest.yield_fixture(scope='session')
-def large_memory_map(tmpdir_factory):
+def large_memory_map(tmpdir_factory, size=100*1024*1024):
     path = (tmpdir_factory.mktemp('data')
             .join('pyarrow-serialization-tmp-file').strpath)
 
     # Create a large memory mapped file
-    SIZE = 100 * 1024 * 1024  # 100 MB
     with open(path, 'wb') as f:
-        f.write(np.random.randint(0, 256, size=SIZE)
+        f.write(np.random.randint(0, 256, size=size)
                 .astype('u1')
                 .tobytes()
-                [:SIZE])
+                [:size])
     return path
 
 
@@ -256,6 +261,49 @@ def test_custom_serialization(large_memory_map):
         for obj in CUSTOM_OBJECTS:
             serialization_roundtrip(obj, mmap)
 
+def test_numpy_serialization(large_memory_map):
+    with pa.memory_map(large_memory_map, mode="r+") as mmap:
+        for t in ["int8", "uint8", "int16", "uint16",
+                  "int32", "uint32", "float32", "float64"]:
+            obj = np.random.randint(0, 10, size=(100, 100)).astype(t)
+            serialization_roundtrip(obj, mmap)
+
+def test_numpy_immutable(large_memory_map):
+    with pa.memory_map(large_memory_map, mode="r+") as mmap:
+        obj = np.zeros([10])
+        mmap.seek(0)
+        pa.serialize_to(obj, mmap, serialization_context)
+        mmap.seek(0)
+        result = pa.deserialize_from(mmap, None, serialization_context)
+        with pytest.raises(ValueError):
+            result[0] = 1.0
+
+@pytest.mark.skip(reason="extensive memory requirements")
+def test_arrow_limits(self):
+    huge_memory_map = lambda temp_dir: large_memory_map(temp_dir, 100 * 1024 * 1024 * 1024)
+    with pa.memory_map(huge_memory_map, mode="r+") as mmap:
+        # Test that objects that are too large for Arrow throw a Python
+        # exception. These tests give out of memory errors on Travis and need
+        # to be run on a machine with lots of RAM.
+        l = 2 ** 29 * [1.0]
+        serialization_roundtrip(l, mmap)
+        del l
+        l = 2 ** 29 * ["s"]
+        serialization_roundtrip(l, mmap)
+        del l
+        l = 2 ** 29 * [["1"], 2, 3, [{"s": 4}]]
+        serialization_roundtrip(l, mmap)
+        del l
+        serialization_roundtrip(l, mmap)
+        l = 2 ** 29 * [{"s": 1}] + 2 ** 29 * [1.0]
+        del l
+        l = np.zeros(2 ** 25)
+        serialization_roundtrip(l, mmap)
+        del l
+        l = [np.zeros(2 ** 18) for _ in range(2 ** 7)]
+        serialization_roundtrip(l, mmap)
+        del l
+
 def test_serialization_callback_error():
 
     class TempClass(object):

From f391950b8c9b52511d62fe82727cc5d98207e11d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 4 Sep 2017 13:31:17 -0400
Subject: [PATCH 0994/1644] ARROW-1452: [C++] Restore DISALLOW_COPY_AND_ASSIGN
 usages removed in ARROW-1452 patch

Also updated the definition of DISALLOW_COPY_AND_ASSIGN to use the one from Apache Kudu, which has seen significantly more scrutiny than ours (https://github.com/apache/kudu/blob/master/src/kudu/gutil/macros.h#L96).

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1036 from wesm/ARROW-1452-followup and squashes the following commits:

bfab15bc [Wes McKinney] clang-format
75a95b9c [Wes McKinney] Restore DISALLOW_COPY_AND_ASSIGN usages removed in ARROW-1452 patch
---
 cpp/src/arrow/array.h         | 3 +++
 cpp/src/arrow/io/interfaces.h | 3 +++
 cpp/src/arrow/type.h          | 3 +++
 cpp/src/arrow/util/macros.h   | 2 +-
 4 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 8052071d5d0ee..8e965e8d9174b 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -254,6 +254,9 @@ class ARROW_EXPORT Array {
     }
     data_ = data;
   }
+
+ private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
 static inline std::ostream& operator<<(std::ostream& os, const Array& x) {
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 59aed75a7630e..11441794e18d2 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -79,6 +79,9 @@ class ARROW_EXPORT FileInterface {
   FileInterface() {}
   FileMode::type mode_;
   void set_mode(FileMode::type mode) { mode_ = mode; }
+
+ private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(FileInterface);
 };
 
 class ARROW_EXPORT Seekable {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 3b7ef19b87547..2a89423d16504 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -157,6 +157,9 @@ class ARROW_EXPORT DataType {
  protected:
   Type::type id_;
   std::vector<std::shared_ptr<Field>> children_;
+
+ private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(DataType);
 };
 
 // TODO(wesm): Remove this from parquet-cpp
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index ec8d3bd1fc4a1..2f1db0924a179 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -22,7 +22,7 @@
 #ifndef ARROW_DISALLOW_COPY_AND_ASSIGN
 #define ARROW_DISALLOW_COPY_AND_ASSIGN(TypeName) \
   TypeName(const TypeName&) = delete;            \
-  TypeName& operator=(const TypeName&) = delete
+  void operator=(const TypeName&) = delete
 #endif
 
 #define ARROW_UNUSED(x) (void)x

From 0e0da74c540bdd24ba9021ce66ea2d236ed3b1c9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 4 Sep 2017 15:39:32 -0400
Subject: [PATCH 0995/1644] ARROW-1460: [C++] Pin clang-format at LLVM 4.0

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1037 from wesm/ARROW-1460 and squashes the following commits:

2cfc3532 [Wes McKinney] Use clang-format-4.0
---
 cpp/CMakeLists.txt                      |  2 +-
 cpp/src/arrow/ipc/ipc-json-test.cc      | 19 ++++++++++++++-----
 cpp/src/arrow/python/arrow_to_python.cc |  6 +++---
 cpp/src/arrow/python/python_to_arrow.cc |  4 ++--
 cpp/src/arrow/util/logging.h            |  3 +--
 cpp/src/plasma/protocol.cc              |  4 ++--
 6 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 12a580374a2aa..f25cc622c8872 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -551,7 +551,7 @@ endif (UNIX)
 # "make format" and "make check-format" targets
 ############################################################
 
-set(CLANG_FORMAT_VERSION 3.9)
+set(CLANG_FORMAT_VERSION 4.0)
 
 # runs clang format and updates files in place.
 add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index ddb2e37288e46..1967e24c4d233 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -112,13 +112,22 @@ TEST(TestJsonSchemaWriter, FlatTypes) {
   // TODO
   // field("f14", date32())
   std::vector<std::shared_ptr<Field>> fields = {
-      field("f0", int8()), field("f1", int16(), false), field("f2", int32()),
-      field("f3", int64(), false), field("f4", uint8()), field("f5", uint16()),
-      field("f6", uint32()), field("f7", uint64()), field("f8", float32()),
-      field("f9", float64()), field("f10", utf8()), field("f11", binary()),
+      field("f0", int8()),
+      field("f1", int16(), false),
+      field("f2", int32()),
+      field("f3", int64(), false),
+      field("f4", uint8()),
+      field("f5", uint16()),
+      field("f6", uint32()),
+      field("f7", uint64()),
+      field("f8", float32()),
+      field("f9", float64()),
+      field("f10", utf8()),
+      field("f11", binary()),
       field("f12", list(int32())),
       field("f13", struct_({field("s1", int32()), field("s2", utf8())})),
-      field("f15", date64()), field("f16", timestamp(TimeUnit::NANO)),
+      field("f15", date64()),
+      field("f16", timestamp(TimeUnit::NANO)),
       field("f17", time64(TimeUnit::MICRO)),
       field("f18", union_({field("u1", int8()), field("u2", time32(TimeUnit::MILLI))},
                           {0, 1}, UnionMode::DENSE))};
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index 3ff1f11018d10..b127971a16f91 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -171,20 +171,20 @@ Status GetValue(PyObject* context, std::shared_ptr<Array> arr, int64_t index,
     }                                                                             \
   }                                                                               \
   *out = result.release();                                                        \
-  return Status::OK();
+  return Status::OK()
 
 Status DeserializeList(PyObject* context, std::shared_ptr<Array> array, int64_t start_idx,
                        int64_t stop_idx, PyObject* base,
                        const std::vector<std::shared_ptr<Tensor>>& tensors,
                        PyObject** out) {
-  DESERIALIZE_SEQUENCE(PyList_New, PyList_SET_ITEM)
+  DESERIALIZE_SEQUENCE(PyList_New, PyList_SET_ITEM);
 }
 
 Status DeserializeTuple(PyObject* context, std::shared_ptr<Array> array,
                         int64_t start_idx, int64_t stop_idx, PyObject* base,
                         const std::vector<std::shared_ptr<Tensor>>& tensors,
                         PyObject** out) {
-  DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SET_ITEM)
+  DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SET_ITEM);
 }
 
 Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out) {
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index d823bad7373c7..65e5f6a46a876 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -522,7 +522,7 @@ Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
         "recursively.");
   }
   SequenceBuilder builder(nullptr);
-  std::vector<PyObject *> sublists, subtuples, subdicts;
+  std::vector<PyObject*> sublists, subtuples, subdicts;
   for (const auto& sequence : sequences) {
     ScopedRef iterator(PyObject_GetIter(sequence));
     RETURN_IF_PYERROR();
@@ -559,7 +559,7 @@ Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
         "This object exceeds the maximum recursion depth. It may contain itself "
         "recursively.");
   }
-  std::vector<PyObject *> key_tuples, key_dicts, val_lists, val_tuples, val_dicts, dummy;
+  std::vector<PyObject*> key_tuples, key_dicts, val_lists, val_tuples, val_dicts, dummy;
   for (const auto& dict : dicts) {
     PyObject *key, *value;
     Py_ssize_t pos = 0;
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 998f7ed7bfaaa..b7e2ceea9f3d8 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -98,8 +98,7 @@ class NullLog {
 class CerrLog {
  public:
   CerrLog(int severity)  // NOLINT(runtime/explicit)
-      : severity_(severity),
-        has_logged_(false) {}
+      : severity_(severity), has_logged_(false) {}
 
   virtual ~CerrLog() {
     if (has_logged_) {
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 77bc8b7aae303..305e3d5d9cef1 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -40,8 +40,8 @@ to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
 Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer) {
   int64_t type;
   RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
-  ARROW_CHECK(type == message_type) << "type = " << type
-                                    << ", message_type = " << message_type;
+  ARROW_CHECK(type == message_type)
+      << "type = " << type << ", message_type = " << message_type;
   return Status::OK();
 }
 

From b1e56a2f5d3fef3d04093fcfd4f279290f597d06 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 4 Sep 2017 15:40:16 -0400
Subject: [PATCH 0996/1644] ARROW-1453: [C++/Python] Support non-contiguous
 Tensors in WriteTensor

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1034 from pcmoritz/write-tensor and squashes the following commits:

7b2ae971 [Wes McKinney] Fix off by one error
084b6b90 [Wes McKinney] Remove outdated comment
f7be2315 [Wes McKinney] Copy strided tensor data into a scratch buffer before writing to output stream
e4be4af3 [Philipp Moritz] fix flake8 linting
3576840f [Philipp Moritz] fixes and more tests
fc443218 [Philipp Moritz] fix python test
57c7aceb [Philipp Moritz] fix linting
23d991f7 [Philipp Moritz] add fixes and tests
357aa730 [Philipp Moritz] fix linting
0e22ebf0 [Philipp Moritz] implement writing strided tensors
---
 cpp/src/arrow/compare.cc                   | 51 ++++++++++++++----
 cpp/src/arrow/ipc/ipc-read-write-test.cc   |  6 +--
 cpp/src/arrow/ipc/writer.cc                | 61 ++++++++++++++++++----
 python/pyarrow/tests/test_serialization.py | 23 +++++---
 python/pyarrow/tests/test_tensor.py        | 29 ++++++++--
 5 files changed, 132 insertions(+), 38 deletions(-)

diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index b78d5d5fb4d2e..876e7f73d3fd2 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -694,6 +694,29 @@ Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_star
   return Status::OK();
 }
 
+bool StridedTensorContentEquals(int dim_index, int64_t left_offset, int64_t right_offset,
+                                int elem_size, const Tensor& left, const Tensor& right) {
+  if (dim_index == left.ndim() - 1) {
+    for (int64_t i = 0; i < left.shape()[dim_index]; ++i) {
+      if (memcmp(left.raw_data() + left_offset + i * left.strides()[dim_index],
+                 right.raw_data() + right_offset + i * right.strides()[dim_index],
+                 elem_size) != 0) {
+        return false;
+      }
+    }
+    return true;
+  }
+  for (int64_t i = 0; i < left.shape()[dim_index]; ++i) {
+    if (!StridedTensorContentEquals(dim_index + 1, left_offset, right_offset, elem_size,
+                                    left, right)) {
+      return false;
+    }
+    left_offset += left.strides()[dim_index];
+    right_offset += right.strides()[dim_index];
+  }
+  return true;
+}
+
 Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
   // The arrays are the same object
   if (&left == &right) {
@@ -704,19 +727,25 @@ Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
     *are_equal = true;
   } else {
     if (!left.is_contiguous() || !right.is_contiguous()) {
-      return Status::NotImplemented(
-          "Comparison not implemented for non-contiguous tensors");
-    }
-
-    const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
-    const int byte_width = size_meta.bit_width() / CHAR_BIT;
-    DCHECK_GT(byte_width, 0);
+      const auto& shape = left.shape();
+      if (shape != right.shape()) {
+        *are_equal = false;
+        return Status::OK();
+      }
+      const auto& type = static_cast<const FixedWidthType&>(*left.type());
+      *are_equal = StridedTensorContentEquals(0, 0, 0, type.bit_width() / 8, left, right);
+      return Status::OK();
+    } else {
+      const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
+      const int byte_width = size_meta.bit_width() / CHAR_BIT;
+      DCHECK_GT(byte_width, 0);
 
-    const uint8_t* left_data = left.data()->data();
-    const uint8_t* right_data = right.data()->data();
+      const uint8_t* left_data = left.data()->data();
+      const uint8_t* right_data = right.data()->data();
 
-    *are_equal =
-        memcmp(left_data, right_data, static_cast<size_t>(byte_width * left.size())) == 0;
+      *are_equal = memcmp(left_data, right_data,
+                          static_cast<size_t>(byte_width * left.size())) == 0;
+    }
   }
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index a0005b591b596..94d8eda292d87 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -739,11 +739,7 @@ TEST_F(TestTensorRoundTrip, NonContiguous) {
   auto data = test::GetBufferFromVector(values);
   Tensor tensor(int64(), data, {4, 3}, {48, 16});
 
-  int32_t metadata_length;
-  int64_t body_length;
-  ASSERT_OK(mmap_->Seek(0));
-  ASSERT_RAISES(Invalid,
-                WriteTensor(tensor, mmap_.get(), &metadata_length, &body_length));
+  CheckTensorRoundTrip(tensor);
 }
 
 }  // namespace ipc
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 6e05b94ffba18..7ac63804ec2a9 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -563,23 +563,62 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
                           pool, kMaxNestingDepth, true);
 }
 
-Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
-                   int64_t* body_length) {
-  if (!tensor.is_contiguous()) {
-    return Status::Invalid("No support yet for writing non-contiguous tensors");
+static Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_size,
+                                     const Tensor& tensor, uint8_t* scratch_space,
+                                     io::OutputStream* dst) {
+  if (dim_index == tensor.ndim() - 1) {
+    const uint8_t* data_ptr = tensor.raw_data() + offset;
+    const int64_t stride = tensor.strides()[dim_index];
+    for (int64_t i = 0; i < tensor.shape()[dim_index]; ++i) {
+      memcpy(scratch_space + i * elem_size, data_ptr, elem_size);
+      data_ptr += stride;
+    }
+    return dst->Write(scratch_space, elem_size * tensor.shape()[dim_index]);
+  }
+  for (int64_t i = 0; i < tensor.shape()[dim_index]; ++i) {
+    RETURN_NOT_OK(WriteStridedTensorData(dim_index + 1, offset, elem_size, tensor,
+                                         scratch_space, dst));
+    offset += tensor.strides()[dim_index];
   }
+  return Status::OK();
+}
 
+Status WriteTensorHeader(const Tensor& tensor, io::OutputStream* dst,
+                         int32_t* metadata_length, int64_t* body_length) {
   RETURN_NOT_OK(AlignStreamPosition(dst));
   std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(WriteTensorMessage(tensor, 0, &metadata));
-  RETURN_NOT_OK(WriteMessage(*metadata, dst, metadata_length));
-  auto data = tensor.data();
-  if (data) {
-    *body_length = data->size();
-    return dst->Write(data->data(), *body_length);
+  return WriteMessage(*metadata, dst, metadata_length);
+}
+
+Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
+                   int64_t* body_length) {
+  if (tensor.is_contiguous()) {
+    RETURN_NOT_OK(WriteTensorHeader(tensor, dst, metadata_length, body_length));
+    auto data = tensor.data();
+    if (data) {
+      *body_length = data->size();
+      return dst->Write(data->data(), *body_length);
+    } else {
+      *body_length = 0;
+      return Status::OK();
+    }
   } else {
-    *body_length = 0;
-    return Status::OK();
+    Tensor dummy(tensor.type(), tensor.data(), tensor.shape());
+    const auto& type = static_cast<const FixedWidthType&>(*tensor.type());
+    RETURN_NOT_OK(WriteTensorHeader(dummy, dst, metadata_length, body_length));
+
+    const int elem_size = type.bit_width() / 8;
+
+    // TODO(wesm): Do we care enough about this temporary allocation to pass in
+    // a MemoryPool to this function?
+    std::shared_ptr<Buffer> scratch_space;
+    RETURN_NOT_OK(AllocateBuffer(default_memory_pool(),
+                                 tensor.shape()[tensor.ndim() - 1] * elem_size,
+                                 &scratch_space));
+
+    return WriteStridedTensorData(0, 0, elem_size, tensor, scratch_space->mutable_data(),
+                                  dst);
   }
 }
 
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index aa6301d7bc86f..8afcf0f7c04b6 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -93,14 +93,17 @@ def assert_equal(obj1, obj2):
     np.uint8(3), np.uint32(4), np.uint64(5), np.float32(1.9),
     np.float64(1.9), np.zeros([100, 100]),
     np.random.normal(size=[100, 100]), np.array(["hi", 3]),
-    np.array(["hi", 3], dtype=object)]
+    np.array(["hi", 3], dtype=object),
+    np.random.normal(size=[45, 22]).T]
+
 
 if sys.version_info >= (3, 0):
     PRIMITIVE_OBJECTS += [0, np.array([["hi", u"hi"], [1.3, 1]])]
 else:
-    PRIMITIVE_OBJECTS += [long(42), long(1 << 62), long(0), \
-                          np.array([["hi", u"hi"], \
-                          [1.3, long(1)]])] # noqa: E501,F821
+    PRIMITIVE_OBJECTS += [long(42), long(1 << 62), long(0),
+                          np.array([["hi", u"hi"],
+                          [1.3, long(1)]])]  # noqa: E501,F821
+
 
 COMPLEX_OBJECTS = [
     [[[[[[[[[[[[]]]]]]]]]]]],
@@ -261,6 +264,7 @@ def test_custom_serialization(large_memory_map):
         for obj in CUSTOM_OBJECTS:
             serialization_roundtrip(obj, mmap)
 
+
 def test_numpy_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for t in ["int8", "uint8", "int16", "uint16",
@@ -268,6 +272,7 @@ def test_numpy_serialization(large_memory_map):
             obj = np.random.randint(0, 10, size=(100, 100)).astype(t)
             serialization_roundtrip(obj, mmap)
 
+
 def test_numpy_immutable(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         obj = np.zeros([10])
@@ -278,9 +283,12 @@ def test_numpy_immutable(large_memory_map):
         with pytest.raises(ValueError):
             result[0] = 1.0
 
+
 @pytest.mark.skip(reason="extensive memory requirements")
 def test_arrow_limits(self):
-    huge_memory_map = lambda temp_dir: large_memory_map(temp_dir, 100 * 1024 * 1024 * 1024)
+    def huge_memory_map(temp_dir):
+        return large_memory_map(temp_dir, 100 * 1024 * 1024 * 1024)
+
     with pa.memory_map(huge_memory_map, mode="r+") as mmap:
         # Test that objects that are too large for Arrow throw a Python
         # exception. These tests give out of memory errors on Travis and need
@@ -294,8 +302,8 @@ def test_arrow_limits(self):
         l = 2 ** 29 * [["1"], 2, 3, [{"s": 4}]]
         serialization_roundtrip(l, mmap)
         del l
-        serialization_roundtrip(l, mmap)
         l = 2 ** 29 * [{"s": 1}] + 2 ** 29 * [1.0]
+        serialization_roundtrip(l, mmap)
         del l
         l = np.zeros(2 ** 25)
         serialization_roundtrip(l, mmap)
@@ -304,6 +312,7 @@ def test_arrow_limits(self):
         serialization_roundtrip(l, mmap)
         del l
 
+
 def test_serialization_callback_error():
 
     class TempClass(object):
@@ -320,7 +329,7 @@ class TempClass(object):
     serialization_context.register_type(TempClass, 20*b"\x00")
     serialized_object = pa.serialize(TempClass(), serialization_context)
     deserialization_context = pa.SerializationContext()
-    
+
     # Pass a Serialization Context into deserialize, but TempClass
     # is not registered
     with pytest.raises(pa.DeserializationCallbackError) as err:
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index c4958342a6f47..1d45dc743b532 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -101,14 +101,35 @@ def test_tensor_ipc_roundtrip(tmpdir):
 
 
 def test_tensor_ipc_strided(tmpdir):
-    data = np.random.randn(10, 4)
-    tensor = pa.Tensor.from_numpy(data[::2])
+    data1 = np.random.randn(10, 4)
+    tensor1 = pa.Tensor.from_numpy(data1[::2])
+
+    data2 = np.random.randn(10, 6, 4)
+    tensor2 = pa.Tensor.from_numpy(data2[::, ::2, ::])
 
     path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-strided')
-    with pytest.raises(ValueError):
-        mmap = pa.create_memory_map(path, 1024)
+    mmap = pa.create_memory_map(path, 2048)
+
+    for tensor in [tensor1, tensor2]:
+        mmap.seek(0)
         pa.write_tensor(tensor, mmap)
 
+        mmap.seek(0)
+        result = pa.read_tensor(mmap)
+
+        assert result.equals(tensor)
+
+
+def test_tensor_equals():
+    data = np.random.randn(10, 6, 4)[::, ::2, ::]
+    tensor1 = pa.Tensor.from_numpy(data)
+    tensor2 = pa.Tensor.from_numpy(np.ascontiguousarray(data))
+    assert tensor1.equals(tensor2)
+    data = data.copy()
+    data[9, 0, 0] = 1.0
+    tensor2 = pa.Tensor.from_numpy(np.ascontiguousarray(data))
+    assert not tensor1.equals(tensor2)
+
 
 def test_tensor_size():
     data = np.random.randn(10, 4)

From ec32013fd6df35b051173f0e9aa8aa8833f1c819 Mon Sep 17 00:00:00 2001
From: fjetter <florian.jetter@blue-yonder.com>
Date: Mon, 4 Sep 2017 19:38:07 -0400
Subject: [PATCH 0997/1644] ARROW-1417: [Python] Allow more generic filesystem
 objects to be passed to ParquetDataset

This way, the `ParquetDataset` accepts both `S3FileSystem` and `LocalFileSystem` objects as they are used in `dask`. By using `issubclass`, external libraries may write their own FS wrappers by inheriting from the arrow FS.

I tested the integration with dask and this will fix the issue blocking https://github.com/dask/dask/pull/2527

Author: fjetter <florian.jetter@blue-yonder.com>

Closes #1032 from fjetter/ARROW-1417 and squashes the following commits:

75f18a5 [fjetter] Remove isinstance check in _ensure_filesystem
302b644 [fjetter] Perform check for type object before issubclass
ed111c9 [fjetter] Allow more generic filesystems to be passed
---
 python/pyarrow/parquet.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 4bc56ebc37404..5dabca982c86c 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -16,13 +16,14 @@
 # under the License.
 
 import os
+import inspect
 import json
 
 import six
 
 import numpy as np
 
-from pyarrow.filesystem import FileSystem, LocalFileSystem
+from pyarrow.filesystem import FileSystem, LocalFileSystem, S3FSWrapper
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
                               RowGroupMetaData, ParquetSchema,
                               ParquetWriter)
@@ -645,13 +646,18 @@ def open_file(path, meta=None):
 
 
 def _ensure_filesystem(fs):
-    if not isinstance(fs, FileSystem):
-        if type(fs).__name__ == 'S3FileSystem':
-            from pyarrow.filesystem import S3FSWrapper
-            return S3FSWrapper(fs)
-        else:
-            raise IOError('Unrecognized filesystem: {0}'
-                          .format(type(fs)))
+    fs_type = type(fs)
+
+    # If the arrow filesystem was subclassed, assume it supports the full interface and return it
+    if not issubclass(fs_type, FileSystem):
+        for mro in inspect.getmro(fs_type):
+            if mro.__name__ is 'S3FileSystem':
+                return S3FSWrapper(fs)
+            # In case its a simple LocalFileSystem (e.g. dask) use native arrow FS
+            elif mro.__name__ is 'LocalFileSystem':
+                return LocalFileSystem.get_instance()
+
+        raise IOError('Unrecognized filesystem: {0}'.format(fs_type))
     else:
         return fs
 

From 338a187a96fc58b72a5b8c295f621149fcd76361 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 4 Sep 2017 21:15:52 -0400
Subject: [PATCH 0998/1644] ARROW-1461: [C++] Disable builds using LLVM apt
 repo until installation issues resolved

It looks like llvm.org made some changes to their apt repository in the last 48 hours and this has been causing our builds to fail. We can re-enable builds using .deb packages from apt.llvm.org at a later time

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1038 from wesm/fix-trusty-toolchain and squashes the following commits:

36ee952b [Wes McKinney] Disable clang until LLVM apt stabilizes
ff26ce6e [Wes McKinney] Update LLVM apt repo URL
---
 .travis.yml                      | 6 +++---
 ci/travis_install_clang_tools.sh | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 016d1110720f4..d0ac073813e2e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -95,9 +95,9 @@ matrix:
     env: ARROW_TEST_GROUP=integration
     jdk: openjdk7
     before_script:
-    - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    - export CC="clang-4.0"
-    - export CXX="clang++-4.0"
+    # - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
+    # - export CC="clang-4.0"
+    # - export CXX="clang++-4.0"
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
index bad1e73d24a4a..693329604d3b9 100644
--- a/ci/travis_install_clang_tools.sh
+++ b/ci/travis_install_clang_tools.sh
@@ -19,6 +19,6 @@
 
 wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
 sudo apt-add-repository -y \
-     "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-4.0 main"
+     "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-4.0 main"
 sudo apt-get update
 sudo apt-get install clang-4.0 clang-format-4.0 clang-tidy-4.0

From ab328ec85b275deff9a7a821a1d8f6e9db5d3292 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 5 Sep 2017 12:18:47 -0400
Subject: [PATCH 0999/1644] ARROW-1401: [C++] Add ARROW_EXTRA_ERROR_CONTEXT
 option

This can be enabled in release or debug builds. The result is slightly longer compile times and larger binaries. The resulting error messages are effectively a crude stack trace, see:

```
Out of memory:
/home/wesm/code/arrow/cpp/src/arrow/builder.cc:281 code: Reserve(length)
/home/wesm/code/arrow/cpp/src/arrow/builder.cc:250 code: ArrayBuilder::Resize(capacity)
ArrayBuilder::Resize
```

We may be able to do better with glog, but this should help for debugging

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1042 from wesm/ARROW-1401 and squashes the following commits:

9d6f4d31 [Wes McKinney] Do not call constructor directly
a0731022 [Wes McKinney] Add to Travis CI
ca6291f2 [Wes McKinney] Add ARROW_EXTRA_ERROR_CONTEXT option with file/line numbers/code
---
 ci/travis_before_script_cpp.sh |  3 ++-
 cpp/CMakeLists.txt             |  8 ++++++++
 cpp/src/arrow/status.h         | 22 ++++++++++++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 7418b7614ae34..a613957ba0db6 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -61,7 +61,8 @@ pushd $ARROW_CPP_BUILD_DIR
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
 -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \
--DARROW_NO_DEPRECATED_API=ON"
+-DARROW_NO_DEPRECATED_API=ON \
+-DARROW_EXTRA_ERROR_CONTEXT=ON"
 CMAKE_LINUX_FLAGS=""
 CMAKE_OSX_FLAGS=""
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f25cc622c8872..9f9d71b8747fd 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -94,6 +94,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Exclude deprecated APIs from build"
     OFF)
 
+  option(ARROW_EXTRA_ERROR_CONTEXT
+    "Compile with extra error context (line numbers, code)"
+    OFF)
+
   option(ARROW_IPC
     "Build the Arrow IPC extensions"
     ON)
@@ -221,6 +225,10 @@ if (ARROW_NO_DEPRECATED_API)
   add_definitions(-DARROW_NO_DEPRECATED_API)
 endif()
 
+if (ARROW_EXTRA_ERROR_CONTEXT)
+  add_definitions(-DARROW_EXTRA_ERROR_CONTEXT)
+endif()
+
 include(SetupCxxFlags)
 
 ############################################################
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index ece83ac932576..0cb0fb101811c 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -19,6 +19,10 @@
 #include <iosfwd>
 #include <string>
 
+#ifdef ARROW_EXTRA_ERROR_CONTEXT
+#include <sstream>
+#endif
+
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -45,6 +49,22 @@
 
 namespace arrow {
 
+#ifdef ARROW_EXTRA_ERROR_CONTEXT
+
+#define RETURN_NOT_OK(s)                        \
+  do {                                          \
+    Status _s = (s);                            \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) {        \
+      std::stringstream ss;                     \
+      ss << __FILE__ << ":" << __LINE__         \
+         << " code: " << #s                     \
+         << "\n" << _s.message();               \
+      return Status(_s.code(), ss.str());       \
+    }                                           \
+  } while (0)
+
+#else
+
 #define RETURN_NOT_OK(s)                 \
   do {                                   \
     Status _s = (s);                     \
@@ -53,6 +73,8 @@ namespace arrow {
     }                                    \
   } while (0)
 
+#endif // ARROW_EXTRA_ERROR_CONTEXT
+
 #define RETURN_NOT_OK_ELSE(s, else_) \
   do {                               \
     Status _s = (s);                 \

From df0ca4e69804f371c32701d1de22732bf003413c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 5 Sep 2017 12:22:23 -0400
Subject: [PATCH 1000/1644] ARROW-1461: [C++] Restore LLVM apt usage

This reverts https://github.com/apache/arrow/commit/338a187a96fc58b72a5b8c295f621149fcd76361, since the apt issues seem to be resolved

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1047 from wesm/restore-llvm-apt and squashes the following commits:

60b7d0eb [Wes McKinney] ARROW-1461: [C++] Restore LLVM apt usage
---
 .travis.yml                      | 6 +++---
 ci/travis_install_clang_tools.sh | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d0ac073813e2e..016d1110720f4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -95,9 +95,9 @@ matrix:
     env: ARROW_TEST_GROUP=integration
     jdk: openjdk7
     before_script:
-    # - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    # - export CC="clang-4.0"
-    # - export CXX="clang++-4.0"
+    - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
+    - export CC="clang-4.0"
+    - export CXX="clang++-4.0"
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
index 693329604d3b9..bad1e73d24a4a 100644
--- a/ci/travis_install_clang_tools.sh
+++ b/ci/travis_install_clang_tools.sh
@@ -19,6 +19,6 @@
 
 wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
 sudo apt-add-repository -y \
-     "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-4.0 main"
+     "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-4.0 main"
 sudo apt-get update
 sudo apt-get install clang-4.0 clang-format-4.0 clang-tidy-4.0

From 3283d2a9cfad9fc95674a5f853519caed0cdd5af Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 5 Sep 2017 12:23:21 -0400
Subject: [PATCH 1001/1644] ARROW-1458: [Python] Document that
 create_parents=False is unsupported in HadoopFileSystem

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1041 from wesm/ARROW-1458 and squashes the following commits:

94b016ff [Wes McKinney] Document that create_parents=False is unsupported in HadoopFileSystem
---
 python/pyarrow/hdfs.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/hdfs.py b/python/pyarrow/hdfs.py
index 1591de5cb3d28..2f20be2867cbf 100644
--- a/python/pyarrow/hdfs.py
+++ b/python/pyarrow/hdfs.py
@@ -44,8 +44,19 @@ def isfile(self, path):
     def delete(self, path, recursive=False):
         return super(HadoopFileSystem, self).delete(path, recursive)
 
-    @implements(FileSystem.mkdir)
-    def mkdir(self, path, create_parents=True):
+    def mkdir(self, path, **kwargs):
+        """
+        Create directory in HDFS
+
+        Parameters
+        ----------
+        path : string
+            Directory path to create, including any parent directories
+
+        Notes
+        -----
+        libhdfs does not support create_parents=False, so we ignore this here
+        """
         return super(HadoopFileSystem, self).mkdir(path)
 
     @implements(FileSystem.rename)

From 1ba19c20c518876b641c33ec14894d9079fced10 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 5 Sep 2017 12:29:51 -0400
Subject: [PATCH 1002/1644] ARROW-1401: [C++] Add note to readme about
 ARROW_EXTRA_ERROR_CONTEXT

Change-Id: I1d1dad715e00d45479efce0239e9e1101b540e3b
---
 cpp/README.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/cpp/README.md b/cpp/README.md
index f37ec0368f410..993efb81af7fa 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -172,6 +172,28 @@ constructors, the circumstances where they would are somewhat esoteric, and it
 is likely that an application would have encountered other more serious
 problems prior to having `std::bad_alloc` thrown in a constructor.
 
+### Extra debugging help
+
+If you use the CMake option `-DARROW_EXTRA_ERROR_CONTEXT=ON` it will compile
+the libraries with extra debugging information on error checks inside the
+`RETURN_NOT_OK` macro. In unit tests with `ASSERT_OK`, this will yield error
+outputs like:
+
+
+```
+../src/arrow/ipc/ipc-read-write-test.cc:609: Failure
+Failed
+NotImplemented: ../src/arrow/ipc/ipc-read-write-test.cc:574 code: writer->WriteRecordBatch(batch)
+../src/arrow/ipc/writer.cc:778 code: CheckStarted()
+../src/arrow/ipc/writer.cc:755 code: schema_writer.Write(&dictionaries_)
+../src/arrow/ipc/writer.cc:730 code: WriteSchema()
+../src/arrow/ipc/writer.cc:697 code: WriteSchemaMessage(schema_, dictionary_memo_, &schema_fb)
+../src/arrow/ipc/metadata-internal.cc:651 code: SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema)
+../src/arrow/ipc/metadata-internal.cc:598 code: FieldToFlatbuffer(fbb, *schema.field(i), dictionary_memo, &offset)
+../src/arrow/ipc/metadata-internal.cc:508 code: TypeToFlatbuffer(fbb, *field.type(), &children, &layout, &type_enum, dictionary_memo, &type_offset)
+Unable to convert type: decimal(19, 4)
+```
+
 ### Deprecations and API Changes
 
 We use the compiler definition `ARROW_NO_DEPRECATED_API` to disable APIs that

From bea5ab0372f0ba377e062855e466739fb19b057b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 5 Sep 2017 15:46:32 -0400
Subject: [PATCH 1003/1644] ARROW-1442: [Website] Add note about nightly builds
 to /install

There are conda nightlies available for Linux now. This will help users provide quicker feedback on whether bug fixes solved the problem for them (since installing the nightly is generally easier than building from source)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1050 from wesm/ARROW-1442 and squashes the following commits:

87c88058 [Wes McKinney] Note that nightlies may not be produced on ASF infrastructure
037d5079 [Wes McKinney] Note that conda nightlies are Linux only
f7602781 [Wes McKinney] Add note about nightly builds to /install
---
 site/install.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/site/install.md b/site/install.md
index bfea0b179d00d..6cb80c1336f87 100644
--- a/site/install.md
+++ b/site/install.md
@@ -136,6 +136,19 @@ These repositories are managed at
 [red-data-tools/arrow-packages][9]. If you have any feedback, please
 send it to the project instead of Apache Arrow project.
 
+### Nightly Development Builds
+
+To assist with development and debugging, some nightly builds are
+available. These builds are not releases and not necessarily produced on ASF
+infrastructure. They are to be used strictly for development.
+
+* **conda packages** for C++ and Python (Linux only)
+
+```
+conda install arrow-cpp -c twosigma
+conda install pyarrow -c twosigma
+```
+
 [1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/
 [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.6.0
 [3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/apache-arrow-0.6.0.tar.gz.md5

From 2433289314249ec8ad4b6fb503fde782372ac8df Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 5 Sep 2017 17:21:08 -0400
Subject: [PATCH 1004/1644] ARROW-696: [C++] Support decimals in IPC and JSON
 reader/writer to enable integration tests

Also adds `arrow::decimal` function and renames `decimal` namespace to `DecimalUtil`. The decimal array tests have been consolidated in array-test.cc. Fixed a signedness bug in HexEncode used in JSON write and pretty printing

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1049 from wesm/ARROW-696 and squashes the following commits:

73b767cf [Wes McKinney] Support + test decimals in JSON reader/writer
c8ca538e [Wes McKinney] Implement IPC roundtrip for decimals
9f7cef6b [Wes McKinney] Some decimal API refinements. Add failing IPC test for decimals
---
 cpp/src/arrow/CMakeLists.txt             |   1 -
 cpp/src/arrow/array-decimal-test.cc      | 132 -----------------------
 cpp/src/arrow/array-test.cc              | 110 +++++++++++++++++--
 cpp/src/arrow/array.cc                   |   4 +-
 cpp/src/arrow/builder.cc                 |   2 +-
 cpp/src/arrow/builder.h                  |  11 +-
 cpp/src/arrow/gpu/cuda_context.cc        |   2 +-
 cpp/src/arrow/ipc/ipc-json-test.cc       |   3 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |   2 +-
 cpp/src/arrow/ipc/json-internal.cc       |  42 +++++---
 cpp/src/arrow/ipc/metadata-internal.cc   |  13 ++-
 cpp/src/arrow/ipc/reader.cc              |   2 -
 cpp/src/arrow/ipc/test-common.h          |  35 +++++-
 cpp/src/arrow/ipc/writer.cc              |   1 +
 cpp/src/arrow/pretty_print.cc            |   5 +-
 cpp/src/arrow/python/arrow_to_pandas.cc  |   4 +-
 cpp/src/arrow/python/builtin_convert.cc  |   4 +-
 cpp/src/arrow/python/helpers.cc          |   2 +-
 cpp/src/arrow/python/pandas_to_arrow.cc  |   4 +-
 cpp/src/arrow/status.h                   |  20 ++--
 cpp/src/arrow/type.cc                    |   2 +-
 cpp/src/arrow/type.h                     |  37 +++++--
 cpp/src/arrow/util/bit-util.cc           |   4 +-
 cpp/src/arrow/util/bit-util.h            |   5 +-
 cpp/src/arrow/util/decimal-test.cc       |  20 ++--
 cpp/src/arrow/util/decimal.cc            |   4 +-
 cpp/src/arrow/util/decimal.h             |   6 +-
 cpp/src/arrow/util/int128.cc             |   2 -
 cpp/src/arrow/util/int128.h              |   2 -
 cpp/src/arrow/util/string.h              |   2 +-
 integration/integration_test.py          |   2 +-
 31 files changed, 256 insertions(+), 229 deletions(-)
 delete mode 100644 cpp/src/arrow/array-decimal-test.cc

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 55fab2dd18db3..6963b11e3ffa5 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -50,7 +50,6 @@ install(
 
 ADD_ARROW_TEST(allocator-test)
 ADD_ARROW_TEST(array-test)
-ADD_ARROW_TEST(array-decimal-test)
 ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(memory_pool-test)
 ADD_ARROW_TEST(pretty_print-test)
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
deleted file mode 100644
index 5f3a4f15dfb46..0000000000000
--- a/cpp/src/arrow/array-decimal-test.cc
+++ /dev/null
@@ -1,132 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstddef>
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include <gtest/gtest.h>
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/test-util.h"
-#include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
-
-using std::size_t;
-
-namespace arrow {
-
-class Buffer;
-
-namespace decimal {
-
-using DecimalVector = std::vector<Int128>;
-
-class DecimalTest : public ::testing::TestWithParam<int> {
- public:
-  DecimalTest() {}
-
-  template <size_t BYTE_WIDTH = 16>
-  void MakeData(const DecimalVector& input, std::vector<uint8_t>* out) const {
-    out->reserve(input.size() * BYTE_WIDTH);
-
-    std::array<uint8_t, BYTE_WIDTH> bytes{{0}};
-
-    for (const auto& value : input) {
-      ASSERT_OK(value.ToBytes(&bytes));
-      out->insert(out->end(), bytes.cbegin(), bytes.cend());
-    }
-  }
-
-  template <size_t BYTE_WIDTH = 16>
-  void TestCreate(int precision, const DecimalVector& draw,
-                  const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
-    auto type = std::make_shared<DecimalType>(precision, 4);
-
-    auto builder = std::make_shared<DecimalBuilder>(type);
-
-    size_t null_count = 0;
-
-    const size_t size = draw.size();
-
-    ASSERT_OK(builder->Reserve(size));
-
-    for (size_t i = 0; i < size; ++i) {
-      if (valid_bytes[i]) {
-        ASSERT_OK(builder->Append(draw[i]));
-      } else {
-        ASSERT_OK(builder->AppendNull());
-        ++null_count;
-      }
-    }
-
-    std::shared_ptr<Array> out;
-    ASSERT_OK(builder->Finish(&out));
-
-    std::vector<uint8_t> raw_bytes;
-
-    raw_bytes.reserve(size * BYTE_WIDTH);
-    MakeData<BYTE_WIDTH>(draw, &raw_bytes);
-
-    auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), BYTE_WIDTH);
-    std::shared_ptr<Buffer> expected_null_bitmap;
-    ASSERT_OK(BitUtil::BytesToBits(valid_bytes, &expected_null_bitmap));
-
-    int64_t expected_null_count = test::null_count(valid_bytes);
-    auto expected = std::make_shared<DecimalArray>(
-        type, size, expected_data, expected_null_bitmap, expected_null_count);
-
-    std::shared_ptr<Array> lhs = out->Slice(offset);
-    std::shared_ptr<Array> rhs = expected->Slice(offset);
-    bool result = lhs->Equals(rhs);
-    ASSERT_TRUE(result);
-  }
-};
-
-TEST_P(DecimalTest, NoNulls) {
-  int precision = GetParam();
-  std::vector<Int128> draw = {Int128(1), Int128(-2), Int128(2389), Int128(4),
-                              Int128(-12348)};
-  std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
-  this->TestCreate(precision, draw, valid_bytes, 0);
-  this->TestCreate(precision, draw, valid_bytes, 2);
-}
-
-TEST_P(DecimalTest, WithNulls) {
-  int precision = GetParam();
-  std::vector<Int128> draw = {Int128(1),  Int128(2), Int128(-1), Int128(4),
-                              Int128(-1), Int128(1), Int128(2)};
-  Int128 big;
-  ASSERT_OK(FromString("230342903942.234234", &big));
-  draw.push_back(big);
-
-  Int128 big_negative;
-  ASSERT_OK(FromString("-23049302932.235234", &big_negative));
-  draw.push_back(big_negative);
-
-  std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
-                                      true, true, true,  true};
-  this->TestCreate(precision, draw, valid_bytes, 0);
-  this->TestCreate(precision, draw, valid_bytes, 2);
-}
-
-INSTANTIATE_TEST_CASE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38));
-
-}  // namespace decimal
-}  // namespace arrow
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 3a3d28c777a69..a73a73e12bf05 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -33,6 +33,8 @@
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
+#include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 
 namespace arrow {
 
@@ -69,7 +71,7 @@ Status MakeArrayFromValidBytes(const vector<uint8_t>& v, MemoryPool* pool,
   int64_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
 
   std::shared_ptr<Buffer> null_buf;
-  RETURN_NOT_OK(BitUtil::BytesToBits(v, &null_buf));
+  RETURN_NOT_OK(BitUtil::BytesToBits(v, default_memory_pool(), &null_buf));
 
   TypedBufferBuilder<int32_t> value_builder(pool);
   for (size_t i = 0; i < v.size(); ++i) {
@@ -157,7 +159,7 @@ TEST_F(TestArray, TestIsNull) {
   }
 
   std::shared_ptr<Buffer> null_buf;
-  ASSERT_OK(BitUtil::BytesToBits(null_bitmap, &null_buf));
+  ASSERT_OK(BitUtil::BytesToBits(null_bitmap, default_memory_pool(), &null_buf));
 
   std::unique_ptr<Array> arr;
   arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_buf, null_count));
@@ -236,7 +238,8 @@ class TestPrimitiveBuilder : public TestBuilder {
     int64_t ex_null_count = 0;
 
     if (nullable) {
-      ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &ex_null_bitmap));
+      ASSERT_OK(
+          BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &ex_null_bitmap));
       ex_null_count = test::null_count(valid_bytes_);
     } else {
       ex_null_bitmap = nullptr;
@@ -327,13 +330,13 @@ void TestPrimitiveBuilder<PBoolean>::Check(const std::unique_ptr<BooleanBuilder>
   int64_t size = builder->length();
 
   std::shared_ptr<Buffer> ex_data;
-  ASSERT_OK(BitUtil::BytesToBits(draws_, &ex_data));
+  ASSERT_OK(BitUtil::BytesToBits(draws_, default_memory_pool(), &ex_data));
 
   std::shared_ptr<Buffer> ex_null_bitmap;
   int64_t ex_null_count = 0;
 
   if (nullable) {
-    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &ex_null_bitmap));
+    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &ex_null_bitmap));
     ex_null_count = test::null_count(valid_bytes_);
   } else {
     ex_null_bitmap = nullptr;
@@ -753,7 +756,7 @@ class TestStringArray : public ::testing::Test {
     length_ = static_cast<int64_t>(offsets_.size()) - 1;
     value_buf_ = test::GetBufferFromVector(chars_);
     offsets_buf_ = test::GetBufferFromVector(offsets_);
-    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_));
+    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &null_bitmap_));
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<StringArray>(length_, offsets_buf_, value_buf_,
@@ -967,7 +970,7 @@ class TestBinaryArray : public ::testing::Test {
     value_buf_ = test::GetBufferFromVector(chars_);
     offsets_buf_ = test::GetBufferFromVector(offsets_);
 
-    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_));
+    ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &null_bitmap_));
     null_count_ = test::null_count(valid_bytes_);
 
     strings_ = std::make_shared<BinaryArray>(length_, offsets_buf_, value_buf_,
@@ -2464,4 +2467,97 @@ TEST(TestUnionArrayAdHoc, TestSliceEquals) {
   CheckUnion(batch->column(2));
 }
 
+using DecimalVector = std::vector<Int128>;
+
+class DecimalTest : public ::testing::TestWithParam<int> {
+ public:
+  DecimalTest() {}
+
+  template <size_t BYTE_WIDTH = 16>
+  void MakeData(const DecimalVector& input, std::vector<uint8_t>* out) const {
+    out->reserve(input.size() * BYTE_WIDTH);
+
+    std::array<uint8_t, BYTE_WIDTH> bytes{{0}};
+
+    for (const auto& value : input) {
+      ASSERT_OK(value.ToBytes(&bytes));
+      out->insert(out->end(), bytes.cbegin(), bytes.cend());
+    }
+  }
+
+  template <size_t BYTE_WIDTH = 16>
+  void TestCreate(int32_t precision, const DecimalVector& draw,
+                  const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
+    auto type = std::make_shared<DecimalType>(precision, 4);
+
+    auto builder = std::make_shared<DecimalBuilder>(type);
+
+    size_t null_count = 0;
+
+    const size_t size = draw.size();
+
+    ASSERT_OK(builder->Reserve(size));
+
+    for (size_t i = 0; i < size; ++i) {
+      if (valid_bytes[i]) {
+        ASSERT_OK(builder->Append(draw[i]));
+      } else {
+        ASSERT_OK(builder->AppendNull());
+        ++null_count;
+      }
+    }
+
+    std::shared_ptr<Array> out;
+    ASSERT_OK(builder->Finish(&out));
+
+    std::vector<uint8_t> raw_bytes;
+
+    raw_bytes.reserve(size * BYTE_WIDTH);
+    MakeData<BYTE_WIDTH>(draw, &raw_bytes);
+
+    auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), BYTE_WIDTH);
+    std::shared_ptr<Buffer> expected_null_bitmap;
+    ASSERT_OK(
+        BitUtil::BytesToBits(valid_bytes, default_memory_pool(), &expected_null_bitmap));
+
+    int64_t expected_null_count = test::null_count(valid_bytes);
+    auto expected = std::make_shared<DecimalArray>(
+        type, size, expected_data, expected_null_bitmap, expected_null_count);
+
+    std::shared_ptr<Array> lhs = out->Slice(offset);
+    std::shared_ptr<Array> rhs = expected->Slice(offset);
+    bool result = lhs->Equals(rhs);
+    ASSERT_TRUE(result);
+  }
+};
+
+TEST_P(DecimalTest, NoNulls) {
+  int32_t precision = GetParam();
+  std::vector<Int128> draw = {Int128(1), Int128(-2), Int128(2389), Int128(4),
+                              Int128(-12348)};
+  std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
+}
+
+TEST_P(DecimalTest, WithNulls) {
+  int32_t precision = GetParam();
+  std::vector<Int128> draw = {Int128(1),  Int128(2), Int128(-1), Int128(4),
+                              Int128(-1), Int128(1), Int128(2)};
+  Int128 big;
+  ASSERT_OK(DecimalUtil::FromString("230342903942.234234", &big));
+  draw.push_back(big);
+
+  Int128 big_negative;
+  ASSERT_OK(DecimalUtil::FromString("-23049302932.235234", &big_negative));
+  draw.push_back(big_negative);
+
+  std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
+                                      true, true, true,  true};
+  this->TestCreate(precision, draw, valid_bytes, 0);
+  this->TestCreate(precision, draw, valid_bytes, 2);
+}
+
+INSTANTIATE_TEST_CASE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38));
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index babf326057e63..34f0868e6407b 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -319,8 +319,8 @@ DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data)
 
 std::string DecimalArray::FormatValue(int64_t i) const {
   const auto& type_ = static_cast<const DecimalType&>(*type());
-  decimal::Int128 value(GetValue(i));
-  return decimal::ToString(value, type_.precision(), type_.scale());
+  Int128 value(GetValue(i));
+  return DecimalUtil::ToString(value, type_.precision(), type_.scale());
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 24d26d4386406..0129dc81294ff 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -1032,7 +1032,7 @@ DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>
     : DecimalBuilder(type, pool) {}
 #endif
 
-Status DecimalBuilder::Append(const decimal::Int128& value) {
+Status DecimalBuilder::Append(const Int128& value) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
   std::array<uint8_t, 16> bytes;
   RETURN_NOT_OK(value.ToBytes(&bytes));
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index a649d628ad130..a99e6825fd30e 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -38,6 +38,7 @@
 namespace arrow {
 
 class Array;
+class Int128;
 
 namespace internal {
 
@@ -45,12 +46,6 @@ struct ArrayData;
 
 }  // namespace internal
 
-namespace decimal {
-
-class Int128;
-
-}  // namespace decimal
-
 static constexpr int64_t kMinBuilderCapacity = 1 << 5;
 
 /// Base class for all data array builders.
@@ -747,7 +742,9 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
   explicit DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
 #endif
 
-  Status Append(const decimal::Int128& val);
+  using FixedSizeBinaryBuilder::Append;
+
+  Status Append(const Int128& val);
 
   Status Finish(std::shared_ptr<Array>* out) override;
 };
diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc
index 430ecab6fbe07..42315cd509c3c 100644
--- a/cpp/src/arrow/gpu/cuda_context.cc
+++ b/cpp/src/arrow/gpu/cuda_context.cc
@@ -185,7 +185,7 @@ class CudaDeviceManager::CudaDeviceManagerImpl {
   // device_number -> CudaContext
   std::unordered_map<int, std::shared_ptr<CudaContext>> contexts_;
 
-  int host_bytes_allocated_;
+  int64_t host_bytes_allocated_;
 };
 
 CudaDeviceManager::CudaDeviceManager() { impl_.reset(new CudaDeviceManagerImpl()); }
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 1967e24c4d233..7855aeafeb094 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -376,7 +376,8 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
   ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
                     &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,                  \
                     &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion, &MakeDates,   \
-                    &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDictionary);
+                    &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDecimal,           \
+                    &MakeDictionary);
 
 class TestJsonRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*> {
  public:
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 94d8eda292d87..ac4d36effa2fa 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -121,7 +121,7 @@ TEST_F(TestSchemaMetadata, NestedFields) {
                     &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,                  \
                     &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion,               \
                     &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes,           \
-                    &MakeFWBinary, &MakeBooleanBatch);
+                    &MakeFWBinary, &MakeDecimal, &MakeBooleanBatch);
 
 static int g_file_number = 0;
 
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index f0fb96369f676..025f6c276541e 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -346,6 +346,7 @@ class SchemaWriter {
     return WritePrimitive("fixedsizebinary", type);
   }
 
+  Status Visit(const DecimalType& type) { return WritePrimitive("decimal", type); }
   Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); }
   Status Visit(const IntervalType& type) { return WritePrimitive("interval", type); }
 
@@ -364,8 +365,6 @@ class SchemaWriter {
     return Status::OK();
   }
 
-  Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
-
   Status Visit(const DictionaryType& type) {
     return VisitType(*type.dictionary()->type());
   }
@@ -437,10 +436,11 @@ class ArrayWriter {
   WriteDataValues(const T& arr) {
     for (int64_t i = 0; i < arr.length(); ++i) {
       int32_t length;
-      const char* buf = reinterpret_cast<const char*>(arr.GetValue(i, &length));
+      const uint8_t* buf = arr.GetValue(i, &length);
 
       if (std::is_base_of<StringArray, T>::value) {
-        writer_->String(buf, length);
+        // Presumed UTF-8
+        writer_->String(reinterpret_cast<const char*>(buf), length);
       } else {
         writer_->String(HexEncode(buf, length));
       }
@@ -450,8 +450,9 @@ class ArrayWriter {
   void WriteDataValues(const FixedSizeBinaryArray& arr) {
     int32_t width = arr.byte_width();
     for (int64_t i = 0; i < arr.length(); ++i) {
-      const char* buf = reinterpret_cast<const char*>(arr.GetValue(i));
-      writer_->String(HexEncode(buf, width));
+      const uint8_t* buf = arr.GetValue(i);
+      std::string encoded = HexEncode(buf, width);
+      writer_->String(encoded);
     }
   }
 
@@ -535,8 +536,6 @@ class ArrayWriter {
     return Status::OK();
   }
 
-  Status Visit(const DecimalArray& array) { return Status::NotImplemented("decimal"); }
-
   Status Visit(const DictionaryArray& array) {
     return VisitArrayValues(*array.indices());
   }
@@ -666,6 +665,17 @@ static Status GetFixedSizeBinary(const RjObject& json_type,
   return Status::OK();
 }
 
+static Status GetDecimal(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+  const auto& it_precision = json_type.FindMember("precision");
+  const auto& it_scale = json_type.FindMember("scale");
+
+  RETURN_NOT_INT("precision", it_precision, json_type);
+  RETURN_NOT_INT("scale", it_scale, json_type);
+
+  *type = decimal(it_precision->value.GetInt(), it_scale->value.GetInt());
+  return Status::OK();
+}
+
 static Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type) {
   const auto& it_unit = json_type.FindMember("unit");
   RETURN_NOT_STRING("unit", it_unit, json_type);
@@ -802,6 +812,8 @@ static Status GetType(const RjObject& json_type,
     *type = binary();
   } else if (type_name == "fixedsizebinary") {
     return GetFixedSizeBinary(json_type, type);
+  } else if (type_name == "decimal") {
+    return GetDecimal(json_type, type);
   } else if (type_name == "null") {
     *type = null();
   } else if (type_name == "date") {
@@ -817,8 +829,12 @@ static Status GetType(const RjObject& json_type,
     *type = list(children[0]);
   } else if (type_name == "struct") {
     *type = struct_(children);
-  } else {
+  } else if (type_name == "union") {
     return GetUnion(json_type, children, type);
+  } else {
+    std::stringstream ss;
+    ss << "Unrecognized type name: " << type_name;
+    return Status::Invalid(ss.str());
   }
   return Status::OK();
 }
@@ -1039,7 +1055,7 @@ class ArrayReader {
   template <typename T>
   typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value, Status>::type
   Visit(const T& type) {
-    FixedSizeBinaryBuilder builder(type_, pool_);
+    typename TypeTraits<T>::BuilderType builder(type_, pool_);
 
     const auto& json_data = obj_->FindMember("DATA");
     RETURN_NOT_ARRAY("DATA", json_data, *obj_);
@@ -1063,8 +1079,10 @@ class ArrayReader {
       const rj::Value& val = json_data_arr[i];
       DCHECK(val.IsString());
       std::string hex_string = val.GetString();
-      DCHECK_EQ(static_cast<int32_t>(hex_string.size()), byte_width * 2)
-          << "Expected size: " << byte_width * 2 << " got: " << hex_string.size();
+      if (static_cast<int32_t>(hex_string.size()) != byte_width * 2) {
+        DCHECK(false) << "Expected size: " << byte_width * 2
+                      << " got: " << hex_string.size();
+      }
       const char* hex_data = hex_string.c_str();
 
       for (int32_t j = 0; j < byte_width; ++j) {
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 1f2340dfbe6f0..962fe79d2a3ac 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -248,8 +248,11 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
     case flatbuf::Type_Bool:
       *out = boolean();
       return Status::OK();
-    case flatbuf::Type_Decimal:
-      return Status::NotImplemented("Decimal");
+    case flatbuf::Type_Decimal: {
+      auto dec_type = static_cast<const flatbuf::Decimal*>(type_data);
+      *out = decimal(dec_type->precision(), dec_type->scale());
+      return Status::OK();
+    }
     case flatbuf::Type_Date: {
       auto date_type = static_cast<const flatbuf::Date*>(type_data);
       if (date_type->unit() == flatbuf::DateUnit_DAY) {
@@ -420,6 +423,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const DataType& type,
       }
       *offset = flatbuf::CreateTimestamp(fbb, fb_unit, fb_timezone).Union();
     } break;
+    case Type::DECIMAL: {
+      const auto& dec_type = static_cast<const DecimalType&>(*value_type);
+      *out_type = flatbuf::Type_Decimal;
+      *offset =
+          flatbuf::CreateDecimal(fbb, dec_type.precision(), dec_type.scale()).Union();
+    } break;
     case Type::LIST:
       *out_type = flatbuf::Type_List;
       return ListToFlatbuffer(fbb, *value_type, children, dictionary_memo, offset);
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index a4cac0b802d2c..919dd7a27ff37 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -189,8 +189,6 @@ class ArrayLoader {
 
   Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
 
-  Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
-
   template <typename T>
   typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
                               !std::is_base_of<FixedSizeBinaryType, T>::value &&
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index c1e79d43cc61d..ae36b5ff4ca55 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -161,12 +161,12 @@ Status MakeRandomBooleanArray(const int length, bool include_nulls,
   std::vector<uint8_t> values(length);
   test::random_null_bytes(length, 0.5, values.data());
   std::shared_ptr<Buffer> data;
-  RETURN_NOT_OK(BitUtil::BytesToBits(values, &data));
+  RETURN_NOT_OK(BitUtil::BytesToBits(values, default_memory_pool(), &data));
 
   if (include_nulls) {
     std::vector<uint8_t> valid_bytes(length);
     std::shared_ptr<Buffer> null_bitmap;
-    RETURN_NOT_OK(BitUtil::BytesToBits(valid_bytes, &null_bitmap));
+    RETURN_NOT_OK(BitUtil::BytesToBits(valid_bytes, default_memory_pool(), &null_bitmap));
     test::random_null_bytes(length, 0.1, valid_bytes.data());
     *out = std::make_shared<BooleanArray>(length, data, null_bitmap, -1);
   } else {
@@ -371,7 +371,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
   std::vector<uint8_t> null_bytes(list_batch->num_rows(), 1);
   null_bytes[0] = 0;
   std::shared_ptr<Buffer> null_bitmask;
-  RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask));
+  RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, default_memory_pool(), &null_bitmask));
   std::shared_ptr<Array> with_nulls(
       new StructArray(type, list_batch->num_rows(), columns, null_bitmask, 1));
 
@@ -431,7 +431,7 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
   std::vector<uint8_t> null_bytes(length, 1);
   null_bytes[2] = 0;
   std::shared_ptr<Buffer> null_bitmask;
-  RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask));
+  RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, default_memory_pool(), &null_bitmask));
 
   // construct individual nullable/non-nullable struct arrays
   auto sparse_no_nulls =
@@ -670,6 +670,33 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
+Status MakeDecimal(std::shared_ptr<RecordBatch>* out) {
+  auto f0 = field("f0", decimal(19, 4));
+  auto schema = ::arrow::schema({f0, f0});
+
+  constexpr int kDecimalSize = 16;
+  constexpr int length = 10;
+
+  std::shared_ptr<Buffer> data, is_valid;
+  std::vector<uint8_t> is_valid_bytes(length);
+
+  RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), kDecimalSize * length, &data));
+
+  test::random_bytes(kDecimalSize * length, 0, data->mutable_data());
+  test::random_null_bytes(length, 0.1, is_valid_bytes.data());
+
+  RETURN_NOT_OK(BitUtil::BytesToBits(is_valid_bytes, default_memory_pool(), &is_valid));
+
+  auto a1 = std::make_shared<DecimalArray>(f0->type(), length, data, is_valid,
+                                           kUnknownNullCount);
+
+  auto a2 = std::make_shared<DecimalArray>(f0->type(), length, data);
+
+  ArrayVector arrays = {a1, a2};
+  *out = std::make_shared<RecordBatch>(schema, a1->length(), arrays);
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 7ac63804ec2a9..3f3d168f08435 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -341,6 +341,7 @@ class RecordBatchSerializer : public ArrayVisitor {
   VISIT_FIXED_WIDTH(Time32Array);
   VISIT_FIXED_WIDTH(Time64Array);
   VISIT_FIXED_WIDTH(FixedSizeBinaryArray);
+  VISIT_FIXED_WIDTH(DecimalArray);
 
 #undef VISIT_FIXED_WIDTH
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index aedad1228dfb2..7794f8d1a457a 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -99,7 +99,7 @@ class ArrayPrinter {
       if (array.IsNull(i)) {
         Write("null");
       } else {
-        const char* buf = reinterpret_cast<const char*>(array.GetValue(i, &length));
+        const uint8_t* buf = array.GetValue(i, &length);
         (*sink_) << HexEncode(buf, length);
       }
     }
@@ -116,8 +116,7 @@ class ArrayPrinter {
       if (array.IsNull(i)) {
         Write("null");
       } else {
-        const char* buf = reinterpret_cast<const char*>(array.GetValue(i));
-        (*sink_) << HexEncode(buf, width);
+        (*sink_) << HexEncode(array.GetValue(i), width);
       }
     }
   }
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index d2d6fb4694de5..769f609ac48ea 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -606,8 +606,8 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
 static Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
                                  std::string* result) {
   DCHECK_NE(result, nullptr);
-  decimal::Int128 decimal(bytes);
-  *result = decimal::ToString(decimal, precision, scale);
+  Int128 decimal(bytes);
+  *result = DecimalUtil::ToString(decimal, precision, scale);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index dca3f457c0f8a..eb17f4efee762 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -589,8 +589,8 @@ class DecimalConverter
       std::string string;
       RETURN_NOT_OK(PythonDecimalToString(item.obj(), &string));
 
-      decimal::Int128 value;
-      RETURN_NOT_OK(decimal::FromString(string, &value));
+      Int128 value;
+      RETURN_NOT_OK(DecimalUtil::FromString(string, &value));
       return typed_builder_->Append(value);
     }
 
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 67f0655b41fc6..deda93f3d7612 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -103,7 +103,7 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision,
   auto size = str.size;
 
   std::string c_string(bytes, size);
-  return decimal::FromString(c_string, nullptr, precision, scale);
+  return DecimalUtil::FromString(c_string, nullptr, precision, scale);
 }
 
 Status DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string,
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index fe8ae80d0215b..8410381860b70 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -634,8 +634,8 @@ Status PandasConverter::ConvertDecimals() {
       std::string string;
       RETURN_NOT_OK(PythonDecimalToString(object, &string));
 
-      decimal::Int128 value;
-      RETURN_NOT_OK(decimal::FromString(string, &value));
+      Int128 value;
+      RETURN_NOT_OK(DecimalUtil::FromString(string, &value));
       RETURN_NOT_OK(builder.Append(value));
     } else if (PandasObjectIsNull(object)) {
       RETURN_NOT_OK(builder.AppendNull());
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 0cb0fb101811c..80285dac2b6f0 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -51,16 +51,14 @@ namespace arrow {
 
 #ifdef ARROW_EXTRA_ERROR_CONTEXT
 
-#define RETURN_NOT_OK(s)                        \
-  do {                                          \
-    Status _s = (s);                            \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) {        \
-      std::stringstream ss;                     \
-      ss << __FILE__ << ":" << __LINE__         \
-         << " code: " << #s                     \
-         << "\n" << _s.message();               \
-      return Status(_s.code(), ss.str());       \
-    }                                           \
+#define RETURN_NOT_OK(s)                                                            \
+  do {                                                                              \
+    Status _s = (s);                                                                \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) {                                            \
+      std::stringstream ss;                                                         \
+      ss << __FILE__ << ":" << __LINE__ << " code: " << #s << "\n" << _s.message(); \
+      return Status(_s.code(), ss.str());                                           \
+    }                                                                               \
   } while (0)
 
 #else
@@ -73,7 +71,7 @@ namespace arrow {
     }                                    \
   } while (0)
 
-#endif // ARROW_EXTRA_ERROR_CONTEXT
+#endif  // ARROW_EXTRA_ERROR_CONTEXT
 
 #define RETURN_NOT_OK_ELSE(s, else_) \
   do {                               \
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index b7963b877d6af..87bb73b1e7cc0 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -455,7 +455,7 @@ std::shared_ptr<Field> field(const std::string& name,
   return std::make_shared<Field>(name, type, nullable, metadata);
 }
 
-std::shared_ptr<DataType> decimal(int precision, int scale) {
+std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale) {
   return std::make_shared<DecimalType>(precision, scale);
 }
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 2a89423d16504..d197817b65d99 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -461,19 +461,19 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
  public:
   static constexpr Type::type type_id = Type::DECIMAL;
 
-  explicit DecimalType(int precision, int scale)
+  explicit DecimalType(int32_t precision, int32_t scale)
       : FixedSizeBinaryType(16, Type::DECIMAL), precision_(precision), scale_(scale) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "decimal"; }
 
-  int precision() const { return precision_; }
-  int scale() const { return scale_; }
+  int32_t precision() const { return precision_; }
+  int32_t scale() const { return scale_; }
 
  private:
-  int precision_;
-  int scale_;
+  int32_t precision_;
+  int32_t scale_;
 };
 
 enum class UnionMode : char { SPARSE, DENSE };
@@ -753,14 +753,29 @@ class ARROW_EXPORT Schema {
 // ----------------------------------------------------------------------
 // Factory functions
 
-std::shared_ptr<DataType> ARROW_EXPORT fixed_size_binary(int32_t byte_width);
+/// \brief Make an instance of FixedSizeBinaryType
+ARROW_EXPORT
+std::shared_ptr<DataType> fixed_size_binary(int32_t byte_width);
 
-std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type);
-std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
+/// \brief Make an instance of DecimalType
+ARROW_EXPORT
+std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale);
 
-std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit);
-std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit,
-                                                 const std::string& timezone);
+/// \brief Make an instance of ListType
+ARROW_EXPORT
+std::shared_ptr<DataType> list(const std::shared_ptr<Field>& value_type);
+
+/// \brief Make an instance of ListType
+ARROW_EXPORT
+std::shared_ptr<DataType> list(const std::shared_ptr<DataType>& value_type);
+
+/// \brief Make an instance of TimestampType
+ARROW_EXPORT
+std::shared_ptr<DataType> timestamp(TimeUnit::type unit);
+
+/// \brief Make an instance of TimestampType
+ARROW_EXPORT
+std::shared_ptr<DataType> timestamp(TimeUnit::type unit, const std::string& timezone);
 
 /// \brief Create an instance of 32-bit time type
 /// Unit can be either SECOND or MILLI
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 0984055167978..15bf3595a9602 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -42,12 +42,12 @@ void BitUtil::FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits
   }
 }
 
-Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes,
+Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes, MemoryPool* pool,
                             std::shared_ptr<Buffer>* out) {
   int64_t bit_length = BitUtil::BytesForBits(bytes.size());
 
   std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), bit_length, &buffer));
+  RETURN_NOT_OK(AllocateBuffer(pool, bit_length, &buffer));
 
   memset(buffer->mutable_data(), 0, static_cast<size_t>(bit_length));
   FillBitsFromBytes(bytes, buffer->mutable_data());
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index da2826f405fac..aa83746e1cf88 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -380,7 +380,10 @@ static T ShiftRightLogical(T v, int shift) {
 }
 
 void FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits);
-ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
+
+/// \brief Convert vector of bytes to bitmap buffer
+ARROW_EXPORT
+Status BytesToBits(const std::vector<uint8_t>&, MemoryPool*, std::shared_ptr<Buffer>*);
 
 }  // namespace BitUtil
 
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 516297147f18a..97af0889a4853 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -27,7 +27,7 @@
 #include "arrow/util/int128.h"
 
 namespace arrow {
-namespace decimal {
+namespace DecimalUtil {
 
 class DecimalTestFixture : public ::testing::Test {
  public:
@@ -163,35 +163,35 @@ TEST(DecimalTest, TestDecimalStringAndBytesRoundTrip) {
 TEST(DecimalTest, TestInvalidInputMinus) {
   std::string invalid_value("-");
   Int128 out;
-  Status status = decimal::FromString(invalid_value, &out);
+  Status status = DecimalUtil::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputDot) {
   std::string invalid_value("0.0.0");
   Int128 out;
-  Status status = decimal::FromString(invalid_value, &out);
+  Status status = DecimalUtil::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputEmbeddedMinus) {
   std::string invalid_value("0-13-32");
   Int128 out;
-  Status status = decimal::FromString(invalid_value, &out);
+  Status status = DecimalUtil::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputSingleChar) {
   std::string invalid_value("a");
   Int128 out;
-  Status status = decimal::FromString(invalid_value, &out);
+  Status status = DecimalUtil::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
   std::string invalid_value("-23092.235-");
   Int128 out;
-  Status status = decimal::FromString(invalid_value, &out);
+  Status status = DecimalUtil::FromString(invalid_value, &out);
   auto msg = status.message();
   ASSERT_RAISES(Invalid, status);
 }
@@ -199,21 +199,21 @@ TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
 TEST(DecimalTest, TestInvalidInputWithMinusPlus) {
   std::string invalid_value("-+23092.235");
   Int128 out;
-  Status status = decimal::FromString(invalid_value, &out);
+  Status status = DecimalUtil::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithPlusMinus) {
   std::string invalid_value("+-23092.235");
   Int128 out;
-  Status status = decimal::FromString(invalid_value, &out);
+  Status status = DecimalUtil::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
   std::string invalid_value("00a");
   Int128 out;
-  Status status = decimal::FromString(invalid_value, &out);
+  Status status = DecimalUtil::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
@@ -282,5 +282,5 @@ TEST(Int128TestFalse, ConstructibleFromBool) {
   ASSERT_EQ(0, value.low_bits());
 }
 
-}  // namespace decimal
+}  // namespace DecimalUtil
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index bfd87a54673c7..3e17bf6274f4a 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -26,7 +26,7 @@
 #include "arrow/util/logging.h"
 
 namespace arrow {
-namespace decimal {
+namespace DecimalUtil {
 
 void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
                      Int128* out) {
@@ -206,5 +206,5 @@ std::string ToString(const Int128& decimal_value, int precision, int scale) {
   return str;
 }
 
-}  // namespace decimal
+}  // namespace DecimalUtil
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 4b7cac894e985..2e948774d4761 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -23,10 +23,11 @@
 #include "arrow/status.h"
 
 namespace arrow {
-namespace decimal {
 
 class Int128;
 
+namespace DecimalUtil {
+
 ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
                                   int8_t sign, Int128* out);
 
@@ -35,6 +36,7 @@ ARROW_EXPORT Status FromString(const std::string& string, Int128* out,
 
 ARROW_EXPORT std::string ToString(const Int128& decimal_value, int precision, int scale);
 
-}  // namespace decimal
+}  // namespace DecimalUtil
 }  // namespace arrow
+
 #endif  // ARROW_DECIMAL_H
diff --git a/cpp/src/arrow/util/int128.cc b/cpp/src/arrow/util/int128.cc
index 552c6cce990ec..d5659a8633ca5 100644
--- a/cpp/src/arrow/util/int128.cc
+++ b/cpp/src/arrow/util/int128.cc
@@ -29,7 +29,6 @@
 #include "arrow/util/logging.h"
 
 namespace arrow {
-namespace decimal {
 
 static constexpr uint64_t kIntMask = 0xFFFFFFFF;
 static constexpr auto kCarryBit = static_cast<uint64_t>(1) << static_cast<uint64_t>(32);
@@ -525,5 +524,4 @@ Int128 operator%(const Int128& left, const Int128& right) {
   return remainder;
 }
 
-}  // namespace decimal
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/int128.h b/cpp/src/arrow/util/int128.h
index f6fafd7a3eae3..dd2ec52f23ddd 100644
--- a/cpp/src/arrow/util/int128.h
+++ b/cpp/src/arrow/util/int128.h
@@ -27,7 +27,6 @@
 #include "arrow/util/visibility.h"
 
 namespace arrow {
-namespace decimal {
 
 /// Represents a signed 128-bit integer in two's complement.
 /// Calculations wrap around and overflow is ignored.
@@ -125,7 +124,6 @@ ARROW_EXPORT Int128 operator*(const Int128& left, const Int128& right);
 ARROW_EXPORT Int128 operator/(const Int128& left, const Int128& right);
 ARROW_EXPORT Int128 operator%(const Int128& left, const Int128& right);
 
-}  // namespace decimal
 }  // namespace arrow
 
 #endif  //  ARROW_INT128_H
diff --git a/cpp/src/arrow/util/string.h b/cpp/src/arrow/util/string.h
index 6e70ddcccefec..a2af87caf59be 100644
--- a/cpp/src/arrow/util/string.h
+++ b/cpp/src/arrow/util/string.h
@@ -27,7 +27,7 @@ namespace arrow {
 
 static const char* kAsciiTable = "0123456789ABCDEF";
 
-static inline std::string HexEncode(const char* data, int32_t length) {
+static inline std::string HexEncode(const uint8_t* data, int32_t length) {
   std::string hex_string;
   hex_string.reserve(length * 2);
   for (int32_t j = 0; j < length; ++j) {
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 465394844882f..4c1734501e029 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -896,7 +896,7 @@ class CPPTester(Tester):
 
     EXE_PATH = os.environ.get(
         'ARROW_CPP_EXE_PATH',
-        os.path.join(ARROW_HOME, 'cpp/test-build/debug'))
+        os.path.join(ARROW_HOME, 'cpp/build/debug'))
 
     CPP_INTEGRATION_EXE = os.path.join(EXE_PATH, 'json-integration-test')
     STREAM_TO_FILE = os.path.join(EXE_PATH, 'stream-to-file')

From 2660dda4082a5b74d82d773b1c35feb69ddbf447 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 5 Sep 2017 19:13:06 -0400
Subject: [PATCH 1005/1644] ARROW-1377: [Python] Add ParquetFile.scan_contents
 function to use for benchmarking

Requires PARQUET-1087: https://github.com/apache/parquet-cpp/pull/387

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1039 from wesm/ARROW-1377 and squashes the following commits:

39caa344 [Wes McKinney] Add ParquetFile.scan_contents function to use for benchmarking
---
 python/pyarrow/_parquet.pxd          |  3 +++
 python/pyarrow/_parquet.pyx          | 19 +++++++++++++++++++
 python/pyarrow/parquet.py            | 27 +++++++++++++++++++++++++--
 python/pyarrow/tests/test_parquet.py | 19 +++++++++++++++++++
 4 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index ced654915e57e..5094232bdc7b4 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -231,6 +231,9 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
         CStatus ReadTable(const vector[int]& column_indices,
                           shared_ptr[CTable]* out)
 
+        CStatus ScanContents(vector[int] columns, int32_t column_batch_size,
+                             int64_t* num_rows)
+
         const ParquetFileReader* parquet_reader()
 
         void set_num_threads(int num_threads)
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index f3b7875045904..aea6fb608501e 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -467,6 +467,25 @@ cdef class ParquetReader:
                              .ReadTable(&ctable))
         return pyarrow_wrap_table(ctable)
 
+    def scan_contents(self, column_indices=None, batch_size=65536):
+        cdef:
+            vector[int] c_column_indices
+            int32_t c_batch_size
+            int64_t c_num_rows
+
+        if column_indices is not None:
+            for index in column_indices:
+                c_column_indices.push_back(index)
+
+        c_batch_size = batch_size
+
+        with nogil:
+            check_status(self.reader.get()
+                         .ScanContents(c_column_indices, c_batch_size,
+                                       &c_num_rows))
+
+        return c_num_rows
+
     def column_name_idx(self, column_name):
         """
         Find the matching index of a column in the schema.
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 5dabca982c86c..568aad4318205 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -118,6 +118,27 @@ def read(self, columns=None, nthreads=1, use_pandas_metadata=False):
         return self.reader.read_all(column_indices=column_indices,
                                     nthreads=nthreads)
 
+    def scan_contents(self, columns=None, batch_size=65536):
+        """
+        Read contents of file with a single thread for indicated columns and
+        batch size. Number of rows in file is returned. This function is used
+        for benchmarking
+
+        Parameters
+        ----------
+        columns : list of integers, default None
+            If None, scan all columns
+        batch_size : int, default 64K
+            Number of rows to read at a time internally
+
+        Returns
+        -------
+        num_rows : number of rows in file
+        """
+        column_indices = self._get_column_indices(columns)
+        return self.reader.scan_contents(column_indices,
+                                         batch_size=batch_size)
+
     def _get_column_indices(self, column_names, use_pandas_metadata=False):
         if column_names is None:
             return None
@@ -648,12 +669,14 @@ def open_file(path, meta=None):
 def _ensure_filesystem(fs):
     fs_type = type(fs)
 
-    # If the arrow filesystem was subclassed, assume it supports the full interface and return it
+    # If the arrow filesystem was subclassed, assume it supports the full
+    # interface and return it
     if not issubclass(fs_type, FileSystem):
         for mro in inspect.getmro(fs_type):
             if mro.__name__ is 'S3FileSystem':
                 return S3FSWrapper(fs)
-            # In case its a simple LocalFileSystem (e.g. dask) use native arrow FS
+            # In case its a simple LocalFileSystem (e.g. dask) use native arrow
+            # FS
             elif mro.__name__ is 'LocalFileSystem':
                 return LocalFileSystem.get_instance()
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index de6f431d44dc1..fa9455b6ca3a3 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -675,6 +675,25 @@ def test_read_single_row_group_with_column_subset():
     tm.assert_frame_equal(df[cols], result.to_pandas())
 
 
+@parquet
+def test_scan_contents():
+    import pyarrow.parquet as pq
+
+    N, K = 10000, 4
+    df = alltypes_sample(size=N)
+    a_table = pa.Table.from_pandas(df)
+
+    buf = io.BytesIO()
+    _write_table(a_table, buf, row_group_size=N / K,
+                 compression='snappy', version='2.0')
+
+    buf.seek(0)
+    pf = pq.ParquetFile(buf)
+
+    assert pf.scan_contents() == 10000
+    assert pf.scan_contents(df.columns[:4]) == 10000
+
+
 @parquet
 def test_parquet_piece_read(tmpdir):
     import pyarrow.parquet as pq

From f355354c24d58ea86558e8fc46f33b47d402ac04 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 5 Sep 2017 19:17:19 -0400
Subject: [PATCH 1006/1644] ARROW-1317: [Python] Attempt to set Hadoop
 CLASSPATH when using JNI

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1040 from wesm/ARROW-1317 and squashes the following commits:

274f839c [Wes McKinney] Add note to documentation
1d664b29 [Wes McKinney] If HADOOP_HOME is not set, see if 'hadoop' is in PATH
c6e52d12 [Wes McKinney] Try to set CLASSPATH if HADOOP_HOME is set but classpath is not
---
 python/doc/source/filesystems.rst |  3 +++
 python/pyarrow/hdfs.py            | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/python/doc/source/filesystems.rst b/python/doc/source/filesystems.rst
index c0530f93c2cdd..5c3297bc0689d 100644
--- a/python/doc/source/filesystems.rst
+++ b/python/doc/source/filesystems.rst
@@ -54,6 +54,9 @@ LD_LIBRARY_PATH), and relies on some environment variables.
 
     export CLASSPATH=`$HADOOP_HOME/bin/hdfs classpath --glob`
 
+If ``CLASSPATH`` is not set, then it will be set automatically if the
+``hadoop`` executable is in your system path, or if ``HADOOP_HOME`` is set.
+
 You can also use libhdfs3, a thirdparty C++ library for HDFS from Pivotal Labs:
 
 .. code-block:: python
diff --git a/python/pyarrow/hdfs.py b/python/pyarrow/hdfs.py
index 2f20be2867cbf..3c9d04188a6ca 100644
--- a/python/pyarrow/hdfs.py
+++ b/python/pyarrow/hdfs.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
 import posixpath
 
 from pyarrow.util import implements
@@ -30,6 +31,9 @@ class HadoopFileSystem(lib.HadoopFileSystem, FileSystem):
 
     def __init__(self, host="default", port=0, user=None, kerb_ticket=None,
                  driver='libhdfs'):
+        if driver == 'libhdfs':
+            _maybe_set_hadoop_classpath()
+
         self._connect(host, port, user, kerb_ticket, driver)
 
     @implements(FileSystem.isdir)
@@ -105,6 +109,21 @@ def walk(self, top_path):
                 yield tup
 
 
+def _maybe_set_hadoop_classpath():
+    import subprocess
+
+    if 'hadoop' in os.environ.get('CLASSPATH', ''):
+        return
+
+    if 'HADOOP_HOME' in os.environ:
+        hadoop_bin = '{0}/bin/hadoop'.format(os.environ['HADOOP_HOME'])
+    else:
+        hadoop_bin = 'hadoop'
+
+    classpath = subprocess.check_output([hadoop_bin, 'classpath', '--glob'])
+    os.environ['CLASSPATH'] = classpath.decode('utf-8')
+
+
 def _libhdfs_walk_files_dirs(top_path, contents):
     files = []
     directories = []

From 9634f4bd09791ef369d63626a1184234a199d76d Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 5 Sep 2017 19:19:46 -0400
Subject: [PATCH 1007/1644] ARROW-1462: [GLib] Add GArrowTime32Array and
 GArrowTime64Array

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1045 from kou/glib-support-time-array and squashes the following commits:

86e2fcab [Kouhei Sutou] [GLib] Add GArrowTime32Array and GArrowTime64Array
---
 c_glib/arrow-glib/array-builder.cpp           | 186 +++++++++++++++
 c_glib/arrow-glib/array-builder.h             | 104 ++++++++
 c_glib/arrow-glib/array.cpp                   | 224 ++++++++++++++++++
 c_glib/arrow-glib/array.h                     | 110 +++++++++
 c_glib/arrow-glib/data-type.cpp               | 193 ++++++++++++++-
 c_glib/arrow-glib/data-type.h                 | 134 +++++++++++
 c_glib/arrow-glib/type.cpp                    |  34 +++
 c_glib/arrow-glib/type.h                      |  16 ++
 c_glib/arrow-glib/type.hpp                    |   2 +
 c_glib/test/helper/buildable.rb               |  45 ++--
 ...-data-type.rb => test-date64-data-type.rb} |   0
 c_glib/test/test-time32-array.rb              |  69 ++++++
 c_glib/test/test-time32-data-type.rb          |  51 ++++
 c_glib/test/test-time64-array.rb              |  57 +++++
 c_glib/test/test-time64-data-type.rb          |  51 ++++
 15 files changed, 1254 insertions(+), 22 deletions(-)
 rename c_glib/test/{test-date64-array-data-type.rb => test-date64-data-type.rb} (100%)
 create mode 100644 c_glib/test/test-time32-array.rb
 create mode 100644 c_glib/test/test-time32-data-type.rb
 create mode 100644 c_glib/test/test-time64-array.rb
 create mode 100644 c_glib/test/test-time64-data-type.rb

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 12c50466c5562..9919bfffd2c42 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -113,6 +113,12 @@ G_BEGIN_DECLS
  * #GArrowDate64ArrayBuilder is the class to create a new
  * #GArrowDate64Array.
  *
+ * #GArrowTime32ArrayBuilder is the class to create a new
+ * #GArrowTime32Array.
+ *
+ * #GArrowTime64ArrayBuilder is the class to create a new
+ * #GArrowTime64Array.
+ *
  * #GArrowListArrayBuilder is the class to create a new
  * #GArrowListArray.
  *
@@ -1308,6 +1314,180 @@ garrow_date64_array_builder_append_null(GArrowDate64ArrayBuilder *builder,
 }
 
 
+G_DEFINE_TYPE(GArrowTime32ArrayBuilder,
+              garrow_time32_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_time32_array_builder_init(GArrowTime32ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_time32_array_builder_class_init(GArrowTime32ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_time32_array_builder_new:
+ * @data_type: A #GArrowTime32DataType.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable):
+ *   A newly created #GArrowTime32ArrayBuilder on success, %NULL on error.
+ *
+ * Since: 0.7.0
+ */
+GArrowTime32ArrayBuilder *
+garrow_time32_array_builder_new(GArrowDataType *data_type,
+                                GError **error)
+{
+  if (!GARROW_IS_TIME32_DATA_TYPE(data_type)) {
+    g_set_error(error,
+                GARROW_ERROR,
+                GARROW_ERROR_INVALID,
+                "[time32-array-builder][new] "
+                "data type must be time32 data type: <%s>",
+                G_OBJECT_TYPE_NAME(data_type));
+    return NULL;
+  }
+
+  auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  auto builder = garrow_array_builder_new(arrow_data_type,
+                                          NULL,
+                                          "[time32-array-builder][new]");
+  return GARROW_TIME32_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_time32_array_builder_append:
+ * @builder: A #GArrowTime32ArrayBuilder.
+ * @value: The number of days since UNIX epoch in signed 32bit integer.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_time32_array_builder_append(GArrowTime32ArrayBuilder *builder,
+                                   gint32 value,
+                                   GError **error)
+{
+  return garrow_array_builder_append<arrow::Time32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[time32-array-builder][append]");
+}
+
+/**
+ * garrow_time32_array_builder_append_null:
+ * @builder: A #GArrowTime32ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_time32_array_builder_append_null(GArrowTime32ArrayBuilder *builder,
+                                        GError **error)
+{
+  return garrow_array_builder_append_null<arrow::Time32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[time32-array-builder][append-null]");
+}
+
+
+G_DEFINE_TYPE(GArrowTime64ArrayBuilder,
+              garrow_time64_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_time64_array_builder_init(GArrowTime64ArrayBuilder *builder)
+{
+}
+
+static void
+garrow_time64_array_builder_class_init(GArrowTime64ArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_time64_array_builder_new:
+ * @data_type: A #GArrowTime64DataType.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable):
+ *   A newly created #GArrowTime64ArrayBuilder on success, %NULL on error.
+ *
+ * Since: 0.7.0
+ */
+GArrowTime64ArrayBuilder *
+garrow_time64_array_builder_new(GArrowDataType *data_type,
+                                GError **error)
+{
+  if (!GARROW_IS_TIME64_DATA_TYPE(data_type)) {
+    g_set_error(error,
+                GARROW_ERROR,
+                GARROW_ERROR_INVALID,
+                "[time64-array-builder][new] "
+                "data type must be time64 data type: <%s>",
+                G_OBJECT_TYPE_NAME(data_type));
+    return NULL;
+  }
+
+  auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  auto builder = garrow_array_builder_new(arrow_data_type,
+                                          NULL,
+                                          "[time64-array-builder][new]");
+  return GARROW_TIME64_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_time64_array_builder_append:
+ * @builder: A #GArrowTime64ArrayBuilder.
+ * @value: The number of milliseconds since UNIX epoch in signed 64bit integer.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_time64_array_builder_append(GArrowTime64ArrayBuilder *builder,
+                                   gint64 value,
+                                   GError **error)
+{
+  return garrow_array_builder_append<arrow::Time64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[time64-array-builder][append]");
+}
+
+/**
+ * garrow_time64_array_builder_append_null:
+ * @builder: A #GArrowTime64ArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_time64_array_builder_append_null(GArrowTime64ArrayBuilder *builder,
+                                        GError **error)
+{
+  return garrow_array_builder_append_null<arrow::Time64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[time64-array-builder][append-null]");
+}
+
+
 typedef struct GArrowListArrayBuilderPrivate_ {
   GArrowArrayBuilder *value_builder;
 } GArrowListArrayBuilderPrivate;
@@ -1703,6 +1883,12 @@ garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder,
     case arrow::Type::type::DATE64:
       type = GARROW_TYPE_DATE64_ARRAY_BUILDER;
       break;
+    case arrow::Type::type::TIME32:
+      type = GARROW_TYPE_TIME32_ARRAY_BUILDER;
+      break;
+    case arrow::Type::type::TIME64:
+      type = GARROW_TYPE_TIME64_ARRAY_BUILDER;
+      break;
     case arrow::Type::type::LIST:
       type = GARROW_TYPE_LIST_ARRAY_BUILDER;
       break;
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index b6d79a8b4dd97..981f0bc10c55d 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -884,6 +884,110 @@ gboolean garrow_date64_array_builder_append_null(GArrowDate64ArrayBuilder *build
                                                  GError **error);
 
 
+#define GARROW_TYPE_TIME32_ARRAY_BUILDER        \
+  (garrow_time32_array_builder_get_type())
+#define GARROW_TIME32_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIME32_ARRAY_BUILDER, \
+                              GArrowTime32ArrayBuilder))
+#define GARROW_TIME32_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIME32_ARRAY_BUILDER,    \
+                           GArrowTime32ArrayBuilderClass))
+#define GARROW_IS_TIME32_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_TIME32_ARRAY_BUILDER))
+#define GARROW_IS_TIME32_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIME32_ARRAY_BUILDER))
+#define GARROW_TIME32_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIME32_ARRAY_BUILDER,  \
+                             GArrowTime32ArrayBuilderClass))
+
+typedef struct _GArrowTime32ArrayBuilder         GArrowTime32ArrayBuilder;
+typedef struct _GArrowTime32ArrayBuilderClass    GArrowTime32ArrayBuilderClass;
+
+/**
+ * GArrowTime32ArrayBuilder:
+ *
+ * It wraps `arrow::Time32Builder`.
+ */
+struct _GArrowTime32ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowTime32ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_time32_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowTime32ArrayBuilder *garrow_time32_array_builder_new(GArrowDataType *data_type,
+                                                          GError **error);
+
+gboolean garrow_time32_array_builder_append(GArrowTime32ArrayBuilder *builder,
+                                            gint32 value,
+                                            GError **error);
+gboolean garrow_time32_array_builder_append_null(GArrowTime32ArrayBuilder *builder,
+                                                 GError **error);
+
+
+#define GARROW_TYPE_TIME64_ARRAY_BUILDER        \
+  (garrow_time64_array_builder_get_type())
+#define GARROW_TIME64_ARRAY_BUILDER(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIME64_ARRAY_BUILDER, \
+                              GArrowTime64ArrayBuilder))
+#define GARROW_TIME64_ARRAY_BUILDER_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIME64_ARRAY_BUILDER,    \
+                           GArrowTime64ArrayBuilderClass))
+#define GARROW_IS_TIME64_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_TIME64_ARRAY_BUILDER))
+#define GARROW_IS_TIME64_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIME64_ARRAY_BUILDER))
+#define GARROW_TIME64_ARRAY_BUILDER_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIME64_ARRAY_BUILDER,  \
+                             GArrowTime64ArrayBuilderClass))
+
+typedef struct _GArrowTime64ArrayBuilder         GArrowTime64ArrayBuilder;
+typedef struct _GArrowTime64ArrayBuilderClass    GArrowTime64ArrayBuilderClass;
+
+/**
+ * GArrowTime64ArrayBuilder:
+ *
+ * It wraps `arrow::Time64Builder`.
+ */
+struct _GArrowTime64ArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowTime64ArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_time64_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowTime64ArrayBuilder *garrow_time64_array_builder_new(GArrowDataType *data_type,
+                                                          GError **error);
+
+gboolean garrow_time64_array_builder_append(GArrowTime64ArrayBuilder *builder,
+                                            gint64 value,
+                                            GError **error);
+gboolean garrow_time64_array_builder_append_null(GArrowTime64ArrayBuilder *builder,
+                                                 GError **error);
+
+
 #define GARROW_TYPE_LIST_ARRAY_BUILDER          \
   (garrow_list_array_builder_get_type())
 #define GARROW_LIST_ARRAY_BUILDER(obj)                          \
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index d61790f44110e..17553ec94cf3e 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -60,6 +60,28 @@ garrow_primitive_array_new(gint64 length,
   return garrow_array_new_raw(&arrow_array);
 };
 
+template <typename T>
+GArrowArray *
+garrow_primitive_array_new(GArrowDataType *data_type,
+                           gint64 length,
+                           GArrowBuffer *data,
+                           GArrowBuffer *null_bitmap,
+                           gint64 n_nulls)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  const auto arrow_data = garrow_buffer_get_raw(data);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_specific_array =
+    std::make_shared<typename arrow::TypeTraits<T>::ArrayType>(arrow_data_type,
+                                                               length,
+                                                               arrow_data,
+                                                               arrow_bitmap,
+                                                               n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_specific_array);
+  return garrow_array_new_raw(&arrow_array);
+};
+
 G_BEGIN_DECLS
 
 /**
@@ -156,6 +178,16 @@ G_BEGIN_DECLS
  * more date data. If you don't have Arrow format data, you need to
  * use #GArrowDate64ArrayBuilder to create a new array.
  *
+ * #GArrowTime32Array is a class for the number of seconds or
+ * milliseconds since midnight in 32-bit signed integer array. It can
+ * store zero or more time data. If you don't have Arrow format data,
+ * you need to use #GArrowTime32ArrayBuilder to create a new array.
+ *
+ * #GArrowTime64Array is a class for the number of microseconds or
+ * nanoseconds since midnight in 64-bit signed integer array. It can
+ * store zero or more time data. If you don't have Arrow format data,
+ * you need to use #GArrowTime64ArrayBuilder to create a new array.
+ *
  * #GArrowListArray is a class for list array. It can store zero or
  * more list data. If you don't have Arrow format data, you need to
  * use #GArrowListArrayBuilder to create a new array.
@@ -1672,6 +1704,192 @@ garrow_date64_array_get_values(GArrowDate64Array *array,
 }
 
 
+G_DEFINE_TYPE(GArrowTime32Array,               \
+              garrow_time32_array,             \
+              GARROW_TYPE_PRIMITIVE_ARRAY)
+
+static void
+garrow_time32_array_init(GArrowTime32Array *object)
+{
+}
+
+static void
+garrow_time32_array_class_init(GArrowTime32ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_time32_array_new:
+ * @data_type: The #GArrowTime32DataType.
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable):
+ *   A newly created #GArrowTime32Array on success, %NULL on error.
+ *
+ * Since: 0.7.0
+ */
+GArrowTime32Array *
+garrow_time32_array_new(GArrowDataType *data_type,
+                        gint64 length,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls,
+                        GError **error)
+{
+  if (!GARROW_IS_TIME32_DATA_TYPE(data_type)) {
+    g_set_error(error,
+                GARROW_ERROR,
+                GARROW_ERROR_INVALID,
+                "[time32-array][new] data type must be time32 data type: <%s>",
+                G_OBJECT_TYPE_NAME(data_type));
+    return NULL;
+  }
+
+  auto array = garrow_primitive_array_new<arrow::Time32Type>(data_type,
+                                                             length,
+                                                             data,
+                                                             null_bitmap,
+                                                             n_nulls);
+  return GARROW_TIME32_ARRAY(array);
+}
+
+/**
+ * garrow_time32_array_get_value:
+ * @array: A #GArrowTime32Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ *
+ * Since: 0.7.0
+ */
+gint32
+garrow_time32_array_get_value(GArrowTime32Array *array,
+                              gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Time32Array *>(arrow_array.get())->Value(i);
+}
+
+/**
+ * garrow_time32_array_get_values:
+ * @array: A #GArrowTime32Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ *
+ * Since: 0.7.0
+ */
+const gint32 *
+garrow_time32_array_get_values(GArrowTime32Array *array,
+                               gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return garrow_array_get_values_raw<arrow::Time32Type>(arrow_array, length);
+}
+
+
+G_DEFINE_TYPE(GArrowTime64Array,               \
+              garrow_time64_array,             \
+              GARROW_TYPE_PRIMITIVE_ARRAY)
+
+static void
+garrow_time64_array_init(GArrowTime64Array *object)
+{
+}
+
+static void
+garrow_time64_array_class_init(GArrowTime64ArrayClass *klass)
+{
+}
+
+/**
+ * garrow_time64_array_new:
+ * @data_type: The #GArrowTime64DataType.
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable):
+ *   A newly created #GArrowTime64Array on success, %NULL on error.
+ *
+ * Since: 0.7.0
+ */
+GArrowTime64Array *
+garrow_time64_array_new(GArrowDataType *data_type,
+                        gint64 length,
+                        GArrowBuffer *data,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls,
+                        GError **error)
+{
+  if (!GARROW_IS_TIME64_DATA_TYPE(data_type)) {
+    g_set_error(error,
+                GARROW_ERROR,
+                GARROW_ERROR_INVALID,
+                "[time64-array][new] data type must be time64 data type: <%s>",
+                G_OBJECT_TYPE_NAME(data_type));
+    return NULL;
+  }
+
+  auto array = garrow_primitive_array_new<arrow::Time64Type>(data_type,
+                                                             length,
+                                                             data,
+                                                             null_bitmap,
+                                                             n_nulls);
+  return GARROW_TIME64_ARRAY(array);
+}
+
+/**
+ * garrow_time64_array_get_value:
+ * @array: A #GArrowTime64Array.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ *
+ * Since: 0.7.0
+ */
+gint64
+garrow_time64_array_get_value(GArrowTime64Array *array,
+                              gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::Time64Array *>(arrow_array.get())->Value(i);
+}
+
+/**
+ * garrow_time64_array_get_values:
+ * @array: A #GArrowTime64Array.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ *
+ * Since: 0.7.0
+ */
+const gint64 *
+garrow_time64_array_get_values(GArrowTime64Array *array,
+                               gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto values =
+    garrow_array_get_values_raw<arrow::Time64Type>(arrow_array, length);
+  return reinterpret_cast<const gint64 *>(values);
+}
+
+
 G_DEFINE_TYPE(GArrowListArray,               \
               garrow_list_array,             \
               GARROW_TYPE_ARRAY)
@@ -1917,6 +2135,12 @@ garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array)
   case arrow::Type::type::DATE64:
     type = GARROW_TYPE_DATE64_ARRAY;
     break;
+  case arrow::Type::type::TIME32:
+    type = GARROW_TYPE_TIME32_ARRAY;
+    break;
+  case arrow::Type::type::TIME64:
+    type = GARROW_TYPE_TIME64_ARRAY;
+    break;
   case arrow::Type::type::LIST:
     type = GARROW_TYPE_LIST_ARRAY;
     break;
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 851c8eabf7dee..9aaca472cb930 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -966,6 +966,116 @@ const gint64 *garrow_date64_array_get_values(GArrowDate64Array *array,
                                              gint64 *length);
 
 
+#define GARROW_TYPE_TIME32_ARRAY                \
+  (garrow_time32_array_get_type())
+#define GARROW_TIME32_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_TIME32_ARRAY, \
+                              GArrowTime32Array))
+#define GARROW_TIME32_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_TIME32_ARRAY,    \
+                           GArrowTime32ArrayClass))
+#define GARROW_IS_TIME32_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME32_ARRAY))
+#define GARROW_IS_TIME32_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_TIME32_ARRAY))
+#define GARROW_TIME32_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_TIME32_ARRAY,  \
+                             GArrowTime32ArrayClass))
+
+typedef struct _GArrowTime32Array         GArrowTime32Array;
+typedef struct _GArrowTime32ArrayClass    GArrowTime32ArrayClass;
+
+/**
+ * GArrowTime32Array:
+ *
+ * It wraps `arrow::Time32Array`.
+ */
+struct _GArrowTime32Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowTime32ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_time32_array_get_type(void) G_GNUC_CONST;
+
+GArrowTime32Array *garrow_time32_array_new(GArrowDataType *data_type,
+                                           gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls,
+                                           GError **error);
+
+gint32 garrow_time32_array_get_value(GArrowTime32Array *array,
+                                     gint64 i);
+const gint32 *garrow_time32_array_get_values(GArrowTime32Array *array,
+                                             gint64 *length);
+
+
+#define GARROW_TYPE_TIME64_ARRAY                \
+  (garrow_time64_array_get_type())
+#define GARROW_TIME64_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_TIME64_ARRAY, \
+                              GArrowTime64Array))
+#define GARROW_TIME64_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_TIME64_ARRAY,    \
+                           GArrowTime64ArrayClass))
+#define GARROW_IS_TIME64_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME64_ARRAY))
+#define GARROW_IS_TIME64_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_TIME64_ARRAY))
+#define GARROW_TIME64_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_TIME64_ARRAY,  \
+                             GArrowTime64ArrayClass))
+
+typedef struct _GArrowTime64Array         GArrowTime64Array;
+typedef struct _GArrowTime64ArrayClass    GArrowTime64ArrayClass;
+
+/**
+ * GArrowTime64Array:
+ *
+ * It wraps `arrow::Time64Array`.
+ */
+struct _GArrowTime64Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowTime64ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_time64_array_get_type(void) G_GNUC_CONST;
+
+GArrowTime64Array *garrow_time64_array_new(GArrowDataType *data_type,
+                                           gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls,
+                                           GError **error);
+
+gint64 garrow_time64_array_get_value(GArrowTime64Array *array,
+                                     gint64 i);
+const gint64 *garrow_time64_array_get_values(GArrowTime64Array *array,
+                                             gint64 *length);
+
+
 #define GARROW_TYPE_LIST_ARRAY                  \
   (garrow_list_array_get_type())
 #define GARROW_LIST_ARRAY(obj)                          \
diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/data-type.cpp
index 7cacf6a40492e..c2c74612fb20f 100644
--- a/c_glib/arrow-glib/data-type.cpp
+++ b/c_glib/arrow-glib/data-type.cpp
@@ -22,6 +22,8 @@
 #endif
 
 #include <arrow-glib/data-type.hpp>
+#include <arrow-glib/enums.h>
+#include <arrow-glib/error.hpp>
 #include <arrow-glib/field.hpp>
 #include <arrow-glib/type.hpp>
 
@@ -67,12 +69,18 @@ G_BEGIN_DECLS
  * #GArrowStringDataType is a class for UTF-8 encoded string data
  * type.
  *
- * #GArrowData32DataType is a class for the number of days since UNIX
+ * #GArrowDate32DataType is a class for the number of days since UNIX
  * epoch in 32-bit signed integer data type.
  *
- * #GArrowData64DataType is a class for the number of milliseconds
+ * #GArrowDate64DataType is a class for the number of milliseconds
  * since UNIX epoch in 64-bit signed integer data type.
  *
+ * #GArrowTime32DataType is a class for the number of seconds or
+ * milliseconds since midnight in 32-bit signed integer data type.
+ *
+ * #GArrowTime64DataType is a class for the number of microseconds or
+ * nanoseconds since midnight in 64-bit signed integer data type.
+ *
  * #GArrowListDataType is a class for list data type.
  *
  * #GArrowStructDataType is a class for struct data type.
@@ -677,7 +685,8 @@ garrow_date32_data_type_class_init(GArrowDate32DataTypeClass *klass)
 /**
  * garrow_date32_data_type_new:
  *
- * Returns: The newly created 64-bit floating point data type.
+ * Returns: A newly created the number of milliseconds
+ *   since UNIX epoch in 32-bit signed integer data type.
  *
  * Since: 0.7.0
  */
@@ -711,7 +720,8 @@ garrow_date64_data_type_class_init(GArrowDate64DataTypeClass *klass)
 /**
  * garrow_date64_data_type_new:
  *
- * Returns: The newly created 64-bit floating point data type.
+ * Returns: A newly created the number of milliseconds
+ *   since UNIX epoch in 64-bit signed integer data type.
  *
  * Since: 0.7.0
  */
@@ -728,6 +738,175 @@ garrow_date64_data_type_new(void)
 }
 
 
+G_DEFINE_TYPE(GArrowTimeDataType,               \
+              garrow_time_data_type,            \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_time_data_type_init(GArrowTimeDataType *object)
+{
+}
+
+static void
+garrow_time_data_type_class_init(GArrowTimeDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_time_data_type_get_unit:
+ * @time_data_type: The #GArrowTimeDataType.
+ *
+ * Returns: The unit of the time data type.
+ *
+ * Since: 0.7.0
+ */
+GArrowTimeUnit
+garrow_time_data_type_get_unit(GArrowTimeDataType *time_data_type)
+{
+  const auto arrow_data_type =
+    garrow_data_type_get_raw(GARROW_DATA_TYPE(time_data_type));
+  const auto arrow_time_data_type =
+    std::static_pointer_cast<arrow::TimeType>(arrow_data_type);
+  return garrow_time_unit_from_raw(arrow_time_data_type->unit());
+}
+
+
+G_DEFINE_TYPE(GArrowTime32DataType,                \
+              garrow_time32_data_type,             \
+              GARROW_TYPE_TIME_DATA_TYPE)
+
+static void
+garrow_time32_data_type_init(GArrowTime32DataType *object)
+{
+}
+
+static void
+garrow_time32_data_type_class_init(GArrowTime32DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_time32_data_type_new:
+ * @unit: %GARROW_TIME_UNIT_SECOND or %GARROW_TIME_UNIT_MILLI.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable):
+ *   A newly created the number of seconds or milliseconds since
+ *   midnight in 32-bit signed integer data type.
+ *
+ * Since: 0.7.0
+ */
+GArrowTime32DataType *
+garrow_time32_data_type_new(GArrowTimeUnit unit, GError **error)
+{
+  switch (unit) {
+  case GARROW_TIME_UNIT_SECOND:
+  case GARROW_TIME_UNIT_MILLI:
+    break;
+  default:
+    {
+      auto enum_class = G_ENUM_CLASS(g_type_class_ref(GARROW_TYPE_TIME_UNIT));
+      GEnumValue *value = g_enum_get_value(enum_class, unit);
+      if (value) {
+        g_set_error(error,
+                    GARROW_ERROR,
+                    GARROW_ERROR_INVALID,
+                    "[time32-data-type][new] time unit must be second or milli: "
+                    "<%s>",
+                    value->value_nick);
+      } else {
+        g_set_error(error,
+                    GARROW_ERROR,
+                    GARROW_ERROR_INVALID,
+                    "[time32-data-type][new] "
+                    "time unit must be second(%d) or milli(%d): <%d>",
+                    GARROW_TIME_UNIT_SECOND,
+                    GARROW_TIME_UNIT_MILLI,
+                    unit);
+      }
+      g_type_class_unref(enum_class);
+    }
+    return NULL;
+  }
+
+  auto arrow_unit = garrow_time_unit_to_raw(unit);
+  auto arrow_data_type = arrow::time32(arrow_unit);
+  auto data_type =
+    GARROW_TIME32_DATA_TYPE(g_object_new(GARROW_TYPE_TIME32_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
+G_DEFINE_TYPE(GArrowTime64DataType,                \
+              garrow_time64_data_type,             \
+              GARROW_TYPE_TIME_DATA_TYPE)
+
+static void
+garrow_time64_data_type_init(GArrowTime64DataType *object)
+{
+}
+
+static void
+garrow_time64_data_type_class_init(GArrowTime64DataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_time64_data_type_new:
+ * @unit: %GARROW_TIME_UNIT_SECOND or %GARROW_TIME_UNIT_MILLI.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable):
+ *   A newly created the number of seconds or milliseconds since
+ *   midnight in 64-bit signed integer data type.
+ *
+ * Since: 0.7.0
+ */
+GArrowTime64DataType *
+garrow_time64_data_type_new(GArrowTimeUnit unit, GError **error)
+{
+  switch (unit) {
+  case GARROW_TIME_UNIT_MICRO:
+  case GARROW_TIME_UNIT_NANO:
+    break;
+  default:
+    {
+      auto enum_class = G_ENUM_CLASS(g_type_class_ref(GARROW_TYPE_TIME_UNIT));
+      auto value = g_enum_get_value(enum_class, unit);
+      if (value) {
+        g_set_error(error,
+                    GARROW_ERROR,
+                    GARROW_ERROR_INVALID,
+                    "[time64-data-type][new] time unit must be micro or nano: "
+                    "<%s>",
+                    value->value_nick);
+      } else {
+        g_set_error(error,
+                    GARROW_ERROR,
+                    GARROW_ERROR_INVALID,
+                    "[time64-data-type][new] "
+                    "time unit must be micro(%d) or nano(%d): <%d>",
+                    GARROW_TIME_UNIT_MICRO,
+                    GARROW_TIME_UNIT_NANO,
+                    unit);
+      }
+      g_type_class_unref(enum_class);
+    }
+    return NULL;
+  }
+
+  auto arrow_unit = garrow_time_unit_to_raw(unit);
+  auto arrow_data_type = arrow::time64(arrow_unit);
+  auto data_type =
+    GARROW_TIME64_DATA_TYPE(g_object_new(GARROW_TYPE_TIME64_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+
 G_DEFINE_TYPE(GArrowListDataType,                \
               garrow_list_data_type,             \
               GARROW_TYPE_DATA_TYPE)
@@ -878,6 +1057,12 @@ garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type)
   case arrow::Type::type::DATE64:
     type = GARROW_TYPE_DATE64_DATA_TYPE;
     break;
+  case arrow::Type::type::TIME32:
+    type = GARROW_TYPE_TIME32_DATA_TYPE;
+    break;
+  case arrow::Type::type::TIME64:
+    type = GARROW_TYPE_TIME64_DATA_TYPE;
+    break;
   case arrow::Type::type::LIST:
     type = GARROW_TYPE_LIST_DATA_TYPE;
     break;
diff --git a/c_glib/arrow-glib/data-type.h b/c_glib/arrow-glib/data-type.h
index 5a0d2bc96026d..c01201db1c621 100644
--- a/c_glib/arrow-glib/data-type.h
+++ b/c_glib/arrow-glib/data-type.h
@@ -780,6 +780,140 @@ GType                 garrow_date64_data_type_get_type (void) G_GNUC_CONST;
 GArrowDate64DataType *garrow_date64_data_type_new      (void);
 
 
+#define GARROW_TYPE_TIME_DATA_TYPE              \
+  (garrow_time_data_type_get_type())
+#define GARROW_TIME_DATA_TYPE(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIME_DATA_TYPE,       \
+                              GArrowTimeDataType))
+#define GARROW_TIME_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_TIME_DATA_TYPE,  \
+                           GArrowTimeDataTypeClass))
+#define GARROW_IS_TIME_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME_DATA_TYPE))
+#define GARROW_IS_TIME_DATA_TYPE_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_TIME_DATA_TYPE))
+#define GARROW_TIME_DATA_TYPE_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIME_DATA_TYPE,        \
+                             GArrowTimeDataTypeClass))
+
+typedef struct _GArrowTimeDataType         GArrowTimeDataType;
+typedef struct _GArrowTimeDataTypeClass    GArrowTimeDataTypeClass;
+
+/**
+ * GArrowTimeDataType:
+ *
+ * It wraps `arrow::TimeType`.
+ */
+struct _GArrowTimeDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowTimeDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType          garrow_time_data_type_get_type (void) G_GNUC_CONST;
+GArrowTimeUnit garrow_time_data_type_get_unit (GArrowTimeDataType *time_data_type);
+
+
+#define GARROW_TYPE_TIME32_DATA_TYPE           \
+  (garrow_time32_data_type_get_type())
+#define GARROW_TIME32_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIME32_DATA_TYPE,     \
+                              GArrowTime32DataType))
+#define GARROW_TIME32_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIME32_DATA_TYPE,        \
+                           GArrowTime32DataTypeClass))
+#define GARROW_IS_TIME32_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME32_DATA_TYPE))
+#define GARROW_IS_TIME32_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIME32_DATA_TYPE))
+#define GARROW_TIME32_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIME32_DATA_TYPE,      \
+                             GArrowTime32DataTypeClass))
+
+typedef struct _GArrowTime32DataType         GArrowTime32DataType;
+typedef struct _GArrowTime32DataTypeClass    GArrowTime32DataTypeClass;
+
+/**
+ * GArrowTime32DataType:
+ *
+ * It wraps `arrow::Time32Type`.
+ */
+struct _GArrowTime32DataType
+{
+  /*< private >*/
+  GArrowTimeDataType parent_instance;
+};
+
+struct _GArrowTime32DataTypeClass
+{
+  GArrowTimeDataTypeClass parent_class;
+};
+
+GType                 garrow_time32_data_type_get_type (void) G_GNUC_CONST;
+GArrowTime32DataType *garrow_time32_data_type_new      (GArrowTimeUnit unit,
+                                                        GError **error);
+
+
+#define GARROW_TYPE_TIME64_DATA_TYPE           \
+  (garrow_time64_data_type_get_type())
+#define GARROW_TIME64_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIME64_DATA_TYPE,     \
+                              GArrowTime64DataType))
+#define GARROW_TIME64_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIME64_DATA_TYPE,        \
+                           GArrowTime64DataTypeClass))
+#define GARROW_IS_TIME64_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME64_DATA_TYPE))
+#define GARROW_IS_TIME64_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIME64_DATA_TYPE))
+#define GARROW_TIME64_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIME64_DATA_TYPE,      \
+                             GArrowTime64DataTypeClass))
+
+typedef struct _GArrowTime64DataType         GArrowTime64DataType;
+typedef struct _GArrowTime64DataTypeClass    GArrowTime64DataTypeClass;
+
+/**
+ * GArrowTime64DataType:
+ *
+ * It wraps `arrow::Time64Type`.
+ */
+struct _GArrowTime64DataType
+{
+  /*< private >*/
+  GArrowTimeDataType parent_instance;
+};
+
+struct _GArrowTime64DataTypeClass
+{
+  GArrowTimeDataTypeClass parent_class;
+};
+
+GType                 garrow_time64_data_type_get_type (void) G_GNUC_CONST;
+GArrowTime64DataType *garrow_time64_data_type_new      (GArrowTimeUnit unit,
+                                                        GError **error);
+
+
 #define GARROW_TYPE_LIST_DATA_TYPE              \
   (garrow_list_data_type_get_type())
 #define GARROW_LIST_DATA_TYPE(obj)                              \
diff --git a/c_glib/arrow-glib/type.cpp b/c_glib/arrow-glib/type.cpp
index 8adbaa90a58c7..68ebc965440f1 100644
--- a/c_glib/arrow-glib/type.cpp
+++ b/c_glib/arrow-glib/type.cpp
@@ -92,3 +92,37 @@ garrow_type_from_raw(arrow::Type::type type)
     return GARROW_TYPE_NA;
   }
 }
+
+GArrowTimeUnit
+garrow_time_unit_from_raw(arrow::TimeUnit::type unit)
+{
+  switch (unit) {
+  case arrow::TimeUnit::type::SECOND:
+    return GARROW_TIME_UNIT_SECOND;
+  case arrow::TimeUnit::type::MILLI:
+    return GARROW_TIME_UNIT_MILLI;
+  case arrow::TimeUnit::type::MICRO:
+    return GARROW_TIME_UNIT_MICRO;
+  case arrow::TimeUnit::type::NANO:
+    return GARROW_TIME_UNIT_NANO;
+  default:
+    return GARROW_TIME_UNIT_SECOND;
+  }
+}
+
+arrow::TimeUnit::type
+garrow_time_unit_to_raw(GArrowTimeUnit unit)
+{
+  switch (unit) {
+  case GARROW_TIME_UNIT_SECOND:
+    return arrow::TimeUnit::type::SECOND;
+  case GARROW_TIME_UNIT_MILLI:
+    return arrow::TimeUnit::type::MILLI;
+  case GARROW_TIME_UNIT_MICRO:
+    return arrow::TimeUnit::type::MICRO;
+  case GARROW_TIME_UNIT_NANO:
+    return arrow::TimeUnit::type::NANO;
+  default:
+    return arrow::TimeUnit::type::SECOND;
+  }
+}
diff --git a/c_glib/arrow-glib/type.h b/c_glib/arrow-glib/type.h
index e171aa3220f05..65d28959e8d78 100644
--- a/c_glib/arrow-glib/type.h
+++ b/c_glib/arrow-glib/type.h
@@ -85,4 +85,20 @@ typedef enum {
   GARROW_TYPE_DICTIONARY
 } GArrowType;
 
+/**
+ * GArrowTimeUnit:
+ * @GARROW_TIME_UNIT_SECOND: Second.
+ * @GARROW_TIME_UNIT_MILLISECOND: Millisecond.
+ * @GARROW_TIME_UNIT_MICRO: Microsecond.
+ * @GARROW_TIME_UNIT_NANO: Nanosecond.
+ *
+ * They are corresponding to `arrow::TimeUnit::type` values.
+ */
+typedef enum {
+  GARROW_TIME_UNIT_SECOND,
+  GARROW_TIME_UNIT_MILLI,
+  GARROW_TIME_UNIT_MICRO,
+  GARROW_TIME_UNIT_NANO
+} GArrowTimeUnit;
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/type.hpp b/c_glib/arrow-glib/type.hpp
index 2a452be6dd854..6f6569712b8da 100644
--- a/c_glib/arrow-glib/type.hpp
+++ b/c_glib/arrow-glib/type.hpp
@@ -24,3 +24,5 @@
 #include <arrow-glib/type.h>
 
 GArrowType garrow_type_from_raw(arrow::Type::type type);
+GArrowTimeUnit garrow_time_unit_from_raw(arrow::TimeUnit::type unit);
+arrow::TimeUnit::type garrow_time_unit_to_raw(GArrowTimeUnit unit);
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index d51503ec34679..cb427d7caa6f0 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -18,67 +18,77 @@
 module Helper
   module Buildable
     def build_boolean_array(values)
-      build_array(Arrow::BooleanArrayBuilder, values)
+      build_array(Arrow::BooleanArrayBuilder.new, values)
     end
 
     def build_int_array(values)
-      build_array(Arrow::IntArrayBuilder, values)
+      build_array(Arrow::IntArrayBuilder.new, values)
     end
 
     def build_int8_array(values)
-      build_array(Arrow::Int8ArrayBuilder, values)
+      build_array(Arrow::Int8ArrayBuilder.new, values)
     end
 
     def build_uint8_array(values)
-      build_array(Arrow::UInt8ArrayBuilder, values)
+      build_array(Arrow::UInt8ArrayBuilder.new, values)
     end
 
     def build_int16_array(values)
-      build_array(Arrow::Int16ArrayBuilder, values)
+      build_array(Arrow::Int16ArrayBuilder.new, values)
     end
 
     def build_uint16_array(values)
-      build_array(Arrow::UInt16ArrayBuilder, values)
+      build_array(Arrow::UInt16ArrayBuilder.new, values)
     end
 
     def build_int32_array(values)
-      build_array(Arrow::Int32ArrayBuilder, values)
+      build_array(Arrow::Int32ArrayBuilder.new, values)
     end
 
     def build_uint32_array(values)
-      build_array(Arrow::UInt32ArrayBuilder, values)
+      build_array(Arrow::UInt32ArrayBuilder.new, values)
     end
 
     def build_int64_array(values)
-      build_array(Arrow::Int64ArrayBuilder, values)
+      build_array(Arrow::Int64ArrayBuilder.new, values)
     end
 
     def build_uint64_array(values)
-      build_array(Arrow::UInt64ArrayBuilder, values)
+      build_array(Arrow::UInt64ArrayBuilder.new, values)
     end
 
     def build_float_array(values)
-      build_array(Arrow::FloatArrayBuilder, values)
+      build_array(Arrow::FloatArrayBuilder.new, values)
     end
 
     def build_double_array(values)
-      build_array(Arrow::DoubleArrayBuilder, values)
+      build_array(Arrow::DoubleArrayBuilder.new, values)
     end
 
     def build_date32_array(values)
-      build_array(Arrow::Date32ArrayBuilder, values)
+      build_array(Arrow::Date32ArrayBuilder.new, values)
     end
 
     def build_date64_array(values)
-      build_array(Arrow::Date64ArrayBuilder, values)
+      build_array(Arrow::Date64ArrayBuilder.new, values)
+    end
+
+    def build_time32_array(unit, values)
+      build_array(Arrow::Time32ArrayBuilder.new(Arrow::Time32DataType.new(unit)),
+                  values)
+    end
+
+    def build_time64_array(unit, values)
+      build_array(Arrow::Time64ArrayBuilder.new(Arrow::Time64DataType.new(unit)),
+                  values)
     end
 
     def build_binary_array(values)
-      build_array(Arrow::BinaryArrayBuilder, values)
+      build_array(Arrow::BinaryArrayBuilder.new, values)
     end
 
     def build_string_array(values)
-      build_array(Arrow::StringArrayBuilder, values)
+      build_array(Arrow::StringArrayBuilder.new, values)
     end
 
     def build_list_array(value_data_type, values_list)
@@ -126,8 +136,7 @@ def build_struct_array(fields, structs)
     end
 
     private
-    def build_array(builder_class, values)
-      builder = builder_class.new
+    def build_array(builder, values)
       values.each do |value|
         if value.nil?
           builder.append_null
diff --git a/c_glib/test/test-date64-array-data-type.rb b/c_glib/test/test-date64-data-type.rb
similarity index 100%
rename from c_glib/test/test-date64-array-data-type.rb
rename to c_glib/test/test-date64-data-type.rb
diff --git a/c_glib/test/test-time32-array.rb b/c_glib/test/test-time32-array.rb
new file mode 100644
index 0000000000000..b648c69f8bfd8
--- /dev/null
+++ b/c_glib/test/test-time32-array.rb
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTime32Array < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_new
+    midnight = 0
+    after_midnight = 60 * 10 # 00:10:00
+    raw_data = [midnight, after_midnight]
+    data_type = Arrow::Time32DataType.new(:second)
+    assert_equal(build_time32_array(:second, [*raw_data, nil]),
+                 Arrow::Time32Array.new(data_type,
+                                        3,
+                                        Arrow::Buffer.new(raw_data.pack("l*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
+  def test_buffer
+    midnight = 0
+    after_midnight = 60 * 10 # 00:10:00
+    raw_data = [midnight, after_midnight]
+    array = build_time32_array(:second, raw_data)
+    assert_equal(raw_data.pack("l*"),
+                 array.buffer.data.to_s)
+  end
+
+  def test_value
+    after_midnight = 60 * 10 # 00:10:00
+    array = build_time32_array(:second, [after_midnight])
+    assert_equal(after_midnight, array.get_value(0))
+  end
+
+  def test_values
+    midnight = 0
+    after_midnight = 60 * 10 # 00:10:00
+    raw_data = [midnight, after_midnight]
+    array = build_time32_array(:second, raw_data)
+    assert_equal(raw_data, array.values)
+  end
+
+  sub_test_case("unit") do
+    def test_second
+      array = build_time32_array(:second, [])
+      assert_equal(Arrow::TimeUnit::SECOND, array.value_data_type.unit)
+    end
+
+    def test_milli
+      array = build_time32_array(:milli, [])
+      assert_equal(Arrow::TimeUnit::MILLI, array.value_data_type.unit)
+    end
+  end
+end
diff --git a/c_glib/test/test-time32-data-type.rb b/c_glib/test/test-time32-data-type.rb
new file mode 100644
index 0000000000000..df3b58bd468e3
--- /dev/null
+++ b/c_glib/test/test-time32-data-type.rb
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTime32DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::Time32DataType.new(:second)
+    assert_equal(Arrow::Type::TIME32, data_type.id)
+  end
+
+  def test_invalid_unit
+    message =
+      "[time32-data-type][new] time unit must be second or milli: <micro>"
+    assert_raise(Arrow::Error::Invalid.new(message)) do
+      Arrow::Time32DataType.new(:micro)
+    end
+  end
+
+  sub_test_case("second") do
+    def setup
+      @data_type = Arrow::Time32DataType.new(:second)
+    end
+
+    def test_to_s
+      assert_equal("time32[s]", @data_type.to_s)
+    end
+  end
+
+  sub_test_case("milli") do
+    def setup
+      @data_type = Arrow::Time32DataType.new(:milli)
+    end
+
+    def test_to_s
+      assert_equal("time32[ms]", @data_type.to_s)
+    end
+  end
+end
diff --git a/c_glib/test/test-time64-array.rb b/c_glib/test/test-time64-array.rb
new file mode 100644
index 0000000000000..775d3153af850
--- /dev/null
+++ b/c_glib/test/test-time64-array.rb
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTime64Array < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_new
+    midnight = 0
+    after_midnight = 60 * 10 * 1000 * 1000 # 00:10:00.000000
+    raw_data = [midnight, after_midnight]
+    data_type = Arrow::Time64DataType.new(:micro)
+    assert_equal(build_time64_array(:micro, [*raw_data, nil]),
+                 Arrow::Time64Array.new(data_type,
+                                        3,
+                                        Arrow::Buffer.new(raw_data.pack("q*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+
+  def test_buffer
+    midnight = 0
+    after_midnight = 60 * 10 * 1000 * 1000 # 00:10:00.000000
+    raw_data = [midnight, after_midnight]
+    array = build_time64_array(:micro, raw_data)
+    assert_equal(raw_data.pack("q*"),
+                 array.buffer.data.to_s)
+  end
+
+  def test_value
+    after_midnight = 60 * 10 * 1000 * 1000 # 00:10:00.000000
+    array = build_time64_array(:micro, [after_midnight])
+    assert_equal(after_midnight, array.get_value(0))
+  end
+
+  def test_values
+    midnight = 0
+    after_midnight = 60 * 10 * 1000 * 1000 # 00:10:00.000000
+    raw_data = [midnight, after_midnight]
+    array = build_time64_array(:micro, raw_data)
+    assert_equal(raw_data, array.values)
+  end
+end
diff --git a/c_glib/test/test-time64-data-type.rb b/c_glib/test/test-time64-data-type.rb
new file mode 100644
index 0000000000000..32e3051b84c7e
--- /dev/null
+++ b/c_glib/test/test-time64-data-type.rb
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTime64DataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::Time64DataType.new(:micro)
+    assert_equal(Arrow::Type::TIME64, data_type.id)
+  end
+
+  def test_invalid_unit
+    message =
+      "[time64-data-type][new] time unit must be micro or nano: <second>"
+    assert_raise(Arrow::Error::Invalid.new(message)) do
+      Arrow::Time64DataType.new(:second)
+    end
+  end
+
+  sub_test_case("micro") do
+    def setup
+      @data_type = Arrow::Time64DataType.new(:micro)
+    end
+
+    def test_to_s
+      assert_equal("time64[us]", @data_type.to_s)
+    end
+  end
+
+  sub_test_case("nano") do
+    def setup
+      @data_type = Arrow::Time64DataType.new(:nano)
+    end
+
+    def test_to_s
+      assert_equal("time64[ns]", @data_type.to_s)
+    end
+  end
+end

From e9f3a12da97a09bcde6cb3bf0be74b66db0110af Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Wed, 6 Sep 2017 00:28:53 -0400
Subject: [PATCH 1008/1644] ARROW-695: Add decimal integration test.

Author: Li Jin <ice.xelloss@gmail.com>

Closes #1048 from icexelloss/decimal-integration and squashes the following commits:

f20ef4a6 [Li Jin] Add decimal integration test.
---
 integration/integration_test.py | 68 +++++++++++++++++++++++++++++++--
 1 file changed, 64 insertions(+), 4 deletions(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 4c1734501e029..4bb0a4bead196 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -21,6 +21,7 @@
 import itertools
 import json
 import os
+import random
 import six
 import string
 import subprocess
@@ -165,17 +166,19 @@ def __init__(self, name, count, is_valid, values):
         self.is_valid = is_valid
         self.values = values
 
+    def _encode_value(self, x):
+        return x
+
     def _get_buffers(self):
         return [
             ('VALIDITY', [int(v) for v in self.is_valid]),
-            ('DATA', list(self.values))
+            ('DATA', list([self._encode_value(x) for x in self.values]))
         ]
 
 
 TEST_INT_MIN = - 2**31 + 1
 TEST_INT_MAX = 2**31 - 1
 
-
 class IntegerType(PrimitiveType):
 
     def __init__(self, name, is_signed, bit_width, nullable=True,
@@ -313,8 +316,54 @@ def generate_column(self, size, name=None):
         return PrimitiveColumn(name, size, is_valid, values)
 
 
-class BooleanType(PrimitiveType):
+class DecimalType(PrimitiveType):
+    def __init__(self, name, bit_width, precision, scale, nullable=True):
+        PrimitiveType.__init__(self, name, nullable=True)
+
+        self.bit_width = bit_width
+        self.precision = precision
+        self.scale = scale
+
+    @property
+    def numpy_type(self):
+        return object
+
+    def _get_type(self):
+        return OrderedDict([
+            ('name', 'decimal'),
+            ('precision', self.precision),
+            ('scale', self.scale),
+        ])
+
+    def _get_type_layout(self):
+        return OrderedDict([
+            ('vectors',
+             [OrderedDict([('type', 'VALIDITY'),
+                           ('typeBitWidth', 1)]),
+              OrderedDict([('type', 'DATA'),
+                           ('typeBitWidth', self.bit_width)])])])
+
+    def generate_column(self, size, name=None):
+        values = [random.randint(0, 2**self.bit_width - 1) for x in range(size)]
+
+        is_valid = self._make_is_valid(size)
+        if name is None:
+            name = self.name
+        return DecimalColumn(name, size, is_valid, values, self.bit_width)
+
+
+class DecimalColumn(PrimitiveColumn):
+    def __init__(self, name, count, is_valid, values, bit_width):
+        PrimitiveColumn.__init__(self, name, count, is_valid, values)
+        self.bit_width = bit_width
+        self.hex_width = bit_width / 4
+
+    def _encode_value(self, x):
+        hex_format_str = '%%0%dx' % self.hex_width
+        return (hex_format_str % x).upper()
+
 
+class BooleanType(PrimitiveType):
     bit_width = 1
 
     def _get_type(self):
@@ -441,7 +490,6 @@ class StringColumn(BinaryColumn):
     def _encode_value(self, x):
         return x
 
-
 class ListType(DataType):
 
     def __init__(self, name, value_type, nullable=True):
@@ -691,6 +739,17 @@ def generate_primitive_case(batch_sizes):
     return _generate_file("primitive", fields, batch_sizes)
 
 
+def generate_decimal_case():
+    fields = [
+        DecimalType('f1', 128, 24, 10, True),
+        DecimalType('f2', 128, 32, -10, True)
+    ]
+
+    batch_sizes = [7, 10]
+
+    return _generate_file('decimal', fields, batch_sizes)
+
+
 def generate_datetime_case():
     fields = [
         DateType('f0', DateType.DAY),
@@ -756,6 +815,7 @@ def _temp_path():
     file_objs = [
         generate_primitive_case([7, 10]),
         generate_primitive_case([0, 0, 0]),
+        generate_decimal_case(),
         generate_datetime_case(),
         generate_nested_case(),
         generate_dictionary_case()

From a3514a3889224125bb115a67ff71ab57fd01cfa0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 6 Sep 2017 07:11:43 -0400
Subject: [PATCH 1009/1644] ARROW-1396: [C++] Add PrettyPrint for schemas that
 outputs dictionaries

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1051 from wesm/ARROW-1396 and squashes the following commits:

11a8b690 [Wes McKinney] Fix cpplint error
81637eae [Wes McKinney] clang-format
dfd46924 [Wes McKinney] Use PrettyPrint with schemas in Python
43652bce [Wes McKinney] Add PrettyPrint functions for Schema that also print dictionaries
f097675e [Wes McKinney] First draft of PrettyPrint for schemas, untested. Name DataType::name a virtual
---
 cpp/src/arrow/pretty_print-test.cc   |  40 +++++++-
 cpp/src/arrow/pretty_print.cc        | 148 +++++++++++++++++++++------
 cpp/src/arrow/pretty_print.h         |  21 +++-
 cpp/src/arrow/type.h                 |  79 ++++++++------
 python/pyarrow/includes/libarrow.pxd |   7 ++
 python/pyarrow/tests/test_schema.py  |  20 +++-
 python/pyarrow/types.pxi             |  14 ++-
 7 files changed, 258 insertions(+), 71 deletions(-)

diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index a687a8fc0e703..d4c92ce4caa84 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -55,6 +55,13 @@ void CheckArray(const Array& arr, int indent, const char* expected) {
   ASSERT_EQ(result, ss.str());
 }
 
+template <typename T>
+void Check(const T& obj, const PrettyPrintOptions& options, const char* expected) {
+  std::string result;
+  ASSERT_OK(PrettyPrint(obj, options, &result));
+  ASSERT_EQ(std::string(expected, strlen(expected)), result);
+}
+
 template <typename TYPE, typename C_TYPE>
 void CheckPrimitive(int indent, const std::vector<bool>& is_valid,
                     const std::vector<C_TYPE>& values, const char* expected) {
@@ -117,7 +124,38 @@ TEST_F(TestPrettyPrint, DictionaryType) {
 -- dictionary: ["foo", "bar", "baz"]
 -- indices: [1, 2, null, 0, 2, 0])expected";
 
-  CheckArray(*arr.get(), 0, expected);
+  CheckArray(*arr, 0, expected);
+}
+
+TEST_F(TestPrettyPrint, SchemaWithDictionary) {
+  std::vector<bool> is_valid = {true, true, false, true, true, true};
+
+  std::shared_ptr<Array> dict;
+  std::vector<std::string> dict_values = {"foo", "bar", "baz"};
+  ArrayFromVector<StringType, std::string>(dict_values, &dict);
+
+  auto simple = field("one", int32());
+  auto simple_dict = field("two", dictionary(int16(), dict));
+  auto list_of_dict = field("three", list(simple_dict));
+
+  auto struct_with_dict = field("four", struct_({simple, simple_dict}));
+
+  auto sch = schema({simple, simple_dict, list_of_dict, struct_with_dict});
+
+  static const char* expected = R"expected(one: int32
+two: dictionary<values=string, indices=int16, ordered=0>
+  dictionary: ["foo", "bar", "baz"]
+three: list<two: dictionary<values=string, indices=int16, ordered=0>>
+  child 0, two: dictionary<values=string, indices=int16, ordered=0>
+      dictionary: ["foo", "bar", "baz"]
+four: struct<one: int32, two: dictionary<values=string, indices=int16, ordered=0>>
+  child 0, one: int32
+  child 1, two: dictionary<values=string, indices=int16, ordered=0>
+      dictionary: ["foo", "bar", "baz"])expected";
+
+  PrettyPrintOptions options{0};
+
+  Check(*sch, options, expected);
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 7794f8d1a457a..f759056330792 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -32,10 +32,56 @@
 
 namespace arrow {
 
-class ArrayPrinter {
+class PrettyPrinter {
+ public:
+  PrettyPrinter(int indent, std::ostream* sink) : indent_(indent), sink_(sink) {}
+
+  void Write(const char* data);
+  void Write(const std::string& data);
+  void WriteIndented(const char* data);
+  void WriteIndented(const std::string& data);
+  void Newline();
+  void Indent();
+  void OpenArray();
+  void CloseArray();
+
+ protected:
+  int indent_;
+  std::ostream* sink_;
+};
+
+void PrettyPrinter::OpenArray() { (*sink_) << "["; }
+
+void PrettyPrinter::CloseArray() { (*sink_) << "]"; }
+
+void PrettyPrinter::Write(const char* data) { (*sink_) << data; }
+void PrettyPrinter::Write(const std::string& data) { (*sink_) << data; }
+
+void PrettyPrinter::WriteIndented(const char* data) {
+  Indent();
+  Write(data);
+}
+
+void PrettyPrinter::WriteIndented(const std::string& data) {
+  Indent();
+  Write(data);
+}
+
+void PrettyPrinter::Newline() {
+  (*sink_) << "\n";
+  Indent();
+}
+
+void PrettyPrinter::Indent() {
+  for (int i = 0; i < indent_; ++i) {
+    (*sink_) << " ";
+  }
+}
+
+class ArrayPrinter : public PrettyPrinter {
  public:
   ArrayPrinter(const Array& array, int indent, std::ostream* sink)
-      : array_(array), indent_(indent), sink_(sink) {}
+      : PrettyPrinter(indent, sink), array_(array) {}
 
   template <typename T>
   inline typename std::enable_if<IsInteger<T>::value, void>::type WriteDataValues(
@@ -136,13 +182,6 @@ class ArrayPrinter {
     }
   }
 
-  void Write(const char* data);
-  void Write(const std::string& data);
-  void Newline();
-  void Indent();
-  void OpenArray();
-  void CloseArray();
-
   Status Visit(const NullArray& array) { return Status::OK(); }
 
   template <typename T>
@@ -250,9 +289,6 @@ class ArrayPrinter {
 
  private:
   const Array& array_;
-  int indent_;
-
-  std::ostream* sink_;
 };
 
 Status ArrayPrinter::WriteValidityBitmap(const Array& array) {
@@ -269,24 +305,6 @@ Status ArrayPrinter::WriteValidityBitmap(const Array& array) {
   }
 }
 
-void ArrayPrinter::OpenArray() { (*sink_) << "["; }
-void ArrayPrinter::CloseArray() { (*sink_) << "]"; }
-
-void ArrayPrinter::Write(const char* data) { (*sink_) << data; }
-
-void ArrayPrinter::Write(const std::string& data) { (*sink_) << data; }
-
-void ArrayPrinter::Newline() {
-  (*sink_) << "\n";
-  Indent();
-}
-
-void ArrayPrinter::Indent() {
-  for (int i = 0; i < indent_; ++i) {
-    (*sink_) << " ";
-  }
-}
-
 Status PrettyPrint(const Array& arr, int indent, std::ostream* sink) {
   ArrayPrinter printer(arr, indent, sink);
   return printer.Print();
@@ -302,8 +320,76 @@ Status PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink) {
   return Status::OK();
 }
 
-Status ARROW_EXPORT DebugPrint(const Array& arr, int indent) {
+Status DebugPrint(const Array& arr, int indent) {
   return PrettyPrint(arr, indent, &std::cout);
 }
 
+class SchemaPrinter : public PrettyPrinter {
+ public:
+  SchemaPrinter(const Schema& schema, int indent, std::ostream* sink)
+      : PrettyPrinter(indent, sink), schema_(schema) {}
+
+  Status PrintType(const DataType& type);
+  Status PrintField(const Field& field);
+
+  Status Print() {
+    for (int i = 0; i < schema_.num_fields(); ++i) {
+      if (i > 0) {
+        Newline();
+      }
+      RETURN_NOT_OK(PrintField(*schema_.field(i)));
+    }
+    return Status::OK();
+  }
+
+ private:
+  const Schema& schema_;
+};
+
+Status SchemaPrinter::PrintType(const DataType& type) {
+  Write(type.ToString());
+  if (type.id() == Type::DICTIONARY) {
+    Newline();
+
+    indent_ += 2;
+    WriteIndented("dictionary: ");
+    const auto& dict_type = static_cast<const DictionaryType&>(type);
+    RETURN_NOT_OK(PrettyPrint(*dict_type.dictionary(), indent_, sink_));
+    indent_ -= 2;
+  } else {
+    for (int i = 0; i < type.num_children(); ++i) {
+      Newline();
+
+      std::stringstream ss;
+      ss << "child " << i << ", ";
+
+      indent_ += 2;
+      WriteIndented(ss.str());
+      RETURN_NOT_OK(PrintField(*type.child(i)));
+      indent_ -= 2;
+    }
+  }
+  return Status::OK();
+}
+
+Status SchemaPrinter::PrintField(const Field& field) {
+  Write(field.name());
+  Write(": ");
+  return PrintType(*field.type());
+}
+
+Status PrettyPrint(const Schema& schema, const PrettyPrintOptions& options,
+                   std::ostream* sink) {
+  SchemaPrinter printer(schema, options.indent, sink);
+  return printer.Print();
+}
+
+Status PrettyPrint(const Schema& schema, const PrettyPrintOptions& options,
+                   std::string* result) {
+  std::ostringstream sink;
+  RETURN_NOT_OK(PrettyPrint(schema, options, &sink));
+  *result = sink.str();
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/pretty_print.h b/cpp/src/arrow/pretty_print.h
index a45c8a8275288..e3aa020c49707 100644
--- a/cpp/src/arrow/pretty_print.h
+++ b/cpp/src/arrow/pretty_print.h
@@ -19,6 +19,7 @@
 #define ARROW_PRETTY_PRINT_H
 
 #include <ostream>
+#include <string>
 
 #include "arrow/type_fwd.h"
 #include "arrow/util/visibility.h"
@@ -32,10 +33,24 @@ struct PrettyPrintOptions {
   int indent;
 };
 
-Status ARROW_EXPORT PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink);
-Status ARROW_EXPORT PrettyPrint(const Array& arr, int indent, std::ostream* sink);
+/// \brief Print human-readable representation of RecordBatch
+ARROW_EXPORT
+Status PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink);
 
-Status ARROW_EXPORT DebugPrint(const Array& arr, int indent);
+/// \brief Print human-readable representation of Array
+ARROW_EXPORT
+Status PrettyPrint(const Array& arr, int indent, std::ostream* sink);
+
+ARROW_EXPORT
+Status PrettyPrint(const Schema& schema, const PrettyPrintOptions& options,
+                   std::ostream* sink);
+
+ARROW_EXPORT
+Status PrettyPrint(const Schema& schema, const PrettyPrintOptions& options,
+                   std::string* result);
+
+ARROW_EXPORT
+Status DebugPrint(const Array& arr, int indent);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index d197817b65d99..b532cd2276531 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -148,8 +148,15 @@ class ARROW_EXPORT DataType {
 
   virtual Status Accept(TypeVisitor* visitor) const = 0;
 
+  /// \brief A string representation of the type, including any children
   virtual std::string ToString() const = 0;
 
+  /// \brief A string name of the type, omitting any child fields
+  ///
+  /// \note Experimental API
+  /// \since 0.7.0
+  virtual std::string name() const = 0;
+
   virtual std::vector<BufferDescr> GetBufferLayout() const = 0;
 
   Type::type id() const { return id_; }
@@ -195,7 +202,6 @@ class ARROW_EXPORT FloatingPoint : public PrimitiveCType {
 class ARROW_EXPORT NestedType : public DataType {
  public:
   using DataType::DataType;
-  static std::string name() { return "nested"; }
 };
 
 class NoExtraMeta {};
@@ -258,7 +264,7 @@ class ARROW_EXPORT CTypeImpl : public BASE {
     return visitor->Visit(*static_cast<const DERIVED*>(this));
   }
 
-  std::string ToString() const override { return std::string(DERIVED::name()); }
+  std::string ToString() const override { return this->name(); }
 };
 
 template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
@@ -277,7 +283,7 @@ class ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
 
-  static std::string name() { return "null"; }
+  std::string name() const override { return "null"; }
 
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
@@ -292,76 +298,76 @@ class ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
   std::string ToString() const override;
 
   int bit_width() const override { return 1; }
-  static std::string name() { return "bool"; }
+  std::string name() const override { return "bool"; }
 };
 
 class ARROW_EXPORT UInt8Type
     : public detail::IntegerTypeImpl<UInt8Type, Type::UINT8, uint8_t> {
  public:
-  static std::string name() { return "uint8"; }
+  std::string name() const override { return "uint8"; }
 };
 
 class ARROW_EXPORT Int8Type
     : public detail::IntegerTypeImpl<Int8Type, Type::INT8, int8_t> {
  public:
-  static std::string name() { return "int8"; }
+  std::string name() const override { return "int8"; }
 };
 
 class ARROW_EXPORT UInt16Type
     : public detail::IntegerTypeImpl<UInt16Type, Type::UINT16, uint16_t> {
  public:
-  static std::string name() { return "uint16"; }
+  std::string name() const override { return "uint16"; }
 };
 
 class ARROW_EXPORT Int16Type
     : public detail::IntegerTypeImpl<Int16Type, Type::INT16, int16_t> {
  public:
-  static std::string name() { return "int16"; }
+  std::string name() const override { return "int16"; }
 };
 
 class ARROW_EXPORT UInt32Type
     : public detail::IntegerTypeImpl<UInt32Type, Type::UINT32, uint32_t> {
  public:
-  static std::string name() { return "uint32"; }
+  std::string name() const override { return "uint32"; }
 };
 
 class ARROW_EXPORT Int32Type
     : public detail::IntegerTypeImpl<Int32Type, Type::INT32, int32_t> {
  public:
-  static std::string name() { return "int32"; }
+  std::string name() const override { return "int32"; }
 };
 
 class ARROW_EXPORT UInt64Type
     : public detail::IntegerTypeImpl<UInt64Type, Type::UINT64, uint64_t> {
  public:
-  static std::string name() { return "uint64"; }
+  std::string name() const override { return "uint64"; }
 };
 
 class ARROW_EXPORT Int64Type
     : public detail::IntegerTypeImpl<Int64Type, Type::INT64, int64_t> {
  public:
-  static std::string name() { return "int64"; }
+  std::string name() const override { return "int64"; }
 };
 
 class ARROW_EXPORT HalfFloatType
     : public detail::CTypeImpl<HalfFloatType, FloatingPoint, Type::HALF_FLOAT, uint16_t> {
  public:
   Precision precision() const override;
-  static std::string name() { return "halffloat"; }
+  std::string name() const override { return "halffloat"; }
 };
 
 class ARROW_EXPORT FloatType
     : public detail::CTypeImpl<FloatType, FloatingPoint, Type::FLOAT, float> {
  public:
   Precision precision() const override;
-  static std::string name() { return "float"; }
+  std::string name() const override { return "float"; }
 };
 
 class ARROW_EXPORT DoubleType
     : public detail::CTypeImpl<DoubleType, FloatingPoint, Type::DOUBLE, double> {
  public:
   Precision precision() const override;
-  static std::string name() { return "double"; }
+  std::string name() const override { return "double"; }
 };
 
 class ARROW_EXPORT ListType : public NestedType {
@@ -383,7 +389,7 @@ class ARROW_EXPORT ListType : public NestedType {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
 
-  static std::string name() { return "list"; }
+  std::string name() const override { return "list"; }
 
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
@@ -397,7 +403,7 @@ class ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "binary"; }
+  std::string name() const override { return "binary"; }
 
   std::vector<BufferDescr> GetBufferLayout() const override;
 
@@ -418,7 +424,7 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "fixed_size_binary"; }
+  std::string name() const override { return "fixed_size_binary"; }
 
   std::vector<BufferDescr> GetBufferLayout() const override;
 
@@ -438,7 +444,7 @@ class ARROW_EXPORT StringType : public BinaryType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "utf8"; }
+  std::string name() const override { return "utf8"; }
 };
 
 class ARROW_EXPORT StructType : public NestedType {
@@ -452,7 +458,7 @@ class ARROW_EXPORT StructType : public NestedType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "struct"; }
+  std::string name() const override { return "struct"; }
 
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
@@ -466,7 +472,7 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "decimal"; }
+  std::string name() const override { return "decimal"; }
 
   int32_t precision() const { return precision_; }
   int32_t scale() const { return scale_; }
@@ -486,7 +492,7 @@ class ARROW_EXPORT UnionType : public NestedType {
             const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
 
   std::string ToString() const override;
-  static std::string name() { return "union"; }
+  std::string name() const override { return "union"; }
   Status Accept(TypeVisitor* visitor) const override;
 
   std::vector<BufferDescr> GetBufferLayout() const override;
@@ -531,6 +537,8 @@ class ARROW_EXPORT Date32Type : public DateType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+
+  std::string name() const override { return "date32"; }
 };
 
 /// Date as int64_t milliseconds since UNIX epoch
@@ -546,7 +554,8 @@ class ARROW_EXPORT Date64Type : public DateType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "date"; }
+
+  std::string name() const override { return "date64"; }
 };
 
 struct TimeUnit {
@@ -591,6 +600,8 @@ class ARROW_EXPORT Time32Type : public TimeType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+
+  std::string name() const override { return "time32"; }
 };
 
 class ARROW_EXPORT Time64Type : public TimeType {
@@ -604,6 +615,8 @@ class ARROW_EXPORT Time64Type : public TimeType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
+
+  std::string name() const override { return "time64"; }
 };
 
 class ARROW_EXPORT TimestampType : public FixedWidthType {
@@ -623,7 +636,7 @@ class ARROW_EXPORT TimestampType : public FixedWidthType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "timestamp"; }
+  std::string name() const override { return "timestamp"; }
 
   TimeUnit::type unit() const { return unit_; }
   const std::string& timezone() const { return timezone_; }
@@ -647,7 +660,7 @@ class ARROW_EXPORT IntervalType : public FixedWidthType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override { return name(); }
-  static std::string name() { return "date"; }
+  std::string name() const override { return "date"; }
 
   Unit unit() const { return unit_; }
 
@@ -673,7 +686,7 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
-  static std::string name() { return "dictionary"; }
+  std::string name() const override { return "dictionary"; }
 
   bool ordered() const { return ordered_; }
 
@@ -814,18 +827,20 @@ std::shared_ptr<Field> ARROW_EXPORT field(
 /// \param fields the schema's fields
 /// \param metadata any custom key-value metadata, default nullptr
 /// \return schema shared_ptr to Schema
-std::shared_ptr<Schema> ARROW_EXPORT
-schema(const std::vector<std::shared_ptr<Field>>& fields,
-       const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+ARROW_EXPORT
+std::shared_ptr<Schema> schema(
+    const std::vector<std::shared_ptr<Field>>& fields,
+    const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
 
 /// \brief Create a Schema instance
 ///
 /// \param fields the schema's fields (rvalue reference)
 /// \param metadata any custom key-value metadata, default nullptr
 /// \return schema shared_ptr to Schema
-std::shared_ptr<Schema> ARROW_EXPORT
-schema(std::vector<std::shared_ptr<Field>>&& fields,
-       const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+ARROW_EXPORT
+std::shared_ptr<Schema> schema(
+    std::vector<std::shared_ptr<Field>>&& fields,
+    const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
 
 // ----------------------------------------------------------------------
 //
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index e032448af9774..6bb8702ad2a2e 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -242,6 +242,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
             const shared_ptr[CKeyValueMetadata]& metadata)
         shared_ptr[CSchema] RemoveMetadata()
 
+    cdef cppclass PrettyPrintOptions:
+        int indent;
+
+    CStatus PrettyPrint(const CSchema& schema,
+                        const PrettyPrintOptions& options,
+                        c_string* result)
+
     cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray):
         c_bool Value(int i)
 
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index f920e8d703bad..4bb6a5af7dccb 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -147,7 +147,8 @@ def test_schema():
     assert repr(sch) == """\
 foo: int32
 bar: string
-baz: list<item: int8>"""
+baz: list<item: int8>
+  child 0, item: int8"""
 
 
 def test_field_empty():
@@ -236,3 +237,20 @@ def test_schema_negative_indexing():
 
     with pytest.raises(IndexError):
         schema[3]
+
+
+def test_schema_repr_with_dictionaries():
+    dct = pa.array(['foo', 'bar', 'baz'], type=pa.string())
+    fields = [
+        pa.field('one', pa.dictionary(pa.int16(), dct)),
+        pa.field('two', pa.int32())
+    ]
+    sch = pa.schema(fields)
+
+    expected = (
+        """\
+one: dictionary<values=string, indices=int16, ordered=0>
+  dictionary: ["foo", "bar", "baz"]
+two: int32""")
+
+    assert repr(sch) == expected
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 30c3aa6bc7d4f..fb6b9618d98df 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -423,7 +423,15 @@ cdef class Schema:
         return pyarrow_wrap_schema(new_schema)
 
     def __str__(self):
-        return frombytes(self.schema.ToString())
+        cdef:
+            PrettyPrintOptions options
+            c_string result
+
+        options.indent = 0
+        with nogil:
+            check_status(PrettyPrint(deref(self.schema), options, &result))
+
+        return frombytes(result)
 
     def __repr__(self):
         return self.__str__()
@@ -835,7 +843,7 @@ cpdef ListType list_(value_type):
     return out
 
 
-cpdef DictionaryType dictionary(DataType index_type, Array dictionary,
+cpdef DictionaryType dictionary(DataType index_type, Array dict_values,
                                 bint ordered=False):
     """
     Dictionary (categorical, or simply encoded) type
@@ -852,7 +860,7 @@ cpdef DictionaryType dictionary(DataType index_type, Array dictionary,
     cdef DictionaryType out = DictionaryType()
     cdef shared_ptr[CDataType] dict_type
     dict_type.reset(new CDictionaryType(index_type.sp_type,
-                                        dictionary.sp_array,
+                                        dict_values.sp_array,
                                         ordered == 1))
     out.init(dict_type)
     return out

From 5abd12321e5730ffd8e16207d86c479f5eb362a5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 6 Sep 2017 07:36:09 -0400
Subject: [PATCH 1010/1644] ARROW-1468: [C++] Add primitive Append variants
 that accept std::vector<T>

Having to always unbox std::vector in user code is a rough edge in practice -- this makes things simpler in third party applications.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1053 from wesm/ARROW-1468 and squashes the following commits:

2546d8fe [Wes McKinney] Add primitive Append variants that accept std::vector<T>
---
 cpp/src/arrow/array-test.cc | 19 +++++++++++++++++--
 cpp/src/arrow/builder.cc    | 34 ++++++++++++++++++++++++++++++++++
 cpp/src/arrow/builder.h     | 29 +++++++++++++++++++++++++++++
 3 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index a73a73e12bf05..5d9eb187e747c 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -614,22 +614,32 @@ TYPED_TEST(TestPrimitiveBuilder, TestAppendVectorStdBool) {
     is_valid.push_back(this->valid_bytes_[i] != 0);
   }
   ASSERT_OK(this->builder_->Append(draws.data(), K, is_valid));
+  ASSERT_OK(this->builder_nn_->Append(draws.data(), K));
 
   ASSERT_EQ(1000, this->builder_->length());
   ASSERT_EQ(1024, this->builder_->capacity());
+  ASSERT_EQ(1000, this->builder_nn_->length());
+  ASSERT_EQ(1024, this->builder_nn_->capacity());
 
   // Append the next 9000
   is_valid.clear();
+  std::vector<T> partial_draws;
   for (int64_t i = K; i < size; ++i) {
+    partial_draws.push_back(draws[i]);
     is_valid.push_back(this->valid_bytes_[i] != 0);
   }
 
-  ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, is_valid));
+  ASSERT_OK(this->builder_->Append(partial_draws, is_valid));
+  ASSERT_OK(this->builder_nn_->Append(partial_draws));
 
   ASSERT_EQ(size, this->builder_->length());
   ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
 
+  ASSERT_EQ(size, this->builder_nn_->length());
+  ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
+
   this->Check(this->builder_, true);
+  this->Check(this->builder_nn_, false);
 }
 
 TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
@@ -671,6 +681,7 @@ TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
 
 TEST(TestBooleanBuilder, TestStdBoolVectorAppend) {
   BooleanBuilder builder;
+  BooleanBuilder builder_nn;
 
   std::vector<bool> values, is_valid;
 
@@ -686,12 +697,15 @@ TEST(TestBooleanBuilder, TestStdBoolVectorAppend) {
       chunk_is_valid.push_back(is_valid[i]);
     }
     ASSERT_OK(builder.Append(chunk_values, chunk_is_valid));
+    ASSERT_OK(builder_nn.Append(chunk_values));
   }
 
-  std::shared_ptr<Array> result;
+  std::shared_ptr<Array> result, result_nn;
   ASSERT_OK(builder.Finish(&result));
+  ASSERT_OK(builder_nn.Finish(&result_nn));
 
   const auto& arr = static_cast<const BooleanArray&>(*result);
+  const auto& arr_nn = static_cast<const BooleanArray&>(*result_nn);
   for (int i = 0; i < length; ++i) {
     if (is_valid[i]) {
       ASSERT_FALSE(arr.IsNull(i));
@@ -699,6 +713,7 @@ TEST(TestBooleanBuilder, TestStdBoolVectorAppend) {
     } else {
       ASSERT_TRUE(arr.IsNull(i));
     }
+    ASSERT_EQ(values[i], arr_nn.Value(i));
   }
 }
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 0129dc81294ff..796624169735c 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -291,6 +291,17 @@ Status PrimitiveBuilder<T>::Append(const value_type* values, int64_t length,
   return Status::OK();
 }
 
+template <typename T>
+Status PrimitiveBuilder<T>::Append(const std::vector<value_type>& values,
+                                   const std::vector<bool>& is_valid) {
+  return Append(values.data(), static_cast<int64_t>(values.size()), is_valid);
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Append(const std::vector<value_type>& values) {
+  return Append(values.data(), static_cast<int64_t>(values.size()));
+}
+
 template <typename T>
 Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
   const int64_t bytes_required = TypeTraits<T>::bytes_required(length_);
@@ -750,6 +761,7 @@ Status BooleanBuilder::Append(const uint8_t* values, int64_t length,
 Status BooleanBuilder::Append(const uint8_t* values, int64_t length,
                               const std::vector<bool>& is_valid) {
   RETURN_NOT_OK(Reserve(length));
+  DCHECK_EQ(length, static_cast<int64_t>(is_valid.size()));
 
   for (int64_t i = 0; i < length; ++i) {
     BitUtil::SetBitTo(raw_data_, length_ + i, values[i] != 0);
@@ -760,10 +772,20 @@ Status BooleanBuilder::Append(const uint8_t* values, int64_t length,
   return Status::OK();
 }
 
+Status BooleanBuilder::Append(const std::vector<uint8_t>& values,
+                              const std::vector<bool>& is_valid) {
+  return Append(values.data(), static_cast<int64_t>(values.size()), is_valid);
+}
+
+Status BooleanBuilder::Append(const std::vector<uint8_t>& values) {
+  return Append(values.data(), static_cast<int64_t>(values.size()));
+}
+
 Status BooleanBuilder::Append(const std::vector<bool>& values,
                               const std::vector<bool>& is_valid) {
   const int64_t length = static_cast<int64_t>(values.size());
   RETURN_NOT_OK(Reserve(length));
+  DCHECK_EQ(length, static_cast<int64_t>(is_valid.size()));
 
   for (int64_t i = 0; i < length; ++i) {
     BitUtil::SetBitTo(raw_data_, length_ + i, values[i]);
@@ -774,6 +796,18 @@ Status BooleanBuilder::Append(const std::vector<bool>& values,
   return Status::OK();
 }
 
+Status BooleanBuilder::Append(const std::vector<bool>& values) {
+  const int64_t length = static_cast<int64_t>(values.size());
+  RETURN_NOT_OK(Reserve(length));
+
+  for (int64_t i = 0; i < length; ++i) {
+    BitUtil::SetBitTo(raw_data_, length_ + i, values[i]);
+  }
+
+  ArrayBuilder::UnsafeSetNotNull(length);
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // DictionaryBuilder
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index a99e6825fd30e..3e8289f049650 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -220,6 +220,18 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   Status Append(const value_type* values, int64_t length,
                 const std::vector<bool>& is_valid);
 
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a std::vector of values
+  /// \param[in] is_valid an std::vector<bool> indicating valid (1) or null
+  /// (0). Equal in length to values
+  /// \return Status
+  Status Append(const std::vector<value_type>& values, const std::vector<bool>& is_valid);
+
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a std::vector of values
+  /// \return Status
+  Status Append(const std::vector<value_type>& values);
+
   Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int64_t capacity) override;
 
@@ -555,6 +567,18 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   /// \return Status
   Status Append(const uint8_t* values, int64_t length, const std::vector<bool>& is_valid);
 
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a std::vector of bytes
+  /// \param[in] is_valid an std::vector<bool> indicating valid (1) or null
+  /// (0). Equal in length to values
+  /// \return Status
+  Status Append(const std::vector<uint8_t>& values, const std::vector<bool>& is_valid);
+
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values a std::vector of bytes
+  /// \return Status
+  Status Append(const std::vector<uint8_t>& values);
+
   /// \brief Append a sequence of elements in one shot
   /// \param[in] values an std::vector<bool> indicating true (1) or false
   /// \param[in] is_valid an std::vector<bool> indicating valid (1) or null
@@ -562,6 +586,11 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   /// \return Status
   Status Append(const std::vector<bool>& values, const std::vector<bool>& is_valid);
 
+  /// \brief Append a sequence of elements in one shot
+  /// \param[in] values an std::vector<bool> indicating true (1) or false
+  /// \return Status
+  Status Append(const std::vector<bool>& values);
+
   Status Finish(std::shared_ptr<Array>* out) override;
   Status Init(int64_t capacity) override;
 

From 54d624b2eeb90afb2724b5137bed5dd0e5d8b0a1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 6 Sep 2017 13:24:09 -0400
Subject: [PATCH 1011/1644] ARROW-1405: [Python] Expose LoggingMemoryPool in
 Python API

I removed some output of pointer contents in the LoggingMemoryPool implementation which was showing up as garbage in the console. Example:

```
n [1]: import pyarrow as pa

In [2]: import pyarrow.parquet as pq

In [3]: pa.log_memory_allocations(True)

In [4]: t = pq.read_table('/home/wesm/Downloads/part-00000-6570e34b-b42c-4a39-8adf-21d3a97fb87d.snappy.parquet')
Allocate: size = 320
Allocate: size = 64
Allocate: size = 192
Free: size = 192
Allocate: size = 320
Allocate: size = 64
Allocate: size = 192
Allocate: size = 192
Allocate: size = 1152
Allocate: size = 64
Allocate: size = 64
Allocate: size = 64
Reallocate: old_size = 64 - new_size = 128
Reallocate: old_size = 128 - new_size = 192
Reallocate: old_size = 192 - new_size = 320
Reallocate: old_size = 320 - new_size = 576
Reallocate: old_size = 64 - new_size = 128
Reallocate: old_size = 576 - new_size = 1088
Reallocate: old_size = 128 - new_size = 256
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1044 from wesm/ARROW-1405 and squashes the following commits:

33111644 [Wes McKinney] Restore includes to prior revision
41d8506f [Wes McKinney] Docstring
9d5e96a4 [Wes McKinney] Expose LoggingMemoryPool in Python API
---
 cpp/src/arrow/memory_pool.cc |  8 ++++----
 python/doc/source/api.rst    |  1 +
 python/pyarrow/__init__.py   |  5 ++++-
 python/pyarrow/lib.pxd       |  4 ----
 python/pyarrow/memory.pxi    | 27 ++++++++++++++++++++++++++-
 5 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index b5b4461cfe7b5..7fd999e2d8a86 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -162,20 +162,20 @@ LoggingMemoryPool::LoggingMemoryPool(MemoryPool* pool) : pool_(pool) {}
 
 Status LoggingMemoryPool::Allocate(int64_t size, uint8_t** out) {
   Status s = pool_->Allocate(size, out);
-  std::cout << "Allocate: size = " << size << " - out = " << *out << std::endl;
+  std::cout << "Allocate: size = " << size << std::endl;
   return s;
 }
 
 Status LoggingMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
   Status s = pool_->Reallocate(old_size, new_size, ptr);
-  std::cout << "Reallocate: old_size = " << old_size << " - new_size = " << new_size
-            << " - ptr = " << *ptr << std::endl;
+  std::cout << "Reallocate: old_size = " << old_size
+            << " - new_size = " << new_size << std::endl;
   return s;
 }
 
 void LoggingMemoryPool::Free(uint8_t* buffer, int64_t size) {
   pool_->Free(buffer, size);
-  std::cout << "Free: buffer = " << buffer << " - size = " << size << std::endl;
+  std::cout << "Free: size = " << size << std::endl;
 }
 
 int64_t LoggingMemoryPool::bytes_allocated() const {
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 473b16d21ec49..cc24e49f8ae61 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -222,6 +222,7 @@ Memory Pools
    default_memory_pool
    total_allocated_bytes
    set_memory_pool
+   log_memory_allocations
 
 .. _api.type_classes:
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index a4a6ed449410a..2b6c9fe7f4793 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -76,9 +76,12 @@
                          have_libhdfs, have_libhdfs3, MockOutputStream)
 
 from pyarrow.lib import (MemoryPool, total_allocated_bytes,
-                         set_memory_pool, default_memory_pool)
+                         set_memory_pool, default_memory_pool,
+                         log_memory_allocations)
+
 from pyarrow.lib import (ChunkedArray, Column, RecordBatch, Table,
                          concat_tables)
+
 from pyarrow.lib import (ArrowException,
                          ArrowKeyError,
                          ArrowInvalid,
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 48a58f7b82660..8fdcf553c13fc 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -35,10 +35,6 @@ cdef class MemoryPool:
     cdef void init(self, CMemoryPool* pool)
 
 
-cdef class LoggingMemoryPool(MemoryPool):
-    pass
-
-
 cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool)
 
 
diff --git a/python/pyarrow/memory.pxi b/python/pyarrow/memory.pxi
index 6671a0125502d..3d2601f89c8f3 100644
--- a/python/pyarrow/memory.pxi
+++ b/python/pyarrow/memory.pxi
@@ -36,7 +36,12 @@ cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool):
 
 
 cdef class LoggingMemoryPool(MemoryPool):
-    pass
+    cdef:
+        unique_ptr[CLoggingMemoryPool] logging_pool
+
+    def __cinit__(self, MemoryPool pool):
+        self.logging_pool.reset(new CLoggingMemoryPool(pool.pool))
+        self.init(self.logging_pool.get())
 
 
 def default_memory_pool():
@@ -50,6 +55,26 @@ def set_memory_pool(MemoryPool pool):
     c_set_default_memory_pool(pool.pool)
 
 
+cdef MemoryPool _default_memory_pool = default_memory_pool()
+cdef LoggingMemoryPool _logging_memory_pool = (
+    LoggingMemoryPool(_default_memory_pool))
+
+
+def log_memory_allocations(enable=True):
+    """
+    Enable or disable memory allocator logging for debugging purposes
+
+    Parameters
+    ----------
+    enable : boolean, default True
+        Pass False to disable logging
+    """
+    if enable:
+        set_memory_pool(_logging_memory_pool)
+    else:
+        set_memory_pool(_default_memory_pool)
+
+
 def total_allocated_bytes():
     cdef CMemoryPool* pool = c_get_memory_pool()
     return pool.bytes_allocated()

From 3d285c69dd7ece1821f4042ed9fdc87dbb1eb623 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 6 Sep 2017 19:25:52 -0400
Subject: [PATCH 1012/1644] ARROW-1451: [C++] Add public API file for IO
 section in arrow/io/api.h

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1057 from wesm/ARROW-1451 and squashes the following commits:

deb802f9 [Wes McKinney] Add public API file for IO tools in arrow/io/api.h, use in pyarrow
---
 cpp/src/arrow/io/CMakeLists.txt      |  1 +
 cpp/src/arrow/io/api.h               | 26 ++++++++++++++++++++++++++
 python/pyarrow/includes/libarrow.pxd | 10 +++-------
 3 files changed, 30 insertions(+), 7 deletions(-)
 create mode 100644 cpp/src/arrow/io/api.h

diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 544b4b79430c2..f35573870191f 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -28,6 +28,7 @@ ADD_ARROW_BENCHMARK(io-memory-benchmark)
 
 # Headers: top level
 install(FILES
+  api.h
   file.h
   hdfs.h
   interfaces.h
diff --git a/cpp/src/arrow/io/api.h b/cpp/src/arrow/io/api.h
new file mode 100644
index 0000000000000..9f26d95e40a6e
--- /dev/null
+++ b/cpp/src/arrow/io/api.h
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_IO_API_H
+#define ARROW_IO_API_H
+
+#include "arrow/io/file.h"
+#include "arrow/io/hdfs.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
+
+#endif  // ARROW_IO_API_H
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 6bb8702ad2a2e..756b213b29e3d 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -419,7 +419,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
                               shared_ptr[CTable]* result)
 
 
-cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
+cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil:
     enum FileMode" arrow::io::FileMode::type":
         FileMode_READ" arrow::io::FileMode::READ"
         FileMode_WRITE" arrow::io::FileMode::WRITE"
@@ -473,9 +473,6 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
     cdef cppclass FileSystem:
         CStatus Stat(const c_string& path, FileStatistics* stat)
 
-
-cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
-
     cdef cppclass FileOutputStream(OutputStream):
         @staticmethod
         CStatus Open(const c_string& path, shared_ptr[FileOutputStream]* file)
@@ -505,8 +502,9 @@ cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
 
         int file_descriptor()
 
+    # ----------------------------------------------------------------------
+    # HDFS
 
-cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
     CStatus HaveLibHdfs()
     CStatus HaveLibHdfs3()
 
@@ -574,8 +572,6 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
                               int64_t default_block_size,
                               shared_ptr[HdfsOutputStream]* handle)
 
-
-cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
     cdef cppclass CBufferReader \
             " arrow::io::BufferReader"(RandomAccessFile):
         CBufferReader(const shared_ptr[CBuffer]& buffer)

From d9f895ebde94a21bd2bf91085a186683fb521e48 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 6 Sep 2017 19:27:23 -0400
Subject: [PATCH 1013/1644] ARROW-1446: [Python] Add (very slow) large memory
 unit test for int32 overflow in PARQUET-1090

Bug fixed in https://github.com/apache/parquet-cpp/pull/389

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1055 from wesm/ARROW-1446 and squashes the following commits:

97c634c9 [Wes McKinney] Add (very slow) large memory unit test for PARQUET-1090
---
 python/pyarrow/tests/test_parquet.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index fa9455b6ca3a3..ae5c28f75c1eb 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1239,3 +1239,17 @@ def test_write_to_dataset_with_partitions(tmpdir):
 @parquet
 def test_write_to_dataset_no_partitions(tmpdir):
     _test_write_to_dataset_no_partitions(str(tmpdir))
+
+
+@pytest.mark.large_memory
+@parquet
+def test_large_table_int32_overflow():
+    size = np.iinfo('int32').max + 1
+
+    arr = np.ones(size, dtype='uint8')
+
+    parr = pa.Array.from_pandas(arr, type=pa.uint8())
+
+    table = pa.Table.from_arrays([parr], names=['one'])
+    f = io.BytesIO()
+    _write_table(table, f)

From 3033eac1056315f22024c1a36f21484789dd8675 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 6 Sep 2017 19:38:56 -0400
Subject: [PATCH 1014/1644] ARROW-1466: [C++] Implement PrettyPrint for
 DecimalArray

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1056 from wesm/ARROW-1466 and squashes the following commits:

1ec7f1cc [Wes McKinney] Implement PrettyPrint for DecimalArray
---
 cpp/src/arrow/pretty_print-test.cc | 26 ++++++++++++++++++++++++++
 cpp/src/arrow/pretty_print.cc      | 17 +++++++++++++++--
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index d4c92ce4caa84..ab0bc16c83448 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -31,6 +31,8 @@
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
+#include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
 
 namespace arrow {
 
@@ -106,6 +108,30 @@ TEST_F(TestPrettyPrint, FixedSizeBinaryType) {
   CheckArray(*array, 0, ex);
 }
 
+TEST_F(TestPrettyPrint, DecimalType) {
+  int32_t p = 19;
+  int32_t s = 4;
+
+  auto type = decimal(p, s);
+
+  DecimalBuilder builder(type);
+
+  Int128 val;
+
+  ASSERT_OK(DecimalUtil::FromString("123.4567", &val));
+  ASSERT_OK(builder.Append(val));
+
+  ASSERT_OK(DecimalUtil::FromString("456.7891", &val));
+  ASSERT_OK(builder.Append(val));
+  ASSERT_OK(builder.AppendNull());
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  static const char* ex = R"expected([123.4567, 456.7891, null])expected";
+  CheckArray(*array, 0, ex);
+}
+
 TEST_F(TestPrettyPrint, DictionaryType) {
   std::vector<bool> is_valid = {true, true, false, true, true, true};
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index f759056330792..1202faa876e48 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -167,6 +167,21 @@ class ArrayPrinter : public PrettyPrinter {
     }
   }
 
+  template <typename T>
+  inline typename std::enable_if<std::is_same<DecimalArray, T>::value, void>::type
+  WriteDataValues(const T& array) {
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) {
+        (*sink_) << ", ";
+      }
+      if (array.IsNull(i)) {
+        Write("null");
+      } else {
+        (*sink_) << array.FormatValue(i);
+      }
+    }
+  }
+
   template <typename T>
   inline typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
   WriteDataValues(const T& array) {
@@ -198,8 +213,6 @@ class ArrayPrinter : public PrettyPrinter {
 
   Status Visit(const IntervalArray& array) { return Status::NotImplemented("interval"); }
 
-  Status Visit(const DecimalArray& array) { return Status::NotImplemented("decimal"); }
-
   Status WriteValidityBitmap(const Array& array);
 
   Status Visit(const ListArray& array) {

From 6e5f7be1088641958163dfe59e06245bd08b22a6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 6 Sep 2017 22:52:13 -0400
Subject: [PATCH 1015/1644] ARROW-1435: [Python] Properly handle time zone
 metadata in Parquet round trips

cc @jreback. Various bugs fixed here, but the bottom line is that this enables tz-aware pandas data to be faithfully round-tripped to Parquet format. We will need to implement compatibility tests in pandas for this, too

example DataFrame that could not be properly written before:

```python
s = pd.Series([datetime.datetime(2017, 9, 6)])
s = s.dt.tz_localize('utc')
s.index = s
# Both a column and an index to hit both use cases
df = pd.DataFrame({'tz_aware': s}, index=s)
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1054 from wesm/ARROW-1435 and squashes the following commits:

6519945f [Wes McKinney] Add test for a non-UTC time zone too
20bb6dc1 [Wes McKinney] Get round trip for tz-aware index to Parquet working. Handle time zones in Column.to_pandas
f92abaa7 [Wes McKinney] Fix, initial test passing
6701cf0e [Wes McKinney] Initial cut at fixing tz aware columns to/from Parquet
---
 cpp/src/arrow/python/pandas_to_arrow.cc     |  6 ++-
 python/pyarrow/__init__.py                  |  3 +-
 python/pyarrow/pandas_compat.py             | 53 +++++++++++++++++----
 python/pyarrow/scalar.pxi                   | 15 ++++--
 python/pyarrow/table.pxi                    | 12 ++++-
 python/pyarrow/tests/test_convert_pandas.py | 33 +++++++++----
 python/pyarrow/tests/test_parquet.py        | 29 ++++++++++-
 python/pyarrow/tests/test_serialization.py  |  4 +-
 python/pyarrow/types.pxi                    | 18 ++++++-
 9 files changed, 144 insertions(+), 29 deletions(-)

diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 8410381860b70..24937795f701b 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -347,8 +347,9 @@ class PandasConverter {
     }
 
     BufferVector buffers = {null_bitmap_, data};
-    return PushArray(
-        std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count, 0));
+    auto arr_data = std::make_shared<ArrayData>(type_, length_, std::move(buffers),
+                                                null_count, 0);
+    return PushArray(arr_data);
   }
 
   template <typename T>
@@ -1158,6 +1159,7 @@ Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
   PandasConverter converter(pool, ao, mo, type);
   RETURN_NOT_OK(converter.Convert());
   *out = converter.result()[0];
+  DCHECK(*out);
   return Status::OK();
 }
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 2b6c9fe7f4793..0d76a35f4ed3c 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -65,7 +65,8 @@
                          FloatValue, DoubleValue, ListValue,
                          BinaryValue, StringValue, FixedSizeBinaryValue,
                          DecimalValue,
-                         Date32Value, Date64Value, TimestampValue)
+                         Date32Value, Date64Value, TimestampValue,
+                         TimestampType)
 
 from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
                          FixedSizeBufferWriter,
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 434b1c9eab90e..d1e6f5a8096e7 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -172,8 +172,8 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
     dict
     """
     ncolumns = len(column_names)
-    df_types = types[:ncolumns]
-    index_types = types[ncolumns:ncolumns + len(index_levels)]
+    df_types = types[:ncolumns - len(index_levels)]
+    index_types = types[ncolumns - len(index_levels):]
 
     column_metadata = [
         get_column_metadata(df[col_name], name=sanitized_name,
@@ -269,13 +269,15 @@ def maybe_coerce_datetime64(values, dtype, type_, timestamps_to_ms=False):
     return values, type_
 
 
+def make_datetimetz(tz):
+    from pyarrow.compat import DatetimeTZDtype
+    return DatetimeTZDtype('ns', tz=tz)
+
+
 def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     import pandas.core.internals as _int
-    from pyarrow.compat import DatetimeTZDtype
     import pyarrow.lib as lib
 
-    block_table = table
-
     index_columns = []
     index_arrays = []
     index_names = []
@@ -286,6 +288,9 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     if metadata is not None and b'pandas' in metadata:
         pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
         index_columns = pandas_metadata['index_columns']
+        table = _add_any_metadata(table, pandas_metadata)
+
+    block_table = table
 
     for name in index_columns:
         i = schema.get_field_index(name)
@@ -293,13 +298,14 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
             col = table.column(i)
             index_name = (None if is_unnamed_index_level(name)
                           else name)
-            values = col.to_pandas().values
+            col_pandas = col.to_pandas()
+            values = col_pandas.values
             if not values.flags.writeable:
                 # ARROW-1054: in pandas 0.19.2, factorize will reject
                 # non-writeable arrays when calling MultiIndex.from_arrays
                 values = values.copy()
 
-            index_arrays.append(values)
+            index_arrays.append(pd.Series(values, dtype=col_pandas.dtype))
             index_names.append(index_name)
             block_table = block_table.remove_column(
                 block_table.schema.get_field_index(name)
@@ -319,7 +325,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
                                     klass=_int.CategoricalBlock,
                                     fastpath=True)
         elif 'timezone' in item:
-            dtype = DatetimeTZDtype('ns', tz=item['timezone'])
+            dtype = make_datetimetz(item['timezone'])
             block = _int.make_block(block_arr, placement=placement,
                                     klass=_int.DatetimeTZBlock,
                                     dtype=dtype, fastpath=True)
@@ -340,3 +346,34 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     ]
 
     return _int.BlockManager(blocks, axes)
+
+
+def _add_any_metadata(table, pandas_metadata):
+    modified_columns = {}
+
+    schema = table.schema
+
+    # Add time zones
+    for i, col_meta in enumerate(pandas_metadata['columns']):
+        if col_meta['pandas_type'] == 'datetimetz':
+            col = table[i]
+            converted = col.to_pandas()
+            tz = col_meta['metadata']['timezone']
+            tz_aware_type = pa.timestamp('ns', tz=tz)
+            with_metadata = pa.Array.from_pandas(converted.values,
+                                                 type=tz_aware_type)
+
+            field = pa.field(schema[i].name, tz_aware_type)
+            modified_columns[i] = pa.Column.from_array(field,
+                                                       with_metadata)
+
+    if len(modified_columns) > 0:
+        columns = []
+        for i in range(len(table.schema)):
+            if i in modified_columns:
+                columns.append(modified_columns[i])
+            else:
+                columns.append(table[i])
+        return pa.Table.from_arrays(columns)
+    else:
+        return table
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index 16d2bad0d2d8d..3a847f77c4f81 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -212,11 +212,18 @@ else:
 
 cdef class TimestampValue(ArrayValue):
 
+    property value:
+
+        def __get__(self):
+            cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
+            cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+            return ap.Value(self.index)
+
     def as_py(self):
-        cdef:
-            CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
-            CTimestampType* dtype = <CTimestampType*> ap.type().get()
-            int64_t value = ap.Value(self.index)
+        cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
+        cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+
+        value = self.value
 
         if not dtype.timezone().empty():
             import pytz
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index dd3359ef13968..245371ffaa124 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -150,6 +150,8 @@ cdef class Column:
 
         if isinstance(field_or_name, Field):
             boxed_field = field_or_name
+            if arr.type != boxed_field.type:
+                raise ValueError('Passed field type does not match array')
         else:
             boxed_field = field(field_or_name, arr.type)
 
@@ -176,7 +178,15 @@ cdef class Column:
                                                         self.sp_column,
                                                         self, &out))
 
-        return pd.Series(wrap_array_output(out), name=self.name)
+        values = wrap_array_output(out)
+        result = pd.Series(values, name=self.name)
+
+        if isinstance(self.type, TimestampType):
+            if self.type.tz is not None:
+                result = (result.dt.tz_localize('utc')
+                          .dt.tz_convert(self.type.tz))
+
+        return result
 
     def equals(self, Column other):
         """
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index f5107c2dfe974..52290d6d85533 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -81,6 +81,16 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
             expected = df
         tm.assert_frame_equal(result, expected, check_dtype=check_dtype)
 
+    def _check_series_roundtrip(self, s, type_=None):
+        arr = pa.Array.from_pandas(s, type=type_)
+
+        result = pd.Series(arr.to_pandas(), name=s.name)
+        if isinstance(arr.type, pa.TimestampType) and arr.type.tz is not None:
+            result = (result.dt.tz_localize('utc')
+                      .dt.tz_convert(arr.type.tz))
+
+        tm.assert_series_equal(s, result)
+
     def _check_array_roundtrip(self, values, expected=None, mask=None,
                                timestamps_to_ms=False, type=None):
         arr = pa.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
@@ -347,9 +357,7 @@ def test_timestamps_notimezone_no_nulls(self):
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
         self._check_pandas_roundtrip(
-            df,
-            timestamps_to_ms=False,
-            expected_schema=schema,
+            df, expected_schema=schema,
         )
 
     def test_timestamps_to_ms_explicit_schema(self):
@@ -389,9 +397,7 @@ def test_timestamps_notimezone_nulls(self):
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
         self._check_pandas_roundtrip(
-            df,
-            timestamps_to_ms=False,
-            expected_schema=schema,
+            df, expected_schema=schema,
         )
 
     def test_timestamps_with_timezone(self):
@@ -406,6 +412,8 @@ def test_timestamps_with_timezone(self):
                             .to_frame())
         self._check_pandas_roundtrip(df, timestamps_to_ms=True)
 
+        self._check_series_roundtrip(df['datetime64'])
+
         # drop-in a null and ns instead of ms
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -417,7 +425,15 @@ def test_timestamps_with_timezone(self):
         })
         df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
                             .to_frame())
-        self._check_pandas_roundtrip(df, timestamps_to_ms=False)
+        self._check_pandas_roundtrip(df)
+
+    def test_timestamp_with_tz_to_pandas_type(self):
+        from pyarrow.compat import DatetimeTZDtype
+
+        tz = 'America/Los_Angeles'
+        t = pa.timestamp('ns', tz=tz)
+
+        assert t.to_pandas_dtype() == DatetimeTZDtype('ns', tz=tz)
 
     def test_date_infer(self):
         df = pd.DataFrame({
@@ -586,8 +602,7 @@ def test_nested_lists_all_none(self):
 
     def test_threaded_conversion(self):
         df = _alltypes_example()
-        self._check_pandas_roundtrip(df, nthreads=2,
-                                     timestamps_to_ms=False)
+        self._check_pandas_roundtrip(df, nthreads=2)
 
     def test_category(self):
         repeats = 5
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index ae5c28f75c1eb..5dfe0a59588a9 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -22,7 +22,7 @@
 import json
 import pytest
 
-from pyarrow.compat import guid, u
+from pyarrow.compat import guid, u, BytesIO
 from pyarrow.filesystem import LocalFileSystem
 import pyarrow as pa
 from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
@@ -114,6 +114,33 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@parquet
+def test_pandas_parquet_datetime_tz():
+    import pyarrow.parquet as pq
+
+    s = pd.Series([datetime.datetime(2017, 9, 6)])
+    s = s.dt.tz_localize('utc')
+
+    s.index = s
+
+    # Both a column and an index to hit both use cases
+    df = pd.DataFrame({'tz_aware': s,
+                       'tz_eastern': s.dt.tz_convert('US/Eastern')},
+                      index=s)
+
+    f = BytesIO()
+
+    arrow_table = pa.Table.from_pandas(df)
+
+    _write_table(arrow_table, f, coerce_timestamps='ms')
+    f.seek(0)
+
+    table_read = pq.read_pandas(f)
+
+    df_read = table_read.to_pandas()
+    tm.assert_frame_equal(df, df_read)
+
+
 @parquet
 def test_pandas_parquet_custom_metadata(tmpdir):
     import pyarrow.parquet as pq
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 8afcf0f7c04b6..27243b0000d18 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -100,9 +100,9 @@ def assert_equal(obj1, obj2):
 if sys.version_info >= (3, 0):
     PRIMITIVE_OBJECTS += [0, np.array([["hi", u"hi"], [1.3, 1]])]
 else:
-    PRIMITIVE_OBJECTS += [long(42), long(1 << 62), long(0),
+    PRIMITIVE_OBJECTS += [long(42), long(1 << 62), long(0),  # noqa
                           np.array([["hi", u"hi"],
-                          [1.3, long(1)]])]  # noqa: E501,F821
+                          [1.3, long(1)]])]  # noqa
 
 
 COMPLEX_OBJECTS = [
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index fb6b9618d98df..3eaee6ced33f7 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -134,6 +134,16 @@ cdef class TimestampType(DataType):
             else:
                 return None
 
+    def to_pandas_dtype(self):
+        """
+        Return the NumPy dtype that would be used for storing this
+        """
+        if self.tz is None:
+            return _pandas_type_map[_Type_TIMESTAMP]
+        else:
+            # Return DatetimeTZ
+            return pdcompat.make_datetimetz(self.tz)
+
 
 cdef class Time32Type(DataType):
 
@@ -431,7 +441,13 @@ cdef class Schema:
         with nogil:
             check_status(PrettyPrint(deref(self.schema), options, &result))
 
-        return frombytes(result)
+        printed = frombytes(result)
+        if self.metadata is not None:
+            import pprint
+            metadata_formatted = pprint.pformat(self.metadata)
+            printed += '\nmetadata\n--------\n' + metadata_formatted
+
+        return printed
 
     def __repr__(self):
         return self.__str__()

From a9a80fef7f930ac7141f3c02f2b1dfbb98dfdd1d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 7 Sep 2017 18:40:48 +0200
Subject: [PATCH 1016/1644] ARROW-941: Add "cold start" instructions for
 running integration tests

This is helpful for users who do not regularly use Python.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1061 from wesm/ARROW-941 and squashes the following commits:

5f18cd7 [Wes McKinney] Add environment setup instruction for integration tests for users without any Python environment already
---
 integration/README.md | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/integration/README.md b/integration/README.md
index 5b6ea45ff7370..6e5a6c15641ec 100644
--- a/integration/README.md
+++ b/integration/README.md
@@ -30,6 +30,31 @@ Our strategy for integration testing between Arrow implementations is as follows
 * The test executable is also capable of validating the contents of a binary
   file against a corresponding JSON file
 
+## Environment setup
+
+The integration test data generator and runner is written in Python and
+currently requires Python 3.5 or higher. You can create a standalone Python
+distribution and environment for running the tests by using [miniconda][1]. On
+Linux this is:
+
+```shell
+MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+wget -O miniconda.sh $MINICONDA_URL
+bash miniconda.sh -b -p miniconda
+export PATH=`pwd`/miniconda/bin:$PATH
+
+conda create -n arrow-integration python=3.6 nomkl numpy six
+source activate arrow-integration
+```
+
+If you are on macOS, instead use the URL:
+
+```shell
+MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+```
+
+After this, you can follow the instructions in the next section.
+
 ## Running the existing integration tests
 
 First, build the Java and C++ projects. For Java, you must run
@@ -43,7 +68,7 @@ Java `arrow-tool` JAR and the build path for the C++ executables:
 
 ```bash
 JAVA_DIR=$ARROW_HOME/java
-CPP_BUILD_DIR=$ARROW_HOME/cpp/test-build
+CPP_BUILD_DIR=$ARROW_HOME/cpp/build
 
 VERSION=0.1.1-SNAPSHOT
 export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
@@ -61,4 +86,6 @@ for additional output)
 python integration_test.py
 
 python integration_test.py --debug  # additional output
-```
\ No newline at end of file
+```
+
+[1]: https://conda.io/miniconda.html
\ No newline at end of file

From b0b125fd74b2bb334e90d9775a670bf18ffd8a22 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 7 Sep 2017 13:00:37 -0400
Subject: [PATCH 1017/1644] ARROW-229: [C++] Implement cast functions for
 numeric types, booleans

Implements safe and unsafe casts amongst booleans and signed/unsigned integers, and single and double precision floating point numbers.

Currently there is only the option to check for integer overflows when casting from a larger integer to a smaller integer. This API should be regarded as experimental in 0.7.0. There are a number of follow up patches we'll want to do quickly after this (exposing this in Python, incorporating into Array.from_pandas)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1027 from wesm/ARROW-229 and squashes the following commits:

82fea97 [Wes McKinney] Fix MSVC warning
ead4a95 [Wes McKinney] Fix overflow check where overflow occurs in a null slot
dc7f8d9 [Wes McKinney] Some basic smoke tests to validate implemented casts
879653d [Wes McKinney] Start test suite for Cast
22308ba [Wes McKinney] Implement cast kernels for numbers. Add helper type traits
ca1c813 [Wes McKinney] Work on context
d05c274 [Wes McKinney] Start some prototyping of a cast implementation
---
 cpp/CMakeLists.txt                    |  37 ++-
 cpp/src/arrow/array.h                 |   2 +-
 cpp/src/arrow/compute/CMakeLists.txt  |  28 +++
 cpp/src/arrow/compute/cast.cc         | 329 ++++++++++++++++++++++++++
 cpp/src/arrow/compute/cast.h          |  55 +++++
 cpp/src/arrow/compute/compute-test.cc | 315 ++++++++++++++++++++++++
 cpp/src/arrow/compute/context.cc      |  46 ++++
 cpp/src/arrow/compute/context.h       |  68 ++++++
 cpp/src/arrow/memory_pool.cc          |   4 +-
 cpp/src/arrow/test-util.h             |  10 +
 cpp/src/arrow/type.h                  |  82 +------
 cpp/src/arrow/type_traits.h           |  68 ++++++
 12 files changed, 956 insertions(+), 88 deletions(-)
 create mode 100644 cpp/src/arrow/compute/CMakeLists.txt
 create mode 100644 cpp/src/arrow/compute/cast.cc
 create mode 100644 cpp/src/arrow/compute/cast.h
 create mode 100644 cpp/src/arrow/compute/compute-test.cc
 create mode 100644 cpp/src/arrow/compute/context.cc
 create mode 100644 cpp/src/arrow/compute/context.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 9f9d71b8747fd..24735ac50c665 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -94,6 +94,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Exclude deprecated APIs from build"
     OFF)
 
+  option(ARROW_COMPUTE
+    "Build the Arrow Compute Modules"
+    ON)
+
   option(ARROW_EXTRA_ERROR_CONTEXT
     "Compile with extra error context (line numbers, code)"
     OFF)
@@ -727,17 +731,6 @@ endif()
 add_subdirectory(src/arrow)
 add_subdirectory(src/arrow/io)
 
-if (ARROW_GPU)
-  # IPC extensions required to build the GPU library
-  set(ARROW_IPC ON)
-  add_subdirectory(src/arrow/gpu)
-endif()
-
-if (ARROW_IPC)
-  add_subdirectory(src/arrow/ipc)
-  add_dependencies(arrow_dependencies metadata_fbs)
-endif()
-
 set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/buffer.cc
@@ -751,6 +744,9 @@ set(ARROW_SRCS
   src/arrow/type.cc
   src/arrow/visitor.cc
 
+  src/arrow/compute/cast.cc
+  src/arrow/compute/context.cc
+
   src/arrow/io/file.cc
   src/arrow/io/interfaces.cc
   src/arrow/io/memory.cc
@@ -763,6 +759,25 @@ set(ARROW_SRCS
   src/arrow/util/key_value_metadata.cc
 )
 
+if (ARROW_COMPUTE)
+  add_subdirectory(src/arrow/compute)
+  set(ARROW_SRCS ${ARROW_SRCS}
+    src/arrow/compute/cast.cc
+    src/arrow/compute/context.cc
+  )
+endif()
+
+if (ARROW_GPU)
+  # IPC extensions required to build the GPU library
+  set(ARROW_IPC ON)
+  add_subdirectory(src/arrow/gpu)
+endif()
+
+if (ARROW_IPC)
+  add_subdirectory(src/arrow/ipc)
+  add_dependencies(arrow_dependencies metadata_fbs)
+endif()
+
 if (ARROW_WITH_BROTLI)
   add_definitions(-DARROW_WITH_BROTLI)
   SET(ARROW_SRCS src/arrow/util/compression_brotli.cc ${ARROW_SRCS})
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 8e965e8d9174b..61ab2ef4aafd9 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -678,4 +678,4 @@ MakePrimitiveArray(const std::shared_ptr<DataType>& type,
 
 }  // namespace arrow
 
-#endif
+#endif  // ARROW_ARRAY_H
diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt
new file mode 100644
index 0000000000000..a154c47c87463
--- /dev/null
+++ b/cpp/src/arrow/compute/CMakeLists.txt
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Headers: top level
+install(FILES
+  cast.h
+  context.h
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute")
+
+#######################################
+# Unit tests
+#######################################
+
+ADD_ARROW_TEST(compute-test)
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
new file mode 100644
index 0000000000000..f610f6b964c31
--- /dev/null
+++ b/cpp/src/arrow/compute/cast.cc
@@ -0,0 +1,329 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/compute/cast.h"
+
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <memory>
+#include <sstream>
+#include <type_traits>
+
+#include "arrow/type_traits.h"
+#include "arrow/util/logging.h"
+
+#include "arrow/compute/context.h"
+
+namespace arrow {
+namespace compute {
+
+struct CastContext {
+  FunctionContext* func_ctx;
+  CastOptions options;
+};
+
+typedef std::function<void(CastContext*, const ArrayData&, ArrayData*)> CastFunction;
+
+template <typename OutType, typename InType, typename Enable = void>
+struct CastFunctor {};
+
+// Type is the same, no computation required
+template <typename O, typename I>
+struct CastFunctor<O, I, typename std::enable_if<std::is_same<I, O>::value>::type> {
+  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+    output->type = input.type;
+    output->buffers = input.buffers;
+    output->length = input.length;
+    output->offset = input.offset;
+    output->null_count = input.null_count;
+    output->child_data = input.child_data;
+  }
+};
+
+// ----------------------------------------------------------------------
+// Null to other things
+
+template <typename T>
+struct CastFunctor<T, NullType,
+                   typename std::enable_if<!std::is_same<T, NullType>::value>::type> {
+  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+    ctx->func_ctx->SetStatus(Status::NotImplemented("NullType"));
+  }
+};
+
+// ----------------------------------------------------------------------
+// Boolean to other things
+
+// Cast from Boolean to other numbers
+template <typename T>
+struct CastFunctor<T, BooleanType,
+                   typename std::enable_if<std::is_base_of<Number, T>::value>::type> {
+  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+    using c_type = typename T::c_type;
+    const uint8_t* data = input.buffers[1]->data();
+    auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
+    constexpr auto kOne = static_cast<c_type>(1);
+    constexpr auto kZero = static_cast<c_type>(0);
+    for (int64_t i = 0; i < input.length; ++i) {
+      *out++ = BitUtil::GetBit(data, i) ? kOne : kZero;
+    }
+  }
+};
+
+// ----------------------------------------------------------------------
+// Integers and Floating Point
+
+template <typename O, typename I>
+struct is_numeric_cast {
+  static constexpr bool value =
+      (std::is_base_of<Number, O>::value && std::is_base_of<Number, I>::value) &&
+      (!std::is_same<O, I>::value);
+};
+
+template <typename O, typename I, typename Enable = void>
+struct is_integer_downcast {
+  static constexpr bool value = false;
+};
+
+template <typename O, typename I>
+struct is_integer_downcast<
+    O, I, typename std::enable_if<std::is_base_of<Integer, O>::value &&
+                                  std::is_base_of<Integer, I>::value>::type> {
+  using O_T = typename O::c_type;
+  using I_T = typename I::c_type;
+
+  static constexpr bool value =
+      ((!std::is_same<O, I>::value) &&
+
+       // same size, but unsigned to signed
+       ((sizeof(O_T) == sizeof(I_T) && std::is_signed<O_T>::value &&
+         std::is_unsigned<I_T>::value) ||
+
+        // Smaller output size
+        (sizeof(O_T) < sizeof(I_T))));
+};
+
+template <typename O, typename I>
+struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::value &&
+                                                 std::is_base_of<Number, I>::value &&
+                                                 !std::is_same<O, I>::value>::type> {
+  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+    using in_type = typename I::c_type;
+    auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data());
+    uint8_t* out_data = reinterpret_cast<uint8_t*>(output->buffers[1]->mutable_data());
+    for (int64_t i = 0; i < input.length; ++i) {
+      BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
+    }
+  }
+};
+
+template <typename O, typename I>
+struct CastFunctor<O, I,
+                   typename std::enable_if<is_integer_downcast<O, I>::value>::type> {
+  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+    using in_type = typename I::c_type;
+    using out_type = typename O::c_type;
+
+    auto in_offset = input.offset;
+
+    auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data()) + in_offset;
+    auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
+
+    if (!ctx->options.allow_int_overflow) {
+      constexpr in_type kMax = static_cast<in_type>(std::numeric_limits<out_type>::max());
+      constexpr in_type kMin = static_cast<in_type>(std::numeric_limits<out_type>::min());
+
+      if (input.null_count > 0) {
+        const uint8_t* is_valid = input.buffers[0]->data();
+        int64_t is_valid_offset = in_offset;
+        for (int64_t i = 0; i < input.length; ++i) {
+          if (ARROW_PREDICT_FALSE(BitUtil::GetBit(is_valid, is_valid_offset++) &&
+                                  (*in_data > kMax || *in_data < kMin))) {
+            ctx->func_ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
+          }
+          *out_data++ = static_cast<out_type>(*in_data++);
+        }
+      } else {
+        for (int64_t i = 0; i < input.length; ++i) {
+          if (ARROW_PREDICT_FALSE(*in_data > kMax || *in_data < kMin)) {
+            ctx->func_ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
+          }
+          *out_data++ = static_cast<out_type>(*in_data++);
+        }
+      }
+    } else {
+      for (int64_t i = 0; i < input.length; ++i) {
+        *out_data++ = static_cast<out_type>(*in_data++);
+      }
+    }
+  }
+};
+
+template <typename O, typename I>
+struct CastFunctor<O, I,
+                   typename std::enable_if<is_numeric_cast<O, I>::value &&
+                                           !is_integer_downcast<O, I>::value>::type> {
+  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+    using in_type = typename I::c_type;
+    using out_type = typename O::c_type;
+
+    auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data());
+    auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
+    for (int64_t i = 0; i < input.length; ++i) {
+      *out_data++ = static_cast<out_type>(*in_data++);
+    }
+  }
+};
+
+// ----------------------------------------------------------------------
+
+#define CAST_CASE(InType, OutType)                                            \
+  case OutType::type_id:                                                      \
+    return [type](CastContext* ctx, const ArrayData& input, ArrayData* out) { \
+      CastFunctor<OutType, InType> func;                                      \
+      func(ctx, input, out);                                                  \
+    }
+
+#define NUMERIC_CASES(FN, IN_TYPE) \
+  FN(IN_TYPE, BooleanType);        \
+  FN(IN_TYPE, UInt8Type);          \
+  FN(IN_TYPE, Int8Type);           \
+  FN(IN_TYPE, UInt16Type);         \
+  FN(IN_TYPE, Int16Type);          \
+  FN(IN_TYPE, UInt32Type);         \
+  FN(IN_TYPE, Int32Type);          \
+  FN(IN_TYPE, UInt64Type);         \
+  FN(IN_TYPE, Int64Type);          \
+  FN(IN_TYPE, FloatType);          \
+  FN(IN_TYPE, DoubleType);
+
+#define GET_CAST_FUNCTION(CapType)                                                    \
+  static CastFunction Get##CapType##CastFunc(const std::shared_ptr<DataType>& type) { \
+    switch (type->id()) {                                                             \
+      NUMERIC_CASES(CAST_CASE, CapType);                                              \
+      default:                                                                        \
+        break;                                                                        \
+    }                                                                                 \
+    return nullptr;                                                                   \
+  }
+
+#define CAST_FUNCTION_CASE(CapType)          \
+  case CapType::type_id:                     \
+    *out = Get##CapType##CastFunc(out_type); \
+    break
+
+GET_CAST_FUNCTION(BooleanType);
+GET_CAST_FUNCTION(UInt8Type);
+GET_CAST_FUNCTION(Int8Type);
+GET_CAST_FUNCTION(UInt16Type);
+GET_CAST_FUNCTION(Int16Type);
+GET_CAST_FUNCTION(UInt32Type);
+GET_CAST_FUNCTION(Int32Type);
+GET_CAST_FUNCTION(UInt64Type);
+GET_CAST_FUNCTION(Int64Type);
+GET_CAST_FUNCTION(FloatType);
+GET_CAST_FUNCTION(DoubleType);
+
+static Status GetCastFunction(const DataType& in_type,
+                              const std::shared_ptr<DataType>& out_type,
+                              CastFunction* out) {
+  switch (in_type.id()) {
+    CAST_FUNCTION_CASE(BooleanType);
+    CAST_FUNCTION_CASE(UInt8Type);
+    CAST_FUNCTION_CASE(Int8Type);
+    CAST_FUNCTION_CASE(UInt16Type);
+    CAST_FUNCTION_CASE(Int16Type);
+    CAST_FUNCTION_CASE(UInt32Type);
+    CAST_FUNCTION_CASE(Int32Type);
+    CAST_FUNCTION_CASE(UInt64Type);
+    CAST_FUNCTION_CASE(Int64Type);
+    CAST_FUNCTION_CASE(FloatType);
+    CAST_FUNCTION_CASE(DoubleType);
+    default:
+      break;
+  }
+  if (*out == nullptr) {
+    std::stringstream ss;
+    ss << "No cast implemented from " << in_type.ToString() << " to "
+       << out_type->ToString();
+    return Status::NotImplemented(ss.str());
+  }
+  return Status::OK();
+}
+
+static Status AllocateLike(FunctionContext* ctx, const Array& array,
+                           const std::shared_ptr<DataType>& out_type,
+                           std::shared_ptr<ArrayData>* out) {
+  if (!is_primitive(out_type->id())) {
+    return Status::NotImplemented(out_type->ToString());
+  }
+
+  const auto& fw_type = static_cast<const FixedWidthType&>(*out_type);
+
+  auto result = std::make_shared<ArrayData>();
+  result->type = out_type;
+  result->length = array.length();
+  result->offset = 0;
+  result->null_count = array.null_count();
+
+  // Propagate null bitmap
+  // TODO(wesm): handling null bitmap when input type is NullType
+  result->buffers.push_back(array.data()->buffers[0]);
+
+  std::shared_ptr<Buffer> out_data;
+
+  int bit_width = fw_type.bit_width();
+
+  if (bit_width == 1) {
+    RETURN_NOT_OK(ctx->Allocate(BitUtil::BytesForBits(array.length()), &out_data));
+  } else if (bit_width % 8 == 0) {
+    RETURN_NOT_OK(ctx->Allocate(array.length() * fw_type.bit_width() / 8, &out_data));
+  } else {
+    DCHECK(false);
+  }
+  result->buffers.push_back(out_data);
+
+  *out = result;
+  return Status::OK();
+}
+
+static Status Cast(CastContext* cast_ctx, const Array& array,
+                   const std::shared_ptr<DataType>& out_type,
+                   std::shared_ptr<Array>* out) {
+  // Dynamic dispatch to obtain right cast function
+  CastFunction func;
+  RETURN_NOT_OK(GetCastFunction(*array.type(), out_type, &func));
+
+  // Allocate memory for output
+  std::shared_ptr<ArrayData> out_data;
+  RETURN_NOT_OK(AllocateLike(cast_ctx->func_ctx, array, out_type, &out_data));
+
+  func(cast_ctx, *array.data(), out_data.get());
+  RETURN_IF_ERROR(cast_ctx->func_ctx);
+  return internal::MakeArray(out_data, out);
+}
+
+Status Cast(FunctionContext* ctx, const Array& array,
+            const std::shared_ptr<DataType>& out_type, const CastOptions& options,
+            std::shared_ptr<Array>* out) {
+  CastContext cast_ctx{ctx, options};
+  return Cast(&cast_ctx, array, out_type, out);
+}
+
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/cast.h b/cpp/src/arrow/compute/cast.h
new file mode 100644
index 0000000000000..9ca70aa43e528
--- /dev/null
+++ b/cpp/src/arrow/compute/cast.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_COMPUTE_CAST_H
+#define ARROW_COMPUTE_CAST_H
+
+#include <memory>
+
+#include "arrow/array.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+using internal::ArrayData;
+
+namespace compute {
+
+class FunctionContext;
+
+struct CastOptions {
+  bool allow_int_overflow;
+};
+
+/// \brief Cast from one array type to another
+/// \param[in] context
+/// \param[in] array
+/// \param[in] to_type
+/// \param[in] options
+/// \param[out] out
+///
+/// \since 0.7.0
+/// \note API not yet finalized
+ARROW_EXPORT
+Status Cast(FunctionContext* context, const Array& array,
+            const std::shared_ptr<DataType>& to_type, const CastOptions& options,
+            std::shared_ptr<Array>* out);
+
+}  // namespace compute
+}  // namespace arrow
+
+#endif  // ARROW_COMPUTE_CAST_H
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
new file mode 100644
index 0000000000000..cda5755d1e901
--- /dev/null
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -0,0 +1,315 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <numeric>
+#include <sstream>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/compare.h"
+#include "arrow/ipc/test-common.h"
+#include "arrow/memory_pool.h"
+#include "arrow/pretty_print.h"
+#include "arrow/status.h"
+#include "arrow/test-common.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+
+#include "arrow/compute/cast.h"
+#include "arrow/compute/context.h"
+
+using std::vector;
+
+namespace arrow {
+namespace compute {
+
+void AssertArraysEqual(const Array& left, const Array& right) {
+  bool are_equal = false;
+  ASSERT_OK(ArrayEquals(left, right, &are_equal));
+
+  if (!are_equal) {
+    std::stringstream ss;
+
+    ss << "Left: ";
+    EXPECT_OK(PrettyPrint(left, 0, &ss));
+    ss << "\nRight: ";
+    EXPECT_OK(PrettyPrint(right, 0, &ss));
+    FAIL() << ss.str();
+  }
+}
+
+class ComputeFixture {
+ public:
+  ComputeFixture() : pool_(default_memory_pool()), ctx_(pool_) {}
+
+ protected:
+  MemoryPool* pool_;
+  FunctionContext ctx_;
+};
+
+// ----------------------------------------------------------------------
+// Cast
+
+class TestCast : public ComputeFixture, public ::testing::Test {
+ public:
+  void CheckPass(const Array& input, const Array& expected,
+                 const std::shared_ptr<DataType>& out_type, const CastOptions& options) {
+    std::shared_ptr<Array> result;
+    ASSERT_OK(Cast(&ctx_, input, out_type, options, &result));
+    AssertArraysEqual(expected, *result);
+  }
+
+  template <typename InType, typename I_TYPE>
+  void CheckFails(const std::shared_ptr<DataType>& in_type,
+                  const std::vector<I_TYPE>& in_values, const std::vector<bool>& is_valid,
+                  const std::shared_ptr<DataType>& out_type, const CastOptions& options) {
+    std::shared_ptr<Array> input, result;
+    if (is_valid.size() > 0) {
+      ArrayFromVector<InType, I_TYPE>(in_type, is_valid, in_values, &input);
+    } else {
+      ArrayFromVector<InType, I_TYPE>(in_type, in_values, &input);
+    }
+    ASSERT_RAISES(Invalid, Cast(&ctx_, *input, out_type, options, &result));
+  }
+
+  template <typename InType, typename I_TYPE, typename OutType, typename O_TYPE>
+  void CheckCase(const std::shared_ptr<DataType>& in_type,
+                 const std::vector<I_TYPE>& in_values, const std::vector<bool>& is_valid,
+                 const std::shared_ptr<DataType>& out_type,
+                 const std::vector<O_TYPE>& out_values, const CastOptions& options) {
+    std::shared_ptr<Array> input, expected;
+    if (is_valid.size() > 0) {
+      ArrayFromVector<InType, I_TYPE>(in_type, is_valid, in_values, &input);
+      ArrayFromVector<OutType, O_TYPE>(out_type, is_valid, out_values, &expected);
+    } else {
+      ArrayFromVector<InType, I_TYPE>(in_type, in_values, &input);
+      ArrayFromVector<OutType, O_TYPE>(out_type, out_values, &expected);
+    }
+    CheckPass(*input, *expected, out_type, options);
+  }
+};
+
+TEST_F(TestCast, SameTypeZeroCopy) {
+  vector<bool> is_valid = {true, false, true, true, true};
+  vector<int32_t> v1 = {0, 1, 2, 3, 4};
+
+  std::shared_ptr<Array> arr;
+  ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(Cast(&this->ctx_, *arr, int32(), {}, &result));
+
+  const auto& lbuffers = arr->data()->buffers;
+  const auto& rbuffers = result->data()->buffers;
+
+  // Buffers are the same
+  ASSERT_EQ(lbuffers[0].get(), rbuffers[0].get());
+  ASSERT_EQ(lbuffers[1].get(), rbuffers[1].get());
+}
+
+TEST_F(TestCast, ToBoolean) {
+  CastOptions options;
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  // int8, should suffice for other integers
+  vector<int8_t> v1 = {0, 1, 127, -1, 0};
+  vector<bool> e1 = {false, true, true, true, false};
+  CheckCase<Int8Type, int8_t, BooleanType, bool>(int8(), v1, is_valid, boolean(), e1,
+                                                 options);
+
+  // floating point
+  vector<double> v2 = {1.0, 0, 0, -1.0, 5.0};
+  vector<bool> e2 = {true, false, false, true, true};
+  CheckCase<DoubleType, double, BooleanType, bool>(float64(), v2, is_valid, boolean(), e2,
+                                                   options);
+}
+
+TEST_F(TestCast, ToIntUpcast) {
+  CastOptions options;
+  options.allow_int_overflow = false;
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  // int8 to int32
+  vector<int8_t> v1 = {0, 1, 127, -1, 0};
+  vector<int32_t> e1 = {0, 1, 127, -1, 0};
+  CheckCase<Int8Type, int8_t, Int32Type, int32_t>(int8(), v1, is_valid, int32(), e1,
+                                                  options);
+
+  // bool to int8
+  vector<bool> v2 = {false, true, false, true, true};
+  vector<int8_t> e2 = {0, 1, 0, 1, 1};
+  CheckCase<BooleanType, bool, Int8Type, int8_t>(boolean(), v2, is_valid, int8(), e2,
+                                                 options);
+
+  // uint8 to int16, no overflow/underrun
+  vector<uint8_t> v3 = {0, 100, 200, 255, 0};
+  vector<int16_t> e3 = {0, 100, 200, 255, 0};
+  CheckCase<UInt8Type, uint8_t, Int16Type, int16_t>(uint8(), v3, is_valid, int16(), e3,
+                                                    options);
+
+  // floating point to integer
+  vector<double> v4 = {1.5, 0, 0.5, -1.5, 5.5};
+  vector<int32_t> e4 = {1, 0, 0, -1, 5};
+  CheckCase<DoubleType, double, Int32Type, int32_t>(float64(), v4, is_valid, int32(), e4,
+                                                    options);
+}
+
+TEST_F(TestCast, OverflowInNullSlot) {
+  CastOptions options;
+  options.allow_int_overflow = false;
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  vector<int32_t> v11 = {0, 70000, 2000, 1000, 0};
+  vector<int16_t> e11 = {0, 0, 2000, 1000, 0};
+
+  std::shared_ptr<Array> expected;
+  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, e11, &expected);
+
+  auto buf = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(v11.data()),
+                                      static_cast<int64_t>(v11.size()));
+  Int32Array tmp11(5, buf, expected->null_bitmap(), -1);
+
+  CheckPass(tmp11, *expected, int16(), options);
+}
+
+TEST_F(TestCast, ToIntDowncastSafe) {
+  CastOptions options;
+  options.allow_int_overflow = false;
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  // int16 to uint8, no overflow/underrun
+  vector<int16_t> v5 = {0, 100, 200, 1, 2};
+  vector<uint8_t> e5 = {0, 100, 200, 1, 2};
+  CheckCase<Int16Type, int16_t, UInt8Type, uint8_t>(int16(), v5, is_valid, uint8(), e5,
+                                                    options);
+
+  // int16 to uint8, with overflow
+  vector<int16_t> v6 = {0, 100, 256, 0, 0};
+  CheckFails<Int16Type>(int16(), v6, is_valid, uint8(), options);
+
+  // underflow
+  vector<int16_t> v7 = {0, 100, -1, 0, 0};
+  CheckFails<Int16Type>(int16(), v7, is_valid, uint8(), options);
+
+  // int32 to int16, no overflow
+  vector<int32_t> v8 = {0, 1000, 2000, 1, 2};
+  vector<int16_t> e8 = {0, 1000, 2000, 1, 2};
+  CheckCase<Int32Type, int32_t, Int16Type, int16_t>(int32(), v8, is_valid, int16(), e8,
+                                                    options);
+
+  // int32 to int16, overflow
+  vector<int32_t> v9 = {0, 1000, 2000, 70000, 0};
+  CheckFails<Int32Type>(int32(), v9, is_valid, int16(), options);
+
+  // underflow
+  vector<int32_t> v10 = {0, 1000, 2000, -70000, 0};
+  CheckFails<Int32Type>(int32(), v9, is_valid, int16(), options);
+}
+
+TEST_F(TestCast, ToIntDowncastUnsafe) {
+  CastOptions options;
+  options.allow_int_overflow = true;
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  // int16 to uint8, no overflow/underrun
+  vector<int16_t> v5 = {0, 100, 200, 1, 2};
+  vector<uint8_t> e5 = {0, 100, 200, 1, 2};
+  CheckCase<Int16Type, int16_t, UInt8Type, uint8_t>(int16(), v5, is_valid, uint8(), e5,
+                                                    options);
+
+  // int16 to uint8, with overflow
+  vector<int16_t> v6 = {0, 100, 256, 0, 0};
+  vector<uint8_t> e6 = {0, 100, 0, 0, 0};
+  CheckCase<Int16Type, int16_t, UInt8Type, uint8_t>(int16(), v6, is_valid, uint8(), e6,
+                                                    options);
+
+  // underflow
+  vector<int16_t> v7 = {0, 100, -1, 0, 0};
+  vector<uint8_t> e7 = {0, 100, 255, 0, 0};
+  CheckCase<Int16Type, int16_t, UInt8Type, uint8_t>(int16(), v7, is_valid, uint8(), e7,
+                                                    options);
+
+  // int32 to int16, no overflow
+  vector<int32_t> v8 = {0, 1000, 2000, 1, 2};
+  vector<int16_t> e8 = {0, 1000, 2000, 1, 2};
+  CheckCase<Int32Type, int32_t, Int16Type, int16_t>(int32(), v8, is_valid, int16(), e8,
+                                                    options);
+
+  // int32 to int16, overflow
+  // TODO(wesm): do we want to allow this? we could set to null
+  vector<int32_t> v9 = {0, 1000, 2000, 70000, 0};
+  vector<int16_t> e9 = {0, 1000, 2000, 4464, 0};
+  CheckCase<Int32Type, int32_t, Int16Type, int16_t>(int32(), v9, is_valid, int16(), e9,
+                                                    options);
+
+  // underflow
+  // TODO(wesm): do we want to allow this? we could set overflow to null
+  vector<int32_t> v10 = {0, 1000, 2000, -70000, 0};
+  vector<int16_t> e10 = {0, 1000, 2000, -4464, 0};
+  CheckCase<Int32Type, int32_t, Int16Type, int16_t>(int32(), v10, is_valid, int16(), e10,
+                                                    options);
+}
+
+TEST_F(TestCast, ToDouble) {
+  CastOptions options;
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  // int16 to double
+  vector<int16_t> v1 = {0, 100, 200, 1, 2};
+  vector<double> e1 = {0, 100, 200, 1, 2};
+  CheckCase<Int16Type, int16_t, DoubleType, double>(int16(), v1, is_valid, float64(), e1,
+                                                    options);
+
+  // float to double
+  vector<float> v2 = {0, 100, 200, 1, 2};
+  vector<double> e2 = {0, 100, 200, 1, 2};
+  CheckCase<FloatType, float, DoubleType, double>(float32(), v2, is_valid, float64(), e2,
+                                                  options);
+
+  // bool to double
+  vector<bool> v3 = {true, true, false, false, true};
+  vector<double> e3 = {1, 1, 0, 0, 1};
+  CheckCase<BooleanType, bool, DoubleType, double>(boolean(), v3, is_valid, float64(), e3,
+                                                   options);
+}
+
+TEST_F(TestCast, UnsupportedTarget) {
+  vector<bool> is_valid = {true, false, true, true, true};
+  vector<int32_t> v1 = {0, 1, 2, 3, 4};
+
+  std::shared_ptr<Array> arr;
+  ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
+
+  std::shared_ptr<Array> result;
+  ASSERT_RAISES(NotImplemented, Cast(&this->ctx_, *arr, utf8(), {}, &result));
+}
+
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/context.cc b/cpp/src/arrow/compute/context.cc
new file mode 100644
index 0000000000000..792dc4f386939
--- /dev/null
+++ b/cpp/src/arrow/compute/context.cc
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/compute/context.h"
+
+#include <memory>
+
+#include "arrow/buffer.h"
+
+namespace arrow {
+namespace compute {
+
+FunctionContext::FunctionContext(MemoryPool* pool) : pool_(pool) {}
+
+MemoryPool* FunctionContext::memory_pool() const { return pool_; }
+
+Status FunctionContext::Allocate(const int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  return AllocateBuffer(pool_, nbytes, out);
+}
+
+void FunctionContext::SetStatus(const Status& status) {
+  if (ARROW_PREDICT_FALSE(!status_.ok())) {
+    return;
+  }
+  status_ = status;
+}
+
+/// \brief Clear any error status
+void FunctionContext::ResetStatus() { status_ = Status::OK(); }
+
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/context.h b/cpp/src/arrow/compute/context.h
new file mode 100644
index 0000000000000..caff2e2b743b2
--- /dev/null
+++ b/cpp/src/arrow/compute/context.h
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_COMPUTE_CONTEXT_H
+#define ARROW_COMPUTE_CONTEXT_H
+
+#include "arrow/status.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+namespace compute {
+
+#define RETURN_IF_ERROR(ctx)                  \
+  if (ARROW_PREDICT_FALSE(ctx->HasError())) { \
+    Status s = ctx->status();                 \
+    ctx->ResetStatus();                       \
+    return s;                                 \
+  }
+
+/// \brief Container for variables and options used by function evaluation
+class ARROW_EXPORT FunctionContext {
+ public:
+  explicit FunctionContext(MemoryPool* pool);
+  MemoryPool* memory_pool() const;
+
+  /// \brief Allocate buffer from the context's memory pool
+  Status Allocate(const int64_t nbytes, std::shared_ptr<Buffer>* out);
+
+  /// \brief Indicate that an error has occurred, to be checked by a parent caller
+  /// \param[in] status a Status instance
+  ///
+  /// \note Will not overwrite a prior set Status, so we will have the first
+  /// error that occurred until FunctionContext::ResetStatus is called
+  void SetStatus(const Status& status);
+
+  /// \brief Clear any error status
+  void ResetStatus();
+
+  /// \brief Return true if an error has occurred
+  bool HasError() const { return !status_.ok(); }
+
+  /// \brief Return the current status of the context
+  const Status& status() const { return status_; }
+
+ private:
+  Status status_;
+  MemoryPool* pool_;
+};
+
+}  // namespace compute
+}  // namespace arrow
+
+#endif  // ARROW_COMPUTE_CONTEXT_H
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 7fd999e2d8a86..d86fb08be8921 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -168,8 +168,8 @@ Status LoggingMemoryPool::Allocate(int64_t size, uint8_t** out) {
 
 Status LoggingMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
   Status s = pool_->Reallocate(old_size, new_size, ptr);
-  std::cout << "Reallocate: old_size = " << old_size
-            << " - new_size = " << new_size << std::endl;
+  std::cout << "Reallocate: old_size = " << old_size << " - new_size = " << new_size
+            << std::endl;
   return s;
 }
 
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 91f2bc9ccc6c1..22a933d374c08 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -232,6 +232,16 @@ void ArrayFromVector(const std::shared_ptr<DataType>& type,
   ASSERT_OK(builder.Finish(out));
 }
 
+template <typename TYPE, typename C_TYPE>
+void ArrayFromVector(const std::shared_ptr<DataType>& type,
+                     const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
+  typename TypeTraits<TYPE>::BuilderType builder(type, default_memory_pool());
+  for (size_t i = 0; i < values.size(); ++i) {
+    ASSERT_OK(builder.Append(values[i]));
+  }
+  ASSERT_OK(builder.Finish(out));
+}
+
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
                      std::shared_ptr<Array>* out) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index b532cd2276531..283e27e02c898 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -186,15 +186,20 @@ class ARROW_EXPORT PrimitiveCType : public FixedWidthType {
   using FixedWidthType::FixedWidthType;
 };
 
-class ARROW_EXPORT Integer : public PrimitiveCType {
+class ARROW_EXPORT Number : public PrimitiveCType {
  public:
   using PrimitiveCType::PrimitiveCType;
+};
+
+class ARROW_EXPORT Integer : public Number {
+ public:
+  using Number::Number;
   virtual bool is_signed() const = 0;
 };
 
-class ARROW_EXPORT FloatingPoint : public PrimitiveCType {
+class ARROW_EXPORT FloatingPoint : public Number {
  public:
-  using PrimitiveCType::PrimitiveCType;
+  using Number::Number;
   enum Precision { HALF, SINGLE, DOUBLE };
   virtual Precision precision() const = 0;
 };
@@ -842,77 +847,6 @@ std::shared_ptr<Schema> schema(
     std::vector<std::shared_ptr<Field>>&& fields,
     const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
 
-// ----------------------------------------------------------------------
-//
-
-static inline bool is_integer(Type::type type_id) {
-  switch (type_id) {
-    case Type::UINT8:
-    case Type::INT8:
-    case Type::UINT16:
-    case Type::INT16:
-    case Type::UINT32:
-    case Type::INT32:
-    case Type::UINT64:
-    case Type::INT64:
-      return true;
-    default:
-      break;
-  }
-  return false;
-}
-
-static inline bool is_floating(Type::type type_id) {
-  switch (type_id) {
-    case Type::HALF_FLOAT:
-    case Type::FLOAT:
-    case Type::DOUBLE:
-      return true;
-    default:
-      break;
-  }
-  return false;
-}
-
-static inline bool is_primitive(Type::type type_id) {
-  switch (type_id) {
-    case Type::NA:
-    case Type::BOOL:
-    case Type::UINT8:
-    case Type::INT8:
-    case Type::UINT16:
-    case Type::INT16:
-    case Type::UINT32:
-    case Type::INT32:
-    case Type::UINT64:
-    case Type::INT64:
-    case Type::HALF_FLOAT:
-    case Type::FLOAT:
-    case Type::DOUBLE:
-    case Type::DATE32:
-    case Type::DATE64:
-    case Type::TIME32:
-    case Type::TIME64:
-    case Type::TIMESTAMP:
-    case Type::INTERVAL:
-      return true;
-    default:
-      break;
-  }
-  return false;
-}
-
-static inline bool is_binary_like(Type::type type_id) {
-  switch (type_id) {
-    case Type::BINARY:
-    case Type::STRING:
-      return true;
-    default:
-      break;
-  }
-  return false;
-}
-
 }  // namespace arrow
 
 #endif  // ARROW_TYPE_H
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index d424cc81ff92a..fbd78398f4579 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -362,6 +362,74 @@ struct IsNumeric {
   static constexpr bool value = std::is_arithmetic<c_type>::value;
 };
 
+static inline bool is_integer(Type::type type_id) {
+  switch (type_id) {
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::UINT64:
+    case Type::INT64:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
+static inline bool is_floating(Type::type type_id) {
+  switch (type_id) {
+    case Type::HALF_FLOAT:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
+static inline bool is_primitive(Type::type type_id) {
+  switch (type_id) {
+    case Type::NA:
+    case Type::BOOL:
+    case Type::UINT8:
+    case Type::INT8:
+    case Type::UINT16:
+    case Type::INT16:
+    case Type::UINT32:
+    case Type::INT32:
+    case Type::UINT64:
+    case Type::INT64:
+    case Type::HALF_FLOAT:
+    case Type::FLOAT:
+    case Type::DOUBLE:
+    case Type::DATE32:
+    case Type::DATE64:
+    case Type::TIME32:
+    case Type::TIME64:
+    case Type::TIMESTAMP:
+    case Type::INTERVAL:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
+static inline bool is_binary_like(Type::type type_id) {
+  switch (type_id) {
+    case Type::BINARY:
+    case Type::STRING:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPE_TRAITS_H

From 3f2fa0381191e55e50d742bec37af0814c390b94 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 7 Sep 2017 13:01:47 -0400
Subject: [PATCH 1018/1644] ARROW-1430: [Python] Python CI build outside of a
 bash function scope, enable flake8 to fail build

This way `set -e` will have effect on any failures during the Python build. This should result in a failing build from flake8 warnings, which I will then fix

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1059 from wesm/ARROW-1430 and squashes the following commits:

33e9d6e [Wes McKinney] Also suppress E999 in Cython linting
ffbee54 [Wes McKinney] Do not lint Cython in Python 2.7
3cf5d25 [Wes McKinney] fix another warning
4149705 [Wes McKinney] Fix flake8 warnings
06b44b6 [Wes McKinney] Move flake8 checks to correct place
b26ddff [Wes McKinney] Remove unused lines
bbc3aeb [Wes McKinney] Split parquet-cpp install into separate script, invoke travis_script_python twice
---
 .travis.yml                                |   8 +-
 ci/travis_build_parquet_cpp.sh             |  50 ++++++++
 ci/travis_env_common.sh                    |   2 +
 ci/travis_script_python.sh                 | 138 +++++++--------------
 python/.flake8.cython                      |   2 +-
 python/pyarrow/includes/libarrow.pxd       |   2 +-
 python/pyarrow/tests/test_serialization.py |   5 +-
 python/pyarrow/tests/test_table.py         |   6 +-
 8 files changed, 114 insertions(+), 99 deletions(-)
 create mode 100755 ci/travis_build_parquet_cpp.sh

diff --git a/.travis.yml b/.travis.yml
index 016d1110720f4..c604fd25a7706 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -61,7 +61,9 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
   - compiler: clang
     osx_image: xcode6.4
     os: osx
@@ -73,7 +75,9 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
   - language: cpp
     before_script:
     - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
diff --git a/ci/travis_build_parquet_cpp.sh b/ci/travis_build_parquet_cpp.sh
new file mode 100755
index 0000000000000..4330a31a077a5
--- /dev/null
+++ b/ci/travis_build_parquet_cpp.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
+export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
+
+# $CPP_TOOLCHAIN set up in before_script_cpp
+export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+
+PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
+mkdir -p $PARQUET_DIR
+
+git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
+
+pushd $PARQUET_DIR
+mkdir build-dir
+cd build-dir
+
+cmake \
+    -GNinja \
+    -DCMAKE_BUILD_TYPE=debug \
+    -DCMAKE_INSTALL_PREFIX=$ARROW_PYTHON_PARQUET_HOME \
+    -DPARQUET_BOOST_USE_SHARED=off \
+    -DPARQUET_BUILD_BENCHMARKS=off \
+    -DPARQUET_BUILD_EXECUTABLES=off \
+    -DPARQUET_BUILD_TESTS=off \
+    ..
+
+ninja
+ninja install
+
+popd
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index ff49cdfc2bba0..52c7da4e0178e 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -34,6 +34,8 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
 export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
 export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
 
+export ARROW_PYTHON_PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
+
 export CMAKE_EXPORT_COMPILE_COMMANDS=1
 
 if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index b5ba136b16f65..e2cf254934892 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -21,114 +21,72 @@ set -e
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 export ARROW_HOME=$ARROW_CPP_INSTALL
-export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
+export PARQUET_HOME=$ARROW_PYTHON_PARQUET_HOME
 export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH
 export PYARROW_CXXFLAGS="-Werror"
 
-build_parquet_cpp() {
-  export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
+PYTHON_VERSION=$1
+CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
 
-  # $CPP_TOOLCHAIN set up in before_script_cpp
-  export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
+source activate $CONDA_ENV_DIR
 
-  PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
-  mkdir -p $PARQUET_DIR
+python --version
+which python
 
-  git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
+# faster builds, please
+conda install -y -q nomkl
 
-  pushd $PARQUET_DIR
-  mkdir build-dir
-  cd build-dir
+# Expensive dependencies install from Continuum package repo
+conda install -y -q pip numpy pandas cython flake8
 
-  cmake \
-      -GNinja \
-      -DCMAKE_BUILD_TYPE=debug \
-      -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
-      -DPARQUET_BOOST_USE_SHARED=off \
-      -DPARQUET_BUILD_BENCHMARKS=off \
-      -DPARQUET_BUILD_EXECUTABLES=off \
-      -DPARQUET_BUILD_TESTS=off \
-      ..
+# Build C++ libraries
+pushd $ARROW_CPP_BUILD_DIR
 
-  ninja
-  ninja install
+# Clear out prior build files
+rm -rf *
 
-  popd
-}
-
-build_parquet_cpp
-
-function rebuild_arrow_libraries() {
-  pushd $ARROW_CPP_BUILD_DIR
-
-  # Clear out prior build files
-  rm -rf *
-
-  cmake -GNinja \
-        -DARROW_BUILD_TESTS=off \
-        -DARROW_BUILD_UTILITIES=off \
-        -DARROW_PLASMA=on \
-        -DARROW_PYTHON=on \
-        -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-        $ARROW_CPP_DIR
-
-  ninja
-  ninja install
+cmake -GNinja \
+      -DARROW_BUILD_TESTS=off \
+      -DARROW_BUILD_UTILITIES=off \
+      -DARROW_PLASMA=on \
+      -DARROW_PYTHON=on \
+      -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+      $ARROW_CPP_DIR
 
-  popd
-}
+ninja
+ninja install
 
-python_version_tests() {
-  PYTHON_VERSION=$1
-  CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
+popd
 
-  conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
-  source activate $CONDA_ENV_DIR
+# Other stuff pip install
+pushd $ARROW_PYTHON_DIR
 
-  python --version
-  which python
+# Fail fast on style checks
+flake8 --count pyarrow
 
-  # faster builds, please
-  conda install -y -q nomkl
+# Check Cython files with some checks turned off
+flake8 --count --config=.flake8.cython pyarrow
 
-  # Expensive dependencies install from Continuum package repo
-  conda install -y -q pip numpy pandas cython flake8
+pip install -r requirements.txt
+python setup.py build_ext --with-parquet --with-plasma \
+       install --single-version-externally-managed --record=record.text
+popd
 
-  # Fail fast on style checks
-  flake8 --count pyarrow
+python -c "import pyarrow.parquet"
+python -c "import pyarrow.plasma"
 
-  # Check Cython files with some checks turned off
-  flake8 --count --config=.flake8.cython pyarrow
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+  export PLASMA_VALGRIND=1
+fi
 
-  # Build C++ libraries
-  rebuild_arrow_libraries
+PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
+python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
 
-  # Other stuff pip install
-  pushd $ARROW_PYTHON_DIR
-  pip install -r requirements.txt
-  python setup.py build_ext --with-parquet --with-plasma \
-         install --single-version-externally-managed --record=record.text
+if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
+  # Build documentation once
+  pushd $ARROW_PYTHON_DIR/doc
+  conda install -y -q --file=requirements.txt
+  sphinx-build -b html -d _build/doctrees -W source _build/html
   popd
-
-  python -c "import pyarrow.parquet"
-  python -c "import pyarrow.plasma"
-
-  if [ $TRAVIS_OS_NAME == "linux" ]; then
-    export PLASMA_VALGRIND=1
-  fi
-
-  PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
-  python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
-
-  if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
-      # Build documentation once
-      pushd $ARROW_PYTHON_DIR/doc
-      conda install -y -q --file=requirements.txt
-      sphinx-build -b html -d _build/doctrees -W source _build/html
-      popd
-  fi
-}
-
-# run tests for python 2.7 and 3.6
-python_version_tests 2.7
-python_version_tests 3.6
+fi
diff --git a/python/.flake8.cython b/python/.flake8.cython
index 53e41323051f9..9255bda624707 100644
--- a/python/.flake8.cython
+++ b/python/.flake8.cython
@@ -17,4 +17,4 @@
 
 [flake8]
 filename = *.pyx,*.pxd,*.pxi
-ignore = E211,E901,E225,E226,E227
+ignore = E211,E901,E999,E225,E226,E227
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 756b213b29e3d..cc356848709a0 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -243,7 +243,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CSchema] RemoveMetadata()
 
     cdef cppclass PrettyPrintOptions:
-        int indent;
+        int indent
 
     CStatus PrettyPrint(const CSchema& schema,
                         const PrettyPrintOptions& options,
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 27243b0000d18..aca3848ae9c29 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -77,8 +77,9 @@ def assert_equal(obj1, obj2):
         for i in range(len(obj1)):
             assert_equal(obj1[i], obj2[i])
     else:
-        assert obj1 == obj2, "Objects {} and {} are different.".format(obj1,
-                                                                       obj2)
+        assert obj1 == obj2, ("Objects {} and {} are different."
+                              .format(obj1, obj2))
+
 
 PRIMITIVE_OBJECTS = [
     0, 0.0, 0.9, 1 << 62, 1 << 100, 1 << 999,
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 424e5184c0d3d..4d5cb364c695d 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -135,9 +135,9 @@ def test_recordbatch_from_to_pandas():
     data = pd.DataFrame({
         'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),
         'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'),
-        'c2': np.random.randn(5),
-        'c3': ['foo', 'bar', None, 'baz', 'qux'],
-        'c4': [False, True, False, True, False]
+        'c3': np.random.randn(5),
+        'c4': ['foo', 'bar', None, 'baz', 'qux'],
+        'c5': [False, True, False, True, False]
     })
 
     batch = pa.RecordBatch.from_pandas(data)

From b698227e9c20bfc24dbf04b747b062881f1f1ad7 Mon Sep 17 00:00:00 2001
From: Jeff Reback <jeff@reback.net>
Date: Thu, 7 Sep 2017 14:09:01 -0400
Subject: [PATCH 1019/1644] ARROW-622 [Python] deprecate timestamps_to_ms in
 .from_pandas()

xref https://github.com/pandas-dev/pandas/issues/17438

this was not fully resolved in https://github.com/apache/arrow/pull/944

Author: Jeff Reback <jeff@reback.net>

Closes #1046 from jreback/warn and squashes the following commits:

382592f [Jeff Reback] deprecate timestamps_to_ms in .from_pandas()
---
 python/pyarrow/array.pxi                    |  2 +
 python/pyarrow/table.pxi                    |  3 +
 python/pyarrow/tests/test_convert_pandas.py | 62 +++++----------------
 3 files changed, 19 insertions(+), 48 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 20e778d068ff8..a693f45e967a5 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -121,6 +121,8 @@ cdef class Array:
             compatibility with other functionality like Parquet I/O which
             only supports milliseconds.
 
+            .. deprecated:: 0.7.0
+
         memory_pool: MemoryPool, optional
             Specific memory pool to use to allocate the resulting Arrow array.
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 245371ffaa124..fc6099fe8430d 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -723,6 +723,9 @@ cdef class Table:
             Convert datetime columns to ms resolution. This is needed for
             compability with other functionality like Parquet I/O which
             only supports milliseconds.
+
+            .. deprecated:: 0.7.0
+
         schema : pyarrow.Schema, optional
             The expected schema of the Arrow Table. This can be used to
             indicate the type of columns if we cannot infer it automatically.
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 52290d6d85533..64424341067bb 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -69,10 +69,10 @@ def tearDown(self):
         pass
 
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
-                                timestamps_to_ms=False, expected_schema=None,
+                                expected_schema=None,
                                 check_dtype=True, schema=None,
                                 check_index=False):
-        table = pa.Table.from_pandas(df, timestamps_to_ms=timestamps_to_ms,
+        table = pa.Table.from_pandas(df,
                                      schema=schema, preserve_index=check_index)
         result = table.to_pandas(nthreads=nthreads)
         if expected_schema:
@@ -92,9 +92,8 @@ def _check_series_roundtrip(self, s, type_=None):
         tm.assert_series_equal(s, result)
 
     def _check_array_roundtrip(self, values, expected=None, mask=None,
-                               timestamps_to_ms=False, type=None):
-        arr = pa.Array.from_pandas(values, timestamps_to_ms=timestamps_to_ms,
-                                   mask=mask, type=type)
+                               type=None):
+        arr = pa.Array.from_pandas(values, mask=mask, type=type)
         result = arr.to_pandas()
 
         values_nulls = pd.isnull(values)
@@ -332,21 +331,6 @@ def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
             pa.Table.from_pandas(df, schema=schema)
 
     def test_timestamps_notimezone_no_nulls(self):
-        df = pd.DataFrame({
-            'datetime64': np.array([
-                '2007-07-13T01:23:34.123',
-                '2006-01-13T12:34:56.432',
-                '2010-08-13T05:46:57.437'],
-                dtype='datetime64[ms]')
-        })
-        field = pa.field('datetime64', pa.timestamp('ms'))
-        schema = pa.schema([field])
-        self._check_pandas_roundtrip(
-            df,
-            timestamps_to_ms=True,
-            expected_schema=schema,
-        )
-
         df = pd.DataFrame({
             'datetime64': np.array([
                 '2007-07-13T01:23:34.123456789',
@@ -357,7 +341,8 @@ def test_timestamps_notimezone_no_nulls(self):
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
         self._check_pandas_roundtrip(
-            df, expected_schema=schema,
+            df,
+            expected_schema=schema,
         )
 
     def test_timestamps_to_ms_explicit_schema(self):
@@ -365,28 +350,15 @@ def test_timestamps_to_ms_explicit_schema(self):
         df = pd.DataFrame({'datetime': [datetime(2017, 1, 1)]})
         pa_type = pa.from_numpy_dtype(df['datetime'].dtype)
 
-        arr = pa.Array.from_pandas(df['datetime'], type=pa_type,
-                                   timestamps_to_ms=True)
+        with tm.assert_produces_warning(FutureWarning,
+                                        check_stacklevel=False):
+            arr = pa.Array.from_pandas(df['datetime'], type=pa_type,
+                                       timestamps_to_ms=True)
 
         tm.assert_almost_equal(df['datetime'].values.astype('M8[ms]'),
                                arr.to_pandas())
 
     def test_timestamps_notimezone_nulls(self):
-        df = pd.DataFrame({
-            'datetime64': np.array([
-                '2007-07-13T01:23:34.123',
-                None,
-                '2010-08-13T05:46:57.437'],
-                dtype='datetime64[ms]')
-        })
-        field = pa.field('datetime64', pa.timestamp('ms'))
-        schema = pa.schema([field])
-        self._check_pandas_roundtrip(
-            df,
-            timestamps_to_ms=True,
-            expected_schema=schema,
-        )
-
         df = pd.DataFrame({
             'datetime64': np.array([
                 '2007-07-13T01:23:34.123456789',
@@ -397,7 +369,8 @@ def test_timestamps_notimezone_nulls(self):
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
         self._check_pandas_roundtrip(
-            df, expected_schema=schema,
+            df,
+            expected_schema=schema,
         )
 
     def test_timestamps_with_timezone(self):
@@ -410,7 +383,7 @@ def test_timestamps_with_timezone(self):
         })
         df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
                             .to_frame())
-        self._check_pandas_roundtrip(df, timestamps_to_ms=True)
+        self._check_pandas_roundtrip(df)
 
         self._check_series_roundtrip(df['datetime64'])
 
@@ -425,15 +398,8 @@ def test_timestamps_with_timezone(self):
         })
         df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
                             .to_frame())
-        self._check_pandas_roundtrip(df)
-
-    def test_timestamp_with_tz_to_pandas_type(self):
-        from pyarrow.compat import DatetimeTZDtype
 
-        tz = 'America/Los_Angeles'
-        t = pa.timestamp('ns', tz=tz)
-
-        assert t.to_pandas_dtype() == DatetimeTZDtype('ns', tz=tz)
+        self._check_pandas_roundtrip(df)
 
     def test_date_infer(self):
         df = pd.DataFrame({

From 6f27a6447171353427a129a5ce88dba181bd8af6 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Thu, 7 Sep 2017 21:48:25 -0400
Subject: [PATCH 1020/1644] =?UTF-8?q?ARROW-1467:=20[JAVA]=20Fix=20reset()?=
 =?UTF-8?q?=20and=20allocateNew()=20in=20Nullable=20Value=20Vectors=20t?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

cc @jacques-n , @icexelloss , @BryanCutler

Things fixed:

1)

allocateNew() in NullableValueVectors allocates extra memory for the validity vector of fixed-width vectors. Instead of doing bits.allocateNew(valueCount + 1), we should simply do bits.allocateNew(valueCount).

AFAIK, the only case where we need an additional valueCount is for the offsetVector and we already do that. Additional valueCount for the validity vector is not needed.

(2)

reset() method should call reset() on the underlying value vector to re-initialize the state (allocation monitor, reader index etc) and zero out the buffers. Right now we just reset the validity vector.

Author: siddharth <siddharth@dremio.com>

Closes #1052 from siddharthteotia/ARROW-1467 and squashes the following commits:

ac903884 [siddharth] addressed review comments
09f899af [siddharth] ARROW-1467: Fix reset() and allocateNew() in Nullable Value Vectors template
---
 .../templates/NullableValueVectors.java       | 20 +++++++------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 319c61c86244b..122cd23648286 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -267,6 +267,12 @@ public void reAlloc() {
     values.reAlloc();
   }
 
+  public void reset() {
+    bits.zeroVector();
+    mutator.reset();
+    accessor.reset();
+  }
+
   <#if type.major == "VarLen">
   @Override
   public void allocateNew(int totalBytes, int valueCount) {
@@ -282,12 +288,6 @@ public void allocateNew(int totalBytes, int valueCount) {
     accessor.reset();
   }
 
-  public void reset() {
-    bits.zeroVector();
-    mutator.reset();
-    accessor.reset();
-  }
-
   @Override
   public int getByteCapacity(){
     return values.getByteCapacity();
@@ -303,7 +303,7 @@ public int getCurrentSizeInBytes(){
   public void allocateNew(int valueCount) {
     try {
       values.allocateNew(valueCount);
-      bits.allocateNew(valueCount+1);
+      bits.allocateNew(valueCount);
     } catch(OutOfMemoryException e) {
       clear();
       throw e;
@@ -313,12 +313,6 @@ public void allocateNew(int valueCount) {
     accessor.reset();
   }
 
-  public void reset() {
-    bits.zeroVector();
-    mutator.reset();
-    accessor.reset();
-  }
-
   /**
    * {@inheritDoc}
    */

From fe45c2bb3718aa5fb0c34c1a244adcdff9477c1a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 8 Sep 2017 00:01:22 -0400
Subject: [PATCH 1021/1644] ARROW-1490: [Java] Allow failures for JDK9 for now

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1064 from wesm/jdk9-allow-failures and squashes the following commits:

dfcaadd5 [Wes McKinney] Allow failures for JDK9 for now
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index c604fd25a7706..76ef8a7d4aea6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,7 +46,7 @@ cache:
 matrix:
   fast_finish: true
   allow_failures:
-  - env: ARROW_TEST_GROUP=packaging
+  - jdk: oraclejdk9
   include:
   - compiler: gcc
     language: cpp

From de2edc8d591ac8e889495392f8ed20d1d7814dea Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 8 Sep 2017 10:09:38 -0400
Subject: [PATCH 1022/1644] ARROW-1156: [C++/Python] Expand casting API, add
 UnaryKernel callable. Use Cast in appropriate places when converting from
 pandas

cc @cloud-fan

With this patch we now try to cast to indicated type on ingest of objects from pandas:

```
In [3]: arr = np.array([None] * 5)

In [4]: pa.Array.from_pandas(arr)
Out[4]:
<pyarrow.lib.NullArray object at 0x7f6cf1485d18>
[
  NA,
  NA,
  NA,
  NA,
  NA
]

In [5]: pa.Array.from_pandas(arr, type=pa.int32())
Out[5]:
<pyarrow.lib.Int32Array object at 0x7f6cf1485d68>
[
  NA,
  NA,
  NA,
  NA,
  NA
]
```

I also added zero-copy casts from integers of the right size to each of the date and time types.

Includes refactoring for ARROW-1481.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1063 from wesm/ARROW-1156 and squashes the following commits:

166d1a50 [Wes McKinney] iwyu
34f5c9d1 [Wes McKinney] Harden default cast options, fix unsafe Python case
1d07b756 [Wes McKinney] Add some basic casting unit tests in Python
c1b45709 [Wes McKinney] Expose arrow::compute::Cast in Python as Array.cast. Still need to write tests
a9a04c9c [Wes McKinney] UnaryKernel::Call returns Status for now for simplicity. Support pre-allocated memory
8903709b [Wes McKinney] Implement casts from null to numbers. Try to cast for types where we do not have an inference rule when converting from arrays of Python objects
a22dd20a [Wes McKinney] Add test to assert zero copy for compatible integer to date/time
a14b83f7 [Wes McKinney] Create callable CastKernel object. Add zero-copy cast rules for date/time types
---
 cpp/README.md                                 |   3 +
 .../iwyu/mappings/arrow-misc.imp              |   5 +-
 cpp/src/arrow/array.h                         |  28 +-
 cpp/src/arrow/compute/CMakeLists.txt          |   2 +
 cpp/src/arrow/compute/api.h                   |  25 ++
 cpp/src/arrow/compute/cast.cc                 | 358 ++++++++++++------
 cpp/src/arrow/compute/cast.h                  |  14 +-
 cpp/src/arrow/compute/compute-test.cc         |  96 ++++-
 cpp/src/arrow/compute/context.h               |   3 +-
 cpp/src/arrow/compute/kernel.h                |  49 +++
 cpp/src/arrow/python/numpy_convert.h          |   3 -
 cpp/src/arrow/python/pandas_to_arrow.cc       | 172 +++++----
 python/pyarrow/array.pxi                      |  41 ++
 python/pyarrow/includes/libarrow.pxd          |  18 +-
 python/pyarrow/tests/test_array.py            |  67 ++++
 python/pyarrow/tests/test_convert_pandas.py   |  20 +
 16 files changed, 687 insertions(+), 217 deletions(-)
 create mode 100644 cpp/src/arrow/compute/api.h
 create mode 100644 cpp/src/arrow/compute/kernel.h

diff --git a/cpp/README.md b/cpp/README.md
index 993efb81af7fa..4a515079ddc01 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -126,6 +126,9 @@ interfaces in this library as needed.
 The CUDA toolchain used to build the library can be customized by using the
 `$CUDA_HOME` environment variable.
 
+This library is still in Alpha stages, and subject to API changes without
+deprecation warnings.
+
 ### API documentation
 
 To generate the (html) API documentation, run the following command in the apidoc
diff --git a/cpp/build-support/iwyu/mappings/arrow-misc.imp b/cpp/build-support/iwyu/mappings/arrow-misc.imp
index 7d9f09c0cef71..f39650d962d58 100644
--- a/cpp/build-support/iwyu/mappings/arrow-misc.imp
+++ b/cpp/build-support/iwyu/mappings/arrow-misc.imp
@@ -24,9 +24,10 @@
   { symbol: ["uint8_t", private, "<cstdint>", public ] },
   { symbol: ["_Node_const_iterator", private, "<flatbuffers/flatbuffers.h>", public ] },
   { symbol: ["unordered_map<>::mapped_type", private, "<flatbuffers/flatbuffers.h>", public ] },
-  { symbol: ["__alloc_traits<>::value_type", private, "<vector>", public ] },
   { symbol: ["move", private, "<utility>", public ] },
   { symbol: ["pair", private, "<utility>", public ] },
   { symbol: ["errno", private, "<cerrno>", public ] },
-  { symbol: ["posix_memalign", private, "<cstdlib>", public ] }
+  { symbol: ["posix_memalign", private, "<cstdlib>", public ] },
+  { include: ["<ext/alloc_traits.h>", private, "<vector>", public ] },
+  { include: ["<string.h>", private, "<cstring>", public ] }
 ]
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 61ab2ef4aafd9..3faff714754c7 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -87,48 +87,48 @@ namespace internal {
 struct ARROW_EXPORT ArrayData {
   ArrayData() {}
 
+  ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
+            int64_t null_count = kUnknownNullCount, int64_t offset = 0)
+      : type(type), length(length), null_count(null_count), offset(offset) {}
+
   ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
             const std::vector<std::shared_ptr<Buffer>>& buffers,
             int64_t null_count = kUnknownNullCount, int64_t offset = 0)
-      : type(type),
-        length(length),
-        buffers(buffers),
-        null_count(null_count),
-        offset(offset) {}
+      : ArrayData(type, length, null_count, offset) {
+    this->buffers = buffers;
+  }
 
   ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
             std::vector<std::shared_ptr<Buffer>>&& buffers,
             int64_t null_count = kUnknownNullCount, int64_t offset = 0)
-      : type(type),
-        length(length),
-        buffers(std::move(buffers)),
-        null_count(null_count),
-        offset(offset) {}
+      : ArrayData(type, length, null_count, offset) {
+    this->buffers = std::move(buffers);
+  }
 
   // Move constructor
   ArrayData(ArrayData&& other) noexcept
       : type(std::move(other.type)),
         length(other.length),
-        buffers(std::move(other.buffers)),
         null_count(other.null_count),
         offset(other.offset),
+        buffers(std::move(other.buffers)),
         child_data(std::move(other.child_data)) {}
 
   ArrayData(const ArrayData& other) noexcept
       : type(other.type),
         length(other.length),
-        buffers(other.buffers),
         null_count(other.null_count),
         offset(other.offset),
+        buffers(other.buffers),
         child_data(other.child_data) {}
 
   // Move assignment
   ArrayData& operator=(ArrayData&& other) {
     type = std::move(other.type);
     length = other.length;
-    buffers = std::move(other.buffers);
     null_count = other.null_count;
     offset = other.offset;
+    buffers = std::move(other.buffers);
     child_data = std::move(other.child_data);
     return *this;
   }
@@ -139,9 +139,9 @@ struct ARROW_EXPORT ArrayData {
 
   std::shared_ptr<DataType> type;
   int64_t length;
-  std::vector<std::shared_ptr<Buffer>> buffers;
   int64_t null_count;
   int64_t offset;
+  std::vector<std::shared_ptr<Buffer>> buffers;
   std::vector<std::shared_ptr<ArrayData>> child_data;
 };
 
diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt
index a154c47c87463..fa475ca9fb0ac 100644
--- a/cpp/src/arrow/compute/CMakeLists.txt
+++ b/cpp/src/arrow/compute/CMakeLists.txt
@@ -17,8 +17,10 @@
 
 # Headers: top level
 install(FILES
+  api.h
   cast.h
   context.h
+  kernel.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute")
 
 #######################################
diff --git a/cpp/src/arrow/compute/api.h b/cpp/src/arrow/compute/api.h
new file mode 100644
index 0000000000000..da7df1cbbfafc
--- /dev/null
+++ b/cpp/src/arrow/compute/api.h
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_COMPUTE_API_H
+#define ARROW_COMPUTE_API_H
+
+#include "arrow/compute/cast.h"
+#include "arrow/compute/context.h"
+#include "arrow/compute/kernel.h"
+
+#endif  // ARROW_COMPUTE_API_H
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index f610f6b964c31..3885fdfb57963 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -18,40 +18,66 @@
 #include "arrow/compute/cast.h"
 
 #include <cstdint>
+#include <cstring>
 #include <functional>
 #include <limits>
 #include <memory>
 #include <sstream>
+#include <string>
 #include <type_traits>
 
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/type.h"
 #include "arrow/type_traits.h"
+#include "arrow/util/bit-util.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
 
 #include "arrow/compute/context.h"
+#include "arrow/compute/kernel.h"
 
 namespace arrow {
 namespace compute {
 
-struct CastContext {
-  FunctionContext* func_ctx;
-  CastOptions options;
+// ----------------------------------------------------------------------
+// Zero copy casts
+
+template <typename O, typename I, typename Enable = void>
+struct is_zero_copy_cast {
+  static constexpr bool value = false;
 };
 
-typedef std::function<void(CastContext*, const ArrayData&, ArrayData*)> CastFunction;
+template <typename O, typename I>
+struct is_zero_copy_cast<O, I, typename std::enable_if<std::is_same<I, O>::value>::type> {
+  static constexpr bool value = true;
+};
+
+// From integers to date/time types with zero copy
+template <typename O, typename I>
+struct is_zero_copy_cast<
+    O, I, typename std::enable_if<std::is_base_of<Integer, I>::value &&
+                                  (std::is_base_of<TimeType, O>::value ||
+                                   std::is_base_of<DateType, O>::value ||
+                                   std::is_base_of<TimestampType, O>::value)>::type> {
+  using O_T = typename O::c_type;
+  using I_T = typename I::c_type;
+
+  static constexpr bool value = sizeof(O_T) == sizeof(I_T);
+};
 
 template <typename OutType, typename InType, typename Enable = void>
 struct CastFunctor {};
 
-// Type is the same, no computation required
+// Indicated no computation required
 template <typename O, typename I>
-struct CastFunctor<O, I, typename std::enable_if<std::is_same<I, O>::value>::type> {
-  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
-    output->type = input.type;
-    output->buffers = input.buffers;
-    output->length = input.length;
-    output->offset = input.offset;
-    output->null_count = input.null_count;
-    output->child_data = input.child_data;
+struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>::type> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    auto in_data = input.data();
+    output->null_count = input.null_count();
+    output->buffers = in_data->buffers;
+    output->child_data = in_data->child_data;
   }
 };
 
@@ -59,10 +85,13 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<I, O>::value>::typ
 // Null to other things
 
 template <typename T>
-struct CastFunctor<T, NullType,
-                   typename std::enable_if<!std::is_same<T, NullType>::value>::type> {
-  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
-    ctx->func_ctx->SetStatus(Status::NotImplemented("NullType"));
+struct CastFunctor<T, NullType, typename std::enable_if<
+                                    std::is_base_of<FixedWidthType, T>::value>::type> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    // Simply initialize data to 0
+    auto buf = output->buffers[1];
+    memset(buf->mutable_data(), 0, buf->size());
   }
 };
 
@@ -73,13 +102,14 @@ struct CastFunctor<T, NullType,
 template <typename T>
 struct CastFunctor<T, BooleanType,
                    typename std::enable_if<std::is_base_of<Number, T>::value>::type> {
-  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
     using c_type = typename T::c_type;
-    const uint8_t* data = input.buffers[1]->data();
+    const uint8_t* data = input.data()->buffers[1]->data();
     auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
     constexpr auto kOne = static_cast<c_type>(1);
     constexpr auto kZero = static_cast<c_type>(0);
-    for (int64_t i = 0; i < input.length; ++i) {
+    for (int64_t i = 0; i < input.length(); ++i) {
       *out++ = BitUtil::GetBit(data, i) ? kOne : kZero;
     }
   }
@@ -122,11 +152,12 @@ template <typename O, typename I>
 struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::value &&
                                                  std::is_base_of<Number, I>::value &&
                                                  !std::is_same<O, I>::value>::type> {
-  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
     using in_type = typename I::c_type;
-    auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data());
+    auto in_data = reinterpret_cast<const in_type*>(input.data()->buffers[1]->data());
     uint8_t* out_data = reinterpret_cast<uint8_t*>(output->buffers[1]->mutable_data());
-    for (int64_t i = 0; i < input.length; ++i) {
+    for (int64_t i = 0; i < input.length(); ++i) {
       BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
     }
   }
@@ -135,39 +166,42 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::v
 template <typename O, typename I>
 struct CastFunctor<O, I,
                    typename std::enable_if<is_integer_downcast<O, I>::value>::type> {
-  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
 
-    auto in_offset = input.offset;
+    auto in_offset = input.offset();
 
-    auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data()) + in_offset;
+    const auto& input_buffers = input.data()->buffers;
+
+    auto in_data = reinterpret_cast<const in_type*>(input_buffers[1]->data()) + in_offset;
     auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
 
-    if (!ctx->options.allow_int_overflow) {
+    if (!options.allow_int_overflow) {
       constexpr in_type kMax = static_cast<in_type>(std::numeric_limits<out_type>::max());
       constexpr in_type kMin = static_cast<in_type>(std::numeric_limits<out_type>::min());
 
-      if (input.null_count > 0) {
-        const uint8_t* is_valid = input.buffers[0]->data();
+      if (input.null_count() > 0) {
+        const uint8_t* is_valid = input_buffers[0]->data();
         int64_t is_valid_offset = in_offset;
-        for (int64_t i = 0; i < input.length; ++i) {
+        for (int64_t i = 0; i < input.length(); ++i) {
           if (ARROW_PREDICT_FALSE(BitUtil::GetBit(is_valid, is_valid_offset++) &&
                                   (*in_data > kMax || *in_data < kMin))) {
-            ctx->func_ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
+            ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
           }
           *out_data++ = static_cast<out_type>(*in_data++);
         }
       } else {
-        for (int64_t i = 0; i < input.length; ++i) {
+        for (int64_t i = 0; i < input.length(); ++i) {
           if (ARROW_PREDICT_FALSE(*in_data > kMax || *in_data < kMin)) {
-            ctx->func_ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
+            ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
           }
           *out_data++ = static_cast<out_type>(*in_data++);
         }
       }
     } else {
-      for (int64_t i = 0; i < input.length; ++i) {
+      for (int64_t i = 0; i < input.length(); ++i) {
         *out_data++ = static_cast<out_type>(*in_data++);
       }
     }
@@ -178,13 +212,14 @@ template <typename O, typename I>
 struct CastFunctor<O, I,
                    typename std::enable_if<is_numeric_cast<O, I>::value &&
                                            !is_integer_downcast<O, I>::value>::type> {
-  void operator()(CastContext* ctx, const ArrayData& input, ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
 
-    auto in_data = reinterpret_cast<const in_type*>(input.buffers[1]->data());
+    auto in_data = reinterpret_cast<const in_type*>(input.data()->buffers[1]->data());
     auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
-    for (int64_t i = 0; i < input.length; ++i) {
+    for (int64_t i = 0; i < input.length(); ++i) {
       *out_data++ = static_cast<out_type>(*in_data++);
     }
   }
@@ -192,12 +227,90 @@ struct CastFunctor<O, I,
 
 // ----------------------------------------------------------------------
 
-#define CAST_CASE(InType, OutType)                                            \
-  case OutType::type_id:                                                      \
-    return [type](CastContext* ctx, const ArrayData& input, ArrayData* out) { \
-      CastFunctor<OutType, InType> func;                                      \
-      func(ctx, input, out);                                                  \
+typedef std::function<void(FunctionContext*, const CastOptions& options, const Array&,
+                           ArrayData*)>
+    CastFunction;
+
+static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input,
+                                        ArrayData* out) {
+  if (!is_primitive(out->type->id())) {
+    return Status::NotImplemented(out->type->ToString());
+  }
+
+  const auto& fw_type = static_cast<const FixedWidthType&>(*out->type);
+
+  const int64_t length = input.length();
+
+  out->null_count = input.null_count();
+
+  // Propagate bitmap unless we are null type
+  std::shared_ptr<Buffer> validity_bitmap = input.data()->buffers[0];
+  if (input.type_id() == Type::NA) {
+    int64_t bitmap_size = BitUtil::BytesForBits(length);
+    RETURN_NOT_OK(ctx->Allocate(bitmap_size, &validity_bitmap));
+    memset(validity_bitmap->mutable_data(), 0, bitmap_size);
+  }
+
+  if (out->buffers.size() == 2) {
+    // Assuming preallocated, propagage bitmap and move on
+    out->buffers[0] = validity_bitmap;
+    return Status::OK();
+  } else {
+    DCHECK_EQ(0, out->buffers.size());
+  }
+
+  out->buffers.push_back(validity_bitmap);
+
+  std::shared_ptr<Buffer> out_data;
+
+  int bit_width = fw_type.bit_width();
+  int64_t buffer_size = 0;
+
+  if (bit_width == 1) {
+    buffer_size = BitUtil::BytesForBits(length);
+  } else if (bit_width % 8 == 0) {
+    buffer_size = length * fw_type.bit_width() / 8;
+  } else {
+    DCHECK(false);
+  }
+
+  RETURN_NOT_OK(ctx->Allocate(buffer_size, &out_data));
+  memset(out_data->mutable_data(), 0, buffer_size);
+
+  out->buffers.push_back(out_data);
+
+  return Status::OK();
+}
+
+class CastKernel : public UnaryKernel {
+ public:
+  CastKernel(const CastOptions& options, const CastFunction& func, bool is_zero_copy)
+      : options_(options), func_(func), is_zero_copy_(is_zero_copy) {}
+
+  Status Call(FunctionContext* ctx, const Array& input, ArrayData* out) override {
+    if (!is_zero_copy_) {
+      RETURN_NOT_OK(AllocateIfNotPreallocated(ctx, input, out));
     }
+    func_(ctx, options_, input, out);
+    RETURN_IF_ERROR(ctx);
+    return Status::OK();
+  }
+
+ private:
+  CastOptions options_;
+  CastFunction func_;
+  bool is_zero_copy_;
+};
+
+#define CAST_CASE(InType, OutType)                                                  \
+  case OutType::type_id:                                                            \
+    is_zero_copy = is_zero_copy_cast<OutType, InType>::value;                       \
+    func = [](FunctionContext* ctx, const CastOptions& options, const Array& input, \
+              ArrayData* out) {                                                     \
+      CastFunctor<OutType, InType> func;                                            \
+      func(ctx, options, input, out);                                               \
+    };                                                                              \
+    break;
 
 #define NUMERIC_CASES(FN, IN_TYPE) \
   FN(IN_TYPE, BooleanType);        \
@@ -212,37 +325,78 @@ struct CastFunctor<O, I,
   FN(IN_TYPE, FloatType);          \
   FN(IN_TYPE, DoubleType);
 
-#define GET_CAST_FUNCTION(CapType)                                                    \
-  static CastFunction Get##CapType##CastFunc(const std::shared_ptr<DataType>& type) { \
-    switch (type->id()) {                                                             \
-      NUMERIC_CASES(CAST_CASE, CapType);                                              \
-      default:                                                                        \
-        break;                                                                        \
-    }                                                                                 \
-    return nullptr;                                                                   \
+#define NULL_CASES(FN, IN_TYPE) \
+  NUMERIC_CASES(FN, IN_TYPE)    \
+  FN(NullType, Time32Type);     \
+  FN(NullType, Date32Type);     \
+  FN(NullType, TimestampType);  \
+  FN(NullType, Time64Type);     \
+  FN(NullType, Date64Type);
+
+#define INT32_CASES(FN, IN_TYPE) \
+  NUMERIC_CASES(FN, IN_TYPE)     \
+  FN(Int32Type, Time32Type);     \
+  FN(Int32Type, Date32Type);
+
+#define INT64_CASES(FN, IN_TYPE) \
+  NUMERIC_CASES(FN, IN_TYPE)     \
+  FN(Int64Type, TimestampType);  \
+  FN(Int64Type, Time64Type);     \
+  FN(Int64Type, Date64Type);
+
+#define DATE32_CASES(FN, IN_TYPE) FN(Date32Type, Date32Type);
+
+#define DATE64_CASES(FN, IN_TYPE) FN(Date64Type, Date64Type);
+
+#define TIME32_CASES(FN, IN_TYPE) FN(Time32Type, Time32Type);
+
+#define TIME64_CASES(FN, IN_TYPE) FN(Time64Type, Time64Type);
+
+#define TIMESTAMP_CASES(FN, IN_TYPE) FN(TimestampType, TimestampType);
+
+#define GET_CAST_FUNCTION(CASE_GENERATOR, InType)                                       \
+  static std::unique_ptr<UnaryKernel> Get##InType##CastFunc(                            \
+      const std::shared_ptr<DataType>& out_type, const CastOptions& options) {          \
+    CastFunction func;                                                                  \
+    bool is_zero_copy = false;                                                          \
+    switch (out_type->id()) {                                                           \
+      CASE_GENERATOR(CAST_CASE, InType);                                                \
+      default:                                                                          \
+        break;                                                                          \
+    }                                                                                   \
+    if (func != nullptr) {                                                              \
+      return std::unique_ptr<UnaryKernel>(new CastKernel(options, func, is_zero_copy)); \
+    }                                                                                   \
+    return nullptr;                                                                     \
   }
 
-#define CAST_FUNCTION_CASE(CapType)          \
-  case CapType::type_id:                     \
-    *out = Get##CapType##CastFunc(out_type); \
+GET_CAST_FUNCTION(NULL_CASES, NullType);
+GET_CAST_FUNCTION(NUMERIC_CASES, BooleanType);
+GET_CAST_FUNCTION(NUMERIC_CASES, UInt8Type);
+GET_CAST_FUNCTION(NUMERIC_CASES, Int8Type);
+GET_CAST_FUNCTION(NUMERIC_CASES, UInt16Type);
+GET_CAST_FUNCTION(NUMERIC_CASES, Int16Type);
+GET_CAST_FUNCTION(NUMERIC_CASES, UInt32Type);
+GET_CAST_FUNCTION(INT32_CASES, Int32Type);
+GET_CAST_FUNCTION(NUMERIC_CASES, UInt64Type);
+GET_CAST_FUNCTION(INT64_CASES, Int64Type);
+GET_CAST_FUNCTION(NUMERIC_CASES, FloatType);
+GET_CAST_FUNCTION(NUMERIC_CASES, DoubleType);
+GET_CAST_FUNCTION(DATE32_CASES, Date32Type);
+GET_CAST_FUNCTION(DATE64_CASES, Date64Type);
+GET_CAST_FUNCTION(TIME32_CASES, Time32Type);
+GET_CAST_FUNCTION(TIME64_CASES, Time64Type);
+GET_CAST_FUNCTION(TIMESTAMP_CASES, TimestampType);
+
+#define CAST_FUNCTION_CASE(InType)                      \
+  case InType::type_id:                                 \
+    *kernel = Get##InType##CastFunc(out_type, options); \
     break
 
-GET_CAST_FUNCTION(BooleanType);
-GET_CAST_FUNCTION(UInt8Type);
-GET_CAST_FUNCTION(Int8Type);
-GET_CAST_FUNCTION(UInt16Type);
-GET_CAST_FUNCTION(Int16Type);
-GET_CAST_FUNCTION(UInt32Type);
-GET_CAST_FUNCTION(Int32Type);
-GET_CAST_FUNCTION(UInt64Type);
-GET_CAST_FUNCTION(Int64Type);
-GET_CAST_FUNCTION(FloatType);
-GET_CAST_FUNCTION(DoubleType);
-
-static Status GetCastFunction(const DataType& in_type,
-                              const std::shared_ptr<DataType>& out_type,
-                              CastFunction* out) {
+Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>& out_type,
+                       const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel) {
   switch (in_type.id()) {
+    CAST_FUNCTION_CASE(NullType);
     CAST_FUNCTION_CASE(BooleanType);
     CAST_FUNCTION_CASE(UInt8Type);
     CAST_FUNCTION_CASE(Int8Type);
@@ -254,10 +408,15 @@ static Status GetCastFunction(const DataType& in_type,
     CAST_FUNCTION_CASE(Int64Type);
     CAST_FUNCTION_CASE(FloatType);
     CAST_FUNCTION_CASE(DoubleType);
+    CAST_FUNCTION_CASE(Date32Type);
+    CAST_FUNCTION_CASE(Date64Type);
+    CAST_FUNCTION_CASE(Time32Type);
+    CAST_FUNCTION_CASE(Time64Type);
+    CAST_FUNCTION_CASE(TimestampType);
     default:
       break;
   }
-  if (*out == nullptr) {
+  if (*kernel == nullptr) {
     std::stringstream ss;
     ss << "No cast implemented from " << in_type.ToString() << " to "
        << out_type->ToString();
@@ -266,64 +425,19 @@ static Status GetCastFunction(const DataType& in_type,
   return Status::OK();
 }
 
-static Status AllocateLike(FunctionContext* ctx, const Array& array,
-                           const std::shared_ptr<DataType>& out_type,
-                           std::shared_ptr<ArrayData>* out) {
-  if (!is_primitive(out_type->id())) {
-    return Status::NotImplemented(out_type->ToString());
-  }
-
-  const auto& fw_type = static_cast<const FixedWidthType&>(*out_type);
-
-  auto result = std::make_shared<ArrayData>();
-  result->type = out_type;
-  result->length = array.length();
-  result->offset = 0;
-  result->null_count = array.null_count();
-
-  // Propagate null bitmap
-  // TODO(wesm): handling null bitmap when input type is NullType
-  result->buffers.push_back(array.data()->buffers[0]);
-
-  std::shared_ptr<Buffer> out_data;
-
-  int bit_width = fw_type.bit_width();
-
-  if (bit_width == 1) {
-    RETURN_NOT_OK(ctx->Allocate(BitUtil::BytesForBits(array.length()), &out_data));
-  } else if (bit_width % 8 == 0) {
-    RETURN_NOT_OK(ctx->Allocate(array.length() * fw_type.bit_width() / 8, &out_data));
-  } else {
-    DCHECK(false);
-  }
-  result->buffers.push_back(out_data);
-
-  *out = result;
-  return Status::OK();
-}
-
-static Status Cast(CastContext* cast_ctx, const Array& array,
-                   const std::shared_ptr<DataType>& out_type,
-                   std::shared_ptr<Array>* out) {
+Status Cast(FunctionContext* ctx, const Array& array,
+            const std::shared_ptr<DataType>& out_type, const CastOptions& options,
+            std::shared_ptr<Array>* out) {
   // Dynamic dispatch to obtain right cast function
-  CastFunction func;
-  RETURN_NOT_OK(GetCastFunction(*array.type(), out_type, &func));
+  std::unique_ptr<UnaryKernel> func;
+  RETURN_NOT_OK(GetCastFunction(*array.type(), out_type, options, &func));
 
-  // Allocate memory for output
-  std::shared_ptr<ArrayData> out_data;
-  RETURN_NOT_OK(AllocateLike(cast_ctx->func_ctx, array, out_type, &out_data));
+  // Data structure for output
+  auto out_data = std::make_shared<ArrayData>(out_type, array.length());
 
-  func(cast_ctx, *array.data(), out_data.get());
-  RETURN_IF_ERROR(cast_ctx->func_ctx);
+  RETURN_NOT_OK(func->Call(ctx, array, out_data.get()));
   return internal::MakeArray(out_data, out);
 }
 
-Status Cast(FunctionContext* ctx, const Array& array,
-            const std::shared_ptr<DataType>& out_type, const CastOptions& options,
-            std::shared_ptr<Array>* out) {
-  CastContext cast_ctx{ctx, options};
-  return Cast(&cast_ctx, array, out_type, out);
-}
-
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/cast.h b/cpp/src/arrow/compute/cast.h
index 9ca70aa43e528..081cdd908326c 100644
--- a/cpp/src/arrow/compute/cast.h
+++ b/cpp/src/arrow/compute/cast.h
@@ -20,21 +20,31 @@
 
 #include <memory>
 
-#include "arrow/array.h"
+#include "arrow/status.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
-using internal::ArrayData;
+class Array;
+class DataType;
 
 namespace compute {
 
 class FunctionContext;
+class UnaryKernel;
 
 struct CastOptions {
+  CastOptions() : allow_int_overflow(false) {}
+
   bool allow_int_overflow;
 };
 
+/// \since 0.7.0
+/// \note API not yet finalized
+ARROW_EXPORT
+Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>& to_type,
+                       const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel);
+
 /// \brief Cast from one array type to another
 /// \param[in] context
 /// \param[in] array
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index cda5755d1e901..ba645c28bd726 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -39,10 +39,14 @@
 
 #include "arrow/compute/cast.h"
 #include "arrow/compute/context.h"
+#include "arrow/compute/kernel.h"
 
 using std::vector;
 
 namespace arrow {
+
+using internal::ArrayData;
+
 namespace compute {
 
 void AssertArraysEqual(const Array& left, const Array& right) {
@@ -72,6 +76,11 @@ class ComputeFixture {
 // ----------------------------------------------------------------------
 // Cast
 
+static void AssertBufferSame(const Array& left, const Array& right, int buffer_index) {
+  ASSERT_EQ(left.data()->buffers[buffer_index].get(),
+            right.data()->buffers[buffer_index].get());
+}
+
 class TestCast : public ComputeFixture, public ::testing::Test {
  public:
   void CheckPass(const Array& input, const Array& expected,
@@ -94,6 +103,13 @@ class TestCast : public ComputeFixture, public ::testing::Test {
     ASSERT_RAISES(Invalid, Cast(&ctx_, *input, out_type, options, &result));
   }
 
+  void CheckZeroCopy(const Array& input, const std::shared_ptr<DataType>& out_type) {
+    std::shared_ptr<Array> result;
+    ASSERT_OK(Cast(&ctx_, input, out_type, {}, &result));
+    AssertBufferSame(input, *result, 0);
+    AssertBufferSame(input, *result, 1);
+  }
+
   template <typename InType, typename I_TYPE, typename OutType, typename O_TYPE>
   void CheckCase(const std::shared_ptr<DataType>& in_type,
                  const std::vector<I_TYPE>& in_values, const std::vector<bool>& is_valid,
@@ -121,12 +137,8 @@ TEST_F(TestCast, SameTypeZeroCopy) {
   std::shared_ptr<Array> result;
   ASSERT_OK(Cast(&this->ctx_, *arr, int32(), {}, &result));
 
-  const auto& lbuffers = arr->data()->buffers;
-  const auto& rbuffers = result->data()->buffers;
-
-  // Buffers are the same
-  ASSERT_EQ(lbuffers[0].get(), rbuffers[0].get());
-  ASSERT_EQ(lbuffers[1].get(), rbuffers[1].get());
+  AssertBufferSame(*arr, *result, 0);
+  AssertBufferSame(*arr, *result, 1);
 }
 
 TEST_F(TestCast, ToBoolean) {
@@ -311,5 +323,77 @@ TEST_F(TestCast, UnsupportedTarget) {
   ASSERT_RAISES(NotImplemented, Cast(&this->ctx_, *arr, utf8(), {}, &result));
 }
 
+TEST_F(TestCast, DateTimeZeroCopy) {
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  std::shared_ptr<Array> arr;
+  vector<int32_t> v1 = {0, 70000, 2000, 1000, 0};
+  ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
+
+  CheckZeroCopy(*arr, time32(TimeUnit::SECOND));
+  CheckZeroCopy(*arr, date32());
+
+  vector<int64_t> v2 = {0, 70000, 2000, 1000, 0};
+  ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, v2, &arr);
+
+  CheckZeroCopy(*arr, time64(TimeUnit::MICRO));
+  CheckZeroCopy(*arr, date64());
+  CheckZeroCopy(*arr, timestamp(TimeUnit::NANO));
+}
+
+TEST_F(TestCast, FromNull) {
+  // Null casts to everything
+  const int length = 10;
+
+  NullArray arr(length);
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(Cast(&ctx_, arr, int32(), {}, &result));
+
+  ASSERT_EQ(length, result->length());
+  ASSERT_EQ(length, result->null_count());
+
+  // OK to look at bitmaps
+  AssertArraysEqual(*result, *result);
+}
+
+TEST_F(TestCast, PreallocatedMemory) {
+  CastOptions options;
+  options.allow_int_overflow = false;
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  const int64_t length = 5;
+
+  std::shared_ptr<Array> arr;
+  vector<int32_t> v1 = {0, 70000, 2000, 1000, 0};
+  vector<int64_t> e1 = {0, 70000, 2000, 1000, 0};
+  ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
+
+  auto out_type = int64();
+
+  std::unique_ptr<UnaryKernel> kernel;
+  ASSERT_OK(GetCastFunction(*int32(), out_type, options, &kernel));
+
+  auto out_data = std::make_shared<ArrayData>(out_type, length);
+
+  std::shared_ptr<Buffer> out_values;
+  ASSERT_OK(this->ctx_.Allocate(length * sizeof(int64_t), &out_values));
+
+  out_data->buffers.push_back(nullptr);
+  out_data->buffers.push_back(out_values);
+
+  ASSERT_OK(kernel->Call(&this->ctx_, *arr, out_data.get()));
+
+  // Buffer address unchanged
+  ASSERT_EQ(out_values.get(), out_data->buffers[1].get());
+
+  std::shared_ptr<Array> result, expected;
+  ASSERT_OK(MakeArray(out_data, &result));
+  ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, e1, &expected);
+
+  AssertArraysEqual(*expected, *result);
+}
+
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/context.h b/cpp/src/arrow/compute/context.h
index caff2e2b743b2..051c91bf049fa 100644
--- a/cpp/src/arrow/compute/context.h
+++ b/cpp/src/arrow/compute/context.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_COMPUTE_CONTEXT_H
 #define ARROW_COMPUTE_CONTEXT_H
 
+#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/type_fwd.h"
 #include "arrow/util/visibility.h"
@@ -35,7 +36,7 @@ namespace compute {
 /// \brief Container for variables and options used by function evaluation
 class ARROW_EXPORT FunctionContext {
  public:
-  explicit FunctionContext(MemoryPool* pool);
+  explicit FunctionContext(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
   MemoryPool* memory_pool() const;
 
   /// \brief Allocate buffer from the context's memory pool
diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h
new file mode 100644
index 0000000000000..c72d4679af2a0
--- /dev/null
+++ b/cpp/src/arrow/compute/kernel.h
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_COMPUTE_KERNEL_H
+#define ARROW_COMPUTE_KERNEL_H
+
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Array;
+using internal::ArrayData;
+
+namespace compute {
+
+class FunctionContext;
+
+/// \class OpKernel
+/// \brief Base class for operator kernels
+class ARROW_EXPORT OpKernel {
+ public:
+  virtual ~OpKernel() = default;
+};
+
+/// \class UnaryKernel
+/// \brief An array-valued function of a single input argument
+class ARROW_EXPORT UnaryKernel : public OpKernel {
+ public:
+  virtual Status Call(FunctionContext* ctx, const Array& input, ArrayData* out) = 0;
+};
+
+}  // namespace compute
+}  // namespace arrow
+
+#endif  // ARROW_COMPUTE_KERNEL_H
diff --git a/cpp/src/arrow/python/numpy_convert.h b/cpp/src/arrow/python/numpy_convert.h
index 7b3b3b7c9a2a0..93c4848926cfc 100644
--- a/cpp/src/arrow/python/numpy_convert.h
+++ b/cpp/src/arrow/python/numpy_convert.h
@@ -57,10 +57,7 @@ bool is_contiguous(PyObject* array);
 ARROW_EXPORT
 Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out);
 
-ARROW_EXPORT
 Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out);
-
-ARROW_EXPORT
 Status GetNumPyType(const DataType& type, int* type_num);
 
 ARROW_EXPORT Status NdarrayToTensor(MemoryPool* pool, PyObject* ao,
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 24937795f701b..1f96f4ed9d67b 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -44,6 +44,9 @@
 #include "arrow/util/macros.h"
 #include "arrow/visitor_inline.h"
 
+#include "arrow/compute/cast.h"
+#include "arrow/compute/context.h"
+
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
 #include "arrow/python/config.h"
@@ -347,8 +350,8 @@ class PandasConverter {
     }
 
     BufferVector buffers = {null_bitmap_, data};
-    auto arr_data = std::make_shared<ArrayData>(type_, length_, std::move(buffers),
-                                                null_count, 0);
+    auto arr_data =
+        std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count, 0);
     return PushArray(arr_data);
   }
 
@@ -419,9 +422,11 @@ class PandasConverter {
   Status ConvertLists(const std::shared_ptr<DataType>& type);
   Status ConvertLists(const std::shared_ptr<DataType>& type, ListBuilder* builder,
                       PyObject* list);
-  Status ConvertObjects();
   Status ConvertDecimals();
   Status ConvertTimes();
+  Status ConvertObjects();
+  Status ConvertObjectsInfer();
+  Status ConvertObjectsInferAndCast();
 
  protected:
   MemoryPool* pool_;
@@ -460,22 +465,32 @@ void CopyStrided<PyObject*, PyObject*>(PyObject** input_data, int64_t length,
   }
 }
 
+static Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t length,
+                         const std::shared_ptr<DataType>& in_type,
+                         const std::shared_ptr<DataType>& out_type, MemoryPool* pool,
+                         std::shared_ptr<Buffer>* out) {
+  // Must cast
+  std::vector<std::shared_ptr<Buffer>> buffers = {nullptr, input};
+  auto tmp_data = std::make_shared<ArrayData>(in_type, length, buffers, 0);
+
+  std::shared_ptr<Array> tmp_array, casted_array;
+  RETURN_NOT_OK(MakeArray(tmp_data, &tmp_array));
+
+  compute::FunctionContext context(pool);
+  compute::CastOptions cast_options;
+  cast_options.allow_int_overflow = false;
+
+  RETURN_NOT_OK(
+      compute::Cast(&context, *tmp_array, out_type, cast_options, &casted_array));
+  *out = casted_array->data()->buffers[1];
+  return Status::OK();
+}
+
 template <typename ArrowType>
 inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   using traits = internal::arrow_traits<ArrowType::type_id>;
   using T = typename traits::T;
 
-  // Handle LONGLONG->INT64 and other fun things
-  int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
-
-  if (NumPyTypeSize(traits::npy_type) != NumPyTypeSize(type_num_compat)) {
-    std::stringstream ss;
-    ss << "NumPy type casts not yet implemented, type sizes differ: ";
-    ss << NumPyTypeSize(traits::npy_type) << " compared to "
-       << NumPyTypeSize(type_num_compat);
-    return Status::NotImplemented(ss.str());
-  }
-
   if (is_strided()) {
     // Strided, must copy into new contiguous memory
     const int64_t stride = PyArray_STRIDES(arr_)[0];
@@ -490,6 +505,15 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
     // Can zero-copy
     *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
   }
+
+  std::shared_ptr<DataType> input_type;
+  RETURN_NOT_OK(
+      NumPyDtypeToArrow(reinterpret_cast<PyObject*>(PyArray_DESCR(arr_)), &input_type));
+
+  if (!input_type->Equals(*type_)) {
+    RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data));
+  }
+
   return Status::OK();
 }
 
@@ -845,6 +869,74 @@ Status PandasConverter::ConvertBooleans() {
   return Status::OK();
 }
 
+Status PandasConverter::ConvertObjectsInfer() {
+  Ndarray1DIndexer<PyObject*> objects;
+
+  PyAcquireGIL lock;
+  objects.Init(arr_);
+  PyDateTime_IMPORT;
+
+  OwnedRef decimal;
+  OwnedRef Decimal;
+  RETURN_NOT_OK(ImportModule("decimal", &decimal));
+  RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+
+  for (int64_t i = 0; i < length_; ++i) {
+    PyObject* obj = objects[i];
+    if (PandasObjectIsNull(obj)) {
+      continue;
+    } else if (PyObject_is_string(obj)) {
+      return ConvertObjectStrings();
+    } else if (PyObject_is_float(obj)) {
+      return ConvertObjectFloats();
+    } else if (PyBool_Check(obj)) {
+      return ConvertBooleans();
+    } else if (PyObject_is_integer(obj)) {
+      return ConvertObjectIntegers();
+    } else if (PyDate_CheckExact(obj)) {
+      // We could choose Date32 or Date64
+      return ConvertDates<Date32Type>();
+    } else if (PyTime_Check(obj)) {
+      return ConvertTimes();
+    } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
+      return ConvertDecimals();
+    } else if (PyList_Check(obj) || PyArray_Check(obj)) {
+      std::shared_ptr<DataType> inferred_type;
+      RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
+      return ConvertLists(inferred_type);
+    } else {
+      const std::string supported_types =
+          "string, bool, float, int, date, time, decimal, list, array";
+      std::stringstream ss;
+      ss << "Error inferring Arrow type for Python object array. ";
+      RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+  out_arrays_.push_back(std::make_shared<NullArray>(length_));
+  return Status::OK();
+}
+
+Status PandasConverter::ConvertObjectsInferAndCast() {
+  size_t position = out_arrays_.size();
+  RETURN_NOT_OK(ConvertObjectsInfer());
+
+  std::shared_ptr<Array> arr = out_arrays_[position];
+
+  // Perform cast
+  compute::FunctionContext context(pool_);
+  compute::CastOptions options;
+  options.allow_int_overflow = false;
+
+  std::shared_ptr<Array> casted;
+  RETURN_NOT_OK(compute::Cast(&context, *arr, type_, options, &casted));
+
+  // Replace with casted values
+  out_arrays_[position] = casted;
+
+  return Status::OK();
+}
+
 Status PandasConverter::ConvertObjects() {
   // Python object arrays are annoying, since we could have one of:
   //
@@ -858,13 +950,6 @@ Status PandasConverter::ConvertObjects() {
 
   RETURN_NOT_OK(InitNullBitmap());
 
-  Ndarray1DIndexer<PyObject*> objects;
-
-  PyAcquireGIL lock;
-  objects.Init(arr_);
-  PyDateTime_IMPORT;
-  lock.release();
-
   // This means we received an explicit type from the user
   if (type_) {
     switch (type_->id()) {
@@ -885,53 +970,12 @@ Status PandasConverter::ConvertObjects() {
       case Type::DECIMAL:
         return ConvertDecimals();
       default:
-        return Status::TypeError("No known conversion to Arrow type");
+        return ConvertObjectsInferAndCast();
     }
   } else {
     // Re-acquire GIL
-    lock.acquire();
-
-    OwnedRef decimal;
-    OwnedRef Decimal;
-    RETURN_NOT_OK(ImportModule("decimal", &decimal));
-    RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
-
-    for (int64_t i = 0; i < length_; ++i) {
-      PyObject* obj = objects[i];
-      if (PandasObjectIsNull(obj)) {
-        continue;
-      } else if (PyObject_is_string(obj)) {
-        return ConvertObjectStrings();
-      } else if (PyObject_is_float(obj)) {
-        return ConvertObjectFloats();
-      } else if (PyBool_Check(obj)) {
-        return ConvertBooleans();
-      } else if (PyObject_is_integer(obj)) {
-        return ConvertObjectIntegers();
-      } else if (PyDate_CheckExact(obj)) {
-        // We could choose Date32 or Date64
-        return ConvertDates<Date32Type>();
-      } else if (PyTime_Check(obj)) {
-        return ConvertTimes();
-      } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
-        return ConvertDecimals();
-      } else if (PyList_Check(obj) || PyArray_Check(obj)) {
-        std::shared_ptr<DataType> inferred_type;
-        RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
-        return ConvertLists(inferred_type);
-      } else {
-        const std::string supported_types =
-            "string, bool, float, int, date, time, decimal, list, array";
-        std::stringstream ss;
-        ss << "Error inferring Arrow type for Python object array. ";
-        RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
-        return Status::Invalid(ss.str());
-      }
-    }
+    return ConvertObjectsInfer();
   }
-
-  out_arrays_.push_back(std::make_shared<NullArray>(length_));
-  return Status::OK();
 }
 
 template <typename T>
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index a693f45e967a5..eec6180165c27 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -88,6 +88,19 @@ def _normalize_slice(object arrow_obj, slice key):
         return arrow_obj.slice(start, stop - start)
 
 
+cdef class _FunctionContext:
+    cdef:
+        unique_ptr[CFunctionContext] ctx
+
+    def __cinit__(self):
+        self.ctx.reset(new CFunctionContext(c_default_memory_pool()))
+
+cdef _FunctionContext _global_ctx = _FunctionContext()
+
+cdef CFunctionContext* _context() nogil:
+    return _global_ctx.ctx.get()
+
+
 cdef class Array:
 
     cdef void init(self, const shared_ptr[CArray]& sp_array):
@@ -99,6 +112,34 @@ cdef class Array:
         with nogil:
             check_status(DebugPrint(deref(self.ap), 0))
 
+    def cast(self, DataType target_type, safe=True):
+        """
+        Cast array values to another data type
+
+        Parameters
+        ----------
+        target_type : DataType
+            Type to cast to
+        safe : boolean, default True
+            Check for overflows or other unsafe conversions
+
+        Returns
+        -------
+        casted : Array
+        """
+        cdef:
+            CCastOptions options
+            shared_ptr[CArray] result
+
+        if not safe:
+            options.allow_int_overflow = 1
+
+        with nogil:
+            check_status(Cast(_context(), self.ap[0], target_type.sp_type,
+                              options, &result))
+
+        return pyarrow_wrap_array(result)
+
     @staticmethod
     def from_pandas(obj, mask=None, DataType type=None,
                     timestamps_to_ms=False,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index cc356848709a0..3b7ddcfa83c7d 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -706,9 +706,6 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
                             InputStream* stream,
                             shared_ptr[CRecordBatch]* out)
 
-
-cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
-
     cdef cppclass CFeatherWriter" arrow::ipc::feather::TableWriter":
         @staticmethod
         CStatus Open(const shared_ptr[OutputStream]& stream,
@@ -737,6 +734,21 @@ cdef extern from "arrow/ipc/feather.h" namespace "arrow::ipc::feather" nogil:
         c_string GetColumnName(int i)
 
 
+cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
+
+    cdef cppclass CFunctionContext" arrow::compute::FunctionContext":
+        CFunctionContext()
+        CFunctionContext(CMemoryPool* pool)
+
+    cdef cppclass CCastOptions" arrow::compute::CastOptions":
+        c_bool allow_int_overflow
+
+    CStatus Cast(CFunctionContext* context, const CArray& array,
+                 const shared_ptr[CDataType]& to_type,
+                 const CCastOptions& options,
+                 shared_ptr[CArray]* out)
+
+
 cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     shared_ptr[CDataType] GetPrimitiveType(Type type)
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 5a373b49a515d..f316417caafb0 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -211,6 +211,73 @@ def test_list_from_arrays():
     assert result.equals(expected)
 
 
+def _check_cast_case(case, safe=True):
+    in_data, in_type, out_data, out_type = case
+
+    in_arr = pa.Array.from_pandas(in_data, type=in_type)
+
+    casted = in_arr.cast(out_type, safe=safe)
+    expected = pa.Array.from_pandas(out_data, type=out_type)
+    assert casted.equals(expected)
+
+
+def test_cast_integers_safe():
+    safe_cases = [
+        (np.array([0, 1, 2, 3], dtype='i1'), pa.int8(),
+         np.array([0, 1, 2, 3], dtype='i4'), pa.int32()),
+        (np.array([0, 1, 2, 3], dtype='i1'), pa.int8(),
+         np.array([0, 1, 2, 3], dtype='u4'), pa.uint16()),
+        (np.array([0, 1, 2, 3], dtype='i1'), pa.int8(),
+         np.array([0, 1, 2, 3], dtype='u1'), pa.uint8()),
+        (np.array([0, 1, 2, 3], dtype='i1'), pa.int8(),
+         np.array([0, 1, 2, 3], dtype='f8'), pa.float64())
+    ]
+
+    for case in safe_cases:
+        _check_cast_case(case)
+
+    unsafe_cases = [
+        (np.array([50000], dtype='i4'), pa.int32(), pa.int16()),
+        (np.array([70000], dtype='i4'), pa.int32(), pa.uint16()),
+        (np.array([-1], dtype='i4'), pa.int32(), pa.uint16()),
+        (np.array([50000], dtype='u2'), pa.uint16(), pa.int16())
+    ]
+    for in_data, in_type, out_type in unsafe_cases:
+        in_arr = pa.Array.from_pandas(in_data, type=in_type)
+
+        with pytest.raises(pa.ArrowInvalid):
+            in_arr.cast(out_type)
+
+
+def test_cast_integers_unsafe():
+    # We let NumPy do the unsafe casting
+    unsafe_cases = [
+        (np.array([50000], dtype='i4'), pa.int32(),
+         np.array([50000], dtype='i2'), pa.int16()),
+        (np.array([70000], dtype='i4'), pa.int32(),
+         np.array([70000], dtype='u2'), pa.uint16()),
+        (np.array([-1], dtype='i4'), pa.int32(),
+         np.array([-1], dtype='u2'), pa.uint16()),
+        (np.array([50000], dtype='u2'), pa.uint16(),
+         np.array([50000], dtype='i2'), pa.int16())
+    ]
+
+    for case in unsafe_cases:
+        _check_cast_case(case, safe=False)
+
+
+def test_cast_signed_to_unsigned():
+    safe_cases = [
+        (np.array([0, 1, 2, 3], dtype='i1'), pa.uint8(),
+         np.array([0, 1, 2, 3], dtype='u1'), pa.uint8()),
+        (np.array([0, 1, 2, 3], dtype='i2'), pa.uint16(),
+         np.array([0, 1, 2, 3], dtype='u2'), pa.uint16())
+    ]
+
+    for case in safe_cases:
+        _check_cast_case(case)
+
+
 def test_simple_type_construction():
     result = pa.lib.TimestampType()
     with pytest.raises(TypeError):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 64424341067bb..e98e83d01a0cd 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -239,6 +239,15 @@ def test_integer_with_nulls(self):
 
         tm.assert_frame_equal(result, ex_frame)
 
+    def test_array_from_pandas_type_cast(self):
+        arr = np.arange(10, dtype='int64')
+
+        target_type = pa.int8()
+
+        result = pa.Array.from_pandas(arr, type=target_type)
+        expected = pa.Array.from_pandas(arr.astype('int8'))
+        assert result.equals(expected)
+
     def test_boolean_no_nulls(self):
         num_values = 100
 
@@ -279,6 +288,17 @@ def test_boolean_object_nulls(self):
         schema = pa.schema([field])
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
+    def test_all_nulls_cast_numeric(self):
+        arr = np.array([None], dtype=object)
+
+        def _check_type(t):
+            a2 = pa.Array.from_pandas(arr, type=t)
+            assert a2.type == t
+            assert a2[0].as_py() is None
+
+        _check_type(pa.int32())
+        _check_type(pa.float64())
+
     def test_unicode(self):
         repeats = 1000
         values = [u'foo', None, u'bar', u'mañana', np.nan]

From 11ebe938744f14c335d8cf5ecfacd64afd492977 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 8 Sep 2017 11:04:05 -0400
Subject: [PATCH 1023/1644] ARROW-1494: [C++] Improve doxygen comments in
 arrow/table.h, note that RecordBatch::column returns new object

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1065 from wesm/ARROW-1494 and squashes the following commits:

ff87fa85 [Wes McKinney] Improve doxygen comments in arrow/table.h, note that RecordBatch::column returns a new object
---
 cpp/src/arrow/table.h | 102 ++++++++++++++++++++++++++----------------
 1 file changed, 64 insertions(+), 38 deletions(-)

diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index ceb564d45c9b2..1145d113e2518 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -34,19 +34,21 @@ class Status;
 
 using ArrayVector = std::vector<std::shared_ptr<Array>>;
 
+/// \class ChunkedArray
 /// \brief A data structure managing a list of primitive Arrow arrays logically
 /// as one large array
 class ARROW_EXPORT ChunkedArray {
  public:
   explicit ChunkedArray(const ArrayVector& chunks);
 
-  // \return the total length of the chunked array; computed on construction
+  /// \return the total length of the chunked array; computed on construction
   int64_t length() const { return length_; }
 
   int64_t null_count() const { return null_count_; }
 
   int num_chunks() const { return static_cast<int>(chunks_.size()); }
 
+  /// \return chunk a particular chunk from the chunked array
   std::shared_ptr<Array> chunk(int i) const { return chunks_[i]; }
 
   const ArrayVector& chunks() const { return chunks_; }
@@ -71,7 +73,7 @@ class ARROW_EXPORT Column {
 
   Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data);
 
-  /// Construct from name and array
+  // Construct from name and array
   Column(const std::string& name, const std::shared_ptr<Array>& data);
 
   int64_t length() const { return data_->length(); }
@@ -80,20 +82,23 @@ class ARROW_EXPORT Column {
 
   std::shared_ptr<Field> field() const { return field_; }
 
-  // \return the column's name in the passed metadata
+  /// \brief The column name
+  /// \return the column's name in the passed metadata
   const std::string& name() const { return field_->name(); }
 
-  // \return the column's type according to the metadata
+  /// \brief The column type
+  /// \return the column's type according to the metadata
   std::shared_ptr<DataType> type() const { return field_->type(); }
 
-  // \return the column's data as a chunked logical array
+  /// \brief The column data as a chunked array
+  /// \return the column's data as a chunked logical array
   std::shared_ptr<ChunkedArray> data() const { return data_; }
 
   bool Equals(const Column& other) const;
   bool Equals(const std::shared_ptr<Column>& other) const;
 
-  // Verify that the column's array data is consistent with the passed field's
-  // metadata
+  /// \brief Verify that the column's array data is consistent with the passed
+  /// field's metadata
   Status ValidateData();
 
  protected:
@@ -108,14 +113,14 @@ class ARROW_EXPORT Column {
 /// sequence of fields, each a contiguous Arrow array
 class ARROW_EXPORT RecordBatch {
  public:
-  /// num_rows is a parameter to allow for record batches of a particular size not
-  /// having any materialized columns. Each array should have the same length as
-  /// num_rows
-
+  /// \param[in] schema
+  /// \param[in] num_rows length of fields in the record batch. Each array
+  /// should have the same length as num_rows
+  /// \param[in] columns the record batch fields as vector of arrays
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
               const std::vector<std::shared_ptr<Array>>& columns);
 
-  /// \brief Deprecated move constructor for a vector of Array instances
+  /// \brief Move-based constructor for a vector of Array instances
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
               std::vector<std::shared_ptr<Array>>&& columns);
 
@@ -137,25 +142,35 @@ class ARROW_EXPORT RecordBatch {
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
               const std::vector<std::shared_ptr<internal::ArrayData>>& columns);
 
+  /// \brief Determine if two record batches are exactly equal
+  /// \return true if batches are equal
   bool Equals(const RecordBatch& other) const;
 
+  /// \brief Determine if two record batches are approximately equal
   bool ApproxEquals(const RecordBatch& other) const;
 
   // \return the table's schema
+  /// \return true if batches are equal
   std::shared_ptr<Schema> schema() const { return schema_; }
 
-  // \return the i-th column
-  // Note: Does not boundscheck
+  /// \brief Retrieve an array from the record batch (new object)
+  /// \param[in] i field index, does not boundscheck
+  /// \return a new Array object
+  ///
+  /// \note This function returns a new object. If you intend to dereference
+  /// the pointer or access the internals, retain a reference to the
+  /// std::shared_ptr returned.
   std::shared_ptr<Array> column(int i) const;
 
   std::shared_ptr<internal::ArrayData> column_data(int i) const { return columns_[i]; }
 
+  /// \brief Name in i-th column
   const std::string& column_name(int i) const;
 
-  // \return the number of columns in the table
+  /// \return the number of columns in the table
   int num_columns() const { return static_cast<int>(columns_.size()); }
 
-  // \return the number of rows (the corresponding length of each column)
+  /// \return the number of rows (the corresponding length of each column)
   int64_t num_rows() const { return num_rows_; }
 
   /// \brief Replace schema key-value metadata with new metadata (EXPERIMENTAL)
@@ -166,12 +181,18 @@ class ARROW_EXPORT RecordBatch {
   std::shared_ptr<RecordBatch> ReplaceSchemaMetadata(
       const std::shared_ptr<const KeyValueMetadata>& metadata) const;
 
-  /// Slice each of the arrays in the record batch and construct a new RecordBatch object
+  /// \brief Slice each of the arrays in the record batch
+  /// \param[in] offset the starting offset to slice, through end of batch
+  /// \return new record batch
   std::shared_ptr<RecordBatch> Slice(int64_t offset) const;
+
+  /// \brief Slice each of the arrays in the record batch
+  /// \param[in] offset the starting offset to slice
+  /// \param[in] length the number of elements to slice from offset
+  /// \return new record batch
   std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length) const;
 
   /// \brief Check for schema or length inconsistencies
-  ///
   /// \return Status
   Status Validate() const;
 
@@ -181,21 +202,22 @@ class ARROW_EXPORT RecordBatch {
   std::vector<std::shared_ptr<internal::ArrayData>> columns_;
 };
 
-// Immutable container of fixed-length columns conforming to a particular schema
+/// \class Table
+/// \brief Logical table as sequence of chunked arrays
 class ARROW_EXPORT Table {
  public:
   /// \brief Construct Table from schema and columns
   /// If columns is zero-length, the table's number of rows is zero
   /// \param schema
   /// \param columns
-  /// \param num_rows number of rows in table, -1 (default) to infer from columns
+  /// \param number of rows in table, -1 (default) to infer from columns
   Table(const std::shared_ptr<Schema>& schema,
         const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows = -1);
 
   /// \brief Construct Table from schema and arrays
   /// \param schema
   /// \param arrays
-  /// \param num_rows number of rows in table, -1 (default) to infer from columns
+  /// \param number of rows in table, -1 (default) to infer from columns
   Table(const std::shared_ptr<Schema>& schema,
         const std::vector<std::shared_ptr<Array>>& arrays, int64_t num_rows = -1);
 
@@ -205,18 +227,17 @@ class ARROW_EXPORT Table {
       const std::vector<std::shared_ptr<RecordBatch>>& batches,
       std::shared_ptr<Table>* table);
 
-  // \return the table's schema
+  /// \return the table's schema
   std::shared_ptr<Schema> schema() const { return schema_; }
 
-  // Note: Does not boundscheck
-  // \return the i-th column
+  /// \param[i] i column index, does not boundscheck
+  /// \return the i-th column
   std::shared_ptr<Column> column(int i) const { return columns_[i]; }
 
-  /// Remove column from the table, producing a new Table (because tables and
-  /// schemas are immutable)
+  /// \brief Remove column from the table, producing a new Table
   Status RemoveColumn(int i, std::shared_ptr<Table>* out) const;
 
-  /// Add column to the table, producing a new Table
+  /// \brief Add column to the table, producing a new Table
   Status AddColumn(int i, const std::shared_ptr<Column>& column,
                    std::shared_ptr<Table>* out) const;
 
@@ -228,15 +249,16 @@ class ARROW_EXPORT Table {
   std::shared_ptr<Table> ReplaceSchemaMetadata(
       const std::shared_ptr<const KeyValueMetadata>& metadata) const;
 
-  // \return the number of columns in the table
+  /// \return the number of columns in the table
   int num_columns() const { return static_cast<int>(columns_.size()); }
 
-  // \return the number of rows (the corresponding length of each column)
+  /// \return the number of rows (the corresponding length of each column)
   int64_t num_rows() const { return num_rows_; }
 
+  /// \brief Determine if semantic contents of tables are exactly equal
   bool Equals(const Table& other) const;
 
-  // After construction, perform any checks to validate the input arguments
+  /// \brief Perform any checks to validate the input arguments
   Status ValidateColumns() const;
 
  private:
@@ -246,14 +268,18 @@ class ARROW_EXPORT Table {
   int64_t num_rows_;
 };
 
-// Construct table from multiple input tables. Return Status::Invalid if
-// schemas are not equal
-Status ARROW_EXPORT ConcatenateTables(const std::vector<std::shared_ptr<Table>>& tables,
-                                      std::shared_ptr<Table>* table);
-
-Status ARROW_EXPORT MakeTable(const std::shared_ptr<Schema>& schema,
-                              const std::vector<std::shared_ptr<Array>>& arrays,
-                              std::shared_ptr<Table>* table);
+/// \brief Construct table from multiple input tables.
+/// \return Status, fails if any schemas are different
+ARROW_EXPORT
+Status ConcatenateTables(const std::vector<std::shared_ptr<Table>>& tables,
+                         std::shared_ptr<Table>* table);
+
+/// \brief Construct table from multiple input tables.
+/// \return Status, fails if any schemas are different
+ARROW_EXPORT
+Status MakeTable(const std::shared_ptr<Schema>& schema,
+                 const std::vector<std::shared_ptr<Array>>& arrays,
+                 std::shared_ptr<Table>* table);
 
 }  // namespace arrow
 

From 5aca7b669530b20121b8dda566ddc20ac1dadbeb Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 8 Sep 2017 18:31:59 -0400
Subject: [PATCH 1024/1644] ARROW-1480: [Python] Improve performance of
 serializing sets

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1060 from pcmoritz/serialize-sets and squashes the following commits:

86707aaa [Wes McKinney] Update RecordBatch::column docstring now that columns are being cached
cb451aab [Wes McKinney] Incorporate code review comments. Add internal caching of boxed arrays to StructArray, UnionArray, RecordBatch
89f191b0 [Philipp Moritz] fix linting
3d335e5a [Philipp Moritz] fix
c705e435 [Philipp Moritz] deserialization
a59cb989 [Philipp Moritz] support serializing sets
---
 cpp/src/arrow/array-test.cc                |   2 +
 cpp/src/arrow/array.cc                     |  21 +--
 cpp/src/arrow/array.h                      |   7 +
 cpp/src/arrow/python/arrow_to_python.cc    | 155 ++++++++++++---------
 cpp/src/arrow/python/python_to_arrow.cc    |  55 ++++++--
 cpp/src/arrow/table.cc                     |  26 +++-
 cpp/src/arrow/table.h                      |  13 +-
 python/pyarrow/tests/test_serialization.py |   1 +
 8 files changed, 183 insertions(+), 97 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 5d9eb187e747c..c92c23dd21dbd 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -2460,6 +2460,8 @@ TEST(TestUnionArrayAdHoc, TestSliceEquals) {
   auto CheckUnion = [&size](std::shared_ptr<Array> array) {
     std::shared_ptr<Array> slice, slice2;
     slice = array->Slice(2);
+    ASSERT_EQ(size - 2, slice->length());
+
     slice2 = array->Slice(2);
     ASSERT_EQ(size - 2, slice->length());
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 34f0868e6407b..2d37274a976c7 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -329,6 +329,7 @@ std::string DecimalArray::FormatValue(int64_t i) const {
 StructArray::StructArray(const std::shared_ptr<ArrayData>& data) {
   DCHECK_EQ(data->type->id(), Type::STRUCT);
   SetData(data);
+  boxed_fields_.resize(data->child_data.size());
 }
 
 StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
@@ -341,12 +342,14 @@ StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
   for (const auto& child : children) {
     data_->child_data.push_back(child->data());
   }
+  boxed_fields_.resize(children.size());
 }
 
-std::shared_ptr<Array> StructArray::field(int pos) const {
-  std::shared_ptr<Array> result;
-  DCHECK(internal::MakeArray(data_->child_data[pos], &result).ok());
-  return result;
+std::shared_ptr<Array> StructArray::field(int i) const {
+  if (!boxed_fields_[i]) {
+    DCHECK(internal::MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
+  }
+  return boxed_fields_[i];
 }
 
 // ----------------------------------------------------------------------
@@ -362,6 +365,7 @@ void UnionArray::SetData(const std::shared_ptr<ArrayData>& data) {
   raw_value_offsets_ = value_offsets == nullptr
                            ? nullptr
                            : reinterpret_cast<const int32_t*>(value_offsets->data());
+  boxed_fields_.resize(data->child_data.size());
 }
 
 UnionArray::UnionArray(const std::shared_ptr<ArrayData>& data) {
@@ -384,10 +388,11 @@ UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
   SetData(internal_data);
 }
 
-std::shared_ptr<Array> UnionArray::child(int pos) const {
-  std::shared_ptr<Array> result;
-  DCHECK(internal::MakeArray(data_->child_data[pos], &result).ok());
-  return result;
+std::shared_ptr<Array> UnionArray::child(int i) const {
+  if (!boxed_fields_[i]) {
+    DCHECK(internal::MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
+  }
+  return boxed_fields_[i];
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 3faff714754c7..bfeedd2b65cd9 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -555,6 +555,10 @@ class ARROW_EXPORT StructArray : public Array {
   // Return a shared pointer in case the requestor desires to share ownership
   // with this array.
   std::shared_ptr<Array> field(int pos) const;
+
+ private:
+  // For caching boxed child data
+  mutable std::vector<std::shared_ptr<Array>> boxed_fields_;
 };
 
 // ----------------------------------------------------------------------
@@ -592,6 +596,9 @@ class ARROW_EXPORT UnionArray : public Array {
 
   const type_id_t* raw_type_ids_;
   const int32_t* raw_value_offsets_;
+
+  // For caching boxed child data
+  mutable std::vector<std::shared_ptr<Array>> boxed_fields_;
 };
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index b127971a16f91..bc12ba74453b3 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -37,26 +37,31 @@ namespace py {
 Status CallDeserializeCallback(PyObject* context, PyObject* value,
                                PyObject** deserialized_object);
 
-Status DeserializeTuple(PyObject* context, std::shared_ptr<Array> array,
-                        int64_t start_idx, int64_t stop_idx, PyObject* base,
+Status DeserializeTuple(PyObject* context, const Array& array, int64_t start_idx,
+                        int64_t stop_idx, PyObject* base,
                         const std::vector<std::shared_ptr<Tensor>>& tensors,
                         PyObject** out);
 
-Status DeserializeList(PyObject* context, std::shared_ptr<Array> array, int64_t start_idx,
+Status DeserializeList(PyObject* context, const Array& array, int64_t start_idx,
                        int64_t stop_idx, PyObject* base,
                        const std::vector<std::shared_ptr<Tensor>>& tensors,
                        PyObject** out);
 
-Status DeserializeDict(PyObject* context, std::shared_ptr<Array> array, int64_t start_idx,
+Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
+                      int64_t stop_idx, PyObject* base,
+                      const std::vector<std::shared_ptr<Tensor>>& tensors,
+                      PyObject** out);
+
+Status DeserializeDict(PyObject* context, const Array& array, int64_t start_idx,
                        int64_t stop_idx, PyObject* base,
                        const std::vector<std::shared_ptr<Tensor>>& tensors,
                        PyObject** out) {
-  auto data = std::dynamic_pointer_cast<StructArray>(array);
+  const auto& data = static_cast<const StructArray&>(array);
   ScopedRef keys, vals;
   ScopedRef result(PyDict_New());
-  RETURN_NOT_OK(DeserializeList(context, data->field(0), start_idx, stop_idx, base,
+  RETURN_NOT_OK(DeserializeList(context, *data.field(0), start_idx, stop_idx, base,
                                 tensors, keys.ref()));
-  RETURN_NOT_OK(DeserializeList(context, data->field(1), start_idx, stop_idx, base,
+  RETURN_NOT_OK(DeserializeList(context, *data.field(1), start_idx, stop_idx, base,
                                 tensors, vals.ref()));
   for (int64_t i = start_idx; i < stop_idx; ++i) {
     // PyDict_SetItem behaves differently from PyList_SetItem and PyTuple_SetItem.
@@ -75,11 +80,10 @@ Status DeserializeDict(PyObject* context, std::shared_ptr<Array> array, int64_t
   return Status::OK();
 }
 
-Status DeserializeArray(std::shared_ptr<Array> array, int64_t offset, PyObject* base,
+Status DeserializeArray(const Array& array, int64_t offset, PyObject* base,
                         const std::vector<std::shared_ptr<arrow::Tensor>>& tensors,
                         PyObject** out) {
-  DCHECK(array);
-  int32_t index = std::static_pointer_cast<Int32Array>(array)->Value(offset);
+  int32_t index = static_cast<const Int32Array&>(array).Value(offset);
   RETURN_NOT_OK(py::TensorToNdarray(*tensors[index], base, out));
   // Mark the array as immutable
   ScopedRef flags(PyObject_GetAttrString(*out, "flags"));
@@ -90,54 +94,51 @@ Status DeserializeArray(std::shared_ptr<Array> array, int64_t offset, PyObject*
   return Status::OK();
 }
 
-Status GetValue(PyObject* context, std::shared_ptr<Array> arr, int64_t index,
-                int32_t type, PyObject* base,
-                const std::vector<std::shared_ptr<Tensor>>& tensors, PyObject** result) {
-  switch (arr->type()->id()) {
+Status GetValue(PyObject* context, const Array& arr, int64_t index, int32_t type,
+                PyObject* base, const std::vector<std::shared_ptr<Tensor>>& tensors,
+                PyObject** result) {
+  switch (arr.type()->id()) {
     case Type::BOOL:
-      *result =
-          PyBool_FromLong(std::static_pointer_cast<BooleanArray>(arr)->Value(index));
+      *result = PyBool_FromLong(static_cast<const BooleanArray&>(arr).Value(index));
       return Status::OK();
     case Type::INT64:
-      *result =
-          PyLong_FromSsize_t(std::static_pointer_cast<Int64Array>(arr)->Value(index));
+      *result = PyLong_FromSsize_t(static_cast<const Int64Array&>(arr).Value(index));
       return Status::OK();
     case Type::BINARY: {
       int32_t nchars;
-      const uint8_t* str =
-          std::static_pointer_cast<BinaryArray>(arr)->GetValue(index, &nchars);
+      const uint8_t* str = static_cast<const BinaryArray&>(arr).GetValue(index, &nchars);
       *result = PyBytes_FromStringAndSize(reinterpret_cast<const char*>(str), nchars);
       return CheckPyError();
     }
     case Type::STRING: {
       int32_t nchars;
-      const uint8_t* str =
-          std::static_pointer_cast<StringArray>(arr)->GetValue(index, &nchars);
+      const uint8_t* str = static_cast<const StringArray&>(arr).GetValue(index, &nchars);
       *result = PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(str), nchars);
       return CheckPyError();
     }
     case Type::FLOAT:
-      *result =
-          PyFloat_FromDouble(std::static_pointer_cast<FloatArray>(arr)->Value(index));
+      *result = PyFloat_FromDouble(static_cast<const FloatArray&>(arr).Value(index));
       return Status::OK();
     case Type::DOUBLE:
-      *result =
-          PyFloat_FromDouble(std::static_pointer_cast<DoubleArray>(arr)->Value(index));
+      *result = PyFloat_FromDouble(static_cast<const DoubleArray&>(arr).Value(index));
       return Status::OK();
     case Type::STRUCT: {
-      auto s = std::static_pointer_cast<StructArray>(arr);
-      auto l = std::static_pointer_cast<ListArray>(s->field(0));
-      if (s->type()->child(0)->name() == "list") {
-        return DeserializeList(context, l->values(), l->value_offset(index),
-                               l->value_offset(index + 1), base, tensors, result);
-      } else if (s->type()->child(0)->name() == "tuple") {
-        return DeserializeTuple(context, l->values(), l->value_offset(index),
-                                l->value_offset(index + 1), base, tensors, result);
-      } else if (s->type()->child(0)->name() == "dict") {
-        return DeserializeDict(context, l->values(), l->value_offset(index),
-                               l->value_offset(index + 1), base, tensors, result);
+      const auto& s = static_cast<const StructArray&>(arr);
+      const auto& l = static_cast<const ListArray&>(*s.field(0));
+      if (s.type()->child(0)->name() == "list") {
+        return DeserializeList(context, *l.values(), l.value_offset(index),
+                               l.value_offset(index + 1), base, tensors, result);
+      } else if (s.type()->child(0)->name() == "tuple") {
+        return DeserializeTuple(context, *l.values(), l.value_offset(index),
+                                l.value_offset(index + 1), base, tensors, result);
+      } else if (s.type()->child(0)->name() == "dict") {
+        return DeserializeDict(context, *l.values(), l.value_offset(index),
+                               l.value_offset(index + 1), base, tensors, result);
+      } else if (s.type()->child(0)->name() == "set") {
+        return DeserializeSet(context, *l.values(), l.value_offset(index),
+                              l.value_offset(index + 1), base, tensors, result);
       } else {
-        DCHECK(false) << "unexpected StructArray type " << s->type()->child(0)->name();
+        DCHECK(false) << "unexpected StructArray type " << s.type()->child(0)->name();
       }
     }
     // We use an Int32Builder here to distinguish the tensor indices from
@@ -151,42 +152,72 @@ Status GetValue(PyObject* context, std::shared_ptr<Array> arr, int64_t index,
   return Status::OK();
 }
 
-#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                              \
-  auto data = std::dynamic_pointer_cast<UnionArray>(array);                       \
-  int64_t size = array->length();                                                 \
-  ScopedRef result(CREATE_FN(stop_idx - start_idx));                              \
-  auto types = std::make_shared<Int8Array>(size, data->type_ids());               \
-  auto offsets = std::make_shared<Int32Array>(size, data->value_offsets());       \
-  for (int64_t i = start_idx; i < stop_idx; ++i) {                                \
-    if (data->IsNull(i)) {                                                        \
-      Py_INCREF(Py_None);                                                         \
-      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                          \
-    } else {                                                                      \
-      int64_t offset = offsets->Value(i);                                         \
-      int8_t type = types->Value(i);                                              \
-      std::shared_ptr<Array> arr = data->child(type);                             \
-      PyObject* value;                                                            \
-      RETURN_NOT_OK(GetValue(context, arr, offset, type, base, tensors, &value)); \
-      SET_ITEM_FN(result.get(), i - start_idx, value);                            \
-    }                                                                             \
-  }                                                                               \
-  *out = result.release();                                                        \
+#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                                  \
+  const auto& data = static_cast<const UnionArray&>(array);                           \
+  int64_t size = array.length();                                                      \
+  ScopedRef result(CREATE_FN(stop_idx - start_idx));                                  \
+  auto types = std::make_shared<Int8Array>(size, data.type_ids());                    \
+  auto offsets = std::make_shared<Int32Array>(size, data.value_offsets());            \
+  for (int64_t i = start_idx; i < stop_idx; ++i) {                                    \
+    if (data.IsNull(i)) {                                                             \
+      Py_INCREF(Py_None);                                                             \
+      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                              \
+    } else {                                                                          \
+      int64_t offset = offsets->Value(i);                                             \
+      int8_t type = types->Value(i);                                                  \
+      PyObject* value;                                                                \
+      RETURN_NOT_OK(                                                                  \
+          GetValue(context, *data.child(type), offset, type, base, tensors, &value)); \
+      SET_ITEM_FN(result.get(), i - start_idx, value);                                \
+    }                                                                                 \
+  }                                                                                   \
+  *out = result.release();                                                            \
   return Status::OK()
 
-Status DeserializeList(PyObject* context, std::shared_ptr<Array> array, int64_t start_idx,
+Status DeserializeList(PyObject* context, const Array& array, int64_t start_idx,
                        int64_t stop_idx, PyObject* base,
                        const std::vector<std::shared_ptr<Tensor>>& tensors,
                        PyObject** out) {
   DESERIALIZE_SEQUENCE(PyList_New, PyList_SET_ITEM);
 }
 
-Status DeserializeTuple(PyObject* context, std::shared_ptr<Array> array,
-                        int64_t start_idx, int64_t stop_idx, PyObject* base,
+Status DeserializeTuple(PyObject* context, const Array& array, int64_t start_idx,
+                        int64_t stop_idx, PyObject* base,
                         const std::vector<std::shared_ptr<Tensor>>& tensors,
                         PyObject** out) {
   DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SET_ITEM);
 }
 
+Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
+                      int64_t stop_idx, PyObject* base,
+                      const std::vector<std::shared_ptr<Tensor>>& tensors,
+                      PyObject** out) {
+  const auto& data = static_cast<const UnionArray&>(array);
+  int64_t size = array.length();
+  ScopedRef result(PySet_New(nullptr));
+  auto types = std::make_shared<Int8Array>(size, data.type_ids());
+  auto offsets = std::make_shared<Int32Array>(size, data.value_offsets());
+  for (int64_t i = start_idx; i < stop_idx; ++i) {
+    if (data.IsNull(i)) {
+      Py_INCREF(Py_None);
+      if (PySet_Add(result.get(), Py_None) < 0) {
+        RETURN_IF_PYERROR();
+      }
+    } else {
+      int64_t offset = offsets->Value(i);
+      int8_t type = types->Value(i);
+      PyObject* value;
+      RETURN_NOT_OK(
+          GetValue(context, *data.child(type), offset, type, base, tensors, &value));
+      if (PySet_Add(result.get(), value) < 0) {
+        RETURN_IF_PYERROR();
+      }
+    }
+  }
+  *out = result.release();
+  return Status::OK();
+}
+
 Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out) {
   int64_t offset;
   int64_t bytes_read;
@@ -213,7 +244,7 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out)
 Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObject* base,
                          PyObject** out) {
   PyAcquireGIL lock;
-  return DeserializeList(context, obj.batch->column(0), 0, obj.batch->num_rows(), base,
+  return DeserializeList(context, *obj.batch->column(0), 0, obj.batch->num_rows(), base,
                          obj.tensors, out);
 }
 
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 65e5f6a46a876..c57091fc069e3 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -62,7 +62,8 @@ class SequenceBuilder {
         tensor_indices_(::arrow::int32(), pool),
         list_offsets_({0}),
         tuple_offsets_({0}),
-        dict_offsets_({0}) {}
+        dict_offsets_({0}),
+        set_offsets_({0}) {}
 
   /// Appending a none to the sequence
   Status AppendNone() {
@@ -163,6 +164,12 @@ class SequenceBuilder {
     return Status::OK();
   }
 
+  Status AppendSet(Py_ssize_t size) {
+    RETURN_NOT_OK(Update(set_offsets_.size() - 1, &set_tag_));
+    set_offsets_.push_back(set_offsets_.back() + static_cast<int32_t>(size));
+    return Status::OK();
+  }
+
   template <typename BuilderType>
   Status AddElement(const int8_t tag, BuilderType* out) {
     if (tag != -1) {
@@ -200,7 +207,7 @@ class SequenceBuilder {
   /// Finish building the sequence and return the result.
   /// Input arrays may be nullptr
   Status Finish(const Array* list_data, const Array* tuple_data, const Array* dict_data,
-                std::shared_ptr<Array>* out) {
+                const Array* set_data, std::shared_ptr<Array>* out) {
     fields_.resize(num_tags_);
     children_.resize(num_tags_);
 
@@ -215,6 +222,7 @@ class SequenceBuilder {
     RETURN_NOT_OK(AddSubsequence(list_tag_, list_data, list_offsets_, "list"));
     RETURN_NOT_OK(AddSubsequence(tuple_tag_, tuple_data, tuple_offsets_, "tuple"));
     RETURN_NOT_OK(AddSubsequence(dict_tag_, dict_data, dict_offsets_, "dict"));
+    RETURN_NOT_OK(AddSubsequence(set_tag_, set_data, set_offsets_, "set"));
 
     auto type = ::arrow::union_(fields_, type_ids_, UnionMode::DENSE);
     out->reset(new UnionArray(type, types_.length(), children_, types_.data(),
@@ -246,6 +254,7 @@ class SequenceBuilder {
   std::vector<int32_t> list_offsets_;
   std::vector<int32_t> tuple_offsets_;
   std::vector<int32_t> dict_offsets_;
+  std::vector<int32_t> set_offsets_;
 
   // Tags for members of the sequence. If they are set to -1 it means
   // they are not used and will not part be of the metadata when we call
@@ -263,6 +272,7 @@ class SequenceBuilder {
   int8_t list_tag_ = -1;
   int8_t tuple_tag_ = -1;
   int8_t dict_tag_ = -1;
+  int8_t set_tag_ = -1;
 
   int8_t num_tags_ = 0;
 
@@ -297,12 +307,14 @@ class DictBuilder {
   ///   value list of the dictionary
   Status Finish(const Array* key_tuple_data, const Array* key_dict_data,
                 const Array* val_list_data, const Array* val_tuple_data,
-                const Array* val_dict_data, std::shared_ptr<Array>* out) {
-    // lists and dicts can't be keys of dicts in Python, that is why for
+                const Array* val_dict_data, const Array* val_set_data,
+                std::shared_ptr<Array>* out) {
+    // lists and sets can't be keys of dicts in Python, that is why for
     // the keys we do not need to collect sublists
     std::shared_ptr<Array> keys, vals;
-    RETURN_NOT_OK(keys_.Finish(nullptr, key_tuple_data, key_dict_data, &keys));
-    RETURN_NOT_OK(vals_.Finish(val_list_data, val_tuple_data, val_dict_data, &vals));
+    RETURN_NOT_OK(keys_.Finish(nullptr, key_tuple_data, key_dict_data, nullptr, &keys));
+    RETURN_NOT_OK(
+        vals_.Finish(val_list_data, val_tuple_data, val_dict_data, val_set_data, &vals));
     auto keys_field = std::make_shared<Field>("keys", keys->type());
     auto vals_field = std::make_shared<Field>("vals", vals->type());
     auto type = std::make_shared<StructType>(
@@ -411,7 +423,8 @@ Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
 
 Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
               std::vector<PyObject*>* sublists, std::vector<PyObject*>* subtuples,
-              std::vector<PyObject*>* subdicts, std::vector<PyObject*>* tensors_out) {
+              std::vector<PyObject*>* subdicts, std::vector<PyObject*>* subsets,
+              std::vector<PyObject*>* tensors_out) {
   // The bool case must precede the int case (PyInt_Check passes for bools)
   if (PyBool_Check(elem)) {
     RETURN_NOT_OK(builder->AppendBool(elem == Py_True));
@@ -463,6 +476,9 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
   } else if (PyTuple_CheckExact(elem)) {
     RETURN_NOT_OK(builder->AppendTuple(PyTuple_Size(elem)));
     subtuples->push_back(elem);
+  } else if (PySet_Check(elem)) {
+    RETURN_NOT_OK(builder->AppendSet(PySet_Size(elem)));
+    subsets->push_back(elem);
   } else if (PyArray_IsScalar(elem, Generic)) {
     RETURN_NOT_OK(AppendScalar(elem, builder));
   } else if (PyArray_Check(elem)) {
@@ -522,14 +538,14 @@ Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
         "recursively.");
   }
   SequenceBuilder builder(nullptr);
-  std::vector<PyObject*> sublists, subtuples, subdicts;
+  std::vector<PyObject*> sublists, subtuples, subdicts, subsets;
   for (const auto& sequence : sequences) {
     ScopedRef iterator(PyObject_GetIter(sequence));
     RETURN_IF_PYERROR();
     ScopedRef item;
     while (item.reset(PyIter_Next(iterator.get())), item.get()) {
       RETURN_NOT_OK(Append(context, item.get(), &builder, &sublists, &subtuples,
-                           &subdicts, tensors_out));
+                           &subdicts, &subsets, tensors_out));
     }
   }
   std::shared_ptr<Array> list;
@@ -547,7 +563,12 @@ Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
     RETURN_NOT_OK(
         SerializeDict(context, subdicts, recursion_depth + 1, &dict, tensors_out));
   }
-  return builder.Finish(list.get(), tuple.get(), dict.get(), out);
+  std::shared_ptr<Array> set;
+  if (subsets.size() > 0) {
+    RETURN_NOT_OK(
+        SerializeSequences(context, subsets, recursion_depth + 1, &set, tensors_out));
+  }
+  return builder.Finish(list.get(), tuple.get(), dict.get(), set.get(), out);
 }
 
 Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
@@ -559,16 +580,17 @@ Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
         "This object exceeds the maximum recursion depth. It may contain itself "
         "recursively.");
   }
-  std::vector<PyObject*> key_tuples, key_dicts, val_lists, val_tuples, val_dicts, dummy;
+  std::vector<PyObject*> key_tuples, key_dicts, val_lists, val_tuples, val_dicts,
+      val_sets, dummy;
   for (const auto& dict : dicts) {
     PyObject *key, *value;
     Py_ssize_t pos = 0;
     while (PyDict_Next(dict, &pos, &key, &value)) {
       RETURN_NOT_OK(Append(context, key, &result.keys(), &dummy, &key_tuples, &key_dicts,
-                           tensors_out));
+                           &dummy, tensors_out));
       DCHECK_EQ(dummy.size(), 0);
       RETURN_NOT_OK(Append(context, value, &result.vals(), &val_lists, &val_tuples,
-                           &val_dicts, tensors_out));
+                           &val_dicts, &val_sets, tensors_out));
     }
   }
   std::shared_ptr<Array> key_tuples_arr;
@@ -596,9 +618,14 @@ Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
     RETURN_NOT_OK(SerializeDict(context, val_dicts, recursion_depth + 1, &val_dict_arr,
                                 tensors_out));
   }
+  std::shared_ptr<Array> val_set_arr;
+  if (val_sets.size() > 0) {
+    RETURN_NOT_OK(SerializeSequences(context, val_sets, recursion_depth + 1, &val_set_arr,
+                                     tensors_out));
+  }
   RETURN_NOT_OK(result.Finish(key_tuples_arr.get(), key_dicts_arr.get(),
                               val_list_arr.get(), val_tuples_arr.get(),
-                              val_dict_arr.get(), out));
+                              val_dict_arr.get(), val_set_arr.get(), out));
 
   // This block is used to decrement the reference counts of the results
   // returned by the serialization callback, which is called in SerializeArray,
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index ae486987601bf..8c7c4e2014465 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -160,9 +160,15 @@ void AssertBatchValid(const RecordBatch& batch) {
   }
 }
 
+RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows)
+    : schema_(schema), num_rows_(num_rows) {
+  boxed_columns_.resize(schema->num_fields());
+}
+
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
                          const std::vector<std::shared_ptr<Array>>& columns)
-    : schema_(schema), num_rows_(num_rows), columns_(columns.size()) {
+    : RecordBatch(schema, num_rows) {
+  columns_.resize(columns.size());
   for (size_t i = 0; i < columns.size(); ++i) {
     columns_[i] = columns[i]->data();
   }
@@ -170,7 +176,8 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
                          std::vector<std::shared_ptr<Array>>&& columns)
-    : schema_(schema), num_rows_(num_rows), columns_(columns.size()) {
+    : RecordBatch(schema, num_rows) {
+  columns_.resize(columns.size());
   for (size_t i = 0; i < columns.size(); ++i) {
     columns_[i] = columns[i]->data();
   }
@@ -178,16 +185,21 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
                          std::vector<std::shared_ptr<internal::ArrayData>>&& columns)
-    : schema_(schema), num_rows_(num_rows), columns_(std::move(columns)) {}
+    : RecordBatch(schema, num_rows) {
+  columns_ = std::move(columns);
+}
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
                          const std::vector<std::shared_ptr<internal::ArrayData>>& columns)
-    : schema_(schema), num_rows_(num_rows), columns_(columns) {}
+    : RecordBatch(schema, num_rows) {
+  columns_ = columns;
+}
 
 std::shared_ptr<Array> RecordBatch::column(int i) const {
-  std::shared_ptr<Array> result;
-  DCHECK(MakeArray(columns_[i], &result).ok());
-  return result;
+  if (!boxed_columns_[i]) {
+    DCHECK(internal::MakeArray(columns_[i], &boxed_columns_[i]).ok());
+  }
+  return boxed_columns_[i];
 }
 
 const std::string& RecordBatch::column_name(int i) const {
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 1145d113e2518..da2722da1a3f0 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -153,13 +153,9 @@ class ARROW_EXPORT RecordBatch {
   /// \return true if batches are equal
   std::shared_ptr<Schema> schema() const { return schema_; }
 
-  /// \brief Retrieve an array from the record batch (new object)
+  /// \brief Retrieve an array from the record batch
   /// \param[in] i field index, does not boundscheck
-  /// \return a new Array object
-  ///
-  /// \note This function returns a new object. If you intend to dereference
-  /// the pointer or access the internals, retain a reference to the
-  /// std::shared_ptr returned.
+  /// \return an Array object
   std::shared_ptr<Array> column(int i) const;
 
   std::shared_ptr<internal::ArrayData> column_data(int i) const { return columns_[i]; }
@@ -197,9 +193,14 @@ class ARROW_EXPORT RecordBatch {
   Status Validate() const;
 
  private:
+  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows);
+
   std::shared_ptr<Schema> schema_;
   int64_t num_rows_;
   std::vector<std::shared_ptr<internal::ArrayData>> columns_;
+
+  // Caching boxed array data
+  mutable std::vector<std::shared_ptr<Array>> boxed_columns_;
 };
 
 /// \class Table
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index aca3848ae9c29..7c8cace8f0762 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -90,6 +90,7 @@ def assert_equal(obj1, obj2):
     [1, 2, 3, None], [(None,), 3, 1.0], ["h", "e", "l", "l", "o", None],
     (None, None), ("hello", None), (True, False),
     {True: "hello", False: "world"}, {"hello": "world", 1: 42, 2.5: 45},
+    {"hello": set([2, 3]), "world": set([42.0]), "this": None},
     np.int8(3), np.int32(4), np.int64(5),
     np.uint8(3), np.uint32(4), np.uint64(5), np.float32(1.9),
     np.float64(1.9), np.zeros([100, 100]),

From 8911a34633d6d6957d07d8b2f579c22ecab43d95 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 8 Sep 2017 18:33:18 -0400
Subject: [PATCH 1025/1644] ARROW-1493: [C++] Flush stream in PrettyPrint
 functions

I'm not sure how to test this.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1066 from wesm/ARROW-1493 and squashes the following commits:

ffbb9af8 [Wes McKinney] Flush stream in PrettyPrint functions
---
 cpp/src/arrow/pretty_print.cc | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 1202faa876e48..3f64d75e01de6 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -45,6 +45,10 @@ class PrettyPrinter {
   void OpenArray();
   void CloseArray();
 
+  void Flush() {
+    (*sink_) << std::flush;
+  }
+
  protected:
   int indent_;
   std::ostream* sink_;
@@ -298,7 +302,11 @@ class ArrayPrinter : public PrettyPrinter {
     return PrettyPrint(*array.indices(), indent_ + 2, sink_);
   }
 
-  Status Print() { return VisitArrayInline(array_, this); }
+  Status Print() {
+    RETURN_NOT_OK(VisitArrayInline(array_, this));
+    Flush();
+    return Status::OK();
+  }
 
  private:
   const Array& array_;
@@ -330,6 +338,7 @@ Status PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink) {
     RETURN_NOT_OK(PrettyPrint(*batch.column(i), indent + 2, sink));
     (*sink) << "\n";
   }
+  (*sink) << std::flush;
   return Status::OK();
 }
 
@@ -352,6 +361,7 @@ class SchemaPrinter : public PrettyPrinter {
       }
       RETURN_NOT_OK(PrintField(*schema_.field(i)));
     }
+    Flush();
     return Status::OK();
   }
 

From 0c8853f90612b485c853cb54acf34a820591ca1d Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Fri, 8 Sep 2017 18:42:08 -0400
Subject: [PATCH 1026/1644] ARROW-1479: [JS] Expand JavaScript implementation

Finally PR'ing https://github.com/graphistry/arrow to the official Arrow project

Primary motivations:
- Multi-part ArrowReader to support async and streaming
- ArrowReader and related types as native JS iterators (and soon, async iterators)
- Unify vectors across RecordBatches into a single logical virtual instance
- Fast vector subarray/slice to dense typed Arrays (handy for GPU compute)

Example:

```js
import { readFileSync } from 'fs';
import { Table } from 'apache-arrow';

const table = Table.from(...[
    'latlong/schema.arrow',
    'latlong/records.arrow'
].map((file) => readFileSync(file)));

const column = table.getColumn('origin_lat');
const typed = column.slice();

assert(typed instanceof Float32Array);

for (let i = -1, n = column.length; ++i < n;) {
    assert(column.get(i) === typed[i]);
}

console.log(table.toString());

/*
        origin_lat,         origin_lon
35.393089294433594,  -97.6007308959961
35.393089294433594,  -97.6007308959961
35.393089294433594,  -97.6007308959961
29.533695220947266, -98.46977996826172
29.533695220947266, -98.46977996826172
*/
```

Todos:
- ArrowWriter
- Map type?
- [`readBuffersAsync`/`readStreamAsync`](https://github.com/graphistry/arrow/issues/8)
- fix prepublish script for Arrow project CI/deploy strategy

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1062 from trxcllnt/new-arrow-js and squashes the following commits:

4f71e769 [Paul Taylor] add more rat excludes and delete changelog
fc01b80a [Paul Taylor] Update license/docs per Wes's feedback
5f66c097 [Paul Taylor] add .npmrc to rat exlusions
927ce614 [Paul Taylor] Merge branch 'master' into new-arrow-js
f6da81eb [Paul Taylor] Fix rat exlusions, add header to prepublish script
aff1fdb9 [Paul Taylor] Tweak the JS CI scripts, fix case-sensitive file name in git
8b9208a1 [Paul Taylor] Merge branch 'master' into new-arrow-js
4a74d856 [Paul Taylor] Update CI scripts for new Arrow JS lib
28261892 [Paul Taylor] chore(js): Move graphistry/arrow into apache/arrow
---
 ci/travis_before_script_js.sh                 |   10 -
 ci/travis_script_js.sh                        |    6 +-
 dev/release/rat_exclude_files.txt             |    5 +
 js/.gitignore                                 |   96 +-
 js/.npmignore                                 |    7 -
 js/.npmrc                                     |    1 +
 js/DEVELOP.md                                 |  140 +
 js/LICENSE                                    |   39 +
 js/README.md                                  |  208 +-
 js/bin/arrow2csv.js                           |   92 +-
 js/bower.json                                 |   17 -
 js/closure-compiler-scripts/File_generated.js |  264 ++
 .../Message_generated.js                      |  486 ++++
 .../Schema_generated.js                       | 2231 ++++++++++++++++
 js/closure-compiler-scripts/flatbuffers.js    | 1204 +++++++++
 js/closure-compiler-scripts/text-encoding.js  |  648 +++++
 js/closure-compiler-scripts/tslib.js          |  151 ++
 js/examples/read_file.html                    |   39 +-
 js/gulpfile.js                                |  285 +++
 js/lerna.json                                 |    9 +
 js/package.json                               |  153 +-
 .../arrows/file}/dictionary.arrow             |  Bin
 js/{spec => perf/arrows/file}/simple.arrow    |  Bin
 .../arrows/file/struct.arrow}                 |  Bin
 js/perf/arrows/multi/count/records.arrow      |  Bin 0 -> 224 bytes
 js/perf/arrows/multi/count/schema.arrow       |  Bin 0 -> 184 bytes
 js/perf/arrows/multi/latlong/records.arrow    |  Bin 0 -> 352 bytes
 js/perf/arrows/multi/latlong/schema.arrow     |  Bin 0 -> 264 bytes
 js/perf/arrows/multi/origins/records.arrow    |  Bin 0 -> 224 bytes
 js/perf/arrows/multi/origins/schema.arrow     |  Bin 0 -> 1604 bytes
 .../arrows/stream/dictionary.arrow}           |  Bin
 .../arrows/stream/simple.arrow}               |  Bin
 .../arrows/stream/struct.arrow}               |  Bin
 js/perf/config.js                             |   38 +
 js/perf/index.js                              |  113 +
 js/{flatbuffers.sh => prepublish.sh}          |   20 +-
 js/spec/arrow.js                              |  179 --
 js/src/Arrow.externs.ts                       |   67 +
 js/src/Arrow.internal.ts                      |  105 +
 js/{bin/arrow_schema.js => src/Arrow.ts}      |   24 +-
 js/src/Arrow_generated.d.ts                   |    5 -
 js/src/arrow.ts                               |  515 ----
 js/src/format/File_generated.ts               |  240 ++
 js/src/format/Message_generated.ts            |  469 ++++
 js/src/format/Schema_generated.ts             | 2254 +++++++++++++++++
 js/src/reader/arrow.ts                        |   78 +
 js/src/reader/dictionary.ts                   |   43 +
 js/src/reader/file.ts                         |   79 +
 js/src/reader/message.ts                      |   63 +
 js/src/reader/stream.ts                       |   43 +
 js/src/reader/vector.ts                       |  271 ++
 js/src/table.ts                               |  133 +
 js/src/types.ts                               |  597 -----
 js/src/vector/dictionary.ts                   |   51 +
 js/src/vector/list.ts                         |  108 +
 js/src/{bitarray.ts => vector/struct.ts}      |   37 +-
 js/src/vector/typed.ts                        |  326 +++
 js/src/vector/vector.ts                       |   91 +
 js/test/Arrow.ts                              |   67 +
 js/test/__snapshots__/reader-tests.ts.snap    |  497 ++++
 js/test/__snapshots__/table-tests.ts.snap     | 1815 +++++++++++++
 js/test/arrows/file/dictionary.arrow          |  Bin 0 -> 2522 bytes
 js/test/arrows/file/dictionary2.arrow         |  Bin 0 -> 2762 bytes
 js/test/arrows/file/multi_dictionary.arrow    |  Bin 0 -> 3482 bytes
 js/test/arrows/file/simple.arrow              |  Bin 0 -> 1642 bytes
 js/test/arrows/file/struct.arrow              |  Bin 0 -> 2354 bytes
 js/test/arrows/multi/count/records.arrow      |  Bin 0 -> 224 bytes
 js/test/arrows/multi/count/schema.arrow       |  Bin 0 -> 184 bytes
 js/test/arrows/multi/latlong/records.arrow    |  Bin 0 -> 352 bytes
 js/test/arrows/multi/latlong/schema.arrow     |  Bin 0 -> 264 bytes
 js/test/arrows/multi/origins/records.arrow    |  Bin 0 -> 224 bytes
 js/test/arrows/multi/origins/schema.arrow     |  Bin 0 -> 1604 bytes
 js/test/arrows/stream/dictionary.arrow        |  Bin 0 -> 1776 bytes
 js/test/arrows/stream/simple.arrow            |  Bin 0 -> 1188 bytes
 js/test/arrows/stream/struct.arrow            |  Bin 0 -> 1884 bytes
 js/test/reader-tests.ts                       |   50 +
 js/test/table-tests.ts                        |   88 +
 js/test/test-config.ts                        |   42 +
 js/test/tsconfig.json                         |   11 +
 js/test/vector-tests.ts                       |  226 ++
 js/tsconfig.json                              |   19 +-
 js/tsconfig/tsconfig.base.json                |   25 +
 js/tsconfig/tsconfig.es2015.cjs.json          |    8 +
 js/tsconfig/tsconfig.es2015.cls.json          |   10 +
 js/tsconfig/tsconfig.es2015.esm.json          |    8 +
 js/tsconfig/tsconfig.es5.cjs.json             |    8 +
 js/tsconfig/tsconfig.es5.cls.json             |   10 +
 js/tsconfig/tsconfig.es5.esm.json             |    8 +
 js/tsconfig/tsconfig.esnext.cjs.json          |    8 +
 js/tsconfig/tsconfig.esnext.cls.json          |   10 +
 js/tsconfig/tsconfig.esnext.esm.json          |    8 +
 js/tslint.json                                |   49 +-
 js/webpack.config.js                          |   54 -
 93 files changed, 13509 insertions(+), 1552 deletions(-)
 delete mode 100644 js/.npmignore
 create mode 100644 js/.npmrc
 create mode 100644 js/DEVELOP.md
 create mode 100644 js/LICENSE
 delete mode 100644 js/bower.json
 create mode 100644 js/closure-compiler-scripts/File_generated.js
 create mode 100644 js/closure-compiler-scripts/Message_generated.js
 create mode 100644 js/closure-compiler-scripts/Schema_generated.js
 create mode 100644 js/closure-compiler-scripts/flatbuffers.js
 create mode 100644 js/closure-compiler-scripts/text-encoding.js
 create mode 100644 js/closure-compiler-scripts/tslib.js
 create mode 100644 js/gulpfile.js
 create mode 100644 js/lerna.json
 rename js/{spec => perf/arrows/file}/dictionary.arrow (100%)
 rename js/{spec => perf/arrows/file}/simple.arrow (100%)
 rename js/{spec/struct_example.arrow => perf/arrows/file/struct.arrow} (100%)
 create mode 100644 js/perf/arrows/multi/count/records.arrow
 create mode 100644 js/perf/arrows/multi/count/schema.arrow
 create mode 100644 js/perf/arrows/multi/latlong/records.arrow
 create mode 100644 js/perf/arrows/multi/latlong/schema.arrow
 create mode 100644 js/perf/arrows/multi/origins/records.arrow
 create mode 100644 js/perf/arrows/multi/origins/schema.arrow
 rename js/{spec/dictionary-stream.arrow => perf/arrows/stream/dictionary.arrow} (100%)
 rename js/{spec/simple-stream.arrow => perf/arrows/stream/simple.arrow} (100%)
 rename js/{spec/struct_example-stream.arrow => perf/arrows/stream/struct.arrow} (100%)
 create mode 100644 js/perf/config.js
 create mode 100644 js/perf/index.js
 rename js/{flatbuffers.sh => prepublish.sh} (66%)
 mode change 100755 => 100644
 delete mode 100644 js/spec/arrow.js
 create mode 100644 js/src/Arrow.externs.ts
 create mode 100644 js/src/Arrow.internal.ts
 rename js/{bin/arrow_schema.js => src/Arrow.ts} (62%)
 mode change 100755 => 100644
 delete mode 100644 js/src/Arrow_generated.d.ts
 delete mode 100644 js/src/arrow.ts
 create mode 100644 js/src/format/File_generated.ts
 create mode 100644 js/src/format/Message_generated.ts
 create mode 100644 js/src/format/Schema_generated.ts
 create mode 100644 js/src/reader/arrow.ts
 create mode 100644 js/src/reader/dictionary.ts
 create mode 100644 js/src/reader/file.ts
 create mode 100644 js/src/reader/message.ts
 create mode 100644 js/src/reader/stream.ts
 create mode 100644 js/src/reader/vector.ts
 create mode 100644 js/src/table.ts
 delete mode 100644 js/src/types.ts
 create mode 100644 js/src/vector/dictionary.ts
 create mode 100644 js/src/vector/list.ts
 rename js/src/{bitarray.ts => vector/struct.ts} (51%)
 create mode 100644 js/src/vector/typed.ts
 create mode 100644 js/src/vector/vector.ts
 create mode 100644 js/test/Arrow.ts
 create mode 100644 js/test/__snapshots__/reader-tests.ts.snap
 create mode 100644 js/test/__snapshots__/table-tests.ts.snap
 create mode 100644 js/test/arrows/file/dictionary.arrow
 create mode 100644 js/test/arrows/file/dictionary2.arrow
 create mode 100644 js/test/arrows/file/multi_dictionary.arrow
 create mode 100644 js/test/arrows/file/simple.arrow
 create mode 100644 js/test/arrows/file/struct.arrow
 create mode 100644 js/test/arrows/multi/count/records.arrow
 create mode 100644 js/test/arrows/multi/count/schema.arrow
 create mode 100644 js/test/arrows/multi/latlong/records.arrow
 create mode 100644 js/test/arrows/multi/latlong/schema.arrow
 create mode 100644 js/test/arrows/multi/origins/records.arrow
 create mode 100644 js/test/arrows/multi/origins/schema.arrow
 create mode 100644 js/test/arrows/stream/dictionary.arrow
 create mode 100644 js/test/arrows/stream/simple.arrow
 create mode 100644 js/test/arrows/stream/struct.arrow
 create mode 100644 js/test/reader-tests.ts
 create mode 100644 js/test/table-tests.ts
 create mode 100644 js/test/test-config.ts
 create mode 100644 js/test/tsconfig.json
 create mode 100644 js/test/vector-tests.ts
 create mode 100644 js/tsconfig/tsconfig.base.json
 create mode 100644 js/tsconfig/tsconfig.es2015.cjs.json
 create mode 100644 js/tsconfig/tsconfig.es2015.cls.json
 create mode 100644 js/tsconfig/tsconfig.es2015.esm.json
 create mode 100644 js/tsconfig/tsconfig.es5.cjs.json
 create mode 100644 js/tsconfig/tsconfig.es5.cls.json
 create mode 100644 js/tsconfig/tsconfig.es5.esm.json
 create mode 100644 js/tsconfig/tsconfig.esnext.cjs.json
 create mode 100644 js/tsconfig/tsconfig.esnext.cls.json
 create mode 100644 js/tsconfig/tsconfig.esnext.esm.json
 delete mode 100644 js/webpack.config.js

diff --git a/ci/travis_before_script_js.sh b/ci/travis_before_script_js.sh
index b72accc219332..1d6bece0e8b54 100755
--- a/ci/travis_before_script_js.sh
+++ b/ci/travis_before_script_js.sh
@@ -21,19 +21,9 @@
 set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-
-# Download flatbuffers
-export FLATBUFFERS_HOME=$TRAVIS_BUILD_DIR/flatbuffers
-conda create -y -q -p $FLATBUFFERS_HOME python=2.7 flatbuffers
-export PATH="$FLATBUFFERS_HOME/bin:$PATH"
-
-npm install -g typescript
-npm install -g webpack
 
 pushd $ARROW_JS_DIR
 
 npm install
-npm run build
 
 popd
diff --git a/ci/travis_script_js.sh b/ci/travis_script_js.sh
index cb1e9e194408a..9f77dec8c53a7 100755
--- a/ci/travis_script_js.sh
+++ b/ci/travis_script_js.sh
@@ -23,6 +23,10 @@ JS_DIR=${TRAVIS_BUILD_DIR}/js
 
 pushd $JS_DIR
 
-npm test
+npm run validate
+
+# Uncomment to use coveralls
+# npm run test:coverage
+# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
 
 popd
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 15a38934e3f5e..83c9f194791ee 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -1,6 +1,10 @@
+*.npmrc
 *.gitignore
 *_generated.h
+*_generated.js
+*_generated.ts
 *.json
+*.snap
 cpp/src/arrow/io/mman.h
 cpp/src/arrow/util/random.h
 cpp/src/arrow/status.cc
@@ -29,6 +33,7 @@ cpp/src/plasma/thirdparty/xxhash.cc
 cpp/src/plasma/thirdparty/xxhash.h
 dev/release/rat_exclude_files.txt
 js/.npmignore
+js/closure-compiler-scripts/*
 python/cmake_modules/BuildUtils.cmake
 python/cmake_modules/FindPythonLibsNew.cmake
 python/cmake_modules/FindNumPy.cmake
diff --git a/js/.gitignore b/js/.gitignore
index ea5514f634d37..b48f35b960ffb 100644
--- a/js/.gitignore
+++ b/js/.gitignore
@@ -1,7 +1,89 @@
-src/Arrow_generated.js
-lib
-lib-esm
-_bundles
-node_modules
-.idea
-*.iml
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+.vscode
+yarn.lock
+package-lock.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# JS package manager files
+yarn.lock
+package-lock.json
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+# compilation targets
+dist
+targets/es5
+targets/es2015
+targets/esnext
diff --git a/js/.npmignore b/js/.npmignore
deleted file mode 100644
index 333aeec571a21..0000000000000
--- a/js/.npmignore
+++ /dev/null
@@ -1,7 +0,0 @@
-.gitignore
-.npmignore
-src/
-spec/
-tsconfig.json
-webpack.config.js
-flatbuffers.sh
diff --git a/js/.npmrc b/js/.npmrc
new file mode 100644
index 0000000000000..43c97e719a5a8
--- /dev/null
+++ b/js/.npmrc
@@ -0,0 +1 @@
+package-lock=false
diff --git a/js/DEVELOP.md b/js/DEVELOP.md
new file mode 100644
index 0000000000000..5b4ac14ed3466
--- /dev/null
+++ b/js/DEVELOP.md
@@ -0,0 +1,140 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# The npm scripts
+
+* `npm run clean` - cleans targets
+* `npm run build` - cleans and compiles all targets
+* `npm test` - executes tests against built targets
+
+These npm scripts accept argument lists of targets × modules:
+
+* Available `targets` are `es5`, `es2015`, `esnext`, and `all` (default: `all`)
+* Available `modules` are `cjs`, `esm`, `umd`, and `all` (default: `all`)
+
+Examples:
+
+* `npm run build` -- builds all ES targets in all module formats
+* `npm run build -- -t es5 -m all` -- builds the ES5 target in all module formats
+* `npm run build -- -t all -m cjs` -- builds all ES targets in the CommonJS module format
+* `npm run build -- --targets es5 es2015 -m all` -- builds the ES5 and ES2015 targets in all module formats
+* `npm run build -- -t es5 --modules cjs esm` -- builds the ES5 target in CommonJS and ESModules module formats
+
+This argument configuration also applies to `clean` and `test` scripts.
+
+* `npm run deploy`
+
+Uses [learna](https://github.com/lerna/lerna) to publish each build target to npm with [conventional](https://conventionalcommits.org/) [changelogs](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-changelog-cli).
+
+# Updating the Arrow format flatbuffers generated code
+
+Once generated, the flatbuffers format code needs to be adjusted for our TS and JS build environments.
+
+## TypeScript
+
+1. Generate the flatbuffers TypeScript source from the Arrow project root directory:
+    ```sh
+    flatc --ts -o ./js/src/format ./format/*.fbs
+    ```
+1. Change all the `flatbuffers` imports to
+    ```ts
+    import { flatbuffers } from "flatbuffers"
+    ```
+1. Delete `Tensor_generated.ts` (remove this step once we support Tensors)
+1. Remove Tensor import from `Schema_generated.ts`
+1. Add `/* tslint:disable:class-name */` to the top of `Schema_generated.ts`
+
+## JavaScript (for Google Closure Compiler builds)
+
+1. Generate the flatbuffers JS source from the Arrow project root directory
+    ```sh
+    flatc --js -o ./js/closure-compiler-scripts ./format/*.fbs
+    ```
+1. Delete `Tensor_generated.js` (remove this step once we support Tensors)
+1. Add `goog.module` declarations to the top of each generated file
+
+    Each file starts with a header that looks like this:
+    ```js
+    // automatically generated by the FlatBuffers compiler, do not modify
+
+    /**
+    * @const
+    * @namespace
+    */
+    var org = org || {};
+    ```
+
+    Update the header of each file to explicitly declare its module.
+
+    `Schema_generated.js`:
+    ```js
+    // automatically generated by the FlatBuffers compiler, do not modify
+    goog.module("module$targets$es5$cls$format$Schema_generated");
+    goog.module.declareLegacyNamespace();
+
+    /**
+    * @const
+    * @namespace
+    */
+    var org = org || {};
+    ```
+
+    `File_generated.js`:
+
+    ```js
+    // automatically generated by the FlatBuffers compiler, do not modify
+    goog.module("module$targets$es5$cls$format$File_generated");
+    goog.module.declareLegacyNamespace();
+    var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
+    /**
+    * @const
+    * @namespace
+    */
+    var org = Schema_.org;
+    ```
+
+    `Message_generated.js`:
+
+    ```js
+    // automatically generated by the FlatBuffers compiler, do not modify
+    goog.module("module$targets$es5$cls$format$Message_generated");
+    goog.module.declareLegacyNamespace();
+    var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
+    /**
+    * @const
+    * @namespace
+    */
+    var org = Schema_.org;
+    ```
+
+1. Replace the last line's export declaration
+
+    The last line of each file is:
+
+    ```js
+    // Exports for Node.js and RequireJS
+    this.org = org;
+    ```
+
+    This should instead read:
+
+    ```js
+    // Exports for Node.js and RequireJS
+    exports.org = org;
+    ```
\ No newline at end of file
diff --git a/js/LICENSE b/js/LICENSE
new file mode 100644
index 0000000000000..02e7948085055
--- /dev/null
+++ b/js/LICENSE
@@ -0,0 +1,39 @@
+## 3rd-party licenses for code that has been adapted for the Arrow JavaScript
+   library
+
+--------------------------------------------------------------------------------
+
+This project includes code from the FlatBuffers project
+
+Copyright 2014 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the tslib project
+
+Copyright 2015 Microsoft Corporation. All rights reserved. 
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
diff --git a/js/README.md b/js/README.md
index 38e8fafcea9c5..bee3a9c7d42db 100644
--- a/js/README.md
+++ b/js/README.md
@@ -17,50 +17,194 @@
   under the License.
 -->
 
-### Installation
+# [Apache Arrow](https://github.com/apache/arrow) in JS
 
-From this directory, run:
+[![Build Status](https://travis-ci.org/apache/arrow.svg?branch=master)](https://travis-ci.org/apache/arrow)
+[![Coverage Status](https://coveralls.io/repos/github/apache/arrow/badge.svg)](https://coveralls.io/github/apache/arrow)
 
-``` bash
-$ npm install   # pull dependencies
-$ npm run lint -- <filename>  # run tslint
-$ npm run build # build typescript (run tsc and webpack)
-$ npm run test  # run the unit tests (node.js only)
+Arrow is a set of technologies that enable big-data systems to process and move data fast.
+
+## install [apache-arrow from npm](https://www.npmjs.com/package/apache-arrow)
+
+`npm install apache-arrow`
+
+(read about how we [package apache-arrow](#packaging) below)
+
+# Powering Columnar In-Memory Analytics
+
+Apache Arrow is a columnar memory layout specification for encoding vectors and table-like containers of flat and nested data. The Arrow spec aligns columnar data in memory to minimize cache misses and take advantage of the latest SIMD (Single input multiple data) and GPU operations on modern processors.
+
+Apache Arrow is the emerging standard for large in-memory columnar data ([Spark](https://spark.apache.org/), [Pandas](http://wesmckinney.com/blog/pandas-and-apache-arrow/), [Drill](https://drill.apache.org/), ...). By standardizing on a common binary interchange format, big data systems can reduce the costs and friction associated with cross-system communication.
+
+# Related Projects
+
+* [GoAI](http://gpuopenanalytics.com/) -- Arrow-powered GPU analytics
+* [rxjs-mapd](https://github.com/graphistry/rxjs-mapd) -- A MapD Core node-driver that returns query results as Arrow columns
+
+# Usage
+
+## Get a table from an Arrow file on disk
+
+```es6
+import { readFileSync } from 'fs';
+import { Table } from 'apache-arrow';
+
+const arrow = readFileSync('simple.arrow');
+const table = Table.from(arrow);
+
+console.log(table.toString());
+
+/*
+ foo,  bar,  baz
+   1,    1,   aa
+null, null, null
+   3, null, null
+   4,    4,  bbb
+   5,    5, cccc
+*/
 ```
 
-### Usage
-The library is designed to be used with node.js or in the browser, this repository contains examples of both.
+## Create a Table when the Arrow file is split across buffers
+
+```es6
+import { readFileSync } from 'fs';
+import { Table } from 'apache-arrow';
 
-#### Node
-Import the arrow module:
+const table = Table.from(...[
+    'latlong/schema.arrow',
+    'latlong/records.arrow'
+].map((file) => readFileSync(file)));
 
-``` js
-var arrow = require("arrow");
+console.log(table.toString());
+
+/*
+        origin_lat,         origin_lon
+35.393089294433594,  -97.6007308959961
+35.393089294433594,  -97.6007308959961
+35.393089294433594,  -97.6007308959961
+29.533695220947266, -98.46977996826172
+29.533695220947266, -98.46977996826172
+*/
 ```
 
-See [bin/arrow_schema.js](bin/arrow_schema.js) and [bin/arrow2csv.js](bin/arrow2csv.js) for usage examples.
+## Columns are what you'd expect
+
+```es6
+import { readFileSync } from 'fs';
+import { Table } from 'apache-arrow';
+
+const table = Table.from(...[
+    'latlong/schema.arrow',
+    'latlong/records.arrow'
+].map(readFileSync));
+
+const column = table.getColumn('origin_lat');
+const typed = column.slice();
+
+assert(typed instanceof Float32Array);
 
-#### Browser
-Include `_bundles/arrow.js` in a `<script />` tag:
-``` html
-<script src="_bundles/arrow.js"/>
+for (let i = -1, n = column.length; ++i < n;) {
+    assert(column.get(i) === typed[i]);
+}
 ```
-See [examples/read_file.html](examples/read_file.html) for a usage example.
 
-### API
-##### `arrow.getReader(buffer)`
-Returns an `ArrowReader` object representing the Arrow file or stream contained in
-the `buffer`.
+## Usage with MapD Core
+
+```es6
+import MapD from 'rxjs-mapd';
+import { Table } from 'apache-arrow';
+
+const port = 9091;
+const host = `localhost`;
+const db = `mapd`;
+const user = `mapd`;
+const password = `HyperInteractive`;
+
+MapD.open(host, port)
+  .connect(db, user, password)
+  .flatMap((session) =>
+    // queryDF returns Arrow buffers
+    session.queryDF(`
+      SELECT origin_city
+      FROM flights
+      WHERE dest_city ILIKE 'dallas'
+      LIMIT 5`
+    ).disconnect()
+  )
+  .map(([schema, records]) =>
+    // Create Arrow Table from results
+    Table.from(schema, records))
+  .map((table) =>
+    // Stringify the table to CSV
+    table.toString({ index: true }))
+  .subscribe((csvStr) =>
+    console.log(csvStr));
+/*
+Index,   origin_city
+    0, Oklahoma City
+    1, Oklahoma City
+    2, Oklahoma City
+    3,   San Antonio
+    4,   San Antonio
+*/
+```
+
+# Getting involved
+
+See [develop.md](https://github.com/apache/arrow/blob/master/develop.md)
+
+Even if you do not plan to contribute to Apache Arrow itself or Arrow
+integrations in other projects, we'd be happy to have you involved:
+
+* Join the mailing list: send an email to
+  [dev-subscribe@arrow.apache.org][1]. Share your ideas and use cases for the
+  project.
+* [Follow our activity on JIRA][3]
+* [Learn the format][2]
+* Contribute code to one of the reference implementations
+
+We prefer to receive contributions in the form of GitHub pull requests. Please send pull requests against the [github.com/apache/arrow][4] repository.
+
+If you are looking for some ideas on what to contribute, check out the [JIRA
+issues][3] for the Apache Arrow project. Comment on the issue and/or contact
+[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+with your questions and ideas.
+
+If you’d like to report a bug but don’t have time to fix it, you can still post
+it on JIRA, or email the mailing list
+[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+
+## Packaging
+
+`apache-arrow` is written in TypeScript, but the project is compiled to multiple JS versions and common module formats.
+
+The base `apache-arrow` package includes all the compilation targets for convenience, but if you're conscientious about your `node_modules` footprint, we got you.
+
+The targets are also published under the `@apache-arrow` namespace:
+
+```sh
+npm install @apache-arrow/es5-cjs # ES5 CommonJS target
+npm install @apache-arrow/es5-esm # ES5 ESModules target
+npm install @apache-arrow/es5-umd # ES5 UMD target
+npm install @apache-arrow/es2015-cjs # ES2015 CommonJS target
+npm install @apache-arrow/es2015-esm # ES2015 ESModules target
+npm install @apache-arrow/es2015-umd # ES2015 UMD target
+npm install @apache-arrow/esnext-esm # ESNext CommonJS target
+npm install @apache-arrow/esnext-esm # ESNext ESModules target
+npm install @apache-arrow/esnext-umd # ESNext UMD target
+```
+
+### Why we package like this
+
+The JS community is a diverse group with a varied list of target environments and tool chains. Publishing multiple packages accommodates projects of all stripes.
 
-##### `ArrowReader.loadNextBatch()`
-Loads the next record batch and returns it's length.
+If you think we missed a compilation target and it's a blocker for adoption, please open an issue. We're here for you ❤️.
 
-##### `ArrowReader.getSchema()`
-Returns a JSON representation of the file's Arrow schema.
+# License
 
-##### `ArrowReader.getVectors()`
-Returns a list of `Vector` objects, one for each column.
-Vector objects have, at minimum, a `get(i)` method and a `length` attribute.
+[Apache 2.0](https://github.com/apache/arrow/blob/master/LICENSE)
 
-##### `ArrowReader.getVector(name: String)`
-Return a Vector object for column `name`
+[1]: mailto:dev-subscribe@arrow.apache.org
+[2]: https://github.com/apache/arrow/tree/master/format
+[3]: https://issues.apache.org/jira/browse/ARROW
+[4]: https://github.com/apache/arrow
\ No newline at end of file
diff --git a/js/bin/arrow2csv.js b/js/bin/arrow2csv.js
index c1801f7127bc0..f316b84b69e58 100755
--- a/js/bin/arrow2csv.js
+++ b/js/bin/arrow2csv.js
@@ -17,37 +17,73 @@
 // specific language governing permissions and limitations
 // under the License.
 
-var fs = require('fs')
-var process = require('process');
-var arrow = require('../lib/arrow.js');
-var program = require('commander');
+var fs = require('fs');
+var Table = require('../dist/Arrow.js').Table;
+var optionList = [
+    {
+        type: String,
+        name: 'schema',
+        alias: 's', multiple: true,
+        typeLabel: '[underline]{columns}',
+        description: 'A space-delimited list of column names'
+    },
+    {
+        type: String,
+        name: 'file', alias: 'f',
+        description: 'The Arrow file to read'
+    }
+];
 
-function list (val) {
-    return val.split(',');
-}
+var argv = require(`command-line-args`)(optionList, { partial: true });
+var files = [argv.file, ...(argv._unknown || [])].filter(Boolean);
 
-program
-  .version('0.1.0')
-  .usage('[options] <file>')
-  .option('-s --schema <list>', 'A comma-separated list of column names', list)
-  .parse(process.argv);
+// console.log(JSON.stringify(argv));
 
-if (!program.schema) {
-    program.outputHelp();
+if (!argv.schema || !files.length) {
+    console.log(require('command-line-usage')([
+        {
+            header: 'arrow2csv',
+            content: 'Print a CSV from an Arrow file'
+        },
+        {
+            header: 'Synopsis',
+            content: [
+                '$ arrow2csv [underline]{file.arrow} [[bold]{--schema} column_name ...]',
+                '$ arrow2csv [[bold]{--schema} column_name ...] [[bold]{--file} [underline]{file.arrow}]',
+                '$ arrow2csv [bold]{-s} column_1 [bold]{-s} column_2 [[bold]{-f} [underline]{file.arrow}]',
+                '$ arrow2csv [[bold]{--help}]'
+            ]
+        },
+        {
+            header: 'Options',
+            optionList: [
+                ...optionList,
+                {
+                    name: 'help',
+                    description: 'Print this usage guide.'
+                }
+            ]
+        },
+        {
+            header: 'Example',
+            content: [
+                '$ arrow2csv --schema foo baz -f simple.arrow',
+                '>  foo,  baz',
+                '>    1,   aa',
+                '> null, null',
+                '>    3, null',
+                '>    4,  bbb',
+                '>    5, cccc',
+            ]
+        }
+    ]));
     process.exit(1);
 }
 
-var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
-var reader = arrow.getReader(buf);
-var nrecords
-
-nrecords = reader.loadNextBatch();
-while (nrecords > 0) {
-  for (var i = 0; i < nrecords; i += 1|0) {
-    console.log(program.schema.map(function (field) {
-      return '' + reader.getVector(field).get(i);
-    }).join(','));
-  }
-  nrecords = reader.loadNextBatch();
-  if (nrecords > 0) console.log('---');
-}
+files.forEach((source) => {
+    var allColumns = Table.from(fs.readFileSync(source));
+    var selectedColumns = new Table(argv.schema.map((columnName) => {
+        return allColumns.getColumn(columnName);
+    }));
+    console.log(selectedColumns.toString());
+});
diff --git a/js/bower.json b/js/bower.json
deleted file mode 100644
index c2099f859c364..0000000000000
--- a/js/bower.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-  "name": "arrow",
-  "description": "",
-  "main": "_bundles/arrow.js",
-  "authors": [],
-  "license": "Apache-2.0",
-  "homepage": "http://arrow.apache.org",
-  "ignore": [
-    ".gitignore",
-    ".npmignore",
-    "src/",
-    "spec/",
-    "tsconfig.json",
-    "webpack.config.js",
-    "flatbuffers.sh"
-  ]
-}
diff --git a/js/closure-compiler-scripts/File_generated.js b/js/closure-compiler-scripts/File_generated.js
new file mode 100644
index 0000000000000..bb82cc4ccc6e4
--- /dev/null
+++ b/js/closure-compiler-scripts/File_generated.js
@@ -0,0 +1,264 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+goog.module("module$targets$es5$cls$format$File_generated");
+goog.module.declareLegacyNamespace();
+var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
+/**
+ * @const
+ * @namespace
+ */
+var org = Schema_.org;
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache = org.apache || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow = org.apache.arrow || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
+
+/**
+ * ----------------------------------------------------------------------
+ * Arrow File metadata
+ *
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Footer = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Footer}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Footer=} obj
+ * @returns {org.apache.arrow.flatbuf.Footer}
+ */
+org.apache.arrow.flatbuf.Footer.getRootAsFooter = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Footer).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.MetadataVersion}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.version = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.MetadataVersion} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.MetadataVersion.V1;
+};
+
+/**
+ * @param {org.apache.arrow.flatbuf.Schema=} obj
+ * @returns {org.apache.arrow.flatbuf.Schema|null}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.schema = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? (obj || new org.apache.arrow.flatbuf.Schema).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
+};
+
+/**
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.Block=} obj
+ * @returns {org.apache.arrow.flatbuf.Block}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.dictionaries = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? (obj || new org.apache.arrow.flatbuf.Block).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.dictionariesLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.Block=} obj
+ * @returns {org.apache.arrow.flatbuf.Block}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.recordBatches = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 10);
+  return offset ? (obj || new org.apache.arrow.flatbuf.Block).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.recordBatchesLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 10);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Footer.startFooter = function(builder) {
+  builder.startObject(4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.MetadataVersion} version
+ */
+org.apache.arrow.flatbuf.Footer.addVersion = function(builder, version) {
+  builder.addFieldInt16(0, version, org.apache.arrow.flatbuf.MetadataVersion.V1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} schemaOffset
+ */
+org.apache.arrow.flatbuf.Footer.addSchema = function(builder, schemaOffset) {
+  builder.addFieldOffset(1, schemaOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dictionariesOffset
+ */
+org.apache.arrow.flatbuf.Footer.addDictionaries = function(builder, dictionariesOffset) {
+  builder.addFieldOffset(2, dictionariesOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Footer.startDictionariesVector = function(builder, numElems) {
+  builder.startVector(24, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} recordBatchesOffset
+ */
+org.apache.arrow.flatbuf.Footer.addRecordBatches = function(builder, recordBatchesOffset) {
+  builder.addFieldOffset(3, recordBatchesOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Footer.startRecordBatchesVector = function(builder, numElems) {
+  builder.startVector(24, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Footer.endFooter = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} offset
+ */
+org.apache.arrow.flatbuf.Footer.finishFooterBuffer = function(builder, offset) {
+  builder.finish(offset);
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Block = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Block}
+ */
+org.apache.arrow.flatbuf.Block.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * Index to the start of the RecordBlock (note this is past the Message header)
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.Block.prototype.offset = function() {
+  return this.bb.readInt64(this.bb_pos);
+};
+
+/**
+ * Length of the metadata
+ *
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Block.prototype.metaDataLength = function() {
+  return this.bb.readInt32(this.bb_pos + 8);
+};
+
+/**
+ * Length of the data (this is aligned so there can be a gap between this and
+ * the metatdata).
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.Block.prototype.bodyLength = function() {
+  return this.bb.readInt64(this.bb_pos + 16);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} offset
+ * @param {number} metaDataLength
+ * @param {flatbuffers.Long} bodyLength
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Block.createBlock = function(builder, offset, metaDataLength, bodyLength) {
+  builder.prep(8, 24);
+  builder.writeInt64(bodyLength);
+  builder.pad(4);
+  builder.writeInt32(metaDataLength);
+  builder.writeInt64(offset);
+  return builder.offset();
+};
+
+// Exports for Node.js and RequireJS
+exports.org = org;
diff --git a/js/closure-compiler-scripts/Message_generated.js b/js/closure-compiler-scripts/Message_generated.js
new file mode 100644
index 0000000000000..0c1a1a99d74ca
--- /dev/null
+++ b/js/closure-compiler-scripts/Message_generated.js
@@ -0,0 +1,486 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+goog.module("module$targets$es5$cls$format$Message_generated");
+goog.module.declareLegacyNamespace();
+var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
+/**
+ * @const
+ * @namespace
+ */
+var org = Schema_.org;
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache = org.apache || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow = org.apache.arrow || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
+
+/**
+ * ----------------------------------------------------------------------
+ * The root Message type
+ * This union enables us to easily send different message types without
+ * redundant storage, and in the future we can easily add new message types.
+ *
+ * Arrow implementations do not need to implement all of the message types,
+ * which may include experimental metadata types. For maximum compatibility,
+ * it is best to send data using RecordBatch
+ *
+ * @enum
+ */
+org.apache.arrow.flatbuf.MessageHeader = {
+  NONE: 0, 0: 'NONE',
+  Schema: 1, 1: 'Schema',
+  DictionaryBatch: 2, 2: 'DictionaryBatch',
+  RecordBatch: 3, 3: 'RecordBatch',
+  Tensor: 4, 4: 'Tensor',
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * Data structures for describing a table row batch (a collection of
+ * equal-length Arrow arrays)
+ * Metadata about a field at some level of a nested type tree (but not
+ * its children).
+ *
+ * For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
+ * would have {length: 5, null_count: 2} for its List node, and {length: 6,
+ * null_count: 0} for its Int16 node, as separate FieldNode structs
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.FieldNode = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.FieldNode}
+ */
+org.apache.arrow.flatbuf.FieldNode.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * The number of value slots in the Arrow array at this level of a nested
+ * tree
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.FieldNode.prototype.length = function() {
+  return this.bb.readInt64(this.bb_pos);
+};
+
+/**
+ * The number of observed nulls. Fields with null_count == 0 may choose not
+ * to write their physical validity bitmap out as a materialized buffer,
+ * instead setting the length of the bitmap buffer to 0.
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.FieldNode.prototype.nullCount = function() {
+  return this.bb.readInt64(this.bb_pos + 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} length
+ * @param {flatbuffers.Long} null_count
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.FieldNode.createFieldNode = function(builder, length, null_count) {
+  builder.prep(8, 16);
+  builder.writeInt64(null_count);
+  builder.writeInt64(length);
+  return builder.offset();
+};
+
+/**
+ * A data header describing the shared memory layout of a "record" or "row"
+ * batch. Some systems call this a "row batch" internally and others a "record
+ * batch".
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.RecordBatch = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.RecordBatch}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.RecordBatch=} obj
+ * @returns {org.apache.arrow.flatbuf.RecordBatch}
+ */
+org.apache.arrow.flatbuf.RecordBatch.getRootAsRecordBatch = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * number of records / rows. The arrays in the batch should all have this
+ * length
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.length = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
+};
+
+/**
+ * Nodes correspond to the pre-ordered flattened logical schema
+ *
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.FieldNode=} obj
+ * @returns {org.apache.arrow.flatbuf.FieldNode}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.nodes = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? (obj || new org.apache.arrow.flatbuf.FieldNode).__init(this.bb.__vector(this.bb_pos + offset) + index * 16, this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.nodesLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * Buffers correspond to the pre-ordered flattened buffer tree
+ *
+ * The number of buffers appended to this list depends on the schema. For
+ * example, most primitive arrays will have 2 buffers, 1 for the validity
+ * bitmap and 1 for the values. For struct arrays, there will only be a
+ * single buffer for the validity (nulls) bitmap
+ *
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.Buffer=} obj
+ * @returns {org.apache.arrow.flatbuf.Buffer}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.buffers = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? (obj || new org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.buffersLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.RecordBatch.startRecordBatch = function(builder) {
+  builder.startObject(3);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} length
+ */
+org.apache.arrow.flatbuf.RecordBatch.addLength = function(builder, length) {
+  builder.addFieldInt64(0, length, builder.createLong(0, 0));
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} nodesOffset
+ */
+org.apache.arrow.flatbuf.RecordBatch.addNodes = function(builder, nodesOffset) {
+  builder.addFieldOffset(1, nodesOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.RecordBatch.startNodesVector = function(builder, numElems) {
+  builder.startVector(16, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} buffersOffset
+ */
+org.apache.arrow.flatbuf.RecordBatch.addBuffers = function(builder, buffersOffset) {
+  builder.addFieldOffset(2, buffersOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.RecordBatch.startBuffersVector = function(builder, numElems) {
+  builder.startVector(24, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.RecordBatch.endRecordBatch = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * For sending dictionary encoding information. Any Field can be
+ * dictionary-encoded, but in this case none of its children may be
+ * dictionary-encoded.
+ * There is one vector / column per dictionary
+ *
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.DictionaryBatch = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.DictionaryBatch}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.DictionaryBatch=} obj
+ * @returns {org.apache.arrow.flatbuf.DictionaryBatch}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.getRootAsDictionaryBatch = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.DictionaryBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.prototype.id = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
+};
+
+/**
+ * @param {org.apache.arrow.flatbuf.RecordBatch=} obj
+ * @returns {org.apache.arrow.flatbuf.RecordBatch|null}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.prototype.data = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.startDictionaryBatch = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} id
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.addId = function(builder, id) {
+  builder.addFieldInt64(0, id, builder.createLong(0, 0));
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(1, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.endDictionaryBatch = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Message = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Message}
+ */
+org.apache.arrow.flatbuf.Message.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Message=} obj
+ * @returns {org.apache.arrow.flatbuf.Message}
+ */
+org.apache.arrow.flatbuf.Message.getRootAsMessage = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Message).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.MetadataVersion}
+ */
+org.apache.arrow.flatbuf.Message.prototype.version = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.MetadataVersion} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.MetadataVersion.V1;
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.MessageHeader}
+ */
+org.apache.arrow.flatbuf.Message.prototype.headerType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? /** @type {org.apache.arrow.flatbuf.MessageHeader} */ (this.bb.readUint8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.MessageHeader.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+org.apache.arrow.flatbuf.Message.prototype.header = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.Message.prototype.bodyLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 10);
+  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Message.startMessage = function(builder) {
+  builder.startObject(4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.MetadataVersion} version
+ */
+org.apache.arrow.flatbuf.Message.addVersion = function(builder, version) {
+  builder.addFieldInt16(0, version, org.apache.arrow.flatbuf.MetadataVersion.V1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.MessageHeader} headerType
+ */
+org.apache.arrow.flatbuf.Message.addHeaderType = function(builder, headerType) {
+  builder.addFieldInt8(1, headerType, org.apache.arrow.flatbuf.MessageHeader.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} headerOffset
+ */
+org.apache.arrow.flatbuf.Message.addHeader = function(builder, headerOffset) {
+  builder.addFieldOffset(2, headerOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} bodyLength
+ */
+org.apache.arrow.flatbuf.Message.addBodyLength = function(builder, bodyLength) {
+  builder.addFieldInt64(3, bodyLength, builder.createLong(0, 0));
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Message.endMessage = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} offset
+ */
+org.apache.arrow.flatbuf.Message.finishMessageBuffer = function(builder, offset) {
+  builder.finish(offset);
+};
+
+// Exports for Node.js and RequireJS
+exports.org = org;
diff --git a/js/closure-compiler-scripts/Schema_generated.js b/js/closure-compiler-scripts/Schema_generated.js
new file mode 100644
index 0000000000000..5b76443886543
--- /dev/null
+++ b/js/closure-compiler-scripts/Schema_generated.js
@@ -0,0 +1,2231 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+goog.module("module$targets$es5$cls$format$Schema_generated");
+goog.module.declareLegacyNamespace();
+
+/**
+ * @const
+ * @namespace
+ */
+var org = org || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache = org.apache || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow = org.apache.arrow || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
+
+/**
+ * @enum
+ */
+org.apache.arrow.flatbuf.MetadataVersion = {
+  V1: 0, 0: 'V1',
+  V2: 1, 1: 'V2',
+  V3: 2, 2: 'V3',
+};
+
+/**
+ * @enum
+ */
+org.apache.arrow.flatbuf.UnionMode = {
+  Sparse: 0, 0: 'Sparse',
+  Dense: 1, 1: 'Dense',
+};
+
+/**
+ * @enum
+ */
+org.apache.arrow.flatbuf.Precision = {
+  HALF: 0, 0: 'HALF',
+  SINGLE: 1, 1: 'SINGLE',
+  DOUBLE: 2, 2: 'DOUBLE',
+};
+
+/**
+ * @enum
+ */
+org.apache.arrow.flatbuf.DateUnit = {
+  DAY: 0, 0: 'DAY',
+  MILLISECOND: 1, 1: 'MILLISECOND',
+};
+
+/**
+ * @enum
+ */
+org.apache.arrow.flatbuf.TimeUnit = {
+  SECOND: 0, 0: 'SECOND',
+  MILLISECOND: 1, 1: 'MILLISECOND',
+  MICROSECOND: 2, 2: 'MICROSECOND',
+  NANOSECOND: 3, 3: 'NANOSECOND',
+};
+
+/**
+ * @enum
+ */
+org.apache.arrow.flatbuf.IntervalUnit = {
+  YEAR_MONTH: 0, 0: 'YEAR_MONTH',
+  DAY_TIME: 1, 1: 'DAY_TIME',
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * Top-level Type value, enabling extensible type-specific metadata. We can
+ * add new logical types to Type without breaking backwards compatibility
+ *
+ * @enum
+ */
+org.apache.arrow.flatbuf.Type = {
+  NONE: 0, 0: 'NONE',
+  Null: 1, 1: 'Null',
+  Int: 2, 2: 'Int',
+  FloatingPoint: 3, 3: 'FloatingPoint',
+  Binary: 4, 4: 'Binary',
+  Utf8: 5, 5: 'Utf8',
+  Bool: 6, 6: 'Bool',
+  Decimal: 7, 7: 'Decimal',
+  Date: 8, 8: 'Date',
+  Time: 9, 9: 'Time',
+  Timestamp: 10, 10: 'Timestamp',
+  Interval: 11, 11: 'Interval',
+  List: 12, 12: 'List',
+  Struct_: 13, 13: 'Struct_',
+  Union: 14, 14: 'Union',
+  FixedSizeBinary: 15, 15: 'FixedSizeBinary',
+  FixedSizeList: 16, 16: 'FixedSizeList',
+  Map: 17, 17: 'Map',
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * The possible types of a vector
+ *
+ * @enum
+ */
+org.apache.arrow.flatbuf.VectorType = {
+  /**
+   * used in List type, Dense Union and variable length primitive types (String, Binary)
+   */
+  OFFSET: 0, 0: 'OFFSET',
+
+  /**
+   * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
+   */
+  DATA: 1, 1: 'DATA',
+
+  /**
+   * Bit vector indicating if each value is null
+   */
+  VALIDITY: 2, 2: 'VALIDITY',
+
+  /**
+   * Type vector used in Union type
+   */
+  TYPE: 3, 3: 'TYPE',
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * Endianness of the platform producing the data
+ *
+ * @enum
+ */
+org.apache.arrow.flatbuf.Endianness = {
+  Little: 0, 0: 'Little',
+  Big: 1, 1: 'Big',
+};
+
+/**
+ * These are stored in the flatbuffer in the Type union below
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Null = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Null}
+ */
+org.apache.arrow.flatbuf.Null.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Null=} obj
+ * @returns {org.apache.arrow.flatbuf.Null}
+ */
+org.apache.arrow.flatbuf.Null.getRootAsNull = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Null).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Null.startNull = function(builder) {
+  builder.startObject(0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Null.endNull = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * A Struct_ in the flatbuffer metadata is the same as an Arrow Struct
+ * (according to the physical memory layout). We used Struct_ here as
+ * Struct is a reserved word in Flatbuffers
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Struct_ = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Struct_}
+ */
+org.apache.arrow.flatbuf.Struct_.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Struct_=} obj
+ * @returns {org.apache.arrow.flatbuf.Struct_}
+ */
+org.apache.arrow.flatbuf.Struct_.getRootAsStruct_ = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Struct_).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Struct_.startStruct_ = function(builder) {
+  builder.startObject(0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Struct_.endStruct_ = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.List = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.List}
+ */
+org.apache.arrow.flatbuf.List.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.List=} obj
+ * @returns {org.apache.arrow.flatbuf.List}
+ */
+org.apache.arrow.flatbuf.List.getRootAsList = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.List).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.List.startList = function(builder) {
+  builder.startObject(0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.List.endList = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.FixedSizeList = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.FixedSizeList}
+ */
+org.apache.arrow.flatbuf.FixedSizeList.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.FixedSizeList=} obj
+ * @returns {org.apache.arrow.flatbuf.FixedSizeList}
+ */
+org.apache.arrow.flatbuf.FixedSizeList.getRootAsFixedSizeList = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.FixedSizeList).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * Number of list items per value
+ *
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.FixedSizeList.prototype.listSize = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.FixedSizeList.startFixedSizeList = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} listSize
+ */
+org.apache.arrow.flatbuf.FixedSizeList.addListSize = function(builder, listSize) {
+  builder.addFieldInt32(0, listSize, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.FixedSizeList.endFixedSizeList = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * A Map is a logical nested type that is represented as
+ *
+ * List<entry: Struct<key: K, value: V>>
+ *
+ * In this layout, the keys and values are each respectively contiguous. We do
+ * not constrain the key and value types, so the application is responsible
+ * for ensuring that the keys are hashable and unique. Whether the keys are sorted
+ * may be set in the metadata for this field
+ *
+ * In a Field with Map type, the Field has a child Struct field, which then
+ * has two children: key type and the second the value type. The names of the
+ * child fields may be respectively "entry", "key", and "value", but this is
+ * not enforced
+ *
+ * Map
+ *   - child[0] entry: Struct
+ *     - child[0] key: K
+ *     - child[1] value: V
+ *
+ * Neither the "entry" field nor the "key" field may be nullable.
+ *
+ * The metadata is structured so that Arrow systems without special handling
+ * for Map can make Map an alias for List. The "layout" attribute for the Map
+ * field must have the same contents as a List.
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Map = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Map}
+ */
+org.apache.arrow.flatbuf.Map.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Map=} obj
+ * @returns {org.apache.arrow.flatbuf.Map}
+ */
+org.apache.arrow.flatbuf.Map.getRootAsMap = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Map).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * Set to true if the keys within each value are sorted
+ *
+ * @returns {boolean}
+ */
+org.apache.arrow.flatbuf.Map.prototype.keysSorted = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Map.startMap = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {boolean} keysSorted
+ */
+org.apache.arrow.flatbuf.Map.addKeysSorted = function(builder, keysSorted) {
+  builder.addFieldInt8(0, +keysSorted, +false);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Map.endMap = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * A union is a complex type with children in Field
+ * By default ids in the type vector refer to the offsets in the children
+ * optionally typeIds provides an indirection between the child offset and the type id
+ * for each child typeIds[offset] is the id used in the type vector
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Union = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Union}
+ */
+org.apache.arrow.flatbuf.Union.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Union=} obj
+ * @returns {org.apache.arrow.flatbuf.Union}
+ */
+org.apache.arrow.flatbuf.Union.getRootAsUnion = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Union).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.UnionMode}
+ */
+org.apache.arrow.flatbuf.Union.prototype.mode = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.UnionMode} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.UnionMode.Sparse;
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Union.prototype.typeIds = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.readInt32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Union.prototype.typeIdsLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Int32Array}
+ */
+org.apache.arrow.flatbuf.Union.prototype.typeIdsArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? new Int32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Union.startUnion = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.UnionMode} mode
+ */
+org.apache.arrow.flatbuf.Union.addMode = function(builder, mode) {
+  builder.addFieldInt16(0, mode, org.apache.arrow.flatbuf.UnionMode.Sparse);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} typeIdsOffset
+ */
+org.apache.arrow.flatbuf.Union.addTypeIds = function(builder, typeIdsOffset) {
+  builder.addFieldOffset(1, typeIdsOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Union.createTypeIdsVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Union.startTypeIdsVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Union.endUnion = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Int = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Int}
+ */
+org.apache.arrow.flatbuf.Int.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Int=} obj
+ * @returns {org.apache.arrow.flatbuf.Int}
+ */
+org.apache.arrow.flatbuf.Int.getRootAsInt = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Int).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Int.prototype.bitWidth = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {boolean}
+ */
+org.apache.arrow.flatbuf.Int.prototype.isSigned = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Int.startInt = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} bitWidth
+ */
+org.apache.arrow.flatbuf.Int.addBitWidth = function(builder, bitWidth) {
+  builder.addFieldInt32(0, bitWidth, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {boolean} isSigned
+ */
+org.apache.arrow.flatbuf.Int.addIsSigned = function(builder, isSigned) {
+  builder.addFieldInt8(1, +isSigned, +false);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Int.endInt = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.FloatingPoint = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.FloatingPoint}
+ */
+org.apache.arrow.flatbuf.FloatingPoint.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.FloatingPoint=} obj
+ * @returns {org.apache.arrow.flatbuf.FloatingPoint}
+ */
+org.apache.arrow.flatbuf.FloatingPoint.getRootAsFloatingPoint = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.FloatingPoint).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.Precision}
+ */
+org.apache.arrow.flatbuf.FloatingPoint.prototype.precision = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.Precision} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Precision.HALF;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.FloatingPoint.startFloatingPoint = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.Precision} precision
+ */
+org.apache.arrow.flatbuf.FloatingPoint.addPrecision = function(builder, precision) {
+  builder.addFieldInt16(0, precision, org.apache.arrow.flatbuf.Precision.HALF);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.FloatingPoint.endFloatingPoint = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * Unicode with UTF-8 encoding
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Utf8 = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Utf8}
+ */
+org.apache.arrow.flatbuf.Utf8.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Utf8=} obj
+ * @returns {org.apache.arrow.flatbuf.Utf8}
+ */
+org.apache.arrow.flatbuf.Utf8.getRootAsUtf8 = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Utf8).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Utf8.startUtf8 = function(builder) {
+  builder.startObject(0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Utf8.endUtf8 = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Binary = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Binary}
+ */
+org.apache.arrow.flatbuf.Binary.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Binary=} obj
+ * @returns {org.apache.arrow.flatbuf.Binary}
+ */
+org.apache.arrow.flatbuf.Binary.getRootAsBinary = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Binary).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Binary.startBinary = function(builder) {
+  builder.startObject(0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Binary.endBinary = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.FixedSizeBinary = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.FixedSizeBinary}
+ */
+org.apache.arrow.flatbuf.FixedSizeBinary.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.FixedSizeBinary=} obj
+ * @returns {org.apache.arrow.flatbuf.FixedSizeBinary}
+ */
+org.apache.arrow.flatbuf.FixedSizeBinary.getRootAsFixedSizeBinary = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.FixedSizeBinary).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * Number of bytes per value
+ *
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.FixedSizeBinary.prototype.byteWidth = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.FixedSizeBinary.startFixedSizeBinary = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} byteWidth
+ */
+org.apache.arrow.flatbuf.FixedSizeBinary.addByteWidth = function(builder, byteWidth) {
+  builder.addFieldInt32(0, byteWidth, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.FixedSizeBinary.endFixedSizeBinary = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Bool = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Bool}
+ */
+org.apache.arrow.flatbuf.Bool.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Bool=} obj
+ * @returns {org.apache.arrow.flatbuf.Bool}
+ */
+org.apache.arrow.flatbuf.Bool.getRootAsBool = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Bool).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Bool.startBool = function(builder) {
+  builder.startObject(0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Bool.endBool = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Decimal = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Decimal}
+ */
+org.apache.arrow.flatbuf.Decimal.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Decimal=} obj
+ * @returns {org.apache.arrow.flatbuf.Decimal}
+ */
+org.apache.arrow.flatbuf.Decimal.getRootAsDecimal = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Decimal).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * Total number of decimal digits
+ *
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Decimal.prototype.precision = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * Number of digits after the decimal point "."
+ *
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Decimal.prototype.scale = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Decimal.startDecimal = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} precision
+ */
+org.apache.arrow.flatbuf.Decimal.addPrecision = function(builder, precision) {
+  builder.addFieldInt32(0, precision, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} scale
+ */
+org.apache.arrow.flatbuf.Decimal.addScale = function(builder, scale) {
+  builder.addFieldInt32(1, scale, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Decimal.endDecimal = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
+ * epoch (1970-01-01), stored in either of two units:
+ *
+ * * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
+ *   leap seconds), where the values are evenly divisible by 86400000
+ * * Days (32 bits) since the UNIX epoch
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Date = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Date}
+ */
+org.apache.arrow.flatbuf.Date.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Date=} obj
+ * @returns {org.apache.arrow.flatbuf.Date}
+ */
+org.apache.arrow.flatbuf.Date.getRootAsDate = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Date).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.DateUnit}
+ */
+org.apache.arrow.flatbuf.Date.prototype.unit = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.DateUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.DateUnit.MILLISECOND;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Date.startDate = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.DateUnit} unit
+ */
+org.apache.arrow.flatbuf.Date.addUnit = function(builder, unit) {
+  builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.DateUnit.MILLISECOND);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Date.endDate = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * Time type. The physical storage type depends on the unit
+ * - SECOND and MILLISECOND: 32 bits
+ * - MICROSECOND and NANOSECOND: 64 bits
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Time = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Time}
+ */
+org.apache.arrow.flatbuf.Time.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Time=} obj
+ * @returns {org.apache.arrow.flatbuf.Time}
+ */
+org.apache.arrow.flatbuf.Time.getRootAsTime = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Time).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.TimeUnit}
+ */
+org.apache.arrow.flatbuf.Time.prototype.unit = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.TimeUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.MILLISECOND;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Time.prototype.bitWidth = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.readInt32(this.bb_pos + offset) : 32;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Time.startTime = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.TimeUnit} unit
+ */
+org.apache.arrow.flatbuf.Time.addUnit = function(builder, unit) {
+  builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.MILLISECOND);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} bitWidth
+ */
+org.apache.arrow.flatbuf.Time.addBitWidth = function(builder, bitWidth) {
+  builder.addFieldInt32(1, bitWidth, 32);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Time.endTime = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding
+ * leap seconds, as a 64-bit integer. Note that UNIX time does not include
+ * leap seconds.
+ *
+ * The Timestamp metadata supports both "time zone naive" and "time zone
+ * aware" timestamps. Read about the timezone attribute for more detail
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Timestamp = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Timestamp}
+ */
+org.apache.arrow.flatbuf.Timestamp.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Timestamp=} obj
+ * @returns {org.apache.arrow.flatbuf.Timestamp}
+ */
+org.apache.arrow.flatbuf.Timestamp.getRootAsTimestamp = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Timestamp).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.TimeUnit}
+ */
+org.apache.arrow.flatbuf.Timestamp.prototype.unit = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.TimeUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.SECOND;
+};
+
+/**
+ * The time zone is a string indicating the name of a time zone, one of:
+ *
+ * * As used in the Olson time zone database (the "tz database" or
+ *   "tzdata"), such as "America/New_York"
+ * * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
+ *
+ * Whether a timezone string is present indicates different semantics about
+ * the data:
+ *
+ * * If the time zone is null or equal to an empty string, the data is "time
+ *   zone naive" and shall be displayed *as is* to the user, not localized
+ *   to the locale of the user. This data can be though of as UTC but
+ *   without having "UTC" as the time zone, it is not considered to be
+ *   localized to any time zone
+ *
+ * * If the time zone is set to a valid value, values can be displayed as
+ *   "localized" to that time zone, even though the underlying 64-bit
+ *   integers are identical to the same data stored in UTC. Converting
+ *   between time zones is a metadata-only operation and does not change the
+ *   underlying values
+ *
+ * @param {flatbuffers.Encoding=} optionalEncoding
+ * @returns {string|Uint8Array|null}
+ */
+org.apache.arrow.flatbuf.Timestamp.prototype.timezone = function(optionalEncoding) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Timestamp.startTimestamp = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.TimeUnit} unit
+ */
+org.apache.arrow.flatbuf.Timestamp.addUnit = function(builder, unit) {
+  builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.SECOND);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} timezoneOffset
+ */
+org.apache.arrow.flatbuf.Timestamp.addTimezone = function(builder, timezoneOffset) {
+  builder.addFieldOffset(1, timezoneOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Timestamp.endTimestamp = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Interval = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Interval}
+ */
+org.apache.arrow.flatbuf.Interval.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Interval=} obj
+ * @returns {org.apache.arrow.flatbuf.Interval}
+ */
+org.apache.arrow.flatbuf.Interval.getRootAsInterval = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Interval).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.IntervalUnit}
+ */
+org.apache.arrow.flatbuf.Interval.prototype.unit = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.IntervalUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Interval.startInterval = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.IntervalUnit} unit
+ */
+org.apache.arrow.flatbuf.Interval.addUnit = function(builder, unit) {
+  builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Interval.endInterval = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * represents the physical layout of a buffer
+ * buffers have fixed width slots of a given type
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.VectorLayout = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.VectorLayout}
+ */
+org.apache.arrow.flatbuf.VectorLayout.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.VectorLayout=} obj
+ * @returns {org.apache.arrow.flatbuf.VectorLayout}
+ */
+org.apache.arrow.flatbuf.VectorLayout.getRootAsVectorLayout = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.VectorLayout).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * the width of a slot in the buffer (typically 1, 8, 16, 32 or 64)
+ *
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.VectorLayout.prototype.bitWidth = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt16(this.bb_pos + offset) : 0;
+};
+
+/**
+ * the purpose of the vector
+ *
+ * @returns {org.apache.arrow.flatbuf.VectorType}
+ */
+org.apache.arrow.flatbuf.VectorLayout.prototype.type = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? /** @type {org.apache.arrow.flatbuf.VectorType} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.VectorType.OFFSET;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.VectorLayout.startVectorLayout = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} bitWidth
+ */
+org.apache.arrow.flatbuf.VectorLayout.addBitWidth = function(builder, bitWidth) {
+  builder.addFieldInt16(0, bitWidth, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.VectorType} type
+ */
+org.apache.arrow.flatbuf.VectorLayout.addType = function(builder, type) {
+  builder.addFieldInt16(1, type, org.apache.arrow.flatbuf.VectorType.OFFSET);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.VectorLayout.endVectorLayout = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * user defined key value pairs to add custom metadata to arrow
+ * key namespacing is the responsibility of the user
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.KeyValue = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.KeyValue}
+ */
+org.apache.arrow.flatbuf.KeyValue.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.KeyValue=} obj
+ * @returns {org.apache.arrow.flatbuf.KeyValue}
+ */
+org.apache.arrow.flatbuf.KeyValue.getRootAsKeyValue = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.KeyValue).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {flatbuffers.Encoding=} optionalEncoding
+ * @returns {string|Uint8Array|null}
+ */
+org.apache.arrow.flatbuf.KeyValue.prototype.key = function(optionalEncoding) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
+};
+
+/**
+ * @param {flatbuffers.Encoding=} optionalEncoding
+ * @returns {string|Uint8Array|null}
+ */
+org.apache.arrow.flatbuf.KeyValue.prototype.value = function(optionalEncoding) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.KeyValue.startKeyValue = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} keyOffset
+ */
+org.apache.arrow.flatbuf.KeyValue.addKey = function(builder, keyOffset) {
+  builder.addFieldOffset(0, keyOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} valueOffset
+ */
+org.apache.arrow.flatbuf.KeyValue.addValue = function(builder, valueOffset) {
+  builder.addFieldOffset(1, valueOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.KeyValue.endKeyValue = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * Dictionary encoding metadata
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.DictionaryEncoding}
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.DictionaryEncoding=} obj
+ * @returns {org.apache.arrow.flatbuf.DictionaryEncoding}
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.getRootAsDictionaryEncoding = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.DictionaryEncoding).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * The known dictionary id in the application where this data is used. In
+ * the file or streaming formats, the dictionary ids are found in the
+ * DictionaryBatch messages
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.prototype.id = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
+};
+
+/**
+ * The dictionary indices are constrained to be positive integers. If this
+ * field is null, the indices must be signed int32
+ *
+ * @param {org.apache.arrow.flatbuf.Int=} obj
+ * @returns {org.apache.arrow.flatbuf.Int|null}
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.prototype.indexType = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? (obj || new org.apache.arrow.flatbuf.Int).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
+};
+
+/**
+ * By default, dictionaries are not ordered, or the order does not have
+ * semantic meaning. In some statistical, applications, dictionary-encoding
+ * is used to represent ordered categorical data, and we provide a way to
+ * preserve that metadata here
+ *
+ * @returns {boolean}
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.prototype.isOrdered = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.startDictionaryEncoding = function(builder) {
+  builder.startObject(3);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} id
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.addId = function(builder, id) {
+  builder.addFieldInt64(0, id, builder.createLong(0, 0));
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} indexTypeOffset
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.addIndexType = function(builder, indexTypeOffset) {
+  builder.addFieldOffset(1, indexTypeOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {boolean} isOrdered
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.addIsOrdered = function(builder, isOrdered) {
+  builder.addFieldInt8(2, +isOrdered, +false);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.DictionaryEncoding.endDictionaryEncoding = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * A field represents a named column in a record / row batch or child of a
+ * nested type.
+ *
+ * - children is only for nested Arrow arrays
+ * - For primitive types, children will have length 0
+ * - nullable should default to true in general
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Field = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Field}
+ */
+org.apache.arrow.flatbuf.Field.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Field=} obj
+ * @returns {org.apache.arrow.flatbuf.Field}
+ */
+org.apache.arrow.flatbuf.Field.getRootAsField = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Field).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {flatbuffers.Encoding=} optionalEncoding
+ * @returns {string|Uint8Array|null}
+ */
+org.apache.arrow.flatbuf.Field.prototype.name = function(optionalEncoding) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
+};
+
+/**
+ * @returns {boolean}
+ */
+org.apache.arrow.flatbuf.Field.prototype.nullable = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.Type}
+ */
+org.apache.arrow.flatbuf.Field.prototype.typeType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? /** @type {org.apache.arrow.flatbuf.Type} */ (this.bb.readUint8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Type.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+org.apache.arrow.flatbuf.Field.prototype.type = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 10);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @param {org.apache.arrow.flatbuf.DictionaryEncoding=} obj
+ * @returns {org.apache.arrow.flatbuf.DictionaryEncoding|null}
+ */
+org.apache.arrow.flatbuf.Field.prototype.dictionary = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 12);
+  return offset ? (obj || new org.apache.arrow.flatbuf.DictionaryEncoding).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
+};
+
+/**
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.Field=} obj
+ * @returns {org.apache.arrow.flatbuf.Field}
+ */
+org.apache.arrow.flatbuf.Field.prototype.children = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 14);
+  return offset ? (obj || new org.apache.arrow.flatbuf.Field).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Field.prototype.childrenLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 14);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * layout of buffers produced for this type (as derived from the Type)
+ * does not include children
+ * each recordbatch will return instances of those Buffers.
+ *
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.VectorLayout=} obj
+ * @returns {org.apache.arrow.flatbuf.VectorLayout}
+ */
+org.apache.arrow.flatbuf.Field.prototype.layout = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 16);
+  return offset ? (obj || new org.apache.arrow.flatbuf.VectorLayout).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Field.prototype.layoutLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 16);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.KeyValue=} obj
+ * @returns {org.apache.arrow.flatbuf.KeyValue}
+ */
+org.apache.arrow.flatbuf.Field.prototype.customMetadata = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 18);
+  return offset ? (obj || new org.apache.arrow.flatbuf.KeyValue).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Field.prototype.customMetadataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 18);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Field.startField = function(builder) {
+  builder.startObject(8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} nameOffset
+ */
+org.apache.arrow.flatbuf.Field.addName = function(builder, nameOffset) {
+  builder.addFieldOffset(0, nameOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {boolean} nullable
+ */
+org.apache.arrow.flatbuf.Field.addNullable = function(builder, nullable) {
+  builder.addFieldInt8(1, +nullable, +false);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.Type} typeType
+ */
+org.apache.arrow.flatbuf.Field.addTypeType = function(builder, typeType) {
+  builder.addFieldInt8(2, typeType, org.apache.arrow.flatbuf.Type.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} typeOffset
+ */
+org.apache.arrow.flatbuf.Field.addType = function(builder, typeOffset) {
+  builder.addFieldOffset(3, typeOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dictionaryOffset
+ */
+org.apache.arrow.flatbuf.Field.addDictionary = function(builder, dictionaryOffset) {
+  builder.addFieldOffset(4, dictionaryOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} childrenOffset
+ */
+org.apache.arrow.flatbuf.Field.addChildren = function(builder, childrenOffset) {
+  builder.addFieldOffset(5, childrenOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Offset>} data
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Field.createChildrenVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addOffset(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Field.startChildrenVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} layoutOffset
+ */
+org.apache.arrow.flatbuf.Field.addLayout = function(builder, layoutOffset) {
+  builder.addFieldOffset(6, layoutOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Offset>} data
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Field.createLayoutVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addOffset(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Field.startLayoutVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} customMetadataOffset
+ */
+org.apache.arrow.flatbuf.Field.addCustomMetadata = function(builder, customMetadataOffset) {
+  builder.addFieldOffset(7, customMetadataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Offset>} data
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Field.createCustomMetadataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addOffset(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Field.startCustomMetadataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Field.endField = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * A Buffer represents a single contiguous memory segment
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Buffer = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Buffer}
+ */
+org.apache.arrow.flatbuf.Buffer.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * The shared memory page id where this buffer is located. Currently this is
+ * not used
+ *
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Buffer.prototype.page = function() {
+  return this.bb.readInt32(this.bb_pos);
+};
+
+/**
+ * The relative offset into the shared memory page where the bytes for this
+ * buffer starts
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.Buffer.prototype.offset = function() {
+  return this.bb.readInt64(this.bb_pos + 8);
+};
+
+/**
+ * The absolute length (in bytes) of the memory buffer. The memory is found
+ * from offset (inclusive) to offset + length (non-inclusive).
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.Buffer.prototype.length = function() {
+  return this.bb.readInt64(this.bb_pos + 16);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} page
+ * @param {flatbuffers.Long} offset
+ * @param {flatbuffers.Long} length
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Buffer.createBuffer = function(builder, page, offset, length) {
+  builder.prep(8, 24);
+  builder.writeInt64(length);
+  builder.writeInt64(offset);
+  builder.pad(4);
+  builder.writeInt32(page);
+  return builder.offset();
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * A Schema describes the columns in a row batch
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Schema = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Schema}
+ */
+org.apache.arrow.flatbuf.Schema.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Schema=} obj
+ * @returns {org.apache.arrow.flatbuf.Schema}
+ */
+org.apache.arrow.flatbuf.Schema.getRootAsSchema = function(bb, obj) {
+  return (obj || new org.apache.arrow.flatbuf.Schema).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * endianness of the buffer
+ * it is Little Endian by default
+ * if endianness doesn't match the underlying system then the vectors need to be converted
+ *
+ * @returns {org.apache.arrow.flatbuf.Endianness}
+ */
+org.apache.arrow.flatbuf.Schema.prototype.endianness = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {org.apache.arrow.flatbuf.Endianness} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Endianness.Little;
+};
+
+/**
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.Field=} obj
+ * @returns {org.apache.arrow.flatbuf.Field}
+ */
+org.apache.arrow.flatbuf.Schema.prototype.fields = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? (obj || new org.apache.arrow.flatbuf.Field).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Schema.prototype.fieldsLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.KeyValue=} obj
+ * @returns {org.apache.arrow.flatbuf.KeyValue}
+ */
+org.apache.arrow.flatbuf.Schema.prototype.customMetadata = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? (obj || new org.apache.arrow.flatbuf.KeyValue).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Schema.prototype.customMetadataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Schema.startSchema = function(builder) {
+  builder.startObject(3);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.Endianness} endianness
+ */
+org.apache.arrow.flatbuf.Schema.addEndianness = function(builder, endianness) {
+  builder.addFieldInt16(0, endianness, org.apache.arrow.flatbuf.Endianness.Little);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} fieldsOffset
+ */
+org.apache.arrow.flatbuf.Schema.addFields = function(builder, fieldsOffset) {
+  builder.addFieldOffset(1, fieldsOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Offset>} data
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Schema.createFieldsVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addOffset(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Schema.startFieldsVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} customMetadataOffset
+ */
+org.apache.arrow.flatbuf.Schema.addCustomMetadata = function(builder, customMetadataOffset) {
+  builder.addFieldOffset(2, customMetadataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Offset>} data
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Schema.createCustomMetadataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addOffset(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Schema.startCustomMetadataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Schema.endSchema = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} offset
+ */
+org.apache.arrow.flatbuf.Schema.finishSchemaBuffer = function(builder, offset) {
+  builder.finish(offset);
+};
+
+// Exports for Node.js and RequireJS
+exports.org = org;
diff --git a/js/closure-compiler-scripts/flatbuffers.js b/js/closure-compiler-scripts/flatbuffers.js
new file mode 100644
index 0000000000000..e51a4a039536f
--- /dev/null
+++ b/js/closure-compiler-scripts/flatbuffers.js
@@ -0,0 +1,1204 @@
+/**
+ * closure-compiler-friendly flatbuffers
+ * copied from node_modules/flatbuffers/js/flatbuffers.js
+ * update as needed
+ */
+
+ /// @file
+/// @addtogroup flatbuffers_javascript_api
+/// @{
+/// @cond FLATBUFFERS_INTERNAL
+
+goog.module("module$flatbuffers");
+goog.module.declareLegacyNamespace();
+/**
+ * @fileoverview
+ *
+ * Need to suppress 'global this' error so the Node.js export line doesn't cause
+ * closure compile to error out.
+ * @suppress {globalThis}
+ */
+
+/**
+ * @const
+ * @namespace
+ */
+var flatbuffers = {};
+
+/**
+ * @typedef {number}
+ */
+flatbuffers.Offset;
+
+/**
+ * @typedef {{
+ *   bb: flatbuffers.ByteBuffer,
+ *   bb_pos: number
+ * }}
+ */
+flatbuffers.Table;
+
+/**
+ * @type {number}
+ * @const
+ */
+flatbuffers.SIZEOF_SHORT = 2;
+
+/**
+ * @type {number}
+ * @const
+ */
+flatbuffers.SIZEOF_INT = 4;
+
+/**
+ * @type {number}
+ * @const
+ */
+flatbuffers.FILE_IDENTIFIER_LENGTH = 4;
+
+/**
+ * @enum {number}
+ */
+flatbuffers.Encoding = {
+  UTF8_BYTES: 1,
+  UTF16_STRING: 2
+};
+
+/**
+ * @type {Int32Array}
+ * @const
+ */
+flatbuffers.int32 = new Int32Array(2);
+
+/**
+ * @type {Float32Array}
+ * @const
+ */
+flatbuffers.float32 = new Float32Array(flatbuffers.int32.buffer);
+
+/**
+ * @type {Float64Array}
+ * @const
+ */
+flatbuffers.float64 = new Float64Array(flatbuffers.int32.buffer);
+
+/**
+ * @type {boolean}
+ * @const
+ */
+flatbuffers.isLittleEndian = new Uint16Array(new Uint8Array([1, 0]).buffer)[0] === 1;
+
+////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * @constructor
+ * @param {number} low
+ * @param {number} high
+ */
+flatbuffers.Long = function(low, high) {
+  /**
+   * @type {number}
+   * @const
+   */
+  this.low = low | 0;
+
+  /**
+   * @type {number}
+   * @const
+   */
+  this.high = high | 0;
+};
+
+/**
+ * @param {number} low
+ * @param {number} high
+ * @returns {flatbuffers.Long}
+ */
+flatbuffers.Long.create = function(low, high) {
+  // Special-case zero to avoid GC overhead for default values
+  return low == 0 && high == 0 ? flatbuffers.Long.ZERO : new flatbuffers.Long(low, high);
+};
+
+/**
+ * @returns {number}
+ */
+flatbuffers.Long.prototype.toFloat64 = function() {
+  return (this.low >>> 0) + this.high * 0x100000000;
+};
+
+/**
+ * @param {flatbuffers.Long} other
+ * @returns {boolean}
+ */
+flatbuffers.Long.prototype.equals = function(other) {
+  return this.low == other.low && this.high == other.high;
+};
+
+/**
+ * @type {flatbuffers.Long}
+ * @const
+ */
+flatbuffers.Long.ZERO = new flatbuffers.Long(0, 0);
+
+/// @endcond
+////////////////////////////////////////////////////////////////////////////////
+/**
+ * Create a FlatBufferBuilder.
+ *
+ * @constructor
+ * @param {number=} opt_initial_size
+ */
+flatbuffers.Builder = function(opt_initial_size) {
+  if (!opt_initial_size) {
+    var initial_size = 1024;
+  } else {
+    var initial_size = opt_initial_size;
+  }
+
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   * @private
+   */
+  this.bb = flatbuffers.ByteBuffer.allocate(initial_size);
+
+  /**
+   * Remaining space in the ByteBuffer.
+   *
+   * @type {number}
+   * @private
+   */
+  this.space = initial_size;
+
+  /**
+   * Minimum alignment encountered so far.
+   *
+   * @type {number}
+   * @private
+   */
+  this.minalign = 1;
+
+  /**
+   * The vtable for the current table.
+   *
+   * @type {Array.<number>}
+   * @private
+   */
+  this.vtable = null;
+
+  /**
+   * The amount of fields we're actually using.
+   *
+   * @type {number}
+   * @private
+   */
+  this.vtable_in_use = 0;
+
+  /**
+   * Whether we are currently serializing a table.
+   *
+   * @type {boolean}
+   * @private
+   */
+  this.isNested = false;
+
+  /**
+   * Starting offset of the current struct/table.
+   *
+   * @type {number}
+   * @private
+   */
+  this.object_start = 0;
+
+  /**
+   * List of offsets of all vtables.
+   *
+   * @type {Array.<number>}
+   * @private
+   */
+  this.vtables = [];
+
+  /**
+   * For the current vector being built.
+   *
+   * @type {number}
+   * @private
+   */
+  this.vector_num_elems = 0;
+
+  /**
+   * False omits default values from the serialized data
+   *
+   * @type {boolean}
+   * @private
+   */
+  this.force_defaults = false;
+};
+
+/**
+ * In order to save space, fields that are set to their default value
+ * don't get serialized into the buffer. Forcing defaults provides a
+ * way to manually disable this optimization.
+ *
+ * @param {boolean} forceDefaults true always serializes default values
+ */
+flatbuffers.Builder.prototype.forceDefaults = function(forceDefaults) {
+  this.force_defaults = forceDefaults;
+};
+
+/**
+ * Get the ByteBuffer representing the FlatBuffer. Only call this after you've
+ * called finish(). The actual data starts at the ByteBuffer's current position,
+ * not necessarily at 0.
+ *
+ * @returns {flatbuffers.ByteBuffer}
+ */
+flatbuffers.Builder.prototype.dataBuffer = function() {
+  return this.bb;
+};
+
+/**
+ * Get the bytes representing the FlatBuffer. Only call this after you've
+ * called finish().
+ *
+ * @returns {Uint8Array}
+ */
+flatbuffers.Builder.prototype.asUint8Array = function() {
+  return this.bb.bytes().subarray(this.bb.position(), this.bb.position() + this.offset());
+};
+
+/// @cond FLATBUFFERS_INTERNAL
+/**
+ * Prepare to write an element of `size` after `additional_bytes` have been
+ * written, e.g. if you write a string, you need to align such the int length
+ * field is aligned to 4 bytes, and the string data follows it directly. If all
+ * you need to do is alignment, `additional_bytes` will be 0.
+ *
+ * @param {number} size This is the of the new element to write
+ * @param {number} additional_bytes The padding size
+ */
+flatbuffers.Builder.prototype.prep = function(size, additional_bytes) {
+  // Track the biggest thing we've ever aligned to.
+  if (size > this.minalign) {
+    this.minalign = size;
+  }
+
+  // Find the amount of alignment needed such that `size` is properly
+  // aligned after `additional_bytes`
+  var align_size = ((~(this.bb.capacity() - this.space + additional_bytes)) + 1) & (size - 1);
+
+  // Reallocate the buffer if needed.
+  while (this.space < align_size + size + additional_bytes) {
+    var old_buf_size = this.bb.capacity();
+    this.bb = flatbuffers.Builder.growByteBuffer(this.bb);
+    this.space += this.bb.capacity() - old_buf_size;
+  }
+
+  this.pad(align_size);
+};
+
+/**
+ * @param {number} byte_size
+ */
+flatbuffers.Builder.prototype.pad = function(byte_size) {
+  for (var i = 0; i < byte_size; i++) {
+    this.bb.writeInt8(--this.space, 0);
+  }
+};
+
+/**
+ * @param {number} value
+ */
+flatbuffers.Builder.prototype.writeInt8 = function(value) {
+  this.bb.writeInt8(this.space -= 1, value);
+};
+
+/**
+ * @param {number} value
+ */
+flatbuffers.Builder.prototype.writeInt16 = function(value) {
+  this.bb.writeInt16(this.space -= 2, value);
+};
+
+/**
+ * @param {number} value
+ */
+flatbuffers.Builder.prototype.writeInt32 = function(value) {
+  this.bb.writeInt32(this.space -= 4, value);
+};
+
+/**
+ * @param {flatbuffers.Long} value
+ */
+flatbuffers.Builder.prototype.writeInt64 = function(value) {
+  this.bb.writeInt64(this.space -= 8, value);
+};
+
+/**
+ * @param {number} value
+ */
+flatbuffers.Builder.prototype.writeFloat32 = function(value) {
+  this.bb.writeFloat32(this.space -= 4, value);
+};
+
+/**
+ * @param {number} value
+ */
+flatbuffers.Builder.prototype.writeFloat64 = function(value) {
+  this.bb.writeFloat64(this.space -= 8, value);
+};
+/// @endcond
+
+/**
+ * Add an `int8` to the buffer, properly aligned, and grows the buffer (if necessary).
+ * @param {number} value The `int8` to add the the buffer.
+ */
+flatbuffers.Builder.prototype.addInt8 = function(value) {
+  this.prep(1, 0);
+  this.writeInt8(value);
+};
+
+/**
+ * Add an `int16` to the buffer, properly aligned, and grows the buffer (if necessary).
+ * @param {number} value The `int16` to add the the buffer.
+ */
+flatbuffers.Builder.prototype.addInt16 = function(value) {
+  this.prep(2, 0);
+  this.writeInt16(value);
+};
+
+/**
+ * Add an `int32` to the buffer, properly aligned, and grows the buffer (if necessary).
+ * @param {number} value The `int32` to add the the buffer.
+ */
+flatbuffers.Builder.prototype.addInt32 = function(value) {
+  this.prep(4, 0);
+  this.writeInt32(value);
+};
+
+/**
+ * Add an `int64` to the buffer, properly aligned, and grows the buffer (if necessary).
+ * @param {flatbuffers.Long} value The `int64` to add the the buffer.
+ */
+flatbuffers.Builder.prototype.addInt64 = function(value) {
+  this.prep(8, 0);
+  this.writeInt64(value);
+};
+
+/**
+ * Add a `float32` to the buffer, properly aligned, and grows the buffer (if necessary).
+ * @param {number} value The `float32` to add the the buffer.
+ */
+flatbuffers.Builder.prototype.addFloat32 = function(value) {
+  this.prep(4, 0);
+  this.writeFloat32(value);
+};
+
+/**
+ * Add a `float64` to the buffer, properly aligned, and grows the buffer (if necessary).
+ * @param {number} value The `float64` to add the the buffer.
+ */
+flatbuffers.Builder.prototype.addFloat64 = function(value) {
+  this.prep(8, 0);
+  this.writeFloat64(value);
+};
+
+/// @cond FLATBUFFERS_INTERNAL
+/**
+ * @param {number} voffset
+ * @param {number} value
+ * @param {number} defaultValue
+ */
+flatbuffers.Builder.prototype.addFieldInt8 = function(voffset, value, defaultValue) {
+  if (this.force_defaults || value != defaultValue) {
+    this.addInt8(value);
+    this.slot(voffset);
+  }
+};
+
+/**
+ * @param {number} voffset
+ * @param {number} value
+ * @param {number} defaultValue
+ */
+flatbuffers.Builder.prototype.addFieldInt16 = function(voffset, value, defaultValue) {
+  if (this.force_defaults || value != defaultValue) {
+    this.addInt16(value);
+    this.slot(voffset);
+  }
+};
+
+/**
+ * @param {number} voffset
+ * @param {number} value
+ * @param {number} defaultValue
+ */
+flatbuffers.Builder.prototype.addFieldInt32 = function(voffset, value, defaultValue) {
+  if (this.force_defaults || value != defaultValue) {
+    this.addInt32(value);
+    this.slot(voffset);
+  }
+};
+
+/**
+ * @param {number} voffset
+ * @param {flatbuffers.Long} value
+ * @param {flatbuffers.Long} defaultValue
+ */
+flatbuffers.Builder.prototype.addFieldInt64 = function(voffset, value, defaultValue) {
+  if (this.force_defaults || !value.equals(defaultValue)) {
+    this.addInt64(value);
+    this.slot(voffset);
+  }
+};
+
+/**
+ * @param {number} voffset
+ * @param {number} value
+ * @param {number} defaultValue
+ */
+flatbuffers.Builder.prototype.addFieldFloat32 = function(voffset, value, defaultValue) {
+  if (this.force_defaults || value != defaultValue) {
+    this.addFloat32(value);
+    this.slot(voffset);
+  }
+};
+
+/**
+ * @param {number} voffset
+ * @param {number} value
+ * @param {number} defaultValue
+ */
+flatbuffers.Builder.prototype.addFieldFloat64 = function(voffset, value, defaultValue) {
+  if (this.force_defaults || value != defaultValue) {
+    this.addFloat64(value);
+    this.slot(voffset);
+  }
+};
+
+/**
+ * @param {number} voffset
+ * @param {flatbuffers.Offset} value
+ * @param {flatbuffers.Offset} defaultValue
+ */
+flatbuffers.Builder.prototype.addFieldOffset = function(voffset, value, defaultValue) {
+  if (this.force_defaults || value != defaultValue) {
+    this.addOffset(value);
+    this.slot(voffset);
+  }
+};
+
+/**
+ * Structs are stored inline, so nothing additional is being added. `d` is always 0.
+ *
+ * @param {number} voffset
+ * @param {flatbuffers.Offset} value
+ * @param {flatbuffers.Offset} defaultValue
+ */
+flatbuffers.Builder.prototype.addFieldStruct = function(voffset, value, defaultValue) {
+  if (value != defaultValue) {
+    this.nested(value);
+    this.slot(voffset);
+  }
+};
+
+/**
+ * Structures are always stored inline, they need to be created right
+ * where they're used.  You'll get this assertion failure if you
+ * created it elsewhere.
+ *
+ * @param {flatbuffers.Offset} obj The offset of the created object
+ */
+flatbuffers.Builder.prototype.nested = function(obj) {
+  if (obj != this.offset()) {
+    throw new Error('FlatBuffers: struct must be serialized inline.');
+  }
+};
+
+/**
+ * Should not be creating any other object, string or vector
+ * while an object is being constructed
+ */
+flatbuffers.Builder.prototype.notNested = function() {
+  if (this.isNested) {
+    throw new Error('FlatBuffers: object serialization must not be nested.');
+  }
+};
+
+/**
+ * Set the current vtable at `voffset` to the current location in the buffer.
+ *
+ * @param {number} voffset
+ */
+flatbuffers.Builder.prototype.slot = function(voffset) {
+  this.vtable[voffset] = this.offset();
+};
+
+/**
+ * @returns {flatbuffers.Offset} Offset relative to the end of the buffer.
+ */
+flatbuffers.Builder.prototype.offset = function() {
+  return this.bb.capacity() - this.space;
+};
+
+/**
+ * Doubles the size of the backing ByteBuffer and copies the old data towards
+ * the end of the new buffer (since we build the buffer backwards).
+ *
+ * @param {flatbuffers.ByteBuffer} bb The current buffer with the existing data
+ * @returns {flatbuffers.ByteBuffer} A new byte buffer with the old data copied
+ * to it. The data is located at the end of the buffer.
+ *
+ * uint8Array.set() formally takes {Array<number>|ArrayBufferView}, so to pass
+ * it a uint8Array we need to suppress the type check:
+ * @suppress {checkTypes}
+ */
+flatbuffers.Builder.growByteBuffer = function(bb) {
+  var old_buf_size = bb.capacity();
+
+  // Ensure we don't grow beyond what fits in an int.
+  if (old_buf_size & 0xC0000000) {
+    throw new Error('FlatBuffers: cannot grow buffer beyond 2 gigabytes.');
+  }
+
+  var new_buf_size = old_buf_size << 1;
+  var nbb = flatbuffers.ByteBuffer.allocate(new_buf_size);
+  nbb.setPosition(new_buf_size - old_buf_size);
+  nbb.bytes().set(bb.bytes(), new_buf_size - old_buf_size);
+  return nbb;
+};
+/// @endcond
+
+/**
+ * Adds on offset, relative to where it will be written.
+ *
+ * @param {flatbuffers.Offset} offset The offset to add.
+ */
+flatbuffers.Builder.prototype.addOffset = function(offset) {
+  this.prep(flatbuffers.SIZEOF_INT, 0); // Ensure alignment is already done.
+  this.writeInt32(this.offset() - offset + flatbuffers.SIZEOF_INT);
+};
+
+/// @cond FLATBUFFERS_INTERNAL
+/**
+ * Start encoding a new object in the buffer.  Users will not usually need to
+ * call this directly. The FlatBuffers compiler will generate helper methods
+ * that call this method internally.
+ *
+ * @param {number} numfields
+ */
+flatbuffers.Builder.prototype.startObject = function(numfields) {
+  this.notNested();
+  if (this.vtable == null) {
+    this.vtable = [];
+  }
+  this.vtable_in_use = numfields;
+  for (var i = 0; i < numfields; i++) {
+    this.vtable[i] = 0; // This will push additional elements as needed
+  }
+  this.isNested = true;
+  this.object_start = this.offset();
+};
+
+/**
+ * Finish off writing the object that is under construction.
+ *
+ * @returns {flatbuffers.Offset} The offset to the object inside `dataBuffer`
+ */
+flatbuffers.Builder.prototype.endObject = function() {
+  if (this.vtable == null || !this.isNested) {
+    throw new Error('FlatBuffers: endObject called without startObject');
+  }
+
+  this.addInt32(0);
+  var vtableloc = this.offset();
+
+  // Write out the current vtable.
+  for (var i = this.vtable_in_use - 1; i >= 0; i--) {
+    // Offset relative to the start of the table.
+    this.addInt16(this.vtable[i] != 0 ? vtableloc - this.vtable[i] : 0);
+  }
+
+  var standard_fields = 2; // The fields below:
+  this.addInt16(vtableloc - this.object_start);
+  this.addInt16((this.vtable_in_use + standard_fields) * flatbuffers.SIZEOF_SHORT);
+
+  // Search for an existing vtable that matches the current one.
+  var existing_vtable = 0;
+outer_loop:
+  for (var i = 0; i < this.vtables.length; i++) {
+    var vt1 = this.bb.capacity() - this.vtables[i];
+    var vt2 = this.space;
+    var len = this.bb.readInt16(vt1);
+    if (len == this.bb.readInt16(vt2)) {
+      for (var j = flatbuffers.SIZEOF_SHORT; j < len; j += flatbuffers.SIZEOF_SHORT) {
+        if (this.bb.readInt16(vt1 + j) != this.bb.readInt16(vt2 + j)) {
+          continue outer_loop;
+        }
+      }
+      existing_vtable = this.vtables[i];
+      break;
+    }
+  }
+
+  if (existing_vtable) {
+    // Found a match:
+    // Remove the current vtable.
+    this.space = this.bb.capacity() - vtableloc;
+
+    // Point table to existing vtable.
+    this.bb.writeInt32(this.space, existing_vtable - vtableloc);
+  } else {
+    // No match:
+    // Add the location of the current vtable to the list of vtables.
+    this.vtables.push(this.offset());
+
+    // Point table to current vtable.
+    this.bb.writeInt32(this.bb.capacity() - vtableloc, this.offset() - vtableloc);
+  }
+
+  this.isNested = false;
+  return vtableloc;
+};
+/// @endcond
+
+/**
+ * Finalize a buffer, poiting to the given `root_table`.
+ *
+ * @param {flatbuffers.Offset} root_table
+ * @param {string=} opt_file_identifier
+ */
+flatbuffers.Builder.prototype.finish = function(root_table, opt_file_identifier) {
+  if (opt_file_identifier) {
+    var file_identifier = opt_file_identifier;
+    this.prep(this.minalign, flatbuffers.SIZEOF_INT +
+      flatbuffers.FILE_IDENTIFIER_LENGTH);
+    if (file_identifier.length != flatbuffers.FILE_IDENTIFIER_LENGTH) {
+      throw new Error('FlatBuffers: file identifier must be length ' +
+        flatbuffers.FILE_IDENTIFIER_LENGTH);
+    }
+    for (var i = flatbuffers.FILE_IDENTIFIER_LENGTH - 1; i >= 0; i--) {
+      this.writeInt8(file_identifier.charCodeAt(i));
+    }
+  }
+  this.prep(this.minalign, flatbuffers.SIZEOF_INT);
+  this.addOffset(root_table);
+  this.bb.setPosition(this.space);
+};
+
+/// @cond FLATBUFFERS_INTERNAL
+/**
+ * This checks a required field has been set in a given table that has
+ * just been constructed.
+ *
+ * @param {flatbuffers.Offset} table
+ * @param {number} field
+ */
+flatbuffers.Builder.prototype.requiredField = function(table, field) {
+  var table_start = this.bb.capacity() - table;
+  var vtable_start = table_start - this.bb.readInt32(table_start);
+  var ok = this.bb.readInt16(vtable_start + field) != 0;
+
+  // If this fails, the caller will show what field needs to be set.
+  if (!ok) {
+    throw new Error('FlatBuffers: field ' + field + ' must be set');
+  }
+};
+
+/**
+ * Start a new array/vector of objects.  Users usually will not call
+ * this directly. The FlatBuffers compiler will create a start/end
+ * method for vector types in generated code.
+ *
+ * @param {number} elem_size The size of each element in the array
+ * @param {number} num_elems The number of elements in the array
+ * @param {number} alignment The alignment of the array
+ */
+flatbuffers.Builder.prototype.startVector = function(elem_size, num_elems, alignment) {
+  this.notNested();
+  this.vector_num_elems = num_elems;
+  this.prep(flatbuffers.SIZEOF_INT, elem_size * num_elems);
+  this.prep(alignment, elem_size * num_elems); // Just in case alignment > int.
+};
+
+/**
+ * Finish off the creation of an array and all its elements. The array must be
+ * created with `startVector`.
+ *
+ * @returns {flatbuffers.Offset} The offset at which the newly created array
+ * starts.
+ */
+flatbuffers.Builder.prototype.endVector = function() {
+  this.writeInt32(this.vector_num_elems);
+  return this.offset();
+};
+/// @endcond
+
+/**
+ * Encode the string `s` in the buffer using UTF-8. If a Uint8Array is passed
+ * instead of a string, it is assumed to contain valid UTF-8 encoded data.
+ *
+ * @param {string|Uint8Array} s The string to encode
+ * @return {flatbuffers.Offset} The offset in the buffer where the encoded string starts
+ */
+flatbuffers.Builder.prototype.createString = function(s) {
+  if (s instanceof Uint8Array) {
+    var utf8 = s;
+  } else {
+    var utf8 = [];
+    var i = 0;
+
+    while (i < s.length) {
+      var codePoint;
+
+      // Decode UTF-16
+      var a = s.charCodeAt(i++);
+      if (a < 0xD800 || a >= 0xDC00) {
+        codePoint = a;
+      } else {
+        var b = s.charCodeAt(i++);
+        codePoint = (a << 10) + b + (0x10000 - (0xD800 << 10) - 0xDC00);
+      }
+
+      // Encode UTF-8
+      if (codePoint < 0x80) {
+        utf8.push(codePoint);
+      } else {
+        if (codePoint < 0x800) {
+          utf8.push(((codePoint >> 6) & 0x1F) | 0xC0);
+        } else {
+          if (codePoint < 0x10000) {
+            utf8.push(((codePoint >> 12) & 0x0F) | 0xE0);
+          } else {
+            utf8.push(
+              ((codePoint >> 18) & 0x07) | 0xF0,
+              ((codePoint >> 12) & 0x3F) | 0x80);
+          }
+          utf8.push(((codePoint >> 6) & 0x3F) | 0x80);
+        }
+        utf8.push((codePoint & 0x3F) | 0x80);
+      }
+    }
+  }
+
+  this.addInt8(0);
+  this.startVector(1, utf8.length, 1);
+  this.bb.setPosition(this.space -= utf8.length);
+  for (var i = 0, offset = this.space, bytes = this.bb.bytes(); i < utf8.length; i++) {
+    bytes[offset++] = utf8[i];
+  }
+  return this.endVector();
+};
+
+/**
+ * A helper function to avoid generated code depending on this file directly.
+ *
+ * @param {number} low
+ * @param {number} high
+ * @returns {flatbuffers.Long}
+ */
+flatbuffers.Builder.prototype.createLong = function(low, high) {
+  return flatbuffers.Long.create(low, high);
+};
+////////////////////////////////////////////////////////////////////////////////
+/// @cond FLATBUFFERS_INTERNAL
+/**
+ * Create a new ByteBuffer with a given array of bytes (`Uint8Array`).
+ *
+ * @constructor
+ * @param {Uint8Array} bytes
+ */
+flatbuffers.ByteBuffer = function(bytes) {
+  /**
+   * @type {Uint8Array}
+   * @private
+   */
+  this.bytes_ = bytes;
+
+  /**
+   * @type {number}
+   * @private
+   */
+  this.position_ = 0;
+};
+
+/**
+ * Create and allocate a new ByteBuffer with a given size.
+ *
+ * @param {number} byte_size
+ * @returns {flatbuffers.ByteBuffer}
+ */
+flatbuffers.ByteBuffer.allocate = function(byte_size) {
+  return new flatbuffers.ByteBuffer(new Uint8Array(byte_size));
+};
+
+/**
+ * Get the underlying `Uint8Array`.
+ *
+ * @returns {Uint8Array}
+ */
+flatbuffers.ByteBuffer.prototype.bytes = function() {
+  return this.bytes_;
+};
+
+/**
+ * Get the buffer's position.
+ *
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.position = function() {
+  return this.position_;
+};
+
+/**
+ * Set the buffer's position.
+ *
+ * @param {number} position
+ */
+flatbuffers.ByteBuffer.prototype.setPosition = function(position) {
+  this.position_ = position;
+};
+
+/**
+ * Get the buffer's capacity.
+ *
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.capacity = function() {
+  return this.bytes_.length;
+};
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.readInt8 = function(offset) {
+  return this.readUint8(offset) << 24 >> 24;
+};
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.readUint8 = function(offset) {
+  return this.bytes_[offset];
+};
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.readInt16 = function(offset) {
+  return this.readUint16(offset) << 16 >> 16;
+};
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.readUint16 = function(offset) {
+  return this.bytes_[offset] | this.bytes_[offset + 1] << 8;
+};
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.readInt32 = function(offset) {
+  return this.bytes_[offset] | this.bytes_[offset + 1] << 8 | this.bytes_[offset + 2] << 16 | this.bytes_[offset + 3] << 24;
+};
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.readUint32 = function(offset) {
+  return this.readInt32(offset) >>> 0;
+};
+
+/**
+ * @param {number} offset
+ * @returns {flatbuffers.Long}
+ */
+flatbuffers.ByteBuffer.prototype.readInt64 = function(offset) {
+  return new flatbuffers.Long(this.readInt32(offset), this.readInt32(offset + 4));
+};
+
+/**
+ * @param {number} offset
+ * @returns {flatbuffers.Long}
+ */
+flatbuffers.ByteBuffer.prototype.readUint64 = function(offset) {
+  return new flatbuffers.Long(this.readUint32(offset), this.readUint32(offset + 4));
+};
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.readFloat32 = function(offset) {
+  flatbuffers.int32[0] = this.readInt32(offset);
+  return flatbuffers.float32[0];
+};
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.readFloat64 = function(offset) {
+  flatbuffers.int32[flatbuffers.isLittleEndian ? 0 : 1] = this.readInt32(offset);
+  flatbuffers.int32[flatbuffers.isLittleEndian ? 1 : 0] = this.readInt32(offset + 4);
+  return flatbuffers.float64[0];
+};
+
+/**
+ * @param {number} offset
+ * @param {number|boolean} value
+ */
+flatbuffers.ByteBuffer.prototype.writeInt8 = function(offset, value) {
+  this.bytes_[offset] = /** @type {number} */(value);
+};
+
+/**
+ * @param {number} offset
+ * @param {number} value
+ */
+flatbuffers.ByteBuffer.prototype.writeUint8 = function(offset, value) {
+  this.bytes_[offset] = value;
+};
+
+/**
+ * @param {number} offset
+ * @param {number} value
+ */
+flatbuffers.ByteBuffer.prototype.writeInt16 = function(offset, value) {
+  this.bytes_[offset] = value;
+  this.bytes_[offset + 1] = value >> 8;
+};
+
+/**
+ * @param {number} offset
+ * @param {number} value
+ */
+flatbuffers.ByteBuffer.prototype.writeUint16 = function(offset, value) {
+    this.bytes_[offset] = value;
+    this.bytes_[offset + 1] = value >> 8;
+};
+
+/**
+ * @param {number} offset
+ * @param {number} value
+ */
+flatbuffers.ByteBuffer.prototype.writeInt32 = function(offset, value) {
+  this.bytes_[offset] = value;
+  this.bytes_[offset + 1] = value >> 8;
+  this.bytes_[offset + 2] = value >> 16;
+  this.bytes_[offset + 3] = value >> 24;
+};
+
+/**
+ * @param {number} offset
+ * @param {number} value
+ */
+flatbuffers.ByteBuffer.prototype.writeUint32 = function(offset, value) {
+    this.bytes_[offset] = value;
+    this.bytes_[offset + 1] = value >> 8;
+    this.bytes_[offset + 2] = value >> 16;
+    this.bytes_[offset + 3] = value >> 24;
+};
+
+/**
+ * @param {number} offset
+ * @param {flatbuffers.Long} value
+ */
+flatbuffers.ByteBuffer.prototype.writeInt64 = function(offset, value) {
+  this.writeInt32(offset, value.low);
+  this.writeInt32(offset + 4, value.high);
+};
+
+/**
+ * @param {number} offset
+ * @param {flatbuffers.Long} value
+ */
+flatbuffers.ByteBuffer.prototype.writeUint64 = function(offset, value) {
+    this.writeUint32(offset, value.low);
+    this.writeUint32(offset + 4, value.high);
+};
+
+/**
+ * @param {number} offset
+ * @param {number} value
+ */
+flatbuffers.ByteBuffer.prototype.writeFloat32 = function(offset, value) {
+  flatbuffers.float32[0] = value;
+  this.writeInt32(offset, flatbuffers.int32[0]);
+};
+
+/**
+ * @param {number} offset
+ * @param {number} value
+ */
+flatbuffers.ByteBuffer.prototype.writeFloat64 = function(offset, value) {
+  flatbuffers.float64[0] = value;
+  this.writeInt32(offset, flatbuffers.int32[flatbuffers.isLittleEndian ? 0 : 1]);
+  this.writeInt32(offset + 4, flatbuffers.int32[flatbuffers.isLittleEndian ? 1 : 0]);
+};
+
+/**
+ * Look up a field in the vtable, return an offset into the object, or 0 if the
+ * field is not present.
+ *
+ * @param {number} bb_pos
+ * @param {number} vtable_offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.__offset = function(bb_pos, vtable_offset) {
+  var vtable = bb_pos - this.readInt32(bb_pos);
+  return vtable_offset < this.readInt16(vtable) ? this.readInt16(vtable + vtable_offset) : 0;
+};
+
+/**
+ * Initialize any Table-derived type to point to the union at the given offset.
+ *
+ * @param {flatbuffers.Table} t
+ * @param {number} offset
+ * @returns {flatbuffers.Table}
+ */
+flatbuffers.ByteBuffer.prototype.__union = function(t, offset) {
+  t.bb_pos = offset + this.readInt32(offset);
+  t.bb = this;
+  return t;
+};
+
+/**
+ * Create a JavaScript string from UTF-8 data stored inside the FlatBuffer.
+ * This allocates a new string and converts to wide chars upon each access.
+ *
+ * To avoid the conversion to UTF-16, pass flatbuffers.Encoding.UTF8_BYTES as
+ * the "optionalEncoding" argument. This is useful for avoiding conversion to
+ * and from UTF-16 when the data will just be packaged back up in another
+ * FlatBuffer later on.
+ *
+ * @param {number} offset
+ * @param {flatbuffers.Encoding=} opt_encoding Defaults to UTF16_STRING
+ * @returns {string|Uint8Array}
+ */
+flatbuffers.ByteBuffer.prototype.__string = function(offset, opt_encoding) {
+  offset += this.readInt32(offset);
+
+  var length = this.readInt32(offset);
+  var result = '';
+  var i = 0;
+
+  offset += flatbuffers.SIZEOF_INT;
+
+  if (opt_encoding === flatbuffers.Encoding.UTF8_BYTES) {
+    return this.bytes_.subarray(offset, offset + length);
+  }
+
+  while (i < length) {
+    var codePoint;
+
+    // Decode UTF-8
+    var a = this.readUint8(offset + i++);
+    if (a < 0xC0) {
+      codePoint = a;
+    } else {
+      var b = this.readUint8(offset + i++);
+      if (a < 0xE0) {
+        codePoint =
+          ((a & 0x1F) << 6) |
+          (b & 0x3F);
+      } else {
+        var c = this.readUint8(offset + i++);
+        if (a < 0xF0) {
+          codePoint =
+            ((a & 0x0F) << 12) |
+            ((b & 0x3F) << 6) |
+            (c & 0x3F);
+        } else {
+          var d = this.readUint8(offset + i++);
+          codePoint =
+            ((a & 0x07) << 18) |
+            ((b & 0x3F) << 12) |
+            ((c & 0x3F) << 6) |
+            (d & 0x3F);
+        }
+      }
+    }
+
+    // Encode UTF-16
+    if (codePoint < 0x10000) {
+      result += String.fromCharCode(codePoint);
+    } else {
+      codePoint -= 0x10000;
+      result += String.fromCharCode(
+        (codePoint >> 10) + 0xD800,
+        (codePoint & ((1 << 10) - 1)) + 0xDC00);
+    }
+  }
+
+  return result;
+};
+
+/**
+ * Retrieve the relative offset stored at "offset"
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.__indirect = function(offset) {
+  return offset + this.readInt32(offset);
+};
+
+/**
+ * Get the start of data of a vector whose offset is stored at "offset" in this object.
+ *
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.__vector = function(offset) {
+  return offset + this.readInt32(offset) + flatbuffers.SIZEOF_INT; // data starts after the length
+};
+
+/**
+ * Get the length of a vector whose offset is stored at "offset" in this object.
+ *
+ * @param {number} offset
+ * @returns {number}
+ */
+flatbuffers.ByteBuffer.prototype.__vector_len = function(offset) {
+  return this.readInt32(offset + this.readInt32(offset));
+};
+
+/**
+ * @param {string} ident
+ * @returns {boolean}
+ */
+flatbuffers.ByteBuffer.prototype.__has_identifier = function(ident) {
+  if (ident.length != flatbuffers.FILE_IDENTIFIER_LENGTH) {
+    throw new Error('FlatBuffers: file identifier must be length ' +
+                    flatbuffers.FILE_IDENTIFIER_LENGTH);
+  }
+  for (var i = 0; i < flatbuffers.FILE_IDENTIFIER_LENGTH; i++) {
+    if (ident.charCodeAt(i) != this.readInt8(this.position_ + flatbuffers.SIZEOF_INT + i)) {
+      return false;
+    }
+  }
+  return true;
+};
+
+/**
+ * A helper function to avoid generated code depending on this file directly.
+ *
+ * @param {number} low
+ * @param {number} high
+ * @returns {flatbuffers.Long}
+ */
+flatbuffers.ByteBuffer.prototype.createLong = function(low, high) {
+  return flatbuffers.Long.create(low, high);
+};
+
+// Exports for Node.js and RequireJS
+exports.flatbuffers = flatbuffers;
+
+/// @endcond
+/// @}
diff --git a/js/closure-compiler-scripts/text-encoding.js b/js/closure-compiler-scripts/text-encoding.js
new file mode 100644
index 0000000000000..ca9154f88ecba
--- /dev/null
+++ b/js/closure-compiler-scripts/text-encoding.js
@@ -0,0 +1,648 @@
+/**
+ * closure-compiler-friendly text-encoding-utf-8
+ * copied from node_modules/text-encoding-utf-8/lib/encoding.cjs.js
+ * update as needed
+ */
+
+ // This is free and unencumbered software released into the public domain.
+// See LICENSE.md for more information.
+
+//
+// Utilities
+//
+
+goog.module("module$text_encoding");
+goog.module.declareLegacyNamespace();
+/**
+ * @param {number} a The number to test.
+ * @param {number} min The minimum value in the range, inclusive.
+ * @param {number} max The maximum value in the range, inclusive.
+ * @return {boolean} True if a >= min and a <= max.
+ */
+function inRange(a, min, max) {
+  return min <= a && a <= max;
+}
+
+/**
+ * @param {*} o
+ * @return {Object}
+ */
+function ToDictionary(o) {
+  if (o === undefined) return {};
+  if (o === Object(o)) return o;
+  throw TypeError('Could not convert argument to dictionary');
+}
+
+/**
+ * @param {string} string Input string of UTF-16 code units.
+ * @return {!Array.<number>} Code points.
+ */
+function stringToCodePoints(string) {
+  // https://heycam.github.io/webidl/#dfn-obtain-unicode
+
+  // 1. Let S be the DOMString value.
+  var s = String(string);
+
+  // 2. Let n be the length of S.
+  var n = s.length;
+
+  // 3. Initialize i to 0.
+  var i = 0;
+
+  // 4. Initialize U to be an empty sequence of Unicode characters.
+  var u = [];
+
+  // 5. While i < n:
+  while (i < n) {
+
+    // 1. Let c be the code unit in S at index i.
+    var c = s.charCodeAt(i);
+
+    // 2. Depending on the value of c:
+
+    // c < 0xD800 or c > 0xDFFF
+    if (c < 0xD800 || c > 0xDFFF) {
+      // Append to U the Unicode character with code point c.
+      u.push(c);
+    }
+
+    // 0xDC00 ≤ c ≤ 0xDFFF
+    else if (0xDC00 <= c && c <= 0xDFFF) {
+      // Append to U a U+FFFD REPLACEMENT CHARACTER.
+      u.push(0xFFFD);
+    }
+
+    // 0xD800 ≤ c ≤ 0xDBFF
+    else if (0xD800 <= c && c <= 0xDBFF) {
+      // 1. If i = n−1, then append to U a U+FFFD REPLACEMENT
+      // CHARACTER.
+      if (i === n - 1) {
+        u.push(0xFFFD);
+      }
+      // 2. Otherwise, i < n−1:
+      else {
+        // 1. Let d be the code unit in S at index i+1.
+        var d = string.charCodeAt(i + 1);
+
+        // 2. If 0xDC00 ≤ d ≤ 0xDFFF, then:
+        if (0xDC00 <= d && d <= 0xDFFF) {
+          // 1. Let a be c & 0x3FF.
+          var a = c & 0x3FF;
+
+          // 2. Let b be d & 0x3FF.
+          var b = d & 0x3FF;
+
+          // 3. Append to U the Unicode character with code point
+          // 2^16+2^10*a+b.
+          u.push(0x10000 + (a << 10) + b);
+
+          // 4. Set i to i+1.
+          i += 1;
+        }
+
+        // 3. Otherwise, d < 0xDC00 or d > 0xDFFF. Append to U a
+        // U+FFFD REPLACEMENT CHARACTER.
+        else  {
+          u.push(0xFFFD);
+        }
+      }
+    }
+
+    // 3. Set i to i+1.
+    i += 1;
+  }
+
+  // 6. Return U.
+  return u;
+}
+
+/**
+ * @param {!Array.<number>} code_points Array of code points.
+ * @return {string} string String of UTF-16 code units.
+ */
+function codePointsToString(code_points) {
+  var s = '';
+  for (var i = 0; i < code_points.length; ++i) {
+    var cp = code_points[i];
+    if (cp <= 0xFFFF) {
+      s += String.fromCharCode(cp);
+    } else {
+      cp -= 0x10000;
+      s += String.fromCharCode((cp >> 10) + 0xD800,
+                               (cp & 0x3FF) + 0xDC00);
+    }
+  }
+  return s;
+}
+
+
+//
+// Implementation of Encoding specification
+// https://encoding.spec.whatwg.org/
+//
+
+//
+// 3. Terminology
+//
+
+/**
+ * End-of-stream is a special token that signifies no more tokens
+ * are in the stream.
+ * @const
+ */ var end_of_stream = -1;
+
+/**
+ * A stream represents an ordered sequence of tokens.
+ *
+ * @constructor
+ * @param {!(Array.<number>|Uint8Array)} tokens Array of tokens that provide the
+ * stream.
+ */
+function Stream(tokens) {
+  /** @type {!Array.<number>} */
+  this.tokens = [].slice.call(tokens);
+}
+
+Stream.prototype = {
+  /**
+   * @return {boolean} True if end-of-stream has been hit.
+   */
+  endOfStream: function() {
+    return !this.tokens.length;
+  },
+
+  /**
+   * When a token is read from a stream, the first token in the
+   * stream must be returned and subsequently removed, and
+   * end-of-stream must be returned otherwise.
+   *
+   * @return {number} Get the next token from the stream, or
+   * end_of_stream.
+   */
+   read: function() {
+    if (!this.tokens.length)
+      return end_of_stream;
+     return this.tokens.shift();
+   },
+
+  /**
+   * When one or more tokens are prepended to a stream, those tokens
+   * must be inserted, in given order, before the first token in the
+   * stream.
+   *
+   * @param {(number|!Array.<number>)} token The token(s) to prepend to the stream.
+   */
+  prepend: function(token) {
+    if (Array.isArray(token)) {
+      var tokens = /**@type {!Array.<number>}*/(token);
+      while (tokens.length)
+        this.tokens.unshift(tokens.pop());
+    } else {
+      this.tokens.unshift(token);
+    }
+  },
+
+  /**
+   * When one or more tokens are pushed to a stream, those tokens
+   * must be inserted, in given order, after the last token in the
+   * stream.
+   *
+   * @param {(number|!Array.<number>)} token The tokens(s) to prepend to the stream.
+   */
+  push: function(token) {
+    if (Array.isArray(token)) {
+      var tokens = /**@type {!Array.<number>}*/(token);
+      while (tokens.length)
+        this.tokens.push(tokens.shift());
+    } else {
+      this.tokens.push(token);
+    }
+  }
+};
+
+//
+// 4. Encodings
+//
+
+// 4.1 Encoders and decoders
+
+/** @const */
+var finished = -1;
+
+/**
+ * @param {boolean} fatal If true, decoding errors raise an exception.
+ * @param {number=} opt_code_point Override the standard fallback code point.
+ * @return {number} The code point to insert on a decoding error.
+ */
+function decoderError(fatal, opt_code_point) {
+  if (fatal)
+    throw TypeError('Decoder error');
+  return opt_code_point || 0xFFFD;
+}
+
+//
+// 7. API
+//
+
+/** @const */ var DEFAULT_ENCODING = 'utf-8';
+
+// 7.1 Interface TextDecoder
+
+/**
+ * @constructor
+ * @param {string=} encoding The label of the encoding;
+ *     defaults to 'utf-8'.
+ * @param {Object=} options
+ */
+function TextDecoder(encoding, options) {
+  if (!(this instanceof TextDecoder)) {
+    return new TextDecoder(encoding, options);
+  }
+  encoding = encoding !== undefined ? String(encoding).toLowerCase() : DEFAULT_ENCODING;
+  if (encoding !== DEFAULT_ENCODING) {
+    throw new Error('Encoding not supported. Only utf-8 is supported');
+  }
+  options = ToDictionary(options);
+
+  /** @private @type {boolean} */
+  this._streaming = false;
+  /** @private @type {boolean} */
+  this._BOMseen = false;
+  /** @private @type {?Decoder} */
+  this._decoder = null;
+  /** @private @type {boolean} */
+  this._fatal = Boolean(options['fatal']);
+  /** @private @type {boolean} */
+  this._ignoreBOM = Boolean(options['ignoreBOM']);
+
+  Object.defineProperty(this, 'encoding', {value: 'utf-8'});
+  Object.defineProperty(this, 'fatal', {value: this._fatal});
+  Object.defineProperty(this, 'ignoreBOM', {value: this._ignoreBOM});
+}
+
+TextDecoder.prototype = {
+  /**
+   * @param {ArrayBufferView=} input The buffer of bytes to decode.
+   * @param {Object=} options
+   * @return {string} The decoded string.
+   */
+  decode: function decode(input, options) {
+    var bytes;
+    if (typeof input === 'object' && input instanceof ArrayBuffer) {
+      bytes = new Uint8Array(input);
+    } else if (typeof input === 'object' && 'buffer' in input &&
+               input.buffer instanceof ArrayBuffer) {
+      bytes = new Uint8Array(input.buffer,
+                             input.byteOffset,
+                             input.byteLength);
+    } else {
+      bytes = new Uint8Array(0);
+    }
+
+    options = ToDictionary(options);
+
+    if (!this._streaming) {
+      this._decoder = new UTF8Decoder({fatal: this._fatal});
+      this._BOMseen = false;
+    }
+    this._streaming = Boolean(options['stream']);
+
+    var input_stream = new Stream(bytes);
+
+    var code_points = [];
+
+    /** @type {?(number|!Array.<number>)} */
+    var result;
+
+    while (!input_stream.endOfStream()) {
+      result = this._decoder.handler(input_stream, input_stream.read());
+      if (result === finished)
+        break;
+      if (result === null)
+        continue;
+      if (Array.isArray(result))
+        code_points.push.apply(code_points, /**@type {!Array.<number>}*/(result));
+      else
+        code_points.push(result);
+    }
+    if (!this._streaming) {
+      do {
+        result = this._decoder.handler(input_stream, input_stream.read());
+        if (result === finished)
+          break;
+        if (result === null)
+          continue;
+        if (Array.isArray(result))
+          code_points.push.apply(code_points, /**@type {!Array.<number>}*/(result));
+        else
+          code_points.push(result);
+      } while (!input_stream.endOfStream());
+      this._decoder = null;
+    }
+
+    if (code_points.length) {
+      // If encoding is one of utf-8, utf-16be, and utf-16le, and
+      // ignore BOM flag and BOM seen flag are unset, run these
+      // subsubsteps:
+      if (['utf-8'].indexOf(this.encoding) !== -1 &&
+          !this._ignoreBOM && !this._BOMseen) {
+        // If token is U+FEFF, set BOM seen flag.
+        if (code_points[0] === 0xFEFF) {
+          this._BOMseen = true;
+          code_points.shift();
+        } else {
+          // Otherwise, if token is not end-of-stream, set BOM seen
+          // flag and append token to output.
+          this._BOMseen = true;
+        }
+      }
+    }
+
+    return codePointsToString(code_points);
+  }
+};
+
+// 7.2 Interface TextEncoder
+
+/**
+ * @constructor
+ * @param {string=} encoding The label of the encoding;
+ *     defaults to 'utf-8'.
+ * @param {Object=} options
+ */
+function TextEncoder(encoding, options) {
+  if (!(this instanceof TextEncoder))
+    return new TextEncoder(encoding, options);
+  encoding = encoding !== undefined ? String(encoding).toLowerCase() : DEFAULT_ENCODING;
+  if (encoding !== DEFAULT_ENCODING) {
+    throw new Error('Encoding not supported. Only utf-8 is supported');
+  }
+  options = ToDictionary(options);
+
+  /** @private @type {boolean} */
+  this._streaming = false;
+  /** @private @type {?Encoder} */
+  this._encoder = null;
+  /** @private @type {{fatal: boolean}} */
+  this._options = {fatal: Boolean(options['fatal'])};
+
+  Object.defineProperty(this, 'encoding', {value: 'utf-8'});
+}
+
+TextEncoder.prototype = {
+  /**
+   * @param {string=} opt_string The string to encode.
+   * @param {Object=} options
+   * @return {Uint8Array} Encoded bytes, as a Uint8Array.
+   */
+  encode: function encode(opt_string, options) {
+    opt_string = opt_string ? String(opt_string) : '';
+    options = ToDictionary(options);
+
+    // NOTE: This option is nonstandard. None of the encodings
+    // permitted for encoding (i.e. UTF-8, UTF-16) are stateful,
+    // so streaming is not necessary.
+    if (!this._streaming)
+      this._encoder = new UTF8Encoder(this._options);
+    this._streaming = Boolean(options['stream']);
+
+    var bytes = [];
+    var input_stream = new Stream(stringToCodePoints(opt_string));
+    /** @type {?(number|!Array.<number>)} */
+    var result;
+    while (!input_stream.endOfStream()) {
+      result = this._encoder.handler(input_stream, input_stream.read());
+      if (result === finished)
+        break;
+      if (Array.isArray(result))
+        bytes.push.apply(bytes, /**@type {!Array.<number>}*/(result));
+      else
+        bytes.push(result);
+    }
+    if (!this._streaming) {
+      while (true) {
+        result = this._encoder.handler(input_stream, input_stream.read());
+        if (result === finished)
+          break;
+        if (Array.isArray(result))
+          bytes.push.apply(bytes, /**@type {!Array.<number>}*/(result));
+        else
+          bytes.push(result);
+      }
+      this._encoder = null;
+    }
+    return new Uint8Array(bytes);
+  }
+};
+
+//
+// 8. The encoding
+//
+
+// 8.1 utf-8
+
+/**
+ * @constructor
+ * @implements {Decoder}
+ * @param {{fatal: boolean}} options
+ */
+function UTF8Decoder(options) {
+  var fatal = options.fatal;
+
+  // utf-8's decoder's has an associated utf-8 code point, utf-8
+  // bytes seen, and utf-8 bytes needed (all initially 0), a utf-8
+  // lower boundary (initially 0x80), and a utf-8 upper boundary
+  // (initially 0xBF).
+  var /** @type {number} */ utf8_code_point = 0,
+      /** @type {number} */ utf8_bytes_seen = 0,
+      /** @type {number} */ utf8_bytes_needed = 0,
+      /** @type {number} */ utf8_lower_boundary = 0x80,
+      /** @type {number} */ utf8_upper_boundary = 0xBF;
+
+  /**
+   * @param {Stream} stream The stream of bytes being decoded.
+   * @param {number} bite The next byte read from the stream.
+   * @return {?(number|!Array.<number>)} The next code point(s)
+   *     decoded, or null if not enough data exists in the input
+   *     stream to decode a complete code point.
+   */
+  this.handler = function(stream, bite) {
+    // 1. If byte is end-of-stream and utf-8 bytes needed is not 0,
+    // set utf-8 bytes needed to 0 and return error.
+    if (bite === end_of_stream && utf8_bytes_needed !== 0) {
+      utf8_bytes_needed = 0;
+      return decoderError(fatal);
+    }
+
+    // 2. If byte is end-of-stream, return finished.
+    if (bite === end_of_stream)
+      return finished;
+
+    // 3. If utf-8 bytes needed is 0, based on byte:
+    if (utf8_bytes_needed === 0) {
+
+      // 0x00 to 0x7F
+      if (inRange(bite, 0x00, 0x7F)) {
+        // Return a code point whose value is byte.
+        return bite;
+      }
+
+      // 0xC2 to 0xDF
+      if (inRange(bite, 0xC2, 0xDF)) {
+        // Set utf-8 bytes needed to 1 and utf-8 code point to byte
+        // − 0xC0.
+        utf8_bytes_needed = 1;
+        utf8_code_point = bite - 0xC0;
+      }
+
+      // 0xE0 to 0xEF
+      else if (inRange(bite, 0xE0, 0xEF)) {
+        // 1. If byte is 0xE0, set utf-8 lower boundary to 0xA0.
+        if (bite === 0xE0)
+          utf8_lower_boundary = 0xA0;
+        // 2. If byte is 0xED, set utf-8 upper boundary to 0x9F.
+        if (bite === 0xED)
+          utf8_upper_boundary = 0x9F;
+        // 3. Set utf-8 bytes needed to 2 and utf-8 code point to
+        // byte − 0xE0.
+        utf8_bytes_needed = 2;
+        utf8_code_point = bite - 0xE0;
+      }
+
+      // 0xF0 to 0xF4
+      else if (inRange(bite, 0xF0, 0xF4)) {
+        // 1. If byte is 0xF0, set utf-8 lower boundary to 0x90.
+        if (bite === 0xF0)
+          utf8_lower_boundary = 0x90;
+        // 2. If byte is 0xF4, set utf-8 upper boundary to 0x8F.
+        if (bite === 0xF4)
+          utf8_upper_boundary = 0x8F;
+        // 3. Set utf-8 bytes needed to 3 and utf-8 code point to
+        // byte − 0xF0.
+        utf8_bytes_needed = 3;
+        utf8_code_point = bite - 0xF0;
+      }
+
+      // Otherwise
+      else {
+        // Return error.
+        return decoderError(fatal);
+      }
+
+      // Then (byte is in the range 0xC2 to 0xF4) set utf-8 code
+      // point to utf-8 code point << (6 × utf-8 bytes needed) and
+      // return continue.
+      utf8_code_point = utf8_code_point << (6 * utf8_bytes_needed);
+      return null;
+    }
+
+    // 4. If byte is not in the range utf-8 lower boundary to utf-8
+    // upper boundary, run these substeps:
+    if (!inRange(bite, utf8_lower_boundary, utf8_upper_boundary)) {
+
+      // 1. Set utf-8 code point, utf-8 bytes needed, and utf-8
+      // bytes seen to 0, set utf-8 lower boundary to 0x80, and set
+      // utf-8 upper boundary to 0xBF.
+      utf8_code_point = utf8_bytes_needed = utf8_bytes_seen = 0;
+      utf8_lower_boundary = 0x80;
+      utf8_upper_boundary = 0xBF;
+
+      // 2. Prepend byte to stream.
+      stream.prepend(bite);
+
+      // 3. Return error.
+      return decoderError(fatal);
+    }
+
+    // 5. Set utf-8 lower boundary to 0x80 and utf-8 upper boundary
+    // to 0xBF.
+    utf8_lower_boundary = 0x80;
+    utf8_upper_boundary = 0xBF;
+
+    // 6. Increase utf-8 bytes seen by one and set utf-8 code point
+    // to utf-8 code point + (byte − 0x80) << (6 × (utf-8 bytes
+    // needed − utf-8 bytes seen)).
+    utf8_bytes_seen += 1;
+    utf8_code_point += (bite - 0x80) << (6 * (utf8_bytes_needed - utf8_bytes_seen));
+
+    // 7. If utf-8 bytes seen is not equal to utf-8 bytes needed,
+    // continue.
+    if (utf8_bytes_seen !== utf8_bytes_needed)
+      return null;
+
+    // 8. Let code point be utf-8 code point.
+    var code_point = utf8_code_point;
+
+    // 9. Set utf-8 code point, utf-8 bytes needed, and utf-8 bytes
+    // seen to 0.
+    utf8_code_point = utf8_bytes_needed = utf8_bytes_seen = 0;
+
+    // 10. Return a code point whose value is code point.
+    return code_point;
+  };
+}
+
+/**
+ * @constructor
+ * @implements {Encoder}
+ * @param {{fatal: boolean}} options
+ */
+function UTF8Encoder(options) {
+  var fatal = options.fatal;
+  /**
+   * @param {Stream} stream Input stream.
+   * @param {number} code_point Next code point read from the stream.
+   * @return {(number|!Array.<number>)} Byte(s) to emit.
+   */
+  this.handler = function(stream, code_point) {
+    // 1. If code point is end-of-stream, return finished.
+    if (code_point === end_of_stream)
+      return finished;
+
+    // 2. If code point is in the range U+0000 to U+007F, return a
+    // byte whose value is code point.
+    if (inRange(code_point, 0x0000, 0x007f))
+      return code_point;
+
+    // 3. Set count and offset based on the range code point is in:
+    var count, offset;
+    // U+0080 to U+07FF:    1 and 0xC0
+    if (inRange(code_point, 0x0080, 0x07FF)) {
+      count = 1;
+      offset = 0xC0;
+    }
+    // U+0800 to U+FFFF:    2 and 0xE0
+    else if (inRange(code_point, 0x0800, 0xFFFF)) {
+      count = 2;
+      offset = 0xE0;
+    }
+    // U+10000 to U+10FFFF: 3 and 0xF0
+    else if (inRange(code_point, 0x10000, 0x10FFFF)) {
+      count = 3;
+      offset = 0xF0;
+    }
+
+    // 4.Let bytes be a byte sequence whose first byte is (code
+    // point >> (6 × count)) + offset.
+    var bytes = [(code_point >> (6 * count)) + offset];
+
+    // 5. Run these substeps while count is greater than 0:
+    while (count > 0) {
+
+      // 1. Set temp to code point >> (6 × (count − 1)).
+      var temp = code_point >> (6 * (count - 1));
+
+      // 2. Append to bytes 0x80 | (temp & 0x3F).
+      bytes.push(0x80 | (temp & 0x3F));
+
+      // 3. Decrease count by one.
+      count -= 1;
+    }
+
+    // 6. Return bytes bytes, in order.
+    return bytes;
+  };
+}
+
+exports.TextEncoder = TextEncoder;
+exports.TextDecoder = TextDecoder;
diff --git a/js/closure-compiler-scripts/tslib.js b/js/closure-compiler-scripts/tslib.js
new file mode 100644
index 0000000000000..b5a722a652c34
--- /dev/null
+++ b/js/closure-compiler-scripts/tslib.js
@@ -0,0 +1,151 @@
+/**
+ * closure-compiler-friendly tslib
+ * copied from node_modules/tslib/tslib.js
+ * update as needed
+ */
+
+var extendStatics = Object.setPrototypeOf ||
+    ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
+    function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; };
+
+function __extends(d, b) {
+    extendStatics(d, b);
+    function __() { this.constructor = d; }
+    d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
+};
+
+var __assign = Object.assign || function (t) {
+    for (var s, i = 1, n = arguments.length; i < n; i++) {
+        s = arguments[i];
+        for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];
+    }
+    return t;
+};
+
+function __rest(s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) if (e.indexOf(p[i]) < 0)
+            t[p[i]] = s[p[i]];
+    return t;
+};
+
+function __decorate(decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+
+function __param(paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+
+function __metadata(metadataKey, metadataValue) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(metadataKey, metadataValue);
+};
+
+function __awaiter(thisArg, _arguments, P, generator) {
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+
+function __generator(thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (_) try {
+            if (f = 1, y && (t = y[op[0] & 2 ? "return" : op[0] ? "throw" : "next"]) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [0, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+
+function __exportStar(m, exports) {
+    for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p];
+};
+
+function __values(o) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator], i = 0;
+    if (m) return m.call(o);
+    return {
+        next: function () {
+            if (o && i >= o.length) o = void 0;
+            return { value: o && o[i++], done: !o };
+        }
+    };
+};
+
+function __read(o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+
+function __spread() {
+    for (var ar = [], i = 0; i < arguments.length; i++)
+        ar = ar.concat(__read(arguments[i]));
+    return ar;
+};
+
+function __await(v) {
+    return this instanceof __await ? (this.v = v, this) : new __await(v);
+};
+
+function __asyncGenerator(thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r);  }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
+
+function __asyncDelegator(o) {
+    var i, p;
+    return i = {}, verb("next"), verb("throw", function (e) { throw e; }), verb("return"), i[Symbol.iterator] = function () { return this; }, i;
+    function verb(n, f) { if (o[n]) i[n] = function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === "return" } : f ? f(v) : v; }; }
+};
+
+function __asyncValues(o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator];
+    return m ? m.call(o) : typeof __values === "function" ? __values(o) : o[Symbol.iterator]();
+};
diff --git a/js/examples/read_file.html b/js/examples/read_file.html
index 5a650a09c8010..2a1ebaba82c50 100644
--- a/js/examples/read_file.html
+++ b/js/examples/read_file.html
@@ -21,7 +21,7 @@
 
 <html>
   <head>
-    <title>arrow.js browser test</title>
+    <title>Arrow.js browser test</title>
     <meta charset="utf-8">
     <style>
 table {
@@ -33,30 +33,41 @@
     </style>
     <script type="text/javascript">
 var reader = new FileReader();
-function addCell (tr, type, name) {
+function addCell (tr, type, value) {
   var td = document.createElement(type)
-  td.textContent = name;
+  td.textContent = value;
   tr.appendChild(td);
 }
 reader.onload = function (evt) {
-  var reader = new arrow.getReader(new Uint8Array(evt.target.result));
-  var schema = reader.getSchema();
-  var length = reader.loadNextBatch();
-console.log(JSON.stringify(schema, null, '\t'));
 
+  var arrowTable = Arrow.Table.from(new Uint8Array(evt.target.result));
   var thead = document.getElementById("thead");
   var tbody = document.getElementById("tbody");
-  var header_row = document.createElement("tr");
 
-  schema.forEach(function (d) {
-    addCell(header_row, "th", d.name);
-  });
+  while (thead.hasChildNodes()) {
+      thead.removeChild(thead.lastChild);
+  }
+
+  while (tbody.hasChildNodes()) {
+      tbody.removeChild(tbody.lastChild);
+  }
+
+  var header_row = document.createElement("tr");
+  for (let column of arrowTable.cols()) {
+    addCell(header_row, "th", column.name);
+  }
 
   thead.appendChild(header_row);
 
-  for (var i = 0; i < length; i += 1|0) {
+  for (let row of arrowTable.rows(true)) {
     var tr = document.createElement("tr");
-    schema.forEach(function (d) { addCell(tr, "td", reader.getVector(d.name).get(i)); });
+    for (let cell of row) {
+      addCell(tr, "td",
+        cell == null ? 'null'
+        : !Array.isArray(cell) ? cell
+        : '[' + cell.map((value) => value == null ? 'null' : value).join(', ') + ']'
+      );
+    }
     tbody.appendChild(tr);
   }
 }
@@ -74,6 +85,6 @@
       <tbody id="tbody">
       </tbody>
     </table>
-    <script type="text/javascript" src="../_bundles/arrow.js"></script>
+    <script type="text/javascript" src="../dist/Arrow.js"></script>
   </body>
 </html>
diff --git a/js/gulpfile.js b/js/gulpfile.js
new file mode 100644
index 0000000000000..90c45b769d9e4
--- /dev/null
+++ b/js/gulpfile.js
@@ -0,0 +1,285 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const del = require(`del`);
+const gulp = require(`gulp`);
+const path = require(`path`);
+const pump = require(`pump`);
+const ts = require(`gulp-typescript`);
+const streamMerge = require(`merge2`);
+const sourcemaps = require(`gulp-sourcemaps`);
+const child_process = require(`child_process`);
+const gulpJsonTransform = require(`gulp-json-transform`);
+const closureCompiler = require(`google-closure-compiler`).gulp();
+
+const knownTargets = [`es5`, `es2015`, `esnext`];
+const knownModules = [`cjs`, `esm`, `cls`, `umd`];
+
+// see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988
+const gCCTargets = {
+    es5: `ECMASCRIPT5`,
+    es2015: `ECMASCRIPT_2015`,
+    es2016: `ECMASCRIPT_2016`,
+    es2017: `ECMASCRIPT_2017`,
+    esnext: `ECMASCRIPT_NEXT`
+};
+
+const tsProjects = [];
+const argv = require(`command-line-args`)([
+    { name: `all`, alias: `a`, type: Boolean },
+    { name: 'update', alias: 'u', type: Boolean },
+    { name: 'verbose', alias: 'v', type: Boolean },
+    { name: `target`, type: String, defaultValue: `` },
+    { name: `module`, type: String, defaultValue: `` },
+    { name: `coverage`, type: Boolean, defaultValue: false },
+    { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
+    { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }
+]);
+
+const { targets, modules } = argv;
+
+argv.target && !targets.length && targets.push(argv.target);
+argv.module && !modules.length && modules.push(argv.module);
+(argv.all || !targets.length) && targets.push(`all`);
+(argv.all || !modules.length) && modules.push(`all`);
+
+for (const [target, format] of combinations([`all`, `all`])) {
+    const combo = `${target}:${format}`;
+    gulp.task(`test:${combo}`, gulp.series(testTask(target, format, combo, `targets/${target}/${format}`)));
+    gulp.task(`clean:${combo}`, gulp.series(cleanTask(target, format, combo, `targets/${target}/${format}`)));
+    gulp.task(`build:${combo}`, gulp.series(buildTask(target, format, combo, `targets/${target}/${format}`)));
+    gulp.task(`bundle:${combo}`, gulp.series(bundleTask(target, format, combo, `targets/${target}/${format}`)));
+    gulp.task(`test:debug:${combo}`, gulp.series(testTask(target, format, combo, `targets/${target}/${format}`, true)));
+}
+
+gulp.task(`test`, gulp.series(runTaskCombos(`test`)));
+gulp.task(`clean`, gulp.parallel(runTaskCombos(`clean`)));
+gulp.task(`build`, gulp.parallel(runTaskCombos(`bundle`)));
+gulp.task(`test:debug`, gulp.series(runTaskCombos(`test:debug`)));
+gulp.task(`default`, gulp.task(`build`));
+
+function runTaskCombos(name) {
+    const combos = [];
+    for (const [target, format] of combinations(targets, modules)) {
+        if (format === `cls`) {
+            continue;
+        }
+        combos.push(`${name}:${target}:${format}`);
+    }
+    return combos;
+}
+
+function cleanTask(target, format, taskName, outDir) {
+    return () => {
+        const globs = [`${outDir}/**`];
+        if (target === `es5` && format === `cjs`) {
+            globs.push(`typings`);
+        }
+        return del(globs);
+    };
+}
+
+function buildTask(target, format, taskName, outDir) {
+    return format === `umd`
+        ? closureTask(target, format, taskName, outDir)
+        : typescriptTask(target, format, taskName, outDir);
+}
+
+function bundleTask(target, format, taskName, outDir) {
+    return [
+        [`build:${taskName}`],
+        (cb) => streamMerge([
+            pump(gulp.src([`LICENSE`, `README.md`, `CHANGELOG.md`]), gulp.dest(outDir)),
+            pump(
+                gulp.src(`package.json`),
+                gulpJsonTransform((orig) => [
+                    `version`, `description`,
+                    `author`, `homepage`, `bugs`, `license`,
+                    `keywords`, `repository`, `peerDependencies`
+                ].reduce((copy, key) => (
+                    (copy[key] = orig[key]) && copy || copy
+                ), {
+                    main: `Arrow.js`,
+                    typings: `Arrow.d.ts`,
+                    name: `@apache-arrow/${target}-${format}`
+                }), 2),
+                gulp.dest(outDir),
+                onError
+            )
+        ])
+    ];
+}
+
+function testTask(target, format, taskName, outDir, debug) {
+    const jestOptions = !debug ? [] : [
+        `--runInBand`, `--env`, `jest-environment-node-debug`];
+    argv.update && jestOptions.unshift(`-u`);
+    argv.verbose && jestOptions.unshift(`--verbose`);
+    argv.coverage && jestOptions.unshift(`--coverage`);
+    const jestPath = `./node_modules/.bin/jest`;
+    const debugOpts = jestOptions.join(' ');
+    const spawnOptions = {
+        stdio: [`ignore`, `inherit`, `inherit`],
+        env: Object.assign({}, process.env, {
+            TEST_TARGET: target, TEST_MODULE: format
+        })
+    };
+    return () => !debug ?
+        child_process.spawn(jestPath, jestOptions, spawnOptions) :
+        child_process.exec(`node --inspect-brk ${jestPath} ${debugOpts}`, spawnOptions);
+}
+
+function closureTask(target, format, taskName, outDir) {
+    const clsTarget = `es5`;
+    const googleRoot = `targets/${clsTarget}/cls`;
+    const languageIn = clsTarget === `es5` ? `es2015` : clsTarget;
+    return [
+        [`clean:${taskName}`, `build:${clsTarget}:cls`],
+        () => {
+            return streamMerge([
+                closureStream(closureSrcs(false), `Arrow`, onError, true),
+                closureStream(closureSrcs(true), `Arrow.internal`, onError)
+            ])
+            .on('end', () => del([`targets/${target}/cls/**`]));
+        }
+    ];
+    function closureSrcs(isInternal) {
+        return gulp.src([
+            `closure-compiler-scripts/*.js`,
+            `${googleRoot}/**/*.js`,
+            `!${googleRoot}/format/*.js`,
+            `!${googleRoot}/Arrow.externs.js`,
+            `!${googleRoot}/Arrow${isInternal ? `` : `.internal`}.js`
+        ], { base: `./` });
+    }
+    function closureStream(sources, entry, onError, copyToDist) {
+        const streams = [
+            sources,
+            sourcemaps.init(),
+            closureCompiler(closureArgs(entry)),
+            sourcemaps.write('.'),
+            gulp.dest(outDir)
+        ];
+        // copy the UMD bundle to dist
+        if (target === `es5` && copyToDist) {
+            streams.push(gulp.dest(`dist`));
+        }
+        return pump(...streams, onError);
+    }
+    function closureArgs(entry) {
+        return {
+            third_party: true,
+            externs: `${googleRoot}/Arrow.externs.js`,
+            warning_level: `QUIET`,
+            dependency_mode: `LOOSE`,
+            rewrite_polyfills: false,
+            // formatting: `PRETTY_PRINT`,
+            compilation_level: `ADVANCED`,
+            assume_function_wrapper: true,
+            js_output_file: `${entry}.js`,
+            language_in: gCCTargets[languageIn],
+            language_out: gCCTargets[clsTarget],
+            entry_point: `${googleRoot}/${entry}.js`,
+            output_wrapper:
+`// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+(function (global, factory) {
+    typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :
+    typeof define === 'function' && define.amd ? define(['exports'], factory) :
+    (factory(global.Arrow = global.Arrow || {}));
+}(this, (function (exports) {%output%}.bind(this))));`
+        };
+    }
+}
+
+function typescriptTask(target, format, taskName, outDir) {
+    return [
+        [`clean:${taskName}`],
+        () => {
+            const tsconfigPath = `tsconfig/tsconfig.${target}.${format}.json`;
+            let { js, dts } = tsProjects.find((p) => p.target === target && p.format === format) || {};
+            if (!js || !dts) {
+                let tsProject = ts.createProject(tsconfigPath);
+                ({ js, dts } = pump(
+                    tsProject.src(),
+                    sourcemaps.init(),
+                    tsProject(ts.reporter.fullReporter(true)),
+                    onError
+                ));
+                dts = [dts, gulp.dest(outDir)];
+                js = [js, sourcemaps.write(), gulp.dest(outDir)];
+                // copy types to the root
+                if (target === `es5` && format === `cjs`) {
+                    dts.push(gulp.dest(`typings`));
+                }
+                tsProjects.push({
+                    target, format, 
+                    js: js = pump(...js, onError),
+                    dts: dts = pump(...dts, onError)
+                });
+            }
+            return streamMerge([ dts, js ]);
+        }
+    ];
+}
+
+function* combinations(_targets, _modules) {
+
+    const targets = known(knownTargets, _targets || [`all`]);
+    const modules = known(knownModules, _modules || [`all`]);
+
+    for (const format of modules) {
+        for (const target of targets) {
+            yield [target, format];
+        }
+    }
+
+    function known(known, values) {
+        return ~values.indexOf(`all`)
+            ? known
+            : Object.keys(
+                values.reduce((map, arg) => ((
+                    (known.indexOf(arg) !== -1) &&
+                    (map[arg.toLowerCase()] = true)
+                    || true) && map
+                ), {})
+            ).sort((a, b) => known.indexOf(a) - known.indexOf(b));
+    }
+}
+
+function onError(err) {
+    if (typeof err === 'number') {
+        process.exit(err);
+    } else if (err) {
+        console.error(err.stack || err.toString());
+        process.exit(1);
+    }
+}
\ No newline at end of file
diff --git a/js/lerna.json b/js/lerna.json
new file mode 100644
index 0000000000000..c8fb8c072c61e
--- /dev/null
+++ b/js/lerna.json
@@ -0,0 +1,9 @@
+{
+  "lerna": "2.0.0",
+  "version": "0.1.1",
+  "packages": [
+    "targets/es5/*",
+    "targets/es2015/*",
+    "targets/esnext/*"
+  ]
+}
diff --git a/js/package.json b/js/package.json
index e46b5bdc23c56..899822130972a 100644
--- a/js/package.json
+++ b/js/package.json
@@ -1,29 +1,140 @@
 {
-  "name": "arrow",
-  "version": "0.0.0",
-  "description": "",
-  "main": "lib/arrow.js",
+  "name": "apache-arrow",
+  "version": "0.1.2",
+  "main": "./targets/es5/cjs/Arrow.js",
+  "module": "./targets/es5/esm/Arrow.js",
+  "browser": "./targets/es5/umd/Arrow.js",
+  "jsnext:main": "./targets/es2015/esm/Arrow.js",
+  "esnext:main": "./targets/esnext/esm/Arrow.js",
+  "typings": "./typings/Arrow.d.ts",
+  "description": "Apache Arrow columnar in-memory format",
   "scripts": {
-    "build": "./flatbuffers.sh && tsc && tsc -m es6 --outDir lib-esm && webpack",
-    "clean": "rm -rf lib lib-esm _bundles",
-    "test": "./node_modules/mocha/bin/mocha ./spec/arrow.js",
-    "lint": "./node_modules/tslint/bin/tslint"
+    "lerna": "lerna",
+    "commit": "git-cz",
+    "test": "gulp test",
+    "build": "gulp build",
+    "clean": "gulp clean",
+    "perf": "node ./perf/index.js",
+    "test:debug": "gulp test:debug",
+    "test:coverage": "gulp test -t esnext -m esm --coverage",
+    "validate": "npm-run-all lint build test",
+    "lerna:publish": "lerna exec --bail=false npm publish",
+    "prepublishOnly": "sh ./prepublish.sh",
+    "commitmsg": "validate-commit-msg",
+    "doc": "shx rm -rf ./doc && esdoc",
+    "lint": "npm-run-all -p lint:*",
+    "lint:src": "tslint --fix --type-check -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
+    "lint:test": "tslint --fix --type-check -p test/tsconfig.json -c tslint.json \"test/**/*.ts\""
   },
-  "author": "",
-  "repository": "https://github.com/apache/arrow/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/apache/arrow.git"
+  },
+  "keywords": [
+    "apache",
+    "arrow"
+  ],
+  "author": "Apache Software Foundation",
   "license": "Apache-2.0",
-  "devDependencies": {
-    "awesome-typescript-loader": "^3.1.3",
-    "chai": "^3.5.0",
-    "mocha": "^3.3.0",
-    "tslint": "^5.3.2",
-    "typescript": "^2.3.2",
-    "uglifyjs-webpack-plugin": "^0.4.3",
-    "webpack": "^2.3.3"
+  "bugs": {
+    "url": "https://issues.apache.org/jira/projects/ARROW"
+  },
+  "homepage": "https://github.com/apache/arrow/blob/master/js/README.md",
+  "files": [
+    "src",
+    "dist",
+    "typings",
+    "targets",
+    "LICENSE",
+    "README.md"
+  ],
+  "peerDependencies": {
+    "tslib": "^1.7.1"
   },
   "dependencies": {
-    "flatbuffers": "^1.5.0",
-    "text-encoding": "^0.6.4",
-    "commander": "^2.9.0"
+    "command-line-args": "~4.0.7",
+    "command-line-usage": "~4.0.1",
+    "flatbuffers": "~1.7.0",
+    "text-encoding": "~0.6.4"
+  },
+  "devDependencies": {
+    "@types/flatbuffers": "~1.6.4",
+    "@types/jest": "~20.0.8",
+    "@types/node": "~8.0.24",
+    "@types/text-encoding": "0.0.32",
+    "benchmark": "~2.1.4",
+    "commitizen": "~2.9.6",
+    "conventional-changelog-cli": "~1.3.2",
+    "conventional-commits-detector": "~0.1.1",
+    "conventional-github-releaser": "~1.1.12",
+    "conventional-recommended-bump": "~1.0.1",
+    "coveralls": "~2.13.1",
+    "cz-conventional-changelog": "~2.0.0",
+    "del": "~3.0.0",
+    "esdoc": "~1.0.1",
+    "esdoc-standard-plugin": "~1.0.0",
+    "google-closure-compiler": "~20170806.0.0",
+    "gulp": "github:gulpjs/gulp#4.0",
+    "gulp-json-transform": "~0.4.2",
+    "gulp-sourcemaps": "~2.6.1",
+    "gulp-typescript": "~3.2.2",
+    "jest": "~20.0.4",
+    "jest-environment-node-debug": "~2.0.0",
+    "json": "~9.0.6",
+    "lerna": "2.0.0",
+    "lint-staged": "~4.0.1",
+    "merge2": "~1.1.0",
+    "mkdirp": "~0.5.1",
+    "npm-run-all": "~4.0.2",
+    "pump": "~1.0.2",
+    "rimraf": "~2.6.1",
+    "shx": "~0.2.2",
+    "text-encoding-utf-8": "~1.0.1",
+    "trash": "~4.0.1",
+    "ts-jest": "~20.0.10",
+    "tslib": "~1.7.1",
+    "tslint": "~5.6.0",
+    "typescript": "~2.4.2",
+    "validate-commit-msg": "~2.14.0"
+  },
+  "config": {
+    "commitizen": {
+      "path": "cz-conventional-changelog"
+    }
+  },
+  "lint-staged": {
+    "*.@(ts)": [
+      "tslint --fix",
+      "git add"
+    ]
+  },
+  "jest": {
+    "verbose": false,
+    "globals": {
+      "ts-jest": {
+        "tsConfigFile": "test/tsconfig.json"
+      }
+    },
+    "roots": [
+      "<rootDir>/test/"
+    ],
+    "moduleFileExtensions": [
+      "js",
+      "ts",
+      "tsx"
+    ],
+    "mapCoverage": true,
+    "coverageReporters": [
+      "lcov"
+    ],
+    "coveragePathIgnorePatterns": [
+      "format\\/(File|Message|Schema|Tensor)_generated\\.(js|ts)$",
+      "test\\/.*\\.(ts|tsx|js)$"
+    ],
+    "transform": {
+      ".(ts|tsx)": "<rootDir>/node_modules/ts-jest/preprocessor.js",
+      ".(js|jsx)": "<rootDir>/node_modules/babel-jest/build/index.js"
+    },
+    "testRegex": "(.*(-|\\.)(test|spec)s?)\\.(ts|tsx|js)$"
   }
 }
diff --git a/js/spec/dictionary.arrow b/js/perf/arrows/file/dictionary.arrow
similarity index 100%
rename from js/spec/dictionary.arrow
rename to js/perf/arrows/file/dictionary.arrow
diff --git a/js/spec/simple.arrow b/js/perf/arrows/file/simple.arrow
similarity index 100%
rename from js/spec/simple.arrow
rename to js/perf/arrows/file/simple.arrow
diff --git a/js/spec/struct_example.arrow b/js/perf/arrows/file/struct.arrow
similarity index 100%
rename from js/spec/struct_example.arrow
rename to js/perf/arrows/file/struct.arrow
diff --git a/js/perf/arrows/multi/count/records.arrow b/js/perf/arrows/multi/count/records.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..00d883762d36959727cd26f5c3e3e4dfafeba981
GIT binary patch
literal 224
zcmbQkz`!5^#2~=KAjZJPz{<b@WbuFmn3)(PfRqD>4F+65J`V#6Pz4u|?*qgFK+Fi1
ggAhzW>OT-53Bj4@;tp^bh#AP{z@!A!VJz}!0H3Q0ZvX%Q

literal 0
HcmV?d00001

diff --git a/js/perf/arrows/multi/count/schema.arrow b/js/perf/arrows/multi/count/schema.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..dfd24e9e0018c471a053b8621cb7ea8b46a5bd9e
GIT binary patch
literal 184
zcmXYqF%E<<3<Ng_?hHcc3W_TzD0l!5;SUt_AR!Rn;(6DFadeJ$?QAs8E$M@5HF~L#
zr5OuZ%B3o0IZ=hY2lPcPZcuM%-_oWG1D^2Ve|IKBBkGWs!5F0fM};2y#Ta|G8z*P4
RzXiyWzHisOZ0C9~dp%Qi3m5<Z

literal 0
HcmV?d00001

diff --git a/js/perf/arrows/multi/latlong/records.arrow b/js/perf/arrows/multi/latlong/records.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..563d12d175d4e39b10386f98c5614cb86a493424
GIT binary patch
literal 352
zcmcb^z`!5^#2~=KAjZJPz{<b@WbuFmn3)(PfK&sB4F+65J`V#6Pz4u|UjxJfK+Fo3
zgAgo0>OT-53Bj4@;to(5m^h3N(||6HEYAc~glsQJ4>}(t);5XP35J<m-#CIWNSs(O
U+4%4w7;gV?<PZoGs~4mW0F=Zhr~m)}

literal 0
HcmV?d00001

diff --git a/js/perf/arrows/multi/latlong/schema.arrow b/js/perf/arrows/multi/latlong/schema.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..638b2ab622f8ed55ff85783a2c1ad54321b90021
GIT binary patch
literal 264
zcmZ8cI|{-;6r7jDH3Sh0ixep=EImMu;4Q>T0wDqM7?xI^BFD2^XWnihKA86z-kaHl
z0XqO2Y!R?R?LCtf&I65e6{&_$Er>DqON^1rWzV@I_L<j?W@#(lX%A1<0}L-#qmne|
z`*0hk%Q#QEZ|N7X=8JSKJwYC4<|FazapK$(DT3by9}J)DRg<l#slWYuULWc53;*jL
ADgXcg

literal 0
HcmV?d00001

diff --git a/js/perf/arrows/multi/origins/records.arrow b/js/perf/arrows/multi/origins/records.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..49a8c407e176e107a9007cbed26b87e3c84608ed
GIT binary patch
literal 224
zcmb7+Q3`+{5JV@-2odTdqT_e~kJw%7t)WPN4UGGCX7Q4=V^h?jSzZeiEf}3nMlvo6
l>`@aJfj#35`_jFSW8A2V8h-QC8fk$0e`;lU^$MH+=mDH!3Sj^M

literal 0
HcmV?d00001

diff --git a/js/perf/arrows/multi/origins/schema.arrow b/js/perf/arrows/multi/origins/schema.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..0d10fb0e2d1354e5a147f49b5283632d4fdde04e
GIT binary patch
literal 1604
zcmeIxJ!lj`6bJCRGapA1Bua|d$Pxq#Q7lBQLUI>l5-+(k7firH-tE4-9d35!oY{?D
z7p!cw6C1I!5^U^j#6|==vD3yTSSZ+OBj<lMIh25nwR8M-zINWcnN7Y4A*R5W32|7A
zh+$F6+jta;V$x5Dv3zVOZwcY+0!+h{IF7jyF^bib=<SVRLKKGY;<*KWYzfYRk2Q%E
zgIIoVAIp~#Vw4cc?~hUl@Hhk);RNjE;`jGIAwKu}{SsD}@EbvYZ(Wf0hcM#%1yK~^
zsEw6vsc(cb?TI(&|3357{+TDRJ4&&b&+YgPP)gz`4CS%>IR?4<b4+7}pBck=1U|nx
z7!;p7QtS`<f7S=b=Zbr?gYiLsQ2SZG_ndDWD5L26^BjW!PQrPZg%!98*MUJn7kY39
z?!iNN3{T-Xyn?sz9zMb+*o80f9e%<vk{O3-n1NGp7B0XnT!uPqK?@XE=)n%$hX?Qo
zp1?DB0WaY-yn_$03t!;}^kEG5bOfeh22R8O^6xiLqmCjS)$oi>LO})pAhm>Is_B%h
zG;7ATjOuK%OM^P8Bq%e5aoyO|FRRB=QN*enORC3)wY*)IY3iDpZO3(Mwq?XR<aJUk
z+wnSUvzVR<R(ot+gLJSUxov!7UZ+M&Q#F}%+fpfRFr_If%q?2dp>$#X;)-z?RB2G9
z4(G`>D72I?0^sG85Q-X@_K`Ly>C#4~93G)FdR{fNn;Ba`lXP!B!4Z`*^O;Mf-jq6W
z#$s*Qab;-}soOCnYvkfQr?Qcb%UP>M%9JyU0G%>Yk%_TfQM}FmMk+>SNi%a5W4oCP
zDzTM_y+BHlTVTD-c7aoCWO_;AK3#qvRmvi^5iY6GH67BtWRQ^2OJo&tU|orEG$h(v
z){!JlU8AI1c1m_v2)P=)U*&05$pzQAO|EJjZlleLT{UYqajGi5jpAS(%~cusbgz(h
ict8K@)(@CUEJJFWRa|<EkIr_DZv{207;paN&+Hd!R}{7Y

literal 0
HcmV?d00001

diff --git a/js/spec/dictionary-stream.arrow b/js/perf/arrows/stream/dictionary.arrow
similarity index 100%
rename from js/spec/dictionary-stream.arrow
rename to js/perf/arrows/stream/dictionary.arrow
diff --git a/js/spec/simple-stream.arrow b/js/perf/arrows/stream/simple.arrow
similarity index 100%
rename from js/spec/simple-stream.arrow
rename to js/perf/arrows/stream/simple.arrow
diff --git a/js/spec/struct_example-stream.arrow b/js/perf/arrows/stream/struct.arrow
similarity index 100%
rename from js/spec/struct_example-stream.arrow
rename to js/perf/arrows/stream/struct.arrow
diff --git a/js/perf/config.js b/js/perf/config.js
new file mode 100644
index 0000000000000..4fbcda3799587
--- /dev/null
+++ b/js/perf/config.js
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const fs = require('fs');
+const path = require('path');
+const arrowFormats = ['file', 'stream'];
+const arrowFileNames = ['simple', 'struct', 'dictionary'];
+const multipartArrows = ['count', 'latlong', 'origins'];
+let arrowTestConfigurations = [];
+
+arrowTestConfigurations = multipartArrows.reduce((configs, folder) => {
+    const schemaPath = path.resolve(__dirname, `./arrows/multi/${folder}/schema.arrow`);
+    const recordsPath = path.resolve(__dirname, `./arrows/multi/${folder}/records.arrow`);
+    return [...configs, [`multipart ${folder}`, fs.readFileSync(schemaPath), fs.readFileSync(recordsPath)]];
+}, arrowTestConfigurations);
+
+arrowTestConfigurations = arrowFormats.reduce((configs, format) => {
+    return arrowFileNames.reduce((configs, name) => {
+        const arrowPath = path.resolve(__dirname, `./arrows/${format}/${name}.arrow`);
+        return [...configs, [`${name} ${format}`, fs.readFileSync(arrowPath)]];
+    }, configs);
+}, arrowTestConfigurations);
+
+module.exports = arrowTestConfigurations;
diff --git a/js/perf/index.js b/js/perf/index.js
new file mode 100644
index 0000000000000..669f690122d10
--- /dev/null
+++ b/js/perf/index.js
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Use the ES5 UMD target as perf baseline
+// ES6/7 iterators are faster in turbofan, but something about the
+// ES5 transpilation (rewriting let and const to var?) JITs better
+const { Table, readBuffers } = require('../dist/Arrow');
+// const { Table, readBuffers } = require('../targets/es5/cjs');
+// const { Table, readBuffers } = require('../targets/es2015/cjs');
+// const { Table, readBuffers } = require('../targets/esnext/cjs');
+
+const Benchmark = require('benchmark');
+const arrowTestConfigurations = require('./config');
+
+const suites = [];
+
+for (let [name, ...buffers] of arrowTestConfigurations) {
+    const parseSuite = new Benchmark.Suite(`Parse ${name}`, { async: true });
+    const sliceSuite = new Benchmark.Suite(`Slice ${name} vectors`, { async: true });
+    const iterateSuite = new Benchmark.Suite(`Iterate ${name} vectors`, { async: true });
+    const getByIndexSuite = new Benchmark.Suite(`Get ${name} values by index`, { async: true });
+    parseSuite.add(createFromTableTest(name, buffers));
+    parseSuite.add(createReadBuffersTest(name, buffers));
+    for (const vector of Table.from(...buffers).cols()) {
+        sliceSuite.add(createSliceTest(vector));
+        iterateSuite.add(createIterateTest(vector));
+        getByIndexSuite.add(createGetByIndexTest(vector));
+    }
+    suites.push(parseSuite, sliceSuite, getByIndexSuite, iterateSuite);
+}
+
+console.log('Running apache-arrow performance tests...\n');
+
+run();
+
+function run() {
+    var suite = suites.shift();
+    suite && suite.on('complete', function() {
+        console.log(suite.name + ':\n' + this.map(function(x) {
+            var str = x.toString();
+            var meanMsPerOp = Math.round(x.stats.mean * 100000)/100;
+            var sliceOf60FPS = Math.round((meanMsPerOp / (1000/60)) * 100000)/1000;
+            return `${str} (avg: ${meanMsPerOp}ms, or ${sliceOf60FPS}% of a frame @ 60FPS) ${x.suffix || ''}`;
+        }).join('\n') + '\n');
+        if (suites.length > 0) {
+            setTimeout(run, 1000);
+        }
+    })
+    .run({ async: true });
+}
+
+function createFromTableTest(name, buffers) {
+    let table;
+    return {
+        async: true,
+        name: `Table.from`,
+        fn() { table = Table.from(...buffers); }
+    };
+}
+
+function createReadBuffersTest(name, buffers) {
+    let vectors;
+    return {
+        async: true,
+        name: `readBuffers`,
+        fn() { for (vectors of readBuffers(...buffers)) {} }
+    };
+}
+
+function createSliceTest(vector) {
+    let xs;
+    return {
+        async: true,
+        name: `name: '${vector.name}', length: ${vector.length}, type: ${vector.type}`,
+        fn() { xs = vector.slice(); }
+    };
+}
+
+function createIterateTest(vector) {
+    let value;
+    return {
+        async: true,
+        name: `name: '${vector.name}', length: ${vector.length}, type: ${vector.type}`,
+        fn() { for (value of vector) {} }
+    };
+}
+
+function createGetByIndexTest(vector) {
+    let value;
+    return {
+        async: true,
+        name: `name: '${vector.name}', length: ${vector.length}, type: ${vector.type}`,
+        fn() {
+            for (let i = -1, n = vector.length; ++i < n;) {
+                value = vector.get(i);
+            }
+        }
+    };
+}
diff --git a/js/flatbuffers.sh b/js/prepublish.sh
old mode 100755
new mode 100644
similarity index 66%
rename from js/flatbuffers.sh
rename to js/prepublish.sh
index 0f8e3f9fe99a0..4ad8db1b439cb
--- a/js/flatbuffers.sh
+++ b/js/prepublish.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -17,14 +17,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-echo "Compiling flatbuffer schemas..."
-mkdir -p lib lib-esm
-DIR=`mktemp -d`
-flatc -o $DIR --js ../format/*.fbs
-cat $DIR/*_generated.js > src/Arrow_generated.js
-
-# Duplicate in the tsc-generated outputs - we can't make tsc pull in .js files
-# and still prooduce declaration files
-cat $DIR/*_generated.js > lib/Arrow_generated.js
-cat $DIR/*_generated.js > lib-esm/Arrow_generated.js
-rm -rf $DIR
+npm run lint
+npm run build
+npm run test
+preset=`conventional-commits-detector` && echo $preset
+bump=`conventional-recommended-bump -p $preset` && echo $bump
+npm --no-git-tag-version version $bump &>/dev/null
+npm run lerna:publish
\ No newline at end of file
diff --git a/js/spec/arrow.js b/js/spec/arrow.js
deleted file mode 100644
index 52c586bfc7467..0000000000000
--- a/js/spec/arrow.js
+++ /dev/null
@@ -1,179 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-var fs = require('fs');
-var chai = require('chai');
-var assert = chai.assert;
-var path= require('path');
-var arrow = require('../lib/arrow.js');
-
-test_files = [
-  {
-    name: 'simple',
-    batches: 1,
-    fields: [
-      {
-        "name": "foo",
-        "type": "Int",
-        "data": [[1, null, 3, 4, 5]]
-      },
-      {
-        "name": "bar",
-        "type": "FloatingPoint",
-        "data": [[1.0, null, null, 4.0, 5.0]]
-      },
-      {
-        "name": "baz",
-        "type": "Utf8",
-        "data": [["aa", null, null, "bbb", "cccc"]]
-      }
-    ]
-  },
-  {
-    name: 'struct_example',
-    batches: 2,
-    fields: [
-      {
-        "name": "struct_nullable",
-        "type": "Struct",
-        "data": [
-          [
-            null,
-            [null, 'MhRNxD4'],
-            [137773603, '3F9HBxK'],
-            [410361374, 'aVd88fp'],
-            null,
-            [null, '3loZrRf'],
-            null
-          ], [
-            null,
-            [null,null],
-            [null,null],
-            null,
-            [null, '78SLiRw'],
-            null,
-            null,
-            [null, '0ilsf82'],
-            [null, 'LjS9MbU'],
-            [null, null],
-          ]
-        ]
-      }
-    ]
-  },
-  {
-    name: 'dictionary',
-    batches: 2,
-    fields: [
-      {
-        "name": "example-csv",
-        "type": "Struct",
-        "data": [
-          [
-            ["Hermione", 25, new Float32Array([-53.235599517822266, 40.231998443603516])],
-            ["Severus", 30, new Float32Array([-62.22999954223633, 3])],
-          ], [
-            ["Harry", 20, new Float32Array([23, -100.23652648925781])]
-          ]
-        ]
-      }
-    ]
-  },
-];
-
-var buf;
-
-function makeSchemaChecks(fields) {
-  describe('schema', function () {
-    var schema;
-    beforeEach(function () {
-      schema = arrow.getSchema(buf);
-    });
-
-    it('should read the number of fields', function () {
-        assert.lengthOf(schema, fields.length);
-    });
-
-    it("should understand fields", function () {
-      for (i = 0; i < fields.length; i += 1|0) {
-          assert.equal(schema[i].name, fields[i].name);
-          assert.equal(schema[i].type, fields[i].type,
-                       'bad type for field ' + schema[i].name);
-      }
-    });
-  });
-}
-
-function makeDataChecks (batches, fields) {
-  describe('data', function() {
-    var reader;
-    beforeEach(function () {
-        reader = arrow.getReader(buf)
-    });
-    it('should read the correct number of record batches', function () {
-        assert.equal(reader.getBatchCount(), batches);
-    });
-    fields.forEach(function (field, i) {
-      it('should read ' + field.type + ' vector ' + field.name, function () {
-        for (var batch_idx = 0; batch_idx < batches; batch_idx += 1|0) {
-          reader.loadNextBatch();
-          var batch = field.data[batch_idx];
-          var vector = reader.getVector(field.name)
-          assert.isDefined(vector, "vector " + field.name);
-          assert.lengthOf(vector, batch.length, "vector " + field.name)
-          for (i = 0; i < vector.length; i += 1|0) {
-            if (field.type == "Date") {
-              assert.equal(vector.get(i).getTime(), batch[i].getTime(),
-                           "vector " + field.name + " index " + i);
-            } else {
-              assert.deepEqual(vector.get(i), batch[i],
-                               "vector " + field.name + " index " + i);
-            }
-          }
-        }
-      });
-    });
-  });
-}
-
-describe('arrow random-access file', function () {
-  test_files.forEach(function (test_file) {
-    describe(test_file.name, function () {
-      var fields = test_file.fields
-      beforeEach(function () {
-        buf = fs.readFileSync(path.resolve(__dirname, test_file.name + '.arrow'));
-      });
-
-      makeSchemaChecks(fields);
-      makeDataChecks(test_file.batches, fields);
-    })
-  });
-});
-
-describe('arrow streaming file format', function () {
-  test_files.forEach(function (test_file) {
-    describe(test_file.name, function () {
-      var fields = test_file.fields
-      beforeEach(function () {
-        buf = fs.readFileSync(path.resolve(__dirname, test_file.name + '-stream.arrow'));
-      });
-
-      makeSchemaChecks(fields);
-      makeDataChecks(test_file.batches, fields);
-    })
-  });
-});
diff --git a/js/src/Arrow.externs.ts b/js/src/Arrow.externs.ts
new file mode 100644
index 0000000000000..7342684e7a660
--- /dev/null
+++ b/js/src/Arrow.externs.ts
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/**
+ * @fileoverview Closure Compiler externs for Arrow
+ * @externs
+ * @suppress {duplicate,checkTypes}
+ */
+/** @type {symbol} */
+Symbol.iterator;
+/** @type {symbol} */
+Symbol.asyncIterator;
+let Table = function() {};
+/** @type {?} */
+Table.prototype.rows;
+/** @type {?} */
+Table.prototype.cols;
+/** @type {?} */
+Table.prototype.getRow;
+/** @type {?} */
+Table.prototype.getCell;
+/** @type {?} */
+Table.prototype.getCellAt;
+/** @type {?} */
+Table.prototype.getColumn;
+/** @type {?} */
+Table.prototype.getColumnAt;
+/** @type {?} */
+Table.prototype.toString;
+
+let Vector = function() {};
+/** @type {?} */
+Vector.prototype.length;
+/** @type {?} */
+Vector.prototype.name;
+/** @type {?} */
+Vector.prototype.type;
+/** @type {?} */
+Vector.prototype.props;
+/** @type {?} */
+Vector.prototype.get;
+/** @type {?} */
+Vector.prototype.concat;
+/** @type {?} */
+Vector.prototype.slice;
+
+let TypedVector = function() {};
+/** @type {?} */
+TypedVector.prototype.arrayType;
+
+let ValidityVector = function() {};
+/** @type {?} */
+(<any> ValidityVector).pack;
diff --git a/js/src/Arrow.internal.ts b/js/src/Arrow.internal.ts
new file mode 100644
index 0000000000000..d8f0c37de8359
--- /dev/null
+++ b/js/src/Arrow.internal.ts
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector as Vector_ } from './vector/vector';
+import { StructVector as StructVector_ } from './vector/struct';
+import { DictionaryVector as DictionaryVector_ } from './vector/dictionary';
+import { ListVector as ListVector_, Utf8Vector as Utf8Vector_, FixedSizeListVector as FixedSizeListVector_ } from './vector/list';
+import {
+    TypedVector as TypedVector_, BitVector as BitVector_,
+    DateVector as DateVector_, IndexVector as IndexVector_,
+    Int8Vector as Int8Vector_, Int16Vector as Int16Vector_,
+    Int32Vector as Int32Vector_, Int64Vector as Int64Vector_,
+    Uint8Vector as Uint8Vector_, Uint16Vector as Uint16Vector_,
+    Uint32Vector as Uint32Vector_, Uint64Vector as Uint64Vector_,
+    Float32Vector as Float32Vector_, Float64Vector as Float64Vector_,
+} from './vector/typed';
+
+export const vectors = {
+    Vector: Vector_,
+    BitVector: BitVector_,
+    ListVector: ListVector_,
+    Utf8Vector: Utf8Vector_,
+    DateVector: DateVector_,
+    IndexVector: IndexVector_,
+    TypedVector: TypedVector_,
+    Int8Vector: Int8Vector_,
+    Int16Vector: Int16Vector_,
+    Int32Vector: Int32Vector_,
+    Int64Vector: Int64Vector_,
+    Uint8Vector: Uint8Vector_,
+    Uint16Vector: Uint16Vector_,
+    Uint32Vector: Uint32Vector_,
+    Uint64Vector: Uint64Vector_,
+    Float32Vector: Float32Vector_,
+    Float64Vector: Float64Vector_,
+    StructVector: StructVector_,
+    DictionaryVector: DictionaryVector_,
+    FixedSizeListVector: FixedSizeListVector_,
+};
+
+export namespace vectors {
+    export type Vector<T> =  Vector_<T>;
+    export type BitVector =  BitVector_;
+    export type ListVector<T> =  ListVector_<T>;
+    export type Utf8Vector =  Utf8Vector_;
+    export type DateVector =  DateVector_;
+    export type IndexVector =  IndexVector_;
+    export type Int8Vector =  Int8Vector_;
+    export type Int16Vector =  Int16Vector_;
+    export type Int32Vector =  Int32Vector_;
+    export type Int64Vector =  Int64Vector_;
+    export type Uint8Vector =  Uint8Vector_;
+    export type Uint16Vector =  Uint16Vector_;
+    export type Uint32Vector =  Uint32Vector_;
+    export type Uint64Vector =  Uint64Vector_;
+    export type Float32Vector =  Float32Vector_;
+    export type Float64Vector =  Float64Vector_;
+    export type StructVector =  StructVector_;
+    export type DictionaryVector<T> =  DictionaryVector_<T>;
+    export type FixedSizeListVector<T> =  FixedSizeListVector_<T>;
+    export type TypedVector<T, TArray> =  TypedVector_<T, TArray>;
+}
+
+/* These exports are needed for the closure umd targets */
+try {
+    const Arrow = eval('exports');
+    if (typeof Arrow === 'object') {
+        // string indexers tell closure compiler not to rename these properties
+        Arrow['vectors'] = {};
+        Arrow['vectors']['Vector'] = Vector_;
+        Arrow['vectors']['BitVector'] = BitVector_;
+        Arrow['vectors']['ListVector'] = ListVector_;
+        Arrow['vectors']['Utf8Vector'] = Utf8Vector_;
+        Arrow['vectors']['DateVector'] = DateVector_;
+        Arrow['vectors']['IndexVector'] = IndexVector_;
+        Arrow['vectors']['Int8Vector'] = Int8Vector_;
+        Arrow['vectors']['Int16Vector'] = Int16Vector_;
+        Arrow['vectors']['Int32Vector'] = Int32Vector_;
+        Arrow['vectors']['Int64Vector'] = Int64Vector_;
+        Arrow['vectors']['Uint8Vector'] = Uint8Vector_;
+        Arrow['vectors']['Uint16Vector'] = Uint16Vector_;
+        Arrow['vectors']['Uint32Vector'] = Uint32Vector_;
+        Arrow['vectors']['Uint64Vector'] = Uint64Vector_;
+        Arrow['vectors']['Float32Vector'] = Float32Vector_;
+        Arrow['vectors']['Float64Vector'] = Float64Vector_;
+        Arrow['vectors']['StructVector'] = StructVector_;
+        Arrow['vectors']['DictionaryVector'] = DictionaryVector_;
+        Arrow['vectors']['FixedSizeListVector'] = FixedSizeListVector_;
+    }
+} catch (e) { /* not the UMD bundle */ }
+/** end closure exports */
diff --git a/js/bin/arrow_schema.js b/js/src/Arrow.ts
old mode 100755
new mode 100644
similarity index 62%
rename from js/bin/arrow_schema.js
rename to js/src/Arrow.ts
index 4917628641907..fe19645781322
--- a/js/bin/arrow_schema.js
+++ b/js/src/Arrow.ts
@@ -1,5 +1,3 @@
-#! /usr/bin/env node
-
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
@@ -17,13 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-var fs = require('fs');
-var process = require('process');
-var arrow = require('../lib/arrow.js');
-
-var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
-var reader = arrow.getReader(buf);
-console.log(JSON.stringify(reader.getSchema(), null, '\t'));
-//console.log(JSON.stringify(reader.getVectors(), null, '\t'));
-console.log('block count: ' + reader.getBatchCount());
+import { Table } from './table';
+import { readBuffers } from './reader/arrow';
+export { Table, readBuffers };
 
+/* These exports are needed for the closure umd targets */
+try {
+    const Arrow = eval('exports');
+    if (typeof Arrow === 'object') {
+        // string indexers tell closure compiler not to rename these properties
+        Arrow['Table'] = Table;
+        Arrow['readBuffers'] = readBuffers;
+    }
+} catch (e) { /* not the UMD bundle */ }
+/** end closure exports */
diff --git a/js/src/Arrow_generated.d.ts b/js/src/Arrow_generated.d.ts
deleted file mode 100644
index 1f5b4547a478c..0000000000000
--- a/js/src/Arrow_generated.d.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-export var org: {
-  apache: {
-    arrow: any
-  }
-}
diff --git a/js/src/arrow.ts b/js/src/arrow.ts
deleted file mode 100644
index ac87a8c0ed766..0000000000000
--- a/js/src/arrow.ts
+++ /dev/null
@@ -1,515 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { flatbuffers } from "flatbuffers";
-import { org } from "./Arrow_generated";
-import { Vector, vectorFromField } from "./types";
-
-import ByteBuffer = flatbuffers.ByteBuffer;
-const Footer = org.apache.arrow.flatbuf.Footer;
-const Message = org.apache.arrow.flatbuf.Message;
-const MessageHeader = org.apache.arrow.flatbuf.MessageHeader;
-const RecordBatch = org.apache.arrow.flatbuf.RecordBatch;
-const DictionaryBatch = org.apache.arrow.flatbuf.DictionaryBatch;
-const Schema = org.apache.arrow.flatbuf.Schema;
-const Type = org.apache.arrow.flatbuf.Type;
-const VectorType = org.apache.arrow.flatbuf.VectorType;
-
-export class ArrowReader {
-
-    private bb;
-    private schema: any = [];
-    private vectors: Vector[];
-    private vectorMap: any = {};
-    private dictionaries: any = {};
-    private batches: any = [];
-    private batchIndex: number = 0;
-
-    constructor(bb, schema, vectors: Vector[], batches, dictionaries) {
-        this.bb = bb;
-        this.schema = schema;
-        this.vectors = vectors;
-        for (let i = 0; i < vectors.length; i++) {
-            this.vectorMap[vectors[i].name] = vectors[i];
-        }
-        this.batches = batches;
-        this.dictionaries = dictionaries;
-    }
-
-    public loadNextBatch() {
-        if (this.batchIndex < this.batches.length) {
-            const batch = this.batches[this.batchIndex];
-            this.batchIndex += 1;
-            loadVectors(this.bb, this.vectors, batch);
-            return batch.length;
-        } else {
-            return 0;
-        }
-    }
-
-    public getSchema() {
-        return this.schema;
-    }
-
-    public getVectors() {
-        return this.vectors;
-    }
-
-    public getVector(name) {
-        return this.vectorMap[name];
-    }
-
-    public getBatchCount() {
-        return this.batches.length;
-    }
-
-    // the index of the next batch to be loaded
-    public getBatchIndex() {
-        return this.batchIndex;
-    }
-
-    // set the index of the next batch to be loaded
-    public setBatchIndex(i: number) {
-        this.batchIndex = i;
-    }
-}
-
-export function getSchema(buf) { return getReader(buf).getSchema(); }
-
-export function getReader(buf): ArrowReader {
-    if (_checkMagic(buf, 0)) {
-        return getFileReader(buf);
-    } else {
-        return getStreamReader(buf);
-    }
-}
-
-export function getStreamReader(buf): ArrowReader {
-    const bb = new ByteBuffer(buf);
-
-    const schema = _loadSchema(bb);
-    let field;
-    const vectors: Vector[] = [];
-    let i;
-    let iLen;
-    let batch;
-    const recordBatches = [];
-    const dictionaryBatches = [];
-    const dictionaries = {};
-
-    for (i = 0, iLen = schema.fieldsLength(); i < iLen; i++) {
-        field = schema.fields(i);
-        _createDictionaryVectors(field, dictionaries);
-        vectors.push(vectorFromField(field, dictionaries));
-    }
-
-    while (bb.position() < bb.capacity()) {
-      batch = _loadBatch(bb);
-      if (batch == null) {
-          break;
-      } else if (batch.type === MessageHeader.DictionaryBatch) {
-          dictionaryBatches.push(batch);
-      } else if (batch.type === MessageHeader.RecordBatch) {
-          recordBatches.push(batch);
-      } else {
-          throw new Error("Expected batch type" + MessageHeader.RecordBatch + " or " +
-              MessageHeader.DictionaryBatch + " but got " + batch.type);
-      }
-    }
-
-    // load dictionary vectors
-    for (i = 0; i < dictionaryBatches.length; i++) {
-      batch = dictionaryBatches[i];
-      loadVectors(bb, [dictionaries[batch.id]], batch);
-    }
-
-    return new ArrowReader(bb, parseSchema(schema), vectors, recordBatches, dictionaries);
-}
-
-export function getFileReader(buf): ArrowReader {
-    const bb = new ByteBuffer(buf);
-
-    const footer = _loadFooter(bb);
-
-    const schema = footer.schema();
-    let i;
-    let len;
-    let field;
-    const vectors: Vector[] = [];
-    let block;
-    let batch;
-    const recordBatchBlocks = [];
-    const dictionaryBatchBlocks = [];
-    const dictionaries = {};
-
-    for (i = 0, len = schema.fieldsLength(); i < len; i++) {
-        field = schema.fields(i);
-        _createDictionaryVectors(field, dictionaries);
-        vectors.push(vectorFromField(field, dictionaries));
-    }
-
-    for (i = 0; i < footer.dictionariesLength(); i++) {
-        block = footer.dictionaries(i);
-        dictionaryBatchBlocks.push({
-            bodyLength: block.bodyLength().low,
-            metaDataLength: block.metaDataLength(),
-            offset: block.offset().low,
-        });
-    }
-
-    for (i = 0; i < footer.recordBatchesLength(); i++) {
-        block = footer.recordBatches(i);
-        recordBatchBlocks.push({
-            bodyLength: block.bodyLength().low,
-            metaDataLength: block.metaDataLength(),
-            offset: block.offset().low,
-        });
-    }
-
-    const dictionaryBatches = dictionaryBatchBlocks.map((batchBlock) => {
-        bb.setPosition(batchBlock.offset);
-        // TODO: Make sure this is a dictionary batch
-        return _loadBatch(bb);
-    });
-
-    const recordBatches = recordBatchBlocks.map((batchBlock) => {
-        bb.setPosition(batchBlock.offset);
-        // TODO: Make sure this is a record batch
-        return _loadBatch(bb);
-    });
-
-    // load dictionary vectors
-    for (i = 0; i < dictionaryBatches.length; i++) {
-        batch = dictionaryBatches[i];
-        loadVectors(bb, [dictionaries[batch.id]], batch);
-    }
-
-    return new ArrowReader(bb, parseSchema(schema), vectors, recordBatches, dictionaries);
-}
-
-function _loadFooter(bb) {
-    const fileLength: number = bb.bytes_.length;
-
-    if (fileLength < MAGIC.length * 2 + 4) {
-      throw new Error("file too small " + fileLength);
-    }
-
-    if (!_checkMagic(bb.bytes_, 0)) {
-      throw new Error("missing magic bytes at beginning of file");
-    }
-
-    if (!_checkMagic(bb.bytes_, fileLength - MAGIC.length)) {
-      throw new Error("missing magic bytes at end of file");
-    }
-
-    const footerLengthOffset: number = fileLength - MAGIC.length - 4;
-    bb.setPosition(footerLengthOffset);
-    const footerLength: number = Int32FromByteBuffer(bb, footerLengthOffset);
-
-    if (footerLength <= 0 || footerLength + MAGIC.length * 2 + 4 > fileLength)  {
-      throw new Error("Invalid footer length: " + footerLength);
-    }
-
-    const footerOffset: number = footerLengthOffset - footerLength;
-    bb.setPosition(footerOffset);
-    const footer = Footer.getRootAsFooter(bb);
-
-    return footer;
-}
-
-function _loadSchema(bb) {
-    const message = _loadMessage(bb);
-    if (message.headerType() !== MessageHeader.Schema) {
-        throw new Error("Expected header type " + MessageHeader.Schema + " but got " + message.headerType());
-    }
-    return message.header(new Schema());
-}
-
-function _loadBatch(bb) {
-    const message = _loadMessage(bb);
-    if (message == null) {
-        return;
-    } else if (message.headerType() === MessageHeader.RecordBatch) {
-        const batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low };
-        return _loadRecordBatch(bb, batch);
-    } else if (message.headerType() === MessageHeader.DictionaryBatch) {
-        const batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low };
-        return _loadDictionaryBatch(bb, batch);
-    } else {
-        throw new Error("Expected header type " + MessageHeader.RecordBatch + " or " + MessageHeader.DictionaryBatch +
-            " but got " + message.headerType());
-    }
-}
-
-function _loadRecordBatch(bb, batch) {
-    const data = batch.header;
-    let i;
-    const nodesLength = data.nodesLength();
-    const nodes = new Array(nodesLength);
-    let buffer;
-    const buffersLength = data.buffersLength();
-    const buffers = new Array(buffersLength);
-
-    for (i = 0; i < nodesLength; i += 1) {
-        nodes[i] = data.nodes(i);
-    }
-
-    for (i = 0; i < buffersLength; i += 1) {
-        buffer = data.buffers(i);
-        buffers[i] = {
-            length: buffer.length().low,
-            offset: bb.position() + buffer.offset().low,
-        };
-    }
-    // position the buffer after the body to read the next message
-    bb.setPosition(bb.position() + batch.length);
-
-    return { nodes, buffers, length: data.length().low, type: MessageHeader.RecordBatch };
-}
-
-function _loadDictionaryBatch(bb, batch) {
-    const id = batch.header.id().toFloat64().toString();
-    const data = batch.header.data();
-    let i;
-    const nodesLength = data.nodesLength();
-    const nodes = new Array(nodesLength);
-    let buffer;
-    const buffersLength = data.buffersLength();
-    const buffers = new Array(buffersLength);
-
-    for (i = 0; i < nodesLength; i += 1) {
-        nodes[i] = data.nodes(i);
-    }
-    for (i = 0; i < buffersLength; i += 1) {
-        buffer = data.buffers(i);
-        buffers[i] = {
-            length: buffer.length().low,
-            offset: bb.position() + buffer.offset().low,
-        };
-    }
-    // position the buffer after the body to read the next message
-    bb.setPosition(bb.position() + batch.length);
-
-    return {
-        buffers,
-        id,
-        length: data.length().low,
-        nodes,
-        type: MessageHeader.DictionaryBatch,
-    };
-}
-
-function _loadMessage(bb) {
-    const messageLength: number = Int32FromByteBuffer(bb, bb.position());
-    if (messageLength === 0) {
-      return;
-    }
-    bb.setPosition(bb.position() + 4);
-    const message = Message.getRootAsMessage(bb);
-    // position the buffer at the end of the message so it's ready to read further
-    bb.setPosition(bb.position() + messageLength);
-
-    return message;
-}
-
-function _createDictionaryVectors(field, dictionaries) {
-    const encoding = field.dictionary();
-    if (encoding != null) {
-        const id = encoding.id().toFloat64().toString();
-        if (dictionaries[id] == null) {
-            // create a field for the dictionary
-            const dictionaryField = _createDictionaryField(id, field);
-            dictionaries[id] = vectorFromField(dictionaryField, null);
-        }
-    }
-
-    // recursively examine child fields
-    for (let i = 0, len = field.childrenLength(); i < len; i++) {
-        _createDictionaryVectors(field.children(i), dictionaries);
-    }
-}
-
-function _createDictionaryField(id, field) {
-    const builder = new flatbuffers.Builder();
-    const nameOffset = builder.createString("dict-" + id);
-
-    const typeType = field.typeType();
-    let typeOffset;
-    if (typeType === Type.Int) {
-        const type = field.type(new org.apache.arrow.flatbuf.Int());
-        org.apache.arrow.flatbuf.Int.startInt(builder);
-        org.apache.arrow.flatbuf.Int.addBitWidth(builder, type.bitWidth());
-        org.apache.arrow.flatbuf.Int.addIsSigned(builder, type.isSigned());
-        typeOffset = org.apache.arrow.flatbuf.Int.endInt(builder);
-    } else if (typeType === Type.FloatingPoint) {
-        const type = field.type(new org.apache.arrow.flatbuf.FloatingPoint());
-        org.apache.arrow.flatbuf.FloatingPoint.startFloatingPoint(builder);
-        org.apache.arrow.flatbuf.FloatingPoint.addPrecision(builder, type.precision());
-        typeOffset = org.apache.arrow.flatbuf.FloatingPoint.endFloatingPoint(builder);
-    } else if (typeType === Type.Utf8) {
-        org.apache.arrow.flatbuf.Utf8.startUtf8(builder);
-        typeOffset = org.apache.arrow.flatbuf.Utf8.endUtf8(builder);
-    } else if (typeType === Type.Date) {
-        const type = field.type(new org.apache.arrow.flatbuf.Date());
-        org.apache.arrow.flatbuf.Date.startDate(builder);
-        org.apache.arrow.flatbuf.Date.addUnit(builder, type.unit());
-        typeOffset = org.apache.arrow.flatbuf.Date.endDate(builder);
-    } else {
-        throw new Error("Unimplemented dictionary type " + typeType);
-    }
-    if (field.childrenLength() > 0) {
-      throw new Error("Dictionary encoded fields can't have children");
-    }
-    const childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, []);
-
-    let layout;
-    const layoutOffsets = [];
-    for (let i = 0, len = field.layoutLength(); i < len; i++) {
-        layout = field.layout(i);
-        org.apache.arrow.flatbuf.VectorLayout.startVectorLayout(builder);
-        org.apache.arrow.flatbuf.VectorLayout.addBitWidth(builder, layout.bitWidth());
-        org.apache.arrow.flatbuf.VectorLayout.addType(builder, layout.type());
-        layoutOffsets.push(org.apache.arrow.flatbuf.VectorLayout.endVectorLayout(builder));
-    }
-    const layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, layoutOffsets);
-
-    org.apache.arrow.flatbuf.Field.startField(builder);
-    org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
-    org.apache.arrow.flatbuf.Field.addNullable(builder, field.nullable());
-    org.apache.arrow.flatbuf.Field.addTypeType(builder, typeType);
-    org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
-    org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
-    org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
-    const offset = org.apache.arrow.flatbuf.Field.endField(builder);
-    builder.finish(offset);
-
-    return org.apache.arrow.flatbuf.Field.getRootAsField(builder.bb);
-}
-
-function Int32FromByteBuffer(bb, offset) {
-    return ((bb.bytes_[offset + 3] & 255) << 24) |
-           ((bb.bytes_[offset + 2] & 255) << 16) |
-           ((bb.bytes_[offset + 1] & 255) << 8) |
-           ((bb.bytes_[offset] & 255));
-}
-
-const MAGIC_STR = "ARROW1";
-const MAGIC = new Uint8Array(MAGIC_STR.length);
-for (let i = 0; i < MAGIC_STR.length; i++) {
-    MAGIC[i] = MAGIC_STR.charCodeAt(i);
-}
-
-function _checkMagic(buf, index) {
-    for (let i = 0; i < MAGIC.length; i++) {
-        if (MAGIC[i] !== buf[index + i]) {
-            return false;
-        }
-    }
-    return true;
-}
-
-const TYPEMAP = {};
-TYPEMAP[Type.NONE]          = "NONE";
-TYPEMAP[Type.Null]          = "Null";
-TYPEMAP[Type.Int]           = "Int";
-TYPEMAP[Type.FloatingPoint] = "FloatingPoint";
-TYPEMAP[Type.Binary]        = "Binary";
-TYPEMAP[Type.Utf8]          = "Utf8";
-TYPEMAP[Type.Bool]          = "Bool";
-TYPEMAP[Type.Decimal]       = "Decimal";
-TYPEMAP[Type.Date]          = "Date";
-TYPEMAP[Type.Time]          = "Time";
-TYPEMAP[Type.Timestamp]     = "Timestamp";
-TYPEMAP[Type.Interval]      = "Interval";
-TYPEMAP[Type.List]          = "List";
-TYPEMAP[Type.FixedSizeList] = "FixedSizeList";
-TYPEMAP[Type.Struct_]       = "Struct";
-TYPEMAP[Type.Union]         = "Union";
-
-const VECTORTYPEMAP = {};
-VECTORTYPEMAP[VectorType.OFFSET]   = "OFFSET";
-VECTORTYPEMAP[VectorType.DATA]     = "DATA";
-VECTORTYPEMAP[VectorType.VALIDITY] = "VALIDITY";
-VECTORTYPEMAP[VectorType.TYPE]     = "TYPE";
-
-function parseField(field) {
-    const children = [];
-    for (let i = 0; i < field.childrenLength(); i++) {
-        children.push(parseField(field.children(i)));
-    }
-
-    const layouts = [];
-    for (let i = 0; i < field.layoutLength(); i++) {
-        layouts.push(VECTORTYPEMAP[field.layout(i).type()]);
-    }
-
-    return {
-      children,
-      layout: layouts,
-      name: field.name(),
-      nullable: field.nullable(),
-      type: TYPEMAP[field.typeType()],
-    };
-}
-
-function parseSchema(schema) {
-    const result = [];
-    for (let i = 0, len = schema.fieldsLength(); i < len; i++) {
-        result.push(parseField(schema.fields(i)));
-    }
-    return result;
-}
-
-function loadVectors(bb, vectors: Vector[], recordBatch) {
-    const indices = { bufferIndex: 0, nodeIndex: 0 };
-    for (const vector of vectors) {
-        loadVector(bb, vector, recordBatch, indices);
-    }
-}
-
-/**
- * Loads a vector with data from a batch
- *   recordBatch: { nodes: org.apache.arrow.flatbuf.FieldNode[], buffers: { offset: number, length: number }[] }
- */
-function loadVector(bb, vector: Vector, recordBatch, indices) {
-    const node = recordBatch.nodes[indices.nodeIndex];
-    let ownBuffersLength;
-    const ownBuffers = [];
-    let i;
-    indices.nodeIndex += 1;
-
-    // dictionary vectors are always ints, so will have a data vector plus optional null vector
-    if (vector.field.dictionary() == null) {
-        ownBuffersLength = vector.field.layoutLength();
-    } else if (vector.field.nullable()) {
-        ownBuffersLength = 2;
-    } else {
-        ownBuffersLength = 1;
-    }
-
-    for (i = 0; i < ownBuffersLength; i += 1) {
-        ownBuffers.push(recordBatch.buffers[indices.bufferIndex + i]);
-    }
-    indices.bufferIndex += ownBuffersLength;
-
-    vector.loadData(bb, node, ownBuffers);
-
-    const children = vector.getChildVectors();
-    for (i = 0; i < children.length; i++) {
-        loadVector(bb, children[i], recordBatch, indices);
-    }
-}
diff --git a/js/src/format/File_generated.ts b/js/src/format/File_generated.ts
new file mode 100644
index 0000000000000..d0b774ae34095
--- /dev/null
+++ b/js/src/format/File_generated.ts
@@ -0,0 +1,240 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+import { flatbuffers } from 'flatbuffers';
+import * as NS16187549871986683199 from './Schema_generated';
+/**
+ * ----------------------------------------------------------------------
+ * Arrow File metadata
+ *
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Footer {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Footer}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Footer {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Footer=} obj
+     * @returns {Footer}
+     */
+    static getRootAsFooter(bb: flatbuffers.ByteBuffer, obj?: Footer): Footer {
+      return (obj || new Footer).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.MetadataVersion}
+     */
+    version(): NS16187549871986683199.org.apache.arrow.flatbuf.MetadataVersion {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.MetadataVersion} */ (this.bb.readInt16(this.bb_pos + offset)) : NS16187549871986683199.org.apache.arrow.flatbuf.MetadataVersion.V1;
+    }
+
+    /**
+     * @param {org.apache.arrow.flatbuf.Schema=} obj
+     * @returns {org.apache.arrow.flatbuf.Schema|null}
+     */
+    schema(obj?: NS16187549871986683199.org.apache.arrow.flatbuf.Schema): NS16187549871986683199.org.apache.arrow.flatbuf.Schema | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? (obj || new NS16187549871986683199.org.apache.arrow.flatbuf.Schema).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
+    }
+
+    /**
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.Block=} obj
+     * @returns {org.apache.arrow.flatbuf.Block}
+     */
+    dictionaries(index: number, obj?: org.apache.arrow.flatbuf.Block): org.apache.arrow.flatbuf.Block | null {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? (obj || new org.apache.arrow.flatbuf.Block).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    dictionariesLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.Block=} obj
+     * @returns {org.apache.arrow.flatbuf.Block}
+     */
+    recordBatches(index: number, obj?: org.apache.arrow.flatbuf.Block): org.apache.arrow.flatbuf.Block | null {
+      let offset = this.bb.__offset(this.bb_pos, 10);
+      return offset ? (obj || new org.apache.arrow.flatbuf.Block).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    recordBatchesLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 10);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startFooter(builder: flatbuffers.Builder) {
+      builder.startObject(4);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.MetadataVersion} version
+     */
+    static addVersion(builder: flatbuffers.Builder, version: NS16187549871986683199.org.apache.arrow.flatbuf.MetadataVersion) {
+      builder.addFieldInt16(0, version, NS16187549871986683199.org.apache.arrow.flatbuf.MetadataVersion.V1);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} schemaOffset
+     */
+    static addSchema(builder: flatbuffers.Builder, schemaOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(1, schemaOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} dictionariesOffset
+     */
+    static addDictionaries(builder: flatbuffers.Builder, dictionariesOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(2, dictionariesOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startDictionariesVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(24, numElems, 8);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} recordBatchesOffset
+     */
+    static addRecordBatches(builder: flatbuffers.Builder, recordBatchesOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(3, recordBatchesOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startRecordBatchesVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(24, numElems, 8);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endFooter(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} offset
+     */
+    static finishFooterBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) {
+      builder.finish(offset);
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Block {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Block}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Block {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * Index to the start of the RecordBlock (note this is past the Message header)
+     *
+     * @returns {flatbuffers.Long}
+     */
+    offset(): flatbuffers.Long {
+      return this.bb.readInt64(this.bb_pos);
+    }
+
+    /**
+     * Length of the metadata
+     *
+     * @returns {number}
+     */
+    metaDataLength(): number {
+      return this.bb.readInt32(this.bb_pos + 8);
+    }
+
+    /**
+     * Length of the data (this is aligned so there can be a gap between this and
+     * the metatdata).
+     *
+     * @returns {flatbuffers.Long}
+     */
+    bodyLength(): flatbuffers.Long {
+      return this.bb.readInt64(this.bb_pos + 16);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Long} offset
+     * @param {number} metaDataLength
+     * @param {flatbuffers.Long} bodyLength
+     * @returns {flatbuffers.Offset}
+     */
+    static createBlock(builder: flatbuffers.Builder, offset: flatbuffers.Long, metaDataLength: number, bodyLength: flatbuffers.Long): flatbuffers.Offset {
+      builder.prep(8, 24);
+      builder.writeInt64(bodyLength);
+      builder.pad(4);
+      builder.writeInt32(metaDataLength);
+      builder.writeInt64(offset);
+      return builder.offset();
+    }
+
+  }
+}
diff --git a/js/src/format/Message_generated.ts b/js/src/format/Message_generated.ts
new file mode 100644
index 0000000000000..daa781f9b9290
--- /dev/null
+++ b/js/src/format/Message_generated.ts
@@ -0,0 +1,469 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+import { flatbuffers } from 'flatbuffers';
+import * as NS16187549871986683199 from './Schema_generated';
+export namespace org.apache.arrow.flatbuf {
+  export import Schema = NS16187549871986683199.org.apache.arrow.flatbuf.Schema;
+}
+/**
+ * ----------------------------------------------------------------------
+ * The root Message type
+ * This union enables us to easily send different message types without
+ * redundant storage, and in the future we can easily add new message types.
+ *
+ * Arrow implementations do not need to implement all of the message types,
+ * which may include experimental metadata types. For maximum compatibility,
+ * it is best to send data using RecordBatch
+ *
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum MessageHeader {
+    NONE = 0,
+    Schema = 1,
+    DictionaryBatch = 2,
+    RecordBatch = 3,
+    Tensor = 4
+  }
+}
+
+/**
+ * ----------------------------------------------------------------------
+ * Data structures for describing a table row batch (a collection of
+ * equal-length Arrow arrays)
+ * Metadata about a field at some level of a nested type tree (but not
+ * its children).
+ *
+ * For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
+ * would have {length: 5, null_count: 2} for its List node, and {length: 6,
+ * null_count: 0} for its Int16 node, as separate FieldNode structs
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class FieldNode {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {FieldNode}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): FieldNode {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * The number of value slots in the Arrow array at this level of a nested
+     * tree
+     *
+     * @returns {flatbuffers.Long}
+     */
+    length(): flatbuffers.Long {
+      return this.bb.readInt64(this.bb_pos);
+    }
+
+    /**
+     * The number of observed nulls. Fields with null_count == 0 may choose not
+     * to write their physical validity bitmap out as a materialized buffer,
+     * instead setting the length of the bitmap buffer to 0.
+     *
+     * @returns {flatbuffers.Long}
+     */
+    nullCount(): flatbuffers.Long {
+      return this.bb.readInt64(this.bb_pos + 8);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Long} length
+     * @param {flatbuffers.Long} null_count
+     * @returns {flatbuffers.Offset}
+     */
+    static createFieldNode(builder: flatbuffers.Builder, length: flatbuffers.Long, null_count: flatbuffers.Long): flatbuffers.Offset {
+      builder.prep(8, 16);
+      builder.writeInt64(null_count);
+      builder.writeInt64(length);
+      return builder.offset();
+    }
+
+  }
+}
+/**
+ * A data header describing the shared memory layout of a "record" or "row"
+ * batch. Some systems call this a "row batch" internally and others a "record
+ * batch".
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class RecordBatch {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {RecordBatch}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): RecordBatch {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {RecordBatch=} obj
+     * @returns {RecordBatch}
+     */
+    static getRootAsRecordBatch(bb: flatbuffers.ByteBuffer, obj?: RecordBatch): RecordBatch {
+      return (obj || new RecordBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * number of records / rows. The arrays in the batch should all have this
+     * length
+     *
+     * @returns {flatbuffers.Long}
+     */
+    length(): flatbuffers.Long {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
+    }
+
+    /**
+     * Nodes correspond to the pre-ordered flattened logical schema
+     *
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.FieldNode=} obj
+     * @returns {org.apache.arrow.flatbuf.FieldNode}
+     */
+    nodes(index: number, obj?: org.apache.arrow.flatbuf.FieldNode): org.apache.arrow.flatbuf.FieldNode | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? (obj || new org.apache.arrow.flatbuf.FieldNode).__init(this.bb.__vector(this.bb_pos + offset) + index * 16, this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    nodesLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * Buffers correspond to the pre-ordered flattened buffer tree
+     *
+     * The number of buffers appended to this list depends on the schema. For
+     * example, most primitive arrays will have 2 buffers, 1 for the validity
+     * bitmap and 1 for the values. For struct arrays, there will only be a
+     * single buffer for the validity (nulls) bitmap
+     *
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.Buffer=} obj
+     * @returns {org.apache.arrow.flatbuf.Buffer}
+     */
+    buffers(index: number, obj?: NS16187549871986683199.org.apache.arrow.flatbuf.Buffer): NS16187549871986683199.org.apache.arrow.flatbuf.Buffer | null {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? (obj || new NS16187549871986683199.org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    buffersLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startRecordBatch(builder: flatbuffers.Builder) {
+      builder.startObject(3);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Long} length
+     */
+    static addLength(builder: flatbuffers.Builder, length: flatbuffers.Long) {
+      builder.addFieldInt64(0, length, builder.createLong(0, 0));
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} nodesOffset
+     */
+    static addNodes(builder: flatbuffers.Builder, nodesOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(1, nodesOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startNodesVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(16, numElems, 8);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} buffersOffset
+     */
+    static addBuffers(builder: flatbuffers.Builder, buffersOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(2, buffersOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startBuffersVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(24, numElems, 8);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endRecordBatch(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * ----------------------------------------------------------------------
+ * For sending dictionary encoding information. Any Field can be
+ * dictionary-encoded, but in this case none of its children may be
+ * dictionary-encoded.
+ * There is one vector / column per dictionary
+ *
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class DictionaryBatch {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {DictionaryBatch}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): DictionaryBatch {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {DictionaryBatch=} obj
+     * @returns {DictionaryBatch}
+     */
+    static getRootAsDictionaryBatch(bb: flatbuffers.ByteBuffer, obj?: DictionaryBatch): DictionaryBatch {
+      return (obj || new DictionaryBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {flatbuffers.Long}
+     */
+    id(): flatbuffers.Long {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
+    }
+
+    /**
+     * @param {org.apache.arrow.flatbuf.RecordBatch=} obj
+     * @returns {org.apache.arrow.flatbuf.RecordBatch|null}
+     */
+    data(obj?: org.apache.arrow.flatbuf.RecordBatch): org.apache.arrow.flatbuf.RecordBatch | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startDictionaryBatch(builder: flatbuffers.Builder) {
+      builder.startObject(2);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Long} id
+     */
+    static addId(builder: flatbuffers.Builder, id: flatbuffers.Long) {
+      builder.addFieldInt64(0, id, builder.createLong(0, 0));
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} dataOffset
+     */
+    static addData(builder: flatbuffers.Builder, dataOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(1, dataOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endDictionaryBatch(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Message {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Message}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Message {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Message=} obj
+     * @returns {Message}
+     */
+    static getRootAsMessage(bb: flatbuffers.ByteBuffer, obj?: Message): Message {
+      return (obj || new Message).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.MetadataVersion}
+     */
+    version(): NS16187549871986683199.org.apache.arrow.flatbuf.MetadataVersion {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.MetadataVersion} */ (this.bb.readInt16(this.bb_pos + offset)) : NS16187549871986683199.org.apache.arrow.flatbuf.MetadataVersion.V1;
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.MessageHeader}
+     */
+    headerType(): org.apache.arrow.flatbuf.MessageHeader {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? /** @type {org.apache.arrow.flatbuf.MessageHeader} */ (this.bb.readUint8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.MessageHeader.NONE;
+    }
+
+    /**
+     * @param {flatbuffers.Table} obj
+     * @returns {?flatbuffers.Table}
+     */
+    header<T extends flatbuffers.Table>(obj: T): T | null {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+    }
+
+    /**
+     * @returns {flatbuffers.Long}
+     */
+    bodyLength(): flatbuffers.Long {
+      let offset = this.bb.__offset(this.bb_pos, 10);
+      return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startMessage(builder: flatbuffers.Builder) {
+      builder.startObject(4);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.MetadataVersion} version
+     */
+    static addVersion(builder: flatbuffers.Builder, version: NS16187549871986683199.org.apache.arrow.flatbuf.MetadataVersion) {
+      builder.addFieldInt16(0, version, NS16187549871986683199.org.apache.arrow.flatbuf.MetadataVersion.V1);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.MessageHeader} headerType
+     */
+    static addHeaderType(builder: flatbuffers.Builder, headerType: org.apache.arrow.flatbuf.MessageHeader) {
+      builder.addFieldInt8(1, headerType, org.apache.arrow.flatbuf.MessageHeader.NONE);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} headerOffset
+     */
+    static addHeader(builder: flatbuffers.Builder, headerOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(2, headerOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Long} bodyLength
+     */
+    static addBodyLength(builder: flatbuffers.Builder, bodyLength: flatbuffers.Long) {
+      builder.addFieldInt64(3, bodyLength, builder.createLong(0, 0));
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endMessage(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} offset
+     */
+    static finishMessageBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) {
+      builder.finish(offset);
+    }
+
+  }
+}
diff --git a/js/src/format/Schema_generated.ts b/js/src/format/Schema_generated.ts
new file mode 100644
index 0000000000000..65493b7f685ec
--- /dev/null
+++ b/js/src/format/Schema_generated.ts
@@ -0,0 +1,2254 @@
+/* tslint:disable:class-name */
+// automatically generated by the FlatBuffers compiler, do not modify
+
+import { flatbuffers } from 'flatbuffers';
+/**
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum MetadataVersion {
+    V1 = 0,
+    V2 = 1,
+    V3 = 2
+  }
+}
+
+/**
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum UnionMode {
+    Sparse = 0,
+    Dense = 1
+  }
+}
+
+/**
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum Precision {
+    HALF = 0,
+    SINGLE = 1,
+    DOUBLE = 2
+  }
+}
+
+/**
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum DateUnit {
+    DAY = 0,
+    MILLISECOND = 1
+  }
+}
+
+/**
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum TimeUnit {
+    SECOND = 0,
+    MILLISECOND = 1,
+    MICROSECOND = 2,
+    NANOSECOND = 3
+  }
+}
+
+/**
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum IntervalUnit {
+    YEAR_MONTH = 0,
+    DAY_TIME = 1
+  }
+}
+
+/**
+ * ----------------------------------------------------------------------
+ * Top-level Type value, enabling extensible type-specific metadata. We can
+ * add new logical types to Type without breaking backwards compatibility
+ *
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum Type {
+    NONE = 0,
+    Null = 1,
+    Int = 2,
+    FloatingPoint = 3,
+    Binary = 4,
+    Utf8 = 5,
+    Bool = 6,
+    Decimal = 7,
+    Date = 8,
+    Time = 9,
+    Timestamp = 10,
+    Interval = 11,
+    List = 12,
+    Struct_ = 13,
+    Union = 14,
+    FixedSizeBinary = 15,
+    FixedSizeList = 16,
+    Map = 17
+  }
+}
+
+/**
+ * ----------------------------------------------------------------------
+ * The possible types of a vector
+ *
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum VectorType {
+    /**
+     * used in List type, Dense Union and variable length primitive types (String, Binary)
+     */
+    OFFSET = 0,
+
+    /**
+     * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
+     */
+    DATA = 1,
+
+    /**
+     * Bit vector indicating if each value is null
+     */
+    VALIDITY = 2,
+
+    /**
+     * Type vector used in Union type
+     */
+    TYPE = 3
+  }
+}
+
+/**
+ * ----------------------------------------------------------------------
+ * Endianness of the platform producing the data
+ *
+ * @enum
+ */
+export namespace org.apache.arrow.flatbuf {
+  export enum Endianness {
+    Little = 0,
+    Big = 1
+  }
+}
+
+/**
+ * These are stored in the flatbuffer in the Type union below
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Null {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Null}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Null {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Null=} obj
+     * @returns {Null}
+     */
+    static getRootAsNull(bb: flatbuffers.ByteBuffer, obj?: Null): Null {
+      return (obj || new Null).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startNull(builder: flatbuffers.Builder) {
+      builder.startObject(0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endNull(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * A Struct_ in the flatbuffer metadata is the same as an Arrow Struct
+ * (according to the physical memory layout). We used Struct_ here as
+ * Struct is a reserved word in Flatbuffers
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Struct_ {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Struct_}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Struct_ {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Struct_=} obj
+     * @returns {Struct_}
+     */
+    static getRootAsStruct_(bb: flatbuffers.ByteBuffer, obj?: Struct_): Struct_ {
+      return (obj || new Struct_).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startStruct_(builder: flatbuffers.Builder) {
+      builder.startObject(0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endStruct_(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class List {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {List}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): List {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {List=} obj
+     * @returns {List}
+     */
+    static getRootAsList(bb: flatbuffers.ByteBuffer, obj?: List): List {
+      return (obj || new List).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startList(builder: flatbuffers.Builder) {
+      builder.startObject(0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endList(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class FixedSizeList {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {FixedSizeList}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): FixedSizeList {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {FixedSizeList=} obj
+     * @returns {FixedSizeList}
+     */
+    static getRootAsFixedSizeList(bb: flatbuffers.ByteBuffer, obj?: FixedSizeList): FixedSizeList {
+      return (obj || new FixedSizeList).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * Number of list items per value
+     *
+     * @returns {number}
+     */
+    listSize(): number {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startFixedSizeList(builder: flatbuffers.Builder) {
+      builder.startObject(1);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} listSize
+     */
+    static addListSize(builder: flatbuffers.Builder, listSize: number) {
+      builder.addFieldInt32(0, listSize, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endFixedSizeList(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * A Map is a logical nested type that is represented as
+ *
+ * List<entry: Struct<key: K, value: V>>
+ *
+ * In this layout, the keys and values are each respectively contiguous. We do
+ * not constrain the key and value types, so the application is responsible
+ * for ensuring that the keys are hashable and unique. Whether the keys are sorted
+ * may be set in the metadata for this field
+ *
+ * In a Field with Map type, the Field has a child Struct field, which then
+ * has two children: key type and the second the value type. The names of the
+ * child fields may be respectively "entry", "key", and "value", but this is
+ * not enforced
+ *
+ * Map
+ *   - child[0] entry: Struct
+ *     - child[0] key: K
+ *     - child[1] value: V
+ *
+ * Neither the "entry" field nor the "key" field may be nullable.
+ *
+ * The metadata is structured so that Arrow systems without special handling
+ * for Map can make Map an alias for List. The "layout" attribute for the Map
+ * field must have the same contents as a List.
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Map {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Map}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Map {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Map=} obj
+     * @returns {Map}
+     */
+    static getRootAsMap(bb: flatbuffers.ByteBuffer, obj?: Map): Map {
+      return (obj || new Map).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * Set to true if the keys within each value are sorted
+     *
+     * @returns {boolean}
+     */
+    keysSorted(): boolean {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startMap(builder: flatbuffers.Builder) {
+      builder.startObject(1);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {boolean} keysSorted
+     */
+    static addKeysSorted(builder: flatbuffers.Builder, keysSorted: boolean) {
+      builder.addFieldInt8(0, +keysSorted, +false);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endMap(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * A union is a complex type with children in Field
+ * By default ids in the type vector refer to the offsets in the children
+ * optionally typeIds provides an indirection between the child offset and the type id
+ * for each child typeIds[offset] is the id used in the type vector
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Union {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Union}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Union {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Union=} obj
+     * @returns {Union}
+     */
+    static getRootAsUnion(bb: flatbuffers.ByteBuffer, obj?: Union): Union {
+      return (obj || new Union).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.UnionMode}
+     */
+    mode(): org.apache.arrow.flatbuf.UnionMode {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.UnionMode} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.UnionMode.Sparse;
+    }
+
+    /**
+     * @param {number} index
+     * @returns {number}
+     */
+    typeIds(index: number): number | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? this.bb.readInt32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+    }
+
+    /**
+     * @returns {number}
+     */
+    typeIdsLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @returns {Int32Array}
+     */
+    typeIdsArray(): Int32Array | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? new Int32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startUnion(builder: flatbuffers.Builder) {
+      builder.startObject(2);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.UnionMode} mode
+     */
+    static addMode(builder: flatbuffers.Builder, mode: org.apache.arrow.flatbuf.UnionMode) {
+      builder.addFieldInt16(0, mode, org.apache.arrow.flatbuf.UnionMode.Sparse);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} typeIdsOffset
+     */
+    static addTypeIds(builder: flatbuffers.Builder, typeIdsOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(1, typeIdsOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {Array.<number>} data
+     * @returns {flatbuffers.Offset}
+     */
+    static createTypeIdsVector(builder: flatbuffers.Builder, data: number[] | Uint8Array): flatbuffers.Offset {
+      builder.startVector(4, data.length, 4);
+      for (let i = data.length - 1; i >= 0; i--) {
+        builder.addInt32(data[i]);
+      }
+      return builder.endVector();
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startTypeIdsVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(4, numElems, 4);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endUnion(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Int {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Int}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Int {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Int=} obj
+     * @returns {Int}
+     */
+    static getRootAsInt(bb: flatbuffers.ByteBuffer, obj?: Int): Int {
+      return (obj || new Int).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {number}
+     */
+    bitWidth(): number {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @returns {boolean}
+     */
+    isSigned(): boolean {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startInt(builder: flatbuffers.Builder) {
+      builder.startObject(2);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} bitWidth
+     */
+    static addBitWidth(builder: flatbuffers.Builder, bitWidth: number) {
+      builder.addFieldInt32(0, bitWidth, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {boolean} isSigned
+     */
+    static addIsSigned(builder: flatbuffers.Builder, isSigned: boolean) {
+      builder.addFieldInt8(1, +isSigned, +false);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endInt(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class FloatingPoint {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {FloatingPoint}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): FloatingPoint {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {FloatingPoint=} obj
+     * @returns {FloatingPoint}
+     */
+    static getRootAsFloatingPoint(bb: flatbuffers.ByteBuffer, obj?: FloatingPoint): FloatingPoint {
+      return (obj || new FloatingPoint).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.Precision}
+     */
+    precision(): org.apache.arrow.flatbuf.Precision {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.Precision} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Precision.HALF;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startFloatingPoint(builder: flatbuffers.Builder) {
+      builder.startObject(1);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.Precision} precision
+     */
+    static addPrecision(builder: flatbuffers.Builder, precision: org.apache.arrow.flatbuf.Precision) {
+      builder.addFieldInt16(0, precision, org.apache.arrow.flatbuf.Precision.HALF);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endFloatingPoint(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * Unicode with UTF-8 encoding
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Utf8 {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Utf8}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Utf8 {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Utf8=} obj
+     * @returns {Utf8}
+     */
+    static getRootAsUtf8(bb: flatbuffers.ByteBuffer, obj?: Utf8): Utf8 {
+      return (obj || new Utf8).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startUtf8(builder: flatbuffers.Builder) {
+      builder.startObject(0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endUtf8(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Binary {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Binary}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Binary {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Binary=} obj
+     * @returns {Binary}
+     */
+    static getRootAsBinary(bb: flatbuffers.ByteBuffer, obj?: Binary): Binary {
+      return (obj || new Binary).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startBinary(builder: flatbuffers.Builder) {
+      builder.startObject(0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endBinary(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class FixedSizeBinary {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {FixedSizeBinary}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): FixedSizeBinary {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {FixedSizeBinary=} obj
+     * @returns {FixedSizeBinary}
+     */
+    static getRootAsFixedSizeBinary(bb: flatbuffers.ByteBuffer, obj?: FixedSizeBinary): FixedSizeBinary {
+      return (obj || new FixedSizeBinary).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * Number of bytes per value
+     *
+     * @returns {number}
+     */
+    byteWidth(): number {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startFixedSizeBinary(builder: flatbuffers.Builder) {
+      builder.startObject(1);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} byteWidth
+     */
+    static addByteWidth(builder: flatbuffers.Builder, byteWidth: number) {
+      builder.addFieldInt32(0, byteWidth, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endFixedSizeBinary(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Bool {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Bool}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Bool {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Bool=} obj
+     * @returns {Bool}
+     */
+    static getRootAsBool(bb: flatbuffers.ByteBuffer, obj?: Bool): Bool {
+      return (obj || new Bool).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startBool(builder: flatbuffers.Builder) {
+      builder.startObject(0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endBool(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Decimal {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Decimal}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Decimal {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Decimal=} obj
+     * @returns {Decimal}
+     */
+    static getRootAsDecimal(bb: flatbuffers.ByteBuffer, obj?: Decimal): Decimal {
+      return (obj || new Decimal).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * Total number of decimal digits
+     *
+     * @returns {number}
+     */
+    precision(): number {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * Number of digits after the decimal point "."
+     *
+     * @returns {number}
+     */
+    scale(): number {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startDecimal(builder: flatbuffers.Builder) {
+      builder.startObject(2);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} precision
+     */
+    static addPrecision(builder: flatbuffers.Builder, precision: number) {
+      builder.addFieldInt32(0, precision, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} scale
+     */
+    static addScale(builder: flatbuffers.Builder, scale: number) {
+      builder.addFieldInt32(1, scale, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endDecimal(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
+ * epoch (1970-01-01), stored in either of two units:
+ *
+ * * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
+ *   leap seconds), where the values are evenly divisible by 86400000
+ * * Days (32 bits) since the UNIX epoch
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Date {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Date}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Date {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Date=} obj
+     * @returns {Date}
+     */
+    static getRootAsDate(bb: flatbuffers.ByteBuffer, obj?: Date): Date {
+      return (obj || new Date).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.DateUnit}
+     */
+    unit(): org.apache.arrow.flatbuf.DateUnit {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.DateUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.DateUnit.MILLISECOND;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startDate(builder: flatbuffers.Builder) {
+      builder.startObject(1);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.DateUnit} unit
+     */
+    static addUnit(builder: flatbuffers.Builder, unit: org.apache.arrow.flatbuf.DateUnit) {
+      builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.DateUnit.MILLISECOND);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endDate(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * Time type. The physical storage type depends on the unit
+ * - SECOND and MILLISECOND: 32 bits
+ * - MICROSECOND and NANOSECOND: 64 bits
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Time {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Time}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Time {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Time=} obj
+     * @returns {Time}
+     */
+    static getRootAsTime(bb: flatbuffers.ByteBuffer, obj?: Time): Time {
+      return (obj || new Time).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.TimeUnit}
+     */
+    unit(): org.apache.arrow.flatbuf.TimeUnit {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.TimeUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.MILLISECOND;
+    }
+
+    /**
+     * @returns {number}
+     */
+    bitWidth(): number {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? this.bb.readInt32(this.bb_pos + offset) : 32;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startTime(builder: flatbuffers.Builder) {
+      builder.startObject(2);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.TimeUnit} unit
+     */
+    static addUnit(builder: flatbuffers.Builder, unit: org.apache.arrow.flatbuf.TimeUnit) {
+      builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.MILLISECOND);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} bitWidth
+     */
+    static addBitWidth(builder: flatbuffers.Builder, bitWidth: number) {
+      builder.addFieldInt32(1, bitWidth, 32);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endTime(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding
+ * leap seconds, as a 64-bit integer. Note that UNIX time does not include
+ * leap seconds.
+ *
+ * The Timestamp metadata supports both "time zone naive" and "time zone
+ * aware" timestamps. Read about the timezone attribute for more detail
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Timestamp {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Timestamp}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Timestamp {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Timestamp=} obj
+     * @returns {Timestamp}
+     */
+    static getRootAsTimestamp(bb: flatbuffers.ByteBuffer, obj?: Timestamp): Timestamp {
+      return (obj || new Timestamp).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.TimeUnit}
+     */
+    unit(): org.apache.arrow.flatbuf.TimeUnit {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.TimeUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.SECOND;
+    }
+
+    /**
+     * The time zone is a string indicating the name of a time zone, one of:
+     *
+     * * As used in the Olson time zone database (the "tz database" or
+     *   "tzdata"), such as "America/New_York"
+     * * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
+     *
+     * Whether a timezone string is present indicates different semantics about
+     * the data:
+     *
+     * * If the time zone is null or equal to an empty string, the data is "time
+     *   zone naive" and shall be displayed *as is* to the user, not localized
+     *   to the locale of the user. This data can be though of as UTC but
+     *   without having "UTC" as the time zone, it is not considered to be
+     *   localized to any time zone
+     *
+     * * If the time zone is set to a valid value, values can be displayed as
+     *   "localized" to that time zone, even though the underlying 64-bit
+     *   integers are identical to the same data stored in UTC. Converting
+     *   between time zones is a metadata-only operation and does not change the
+     *   underlying values
+     *
+     * @param {flatbuffers.Encoding=} optionalEncoding
+     * @returns {string|Uint8Array|null}
+     */
+    timezone(): string | null;
+    timezone(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null;
+    timezone(optionalEncoding?: any): string | Uint8Array | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startTimestamp(builder: flatbuffers.Builder) {
+      builder.startObject(2);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.TimeUnit} unit
+     */
+    static addUnit(builder: flatbuffers.Builder, unit: org.apache.arrow.flatbuf.TimeUnit) {
+      builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.SECOND);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} timezoneOffset
+     */
+    static addTimezone(builder: flatbuffers.Builder, timezoneOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(1, timezoneOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endTimestamp(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Interval {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Interval}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Interval {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Interval=} obj
+     * @returns {Interval}
+     */
+    static getRootAsInterval(bb: flatbuffers.ByteBuffer, obj?: Interval): Interval {
+      return (obj || new Interval).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.IntervalUnit}
+     */
+    unit(): org.apache.arrow.flatbuf.IntervalUnit {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.IntervalUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startInterval(builder: flatbuffers.Builder) {
+      builder.startObject(1);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.IntervalUnit} unit
+     */
+    static addUnit(builder: flatbuffers.Builder, unit: org.apache.arrow.flatbuf.IntervalUnit) {
+      builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endInterval(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * ----------------------------------------------------------------------
+ * represents the physical layout of a buffer
+ * buffers have fixed width slots of a given type
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class VectorLayout {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {VectorLayout}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): VectorLayout {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {VectorLayout=} obj
+     * @returns {VectorLayout}
+     */
+    static getRootAsVectorLayout(bb: flatbuffers.ByteBuffer, obj?: VectorLayout): VectorLayout {
+      return (obj || new VectorLayout).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * the width of a slot in the buffer (typically 1, 8, 16, 32 or 64)
+     *
+     * @returns {number}
+     */
+    bitWidth(): number {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.readInt16(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * the purpose of the vector
+     *
+     * @returns {org.apache.arrow.flatbuf.VectorType}
+     */
+    type(): org.apache.arrow.flatbuf.VectorType {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? /** @type {org.apache.arrow.flatbuf.VectorType} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.VectorType.OFFSET;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startVectorLayout(builder: flatbuffers.Builder) {
+      builder.startObject(2);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} bitWidth
+     */
+    static addBitWidth(builder: flatbuffers.Builder, bitWidth: number) {
+      builder.addFieldInt16(0, bitWidth, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.VectorType} type
+     */
+    static addType(builder: flatbuffers.Builder, type: org.apache.arrow.flatbuf.VectorType) {
+      builder.addFieldInt16(1, type, org.apache.arrow.flatbuf.VectorType.OFFSET);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endVectorLayout(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * ----------------------------------------------------------------------
+ * user defined key value pairs to add custom metadata to arrow
+ * key namespacing is the responsibility of the user
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class KeyValue {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {KeyValue}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): KeyValue {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {KeyValue=} obj
+     * @returns {KeyValue}
+     */
+    static getRootAsKeyValue(bb: flatbuffers.ByteBuffer, obj?: KeyValue): KeyValue {
+      return (obj || new KeyValue).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @param {flatbuffers.Encoding=} optionalEncoding
+     * @returns {string|Uint8Array|null}
+     */
+    key(): string | null;
+    key(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null;
+    key(optionalEncoding?: any): string | Uint8Array | null {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
+    }
+
+    /**
+     * @param {flatbuffers.Encoding=} optionalEncoding
+     * @returns {string|Uint8Array|null}
+     */
+    value(): string | null;
+    value(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null;
+    value(optionalEncoding?: any): string | Uint8Array | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startKeyValue(builder: flatbuffers.Builder) {
+      builder.startObject(2);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} keyOffset
+     */
+    static addKey(builder: flatbuffers.Builder, keyOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(0, keyOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} valueOffset
+     */
+    static addValue(builder: flatbuffers.Builder, valueOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(1, valueOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endKeyValue(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * ----------------------------------------------------------------------
+ * Dictionary encoding metadata
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class DictionaryEncoding {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {DictionaryEncoding}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): DictionaryEncoding {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {DictionaryEncoding=} obj
+     * @returns {DictionaryEncoding}
+     */
+    static getRootAsDictionaryEncoding(bb: flatbuffers.ByteBuffer, obj?: DictionaryEncoding): DictionaryEncoding {
+      return (obj || new DictionaryEncoding).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * The known dictionary id in the application where this data is used. In
+     * the file or streaming formats, the dictionary ids are found in the
+     * DictionaryBatch messages
+     *
+     * @returns {flatbuffers.Long}
+     */
+    id(): flatbuffers.Long {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
+    }
+
+    /**
+     * The dictionary indices are constrained to be positive integers. If this
+     * field is null, the indices must be signed int32
+     *
+     * @param {org.apache.arrow.flatbuf.Int=} obj
+     * @returns {org.apache.arrow.flatbuf.Int|null}
+     */
+    indexType(obj?: org.apache.arrow.flatbuf.Int): org.apache.arrow.flatbuf.Int | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? (obj || new org.apache.arrow.flatbuf.Int).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
+    }
+
+    /**
+     * By default, dictionaries are not ordered, or the order does not have
+     * semantic meaning. In some statistical, applications, dictionary-encoding
+     * is used to represent ordered categorical data, and we provide a way to
+     * preserve that metadata here
+     *
+     * @returns {boolean}
+     */
+    isOrdered(): boolean {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startDictionaryEncoding(builder: flatbuffers.Builder) {
+      builder.startObject(3);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Long} id
+     */
+    static addId(builder: flatbuffers.Builder, id: flatbuffers.Long) {
+      builder.addFieldInt64(0, id, builder.createLong(0, 0));
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} indexTypeOffset
+     */
+    static addIndexType(builder: flatbuffers.Builder, indexTypeOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(1, indexTypeOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {boolean} isOrdered
+     */
+    static addIsOrdered(builder: flatbuffers.Builder, isOrdered: boolean) {
+      builder.addFieldInt8(2, +isOrdered, +false);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endDictionaryEncoding(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * ----------------------------------------------------------------------
+ * A field represents a named column in a record / row batch or child of a
+ * nested type.
+ *
+ * - children is only for nested Arrow arrays
+ * - For primitive types, children will have length 0
+ * - nullable should default to true in general
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Field {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Field}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Field {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Field=} obj
+     * @returns {Field}
+     */
+    static getRootAsField(bb: flatbuffers.ByteBuffer, obj?: Field): Field {
+      return (obj || new Field).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * @param {flatbuffers.Encoding=} optionalEncoding
+     * @returns {string|Uint8Array|null}
+     */
+    name(): string | null;
+    name(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null;
+    name(optionalEncoding?: any): string | Uint8Array | null {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
+    }
+
+    /**
+     * @returns {boolean}
+     */
+    nullable(): boolean {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+    }
+
+    /**
+     * @returns {org.apache.arrow.flatbuf.Type}
+     */
+    typeType(): org.apache.arrow.flatbuf.Type {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? /** @type {org.apache.arrow.flatbuf.Type} */ (this.bb.readUint8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Type.NONE;
+    }
+
+    /**
+     * @param {flatbuffers.Table} obj
+     * @returns {?flatbuffers.Table}
+     */
+    type<T extends flatbuffers.Table>(obj: T): T | null {
+      let offset = this.bb.__offset(this.bb_pos, 10);
+      return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+    }
+
+    /**
+     * @param {org.apache.arrow.flatbuf.DictionaryEncoding=} obj
+     * @returns {org.apache.arrow.flatbuf.DictionaryEncoding|null}
+     */
+    dictionary(obj?: org.apache.arrow.flatbuf.DictionaryEncoding): org.apache.arrow.flatbuf.DictionaryEncoding | null {
+      let offset = this.bb.__offset(this.bb_pos, 12);
+      return offset ? (obj || new org.apache.arrow.flatbuf.DictionaryEncoding).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
+    }
+
+    /**
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.Field=} obj
+     * @returns {org.apache.arrow.flatbuf.Field}
+     */
+    children(index: number, obj?: org.apache.arrow.flatbuf.Field): org.apache.arrow.flatbuf.Field | null {
+      let offset = this.bb.__offset(this.bb_pos, 14);
+      return offset ? (obj || new org.apache.arrow.flatbuf.Field).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    childrenLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 14);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * layout of buffers produced for this type (as derived from the Type)
+     * does not include children
+     * each recordbatch will return instances of those Buffers.
+     *
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.VectorLayout=} obj
+     * @returns {org.apache.arrow.flatbuf.VectorLayout}
+     */
+    layout(index: number, obj?: org.apache.arrow.flatbuf.VectorLayout): org.apache.arrow.flatbuf.VectorLayout | null {
+      let offset = this.bb.__offset(this.bb_pos, 16);
+      return offset ? (obj || new org.apache.arrow.flatbuf.VectorLayout).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    layoutLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 16);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.KeyValue=} obj
+     * @returns {org.apache.arrow.flatbuf.KeyValue}
+     */
+    customMetadata(index: number, obj?: org.apache.arrow.flatbuf.KeyValue): org.apache.arrow.flatbuf.KeyValue | null {
+      let offset = this.bb.__offset(this.bb_pos, 18);
+      return offset ? (obj || new org.apache.arrow.flatbuf.KeyValue).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    customMetadataLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 18);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startField(builder: flatbuffers.Builder) {
+      builder.startObject(8);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} nameOffset
+     */
+    static addName(builder: flatbuffers.Builder, nameOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(0, nameOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {boolean} nullable
+     */
+    static addNullable(builder: flatbuffers.Builder, nullable: boolean) {
+      builder.addFieldInt8(1, +nullable, +false);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.Type} typeType
+     */
+    static addTypeType(builder: flatbuffers.Builder, typeType: org.apache.arrow.flatbuf.Type) {
+      builder.addFieldInt8(2, typeType, org.apache.arrow.flatbuf.Type.NONE);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} typeOffset
+     */
+    static addType(builder: flatbuffers.Builder, typeOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(3, typeOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} dictionaryOffset
+     */
+    static addDictionary(builder: flatbuffers.Builder, dictionaryOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(4, dictionaryOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} childrenOffset
+     */
+    static addChildren(builder: flatbuffers.Builder, childrenOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(5, childrenOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {Array.<flatbuffers.Offset>} data
+     * @returns {flatbuffers.Offset}
+     */
+    static createChildrenVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset {
+      builder.startVector(4, data.length, 4);
+      for (let i = data.length - 1; i >= 0; i--) {
+        builder.addOffset(data[i]);
+      }
+      return builder.endVector();
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startChildrenVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(4, numElems, 4);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} layoutOffset
+     */
+    static addLayout(builder: flatbuffers.Builder, layoutOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(6, layoutOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {Array.<flatbuffers.Offset>} data
+     * @returns {flatbuffers.Offset}
+     */
+    static createLayoutVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset {
+      builder.startVector(4, data.length, 4);
+      for (let i = data.length - 1; i >= 0; i--) {
+        builder.addOffset(data[i]);
+      }
+      return builder.endVector();
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startLayoutVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(4, numElems, 4);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} customMetadataOffset
+     */
+    static addCustomMetadata(builder: flatbuffers.Builder, customMetadataOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(7, customMetadataOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {Array.<flatbuffers.Offset>} data
+     * @returns {flatbuffers.Offset}
+     */
+    static createCustomMetadataVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset {
+      builder.startVector(4, data.length, 4);
+      for (let i = data.length - 1; i >= 0; i--) {
+        builder.addOffset(data[i]);
+      }
+      return builder.endVector();
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startCustomMetadataVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(4, numElems, 4);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endField(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+  }
+}
+/**
+ * ----------------------------------------------------------------------
+ * A Buffer represents a single contiguous memory segment
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Buffer {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Buffer}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Buffer {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * The shared memory page id where this buffer is located. Currently this is
+     * not used
+     *
+     * @returns {number}
+     */
+    page(): number {
+      return this.bb.readInt32(this.bb_pos);
+    }
+
+    /**
+     * The relative offset into the shared memory page where the bytes for this
+     * buffer starts
+     *
+     * @returns {flatbuffers.Long}
+     */
+    offset(): flatbuffers.Long {
+      return this.bb.readInt64(this.bb_pos + 8);
+    }
+
+    /**
+     * The absolute length (in bytes) of the memory buffer. The memory is found
+     * from offset (inclusive) to offset + length (non-inclusive).
+     *
+     * @returns {flatbuffers.Long}
+     */
+    length(): flatbuffers.Long {
+      return this.bb.readInt64(this.bb_pos + 16);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} page
+     * @param {flatbuffers.Long} offset
+     * @param {flatbuffers.Long} length
+     * @returns {flatbuffers.Offset}
+     */
+    static createBuffer(builder: flatbuffers.Builder, page: number, offset: flatbuffers.Long, length: flatbuffers.Long): flatbuffers.Offset {
+      builder.prep(8, 24);
+      builder.writeInt64(length);
+      builder.writeInt64(offset);
+      builder.pad(4);
+      builder.writeInt32(page);
+      return builder.offset();
+    }
+
+  }
+}
+/**
+ * ----------------------------------------------------------------------
+ * A Schema describes the columns in a row batch
+ *
+ * @constructor
+ */
+export namespace org.apache.arrow.flatbuf {
+  export class Schema {
+    /**
+     * @type {flatbuffers.ByteBuffer}
+     */
+    bb: flatbuffers.ByteBuffer;
+
+    /**
+     * @type {number}
+     */
+    bb_pos: number = 0;
+    /**
+     * @param {number} i
+     * @param {flatbuffers.ByteBuffer} bb
+     * @returns {Schema}
+     */
+    __init(i: number, bb: flatbuffers.ByteBuffer): Schema {
+      this.bb_pos = i;
+      this.bb = bb;
+      return this;
+    }
+
+    /**
+     * @param {flatbuffers.ByteBuffer} bb
+     * @param {Schema=} obj
+     * @returns {Schema}
+     */
+    static getRootAsSchema(bb: flatbuffers.ByteBuffer, obj?: Schema): Schema {
+      return (obj || new Schema).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+    }
+
+    /**
+     * endianness of the buffer
+     * it is Little Endian by default
+     * if endianness doesn't match the underlying system then the vectors need to be converted
+     *
+     * @returns {org.apache.arrow.flatbuf.Endianness}
+     */
+    endianness(): org.apache.arrow.flatbuf.Endianness {
+      let offset = this.bb.__offset(this.bb_pos, 4);
+      return offset ? /** @type {org.apache.arrow.flatbuf.Endianness} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Endianness.Little;
+    }
+
+    /**
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.Field=} obj
+     * @returns {org.apache.arrow.flatbuf.Field}
+     */
+    fields(index: number, obj?: org.apache.arrow.flatbuf.Field): org.apache.arrow.flatbuf.Field | null {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? (obj || new org.apache.arrow.flatbuf.Field).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    fieldsLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 6);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {number} index
+     * @param {org.apache.arrow.flatbuf.KeyValue=} obj
+     * @returns {org.apache.arrow.flatbuf.KeyValue}
+     */
+    customMetadata(index: number, obj?: org.apache.arrow.flatbuf.KeyValue): org.apache.arrow.flatbuf.KeyValue | null {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? (obj || new org.apache.arrow.flatbuf.KeyValue).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+    }
+
+    /**
+     * @returns {number}
+     */
+    customMetadataLength(): number {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     */
+    static startSchema(builder: flatbuffers.Builder) {
+      builder.startObject(3);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {org.apache.arrow.flatbuf.Endianness} endianness
+     */
+    static addEndianness(builder: flatbuffers.Builder, endianness: org.apache.arrow.flatbuf.Endianness) {
+      builder.addFieldInt16(0, endianness, org.apache.arrow.flatbuf.Endianness.Little);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} fieldsOffset
+     */
+    static addFields(builder: flatbuffers.Builder, fieldsOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(1, fieldsOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {Array.<flatbuffers.Offset>} data
+     * @returns {flatbuffers.Offset}
+     */
+    static createFieldsVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset {
+      builder.startVector(4, data.length, 4);
+      for (let i = data.length - 1; i >= 0; i--) {
+        builder.addOffset(data[i]);
+      }
+      return builder.endVector();
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startFieldsVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(4, numElems, 4);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} customMetadataOffset
+     */
+    static addCustomMetadata(builder: flatbuffers.Builder, customMetadataOffset: flatbuffers.Offset) {
+      builder.addFieldOffset(2, customMetadataOffset, 0);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {Array.<flatbuffers.Offset>} data
+     * @returns {flatbuffers.Offset}
+     */
+    static createCustomMetadataVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset {
+      builder.startVector(4, data.length, 4);
+      for (let i = data.length - 1; i >= 0; i--) {
+        builder.addOffset(data[i]);
+      }
+      return builder.endVector();
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {number} numElems
+     */
+    static startCustomMetadataVector(builder: flatbuffers.Builder, numElems: number) {
+      builder.startVector(4, numElems, 4);
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @returns {flatbuffers.Offset}
+     */
+    static endSchema(builder: flatbuffers.Builder): flatbuffers.Offset {
+      let offset = builder.endObject();
+      return offset;
+    }
+
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {flatbuffers.Offset} offset
+     */
+    static finishSchemaBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) {
+      builder.finish(offset);
+    }
+
+  }
+}
diff --git a/js/src/reader/arrow.ts b/js/src/reader/arrow.ts
new file mode 100644
index 0000000000000..9716c7fb60150
--- /dev/null
+++ b/js/src/reader/arrow.ts
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import * as Schema_ from '../format/Schema_generated';
+import * as Message_ from '../format/Message_generated';
+
+import { readFile } from './file';
+import { readStream } from './stream';
+import { readVector } from './vector';
+import { Vector } from '../vector/vector';
+import { readDictionaries } from './dictionary';
+
+import ByteBuffer = flatbuffers.ByteBuffer;
+export import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
+export type Dictionaries = { [k: string]: Vector<any> };
+export type IteratorState = { nodeIndex: number; bufferIndex: number };
+
+export function* readRecords(...bytes: ByteBuffer[]) {
+    try {
+        yield* readFile(...bytes);
+    } catch (e) {
+        try {
+            yield* readStream(...bytes);
+        } catch (e) {
+            throw new Error('Invalid Arrow buffer');
+        }
+    }
+}
+
+export function* readBuffers(...bytes: Array<Uint8Array | Buffer | string>) {
+    const dictionaries: Dictionaries = {};
+    const byteBuffers = bytes.map(toByteBuffer);
+    for (let { schema, batch } of readRecords(...byteBuffers)) {
+        let vectors: Vector<any>[] = [];
+        let state = { nodeIndex: 0, bufferIndex: 0 };
+        let index = -1, fieldsLength = schema.fieldsLength();
+        if (batch.id) {
+            while (++index < fieldsLength) {
+                for (let [id, vector] of readDictionaries(schema.fields(index), batch, state, dictionaries)) {
+                    dictionaries[id] = dictionaries[id] && dictionaries[id].concat(vector) || vector;
+                }
+            }
+        } else {
+            while (++index < fieldsLength) {
+                vectors[index] = readVector(schema.fields(index), batch, state, dictionaries);
+            }
+            yield vectors;
+        }
+    }
+}
+
+function toByteBuffer(bytes?: Uint8Array | Buffer | string) {
+    let arr: Uint8Array = bytes as any || new Uint8Array(0);
+    if (typeof bytes === 'string') {
+        arr = new Uint8Array(bytes.length);
+        for (let i = -1, n = bytes.length; ++i < n;) {
+            arr[i] = bytes.charCodeAt(i);
+        }
+        return new ByteBuffer(arr);
+    }
+    return new ByteBuffer(arr);
+}
diff --git a/js/src/reader/dictionary.ts b/js/src/reader/dictionary.ts
new file mode 100644
index 0000000000000..aef2bc9d1cdbf
--- /dev/null
+++ b/js/src/reader/dictionary.ts
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { readVector } from './vector';
+import { MessageBatch } from './message';
+import * as Schema_ from '../format/Schema_generated';
+import { IteratorState, Dictionaries } from './arrow';
+
+import Field = Schema_.org.apache.arrow.flatbuf.Field;
+import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
+
+export function* readDictionaries(field: Field,
+                                  batch: MessageBatch,
+                                  iterator: IteratorState,
+                                  dictionaries: Dictionaries) {
+    let id: string, encoding: DictionaryEncoding;
+    if ((encoding = field.dictionary()) &&
+        batch.id === (id = encoding.id().toFloat64().toString())) {
+        yield [id, readVector(field, batch, iterator, null)];
+        return;
+    }
+    for (let i = -1, n = field.childrenLength(); ++i < n;) {
+        // Since a dictionary batch can only contain a single vector, return early after we find it
+        for (let result of readDictionaries(field.children(i), batch, iterator, dictionaries)) {
+            yield result;
+            return;
+        }
+    }
+}
diff --git a/js/src/reader/file.ts b/js/src/reader/file.ts
new file mode 100644
index 0000000000000..b05b99a5e6dcf
--- /dev/null
+++ b/js/src/reader/file.ts
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import * as File_ from '../format/File_generated';
+import * as Schema_ from '../format/Schema_generated';
+import * as Message_ from '../format/Message_generated';
+import { PADDING, readMessageBatches } from './message';
+
+import ByteBuffer = flatbuffers.ByteBuffer;
+import Footer = File_.org.apache.arrow.flatbuf.Footer;
+export import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
+
+const MAGIC_STR = 'ARROW1';
+const MAGIC = new Uint8Array(MAGIC_STR.length);
+for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
+    MAGIC[i] = MAGIC_STR.charCodeAt(i);
+}
+
+export function _checkMagic(buffer: Uint8Array, index = 0) {
+    for (let i = -1, n = MAGIC.length; ++i < n;) {
+        if (MAGIC[i] !== buffer[index + i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+const magicLength = MAGIC.length;
+const magicAndPadding = magicLength + PADDING;
+const magicX2AndPadding = magicLength * 2 + PADDING;
+
+export function* readFile(...bbs: ByteBuffer[]) {
+    for (let bb of bbs) {
+        let fileLength = bb.capacity();
+        let footerLength: number, footerOffset: number;
+        if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
+            (!_checkMagic(bb.bytes(), 0) /*                        Missing magic start    */) ||
+            (!_checkMagic(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
+            (/*                                                    Invalid footer length  */
+            (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
+            (footerLength + magicX2AndPadding > fileLength))) {
+            throw new Error('Invalid file');
+        }
+        bb.setPosition(footerOffset - footerLength);
+        let footer = Footer.getRootAsFooter(bb), schema = footer.schema();
+        for (let i = -1, n = footer.dictionariesLength(); ++i < n;) {
+            let block = footer.dictionaries(i);
+            bb.setPosition(block.offset().low);
+            for (let batch of readMessageBatches(bb)) {
+                yield { schema, batch };
+                break;
+            }
+        }
+        for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) {
+            const block = footer.recordBatches(i);
+            bb.setPosition(block.offset().low);
+            for (let batch of readMessageBatches(bb)) {
+                yield { schema, batch };
+                break;
+            }
+        }
+    }
+}
diff --git a/js/src/reader/message.ts b/js/src/reader/message.ts
new file mode 100644
index 0000000000000..5472f10833878
--- /dev/null
+++ b/js/src/reader/message.ts
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import * as Message_ from '../format/Message_generated';
+import ByteBuffer = flatbuffers.ByteBuffer;
+import Message = Message_.org.apache.arrow.flatbuf.Message;
+import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
+import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
+import DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch;
+
+export const PADDING = 4;
+export type MessageBatch = {
+    id?: string;
+    offset: number;
+    bytes: Uint8Array;
+    data: RecordBatch;
+};
+
+export function* readMessages(bb: ByteBuffer) {
+    let message, length;
+    while (bb.position() < bb.capacity() &&
+          (length = bb.readInt32(bb.position())) > 0) {
+        bb.setPosition(bb.position() + PADDING);
+        message = Message.getRootAsMessage(bb);
+        bb.setPosition(bb.position() + length);
+        yield message;
+    }
+}
+
+export function* readMessageBatches(bb: ByteBuffer) {
+    let bytes = bb.bytes();
+    for (let message of readMessages(bb)) {
+        let type = message.headerType();
+        let id: string, data: RecordBatch;
+        if (type === MessageHeader.RecordBatch) {
+            data = message.header(new RecordBatch());
+        } else if (type === MessageHeader.DictionaryBatch) {
+            let header = message.header(new DictionaryBatch());
+            id = header.id().toFloat64().toString();
+            data = header.data();
+        } else {
+            continue;
+        }
+        yield <MessageBatch> { id, data, bytes, offset: bytes.byteOffset + bb.position() };
+        // position the buffer after the body to read the next message
+        bb.setPosition(bb.position() + message.bodyLength().low);
+    }
+}
diff --git a/js/src/reader/stream.ts b/js/src/reader/stream.ts
new file mode 100644
index 0000000000000..9869f633d08f4
--- /dev/null
+++ b/js/src/reader/stream.ts
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { _checkMagic } from './file';
+import { flatbuffers } from 'flatbuffers';
+import * as Schema_ from '../format/Schema_generated';
+import * as Message_ from '../format/Message_generated';
+import { readMessages, readMessageBatches } from './message';
+
+import ByteBuffer = flatbuffers.ByteBuffer;
+import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
+
+export function* readStream(...bbs: ByteBuffer[]) {
+    if (!bbs.length || _checkMagic(bbs[0].bytes(), 0)) {
+        throw new Error('Invalid Arrow Stream');
+    }
+    for (const message of readMessages(bbs[0])) {
+        if (message.headerType() === MessageHeader.Schema) {
+            const schema = message.header(new Schema());
+            for (const bb of bbs) {
+                for (const batch of readMessageBatches(bb)) {
+                    yield { schema, batch };
+                }
+            }
+            break;
+        }
+    }
+}
diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts
new file mode 100644
index 0000000000000..a3cd79803641f
--- /dev/null
+++ b/js/src/reader/vector.ts
@@ -0,0 +1,271 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import { MessageBatch } from './message';
+import { Vector } from '../vector/vector';
+import * as Schema_ from '../format/Schema_generated';
+import { StructVector } from '../vector/struct';
+import { IteratorState, Dictionaries } from './arrow';
+import { DictionaryVector } from '../vector/dictionary';
+import { Utf8Vector, ListVector, FixedSizeListVector } from '../vector/list';
+import {
+    TypedArray, TypedArrayCtor, IntArray, FloatArray,
+    Int8Vector, Int16Vector, Int32Vector, Int64Vector,
+    Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector,
+    Float32Vector, Float64Vector, IndexVector, DateVector,
+} from '../vector/typed';
+
+import Int = Schema_.org.apache.arrow.flatbuf.Int;
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+import Field = Schema_.org.apache.arrow.flatbuf.Field;
+import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
+import VectorType = Schema_.org.apache.arrow.flatbuf.VectorType;
+import VectorLayout = Schema_.org.apache.arrow.flatbuf.VectorLayout;
+import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
+import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
+import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
+
+export function readVector(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries) {
+    return readDictionaryVector(field, batch, state, dictionaries) ||
+                readTypedVector(field, batch, state, dictionaries);
+}
+
+function readTypedVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) {
+    let typeType = field.typeType(), readTyped = typedVectorReaders[typeType];
+    if (!readTyped) {
+        throw new Error('Unrecognized vector name "' + Type[typeType] + '" type "' + typeType + '"');
+    }
+    return readTyped(field, batch, iterator, dictionaries);
+}
+
+function readDictionaryVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) {
+    let encoding: DictionaryEncoding;
+    if (dictionaries && (encoding = field.dictionary())) {
+        let id = encoding.id().toFloat64().toString();
+        let fieldType =  encoding.indexType() ||
+            /* a dictionary index defaults to signed 32 bit int if unspecified */
+            { bitWidth: () => 32, isSigned: () => true };
+        let indexField = createSyntheticDictionaryIndexField(field, fieldType);
+        let index = readIntVector(indexField, batch, iterator, null, fieldType);
+        return DictionaryVector.create(field, index.length, index, dictionaries[id]);
+    }
+}
+
+const IntViews    = [Int8Array,    Int16Array,   Int32Array,   Int32Array  ];
+const Int32Views  = [Int32Array,   Int32Array,   Int32Array,   Int32Array  ];
+const UintViews   = [Uint8Array,   Uint16Array,  Uint32Array,  Uint32Array ];
+const Uint8Views  = [Uint8Array,   Uint8Array,   Uint8Array,   Uint8Array  ];
+const Uint32Views = [Uint32Array,  Uint32Array,  Uint32Array,  Uint32Array ];
+const FloatViews  = [Int8Array,    Int16Array,   Float32Array, Float64Array];
+
+const createIntDataViews = createDataView.bind(null, IntViews, null);
+const createUintDataViews = createDataView.bind(null, UintViews, null);
+const createDateDataViews = createDataView.bind(null, Uint32Views, null);
+const createFloatDataViews = createDataView.bind(null, FloatViews, null);
+const createNestedDataViews = createDataView.bind(null, Uint32Views, null);
+const createValidityDataViews = createDataView.bind(null, Uint8Views, null);
+const createUtf8DataViews = createDataView.bind(null, Uint8Views, Int32Views);
+
+const floatVectors = {
+    [Precision.SINGLE]: Float32Vector,
+    [Precision.DOUBLE]: Float64Vector
+};
+const intVectors = [
+    [/* unsigned */ Uint8Vector,   /* signed */ Int8Vector ],
+    [/* unsigned */ Uint16Vector,  /* signed */ Int16Vector],
+    [/* unsigned */ Uint32Vector,  /* signed */ Int32Vector],
+    [/* unsigned */ Uint64Vector,  /* signed */ Int64Vector]
+];
+
+function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, fieldType?: FieldType) {
+    let type = (fieldType || field.type(new Int()));
+    return type.isSigned() ?
+        read_IntVector(field, batch, iterator, dictionaries, type) :
+        readUintVector(field, batch, iterator, dictionaries, type);
+}
+
+const read_IntVector = readVectorLayout<number, IntArray>(createIntDataViews, createIntVector);
+const readUintVector = readVectorLayout<number, IntArray>(createUintDataViews, createIntVector);
+function createIntVector(field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) {
+    let type = fieldType || field.type(new Int()), bitWidth = type.bitWidth();
+    let Vector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()];
+    return Vector.create(field, length, validity, data || offsets);
+    // ---------------------- so this is kinda strange 👆:
+    // The dictionary encoded vectors I generated from sample mapd-core queries have the indicies' data buffers
+    // tagged as VectorType.OFFSET (0) in the field metadata. The current TS impl ignores buffers' layout type,
+    // and assumes the second buffer is the data for a NullableIntVector. Since we've been stricter about enforcing
+    // the Arrow spec while parsing, the IntVector's data buffer reads empty in this case. If so, fallback to using
+    // the offsets buffer as the data, since IntVectors don't have offsets.
+}
+
+const readFloatVector = readVectorLayout<number, FloatArray>(
+    createFloatDataViews,
+    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+        let type = field.type(new FloatingPoint());
+        let Vector = floatVectors[type.precision()];
+        return Vector.create(field, length, validity, data);
+    }
+);
+
+const readDateVector = readVectorLayout<Date, Uint32Array>(
+    createDateDataViews,
+    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+        return DateVector.create(field, length, validity, data);
+    }
+);
+
+const readUtf8Vector = readVectorLayout<string, Uint8Array>(
+    createUtf8DataViews,
+    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+        let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1);
+        return Utf8Vector.create(
+            field, length, validity,
+            Uint8Vector.create(field, data.length, null, data),
+            IndexVector.create(field, length + 1, null, offsetsAdjusted)
+        );
+    }
+);
+
+const readListVector = readVectorLayout<any[], Uint32Array>(
+    createNestedDataViews,
+    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+        let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1);
+        return ListVector.create(
+            field, length, validity,
+            readVector(field.children(0), batch, iterator, dictionaries),
+            IndexVector.create(field, length + 1, null, offsetsAdjusted)
+        );
+    }
+);
+
+const readFixedSizeListVector = readVectorLayout<any[], Uint32Array>(
+    createNestedDataViews,
+    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+        let size = field.type(new FixedSizeList()).listSize();
+        return FixedSizeListVector.create(
+            field, length, size, validity,
+            readVector(field.children(0), batch, iterator, dictionaries)
+        );
+    }
+);
+
+const readStructVector = readVectorLayout<any[], ArrayLike<any>>(
+    createNestedDataViews,
+    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+        let vectors: Vector<any>[] = [];
+        for (let i = -1, n = field.childrenLength(); ++i < n;) {
+            vectors[i] = readVector(field.children(i), batch, iterator, dictionaries);
+        }
+        return StructVector.create(field, length, validity, ...vectors);
+    }
+);
+
+const typedVectorReaders = {
+    [Type.Int]: readIntVector,
+    [Type.Date]: readDateVector,
+    [Type.List]: readListVector,
+    [Type.Utf8]: readUtf8Vector,
+    [Type.Struct_]: readStructVector,
+    [Type.FloatingPoint]: readFloatVector,
+    [Type.FixedSizeList]: readFixedSizeListVector,
+};
+
+type FieldType = { bitWidth(): number; isSigned(): boolean };
+type dataViewFactory<V = TypedArray> = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => V;
+type vectorFactory<TList, V = Vector<any>> = (field: Field,
+                                              length: number,
+                                              data: TList,
+                                              nulls: Uint8Array,
+                                              offsets: TypedArray,
+                                              fieldType: FieldType,
+                                              chunk: MessageBatch,
+                                              iterable: IteratorState,
+                                              dictionaries: Dictionaries) => V;
+
+function readVectorLayout<T, TList>(createDataView: dataViewFactory<TList>, createVector: vectorFactory<TList, Vector<T>>) {
+    return function readLayout(
+            field: Field,
+            chunk: MessageBatch,
+            iterator: IteratorState,
+            dictionaries: Dictionaries,
+            integerFieldType?: FieldType
+    ) {
+        let batch = chunk.data;
+        let layoutLength = field.layoutLength();
+        let node = batch.nodes(iterator.nodeIndex++);
+        let data: TList, offsets: any, validity: Uint8Array;
+        let type, bitWidth, bufferLength, nodeLength = node.length().low;
+        for (let i = -1; ++i < layoutLength;) {
+            let layout = field.layout(i);
+            let buffer = batch.buffers(iterator.bufferIndex++);
+            if ((type = layout.type()) === VectorType.TYPE ||
+                (bufferLength = buffer.length().low) <= 0  ||
+                (bitWidth = layout.bitWidth()) <= 0) {
+                continue;
+            } else if (type === VectorType.DATA) {
+                data = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength);
+            } else if (type === VectorType.OFFSET) {
+                offsets = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength);
+            } else if (node.nullCount().low > 0) {
+                validity = createValidityDataViews(chunk, type, bitWidth, buffer.offset().low, nodeLength);
+            }
+        }
+        return createVector(field, nodeLength, data, validity, offsets, integerFieldType, chunk, iterator, dictionaries);
+    };
+}
+
+function createDataView(
+    dataViews: TypedArrayCtor<any>[], offsetViews: TypedArrayCtor<any>[] | null,
+    batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number
+) {
+    const buffer = batch.bytes.buffer;
+    const byteLength = buffer.byteLength;
+    const byteOffset = batch.offset + offset;
+    const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || dataViews);
+    const dataViewLength = ((byteOffset + length) <= byteLength
+        ? length
+        : byteLength - byteOffset
+    ) / DataViewType['BYTES_PER_ELEMENT'];
+    return new DataViewType(buffer, byteOffset, dataViewLength);
+}
+
+function valueForBitWidth(bitWidth: number, values: any[]) {
+    return values[bitWidth >> 4] || values[3];
+}
+
+function createSyntheticDictionaryIndexField(field: Field, type: FieldType) {
+    let layouts = [];
+    let builder = new flatbuffers.Builder();
+    if (field.nullable()) {
+        VectorLayout.startVectorLayout(builder);
+        VectorLayout.addBitWidth(builder, 8);
+        VectorLayout.addType(builder, VectorType.VALIDITY);
+        builder.finish(VectorLayout.endVectorLayout(builder));
+        layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer()));
+        builder = new flatbuffers.Builder();
+    }
+    VectorLayout.startVectorLayout(builder);
+    VectorLayout.addBitWidth(builder, type.bitWidth());
+    VectorLayout.addType(builder, VectorType.DATA);
+    builder.finish(VectorLayout.endVectorLayout(builder));
+    layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer()));
+    return Object.create(field, {
+        layout: { value(i) { return layouts[i]; } },
+        layoutLength: { value() { return layouts.length; } }
+    });
+}
\ No newline at end of file
diff --git a/js/src/table.ts b/js/src/table.ts
new file mode 100644
index 0000000000000..999bb245182d4
--- /dev/null
+++ b/js/src/table.ts
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { readBuffers } from './reader/arrow';
+import { StructVector } from './vector/struct';
+import { Vector, sliceToRangeArgs } from './vector/vector';
+
+export class Table implements Iterable<Map<string, any>> {
+    public length: number;
+    protected _columns: Vector<any>[];
+    protected _columnsMap: { [k: string]: Vector<any> };
+    static from(...bytes: Array<Uint8Array | Buffer | string>) {
+        let columns: Vector<any>[];
+        for (let vectors of readBuffers(...bytes)) {
+            columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i]));
+        }
+        return new Table(columns);
+    }
+    static fromStruct(vector: StructVector) {
+        return new Table((<any> vector).vectors);
+    }
+    constructor(columns: Vector<any>[]) {
+        this._columns = columns || [];
+        this.length = Math.max(...this._columns.map((v) => v.length));
+        this._columnsMap = this._columns.reduce((map, vec) => {
+            return (map[vec.name] = vec) && map || map;
+        }, <any> {});
+    }
+    *[Symbol.iterator]() {
+        for (let cols = this._columns, i = -1, n = this.length; ++i < n;) {
+            yield rowAsMap(i, cols);
+        }
+    }
+    *rows(startRow?: number | boolean, endRow?: number | boolean, compact?: boolean) {
+        let start = startRow as number, end = endRow as number;
+        if (typeof startRow === 'boolean') {
+            compact = startRow;
+            start = end;
+            end = undefined;
+        } else if (typeof endRow === 'boolean') {
+            compact = endRow;
+            end = undefined;
+        }
+        let rowIndex = -1, { length } = this;
+        const [rowOffset, rowsTotal] = sliceToRangeArgs(length, start, end);
+        while (++rowIndex < rowsTotal) {
+            yield this.getRow((rowIndex + rowOffset) % length, compact);
+        }
+    }
+    *cols(startCol?: number, endCol?: number) {
+        for (const column of this._columns.slice(startCol, endCol)) {
+            yield column;
+        }
+    }
+    getRow(rowIndex: number, compact?: boolean) {
+        return (compact && rowAsArray || rowAsObject)(rowIndex, this._columns);
+    }
+    getCell(columnName: string, rowIndex: number) {
+        return this.getColumn(columnName).get(rowIndex);
+    }
+    getCellAt(columnIndex: number, rowIndex: number) {
+        return this.getColumnAt(columnIndex).get(rowIndex);
+    }
+    getColumn<T = any>(columnName: string) {
+        return this._columnsMap[columnName] as Vector<T>;
+    }
+    getColumnAt<T = any>(columnIndex: number) {
+        return this._columns[columnIndex] as Vector<T>;
+    }
+    toString({ index = false } = {}) {
+        const { length } = this;
+        if (length <= 0) { return ''; }
+        const maxColumnWidths = [];
+        const rows = new Array(length + 1);
+        rows[0] = this._columns.map((c) => c.name);
+        index && rows[0].unshift('Index');
+        for (let i = -1, n = rows.length - 1; ++i < n;) {
+            rows[i + 1] = this.getRow(i, true);
+            index && rows[i + 1].unshift(i);
+        }
+        // Pass one to convert to strings and count max column widths
+        for (let i = -1, n = rows.length; ++i < n;) {
+            const row = rows[i];
+            for (let j = -1, k = row.length; ++j < k;) {
+                const val = row[j] = `${row[j]}`;
+                maxColumnWidths[j] = !maxColumnWidths[j]
+                    ? val.length
+                    : Math.max(maxColumnWidths[j], val.length);
+            }
+        }
+        // Pass two to pad each one to max column width
+        for (let i = -1, n = rows.length; ++i < n;) {
+            const row = rows[i];
+            for (let j = -1, k = row.length; ++j < k;) {
+                row[j] = leftPad(row[j], ' ', maxColumnWidths[j]);
+            }
+            rows[i] = row.join(', ');
+        }
+        return rows.join('\n');
+    }
+}
+
+Table.prototype.length = 0;
+
+function leftPad(str, fill, n) {
+    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
+}
+
+function rowAsMap(row: number, columns: Vector<any>[]) {
+    return columns.reduce((map, vector) => map.set(vector.name, vector.get(row)), new Map());
+}
+
+function rowAsObject(rowIndex: number, columns: Vector<any>[]) {
+    return columns.reduce((row, vector) => (row[vector.name] = vector.get(rowIndex)) && row || row, Object.create(null));
+}
+
+function rowAsArray(rowIndex: number, columns: Vector<any>[]) {
+    return columns.reduce((row, vector, columnIndex) => (row[columnIndex] = vector.get(rowIndex)) && row || row, new Array(columns.length));
+}
diff --git a/js/src/types.ts b/js/src/types.ts
deleted file mode 100644
index c541098d53426..0000000000000
--- a/js/src/types.ts
+++ /dev/null
@@ -1,597 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { org } from "./Arrow_generated";
-import { BitArray } from "./bitarray";
-
-import { TextDecoder } from "text-encoding";
-
-const Type = org.apache.arrow.flatbuf.Type;
-
-interface IArrayView {
-    slice(start: number, end: number): IArrayView;
-    toString(): string;
-}
-
-interface IViewConstructor<T extends IArrayView> {
-    BYTES_PER_ELEMENT: number;
-    new(buffer: any, offset: number, length: number): T;
-}
-
-export abstract class Vector {
-    /**
-     * Helper function for loading a VALIDITY buffer (for Nullable types)
-     *   bb: flatbuffers.ByteBuffer
-     *   buffer: org.apache.arrow.flatbuf.Buffer
-     */
-    public static loadValidityBuffer(bb, buffer): BitArray {
-        const arrayBuffer = bb.bytes_.buffer;
-        const offset = bb.bytes_.byteOffset + buffer.offset;
-        return new BitArray(arrayBuffer, offset, buffer.length * 8);
-    }
-
-    /**
-     * Helper function for loading an OFFSET buffer
-     *   buffer: org.apache.arrow.flatbuf.Buffer
-     */
-    public static loadOffsetBuffer(bb, buffer): Int32Array {
-        const arrayBuffer = bb.bytes_.buffer;
-        const offset  = bb.bytes_.byteOffset + buffer.offset;
-        const length = buffer.length / Int32Array.BYTES_PER_ELEMENT;
-        return new Int32Array(arrayBuffer, offset, length);
-    }
-
-    public field: any;
-    public name: string;
-    public length: number;
-    public nullCount: number;
-
-    constructor(field) {
-        this.field = field;
-        this.name = field.name();
-    }
-
-    /* Access datum at index i */
-    public abstract get(i);
-    /* Return array representing data in the range [start, end) */
-    public abstract slice(start: number, end: number);
-    /* Return array of child vectors, for container types */
-    public getChildVectors() {
-        return [];
-    }
-
-    /**
-     * Use recordBatch fieldNodes and Buffers to construct this Vector
-     *   bb: flatbuffers.ByteBuffer
-     *   node: org.apache.arrow.flatbuf.FieldNode
-     *   buffers: { offset: number, length: number }[]
-     */
-    public loadData(bb, node, buffers) {
-        this.length = node.length().low;
-        this.nullCount = node.nullCount().low;
-        this.loadBuffers(bb, node, buffers);
-    }
-
-    protected abstract loadBuffers(bb, node, buffers);
-}
-
-class SimpleVector<T extends IArrayView> extends Vector {
-    protected dataView: T;
-    private TypedArray: IViewConstructor<T>;
-
-    constructor(field, TypedArray: IViewConstructor<T>) {
-        super(field);
-        this.TypedArray = TypedArray;
-    }
-
-    public get(i) {
-        return this.dataView[i];
-    }
-
-    public getDataView() {
-        return this.dataView;
-    }
-
-    public toString() {
-        return this.dataView.toString();
-    }
-
-    public slice(start, end) {
-        return this.dataView.slice(start, end);
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.loadDataBuffer(bb, buffers[0]);
-    }
-
-    /*
-     * buffer: org.apache.arrow.flatbuf.Buffer
-     */
-    protected loadDataBuffer(bb, buffer) {
-        const arrayBuffer = bb.bytes_.buffer;
-        const offset  = bb.bytes_.byteOffset + buffer.offset;
-        const length = buffer.length / this.TypedArray.BYTES_PER_ELEMENT;
-        this.dataView = new this.TypedArray(arrayBuffer, offset, length);
-    }
-
-}
-
-class NullableSimpleVector<T extends IArrayView> extends SimpleVector<T> {
-    protected validityView: BitArray;
-
-    public get(i: number) {
-        if (this.validityView.get(i)) {
-            return this.dataView[i];
-        } else {
-          return null;
-        }
-    }
-
-    public getValidityVector() {
-        return this.validityView;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-        this.loadDataBuffer(bb, buffers[1]);
-    }
-}
-
-/* tslint:disable max-line-length */
-class Uint8Vector   extends SimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   } }
-class Uint16Vector  extends SimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  } }
-class Uint32Vector  extends SimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  } }
-class Int8Vector    extends SimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   } }
-class Int16Vector   extends SimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  } }
-class Int32Vector   extends SimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  } }
-class Float32Vector extends SimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); } }
-class Float64Vector extends SimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); } }
-
-class NullableUint8Vector   extends NullableSimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   } }
-class NullableUint16Vector  extends NullableSimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  } }
-class NullableUint32Vector  extends NullableSimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  } }
-class NullableInt8Vector    extends NullableSimpleVector<Uint8Array>   { constructor(field) { super(field, Uint8Array);   } }
-class NullableInt16Vector   extends NullableSimpleVector<Uint16Array>  { constructor(field) { super(field, Uint16Array);  } }
-class NullableInt32Vector   extends NullableSimpleVector<Uint32Array>  { constructor(field) { super(field, Uint32Array);  } }
-class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); } }
-class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); } }
-/* tslint:enable max-line-length */
-
-class Uint64Vector extends SimpleVector<Uint32Array>  {
-    constructor(field) {
-        super(field, Uint32Array);
-    }
-
-    public get(i: number) {
-        return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
-    }
-}
-
-class NullableUint64Vector extends NullableSimpleVector<Uint32Array>  {
-    constructor(field) {
-        super(field, Uint32Array);
-    }
-
-    public get(i: number) {
-        if (this.validityView.get(i)) {
-            return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
-        } else {
-          return null;
-        }
-    }
-}
-
-class Int64Vector extends NullableSimpleVector<Uint32Array>  {
-    constructor(field) {
-        super(field, Uint32Array);
-    }
-
-    public get(i: number) {
-        return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
-    }
-}
-
-class NullableInt64Vector extends NullableSimpleVector<Uint32Array>  {
-    constructor(field) {
-        super(field, Uint32Array);
-    }
-
-    public get(i: number) {
-        if (this.validityView.get(i)) {
-            return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
-        } else {
-          return null;
-        }
-    }
-}
-
-class DateVector extends SimpleVector<Uint32Array> {
-    constructor(field) {
-        super(field, Uint32Array);
-    }
-
-    public get(i) {
-        return new Date(super.get(2 * i + 1) * Math.pow(2, 32) + super.get(2 * i));
-    }
-}
-
-class NullableDateVector extends DateVector {
-    private validityView: BitArray;
-
-    public get(i) {
-        if (this.validityView.get(i)) {
-            return super.get(i);
-        } else {
-            return null;
-        }
-    }
-
-    public getValidityVector() {
-        return this.validityView;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-        this.loadDataBuffer(bb, buffers[1]);
-    }
-}
-
-class Utf8Vector extends SimpleVector<Uint8Array> {
-    private static decoder: TextDecoder = new TextDecoder("utf8");
-
-    protected offsetView: Int32Array;
-
-    constructor(field) {
-        super(field, Uint8Array);
-    }
-
-    public get(i) {
-        return Utf8Vector.decoder.decode(this.dataView.slice(this.offsetView[i], this.offsetView[i + 1]));
-    }
-
-    public slice(start: number, end: number) {
-        const result: string[] = [];
-        for (let i: number = start; i < end; i++) {
-            result.push(this.get(i));
-        }
-        return result;
-    }
-
-    public getOffsetView() {
-        return this.offsetView;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]);
-        this.loadDataBuffer(bb, buffers[1]);
-    }
-}
-
-class NullableUtf8Vector extends Utf8Vector {
-    private validityView: BitArray;
-
-    public get(i) {
-        if (this.validityView.get(i)) {
-            return super.get(i);
-        } else {
-            return null;
-        }
-    }
-
-    public getValidityVector() {
-        return this.validityView;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-        this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]);
-        this.loadDataBuffer(bb, buffers[2]);
-    }
-}
-
-// Nested Types
-class ListVector extends Uint32Vector {
-    private dataVector: Vector;
-
-    constructor(field, dataVector: Vector) {
-        super(field);
-        this.dataVector = dataVector;
-    }
-
-    public getChildVectors() {
-        return [this.dataVector];
-    }
-
-    public get(i) {
-        const offset = super.get(i);
-        if (offset === null) {
-            return null;
-        }
-        const nextOffset = super.get(i + 1);
-        return this.dataVector.slice(offset, nextOffset);
-    }
-
-    public toString() {
-        return "length: " + (this.length);
-    }
-
-    public slice(start: number, end: number) {
-        const result = [];
-        for (let i = start; i < end; i++) {
-            result.push(this.get(i));
-        }
-        return result;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        super.loadBuffers(bb, node, buffers);
-        this.length -= 1;
-    }
-}
-
-class NullableListVector extends ListVector {
-    private validityView: BitArray;
-
-    public get(i) {
-        if (this.validityView.get(i)) {
-            return super.get(i);
-        } else {
-            return null;
-        }
-    }
-
-    public getValidityVector() {
-        return this.validityView;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-        this.loadDataBuffer(bb, buffers[1]);
-        this.length -= 1;
-    }
-}
-
-class FixedSizeListVector extends Vector {
-    public size: number;
-    private dataVector: Vector;
-
-    constructor(field, size: number, dataVector: Vector) {
-        super(field);
-        this.size = size;
-        this.dataVector = dataVector;
-    }
-
-    public getChildVectors() {
-        return [this.dataVector];
-    }
-
-    public get(i: number) {
-        return this.dataVector.slice(i * this.size, (i + 1) * this.size);
-    }
-
-    public slice(start: number, end: number) {
-        const result = [];
-        for (let i = start; i < end; i++) {
-            result.push(this.get(i));
-        }
-        return result;
-    }
-
-    public getListSize() {
-        return this.size;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        // no buffers to load
-    }
-}
-
-class NullableFixedSizeListVector extends FixedSizeListVector {
-    private validityView: BitArray;
-
-    public get(i: number) {
-        if (this.validityView.get(i)) {
-            return super.get(i);
-        } else {
-            return null;
-        }
-    }
-
-    public getValidityVector() {
-        return this.validityView;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-    }
-}
-
-class StructVector extends Vector {
-    private validityView: BitArray;
-    private vectors: Vector[];
-
-    constructor(field, vectors: Vector[]) {
-        super(field);
-        this.vectors = vectors;
-    }
-
-    public getChildVectors() {
-        return this.vectors;
-    }
-
-    public get(i: number) {
-        if (this.validityView.get(i)) {
-          return this.vectors.map((v: Vector) => v.get(i));
-        } else {
-            return null;
-        }
-    }
-
-    public slice(start: number, end: number) {
-        const result = [];
-        for (let i = start; i < end; i++) {
-            result.push(this.get(i));
-        }
-        return result;
-    }
-
-    public getValidityVector() {
-        return this.validityView;
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
-    }
-}
-
-class DictionaryVector extends Vector {
-    private indices: Vector;
-    private dictionary: Vector;
-
-    constructor(field, indices: Vector, dictionary: Vector) {
-        super(field);
-        this.indices = indices;
-        this.dictionary = dictionary;
-    }
-
-    public get(i) {
-        const encoded = this.indices.get(i);
-        if (encoded == null) {
-            return null;
-        } else {
-            return this.dictionary.get(encoded);
-        }
-    }
-
-    /** Get the dictionary encoded value */
-    public getEncoded(i) {
-        return this.indices.get(i);
-    }
-
-    public slice(start, end) {
-        return this.indices.slice(start, end); // TODO decode
-    }
-
-    public getChildVectors() {
-        return this.indices.getChildVectors();
-    }
-
-    /** Get the index (encoded) vector */
-    public getIndexVector() {
-        return this.indices;
-    }
-
-    /** Get the dictionary vector */
-    public getDictionaryVector() {
-        return this.dictionary;
-    }
-
-    public toString() {
-        return this.indices.toString();
-    }
-
-    protected loadBuffers(bb, node, buffers) {
-        this.indices.loadData(bb, node, buffers);
-    }
-}
-
-export function vectorFromField(field, dictionaries): Vector {
-    const dictionary = field.dictionary();
-    const nullable = field.nullable();
-    if (dictionary == null) {
-        const typeType = field.typeType();
-        if (typeType === Type.List) {
-            const dataVector = vectorFromField(field.children(0), dictionaries);
-            return nullable ? new NullableListVector(field, dataVector) : new ListVector(field, dataVector);
-        } else if (typeType === Type.FixedSizeList) {
-            const dataVector = vectorFromField(field.children(0), dictionaries);
-            const size = field.type(new org.apache.arrow.flatbuf.FixedSizeList()).listSize();
-            if (nullable) {
-              return new NullableFixedSizeListVector(field, size, dataVector);
-            } else {
-              return new FixedSizeListVector(field, size, dataVector);
-            }
-         } else if (typeType === Type.Struct_) {
-            const vectors: Vector[] = [];
-            for (let i: number = 0; i < field.childrenLength(); i++) {
-                vectors.push(vectorFromField(field.children(i), dictionaries));
-            }
-            return new StructVector(field, vectors);
-        } else {
-            if (typeType === Type.Int) {
-                const type = field.type(new org.apache.arrow.flatbuf.Int());
-                return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable);
-            } else if (typeType === Type.FloatingPoint) {
-                const precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision();
-                if (precision === org.apache.arrow.flatbuf.Precision.SINGLE) {
-                    return nullable ? new NullableFloat32Vector(field) : new Float32Vector(field);
-                } else if (precision === org.apache.arrow.flatbuf.Precision.DOUBLE) {
-                    return nullable ? new NullableFloat64Vector(field) : new Float64Vector(field);
-                } else {
-                    throw new Error("Unimplemented FloatingPoint precision " + precision);
-                }
-            } else if (typeType === Type.Utf8) {
-                return nullable ? new NullableUtf8Vector(field) : new Utf8Vector(field);
-            } else if (typeType === Type.Date) {
-                return nullable ? new NullableDateVector(field) : new DateVector(field);
-            } else {
-                throw new Error("Unimplemented type " + typeType);
-            }
-        }
-    } else {
-        // determine arrow type - default is signed 32 bit int
-        const type = dictionary.indexType();
-        let bitWidth = 32;
-        let signed = true;
-        if (type != null) {
-            bitWidth = type.bitWidth();
-            signed = type.isSigned();
-        }
-        const indices = _createIntVector(field, bitWidth, signed, nullable);
-        return new DictionaryVector(field, indices, dictionaries[dictionary.id().toFloat64().toString()]);
-    }
-}
-
-function _createIntVector(field, bitWidth, signed, nullable) {
-    if (bitWidth === 64) {
-        if (signed) {
-            return nullable ? new NullableInt64Vector(field) : new Int64Vector(field);
-        } else {
-            return nullable ? new NullableUint64Vector(field) : new Uint64Vector(field);
-        }
-    } else if (bitWidth === 32) {
-        if (signed) {
-            return nullable ? new NullableInt32Vector(field) : new Int32Vector(field);
-        } else {
-            return nullable ? new NullableUint32Vector(field) : new Uint32Vector(field);
-        }
-    } else if (bitWidth === 16) {
-        if (signed) {
-            return nullable ? new NullableInt16Vector(field) : new Int16Vector(field);
-        } else {
-            return nullable ? new NullableUint16Vector(field) : new Uint16Vector(field);
-        }
-    } else if (bitWidth === 8) {
-        if (signed) {
-            return nullable ? new NullableInt8Vector(field) : new Int8Vector(field);
-        } else {
-            return nullable ? new NullableUint8Vector(field) : new Uint8Vector(field);
-        }
-    } else {
-         throw new Error("Unimplemented Int bit width " + bitWidth);
-    }
-}
diff --git a/js/src/vector/dictionary.ts b/js/src/vector/dictionary.ts
new file mode 100644
index 0000000000000..de811eaf5b050
--- /dev/null
+++ b/js/src/vector/dictionary.ts
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+
+export class DictionaryVector<T> extends Vector<T> {
+    protected data: Vector<T>;
+    protected keys: Vector<number>;
+    constructor(index: Vector<number>, dictionary: Vector<T>) {
+        super();
+        this.keys = index;
+        this.data = dictionary;
+        this.length = index && index.length || 0;
+    }
+    index(index: number) {
+        return this.keys.get(index);
+    }
+    value(index: number) {
+        return this.data.get(index);
+    }
+    get(index: number) {
+        return this.value(this.index(index));
+    }
+    concat(vector: DictionaryVector<T>) {
+        return DictionaryVector.from(this,
+            this.length + vector.length,
+            this.keys.concat(vector.keys),
+            this.data
+        );
+    }
+    *[Symbol.iterator]() {
+        let { data } = this;
+        for (const loc of this.keys) {
+            yield data.get(loc);
+        }
+    }
+}
diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts
new file mode 100644
index 0000000000000..7360d968b0250
--- /dev/null
+++ b/js/src/vector/list.ts
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { TextDecoder } from 'text-encoding';
+import { IndexVector, BitVector, ValidityArgs } from './typed';
+
+export class ListVectorBase<T> extends Vector<T> {
+    protected values: Vector<T>;
+    protected offsets: IndexVector;
+    constructor(validity: ValidityArgs, values: Vector<any>, offsets: IndexVector) {
+        super();
+        this.values = values;
+        this.offsets = offsets;
+        validity && (this.validity = BitVector.from(validity));
+    }
+    get(index: number) {
+        let batch, from, to, { offsets } = this;
+        if (!this.validity.get(index) ||
+            /* return null if `to` is null */
+            ((to = offsets.get(index + 1)) === null) || !(
+            /*
+            return null if `batch` is less than than 0. this check is placed
+            second to avoid creating the [from, batch] tuple if `to` is null
+            */
+            ([from, batch] = offsets.get(index, true) as number[]) && batch > -1)) {
+            return null;
+        }
+        return this.values.slice(from, to, batch) as any;
+    }
+    concat(vector: ListVectorBase<T>) {
+        return (this.constructor as typeof ListVectorBase).from(this,
+            this.length + vector.length,
+            this.validity.concat(vector.validity),
+            this.values.concat(vector.values),
+            this.offsets.concat(vector.offsets)
+        );
+    }
+    *[Symbol.iterator]() {
+        let v, r1, r2, { values } = this;
+        let it = this.offsets[Symbol.iterator]();
+        let iv = this.validity[Symbol.iterator]();
+        while (!(v = iv.next()).done && !(r1 = it.next()).done && !(r2 = it.next()).done) {
+            yield !v.value ? null : values.slice(r1.value[0], r2.value, r1.value[1]) as any;
+        }
+    }
+}
+
+export class ListVector<T> extends ListVectorBase<T[]> {}
+export class Utf8Vector extends ListVectorBase<string> {
+    protected static decoder = new TextDecoder(`utf-8`);
+    get(index: number) {
+        let chars = super.get(index) as any;
+        return chars ? Utf8Vector.decoder.decode(chars) : null;
+    }
+    *[Symbol.iterator]() {
+        let decoder = Utf8Vector.decoder;
+        for (const chars of super[Symbol.iterator]()) {
+            yield !chars ? null : decoder.decode(chars);
+        }
+    }
+}
+
+export class FixedSizeListVector<T> extends Vector<T[]> {
+    protected size: number;
+    protected values: Vector<T>;
+    constructor(size: number, validity: ValidityArgs, values: Vector<T>) {
+        super();
+        this.values = values;
+        this.size = Math.abs(size | 0) || 1;
+        validity && (this.validity = BitVector.from(validity));
+    }
+    get(index: number) {
+        return !this.validity.get(index) ? null : this.values.slice(
+            this.size * index, this.size * (index + 1)
+        ) as T[];
+    }
+    concat(vector: FixedSizeListVector<T>) {
+        return FixedSizeListVector.from(this,
+            this.length + vector.length,
+            this.size,
+            this.validity.concat(vector.validity),
+            this.values.concat(vector.values)
+        );
+    }
+    *[Symbol.iterator]() {
+        let v, i = -1;
+        let { size, length, values } = this;
+        let iv = this.validity[Symbol.iterator]();
+        while (!(v = iv.next()).done && ++i < length) {
+            yield !v.value ? null : values.slice(size * i, size * (i + 1)) as T[];
+        }
+    }
+}
diff --git a/js/src/bitarray.ts b/js/src/vector/struct.ts
similarity index 51%
rename from js/src/bitarray.ts
rename to js/src/vector/struct.ts
index 6b0a91aed606a..e59ac91e9cd08 100644
--- a/js/src/bitarray.ts
+++ b/js/src/vector/struct.ts
@@ -15,28 +15,25 @@
 // specific language governing permissions and limitations
 // under the License.
 
-export class BitArray {
-    private view: Uint8Array;
+import { Vector } from './vector';
+import { BitVector, ValidityArgs } from './typed';
 
-    constructor(buffer: ArrayBuffer, offset: number, length: number) {
-        this.view = new Uint8Array(buffer, offset || 0, Math.ceil(length / 8));
+export class StructVector extends Vector<any[]> {
+    protected vectors: Vector<any>[];
+    constructor(validity: ValidityArgs, ...vectors: Vector<any>[]) {
+        super();
+        this.vectors = vectors;
+        this.length = Math.max(0, ...vectors.map((v) => v.length));
+        validity && (this.validity = BitVector.from(validity));
     }
-
-    public get(i) {
-        const index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too.
-        const bit = i % 8;  // i % 8 is just as fast as i & 7
-        return (this.view[index] & (1 << bit)) !== 0;
-    }
-
-    public set(i) {
-        const index = (i >> 3) | 0;
-        const bit = i % 8;
-        this.view[index] |= 1 << bit;
+    get(index: number) {
+        return this.validity.get(index) ? this.vectors.map((v) => v.get(index)) : null;
     }
-
-    public unset(i) {
-        const index = (i >> 3) | 0;
-        const bit = i % 8;
-        this.view[index] &= ~(1 << bit);
+    concat(vector: StructVector) {
+        return StructVector.from(this,
+            this.length + vector.length,
+            this.validity.concat(vector.validity),
+            ...this.vectors.map((v, i) => v.concat(vector.vectors[i]))
+        );
     }
 }
diff --git a/js/src/vector/typed.ts b/js/src/vector/typed.ts
new file mode 100644
index 0000000000000..b38812e07d065
--- /dev/null
+++ b/js/src/vector/typed.ts
@@ -0,0 +1,326 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { flatbuffers } from 'flatbuffers';
+
+import Long = flatbuffers.Long;
+
+export type VArray<T = any> = {
+    [k: number]: T; length: number;
+    constructor: VArrayCtor<VArray<T>>;
+};
+
+export type VArrayCtor<VArray> = {
+    readonly prototype: VArray;
+    BYTES_PER_ELEMENT?: number;
+    new(...args: any[]): VArray;
+};
+
+export class VirtualVector<T, TArrayType = VArray<T>> extends Vector<T> {
+    protected lists: TArrayType[];
+    protected _arrayType: VArrayCtor<TArrayType>;
+    public get arrayType() { return this._arrayType; }
+    constructor(...lists: TArrayType[]) {
+        super();
+        this.lists = lists.filter(Boolean);
+    }
+    get(index: number): T {
+        /* inlined `findVirtual` impl */
+        let rows, length, lists = this.lists;
+        for (let batch = -1;
+            (rows = lists[++batch]) &&
+            (length = rows.length) <= index &&
+            0 <= (index -= length);) {}
+        return rows && -1 < index ? rows[index] : null;
+    }
+    protected range(from: number, total: number, batch?: number) {
+        /* inlined `findVirtual` impl */
+        let rows, local = from, length;
+        let { lists, _arrayType } = this;
+        for (batch = (batch || 0) - 1;
+            (rows = lists[++batch]) &&
+            (length = rows.length) <= local &&
+            0 <= (local -= length);) {}
+        if (rows && local > -1) {
+            let index = 0, listsLength = lists.length;
+            let set: any = Array.isArray(rows) ? arraySet : typedArraySet;
+            let slice = _arrayType['prototype']['subarray'] || _arrayType['prototype']['slice'];
+            let source = slice.call(rows, local, local + total), target = source;
+            // Perf optimization: if the first slice contains all the values we're looking for,
+            // we don't have to copy values to a target Array. If we're slicing a TypedArray,
+            // this is a significant improvement as we avoid the memcpy 🎉
+            if (source.length < total) {
+                target = new _arrayType(total);
+                while ((index = set(source, target, index)) < total) {
+                    rows = lists[batch = ((batch + 1) % listsLength)];
+                    source = slice.call(rows, 0, Math.min(rows.length, total - index));
+                }
+            }
+            return target as any;
+        }
+        return new _arrayType(0);
+    }
+    *[Symbol.iterator]() {
+        let index = -1, { lists, length } = this;
+        for (let outer = -1, n = lists.length; ++outer < n;) {
+            let list = lists[outer] as any;
+            for (let inner = -1, k = list.length; ++index < length && ++inner < k;) {
+                yield list[inner];
+            }
+        }
+    }
+}
+
+export type ValidityArgs = Vector<boolean> | Uint8Array;
+export class BitVector extends VirtualVector<boolean, Uint8Array> {
+    static constant: Vector<boolean> = new (class ValidVector extends Vector<boolean> {
+        get() { return true; }
+        *[Symbol.iterator]() {
+            do { yield true; } while (true);
+        }
+    })();
+    static from(src: any) {
+        return src instanceof BitVector   ? src
+             : src === BitVector.constant ? src
+             : src instanceof Uint8Array       ? new BitVector(src)
+             : src instanceof Array            ? new BitVector(BitVector.pack(src))
+             : src instanceof Vector           ? new BitVector(BitVector.pack(src))
+                                               : BitVector.constant as Vector<any>;
+    }
+    static pack(values: Iterable<any>) {
+        let xs = [], n, i = 0;
+        let bit = 0, byte = 0;
+        for (const value of values) {
+            value && (byte |= 1 << bit);
+            if (++bit === 8) {
+                xs[i++] = byte;
+                byte = bit = 0;
+            }
+        }
+        if (i === 0 || bit > 0) { xs[i++] = byte; }
+        if (i % 8 && (n = n = i + 8 - i % 8)) {
+            do { xs[i] = 0; } while (++i < n);
+        }
+        return new Uint8Array(xs);
+    }
+    constructor(...lists: Uint8Array[]) {
+        super(...lists);
+        this.length = this.lists.reduce((l, xs) => l + xs['length'], 0);
+    }
+    get(index: number) {
+        /* inlined `findVirtual` impl */
+        let rows, length, lists = this.lists;
+        for (let batch = -1;
+            (rows = lists[++batch]) &&
+            (length = rows.length) <= index &&
+            0 <= (index -= length);) {}
+        return !(!rows || index < 0 || (rows[index >> 3 | 0] & 1 << index % 8) === 0);
+    }
+    set(index: number, value: boolean) {
+        /* inlined `findVirtual` impl */
+        let rows, length, lists = this.lists;
+        for (let batch = -1;
+            (rows = lists[++batch]) &&
+            (length = rows.length) <= index &&
+            0 <= (index -= length);) {}
+        if (rows && index > -1) {
+            value
+                ? (rows[index >> 3 | 0] |=  (1 << (index % 8)))
+                : (rows[index >> 3 | 0] &= ~(1 << (index % 8)));
+        }
+    }
+    concat(vector: BitVector) {
+        return new BitVector(...this.lists, ...vector.lists);
+    }
+    *[Symbol.iterator]() {
+        for (const byte of super[Symbol.iterator]()) {
+            for (let i = -1; ++i < 8;) {
+                yield (byte & 1 << i) !== 0;
+            }
+        }
+    }
+}
+
+export class TypedVector<T, TArrayType> extends VirtualVector<T, TArrayType> {
+    constructor(validity: ValidityArgs, ...lists: TArrayType[]) {
+        super(...lists);
+        validity && (this.validity = BitVector.from(validity));
+    }
+    concat(vector: TypedVector<T, TArrayType>) {
+        return (this.constructor as typeof TypedVector).from(this,
+            this.length + vector.length,
+            this.validity.concat(vector.validity),
+            ...this.lists, ...vector.lists
+        );
+    }
+}
+
+export class DateVector extends TypedVector<Date, Uint32Array> {
+    get(index: number) {
+        return !this.validity.get(index) ? null : new Date(
+            Math.pow(2, 32) *
+                <any> super.get(2 * index + 1) +
+                <any> super.get(2 * index)
+        );
+    }
+    *[Symbol.iterator]() {
+        let v, low, high;
+        let it = super[Symbol.iterator]();
+        let iv = this.validity[Symbol.iterator]();
+        while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) {
+            yield !v.value ? null : new Date(Math.pow(2, 32) * high.value + low.value);
+        }
+    }
+}
+
+export class IndexVector extends TypedVector<number | number[], Int32Array> {
+    get(index: number, returnWithBatchIndex = false) {
+        /* inlined `findVirtual` impl */
+        let rows, length, batch = -1, lists = this.lists;
+        for (;
+            (rows = lists[++batch]) &&
+            (length = rows.length) <= index &&
+            0 <= (index -= length);) {}
+        return !returnWithBatchIndex
+            ? (rows && -1 < index ? rows[index + batch] : null) as number
+            : (rows && -1 < index ? [rows[index + batch], batch] : [0, -1]) as number[];
+    }
+    *[Symbol.iterator]() {
+        // Alternate between iterating a tuple of [from, batch], and to. The from
+        // and to values are relative to the record batch they're defined in, so
+        // `ListVectorBase` needs to know the right batch to read.
+        let xs = new Int32Array(2), { lists } = this;
+        for (let i = -1, n = lists.length; ++i < n;) {
+            let list = lists[i] as any;
+            for (let j = -1, k = list.length - 1; ++j < k;) {
+                xs[1] = i;
+                xs[0] = list[j];
+                yield xs;
+                yield list[j + 1];
+            }
+        }
+    }
+}
+
+export class ByteVector<TList> extends TypedVector<number, TList> {
+    get(index: number) {
+        return this.validity.get(index) ? super.get(index) : null;
+    }
+    *[Symbol.iterator]() {
+        let v, r, { validity } = this;
+        let it = super[Symbol.iterator]();
+        // fast path the case of no nulls
+        if (validity === BitVector.constant) {
+            yield* it;
+        } else {
+            let iv = validity[Symbol.iterator]();
+            while (!(v = iv.next()).done && !(r = it.next()).done) {
+                yield !v.value ? null : r.value;
+            }
+        }
+    }
+}
+
+export class LongVector<TList> extends TypedVector<Long, TList> {
+    get(index: number) {
+        return !this.validity.get(index) ? null : new Long(
+            <any> super.get(index * 2),     /* low */
+            <any> super.get(index * 2 + 1) /* high */
+        );
+    }
+    *[Symbol.iterator]() {
+        let v, low, high;
+        let it = super[Symbol.iterator]();
+        let iv = this.validity[Symbol.iterator]();
+        while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) {
+            yield !v.value ? null : new Long(low.value, high.value);
+        }
+    }
+}
+
+export class Int8Vector    extends ByteVector<Int8Array>    {}
+export class Int16Vector   extends ByteVector<Int16Array>   {}
+export class Int32Vector   extends ByteVector<Int32Array>   {}
+export class Int64Vector   extends LongVector<Int32Array>   {}
+export class Uint8Vector   extends ByteVector<Uint8Array>   {}
+export class Uint16Vector  extends ByteVector<Uint16Array>  {}
+export class Uint32Vector  extends ByteVector<Uint32Array>  {}
+export class Uint64Vector  extends LongVector<Uint32Array>  {}
+export class Float32Vector extends ByteVector<Float32Array> {}
+export class Float64Vector extends ByteVector<Float64Array> {}
+
+LongVector.prototype.stride = 2;
+(Vector.prototype as any).lists = [];
+(Vector.prototype as any).validity = BitVector.constant;
+(VirtualVector.prototype as any)._arrayType = Array;
+(BitVector.prototype as any)._arrayType = Uint8Array;
+(Int8Vector.prototype as any)._arrayType = Int8Array;
+(Int16Vector.prototype as any)._arrayType = Int16Array;
+(Int32Vector.prototype as any)._arrayType = Int32Array;
+(Int64Vector.prototype as any)._arrayType = Int32Array;
+(Uint8Vector.prototype as any)._arrayType = Uint8Array;
+(Uint16Vector.prototype as any)._arrayType = Uint16Array;
+(Uint32Vector.prototype as any)._arrayType = Uint32Array;
+(Uint64Vector.prototype as any)._arrayType = Uint32Array;
+(DateVector.prototype as any)._arrayType = Uint32Array;
+(IndexVector.prototype as any)._arrayType = Int32Array;
+(Float32Vector.prototype as any)._arrayType = Float32Array;
+(Float64Vector.prototype as any)._arrayType = Float64Array;
+
+function arraySet<T>(source: Array<T>, target: Array<T>, index: number) {
+    for (let i = 0, n = source.length; i < n;) {
+        target[index++] = source[i++];
+    }
+    return index;
+}
+
+function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
+    return target.set(source, index) || index + source.length;
+}
+
+// Rather than eat the iterator cost, we've inlined this function into the relevant functions
+// function* findVirtual<TList>(index: number, lists: TList[], batch?: number) {
+//     let rows, length;
+//     for (batch = (batch || 0) - 1;
+//         (rows = lists[++batch]) &&
+//         (length = rows.length) <= index &&
+//         0 <= (index -= length);) {}
+//     return rows && -1 < index ? yield [rows, index, batch] : null;
+// }
+
+export type TypedArrayCtor<T extends TypedArray> = {
+    readonly prototype: T;
+    readonly BYTES_PER_ELEMENT: number;
+    new(length: number): T;
+    new(array: ArrayLike<number>): T;
+    new(buffer: ArrayBufferLike, byteOffset?: number, length?: number): T;
+};
+
+export type FloatArray = Float32Array | Float64Array;
+export type IntArray = Int8Array | Int16Array | Int32Array | Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array;
+
+export type TypedArray = (
+            Int8Array        |
+            Uint8Array       |
+            Int16Array       |
+            Int32Array       |
+            Uint16Array      |
+            Uint32Array      |
+            Float32Array     |
+            Float64Array     |
+            Uint8ClampedArray);
diff --git a/js/src/vector/vector.ts b/js/src/vector/vector.ts
new file mode 100644
index 0000000000000..1f39f87cbefc8
--- /dev/null
+++ b/js/src/vector/vector.ts
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Schema_ from '../format/Schema_generated';
+export import Type = Schema_.org.apache.arrow.flatbuf.Type;
+export import Field = Schema_.org.apache.arrow.flatbuf.Field;
+
+export function sliceToRangeArgs(length: number, start: number, end?: number) {
+    let total = length, from = start || 0;
+    let to = end === end && typeof end == 'number' ? end : total;
+    if (to < 0) { to = total + to; }
+    if (from < 0) { from = total - (from * -1) % total; }
+    if (to < from) { from = to; to = start; }
+    total = !isFinite(total = (to - from)) || total < 0 ? 0 : total;
+    return [from, total];
+}
+
+export class Vector<T> implements Iterable<T> {
+    static defaultName = '';
+    static defaultProps = new Map();
+    static defaultType = Type[Type.NONE];
+    static create<T = any>(field: Field, length: number, ...args: any[]) {
+        let vector = new this<T>(...args), m;
+        vector.length = length;
+        vector.name = field.name();
+        vector.type = Type[field.typeType()];
+        if ((m = field.customMetadataLength()) > 0) {
+            let entry, i = 0, data = vector.props = new Map();
+            do {
+                entry = field.customMetadata(i);
+                data[entry.key()] = entry.value();
+            } while (++i < m);
+        }
+        return vector;
+    }
+    static from<T = any>(source: Vector<T>, length: number, ...args: any[]) {
+        let vector = new this<T>(...args);
+        vector.length = length;
+        source.name !== Vector.defaultName && (vector.name = source.name);
+        source.type !== Vector.defaultType && (vector.type = source.type);
+        source.props !== Vector.defaultProps && (vector.props = source.props);
+        return vector;
+    }
+    public name: string;
+    public type: string;
+    public length: number;
+    public stride: number;
+    public props: Map<PropertyKey, any>;
+    protected validity: Vector<boolean>;
+    get(index: number): T { return null; }
+    concat(vector: Vector<T>) { return vector; }
+    slice<R = T>(start?: number, end?: number, batch?: number) {
+        const { stride } = this;
+        const [offset, length] = sliceToRangeArgs(
+            stride * this.length, stride * (start || 0), stride * end
+        );
+        return this.range<R>(offset, length, batch);
+    }
+    protected range<R = T>(index: number, length: number, batch?: number) {
+        const result = new Array<R>(length);
+        for (let i = -1, n = this.length; ++i < length;) {
+            result[i] = this.get((i + index) % n) as any;
+        }
+        return result as Iterable<R>;
+    }
+    *[Symbol.iterator]() {
+        for (let i = -1, n = this.length; ++i < n;) {
+            yield this.get(i);
+        }
+    }
+}
+
+Vector.prototype.length = 0;
+Vector.prototype.stride = 1;
+Vector.prototype.name = Vector.defaultName;
+Vector.prototype.type = Vector.defaultType;
+Vector.prototype.props = Vector.defaultProps;
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
new file mode 100644
index 0000000000000..a9ab2b783fecc
--- /dev/null
+++ b/js/test/Arrow.ts
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/* tslint:disable */
+// Dynamically load an Ix target build based on command line arguments
+
+const target = process.env.TEST_TARGET;
+const format = process.env.TEST_MODULE;
+const resolve = require('path').resolve;
+
+// these are duplicated in the gulpfile :<
+const targets = [`es5`, `es2015`, `esnext`];
+const formats = [`cjs`, `esm`, `cls`, `umd`];
+
+function throwInvalidImportError(name: string, value: string, values: string[]) {
+    throw new Error('Unrecognized ' + name + ' \'' + value + '\'. Please run tests with \'--' + name + ' <any of ' + values.join(', ') + '>\'');
+}
+
+if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets);
+if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats);
+
+let Arrow: any = require(resolve(`./targets/${target}/${format}/Arrow.js`));
+let ArrowInternal: any = require(resolve(`./targets/${target}/${format}/Arrow.internal.js`));
+
+import { vectors as vectors_ } from '../src/Arrow.internal';
+import { Table as Table_, readBuffers as readBuffers_ } from '../src/Arrow';
+
+export let Table = Arrow.Table as typeof Table_;
+export let readBuffers = Arrow.readBuffers as typeof readBuffers_;
+
+export let vectors: typeof vectors_ = ArrowInternal.vectors;
+export namespace vectors {
+    export type Vector<T> = vectors_.Vector<T>;
+    export type BitVector = vectors_.BitVector;
+    export type ListVector<T> = vectors_.ListVector<T>;
+    export type Utf8Vector = vectors_.Utf8Vector;
+    export type DateVector = vectors_.DateVector;
+    export type IndexVector = vectors_.IndexVector;
+    export type Int8Vector = vectors_.Int8Vector;
+    export type Int16Vector = vectors_.Int16Vector;
+    export type Int32Vector = vectors_.Int32Vector;
+    export type Int64Vector = vectors_.Int64Vector;
+    export type Uint8Vector = vectors_.Uint8Vector;
+    export type Uint16Vector = vectors_.Uint16Vector;
+    export type Uint32Vector = vectors_.Uint32Vector;
+    export type Uint64Vector = vectors_.Uint64Vector;
+    export type Float32Vector = vectors_.Float32Vector;
+    export type Float64Vector = vectors_.Float64Vector;
+    export type StructVector = vectors_.StructVector;
+    export type DictionaryVector<T> = vectors_.DictionaryVector<T>;
+    export type FixedSizeListVector<T> = vectors_.FixedSizeListVector<T>;
+};
+
diff --git a/js/test/__snapshots__/reader-tests.ts.snap b/js/test/__snapshots__/reader-tests.ts.snap
new file mode 100644
index 0000000000000..961ce87861c3f
--- /dev/null
+++ b/js/test/__snapshots__/reader-tests.ts.snap
@@ -0,0 +1,497 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
+Array [
+  "Hermione",
+  25,
+  Float32Array [
+    -53.235599517822266,
+    40.231998443603516,
+  ],
+]
+`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+  "Severus",
+  30,
+  Float32Array [
+    -62.22999954223633,
+    3,
+  ],
+]
+`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
+Array [
+  "Harry",
+  20,
+  Float32Array [
+    23,
+    -100.23652648925781,
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
+Array [
+  "Hermione",
+  25,
+  Float32Array [
+    -53.235599517822266,
+    40.231998443603516,
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+  "Severus",
+  30,
+  Float32Array [
+    -62.22999954223633,
+    3,
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
+Array [
+  "Harry",
+  20,
+  Float32Array [
+    23,
+    -100.23652648925781,
+  ],
+]
+`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
+Array [
+  "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+  "Airbus",
+  1502880750,
+  Float32Array [
+    32.45663833618164,
+    1.8712350130081177,
+  ],
+]
+`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+  "50fb46f4-fefa-42c1-919c-0121974cdd00",
+  "Boeing",
+  1502880750,
+  Float32Array [
+    38.766666412353516,
+    -4.181231498718262,
+  ],
+]
+`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
+Array [
+  "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+  "12345",
+  "Airbus",
+  1502880750,
+  Float32Array [
+    32.45663833618164,
+    1.8712350130081177,
+  ],
+]
+`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+  "50fb46f4-fefa-42c1-919c-0121974cdd00",
+  "67890",
+  "Boeing",
+  1502880750,
+  Float32Array [
+    38.766666412353516,
+    -4.181231498718262,
+  ],
+]
+`;
+
+exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"row_count"`;
+
+exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
+
+exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `1`;
+
+exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `10000`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_lat"`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"FloatingPoint"`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `29.533695220947266`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `29.533695220947266`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"origin_lon"`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `-98.46977996826172`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `-98.46977996826172`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_city"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Utf8"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"San Antonio"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `"San Antonio"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+  null,
+  "MhRNxD4",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `
+Array [
+  137773603,
+  "3F9HBxK",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `
+Array [
+  410361374,
+  "aVd88fp",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
+Array [
+  null,
+  "3loZrRf",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `
+Array [
+  null,
+  "78SLiRw",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `
+Array [
+  null,
+  "0ilsf82",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `
+Array [
+  null,
+  "LjS9MbU",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+  null,
+  "MhRNxD4",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `
+Array [
+  137773603,
+  "3F9HBxK",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `
+Array [
+  410361374,
+  "aVd88fp",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
+Array [
+  null,
+  "3loZrRf",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `
+Array [
+  null,
+  "78SLiRw",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `
+Array [
+  null,
+  "0ilsf82",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `
+Array [
+  null,
+  "LjS9MbU",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `
+Array [
+  null,
+  null,
+]
+`;
diff --git a/js/test/__snapshots__/table-tests.ts.snap b/js/test/__snapshots__/table-tests.ts.snap
new file mode 100644
index 0000000000000..a7fb9c5a64c24
--- /dev/null
+++ b/js/test/__snapshots__/table-tests.ts.snap
@@ -0,0 +1,1815 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`dictionary file Arrow Table creates a Table from Arrow buffers 1`] = `"example-csv"`;
+
+exports[`dictionary file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
+
+exports[`dictionary file Arrow Table creates a Table from Arrow buffers 3`] = `3`;
+
+exports[`dictionary file Arrow Table creates a Table from Arrow buffers 4`] = `
+Array [
+  "Hermione",
+  25,
+  Float32Array [
+    -53.235599517822266,
+    40.231998443603516,
+  ],
+]
+`;
+
+exports[`dictionary file Arrow Table creates a Table from Arrow buffers 5`] = `
+Array [
+  "Severus",
+  30,
+  Float32Array [
+    -62.22999954223633,
+    3,
+  ],
+]
+`;
+
+exports[`dictionary file Arrow Table creates a Table from Arrow buffers 6`] = `
+Array [
+  "Harry",
+  20,
+  Float32Array [
+    23,
+    -100.23652648925781,
+  ],
+]
+`;
+
+exports[`dictionary file Arrow Table enumerates Table rows 1`] = `
+Object {
+  "example-csv": Array [
+    "Hermione",
+    25,
+    Float32Array [
+      -53.235599517822266,
+      40.231998443603516,
+    ],
+  ],
+}
+`;
+
+exports[`dictionary file Arrow Table enumerates Table rows 2`] = `
+Object {
+  "example-csv": Array [
+    "Severus",
+    30,
+    Float32Array [
+      -62.22999954223633,
+      3,
+    ],
+  ],
+}
+`;
+
+exports[`dictionary file Arrow Table enumerates Table rows 3`] = `
+Object {
+  "example-csv": Array [
+    "Harry",
+    20,
+    Float32Array [
+      23,
+      -100.23652648925781,
+    ],
+  ],
+}
+`;
+
+exports[`dictionary file Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  Array [
+    "Hermione",
+    25,
+    Float32Array [
+      -53.235599517822266,
+      40.231998443603516,
+    ],
+  ],
+]
+`;
+
+exports[`dictionary file Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  Array [
+    "Severus",
+    30,
+    Float32Array [
+      -62.22999954223633,
+      3,
+    ],
+  ],
+]
+`;
+
+exports[`dictionary file Arrow Table enumerates Table rows compact 3`] = `
+Array [
+  Array [
+    "Harry",
+    20,
+    Float32Array [
+      23,
+      -100.23652648925781,
+    ],
+  ],
+]
+`;
+
+exports[`dictionary file Arrow Table toString() prints a pretty Table 1`] = `
+"                                       example-csv
+Hermione,25,-53.235599517822266,40.231998443603516
+                   Severus,30,-62.22999954223633,3
+                   Harry,20,23,-100.23652648925781"
+`;
+
+exports[`dictionary file Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,                                        example-csv
+    0, Hermione,25,-53.235599517822266,40.231998443603516
+    1,                    Severus,30,-62.22999954223633,3
+    2,                    Harry,20,23,-100.23652648925781"
+`;
+
+exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 1`] = `"example-csv"`;
+
+exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
+
+exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 3`] = `3`;
+
+exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 4`] = `
+Array [
+  "Hermione",
+  25,
+  Float32Array [
+    -53.235599517822266,
+    40.231998443603516,
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 5`] = `
+Array [
+  "Severus",
+  30,
+  Float32Array [
+    -62.22999954223633,
+    3,
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 6`] = `
+Array [
+  "Harry",
+  20,
+  Float32Array [
+    23,
+    -100.23652648925781,
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow Table enumerates Table rows 1`] = `
+Object {
+  "example-csv": Array [
+    "Hermione",
+    25,
+    Float32Array [
+      -53.235599517822266,
+      40.231998443603516,
+    ],
+  ],
+}
+`;
+
+exports[`dictionary stream Arrow Table enumerates Table rows 2`] = `
+Object {
+  "example-csv": Array [
+    "Severus",
+    30,
+    Float32Array [
+      -62.22999954223633,
+      3,
+    ],
+  ],
+}
+`;
+
+exports[`dictionary stream Arrow Table enumerates Table rows 3`] = `
+Object {
+  "example-csv": Array [
+    "Harry",
+    20,
+    Float32Array [
+      23,
+      -100.23652648925781,
+    ],
+  ],
+}
+`;
+
+exports[`dictionary stream Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  Array [
+    "Hermione",
+    25,
+    Float32Array [
+      -53.235599517822266,
+      40.231998443603516,
+    ],
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  Array [
+    "Severus",
+    30,
+    Float32Array [
+      -62.22999954223633,
+      3,
+    ],
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow Table enumerates Table rows compact 3`] = `
+Array [
+  Array [
+    "Harry",
+    20,
+    Float32Array [
+      23,
+      -100.23652648925781,
+    ],
+  ],
+]
+`;
+
+exports[`dictionary stream Arrow Table toString() prints a pretty Table 1`] = `
+"                                       example-csv
+Hermione,25,-53.235599517822266,40.231998443603516
+                   Severus,30,-62.22999954223633,3
+                   Harry,20,23,-100.23652648925781"
+`;
+
+exports[`dictionary stream Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`dictionary stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,                                        example-csv
+    0, Hermione,25,-53.235599517822266,40.231998443603516
+    1,                    Severus,30,-62.22999954223633,3
+    2,                    Harry,20,23,-100.23652648925781"
+`;
+
+exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`;
+
+exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
+
+exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 3`] = `2`;
+
+exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 4`] = `
+Array [
+  "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+  "Airbus",
+  1502880750,
+  Float32Array [
+    32.45663833618164,
+    1.8712350130081177,
+  ],
+]
+`;
+
+exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 5`] = `
+Array [
+  "50fb46f4-fefa-42c1-919c-0121974cdd00",
+  "Boeing",
+  1502880750,
+  Float32Array [
+    38.766666412353516,
+    -4.181231498718262,
+  ],
+]
+`;
+
+exports[`dictionary2 file Arrow Table enumerates Table rows 1`] = `
+Object {
+  "struct": Array [
+    "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+    "Airbus",
+    1502880750,
+    Float32Array [
+      32.45663833618164,
+      1.8712350130081177,
+    ],
+  ],
+}
+`;
+
+exports[`dictionary2 file Arrow Table enumerates Table rows 2`] = `
+Object {
+  "struct": Array [
+    "50fb46f4-fefa-42c1-919c-0121974cdd00",
+    "Boeing",
+    1502880750,
+    Float32Array [
+      38.766666412353516,
+      -4.181231498718262,
+    ],
+  ],
+}
+`;
+
+exports[`dictionary2 file Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  Array [
+    "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+    "Airbus",
+    1502880750,
+    Float32Array [
+      32.45663833618164,
+      1.8712350130081177,
+    ],
+  ],
+]
+`;
+
+exports[`dictionary2 file Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  Array [
+    "50fb46f4-fefa-42c1-919c-0121974cdd00",
+    "Boeing",
+    1502880750,
+    Float32Array [
+      38.766666412353516,
+      -4.181231498718262,
+    ],
+  ],
+]
+`;
+
+exports[`dictionary2 file Arrow Table toString() prints a pretty Table 1`] = `
+"                                                                                      struct
+ a0fb47f9-f8fb-4403-a64a-786d7611f8ef,Airbus,1502880750,32.45663833618164,1.8712350130081177
+50fb46f4-fefa-42c1-919c-0121974cdd00,Boeing,1502880750,38.766666412353516,-4.181231498718262"
+`;
+
+exports[`dictionary2 file Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`dictionary2 file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,                                                                                       struct
+    0,  a0fb47f9-f8fb-4403-a64a-786d7611f8ef,Airbus,1502880750,32.45663833618164,1.8712350130081177
+    1, 50fb46f4-fefa-42c1-919c-0121974cdd00,Boeing,1502880750,38.766666412353516,-4.181231498718262"
+`;
+
+exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`;
+
+exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
+
+exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 3`] = `2`;
+
+exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 4`] = `
+Array [
+  "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+  "12345",
+  "Airbus",
+  1502880750,
+  Float32Array [
+    32.45663833618164,
+    1.8712350130081177,
+  ],
+]
+`;
+
+exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 5`] = `
+Array [
+  "50fb46f4-fefa-42c1-919c-0121974cdd00",
+  "67890",
+  "Boeing",
+  1502880750,
+  Float32Array [
+    38.766666412353516,
+    -4.181231498718262,
+  ],
+]
+`;
+
+exports[`multi_dictionary file Arrow Table enumerates Table rows 1`] = `
+Object {
+  "struct": Array [
+    "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+    "12345",
+    "Airbus",
+    1502880750,
+    Float32Array [
+      32.45663833618164,
+      1.8712350130081177,
+    ],
+  ],
+}
+`;
+
+exports[`multi_dictionary file Arrow Table enumerates Table rows 2`] = `
+Object {
+  "struct": Array [
+    "50fb46f4-fefa-42c1-919c-0121974cdd00",
+    "67890",
+    "Boeing",
+    1502880750,
+    Float32Array [
+      38.766666412353516,
+      -4.181231498718262,
+    ],
+  ],
+}
+`;
+
+exports[`multi_dictionary file Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  Array [
+    "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+    "12345",
+    "Airbus",
+    1502880750,
+    Float32Array [
+      32.45663833618164,
+      1.8712350130081177,
+    ],
+  ],
+]
+`;
+
+exports[`multi_dictionary file Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  Array [
+    "50fb46f4-fefa-42c1-919c-0121974cdd00",
+    "67890",
+    "Boeing",
+    1502880750,
+    Float32Array [
+      38.766666412353516,
+      -4.181231498718262,
+    ],
+  ],
+]
+`;
+
+exports[`multi_dictionary file Arrow Table toString() prints a pretty Table 1`] = `
+"                                                                                            struct
+ a0fb47f9-f8fb-4403-a64a-786d7611f8ef,12345,Airbus,1502880750,32.45663833618164,1.8712350130081177
+50fb46f4-fefa-42c1-919c-0121974cdd00,67890,Boeing,1502880750,38.766666412353516,-4.181231498718262"
+`;
+
+exports[`multi_dictionary file Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`multi_dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,                                                                                             struct
+    0,  a0fb47f9-f8fb-4403-a64a-786d7611f8ef,12345,Airbus,1502880750,32.45663833618164,1.8712350130081177
+    1, 50fb46f4-fefa-42c1-919c-0121974cdd00,67890,Boeing,1502880750,38.766666412353516,-4.181231498718262"
+`;
+
+exports[`multipart count Arrow Table creates a Table from Arrow buffers 1`] = `"row_count"`;
+
+exports[`multipart count Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`;
+
+exports[`multipart count Arrow Table creates a Table from Arrow buffers 3`] = `1`;
+
+exports[`multipart count Arrow Table creates a Table from Arrow buffers 4`] = `10000`;
+
+exports[`multipart count Arrow Table enumerates Table rows 1`] = `
+Object {
+  "row_count": 10000,
+}
+`;
+
+exports[`multipart count Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  10000,
+]
+`;
+
+exports[`multipart count Arrow Table toString() prints a pretty Table 1`] = `
+"row_count
+    10000"
+`;
+
+exports[`multipart count Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`multipart count Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index, row_count
+    0,     10000"
+`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 1`] = `"origin_lat"`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 2`] = `"FloatingPoint"`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 3`] = `5`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 4`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 5`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 6`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 7`] = `29.533695220947266`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 8`] = `29.533695220947266`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 9`] = `"origin_lon"`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 11`] = `5`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 12`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 13`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 14`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 15`] = `-98.46977996826172`;
+
+exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 16`] = `-98.46977996826172`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows 1`] = `
+Object {
+  "origin_lat": 35.393089294433594,
+  "origin_lon": -97.6007308959961,
+}
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows 2`] = `
+Object {
+  "origin_lat": 35.393089294433594,
+  "origin_lon": -97.6007308959961,
+}
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows 3`] = `
+Object {
+  "origin_lat": 35.393089294433594,
+  "origin_lon": -97.6007308959961,
+}
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows 4`] = `
+Object {
+  "origin_lat": 29.533695220947266,
+  "origin_lon": -98.46977996826172,
+}
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows 5`] = `
+Object {
+  "origin_lat": 29.533695220947266,
+  "origin_lon": -98.46977996826172,
+}
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  35.393089294433594,
+  -97.6007308959961,
+]
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  35.393089294433594,
+  -97.6007308959961,
+]
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows compact 3`] = `
+Array [
+  35.393089294433594,
+  -97.6007308959961,
+]
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows compact 4`] = `
+Array [
+  29.533695220947266,
+  -98.46977996826172,
+]
+`;
+
+exports[`multipart latlong Arrow Table enumerates Table rows compact 5`] = `
+Array [
+  29.533695220947266,
+  -98.46977996826172,
+]
+`;
+
+exports[`multipart latlong Arrow Table toString() prints a pretty Table 1`] = `
+"        origin_lat,         origin_lon
+35.393089294433594,  -97.6007308959961
+35.393089294433594,  -97.6007308959961
+35.393089294433594,  -97.6007308959961
+29.533695220947266, -98.46977996826172
+29.533695220947266, -98.46977996826172"
+`;
+
+exports[`multipart latlong Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`multipart latlong Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,         origin_lat,         origin_lon
+    0, 35.393089294433594,  -97.6007308959961
+    1, 35.393089294433594,  -97.6007308959961
+    2, 35.393089294433594,  -97.6007308959961
+    3, 29.533695220947266, -98.46977996826172
+    4, 29.533695220947266, -98.46977996826172"
+`;
+
+exports[`multipart origins Arrow Table creates a Table from Arrow buffers 1`] = `"origin_city"`;
+
+exports[`multipart origins Arrow Table creates a Table from Arrow buffers 2`] = `"Utf8"`;
+
+exports[`multipart origins Arrow Table creates a Table from Arrow buffers 3`] = `5`;
+
+exports[`multipart origins Arrow Table creates a Table from Arrow buffers 4`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow Table creates a Table from Arrow buffers 5`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow Table creates a Table from Arrow buffers 6`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow Table creates a Table from Arrow buffers 7`] = `"San Antonio"`;
+
+exports[`multipart origins Arrow Table creates a Table from Arrow buffers 8`] = `"San Antonio"`;
+
+exports[`multipart origins Arrow Table enumerates Table rows 1`] = `
+Object {
+  "origin_city": "Oklahoma City",
+}
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows 2`] = `
+Object {
+  "origin_city": "Oklahoma City",
+}
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows 3`] = `
+Object {
+  "origin_city": "Oklahoma City",
+}
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows 4`] = `
+Object {
+  "origin_city": "San Antonio",
+}
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows 5`] = `
+Object {
+  "origin_city": "San Antonio",
+}
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  "Oklahoma City",
+]
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  "Oklahoma City",
+]
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows compact 3`] = `
+Array [
+  "Oklahoma City",
+]
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows compact 4`] = `
+Array [
+  "San Antonio",
+]
+`;
+
+exports[`multipart origins Arrow Table enumerates Table rows compact 5`] = `
+Array [
+  "San Antonio",
+]
+`;
+
+exports[`multipart origins Arrow Table toString() prints a pretty Table 1`] = `
+"  origin_city
+Oklahoma City
+Oklahoma City
+Oklahoma City
+  San Antonio
+  San Antonio"
+`;
+
+exports[`multipart origins Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`multipart origins Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,   origin_city
+    0, Oklahoma City
+    1, Oklahoma City
+    2, Oklahoma City
+    3,   San Antonio
+    4,   San Antonio"
+`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 1`] = `"foo"`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 3`] = `5`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 4`] = `1`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 5`] = `null`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 6`] = `3`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 7`] = `4`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 8`] = `5`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 9`] = `"bar"`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 11`] = `5`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 12`] = `1`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 13`] = `null`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 14`] = `null`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 15`] = `4`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 16`] = `5`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 17`] = `"baz"`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 18`] = `"Utf8"`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 19`] = `5`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 20`] = `"aa"`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 21`] = `null`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 22`] = `null`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 23`] = `"bbb"`;
+
+exports[`simple file Arrow Table creates a Table from Arrow buffers 24`] = `"cccc"`;
+
+exports[`simple file Arrow Table enumerates Table rows 1`] = `
+Object {
+  "bar": 1,
+  "baz": "aa",
+  "foo": 1,
+}
+`;
+
+exports[`simple file Arrow Table enumerates Table rows 2`] = `
+Object {
+  "bar": null,
+  "baz": null,
+  "foo": null,
+}
+`;
+
+exports[`simple file Arrow Table enumerates Table rows 3`] = `
+Object {
+  "bar": null,
+  "baz": null,
+  "foo": 3,
+}
+`;
+
+exports[`simple file Arrow Table enumerates Table rows 4`] = `
+Object {
+  "bar": 4,
+  "baz": "bbb",
+  "foo": 4,
+}
+`;
+
+exports[`simple file Arrow Table enumerates Table rows 5`] = `
+Object {
+  "bar": 5,
+  "baz": "cccc",
+  "foo": 5,
+}
+`;
+
+exports[`simple file Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  1,
+  1,
+  "aa",
+]
+`;
+
+exports[`simple file Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  null,
+  null,
+  null,
+]
+`;
+
+exports[`simple file Arrow Table enumerates Table rows compact 3`] = `
+Array [
+  3,
+  null,
+  null,
+]
+`;
+
+exports[`simple file Arrow Table enumerates Table rows compact 4`] = `
+Array [
+  4,
+  4,
+  "bbb",
+]
+`;
+
+exports[`simple file Arrow Table enumerates Table rows compact 5`] = `
+Array [
+  5,
+  5,
+  "cccc",
+]
+`;
+
+exports[`simple file Arrow Table toString() prints a pretty Table 1`] = `
+" foo,  bar,  baz
+   1,    1,   aa
+null, null, null
+   3, null, null
+   4,    4,  bbb
+   5,    5, cccc"
+`;
+
+exports[`simple file Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`simple file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,  foo,  bar,  baz
+    0,    1,    1,   aa
+    1, null, null, null
+    2,    3, null, null
+    3,    4,    4,  bbb
+    4,    5,    5, cccc"
+`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 1`] = `"foo"`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 3`] = `5`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 4`] = `1`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 5`] = `null`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 6`] = `3`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 7`] = `4`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 8`] = `5`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 9`] = `"bar"`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 11`] = `5`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 12`] = `1`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 13`] = `null`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 14`] = `null`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 15`] = `4`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 16`] = `5`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 17`] = `"baz"`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 18`] = `"Utf8"`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 19`] = `5`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 20`] = `"aa"`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 21`] = `null`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 22`] = `null`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 23`] = `"bbb"`;
+
+exports[`simple stream Arrow Table creates a Table from Arrow buffers 24`] = `"cccc"`;
+
+exports[`simple stream Arrow Table enumerates Table rows 1`] = `
+Object {
+  "bar": 1,
+  "baz": "aa",
+  "foo": 1,
+}
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows 2`] = `
+Object {
+  "bar": null,
+  "baz": null,
+  "foo": null,
+}
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows 3`] = `
+Object {
+  "bar": null,
+  "baz": null,
+  "foo": 3,
+}
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows 4`] = `
+Object {
+  "bar": 4,
+  "baz": "bbb",
+  "foo": 4,
+}
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows 5`] = `
+Object {
+  "bar": 5,
+  "baz": "cccc",
+  "foo": 5,
+}
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  1,
+  1,
+  "aa",
+]
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  null,
+  null,
+  null,
+]
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows compact 3`] = `
+Array [
+  3,
+  null,
+  null,
+]
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows compact 4`] = `
+Array [
+  4,
+  4,
+  "bbb",
+]
+`;
+
+exports[`simple stream Arrow Table enumerates Table rows compact 5`] = `
+Array [
+  5,
+  5,
+  "cccc",
+]
+`;
+
+exports[`simple stream Arrow Table toString() prints a pretty Table 1`] = `
+" foo,  bar,  baz
+   1,    1,   aa
+null, null, null
+   3, null, null
+   4,    4,  bbb
+   5,    5, cccc"
+`;
+
+exports[`simple stream Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`simple stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,  foo,  bar,  baz
+    0,    1,    1,   aa
+    1, null, null, null
+    2,    3, null, null
+    3,    4,    4,  bbb
+    4,    5,    5, cccc"
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 1`] = `"struct_nullable"`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 3`] = `17`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 4`] = `null`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 5`] = `
+Array [
+  null,
+  "MhRNxD4",
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 6`] = `
+Array [
+  137773603,
+  "3F9HBxK",
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 7`] = `
+Array [
+  410361374,
+  "aVd88fp",
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 8`] = `null`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 9`] = `
+Array [
+  null,
+  "3loZrRf",
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 10`] = `null`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 11`] = `null`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 12`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 13`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 14`] = `null`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 15`] = `
+Array [
+  null,
+  "78SLiRw",
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 16`] = `null`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 17`] = `null`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 18`] = `
+Array [
+  null,
+  "0ilsf82",
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 19`] = `
+Array [
+  null,
+  "LjS9MbU",
+]
+`;
+
+exports[`struct file Arrow Table creates a Table from Arrow buffers 20`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 1`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 2`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "MhRNxD4",
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 3`] = `
+Object {
+  "struct_nullable": Array [
+    137773603,
+    "3F9HBxK",
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 4`] = `
+Object {
+  "struct_nullable": Array [
+    410361374,
+    "aVd88fp",
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 5`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 6`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "3loZrRf",
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 7`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 8`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 9`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    null,
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 10`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    null,
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 11`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 12`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "78SLiRw",
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 13`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 14`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 15`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "0ilsf82",
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 16`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "LjS9MbU",
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows 17`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    null,
+  ],
+}
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  Array [
+    null,
+    "MhRNxD4",
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 3`] = `
+Array [
+  Array [
+    137773603,
+    "3F9HBxK",
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 4`] = `
+Array [
+  Array [
+    410361374,
+    "aVd88fp",
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 5`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 6`] = `
+Array [
+  Array [
+    null,
+    "3loZrRf",
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 7`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 8`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 9`] = `
+Array [
+  Array [
+    null,
+    null,
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 10`] = `
+Array [
+  Array [
+    null,
+    null,
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 11`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 12`] = `
+Array [
+  Array [
+    null,
+    "78SLiRw",
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 13`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 14`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 15`] = `
+Array [
+  Array [
+    null,
+    "0ilsf82",
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 16`] = `
+Array [
+  Array [
+    null,
+    "LjS9MbU",
+  ],
+]
+`;
+
+exports[`struct file Arrow Table enumerates Table rows compact 17`] = `
+Array [
+  Array [
+    null,
+    null,
+  ],
+]
+`;
+
+exports[`struct file Arrow Table toString() prints a pretty Table 1`] = `
+"  struct_nullable
+             null
+         ,MhRNxD4
+137773603,3F9HBxK
+410361374,aVd88fp
+             null
+         ,3loZrRf
+             null
+             null
+                ,
+                ,
+             null
+         ,78SLiRw
+             null
+             null
+         ,0ilsf82
+         ,LjS9MbU
+                ,"
+`;
+
+exports[`struct file Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`struct file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,   struct_nullable
+    0,              null
+    1,          ,MhRNxD4
+    2, 137773603,3F9HBxK
+    3, 410361374,aVd88fp
+    4,              null
+    5,          ,3loZrRf
+    6,              null
+    7,              null
+    8,                 ,
+    9,                 ,
+   10,              null
+   11,          ,78SLiRw
+   12,              null
+   13,              null
+   14,          ,0ilsf82
+   15,          ,LjS9MbU
+   16,                 ,"
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 1`] = `"struct_nullable"`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 3`] = `17`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 4`] = `null`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 5`] = `
+Array [
+  null,
+  "MhRNxD4",
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 6`] = `
+Array [
+  137773603,
+  "3F9HBxK",
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 7`] = `
+Array [
+  410361374,
+  "aVd88fp",
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 8`] = `null`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 9`] = `
+Array [
+  null,
+  "3loZrRf",
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 10`] = `null`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 11`] = `null`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 12`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 13`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 14`] = `null`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 15`] = `
+Array [
+  null,
+  "78SLiRw",
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 16`] = `null`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 17`] = `null`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 18`] = `
+Array [
+  null,
+  "0ilsf82",
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 19`] = `
+Array [
+  null,
+  "LjS9MbU",
+]
+`;
+
+exports[`struct stream Arrow Table creates a Table from Arrow buffers 20`] = `
+Array [
+  null,
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 1`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 2`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "MhRNxD4",
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 3`] = `
+Object {
+  "struct_nullable": Array [
+    137773603,
+    "3F9HBxK",
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 4`] = `
+Object {
+  "struct_nullable": Array [
+    410361374,
+    "aVd88fp",
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 5`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 6`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "3loZrRf",
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 7`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 8`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 9`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    null,
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 10`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    null,
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 11`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 12`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "78SLiRw",
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 13`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 14`] = `
+Object {
+  "struct_nullable": null,
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 15`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "0ilsf82",
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 16`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    "LjS9MbU",
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows 17`] = `
+Object {
+  "struct_nullable": Array [
+    null,
+    null,
+  ],
+}
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 1`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 2`] = `
+Array [
+  Array [
+    null,
+    "MhRNxD4",
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 3`] = `
+Array [
+  Array [
+    137773603,
+    "3F9HBxK",
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 4`] = `
+Array [
+  Array [
+    410361374,
+    "aVd88fp",
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 5`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 6`] = `
+Array [
+  Array [
+    null,
+    "3loZrRf",
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 7`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 8`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 9`] = `
+Array [
+  Array [
+    null,
+    null,
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 10`] = `
+Array [
+  Array [
+    null,
+    null,
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 11`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 12`] = `
+Array [
+  Array [
+    null,
+    "78SLiRw",
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 13`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 14`] = `
+Array [
+  null,
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 15`] = `
+Array [
+  Array [
+    null,
+    "0ilsf82",
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 16`] = `
+Array [
+  Array [
+    null,
+    "LjS9MbU",
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table enumerates Table rows compact 17`] = `
+Array [
+  Array [
+    null,
+    null,
+  ],
+]
+`;
+
+exports[`struct stream Arrow Table toString() prints a pretty Table 1`] = `
+"  struct_nullable
+             null
+         ,MhRNxD4
+137773603,3F9HBxK
+410361374,aVd88fp
+             null
+         ,3loZrRf
+             null
+             null
+                ,
+                ,
+             null
+         ,78SLiRw
+             null
+             null
+         ,0ilsf82
+         ,LjS9MbU
+                ,"
+`;
+
+exports[`struct stream Arrow Table toString() prints an empty Table 1`] = `""`;
+
+exports[`struct stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
+"Index,   struct_nullable
+    0,              null
+    1,          ,MhRNxD4
+    2, 137773603,3F9HBxK
+    3, 410361374,aVd88fp
+    4,              null
+    5,          ,3loZrRf
+    6,              null
+    7,              null
+    8,                 ,
+    9,                 ,
+   10,              null
+   11,          ,78SLiRw
+   12,              null
+   13,              null
+   14,          ,0ilsf82
+   15,          ,LjS9MbU
+   16,                 ,"
+`;
diff --git a/js/test/arrows/file/dictionary.arrow b/js/test/arrows/file/dictionary.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..34d41db1f200159406c459b8c076d5ce0c3d30ec
GIT binary patch
literal 2522
zcmeHJ&1(};5T9+*G{mHqHnv(ta_B)s!H*ykC?w(`f?#L`@#bQlX&@hEla^FG<l;rV
z_&@kZc+N?|lLzTZaumda;Mw^5y?s;GjipqtoiK0S%)I%2yPL|!#)C)KhchAzB9fJP
zNl8x9c8^77tX#Z|l_ep`07*UISzsP(9&`!Tf>YFyCl*NI8<1N7r49A1Uz3Rk-wEvR
zLio|iYm>=D?S_ycE00v|-a0$whwl6poGAiD5*{f-vL43ThfM(<#?BTZKnj_r@FRGZ
zuGRg%zeb+Aq?OUn8Q|rH>9$L?ZnM)b^@3)7y?FD+Ey%;#mt^rTwKFFovhl8Q1!_($
z<jtJ=pzH(TKK3A90m}-|Z<x}(PTw?4P-=GSX0ZOuZ}kjfUO>lJWEtL)h|5{AXHOih
zg1!KX%z6W?>@oA;tk{pQh?7R_wCw;VMkrXGRg>48IPE+nwBy|A<L8$TrndV0F}Bb!
z{+zV|>(O=$|I45qzimpJ{h--txcvLUj$sb`iJStOf#2S4nX6BGyYdM<>mX!N<D#4g
z&kD7VR)xjgNo>+M#q1dO&)hQLnOoB4t^Hvy6ry{T@m@tC>`gegsGP9~nlJaXo%T6L
zm4jK(9FSI3KM3}!CTKUi9kXe6O|aA3f(!MS#eC;+_re+Dyz_BTePX)8eVhsJQg~Ri
zKQP7SunOaH-c&-=A9<}je!f5@ARhBlD4vSTCxQ_7b2}8<C&g`reT(ODT%I>B3OXsJ
zo^z=DIUK$SRQ|LxuKaj7dh@+-=jFG-NTghDp@73?R(3W=%`Do3c>m}kswb~!;&Po?
zTz=Hd;=TFTnQ30kku!_uaa=w=WBOGeN+}~Kud`5I=O^;6vby(vr24GAVc{EsZx~`3
zxCLa~*UpghB~(KqWflh5cJi|2!iAz&F6qxsUX1AZ4rPz@ZAwZT&gbMSdh!)@&(D8-
MMaQ^*75ks=H#cC^v;Y7A

literal 0
HcmV?d00001

diff --git a/js/test/arrows/file/dictionary2.arrow b/js/test/arrows/file/dictionary2.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..1537f54db79eeeb2243e91846994737099a8ea96
GIT binary patch
literal 2762
zcmeHJJ!n%=6h5zwHMFU=lGb7~Jg`F%14;YL9~5cn(4k<d0TmpY=Dk+~wh75A)XkxS
zgF=T66&w^qGP?^M96Gq^;NU3YAiB6}{J#6{H8CYq*Pig@+<Wdl=X~efoA)xmx_aYQ
zT4c9hBrc)E<eUu1utYJ9DN~<}P+Nm)9+ij;V#;-bVgV7L_`B7bfF`DZ1mcPeTN6sR
z_u1HINuTn0(kIeHfM3|j7za8=5#u%B2oM>us7;==KdlyX+rSubWy7m(7FMd2t)S4T
zSKK*gb|wQmdAeSsT(8uEYQ63MLgzQ}Vu)vWr{I$VMEb2z3z&idKz^U$YlF!b9RnuF
z0D3Wi55-b?$}I&YHbJdLl#Ov1gLJvFtMlH*xTb+@R)DeLw%iJ9WvENQVkz*7l})d}
zLBaL>()RseE~syN$LsCny^mbv{T!OTr%WR@lx5?K{WP&L#C@<OA(O}%2j>x-eN^3%
zJ*=@;D*$;m0g*G-D}0kP?@Uzfj;NT78wZAf;)9x3xEa(dTN}sDhF2s3m_9G6x-Ge@
zINDN#HoRGB9MRcJo-z4+|N7?X+nu3V40ZmqDnF(cIRx<MesX|<e*%6NWn5)sbrHpr
zv4~|2*0zK(g&eN}TxYCrnqw5+JbUI5^o?_#IOm4*FYIO4D=vNc;mr_f;ONS1B;+H&
zUQuG;cLDup_u12FWi?;=H;Ky<#V%%JuK)Wmh<R8Sd&IbFKzQ;PKv++=@C-X8tjRnJ
z+AGXOAvYuR?Tl~ioq1TZ@tLpG*S8xBRgV_}@(uyo^^fC8G=|d&>u;K1$iO=y{m1LG
z;oq5`cdlD{)z@uTIpA!#nmD!|EE$8=_q6VgQ`7cNUNr6W*#A)5P2crY9{1S4*Ta6G
z?WPV++52uDC!O;icZ)f1qq#l>n4XnV{<@R(^GQGFuO}TRHJdDDoKiBI%edK0I_>8?
z{~GsXd?)GS5tnpk%IRc2oi8U->6vss>y%wLmHHQ7H{HyCT;9J!nE824@Yx4z;pOhn
z>!YKS3$NclI&9xHb}9U=@Vtp>U>RWOJzeCH&eMd=h+5Z)K4~L;LpgnCvUR8L%<$Q~
Yf?fnbcz;gcnJ0Z`9^!X5ul~dS1zMm*_5c6?

literal 0
HcmV?d00001

diff --git a/js/test/arrows/file/multi_dictionary.arrow b/js/test/arrows/file/multi_dictionary.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..113d30da7856560455dc0b63229671825d1a9fdd
GIT binary patch
literal 3482
zcmeHK%}Z2K6hALb2R}+fEwaUcE+ib78K3h-l8Ot1Br%}^7mhRU>3}-J_ymI%Eec!|
zxNuR>!XUzBv}zNyEN~I%qCX&_MO2HH)%5$ld$02t5n4!_JaXQ-_uO;O$M2r|<~cGo
zb?rt>#I6@<6Ok4<B@NOlA%BdAobdHxHDOJlua_oYa;(ULz;@aCS*e_ZB=>^f*Wn?p
z){ZYljNKyaEciW%)nTm7oDkW>;t$N}=d<=Rjqxn_Eg0EZl*Y%4UzG~uhrylTi}Ox#
zDKl9tEPI)iav?ut55yDDGuOPck}DTVUa?%=|6S)`{LR3h$vuiaDX>VLr6f^M_%OeI
z*oKjb4aqtbH2{wW{E;kQ_U5x*mOx-e+_*+-`e3>cvZv$T#JHpxR+fjc$?eH+RV!Vc
z2ajbvXS%TDWH`v=9XGqW=nZ-0Rp(&6!Ms1}yuSqUUW0Fj3gbaMjc*KPvQnwsfg~>D
z;6?BQu*gYk9Z!?rCD?Iq2e=tLeY@mju6yOe^85k(*$NlH>^0)R_#t@O@bd`EI;6og
z4!^Yf^6BNn*K5rK7)l>>5EkQY!j?W54+8PS@ZAlJI^eqv{TlLf+Q1rXm;@8oZ7^$i
zh9_%KeNF3nAA?{m6BF03{B12`J`R@5_&J7U8q(Cllj-mys54?d3#o<fQ}1?rFvJSu
zH~R50wOF@@{j85UKw$p`)Nxk2^i5^H#GJm!S2eS?ag0gCyC2MT`uePM3?UNxk+lkN
z<D4DJxyktlY}r4BJooaCoh-6~R?87c&>omMkXn%68Q@R!iD{p&H4ptBL34*-Ww2&m
z)$?sap6VQUZ|QdyY;G8aem#^y;u!`>)uhffsAnt^b2G%fLH|Zi+zhd1vuC7Go?BfR
zEjny&&{5Q1?Sp?)Ke09RH}&&9ZT7_C1NLAdnMy~MhkAflJ^wEJdUT>Y)%m|NL4Yv(
zS?UhfH;Dv-__yxwsSXZ1-&(yN1KZ4584qDS7&i(n7qqPPQ__0&dyv-jq}Kkf)(7jL
zYF+DV?KiaE^r@c8qeJYs4$*IGy{VhiMsRH&M?J5_g>lV&F_OAo@HhvcY}B2zlWscV
zrrf!RZAS+p*@T^qBvXleG7*cpDaRe8O~SP!E`HJ?c03o0q+{t^BpQpy(n&j)&qt$G
z{|eRpxdz$HzxHh2BAfYXm4-atwML)b`*yj#y?gZe&iyyl-X)6a-NXJN_kkC{bUn<M
z04SuvtoF9Tdsu_=L7DBp`-XXWK(%_MJ^rR5>V)q2o9g(Rir?qFufgxb|LvP<1D9dM
H|DWs+-cqT+

literal 0
HcmV?d00001

diff --git a/js/test/arrows/file/simple.arrow b/js/test/arrows/file/simple.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..838db6dc8eda5376aaaf977c5f96c50397582a5f
GIT binary patch
literal 1642
zcmeHIJxc>Y5S{x-E`~!=1UW$=g@vVpg@xdyvk-(J*x1}DwgF44l$Ms3{sezQ3QK>C
zrQP}7+>FLVB_!H8VfN$A?7W@X-AjKkI62#uh2=aD>0os#tJ0K)v~-Ln2o)D;D;5K4
zL)k~rXXvHPPphiJg2plCThPpK`t2A-4oa*);8@d=0I^%(73C1=$r12CLggrn8+oi_
zM>%{Ly9?zWLYtGTmZnBxEL!idE=T<PPw}p?K2yI088dUnqcUON|0D<ZzoB;SF@+{j
zm7s_274dtPM~qBp^Zj29hq8zI*6~^6HLG!|>=K1qD@_LLSm6wo7*Sa3Cn)RIfVw0Y
zfD-loklcoj``mlSN9H$;kH5i@h)QmDkCPidn!?9Z{xtIr`2NlOKIa)y-0pK?n{1X4
z=ORDvi*w8ya(!OMAN^hqzczRB8t+;^>-8J_tfwpUUzw%(Go&cWvb=yS$iRD_;^pu~
prFaE=t<2^C?KYHW$}@hjgWmT8pS(n{|9`)h`+nV`u6_NFc>}#ma&iCw

literal 0
HcmV?d00001

diff --git a/js/test/arrows/file/struct.arrow b/js/test/arrows/file/struct.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..3d2c018e6c27c95057f2a18395a9dc6031e6b201
GIT binary patch
literal 2354
zcmeHJJ5L)y5MCPt7wZBk2y#J?B~nBw5RVv4cSZyvVqWKjR}<Ng3!qal*iI-Sg_1T>
z6s4k~Oa4I`{{(&lB_(a7NY3}|?Kp^s{6L5%V|}|jbMx4po7-{9b?24_Rns&dQtAP$
zZPlULRI9S|8bc_o<w`}hECNv!;0zNz7b}$txch&=#{v40=O<W)flmN^9z*vi&~HtH
ze+<L`>bC#`0A=(+is+wGdDvKpXyMleIejtO0puY>U<_|asV;b<MmtVIFI^Z39%KR|
zeUoQA+B^Yz0Fnh83zmUz6^cW43jcA$Jv$!<dFDPMh(le*@5Edv?v~byZ{C-DKb!O4
zsu&U!#AaT*O<rxiGu0B-=Pomenxf!s__TC>e?rC_v<DCM$a7;G$n`&R%u?jS%PjJJ
zqjynWzDABiY1%i>tA32!F>+3PZ?aFWNwxU?K&|QVQPY-z;pc@f8Ls-qN37n%QY`%I
zxY&rxo9<s?F;5-9Lj%^L)FYO+Ml&1k?B2w);mNTV<9jc&i|fgxmml`GS9e{no}fK-
z)VBV>w6m*goA-#&UDS5j1ee;DZ_$xWVP-wcK6z)5gufTCRbFbJ;&0~}qu0pfG@{&q
z?#j*&Ubi*rclBPXsd}vfYg-^S&f4w+o&s!FH)vU~zCB8165A>F`SjF!_TAd@Xfl)D
zbU%K`O?jhpD~ZxZ!h2O1*z^ltawxr(8Jo$yzOWVE-8OE2b_g3cIlB=%&N4;^kpA=4
j<eA?SNop`}iY+ehfz9d?{qM=u-d|@={*HX6^nazl^uV^z

literal 0
HcmV?d00001

diff --git a/js/test/arrows/multi/count/records.arrow b/js/test/arrows/multi/count/records.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..00d883762d36959727cd26f5c3e3e4dfafeba981
GIT binary patch
literal 224
zcmbQkz`!5^#2~=KAjZJPz{<b@WbuFmn3)(PfRqD>4F+65J`V#6Pz4u|?*qgFK+Fi1
ggAhzW>OT-53Bj4@;tp^bh#AP{z@!A!VJz}!0H3Q0ZvX%Q

literal 0
HcmV?d00001

diff --git a/js/test/arrows/multi/count/schema.arrow b/js/test/arrows/multi/count/schema.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..dfd24e9e0018c471a053b8621cb7ea8b46a5bd9e
GIT binary patch
literal 184
zcmXYqF%E<<3<Ng_?hHcc3W_TzD0l!5;SUt_AR!Rn;(6DFadeJ$?QAs8E$M@5HF~L#
zr5OuZ%B3o0IZ=hY2lPcPZcuM%-_oWG1D^2Ve|IKBBkGWs!5F0fM};2y#Ta|G8z*P4
RzXiyWzHisOZ0C9~dp%Qi3m5<Z

literal 0
HcmV?d00001

diff --git a/js/test/arrows/multi/latlong/records.arrow b/js/test/arrows/multi/latlong/records.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..563d12d175d4e39b10386f98c5614cb86a493424
GIT binary patch
literal 352
zcmcb^z`!5^#2~=KAjZJPz{<b@WbuFmn3)(PfK&sB4F+65J`V#6Pz4u|UjxJfK+Fo3
zgAgo0>OT-53Bj4@;to(5m^h3N(||6HEYAc~glsQJ4>}(t);5XP35J<m-#CIWNSs(O
U+4%4w7;gV?<PZoGs~4mW0F=Zhr~m)}

literal 0
HcmV?d00001

diff --git a/js/test/arrows/multi/latlong/schema.arrow b/js/test/arrows/multi/latlong/schema.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..638b2ab622f8ed55ff85783a2c1ad54321b90021
GIT binary patch
literal 264
zcmZ8cI|{-;6r7jDH3Sh0ixep=EImMu;4Q>T0wDqM7?xI^BFD2^XWnihKA86z-kaHl
z0XqO2Y!R?R?LCtf&I65e6{&_$Er>DqON^1rWzV@I_L<j?W@#(lX%A1<0}L-#qmne|
z`*0hk%Q#QEZ|N7X=8JSKJwYC4<|FazapK$(DT3by9}J)DRg<l#slWYuULWc53;*jL
ADgXcg

literal 0
HcmV?d00001

diff --git a/js/test/arrows/multi/origins/records.arrow b/js/test/arrows/multi/origins/records.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..49a8c407e176e107a9007cbed26b87e3c84608ed
GIT binary patch
literal 224
zcmb7+Q3`+{5JV@-2odTdqT_e~kJw%7t)WPN4UGGCX7Q4=V^h?jSzZeiEf}3nMlvo6
l>`@aJfj#35`_jFSW8A2V8h-QC8fk$0e`;lU^$MH+=mDH!3Sj^M

literal 0
HcmV?d00001

diff --git a/js/test/arrows/multi/origins/schema.arrow b/js/test/arrows/multi/origins/schema.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..0d10fb0e2d1354e5a147f49b5283632d4fdde04e
GIT binary patch
literal 1604
zcmeIxJ!lj`6bJCRGapA1Bua|d$Pxq#Q7lBQLUI>l5-+(k7firH-tE4-9d35!oY{?D
z7p!cw6C1I!5^U^j#6|==vD3yTSSZ+OBj<lMIh25nwR8M-zINWcnN7Y4A*R5W32|7A
zh+$F6+jta;V$x5Dv3zVOZwcY+0!+h{IF7jyF^bib=<SVRLKKGY;<*KWYzfYRk2Q%E
zgIIoVAIp~#Vw4cc?~hUl@Hhk);RNjE;`jGIAwKu}{SsD}@EbvYZ(Wf0hcM#%1yK~^
zsEw6vsc(cb?TI(&|3357{+TDRJ4&&b&+YgPP)gz`4CS%>IR?4<b4+7}pBck=1U|nx
z7!;p7QtS`<f7S=b=Zbr?gYiLsQ2SZG_ndDWD5L26^BjW!PQrPZg%!98*MUJn7kY39
z?!iNN3{T-Xyn?sz9zMb+*o80f9e%<vk{O3-n1NGp7B0XnT!uPqK?@XE=)n%$hX?Qo
zp1?DB0WaY-yn_$03t!;}^kEG5bOfeh22R8O^6xiLqmCjS)$oi>LO})pAhm>Is_B%h
zG;7ATjOuK%OM^P8Bq%e5aoyO|FRRB=QN*enORC3)wY*)IY3iDpZO3(Mwq?XR<aJUk
z+wnSUvzVR<R(ot+gLJSUxov!7UZ+M&Q#F}%+fpfRFr_If%q?2dp>$#X;)-z?RB2G9
z4(G`>D72I?0^sG85Q-X@_K`Ly>C#4~93G)FdR{fNn;Ba`lXP!B!4Z`*^O;Mf-jq6W
z#$s*Qab;-}soOCnYvkfQr?Qcb%UP>M%9JyU0G%>Yk%_TfQM}FmMk+>SNi%a5W4oCP
zDzTM_y+BHlTVTD-c7aoCWO_;AK3#qvRmvi^5iY6GH67BtWRQ^2OJo&tU|orEG$h(v
z){!JlU8AI1c1m_v2)P=)U*&05$pzQAO|EJjZlleLT{UYqajGi5jpAS(%~cusbgz(h
ict8K@)(@CUEJJFWRa|<EkIr_DZv{207;paN&+Hd!R}{7Y

literal 0
HcmV?d00001

diff --git a/js/test/arrows/stream/dictionary.arrow b/js/test/arrows/stream/dictionary.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..17ca48b3a97f51ad016faa004cc53a2a5e809eec
GIT binary patch
literal 1776
zcmbVLL2DCH5T0$)G{mHqHnv(t_Rxcff@l#56jJdJ!9$>4yt$ZX2qYWXY)dL0a`6ZF
zAN&!Xds6V=u_x(K5D$W9*YA7#rmU}Osk6+RH}l@id^7WQHYc(qB6V4miZrBZ_gG}k
z+9jx2TN#!bm@)ug1UIoZAy?onxWzN-<d71vL5(Fy`tW@CGs^<RE@A%w#*fb4&9Y4W
zX0QQOK&gIjouBG6cmD?NY=T8f0VPwi9_BiPPlyOd&KD+71xPFSNP(3by(o<~sPn9B
zZH#jPe7$eRqgHo3h||_28T7U`@7%r%dtUpBEdQZ)Vj=@gaEmKgF%5y6m{Lggf%FJ_
zh#<tW29Els^)OCN-z2R;+%way=h1Ltkn;*Uz9y@PRz_aViamSkWC!vUSY$q!Vr7qs
zgR^2kz93H(xvRDVoERyzI;*Cxm^kewEcD~t8RN(252m~R@(H#$Vg3c{12v%U5#p~x
z#?i>McG6@J_Z|K;*)_z#PvjifOrz0@p}F~NvL~OQvkp=nH7?6#=&VrtXpgYCH;GLZ
zw}_5;f5%pXPHbh1Tl>RaC>8f=<GU(S-kW@GMLTnm6fgJm^<ysZz|lNp15BSzlqCBd
zlZ*!A*zA}+lk8502%z!mnC>E8UOr*mcQKA@FP*IK5O?yIlwVft$FA7?tv=t*cS<V8
zqpo%P_d_0n{G8Wv^NGCtA4q<jd%4;ECvGe3n~&qHI^Ua5IxCfd3#$4hI9vs*e%=Mx
z{{Hyj&A0IW>#x%Tk#_rEb89=F)8<y}gWp4jD4tW-JHB1#=G#x3o8O=Rotxq!M$XO0
faaP@*GvjIum9&vm*MCTLouA0N?e+cln3>2g$Le<8

literal 0
HcmV?d00001

diff --git a/js/test/arrows/stream/simple.arrow b/js/test/arrows/stream/simple.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..2c68c0e44b0af2ed78a47edbd038ae8fa8ab574b
GIT binary patch
literal 1188
zcmcIiJ5Iw;5M2La6IsYbgd7PZ7aV|so>x*(asiu&HWDROj*yaLaD)_;9EOt4%&d3$
zIDjGrHd;TwnSC>FH(o>{m57vbC=<z~(04qdSaFdlIFS_0)VLHYBSAhR7f52)ec$6?
zc7^v-WNkR>&J3d$Bh?>poGB!NdI5h?f00HmfkzUnN7vrV<Di`c)KGhdq-Sh<dJQpA
zC2Emf<6MCL!xz0foUgSm*X}CSqb3{o`+xS}{g0LB9ZO`6)QB1^uh4I-j})Dd^ZVZ}
zmvW9d5Bi+-k<GePcS&NMl{SNOoOun#XZ{3)bIV9s<XB;ed4EK(SI2$ttK%!H8{?ZF
za5SP3jCY-2_-cf2M)jom4fy?ye&6#yWH|48Vkdm;5$;8QelPB^tK@vw!e8rNF21L$
fFyJ#=x2yibpY!x&{|DPP|Aw?}*L63DJsjQuh?Gj^

literal 0
HcmV?d00001

diff --git a/js/test/arrows/stream/struct.arrow b/js/test/arrows/stream/struct.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..4e97b7084f6b7c62df23acfe9e200661946314e3
GIT binary patch
literal 1884
zcmd5+O;1`u6us|>_(~{cQ4I?<B)Di?G*n8(9U7~~qNY$AbYnn$)#jys0P4b|F1zif
zOIPmnAGqNs@DsZ1vfCz2@SJ%w)HE8vrqN5z%-l2gW9D+-6fKdch=e5~LFpAotua`(
z;zdH(6A54^r1t`tDR2cZQf)Sy&>sKBIu0-%dG3D)Y~UDR>;`PleEZ%U^bsHi(7p#q
z0^}Hj5>P*pUHCYN=-?Lw&sfa%1-yj}%;AoSJVi8G^r=%vp_8Ln_B2Murp|oyc?Jvu
z6bC*IJpI^W#lihR{5bOdyq@O;Z5_l2DqH4%0^3S;zqVdo`&9G1QrX*-7z*I$W_<^A
zeZ%Tx(qpT8*I6VbA?RVm^y_>9jy34>H`T~{;~1#*AIa*a;f-!)!=GrozxnztJg%(d
zMDMraXg;fXPW<1@Ik~2^)1UihcOHlC?CCpxe*3k+RbuLuGkjW$*_VxKHzEJ^{3aLc
z6ao4**o&qgvHWkcu$5mtntqv@nV6e8dQ*D8k;%BbDQ{<WKks&ubg+ZoRv(xOc5`p@
zJuz}0y<OMIO>diTF>4Nmnf+|e$v1;y^hdth(oOG^{&ro_cHK<bZj8IIUD-S5bK6#a
zwce$+rneffw+&2>v$qq#D1c{F0n>w%$wGQ(GXHvhcBAxRy*QpJ<hJvN2jy9Jd}$?J
Z+e*9dD#>lH;%3Hj9}5!;<>ec0a0N6*h?f8W

literal 0
HcmV?d00001

diff --git a/js/test/reader-tests.ts b/js/test/reader-tests.ts
new file mode 100644
index 0000000000000..a7f9f4110237f
--- /dev/null
+++ b/js/test/reader-tests.ts
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { readBuffers } from './Arrow';
+import arrowTestConfigurations from './test-config';
+
+for (let [name, ...buffers] of arrowTestConfigurations) {
+    describe(`${name} readBuffers`, () => {
+        test(`enumerates each batch as an Array of Vectors`, () => {
+            expect.hasAssertions();
+            for (let vectors of readBuffers(...buffers)) {
+                for (let vector of vectors) {
+                    expect(vector.name).toMatchSnapshot();
+                    expect(vector.type).toMatchSnapshot();
+                    expect(vector.length).toMatchSnapshot();
+                        for (let i = -1, n = vector.length; ++i < n;) {
+                        expect(vector.get(i)).toMatchSnapshot();
+                    }
+                }
+            }
+        });
+        test(`vector iterators report the same values as get`, () => {
+            expect.hasAssertions();
+            for (let vectors of readBuffers(...buffers)) {
+                for (let vector of vectors) {
+                    let i = -1, n = vector.length;
+                    for (let v of vector) {
+                        expect(++i).toBeLessThan(n);
+                        expect(v).toEqual(vector.get(i));
+                    }
+                    expect(++i).toEqual(n);
+                }
+            }
+        });
+    });
+}
diff --git a/js/test/table-tests.ts b/js/test/table-tests.ts
new file mode 100644
index 0000000000000..c840299155af4
--- /dev/null
+++ b/js/test/table-tests.ts
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Table, readBuffers } from './Arrow';
+import arrowTestConfigurations from './test-config';
+
+for (let [name, ...buffers] of arrowTestConfigurations) {
+    describe(`${name} Table`, () => {
+        test(`creates a Table from Arrow buffers`, () => {
+            expect.hasAssertions();
+            const table = Table.from(...buffers);
+            for (const vector of table.cols()) {
+                expect(vector.name).toMatchSnapshot();
+                expect(vector.type).toMatchSnapshot();
+                expect(vector.length).toMatchSnapshot();
+                for (let i = -1, n = vector.length; ++i < n;) {
+                    expect(vector.get(i)).toMatchSnapshot();
+                }
+            }
+        });
+        test(`vector iterators report the same values as get`, () => {
+            expect.hasAssertions();
+            const table = Table.from(...buffers);
+            for (const vector of table.cols()) {
+                let i = -1, n = vector.length;
+                for (let v of vector) {
+                    expect(++i).toBeLessThan(n);
+                    expect(v).toEqual(vector.get(i));
+                }
+                expect(++i).toEqual(n);
+            }
+        });
+        test(`batch and Table Vectors report the same values`, () => {
+            expect.hasAssertions();
+            let rowsTotal = 0, table = Table.from(...buffers);
+            for (let vectors of readBuffers(...buffers)) {
+                let rowsNow = Math.max(...vectors.map((v) => v.length));
+                for (let vi = -1, vn = vectors.length; ++vi < vn;) {
+                    let v1 = vectors[vi];
+                    let v2 = table.getColumnAt(vi);
+                    expect(v1.name).toEqual(v2.name);
+                    expect(v1.type).toEqual(v2.type);
+                    for (let i = -1, n = v1.length; ++i < n;) {
+                        expect(v1.get(i)).toEqual(v2.get(i + rowsTotal));
+                    }
+                }
+                rowsTotal += rowsNow;
+            }
+        });
+        test(`enumerates Table rows`, () => {
+            expect.hasAssertions();
+            const table = Table.from(...buffers);
+            for (const row of table.rows()) {
+                expect(row).toMatchSnapshot();
+            }
+        });
+        test(`enumerates Table rows compact`, () => {
+            expect.hasAssertions();
+            const table = Table.from(...buffers);
+            for (const row of table.rows(true)) {
+                expect(row).toMatchSnapshot();
+            }
+        });
+        test(`toString() prints an empty Table`, () => {
+            expect(Table.from().toString()).toMatchSnapshot();
+        });
+        test(`toString() prints a pretty Table`, () => {
+            expect(Table.from(...buffers).toString()).toMatchSnapshot();
+        });
+        test(`toString({ index: true }) prints a pretty Table with an Index column`, () => {
+            expect(Table.from(...buffers).toString({ index: true })).toMatchSnapshot();
+        });
+    });
+}
diff --git a/js/test/test-config.ts b/js/test/test-config.ts
new file mode 100644
index 0000000000000..b31ff11ad4173
--- /dev/null
+++ b/js/test/test-config.ts
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as fs from 'fs';
+import * as path from 'path';
+const arrowFormats = ['file', 'stream'];
+const arrowFileNames = ['simple', 'struct', 'dictionary', 'dictionary2', 'multi_dictionary'];
+const multipartArrows = ['count', 'latlong', 'origins'];
+export let arrowTestConfigurations = [];
+
+arrowTestConfigurations = arrowFormats.reduce((configs, format) => {
+    return arrowFileNames.reduce((configs, name) => {
+        const arrowPath = path.resolve(__dirname, `./arrows/${format}/${name}.arrow`);
+        try {
+            const arrowFile = fs.readFileSync(arrowPath);
+            return [...configs, [`${name} ${format} Arrow`, arrowFile]];
+        } catch (e) {}
+        return configs;
+    }, configs);
+}, arrowTestConfigurations);
+
+arrowTestConfigurations = multipartArrows.reduce((configs, folder) => {
+    const schemaPath = path.resolve(__dirname, `./arrows/multi/${folder}/schema.arrow`);
+    const recordsPath = path.resolve(__dirname, `./arrows/multi/${folder}/records.arrow`);
+    return [...configs, [`multipart ${folder} Arrow`, fs.readFileSync(schemaPath), fs.readFileSync(recordsPath)]];
+}, arrowTestConfigurations);
+
+export default arrowTestConfigurations;
diff --git a/js/test/tsconfig.json b/js/test/tsconfig.json
new file mode 100644
index 0000000000000..c1ae204212b8f
--- /dev/null
+++ b/js/test/tsconfig.json
@@ -0,0 +1,11 @@
+{
+  "extends": "../tsconfig.json",
+  "include": ["./**/*.ts"],
+  "compilerOptions": {
+    "target": "ESNEXT",
+    "module": "commonjs",
+    "allowJs": true,
+    "importHelpers": false,
+    "noEmitHelpers": false
+  }
+}
diff --git a/js/test/vector-tests.ts b/js/test/vector-tests.ts
new file mode 100644
index 0000000000000..54d50d1af2754
--- /dev/null
+++ b/js/test/vector-tests.ts
@@ -0,0 +1,226 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { vectors } from './Arrow';
+import { flatbuffers } from 'flatbuffers';
+import Long = flatbuffers.Long;
+const BitVector = vectors.BitVector;
+const TypedVector = vectors.TypedVector;
+
+const LongVectors = {
+    Int64Vector: vectors.Int64Vector,
+    Uint64Vector: vectors.Uint64Vector,
+};
+
+const ByteVectors = {
+    Int8Vector: vectors.Int8Vector,
+    Int16Vector: vectors.Int16Vector,
+    Int32Vector: vectors.Int32Vector,
+    Uint8Vector: vectors.Uint8Vector,
+    Uint16Vector: vectors.Uint16Vector,
+    Uint32Vector: vectors.Uint32Vector,
+    Float32Vector: vectors.Float32Vector,
+    Float64Vector: vectors.Float64Vector,
+};
+
+const longVectors = toMap<typeof TypedVector>(vectors, Object.keys(LongVectors));
+const byteVectors = toMap<typeof TypedVector>(vectors, Object.keys(ByteVectors));
+const bytes = Array.from(
+    { length: 5 },
+    () => Uint8Array.from(
+        { length: 64 },
+        () => Math.random() * 255 | 0));
+
+describe(`BitVector`, () => {
+    const vector = new BitVector(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]));
+    const values = [true, true, false, true, true, false, false, false];
+    const n = values.length;
+    vector.length = 1;
+    test(`gets expected values`, () => {
+        let i = -1;
+        while (++i < n) {
+            expect(vector.get(i)).toEqual(values[i]);
+        }
+    });
+    test(`iterates expected values`, () => {
+        let i = -1;
+        for (let v of vector) {
+            expect(++i).toBeLessThan(n);
+            expect(v).toEqual(values[i]);
+        }
+    });
+    test(`can set values to true and false`, () => {
+        const v = new BitVector(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]));
+        const expected1 = [true, true, false, true, true, false, false, false];
+        const expected2 = [true, true,  true, true, true, false, false, false];
+        const expected3 = [true, true, false, false, false, false, true, true];
+        function validate(expected) {
+            for (let i = -1; ++i < n;) {
+                expect(v.get(i)).toEqual(expected[i]);
+            }
+        }
+        validate(expected1);
+        v.set(2, true);
+        validate(expected2);
+        v.set(2, false);
+        validate(expected1);
+        v.set(3, false);
+        v.set(4, false);
+        v.set(6, true);
+        v.set(7, true);
+        validate(expected3);
+        v.set(3, true);
+        v.set(4, true);
+        v.set(6, false);
+        v.set(7, false);
+        validate(expected1);
+    });
+    test(`packs 0 values`, () => {
+        expect(BitVector.pack([])).toEqual(
+            new Uint8Array([0, 0, 0, 0, 0, 0, 0, 0]));
+    });
+    test(`packs 3 values`, () => {
+        expect(BitVector.pack([
+            true, false, true
+        ])).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
+    });
+    test(`packs 8 values`, () => {
+        expect(BitVector.pack([
+            true, true, false, true, true, false, false, false
+        ])).toEqual(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]));
+    });
+    test(`packs 25 values`, () => {
+        expect(BitVector.pack([
+            true, true, false, true, true, false, false, false,
+            false, false, false, true, true, false, true, true,
+            false
+        ])).toEqual(new Uint8Array([27, 216, 0, 0, 0, 0, 0, 0]));
+    });
+    test(`from with boolean Array packs values`, () => {
+        expect(BitVector.from([
+            true, false, true
+        ]).slice()).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
+    });
+});
+
+for (const [VectorName, VectorType] of longVectors) {
+    const ArrayType = VectorType.prototype.arrayType;
+    describe(`${VectorName}`, () => {
+        const values = concatTyped(ArrayType, ...bytes);
+        const bLists = bytes.map((b) => new ArrayType(b.buffer));
+        const vector = new VectorType(null, ...bLists);
+        const n = vector.length = values.length * 0.5;
+        test(`gets expected values`, () => {
+            let i = -1;
+            while (++i < n) {
+                expect(vector.get(i)).toEqual(new Long(
+                    values[i * 2], values[i * 2 + 1]
+                ));
+            }
+        });
+        test(`iterates expected values`, () => {
+            let i = -1;
+            for (let v of vector) {
+                expect(++i).toBeLessThan(n);
+                expect(v).toEqual(new Long(
+                    values[i * 2], values[i * 2 + 1]
+                ));
+            }
+        });
+        test(`slices the entire array`, () => {
+            expect(vector.slice()).toEqual(values);
+        });
+        test(`slice returns a TypedArray`, () => {
+            expect(vector.slice()).toBeInstanceOf(ArrayType);
+        });
+        test(`slices from -20 to length`, () => {
+            expect(vector.slice(-20)).toEqual(values.slice(-40));
+        });
+        test(`slices from 0 to -20`, () => {
+            expect(vector.slice(0, -20)).toEqual(values.slice(0, -40));
+        });
+        test(`slices the array from 0 to length - 20`, () => {
+            expect(vector.slice(0, n - 20)).toEqual(values.slice(0, values.length - 40));
+        });
+        test(`slices the array from 0 to length + 20`, () => {
+            expect(vector.slice(0, n + 20)).toEqual(
+                concatTyped(ArrayType, values, values.slice(0, 40)));
+        });
+    });
+}
+
+for (const [VectorName, VectorType] of byteVectors) {
+    const ArrayType = VectorType.prototype.arrayType;
+    describe(`${VectorName}`, () => {
+        const values = concatTyped(ArrayType, ...bytes);
+        const bLists = bytes.map((b) => new ArrayType(b.buffer));
+        const vector = new VectorType(null, ...bLists);
+        const n = vector.length = values.length;
+        test(`gets expected values`, () => {
+            let i = -1;
+            while (++i < n) {
+                expect(vector.get(i)).toEqual(values[i]);
+            }
+        });
+        test(`iterates expected values`, () => {
+            expect.hasAssertions();
+            let i = -1;
+            for (let v of vector) {
+                expect(++i).toBeLessThan(n);
+                expect(v).toEqual(values[i]);
+            }
+        });
+        test(`slices the entire array`, () => {
+            expect(vector.slice()).toEqual(values);
+        });
+        test(`slice returns a TypedArray`, () => {
+            expect(vector.slice()).toBeInstanceOf(ArrayType);
+        });
+        test(`slices from -20 to length`, () => {
+            expect(vector.slice(-20)).toEqual(values.slice(-20));
+        });
+        test(`slices from 0 to -20`, () => {
+            expect(vector.slice(0, -20)).toEqual(values.slice(0, -20));
+        });
+        test(`slices the array from 0 to length - 20`, () => {
+            expect(vector.slice(0, n - 20)).toEqual(values.slice(0, n - 20));
+        });
+        test(`slices the array from 0 to length + 20`, () => {
+            expect(vector.slice(0, n + 20)).toEqual(
+                concatTyped(ArrayType, values, values.slice(0, 20)));
+        });
+    });
+}
+
+function toMap<T>(entries: any, keys: string[]) {
+    return keys.reduce((map, key) => {
+        map.set(key, entries[key] as T);
+        return map;
+    }, new Map<string, T>());
+}
+
+function concatTyped(ArrayType: any, ...bytes: any[]) {
+    const BPM = ArrayType.BYTES_PER_ELEMENT;
+    return bytes.reduce((v, bytes) => {
+        const l = bytes.byteLength / BPM;
+        const a = new ArrayType(v.length + l);
+        const b = new ArrayType(bytes.buffer);
+        a.set(v);
+        a.set(b, v.length);
+        return a;
+    }, new ArrayType(0)) as Array<number>;
+}
\ No newline at end of file
diff --git a/js/tsconfig.json b/js/tsconfig.json
index f2ad0e85fe842..a56166b886b36 100644
--- a/js/tsconfig.json
+++ b/js/tsconfig.json
@@ -1,14 +1,11 @@
 {
-  "compilerOptions": {
-    "module": "commonjs",
-    "target": "es5",
-    "lib": ["es2015", "dom"],
-    "outDir": "lib",
-    "moduleResolution": "node",
-    "sourceMap": true,
-    "declaration": true
+  "extends": "./tsconfig/tsconfig.base.json",
+  "formatCodeOptions": {
+    "tabSize": 4,
+    "indentSize": 4
   },
-  "include": [
-    "src/*.ts"
-  ]
+  "compilerOptions": {
+    "target": "ESNEXT",
+    "module": "es2015"
+  }
 }
diff --git a/js/tsconfig/tsconfig.base.json b/js/tsconfig/tsconfig.base.json
new file mode 100644
index 0000000000000..47238543a7c3a
--- /dev/null
+++ b/js/tsconfig/tsconfig.base.json
@@ -0,0 +1,25 @@
+{
+  "exclude": ["../node_modules"],
+  "include": ["../src/**/*.ts"],
+  "compilerOptions": {
+    "lib": ["dom", "esnext", "esnext.asynciterable"],
+    "moduleResolution": "node",
+    "sourceMap": false,
+    "inlineSources": true,
+    "inlineSourceMap": true,
+    "declaration": true,
+    "skipLibCheck": true,
+    "importHelpers": true,
+    "noEmitHelpers": true,
+    "noImplicitAny": false,
+    "noEmitOnError": false,
+    "noImplicitThis": true,
+    "noUnusedLocals": true,
+    "removeComments": false,
+    "downlevelIteration": true,
+    "noImplicitUseStrict": true,
+    "preserveConstEnums": false,
+    "noFallthroughCasesInSwitch": true,
+    "forceConsistentCasingInFileNames": true
+  }
+}
diff --git a/js/tsconfig/tsconfig.es2015.cjs.json b/js/tsconfig/tsconfig.es2015.cjs.json
new file mode 100644
index 0000000000000..7e92d5d2625a2
--- /dev/null
+++ b/js/tsconfig/tsconfig.es2015.cjs.json
@@ -0,0 +1,8 @@
+//Compiler configuaration to build the ES2015 CommonJS target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ES2015",
+    "module": "commonjs"
+  }
+}
diff --git a/js/tsconfig/tsconfig.es2015.cls.json b/js/tsconfig/tsconfig.es2015.cls.json
new file mode 100644
index 0000000000000..11ccc04d58375
--- /dev/null
+++ b/js/tsconfig/tsconfig.es2015.cls.json
@@ -0,0 +1,10 @@
+//Compiler configuaration to build the ES2015 Closure Compiler target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ES2015",
+    "module": "es2015",
+    "noEmitHelpers": true,
+    "importHelpers": false
+  }
+}
diff --git a/js/tsconfig/tsconfig.es2015.esm.json b/js/tsconfig/tsconfig.es2015.esm.json
new file mode 100644
index 0000000000000..a8b34547ace6e
--- /dev/null
+++ b/js/tsconfig/tsconfig.es2015.esm.json
@@ -0,0 +1,8 @@
+//Compiler configuaration to build the ES2015 ESModules target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ES2015",
+    "module": "es2015"
+  }
+}
diff --git a/js/tsconfig/tsconfig.es5.cjs.json b/js/tsconfig/tsconfig.es5.cjs.json
new file mode 100644
index 0000000000000..cab55c1ab5f07
--- /dev/null
+++ b/js/tsconfig/tsconfig.es5.cjs.json
@@ -0,0 +1,8 @@
+//Compiler configuaration to build the ES5 CommonJS target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ES5",
+    "module": "commonjs"
+  }
+}
diff --git a/js/tsconfig/tsconfig.es5.cls.json b/js/tsconfig/tsconfig.es5.cls.json
new file mode 100644
index 0000000000000..55f7ea52cf362
--- /dev/null
+++ b/js/tsconfig/tsconfig.es5.cls.json
@@ -0,0 +1,10 @@
+//Compiler configuaration to build the ES5 Closure Compiler target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ES5",
+    "module": "es2015",
+    "noEmitHelpers": true,
+    "importHelpers": false
+  }
+}
diff --git a/js/tsconfig/tsconfig.es5.esm.json b/js/tsconfig/tsconfig.es5.esm.json
new file mode 100644
index 0000000000000..7780883b3f76e
--- /dev/null
+++ b/js/tsconfig/tsconfig.es5.esm.json
@@ -0,0 +1,8 @@
+//Compiler configuaration to build the ES5 ESModules target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ES5",
+    "module": "es2015"
+  }
+}
diff --git a/js/tsconfig/tsconfig.esnext.cjs.json b/js/tsconfig/tsconfig.esnext.cjs.json
new file mode 100644
index 0000000000000..06b2ee5c33009
--- /dev/null
+++ b/js/tsconfig/tsconfig.esnext.cjs.json
@@ -0,0 +1,8 @@
+//Compiler configuaration to build the ESNext CommonJS target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ESNEXT",
+    "module": "commonjs"
+  }
+}
diff --git a/js/tsconfig/tsconfig.esnext.cls.json b/js/tsconfig/tsconfig.esnext.cls.json
new file mode 100644
index 0000000000000..009a5ac10d644
--- /dev/null
+++ b/js/tsconfig/tsconfig.esnext.cls.json
@@ -0,0 +1,10 @@
+//Compiler configuaration to build the ESNext Closure Compiler target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ESNEXT",
+    "module": "es2015",
+    "noEmitHelpers": true,
+    "importHelpers": false
+  }
+}
diff --git a/js/tsconfig/tsconfig.esnext.esm.json b/js/tsconfig/tsconfig.esnext.esm.json
new file mode 100644
index 0000000000000..c614f2ecadf83
--- /dev/null
+++ b/js/tsconfig/tsconfig.esnext.esm.json
@@ -0,0 +1,8 @@
+//Compiler configuaration to build the ESNext ESModules target
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "target": "ESNEXT",
+    "module": "es2015"
+  }
+}
diff --git a/js/tslint.json b/js/tslint.json
index 5f2f6cdcf36e0..705ef8cb7440f 100644
--- a/js/tslint.json
+++ b/js/tslint.json
@@ -1,12 +1,39 @@
 {
-    "defaultSeverity": "error",
-    "extends": [
-        "tslint:recommended"
-    ],
-    "jsRules": {},
-    "rules": {
-      "no-bitwise": false,
-      "max-classes-per-file": false
-    },
-    "rulesDirectory": []
-}
+  "rules": {
+    "curly": true,
+    "eofline": false,
+    "align": [true, "parameters"],
+    "class-name": true,
+    "indent": [true, "spaces"],
+    "max-line-length": [false, 150],
+    "no-consecutive-blank-lines": [true],
+    "no-trailing-whitespace": true,
+    "no-duplicate-variable": true,
+    "no-var-keyword": true,
+    "no-empty": false,
+    "no-unused-expression": false,
+    "no-use-before-declare": false,
+    "no-var-requires": true,
+    "no-require-imports": true,
+    "one-line": [true,
+      "check-else",
+      "check-whitespace",
+      "check-open-brace"],
+    "quotemark": [true,
+      "single",
+      "avoid-escape"],
+    "semicolon": [true, "always"],
+    "typedef-whitespace": [true, {
+      "call-signature": "nospace",
+      "index-signature": "nospace",
+      "parameter": "nospace",
+      "property-declaration": "nospace",
+      "variable-declaration": "nospace"
+    }],
+    "whitespace": [true,
+      "check-branch",
+      "check-decl",
+      "check-operator",
+      "check-type"]
+  }
+}
\ No newline at end of file
diff --git a/js/webpack.config.js b/js/webpack.config.js
deleted file mode 100644
index aa123bd39f9b0..0000000000000
--- a/js/webpack.config.js
+++ /dev/null
@@ -1,54 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-var path = require('path');
-var UglifyJSPlugin = require('uglifyjs-webpack-plugin');
-
-module.exports = {
-  entry: {
-    'arrow': './src/arrow.ts',
-    'arrow.min': './src/arrow.ts'
-  },
-  output: {
-    path: path.resolve(__dirname, '_bundles'),
-    filename: '[name].js',
-    libraryTarget: 'umd',
-    library: 'arrow',
-    umdNamedDefine: true
-  },
-  resolve: {
-    extensions: ['.ts', '.js']
-  },
-  devtool: 'source-map',
-  plugins: [
-    new UglifyJSPlugin({
-      minimize: true,
-      sourceMap: true,
-      include: /\.min\.js$/
-    })
-  ],
-  module: {
-    loaders: [{
-      test: /\.ts$/,
-      loader: 'awesome-typescript-loader',
-      exclude: /node_modules/,
-      query: {
-        declaration: false
-      }
-    }]
-  }
-};

From 8b70ab07c33c68131e8bc68981fcb2f9c22f0bef Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 9 Sep 2017 13:45:52 -0400
Subject: [PATCH 1027/1644] ARROW-1507: [C++] Include arrow/array.h for
 arrow::internal::ArrayData

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1072 from kou/cpp-fix-include-for-compute and squashes the following commits:

a17ea6f8 [Kouhei Sutou] [C++] Include arrow/array.h for arrow::internal::ArrayData
---
 cpp/src/arrow/compute/kernel.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h
index c72d4679af2a0..521421ef37b41 100644
--- a/cpp/src/arrow/compute/kernel.h
+++ b/cpp/src/arrow/compute/kernel.h
@@ -18,11 +18,10 @@
 #ifndef ARROW_COMPUTE_KERNEL_H
 #define ARROW_COMPUTE_KERNEL_H
 
-#include "arrow/util/visibility.h"
+#include "arrow/array.h"
 
 namespace arrow {
 
-class Array;
 using internal::ArrayData;
 
 namespace compute {

From 875be96da78d9503ce4aeb37fd7104e5662feab6 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 9 Sep 2017 13:47:44 -0400
Subject: [PATCH 1028/1644] ARROW-1504: [GLib] Add GArrowTimestampArray

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1068 from kou/glib-add-timestamp-array and squashes the following commits:

27f0227f [Kouhei Sutou] [GLib] Add GArrowTimestampArray
---
 c_glib/arrow-glib/array-builder.cpp     | 80 +++++++++++++++++++++
 c_glib/arrow-glib/array-builder.h       | 52 ++++++++++++++
 c_glib/arrow-glib/array.cpp             | 92 +++++++++++++++++++++++++
 c_glib/arrow-glib/array.h               | 54 +++++++++++++++
 c_glib/arrow-glib/data-type.cpp         | 44 ++++++++++++
 c_glib/arrow-glib/data-type.h           | 44 ++++++++++++
 c_glib/test/helper/buildable.rb         |  6 ++
 c_glib/test/test-timestamp-array.rb     | 57 +++++++++++++++
 c_glib/test/test-timestamp-data-type.rb | 63 +++++++++++++++++
 9 files changed, 492 insertions(+)
 create mode 100644 c_glib/test/test-timestamp-array.rb
 create mode 100644 c_glib/test/test-timestamp-data-type.rb

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 9919bfffd2c42..f550eb559841d 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -113,6 +113,9 @@ G_BEGIN_DECLS
  * #GArrowDate64ArrayBuilder is the class to create a new
  * #GArrowDate64Array.
  *
+ * #GArrowTimestampArrayBuilder is the class to create a new
+ * #GArrowTimestampArray.
+ *
  * #GArrowTime32ArrayBuilder is the class to create a new
  * #GArrowTime32Array.
  *
@@ -1314,6 +1317,80 @@ garrow_date64_array_builder_append_null(GArrowDate64ArrayBuilder *builder,
 }
 
 
+G_DEFINE_TYPE(GArrowTimestampArrayBuilder,
+              garrow_timestamp_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_timestamp_array_builder_init(GArrowTimestampArrayBuilder *builder)
+{
+}
+
+static void
+garrow_timestamp_array_builder_class_init(GArrowTimestampArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_timestamp_array_builder_new:
+ * @data_type: A #GArrowTimestampDataType.
+ *
+ * Returns: A newly created #GArrowTimestampArrayBuilder.
+ *
+ * Since: 0.7.0
+ */
+GArrowTimestampArrayBuilder *
+garrow_timestamp_array_builder_new(GArrowTimestampDataType *data_type)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto builder = garrow_array_builder_new(arrow_data_type,
+                                          NULL,
+                                          "[timestamp-array-builder][new]");
+  return GARROW_TIMESTAMP_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_timestamp_array_builder_append:
+ * @builder: A #GArrowTimestampArrayBuilder.
+ * @value: The number of milliseconds since UNIX epoch in signed 64bit integer.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_timestamp_array_builder_append(GArrowTimestampArrayBuilder *builder,
+                                      gint64 value,
+                                      GError **error)
+{
+  return garrow_array_builder_append<arrow::TimestampBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[timestamp-array-builder][append]");
+}
+
+/**
+ * garrow_timestamp_array_builder_append_null:
+ * @builder: A #GArrowTimestampArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.7.0
+ */
+gboolean
+garrow_timestamp_array_builder_append_null(GArrowTimestampArrayBuilder *builder,
+                                           GError **error)
+{
+  return garrow_array_builder_append_null<arrow::TimestampBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[timestamp-array-builder][append-null]");
+}
+
+
 G_DEFINE_TYPE(GArrowTime32ArrayBuilder,
               garrow_time32_array_builder,
               GARROW_TYPE_ARRAY_BUILDER)
@@ -1883,6 +1960,9 @@ garrow_array_builder_new_raw(arrow::ArrayBuilder *arrow_builder,
     case arrow::Type::type::DATE64:
       type = GARROW_TYPE_DATE64_ARRAY_BUILDER;
       break;
+    case arrow::Type::type::TIMESTAMP:
+      type = GARROW_TYPE_TIMESTAMP_ARRAY_BUILDER;
+      break;
     case arrow::Type::type::TIME32:
       type = GARROW_TYPE_TIME32_ARRAY_BUILDER;
       break;
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index 981f0bc10c55d..4a19925bc68f7 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -884,6 +884,58 @@ gboolean garrow_date64_array_builder_append_null(GArrowDate64ArrayBuilder *build
                                                  GError **error);
 
 
+#define GARROW_TYPE_TIMESTAMP_ARRAY_BUILDER     \
+  (garrow_timestamp_array_builder_get_type())
+#define GARROW_TIMESTAMP_ARRAY_BUILDER(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
+                              GARROW_TYPE_TIMESTAMP_ARRAY_BUILDER,      \
+                              GArrowTimestampArrayBuilder))
+#define GARROW_TIMESTAMP_ARRAY_BUILDER_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIMESTAMP_ARRAY_BUILDER, \
+                           GArrowTimestampArrayBuilderClass))
+#define GARROW_IS_TIMESTAMP_ARRAY_BUILDER(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
+                              GARROW_TYPE_TIMESTAMP_ARRAY_BUILDER))
+#define GARROW_IS_TIMESTAMP_ARRAY_BUILDER_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                                     \
+                           GARROW_TYPE_TIMESTAMP_ARRAY_BUILDER))
+#define GARROW_TIMESTAMP_ARRAY_BUILDER_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                                     \
+                             GARROW_TYPE_TIMESTAMP_ARRAY_BUILDER,       \
+                             GArrowTimestampArrayBuilderClass))
+
+typedef struct _GArrowTimestampArrayBuilder      GArrowTimestampArrayBuilder;
+typedef struct _GArrowTimestampArrayBuilderClass GArrowTimestampArrayBuilderClass;
+
+/**
+ * GArrowTimestampArrayBuilder:
+ *
+ * It wraps `arrow::TimestampBuilder`.
+ */
+struct _GArrowTimestampArrayBuilder
+{
+  /*< private >*/
+  GArrowArrayBuilder parent_instance;
+};
+
+struct _GArrowTimestampArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
+
+GType garrow_timestamp_array_builder_get_type(void) G_GNUC_CONST;
+
+GArrowTimestampArrayBuilder *
+garrow_timestamp_array_builder_new(GArrowTimestampDataType *data_type);
+
+gboolean garrow_timestamp_array_builder_append(GArrowTimestampArrayBuilder *builder,
+                                               gint64 value,
+                                               GError **error);
+gboolean garrow_timestamp_array_builder_append_null(GArrowTimestampArrayBuilder *builder,
+                                                    GError **error);
+
+
 #define GARROW_TYPE_TIME32_ARRAY_BUILDER        \
   (garrow_time32_array_builder_get_type())
 #define GARROW_TIME32_ARRAY_BUILDER(obj)                        \
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 17553ec94cf3e..a52ea05d3fb9c 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -178,6 +178,12 @@ G_BEGIN_DECLS
  * more date data. If you don't have Arrow format data, you need to
  * use #GArrowDate64ArrayBuilder to create a new array.
  *
+ * #GArrowTimestampArray is a class for the number of
+ * seconds/milliseconds/microseconds/nanoseconds since UNIX epoch in
+ * 64-bit signed integer array. It can store zero or more timestamp
+ * data. If you don't have Arrow format data, you need to use
+ * #GArrowTimestampArrayBuilder to create a new array.
+ *
  * #GArrowTime32Array is a class for the number of seconds or
  * milliseconds since midnight in 32-bit signed integer array. It can
  * store zero or more time data. If you don't have Arrow format data,
@@ -1704,6 +1710,89 @@ garrow_date64_array_get_values(GArrowDate64Array *array,
 }
 
 
+G_DEFINE_TYPE(GArrowTimestampArray,             \
+              garrow_timestamp_array,           \
+              GARROW_TYPE_PRIMITIVE_ARRAY)
+
+static void
+garrow_timestamp_array_init(GArrowTimestampArray *object)
+{
+}
+
+static void
+garrow_timestamp_array_class_init(GArrowTimestampArrayClass *klass)
+{
+}
+
+/**
+ * garrow_timestamp_array_new:
+ * @data_type: The #GArrowTimestampDataType.
+ * @length: The number of elements.
+ * @data: The binary data in Arrow format of the array.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowTimestampArray.
+ *
+ * Since: 0.7.0
+ */
+GArrowTimestampArray *
+garrow_timestamp_array_new(GArrowTimestampDataType *data_type,
+                           gint64 length,
+                           GArrowBuffer *data,
+                           GArrowBuffer *null_bitmap,
+                           gint64 n_nulls)
+{
+  auto array =
+    garrow_primitive_array_new<arrow::TimestampType>(GARROW_DATA_TYPE(data_type),
+                                                     length,
+                                                     data,
+                                                     null_bitmap,
+                                                     n_nulls);
+  return GARROW_TIMESTAMP_ARRAY(array);
+}
+
+/**
+ * garrow_timestamp_array_get_value:
+ * @array: A #GArrowTimestampArray.
+ * @i: The index of the target value.
+ *
+ * Returns: The i-th value.
+ *
+ * Since: 0.7.0
+ */
+gint64
+garrow_timestamp_array_get_value(GArrowTimestampArray *array,
+                                 gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  return static_cast<arrow::TimestampArray *>(arrow_array.get())->Value(i);
+}
+
+/**
+ * garrow_timestamp_array_get_values:
+ * @array: A #GArrowTimestampArray.
+ * @length: (out): The number of values.
+ *
+ * Returns: (array length=length): The raw values.
+ *
+ * Since: 0.7.0
+ */
+const gint64 *
+garrow_timestamp_array_get_values(GArrowTimestampArray *array,
+                                  gint64 *length)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto values =
+    garrow_array_get_values_raw<arrow::TimestampType>(arrow_array, length);
+  return reinterpret_cast<const gint64 *>(values);
+}
+
+
 G_DEFINE_TYPE(GArrowTime32Array,               \
               garrow_time32_array,             \
               GARROW_TYPE_PRIMITIVE_ARRAY)
@@ -2135,6 +2224,9 @@ garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array)
   case arrow::Type::type::DATE64:
     type = GARROW_TYPE_DATE64_ARRAY;
     break;
+  case arrow::Type::type::TIMESTAMP:
+    type = GARROW_TYPE_TIMESTAMP_ARRAY;
+    break;
   case arrow::Type::type::TIME32:
     type = GARROW_TYPE_TIME32_ARRAY;
     break;
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 9aaca472cb930..da6920739da86 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -966,6 +966,60 @@ const gint64 *garrow_date64_array_get_values(GArrowDate64Array *array,
                                              gint64 *length);
 
 
+#define GARROW_TYPE_TIMESTAMP_ARRAY             \
+  (garrow_timestamp_array_get_type())
+#define GARROW_TIMESTAMP_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIMESTAMP_ARRAY,      \
+                              GArrowTimestampArray))
+#define GARROW_TIMESTAMP_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_TIMESTAMP_ARRAY, \
+                           GArrowTimestampArrayClass))
+#define GARROW_IS_TIMESTAMP_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIMESTAMP_ARRAY))
+#define GARROW_IS_TIMESTAMP_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIMESTAMP_ARRAY))
+#define GARROW_TIMESTAMP_ARRAY_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIMESTAMP_ARRAY,       \
+                             GArrowTimestampArrayClass))
+
+typedef struct _GArrowTimestampArray         GArrowTimestampArray;
+typedef struct _GArrowTimestampArrayClass    GArrowTimestampArrayClass;
+
+/**
+ * GArrowTimestampArray:
+ *
+ * It wraps `arrow::TimestampArray`.
+ */
+struct _GArrowTimestampArray
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowTimestampArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_timestamp_array_get_type(void) G_GNUC_CONST;
+
+GArrowTimestampArray *garrow_timestamp_array_new(GArrowTimestampDataType *data_type,
+                                                 gint64 length,
+                                                 GArrowBuffer *data,
+                                                 GArrowBuffer *null_bitmap,
+                                                 gint64 n_nulls);
+
+gint64 garrow_timestamp_array_get_value(GArrowTimestampArray *array,
+                                        gint64 i);
+const gint64 *garrow_timestamp_array_get_values(GArrowTimestampArray *array,
+                                                gint64 *length);
+
+
 #define GARROW_TYPE_TIME32_ARRAY                \
   (garrow_time32_array_get_type())
 #define GARROW_TIME32_ARRAY(obj)                        \
diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/data-type.cpp
index c2c74612fb20f..acb1484311988 100644
--- a/c_glib/arrow-glib/data-type.cpp
+++ b/c_glib/arrow-glib/data-type.cpp
@@ -75,6 +75,10 @@ G_BEGIN_DECLS
  * #GArrowDate64DataType is a class for the number of milliseconds
  * since UNIX epoch in 64-bit signed integer data type.
  *
+ * #GArrowTimestampDataType is a class for the number of
+ * seconds/milliseconds/microseconds/nanoseconds since UNIX epoch in
+ * 64-bit signed integer data type.
+ *
  * #GArrowTime32DataType is a class for the number of seconds or
  * milliseconds since midnight in 32-bit signed integer data type.
  *
@@ -738,6 +742,43 @@ garrow_date64_data_type_new(void)
 }
 
 
+G_DEFINE_TYPE(GArrowTimestampDataType,                \
+              garrow_timestamp_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_timestamp_data_type_init(GArrowTimestampDataType *object)
+{
+}
+
+static void
+garrow_timestamp_data_type_class_init(GArrowTimestampDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_timestamp_data_type_new:
+ * @unit: The unit of the timestamp data.
+ *
+ * Returns: A newly created the number of
+ *   seconds/milliseconds/microseconds/nanoseconds since UNIX epoch in
+ *   64-bit signed integer data type.
+ *
+ * Since: 0.7.0
+ */
+GArrowTimestampDataType *
+garrow_timestamp_data_type_new(GArrowTimeUnit unit)
+{
+  auto arrow_unit = garrow_time_unit_to_raw(unit);
+  auto arrow_data_type = arrow::timestamp(arrow_unit);
+  auto data_type =
+    GARROW_TIMESTAMP_DATA_TYPE(g_object_new(GARROW_TYPE_TIMESTAMP_DATA_TYPE,
+                                            "data-type", &arrow_data_type,
+                                            NULL));
+  return data_type;
+}
+
+
 G_DEFINE_TYPE(GArrowTimeDataType,               \
               garrow_time_data_type,            \
               GARROW_TYPE_DATA_TYPE)
@@ -1057,6 +1098,9 @@ garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type)
   case arrow::Type::type::DATE64:
     type = GARROW_TYPE_DATE64_DATA_TYPE;
     break;
+  case arrow::Type::type::TIMESTAMP:
+    type = GARROW_TYPE_TIMESTAMP_DATA_TYPE;
+    break;
   case arrow::Type::type::TIME32:
     type = GARROW_TYPE_TIME32_DATA_TYPE;
     break;
diff --git a/c_glib/arrow-glib/data-type.h b/c_glib/arrow-glib/data-type.h
index c01201db1c621..a450ab3bc8dc5 100644
--- a/c_glib/arrow-glib/data-type.h
+++ b/c_glib/arrow-glib/data-type.h
@@ -780,6 +780,50 @@ GType                 garrow_date64_data_type_get_type (void) G_GNUC_CONST;
 GArrowDate64DataType *garrow_date64_data_type_new      (void);
 
 
+#define GARROW_TYPE_TIMESTAMP_DATA_TYPE         \
+  (garrow_timestamp_data_type_get_type())
+#define GARROW_TIMESTAMP_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIMESTAMP_DATA_TYPE,  \
+                              GArrowTimestampDataType))
+#define GARROW_TIMESTAMP_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIMESTAMP_DATA_TYPE,     \
+                           GArrowTimestampDataTypeClass))
+#define GARROW_IS_TIMESTAMP_DATA_TYPE(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIMESTAMP_DATA_TYPE))
+#define GARROW_IS_TIMESTAMP_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIMESTAMP_DATA_TYPE))
+#define GARROW_TIMESTAMP_DATA_TYPE_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIMESTAMP_DATA_TYPE,   \
+                             GArrowTimestampDataTypeClass))
+
+typedef struct _GArrowTimestampDataType         GArrowTimestampDataType;
+typedef struct _GArrowTimestampDataTypeClass    GArrowTimestampDataTypeClass;
+
+/**
+ * GArrowTimestampDataType:
+ *
+ * It wraps `arrow::TimestampType`.
+ */
+struct _GArrowTimestampDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowTimestampDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_timestamp_data_type_get_type (void) G_GNUC_CONST;
+GArrowTimestampDataType *garrow_timestamp_data_type_new   (GArrowTimeUnit unit);
+
+
 #define GARROW_TYPE_TIME_DATA_TYPE              \
   (garrow_time_data_type_get_type())
 #define GARROW_TIME_DATA_TYPE(obj)                              \
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index cb427d7caa6f0..0ed5759412669 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -73,6 +73,12 @@ def build_date64_array(values)
       build_array(Arrow::Date64ArrayBuilder.new, values)
     end
 
+    def build_timestamp_array(unit, values)
+      data_type = Arrow::TimestampDataType.new(unit)
+      build_array(Arrow::TimestampArrayBuilder.new(data_type),
+                  values)
+    end
+
     def build_time32_array(unit, values)
       build_array(Arrow::Time32ArrayBuilder.new(Arrow::Time32DataType.new(unit)),
                   values)
diff --git a/c_glib/test/test-timestamp-array.rb b/c_glib/test/test-timestamp-array.rb
new file mode 100644
index 0000000000000..79340232362d4
--- /dev/null
+++ b/c_glib/test/test-timestamp-array.rb
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTimestampArray < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_new
+    epoch = 0
+    after_epoch = 1504953190854 # 2017-09-09T10:33:10.854Z
+    raw_data = [epoch, after_epoch]
+    data_type = Arrow::TimestampDataType.new(:milli)
+    assert_equal(build_timestamp_array(:milli, [*raw_data, nil]),
+                 Arrow::TimestampArray.new(data_type,
+                                           3,
+                                           Arrow::Buffer.new(raw_data.pack("q*")),
+                                           Arrow::Buffer.new([0b011].pack("C*")),
+                                           -1))
+  end
+
+  def test_buffer
+    epoch = 0
+    after_epoch = 1504953190854 # 2017-09-09T10:33:10.854Z
+    raw_data = [epoch, after_epoch]
+    array = build_timestamp_array(:milli, raw_data)
+    assert_equal(raw_data.pack("q*"),
+                 array.buffer.data.to_s)
+  end
+
+  def test_value
+    after_epoch = 1504953190854 # 2017-09-09T10:33:10.854Z
+    array = build_timestamp_array(:milli, [after_epoch])
+    assert_equal(after_epoch, array.get_value(0))
+  end
+
+  def test_values
+    epoch = 0
+    after_epoch = 1504953190854 # 2017-09-09T10:33:10.854Z
+    raw_data = [epoch, after_epoch]
+    array = build_timestamp_array(:milli, raw_data)
+    assert_equal(raw_data, array.values)
+  end
+end
diff --git a/c_glib/test/test-timestamp-data-type.rb b/c_glib/test/test-timestamp-data-type.rb
new file mode 100644
index 0000000000000..83038876926f0
--- /dev/null
+++ b/c_glib/test/test-timestamp-data-type.rb
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTimestampDataType < Test::Unit::TestCase
+  def test_type
+    data_type = Arrow::TimestampDataType.new(:micro)
+    assert_equal(Arrow::Type::TIMESTAMP, data_type.id)
+  end
+
+  sub_test_case("second") do
+    def setup
+      @data_type = Arrow::TimestampDataType.new(:second)
+    end
+
+    def test_to_s
+      assert_equal("timestamp[s]", @data_type.to_s)
+    end
+  end
+
+  sub_test_case("millisecond") do
+    def setup
+      @data_type = Arrow::TimestampDataType.new(:milli)
+    end
+
+    def test_to_s
+      assert_equal("timestamp[ms]", @data_type.to_s)
+    end
+  end
+
+  sub_test_case("micro") do
+    def setup
+      @data_type = Arrow::TimestampDataType.new(:micro)
+    end
+
+    def test_to_s
+      assert_equal("timestamp[us]", @data_type.to_s)
+    end
+  end
+
+  sub_test_case("nano") do
+    def setup
+      @data_type = Arrow::TimestampDataType.new(:nano)
+    end
+
+    def test_to_s
+      assert_equal("timestamp[ns]", @data_type.to_s)
+    end
+  end
+end

From 1706aabab54b1c0ed1f4a259842f82f4f5edce9e Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 9 Sep 2017 13:48:46 -0400
Subject: [PATCH 1029/1644] ARROW-1505: [GLib] Simplify arguments check

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1069 from kou/glib-simplify-arguments-check and squashes the following commits:

46c07f1f [Kouhei Sutou] [GLib] Simplify arguments check
---
 c_glib/arrow-glib/array-builder.cpp | 38 +++---------------
 c_glib/arrow-glib/array-builder.h   |  6 +--
 c_glib/arrow-glib/array.cpp         | 62 ++++++++++-------------------
 c_glib/arrow-glib/array.h           | 10 ++---
 4 files changed, 32 insertions(+), 84 deletions(-)

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index f550eb559841d..126986d1fb007 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -1408,28 +1408,15 @@ garrow_time32_array_builder_class_init(GArrowTime32ArrayBuilderClass *klass)
 /**
  * garrow_time32_array_builder_new:
  * @data_type: A #GArrowTime32DataType.
- * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: (nullable):
- *   A newly created #GArrowTime32ArrayBuilder on success, %NULL on error.
+ * Returns: A newly created #GArrowTime32ArrayBuilder.
  *
  * Since: 0.7.0
  */
 GArrowTime32ArrayBuilder *
-garrow_time32_array_builder_new(GArrowDataType *data_type,
-                                GError **error)
+garrow_time32_array_builder_new(GArrowTime32DataType *data_type)
 {
-  if (!GARROW_IS_TIME32_DATA_TYPE(data_type)) {
-    g_set_error(error,
-                GARROW_ERROR,
-                GARROW_ERROR_INVALID,
-                "[time32-array-builder][new] "
-                "data type must be time32 data type: <%s>",
-                G_OBJECT_TYPE_NAME(data_type));
-    return NULL;
-  }
-
-  auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
   auto builder = garrow_array_builder_new(arrow_data_type,
                                           NULL,
                                           "[time32-array-builder][new]");
@@ -1495,28 +1482,15 @@ garrow_time64_array_builder_class_init(GArrowTime64ArrayBuilderClass *klass)
 /**
  * garrow_time64_array_builder_new:
  * @data_type: A #GArrowTime64DataType.
- * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: (nullable):
- *   A newly created #GArrowTime64ArrayBuilder on success, %NULL on error.
+ * Returns: A newly created #GArrowTime64ArrayBuilder.
  *
  * Since: 0.7.0
  */
 GArrowTime64ArrayBuilder *
-garrow_time64_array_builder_new(GArrowDataType *data_type,
-                                GError **error)
+garrow_time64_array_builder_new(GArrowTime64DataType *data_type)
 {
-  if (!GARROW_IS_TIME64_DATA_TYPE(data_type)) {
-    g_set_error(error,
-                GARROW_ERROR,
-                GARROW_ERROR_INVALID,
-                "[time64-array-builder][new] "
-                "data type must be time64 data type: <%s>",
-                G_OBJECT_TYPE_NAME(data_type));
-    return NULL;
-  }
-
-  auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
   auto builder = garrow_array_builder_new(arrow_data_type,
                                           NULL,
                                           "[time64-array-builder][new]");
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index 4a19925bc68f7..f28959ffb5f96 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -978,8 +978,7 @@ struct _GArrowTime32ArrayBuilderClass
 
 GType garrow_time32_array_builder_get_type(void) G_GNUC_CONST;
 
-GArrowTime32ArrayBuilder *garrow_time32_array_builder_new(GArrowDataType *data_type,
-                                                          GError **error);
+GArrowTime32ArrayBuilder *garrow_time32_array_builder_new(GArrowTime32DataType *data_type);
 
 gboolean garrow_time32_array_builder_append(GArrowTime32ArrayBuilder *builder,
                                             gint32 value,
@@ -1030,8 +1029,7 @@ struct _GArrowTime64ArrayBuilderClass
 
 GType garrow_time64_array_builder_get_type(void) G_GNUC_CONST;
 
-GArrowTime64ArrayBuilder *garrow_time64_array_builder_new(GArrowDataType *data_type,
-                                                          GError **error);
+GArrowTime64ArrayBuilder *garrow_time64_array_builder_new(GArrowTime64DataType *data_type);
 
 gboolean garrow_time64_array_builder_append(GArrowTime64ArrayBuilder *builder,
                                             gint64 value,
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index a52ea05d3fb9c..007f23314f7d8 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -1818,35 +1818,24 @@ garrow_time32_array_class_init(GArrowTime32ArrayClass *klass)
  *   @n_nulls is 0.
  * @n_nulls: The number of null elements. If -1 is specified, the
  *   number of nulls are computed from @null_bitmap.
- * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: (nullable):
- *   A newly created #GArrowTime32Array on success, %NULL on error.
+ * Returns: A newly created #GArrowTime32Array.
  *
  * Since: 0.7.0
  */
 GArrowTime32Array *
-garrow_time32_array_new(GArrowDataType *data_type,
+garrow_time32_array_new(GArrowTime32DataType *data_type,
                         gint64 length,
                         GArrowBuffer *data,
                         GArrowBuffer *null_bitmap,
-                        gint64 n_nulls,
-                        GError **error)
-{
-  if (!GARROW_IS_TIME32_DATA_TYPE(data_type)) {
-    g_set_error(error,
-                GARROW_ERROR,
-                GARROW_ERROR_INVALID,
-                "[time32-array][new] data type must be time32 data type: <%s>",
-                G_OBJECT_TYPE_NAME(data_type));
-    return NULL;
-  }
-
-  auto array = garrow_primitive_array_new<arrow::Time32Type>(data_type,
-                                                             length,
-                                                             data,
-                                                             null_bitmap,
-                                                             n_nulls);
+                        gint64 n_nulls)
+{
+  auto array =
+    garrow_primitive_array_new<arrow::Time32Type>(GARROW_DATA_TYPE(data_type),
+                                                  length,
+                                                  data,
+                                                  null_bitmap,
+                                                  n_nulls);
   return GARROW_TIME32_ARRAY(array);
 }
 
@@ -1910,35 +1899,24 @@ garrow_time64_array_class_init(GArrowTime64ArrayClass *klass)
  *   @n_nulls is 0.
  * @n_nulls: The number of null elements. If -1 is specified, the
  *   number of nulls are computed from @null_bitmap.
- * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: (nullable):
- *   A newly created #GArrowTime64Array on success, %NULL on error.
+ * Returns: A newly created #GArrowTime64Array.
  *
  * Since: 0.7.0
  */
 GArrowTime64Array *
-garrow_time64_array_new(GArrowDataType *data_type,
+garrow_time64_array_new(GArrowTime64DataType *data_type,
                         gint64 length,
                         GArrowBuffer *data,
                         GArrowBuffer *null_bitmap,
-                        gint64 n_nulls,
-                        GError **error)
-{
-  if (!GARROW_IS_TIME64_DATA_TYPE(data_type)) {
-    g_set_error(error,
-                GARROW_ERROR,
-                GARROW_ERROR_INVALID,
-                "[time64-array][new] data type must be time64 data type: <%s>",
-                G_OBJECT_TYPE_NAME(data_type));
-    return NULL;
-  }
-
-  auto array = garrow_primitive_array_new<arrow::Time64Type>(data_type,
-                                                             length,
-                                                             data,
-                                                             null_bitmap,
-                                                             n_nulls);
+                        gint64 n_nulls)
+{
+  auto array =
+    garrow_primitive_array_new<arrow::Time64Type>(GARROW_DATA_TYPE(data_type),
+                                                  length,
+                                                  data,
+                                                  null_bitmap,
+                                                  n_nulls);
   return GARROW_TIME64_ARRAY(array);
 }
 
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index da6920739da86..ebfcf4335a9ae 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -1062,12 +1062,11 @@ struct _GArrowTime32ArrayClass
 
 GType garrow_time32_array_get_type(void) G_GNUC_CONST;
 
-GArrowTime32Array *garrow_time32_array_new(GArrowDataType *data_type,
+GArrowTime32Array *garrow_time32_array_new(GArrowTime32DataType *data_type,
                                            gint64 length,
                                            GArrowBuffer *data,
                                            GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls,
-                                           GError **error);
+                                           gint64 n_nulls);
 
 gint32 garrow_time32_array_get_value(GArrowTime32Array *array,
                                      gint64 i);
@@ -1117,12 +1116,11 @@ struct _GArrowTime64ArrayClass
 
 GType garrow_time64_array_get_type(void) G_GNUC_CONST;
 
-GArrowTime64Array *garrow_time64_array_new(GArrowDataType *data_type,
+GArrowTime64Array *garrow_time64_array_new(GArrowTime64DataType *data_type,
                                            gint64 length,
                                            GArrowBuffer *data,
                                            GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls,
-                                           GError **error);
+                                           gint64 n_nulls);
 
 gint64 garrow_time64_array_get_value(GArrowTime64Array *array,
                                      gint64 i);

From 675e251586981eaedb01bec039c88f6540571a59 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 9 Sep 2017 13:50:30 -0400
Subject: [PATCH 1030/1644] ARROW-1506: [C++] Add .pc for compute modules

If .pc for compute modules exist, we can detect whether compute modules
are built or not by pkg-config:

    pkg-config --exists arrow-compute

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1070 from kou/cpp-add-pc-for-arrow-compute and squashes the following commits:

59fcd346 [Kouhei Sutou] [C++] Add .pc for compute modules
---
 cpp/src/arrow/compute/CMakeLists.txt      |  8 ++++++++
 cpp/src/arrow/compute/arrow-compute.pc.in | 21 +++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 cpp/src/arrow/compute/arrow-compute.pc.in

diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt
index fa475ca9fb0ac..4589afb9574d3 100644
--- a/cpp/src/arrow/compute/CMakeLists.txt
+++ b/cpp/src/arrow/compute/CMakeLists.txt
@@ -23,6 +23,14 @@ install(FILES
   kernel.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute")
 
+# pkg-config support
+configure_file(arrow-compute.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/arrow-compute.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/arrow-compute.pc"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
+
 #######################################
 # Unit tests
 #######################################
diff --git a/cpp/src/arrow/compute/arrow-compute.pc.in b/cpp/src/arrow/compute/arrow-compute.pc.in
new file mode 100644
index 0000000000000..bbdb12c470043
--- /dev/null
+++ b/cpp/src/arrow/compute/arrow-compute.pc.in
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+Name: Apache Arrow Compute
+Description: Compute modules for Apache Arrow
+Version: @ARROW_VERSION@
+Requires: arrow

From b5d510f72bf0bbe2c9a38122a0d68e550f0c12b8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 10 Sep 2017 08:03:59 +0200
Subject: [PATCH 1031/1644] ARROW-1358: Update sha{1, 256, 512} checksums per
 latest ASF release policy

See http://www.apache.org/dev/release-distribution#sigs-and-sums

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1077 from wesm/ARROW-1358 and squashes the following commits:

61adb01 [Wes McKinney] Add SHA256 and SHA512 checksums named in accordance with updated ASF release checksum policy
---
 dev/release/02-source.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index 6ab160ca5169b..62478131d8c93 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -98,7 +98,9 @@ ${SOURCE_DIR}/run-rat.sh ${tarball}
 # sign the archive
 gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
 gpg --print-md MD5 ${tarball} > ${tarball}.md5
-shasum $tarball > ${tarball}.sha
+sha1sum $tarball > ${tarball}.sha1
+sha256sum $tarball > ${tarball}.sha256
+sha512sum $tarball > ${tarball}.sha512
 
 # check out the arrow RC folder
 svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp

From 947ca871c60fbd8680b8d477be2a5bee54fa8e0e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 10 Sep 2017 08:30:46 +0200
Subject: [PATCH 1032/1644] ARROW-594: [C++/Python] Write arrow::Table to
 stream and file writers

* Add TableBatchReader class which creates a sequence of record batches from a table with chunked columns
* Write Table to stream/file using the TableBatchReader
* Add Python bindings

Also resolves ARROW-989

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1074 from wesm/ARROW-594 and squashes the following commits:

8126372 [Wes McKinney] Add write_table method to record batch writer classes
ad30059 [Wes McKinney] Add RecordBatchWriter::WriteTable method
00a317b [Wes McKinney] Fix go example
d643877 [Wes McKinney] Add Table::IsChunked method
84a9dcb [Wes McKinney] Add unit tests for TableBatchIterator
1748b39 [Wes McKinney] Draft TableBatchIterator implementation. Move RecordBatchReader interface to arrow/table.h
---
 c_glib/arrow-glib/reader.cpp             |   2 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  42 +++++---
 cpp/src/arrow/ipc/reader.cc              |   8 +-
 cpp/src/arrow/ipc/reader.h               |  19 +---
 cpp/src/arrow/ipc/stream-to-file.cc      |   2 +-
 cpp/src/arrow/ipc/writer.cc              |  15 +++
 cpp/src/arrow/ipc/writer.h               |  21 ++--
 cpp/src/arrow/pretty_print.cc            |   4 +-
 cpp/src/arrow/python/arrow_to_pandas.cc  |   2 +-
 cpp/src/arrow/python/arrow_to_python.cc  |   4 +-
 cpp/src/arrow/table-test.cc              |  77 ++++++++++++++
 cpp/src/arrow/table.cc                   | 128 +++++++++++++++++++++--
 cpp/src/arrow/table.h                    |  41 ++++++++
 python/pyarrow/includes/libarrow.pxd     |   6 +-
 python/pyarrow/ipc.pxi                   |  32 +++++-
 python/pyarrow/tests/test_ipc.py         |  21 +++-
 16 files changed, 356 insertions(+), 68 deletions(-)

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index ba835fd33a6bd..1941f6435ced6 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -193,7 +193,7 @@ garrow_record_batch_reader_read_next_record_batch(GArrowRecordBatchReader *reade
 {
   auto arrow_reader = garrow_record_batch_reader_get_raw(reader);
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
-  auto status = arrow_reader->ReadNextRecordBatch(&arrow_record_batch);
+  auto status = arrow_reader->ReadNext(&arrow_record_batch);
 
   if (garrow_error_check(error,
                          status,
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index ac4d36effa2fa..ad3af0fb69f6c 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -544,8 +544,8 @@ TEST_P(TestFileFormat, RoundTrip) {
   ASSERT_OK((*GetParam())(&batch1));  // NOLINT clang-tidy gtest issue
   ASSERT_OK((*GetParam())(&batch2));  // NOLINT clang-tidy gtest issue
 
-  std::vector<std::shared_ptr<RecordBatch>> in_batches = {batch1, batch2};
-  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+  BatchVector in_batches = {batch1, batch2};
+  BatchVector out_batches;
 
   ASSERT_OK(RoundTripHelper(in_batches, &out_batches));
 
@@ -564,14 +564,14 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
   }
   void TearDown() {}
 
-  Status RoundTripHelper(const RecordBatch& batch,
-                         std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
+  Status RoundTripHelper(const BatchVector& batches, BatchVector* out_batches) {
     // Write the file
     std::shared_ptr<RecordBatchWriter> writer;
-    RETURN_NOT_OK(RecordBatchStreamWriter::Open(sink_.get(), batch.schema(), &writer));
-    int num_batches = 5;
-    for (int i = 0; i < num_batches; ++i) {
-      RETURN_NOT_OK(writer->WriteRecordBatch(batch));
+    RETURN_NOT_OK(
+        RecordBatchStreamWriter::Open(sink_.get(), batches[0]->schema(), &writer));
+
+    for (const auto& batch : batches) {
+      RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
     }
     RETURN_NOT_OK(writer->Close());
     RETURN_NOT_OK(sink_->Close());
@@ -584,7 +584,7 @@ class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
 
     std::shared_ptr<RecordBatch> chunk;
     while (true) {
-      RETURN_NOT_OK(reader->ReadNextRecordBatch(&chunk));
+      RETURN_NOT_OK(reader->ReadNext(&chunk));
       if (chunk == nullptr) {
         break;
       }
@@ -604,9 +604,9 @@ TEST_P(TestStreamFormat, RoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK((*GetParam())(&batch));  // NOLINT clang-tidy gtest issue
 
-  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+  BatchVector out_batches;
 
-  ASSERT_OK(RoundTripHelper(*batch, &out_batches));
+  ASSERT_OK(RoundTripHelper({batch, batch, batch}, &out_batches));
 
   // Compare batches. Same
   for (size_t i = 0; i < out_batches.size(); ++i) {
@@ -666,17 +666,31 @@ TEST_F(TestStreamFormat, DictionaryRoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK(MakeDictionary(&batch));
 
-  std::vector<std::shared_ptr<RecordBatch>> out_batches;
-  ASSERT_OK(RoundTripHelper(*batch, &out_batches));
+  BatchVector out_batches;
+  ASSERT_OK(RoundTripHelper({batch}, &out_batches));
 
   CheckBatchDictionaries(*out_batches[0]);
 }
 
+TEST_F(TestStreamFormat, WriteTable) {
+  std::shared_ptr<RecordBatch> b1, b2, b3;
+  ASSERT_OK(MakeIntRecordBatch(&b1));
+  ASSERT_OK(MakeIntRecordBatch(&b2));
+  ASSERT_OK(MakeIntRecordBatch(&b3));
+
+  BatchVector out_batches;
+  ASSERT_OK(RoundTripHelper({b1, b2, b3}, &out_batches));
+
+  ASSERT_TRUE(b1->Equals(*out_batches[0]));
+  ASSERT_TRUE(b2->Equals(*out_batches[1]));
+  ASSERT_TRUE(b3->Equals(*out_batches[2]));
+}
+
 TEST_F(TestFileFormat, DictionaryRoundTrip) {
   std::shared_ptr<RecordBatch> batch;
   ASSERT_OK(MakeDictionary(&batch));
 
-  std::vector<std::shared_ptr<RecordBatch>> out_batches;
+  BatchVector out_batches;
   ASSERT_OK(RoundTripHelper({batch}, &out_batches));
 
   CheckBatchDictionaries(*out_batches[0]);
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 919dd7a27ff37..2a0633f31c876 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -389,8 +389,6 @@ static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
   return FileBlock{block->offset(), block->metaDataLength(), block->bodyLength()};
 }
 
-RecordBatchReader::~RecordBatchReader() {}
-
 class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
  public:
   RecordBatchStreamReaderImpl() {}
@@ -432,7 +430,7 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
     return GetSchema(message->header(), dictionary_memo_, &schema_);
   }
 
-  Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+  Status ReadNext(std::shared_ptr<RecordBatch>* batch) {
     std::unique_ptr<Message> message;
     RETURN_NOT_OK(ReadMessageAndValidate(message_reader_.get(), Message::RECORD_BATCH,
                                          true, &message));
@@ -504,8 +502,8 @@ std::shared_ptr<Schema> RecordBatchStreamReader::schema() const {
   return impl_->schema();
 }
 
-Status RecordBatchStreamReader::ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
-  return impl_->ReadNextRecordBatch(batch);
+Status RecordBatchStreamReader::ReadNext(std::shared_ptr<RecordBatch>* batch) {
+  return impl_->ReadNext(batch);
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 52b04abe2256b..54174f9ca1733 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -24,6 +24,7 @@
 #include <memory>
 
 #include "arrow/ipc/message.h"
+#include "arrow/table.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -43,21 +44,7 @@ class RandomAccessFile;
 
 namespace ipc {
 
-/// \brief Abstract interface for reading stream of record batches
-class ARROW_EXPORT RecordBatchReader {
- public:
-  virtual ~RecordBatchReader();
-
-  /// \return the shared schema of the record batches in the stream
-  virtual std::shared_ptr<Schema> schema() const = 0;
-
-  /// Read the next record batch in the stream. Return nullptr for batch when
-  /// reaching end of stream
-  ///
-  /// \param(out) batch the next loaded batch, nullptr at end of stream
-  /// \return Status
-  virtual Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) = 0;
-};
+using RecordBatchReader = ::arrow::RecordBatchReader;
 
 /// \class RecordBatchStreamReader
 /// \brief Synchronous batch stream reader that reads from io::InputStream
@@ -96,7 +83,7 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
                      std::shared_ptr<RecordBatchReader>* out);
 
   std::shared_ptr<Schema> schema() const override;
-  Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) override;
+  Status ReadNext(std::shared_ptr<RecordBatch>* batch) override;
 
  private:
   RecordBatchStreamReader();
diff --git a/cpp/src/arrow/ipc/stream-to-file.cc b/cpp/src/arrow/ipc/stream-to-file.cc
index 4f8cd80a12f76..859969b3aee54 100644
--- a/cpp/src/arrow/ipc/stream-to-file.cc
+++ b/cpp/src/arrow/ipc/stream-to-file.cc
@@ -45,7 +45,7 @@ Status ConvertToFile() {
 
   std::shared_ptr<RecordBatch> batch;
   while (true) {
-    RETURN_NOT_OK(reader->ReadNextRecordBatch(&batch));
+    RETURN_NOT_OK(reader->ReadNext(&batch));
     if (batch == nullptr) break;
     RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
   }
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 3f3d168f08435..9f557f6659eb4 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -655,6 +655,21 @@ Status GetTensorSize(const Tensor& tensor, int64_t* size) {
 
 RecordBatchWriter::~RecordBatchWriter() {}
 
+Status RecordBatchWriter::WriteTable(const Table& table) {
+  TableBatchReader reader(table);
+
+  std::shared_ptr<RecordBatch> batch;
+  while (true) {
+    RETURN_NOT_OK(reader.ReadNext(&batch));
+    if (batch == nullptr) {
+      break;
+    }
+    RETURN_NOT_OK(WriteRecordBatch(*batch, true));
+  }
+
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // Stream writer implementation
 
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 3f110fe26fbdc..bff7bb0996774 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -37,6 +37,7 @@ class MemoryPool;
 class RecordBatch;
 class Schema;
 class Status;
+class Table;
 class Tensor;
 
 namespace io {
@@ -53,15 +54,20 @@ class ARROW_EXPORT RecordBatchWriter {
  public:
   virtual ~RecordBatchWriter();
 
-  /// Write a record batch to the stream
+  /// \brief Write a record batch to the stream
   ///
   /// \param allow_64bit boolean permitting field lengths exceeding INT32_MAX
-  /// \return Status indicate success or failure
+  /// \return Status
   virtual Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) = 0;
 
-  /// Perform any logic necessary to finish the stream
+  /// \brief Write possibly-chunked table by creating sequence of record batches
+  /// \param[in] table
+  /// \return Status
+  Status WriteTable(const Table& table);
+
+  /// \brief Perform any logic necessary to finish the stream
   ///
-  /// \return Status indicate success or failure
+  /// \return Status
   virtual Status Close() = 0;
 
   /// In some cases, writing may require memory allocation. We use the default
@@ -84,7 +90,7 @@ class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
   /// \param(in) sink output stream to write to
   /// \param(in) schema the schema of the record batches to be written
   /// \param(out) out the created stream writer
-  /// \return Status indicating success or failure
+  /// \return Status
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchWriter>* out);
 
@@ -118,7 +124,7 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
   /// \param(in) sink output stream to write to
   /// \param(in) schema the schema of the record batches to be written
   /// \param(out) out the created stream writer
-  /// \return Status indicating success or failure
+  /// \return Status
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchWriter>* out);
 
@@ -195,7 +201,7 @@ Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
 /// \param[in] scheam the schema to write
 /// \param[in] pool a MemoryPool to allocate memory from
 /// \param[out] out the serialized schema
-/// \return Status
+/// \returnn Status
 ARROW_EXPORT
 Status SerializeSchema(const Schema& schema, MemoryPool* pool,
                        std::shared_ptr<Buffer>* out);
@@ -203,6 +209,7 @@ Status SerializeSchema(const Schema& schema, MemoryPool* pool,
 /// \brief Write multiple record batches to OutputStream
 /// \param[in] batches a vector of batches. Must all have same schema
 /// \param[out] dst an OutputStream
+/// \return Status
 ARROW_EXPORT
 Status WriteRecordBatchStream(const std::vector<std::shared_ptr<RecordBatch>>& batches,
                               io::OutputStream* dst);
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 3f64d75e01de6..cc1acf4befd3a 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -45,9 +45,7 @@ class PrettyPrinter {
   void OpenArray();
   void CloseArray();
 
-  void Flush() {
-    (*sink_) << std::flush;
-  }
+  void Flush() { (*sink_) << std::flush; }
 
  protected:
   int indent_;
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 769f609ac48ea..117bf232f2974 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -1192,7 +1192,7 @@ class DataFrameBlockCreator {
   Status CreateBlocks() {
     for (int i = 0; i < table_->num_columns(); ++i) {
       std::shared_ptr<Column> col = table_->column(i);
-      PandasBlock::type output_type;
+      PandasBlock::type output_type = PandasBlock::OBJECT;
       RETURN_NOT_OK(GetPandasBlockType(*col, options_, &output_type));
 
       int block_placement = 0;
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index bc12ba74453b3..a281fe3c62902 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -226,9 +226,9 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out)
   RETURN_NOT_OK(
       src->Read(sizeof(int32_t), &bytes_read, reinterpret_cast<uint8_t*>(&num_tensors)));
 
-  std::shared_ptr<ipc::RecordBatchReader> reader;
+  std::shared_ptr<RecordBatchReader> reader;
   RETURN_NOT_OK(ipc::RecordBatchStreamReader::Open(src, &reader));
-  RETURN_NOT_OK(reader->ReadNextRecordBatch(&out->batch));
+  RETURN_NOT_OK(reader->ReadNext(&out->batch));
 
   RETURN_NOT_OK(src->Tell(&offset));
   offset += 4;  // Skip the end-of-stream message
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 4b67492b7ed4a..a9c7e6da0ae73 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -33,6 +33,11 @@ using std::vector;
 
 namespace arrow {
 
+std::shared_ptr<Column> column(const std::shared_ptr<Field>& field,
+                               const std::vector<std::shared_ptr<Array>>& arrays) {
+  return std::make_shared<Column>(field, arrays);
+}
+
 class TestChunkedArray : public TestBase {
  protected:
   virtual void Construct() {
@@ -437,6 +442,29 @@ TEST_F(TestTable, AddColumn) {
   ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
 }
 
+TEST_F(TestTable, IsChunked) {
+  ArrayVector c1, c2;
+
+  auto a1 = MakePrimitive<Int32Array>(10);
+  auto a2 = MakePrimitive<Int32Array>(20);
+
+  auto sch1 = arrow::schema({field("f1", int32()), field("f2", int32())});
+
+  std::vector<std::shared_ptr<Column>> columns;
+
+  std::shared_ptr<RecordBatch> batch;
+
+  columns = {column(sch1->field(0), {a1}), column(sch1->field(1), {a1})};
+  auto t1 = std::make_shared<Table>(sch1, columns);
+
+  ASSERT_FALSE(t1->IsChunked());
+
+  columns = {column(sch1->field(0), {a2}), column(sch1->field(1), {a1, a1})};
+  auto t2 = std::make_shared<Table>(sch1, columns);
+
+  ASSERT_TRUE(t2->IsChunked());
+}
+
 class TestRecordBatch : public TestBase {};
 
 TEST_F(TestRecordBatch, Equals) {
@@ -522,4 +550,53 @@ TEST_F(TestRecordBatch, Slice) {
   }
 }
 
+class TestTableBatchReader : public TestBase {};
+
+TEST_F(TestTableBatchReader, ReadNext) {
+  ArrayVector c1, c2;
+
+  auto a1 = MakePrimitive<Int32Array>(10);
+  auto a2 = MakePrimitive<Int32Array>(20);
+  auto a3 = MakePrimitive<Int32Array>(30);
+  auto a4 = MakePrimitive<Int32Array>(10);
+
+  auto sch1 = arrow::schema({field("f1", int32()), field("f2", int32())});
+
+  std::vector<std::shared_ptr<Column>> columns;
+
+  std::shared_ptr<RecordBatch> batch;
+
+  columns = {column(sch1->field(0), {a1, a4, a2}), column(sch1->field(1), {a2, a2})};
+  Table t1(sch1, columns);
+
+  TableBatchReader i1(t1);
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_EQ(10, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_EQ(10, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_EQ(20, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_EQ(nullptr, batch);
+
+  columns = {column(sch1->field(0), {a1}), column(sch1->field(1), {a4})};
+  Table t2(sch1, columns);
+
+  TableBatchReader i2(t2);
+
+  ASSERT_OK(i2.ReadNext(&batch));
+  ASSERT_EQ(10, batch->num_rows());
+
+  // Ensure non-sliced
+  ASSERT_EQ(a1->data().get(), batch->column_data(0).get());
+  ASSERT_EQ(a4->data().get(), batch->column_data(1).get());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_EQ(nullptr, batch);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 8c7c4e2014465..4a08dc540a4d1 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -30,6 +30,8 @@
 
 namespace arrow {
 
+using internal::ArrayData;
+
 // ----------------------------------------------------------------------
 // ChunkedArray and Column methods
 
@@ -184,13 +186,13 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
 }
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-                         std::vector<std::shared_ptr<internal::ArrayData>>&& columns)
+                         std::vector<std::shared_ptr<ArrayData>>&& columns)
     : RecordBatch(schema, num_rows) {
   columns_ = std::move(columns);
 }
 
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-                         const std::vector<std::shared_ptr<internal::ArrayData>>& columns)
+                         const std::vector<std::shared_ptr<ArrayData>>& columns)
     : RecordBatch(schema, num_rows) {
   columns_ = columns;
 }
@@ -245,13 +247,13 @@ std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset) const {
 }
 
 std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length) const {
-  std::vector<std::shared_ptr<internal::ArrayData>> arrays;
+  std::vector<std::shared_ptr<ArrayData>> arrays;
   arrays.reserve(num_columns());
   for (const auto& field : columns_) {
     int64_t col_length = std::min(field->length - offset, length);
     int64_t col_offset = field->offset + offset;
 
-    auto new_data = std::make_shared<internal::ArrayData>(*field);
+    auto new_data = std::make_shared<ArrayData>(*field);
     new_data->length = col_length;
     new_data->offset = col_offset;
     new_data->null_count = kUnknownNullCount;
@@ -263,7 +265,7 @@ std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length)
 
 Status RecordBatch::Validate() const {
   for (int i = 0; i < num_columns(); ++i) {
-    const internal::ArrayData& arr = *columns_[i];
+    const ArrayData& arr = *columns_[i];
     if (arr.length != num_rows_) {
       std::stringstream ss;
       ss << "Number of rows in column " << i << " did not match batch: " << arr.length
@@ -471,9 +473,18 @@ Status Table::ValidateColumns() const {
   return Status::OK();
 }
 
-Status ARROW_EXPORT MakeTable(const std::shared_ptr<Schema>& schema,
-                              const std::vector<std::shared_ptr<Array>>& arrays,
-                              std::shared_ptr<Table>* table) {
+bool Table::IsChunked() const {
+  for (size_t i = 0; i < columns_.size(); ++i) {
+    if (columns_[i]->data()->num_chunks() > 1) {
+      return true;
+    }
+  }
+  return false;
+}
+
+Status MakeTable(const std::shared_ptr<Schema>& schema,
+                 const std::vector<std::shared_ptr<Array>>& arrays,
+                 std::shared_ptr<Table>* table) {
   // Make sure the length of the schema corresponds to the length of the vector
   if (schema->num_fields() != static_cast<int>(arrays.size())) {
     std::stringstream ss;
@@ -493,4 +504,105 @@ Status ARROW_EXPORT MakeTable(const std::shared_ptr<Schema>& schema,
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// Base record batch reader
+
+RecordBatchReader::~RecordBatchReader() {}
+
+#ifndef ARROW_NO_DEPRECATED_API
+Status RecordBatchReader::ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
+  return ReadNext(batch);
+}
+#endif
+
+// ----------------------------------------------------------------------
+// Convert a table to a sequence of record batches
+
+class TableBatchReader::TableBatchReaderImpl {
+ public:
+  explicit TableBatchReaderImpl(const Table& table)
+      : table_(table),
+        column_data_(table.num_columns()),
+        chunk_numbers_(table.num_columns(), 0),
+        chunk_offsets_(table.num_columns(), 0),
+        absolute_row_position_(0) {
+    for (int i = 0; i < table.num_columns(); ++i) {
+      column_data_[i] = table.column(i)->data().get();
+    }
+  }
+
+  Status ReadNext(std::shared_ptr<RecordBatch>* out) {
+    if (absolute_row_position_ == table_.num_rows()) {
+      *out = nullptr;
+      return Status::OK();
+    }
+
+    // Determine the minimum contiguous slice across all columns
+    int64_t chunksize = table_.num_rows();
+    std::vector<const Array*> chunks(table_.num_columns());
+    for (int i = 0; i < table_.num_columns(); ++i) {
+      auto chunk = column_data_[i]->chunk(chunk_numbers_[i]).get();
+      int64_t chunk_remaining = chunk->length() - chunk_offsets_[i];
+
+      if (chunk_remaining < chunksize) {
+        chunksize = chunk_remaining;
+      }
+
+      chunks[i] = chunk;
+    }
+
+    // Slice chunks and advance chunk index as appropriate
+    std::vector<std::shared_ptr<ArrayData>> batch_data;
+    batch_data.reserve(table_.num_columns());
+
+    for (int i = 0; i < table_.num_columns(); ++i) {
+      // Exhausted chunk
+      const Array* chunk = chunks[i];
+      const int64_t offset = chunk_offsets_[i];
+      std::shared_ptr<ArrayData> slice_data;
+      if ((chunk->length() - offset) == chunksize) {
+        ++chunk_numbers_[i];
+        chunk_offsets_[i] = 0;
+        if (chunk_offsets_[i] > 0) {
+          // Need to slice
+          slice_data = chunk->Slice(offset, chunksize)->data();
+        } else {
+          // No slice
+          slice_data = chunk->data();
+        }
+      } else {
+        slice_data = chunk->Slice(offset, chunksize)->data();
+      }
+      batch_data.emplace_back(std::move(slice_data));
+    }
+
+    absolute_row_position_ += chunksize;
+    *out =
+        std::make_shared<RecordBatch>(table_.schema(), chunksize, std::move(batch_data));
+
+    return Status::OK();
+  }
+
+  std::shared_ptr<Schema> schema() const { return table_.schema(); }
+
+ private:
+  const Table& table_;
+  std::vector<ChunkedArray*> column_data_;
+  std::vector<int> chunk_numbers_;
+  std::vector<int64_t> chunk_offsets_;
+  int64_t absolute_row_position_;
+};
+
+TableBatchReader::TableBatchReader(const Table& table) {
+  impl_.reset(new TableBatchReaderImpl(table));
+}
+
+TableBatchReader::~TableBatchReader() {}
+
+std::shared_ptr<Schema> TableBatchReader::schema() const { return impl_->schema(); }
+
+Status TableBatchReader::ReadNext(std::shared_ptr<RecordBatch>* out) {
+  return impl_->ReadNext(out);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index da2722da1a3f0..90336e982c7ff 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -262,6 +262,9 @@ class ARROW_EXPORT Table {
   /// \brief Perform any checks to validate the input arguments
   Status ValidateColumns() const;
 
+  /// \brief Return true if any column has multiple chunks
+  bool IsChunked() const;
+
  private:
   std::shared_ptr<Schema> schema_;
   std::vector<std::shared_ptr<Column>> columns_;
@@ -269,6 +272,44 @@ class ARROW_EXPORT Table {
   int64_t num_rows_;
 };
 
+/// \brief Abstract interface for reading stream of record batches
+class ARROW_EXPORT RecordBatchReader {
+ public:
+  virtual ~RecordBatchReader();
+
+  /// \return the shared schema of the record batches in the stream
+  virtual std::shared_ptr<Schema> schema() const = 0;
+
+  /// Read the next record batch in the stream. Return nullptr for batch when
+  /// reaching end of stream
+  ///
+  /// \param(out) batch the next loaded batch, nullptr at end of stream
+  /// \return Status
+  virtual Status ReadNext(std::shared_ptr<RecordBatch>* batch) = 0;
+
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \deprecated Since 0.7.0
+  Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch);
+#endif
+};
+
+/// \brief Compute a sequence of record batches from a (possibly chunked) Table
+class ARROW_EXPORT TableBatchReader : public RecordBatchReader {
+ public:
+  ~TableBatchReader();
+
+  /// \brief Read batches with the maximum possible size
+  explicit TableBatchReader(const Table& table);
+
+  std::shared_ptr<Schema> schema() const override;
+
+  Status ReadNext(std::shared_ptr<RecordBatch>* out) override;
+
+ private:
+  class TableBatchReaderImpl;
+  std::unique_ptr<TableBatchReaderImpl> impl_;
+};
+
 /// \brief Construct table from multiple input tables.
 /// \return Status, fails if any schemas are different
 ARROW_EXPORT
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 3b7ddcfa83c7d..5e6708871e650 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -632,11 +632,13 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
 
     cdef cppclass CRecordBatchWriter" arrow::ipc::RecordBatchWriter":
         CStatus Close()
-        CStatus WriteRecordBatch(const CRecordBatch& batch)
+        CStatus WriteRecordBatch(const CRecordBatch& batch,
+                                 c_bool allow_64bit)
+        CStatus WriteTable(const CTable& table)
 
     cdef cppclass CRecordBatchReader" arrow::ipc::RecordBatchReader":
         shared_ptr[CSchema] schema()
-        CStatus ReadNextRecordBatch(shared_ptr[CRecordBatch]* batch)
+        CStatus ReadNext(shared_ptr[CRecordBatch]* batch)
 
     cdef cppclass CRecordBatchStreamReader \
             " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index 027a00d8d4992..27e9167750991 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -175,6 +175,21 @@ cdef class _RecordBatchWriter:
                                               &self.writer))
         self.closed = False
 
+    def write(self, table_or_batch):
+        """
+        Write RecordBatch or Table to stream
+
+        Parameters
+        ----------
+        table_or_batch : {RecordBatch, Table}
+        """
+        if isinstance(table_or_batch, RecordBatch):
+            self.write_batch(table_or_batch)
+        elif isinstance(table_or_batch, Table):
+            self.write_table(table_or_batch)
+        else:
+            raise ValueError(type(table_or_batch))
+
     def write_batch(self, RecordBatch batch):
         """
         Write RecordBatch to stream
@@ -185,7 +200,18 @@ cdef class _RecordBatchWriter:
         """
         with nogil:
             check_status(self.writer.get()
-                         .WriteRecordBatch(deref(batch.batch)))
+                         .WriteRecordBatch(deref(batch.batch), 1))
+
+    def write_table(self, Table table):
+        """
+        Write RecordBatch to stream
+
+        Parameters
+        ----------
+        batch : RecordBatch
+        """
+        with nogil:
+            check_status(self.writer.get().WriteTable(table.table[0]))
 
     def close(self):
         """
@@ -245,7 +271,7 @@ cdef class _RecordBatchReader:
         cdef shared_ptr[CRecordBatch] batch
 
         with nogil:
-            check_status(self.reader.get().ReadNextRecordBatch(&batch))
+            check_status(self.reader.get().ReadNext(&batch))
 
         if batch.get() == NULL:
             raise StopIteration
@@ -263,7 +289,7 @@ cdef class _RecordBatchReader:
 
         with nogil:
             while True:
-                check_status(self.reader.get().ReadNextRecordBatch(&batch))
+                check_status(self.reader.get().ReadNext(&batch))
                 if batch.get() == NULL:
                     break
                 batches.push_back(batch)
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index ecdbe62113392..6ac44faba12b7 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -40,7 +40,7 @@ def _get_sink(self):
     def _get_source(self):
         return self.sink.getvalue()
 
-    def write_batches(self, num_batches=5):
+    def write_batches(self, num_batches=5, as_table=False):
         nrows = 5
         df = pd.DataFrame({
             'one': np.random.randn(nrows),
@@ -54,12 +54,17 @@ def write_batches(self, num_batches=5):
         for i in range(num_batches):
             unique_df = df.copy()
             unique_df['one'] = np.random.randn(len(df))
-
             batch = pa.RecordBatch.from_pandas(unique_df)
-            writer.write_batch(batch)
             frames.append(unique_df)
             batches.append(batch)
 
+        if as_table:
+            table = pa.Table.from_batches(batches)
+            writer.write_table(table)
+        else:
+            for batch in batches:
+                writer.write_batch(batch)
+
         writer.close()
         return frames, batches
 
@@ -75,8 +80,8 @@ def test_empty_file(self):
         with pytest.raises(pa.ArrowInvalid):
             pa.open_file(buf)
 
-    def test_simple_roundtrip(self):
-        _, batches = self.write_batches()
+    def _check_roundtrip(self, as_table=False):
+        _, batches = self.write_batches(as_table=as_table)
         file_contents = pa.BufferReader(self._get_source())
 
         reader = pa.open_file(file_contents)
@@ -89,6 +94,12 @@ def test_simple_roundtrip(self):
             assert batches[i].equals(batch)
             assert reader.schema.equals(batches[0].schema)
 
+    def test_simple_roundtrip(self):
+        self._check_roundtrip(as_table=False)
+
+    def test_write_table(self):
+        self._check_roundtrip(as_table=True)
+
     def test_read_all(self):
         _, batches = self.write_batches()
         file_contents = pa.BufferReader(self._get_source())

From 4a6a6cb47dcf832213f1fb31f2325ad10a3864bd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 10 Sep 2017 08:34:43 +0200
Subject: [PATCH 1033/1644] ARROW-1359: [C++] Add flavor='spark' option to
 write_parquet that sanitizes schema field names

I also made the default for `use_deprecated_int96_timestamps` None so that we can distinguish between unspecified and explicitly False. In the event that the user passes `flavor='spark'`, this is enabled. Once Spark processes the int96 deprecation in the future, we can remove this part.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1076 from wesm/ARROW-1359 and squashes the following commits:

8a60b66 [Wes McKinney] Use composition rather than inheritance
e3fa8ec [Wes McKinney] Add note about spark flavor to Sphinx docs
8159a51 [Wes McKinney] Add flavor='spark' option to write_parquet that sanitizes schema field names, turns on int96 timestamps
---
 python/doc/source/parquet.rst        |  7 ++
 python/pyarrow/parquet.py            | 96 +++++++++++++++++++++++++---
 python/pyarrow/table.pxi             | 31 +++++++--
 python/pyarrow/tests/test_parquet.py | 28 +++++++-
 python/pyarrow/types.pxi             |  5 +-
 5 files changed, 149 insertions(+), 18 deletions(-)

diff --git a/python/doc/source/parquet.rst b/python/doc/source/parquet.rst
index 7626c15048157..d466ba128cddc 100644
--- a/python/doc/source/parquet.rst
+++ b/python/doc/source/parquet.rst
@@ -217,6 +217,13 @@ such as those produced by Hive:
    dataset = pq.ParquetDataset('dataset_name/')
    table = dataset.read()
 
+Using with Spark
+----------------
+
+Spark places some constraints on the types of Parquet files it will read. The
+option ``flavor='spark'`` will set these options automatically and also
+sanitize field characters unsupported by Spark SQL.
+
 Multithreaded Reads
 -------------------
 
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 568aad4318205..1584b849ad7c4 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -18,17 +18,17 @@
 import os
 import inspect
 import json
-
+import re
 import six
 
 import numpy as np
 
 from pyarrow.filesystem import FileSystem, LocalFileSystem, S3FSWrapper
 from pyarrow._parquet import (ParquetReader, FileMetaData,  # noqa
-                              RowGroupMetaData, ParquetSchema,
-                              ParquetWriter)
+                              RowGroupMetaData, ParquetSchema)
 import pyarrow._parquet as _parquet  # noqa
 import pyarrow.lib as lib
+import pyarrow as pa
 
 
 # ----------------------------------------------------------------------
@@ -164,6 +164,73 @@ def _get_column_indices(self, column_names, use_pandas_metadata=False):
         return indices
 
 
+_SPARK_DISALLOWED_CHARS = re.compile('[ ,;{}()\n\t=]')
+
+
+def _sanitized_spark_field_name(name):
+    return _SPARK_DISALLOWED_CHARS.sub('_', name)
+
+
+def _sanitize_schema(schema, flavor):
+    if 'spark' in flavor:
+        sanitized_fields = []
+
+        schema_changed = False
+
+        for field in schema:
+            name = field.name
+            sanitized_name = _sanitized_spark_field_name(name)
+
+            if sanitized_name != name:
+                schema_changed = True
+                sanitized_field = pa.field(sanitized_name, field.type,
+                                           field.nullable, field.metadata)
+                sanitized_fields.append(sanitized_field)
+            else:
+                sanitized_fields.append(field)
+        return pa.schema(sanitized_fields), schema_changed
+    else:
+        return schema, False
+
+
+def _sanitize_table(table, new_schema, flavor):
+    # TODO: This will not handle prohibited characters in nested field names
+    if 'spark' in flavor:
+        column_data = [table[i].data for i in range(table.num_columns)]
+        return pa.Table.from_arrays(column_data, schema=new_schema)
+    else:
+        return table
+
+
+class ParquetWriter(object):
+    """
+
+    Parameters
+    ----------
+    where
+    schema
+    flavor : {'spark', ...}
+        Set options for compatibility with a particular reader
+    """
+    def __init__(self, where, schema, flavor=None, **options):
+        self.flavor = flavor
+        if flavor is not None:
+            schema, self.schema_changed = _sanitize_schema(schema, flavor)
+        else:
+            self.schema_changed = False
+
+        self.schema = schema
+        self.writer = _parquet.ParquetWriter(where, schema, **options)
+
+    def write_table(self, table, row_group_size=None):
+        if self.schema_changed:
+            table = _sanitize_table(table, self.schema, self.flavor)
+        self.writer.write_table(table, row_group_size=row_group_size)
+
+    def close(self):
+        self.writer.close()
+
+
 def _get_pandas_index_columns(keyvalues):
     return (json.loads(keyvalues[b'pandas'].decode('utf8'))
             ['index_columns'])
@@ -787,8 +854,9 @@ def read_pandas(source, columns=None, nthreads=1, metadata=None):
 
 def write_table(table, where, row_group_size=None, version='1.0',
                 use_dictionary=True, compression='snappy',
-                use_deprecated_int96_timestamps=False,
-                coerce_timestamps=None, **kwargs):
+                use_deprecated_int96_timestamps=None,
+                coerce_timestamps=None,
+                flavor=None, **kwargs):
     """
     Write a Table to Parquet format
 
@@ -804,15 +872,26 @@ def write_table(table, where, row_group_size=None, version='1.0',
     use_dictionary : bool or list
         Specify if we should use dictionary encoding in general or only for
         some columns.
-    use_deprecated_int96_timestamps : boolean, default False
-        Write nanosecond resolution timestamps to INT96 Parquet format
+    use_deprecated_int96_timestamps : boolean, default None
+        Write nanosecond resolution timestamps to INT96 Parquet
+        format. Defaults to False unless enabled by flavor argument
     coerce_timestamps : string, default None
         Cast timestamps a particular resolution.
         Valid values: {None, 'ms', 'us'}
     compression : str or dict
         Specify the compression codec, either on a general basis or per-column.
+    flavor : {'spark'}, default None
+        Sanitize schema or set other compatibility options for compatibility
     """
     row_group_size = kwargs.get('chunk_size', row_group_size)
+
+    if use_deprecated_int96_timestamps is None:
+        # Use int96 timestamps for Spark
+        if flavor is not None and 'spark' in flavor:
+            use_deprecated_int96_timestamps = True
+        else:
+            use_deprecated_int96_timestamps = False
+
     options = dict(
         use_dictionary=use_dictionary,
         compression=compression,
@@ -822,7 +901,8 @@ def write_table(table, where, row_group_size=None, version='1.0',
 
     writer = None
     try:
-        writer = ParquetWriter(where, table.schema, **options)
+        writer = ParquetWriter(where, table.schema, flavor=flavor,
+                               **options)
         writer.write_table(table, row_group_size=row_group_size)
     except:
         if writer is not None:
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index fc6099fe8430d..68eb5cbdcac6c 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -758,7 +758,7 @@ cdef class Table:
         return cls.from_arrays(arrays, names=names, metadata=metadata)
 
     @staticmethod
-    def from_arrays(arrays, names=None, dict metadata=None):
+    def from_arrays(arrays, names=None, schema=None, dict metadata=None):
         """
         Construct a Table from Arrow arrays or columns
 
@@ -777,11 +777,22 @@ cdef class Table:
         """
         cdef:
             vector[shared_ptr[CColumn]] columns
-            shared_ptr[CSchema] schema
+            Schema cy_schema
+            shared_ptr[CSchema] c_schema
             shared_ptr[CTable] table
             int i, K = <int> len(arrays)
 
-        _schema_from_arrays(arrays, names, metadata, &schema)
+        if schema is None:
+            _schema_from_arrays(arrays, names, metadata, &c_schema)
+        elif schema is not None:
+            if names is not None:
+                raise ValueError('Cannot pass schema and arrays')
+            cy_schema = schema
+
+            if len(schema) != len(arrays):
+                raise ValueError('Schema and number of arrays unequal')
+
+            c_schema = cy_schema.sp_schema
 
         columns.reserve(K)
 
@@ -789,23 +800,29 @@ cdef class Table:
             if isinstance(arrays[i], Array):
                 columns.push_back(
                     make_shared[CColumn](
-                        schema.get().field(i),
+                        c_schema.get().field(i),
                         (<Array> arrays[i]).sp_array
                     )
                 )
             elif isinstance(arrays[i], ChunkedArray):
                 columns.push_back(
                     make_shared[CColumn](
-                        schema.get().field(i),
+                        c_schema.get().field(i),
                         (<ChunkedArray> arrays[i]).sp_chunked_array
                     )
                 )
             elif isinstance(arrays[i], Column):
-                columns.push_back((<Column> arrays[i]).sp_column)
+                # Make sure schema field and column are consistent
+                columns.push_back(
+                    make_shared[CColumn](
+                        c_schema.get().field(i),
+                        (<Column> arrays[i]).sp_column.get().data()
+                    )
+                )
             else:
                 raise ValueError(type(arrays[i]))
 
-        table.reset(new CTable(schema, columns))
+        table.reset(new CTable(c_schema, columns))
         return pyarrow_wrap_table(table)
 
     @staticmethod
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 5dfe0a59588a9..9b5a4bcc43c51 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -17,6 +17,7 @@
 
 from os.path import join as pjoin
 import datetime
+import gc
 import io
 import os
 import json
@@ -562,6 +563,10 @@ def test_date_time_types():
     _check_roundtrip(table, expected=expected, version='2.0',
                      use_deprecated_int96_timestamps=True)
 
+    # Check that setting flavor to 'spark' uses int96 timestamps
+    _check_roundtrip(table, expected=expected, version='2.0',
+                     flavor='spark')
+
     # Unsupported stuff
     def _assert_unsupported(array):
         table = pa.Table.from_arrays([array], ['unsupported'])
@@ -576,6 +581,18 @@ def _assert_unsupported(array):
     _assert_unsupported(a7)
 
 
+@parquet
+def test_sanitized_spark_field_names():
+    a0 = pa.array([0, 1, 2, 3, 4])
+    name = 'prohib; ,\t{}'
+    table = pa.Table.from_arrays([a0], [name])
+
+    result = _roundtrip_table(table, flavor='spark')
+
+    expected_name = 'prohib______'
+    assert result.schema[0].name == expected_name
+
+
 @parquet
 def test_fixed_size_binary():
     t0 = pa.binary(10)
@@ -587,15 +604,19 @@ def test_fixed_size_binary():
     _check_roundtrip(table)
 
 
-def _check_roundtrip(table, expected=None, **params):
+def _roundtrip_table(table, **params):
     buf = io.BytesIO()
     _write_table(table, buf, **params)
     buf.seek(0)
 
+    return _read_table(buf)
+
+
+def _check_roundtrip(table, expected=None, **params):
     if expected is None:
         expected = table
 
-    result = _read_table(buf)
+    result = _roundtrip_table(table, **params)
     assert result.equals(expected)
 
 
@@ -1181,6 +1202,9 @@ def test_write_error_deletes_incomplete_file(tmpdir):
     except pa.ArrowException:
         pass
 
+    # Ensure that object has been destructed; this causes test failures on
+    # Windows
+    gc.collect()
     assert not os.path.exists(filename)
 
 
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 3eaee6ced33f7..56670f64a7872 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -299,7 +299,6 @@ cdef class Schema:
         return self.schema.num_fields()
 
     def __getitem__(self, int i):
-
         cdef:
             Field result = Field()
             int num_fields = self.schema.num_fields()
@@ -318,6 +317,10 @@ cdef class Schema:
 
         return result
 
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self[i]
+
     def _check_null(self):
         if self.schema == NULL:
             raise ReferenceError(

From f211463866a680b5e58394955a610f6765a02a27 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 10 Sep 2017 11:26:06 -0400
Subject: [PATCH 1034/1644] ARROW-1514: [C++] Fix a typo in document

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1079 from kou/cpp-fix-typo and squashes the following commits:

be5664b6 [Kouhei Sutou] [C++] Fix a typo in document
---
 cpp/src/arrow/ipc/writer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index bff7bb0996774..df03f4f1381f7 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -201,7 +201,7 @@ Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
 /// \param[in] scheam the schema to write
 /// \param[in] pool a MemoryPool to allocate memory from
 /// \param[out] out the serialized schema
-/// \returnn Status
+/// \return Status
 ARROW_EXPORT
 Status SerializeSchema(const Schema& schema, MemoryPool* pool,
                        std::shared_ptr<Buffer>* out);

From ce623a979a3354d12918b35425c6c78e9e0c70f2 Mon Sep 17 00:00:00 2001
From: rvernica <rvernica@gmail.com>
Date: Sat, 9 Sep 2017 18:10:24 -0700
Subject: [PATCH 1035/1644] ARROW-1512: [C++] Fix API change in documentation

Closes #1078

Change-Id: I0c3263757088ccb352c5428a86d6e756fc54018f
---
 cpp/apidoc/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
index ab9bbaa405abc..293241c0dc767 100644
--- a/cpp/apidoc/index.md
+++ b/cpp/apidoc/index.md
@@ -84,7 +84,7 @@ undefined.
     const uint8_t* null_bitmap = int64_array->null_bitmap_data();
 
     // Get the pointer to the actual data
-    const int64_t* data = int64_array->raw_data();
+    const int64_t* data = int64_array->raw_values();
 
 In the above example, we have yet skipped explaining two things in the code.
 On constructing the builder, we have passed `arrow::int64()` to it. This is

From c1492bf4a01600c838c59ebef7dd88694cca41c3 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 10 Sep 2017 11:29:37 -0400
Subject: [PATCH 1036/1644] ARROW-1510: [GLib] Support cast

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1075 from kou/glib-add-array-cast and squashes the following commits:

dead23ee [Kouhei Sutou] [GLib] Omit tests with old GObject Introspection
29a9fd81 [Kouhei Sutou] [GLib] Use GLib 2.44 or later on Travis CI
cd6a6436 [Kouhei Sutou] [GLib] Support cast
---
 c_glib/arrow-glib/Makefile.am             |   9 ++
 c_glib/arrow-glib/array.cpp               |  56 +++++++++
 c_glib/arrow-glib/array.h                 |   6 +
 c_glib/arrow-glib/arrow-glib.h            |   1 +
 c_glib/arrow-glib/compute.cpp             | 145 ++++++++++++++++++++++
 c_glib/arrow-glib/compute.h               |  39 ++++++
 c_glib/arrow-glib/compute.hpp             |  27 ++++
 c_glib/configure.ac                       |   2 +-
 c_glib/doc/reference/arrow-glib-docs.sgml |  16 +++
 c_glib/test/helper/omittable.rb           |  10 +-
 c_glib/test/test-boolean-array.rb         |   2 +-
 c_glib/test/test-buffer.rb                |   2 +-
 c_glib/test/test-cast.rb                  |  45 +++++++
 c_glib/test/test-double-array.rb          |   2 +-
 c_glib/test/test-float-array.rb           |   2 +-
 c_glib/test/test-int16-array.rb           |   2 +-
 c_glib/test/test-tensor.rb                |   4 +-
 c_glib/test/test-uint16-array.rb          |   2 +-
 c_glib/test/test-uint32-array.rb          |   2 +-
 c_glib/test/test-uint64-array.rb          |   2 +-
 ci/travis_before_script_c_glib.sh         |   4 +
 21 files changed, 368 insertions(+), 12 deletions(-)
 create mode 100644 c_glib/arrow-glib/compute.cpp
 create mode 100644 c_glib/arrow-glib/compute.h
 create mode 100644 c_glib/arrow-glib/compute.hpp
 create mode 100644 c_glib/test/test-cast.rb

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index f33a8d197875c..fafd6d27fa950 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -70,6 +70,9 @@ libarrow_glib_la_headers +=			\
 	reader.h				\
 	writer.h
 
+libarrow_glib_la_headers +=			\
+	compute.h
+
 libarrow_glib_la_generated_headers =		\
 	enums.h
 
@@ -108,6 +111,9 @@ libarrow_glib_la_sources +=			\
 	reader.cpp				\
 	writer.cpp
 
+libarrow_glib_la_sources +=			\
+	compute.cpp
+
 libarrow_glib_la_cpp_headers =			\
 	array.hpp				\
 	array-builder.hpp			\
@@ -138,6 +144,9 @@ libarrow_glib_la_cpp_headers +=			\
 	reader.hpp				\
 	writer.hpp
 
+libarrow_glib_la_cpp_headers +=			\
+	compute.hpp
+
 libarrow_glib_la_SOURCES =			\
 	$(libarrow_glib_la_sources)		\
 	$(libarrow_glib_la_cpp_headers)
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 007f23314f7d8..1bdbb0c0da9c9 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -23,6 +23,7 @@
 
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/buffer.hpp>
+#include <arrow-glib/compute.hpp>
 #include <arrow-glib/data-type.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/type.hpp>
@@ -500,6 +501,61 @@ garrow_array_to_string(GArrowArray *array, GError **error)
   }
 }
 
+/**
+ * garrow_array_cast:
+ * @array: A #GArrowArray.
+ * @target_data_type: A #GArrowDataType of cast target data.
+ * @options: (nullable): A #GArrowCastOptions.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   A newly created casted array on success, %NULL on error.
+ *
+ * Since: 0.7.0
+ */
+GArrowArray *
+garrow_array_cast(GArrowArray *array,
+                  GArrowDataType *target_data_type,
+                  GArrowCastOptions *options,
+                  GError **error)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto arrow_array_raw = arrow_array.get();
+  auto memory_pool = arrow::default_memory_pool();
+  arrow::compute::FunctionContext context(memory_pool);
+  auto arrow_target_data_type = garrow_data_type_get_raw(target_data_type);
+  std::shared_ptr<arrow::Array> arrow_casted_array;
+  arrow::Status status;
+  if (options) {
+    auto arrow_options = garrow_cast_options_get_raw(options);
+    status = arrow::compute::Cast(&context,
+                                  *arrow_array_raw,
+                                  arrow_target_data_type,
+                                  *arrow_options,
+                                  &arrow_casted_array);
+  } else {
+    arrow::compute::CastOptions arrow_options;
+    status = arrow::compute::Cast(&context,
+                                  *arrow_array_raw,
+                                  arrow_target_data_type,
+                                  arrow_options,
+                                  &arrow_casted_array);
+  }
+
+  if (!status.ok()) {
+    std::stringstream message;
+    message << "[array][cast] <";
+    message << arrow_array->type()->ToString();
+    message << "> -> <";
+    message << arrow_target_data_type->ToString();
+    message << ">";
+    garrow_error_check(error, status, message.str().c_str());
+    return NULL;
+  }
+
+  return garrow_array_new_raw(&arrow_casted_array);
+}
+
 
 G_DEFINE_TYPE(GArrowNullArray,               \
               garrow_null_array,             \
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index ebfcf4335a9ae..8a611225112e5 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include <arrow-glib/buffer.h>
+#include <arrow-glib/compute.h>
 #include <arrow-glib/data-type.h>
 
 G_BEGIN_DECLS
@@ -82,6 +83,11 @@ GArrowArray   *garrow_array_slice       (GArrowArray *array,
 gchar         *garrow_array_to_string   (GArrowArray *array,
                                          GError **error);
 
+GArrowArray   *garrow_array_cast        (GArrowArray *array,
+                                         GArrowDataType *target_data_type,
+                                         GArrowCastOptions *options,
+                                         GError **error);
+
 #define GARROW_TYPE_NULL_ARRAY                  \
   (garrow_null_array_get_type())
 #define GARROW_NULL_ARRAY(obj)                          \
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index 7ab95317210e7..f15f16db9a35d 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -23,6 +23,7 @@
 #include <arrow-glib/array-builder.h>
 #include <arrow-glib/chunked-array.h>
 #include <arrow-glib/column.h>
+#include <arrow-glib/compute.h>
 #include <arrow-glib/data-type.h>
 #include <arrow-glib/enums.h>
 #include <arrow-glib/error.h>
diff --git a/c_glib/arrow-glib/compute.cpp b/c_glib/arrow-glib/compute.cpp
new file mode 100644
index 0000000000000..f7a48d148b0a4
--- /dev/null
+++ b/c_glib/arrow-glib/compute.cpp
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/compute.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: compute
+ * @section_id: compute-classes
+ * @title: Compute classes
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowCastOptions is a class to custom garrow_array_cast().
+ */
+
+typedef struct GArrowCastOptionsPrivate_ {
+  arrow::compute::CastOptions options;
+} GArrowCastOptionsPrivate;
+
+enum {
+  PROP_0,
+  PROP_ALLOW_INT_OVERFLOW
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowCastOptions,
+                           garrow_cast_options,
+                           G_TYPE_OBJECT)
+
+#define GARROW_CAST_OPTIONS_GET_PRIVATE(object) \
+  static_cast<GArrowCastOptionsPrivate *>(      \
+    garrow_cast_options_get_instance_private(   \
+      GARROW_CAST_OPTIONS(object)))
+
+static void
+garrow_cast_options_set_property(GObject *object,
+                                 guint prop_id,
+                                 const GValue *value,
+                                 GParamSpec *pspec)
+{
+  auto priv = GARROW_CAST_OPTIONS_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_ALLOW_INT_OVERFLOW:
+    priv->options.allow_int_overflow = g_value_get_boolean(value);
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_cast_options_get_property(GObject *object,
+                                 guint prop_id,
+                                 GValue *value,
+                                 GParamSpec *pspec)
+{
+  auto priv = GARROW_CAST_OPTIONS_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_ALLOW_INT_OVERFLOW:
+    g_value_set_boolean(value, priv->options.allow_int_overflow);
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_cast_options_init(GArrowCastOptions *object)
+{
+}
+
+static void
+garrow_cast_options_class_init(GArrowCastOptionsClass *klass)
+{
+  GParamSpec *spec;
+
+  auto gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->set_property = garrow_cast_options_set_property;
+  gobject_class->get_property = garrow_cast_options_get_property;
+
+  spec = g_param_spec_boolean("allow_int_overflow",
+                              "Allow int overflow",
+                              "Whether integer overflow is allowed or not",
+                              FALSE,
+                              static_cast<GParamFlags>(G_PARAM_READWRITE));
+  g_object_class_install_property(gobject_class, PROP_ALLOW_INT_OVERFLOW, spec);
+}
+
+/**
+ * garrow_cast_options_new:
+ *
+ * Returns: A newly created #GArrowCastOptions.
+ *
+ * Since: 0.7.0
+ */
+GArrowCastOptions *
+garrow_cast_options_new(void)
+{
+  auto cast_options = g_object_new(GARROW_TYPE_CAST_OPTIONS, NULL);
+  return GARROW_CAST_OPTIONS(cast_options);
+}
+
+G_END_DECLS
+
+GArrowCastOptions *
+garrow_cast_options_new_raw(arrow::compute::CastOptions *arrow_cast_options)
+{
+  auto cast_options =
+    g_object_new(GARROW_TYPE_CAST_OPTIONS,
+                 "allow-int-overflow", arrow_cast_options->allow_int_overflow,
+                 NULL);
+  return GARROW_CAST_OPTIONS(cast_options);
+}
+
+arrow::compute::CastOptions *
+garrow_cast_options_get_raw(GArrowCastOptions *cast_options)
+{
+  auto priv = GARROW_CAST_OPTIONS_GET_PRIVATE(cast_options);
+  return &(priv->options);
+}
diff --git a/c_glib/arrow-glib/compute.h b/c_glib/arrow-glib/compute.h
new file mode 100644
index 0000000000000..4b944ad30d320
--- /dev/null
+++ b/c_glib/arrow-glib/compute.h
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_CAST_OPTIONS (garrow_cast_options_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowCastOptions,
+                         garrow_cast_options,
+                         GARROW,
+                         CAST_OPTIONS,
+                         GObject)
+struct _GArrowCastOptionsClass
+{
+  GObjectClass parent_class;
+};
+
+GArrowCastOptions *garrow_cast_options_new(void);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/compute.hpp b/c_glib/arrow-glib/compute.hpp
new file mode 100644
index 0000000000000..bdeaad2922ece
--- /dev/null
+++ b/c_glib/arrow-glib/compute.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/compute/api.h>
+
+#include <arrow-glib/compute.h>
+
+GArrowCastOptions *garrow_cast_options_new_raw(arrow::compute::CastOptions *arrow_cast_options);
+arrow::compute::CastOptions *garrow_cast_options_get_raw(GArrowCastOptions *cast_options);
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index 103f11b4bbedd..b4e30166ef245 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -73,7 +73,7 @@ AC_ARG_WITH(arrow-cpp-build-dir,
   [GARROW_ARROW_CPP_BUILD_DIR="$withval"],
   [GARROW_ARROW_CPP_BUILD_DIR=""])
 if test "x$GARROW_ARROW_CPP_BUILD_DIR" = "x"; then
-  PKG_CHECK_MODULES([ARROW], [arrow])
+  PKG_CHECK_MODULES([ARROW], [arrow arrow-compute])
 else
   ARROW_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src"
   ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}"
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 26fd2f6262ba5..9c0475cc76c43 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -128,6 +128,22 @@
     <title>Index of deprecated API</title>
     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
   </index>
+  <index id="api-index-0-7-0" role="0.7.0">
+    <title>Index of new symbols in 0.7.0</title>
+    <xi:include href="xml/api-index-0.7.0.xml"><xi:fallback /></xi:include>
+  </index>
+  <index id="api-index-0-6-0" role="0.6.0">
+    <title>Index of new symbols in 0.6.0</title>
+    <xi:include href="xml/api-index-0.6.0.xml"><xi:fallback /></xi:include>
+  </index>
+  <index id="api-index-0-5-0" role="0.5.0">
+    <title>Index of new symbols in 0.5.0</title>
+    <xi:include href="xml/api-index-0.5.0.xml"><xi:fallback /></xi:include>
+  </index>
+  <index id="api-index-0-4-0" role="0.4.0">
+    <title>Index of new symbols in 0.4.0</title>
+    <xi:include href="xml/api-index-0.4.0.xml"><xi:fallback /></xi:include>
+  </index>
   <index id="api-index-0-3-0" role="0.3.0">
     <title>Index of new symbols in 0.3.0</title>
     <xi:include href="xml/api-index-0.3.0.xml"><xi:fallback /></xi:include>
diff --git a/c_glib/test/helper/omittable.rb b/c_glib/test/helper/omittable.rb
index a16ad32485e15..a1c0334b63a2b 100644
--- a/c_glib/test/helper/omittable.rb
+++ b/c_glib/test/helper/omittable.rb
@@ -17,12 +17,20 @@
 
 module Helper
   module Omittable
-    def require_gi(major, minor, micro)
+    def require_gi_bindings(major, minor, micro)
       return if GLib.check_binding_version?(major, minor, micro)
       message =
         "Require gobject-introspection #{major}.#{minor}.#{micro} or later: " +
         GLib::BINDING_VERSION.join(".")
       omit(message)
     end
+
+    def require_gi(major, minor, micro)
+      return if GObjectIntrospection::Version.or_later?(major, minor, micro)
+      message =
+        "Require GObject Introspection #{major}.#{minor}.#{micro} or later: " +
+        GObjectIntrospection::Version::STRING
+      omit(message)
+    end
   end
 end
diff --git a/c_glib/test/test-boolean-array.rb b/c_glib/test/test-boolean-array.rb
index 622e4e90c482b..ae22bce48b64a 100644
--- a/c_glib/test/test-boolean-array.rb
+++ b/c_glib/test/test-boolean-array.rb
@@ -44,7 +44,7 @@ def test_value
   end
 
   def test_values
-    require_gi(3, 1, 9)
+    require_gi_bindings(3, 1, 9)
     builder = Arrow::BooleanArrayBuilder.new
     builder.append(true)
     builder.append(false)
diff --git a/c_glib/test/test-buffer.rb b/c_glib/test/test-buffer.rb
index 39ae631a0f68d..3b02df7ba7e84 100644
--- a/c_glib/test/test-buffer.rb
+++ b/c_glib/test/test-buffer.rb
@@ -51,7 +51,7 @@ def test_data
   end
 
   def test_mutable_data
-    require_gi(3, 1, 2)
+    require_gi_bindings(3, 1, 2)
     assert_nil(@buffer.mutable_data)
   end
 
diff --git a/c_glib/test/test-cast.rb b/c_glib/test/test-cast.rb
new file mode 100644
index 0000000000000..f32cd83cc9d8a
--- /dev/null
+++ b/c_glib/test/test-cast.rb
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestCast < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_safe
+    require_gi(1, 42, 0)
+    data = [-1, 2, nil]
+    assert_equal(build_int32_array(data),
+                 build_int8_array(data).cast(Arrow::Int32DataType.new))
+  end
+
+  sub_test_case("allow-int-overflow") do
+    def test_default
+      require_gi(1, 42, 0)
+      assert_raise(Arrow::Error::Invalid) do
+        build_int32_array([128]).cast(Arrow::Int8DataType.new)
+      end
+    end
+
+    def test_true
+      options = Arrow::CastOptions.new
+      options.allow_int_overflow = true
+      assert_equal(build_int8_array([-128]),
+                   build_int32_array([128]).cast(Arrow::Int8DataType.new,
+                                                 options))
+    end
+  end
+end
diff --git a/c_glib/test/test-double-array.rb b/c_glib/test/test-double-array.rb
index c4a2d3017b6d0..1213a5dfe53d6 100644
--- a/c_glib/test/test-double-array.rb
+++ b/c_glib/test/test-double-array.rb
@@ -44,7 +44,7 @@ def test_value
   end
 
   def test_values
-    require_gi(3, 1, 7)
+    require_gi_bindings(3, 1, 7)
     builder = Arrow::DoubleArrayBuilder.new
     builder.append(1.5)
     builder.append(3)
diff --git a/c_glib/test/test-float-array.rb b/c_glib/test/test-float-array.rb
index ce77c8b3defba..c8e1b4d864c08 100644
--- a/c_glib/test/test-float-array.rb
+++ b/c_glib/test/test-float-array.rb
@@ -44,7 +44,7 @@ def test_value
   end
 
   def test_values
-    require_gi(3, 1, 7)
+    require_gi_bindings(3, 1, 7)
     builder = Arrow::FloatArrayBuilder.new
     builder.append(1.5)
     builder.append(3)
diff --git a/c_glib/test/test-int16-array.rb b/c_glib/test/test-int16-array.rb
index 8703e803ff0f4..13646e0d5b818 100644
--- a/c_glib/test/test-int16-array.rb
+++ b/c_glib/test/test-int16-array.rb
@@ -44,7 +44,7 @@ def test_value
   end
 
   def test_values
-    require_gi(3, 1, 7)
+    require_gi_bindings(3, 1, 7)
     builder = Arrow::Int16ArrayBuilder.new
     builder.append(-1)
     builder.append(2)
diff --git a/c_glib/test/test-tensor.rb b/c_glib/test/test-tensor.rb
index bdfc6576a3f1e..6cead586695f6 100644
--- a/c_glib/test/test-tensor.rb
+++ b/c_glib/test/test-tensor.rb
@@ -66,12 +66,12 @@ def test_buffer
   end
 
   def test_shape
-    require_gi(3, 1, 2)
+    require_gi_bindings(3, 1, 2)
     assert_equal(@shape, @tensor.shape)
   end
 
   def test_strides
-    require_gi(3, 1, 2)
+    require_gi_bindings(3, 1, 2)
     assert_equal([4, 2, 1], @tensor.strides)
   end
 
diff --git a/c_glib/test/test-uint16-array.rb b/c_glib/test/test-uint16-array.rb
index 1c79c999c30cf..1362c8e7ff507 100644
--- a/c_glib/test/test-uint16-array.rb
+++ b/c_glib/test/test-uint16-array.rb
@@ -44,7 +44,7 @@ def test_value
   end
 
   def test_values
-    require_gi(3, 1, 7)
+    require_gi_bindings(3, 1, 7)
     builder = Arrow::UInt16ArrayBuilder.new
     builder.append(1)
     builder.append(2)
diff --git a/c_glib/test/test-uint32-array.rb b/c_glib/test/test-uint32-array.rb
index a4d1e3257c98e..01b3edb353ff2 100644
--- a/c_glib/test/test-uint32-array.rb
+++ b/c_glib/test/test-uint32-array.rb
@@ -44,7 +44,7 @@ def test_value
   end
 
   def test_values
-    require_gi(3, 1, 7)
+    require_gi_bindings(3, 1, 7)
     builder = Arrow::UInt32ArrayBuilder.new
     builder.append(1)
     builder.append(2)
diff --git a/c_glib/test/test-uint64-array.rb b/c_glib/test/test-uint64-array.rb
index b7f9c403e2494..a002af269293c 100644
--- a/c_glib/test/test-uint64-array.rb
+++ b/c_glib/test/test-uint64-array.rb
@@ -44,7 +44,7 @@ def test_value
   end
 
   def test_values
-    require_gi(3, 1, 7)
+    require_gi_bindings(3, 1, 7)
     builder = Arrow::UInt64ArrayBuilder.new
     builder.append(1)
     builder.append(2)
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 7ab8e2eaa03ee..6799ae4ce41b5 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -26,6 +26,10 @@ if [ $TRAVIS_OS_NAME == "osx" ]; then
   brew upgrade git cmake wget libtool
 
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
+else
+  sudo apt-add-repository -y ppa:jonathonf/gtk3.18
+  sudo apt-get update
+  sudo apt-get install -V -y libglib2.0-dev
 fi
 
 gem install test-unit gobject-introspection

From 0ac81b5c05aeb7c1c01a30d38313f78408fcf925 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 10 Sep 2017 13:58:05 -0400
Subject: [PATCH 1037/1644] ARROW-1515: [GLib] Detect version directly

Running ./autogen.sh isn't required for version update in java/pom.xml.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1080 from kou/glib-detect-version-directly and squashes the following commits:

34015083 [Kouhei Sutou] [GLib] Detect version directly
---
 c_glib/.gitignore   | 1 -
 c_glib/Makefile.am  | 3 +--
 c_glib/autogen.sh   | 5 -----
 c_glib/configure.ac | 5 ++++-
 4 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/c_glib/.gitignore b/c_glib/.gitignore
index 796b842083f9e..ff3615a3e4920 100644
--- a/c_glib/.gitignore
+++ b/c_glib/.gitignore
@@ -33,7 +33,6 @@ Makefile.in
 /libtool
 /m4/
 /stamp-h1
-/version
 /arrow-glib/enums.c
 /arrow-glib/enums.h
 /arrow-glib/stamp-*
diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am
index 2e23f125683ba..d059d122ba412 100644
--- a/c_glib/Makefile.am
+++ b/c_glib/Makefile.am
@@ -24,8 +24,7 @@ SUBDIRS =					\
 
 EXTRA_DIST =					\
 	README.md				\
-	test					\
-	version
+	test
 
 arrow_glib_docdir = ${datarootdir}/doc/arrow-glib
 arrow_glib_doc_DATA =				\
diff --git a/c_glib/autogen.sh b/c_glib/autogen.sh
index 08e33e6ca07c0..31c22850ddad1 100755
--- a/c_glib/autogen.sh
+++ b/c_glib/autogen.sh
@@ -20,11 +20,6 @@
 set -u
 set -e
 
-ruby \
-    -e 'print ARGF.read.scan(/^  <version>(.+?)<\/version>/)[0][0]' \
-    ../java/pom.xml > \
-    version
-
 mkdir -p m4
 
 gtkdocize --copy --docdir doc/reference
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index b4e30166ef245..938064c16a7dc 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -17,7 +17,10 @@
 
 AC_PREREQ(2.65)
 
-m4_define([arrow_glib_version], m4_include(version))
+m4_define([arrow_glib_version],
+           m4_esyscmd(grep "^  <version>" "$(dirname $0)/../java/pom.xml" | \
+                        sed -e 's/\(^  <version>\|<\/version>$\)//g' | \
+                        tr -d '\n'))
 AC_INIT([arrow-glib],
         arrow_glib_version,
         [https://issues.apache.org/jira/browse/ARROW],

From a534e5d53ed9530c218f12a09bd8e7b1ea4c8935 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 10 Sep 2017 13:58:45 -0400
Subject: [PATCH 1038/1644] ARROW-1516: [GLib] Update document

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1081 from kou/glib-update-document and squashes the following commits:

d3e123ab [Kouhei Sutou] [GLib] Update document
---
 c_glib/arrow-glib/input-stream.cpp | 4 ++--
 c_glib/arrow-glib/type.h           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index db3efd4b0b24b..11c63f265cde4 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -47,8 +47,8 @@ G_BEGIN_DECLS
  *
  * #GArrowBufferInputStream is a class to read data on buffer.
  *
- * #GArrowMemoryMappedFile is a class to read data in file by mapping
- * the file on memory. It supports zero copy.
+ * #GArrowMemoryMappedInputStream is a class to read data in file by
+ * mapping the file on memory. It supports zero copy.
  *
  * #GArrowGIOInputStream is a class for `GInputStream` based input
  * stream.
diff --git a/c_glib/arrow-glib/type.h b/c_glib/arrow-glib/type.h
index 65d28959e8d78..378d821ef2118 100644
--- a/c_glib/arrow-glib/type.h
+++ b/c_glib/arrow-glib/type.h
@@ -88,7 +88,7 @@ typedef enum {
 /**
  * GArrowTimeUnit:
  * @GARROW_TIME_UNIT_SECOND: Second.
- * @GARROW_TIME_UNIT_MILLISECOND: Millisecond.
+ * @GARROW_TIME_UNIT_MILLI: Millisecond.
  * @GARROW_TIME_UNIT_MICRO: Microsecond.
  * @GARROW_TIME_UNIT_NANO: Nanosecond.
  *

From 840b68744ac65c3e151bbc03f0e195fce2bb0f30 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 10 Sep 2017 13:59:49 -0400
Subject: [PATCH 1039/1644] ARROW-1517: Remove unnecessary temporary in
 DecimalUtil::ToString function

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1082 from cpcloud/ARROW-1517 and squashes the following commits:

49e30594 [Phillip Cloud] ARROW-1517: Remove unnecessary temporary in DecimalUtil::ToString function
---
 cpp/src/arrow/util/decimal.cc | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 3e17bf6274f4a..7dd5807330747 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -153,25 +153,24 @@ Status FromString(const std::string& s, Int128* out, int* precision, int* scale)
 }
 
 std::string ToString(const Int128& decimal_value, int precision, int scale) {
-  Int128 value(decimal_value);
+  const bool is_negative = decimal_value < 0;
 
   // Decimal values are sent to clients as strings so in the interest of
   // speed the string will be created without the using stringstream with the
   // whole/fractional_part().
   size_t last_char_idx = precision + (scale > 0)  // Add a space for decimal place
                          + (scale == precision)   // Add a space for leading 0
-                         + (value < 0);           // Add a space for negative sign
+                         + is_negative;           // Add a space for negative sign
   std::string str(last_char_idx, '0');
 
   // Start filling in the values in reverse order by taking the last digit
   // of the value. Use a positive value and worry about the sign later. At this
   // point the last_char_idx points to the string terminator.
-  Int128 remaining_value(value);
+  Int128 remaining_value(decimal_value);
 
-  size_t first_digit_idx = 0;
-  if (value < 0) {
-    remaining_value = -value;
-    first_digit_idx = 1;
+  const auto first_digit_idx = static_cast<size_t>(is_negative);
+  if (is_negative) {
+    remaining_value.Negate();
   }
 
   if (scale > 0) {
@@ -199,7 +198,7 @@ std::string ToString(const Int128& decimal_value, int precision, int scale) {
     // For safety, enforce string length independent of remaining_value.
   } while (last_char_idx > first_digit_idx);
 
-  if (value < 0) {
+  if (is_negative) {
     str[0] = '-';
   }
 

From a45b0471223904a0d07bf034cf29960935f86ef3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 11 Sep 2017 00:19:32 -0400
Subject: [PATCH 1040/1644] ARROW-1307: [Python] Expand IPC section to include
 object serialization, Feather format. Add Feather functions to API listing

Close #1084

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1085 from wesm/ARROW-1307 and squashes the following commits:

6a461107 [Wes McKinney] typo
c161ddf1 [Wes McKinney] Expand IPC section to include object serialization, Feather format. Add feather functions to API
---
 python/doc/source/api.rst |  19 ++++-
 python/doc/source/ipc.rst | 170 ++++++++++++++++++++++++++++++++++++--
 python/pyarrow/feather.py |  10 ++-
 3 files changed, 188 insertions(+), 11 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index cc24e49f8ae61..fa07d203df397 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -182,8 +182,8 @@ File Systems
 
 .. _api.ipc:
 
-Interprocess Communication and Serialization
---------------------------------------------
+Serialization and IPC
+---------------------
 
 .. autosummary::
    :toctree: generated/
@@ -210,11 +210,26 @@ Interprocess Communication and Serialization
    SerializedPyObject
    SerializationContext
 
+.. _api.feather:
+
+Feather Format
+~~~~~~~~~~~~~~
+
+.. currentmodule:: pyarrow.feather
+
 .. _api.memory_pool:
 
+.. autosummary::
+   :toctree: generated/
+
+   read_feather
+   write_feather
+
 Memory Pools
 ------------
 
+.. currentmodule:: pyarrow
+
 .. autosummary::
    :toctree: generated/
 
diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index dca776b792059..17fe84e03633d 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -19,8 +19,11 @@
 
 .. _ipc:
 
-IPC: Fast Streaming and Serialization
-=====================================
+Streaming, Serialization, and IPC
+=================================
+
+Writing and Reading Streams
+---------------------------
 
 Arrow defines two types of binary formats for serializing record batches:
 
@@ -35,8 +38,8 @@ Arrow defines two types of binary formats for serializing record batches:
 To follow this section, make sure to first read the section on :ref:`Memory and
 IO <io>`.
 
-Writing and Reading Streams
----------------------------
+Using streams
+~~~~~~~~~~~~~
 
 First, let's create a small record batch:
 
@@ -102,7 +105,7 @@ An important point is that if the input source supports zero-copy reads
 batches are also zero-copy and do not allocate any new memory on read.
 
 Writing and Reading Random Access Files
----------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The :class:`~pyarrow.RecordBatchFileWriter` has the same API as
 :class:`~pyarrow.RecordBatchStreamWriter`:
@@ -138,7 +141,7 @@ batches in the file, and can read any at random:
    b.equals(batch)
 
 Reading from Stream and File Format for pandas
-----------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The stream and file reader classes have a special ``read_pandas`` method to
 simplify reading multiple record batches and converting them to a single
@@ -147,4 +150,157 @@ DataFrame output:
 .. ipython:: python
 
    df = pa.open_file(buf).read_pandas()
-   df
+   df[:5]
+
+Arbitrary Object Serialization
+------------------------------
+
+In ``pyarrow`` we are able to serialize and deserialize many kinds of Python
+objects. While not a complete replacement for the ``pickle`` module, these
+functions can be significantly faster, particular when dealing with collections
+of NumPy arrays.
+
+As an example, consider a dictionary containing NumPy arrays:
+
+.. ipython:: python
+
+   import numpy as np
+
+   data = {
+       i: np.random.randn(500, 500)
+       for i in range(100)
+   }
+
+We use the ``pyarrow.serialize`` function to convert this data to a byte
+buffer:
+
+.. ipython:: python
+
+   buf = pa.serialize(data).to_buffer()
+   type(buf)
+   buf.size
+
+``pyarrow.serialize`` creates an intermediate object which can be converted to
+a buffer (the ``to_buffer`` method) or written directly to an output stream.
+
+``pyarrow.deserialize`` converts a buffer-like object back to the original
+Python object:
+
+.. ipython:: python
+
+   restored_data = pa.deserialize(buf)
+   restored_data[0]
+
+When dealing with NumPy arrays, ``pyarrow.deserialize`` can be significantly
+faster than ``pickle`` because the resulting arrays are zero-copy references
+into the input buffer. The larger the arrays, the larger the performance
+savings.
+
+Consider this example, we have for ``pyarrow.deserialize``
+
+.. ipython:: python
+
+   %timeit restored_data = pa.deserialize(buf)
+
+And for pickle:
+
+.. ipython:: python
+
+   import pickle
+   pickled = pickle.dumps(data)
+   %timeit unpickled_data = pickle.loads(pickled)
+
+We aspire to make these functions a high-speed alternative to pickle for
+transient serialization in Python big data applications.
+
+Serializing Custom Data Types
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If an unrecognized data type is encountered when serializing an object,
+``pyarrow`` will fall back on using ``pickle`` for converting that type to a
+byte string. There may be a more efficient way, though.
+
+Consider a class with two members, one of which is a NumPy array:
+
+.. code-block:: python
+
+   class MyData:
+       def __init__(self, name, data):
+           self.name = name
+           self.data = data
+
+We write functions to convert this to and from a dictionary with simpler types:
+
+.. code-block:: python
+
+   def _serialize_MyData(val):
+       return {'name': val.name, 'data': val.data}
+
+   def _deserialize_MyData(data):
+       return MyData(data['name'], data['data']
+
+then, we must register these functions in a ``SerializationContext`` so that
+``MyData`` can be recognized:
+
+.. code-block:: python
+
+   context = pa.SerializationContext()
+   context.register_type(MyData, 'MyData',
+                         custom_serializer=_serialize_MyData,
+                         custom_deserializer=_deserialize_MyData)
+
+Lastly, we use this context as an additioanl argument to ``pyarrow.serialize``:
+
+.. code-block:: python
+
+   buf = pa.serialize(val, context=context).to_buffer()
+   restored_val = pa.deserialize(buf, context=context)
+
+Feather Format
+--------------
+
+Feather is a lightweight file-format for data frames that uses the Arrow memory
+layout for data representation on disk. It was created early in the Arrow
+project as a proof of concept for fast, language-agnostic data frame storage
+for Python (pandas) and R.
+
+Compared with Arrow streams and files, Feather has some limitations:
+
+* Only non-nested data types and categorical (dictionary-encoded) types are
+  supported
+* Supports only a single batch of rows, where general Arrow streams support an
+  arbitrary number
+* Supports limited scalar value types, adequate only for representing typical
+  data found in R and pandas
+
+We would like to continue to innovate in the Feather format, but we must wait
+for an R implementation for Arrow to mature.
+
+The ``pyarrow.feather`` module contains the read and write functions for the
+format. The input and output are ``pandas.DataFrame`` objects:
+
+.. code-block:: python
+
+   import pyarrow.feather as feather
+
+   feather.write_feather(df, '/path/to/file')
+   read_df = feather.read_feather('/path/to/file')
+
+``read_feather`` supports multithreaded reads, and may yield faster performance
+on some files:
+
+.. code-block:: python
+
+   read_df = feather.read_feather('/path/to/file', nthreads=4)
+
+These functions can read and write with file-like objects. For example:
+
+.. code-block:: python
+
+   with open('/path/to/file', 'wb') as f:
+       feather.write_feather(df, f)
+
+   with open('/path/to/file', 'rb') as f:
+       read_df = feather.read_feather(f)
+
+A file input to ``read_feather`` must support seeking.
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 9f40ab419ad5a..aba76a008a072 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -104,9 +104,15 @@ def write(self, df):
 
 
 def write_feather(df, dest):
-    '''
+    """
     Write a pandas.DataFrame to Feather format
-    '''
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+    dest : string
+        Local file path
+    """
     writer = FeatherWriter(dest)
     try:
         writer.write(df)

From 94f6247c4ef49b9b4fe1cccdc172810c55c34a10 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 11 Sep 2017 12:46:37 +0200
Subject: [PATCH 1041/1644] ARROW-1508: C++: Add support for
 FixedSizeBinaryType in DictionaryBuilder

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1073 from xhochy/ARROW-1508 and squashes the following commits:

cefee2c [Uwe L. Korn] ninja format
c471bfa [Uwe L. Korn] Review comments
4bf354e [Uwe L. Korn] Use non-deprecated interfaces
2538926 [Uwe L. Korn] ARROW-1508: C++: Add support for FixedSizeBinaryType in DictionaryBuilder
---
 cpp/src/arrow/array-test.cc | 84 +++++++++++++++++++++++++++++++++++++
 cpp/src/arrow/array.h       |  1 +
 cpp/src/arrow/builder.cc    | 71 ++++++++++++++++++++++++++++++-
 cpp/src/arrow/builder.h     | 11 +++++
 4 files changed, 165 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index c92c23dd21dbd..a9596c5b878d6 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1764,6 +1764,90 @@ TEST(TestStringDictionaryBuilder, DoubleTableSize) {
   ASSERT_TRUE(expected.Equals(result));
 }
 
+TEST(TestFixedSizeBinaryDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  std::vector<uint8_t> test{12, 12, 11, 12};
+  std::vector<uint8_t> test2{12, 12, 11, 11};
+  ASSERT_OK(builder.Append(test.data()));
+  ASSERT_OK(builder.Append(test2.data()));
+  ASSERT_OK(builder.Append(test.data()));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4));
+  ASSERT_OK(fsb_builder.Append(test.data()));
+  ASSERT_OK(fsb_builder.Append(test2.data()));
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), fsb_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4));
+  Int16Builder int_builder;
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    std::vector<uint8_t> value{12, 12, static_cast<uint8_t>(i / 128),
+                               static_cast<uint8_t>(i % 128)};
+    ASSERT_OK(builder.Append(value.data()));
+    ASSERT_OK(fsb_builder.Append(value.data()));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  std::vector<uint8_t> known_value{12, 12, 0, 1};
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append(known_value.data()));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+  auto dtype = std::make_shared<DictionaryType>(int16(), fsb_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  // Build an array with different byte width
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(5));
+  std::vector<uint8_t> value{100, 1, 1, 1, 1};
+  ASSERT_OK(fsb_builder.Append(value.data()));
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+
+  ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array));
+}
+
 // ----------------------------------------------------------------------
 // List tests
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index bfeedd2b65cd9..994270db32387 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -512,6 +512,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
                        int64_t null_count = 0, int64_t offset = 0);
 
   const uint8_t* GetValue(int64_t i) const;
+  const uint8_t* Value(int64_t i) const { return GetValue(i); }
 
   int32_t byte_width() const { return byte_width_; }
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 796624169735c..5945677e72bfe 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -26,6 +26,7 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
+#include "arrow/compare.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
@@ -814,6 +815,20 @@ Status BooleanBuilder::Append(const std::vector<bool>& values) {
 template <typename T>
 DictionaryBuilder<T>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
                                         MemoryPool* pool)
+    : ArrayBuilder(type, pool),
+      hash_table_(new PoolBuffer(pool)),
+      hash_slots_(nullptr),
+      dict_builder_(type, pool),
+      values_builder_(pool),
+      byte_width_(-1) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
+
+template <>
+DictionaryBuilder<FixedSizeBinaryType>::DictionaryBuilder(
+    const std::shared_ptr<DataType>& type, MemoryPool* pool)
     : ArrayBuilder(type, pool),
       hash_table_(new PoolBuffer(pool)),
       hash_slots_(nullptr),
@@ -822,6 +837,7 @@ DictionaryBuilder<T>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
   if (!::arrow::CpuInfo::initialized()) {
     ::arrow::CpuInfo::Init();
   }
+  byte_width_ = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
 }
 
 #ifndef ARROW_NO_DEPRECATED_API
@@ -918,6 +934,24 @@ Status DictionaryBuilder<T>::AppendArray(const Array& array) {
   return Status::OK();
 }
 
+template <>
+Status DictionaryBuilder<FixedSizeBinaryType>::AppendArray(const Array& array) {
+  if (!type_->Equals(*array.type())) {
+    return Status::Invalid("Cannot append FixedSizeBinary array with non-matching type");
+  }
+
+  const FixedSizeBinaryArray& numeric_array =
+      static_cast<const FixedSizeBinaryArray&>(array);
+  for (int64_t i = 0; i < array.length(); i++) {
+    if (array.IsNull(i)) {
+      RETURN_NOT_OK(AppendNull());
+    } else {
+      RETURN_NOT_OK(Append(numeric_array.Value(i)));
+    }
+  }
+  return Status::OK();
+}
+
 template <typename T>
 Status DictionaryBuilder<T>::AppendNull() {
   return values_builder_.AppendNull();
@@ -975,17 +1009,35 @@ typename DictionaryBuilder<T>::Scalar DictionaryBuilder<T>::GetDictionaryValue(
   return data[index];
 }
 
+template <>
+const uint8_t* DictionaryBuilder<FixedSizeBinaryType>::GetDictionaryValue(int64_t index) {
+  return dict_builder_.GetValue(index);
+}
+
 template <typename T>
 int DictionaryBuilder<T>::HashValue(const Scalar& value) {
   return HashUtil::Hash(&value, sizeof(Scalar), 0);
 }
 
+template <>
+int DictionaryBuilder<FixedSizeBinaryType>::HashValue(const Scalar& value) {
+  return HashUtil::Hash(value, byte_width_, 0);
+}
+
 template <typename T>
 bool DictionaryBuilder<T>::SlotDifferent(hash_slot_t index, const Scalar& value) {
   const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
   return other != value;
 }
 
+template <>
+bool DictionaryBuilder<FixedSizeBinaryType>::SlotDifferent(hash_slot_t index,
+                                                           const Scalar& value) {
+  int32_t width = static_cast<const FixedSizeBinaryType&>(*type_).byte_width();
+  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
+  return memcmp(other, value, width) != 0;
+}
+
 template <typename T>
 Status DictionaryBuilder<T>::AppendDictionary(const Scalar& value) {
   return dict_builder_.Append(value);
@@ -1052,6 +1104,7 @@ template class DictionaryBuilder<Time64Type>;
 template class DictionaryBuilder<TimestampType>;
 template class DictionaryBuilder<FloatType>;
 template class DictionaryBuilder<DoubleType>;
+template class DictionaryBuilder<FixedSizeBinaryType>;
 template class DictionaryBuilder<BinaryType>;
 template class DictionaryBuilder<StringType>;
 
@@ -1315,6 +1368,11 @@ Status FixedSizeBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
   return Status::OK();
 }
 
+const uint8_t* FixedSizeBinaryBuilder::GetValue(int64_t i) const {
+  const uint8_t* data_ptr = byte_builder_.data();
+  return data_ptr + i * byte_width_;
+}
+
 // ----------------------------------------------------------------------
 // Struct
 
@@ -1438,6 +1496,7 @@ Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>&
     DICTIONARY_BUILDER_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
     DICTIONARY_BUILDER_CASE(STRING, StringDictionaryBuilder);
     DICTIONARY_BUILDER_CASE(BINARY, BinaryDictionaryBuilder);
+    DICTIONARY_BUILDER_CASE(FIXED_SIZE_BINARY, DictionaryBuilder<FixedSizeBinaryType>);
     default:
       return Status::NotImplemented(type->ToString());
   }
@@ -1472,8 +1531,12 @@ Status EncodeArrayToDictionary(const Array& input, MemoryPool* pool,
     DICTIONARY_ARRAY_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
     DICTIONARY_ARRAY_CASE(STRING, StringDictionaryBuilder);
     DICTIONARY_ARRAY_CASE(BINARY, BinaryDictionaryBuilder);
+    DICTIONARY_ARRAY_CASE(FIXED_SIZE_BINARY, DictionaryBuilder<FixedSizeBinaryType>);
     default:
-      return Status::NotImplemented(type->ToString());
+      std::stringstream ss;
+      ss << "Cannot encode array of type " << type->ToString();
+      ss << " to dictionary";
+      return Status::NotImplemented(ss.str());
   }
 }
 #define DICTIONARY_COLUMN_CASE(ENUM, BuilderType)                             \
@@ -1516,8 +1579,12 @@ Status EncodeColumnToDictionary(const Column& input, MemoryPool* pool,
     DICTIONARY_COLUMN_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
     DICTIONARY_COLUMN_CASE(STRING, StringDictionaryBuilder);
     DICTIONARY_COLUMN_CASE(BINARY, BinaryDictionaryBuilder);
+    DICTIONARY_COLUMN_CASE(FIXED_SIZE_BINARY, DictionaryBuilder<FixedSizeBinaryType>);
     default:
-      return Status::NotImplemented(type->ToString());
+      std::stringstream ss;
+      ss << "Cannot encode column of type " << type->ToString();
+      ss << " to dictionary";
+      return Status::NotImplemented(ss.str());
   }
 }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 3e8289f049650..bf7b317f68c60 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -756,6 +756,11 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
   /// \return size of values buffer so far
   int64_t value_data_length() const { return byte_builder_.length(); }
 
+  /// Temporary access to a value.
+  ///
+  /// This pointer becomes invalid on the next modifying operation.
+  const uint8_t* GetValue(int64_t i) const;
+
  protected:
   int32_t byte_width_;
   BufferBuilder byte_builder_;
@@ -866,6 +871,11 @@ struct DictionaryScalar<StringType> {
   using type = WrappedBinary;
 };
 
+template <>
+struct DictionaryScalar<FixedSizeBinaryType> {
+  using type = uint8_t const*;
+};
+
 }  // namespace internal
 
 /// \brief Array builder for created encoded DictionaryArray from dense array
@@ -921,6 +931,7 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
 
   typename TypeTraits<T>::BuilderType dict_builder_;
   AdaptiveIntBuilder values_builder_;
+  int32_t byte_width_;
 };
 
 class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {

From 7e95764263c34a3bfabf0ca9909f425c16042141 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 11 Sep 2017 13:11:37 -0400
Subject: [PATCH 1042/1644] ARROW-1513: C++: Add cast from Dictionary to plain
 arrays

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1086 from xhochy/ARROW-1513 and squashes the following commits:

769b205e [Wes McKinney] Fix compiler error / warning. Change type checks to DCHECK
05c539f2 [Uwe L. Korn] ARROW-1513: C++: Add cast from Dictionary to plain arrays

Change-Id: I80d444e7a0661d58909801004ac9818dece04a15
---
 cpp/src/arrow/compute/cast.cc         | 344 +++++++++++++++++++++++---
 cpp/src/arrow/compute/compute-test.cc |  26 +-
 cpp/src/arrow/table-test.cc           |  66 ++---
 cpp/src/arrow/test-common.h           |  66 ++++-
 cpp/src/arrow/type.h                  |   5 +
 cpp/src/arrow/util/bit-util.h         |   6 +-
 6 files changed, 429 insertions(+), 84 deletions(-)

diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index 3885fdfb57963..c651244b3ce52 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -28,6 +28,8 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/compare.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -37,6 +39,32 @@
 #include "arrow/compute/context.h"
 #include "arrow/compute/kernel.h"
 
+#ifdef ARROW_EXTRA_ERROR_CONTEXT
+
+#define FUNC_RETURN_NOT_OK(s)                                                       \
+  do {                                                                              \
+    Status _s = (s);                                                                \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) {                                            \
+      std::stringstream ss;                                                         \
+      ss << __FILE__ << ":" << __LINE__ << " code: " << #s << "\n" << _s.message(); \
+      ctx->SetStatus(Status(_s.code(), ss.str()));                                  \
+      return;                                                                       \
+    }                                                                               \
+  } while (0)
+
+#else
+
+#define FUNC_RETURN_NOT_OK(s)            \
+  do {                                   \
+    Status _s = (s);                     \
+    if (ARROW_PREDICT_FALSE(!_s.ok())) { \
+      ctx->SetStatus(_s);                \
+      return;                            \
+    }                                    \
+  } while (0)
+
+#endif  // ARROW_EXTRA_ERROR_CONTEXT
+
 namespace arrow {
 namespace compute {
 
@@ -225,6 +253,212 @@ struct CastFunctor<O, I,
   }
 };
 
+// ----------------------------------------------------------------------
+// Dictionary to other things
+
+template <typename IndexType>
+void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices,
+                                     const FixedSizeBinaryArray& dictionary,
+                                     ArrayData* output) {
+  using index_c_type = typename IndexType::c_type;
+  const uint8_t* valid_bits = indices.null_bitmap_data();
+  INIT_BITSET(valid_bits, indices.offset());
+
+  const index_c_type* in =
+      reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
+      indices.offset();
+  uint8_t* out = output->buffers[1]->mutable_data();
+  int32_t byte_width =
+      static_cast<const FixedSizeBinaryType&>(*output->type).byte_width();
+  for (int64_t i = 0; i < indices.length(); ++i) {
+    if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+      const uint8_t* value = dictionary.Value(in[i]);
+      memcpy(out + i * byte_width, value, byte_width);
+    }
+    READ_NEXT_BITSET(valid_bits);
+  }
+}
+
+template <typename T>
+struct CastFunctor<
+    T, DictionaryType,
+    typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value>::type> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    const DictionaryArray& dict_array = static_cast<const DictionaryArray&>(input);
+    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type());
+    const DataType& values_type = *type.dictionary()->type();
+    const FixedSizeBinaryArray& dictionary =
+        static_cast<const FixedSizeBinaryArray&>(*type.dictionary());
+
+    // Check if values and output type match
+    DCHECK(values_type.Equals(*output->type))
+      << "Dictionary type: " << values_type
+      << " target type: " << (*output->type);
+
+    const Array& indices = *dict_array.indices();
+    switch (indices.type()->id()) {
+      case Type::INT8:
+        UnpackFixedSizeBinaryDictionary<Int8Type>(ctx, indices, dictionary, output);
+        break;
+      case Type::INT16:
+        UnpackFixedSizeBinaryDictionary<Int16Type>(ctx, indices, dictionary, output);
+        break;
+      case Type::INT32:
+        UnpackFixedSizeBinaryDictionary<Int32Type>(ctx, indices, dictionary, output);
+        break;
+      case Type::INT64:
+        UnpackFixedSizeBinaryDictionary<Int64Type>(ctx, indices, dictionary, output);
+        break;
+      default:
+        std::stringstream ss;
+        ss << "Invalid index type: " << indices.type()->ToString();
+        ctx->SetStatus(Status::Invalid(ss.str()));
+        return;
+    }
+  }
+};
+
+template <typename IndexType>
+Status UnpackBinaryDictionary(FunctionContext* ctx, const Array& indices,
+                              const BinaryArray& dictionary, ArrayData* output) {
+  using index_c_type = typename IndexType::c_type;
+  std::unique_ptr<ArrayBuilder> builder;
+  RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), output->type, &builder));
+  BinaryBuilder* binary_builder = static_cast<BinaryBuilder*>(builder.get());
+
+  const uint8_t* valid_bits = indices.null_bitmap_data();
+  INIT_BITSET(valid_bits, indices.offset());
+
+  const index_c_type* in =
+      reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
+      indices.offset();
+  for (int64_t i = 0; i < indices.length(); ++i) {
+    if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+      int32_t length;
+      const uint8_t* value = dictionary.GetValue(in[i], &length);
+      RETURN_NOT_OK(binary_builder->Append(value, length));
+    } else {
+      RETURN_NOT_OK(binary_builder->AppendNull());
+    }
+    READ_NEXT_BITSET(valid_bits);
+  }
+
+  std::shared_ptr<Array> plain_array;
+  RETURN_NOT_OK(binary_builder->Finish(&plain_array));
+  // Copy all buffer except the valid bitmap
+  for (size_t i = 1; i < plain_array->data()->buffers.size(); i++) {
+    output->buffers.push_back(plain_array->data()->buffers[i]);
+  }
+
+  return Status::OK();
+}
+
+template <typename T>
+struct CastFunctor<T, DictionaryType,
+                   typename std::enable_if<std::is_base_of<BinaryType, T>::value>::type> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    const DictionaryArray& dict_array = static_cast<const DictionaryArray&>(input);
+    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type());
+    const DataType& values_type = *type.dictionary()->type();
+    const BinaryArray& dictionary = static_cast<const BinaryArray&>(*type.dictionary());
+
+    // Check if values and output type match
+    DCHECK(values_type.Equals(*output->type))
+      << "Dictionary type: " << values_type
+      << " target type: " << (*output->type);
+
+    const Array& indices = *dict_array.indices();
+    switch (indices.type()->id()) {
+      case Type::INT8:
+        FUNC_RETURN_NOT_OK(
+            (UnpackBinaryDictionary<Int8Type>(ctx, indices, dictionary, output)));
+        break;
+      case Type::INT16:
+        FUNC_RETURN_NOT_OK(
+            (UnpackBinaryDictionary<Int16Type>(ctx, indices, dictionary, output)));
+        break;
+      case Type::INT32:
+        FUNC_RETURN_NOT_OK(
+            (UnpackBinaryDictionary<Int32Type>(ctx, indices, dictionary, output)));
+        break;
+      case Type::INT64:
+        FUNC_RETURN_NOT_OK(
+            (UnpackBinaryDictionary<Int64Type>(ctx, indices, dictionary, output)));
+        break;
+      default:
+        std::stringstream ss;
+        ss << "Invalid index type: " << indices.type()->ToString();
+        ctx->SetStatus(Status::Invalid(ss.str()));
+        return;
+    }
+  }
+};
+
+template <typename IndexType, typename c_type>
+void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
+                               c_type* out) {
+  using index_c_type = typename IndexType::c_type;
+
+  const uint8_t* valid_bits = indices.null_bitmap_data();
+  INIT_BITSET(valid_bits, indices.offset());
+
+  const index_c_type* in =
+      reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
+      indices.offset();
+  for (int64_t i = 0; i < indices.length(); ++i) {
+    if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+      out[i] = dictionary[in[i]];
+    }
+    READ_NEXT_BITSET(valid_bits);
+  }
+}
+
+// Cast from dictionary to plain representation
+template <typename T>
+struct CastFunctor<T, DictionaryType,
+                   typename std::enable_if<IsNumeric<T>::value>::type> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    using c_type = typename T::c_type;
+
+    const DictionaryArray& dict_array = static_cast<const DictionaryArray&>(input);
+    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type());
+    const DataType& values_type = *type.dictionary()->type();
+
+    // Check if values and output type match
+    DCHECK(values_type.Equals(*output->type))
+      << "Dictionary type: " << values_type
+      << " target type: " << (*output->type);
+
+    auto dictionary =
+        reinterpret_cast<const c_type*>(type.dictionary()->data()->buffers[1]->data()) +
+        type.dictionary()->offset();
+    auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
+    const Array& indices = *dict_array.indices();
+    switch (indices.type()->id()) {
+      case Type::INT8:
+        UnpackPrimitiveDictionary<Int8Type, c_type>(indices, dictionary, out);
+        break;
+      case Type::INT16:
+        UnpackPrimitiveDictionary<Int16Type, c_type>(indices, dictionary, out);
+        break;
+      case Type::INT32:
+        UnpackPrimitiveDictionary<Int32Type, c_type>(indices, dictionary, out);
+        break;
+      case Type::INT64:
+        UnpackPrimitiveDictionary<Int64Type, c_type>(indices, dictionary, out);
+        break;
+      default:
+        std::stringstream ss;
+        ss << "Invalid index type: " << indices.type()->ToString();
+        ctx->SetStatus(Status::Invalid(ss.str()));
+        return;
+    }
+  }
+};
+
 // ----------------------------------------------------------------------
 
 typedef std::function<void(FunctionContext*, const CastOptions& options, const Array&,
@@ -232,13 +466,7 @@ typedef std::function<void(FunctionContext*, const CastOptions& options, const A
     CastFunction;
 
 static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input,
-                                        ArrayData* out) {
-  if (!is_primitive(out->type->id())) {
-    return Status::NotImplemented(out->type->ToString());
-  }
-
-  const auto& fw_type = static_cast<const FixedWidthType&>(*out->type);
-
+                                        bool can_pre_allocate_values, ArrayData* out) {
   const int64_t length = input.length();
 
   out->null_count = input.null_count();
@@ -261,35 +489,49 @@ static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input
 
   out->buffers.push_back(validity_bitmap);
 
-  std::shared_ptr<Buffer> out_data;
+  if (can_pre_allocate_values) {
+    std::shared_ptr<Buffer> out_data;
 
-  int bit_width = fw_type.bit_width();
-  int64_t buffer_size = 0;
+    if (!(is_primitive(out->type->id()) || out->type->id() == Type::FIXED_SIZE_BINARY)) {
+      std::stringstream ss;
+      ss << "Cannot pre-allocate memory for type: " << out->type->ToString();
+      return Status::NotImplemented(ss.str());
+    }
 
-  if (bit_width == 1) {
-    buffer_size = BitUtil::BytesForBits(length);
-  } else if (bit_width % 8 == 0) {
-    buffer_size = length * fw_type.bit_width() / 8;
-  } else {
-    DCHECK(false);
-  }
+    const auto& fw_type = static_cast<const FixedWidthType&>(*out->type);
+
+    int bit_width = fw_type.bit_width();
+    int64_t buffer_size = 0;
 
-  RETURN_NOT_OK(ctx->Allocate(buffer_size, &out_data));
-  memset(out_data->mutable_data(), 0, buffer_size);
+    if (bit_width == 1) {
+      buffer_size = BitUtil::BytesForBits(length);
+    } else if (bit_width % 8 == 0) {
+      buffer_size = length * fw_type.bit_width() / 8;
+    } else {
+      DCHECK(false);
+    }
 
-  out->buffers.push_back(out_data);
+    RETURN_NOT_OK(ctx->Allocate(buffer_size, &out_data));
+    memset(out_data->mutable_data(), 0, buffer_size);
+
+    out->buffers.push_back(out_data);
+  }
 
   return Status::OK();
 }
 
 class CastKernel : public UnaryKernel {
  public:
-  CastKernel(const CastOptions& options, const CastFunction& func, bool is_zero_copy)
-      : options_(options), func_(func), is_zero_copy_(is_zero_copy) {}
+  CastKernel(const CastOptions& options, const CastFunction& func, bool is_zero_copy,
+             bool can_pre_allocate_values)
+      : options_(options),
+        func_(func),
+        is_zero_copy_(is_zero_copy),
+        can_pre_allocate_values_(can_pre_allocate_values) {}
 
   Status Call(FunctionContext* ctx, const Array& input, ArrayData* out) override {
     if (!is_zero_copy_) {
-      RETURN_NOT_OK(AllocateIfNotPreallocated(ctx, input, out));
+      RETURN_NOT_OK(AllocateIfNotPreallocated(ctx, input, can_pre_allocate_values_, out));
     }
     func_(ctx, options_, input, out);
     RETURN_IF_ERROR(ctx);
@@ -300,11 +542,14 @@ class CastKernel : public UnaryKernel {
   CastOptions options_;
   CastFunction func_;
   bool is_zero_copy_;
+  bool can_pre_allocate_values_;
 };
 
 #define CAST_CASE(InType, OutType)                                                  \
   case OutType::type_id:                                                            \
     is_zero_copy = is_zero_copy_cast<OutType, InType>::value;                       \
+    can_pre_allocate_values =                                                       \
+        !(!is_binary_like(InType::type_id) && is_binary_like(OutType::type_id));    \
     func = [](FunctionContext* ctx, const CastOptions& options, const Array& input, \
               ArrayData* out) {                                                     \
       CastFunctor<OutType, InType> func;                                            \
@@ -354,20 +599,42 @@ class CastKernel : public UnaryKernel {
 
 #define TIMESTAMP_CASES(FN, IN_TYPE) FN(TimestampType, TimestampType);
 
-#define GET_CAST_FUNCTION(CASE_GENERATOR, InType)                                       \
-  static std::unique_ptr<UnaryKernel> Get##InType##CastFunc(                            \
-      const std::shared_ptr<DataType>& out_type, const CastOptions& options) {          \
-    CastFunction func;                                                                  \
-    bool is_zero_copy = false;                                                          \
-    switch (out_type->id()) {                                                           \
-      CASE_GENERATOR(CAST_CASE, InType);                                                \
-      default:                                                                          \
-        break;                                                                          \
-    }                                                                                   \
-    if (func != nullptr) {                                                              \
-      return std::unique_ptr<UnaryKernel>(new CastKernel(options, func, is_zero_copy)); \
-    }                                                                                   \
-    return nullptr;                                                                     \
+#define DICTIONARY_CASES(FN, IN_TYPE) \
+  FN(IN_TYPE, Time32Type);            \
+  FN(IN_TYPE, Date32Type);            \
+  FN(IN_TYPE, TimestampType);         \
+  FN(IN_TYPE, Time64Type);            \
+  FN(IN_TYPE, Date64Type);            \
+  FN(IN_TYPE, UInt8Type);             \
+  FN(IN_TYPE, Int8Type);              \
+  FN(IN_TYPE, UInt16Type);            \
+  FN(IN_TYPE, Int16Type);             \
+  FN(IN_TYPE, UInt32Type);            \
+  FN(IN_TYPE, Int32Type);             \
+  FN(IN_TYPE, UInt64Type);            \
+  FN(IN_TYPE, Int64Type);             \
+  FN(IN_TYPE, FloatType);             \
+  FN(IN_TYPE, DoubleType);            \
+  FN(IN_TYPE, FixedSizeBinaryType);   \
+  FN(IN_TYPE, BinaryType);            \
+  FN(IN_TYPE, StringType);
+
+#define GET_CAST_FUNCTION(CASE_GENERATOR, InType)                                \
+  static std::unique_ptr<UnaryKernel> Get##InType##CastFunc(                     \
+      const std::shared_ptr<DataType>& out_type, const CastOptions& options) {   \
+    CastFunction func;                                                           \
+    bool is_zero_copy = false;                                                   \
+    bool can_pre_allocate_values = true;                                         \
+    switch (out_type->id()) {                                                    \
+      CASE_GENERATOR(CAST_CASE, InType);                                         \
+      default:                                                                   \
+        break;                                                                   \
+    }                                                                            \
+    if (func != nullptr) {                                                       \
+      return std::unique_ptr<UnaryKernel>(                                       \
+          new CastKernel(options, func, is_zero_copy, can_pre_allocate_values)); \
+    }                                                                            \
+    return nullptr;                                                              \
   }
 
 GET_CAST_FUNCTION(NULL_CASES, NullType);
@@ -388,6 +655,8 @@ GET_CAST_FUNCTION(TIME32_CASES, Time32Type);
 GET_CAST_FUNCTION(TIME64_CASES, Time64Type);
 GET_CAST_FUNCTION(TIMESTAMP_CASES, TimestampType);
 
+GET_CAST_FUNCTION(DICTIONARY_CASES, DictionaryType);
+
 #define CAST_FUNCTION_CASE(InType)                      \
   case InType::type_id:                                 \
     *kernel = Get##InType##CastFunc(out_type, options); \
@@ -413,6 +682,7 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>&
     CAST_FUNCTION_CASE(Time32Type);
     CAST_FUNCTION_CASE(Time64Type);
     CAST_FUNCTION_CASE(TimestampType);
+    CAST_FUNCTION_CASE(DictionaryType);
     default:
       break;
   }
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index ba645c28bd726..5898aeebde414 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -66,10 +66,9 @@ void AssertArraysEqual(const Array& left, const Array& right) {
 
 class ComputeFixture {
  public:
-  ComputeFixture() : pool_(default_memory_pool()), ctx_(pool_) {}
+  ComputeFixture() : ctx_(default_memory_pool()) {}
 
  protected:
-  MemoryPool* pool_;
   FunctionContext ctx_;
 };
 
@@ -81,7 +80,7 @@ static void AssertBufferSame(const Array& left, const Array& right, int buffer_i
             right.data()->buffers[buffer_index].get());
 }
 
-class TestCast : public ComputeFixture, public ::testing::Test {
+class TestCast : public ComputeFixture, public TestBase {
  public:
   void CheckPass(const Array& input, const Array& expected,
                  const std::shared_ptr<DataType>& out_type, const CastOptions& options) {
@@ -395,5 +394,26 @@ TEST_F(TestCast, PreallocatedMemory) {
   AssertArraysEqual(*expected, *result);
 }
 
+template <typename TestType>
+class TestDictionaryCast : public TestCast {};
+
+typedef ::testing::Types<UInt8Type, Int8Type, UInt16Type, Int16Type, Int32Type,
+                         UInt32Type, UInt64Type, Int64Type, FloatType, DoubleType,
+                         Date32Type, Date64Type, FixedSizeBinaryType, BinaryType>
+    TestTypes;
+
+TYPED_TEST_CASE(TestDictionaryCast, TestTypes);
+
+TYPED_TEST(TestDictionaryCast, Basic) {
+  CastOptions options;
+  std::shared_ptr<Array> plain_array =
+      TestBase::MakeRandomArray<typename TypeTraits<TypeParam>::ArrayType>(10, 2);
+
+  std::shared_ptr<Array> dict_array;
+  ASSERT_OK(EncodeArrayToDictionary(*plain_array, this->pool_, &dict_array));
+
+  this->CheckPass(*dict_array, *plain_array, plain_array->type(), options);
+}
+
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index a9c7e6da0ae73..b0aeed1925ece 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -128,9 +128,9 @@ class TestColumn : public TestChunkedArray {
 
 TEST_F(TestColumn, BasicAPI) {
   ArrayVector arrays;
-  arrays.push_back(MakePrimitive<Int32Array>(100));
-  arrays.push_back(MakePrimitive<Int32Array>(100, 10));
-  arrays.push_back(MakePrimitive<Int32Array>(100, 20));
+  arrays.push_back(MakeRandomArray<Int32Array>(100));
+  arrays.push_back(MakeRandomArray<Int32Array>(100, 10));
+  arrays.push_back(MakeRandomArray<Int32Array>(100, 20));
 
   auto f0 = field("c0", int32());
   column_.reset(new Column(f0, arrays));
@@ -148,15 +148,15 @@ TEST_F(TestColumn, BasicAPI) {
 
 TEST_F(TestColumn, ChunksInhomogeneous) {
   ArrayVector arrays;
-  arrays.push_back(MakePrimitive<Int32Array>(100));
-  arrays.push_back(MakePrimitive<Int32Array>(100, 10));
+  arrays.push_back(MakeRandomArray<Int32Array>(100));
+  arrays.push_back(MakeRandomArray<Int32Array>(100, 10));
 
   auto f0 = field("c0", int32());
   column_.reset(new Column(f0, arrays));
 
   ASSERT_OK(column_->ValidateData());
 
-  arrays.push_back(MakePrimitive<Int16Array>(100, 10));
+  arrays.push_back(MakeRandomArray<Int16Array>(100, 10));
   column_.reset(new Column(f0, arrays));
   ASSERT_RAISES(Invalid, column_->ValidateData());
 }
@@ -202,8 +202,8 @@ class TestTable : public TestBase {
     vector<shared_ptr<Field>> fields = {f0, f1, f2};
     schema_ = std::make_shared<Schema>(fields);
 
-    arrays_ = {MakePrimitive<Int32Array>(length), MakePrimitive<UInt8Array>(length),
-               MakePrimitive<Int16Array>(length)};
+    arrays_ = {MakeRandomArray<Int32Array>(length), MakeRandomArray<UInt8Array>(length),
+               MakeRandomArray<Int16Array>(length)};
 
     columns_ = {std::make_shared<Column>(schema_->field(0), arrays_[0]),
                 std::make_shared<Column>(schema_->field(1), arrays_[1]),
@@ -276,9 +276,10 @@ TEST_F(TestTable, InvalidColumns) {
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
 
   columns_ = {
-      std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length)),
-      std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length)),
-      std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length - 1))};
+      std::make_shared<Column>(schema_->field(0), MakeRandomArray<Int32Array>(length)),
+      std::make_shared<Column>(schema_->field(1), MakeRandomArray<UInt8Array>(length)),
+      std::make_shared<Column>(schema_->field(2),
+                               MakeRandomArray<Int16Array>(length - 1))};
 
   table_.reset(new Table(schema_, columns_, length));
   ASSERT_RAISES(Invalid, table_->ValidateColumns());
@@ -300,9 +301,12 @@ TEST_F(TestTable, Equals) {
   ASSERT_FALSE(table_->Equals(Table(other_schema, columns_)));
   // Differing columns
   std::vector<std::shared_ptr<Column>> other_columns = {
-      std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length, 10)),
-      std::make_shared<Column>(schema_->field(1), MakePrimitive<UInt8Array>(length, 10)),
-      std::make_shared<Column>(schema_->field(2), MakePrimitive<Int16Array>(length, 10))};
+      std::make_shared<Column>(schema_->field(0),
+                               MakeRandomArray<Int32Array>(length, 10)),
+      std::make_shared<Column>(schema_->field(1),
+                               MakeRandomArray<UInt8Array>(length, 10)),
+      std::make_shared<Column>(schema_->field(2),
+                               MakeRandomArray<Int16Array>(length, 10))};
   ASSERT_FALSE(table_->Equals(Table(schema_, other_columns)));
 }
 
@@ -410,8 +414,8 @@ TEST_F(TestTable, AddColumn) {
   ASSERT_TRUE(status.IsInvalid());
 
   // Add column with wrong length
-  auto longer_col =
-      std::make_shared<Column>(schema_->field(0), MakePrimitive<Int32Array>(length + 1));
+  auto longer_col = std::make_shared<Column>(schema_->field(0),
+                                             MakeRandomArray<Int32Array>(length + 1));
   status = table.AddColumn(0, longer_col, &result);
   ASSERT_TRUE(status.IsInvalid());
 
@@ -445,8 +449,8 @@ TEST_F(TestTable, AddColumn) {
 TEST_F(TestTable, IsChunked) {
   ArrayVector c1, c2;
 
-  auto a1 = MakePrimitive<Int32Array>(10);
-  auto a2 = MakePrimitive<Int32Array>(20);
+  auto a1 = MakeRandomArray<Int32Array>(10);
+  auto a2 = MakeRandomArray<Int32Array>(20);
 
   auto sch1 = arrow::schema({field("f1", int32()), field("f2", int32())});
 
@@ -477,9 +481,9 @@ TEST_F(TestRecordBatch, Equals) {
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto schema = std::make_shared<Schema>(fields);
 
-  auto a0 = MakePrimitive<Int32Array>(length);
-  auto a1 = MakePrimitive<UInt8Array>(length);
-  auto a2 = MakePrimitive<Int16Array>(length);
+  auto a0 = MakeRandomArray<Int32Array>(length);
+  auto a1 = MakeRandomArray<UInt8Array>(length);
+  auto a2 = MakeRandomArray<Int16Array>(length);
 
   RecordBatch b1(schema, length, {a0, a1, a2});
   RecordBatch b3(schema, length, {a0, a1});
@@ -502,10 +506,10 @@ TEST_F(TestRecordBatch, Validate) {
 
   auto schema = ::arrow::schema({f0, f1, f2});
 
-  auto a0 = MakePrimitive<Int32Array>(length);
-  auto a1 = MakePrimitive<UInt8Array>(length);
-  auto a2 = MakePrimitive<Int16Array>(length);
-  auto a3 = MakePrimitive<Int16Array>(5);
+  auto a0 = MakeRandomArray<Int32Array>(length);
+  auto a1 = MakeRandomArray<UInt8Array>(length);
+  auto a2 = MakeRandomArray<Int16Array>(length);
+  auto a3 = MakeRandomArray<Int16Array>(5);
 
   RecordBatch b1(schema, length, {a0, a1, a2});
 
@@ -531,8 +535,8 @@ TEST_F(TestRecordBatch, Slice) {
   vector<shared_ptr<Field>> fields = {f0, f1};
   auto schema = std::make_shared<Schema>(fields);
 
-  auto a0 = MakePrimitive<Int32Array>(length);
-  auto a1 = MakePrimitive<UInt8Array>(length);
+  auto a0 = MakeRandomArray<Int32Array>(length);
+  auto a1 = MakeRandomArray<UInt8Array>(length);
 
   RecordBatch batch(schema, length, {a0, a1});
 
@@ -555,10 +559,10 @@ class TestTableBatchReader : public TestBase {};
 TEST_F(TestTableBatchReader, ReadNext) {
   ArrayVector c1, c2;
 
-  auto a1 = MakePrimitive<Int32Array>(10);
-  auto a2 = MakePrimitive<Int32Array>(20);
-  auto a3 = MakePrimitive<Int32Array>(30);
-  auto a4 = MakePrimitive<Int32Array>(10);
+  auto a1 = MakeRandomArray<Int32Array>(10);
+  auto a2 = MakeRandomArray<Int32Array>(20);
+  auto a3 = MakeRandomArray<Int32Array>(30);
+  auto a4 = MakeRandomArray<Int32Array>(10);
 
   auto sch1 = arrow::schema({field("f1", int32()), field("f2", int32())});
 
diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h
index 4ce06408d17d0..3dc39fc3d07ca 100644
--- a/cpp/src/arrow/test-common.h
+++ b/cpp/src/arrow/test-common.h
@@ -42,15 +42,7 @@ class TestBase : public ::testing::Test {
     random_seed_ = 0;
   }
 
-  template <typename ArrayType>
-  std::shared_ptr<Array> MakePrimitive(int64_t length, int64_t null_count = 0) {
-    auto data = std::make_shared<PoolBuffer>(pool_);
-    const int64_t data_nbytes = length * sizeof(typename ArrayType::value_type);
-    EXPECT_OK(data->Resize(data_nbytes));
-
-    // Fill with random data
-    test::random_bytes(data_nbytes, random_seed_++, data->mutable_data());
-
+  std::shared_ptr<Buffer> MakeRandomNullBitmap(int64_t length, int64_t null_count) {
     auto null_bitmap = std::make_shared<PoolBuffer>(pool_);
     const int64_t null_nbytes = BitUtil::BytesForBits(length);
     EXPECT_OK(null_bitmap->Resize(null_nbytes));
@@ -58,14 +50,68 @@ class TestBase : public ::testing::Test {
     for (int64_t i = 0; i < null_count; i++) {
       BitUtil::ClearBit(null_bitmap->mutable_data(), i * (length / null_count));
     }
-    return std::make_shared<ArrayType>(length, data, null_bitmap, null_count);
+    return null_bitmap;
   }
 
+  template <typename ArrayType>
+  std::shared_ptr<Array> MakeRandomArray(int64_t length, int64_t null_count = 0);
+
  protected:
   uint32_t random_seed_;
   MemoryPool* pool_;
 };
 
+template <typename ArrayType>
+std::shared_ptr<Array> TestBase::MakeRandomArray(int64_t length, int64_t null_count) {
+  auto data = std::make_shared<PoolBuffer>(pool_);
+  const int64_t data_nbytes = length * sizeof(typename ArrayType::value_type);
+  EXPECT_OK(data->Resize(data_nbytes));
+
+  // Fill with random data
+  test::random_bytes(data_nbytes, random_seed_++, data->mutable_data());
+  std::shared_ptr<Buffer> null_bitmap = MakeRandomNullBitmap(length, null_count);
+
+  return std::make_shared<ArrayType>(length, data, null_bitmap, null_count);
+}
+
+template <>
+std::shared_ptr<Array> TestBase::MakeRandomArray<FixedSizeBinaryArray>(
+    int64_t length, int64_t null_count) {
+  const int byte_width = 10;
+  std::shared_ptr<Buffer> null_bitmap = MakeRandomNullBitmap(length, null_count);
+  std::shared_ptr<PoolBuffer> data = std::make_shared<PoolBuffer>(pool_);
+
+  EXPECT_OK(data->Resize(byte_width * length));
+  ::arrow::test::random_bytes(data->size(), 0, data->mutable_data());
+  return std::make_shared<FixedSizeBinaryArray>(fixed_size_binary(byte_width), length,
+                                                data, null_bitmap, null_count);
+}
+
+template <>
+std::shared_ptr<Array> TestBase::MakeRandomArray<BinaryArray>(int64_t length,
+                                                              int64_t null_count) {
+  std::vector<uint8_t> valid_bytes(length, 1);
+  for (int64_t i = 0; i < null_count; i++) {
+    valid_bytes[i * 2] = 0;
+  }
+  BinaryBuilder builder(pool_);
+
+  const int kBufferSize = 10;
+  uint8_t buffer[kBufferSize];
+  for (int64_t i = 0; i < length; i++) {
+    if (!valid_bytes[i]) {
+      EXPECT_OK(builder.AppendNull());
+    } else {
+      ::arrow::test::random_bytes(kBufferSize, static_cast<uint32_t>(i), buffer);
+      EXPECT_OK(builder.Append(buffer, kBufferSize));
+    }
+  }
+
+  std::shared_ptr<Array> out;
+  EXPECT_OK(builder.Finish(&out));
+  return out;
+}
+
 class TestBuilder : public ::testing::Test {
  public:
   void SetUp() {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 283e27e02c898..7630f48151b28 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -169,6 +169,11 @@ class ARROW_EXPORT DataType {
   ARROW_DISALLOW_COPY_AND_ASSIGN(DataType);
 };
 
+inline std::ostream& operator<<(std::ostream& os, const DataType& type) {
+  os << type.ToString();
+  return os;
+}
+
 // TODO(wesm): Remove this from parquet-cpp
 using TypePtr = std::shared_ptr<DataType>;
 
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index aa83746e1cf88..b8a8efabe5fef 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -49,9 +49,9 @@
 
 namespace arrow {
 
-#define INIT_BITSET(valid_bits_vector, valid_bits_index)        \
-  int byte_offset_##valid_bits_vector = (valid_bits_index) / 8; \
-  int bit_offset_##valid_bits_vector = (valid_bits_index) % 8;  \
+#define INIT_BITSET(valid_bits_vector, valid_bits_index)            \
+  int64_t byte_offset_##valid_bits_vector = (valid_bits_index) / 8; \
+  int64_t bit_offset_##valid_bits_vector = (valid_bits_index) % 8;  \
   uint8_t bitset_##valid_bits_vector = valid_bits_vector[byte_offset_##valid_bits_vector];
 
 #define READ_NEXT_BITSET(valid_bits_vector)                                          \

From 9ede7fbab43461e0774f710c784e5359623347ad Mon Sep 17 00:00:00 2001
From: Laurent Goujon <laurent@dremio.com>
Date: Mon, 11 Sep 2017 21:47:55 -0400
Subject: [PATCH 1043/1644] ARROW-1527: Fix Travis CI JDK9 build

Update `.travis.yml` to install JDK9 package as it seems it is not
part of the default trusty image.

Author: Laurent Goujon <laurent@dremio.com>

Closes #1089 from laurentgo/laurent/travis-jdk9 and squashes the following commits:

19299b31 [Laurent Goujon] ARROW-1527: Fix Travis CI JDK9 build
---
 .travis.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 76ef8a7d4aea6..c4a8f4120c732 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -94,6 +94,10 @@ matrix:
     jdk: oraclejdk9
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+    addons:
+      apt:
+        packages:
+          - oracle-java9-installer
   - language: java
     os: linux
     env: ARROW_TEST_GROUP=integration

From b590c245493084e584a7f79defc395282fd5c2a4 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 11 Sep 2017 21:49:13 -0400
Subject: [PATCH 1044/1644] ARROW-1519: [C++] Move DecimalUtil functions to
 methods on the Int128 class

This is mostly moving things around so that there's one place to access all Int128 functionality from third parties, the original impetus being `parquet-cpp` decimal reads/writes.

Author: Phillip Cloud <cpcloud@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1083 from cpcloud/ARROW-1519 and squashes the following commits:

27ae41dc [Wes McKinney] Rename Int128 to Decimal128
dd401c6d [Phillip Cloud] IWYU
51a3f240 [Phillip Cloud] Use a bool for negativity
94b5b37a [Phillip Cloud] Doc for Int128::FromString
db4d2070 [Phillip Cloud] Add doc and const
c06ebe83 [Phillip Cloud] Add DCHECK for Int128 pointer in StringToInteger
bec1c8b9 [Phillip Cloud] Add DCHECK for Int128 initial value in StringToInteger
d9829c9b [Phillip Cloud] Refactor Int128 construction from string
d7c2068c [Phillip Cloud] ARROW-1519: [C++] Move DecimalUtil functions to methods on the Int128 class
---
 cpp/CMakeLists.txt                      |   1 -
 cpp/src/arrow/array-test.cc             |  20 +-
 cpp/src/arrow/array.cc                  |   5 +-
 cpp/src/arrow/builder.cc                |   3 +-
 cpp/src/arrow/builder.h                 |   4 +-
 cpp/src/arrow/compare.cc                |   1 -
 cpp/src/arrow/pretty_print-test.cc      |   7 +-
 cpp/src/arrow/python/arrow_to_pandas.cc |   5 +-
 cpp/src/arrow/python/builtin_convert.cc |   5 +-
 cpp/src/arrow/python/helpers.cc         |   2 +-
 cpp/src/arrow/python/pandas_to_arrow.cc |   5 +-
 cpp/src/arrow/python/python-test.cc     |   2 -
 cpp/src/arrow/util/CMakeLists.txt       |   2 +-
 cpp/src/arrow/util/decimal-test.cc      | 140 +++---
 cpp/src/arrow/util/decimal.cc           | 617 +++++++++++++++++++++---
 cpp/src/arrow/util/decimal.h            | 118 ++++-
 cpp/src/arrow/util/int128.cc            | 527 --------------------
 cpp/src/arrow/util/int128.h             | 129 -----
 18 files changed, 755 insertions(+), 838 deletions(-)
 delete mode 100644 cpp/src/arrow/util/int128.cc
 delete mode 100644 cpp/src/arrow/util/int128.h

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 24735ac50c665..577a4bbea83fa 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -755,7 +755,6 @@ set(ARROW_SRCS
   src/arrow/util/compression.cc
   src/arrow/util/cpu-info.cc
   src/arrow/util/decimal.cc
-  src/arrow/util/int128.cc
   src/arrow/util/key_value_metadata.cc
 )
 
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index a9596c5b878d6..39db715b27080 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -34,7 +34,6 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
 
 namespace arrow {
 
@@ -2568,7 +2567,7 @@ TEST(TestUnionArrayAdHoc, TestSliceEquals) {
   CheckUnion(batch->column(2));
 }
 
-using DecimalVector = std::vector<Int128>;
+using DecimalVector = std::vector<Decimal128>;
 
 class DecimalTest : public ::testing::TestWithParam<int> {
  public:
@@ -2634,8 +2633,8 @@ class DecimalTest : public ::testing::TestWithParam<int> {
 
 TEST_P(DecimalTest, NoNulls) {
   int32_t precision = GetParam();
-  std::vector<Int128> draw = {Int128(1), Int128(-2), Int128(2389), Int128(4),
-                              Int128(-12348)};
+  std::vector<Decimal128> draw = {Decimal128(1), Decimal128(-2), Decimal128(2389),
+                                  Decimal128(4), Decimal128(-12348)};
   std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
   this->TestCreate(precision, draw, valid_bytes, 0);
   this->TestCreate(precision, draw, valid_bytes, 2);
@@ -2643,14 +2642,15 @@ TEST_P(DecimalTest, NoNulls) {
 
 TEST_P(DecimalTest, WithNulls) {
   int32_t precision = GetParam();
-  std::vector<Int128> draw = {Int128(1),  Int128(2), Int128(-1), Int128(4),
-                              Int128(-1), Int128(1), Int128(2)};
-  Int128 big;
-  ASSERT_OK(DecimalUtil::FromString("230342903942.234234", &big));
+  std::vector<Decimal128> draw = {Decimal128(1), Decimal128(2),  Decimal128(-1),
+                                  Decimal128(4), Decimal128(-1), Decimal128(1),
+                                  Decimal128(2)};
+  Decimal128 big;
+  ASSERT_OK(Decimal128::FromString("230342903942.234234", &big));
   draw.push_back(big);
 
-  Int128 big_negative;
-  ASSERT_OK(DecimalUtil::FromString("-23049302932.235234", &big_negative));
+  Decimal128 big_negative;
+  ASSERT_OK(Decimal128::FromString("-23049302932.235234", &big_negative));
   draw.push_back(big_negative);
 
   std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 2d37274a976c7..a1d3bed255d1d 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -29,7 +29,6 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/visitor.h"
@@ -319,8 +318,8 @@ DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data)
 
 std::string DecimalArray::FormatValue(int64_t i) const {
   const auto& type_ = static_cast<const DecimalType&>(*type());
-  Int128 value(GetValue(i));
-  return DecimalUtil::ToString(value, type_.precision(), type_.scale());
+  Decimal128 value(GetValue(i));
+  return value.ToString(type_.precision(), type_.scale());
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 5945677e72bfe..7f0f402bb622e 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -35,7 +35,6 @@
 #include "arrow/util/cpu-info.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/hash-util.h"
-#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -1119,7 +1118,7 @@ DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>
     : DecimalBuilder(type, pool) {}
 #endif
 
-Status DecimalBuilder::Append(const Int128& value) {
+Status DecimalBuilder::Append(const Decimal128& value) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
   std::array<uint8_t, 16> bytes;
   RETURN_NOT_OK(value.ToBytes(&bytes));
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index bf7b317f68c60..7df8899a31b63 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -38,7 +38,7 @@
 namespace arrow {
 
 class Array;
-class Int128;
+class Decimal128;
 
 namespace internal {
 
@@ -778,7 +778,7 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
 
   using FixedSizeBinaryBuilder::Append;
 
-  Status Append(const Int128& val);
+  Status Append(const Decimal128& val);
 
   Status Finish(std::shared_ptr<Array>* out) override;
 };
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 876e7f73d3fd2..2aeb03b8b9a12 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -29,7 +29,6 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
-#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 #include "arrow/visitor_inline.h"
 
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index ab0bc16c83448..8b9a24fecd332 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -32,7 +32,6 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
 
 namespace arrow {
 
@@ -116,12 +115,12 @@ TEST_F(TestPrettyPrint, DecimalType) {
 
   DecimalBuilder builder(type);
 
-  Int128 val;
+  Decimal128 val;
 
-  ASSERT_OK(DecimalUtil::FromString("123.4567", &val));
+  ASSERT_OK(Decimal128::FromString("123.4567", &val));
   ASSERT_OK(builder.Append(val));
 
-  ASSERT_OK(DecimalUtil::FromString("456.7891", &val));
+  ASSERT_OK(Decimal128::FromString("456.7891", &val));
   ASSERT_OK(builder.Append(val));
   ASSERT_OK(builder.AppendNull());
 
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 117bf232f2974..148963fc2a891 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -37,7 +37,6 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/parallel.h"
@@ -606,8 +605,8 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
 static Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
                                  std::string* result) {
   DCHECK_NE(result, nullptr);
-  Int128 decimal(bytes);
-  *result = DecimalUtil::ToString(decimal, precision, scale);
+  Decimal128 decimal(bytes);
+  *result = decimal.ToString(precision, scale);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index eb17f4efee762..747b872af0a30 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -28,7 +28,6 @@
 #include "arrow/api.h"
 #include "arrow/status.h"
 #include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 
 #include "arrow/python/helpers.h"
@@ -589,8 +588,8 @@ class DecimalConverter
       std::string string;
       RETURN_NOT_OK(PythonDecimalToString(item.obj(), &string));
 
-      Int128 value;
-      RETURN_NOT_OK(DecimalUtil::FromString(string, &value));
+      Decimal128 value;
+      RETURN_NOT_OK(Decimal128::FromString(string, &value));
       return typed_builder_->Append(value);
     }
 
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index deda93f3d7612..fb2fed7f0ca44 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -103,7 +103,7 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision,
   auto size = str.size;
 
   std::string c_string(bytes, size);
-  return DecimalUtil::FromString(c_string, nullptr, precision, scale);
+  return Decimal128::FromString(c_string, nullptr, precision, scale);
 }
 
 Status DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string,
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 1f96f4ed9d67b..64f753e53d2c1 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -39,7 +39,6 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/visitor_inline.h"
@@ -659,8 +658,8 @@ Status PandasConverter::ConvertDecimals() {
       std::string string;
       RETURN_NOT_OK(PythonDecimalToString(object, &string));
 
-      Int128 value;
-      RETURN_NOT_OK(DecimalUtil::FromString(string, &value));
+      Decimal128 value;
+      RETURN_NOT_OK(Decimal128::FromString(string, &value));
       RETURN_NOT_OK(builder.Append(value));
     } else if (PandasObjectIsNull(object)) {
       RETURN_NOT_OK(builder.AppendNull());
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 0241ff0579563..e1796c097d660 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -29,8 +29,6 @@
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/helpers.h"
 
-#include "arrow/util/decimal.h"
-
 namespace arrow {
 namespace py {
 
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 070582008aec0..44be1c92ae64a 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -32,8 +32,8 @@ install(FILES
   compression_zlib.h
   compression_zstd.h
   cpu-info.h
+  decimal.h
   hash-util.h
-  int128.h
   key_value_metadata.h
   logging.h
   macros.h
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 97af0889a4853..6f5118ea34c56 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -24,31 +24,29 @@
 #include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
 
 namespace arrow {
-namespace DecimalUtil {
 
 class DecimalTestFixture : public ::testing::Test {
  public:
   DecimalTestFixture() : integer_value_(23423445), string_value_("234.23445") {}
-  Int128 integer_value_;
+  Decimal128 integer_value_;
   std::string string_value_;
 };
 
 TEST_F(DecimalTestFixture, TestToString) {
-  Int128 decimal(this->integer_value_);
+  Decimal128 decimal(this->integer_value_);
   int precision = 8;
   int scale = 5;
-  std::string result = ToString(decimal, precision, scale);
+  std::string result = decimal.ToString(precision, scale);
   ASSERT_EQ(result, this->string_value_);
 }
 
 TEST_F(DecimalTestFixture, TestFromString) {
-  Int128 expected(this->integer_value_);
-  Int128 result;
+  Decimal128 expected(this->integer_value_);
+  Decimal128 result;
   int precision, scale;
-  ASSERT_OK(FromString(this->string_value_, &result, &precision, &scale));
+  ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
   ASSERT_EQ(result, expected);
   ASSERT_EQ(precision, 8);
   ASSERT_EQ(scale, 5);
@@ -56,10 +54,10 @@ TEST_F(DecimalTestFixture, TestFromString) {
 
 TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
   std::string plus_value("+234.234");
-  Int128 out;
+  Decimal128 out;
   int scale;
   int precision;
-  ASSERT_OK(FromString(plus_value, &out, &precision, &scale));
+  ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
   ASSERT_EQ(234234, out);
   ASSERT_EQ(6, precision);
   ASSERT_EQ(3, scale);
@@ -67,39 +65,20 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
 
 TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
   std::string plus_value("+2342394230592.232349023094");
-  Int128 expected_value("2342394230592232349023094");
-  Int128 out;
+  Decimal128 expected_value("2342394230592232349023094");
+  Decimal128 out;
   int scale;
   int precision;
-  ASSERT_OK(FromString(plus_value, &out, &precision, &scale));
+  ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
   ASSERT_EQ(expected_value, out);
   ASSERT_EQ(25, precision);
   ASSERT_EQ(12, scale);
 }
 
-TEST(DecimalTest, TestStringToInt32) {
-  Int128 value;
-  StringToInteger("123", "456", 1, &value);
-  ASSERT_EQ(value, 123456);
-}
-
-TEST(DecimalTest, TestStringToInt64) {
-  Int128 value;
-  StringToInteger("123456789", "456", -1, &value);
-  ASSERT_EQ(value, -123456789456);
-}
-
-TEST(DecimalTest, TestStringToInt128) {
-  Int128 value;
-  StringToInteger("123456789", "456789123", 1, &value);
-  ASSERT_EQ(value.high_bits(), 0);
-  ASSERT_EQ(value.low_bits(), 123456789456789123);
-}
-
-TEST(DecimalTest, TestFromString128) {
-  static const std::string string_value("-23049223942343532412");
-  Int128 result(string_value);
-  Int128 expected(static_cast<int64_t>(-230492239423435324));
+TEST(DecimalTest, TestFromStringDecimal128) {
+  std::string string_value("-23049223942343532412");
+  Decimal128 result(string_value);
+  Decimal128 expected(static_cast<int64_t>(-230492239423435324));
   ASSERT_EQ(result, expected * 100 - 12);
 
   // Sanity check that our number is actually using more than 64 bits
@@ -108,9 +87,9 @@ TEST(DecimalTest, TestFromString128) {
 
 TEST(DecimalTest, TestFromDecimalString128) {
   std::string string_value("-23049223942343.532412");
-  Int128 result;
-  ASSERT_OK(FromString(string_value, &result));
-  Int128 expected(static_cast<int64_t>(-230492239423435324));
+  Decimal128 result;
+  ASSERT_OK(Decimal128::FromString(string_value, &result));
+  Decimal128 expected(static_cast<int64_t>(-230492239423435324));
   expected *= 100;
   expected -= 12;
   ASSERT_EQ(result, expected);
@@ -120,109 +99,109 @@ TEST(DecimalTest, TestFromDecimalString128) {
 }
 
 TEST(DecimalTest, TestDecimal32SignedRoundTrip) {
-  Int128 expected("-3402692");
+  Decimal128 expected("-3402692");
 
   std::array<uint8_t, 16> bytes;
   ASSERT_OK(expected.ToBytes(&bytes));
 
-  Int128 result(bytes.data());
+  Decimal128 result(bytes.data());
   ASSERT_EQ(expected, result);
 }
 
 TEST(DecimalTest, TestDecimal64SignedRoundTrip) {
-  Int128 expected;
+  Decimal128 expected;
   std::string string_value("-34034293045.921");
-  ASSERT_OK(FromString(string_value, &expected));
+  ASSERT_OK(Decimal128::FromString(string_value, &expected));
 
   std::array<uint8_t, 16> bytes;
   ASSERT_OK(expected.ToBytes(&bytes));
 
-  Int128 result(bytes.data());
+  Decimal128 result(bytes.data());
 
   ASSERT_EQ(expected, result);
 }
 
 TEST(DecimalTest, TestDecimalStringAndBytesRoundTrip) {
-  Int128 expected;
+  Decimal128 expected;
   std::string string_value("-340282366920938463463374607431.711455");
-  ASSERT_OK(FromString(string_value, &expected));
+  ASSERT_OK(Decimal128::FromString(string_value, &expected));
 
   std::string expected_string_value("-340282366920938463463374607431711455");
-  Int128 expected_underlying_value(expected_string_value);
+  Decimal128 expected_underlying_value(expected_string_value);
 
   ASSERT_EQ(expected, expected_underlying_value);
 
   std::array<uint8_t, 16> bytes;
   ASSERT_OK(expected.ToBytes(&bytes));
 
-  Int128 result(bytes.data());
+  Decimal128 result(bytes.data());
 
   ASSERT_EQ(expected, result);
 }
 
 TEST(DecimalTest, TestInvalidInputMinus) {
   std::string invalid_value("-");
-  Int128 out;
-  Status status = DecimalUtil::FromString(invalid_value, &out);
+  Decimal128 out;
+  Status status = Decimal128::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputDot) {
   std::string invalid_value("0.0.0");
-  Int128 out;
-  Status status = DecimalUtil::FromString(invalid_value, &out);
+  Decimal128 out;
+  Status status = Decimal128::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputEmbeddedMinus) {
   std::string invalid_value("0-13-32");
-  Int128 out;
-  Status status = DecimalUtil::FromString(invalid_value, &out);
+  Decimal128 out;
+  Status status = Decimal128::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputSingleChar) {
   std::string invalid_value("a");
-  Int128 out;
-  Status status = DecimalUtil::FromString(invalid_value, &out);
+  Decimal128 out;
+  Status status = Decimal128::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
   std::string invalid_value("-23092.235-");
-  Int128 out;
-  Status status = DecimalUtil::FromString(invalid_value, &out);
+  Decimal128 out;
+  Status status = Decimal128::FromString(invalid_value, &out);
   auto msg = status.message();
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithMinusPlus) {
   std::string invalid_value("-+23092.235");
-  Int128 out;
-  Status status = DecimalUtil::FromString(invalid_value, &out);
+  Decimal128 out;
+  Status status = Decimal128::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithPlusMinus) {
   std::string invalid_value("+-23092.235");
-  Int128 out;
-  Status status = DecimalUtil::FromString(invalid_value, &out);
+  Decimal128 out;
+  Status status = Decimal128::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
   std::string invalid_value("00a");
-  Int128 out;
-  Status status = DecimalUtil::FromString(invalid_value, &out);
+  Decimal128 out;
+  Status status = Decimal128::FromString(invalid_value, &out);
   ASSERT_RAISES(Invalid, status);
 }
 
 TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
   std::string string_value("0000000");
-  Int128 d;
+  Decimal128 d;
   int precision;
   int scale;
-  ASSERT_OK(FromString(string_value, &d, &precision, &scale));
+  ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
   ASSERT_EQ(precision, 7);
   ASSERT_EQ(scale, 0);
   ASSERT_EQ(d, 0);
@@ -230,10 +209,10 @@ TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
 
 TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
   std::string string_value("000.0000");
-  Int128 d;
+  Decimal128 d;
   int precision;
   int scale;
-  ASSERT_OK(FromString(string_value, &d, &precision, &scale));
+  ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
   // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
   ASSERT_NE(precision, 7);
 
@@ -243,44 +222,43 @@ TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
 
 TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
   std::string string_value(".00000");
-  Int128 d;
+  Decimal128 d;
   int precision;
   int scale;
-  ASSERT_OK(FromString(string_value, &d, &precision, &scale));
+  ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
   ASSERT_EQ(precision, 5);
   ASSERT_EQ(scale, 5);
   ASSERT_EQ(d, 0);
 }
 
 template <typename T>
-class Int128Test : public ::testing::Test {
+class Decimal128Test : public ::testing::Test {
  public:
-  Int128Test() : value_(42) {}
+  Decimal128Test() : value_(42) {}
   const T value_;
 };
 
-using Int128Types =
+using Decimal128Types =
     ::testing::Types<char, unsigned char, short, unsigned short,  // NOLINT
                      int, unsigned int, long, unsigned long,      // NOLINT
                      long long, unsigned long long                // NOLINT
                      >;
 
-TYPED_TEST_CASE(Int128Test, Int128Types);
+TYPED_TEST_CASE(Decimal128Test, Decimal128Types);
 
-TYPED_TEST(Int128Test, ConstructibleFromAnyIntegerType) {
-  Int128 value(this->value_);
+TYPED_TEST(Decimal128Test, ConstructibleFromAnyIntegerType) {
+  Decimal128 value(this->value_);
   ASSERT_EQ(42, value.low_bits());
 }
 
-TEST(Int128TestTrue, ConstructibleFromBool) {
-  Int128 value(true);
+TEST(Decimal128TestTrue, ConstructibleFromBool) {
+  Decimal128 value(true);
   ASSERT_EQ(1, value.low_bits());
 }
 
-TEST(Int128TestFalse, ConstructibleFromBool) {
-  Int128 value(false);
+TEST(Decimal128TestFalse, ConstructibleFromBool) {
+  Decimal128 value(false);
   ASSERT_EQ(0, value.low_bits());
 }
 
-}  // namespace DecimalUtil
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 7dd5807330747..123482e75a37d 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -17,40 +17,140 @@
 
 #include <algorithm>
 #include <cctype>
-#include <cstdlib>
+#include <cmath>
+#include <cstring>
+#include <limits>
 #include <sstream>
 
-#include "arrow/util/bit-util.h"
+#ifdef _MSC_VER
+#include <intrin.h>
+#pragma intrinsic(_BitScanReverse)
+#endif
+
 #include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
-namespace DecimalUtil {
 
-void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
-                     Int128* out) {
-  DCHECK(sign == -1 || sign == 1);
-  DCHECK_NE(out, nullptr);
-  DCHECK(!whole.empty() || !fractional.empty());
-  *out = Int128(whole + fractional) * sign;
+static constexpr uint64_t kIntMask = 0xFFFFFFFF;
+static constexpr auto kCarryBit = static_cast<uint64_t>(1) << static_cast<uint64_t>(32);
+
+Decimal128::Decimal128(const std::string& str) : Decimal128() {
+  Status status(Decimal128::FromString(str, this));
+  DCHECK(status.ok()) << status.message();
+}
+
+Decimal128::Decimal128(const uint8_t* bytes)
+    : Decimal128(reinterpret_cast<const int64_t*>(bytes)[0],
+                 reinterpret_cast<const uint64_t*>(bytes)[1]) {}
+
+Status Decimal128::ToBytes(std::array<uint8_t, 16>* out) const {
+  if (out == nullptr) {
+    return Status::Invalid("Cannot fill nullptr of bytes from Decimal128");
+  }
+
+  const uint64_t raw[] = {static_cast<uint64_t>(high_bits_), low_bits_};
+  const auto* raw_data = reinterpret_cast<const uint8_t*>(raw);
+  std::copy(raw_data, raw_data + out->size(), out->begin());
+  return Status::OK();
+}
+
+std::string Decimal128::ToString(int precision, int scale) const {
+  using std::size_t;
+
+  const bool is_negative = *this < 0;
+
+  // Decimal values are sent to clients as strings so in the interest of
+  // speed the string will be created without the using stringstream with the
+  // whole/fractional_part().
+  size_t last_char_idx = precision + (scale > 0)  // Add a space for decimal place
+                         + (scale == precision)   // Add a space for leading 0
+                         + is_negative;           // Add a space for negative sign
+
+  std::string str(last_char_idx, '0');
+
+  // Start filling in the values in reverse order by taking the last digit
+  // of the value. Use a positive value and worry about the sign later. At this
+  // point the last_char_idx points to the string terminator.
+  Decimal128 remaining_value(*this);
+
+  const auto first_digit_idx = static_cast<size_t>(is_negative);
+  if (is_negative) {
+    remaining_value.Negate();
+  }
+
+  if (scale > 0) {
+    int remaining_scale = scale;
+    do {
+      str[--last_char_idx] =
+          static_cast<char>(remaining_value % 10 + '0');  // Ascii offset
+      remaining_value /= 10;
+    } while (--remaining_scale > 0);
+    str[--last_char_idx] = '.';
+    DCHECK_GT(last_char_idx, first_digit_idx) << "Not enough space remaining";
+  }
+
+  do {
+    str[--last_char_idx] = static_cast<char>(remaining_value % 10 + '0');  // Ascii offset
+    remaining_value /= 10;
+    if (remaining_value == 0) {
+      // Trim any extra leading 0's.
+      if (last_char_idx > first_digit_idx) {
+        str.erase(0, last_char_idx - first_digit_idx);
+      }
+
+      break;
+    }
+    // For safety, enforce string length independent of remaining_value.
+  } while (last_char_idx > first_digit_idx);
+
+  if (is_negative) {
+    str[0] = '-';
+  }
+
+  return str;
+}
+
+static void StringToInteger(const std::string& str, Decimal128* out) {
+  using std::size_t;
+
+  DCHECK_NE(out, nullptr) << "Decimal128 output variable cannot be nullptr";
+  DCHECK_EQ(*out, 0)
+      << "When converting a string to Decimal128 the initial output must be 0";
+
+  const size_t length = str.length();
+
+  DCHECK_GT(length, 0) << "length of parsed decimal string should be greater than 0";
+
+  size_t posn = 0;
+
+  while (posn < length) {
+    const size_t group = std::min(static_cast<size_t>(18), length - posn);
+    const auto chunk = static_cast<int64_t>(std::stoll(str.substr(posn, group)));
+    const auto multiple =
+        static_cast<int64_t>(std::pow(10.0, static_cast<double>(group)));
+
+    *out *= multiple;
+    *out += chunk;
+
+    posn += group;
+  }
 }
 
-Status FromString(const std::string& s, Int128* out, int* precision, int* scale) {
+Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precision,
+                              int* scale) {
   // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?";
   if (s.empty()) {
     return Status::Invalid("Empty string cannot be converted to decimal");
   }
 
-  int8_t sign = 1;
   std::string::const_iterator charp = s.cbegin();
   std::string::const_iterator end = s.cend();
 
   char first_char = *charp;
+  bool is_negative = false;
   if (first_char == '+' || first_char == '-') {
-    if (first_char == '-') {
-      sign = -1;
-    }
+    is_negative = first_char == '-';
     ++charp;
   }
 
@@ -70,7 +170,7 @@ Status FromString(const std::string& s, Int128* out, int* precision, int* scale)
   // all zeros and no decimal point
   if (charp == end) {
     if (out != nullptr) {
-      *out = Int128(0);
+      *out = 0;
     }
 
     // Not sure what other libraries assign precision to for this case (this case of
@@ -146,64 +246,469 @@ Status FromString(const std::string& s, Int128* out, int* precision, int* scale)
   }
 
   if (out != nullptr) {
-    StringToInteger(whole_part, fractional_part, sign, out);
+    // zero out in case we've passed in a previously used value
+    *out = 0;
+    StringToInteger(whole_part + fractional_part, out);
+    if (is_negative) {
+      out->Negate();
+    }
   }
 
   return Status::OK();
 }
 
-std::string ToString(const Int128& decimal_value, int precision, int scale) {
-  const bool is_negative = decimal_value < 0;
+Decimal128& Decimal128::Negate() {
+  low_bits_ = ~low_bits_ + 1;
+  high_bits_ = ~high_bits_;
+  if (low_bits_ == 0) {
+    ++high_bits_;
+  }
+  return *this;
+}
 
-  // Decimal values are sent to clients as strings so in the interest of
-  // speed the string will be created without the using stringstream with the
-  // whole/fractional_part().
-  size_t last_char_idx = precision + (scale > 0)  // Add a space for decimal place
-                         + (scale == precision)   // Add a space for leading 0
-                         + is_negative;           // Add a space for negative sign
-  std::string str(last_char_idx, '0');
+Decimal128& Decimal128::operator+=(const Decimal128& right) {
+  const uint64_t sum = low_bits_ + right.low_bits_;
+  high_bits_ += right.high_bits_;
+  if (sum < low_bits_) {
+    ++high_bits_;
+  }
+  low_bits_ = sum;
+  return *this;
+}
 
-  // Start filling in the values in reverse order by taking the last digit
-  // of the value. Use a positive value and worry about the sign later. At this
-  // point the last_char_idx points to the string terminator.
-  Int128 remaining_value(decimal_value);
+Decimal128& Decimal128::operator-=(const Decimal128& right) {
+  const uint64_t diff = low_bits_ - right.low_bits_;
+  high_bits_ -= right.high_bits_;
+  if (diff > low_bits_) {
+    --high_bits_;
+  }
+  low_bits_ = diff;
+  return *this;
+}
 
-  const auto first_digit_idx = static_cast<size_t>(is_negative);
-  if (is_negative) {
-    remaining_value.Negate();
+Decimal128& Decimal128::operator/=(const Decimal128& right) {
+  Decimal128 remainder;
+  DCHECK(Divide(right, this, &remainder).ok());
+  return *this;
+}
+
+Decimal128::operator char() const {
+  DCHECK(high_bits_ == 0 || high_bits_ == -1)
+      << "Trying to cast an Decimal128 greater than the value range of a "
+         "char. high_bits_ must be equal to 0 or -1, got: "
+      << high_bits_;
+  DCHECK_LE(low_bits_, std::numeric_limits<char>::max())
+      << "low_bits_ too large for C type char, got: " << low_bits_;
+  return static_cast<char>(low_bits_);
+}
+
+Decimal128& Decimal128::operator|=(const Decimal128& right) {
+  low_bits_ |= right.low_bits_;
+  high_bits_ |= right.high_bits_;
+  return *this;
+}
+
+Decimal128& Decimal128::operator&=(const Decimal128& right) {
+  low_bits_ &= right.low_bits_;
+  high_bits_ &= right.high_bits_;
+  return *this;
+}
+
+Decimal128& Decimal128::operator<<=(uint32_t bits) {
+  if (bits != 0) {
+    if (bits < 64) {
+      high_bits_ <<= bits;
+      high_bits_ |= (low_bits_ >> (64 - bits));
+      low_bits_ <<= bits;
+    } else if (bits < 128) {
+      high_bits_ = static_cast<int64_t>(low_bits_) << (bits - 64);
+      low_bits_ = 0;
+    } else {
+      high_bits_ = 0;
+      low_bits_ = 0;
+    }
   }
+  return *this;
+}
 
-  if (scale > 0) {
-    int remaining_scale = scale;
-    do {
-      str[--last_char_idx] =
-          static_cast<char>(remaining_value % 10 + '0');  // Ascii offset
-      remaining_value /= 10;
-    } while (--remaining_scale > 0);
-    str[--last_char_idx] = '.';
-    DCHECK_GT(last_char_idx, first_digit_idx) << "Not enough space remaining";
+Decimal128& Decimal128::operator>>=(uint32_t bits) {
+  if (bits != 0) {
+    if (bits < 64) {
+      low_bits_ >>= bits;
+      low_bits_ |= static_cast<uint64_t>(high_bits_ << (64 - bits));
+      high_bits_ = static_cast<int64_t>(static_cast<uint64_t>(high_bits_) >> bits);
+    } else if (bits < 128) {
+      low_bits_ = static_cast<uint64_t>(high_bits_ >> (bits - 64));
+      high_bits_ = static_cast<int64_t>(high_bits_ >= 0L ? 0L : -1L);
+    } else {
+      high_bits_ = static_cast<int64_t>(high_bits_ >= 0L ? 0L : -1L);
+      low_bits_ = static_cast<uint64_t>(high_bits_);
+    }
   }
+  return *this;
+}
 
-  do {
-    str[--last_char_idx] = static_cast<char>(remaining_value % 10 + '0');  // Ascii offset
-    remaining_value /= 10;
-    if (remaining_value == 0) {
-      // Trim any extra leading 0's.
-      if (last_char_idx > first_digit_idx) {
-        str.erase(0, last_char_idx - first_digit_idx);
-      }
+Decimal128& Decimal128::operator*=(const Decimal128& right) {
+  // Break the left and right numbers into 32 bit chunks
+  // so that we can multiply them without overflow.
+  const uint64_t L0 = static_cast<uint64_t>(high_bits_) >> 32;
+  const uint64_t L1 = static_cast<uint64_t>(high_bits_) & kIntMask;
+  const uint64_t L2 = low_bits_ >> 32;
+  const uint64_t L3 = low_bits_ & kIntMask;
+
+  const uint64_t R0 = static_cast<uint64_t>(right.high_bits_) >> 32;
+  const uint64_t R1 = static_cast<uint64_t>(right.high_bits_) & kIntMask;
+  const uint64_t R2 = right.low_bits_ >> 32;
+  const uint64_t R3 = right.low_bits_ & kIntMask;
+
+  uint64_t product = L3 * R3;
+  low_bits_ = product & kIntMask;
+
+  uint64_t sum = product >> 32;
+
+  product = L2 * R3;
+  sum += product;
+
+  product = L3 * R2;
+  sum += product;
+
+  low_bits_ += sum << 32;
+
+  high_bits_ = static_cast<int64_t>(sum < product ? kCarryBit : 0);
+  if (sum < product) {
+    high_bits_ += kCarryBit;
+  }
+
+  high_bits_ += static_cast<int64_t>(sum >> 32);
+  high_bits_ += L1 * R3 + L2 * R2 + L3 * R1;
+  high_bits_ += (L0 * R3 + L1 * R2 + L2 * R1 + L3 * R0) << 32;
+  return *this;
+}
+
+/// Expands the given value into an array of ints so that we can work on
+/// it. The array will be converted to an absolute value and the wasNegative
+/// flag will be set appropriately. The array will remove leading zeros from
+/// the value.
+/// \param array an array of length 4 to set with the value
+/// \param was_negative a flag for whether the value was original negative
+/// \result the output length of the array
+static int64_t FillInArray(const Decimal128& value, uint32_t* array, bool& was_negative) {
+  uint64_t high;
+  uint64_t low;
+  const int64_t highbits = value.high_bits();
+  const uint64_t lowbits = value.low_bits();
+
+  if (highbits < 0) {
+    low = ~lowbits + 1;
+    high = static_cast<uint64_t>(~highbits);
+    if (low == 0) {
+      ++high;
+    }
+    was_negative = true;
+  } else {
+    low = lowbits;
+    high = static_cast<uint64_t>(highbits);
+    was_negative = false;
+  }
+
+  if (high != 0) {
+    if (high > std::numeric_limits<uint32_t>::max()) {
+      array[0] = static_cast<uint32_t>(high >> 32);
+      array[1] = static_cast<uint32_t>(high);
+      array[2] = static_cast<uint32_t>(low >> 32);
+      array[3] = static_cast<uint32_t>(low);
+      return 4;
+    }
+
+    array[0] = static_cast<uint32_t>(high);
+    array[1] = static_cast<uint32_t>(low >> 32);
+    array[2] = static_cast<uint32_t>(low);
+    return 3;
+  }
 
+  if (low >= std::numeric_limits<uint32_t>::max()) {
+    array[0] = static_cast<uint32_t>(low >> 32);
+    array[1] = static_cast<uint32_t>(low);
+    return 2;
+  }
+
+  if (low == 0) {
+    return 0;
+  }
+
+  array[0] = static_cast<uint32_t>(low);
+  return 1;
+}
+
+/// \brief Find last set bit in a 32 bit integer. Bit 1 is the LSB and bit 32 is the MSB.
+static int64_t FindLastSetBit(uint32_t value) {
+#if defined(__clang__) || defined(__GNUC__)
+  // Count leading zeros
+  return __builtin_clz(value) + 1;
+#elif defined(_MSC_VER)
+  unsigned long index;                                         // NOLINT
+  _BitScanReverse(&index, static_cast<unsigned long>(value));  // NOLINT
+  return static_cast<int64_t>(index + 1UL);
+#endif
+}
+
+/// Shift the number in the array left by bits positions.
+/// \param array the number to shift, must have length elements
+/// \param length the number of entries in the array
+/// \param bits the number of bits to shift (0 <= bits < 32)
+static void ShiftArrayLeft(uint32_t* array, int64_t length, int64_t bits) {
+  if (length > 0 && bits != 0) {
+    for (int64_t i = 0; i < length - 1; ++i) {
+      array[i] = (array[i] << bits) | (array[i + 1] >> (32 - bits));
+    }
+    array[length - 1] <<= bits;
+  }
+}
+
+/// Shift the number in the array right by bits positions.
+/// \param array the number to shift, must have length elements
+/// \param length the number of entries in the array
+/// \param bits the number of bits to shift (0 <= bits < 32)
+static void ShiftArrayRight(uint32_t* array, int64_t length, int64_t bits) {
+  if (length > 0 && bits != 0) {
+    for (int64_t i = length - 1; i > 0; --i) {
+      array[i] = (array[i] >> bits) | (array[i - 1] << (32 - bits));
+    }
+    array[0] >>= bits;
+  }
+}
+
+/// \brief Fix the signs of the result and remainder at the end of the division based on
+/// the signs of the dividend and divisor.
+static void FixDivisionSigns(Decimal128* result, Decimal128* remainder,
+                             bool dividend_was_negative, bool divisor_was_negative) {
+  if (dividend_was_negative != divisor_was_negative) {
+    result->Negate();
+  }
+
+  if (dividend_was_negative) {
+    remainder->Negate();
+  }
+}
+
+/// \brief Build a Decimal128 from a list of ints.
+static Status BuildFromArray(Decimal128* value, uint32_t* array, int64_t length) {
+  switch (length) {
+    case 0:
+      *value = {static_cast<int64_t>(0)};
+      break;
+    case 1:
+      *value = {static_cast<int64_t>(array[0])};
+      break;
+    case 2:
+      *value = {static_cast<int64_t>(0),
+                (static_cast<uint64_t>(array[0]) << 32) + array[1]};
+      break;
+    case 3:
+      *value = {static_cast<int64_t>(array[0]),
+                (static_cast<uint64_t>(array[1]) << 32) + array[2]};
+      break;
+    case 4:
+      *value = {(static_cast<int64_t>(array[0]) << 32) + array[1],
+                (static_cast<uint64_t>(array[2]) << 32) + array[3]};
+      break;
+    case 5:
+      if (array[0] != 0) {
+        return Status::Invalid("Can't build Decimal128 with 5 ints.");
+      }
+      *value = {(static_cast<int64_t>(array[1]) << 32) + array[2],
+                (static_cast<uint64_t>(array[3]) << 32) + array[4]};
       break;
+    default:
+      return Status::Invalid("Unsupported length for building Decimal128");
+  }
+
+  return Status::OK();
+}
+
+/// \brief Do a division where the divisor fits into a single 32 bit value.
+static Status SingleDivide(const uint32_t* dividend, int64_t dividend_length,
+                           uint32_t divisor, Decimal128* remainder,
+                           bool dividend_was_negative, bool divisor_was_negative,
+                           Decimal128* result) {
+  uint64_t r = 0;
+  uint32_t result_array[5];
+  for (int64_t j = 0; j < dividend_length; j++) {
+    r <<= 32;
+    r += dividend[j];
+    result_array[j] = static_cast<uint32_t>(r / divisor);
+    r %= divisor;
+  }
+  RETURN_NOT_OK(BuildFromArray(result, result_array, dividend_length));
+  *remainder = static_cast<int64_t>(r);
+  FixDivisionSigns(result, remainder, dividend_was_negative, divisor_was_negative);
+  return Status::OK();
+}
+
+Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result,
+                          Decimal128* remainder) const {
+  // Split the dividend and divisor into integer pieces so that we can
+  // work on them.
+  uint32_t dividend_array[5];
+  uint32_t divisor_array[4];
+  bool dividend_was_negative;
+  bool divisor_was_negative;
+  // leave an extra zero before the dividend
+  dividend_array[0] = 0;
+  int64_t dividend_length =
+      FillInArray(*this, dividend_array + 1, dividend_was_negative) + 1;
+  int64_t divisor_length = FillInArray(divisor, divisor_array, divisor_was_negative);
+
+  // Handle some of the easy cases.
+  if (dividend_length <= divisor_length) {
+    *remainder = *this;
+    *result = 0;
+    return Status::OK();
+  }
+
+  if (divisor_length == 0) {
+    return Status::Invalid("Division by 0 in Decimal128");
+  }
+
+  if (divisor_length == 1) {
+    return SingleDivide(dividend_array, dividend_length, divisor_array[0], remainder,
+                        dividend_was_negative, divisor_was_negative, result);
+  }
+
+  int64_t result_length = dividend_length - divisor_length;
+  uint32_t result_array[4];
+
+  // Normalize by shifting both by a multiple of 2 so that
+  // the digit guessing is better. The requirement is that
+  // divisor_array[0] is greater than 2**31.
+  int64_t normalize_bits = 32 - FindLastSetBit(divisor_array[0]);
+  ShiftArrayLeft(divisor_array, divisor_length, normalize_bits);
+  ShiftArrayLeft(dividend_array, dividend_length, normalize_bits);
+
+  // compute each digit in the result
+  for (int64_t j = 0; j < result_length; ++j) {
+    // Guess the next digit. At worst it is two too large
+    uint32_t guess = std::numeric_limits<uint32_t>::max();
+    auto high_dividend =
+        static_cast<uint64_t>(dividend_array[j]) << 32 | dividend_array[j + 1];
+    if (dividend_array[j] != divisor_array[0]) {
+      guess = static_cast<uint32_t>(high_dividend / divisor_array[0]);
     }
-    // For safety, enforce string length independent of remaining_value.
-  } while (last_char_idx > first_digit_idx);
 
-  if (is_negative) {
-    str[0] = '-';
+    // catch all of the cases where guess is two too large and most of the
+    // cases where it is one too large
+    auto rhat = static_cast<uint32_t>(high_dividend -
+                                      guess * static_cast<uint64_t>(divisor_array[0]));
+    while (static_cast<uint64_t>(divisor_array[1]) * guess >
+           (static_cast<uint64_t>(rhat) << 32) + dividend_array[j + 2]) {
+      --guess;
+      rhat += divisor_array[0];
+      if (static_cast<uint64_t>(rhat) < divisor_array[0]) {
+        break;
+      }
+    }
+
+    // subtract off the guess * divisor from the dividend
+    uint64_t mult = 0;
+    for (int64_t i = divisor_length - 1; i >= 0; --i) {
+      mult += static_cast<uint64_t>(guess) * divisor_array[i];
+      uint32_t prev = dividend_array[j + i + 1];
+      dividend_array[j + i + 1] -= static_cast<uint32_t>(mult);
+      mult >>= 32;
+      if (dividend_array[j + i + 1] > prev) {
+        ++mult;
+      }
+    }
+    uint32_t prev = dividend_array[j];
+    dividend_array[j] -= static_cast<uint32_t>(mult);
+
+    // if guess was too big, we add back divisor
+    if (dividend_array[j] > prev) {
+      --guess;
+
+      uint32_t carry = 0;
+      for (int64_t i = divisor_length - 1; i >= 0; --i) {
+        uint64_t sum =
+            static_cast<uint64_t>(divisor_array[i]) + dividend_array[j + i + 1] + carry;
+        dividend_array[j + i + 1] = static_cast<uint32_t>(sum);
+        carry = static_cast<uint32_t>(sum >> 32);
+      }
+      dividend_array[j] += carry;
+    }
+
+    result_array[j] = guess;
   }
 
-  return str;
+  // denormalize the remainder
+  ShiftArrayRight(dividend_array, dividend_length, normalize_bits);
+
+  // return result and remainder
+  RETURN_NOT_OK(BuildFromArray(result, result_array, result_length));
+  RETURN_NOT_OK(BuildFromArray(remainder, dividend_array, dividend_length));
+  FixDivisionSigns(result, remainder, dividend_was_negative, divisor_was_negative);
+  return Status::OK();
+}
+
+bool operator==(const Decimal128& left, const Decimal128& right) {
+  return left.high_bits() == right.high_bits() && left.low_bits() == right.low_bits();
+}
+
+bool operator!=(const Decimal128& left, const Decimal128& right) {
+  return !operator==(left, right);
+}
+
+bool operator<(const Decimal128& left, const Decimal128& right) {
+  return left.high_bits() < right.high_bits() ||
+         (left.high_bits() == right.high_bits() && left.low_bits() < right.low_bits());
+}
+
+bool operator<=(const Decimal128& left, const Decimal128& right) {
+  return !operator>(left, right);
+}
+
+bool operator>(const Decimal128& left, const Decimal128& right) {
+  return operator<(right, left);
+}
+
+bool operator>=(const Decimal128& left, const Decimal128& right) {
+  return !operator<(left, right);
+}
+
+Decimal128 operator-(const Decimal128& operand) {
+  Decimal128 result(operand.high_bits(), operand.low_bits());
+  return result.Negate();
+}
+
+Decimal128 operator+(const Decimal128& left, const Decimal128& right) {
+  Decimal128 result(left.high_bits(), left.low_bits());
+  result += right;
+  return result;
+}
+
+Decimal128 operator-(const Decimal128& left, const Decimal128& right) {
+  Decimal128 result(left.high_bits(), left.low_bits());
+  result -= right;
+  return result;
+}
+
+Decimal128 operator*(const Decimal128& left, const Decimal128& right) {
+  Decimal128 result(left.high_bits(), left.low_bits());
+  result *= right;
+  return result;
+}
+
+Decimal128 operator/(const Decimal128& left, const Decimal128& right) {
+  Decimal128 remainder;
+  Decimal128 result;
+  DCHECK(left.Divide(right, &result, &remainder).ok());
+  return result;
+}
+
+Decimal128 operator%(const Decimal128& left, const Decimal128& right) {
+  Decimal128 remainder;
+  Decimal128 result;
+  DCHECK(left.Divide(right, &result, &remainder).ok());
+  return remainder;
 }
 
-}  // namespace DecimalUtil
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 2e948774d4761..ff30d441a38d0 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -18,25 +18,125 @@
 #ifndef ARROW_DECIMAL_H
 #define ARROW_DECIMAL_H
 
+#include <array>
+#include <cstdint>
 #include <string>
+#include <type_traits>
 
 #include "arrow/status.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
-class Int128;
+/// Represents a signed 128-bit integer in two's complement.
+/// Calculations wrap around and overflow is ignored.
+///
+/// For a discussion of the algorithms, look at Knuth's volume 2,
+/// Semi-numerical Algorithms section 4.3.1.
+///
+/// Adapted from the Apache ORC C++ implementation
+class ARROW_EXPORT Decimal128 {
+ public:
+  /// \brief Create an Decimal128 from the two's complement representation.
+  constexpr Decimal128(int64_t high, uint64_t low) : high_bits_(high), low_bits_(low) {}
 
-namespace DecimalUtil {
+  /// \brief Empty constructor creates an Decimal128 with a value of 0.
+  constexpr Decimal128() : Decimal128(0, 0) {}
 
-ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
-                                  int8_t sign, Int128* out);
+  /// \brief Convert any integer value into an Decimal128.
+  template <typename T,
+            typename = typename std::enable_if<std::is_integral<T>::value, T>::type>
+  constexpr Decimal128(T value)
+      : Decimal128(static_cast<int64_t>(value) >= 0 ? 0 : -1,
+                   static_cast<uint64_t>(value)) {}
 
-ARROW_EXPORT Status FromString(const std::string& string, Int128* out,
-                               int* precision = nullptr, int* scale = nullptr);
+  /// \brief Parse the number from a base 10 string representation.
+  explicit Decimal128(const std::string& value);
 
-ARROW_EXPORT std::string ToString(const Int128& decimal_value, int precision, int scale);
+  /// \brief Create an Decimal128 from an array of bytes
+  explicit Decimal128(const uint8_t* bytes);
+
+  /// \brief Negate the current value
+  Decimal128& Negate();
+
+  /// \brief Add a number to this one. The result is truncated to 128 bits.
+  Decimal128& operator+=(const Decimal128& right);
+
+  /// \brief Subtract a number from this one. The result is truncated to 128 bits.
+  Decimal128& operator-=(const Decimal128& right);
+
+  /// \brief Multiply this number by another number. The result is truncated to 128 bits.
+  Decimal128& operator*=(const Decimal128& right);
+
+  /// Divide this number by right and return the result. This operation is
+  /// not destructive.
+  /// The answer rounds to zero. Signs work like:
+  ///   21 /  5 ->  4,  1
+  ///  -21 /  5 -> -4, -1
+  ///   21 / -5 -> -4,  1
+  ///  -21 / -5 ->  4, -1
+  /// @param right the number to divide by
+  /// @param remainder the remainder after the division
+  Status Divide(const Decimal128& divisor, Decimal128* result,
+                Decimal128* remainder) const;
+
+  /// \brief In-place division.
+  Decimal128& operator/=(const Decimal128& right);
+
+  /// \brief Cast the value to char. This is used when converting the value a string.
+  explicit operator char() const;
+
+  /// \brief Bitwise or between two Decimal128.
+  Decimal128& operator|=(const Decimal128& right);
+
+  /// \brief Bitwise and between two Decimal128.
+  Decimal128& operator&=(const Decimal128& right);
+
+  /// \brief Shift left by the given number of bits.
+  Decimal128& operator<<=(uint32_t bits);
+
+  /// \brief Shift right by the given number of bits. Negative values will
+  Decimal128& operator>>=(uint32_t bits);
+
+  /// \brief Get the high bits of the two's complement representation of the number.
+  int64_t high_bits() const { return high_bits_; }
+
+  /// \brief Get the low bits of the two's complement representation of the number.
+  uint64_t low_bits() const { return low_bits_; }
+
+  /// \brief Put the raw bytes of the value into a pointer to uint8_t.
+  Status ToBytes(std::array<uint8_t, 16>* out) const;
+
+  /// \brief Convert the Decimal128 value to a base 10 decimal string with the given
+  /// precision
+  /// and scale.
+  std::string ToString(int precision, int scale) const;
+
+  /// \brief Convert a decimal string to an Decimal128 value, optionally including
+  /// precision
+  /// and scale if they're passed in and not null.
+  static Status FromString(const std::string& s, Decimal128* out,
+                           int* precision = nullptr, int* scale = nullptr);
+
+ private:
+  int64_t high_bits_;
+  uint64_t low_bits_;
+};
+
+ARROW_EXPORT bool operator==(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT bool operator!=(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT bool operator<(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT bool operator<=(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT bool operator>(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT bool operator>=(const Decimal128& left, const Decimal128& right);
+
+ARROW_EXPORT Decimal128 operator-(const Decimal128& operand);
+ARROW_EXPORT Decimal128 operator+(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT Decimal128 operator-(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT Decimal128 operator*(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT Decimal128 operator/(const Decimal128& left, const Decimal128& right);
+ARROW_EXPORT Decimal128 operator%(const Decimal128& left, const Decimal128& right);
 
-}  // namespace DecimalUtil
 }  // namespace arrow
 
-#endif  // ARROW_DECIMAL_H
+#endif  //  ARROW_DECIMAL_H
diff --git a/cpp/src/arrow/util/int128.cc b/cpp/src/arrow/util/int128.cc
deleted file mode 100644
index d5659a8633ca5..0000000000000
--- a/cpp/src/arrow/util/int128.cc
+++ /dev/null
@@ -1,527 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <algorithm>
-#include <cmath>
-#include <cstring>
-#include <limits>
-
-#ifdef _MSC_VER
-#include <intrin.h>
-#pragma intrinsic(_BitScanReverse)
-#endif
-
-#include "arrow/util/int128.h"
-#include "arrow/util/logging.h"
-
-namespace arrow {
-
-static constexpr uint64_t kIntMask = 0xFFFFFFFF;
-static constexpr auto kCarryBit = static_cast<uint64_t>(1) << static_cast<uint64_t>(32);
-
-Int128::Int128(const std::string& str) : Int128() {
-  const size_t length = str.length();
-
-  if (length > 0) {
-    bool is_negative = str[0] == '-';
-    auto posn = static_cast<size_t>(is_negative);
-
-    while (posn < length) {
-      const size_t group = std::min(static_cast<size_t>(18), length - posn);
-      const auto chunk = static_cast<int64_t>(std::stoll(str.substr(posn, group)));
-      const auto multiple =
-          static_cast<int64_t>(std::pow(10.0, static_cast<double>(group)));
-
-      *this *= multiple;
-      *this += chunk;
-
-      posn += group;
-    }
-
-    if (is_negative) {
-      Negate();
-    }
-  }
-}
-
-Int128::Int128(const uint8_t* bytes)
-    : Int128(reinterpret_cast<const int64_t*>(bytes)[0],
-             reinterpret_cast<const uint64_t*>(bytes)[1]) {}
-
-Status Int128::ToBytes(std::array<uint8_t, 16>* out) const {
-  if (out == nullptr) {
-    return Status::Invalid("Cannot fill nullptr of bytes from Int128");
-  }
-
-  const uint64_t raw[] = {static_cast<uint64_t>(high_bits_), low_bits_};
-  const auto* raw_data = reinterpret_cast<const uint8_t*>(raw);
-  std::copy(raw_data, raw_data + out->size(), out->begin());
-  return Status::OK();
-}
-
-Int128& Int128::Negate() {
-  low_bits_ = ~low_bits_ + 1;
-  high_bits_ = ~high_bits_;
-  if (low_bits_ == 0) {
-    ++high_bits_;
-  }
-  return *this;
-}
-
-Int128& Int128::operator+=(const Int128& right) {
-  const uint64_t sum = low_bits_ + right.low_bits_;
-  high_bits_ += right.high_bits_;
-  if (sum < low_bits_) {
-    ++high_bits_;
-  }
-  low_bits_ = sum;
-  return *this;
-}
-
-Int128& Int128::operator-=(const Int128& right) {
-  const uint64_t diff = low_bits_ - right.low_bits_;
-  high_bits_ -= right.high_bits_;
-  if (diff > low_bits_) {
-    --high_bits_;
-  }
-  low_bits_ = diff;
-  return *this;
-}
-
-Int128& Int128::operator/=(const Int128& right) {
-  Int128 remainder;
-  DCHECK(Divide(right, this, &remainder).ok());
-  return *this;
-}
-
-Int128::operator char() const {
-  DCHECK(high_bits_ == 0 || high_bits_ == -1)
-      << "Trying to cast an Int128 greater than the value range of a "
-         "char. high_bits_ must be equal to 0 or -1, got: "
-      << high_bits_;
-  DCHECK_LE(low_bits_, std::numeric_limits<char>::max())
-      << "low_bits_ too large for C type char, got: " << low_bits_;
-  return static_cast<char>(low_bits_);
-}
-
-Int128& Int128::operator|=(const Int128& right) {
-  low_bits_ |= right.low_bits_;
-  high_bits_ |= right.high_bits_;
-  return *this;
-}
-
-Int128& Int128::operator&=(const Int128& right) {
-  low_bits_ &= right.low_bits_;
-  high_bits_ &= right.high_bits_;
-  return *this;
-}
-
-Int128& Int128::operator<<=(uint32_t bits) {
-  if (bits != 0) {
-    if (bits < 64) {
-      high_bits_ <<= bits;
-      high_bits_ |= (low_bits_ >> (64 - bits));
-      low_bits_ <<= bits;
-    } else if (bits < 128) {
-      high_bits_ = static_cast<int64_t>(low_bits_) << (bits - 64);
-      low_bits_ = 0;
-    } else {
-      high_bits_ = 0;
-      low_bits_ = 0;
-    }
-  }
-  return *this;
-}
-
-Int128& Int128::operator>>=(uint32_t bits) {
-  if (bits != 0) {
-    if (bits < 64) {
-      low_bits_ >>= bits;
-      low_bits_ |= static_cast<uint64_t>(high_bits_ << (64 - bits));
-      high_bits_ = static_cast<int64_t>(static_cast<uint64_t>(high_bits_) >> bits);
-    } else if (bits < 128) {
-      low_bits_ = static_cast<uint64_t>(high_bits_ >> (bits - 64));
-      high_bits_ = static_cast<int64_t>(high_bits_ >= 0L ? 0L : -1L);
-    } else {
-      high_bits_ = static_cast<int64_t>(high_bits_ >= 0L ? 0L : -1L);
-      low_bits_ = static_cast<uint64_t>(high_bits_);
-    }
-  }
-  return *this;
-}
-
-Int128& Int128::operator*=(const Int128& right) {
-  // Break the left and right numbers into 32 bit chunks
-  // so that we can multiply them without overflow.
-  const uint64_t L0 = static_cast<uint64_t>(high_bits_) >> 32;
-  const uint64_t L1 = static_cast<uint64_t>(high_bits_) & kIntMask;
-  const uint64_t L2 = low_bits_ >> 32;
-  const uint64_t L3 = low_bits_ & kIntMask;
-
-  const uint64_t R0 = static_cast<uint64_t>(right.high_bits_) >> 32;
-  const uint64_t R1 = static_cast<uint64_t>(right.high_bits_) & kIntMask;
-  const uint64_t R2 = right.low_bits_ >> 32;
-  const uint64_t R3 = right.low_bits_ & kIntMask;
-
-  uint64_t product = L3 * R3;
-  low_bits_ = product & kIntMask;
-
-  uint64_t sum = product >> 32;
-
-  product = L2 * R3;
-  sum += product;
-
-  product = L3 * R2;
-  sum += product;
-
-  low_bits_ += sum << 32;
-
-  high_bits_ = static_cast<int64_t>(sum < product ? kCarryBit : 0);
-  if (sum < product) {
-    high_bits_ += kCarryBit;
-  }
-
-  high_bits_ += static_cast<int64_t>(sum >> 32);
-  high_bits_ += L1 * R3 + L2 * R2 + L3 * R1;
-  high_bits_ += (L0 * R3 + L1 * R2 + L2 * R1 + L3 * R0) << 32;
-  return *this;
-}
-
-/// Expands the given value into an array of ints so that we can work on
-/// it. The array will be converted to an absolute value and the wasNegative
-/// flag will be set appropriately. The array will remove leading zeros from
-/// the value.
-/// \param array an array of length 4 to set with the value
-/// \param was_negative a flag for whether the value was original negative
-/// \result the output length of the array
-static int64_t FillInArray(const Int128& value, uint32_t* array, bool& was_negative) {
-  uint64_t high;
-  uint64_t low;
-  const int64_t highbits = value.high_bits();
-  const uint64_t lowbits = value.low_bits();
-
-  if (highbits < 0) {
-    low = ~lowbits + 1;
-    high = static_cast<uint64_t>(~highbits);
-    if (low == 0) {
-      ++high;
-    }
-    was_negative = true;
-  } else {
-    low = lowbits;
-    high = static_cast<uint64_t>(highbits);
-    was_negative = false;
-  }
-
-  if (high != 0) {
-    if (high > std::numeric_limits<uint32_t>::max()) {
-      array[0] = static_cast<uint32_t>(high >> 32);
-      array[1] = static_cast<uint32_t>(high);
-      array[2] = static_cast<uint32_t>(low >> 32);
-      array[3] = static_cast<uint32_t>(low);
-      return 4;
-    }
-
-    array[0] = static_cast<uint32_t>(high);
-    array[1] = static_cast<uint32_t>(low >> 32);
-    array[2] = static_cast<uint32_t>(low);
-    return 3;
-  }
-
-  if (low >= std::numeric_limits<uint32_t>::max()) {
-    array[0] = static_cast<uint32_t>(low >> 32);
-    array[1] = static_cast<uint32_t>(low);
-    return 2;
-  }
-
-  if (low == 0) {
-    return 0;
-  }
-
-  array[0] = static_cast<uint32_t>(low);
-  return 1;
-}
-
-/// \brief Find last set bit in a 32 bit integer. Bit 1 is the LSB and bit 32 is the MSB.
-static int64_t FindLastSetBit(uint32_t value) {
-#if defined(__clang__) || defined(__GNUC__)
-  // Count leading zeros
-  return __builtin_clz(value) + 1;
-#elif defined(_MSC_VER)
-  unsigned long index;                                         // NOLINT
-  _BitScanReverse(&index, static_cast<unsigned long>(value));  // NOLINT
-  return static_cast<int64_t>(index + 1UL);
-#endif
-}
-
-/// Shift the number in the array left by bits positions.
-/// \param array the number to shift, must have length elements
-/// \param length the number of entries in the array
-/// \param bits the number of bits to shift (0 <= bits < 32)
-static void ShiftArrayLeft(uint32_t* array, int64_t length, int64_t bits) {
-  if (length > 0 && bits != 0) {
-    for (int64_t i = 0; i < length - 1; ++i) {
-      array[i] = (array[i] << bits) | (array[i + 1] >> (32 - bits));
-    }
-    array[length - 1] <<= bits;
-  }
-}
-
-/// Shift the number in the array right by bits positions.
-/// \param array the number to shift, must have length elements
-/// \param length the number of entries in the array
-/// \param bits the number of bits to shift (0 <= bits < 32)
-static void ShiftArrayRight(uint32_t* array, int64_t length, int64_t bits) {
-  if (length > 0 && bits != 0) {
-    for (int64_t i = length - 1; i > 0; --i) {
-      array[i] = (array[i] >> bits) | (array[i - 1] << (32 - bits));
-    }
-    array[0] >>= bits;
-  }
-}
-
-/// \brief Fix the signs of the result and remainder at the end of the division based on
-/// the signs of the dividend and divisor.
-static void FixDivisionSigns(Int128* result, Int128* remainder,
-                             bool dividend_was_negative, bool divisor_was_negative) {
-  if (dividend_was_negative != divisor_was_negative) {
-    result->Negate();
-  }
-
-  if (dividend_was_negative) {
-    remainder->Negate();
-  }
-}
-
-/// \brief Build a Int128 from a list of ints.
-static Status BuildFromArray(Int128* value, uint32_t* array, int64_t length) {
-  switch (length) {
-    case 0:
-      *value = {static_cast<int64_t>(0)};
-      break;
-    case 1:
-      *value = {static_cast<int64_t>(array[0])};
-      break;
-    case 2:
-      *value = {static_cast<int64_t>(0),
-                (static_cast<uint64_t>(array[0]) << 32) + array[1]};
-      break;
-    case 3:
-      *value = {static_cast<int64_t>(array[0]),
-                (static_cast<uint64_t>(array[1]) << 32) + array[2]};
-      break;
-    case 4:
-      *value = {(static_cast<int64_t>(array[0]) << 32) + array[1],
-                (static_cast<uint64_t>(array[2]) << 32) + array[3]};
-      break;
-    case 5:
-      if (array[0] != 0) {
-        return Status::Invalid("Can't build Int128 with 5 ints.");
-      }
-      *value = {(static_cast<int64_t>(array[1]) << 32) + array[2],
-                (static_cast<uint64_t>(array[3]) << 32) + array[4]};
-      break;
-    default:
-      return Status::Invalid("Unsupported length for building Int128");
-  }
-
-  return Status::OK();
-}
-
-/// \brief Do a division where the divisor fits into a single 32 bit value.
-static Status SingleDivide(const uint32_t* dividend, int64_t dividend_length,
-                           uint32_t divisor, Int128* remainder,
-                           bool dividend_was_negative, bool divisor_was_negative,
-                           Int128* result) {
-  uint64_t r = 0;
-  uint32_t result_array[5];
-  for (int64_t j = 0; j < dividend_length; j++) {
-    r <<= 32;
-    r += dividend[j];
-    result_array[j] = static_cast<uint32_t>(r / divisor);
-    r %= divisor;
-  }
-  RETURN_NOT_OK(BuildFromArray(result, result_array, dividend_length));
-  *remainder = static_cast<int64_t>(r);
-  FixDivisionSigns(result, remainder, dividend_was_negative, divisor_was_negative);
-  return Status::OK();
-}
-
-Status Int128::Divide(const Int128& divisor, Int128* result, Int128* remainder) const {
-  // Split the dividend and divisor into integer pieces so that we can
-  // work on them.
-  uint32_t dividend_array[5];
-  uint32_t divisor_array[4];
-  bool dividend_was_negative;
-  bool divisor_was_negative;
-  // leave an extra zero before the dividend
-  dividend_array[0] = 0;
-  int64_t dividend_length =
-      FillInArray(*this, dividend_array + 1, dividend_was_negative) + 1;
-  int64_t divisor_length = FillInArray(divisor, divisor_array, divisor_was_negative);
-
-  // Handle some of the easy cases.
-  if (dividend_length <= divisor_length) {
-    *remainder = *this;
-    *result = 0;
-    return Status::OK();
-  }
-
-  if (divisor_length == 0) {
-    return Status::Invalid("Division by 0 in Int128");
-  }
-
-  if (divisor_length == 1) {
-    return SingleDivide(dividend_array, dividend_length, divisor_array[0], remainder,
-                        dividend_was_negative, divisor_was_negative, result);
-  }
-
-  int64_t result_length = dividend_length - divisor_length;
-  uint32_t result_array[4];
-
-  // Normalize by shifting both by a multiple of 2 so that
-  // the digit guessing is better. The requirement is that
-  // divisor_array[0] is greater than 2**31.
-  int64_t normalize_bits = 32 - FindLastSetBit(divisor_array[0]);
-  ShiftArrayLeft(divisor_array, divisor_length, normalize_bits);
-  ShiftArrayLeft(dividend_array, dividend_length, normalize_bits);
-
-  // compute each digit in the result
-  for (int64_t j = 0; j < result_length; ++j) {
-    // Guess the next digit. At worst it is two too large
-    uint32_t guess = std::numeric_limits<uint32_t>::max();
-    auto high_dividend =
-        static_cast<uint64_t>(dividend_array[j]) << 32 | dividend_array[j + 1];
-    if (dividend_array[j] != divisor_array[0]) {
-      guess = static_cast<uint32_t>(high_dividend / divisor_array[0]);
-    }
-
-    // catch all of the cases where guess is two too large and most of the
-    // cases where it is one too large
-    auto rhat = static_cast<uint32_t>(high_dividend -
-                                      guess * static_cast<uint64_t>(divisor_array[0]));
-    while (static_cast<uint64_t>(divisor_array[1]) * guess >
-           (static_cast<uint64_t>(rhat) << 32) + dividend_array[j + 2]) {
-      --guess;
-      rhat += divisor_array[0];
-      if (static_cast<uint64_t>(rhat) < divisor_array[0]) {
-        break;
-      }
-    }
-
-    // subtract off the guess * divisor from the dividend
-    uint64_t mult = 0;
-    for (int64_t i = divisor_length - 1; i >= 0; --i) {
-      mult += static_cast<uint64_t>(guess) * divisor_array[i];
-      uint32_t prev = dividend_array[j + i + 1];
-      dividend_array[j + i + 1] -= static_cast<uint32_t>(mult);
-      mult >>= 32;
-      if (dividend_array[j + i + 1] > prev) {
-        ++mult;
-      }
-    }
-    uint32_t prev = dividend_array[j];
-    dividend_array[j] -= static_cast<uint32_t>(mult);
-
-    // if guess was too big, we add back divisor
-    if (dividend_array[j] > prev) {
-      --guess;
-
-      uint32_t carry = 0;
-      for (int64_t i = divisor_length - 1; i >= 0; --i) {
-        uint64_t sum =
-            static_cast<uint64_t>(divisor_array[i]) + dividend_array[j + i + 1] + carry;
-        dividend_array[j + i + 1] = static_cast<uint32_t>(sum);
-        carry = static_cast<uint32_t>(sum >> 32);
-      }
-      dividend_array[j] += carry;
-    }
-
-    result_array[j] = guess;
-  }
-
-  // denormalize the remainder
-  ShiftArrayRight(dividend_array, dividend_length, normalize_bits);
-
-  // return result and remainder
-  RETURN_NOT_OK(BuildFromArray(result, result_array, result_length));
-  RETURN_NOT_OK(BuildFromArray(remainder, dividend_array, dividend_length));
-  FixDivisionSigns(result, remainder, dividend_was_negative, divisor_was_negative);
-  return Status::OK();
-}
-
-bool operator==(const Int128& left, const Int128& right) {
-  return left.high_bits() == right.high_bits() && left.low_bits() == right.low_bits();
-}
-
-bool operator!=(const Int128& left, const Int128& right) {
-  return !operator==(left, right);
-}
-
-bool operator<(const Int128& left, const Int128& right) {
-  return left.high_bits() < right.high_bits() ||
-         (left.high_bits() == right.high_bits() && left.low_bits() < right.low_bits());
-}
-
-bool operator<=(const Int128& left, const Int128& right) {
-  return !operator>(left, right);
-}
-
-bool operator>(const Int128& left, const Int128& right) { return operator<(right, left); }
-
-bool operator>=(const Int128& left, const Int128& right) {
-  return !operator<(left, right);
-}
-
-Int128 operator-(const Int128& operand) {
-  Int128 result(operand.high_bits(), operand.low_bits());
-  return result.Negate();
-}
-
-Int128 operator+(const Int128& left, const Int128& right) {
-  Int128 result(left.high_bits(), left.low_bits());
-  result += right;
-  return result;
-}
-
-Int128 operator-(const Int128& left, const Int128& right) {
-  Int128 result(left.high_bits(), left.low_bits());
-  result -= right;
-  return result;
-}
-
-Int128 operator*(const Int128& left, const Int128& right) {
-  Int128 result(left.high_bits(), left.low_bits());
-  result *= right;
-  return result;
-}
-
-Int128 operator/(const Int128& left, const Int128& right) {
-  Int128 remainder;
-  Int128 result;
-  DCHECK(left.Divide(right, &result, &remainder).ok());
-  return result;
-}
-
-Int128 operator%(const Int128& left, const Int128& right) {
-  Int128 remainder;
-  Int128 result;
-  DCHECK(left.Divide(right, &result, &remainder).ok());
-  return remainder;
-}
-
-}  // namespace arrow
diff --git a/cpp/src/arrow/util/int128.h b/cpp/src/arrow/util/int128.h
deleted file mode 100644
index dd2ec52f23ddd..0000000000000
--- a/cpp/src/arrow/util/int128.h
+++ /dev/null
@@ -1,129 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_INT128_H
-#define ARROW_INT128_H
-
-#include <array>
-#include <cstdint>
-#include <string>
-#include <type_traits>
-
-#include "arrow/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-/// Represents a signed 128-bit integer in two's complement.
-/// Calculations wrap around and overflow is ignored.
-///
-/// For a discussion of the algorithms, look at Knuth's volume 2,
-/// Semi-numerical Algorithms section 4.3.1.
-///
-/// Adapted from the Apache ORC C++ implementation
-class ARROW_EXPORT Int128 {
- public:
-  /// \brief Create an Int128 from the two's complement representation.
-  constexpr Int128(int64_t high, uint64_t low) : high_bits_(high), low_bits_(low) {}
-
-  /// \brief Empty constructor creates an Int128 with a value of 0.
-  constexpr Int128() : Int128(0, 0) {}
-
-  /// \brief Convert any integer value into an Int128.
-  template <typename T,
-            typename = typename std::enable_if<std::is_integral<T>::value, T>::type>
-  constexpr Int128(T value)
-      : Int128(static_cast<int64_t>(value) >= 0 ? 0 : -1, static_cast<uint64_t>(value)) {}
-
-  /// \brief Parse the number from a base 10 string representation.
-  explicit Int128(const std::string& value);
-
-  /// \brief Create an Int128 from an array of bytes
-  explicit Int128(const uint8_t* bytes);
-
-  /// \brief Negate the current value
-  Int128& Negate();
-
-  /// \brief Add a number to this one. The result is truncated to 128 bits.
-  Int128& operator+=(const Int128& right);
-
-  /// \brief Subtract a number from this one. The result is truncated to 128 bits.
-  Int128& operator-=(const Int128& right);
-
-  /// \brief Multiply this number by another number. The result is truncated to 128 bits.
-  Int128& operator*=(const Int128& right);
-
-  /// Divide this number by right and return the result. This operation is
-  /// not destructive.
-  /// The answer rounds to zero. Signs work like:
-  ///   21 /  5 ->  4,  1
-  ///  -21 /  5 -> -4, -1
-  ///   21 / -5 -> -4,  1
-  ///  -21 / -5 ->  4, -1
-  /// @param right the number to divide by
-  /// @param remainder the remainder after the division
-  Status Divide(const Int128& divisor, Int128* result, Int128* remainder) const;
-
-  /// \brief In-place division.
-  Int128& operator/=(const Int128& right);
-
-  /// \brief Cast the value to char. This is used when converting the value a string.
-  explicit operator char() const;
-
-  /// \brief Bitwise or between two Int128.
-  Int128& operator|=(const Int128& right);
-
-  /// \brief Bitwise and between two Int128.
-  Int128& operator&=(const Int128& right);
-
-  /// \brief Shift left by the given number of bits.
-  Int128& operator<<=(uint32_t bits);
-
-  /// \brief Shift right by the given number of bits. Negative values will
-  Int128& operator>>=(uint32_t bits);
-
-  /// \brief Get the high bits of the two's complement representation of the number.
-  int64_t high_bits() const { return high_bits_; }
-
-  /// \brief Get the low bits of the two's complement representation of the number.
-  uint64_t low_bits() const { return low_bits_; }
-
-  /// \brief Put the raw bytes of the value into a pointer to uint8_t.
-  Status ToBytes(std::array<uint8_t, 16>* out) const;
-
- private:
-  int64_t high_bits_;
-  uint64_t low_bits_;
-};
-
-ARROW_EXPORT bool operator==(const Int128& left, const Int128& right);
-ARROW_EXPORT bool operator!=(const Int128& left, const Int128& right);
-ARROW_EXPORT bool operator<(const Int128& left, const Int128& right);
-ARROW_EXPORT bool operator<=(const Int128& left, const Int128& right);
-ARROW_EXPORT bool operator>(const Int128& left, const Int128& right);
-ARROW_EXPORT bool operator>=(const Int128& left, const Int128& right);
-
-ARROW_EXPORT Int128 operator-(const Int128& operand);
-ARROW_EXPORT Int128 operator+(const Int128& left, const Int128& right);
-ARROW_EXPORT Int128 operator-(const Int128& left, const Int128& right);
-ARROW_EXPORT Int128 operator*(const Int128& left, const Int128& right);
-ARROW_EXPORT Int128 operator/(const Int128& left, const Int128& right);
-ARROW_EXPORT Int128 operator%(const Int128& left, const Int128& right);
-
-}  // namespace arrow
-
-#endif  //  ARROW_INT128_H

From 18388d65000745b60bee71f14309b8c2e2f2a445 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 11 Sep 2017 21:50:33 -0400
Subject: [PATCH 1045/1644] ARROW-1331: [JAVA] include package statement

TestSplitAndTransfer was merged as part of https://github.com/apache/arrow/pull/955
However, the class did not have the package statement.

This small patch fixes that.

Author: siddharth <siddharth@dremio.com>

Closes #1088 from siddharthteotia/ARROW-1331-fix and squashes the following commits:

78882b52 [siddharth] ARROW-1331: include package statement
---
 .../test/java/org/apache/arrow/vector/TestSplitAndTransfer.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
index 4ff42b5b0e0aa..66e5375e3bd0d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+package org.apache.arrow.vector;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;

From 4c008c2b9481ae0bf05f29d30539edd7266779b0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Sep 2017 09:10:30 +0200
Subject: [PATCH 1046/1644] ARROW-1511: [C++] Promote ArrayData, MakeArray to
 public API, deprecate MakePrimitiveArray

Since `ArrayData` will become more exposed to users with the advent of `arrow::compute`, it should be a part of the public API.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1087 from wesm/ARROW-1511 and squashes the following commits:

67e2996 [Wes McKinney] Remove internal:: symbols for ArrayData, MakeArray
da7d9e9 [Wes McKinney] Add docstring for MakeArray
eee4c2f [Wes McKinney] Promote ArrayData, MakeArray to public API, deprecate MakePrimitiveArray
---
 cpp/src/arrow/array.cc                  | 49 +++++++-------
 cpp/src/arrow/array.h                   | 87 +++++++++++++------------
 cpp/src/arrow/builder.cc                |  1 -
 cpp/src/arrow/builder.h                 |  2 +-
 cpp/src/arrow/compute/cast.cc           |  2 +-
 cpp/src/arrow/compute/compute-test.cc   |  3 -
 cpp/src/arrow/compute/kernel.h          |  3 -
 cpp/src/arrow/ipc/feather-test.cc       |  4 +-
 cpp/src/arrow/ipc/feather.cc            |  5 +-
 cpp/src/arrow/ipc/reader.cc             | 18 ++---
 cpp/src/arrow/python/pandas_to_arrow.cc |  4 --
 cpp/src/arrow/table.cc                  |  4 +-
 cpp/src/arrow/table.h                   |  8 +--
 13 files changed, 94 insertions(+), 96 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index a1d3bed255d1d..80188a14a9588 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -36,8 +36,6 @@
 
 namespace arrow {
 
-using internal::ArrayData;
-
 // ----------------------------------------------------------------------
 // Base array class
 
@@ -158,7 +156,7 @@ const uint8_t* PrimitiveArray::raw_values() const {
 }
 
 template <typename T>
-NumericArray<T>::NumericArray(const std::shared_ptr<internal::ArrayData>& data)
+NumericArray<T>::NumericArray(const std::shared_ptr<ArrayData>& data)
     : PrimitiveArray(data) {
   DCHECK_EQ(data->type->id(), T::type_id);
 }
@@ -166,7 +164,7 @@ NumericArray<T>::NumericArray(const std::shared_ptr<internal::ArrayData>& data)
 // ----------------------------------------------------------------------
 // BooleanArray
 
-BooleanArray::BooleanArray(const std::shared_ptr<internal::ArrayData>& data)
+BooleanArray::BooleanArray(const std::shared_ptr<ArrayData>& data)
     : PrimitiveArray(data) {
   DCHECK_EQ(data->type->id(), Type::BOOL);
 }
@@ -214,9 +212,9 @@ Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPo
                           static_cast<const Int32Array&>(offsets).values()};
 
   auto list_type = list(values.type());
-  auto internal_data = std::make_shared<internal::ArrayData>(
-      list_type, offsets.length() - 1, std::move(buffers), offsets.null_count(),
-      offsets.offset());
+  auto internal_data =
+      std::make_shared<ArrayData>(list_type, offsets.length() - 1, std::move(buffers),
+                                  offsets.null_count(), offsets.offset());
   internal_data->child_data.push_back(values.data());
 
   *out = std::make_shared<ListArray>(internal_data);
@@ -229,7 +227,7 @@ void ListArray::SetData(const std::shared_ptr<ArrayData>& data) {
   raw_value_offsets_ = value_offsets == nullptr
                            ? nullptr
                            : reinterpret_cast<const int32_t*>(value_offsets->data());
-  DCHECK(internal::MakeArray(data_->child_data[0], &values_).ok());
+  DCHECK(MakeArray(data_->child_data[0], &values_).ok());
 }
 
 std::shared_ptr<DataType> ListArray::value_type() const {
@@ -244,7 +242,7 @@ std::shared_ptr<Array> ListArray::values() const { return values_; }
 static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
 static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
 
-BinaryArray::BinaryArray(const std::shared_ptr<internal::ArrayData>& data) {
+BinaryArray::BinaryArray(const std::shared_ptr<ArrayData>& data) {
   DCHECK_EQ(data->type->id(), Type::BINARY);
   SetData(data);
 }
@@ -276,7 +274,7 @@ BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
       std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
 }
 
-StringArray::StringArray(const std::shared_ptr<internal::ArrayData>& data) {
+StringArray::StringArray(const std::shared_ptr<ArrayData>& data) {
   DCHECK_EQ(data->type->id(), Type::STRING);
   SetData(data);
 }
@@ -291,8 +289,7 @@ StringArray::StringArray(int64_t length, const std::shared_ptr<Buffer>& value_of
 // ----------------------------------------------------------------------
 // Fixed width binary
 
-FixedSizeBinaryArray::FixedSizeBinaryArray(
-    const std::shared_ptr<internal::ArrayData>& data) {
+FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<ArrayData>& data) {
   SetData(data);
 }
 
@@ -311,7 +308,7 @@ const uint8_t* FixedSizeBinaryArray::GetValue(int64_t i) const {
 // ----------------------------------------------------------------------
 // Decimal
 
-DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data)
+DecimalArray::DecimalArray(const std::shared_ptr<ArrayData>& data)
     : FixedSizeBinaryArray(data) {
   DCHECK_EQ(data->type->id(), Type::DECIMAL);
 }
@@ -346,7 +343,7 @@ StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
 
 std::shared_ptr<Array> StructArray::field(int i) const {
   if (!boxed_fields_[i]) {
-    DCHECK(internal::MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
+    DCHECK(MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
   }
   return boxed_fields_[i];
 }
@@ -389,7 +386,7 @@ UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
 
 std::shared_ptr<Array> UnionArray::child(int i) const {
   if (!boxed_fields_[i]) {
-    DCHECK(internal::MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
+    DCHECK(MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
   }
   return boxed_fields_[i];
 }
@@ -418,7 +415,7 @@ void DictionaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
   auto indices_data = data_->ShallowCopy();
   indices_data->type = dict_type_->index_type();
   std::shared_ptr<Array> result;
-  DCHECK(internal::MakeArray(indices_data, &indices_).ok());
+  DCHECK(MakeArray(indices_data, &indices_).ok());
 }
 
 std::shared_ptr<Array> DictionaryArray::indices() const { return indices_; }
@@ -598,21 +595,24 @@ class ArrayDataWrapper {
   std::shared_ptr<Array>* out_;
 };
 
+}  // namespace internal
+
+// Remove enclosing namespace after 0.7.0
 Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out) {
-  ArrayDataWrapper wrapper_visitor(data, out);
+  internal::ArrayDataWrapper wrapper_visitor(data, out);
   return VisitTypeInline(*data->type, &wrapper_visitor);
 }
 
-}  // namespace internal
-
+#ifndef ARROW_NO_DEPRECATED_API
+// \deprecated Since 0.7.0
 Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
                           const std::shared_ptr<Buffer>& data,
                           const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                           int64_t offset, std::shared_ptr<Array>* out) {
   BufferVector buffers = {null_bitmap, data};
-  auto internal_data = std::make_shared<internal::ArrayData>(
-      type, length, std::move(buffers), null_count, offset);
-  return internal::MakeArray(internal_data, out);
+  auto internal_data =
+      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
+  return MakeArray(internal_data, out);
 }
 
 Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
@@ -620,9 +620,10 @@ Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
                           int64_t length, int64_t null_count, int64_t offset,
                           std::shared_ptr<Array>* out) {
   auto internal_data =
-      std::make_shared<internal::ArrayData>(type, length, buffers, null_count, offset);
-  return internal::MakeArray(internal_data, out);
+      std::make_shared<ArrayData>(type, length, buffers, null_count, offset);
+  return MakeArray(internal_data, out);
 }
+#endif
 
 // ----------------------------------------------------------------------
 // Instantiate templates
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 994270db32387..e801b3586df30 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -52,9 +52,8 @@ struct Decimal;
 // ----------------------------------------------------------------------
 // Generic array data container
 
-namespace internal {
-
-/// \brief Mutable internal container for generic Arrow array data
+/// \class ArrayData
+/// \brief Mutable container for generic Arrow array data
 ///
 /// This data structure is a self-contained representation of the memory and
 /// metadata inside an Arrow array data structure (called vectors in Java). The
@@ -145,11 +144,13 @@ struct ARROW_EXPORT ArrayData {
   std::vector<std::shared_ptr<ArrayData>> child_data;
 };
 
+/// \brief Create a strongly-typed Array instance from generic ArrayData
+/// \param[in] data the array contents
+/// \param[out] out the resulting Array instance
+/// \return Status
 ARROW_EXPORT
 Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out);
 
-}  // namespace internal
-
 // ----------------------------------------------------------------------
 // User array accessor types
 
@@ -232,7 +233,7 @@ class ARROW_EXPORT Array {
   /// Slice from offset until end of the array
   std::shared_ptr<Array> Slice(int64_t offset) const;
 
-  std::shared_ptr<internal::ArrayData> data() const { return data_; }
+  std::shared_ptr<ArrayData> data() const { return data_; }
 
   int num_fields() const { return static_cast<int>(data_->child_data.size()); }
 
@@ -242,11 +243,11 @@ class ARROW_EXPORT Array {
  protected:
   Array() {}
 
-  std::shared_ptr<internal::ArrayData> data_;
+  std::shared_ptr<ArrayData> data_;
   const uint8_t* null_bitmap_data_;
 
   /// Protected method for constructors
-  inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
+  inline void SetData(const std::shared_ptr<ArrayData>& data) {
     if (data->buffers.size() > 0 && data->buffers[0]) {
       null_bitmap_data_ = data->buffers[0]->data();
     } else {
@@ -274,11 +275,11 @@ class ARROW_EXPORT NullArray : public FlatArray {
  public:
   using TypeClass = NullType;
 
-  explicit NullArray(const std::shared_ptr<internal::ArrayData>& data) { SetData(data); }
+  explicit NullArray(const std::shared_ptr<ArrayData>& data) { SetData(data); }
   explicit NullArray(int64_t length);
 
  private:
-  inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
+  inline void SetData(const std::shared_ptr<ArrayData>& data) {
     null_bitmap_data_ = nullptr;
     data->null_count = data->length;
     data_ = data;
@@ -302,13 +303,13 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray {
  protected:
   PrimitiveArray() {}
 
-  inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
+  inline void SetData(const std::shared_ptr<ArrayData>& data) {
     auto values = data->buffers[1];
     this->Array::SetData(data);
     raw_values_ = values == nullptr ? nullptr : values->data();
   }
 
-  explicit inline PrimitiveArray(const std::shared_ptr<internal::ArrayData>& data) {
+  explicit inline PrimitiveArray(const std::shared_ptr<ArrayData>& data) {
     SetData(data);
   }
 
@@ -321,7 +322,7 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   using TypeClass = TYPE;
   using value_type = typename TypeClass::c_type;
 
-  explicit NumericArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit NumericArray(const std::shared_ptr<ArrayData>& data);
 
   // Only enable this constructor without a type argument for types without additional
   // metadata
@@ -348,7 +349,7 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
  public:
   using TypeClass = BooleanType;
 
-  explicit BooleanArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit BooleanArray(const std::shared_ptr<ArrayData>& data);
 
   BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
                const std::shared_ptr<Buffer>& null_bitmap = nullptr,
@@ -370,7 +371,7 @@ class ARROW_EXPORT ListArray : public Array {
  public:
   using TypeClass = ListType;
 
-  explicit ListArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit ListArray(const std::shared_ptr<ArrayData>& data);
 
   ListArray(const std::shared_ptr<DataType>& type, int64_t length,
             const std::shared_ptr<Buffer>& value_offsets,
@@ -410,7 +411,7 @@ class ARROW_EXPORT ListArray : public Array {
   }
 
  protected:
-  void SetData(const std::shared_ptr<internal::ArrayData>& data);
+  void SetData(const std::shared_ptr<ArrayData>& data);
   const int32_t* raw_value_offsets_;
 
  private:
@@ -424,7 +425,7 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
  public:
   using TypeClass = BinaryType;
 
-  explicit BinaryArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit BinaryArray(const std::shared_ptr<ArrayData>& data);
 
   BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
               const std::shared_ptr<Buffer>& data,
@@ -463,7 +464,7 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
   BinaryArray() {}
 
   /// Protected method for constructors
-  void SetData(const std::shared_ptr<internal::ArrayData>& data);
+  void SetData(const std::shared_ptr<ArrayData>& data);
 
   // Constructor that allows sub-classes/builders to propagate there logical type up the
   // class hierarchy.
@@ -481,7 +482,7 @@ class ARROW_EXPORT StringArray : public BinaryArray {
  public:
   using TypeClass = StringType;
 
-  explicit StringArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit StringArray(const std::shared_ptr<ArrayData>& data);
 
   StringArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
               const std::shared_ptr<Buffer>& data,
@@ -504,7 +505,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
  public:
   using TypeClass = FixedSizeBinaryType;
 
-  explicit FixedSizeBinaryArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit FixedSizeBinaryArray(const std::shared_ptr<ArrayData>& data);
 
   FixedSizeBinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
                        const std::shared_ptr<Buffer>& data,
@@ -517,7 +518,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
   int32_t byte_width() const { return byte_width_; }
 
  protected:
-  inline void SetData(const std::shared_ptr<internal::ArrayData>& data) {
+  inline void SetData(const std::shared_ptr<ArrayData>& data) {
     this->PrimitiveArray::SetData(data);
     byte_width_ = static_cast<const FixedSizeBinaryType&>(*type()).byte_width();
   }
@@ -533,8 +534,8 @@ class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray {
 
   using FixedSizeBinaryArray::FixedSizeBinaryArray;
 
-  /// \brief Construct DecimalArray from internal::ArrayData instance
-  explicit DecimalArray(const std::shared_ptr<internal::ArrayData>& data);
+  /// \brief Construct DecimalArray from ArrayData instance
+  explicit DecimalArray(const std::shared_ptr<ArrayData>& data);
 
   std::string FormatValue(int64_t i) const;
 };
@@ -546,7 +547,7 @@ class ARROW_EXPORT StructArray : public Array {
  public:
   using TypeClass = StructType;
 
-  explicit StructArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit StructArray(const std::shared_ptr<ArrayData>& data);
 
   StructArray(const std::shared_ptr<DataType>& type, int64_t length,
               const std::vector<std::shared_ptr<Array>>& children,
@@ -570,7 +571,7 @@ class ARROW_EXPORT UnionArray : public Array {
   using TypeClass = UnionType;
   using type_id_t = uint8_t;
 
-  explicit UnionArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit UnionArray(const std::shared_ptr<ArrayData>& data);
 
   UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
              const std::vector<std::shared_ptr<Array>>& children,
@@ -593,7 +594,7 @@ class ARROW_EXPORT UnionArray : public Array {
   std::shared_ptr<Array> child(int pos) const;
 
  protected:
-  void SetData(const std::shared_ptr<internal::ArrayData>& data);
+  void SetData(const std::shared_ptr<ArrayData>& data);
 
   const type_id_t* raw_type_ids_;
   const int32_t* raw_value_offsets_;
@@ -624,7 +625,7 @@ class ARROW_EXPORT DictionaryArray : public Array {
  public:
   using TypeClass = DictionaryType;
 
-  explicit DictionaryArray(const std::shared_ptr<internal::ArrayData>& data);
+  explicit DictionaryArray(const std::shared_ptr<ArrayData>& data);
 
   DictionaryArray(const std::shared_ptr<DataType>& type,
                   const std::shared_ptr<Array>& indices);
@@ -635,7 +636,7 @@ class ARROW_EXPORT DictionaryArray : public Array {
   const DictionaryType* dict_type() const { return dict_type_; }
 
  private:
-  void SetData(const std::shared_ptr<internal::ArrayData>& data);
+  void SetData(const std::shared_ptr<ArrayData>& data);
 
   const DictionaryType* dict_type_;
   std::shared_ptr<Array> indices_;
@@ -669,20 +670,26 @@ ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
 ///
 /// \param array an Array instance
 /// \return Status
-Status ARROW_EXPORT ValidateArray(const Array& array);
+ARROW_EXPORT
+Status ValidateArray(const Array& array);
+
+#ifndef ARROW_NO_DEPRECATED_API
+// \deprecated Since 0.7.0
 
 /// Create new arrays for logical types that are backed by primitive arrays.
-Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-                                       int64_t length,
-                                       const std::shared_ptr<Buffer>& data,
-                                       const std::shared_ptr<Buffer>& null_bitmap,
-                                       int64_t null_count, int64_t offset,
-                                       std::shared_ptr<Array>* out);
-
-Status ARROW_EXPORT
-MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-                   const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length,
-                   int64_t null_count, int64_t offset, std::shared_ptr<Array>* out);
+ARROW_EXPORT
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
+                          const std::shared_ptr<Buffer>& data,
+                          const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
+                          int64_t offset, std::shared_ptr<Array>* out);
+
+ARROW_EXPORT
+Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+                          const std::vector<std::shared_ptr<Buffer>>& buffers,
+                          int64_t length, int64_t null_count, int64_t offset,
+                          std::shared_ptr<Array>* out);
+
+#endif
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 7f0f402bb622e..b27b2c74d7b10 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -40,7 +40,6 @@
 namespace arrow {
 
 using internal::AdaptiveIntBuilderBase;
-using internal::ArrayData;
 using internal::WrappedBinary;
 
 Status ArrayBuilder::AppendToBitmap(bool is_valid) {
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 7df8899a31b63..80c63a5b67299 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -704,7 +704,7 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
   static constexpr int64_t kMaximumCapacity = std::numeric_limits<int32_t>::max() - 1;
 
   Status AppendNextOffset();
-  Status FinishInternal(std::shared_ptr<internal::ArrayData>* out);
+  Status FinishInternal(std::shared_ptr<ArrayData>* out);
   void Reset();
 };
 
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index c651244b3ce52..5283bf0a4c297 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -706,7 +706,7 @@ Status Cast(FunctionContext* ctx, const Array& array,
   auto out_data = std::make_shared<ArrayData>(out_type, array.length());
 
   RETURN_NOT_OK(func->Call(ctx, array, out_data.get()));
-  return internal::MakeArray(out_data, out);
+  return MakeArray(out_data, out);
 }
 
 }  // namespace compute
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 5898aeebde414..9df4573deba4a 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -44,9 +44,6 @@
 using std::vector;
 
 namespace arrow {
-
-using internal::ArrayData;
-
 namespace compute {
 
 void AssertArraysEqual(const Array& left, const Array& right) {
diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h
index 521421ef37b41..4e072a7c143ed 100644
--- a/cpp/src/arrow/compute/kernel.h
+++ b/cpp/src/arrow/compute/kernel.h
@@ -21,9 +21,6 @@
 #include "arrow/array.h"
 
 namespace arrow {
-
-using internal::ArrayData;
-
 namespace compute {
 
 class FunctionContext;
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index e74a60dd48925..6bd16462df94d 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -368,11 +368,11 @@ TEST_F(TestTableWriter, TimeTypes) {
   std::vector<std::shared_ptr<Buffer>> buffers = {prim_values.null_bitmap(),
                                                   prim_values.values()};
 
-  std::vector<std::shared_ptr<internal::ArrayData>> arrays;
+  std::vector<std::shared_ptr<ArrayData>> arrays;
   arrays.push_back(date_array->data());
 
   for (int i = 1; i < schema->num_fields(); ++i) {
-    arrays.emplace_back(std::make_shared<internal::ArrayData>(
+    arrays.emplace_back(std::make_shared<ArrayData>(
         schema->field(i)->type(), values->length(), buffers, values->null_count(), 0));
   }
 
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 5c6e2520766d5..31dc0e73e0e3d 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -368,7 +368,10 @@ class TableReader::TableReaderImpl {
     }
 
     buffers.push_back(SliceBuffer(buffer, offset, buffer->size() - offset));
-    return MakePrimitiveArray(type, buffers, meta->length(), meta->null_count(), 0, out);
+
+    auto arr_data =
+        std::make_shared<ArrayData>(type, meta->length(), buffers, meta->null_count());
+    return MakeArray(arr_data, out);
   }
 
   bool HasDescription() const { return metadata_->HasDescription(); }
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 2a0633f31c876..09def6ea6ed4b 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -74,7 +74,7 @@ class IpcComponentSource {
     }
   }
 
-  Status GetFieldMetadata(int field_index, internal::ArrayData* out) {
+  Status GetFieldMetadata(int field_index, ArrayData* out) {
     auto nodes = metadata_->nodes();
     // pop off a field
     if (field_index >= static_cast<int>(nodes->size())) {
@@ -106,11 +106,11 @@ struct ArrayLoaderContext {
 };
 
 static Status LoadArray(const std::shared_ptr<DataType>& type,
-                        ArrayLoaderContext* context, internal::ArrayData* out);
+                        ArrayLoaderContext* context, ArrayData* out);
 
 class ArrayLoader {
  public:
-  ArrayLoader(const std::shared_ptr<DataType>& type, internal::ArrayData* out,
+  ArrayLoader(const std::shared_ptr<DataType>& type, ArrayData* out,
               ArrayLoaderContext* context)
       : type_(type), context_(context), out_(out) {}
 
@@ -168,7 +168,7 @@ class ArrayLoader {
     return GetBuffer(context_->buffer_index++, &out_->buffers[2]);
   }
 
-  Status LoadChild(const Field& field, internal::ArrayData* out) {
+  Status LoadChild(const Field& field, ArrayData* out) {
     ArrayLoader loader(field.type(), out, context_);
     --context_->max_recursion_depth;
     RETURN_NOT_OK(loader.Load());
@@ -180,7 +180,7 @@ class ArrayLoader {
     out_->child_data.reserve(static_cast<int>(child_fields.size()));
 
     for (const auto& child_field : child_fields) {
-      auto field_array = std::make_shared<internal::ArrayData>();
+      auto field_array = std::make_shared<ArrayData>();
       RETURN_NOT_OK(LoadChild(*child_field.get(), field_array.get()));
       out_->child_data.emplace_back(field_array);
     }
@@ -257,11 +257,11 @@ class ArrayLoader {
   ArrayLoaderContext* context_;
 
   // Used in visitor pattern
-  internal::ArrayData* out_;
+  ArrayData* out_;
 };
 
 static Status LoadArray(const std::shared_ptr<DataType>& type,
-                        ArrayLoaderContext* context, internal::ArrayData* out) {
+                        ArrayLoaderContext* context, ArrayData* out) {
   ArrayLoader loader(type, out, context);
   return loader.Load();
 }
@@ -291,9 +291,9 @@ static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
   context.buffer_index = 0;
   context.max_recursion_depth = max_recursion_depth;
 
-  std::vector<std::shared_ptr<internal::ArrayData>> arrays(schema->num_fields());
+  std::vector<std::shared_ptr<ArrayData>> arrays(schema->num_fields());
   for (int i = 0; i < schema->num_fields(); ++i) {
-    auto arr = std::make_shared<internal::ArrayData>();
+    auto arr = std::make_shared<ArrayData>();
     RETURN_NOT_OK(LoadArray(schema->field(i)->type(), &context, arr.get()));
     DCHECK_EQ(num_rows, arr->length) << "Array length did not match record batch length";
     arrays[i] = std::move(arr);
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index 64f753e53d2c1..dc5b67f53e44d 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -56,10 +56,6 @@
 #include "arrow/python/util/datetime.h"
 
 namespace arrow {
-
-using internal::ArrayData;
-using internal::MakeArray;
-
 namespace py {
 
 using internal::NumPyTypeSize;
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 4a08dc540a4d1..3d3ecd2734eba 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -30,8 +30,6 @@
 
 namespace arrow {
 
-using internal::ArrayData;
-
 // ----------------------------------------------------------------------
 // ChunkedArray and Column methods
 
@@ -199,7 +197,7 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
 
 std::shared_ptr<Array> RecordBatch::column(int i) const {
   if (!boxed_columns_[i]) {
-    DCHECK(internal::MakeArray(columns_[i], &boxed_columns_[i]).ok());
+    DCHECK(MakeArray(columns_[i], &boxed_columns_[i]).ok());
   }
   return boxed_columns_[i];
 }
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 90336e982c7ff..a66772e5a7123 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -135,12 +135,12 @@ class ARROW_EXPORT RecordBatch {
   /// should be equal to the length of each field
   /// \param columns the data for the batch's columns
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-              std::vector<std::shared_ptr<internal::ArrayData>>&& columns);
+              std::vector<std::shared_ptr<ArrayData>>&& columns);
 
   /// \brief Construct record batch by copying vector of array data
   /// \since 0.5.0
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-              const std::vector<std::shared_ptr<internal::ArrayData>>& columns);
+              const std::vector<std::shared_ptr<ArrayData>>& columns);
 
   /// \brief Determine if two record batches are exactly equal
   /// \return true if batches are equal
@@ -158,7 +158,7 @@ class ARROW_EXPORT RecordBatch {
   /// \return an Array object
   std::shared_ptr<Array> column(int i) const;
 
-  std::shared_ptr<internal::ArrayData> column_data(int i) const { return columns_[i]; }
+  std::shared_ptr<ArrayData> column_data(int i) const { return columns_[i]; }
 
   /// \brief Name in i-th column
   const std::string& column_name(int i) const;
@@ -197,7 +197,7 @@ class ARROW_EXPORT RecordBatch {
 
   std::shared_ptr<Schema> schema_;
   int64_t num_rows_;
-  std::vector<std::shared_ptr<internal::ArrayData>> columns_;
+  std::vector<std::shared_ptr<ArrayData>> columns_;
 
   // Caching boxed array data
   mutable std::vector<std::shared_ptr<Array>> boxed_columns_;

From 6a020a1f8297c7b6100538d395fa8853adef8f18 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Sep 2017 09:12:51 +0200
Subject: [PATCH 1047/1644] ARROW-1459: [Python] Use list values length to
 advance offset when reconstructing array of ndarrays

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1090 from wesm/ARROW-1459 and squashes the following commits:

39dd025 [Wes McKinney] Use list values length to advance offset when reconstructing array of ndarrays
---
 cpp/src/arrow/python/arrow_to_pandas.cc     |  2 +-
 python/pyarrow/tests/test_convert_pandas.py | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 148963fc2a891..be738e7f92247 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -519,7 +519,7 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
       ++out_values;
     }
 
-    chunk_offset += arr->length();
+    chunk_offset += arr->values()->length();
   }
 
   Py_XDECREF(numpy_array);
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index e98e83d01a0cd..5d56cde7d48b9 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -562,6 +562,24 @@ def test_column_of_lists_chunked(self):
 
         tm.assert_frame_equal(result, df)
 
+    def test_column_of_lists_chunked2(self):
+        data1 = [[0, 1], [2, 3], [4, 5], [6, 7], [10, 11],
+                 [12, 13], [14, 15], [16, 17]]
+        data2 = [[8, 9], [18, 19]]
+
+        a1 = pa.array(data1)
+        a2 = pa.array(data2)
+
+        t1 = pa.Table.from_arrays([a1], names=['a'])
+        t2 = pa.Table.from_arrays([a2], names=['a'])
+
+        concatenated = pa.concat_tables([t1, t2])
+
+        result = concatenated.to_pandas()
+        expected = pd.DataFrame({'a': data1 + data2})
+
+        tm.assert_frame_equal(result, expected)
+
     def test_column_of_lists_strided(self):
         df, schema = dataframe_with_lists()
         df = pd.concat([df] * 6, ignore_index=True)

From b77af1a1b6f5a6532d39e4b228bb939e8837d09f Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 12 Sep 2017 11:00:11 -0400
Subject: [PATCH 1048/1644] ARROW-1528: [GLib] Resolve recursive include
 dependency

Before:

  * arrow-glib/field.h depends on arrow-glib/data-type.h
  * arrow-glib/data-type.h depends on arrow-glib/field.h

After:

  * arrow-glib/field.h depends on arrow-glib/basic-data-type.h
  * arrow-glib/basic-data-type.h doesn't depend on arrow-glib/field.h
  * arrow-glib/composite-data-type.h depends on
    arrow-glib/basic-data-type.h and
    arrow-glib/composite-data-type.h

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1091 from kou/glib-resolve-recursive-dependency and squashes the following commits:

a43c29b3 [Kouhei Sutou] [GLib] Fix document build error
490c6259 [Kouhei Sutou] [GLib] Resolve recursive include dependency
---
 c_glib/arrow-glib/Makefile.am                 |    6 +-
 .../{data-type.cpp => basic-data-type.cpp}    |  104 +-
 c_glib/arrow-glib/basic-data-type.h           |  954 +++++++++++++++
 c_glib/arrow-glib/basic-data-type.hpp         |   27 +
 c_glib/arrow-glib/composite-data-type.cpp     |  136 +++
 c_glib/arrow-glib/composite-data-type.h       |  115 ++
 c_glib/arrow-glib/compute.cpp                 |    2 +-
 c_glib/arrow-glib/data-type.h                 | 1030 +----------------
 c_glib/arrow-glib/data-type.hpp               |    8 +-
 c_glib/arrow-glib/field.h                     |    2 +-
 c_glib/doc/reference/arrow-glib-docs.sgml     |    7 +-
 11 files changed, 1252 insertions(+), 1139 deletions(-)
 rename c_glib/arrow-glib/{data-type.cpp => basic-data-type.cpp} (90%)
 create mode 100644 c_glib/arrow-glib/basic-data-type.h
 create mode 100644 c_glib/arrow-glib/basic-data-type.hpp
 create mode 100644 c_glib/arrow-glib/composite-data-type.cpp
 create mode 100644 c_glib/arrow-glib/composite-data-type.h

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index fafd6d27fa950..cde62cde08014 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -44,9 +44,11 @@ libarrow_glib_la_headers =			\
 	array.h					\
 	array-builder.h				\
 	arrow-glib.h				\
+	basic-data-type.h			\
 	buffer.h				\
 	chunked-array.h				\
 	column.h				\
+	composite-data-type.h			\
 	data-type.h				\
 	error.h					\
 	field.h					\
@@ -83,10 +85,11 @@ libarrow_glib_la_generated_sources =		\
 libarrow_glib_la_sources =			\
 	array.cpp				\
 	array-builder.cpp			\
+	basic-data-type.cpp			\
 	buffer.cpp				\
 	chunked-array.cpp			\
 	column.cpp				\
-	data-type.cpp				\
+	composite-data-type.cpp			\
 	error.cpp				\
 	field.cpp				\
 	record-batch.cpp			\
@@ -118,6 +121,7 @@ libarrow_glib_la_cpp_headers =			\
 	array.hpp				\
 	array-builder.hpp			\
 	arrow-glib.hpp				\
+	basic-data-type.hpp			\
 	buffer.hpp				\
 	chunked-array.hpp			\
 	column.hpp				\
diff --git a/c_glib/arrow-glib/data-type.cpp b/c_glib/arrow-glib/basic-data-type.cpp
similarity index 90%
rename from c_glib/arrow-glib/data-type.cpp
rename to c_glib/arrow-glib/basic-data-type.cpp
index acb1484311988..62f8a7e4d2fbe 100644
--- a/c_glib/arrow-glib/data-type.cpp
+++ b/c_glib/arrow-glib/basic-data-type.cpp
@@ -30,9 +30,9 @@
 G_BEGIN_DECLS
 
 /**
- * SECTION: data-type
- * @section_id: data-type-classes
- * @title: Data type classes
+ * SECTION: basic-data-type
+ * @section_id: basic-data-type-classes
+ * @title: Basic data type classes
  * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowDataType is a base class for all data type classes such as
@@ -84,10 +84,6 @@ G_BEGIN_DECLS
  *
  * #GArrowTime64DataType is a class for the number of microseconds or
  * nanoseconds since midnight in 64-bit signed integer data type.
- *
- * #GArrowListDataType is a class for list data type.
- *
- * #GArrowStructDataType is a class for struct data type.
  */
 
 typedef struct GArrowDataTypePrivate_ {
@@ -947,100 +943,6 @@ garrow_time64_data_type_new(GArrowTimeUnit unit, GError **error)
   return data_type;
 }
 
-
-G_DEFINE_TYPE(GArrowListDataType,                \
-              garrow_list_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_list_data_type_init(GArrowListDataType *object)
-{
-}
-
-static void
-garrow_list_data_type_class_init(GArrowListDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_list_data_type_new:
- * @field: The field of elements
- *
- * Returns: The newly created list data type.
- */
-GArrowListDataType *
-garrow_list_data_type_new(GArrowField *field)
-{
-  auto arrow_field = garrow_field_get_raw(field);
-  auto arrow_data_type =
-    std::make_shared<arrow::ListType>(arrow_field);
-
-  GArrowListDataType *data_type =
-    GARROW_LIST_DATA_TYPE(g_object_new(GARROW_TYPE_LIST_DATA_TYPE,
-                                       "data-type", &arrow_data_type,
-                                       NULL));
-  return data_type;
-}
-
-/**
- * garrow_list_data_type_get_value_field:
- * @list_data_type: A #GArrowListDataType.
- *
- * Returns: (transfer full): The field of value.
- */
-GArrowField *
-garrow_list_data_type_get_value_field(GArrowListDataType *list_data_type)
-{
-  auto arrow_data_type =
-    garrow_data_type_get_raw(GARROW_DATA_TYPE(list_data_type));
-  auto arrow_list_data_type =
-    static_cast<arrow::ListType *>(arrow_data_type.get());
-
-  auto arrow_field = arrow_list_data_type->value_field();
-  auto field = garrow_field_new_raw(&arrow_field);
-
-  return field;
-}
-
-
-G_DEFINE_TYPE(GArrowStructDataType,                \
-              garrow_struct_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
-
-static void
-garrow_struct_data_type_init(GArrowStructDataType *object)
-{
-}
-
-static void
-garrow_struct_data_type_class_init(GArrowStructDataTypeClass *klass)
-{
-}
-
-/**
- * garrow_struct_data_type_new:
- * @fields: (element-type GArrowField): The fields of the struct.
- *
- * Returns: The newly created struct data type.
- */
-GArrowStructDataType *
-garrow_struct_data_type_new(GList *fields)
-{
-  std::vector<std::shared_ptr<arrow::Field>> arrow_fields;
-  for (GList *node = fields; node; node = g_list_next(node)) {
-    auto field = GARROW_FIELD(node->data);
-    auto arrow_field = garrow_field_get_raw(field);
-    arrow_fields.push_back(arrow_field);
-  }
-
-  auto arrow_data_type = std::make_shared<arrow::StructType>(arrow_fields);
-  GArrowStructDataType *data_type =
-    GARROW_STRUCT_DATA_TYPE(g_object_new(GARROW_TYPE_STRUCT_DATA_TYPE,
-                                         "data-type", &arrow_data_type,
-                                         NULL));
-  return data_type;
-}
-
 G_END_DECLS
 
 GArrowDataType *
diff --git a/c_glib/arrow-glib/basic-data-type.h b/c_glib/arrow-glib/basic-data-type.h
new file mode 100644
index 0000000000000..d3d52c6120de7
--- /dev/null
+++ b/c_glib/arrow-glib/basic-data-type.h
@@ -0,0 +1,954 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_DATA_TYPE                   \
+  (garrow_data_type_get_type())
+#define GARROW_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DATA_TYPE,    \
+                              GArrowDataType))
+#define GARROW_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DATA_TYPE,       \
+                           GArrowDataTypeClass))
+#define GARROW_IS_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_DATA_TYPE))
+#define GARROW_IS_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DATA_TYPE))
+#define GARROW_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DATA_TYPE,     \
+                             GArrowDataTypeClass))
+
+typedef struct _GArrowDataType         GArrowDataType;
+typedef struct _GArrowDataTypeClass    GArrowDataTypeClass;
+
+/**
+ * GArrowDataType:
+ *
+ * It wraps `arrow::DataType`.
+ */
+struct _GArrowDataType
+{
+  /*< private >*/
+  GObject parent_instance;
+};
+
+struct _GArrowDataTypeClass
+{
+  GObjectClass parent_class;
+};
+
+GType      garrow_data_type_get_type  (void) G_GNUC_CONST;
+gboolean   garrow_data_type_equal     (GArrowDataType *data_type,
+                                       GArrowDataType *other_data_type);
+gchar     *garrow_data_type_to_string (GArrowDataType *data_type);
+GArrowType garrow_data_type_get_id    (GArrowDataType *data_type);
+
+
+#define GARROW_TYPE_NULL_DATA_TYPE              \
+  (garrow_null_data_type_get_type())
+#define GARROW_NULL_DATA_TYPE(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_NULL_DATA_TYPE,       \
+                              GArrowNullDataType))
+#define GARROW_NULL_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_NULL_DATA_TYPE,  \
+                           GArrowNullDataTypeClass))
+#define GARROW_IS_NULL_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_NULL_DATA_TYPE))
+#define GARROW_IS_NULL_DATA_TYPE_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_NULL_DATA_TYPE))
+#define GARROW_NULL_DATA_TYPE_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_NULL_DATA_TYPE,        \
+                             GArrowNullDataTypeClass))
+
+typedef struct _GArrowNullDataType         GArrowNullDataType;
+typedef struct _GArrowNullDataTypeClass    GArrowNullDataTypeClass;
+
+/**
+ * GArrowNullDataType:
+ *
+ * It wraps `arrow::NullType`.
+ */
+struct _GArrowNullDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowNullDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType               garrow_null_data_type_get_type (void) G_GNUC_CONST;
+GArrowNullDataType *garrow_null_data_type_new      (void);
+
+
+#define GARROW_TYPE_BOOLEAN_DATA_TYPE           \
+  (garrow_boolean_data_type_get_type())
+#define GARROW_BOOLEAN_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BOOLEAN_DATA_TYPE,    \
+                              GArrowBooleanDataType))
+#define GARROW_BOOLEAN_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BOOLEAN_DATA_TYPE,       \
+                           GArrowBooleanDataTypeClass))
+#define GARROW_IS_BOOLEAN_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BOOLEAN_DATA_TYPE))
+#define GARROW_IS_BOOLEAN_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BOOLEAN_DATA_TYPE))
+#define GARROW_BOOLEAN_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BOOLEAN_DATA_TYPE,     \
+                             GArrowBooleanDataTypeClass))
+
+typedef struct _GArrowBooleanDataType         GArrowBooleanDataType;
+typedef struct _GArrowBooleanDataTypeClass    GArrowBooleanDataTypeClass;
+
+/**
+ * GArrowBooleanDataType:
+ *
+ * It wraps `arrow::BooleanType`.
+ */
+struct _GArrowBooleanDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowBooleanDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                  garrow_boolean_data_type_get_type (void) G_GNUC_CONST;
+GArrowBooleanDataType *garrow_boolean_data_type_new      (void);
+
+
+#define GARROW_TYPE_INT8_DATA_TYPE            \
+  (garrow_int8_data_type_get_type())
+#define GARROW_INT8_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
+                              GARROW_TYPE_INT8_DATA_TYPE,     \
+                              GArrowInt8DataType))
+#define GARROW_INT8_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
+                           GARROW_TYPE_INT8_DATA_TYPE,        \
+                           GArrowInt8DataTypeClass))
+#define GARROW_IS_INT8_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
+                              GARROW_TYPE_INT8_DATA_TYPE))
+#define GARROW_IS_INT8_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
+                           GARROW_TYPE_INT8_DATA_TYPE))
+#define GARROW_INT8_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
+                             GARROW_TYPE_INT8_DATA_TYPE,      \
+                             GArrowInt8DataTypeClass))
+
+typedef struct _GArrowInt8DataType         GArrowInt8DataType;
+typedef struct _GArrowInt8DataTypeClass    GArrowInt8DataTypeClass;
+
+/**
+ * GArrowInt8DataType:
+ *
+ * It wraps `arrow::Int8Type`.
+ */
+struct _GArrowInt8DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt8DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int8_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt8DataType   *garrow_int8_data_type_new      (void);
+
+
+#define GARROW_TYPE_UINT8_DATA_TYPE            \
+  (garrow_uint8_data_type_get_type())
+#define GARROW_UINT8_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
+                              GARROW_TYPE_UINT8_DATA_TYPE,     \
+                              GArrowUInt8DataType))
+#define GARROW_UINT8_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
+                           GARROW_TYPE_UINT8_DATA_TYPE,        \
+                           GArrowUInt8DataTypeClass))
+#define GARROW_IS_UINT8_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
+                              GARROW_TYPE_UINT8_DATA_TYPE))
+#define GARROW_IS_UINT8_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
+                           GARROW_TYPE_UINT8_DATA_TYPE))
+#define GARROW_UINT8_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
+                             GARROW_TYPE_UINT8_DATA_TYPE,      \
+                             GArrowUInt8DataTypeClass))
+
+typedef struct _GArrowUInt8DataType         GArrowUInt8DataType;
+typedef struct _GArrowUInt8DataTypeClass    GArrowUInt8DataTypeClass;
+
+/**
+ * GArrowUInt8DataType:
+ *
+ * It wraps `arrow::UInt8Type`.
+ */
+struct _GArrowUInt8DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt8DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint8_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt8DataType  *garrow_uint8_data_type_new      (void);
+
+
+#define GARROW_TYPE_INT16_DATA_TYPE            \
+  (garrow_int16_data_type_get_type())
+#define GARROW_INT16_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
+                              GARROW_TYPE_INT16_DATA_TYPE,     \
+                              GArrowInt16DataType))
+#define GARROW_INT16_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
+                           GARROW_TYPE_INT16_DATA_TYPE,        \
+                           GArrowInt16DataTypeClass))
+#define GARROW_IS_INT16_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
+                              GARROW_TYPE_INT16_DATA_TYPE))
+#define GARROW_IS_INT16_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
+                           GARROW_TYPE_INT16_DATA_TYPE))
+#define GARROW_INT16_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
+                             GARROW_TYPE_INT16_DATA_TYPE,      \
+                             GArrowInt16DataTypeClass))
+
+typedef struct _GArrowInt16DataType         GArrowInt16DataType;
+typedef struct _GArrowInt16DataTypeClass    GArrowInt16DataTypeClass;
+
+/**
+ * GArrowInt16DataType:
+ *
+ * It wraps `arrow::Int16Type`.
+ */
+struct _GArrowInt16DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt16DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int16_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt16DataType  *garrow_int16_data_type_new      (void);
+
+
+#define GARROW_TYPE_UINT16_DATA_TYPE            \
+  (garrow_uint16_data_type_get_type())
+#define GARROW_UINT16_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT16_DATA_TYPE,     \
+                              GArrowUInt16DataType))
+#define GARROW_UINT16_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT16_DATA_TYPE,        \
+                           GArrowUInt16DataTypeClass))
+#define GARROW_IS_UINT16_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT16_DATA_TYPE))
+#define GARROW_IS_UINT16_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT16_DATA_TYPE))
+#define GARROW_UINT16_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT16_DATA_TYPE,      \
+                             GArrowUInt16DataTypeClass))
+
+typedef struct _GArrowUInt16DataType         GArrowUInt16DataType;
+typedef struct _GArrowUInt16DataTypeClass    GArrowUInt16DataTypeClass;
+
+/**
+ * GArrowUInt16DataType:
+ *
+ * It wraps `arrow::UInt16Type`.
+ */
+struct _GArrowUInt16DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt16DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint16_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt16DataType *garrow_uint16_data_type_new      (void);
+
+
+#define GARROW_TYPE_INT32_DATA_TYPE            \
+  (garrow_int32_data_type_get_type())
+#define GARROW_INT32_DATA_TYPE(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT32_DATA_TYPE,      \
+                              GArrowInt32DataType))
+#define GARROW_INT32_DATA_TYPE_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT32_DATA_TYPE, \
+                           GArrowInt32DataTypeClass))
+#define GARROW_IS_INT32_DATA_TYPE(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT32_DATA_TYPE))
+#define GARROW_IS_INT32_DATA_TYPE_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT32_DATA_TYPE))
+#define GARROW_INT32_DATA_TYPE_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT32_DATA_TYPE,       \
+                             GArrowInt32DataTypeClass))
+
+typedef struct _GArrowInt32DataType         GArrowInt32DataType;
+typedef struct _GArrowInt32DataTypeClass    GArrowInt32DataTypeClass;
+
+/**
+ * GArrowInt32DataType:
+ *
+ * It wraps `arrow::Int32Type`.
+ */
+struct _GArrowInt32DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt32DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int32_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt32DataType  *garrow_int32_data_type_new      (void);
+
+
+#define GARROW_TYPE_UINT32_DATA_TYPE            \
+  (garrow_uint32_data_type_get_type())
+#define GARROW_UINT32_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT32_DATA_TYPE,     \
+                              GArrowUInt32DataType))
+#define GARROW_UINT32_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT32_DATA_TYPE,        \
+                           GArrowUInt32DataTypeClass))
+#define GARROW_IS_UINT32_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT32_DATA_TYPE))
+#define GARROW_IS_UINT32_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT32_DATA_TYPE))
+#define GARROW_UINT32_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT32_DATA_TYPE,      \
+                             GArrowUInt32DataTypeClass))
+
+typedef struct _GArrowUInt32DataType         GArrowUInt32DataType;
+typedef struct _GArrowUInt32DataTypeClass    GArrowUInt32DataTypeClass;
+
+/**
+ * GArrowUInt32DataType:
+ *
+ * It wraps `arrow::UInt32Type`.
+ */
+struct _GArrowUInt32DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt32DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint32_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt32DataType *garrow_uint32_data_type_new      (void);
+
+
+#define GARROW_TYPE_INT64_DATA_TYPE            \
+  (garrow_int64_data_type_get_type())
+#define GARROW_INT64_DATA_TYPE(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_INT64_DATA_TYPE,      \
+                              GArrowInt64DataType))
+#define GARROW_INT64_DATA_TYPE_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT64_DATA_TYPE, \
+                           GArrowInt64DataTypeClass))
+#define GARROW_IS_INT64_DATA_TYPE(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_INT64_DATA_TYPE))
+#define GARROW_IS_INT64_DATA_TYPE_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_INT64_DATA_TYPE))
+#define GARROW_INT64_DATA_TYPE_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_INT64_DATA_TYPE,       \
+                             GArrowInt64DataTypeClass))
+
+typedef struct _GArrowInt64DataType         GArrowInt64DataType;
+typedef struct _GArrowInt64DataTypeClass    GArrowInt64DataTypeClass;
+
+/**
+ * GArrowInt64DataType:
+ *
+ * It wraps `arrow::Int64Type`.
+ */
+struct _GArrowInt64DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowInt64DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_int64_data_type_get_type (void) G_GNUC_CONST;
+GArrowInt64DataType  *garrow_int64_data_type_new      (void);
+
+
+#define GARROW_TYPE_UINT64_DATA_TYPE            \
+  (garrow_uint64_data_type_get_type())
+#define GARROW_UINT64_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_UINT64_DATA_TYPE,     \
+                              GArrowUInt64DataType))
+#define GARROW_UINT64_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_UINT64_DATA_TYPE,        \
+                           GArrowUInt64DataTypeClass))
+#define GARROW_IS_UINT64_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT64_DATA_TYPE))
+#define GARROW_IS_UINT64_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_UINT64_DATA_TYPE))
+#define GARROW_UINT64_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_UINT64_DATA_TYPE,      \
+                             GArrowUInt64DataTypeClass))
+
+typedef struct _GArrowUInt64DataType         GArrowUInt64DataType;
+typedef struct _GArrowUInt64DataTypeClass    GArrowUInt64DataTypeClass;
+
+/**
+ * GArrowUInt64DataType:
+ *
+ * It wraps `arrow::UInt64Type`.
+ */
+struct _GArrowUInt64DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowUInt64DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_uint64_data_type_get_type (void) G_GNUC_CONST;
+GArrowUInt64DataType *garrow_uint64_data_type_new      (void);
+
+
+#define GARROW_TYPE_FLOAT_DATA_TYPE           \
+  (garrow_float_data_type_get_type())
+#define GARROW_FLOAT_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
+                              GARROW_TYPE_FLOAT_DATA_TYPE,    \
+                              GArrowFloatDataType))
+#define GARROW_FLOAT_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
+                           GARROW_TYPE_FLOAT_DATA_TYPE,       \
+                           GArrowFloatDataTypeClass))
+#define GARROW_IS_FLOAT_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
+                              GARROW_TYPE_FLOAT_DATA_TYPE))
+#define GARROW_IS_FLOAT_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
+                           GARROW_TYPE_FLOAT_DATA_TYPE))
+#define GARROW_FLOAT_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
+                             GARROW_TYPE_FLOAT_DATA_TYPE,     \
+                             GArrowFloatDataTypeClass))
+
+typedef struct _GArrowFloatDataType         GArrowFloatDataType;
+typedef struct _GArrowFloatDataTypeClass    GArrowFloatDataTypeClass;
+
+/**
+ * GArrowFloatDataType:
+ *
+ * It wraps `arrow::FloatType`.
+ */
+struct _GArrowFloatDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowFloatDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                garrow_float_data_type_get_type (void) G_GNUC_CONST;
+GArrowFloatDataType *garrow_float_data_type_new      (void);
+
+
+#define GARROW_TYPE_DOUBLE_DATA_TYPE           \
+  (garrow_double_data_type_get_type())
+#define GARROW_DOUBLE_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DOUBLE_DATA_TYPE,     \
+                              GArrowDoubleDataType))
+#define GARROW_DOUBLE_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DOUBLE_DATA_TYPE,        \
+                           GArrowDoubleDataTypeClass))
+#define GARROW_IS_DOUBLE_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DOUBLE_DATA_TYPE))
+#define GARROW_IS_DOUBLE_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DOUBLE_DATA_TYPE))
+#define GARROW_DOUBLE_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DOUBLE_DATA_TYPE,      \
+                             GArrowDoubleDataTypeClass))
+
+typedef struct _GArrowDoubleDataType         GArrowDoubleDataType;
+typedef struct _GArrowDoubleDataTypeClass    GArrowDoubleDataTypeClass;
+
+/**
+ * GArrowDoubleDataType:
+ *
+ * It wraps `arrow::DoubleType`.
+ */
+struct _GArrowDoubleDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowDoubleDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_double_data_type_get_type (void) G_GNUC_CONST;
+GArrowDoubleDataType *garrow_double_data_type_new      (void);
+
+
+#define GARROW_TYPE_BINARY_DATA_TYPE            \
+  (garrow_binary_data_type_get_type())
+#define GARROW_BINARY_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BINARY_DATA_TYPE,     \
+                              GArrowBinaryDataType))
+#define GARROW_BINARY_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_BINARY_DATA_TYPE,        \
+                           GArrowBinaryDataTypeClass))
+#define GARROW_IS_BINARY_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BINARY_DATA_TYPE))
+#define GARROW_IS_BINARY_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_BINARY_DATA_TYPE))
+#define GARROW_BINARY_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_BINARY_DATA_TYPE,      \
+                             GArrowBinaryDataTypeClass))
+
+typedef struct _GArrowBinaryDataType         GArrowBinaryDataType;
+typedef struct _GArrowBinaryDataTypeClass    GArrowBinaryDataTypeClass;
+
+/**
+ * GArrowBinaryDataType:
+ *
+ * It wraps `arrow::BinaryType`.
+ */
+struct _GArrowBinaryDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowBinaryDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_binary_data_type_get_type (void) G_GNUC_CONST;
+GArrowBinaryDataType *garrow_binary_data_type_new      (void);
+
+
+#define GARROW_TYPE_STRING_DATA_TYPE            \
+  (garrow_string_data_type_get_type())
+#define GARROW_STRING_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
+                              GARROW_TYPE_STRING_DATA_TYPE,    \
+                              GArrowStringDataType))
+#define GARROW_STRING_DATA_TYPE_CLASS(klass)                   \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
+                           GARROW_TYPE_STRING_DATA_TYPE,       \
+                           GArrowStringDataTypeClass))
+#define GARROW_IS_STRING_DATA_TYPE(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
+                              GARROW_TYPE_STRING_DATA_TYPE))
+#define GARROW_IS_STRING_DATA_TYPE_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
+                           GARROW_TYPE_STRING_DATA_TYPE))
+#define GARROW_STRING_DATA_TYPE_GET_CLASS(obj)                 \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
+                             GARROW_TYPE_STRING_DATA_TYPE,     \
+                             GArrowStringDataTypeClass))
+
+typedef struct _GArrowStringDataType         GArrowStringDataType;
+typedef struct _GArrowStringDataTypeClass    GArrowStringDataTypeClass;
+
+/**
+ * GArrowStringDataType:
+ *
+ * It wraps `arrow::StringType`.
+ */
+struct _GArrowStringDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowStringDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_string_data_type_get_type (void) G_GNUC_CONST;
+GArrowStringDataType *garrow_string_data_type_new      (void);
+
+
+#define GARROW_TYPE_DATE32_DATA_TYPE           \
+  (garrow_date32_data_type_get_type())
+#define GARROW_DATE32_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DATE32_DATA_TYPE,     \
+                              GArrowDate32DataType))
+#define GARROW_DATE32_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DATE32_DATA_TYPE,        \
+                           GArrowDate32DataTypeClass))
+#define GARROW_IS_DATE32_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DATE32_DATA_TYPE))
+#define GARROW_IS_DATE32_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DATE32_DATA_TYPE))
+#define GARROW_DATE32_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DATE32_DATA_TYPE,      \
+                             GArrowDate32DataTypeClass))
+
+typedef struct _GArrowDate32DataType         GArrowDate32DataType;
+typedef struct _GArrowDate32DataTypeClass    GArrowDate32DataTypeClass;
+
+/**
+ * GArrowDate32DataType:
+ *
+ * It wraps `arrow::Date32Type`.
+ */
+struct _GArrowDate32DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowDate32DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_date32_data_type_get_type (void) G_GNUC_CONST;
+GArrowDate32DataType *garrow_date32_data_type_new      (void);
+
+
+#define GARROW_TYPE_DATE64_DATA_TYPE           \
+  (garrow_date64_data_type_get_type())
+#define GARROW_DATE64_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_DATE64_DATA_TYPE,     \
+                              GArrowDate64DataType))
+#define GARROW_DATE64_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_DATE64_DATA_TYPE,        \
+                           GArrowDate64DataTypeClass))
+#define GARROW_IS_DATE64_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DATE64_DATA_TYPE))
+#define GARROW_IS_DATE64_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_DATE64_DATA_TYPE))
+#define GARROW_DATE64_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_DATE64_DATA_TYPE,      \
+                             GArrowDate64DataTypeClass))
+
+typedef struct _GArrowDate64DataType         GArrowDate64DataType;
+typedef struct _GArrowDate64DataTypeClass    GArrowDate64DataTypeClass;
+
+/**
+ * GArrowDate64DataType:
+ *
+ * It wraps `arrow::Date64Type`.
+ */
+struct _GArrowDate64DataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowDate64DataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_date64_data_type_get_type (void) G_GNUC_CONST;
+GArrowDate64DataType *garrow_date64_data_type_new      (void);
+
+
+#define GARROW_TYPE_TIMESTAMP_DATA_TYPE         \
+  (garrow_timestamp_data_type_get_type())
+#define GARROW_TIMESTAMP_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIMESTAMP_DATA_TYPE,  \
+                              GArrowTimestampDataType))
+#define GARROW_TIMESTAMP_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIMESTAMP_DATA_TYPE,     \
+                           GArrowTimestampDataTypeClass))
+#define GARROW_IS_TIMESTAMP_DATA_TYPE(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIMESTAMP_DATA_TYPE))
+#define GARROW_IS_TIMESTAMP_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIMESTAMP_DATA_TYPE))
+#define GARROW_TIMESTAMP_DATA_TYPE_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIMESTAMP_DATA_TYPE,   \
+                             GArrowTimestampDataTypeClass))
+
+typedef struct _GArrowTimestampDataType         GArrowTimestampDataType;
+typedef struct _GArrowTimestampDataTypeClass    GArrowTimestampDataTypeClass;
+
+/**
+ * GArrowTimestampDataType:
+ *
+ * It wraps `arrow::TimestampType`.
+ */
+struct _GArrowTimestampDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowTimestampDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_timestamp_data_type_get_type (void) G_GNUC_CONST;
+GArrowTimestampDataType *garrow_timestamp_data_type_new   (GArrowTimeUnit unit);
+
+
+#define GARROW_TYPE_TIME_DATA_TYPE              \
+  (garrow_time_data_type_get_type())
+#define GARROW_TIME_DATA_TYPE(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIME_DATA_TYPE,       \
+                              GArrowTimeDataType))
+#define GARROW_TIME_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_TIME_DATA_TYPE,  \
+                           GArrowTimeDataTypeClass))
+#define GARROW_IS_TIME_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME_DATA_TYPE))
+#define GARROW_IS_TIME_DATA_TYPE_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_TIME_DATA_TYPE))
+#define GARROW_TIME_DATA_TYPE_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIME_DATA_TYPE,        \
+                             GArrowTimeDataTypeClass))
+
+typedef struct _GArrowTimeDataType         GArrowTimeDataType;
+typedef struct _GArrowTimeDataTypeClass    GArrowTimeDataTypeClass;
+
+/**
+ * GArrowTimeDataType:
+ *
+ * It wraps `arrow::TimeType`.
+ */
+struct _GArrowTimeDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowTimeDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType          garrow_time_data_type_get_type (void) G_GNUC_CONST;
+GArrowTimeUnit garrow_time_data_type_get_unit (GArrowTimeDataType *time_data_type);
+
+
+#define GARROW_TYPE_TIME32_DATA_TYPE           \
+  (garrow_time32_data_type_get_type())
+#define GARROW_TIME32_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIME32_DATA_TYPE,     \
+                              GArrowTime32DataType))
+#define GARROW_TIME32_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIME32_DATA_TYPE,        \
+                           GArrowTime32DataTypeClass))
+#define GARROW_IS_TIME32_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME32_DATA_TYPE))
+#define GARROW_IS_TIME32_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIME32_DATA_TYPE))
+#define GARROW_TIME32_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIME32_DATA_TYPE,      \
+                             GArrowTime32DataTypeClass))
+
+typedef struct _GArrowTime32DataType         GArrowTime32DataType;
+typedef struct _GArrowTime32DataTypeClass    GArrowTime32DataTypeClass;
+
+/**
+ * GArrowTime32DataType:
+ *
+ * It wraps `arrow::Time32Type`.
+ */
+struct _GArrowTime32DataType
+{
+  /*< private >*/
+  GArrowTimeDataType parent_instance;
+};
+
+struct _GArrowTime32DataTypeClass
+{
+  GArrowTimeDataTypeClass parent_class;
+};
+
+GType                 garrow_time32_data_type_get_type (void) G_GNUC_CONST;
+GArrowTime32DataType *garrow_time32_data_type_new      (GArrowTimeUnit unit,
+                                                        GError **error);
+
+
+#define GARROW_TYPE_TIME64_DATA_TYPE           \
+  (garrow_time64_data_type_get_type())
+#define GARROW_TIME64_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIME64_DATA_TYPE,     \
+                              GArrowTime64DataType))
+#define GARROW_TIME64_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_TIME64_DATA_TYPE,        \
+                           GArrowTime64DataTypeClass))
+#define GARROW_IS_TIME64_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME64_DATA_TYPE))
+#define GARROW_IS_TIME64_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIME64_DATA_TYPE))
+#define GARROW_TIME64_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIME64_DATA_TYPE,      \
+                             GArrowTime64DataTypeClass))
+
+typedef struct _GArrowTime64DataType         GArrowTime64DataType;
+typedef struct _GArrowTime64DataTypeClass    GArrowTime64DataTypeClass;
+
+/**
+ * GArrowTime64DataType:
+ *
+ * It wraps `arrow::Time64Type`.
+ */
+struct _GArrowTime64DataType
+{
+  /*< private >*/
+  GArrowTimeDataType parent_instance;
+};
+
+struct _GArrowTime64DataTypeClass
+{
+  GArrowTimeDataTypeClass parent_class;
+};
+
+GType                 garrow_time64_data_type_get_type (void) G_GNUC_CONST;
+GArrowTime64DataType *garrow_time64_data_type_new      (GArrowTimeUnit unit,
+                                                        GError **error);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/basic-data-type.hpp b/c_glib/arrow-glib/basic-data-type.hpp
new file mode 100644
index 0000000000000..aa95cecec408d
--- /dev/null
+++ b/c_glib/arrow-glib/basic-data-type.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/basic-data-type.h>
+
+GArrowDataType *garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type);
+std::shared_ptr<arrow::DataType> garrow_data_type_get_raw(GArrowDataType *data_type);
diff --git a/c_glib/arrow-glib/composite-data-type.cpp b/c_glib/arrow-glib/composite-data-type.cpp
new file mode 100644
index 0000000000000..ce3d78ca08332
--- /dev/null
+++ b/c_glib/arrow-glib/composite-data-type.cpp
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/enums.h>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/field.hpp>
+#include <arrow-glib/type.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: composite-data-type
+ * @section_id: composite-data-type-classes
+ * @title: Composite data type classes
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowListDataType is a class for list data type.
+ *
+ * #GArrowStructDataType is a class for struct data type.
+ */
+
+G_DEFINE_TYPE(GArrowListDataType,                \
+              garrow_list_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_list_data_type_init(GArrowListDataType *object)
+{
+}
+
+static void
+garrow_list_data_type_class_init(GArrowListDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_list_data_type_new:
+ * @field: The field of elements
+ *
+ * Returns: The newly created list data type.
+ */
+GArrowListDataType *
+garrow_list_data_type_new(GArrowField *field)
+{
+  auto arrow_field = garrow_field_get_raw(field);
+  auto arrow_data_type =
+    std::make_shared<arrow::ListType>(arrow_field);
+
+  GArrowListDataType *data_type =
+    GARROW_LIST_DATA_TYPE(g_object_new(GARROW_TYPE_LIST_DATA_TYPE,
+                                       "data-type", &arrow_data_type,
+                                       NULL));
+  return data_type;
+}
+
+/**
+ * garrow_list_data_type_get_value_field:
+ * @list_data_type: A #GArrowListDataType.
+ *
+ * Returns: (transfer full): The field of value.
+ */
+GArrowField *
+garrow_list_data_type_get_value_field(GArrowListDataType *list_data_type)
+{
+  auto arrow_data_type =
+    garrow_data_type_get_raw(GARROW_DATA_TYPE(list_data_type));
+  auto arrow_list_data_type =
+    static_cast<arrow::ListType *>(arrow_data_type.get());
+
+  auto arrow_field = arrow_list_data_type->value_field();
+  auto field = garrow_field_new_raw(&arrow_field);
+
+  return field;
+}
+
+
+G_DEFINE_TYPE(GArrowStructDataType,                \
+              garrow_struct_data_type,             \
+              GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_struct_data_type_init(GArrowStructDataType *object)
+{
+}
+
+static void
+garrow_struct_data_type_class_init(GArrowStructDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_struct_data_type_new:
+ * @fields: (element-type GArrowField): The fields of the struct.
+ *
+ * Returns: The newly created struct data type.
+ */
+GArrowStructDataType *
+garrow_struct_data_type_new(GList *fields)
+{
+  std::vector<std::shared_ptr<arrow::Field>> arrow_fields;
+  for (GList *node = fields; node; node = g_list_next(node)) {
+    auto field = GARROW_FIELD(node->data);
+    auto arrow_field = garrow_field_get_raw(field);
+    arrow_fields.push_back(arrow_field);
+  }
+
+  auto arrow_data_type = std::make_shared<arrow::StructType>(arrow_fields);
+  GArrowStructDataType *data_type =
+    GARROW_STRUCT_DATA_TYPE(g_object_new(GARROW_TYPE_STRUCT_DATA_TYPE,
+                                         "data-type", &arrow_data_type,
+                                         NULL));
+  return data_type;
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/composite-data-type.h b/c_glib/arrow-glib/composite-data-type.h
new file mode 100644
index 0000000000000..9dac5bd84bed2
--- /dev/null
+++ b/c_glib/arrow-glib/composite-data-type.h
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/basic-data-type.h>
+#include <arrow-glib/field.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_LIST_DATA_TYPE              \
+  (garrow_list_data_type_get_type())
+#define GARROW_LIST_DATA_TYPE(obj)                              \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_LIST_DATA_TYPE,       \
+                              GArrowListDataType))
+#define GARROW_LIST_DATA_TYPE_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_LIST_DATA_TYPE,  \
+                           GArrowListDataTypeClass))
+#define GARROW_IS_LIST_DATA_TYPE(obj)                           \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_LIST_DATA_TYPE))
+#define GARROW_IS_LIST_DATA_TYPE_CLASS(klass)           \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_LIST_DATA_TYPE))
+#define GARROW_LIST_DATA_TYPE_GET_CLASS(obj)                    \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_LIST_DATA_TYPE,        \
+                             GArrowListDataTypeClass))
+
+typedef struct _GArrowListDataType         GArrowListDataType;
+typedef struct _GArrowListDataTypeClass    GArrowListDataTypeClass;
+
+/**
+ * GArrowListDataType:
+ *
+ * It wraps `arrow::ListType`.
+ */
+struct _GArrowListDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowListDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType               garrow_list_data_type_get_type (void) G_GNUC_CONST;
+GArrowListDataType *garrow_list_data_type_new      (GArrowField *field);
+GArrowField *garrow_list_data_type_get_value_field (GArrowListDataType *list_data_type);
+
+
+#define GARROW_TYPE_STRUCT_DATA_TYPE            \
+  (garrow_struct_data_type_get_type())
+#define GARROW_STRUCT_DATA_TYPE(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_STRUCT_DATA_TYPE,     \
+                              GArrowStructDataType))
+#define GARROW_STRUCT_DATA_TYPE_CLASS(klass)                    \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
+                           GARROW_TYPE_STRUCT_DATA_TYPE,        \
+                           GArrowStructDataTypeClass))
+#define GARROW_IS_STRUCT_DATA_TYPE(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRUCT_DATA_TYPE))
+#define GARROW_IS_STRUCT_DATA_TYPE_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_STRUCT_DATA_TYPE))
+#define GARROW_STRUCT_DATA_TYPE_GET_CLASS(obj)                  \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_STRUCT_DATA_TYPE,      \
+                             GArrowStructDataTypeClass))
+
+typedef struct _GArrowStructDataType         GArrowStructDataType;
+typedef struct _GArrowStructDataTypeClass    GArrowStructDataTypeClass;
+
+/**
+ * GArrowStructDataType:
+ *
+ * It wraps `arrow::StructType`.
+ */
+struct _GArrowStructDataType
+{
+  /*< private >*/
+  GArrowDataType parent_instance;
+};
+
+struct _GArrowStructDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+GType                 garrow_struct_data_type_get_type (void) G_GNUC_CONST;
+GArrowStructDataType *garrow_struct_data_type_new      (GList *fields);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/compute.cpp b/c_glib/arrow-glib/compute.cpp
index f7a48d148b0a4..9134aa675c50f 100644
--- a/c_glib/arrow-glib/compute.cpp
+++ b/c_glib/arrow-glib/compute.cpp
@@ -28,7 +28,7 @@ G_BEGIN_DECLS
 /**
  * SECTION: compute
  * @section_id: compute-classes
- * @title: Compute classes
+ * @title: Classes for computation
  * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowCastOptions is a class to custom garrow_array_cast().
diff --git a/c_glib/arrow-glib/data-type.h b/c_glib/arrow-glib/data-type.h
index a450ab3bc8dc5..6067b7234b28e 100644
--- a/c_glib/arrow-glib/data-type.h
+++ b/c_glib/arrow-glib/data-type.h
@@ -19,1031 +19,5 @@
 
 #pragma once
 
-#include <glib-object.h>
-
-#include <arrow-glib/type.h>
-
-G_BEGIN_DECLS
-
-#ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowField         GArrowField;
-#endif
-
-#define GARROW_TYPE_DATA_TYPE                   \
-  (garrow_data_type_get_type())
-#define GARROW_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DATA_TYPE,    \
-                              GArrowDataType))
-#define GARROW_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DATA_TYPE,       \
-                           GArrowDataTypeClass))
-#define GARROW_IS_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_DATA_TYPE))
-#define GARROW_IS_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DATA_TYPE))
-#define GARROW_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DATA_TYPE,     \
-                             GArrowDataTypeClass))
-
-typedef struct _GArrowDataType         GArrowDataType;
-typedef struct _GArrowDataTypeClass    GArrowDataTypeClass;
-
-/**
- * GArrowDataType:
- *
- * It wraps `arrow::DataType`.
- */
-struct _GArrowDataType
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowDataTypeClass
-{
-  GObjectClass parent_class;
-};
-
-GType      garrow_data_type_get_type  (void) G_GNUC_CONST;
-gboolean   garrow_data_type_equal     (GArrowDataType *data_type,
-                                       GArrowDataType *other_data_type);
-gchar     *garrow_data_type_to_string (GArrowDataType *data_type);
-GArrowType garrow_data_type_get_id    (GArrowDataType *data_type);
-
-
-#define GARROW_TYPE_NULL_DATA_TYPE              \
-  (garrow_null_data_type_get_type())
-#define GARROW_NULL_DATA_TYPE(obj)                              \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_NULL_DATA_TYPE,       \
-                              GArrowNullDataType))
-#define GARROW_NULL_DATA_TYPE_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_NULL_DATA_TYPE,  \
-                           GArrowNullDataTypeClass))
-#define GARROW_IS_NULL_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_NULL_DATA_TYPE))
-#define GARROW_IS_NULL_DATA_TYPE_CLASS(klass)           \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_NULL_DATA_TYPE))
-#define GARROW_NULL_DATA_TYPE_GET_CLASS(obj)                    \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_NULL_DATA_TYPE,        \
-                             GArrowNullDataTypeClass))
-
-typedef struct _GArrowNullDataType         GArrowNullDataType;
-typedef struct _GArrowNullDataTypeClass    GArrowNullDataTypeClass;
-
-/**
- * GArrowNullDataType:
- *
- * It wraps `arrow::NullType`.
- */
-struct _GArrowNullDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowNullDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType               garrow_null_data_type_get_type (void) G_GNUC_CONST;
-GArrowNullDataType *garrow_null_data_type_new      (void);
-
-
-#define GARROW_TYPE_BOOLEAN_DATA_TYPE           \
-  (garrow_boolean_data_type_get_type())
-#define GARROW_BOOLEAN_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BOOLEAN_DATA_TYPE,    \
-                              GArrowBooleanDataType))
-#define GARROW_BOOLEAN_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BOOLEAN_DATA_TYPE,       \
-                           GArrowBooleanDataTypeClass))
-#define GARROW_IS_BOOLEAN_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BOOLEAN_DATA_TYPE))
-#define GARROW_IS_BOOLEAN_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BOOLEAN_DATA_TYPE))
-#define GARROW_BOOLEAN_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BOOLEAN_DATA_TYPE,     \
-                             GArrowBooleanDataTypeClass))
-
-typedef struct _GArrowBooleanDataType         GArrowBooleanDataType;
-typedef struct _GArrowBooleanDataTypeClass    GArrowBooleanDataTypeClass;
-
-/**
- * GArrowBooleanDataType:
- *
- * It wraps `arrow::BooleanType`.
- */
-struct _GArrowBooleanDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowBooleanDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                  garrow_boolean_data_type_get_type (void) G_GNUC_CONST;
-GArrowBooleanDataType *garrow_boolean_data_type_new      (void);
-
-
-#define GARROW_TYPE_INT8_DATA_TYPE            \
-  (garrow_int8_data_type_get_type())
-#define GARROW_INT8_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
-                              GARROW_TYPE_INT8_DATA_TYPE,     \
-                              GArrowInt8DataType))
-#define GARROW_INT8_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
-                           GARROW_TYPE_INT8_DATA_TYPE,        \
-                           GArrowInt8DataTypeClass))
-#define GARROW_IS_INT8_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
-                              GARROW_TYPE_INT8_DATA_TYPE))
-#define GARROW_IS_INT8_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
-                           GARROW_TYPE_INT8_DATA_TYPE))
-#define GARROW_INT8_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
-                             GARROW_TYPE_INT8_DATA_TYPE,      \
-                             GArrowInt8DataTypeClass))
-
-typedef struct _GArrowInt8DataType         GArrowInt8DataType;
-typedef struct _GArrowInt8DataTypeClass    GArrowInt8DataTypeClass;
-
-/**
- * GArrowInt8DataType:
- *
- * It wraps `arrow::Int8Type`.
- */
-struct _GArrowInt8DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowInt8DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_int8_data_type_get_type (void) G_GNUC_CONST;
-GArrowInt8DataType   *garrow_int8_data_type_new      (void);
-
-
-#define GARROW_TYPE_UINT8_DATA_TYPE            \
-  (garrow_uint8_data_type_get_type())
-#define GARROW_UINT8_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
-                              GARROW_TYPE_UINT8_DATA_TYPE,     \
-                              GArrowUInt8DataType))
-#define GARROW_UINT8_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
-                           GARROW_TYPE_UINT8_DATA_TYPE,        \
-                           GArrowUInt8DataTypeClass))
-#define GARROW_IS_UINT8_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
-                              GARROW_TYPE_UINT8_DATA_TYPE))
-#define GARROW_IS_UINT8_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
-                           GARROW_TYPE_UINT8_DATA_TYPE))
-#define GARROW_UINT8_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
-                             GARROW_TYPE_UINT8_DATA_TYPE,      \
-                             GArrowUInt8DataTypeClass))
-
-typedef struct _GArrowUInt8DataType         GArrowUInt8DataType;
-typedef struct _GArrowUInt8DataTypeClass    GArrowUInt8DataTypeClass;
-
-/**
- * GArrowUInt8DataType:
- *
- * It wraps `arrow::UInt8Type`.
- */
-struct _GArrowUInt8DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowUInt8DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_uint8_data_type_get_type (void) G_GNUC_CONST;
-GArrowUInt8DataType  *garrow_uint8_data_type_new      (void);
-
-
-#define GARROW_TYPE_INT16_DATA_TYPE            \
-  (garrow_int16_data_type_get_type())
-#define GARROW_INT16_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
-                              GARROW_TYPE_INT16_DATA_TYPE,     \
-                              GArrowInt16DataType))
-#define GARROW_INT16_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
-                           GARROW_TYPE_INT16_DATA_TYPE,        \
-                           GArrowInt16DataTypeClass))
-#define GARROW_IS_INT16_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
-                              GARROW_TYPE_INT16_DATA_TYPE))
-#define GARROW_IS_INT16_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
-                           GARROW_TYPE_INT16_DATA_TYPE))
-#define GARROW_INT16_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
-                             GARROW_TYPE_INT16_DATA_TYPE,      \
-                             GArrowInt16DataTypeClass))
-
-typedef struct _GArrowInt16DataType         GArrowInt16DataType;
-typedef struct _GArrowInt16DataTypeClass    GArrowInt16DataTypeClass;
-
-/**
- * GArrowInt16DataType:
- *
- * It wraps `arrow::Int16Type`.
- */
-struct _GArrowInt16DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowInt16DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_int16_data_type_get_type (void) G_GNUC_CONST;
-GArrowInt16DataType  *garrow_int16_data_type_new      (void);
-
-
-#define GARROW_TYPE_UINT16_DATA_TYPE            \
-  (garrow_uint16_data_type_get_type())
-#define GARROW_UINT16_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT16_DATA_TYPE,     \
-                              GArrowUInt16DataType))
-#define GARROW_UINT16_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT16_DATA_TYPE,        \
-                           GArrowUInt16DataTypeClass))
-#define GARROW_IS_UINT16_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT16_DATA_TYPE))
-#define GARROW_IS_UINT16_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT16_DATA_TYPE))
-#define GARROW_UINT16_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT16_DATA_TYPE,      \
-                             GArrowUInt16DataTypeClass))
-
-typedef struct _GArrowUInt16DataType         GArrowUInt16DataType;
-typedef struct _GArrowUInt16DataTypeClass    GArrowUInt16DataTypeClass;
-
-/**
- * GArrowUInt16DataType:
- *
- * It wraps `arrow::UInt16Type`.
- */
-struct _GArrowUInt16DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowUInt16DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_uint16_data_type_get_type (void) G_GNUC_CONST;
-GArrowUInt16DataType *garrow_uint16_data_type_new      (void);
-
-
-#define GARROW_TYPE_INT32_DATA_TYPE            \
-  (garrow_int32_data_type_get_type())
-#define GARROW_INT32_DATA_TYPE(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT32_DATA_TYPE,      \
-                              GArrowInt32DataType))
-#define GARROW_INT32_DATA_TYPE_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT32_DATA_TYPE, \
-                           GArrowInt32DataTypeClass))
-#define GARROW_IS_INT32_DATA_TYPE(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT32_DATA_TYPE))
-#define GARROW_IS_INT32_DATA_TYPE_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT32_DATA_TYPE))
-#define GARROW_INT32_DATA_TYPE_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT32_DATA_TYPE,       \
-                             GArrowInt32DataTypeClass))
-
-typedef struct _GArrowInt32DataType         GArrowInt32DataType;
-typedef struct _GArrowInt32DataTypeClass    GArrowInt32DataTypeClass;
-
-/**
- * GArrowInt32DataType:
- *
- * It wraps `arrow::Int32Type`.
- */
-struct _GArrowInt32DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowInt32DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_int32_data_type_get_type (void) G_GNUC_CONST;
-GArrowInt32DataType  *garrow_int32_data_type_new      (void);
-
-
-#define GARROW_TYPE_UINT32_DATA_TYPE            \
-  (garrow_uint32_data_type_get_type())
-#define GARROW_UINT32_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT32_DATA_TYPE,     \
-                              GArrowUInt32DataType))
-#define GARROW_UINT32_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT32_DATA_TYPE,        \
-                           GArrowUInt32DataTypeClass))
-#define GARROW_IS_UINT32_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT32_DATA_TYPE))
-#define GARROW_IS_UINT32_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT32_DATA_TYPE))
-#define GARROW_UINT32_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT32_DATA_TYPE,      \
-                             GArrowUInt32DataTypeClass))
-
-typedef struct _GArrowUInt32DataType         GArrowUInt32DataType;
-typedef struct _GArrowUInt32DataTypeClass    GArrowUInt32DataTypeClass;
-
-/**
- * GArrowUInt32DataType:
- *
- * It wraps `arrow::UInt32Type`.
- */
-struct _GArrowUInt32DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowUInt32DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_uint32_data_type_get_type (void) G_GNUC_CONST;
-GArrowUInt32DataType *garrow_uint32_data_type_new      (void);
-
-
-#define GARROW_TYPE_INT64_DATA_TYPE            \
-  (garrow_int64_data_type_get_type())
-#define GARROW_INT64_DATA_TYPE(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT64_DATA_TYPE,      \
-                              GArrowInt64DataType))
-#define GARROW_INT64_DATA_TYPE_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT64_DATA_TYPE, \
-                           GArrowInt64DataTypeClass))
-#define GARROW_IS_INT64_DATA_TYPE(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT64_DATA_TYPE))
-#define GARROW_IS_INT64_DATA_TYPE_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT64_DATA_TYPE))
-#define GARROW_INT64_DATA_TYPE_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT64_DATA_TYPE,       \
-                             GArrowInt64DataTypeClass))
-
-typedef struct _GArrowInt64DataType         GArrowInt64DataType;
-typedef struct _GArrowInt64DataTypeClass    GArrowInt64DataTypeClass;
-
-/**
- * GArrowInt64DataType:
- *
- * It wraps `arrow::Int64Type`.
- */
-struct _GArrowInt64DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowInt64DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_int64_data_type_get_type (void) G_GNUC_CONST;
-GArrowInt64DataType  *garrow_int64_data_type_new      (void);
-
-
-#define GARROW_TYPE_UINT64_DATA_TYPE            \
-  (garrow_uint64_data_type_get_type())
-#define GARROW_UINT64_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT64_DATA_TYPE,     \
-                              GArrowUInt64DataType))
-#define GARROW_UINT64_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT64_DATA_TYPE,        \
-                           GArrowUInt64DataTypeClass))
-#define GARROW_IS_UINT64_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT64_DATA_TYPE))
-#define GARROW_IS_UINT64_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT64_DATA_TYPE))
-#define GARROW_UINT64_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT64_DATA_TYPE,      \
-                             GArrowUInt64DataTypeClass))
-
-typedef struct _GArrowUInt64DataType         GArrowUInt64DataType;
-typedef struct _GArrowUInt64DataTypeClass    GArrowUInt64DataTypeClass;
-
-/**
- * GArrowUInt64DataType:
- *
- * It wraps `arrow::UInt64Type`.
- */
-struct _GArrowUInt64DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowUInt64DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_uint64_data_type_get_type (void) G_GNUC_CONST;
-GArrowUInt64DataType *garrow_uint64_data_type_new      (void);
-
-
-#define GARROW_TYPE_FLOAT_DATA_TYPE           \
-  (garrow_float_data_type_get_type())
-#define GARROW_FLOAT_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
-                              GARROW_TYPE_FLOAT_DATA_TYPE,    \
-                              GArrowFloatDataType))
-#define GARROW_FLOAT_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
-                           GARROW_TYPE_FLOAT_DATA_TYPE,       \
-                           GArrowFloatDataTypeClass))
-#define GARROW_IS_FLOAT_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
-                              GARROW_TYPE_FLOAT_DATA_TYPE))
-#define GARROW_IS_FLOAT_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
-                           GARROW_TYPE_FLOAT_DATA_TYPE))
-#define GARROW_FLOAT_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
-                             GARROW_TYPE_FLOAT_DATA_TYPE,     \
-                             GArrowFloatDataTypeClass))
-
-typedef struct _GArrowFloatDataType         GArrowFloatDataType;
-typedef struct _GArrowFloatDataTypeClass    GArrowFloatDataTypeClass;
-
-/**
- * GArrowFloatDataType:
- *
- * It wraps `arrow::FloatType`.
- */
-struct _GArrowFloatDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowFloatDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                garrow_float_data_type_get_type (void) G_GNUC_CONST;
-GArrowFloatDataType *garrow_float_data_type_new      (void);
-
-
-#define GARROW_TYPE_DOUBLE_DATA_TYPE           \
-  (garrow_double_data_type_get_type())
-#define GARROW_DOUBLE_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_DOUBLE_DATA_TYPE,     \
-                              GArrowDoubleDataType))
-#define GARROW_DOUBLE_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_DOUBLE_DATA_TYPE,        \
-                           GArrowDoubleDataTypeClass))
-#define GARROW_IS_DOUBLE_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DOUBLE_DATA_TYPE))
-#define GARROW_IS_DOUBLE_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_DOUBLE_DATA_TYPE))
-#define GARROW_DOUBLE_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_DOUBLE_DATA_TYPE,      \
-                             GArrowDoubleDataTypeClass))
-
-typedef struct _GArrowDoubleDataType         GArrowDoubleDataType;
-typedef struct _GArrowDoubleDataTypeClass    GArrowDoubleDataTypeClass;
-
-/**
- * GArrowDoubleDataType:
- *
- * It wraps `arrow::DoubleType`.
- */
-struct _GArrowDoubleDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowDoubleDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_double_data_type_get_type (void) G_GNUC_CONST;
-GArrowDoubleDataType *garrow_double_data_type_new      (void);
-
-
-#define GARROW_TYPE_BINARY_DATA_TYPE            \
-  (garrow_binary_data_type_get_type())
-#define GARROW_BINARY_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BINARY_DATA_TYPE,     \
-                              GArrowBinaryDataType))
-#define GARROW_BINARY_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BINARY_DATA_TYPE,        \
-                           GArrowBinaryDataTypeClass))
-#define GARROW_IS_BINARY_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BINARY_DATA_TYPE))
-#define GARROW_IS_BINARY_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BINARY_DATA_TYPE))
-#define GARROW_BINARY_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BINARY_DATA_TYPE,      \
-                             GArrowBinaryDataTypeClass))
-
-typedef struct _GArrowBinaryDataType         GArrowBinaryDataType;
-typedef struct _GArrowBinaryDataTypeClass    GArrowBinaryDataTypeClass;
-
-/**
- * GArrowBinaryDataType:
- *
- * It wraps `arrow::BinaryType`.
- */
-struct _GArrowBinaryDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowBinaryDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_binary_data_type_get_type (void) G_GNUC_CONST;
-GArrowBinaryDataType *garrow_binary_data_type_new      (void);
-
-
-#define GARROW_TYPE_STRING_DATA_TYPE            \
-  (garrow_string_data_type_get_type())
-#define GARROW_STRING_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
-                              GARROW_TYPE_STRING_DATA_TYPE,    \
-                              GArrowStringDataType))
-#define GARROW_STRING_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
-                           GARROW_TYPE_STRING_DATA_TYPE,       \
-                           GArrowStringDataTypeClass))
-#define GARROW_IS_STRING_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
-                              GARROW_TYPE_STRING_DATA_TYPE))
-#define GARROW_IS_STRING_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
-                           GARROW_TYPE_STRING_DATA_TYPE))
-#define GARROW_STRING_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
-                             GARROW_TYPE_STRING_DATA_TYPE,     \
-                             GArrowStringDataTypeClass))
-
-typedef struct _GArrowStringDataType         GArrowStringDataType;
-typedef struct _GArrowStringDataTypeClass    GArrowStringDataTypeClass;
-
-/**
- * GArrowStringDataType:
- *
- * It wraps `arrow::StringType`.
- */
-struct _GArrowStringDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowStringDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_string_data_type_get_type (void) G_GNUC_CONST;
-GArrowStringDataType *garrow_string_data_type_new      (void);
-
-
-#define GARROW_TYPE_DATE32_DATA_TYPE           \
-  (garrow_date32_data_type_get_type())
-#define GARROW_DATE32_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_DATE32_DATA_TYPE,     \
-                              GArrowDate32DataType))
-#define GARROW_DATE32_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_DATE32_DATA_TYPE,        \
-                           GArrowDate32DataTypeClass))
-#define GARROW_IS_DATE32_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DATE32_DATA_TYPE))
-#define GARROW_IS_DATE32_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_DATE32_DATA_TYPE))
-#define GARROW_DATE32_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_DATE32_DATA_TYPE,      \
-                             GArrowDate32DataTypeClass))
-
-typedef struct _GArrowDate32DataType         GArrowDate32DataType;
-typedef struct _GArrowDate32DataTypeClass    GArrowDate32DataTypeClass;
-
-/**
- * GArrowDate32DataType:
- *
- * It wraps `arrow::Date32Type`.
- */
-struct _GArrowDate32DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowDate32DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_date32_data_type_get_type (void) G_GNUC_CONST;
-GArrowDate32DataType *garrow_date32_data_type_new      (void);
-
-
-#define GARROW_TYPE_DATE64_DATA_TYPE           \
-  (garrow_date64_data_type_get_type())
-#define GARROW_DATE64_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_DATE64_DATA_TYPE,     \
-                              GArrowDate64DataType))
-#define GARROW_DATE64_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_DATE64_DATA_TYPE,        \
-                           GArrowDate64DataTypeClass))
-#define GARROW_IS_DATE64_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DATE64_DATA_TYPE))
-#define GARROW_IS_DATE64_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_DATE64_DATA_TYPE))
-#define GARROW_DATE64_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_DATE64_DATA_TYPE,      \
-                             GArrowDate64DataTypeClass))
-
-typedef struct _GArrowDate64DataType         GArrowDate64DataType;
-typedef struct _GArrowDate64DataTypeClass    GArrowDate64DataTypeClass;
-
-/**
- * GArrowDate64DataType:
- *
- * It wraps `arrow::Date64Type`.
- */
-struct _GArrowDate64DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowDate64DataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_date64_data_type_get_type (void) G_GNUC_CONST;
-GArrowDate64DataType *garrow_date64_data_type_new      (void);
-
-
-#define GARROW_TYPE_TIMESTAMP_DATA_TYPE         \
-  (garrow_timestamp_data_type_get_type())
-#define GARROW_TIMESTAMP_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_TIMESTAMP_DATA_TYPE,  \
-                              GArrowTimestampDataType))
-#define GARROW_TIMESTAMP_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_TIMESTAMP_DATA_TYPE,     \
-                           GArrowTimestampDataTypeClass))
-#define GARROW_IS_TIMESTAMP_DATA_TYPE(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIMESTAMP_DATA_TYPE))
-#define GARROW_IS_TIMESTAMP_DATA_TYPE_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_TIMESTAMP_DATA_TYPE))
-#define GARROW_TIMESTAMP_DATA_TYPE_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_TIMESTAMP_DATA_TYPE,   \
-                             GArrowTimestampDataTypeClass))
-
-typedef struct _GArrowTimestampDataType         GArrowTimestampDataType;
-typedef struct _GArrowTimestampDataTypeClass    GArrowTimestampDataTypeClass;
-
-/**
- * GArrowTimestampDataType:
- *
- * It wraps `arrow::TimestampType`.
- */
-struct _GArrowTimestampDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowTimestampDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_timestamp_data_type_get_type (void) G_GNUC_CONST;
-GArrowTimestampDataType *garrow_timestamp_data_type_new   (GArrowTimeUnit unit);
-
-
-#define GARROW_TYPE_TIME_DATA_TYPE              \
-  (garrow_time_data_type_get_type())
-#define GARROW_TIME_DATA_TYPE(obj)                              \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_TIME_DATA_TYPE,       \
-                              GArrowTimeDataType))
-#define GARROW_TIME_DATA_TYPE_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_TIME_DATA_TYPE,  \
-                           GArrowTimeDataTypeClass))
-#define GARROW_IS_TIME_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIME_DATA_TYPE))
-#define GARROW_IS_TIME_DATA_TYPE_CLASS(klass)           \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_TIME_DATA_TYPE))
-#define GARROW_TIME_DATA_TYPE_GET_CLASS(obj)                    \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_TIME_DATA_TYPE,        \
-                             GArrowTimeDataTypeClass))
-
-typedef struct _GArrowTimeDataType         GArrowTimeDataType;
-typedef struct _GArrowTimeDataTypeClass    GArrowTimeDataTypeClass;
-
-/**
- * GArrowTimeDataType:
- *
- * It wraps `arrow::TimeType`.
- */
-struct _GArrowTimeDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowTimeDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType          garrow_time_data_type_get_type (void) G_GNUC_CONST;
-GArrowTimeUnit garrow_time_data_type_get_unit (GArrowTimeDataType *time_data_type);
-
-
-#define GARROW_TYPE_TIME32_DATA_TYPE           \
-  (garrow_time32_data_type_get_type())
-#define GARROW_TIME32_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_TIME32_DATA_TYPE,     \
-                              GArrowTime32DataType))
-#define GARROW_TIME32_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_TIME32_DATA_TYPE,        \
-                           GArrowTime32DataTypeClass))
-#define GARROW_IS_TIME32_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIME32_DATA_TYPE))
-#define GARROW_IS_TIME32_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_TIME32_DATA_TYPE))
-#define GARROW_TIME32_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_TIME32_DATA_TYPE,      \
-                             GArrowTime32DataTypeClass))
-
-typedef struct _GArrowTime32DataType         GArrowTime32DataType;
-typedef struct _GArrowTime32DataTypeClass    GArrowTime32DataTypeClass;
-
-/**
- * GArrowTime32DataType:
- *
- * It wraps `arrow::Time32Type`.
- */
-struct _GArrowTime32DataType
-{
-  /*< private >*/
-  GArrowTimeDataType parent_instance;
-};
-
-struct _GArrowTime32DataTypeClass
-{
-  GArrowTimeDataTypeClass parent_class;
-};
-
-GType                 garrow_time32_data_type_get_type (void) G_GNUC_CONST;
-GArrowTime32DataType *garrow_time32_data_type_new      (GArrowTimeUnit unit,
-                                                        GError **error);
-
-
-#define GARROW_TYPE_TIME64_DATA_TYPE           \
-  (garrow_time64_data_type_get_type())
-#define GARROW_TIME64_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_TIME64_DATA_TYPE,     \
-                              GArrowTime64DataType))
-#define GARROW_TIME64_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_TIME64_DATA_TYPE,        \
-                           GArrowTime64DataTypeClass))
-#define GARROW_IS_TIME64_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIME64_DATA_TYPE))
-#define GARROW_IS_TIME64_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_TIME64_DATA_TYPE))
-#define GARROW_TIME64_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_TIME64_DATA_TYPE,      \
-                             GArrowTime64DataTypeClass))
-
-typedef struct _GArrowTime64DataType         GArrowTime64DataType;
-typedef struct _GArrowTime64DataTypeClass    GArrowTime64DataTypeClass;
-
-/**
- * GArrowTime64DataType:
- *
- * It wraps `arrow::Time64Type`.
- */
-struct _GArrowTime64DataType
-{
-  /*< private >*/
-  GArrowTimeDataType parent_instance;
-};
-
-struct _GArrowTime64DataTypeClass
-{
-  GArrowTimeDataTypeClass parent_class;
-};
-
-GType                 garrow_time64_data_type_get_type (void) G_GNUC_CONST;
-GArrowTime64DataType *garrow_time64_data_type_new      (GArrowTimeUnit unit,
-                                                        GError **error);
-
-
-#define GARROW_TYPE_LIST_DATA_TYPE              \
-  (garrow_list_data_type_get_type())
-#define GARROW_LIST_DATA_TYPE(obj)                              \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_LIST_DATA_TYPE,       \
-                              GArrowListDataType))
-#define GARROW_LIST_DATA_TYPE_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_LIST_DATA_TYPE,  \
-                           GArrowListDataTypeClass))
-#define GARROW_IS_LIST_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_LIST_DATA_TYPE))
-#define GARROW_IS_LIST_DATA_TYPE_CLASS(klass)           \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_LIST_DATA_TYPE))
-#define GARROW_LIST_DATA_TYPE_GET_CLASS(obj)                    \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_LIST_DATA_TYPE,        \
-                             GArrowListDataTypeClass))
-
-typedef struct _GArrowListDataType         GArrowListDataType;
-typedef struct _GArrowListDataTypeClass    GArrowListDataTypeClass;
-
-/**
- * GArrowListDataType:
- *
- * It wraps `arrow::ListType`.
- */
-struct _GArrowListDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowListDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType               garrow_list_data_type_get_type (void) G_GNUC_CONST;
-GArrowListDataType *garrow_list_data_type_new      (GArrowField *field);
-GArrowField *garrow_list_data_type_get_value_field (GArrowListDataType *list_data_type);
-
-
-#define GARROW_TYPE_STRUCT_DATA_TYPE            \
-  (garrow_struct_data_type_get_type())
-#define GARROW_STRUCT_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_STRUCT_DATA_TYPE,     \
-                              GArrowStructDataType))
-#define GARROW_STRUCT_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_STRUCT_DATA_TYPE,        \
-                           GArrowStructDataTypeClass))
-#define GARROW_IS_STRUCT_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STRUCT_DATA_TYPE))
-#define GARROW_IS_STRUCT_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_STRUCT_DATA_TYPE))
-#define GARROW_STRUCT_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_STRUCT_DATA_TYPE,      \
-                             GArrowStructDataTypeClass))
-
-typedef struct _GArrowStructDataType         GArrowStructDataType;
-typedef struct _GArrowStructDataTypeClass    GArrowStructDataTypeClass;
-
-/**
- * GArrowStructDataType:
- *
- * It wraps `arrow::StructType`.
- */
-struct _GArrowStructDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
-struct _GArrowStructDataTypeClass
-{
-  GArrowDataTypeClass parent_class;
-};
-
-GType                 garrow_struct_data_type_get_type (void) G_GNUC_CONST;
-GArrowStructDataType *garrow_struct_data_type_new      (GList *fields);
-
-G_END_DECLS
+#include <arrow-glib/basic-data-type.h>
+#include <arrow-glib/composite-data-type.h>
diff --git a/c_glib/arrow-glib/data-type.hpp b/c_glib/arrow-glib/data-type.hpp
index fddcb2eb1ac59..77ab90321d7d8 100644
--- a/c_glib/arrow-glib/data-type.hpp
+++ b/c_glib/arrow-glib/data-type.hpp
@@ -19,9 +19,5 @@
 
 #pragma once
 
-#include <arrow/api.h>
-
-#include <arrow-glib/data-type.h>
-
-GArrowDataType *garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type);
-std::shared_ptr<arrow::DataType> garrow_data_type_get_raw(GArrowDataType *data_type);
+#include <arrow-glib/basic-data-type.hpp>
+#include <arrow-glib/composite-data-type.h>
diff --git a/c_glib/arrow-glib/field.h b/c_glib/arrow-glib/field.h
index e724dce49da5c..b1844f49a6edb 100644
--- a/c_glib/arrow-glib/field.h
+++ b/c_glib/arrow-glib/field.h
@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <arrow-glib/data-type.h>
+#include <arrow-glib/basic-data-type.h>
 
 G_BEGIN_DECLS
 
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 9c0475cc76c43..3366fc8ebe54a 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -46,6 +46,10 @@
       <title>Array builder</title>
       <xi:include href="xml/array-builder.xml"/>
     </chapter>
+    <chapter id="compute">
+      <title>Compute</title>
+      <xi:include href="xml/compute.xml"/>
+    </chapter>
     <chapter id="tensor">
       <title>Tensor</title>
       <xi:include href="xml/tensor.xml"/>
@@ -53,7 +57,8 @@
     <chapter id="type">
       <title>Type</title>
       <xi:include href="xml/type.xml"/>
-      <xi:include href="xml/data-type.xml"/>
+      <xi:include href="xml/basic-data-type.xml"/>
+      <xi:include href="xml/composite-data-type.xml"/>
     </chapter>
     <chapter id="schema">
       <title>Schema</title>

From 3fbf760413681889f2d9335417fca4a60d0c73a4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Sep 2017 13:10:05 -0400
Subject: [PATCH 1049/1644] ARROW-1530: [C++] Install arrow/util/parallel.h

We can use this in parquet-cpp or other places where a thread pool executor would be useful

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1093 from wesm/install-util-parallel and squashes the following commits:

1d7aeaf7 [Wes McKinney] Install arrow/util/parallel.h
---
 cpp/src/arrow/util/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 44be1c92ae64a..1178c658c4800 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -37,6 +37,7 @@ install(FILES
   key_value_metadata.h
   logging.h
   macros.h
+  parallel.h
   random.h
   rle-encoding.h
   sse-util.h

From cf1ac9cf4d82befadec3e40386daeff6aab0de90 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Tue, 12 Sep 2017 17:24:19 -0400
Subject: [PATCH 1050/1644] ARROW-1531: [C++] Return ToBytes by value from
 Decimal128

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1094 from cpcloud/fix-decimal128-to-bytes and squashes the following commits:

622fd99f [Phillip Cloud] ARROW-1531: [C++] Use forward references for appending std::array in FixedSizeBinaryBuilder
---
 cpp/src/arrow/array-test.cc        |  4 +---
 cpp/src/arrow/buffer.h             |  2 +-
 cpp/src/arrow/builder.cc           |  4 +---
 cpp/src/arrow/builder.h            |  1 +
 cpp/src/arrow/util/decimal-test.cc | 11 +++--------
 cpp/src/arrow/util/decimal.cc      | 11 ++++-------
 cpp/src/arrow/util/decimal.h       | 10 ++++------
 7 files changed, 15 insertions(+), 28 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 39db715b27080..97310830ddc7c 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -2577,10 +2577,8 @@ class DecimalTest : public ::testing::TestWithParam<int> {
   void MakeData(const DecimalVector& input, std::vector<uint8_t>* out) const {
     out->reserve(input.size() * BYTE_WIDTH);
 
-    std::array<uint8_t, BYTE_WIDTH> bytes{{0}};
-
     for (const auto& value : input) {
-      ASSERT_OK(value.ToBytes(&bytes));
+      auto bytes = value.ToBytes();
       out->insert(out->end(), bytes.cbegin(), bytes.cend());
     }
   }
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 4c3bd79599241..d215267786050 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -219,7 +219,7 @@ class ARROW_EXPORT BufferBuilder {
   template <size_t NBYTES>
   Status Append(const std::array<uint8_t, NBYTES>& data) {
     constexpr auto nbytes = static_cast<int64_t>(NBYTES);
-    if (capacity_ < static_cast<int64_t>(nbytes) + size_) {
+    if (capacity_ < nbytes + size_) {
       int64_t new_capacity = BitUtil::NextPower2(nbytes + size_);
       RETURN_NOT_OK(Resize(new_capacity));
     }
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index b27b2c74d7b10..a194ab7457225 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -1119,9 +1119,7 @@ DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>
 
 Status DecimalBuilder::Append(const Decimal128& value) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
-  std::array<uint8_t, 16> bytes;
-  RETURN_NOT_OK(value.ToBytes(&bytes));
-  return FixedSizeBinaryBuilder::Append(bytes);
+  return FixedSizeBinaryBuilder::Append(value.ToBytes());
 }
 
 Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 80c63a5b67299..4d18a3e3bbf49 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_BUILDER_H
 #define ARROW_BUILDER_H
 
+#include <array>
 #include <cstdint>
 #include <functional>
 #include <limits>
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 6f5118ea34c56..565a1bbb939b6 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -101,9 +101,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
 TEST(DecimalTest, TestDecimal32SignedRoundTrip) {
   Decimal128 expected("-3402692");
 
-  std::array<uint8_t, 16> bytes;
-  ASSERT_OK(expected.ToBytes(&bytes));
-
+  auto bytes = expected.ToBytes();
   Decimal128 result(bytes.data());
   ASSERT_EQ(expected, result);
 }
@@ -113,9 +111,7 @@ TEST(DecimalTest, TestDecimal64SignedRoundTrip) {
   std::string string_value("-34034293045.921");
   ASSERT_OK(Decimal128::FromString(string_value, &expected));
 
-  std::array<uint8_t, 16> bytes;
-  ASSERT_OK(expected.ToBytes(&bytes));
-
+  auto bytes = expected.ToBytes();
   Decimal128 result(bytes.data());
 
   ASSERT_EQ(expected, result);
@@ -131,8 +127,7 @@ TEST(DecimalTest, TestDecimalStringAndBytesRoundTrip) {
 
   ASSERT_EQ(expected, expected_underlying_value);
 
-  std::array<uint8_t, 16> bytes;
-  ASSERT_OK(expected.ToBytes(&bytes));
+  auto bytes = expected.ToBytes();
 
   Decimal128 result(bytes.data());
 
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 123482e75a37d..49d5c024950f3 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -44,15 +44,12 @@ Decimal128::Decimal128(const uint8_t* bytes)
     : Decimal128(reinterpret_cast<const int64_t*>(bytes)[0],
                  reinterpret_cast<const uint64_t*>(bytes)[1]) {}
 
-Status Decimal128::ToBytes(std::array<uint8_t, 16>* out) const {
-  if (out == nullptr) {
-    return Status::Invalid("Cannot fill nullptr of bytes from Decimal128");
-  }
-
+std::array<uint8_t, 16> Decimal128::ToBytes() const {
   const uint64_t raw[] = {static_cast<uint64_t>(high_bits_), low_bits_};
   const auto* raw_data = reinterpret_cast<const uint8_t*>(raw);
-  std::copy(raw_data, raw_data + out->size(), out->begin());
-  return Status::OK();
+  std::array<uint8_t, 16> out{{0}};
+  std::copy(raw_data, raw_data + out.size(), out.begin());
+  return out;
 }
 
 std::string Decimal128::ToString(int precision, int scale) const {
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index ff30d441a38d0..a0dea09216a61 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -104,17 +104,15 @@ class ARROW_EXPORT Decimal128 {
   /// \brief Get the low bits of the two's complement representation of the number.
   uint64_t low_bits() const { return low_bits_; }
 
-  /// \brief Put the raw bytes of the value into a pointer to uint8_t.
-  Status ToBytes(std::array<uint8_t, 16>* out) const;
+  /// \brief Return the raw bytes of the value.
+  std::array<uint8_t, 16> ToBytes() const;
 
   /// \brief Convert the Decimal128 value to a base 10 decimal string with the given
-  /// precision
-  /// and scale.
+  /// precision and scale.
   std::string ToString(int precision, int scale) const;
 
   /// \brief Convert a decimal string to an Decimal128 value, optionally including
-  /// precision
-  /// and scale if they're passed in and not null.
+  /// precision and scale if they're passed in and not null.
   static Status FromString(const std::string& s, Decimal128* out,
                            int* precision = nullptr, int* scale = nullptr);
 

From 576fb6651257c7c967c613fd20232ee80a434f0e Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Tue, 12 Sep 2017 17:54:13 -0400
Subject: [PATCH 1051/1644] ARROW-1532: [Python] Referencing an Empty Schema
 causes a SegFault

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1096 from cpcloud/ARROW-1532 and squashes the following commits:

7f43c6c7 [Phillip Cloud] ARROW-1532: [Python] Referencing an Empty Schema causes a SegFault
---
 python/pyarrow/tests/test_table.py | 9 +++++++++
 python/pyarrow/types.pxi           | 2 ++
 2 files changed, 11 insertions(+)

diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 4d5cb364c695d..311d64e8a8878 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -285,3 +285,12 @@ def test_table_negative_indexing():
 def test_table_ctor_errors():
     with pytest.raises(ReferenceError):
         repr(pa.Table())
+    with pytest.raises(ReferenceError):
+        str(pa.Table())
+
+
+def test_schema_ctor_errors():
+    with pytest.raises(ReferenceError):
+        repr(pa.Schema())
+    with pytest.raises(ReferenceError):
+        str(pa.Schema())
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 56670f64a7872..b298e740250af 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -436,6 +436,8 @@ cdef class Schema:
         return pyarrow_wrap_schema(new_schema)
 
     def __str__(self):
+        self._check_null()
+
         cdef:
             PrettyPrintOptions options
             c_string result

From b671dccbffd69f4c2177ec469e3cd1369ede2af5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Sep 2017 17:58:08 -0400
Subject: [PATCH 1052/1644] [Release] Update CHANGELOG.md for 0.7.0

Change-Id: I29a15bf02595ac5de6615b97597bb3ff3c493f9a
---
 CHANGELOG.md | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 142 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8fa4e4242e598..4858df8854181 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,147 @@
   under the License.
 -->
 
+# Apache Arrow 0.7.0 (12 September 2017)
+
+## Bug
+
+* ARROW-1302 - C++: ${MAKE} variable not set sometimes on older MacOS installations
+* ARROW-1354 - [Python] Segfault in Table.from\_pandas with Mixed-Type Categories
+* ARROW-1357 - [Python] Data corruption in reading multi-file parquet dataset
+* ARROW-1363 - [C++] IPC writer sends buffer layout for dictionary rather than indices
+* ARROW-1365 - [Python] Remove usage of removed jemalloc\_memory\_pool in Python API docs
+* ARROW-1373 - [Java] Implement get<type>Buffer() methods at the ValueVector interface
+* ARROW-1375 - [C++] Visual Studio 2017 Appveyor builds failing
+* ARROW-1379 - [Java] maven dependency issues - both unused and undeclared
+* ARROW-1407 - Dictionaries can only hold a maximum of 4096 indices
+* ARROW-1411 - [Python] Booleans in Float Columns cause Segfault
+* ARROW-1414 - [GLib] Cast after status check
+* ARROW-1421 - [Python] pyarrow.serialize cannot serialize a Python dict input
+* ARROW-1426 - [Website] The title element of the top page is empty
+* ARROW-1429 - [Python] Error loading parquet file with \_metadata from HDFS
+* ARROW-1430 - [Python] flake8 warnings are not failing CI builds
+* ARROW-1434 - [C++/Python] pyarrow.Array.from\_pandas does not support datetime64[D] arrays
+* ARROW-1435 - [Python] PyArrow not propagating timezone information from Parquet to Python
+* ARROW-1439 - [Packaging] Automate updating RPM in RPM build
+* ARROW-1443 - [Java] Bug on ArrowBuf.setBytes with unsliced ByteBuffers
+* ARROW-1444 - BitVector.splitAndTransfer copies last byte incorrectly 
+* ARROW-1446 - Python: Writing more than 2^31 rows from pandas dataframe causes row count overflow error
+* ARROW-1450 - [Python] Raise proper error if custom serialization handler fails
+* ARROW-1452 - [C++] Make UNUSED macro name more unique so it does not conflict with thirdparty projects
+* ARROW-1453 - [Python] Implement WriteTensor for non-contiguous tensors
+* ARROW-1458 - [Python] Document that HadoopFileSystem.mkdir with create\_parents=False has no effect
+* ARROW-1459 - [Python] PyArrow fails to load partitioned parquet files with non-primitive types
+* ARROW-1461 - [C++] Disable builds using LLVM apt packages temporarily
+* ARROW-1467 - [JAVA]: Fix reset() and allocateNew() in Nullable Value Vectors template
+* ARROW-1490 - [Java] Allow Travis CI failures for JDK9 for now
+* ARROW-1493 - [C++] Flush the output stream at the end of each PrettyPrint function
+* ARROW-1495 - [C++] Store shared\_ptr to boxed arrays in RecordBatch
+* ARROW-1507 - [C++] arrow/compute/api.h can't be used without arrow/array.h
+* ARROW-1512 - [Docs] NumericArray has no member named 'raw\_data'
+* ARROW-1514 - [C++] Fix a typo in document
+* ARROW-1527 - Fix Travis JDK9 build
+* ARROW-1531 - [C++] Return ToBytes by value from Decimal128
+* ARROW-1532 - [Python] Referencing an Empty Schema causes a SegFault
+* ARROW-407 - BitVector.copyFromSafe() should re-allocate if necessary instead of returning false
+* ARROW-801 - [JAVA] Provide direct access to underlying buffer memory addresses in consistent way without generating garbage or large amount indirections
+
+## Improvement
+
+* ARROW-1307 - [Python] Add pandas serialization section + Feather API to Sphinx docs
+* ARROW-1317 - [Python] Add function to set Hadoop CLASSPATH 
+* ARROW-1331 - [Java] Refactor tests
+* ARROW-1339 - [C++] Use boost::filesystem for handling of platform-specific file path encodings
+* ARROW-1344 - [C++] Calling BufferOutputStream::Write after calling Finish crashes
+* ARROW-1348 - [C++/Python] Add release verification script for Windows
+* ARROW-1351 - Automate updating CHANGELOG.md as part of release scripts
+* ARROW-1352 - [Integration] Improve print formatting for producer, consumer line
+* ARROW-1355 - Make arrow buildable with java9
+* ARROW-1356 - [Website] Add new committers
+* ARROW-1358 - Update source release scripts to account for new SHA checksum policy
+* ARROW-1359 - [Python] Add Parquet writer option to normalize field names for use in Spark
+* ARROW-1366 - [Python] Add instructions for starting the Plasma store when installing pyarrow from wheels
+* ARROW-1372 - [Plasma] Support for storing data in huge pages
+* ARROW-1376 - [C++] RecordBatchStreamReader::Open API is inconsistent with writer
+* ARROW-1381 - [Python] Improve performance of SerializedPyObject.to\_buffer
+* ARROW-1383 - [C++] Support std::vector<bool> in builder vector appends
+* ARROW-1384 - [C++] Add convenience function for serializing a record batch to an IPC message 
+* ARROW-1386 - [C++] Unpin CMake version in MSVC build toolchain
+* ARROW-1395 - [C++] Remove APIs deprecated as of 0.5.0 and later versions
+* ARROW-1397 - [Packaging] Use Docker instead of Vagrant
+* ARROW-1401 - [C++] Add extra debugging context to failures in RETURN\_NOT\_OK in debug builds
+* ARROW-1402 - [C++] Possibly deprecate public APIs that use MutableBuffer
+* ARROW-1404 - [Packaging] Build .deb and .rpm on Travis CI
+* ARROW-1405 - [Python] Add logging option for verbose memory allocations
+* ARROW-1406 - [Python] Harden user API for generating serialized schema and record batch messages as memoryview-compatible objects
+* ARROW-1408 - [C++] Refactor and make IPC read / write APIs more consistent, add appropriate deprecations
+* ARROW-1410 - Plasma object store occasionally pauses for a long time
+* ARROW-1412 - [Plasma] Add higher level API for putting and getting Python objects
+* ARROW-1413 - [C++] Add include-what-you-use configuration
+* ARROW-1416 - [Format] Clarify example array in memory layout documentation
+* ARROW-1418 - [Python] Introduce SerializationContext to register custom serialization callbacks
+* ARROW-1419 - [GLib] Suppress sign-conversion warning on Clang
+* ARROW-1427 - [GLib] Add a link to readme of Arrow GLib
+* ARROW-1428 - [C++] Append steps to clone source code to README.mb
+* ARROW-1432 - [C++] Build bundled jemalloc functions with private prefix
+* ARROW-1433 - [C++] Simplify implementation of Array::Slice
+* ARROW-1438 - [Plasma] Pull SerializationContext through PlasmaClient put and get
+* ARROW-1441 - [Site] Add Ruby to Flexible section
+* ARROW-1442 - [Website] Add pointer to nightly conda packages on /install
+* ARROW-1447 - [C++] Round of include-what-you-use include cleanups
+* ARROW-1448 - [Packaging] Support uploading built .deb and .rpm to Bintray
+* ARROW-1449 - Implement Decimal using only Int128
+* ARROW-1451 - [C++] Create arrow/io/api.h
+* ARROW-1460 - [C++] Upgrade clang-format used to LLVM 4.0
+* ARROW-1466 - [C++] Support DecimalArray in arrow::PrettyPrint
+* ARROW-1468 - [C++] Append to PrimitiveBuilder from std::vector<CTYPE>
+* ARROW-1480 - [Python] Improve performance of serializing sets
+* ARROW-1494 - [C++] Document that shared\_ptr returned by RecordBatch::column needs to be retained
+* ARROW-1499 - [Python] Consider adding option to parquet.write\_table that sets options for maximum Spark compatibility
+* ARROW-1505 - [GLib] Simplify arguments check
+* ARROW-1506 - [C++] Support pkg-config for compute modules
+* ARROW-1508 - C++: Add support for FixedSizeBinaryType in DictionaryBuilder
+* ARROW-1511 - [C++] Deprecate arrow::MakePrimitiveArray
+* ARROW-1513 - C++: Add cast from Dictionary to plain arrays
+* ARROW-1515 - [GLib] Detect version directly
+* ARROW-1516 - [GLib] Update document
+* ARROW-1517 - Remove unnecessary temporary in DecimalUtil::ToString function
+* ARROW-1519 - [C++] Move DecimalUtil functions to methods on the Int128 class
+* ARROW-1528 - [GLib] Resolve include dependency
+* ARROW-1530 - [C++] Install arrow/util/parallel.h
+* ARROW-594 - [Python] Provide interface to write pyarrow.Table to a stream
+* ARROW-786 - [Format] In-memory format for 128-bit Decimals, handling of sign bit
+* ARROW-837 - [Python] Expose buffer allocation, FixedSizeBufferWriter
+* ARROW-941 - [Docs] Improve "cold start" integration testing instructions
+
+## New Feature
+
+* ARROW-1156 - [Python] pyarrow.Array.from\_pandas should take a type parameter
+* ARROW-1238 - [Java] Add JSON read/write support for decimals for integration tests
+* ARROW-1364 - [C++] IPC reader and writer specialized for GPU device memory
+* ARROW-1377 - [Python] Add function to assist with benchmarking Parquet scan performance
+* ARROW-1387 - [C++] Set up GPU leaf library build toolchain
+* ARROW-1392 - [C++] Implement reader and writer IO interfaces for GPU buffers
+* ARROW-1396 - [C++] Add PrettyPrint function for Schemas, which also outputs any dictionaries
+* ARROW-1399 - [C++] Add CUDA build version in a public header to help prevent ABI conflicts
+* ARROW-1400 - [Python] Ability to create partitions when writing to Parquet
+* ARROW-1415 - [GLib] Support date32 and date64
+* ARROW-1417 - [Python] Allow more generic filesystem objects to be passed to ParquetDataset
+* ARROW-1462 - [GLib] Support time array
+* ARROW-1479 - [JS] Expand JavaScript implementation
+* ARROW-1481 - [C++] Expose type casts as generic callable object that can write into pre-allocated memory
+* ARROW-1504 - [GLib] Support timestamp
+* ARROW-1510 - [C++] Support cast
+* ARROW-229 - [C++] Implement safe casts for primitive types
+* ARROW-592 - [C++] Provide .deb and .rpm packages
+* ARROW-695 - Integration tests for Decimal types
+* ARROW-696 - [C++] Add JSON read/write support for decimals for integration tests
+* ARROW-759 - [Python] Implement a transient list serialization function that can handle a mix of scalars, lists, ndarrays, dicts
+* ARROW-989 - [Python] Write pyarrow.Table to FileWriter or StreamWriter
+
+## Test
+
+* ARROW-1390 - [Python] Extend tests for python serialization
+
 # Apache Arrow 0.6.0 (14 August 2017)
 
 ## Bug
@@ -1102,3 +1243,4 @@
 
 * ARROW-260 - TestValueVector.testFixedVectorReallocation and testVariableVectorReallocation are flaky
 * ARROW-83 - Add basic test infrastructure for DecimalType
+

From 97f9029ce835dfc2655ca91b9820a2e6aed89107 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Sep 2017 18:01:14 -0400
Subject: [PATCH 1053/1644] [maven-release-plugin] prepare release
 apache-arrow-0.7.0

Change-Id: Ib082c383446182630abbb6c8e2e154006ff4560f
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 647b716039e7b..684ed3c525454 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.7.0-SNAPSHOT</version>
+  <version>0.7.0</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 1c4eef71fc009..dcd3572f7bf06 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>0.7.0</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index c4c301d1a419e..acd562625faa5 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.7.0-SNAPSHOT</version>
+  <version>0.7.0</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -44,7 +44,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>HEAD</tag>
+    <tag>apache-arrow-0.7.0</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 48b507afdd1ca..a1a63082d1e6c 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.7.0-SNAPSHOT</version>
+        <version>0.7.0</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index bf33823fef52f..dd1a0ea55d92f 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>0.7.0</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From d0b0b74181fd94c22996014b9a13e953ed7bbadd Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 13 Sep 2017 09:09:38 -0400
Subject: [PATCH 1054/1644] ARROW-1529: [GLib] Use Xcode 8.3 on Travis CI

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1092 from kou/glib-travis-macos and squashes the following commits:

291808b2 [Kouhei Sutou] [GLib] Use Xcode 8.3 on Travis CI
---
 .travis.yml                       |  2 +-
 ci/travis_before_script_c_glib.sh | 13 +++++++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c4a8f4120c732..f6f64003d273d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -128,7 +128,7 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
   - compiler: clang
-    osx_image: xcode6.4
+    osx_image: xcode8.3
     os: osx
     cache:
     addons:
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 6799ae4ce41b5..533c892712dbb 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -23,7 +23,9 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
   brew install gtk-doc autoconf-archive gobject-introspection
-  brew upgrade git cmake wget libtool
+  brew upgrade git cmake
+  brew outdated || brew upgrade wget
+  brew outdated || brew upgrade libtool
 
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
 else
@@ -36,6 +38,7 @@ gem install test-unit gobject-introspection
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
   brew install lua
+  sudo env PKG_CONFIG_PATH=$PKG_CONFIG_PATH luarocks install lgi
 else
   git clone \
     --quiet \
@@ -47,8 +50,8 @@ else
   echo "yes" | ./install.sh > /dev/null
   . ~/torch/install/bin/torch-activate
   popd
+  luarocks install lgi
 fi
-luarocks install lgi
 
 go get github.com/linuxdeepin/go-gir-generator || :
 pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
@@ -74,6 +77,12 @@ pushd $ARROW_C_GLIB_DIR
 export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
 
+if [ $TRAVIS_OS_NAME == "osx" ]; then
+  install_name_tool \
+    -id $ARROW_CPP_INSTALL/lib/libarrow.dylib \
+    $ARROW_CPP_INSTALL/lib/libarrow.dylib
+fi
+
 CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"
 if [ $TRAVIS_OS_NAME != "osx" ]; then
   CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"

From 96d451f37eb23d9c9cdc997032d7534a585537ff Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 15 Sep 2017 10:47:04 -0400
Subject: [PATCH 1055/1644] ARROW-1537: [C++] Support building with full path
 install_name on macOS

If you use `@rpath` for install_name (default), you can use the
DYLD_LIBRARY_PATH environment variable to find libarrow.dylib. But the
DYLD_LIBRARY_PATH environment variable isn't inherited to sub process by
System Integration Protection (SIP). It's difficult to use
libarrow.dylib.

You can use full path install_name by -DARROW_INSTALL_NAME_RPATH=OFF
CMake option. If you use it, you can find libarrow.dylib without
DYLD_LIBRARY_PATH environment variable.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1100 from kou/cpp-macos-support-install-name and squashes the following commits:

8207ace [Kouhei Sutou] [C++] Support building with full path install_name on macOS
---
 ci/travis_before_script_c_glib.sh  | 6 ------
 ci/travis_before_script_cpp.sh     | 3 ++-
 cpp/CMakeLists.txt                 | 4 ++++
 cpp/cmake_modules/BuildUtils.cmake | 9 +++++++--
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 533c892712dbb..52bfe87eba75f 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -77,12 +77,6 @@ pushd $ARROW_C_GLIB_DIR
 export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
 
-if [ $TRAVIS_OS_NAME == "osx" ]; then
-  install_name_tool \
-    -id $ARROW_CPP_INSTALL/lib/libarrow.dylib \
-    $ARROW_CPP_INSTALL/lib/libarrow.dylib
-fi
-
 CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"
 if [ $TRAVIS_OS_NAME != "osx" ]; then
   CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index a613957ba0db6..d46fa2ff9433c 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -70,7 +70,8 @@ if [ $only_library_mode == "yes" ]; then
   CMAKE_COMMON_FLAGS="\
 $CMAKE_COMMON_FLAGS \
 -DARROW_BUILD_TESTS=OFF \
--DARROW_BUILD_UTILITIES=OFF"
+-DARROW_BUILD_UTILITIES=OFF \
+-DARROW_INSTALL_NAME_RPATH=OFF"
 fi
 
 # Use Ninja for faster builds when using toolchain
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 577a4bbea83fa..972132f29e901 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -150,6 +150,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build Arrow libraries with RATH set to \$ORIGIN"
     OFF)
 
+  option(ARROW_INSTALL_NAME_RPATH
+    "Build Arrow libraries with install_name set to @rpath"
+    ON)
+
   option(ARROW_PLASMA
     "Build the plasma object store along with Arrow"
     OFF)
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 6b2be41d24aa7..8f92d73baea31 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -176,10 +176,15 @@ function(ADD_ARROW_LIB LIB_NAME)
   endif()
 
   if (APPLE)
-      set_target_properties(${LIB_NAME}_shared
+    if (ARROW_INSTALL_NAME_RPATH)
+      set(_lib_install_name "@rpath")
+    else()
+      set(_lib_install_name "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+    endif()
+    set_target_properties(${LIB_NAME}_shared
       PROPERTIES
       BUILD_WITH_INSTALL_RPATH ON
-      INSTALL_NAME_DIR "@rpath")
+      INSTALL_NAME_DIR "${_lib_install_name}")
   endif()
 
 endfunction()

From 72ad07e96648cdc3bc7d2c6f48c732dbe3133080 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 15 Sep 2017 10:48:22 -0400
Subject: [PATCH 1056/1644] ARROW-1542: [C++] Install packages in temporary
 directory in MSVC build verification script

I found that the script did not work due to the remnants of the last time I ran it.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1101 from wesm/ARROW-1542 and squashes the following commits:

0718370 [Wes McKinney] Install packages in temporary directory in MSVC build verification script
---
 dev/release/verify-release-candidate.bat | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat
index 503f1fd40b668..27157da6687a2 100644
--- a/dev/release/verify-release-candidate.bat
+++ b/dev/release/verify-release-candidate.bat
@@ -45,15 +45,18 @@ tar xvf %1.tar.gz -C "C:/tmp/"
 set GENERATOR=Visual Studio 14 2015 Win64
 set CONFIGURATION=release
 set ARROW_SOURCE=C:\tmp\%1
+set INSTALL_DIR=C:\tmp\%1\install
 
 pushd %ARROW_SOURCE%
 
 call activate arrow-verify-release
 
 set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
-set ARROW_HOME=%CONDA_PREFIX%\Library
 set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
-set PARQUET_HOME=%CONDA_PREFIX%\Library
+
+set ARROW_HOME=%INSTALL_DIR%
+set PARQUET_HOME=%INSTALL_DIR%
+set PATH=%INSTALL_DIR%\bin;%PATH%
 
 @rem Build and test Arrow C++ libraries
 mkdir cpp\build

From 59b24ba3b81bb8ead4fa74ff15ebe633d408415e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 15 Sep 2017 12:29:10 -0400
Subject: [PATCH 1057/1644] ARROW-559: Add release verification script for
 Linux

Since we're accumulating a bunch of components, I started this script which we can refine to make verifying releases easier for others.

I bootstrapped some pieces off https://github.com/apache/parquet-cpp/blob/master/dev/release/verify-release-candidate, very helpful!

This script:

* Checks GPG signature, checksums
* Installs temporary Python install for the duration of these tests
* Builds/install C++ and runs tests (with Python and Plasma)
* Builds parquet-cpp against the Arrow RC
* Python (with Parquet and Plasma extensions)
* C GLib (requires Ruby in PATH and the gems indicated in README)
* Integration tests
* JavaScript (requires NodeJS >= 6.0.0)

There are some potentially snowflake-y aspects to my environment:

* BOOST_ROOT is set to a Boost install location containing libraries built with `-fPIC`. I'm not sure what to do about this one. One maybe better option is to use system level boost and shared libraries
* Maven 3.3.9 is in PATH
* NodeJS 6.11.3 is in PATH

There are probably some other things that Linux users will run into as they run this script.

I had to compile GLib libraries in this since the ones at system level (Ubuntu 14.04) are too old.

cc @kou @xhochy

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1102 from wesm/ARROW-559 and squashes the following commits:

8fd6530 [Wes McKinney] Use Boost shared libraries
3531927 [Wes McKinney] Add note to dev/README.md
079b5e4 [Wes McKinney] Fix comments
17f7ac0 [Wes McKinney] More fixes, finally works
adb3146 [Wes McKinney] More work on release verification script
86ef171 [Wes McKinney] Start Linux release verification script
---
 dev/README.md                           |  18 ++
 dev/release/verify-release-candidate.sh | 244 ++++++++++++++++++++++++
 2 files changed, 262 insertions(+)
 create mode 100755 dev/release/verify-release-candidate.sh

diff --git a/dev/README.md b/dev/README.md
index e986abef1913f..91d688dfee734 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -92,3 +92,21 @@ Merge hash: 485658a5
 Would you like to pick 485658a5 into another branch? (y/n):
 ```
 For now just say n as we have 1 branch
+
+## Verifying Release Candidates
+
+We have provided a script to assist with verifying release candidates:
+
+```shell
+bash dev/release/verify-release-candidate.sh 0.7.0 0
+```
+
+Currently this only works on Linux (patches to expand to macOS welcome!). Read
+the script for information about system dependencies.
+
+On Windows, we have a script that verifies C++ and Python (requires Visual
+Studio 2015):
+
+```
+dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
+```
\ No newline at end of file
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
new file mode 100755
index 0000000000000..38680f429ed77
--- /dev/null
+++ b/dev/release/verify-release-candidate.sh
@@ -0,0 +1,244 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Requirements
+# - Ruby 2.x
+#   - Plus gem dependencies, see c_glib/README
+# - Maven >= 3.3.9
+# - JDK >=7
+# - gcc >= 4.8
+# - nodejs >= 6.0.0 (best way is to use nvm)
+#
+# If using a non-system Boost, set BOOST_ROOT and add Boost libraries to
+# LD_LIBRARY_PATH
+
+case $# in
+  2) VERSION="$1"
+     RC_NUMBER="$2"
+     ;;
+
+  *) echo "Usage: $0 X.Y.Z RC_NUMBER"
+     exit 1
+     ;;
+esac
+
+set -ex
+
+HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
+
+ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
+
+download_dist_file() {
+  curl -f -O $ARROW_DIST_URL/$1
+}
+
+download_rc_file() {
+  download_dist_file apache-arrow-${VERSION}-rc${RC_NUMBER}/$1
+}
+
+import_gpg_keys() {
+  download_dist_file KEYS
+  gpg --import KEYS
+}
+
+fetch_archive() {
+  local dist_name=$1
+  download_rc_file ${dist_name}.tar.gz
+  download_rc_file ${dist_name}.tar.gz.asc
+  download_rc_file ${dist_name}.tar.gz.md5
+  download_rc_file ${dist_name}.tar.gz.sha512
+  gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
+  gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
+  sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+}
+
+setup_tempdir() {
+  cleanup() {
+    rm -fr "$TMPDIR"
+  }
+  trap cleanup EXIT
+  TMPDIR=$(mktemp -d -t "$1.XXXXX")
+}
+
+
+setup_miniconda() {
+  # Setup short-lived miniconda for Python and integration tests
+  MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+
+  MINICONDA=`pwd`/test-miniconda
+
+  wget -O miniconda.sh $MINICONDA_URL
+  bash miniconda.sh -b -p $MINICONDA
+  rm -f miniconda.sh
+
+  export PATH=$MINICONDA/bin:$PATH
+
+  conda create -n arrow-test -y -q python=3.6 \
+        nomkl numpy pandas six cython
+  source activate arrow-test
+}
+
+# Build and test C++
+
+test_and_install_cpp() {
+  mkdir cpp/build
+  pushd cpp/build
+
+  cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+        -DARROW_PLASMA=on \
+        -DARROW_PYTHON=on \
+        -DARROW_BOOST_USE_SHARED=on \
+        -DCMAKE_BUILD_TYPE=release \
+        -DARROW_BUILD_BENCHMARKS=on \
+        ..
+
+  make -j$NPROC
+  make install
+
+  ctest -L unittest
+  popd
+}
+
+# Build and install Parquet master so we can test the Python bindings
+
+install_parquet_cpp() {
+  git clone git@github.com:apache/parquet-cpp.git
+
+  mkdir parquet-cpp/build
+  pushd parquet-cpp/build
+
+  cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+        -DCMAKE_BUILD_TYPE=release \
+        -DPARQUET_BOOST_USE_SHARED=on \
+        -DPARQUET_BUILD_TESTS=off \
+        ..
+
+  make -j$NPROC
+  make install
+
+  popd
+}
+
+# Build and test Python
+
+test_python() {
+  pushd python
+
+  pip install -r requirements.txt
+
+  python setup.py build_ext --inplace --with-parquet --with-plasma
+  py.test pyarrow -v --pdb
+
+  popd
+}
+
+
+test_glib() {
+  # Build and test GLib, requires GLib >= 2.44 , so install that
+  # here
+  GLIB_VERSION=glib-2.53.7
+  GLIB_URL=https://gensho.ftp.acc.umu.se/pub/gnome/sources/glib/2.53/$GLIB_VERSION.tar.xz
+  curl -f -O $GLIB_URL
+  tar xf $GLIB_VERSION.tar.xz
+  pushd $GLIB_VERSION
+
+  ./configure --disable-libelf --enable-libmount=no --prefix=$ARROW_HOME
+  make -j$NPROC
+  make install
+
+  popd
+
+  pushd c_glib
+
+  ./configure --prefix=$ARROW_HOME
+  make -j$NPROC
+  make install
+
+  NO_MAKE=yes test/run-test.sh
+
+  popd
+}
+
+test_js() {
+  pushd js
+  npm install
+  npm run validate
+  popd
+}
+
+# Build and test Java (Requires newer Maven -- I used 3.3.9)
+
+test_package_java() {
+  pushd java
+
+  mvn test
+  mvn package
+
+  popd
+}
+
+# Run integration tests
+test_integration() {
+  JAVA_DIR=`pwd`/java
+  CPP_BUILD_DIR=`pwd`/cpp/build
+
+  export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
+  export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release
+
+  pushd integration
+
+  python integration_test.py
+
+  popd
+}
+
+setup_tempdir "arrow-$VERSION"
+echo "Working in sandbox $TMPDIR"
+cd $TMPDIR
+
+export ARROW_HOME=$TMPDIR/install
+export PARQUET_HOME=$TMPDIR/install
+export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH
+export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH
+
+NPROC=$(nproc)
+VERSION=$1
+RC_NUMBER=$2
+
+TARBALL=apache-arrow-$1.tar.gz
+
+import_gpg_keys
+
+DIST_NAME="apache-arrow-${VERSION}"
+fetch_archive $DIST_NAME
+tar xvzf ${DIST_NAME}.tar.gz
+cd ${DIST_NAME}
+
+setup_miniconda
+test_and_install_cpp
+install_parquet_cpp
+test_python
+test_glib
+test_package_java
+test_integration
+test_js
+
+echo 'Release candidate looks good!'
+exit 0

From bf73d2724cbdf402e2fbff5304fd9a842e9d6ddb Mon Sep 17 00:00:00 2001
From: rvernica <rvernica@gmail.com>
Date: Sun, 17 Sep 2017 09:11:04 -0700
Subject: [PATCH 1058/1644] ARROW-1545: Remove deprecated args of builder

Closes #1107

Change-Id: I9cb83279900aed8e04ef8baf049e30c5007e6538
---
 cpp/apidoc/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md
index 293241c0dc767..25be1f284fc4e 100644
--- a/cpp/apidoc/index.md
+++ b/cpp/apidoc/index.md
@@ -55,7 +55,7 @@ build these objects. To build an array of `int64_t` elements, we can use the
 `arrow::Int64Builder`. In the following example, we build an array of the range
 1 to 8 where the element that should hold the number 4 is nulled.
 
-    Int64Builder builder(arrow::default_memory_pool(), arrow::int64());
+    Int64Builder builder;
     builder.Append(1);
     builder.Append(2);
     builder.Append(3);

From bfe657909f5e7d96b7b8e5179baa17044b6ea375 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 17 Sep 2017 18:32:16 -0400
Subject: [PATCH 1059/1644] ARROW-1546: [GLib] Support GLib 2.40 again

Ubuntu 14.04 ships GLib 2.40.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1106 from kou/glib-support-glib-2.40-again and squashes the following commits:

cbcdf9a [Kouhei Sutou] [GLib] Support GLib 2.40 again
---
 c_glib/arrow-glib/Makefile.am           |  1 +
 c_glib/arrow-glib/arrow-glib.h          |  2 +
 c_glib/arrow-glib/compute.h             | 16 ++--
 c_glib/arrow-glib/gobject-type.h        | 97 +++++++++++++++++++++++++
 ci/travis_before_script_c_glib.sh       |  4 -
 dev/release/verify-release-candidate.sh | 14 ----
 6 files changed, 106 insertions(+), 28 deletions(-)
 create mode 100644 c_glib/arrow-glib/gobject-type.h

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index cde62cde08014..4c22d11c76ca9 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -52,6 +52,7 @@ libarrow_glib_la_headers =			\
 	data-type.h				\
 	error.h					\
 	field.h					\
+	gobject-type.h				\
 	record-batch.h				\
 	schema.h				\
 	table.h					\
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index f15f16db9a35d..47f324dc04640 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include <arrow-glib/gobject-type.h>
+
 #include <arrow-glib/array.h>
 #include <arrow-glib/array-builder.h>
 #include <arrow-glib/chunked-array.h>
diff --git a/c_glib/arrow-glib/compute.h b/c_glib/arrow-glib/compute.h
index 4b944ad30d320..cdae5790e8346 100644
--- a/c_glib/arrow-glib/compute.h
+++ b/c_glib/arrow-glib/compute.h
@@ -19,20 +19,16 @@
 
 #pragma once
 
-#include <glib-object.h>
+#include <arrow-glib/gobject-type.h>
 
 G_BEGIN_DECLS
 
 #define GARROW_TYPE_CAST_OPTIONS (garrow_cast_options_get_type())
-G_DECLARE_DERIVABLE_TYPE(GArrowCastOptions,
-                         garrow_cast_options,
-                         GARROW,
-                         CAST_OPTIONS,
-                         GObject)
-struct _GArrowCastOptionsClass
-{
-  GObjectClass parent_class;
-};
+GARROW_DECLARE_TYPE(GArrowCastOptions,
+                    garrow_cast_options,
+                    GARROW,
+                    CAST_OPTIONS,
+                    GObject)
 
 GArrowCastOptions *garrow_cast_options_new(void);
 
diff --git a/c_glib/arrow-glib/gobject-type.h b/c_glib/arrow-glib/gobject-type.h
new file mode 100644
index 0000000000000..a2f8397b941a9
--- /dev/null
+++ b/c_glib/arrow-glib/gobject-type.h
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <glib-object.h>
+
+#ifdef G_DECLARE_DERIVABLE_TYPE
+#  define GARROW_DECLARE_TYPE(ObjectName,       \
+                              object_name,      \
+                              MODULE_NAME,      \
+                              OBJECT_NAME,      \
+                              ParentName)       \
+  G_DECLARE_DERIVABLE_TYPE(ObjectName,          \
+                           object_name,         \
+                           MODULE_NAME,         \
+                           OBJECT_NAME,         \
+                           ParentName)          \
+  struct _ ## ObjectName ## Class               \
+  {                                             \
+    ParentName ## Class parent_class;           \
+  };
+#else
+#  define GARROW_DECLARE_TYPE(ObjectName,                               \
+                              object_name,                              \
+                              MODULE_NAME,                              \
+                              OBJECT_NAME,                              \
+                              ParentName)                               \
+  typedef struct _ ## ObjectName ObjectName;                            \
+  typedef struct _ ## ObjectName ## Class ObjectName ## Class;          \
+                                                                        \
+  struct _ ## ObjectName                                                \
+  {                                                                     \
+    ParentName parent_instance;                                         \
+  };                                                                    \
+                                                                        \
+  struct _ ## ObjectName ## Class                                       \
+  {                                                                     \
+    ParentName ## Class parent_class;                                   \
+  };                                                                    \
+                                                                        \
+  GType object_name ## _get_type(void) G_GNUC_CONST;                    \
+                                                                        \
+  static inline ObjectName *                                            \
+  MODULE_NAME ## _ ## OBJECT_NAME(gpointer object)                      \
+  {                                                                     \
+    return G_TYPE_CHECK_INSTANCE_CAST(object,                           \
+                                      object_name ## _get_type(),       \
+                                      ObjectName);                      \
+  }                                                                     \
+                                                                        \
+  static inline ObjectName ## Class *                                   \
+  MODULE_NAME ## _ ## OBJECT_NAME ## _CLASS(gpointer klass)             \
+  {                                                                     \
+    return G_TYPE_CHECK_CLASS_CAST(klass,                               \
+                                   object_name ## _get_type(),          \
+                                   ObjectName ## Class);                \
+  }                                                                     \
+                                                                        \
+  static inline gboolean                                                \
+  MODULE_NAME ## _IS_ ## OBJECT_NAME(gpointer object)                   \
+  {                                                                     \
+    return G_TYPE_CHECK_INSTANCE_TYPE(object,                           \
+                                      object_name ## _get_type());      \
+  }                                                                     \
+                                                                        \
+  static inline gboolean                                                \
+  MODULE_NAME ## _IS_ ## OBJECT_NAME ## _CLASS(gpointer klass)          \
+  {                                                                     \
+    return G_TYPE_CHECK_CLASS_TYPE(klass,                               \
+                                   object_name ## _get_type());         \
+  }                                                                     \
+                                                                        \
+  static inline ObjectName ## Class *                                   \
+  MODULE_NAME ## _ ## ObjectName ## _GET_CLASS(gpointer object)         \
+  {                                                                     \
+    return G_TYPE_INSTANCE_GET_CLASS(object,                            \
+                                     object_name ## _get_type(),        \
+                                     ObjectName ## Class);              \
+  }
+#endif
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 52bfe87eba75f..7100946d76ae9 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -28,10 +28,6 @@ if [ $TRAVIS_OS_NAME == "osx" ]; then
   brew outdated || brew upgrade libtool
 
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
-else
-  sudo apt-add-repository -y ppa:jonathonf/gtk3.18
-  sudo apt-get update
-  sudo apt-get install -V -y libglib2.0-dev
 fi
 
 gem install test-unit gobject-introspection
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 38680f429ed77..9ebffdb6401ef 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -151,20 +151,6 @@ test_python() {
 
 
 test_glib() {
-  # Build and test GLib, requires GLib >= 2.44 , so install that
-  # here
-  GLIB_VERSION=glib-2.53.7
-  GLIB_URL=https://gensho.ftp.acc.umu.se/pub/gnome/sources/glib/2.53/$GLIB_VERSION.tar.xz
-  curl -f -O $GLIB_URL
-  tar xf $GLIB_VERSION.tar.xz
-  pushd $GLIB_VERSION
-
-  ./configure --disable-libelf --enable-libmount=no --prefix=$ARROW_HOME
-  make -j$NPROC
-  make install
-
-  popd
-
   pushd c_glib
 
   ./configure --prefix=$ARROW_HOME

From e093772450ed90ff0ecc76c122cee2af8a56b655 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Sun, 17 Sep 2017 19:12:14 -0400
Subject: [PATCH 1060/1644] ARROW-1544: [JS] Export Vector types

Resolves https://issues.apache.org/jira/browse/ARROW-1544

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1103 from trxcllnt/js-export-vector-typedefs and squashes the following commits:

91a0625 [Paul Taylor] use gulp 4 from github. thought 4-alpha was on npm already.
e5a1034 [Paul Taylor] fix jest test coverage script
c6b09ee [Paul Taylor] export Vector types on root Arrow export
032ad27 [Paul Taylor] add compileOnSave (now required by TS 2.5?)
eb96552 [Paul Taylor] update dependencies
---
 js/gulpfile.js                 |  80 +++++++++++++------------
 js/package.json                |  21 +++----
 js/src/Arrow.externs.ts        |   8 +++
 js/src/Arrow.internal.ts       | 105 ---------------------------------
 js/src/Arrow.ts                |  60 ++++++++++++++++++-
 js/test/Arrow.ts               |  71 +++++++++++++---------
 js/test/vector-tests.ts        |  38 ++++++------
 js/tsconfig/tsconfig.base.json |   1 +
 8 files changed, 181 insertions(+), 203 deletions(-)
 delete mode 100644 js/src/Arrow.internal.ts

diff --git a/js/gulpfile.js b/js/gulpfile.js
index 90c45b769d9e4..9c945aa53af6e 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -84,7 +84,7 @@ function runTaskCombos(name) {
 }
 
 function cleanTask(target, format, taskName, outDir) {
-    return () => {
+    return function cleanTask() {
         const globs = [`${outDir}/**`];
         if (target === `es5` && format === `cjs`) {
             globs.push(`typings`);
@@ -102,25 +102,27 @@ function buildTask(target, format, taskName, outDir) {
 function bundleTask(target, format, taskName, outDir) {
     return [
         [`build:${taskName}`],
-        (cb) => streamMerge([
-            pump(gulp.src([`LICENSE`, `README.md`, `CHANGELOG.md`]), gulp.dest(outDir)),
-            pump(
-                gulp.src(`package.json`),
-                gulpJsonTransform((orig) => [
-                    `version`, `description`,
-                    `author`, `homepage`, `bugs`, `license`,
-                    `keywords`, `repository`, `peerDependencies`
-                ].reduce((copy, key) => (
-                    (copy[key] = orig[key]) && copy || copy
-                ), {
-                    main: `Arrow.js`,
-                    typings: `Arrow.d.ts`,
-                    name: `@apache-arrow/${target}-${format}`
-                }), 2),
-                gulp.dest(outDir),
-                onError
-            )
-        ])
+        function bundleTask() {
+            return streamMerge([
+                pump(gulp.src([`LICENSE`, `README.md`]), gulp.dest(outDir), onError),
+                pump(
+                    gulp.src(`package.json`),
+                    gulpJsonTransform((orig) => [
+                        `version`, `description`,
+                        `author`, `homepage`, `bugs`, `license`,
+                        `keywords`, `repository`, `peerDependencies`
+                    ].reduce((copy, key) => (
+                        (copy[key] = orig[key]) && copy || copy
+                    ), {
+                        main: `Arrow.js`,
+                        typings: `Arrow.d.ts`,
+                        name: `@apache-arrow/${target}-${format}`
+                    }), 2),
+                    gulp.dest(outDir),
+                    onError
+                )
+            ])
+        }
     ];
 }
 
@@ -138,9 +140,11 @@ function testTask(target, format, taskName, outDir, debug) {
             TEST_TARGET: target, TEST_MODULE: format
         })
     };
-    return () => !debug ?
-        child_process.spawn(jestPath, jestOptions, spawnOptions) :
-        child_process.exec(`node --inspect-brk ${jestPath} ${debugOpts}`, spawnOptions);
+    return function testTask() {
+        return !debug ?
+            child_process.spawn(jestPath, jestOptions, spawnOptions) :
+            child_process.exec(`node --inspect-brk ${jestPath} ${debugOpts}`, spawnOptions);
+    }
 }
 
 function closureTask(target, format, taskName, outDir) {
@@ -149,38 +153,36 @@ function closureTask(target, format, taskName, outDir) {
     const languageIn = clsTarget === `es5` ? `es2015` : clsTarget;
     return [
         [`clean:${taskName}`, `build:${clsTarget}:cls`],
-        () => {
-            return streamMerge([
-                closureStream(closureSrcs(false), `Arrow`, onError, true),
-                closureStream(closureSrcs(true), `Arrow.internal`, onError)
-            ])
-            .on('end', () => del([`targets/${target}/cls/**`]));
+        function closureTask() {
+            return closureStream(
+                closureSrcs(),
+                closureCompiler(closureArgs())
+            ).on('end', () => del([`targets/${target}/cls/**`]));
         }
     ];
-    function closureSrcs(isInternal) {
+    function closureSrcs() {
         return gulp.src([
             `closure-compiler-scripts/*.js`,
             `${googleRoot}/**/*.js`,
             `!${googleRoot}/format/*.js`,
             `!${googleRoot}/Arrow.externs.js`,
-            `!${googleRoot}/Arrow${isInternal ? `` : `.internal`}.js`
         ], { base: `./` });
     }
-    function closureStream(sources, entry, onError, copyToDist) {
+    function closureStream(sources, compiler) {
         const streams = [
             sources,
             sourcemaps.init(),
-            closureCompiler(closureArgs(entry)),
+            compiler,
             sourcemaps.write('.'),
             gulp.dest(outDir)
         ];
-        // copy the UMD bundle to dist
-        if (target === `es5` && copyToDist) {
+        // copy the ES5 UMD bundle to dist
+        if (target === `es5`) {
             streams.push(gulp.dest(`dist`));
         }
         return pump(...streams, onError);
     }
-    function closureArgs(entry) {
+    function closureArgs() {
         return {
             third_party: true,
             externs: `${googleRoot}/Arrow.externs.js`,
@@ -190,10 +192,10 @@ function closureTask(target, format, taskName, outDir) {
             // formatting: `PRETTY_PRINT`,
             compilation_level: `ADVANCED`,
             assume_function_wrapper: true,
-            js_output_file: `${entry}.js`,
+            js_output_file: `Arrow.js`,
             language_in: gCCTargets[languageIn],
             language_out: gCCTargets[clsTarget],
-            entry_point: `${googleRoot}/${entry}.js`,
+            entry_point: `${googleRoot}/Arrow.js`,
             output_wrapper:
 `// Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
@@ -223,7 +225,7 @@ function closureTask(target, format, taskName, outDir) {
 function typescriptTask(target, format, taskName, outDir) {
     return [
         [`clean:${taskName}`],
-        () => {
+        function typescriptTask() {
             const tsconfigPath = `tsconfig/tsconfig.${target}.${format}.json`;
             let { js, dts } = tsProjects.find((p) => p.target === target && p.format === format) || {};
             if (!js || !dts) {
diff --git a/js/package.json b/js/package.json
index 899822130972a..e6848fd641d0e 100644
--- a/js/package.json
+++ b/js/package.json
@@ -73,28 +73,28 @@
     "del": "~3.0.0",
     "esdoc": "~1.0.1",
     "esdoc-standard-plugin": "~1.0.0",
-    "google-closure-compiler": "~20170806.0.0",
+    "google-closure-compiler": "~20170910.0.0",
     "gulp": "github:gulpjs/gulp#4.0",
     "gulp-json-transform": "~0.4.2",
     "gulp-sourcemaps": "~2.6.1",
     "gulp-typescript": "~3.2.2",
-    "jest": "~20.0.4",
+    "jest": "~21.1.0",
     "jest-environment-node-debug": "~2.0.0",
     "json": "~9.0.6",
-    "lerna": "2.0.0",
-    "lint-staged": "~4.0.1",
-    "merge2": "~1.1.0",
+    "lerna": "~2.1.2",
+    "lint-staged": "~4.2.1",
+    "merge2": "~1.2.0",
     "mkdirp": "~0.5.1",
-    "npm-run-all": "~4.0.2",
+    "npm-run-all": "~4.1.1",
     "pump": "~1.0.2",
     "rimraf": "~2.6.1",
     "shx": "~0.2.2",
     "text-encoding-utf-8": "~1.0.1",
     "trash": "~4.0.1",
-    "ts-jest": "~20.0.10",
+    "ts-jest": "~21.0.1",
     "tslib": "~1.7.1",
-    "tslint": "~5.6.0",
-    "typescript": "~2.4.2",
+    "tslint": "~5.7.0",
+    "typescript": "~2.5.2",
     "validate-commit-msg": "~2.14.0"
   },
   "config": {
@@ -129,7 +129,8 @@
     ],
     "coveragePathIgnorePatterns": [
       "format\\/(File|Message|Schema|Tensor)_generated\\.(js|ts)$",
-      "test\\/.*\\.(ts|tsx|js)$"
+      "test\\/.*\\.(ts|tsx|js)$",
+      "/node_modules/"
     ],
     "transform": {
       ".(ts|tsx)": "<rootDir>/node_modules/ts-jest/preprocessor.js",
diff --git a/js/src/Arrow.externs.ts b/js/src/Arrow.externs.ts
index 7342684e7a660..7289d6d2732b6 100644
--- a/js/src/Arrow.externs.ts
+++ b/js/src/Arrow.externs.ts
@@ -26,6 +26,8 @@ Symbol.iterator;
 Symbol.asyncIterator;
 let Table = function() {};
 /** @type {?} */
+Table.prototype.length;
+/** @type {?} */
 Table.prototype.rows;
 /** @type {?} */
 Table.prototype.cols;
@@ -65,3 +67,9 @@ TypedVector.prototype.arrayType;
 let ValidityVector = function() {};
 /** @type {?} */
 (<any> ValidityVector).pack;
+
+let DictionaryVector = function() {};
+/** @type {?} */
+DictionaryVector.prototype.index;
+/** @type {?} */
+DictionaryVector.prototype.value;
diff --git a/js/src/Arrow.internal.ts b/js/src/Arrow.internal.ts
deleted file mode 100644
index d8f0c37de8359..0000000000000
--- a/js/src/Arrow.internal.ts
+++ /dev/null
@@ -1,105 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector as Vector_ } from './vector/vector';
-import { StructVector as StructVector_ } from './vector/struct';
-import { DictionaryVector as DictionaryVector_ } from './vector/dictionary';
-import { ListVector as ListVector_, Utf8Vector as Utf8Vector_, FixedSizeListVector as FixedSizeListVector_ } from './vector/list';
-import {
-    TypedVector as TypedVector_, BitVector as BitVector_,
-    DateVector as DateVector_, IndexVector as IndexVector_,
-    Int8Vector as Int8Vector_, Int16Vector as Int16Vector_,
-    Int32Vector as Int32Vector_, Int64Vector as Int64Vector_,
-    Uint8Vector as Uint8Vector_, Uint16Vector as Uint16Vector_,
-    Uint32Vector as Uint32Vector_, Uint64Vector as Uint64Vector_,
-    Float32Vector as Float32Vector_, Float64Vector as Float64Vector_,
-} from './vector/typed';
-
-export const vectors = {
-    Vector: Vector_,
-    BitVector: BitVector_,
-    ListVector: ListVector_,
-    Utf8Vector: Utf8Vector_,
-    DateVector: DateVector_,
-    IndexVector: IndexVector_,
-    TypedVector: TypedVector_,
-    Int8Vector: Int8Vector_,
-    Int16Vector: Int16Vector_,
-    Int32Vector: Int32Vector_,
-    Int64Vector: Int64Vector_,
-    Uint8Vector: Uint8Vector_,
-    Uint16Vector: Uint16Vector_,
-    Uint32Vector: Uint32Vector_,
-    Uint64Vector: Uint64Vector_,
-    Float32Vector: Float32Vector_,
-    Float64Vector: Float64Vector_,
-    StructVector: StructVector_,
-    DictionaryVector: DictionaryVector_,
-    FixedSizeListVector: FixedSizeListVector_,
-};
-
-export namespace vectors {
-    export type Vector<T> =  Vector_<T>;
-    export type BitVector =  BitVector_;
-    export type ListVector<T> =  ListVector_<T>;
-    export type Utf8Vector =  Utf8Vector_;
-    export type DateVector =  DateVector_;
-    export type IndexVector =  IndexVector_;
-    export type Int8Vector =  Int8Vector_;
-    export type Int16Vector =  Int16Vector_;
-    export type Int32Vector =  Int32Vector_;
-    export type Int64Vector =  Int64Vector_;
-    export type Uint8Vector =  Uint8Vector_;
-    export type Uint16Vector =  Uint16Vector_;
-    export type Uint32Vector =  Uint32Vector_;
-    export type Uint64Vector =  Uint64Vector_;
-    export type Float32Vector =  Float32Vector_;
-    export type Float64Vector =  Float64Vector_;
-    export type StructVector =  StructVector_;
-    export type DictionaryVector<T> =  DictionaryVector_<T>;
-    export type FixedSizeListVector<T> =  FixedSizeListVector_<T>;
-    export type TypedVector<T, TArray> =  TypedVector_<T, TArray>;
-}
-
-/* These exports are needed for the closure umd targets */
-try {
-    const Arrow = eval('exports');
-    if (typeof Arrow === 'object') {
-        // string indexers tell closure compiler not to rename these properties
-        Arrow['vectors'] = {};
-        Arrow['vectors']['Vector'] = Vector_;
-        Arrow['vectors']['BitVector'] = BitVector_;
-        Arrow['vectors']['ListVector'] = ListVector_;
-        Arrow['vectors']['Utf8Vector'] = Utf8Vector_;
-        Arrow['vectors']['DateVector'] = DateVector_;
-        Arrow['vectors']['IndexVector'] = IndexVector_;
-        Arrow['vectors']['Int8Vector'] = Int8Vector_;
-        Arrow['vectors']['Int16Vector'] = Int16Vector_;
-        Arrow['vectors']['Int32Vector'] = Int32Vector_;
-        Arrow['vectors']['Int64Vector'] = Int64Vector_;
-        Arrow['vectors']['Uint8Vector'] = Uint8Vector_;
-        Arrow['vectors']['Uint16Vector'] = Uint16Vector_;
-        Arrow['vectors']['Uint32Vector'] = Uint32Vector_;
-        Arrow['vectors']['Uint64Vector'] = Uint64Vector_;
-        Arrow['vectors']['Float32Vector'] = Float32Vector_;
-        Arrow['vectors']['Float64Vector'] = Float64Vector_;
-        Arrow['vectors']['StructVector'] = StructVector_;
-        Arrow['vectors']['DictionaryVector'] = DictionaryVector_;
-        Arrow['vectors']['FixedSizeListVector'] = FixedSizeListVector_;
-    }
-} catch (e) { /* not the UMD bundle */ }
-/** end closure exports */
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index fe19645781322..ea8a5c3e1d9bb 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -17,7 +17,43 @@
 
 import { Table } from './table';
 import { readBuffers } from './reader/arrow';
-export { Table, readBuffers };
+import { Vector } from './vector/vector';
+import { StructVector } from './vector/struct';
+import { DictionaryVector } from './vector/dictionary';
+import { ListVector, Utf8Vector, FixedSizeListVector } from './vector/list';
+import {
+    TypedVector, BitVector,
+    DateVector, IndexVector,
+    Int8Vector, Int16Vector,
+    Int32Vector, Int64Vector,
+    Uint8Vector, Uint16Vector,
+    Uint32Vector, Uint64Vector,
+    Float32Vector, Float64Vector,
+} from './vector/typed';
+
+export {
+    Table, readBuffers,
+    Vector,
+    BitVector,
+    ListVector,
+    Utf8Vector,
+    DateVector,
+    IndexVector,
+    TypedVector,
+    Int8Vector,
+    Int16Vector,
+    Int32Vector,
+    Int64Vector,
+    Uint8Vector,
+    Uint16Vector,
+    Uint32Vector,
+    Uint64Vector,
+    Float32Vector,
+    Float64Vector,
+    StructVector,
+    DictionaryVector,
+    FixedSizeListVector,
+};
 
 /* These exports are needed for the closure umd targets */
 try {
@@ -26,6 +62,26 @@ try {
         // string indexers tell closure compiler not to rename these properties
         Arrow['Table'] = Table;
         Arrow['readBuffers'] = readBuffers;
+        Arrow['Vector'] = Vector;
+        Arrow['BitVector'] = BitVector;
+        Arrow['ListVector'] = ListVector;
+        Arrow['Utf8Vector'] = Utf8Vector;
+        Arrow['DateVector'] = DateVector;
+        Arrow['IndexVector'] = IndexVector;
+        Arrow['TypedVector'] = TypedVector;
+        Arrow['Int8Vector'] = Int8Vector;
+        Arrow['Int16Vector'] = Int16Vector;
+        Arrow['Int32Vector'] = Int32Vector;
+        Arrow['Int64Vector'] = Int64Vector;
+        Arrow['Uint8Vector'] = Uint8Vector;
+        Arrow['Uint16Vector'] = Uint16Vector;
+        Arrow['Uint32Vector'] = Uint32Vector;
+        Arrow['Uint64Vector'] = Uint64Vector;
+        Arrow['Float32Vector'] = Float32Vector;
+        Arrow['Float64Vector'] = Float64Vector;
+        Arrow['StructVector'] = StructVector;
+        Arrow['DictionaryVector'] = DictionaryVector;
+        Arrow['FixedSizeListVector'] = FixedSizeListVector;
     }
 } catch (e) { /* not the UMD bundle */ }
-/** end closure exports */
+/* end closure exports */
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
index a9ab2b783fecc..3f29c5409ab26 100644
--- a/js/test/Arrow.ts
+++ b/js/test/Arrow.ts
@@ -34,34 +34,51 @@ if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets
 if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats);
 
 let Arrow: any = require(resolve(`./targets/${target}/${format}/Arrow.js`));
-let ArrowInternal: any = require(resolve(`./targets/${target}/${format}/Arrow.internal.js`));
 
-import { vectors as vectors_ } from '../src/Arrow.internal';
-import { Table as Table_, readBuffers as readBuffers_ } from '../src/Arrow';
+import {
+    Table as Table_,
+    readBuffers as readBuffers_,
+    Vector as Vector_,
+    BitVector as BitVector_,
+    ListVector as ListVector_,
+    Utf8Vector as Utf8Vector_,
+    DateVector as DateVector_,
+    IndexVector as IndexVector_,
+    TypedVector as TypedVector_,
+    Int8Vector as Int8Vector_,
+    Int16Vector as Int16Vector_,
+    Int32Vector as Int32Vector_,
+    Int64Vector as Int64Vector_,
+    Uint8Vector as Uint8Vector_,
+    Uint16Vector as Uint16Vector_,
+    Uint32Vector as Uint32Vector_,
+    Uint64Vector as Uint64Vector_,
+    Float32Vector as Float32Vector_,
+    Float64Vector as Float64Vector_,
+    StructVector as StructVector_,
+    DictionaryVector as DictionaryVector_,
+    FixedSizeListVector as FixedSizeListVector_,
+} from '../src/Arrow';
 
 export let Table = Arrow.Table as typeof Table_;
 export let readBuffers = Arrow.readBuffers as typeof readBuffers_;
-
-export let vectors: typeof vectors_ = ArrowInternal.vectors;
-export namespace vectors {
-    export type Vector<T> = vectors_.Vector<T>;
-    export type BitVector = vectors_.BitVector;
-    export type ListVector<T> = vectors_.ListVector<T>;
-    export type Utf8Vector = vectors_.Utf8Vector;
-    export type DateVector = vectors_.DateVector;
-    export type IndexVector = vectors_.IndexVector;
-    export type Int8Vector = vectors_.Int8Vector;
-    export type Int16Vector = vectors_.Int16Vector;
-    export type Int32Vector = vectors_.Int32Vector;
-    export type Int64Vector = vectors_.Int64Vector;
-    export type Uint8Vector = vectors_.Uint8Vector;
-    export type Uint16Vector = vectors_.Uint16Vector;
-    export type Uint32Vector = vectors_.Uint32Vector;
-    export type Uint64Vector = vectors_.Uint64Vector;
-    export type Float32Vector = vectors_.Float32Vector;
-    export type Float64Vector = vectors_.Float64Vector;
-    export type StructVector = vectors_.StructVector;
-    export type DictionaryVector<T> = vectors_.DictionaryVector<T>;
-    export type FixedSizeListVector<T> = vectors_.FixedSizeListVector<T>;
-};
-
+export let Vector = Arrow.Vector as typeof Vector_;
+export let BitVector = Arrow.BitVector as typeof BitVector_;
+export let ListVector = Arrow.ListVector as typeof ListVector_;
+export let Utf8Vector = Arrow.Utf8Vector as typeof Utf8Vector_;
+export let DateVector = Arrow.DateVector as typeof DateVector_;
+export let IndexVector = Arrow.IndexVector as typeof IndexVector_;
+export let TypedVector = Arrow.TypedVector as typeof TypedVector_;
+export let Int8Vector = Arrow.Int8Vector as typeof Int8Vector_;
+export let Int16Vector = Arrow.Int16Vector as typeof Int16Vector_;
+export let Int32Vector = Arrow.Int32Vector as typeof Int32Vector_;
+export let Int64Vector = Arrow.Int64Vector as typeof Int64Vector_;
+export let Uint8Vector = Arrow.Uint8Vector as typeof Uint8Vector_;
+export let Uint16Vector = Arrow.Uint16Vector as typeof Uint16Vector_;
+export let Uint32Vector = Arrow.Uint32Vector as typeof Uint32Vector_;
+export let Uint64Vector = Arrow.Uint64Vector as typeof Uint64Vector_;
+export let Float32Vector = Arrow.Float32Vector as typeof Float32Vector_;
+export let Float64Vector = Arrow.Float64Vector as typeof Float64Vector_;
+export let StructVector = Arrow.StructVector as typeof StructVector_;
+export let DictionaryVector = Arrow.DictionaryVector as typeof DictionaryVector_;
+export let FixedSizeListVector = Arrow.FixedSizeListVector as typeof FixedSizeListVector_;
diff --git a/js/test/vector-tests.ts b/js/test/vector-tests.ts
index 54d50d1af2754..0c9ef4404ed6a 100644
--- a/js/test/vector-tests.ts
+++ b/js/test/vector-tests.ts
@@ -15,30 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { vectors } from './Arrow';
 import { flatbuffers } from 'flatbuffers';
 import Long = flatbuffers.Long;
-const BitVector = vectors.BitVector;
-const TypedVector = vectors.TypedVector;
+import {
+    BitVector,
+    TypedVector,
+    Int64Vector,
+    Uint64Vector,
+    Int8Vector,
+    Int16Vector,
+    Int32Vector,
+    Uint8Vector,
+    Uint16Vector,
+    Uint32Vector,
+    Float32Vector,
+    Float64Vector,
+} from './Arrow';
 
-const LongVectors = {
-    Int64Vector: vectors.Int64Vector,
-    Uint64Vector: vectors.Uint64Vector,
-};
+const LongVectors = { Int64Vector, Uint64Vector };
+const ByteVectors = { Int8Vector, Int16Vector, Int32Vector, Uint8Vector, Uint16Vector, Uint32Vector, Float32Vector, Float64Vector };
 
-const ByteVectors = {
-    Int8Vector: vectors.Int8Vector,
-    Int16Vector: vectors.Int16Vector,
-    Int32Vector: vectors.Int32Vector,
-    Uint8Vector: vectors.Uint8Vector,
-    Uint16Vector: vectors.Uint16Vector,
-    Uint32Vector: vectors.Uint32Vector,
-    Float32Vector: vectors.Float32Vector,
-    Float64Vector: vectors.Float64Vector,
-};
-
-const longVectors = toMap<typeof TypedVector>(vectors, Object.keys(LongVectors));
-const byteVectors = toMap<typeof TypedVector>(vectors, Object.keys(ByteVectors));
+const longVectors = toMap<typeof TypedVector>(LongVectors, Object.keys(LongVectors));
+const byteVectors = toMap<typeof TypedVector>(ByteVectors, Object.keys(ByteVectors));
 const bytes = Array.from(
     { length: 5 },
     () => Uint8Array.from(
diff --git a/js/tsconfig/tsconfig.base.json b/js/tsconfig/tsconfig.base.json
index 47238543a7c3a..4a46ed1f0af9b 100644
--- a/js/tsconfig/tsconfig.base.json
+++ b/js/tsconfig/tsconfig.base.json
@@ -1,6 +1,7 @@
 {
   "exclude": ["../node_modules"],
   "include": ["../src/**/*.ts"],
+  "compileOnSave": false,
   "compilerOptions": {
     "lib": ["dom", "esnext", "esnext.asynciterable"],
     "moduleResolution": "node",

From b635d4c9046481422aaa3a93a4c8f83f73d16c64 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 18 Sep 2017 08:57:04 +0900
Subject: [PATCH 1061/1644] Remove garbage ")"

---
 cpp/src/arrow/builder.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 4d18a3e3bbf49..28f3cb9723236 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -207,7 +207,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   /// \param[in] values a contiguous C array of values
   /// \param[in] length the number of values to append
   /// \param[in] valid_bytes an optional sequence of bytes where non-zero
-  /// indicates a valid (non-null) value)
+  /// indicates a valid (non-null) value
   /// \return Status
   Status Append(const value_type* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
@@ -423,7 +423,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
   /// \param[in] values a contiguous C array of values
   /// \param[in] length the number of values to append
   /// \param[in] valid_bytes an optional sequence of bytes where non-zero
-  /// indicates a valid (non-null) value)
+  /// indicates a valid (non-null) value
   /// \return Status
   Status Append(const uint64_t* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
@@ -485,7 +485,7 @@ class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase
   /// \param[in] values a contiguous C array of values
   /// \param[in] length the number of values to append
   /// \param[in] valid_bytes an optional sequence of bytes where non-zero
-  /// indicates a valid (non-null) value)
+  /// indicates a valid (non-null) value
   /// \return Status
   Status Append(const int64_t* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
@@ -555,7 +555,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   /// \param[in] values a contiguous array of bytes (non-zero is 1)
   /// \param[in] length the number of values to append
   /// \param[in] valid_bytes an optional sequence of bytes where non-zero
-  /// indicates a valid (non-null) value)
+  /// indicates a valid (non-null) value
   /// \return Status
   Status Append(const uint8_t* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);

From 63e7966adc8c2664be76928c16555d818ca1bba3 Mon Sep 17 00:00:00 2001
From: Wataru Shimizu <waruzilla@gmail.com>
Date: Tue, 19 Sep 2017 00:00:58 +0900
Subject: [PATCH 1062/1644] ARROW-1464: [GLib] Add "Common build problems"
 section into the README.md of c_glib

Add some detailed explanation of common build problems especially on macOS because it requires some tweaks.

Author: Wataru Shimizu <waruzilla@gmail.com>

Closes #1104 from wagavulin/build-troubleshooting and squashes the following commits:

9b655426 [Wataru Shimizu] Improve format and the explanation of installing/linking autoconf archive on macOS.
b6c5274f [Wataru Shimizu] Add "Common build problems" section in the README.md of c_glib
---
 c_glib/README.md | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/c_glib/README.md b/c_glib/README.md
index 5062ceab7a76f..2a9d5d64ce2cd 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -197,3 +197,43 @@ Now, you can run unit tests by the followings:
 % cd c_glib
 % test/run-test.sh
 ```
+
+## Common build problems
+
+### configure failed - `AX_CXX_COMPILE_STDCXX_11(ext, mandatory)'
+
+* Check whether `autoconf-archive` is installed.
+* [macOS] `autoconf-archive` must be linked, but may not be linked. You can check it by running `brew install autoconf-archive` again. If it's not linked, it will show a warning message like:
+
+```console
+% brew install autoconf-archive
+Warning: autoconf-archive 2017.03.21 is already installed, it's just not linked.
+You can use `brew link autoconf-archive` to link this version.
+```
+
+In this case, you need to run `brew link autoconf-archive`. It may fail with the following message if you have install conflicted packages (e.g. `gnome-common`).
+
+```console
+% brew link autoconf-archive
+Linking /usr/local/Cellar/autoconf-archive/2017.03.21... 
+Error: Could not symlink share/aclocal/ax_check_enable_debug.m4
+Target /usr/local/share/aclocal/ax_check_enable_debug.m4
+is a symlink belonging to gnome-common. You can unlink it:
+  brew unlink gnome-common
+```
+
+You need to run `brew unlink <pkgname>`, then run `brew link autoconf-archive` again.
+
+After installing/linking `autoconf-archive`, run `./autogen.sh` again.
+
+### [macOS] configure failed - gobject-introspection-1.0 is not installed
+
+gobject-introspection requires libffi, and it's automatically installed with gobject-introspection. However it can't be found because it's [keg-only](https://docs.brew.sh/FAQ.html#what-does-keg-only-mean). You need to set `PKG_CONFIG_PATH` when executing configure.
+
+```console
+% ./configure PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig
+```
+
+### build failed - /usr/bin/ld: cannot find -larrow
+
+Arrow C++ must be installed to build Arrow GLib. Run `make install` on Arrow C++ build directory. In addtion, on linux, you may need to run `sudo ldconfig`.

From 4a65fea011fe6e5f016a3170c778d53ab98356b3 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 18 Sep 2017 17:54:15 -0400
Subject: [PATCH 1063/1644] ARROW-1548: [GLib] Support bulk append in builder

`append_values()` are for bulk values append.
`append_nulls()` are for bulk nulls append.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1110 from kou/glib-support-bulk-append-in-builder and squashes the following commits:

49260319 [Kouhei Sutou] [GLib] Support bulk append in builder
---
 c_glib/arrow-glib/array-builder.cpp       | 1147 ++++++++++++++++++++-
 c_glib/arrow-glib/array-builder.h         |  153 +++
 c_glib/doc/reference/arrow-glib-docs.sgml |    4 +
 c_glib/test/test-array-builder.rb         |  485 +++++++++
 4 files changed, 1780 insertions(+), 9 deletions(-)
 create mode 100644 c_glib/test/test-array-builder.rb

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 126986d1fb007..a6fad87ab8f51 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -38,6 +38,43 @@ garrow_array_builder_append(GArrowArrayBuilder *builder,
   return garrow_error_check(error, status, context);
 }
 
+template <typename BUILDER, typename VALUE>
+gboolean
+garrow_array_builder_append_values(GArrowArrayBuilder *builder,
+                                   const VALUE *values,
+                                   gint64 values_length,
+                                   const gboolean *is_valids,
+                                   gint64 is_valids_length,
+                                   GError **error,
+                                   const gchar *context)
+{
+  auto arrow_builder =
+    static_cast<BUILDER>(garrow_array_builder_get_raw(builder));
+  arrow::Status status;
+  if (is_valids_length > 0) {
+    if (values_length != is_valids_length) {
+      g_set_error(error,
+                  GARROW_ERROR,
+                  GARROW_ERROR_INVALID,
+                  "%s: values length and is_valids length must be equal: "
+                  "<%" G_GINT64_FORMAT "> != "
+                  "<%" G_GINT64_FORMAT ">",
+                  context,
+                  values_length,
+                  is_valids_length);
+      return FALSE;
+    }
+    uint8_t valid_bytes[is_valids_length];
+    for (gint64 i = 0; i < is_valids_length; ++i) {
+      valid_bytes[i] = is_valids[i];
+    }
+    status = arrow_builder->Append(values, values_length, valid_bytes);
+  } else {
+    status = arrow_builder->Append(values, values_length, nullptr);
+  }
+  return garrow_error_check(error, status, context);
+}
+
 template <typename BUILDER>
 gboolean
 garrow_array_builder_append_null(GArrowArrayBuilder *builder,
@@ -50,6 +87,35 @@ garrow_array_builder_append_null(GArrowArrayBuilder *builder,
   return garrow_error_check(error, status, context);
 }
 
+template <typename BUILDER>
+gboolean
+garrow_array_builder_append_nulls(GArrowArrayBuilder *builder,
+                                  gint64 n,
+                                  GError **error,
+                                  const gchar *context)
+{
+  if (n < 0) {
+    g_set_error(error,
+                GARROW_ERROR,
+                GARROW_ERROR_INVALID,
+                "%s: the number of nulls must be 0 or larger: "
+                "<%" G_GINT64_FORMAT ">",
+                context,
+                n);
+    return FALSE;
+  }
+  if (n == 0) {
+    return TRUE;
+  }
+
+  auto arrow_builder =
+    static_cast<BUILDER>(garrow_array_builder_get_raw(builder));
+  uint8_t valid_bytes[n];
+  memset(valid_bytes, 0, sizeof(uint8_t) * n);
+  auto status = arrow_builder->AppendNulls(valid_bytes, n);
+  return garrow_error_check(error, status, context);
+}
+
 G_BEGIN_DECLS
 
 /**
@@ -302,6 +368,47 @@ garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
      "[boolean-array-builder][append]");
 }
 
+/**
+ * garrow_boolean_array_builder_append_values:
+ * @builder: A #GArrowBooleanArrayBuilder.
+ * @values: (array length=values_length): The array of boolean.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_boolean_array_builder_append_values(GArrowBooleanArrayBuilder *builder,
+                                           const gboolean *values,
+                                           gint64 values_length,
+                                           const gboolean *is_valids,
+                                           gint64 is_valids_length,
+                                           GError **error)
+{
+  guint8 arrow_values[values_length];
+  for (gint64 i = 0; i < values_length; ++i) {
+    arrow_values[i] = values[i];
+  }
+  return garrow_array_builder_append_values<arrow::BooleanBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     arrow_values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[boolean-array-builder][append-values]");
+}
+
 /**
  * garrow_boolean_array_builder_append_null:
  * @builder: A #GArrowBooleanArrayBuilder.
@@ -319,6 +426,31 @@ garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
      "[boolean-array-builder][append-null]");
 }
 
+/**
+ * garrow_boolean_array_builder_append_nulls:
+ * @builder: A #GArrowBooleanArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_boolean_array_builder_append_nulls(GArrowBooleanArrayBuilder *builder,
+                                          gint64 n,
+                                          GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::BooleanBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[boolean-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowIntArrayBuilder,
               garrow_int_array_builder,
@@ -373,6 +505,43 @@ garrow_int_array_builder_append(GArrowIntArrayBuilder *builder,
      "[int-array-builder][append]");
 }
 
+/**
+ * garrow_int_array_builder_append_values:
+ * @builder: A #GArrowIntArrayBuilder.
+ * @values: (array length=values_length): The array of int.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int_array_builder_append_values(GArrowIntArrayBuilder *builder,
+                                       const gint64 *values,
+                                       gint64 values_length,
+                                       const gboolean *is_valids,
+                                       gint64 is_valids_length,
+                                       GError **error)
+{
+  return garrow_array_builder_append_values<arrow::AdaptiveIntBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     reinterpret_cast<const int64_t *>(values),
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[int-array-builder][append-values]");
+}
+
 /**
  * garrow_int_array_builder_append_null:
  * @builder: A #GArrowIntArrayBuilder.
@@ -392,6 +561,31 @@ garrow_int_array_builder_append_null(GArrowIntArrayBuilder *builder,
      "[int-array-builder][append-null]");
 }
 
+/**
+ * garrow_int_array_builder_append_nulls:
+ * @builder: A #GArrowIntArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int_array_builder_append_nulls(GArrowIntArrayBuilder *builder,
+                                      gint64 n,
+                                      GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::AdaptiveIntBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[int-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowInt8ArrayBuilder,
               garrow_int8_array_builder,
@@ -441,6 +635,43 @@ garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
      "[int8-array-builder][append]");
 }
 
+/**
+ * garrow_int8_array_builder_append_values:
+ * @builder: A #GArrowInt8ArrayBuilder.
+ * @values: (array length=values_length): The array of int8.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int8_array_builder_append_values(GArrowInt8ArrayBuilder *builder,
+                                        const gint8 *values,
+                                        gint64 values_length,
+                                        const gboolean *is_valids,
+                                        gint64 is_valids_length,
+                                        GError **error)
+{
+  return garrow_array_builder_append_values<arrow::Int8Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[int8-array-builder][append-values]");
+}
+
 /**
  * garrow_int8_array_builder_append_null:
  * @builder: A #GArrowInt8ArrayBuilder.
@@ -458,6 +689,31 @@ garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
      "[int8-array-builder][append-null]");
 }
 
+/**
+ * garrow_int8_array_builder_append_nulls:
+ * @builder: A #GArrowInt8ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int8_array_builder_append_nulls(GArrowInt8ArrayBuilder *builder,
+                                       gint64 n,
+                                       GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::Int8Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[int8-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowUInt8ArrayBuilder,
               garrow_uint8_array_builder,
@@ -507,6 +763,43 @@ garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
      "[uint8-array-builder][append]");
 }
 
+/**
+ * garrow_uint8_array_builder_append_values:
+ * @builder: A #GArrowUInt8ArrayBuilder.
+ * @values: (array length=values_length): The array of uint8.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint8_array_builder_append_values(GArrowUInt8ArrayBuilder *builder,
+                                         const guint8 *values,
+                                         gint64 values_length,
+                                         const gboolean *is_valids,
+                                         gint64 is_valids_length,
+                                         GError **error)
+{
+  return garrow_array_builder_append_values<arrow::UInt8Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[uint8-array-builder][append-values]");
+}
+
 /**
  * garrow_uint8_array_builder_append_null:
  * @builder: A #GArrowUInt8ArrayBuilder.
@@ -524,6 +817,31 @@ garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
      "[uint8-array-builder][append-null]");
 }
 
+/**
+ * garrow_uint8_array_builder_append_nulls:
+ * @builder: A #GArrowUInt8ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint8_array_builder_append_nulls(GArrowUInt8ArrayBuilder *builder,
+                                        gint64 n,
+                                        GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::UInt8Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[uint8-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowInt16ArrayBuilder,
               garrow_int16_array_builder,
@@ -573,6 +891,43 @@ garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
      "[int16-array-builder][append]");
 }
 
+/**
+ * garrow_int16_array_builder_append_values:
+ * @builder: A #GArrowInt16ArrayBuilder.
+ * @values: (array length=values_length): The array of int16.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int16_array_builder_append_values(GArrowInt16ArrayBuilder *builder,
+                                         const gint16 *values,
+                                         gint64 values_length,
+                                         const gboolean *is_valids,
+                                         gint64 is_valids_length,
+                                         GError **error)
+{
+  return garrow_array_builder_append_values<arrow::Int16Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[int16-array-builder][append-values]");
+}
+
 /**
  * garrow_int16_array_builder_append_null:
  * @builder: A #GArrowInt16ArrayBuilder.
@@ -590,6 +945,31 @@ garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
      "[int16-array-builder][append-null]");
 }
 
+/**
+ * garrow_int16_array_builder_append_nulls:
+ * @builder: A #GArrowInt16ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int16_array_builder_append_nulls(GArrowInt16ArrayBuilder *builder,
+                                        gint64 n,
+                                        GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::Int16Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[int16-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowUInt16ArrayBuilder,
               garrow_uint16_array_builder,
@@ -639,6 +1019,43 @@ garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
      "[uint16-array-builder][append]");
 }
 
+/**
+ * garrow_uint16_array_builder_append_values:
+ * @builder: A #GArrowUInt16ArrayBuilder.
+ * @values: (array length=values_length): The array of uint16.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint16_array_builder_append_values(GArrowUInt16ArrayBuilder *builder,
+                                          const guint16 *values,
+                                          gint64 values_length,
+                                          const gboolean *is_valids,
+                                          gint64 is_valids_length,
+                                          GError **error)
+{
+  return garrow_array_builder_append_values<arrow::UInt16Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[uint16-array-builder][append-values]");
+}
+
 /**
  * garrow_uint16_array_builder_append_null:
  * @builder: A #GArrowUInt16ArrayBuilder.
@@ -656,6 +1073,31 @@ garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
      "[uint16-array-builder][append-null]");
 }
 
+/**
+ * garrow_uint16_array_builder_append_nulls:
+ * @builder: A #GArrowUInt16ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint16_array_builder_append_nulls(GArrowUInt16ArrayBuilder *builder,
+                                         gint64 n,
+                                         GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::UInt16Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[uint16-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowInt32ArrayBuilder,
               garrow_int32_array_builder,
@@ -705,6 +1147,43 @@ garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
      "[int32-array-builder][append]");
 }
 
+/**
+ * garrow_int32_array_builder_append_values:
+ * @builder: A #GArrowInt32ArrayBuilder.
+ * @values: (array length=values_length): The array of int32.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int32_array_builder_append_values(GArrowInt32ArrayBuilder *builder,
+                                         const gint32 *values,
+                                         gint64 values_length,
+                                         const gboolean *is_valids,
+                                         gint64 is_valids_length,
+                                         GError **error)
+{
+  return garrow_array_builder_append_values<arrow::Int32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[int32-array-builder][append-values]");
+}
+
 /**
  * garrow_int32_array_builder_append_null:
  * @builder: A #GArrowInt32ArrayBuilder.
@@ -714,7 +1193,7 @@ garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
  */
 gboolean
 garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
-                                      GError **error)
+                                       GError **error)
 {
   return garrow_array_builder_append_null<arrow::Int32Builder *>
     (GARROW_ARRAY_BUILDER(builder),
@@ -722,6 +1201,31 @@ garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
      "[int32-array-builder][append-null]");
 }
 
+/**
+ * garrow_int32_array_builder_append_nulls:
+ * @builder: A #GArrowInt32ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int32_array_builder_append_nulls(GArrowInt32ArrayBuilder *builder,
+                                        gint64 n,
+                                        GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::Int32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[int32-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowUInt32ArrayBuilder,
               garrow_uint32_array_builder,
@@ -771,6 +1275,43 @@ garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
      "[uint32-array-builder][append]");
 }
 
+/**
+ * garrow_uint32_array_builder_append_values:
+ * @builder: A #GArrowUInt32ArrayBuilder.
+ * @values: (array length=values_length): The array of uint32.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint32_array_builder_append_values(GArrowUInt32ArrayBuilder *builder,
+                                          const guint32 *values,
+                                          gint64 values_length,
+                                          const gboolean *is_valids,
+                                          gint64 is_valids_length,
+                                          GError **error)
+{
+  return garrow_array_builder_append_values<arrow::UInt32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[uint32-array-builder][append-values]");
+}
+
 /**
  * garrow_uint32_array_builder_append_null:
  * @builder: A #GArrowUInt32ArrayBuilder.
@@ -788,13 +1329,38 @@ garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
      "[uint32-array-builder][append-null]");
 }
 
-
-G_DEFINE_TYPE(GArrowInt64ArrayBuilder,
-              garrow_int64_array_builder,
-              GARROW_TYPE_ARRAY_BUILDER)
-
-static void
-garrow_int64_array_builder_init(GArrowInt64ArrayBuilder *builder)
+/**
+ * garrow_uint32_array_builder_append_nulls:
+ * @builder: A #GArrowUInt32ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint32_array_builder_append_nulls(GArrowUInt32ArrayBuilder *builder,
+                                         gint64 n,
+                                         GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::UInt32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[uint32-array-builder][append-nulls]");
+}
+
+
+G_DEFINE_TYPE(GArrowInt64ArrayBuilder,
+              garrow_int64_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_int64_array_builder_init(GArrowInt64ArrayBuilder *builder)
 {
 }
 
@@ -837,6 +1403,43 @@ garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
      "[int64-array-builder][append]");
 }
 
+/**
+ * garrow_int64_array_builder_append_values:
+ * @builder: A #GArrowInt64ArrayBuilder.
+ * @values: (array length=values_length): The array of int64.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int64_array_builder_append_values(GArrowInt64ArrayBuilder *builder,
+                                         const gint64 *values,
+                                         gint64 values_length,
+                                         const gboolean *is_valids,
+                                         gint64 is_valids_length,
+                                         GError **error)
+{
+  return garrow_array_builder_append_values<arrow::Int64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     reinterpret_cast<const int64_t *>(values),
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[int64-array-builder][append-values]");
+}
+
 /**
  * garrow_int64_array_builder_append_null:
  * @builder: A #GArrowInt64ArrayBuilder.
@@ -854,6 +1457,31 @@ garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
      "[int64-array-builder][append-null]");
 }
 
+/**
+ * garrow_int64_array_builder_append_nulls:
+ * @builder: A #GArrowInt64ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_int64_array_builder_append_nulls(GArrowInt64ArrayBuilder *builder,
+                                        gint64 n,
+                                        GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::Int64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[int64-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowUInt64ArrayBuilder,
               garrow_uint64_array_builder,
@@ -903,6 +1531,43 @@ garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
      "[uint64-array-builder][append]");
 }
 
+/**
+ * garrow_uint64_array_builder_append_values:
+ * @builder: A #GArrowUInt64ArrayBuilder.
+ * @values: (array length=values_length): The array of uint64.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint64_array_builder_append_values(GArrowUInt64ArrayBuilder *builder,
+                                          const guint64 *values,
+                                          gint64 values_length,
+                                          const gboolean *is_valids,
+                                          gint64 is_valids_length,
+                                          GError **error)
+{
+  return garrow_array_builder_append_values<arrow::UInt64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     reinterpret_cast<const uint64_t *>(values),
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[uint64-array-builder][append-values]");
+}
+
 /**
  * garrow_uint64_array_builder_append_null:
  * @builder: A #GArrowUInt64ArrayBuilder.
@@ -912,7 +1577,7 @@ garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
  */
 gboolean
 garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
-                                       GError **error)
+                                        GError **error)
 {
   return garrow_array_builder_append_null<arrow::UInt64Builder *>
     (GARROW_ARRAY_BUILDER(builder),
@@ -920,6 +1585,31 @@ garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
      "[uint64-array-builder][append-null]");
 }
 
+/**
+ * garrow_uint64_array_builder_append_nulls:
+ * @builder: A #GArrowUInt64ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint64_array_builder_append_nulls(GArrowUInt64ArrayBuilder *builder,
+                                         gint64 n,
+                                         GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::UInt64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[uint64-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowFloatArrayBuilder,
               garrow_float_array_builder,
@@ -969,6 +1659,43 @@ garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
      "[float-array-builder][append]");
 }
 
+/**
+ * garrow_float_array_builder_append_values:
+ * @builder: A #GArrowFloatArrayBuilder.
+ * @values: (array length=values_length): The array of float.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_float_array_builder_append_values(GArrowFloatArrayBuilder *builder,
+                                         const gfloat *values,
+                                         gint64 values_length,
+                                         const gboolean *is_valids,
+                                         gint64 is_valids_length,
+                                         GError **error)
+{
+  return garrow_array_builder_append_values<arrow::FloatBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[float-array-builder][append-values]");
+}
+
 /**
  * garrow_float_array_builder_append_null:
  * @builder: A #GArrowFloatArrayBuilder.
@@ -986,6 +1713,31 @@ garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
      "[float-array-builder][append-null]");
 }
 
+/**
+ * garrow_float_array_builder_append_nulls:
+ * @builder: A #GArrowFloatArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_float_array_builder_append_nulls(GArrowFloatArrayBuilder *builder,
+                                        gint64 n,
+                                        GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::FloatBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[float-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowDoubleArrayBuilder,
               garrow_double_array_builder,
@@ -1035,6 +1787,43 @@ garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
      "[double-array-builder][append]");
 }
 
+/**
+ * garrow_double_array_builder_append_values:
+ * @builder: A #GArrowDoubleArrayBuilder.
+ * @values: (array length=values_length): The array of double.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_double_array_builder_append_values(GArrowDoubleArrayBuilder *builder,
+                                          const gdouble *values,
+                                          gint64 values_length,
+                                          const gboolean *is_valids,
+                                          gint64 is_valids_length,
+                                          GError **error)
+{
+  return garrow_array_builder_append_values<arrow::DoubleBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[double-array-builder][append-values]");
+}
+
 /**
  * garrow_double_array_builder_append_null:
  * @builder: A #GArrowDoubleArrayBuilder.
@@ -1052,6 +1841,31 @@ garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
      "[double-array-builder][append-null]");
 }
 
+/**
+ * garrow_double_array_builder_append_nulls:
+ * @builder: A #GArrowDoubleArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_double_array_builder_append_nulls(GArrowDoubleArrayBuilder *builder,
+                                         gint64 n,
+                                         GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::DoubleBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[double-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowBinaryArrayBuilder,
               garrow_binary_array_builder,
@@ -1225,6 +2039,44 @@ garrow_date32_array_builder_append(GArrowDate32ArrayBuilder *builder,
      "[date32-array-builder][append]");
 }
 
+/**
+ * garrow_date32_array_builder_append_values:
+ * @builder: A #GArrowDate32ArrayBuilder.
+ * @values: (array length=values_length): The array of
+ *   the number of days since UNIX epoch in signed 32bit integer.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_date32_array_builder_append_values(GArrowDate32ArrayBuilder *builder,
+                                          const gint32 *values,
+                                          gint64 values_length,
+                                          const gboolean *is_valids,
+                                          gint64 is_valids_length,
+                                          GError **error)
+{
+  return garrow_array_builder_append_values<arrow::Date32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[date32-array-builder][append-values]");
+}
+
 /**
  * garrow_date32_array_builder_append_null:
  * @builder: A #GArrowDate32ArrayBuilder.
@@ -1244,6 +2096,31 @@ garrow_date32_array_builder_append_null(GArrowDate32ArrayBuilder *builder,
      "[date32-array-builder][append-null]");
 }
 
+/**
+ * garrow_date32_array_builder_append_nulls:
+ * @builder: A #GArrowDate32ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_date32_array_builder_append_nulls(GArrowDate32ArrayBuilder *builder,
+                                         gint64 n,
+                                         GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::Date32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[date32-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowDate64ArrayBuilder,
               garrow_date64_array_builder,
@@ -1297,6 +2174,44 @@ garrow_date64_array_builder_append(GArrowDate64ArrayBuilder *builder,
      "[date64-array-builder][append]");
 }
 
+/**
+ * garrow_date64_array_builder_append_values:
+ * @builder: A #GArrowDate64ArrayBuilder.
+ * @values: (array length=values_length): The array of
+ *   the number of milliseconds since UNIX epoch in signed 64bit integer.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_date64_array_builder_append_values(GArrowDate64ArrayBuilder *builder,
+                                          const gint64 *values,
+                                          gint64 values_length,
+                                          const gboolean *is_valids,
+                                          gint64 is_valids_length,
+                                          GError **error)
+{
+  return garrow_array_builder_append_values<arrow::Date64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     reinterpret_cast<const int64_t *>(values),
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[date64-array-builder][append-values]");
+}
+
 /**
  * garrow_date64_array_builder_append_null:
  * @builder: A #GArrowDate64ArrayBuilder.
@@ -1316,6 +2231,31 @@ garrow_date64_array_builder_append_null(GArrowDate64ArrayBuilder *builder,
      "[date64-array-builder][append-null]");
 }
 
+/**
+ * garrow_date64_array_builder_append_nulls:
+ * @builder: A #GArrowDate64ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_date64_array_builder_append_nulls(GArrowDate64ArrayBuilder *builder,
+                                         gint64 n,
+                                         GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::Date64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[date64-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowTimestampArrayBuilder,
               garrow_timestamp_array_builder,
@@ -1371,6 +2311,44 @@ garrow_timestamp_array_builder_append(GArrowTimestampArrayBuilder *builder,
      "[timestamp-array-builder][append]");
 }
 
+/**
+ * garrow_timestamp_array_builder_append_values:
+ * @builder: A #GArrowTimestampArrayBuilder.
+ * @values: (array length=values_length): The array of
+ *   the number of milliseconds since UNIX epoch in signed 64bit integer.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_timestamp_array_builder_append_values(GArrowTimestampArrayBuilder *builder,
+                                             const gint64 *values,
+                                             gint64 values_length,
+                                             const gboolean *is_valids,
+                                             gint64 is_valids_length,
+                                             GError **error)
+{
+  return garrow_array_builder_append_values<arrow::TimestampBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     reinterpret_cast<const int64_t *>(values),
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[timestamp-array-builder][append-values]");
+}
+
 /**
  * garrow_timestamp_array_builder_append_null:
  * @builder: A #GArrowTimestampArrayBuilder.
@@ -1390,6 +2368,31 @@ garrow_timestamp_array_builder_append_null(GArrowTimestampArrayBuilder *builder,
      "[timestamp-array-builder][append-null]");
 }
 
+/**
+ * garrow_timestamp_array_builder_append_nulls:
+ * @builder: A #GArrowTimestampArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_timestamp_array_builder_append_nulls(GArrowTimestampArrayBuilder *builder,
+                                            gint64 n,
+                                            GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::TimestampBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[timestamp-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowTime32ArrayBuilder,
               garrow_time32_array_builder,
@@ -1445,6 +2448,44 @@ garrow_time32_array_builder_append(GArrowTime32ArrayBuilder *builder,
      "[time32-array-builder][append]");
 }
 
+/**
+ * garrow_time32_array_builder_append_values:
+ * @builder: A #GArrowTime32ArrayBuilder.
+ * @values: (array length=values_length): The array of
+ *   the number of days since UNIX epoch in signed 32bit integer.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_time32_array_builder_append_values(GArrowTime32ArrayBuilder *builder,
+                                          const gint32 *values,
+                                          gint64 values_length,
+                                          const gboolean *is_valids,
+                                          gint64 is_valids_length,
+                                          GError **error)
+{
+  return garrow_array_builder_append_values<arrow::Time32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     values,
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[time32-array-builder][append-values]");
+}
+
 /**
  * garrow_time32_array_builder_append_null:
  * @builder: A #GArrowTime32ArrayBuilder.
@@ -1464,6 +2505,31 @@ garrow_time32_array_builder_append_null(GArrowTime32ArrayBuilder *builder,
      "[time32-array-builder][append-null]");
 }
 
+/**
+ * garrow_time32_array_builder_append_nulls:
+ * @builder: A #GArrowTime32ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_time32_array_builder_append_nulls(GArrowTime32ArrayBuilder *builder,
+                                         gint64 n,
+                                         GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::Time32Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[time32-array-builder][append-nulls]");
+}
+
 
 G_DEFINE_TYPE(GArrowTime64ArrayBuilder,
               garrow_time64_array_builder,
@@ -1519,6 +2585,44 @@ garrow_time64_array_builder_append(GArrowTime64ArrayBuilder *builder,
      "[time64-array-builder][append]");
 }
 
+/**
+ * garrow_time64_array_builder_append_values:
+ * @builder: A #GArrowTime64ArrayBuilder.
+ * @values: (array length=values_length): The array of
+ *   the number of milliseconds since UNIX epoch in signed 64bit integer.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_time64_array_builder_append_values(GArrowTime64ArrayBuilder *builder,
+                                          const gint64 *values,
+                                          gint64 values_length,
+                                          const gboolean *is_valids,
+                                          gint64 is_valids_length,
+                                          GError **error)
+{
+  return garrow_array_builder_append_values<arrow::Time64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     reinterpret_cast<const int64_t *>(values),
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[time64-array-builder][append-values]");
+}
+
 /**
  * garrow_time64_array_builder_append_null:
  * @builder: A #GArrowTime64ArrayBuilder.
@@ -1538,6 +2642,31 @@ garrow_time64_array_builder_append_null(GArrowTime64ArrayBuilder *builder,
      "[time64-array-builder][append-null]");
 }
 
+/**
+ * garrow_time64_array_builder_append_nulls:
+ * @builder: A #GArrowTime64ArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_time64_array_builder_append_nulls(GArrowTime64ArrayBuilder *builder,
+                                         gint64 n,
+                                         GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::Time64Builder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[time64-array-builder][append-nulls]");
+}
+
 
 typedef struct GArrowListArrayBuilderPrivate_ {
   GArrowArrayBuilder *value_builder;
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index f28959ffb5f96..87808549afa97 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -116,8 +116,17 @@ GArrowBooleanArrayBuilder *garrow_boolean_array_builder_new(void);
 gboolean garrow_boolean_array_builder_append(GArrowBooleanArrayBuilder *builder,
                                              gboolean value,
                                              GError **error);
+gboolean garrow_boolean_array_builder_append_values(GArrowBooleanArrayBuilder *builder,
+                                                    const gboolean *values,
+                                                    gint64 values_length,
+                                                    const gboolean *is_valids,
+                                                    gint64 is_valids_length,
+                                                    GError **error);
 gboolean garrow_boolean_array_builder_append_null(GArrowBooleanArrayBuilder *builder,
                                                   GError **error);
+gboolean garrow_boolean_array_builder_append_nulls(GArrowBooleanArrayBuilder *builder,
+                                                   gint64 n,
+                                                   GError **error);
 
 
 #define GARROW_TYPE_INT_ARRAY_BUILDER           \
@@ -167,8 +176,17 @@ GArrowIntArrayBuilder *garrow_int_array_builder_new(void);
 gboolean garrow_int_array_builder_append(GArrowIntArrayBuilder *builder,
                                          gint64 value,
                                          GError **error);
+gboolean garrow_int_array_builder_append_values(GArrowIntArrayBuilder *builder,
+                                                const gint64 *values,
+                                                gint64 values_length,
+                                                const gboolean *is_valids,
+                                                gint64 is_valids_length,
+                                                GError **error);
 gboolean garrow_int_array_builder_append_null(GArrowIntArrayBuilder *builder,
                                               GError **error);
+gboolean garrow_int_array_builder_append_nulls(GArrowIntArrayBuilder *builder,
+                                               gint64 n,
+                                               GError **error);
 
 
 #define GARROW_TYPE_INT8_ARRAY_BUILDER          \
@@ -218,8 +236,17 @@ GArrowInt8ArrayBuilder *garrow_int8_array_builder_new(void);
 gboolean garrow_int8_array_builder_append(GArrowInt8ArrayBuilder *builder,
                                           gint8 value,
                                           GError **error);
+gboolean garrow_int8_array_builder_append_values(GArrowInt8ArrayBuilder *builder,
+                                                 const gint8 *values,
+                                                 gint64 values_length,
+                                                 const gboolean *is_valids,
+                                                 gint64 is_valids_length,
+                                                 GError **error);
 gboolean garrow_int8_array_builder_append_null(GArrowInt8ArrayBuilder *builder,
                                                GError **error);
+gboolean garrow_int8_array_builder_append_nulls(GArrowInt8ArrayBuilder *builder,
+                                                gint64 n,
+                                                GError **error);
 
 
 #define GARROW_TYPE_UINT8_ARRAY_BUILDER         \
@@ -269,8 +296,17 @@ GArrowUInt8ArrayBuilder *garrow_uint8_array_builder_new(void);
 gboolean garrow_uint8_array_builder_append(GArrowUInt8ArrayBuilder *builder,
                                            guint8 value,
                                            GError **error);
+gboolean garrow_uint8_array_builder_append_values(GArrowUInt8ArrayBuilder *builder,
+                                                  const guint8 *values,
+                                                  gint64 values_length,
+                                                  const gboolean *is_valids,
+                                                  gint64 is_valids_length,
+                                                  GError **error);
 gboolean garrow_uint8_array_builder_append_null(GArrowUInt8ArrayBuilder *builder,
                                                 GError **error);
+gboolean garrow_uint8_array_builder_append_nulls(GArrowUInt8ArrayBuilder *builder,
+                                                 gint64 n,
+                                                 GError **error);
 
 
 #define GARROW_TYPE_INT16_ARRAY_BUILDER         \
@@ -320,8 +356,17 @@ GArrowInt16ArrayBuilder *garrow_int16_array_builder_new(void);
 gboolean garrow_int16_array_builder_append(GArrowInt16ArrayBuilder *builder,
                                            gint16 value,
                                            GError **error);
+gboolean garrow_int16_array_builder_append_values(GArrowInt16ArrayBuilder *builder,
+                                                  const gint16 *values,
+                                                  gint64 values_length,
+                                                  const gboolean *is_valids,
+                                                  gint64 is_valids_length,
+                                                  GError **error);
 gboolean garrow_int16_array_builder_append_null(GArrowInt16ArrayBuilder *builder,
                                                 GError **error);
+gboolean garrow_int16_array_builder_append_nulls(GArrowInt16ArrayBuilder *builder,
+                                                 gint64 n,
+                                                 GError **error);
 
 
 #define GARROW_TYPE_UINT16_ARRAY_BUILDER        \
@@ -371,8 +416,17 @@ GArrowUInt16ArrayBuilder *garrow_uint16_array_builder_new(void);
 gboolean garrow_uint16_array_builder_append(GArrowUInt16ArrayBuilder *builder,
                                             guint16 value,
                                             GError **error);
+gboolean garrow_uint16_array_builder_append_values(GArrowUInt16ArrayBuilder *builder,
+                                                   const guint16 *values,
+                                                   gint64 values_length,
+                                                   const gboolean *is_valids,
+                                                   gint64 is_valids_length,
+                                                   GError **error);
 gboolean garrow_uint16_array_builder_append_null(GArrowUInt16ArrayBuilder *builder,
                                                  GError **error);
+gboolean garrow_uint16_array_builder_append_nulls(GArrowUInt16ArrayBuilder *builder,
+                                                  gint64 n,
+                                                  GError **error);
 
 
 #define GARROW_TYPE_INT32_ARRAY_BUILDER         \
@@ -422,8 +476,17 @@ GArrowInt32ArrayBuilder *garrow_int32_array_builder_new(void);
 gboolean garrow_int32_array_builder_append(GArrowInt32ArrayBuilder *builder,
                                            gint32 value,
                                            GError **error);
+gboolean garrow_int32_array_builder_append_values(GArrowInt32ArrayBuilder *builder,
+                                                  const gint32 *values,
+                                                  gint64 values_length,
+                                                  const gboolean *is_valids,
+                                                  gint64 is_valids_length,
+                                                  GError **error);
 gboolean garrow_int32_array_builder_append_null(GArrowInt32ArrayBuilder *builder,
                                                 GError **error);
+gboolean garrow_int32_array_builder_append_nulls(GArrowInt32ArrayBuilder *builder,
+                                                 gint64 n,
+                                                 GError **error);
 
 
 #define GARROW_TYPE_UINT32_ARRAY_BUILDER        \
@@ -473,8 +536,17 @@ GArrowUInt32ArrayBuilder *garrow_uint32_array_builder_new(void);
 gboolean garrow_uint32_array_builder_append(GArrowUInt32ArrayBuilder *builder,
                                             guint32 value,
                                             GError **error);
+gboolean garrow_uint32_array_builder_append_values(GArrowUInt32ArrayBuilder *builder,
+                                                   const guint32 *values,
+                                                   gint64 values_length,
+                                                   const gboolean *is_valids,
+                                                   gint64 is_valids_length,
+                                                   GError **error);
 gboolean garrow_uint32_array_builder_append_null(GArrowUInt32ArrayBuilder *builder,
                                                  GError **error);
+gboolean garrow_uint32_array_builder_append_nulls(GArrowUInt32ArrayBuilder *builder,
+                                                  gint64 n,
+                                                  GError **error);
 
 
 #define GARROW_TYPE_INT64_ARRAY_BUILDER         \
@@ -524,8 +596,17 @@ GArrowInt64ArrayBuilder *garrow_int64_array_builder_new(void);
 gboolean garrow_int64_array_builder_append(GArrowInt64ArrayBuilder *builder,
                                            gint64 value,
                                            GError **error);
+gboolean garrow_int64_array_builder_append_values(GArrowInt64ArrayBuilder *builder,
+                                                  const gint64 *values,
+                                                  gint64 values_length,
+                                                  const gboolean *is_valids,
+                                                  gint64 is_valids_length,
+                                                  GError **error);
 gboolean garrow_int64_array_builder_append_null(GArrowInt64ArrayBuilder *builder,
                                                 GError **error);
+gboolean garrow_int64_array_builder_append_nulls(GArrowInt64ArrayBuilder *builder,
+                                                 gint64 n,
+                                                 GError **error);
 
 
 #define GARROW_TYPE_UINT64_ARRAY_BUILDER        \
@@ -575,8 +656,17 @@ GArrowUInt64ArrayBuilder *garrow_uint64_array_builder_new(void);
 gboolean garrow_uint64_array_builder_append(GArrowUInt64ArrayBuilder *builder,
                                             guint64 value,
                                             GError **error);
+gboolean garrow_uint64_array_builder_append_values(GArrowUInt64ArrayBuilder *builder,
+                                                   const guint64 *values,
+                                                   gint64 values_length,
+                                                   const gboolean *is_valids,
+                                                   gint64 is_valids_length,
+                                                   GError **error);
 gboolean garrow_uint64_array_builder_append_null(GArrowUInt64ArrayBuilder *builder,
                                                  GError **error);
+gboolean garrow_uint64_array_builder_append_nulls(GArrowUInt64ArrayBuilder *builder,
+                                                  gint64 n,
+                                                  GError **error);
 
 
 #define GARROW_TYPE_FLOAT_ARRAY_BUILDER         \
@@ -626,8 +716,17 @@ GArrowFloatArrayBuilder *garrow_float_array_builder_new(void);
 gboolean garrow_float_array_builder_append(GArrowFloatArrayBuilder *builder,
                                            gfloat value,
                                            GError **error);
+gboolean garrow_float_array_builder_append_values(GArrowFloatArrayBuilder *builder,
+                                                  const gfloat *values,
+                                                  gint64 values_length,
+                                                  const gboolean *is_valids,
+                                                  gint64 is_valids_length,
+                                                  GError **error);
 gboolean garrow_float_array_builder_append_null(GArrowFloatArrayBuilder *builder,
                                                 GError **error);
+gboolean garrow_float_array_builder_append_nulls(GArrowFloatArrayBuilder *builder,
+                                                 gint64 n,
+                                                 GError **error);
 
 
 #define GARROW_TYPE_DOUBLE_ARRAY_BUILDER        \
@@ -677,8 +776,17 @@ GArrowDoubleArrayBuilder *garrow_double_array_builder_new(void);
 gboolean garrow_double_array_builder_append(GArrowDoubleArrayBuilder *builder,
                                             gdouble value,
                                             GError **error);
+gboolean garrow_double_array_builder_append_values(GArrowDoubleArrayBuilder *builder,
+                                                   const gdouble *values,
+                                                   gint64 values_length,
+                                                   const gboolean *is_valids,
+                                                   gint64 is_valids_length,
+                                                   GError **error);
 gboolean garrow_double_array_builder_append_null(GArrowDoubleArrayBuilder *builder,
                                                  GError **error);
+gboolean garrow_double_array_builder_append_nulls(GArrowDoubleArrayBuilder *builder,
+                                                  gint64 n,
+                                                  GError **error);
 
 
 #define GARROW_TYPE_BINARY_ARRAY_BUILDER        \
@@ -829,8 +937,17 @@ GArrowDate32ArrayBuilder *garrow_date32_array_builder_new(void);
 gboolean garrow_date32_array_builder_append(GArrowDate32ArrayBuilder *builder,
                                             gint32 value,
                                             GError **error);
+gboolean garrow_date32_array_builder_append_values(GArrowDate32ArrayBuilder *builder,
+                                                   const gint32 *values,
+                                                   gint64 values_length,
+                                                   const gboolean *is_valids,
+                                                   gint64 is_valids_length,
+                                                   GError **error);
 gboolean garrow_date32_array_builder_append_null(GArrowDate32ArrayBuilder *builder,
                                                  GError **error);
+gboolean garrow_date32_array_builder_append_nulls(GArrowDate32ArrayBuilder *builder,
+                                                  gint64 n,
+                                                  GError **error);
 
 
 #define GARROW_TYPE_DATE64_ARRAY_BUILDER        \
@@ -880,8 +997,17 @@ GArrowDate64ArrayBuilder *garrow_date64_array_builder_new(void);
 gboolean garrow_date64_array_builder_append(GArrowDate64ArrayBuilder *builder,
                                             gint64 value,
                                             GError **error);
+gboolean garrow_date64_array_builder_append_values(GArrowDate64ArrayBuilder *builder,
+                                                   const gint64 *values,
+                                                   gint64 values_length,
+                                                   const gboolean *is_valids,
+                                                   gint64 is_valids_length,
+                                                   GError **error);
 gboolean garrow_date64_array_builder_append_null(GArrowDate64ArrayBuilder *builder,
                                                  GError **error);
+gboolean garrow_date64_array_builder_append_nulls(GArrowDate64ArrayBuilder *builder,
+                                                  gint64 n,
+                                                  GError **error);
 
 
 #define GARROW_TYPE_TIMESTAMP_ARRAY_BUILDER     \
@@ -932,8 +1058,17 @@ garrow_timestamp_array_builder_new(GArrowTimestampDataType *data_type);
 gboolean garrow_timestamp_array_builder_append(GArrowTimestampArrayBuilder *builder,
                                                gint64 value,
                                                GError **error);
+gboolean garrow_timestamp_array_builder_append_values(GArrowTimestampArrayBuilder *builder,
+                                                      const gint64 *values,
+                                                      gint64 values_length,
+                                                      const gboolean *is_valids,
+                                                      gint64 is_valids_length,
+                                                      GError **error);
 gboolean garrow_timestamp_array_builder_append_null(GArrowTimestampArrayBuilder *builder,
                                                     GError **error);
+gboolean garrow_timestamp_array_builder_append_nulls(GArrowTimestampArrayBuilder *builder,
+                                                     gint64 n,
+                                                     GError **error);
 
 
 #define GARROW_TYPE_TIME32_ARRAY_BUILDER        \
@@ -983,8 +1118,17 @@ GArrowTime32ArrayBuilder *garrow_time32_array_builder_new(GArrowTime32DataType *
 gboolean garrow_time32_array_builder_append(GArrowTime32ArrayBuilder *builder,
                                             gint32 value,
                                             GError **error);
+gboolean garrow_time32_array_builder_append_values(GArrowTime32ArrayBuilder *builder,
+                                                   const gint32 *values,
+                                                   gint64 values_length,
+                                                   const gboolean *is_valids,
+                                                   gint64 is_valids_length,
+                                                   GError **error);
 gboolean garrow_time32_array_builder_append_null(GArrowTime32ArrayBuilder *builder,
                                                  GError **error);
+gboolean garrow_time32_array_builder_append_nulls(GArrowTime32ArrayBuilder *builder,
+                                                  gint64 n,
+                                                  GError **error);
 
 
 #define GARROW_TYPE_TIME64_ARRAY_BUILDER        \
@@ -1034,8 +1178,17 @@ GArrowTime64ArrayBuilder *garrow_time64_array_builder_new(GArrowTime64DataType *
 gboolean garrow_time64_array_builder_append(GArrowTime64ArrayBuilder *builder,
                                             gint64 value,
                                             GError **error);
+gboolean garrow_time64_array_builder_append_values(GArrowTime64ArrayBuilder *builder,
+                                                   const gint64 *values,
+                                                   gint64 values_length,
+                                                   const gboolean *is_valids,
+                                                   gint64 is_valids_length,
+                                                   GError **error);
 gboolean garrow_time64_array_builder_append_null(GArrowTime64ArrayBuilder *builder,
                                                  GError **error);
+gboolean garrow_time64_array_builder_append_nulls(GArrowTime64ArrayBuilder *builder,
+                                                  gint64 n,
+                                                  GError **error);
 
 
 #define GARROW_TYPE_LIST_ARRAY_BUILDER          \
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index 3366fc8ebe54a..fc161a5864c24 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -133,6 +133,10 @@
     <title>Index of deprecated API</title>
     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
   </index>
+  <index id="api-index-0-8-0" role="0.8.0">
+    <title>Index of new symbols in 0.8.0</title>
+    <xi:include href="xml/api-index-0.8.0.xml"><xi:fallback /></xi:include>
+  </index>
   <index id="api-index-0-7-0" role="0.7.0">
     <title>Index of new symbols in 0.7.0</title>
     <xi:include href="xml/api-index-0.7.0.xml"><xi:fallback /></xi:include>
diff --git a/c_glib/test/test-array-builder.rb b/c_glib/test/test-array-builder.rb
new file mode 100644
index 0000000000000..6c0f984854d87
--- /dev/null
+++ b/c_glib/test/test-array-builder.rb
@@ -0,0 +1,485 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module ArrayBuilderAppendValuesTests
+  def test_empty
+    builder = create_builder
+    builder.append_values([])
+    assert_equal(build_array([]),
+                 builder.finish)
+  end
+
+  def test_values_only
+    builder = create_builder
+    builder.append_values(sample_values)
+    assert_equal(build_array(sample_values),
+                 builder.finish)
+  end
+
+  def test_with_is_valids
+    builder = create_builder
+    builder.append_values(sample_values, [true, true, false])
+    sample_values_with_null = sample_values
+    sample_values_with_null[2] = nil
+    assert_equal(build_array(sample_values_with_null),
+                 builder.finish)
+  end
+
+  def test_mismatch_length
+    builder = create_builder
+    message = "[#{builder_class_name}][append-values]: " +
+      "values length and is_valids length must be equal: <3> != <2>"
+    assert_raise(Arrow::Error::Invalid.new(message)) do
+      builder.append_values(sample_values, [true, true])
+    end
+  end
+end
+
+module ArrayBuilderAppendNullsTests
+  def test_zero
+    builder = create_builder
+    builder.append_nulls(0)
+    assert_equal(build_array([]),
+                 builder.finish)
+  end
+
+  def test_positive
+    builder = create_builder
+    builder.append_nulls(3)
+    assert_equal(build_array([nil, nil, nil]),
+                 builder.finish)
+  end
+
+  def test_negative
+    builder = create_builder
+    message = "[#{builder_class_name}][append-nulls]: " +
+      "the number of nulls must be 0 or larger: <-1>"
+    assert_raise(Arrow::Error::Invalid.new(message)) do
+      builder.append_nulls(-1)
+    end
+  end
+end
+
+class TestArrayBuilder < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def setup
+    require_gi_bindings(3, 1, 9)
+  end
+
+  def build_array(values)
+    super(create_builder, values)
+  end
+
+  sub_test_case("BooleanArrayBuilder") do
+    def create_builder
+      Arrow::BooleanArrayBuilder.new
+    end
+
+    def builder_class_name
+      "boolean-array-builder"
+    end
+
+    def sample_values
+      [true, false, true]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("IntArrayBuilder") do
+    def create_builder
+      Arrow::IntArrayBuilder.new
+    end
+
+    def builder_class_name
+      "int-array-builder"
+    end
+
+    def sample_values
+      [1, -2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("Int8ArrayBuilder") do
+    def create_builder
+      Arrow::Int8ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "int8-array-builder"
+    end
+
+    def sample_values
+      [1, -2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("UInt8ArrayBuilder") do
+    def create_builder
+      Arrow::UInt8ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "uint8-array-builder"
+    end
+
+    def sample_values
+      [1, 2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("Int16ArrayBuilder") do
+    def create_builder
+      Arrow::Int16ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "int16-array-builder"
+    end
+
+    def sample_values
+      [1, -2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("UInt16ArrayBuilder") do
+    def create_builder
+      Arrow::UInt16ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "uint16-array-builder"
+    end
+
+    def sample_values
+      [1, 2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("Int32ArrayBuilder") do
+    def create_builder
+      Arrow::Int32ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "int32-array-builder"
+    end
+
+    def sample_values
+      [1, -2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("UInt32ArrayBuilder") do
+    def create_builder
+      Arrow::UInt32ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "uint32-array-builder"
+    end
+
+    def sample_values
+      [1, 2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("Int64ArrayBuilder") do
+    def create_builder
+      Arrow::Int64ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "int64-array-builder"
+    end
+
+    def sample_values
+      [1, -2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("UInt64ArrayBuilder") do
+    def create_builder
+      Arrow::UInt64ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "uint64-array-builder"
+    end
+
+    def sample_values
+      [1, 2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("FloatArrayBuilder") do
+    def create_builder
+      Arrow::FloatArrayBuilder.new
+    end
+
+    def builder_class_name
+      "float-array-builder"
+    end
+
+    def sample_values
+      [1.1, -2.2, 3.3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("DoubleArrayBuilder") do
+    def create_builder
+      Arrow::DoubleArrayBuilder.new
+    end
+
+    def builder_class_name
+      "double-array-builder"
+    end
+
+    def sample_values
+      [1.1, -2.2, 3.3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("Date32ArrayBuilder") do
+    def create_builder
+      Arrow::Date32ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "date32-array-builder"
+    end
+
+    def sample_values
+      [
+        0,     # epoch
+        17406, # 2017-08-28
+        17427, # 2017-09-18
+      ]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("Date64ArrayBuilder") do
+    def create_builder
+      Arrow::Date64ArrayBuilder.new
+    end
+
+    def builder_class_name
+      "date64-array-builder"
+    end
+
+    def sample_values
+      [
+        -315619200,    # 1960-01-01T00:00:00Z
+        0,             # epoch
+        1503878400000, # 2017-08-28T00:00:00Z
+      ]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("TimestampArrayBuilder") do
+    def create_builder
+      data_type = Arrow::TimestampDataType.new(:milli)
+      Arrow::TimestampArrayBuilder.new(data_type)
+    end
+
+    def builder_class_name
+      "timestamp-array-builder"
+    end
+
+    def sample_values
+      [
+        0,             # epoch
+        1504953190854, # 2017-09-09T10:33:10.854Z
+        1505660812942, # 2017-09-17T15:06:52.942Z
+      ]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("Time32ArrayBuilder") do
+    def create_builder
+      data_type = Arrow::Time32DataType.new(:second)
+      Arrow::Time32ArrayBuilder.new(data_type)
+    end
+
+    def builder_class_name
+      "time32-array-builder"
+    end
+
+    def sample_values
+      [
+        0,                # midnight
+        60 * 10,          # 00:10:00
+        60 * 60 * 2 + 30, # 02:00:30
+      ]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
+  sub_test_case("Time64ArrayBuilder") do
+    def create_builder
+      data_type = Arrow::Time64DataType.new(:micro)
+      Arrow::Time64ArrayBuilder.new(data_type)
+    end
+
+    def builder_class_name
+      "time64-array-builder"
+    end
+
+    def sample_values
+      [
+        0,                                # midnight
+        60 * 10 * 1000 * 1000,            # 00:10:00.000000
+        (60 * 60 * 2 + 30) * 1000 * 1000, # 02:00:30.000000
+      ]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+end

From e1d9c7fc7b3bbbb390f336551b95d3ca881c9285 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 19 Sep 2017 00:23:03 -0400
Subject: [PATCH 1064/1644] ARROW-1550: [Python] Explicitly close owned file
 handles in ParquetWriter.close to avoid Windows flakiness

I can reproduce this failure locally, but I'm unsure why this just now started happening. The 0.7.0 release build passed (https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/build/1.0.3357/job/477b1iicmwuy51l8) and there haven't been related code changes since then. Either way it's better to close the sink explicitly

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1114 from wesm/ARROW-1550 and squashes the following commits:

863827c4 [Wes McKinney] Check status
7248c793 [Wes McKinney] Explicitly close owned file handles in ParquetWriter.close to avoid flakiness on Windows
---
 python/pyarrow/_parquet.pyx          | 5 +++++
 python/pyarrow/tests/test_parquet.py | 3 ---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index aea6fb608501e..b096fa1b4d337 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -562,6 +562,7 @@ cdef class ParquetWriter:
     cdef:
         unique_ptr[FileWriter] writer
         shared_ptr[OutputStream] sink
+        bint own_sink
 
     cdef readonly:
         object use_dictionary
@@ -588,8 +589,10 @@ cdef class ParquetWriter:
                 check_status(FileOutputStream.Open(c_where,
                                                    &filestream))
             self.sink = <shared_ptr[OutputStream]> filestream
+            self.own_sink = True
         else:
             get_writer(where, &self.sink)
+            self.own_sink = False
 
         self.use_dictionary = use_dictionary
         self.compression = compression
@@ -664,6 +667,8 @@ cdef class ParquetWriter:
     def close(self):
         with nogil:
             check_status(self.writer.get().Close())
+            if self.own_sink:
+                check_status(self.sink.get().Close())
 
     def write_table(self, Table table, row_group_size=None):
         cdef CTable* ctable = table.table
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 9b5a4bcc43c51..790b47832f226 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1202,9 +1202,6 @@ def test_write_error_deletes_incomplete_file(tmpdir):
     except pa.ArrowException:
         pass
 
-    # Ensure that object has been destructed; this causes test failures on
-    # Windows
-    gc.collect()
     assert not os.path.exists(filename)
 
 

From 0d5e699c2d8a52f38f1c061b785e5af39ee30f95 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 19 Sep 2017 09:10:06 -0400
Subject: [PATCH 1065/1644] ARROW-1551: [Website] Website updates, blog post
 for 0.7.0

I drafted a post to publish tomorrow. If anyone would like to make some changes or additions please post a link to a git commit here for me to cherry pick

cc @kou @trxcllnt

@pcmoritz I think we should write a whole blog post about the object serialization functions. The perf wins over pickle when working with large datasets are a pretty big deal

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1111 from wesm/ARROW-1551 and squashes the following commits:

3e05047e [Wes McKinney] Update publication date to 19 September
a9f8770e [Wes McKinney] More edits, links
8c877d9c [Wes McKinney] Draft 0.7.0 release post
---
 site/_posts/2017-09-19-0.7.0-release.md | 190 ++++++++++++++++++++++++
 site/_release/index.md                  |   2 +
 site/index.html                         |  27 +++-
 site/install.md                         |  32 ++--
 4 files changed, 227 insertions(+), 24 deletions(-)
 create mode 100644 site/_posts/2017-09-19-0.7.0-release.md

diff --git a/site/_posts/2017-09-19-0.7.0-release.md b/site/_posts/2017-09-19-0.7.0-release.md
new file mode 100644
index 0000000000000..dd253df61cf3a
--- /dev/null
+++ b/site/_posts/2017-09-19-0.7.0-release.md
@@ -0,0 +1,190 @@
+---
+layout: post
+title: "Apache Arrow 0.7.0 Release"
+date: "2017-09-19 00:00:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.7.0 release. It includes
+[**133 resolved JIRAs**][1] many new features and bug fixes to the various
+language implementations. The Arrow memory format remains stable since the
+0.3.x release.
+
+See the [Install Page][2] to learn how to get the libraries for your
+platform. The [complete changelog][3] is also available.
+
+We include some highlights from the release in this post.
+
+## New PMC Member: Kouhei Sutou
+
+Since the last release we have added [Kou][4] to the Arrow Project Management
+Committee. He is also a PMC for Apache Subversion, and a major contributor to
+many other open source projects.
+
+As an active member of the Ruby community in Japan, Kou has been developing the
+GLib-based C bindings for Arrow with associated Ruby wrappers, to enable Ruby
+users to benefit from the work that's happening in Apache Arrow.
+
+We are excited to be collaborating with the Ruby community on shared
+infrastructure for in-memory analytics and data science.
+
+## Expanded JavaScript (TypeScript) Implementation
+
+[Paul Taylor][5] from the [Falcor][7] and [ReactiveX][6] projects has worked to
+expand the JavaScript implementation (which is written in TypeScript), using
+the latest in modern JavaScript build and packaging technology. We are looking
+forward to building out the JS implementation and bringing it up to full
+functionality with the C++ and Java implementations.
+
+We are looking for more JavaScript developers to join the project and work
+together to make Arrow for JS work well with many kinds of front end use cases,
+like real time data visualization.
+
+## Type casting for C++ and Python
+
+As part of longer-term efforts to build an Arrow-native in-memory analytics
+library, we implemented a variety of type conversion functions. These functions
+are essential in ETL tasks when conforming one table schema to another. These
+are similar to the `astype` function in NumPy.
+
+```python
+In [17]: import pyarrow as pa
+
+In [18]: arr = pa.array([True, False, None, True])
+
+In [19]: arr
+Out[19]:
+<pyarrow.lib.BooleanArray object at 0x7ff6fb069b88>
+[
+  True,
+  False,
+  NA,
+  True
+]
+
+In [20]: arr.cast(pa.int32())
+Out[20]:
+<pyarrow.lib.Int32Array object at 0x7ff6fb0383b8>
+[
+  1,
+  0,
+  NA,
+  1
+]
+```
+
+Over time these will expand to support as many input-and-output type
+combinations with optimized conversions.
+
+## New Arrow GPU (CUDA) Extension Library for C++
+
+To help with GPU-related projects using Arrow, like the [GPU Open Analytics
+Initiative][8], we have started a C++ add-on library to simplify Arrow memory
+management on CUDA-enabled graphics cards. We would like to expand this to
+include a library of reusable CUDA kernel functions for GPU analytics on Arrow
+columnar memory.
+
+For example, we could write a record batch from CPU memory to GPU device memory
+like so (some error checking omitted):
+
+```c++
+#include <arrow/api.h>
+#include <arrow/gpu/cuda_api.h>
+
+using namespace arrow;
+
+gpu::CudaDeviceManager* manager;
+std::shared_ptr<gpu::CudaContext> context;
+
+gpu::CudaDeviceManager::GetInstance(&manager)
+manager_->GetContext(kGpuNumber, &context);
+
+std::shared_ptr<RecordBatch> batch = GetCpuData();
+
+std::shared_ptr<gpu::CudaBuffer> device_serialized;
+gpu::SerializeRecordBatch(*batch, context_.get(), &device_serialized));
+```
+
+We can then "read" the GPU record batch, but the returned `arrow::RecordBatch`
+internally will contain GPU device pointers that you can use for CUDA kernel
+calls:
+
+```
+std::shared_ptr<RecordBatch> device_batch;
+gpu::ReadRecordBatch(batch->schema(), device_serialized,
+                     default_memory_pool(), &device_batch));
+
+// Now run some CUDA kernels on device_batch
+```
+
+## Decimal Integration Tests
+
+[Phillip Cloud][9] has been working on decimal support in C++ to enable Parquet
+read/write support in C++ and Python, and also end-to-end testing against the
+Arrow Java libraries.
+
+In the upcoming releases, we hope to complete the remaining data types that
+need end-to-end testing between Java and C++:
+
+* Fixed size lists (variable-size lists already implemented)
+* Fixes size binary
+* Unions
+* Maps
+* Time intervals
+
+## Other Notable Python Changes
+
+Some highlights of Python development outside of bug fixes and general API
+improvements include:
+
+* Simplified `put` and `get` arbitrary Python objects in Plasma objects
+* [High-speed, memory efficient object serialization][10]. This is important
+  enough that we will likely write a dedicated blog post about it.
+* New `flavor='spark'` option to `pyarrow.parquet.write_table` to enable easy
+  writing of Parquet files maximized for Spark compatibility
+* `parquet.write_to_dataset` function with support for partitioned writes
+* Improved support for Dask filesystems
+* Improved Python usability for IPC: read and write schemas and record batches
+  more easily. See the [API docs][11] for more about these.
+
+## The Road Ahead
+
+Upcoming Arrow releases will continue to expand the project to cover more use
+cases. In addition to completing end-to-end testing for all the major data
+types, some of us will be shifting attention to building Arrow-native in-memory
+analytics libraries.
+
+We are looking for more JavaScript, R, and other programming language
+developers to join the project and expand the available implementations and
+bindings to more languages.
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.7.0
+[2]: http://arrow.apache.org/install
+[3]: http://arrow.apache.org/release/0.7.0.html
+[4]: https://github.com/kou
+[5]: https://github.com/trxcllnt
+[6]: http://reactivex.io
+[7]: https://github.com/netflix/falcor
+[8]: http://gpuopenanalytics.com/
+[9]: http://github.com/cpcloud
+[10]: http://arrow.apache.org/docs/python/ipc.html
+[11]: http://arrow.apache.org/docs/python/api.html
\ No newline at end of file
diff --git a/site/_release/index.md b/site/_release/index.md
index b373d8bfe1993..ad78de8f3955e 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -26,6 +26,7 @@ limitations under the License.
 
 Navigate to the release page for downloads and the changelog.
 
+* [0.7.0 (17 September 2017)][8]
 * [0.6.0 (14 August 2017)][7]
 * [0.5.0 (23 July 2017)][6]
 * [0.4.1 (9 June 2017)][5]
@@ -41,3 +42,4 @@ Navigate to the release page for downloads and the changelog.
 [5]: {{ site.baseurl }}/release/0.4.1.html
 [6]: {{ site.baseurl }}/release/0.5.0.html
 [7]: {{ site.baseurl }}/release/0.6.0.html
+[8]: {{ site.baseurl }}/release/0.7.0.html
diff --git a/site/index.html b/site/index.html
index 95d9f4df1e507..01836d7bf5b51 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,26 +7,37 @@ <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
         <p>
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
-          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.6.0 Release - August 14, 2017)</a>
+          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.7.0 Release - September 17, 2017)</a>
         </p>
       </div>
-      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.6.0 release</a></h4>
+      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.7.0 release</a></h4>
       <div class="row">
         <div class="col-lg-4">
           <h2>Fast</h2>
-          <p>Apache Arrow&#8482; enables execution engines to take advantage of the latest SIM
-D (Single input multiple data) operations included in modern processors, for native vectorized optimization of analytical data processing. Columnar layout of data also allows for a better use of CPU caches by placing all data relevant to a column operation in as compact of a format
- as possible.</p>
+          <p>Apache Arrow&#8482; enables execution engines to take advantage of
+ the latest SIMD (Single input multiple data) operations included in modern
+ processors, for native vectorized optimization of analytical data
+ processing. Columnar layout is optimized for data locality for better
+ performance on modern hardware like CPUs and GPUs.</p>
+
           <p>The Arrow memory format supports <strong>zero-copy reads</strong>
           for lightning-fast data access without serialization overhead.</p>
+
         </div>
         <div class="col-lg-4">
           <h2>Flexible</h2>
-          <p>Arrow acts as a new high-performance interface between various systems. It is also focused on supporting a wide variety of industry-standard programming languages. Java, C, C++, Python, Ruby, and JavaScript implementations are in progress and more languages are welcome.</p>
+          <p>Arrow acts as a new high-performance interface between various
+          systems. It is also focused on supporting a wide variety of
+          industry-standard programming languages. Java, C, C++, Python, Ruby,
+          and JavaScript implementations are in progress and more languages are
+          welcome.</p>
         </div>
         <div class="col-lg-4">
           <h2>Standard</h2>
-          <p>Apache Arrow is backed by key developers of 13 major open source projects, including Calcite, Cassandra, Drill, Hadoop, HBase, Ibis, Impala, Kudu, Pandas, Parquet, Phoenix, Spark, and Storm making it the de-facto standard for columnar in-memory analytics.</p>
+          <p>Apache Arrow is backed by key developers of 13 major open source
+          projects, including Calcite, Cassandra, Drill, Hadoop, HBase, Ibis,
+          Impala, Kudu, Pandas, Parquet, Phoenix, Spark, and Storm making it
+          the de-facto standard for columnar in-memory analytics.</p>
         </div>
      </div> <!-- close "row" div -->
 
@@ -41,7 +52,7 @@ <h2>Advantages of a Common Data Layer</h2>
 <img src="img/copy2.png" alt="common data layer" style="width:100%" />
 <ul>
     <li>Each system has its own internal memory format</li>
-    <li>70-80% CPU wasted on serialization and deserialization</li>
+    <li>70-80% computation wasted on serialization and deserialization</li>
     <li>Similar functionality implemented in multiple projects</li>
 </ul>
 </div>
diff --git a/site/install.md b/site/install.md
index 6cb80c1336f87..74d298667d68f 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,17 +20,17 @@ limitations under the License.
 {% endcomment %}
 -->
 
-## Current Version: 0.6.0
+## Current Version: 0.7.0
 
-### Released: 14 August 2017
+### Released: 17 September 2017
 
 See the [release notes][10] for more about what's new.
 
 ### Source release
 
-* **Source Release**: [apache-arrow-0.6.0.tar.gz][6]
-* **Verification**: [md5][3], [asc][7]
-* [Git tag b173334][2]
+* **Source Release**: [apache-arrow-0.7.0.tar.gz][6]
+* **Verification**: [sha512][3], [asc][7]
+* [Git tag 97f9029][2]
 
 ### Java Packages
 
@@ -52,8 +52,8 @@ Install them with:
 
 
 ```shell
-conda install arrow-cpp=0.6.* -c conda-forge
-conda install pyarrow==0.6.* -c conda-forge
+conda install arrow-cpp=0.7.* -c conda-forge
+conda install pyarrow==0.7.* -c conda-forge
 ```
 
 ### Python Wheels on PyPI (Unofficial)
@@ -61,10 +61,10 @@ conda install pyarrow==0.6.* -c conda-forge
 We have provided binary wheels on PyPI for Linux, macOS, and Windows:
 
 ```shell
-pip install pyarrow==0.6.*
+pip install pyarrow==0.7.*
 ```
 
-We recommend pinning `0.6.*` in `requirements.txt` to install the latest patch
+We recommend pinning `0.7.*` in `requirements.txt` to install the latest patch
 release.
 
 These include the Apache Arrow and Apache Parquet C++ binary libraries bundled
@@ -149,13 +149,13 @@ conda install arrow-cpp -c twosigma
 conda install pyarrow -c twosigma
 ```
 
-[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.6.0
-[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/apache-arrow-0.6.0.tar.gz.md5
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.6.0%22
+[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.7.0
+[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/apache-arrow-0.7.0.tar.gz.sha512
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.7.0%22
 [5]: http://conda-forge.github.io
-[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/apache-arrow-0.6.0.tar.gz
-[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.6.0/apache-arrow-0.6.0.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/apache-arrow-0.7.0.tar.gz
+[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/apache-arrow-0.7.0.tar.gz.asc
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
-[10]: http://arrow.apache.org/release/0.6.0.html
+[10]: http://arrow.apache.org/release/0.7.0.html

From b448f6651c286722f32f9c633d830845c40291e1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 19 Sep 2017 09:12:56 -0400
Subject: [PATCH 1066/1644] ARROW-1550: [Python] Followup: fix flake8 warning

Change-Id: I8842358bbdc66635380891982ab3842018615fd9
---
 python/pyarrow/tests/test_parquet.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 790b47832f226..6266878506059 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -17,7 +17,6 @@
 
 from os.path import join as pjoin
 import datetime
-import gc
 import io
 import os
 import json

From aebc4128c9a14418b4f88edf34821318738dd8fa Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 19 Sep 2017 09:46:01 -0400
Subject: [PATCH 1067/1644] ARROW-1551: [Website] Add 0.7.0 changelog

Change-Id: I9c27893ebfee46364d78963fe20a43f06a1aa700
---
 site/_release/0.7.0.md | 205 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 205 insertions(+)
 create mode 100644 site/_release/0.7.0.md

diff --git a/site/_release/0.7.0.md b/site/_release/0.7.0.md
new file mode 100644
index 0000000000000..2995021675b8e
--- /dev/null
+++ b/site/_release/0.7.0.md
@@ -0,0 +1,205 @@
+---
+layout: default
+title: Apache Arrow 0.7.0 Release
+permalink: /release/0.7.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.7.0 (17 September 2017)
+
+This is a major release. Read more in the [release blog post][3].
+
+## Download
+
+* [**Source Artifacts**][2]
+* [Git tag][1]
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.6.0..apache-arrow-0.7.0
+    58  Wes McKinney
+    14  Kouhei Sutou
+    11  Philipp Moritz
+     7  Phillip Cloud
+     6  siddharth
+     5  Uwe L. Korn
+     2  Bryan Cutler
+     2  HorimotoYasuhiro
+     2  Laurent Goujon
+     2  Li Jin
+     2  Max Risuhin
+     2  fjetter
+     1  Antony Mayi
+     1  Brecht Machiels
+     1  Fritz Obermeyer
+     1  Gonzalo Ortiz
+     1  Jeff Reback
+     1  Kyle Kelley
+     1  Paul Taylor
+     1  Robert Nishihara
+     1  Wataru Shimizu
+     1  hatappi
+     1  mnc
+     1  rvernica
+     1  tetsutaroendo
+```
+
+# Changelog
+
+## New Features and Improvements
+
+* [ARROW-1156](https://issues.apache.org/jira/browse/ARROW-1156) - [Python] pyarrow.Array.from_pandas should take a type parameter
+* [ARROW-1238](https://issues.apache.org/jira/browse/ARROW-1238) - [Java] Add JSON read/write support for decimals for integration tests
+* [ARROW-1307](https://issues.apache.org/jira/browse/ARROW-1307) - [Python] Add pandas serialization section + Feather API to Sphinx docs
+* [ARROW-1317](https://issues.apache.org/jira/browse/ARROW-1317) - [Python] Add function to set Hadoop CLASSPATH
+* [ARROW-1331](https://issues.apache.org/jira/browse/ARROW-1331) - [Java] Refactor tests
+* [ARROW-1339](https://issues.apache.org/jira/browse/ARROW-1339) - [C++] Use boost::filesystem for handling of platform-specific file path encodings
+* [ARROW-1344](https://issues.apache.org/jira/browse/ARROW-1344) - [C++] Calling BufferOutputStream::Write after calling Finish crashes
+* [ARROW-1348](https://issues.apache.org/jira/browse/ARROW-1348) - [C++/Python] Add release verification script for Windows
+* [ARROW-1351](https://issues.apache.org/jira/browse/ARROW-1351) - Automate updating CHANGELOG.md as part of release scripts
+* [ARROW-1352](https://issues.apache.org/jira/browse/ARROW-1352) - [Integration] Improve print formatting for producer, consumer line
+* [ARROW-1355](https://issues.apache.org/jira/browse/ARROW-1355) - Make arrow buildable with java9
+* [ARROW-1356](https://issues.apache.org/jira/browse/ARROW-1356) - [Website] Add new committers
+* [ARROW-1358](https://issues.apache.org/jira/browse/ARROW-1358) - Update source release scripts to account for new SHA checksum policy
+* [ARROW-1359](https://issues.apache.org/jira/browse/ARROW-1359) - [Python] Add Parquet writer option to normalize field names for use in Spark
+* [ARROW-1364](https://issues.apache.org/jira/browse/ARROW-1364) - [C++] IPC reader and writer specialized for GPU device memory
+* [ARROW-1366](https://issues.apache.org/jira/browse/ARROW-1366) - [Python] Add instructions for starting the Plasma store when installing pyarrow from wheels
+* [ARROW-1372](https://issues.apache.org/jira/browse/ARROW-1372) - [Plasma] Support for storing data in huge pages
+* [ARROW-1376](https://issues.apache.org/jira/browse/ARROW-1376) - [C++] RecordBatchStreamReader::Open API is inconsistent with writer
+* [ARROW-1377](https://issues.apache.org/jira/browse/ARROW-1377) - [Python] Add function to assist with benchmarking Parquet scan performance
+* [ARROW-1381](https://issues.apache.org/jira/browse/ARROW-1381) - [Python] Improve performance of SerializedPyObject.to_buffer
+* [ARROW-1383](https://issues.apache.org/jira/browse/ARROW-1383) - [C++] Support std::vector<bool> in builder vector appends
+* [ARROW-1384](https://issues.apache.org/jira/browse/ARROW-1384) - [C++] Add convenience function for serializing a record batch to an IPC message
+* [ARROW-1386](https://issues.apache.org/jira/browse/ARROW-1386) - [C++] Unpin CMake version in MSVC build toolchain
+* [ARROW-1387](https://issues.apache.org/jira/browse/ARROW-1387) - [C++] Set up GPU leaf library build toolchain
+* [ARROW-1390](https://issues.apache.org/jira/browse/ARROW-1390) - [Python] Extend tests for python serialization
+* [ARROW-1392](https://issues.apache.org/jira/browse/ARROW-1392) - [C++] Implement reader and writer IO interfaces for GPU buffers
+* [ARROW-1395](https://issues.apache.org/jira/browse/ARROW-1395) - [C++] Remove APIs deprecated as of 0.5.0 and later versions
+* [ARROW-1396](https://issues.apache.org/jira/browse/ARROW-1396) - [C++] Add PrettyPrint function for Schemas, which also outputs any dictionaries
+* [ARROW-1397](https://issues.apache.org/jira/browse/ARROW-1397) - [Packaging] Use Docker instead of Vagrant
+* [ARROW-1399](https://issues.apache.org/jira/browse/ARROW-1399) - [C++] Add CUDA build version in a public header to help prevent ABI conflicts
+* [ARROW-1400](https://issues.apache.org/jira/browse/ARROW-1400) - [Python] Ability to create partitions when writing to Parquet
+* [ARROW-1401](https://issues.apache.org/jira/browse/ARROW-1401) - [C++] Add extra debugging context to failures in RETURN_NOT_OK in debug builds
+* [ARROW-1402](https://issues.apache.org/jira/browse/ARROW-1402) - [C++] Possibly deprecate public APIs that use MutableBuffer
+* [ARROW-1404](https://issues.apache.org/jira/browse/ARROW-1404) - [Packaging] Build .deb and .rpm on Travis CI
+* [ARROW-1405](https://issues.apache.org/jira/browse/ARROW-1405) - [Python] Add logging option for verbose memory allocations
+* [ARROW-1406](https://issues.apache.org/jira/browse/ARROW-1406) - [Python] Harden user API for generating serialized schema and record batch messages as memoryview-compatible objects
+* [ARROW-1408](https://issues.apache.org/jira/browse/ARROW-1408) - [C++] Refactor and make IPC read / write APIs more consistent, add appropriate deprecations
+* [ARROW-1410](https://issues.apache.org/jira/browse/ARROW-1410) - Plasma object store occasionally pauses for a long time
+* [ARROW-1412](https://issues.apache.org/jira/browse/ARROW-1412) - [Plasma] Add higher level API for putting and getting Python objects
+* [ARROW-1413](https://issues.apache.org/jira/browse/ARROW-1413) - [C++] Add include-what-you-use configuration
+* [ARROW-1415](https://issues.apache.org/jira/browse/ARROW-1415) - [GLib] Support date32 and date64
+* [ARROW-1416](https://issues.apache.org/jira/browse/ARROW-1416) - [Format] Clarify example array in memory layout documentation
+* [ARROW-1417](https://issues.apache.org/jira/browse/ARROW-1417) - [Python] Allow more generic filesystem objects to be passed to ParquetDataset
+* [ARROW-1418](https://issues.apache.org/jira/browse/ARROW-1418) - [Python] Introduce SerializationContext to register custom serialization callbacks
+* [ARROW-1419](https://issues.apache.org/jira/browse/ARROW-1419) - [GLib] Suppress sign-conversion warning on Clang
+* [ARROW-1427](https://issues.apache.org/jira/browse/ARROW-1427) - [GLib] Add a link to readme of Arrow GLib
+* [ARROW-1428](https://issues.apache.org/jira/browse/ARROW-1428) - [C++] Append steps to clone source code to README.mb
+* [ARROW-1432](https://issues.apache.org/jira/browse/ARROW-1432) - [C++] Build bundled jemalloc functions with private prefix
+* [ARROW-1433](https://issues.apache.org/jira/browse/ARROW-1433) - [C++] Simplify implementation of Array::Slice
+* [ARROW-1438](https://issues.apache.org/jira/browse/ARROW-1438) - [Plasma] Pull SerializationContext through PlasmaClient put and get
+* [ARROW-1441](https://issues.apache.org/jira/browse/ARROW-1441) - [Site] Add Ruby to Flexible section
+* [ARROW-1442](https://issues.apache.org/jira/browse/ARROW-1442) - [Website] Add pointer to nightly conda packages on /install
+* [ARROW-1447](https://issues.apache.org/jira/browse/ARROW-1447) - [C++] Round of include-what-you-use include cleanups
+* [ARROW-1448](https://issues.apache.org/jira/browse/ARROW-1448) - [Packaging] Support uploading built .deb and .rpm to Bintray
+* [ARROW-1449](https://issues.apache.org/jira/browse/ARROW-1449) - Implement Decimal using only Int128
+* [ARROW-1451](https://issues.apache.org/jira/browse/ARROW-1451) - [C++] Create arrow/io/api.h
+* [ARROW-1460](https://issues.apache.org/jira/browse/ARROW-1460) - [C++] Upgrade clang-format used to LLVM 4.0
+* [ARROW-1462](https://issues.apache.org/jira/browse/ARROW-1462) - [GLib] Support time array
+* [ARROW-1466](https://issues.apache.org/jira/browse/ARROW-1466) - [C++] Support DecimalArray in arrow::PrettyPrint
+* [ARROW-1468](https://issues.apache.org/jira/browse/ARROW-1468) - [C++] Append to PrimitiveBuilder from std::vector<CTYPE>
+* [ARROW-1479](https://issues.apache.org/jira/browse/ARROW-1479) - [JS] Expand JavaScript implementation
+* [ARROW-1480](https://issues.apache.org/jira/browse/ARROW-1480) - [Python] Improve performance of serializing sets
+* [ARROW-1481](https://issues.apache.org/jira/browse/ARROW-1481) - [C++] Expose type casts as generic callable object that can write into pre-allocated memory
+* [ARROW-1494](https://issues.apache.org/jira/browse/ARROW-1494) - [C++] Document that shared_ptr returned by RecordBatch::column needs to be retained
+* [ARROW-1499](https://issues.apache.org/jira/browse/ARROW-1499) - [Python] Consider adding option to parquet.write_table that sets options for maximum Spark compatibility
+* [ARROW-1504](https://issues.apache.org/jira/browse/ARROW-1504) - [GLib] Support timestamp
+* [ARROW-1505](https://issues.apache.org/jira/browse/ARROW-1505) - [GLib] Simplify arguments check
+* [ARROW-1506](https://issues.apache.org/jira/browse/ARROW-1506) - [C++] Support pkg-config for compute modules
+* [ARROW-1508](https://issues.apache.org/jira/browse/ARROW-1508) - C++: Add support for FixedSizeBinaryType in DictionaryBuilder
+* [ARROW-1510](https://issues.apache.org/jira/browse/ARROW-1510) - [C++] Support cast
+* [ARROW-1511](https://issues.apache.org/jira/browse/ARROW-1511) - [C++] Deprecate arrow::MakePrimitiveArray
+* [ARROW-1513](https://issues.apache.org/jira/browse/ARROW-1513) - C++: Add cast from Dictionary to plain arrays
+* [ARROW-1515](https://issues.apache.org/jira/browse/ARROW-1515) - [GLib] Detect version directly
+* [ARROW-1516](https://issues.apache.org/jira/browse/ARROW-1516) - [GLib] Update document
+* [ARROW-1517](https://issues.apache.org/jira/browse/ARROW-1517) - Remove unnecessary temporary in DecimalUtil::ToString function
+* [ARROW-1519](https://issues.apache.org/jira/browse/ARROW-1519) - [C++] Move DecimalUtil functions to methods on the Int128 class
+* [ARROW-1528](https://issues.apache.org/jira/browse/ARROW-1528) - [GLib] Resolve include dependency
+* [ARROW-1530](https://issues.apache.org/jira/browse/ARROW-1530) - [C++] Install arrow/util/parallel.h
+* [ARROW-1551](https://issues.apache.org/jira/browse/ARROW-1551) - [Website] Updates for 0.7.0 release
+* [ARROW-229](https://issues.apache.org/jira/browse/ARROW-229) - [C++] Implement safe casts for primitive types
+* [ARROW-592](https://issues.apache.org/jira/browse/ARROW-592) - [C++] Provide .deb and .rpm packages
+* [ARROW-594](https://issues.apache.org/jira/browse/ARROW-594) - [Python] Provide interface to write pyarrow.Table to a stream
+* [ARROW-695](https://issues.apache.org/jira/browse/ARROW-695) - Integration tests for Decimal types
+* [ARROW-696](https://issues.apache.org/jira/browse/ARROW-696) - [C++] Add JSON read/write support for decimals for integration tests
+* [ARROW-759](https://issues.apache.org/jira/browse/ARROW-759) - [Python] Implement a transient list serialization function that can handle a mix of scalars, lists, ndarrays, dicts
+* [ARROW-786](https://issues.apache.org/jira/browse/ARROW-786) - [Format] In-memory format for 128-bit Decimals, handling of sign bit
+* [ARROW-837](https://issues.apache.org/jira/browse/ARROW-837) - [Python] Expose buffer allocation, FixedSizeBufferWriter
+* [ARROW-941](https://issues.apache.org/jira/browse/ARROW-941) - [Docs] Improve "cold start" integration testing instructions
+* [ARROW-989](https://issues.apache.org/jira/browse/ARROW-989) - [Python] Write pyarrow.Table to FileWriter or StreamWriter
+
+## Bug Fixes
+
+* [ARROW-12](https://issues.apache.org/jira/browse/ARROW-12) - Get Github activity mirrored to JIRA
+* [ARROW-1302](https://issues.apache.org/jira/browse/ARROW-1302) - C++: ${MAKE} variable not set sometimes on older MacOS installations
+* [ARROW-1354](https://issues.apache.org/jira/browse/ARROW-1354) - [Python] Segfault in Table.from_pandas with Mixed-Type Categories
+* [ARROW-1357](https://issues.apache.org/jira/browse/ARROW-1357) - [Python] Data corruption in reading multi-file parquet dataset
+* [ARROW-1363](https://issues.apache.org/jira/browse/ARROW-1363) - [C++] IPC writer sends buffer layout for dictionary rather than indices
+* [ARROW-1365](https://issues.apache.org/jira/browse/ARROW-1365) - [Python] Remove usage of removed jemalloc_memory_pool in Python API docs
+* [ARROW-1373](https://issues.apache.org/jira/browse/ARROW-1373) - [Java] Implement get<type>Buffer() methods at the ValueVector interface
+* [ARROW-1375](https://issues.apache.org/jira/browse/ARROW-1375) - [C++] Visual Studio 2017 Appveyor builds failing
+* [ARROW-1379](https://issues.apache.org/jira/browse/ARROW-1379) - [Java] maven dependency issues - both unused and undeclared
+* [ARROW-1407](https://issues.apache.org/jira/browse/ARROW-1407) - Dictionaries can only hold a maximum of 4096 indices
+* [ARROW-1411](https://issues.apache.org/jira/browse/ARROW-1411) - [Python] Booleans in Float Columns cause Segfault
+* [ARROW-1414](https://issues.apache.org/jira/browse/ARROW-1414) - [GLib] Cast after status check
+* [ARROW-1421](https://issues.apache.org/jira/browse/ARROW-1421) - [Python] pyarrow.serialize cannot serialize a Python dict input
+* [ARROW-1426](https://issues.apache.org/jira/browse/ARROW-1426) - [Website] The title element of the top page is empty
+* [ARROW-1429](https://issues.apache.org/jira/browse/ARROW-1429) - [Python] Error loading parquet file with _metadata from HDFS
+* [ARROW-1430](https://issues.apache.org/jira/browse/ARROW-1430) - [Python] flake8 warnings are not failing CI builds
+* [ARROW-1434](https://issues.apache.org/jira/browse/ARROW-1434) - [C++/Python] pyarrow.Array.from_pandas does not support datetime64[D] arrays
+* [ARROW-1435](https://issues.apache.org/jira/browse/ARROW-1435) - [Python] PyArrow not propagating timezone information from Parquet to Python
+* [ARROW-1439](https://issues.apache.org/jira/browse/ARROW-1439) - [Packaging] Automate updating RPM in RPM build
+* [ARROW-1443](https://issues.apache.org/jira/browse/ARROW-1443) - [Java] Bug on ArrowBuf.setBytes with unsliced ByteBuffers
+* [ARROW-1444](https://issues.apache.org/jira/browse/ARROW-1444) - [JAVA] BitVector.splitAndTransfer copies last byte incorrectly
+* [ARROW-1446](https://issues.apache.org/jira/browse/ARROW-1446) - Python: Writing more than 2^31 rows from pandas dataframe causes row count overflow error
+* [ARROW-1450](https://issues.apache.org/jira/browse/ARROW-1450) - [Python] Raise proper error if custom serialization handler fails
+* [ARROW-1452](https://issues.apache.org/jira/browse/ARROW-1452) - [C++] Make UNUSED macro name more unique so it does not conflict with thirdparty projects
+* [ARROW-1453](https://issues.apache.org/jira/browse/ARROW-1453) - [Python] Implement WriteTensor for non-contiguous tensors
+* [ARROW-1457](https://issues.apache.org/jira/browse/ARROW-1457) - [C++] Optimize strided WriteTensor
+* [ARROW-1458](https://issues.apache.org/jira/browse/ARROW-1458) - [Python] Document that HadoopFileSystem.mkdir with create_parents=False has no effect
+* [ARROW-1459](https://issues.apache.org/jira/browse/ARROW-1459) - [Python] PyArrow fails to load partitioned parquet files with non-primitive types
+* [ARROW-1461](https://issues.apache.org/jira/browse/ARROW-1461) - [C++] Disable builds using LLVM apt packages temporarily
+* [ARROW-1467](https://issues.apache.org/jira/browse/ARROW-1467) - [JAVA]: Fix reset() and allocateNew() in Nullable Value Vectors template
+* [ARROW-1490](https://issues.apache.org/jira/browse/ARROW-1490) - [Java] Allow Travis CI failures for JDK9 for now
+* [ARROW-1493](https://issues.apache.org/jira/browse/ARROW-1493) - [C++] Flush the output stream at the end of each PrettyPrint function
+* [ARROW-1495](https://issues.apache.org/jira/browse/ARROW-1495) - [C++] Store shared_ptr to boxed arrays in RecordBatch
+* [ARROW-1507](https://issues.apache.org/jira/browse/ARROW-1507) - [C++] arrow/compute/api.h can't be used without arrow/array.h
+* [ARROW-1512](https://issues.apache.org/jira/browse/ARROW-1512) - [Docs] NumericArray has no member named 'raw_data'
+* [ARROW-1514](https://issues.apache.org/jira/browse/ARROW-1514) - [C++] Fix a typo in document
+* [ARROW-1527](https://issues.apache.org/jira/browse/ARROW-1527) - Fix Travis JDK9 build
+* [ARROW-1531](https://issues.apache.org/jira/browse/ARROW-1531) - [C++] Return ToBytes by value from Decimal128
+* [ARROW-1532](https://issues.apache.org/jira/browse/ARROW-1532) - [Python] Referencing an Empty Schema causes a SegFault
+* [ARROW-407](https://issues.apache.org/jira/browse/ARROW-407) - BitVector.copyFromSafe() should re-allocate if necessary instead of returning false
+* [ARROW-801](https://issues.apache.org/jira/browse/ARROW-801) - [JAVA] Provide direct access to underlying buffer memory addresses in consistent way without generating garbage or large amount indirections
+
+[1]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.7.0
+[2]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/
+[3]: http://arrow.apache.org/blog/2017/09/19/0.7.0-release/
\ No newline at end of file

From 2706b7fc5224ff56441d46eec516b2072090a984 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Tue, 19 Sep 2017 12:10:07 -0400
Subject: [PATCH 1068/1644] ARROW-1533: [JAVA] realloc should consider the
 existing buffer capacity for computing target memory requirement

cc @jacques-n , This is same as https://github.com/apache/arrow/pull/1097

The latter one was closed as I had to rename the branch correctly and use the correct JIRA number.

Author: siddharth <siddharth@dremio.com>

Closes #1112 from siddharthteotia/ARROW-1533 and squashes the following commits:

4c97be44 [siddharth] ARROW-1533: realloc should consider the existing buffer capacity for computing target memory requirement
---
 .../apache/arrow/memory/BaseAllocator.java    |  17 +-
 .../codegen/templates/FixedValueVectors.java  |  13 +-
 .../templates/VariableLengthVectors.java      |  11 +-
 .../org/apache/arrow/vector/BitVector.java    |  12 +-
 .../apache/arrow/vector/TestBitVector.java    | 193 ++++++++++++
 .../apache/arrow/vector/TestValueVector.java  | 274 +++++++++++++++++-
 6 files changed, 511 insertions(+), 9 deletions(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 2749b6fe030f4..5411baf7bdfe3 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -133,7 +133,7 @@ private static String createErrorMsg(final BufferAllocator allocator, final int
    * @param val An integer value.
    * @return The closest power of two of that value.
    */
-  static int nextPowerOfTwo(int val) {
+  public static int nextPowerOfTwo(int val) {
     int highestBit = Integer.highestOneBit(val);
     if (highestBit == val) {
       return val;
@@ -142,6 +142,21 @@ static int nextPowerOfTwo(int val) {
     }
   }
 
+  /**
+   * Rounds up the provided value to the nearest power of two.
+   *
+   * @param val A long value.
+   * @return The closest power of two of that value.
+   */
+  public static long nextPowerOfTwo(long val) {
+    long highestBit = Long.highestOneBit(val);
+    if (highestBit == val) {
+      return val;
+    } else {
+      return highestBit << 1;
+    }
+  }
+
   public static StringBuilder indent(StringBuilder sb, int indent) {
     final char[] indentation = new char[indent * 2];
     Arrays.fill(indentation, ' ');
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index ffd8cad02e2c3..e07416ba984d4 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -208,14 +208,21 @@ private void allocateBytes(final long size) {
    * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
    */
   public void reAlloc() {
-    final long newAllocationSize = allocationSizeInBytes * 2L;
-    if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
+    long baseSize  = allocationSizeInBytes;
+    final int currentBufferCapacity = data.capacity();
+    if (baseSize < (long)currentBufferCapacity) {
+        baseSize = (long)currentBufferCapacity;
+    }
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
     }
 
     logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", name, allocationSizeInBytes, newAllocationSize);
     final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
-    newBuf.setBytes(0, data, 0, data.capacity());
+    newBuf.setBytes(0, data, 0, currentBufferCapacity);
     final int halfNewCapacity = newBuf.capacity() / 2;
     newBuf.setZero(halfNewCapacity, halfNewCapacity);
     newBuf.writerIndex(data.writerIndex());
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index c276f11d79645..3934e74f11b2d 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -370,13 +370,20 @@ public void reset() {
   }
 
   public void reAlloc() {
-    final long newAllocationSize = allocationSizeInBytes*2L;
+    long baseSize = allocationSizeInBytes;
+    final int currentBufferCapacity = data.capacity();
+    if (baseSize < (long)currentBufferCapacity) {
+      baseSize = (long)currentBufferCapacity;
+    }
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
     if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
       throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
     }
 
     final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
-    newBuf.setBytes(0, data, 0, data.capacity());
+    newBuf.setBytes(0, data, 0, currentBufferCapacity);
     data.release();
     data = newBuf;
     allocationSizeInBytes = (int)newAllocationSize;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 8a60273e179ef..e1911169fb6c5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -19,6 +19,7 @@
 package org.apache.arrow.vector;
 
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.BitHolder;
@@ -208,7 +209,14 @@ private void allocateBytes(final long size) {
    * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
    */
   public void reAlloc() {
-    final long newAllocationSize = allocationSizeInBytes * 2L;
+    long baseSize  = allocationSizeInBytes;
+    final int currentBufferCapacity = data.capacity();
+    if (baseSize < (long)currentBufferCapacity) {
+      baseSize = (long)currentBufferCapacity;
+    }
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
     if (newAllocationSize > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
     }
@@ -216,7 +224,7 @@ public void reAlloc() {
     final int curSize = (int) newAllocationSize;
     final ArrowBuf newBuf = allocator.buffer(curSize);
     newBuf.setZero(0, newBuf.capacity());
-    newBuf.setBytes(0, data, 0, data.capacity());
+    newBuf.setBytes(0, data, 0, currentBufferCapacity);
     data.release();
     data = newBuf;
     allocationSizeInBytes = curSize;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
index 82e61be45c67b..17fcf05fcd9bd 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -21,6 +21,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -234,6 +235,198 @@ public void testSplitAndTransfer2() throws Exception {
     }
   }
 
+  @Test
+  public void testReallocAfterVectorTransfer1() {
+    try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
+      vector.allocateNew(4096);
+      int valueCapacity = vector.getValueCapacity();
+      assertEquals(4096, valueCapacity);
+
+      final BitVector.Mutator mutator = vector.getMutator();
+      final BitVector.Accessor accessor = vector.getAccessor();
+
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          mutator.setToOne(i);
+        }
+      }
+
+      for (int i = 0; i < valueCapacity; i++) {
+        int val = accessor.get(i);
+        if ((i & 1) == 1) {
+          assertEquals("unexpected cleared bit at index: " + i, 1, val);
+        }
+        else {
+          assertEquals("unexpected set bit at index: " + i, 0, val);
+        }
+      }
+
+      /* trigger first realloc */
+      mutator.setSafeToOne(valueCapacity);
+      assertEquals(valueCapacity * 2, vector.getValueCapacity());
+
+      for (int i = valueCapacity; i < valueCapacity*2; i++) {
+        if ((i & 1) == 1) {
+          mutator.setToOne(i);
+        }
+      }
+
+      for (int i = 0; i < valueCapacity*2; i++) {
+        int val = accessor.get(i);
+        if (((i & 1) == 1) || (i == valueCapacity)) {
+          assertEquals("unexpected cleared bit at index: " + i, 1, val);
+        }
+        else {
+          assertEquals("unexpected set bit at index: " + i, 0, val);
+        }
+      }
+
+      /* trigger second realloc */
+      mutator.setSafeToOne(valueCapacity*2);
+      assertEquals(valueCapacity * 4, vector.getValueCapacity());
+
+      for (int i = valueCapacity*2; i < valueCapacity*4; i++) {
+        if ((i & 1) == 1) {
+          mutator.setToOne(i);
+        }
+      }
+
+      for (int i = 0; i < valueCapacity*4; i++) {
+        int val = accessor.get(i);
+        if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2)) {
+          assertEquals("unexpected cleared bit at index: " + i, 1, val);
+        }
+        else {
+          assertEquals("unexpected set bit at index: " + i, 0, val);
+        }
+      }
+
+      /* now transfer the vector */
+      TransferPair transferPair = vector.getTransferPair(allocator);
+      transferPair.transfer();
+      final BitVector toVector = (BitVector)transferPair.getTo();
+      final BitVector.Accessor toAccessor = toVector.getAccessor();
+      final BitVector.Mutator toMutator = toVector.getMutator();
+
+      assertEquals(valueCapacity * 4, toVector.getValueCapacity());
+
+      /* realloc the toVector */
+      toMutator.setSafeToOne(valueCapacity * 4);
+
+      for (int i = 0; i < toVector.getValueCapacity(); i++) {
+        int val = toAccessor.get(i);
+        if (i <= valueCapacity * 4) {
+          if (((i & 1) == 1) || (i == valueCapacity) ||
+                  (i == valueCapacity*2) || (i == valueCapacity*4)) {
+            assertEquals("unexpected cleared bit at index: " + i, 1, val);
+          }
+          else {
+            assertEquals("unexpected set bit at index: " + i, 0, val);
+          }
+        }
+        else {
+          assertEquals("unexpected set bit at index: " + i, 0, val);
+        }
+      }
+
+      toVector.close();
+    }
+  }
+
+  @Test
+  public void testReallocAfterVectorTransfer2() {
+    try (final NullableBitVector vector = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator)) {
+      vector.allocateNew(4096);
+      int valueCapacity = vector.getValueCapacity();
+      assertEquals(4096, valueCapacity);
+
+      final NullableBitVector.Mutator mutator = vector.getMutator();
+      final NullableBitVector.Accessor accessor = vector.getAccessor();
+
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          mutator.set(i, 1);
+        }
+      }
+
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i));
+        }
+        else {
+          assertTrue("unexpected set bit at index: " + i, accessor.isNull(i));
+        }
+      }
+
+      /* trigger first realloc */
+      mutator.setSafe(valueCapacity, 1, 1);
+      assertEquals(valueCapacity * 2, vector.getValueCapacity());
+
+      for (int i = valueCapacity; i < valueCapacity*2; i++) {
+        if ((i & 1) == 1) {
+          mutator.set(i, 1);
+        }
+      }
+
+      for (int i = 0; i < valueCapacity*2; i++) {
+        if (((i & 1) == 1) || (i == valueCapacity)) {
+          assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i));
+        }
+        else {
+          assertTrue("unexpected set bit at index: " + i, accessor.isNull(i));
+        }
+      }
+
+      /* trigger second realloc */
+      mutator.setSafe(valueCapacity*2, 1, 1);
+      assertEquals(valueCapacity * 4, vector.getValueCapacity());
+
+      for (int i = valueCapacity*2; i < valueCapacity*4; i++) {
+        if ((i & 1) == 1) {
+          mutator.set(i, 1);
+        }
+      }
+
+      for (int i = 0; i < valueCapacity*4; i++) {
+        if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2)) {
+          assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i));
+        }
+        else {
+          assertTrue("unexpected set bit at index: " + i, accessor.isNull(i));
+        }
+      }
+
+      /* now transfer the vector */
+      TransferPair transferPair = vector.getTransferPair(allocator);
+      transferPair.transfer();
+      final NullableBitVector toVector = (NullableBitVector)transferPair.getTo();
+      final NullableBitVector.Accessor toAccessor = toVector.getAccessor();
+      final NullableBitVector.Mutator toMutator = toVector.getMutator();
+
+      assertEquals(valueCapacity * 4, toVector.getValueCapacity());
+
+      /* realloc the toVector */
+      toMutator.setSafe(valueCapacity * 4, 1, 1);
+
+      for (int i = 0; i < toVector.getValueCapacity(); i++) {
+        if (i <= valueCapacity * 4) {
+          if (((i & 1) == 1) || (i == valueCapacity) ||
+                  (i == valueCapacity*2) || (i == valueCapacity*4)) {
+            assertFalse("unexpected cleared bit at index: " + i, toAccessor.isNull(i));
+          }
+          else {
+            assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i));
+          }
+        }
+        else {
+          assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i));
+        }
+      }
+
+      toVector.close();
+    }
+  }
+
   @Test
   public void testBitVector() {
     // Create a new value vector for 1024 integers
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 57119bfdae24e..a239861d9b32f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -44,6 +44,7 @@
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.TransferPair;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -889,9 +890,280 @@ public void testNullableVarType2() {
    *  TODO:
    *
    *  The realloc() related tests below should be moved up and we need to
-   *  realloc related tests (edge cases) for more vector types.
+   *  add realloc related tests (edge cases) for more vector types.
    */
 
+  @Test /* Float8Vector */
+  public void testReallocAfterVectorTransfer1() {
+    try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
+      final Float8Vector.Mutator mutator = vector.getMutator();
+      final Float8Vector.Accessor accessor = vector.getAccessor();
+      final int initialDefaultCapacity = 4096;
+      boolean error = false;
+
+      /* use the default capacity; 4096*8 => 32KB */
+      vector.allocateNew();
+
+      assertEquals(initialDefaultCapacity, vector.getValueCapacity());
+
+      double baseValue = 100.375;
+
+      for (int i = 0; i < initialDefaultCapacity; i++) {
+        mutator.setSafe(i, baseValue + (double)i);
+      }
+
+      /* the above setSafe calls should not have triggered a realloc as
+       * we are within the capacity. check the vector contents
+       */
+      assertEquals(initialDefaultCapacity, vector.getValueCapacity());
+
+      for (int i = 0; i < initialDefaultCapacity; i++) {
+        double value = accessor.get(i);
+        assertEquals(baseValue + (double)i, value, 0);
+      }
+
+      /* this should trigger a realloc */
+      mutator.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity);
+      assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity());
+
+      for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) {
+        mutator.setSafe(i, baseValue + (double)i);
+      }
+
+      for (int i = 0; i < (initialDefaultCapacity * 2); i++) {
+        double value = accessor.get(i);
+        assertEquals(baseValue + (double)i, value, 0);
+      }
+
+      /* this should trigger a realloc */
+      mutator.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2));
+      assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity());
+
+      for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) {
+        mutator.setSafe(i, baseValue + (double)i);
+      }
+
+      for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
+        double value = accessor.get(i);
+        assertEquals(baseValue + (double)i, value, 0);
+      }
+
+      /* at this point we are working with a 128KB buffer data for this
+       * vector. now let's transfer this vector
+       */
+
+      TransferPair transferPair = vector.getTransferPair(allocator);
+      transferPair.transfer();
+
+      Float8Vector toVector = (Float8Vector)transferPair.getTo();
+
+      /* now let's realloc the toVector */
+      toVector.reAlloc();
+      assertEquals(initialDefaultCapacity * 8, toVector.getValueCapacity());
+
+      final Float8Vector.Accessor toAccessor = toVector.getAccessor();
+
+      for (int i = 0; i < (initialDefaultCapacity * 8); i++) {
+        double value = toAccessor.get(i);
+        if (i < (initialDefaultCapacity * 4)) {
+          assertEquals(baseValue + (double)i, value, 0);
+        }
+        else {
+          assertEquals(0, value, 0);
+        }
+      }
+
+      toVector.close();
+    }
+  }
+
+  @Test /* NullableFloat8Vector */
+  public void testReallocAfterVectorTransfer2() {
+    try (final NullableFloat8Vector vector = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) {
+      final NullableFloat8Vector.Mutator mutator = vector.getMutator();
+      final NullableFloat8Vector.Accessor accessor = vector.getAccessor();
+      final int initialDefaultCapacity = 4096;
+      boolean error = false;
+
+      vector.allocateNew(initialDefaultCapacity);
+
+      assertEquals(initialDefaultCapacity, vector.getValueCapacity());
+
+      double baseValue = 100.375;
+
+      for (int i = 0; i < initialDefaultCapacity; i++) {
+        mutator.setSafe(i, baseValue + (double)i);
+      }
+
+      /* the above setSafe calls should not have triggered a realloc as
+       * we are within the capacity. check the vector contents
+       */
+      assertEquals(initialDefaultCapacity, vector.getValueCapacity());
+
+      for (int i = 0; i < initialDefaultCapacity; i++) {
+        double value = accessor.get(i);
+        assertEquals(baseValue + (double)i, value, 0);
+      }
+
+      /* this should trigger a realloc */
+      mutator.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity);
+      assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity());
+
+      for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) {
+        mutator.setSafe(i, baseValue + (double)i);
+      }
+
+      for (int i = 0; i < (initialDefaultCapacity * 2); i++) {
+        double value = accessor.get(i);
+        assertEquals(baseValue + (double)i, value, 0);
+      }
+
+      /* this should trigger a realloc */
+      mutator.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2));
+      assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity());
+
+      for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) {
+        mutator.setSafe(i, baseValue + (double)i);
+      }
+
+      for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
+        double value = accessor.get(i);
+        assertEquals(baseValue + (double)i, value, 0);
+      }
+
+      /* at this point we are working with a 128KB buffer data for this
+       * vector. now let's transfer this vector
+       */
+
+      TransferPair transferPair = vector.getTransferPair(allocator);
+      transferPair.transfer();
+
+      NullableFloat8Vector toVector = (NullableFloat8Vector)transferPair.getTo();
+      final NullableFloat8Vector.Accessor toAccessor = toVector.getAccessor();
+
+      /* check toVector contents before realloc */
+      for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
+        assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i));
+        double value = toAccessor.get(i);
+        assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0);
+      }
+
+      /* now let's realloc the toVector and check contents again */
+      toVector.reAlloc();
+      assertEquals(initialDefaultCapacity * 8, toVector.getValueCapacity());
+
+      for (int i = 0; i < (initialDefaultCapacity * 8); i++) {
+        if (i < (initialDefaultCapacity * 4)) {
+          assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i));
+          double value = toAccessor.get(i);
+          assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0);
+        }
+        else {
+          assertTrue("unexpected non-null value at index: " + i, toAccessor.isNull(i));
+        }
+      }
+
+      toVector.close();
+    }
+  }
+
+  @Test /* NullableVarCharVector */
+  public void testReallocAfterVectorTransfer3() {
+    try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
+      final NullableVarCharVector.Mutator mutator = vector.getMutator();
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+
+      /* 4096 values with 10 byte per record */
+      vector.allocateNew(4096 * 10, 4096);
+      int valueCapacity = vector.getValueCapacity();
+
+      /* populate the vector */
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          mutator.set(i, STR1);
+        }
+        else {
+          mutator.set(i, STR2);
+        }
+      }
+
+      /* Check the vector output */
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          assertArrayEquals(STR1, accessor.get(i));
+        }
+        else {
+          assertArrayEquals(STR2, accessor.get(i));
+        }
+      }
+
+      /* trigger first realloc */
+      mutator.setSafe(valueCapacity, STR2, 0, STR2.length);
+
+      /* populate the remaining vector */
+      for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
+        if ((i & 1) == 1) {
+          mutator.set(i, STR1);
+        }
+        else {
+          mutator.set(i, STR2);
+        }
+      }
+
+      /* Check the vector output */
+      valueCapacity = vector.getValueCapacity();
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          assertArrayEquals(STR1, accessor.get(i));
+        }
+        else {
+          assertArrayEquals(STR2, accessor.get(i));
+        }
+      }
+
+      /* trigger second realloc */
+      mutator.setSafe(valueCapacity + 10, STR2, 0, STR2.length);
+
+      /* populate the remaining vector */
+      for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
+        if ((i & 1) == 1) {
+          mutator.set(i, STR1);
+        }
+        else {
+          mutator.set(i, STR2);
+        }
+      }
+
+      /* Check the vector output */
+      valueCapacity = vector.getValueCapacity();
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          assertArrayEquals(STR1, accessor.get(i));
+        }
+        else {
+          assertArrayEquals(STR2, accessor.get(i));
+        }
+      }
+
+      /* we are potentially working with 4x the size of vector buffer
+       * that we initially started with. Now let's transfer the vector.
+       */
+
+      TransferPair transferPair = vector.getTransferPair(allocator);
+      transferPair.transfer();
+      NullableVarCharVector toVector = (NullableVarCharVector)transferPair.getTo();
+      NullableVarCharVector.Mutator toMutator = toVector.getMutator();
+      NullableVarCharVector.Accessor toAccessor = toVector.getAccessor();
+
+      valueCapacity = toVector.getValueCapacity();
+
+      /* trigger a realloc of this toVector */
+      toMutator.setSafe(valueCapacity + 10, STR2, 0, STR2.length);
+
+      toVector.close();
+    }
+  }
+
   @Test
   public void testReAllocNullableFixedWidthVector() {
     // Create a new value vector for 1024 integers

From d4685f420a5bb75cf440e6e687f3ae279e66b73f Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Tue, 19 Sep 2017 12:13:54 -0400
Subject: [PATCH 1069/1644] ARROW-1547: [JAVA] Fix 8x memory over-allocation in
 BitVector
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Problem:

Typically there are 3 ways of specifying the amount of memory needed for vectors.
CASE (1) allocateNew() – here the application doesn't really specify the size of memory or value count. Each vector type has a default value count (4096) and therefore a default size (in bytes) is used in such cases.

For example, for a 4 byte fixed-width vector, we will allocate 32KB of memory for a call to allocateNew().

CASE (2) setInitialCapacity(count) followed by allocateNew() - In this case also the application doesn't specify the value count or size in allocateNew(). However, the call to setInitialCapacity() dictates the amount of memory the subsequent call to allocateNew() will allocate.

For example, we can do setInitialCapacity(1024) and the call to allocateNew() will allocate 4KB of memory for the 4 byte fixed-width vector.

CASE (3) allocateNew(count) - The application is specific about requirements.
For nullable vectors, the above calls also allocate the memory for validity vector.

The problem is that Bit Vector uses a default memory size in bytes of 4096. In other words, we allocate a vector for 4096*8 value count.

In the default case (as explained above), the vector types have a value count of 4096 so we need only 4096 bits (512 bytes) in the bit vector and not really 4096 as the size in bytes.

This happens in CASE 1 where the application depends on the default memory allocation . In such cases, the size of buffer for bit vector is 8x than actually needed

Author: siddharth <siddharth@dremio.com>

Closes #1109 from siddharthteotia/ARROW-1547 and squashes the following commits:

c92164a0 [siddharth] addressed review comments
f3d12348 [siddharth] ARROW-1547: Fix 8x memory over-allocation in BitVector
---
 .../src/main/java/org/apache/arrow/vector/BitVector.java | 4 ++--
 .../apache/arrow/vector/TestBufferOwnershipTransfer.java | 9 +++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index e1911169fb6c5..591d13c4bd5b4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -44,7 +44,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
   private final Mutator mutator = new Mutator();
 
   int valueCount;
-  private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+  private int allocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION);
   private int allocationMonitor = 0;
 
   public BitVector(String name, BufferAllocator allocator) {
@@ -176,7 +176,7 @@ public boolean allocateNewSafe() {
   @Override
   public void reset() {
     valueCount = 0;
-    allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+    allocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION);
     allocationMonitor = 0;
     zeroVector();
     super.reset();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
index 54fc306717088..1a801a63ec688 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
@@ -47,7 +47,9 @@ public void testTransferFixedWidth() {
     v1.makeTransferPair(v2).transfer();
 
     assertEquals(0, childAllocator1.getAllocatedMemory());
-    assertEquals(5 * 4096, childAllocator2.getAllocatedMemory());
+    int expectedBitVector = 512;
+    int expectedValueVector = 4096*4;
+    assertEquals(expectedBitVector + expectedValueVector, childAllocator2.getAllocatedMemory());
   }
 
   @Test
@@ -66,7 +68,10 @@ public void testTransferVariableidth() {
     v1.makeTransferPair(v2).transfer();
 
     assertEquals(0, childAllocator1.getAllocatedMemory());
-    int expected = 8 * 4096 + 4 * 4096 + 4096;
+    int expectedValueVector = 4096*8;
+    int expectedOffsetVector = 4096*4;
+    int expectedBitVector = 512;
+    int expected = expectedBitVector + expectedOffsetVector + expectedValueVector;
     assertEquals(expected, childAllocator2.getAllocatedMemory());
   }
 

From c4f5a1237380d70c21bbf415e39ae0b4f4ca97b1 Mon Sep 17 00:00:00 2001
From: Deepak Majeti <deepak.majeti@hpe.com>
Date: Tue, 19 Sep 2017 23:42:33 -0400
Subject: [PATCH 1070/1644] ARROW-1536:[C++] Do not transitively depend on
 libboost_system

Author: Deepak Majeti <deepak.majeti@hpe.com>

Closes #1105 from majetideepak/ARROW-1536 and squashes the following commits:

9f4ed610 [Deepak Majeti] Review comments
d49e1aa0 [Deepak Majeti] Fix failure
055dc308 [Deepak Majeti] ARROW-1536:[C++] Do not transitively depend on libboost_system
---
 cpp/src/arrow/io/file.cc | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 0abadbb16af41..843cfe89fc086 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -107,16 +107,35 @@
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
+#if defined(_MSC_VER)
 #include <boost/filesystem.hpp>           // NOLINT
 #include <boost/system/system_error.hpp>  // NOLINT
-
 namespace fs = boost::filesystem;
+#define PlatformFilename fs::path
+
+namespace arrow {
+namespace io {
 
+#else
 namespace arrow {
 namespace io {
 
+struct PlatformFilename {
+  PlatformFilename() {}
+  explicit PlatformFilename(const std::string& path) { utf8_path = path; }
+
+  const char* c_str() const { return utf8_path.c_str(); }
+
+  const std::string& string() const { return utf8_path; }
+
+  size_t length() const { return utf8_path.size(); }
+
+  std::string utf8_path;
+};
+#endif
+
 static inline Status CheckOpenResult(int ret, int errno_actual,
-                                     const fs::path& file_name) {
+                                     const PlatformFilename& file_name) {
   if (ret == -1) {
     // TODO: errno codes to strings
     std::stringstream ss;
@@ -137,7 +156,7 @@ static inline int64_t lseek64_compat(int fd, int64_t pos, int whence) {
 #endif
 }
 
-static inline Status FileOpenReadable(const fs::path& file_name, int* fd) {
+static inline Status FileOpenReadable(const PlatformFilename& file_name, int* fd) {
   int ret;
   errno_t errno_actual = 0;
 #if defined(_MSC_VER)
@@ -152,7 +171,7 @@ static inline Status FileOpenReadable(const fs::path& file_name, int* fd) {
   return CheckOpenResult(ret, errno_actual, file_name);
 }
 
-static inline Status FileOpenWriteable(const fs::path& file_name, bool write_only,
+static inline Status FileOpenWriteable(const PlatformFilename& file_name, bool write_only,
                                        bool truncate, int* fd) {
   int ret;
   errno_t errno_actual = 0;
@@ -366,20 +385,20 @@ class OSFile {
 
  protected:
   Status SetFileName(const std::string& file_name) {
-    try {
 #if defined(_MSC_VER)
+    try {
       std::codecvt_utf8_utf16<wchar_t> utf16_converter;
       file_name_.assign(file_name, utf16_converter);
-#else
-      file_name_ = file_name;
-#endif
     } catch (boost::system::system_error& e) {
       return Status::Invalid(e.what());
     }
+#else
+    file_name_ = PlatformFilename(file_name);
+#endif
     return Status::OK();
   }
 
-  fs::path file_name_;
+  PlatformFilename file_name_;
 
   std::mutex lock_;
 

From 2551050b358f572a18ac7edd964c55c092ed5b1a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 19 Sep 2017 23:46:15 -0400
Subject: [PATCH 1071/1644] ARROW-1554: [Python] Update Sphinx install page to
 note that VC14 runtime may need to be installed on Windows

Close #819 (tidying)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1115 from wesm/ARROW-1554 and squashes the following commits:

a7c3e279 [Wes McKinney] Update Sphinx install page to note that VC14 runtime may need to be installed separately when using pip on Windows
---
 python/doc/source/install.rst | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/python/doc/source/install.rst b/python/doc/source/install.rst
index a2a6520be4884..d07d9004d2632 100644
--- a/python/doc/source/install.rst
+++ b/python/doc/source/install.rst
@@ -30,19 +30,20 @@ To install the latest version of PyArrow from conda-forge using conda:
 Pip
 ---
 
-Install the latest version from PyPI:
+Install the latest version from PyPI (Windows, Linux, and macOS):
 
 .. code-block:: bash
 
     pip install pyarrow
 
+If you encounter any importing issues of the pip wheels on Windows, you may
+need to install the `Visual C++ Redistributable for Visual Studio 2015
+<https://www.microsoft.com/en-us/download/details.aspx?id=48145>`_.
+
 .. note::
 
-    Currently there are only binary artifacts available for Linux and MacOS.
-    Otherwise this will only pull the python sources and assumes an existing
-    installation of the C++ part of Arrow.  To retrieve the binary artifacts,
-    you'll need a recent ``pip`` version that supports features like the
-    ``manylinux1`` tag.
+   Windows packages are only available for Python 3.5 and higher (this is also
+   true for TensorFlow and any package that is implemented with modern C++).
 
 Installing from source
 ----------------------

From 903d03b0bab82fe6b8724d98cccedf582e2b4f86 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Wed, 20 Sep 2017 08:01:49 -0400
Subject: [PATCH 1072/1644] ARROW-1553: [JAVA] Implement setInitialCapacity for
 MapWriter

 Implement setInitialCapacity for MapWriter and pass on this capacity during lazy creation of child vectors

cc @jacques-n , @StevenMPhillips

Author: siddharth <siddharth@dremio.com>

Closes #1113 from siddharthteotia/ARROW-1553 and squashes the following commits:

5a759be7 [siddharth] ARROW-1553:  Implement setInitialCapacity for MapWriter and pass on this capacity during lazy creation of child vectors
---
 .../main/codegen/templates/MapWriters.java    | 10 +++
 .../complex/writer/TestComplexWriter.java     | 88 +++++++++++++++++++
 2 files changed, 98 insertions(+)

diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 14cc08d7db0e9..b89f91457e8b2 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -47,6 +47,7 @@
 public class ${mode}MapWriter extends AbstractFieldWriter {
 
   protected final ${containerClass} container;
+  private int initialCapacity;
   private final Map<String, FieldWriter> fields = Maps.newHashMap();
   public ${mode}MapWriter(${containerClass} container) {
     <#if mode == "Single">
@@ -55,6 +56,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
     }
     </#if>
     this.container = container;
+    this.initialCapacity = 0;
     for (Field child : container.getField().getChildren()) {
       MinorType minorType = Types.getMinorTypeForArrowType(child.getType());
       switch (minorType) {
@@ -101,6 +103,11 @@ public int getValueCapacity() {
     return container.getValueCapacity();
   }
 
+  public void setInitialCapacity(int initialCapacity) {
+    this.initialCapacity = initialCapacity;
+    container.setInitialCapacity(initialCapacity);
+  }
+
   @Override
   public boolean isEmptyMap() {
     return 0 == container.size();
@@ -248,6 +255,9 @@ public void end() {
       writer = new PromotableWriter(v, container, getNullableMapWriterFactory());
       vector = v;
       if (currentVector == null || currentVector != vector) {
+        if(this.initialCapacity > 0) {
+          vector.setInitialCapacity(this.initialCapacity);
+        }
         vector.allocateNewSafe();
       } 
       writer.setPosition(idx());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index f81cd557a9d8f..856d60724b085 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -28,6 +28,10 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.SchemaChangeCallBack;
+import org.apache.arrow.vector.NullableFloat8Vector;
+import org.apache.arrow.vector.NullableFloat4Vector;
+import org.apache.arrow.vector.NullableBigIntVector;
+import org.apache.arrow.vector.NullableIntVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
@@ -38,6 +42,11 @@
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.impl.UnionReader;
 import org.apache.arrow.vector.complex.impl.UnionWriter;
+import org.apache.arrow.vector.complex.impl.SingleMapWriter;
+import org.apache.arrow.vector.complex.reader.IntReader;
+import org.apache.arrow.vector.complex.reader.Float8Reader;
+import org.apache.arrow.vector.complex.reader.Float4Reader;
+import org.apache.arrow.vector.complex.reader.BigIntReader;
 import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
@@ -834,4 +843,83 @@ public void complexCopierWithList() {
     innerMap = (JsonStringHashMap<?, ?>) object.get(3);
     assertEquals(2, innerMap.get("a"));
   }
+
+  @Test
+  public void testSingleMapWriter1() {
+    /* initialize a SingleMapWriter with empty MapVector and then lazily
+     * create all vectors with expected initialCapacity.
+     */
+    MapVector parent = MapVector.empty("parent", allocator);
+    SingleMapWriter singleMapWriter = new SingleMapWriter(parent);
+
+    int initialCapacity = 1024;
+    singleMapWriter.setInitialCapacity(initialCapacity);
+
+    IntWriter intWriter = singleMapWriter.integer("intField");
+    BigIntWriter bigIntWriter = singleMapWriter.bigInt("bigIntField");
+    Float4Writer float4Writer = singleMapWriter.float4("float4Field");
+    Float8Writer float8Writer = singleMapWriter.float8("float8Field");
+    ListWriter listWriter = singleMapWriter.list("listField");
+
+    int intValue = 100;
+    long bigIntValue = 10000;
+    float float4Value = 100.5f;
+    double float8Value = 100.375;
+
+    for (int i = 0; i < initialCapacity; i++) {
+      singleMapWriter.start();
+
+      intWriter.writeInt(intValue + i);
+      bigIntWriter.writeBigInt(bigIntValue + (long)i);
+      float4Writer.writeFloat4(float4Value + (float)i);
+      float8Writer.writeFloat8(float8Value + (double)i);
+
+      listWriter.setPosition(i);
+      listWriter.startList();
+      listWriter.integer().writeInt(intValue + i);
+      listWriter.integer().writeInt(intValue + i + 1);
+      listWriter.integer().writeInt(intValue + i + 2);
+      listWriter.integer().writeInt(intValue + i + 3);
+      listWriter.endList();
+
+      singleMapWriter.end();
+    }
+
+    NullableIntVector intVector = (NullableIntVector)parent.getChild("intField");
+    NullableBigIntVector bigIntVector = (NullableBigIntVector)parent.getChild("bigIntField");
+    NullableFloat4Vector float4Vector = (NullableFloat4Vector)parent.getChild("float4Field");
+    NullableFloat8Vector float8Vector = (NullableFloat8Vector)parent.getChild("float8Field");
+
+    assertEquals(initialCapacity, singleMapWriter.getValueCapacity());
+    assertEquals(initialCapacity, intVector.getValueCapacity());
+    assertEquals(initialCapacity, bigIntVector.getValueCapacity());
+    assertEquals(initialCapacity, float4Vector.getValueCapacity());
+    assertEquals(initialCapacity, float8Vector.getValueCapacity());
+
+    MapReader singleMapReader = new SingleMapReaderImpl(parent);
+
+    IntReader intReader = singleMapReader.reader("intField");
+    BigIntReader bigIntReader = singleMapReader.reader("bigIntField");
+    Float4Reader float4Reader = singleMapReader.reader("float4Field");
+    Float8Reader float8Reader = singleMapReader.reader("float8Field");
+    UnionListReader listReader = (UnionListReader)singleMapReader.reader("listField");
+
+    for (int i = 0; i < initialCapacity; i++) {
+      intReader.setPosition(i);
+      bigIntReader.setPosition(i);
+      float4Reader.setPosition(i);
+      float8Reader.setPosition(i);
+      listReader.setPosition(i);
+
+      assertEquals(intValue + i, intReader.readInteger().intValue());
+      assertEquals(bigIntValue + (long)i, bigIntReader.readLong().longValue());
+      assertEquals(float4Value + (float)i, float4Reader.readFloat().floatValue(), 0);
+      assertEquals(float8Value + (double)i, float8Reader.readDouble().doubleValue(), 0);
+
+      for (int j = 0; j < 4; j++) {
+        listReader.next();
+        assertEquals(intValue + i + j, listReader.reader().readInteger().intValue());
+      }
+    }
+  }
 }

From 9997a1a3bacc397dc58645b2d8cfd0c3923efe01 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <tom.w.augspurger@gmail.com>
Date: Wed, 20 Sep 2017 13:36:28 -0400
Subject: [PATCH 1073/1644] ARROW-1557 [Python] Validate names length in
 Table.from_arrays

We now raise a ValueError when the length of the names doesn't match
the length of the arrays.

```python
In [1]: import pyarrow as pa

In [2]: pa.Table.from_arrays([pa.array([1, 2]), pa.array([3, 4])], names=['a', 'b', 'c'])
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-2-cda803f3f774> in <module>()
----> 1 pa.Table.from_arrays([pa.array([1, 2]), pa.array([3, 4])], names=['a', 'b', 'c'])

table.pxi in pyarrow.lib.Table.from_arrays()

table.pxi in pyarrow.lib._schema_from_arrays()

ValueError: Length of names (3) does not match length of arrays (2)
```

This affected `RecordBatch.from_arrays` and `Table.from_arrays`.

Author: Tom Augspurger <tom.w.augspurger@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1117 from TomAugspurger/validate-names and squashes the following commits:

4df6f593 [Tom Augspurger] REF: avoid redundant len calculation
965a5608 [Wes McKinney] Fix test failure exposed in test_parquet.py
ed74d522 [Tom Augspurger] ARROW-1557 [Python] Validate names length in Table.from_arrays
---
 python/pyarrow/table.pxi             |  5 ++++-
 python/pyarrow/tests/test_parquet.py |  2 +-
 python/pyarrow/tests/test_table.py   | 24 ++++++++++++++++++++++++
 3 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 68eb5cbdcac6c..028797e45b8d9 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -315,7 +315,7 @@ cdef int _schema_from_arrays(
 
     fields.resize(K)
 
-    if len(arrays) == 0:
+    if not K:
         raise ValueError('Must pass at least one array')
 
     if isinstance(arrays[0], Column):
@@ -328,6 +328,9 @@ cdef int _schema_from_arrays(
         if names is None:
             raise ValueError('Must pass names when constructing '
                              'from Array objects')
+        if len(names) != K:
+            raise ValueError("Length of names ({}) does not match "
+                             "length of arrays ({})".format(len(names), K))
         for i in range(K):
             val = arrays[i]
             if isinstance(val, (Array, ChunkedArray)):
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 6266878506059..eb23894f48082 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -57,7 +57,7 @@ def test_single_pylist_column_roundtrip(tmpdir):
         filename = tmpdir.join('single_{}_column.parquet'
                                .format(dtype.__name__))
         data = [pa.array(list(map(dtype, range(5))))]
-        table = pa.Table.from_arrays(data, names=('a', 'b'))
+        table = pa.Table.from_arrays(data, names=['a'])
         _write_table(table, filename.strpath)
         table_read = _read_table(filename.strpath)
         for col_written, col_read in zip(table.itercolumns(),
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 311d64e8a8878..4a2868a3c1de3 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -82,6 +82,18 @@ def test_recordbatch_basics():
         batch[2]
 
 
+def test_recordbatch_from_arrays_invalid_names():
+    data = [
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10])
+    ]
+    with pytest.raises(ValueError):
+        pa.RecordBatch.from_arrays(data, names=['a', 'b', 'c'])
+
+    with pytest.raises(ValueError):
+        pa.RecordBatch.from_arrays(data, names=['a'])
+
+
 def test_recordbatch_empty_metadata():
     data = [
         pa.array(range(5)),
@@ -200,6 +212,18 @@ def test_table_basics():
             assert chunk is not None
 
 
+def test_table_from_arrays_invalid_names():
+    data = [
+        pa.array(range(5)),
+        pa.array([-10, -5, 0, 5, 10])
+    ]
+    with pytest.raises(ValueError):
+        pa.Table.from_arrays(data, names=['a', 'b', 'c'])
+
+    with pytest.raises(ValueError):
+        pa.Table.from_arrays(data, names=['a'])
+
+
 def test_table_add_column():
     data = [
         pa.array(range(5)),

From 975f32b1eff2aac48eb439938ae16c61ba44b560 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Wed, 20 Sep 2017 13:40:10 -0400
Subject: [PATCH 1074/1644] ARROW-1497: [Java] Fix JsonReader to initialize
 count correctly

Author: Li Jin <ice.xelloss@gmail.com>

Closes #1067 from icexelloss/json-reader-ARROW-1497 and squashes the following commits:

6d4e1dfb [Li Jin] Fix JsonReader to read union vectors correctly
---
 .../vector/file/json/JsonFileReader.java      | 11 ++++---
 .../arrow/vector/file/json/TestJSONFile.java  | 32 +++++++++----------
 2 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 71685d1358923..8bb0f26d978f7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -67,7 +67,6 @@
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.schema.ArrowVectorType;
@@ -217,6 +216,11 @@ public VectorSchemaRoot read() throws IOException {
     }
   }
 
+  /*
+   * TODO: This method doesn't load some vectors correctly. For instance, it doesn't initialize
+   * `lastSet` in ListVector, VarCharVector, NullableVarBinaryVector A better way of implementing
+   * this function is to use `loadFieldBuffers` methods in FieldVector.
+   */
   private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException {
     List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
     List<BufferBacked> fieldInnerVectors = vector.getFieldInnerVectors();
@@ -231,6 +235,8 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name);
       }
       int count = readNextField("count", Integer.class);
+      vector.allocateNew();
+      vector.getMutator().setValueCount(count);
       for (int v = 0; v < vectorTypes.size(); v++) {
         ArrowVectorType vectorType = vectorTypes.get(v);
         BufferBacked innerVector = fieldInnerVectors.get(v);
@@ -266,9 +272,6 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         }
         readToken(END_ARRAY);
       }
-      if (vector instanceof NullableMapVector) {
-        ((NullableMapVector) vector).valueCount = count;
-      }
     }
     readToken(END_OBJECT);
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index b7c06327291e8..960567fc87087 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -30,6 +30,7 @@
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.file.BaseFileTest;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.Validator;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -96,28 +97,25 @@ public void testWriteReadUnionJSON() throws IOException {
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
         NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
-
       writeUnionData(count, parent);
-
       printVectors(parent.getChildrenFromFields());
 
-      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
-      validateUnionData(count, root);
+      try (VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"))) {
+        validateUnionData(count, root);
+        writeJSON(file, root, null);
 
-      writeJSON(file, root, null);
-    }
-    // read
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
-    ) {
-      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
-      Schema schema = reader.start();
-      LOGGER.debug("reading schema: " + schema);
+        // read
+        try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE)) {
+          JsonFileReader reader = new JsonFileReader(file, readerAllocator);
 
-      // initialize vectors
-      try (VectorSchemaRoot root = reader.read();) {
-        validateUnionData(count, root);
+          Schema schema = reader.start();
+          LOGGER.debug("reading schema: " + schema);
+
+          try (VectorSchemaRoot rootFromJson = reader.read();) {
+            validateUnionData(count, rootFromJson);
+            Validator.compareVectorSchemaRoot(root, rootFromJson);
+          }
+        }
       }
     }
   }

From 203fb635505ad9ffe6a205f9473cfc6e6f47e500 Mon Sep 17 00:00:00 2001
From: Amir Malekpour <a.malekpour@gmail.com>
Date: Wed, 20 Sep 2017 20:51:29 -0400
Subject: [PATCH 1075/1644] =?UTF-8?q?ARROW-1500:=20[C++]=20Do=20not=20igno?=
 =?UTF-8?q?re=20return=20value=20from=20truncate=20in=20MemoryMa=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ppedFile::Create

Author: Amir Malekpour <a.malekpour@gmail.com>

Author: Amir Malekpour <a.malekpour@gmail.com>

Closes #1116 from amirma/arrow-1500 and squashes the following commits:

689aaa9 [Amir Malekpour]  RROW-1500: [C++] Do not ignore return value from truncate in MemoryMappedFile::Create
---
 cpp/src/arrow/io/file.cc | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 843cfe89fc086..ca536321ba3d8 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -134,12 +134,13 @@ struct PlatformFilename {
 };
 #endif
 
-static inline Status CheckOpenResult(int ret, int errno_actual,
-                                     const PlatformFilename& file_name) {
+static inline Status CheckFileOpResult(int ret, int errno_actual,
+                                       const PlatformFilename& file_name,
+                                       const std::string& opname) {
   if (ret == -1) {
-    // TODO: errno codes to strings
     std::stringstream ss;
-    ss << "Failed to open local file: " << file_name.string();
+    ss << "Failed to " << opname << " file: " << file_name.string();
+    ss << " , error: " << std::strerror(errno_actual);
     return Status::IOError(ss.str());
   }
   return Status::OK();
@@ -168,7 +169,7 @@ static inline Status FileOpenReadable(const PlatformFilename& file_name, int* fd
   errno_actual = errno;
 #endif
 
-  return CheckOpenResult(ret, errno_actual, file_name);
+  return CheckFileOpResult(ret, errno_actual, file_name, "open local");
 }
 
 static inline Status FileOpenWriteable(const PlatformFilename& file_name, bool write_only,
@@ -211,7 +212,7 @@ static inline Status FileOpenWriteable(const PlatformFilename& file_name, bool w
 
   ret = *fd = open(file_name.c_str(), oflag, ARROW_WRITE_SHMODE);
 #endif
-  return CheckOpenResult(ret, errno_actual, file_name);
+  return CheckFileOpResult(ret, errno_actual, file_name, "open local");
 }
 
 static inline Status FileTell(int fd, int64_t* pos) {
@@ -599,13 +600,21 @@ MemoryMappedFile::~MemoryMappedFile() {}
 
 Status MemoryMappedFile::Create(const std::string& path, int64_t size,
                                 std::shared_ptr<MemoryMappedFile>* out) {
+  int ret;
+  errno_t errno_actual;
   std::shared_ptr<FileOutputStream> file;
   RETURN_NOT_OK(FileOutputStream::Open(path, &file));
+
 #ifdef _MSC_VER
-  _chsize_s(file->file_descriptor(), static_cast<size_t>(size));
+  errno_actual = _chsize_s(file->file_descriptor(), static_cast<size_t>(size));
+  ret = errno_actual == 0 ? 0 : -1;
 #else
-  ftruncate(file->file_descriptor(), static_cast<size_t>(size));
+  ret = ftruncate(file->file_descriptor(), static_cast<size_t>(size));
+  errno_actual = errno;
 #endif
+
+  RETURN_NOT_OK(CheckFileOpResult(ret, errno_actual, PlatformFilename(path), "truncate"));
+
   RETURN_NOT_OK(file->Close());
   return MemoryMappedFile::Open(path, FileMode::READWRITE, out);
 }

From d154c10368d86abc03241982ea43a1a872cbea73 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 20 Sep 2017 22:59:26 -0400
Subject: [PATCH 1076/1644] ARROW-1578: [C++] Run lint checks in Travis CI much
 earlier at before_script stage to fail faster

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1118 from wesm/ARROW-1578 and squashes the following commits:

0bb5202 [Wes McKinney] System python not available on xcode 6.4 machines
d1cf679 [Wes McKinney] Set language: python when linting on macOS
910f684 [Wes McKinney] Fixes for linting. Do not cache .conda_packages
ed9e23a [Wes McKinney] Move linting to separate shell script
b7db083 [Wes McKinney] Only run lint checks when not running in --only-library mode
7e50fad [Wes McKinney] Revert cpplint failure
28fc3fb [Wes McKinney] Typo
329f017 [Wes McKinney] Run lint checks before compiling anything. Make cpplint warning
---
 .travis.yml                    |  6 +++--
 ci/travis_before_script_cpp.sh |  1 +
 ci/travis_lint.sh              | 40 ++++++++++++++++++++++++++++++++++
 ci/travis_script_cpp.sh        |  2 --
 ci/travis_script_python.sh     |  8 +------
 5 files changed, 46 insertions(+), 11 deletions(-)
 create mode 100755 ci/travis_lint.sh

diff --git a/.travis.yml b/.travis.yml
index f6f64003d273d..4691eeb0656e0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,8 +40,6 @@ services:
 
 cache:
   ccache: true
-  directories:
-    - $HOME/.conda_packages
 
 matrix:
   fast_finish: true
@@ -58,6 +56,7 @@ matrix:
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_VALGRIND=1
     - export ARROW_TRAVIS_PLASMA=1
+    - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
@@ -65,6 +64,7 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
   - compiler: clang
+    language: cpp
     osx_image: xcode6.4
     os: osx
     cache:
@@ -106,6 +106,7 @@ matrix:
     - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - export CC="clang-4.0"
     - export CXX="clang++-4.0"
+    - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
@@ -123,6 +124,7 @@ matrix:
     before_script:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
+    - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index d46fa2ff9433c..a7f1d2756344b 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -99,6 +99,7 @@ else
           $ARROW_CPP_DIR
 fi
 
+# Build and install libraries
 $TRAVIS_MAKE -j4
 $TRAVIS_MAKE install
 
diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
new file mode 100755
index 0000000000000..9b7b474524fb4
--- /dev/null
+++ b/ci/travis_lint.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+# Fail fast for code linting issues
+mkdir $TRAVIS_BUILD_DIR/cpp/lint
+pushd $TRAVIS_BUILD_DIR/cpp/lint
+
+cmake ..
+make lint
+
+popd
+
+# Fail fast on style checks
+sudo pip install flake8
+
+PYARROW_DIR=$TRAVIS_BUILD_DIR/python
+
+flake8 --count $PYTHON_DIR/pyarrow
+
+# Check Cython files with some checks turned off
+flake8 --count --config=$PYTHON_DIR/.flake8.cython \
+       $PYTHON_DIR/pyarrow
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index 4e3e7bbea1c08..a2079036c4549 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -27,8 +27,6 @@ git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
 
 pushd $CPP_BUILD_DIR
 
-$TRAVIS_MAKE lint
-
 # ARROW-209: checks depending on the LLVM toolchain are disabled temporarily
 # until we are able to install the full LLVM toolchain in Travis CI again
 
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index e2cf254934892..f6ce18a2abe54 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -38,7 +38,7 @@ which python
 conda install -y -q nomkl
 
 # Expensive dependencies install from Continuum package repo
-conda install -y -q pip numpy pandas cython flake8
+conda install -y -q pip numpy pandas cython
 
 # Build C++ libraries
 pushd $ARROW_CPP_BUILD_DIR
@@ -62,12 +62,6 @@ popd
 # Other stuff pip install
 pushd $ARROW_PYTHON_DIR
 
-# Fail fast on style checks
-flake8 --count pyarrow
-
-# Check Cython files with some checks turned off
-flake8 --count --config=.flake8.cython pyarrow
-
 pip install -r requirements.txt
 python setup.py build_ext --with-parquet --with-plasma \
        install --single-version-externally-managed --record=record.text

From c470c9c2d39ff81fff5ee3c5dc6e21d03da5f304 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 21 Sep 2017 09:49:02 -0400
Subject: [PATCH 1077/1644] ARROW-1591: C++: Xcode 9 is not correctly detected

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1121 from xhochy/ARROW-1591 and squashes the following commits:

0b3a11a [Uwe L. Korn] ARROW-1591: C++: Xcode 9 is not correctly detected
---
 cpp/cmake_modules/CompilerInfo.cmake | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index 4a18376df59f4..a1b470182a8dc 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -68,6 +68,11 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-8")
   set(COMPILER_FAMILY "clang")
   set(COMPILER_VERSION "3.8.0svn")
 
+# clang on Mac OS X, XCode 8.
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-9")
+  set(COMPILER_FAMILY "clang")
+  set(COMPILER_VERSION "4.0.0svn")
+
 # gcc
 elseif("${COMPILER_VERSION_FULL_LOWER}" MATCHES ".*gcc[ -]version.*")
   set(COMPILER_FAMILY "gcc")

From cfcee74ba5a75668da22e812289b3e1bf816aaf7 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 21 Sep 2017 15:37:45 -0700
Subject: [PATCH 1078/1644] ARROW-1347: [JAVA] Return consistent child field
 name for List Vectors

This makes the child fields of ListVector have consistent names of `ListVector.DATA_VECTOR_NAME`. Previously, an empty ListVector would have a child name of `ZeroVector.name` which is "[DEFAULT]".

Author: Bryan Cutler <cutlerb@gmail.com>
Author: Steven Phillips <steven@dremio.com>

Closes #1119 from BryanCutler/java-ListVector-child-name-ARROW-1347 and squashes the following commits:

c240378 [Bryan Cutler] changed to use instanceof and added test
2923a45 [Steven Phillips] ARROW-1347: [JAVA] return consistent child field name for List vectors
---
 .../org/apache/arrow/vector/complex/ListVector.java  |  4 ++++
 .../java/org/apache/arrow/vector/TestListVector.java | 12 ++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 79aca0c7b307c..470317f8c07a6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -49,6 +49,7 @@
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -323,6 +324,9 @@ public int getBufferSize() {
 
   @Override
   public Field getField() {
+    if (getDataVector() instanceof ZeroVector) {
+      return new Field(name, fieldType, ImmutableList.of(new Field(DATA_VECTOR_NAME, FieldType.nullable(Null.INSTANCE), null)));
+    }
     return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 276121c0d1f3b..1c9b574998018 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -628,4 +628,16 @@ public void testGetBufferAddress() throws Exception {
       assertEquals(offsetAddress, buffers.get(1).memoryAddress());
     }
   }
+
+  @Test
+  public void testConsistentChildName() throws Exception {
+    try (ListVector listVector = ListVector.empty("sourceVector", allocator)) {
+      String emptyListStr = listVector.getField().toString();
+      assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME));
+
+      listVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+      String emptyVectorStr = listVector.getField().toString();
+      assertTrue(emptyVectorStr.contains(ListVector.DATA_VECTOR_NAME));
+    }
+  }
 }

From 8fd73b48c852ea5d419dea48dd7d7eded00674dd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 21 Sep 2017 19:26:28 -0400
Subject: [PATCH 1079/1644] ARROW-1595: [Python] Fix package dependency
 resolution issue causing broken builds

One of the dependencies installed in the docs requirements is causing NumPy to get downgraded by the SAT solver, and this is then causing an ABI conflict with the pyarrow build (which was built with a different version of NumPy). This installs everything in one `conda install` call

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1123 from wesm/ARROW-1595 and squashes the following commits:

60b05ad5 [Wes McKinney] Install conda dependencies all at once, pin NumPy version
---
 ci/travis_script_python.sh | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index f6ce18a2abe54..b779aec195637 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -34,11 +34,16 @@ source activate $CONDA_ENV_DIR
 python --version
 which python
 
-# faster builds, please
-conda install -y -q nomkl
-
-# Expensive dependencies install from Continuum package repo
-conda install -y -q pip numpy pandas cython
+conda install -y -q pip \
+      nomkl \
+      numpy=1.13.1 \
+      pandas \
+      cython \
+      ipython \
+      matplotlib \
+      numpydoc \
+      sphinx \
+      sphinx_bootstrap_theme
 
 # Build C++ libraries
 pushd $ARROW_CPP_BUILD_DIR
@@ -80,7 +85,6 @@ python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
 if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
   # Build documentation once
   pushd $ARROW_PYTHON_DIR/doc
-  conda install -y -q --file=requirements.txt
   sphinx-build -b html -d _build/doctrees -W source _build/html
   popd
 fi

From 8996a4ff3ca006983fb19f07d3530180e65a93d1 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Thu, 21 Sep 2017 21:28:07 -0400
Subject: [PATCH 1080/1644] ARROW-1590: [JS] Flow TS Table method generics

This PR fixes the Table generics to infer the types from the call site:

![kapture 2017-09-21 at 4 03 34](https://user-images.githubusercontent.com/178183/30692953-5b8638d6-9e82-11e7-9d66-b87eb50f0e3f.gif)

@wesm this PR also includes the fixes to the prepublish script I mentioned yesterday.

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1120 from trxcllnt/fix-ts-typings and squashes the following commits:

73d8eee [Paul Taylor] make package the default gulp task
1d269fe [Paul Taylor] flow table method generics
dd1e819 [Paul Taylor] more defensively typed reader internal values
ac6a778 [Paul Taylor] add comments explaining ARROW-1363 reader workaround
e37f885 [Paul Taylor] fix gulp and prepublish scripts
58fa201 [Paul Taylor] enforce exact dependency package versions
---
 js/.gitignore               |  3 +-
 js/.npmrc                   |  1 +
 js/gulpfile.js              | 63 +++++++++++++------------
 js/package.json             | 91 ++++++++++++++++---------------------
 js/prepublish.sh            |  6 +--
 js/src/reader/dictionary.ts | 11 ++---
 js/src/reader/vector.ts     | 19 +++++---
 js/src/table.ts             | 28 ++++++++----
 8 files changed, 115 insertions(+), 107 deletions(-)

diff --git a/js/.gitignore b/js/.gitignore
index b48f35b960ffb..6d0f88d191cb0 100644
--- a/js/.gitignore
+++ b/js/.gitignore
@@ -57,7 +57,8 @@ build/Release
 node_modules/
 jspm_packages/
 
-# Typescript v1 declaration files
+# Typescript declaration files
+types/
 typings/
 
 # Optional npm cache directory
diff --git a/js/.npmrc b/js/.npmrc
index 43c97e719a5a8..71ffabdd55d95 100644
--- a/js/.npmrc
+++ b/js/.npmrc
@@ -1 +1,2 @@
+save-prefix=
 package-lock=false
diff --git a/js/gulpfile.js b/js/gulpfile.js
index 9c945aa53af6e..9f8e564bd9e3a 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -63,14 +63,17 @@ for (const [target, format] of combinations([`all`, `all`])) {
     gulp.task(`clean:${combo}`, gulp.series(cleanTask(target, format, combo, `targets/${target}/${format}`)));
     gulp.task(`build:${combo}`, gulp.series(buildTask(target, format, combo, `targets/${target}/${format}`)));
     gulp.task(`bundle:${combo}`, gulp.series(bundleTask(target, format, combo, `targets/${target}/${format}`)));
+    gulp.task(`package:${combo}`, gulp.series(packageTask(target, format, combo, `targets/${target}/${format}`)));
     gulp.task(`test:debug:${combo}`, gulp.series(testTask(target, format, combo, `targets/${target}/${format}`, true)));
 }
 
 gulp.task(`test`, gulp.series(runTaskCombos(`test`)));
 gulp.task(`clean`, gulp.parallel(runTaskCombos(`clean`)));
-gulp.task(`build`, gulp.parallel(runTaskCombos(`bundle`)));
+gulp.task(`build`, gulp.parallel(runTaskCombos(`build`)));
+gulp.task(`bundle`, gulp.parallel(runTaskCombos(`bundle`)));
+gulp.task(`package`, gulp.parallel(runTaskCombos(`package`)));
 gulp.task(`test:debug`, gulp.series(runTaskCombos(`test:debug`)));
-gulp.task(`default`, gulp.task(`build`));
+gulp.task(`default`, gulp.task(`package`));
 
 function runTaskCombos(name) {
     const combos = [];
@@ -87,7 +90,7 @@ function cleanTask(target, format, taskName, outDir) {
     return function cleanTask() {
         const globs = [`${outDir}/**`];
         if (target === `es5` && format === `cjs`) {
-            globs.push(`typings`);
+            globs.push(`types`, `typings`);
         }
         return del(globs);
     };
@@ -100,30 +103,32 @@ function buildTask(target, format, taskName, outDir) {
 }
 
 function bundleTask(target, format, taskName, outDir) {
-    return [
-        [`build:${taskName}`],
-        function bundleTask() {
-            return streamMerge([
-                pump(gulp.src([`LICENSE`, `README.md`]), gulp.dest(outDir), onError),
-                pump(
-                    gulp.src(`package.json`),
-                    gulpJsonTransform((orig) => [
-                        `version`, `description`,
-                        `author`, `homepage`, `bugs`, `license`,
-                        `keywords`, `repository`, `peerDependencies`
-                    ].reduce((copy, key) => (
-                        (copy[key] = orig[key]) && copy || copy
-                    ), {
-                        main: `Arrow.js`,
-                        typings: `Arrow.d.ts`,
-                        name: `@apache-arrow/${target}-${format}`
-                    }), 2),
-                    gulp.dest(outDir),
-                    onError
-                )
-            ])
-        }
-    ];
+    return function bundleTask() {
+        return streamMerge([
+            pump(gulp.src([`LICENSE`, `README.md`]), gulp.dest(outDir), onError),
+            pump(
+                gulp.src(`package.json`),
+                gulpJsonTransform((orig) => [
+                    `version`, `description`, `keywords`,
+                    `repository`, `author`, `homepage`, `bugs`, `license`,
+                    `dependencies`, `peerDependencies`
+                ].reduce((copy, key) => (
+                    (copy[key] = orig[key]) && copy || copy
+                ), {
+                    main: `Arrow.js`,
+                    types: `Arrow.d.ts`,
+                    typings: `Arrow.d.ts`,
+                    name: `@apache-arrow/${target}-${format}`
+                }), 2),
+                gulp.dest(outDir),
+                onError
+            )
+        ]);
+    }
+}
+
+function packageTask(target, format, taskName, outDir) {
+    return [`build:${taskName}`, `bundle:${taskName}`];
 }
 
 function testTask(target, format, taskName, outDir, debug) {
@@ -240,10 +245,10 @@ function typescriptTask(target, format, taskName, outDir) {
                 js = [js, sourcemaps.write(), gulp.dest(outDir)];
                 // copy types to the root
                 if (target === `es5` && format === `cjs`) {
-                    dts.push(gulp.dest(`typings`));
+                    dts.push(gulp.dest(`types`));
                 }
                 tsProjects.push({
-                    target, format, 
+                    target, format,
                     js: js = pump(...js, onError),
                     dts: dts = pump(...dts, onError)
                 });
diff --git a/js/package.json b/js/package.json
index e6848fd641d0e..03687a8b25ca2 100644
--- a/js/package.json
+++ b/js/package.json
@@ -1,26 +1,27 @@
 {
   "name": "apache-arrow",
   "version": "0.1.2",
+  "types": "./types/Arrow.d.ts",
+  "typings": "./types/Arrow.d.ts",
   "main": "./targets/es5/cjs/Arrow.js",
   "module": "./targets/es5/esm/Arrow.js",
   "browser": "./targets/es5/umd/Arrow.js",
   "jsnext:main": "./targets/es2015/esm/Arrow.js",
   "esnext:main": "./targets/esnext/esm/Arrow.js",
-  "typings": "./typings/Arrow.d.ts",
   "description": "Apache Arrow columnar in-memory format",
   "scripts": {
     "lerna": "lerna",
-    "commit": "git-cz",
     "test": "gulp test",
     "build": "gulp build",
     "clean": "gulp clean",
+    "bundle": "gulp bundle",
+    "package": "gulp package",
     "perf": "node ./perf/index.js",
     "test:debug": "gulp test:debug",
     "test:coverage": "gulp test -t esnext -m esm --coverage",
-    "validate": "npm-run-all lint build test",
+    "validate": "npm-run-all clean lint build test bundle",
     "lerna:publish": "lerna exec --bail=false npm publish",
     "prepublishOnly": "sh ./prepublish.sh",
-    "commitmsg": "validate-commit-msg",
     "doc": "shx rm -rf ./doc && esdoc",
     "lint": "npm-run-all -p lint:*",
     "lint:src": "tslint --fix --type-check -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
@@ -43,64 +44,52 @@
   "files": [
     "src",
     "dist",
-    "typings",
+    "types",
     "targets",
     "LICENSE",
     "README.md"
   ],
   "peerDependencies": {
-    "tslib": "^1.7.1"
+    "tslib": "~1.7.1",
+    "command-line-usage": "4.0.1"
   },
   "dependencies": {
-    "command-line-args": "~4.0.7",
-    "command-line-usage": "~4.0.1",
-    "flatbuffers": "~1.7.0",
-    "text-encoding": "~0.6.4"
+    "flatbuffers": "1.7.0",
+    "text-encoding": "0.6.4"
   },
   "devDependencies": {
-    "@types/flatbuffers": "~1.6.4",
-    "@types/jest": "~20.0.8",
-    "@types/node": "~8.0.24",
+    "@types/flatbuffers": "1.6.4",
+    "@types/jest": "20.0.8",
+    "@types/node": "^8.0.24",
     "@types/text-encoding": "0.0.32",
-    "benchmark": "~2.1.4",
-    "commitizen": "~2.9.6",
-    "conventional-changelog-cli": "~1.3.2",
-    "conventional-commits-detector": "~0.1.1",
-    "conventional-github-releaser": "~1.1.12",
-    "conventional-recommended-bump": "~1.0.1",
-    "coveralls": "~2.13.1",
-    "cz-conventional-changelog": "~2.0.0",
-    "del": "~3.0.0",
-    "esdoc": "~1.0.1",
-    "esdoc-standard-plugin": "~1.0.0",
-    "google-closure-compiler": "~20170910.0.0",
+    "benchmark": "2.1.4",
+    "coveralls": "2.13.1",
+    "command-line-args": "4.0.7",
+    "del": "3.0.0",
+    "esdoc": "1.0.3",
+    "esdoc-standard-plugin": "1.0.0",
+    "google-closure-compiler": "20170910.0.0",
     "gulp": "github:gulpjs/gulp#4.0",
-    "gulp-json-transform": "~0.4.2",
-    "gulp-sourcemaps": "~2.6.1",
-    "gulp-typescript": "~3.2.2",
-    "jest": "~21.1.0",
-    "jest-environment-node-debug": "~2.0.0",
-    "json": "~9.0.6",
-    "lerna": "~2.1.2",
-    "lint-staged": "~4.2.1",
-    "merge2": "~1.2.0",
-    "mkdirp": "~0.5.1",
-    "npm-run-all": "~4.1.1",
-    "pump": "~1.0.2",
-    "rimraf": "~2.6.1",
-    "shx": "~0.2.2",
-    "text-encoding-utf-8": "~1.0.1",
-    "trash": "~4.0.1",
-    "ts-jest": "~21.0.1",
-    "tslib": "~1.7.1",
-    "tslint": "~5.7.0",
-    "typescript": "~2.5.2",
-    "validate-commit-msg": "~2.14.0"
-  },
-  "config": {
-    "commitizen": {
-      "path": "cz-conventional-changelog"
-    }
+    "gulp-json-transform": "0.4.2",
+    "gulp-sourcemaps": "2.6.1",
+    "gulp-typescript": "3.2.2",
+    "jest": "21.1.0",
+    "jest-environment-node-debug": "2.0.0",
+    "json": "9.0.6",
+    "lerna": "2.2.0",
+    "lint-staged": "4.2.1",
+    "merge2": "1.2.0",
+    "mkdirp": "0.5.1",
+    "npm-run-all": "4.1.1",
+    "pump": "1.0.2",
+    "rimraf": "2.6.2",
+    "shx": "0.2.2",
+    "text-encoding-utf-8": "1.0.1",
+    "trash": "4.1.0",
+    "ts-jest": "21.0.1",
+    "tslib": "1.7.1",
+    "tslint": "5.7.0",
+    "typescript": "2.5.2"
   },
   "lint-staged": {
     "*.@(ts)": [
diff --git a/js/prepublish.sh b/js/prepublish.sh
index 4ad8db1b439cb..b40504ae808cb 100644
--- a/js/prepublish.sh
+++ b/js/prepublish.sh
@@ -17,10 +17,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
+npm run clean
 npm run lint
 npm run build
 npm run test
-preset=`conventional-commits-detector` && echo $preset
-bump=`conventional-recommended-bump -p $preset` && echo $bump
-npm --no-git-tag-version version $bump &>/dev/null
+npm --no-git-tag-version version patch &>/dev/null
+npm run bundle
 npm run lerna:publish
\ No newline at end of file
diff --git a/js/src/reader/dictionary.ts b/js/src/reader/dictionary.ts
index aef2bc9d1cdbf..abf7ac3dfb976 100644
--- a/js/src/reader/dictionary.ts
+++ b/js/src/reader/dictionary.ts
@@ -19,21 +19,18 @@ import { readVector } from './vector';
 import { MessageBatch } from './message';
 import * as Schema_ from '../format/Schema_generated';
 import { IteratorState, Dictionaries } from './arrow';
-
 import Field = Schema_.org.apache.arrow.flatbuf.Field;
-import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
 
-export function* readDictionaries(field: Field,
+export function* readDictionaries(field: Field | null,
                                   batch: MessageBatch,
                                   iterator: IteratorState,
                                   dictionaries: Dictionaries) {
-    let id: string, encoding: DictionaryEncoding;
-    if ((encoding = field.dictionary()) &&
-        batch.id === (id = encoding.id().toFloat64().toString())) {
+    let id: string, encoding = field && field.dictionary();
+    if (encoding && batch.id === (id = encoding.id().toFloat64().toString())) {
         yield [id, readVector(field, batch, iterator, null)];
         return;
     }
-    for (let i = -1, n = field.childrenLength(); ++i < n;) {
+    for (let i = -1, n = field && field.childrenLength() || 0; ++i < n;) {
         // Since a dictionary batch can only contain a single vector, return early after we find it
         for (let result of readDictionaries(field.children(i), batch, iterator, dictionaries)) {
             yield result;
diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts
index a3cd79803641f..3b6663be89bdc 100644
--- a/js/src/reader/vector.ts
+++ b/js/src/reader/vector.ts
@@ -54,12 +54,13 @@ function readTypedVector(field: Field, batch: MessageBatch, iterator: IteratorSt
 }
 
 function readDictionaryVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) {
-    let encoding: DictionaryEncoding;
+    let encoding: DictionaryEncoding | null;
     if (dictionaries && (encoding = field.dictionary())) {
         let id = encoding.id().toFloat64().toString();
         let fieldType =  encoding.indexType() ||
             /* a dictionary index defaults to signed 32 bit int if unspecified */
             { bitWidth: () => 32, isSigned: () => true };
+        // workaround for https://issues.apache.org/jira/browse/ARROW-1363
         let indexField = createSyntheticDictionaryIndexField(field, fieldType);
         let index = readIntVector(indexField, batch, iterator, null, fieldType);
         return DictionaryVector.create(field, index.length, index, dictionaries[id]);
@@ -105,12 +106,16 @@ function createIntVector(field, length, data, validity, offsets, fieldType, batc
     let type = fieldType || field.type(new Int()), bitWidth = type.bitWidth();
     let Vector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()];
     return Vector.create(field, length, validity, data || offsets);
-    // ---------------------- so this is kinda strange 👆:
-    // The dictionary encoded vectors I generated from sample mapd-core queries have the indicies' data buffers
-    // tagged as VectorType.OFFSET (0) in the field metadata. The current TS impl ignores buffers' layout type,
-    // and assumes the second buffer is the data for a NullableIntVector. Since we've been stricter about enforcing
-    // the Arrow spec while parsing, the IntVector's data buffer reads empty in this case. If so, fallback to using
-    // the offsets buffer as the data, since IntVectors don't have offsets.
+    // ----------------------------------------------- 👆:
+    // Workaround for https://issues.apache.org/jira/browse/ARROW-1363
+    // This bug causes dictionary encoded vector indicies' IntVector data
+    // buffers to be tagged as VectorType.OFFSET (0) in the field metadata
+    // instead of VectorType.DATA. The `readVectorLayout` routine strictly
+    // obeys the types in the field metadata, so if we're parsing an Arrow
+    // file written by a version of the library published before ARROW-1363
+    // was fixed, the IntVector's data buffer will be null, and the offset
+    // buffer will be the actual data. If data is null, it's safe to assume
+    // the offset buffer is the data, because IntVectors don't have offsets.
 }
 
 const readFloatVector = readVectorLayout<number, FloatArray>(
diff --git a/js/src/table.ts b/js/src/table.ts
index 999bb245182d4..5e781054daf31 100644
--- a/js/src/table.ts
+++ b/js/src/table.ts
@@ -19,6 +19,8 @@ import { readBuffers } from './reader/arrow';
 import { StructVector } from './vector/struct';
 import { Vector, sliceToRangeArgs } from './vector/vector';
 
+export type RowObject = { [k: string]: any };
+
 export class Table implements Iterable<Map<string, any>> {
     public length: number;
     protected _columns: Vector<any>[];
@@ -66,22 +68,30 @@ export class Table implements Iterable<Map<string, any>> {
             yield column;
         }
     }
+    getRow(rowIndex: number): RowObject;
+    getRow(rowIndex: number, compact: boolean): Array<any>;
     getRow(rowIndex: number, compact?: boolean) {
         return (compact && rowAsArray || rowAsObject)(rowIndex, this._columns);
     }
-    getCell(columnName: string, rowIndex: number) {
-        return this.getColumn(columnName).get(rowIndex);
+    getCell<T extends any>(columnName: string, rowIndex: number) {
+        return this.getColumn<Vector<T>>(columnName).get(rowIndex);
     }
-    getCellAt(columnIndex: number, rowIndex: number) {
-        return this.getColumnAt(columnIndex).get(rowIndex);
+    getCellAt<T extends any>(columnIndex: number, rowIndex: number) {
+        return this.getColumnAt<Vector<T>>(columnIndex).get(rowIndex);
     }
-    getColumn<T = any>(columnName: string) {
-        return this._columnsMap[columnName] as Vector<T>;
+    getColumn<T extends Vector<any>>(columnName: string) {
+        return this._columnsMap[columnName] as T;
     }
-    getColumnAt<T = any>(columnIndex: number) {
-        return this._columns[columnIndex] as Vector<T>;
+    getColumnAt<T extends Vector<any>>(columnIndex: number) {
+        return this._columns[columnIndex] as T;
     }
-    toString({ index = false } = {}) {
+    toString(): string;
+    toString(index: boolean): string;
+    toString(options: { index: boolean }): string;
+    toString(options?: any) {
+        const index = typeof options === 'object' ? options && !!options.index
+                    : typeof options === 'boolean' ? !!options
+                    : false;
         const { length } = this;
         if (length <= 0) { return ''; }
         const maxColumnWidths = [];

From c0a5019bf86f846d7f85dff88cc45aeefd96ec42 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 22 Sep 2017 10:13:09 -0400
Subject: [PATCH 1081/1644] ARROW-1592: [GLib] Add GArrowUIntArrayBuilder

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1122 from kou/glib-add-uint-array-builder and squashes the following commits:

24bb9a7 [Kouhei Sutou] [GLib] Add missing "unsigned"
fd23f24 [Kouhei Sutou] [GLib] Fix build error on macOS
5b59775 [Kouhei Sutou] [GLib] Add UIntArrayBuilder
---
 c_glib/arrow-glib/array-builder.cpp    | 140 +++++++++++++++++++++++++
 c_glib/arrow-glib/array-builder.h      |  73 ++++++-------
 c_glib/test/helper/buildable.rb        |   4 +
 c_glib/test/test-array-builder.rb      |  22 ++++
 c_glib/test/test-uint-array-builder.rb |  59 +++++++++++
 5 files changed, 257 insertions(+), 41 deletions(-)
 create mode 100644 c_glib/test/test-uint-array-builder.rb

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index a6fad87ab8f51..bea7e8342c797 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -137,6 +137,11 @@ G_BEGIN_DECLS
  * you use this builder instead of specific integer size builder such
  * as #GArrowInt8ArrayBuilder.
  *
+ * #GArrowUIntArrayBuilder is the class to create a new unsigned
+ * integer array. Unsigned integer size is automatically chosen. It's
+ * recommend that you use this builder instead of specific unsigned
+ * integer size builder such as #GArrowUInt8ArrayBuilder.
+ *
  * #GArrowInt8ArrayBuilder is the class to create a new
  * #GArrowInt8Array.
  *
@@ -587,6 +592,141 @@ garrow_int_array_builder_append_nulls(GArrowIntArrayBuilder *builder,
 }
 
 
+G_DEFINE_TYPE(GArrowUIntArrayBuilder,
+              garrow_uint_array_builder,
+              GARROW_TYPE_ARRAY_BUILDER)
+
+static void
+garrow_uint_array_builder_init(GArrowUIntArrayBuilder *builder)
+{
+}
+
+static void
+garrow_uint_array_builder_class_init(GArrowUIntArrayBuilderClass *klass)
+{
+}
+
+/**
+ * garrow_uint_array_builder_new:
+ *
+ * Returns: A newly created #GArrowUIntArrayBuilder.
+ *
+ * Since: 0.8.0
+ */
+GArrowUIntArrayBuilder *
+garrow_uint_array_builder_new(void)
+{
+  auto memory_pool = arrow::default_memory_pool();
+  auto arrow_builder = new arrow::AdaptiveUIntBuilder(memory_pool);
+  auto builder = garrow_array_builder_new_raw(arrow_builder,
+                                              GARROW_TYPE_UINT_ARRAY_BUILDER);
+  return GARROW_UINT_ARRAY_BUILDER(builder);
+}
+
+/**
+ * garrow_uint_array_builder_append:
+ * @builder: A #GArrowUIntArrayBuilder.
+ * @value: A unsigned int value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint_array_builder_append(GArrowUIntArrayBuilder *builder,
+                                 guint64 value,
+                                 GError **error)
+{
+  return garrow_array_builder_append<arrow::AdaptiveUIntBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     value,
+     error,
+     "[uint-array-builder][append]");
+}
+
+/**
+ * garrow_uint_array_builder_append_values:
+ * @builder: A #GArrowUIntArrayBuilder.
+ * @values: (array length=values_length): The array of unsigned int.
+ * @values_length: The length of `values`.
+ * @is_valids: (nullable) (array length=is_valids_length): The array of
+ *   boolean that shows whether the Nth value is valid or not. If the
+ *   Nth `is_valids` is %TRUE, the Nth `values` is valid value. Otherwise
+ *   the Nth value is null value.
+ * @is_valids_length: The length of `is_valids`.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple values at once. It's efficient than multiple
+ * `append()` and `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint_array_builder_append_values(GArrowUIntArrayBuilder *builder,
+                                        const guint64 *values,
+                                        gint64 values_length,
+                                        const gboolean *is_valids,
+                                        gint64 is_valids_length,
+                                        GError **error)
+{
+  return garrow_array_builder_append_values<arrow::AdaptiveUIntBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     reinterpret_cast<const uint64_t *>(values),
+     values_length,
+     is_valids,
+     is_valids_length,
+     error,
+     "[uint-array-builder][append-values]");
+}
+
+/**
+ * garrow_uint_array_builder_append_null:
+ * @builder: A #GArrowUIntArrayBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint_array_builder_append_null(GArrowUIntArrayBuilder *builder,
+                                      GError **error)
+{
+  return garrow_array_builder_append_null<arrow::AdaptiveUIntBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     error,
+     "[uint-array-builder][append-null]");
+}
+
+/**
+ * garrow_uint_array_builder_append_nulls:
+ * @builder: A #GArrowUIntArrayBuilder.
+ * @n: The number of null values to be appended.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Append multiple nulls at once. It's efficient than multiple
+ * `append_null()` calls.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_uint_array_builder_append_nulls(GArrowUIntArrayBuilder *builder,
+                                       gint64 n,
+                                       GError **error)
+{
+  return garrow_array_builder_append_nulls<arrow::AdaptiveUIntBuilder *>
+    (GARROW_ARRAY_BUILDER(builder),
+     n,
+     error,
+     "[uint-array-builder][append-nulls]");
+}
+
+
 G_DEFINE_TYPE(GArrowInt8ArrayBuilder,
               garrow_int8_array_builder,
               GARROW_TYPE_ARRAY_BUILDER)
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index 87808549afa97..7a53bff11de1f 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -20,50 +20,16 @@
 #pragma once
 
 #include <arrow-glib/array.h>
+#include <arrow-glib/gobject-type.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_ARRAY_BUILDER               \
-  (garrow_array_builder_get_type())
-#define GARROW_ARRAY_BUILDER(obj)                               \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_ARRAY_BUILDER,        \
-                              GArrowArrayBuilder))
-#define GARROW_ARRAY_BUILDER_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_ARRAY_BUILDER,   \
-                           GArrowArrayBuilderClass))
-#define GARROW_IS_ARRAY_BUILDER(obj)            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),            \
-                              GARROW_TYPE_ARRAY_BUILDER))
-#define GARROW_IS_ARRAY_BUILDER_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_ARRAY_BUILDER))
-#define GARROW_ARRAY_BUILDER_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_ARRAY_BUILDER, \
-                             GArrowArrayBuilderClass))
-
-typedef struct _GArrowArrayBuilder         GArrowArrayBuilder;
-typedef struct _GArrowArrayBuilderClass    GArrowArrayBuilderClass;
-
-/**
- * GArrowArrayBuilder:
- *
- * It wraps `arrow::ArrayBuilder`.
- */
-struct _GArrowArrayBuilder
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowArrayBuilderClass
-{
-  GObjectClass parent_class;
-};
-
-GType               garrow_array_builder_get_type (void) G_GNUC_CONST;
+#define GARROW_TYPE_ARRAY_BUILDER (garrow_array_builder_get_type())
+GARROW_DECLARE_TYPE(GArrowArrayBuilder,
+                    garrow_array_builder,
+                    GARROW,
+                    ARRAY_BUILDER,
+                    GObject)
 
 GArrowArray        *garrow_array_builder_finish   (GArrowArrayBuilder *builder,
                                                    GError **error);
@@ -189,6 +155,31 @@ gboolean garrow_int_array_builder_append_nulls(GArrowIntArrayBuilder *builder,
                                                GError **error);
 
 
+#define GARROW_TYPE_UINT_ARRAY_BUILDER (garrow_uint_array_builder_get_type())
+GARROW_DECLARE_TYPE(GArrowUIntArrayBuilder,
+                    garrow_uint_array_builder,
+                    GARROW,
+                    UINT_ARRAY_BUILDER,
+                    GArrowArrayBuilder)
+
+GArrowUIntArrayBuilder *garrow_uint_array_builder_new(void);
+
+gboolean garrow_uint_array_builder_append(GArrowUIntArrayBuilder *builder,
+                                          guint64 value,
+                                          GError **error);
+gboolean garrow_uint_array_builder_append_values(GArrowUIntArrayBuilder *builder,
+                                                 const guint64 *values,
+                                                 gint64 values_length,
+                                                 const gboolean *is_valids,
+                                                 gint64 is_valids_length,
+                                                 GError **error);
+gboolean garrow_uint_array_builder_append_null(GArrowUIntArrayBuilder *builder,
+                                               GError **error);
+gboolean garrow_uint_array_builder_append_nulls(GArrowUIntArrayBuilder *builder,
+                                                gint64 n,
+                                                GError **error);
+
+
 #define GARROW_TYPE_INT8_ARRAY_BUILDER          \
   (garrow_int8_array_builder_get_type())
 #define GARROW_INT8_ARRAY_BUILDER(obj)                          \
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index 0ed5759412669..bdb7e11caa380 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -25,6 +25,10 @@ def build_int_array(values)
       build_array(Arrow::IntArrayBuilder.new, values)
     end
 
+    def build_uint_array(values)
+      build_array(Arrow::UIntArrayBuilder.new, values)
+    end
+
     def build_int8_array(values)
       build_array(Arrow::Int8ArrayBuilder.new, values)
     end
diff --git a/c_glib/test/test-array-builder.rb b/c_glib/test/test-array-builder.rb
index 6c0f984854d87..0c47b0a75f488 100644
--- a/c_glib/test/test-array-builder.rb
+++ b/c_glib/test/test-array-builder.rb
@@ -130,6 +130,28 @@ def sample_values
     end
   end
 
+  sub_test_case("UIntArrayBuilder") do
+    def create_builder
+      Arrow::UIntArrayBuilder.new
+    end
+
+    def builder_class_name
+      "uint-array-builder"
+    end
+
+    def sample_values
+      [1, 2, 3]
+    end
+
+    sub_test_case("#append_values") do
+      include ArrayBuilderAppendValuesTests
+    end
+
+    sub_test_case("#append_nulls") do
+      include ArrayBuilderAppendNullsTests
+    end
+  end
+
   sub_test_case("Int8ArrayBuilder") do
     def create_builder
       Arrow::Int8ArrayBuilder.new
diff --git a/c_glib/test/test-uint-array-builder.rb b/c_glib/test/test-uint-array-builder.rb
new file mode 100644
index 0000000000000..89621189b4571
--- /dev/null
+++ b/c_glib/test/test-uint-array-builder.rb
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUIntArrayBuilder < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_uint8
+    values = [0, 2]
+    assert_equal(build_uint_array([*values, nil]),
+                 Arrow::UInt8Array.new(3,
+                                       Arrow::Buffer.new(values.pack("C*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
+  def test_uint16
+    border_value = 2 ** 8
+    values = [0, border_value]
+    assert_equal(build_uint_array([*values, nil]),
+                 Arrow::UInt16Array.new(3,
+                                       Arrow::Buffer.new(values.pack("S*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
+  def test_uint32
+    border_value = 2 ** 16
+    values = [0, border_value]
+    assert_equal(build_uint_array([*values, nil]),
+                 Arrow::UInt32Array.new(3,
+                                       Arrow::Buffer.new(values.pack("L*")),
+                                       Arrow::Buffer.new([0b011].pack("C*")),
+                                       -1))
+  end
+
+  def test_uint64
+    border_value = 2 ** 32
+    values = [0, border_value]
+    assert_equal(build_uint_array([*values, nil]),
+                 Arrow::UInt64Array.new(3,
+                                        Arrow::Buffer.new(values.pack("Q*")),
+                                        Arrow::Buffer.new([0b011].pack("C*")),
+                                        -1))
+  end
+end

From b41a4ee2322d0084ff78b78ccfebc4536f7e0a62 Mon Sep 17 00:00:00 2001
From: Kentaro Hayashi <hayashi@clear-code.com>
Date: Fri, 22 Sep 2017 12:09:09 -0700
Subject: [PATCH 1082/1644] ARROW-1598: [C++] Fix diverged code comment in
 plasma tutorial

Even though fixed object id is used in implementation, comment says random object id is created.

Author: Kentaro Hayashi <hayashi@clear-code.com>

Closes #1124 from kenhys/arrow-1598 and squashes the following commits:

dc5934e [Kentaro Hayashi] ARROW-1598: [C++] Fix diverged code comment in plasma tutorial
---
 cpp/apidoc/tutorials/plasma.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/apidoc/tutorials/plasma.md b/cpp/apidoc/tutorials/plasma.md
index aed902413bb36..8d54a10215747 100644
--- a/cpp/apidoc/tutorials/plasma.md
+++ b/cpp/apidoc/tutorials/plasma.md
@@ -219,7 +219,7 @@ int main(int argc, char** argv) {
   // Start up and connect a Plasma client.
   PlasmaClient client;
   ARROW_CHECK_OK(client.Connect("/tmp/plasma", "", PLASMA_DEFAULT_RELEASE_DELAY));
-  // Create an object with a random ObjectID.
+  // Create an object with a fixed ObjectID.
   ObjectID object_id = ObjectID::from_binary("00000000000000000000");
   int64_t data_size = 1000;
   uint8_t *data;

From 096b8770a67d1911bee2d4d70653b8539bbd74c7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 25 Sep 2017 14:24:50 +0200
Subject: [PATCH 1083/1644] ARROW-1601: [C++] Do not read extra byte from
 validity bitmap, add internal::BitmapReader in lieu of macros

@xhochy since this is causing the crash reported in ARROW-1601 we may want to do a patch release 0.7.1 and parquet-cpp 1.3.1

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1126 from wesm/ARROW-1601 and squashes the following commits:

6cec81c [Wes McKinney] Fix RleDecoder logic with BitmapReader
ba58b8a [Wes McKinney] Fix test name
fa47865 [Wes McKinney] Add BitmapReader class to replace the bitset macros
---
 cpp/build-support/run_clang_format.py |  7 ++--
 cpp/src/arrow/compute/cast.cc         | 34 +++++++++----------
 cpp/src/arrow/util/bit-util-test.cc   | 21 ++++++++++++
 cpp/src/arrow/util/bit-util.h         | 49 +++++++++++++++++++++++++++
 cpp/src/arrow/util/rle-encoding.h     | 15 ++++----
 5 files changed, 98 insertions(+), 28 deletions(-)

diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py
index ac4954ca5708f..f1a448f536101 100755
--- a/cpp/build-support/run_clang_format.py
+++ b/cpp/build-support/run_clang_format.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -58,8 +58,9 @@
 # fi
 
 try:
-    subprocess.check_output([CLANG_FORMAT, '-i'] + files_to_format,
-                            stderr=subprocess.STDOUT)
+    cmd = [CLANG_FORMAT, '-i'] + files_to_format
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 except Exception as e:
     print(e)
+    print(' '.join(cmd))
     raise
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index 5283bf0a4c297..ee838fa38f416 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -261,8 +261,9 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices,
                                      const FixedSizeBinaryArray& dictionary,
                                      ArrayData* output) {
   using index_c_type = typename IndexType::c_type;
-  const uint8_t* valid_bits = indices.null_bitmap_data();
-  INIT_BITSET(valid_bits, indices.offset());
+
+  internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
+                                           indices.length());
 
   const index_c_type* in =
       reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
@@ -271,11 +272,11 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices,
   int32_t byte_width =
       static_cast<const FixedSizeBinaryType&>(*output->type).byte_width();
   for (int64_t i = 0; i < indices.length(); ++i) {
-    if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+    if (valid_bits_reader.IsSet()) {
       const uint8_t* value = dictionary.Value(in[i]);
       memcpy(out + i * byte_width, value, byte_width);
     }
-    READ_NEXT_BITSET(valid_bits);
+    valid_bits_reader.Next();
   }
 }
 
@@ -293,8 +294,7 @@ struct CastFunctor<
 
     // Check if values and output type match
     DCHECK(values_type.Equals(*output->type))
-      << "Dictionary type: " << values_type
-      << " target type: " << (*output->type);
+        << "Dictionary type: " << values_type << " target type: " << (*output->type);
 
     const Array& indices = *dict_array.indices();
     switch (indices.type()->id()) {
@@ -327,21 +327,21 @@ Status UnpackBinaryDictionary(FunctionContext* ctx, const Array& indices,
   RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), output->type, &builder));
   BinaryBuilder* binary_builder = static_cast<BinaryBuilder*>(builder.get());
 
-  const uint8_t* valid_bits = indices.null_bitmap_data();
-  INIT_BITSET(valid_bits, indices.offset());
+  internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
+                                           indices.length());
 
   const index_c_type* in =
       reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
       indices.offset();
   for (int64_t i = 0; i < indices.length(); ++i) {
-    if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+    if (valid_bits_reader.IsSet()) {
       int32_t length;
       const uint8_t* value = dictionary.GetValue(in[i], &length);
       RETURN_NOT_OK(binary_builder->Append(value, length));
     } else {
       RETURN_NOT_OK(binary_builder->AppendNull());
     }
-    READ_NEXT_BITSET(valid_bits);
+    valid_bits_reader.Next();
   }
 
   std::shared_ptr<Array> plain_array;
@@ -366,8 +366,7 @@ struct CastFunctor<T, DictionaryType,
 
     // Check if values and output type match
     DCHECK(values_type.Equals(*output->type))
-      << "Dictionary type: " << values_type
-      << " target type: " << (*output->type);
+        << "Dictionary type: " << values_type << " target type: " << (*output->type);
 
     const Array& indices = *dict_array.indices();
     switch (indices.type()->id()) {
@@ -401,17 +400,17 @@ void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
                                c_type* out) {
   using index_c_type = typename IndexType::c_type;
 
-  const uint8_t* valid_bits = indices.null_bitmap_data();
-  INIT_BITSET(valid_bits, indices.offset());
+  internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
+                                           indices.length());
 
   const index_c_type* in =
       reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
       indices.offset();
   for (int64_t i = 0; i < indices.length(); ++i) {
-    if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+    if (valid_bits_reader.IsSet()) {
       out[i] = dictionary[in[i]];
     }
-    READ_NEXT_BITSET(valid_bits);
+    valid_bits_reader.Next();
   }
 }
 
@@ -429,8 +428,7 @@ struct CastFunctor<T, DictionaryType,
 
     // Check if values and output type match
     DCHECK(values_type.Equals(*output->type))
-      << "Dictionary type: " << values_type
-      << " target type: " << (*output->type);
+        << "Dictionary type: " << values_type << " target type: " << (*output->type);
 
     auto dictionary =
         reinterpret_cast<const c_type*>(type.dictionary()->data()->buffers[1]->data()) +
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index d838ab9d7a7b1..a5c6cecc5977b 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -72,6 +72,27 @@ TEST(BitUtilTests, TestNextPower2) {
   ASSERT_EQ(1LL << 62, NextPower2((1LL << 62) - 1));
 }
 
+TEST(BitmapReader, DoesNotReadOutOfBounds) {
+  uint8_t bitmap[16] = {0};
+
+  const int length = 128;
+
+  internal::BitmapReader r1(bitmap, 0, length);
+
+  // If this were to read out of bounds, valgrind would tell us
+  for (int i = 0; i < length; ++i) {
+    ASSERT_TRUE(r1.IsNotSet());
+    r1.Next();
+  }
+
+  internal::BitmapReader r2(bitmap, 5, length - 5);
+
+  for (int i = 0; i < (length - 5); ++i) {
+    ASSERT_TRUE(r2.IsNotSet());
+    r2.Next();
+  }
+}
+
 static inline int64_t SlowCountBits(const uint8_t* data, int64_t bit_offset,
                                     int64_t length) {
   int64_t count = 0;
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index b8a8efabe5fef..fa0d7a4d566d0 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -48,6 +48,53 @@
 #endif
 
 namespace arrow {
+namespace internal {
+
+class BitmapReader {
+ public:
+  BitmapReader(const uint8_t* bitmap, int64_t start_offset, int64_t length)
+      : bitmap_(bitmap), position_(0), length_(length) {
+    byte_offset_ = start_offset / 8;
+    bit_offset_ = start_offset % 8;
+    current_byte_ = bitmap[byte_offset_];
+  }
+
+#if defined(_MSC_VER)
+  // MSVC is finicky about this cast
+  bool IsSet() const { return (current_byte_ & (1 << bit_offset_)) != 0; }
+#else
+  bool IsSet() const { return current_byte_ & (1 << bit_offset_); }
+#endif
+
+  bool IsNotSet() const { return (current_byte_ & (1 << bit_offset_)) == 0; }
+
+  void Next() {
+    ++bit_offset_;
+    ++position_;
+    if (bit_offset_ == 8) {
+      bit_offset_ = 0;
+      ++byte_offset_;
+      if (ARROW_PREDICT_TRUE(position_ < length_)) {
+        current_byte_ = bitmap_[byte_offset_];
+      }
+    }
+  }
+
+ private:
+  const uint8_t* bitmap_;
+  int64_t position_;
+  int64_t length_;
+
+  uint8_t current_byte_;
+  int64_t byte_offset_;
+  int64_t bit_offset_;
+};
+
+}  // namespace internal
+
+#ifndef ARROW_NO_DEPRECATED_API
+
+// \deprecated Since > 0.7.0
 
 #define INIT_BITSET(valid_bits_vector, valid_bits_index)            \
   int64_t byte_offset_##valid_bits_vector = (valid_bits_index) / 8; \
@@ -62,6 +109,8 @@ namespace arrow {
     bitset_##valid_bits_vector = valid_bits_vector[byte_offset_##valid_bits_vector]; \
   }
 
+#endif
+
 // TODO(wesm): The source from Impala was depending on boost::make_unsigned
 //
 // We add a partial stub implementation here
diff --git a/cpp/src/arrow/util/rle-encoding.h b/cpp/src/arrow/util/rle-encoding.h
index f4c8a772efc8c..f343b74cc788f 100644
--- a/cpp/src/arrow/util/rle-encoding.h
+++ b/cpp/src/arrow/util/rle-encoding.h
@@ -352,11 +352,12 @@ inline int RleDecoder::GetBatchWithDictSpaced(const T* dictionary, T* values,
   DCHECK_GE(bit_width_, 0);
   int values_read = 0;
   int remaining_nulls = null_count;
-  INIT_BITSET(valid_bits, static_cast<int>(valid_bits_offset));
+
+  internal::BitmapReader bit_reader(valid_bits, valid_bits_offset, batch_size);
 
   while (values_read < batch_size) {
-    bool is_valid = (bitset_valid_bits & (1 << bit_offset_valid_bits)) != 0;
-    READ_NEXT_BITSET(valid_bits);
+    bool is_valid = bit_reader.IsSet();
+    bit_reader.Next();
 
     if (is_valid) {
       if ((repeat_count_ == 0) && (literal_count_ == 0)) {
@@ -369,14 +370,14 @@ inline int RleDecoder::GetBatchWithDictSpaced(const T* dictionary, T* values,
         repeat_count_--;
 
         while (repeat_count_ > 0 && (values_read + repeat_batch) < batch_size) {
-          if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+          if (bit_reader.IsSet()) {
             repeat_count_--;
           } else {
             remaining_nulls--;
           }
           repeat_batch++;
 
-          READ_NEXT_BITSET(valid_bits);
+          bit_reader.Next();
         }
         std::fill(values + values_read, values + values_read + repeat_batch, value);
         values_read += repeat_batch;
@@ -397,7 +398,7 @@ inline int RleDecoder::GetBatchWithDictSpaced(const T* dictionary, T* values,
 
         // Read the first bitset to the end
         while (literals_read < literal_batch) {
-          if (bitset_valid_bits & (1 << bit_offset_valid_bits)) {
+          if (bit_reader.IsSet()) {
             values[values_read + literals_read + skipped] =
                 dictionary[indices[literals_read]];
             literals_read++;
@@ -405,7 +406,7 @@ inline int RleDecoder::GetBatchWithDictSpaced(const T* dictionary, T* values,
             skipped++;
           }
 
-          READ_NEXT_BITSET(valid_bits);
+          bit_reader.Next();
         }
         literal_count_ -= literal_batch;
         values_read += literal_batch + skipped;

From 39e487c3017c8150ea96c3fda1b2aab7f816b788 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 25 Sep 2017 22:59:36 -0400
Subject: [PATCH 1084/1644] ARROW-1608: Support Release verification script on
 macOS

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1129 from xhochy/ARROW-1608 and squashes the following commits:

cb768315 [Uwe L. Korn] ARROW-1608: Support Release verification script on macOS
---
 dev/release/verify-release-candidate.sh | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 9ebffdb6401ef..8d1bed75b8b05 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -66,7 +66,11 @@ fetch_archive() {
   download_rc_file ${dist_name}.tar.gz.sha512
   gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
   gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
-  sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+  if [ "$(uname)" == "Darwin" ]; then
+    shasum -a 512 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+  else
+    sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+  fi
 }
 
 setup_tempdir() {
@@ -80,7 +84,11 @@ setup_tempdir() {
 
 setup_miniconda() {
   # Setup short-lived miniconda for Python and integration tests
-  MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+  if [ "$(uname)" == "Darwin" ]; then
+    MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+  else
+    MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+  fi
 
   MINICONDA=`pwd`/test-miniconda
 
@@ -204,7 +212,11 @@ export PARQUET_HOME=$TMPDIR/install
 export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH
 export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH
 
-NPROC=$(nproc)
+if [ "$(uname)" == "Darwin" ]; then
+  NPROC=$(sysctl -n hw.ncpu)
+else
+  NPROC=$(nproc)
+fi
 VERSION=$1
 RC_NUMBER=$2
 

From b640cc0e78b41cb26e8e5d11a76d20d759654d44 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Mon, 25 Sep 2017 23:01:33 -0400
Subject: [PATCH 1085/1644] ARROW-1610: C++/Python: Only call python-prefix if
 the default PYTHON_LIBRARY is not present

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1130 from xhochy/ARROW-1610 and squashes the following commits:

8c7c190e [Korn, Uwe] ARROW-1610: C++/Python: Only call python-prefix if the default PYTHON_LIBRARY is not present
---
 cpp/cmake_modules/FindPythonLibsNew.cmake | 26 ++++++++++++-----------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/cpp/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake
index 09124aa17bb9c..40042561cccef 100644
--- a/cpp/cmake_modules/FindPythonLibsNew.cmake
+++ b/cpp/cmake_modules/FindPythonLibsNew.cmake
@@ -149,19 +149,21 @@ if(CMAKE_HOST_WIN32)
     set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/libpython${PYTHON_LIBRARY_SUFFIX}.a")
   endif()
 elseif(APPLE)
-  # In some cases libpythonX.X.dylib is not part of the PYTHON_PREFIX and we
-  # need to call `python-config --prefix` to determine the correct location.
 
-  find_program(PYTHON_CONFIG python-config
-      NO_CMAKE_SYSTEM_PATH)
-  if (PYTHON_CONFIG)
-    execute_process(
-        COMMAND "${PYTHON_CONFIG}" "--prefix"
-        OUTPUT_VARIABLE PYTHON_CONFIG_PREFIX
-        OUTPUT_STRIP_TRAILING_WHITESPACE)
-    set(PYTHON_LIBRARY "${PYTHON_CONFIG_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
-  else()
-    set(PYTHON_LIBRARY "${PYTHON_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
+  set(PYTHON_LIBRARY "${PYTHON_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
+
+  if (NOT EXISTS ${PYTHON_LIBRARY})
+    # In some cases libpythonX.X.dylib is not part of the PYTHON_PREFIX and we
+    # need to call `python-config --prefix` to determine the correct location.
+    find_program(PYTHON_CONFIG python-config
+        NO_CMAKE_SYSTEM_PATH)
+    if (PYTHON_CONFIG)
+      execute_process(
+          COMMAND "${PYTHON_CONFIG}" "--prefix"
+          OUTPUT_VARIABLE PYTHON_CONFIG_PREFIX
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+      set(PYTHON_LIBRARY "${PYTHON_CONFIG_PREFIX}/lib/libpython${PYTHON_LIBRARY_SUFFIX}.dylib")
+    endif()
   endif()
 else()
     if(${PYTHON_SIZEOF_VOID_P} MATCHES 8)

From 5da6b87950c5bc6658bc49aef29dcf9bf67e0012 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 26 Sep 2017 10:21:28 +0200
Subject: [PATCH 1086/1644] ARROW-1606: [Python] Copy .lib files in addition to
 .dll when bundling libraries for Windows

I don't have access to a Windows machine at the moment to try this out locally. I will try from wesm/arrow-dist on Appveyor in the meantime

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1127 from wesm/ARROW-1606 and squashes the following commits:

9edd85d [Wes McKinney] Bundle .lib files on Windows in CMakeLists.txt
e8f6c73 [Wes McKinney] Copy all suffixes for libraries on Windows to get .dll and .lib files
---
 python/CMakeLists.txt                | 83 +++++++++++++++-------------
 python/cmake_modules/FindArrow.cmake | 26 ++++++---
 python/doc/source/development.rst    |  3 +-
 python/setup.py                      | 81 +++++++++++++++------------
 4 files changed, 113 insertions(+), 80 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index af95073f5da35..a636d51b4209e 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -176,6 +176,14 @@ find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 
 function(bundle_arrow_lib library_path)
+  set(options)
+  set(one_value_args ABI_VERSION SO_VERSION)
+  set(multi_value_args)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
   get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY)
   get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
   configure_file(${${library_path}}
@@ -183,30 +191,45 @@ function(bundle_arrow_lib library_path)
       COPYONLY)
 
   if (APPLE)
-    configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+    configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARG_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARG_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
         COPYONLY)
-    configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+    configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARG_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARG_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
         COPYONLY)
-  else(NOT MSVC)
-    configure_file(${${library_path}}.${ARROW_ABI_VERSION}
-        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
+  elseif(NOT MSVC)
+    configure_file(${${library_path}}.${ARG_ABI_VERSION}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARG_ABI_VERSION}
         COPYONLY)
-    configure_file(${${library_path}}.${ARROW_SO_VERSION}
-        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}
+    configure_file(${${library_path}}.${ARG_SO_VERSION}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARG_SO_VERSION}
         COPYONLY)
   endif()
 
-  SET(ARROW_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 endfunction(bundle_arrow_lib)
 
+function(bundle_arrow_implib library_path)
+  get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY)
+  get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
+  configure_file(${${library_path}}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.lib
+      COPYONLY)
+endfunction(bundle_arrow_implib)
+
 if (PYARROW_BUNDLE_ARROW_CPP)
   # arrow
   file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
-  bundle_arrow_lib(ARROW_SHARED_LIB)
-  bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
+  bundle_arrow_lib(ARROW_SHARED_LIB
+    ABI_VERSION ${ARROW_ABI_VERSION}
+    SO_VERSION ${ARROW_SO_VERSION})
+  bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB
+    ABI_VERSION ${ARROW_ABI_VERSION}
+    SO_VERSION ${ARROW_SO_VERSION})
+
+  if (MSVC)
+    bundle_arrow_implib(ARROW_SHARED_IMP_LIB)
+    bundle_arrow_implib(ARROW_PYTHON_SHARED_IMP_LIB)
+  endif()
 endif()
 
 if (MSVC)
@@ -248,30 +271,12 @@ if (PYARROW_BUILD_PARQUET)
   include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
 
   if (PYARROW_BUNDLE_ARROW_CPP)
-      get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY)
-      get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE)
-      message(STATUS "Configure start")
-      configure_file(${PARQUET_SHARED_LIB}
-          ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
-          COPYONLY)
-      if (APPLE)
-          configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-              COPYONLY)
-          configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
-              COPYONLY)
-      elseif (NOT MSVC)
-          configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
-              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
-              COPYONLY)
-          configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
-              ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
-              COPYONLY)
-      endif()
-      message(STATUS "Configure end")
-      #SET(PARQUET_SHARED_LIB
-      #    ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX})
+    bundle_arrow_lib(PARQUET_SHARED_LIB
+      ABI_VERSION ${PARQUET_ABI_VERSION}
+      SO_VERSION ${PARQUET_SO_VERSION})
+    if (MSVC)
+      bundle_arrow_implib(PARQUET_SHARED_IMP_LIB)
+    endif()
   endif()
   if (MSVC)
     ADD_THIRDPARTY_LIB(parquet
@@ -301,7 +306,9 @@ if (PYARROW_BUILD_PLASMA)
     SHARED_LIB ${PLASMA_SHARED_LIB})
 
   if (PYARROW_BUNDLE_ARROW_CPP)
-    bundle_arrow_lib(PLASMA_SHARED_LIB)
+    bundle_arrow_lib(PLASMA_SHARED_LIB
+      ABI_VERSION ${ARROW_ABI_VERSION}
+      SO_VERSION ${ARROW_SO_VERSION})
   endif()
   set(LINK_LIBS
     ${LINK_LIBS}
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index c893a1aadb0e2..12f76b6c2b3e8 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -124,10 +124,22 @@ else ()
   set(ARROW_FOUND FALSE)
 endif ()
 
-mark_as_advanced(
-  ARROW_INCLUDE_DIR
-  ARROW_STATIC_LIB
-  ARROW_SHARED_LIB
-  ARROW_PYTHON_STATIC_LIB
-  ARROW_PYTHON_SHARED_LIB
-)
+if (MSVC)
+  mark_as_advanced(
+    ARROW_INCLUDE_DIR
+    ARROW_STATIC_LIB
+    ARROW_SHARED_LIB
+    ARROW_SHARED_IMP_LIB
+    ARROW_PYTHON_STATIC_LIB
+    ARROW_PYTHON_SHARED_LIB
+    ARROW_PYTHON_SHARED_IMP_LIB
+  )
+else()
+  mark_as_advanced(
+    ARROW_INCLUDE_DIR
+    ARROW_STATIC_LIB
+    ARROW_SHARED_LIB
+    ARROW_PYTHON_STATIC_LIB
+    ARROW_PYTHON_SHARED_LIB
+  )
+endif()
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 53544ba7a6ac3..2d7132303a314 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -272,7 +272,8 @@ Now, we build and install Arrow C++ libraries
    cmake -G "Visual Studio 14 2015 Win64" ^
          -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
          -DCMAKE_BUILD_TYPE=Release ^
-         -DARROW_BUILD_TESTS=off ^
+         -DARROW_BUILD_TESTS=on ^
+         -DARROW_CXXFLAGS="/WX" ^
          -DARROW_PYTHON=on ..
    cmake --build . --target INSTALL --config Release
    cd ..\..
diff --git a/python/setup.py b/python/setup.py
index 4657da0bf86f2..8966fb873a630 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -208,14 +208,6 @@ def _run_cmake(self):
 
         # Move the libraries to the place expected by the Python
         # build
-        shared_library_prefix = 'lib'
-        if sys.platform == 'darwin':
-            shared_library_suffix = '.dylib'
-        elif sys.platform == 'win32':
-            shared_library_suffix = '.dll'
-            shared_library_prefix = ''
-        else:
-            shared_library_suffix = '.so'
 
         try:
             os.makedirs(pjoin(build_lib, 'pyarrow'))
@@ -227,40 +219,18 @@ def _run_cmake(self):
         else:
             build_prefix = self.build_type
 
-        def move_lib(lib_name):
-            lib_filename = (shared_library_prefix + lib_name +
-                            shared_library_suffix)
-            # Also copy libraries with ABI/SO version suffix
-            if sys.platform == 'darwin':
-                lib_pattern = (shared_library_prefix + lib_name +
-                               ".*" + shared_library_suffix[1:])
-                libs = glob.glob(pjoin(build_prefix, lib_pattern))
-            else:
-                libs = glob.glob(pjoin(build_prefix, lib_filename) + '*')
-            # Longest suffix library should be copied, all others symlinked
-            libs.sort(key=lambda s: -len(s))
-            print(libs, libs[0])
-            lib_filename = os.path.basename(libs[0])
-            shutil.move(pjoin(build_prefix, lib_filename),
-                        pjoin(build_lib, 'pyarrow', lib_filename))
-            for lib in libs[1:]:
-                filename = os.path.basename(lib)
-                link_name = pjoin(build_lib, 'pyarrow', filename)
-                if not os.path.exists(link_name):
-                    os.symlink(lib_filename, link_name)
-
         if self.bundle_arrow_cpp:
             print(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
             if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
                 shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
             shutil.move(pjoin(build_prefix, 'include'),
                         pjoin(build_lib, 'pyarrow'))
-            move_lib("arrow")
-            move_lib("arrow_python")
+            move_shared_libs(build_prefix, build_lib, "arrow")
+            move_shared_libs(build_prefix, build_lib, "arrow_python")
             if self.with_plasma:
-                move_lib("plasma")
+                move_shared_libs(build_prefix, build_lib, "plasma")
             if self.with_parquet:
-                move_lib("parquet")
+                move_shared_libs(build_prefix, build_lib, "parquet")
 
         # Move the built C-extension to the place expected by the Python build
         self._found_names = []
@@ -344,6 +314,49 @@ def get_outputs(self):
         return [self._get_cmake_ext_path(name)
                 for name in self.get_names()]
 
+
+def move_shared_libs(build_prefix, build_lib, lib_name):
+    if sys.platform == 'win32':
+        # Move all .dll and .lib files
+        libs = glob.glob(pjoin(build_prefix, lib_name) + '*')
+
+        for filename in libs:
+            shutil.move(pjoin(build_prefix, filename),
+                        pjoin(build_lib, 'pyarrow', filename))
+    else:
+        _move_shared_libs_unix(build_prefix, build_lib, lib_name)
+
+
+def _move_shared_libs_unix(build_prefix, build_lib, lib_name):
+    shared_library_prefix = 'lib'
+    if sys.platform == 'darwin':
+        shared_library_suffix = '.dylib'
+    else:
+        shared_library_suffix = '.so'
+
+    lib_filename = (shared_library_prefix + lib_name +
+                    shared_library_suffix)
+    # Also copy libraries with ABI/SO version suffix
+    if sys.platform == 'darwin':
+        lib_pattern = (shared_library_prefix + lib_name +
+                       ".*" + shared_library_suffix[1:])
+        libs = glob.glob(pjoin(build_prefix, lib_pattern))
+    else:
+        libs = glob.glob(pjoin(build_prefix, lib_filename) + '*')
+
+    # Longest suffix library should be copied, all others symlinked
+    libs.sort(key=lambda s: -len(s))
+    print(libs, libs[0])
+    lib_filename = os.path.basename(libs[0])
+    shutil.move(pjoin(build_prefix, lib_filename),
+                pjoin(build_lib, 'pyarrow', lib_filename))
+    for lib in libs[1:]:
+        filename = os.path.basename(lib)
+        link_name = pjoin(build_lib, 'pyarrow', filename)
+        if not os.path.exists(link_name):
+            os.symlink(lib_filename, link_name)
+
+
 # In the case of a git-archive, we don't have any version information
 # from the SCM to infer a version. The only source is the java/pom.xml.
 #

From f9d1e1be756636cb8a280ac3723ed3dea2abe204 Mon Sep 17 00:00:00 2001
From: Rene Sugar <rene.sugar@gmail.com>
Date: Tue, 26 Sep 2017 21:56:33 -0400
Subject: [PATCH 1087/1644] ARROW-1611: [C++] Add BitmapWriter, do not perform
 out of bounds read in BitmapReader when length is 0

close #1133
close #1131

Author: Rene Sugar <rene.sugar@gmail.com>

Closes #1137 from wesm/ARROW-1611 and squashes the following commits:

5bff0bd [Rene Sugar] ARROW-1611 Do not read bitmap when length is zero, add internal::BitmapWriter
---
 cpp/src/arrow/util/bit-util-test.cc |  36 +++++++
 cpp/src/arrow/util/bit-util.h       | 139 +++++++++++++++++++---------
 2 files changed, 132 insertions(+), 43 deletions(-)

diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index a5c6cecc5977b..5a66d7e85ad5a 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -91,6 +91,42 @@ TEST(BitmapReader, DoesNotReadOutOfBounds) {
     ASSERT_TRUE(r2.IsNotSet());
     r2.Next();
   }
+
+  // Does not access invalid memory
+  internal::BitmapReader r3(nullptr, 0, 0);
+}
+
+TEST(BitmapWriter, DoesNotWriteOutOfBounds) {
+  uint8_t bitmap[16] = {0};
+
+  const int length = 128;
+
+  int64_t num_values = 0;
+
+  internal::BitmapWriter r1(bitmap, 0, length);
+
+  // If this were to write out of bounds, valgrind would tell us
+  for (int i = 0; i < length; ++i) {
+    r1.Set();
+    r1.Clear();
+    r1.Next();
+  }
+  r1.Finish();
+  num_values = r1.position();
+
+  ASSERT_EQ(length, num_values);
+
+  internal::BitmapWriter r2(bitmap, 5, length - 5);
+
+  for (int i = 0; i < (length - 5); ++i) {
+    r2.Set();
+    r2.Clear();
+    r2.Next();
+  }
+  r2.Finish();
+  num_values = r2.position();
+
+  ASSERT_EQ((length - 5), num_values);
 }
 
 static inline int64_t SlowCountBits(const uint8_t* data, int64_t bit_offset,
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index fa0d7a4d566d0..a85aff7c00d90 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -48,49 +48,6 @@
 #endif
 
 namespace arrow {
-namespace internal {
-
-class BitmapReader {
- public:
-  BitmapReader(const uint8_t* bitmap, int64_t start_offset, int64_t length)
-      : bitmap_(bitmap), position_(0), length_(length) {
-    byte_offset_ = start_offset / 8;
-    bit_offset_ = start_offset % 8;
-    current_byte_ = bitmap[byte_offset_];
-  }
-
-#if defined(_MSC_VER)
-  // MSVC is finicky about this cast
-  bool IsSet() const { return (current_byte_ & (1 << bit_offset_)) != 0; }
-#else
-  bool IsSet() const { return current_byte_ & (1 << bit_offset_); }
-#endif
-
-  bool IsNotSet() const { return (current_byte_ & (1 << bit_offset_)) == 0; }
-
-  void Next() {
-    ++bit_offset_;
-    ++position_;
-    if (bit_offset_ == 8) {
-      bit_offset_ = 0;
-      ++byte_offset_;
-      if (ARROW_PREDICT_TRUE(position_ < length_)) {
-        current_byte_ = bitmap_[byte_offset_];
-      }
-    }
-  }
-
- private:
-  const uint8_t* bitmap_;
-  int64_t position_;
-  int64_t length_;
-
-  uint8_t current_byte_;
-  int64_t byte_offset_;
-  int64_t bit_offset_;
-};
-
-}  // namespace internal
 
 #ifndef ARROW_NO_DEPRECATED_API
 
@@ -436,6 +393,102 @@ Status BytesToBits(const std::vector<uint8_t>&, MemoryPool*, std::shared_ptr<Buf
 
 }  // namespace BitUtil
 
+namespace internal {
+
+class BitmapReader {
+ public:
+  BitmapReader(const uint8_t* bitmap, int64_t start_offset, int64_t length)
+      : bitmap_(bitmap), position_(0), length_(length) {
+    current_byte_ = 0;
+    byte_offset_ = start_offset / 8;
+    bit_offset_ = start_offset % 8;
+    if (length > 0) {
+      current_byte_ = bitmap[byte_offset_];
+    }
+  }
+
+#if defined(_MSC_VER)
+  // MSVC is finicky about this cast
+  bool IsSet() const { return (current_byte_ & (1 << bit_offset_)) != 0; }
+#else
+  bool IsSet() const { return current_byte_ & (1 << bit_offset_); }
+#endif
+
+  bool IsNotSet() const { return (current_byte_ & (1 << bit_offset_)) == 0; }
+
+  void Next() {
+    ++bit_offset_;
+    ++position_;
+    if (bit_offset_ == 8) {
+      bit_offset_ = 0;
+      ++byte_offset_;
+      if (ARROW_PREDICT_TRUE(position_ < length_)) {
+        current_byte_ = bitmap_[byte_offset_];
+      }
+    }
+  }
+
+ private:
+  const uint8_t* bitmap_;
+  int64_t position_;
+  int64_t length_;
+
+  uint8_t current_byte_;
+  int64_t byte_offset_;
+  int64_t bit_offset_;
+};
+
+class BitmapWriter {
+ public:
+  BitmapWriter(uint8_t* bitmap, int64_t start_offset, int64_t length)
+      : bitmap_(bitmap), position_(0), length_(length) {
+    current_byte_ = 0;
+    byte_offset_ = start_offset / 8;
+    bit_offset_ = start_offset % 8;
+    if (length > 0) {
+      current_byte_ = bitmap[byte_offset_];
+    }
+  }
+
+  void Set() { current_byte_ |= BitUtil::kBitmask[bit_offset_]; }
+
+  void Clear() { current_byte_ &= BitUtil::kFlippedBitmask[bit_offset_]; }
+
+  void Next() {
+    ++bit_offset_;
+    ++position_;
+    bitmap_[byte_offset_] = current_byte_;
+    if (bit_offset_ == 8) {
+      bit_offset_ = 0;
+      ++byte_offset_;
+      if (ARROW_PREDICT_TRUE(position_ < length_)) {
+        current_byte_ = bitmap_[byte_offset_];
+      }
+    }
+  }
+
+  void Finish() {
+    if (ARROW_PREDICT_TRUE(position_ < length_)) {
+      if (bit_offset_ != 0) {
+        bitmap_[byte_offset_] = current_byte_;
+      }
+    }
+  }
+
+  int64_t position() const { return position_; }
+
+ private:
+  uint8_t* bitmap_;
+  int64_t position_;
+  int64_t length_;
+
+  uint8_t current_byte_;
+  int64_t byte_offset_;
+  int64_t bit_offset_;
+};
+
+}  // namespace internal
+
 // ----------------------------------------------------------------------
 // Bitmap utilities
 

From 808a1433005ce5325ba69b1a65d05e1b547eea2c Mon Sep 17 00:00:00 2001
From: m-nakamura145 <masato.nakamura145@gmail.com>
Date: Tue, 26 Sep 2017 22:21:47 -0400
Subject: [PATCH 1088/1644] ARROW-1612:[GLib] Update readme for mac os

I tried to build Arrow GLib released source archive on mac os. but build failed.
Accutally, We need to install `gobject-introspection` and set `PKG_CONFIG_PATH ` before build Arrow Glib.
so, I added some commands to README.md.

Author: m-nakamura145 <masato.nakamura145@gmail.com>

Closes #1134 from m-nakamura145/update_c_glib_readme_for_mac_os and squashes the following commits:

3bd886b [m-nakamura145] ARROW-1612:[GLib] Fix PKG_CONFIG_PATH
bbd95d4 [m-nakamura145] ARROW-1612:[GLib] Update readme for mac os
---
 c_glib/README.md | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/c_glib/README.md b/c_glib/README.md
index 2a9d5d64ce2cd..1f67d7ea52ccb 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -71,7 +71,19 @@ GLib (replace the version number in the following commands with the one you use)
 You need to build and install Arrow C++ before you build and install
 Arrow GLib. See Arrow C++ document about how to install Arrow C++.
 
-You can build and install Arrow GLib after you install Arrow C++:
+You can build and install Arrow GLib after you install Arrow C++.
+
+If you use macOS with [Homebrew](https://brew.sh/), you must install `gobject-introspection` and set `PKG_CONFIG_PATH` before build Arrow GLib:
+
+```text
+% cd c_glib
+% brew install -y gobject-introspection
+% ./configure PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig:$PKG_CONFIG_PATH
+% make
+% sudo make install
+```
+
+Others:
 
 ```text
 % cd c_glib

From 3a53f93157f72e04739ff8ea14b3afcb276ebaa4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 26 Sep 2017 23:34:39 -0400
Subject: [PATCH 1089/1644] [Release] Update version to 0.7.1-SNAPSHOT

Change-Id: I66fdf1e7e9aeb8cc4fffe72c60802087a8623c3e
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 2 +-
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 684ed3c525454..5f27a05b3c24c 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.7.0</version>
+  <version>0.7.1-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index dcd3572f7bf06..6642232d2d52e 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.7.0</version>
+    <version>0.7.1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index acd562625faa5..5459071aa0fa3 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.7.0</version>
+  <version>0.7.1-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index a1a63082d1e6c..a0268825fb9bd 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.7.0</version>
+        <version>0.7.1-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index dd1a0ea55d92f..3e2faf9fd84c7 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.7.0</version>
+    <version>0.7.1-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From fbabde5b57224c90b0929d0c041869dc671d5007 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 27 Sep 2017 09:16:09 -0400
Subject: [PATCH 1090/1644] [Release] Update CHANGELOG.md for 0.7.1

Change-Id: Ia63c021f7727fd7c4b849d693f883564dbf22ee1
---
 CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4858df8854181..485afc1497ebe 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,47 @@
   under the License.
 -->
 
+# Apache Arrow 0.7.1 (27 September 2017)
+
+## Bug
+
+* ARROW-1497 - [Java] JsonFileReader doesn't set value count for some vectors
+* ARROW-1500 - [C++] Result of ftruncate ignored in MemoryMappedFile::Create
+* ARROW-1536 - [C++] Do not transitively depend on libboost\_system
+* ARROW-1542 - [C++] Windows release verification script should not modify conda environment
+* ARROW-1544 - [JS] Export Vector type definitions
+* ARROW-1545 - Int64Builder should not need int64() as arg
+* ARROW-1550 - [Python] Fix flaky test on Windows 
+* ARROW-1554 - [Python] Document that pip wheels depend on MSVC14 runtime
+* ARROW-1557 - [PYTHON] pyarrow.Table.from\_arrays doesn't validate names length
+* ARROW-1591 - C++: Xcode 9 is not correctly detected
+* ARROW-1595 - [Python] Fix package dependency issues causing build failures
+* ARROW-1601 - [C++] READ\_NEXT\_BITSET reads one byte past the last byte on last iteration
+* ARROW-1606 - Python: Windows wheels don't include .lib files.
+* ARROW-1610 - C++/Python: Only call python-prefix if the default PYTHON\_LIBRARY is not present
+* ARROW-1611 - Crash in BitmapReader when length is zero
+
+## Improvement
+
+* ARROW-1537 - [C++] Support building with full path install\_name on macOS
+* ARROW-1546 - [GLib] Support GLib 2.40 again
+* ARROW-1578 - [C++/Python] Run lint checks in Travis CI to fail for linting issues as early as possible
+* ARROW-1608 - Support Release verification script on macOS
+* ARROW-1612 - [GLib] add how to install for mac os to README
+
+## New Feature
+
+* ARROW-1548 - [GLib] Support build append in builder
+* ARROW-1592 - [GLib] Add GArrowUIntArrayBuilder
+
+## Test
+
+* ARROW-1529 - [GLib] Fix failure on macOS on Travis CI
+
+## Wish
+
+* ARROW-559 - Script to easily verify release in all languages
+
 # Apache Arrow 0.7.0 (12 September 2017)
 
 ## Bug
@@ -1244,3 +1285,4 @@
 * ARROW-260 - TestValueVector.testFixedVectorReallocation and testVariableVectorReallocation are flaky
 * ARROW-83 - Add basic test infrastructure for DecimalType
 
+

From 0e21f84c2fc26dba949a03ee7d7ebfade0a65b81 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 27 Sep 2017 09:19:05 -0400
Subject: [PATCH 1091/1644] [maven-release-plugin] prepare release
 apache-arrow-0.7.1

Change-Id: I259bdeee24cca1a01d6bcb928fe2529d3cf6cd16
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 5f27a05b3c24c..12f67ad9791c5 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.7.1-SNAPSHOT</version>
+  <version>0.7.1</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 6642232d2d52e..40671fff46c73 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.7.1-SNAPSHOT</version>
+    <version>0.7.1</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 5459071aa0fa3..354a611c8c601 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.7.1-SNAPSHOT</version>
+  <version>0.7.1</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -44,7 +44,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.7.0</tag>
+    <tag>apache-arrow-0.7.1</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index a0268825fb9bd..cadc5c0d4a1e2 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.7.1-SNAPSHOT</version>
+        <version>0.7.1</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 3e2faf9fd84c7..f6bba2e350f37 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.7.1-SNAPSHOT</version>
+    <version>0.7.1</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 686a8f7937604bc84eabde64847ae73a984067f2 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Wed, 27 Sep 2017 20:58:08 -0400
Subject: [PATCH 1092/1644] ARROW-1607: [C++] Implement DictionaryBuilder for
 Decimals

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1128 from cpcloud/ARROW-1607 and squashes the following commits:

3137566 [Phillip Cloud] ARROW-1607: [C++] Implement DictionaryBuilder for Decimals
---
 cpp/src/arrow/array-test.cc      | 92 ++++++++++++++++++++++++++++++--
 cpp/src/arrow/builder.cc         | 11 ++--
 cpp/src/arrow/builder.h          |  4 +-
 cpp/src/arrow/compute/cast.cc    |  6 ++-
 cpp/src/arrow/io/io-file-test.cc |  8 +--
 cpp/src/arrow/io/io-hdfs-test.cc |  2 +-
 6 files changed, 107 insertions(+), 16 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 97310830ddc7c..4ea9248f3e4d2 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1200,7 +1200,7 @@ class TestFWBinaryArray : public ::testing::Test {
 };
 
 TEST_F(TestFWBinaryArray, Builder) {
-  const int32_t byte_width = 10;
+  constexpr int32_t byte_width = 10;
   int64_t length = 4096;
 
   int64_t nbytes = length * byte_width;
@@ -1215,8 +1215,7 @@ TEST_F(TestFWBinaryArray, Builder) {
 
   std::shared_ptr<Array> result;
 
-  auto CheckResult = [this, &length, &is_valid, &raw_data,
-                      &byte_width](const Array& result) {
+  auto CheckResult = [&length, &is_valid, &raw_data, byte_width](const Array& result) {
     // Verify output
     const auto& fw_result = static_cast<const FixedSizeBinaryArray&>(result);
 
@@ -1847,6 +1846,93 @@ TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) {
   ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array));
 }
 
+TEST(TestDecimalDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  const auto& decimal_type = arrow::decimal(2, 0);
+  DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type, default_memory_pool());
+
+  // Test data
+  std::vector<Decimal128> test{12, 12, 11, 12};
+  for (const auto& value : test) {
+    ASSERT_OK(builder.Append(value.ToBytes().data()));
+  }
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  FixedSizeBinaryBuilder decimal_builder(decimal_type);
+  ASSERT_OK(decimal_builder.Append(Decimal128(12).ToBytes()));
+  ASSERT_OK(decimal_builder.Append(Decimal128(11).ToBytes()));
+
+  std::shared_ptr<Array> decimal_array;
+  ASSERT_OK(decimal_builder.Finish(&decimal_array));
+  auto dtype = arrow::dictionary(int8(), decimal_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append({0, 0, 1, 0}));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
+  const auto& decimal_type = arrow::decimal(21, 0);
+
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type, default_memory_pool());
+
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(decimal_type);
+  Int16Builder int_builder;
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    const uint8_t bytes[] = {0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             12,
+                             12,
+                             static_cast<uint8_t>(i / 128),
+                             static_cast<uint8_t>(i % 128)};
+    ASSERT_OK(builder.Append(bytes));
+    ASSERT_OK(fsb_builder.Append(bytes));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  const uint8_t known_value[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1};
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append(known_value));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+
+  auto dtype = std::make_shared<DictionaryType>(int16(), fsb_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
 // ----------------------------------------------------------------------
 // List tests
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index a194ab7457225..0479dc555924b 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -831,11 +831,11 @@ DictionaryBuilder<FixedSizeBinaryType>::DictionaryBuilder(
       hash_table_(new PoolBuffer(pool)),
       hash_slots_(nullptr),
       dict_builder_(type, pool),
-      values_builder_(pool) {
+      values_builder_(pool),
+      byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {
   if (!::arrow::CpuInfo::initialized()) {
     ::arrow::CpuInfo::Init();
   }
-  byte_width_ = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
 }
 
 #ifndef ARROW_NO_DEPRECATED_API
@@ -921,7 +921,7 @@ Status DictionaryBuilder<T>::Append(const Scalar& value) {
 
 template <typename T>
 Status DictionaryBuilder<T>::AppendArray(const Array& array) {
-  const NumericArray<T>& numeric_array = static_cast<const NumericArray<T>&>(array);
+  const auto& numeric_array = static_cast<const NumericArray<T>&>(array);
   for (int64_t i = 0; i < array.length(); i++) {
     if (array.IsNull(i)) {
       RETURN_NOT_OK(AppendNull());
@@ -938,8 +938,7 @@ Status DictionaryBuilder<FixedSizeBinaryType>::AppendArray(const Array& array) {
     return Status::Invalid("Cannot append FixedSizeBinary array with non-matching type");
   }
 
-  const FixedSizeBinaryArray& numeric_array =
-      static_cast<const FixedSizeBinaryArray&>(array);
+  const auto& numeric_array = static_cast<const FixedSizeBinaryArray&>(array);
   for (int64_t i = 0; i < array.length(); i++) {
     if (array.IsNull(i)) {
       RETURN_NOT_OK(AppendNull());
@@ -1493,6 +1492,7 @@ Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>&
     DICTIONARY_BUILDER_CASE(STRING, StringDictionaryBuilder);
     DICTIONARY_BUILDER_CASE(BINARY, BinaryDictionaryBuilder);
     DICTIONARY_BUILDER_CASE(FIXED_SIZE_BINARY, DictionaryBuilder<FixedSizeBinaryType>);
+    DICTIONARY_BUILDER_CASE(DECIMAL, DictionaryBuilder<FixedSizeBinaryType>);
     default:
       return Status::NotImplemented(type->ToString());
   }
@@ -1528,6 +1528,7 @@ Status EncodeArrayToDictionary(const Array& input, MemoryPool* pool,
     DICTIONARY_ARRAY_CASE(STRING, StringDictionaryBuilder);
     DICTIONARY_ARRAY_CASE(BINARY, BinaryDictionaryBuilder);
     DICTIONARY_ARRAY_CASE(FIXED_SIZE_BINARY, DictionaryBuilder<FixedSizeBinaryType>);
+    DICTIONARY_ARRAY_CASE(DECIMAL, DictionaryBuilder<FixedSizeBinaryType>);
     default:
       std::stringstream ss;
       ss << "Cannot encode array of type " << type->ToString();
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 28f3cb9723236..da7386aafefc4 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -937,8 +937,8 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
 
 class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
  public:
-  using DictionaryBuilder::DictionaryBuilder;
   using DictionaryBuilder::Append;
+  using DictionaryBuilder::DictionaryBuilder;
 
   Status Append(const uint8_t* value, int32_t length) {
     return Append(internal::WrappedBinary(value, length));
@@ -958,8 +958,8 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 /// \brief Dictionary array builder with convenience methods for strings
 class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
  public:
-  using DictionaryBuilder::DictionaryBuilder;
   using DictionaryBuilder::Append;
+  using DictionaryBuilder::DictionaryBuilder;
 
   Status Append(const uint8_t* value, int32_t length) {
     return Append(internal::WrappedBinary(value, length));
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index ee838fa38f416..149cc36b61597 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -490,7 +490,10 @@ static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input
   if (can_pre_allocate_values) {
     std::shared_ptr<Buffer> out_data;
 
-    if (!(is_primitive(out->type->id()) || out->type->id() == Type::FIXED_SIZE_BINARY)) {
+    const Type::type type_id = out->type->id();
+
+    if (!(is_primitive(type_id) || type_id == Type::FIXED_SIZE_BINARY ||
+          type_id == Type::DECIMAL)) {
       std::stringstream ss;
       ss << "Cannot pre-allocate memory for type: " << out->type->ToString();
       return Status::NotImplemented(ss.str());
@@ -614,6 +617,7 @@ class CastKernel : public UnaryKernel {
   FN(IN_TYPE, FloatType);             \
   FN(IN_TYPE, DoubleType);            \
   FN(IN_TYPE, FixedSizeBinaryType);   \
+  FN(IN_TYPE, DecimalType);           \
   FN(IN_TYPE, BinaryType);            \
   FN(IN_TYPE, StringType);
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 636fbd842d049..ee3beabd9a80e 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -393,9 +393,9 @@ TEST_F(TestReadableFile, ThreadSafety) {
   ASSERT_OK(ReadableFile::Open(path_, &pool, &file_));
 
   std::atomic<int> correct_count(0);
-  const int niter = 10000;
+  constexpr int niter = 10000;
 
-  auto ReadData = [&correct_count, &data, niter, this]() {
+  auto ReadData = [&correct_count, &data, this, niter]() {
     std::shared_ptr<Buffer> buffer;
 
     for (int i = 0; i < niter; ++i) {
@@ -587,9 +587,9 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) {
                         static_cast<int64_t>(data.size())));
 
   std::atomic<int> correct_count(0);
-  const int niter = 10000;
+  constexpr int niter = 10000;
 
-  auto ReadData = [&correct_count, &data, niter, &file]() {
+  auto ReadData = [&correct_count, &data, &file, niter]() {
     std::shared_ptr<Buffer> buffer;
 
     for (int i = 0; i < niter; ++i) {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index eaf638f200251..5305b4774624d 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -437,7 +437,7 @@ TYPED_TEST(TestHadoopFileSystem, ThreadSafety) {
   ASSERT_OK(this->client_->OpenReadable(src_path, &file));
 
   std::atomic<int> correct_count(0);
-  const int niter = 1000;
+  constexpr int niter = 1000;
 
   auto ReadData = [&file, &correct_count, &data, niter]() {
     for (int i = 0; i < niter; ++i) {

From c35815474ef6f1674fb1f5fcba0cda7986edeffb Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Thu, 28 Sep 2017 15:46:02 +0200
Subject: [PATCH 1093/1644] ARROW-1609: [Plasma] Xcode 9 compilation workaround

see https://github.com/apache/arrow/pull/1139

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1144 from pcmoritz/plasma-xcode-9-workaround and squashes the following commits:

7642499 [Philipp Moritz] fix on other platforms
c5cdddd [Philipp Moritz] xcode 9 compilation workaround
---
 cpp/src/plasma/CMakeLists.txt |  1 +
 cpp/src/plasma/common.h       |  2 ++
 cpp/src/plasma/compat.h       | 35 +++++++++++++++++++++++++++++++++++
 cpp/src/plasma/io.h           |  1 +
 cpp/src/plasma/plasma.h       |  2 ++
 5 files changed, 41 insertions(+)
 create mode 100644 cpp/src/plasma/compat.h

diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 7e91202623e24..ed1762b32e5f4 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -111,6 +111,7 @@ target_link_libraries(plasma_store plasma_static)
 install(FILES
   common.h
   common_generated.h
+  compat.h
   client.h
   events.h
   plasma.h
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
index 66d5f3069d0dd..cc67ffe4686be 100644
--- a/cpp/src/plasma/common.h
+++ b/cpp/src/plasma/common.h
@@ -26,6 +26,8 @@
 #define __STDC_FORMAT_MACROS
 #endif
 
+#include "plasma/compat.h"
+
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 
diff --git a/cpp/src/plasma/compat.h b/cpp/src/plasma/compat.h
new file mode 100644
index 0000000000000..ce751da1d710f
--- /dev/null
+++ b/cpp/src/plasma/compat.h
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_COMPAT_H
+#define PLASMA_COMPAT_H
+
+// Workaround for multithreading on XCode 9, see
+// https://issues.apache.org/jira/browse/ARROW-1622 and
+// https://github.com/tensorflow/tensorflow/issues/13220#issuecomment-331579775
+// This should be a short-term fix until the problem is fixed upstream.
+#ifdef __APPLE__
+#ifndef _MACH_PORT_T
+#define _MACH_PORT_T
+#include <sys/_types.h> /* __darwin_mach_port_t */
+typedef __darwin_mach_port_t mach_port_t;
+#include <pthread.h>
+mach_port_t pthread_mach_thread_np(pthread_t);
+#endif /* _MACH_PORT_T */
+#endif /* __APPLE__ */
+
+#endif  // PLASMA_COMPAT_H
diff --git a/cpp/src/plasma/io.h b/cpp/src/plasma/io.h
index ef96c06ccea57..4beb1346be42a 100644
--- a/cpp/src/plasma/io.h
+++ b/cpp/src/plasma/io.h
@@ -27,6 +27,7 @@
 #include <vector>
 
 #include "arrow/status.h"
+#include "plasma/compat.h"
 
 // TODO(pcm): Replace our own custom message header (message type,
 // message length, plasma protocol verion) with one that is serialized
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
index 476002f68c05f..603ff8a4fac6c 100644
--- a/cpp/src/plasma/plasma.h
+++ b/cpp/src/plasma/plasma.h
@@ -31,6 +31,8 @@
 #include <unordered_map>
 #include <unordered_set>
 
+#include "plasma/compat.h"
+
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
 #include "plasma/common.h"

From bdfa65e3862fd674c6900b5c65230177185fe8a6 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 28 Sep 2017 15:47:11 +0200
Subject: [PATCH 1094/1644] ARROW-1620: Python: Download Boost in manylinux1
 build from bintray

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1141 from xhochy/ARROW-1620 and squashes the following commits:

30da182 [Uwe L. Korn] ARROW-1620: Python: Download Boost in manylinux1 build from bintray
---
 python/manylinux1/scripts/build_boost.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh
index 3c11f3aeb6f7e..4650cde9532ea 100755
--- a/python/manylinux1/scripts/build_boost.sh
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -16,10 +16,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
-tar xf boost_1_60_0.tar.gz
-pushd /boost_1_60_0
+BOOST_VERSION=1.65.1
+BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_}
+
+wget --no-check-certificate https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz -O /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
+tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
+pushd /boost_${BOOST_VERSION_UNDERSCORE}
 ./bootstrap.sh
 ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install
 popd
-rm -rf boost_1_60_0.tar.gz boost_1_60_0
+rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE}

From ac997fb5cc132f7ef5428249f15c41fe1a25453b Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Fri, 29 Sep 2017 08:58:09 -0500
Subject: [PATCH 1095/1644] ARROW-1618: [JAVA] Reduce Heap Usage (Phase 1)

cc @jacques-n , @icexelloss , @BryanCutler

This is initial small phase of our attempt to reduce heap usage per vector.

As part of investigation, we realized its better to address few things as part of subtasks for ARROW-1463. Accordingly, I need to update the requirements document w.r.t heap usage for ARROW-1471.

This patch gets rid of Release Listener object in Allocation Manager as all the logic is implemented as part of AllocationManager itself.

https://docs.google.com/document/d/1MU-ah_bBHIxXNrd7SkwewGCOOexkXJ7cgKaCis5f-PI/edit

Author: siddharth <siddharth@dremio.com>

Closes #1142 from siddharthteotia/ARROW-1618 and squashes the following commits:

77151a27 [siddharth] ARROW-1618: Reduce Heap Usage (Phase 1)
---
 .../arrow/memory/AllocationManager.java       | 81 +++++++++----------
 1 file changed, 37 insertions(+), 44 deletions(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index c528937bfdcaa..6877c18f624ee 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -140,7 +140,7 @@ private BufferLedger associate(final BaseAllocator allocator, final boolean reta
         return existingLedger;
       }
 
-      final BufferLedger ledger = new BufferLedger(allocator, new ReleaseListener(allocator));
+      final BufferLedger ledger = new BufferLedger(allocator);
       if (retain) {
         ledger.inc();
       }
@@ -151,54 +151,41 @@ private BufferLedger associate(final BaseAllocator allocator, final boolean reta
     }
   }
 
-
   /**
    * The way that a particular BufferLedger communicates back to the AllocationManager that it
    * now longer needs to hold
    * a reference to particular piece of memory.
+   * Can only be called when you already hold the writeLock.
    */
-  private class ReleaseListener {
-
-    private final BufferAllocator allocator;
-
-    public ReleaseListener(BufferAllocator allocator) {
-      this.allocator = allocator;
-    }
-
-    /**
-     * Can only be called when you already hold the writeLock.
-     */
-    public void release() {
-      allocator.assertOpen();
+  private void release(final BufferLedger ledger) {
+    final BaseAllocator allocator = ledger.getAllocator();
+    allocator.assertOpen();
 
-      final BufferLedger oldLedger = map.remove(allocator);
-      oldLedger.allocator.dissociateLedger(oldLedger);
+    final BufferLedger oldLedger = map.remove(allocator);
+    oldLedger.allocator.dissociateLedger(oldLedger);
 
-      if (oldLedger == owningLedger) {
-        if (map.isEmpty()) {
-          // no one else owns, lets release.
-          oldLedger.allocator.releaseBytes(size);
-          underlying.release();
-          amDestructionTime = System.nanoTime();
-          owningLedger = null;
-        } else {
-          // we need to change the owning allocator. we've been removed so we'll get whatever is
-          // top of list
-          BufferLedger newLedger = map.values().iterator().next();
-
-          // we'll forcefully transfer the ownership and not worry about whether we exceeded the
-          // limit
-          // since this consumer can't do anything with this.
-          oldLedger.transferBalance(newLedger);
-        }
+    if (oldLedger == owningLedger) {
+      if (map.isEmpty()) {
+        // no one else owns, lets release.
+        oldLedger.allocator.releaseBytes(size);
+        underlying.release();
+        amDestructionTime = System.nanoTime();
+        owningLedger = null;
       } else {
-        if (map.isEmpty()) {
-          throw new IllegalStateException("The final removal of a ledger should be connected to " +
-              "the owning ledger.");
-        }
+        // we need to change the owning allocator. we've been removed so we'll get whatever is
+        // top of list
+        BufferLedger newLedger = map.values().iterator().next();
+
+        // we'll forcefully transfer the ownership and not worry about whether we exceeded the
+        // limit
+        // since this consumer can't do anything with this.
+        oldLedger.transferBalance(newLedger);
+      }
+    } else {
+      if (map.isEmpty()) {
+        throw new IllegalStateException("The final removal of a ledger should be connected to " +
+                "the owning ledger.");
       }
-
-
     }
   }
 
@@ -221,16 +208,22 @@ public class BufferLedger {
     // correctly
     private final long lCreationTime = System.nanoTime();
     private final BaseAllocator allocator;
-    private final ReleaseListener listener;
     private final HistoricalLog historicalLog = BaseAllocator.DEBUG ? new HistoricalLog
         (BaseAllocator.DEBUG_LOG_LENGTH,
             "BufferLedger[%d]", 1)
         : null;
     private volatile long lDestructionTime = 0;
 
-    private BufferLedger(BaseAllocator allocator, ReleaseListener listener) {
+    private BufferLedger(BaseAllocator allocator) {
       this.allocator = allocator;
-      this.listener = listener;
+    }
+
+    /**
+     * Get the allocator for this ledger
+     * @return allocator
+     */
+    private BaseAllocator getAllocator() {
+      return allocator;
     }
 
     /**
@@ -340,7 +333,7 @@ public int decrement(int decrement) {
         outcome = bufRefCnt.addAndGet(-decrement);
         if (outcome == 0) {
           lDestructionTime = System.nanoTime();
-          listener.release();
+          release(this);
         }
       }
 

From 545496c8346397cd73f9f161f658c0e5e0e5b583 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Fri, 29 Sep 2017 10:31:33 -0500
Subject: [PATCH 1096/1644] ARROW-1619: [Java] Set lastSet in JsonFileReader

When reading a vector in JsonFileReader, lastSet should be set in VariableWidthVectors after reading inner vectors or else subsequent operations could corrupt the offsets.  This also allows to simplify some of the related code.  Additionally, ListVector.lastSet should be explicitly initialized to 0, which is it's starting offset.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1140 from BryanCutler/java-JsonReader-setLast-ARROW-1619 and squashes the following commits:

8f97a3db [Bryan Cutler] added test for VarBinaryVector that checks lastSet after reading
70df0cc4 [Bryan Cutler] set lastSet in JsonFileReader and initialize lastSet for ListVector
---
 .../arrow/vector/complex/ListVector.java      |  2 +-
 .../vector/file/json/JsonFileReader.java      | 46 ++++++++++-------
 .../arrow/vector/file/BaseFileTest.java       | 49 +++++++++++++++++++
 .../arrow/vector/file/TestArrowFile.java      | 41 ++++++++++++++++
 .../arrow/vector/file/json/TestJSONFile.java  | 29 +++++++++++
 5 files changed, 149 insertions(+), 18 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 470317f8c07a6..6511efcb7d513 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -368,7 +368,7 @@ public UnionVector promoteToUnion() {
     return vector;
   }
 
-  private int lastSet;
+  private int lastSet = 0;
 
   public class Accessor extends BaseRepeatedAccessor {
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 8bb0f26d978f7..c6ebd61aa07b9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -44,6 +44,8 @@
 import org.apache.arrow.vector.Float4Vector;
 import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.NullableVarBinaryVector;
+import org.apache.arrow.vector.NullableVarCharVector;
 import org.apache.arrow.vector.SmallIntVector;
 import org.apache.arrow.vector.TimeMicroVector;
 import org.apache.arrow.vector.TimeMilliVector;
@@ -63,10 +65,10 @@
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.UInt8Vector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.ValueVector.Mutator;
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.schema.ArrowVectorType;
@@ -84,7 +86,6 @@
 import com.google.common.base.Objects;
 
 public class JsonFileReader implements AutoCloseable, DictionaryProvider {
-  private final File inputFile;
   private final JsonParser parser;
   private final BufferAllocator allocator;
   private Schema schema;
@@ -93,7 +94,6 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider {
 
   public JsonFileReader(File inputFile, BufferAllocator allocator) throws JsonParseException, IOException {
     super();
-    this.inputFile = inputFile;
     this.allocator = allocator;
     MappingJsonFactory jsonFactory = new MappingJsonFactory();
     this.parser = jsonFactory.createParser(inputFile);
@@ -216,10 +216,9 @@ public VectorSchemaRoot read() throws IOException {
     }
   }
 
-  /*
-   * TODO: This method doesn't load some vectors correctly. For instance, it doesn't initialize
-   * `lastSet` in ListVector, VarCharVector, NullableVarBinaryVector A better way of implementing
-   * this function is to use `loadFieldBuffers` methods in FieldVector.
+  /**
+   * TODO: A better way of implementing this function is to use `loadFieldBuffers` methods in
+   * FieldVector to set the inner-vector data as done in `ArrowFileReader`.
    */
   private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException {
     List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
@@ -234,29 +233,42 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
       if (started && !Objects.equal(field.getName(), name)) {
         throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name);
       }
+
+      // Initialize the vector with required capacity
       int count = readNextField("count", Integer.class);
+      vector.setInitialCapacity(count);
       vector.allocateNew();
-      vector.getMutator().setValueCount(count);
+
+      // Read inner vectors
       for (int v = 0; v < vectorTypes.size(); v++) {
         ArrowVectorType vectorType = vectorTypes.get(v);
-        BufferBacked innerVector = fieldInnerVectors.get(v);
+        ValueVector valueVector = (ValueVector) fieldInnerVectors.get(v);
         nextFieldIs(vectorType.getName());
         readToken(START_ARRAY);
-        ValueVector valueVector = (ValueVector) innerVector;
-
         int innerVectorCount = vectorType.equals(OFFSET) ? count + 1 : count;
-        valueVector.setInitialCapacity(innerVectorCount);
-        valueVector.allocateNew();
-
         for (int i = 0; i < innerVectorCount; i++) {
           parser.nextToken();
           setValueFromParser(valueVector, i);
         }
-        Mutator mutator = valueVector.getMutator();
-        mutator.setValueCount(innerVectorCount);
         readToken(END_ARRAY);
       }
-      // if children
+
+      // Set lastSet before valueCount to prevent setValueCount from filling empty values
+      switch (vector.getMinorType()) {
+        case LIST:
+          // ListVector starts lastSet from index 0, so lastSet value is always last index written + 1
+          ((ListVector) vector).getMutator().setLastSet(count);
+          break;
+        case VARBINARY:
+          ((NullableVarBinaryVector) vector).getMutator().setLastSet(count - 1);
+          break;
+        case VARCHAR:
+          ((NullableVarCharVector) vector).getMutator().setLastSet(count - 1);
+          break;
+      }
+      vector.getMutator().setValueCount(count);
+
+      // read child vectors, if any
       List<Field> fields = field.getChildren();
       if (!fields.isEmpty()) {
         List<FieldVector> vectorChildren = vector.getChildrenFromFields();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index c05d59049776f..ba62de0a6d93a 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -32,6 +32,7 @@
 import org.apache.arrow.vector.NullableDecimalVector;
 import org.apache.arrow.vector.NullableIntVector;
 import org.apache.arrow.vector.NullableTimeMilliVector;
+import org.apache.arrow.vector.NullableVarBinaryVector;
 import org.apache.arrow.vector.NullableVarCharVector;
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VectorSchemaRoot;
@@ -541,4 +542,52 @@ public void writeUnionData(int count, NullableMapVector parent) {
     writer.setValueCount(count);
     varchar.release();
   }
+
+  protected void writeVarBinaryData(int count, NullableMapVector parent) {
+    Assert.assertTrue(count < 100);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    ListWriter listWriter = rootWriter.list("list");
+    ArrowBuf varbin = allocator.buffer(count);
+    for (int i = 0; i < count; i++) {
+      varbin.setByte(i, i);
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 3; j++) {
+        listWriter.varBinary().writeVarBinary(0, i + 1, varbin);
+      }
+      listWriter.endList();
+    }
+    writer.setValueCount(count);
+    varbin.release();
+  }
+
+  protected void validateVarBinary(int count, VectorSchemaRoot root) {
+    Assert.assertEquals(count, root.getRowCount());
+    ListVector listVector = (ListVector) root.getVector("list");
+    byte[] expectedArray = new byte[count];
+    int numVarBinaryValues = 0;
+    for (int i = 0; i < count; i++) {
+      expectedArray[i] = (byte) i;
+      Object obj = listVector.getAccessor().getObject(i);
+      List<?> objList = (List) obj;
+      if (i % 3 == 0) {
+        Assert.assertTrue(objList.isEmpty());
+      } else {
+        byte[] expected = Arrays.copyOfRange(expectedArray, 0, i + 1);
+        for (int j = 0; j < i % 3; j++) {
+          byte[] result = (byte[]) objList.get(j);
+          Assert.assertArrayEquals(result, expected);
+          numVarBinaryValues++;
+        }
+      }
+    }
+
+    // ListVector lastSet should be the index of last value + 1
+    Assert.assertEquals(listVector.getMutator().getLastSet(), count);
+
+    // NullableVarBinaryVector lastSet should be the index of last value
+    NullableVarBinaryVector binaryVector = (NullableVarBinaryVector) listVector.getChildrenFromFields().get(0);
+    Assert.assertEquals(binaryVector.getMutator().getLastSet(), numVarBinaryValues - 1);
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index c483ba7de91c6..81e58989fccc4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -622,6 +622,47 @@ public void testWriteReadFixedSizeList() throws IOException {
     }
   }
 
+  @Test
+  public void testWriteReadVarBin() throws IOException {
+    File file = new File("target/mytest_varbin.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+    int count = COUNT;
+
+    // write
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+      writeVarBinaryData(count, parent);
+      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
+      validateVarBinary(count, root);
+      write(parent.getChild("root"), file, stream);
+    }
+
+    // read from file
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        FileInputStream fileInputStream = new FileInputStream(file);
+        ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      Assert.assertTrue(arrowReader.loadNextBatch());
+      validateVarBinary(count, root);
+    }
+
+    // read from stream
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+        ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      Assert.assertTrue(arrowReader.loadNextBatch());
+      validateVarBinary(count, root);
+    }
+  }
+
 
   /**
    * Writes the contents of parents to file. If outStream is non-null, also writes it
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 960567fc87087..ee90d340d7cc8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -285,4 +285,33 @@ public void testSetStructLength() throws IOException {
     }
   }
 
+  @Test
+  public void testWriteReadVarBinJSON() throws IOException {
+    File file = new File("target/mytest_varbin.json");
+    int count = COUNT;
+
+    // write
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+      writeVarBinaryData(count, parent);
+      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
+      validateVarBinary(count, root);
+      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE)) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateVarBinary(count, root);
+      }
+      reader.close();
+    }
+  }
 }

From 7045b4213bdfb9d107ca3ae72eecfb424c8e19b1 Mon Sep 17 00:00:00 2001
From: Rene Sugar <rene.sugar@gmail.com>
Date: Fri, 29 Sep 2017 15:29:38 -0500
Subject: [PATCH 1097/1644] =?UTF-8?q?ARROW-1615=20Added=20BUILD=5FWARNING?=
 =?UTF-8?q?=5FLEVEL=20and=20BUILD=5FWARNING=5FFLAGS=20to=20Setup=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…CxxFlags.cmake

Author: Rene Sugar <rene.sugar@gmail.com>

Closes #1145 from renesugar/ARROW-1615 and squashes the following commits:

71a615e3 [Rene Sugar] ARROW-1615 Add -Wno-vla-extension and change non-checkin builds back to -Wall
18958430 [Rene Sugar] ARROW-1615 Add -Wno-cast-align
5fe4e8e8 [Rene Sugar] ARROW-1615 Move -Wno-shorten-64-to-32 after -Wconversion
9d3c7ec3 [Rene Sugar] ARROW-1615 Identify compiler version for clang-802 plus more warning entries
5ebaf86e [Rene Sugar] ARROW-1615 Moved version specific warning entry
971e61aa [Rene Sugar] ARROW-1615 Fixed version specific warning entry
6cf24977 [Rene Sugar] ARROW-1615 Added more version specific Clang warning entries
50def439 [Rene Sugar] ARROW-1615 Updated build warning level terminology
ea906eb4 [Rene Sugar] ARROW-1615 Check compiler version before disabling some warnings
159e1897 [Rene Sugar] ARROW-1615 Include CompilerInfo before SetupCxxFlags in arrow/python
8359c966 [Rene Sugar] ARROW-1615 Added BUILD_WARNING_LEVEL and BUILD_WARNING_FLAGS to SetupCxxFlags.cmake
---
 cpp/CMakeLists.txt                    |   6 +-
 cpp/cmake_modules/CompilerInfo.cmake  |   5 ++
 cpp/cmake_modules/SetupCxxFlags.cmake | 105 +++++++++++++++++++++++++-
 python/CMakeLists.txt                 |   2 +-
 4 files changed, 113 insertions(+), 5 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 972132f29e901..ad99970e9b25f 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -229,6 +229,9 @@ include(BuildUtils)
 # Compiler flags
 ############################################################
 
+# Determine compiler version
+include(CompilerInfo)
+
 if (ARROW_NO_DEPRECATED_API)
   add_definitions(-DARROW_NO_DEPRECATED_API)
 endif()
@@ -253,9 +256,6 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}")
 
 message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
 
-# Determine compiler version
-include(CompilerInfo)
-
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
   # Using Clang with ccache causes a bunch of spurious warnings that are
   # purportedly fixed in the next version of ccache. See the following for details:
diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
index a1b470182a8dc..5ff1d861415d4 100644
--- a/cpp/cmake_modules/CompilerInfo.cmake
+++ b/cpp/cmake_modules/CompilerInfo.cmake
@@ -63,6 +63,11 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-7")
   set(COMPILER_FAMILY "clang")
   set(COMPILER_VERSION "3.7.0svn")
 
+# clang on Mac OS X, XCode 8.
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-802")
+  set(COMPILER_FAMILY "clang")
+  set(COMPILER_VERSION "3.9.0svn")
+
 # clang on Mac OS X, XCode 8.
 elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-8")
   set(COMPILER_FAMILY "clang")
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 6e92c4b1c1b0d..1f4c898cfcf3f 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -48,7 +48,110 @@ if (MSVC)
     set(CXX_COMMON_FLAGS "/W3")
   endif()
 else()
-  set(CXX_COMMON_FLAGS "-Wall -std=c++11")
+  # Common flags set below with warning level
+  set(CXX_COMMON_FLAGS "")
+endif()
+
+# Build warning level (CHECKIN, EVERYTHING, etc.)
+
+# if no build warning level is specified, default to development warning level
+if (NOT BUILD_WARNING_LEVEL)
+  set(BUILD_WARNING_LEVEL Production)
+endif(NOT BUILD_WARNING_LEVEL)
+
+string(TOUPPER ${BUILD_WARNING_LEVEL} UPPERCASE_BUILD_WARNING_LEVEL)
+
+if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
+  # Pre-checkin builds
+  if ("${COMPILER_FAMILY}" STREQUAL "msvc")
+    string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
+    # Treat all compiler warnings as errors
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
+  elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
+    -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
+    -Wno-unused-parameter -Wno-undef -Wno-documentation-deprecated-sync \
+    -Wno-shadow -Wno-switch-enum -Wno-documentation -Wno-exit-time-destructors \
+    -Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \
+    -Wno-implicit-fallthrough -Wno-old-style-cast -Wno-unreachable-code-return \
+    -Wno-float-equal -Wno-missing-prototypes -Wno-non-virtual-dtor \
+    -Wno-unused-macros -Wno-covered-switch-default -Wno-unreachable-code-break \
+    -Wno-extra-semi -Wno-cast-align -Wno-vla-extension -Wno-shift-sign-overflow \
+    -Wno-used-but-marked-unused -Wno-missing-variable-declarations \
+    -Wno-gnu-zero-variadic-macro-arguments -Wconversion -Wno-sign-conversion \
+    -Wno-disabled-macro-expansion -Wc++11-narrowing -Wnarrowing -Wno-shorten-64-to-32")
+
+    # Version numbers where warnings are introduced
+    if ("${COMPILER_VERSION}" VERSION_GREATER "3.3")
+      set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-gnu-folding-constant")
+    endif()
+    if ("${COMPILER_VERSION}" VERSION_GREATER "3.6")
+      set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-reserved-id-macro")
+      set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-range-loop-analysis")
+    endif()
+    if ("${COMPILER_VERSION}" VERSION_GREATER "3.7")
+      set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-double-promotion")
+    endif()
+    if ("${COMPILER_VERSION}" VERSION_GREATER "3.8")
+      set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-undefined-func-template")
+    endif()
+
+    # Treat all compiler warnings as errors
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wunknown-warning-option -Werror")
+  elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wconversion -Wno-sign-conversion")
+    # Treat all compiler warnings as errors
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wunknown-warning-option -Werror")
+  else()
+    message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
+  endif()
+elseif ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "EVERYTHING")
+  # Pedantic builds for fixing warnings
+  if ("${COMPILER_FAMILY}" STREQUAL "msvc")
+    string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /Wall")
+    # https://docs.microsoft.com/en-us/cpp/build/reference/compiler-option-warning-level
+    # /wdnnnn disables a warning where "nnnn" is a warning number
+    # Treat all compiler warnings as errors
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}  /WX")
+  elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic")
+    # Treat all compiler warnings as errors
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror")
+  elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wpedantic -Wextra -Wno-unused-parameter")
+    # Treat all compiler warnings as errors
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror")
+  else()
+    message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
+  endif()
+else()
+  # Production builds (warning are not treated as errors)
+  if ("${COMPILER_FAMILY}" STREQUAL "msvc")
+    # https://docs.microsoft.com/en-us/cpp/build/reference/compiler-option-warning-level
+    # TODO: Enable /Wall and disable individual warnings until build compiles without errors
+    # /wdnnnn disables a warning where "nnnn" is a warning number
+    string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
+  elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
+  elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
+  else()
+    message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
+  endif()
+endif()
+
+# if build warning flags is set, add to CXX_COMMON_FLAGS
+if (BUILD_WARNING_FLAGS)
+  # Use BUILD_WARNING_FLAGS with BUILD_WARNING_LEVEL=everything to disable
+  # warnings (use with Clang's -Weverything flag to find potential errors)
+  set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} ${BUILD_WARNING_FLAGS}")
+endif(BUILD_WARNING_FLAGS)
+
+if (NOT ("${COMPILER_FAMILY}" STREQUAL "msvc"))
+set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=c++11")
 endif()
 
 # Only enable additional instruction sets if they are supported
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index a636d51b4209e..d148d1105b64e 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -72,8 +72,8 @@ endif(CCACHE_FOUND)
 ############################################################
 
 include(BuildUtils)
-include(SetupCxxFlags)
 include(CompilerInfo)
+include(SetupCxxFlags)
 
 # Add common flags
 set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")

From a03e093b16c7631f5ba4354df93889d257429d57 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 29 Sep 2017 23:01:42 -0500
Subject: [PATCH 1098/1644] ARROW-1600: [C++] Add Buffer constructor that wraps
 std::string

Many other libraries interchange binary data with `std::string`. This makes it easy to wrap such data in an `arrow::Buffer`.

It may be worth adding a function that creates a buffer from a string, but owns its memory.

I also deprecated `arrow::GetBufferFromString`, which shouldn't have been public in the first place, since the new ctor is more general

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1147 from wesm/ARROW-1600 and squashes the following commits:

f60f502c [Wes McKinney] Remove TestBuffer fixture
644bf2b7 [Wes McKinney] Add Buffer ctor that wraps std::string
---
 cpp/src/arrow/buffer-test.cc             | 28 +++++++++------
 cpp/src/arrow/buffer.h                   | 44 +++++++++++++++++-------
 cpp/src/arrow/ipc/ipc-json-test.cc       |  6 ++--
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  8 ++---
 4 files changed, 55 insertions(+), 31 deletions(-)

diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index 334ad7bf7143d..5fd2706f0466b 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -32,9 +32,7 @@ using std::string;
 
 namespace arrow {
 
-class TestBuffer : public ::testing::Test {};
-
-TEST_F(TestBuffer, IsMutableFlag) {
+TEST(TestBuffer, IsMutableFlag) {
   Buffer buf(nullptr, 0);
 
   ASSERT_FALSE(buf.is_mutable());
@@ -46,7 +44,15 @@ TEST_F(TestBuffer, IsMutableFlag) {
   ASSERT_TRUE(pbuf.is_mutable());
 }
 
-TEST_F(TestBuffer, Resize) {
+TEST(TestBuffer, FromStdString) {
+  std::string val = "hello, world";
+
+  Buffer buf(val);
+  ASSERT_EQ(0, memcmp(buf.data(), val.c_str(), val.size()));
+  ASSERT_EQ(static_cast<int64_t>(val.size()), buf.size());
+}
+
+TEST(TestBuffer, Resize) {
   PoolBuffer buf;
 
   ASSERT_EQ(0, buf.size());
@@ -69,7 +75,7 @@ TEST_F(TestBuffer, Resize) {
   ASSERT_EQ(128, buf.capacity());
 }
 
-TEST_F(TestBuffer, TypedResize) {
+TEST(TestBuffer, TypedResize) {
   PoolBuffer buf;
 
   ASSERT_EQ(0, buf.size());
@@ -88,7 +94,7 @@ TEST_F(TestBuffer, TypedResize) {
   ASSERT_EQ(832, buf.capacity());
 }
 
-TEST_F(TestBuffer, ResizeOOM) {
+TEST(TestBuffer, ResizeOOM) {
 // This test doesn't play nice with AddressSanitizer
 #ifndef ADDRESS_SANITIZER
   // realloc fails, even though there may be no explicit limit
@@ -99,7 +105,7 @@ TEST_F(TestBuffer, ResizeOOM) {
 #endif
 }
 
-TEST_F(TestBuffer, EqualsWithSameContent) {
+TEST(TestBuffer, EqualsWithSameContent) {
   MemoryPool* pool = default_memory_pool();
   const int32_t bufferSize = 128 * 1024;
   uint8_t* rawBuffer1;
@@ -123,7 +129,7 @@ TEST_F(TestBuffer, EqualsWithSameContent) {
   pool->Free(rawBuffer3, bufferSize);
 }
 
-TEST_F(TestBuffer, EqualsWithSameBuffer) {
+TEST(TestBuffer, EqualsWithSameBuffer) {
   MemoryPool* pool = default_memory_pool();
   const int32_t bufferSize = 128 * 1024;
   uint8_t* rawBuffer;
@@ -142,7 +148,7 @@ TEST_F(TestBuffer, EqualsWithSameBuffer) {
   pool->Free(rawBuffer, bufferSize);
 }
 
-TEST_F(TestBuffer, Copy) {
+TEST(TestBuffer, Copy) {
   std::string data_str = "some data to copy";
 
   auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
@@ -157,7 +163,7 @@ TEST_F(TestBuffer, Copy) {
   ASSERT_TRUE(out->Equals(expected));
 }
 
-TEST_F(TestBuffer, SliceBuffer) {
+TEST(TestBuffer, SliceBuffer) {
   std::string data_str = "some data to slice";
 
   auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
@@ -171,7 +177,7 @@ TEST_F(TestBuffer, SliceBuffer) {
   ASSERT_EQ(2, buf.use_count());
 }
 
-TEST_F(TestBuffer, SliceMutableBuffer) {
+TEST(TestBuffer, SliceMutableBuffer) {
   std::string data_str = "some data to slice";
   auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
 
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index d215267786050..dbd93763638b3 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -47,9 +47,25 @@ class MemoryPool;
 /// The following invariant is always true: Size < Capacity
 class ARROW_EXPORT Buffer {
  public:
+  /// \brief Construct from buffer and size without copying memory
+  ///
+  /// \param[in] data a memory buffer
+  /// \param[in] size buffer size
+  ///
+  /// \note The passed memory must be kept alive through some other means
   Buffer(const uint8_t* data, int64_t size)
       : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
 
+  /// \brief Construct from std::string without copying memory
+  ///
+  /// \param[in] data a std::string object
+  ///
+  /// \note The std::string must stay alive for the lifetime of the Buffer, so
+  /// temporary rvalue strings must be stored in an lvalue somewhere
+  explicit Buffer(const std::string& data)
+      : Buffer(reinterpret_cast<const uint8_t*>(data.c_str()),
+               static_cast<int64_t>(data.size())) {}
+
   virtual ~Buffer() = default;
 
   /// An offset into data that is owned by another buffer, but we want to be
@@ -69,6 +85,8 @@ class ARROW_EXPORT Buffer {
   /// Return true if both buffers are the same size and contain the same bytes
   /// up to the number of compared bytes
   bool Equals(const Buffer& other, int64_t nbytes) const;
+
+  /// Return true if both buffers are the same size and contain the same bytes
   bool Equals(const Buffer& other) const;
 
   /// Copy a section of the buffer into a new Buffer.
@@ -101,17 +119,6 @@ class ARROW_EXPORT Buffer {
   ARROW_DISALLOW_COPY_AND_ASSIGN(Buffer);
 };
 
-/// \brief Create Buffer referencing std::string memory
-///
-/// Warning: string instance must stay alive
-///
-/// \param str std::string instance
-/// \return std::shared_ptr<Buffer>
-static inline std::shared_ptr<Buffer> GetBufferFromString(const std::string& str) {
-  return std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(str.c_str()),
-                                  static_cast<int64_t>(str.size()));
-}
-
 /// Construct a view on passed buffer at the indicated offset and length. This
 /// function cannot fail and does not error checking (except in debug builds)
 static inline std::shared_ptr<Buffer> SliceBuffer(const std::shared_ptr<Buffer>& buffer,
@@ -331,11 +338,24 @@ Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
                                std::shared_ptr<ResizableBuffer>* out);
 
 #ifndef ARROW_NO_DEPRECATED_API
+
 /// \deprecated Since 0.7.0
 ARROW_EXPORT
 Status AllocateBuffer(MemoryPool* pool, const int64_t size,
                       std::shared_ptr<MutableBuffer>* out);
-#endif
+
+/// \brief Create Buffer referencing std::string memory
+/// \deprecated Since 0.8.0
+///
+/// Warning: string instance must stay alive
+///
+/// \param str std::string instance
+/// \return std::shared_ptr<Buffer>
+static inline std::shared_ptr<Buffer> GetBufferFromString(const std::string& str) {
+  return std::make_shared<Buffer>(str);
+}
+
+#endif  // ARROW_NO_DEPRECATED_API
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 7855aeafeb094..f2dd9e74e335d 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -279,8 +279,7 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
 
   std::unique_ptr<JsonReader> reader;
 
-  auto buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(result.c_str()),
-                                         static_cast<int>(result.size()));
+  auto buffer = std::make_shared<Buffer>(result);
 
   ASSERT_OK(JsonReader::Open(buffer, &reader));
   ASSERT_TRUE(reader->schema()->Equals(*schema));
@@ -395,8 +394,7 @@ void CheckRoundtrip(const RecordBatch& batch) {
   std::string result;
   ASSERT_OK(writer->Finish(&result));
 
-  auto buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(result.c_str()),
-                                         static_cast<int64_t>(result.size()));
+  auto buffer = std::make_shared<Buffer>(result);
 
   std::unique_ptr<JsonReader> reader;
   ASSERT_OK(JsonReader::Open(buffer, &reader));
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index ad3af0fb69f6c..d454d59b285fd 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -63,10 +63,10 @@ TEST(TestMessage, Equals) {
   std::string metadata = "foo";
   std::string body = "bar";
 
-  auto b1 = GetBufferFromString(metadata);
-  auto b2 = GetBufferFromString(metadata);
-  auto b3 = GetBufferFromString(body);
-  auto b4 = GetBufferFromString(body);
+  auto b1 = std::make_shared<Buffer>(metadata);
+  auto b2 = std::make_shared<Buffer>(metadata);
+  auto b3 = std::make_shared<Buffer>(body);
+  auto b4 = std::make_shared<Buffer>(body);
 
   Message msg1(b1, b3);
   Message msg2(b2, b4);

From ccbf6446bccda9856f7e86f5d9ccccd80273eba2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 29 Sep 2017 23:02:58 -0500
Subject: [PATCH 1099/1644] ARROW-838: [Python] Expand pyarrow.array to handle
 NumPy arrays not originating in pandas
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This unifies the ingest path for 1D data into `pyarrow.array`. I added the argument `from_pandas` to turn null sentinel checking on or off:

```
In [8]: arr = np.random.randn(10000000)

In [9]: arr[::3] = np.nan

In [10]: arr2 = pa.array(arr)

In [11]: arr2.null_count
Out[11]: 0

In [12]: %timeit arr2 = pa.array(arr)
The slowest run took 5.43 times longer than the fastest. This could mean that an intermediate result is being cached.
10000 loops, best of 3: 68.4 µs per loop

In [13]: arr2 = pa.array(arr, from_pandas=True)

In [14]: arr2.null_count
Out[14]: 3333334

In [15]: %timeit arr2 = pa.array(arr, from_pandas=True)
1 loop, best of 3: 228 ms per loop
```

When the data is contiguous, it is always zero-copy, but then `from_pandas=True` and no null mask is passed, then a null bitmap is constructed and populated.

This also permits sequence reads into integers smaller than int64:

```
In [17]: pa.array([1, 2, 3, 4], type='i1')
Out[17]:
<pyarrow.lib.Int8Array object at 0x7ffa1c1c65e8>
[
  1,
  2,
  3,
  4
]
```

Oh, I also added NumPy-like string type aliases:

```
In [18]: pa.int32() == 'i4'
Out[18]: True
```

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1146 from wesm/expand-py-array-method and squashes the following commits:

1570e525 [Wes McKinney] Code review comments
d3bbb3c3 [Wes McKinney] Handle type aliases in cast, too
797f0151 [Wes McKinney] Allow null checking to be skipped with from_pandas=False in pyarrow.array
f2802fc7 [Wes McKinney] Cleaner codepath for numpy->arrow conversions
587c575a [Wes McKinney] Add direct types sequence converters for more data types
cf40b767 [Wes McKinney] Add type aliases, some unit tests
7b530e4b [Wes McKinney] Consolidate both sequence and ndarray/Series/Index conversion in pyarrow.Array
---
 cpp/src/arrow/python/CMakeLists.txt           |   4 +-
 cpp/src/arrow/python/api.h                    |   2 +-
 cpp/src/arrow/python/builtin_convert.cc       | 223 +++++++++-----
 .../{pandas_to_arrow.cc => numpy_to_arrow.cc} | 251 ++++++++--------
 .../{pandas_to_arrow.h => numpy_to_arrow.h}   |  21 +-
 python/pyarrow/__init__.py                    |   2 +-
 python/pyarrow/array.pxi                      | 279 ++++++++++--------
 python/pyarrow/includes/libarrow.pxd          |  11 +-
 python/pyarrow/pandas_compat.py               |  22 +-
 python/pyarrow/scalar.pxi                     |   8 +-
 python/pyarrow/table.pxi                      |  13 +-
 python/pyarrow/tests/test_array.py            |  58 ++--
 python/pyarrow/tests/test_convert_pandas.py   |  95 +++---
 python/pyarrow/tests/test_parquet.py          |  42 ++-
 python/pyarrow/tests/test_schema.py           |  50 ++++
 python/pyarrow/types.pxi                      |  72 ++++-
 16 files changed, 698 insertions(+), 455 deletions(-)
 rename cpp/src/arrow/python/{pandas_to_arrow.cc => numpy_to_arrow.cc} (90%)
 rename cpp/src/arrow/python/{pandas_to_arrow.h => numpy_to_arrow.h} (70%)

diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 84aad82e2a90e..7938d8473b6bf 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -57,7 +57,7 @@ set(ARROW_PYTHON_SRCS
   init.cc
   io.cc
   numpy_convert.cc
-  pandas_to_arrow.cc
+  numpy_to_arrow.cc
   python_to_arrow.cc
   pyarrow.cc
 )
@@ -100,7 +100,7 @@ install(FILES
   io.h
   numpy_convert.h
   numpy_interop.h
-  pandas_to_arrow.h
+  numpy_to_arrow.h
   python_to_arrow.h
   platform.h
   pyarrow.h
diff --git a/cpp/src/arrow/python/api.h b/cpp/src/arrow/python/api.h
index 4ceb3f1a45dc1..a000ac5fa5ae5 100644
--- a/cpp/src/arrow/python/api.h
+++ b/cpp/src/arrow/python/api.h
@@ -25,7 +25,7 @@
 #include "arrow/python/helpers.h"
 #include "arrow/python/io.h"
 #include "arrow/python/numpy_convert.h"
-#include "arrow/python/pandas_to_arrow.h"
+#include "arrow/python/numpy_to_arrow.h"
 #include "arrow/python/python_to_arrow.h"
 
 #endif  // ARROW_PYTHON_API_H
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 747b872af0a30..f9d7361e00425 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -20,6 +20,7 @@
 #include <datetime.h>
 
 #include <algorithm>
+#include <limits>
 #include <sstream>
 #include <string>
 
@@ -359,7 +360,11 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
     if (PySequence_Check(obj)) {
       for (int64_t i = 0; i < size; ++i) {
         OwnedRef ref(PySequence_GetItem(obj, i));
-        RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
+        if (ref.obj() == Py_None) {
+          RETURN_NOT_OK(this->typed_builder_->AppendNull());
+        } else {
+          RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
+        }
       }
     } else if (PyObject_HasAttrString(obj, "__iter__")) {
       PyObject* iter = PyObject_GetIter(obj);
@@ -370,7 +375,11 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
       // consuming at size.
       while ((item = PyIter_Next(iter)) && i < size) {
         OwnedRef ref(item);
-        RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
+        if (ref.obj() == Py_None) {
+          RETURN_NOT_OK(this->typed_builder_->AppendNull());
+        } else {
+          RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
+        }
         ++i;
       }
       if (size != i) {
@@ -388,52 +397,136 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
 class NullConverter : public TypedConverterVisitor<NullBuilder, NullConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
-    if (item.obj() == Py_None) {
-      return typed_builder_->AppendNull();
-    } else {
-      return Status::Invalid("NullConverter: passed non-None value");
-    }
+    return Status::Invalid("NullConverter: passed non-None value");
   }
 };
 
 class BoolConverter : public TypedConverterVisitor<BooleanBuilder, BoolConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
-    if (item.obj() == Py_None) {
-      return typed_builder_->AppendNull();
-    } else {
-      if (item.obj() == Py_True) {
-        return typed_builder_->Append(true);
-      } else {
-        return typed_builder_->Append(false);
-      }
+    return typed_builder_->Append(item.obj() == Py_True);
+  }
+};
+
+class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int8_t>::max() ||
+                            val < std::numeric_limits<int8_t>::min())) {
+      return Status::Invalid(
+          "Cannot coerce values to array type that would "
+          "lose data");
     }
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(static_cast<int8_t>(val));
+  }
+};
+
+class Int16Converter : public TypedConverterVisitor<Int16Builder, Int16Converter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int16_t>::max() ||
+                            val < std::numeric_limits<int16_t>::min())) {
+      return Status::Invalid(
+          "Cannot coerce values to array type that would "
+          "lose data");
+    }
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(static_cast<int16_t>(val));
+  }
+};
+
+class Int32Converter : public TypedConverterVisitor<Int32Builder, Int32Converter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int32_t>::max() ||
+                            val < std::numeric_limits<int32_t>::min())) {
+      return Status::Invalid(
+          "Cannot coerce values to array type that would "
+          "lose data");
+    }
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(static_cast<int32_t>(val));
   }
 };
 
 class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
-    int64_t val;
-    if (item.obj() == Py_None) {
-      return typed_builder_->AppendNull();
-    } else {
-      val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
-      RETURN_IF_PYERROR();
-      return typed_builder_->Append(val);
+    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(val);
+  }
+};
+
+class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint8_t>::max() ||
+                            val < std::numeric_limits<uint8_t>::min())) {
+      return Status::Invalid(
+          "Cannot coerce values to array type that would "
+          "lose data");
     }
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(static_cast<uint8_t>(val));
   }
 };
 
-class DateConverter : public TypedConverterVisitor<Date64Builder, DateConverter> {
+class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Converter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
-    if (item.obj() == Py_None) {
-      return typed_builder_->AppendNull();
-    } else {
-      PyDateTime_Date* pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
-      return typed_builder_->Append(PyDate_to_ms(pydate));
+    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint16_t>::max() ||
+                            val < std::numeric_limits<uint16_t>::min())) {
+      return Status::Invalid(
+          "Cannot coerce values to array type that would "
+          "lose data");
     }
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(static_cast<uint16_t>(val));
+  }
+};
+
+class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Converter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint32_t>::max() ||
+                            val < std::numeric_limits<uint32_t>::min())) {
+      return Status::Invalid(
+          "Cannot coerce values to array type that would "
+          "lose data");
+    }
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(static_cast<uint32_t>(val));
+  }
+};
+
+class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Converter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(val);
+  }
+};
+
+class DateConverter : public TypedConverterVisitor<Date64Builder, DateConverter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
+    return typed_builder_->Append(PyDate_to_ms(pydate));
   }
 };
 
@@ -441,27 +534,17 @@ class TimestampConverter
     : public TypedConverterVisitor<Date64Builder, TimestampConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
-    if (item.obj() == Py_None) {
-      return typed_builder_->AppendNull();
-    } else {
-      PyDateTime_DateTime* pydatetime =
-          reinterpret_cast<PyDateTime_DateTime*>(item.obj());
-      return typed_builder_->Append(PyDateTime_to_us(pydatetime));
-    }
+    auto pydatetime = reinterpret_cast<PyDateTime_DateTime*>(item.obj());
+    return typed_builder_->Append(PyDateTime_to_us(pydatetime));
   }
 };
 
 class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
-    double val;
-    if (item.obj() == Py_None) {
-      return typed_builder_->AppendNull();
-    } else {
-      val = PyFloat_AsDouble(item.obj());
-      RETURN_IF_PYERROR();
-      return typed_builder_->Append(val);
-    }
+    double val = PyFloat_AsDouble(item.obj());
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(val);
   }
 };
 
@@ -473,10 +556,7 @@ class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverte
     Py_ssize_t length;
     OwnedRef tmp;
 
-    if (item.obj() == Py_None) {
-      RETURN_NOT_OK(typed_builder_->AppendNull());
-      return Status::OK();
-    } else if (PyUnicode_Check(item.obj())) {
+    if (PyUnicode_Check(item.obj())) {
       tmp.reset(PyUnicode_AsUTF8String(item.obj()));
       RETURN_IF_PYERROR();
       bytes_obj = tmp.obj();
@@ -504,10 +584,7 @@ class FixedWidthBytesConverter
     Py_ssize_t expected_length =
         std::dynamic_pointer_cast<FixedSizeBinaryType>(typed_builder_->type())
             ->byte_width();
-    if (item.obj() == Py_None) {
-      RETURN_NOT_OK(typed_builder_->AppendNull());
-      return Status::OK();
-    } else if (PyUnicode_Check(item.obj())) {
+    if (PyUnicode_Check(item.obj())) {
       tmp.reset(PyUnicode_AsUTF8String(item.obj()));
       RETURN_IF_PYERROR();
       bytes_obj = tmp.obj();
@@ -535,9 +612,7 @@ class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter>
     Py_ssize_t length;
 
     PyObject* obj = item.obj();
-    if (obj == Py_None) {
-      return typed_builder_->AppendNull();
-    } else if (PyBytes_Check(obj)) {
+    if (PyBytes_Check(obj)) {
       tmp.reset(
           PyUnicode_FromStringAndSize(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)));
       RETURN_IF_PYERROR();
@@ -565,14 +640,10 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
   Status Init(ArrayBuilder* builder) override;
 
   inline Status AppendItem(const OwnedRef& item) override {
-    if (item.obj() == Py_None) {
-      return typed_builder_->AppendNull();
-    } else {
-      RETURN_NOT_OK(typed_builder_->Append());
-      PyObject* item_obj = item.obj();
-      int64_t list_size = static_cast<int64_t>(PySequence_Size(item_obj));
-      return value_converter_->AppendData(item_obj, list_size);
-    }
+    RETURN_NOT_OK(typed_builder_->Append());
+    PyObject* item_obj = item.obj();
+    int64_t list_size = static_cast<int64_t>(PySequence_Size(item_obj));
+    return value_converter_->AppendData(item_obj, list_size);
   }
 
  protected:
@@ -584,16 +655,12 @@ class DecimalConverter
  public:
   inline Status AppendItem(const OwnedRef& item) {
     /// TODO(phillipc): Check for nan?
-    if (item.obj() != Py_None) {
-      std::string string;
-      RETURN_NOT_OK(PythonDecimalToString(item.obj(), &string));
-
-      Decimal128 value;
-      RETURN_NOT_OK(Decimal128::FromString(string, &value));
-      return typed_builder_->Append(value);
-    }
+    std::string string;
+    RETURN_NOT_OK(PythonDecimalToString(item.obj(), &string));
 
-    return typed_builder_->AppendNull();
+    Decimal128 value;
+    RETURN_NOT_OK(Decimal128::FromString(string, &value));
+    return typed_builder_->Append(value);
   }
 };
 
@@ -604,8 +671,22 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
       return std::make_shared<NullConverter>();
     case Type::BOOL:
       return std::make_shared<BoolConverter>();
+    case Type::INT8:
+      return std::make_shared<Int8Converter>();
+    case Type::INT16:
+      return std::make_shared<Int16Converter>();
+    case Type::INT32:
+      return std::make_shared<Int32Converter>();
     case Type::INT64:
       return std::make_shared<Int64Converter>();
+    case Type::UINT8:
+      return std::make_shared<UInt8Converter>();
+    case Type::UINT16:
+      return std::make_shared<UInt16Converter>();
+    case Type::UINT32:
+      return std::make_shared<UInt32Converter>();
+    case Type::UINT64:
+      return std::make_shared<UInt64Converter>();
     case Type::DATE64:
       return std::make_shared<DateConverter>();
     case Type::TIMESTAMP:
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
similarity index 90%
rename from cpp/src/arrow/python/pandas_to_arrow.cc
rename to cpp/src/arrow/python/numpy_to_arrow.cc
index dc5b67f53e44d..7151c94c0dbcd 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -19,10 +19,9 @@
 
 #define ARROW_NO_DEFAULT_MEMORY_POOL
 
+#include "arrow/python/numpy_to_arrow.h"
 #include "arrow/python/numpy_interop.h"
 
-#include "arrow/python/pandas_to_arrow.h"
-
 #include <algorithm>
 #include <cmath>
 #include <cstdint>
@@ -60,10 +59,14 @@ namespace py {
 
 using internal::NumPyTypeSize;
 
+constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max();
+
 // ----------------------------------------------------------------------
 // Conversion utilities
 
-static inline bool PyFloat_isnan(const PyObject* obj) {
+namespace {
+
+inline bool PyFloat_isnan(const PyObject* obj) {
   if (PyFloat_Check(obj)) {
     double val = PyFloat_AS_DOUBLE(obj);
     return val != val;
@@ -71,11 +74,12 @@ static inline bool PyFloat_isnan(const PyObject* obj) {
     return false;
   }
 }
-static inline bool PandasObjectIsNull(const PyObject* obj) {
+
+inline bool PandasObjectIsNull(const PyObject* obj) {
   return obj == Py_None || obj == numpy_nan || PyFloat_isnan(obj);
 }
 
-static inline bool PyObject_is_string(const PyObject* obj) {
+inline bool PyObject_is_string(const PyObject* obj) {
 #if PY_MAJOR_VERSION >= 3
   return PyUnicode_Check(obj) || PyBytes_Check(obj);
 #else
@@ -83,14 +87,14 @@ static inline bool PyObject_is_string(const PyObject* obj) {
 #endif
 }
 
-static inline bool PyObject_is_float(const PyObject* obj) { return PyFloat_Check(obj); }
+inline bool PyObject_is_float(const PyObject* obj) { return PyFloat_Check(obj); }
 
-static inline bool PyObject_is_integer(const PyObject* obj) {
+inline bool PyObject_is_integer(const PyObject* obj) {
   return (!PyBool_Check(obj)) && PyArray_IsIntegerScalar(obj);
 }
 
 template <int TYPE>
-static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
+inline int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
   typedef internal::npy_traits<TYPE> traits;
   typedef typename traits::value_type T;
 
@@ -109,7 +113,7 @@ static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
 }
 
 // Returns null count
-static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
+int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
   int64_t null_count = 0;
 
   Ndarray1DIndexer<uint8_t> mask_values(mask);
@@ -123,29 +127,6 @@ static int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap
   return null_count;
 }
 
-template <int TYPE, typename BuilderType>
-static Status AppendNdarrayToBuilder(PyArrayObject* array, BuilderType* builder) {
-  typedef internal::npy_traits<TYPE> traits;
-  typedef typename traits::value_type T;
-
-  // TODO(wesm): Vector append when not strided
-  Ndarray1DIndexer<T> values(array);
-  if (traits::supports_nulls) {
-    for (int64_t i = 0; i < values.size(); ++i) {
-      if (traits::isnull(values[i])) {
-        RETURN_NOT_OK(builder->AppendNull());
-      } else {
-        RETURN_NOT_OK(builder->Append(values[i]));
-      }
-    }
-  } else {
-    for (int64_t i = 0; i < values.size(); ++i) {
-      RETURN_NOT_OK(builder->Append(values[i]));
-    }
-  }
-  return Status::OK();
-}
-
 Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
   if (PyArray_NDIM(numpy_array) != 1) {
     return Status::Invalid("only handle 1-dimensional arrays");
@@ -162,7 +143,7 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
   return Status::OK();
 }
 
-constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max();
+}  // namespace
 
 /// Append as many string objects from NumPy arrays to a `StringBuilder` as we
 /// can fit
@@ -272,14 +253,15 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
 // ----------------------------------------------------------------------
 // Conversion from NumPy-in-Pandas to Arrow
 
-class PandasConverter {
+class NumPyConverter {
  public:
-  PandasConverter(MemoryPool* pool, PyObject* ao, PyObject* mo,
-                  const std::shared_ptr<DataType>& type)
+  NumPyConverter(MemoryPool* pool, PyObject* ao, PyObject* mo,
+                 const std::shared_ptr<DataType>& type, bool use_pandas_null_sentinels)
       : pool_(pool),
         type_(type),
         arr_(reinterpret_cast<PyArrayObject*>(ao)),
-        mask_(nullptr) {
+        mask_(nullptr),
+        use_pandas_null_sentinels_(use_pandas_null_sentinels) {
     if (mo != nullptr && mo != Py_None) {
       mask_ = reinterpret_cast<PyArrayObject*>(mo);
     }
@@ -291,6 +273,39 @@ class PandasConverter {
     return astrides[0] != PyArray_DESCR(arr_)->elsize;
   }
 
+  Status Convert();
+
+  const std::vector<std::shared_ptr<Array>>& result() const { return out_arrays_; }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
+                              std::is_same<BooleanType, T>::value,
+                          Status>::type
+  Visit(const T& type) {
+    return VisitNative<T>();
+  }
+
+  Status Visit(const Date32Type& type) { return VisitNative<Date32Type>(); }
+  Status Visit(const Date64Type& type) { return VisitNative<Int64Type>(); }
+  Status Visit(const TimestampType& type) { return VisitNative<TimestampType>(); }
+  Status Visit(const Time32Type& type) { return VisitNative<Int32Type>(); }
+  Status Visit(const Time64Type& type) { return VisitNative<Int64Type>(); }
+
+  Status Visit(const NullType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const BinaryType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const FixedSizeBinaryType& type) {
+    return TypeNotImplemented(type.ToString());
+  }
+
+  Status Visit(const DecimalType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const DictionaryType& type) { return TypeNotImplemented(type.ToString()); }
+
+  Status Visit(const NestedType& type) { return TypeNotImplemented(type.ToString()); }
+
+ protected:
   Status InitNullBitmap() {
     int64_t null_bytes = BitUtil::BytesForBits(length_);
 
@@ -317,6 +332,32 @@ class PandasConverter {
     return Status::OK();
   }
 
+  template <int TYPE, typename BuilderType>
+  Status AppendNdarrayToBuilder(PyArrayObject* array, BuilderType* builder) {
+    typedef internal::npy_traits<TYPE> traits;
+    typedef typename traits::value_type T;
+
+    const bool null_sentinels_possible =
+        (use_pandas_null_sentinels_ && traits::supports_nulls);
+
+    // TODO(wesm): Vector append when not strided
+    Ndarray1DIndexer<T> values(array);
+    if (null_sentinels_possible) {
+      for (int64_t i = 0; i < values.size(); ++i) {
+        if (traits::isnull(values[i])) {
+          RETURN_NOT_OK(builder->AppendNull());
+        } else {
+          RETURN_NOT_OK(builder->Append(values[i]));
+        }
+      }
+    } else {
+      for (int64_t i = 0; i < values.size(); ++i) {
+        RETURN_NOT_OK(builder->Append(values[i]));
+      }
+    }
+    return Status::OK();
+  }
+
   Status PushArray(const std::shared_ptr<ArrayData>& data) {
     std::shared_ptr<Array> result;
     RETURN_NOT_OK(MakeArray(data, &result));
@@ -328,7 +369,10 @@ class PandasConverter {
   Status VisitNative() {
     using traits = internal::arrow_traits<ArrowType::type_id>;
 
-    if (mask_ != nullptr || traits::supports_nulls) {
+    const bool null_sentinels_possible =
+        (use_pandas_null_sentinels_ && traits::supports_nulls);
+
+    if (mask_ != nullptr || null_sentinels_possible) {
       RETURN_NOT_OK(InitNullBitmap());
     }
 
@@ -338,7 +382,7 @@ class PandasConverter {
     int64_t null_count = 0;
     if (mask_ != nullptr) {
       null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
-    } else if (traits::supports_nulls) {
+    } else if (null_sentinels_possible) {
       // TODO(wesm): this presumes the NumPy C type and arrow C type are the
       // same
       null_count = ValuesToBitmap<traits::npy_type>(arr_, null_bitmap_data_);
@@ -350,58 +394,17 @@ class PandasConverter {
     return PushArray(arr_data);
   }
 
-  template <typename T>
-  typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
-                              std::is_same<BooleanType, T>::value,
-                          Status>::type
-  Visit(const T& type) {
-    return VisitNative<T>();
-  }
-
-  Status Visit(const Date32Type& type) { return VisitNative<Date32Type>(); }
-  Status Visit(const Date64Type& type) { return VisitNative<Int64Type>(); }
-  Status Visit(const TimestampType& type) { return VisitNative<TimestampType>(); }
-  Status Visit(const Time32Type& type) { return VisitNative<Int32Type>(); }
-  Status Visit(const Time64Type& type) { return VisitNative<Int64Type>(); }
-
   Status TypeNotImplemented(std::string type_name) {
     std::stringstream ss;
-    ss << "PandasConverter doesn't implement <" << type_name << "> conversion. ";
+    ss << "NumPyConverter doesn't implement <" << type_name << "> conversion. ";
     return Status::NotImplemented(ss.str());
   }
 
-  Status Visit(const NullType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Visit(const BinaryType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Visit(const FixedSizeBinaryType& type) {
-    return TypeNotImplemented(type.ToString());
-  }
-
-  Status Visit(const DecimalType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Visit(const DictionaryType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Visit(const NestedType& type) { return TypeNotImplemented(type.ToString()); }
-
-  Status Convert() {
-    if (PyArray_NDIM(arr_) != 1) {
-      return Status::Invalid("only handle 1-dimensional arrays");
-    }
-
-    if (type_ == nullptr) {
-      return Status::Invalid("Must pass data type");
-    }
-
-    // Visit the type to perform conversion
-    return VisitTypeInline(*type_, this);
-  }
-
-  const std::vector<std::shared_ptr<Array>>& result() const { return out_arrays_; }
-
   // ----------------------------------------------------------------------
   // Conversion logic for various object dtype arrays
 
+  Status ConvertObjects();
+
   template <int ITEM_TYPE, typename ArrowType>
   Status ConvertTypedLists(const std::shared_ptr<DataType>& type, ListBuilder* builder,
                            PyObject* list);
@@ -419,17 +422,17 @@ class PandasConverter {
                       PyObject* list);
   Status ConvertDecimals();
   Status ConvertTimes();
-  Status ConvertObjects();
   Status ConvertObjectsInfer();
   Status ConvertObjectsInferAndCast();
 
- protected:
   MemoryPool* pool_;
   std::shared_ptr<DataType> type_;
   PyArrayObject* arr_;
   PyArrayObject* mask_;
   int64_t length_;
 
+  bool use_pandas_null_sentinels_;
+
   // Used in visitor pattern
   std::vector<std::shared_ptr<Array>> out_arrays_;
 
@@ -437,6 +440,23 @@ class PandasConverter {
   uint8_t* null_bitmap_data_;
 };
 
+Status NumPyConverter::Convert() {
+  if (PyArray_NDIM(arr_) != 1) {
+    return Status::Invalid("only handle 1-dimensional arrays");
+  }
+
+  if (PyArray_DESCR(arr_)->type_num == NPY_OBJECT) {
+    return ConvertObjects();
+  }
+
+  if (type_ == nullptr) {
+    return Status::Invalid("Must pass data type for non-object arrays");
+  }
+
+  // Visit the type to perform conversion
+  return VisitTypeInline(*type_, this);
+}
+
 template <typename T, typename T2>
 void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) {
   // Passing input_data as non-const is a concession to PyObject*
@@ -482,7 +502,7 @@ static Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t len
 }
 
 template <typename ArrowType>
-inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
+inline Status NumPyConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   using traits = internal::arrow_traits<ArrowType::type_id>;
   using T = typename traits::T;
 
@@ -513,7 +533,7 @@ inline Status PandasConverter::ConvertData(std::shared_ptr<Buffer>* data) {
 }
 
 template <>
-inline Status PandasConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* data) {
+inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* data) {
   // Handle LONGLONG->INT64 and other fun things
   int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
   int type_size = NumPyTypeSize(type_num_compat);
@@ -552,7 +572,7 @@ inline Status PandasConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>*
 }
 
 template <>
-inline Status PandasConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>* data) {
+inline Status NumPyConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>* data) {
   int64_t nbytes = BitUtil::BytesForBits(length_);
   auto buffer = std::make_shared<PoolBuffer>(pool_);
   RETURN_NOT_OK(buffer->Resize(nbytes));
@@ -590,7 +610,7 @@ struct UnboxDate<Date64Type> {
 };
 
 template <typename ArrowType>
-Status PandasConverter::ConvertDates() {
+Status NumPyConverter::ConvertDates() {
   PyAcquireGIL lock;
 
   using BuilderType = typename TypeTraits<ArrowType>::BuilderType;
@@ -626,7 +646,7 @@ Status PandasConverter::ConvertDates() {
   return PushBuilderResult(&builder);
 }
 
-Status PandasConverter::ConvertDecimals() {
+Status NumPyConverter::ConvertDecimals() {
   PyAcquireGIL lock;
 
   // Import the decimal module and Decimal class
@@ -669,7 +689,7 @@ Status PandasConverter::ConvertDecimals() {
   return PushBuilderResult(&builder);
 }
 
-Status PandasConverter::ConvertTimes() {
+Status NumPyConverter::ConvertTimes() {
   // Convert array of datetime.time objects to Arrow
   PyAcquireGIL lock;
   PyDateTime_IMPORT;
@@ -697,7 +717,7 @@ Status PandasConverter::ConvertTimes() {
   return PushBuilderResult(&builder);
 }
 
-Status PandasConverter::ConvertObjectStrings() {
+Status NumPyConverter::ConvertObjectStrings() {
   PyAcquireGIL lock;
 
   // The output type at this point is inconclusive because there may be bytes
@@ -729,7 +749,7 @@ Status PandasConverter::ConvertObjectStrings() {
   return Status::OK();
 }
 
-Status PandasConverter::ConvertObjectFloats() {
+Status NumPyConverter::ConvertObjectFloats() {
   PyAcquireGIL lock;
 
   Ndarray1DIndexer<PyObject*> objects(arr_);
@@ -764,7 +784,7 @@ Status PandasConverter::ConvertObjectFloats() {
   return PushBuilderResult(&builder);
 }
 
-Status PandasConverter::ConvertObjectIntegers() {
+Status NumPyConverter::ConvertObjectIntegers() {
   PyAcquireGIL lock;
 
   Int64Builder builder(pool_);
@@ -799,7 +819,7 @@ Status PandasConverter::ConvertObjectIntegers() {
   return PushBuilderResult(&builder);
 }
 
-Status PandasConverter::ConvertObjectFixedWidthBytes(
+Status NumPyConverter::ConvertObjectFixedWidthBytes(
     const std::shared_ptr<DataType>& type) {
   PyAcquireGIL lock;
 
@@ -822,7 +842,7 @@ Status PandasConverter::ConvertObjectFixedWidthBytes(
   return Status::OK();
 }
 
-Status PandasConverter::ConvertBooleans() {
+Status NumPyConverter::ConvertBooleans() {
   PyAcquireGIL lock;
 
   Ndarray1DIndexer<PyObject*> objects(arr_);
@@ -864,7 +884,7 @@ Status PandasConverter::ConvertBooleans() {
   return Status::OK();
 }
 
-Status PandasConverter::ConvertObjectsInfer() {
+Status NumPyConverter::ConvertObjectsInfer() {
   Ndarray1DIndexer<PyObject*> objects;
 
   PyAcquireGIL lock;
@@ -912,7 +932,7 @@ Status PandasConverter::ConvertObjectsInfer() {
   return Status::OK();
 }
 
-Status PandasConverter::ConvertObjectsInferAndCast() {
+Status NumPyConverter::ConvertObjectsInferAndCast() {
   size_t position = out_arrays_.size();
   RETURN_NOT_OK(ConvertObjectsInfer());
 
@@ -932,7 +952,7 @@ Status PandasConverter::ConvertObjectsInferAndCast() {
   return Status::OK();
 }
 
-Status PandasConverter::ConvertObjects() {
+Status NumPyConverter::ConvertObjects() {
   // Python object arrays are annoying, since we could have one of:
   //
   // * Strings
@@ -1005,8 +1025,8 @@ Status LoopPySequence(PyObject* sequence, T func) {
 }
 
 template <int ITEM_TYPE, typename ArrowType>
-inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>& type,
-                                                 ListBuilder* builder, PyObject* list) {
+inline Status NumPyConverter::ConvertTypedLists(const std::shared_ptr<DataType>& type,
+                                                ListBuilder* builder, PyObject* list) {
   typedef internal::npy_traits<ITEM_TYPE> traits;
   typedef typename traits::BuilderClass BuilderT;
 
@@ -1050,7 +1070,7 @@ inline Status PandasConverter::ConvertTypedLists(const std::shared_ptr<DataType>
 }
 
 template <>
-inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
+inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
     const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
   PyAcquireGIL lock;
 
@@ -1091,7 +1111,7 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
 }
 
 template <>
-inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
+inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
   PyAcquireGIL lock;
   // TODO: If there are bytes involed, convert to Binary representation
@@ -1145,8 +1165,8 @@ inline Status PandasConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     return ConvertTypedLists<NUMPY_TYPE, ArrowType>(type, builder, list); \
   }
 
-Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type,
-                                     ListBuilder* builder, PyObject* list) {
+Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type,
+                                    ListBuilder* builder, PyObject* list) {
   switch (type->id()) {
     LIST_CASE(NA, NPY_OBJECT, NullType)
     LIST_CASE(UINT8, NPY_UINT8, UInt8Type)
@@ -1185,7 +1205,7 @@ Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type,
   }
 }
 
-Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
+Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
   std::unique_ptr<ArrayBuilder> array_builder;
   RETURN_NOT_OK(MakeBuilder(pool_, arrow::list(type), &array_builder));
   ListBuilder* list_builder = static_cast<ListBuilder*>(array_builder.get());
@@ -1193,20 +1213,13 @@ Status PandasConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
   return PushBuilderResult(list_builder);
 }
 
-Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-                     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out) {
-  PandasConverter converter(pool, ao, mo, type);
+Status NdarrayToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+                      bool use_pandas_null_sentinels,
+                      const std::shared_ptr<DataType>& type,
+                      std::shared_ptr<ChunkedArray>* out) {
+  NumPyConverter converter(pool, ao, mo, type, use_pandas_null_sentinels);
   RETURN_NOT_OK(converter.Convert());
-  *out = converter.result()[0];
-  DCHECK(*out);
-  return Status::OK();
-}
-
-Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-                            const std::shared_ptr<DataType>& type,
-                            std::shared_ptr<ChunkedArray>* out) {
-  PandasConverter converter(pool, ao, mo, type);
-  RETURN_NOT_OK(converter.ConvertObjects());
+  DCHECK(converter.result()[0]);
   *out = std::make_shared<ChunkedArray>(converter.result());
   return Status::OK();
 }
diff --git a/cpp/src/arrow/python/pandas_to_arrow.h b/cpp/src/arrow/python/numpy_to_arrow.h
similarity index 70%
rename from cpp/src/arrow/python/pandas_to_arrow.h
rename to cpp/src/arrow/python/numpy_to_arrow.h
index 3e655ba3feec0..4a70b4bc5333b 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.h
+++ b/cpp/src/arrow/python/numpy_to_arrow.h
@@ -17,8 +17,8 @@
 
 // Converting from pandas memory representation to Arrow data structures
 
-#ifndef ARROW_PYTHON_PANDAS_TO_ARROW_H
-#define ARROW_PYTHON_PANDAS_TO_ARROW_H
+#ifndef ARROW_PYTHON_NUMPY_TO_ARROW_H
+#define ARROW_PYTHON_NUMPY_TO_ARROW_H
 
 #include "arrow/python/platform.h"
 
@@ -36,12 +36,8 @@ class Status;
 
 namespace py {
 
-ARROW_EXPORT
-Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-                     const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* out);
-
-/// Convert dtype=object arrays. If target data type is not known, pass a type
-/// with nullptr
+/// Convert NumPy arrays to Arrow. If target data type is not known, pass a
+/// type with nullptr
 ///
 /// \param[in] pool Memory pool for any memory allocations
 /// \param[in] ao an ndarray with the array data
@@ -49,11 +45,12 @@ Status PandasToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
 /// \param[in] type
 /// \param[out] out a ChunkedArray, to accommodate chunked output
 ARROW_EXPORT
-Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
-                            const std::shared_ptr<DataType>& type,
-                            std::shared_ptr<ChunkedArray>* out);
+Status NdarrayToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
+                      bool use_pandas_null_sentinels,
+                      const std::shared_ptr<DataType>& type,
+                      std::shared_ptr<ChunkedArray>* out);
 
 }  // namespace py
 }  // namespace arrow
 
-#endif  // ARROW_PYTHON_PANDAS_TO_ARROW_H
+#endif  // ARROW_PYTHON_NUMPY_TO_ARROW_H
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 0d76a35f4ed3c..ac06948227454 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -36,7 +36,7 @@
                          time32, time64, timestamp, date32, date64,
                          float16, float32, float64,
                          binary, string, decimal,
-                         list_, struct, dictionary, field,
+                         list_, struct, dictionary, field, type_for_alias,
                          DataType, NAType,
                          Field,
                          Schema,
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index eec6180165c27..f402defc9b06f 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -16,58 +16,161 @@
 # under the License.
 
 
-def array(object sequence, DataType type=None, MemoryPool memory_pool=None,
-          size=None):
+cdef _sequence_to_array(object sequence, object size, DataType type,
+                        CMemoryPool* pool):
+    cdef shared_ptr[CArray] out
+    cdef int64_t c_size
+    if type is None:
+        with nogil:
+            check_status(ConvertPySequence(sequence, pool, &out))
+    else:
+        if size is None:
+            with nogil:
+                check_status(
+                    ConvertPySequence(
+                        sequence, pool, &out, type.sp_type
+                    )
+                )
+        else:
+            c_size = size
+            with nogil:
+                check_status(
+                    ConvertPySequence(
+                        sequence, pool, &out, type.sp_type, c_size
+                    )
+                )
+
+    return pyarrow_wrap_array(out)
+
+
+cdef _is_array_like(obj):
+    try:
+        import pandas
+        return isinstance(obj, (np.ndarray, pd.Series, pd.Index, Categorical))
+    except:
+        return isinstance(obj, np.ndarray)
+
+
+cdef _ndarray_to_array(object values, object mask, DataType type,
+                       c_bool use_pandas_null_sentinels,
+                       CMemoryPool* pool):
+    cdef shared_ptr[CChunkedArray] chunked_out
+    cdef shared_ptr[CDataType] c_type
+
+    dtype = values.dtype
+
+    if type is None and dtype != object:
+        with nogil:
+            check_status(NumPyDtypeToArrow(dtype, &c_type))
+
+    if type is not None:
+        c_type = type.sp_type
+
+    with nogil:
+        check_status(NdarrayToArrow(pool, values, mask,
+                                    use_pandas_null_sentinels,
+                                    c_type, &chunked_out))
+
+    if chunked_out.get().num_chunks() > 1:
+        return pyarrow_wrap_chunked_array(chunked_out)
+    else:
+        return pyarrow_wrap_array(chunked_out.get().chunk(0))
+
+
+cdef DataType _ensure_type(object type):
+    if type is None:
+        return None
+    elif not isinstance(type, DataType):
+        return type_for_alias(type)
+    else:
+        return type
+
+
+def array(object obj, type=None, mask=None,
+          MemoryPool memory_pool=None, size=None,
+          from_pandas=False):
     """
-    Create pyarrow.Array instance from a Python sequence
+    Create pyarrow.Array instance from a Python object
 
     Parameters
     ----------
-    sequence : sequence-like or iterable object of Python objects.
-        If both type and size are specified may be a single use iterable.
-    type : pyarrow.DataType, optional
-        If not passed, will be inferred from the data
+    obj : sequence, iterable, ndarray or Series
+        If both type and size are specified may be a single use iterable. If
+        not strongly-typed, Arrow type will be inferred for resulting array
+    mask : array (boolean), optional
+        Indicate which values are null (True) or not null (False).
+    type : pyarrow.DataType
+        Explicit type to attempt to coerce to, otherwise will be inferred from
+        the data
     memory_pool : pyarrow.MemoryPool, optional
         If not passed, will allocate memory from the currently-set default
         memory pool
+
     size : int64, optional
         Size of the elements. If the imput is larger than size bail at this
         length. For iterators, if size is larger than the input iterator this
         will be treated as a "max size", but will involve an initial allocation
         of size followed by a resize to the actual size (so if you know the
         exact size specifying it correctly will give you better performance).
+    from_pandas : boolean, default False
+        Use pandas's semantics for inferring nulls from values in ndarray-like
+        data. If passed, the mask tasks precendence, but if a value is unmasked
+        (not-null), but still null according to pandas semantics, then it is
+        null
+
+    Notes
+    -----
+    Localized timestamps will currently be returned as UTC (pandas's native
+    representation).  Timezone-naive data will be implicitly interpreted as
+    UTC.
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> import pyarrow as pa
+    >>> pa.array(pd.Series([1, 2]))
+    <pyarrow.array.Int64Array object at 0x7f674e4c0e10>
+    [
+      1,
+      2
+    ]
+
+    >>> import numpy as np
+    >>> pa.array(pd.Series([1, 2]), np.array([0, 1],
+    ... dtype=bool))
+    <pyarrow.array.Int64Array object at 0x7f9019e11208>
+    [
+      1,
+      NA
+    ]
 
     Returns
     -------
-    array : pyarrow.Array
+    array : pyarrow.Array or pyarrow.ChunkedArray (if object data
+    overflowed binary storage)
     """
-    cdef:
-        shared_ptr[CArray] sp_array
-        CMemoryPool* pool
-        int64_t c_size
+    type = _ensure_type(type)
+    cdef CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)
 
-    pool = maybe_unbox_memory_pool(memory_pool)
-    if type is None:
-        with nogil:
-            check_status(ConvertPySequence(sequence, pool, &sp_array))
-    else:
-        if size is None:
-            with nogil:
-                check_status(
-                    ConvertPySequence(
-                        sequence, pool, &sp_array, type.sp_type
-                    )
-                )
-        else:
-            c_size = size
-            with nogil:
-                check_status(
-                    ConvertPySequence(
-                        sequence, pool, &sp_array, type.sp_type, c_size
-                    )
-                )
+    if _is_array_like(obj):
+        if mask is not None:
+            mask = get_series_values(mask)
+
+        values = get_series_values(obj)
 
-    return pyarrow_wrap_array(sp_array)
+        if isinstance(values, Categorical):
+            return DictionaryArray.from_arrays(
+                values.codes, values.categories.values,
+                mask=mask, ordered=values.ordered,
+                memory_pool=memory_pool)
+        else:
+            values, type = pdcompat.get_datetimetz_type(values, obj.dtype,
+                                                        type)
+            return _ndarray_to_array(values, mask, type, from_pandas, pool)
+    else:
+        if mask is not None:
+            raise ValueError("Masks only supported with ndarray-like inputs")
+        return _sequence_to_array(obj, size, type, pool)
 
 
 def _normalize_slice(object arrow_obj, slice key):
@@ -112,7 +215,7 @@ cdef class Array:
         with nogil:
             check_status(DebugPrint(deref(self.ap), 0))
 
-    def cast(self, DataType target_type, safe=True):
+    def cast(self, object target_type, safe=True):
         """
         Cast array values to another data type
 
@@ -130,42 +233,37 @@ cdef class Array:
         cdef:
             CCastOptions options
             shared_ptr[CArray] result
+            DataType type
+
+        type = _ensure_type(target_type)
 
         if not safe:
             options.allow_int_overflow = 1
 
         with nogil:
-            check_status(Cast(_context(), self.ap[0], target_type.sp_type,
+            check_status(Cast(_context(), self.ap[0], type.sp_type,
                               options, &result))
 
         return pyarrow_wrap_array(result)
 
     @staticmethod
-    def from_pandas(obj, mask=None, DataType type=None,
-                    timestamps_to_ms=False,
-                    MemoryPool memory_pool=None):
+    def from_pandas(obj, mask=None, type=None, MemoryPool memory_pool=None):
         """
-        Convert pandas.Series to an Arrow Array.
+        Convert pandas.Series to an Arrow Array, using pandas's semantics about
+        what values indicate nulls. See pyarrow.array for more general
+        conversion from arrays or sequences to Arrow arrays
 
         Parameters
         ----------
-        series : pandas.Series or numpy.ndarray
-
-        mask : pandas.Series or numpy.ndarray, optional
-            boolean mask if the object is null (True) or valid (False)
-
+        sequence : ndarray, Inded Series
+        mask : array (boolean), optional
+            Indicate which values are null (True) or not null (False)
         type : pyarrow.DataType
-            Explicit type to attempt to coerce to
-
-        timestamps_to_ms : bool, optional
-            Convert datetime columns to ms resolution. This is needed for
-            compatibility with other functionality like Parquet I/O which
-            only supports milliseconds.
-
-            .. deprecated:: 0.7.0
-
-        memory_pool: MemoryPool, optional
-            Specific memory pool to use to allocate the resulting Arrow array.
+            Explicit type to attempt to coerce to, otherwise will be inferred
+            from the data
+        memory_pool : pyarrow.MemoryPool, optional
+            If not passed, will allocate memory from the currently-set default
+            memory pool
 
         Notes
         -----
@@ -173,78 +271,13 @@ cdef class Array:
         representation).  Timezone-naive data will be implicitly interpreted as
         UTC.
 
-        Examples
-        --------
-
-        >>> import pandas as pd
-        >>> import pyarrow as pa
-        >>> pa.Array.from_pandas(pd.Series([1, 2]))
-        <pyarrow.array.Int64Array object at 0x7f674e4c0e10>
-        [
-          1,
-          2
-        ]
-
-        >>> import numpy as np
-        >>> pa.Array.from_pandas(pd.Series([1, 2]), np.array([0, 1],
-        ... dtype=bool))
-        <pyarrow.array.Int64Array object at 0x7f9019e11208>
-        [
-          1,
-          NA
-        ]
-
         Returns
         -------
         array : pyarrow.Array or pyarrow.ChunkedArray (if object data
-        overflowed binary storage)
+        overflows binary buffer)
         """
-        cdef:
-            shared_ptr[CArray] out
-            shared_ptr[CChunkedArray] chunked_out
-            shared_ptr[CDataType] c_type
-            CMemoryPool* pool
-
-        if mask is not None:
-            mask = get_series_values(mask)
-
-        values = get_series_values(obj)
-        pool = maybe_unbox_memory_pool(memory_pool)
-
-        if isinstance(values, Categorical):
-            return DictionaryArray.from_arrays(
-                values.codes, values.categories.values,
-                mask=mask, ordered=values.ordered,
-                memory_pool=memory_pool)
-        elif values.dtype == object:
-            # Object dtype undergoes a different conversion path as more type
-            # inference may be needed
-            if type is not None:
-                c_type = type.sp_type
-            with nogil:
-                check_status(PandasObjectsToArrow(
-                    pool, values, mask, c_type, &chunked_out))
-
-            if chunked_out.get().num_chunks() > 1:
-                return pyarrow_wrap_chunked_array(chunked_out)
-            else:
-                out = chunked_out.get().chunk(0)
-        else:
-            values, type = pdcompat.maybe_coerce_datetime64(
-                values, obj.dtype, type, timestamps_to_ms=timestamps_to_ms)
-
-            if type is None:
-                dtype = values.dtype
-                with nogil:
-                    check_status(NumPyDtypeToArrow(dtype, &c_type))
-            else:
-                c_type = type.sp_type
-
-            with nogil:
-                check_status(PandasToArrow(
-                    pool, values, mask, c_type, &out))
-
-        return pyarrow_wrap_array(out)
+        return array(obj, mask=mask, type=type, memory_pool=memory_pool,
+                     from_pandas=True)
 
     property null_count:
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 5e6708871e650..fc17d1c06ae0a 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -766,13 +766,10 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
 
     CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
 
-    CStatus PandasToArrow(CMemoryPool* pool, object ao, object mo,
-                          const shared_ptr[CDataType]& type,
-                          shared_ptr[CArray]* out)
-
-    CStatus PandasObjectsToArrow(CMemoryPool* pool, object ao, object mo,
-                                 const shared_ptr[CDataType]& type,
-                                 shared_ptr[CChunkedArray]* out)
+    CStatus NdarrayToArrow(CMemoryPool* pool, object ao, object mo,
+                           c_bool use_pandas_null_sentinels,
+                           const shared_ptr[CDataType]& type,
+                           shared_ptr[CChunkedArray]* out)
 
     CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
                             shared_ptr[CTensor]* out)
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index d1e6f5a8096e7..be48aeb442d38 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -203,7 +203,7 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
     }
 
 
-def dataframe_to_arrays(df, timestamps_to_ms, schema, preserve_index):
+def dataframe_to_arrays(df, schema, preserve_index):
     names = []
     arrays = []
     index_columns = []
@@ -223,15 +223,13 @@ def dataframe_to_arrays(df, timestamps_to_ms, schema, preserve_index):
             field = schema.field_by_name(name)
             type = getattr(field, "type", None)
 
-        array = pa.Array.from_pandas(
-            col, type=type, timestamps_to_ms=timestamps_to_ms
-        )
+        array = pa.array(col, from_pandas=True, type=type)
         arrays.append(array)
         names.append(name)
         types.append(array.type)
 
     for i, column in enumerate(index_columns):
-        array = pa.Array.from_pandas(column, timestamps_to_ms=timestamps_to_ms)
+        array = pa.array(column)
         arrays.append(array)
         names.append(index_level_name(column, i))
         types.append(array.type)
@@ -242,25 +240,15 @@ def dataframe_to_arrays(df, timestamps_to_ms, schema, preserve_index):
     return names, arrays, metadata
 
 
-def maybe_coerce_datetime64(values, dtype, type_, timestamps_to_ms=False):
-    if timestamps_to_ms:
-        import warnings
-        warnings.warn('timestamps_to_ms=True is deprecated', FutureWarning)
-
+def get_datetimetz_type(values, dtype, type_):
     from pyarrow.compat import DatetimeTZDtype
 
     if values.dtype.type != np.datetime64:
         return values, type_
 
-    coerce_ms = timestamps_to_ms and values.dtype != 'datetime64[ms]'
-
-    if coerce_ms:
-        values = values.astype('datetime64[ms]')
-        type_ = pa.timestamp('ms')
-
     if isinstance(dtype, DatetimeTZDtype):
         tz = dtype.tz
-        unit = 'ms' if coerce_ms else dtype.unit
+        unit = dtype.unit
         type_ = pa.timestamp(unit, tz)
     elif type_ is None:
         # Trust the NumPy dtype
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index 3a847f77c4f81..c37ed3b200ea3 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -348,10 +348,10 @@ cdef class StructValue(ArrayValue):
 
 cdef dict _scalar_classes = {
     _Type_BOOL: BooleanValue,
-    _Type_UINT8: Int8Value,
-    _Type_UINT16: Int16Value,
-    _Type_UINT32: Int32Value,
-    _Type_UINT64: Int64Value,
+    _Type_UINT8: UInt8Value,
+    _Type_UINT16: UInt16Value,
+    _Type_UINT32: UInt32Value,
+    _Type_UINT64: UInt64Value,
     _Type_INT8: Int8Value,
     _Type_INT16: Int16Value,
     _Type_INT32: Int32Value,
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 028797e45b8d9..e5422a5beca3b 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -575,7 +575,7 @@ cdef class RecordBatch:
         pyarrow.RecordBatch
         """
         names, arrays, metadata = pdcompat.dataframe_to_arrays(
-            df, False, schema, preserve_index
+            df, schema, preserve_index
         )
         return cls.from_arrays(arrays, names, metadata)
 
@@ -714,21 +714,13 @@ cdef class Table:
         return result
 
     @classmethod
-    def from_pandas(cls, df, bint timestamps_to_ms=False,
-                    Schema schema=None, bint preserve_index=True):
+    def from_pandas(cls, df, Schema schema=None, bint preserve_index=True):
         """
         Convert pandas.DataFrame to an Arrow Table
 
         Parameters
         ----------
         df : pandas.DataFrame
-        timestamps_to_ms : bool
-            Convert datetime columns to ms resolution. This is needed for
-            compability with other functionality like Parquet I/O which
-            only supports milliseconds.
-
-            .. deprecated:: 0.7.0
-
         schema : pyarrow.Schema, optional
             The expected schema of the Arrow Table. This can be used to
             indicate the type of columns if we cannot infer it automatically.
@@ -754,7 +746,6 @@ cdef class Table:
         """
         names, arrays, metadata = pdcompat.dataframe_to_arrays(
             df,
-            timestamps_to_ms=timestamps_to_ms,
             schema=schema,
             preserve_index=preserve_index
         )
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index f316417caafb0..3bf392686f0bc 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -149,6 +149,14 @@ def test_array_factory_invalid_type():
         pa.array(arr)
 
 
+def test_array_ref_to_ndarray_base():
+    arr = np.array([1, 2, 3])
+
+    refcount = sys.getrefcount(arr)
+    arr2 = pa.array(arr)  # noqa
+    assert sys.getrefcount(arr) == (refcount + 1)
+
+
 def test_dictionary_from_numpy():
     indices = np.repeat([0, 1, 2], 2)
     dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
@@ -170,8 +178,8 @@ def test_dictionary_from_boxed_arrays():
     indices = np.repeat([0, 1, 2], 2)
     dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
 
-    iarr = pa.Array.from_pandas(indices)
-    darr = pa.Array.from_pandas(dictionary)
+    iarr = pa.array(indices)
+    darr = pa.array(dictionary)
 
     d1 = pa.DictionaryArray.from_arrays(iarr, darr)
 
@@ -201,9 +209,9 @@ def test_dictionary_with_pandas():
 
 def test_list_from_arrays():
     offsets_arr = np.array([0, 2, 5, 8], dtype='i4')
-    offsets = pa.Array.from_pandas(offsets_arr, type=pa.int32())
+    offsets = pa.array(offsets_arr, type='int32')
     pyvalues = [b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h']
-    values = pa.array(pyvalues, type=pa.binary())
+    values = pa.array(pyvalues, type='binary')
 
     result = pa.ListArray.from_arrays(offsets, values)
     expected = pa.array([pyvalues[:2], pyvalues[2:5], pyvalues[5:8]])
@@ -214,22 +222,22 @@ def test_list_from_arrays():
 def _check_cast_case(case, safe=True):
     in_data, in_type, out_data, out_type = case
 
-    in_arr = pa.Array.from_pandas(in_data, type=in_type)
+    in_arr = pa.array(in_data, type=in_type)
 
     casted = in_arr.cast(out_type, safe=safe)
-    expected = pa.Array.from_pandas(out_data, type=out_type)
+    expected = pa.array(out_data, type=out_type)
     assert casted.equals(expected)
 
 
 def test_cast_integers_safe():
     safe_cases = [
-        (np.array([0, 1, 2, 3], dtype='i1'), pa.int8(),
+        (np.array([0, 1, 2, 3], dtype='i1'), 'int8',
          np.array([0, 1, 2, 3], dtype='i4'), pa.int32()),
-        (np.array([0, 1, 2, 3], dtype='i1'), pa.int8(),
+        (np.array([0, 1, 2, 3], dtype='i1'), 'int8',
          np.array([0, 1, 2, 3], dtype='u4'), pa.uint16()),
-        (np.array([0, 1, 2, 3], dtype='i1'), pa.int8(),
+        (np.array([0, 1, 2, 3], dtype='i1'), 'int8',
          np.array([0, 1, 2, 3], dtype='u1'), pa.uint8()),
-        (np.array([0, 1, 2, 3], dtype='i1'), pa.int8(),
+        (np.array([0, 1, 2, 3], dtype='i1'), 'int8',
          np.array([0, 1, 2, 3], dtype='f8'), pa.float64())
     ]
 
@@ -237,13 +245,13 @@ def test_cast_integers_safe():
         _check_cast_case(case)
 
     unsafe_cases = [
-        (np.array([50000], dtype='i4'), pa.int32(), pa.int16()),
-        (np.array([70000], dtype='i4'), pa.int32(), pa.uint16()),
-        (np.array([-1], dtype='i4'), pa.int32(), pa.uint16()),
-        (np.array([50000], dtype='u2'), pa.uint16(), pa.int16())
+        (np.array([50000], dtype='i4'), 'int32', 'int16'),
+        (np.array([70000], dtype='i4'), 'int32', 'uint16'),
+        (np.array([-1], dtype='i4'), 'int32', 'uint16'),
+        (np.array([50000], dtype='u2'), 'uint16', 'int16')
     ]
     for in_data, in_type, out_type in unsafe_cases:
-        in_arr = pa.Array.from_pandas(in_data, type=in_type)
+        in_arr = pa.array(in_data, type=in_type)
 
         with pytest.raises(pa.ArrowInvalid):
             in_arr.cast(out_type)
@@ -252,11 +260,11 @@ def test_cast_integers_safe():
 def test_cast_integers_unsafe():
     # We let NumPy do the unsafe casting
     unsafe_cases = [
-        (np.array([50000], dtype='i4'), pa.int32(),
+        (np.array([50000], dtype='i4'), 'int32',
          np.array([50000], dtype='i2'), pa.int16()),
-        (np.array([70000], dtype='i4'), pa.int32(),
+        (np.array([70000], dtype='i4'), 'int32',
          np.array([70000], dtype='u2'), pa.uint16()),
-        (np.array([-1], dtype='i4'), pa.int32(),
+        (np.array([-1], dtype='i4'), 'int32',
          np.array([-1], dtype='u2'), pa.uint16()),
         (np.array([50000], dtype='u2'), pa.uint16(),
          np.array([50000], dtype='i2'), pa.int16())
@@ -315,3 +323,17 @@ def test_simple_type_construction():
 )
 def test_logical_type(type, expected):
     assert get_logical_type(type) == expected
+
+
+def test_array_conversions_no_sentinel_values():
+    arr = np.array([1, 2, 3, 4], dtype='int8')
+    refcount = sys.getrefcount(arr)
+    arr2 = pa.array(arr)  # noqa
+    assert sys.getrefcount(arr) == (refcount + 1)
+
+    assert arr2.type == 'int8'
+
+    arr3 = pa.array(np.array([1, np.nan, 2, 3, np.nan, 4], dtype='float32'),
+                    type='float32')
+    assert arr3.type == 'float32'
+    assert arr3.null_count == 0
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 5d56cde7d48b9..182f3afc7e63e 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -18,7 +18,7 @@
 
 from collections import OrderedDict
 
-from datetime import datetime, date, time
+from datetime import date, time
 import unittest
 import decimal
 import json
@@ -82,7 +82,7 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
         tm.assert_frame_equal(result, expected, check_dtype=check_dtype)
 
     def _check_series_roundtrip(self, s, type_=None):
-        arr = pa.Array.from_pandas(s, type=type_)
+        arr = pa.array(s, from_pandas=True, type=type_)
 
         result = pd.Series(arr.to_pandas(), name=s.name)
         if isinstance(arr.type, pa.TimestampType) and arr.type.tz is not None:
@@ -93,7 +93,7 @@ def _check_series_roundtrip(self, s, type_=None):
 
     def _check_array_roundtrip(self, values, expected=None, mask=None,
                                type=None):
-        arr = pa.Array.from_pandas(values, mask=mask, type=type)
+        arr = pa.array(values, from_pandas=True, mask=mask, type=type)
         result = arr.to_pandas()
 
         values_nulls = pd.isnull(values)
@@ -152,7 +152,7 @@ def test_float_nulls(self):
         for name, arrow_dtype in dtypes:
             values = np.random.randn(num_values).astype(name)
 
-            arr = pa.Array.from_pandas(values, null_mask)
+            arr = pa.array(values, from_pandas=True, mask=null_mask)
             arrays.append(arr)
             fields.append(pa.field(name, arrow_dtype))
             values[null_mask] = np.nan
@@ -223,7 +223,7 @@ def test_integer_with_nulls(self):
         for name in int_dtypes:
             values = np.random.randint(0, 100, size=num_values)
 
-            arr = pa.Array.from_pandas(values, null_mask)
+            arr = pa.array(values, mask=null_mask)
             arrays.append(arr)
 
             expected = values.astype('f8')
@@ -244,8 +244,8 @@ def test_array_from_pandas_type_cast(self):
 
         target_type = pa.int8()
 
-        result = pa.Array.from_pandas(arr, type=target_type)
-        expected = pa.Array.from_pandas(arr.astype('int8'))
+        result = pa.array(arr, type=target_type)
+        expected = pa.array(arr.astype('int8'))
         assert result.equals(expected)
 
     def test_boolean_no_nulls(self):
@@ -266,7 +266,7 @@ def test_boolean_nulls(self):
         mask = np.random.randint(0, 10, size=num_values) < 3
         values = np.random.randint(0, 10, size=num_values) < 5
 
-        arr = pa.Array.from_pandas(values, mask)
+        arr = pa.array(values, mask=mask)
 
         expected = values.astype(object)
         expected[mask] = None
@@ -292,7 +292,7 @@ def test_all_nulls_cast_numeric(self):
         arr = np.array([None], dtype=object)
 
         def _check_type(t):
-            a2 = pa.Array.from_pandas(arr, type=t)
+            a2 = pa.array(arr, type=t)
             assert a2.type == t
             assert a2[0].as_py() is None
 
@@ -325,7 +325,7 @@ def test_bytes_exceed_2gb(self):
         df = pd.DataFrame({
             'strings': np.array([val] * 4000, dtype=object)
         })
-        arr = pa.Array.from_pandas(df['strings'])
+        arr = pa.array(df['strings'])
         assert isinstance(arr, pa.ChunkedArray)
         assert arr.num_chunks == 2
         arr = None
@@ -365,19 +365,6 @@ def test_timestamps_notimezone_no_nulls(self):
             expected_schema=schema,
         )
 
-    def test_timestamps_to_ms_explicit_schema(self):
-        # ARROW-1328
-        df = pd.DataFrame({'datetime': [datetime(2017, 1, 1)]})
-        pa_type = pa.from_numpy_dtype(df['datetime'].dtype)
-
-        with tm.assert_produces_warning(FutureWarning,
-                                        check_stacklevel=False):
-            arr = pa.Array.from_pandas(df['datetime'], type=pa_type,
-                                       timestamps_to_ms=True)
-
-        tm.assert_almost_equal(df['datetime'].values.astype('M8[ms]'),
-                               arr.to_pandas())
-
     def test_timestamps_notimezone_nulls(self):
         df = pd.DataFrame({
             'datetime64': np.array([
@@ -450,11 +437,11 @@ def test_date_objects_typed(self):
         t32 = pa.date32()
         t64 = pa.date64()
 
-        a32 = pa.Array.from_pandas(arr, type=t32)
-        a64 = pa.Array.from_pandas(arr, type=t64)
+        a32 = pa.array(arr, type=t32)
+        a64 = pa.array(arr, type=t64)
 
-        a32_expected = pa.Array.from_pandas(arr_i4, mask=mask, type=t32)
-        a64_expected = pa.Array.from_pandas(arr_i8, mask=mask, type=t64)
+        a32_expected = pa.array(arr_i4, mask=mask, type=t32)
+        a64_expected = pa.array(arr_i8, mask=mask, type=t64)
 
         assert a32.equals(a32_expected)
         assert a64.equals(a64_expected)
@@ -481,8 +468,8 @@ def test_dates_from_integers(self):
         arr = np.array([17259, 17260, 17261], dtype='int32')
         arr2 = arr.astype('int64') * 86400000
 
-        a1 = pa.Array.from_pandas(arr, type=t1)
-        a2 = pa.Array.from_pandas(arr2, type=t2)
+        a1 = pa.array(arr, type=t1)
+        a2 = pa.array(arr2, type=t2)
 
         expected = date(2017, 4, 3)
         assert a1[0].as_py() == expected
@@ -520,7 +507,7 @@ def test_column_of_arrays_to_py(self):
             np.arange(1, dtype=dtype)
         ])
         type_ = pa.list_(pa.int8())
-        parr = pa.Array.from_pandas(arr, type=type_)
+        parr = pa.array(arr, type=type_)
 
         assert parr[0].as_py() == list(range(10))
         assert parr[1].as_py() == list(range(5))
@@ -592,7 +579,7 @@ def test_column_of_lists_strided(self):
     def test_nested_lists_all_none(self):
         data = np.array([[None, None], None], dtype=object)
 
-        arr = pa.Array.from_pandas(data)
+        arr = pa.array(data)
         expected = pa.array(list(data))
         assert arr.equals(expected)
         assert arr.type == pa.list_(pa.null())
@@ -600,7 +587,7 @@ def test_nested_lists_all_none(self):
         data2 = np.array([None, None, [None, None],
                           np.array([None, None], dtype=object)],
                          dtype=object)
-        arr = pa.Array.from_pandas(data2)
+        arr = pa.array(data2)
         expected = pa.array([None, None, [None, None], [None, None]])
         assert arr.equals(expected)
 
@@ -760,7 +747,7 @@ def test_pytime_from_pandas(self):
         t1 = pa.time64('us')
 
         aobjs = np.array(pytimes + [None], dtype=object)
-        parr = pa.Array.from_pandas(aobjs)
+        parr = pa.array(aobjs)
         assert parr.type == t1
         assert parr[0].as_py() == pytimes[0]
         assert parr[1].as_py() == pytimes[1]
@@ -775,18 +762,18 @@ def test_pytime_from_pandas(self):
         arr = np.array([_pytime_to_micros(v) for v in pytimes],
                        dtype='int64')
 
-        a1 = pa.Array.from_pandas(arr, type=pa.time64('us'))
+        a1 = pa.array(arr, type=pa.time64('us'))
         assert a1[0].as_py() == pytimes[0]
 
-        a2 = pa.Array.from_pandas(arr * 1000, type=pa.time64('ns'))
+        a2 = pa.array(arr * 1000, type=pa.time64('ns'))
         assert a2[0].as_py() == pytimes[0]
 
-        a3 = pa.Array.from_pandas((arr / 1000).astype('i4'),
-                                  type=pa.time32('ms'))
+        a3 = pa.array((arr / 1000).astype('i4'),
+                      type=pa.time32('ms'))
         assert a3[0].as_py() == pytimes[0].replace(microsecond=1000)
 
-        a4 = pa.Array.from_pandas((arr / 1000000).astype('i4'),
-                                  type=pa.time32('s'))
+        a4 = pa.array((arr / 1000000).astype('i4'),
+                      type=pa.time32('s'))
         assert a4[0].as_py() == pytimes[0].replace(microsecond=0)
 
     def test_arrow_time_to_pandas(self):
@@ -809,14 +796,14 @@ def test_arrow_time_to_pandas(self):
 
         null_mask = np.array([False, False, True], dtype=bool)
 
-        a1 = pa.Array.from_pandas(arr, mask=null_mask, type=pa.time64('us'))
-        a2 = pa.Array.from_pandas(arr * 1000, mask=null_mask,
-                                  type=pa.time64('ns'))
+        a1 = pa.array(arr, mask=null_mask, type=pa.time64('us'))
+        a2 = pa.array(arr * 1000, mask=null_mask,
+                      type=pa.time64('ns'))
 
-        a3 = pa.Array.from_pandas((arr / 1000).astype('i4'), mask=null_mask,
-                                  type=pa.time32('ms'))
-        a4 = pa.Array.from_pandas((arr / 1000000).astype('i4'), mask=null_mask,
-                                  type=pa.time32('s'))
+        a3 = pa.array((arr / 1000).astype('i4'), mask=null_mask,
+                      type=pa.time32('ms'))
+        a4 = pa.array((arr / 1000000).astype('i4'), mask=null_mask,
+                      type=pa.time32('s'))
 
         names = ['time64[us]', 'time64[ns]', 'time32[ms]', 'time32[s]']
         batch = pa.RecordBatch.from_arrays([a1, a2, a3, a4], names)
@@ -841,8 +828,8 @@ def test_arrow_time_to_pandas(self):
 
         tm.assert_frame_equal(df, expected_df)
 
-    def _check_numpy_array_roundtrip(self, np_array):
-        arr = pa.Array.from_pandas(np_array)
+    def _check_array_from_pandas_roundtrip(self, np_array):
+        arr = pa.array(np_array, from_pandas=True)
         result = arr.to_pandas()
         npt.assert_array_equal(result, np_array)
 
@@ -853,7 +840,7 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-13T12:34:56.432539784',
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
-        self._check_numpy_array_roundtrip(datetime64_ns)
+        self._check_array_from_pandas_roundtrip(datetime64_ns)
 
         datetime64_us = np.array([
                 '2007-07-13T01:23:34.123456',
@@ -861,7 +848,7 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-13T12:34:56.432539',
                 '2010-08-13T05:46:57.437699'],
                 dtype='datetime64[us]')
-        self._check_numpy_array_roundtrip(datetime64_us)
+        self._check_array_from_pandas_roundtrip(datetime64_us)
 
         datetime64_ms = np.array([
                 '2007-07-13T01:23:34.123',
@@ -869,7 +856,7 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-13T12:34:56.432',
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
-        self._check_numpy_array_roundtrip(datetime64_ms)
+        self._check_array_from_pandas_roundtrip(datetime64_ms)
 
         datetime64_s = np.array([
                 '2007-07-13T01:23:34',
@@ -877,7 +864,7 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-13T12:34:56',
                 '2010-08-13T05:46:57'],
                 dtype='datetime64[s]')
-        self._check_numpy_array_roundtrip(datetime64_s)
+        self._check_array_from_pandas_roundtrip(datetime64_s)
 
         datetime64_d = np.array([
                 '2007-07-13',
@@ -885,11 +872,11 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-15',
                 '2010-08-19'],
                 dtype='datetime64[D]')
-        self._check_numpy_array_roundtrip(datetime64_d)
+        self._check_array_from_pandas_roundtrip(datetime64_d)
 
     def test_all_nones(self):
         def _check_series(s):
-            converted = pa.Array.from_pandas(s)
+            converted = pa.array(s)
             assert isinstance(converted, pa.NullArray)
             assert len(converted) == 3
             assert converted.null_count == 3
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index eb23894f48082..b0593fe885dec 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -457,8 +457,26 @@ def test_column_of_arrays(tmpdir):
 
 @parquet
 def test_coerce_timestamps(tmpdir):
+    from collections import OrderedDict
     # ARROW-622
-    df, schema = dataframe_with_arrays()
+    arrays = OrderedDict()
+    fields = [pa.field('datetime64',
+                       pa.list_(pa.timestamp('ms')))]
+    arrays['datetime64'] = [
+        np.array(['2007-07-13T01:23:34.123456789',
+                  None,
+                  '2010-08-13T05:46:57.437699912'],
+                 dtype='datetime64[ms]'),
+        None,
+        None,
+        np.array(['2007-07-13T02',
+                  None,
+                  '2010-08-13T05:46:57.437699912'],
+                 dtype='datetime64[ms]'),
+    ]
+
+    df = pd.DataFrame(arrays)
+    schema = pa.schema(fields)
 
     filename = tmpdir.join('pandas_rountrip.parquet')
     arrow_table = pa.Table.from_pandas(df, schema=schema)
@@ -497,41 +515,41 @@ def test_column_of_lists(tmpdir):
 def test_date_time_types():
     t1 = pa.date32()
     data1 = np.array([17259, 17260, 17261], dtype='int32')
-    a1 = pa.Array.from_pandas(data1, type=t1)
+    a1 = pa.array(data1, type=t1)
 
     t2 = pa.date64()
     data2 = data1.astype('int64') * 86400000
-    a2 = pa.Array.from_pandas(data2, type=t2)
+    a2 = pa.array(data2, type=t2)
 
     t3 = pa.timestamp('us')
     start = pd.Timestamp('2000-01-01').value / 1000
     data3 = np.array([start, start + 1, start + 2], dtype='int64')
-    a3 = pa.Array.from_pandas(data3, type=t3)
+    a3 = pa.array(data3, type=t3)
 
     t4 = pa.time32('ms')
     data4 = np.arange(3, dtype='i4')
-    a4 = pa.Array.from_pandas(data4, type=t4)
+    a4 = pa.array(data4, type=t4)
 
     t5 = pa.time64('us')
-    a5 = pa.Array.from_pandas(data4.astype('int64'), type=t5)
+    a5 = pa.array(data4.astype('int64'), type=t5)
 
     t6 = pa.time32('s')
-    a6 = pa.Array.from_pandas(data4, type=t6)
+    a6 = pa.array(data4, type=t6)
 
     ex_t6 = pa.time32('ms')
-    ex_a6 = pa.Array.from_pandas(data4 * 1000, type=ex_t6)
+    ex_a6 = pa.array(data4 * 1000, type=ex_t6)
 
     t7 = pa.timestamp('ns')
     start = pd.Timestamp('2001-01-01').value
     data7 = np.array([start, start + 1000, start + 2000],
                      dtype='int64')
-    a7 = pa.Array.from_pandas(data7, type=t7)
+    a7 = pa.array(data7, type=t7)
 
     t7_us = pa.timestamp('us')
     start = pd.Timestamp('2001-01-01').value
     data7_us = np.array([start, start + 1000, start + 2000],
                         dtype='int64') // 1000
-    a7_us = pa.Array.from_pandas(data7_us, type=t7_us)
+    a7_us = pa.array(data7_us, type=t7_us)
 
     table = pa.Table.from_arrays([a1, a2, a3, a4, a5, a6, a7],
                                  ['date32', 'date64', 'timestamp[us]',
@@ -575,7 +593,7 @@ def _assert_unsupported(array):
             _write_table(table, buf, version="2.0")
 
     t7 = pa.time64('ns')
-    a7 = pa.Array.from_pandas(data4.astype('int64'), type=t7)
+    a7 = pa.array(data4.astype('int64'), type=t7)
 
     _assert_unsupported(a7)
 
@@ -1295,7 +1313,7 @@ def test_large_table_int32_overflow():
 
     arr = np.ones(size, dtype='uint8')
 
-    parr = pa.Array.from_pandas(arr, type=pa.uint8())
+    parr = pa.array(arr, type=pa.uint8())
 
     table = pa.Table.from_arrays([parr], names=['one'])
     f = io.BytesIO()
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 4bb6a5af7dccb..c77be98054c6f 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -69,6 +69,56 @@ def test_type_list():
     assert str(l2) == 'list<my_item: string>'
 
 
+def test_type_comparisons():
+    val = pa.int32()
+    assert val == pa.int32()
+    assert val == 'int32'
+
+    with pytest.raises(TypeError):
+        val == 5
+
+
+def test_type_for_alias():
+    cases = [
+        ('i1', pa.int8()),
+        ('int8', pa.int8()),
+        ('i2', pa.int16()),
+        ('int16', pa.int16()),
+        ('i4', pa.int32()),
+        ('int32', pa.int32()),
+        ('i8', pa.int64()),
+        ('int64', pa.int64()),
+        ('u1', pa.uint8()),
+        ('uint8', pa.uint8()),
+        ('u2', pa.uint16()),
+        ('uint16', pa.uint16()),
+        ('u4', pa.uint32()),
+        ('uint32', pa.uint32()),
+        ('u8', pa.uint64()),
+        ('uint64', pa.uint64()),
+        ('f4', pa.float32()),
+        ('float32', pa.float32()),
+        ('f8', pa.float64()),
+        ('float64', pa.float64()),
+        ('date32', pa.date32()),
+        ('date64', pa.date64()),
+        ('string', pa.string()),
+        ('str', pa.string()),
+        ('binary', pa.binary()),
+        ('time32[s]', pa.time32('s')),
+        ('time32[ms]', pa.time32('ms')),
+        ('time64[us]', pa.time64('us')),
+        ('time64[ns]', pa.time64('ns')),
+        ('timestamp[s]', pa.timestamp('s')),
+        ('timestamp[ms]', pa.timestamp('ms')),
+        ('timestamp[us]', pa.timestamp('us')),
+        ('timestamp[ns]', pa.timestamp('ns')),
+    ]
+
+    for val, expected in cases:
+        assert pa.type_for_alias(val) == expected
+
+
 def test_type_string():
     t = pa.string()
     assert str(t) == 'string'
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index b298e740250af..316e09a6efda7 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -72,11 +72,19 @@ cdef class DataType:
     def __repr__(self):
         return '{0.__class__.__name__}({0})'.format(self)
 
-    def __richcmp__(DataType self, DataType other, int op):
+    def __richcmp__(DataType self, object other, int op):
+        cdef DataType other_type
+        if not isinstance(other, DataType):
+            if not isinstance(other, six.string_types):
+                raise TypeError(other)
+            other_type = type_for_alias(other)
+        else:
+            other_type = other
+
         if op == cp.Py_EQ:
-            return self.type.Equals(deref(other.type))
+            return self.type.Equals(deref(other_type.type))
         elif op == cp.Py_NE:
-            return not self.type.Equals(deref(other.type))
+            return not self.type.Equals(deref(other_type.type))
         else:
             raise TypeError('Invalid comparison')
 
@@ -922,6 +930,64 @@ def struct(fields):
     return pyarrow_wrap_data_type(struct_type)
 
 
+cdef dict _type_aliases = {
+    'null': null,
+    'i1': int8,
+    'int8': int8,
+    'i2': int16,
+    'int16': int16,
+    'i4': int32,
+    'int32': int32,
+    'i8': int64,
+    'int64': int64,
+    'u1': uint8,
+    'uint8': uint8,
+    'u2': uint16,
+    'uint16': uint16,
+    'u4': uint32,
+    'uint32': uint32,
+    'u8': uint64,
+    'uint64': uint64,
+    'f4': float32,
+    'float32': float32,
+    'f8': float64,
+    'float64': float64,
+    'string': string,
+    'str': string,
+    'utf8': string,
+    'binary': binary,
+    'date32': date32,
+    'date64': date64,
+    'time32[s]': time32('s'),
+    'time32[ms]': time32('ms'),
+    'time64[us]': time64('us'),
+    'time64[ns]': time64('ns'),
+    'timestamp[s]': timestamp('s'),
+    'timestamp[ms]': timestamp('ms'),
+    'timestamp[us]': timestamp('us'),
+    'timestamp[ns]': timestamp('ns'),
+}
+
+
+def type_for_alias(name):
+    """
+    Return DataType given a string alias if one exists
+
+    Returns
+    -------
+    type : DataType
+    """
+    name = name.lower()
+    try:
+        alias = _type_aliases[name]
+    except KeyError:
+        raise ValueError('No type alias for {0}'.format(name))
+
+    if isinstance(alias, DataType):
+        return alias
+    return alias()
+
+
 def schema(fields):
     """
     Construct pyarrow.Schema from collection of fields

From cc3b27cc5b23347eb1e5b8d175317230088b586e Mon Sep 17 00:00:00 2001
From: Rene Sugar <rene.sugar@gmail.com>
Date: Sat, 30 Sep 2017 11:42:37 -0500
Subject: [PATCH 1100/1644] =?UTF-8?q?ARROW-1626=20Add=20make=20targets=20t?=
 =?UTF-8?q?o=20run=20the=20inter-procedural=20static=20analys=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…is tool called infer

Author: Rene Sugar <rene.sugar@gmail.com>

Closes #1149 from renesugar/infer and squashes the following commits:

8591b5ff [Rene Sugar] ARROW-1626 Add make targets to run the inter-procedural static analysis tool called infer
---
 cpp/CMakeLists.txt                     | 21 +++++++++++
 cpp/build-support/run-infer.sh         | 48 ++++++++++++++++++++++++++
 cpp/cmake_modules/FindInferTools.cmake | 45 ++++++++++++++++++++++++
 3 files changed, 114 insertions(+)
 create mode 100755 cpp/build-support/run-infer.sh
 create mode 100644 cpp/cmake_modules/FindInferTools.cmake

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index ad99970e9b25f..d488646fd9694 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -56,6 +56,14 @@ if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
   set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 endif()
 
+find_package(InferTools)
+if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR INFER_FOUND)
+  # Generate a Clang compile_commands.json "compilation database" file for use
+  # with various development tools, such as Vim's YouCompleteMe plugin.
+  # See http://clang.llvm.org/docs/JSONCompilationDatabase.html
+  set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
+endif()
+
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
   set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
@@ -596,6 +604,19 @@ if (${CLANG_TIDY_FOUND})
 
 endif()
 
+############################################################
+# "make infer" target
+############################################################
+
+if (${INFER_FOUND})
+  # runs infer capture
+  add_custom_target(infer ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 1)
+  # runs infer analyze
+  add_custom_target(infer-analyze ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 2)
+  # runs infer report
+  add_custom_target(infer-report ${BUILD_SUPPORT_DIR}/run-infer.sh ${INFER_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 3)
+endif()
+
 ############################################################
 # "make iwyu" target
 ############################################################
diff --git a/cpp/build-support/run-infer.sh b/cpp/build-support/run-infer.sh
new file mode 100755
index 0000000000000..823685aef146e
--- /dev/null
+++ b/cpp/build-support/run-infer.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# Runs infer in the given directory
+# Arguments:
+#   $1 - Path to the infer binary
+#   $2 - Path to the compile_commands.json to use
+#   $3 - Apply infer step (1=capture, 2=analyze, 3=report)
+#
+INFER=$1
+shift
+COMPILE_COMMANDS=$1
+shift
+APPLY_STEP=$1
+shift
+
+if [ "$APPLY_STEP" == "1" ]; then
+  $INFER capture --compilation-database $COMPILE_COMMANDS
+  echo ""
+  echo "Run 'make infer-analyze' next."
+elif [ "$APPLY_STEP" == "2" ]; then
+  # infer's analyze step can take a very long time to complete
+  $INFER analyze
+  echo ""
+  echo "Run 'make infer-report' next."
+  echo "See: http://fbinfer.com/docs/steps-for-ci.html"
+elif [ "$APPLY_STEP" == "3" ]; then
+  $INFER report --issues-csv ./infer-out/report.csv 1> /dev/null
+  $INFER report --issues-txt ./infer-out/report.txt 1> /dev/null
+  $INFER report --issues-json ./infer-out/report.json 1> /dev/null
+  echo ""
+  echo "Reports (report.txt, report.csv, report.json) can be found in the infer-out subdirectory."
+else
+  echo ""
+  echo "See: http://fbinfer.com/docs/steps-for-ci.html"
+fi
diff --git a/cpp/cmake_modules/FindInferTools.cmake b/cpp/cmake_modules/FindInferTools.cmake
new file mode 100644
index 0000000000000..00c6709c67703
--- /dev/null
+++ b/cpp/cmake_modules/FindInferTools.cmake
@@ -0,0 +1,45 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find the infer module
+#
+# Usage of this module as follows:
+#
+#  find_package(InferTools)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  InferTools_PATH -
+#   When set, this path is inspected instead of standard library binary locations
+#   to find infer
+#
+# This module defines
+#  INFER_BIN, The  path to the clang tidy binary
+#  INFER_FOUND, Whether clang tidy was found
+
+find_program(INFER_BIN
+  NAMES infer
+  PATHS ${InferTools_PATH} $ENV{INFER_TOOLS_PATH} /usr/local/bin /usr/bin
+  /usr/local/homebrew/bin
+  /opt/local/bin
+  NO_DEFAULT_PATH
+)
+
+if ( "${INFER_BIN}" STREQUAL "INFER_BIN-NOTFOUND" )
+  set(INFER_FOUND 0)
+  message("infer not found")
+else()
+  set(INFER_FOUND 1)
+  message("infer found at ${INFER_BIN}")
+endif()

From 9aa6eb58246ea6f6b7290881d57c80d515a6aa6d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 30 Sep 2017 13:25:04 -0500
Subject: [PATCH 1101/1644] ARROW-1624: [C++] Fix build on LLVM 4.0, remove
 some clang warning suppressions

I'm going to quick make a pass through later today and see how many of these warning suppressions I can remove

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1148 from wesm/warning-fixes and squashes the following commits:

e930152a [Wes McKinney] Only build compute modules if -DARROW_COMPUTE=ON
d6ca7ac0 [Wes McKinney] Slight refactor of CMakeLists.txt to move Arrow library setup to src/arrow
e2c61a2e [Wes McKinney] Use -Wno-unknown-warning-option
3d2d7265 [Wes McKinney] Fix travis CI script
6d0d4117 [Wes McKinney] Use BUILD_WARNING_LEVEL in Travis CI
1bec4a76 [Wes McKinney] Fix some more compiler warnings
cae05fb5 [Wes McKinney] Fix documentation compiler warnings
f76b2b93 [Wes McKinney] Fix a bunch of documentation warnings
ac54e2df [Wes McKinney] Remove some clang warning suppressions, fix warnings
6935b8c8 [Wes McKinney] Fix compiler warnings with clang-4.0
---
 ci/travis_before_script_cpp.sh              |   4 +-
 cpp/CMakeLists.txt                          | 267 +-------------------
 cpp/README.md                               |   7 +
 cpp/cmake_modules/BuildUtils.cmake          | 146 +++++++++++
 cpp/cmake_modules/SetupCxxFlags.cmake       |  26 +-
 cpp/cmake_modules/ThirdpartyToolchain.cmake |   1 +
 cpp/src/arrow/CMakeLists.txt                | 118 +++++++++
 cpp/src/arrow/array-test.cc                 |   8 +-
 cpp/src/arrow/array.cc                      |  23 +-
 cpp/src/arrow/array.h                       |   2 +-
 cpp/src/arrow/buffer.cc                     |   4 +
 cpp/src/arrow/compare.cc                    |   2 +-
 cpp/src/arrow/compute/cast.h                |  10 +-
 cpp/src/arrow/io/file.h                     |   2 +-
 cpp/src/arrow/io/interfaces.h               |   5 +
 cpp/src/arrow/ipc/feather.cc                |  27 +-
 cpp/src/arrow/ipc/message.h                 |   7 +-
 cpp/src/arrow/ipc/reader.h                  |  49 ++--
 cpp/src/arrow/ipc/writer.cc                 |  36 +--
 cpp/src/arrow/ipc/writer.h                  |  24 +-
 cpp/src/arrow/pretty_print.cc               |   4 +-
 cpp/src/arrow/python/arrow_to_python.h      |   2 +-
 cpp/src/arrow/python/numpy_to_arrow.cc      |   2 +-
 cpp/src/arrow/python/numpy_to_arrow.h       |   2 +-
 cpp/src/arrow/python/python_to_arrow.cc     |  11 +-
 cpp/src/arrow/python/util/datetime.h        |  13 +-
 cpp/src/arrow/table.cc                      |   7 -
 cpp/src/arrow/table.h                       |  18 +-
 cpp/src/arrow/type-test.cc                  |   6 +-
 cpp/src/arrow/type.cc                       |   8 +
 cpp/src/arrow/type.h                        |   8 +-
 cpp/src/arrow/util/compression_brotli.cc    |   4 +-
 cpp/src/arrow/util/compression_lz4.cc       |   4 +-
 cpp/src/arrow/util/compression_snappy.cc    |  10 +-
 cpp/src/arrow/util/compression_zlib.cc      |   2 +-
 cpp/src/arrow/util/compression_zstd.cc      |   4 +-
 cpp/src/arrow/util/cpu-info.cc              |   4 +
 cpp/src/arrow/util/decimal.h                |   4 +-
 cpp/src/arrow/util/hash-util.h              |   2 +-
 cpp/src/arrow/util/logging.h                |   2 +-
 cpp/src/arrow/util/macros.h                 |   2 +-
 cpp/src/arrow/util/sse-util.h               |  20 +-
 cpp/src/arrow/visitor.cc                    |   5 +-
 cpp/src/plasma/CMakeLists.txt               |   9 +-
 cpp/src/plasma/client.cc                    |  14 +-
 cpp/src/plasma/client.h                     | 116 ++++-----
 cpp/src/plasma/events.h                     |   3 -
 cpp/src/plasma/eviction_policy.h            |   3 -
 cpp/src/plasma/fling.cc                     |   5 +-
 cpp/src/plasma/io.cc                        |  11 +-
 cpp/src/plasma/protocol.cc                  |  15 +-
 cpp/src/plasma/store.cc                     |   3 +-
 cpp/src/plasma/store.h                      |   7 -
 cpp/src/plasma/test/serialization_tests.cc  |  13 +-
 cpp/src/plasma/thirdparty/ae/ae_epoll.c     |   6 +-
 55 files changed, 570 insertions(+), 547 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index a7f1d2756344b..65171a735575f 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -90,12 +90,12 @@ fi
 if [ $TRAVIS_OS_NAME == "linux" ]; then
     cmake $CMAKE_COMMON_FLAGS \
           $CMAKE_LINUX_FLAGS \
-          -DARROW_CXXFLAGS="-Wconversion -Wno-sign-conversion -Werror" \
+          -DBUILD_WARNING_LEVEL=CHECKIN \
           $ARROW_CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
           $CMAKE_OSX_FLAGS \
-          -DARROW_CXXFLAGS=-Werror \
+          -DBUILD_WARNING_LEVEL=CHECKIN \
           $ARROW_CPP_DIR
 fi
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d488646fd9694..0930ae4ced92c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -231,8 +231,6 @@ else()
   set(ARROW_BOOST_HEADER_ONLY 1)
 endif()
 
-include(BuildUtils)
-
 ############################################################
 # Compiler flags
 ############################################################
@@ -256,6 +254,9 @@ include(SetupCxxFlags)
 
 add_custom_target(arrow_dependencies)
 
+include(BuildUtils)
+enable_testing()
+
 include(ThirdpartyToolchain)
 
 # Add common flags
@@ -362,153 +363,6 @@ if (PARQUET_BUILD_SHARED)
   endif()
 endif()
 
-############################################################
-# Benchmarking
-############################################################
-# Add a new micro benchmark, with or without an executable that should be built.
-# If benchmarks are enabled then they will be run along side unit tests with ctest.
-# 'make runbenchmark' and 'make unittest' to build/run only benchmark or unittests,
-# respectively.
-#
-# REL_BENCHMARK_NAME is the name of the benchmark app. It may be a single component
-# (e.g. monotime-benchmark) or contain additional components (e.g.
-# net/net_util-benchmark). Either way, the last component must be a globally
-# unique name.
-
-# The benchmark will registered as unit test with ctest with a label
-# of 'benchmark'.
-#
-# Arguments after the test name will be passed to set_tests_properties().
-function(ADD_ARROW_BENCHMARK REL_BENCHMARK_NAME)
-  if(NO_BENCHMARKS)
-    return()
-  endif()
-  get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
-
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc)
-    # This benchmark has a corresponding .cc file, set it up as an executable.
-    set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}")
-    add_executable(${BENCHMARK_NAME} "${REL_BENCHMARK_NAME}.cc")
-    target_link_libraries(${BENCHMARK_NAME} ${ARROW_BENCHMARK_LINK_LIBS})
-    add_dependencies(runbenchmark ${BENCHMARK_NAME})
-    set(NO_COLOR "--color_print=false")
-  else()
-    # No executable, just invoke the benchmark (probably a script) directly.
-    set(BENCHMARK_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME})
-    set(NO_COLOR "")
-  endif()
-
-  add_test(${BENCHMARK_NAME}
-    ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} benchmark ${BENCHMARK_PATH} ${NO_COLOR})
-  set_tests_properties(${BENCHMARK_NAME} PROPERTIES LABELS "benchmark")
-  if(ARGN)
-    set_tests_properties(${BENCHMARK_NAME} PROPERTIES ${ARGN})
-  endif()
-endfunction()
-
-# A wrapper for add_dependencies() that is compatible with NO_BENCHMARKS.
-function(ADD_ARROW_BENCHMARK_DEPENDENCIES REL_BENCHMARK_NAME)
-  if(NO_BENCHMARKS)
-    return()
-  endif()
-  get_filename_component(BENCMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
-
-  add_dependencies(${BENCHMARK_NAME} ${ARGN})
-endfunction()
-
-# A wrapper for target_link_libraries() that is compatible with NO_BENCHMARKS.
-function(ARROW_BENCHMARK_LINK_LIBRARIES REL_BENCHMARK_NAME)
-    if(NO_BENCHMARKS)
-    return()
-  endif()
-  get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
-
-  target_link_libraries(${BENCHMARK_NAME} ${ARGN})
-endfunction()
-
-
-############################################################
-# Testing
-############################################################
-# Add a new test case, with or without an executable that should be built.
-#
-# REL_TEST_NAME is the name of the test. It may be a single component
-# (e.g. monotime-test) or contain additional components (e.g.
-# net/net_util-test). Either way, the last component must be a globally
-# unique name.
-#
-# The unit test is added with a label of "unittest" to support filtering with
-# ctest.
-#
-# Arguments after the test name will be passed to set_tests_properties().
-function(ADD_ARROW_TEST REL_TEST_NAME)
-  set(options)
-  set(single_value_args)
-  set(multi_value_args STATIC_LINK_LIBS)
-  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
-  if(ARG_UNPARSED_ARGUMENTS)
-    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
-  endif()
-
-  if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
-    # This test has a corresponding .cc file, set it up as an executable.
-    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-    add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
-
-    if (ARG_STATIC_LINK_LIBS)
-      # Customize link libraries
-      target_link_libraries(${TEST_NAME} ${ARG_STATIC_LINK_LIBS})
-    else()
-      target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
-    endif()
-    add_dependencies(unittest ${TEST_NAME})
-  else()
-    # No executable, just invoke the test (probably a script) directly.
-    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
-  endif()
-
-  if (ARROW_TEST_MEMCHECK)
-    SET_PROPERTY(TARGET ${TEST_NAME}
-      APPEND_STRING PROPERTY
-      COMPILE_FLAGS " -DARROW_VALGRIND")
-    add_test(${TEST_NAME}
-      bash -c "cd ${EXECUTABLE_OUTPUT_PATH}; valgrind --tool=memcheck --leak-check=full --leak-check-heuristics=stdstring --error-exitcode=1 ${TEST_PATH}")
-  elseif(MSVC)
-    add_test(${TEST_NAME} ${TEST_PATH})
-  else()
-    add_test(${TEST_NAME}
-      ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
-  endif()
-  set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
-endfunction()
-
-# A wrapper for add_dependencies() that is compatible with NO_TESTS.
-function(ADD_ARROW_TEST_DEPENDENCIES REL_TEST_NAME)
-  if(NO_TESTS)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  add_dependencies(${TEST_NAME} ${ARGN})
-endfunction()
-
-# A wrapper for target_link_libraries() that is compatible with NO_TESTS.
-function(ARROW_TEST_LINK_LIBRARIES REL_TEST_NAME)
-  if(NO_TESTS)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  target_link_libraries(${TEST_NAME} ${ARGN})
-endfunction()
-
-enable_testing()
-
 ############################################################
 # "make ctags" target
 ############################################################
@@ -754,121 +608,6 @@ if(NOT WIN32 AND ARROW_PLASMA)
 endif()
 
 add_subdirectory(src/arrow)
-add_subdirectory(src/arrow/io)
-
-set(ARROW_SRCS
-  src/arrow/array.cc
-  src/arrow/buffer.cc
-  src/arrow/builder.cc
-  src/arrow/compare.cc
-  src/arrow/memory_pool.cc
-  src/arrow/pretty_print.cc
-  src/arrow/status.cc
-  src/arrow/table.cc
-  src/arrow/tensor.cc
-  src/arrow/type.cc
-  src/arrow/visitor.cc
-
-  src/arrow/compute/cast.cc
-  src/arrow/compute/context.cc
-
-  src/arrow/io/file.cc
-  src/arrow/io/interfaces.cc
-  src/arrow/io/memory.cc
-
-  src/arrow/util/bit-util.cc
-  src/arrow/util/compression.cc
-  src/arrow/util/cpu-info.cc
-  src/arrow/util/decimal.cc
-  src/arrow/util/key_value_metadata.cc
-)
-
-if (ARROW_COMPUTE)
-  add_subdirectory(src/arrow/compute)
-  set(ARROW_SRCS ${ARROW_SRCS}
-    src/arrow/compute/cast.cc
-    src/arrow/compute/context.cc
-  )
-endif()
-
-if (ARROW_GPU)
-  # IPC extensions required to build the GPU library
-  set(ARROW_IPC ON)
-  add_subdirectory(src/arrow/gpu)
-endif()
-
-if (ARROW_IPC)
-  add_subdirectory(src/arrow/ipc)
-  add_dependencies(arrow_dependencies metadata_fbs)
-endif()
-
-if (ARROW_WITH_BROTLI)
-  add_definitions(-DARROW_WITH_BROTLI)
-  SET(ARROW_SRCS src/arrow/util/compression_brotli.cc ${ARROW_SRCS})
-endif()
-
-if (ARROW_WITH_LZ4)
-  add_definitions(-DARROW_WITH_LZ4)
-  SET(ARROW_SRCS src/arrow/util/compression_lz4.cc ${ARROW_SRCS})
-endif()
-
-if (ARROW_WITH_SNAPPY)
-  add_definitions(-DARROW_WITH_SNAPPY)
-  SET(ARROW_SRCS src/arrow/util/compression_snappy.cc ${ARROW_SRCS})
-endif()
-
-if (ARROW_WITH_ZLIB)
-  add_definitions(-DARROW_WITH_ZLIB)
-  SET(ARROW_SRCS src/arrow/util/compression_zlib.cc ${ARROW_SRCS})
-endif()
-
-if (ARROW_WITH_ZSTD)
-  add_definitions(-DARROW_WITH_ZSTD)
-  SET(ARROW_SRCS src/arrow/util/compression_zstd.cc ${ARROW_SRCS})
-endif()
-
-if (NOT ARROW_BOOST_HEADER_ONLY)
-  set(ARROW_SRCS ${ARROW_SRCS}
-    src/arrow/io/hdfs.cc
-    src/arrow/io/hdfs-internal.cc
-  )
-endif()
-
-if (ARROW_IPC)
-  set(ARROW_SRCS ${ARROW_SRCS}
-    src/arrow/ipc/dictionary.cc
-    src/arrow/ipc/feather.cc
-    src/arrow/ipc/json.cc
-    src/arrow/ipc/json-internal.cc
-    src/arrow/ipc/message.cc
-    src/arrow/ipc/metadata-internal.cc
-    src/arrow/ipc/reader.cc
-    src/arrow/ipc/writer.cc
-  )
-endif()
-
-
-if(NOT APPLE AND NOT MSVC)
-  # Localize thirdparty symbols using a linker version script. This hides them
-  # from the client application. The OS X linker does not support the
-  # version-script option.
-  set(ARROW_SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map")
-endif()
-
-set(ARROW_ALL_SRCS
-  ${ARROW_SRCS})
-
-ADD_ARROW_LIB(arrow
-  SOURCES ${ARROW_ALL_SRCS}
-  DEPENDENCIES arrow_dependencies
-  SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
-  SHARED_LINK_LIBS ${ARROW_LINK_LIBS}
-  SHARED_PRIVATE_LINK_LIBS ${ARROW_SHARED_PRIVATE_LINK_LIBS}
-  STATIC_LINK_LIBS ${ARROW_STATIC_LINK_LIBS}
-  STATIC_PRIVATE_LINK_LIBS ${ARROW_STATIC_PRIVATE_LINK_LIBS}
-)
-
-add_subdirectory(src/arrow/util)
 
 if(ARROW_PYTHON)
   find_package(PythonLibsNew REQUIRED)
diff --git a/cpp/README.md b/cpp/README.md
index 4a515079ddc01..6e29e6f78f4d1 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -238,6 +238,13 @@ build failures by running the following checks before submitting your pull reque
     # before running it.
     make format # requires clang-format is installed
 
+We run our CI builds with more compiler warnings enabled for the Clang
+compiler. Please run CMake with
+
+`-DBUILD_WARNING_LEVEL=CHECKIN`
+
+to avoid failures due to compiler warnings.
+
 Note that the clang-tidy target may take a while to run.  You might consider
 running clang-tidy separately on the files you have added/changed before
 invoking the make target to reduce iteration time.  Also, it might generate warnings
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 8f92d73baea31..e398dc1827cbb 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -188,3 +188,149 @@ function(ADD_ARROW_LIB LIB_NAME)
   endif()
 
 endfunction()
+
+
+############################################################
+# Benchmarking
+############################################################
+# Add a new micro benchmark, with or without an executable that should be built.
+# If benchmarks are enabled then they will be run along side unit tests with ctest.
+# 'make runbenchmark' and 'make unittest' to build/run only benchmark or unittests,
+# respectively.
+#
+# REL_BENCHMARK_NAME is the name of the benchmark app. It may be a single component
+# (e.g. monotime-benchmark) or contain additional components (e.g.
+# net/net_util-benchmark). Either way, the last component must be a globally
+# unique name.
+
+# The benchmark will registered as unit test with ctest with a label
+# of 'benchmark'.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+function(ADD_ARROW_BENCHMARK REL_BENCHMARK_NAME)
+  if(NO_BENCHMARKS)
+    return()
+  endif()
+  get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
+
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc)
+    # This benchmark has a corresponding .cc file, set it up as an executable.
+    set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}")
+    add_executable(${BENCHMARK_NAME} "${REL_BENCHMARK_NAME}.cc")
+    target_link_libraries(${BENCHMARK_NAME} ${ARROW_BENCHMARK_LINK_LIBS})
+    add_dependencies(runbenchmark ${BENCHMARK_NAME})
+    set(NO_COLOR "--color_print=false")
+  else()
+    # No executable, just invoke the benchmark (probably a script) directly.
+    set(BENCHMARK_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME})
+    set(NO_COLOR "")
+  endif()
+
+  add_test(${BENCHMARK_NAME}
+    ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} benchmark ${BENCHMARK_PATH} ${NO_COLOR})
+  set_tests_properties(${BENCHMARK_NAME} PROPERTIES LABELS "benchmark")
+  if(ARGN)
+    set_tests_properties(${BENCHMARK_NAME} PROPERTIES ${ARGN})
+  endif()
+endfunction()
+
+# A wrapper for add_dependencies() that is compatible with NO_BENCHMARKS.
+function(ADD_ARROW_BENCHMARK_DEPENDENCIES REL_BENCHMARK_NAME)
+  if(NO_BENCHMARKS)
+    return()
+  endif()
+  get_filename_component(BENCMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
+
+  add_dependencies(${BENCHMARK_NAME} ${ARGN})
+endfunction()
+
+# A wrapper for target_link_libraries() that is compatible with NO_BENCHMARKS.
+function(ARROW_BENCHMARK_LINK_LIBRARIES REL_BENCHMARK_NAME)
+    if(NO_BENCHMARKS)
+    return()
+  endif()
+  get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
+
+  target_link_libraries(${BENCHMARK_NAME} ${ARGN})
+endfunction()
+
+
+############################################################
+# Testing
+############################################################
+# Add a new test case, with or without an executable that should be built.
+#
+# REL_TEST_NAME is the name of the test. It may be a single component
+# (e.g. monotime-test) or contain additional components (e.g.
+# net/net_util-test). Either way, the last component must be a globally
+# unique name.
+#
+# The unit test is added with a label of "unittest" to support filtering with
+# ctest.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+function(ADD_ARROW_TEST REL_TEST_NAME)
+  set(options)
+  set(single_value_args)
+  set(multi_value_args STATIC_LINK_LIBS)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
+    # This test has a corresponding .cc file, set it up as an executable.
+    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+    add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
+
+    if (ARG_STATIC_LINK_LIBS)
+      # Customize link libraries
+      target_link_libraries(${TEST_NAME} ${ARG_STATIC_LINK_LIBS})
+    else()
+      target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+    endif()
+    add_dependencies(unittest ${TEST_NAME})
+  else()
+    # No executable, just invoke the test (probably a script) directly.
+    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
+  endif()
+
+  if (ARROW_TEST_MEMCHECK)
+    SET_PROPERTY(TARGET ${TEST_NAME}
+      APPEND_STRING PROPERTY
+      COMPILE_FLAGS " -DARROW_VALGRIND")
+    add_test(${TEST_NAME}
+      bash -c "cd ${EXECUTABLE_OUTPUT_PATH}; valgrind --tool=memcheck --leak-check=full --leak-check-heuristics=stdstring --error-exitcode=1 ${TEST_PATH}")
+  elseif(MSVC)
+    add_test(${TEST_NAME} ${TEST_PATH})
+  else()
+    add_test(${TEST_NAME}
+      ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
+  endif()
+  set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
+endfunction()
+
+# A wrapper for add_dependencies() that is compatible with NO_TESTS.
+function(ADD_ARROW_TEST_DEPENDENCIES REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  add_dependencies(${TEST_NAME} ${ARGN})
+endfunction()
+
+# A wrapper for target_link_libraries() that is compatible with NO_TESTS.
+function(ARROW_TEST_LINK_LIBRARIES REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  target_link_libraries(${TEST_NAME} ${ARGN})
+endfunction()
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 1f4c898cfcf3f..77bfac83555b0 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -70,17 +70,17 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
-    -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
-    -Wno-unused-parameter -Wno-undef -Wno-documentation-deprecated-sync \
-    -Wno-shadow -Wno-switch-enum -Wno-documentation -Wno-exit-time-destructors \
-    -Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \
-    -Wno-implicit-fallthrough -Wno-old-style-cast -Wno-unreachable-code-return \
-    -Wno-float-equal -Wno-missing-prototypes -Wno-non-virtual-dtor \
-    -Wno-unused-macros -Wno-covered-switch-default -Wno-unreachable-code-break \
-    -Wno-extra-semi -Wno-cast-align -Wno-vla-extension -Wno-shift-sign-overflow \
-    -Wno-used-but-marked-unused -Wno-missing-variable-declarations \
-    -Wno-gnu-zero-variadic-macro-arguments -Wconversion -Wno-sign-conversion \
-    -Wno-disabled-macro-expansion -Wc++11-narrowing -Wnarrowing -Wno-shorten-64-to-32")
+-Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
+-Wno-unused-parameter -Wno-undef \
+-Wno-shadow -Wno-switch-enum -Wno-exit-time-destructors \
+-Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \
+-Wno-implicit-fallthrough -Wno-unreachable-code-return \
+-Wno-float-equal -Wno-missing-prototypes \
+-Wno-old-style-cast -Wno-covered-switch-default \
+-Wno-cast-align -Wno-vla-extension -Wno-shift-sign-overflow \
+-Wno-used-but-marked-unused -Wno-missing-variable-declarations \
+-Wno-gnu-zero-variadic-macro-arguments -Wconversion -Wno-sign-conversion \
+-Wno-disabled-macro-expansion -Wno-shorten-64-to-32")
 
     # Version numbers where warnings are introduced
     if ("${COMPILER_VERSION}" VERSION_GREATER "3.3")
@@ -98,11 +98,11 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
     endif()
 
     # Treat all compiler warnings as errors
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wunknown-warning-option -Werror")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option -Werror")
   elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wconversion -Wno-sign-conversion")
     # Treat all compiler warnings as errors
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wunknown-warning-option -Werror")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option -Werror")
   else()
     message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
   endif()
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 194156cc65643..98186d01415bc 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -126,6 +126,7 @@ endif()
 set(Boost_DEBUG TRUE)
 set(Boost_USE_MULTITHREADED ON)
 set(Boost_ADDITIONAL_VERSIONS
+  "1.65.0" "1.65"
   "1.64.0" "1.64"
   "1.63.0" "1.63"
   "1.62.0" "1.61"
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 6963b11e3ffa5..5c90331418142 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -15,6 +15,121 @@
 # specific language governing permissions and limitations
 # under the License.
 
+set(ARROW_SRCS
+  array.cc
+  buffer.cc
+  builder.cc
+  compare.cc
+  memory_pool.cc
+  pretty_print.cc
+  status.cc
+  table.cc
+  tensor.cc
+  type.cc
+  visitor.cc
+
+  io/file.cc
+  io/interfaces.cc
+  io/memory.cc
+
+  util/bit-util.cc
+  util/compression.cc
+  util/cpu-info.cc
+  util/decimal.cc
+  util/key_value_metadata.cc
+)
+
+if ("${COMPILER_FAMILY}" STREQUAL "clang")
+  set_property(SOURCE io/file.cc
+    APPEND_STRING
+    PROPERTY COMPILE_FLAGS
+    " -Wno-unused-macros ")
+endif()
+
+if (ARROW_COMPUTE)
+  add_subdirectory(compute)
+  set(ARROW_SRCS ${ARROW_SRCS}
+    compute/cast.cc
+    compute/context.cc
+  )
+endif()
+
+if (ARROW_GPU)
+  # IPC extensions required to build the GPU library
+  set(ARROW_IPC ON)
+  add_subdirectory(gpu)
+endif()
+
+if (ARROW_IPC)
+  add_subdirectory(ipc)
+  add_dependencies(arrow_dependencies metadata_fbs)
+endif()
+
+if (ARROW_WITH_BROTLI)
+  add_definitions(-DARROW_WITH_BROTLI)
+  SET(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS})
+endif()
+
+if (ARROW_WITH_LZ4)
+  add_definitions(-DARROW_WITH_LZ4)
+  SET(ARROW_SRCS util/compression_lz4.cc ${ARROW_SRCS})
+endif()
+
+if (ARROW_WITH_SNAPPY)
+  add_definitions(-DARROW_WITH_SNAPPY)
+  SET(ARROW_SRCS util/compression_snappy.cc ${ARROW_SRCS})
+endif()
+
+if (ARROW_WITH_ZLIB)
+  add_definitions(-DARROW_WITH_ZLIB)
+  SET(ARROW_SRCS util/compression_zlib.cc ${ARROW_SRCS})
+endif()
+
+if (ARROW_WITH_ZSTD)
+  add_definitions(-DARROW_WITH_ZSTD)
+  SET(ARROW_SRCS util/compression_zstd.cc ${ARROW_SRCS})
+endif()
+
+if (NOT ARROW_BOOST_HEADER_ONLY)
+  set(ARROW_SRCS ${ARROW_SRCS}
+    io/hdfs.cc
+    io/hdfs-internal.cc
+  )
+endif()
+
+if (ARROW_IPC)
+  set(ARROW_SRCS ${ARROW_SRCS}
+    ipc/dictionary.cc
+    ipc/feather.cc
+    ipc/json.cc
+    ipc/json-internal.cc
+    ipc/message.cc
+    ipc/metadata-internal.cc
+    ipc/reader.cc
+    ipc/writer.cc
+  )
+endif()
+
+if(NOT APPLE AND NOT MSVC)
+  # Localize thirdparty symbols using a linker version script. This hides them
+  # from the client application. The OS X linker does not support the
+  # version-script option.
+  set(ARROW_SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
+endif()
+
+set(ARROW_ALL_SRCS
+  ${ARROW_SRCS})
+
+ADD_ARROW_LIB(arrow
+  SOURCES ${ARROW_ALL_SRCS}
+  DEPENDENCIES arrow_dependencies
+  SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
+  SHARED_LINK_LIBS ${ARROW_LINK_LIBS}
+  SHARED_PRIVATE_LINK_LIBS ${ARROW_SHARED_PRIVATE_LINK_LIBS}
+  STATIC_LINK_LIBS ${ARROW_STATIC_LINK_LIBS}
+  STATIC_PRIVATE_LINK_LIBS ${ARROW_STATIC_PRIVATE_LINK_LIBS}
+)
+
 # Headers: top level
 install(FILES
   allocator.h
@@ -60,3 +175,6 @@ ADD_ARROW_TEST(tensor-test)
 
 ADD_ARROW_BENCHMARK(builder-benchmark)
 ADD_ARROW_BENCHMARK(column-benchmark)
+
+add_subdirectory(io)
+add_subdirectory(util)
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 4ea9248f3e4d2..4ecf0f92b2a33 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -283,7 +283,7 @@ class TestPrimitiveBuilder : public TestBuilder {
 
 #define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
   struct P##CapType {                               \
-    PTYPE_DECL(CapType, c_type);                    \
+    PTYPE_DECL(CapType, c_type)                     \
     static void draw(int64_t N, vector<T>* draws) { \
       test::randint<T>(N, LOWER, UPPER, draws);     \
     }                                               \
@@ -291,7 +291,7 @@ class TestPrimitiveBuilder : public TestBuilder {
 
 #define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)     \
   struct P##CapType {                                  \
-    PTYPE_DECL(CapType, c_type);                       \
+    PTYPE_DECL(CapType, c_type)                        \
     static void draw(int64_t N, vector<T>* draws) {    \
       test::random_real<T>(N, 0, LOWER, UPPER, draws); \
     }                                                  \
@@ -311,7 +311,7 @@ PFLOAT_DECL(Float, float, -1000, 1000);
 PFLOAT_DECL(Double, double, -1000, 1000);
 
 struct PBoolean {
-  PTYPE_DECL(Boolean, uint8_t);
+  PTYPE_DECL(Boolean, uint8_t)
 };
 
 template <>
@@ -378,8 +378,6 @@ TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
 #define DECL_TYPE() typedef typename TestFixture::Type Type;
 
-#define DECL_ARRAYTYPE() typedef typename TestFixture::ArrayType ArrayType;
-
 TYPED_TEST(TestPrimitiveBuilder, TestInit) {
   DECL_TYPE();
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 80188a14a9588..92e8d0f0fa8f6 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -194,7 +194,8 @@ ListArray::ListArray(const std::shared_ptr<DataType>& type, int64_t length,
   SetData(internal_data);
 }
 
-Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPool* pool,
+Status ListArray::FromArrays(const Array& offsets, const Array& values,
+                             MemoryPool* ARROW_ARG_UNUSED(pool),
                              std::shared_ptr<Array>* out) {
   if (ARROW_PREDICT_FALSE(offsets.length() == 0)) {
     return Status::Invalid("List offsets must have non-zero length");
@@ -239,9 +240,6 @@ std::shared_ptr<Array> ListArray::values() const { return values_; }
 // ----------------------------------------------------------------------
 // String and binary
 
-static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
-static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
-
 BinaryArray::BinaryArray(const std::shared_ptr<ArrayData>& data) {
   DCHECK_EQ(data->type->id(), Type::BINARY);
   SetData(data);
@@ -261,8 +259,8 @@ BinaryArray::BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_of
                          const std::shared_ptr<Buffer>& data,
                          const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                          int64_t offset)
-    : BinaryArray(kBinary, length, value_offsets, data, null_bitmap, null_count, offset) {
-}
+    : BinaryArray(binary(), length, value_offsets, data, null_bitmap, null_count,
+                  offset) {}
 
 BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
                          const std::shared_ptr<Buffer>& value_offsets,
@@ -283,8 +281,7 @@ StringArray::StringArray(int64_t length, const std::shared_ptr<Buffer>& value_of
                          const std::shared_ptr<Buffer>& data,
                          const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                          int64_t offset)
-    : BinaryArray(kString, length, value_offsets, data, null_bitmap, null_count, offset) {
-}
+    : BinaryArray(utf8(), length, value_offsets, data, null_bitmap, null_count, offset) {}
 
 // ----------------------------------------------------------------------
 // Fixed width binary
@@ -437,13 +434,13 @@ Status Array::Accept(ArrayVisitor* visitor) const {
 namespace internal {
 
 struct ValidateVisitor {
-  Status Visit(const NullArray& array) { return Status::OK(); }
+  Status Visit(const NullArray&) { return Status::OK(); }
 
-  Status Visit(const PrimitiveArray& array) { return Status::OK(); }
+  Status Visit(const PrimitiveArray&) { return Status::OK(); }
 
-  Status Visit(const DecimalArray& array) { return Status::OK(); }
+  Status Visit(const DecimalArray&) { return Status::OK(); }
 
-  Status Visit(const BinaryArray& array) {
+  Status Visit(const BinaryArray&) {
     // TODO(wesm): what to do here?
     return Status::OK();
   }
@@ -585,7 +582,7 @@ class ArrayDataWrapper {
       : data_(data), out_(out) {}
 
   template <typename T>
-  Status Visit(const T& type) {
+  Status Visit(const T&) {
     using ArrayType = typename TypeTraits<T>::ArrayType;
     *out_ = std::make_shared<ArrayType>(data_);
     return Status::OK();
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index e801b3586df30..ee29d95068276 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -674,7 +674,7 @@ ARROW_EXPORT
 Status ValidateArray(const Array& array);
 
 #ifndef ARROW_NO_DEPRECATED_API
-// \deprecated Since 0.7.0
+// \note Deprecated since 0.7.0
 
 /// Create new arrays for logical types that are backed by primitive arrays.
 ARROW_EXPORT
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index cf533eb3be439..e308ed260497f 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -132,6 +132,8 @@ Status AllocateBuffer(MemoryPool* pool, const int64_t size,
   return Status::OK();
 }
 
+#ifndef ARROW_NO_DEPRECATED_API
+
 Status AllocateBuffer(MemoryPool* pool, const int64_t size,
                       std::shared_ptr<MutableBuffer>* out) {
   std::shared_ptr<Buffer> buffer;
@@ -140,6 +142,8 @@ Status AllocateBuffer(MemoryPool* pool, const int64_t size,
   return Status::OK();
 }
 
+#endif
+
 Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
                                std::shared_ptr<ResizableBuffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 2aeb03b8b9a12..515b8f62f5640 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -584,7 +584,7 @@ class TypeEqualsVisitor {
   typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
                               std::is_base_of<PrimitiveCType, T>::value,
                           Status>::type
-  Visit(const T& type) {
+  Visit(const T&) {
     result_ = true;
     return Status::OK();
   }
diff --git a/cpp/src/arrow/compute/cast.h b/cpp/src/arrow/compute/cast.h
index 081cdd908326c..7a07512b2adcf 100644
--- a/cpp/src/arrow/compute/cast.h
+++ b/cpp/src/arrow/compute/cast.h
@@ -46,11 +46,11 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>&
                        const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel);
 
 /// \brief Cast from one array type to another
-/// \param[in] context
-/// \param[in] array
-/// \param[in] to_type
-/// \param[in] options
-/// \param[out] out
+/// \param[in] context the FunctionContext
+/// \param[in] array array to cast
+/// \param[in] to_type type to cast to
+/// \param[in] options casting options
+/// \param[out] out resulting array
 ///
 /// \since 0.7.0
 /// \note API not yet finalized
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 1b1bbe0e4955d..4fb09634a2b78 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -86,7 +86,7 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile {
   /// \param[in] pool a MemoryPool for memory allocations
   /// \param[out] file ReadableFile instance
   /// Open file with one's own memory pool for memory allocations
-  static Status Open(const std::string& path, MemoryPool* memory_pool,
+  static Status Open(const std::string& path, MemoryPool* pool,
                      std::shared_ptr<ReadableFile>* file);
 
   Status Close() override;
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 11441794e18d2..2c5b351e2082a 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -86,11 +86,14 @@ class ARROW_EXPORT FileInterface {
 
 class ARROW_EXPORT Seekable {
  public:
+  virtual ~Seekable() = default;
   virtual Status Seek(int64_t position) = 0;
 };
 
 class ARROW_EXPORT Writeable {
  public:
+  virtual ~Writeable() = default;
+
   virtual Status Write(const uint8_t* data, int64_t nbytes) = 0;
 
   /// \brief Flush buffered bytes, if any
@@ -101,6 +104,8 @@ class ARROW_EXPORT Writeable {
 
 class ARROW_EXPORT Readable {
  public:
+  virtual ~Readable() = default;
+
   virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0;
 
   // Does not copy if not necessary
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 31dc0e73e0e3d..9d244f1154416 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -477,7 +477,6 @@ fbs::Type ToFlatbufferType(Type::type type) {
       return fbs::Type_INT64;
     default:
       DCHECK(false) << "Cannot reach this code";
-      break;
   }
   // prevent compiler warning
   return fbs::Type_MIN;
@@ -632,19 +631,19 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
 #define VISIT_PRIMITIVE(TYPE) \
   Status Visit(const TYPE& values) override { return WritePrimitiveValues(values); }
 
-  VISIT_PRIMITIVE(BooleanArray);
-  VISIT_PRIMITIVE(Int8Array);
-  VISIT_PRIMITIVE(Int16Array);
-  VISIT_PRIMITIVE(Int32Array);
-  VISIT_PRIMITIVE(Int64Array);
-  VISIT_PRIMITIVE(UInt8Array);
-  VISIT_PRIMITIVE(UInt16Array);
-  VISIT_PRIMITIVE(UInt32Array);
-  VISIT_PRIMITIVE(UInt64Array);
-  VISIT_PRIMITIVE(FloatArray);
-  VISIT_PRIMITIVE(DoubleArray);
-  VISIT_PRIMITIVE(BinaryArray);
-  VISIT_PRIMITIVE(StringArray);
+  VISIT_PRIMITIVE(BooleanArray)
+  VISIT_PRIMITIVE(Int8Array)
+  VISIT_PRIMITIVE(Int16Array)
+  VISIT_PRIMITIVE(Int32Array)
+  VISIT_PRIMITIVE(Int64Array)
+  VISIT_PRIMITIVE(UInt8Array)
+  VISIT_PRIMITIVE(UInt16Array)
+  VISIT_PRIMITIVE(UInt32Array)
+  VISIT_PRIMITIVE(UInt64Array)
+  VISIT_PRIMITIVE(FloatArray)
+  VISIT_PRIMITIVE(DoubleArray)
+  VISIT_PRIMITIVE(BinaryArray)
+  VISIT_PRIMITIVE(StringArray)
 
 #undef VISIT_PRIMITIVE
 
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index 4bc4384f64eab..522b3bdcdbb47 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -82,11 +82,10 @@ class ARROW_EXPORT Message {
   static Status ReadFrom(const std::shared_ptr<Buffer>& metadata, io::InputStream* stream,
                          std::unique_ptr<Message>* out);
 
-  /// \brief Write length-prefixed metadata and body to output stream
+  /// \brief Return true if message type and contents are equal
   ///
-  /// \param[in] file output stream to write to
-  /// \param[out] output_length the number of bytes written
-  /// \return Status
+  /// \param other another message
+  /// \return true if contents equal
   bool Equals(const Message& other) const;
 
   /// \brief the Message metadata
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 54174f9ca1733..e90dc1e5234ee 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -54,8 +54,8 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
 
   /// Create batch reader from generic MessageReader
   ///
-  /// \param(in) message_reader a MessageReader implementation
-  /// \param(out) out the created RecordBatchReader object
+  /// \param[in] message_reader a MessageReader implementation
+  /// \param[out] out the created RecordBatchReader object
   /// \return Status
   static Status Open(std::unique_ptr<MessageReader> message_reader,
                      std::shared_ptr<RecordBatchReader>* out);
@@ -72,9 +72,9 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
 
   /// \brief Record batch stream reader from InputStream
   ///
-  /// \param(in) stream an input stream instance. Must stay alive throughout
+  /// \param[in] stream an input stream instance. Must stay alive throughout
   /// lifetime of stream reader
-  /// \param(out) out the created RecordBatchStreamReader object
+  /// \param[out] out the created RecordBatchStreamReader object
   /// \return Status
   static Status Open(io::InputStream* stream, std::shared_ptr<RecordBatchReader>* out);
 
@@ -112,8 +112,8 @@ class ARROW_EXPORT RecordBatchFileReader {
   /// metadata footer). The metadata must have been written with memory offsets
   /// relative to the start of the containing file
   ///
-  /// @param file: the data source
-  /// @param footer_offset: the position of the end of the Arrow "file"
+  /// @param file the data source
+  /// @param footer_offset the position of the end of the Arrow "file"
   static Status Open(io::RandomAccessFile* file, int64_t footer_offset,
                      std::shared_ptr<RecordBatchFileReader>* reader);
 
@@ -138,8 +138,8 @@ class ARROW_EXPORT RecordBatchFileReader {
   /// Read a record batch from the file. Does not copy memory if the input
   /// source supports zero-copy.
   ///
-  /// \param(in) i the index of the record batch to return
-  /// \param(out) batch the read batch
+  /// \param[in] i the index of the record batch to return
+  /// \param[out] batch the read batch
   /// \return Status
   Status ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch);
 
@@ -165,20 +165,19 @@ Status ReadSchema(io::InputStream* stream, std::shared_ptr<Schema>* out);
 /// Read record batch as encapsulated IPC message with metadata size prefix and
 /// header
 ///
-/// \param(in) schema the record batch schema
-/// \param(in) offset the file location of the start of the message
-/// \param(in) file the file where the batch is located
-/// \param(out) out the read record batch
+/// \param[in] schema the record batch schema
+/// \param[in] stream the file where the batch is located
+/// \param[out] out the read record batch
 ARROW_EXPORT
 Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, io::InputStream* stream,
                        std::shared_ptr<RecordBatch>* out);
 
 /// \brief Read record batch from file given metadata and schema
 ///
-/// \param(in) metadata a Message containing the record batch metadata
-/// \param(in) schema the record batch schema
-/// \param(in) file a random access file
-/// \param(out) out the read record batch
+/// \param[in] metadata a Message containing the record batch metadata
+/// \param[in] schema the record batch schema
+/// \param[in] file a random access file
+/// \param[out] out the read record batch
 ARROW_EXPORT
 Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
                        io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
@@ -186,7 +185,7 @@ Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& sc
 /// \brief Read record batch from fully encapulated Message
 ///
 /// \param[in] message a message instance containing metadata and body
-/// \param[in] schema
+/// \param[in] schema the record batch schema
 /// \param[out] out the resulting RecordBatch
 /// \return Status
 ARROW_EXPORT
@@ -195,11 +194,11 @@ Status ReadRecordBatch(const Message& message, const std::shared_ptr<Schema>& sc
 
 /// Read record batch from file given metadata and schema
 ///
-/// \param(in) metadata a Message containing the record batch metadata
-/// \param(in) schema the record batch schema
-/// \param(in) file a random access file
-/// \param(in) max_recursion_depth the maximum permitted nesting depth
-/// \param(out) out the read record batch
+/// \param[in] metadata a Message containing the record batch metadata
+/// \param[in] schema the record batch schema
+/// \param[in] file a random access file
+/// \param[in] max_recursion_depth the maximum permitted nesting depth
+/// \param[out] out the read record batch
 ARROW_EXPORT
 Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
                        int max_recursion_depth, io::RandomAccessFile* file,
@@ -207,9 +206,9 @@ Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& sc
 
 /// EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
 ///
-/// \param(in) offset the file location of the start of the message
-/// \param(in) file the file where the batch is located
-/// \param(out) out the read tensor
+/// \param[in] offset the file location of the start of the message
+/// \param[in] file the file where the batch is located
+/// \param[out] out the read tensor
 ARROW_EXPORT
 Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
                   std::shared_ptr<Tensor>* out);
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 9f557f6659eb4..9531fd77a24de 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -324,24 +324,24 @@ class RecordBatchSerializer : public ArrayVisitor {
 #define VISIT_FIXED_WIDTH(TYPE) \
   Status Visit(const TYPE& array) override { return VisitFixedWidth<TYPE>(array); }
 
-  VISIT_FIXED_WIDTH(Int8Array);
-  VISIT_FIXED_WIDTH(Int16Array);
-  VISIT_FIXED_WIDTH(Int32Array);
-  VISIT_FIXED_WIDTH(Int64Array);
-  VISIT_FIXED_WIDTH(UInt8Array);
-  VISIT_FIXED_WIDTH(UInt16Array);
-  VISIT_FIXED_WIDTH(UInt32Array);
-  VISIT_FIXED_WIDTH(UInt64Array);
-  VISIT_FIXED_WIDTH(HalfFloatArray);
-  VISIT_FIXED_WIDTH(FloatArray);
-  VISIT_FIXED_WIDTH(DoubleArray);
-  VISIT_FIXED_WIDTH(Date32Array);
-  VISIT_FIXED_WIDTH(Date64Array);
-  VISIT_FIXED_WIDTH(TimestampArray);
-  VISIT_FIXED_WIDTH(Time32Array);
-  VISIT_FIXED_WIDTH(Time64Array);
-  VISIT_FIXED_WIDTH(FixedSizeBinaryArray);
-  VISIT_FIXED_WIDTH(DecimalArray);
+  VISIT_FIXED_WIDTH(Int8Array)
+  VISIT_FIXED_WIDTH(Int16Array)
+  VISIT_FIXED_WIDTH(Int32Array)
+  VISIT_FIXED_WIDTH(Int64Array)
+  VISIT_FIXED_WIDTH(UInt8Array)
+  VISIT_FIXED_WIDTH(UInt16Array)
+  VISIT_FIXED_WIDTH(UInt32Array)
+  VISIT_FIXED_WIDTH(UInt64Array)
+  VISIT_FIXED_WIDTH(HalfFloatArray)
+  VISIT_FIXED_WIDTH(FloatArray)
+  VISIT_FIXED_WIDTH(DoubleArray)
+  VISIT_FIXED_WIDTH(Date32Array)
+  VISIT_FIXED_WIDTH(Date64Array)
+  VISIT_FIXED_WIDTH(TimestampArray)
+  VISIT_FIXED_WIDTH(Time32Array)
+  VISIT_FIXED_WIDTH(Time64Array)
+  VISIT_FIXED_WIDTH(FixedSizeBinaryArray)
+  VISIT_FIXED_WIDTH(DecimalArray)
 
 #undef VISIT_FIXED_WIDTH
 
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index df03f4f1381f7..89d3f5f268489 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -61,7 +61,7 @@ class ARROW_EXPORT RecordBatchWriter {
   virtual Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) = 0;
 
   /// \brief Write possibly-chunked table by creating sequence of record batches
-  /// \param[in] table
+  /// \param[in] table table to write
   /// \return Status
   Status WriteTable(const Table& table);
 
@@ -87,9 +87,9 @@ class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
   /// Create a new writer from stream sink and schema. User is responsible for
   /// closing the actual OutputStream.
   ///
-  /// \param(in) sink output stream to write to
-  /// \param(in) schema the schema of the record batches to be written
-  /// \param(out) out the created stream writer
+  /// \param[in] sink output stream to write to
+  /// \param[in] schema the schema of the record batches to be written
+  /// \param[out] out the created stream writer
   /// \return Status
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchWriter>* out);
@@ -121,9 +121,9 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
 
   /// Create a new writer from stream sink and schema
   ///
-  /// \param(in) sink output stream to write to
-  /// \param(in) schema the schema of the record batches to be written
-  /// \param(out) out the created stream writer
+  /// \param[in] sink output stream to write to
+  /// \param[in] schema the schema of the record batches to be written
+  /// \param[out] out the created stream writer
   /// \return Status
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchWriter>* out);
@@ -155,13 +155,13 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
 /// to the end of the body and end of the metadata / data header (suffixed by
 /// the header size) is returned in out-variables
 ///
-/// \param(in) buffer_start_offset the start offset to use in the buffer metadata,
+/// \param[in] buffer_start_offset the start offset to use in the buffer metadata,
 /// default should be 0
-/// \param(in) allow_64bit permit field lengths exceeding INT32_MAX. May not be
+/// \param[in] allow_64bit permit field lengths exceeding INT32_MAX. May not be
 /// readable by other Arrow implementations
-/// \param(out) metadata_length: the size of the length-prefixed flatbuffer
+/// \param[out] metadata_length the size of the length-prefixed flatbuffer
 /// including padding to a 64-byte boundary
-/// \param(out) body_length: the size of the contiguous buffer block plus
+/// \param[out] body_length the size of the contiguous buffer block plus
 /// padding bytes
 /// \return Status
 ///
@@ -198,7 +198,7 @@ Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
 /// \brief Serialize schema using stream writer as a sequence of one or more
 /// IPC messages
 ///
-/// \param[in] scheam the schema to write
+/// \param[in] schema the schema to write
 /// \param[in] pool a MemoryPool to allocate memory from
 /// \param[out] out the serialized schema
 /// \return Status
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index cc1acf4befd3a..31b9344240262 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -199,7 +199,7 @@ class ArrayPrinter : public PrettyPrinter {
     }
   }
 
-  Status Visit(const NullArray& array) { return Status::OK(); }
+  Status Visit(const NullArray&) { return Status::OK(); }
 
   template <typename T>
   typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value ||
@@ -213,7 +213,7 @@ class ArrayPrinter : public PrettyPrinter {
     return Status::OK();
   }
 
-  Status Visit(const IntervalArray& array) { return Status::NotImplemented("interval"); }
+  Status Visit(const IntervalArray&) { return Status::NotImplemented("interval"); }
 
   Status WriteValidityBitmap(const Array& array);
 
diff --git a/cpp/src/arrow/python/arrow_to_python.h b/cpp/src/arrow/python/arrow_to_python.h
index 3650c058efd97..7509f30eb4e90 100644
--- a/cpp/src/arrow/python/arrow_to_python.h
+++ b/cpp/src/arrow/python/arrow_to_python.h
@@ -54,7 +54,7 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out);
 /// _serialize_callback method for serialization and a _deserialize_callback
 /// method for deserialization. If context is None, no custom serialization
 /// will be attempted.
-/// \param[in] object
+/// \param[in] object object to deserialize
 /// \param[in] base a Python object holding the underlying data that any NumPy
 /// arrays will reference, to avoid premature deallocation
 /// \param[out] out the returned object
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 7151c94c0dbcd..8845ee7838e16 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -149,7 +149,7 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
 /// can fit
 ///
 /// \param[in] offset starting offset for appending
-/// \param[out] values_consumed ending offset where we stopped appending. Will
+/// \param[out] end_offset ending offset where we stopped appending. Will
 /// be length of arr if fully consumed
 /// \param[out] have_bytes true if we encountered any PyBytes object
 static Status AppendObjectStrings(PyArrayObject* arr, PyArrayObject* mask, int64_t offset,
diff --git a/cpp/src/arrow/python/numpy_to_arrow.h b/cpp/src/arrow/python/numpy_to_arrow.h
index 4a70b4bc5333b..4a870fff9fcf3 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.h
+++ b/cpp/src/arrow/python/numpy_to_arrow.h
@@ -42,7 +42,7 @@ namespace py {
 /// \param[in] pool Memory pool for any memory allocations
 /// \param[in] ao an ndarray with the array data
 /// \param[in] mo an ndarray with a null mask (True is null), optional
-/// \param[in] type
+/// \param[in] type a specific type to cast to, may be null
 /// \param[out] out a ChunkedArray, to accommodate chunked output
 ARROW_EXPORT
 Status NdarrayToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index c57091fc069e3..9ba78213b2ccd 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -297,12 +297,11 @@ class DictBuilder {
 
   /// Construct an Arrow StructArray representing the dictionary.
   /// Contains a field "keys" for the keys and "vals" for the values.
-
-  /// \param list_data
+  /// \param val_list_data
   ///    List containing the data from nested lists in the value
   ///   list of the dictionary
   ///
-  /// \param dict_data
+  /// \param val_dict_data
   ///   List containing the data from nested dictionaries in the
   ///   value list of the dictionary
   Status Finish(const Array* key_tuple_data, const Array* key_dict_data,
@@ -543,7 +542,11 @@ Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
     ScopedRef iterator(PyObject_GetIter(sequence));
     RETURN_IF_PYERROR();
     ScopedRef item;
-    while (item.reset(PyIter_Next(iterator.get())), item.get()) {
+    while (true) {
+      item.reset(PyIter_Next(iterator.get()));
+      if (!item.get()) {
+        break;
+      }
       RETURN_NOT_OK(Append(context, item.get(), &builder, &sublists, &subtuples,
                            &subdicts, &subsets, tensors_out));
     }
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index de7515101518a..4ebef720f7a19 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -70,11 +70,14 @@ static inline Status PyTime_from_int(int64_t val, const TimeUnit::type unit,
 }
 
 static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
-  struct tm date = {0};
+  struct tm date;
+  memset(&date, 0, sizeof(struct tm));
   date.tm_year = PyDateTime_GET_YEAR(pydate) - 1900;
   date.tm_mon = PyDateTime_GET_MONTH(pydate) - 1;
   date.tm_mday = PyDateTime_GET_DAY(pydate);
-  struct tm epoch = {0};
+  struct tm epoch;
+  memset(&epoch, 0, sizeof(struct tm));
+
   epoch.tm_year = 70;
   epoch.tm_mday = 1;
 #ifdef _MSC_VER
@@ -88,7 +91,8 @@ static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
 }
 
 static inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) {
-  struct tm datetime = {0};
+  struct tm datetime;
+  memset(&datetime, 0, sizeof(struct tm));
   datetime.tm_year = PyDateTime_GET_YEAR(pydatetime) - 1900;
   datetime.tm_mon = PyDateTime_GET_MONTH(pydatetime) - 1;
   datetime.tm_mday = PyDateTime_GET_DAY(pydatetime);
@@ -96,7 +100,8 @@ static inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) {
   datetime.tm_min = PyDateTime_DATE_GET_MINUTE(pydatetime);
   datetime.tm_sec = PyDateTime_DATE_GET_SECOND(pydatetime);
   int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
-  struct tm epoch = {0};
+  struct tm epoch;
+  memset(&epoch, 0, sizeof(struct tm));
   epoch.tm_year = 70;
   epoch.tm_mday = 1;
 #ifdef _MSC_VER
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 3d3ecd2734eba..d0bbe7e0ec9a5 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -153,13 +153,6 @@ Status Column::ValidateData() {
 // ----------------------------------------------------------------------
 // RecordBatch methods
 
-void AssertBatchValid(const RecordBatch& batch) {
-  Status s = batch.Validate();
-  if (!s.ok()) {
-    DCHECK(false) << s.ToString();
-  }
-}
-
 RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows)
     : schema_(schema), num_rows_(num_rows) {
   boxed_columns_.resize(schema->num_fields());
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index a66772e5a7123..85fa2341a8206 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -113,7 +113,7 @@ class ARROW_EXPORT Column {
 /// sequence of fields, each a contiguous Arrow array
 class ARROW_EXPORT RecordBatch {
  public:
-  /// \param[in] schema
+  /// \param[in] schema The record batch schema
   /// \param[in] num_rows length of fields in the record batch. Each array
   /// should have the same length as num_rows
   /// \param[in] columns the record batch fields as vector of arrays
@@ -209,16 +209,16 @@ class ARROW_EXPORT Table {
  public:
   /// \brief Construct Table from schema and columns
   /// If columns is zero-length, the table's number of rows is zero
-  /// \param schema
-  /// \param columns
-  /// \param number of rows in table, -1 (default) to infer from columns
+  /// \param schema The table schema (column types)
+  /// \param columns The table's columns
+  /// \param num_rows number of rows in table, -1 (default) to infer from columns
   Table(const std::shared_ptr<Schema>& schema,
         const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows = -1);
 
   /// \brief Construct Table from schema and arrays
-  /// \param schema
-  /// \param arrays
-  /// \param number of rows in table, -1 (default) to infer from columns
+  /// \param schema The table schema (column types)
+  /// \param arrays The table's columns as arrays
+  /// \param num_rows number of rows in table, -1 (default) to infer from columns
   Table(const std::shared_ptr<Schema>& schema,
         const std::vector<std::shared_ptr<Array>>& arrays, int64_t num_rows = -1);
 
@@ -231,7 +231,7 @@ class ARROW_EXPORT Table {
   /// \return the table's schema
   std::shared_ptr<Schema> schema() const { return schema_; }
 
-  /// \param[i] i column index, does not boundscheck
+  /// \param[in] i column index, does not boundscheck
   /// \return the i-th column
   std::shared_ptr<Column> column(int i) const { return columns_[i]; }
 
@@ -283,7 +283,7 @@ class ARROW_EXPORT RecordBatchReader {
   /// Read the next record batch in the stream. Return nullptr for batch when
   /// reaching end of stream
   ///
-  /// \param(out) batch the next loaded batch, nullptr at end of stream
+  /// \param[out] batch the next loaded batch, nullptr at end of stream
   /// \return Status
   virtual Status ReadNext(std::shared_ptr<RecordBatch>* batch) = 0;
 
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 710344ab0d21e..3242fadd50dd6 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -68,8 +68,7 @@ TEST(TestField, TestAddMetadata) {
       new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
   auto f0 = field("f0", int32());
   auto f1 = field("f0", int32(), true, metadata);
-  std::shared_ptr<Field> f2;
-  ASSERT_OK(f0->AddMetadata(metadata, &f2));
+  std::shared_ptr<Field> f2 = f0->AddMetadata(metadata);
 
   ASSERT_FALSE(f2->Equals(*f0));
   ASSERT_TRUE(f2->Equals(*f1));
@@ -184,8 +183,7 @@ TEST_F(TestSchema, TestAddMetadata) {
   auto metadata = std::shared_ptr<KeyValueMetadata>(
       new KeyValueMetadata({"foo", "bar"}, {"bizz", "buzz"}));
   auto schema = std::make_shared<Schema>(fields);
-  std::shared_ptr<Schema> new_schema;
-  ASSERT_OK(schema->AddMetadata(metadata, &new_schema));
+  std::shared_ptr<Schema> new_schema = schema->AddMetadata(metadata);
   ASSERT_TRUE(metadata->Equals(*new_schema->metadata()));
 
   // Not copied
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 87bb73b1e7cc0..b9e314440f54f 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -36,12 +36,16 @@ std::shared_ptr<Field> Field::AddMetadata(
   return std::make_shared<Field>(name_, type_, nullable_, metadata);
 }
 
+#ifndef ARROW_NO_DEPRECATED_API
+
 Status Field::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
                           std::shared_ptr<Field>* out) const {
   *out = AddMetadata(metadata);
   return Status::OK();
 }
 
+#endif
+
 std::shared_ptr<Field> Field::RemoveMetadata() const {
   return std::make_shared<Field>(name_, type_, nullable_);
 }
@@ -307,12 +311,16 @@ std::shared_ptr<Schema> Schema::AddMetadata(
   return std::make_shared<Schema>(fields_, metadata);
 }
 
+#ifndef ARROW_NO_DEPRECATED_API
+
 Status Schema::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
                            std::shared_ptr<Schema>* out) const {
   *out = AddMetadata(metadata);
   return Status::OK();
 }
 
+#endif
+
 std::shared_ptr<const KeyValueMetadata> Schema::metadata() const { return metadata_; }
 
 std::shared_ptr<Schema> Schema::RemoveMetadata() const {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 7630f48151b28..c1da6e2704389 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -227,9 +227,11 @@ class ARROW_EXPORT Field {
 
   std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
 
-  /// \deprecated
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \note Deprecated since 0.8.0
   Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
                      std::shared_ptr<Field>* out) const;
+#endif
 
   std::shared_ptr<Field> AddMetadata(
       const std::shared_ptr<const KeyValueMetadata>& metadata) const;
@@ -749,9 +751,11 @@ class ARROW_EXPORT Schema {
                   std::shared_ptr<Schema>* out) const;
   Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
 
-  /// \deprecated
+#ifndef ARROW_NO_DEPRECATED_API
+  /// \note Deprecated since 0.8.0
   Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
                      std::shared_ptr<Schema>* out) const;
+#endif
 
   /// \brief Replace key-value metadata with new metadata
   ///
diff --git a/cpp/src/arrow/util/compression_brotli.cc b/cpp/src/arrow/util/compression_brotli.cc
index 1aaec11d9caa1..196c19a660b4d 100644
--- a/cpp/src/arrow/util/compression_brotli.cc
+++ b/cpp/src/arrow/util/compression_brotli.cc
@@ -25,6 +25,7 @@
 #include <brotli/types.h>
 
 #include "arrow/status.h"
+#include "arrow/util/macros.h"
 
 namespace arrow {
 
@@ -41,7 +42,8 @@ Status BrotliCodec::Decompress(int64_t input_len, const uint8_t* input,
   return Status::OK();
 }
 
-int64_t BrotliCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+int64_t BrotliCodec::MaxCompressedLen(int64_t input_len,
+                                      const uint8_t* ARROW_ARG_UNUSED(input)) {
   return BrotliEncoderMaxCompressedSize(input_len);
 }
 
diff --git a/cpp/src/arrow/util/compression_lz4.cc b/cpp/src/arrow/util/compression_lz4.cc
index cda40ad8c3a53..001edeb01b662 100644
--- a/cpp/src/arrow/util/compression_lz4.cc
+++ b/cpp/src/arrow/util/compression_lz4.cc
@@ -22,6 +22,7 @@
 #include <lz4.h>
 
 #include "arrow/status.h"
+#include "arrow/util/macros.h"
 
 namespace arrow {
 
@@ -39,7 +40,8 @@ Status Lz4Codec::Decompress(int64_t input_len, const uint8_t* input, int64_t out
   return Status::OK();
 }
 
-int64_t Lz4Codec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+int64_t Lz4Codec::MaxCompressedLen(int64_t input_len,
+                                   const uint8_t* ARROW_ARG_UNUSED(input)) {
   return LZ4_compressBound(static_cast<int>(input_len));
 }
 
diff --git a/cpp/src/arrow/util/compression_snappy.cc b/cpp/src/arrow/util/compression_snappy.cc
index e284bd4358dd6..2edaef7cf4b72 100644
--- a/cpp/src/arrow/util/compression_snappy.cc
+++ b/cpp/src/arrow/util/compression_snappy.cc
@@ -33,7 +33,8 @@ namespace arrow {
 // Snappy implementation
 
 Status SnappyCodec::Decompress(int64_t input_len, const uint8_t* input,
-                               int64_t output_len, uint8_t* output_buffer) {
+                               int64_t ARROW_ARG_UNUSED(output_len),
+                               uint8_t* output_buffer) {
   if (!snappy::RawUncompress(reinterpret_cast<const char*>(input),
                              static_cast<size_t>(input_len),
                              reinterpret_cast<char*>(output_buffer))) {
@@ -42,13 +43,14 @@ Status SnappyCodec::Decompress(int64_t input_len, const uint8_t* input,
   return Status::OK();
 }
 
-int64_t SnappyCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+int64_t SnappyCodec::MaxCompressedLen(int64_t input_len,
+                                      const uint8_t* ARROW_ARG_UNUSED(input)) {
   return snappy::MaxCompressedLength(input_len);
 }
 
 Status SnappyCodec::Compress(int64_t input_len, const uint8_t* input,
-                             int64_t output_buffer_len, uint8_t* output_buffer,
-                             int64_t* output_length) {
+                             int64_t ARROW_ARG_UNUSED(output_buffer_len),
+                             uint8_t* output_buffer, int64_t* output_length) {
   size_t output_len;
   snappy::RawCompress(reinterpret_cast<const char*>(input),
                       static_cast<size_t>(input_len),
diff --git a/cpp/src/arrow/util/compression_zlib.cc b/cpp/src/arrow/util/compression_zlib.cc
index 0656fd64b5cbd..3a520240ec61e 100644
--- a/cpp/src/arrow/util/compression_zlib.cc
+++ b/cpp/src/arrow/util/compression_zlib.cc
@@ -165,7 +165,7 @@ class GZipCodec::GZipCodecImpl {
     return Status::OK();
   }
 
-  int64_t MaxCompressedLen(int64_t input_length, const uint8_t* input) {
+  int64_t MaxCompressedLen(int64_t input_length, const uint8_t* ARROW_ARG_UNUSED(input)) {
     // Most be in compression mode
     if (!compressor_initialized_) {
       Status s = InitCompressor();
diff --git a/cpp/src/arrow/util/compression_zstd.cc b/cpp/src/arrow/util/compression_zstd.cc
index d19ac43449b66..20306f48efb8f 100644
--- a/cpp/src/arrow/util/compression_zstd.cc
+++ b/cpp/src/arrow/util/compression_zstd.cc
@@ -23,6 +23,7 @@
 #include <zstd.h>
 
 #include "arrow/status.h"
+#include "arrow/util/macros.h"
 
 using std::size_t;
 
@@ -42,7 +43,8 @@ Status ZSTDCodec::Decompress(int64_t input_len, const uint8_t* input, int64_t ou
   return Status::OK();
 }
 
-int64_t ZSTDCodec::MaxCompressedLen(int64_t input_len, const uint8_t* input) {
+int64_t ZSTDCodec::MaxCompressedLen(int64_t input_len,
+                                    const uint8_t* ARROW_ARG_UNUSED(input)) {
   return ZSTD_compressBound(input_len);
 }
 
diff --git a/cpp/src/arrow/util/cpu-info.cc b/cpp/src/arrow/util/cpu-info.cc
index 639f02e482be4..822fcaea74e1c 100644
--- a/cpp/src/arrow/util/cpu-info.cc
+++ b/cpp/src/arrow/util/cpu-info.cc
@@ -76,6 +76,8 @@ static struct {
 };
 static const int64_t num_flags = sizeof(flag_mappings) / sizeof(flag_mappings[0]);
 
+namespace {
+
 // Helper function to parse for hardware flags.
 // values contains a list of space-seperated flags.  check to see if the flags we
 // care about are present.
@@ -90,6 +92,8 @@ int64_t ParseCPUFlags(const string& values) {
   return flags;
 }
 
+}  // namespace
+
 #ifdef _WIN32
 bool RetrieveCacheSize(int64_t* cache_sizes) {
   if (!cache_sizes) {
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index a0dea09216a61..58496a874f17d 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -75,8 +75,8 @@ class ARROW_EXPORT Decimal128 {
   ///  -21 /  5 -> -4, -1
   ///   21 / -5 -> -4,  1
   ///  -21 / -5 ->  4, -1
-  /// @param right the number to divide by
-  /// @param remainder the remainder after the division
+  /// \param divisor the number to divide by
+  /// \param remainder the remainder after the division
   Status Divide(const Decimal128& divisor, Decimal128* result,
                 Decimal128* remainder) const;
 
diff --git a/cpp/src/arrow/util/hash-util.h b/cpp/src/arrow/util/hash-util.h
index d5fb212f33a88..3bba07bb20fb5 100644
--- a/cpp/src/arrow/util/hash-util.h
+++ b/cpp/src/arrow/util/hash-util.h
@@ -199,7 +199,7 @@ class HashUtil {
   static uint32_t FnvHash64to32(const void* data, int32_t bytes, uint32_t hash) {
     // IMPALA-2270: this function should never be used for zero-byte inputs.
     DCHECK_GT(bytes, 0);
-    uint64_t hash_u64 = hash | ((uint64_t)hash << 32);
+    uint64_t hash_u64 = hash | (static_cast<uint64_t>(hash) << 32);
     hash_u64 = FnvHash64(data, bytes, hash_u64);
     return static_cast<uint32_t>((hash_u64 >> 32) ^ (hash_u64 & 0xFFFFFFFF));
   }
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index b7e2ceea9f3d8..40a51cb5569c1 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -90,7 +90,7 @@ namespace internal {
 class NullLog {
  public:
   template <class T>
-  NullLog& operator<<(const T& t) {
+  NullLog& operator<<(const T& ARROW_ARG_UNUSED(t)) {
     return *this;
   }
 };
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index 2f1db0924a179..a5f6e57079a43 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -26,7 +26,7 @@
 #endif
 
 #define ARROW_UNUSED(x) (void)x
-
+#define ARROW_ARG_UNUSED(x)
 //
 // GCC can be told that a certain branch is not likely to be taken (for
 // instance, a CHECK failure), and use that information in static analysis.
diff --git a/cpp/src/arrow/util/sse-util.h b/cpp/src/arrow/util/sse-util.h
index a0ec8a2e93911..32ac43f919bfa 100644
--- a/cpp/src/arrow/util/sse-util.h
+++ b/cpp/src/arrow/util/sse-util.h
@@ -176,27 +176,27 @@ static inline int SSE4_cmpestri(__m128i str1, int len1, __m128i str2, int len2)
   return 0;
 }
 
-static inline uint32_t SSE4_crc32_u8(uint32_t crc, uint8_t v) {
+static inline uint32_t SSE4_crc32_u8(uint32_t, uint8_t) {
   DCHECK(false) << "CPU doesn't support SSE 4.2";
   return 0;
 }
 
-static inline uint32_t SSE4_crc32_u16(uint32_t crc, uint16_t v) {
+static inline uint32_t SSE4_crc32_u16(uint32_t, uint16_t) {
   DCHECK(false) << "CPU doesn't support SSE 4.2";
   return 0;
 }
 
-static inline uint32_t SSE4_crc32_u32(uint32_t crc, uint32_t v) {
+static inline uint32_t SSE4_crc32_u32(uint32_t, uint32_t) {
   DCHECK(false) << "CPU doesn't support SSE 4.2";
   return 0;
 }
 
-static inline uint32_t SSE4_crc32_u64(uint32_t crc, uint64_t v) {
+static inline uint32_t SSE4_crc32_u64(uint32_t, uint64_t) {
   DCHECK(false) << "CPU doesn't support SSE 4.2";
   return 0;
 }
 
-static inline int64_t POPCNT_popcnt_u64(uint64_t a) {
+static inline int64_t POPCNT_popcnt_u64(uint64_t) {
   DCHECK(false) << "CPU doesn't support SSE 4.2";
   return 0;
 }
@@ -205,27 +205,27 @@ static inline int64_t POPCNT_popcnt_u64(uint64_t a) {
 
 #else
 
-static inline uint32_t SSE4_crc32_u8(uint32_t crc, uint8_t v) {
+static inline uint32_t SSE4_crc32_u8(uint32_t, uint8_t) {
   DCHECK(false) << "SSE support is not enabled";
   return 0;
 }
 
-static inline uint32_t SSE4_crc32_u16(uint32_t crc, uint16_t v) {
+static inline uint32_t SSE4_crc32_u16(uint32_t, uint16_t) {
   DCHECK(false) << "SSE support is not enabled";
   return 0;
 }
 
-static inline uint32_t SSE4_crc32_u32(uint32_t crc, uint32_t v) {
+static inline uint32_t SSE4_crc32_u32(uint32_t, uint32_t) {
   DCHECK(false) << "SSE support is not enabled";
   return 0;
 }
 
-static inline uint32_t SSE4_crc32_u64(uint32_t crc, uint64_t v) {
+static inline uint32_t SSE4_crc32_u64(uint32_t, uint64_t) {
   DCHECK(false) << "SSE support is not enabled";
   return 0;
 }
 
-static inline int64_t POPCNT_popcnt_u64(uint64_t a) {
+static inline int64_t POPCNT_popcnt_u64(uint64_t) {
   DCHECK(false) << "SSE support is not enabled";
   return 0;
 }
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index 53a0fee0aae2d..a7b01b0f6315a 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -56,10 +56,7 @@ ARRAY_VISITOR_DEFAULT(ListArray);
 ARRAY_VISITOR_DEFAULT(StructArray);
 ARRAY_VISITOR_DEFAULT(UnionArray);
 ARRAY_VISITOR_DEFAULT(DictionaryArray);
-
-Status ArrayVisitor::Visit(const DecimalArray& array) {
-  return Status::NotImplemented("decimal");
-}
+ARRAY_VISITOR_DEFAULT(DecimalArray);
 
 #undef ARRAY_VISITOR_DEFAULT
 
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index ed1762b32e5f4..c9339545fcaa9 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -94,7 +94,14 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   set_property(SOURCE malloc.cc
     APPEND_STRING
     PROPERTY COMPILE_FLAGS
-    " -Wno-parentheses-equality -Wno-shorten-64-to-32")
+    " -Wno-parentheses-equality \
+-Wno-shorten-64-to-32 \
+-Wno-unused-macros ")
+
+  set_property(SOURCE thirdparty/xxhash.cc
+    APPEND_STRING
+    PROPERTY COMPILE_FLAGS
+    "-Wno-unused-macros")
 endif()
 
 if ("${COMPILER_FAMILY}" STREQUAL "gcc")
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 5e28d4f2af7be..3f99fe04708ac 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -286,7 +286,6 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
 /// calls will not do anything. The client will only send a message to the store
 /// releasing the object when the client is truly done with the object.
 ///
-/// @param conn The plasma connection.
 /// @param object_id The object ID to attempt to release.
 Status PlasmaClient::PerformRelease(const ObjectID& object_id) {
   // Decrement the count of the number of instances of this object that are
@@ -401,7 +400,8 @@ static inline bool compute_object_hash_parallel(XXH64_state_t* hash_state,
     }
   }
 
-  XXH64_update(hash_state, (unsigned char*)threadhash, sizeof(threadhash));
+  XXH64_update(hash_state, reinterpret_cast<unsigned char*>(threadhash),
+               sizeof(threadhash));
   return true;
 }
 
@@ -409,12 +409,14 @@ static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
   XXH64_state_t hash_state;
   XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
   if (obj_buffer.data_size >= kBytesInMB) {
-    compute_object_hash_parallel(&hash_state, (unsigned char*)obj_buffer.data,
+    compute_object_hash_parallel(&hash_state,
+                                 reinterpret_cast<unsigned char*>(obj_buffer.data),
                                  obj_buffer.data_size);
   } else {
-    XXH64_update(&hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
+    XXH64_update(&hash_state, reinterpret_cast<unsigned char*>(obj_buffer.data),
+                 obj_buffer.data_size);
   }
-  XXH64_update(&hash_state, (unsigned char*)obj_buffer.metadata,
+  XXH64_update(&hash_state, reinterpret_cast<unsigned char*>(obj_buffer.metadata),
                obj_buffer.metadata_size);
   return XXH64_digest(&hash_state);
 }
@@ -548,8 +550,6 @@ Status PlasmaClient::Disconnect() {
   return Status::OK();
 }
 
-#define h_addr h_addr_list[0]
-
 Status PlasmaClient::Transfer(const char* address, int port, const ObjectID& object_id) {
   return SendDataRequest(manager_conn_, object_id, address, port);
 }
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index 50ec55f5ec8f3..145942441c9f1 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -82,15 +82,15 @@ class ARROW_EXPORT PlasmaClient {
   /// Connect to the local plasma store and plasma manager. Return
   /// the resulting connection.
   ///
-  /// @param store_socket_name The name of the UNIX domain socket to use to
+  /// \param store_socket_name The name of the UNIX domain socket to use to
   ///        connect to the Plasma store.
-  /// @param manager_socket_name The name of the UNIX domain socket to use to
+  /// \param manager_socket_name The name of the UNIX domain socket to use to
   ///        connect to the local Plasma manager. If this is "", then this
   ///        function will not connect to a manager.
-  /// @param release_delay Number of released objects that are kept around
+  /// \param release_delay Number of released objects that are kept around
   ///        and not evicted to avoid too many munmaps.
-  /// @param num_retries number of attempts to connect to IPC socket, default 50
-  /// @return The return status.
+  /// \param num_retries number of attempts to connect to IPC socket, default 50
+  /// \return The return status.
   Status Connect(const std::string& store_socket_name,
                  const std::string& manager_socket_name, int release_delay,
                  int num_retries = -1);
@@ -98,17 +98,17 @@ class ARROW_EXPORT PlasmaClient {
   /// Create an object in the Plasma Store. Any metadata for this object must be
   /// be passed in when the object is created.
   ///
-  /// @param object_id The ID to use for the newly created object.
-  /// @param data_size The size in bytes of the space to be allocated for this
+  /// \param object_id The ID to use for the newly created object.
+  /// \param data_size The size in bytes of the space to be allocated for this
   /// object's
   ///        data (this does not include space used for metadata).
-  /// @param metadata The object's metadata. If there is no metadata, this
+  /// \param metadata The object's metadata. If there is no metadata, this
   /// pointer
   ///        should be NULL.
-  /// @param metadata_size The size in bytes of the metadata. If there is no
+  /// \param metadata_size The size in bytes of the metadata. If there is no
   ///        metadata, this should be 0.
-  /// @param data The address of the newly created object will be written here.
-  /// @return The return status.
+  /// \param data The address of the newly created object will be written here.
+  /// \return The return status.
   Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
                 int64_t metadata_size, uint8_t** data);
 
@@ -119,14 +119,14 @@ class ARROW_EXPORT PlasmaClient {
   /// but
   /// the caller should not release objects that were not retrieved.
   ///
-  /// @param object_ids The IDs of the objects to get.
-  /// @param num_object_ids The number of object IDs to get.
-  /// @param timeout_ms The amount of time in milliseconds to wait before this
+  /// \param object_ids The IDs of the objects to get.
+  /// \param num_objects The number of object IDs to get.
+  /// \param timeout_ms The amount of time in milliseconds to wait before this
   ///        request times out. If this value is -1, then no timeout is set.
-  /// @param object_buffers An array where the results will be stored. If the
+  /// \param object_buffers An array where the results will be stored. If the
   /// data
   ///        size field is -1, then the object was not retrieved.
-  /// @return The return status.
+  /// \return The return status.
   Status Get(const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms,
              ObjectBuffer* object_buffers);
 
@@ -136,8 +136,8 @@ class ARROW_EXPORT PlasmaClient {
   /// the address returned by Get is no longer valid. This should be called
   /// once for each call to Get (with the same object ID).
   ///
-  /// @param object_id The ID of the object that is no longer needed.
-  /// @return The return status.
+  /// \param object_id The ID of the object that is no longer needed.
+  /// \return The return status.
   Status Release(const ObjectID& object_id);
 
   /// Check if the object store contains a particular object and the object has
@@ -146,18 +146,18 @@ class ARROW_EXPORT PlasmaClient {
   /// @todo: We may want to indicate if the object has been created but not
   /// sealed.
   ///
-  /// @param object_id The ID of the object whose presence we are checking.
-  /// @param has_object The function will write true at this address if
+  /// \param object_id The ID of the object whose presence we are checking.
+  /// \param has_object The function will write true at this address if
   ///        the object is present and false if it is not present.
-  /// @return The return status.
+  /// \return The return status.
   Status Contains(const ObjectID& object_id, bool* has_object);
 
   /// Seal an object in the object store. The object will be immutable after
   /// this
   /// call.
   ///
-  /// @param object_id The ID of the object to seal.
-  /// @return The return status.
+  /// \param object_id The ID of the object to seal.
+  /// \return The return status.
   Status Seal(const ObjectID& object_id);
 
   /// Delete an object from the object store. This currently assumes that the
@@ -166,52 +166,51 @@ class ARROW_EXPORT PlasmaClient {
   /// @todo We may want to allow the deletion of objects that are not present or
   ///       haven't been sealed.
   ///
-  /// @param object_id The ID of the object to delete.
-  /// @return The return status.
+  /// \param object_id The ID of the object to delete.
+  /// \return The return status.
   Status Delete(const ObjectID& object_id);
 
   /// Delete objects until we have freed up num_bytes bytes or there are no more
   /// released objects that can be deleted.
   ///
-  /// @param num_bytes The number of bytes to try to free up.
-  /// @param num_bytes_evicted Out parameter for total number of bytes of space
+  /// \param num_bytes The number of bytes to try to free up.
+  /// \param num_bytes_evicted Out parameter for total number of bytes of space
   /// retrieved.
-  /// @return The return status.
+  /// \return The return status.
   Status Evict(int64_t num_bytes, int64_t& num_bytes_evicted);
 
   /// Compute the hash of an object in the object store.
   ///
-  /// @param conn The object containing the connection state.
-  /// @param object_id The ID of the object we want to hash.
-  /// @param digest A pointer at which to return the hash digest of the object.
+  /// \param object_id The ID of the object we want to hash.
+  /// \param digest A pointer at which to return the hash digest of the object.
   ///        The pointer must have at least kDigestSize bytes allocated.
-  /// @return The return status.
+  /// \return The return status.
   Status Hash(const ObjectID& object_id, uint8_t* digest);
 
   /// Subscribe to notifications when objects are sealed in the object store.
   /// Whenever an object is sealed, a message will be written to the client
   /// socket that is returned by this method.
   ///
-  /// @param fd Out parameter for the file descriptor the client should use to
+  /// \param fd Out parameter for the file descriptor the client should use to
   /// read notifications
   ///         from the object store about sealed objects.
-  /// @return The return status.
+  /// \return The return status.
   Status Subscribe(int* fd);
 
   /// Receive next object notification for this client if Subscribe has been called.
   ///
-  /// @param fd The file descriptor we are reading the notification from.
-  /// @param object_id Out parameter, the object_id of the object that was sealed.
-  /// @param data_size Out parameter, the data size of the object that was sealed.
-  /// @param metadata_size Out parameter, the metadata size of the object that was sealed.
-  /// @return The return status.
+  /// \param fd The file descriptor we are reading the notification from.
+  /// \param object_id Out parameter, the object_id of the object that was sealed.
+  /// \param data_size Out parameter, the data size of the object that was sealed.
+  /// \param metadata_size Out parameter, the metadata size of the object that was sealed.
+  /// \return The return status.
   Status GetNotification(int fd, ObjectID* object_id, int64_t* data_size,
                          int64_t* metadata_size);
 
   /// Disconnect from the local plasma instance, including the local store and
   /// manager.
   ///
-  /// @return The return status.
+  /// \return The return status.
   Status Disconnect();
 
   /// Attempt to initiate the transfer of some objects from remote Plasma
@@ -236,17 +235,17 @@ class ARROW_EXPORT PlasmaClient {
   /// This method is idempotent in the sense that it is ok to call it multiple
   /// times.
   ///
-  /// @param num_object_ids The number of object IDs fetch is being called on.
-  /// @param object_ids The IDs of the objects that fetch is being called on.
-  /// @return The return status.
+  /// \param num_object_ids The number of object IDs fetch is being called on.
+  /// \param object_ids The IDs of the objects that fetch is being called on.
+  /// \return The return status.
   Status Fetch(int num_object_ids, const ObjectID* object_ids);
 
   /// Wait for (1) a specified number of objects to be available (sealed) in the
   /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to
   /// expire. This is a blocking call.
   ///
-  /// @param num_object_requests Size of the object_requests array.
-  /// @param object_requests Object event array. Each element contains a request
+  /// \param num_object_requests Size of the object_requests array.
+  /// \param object_requests Object event array. Each element contains a request
   ///        for a particular object_id. The type of request is specified in the
   ///        "type" field.
   ///        - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes
@@ -260,36 +259,34 @@ class ARROW_EXPORT PlasmaClient {
   ///          available either at the local Plasma Store or on a remote Plasma
   ///          Store. In this case, the functions sets the "status" field to
   ///          ObjectStatus_Local or ObjectStatus_Remote.
-  /// @param num_ready_objects The number of requests in object_requests array
+  /// \param num_ready_objects The number of requests in object_requests array
   /// that
   ///        must be satisfied before the function returns, unless it timeouts.
   ///        The num_ready_objects should be no larger than num_object_requests.
-  /// @param timeout_ms Timeout value in milliseconds. If this timeout expires
+  /// \param timeout_ms Timeout value in milliseconds. If this timeout expires
   ///        before min_num_ready_objects of requests are satisfied, the
   ///        function
   ///        returns.
-  /// @param num_objects_ready Out parameter for number of satisfied requests in
+  /// \param num_objects_ready Out parameter for number of satisfied requests in
   ///        the object_requests list. If the returned number is less than
   ///        min_num_ready_objects this means that timeout expired.
-  /// @return The return status.
+  /// \return The return status.
   Status Wait(int64_t num_object_requests, ObjectRequest* object_requests,
               int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
 
   /// Transfer local object to a different plasma manager.
   ///
-  /// @param conn The object containing the connection state.
-  /// @param addr IP address of the plasma manager we are transfering to.
-  /// @param port Port of the plasma manager we are transfering to.
-  /// @object_id ObjectID of the object we are transfering.
-  /// @return The return status.
+  /// \param addr IP address of the plasma manager we are transfering to.
+  /// \param port Port of the plasma manager we are transfering to.
+  /// \param object_id ObjectID of the object we are transfering.
+  /// \return The return status.
   Status Transfer(const char* addr, int port, const ObjectID& object_id);
 
   /// Return the status of a given object. This method may query the object
   /// table.
   ///
-  /// @param conn The object containing the connection state.
-  /// @param object_id The ID of the object whose status we query.
-  /// @param object_status Out parameter for object status. Can take the
+  /// \param object_id The ID of the object whose status we query.
+  /// \param object_status Out parameter for object status. Can take the
   ///         following values.
   ///         - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma
   ///         Store.
@@ -300,13 +297,12 @@ class ARROW_EXPORT PlasmaClient {
   ///           Plasma Store.
   ///         - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the
   ///           system.
-  /// @return The return status.
+  /// \return The return status.
   Status Info(const ObjectID& object_id, int* object_status);
 
   /// Get the file descriptor for the socket connection to the plasma manager.
   ///
-  /// @param conn The plasma connection.
-  /// @return The file descriptor for the manager connection. If there is no
+  /// \return The file descriptor for the manager connection. If there is no
   ///         connection to the manager, this is -1.
   int get_manager_fd();
 
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
index 42914848645de..419c2ebfba7fe 100644
--- a/cpp/src/plasma/events.h
+++ b/cpp/src/plasma/events.h
@@ -66,7 +66,6 @@ class EventLoop {
   /// Remove a file event handler from the event loop.
   ///
   /// @param fd The file descriptor of the event handler.
-  /// @return Void.
   void RemoveFileEvent(int fd);
 
   /// Register a handler that will be called after a time slice of
@@ -84,8 +83,6 @@ class EventLoop {
   int RemoveTimer(int64_t timer_id);
 
   /// \brief Run the event loop.
-  ///
-  /// @return Void.
   void Start();
 
   /// \brief Stop the event loop
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
index dd1c873466ec9..de33dabcbafb3 100644
--- a/cpp/src/plasma/eviction_policy.h
+++ b/cpp/src/plasma/eviction_policy.h
@@ -70,7 +70,6 @@ class EvictionPolicy {
   /// cache.
   ///
   /// @param object_id The object ID of the object that was created.
-  /// @return Void.
   void object_created(const ObjectID& object_id);
 
   /// This method will be called when the Plasma store needs more space, perhaps
@@ -94,7 +93,6 @@ class EvictionPolicy {
   /// @param object_id The ID of the object that is now being used.
   /// @param objects_to_evict The object IDs that were chosen for eviction will
   ///        be stored into this vector.
-  /// @return Void.
   void begin_object_access(const ObjectID& object_id,
                            std::vector<ObjectID>* objects_to_evict);
 
@@ -106,7 +104,6 @@ class EvictionPolicy {
   /// @param object_id The ID of the object that is no longer being used.
   /// @param objects_to_evict The object IDs that were chosen for eviction will
   ///        be stored into this vector.
-  /// @return Void.
   void end_object_access(const ObjectID& object_id,
                          std::vector<ObjectID>* objects_to_evict);
 
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
index 79da4f43a192a..14db32085c0d7 100644
--- a/cpp/src/plasma/fling.cc
+++ b/cpp/src/plasma/fling.cc
@@ -64,8 +64,9 @@ int recv_fd(int conn) {
   for (struct cmsghdr* header = CMSG_FIRSTHDR(&msg); header != NULL;
        header = CMSG_NXTHDR(&msg, header))
     if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) {
-      ssize_t count =
-          (header->cmsg_len - (CMSG_DATA(header) - (unsigned char*)header)) / sizeof(int);
+      ssize_t count = (header->cmsg_len -
+                       (CMSG_DATA(header) - reinterpret_cast<unsigned char*>(header))) /
+                      sizeof(int);
       for (int i = 0; i < count; ++i) {
         int fd = (reinterpret_cast<int*>(CMSG_DATA(header)))[i];
         if (found_fd == -1) {
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
index 9bb433990820c..fc0010b0631e2 100644
--- a/cpp/src/plasma/io.cc
+++ b/cpp/src/plasma/io.cc
@@ -26,10 +26,6 @@
 
 using arrow::Status;
 
-/* Number of times we try binding to a socket. */
-#define NUM_BIND_ATTEMPTS 5
-#define BIND_TIMEOUT_MS 100
-
 /* Number of times we try connecting to a socket. */
 #define NUM_CONNECT_ATTEMPTS 50
 #define CONNECT_TIMEOUT_MS 100
@@ -134,7 +130,8 @@ int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
   }
   strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
 
-  if (bind(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) != 0) {
+  if (bind(socket_fd, reinterpret_cast<struct sockaddr*>(&socket_address),
+           sizeof(socket_address)) != 0) {
     ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname;
     close(socket_fd);
     return -1;
@@ -197,8 +194,8 @@ int connect_ipc_sock(const std::string& pathname) {
   }
   strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
 
-  if (connect(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) !=
-      0) {
+  if (connect(socket_fd, reinterpret_cast<struct sockaddr*>(&socket_address),
+              sizeof(socket_address)) != 0) {
     close(socket_fd);
     return -1;
   }
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 305e3d5d9cef1..2261b6a624a8c 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -79,8 +79,9 @@ Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object,
   PlasmaObjectSpec plasma_object(object->handle.store_fd, object->handle.mmap_size,
                                  object->data_offset, object->data_size,
                                  object->metadata_offset, object->metadata_size);
-  auto message = CreatePlasmaCreateReply(fbb, fbb.CreateString(object_id.binary()),
-                                         &plasma_object, (PlasmaError)error_code);
+  auto message =
+      CreatePlasmaCreateReply(fbb, fbb.CreateString(object_id.binary()), &plasma_object,
+                              static_cast<PlasmaError>(error_code));
   return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
 }
 
@@ -123,7 +124,7 @@ Status ReadSealRequest(uint8_t* data, size_t size, ObjectID* object_id,
 Status SendSealReply(int sock, ObjectID object_id, int error) {
   flatbuffers::FlatBufferBuilder fbb;
   auto message = CreatePlasmaSealReply(fbb, fbb.CreateString(object_id.binary()),
-                                       (PlasmaError)error);
+                                       static_cast<PlasmaError>(error));
   return PlasmaSend(sock, MessageType_PlasmaSealReply, &fbb, message);
 }
 
@@ -154,7 +155,7 @@ Status ReadReleaseRequest(uint8_t* data, size_t size, ObjectID* object_id) {
 Status SendReleaseReply(int sock, ObjectID object_id, int error) {
   flatbuffers::FlatBufferBuilder fbb;
   auto message = CreatePlasmaReleaseReply(fbb, fbb.CreateString(object_id.binary()),
-                                          (PlasmaError)error);
+                                          static_cast<PlasmaError>(error));
   return PlasmaSend(sock, MessageType_PlasmaReleaseReply, &fbb, message);
 }
 
@@ -185,7 +186,7 @@ Status ReadDeleteRequest(uint8_t* data, size_t size, ObjectID* object_id) {
 Status SendDeleteReply(int sock, ObjectID object_id, int error) {
   flatbuffers::FlatBufferBuilder fbb;
   auto message = CreatePlasmaDeleteReply(fbb, fbb.CreateString(object_id.binary()),
-                                         (PlasmaError)error);
+                                         static_cast<PlasmaError>(error));
   return PlasmaSend(sock, MessageType_PlasmaDeleteReply, &fbb, message);
 }
 
@@ -526,8 +527,8 @@ Status ReadDataReply(uint8_t* data, size_t size, ObjectID* object_id,
   auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
   DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
-  *object_size = (int64_t)message->object_size();
-  *metadata_size = (int64_t)message->metadata_size();
+  *object_size = static_cast<int64_t>(message->object_size());
+  *metadata_size = static_cast<int64_t>(message->metadata_size());
   return Status::OK();
 }
 
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index aaa2bad67c38b..72d199ba4658b 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -458,8 +458,7 @@ void PlasmaStore::disconnect_client(int client_fd) {
 /// be
 /// buffered, and this will be called again when the send buffer has room.
 ///
-/// @param client The client to send the notification to.
-/// @return Void.
+/// @param client_fd The client to send the notification to.
 void PlasmaStore::send_notifications(int client_fd) {
   auto it = pending_notifications_.find(client_fd);
 
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index 61a3a245610c3..d03d11f4ef0c4 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -77,7 +77,6 @@ class PlasmaStore {
   /// be called on objects that are returned by the eviction policy to evict.
   ///
   /// @param object_ids Object IDs of the objects to be deleted.
-  /// @return Void.
   void delete_objects(const std::vector<ObjectID>& object_ids);
 
   /// Process a get request from a client. This method assumes that we will
@@ -91,7 +90,6 @@ class PlasmaStore {
   /// @param client The client making this request.
   /// @param object_ids Object IDs of the objects to be gotten.
   /// @param timeout_ms The timeout for the get request in milliseconds.
-  /// @return Void.
   void process_get_request(Client* client, const std::vector<ObjectID>& object_ids,
                            int64_t timeout_ms);
 
@@ -101,7 +99,6 @@ class PlasmaStore {
   /// @param digest The digest of the object. This is used to tell if two
   /// objects
   ///        with the same object ID are the same.
-  /// @return Void.
   void seal_object(const ObjectID& object_id, unsigned char digest[]);
 
   /// Check if the plasma store contains an object:
@@ -115,25 +112,21 @@ class PlasmaStore {
   ///
   /// @param object_id The object ID of the object that is being released.
   /// @param client The client making this request.
-  /// @param Void.
   void release_object(const ObjectID& object_id, Client* client);
 
   /// Subscribe a file descriptor to updates about new sealed objects.
   ///
   /// @param client The client making this request.
-  /// @return Void.
   void subscribe_to_updates(Client* client);
 
   /// Connect a new client to the PlasmaStore.
   ///
   /// @param listener_sock The socket that is listening to incoming connections.
-  /// @return Void.
   void connect_client(int listener_sock);
 
   /// Disconnect a client from the PlasmaStore.
   ///
   /// @param client_fd The client file descriptor that is disconnected.
-  /// @return Void.
   void disconnect_client(int client_fd);
 
   void send_notifications(int client_fd);
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
index c76f5ce109251..7c9d90133a6d4 100644
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -43,7 +43,7 @@ int create_temp_file(void) {
  * Seek to the beginning of a file and read a message from it.
  *
  * @param fd File descriptor of the file.
- * @param message type Message type that we expect in the file.
+ * @param message_type Message type that we expect in the file.
  *
  * @return Pointer to the content of the message. Needs to be freed by the
  * caller.
@@ -226,7 +226,7 @@ TEST(PlasmaSerialization, DeleteReply) {
 
 TEST(PlasmaSerialization, StatusRequest) {
   int fd = create_temp_file();
-  int64_t num_objects = 2;
+  constexpr int64_t num_objects = 2;
   ObjectID object_ids[num_objects];
   object_ids[0] = ObjectID::from_random();
   object_ids[1] = ObjectID::from_random();
@@ -249,10 +249,11 @@ TEST(PlasmaSerialization, StatusReply) {
   ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusReply);
   int64_t num_objects = ReadStatusReply_num_objects(data.data(), data.size());
-  ObjectID object_ids_read[num_objects];
-  int object_statuses_read[num_objects];
-  ARROW_CHECK_OK(ReadStatusReply(data.data(), data.size(), object_ids_read,
-                                 object_statuses_read, num_objects));
+
+  std::vector<ObjectID> object_ids_read(num_objects);
+  std::vector<int> object_statuses_read(num_objects);
+  ARROW_CHECK_OK(ReadStatusReply(data.data(), data.size(), object_ids_read.data(),
+                                 object_statuses_read.data(), num_objects));
   ASSERT_EQ(object_ids[0], object_ids_read[0]);
   ASSERT_EQ(object_ids[1], object_ids_read[1]);
   ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
diff --git a/cpp/src/plasma/thirdparty/ae/ae_epoll.c b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
index 410aac70dc5af..2f70550a9803a 100644
--- a/cpp/src/plasma/thirdparty/ae/ae_epoll.c
+++ b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
@@ -72,7 +72,8 @@ static void aeApiFree(aeEventLoop *eventLoop) {
 
 static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
     aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee = {0}; /* avoid valgrind warning */
+    struct epoll_event ee;
+    memset(&ee, 0, sizeof(struct epoll_event)); // avoid valgrind warning
     /* If the fd was already monitored for some event, we need a MOD
      * operation. Otherwise we need an ADD operation. */
     int op = eventLoop->events[fd].mask == AE_NONE ?
@@ -89,7 +90,8 @@ static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
 
 static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
     aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee = {0}; /* avoid valgrind warning */
+    struct epoll_event ee;
+    memset(&ee, 0, sizeof(struct epoll_event)); // avoid valgrind warning
     int mask = eventLoop->events[fd].mask & (~delmask);
 
     ee.events = 0;

From 811e668e8a2af7ed4d0f2f237a223a7297f4bdc9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 1 Oct 2017 19:32:53 -0500
Subject: [PATCH 1102/1644] ARROW-1629: [C++] Add miscellaneous DCHECKs and
 minor changes based on infer tool output

This was an interesting journey through some esoterica. I went through all the warnings/errors that infer (fbinfer.com) outputs and made changes if it seemed warranted. Some of the checks might be overkill.

See https://gist.github.com/wesm/fc6809e4f4aaef3ecfeb21b8123627bc for a summary of actions on each warning

Most of the errors that Infer wasn't happy about were already addressed by DCHECKs. This was useful to go through all these cases -- in nearly all cases the null references are impossible or would be the result of an error on behalf of the application programmer. For example: we do not do array boundschecking in most cases in production builds, but these boundschecks are included in debug builds to assist with catching bugs caused by improper use by application developers.

As a matter of convention, we should not use error `Status` to do parameter validation or asserting pre-conditions that are the responsibility of the library user. If parameter validation is required in binding code (e.g. Python), then this validation should happen in the binding layer, not in the core C++ library.

There are some other cases where we have a `std::shared_ptr<T>` out variable with code like:

```
RETURN_NOT_OK(Foo(..., &out));
out->Method(...);
```

Here, infer complains that `out` could contain a null pointer, but our contract with developers is that if `Foo` returns successfully that `out` is non-null.

Interestingly, infer doesn't like some stack variables that are bound in C++11 lambda expressions. I noted these in the gist with `LAMBDA`.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1151 from wesm/fix-infer-issues and squashes the following commits:

f285be95 [Wes McKinney] Restore code paths for empty chunked arrays for backwards compat
5aa86ce2 [Wes McKinney] More DCHECK esoterica / tweaks based on infer report
22c5d361 [Wes McKinney] Address a couple more infer warnings
75131a6b [Wes McKinney] Some more minor infer fixes
5ff3e3a5 [Wes McKinney] Compilation fix
05316ce4 [Wes McKinney] Fix miscellaneous things that infer does not like. Make some Python helper functions internal / non-exported
---
 cpp/src/arrow/array.cc                   | 12 ++++++++++--
 cpp/src/arrow/array.h                    |  2 +-
 cpp/src/arrow/buffer.h                   |  2 +-
 cpp/src/arrow/io/hdfs-internal.cc        |  2 ++
 cpp/src/arrow/ipc/message.cc             | 10 ++++++++++
 cpp/src/arrow/ipc/reader.cc              |  8 +++++++-
 cpp/src/arrow/memory_pool.cc             |  3 ++-
 cpp/src/arrow/python/CMakeLists.txt      |  2 +-
 cpp/src/arrow/python/arrow_to_pandas.cc  |  7 ++++---
 cpp/src/arrow/python/arrow_to_python.cc  |  4 ++++
 cpp/src/arrow/python/builtin_convert.cc  |  2 +-
 cpp/src/arrow/python/helpers.cc          | 12 ++++++------
 cpp/src/arrow/python/helpers.h           | 24 ++++++++++++-----------
 cpp/src/arrow/python/numpy_to_arrow.cc   | 25 ++++++++++++------------
 cpp/src/arrow/python/python-test.cc      |  6 +++---
 cpp/src/arrow/table-test.cc              |  4 ----
 cpp/src/arrow/table.cc                   |  9 ++++++---
 cpp/src/arrow/table.h                    |  2 +-
 cpp/src/arrow/util/bit-util.cc           |  7 ++++---
 cpp/src/arrow/util/key_value_metadata.cc |  6 ++++--
 cpp/src/arrow/util/logging.h             |  2 +-
 cpp/src/plasma/client.cc                 |  1 +
 cpp/src/plasma/fling.cc                  |  3 +++
 cpp/src/plasma/io.cc                     |  1 +
 cpp/src/plasma/store.cc                  |  7 +++----
 25 files changed, 102 insertions(+), 61 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 92e8d0f0fa8f6..12922ae7b0ac6 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -224,6 +224,8 @@ Status ListArray::FromArrays(const Array& offsets, const Array& values,
 
 void ListArray::SetData(const std::shared_ptr<ArrayData>& data) {
   this->Array::SetData(data);
+  DCHECK_EQ(data->buffers.size(), 2);
+
   auto value_offsets = data->buffers[1];
   raw_value_offsets_ = value_offsets == nullptr
                            ? nullptr
@@ -246,6 +248,7 @@ BinaryArray::BinaryArray(const std::shared_ptr<ArrayData>& data) {
 }
 
 void BinaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
+  DCHECK_EQ(data->buffers.size(), 3);
   auto value_offsets = data->buffers[1];
   auto value_data = data->buffers[2];
   this->Array::SetData(data);
@@ -342,6 +345,7 @@ std::shared_ptr<Array> StructArray::field(int i) const {
   if (!boxed_fields_[i]) {
     DCHECK(MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
   }
+  DCHECK(boxed_fields_[i]);
   return boxed_fields_[i];
 }
 
@@ -351,6 +355,8 @@ std::shared_ptr<Array> StructArray::field(int i) const {
 void UnionArray::SetData(const std::shared_ptr<ArrayData>& data) {
   this->Array::SetData(data);
 
+  DCHECK_EQ(data->buffers.size(), 3);
+
   auto type_ids = data_->buffers[1];
   auto value_offsets = data_->buffers[2];
   raw_type_ids_ =
@@ -385,6 +391,7 @@ std::shared_ptr<Array> UnionArray::child(int i) const {
   if (!boxed_fields_[i]) {
     DCHECK(MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
   }
+  DCHECK(boxed_fields_[i]);
   return boxed_fields_[i];
 }
 
@@ -594,10 +601,11 @@ class ArrayDataWrapper {
 
 }  // namespace internal
 
-// Remove enclosing namespace after 0.7.0
 Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out) {
   internal::ArrayDataWrapper wrapper_visitor(data, out);
-  return VisitTypeInline(*data->type, &wrapper_visitor);
+  RETURN_NOT_OK(VisitTypeInline(*data->type, &wrapper_visitor));
+  DCHECK(out);
+  return Status::OK();
 }
 
 #ifndef ARROW_NO_DEPRECATED_API
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index ee29d95068276..4ad60eb77f48e 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -84,7 +84,7 @@ struct Decimal;
 /// input array and replace them with newly-allocated data, changing the output
 /// data type as well.
 struct ARROW_EXPORT ArrayData {
-  ArrayData() {}
+  ArrayData() : length(0) {}
 
   ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
             int64_t null_count = kUnknownNullCount, int64_t offset = 0)
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index dbd93763638b3..5f61ade956bef 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -201,7 +201,7 @@ class ARROW_EXPORT BufferBuilder {
     if (elements == 0) {
       return Status::OK();
     }
-    if (capacity_ == 0) {
+    if (buffer_ == nullptr) {
       buffer_ = std::make_shared<PoolBuffer>(pool_);
     }
     int64_t old_capacity = capacity_;
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index e6d0487a9db5c..9cd1c5052fe8d 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -44,6 +44,7 @@
 #include <boost/filesystem.hpp>  // NOLINT
 
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 namespace fs = boost::filesystem;
 
@@ -346,6 +347,7 @@ bool LibHdfsShim::HasPread() {
 tSize LibHdfsShim::Pread(hdfsFS fs, hdfsFile file, tOffset position, void* buffer,
                          tSize length) {
   GET_SYMBOL(this, hdfsPread);
+  DCHECK(this->hdfsPread);
   return this->hdfsPread(fs, file, position, buffer, length);
 }
 
diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc
index 082c92556b755..0c587ab7b234f 100644
--- a/cpp/src/arrow/ipc/message.cc
+++ b/cpp/src/arrow/ipc/message.cc
@@ -29,6 +29,7 @@
 #include "arrow/ipc/Schema_generated.h"
 #include "arrow/ipc/metadata-internal.h"
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace ipc {
@@ -194,9 +195,18 @@ std::string FormatMessageType(Message::Type type) {
 
 Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
                    std::unique_ptr<Message>* message) {
+  DCHECK_GT(static_cast<size_t>(metadata_length), sizeof(int32_t));
+
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
 
+  if (buffer->size() < metadata_length) {
+    std::stringstream ss;
+    ss << "Expected to read " << metadata_length << " metadata bytes but got "
+       << buffer->size();
+    return Status::Invalid(ss.str());
+  }
+
   int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
 
   if (flatbuffer_size + static_cast<int>(sizeof(int32_t)) > metadata_length) {
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 09def6ea6ed4b..e6ba50e742f7b 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -349,7 +349,6 @@ Status ReadDictionary(const Buffer& metadata, const DictionaryTypeMap& dictionar
       reinterpret_cast<const flatbuf::RecordBatch*>(dictionary_batch->data());
   RETURN_NOT_OK(
       ReadRecordBatch(batch_meta, dummy_schema, kMaxNestingDepth, file, &batch));
-
   if (batch->num_columns() != 1) {
     return Status::Invalid("Dictionary record batch must only contain one field");
   }
@@ -526,6 +525,13 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
     int file_end_size = static_cast<int>(magic_size + sizeof(int32_t));
     RETURN_NOT_OK(file_->ReadAt(footer_offset_ - file_end_size, file_end_size, &buffer));
 
+    const int64_t expected_footer_size = magic_size + sizeof(int32_t);
+    if (buffer->size() < expected_footer_size) {
+      std::stringstream ss;
+      ss << "Unable to read " << expected_footer_size << "from end of file";
+      return Status::Invalid(ss.str());
+    }
+
     if (memcmp(buffer->data() + sizeof(int32_t), kArrowMagicBytes, magic_size)) {
       return Status::Invalid("Not an Arrow file");
     }
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index d86fb08be8921..851065b521118 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -112,8 +112,9 @@ Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t
   // Note: We cannot use realloc() here as it doesn't guarantee alignment.
 
   // Allocate new chunk
-  uint8_t* out;
+  uint8_t* out = nullptr;
   RETURN_NOT_OK(AllocateAligned(new_size, &out));
+  DCHECK(out);
   // Copy contents and release old memory chunk
   memcpy(out, *ptr, static_cast<size_t>(std::min(new_size, old_size)));
 #ifdef _MSC_VER
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 7938d8473b6bf..af53a1631f708 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -40,7 +40,7 @@ endif()
 
 set(ARROW_PYTHON_MIN_TEST_LIBS
   arrow_python_test_main
-  arrow_python_shared
+  arrow_python_static
   arrow_shared)
 
 set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index be738e7f92247..88b594cac9428 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -615,8 +615,8 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
   PyAcquireGIL lock;
   OwnedRef decimal_ref;
   OwnedRef Decimal_ref;
-  RETURN_NOT_OK(ImportModule("decimal", &decimal_ref));
-  RETURN_NOT_OK(ImportFromModule(decimal_ref, "Decimal", &Decimal_ref));
+  RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_ref));
+  RETURN_NOT_OK(internal::ImportFromModule(decimal_ref, "Decimal", &Decimal_ref));
   PyObject* Decimal = Decimal_ref.obj();
 
   for (int c = 0; c < data.num_chunks(); c++) {
@@ -633,7 +633,8 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
         const uint8_t* raw_value = arr->GetValue(i);
         std::string decimal_string;
         RETURN_NOT_OK(RawDecimalToString(raw_value, precision, scale, &decimal_string));
-        RETURN_NOT_OK(DecimalFromString(Decimal, decimal_string, out_values++));
+        *out_values++ = internal::DecimalFromString(Decimal, decimal_string);
+        RETURN_IF_PYERROR();
       }
     }
   }
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index a281fe3c62902..b4f4a41f44068 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -59,6 +59,10 @@ Status DeserializeDict(PyObject* context, const Array& array, int64_t start_idx,
   const auto& data = static_cast<const StructArray&>(array);
   ScopedRef keys, vals;
   ScopedRef result(PyDict_New());
+  RETURN_IF_PYERROR();
+
+  DCHECK_EQ(2, data.num_fields());
+
   RETURN_NOT_OK(DeserializeList(context, *data.field(0), start_idx, stop_idx, base,
                                 tensors, keys.ref()));
   RETURN_NOT_OK(DeserializeList(context, *data.field(1), start_idx, stop_idx, base,
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index f9d7361e00425..69a19e7a36707 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -656,7 +656,7 @@ class DecimalConverter
   inline Status AppendItem(const OwnedRef& item) {
     /// TODO(phillipc): Check for nan?
     std::string string;
-    RETURN_NOT_OK(PythonDecimalToString(item.obj(), &string));
+    RETURN_NOT_OK(internal::PythonDecimalToString(item.obj(), &string));
 
     Decimal128 value;
     RETURN_NOT_OK(Decimal128::FromString(string, &value));
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index fb2fed7f0ca44..ad6a7f125e0a0 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -55,6 +55,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
   }
 }
 
+namespace internal {
+
 Status ImportModule(const std::string& module_name, OwnedRef* ref) {
   PyObject* module = PyImport_ImportModule(module_name.c_str());
   RETURN_IF_PYERROR();
@@ -106,10 +108,9 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision,
   return Decimal128::FromString(c_string, nullptr, precision, scale);
 }
 
-Status DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string,
-                         PyObject** out) {
+PyObject* DecimalFromString(PyObject* decimal_constructor,
+                            const std::string& decimal_string) {
   DCHECK_NE(decimal_constructor, nullptr);
-  DCHECK_NE(out, nullptr);
 
   auto string_size = decimal_string.size();
   DCHECK_GT(string_size, 0);
@@ -117,11 +118,10 @@ Status DecimalFromString(PyObject* decimal_constructor, const std::string& decim
   auto string_bytes = decimal_string.c_str();
   DCHECK_NE(string_bytes, nullptr);
 
-  *out = PyObject_CallFunction(decimal_constructor, const_cast<char*>("s#"), string_bytes,
+  return PyObject_CallFunction(decimal_constructor, const_cast<char*>("s#"), string_bytes,
                                string_size);
-  RETURN_IF_PYERROR();
-  return Status::OK();
 }
 
+}  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index 8b8c6673c8ebb..01ab91657d9fc 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -28,26 +28,28 @@
 #include "arrow/util/visibility.h"
 
 namespace arrow {
-
 namespace py {
 
 class OwnedRef;
 
-ARROW_EXPORT std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
+ARROW_EXPORT
+std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
+
+namespace internal {
 
-Status ARROW_EXPORT ImportModule(const std::string& module_name, OwnedRef* ref);
-Status ARROW_EXPORT ImportFromModule(const OwnedRef& module,
-                                     const std::string& module_name, OwnedRef* ref);
+Status ImportModule(const std::string& module_name, OwnedRef* ref);
+Status ImportFromModule(const OwnedRef& module, const std::string& module_name,
+                        OwnedRef* ref);
 
-Status ARROW_EXPORT PythonDecimalToString(PyObject* python_decimal, std::string* out);
+Status PythonDecimalToString(PyObject* python_decimal, std::string* out);
 
-Status ARROW_EXPORT InferDecimalPrecisionAndScale(PyObject* python_decimal,
-                                                  int* precision = nullptr,
-                                                  int* scale = nullptr);
+Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision = nullptr,
+                                     int* scale = nullptr);
 
-Status ARROW_EXPORT DecimalFromString(PyObject* decimal_constructor,
-                                      const std::string& decimal_string, PyObject** out);
+PyObject* DecimalFromString(PyObject* decimal_constructor,
+                            const std::string& decimal_string);
 
+}  // namespace internal
 }  // namespace py
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 8845ee7838e16..c0ce61cca55c3 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -652,8 +652,8 @@ Status NumPyConverter::ConvertDecimals() {
   // Import the decimal module and Decimal class
   OwnedRef decimal;
   OwnedRef Decimal;
-  RETURN_NOT_OK(ImportModule("decimal", &decimal));
-  RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+  RETURN_NOT_OK(internal::ImportModule("decimal", &decimal));
+  RETURN_NOT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal));
 
   Ndarray1DIndexer<PyObject*> objects(arr_);
   PyObject* object = objects[0];
@@ -661,7 +661,7 @@ Status NumPyConverter::ConvertDecimals() {
   int precision;
   int scale;
 
-  RETURN_NOT_OK(InferDecimalPrecisionAndScale(object, &precision, &scale));
+  RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(object, &precision, &scale));
 
   type_ = std::make_shared<DecimalType>(precision, scale);
 
@@ -672,7 +672,7 @@ Status NumPyConverter::ConvertDecimals() {
     object = objects[i];
     if (PyObject_IsInstance(object, Decimal.obj())) {
       std::string string;
-      RETURN_NOT_OK(PythonDecimalToString(object, &string));
+      RETURN_NOT_OK(internal::PythonDecimalToString(object, &string));
 
       Decimal128 value;
       RETURN_NOT_OK(Decimal128::FromString(string, &value));
@@ -823,7 +823,7 @@ Status NumPyConverter::ConvertObjectFixedWidthBytes(
     const std::shared_ptr<DataType>& type) {
   PyAcquireGIL lock;
 
-  int32_t byte_width = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
+  const int32_t byte_width = static_cast<const FixedSizeBinaryType&>(*type).byte_width();
 
   // The output type at this point is inconclusive because there may be bytes
   // and unicode mixed in the object array
@@ -893,8 +893,8 @@ Status NumPyConverter::ConvertObjectsInfer() {
 
   OwnedRef decimal;
   OwnedRef Decimal;
-  RETURN_NOT_OK(ImportModule("decimal", &decimal));
-  RETURN_NOT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+  RETURN_NOT_OK(internal::ImportModule("decimal", &decimal));
+  RETURN_NOT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal));
 
   for (int64_t i = 0; i < length_; ++i) {
     PyObject* obj = objects[i];
@@ -935,7 +935,7 @@ Status NumPyConverter::ConvertObjectsInfer() {
 Status NumPyConverter::ConvertObjectsInferAndCast() {
   size_t position = out_arrays_.size();
   RETURN_NOT_OK(ConvertObjectsInfer());
-
+  DCHECK_EQ(position + 1, out_arrays_.size());
   std::shared_ptr<Array> arr = out_arrays_[position];
 
   // Perform cast
@@ -1182,10 +1182,10 @@ Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type,
     LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
     LIST_CASE(STRING, NPY_OBJECT, StringType)
     case Type::LIST: {
-      const ListType& list_type = static_cast<const ListType&>(*type);
+      const auto& list_type = static_cast<const ListType&>(*type);
       auto value_builder = static_cast<ListBuilder*>(builder->value_builder());
 
-      auto foreach_item = [&](PyObject* object) {
+      auto foreach_item = [this, &builder, &value_builder, &list_type](PyObject* object) {
         if (PandasObjectIsNull(object)) {
           return builder->AppendNull();
         } else {
@@ -1219,8 +1219,9 @@ Status NdarrayToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
                       std::shared_ptr<ChunkedArray>* out) {
   NumPyConverter converter(pool, ao, mo, type, use_pandas_null_sentinels);
   RETURN_NOT_OK(converter.Convert());
-  DCHECK(converter.result()[0]);
-  *out = std::make_shared<ChunkedArray>(converter.result());
+  const auto& output_arrays = converter.result();
+  DCHECK_GT(output_arrays.size(), 0);
+  *out = std::make_shared<ChunkedArray>(output_arrays);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index e1796c097d660..86391a18598fd 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -39,10 +39,10 @@ TEST(DecimalTest, TestPythonDecimalToString) {
 
   OwnedRef decimal;
   OwnedRef Decimal;
-  ASSERT_OK(ImportModule("decimal", &decimal));
+  ASSERT_OK(internal::ImportModule("decimal", &decimal));
   ASSERT_NE(decimal.obj(), nullptr);
 
-  ASSERT_OK(ImportFromModule(decimal, "Decimal", &Decimal));
+  ASSERT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal));
   ASSERT_NE(Decimal.obj(), nullptr);
 
   std::string decimal_string("-39402950693754869342983");
@@ -61,7 +61,7 @@ TEST(DecimalTest, TestPythonDecimalToString) {
   ASSERT_NE(python_object, nullptr);
 
   std::string string_result;
-  ASSERT_OK(PythonDecimalToString(python_object, &string_result));
+  ASSERT_OK(internal::PythonDecimalToString(python_object, &string_result));
 }
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index b0aeed1925ece..b490310c26ae6 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -140,10 +140,6 @@ TEST_F(TestColumn, BasicAPI) {
   ASSERT_EQ(300, column_->length());
   ASSERT_EQ(30, column_->null_count());
   ASSERT_EQ(3, column_->data()->num_chunks());
-
-  // nullptr array should not break
-  column_.reset(new Column(f0, std::shared_ptr<Array>(nullptr)));
-  ASSERT_NE(column_.get(), nullptr);
 }
 
 TEST_F(TestColumn, ChunksInhomogeneous) {
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index d0bbe7e0ec9a5..009b5cf6373de 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -42,6 +42,8 @@ ChunkedArray::ChunkedArray(const ArrayVector& chunks) : chunks_(chunks) {
   }
 }
 
+std::shared_ptr<DataType> ChunkedArray::type() const { return chunks_[0]->type(); }
+
 bool ChunkedArray::Equals(const ChunkedArray& other) const {
   if (length_ != other.length()) {
     return false;
@@ -105,10 +107,10 @@ Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks)
 
 Column::Column(const std::shared_ptr<Field>& field, const std::shared_ptr<Array>& data)
     : field_(field) {
-  if (data) {
-    data_ = std::make_shared<ChunkedArray>(ArrayVector({data}));
-  } else {
+  if (!data) {
     data_ = std::make_shared<ChunkedArray>(ArrayVector({}));
+  } else {
+    data_ = std::make_shared<ChunkedArray>(ArrayVector({data}));
   }
 }
 
@@ -192,6 +194,7 @@ std::shared_ptr<Array> RecordBatch::column(int i) const {
   if (!boxed_columns_[i]) {
     DCHECK(MakeArray(columns_[i], &boxed_columns_[i]).ok());
   }
+  DCHECK(boxed_columns_[i]);
   return boxed_columns_[i];
 }
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 85fa2341a8206..324112bfb3e1d 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -53,7 +53,7 @@ class ARROW_EXPORT ChunkedArray {
 
   const ArrayVector& chunks() const { return chunks_; }
 
-  std::shared_ptr<DataType> type() const { return chunks_[0]->type(); }
+  std::shared_ptr<DataType> type() const;
 
   bool Equals(const ChunkedArray& other) const;
   bool Equals(const std::shared_ptr<ChunkedArray>& other) const;
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 15bf3595a9602..e0116cc567b9c 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -31,6 +31,7 @@
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 
@@ -48,9 +49,9 @@ Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes, MemoryPool* pool,
 
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(AllocateBuffer(pool, bit_length, &buffer));
-
-  memset(buffer->mutable_data(), 0, static_cast<size_t>(bit_length));
-  FillBitsFromBytes(bytes, buffer->mutable_data());
+  uint8_t* out_buf = buffer->mutable_data();
+  memset(out_buf, 0, static_cast<size_t>(bit_length));
+  FillBitsFromBytes(bytes, out_buf);
 
   *out = buffer;
   return Status::OK();
diff --git a/cpp/src/arrow/util/key_value_metadata.cc b/cpp/src/arrow/util/key_value_metadata.cc
index cf74ddf37b4df..4f379537c48a0 100644
--- a/cpp/src/arrow/util/key_value_metadata.cc
+++ b/cpp/src/arrow/util/key_value_metadata.cc
@@ -89,12 +89,14 @@ int64_t KeyValueMetadata::size() const {
 
 std::string KeyValueMetadata::key(int64_t i) const {
   DCHECK_GE(i, 0);
-  return keys_[static_cast<size_t>(i)];
+  DCHECK_LT(static_cast<size_t>(i), keys_.size());
+  return keys_[i];
 }
 
 std::string KeyValueMetadata::value(int64_t i) const {
   DCHECK_GE(i, 0);
-  return values_[static_cast<size_t>(i)];
+  DCHECK_LT(static_cast<size_t>(i), values_.size());
+  return values_[i];
 }
 
 std::shared_ptr<KeyValueMetadata> KeyValueMetadata::Copy() const {
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 40a51cb5569c1..39815f303151e 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -45,7 +45,7 @@ namespace arrow {
 #define ARROW_CHECK(condition)                           \
   (condition) ? 0                                        \
               : ::arrow::internal::FatalLog(ARROW_FATAL) \
-                    << __FILE__ << __LINE__ << " Check failed: " #condition " "
+                    << __FILE__ << ":" << __LINE__ << " Check failed: " #condition " "
 
 #ifdef NDEBUG
 #define ARROW_DFATAL ARROW_WARNING
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 3f99fe04708ac..e57a2a6f3008c 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -356,6 +356,7 @@ Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) {
     std::vector<uint8_t> buffer;
     RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaContainsReply, &buffer));
     ObjectID object_id2;
+    DCHECK_GT(buffer.size(), 0);
     RETURN_NOT_OK(
         ReadContainsReply(buffer.data(), buffer.size(), &object_id2, has_object));
   }
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
index 14db32085c0d7..b84648b25a9e7 100644
--- a/cpp/src/plasma/fling.cc
+++ b/cpp/src/plasma/fling.cc
@@ -37,6 +37,9 @@ int send_fd(int conn, int fd) {
   init_msg(&msg, &iov, buf, sizeof(buf));
 
   struct cmsghdr* header = CMSG_FIRSTHDR(&msg);
+  if (header == nullptr) {
+    return -1;
+  }
   header->cmsg_level = SOL_SOCKET;
   header->cmsg_type = SCM_RIGHTS;
   header->cmsg_len = CMSG_LEN(sizeof(int));
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
index fc0010b0631e2..afe7053329b1e 100644
--- a/cpp/src/plasma/io.cc
+++ b/cpp/src/plasma/io.cc
@@ -224,6 +224,7 @@ uint8_t* read_message_async(int sock) {
   uint8_t* message = reinterpret_cast<uint8_t*>(malloc(size));
   s = ReadBytes(sock, message, size);
   if (!s.ok()) {
+    free(message);
     /* The other side has closed the socket. */
     ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
     close(sock);
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 72d199ba4658b..210cce16238f8 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -665,7 +665,7 @@ class PlasmaStoreRunner {
   std::unique_ptr<PlasmaStore> store_;
 };
 
-static PlasmaStoreRunner* g_runner = nullptr;
+static std::unique_ptr<PlasmaStoreRunner> g_runner = nullptr;
 
 void HandleSignal(int signal) {
   if (signal == SIGTERM) {
@@ -683,10 +683,9 @@ void start_server(char* socket_name, int64_t system_memory, std::string plasma_d
   // to a client that has already died, the store could die.
   signal(SIGPIPE, SIG_IGN);
 
-  PlasmaStoreRunner runner;
-  g_runner = &runner;
+  g_runner.reset(new PlasmaStoreRunner());
   signal(SIGTERM, HandleSignal);
-  runner.Start(socket_name, system_memory, plasma_directory, hugepages_enabled);
+  g_runner->Start(socket_name, system_memory, plasma_directory, hugepages_enabled);
 }
 
 }  // namespace plasma

From af167fdcda9bbb4fe159fd25af440b1f0620721c Mon Sep 17 00:00:00 2001
From: Ofek Lev <ofekmeister@gmail.com>
Date: Tue, 26 Sep 2017 13:11:49 -0400
Subject: [PATCH 1103/1644] [Python] Update README.md to reflect that wheels
 are available on all platforms

Close #1136

Change-Id: Ib9261972fb720df351931902a1666f23b0a0132f
---
 python/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/README.md b/python/README.md
index 3c48d5d30b595..38c994013755a 100644
--- a/python/README.md
+++ b/python/README.md
@@ -32,7 +32,7 @@ package manager:
 conda install pyarrow -c conda-forge
 ```
 
-On Linux, you can also install binary wheels from PyPI with pip:
+On Linux/macOS and Windows, you can also install binary wheels from PyPI with pip:
 
 ```shell
 pip install pyarrow
@@ -65,4 +65,4 @@ python setup.py build_sphinx -s doc/source
 
 [1]: https://github.com/apache/parquet-cpp
 [2]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst
-[3]: https://github.com/pandas-dev/pandas
\ No newline at end of file
+[3]: https://github.com/pandas-dev/pandas

From c905783fd6b5a173fbf994fb5c9c17477a786554 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 2 Oct 2017 08:09:47 -0400
Subject: [PATCH 1104/1644] ARROW-1625: [Serialization] Support OrderedDict and
 defaultdict serialization

This PR adds support for OrderedDicts and default dicts using custom serialization handlers.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1152 from pcmoritz/pydict-exact2 and squashes the following commits:

431e0272 [Philipp Moritz] make cloudpickle optional
052b1aa9 [Philipp Moritz] I'd prefer this not to be a runtime dependency
db19ab9b [Philipp Moritz] add tests
799d983e [Philipp Moritz] do not interpret OrderedDict as dict
---
 ci/travis_script_python.sh                 |  1 +
 cpp/src/arrow/python/python_to_arrow.cc    |  2 +-
 python/pyarrow/tests/test_serialization.py | 38 ++++++++++++++++++++--
 python/requirements.txt                    |  1 +
 4 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index b779aec195637..694154335f179 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -36,6 +36,7 @@ which python
 
 conda install -y -q pip \
       nomkl \
+      cloudpickle \
       numpy=1.13.1 \
       pandas \
       cython \
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 9ba78213b2ccd..a693a081f787a 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -469,7 +469,7 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
   } else if (PyList_Check(elem)) {
     RETURN_NOT_OK(builder->AppendList(PyList_Size(elem)));
     sublists->push_back(elem);
-  } else if (PyDict_Check(elem)) {
+  } else if (PyDict_CheckExact(elem)) {
     RETURN_NOT_OK(builder->AppendDict(PyDict_Size(elem)));
     subdicts->push_back(elem);
   } else if (PyTuple_CheckExact(elem)) {
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 7c8cace8f0762..eab81c2b5b42d 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -19,7 +19,7 @@
 
 import pytest
 
-from collections import namedtuple
+from collections import namedtuple, OrderedDict, defaultdict
 import string
 import sys
 
@@ -50,6 +50,12 @@ def assert_equal(obj1, obj2):
                                                                   .format(
                                                                       obj1,
                                                                       obj2))
+        try:
+            # Workaround to make comparison of OrderedDicts work on Python 2.7
+            if obj1 == obj2:
+                return
+        except:
+            pass
         for key in obj1.__dict__.keys():
             if key not in special_keys:
                 assert_equal(obj1.__dict__[key], obj2.__dict__[key])
@@ -168,7 +174,8 @@ class CustomError(Exception):
 
 CUSTOM_OBJECTS = [Exception("Test object."), CustomError(), Point(11, y=22),
                   Foo(), Bar(), Baz(), Qux(), SubQux(), SubQuxPickle(),
-                  NamedTupleExample(1, 1.0, "hi", np.zeros([3, 5]), [1, 2, 3])]
+                  NamedTupleExample(1, 1.0, "hi", np.zeros([3, 5]), [1, 2, 3]),
+                  OrderedDict([("hello", 1), ("world", 2)])]
 
 
 def make_serialization_context():
@@ -213,6 +220,28 @@ def array_custom_deserializer(serialized_obj):
                               custom_serializer=lambda obj: str(obj),
                               custom_deserializer=deserializer)
 
+    def ordered_dict_custom_serializer(obj):
+        return list(obj.keys()), list(obj.values())
+
+    def ordered_dict_custom_deserializer(obj):
+        return OrderedDict(zip(obj[0], obj[1]))
+
+    context.register_type(OrderedDict, 20 * b"\x12", pickle=False,
+                          custom_serializer=ordered_dict_custom_serializer,
+                          custom_deserializer=ordered_dict_custom_deserializer)
+
+    def default_dict_custom_serializer(obj):
+        return list(obj.keys()), list(obj.values()), obj.default_factory
+
+    def default_dict_custom_deserializer(obj):
+        return defaultdict(obj[2], zip(obj[0], obj[1]))
+
+    context.register_type(defaultdict, 20 * b"\x13", pickle=False,
+                          custom_serializer=default_dict_custom_serializer,
+                          custom_deserializer=default_dict_custom_deserializer)
+
+    context.register_type(type(lambda: 0), 20 * b"\x14", pickle=True)
+
     return context
 
 
@@ -266,6 +295,11 @@ def test_custom_serialization(large_memory_map):
         for obj in CUSTOM_OBJECTS:
             serialization_roundtrip(obj, mmap)
 
+def test_default_dict_serialization(large_memory_map):
+    cloudpickle = pytest.importorskip("cloudpickle")
+    with pa.memory_map(large_memory_map, mode="r+") as mmap:
+        obj = defaultdict(lambda: 0, [("hello", 1), ("world", 2)])
+        serialization_roundtrip(obj, mmap)
 
 def test_numpy_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
diff --git a/python/requirements.txt b/python/requirements.txt
index 103f490588a5b..d2e28a7747ba8 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,3 +1,4 @@
 pytest
+cloudpickle
 numpy>=1.10.0
 six

From 82eea49b3eea6047f53478113ab3ff9a38f0d344 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 3 Oct 2017 09:01:05 -0400
Subject: [PATCH 1105/1644] [Java] Update pom versions to 0.8.0-SNAPSHOT

Change-Id: If33561a7858b8a16edd878fce2da6ec0af8a93c4
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 2 +-
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 12f67ad9791c5..cd2b6c0de6710 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.7.1</version>
+  <version>0.8.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 40671fff46c73..7efc8e6aa470c 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.7.1</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 354a611c8c601..0a0f2e0ce8f65 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.7.1</version>
+  <version>0.8.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index cadc5c0d4a1e2..f3c86688134dc 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.7.1</version>
+        <version>0.8.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index f6bba2e350f37..46e06aa1e3f97 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.7.1</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 988338c544580ffd367a5540f1061dd7b0fccc0e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 3 Oct 2017 08:59:03 -0400
Subject: [PATCH 1106/1644] ARROW-1634: [Website] Add release page for 0.7.1,
 update front page

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1154 from wesm/ARROW-1634 and squashes the following commits:

da21ff78 [Wes McKinney] Update install page
8468c71a [Wes McKinney] Add release page for 0.7.1, update front matter
---
 dev/release/changelog.py |  2 +-
 site/_release/0.7.1.md   | 91 ++++++++++++++++++++++++++++++++++++++++
 site/_release/index.md   |  2 +
 site/index.html          |  4 +-
 site/install.md          | 22 +++++-----
 5 files changed, 107 insertions(+), 14 deletions(-)
 create mode 100644 site/_release/0.7.1.md

diff --git a/dev/release/changelog.py b/dev/release/changelog.py
index 65881503abe27..23ac4189da9d9 100644
--- a/dev/release/changelog.py
+++ b/dev/release/changelog.py
@@ -83,7 +83,7 @@ def format_changelog_website(issues, out):
         'Improvement': NEW_FEATURE,
         'Wish': NEW_FEATURE,
         'Task': NEW_FEATURE,
-        'Test': NEW_FEATURE,
+        'Test': BUGFIX,
         'Bug': BUGFIX
     }
 
diff --git a/site/_release/0.7.1.md b/site/_release/0.7.1.md
new file mode 100644
index 0000000000000..6eb6bff2e8ebe
--- /dev/null
+++ b/site/_release/0.7.1.md
@@ -0,0 +1,91 @@
+---
+layout: default
+title: Apache Arrow 0.7.1 Release
+permalink: /release/0.7.1.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.7.1 (1 October 2017)
+
+This is a minor bug release. It was motivated by [ARROW-1601][3], but see the
+complete changelog.
+
+## Download
+
+* [**Source Artifacts**][2]
+* [Git tag][1]
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.7.0..apache-arrow-0.7.1
+    14  Wes McKinney
+     6  Kouhei Sutou
+     3  siddharth
+     2  Paul Taylor
+     2  Uwe L. Korn
+     1  Amir Malekpour
+     1  Bryan Cutler
+     1  Deepak Majeti
+     1  Kentaro Hayashi
+     1  Korn, Uwe
+     1  Li Jin
+     1  Rene Sugar
+     1  Tom Augspurger
+     1  Wataru Shimizu
+     1  m-nakamura145
+     1  rvernica
+```
+
+# Changelog
+
+## New Features and Improvements
+
+* [ARROW-1537](https://issues.apache.org/jira/browse/ARROW-1537) - [C++] Support building with full path install_name on macOS
+* [ARROW-1546](https://issues.apache.org/jira/browse/ARROW-1546) - [GLib] Support GLib 2.40 again
+* [ARROW-1548](https://issues.apache.org/jira/browse/ARROW-1548) - [GLib] Support build append in builder
+* [ARROW-1578](https://issues.apache.org/jira/browse/ARROW-1578) - [C++/Python] Run lint checks in Travis CI to fail for linting issues as early as possible
+* [ARROW-1592](https://issues.apache.org/jira/browse/ARROW-1592) - [GLib] Add GArrowUIntArrayBuilder
+* [ARROW-1608](https://issues.apache.org/jira/browse/ARROW-1608) - Support Release verification script on macOS
+* [ARROW-1612](https://issues.apache.org/jira/browse/ARROW-1612) - [GLib] add how to install for mac os to README
+* [ARROW-559](https://issues.apache.org/jira/browse/ARROW-559) - Script to easily verify release in all languages
+
+## Bug Fixes
+
+* [ARROW-1497](https://issues.apache.org/jira/browse/ARROW-1497) - [Java] JsonFileReader doesn't set value count for some vectors
+* [ARROW-1500](https://issues.apache.org/jira/browse/ARROW-1500) - [C++] Result of ftruncate ignored in MemoryMappedFile::Create
+* [ARROW-1529](https://issues.apache.org/jira/browse/ARROW-1529) - [GLib] Fix failure on macOS on Travis CI
+* [ARROW-1536](https://issues.apache.org/jira/browse/ARROW-1536) - [C++] Do not transitively depend on libboost_system
+* [ARROW-1542](https://issues.apache.org/jira/browse/ARROW-1542) - [C++] Windows release verification script should not modify conda environment
+* [ARROW-1544](https://issues.apache.org/jira/browse/ARROW-1544) - [JS] Export Vector type definitions
+* [ARROW-1545](https://issues.apache.org/jira/browse/ARROW-1545) - Int64Builder should not need int64() as arg
+* [ARROW-1550](https://issues.apache.org/jira/browse/ARROW-1550) - [Python] Fix flaky test on Windows
+* [ARROW-1554](https://issues.apache.org/jira/browse/ARROW-1554) - [Python] Document that pip wheels depend on MSVC14 runtime
+* [ARROW-1557](https://issues.apache.org/jira/browse/ARROW-1557) - [PYTHON] pyarrow.Table.from_arrays doesn't validate names length
+* [ARROW-1591](https://issues.apache.org/jira/browse/ARROW-1591) - C++: Xcode 9 is not correctly detected
+* [ARROW-1595](https://issues.apache.org/jira/browse/ARROW-1595) - [Python] Fix package dependency issues causing build failures
+* [ARROW-1601](https://issues.apache.org/jira/browse/ARROW-1601) - [C++] READ_NEXT_BITSET reads one byte past the last byte on last iteration
+* [ARROW-1606](https://issues.apache.org/jira/browse/ARROW-1606) - Python: Windows wheels don't include .lib files.
+* [ARROW-1610](https://issues.apache.org/jira/browse/ARROW-1610) - C++/Python: Only call python-prefix if the default PYTHON_LIBRARY is not present
+* [ARROW-1611](https://issues.apache.org/jira/browse/ARROW-1611) - Crash in BitmapReader when length is zero
+
+[1]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.7.1
+[2]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/
+[3]: https://issues.apache.org/jira/browse/ARROW-1601
\ No newline at end of file
diff --git a/site/_release/index.md b/site/_release/index.md
index ad78de8f3955e..4752e980b1916 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -26,6 +26,7 @@ limitations under the License.
 
 Navigate to the release page for downloads and the changelog.
 
+* [0.7.1 (1 October 2017)][9]
 * [0.7.0 (17 September 2017)][8]
 * [0.6.0 (14 August 2017)][7]
 * [0.5.0 (23 July 2017)][6]
@@ -43,3 +44,4 @@ Navigate to the release page for downloads and the changelog.
 [6]: {{ site.baseurl }}/release/0.5.0.html
 [7]: {{ site.baseurl }}/release/0.6.0.html
 [8]: {{ site.baseurl }}/release/0.7.0.html
+[9]: {{ site.baseurl }}/release/0.7.1.html
diff --git a/site/index.html b/site/index.html
index 01836d7bf5b51..54f67c1a245d0 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,10 +7,10 @@ <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
         <p>
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
-          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.7.0 Release - September 17, 2017)</a>
+          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.7.1 Release - October 1, 2017)</a>
         </p>
       </div>
-      <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.7.0 release</a></h4>
+      <h4><a href="{{ site.baseurl }}/blog/"><strong>See Latest News</strong></a></h4>
       <div class="row">
         <div class="col-lg-4">
           <h2>Fast</h2>
diff --git a/site/install.md b/site/install.md
index 74d298667d68f..1513fe819789c 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,17 +20,17 @@ limitations under the License.
 {% endcomment %}
 -->
 
-## Current Version: 0.7.0
+## Current Version: 0.7.1
 
-### Released: 17 September 2017
+### Released: 1 October 2017
 
 See the [release notes][10] for more about what's new.
 
 ### Source release
 
-* **Source Release**: [apache-arrow-0.7.0.tar.gz][6]
+* **Source Release**: [apache-arrow-0.7.1.tar.gz][6]
 * **Verification**: [sha512][3], [asc][7]
-* [Git tag 97f9029][2]
+* [Git tag 0e21f84][2]
 
 ### Java Packages
 
@@ -149,13 +149,13 @@ conda install arrow-cpp -c twosigma
 conda install pyarrow -c twosigma
 ```
 
-[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.7.0
-[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/apache-arrow-0.7.0.tar.gz.sha512
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.7.0%22
+[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.7.1
+[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz.sha512
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.7.1%22
 [5]: http://conda-forge.github.io
-[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/apache-arrow-0.7.0.tar.gz
-[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.0/apache-arrow-0.7.0.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz
+[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz.asc
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
-[10]: http://arrow.apache.org/release/0.7.0.html
+[10]: http://arrow.apache.org/release/0.7.1.html

From ff39cb5e1f56decbfd3369b562b6286d8c3fcb3e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 4 Oct 2017 11:00:12 -0400
Subject: [PATCH 1107/1644] ARROW-1640: Fix HTTPS failures in cmake / libcurl
 caused by ca-certificates clash

See discussion in https://github.com/apache/arrow/pull/1155

This seems to have done the trick, here is the Travis CI build running on my fork: https://travis-ci.org/wesm/arrow/builds/283203983

I will merge this as soon as it passes so that other PRs can rebase and stop failing

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1160 from wesm/fix-openssl-issues and squashes the following commits:

e218025d [Wes McKinney] Fix conda update statement
b2e3da19 [Wes McKinney] Do not install additional stuff in the root conda env
abb048ed [Wes McKinney] Use ca-certificates from defaults
---
 ci/travis_before_script_cpp.sh | 4 ++++
 ci/travis_install_conda.sh     | 5 -----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 65171a735575f..5dd8237f909b5 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -47,6 +47,10 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
         curl \
         thrift-cpp \
         ninja
+
+  # HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
+  # updated sometime on October 2 or October 3
+  conda update -y -p $CPP_TOOLCHAIN ca-certificates -c defaults
 fi
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index c2502a3744c11..3faa548817017 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -44,8 +44,3 @@ conda config --set remote_connect_timeout_secs 12
 conda config --add channels https://repo.continuum.io/pkgs/free
 conda config --add channels conda-forge
 conda info -a
-
-# faster builds, please
-conda install -y nomkl
-
-conda install --y conda-build jinja2 anaconda-client cmake curl

From 8ceee56622c9503b75d1c1023747f2e3c28be109 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 4 Oct 2017 11:04:57 -0400
Subject: [PATCH 1108/1644] ARROW-1543: [C++] Correct C++ tutorial to use
 std::unique_ptr instead of std::shared_ptr

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1157 from wesm/ARROW-1543 and squashes the following commits:

82e7593f [Wes McKinney] Add missing namespace
8afcc602 [Wes McKinney] Tweaks
7df6dd23 [Wes McKinney] Correct C++ tutorial to use std::unique_ptr instead of std::shared_ptr
---
 cpp/apidoc/tutorials/row_wise_conversion.md | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/cpp/apidoc/tutorials/row_wise_conversion.md b/cpp/apidoc/tutorials/row_wise_conversion.md
index e91c26e9da120..5d61eca85732c 100644
--- a/cpp/apidoc/tutorials/row_wise_conversion.md
+++ b/cpp/apidoc/tutorials/row_wise_conversion.md
@@ -57,12 +57,15 @@ is referenced by the offsets in the former array.
 // the underlying memory regions in-place. At the moment, arrow::jemalloc is only
 // supported on Unix systems, not Windows.
 
-arrow::Int64Builder id_builder(arrow::default_memory_pool());
-arrow::DoubleBuilder cost_builder(arrow::default_memory_pool());
-std::shared_ptr<DoubleBuilder> components_values_builder =
-    std::make_shared<DoubleBuilder>(arrow::default_memory_pool());
-arrow::ListBuilder components_builder(arrow::default_memory_pool(),
-    components_values_builder);
+using arrow::DoubleBuilder;
+using arrow::Int64Builder;
+using arrow::ListBuilder;
+
+arrow::MemoryPool* pool = arrow::default_memory_pool();
+Int64Builder id_builder(pool);
+DoubleBuilder cost_builder(pool);
+std::unique_ptr<DoubleBuilder> components_values_builder(new DoubleBuilder(pool));
+ListBuilder components_builder(pool, std::move(components_values_builder));
 ```
 
 Now we can loop over our existing data and insert it into the builders. The

From 67c631740b4c3cc895a712a5f6eed9dc7185e942 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 4 Oct 2017 16:06:49 -0400
Subject: [PATCH 1109/1644] ARROW-950: [Website] Add Google Analytics tag to
 site

I created this GA property. I will be happy to give access to any PMC who wishes to look at the analytics data. It would be better to be collecting the site visitor data than not.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1158 from wesm/ARROW-950 and squashes the following commits:

fd60e834 [Wes McKinney] Add Google Analytics tag to site
---
 site/_includes/analytics.html | 9 +++++++++
 site/_includes/top.html       | 3 +++
 2 files changed, 12 insertions(+)
 create mode 100644 site/_includes/analytics.html

diff --git a/site/_includes/analytics.html b/site/_includes/analytics.html
new file mode 100644
index 0000000000000..db0e0b762ff80
--- /dev/null
+++ b/site/_includes/analytics.html
@@ -0,0 +1,9 @@
+<!-- Global Site Tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script>
+<script>
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments)};
+  gtag('js', new Date());
+
+  gtag('config', 'UA-107500873-1');
+</script>
diff --git a/site/_includes/top.html b/site/_includes/top.html
index 988ae6c742b1f..af895c9d437aa 100644
--- a/site/_includes/top.html
+++ b/site/_includes/top.html
@@ -17,4 +17,7 @@
             integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
             crossorigin="anonymous"></script>
     <script src="{{ site.baseurl }}/assets/javascripts/bootstrap.min.js"></script>
+    {% if jekyll.environment == 'production' %}
+    {% include analytics.html %}
+    {% endif %}
   </head>

From 592c4e86eeb47f3562a0007d1a79a731b00af7a0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 4 Oct 2017 16:09:24 -0400
Subject: [PATCH 1110/1644] [Website] jekyll must be run with
 JEKYLL_ENV=production

Change-Id: I65c4cc4e2cb10d722bd64ebdb6e47a144e4be406
---
 site/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/README.md b/site/README.md
index 2c5f979197067..c1522651c506f 100644
--- a/site/README.md
+++ b/site/README.md
@@ -45,7 +45,7 @@ bundle exec jekyll serve
 After following the above instructions the base `site/` directory, run:
 
 ```shell
-bundle exec jekyll build
+JEKYLL_ENV=production bundle exec jekyll build
 rsync -r build/ asf-site/
 cd asf-site
 git status

From 87fc57777687d8af8c92d18d38f654eeb293a532 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 4 Oct 2017 16:14:46 -0400
Subject: [PATCH 1111/1644] ARROW-1584: [C++/Python] Support Null type in IPC
 round trips, fix serialize_pandas on empty DataFrame

cc @TomAugspurger @cpcloud -- this turned out to be caused by the index in the empty DataFrame having null type:

```
In [3]: arr = pa.array([])

In [4]: arr.type
Out[4]: DataType(null)

In [5]: arr = pa.array([None, None, None])

In [6]: arr
Out[6]:
<pyarrow.lib.NullArray object at 0x7f28d3d205e8>
[
  NA,
  NA,
  NA
]
```

This type had been unhandled in the IPC machinery, so this resolves ARROW-1637. We need to implement integration tests for this type to make sure such data can be accurately messages between Java and C++/Python

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1155 from wesm/ARROW-1584 and squashes the following commits:

b7cebf29 [Wes McKinney] Linting
a71c5440 [Wes McKinney] Add non-trivial field after null type and fix buffer handling bug
57dc9622 [Wes McKinney] Support Null type in IPC round trips, fix serialize_pandas on empty DataFrame
---
 cpp/src/arrow/ipc/ipc-read-write-test.cc   |  2 +-
 cpp/src/arrow/ipc/metadata-internal.cc     |  7 +++++++
 cpp/src/arrow/ipc/reader.cc                |  7 ++++++-
 cpp/src/arrow/ipc/test-common.h            | 21 ++++++++++++++++++++
 cpp/src/arrow/ipc/writer.cc                |  5 +++++
 cpp/src/arrow/python/builtin_convert.cc    |  9 +++------
 python/pyarrow/tests/test_ipc.py           | 23 +++++++++++++---------
 python/pyarrow/tests/test_serialization.py |  4 +++-
 8 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index d454d59b285fd..adf34a9eb5422 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -121,7 +121,7 @@ TEST_F(TestSchemaMetadata, NestedFields) {
                     &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList,                  \
                     &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion,               \
                     &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes,           \
-                    &MakeFWBinary, &MakeDecimal, &MakeBooleanBatch);
+                    &MakeFWBinary, &MakeNull, &MakeDecimal, &MakeBooleanBatch);
 
 static int g_file_number = 0;
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 962fe79d2a3ac..8593dca79e2e7 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -229,6 +229,9 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
   switch (type) {
     case flatbuf::Type_NONE:
       return Status::Invalid("Type metadata cannot be none");
+    case flatbuf::Type_Null:
+      *out = null();
+      return Status::OK();
     case flatbuf::Type_Int:
       return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
     case flatbuf::Type_FloatingPoint:
@@ -353,6 +356,10 @@ static Status TypeToFlatbuffer(FBB& fbb, const DataType& type,
   }
 
   switch (value_type->id()) {
+    case Type::NA:
+      *out_type = flatbuf::Type_Null;
+      *offset = flatbuf::CreateNull(fbb).Union();
+      break;
     case Type::BOOL:
       *out_type = flatbuf::Type_Bool;
       *offset = flatbuf::CreateBool(fbb).Union();
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index e6ba50e742f7b..5d0f57085cad1 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -187,7 +187,12 @@ class ArrayLoader {
     return Status::OK();
   }
 
-  Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
+  Status Visit(const NullType& type) {
+    out_->buffers.resize(1);
+    RETURN_NOT_OK(LoadCommon());
+    RETURN_NOT_OK(GetBuffer(context_->buffer_index++, &out_->buffers[0]));
+    return Status::OK();
+  }
 
   template <typename T>
   typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index ae36b5ff4ca55..b2137b7dbef6a 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -697,6 +697,27 @@ Status MakeDecimal(std::shared_ptr<RecordBatch>* out) {
   return Status::OK();
 }
 
+Status MakeNull(std::shared_ptr<RecordBatch>* out) {
+  auto f0 = field("f0", null());
+
+  // Also put a non-null field to make sure we handle the null array buffers properly
+  auto f1 = field("f1", int64());
+
+  auto schema = ::arrow::schema({f0, f1});
+
+  auto a1 = std::make_shared<NullArray>(10);
+
+  std::vector<int64_t> int_values = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+  std::vector<bool> is_valid = {true, true, true, false, false,
+                                true, true, true, true,  true};
+  std::shared_ptr<Array> a2;
+  ArrayFromVector<Int64Type, int64_t>(f1->type(), is_valid, int_values, &a2);
+
+  ArrayVector arrays = {a1, a2};
+  *out = std::make_shared<RecordBatch>(schema, a1->length(), arrays);
+  return Status::OK();
+}
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 9531fd77a24de..e415c2982f7b0 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -321,6 +321,11 @@ class RecordBatchSerializer : public ArrayVisitor {
     return Status::OK();
   }
 
+  Status Visit(const NullArray& array) override {
+    buffers_.push_back(nullptr);
+    return Status::OK();
+  }
+
 #define VISIT_FIXED_WIDTH(TYPE) \
   Status Visit(const TYPE& array) override { return VisitFixedWidth<TYPE>(array); }
 
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 69a19e7a36707..f7862d1516230 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -470,8 +470,7 @@ class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter
   inline Status AppendItem(const OwnedRef& item) {
     uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
 
-    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint8_t>::max() ||
-                            val < std::numeric_limits<uint8_t>::min())) {
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint8_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
@@ -486,8 +485,7 @@ class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Conver
   inline Status AppendItem(const OwnedRef& item) {
     uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
 
-    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint16_t>::max() ||
-                            val < std::numeric_limits<uint16_t>::min())) {
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint16_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
@@ -502,8 +500,7 @@ class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Conver
   inline Status AppendItem(const OwnedRef& item) {
     uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
 
-    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint32_t>::max() ||
-                            val < std::numeric_limits<uint32_t>::min())) {
+    if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint32_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 6ac44faba12b7..2c165575453e7 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -344,6 +344,12 @@ def test_get_record_batch_size():
     assert pa.get_record_batch_size(batch) > (N * itemsize)
 
 
+def _check_serialize_pandas_round_trip(df, nthreads=1):
+    buf = pa.serialize_pandas(df)
+    result = pa.deserialize_pandas(buf, nthreads=nthreads)
+    assert_frame_equal(result, df)
+
+
 def test_pandas_serialize_round_trip():
     index = pd.Index([1, 2, 3], name='my_index')
     columns = ['foo', 'bar']
@@ -351,9 +357,7 @@ def test_pandas_serialize_round_trip():
         {'foo': [1.5, 1.6, 1.7], 'bar': list('abc')},
         index=index, columns=columns
     )
-    buf = pa.serialize_pandas(df)
-    result = pa.deserialize_pandas(buf)
-    assert_frame_equal(result, df)
+    _check_serialize_pandas_round_trip(df)
 
 
 def test_pandas_serialize_round_trip_nthreads():
@@ -363,9 +367,7 @@ def test_pandas_serialize_round_trip_nthreads():
         {'foo': [1.5, 1.6, 1.7], 'bar': list('abc')},
         index=index, columns=columns
     )
-    buf = pa.serialize_pandas(df)
-    result = pa.deserialize_pandas(buf, nthreads=2)
-    assert_frame_equal(result, df)
+    _check_serialize_pandas_round_trip(df, nthreads=2)
 
 
 def test_pandas_serialize_round_trip_multi_index():
@@ -379,9 +381,12 @@ def test_pandas_serialize_round_trip_multi_index():
         index=index,
         columns=columns,
     )
-    buf = pa.serialize_pandas(df)
-    result = pa.deserialize_pandas(buf)
-    assert_frame_equal(result, df)
+    _check_serialize_pandas_round_trip(df)
+
+
+def test_serialize_pandas_empty_dataframe():
+    df = pd.DataFrame()
+    _check_serialize_pandas_round_trip(df)
 
 
 @pytest.mark.xfail(
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index eab81c2b5b42d..09fc4aaa2aa36 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -295,12 +295,14 @@ def test_custom_serialization(large_memory_map):
         for obj in CUSTOM_OBJECTS:
             serialization_roundtrip(obj, mmap)
 
+
 def test_default_dict_serialization(large_memory_map):
-    cloudpickle = pytest.importorskip("cloudpickle")
+    pytest.importorskip("cloudpickle")
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         obj = defaultdict(lambda: 0, [("hello", 1), ("world", 2)])
         serialization_roundtrip(obj, mmap)
 
+
 def test_numpy_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for t in ["int8", "uint8", "int16", "uint16",

From 31d33e079abf57d9d55f8141eb9b64d1a899cfff Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 4 Oct 2017 16:46:22 -0400
Subject: [PATCH 1112/1644] [Website] Update website with new committers

Change-Id: I0c2c78b164f510b923d90a8e862b80e65c21b059
---
 site/committers.html | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/site/committers.html b/site/committers.html
index 3909922c744fd..990260ef70d3e 100644
--- a/site/committers.html
+++ b/site/committers.html
@@ -101,6 +101,26 @@ <h2>Committers</h2>
 <td>Philipp Moritz</td>
 <td>pcmoritz</td>
 </tr>
+<tr>
+<td>Phillip Cloud</td>
+<td>cpcloud</td>
+</tr>
+<tr>
+<td>Bryan Cutler</td>
+<td>cutlerb</td>
+</tr>
+<tr>
+<td>Li Jin</td>
+<td>icexelloss</td>
+</tr>
+<tr>
+<td>Siddharth Teotia</td>
+<td>siddteotia</td>
+</tr>
+<tr>
+<td>Paul Taylor</td>
+<td>ptaylor</td>
+</tr>
 </tbody></table>
 
     </div> <!-- /container -->

From dc129d60fbffbf3a5b71b1f7987f7dab948b3d61 Mon Sep 17 00:00:00 2001
From: Yuliya Feldman <yuliya@dremio.com>
Date: Wed, 4 Oct 2017 23:18:58 -0400
Subject: [PATCH 1113/1644] =?UTF-8?q?ARROW-1627:=20New=20class=20to=20hand?=
 =?UTF-8?q?le=20collection=20of=20BufferLedger(s)=20within=20=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…AllocationManager

LowCostIdentityHashMap uses array only to store objects since key (BaseAllocator) is already included in the value (BufferLedger)
To make this class a bit more generic it uses values that implement ValueWithKeyIncluded interface that provides API : getKey()
BufferLedger provides implementation for it.
LowCostIdentityHashMap is not general purpose map it just provides some “map like” APIs to simplify switch over from IdentityHashMap.

Author: Yuliya Feldman <yuliya@dremio.com>

Closes #1150 from yufeldman/ARROW-1627-master and squashes the following commits:

dcd5e293 [Yuliya Feldman] ARROW-1627: New class to handle collection of BufferLedger(s) within AllocationManager LowCostIdentityHashMap uses array only to store objects since key (BaseAllocator) is already included in the value (BufferLedger) To make this class a bit more generic it uses values that implement ValueWithKeyIncluded interface that provides API : getKey() BufferLedger provides implementation for it. LowCostIdentityHashMap is not general purpose map it just provides some “map like” APIs to simplify switch over from IdentityHashMap.
---
 .../arrow/memory/AllocationManager.java       |  17 +-
 .../arrow/memory/LowCostIdentityHasMap.java   | 334 ++++++++++++++++++
 .../arrow/memory/ValueWithKeyIncluded.java    |  26 ++
 .../memory/TestLowCostIdentityHasMap.java     | 169 +++++++++
 4 files changed, 541 insertions(+), 5 deletions(-)
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java
 create mode 100644 java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index 6877c18f624ee..14687b54634ac 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -74,7 +74,9 @@ public class AllocationManager {
   private final long allocatorManagerId = MANAGER_ID_GENERATOR.incrementAndGet();
   private final int size;
   private final UnsafeDirectLittleEndian underlying;
-  private final IdentityHashMap<BufferAllocator, BufferLedger> map = new IdentityHashMap<>();
+  // ARROW-1627 Trying to minimize memory overhead caused by previously used IdentityHashMap
+  // see JIRA for details
+  private final LowCostIdentityHasMap<BaseAllocator, BufferLedger> map = new LowCostIdentityHasMap<>();
   private final ReadWriteLock lock = new ReentrantReadWriteLock();
   private final AutoCloseableLock readLock = new AutoCloseableLock(lock.readLock());
   private final AutoCloseableLock writeLock = new AutoCloseableLock(lock.writeLock());
@@ -144,7 +146,7 @@ private BufferLedger associate(final BaseAllocator allocator, final boolean reta
       if (retain) {
         ledger.inc();
       }
-      BufferLedger oldLedger = map.put(allocator, ledger);
+      BufferLedger oldLedger = map.put(ledger);
       Preconditions.checkArgument(oldLedger == null);
       allocator.associateLedger(ledger);
       return ledger;
@@ -174,7 +176,7 @@ private void release(final BufferLedger ledger) {
       } else {
         // we need to change the owning allocator. we've been removed so we'll get whatever is
         // top of list
-        BufferLedger newLedger = map.values().iterator().next();
+        BufferLedger newLedger = map.getNextValue();
 
         // we'll forcefully transfer the ownership and not worry about whether we exceeded the
         // limit
@@ -196,7 +198,7 @@ private void release(final BufferLedger ledger) {
    * As with AllocationManager, the only reason this is public is due to ArrowBuf being in io
    * .netty.buffer package.
    */
-  public class BufferLedger {
+  public class BufferLedger implements ValueWithKeyIncluded<BaseAllocator> {
 
     private final IdentityHashMap<ArrowBuf, Object> buffers =
         BaseAllocator.DEBUG ? new IdentityHashMap<ArrowBuf, Object>() : null;
@@ -226,6 +228,11 @@ private BaseAllocator getAllocator() {
       return allocator;
     }
 
+    @Override
+    public BaseAllocator getKey() {
+      return allocator;
+    }
+
     /**
      * Transfer any balance the current ledger has to the target ledger. In the case that the
      * current ledger holds no
@@ -457,4 +464,4 @@ boolean isOwningLedger() {
 
   }
 
-}
\ No newline at end of file
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java b/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java
new file mode 100644
index 0000000000000..1153fb5936471
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java
@@ -0,0 +1,334 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.arrow.memory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+/**
+ * Highly specialized IdentityHashMap that implements only partial
+ * Map APIs
+ * It incurs low initial cost (just two elements by default)
+ * It assumes Value includes the Key - Implements @ValueWithKeyIncluded iface
+ * that provides "getKey" method
+ * @param <V>
+ */
+public class LowCostIdentityHasMap<K, V extends ValueWithKeyIncluded<K>> {
+
+  /*
+   * The internal data structure to hold values.
+   */
+  private Object[] elementData;
+
+  /* Actual number of values. */
+  private int size;
+
+  /*
+   * maximum number of elements that can be put in this map before having to
+   * rehash.
+   */
+  private int threshold;
+
+  private static final int DEFAULT_MIN_SIZE = 1;
+
+  /* Default load factor of 0.75; */
+  private static final int loadFactor = 7500;
+
+  /**
+   * Creates an Map with default expected maximum size.
+   */
+  public LowCostIdentityHasMap() {
+    this(DEFAULT_MIN_SIZE);
+  }
+
+  /**
+   * Creates an Map with the specified maximum size parameter.
+   *
+   * @param maxSize
+   *            The estimated maximum number of entries that will be put in
+   *            this map.
+   */
+  public LowCostIdentityHasMap(int maxSize) {
+    if (maxSize >= 0) {
+      this.size = 0;
+      threshold = getThreshold(maxSize);
+      elementData = newElementArray(computeElementArraySize());
+    } else {
+      throw new IllegalArgumentException();
+    }
+  }
+
+  private int getThreshold(int maxSize) {
+    // assign the threshold to maxSize initially, this will change to a
+    // higher value if rehashing occurs.
+    return maxSize > 2 ? maxSize : 2;
+  }
+
+  private int computeElementArraySize() {
+    int arraySize = (int) (((long) threshold * 10000) / loadFactor);
+    // ensure arraySize is positive, the above cast from long to int type
+    // leads to overflow and negative arraySize if threshold is too big
+    return arraySize < 0 ? -arraySize : arraySize;
+  }
+
+  /**
+   * Create a new element array
+   *
+   * @param s
+   *            the number of elements
+   * @return Reference to the element array
+   */
+  private Object[] newElementArray(int s) {
+    return new Object[s];
+  }
+  
+  /**
+   * Removes all elements from this map, leaving it empty.
+   *
+   * @see #isEmpty()
+   * @see #size()
+   */
+  public void clear() {
+    size = 0;
+    for (int i = 0; i < elementData.length; i++) {
+      elementData[i] = null;
+    }
+  }
+
+  /**
+   * Returns whether this map contains the specified key.
+   *
+   * @param key
+   *            the key to search for.
+   * @return {@code true} if this map contains the specified key,
+   *         {@code false} otherwise.
+   */
+  public boolean containsKey(K key) {
+    Preconditions.checkNotNull(key);
+
+    int index = findIndex(key, elementData);
+    return (elementData[index] == null) ? false : ((V)elementData[index]).getKey() == key;
+  }
+
+  /**
+   * Returns whether this map contains the specified value.
+   *
+   * @param value
+   *            the value to search for.
+   * @return {@code true} if this map contains the specified value,
+   *         {@code false} otherwise.
+   */
+  public boolean containsValue(V value) {
+    Preconditions.checkNotNull(value);
+
+    for (int i = 0; i < elementData.length; i++) {
+      if (elementData[i] == value) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Returns the value of the mapping with the specified key.
+   *
+   * @param key
+   *            the key.
+   * @return the value of the mapping with the specified key.
+   */
+  public V get(K key) {
+    Preconditions.checkNotNull(key);
+
+    int index = findIndex(key, elementData);
+
+    return (elementData[index] == null) ? null :
+      (((V)elementData[index]).getKey() == key) ? (V)elementData[index] : null;
+  }
+
+  /**
+   * Returns the index where the key is found at, or the index of the next
+   * empty spot if the key is not found in this table.
+   */
+  @VisibleForTesting
+  int findIndex(Object key, Object[] array) {
+    int length = array.length;
+    int index = getModuloHash(key, length);
+    int last = (index + length - 1) % length;
+    while (index != last) {
+      if ((array[index] == null) || ((V)array[index]).getKey() == key) {
+        /*
+         * Found the key, or the next empty spot (which means key is not
+         * in the table)
+         */
+        break;
+      }
+      index = (index+1) % length;
+    }
+    return index;
+  }
+
+  @VisibleForTesting
+  static int getModuloHash(Object key, int length) {
+    return ((System.identityHashCode(key) & 0x7FFFFFFF) % length);
+  }
+
+  /**
+   * Maps the specified key to the specified value.
+   *
+   * @param value
+   *            the value.
+   * @return the value of any previous mapping with the specified key or
+   *         {@code null} if there was no such mapping.
+   */
+  public V put(V value) {
+    Preconditions.checkNotNull(value);
+    K _key = value.getKey();
+    Preconditions.checkNotNull(_key);
+    V _value = value;
+
+    int index = findIndex(_key, elementData);
+
+    // if the key doesn't exist in the table
+    if (elementData[index] == null || ((V)elementData[index]).getKey() != _key) {
+      if (++size > threshold) {
+        rehash();
+        index = findIndex(_key, elementData);
+      }
+
+      // insert the key and assign the value to null initially
+      elementData[index] = null;
+    }
+
+    // insert value to where it needs to go, return the old value
+    Object result = elementData[index];
+    elementData[index] = _value;
+
+    return (V) result;
+  }
+
+  @VisibleForTesting
+  void rehash() {
+    int newlength = elementData.length * 15 / 10;
+    if (newlength == 0) {
+      newlength = 1;
+    }
+    Object[] newData = newElementArray(newlength);
+    for (int i = 0; i < elementData.length; i++) {
+      Object key = (elementData[i] == null) ? null : ((V)elementData[i]).getKey();
+      if (key != null) {
+        // if not empty
+        int index = findIndex(key, newData);
+        newData[index] = elementData[i];
+      }
+    }
+    elementData = newData;
+    computeMaxSize();
+  }
+
+  private void computeMaxSize() {
+    threshold = (int) ((long) (elementData.length) * loadFactor / 10000);
+  }
+
+  /**
+   * Removes the mapping with the specified key from this map.
+   *
+   * @param key
+   *            the key of the mapping to remove.
+   * @return the value of the removed mapping, or {@code null} if no mapping
+   *         for the specified key was found.
+   */
+  public V remove(K key) {
+    Preconditions.checkNotNull(key);
+
+    boolean hashedOk;
+    int index, next, hash;
+    Object result, object;
+    index = next = findIndex(key, elementData);
+
+    if (elementData[index] == null || ((V)elementData[index]).getKey() != key) {
+      return null;
+    }
+
+    // store the value for this key
+    result = elementData[index];
+    // clear value to allow movement of the rest of the elements
+    elementData[index] = null;
+    size--;
+
+    // shift the following elements up if needed
+    // until we reach an empty spot
+    int length = elementData.length;
+    while (true) {
+      next = (next + 1) % length;
+      object = elementData[next];
+      if (object == null) {
+        break;
+      }
+
+      hash = getModuloHash(((V)object).getKey(), length);
+      hashedOk = hash > index;
+      if (next < index) {
+        hashedOk = hashedOk || (hash <= next);
+      } else {
+        hashedOk = hashedOk && (hash <= next);
+      }
+      if (!hashedOk) {
+        elementData[index] = object;
+        index = next;
+        elementData[index] = null;
+      }
+    }
+
+    return (V) result;
+  }
+
+
+
+  /**
+   * Returns whether this Map has no elements.
+   *
+   * @return {@code true} if this Map has no elements,
+   *         {@code false} otherwise.
+   * @see #size()
+   */
+  public boolean isEmpty() {
+    return size == 0;
+  }
+
+  /**
+   * Returns the number of mappings in this Map.
+   *
+   * @return the number of mappings in this Map.
+   */
+  public int size() {
+    return size;
+  }
+
+  /**
+   * Special API to return next value - substitute of regular Map.values.iterator().next()
+   * @return next available value or null if none available
+   */
+  public V getNextValue() {
+    for (int i = 0; i < elementData.length; i++) {
+      if (elementData[i] != null) {
+        return (V)elementData[i];
+      }
+    }
+    return null;
+  }
+}
\ No newline at end of file
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java b/java/memory/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java
new file mode 100644
index 0000000000000..7bd9cecf97088
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+/**
+ * Helper Iface to generify a value to be included in the map where
+ * key is part of the value
+ */
+public interface ValueWithKeyIncluded<K> {
+    K getKey();
+}
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java b/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java
new file mode 100644
index 0000000000000..c1196147b4925
--- /dev/null
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.junit.Test;
+
+/**
+ * To test simplified implementation of IdentityHashMap
+ */
+public class TestLowCostIdentityHasMap {
+
+  @Test
+  public void testIdentityHashMap() throws Exception {
+    LowCostIdentityHasMap<String, StringWithKey> hashMap = new LowCostIdentityHasMap<>();
+
+    StringWithKey obj1 = new StringWithKey("s1key", "s1value");
+    StringWithKey obj2 = new StringWithKey("s2key", "s2value");
+    StringWithKey obj3 = new StringWithKey("s3key", "s3value");
+    StringWithKey obj4 = new StringWithKey("s1key", "s4value");
+    StringWithKey obj5 = new StringWithKey("s5key", "s5value");
+
+    assertNull(hashMap.put(obj1));
+    assertNull(hashMap.put(obj2));
+    assertNull(hashMap.put(obj3));
+    assertEquals(obj1, hashMap.put(obj4));
+    assertNull(hashMap.put(obj5));
+
+    assertEquals(4, hashMap.size());
+
+    assertEquals(obj4,hashMap.get("s1key"));
+
+    assertNull(hashMap.remove("abc"));
+
+    assertEquals(obj3,hashMap.remove("s3key"));
+
+    assertEquals(3, hashMap.size());
+
+    assertTrue(!hashMap.isEmpty());
+
+    StringWithKey nextValue = hashMap.getNextValue();
+
+    assertNotNull(nextValue);
+
+    assertTrue((hashMap.get("s1key") == nextValue || hashMap.get("s2key") == nextValue ||
+      hashMap.get("s5key") == nextValue));
+
+    assertTrue(hashMap.containsValue(obj4));
+    assertTrue(hashMap.containsValue(obj2));
+    assertTrue(hashMap.containsValue(obj5));
+
+    assertEquals(obj4,hashMap.remove("s1key"));
+
+    nextValue = hashMap.getNextValue();
+
+    assertNotNull(nextValue);
+
+    assertTrue(hashMap.get("s2key") == nextValue || hashMap.get("s5key") == nextValue);
+
+    assertEquals(2, hashMap.size());
+
+    assertEquals(obj2,hashMap.remove("s2key"));
+    assertEquals(obj5,hashMap.remove("s5key"));
+
+    assertEquals(0, hashMap.size());
+
+    assertTrue(hashMap.isEmpty());
+  }
+
+  @Test
+  public void testLargeMap() throws Exception {
+    LowCostIdentityHasMap<String, StringWithKey> hashMap = new LowCostIdentityHasMap<>();
+
+    String [] keys = new String[200];
+    for (int i = 0; i < 200; i++) {
+      keys[i] = "s"+i+"key";
+    }
+
+    for (int i = 0; i < 100; i++) {
+      if (i % 5 == 0 && i != 0) {
+        StringWithKey obj = new StringWithKey(keys[i-5], "s" + i + "value");
+        StringWithKey retObj = hashMap.put(obj);
+        assertNotNull(retObj);
+        StringWithKey obj1 = new StringWithKey(keys[i], "s" + 2*i + "value");
+        StringWithKey retObj1 = hashMap.put(obj1);
+        assertNull(retObj1);
+      } else {
+        StringWithKey obj = new StringWithKey(keys[i], "s" + i + "value");
+        StringWithKey retObj = hashMap.put(obj);
+        assertNull(retObj);
+      }
+    }
+    assertEquals(100, hashMap.size());
+    for (int i = 0; i < 100; i++) {
+      StringWithKey returnObj = hashMap.get(keys[i]);
+      assertNotNull(returnObj);
+      if (i == 95) {
+        assertEquals("s190value", returnObj.getValue());
+        continue;
+      }
+      if (i % 5 == 0) {
+        assertEquals("s" + (i+5) + "value", returnObj.getValue());
+      } else {
+        assertEquals("s" + i + "value", returnObj.getValue());
+      }
+    }
+
+    for (int i = 0; i < 100; i++) {
+      if (i % 4 == 0) {
+        StringWithKey returnObj = hashMap.remove(keys[i]);
+        assertNotNull(returnObj);
+        assertTrue(!hashMap.containsKey(keys[i]));
+      }
+      StringWithKey obj = new StringWithKey(keys[100+i], "s" + (100+i) + "value");
+      StringWithKey retObj = hashMap.put(obj);
+      assertNull(retObj);
+      assertTrue(hashMap.containsKey(keys[100+i]));
+    }
+    assertEquals(175, hashMap.size());
+    for (int i = 0; i < 100; i++) {
+      StringWithKey retObj = hashMap.getNextValue();
+      assertNotNull(retObj);
+      hashMap.remove(retObj.getKey());
+    }
+    assertTrue(!hashMap.isEmpty());
+    assertEquals(75, hashMap.size());
+    hashMap.clear();
+    assertTrue(hashMap.isEmpty());
+  }
+
+  private class StringWithKey implements ValueWithKeyIncluded<String> {
+
+    private String myValue;
+    private String myKey;
+
+    StringWithKey(String myKey, String myValue) {
+      this.myKey = myKey;
+      this.myValue = myValue;
+    }
+
+    @Override
+    public String getKey() {
+      return myKey;
+    }
+
+    String getValue() {
+      return myValue;
+    }
+  }
+}

From 81319d9c2784a626af318911c183701d320888f4 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Wed, 4 Oct 2017 23:14:11 -0700
Subject: [PATCH 1114/1644] ARROW-1647: [Plasma] Make sure to read length
 header as int64_t instead of size_t.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1172 from robertnishihara/plasmaiofixes and squashes the following commits:

9bb0c00 [Robert Nishihara] Make sure to read length header as int64_t instead of size_t.
---
 cpp/src/plasma/io.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
index afe7053329b1e..d604078114e38 100644
--- a/cpp/src/plasma/io.cc
+++ b/cpp/src/plasma/io.cc
@@ -92,11 +92,14 @@ Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
   RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)),
                      *type = DISCONNECT_CLIENT);
   ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version;
-  size_t length;
   RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(type), sizeof(*type)),
                      *type = DISCONNECT_CLIENT);
-  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)),
+  int64_t length_temp;
+  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length_temp),
+                               sizeof(length_temp)),
                      *type = DISCONNECT_CLIENT);
+  // The length must be read as an int64_t, but it should be used as a size_t.
+  size_t length = length_temp;
   if (length > buffer->size()) {
     buffer->resize(length);
   }

From 8b5b22b15d2df9c7e7fc52842c1f7b8794c7d40e Mon Sep 17 00:00:00 2001
From: Amir Malekpour <a.malekpour@gmail.com>
Date: Thu, 5 Oct 2017 10:57:03 +0200
Subject: [PATCH 1115/1644] ARROW-1525: [C++] New compare functions that return
 boolean instead of Status

Comparison should always succeed so, it makes better sense to return boolean
values as the result of comparison instead of Status.
This changeset introduces a set of new compare functions and deprecates the
current ones.

Author: Amir Malekpour <a.malekpour@gmail.com>

Author: Amir Malekpour <a.malekpour@gmail.com>

Closes #1156 from amirma/arrow-1525 and squashes the following commits:

b94ee28 [Amir Malekpour] ARROW-1525: [C++] New compare functions that return boolean instead of Status
---
 cpp/README.md                         |   8 +-
 cpp/src/arrow/array.cc                |  22 +----
 cpp/src/arrow/compare.cc              | 114 +++++++++++++++++---------
 cpp/src/arrow/compare.h               |  24 ++++++
 cpp/src/arrow/compute/compute-test.cc |   3 +-
 cpp/src/arrow/tensor.cc               |   7 +-
 cpp/src/arrow/type.cc                 |   7 +-
 7 files changed, 111 insertions(+), 74 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 6e29e6f78f4d1..a94699b6e1a5a 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -248,15 +248,15 @@ to avoid failures due to compiler warnings.
 Note that the clang-tidy target may take a while to run.  You might consider
 running clang-tidy separately on the files you have added/changed before
 invoking the make target to reduce iteration time.  Also, it might generate warnings
-that aren't valid.  To avoid these you can use add a line comment `// NOLINT`. If
+that aren't valid.  To avoid these you can add a line comment `// NOLINT`. If
 NOLINT doesn't suppress the warnings, you add the file in question to
 the .clang-tidy-ignore file.  This will allow `make check-clang-tidy` to pass in
-travis-CI (but still surface the potential warnings in `make clang-tidy`).   Ideally,
-both of these options would be used rarely.  Current known uses-cases whent hey are required:
+travis-CI (but still surface the potential warnings in `make clang-tidy`). Ideally,
+both of these options would be used rarely. Current known uses-cases when they are required:
 
 *  Parameterized tests in google test.
 
 [1]: https://brew.sh/
 [2]: https://github.com/apache/arrow/blob/master/cpp/apidoc/Windows.md
 [3]: https://google.github.io/styleguide/cppguide.html
-[4]: https://github.com/include-what-you-use/include-what-you-use
\ No newline at end of file
+[4]: https://github.com/include-what-you-use/include-what-you-use
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 12922ae7b0ac6..a3a1e62ff2ba2 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -53,12 +53,7 @@ int64_t Array::null_count() const {
 }
 
 bool Array::Equals(const Array& arr) const {
-  bool are_equal = false;
-  Status error = ArrayEquals(*this, arr, &are_equal);
-  if (!error.ok()) {
-    DCHECK(false) << "Arrays not comparable: " << error.ToString();
-  }
-  return are_equal;
+  return ArrayEquals(*this, arr);
 }
 
 bool Array::Equals(const std::shared_ptr<Array>& arr) const {
@@ -69,12 +64,7 @@ bool Array::Equals(const std::shared_ptr<Array>& arr) const {
 }
 
 bool Array::ApproxEquals(const Array& arr) const {
-  bool are_equal = false;
-  Status error = ArrayApproxEquals(*this, arr, &are_equal);
-  if (!error.ok()) {
-    DCHECK(false) << "Arrays not comparable: " << error.ToString();
-  }
-  return are_equal;
+  return ArrayApproxEquals(*this, arr);
 }
 
 bool Array::ApproxEquals(const std::shared_ptr<Array>& arr) const {
@@ -94,13 +84,7 @@ bool Array::RangeEquals(int64_t start_idx, int64_t end_idx, int64_t other_start_
 
 bool Array::RangeEquals(const Array& other, int64_t start_idx, int64_t end_idx,
                         int64_t other_start_idx) const {
-  bool are_equal = false;
-  Status error =
-      ArrayRangeEquals(*this, other, start_idx, end_idx, other_start_idx, &are_equal);
-  if (!error.ok()) {
-    DCHECK(false) << "Arrays not comparable: " << error.ToString();
-  }
-  return are_equal;
+  return ArrayRangeEquals(*this, other, start_idx, end_idx, other_start_idx);
 }
 
 static inline std::shared_ptr<ArrayData> SliceData(const ArrayData& data, int64_t offset,
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 515b8f62f5640..984537fc94c01 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -542,22 +542,26 @@ static bool BaseDataEquals(const Array& left, const Array& right) {
 }
 
 template <typename VISITOR>
-inline Status ArrayEqualsImpl(const Array& left, const Array& right, bool* are_equal) {
+inline bool ArrayEqualsImpl(const Array& left, const Array& right) {
+  bool are_equal;
   // The arrays are the same object
   if (&left == &right) {
-    *are_equal = true;
+    are_equal = true;
   } else if (!BaseDataEquals(left, right)) {
-    *are_equal = false;
+    are_equal = false;
   } else if (left.length() == 0) {
-    *are_equal = true;
+    are_equal = true;
   } else if (left.null_count() == left.length()) {
-    *are_equal = true;
+    are_equal = true;
   } else {
     VISITOR visitor(right);
-    RETURN_NOT_OK(VisitArrayInline(left, &visitor));
-    *are_equal = visitor.result();
+    auto error = VisitArrayInline(left, &visitor);
+    if (!error.ok()) {
+      DCHECK(false) << "Arrays are not comparable: " << error.ToString();
+    }
+    are_equal = visitor.result();
   }
-  return Status::OK();
+  return are_equal;
 }
 
 class TypeEqualsVisitor {
@@ -668,29 +672,33 @@ class TypeEqualsVisitor {
 
 }  // namespace internal
 
-Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
-  return internal::ArrayEqualsImpl<internal::ArrayEqualsVisitor>(left, right, are_equal);
+bool ArrayEquals(const Array& left, const Array& right) {
+  return internal::ArrayEqualsImpl<internal::ArrayEqualsVisitor>(left, right);
 }
 
-Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal) {
-  return internal::ArrayEqualsImpl<internal::ApproxEqualsVisitor>(left, right, are_equal);
+bool ArrayApproxEquals(const Array& left, const Array& right) {
+  return internal::ArrayEqualsImpl<internal::ApproxEqualsVisitor>(left, right);
 }
 
-Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_start_idx,
-                        int64_t left_end_idx, int64_t right_start_idx, bool* are_equal) {
+bool ArrayRangeEquals(const Array& left, const Array& right, int64_t left_start_idx,
+                      int64_t left_end_idx, int64_t right_start_idx) {
+  bool are_equal;
   if (&left == &right) {
-    *are_equal = true;
+    are_equal = true;
   } else if (left.type_id() != right.type_id()) {
-    *are_equal = false;
+    are_equal = false;
   } else if (left.length() == 0) {
-    *are_equal = true;
+    are_equal = true;
   } else {
     internal::RangeEqualsVisitor visitor(right, left_start_idx, left_end_idx,
                                          right_start_idx);
-    RETURN_NOT_OK(VisitArrayInline(left, &visitor));
-    *are_equal = visitor.result();
+    auto error = VisitArrayInline(left, &visitor);
+    if (!error.ok()) {
+      DCHECK(false) << "Arrays are not comparable: " << error.ToString();
+    }
+    are_equal = visitor.result();
   }
-  return Status::OK();
+  return are_equal;
 }
 
 bool StridedTensorContentEquals(int dim_index, int64_t left_offset, int64_t right_offset,
@@ -716,24 +724,25 @@ bool StridedTensorContentEquals(int dim_index, int64_t left_offset, int64_t righ
   return true;
 }
 
-Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
+bool TensorEquals(const Tensor& left, const Tensor& right) {
+  bool are_equal;
   // The arrays are the same object
   if (&left == &right) {
-    *are_equal = true;
+    are_equal = true;
   } else if (left.type_id() != right.type_id()) {
-    *are_equal = false;
+    are_equal = false;
   } else if (left.size() == 0) {
-    *are_equal = true;
+    are_equal = true;
   } else {
     if (!left.is_contiguous() || !right.is_contiguous()) {
       const auto& shape = left.shape();
       if (shape != right.shape()) {
-        *are_equal = false;
-        return Status::OK();
+        are_equal = false;
+      } else {
+        const auto& type = static_cast<const FixedWidthType&>(*left.type());
+        are_equal = StridedTensorContentEquals(0, 0, 0,
+                                               type.bit_width() / 8, left, right);
       }
-      const auto& type = static_cast<const FixedWidthType&>(*left.type());
-      *are_equal = StridedTensorContentEquals(0, 0, 0, type.bit_width() / 8, left, right);
-      return Status::OK();
     } else {
       const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
       const int byte_width = size_meta.bit_width() / CHAR_BIT;
@@ -742,24 +751,55 @@ Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
       const uint8_t* left_data = left.data()->data();
       const uint8_t* right_data = right.data()->data();
 
-      *are_equal = memcmp(left_data, right_data,
-                          static_cast<size_t>(byte_width * left.size())) == 0;
+      are_equal = memcmp(left_data, right_data,
+                         static_cast<size_t>(byte_width * left.size())) == 0;
     }
   }
-  return Status::OK();
+  return are_equal;
 }
 
-Status TypeEquals(const DataType& left, const DataType& right, bool* are_equal) {
+bool TypeEquals(const DataType& left, const DataType& right) {
+  bool are_equal;
   // The arrays are the same object
   if (&left == &right) {
-    *are_equal = true;
+    are_equal = true;
   } else if (left.id() != right.id()) {
-    *are_equal = false;
+    are_equal = false;
   } else {
     internal::TypeEqualsVisitor visitor(right);
-    RETURN_NOT_OK(VisitTypeInline(left, &visitor));
-    *are_equal = visitor.result();
+    auto error = VisitTypeInline(left, &visitor);
+    if (!error.ok()) {
+      DCHECK(false) << "Types are not comparable: " << error.ToString();
+    }
+    are_equal = visitor.result();
   }
+  return are_equal;
+}
+
+Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
+  *are_equal = ArrayEquals(left, right);
+  return Status::OK();
+}
+
+Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
+  *are_equal = TensorEquals(left, right);
+  return Status::OK();
+}
+
+Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal) {
+  *are_equal = ArrayApproxEquals(left, right);
+  return Status::OK();
+}
+
+Status ArrayRangeEquals(const Array& left, const Array& right,
+                        int64_t start_idx, int64_t end_idx,
+                        int64_t other_start_idx, bool* are_equal) {
+  *are_equal = ArrayRangeEquals(left, right, start_idx, end_idx, other_start_idx);
+  return Status::OK();
+}
+
+Status TypeEquals(const DataType& left, const DataType& right, bool* are_equal) {
+  *are_equal = TypeEquals(left, right);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index a36b55320b5a2..27176ed864cdd 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -31,25 +31,49 @@ class DataType;
 class Status;
 class Tensor;
 
+#ifndef ARROW_NO_DEPRECATED_API
 /// Returns true if the arrays are exactly equal
+/// \deprecated Since 0.8.0
 Status ARROW_EXPORT ArrayEquals(const Array& left, const Array& right, bool* are_equal);
 
+/// \deprecated Since 0.8.0
 Status ARROW_EXPORT TensorEquals(const Tensor& left, const Tensor& right,
                                  bool* are_equal);
 
 /// Returns true if the arrays are approximately equal. For non-floating point
 /// types, this is equivalent to ArrayEquals(left, right)
+/// \deprecated Since 0.8.0
 Status ARROW_EXPORT ArrayApproxEquals(const Array& left, const Array& right,
                                       bool* are_equal);
 
 /// Returns true if indicated equal-length segment of arrays is exactly equal
+/// \deprecated Since 0.8.0
 Status ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right,
                                      int64_t start_idx, int64_t end_idx,
                                      int64_t other_start_idx, bool* are_equal);
 
 /// Returns true if the type metadata are exactly equal
+/// \deprecated Since 0.8.0
 Status ARROW_EXPORT TypeEquals(const DataType& left, const DataType& right,
                                bool* are_equal);
+#endif
+
+/// Returns true if the arrays are exactly equal
+bool ARROW_EXPORT ArrayEquals(const Array& left, const Array& right);
+
+bool ARROW_EXPORT TensorEquals(const Tensor& left, const Tensor& right);
+
+/// Returns true if the arrays are approximately equal. For non-floating point
+/// types, this is equivalent to ArrayEquals(left, right)
+bool ARROW_EXPORT ArrayApproxEquals(const Array& left, const Array& right);
+
+/// Returns true if indicated equal-length segment of arrays is exactly equal
+bool ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right,
+                                   int64_t start_idx, int64_t end_idx,
+                                   int64_t other_start_idx);
+
+/// Returns true if the type metadata are exactly equal
+bool ARROW_EXPORT TypeEquals(const DataType& left, const DataType& right);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 9df4573deba4a..9db0a0e13dd59 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -47,8 +47,7 @@ namespace arrow {
 namespace compute {
 
 void AssertArraysEqual(const Array& left, const Array& right) {
-  bool are_equal = false;
-  ASSERT_OK(ArrayEquals(left, right, &are_equal));
+  bool are_equal = ArrayEquals(left, right);
 
   if (!are_equal) {
     std::stringstream ss;
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index efadd7cea224d..ed7da8b2c0c7d 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -118,12 +118,7 @@ bool Tensor::is_column_major() const {
 Type::type Tensor::type_id() const { return type_->id(); }
 
 bool Tensor::Equals(const Tensor& other) const {
-  bool are_equal = false;
-  Status error = TensorEquals(*this, other, &are_equal);
-  if (!error.ok()) {
-    DCHECK(false) << "Tensors not comparable: " << error.ToString();
-  }
-  return are_equal;
+  return TensorEquals(*this, other);
 }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index b9e314440f54f..14ddd2a5910b0 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -83,12 +83,7 @@ std::string Field::ToString() const {
 DataType::~DataType() {}
 
 bool DataType::Equals(const DataType& other) const {
-  bool are_equal = false;
-  Status error = TypeEquals(*this, other, &are_equal);
-  if (!error.ok()) {
-    DCHECK(false) << "Types not comparable: " << error.ToString();
-  }
-  return are_equal;
+  return TypeEquals(*this, other);
 }
 
 bool DataType::Equals(const std::shared_ptr<DataType>& other) const {

From 0f819fa7355d3b391b0083168f0ffd6f329e8d1b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 11:08:50 +0200
Subject: [PATCH 1116/1644] ARROW-1603: [C++] Add BinaryArray::GetString helper
 method

In some applications it is useful to get values as `std::string`

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1159 from wesm/ARROW-1603 and squashes the following commits:

a94099d [Wes McKinney] Add BinaryArray::GetString method
---
 cpp/src/arrow/array-test.cc | 12 ++++++++++++
 cpp/src/arrow/array.h       | 11 +++++++++++
 2 files changed, 23 insertions(+)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 4ecf0f92b2a33..ef10257b59c62 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1045,6 +1045,18 @@ TEST_F(TestBinaryArray, TestGetValue) {
   }
 }
 
+TEST_F(TestBinaryArray, TestGetString) {
+  for (size_t i = 0; i < expected_.size(); ++i) {
+    if (valid_bytes_[i] == 0) {
+      ASSERT_TRUE(strings_->IsNull(i));
+    } else {
+      std::string val = strings_->GetString(i);
+      ASSERT_EQ(0, std::memcmp(expected_[i].data(), val.c_str(),
+                               val.size()));
+    }
+  }
+}
+
 TEST_F(TestBinaryArray, TestEqualsEmptyStrings) {
   BinaryBuilder builder;
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 4ad60eb77f48e..f668b6de929c0 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -444,6 +444,17 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
     return raw_data_ + pos;
   }
 
+  /// \brief Get binary value as a std::string
+  ///
+  /// \param i the value index
+  /// \return the value copied into a std::string
+  std::string GetString(int64_t i) const {
+    int32_t length = 0;
+    const uint8_t* bytes = GetValue(i, &length);
+    return std::string(reinterpret_cast<const char*>(bytes),
+                       static_cast<size_t>(length));
+  }
+
   /// Note that this buffer does not account for any slice offset
   std::shared_ptr<Buffer> value_offsets() const { return data_->buffers[1]; }
 

From 7511cfdd5e625e310b7943bfe1ab5f6e8a67b3b2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 11:11:01 +0200
Subject: [PATCH 1117/1644] ARROW-226: [C++] If opening an HDFS file fails and
 it does not exist, say so to help with debugging

This was a rough edge that had affected some users

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1166 from wesm/ARROW-226 and squashes the following commits:

7ab2891 [Wes McKinney] If opening an HDFS file fails and it does not exist, say so to help with debugging
---
 cpp/src/arrow/io/hdfs.cc          | 8 ++++++--
 python/pyarrow/tests/test_hdfs.py | 9 +++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index d9d420622dad9..77d1f524aae85 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -455,9 +455,13 @@ class HadoopFileSystem::HadoopFileSystemImpl {
     hdfsFile handle = driver_->OpenFile(fs_, path.c_str(), O_RDONLY, buffer_size, 0, 0);
 
     if (handle == nullptr) {
-      // TODO(wesm): determine cause of failure
       std::stringstream ss;
-      ss << "Unable to open file " << path;
+      if (!Exists(path)) {
+        ss << "HDFS file does not exist: " << path;
+      } else {
+        // TODO(wesm): determine other causes of failure
+        ss << "HDFS path exists, but opening file failed: " << path;
+      }
       return Status::IOError(ss.str());
     }
 
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index cbc277373c707..51b6ba25bd657 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -213,6 +213,15 @@ def test_file_context_manager(self):
             result = f.read(10)
             assert result == data
 
+    def test_open_not_exist_error_message(self):
+        # ARROW-226
+        path = pjoin(self.tmp_path, 'does-not-exist-123')
+
+        try:
+            self.hdfs.open(path)
+        except Exception as e:
+            assert 'file does not exist' in e.args[0].lower()
+
     def test_read_whole_file(self):
         path = pjoin(self.tmp_path, 'read-whole-file')
 

From f0873a99aecdfda4b3d8ec04a41491af3442b827 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 11:13:26 +0200
Subject: [PATCH 1118/1644] ARROW-942: Support running integration tests with
 both Python 2.7 and 3.6

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1168 from wesm/ARROW-942 and squashes the following commits:

e4ef049 [Wes McKinney] Support running integration tests in both Python 2.7 and 3.6
---
 integration/integration_test.py | 43 +++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 4bb0a4bead196..59a1de5a4639d 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -17,6 +17,7 @@
 
 from collections import OrderedDict
 import argparse
+import binascii
 import glob
 import itertools
 import json
@@ -30,6 +31,7 @@
 
 import numpy as np
 
+
 ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
 
 # Control for flakiness
@@ -63,11 +65,24 @@ def rands(nchars):
     return ''.join(np.random.choice(RANDS_CHARS, nchars))
 
 
-def str_from_bytes(x):
-    if six.PY2:
-        return x
-    else:
-        return x.decode('utf-8')
+if six.PY2:
+    def frombytes(o):
+        return o
+
+    def tobytes(o):
+        if isinstance(o, unicode):
+            return o.encode('utf8')
+        else:
+            return o
+else:
+    def tobytes(o):
+        if isinstance(o, str):
+            return o.encode('utf8')
+        else:
+            return o
+
+    def frombytes(o):
+        return o.decode('utf8')
 
 
 # from the merge_arrow_pr.py script
@@ -82,11 +97,11 @@ def run_cmd(cmd):
         print('Command failed: %s' % ' '.join(cmd))
         print('With output:')
         print('--------------')
-        print(str_from_bytes(e.output))
+        print(frombytes(e.output))
         print('--------------')
         raise e
 
-    return str_from_bytes(output)
+    return frombytes(output)
 
 # ----------------------------------------------------------------------
 # Data generation
@@ -353,6 +368,7 @@ def generate_column(self, size, name=None):
 
 
 class DecimalColumn(PrimitiveColumn):
+
     def __init__(self, name, count, is_valid, values, bit_width):
         PrimitiveColumn.__init__(self, name, count, is_valid, values)
         self.bit_width = bit_width
@@ -416,7 +432,7 @@ def generate_column(self, size, name=None):
                         .tostring())
                 values.append(draw)
             else:
-                values.append("")
+                values.append(b"")
 
         if name is None:
             name = self.name
@@ -439,9 +455,9 @@ def generate_column(self, size, name=None):
 
         for i in range(size):
             if is_valid[i]:
-                values.append(rands(K))
+                values.append(tobytes(rands(K)))
             else:
-                values.append("")
+                values.append(b"")
 
         if name is None:
             name = self.name
@@ -462,7 +478,7 @@ def get_json(self):
 class BinaryColumn(PrimitiveColumn):
 
     def _encode_value(self, x):
-        return ''.join('{:02x}'.format(c).upper() for c in x)
+        return frombytes(binascii.hexlify(x))
 
     def _get_buffers(self):
         offset = 0
@@ -473,7 +489,7 @@ def _get_buffers(self):
             if self.is_valid[i]:
                 offset += len(v)
             else:
-                v = ""
+                v = b""
 
             offsets.append(offset)
             data.append(self._encode_value(v))
@@ -488,7 +504,8 @@ def _get_buffers(self):
 class StringColumn(BinaryColumn):
 
     def _encode_value(self, x):
-        return x
+        return frombytes(x)
+
 
 class ListType(DataType):
 

From 9805ada853ae63580c59d765941f1e36a86e554c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 11:16:41 +0200
Subject: [PATCH 1119/1644] ARROW-1633: [Python] Support NumPy string and
 unicode types in pyarrow.array, Array.from_pandas

I suppose this could have been worse. If anyone has any better ideas about alternatives to my onion router of UTF32 bytes -> PyUnicode -> PyBytes (UTF8) -> StringBuilder, I'm open to them. Since we support gcc 4.8 we don't have the option of using `std::codecvt`.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1167 from wesm/ARROW-1633 and squashes the following commits:

f0840b4 [Wes McKinney] Also test big-endian unicode
ca59ada [Wes McKinney] Implement NumPy unicode to Arrow utf8 conversion, deal with masks, truncated values
9304216 [Wes McKinney] Convert NumPy string (ascii) arrays to arrow binary arrays, handle mask and strided versions
---
 cpp/src/arrow/python/numpy_convert.cc  |   2 +
 cpp/src/arrow/python/numpy_to_arrow.cc | 130 ++++++++++++++++++++++++-
 python/pyarrow/tests/test_array.py     |  61 ++++++++++++
 3 files changed, 188 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 4c64cc4c32b4f..bef80542ab0ba 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -167,6 +167,8 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
 #endif
     TO_ARROW_TYPE_CASE(FLOAT32, float32);
     TO_ARROW_TYPE_CASE(FLOAT64, float64);
+    TO_ARROW_TYPE_CASE(STRING, binary);
+    TO_ARROW_TYPE_CASE(UNICODE, utf8);
     case NPY_DATETIME: {
       auto date_dtype =
           reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index c0ce61cca55c3..e935b45b5e8d2 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -266,12 +266,11 @@ class NumPyConverter {
       mask_ = reinterpret_cast<PyArrayObject*>(mo);
     }
     length_ = static_cast<int64_t>(PyArray_SIZE(arr_));
+    itemsize_ = static_cast<int>(PyArray_DESCR(arr_)->elsize);
+    stride_ = static_cast<int64_t>(PyArray_STRIDES(arr_)[0]);
   }
 
-  bool is_strided() const {
-    npy_intp* astrides = PyArray_STRIDES(arr_);
-    return astrides[0] != PyArray_DESCR(arr_)->elsize;
-  }
+  bool is_strided() const { return itemsize_ != stride_; }
 
   Status Convert();
 
@@ -293,7 +292,11 @@ class NumPyConverter {
 
   Status Visit(const NullType& type) { return TypeNotImplemented(type.ToString()); }
 
-  Status Visit(const BinaryType& type) { return TypeNotImplemented(type.ToString()); }
+  // NumPy ascii string arrays
+  Status Visit(const BinaryType& type);
+
+  // NumPy unicode arrays
+  Status Visit(const StringType& type);
 
   Status Visit(const FixedSizeBinaryType& type) {
     return TypeNotImplemented(type.ToString());
@@ -430,6 +433,8 @@ class NumPyConverter {
   PyArrayObject* arr_;
   PyArrayObject* mask_;
   int64_t length_;
+  int64_t stride_;
+  int itemsize_;
 
   bool use_pandas_null_sentinels_;
 
@@ -1213,10 +1218,125 @@ Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
   return PushBuilderResult(list_builder);
 }
 
+Status NumPyConverter::Visit(const BinaryType& type) {
+  BinaryBuilder builder(pool_);
+
+  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
+
+  int item_length = 0;
+  if (mask_ != nullptr) {
+    Ndarray1DIndexer<uint8_t> mask_values(mask_);
+    for (int64_t i = 0; i < length_; ++i) {
+      if (mask_values[i]) {
+        RETURN_NOT_OK(builder.AppendNull());
+      } else {
+        // This is annoying. NumPy allows strings to have nul-terminators, so
+        // we must check for them here
+        for (item_length = 0; item_length < itemsize_; ++item_length) {
+          if (data[item_length] == 0) {
+            break;
+          }
+        }
+        RETURN_NOT_OK(builder.Append(data, item_length));
+      }
+      data += stride_;
+    }
+  } else {
+    for (int64_t i = 0; i < length_; ++i) {
+      for (item_length = 0; item_length < itemsize_; ++item_length) {
+        // Look for nul-terminator
+        if (data[item_length] == 0) {
+          break;
+        }
+      }
+      RETURN_NOT_OK(builder.Append(data, item_length));
+      data += stride_;
+    }
+  }
+
+  std::shared_ptr<Array> result;
+  RETURN_NOT_OK(builder.Finish(&result));
+  return PushArray(result->data());
+}
+
+namespace {
+
+// NumPy unicode is UCS4/UTF32 always
+constexpr int kNumPyUnicodeSize = 4;
+
+Status AppendUTF32(const char* data, int itemsize, int byteorder,
+                   StringBuilder* builder) {
+  // The binary \x00\x00\x00\x00 indicates a nul terminator in NumPy unicode,
+  // so we need to detect that here to truncate if necessary. Yep.
+  int actual_length = 0;
+  for (; actual_length < itemsize / kNumPyUnicodeSize; ++actual_length) {
+    const char* code_point = data + actual_length * kNumPyUnicodeSize;
+    if ((*code_point == '\0') && (*(code_point + 1) == '\0') &&
+        (*(code_point + 2) == '\0') && (*(code_point + 3) == '\0')) {
+      break;
+    }
+  }
+
+  ScopedRef unicode_obj(PyUnicode_DecodeUTF32(data, actual_length * kNumPyUnicodeSize,
+                                              nullptr, &byteorder));
+  RETURN_IF_PYERROR();
+  ScopedRef utf8_obj(PyUnicode_AsUTF8String(unicode_obj.get()));
+  if (utf8_obj.get() == NULL) {
+    PyErr_Clear();
+    return Status::Invalid("failed converting UTF32 to UTF8");
+  }
+
+  const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(utf8_obj.get()));
+  if (builder->value_data_length() + length > kBinaryMemoryLimit) {
+    return Status::Invalid("Encoded string length exceeds maximum size (2GB)");
+  }
+  return builder->Append(PyBytes_AS_STRING(utf8_obj.get()), length);
+}
+
+}  // namespace
+
+Status NumPyConverter::Visit(const StringType& type) {
+  StringBuilder builder(pool_);
+
+  auto data = reinterpret_cast<const char*>(PyArray_DATA(arr_));
+
+  char numpy_byteorder = PyArray_DESCR(arr_)->byteorder;
+
+  // For Python C API, -1 is little-endian, 1 is big-endian
+  int byteorder = numpy_byteorder == '>' ? 1 : -1;
+
+  PyAcquireGIL gil_lock;
+
+  if (mask_ != nullptr) {
+    Ndarray1DIndexer<uint8_t> mask_values(mask_);
+    for (int64_t i = 0; i < length_; ++i) {
+      if (mask_values[i]) {
+        RETURN_NOT_OK(builder.AppendNull());
+      } else {
+        RETURN_NOT_OK(AppendUTF32(data, itemsize_, byteorder, &builder));
+      }
+      data += stride_;
+    }
+  } else {
+    for (int64_t i = 0; i < length_; ++i) {
+      RETURN_NOT_OK(AppendUTF32(data, itemsize_, byteorder, &builder));
+      data += stride_;
+    }
+  }
+
+  std::shared_ptr<Array> result;
+  RETURN_NOT_OK(builder.Finish(&result));
+  return PushArray(result->data());
+}
+
 Status NdarrayToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
                       bool use_pandas_null_sentinels,
                       const std::shared_ptr<DataType>& type,
                       std::shared_ptr<ChunkedArray>* out) {
+  if (!PyArray_Check(ao)) {
+    return Status::Invalid("Input object was not a NumPy array");
+  }
+
   NumPyConverter converter(pool, ao, mo, type, use_pandas_null_sentinels);
   RETURN_NOT_OK(converter.Convert());
   const auto& output_arrays = converter.result();
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 3bf392686f0bc..414a268ce1bb8 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -337,3 +337,64 @@ def test_array_conversions_no_sentinel_values():
                     type='float32')
     assert arr3.type == 'float32'
     assert arr3.null_count == 0
+
+
+def test_array_from_numpy_ascii():
+    arr = np.array(['abcde', 'abc', ''], dtype='|S5')
+
+    arrow_arr = pa.array(arr)
+    assert arrow_arr.type == 'binary'
+    expected = pa.array(['abcde', 'abc', ''], type='binary')
+    assert arrow_arr.equals(expected)
+
+    mask = np.array([False, True, False])
+    arrow_arr = pa.array(arr, mask=mask)
+    expected = pa.array(['abcde', None, ''], type='binary')
+    assert arrow_arr.equals(expected)
+
+    # Strided variant
+    arr = np.array(['abcde', 'abc', ''] * 5, dtype='|S5')[::2]
+    mask = np.array([False, True, False] * 5)[::2]
+    arrow_arr = pa.array(arr, mask=mask)
+
+    expected = pa.array(['abcde', '', None, 'abcde', '', None, 'abcde', ''],
+                        type='binary')
+    assert arrow_arr.equals(expected)
+
+    # 0 itemsize
+    arr = np.array(['', '', ''], dtype='|S0')
+    arrow_arr = pa.array(arr)
+    expected = pa.array(['', '', ''], type='binary')
+    assert arrow_arr.equals(expected)
+
+
+def test_array_from_numpy_unicode():
+    dtypes = ['<U5', '>U5']
+
+    for dtype in dtypes:
+        arr = np.array(['abcde', 'abc', ''], dtype=dtype)
+
+        arrow_arr = pa.array(arr)
+        assert arrow_arr.type == 'utf8'
+        expected = pa.array(['abcde', 'abc', ''], type='utf8')
+        assert arrow_arr.equals(expected)
+
+        mask = np.array([False, True, False])
+        arrow_arr = pa.array(arr, mask=mask)
+        expected = pa.array(['abcde', None, ''], type='utf8')
+        assert arrow_arr.equals(expected)
+
+        # Strided variant
+        arr = np.array(['abcde', 'abc', ''] * 5, dtype=dtype)[::2]
+        mask = np.array([False, True, False] * 5)[::2]
+        arrow_arr = pa.array(arr, mask=mask)
+
+        expected = pa.array(['abcde', '', None, 'abcde', '', None,
+                             'abcde', ''], type='utf8')
+        assert arrow_arr.equals(expected)
+
+    # 0 itemsize
+    arr = np.array(['', '', ''], dtype='<U0')
+    arrow_arr = pa.array(arr)
+    expected = pa.array(['', '', ''], type='utf8')
+    assert arrow_arr.equals(expected)

From ab6aa9a2dd885feb9750b88e6f5b48d4d5467dac Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 13:58:33 +0200
Subject: [PATCH 1120/1644] ARROW-1486: [C++] Make Column, RecordBatch, and
 Table non-copyable

Since these data structures are immutable, there is little benefit to permitting copying or assignment in user code, and may lead to unintended technical debt in API design.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1169 from wesm/ARROW-1486 and squashes the following commits:

1602e53 [Wes McKinney] Make Column, RecordBatch, and Table non-copyable
---
 cpp/src/arrow/table.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 324112bfb3e1d..709a6666f9942 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -25,6 +25,7 @@
 
 #include "arrow/array.h"
 #include "arrow/type.h"
+#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -104,6 +105,9 @@ class ARROW_EXPORT Column {
  protected:
   std::shared_ptr<Field> field_;
   std::shared_ptr<ChunkedArray> data_;
+
+ private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(Column);
 };
 
 /// \class RecordBatch
@@ -193,6 +197,8 @@ class ARROW_EXPORT RecordBatch {
   Status Validate() const;
 
  private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(RecordBatch);
+
   RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows);
 
   std::shared_ptr<Schema> schema_;
@@ -266,6 +272,8 @@ class ARROW_EXPORT Table {
   bool IsChunked() const;
 
  private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(Table);
+
   std::shared_ptr<Schema> schema_;
   std::vector<std::shared_ptr<Column>> columns_;
 

From 909a6f68a0d918737797f7052e6145ce9ba0ea0e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 08:40:34 -0400
Subject: [PATCH 1121/1644] ARROW-1616: [Python] Add unit test for
 RecordBatchWriter.write dispatching to write_table or write_batch

I'd actually already implemented this, but it was untested.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1170 from wesm/ARROW-1616 and squashes the following commits:

32599fe3 [Wes McKinney] Add unit test for _RecordBatchWriter.write
---
 python/pyarrow/tests/test_ipc.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 2c165575453e7..7a771654e4d4d 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -147,6 +147,26 @@ def test_categorical_roundtrip(self):
                  .read_all())
         assert_frame_equal(table.to_pandas(), df)
 
+    def test_stream_write_dispatch(self):
+        # ARROW-1616
+        df = pd.DataFrame({
+            'one': np.random.randn(5),
+            'two': pd.Categorical(['foo', np.nan, 'bar', 'foo', 'foo'],
+                                  categories=['foo', 'bar'],
+                                  ordered=True)
+        })
+        table = pa.Table.from_pandas(df, preserve_index=False)
+        batch = pa.RecordBatch.from_pandas(df, preserve_index=False)
+        writer = self._get_writer(self.sink, table.schema)
+        writer.write(table)
+        writer.write(batch)
+        writer.close()
+
+        table = (pa.open_stream(pa.BufferReader(self._get_source()))
+                 .read_all())
+        assert_frame_equal(table.to_pandas(),
+                           pd.concat([df, df], ignore_index=True))
+
     def test_simple_roundtrip(self):
         _, batches = self.write_batches()
         file_contents = pa.BufferReader(self._get_source())

From bd73166bde3d015118266a56d7db50eb20562857 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 08:55:51 -0400
Subject: [PATCH 1122/1644] ARROW-1526: [Python] Add unit test for fix in
 PARQUET-1100

This generates a table with large list elements to exercise the case where there were incomplete number of repeated values decoded

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1171 from wesm/ARROW-1526 and squashes the following commits:

bf260b9c [Wes McKinney] Add unit test for fix in PARQUET-1100
---
 python/pyarrow/tests/test_parquet.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index b0593fe885dec..d51b85d8ee4ed 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -598,6 +598,23 @@ def _assert_unsupported(array):
     _assert_unsupported(a7)
 
 
+@parquet
+def test_large_list_records():
+    # This was fixed in PARQUET-1100
+
+    list_lengths = np.random.randint(0, 500, size=50)
+    list_lengths[::10] = 0
+
+    list_values = [list(map(int, np.random.randint(0, 100, size=x)))
+                   if i % 8 else None
+                   for i, x in enumerate(list_lengths)]
+
+    a1 = pa.array(list_values)
+
+    table = pa.Table.from_arrays([a1], ['int_lists'])
+    _check_roundtrip(table)
+
+
 @parquet
 def test_sanitized_spark_field_names():
     a0 = pa.array([0, 1, 2, 3, 4])

From 51905e5e43d137b720df4501c5fe80b04585ee4c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 15:15:28 -0400
Subject: [PATCH 1123/1644] ARROW-1498: Add CONTRIBUTING.md to .github special
 directory

I don't think it's necessary to have an ISSUE_TEMPLATE.md right now, but we can add one in the future if it is deemed useful

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1176 from wesm/ARROW-1498 and squashes the following commits:

462e766e [Wes McKinney] Add CONTRIBUTING.md to .github special directory
---
 .github/CONTRIBUTING.md | 71 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 .github/CONTRIBUTING.md

diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
new file mode 100644
index 0000000000000..740f85320c45e
--- /dev/null
+++ b/.github/CONTRIBUTING.md
@@ -0,0 +1,71 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+## Contributing to Apache Arrow
+
+There are many ways to contribute to Apache Arrow:
+
+* Contributing code (we call them "patches")
+* Writing documentation (another form of code, in a way)
+* Participating in discussions on JIRA or the mailing list
+* Helping users of the libraries
+
+## Reporting bugs and asking questions
+
+We support GitHub issues as a lightweight way to ask questions and engage with
+the Arrow developer community. We use [JIRA][3] for maintaining a queue of
+development work and as the public record for work on the project. So, feel
+free to open GitHub issues, but bugs and feature requests will eventually need
+to end up in JIRA, either before or after completing a pull request.
+
+## How to contribute patches
+
+We prefer to receive contributions in the form of GitHub pull requests. Please
+send pull requests against the [github.com/apache/arrow][4] repository.
+
+If you are looking for some ideas on what to contribute, check out the [JIRA
+issues][3] for the Apache Arrow project. Comment on the issue and/or contact
+[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+with your questions and ideas.
+
+If you’d like to report a bug but don’t have time to fix it, you can still post
+it on JIRA, or email the mailing list
+[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+
+To contribute a patch:
+
+1. Break your work into small, single-purpose patches if possible. It’s much
+harder to merge in a large change with a lot of disjoint features.
+2. Create a JIRA for your patch on the [Arrow Project
+JIRA](https://issues.apache.org/jira/browse/ARROW).
+3. Submit the patch as a GitHub pull request against the master branch. For a
+tutorial, see the GitHub guides on forking a repo and sending a pull
+request. Prefix your pull request name with the JIRA name (ex:
+https://github.com/apache/arrow/pull/240).
+4. Make sure that your code passes the unit tests. You can find instructions
+how to run the unit tests for each Arrow component in its respective README
+file.
+5. Add new unit tests for your code.
+
+Thank you in advance for your contributions!
+
+[1]: mailto:dev-subscribe@arrow.apache.org
+[2]: https://github.com/apache/arrow/tree/master/format
+[3]: https://issues.apache.org/jira/browse/ARROW
+[4]: https://github.com/apache/arrow
\ No newline at end of file

From bea3495dd9ed18f6257f2f89aaea994e08707aac Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 15:54:49 -0400
Subject: [PATCH 1124/1644] ARROW-1539: [C++] Remove APIs deprecated as of
 0.7.0 or prior releases

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1174 from wesm/ARROW-1539 and squashes the following commits:

188ad705 [Wes McKinney] Remove APIs deprecated as of 0.7.0 or prior releases
---
 cpp/src/arrow/array-test.cc    |  3 +--
 cpp/src/arrow/array.cc         | 30 ++--------------------------
 cpp/src/arrow/array.h          | 21 +-------------------
 cpp/src/arrow/buffer.cc        | 12 ------------
 cpp/src/arrow/buffer.h         |  5 -----
 cpp/src/arrow/builder.cc       | 34 --------------------------------
 cpp/src/arrow/builder.h        | 36 ----------------------------------
 cpp/src/arrow/compare.cc       |  9 ++++-----
 cpp/src/arrow/io/hdfs.h        |  5 -----
 cpp/src/arrow/ipc/reader.cc    | 23 ----------------------
 cpp/src/arrow/ipc/reader.h     | 19 ------------------
 cpp/src/arrow/ipc/writer.cc    | 22 ---------------------
 cpp/src/arrow/ipc/writer.h     | 22 ---------------------
 cpp/src/arrow/table.cc         |  6 ------
 cpp/src/arrow/table.h          |  5 -----
 cpp/src/arrow/tensor.cc        |  4 +---
 cpp/src/arrow/type.cc          |  4 +---
 cpp/src/arrow/util/bit-util.cc |  8 --------
 cpp/src/arrow/util/bit-util.h  | 26 ------------------------
 cpp/src/plasma/io.cc           |  6 +++---
 20 files changed, 13 insertions(+), 287 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index ef10257b59c62..0b02c675d3537 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1051,8 +1051,7 @@ TEST_F(TestBinaryArray, TestGetString) {
       ASSERT_TRUE(strings_->IsNull(i));
     } else {
       std::string val = strings_->GetString(i);
-      ASSERT_EQ(0, std::memcmp(expected_[i].data(), val.c_str(),
-                               val.size()));
+      ASSERT_EQ(0, std::memcmp(expected_[i].data(), val.c_str(), val.size()));
     }
   }
 }
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index a3a1e62ff2ba2..cd1721f07fe9e 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -52,9 +52,7 @@ int64_t Array::null_count() const {
   return data_->null_count;
 }
 
-bool Array::Equals(const Array& arr) const {
-  return ArrayEquals(*this, arr);
-}
+bool Array::Equals(const Array& arr) const { return ArrayEquals(*this, arr); }
 
 bool Array::Equals(const std::shared_ptr<Array>& arr) const {
   if (!arr) {
@@ -63,9 +61,7 @@ bool Array::Equals(const std::shared_ptr<Array>& arr) const {
   return Equals(*arr);
 }
 
-bool Array::ApproxEquals(const Array& arr) const {
-  return ArrayApproxEquals(*this, arr);
-}
+bool Array::ApproxEquals(const Array& arr) const { return ArrayApproxEquals(*this, arr); }
 
 bool Array::ApproxEquals(const std::shared_ptr<Array>& arr) const {
   if (!arr) {
@@ -592,28 +588,6 @@ Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>*
   return Status::OK();
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-// \deprecated Since 0.7.0
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
-                          const std::shared_ptr<Buffer>& data,
-                          const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
-                          int64_t offset, std::shared_ptr<Array>* out) {
-  BufferVector buffers = {null_bitmap, data};
-  auto internal_data =
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
-  return MakeArray(internal_data, out);
-}
-
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-                          const std::vector<std::shared_ptr<Buffer>>& buffers,
-                          int64_t length, int64_t null_count, int64_t offset,
-                          std::shared_ptr<Array>* out) {
-  auto internal_data =
-      std::make_shared<ArrayData>(type, length, buffers, null_count, offset);
-  return MakeArray(internal_data, out);
-}
-#endif
-
 // ----------------------------------------------------------------------
 // Instantiate templates
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index f668b6de929c0..882c63cddf15d 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -451,8 +451,7 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
   std::string GetString(int64_t i) const {
     int32_t length = 0;
     const uint8_t* bytes = GetValue(i, &length);
-    return std::string(reinterpret_cast<const char*>(bytes),
-                       static_cast<size_t>(length));
+    return std::string(reinterpret_cast<const char*>(bytes), static_cast<size_t>(length));
   }
 
   /// Note that this buffer does not account for any slice offset
@@ -684,24 +683,6 @@ ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
 ARROW_EXPORT
 Status ValidateArray(const Array& array);
 
-#ifndef ARROW_NO_DEPRECATED_API
-// \note Deprecated since 0.7.0
-
-/// Create new arrays for logical types that are backed by primitive arrays.
-ARROW_EXPORT
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
-                          const std::shared_ptr<Buffer>& data,
-                          const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
-                          int64_t offset, std::shared_ptr<Array>* out);
-
-ARROW_EXPORT
-Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
-                          const std::vector<std::shared_ptr<Buffer>>& buffers,
-                          int64_t length, int64_t null_count, int64_t offset,
-                          std::shared_ptr<Array>* out);
-
-#endif
-
 }  // namespace arrow
 
 #endif  // ARROW_ARRAY_H
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index e308ed260497f..1b8e4375445bb 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -132,18 +132,6 @@ Status AllocateBuffer(MemoryPool* pool, const int64_t size,
   return Status::OK();
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-Status AllocateBuffer(MemoryPool* pool, const int64_t size,
-                      std::shared_ptr<MutableBuffer>* out) {
-  std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(AllocateBuffer(pool, size, &buffer));
-  *out = std::dynamic_pointer_cast<MutableBuffer>(buffer);
-  return Status::OK();
-}
-
-#endif
-
 Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
                                std::shared_ptr<ResizableBuffer>* out) {
   auto buffer = std::make_shared<PoolBuffer>(pool);
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 5f61ade956bef..b745812b10366 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -339,11 +339,6 @@ Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
 
 #ifndef ARROW_NO_DEPRECATED_API
 
-/// \deprecated Since 0.7.0
-ARROW_EXPORT
-Status AllocateBuffer(MemoryPool* pool, const int64_t size,
-                      std::shared_ptr<MutableBuffer>* out);
-
 /// \brief Create Buffer referencing std::string memory
 /// \deprecated Since 0.8.0
 ///
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 0479dc555924b..55e7873e8308c 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -691,11 +691,6 @@ BooleanBuilder::BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool
   DCHECK_EQ(Type::BOOL, type->id());
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-BooleanBuilder::BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-    : BooleanBuilder(type, pool) {}
-#endif
-
 Status BooleanBuilder::Init(int64_t capacity) {
   RETURN_NOT_OK(ArrayBuilder::Init(capacity));
   data_ = std::make_shared<PoolBuffer>(pool_);
@@ -838,13 +833,6 @@ DictionaryBuilder<FixedSizeBinaryType>::DictionaryBuilder(
   }
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-template <typename T>
-DictionaryBuilder<T>::DictionaryBuilder(MemoryPool* pool,
-                                        const std::shared_ptr<DataType>& type)
-    : DictionaryBuilder(type, pool) {}
-#endif
-
 template <typename T>
 Status DictionaryBuilder<T>::Init(int64_t elements) {
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
@@ -1111,11 +1099,6 @@ template class DictionaryBuilder<StringType>;
 DecimalBuilder::DecimalBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
     : FixedSizeBinaryBuilder(type, pool) {}
 
-#ifndef ARROW_NO_DEPRECATED_API
-DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-    : DecimalBuilder(type, pool) {}
-#endif
-
 Status DecimalBuilder::Append(const Decimal128& value) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
   return FixedSizeBinaryBuilder::Append(value.ToBytes());
@@ -1212,11 +1195,6 @@ ArrayBuilder* ListBuilder::value_builder() const {
 BinaryBuilder::BinaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
     : ArrayBuilder(type, pool), offsets_builder_(pool), value_data_builder_(pool) {}
 
-#ifndef ARROW_NO_DEPRECATED_API
-BinaryBuilder::BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-    : BinaryBuilder(type, pool) {}
-#endif
-
 BinaryBuilder::BinaryBuilder(MemoryPool* pool) : BinaryBuilder(binary(), pool) {}
 
 Status BinaryBuilder::Init(int64_t elements) {
@@ -1316,12 +1294,6 @@ FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(const std::shared_ptr<DataType>&
       byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()),
       byte_builder_(pool) {}
 
-#ifndef ARROW_NO_DEPRECATED_API
-FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(MemoryPool* pool,
-                                               const std::shared_ptr<DataType>& type)
-    : FixedSizeBinaryBuilder(type, pool) {}
-#endif
-
 Status FixedSizeBinaryBuilder::Append(const uint8_t* value) {
   RETURN_NOT_OK(Reserve(1));
   UnsafeAppendToBitmap(true);
@@ -1377,12 +1349,6 @@ StructBuilder::StructBuilder(const std::shared_ptr<DataType>& type, MemoryPool*
   field_builders_ = std::move(field_builders);
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-StructBuilder::StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-                             std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders)
-    : StructBuilder(type, pool, std::move(field_builders)) {}
-#endif
-
 Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
   std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
   for (size_t i = 0; i < field_builders_.size(); ++i) {
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index da7386aafefc4..1ddcf27143b53 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -181,11 +181,6 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   explicit PrimitiveBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
       : ArrayBuilder(type, pool), data_(nullptr), raw_data_(nullptr) {}
 
-#ifndef ARROW_NO_DEPRECATED_API
-  explicit PrimitiveBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type)
-      : PrimitiveBuilder(type, pool) {}
-#endif
-
   using ArrayBuilder::Advance;
 
   /// Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
@@ -514,11 +509,6 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 
   explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.6.0
-  explicit BooleanBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
-#endif
-
   using ArrayBuilder::Advance;
 
   /// Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
@@ -667,11 +657,6 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
  public:
   explicit BinaryBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.6.0
-  BinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
-#endif
-
   BinaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
 
   Status Append(const uint8_t* value, int32_t length);
@@ -728,11 +713,6 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
 
 class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
  public:
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.6.0
-  FixedSizeBinaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
-#endif
-
   FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
                          MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
@@ -772,11 +752,6 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
   explicit DecimalBuilder(const std::shared_ptr<DataType>& type,
                           MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.6.0
-  explicit DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
-#endif
-
   using FixedSizeBinaryBuilder::Append;
 
   Status Append(const Decimal128& val);
@@ -794,12 +769,6 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
 /// called to maintain data-structure consistency.
 class ARROW_EXPORT StructBuilder : public ArrayBuilder {
  public:
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.6.0
-  StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-                std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders);
-#endif
-
   StructBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool,
                 std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders);
 
@@ -888,11 +857,6 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
 
   ~DictionaryBuilder() {}
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.6.0
-  explicit DictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
-#endif
-
   DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
 
   template <typename T1 = T>
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 984537fc94c01..94ea339caf774 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -740,8 +740,8 @@ bool TensorEquals(const Tensor& left, const Tensor& right) {
         are_equal = false;
       } else {
         const auto& type = static_cast<const FixedWidthType&>(*left.type());
-        are_equal = StridedTensorContentEquals(0, 0, 0,
-                                               type.bit_width() / 8, left, right);
+        are_equal =
+            StridedTensorContentEquals(0, 0, 0, type.bit_width() / 8, left, right);
       }
     } else {
       const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
@@ -791,9 +791,8 @@ Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal)
   return Status::OK();
 }
 
-Status ArrayRangeEquals(const Array& left, const Array& right,
-                        int64_t start_idx, int64_t end_idx,
-                        int64_t other_start_idx, bool* are_equal) {
+Status ArrayRangeEquals(const Array& left, const Array& right, int64_t start_idx,
+                        int64_t end_idx, int64_t other_start_idx, bool* are_equal) {
   *are_equal = ArrayRangeEquals(left, right, start_idx, end_idx, other_start_idx);
   return Status::OK();
 }
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 431ff5e7fe463..aaaafc819f217 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -172,11 +172,6 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
   ARROW_DISALLOW_COPY_AND_ASSIGN(HadoopFileSystem);
 };
 
-#ifndef ARROW_NO_DEPRECATED_API
-/// \deprecated Since 0.6.0
-using HdfsClient = HadoopFileSystem;
-#endif
-
 class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
  public:
   ~HdfsReadableFile();
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 5d0f57085cad1..5fe908240178a 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -487,21 +487,6 @@ Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& str
   return Open(std::move(message_reader), out);
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-Status RecordBatchStreamReader::Open(std::unique_ptr<MessageReader> message_reader,
-                                     std::shared_ptr<RecordBatchStreamReader>* reader) {
-  // Private ctor
-  *reader = std::shared_ptr<RecordBatchStreamReader>(new RecordBatchStreamReader());
-  return (*reader)->impl_->Open(std::move(message_reader));
-}
-
-Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
-                                     std::shared_ptr<RecordBatchStreamReader>* out) {
-  std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
-  return Open(std::move(message_reader), out);
-}
-#endif
-
 std::shared_ptr<Schema> RecordBatchStreamReader::schema() const {
   return impl_->schema();
 }
@@ -646,7 +631,6 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
  private:
   io::RandomAccessFile* file_;
 
-  // Deprecated as of 0.7.0
   std::shared_ptr<io::RandomAccessFile> owned_file_;
 
   // The location where the Arrow file layout ends. May be the end of the file
@@ -735,13 +719,6 @@ Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, io::InputStream* f
                          out);
 }
 
-// Deprecated
-Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
-                       io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
-  RETURN_NOT_OK(file->Seek(offset));
-  return ReadRecordBatch(schema, file, out);
-}
-
 Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
                   std::shared_ptr<Tensor>* out) {
   // Respect alignment of Tensor messages (see WriteTensor)
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index e90dc1e5234ee..1db282f398a92 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -60,16 +60,6 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
   static Status Open(std::unique_ptr<MessageReader> message_reader,
                      std::shared_ptr<RecordBatchReader>* out);
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.7.0
-  static Status Open(std::unique_ptr<MessageReader> message_reader,
-                     std::shared_ptr<RecordBatchStreamReader>* out);
-
-  /// \deprecated Since 0.7.0
-  static Status Open(const std::shared_ptr<io::InputStream>& stream,
-                     std::shared_ptr<RecordBatchStreamReader>* out);
-#endif
-
   /// \brief Record batch stream reader from InputStream
   ///
   /// \param[in] stream an input stream instance. Must stay alive throughout
@@ -213,15 +203,6 @@ ARROW_EXPORT
 Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
                   std::shared_ptr<Tensor>* out);
 
-#ifndef ARROW_NO_DEPRECATED_API
-/// \deprecated Since 0.7.0
-///
-/// Deprecated in favor of more general InputStream-based API
-ARROW_EXPORT
-Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
-                       io::RandomAccessFile* stream, std::shared_ptr<RecordBatch>* out);
-#endif
-
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index e415c2982f7b0..c32161434c447 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -858,17 +858,6 @@ Status RecordBatchStreamWriter::Open(io::OutputStream* sink,
   return Status::OK();
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-Status RecordBatchStreamWriter::Open(io::OutputStream* sink,
-                                     const std::shared_ptr<Schema>& schema,
-                                     std::shared_ptr<RecordBatchStreamWriter>* out) {
-  // ctor is private
-  *out = std::shared_ptr<RecordBatchStreamWriter>(new RecordBatchStreamWriter());
-  (*out)->impl_.reset(new RecordBatchStreamWriterImpl(sink, schema));
-  return Status::OK();
-}
-#endif
-
 Status RecordBatchStreamWriter::Close() { return impl_->Close(); }
 
 // ----------------------------------------------------------------------
@@ -932,17 +921,6 @@ Status RecordBatchFileWriter::Open(io::OutputStream* sink,
   return Status::OK();
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-Status RecordBatchFileWriter::Open(io::OutputStream* sink,
-                                   const std::shared_ptr<Schema>& schema,
-                                   std::shared_ptr<RecordBatchFileWriter>* out) {
-  // ctor is private
-  *out = std::shared_ptr<RecordBatchFileWriter>(new RecordBatchFileWriter());
-  (*out)->impl_.reset(new RecordBatchFileWriterImpl(sink, schema));
-  return Status::OK();
-}
-#endif
-
 Status RecordBatchFileWriter::WriteRecordBatch(const RecordBatch& batch,
                                                bool allow_64bit) {
   return impl_->WriteRecordBatch(batch, allow_64bit);
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 89d3f5f268489..585373b2284cf 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -94,12 +94,6 @@ class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchWriter>* out);
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.7.0
-  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-                     std::shared_ptr<RecordBatchStreamWriter>* out);
-#endif
-
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
   Status Close() override;
   void set_memory_pool(MemoryPool* pool) override;
@@ -128,12 +122,6 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchWriter>* out);
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.7.0
-  static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-                     std::shared_ptr<RecordBatchFileWriter>* out);
-#endif
-
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
   Status Close() override;
 
@@ -231,16 +219,6 @@ ARROW_EXPORT
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
                    int64_t* body_length);
 
-#ifndef ARROW_NO_DEPRECATED_API
-/// EXPERIMENTAL: Write RecordBatch allowing lengths over INT32_MAX. This data
-/// may not be readable by all Arrow implementations
-/// \deprecated Since 0.7.0
-ARROW_EXPORT
-Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
-                             io::OutputStream* dst, int32_t* metadata_length,
-                             int64_t* body_length, MemoryPool* pool);
-#endif
-
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 009b5cf6373de..aac5dad1fbe1f 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -503,12 +503,6 @@ Status MakeTable(const std::shared_ptr<Schema>& schema,
 
 RecordBatchReader::~RecordBatchReader() {}
 
-#ifndef ARROW_NO_DEPRECATED_API
-Status RecordBatchReader::ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch) {
-  return ReadNext(batch);
-}
-#endif
-
 // ----------------------------------------------------------------------
 // Convert a table to a sequence of record batches
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 709a6666f9942..d40bdb8569307 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -294,11 +294,6 @@ class ARROW_EXPORT RecordBatchReader {
   /// \param[out] batch the next loaded batch, nullptr at end of stream
   /// \return Status
   virtual Status ReadNext(std::shared_ptr<RecordBatch>* batch) = 0;
-
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \deprecated Since 0.7.0
-  Status ReadNextRecordBatch(std::shared_ptr<RecordBatch>* batch);
-#endif
 };
 
 /// \brief Compute a sequence of record batches from a (possibly chunked) Table
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index ed7da8b2c0c7d..465f724823446 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -117,8 +117,6 @@ bool Tensor::is_column_major() const {
 
 Type::type Tensor::type_id() const { return type_->id(); }
 
-bool Tensor::Equals(const Tensor& other) const {
-  return TensorEquals(*this, other);
-}
+bool Tensor::Equals(const Tensor& other) const { return TensorEquals(*this, other); }
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 14ddd2a5910b0..a9bf591918558 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -82,9 +82,7 @@ std::string Field::ToString() const {
 
 DataType::~DataType() {}
 
-bool DataType::Equals(const DataType& other) const {
-  return TypeEquals(*this, other);
-}
+bool DataType::Equals(const DataType& other) const { return TypeEquals(*this, other); }
 
 bool DataType::Equals(const std::shared_ptr<DataType>& other) const {
   if (!other) {
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index e0116cc567b9c..4dd91e99ad9da 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -104,14 +104,6 @@ Status GetEmptyBitmap(MemoryPool* pool, int64_t length, std::shared_ptr<Buffer>*
   return Status::OK();
 }
 
-Status GetEmptyBitmap(MemoryPool* pool, int64_t length,
-                      std::shared_ptr<MutableBuffer>* result) {
-  std::shared_ptr<Buffer> buffer;
-  RETURN_NOT_OK(GetEmptyBitmap(pool, length, &buffer));
-  *result = std::dynamic_pointer_cast<MutableBuffer>(buffer);
-  return Status::OK();
-}
-
 Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t length,
                   std::shared_ptr<Buffer>* out) {
   std::shared_ptr<Buffer> buffer;
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index a85aff7c00d90..dd1024596974e 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -49,25 +49,6 @@
 
 namespace arrow {
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-// \deprecated Since > 0.7.0
-
-#define INIT_BITSET(valid_bits_vector, valid_bits_index)            \
-  int64_t byte_offset_##valid_bits_vector = (valid_bits_index) / 8; \
-  int64_t bit_offset_##valid_bits_vector = (valid_bits_index) % 8;  \
-  uint8_t bitset_##valid_bits_vector = valid_bits_vector[byte_offset_##valid_bits_vector];
-
-#define READ_NEXT_BITSET(valid_bits_vector)                                          \
-  bit_offset_##valid_bits_vector++;                                                  \
-  if (bit_offset_##valid_bits_vector == 8) {                                         \
-    bit_offset_##valid_bits_vector = 0;                                              \
-    byte_offset_##valid_bits_vector++;                                               \
-    bitset_##valid_bits_vector = valid_bits_vector[byte_offset_##valid_bits_vector]; \
-  }
-
-#endif
-
 // TODO(wesm): The source from Impala was depending on boost::make_unsigned
 //
 // We add a partial stub implementation here
@@ -522,13 +503,6 @@ ARROW_EXPORT
 bool BitmapEquals(const uint8_t* left, int64_t left_offset, const uint8_t* right,
                   int64_t right_offset, int64_t bit_length);
 
-#ifndef ARROW_NO_DEPRECATED_API
-/// \deprecated Since 0.7.0
-ARROW_EXPORT
-Status GetEmptyBitmap(MemoryPool* pool, int64_t length,
-                      std::shared_ptr<MutableBuffer>* result);
-#endif
-
 }  // namespace arrow
 
 #endif  // ARROW_UTIL_BIT_UTIL_H
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
index d604078114e38..3134ff10757ee 100644
--- a/cpp/src/plasma/io.cc
+++ b/cpp/src/plasma/io.cc
@@ -95,9 +95,9 @@ Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
   RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(type), sizeof(*type)),
                      *type = DISCONNECT_CLIENT);
   int64_t length_temp;
-  RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length_temp),
-                               sizeof(length_temp)),
-                     *type = DISCONNECT_CLIENT);
+  RETURN_NOT_OK_ELSE(
+      ReadBytes(fd, reinterpret_cast<uint8_t*>(&length_temp), sizeof(length_temp)),
+      *type = DISCONNECT_CLIENT);
   // The length must be read as an int64_t, but it should be used as a size_t.
   size_t length = length_temp;
   if (length > buffer->size()) {

From eaa9538522ce3f3f5bf5fdaa9422fd267e1285a4 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Thu, 5 Oct 2017 15:55:55 -0400
Subject: [PATCH 1125/1644] ARROW-1649: C++: Print number of nulls in
 PrettyPrint for NullArray

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1173 from xhochy/ARROW-1649 and squashes the following commits:

d1e169a3 [Korn, Uwe] ARROW-1649: C++: Print number of nulls in PrettyPrint for NullArray
---
 cpp/src/arrow/pretty_print.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 31b9344240262..aaea34a51388c 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -199,7 +199,10 @@ class ArrayPrinter : public PrettyPrinter {
     }
   }
 
-  Status Visit(const NullArray&) { return Status::OK(); }
+  Status Visit(const NullArray& array) {
+    (*sink_) << array.length() << " nulls";
+    return Status::OK();
+  }
 
   template <typename T>
   typename std::enable_if<std::is_base_of<PrimitiveArray, T>::value ||

From 3ae43551fbf22f1bd8a6e95436e96e99991134e8 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Thu, 5 Oct 2017 16:36:17 -0700
Subject: [PATCH 1126/1644] ARROW-1653: [Plasma] Use static cast to avoid
 compiler warning.

See https://github.com/apache/arrow/pull/1172#discussion_r142931449

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1178 from robertnishihara/fixwarning and squashes the following commits:

638a2f8 [Robert Nishihara] Fix bug.
16012a9 [Robert Nishihara] Use static cast to avoid compiler warning.
---
 cpp/src/plasma/io.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
index 3134ff10757ee..2228ad94c6346 100644
--- a/cpp/src/plasma/io.cc
+++ b/cpp/src/plasma/io.cc
@@ -99,7 +99,7 @@ Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
       ReadBytes(fd, reinterpret_cast<uint8_t*>(&length_temp), sizeof(length_temp)),
       *type = DISCONNECT_CLIENT);
   // The length must be read as an int64_t, but it should be used as a size_t.
-  size_t length = length_temp;
+  size_t length = static_cast<size_t>(length_temp);
   if (length > buffer->size()) {
     buffer->resize(length);
   }

From f8cdafae40e765cdb35ba9b8e330752325eabcd7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 21:33:08 -0400
Subject: [PATCH 1127/1644] ARROW-1541: [C++] Fix race conditions in arrow_gpu
 with generated Flatbuffers files. Do not put generated files in source tree

It's a better practice to put generated files in the build directory rather than in the main source tree. There was also a race condition with `arrow/gpu/cuda_version.h`, so fixed that also

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1180 from wesm/ARROW-1541 and squashes the following commits:

1e95ecae [Wes McKinney] Put generated Flatbuffers sources in the binary dir rather than source dir. Fix race conditions in arrow_gpu build
---
 cpp/CMakeLists.txt               |  2 ++
 cpp/src/arrow/CMakeLists.txt     | 13 +++++++------
 cpp/src/arrow/gpu/CMakeLists.txt |  5 ++++-
 cpp/src/arrow/ipc/.gitignore     |  1 -
 cpp/src/arrow/ipc/CMakeLists.txt |  2 +-
 5 files changed, 14 insertions(+), 9 deletions(-)
 delete mode 100644 cpp/src/arrow/ipc/.gitignore

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0930ae4ced92c..212d1f5f3a534 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -332,6 +332,8 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/src)
 include_directories(src)
 
 ############################################################
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 5c90331418142..9a676510e4bf9 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -60,11 +60,6 @@ if (ARROW_GPU)
   add_subdirectory(gpu)
 endif()
 
-if (ARROW_IPC)
-  add_subdirectory(ipc)
-  add_dependencies(arrow_dependencies metadata_fbs)
-endif()
-
 if (ARROW_WITH_BROTLI)
   add_definitions(-DARROW_WITH_BROTLI)
   SET(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS})
@@ -98,7 +93,9 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
 endif()
 
 if (ARROW_IPC)
-  set(ARROW_SRCS ${ARROW_SRCS}
+  add_subdirectory(ipc)
+
+  set(ARROW_IPC_SRCS
     ipc/dictionary.cc
     ipc/feather.cc
     ipc/json.cc
@@ -108,6 +105,10 @@ if (ARROW_IPC)
     ipc/reader.cc
     ipc/writer.cc
   )
+  SET(ARROW_SRCS ${ARROW_SRCS}
+    ${ARROW_IPC_SRCS})
+
+  add_dependencies(arrow_dependencies metadata_fbs)
 endif()
 
 if(NOT APPLE AND NOT MSVC)
diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt
index 176916e282292..c2f423378b876 100644
--- a/cpp/src/arrow/gpu/CMakeLists.txt
+++ b/cpp/src/arrow/gpu/CMakeLists.txt
@@ -32,6 +32,9 @@ set(ARROW_GPU_SRCS
   cuda_memory.cc
 )
 
+add_custom_target(arrow_gpu_sources DEPENDS ${ARROW_GPU_SRCS})
+add_dependencies(arrow_gpu_sources metadata_fbs)
+
 set(ARROW_GPU_SHARED_LINK_LIBS
   arrow_shared
   ${CUDA_LIBRARIES}
@@ -47,7 +50,7 @@ ADD_ARROW_LIB(arrow_gpu
 
 # CUDA build version
 configure_file(cuda_version.h.in
-  "${CMAKE_CURRENT_SOURCE_DIR}/cuda_version.h"
+  "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h"
   @ONLY)
 
 install(FILES
diff --git a/cpp/src/arrow/ipc/.gitignore b/cpp/src/arrow/ipc/.gitignore
deleted file mode 100644
index b2ddb055dcbc6..0000000000000
--- a/cpp/src/arrow/ipc/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*_generated.h
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 9689970a183fe..a24349fd6c928 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -46,7 +46,7 @@ set_source_files_properties(Message_generated.h PROPERTIES GENERATED TRUE)
 set_source_files_properties(feather_generated.h PROPERTIES GENERATED TRUE)
 set_source_files_properties(File_generated.h PROPERTIES GENERATED TRUE)
 
-set(OUTPUT_DIR ${CMAKE_SOURCE_DIR}/src/arrow/ipc)
+set(OUTPUT_DIR ${CMAKE_BINARY_DIR}/src/arrow/ipc)
 set(FBS_OUTPUT_FILES
   "${OUTPUT_DIR}/File_generated.h"
   "${OUTPUT_DIR}/Message_generated.h"

From ac1b66d051769a11d08d37c28215945a9c219b2d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 5 Oct 2017 21:34:10 -0400
Subject: [PATCH 1128/1644] ARROW-1540: Add NO_VALGRIND option to
 ADD_ARROW_TEST and disable valgrind in a few problematic tests

It turns out that using valgrind with CUDA out of the box just doesn't work that well. There were a couple other tests that fail in some valgrind setups so I disabled those, too. Everything else should valgrind cleanly.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1181 from wesm/ARROW-1540 and squashes the following commits:

f8a6aced [Wes McKinney] Add NO_VALGRIND option to ADD_ARROW_TEST so certain tests can be excluded from valgrind-enabled runs
---
 cpp/cmake_modules/BuildUtils.cmake  | 4 ++--
 cpp/src/arrow/allocator-test.cc     | 2 +-
 cpp/src/arrow/gpu/CMakeLists.txt    | 3 ++-
 cpp/src/arrow/io/CMakeLists.txt     | 2 +-
 cpp/src/arrow/python/CMakeLists.txt | 3 ++-
 5 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index e398dc1827cbb..fb09e4e95c6ae 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -270,7 +270,7 @@ endfunction()
 #
 # Arguments after the test name will be passed to set_tests_properties().
 function(ADD_ARROW_TEST REL_TEST_NAME)
-  set(options)
+  set(options NO_VALGRIND)
   set(single_value_args)
   set(multi_value_args STATIC_LINK_LIBS)
   cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
@@ -300,7 +300,7 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
     set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
   endif()
 
-  if (ARROW_TEST_MEMCHECK)
+  if (ARROW_TEST_MEMCHECK AND NOT ARG_NO_VALGRIND)
     SET_PROPERTY(TARGET ${TEST_NAME}
       APPEND_STRING PROPERTY
       COMPILE_FLAGS " -DARROW_VALGRIND")
diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
index e02741ec6aaf3..f3af47c1c95a7 100644
--- a/cpp/src/arrow/allocator-test.cc
+++ b/cpp/src/arrow/allocator-test.cc
@@ -41,7 +41,7 @@ TEST(stl_allocator, MemoryTracking) {
 
 TEST(stl_allocator, TestOOM) {
   stl_allocator<uint64_t> alloc;
-  uint64_t to_alloc = std::numeric_limits<uint64_t>::max();
+  uint64_t to_alloc = std::numeric_limits<uint64_t>::max() / 2;
   ASSERT_THROW(alloc.allocate(to_alloc), std::bad_alloc);
 }
 
diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt
index c2f423378b876..3f3069b91974b 100644
--- a/cpp/src/arrow/gpu/CMakeLists.txt
+++ b/cpp/src/arrow/gpu/CMakeLists.txt
@@ -79,7 +79,8 @@ set(ARROW_GPU_TEST_LINK_LIBS
 
 if (ARROW_BUILD_TESTS)
   ADD_ARROW_TEST(cuda-test
-    STATIC_LINK_LIBS ${ARROW_GPU_TEST_LINK_LIBS})
+    STATIC_LINK_LIBS ${ARROW_GPU_TEST_LINK_LIBS}
+    NO_VALGRIND)
 endif()
 
 if (ARROW_BUILD_BENCHMARKS)
diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index f35573870191f..98e3859910d43 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -20,7 +20,7 @@
 
 ADD_ARROW_TEST(io-file-test)
 if (NOT ARROW_BOOST_HEADER_ONLY)
-  ADD_ARROW_TEST(io-hdfs-test)
+  ADD_ARROW_TEST(io-hdfs-test NO_VALGRIND)
 endif()
 ADD_ARROW_TEST(io-memory-test)
 
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index af53a1631f708..aa12baacce5fc 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -117,7 +117,8 @@ install(
 
 if (ARROW_BUILD_TESTS)
   ADD_ARROW_TEST(python-test
-    STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}")
+    STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}"
+    NO_VALGRIND)
   target_link_libraries(python-test
     ${PYTHON_LIBRARIES})
 endif()

From 898f5e2736aee85f038ed23efb2cef9b0d5af7fa Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 6 Oct 2017 11:34:02 +0200
Subject: [PATCH 1129/1644] ARROW-1602: [C++] Add IsValid method to pair with
 IsNull

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1177 from wesm/ARROW-1602 and squashes the following commits:

0b97e18 [Wes McKinney] Add IsValid method to pair with IsNull
---
 cpp/src/arrow/array-test.cc | 3 ++-
 cpp/src/arrow/array.h       | 9 ++++++++-
 cpp/src/arrow/compare.cc    | 2 +-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 0b02c675d3537..ae9e9fd0091bd 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -142,7 +142,7 @@ TEST_F(TestArray, SliceRecomputeNullCount) {
   ASSERT_EQ(6, null_arr_sliced->null_count());
 }
 
-TEST_F(TestArray, TestIsNull) {
+TEST_F(TestArray, TestIsNullIsValid) {
   // clang-format off
   vector<uint8_t> null_bitmap = {1, 0, 1, 1, 0, 1, 0, 0,
                                  1, 0, 1, 1, 0, 1, 0, 0,
@@ -170,6 +170,7 @@ TEST_F(TestArray, TestIsNull) {
 
   for (size_t i = 0; i < null_bitmap.size(); ++i) {
     EXPECT_EQ(null_bitmap[i] != 0, !arr->IsNull(i)) << i;
+    EXPECT_EQ(null_bitmap[i] != 0, arr->IsValid(i)) << i;
   }
 }
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 882c63cddf15d..975cd5be1ecd4 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -169,12 +169,19 @@ class ARROW_EXPORT Array {
  public:
   virtual ~Array() = default;
 
-  /// Determine if a slot is null. For inner loops. Does *not* boundscheck
+  /// \brief Return true if value at index is null. Does not boundscheck
   bool IsNull(int64_t i) const {
     return null_bitmap_data_ != nullptr &&
            BitUtil::BitNotSet(null_bitmap_data_, i + data_->offset);
   }
 
+  /// \brief Return true if value at index is valid (not null). Does not
+  /// boundscheck
+  bool IsValid(int64_t i) const {
+    return null_bitmap_data_ != nullptr &&
+      BitUtil::GetBit(null_bitmap_data_, i + data_->offset);
+  }
+
   /// Size in the number of elements this array contains.
   int64_t length() const { return data_->length; }
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 94ea339caf774..2ec86c3695aa5 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -351,7 +351,7 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       const uint8_t* right_data = right.values()->data();
 
       for (int64_t i = 0; i < left.length(); ++i) {
-        if (!left.IsNull(i) &&
+        if (left.IsValid(i) &&
             BitUtil::GetBit(left_data, i + left.offset()) !=
                 BitUtil::GetBit(right_data, i + right.offset())) {
           result_ = false;

From 0a4c5b17fb1e6268f6fa9d56f9644ce24aaef064 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 6 Oct 2017 11:35:43 +0200
Subject: [PATCH 1130/1644] ARROW-1226: [C++] Docs cleaning in arrow/ipc.
 Doxyfile fixes, move ipc/metadata-internal.h symbols to internal NS

There were also some compiler warnings with clang-4.0 that I fixed here.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1179 from wesm/ARROW-1226 and squashes the following commits:

b6e7131 [Wes McKinney] Fix documentation warnings
7b01563 [Wes McKinney] Clean up doxygen docs in arrow/ipc. Move metadata-internal.h symbols to internal namespace. Fix Doxyfile so that full module paths show in generated files, like arrow/ipc/reader.h
---
 cpp/apidoc/Doxyfile                    |  2 +-
 cpp/cmake_modules/SetupCxxFlags.cmake  |  2 +-
 cpp/src/arrow/ipc/dictionary.h         | 14 +++--
 cpp/src/arrow/ipc/feather.h            | 44 ++++++++++++++--
 cpp/src/arrow/ipc/json.h               | 39 ++++++++++++--
 cpp/src/arrow/ipc/message.cc           |  4 +-
 cpp/src/arrow/ipc/message.h            |  5 ++
 cpp/src/arrow/ipc/metadata-internal.cc |  2 +
 cpp/src/arrow/ipc/metadata-internal.h  |  2 +
 cpp/src/arrow/ipc/reader.cc            | 15 +++---
 cpp/src/arrow/ipc/reader.h             | 57 +++++++++++++++------
 cpp/src/arrow/ipc/writer.cc            | 17 +++---
 cpp/src/arrow/ipc/writer.h             | 71 +++++++++++++++++++-------
 cpp/src/arrow/python/helpers.cc        |  3 +-
 cpp/src/plasma/CMakeLists.txt          |  5 +-
 15 files changed, 214 insertions(+), 68 deletions(-)

diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile
index fadb6d5be4af2..d5fd1ac0024b9 100644
--- a/cpp/apidoc/Doxyfile
+++ b/cpp/apidoc/Doxyfile
@@ -188,7 +188,7 @@ STRIP_FROM_PATH        =
 # specify the list of include paths that are normally passed to the compiler
 # using the -I flag.
 
-STRIP_FROM_INC_PATH    =
+STRIP_FROM_INC_PATH  = ../src
 
 # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
 # less readable) file names. This can be useful is your file systems doesn't
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 77bfac83555b0..4b1950f7a17bd 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -71,7 +71,7 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
 -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
--Wno-unused-parameter -Wno-undef \
+-Wno-comma -Wno-unused-parameter -Wno-undef \
 -Wno-shadow -Wno-switch-enum -Wno-exit-time-destructors \
 -Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \
 -Wno-implicit-fallthrough -Wno-unreachable-code-return \
diff --git a/cpp/src/arrow/ipc/dictionary.h b/cpp/src/arrow/ipc/dictionary.h
index 4130e9ba8a0b9..4494b134cbff8 100644
--- a/cpp/src/arrow/ipc/dictionary.h
+++ b/cpp/src/arrow/ipc/dictionary.h
@@ -38,26 +38,30 @@ namespace ipc {
 using DictionaryMap = std::unordered_map<int64_t, std::shared_ptr<Array>>;
 using DictionaryTypeMap = std::unordered_map<int64_t, std::shared_ptr<Field>>;
 
-// Memoization data structure for handling shared dictionaries
+/// \brief Memoization data structure for handling shared dictionaries
 class ARROW_EXPORT DictionaryMemo {
  public:
   DictionaryMemo();
 
-  // Returns KeyError if dictionary not found
+  /// \brief Returns KeyError if dictionary not found
   Status GetDictionary(int64_t id, std::shared_ptr<Array>* dictionary) const;
 
-  /// Return id for dictionary, computing new id if necessary
+  /// \brief Return id for dictionary, computing new id if necessary
   int64_t GetId(const std::shared_ptr<Array>& dictionary);
 
+  /// \brief Return true if dictionary array object is in this memo
   bool HasDictionary(const std::shared_ptr<Array>& dictionary) const;
+
+  /// \brief Return true if we have a dictionary for the input id
   bool HasDictionaryId(int64_t id) const;
 
-  // Add a dictionary to the memo with a particular id. Returns KeyError if
-  // that dictionary already exists
+  /// \brief Add a dictionary to the memo with a particular id. Returns
+  /// KeyError if that dictionary already exists
   Status AddDictionary(int64_t id, const std::shared_ptr<Array>& dictionary);
 
   const DictionaryMap& id_to_dictionary() const { return id_to_dictionary_; }
 
+  /// \brief The number of dictionaries stored in the memo
   int size() const { return static_cast<int>(id_to_dictionary_.size()); }
 
  private:
diff --git a/cpp/src/arrow/ipc/feather.h b/cpp/src/arrow/ipc/feather.h
index 83cd151172c5c..7c884e9f52d8c 100644
--- a/cpp/src/arrow/ipc/feather.h
+++ b/cpp/src/arrow/ipc/feather.h
@@ -48,28 +48,47 @@ static constexpr const int kFeatherVersion = 2;
 // ----------------------------------------------------------------------
 // Metadata accessor classes
 
+/// \class TableReader
+/// \brief An interface for reading columns from Feather files
 class ARROW_EXPORT TableReader {
  public:
   TableReader();
   ~TableReader();
 
+  /// \brief Open a Feather file from a RandomAccessFile interface
+  ///
+  /// \param[in] source a RandomAccessFile instance
+  /// \param[out] out the table reader
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& source,
                      std::unique_ptr<TableReader>* out);
 
-  // Optional table description
-  //
-  // This does not return a const std::string& because a string has to be
-  // copied from the flatbuffer to be able to return a non-flatbuffer type
+  /// \brief Optional table description
+  ///
+  /// This does not return a const std::string& because a string has to be
+  /// copied from the flatbuffer to be able to return a non-flatbuffer type
   std::string GetDescription() const;
+
+  /// \brief Return true if the table has a description field populated
   bool HasDescription() const;
 
+  /// \brief Return the version number of the Feather file
   int version() const;
 
+  /// \brief Return the number of rows in the file
   int64_t num_rows() const;
+
+  /// \brief Return the number of columns in the file
   int64_t num_columns() const;
 
   std::string GetColumnName(int i) const;
 
+  /// \brief Read a column from the file as an arrow::Column.
+  ///
+  /// \param[in] i the column index to read
+  /// \param[out] out the returned column
+  /// \return Status
+  ///
+  /// This function is zero-copy if the file source supports zero-copy reads
   Status GetColumn(int i, std::shared_ptr<Column>* out);
 
  private:
@@ -77,19 +96,34 @@ class ARROW_EXPORT TableReader {
   std::unique_ptr<TableReaderImpl> impl_;
 };
 
+/// \class TableWriter
+/// \brief Interface for writing Feather files
 class ARROW_EXPORT TableWriter {
  public:
   ~TableWriter();
 
+  /// \brief Create a new TableWriter that writes to an OutputStream
+  /// \param[in] stream an output stream
+  /// \param[out] out the returned table writer
+  /// \return Status
   static Status Open(const std::shared_ptr<io::OutputStream>& stream,
                      std::unique_ptr<TableWriter>* out);
 
+  /// \brief Set the description field in the file metadata
   void SetDescription(const std::string& desc);
+
+  /// \brief Set the number of rows in the file
   void SetNumRows(int64_t num_rows);
 
+  /// \brief Append a column to the file
+  ///
+  /// \param[in] name the column name
+  /// \param[in] values the column values as a contiguous arrow::Array
+  /// \return Status
   Status Append(const std::string& name, const Array& values);
 
-  // We are done, write the file metadata and footer
+  /// \brief Finalize the file by writing the file metadata and footer
+  /// \return Status
   Status Finalize();
 
  private:
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
index 0391172075837..51f30f0c109f3 100644
--- a/cpp/src/arrow/ipc/json.h
+++ b/cpp/src/arrow/ipc/json.h
@@ -35,14 +35,29 @@ class Schema;
 
 namespace ipc {
 
+/// \class JsonWriter
+/// \brief Write the JSON representation of an Arrow record batch file or stream
+///
+/// This is used for integration testing
 class ARROW_EXPORT JsonWriter {
  public:
   ~JsonWriter();
 
+  /// \brief Create a new JSON writer that writes to memory
+  ///
+  /// \param[in] schema the schema of record batches
+  /// \param[out] out the returned writer object
+  /// \return Status
   static Status Open(const std::shared_ptr<Schema>& schema,
                      std::unique_ptr<JsonWriter>* out);
 
+  /// \brief Append a record batch
   Status WriteRecordBatch(const RecordBatch& batch);
+
+  /// \brief Finish the JSON payload and return as a std::string
+  ///
+  /// \param[out] result the JSON as as a std::string
+  /// \return Status
   Status Finish(std::string* result);
 
  private:
@@ -53,23 +68,41 @@ class ARROW_EXPORT JsonWriter {
   std::unique_ptr<JsonWriterImpl> impl_;
 };
 
-// TODO(wesm): Read from a file stream rather than an in-memory buffer
+/// \class JsonReader
+/// \brief Read the JSON representation of an Arrow record batch file or stream
+///
+/// This is used for integration testing
 class ARROW_EXPORT JsonReader {
  public:
   ~JsonReader();
 
+  /// \brief Create a new JSON reader
+  ///
+  /// \param[in] pool a MemoryPool to use for buffer allocations
+  /// \param[in] data a Buffer containing the JSON data
+  /// \param[out] reader the returned reader object
+  /// \return Status
   static Status Open(MemoryPool* pool, const std::shared_ptr<Buffer>& data,
                      std::unique_ptr<JsonReader>* reader);
 
-  // Use the default memory pool
+  /// \brief Create a new JSON reader that uses the default memory pool
+  ///
+  /// \param[in] data a Buffer containing the JSON data
+  /// \param[out] reader the returned reader object
+  /// \return Status
   static Status Open(const std::shared_ptr<Buffer>& data,
                      std::unique_ptr<JsonReader>* reader);
 
+  /// \brief Return the schema read from the JSON
   std::shared_ptr<Schema> schema() const;
 
+  /// \brief Return the number of record batches
   int num_record_batches() const;
 
-  // Read a record batch from the file
+  /// \brief Read a particular record batch from the file
+  ///
+  /// \param[in] i the record batch index, does not boundscheck
+  /// \param[out] batch the read record batch
   Status ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) const;
 
  private:
diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc
index 0c587ab7b234f..0dd5c72e51980 100644
--- a/cpp/src/arrow/ipc/message.cc
+++ b/cpp/src/arrow/ipc/message.cc
@@ -44,7 +44,7 @@ class Message::MessageImpl {
     message_ = flatbuf::GetMessage(metadata_->data());
 
     // Check that the metadata version is supported
-    if (message_->version() < kMinMetadataVersion) {
+    if (message_->version() < internal::kMinMetadataVersion) {
       return Status::Invalid("Old metadata version not supported");
     }
 
@@ -166,7 +166,7 @@ Status Message::ReadFrom(const std::shared_ptr<Buffer>& metadata, io::InputStrea
 
 Status Message::SerializeTo(io::OutputStream* file, int64_t* output_length) const {
   int32_t metadata_length = 0;
-  RETURN_NOT_OK(WriteMessage(*metadata(), file, &metadata_length));
+  RETURN_NOT_OK(internal::WriteMessage(*metadata(), file, &metadata_length));
 
   *output_length = metadata_length;
 
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index 522b3bdcdbb47..67a95c7d24761 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -53,6 +53,7 @@ constexpr int kMaxNestingDepth = 64;
 // individual fields metadata can be retrieved from very large schema without
 //
 
+/// \class Message
 /// \brief An IPC message including metadata and body
 class ARROW_EXPORT Message {
  public:
@@ -70,6 +71,7 @@ class ARROW_EXPORT Message {
   /// \param[in] metadata a buffer containing the Flatbuffer metadata
   /// \param[in] body a buffer containing the message body, which may be nullptr
   /// \param[out] out the created message
+  /// \return Status
   static Status Open(const std::shared_ptr<Buffer>& metadata,
                      const std::shared_ptr<Buffer>& body, std::unique_ptr<Message>* out);
 
@@ -77,6 +79,7 @@ class ARROW_EXPORT Message {
   /// \param[in] metadata containing a serialized Message flatbuffer
   /// \param[in] stream an InputStream
   /// \param[out] out the created Message
+  /// \return Status
   ///
   /// \note If stream supports zero-copy, this is zero-copy
   static Status ReadFrom(const std::shared_ptr<Buffer>& metadata, io::InputStream* stream,
@@ -98,8 +101,10 @@ class ARROW_EXPORT Message {
   /// \return buffer is nullptr if no body
   std::shared_ptr<Buffer> body() const;
 
+  /// \brief The Message type
   Type type() const;
 
+  /// \brief The Message metadata version
   MetadataVersion metadata_version() const;
 
   const void* header() const;
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 8593dca79e2e7..162afb94bfc02 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -46,6 +46,7 @@ namespace arrow {
 namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
+namespace internal {
 
 using FBB = flatbuffers::FlatBufferBuilder;
 using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
@@ -933,5 +934,6 @@ Status WriteMessage(const Buffer& message, io::OutputStream* file,
   return Status::OK();
 }
 
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index 83d1f13012714..309e7587a754c 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -45,6 +45,7 @@ class OutputStream;
 }  // namespace io
 
 namespace ipc {
+namespace internal {
 
 static constexpr flatbuf::MetadataVersion kCurrentMetadataVersion =
     flatbuf::MetadataVersion_V3;
@@ -130,6 +131,7 @@ Status WriteDictionaryMessage(const int64_t id, const int64_t length,
                               const std::vector<BufferMetadata>& buffers,
                               std::shared_ptr<Buffer>* out);
 
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 5fe908240178a..50eb9039c6ab6 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -51,6 +51,9 @@ namespace flatbuf = org::apache::arrow::flatbuf;
 
 namespace ipc {
 
+using internal::FileBlock;
+using internal::kArrowMagicBytes;
+
 // ----------------------------------------------------------------------
 // Record batch read path
 
@@ -422,7 +425,7 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
     RETURN_NOT_OK(
         ReadMessageAndValidate(message_reader_.get(), Message::SCHEMA, false, &message));
 
-    RETURN_NOT_OK(GetDictionaryTypes(message->header(), &dictionary_types_));
+    RETURN_NOT_OK(internal::GetDictionaryTypes(message->header(), &dictionary_types_));
 
     // TODO(wesm): In future, we may want to reconcile the ids in the stream with
     // those found in the schema
@@ -431,7 +434,7 @@ class RecordBatchStreamReader::RecordBatchStreamReaderImpl {
       RETURN_NOT_OK(ReadNextDictionary());
     }
 
-    return GetSchema(message->header(), dictionary_memo_, &schema_);
+    return internal::GetSchema(message->header(), dictionary_memo_, &schema_);
   }
 
   Status ReadNext(std::shared_ptr<RecordBatch>* batch) {
@@ -588,7 +591,7 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
   }
 
   Status ReadSchema() {
-    RETURN_NOT_OK(GetDictionaryTypes(footer_->schema(), &dictionary_fields_));
+    RETURN_NOT_OK(internal::GetDictionaryTypes(footer_->schema(), &dictionary_fields_));
 
     // Read all the dictionaries
     for (int i = 0; i < num_dictionaries(); ++i) {
@@ -611,7 +614,7 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
     }
 
     // Get the schema
-    return GetSchema(footer_->schema(), *dictionary_memo_, &schema_);
+    return internal::GetSchema(footer_->schema(), *dictionary_memo_, &schema_);
   }
 
   Status Open(const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset) {
@@ -732,8 +735,8 @@ Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
   std::vector<int64_t> shape;
   std::vector<int64_t> strides;
   std::vector<std::string> dim_names;
-  RETURN_NOT_OK(
-      GetTensorMetadata(*message->metadata(), &type, &shape, &strides, &dim_names));
+  RETURN_NOT_OK(internal::GetTensorMetadata(*message->metadata(), &type, &shape, &strides,
+                                            &dim_names));
   *out = std::make_shared<Tensor>(type, message->body(), shape, strides, dim_names);
   return Status::OK();
 }
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 1db282f398a92..7581fbda5b140 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -48,6 +48,10 @@ using RecordBatchReader = ::arrow::RecordBatchReader;
 
 /// \class RecordBatchStreamReader
 /// \brief Synchronous batch stream reader that reads from io::InputStream
+///
+/// This class reads the schema (plus any dictionaries) as the first messages
+/// in the stream, followed by record batches. For more granular zero-copy
+/// reads see the ReadRecordBatch functions
 class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
  public:
   virtual ~RecordBatchStreamReader();
@@ -68,11 +72,16 @@ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
   /// \return Status
   static Status Open(io::InputStream* stream, std::shared_ptr<RecordBatchReader>* out);
 
-  /// \brief Version of Open that retains ownership of stream
+  /// \brief Open stream and retain ownership of stream object
+  /// \param[in] stream the input stream
+  /// \param[out] out the batch reader
+  /// \return Status
   static Status Open(const std::shared_ptr<io::InputStream>& stream,
                      std::shared_ptr<RecordBatchReader>* out);
 
+  /// \brief Returns the schema read from the stream
   std::shared_ptr<Schema> schema() const override;
+
   Status ReadNext(std::shared_ptr<RecordBatch>* batch) override;
 
  private:
@@ -88,11 +97,12 @@ class ARROW_EXPORT RecordBatchFileReader {
   ~RecordBatchFileReader();
 
   /// \brief Open a RecordBatchFileReader
-  // Open a file-like object that is assumed to be self-contained; i.e., the
-  // end of the file interface is the end of the Arrow file. Note that there
-  // can be any amount of data preceding the Arrow-formatted data, because we
-  // need only locate the end of the Arrow file stream to discover the metadata
-  // and then proceed to read the data into memory.
+  ///
+  /// Open a file-like object that is assumed to be self-contained; i.e., the
+  /// end of the file interface is the end of the Arrow file. Note that there
+  /// can be any amount of data preceding the Arrow-formatted data, because we
+  /// need only locate the end of the Arrow file stream to discover the metadata
+  /// and then proceed to read the data into memory.
   static Status Open(io::RandomAccessFile* file,
                      std::shared_ptr<RecordBatchFileReader>* reader);
 
@@ -102,31 +112,42 @@ class ARROW_EXPORT RecordBatchFileReader {
   /// metadata footer). The metadata must have been written with memory offsets
   /// relative to the start of the containing file
   ///
-  /// @param file the data source
-  /// @param footer_offset the position of the end of the Arrow "file"
+  /// \param[in] file the data source
+  /// \param[in] footer_offset the position of the end of the Arrow file
+  /// \param[out] reader the returned reader
+  /// \return Status
   static Status Open(io::RandomAccessFile* file, int64_t footer_offset,
                      std::shared_ptr<RecordBatchFileReader>* reader);
 
   /// \brief Version of Open that retains ownership of file
+  ///
+  /// \param[in] file the data source
+  /// \param[out] reader the returned reader
+  /// \return Status
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
                      std::shared_ptr<RecordBatchFileReader>* reader);
 
   /// \brief Version of Open that retains ownership of file
+  ///
+  /// \param[in] file the data source
+  /// \param[in] footer_offset the position of the end of the Arrow file
+  /// \param[out] reader the returned reader
+  /// \return Status
   static Status Open(const std::shared_ptr<io::RandomAccessFile>& file,
                      int64_t footer_offset,
                      std::shared_ptr<RecordBatchFileReader>* reader);
 
-  /// The schema includes any dictionaries
+  /// \brief The schema read from the file
   std::shared_ptr<Schema> schema() const;
 
-  /// Returns number of record batches in the file
+  /// \brief Returns the number of record batches in the file
   int num_record_batches() const;
 
-  /// Returns MetadataVersion in the file metadata
+  /// \brief Return the metadata version from the file metadata
   MetadataVersion version() const;
 
-  /// Read a record batch from the file. Does not copy memory if the input
-  /// source supports zero-copy.
+  /// \brief Read a particular record batch from the file. Does not copy memory
+  /// if the input source supports zero-copy.
   ///
   /// \param[in] i the index of the record batch to return
   /// \param[out] batch the read batch
@@ -142,10 +163,12 @@ class ARROW_EXPORT RecordBatchFileReader {
 
 // Generic read functions; does not copy data if the input supports zero copy reads
 
-/// \brief Read Schema from stream serialized as a sequence of IPC messages
+/// \brief Read Schema from stream serialized as a sequence of one or more IPC
+/// messages
 ///
 /// \param[in] stream an InputStream
 /// \param[out] out the output Schema
+/// \return Status
 ///
 /// If record batches follow the schema, it is better to use
 /// RecordBatchStreamReader
@@ -158,6 +181,7 @@ Status ReadSchema(io::InputStream* stream, std::shared_ptr<Schema>* out);
 /// \param[in] schema the record batch schema
 /// \param[in] stream the file where the batch is located
 /// \param[out] out the read record batch
+/// \return Status
 ARROW_EXPORT
 Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, io::InputStream* stream,
                        std::shared_ptr<RecordBatch>* out);
@@ -168,11 +192,12 @@ Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, io::InputStream* s
 /// \param[in] schema the record batch schema
 /// \param[in] file a random access file
 /// \param[out] out the read record batch
+/// \return Status
 ARROW_EXPORT
 Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
                        io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
 
-/// \brief Read record batch from fully encapulated Message
+/// \brief Read record batch from encapulated Message
 ///
 /// \param[in] message a message instance containing metadata and body
 /// \param[in] schema the record batch schema
@@ -189,6 +214,7 @@ Status ReadRecordBatch(const Message& message, const std::shared_ptr<Schema>& sc
 /// \param[in] file a random access file
 /// \param[in] max_recursion_depth the maximum permitted nesting depth
 /// \param[out] out the read record batch
+/// \return Status
 ARROW_EXPORT
 Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& schema,
                        int max_recursion_depth, io::RandomAccessFile* file,
@@ -199,6 +225,7 @@ Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& sc
 /// \param[in] offset the file location of the start of the message
 /// \param[in] file the file where the batch is located
 /// \param[out] out the read tensor
+/// \return Status
 ARROW_EXPORT
 Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
                   std::shared_ptr<Tensor>* out);
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index c32161434c447..279a69544faf2 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -42,6 +42,9 @@
 namespace arrow {
 namespace ipc {
 
+using internal::FileBlock;
+using internal::kArrowMagicBytes;
+
 // ----------------------------------------------------------------------
 // Record batch write path
 
@@ -201,7 +204,7 @@ class RecordBatchSerializer : public ArrayVisitor {
     // itself as an int32_t.
     std::shared_ptr<Buffer> metadata_fb;
     RETURN_NOT_OK(WriteMetadataMessage(batch.num_rows(), *body_length, &metadata_fb));
-    RETURN_NOT_OK(WriteMessage(*metadata_fb, dst, metadata_length));
+    RETURN_NOT_OK(internal::WriteMessage(*metadata_fb, dst, metadata_length));
 
 #ifndef NDEBUG
     RETURN_NOT_OK(dst->Tell(&current_position));
@@ -491,8 +494,8 @@ class RecordBatchSerializer : public ArrayVisitor {
   // In some cases, intermediate buffers may need to be allocated (with sliced arrays)
   MemoryPool* pool_;
 
-  std::vector<FieldMetadata> field_nodes_;
-  std::vector<BufferMetadata> buffer_meta_;
+  std::vector<internal::FieldMetadata> field_nodes_;
+  std::vector<internal::BufferMetadata> buffer_meta_;
   std::vector<std::shared_ptr<Buffer>> buffers_;
 
   int64_t max_recursion_depth_;
@@ -593,8 +596,8 @@ Status WriteTensorHeader(const Tensor& tensor, io::OutputStream* dst,
                          int32_t* metadata_length, int64_t* body_length) {
   RETURN_NOT_OK(AlignStreamPosition(dst));
   std::shared_ptr<Buffer> metadata;
-  RETURN_NOT_OK(WriteTensorMessage(tensor, 0, &metadata));
-  return WriteMessage(*metadata, dst, metadata_length);
+  RETURN_NOT_OK(internal::WriteTensorMessage(tensor, 0, &metadata));
+  return internal::WriteMessage(*metadata, dst, metadata_length);
 }
 
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
@@ -715,10 +718,10 @@ class SchemaWriter : public StreamBookKeeper {
 
   Status WriteSchema() {
     std::shared_ptr<Buffer> schema_fb;
-    RETURN_NOT_OK(WriteSchemaMessage(schema_, dictionary_memo_, &schema_fb));
+    RETURN_NOT_OK(internal::WriteSchemaMessage(schema_, dictionary_memo_, &schema_fb));
 
     int32_t metadata_length = 0;
-    RETURN_NOT_OK(WriteMessage(*schema_fb, sink_, &metadata_length));
+    RETURN_NOT_OK(internal::WriteMessage(*schema_fb, sink_, &metadata_length));
     RETURN_NOT_OK(UpdatePosition());
     DCHECK_EQ(0, position_ % 8) << "WriteSchema did not perform an aligned write";
     return Status::OK();
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 585373b2284cf..cedac45e712d6 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -94,8 +94,17 @@ class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchWriter>* out);
 
+  /// \brief Write a record batch to the stream
+  ///
+  /// \param[in] batch the record batch to write
+  /// \param[in] allow_64bit allow array lengths over INT32_MAX - 1
+  /// \return Status
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
+
+  /// \brief Close the stream by writing a 4-byte int32 0 EOS market
+  /// \return Status
   Status Close() override;
+
   void set_memory_pool(MemoryPool* pool) override;
 
  protected:
@@ -122,7 +131,15 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
   static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
                      std::shared_ptr<RecordBatchWriter>* out);
 
+  /// \brief Write a record batch to the file
+  ///
+  /// \param[in] batch the record batch to write
+  /// \param[in] allow_64bit allow array lengths over INT32_MAX - 1
+  /// \return Status
   Status WriteRecordBatch(const RecordBatch& batch, bool allow_64bit = false) override;
+
+  /// \brief Close the file stream by writing the file footer and magic number
+  /// \return Status
   Status Close() override;
 
  private:
@@ -131,6 +148,21 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
   std::unique_ptr<RecordBatchFileWriterImpl> impl_;
 };
 
+/// \brief Low-level API for writing a record batch (without schema) to an OutputStream
+///
+/// \param[in] batch the record batch to write
+/// \param[in] buffer_start_offset the start offset to use in the buffer metadata,
+/// generally should be 0
+/// \param[in] dst an OutputStream
+/// \param[out] metadata_length the size of the length-prefixed flatbuffer
+/// including padding to a 64-byte boundary
+/// \param[out] body_length the size of the contiguous buffer block plus
+/// \param[in] max_recursion_depth the maximum permitted nesting schema depth
+/// \param[in] allow_64bit permit field lengths exceeding INT32_MAX. May not be
+/// readable by other Arrow implementations
+/// padding bytes
+/// \return Status
+///
 /// Write the RecordBatch (collection of equal-length Arrow arrays) to the
 /// output stream in a contiguous block. The record batch metadata is written as
 /// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
@@ -142,18 +174,6 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
 /// Finally, the absolute offsets (relative to the start of the output stream)
 /// to the end of the body and end of the metadata / data header (suffixed by
 /// the header size) is returned in out-variables
-///
-/// \param[in] buffer_start_offset the start offset to use in the buffer metadata,
-/// default should be 0
-/// \param[in] allow_64bit permit field lengths exceeding INT32_MAX. May not be
-/// readable by other Arrow implementations
-/// \param[out] metadata_length the size of the length-prefixed flatbuffer
-/// including padding to a 64-byte boundary
-/// \param[out] body_length the size of the contiguous buffer block plus
-/// padding bytes
-/// \return Status
-///
-/// Low-level API
 ARROW_EXPORT
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
                         io::OutputStream* dst, int32_t* metadata_length,
@@ -174,6 +194,7 @@ Status SerializeRecordBatch(const RecordBatch& batch, MemoryPool* pool,
 /// \brief Write record batch to OutputStream
 ///
 /// \param[in] batch the record batch to write
+/// \param[in] pool a MemoryPool to use for temporary allocations, if needed
 /// \param[in] out the OutputStream to write the output to
 /// \return Status
 ///
@@ -194,7 +215,7 @@ ARROW_EXPORT
 Status SerializeSchema(const Schema& schema, MemoryPool* pool,
                        std::shared_ptr<Buffer>* out);
 
-/// \brief Write multiple record batches to OutputStream
+/// \brief Write multiple record batches to OutputStream, including schema
 /// \param[in] batches a vector of batches. Must all have same schema
 /// \param[out] dst an OutputStream
 /// \return Status
@@ -202,18 +223,30 @@ ARROW_EXPORT
 Status WriteRecordBatchStream(const std::vector<std::shared_ptr<RecordBatch>>& batches,
                               io::OutputStream* dst);
 
-// Compute the precise number of bytes needed in a contiguous memory segment to
-// write the record batch. This involves generating the complete serialized
-// Flatbuffers metadata.
+/// \brief Compute the number of bytes needed to write a record batch including metadata
+///
+/// \param[in] batch the record batch to write
+/// \param[out] size the size of the complete encapsulated message
+/// \return Status
 ARROW_EXPORT
 Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 
-// Compute the precise number of bytes needed in a contiguous memory segment to
-// write the tensor including metadata, padding, and data
+/// \brief Compute the number of bytes needed to write a tensor including metadata
+///
+/// \param[in] tensor the tenseor to write
+/// \param[out] size the size of the complete encapsulated message
+/// \return Status
 ARROW_EXPORT
 Status GetTensorSize(const Tensor& tensor, int64_t* size);
 
-/// EXPERIMENTAL: Write arrow::Tensor as a contiguous message
+/// \brief EXPERIMENTAL: Write arrow::Tensor as a contiguous message
+///
+/// \param[in] tensor the Tensor to write
+/// \param[in] dst the OutputStream to write to
+/// \param[out] metadata_length the actual metadata length
+/// \param[out] body_length the acutal message body length
+/// \return Status
+///
 /// <metadata size><metadata><tensor data>
 ARROW_EXPORT
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index ad6a7f125e0a0..708d9916f46d3 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -27,8 +27,7 @@ namespace py {
 
 #define GET_PRIMITIVE_TYPE(NAME, FACTORY) \
   case Type::NAME:                        \
-    return FACTORY();                     \
-    break;
+    return FACTORY()
 
 std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
   switch (type) {
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index c9339545fcaa9..4b76f25ba6e32 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -96,12 +96,13 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
     PROPERTY COMPILE_FLAGS
     " -Wno-parentheses-equality \
 -Wno-shorten-64-to-32 \
--Wno-unused-macros ")
+-Wno-unused-macros")
 
   set_property(SOURCE thirdparty/xxhash.cc
     APPEND_STRING
     PROPERTY COMPILE_FLAGS
-    "-Wno-unused-macros")
+    "-Wno-unused-macros \
+-Wno-unreachable-code")
 endif()
 
 if ("${COMPILER_FAMILY}" STREQUAL "gcc")

From 8309556c7d2b0e14df1422baa574cf2de8c1bd3b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 6 Oct 2017 07:38:56 -0400
Subject: [PATCH 1131/1644] ARROW-1556: [C++] Move verbose AssertArraysEqual
 function used in PARQUET-1100 into arrow/test-util.h

We already had a variant of this in compute-test.cc, so this consolidates the code and makes it easier for thirdparty arrow users

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1175 from wesm/ARROW-1556 and squashes the following commits:

0605b466 [Wes McKinney] Move verbose AssertArraysEqual function from PARQUET-1100 into arrow/test-util.h
---
 cpp/CMakeLists.txt                    |  1 +
 cpp/src/arrow/compute/compute-test.cc | 14 --------------
 cpp/src/arrow/test-util.h             | 12 ++++++++++++
 3 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 212d1f5f3a534..12d48549e3b6c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -517,6 +517,7 @@ endif()
 set(ARROW_BENCHMARK_LINK_LIBS
   arrow_static
   arrow_benchmark_main
+  gtest
   ${ARROW_STATIC_LINK_LIBS})
 
 set(ARROW_LINK_LIBS
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 9db0a0e13dd59..a4b502d2ae613 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -46,20 +46,6 @@ using std::vector;
 namespace arrow {
 namespace compute {
 
-void AssertArraysEqual(const Array& left, const Array& right) {
-  bool are_equal = ArrayEquals(left, right);
-
-  if (!are_equal) {
-    std::stringstream ss;
-
-    ss << "Left: ";
-    EXPECT_OK(PrettyPrint(left, 0, &ss));
-    ss << "\nRight: ";
-    EXPECT_OK(PrettyPrint(right, 0, &ss));
-    FAIL() << ss.str();
-  }
-}
-
 class ComputeFixture {
  public:
   ComputeFixture() : ctx_(default_memory_pool()) {}
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 22a933d374c08..80e4feb6c329a 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -31,6 +31,7 @@
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
 #include "arrow/memory_pool.h"
+#include "arrow/pretty_print.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
@@ -279,6 +280,17 @@ Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>&
   return builder->Finish(out);
 }
 
+void AssertArraysEqual(const Array& expected, const Array& actual) {
+  if (!actual.Equals(expected)) {
+    std::stringstream pp_result;
+    std::stringstream pp_expected;
+
+    EXPECT_OK(PrettyPrint(actual, 0, &pp_result));
+    EXPECT_OK(PrettyPrint(expected, 0, &pp_expected));
+    FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str();
+  }
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_TEST_UTIL_H_

From b29b06586ff89bee93062ad082a930d30d5f45df Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 7 Oct 2017 15:48:18 -0400
Subject: [PATCH 1132/1644] ARROW-1641: [C++] Hide std::mutex from public
 headers

This was one part of ARROW-1134 that we can push through. I had to do some refactoring since there was a mutex in one of the base file interfaces. It doesn't appear that this will impact parquet-cpp or other Arrow users

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1165 from wesm/ARROW-1641 and squashes the following commits:

9f4a466d [Wes McKinney] Make PyReadableFile C++ threadsafe also
002c5a62 [Wes McKinney] Make ReadableFile::Read threadsafe again
e373ff2a [Wes McKinney] Rewrite CudaBufferWriter to not inherit from FixedSizeBufferWriter. Fix deadlocks in ReadableFile
6ab45e8f [Wes McKinney] Hide mutexes from public headers
---
 cpp/src/arrow/allocator-test.cc   |  15 ++-
 cpp/src/arrow/array.h             |   2 +-
 cpp/src/arrow/gpu/cuda_memory.cc  | 161 +++++++++++++++++++++---------
 cpp/src/arrow/gpu/cuda_memory.h   |  21 ++--
 cpp/src/arrow/io/file.cc          |  43 +++++++-
 cpp/src/arrow/io/file.h           |  13 +++
 cpp/src/arrow/io/interfaces.cc    |   2 -
 cpp/src/arrow/io/interfaces.h     |   9 +-
 cpp/src/arrow/io/memory.cc        | 133 ++++++++++++++++--------
 cpp/src/arrow/io/memory.h         |  18 ++--
 cpp/src/arrow/memory_pool-test.cc |  29 +++---
 cpp/src/arrow/memory_pool.cc      | 100 ++++++++++---------
 cpp/src/arrow/memory_pool.h       |  21 ----
 cpp/src/arrow/python/io.cc        | 128 ++++++++++++++----------
 cpp/src/arrow/python/io.h         |  24 ++---
 15 files changed, 442 insertions(+), 277 deletions(-)

diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
index f3af47c1c95a7..7c768be92812c 100644
--- a/cpp/src/arrow/allocator-test.cc
+++ b/cpp/src/arrow/allocator-test.cc
@@ -59,17 +59,16 @@ TEST(stl_allocator, FreeLargeMemory) {
 }
 
 TEST(stl_allocator, MaxMemory) {
-  DefaultMemoryPool pool;
+  auto pool = default_memory_pool();
 
-  ASSERT_EQ(0, pool.max_memory());
-  stl_allocator<uint8_t> alloc(&pool);
-  uint8_t* data = alloc.allocate(100);
-  uint8_t* data2 = alloc.allocate(100);
+  stl_allocator<uint8_t> alloc(pool);
+  uint8_t* data = alloc.allocate(1000);
+  uint8_t* data2 = alloc.allocate(1000);
 
-  alloc.deallocate(data, 100);
-  alloc.deallocate(data2, 100);
+  alloc.deallocate(data, 1000);
+  alloc.deallocate(data2, 1000);
 
-  ASSERT_EQ(200, pool.max_memory());
+  ASSERT_EQ(2000, pool->max_memory());
 }
 
 #endif  // ARROW_VALGRIND
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 975cd5be1ecd4..36bceeb8f4aa0 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -179,7 +179,7 @@ class ARROW_EXPORT Array {
   /// boundscheck
   bool IsValid(int64_t i) const {
     return null_bitmap_data_ != nullptr &&
-      BitUtil::GetBit(null_bitmap_data_, i + data_->offset);
+           BitUtil::GetBit(null_bitmap_data_, i + data_->offset);
   }
 
   /// Size in the number of elements this array contains.
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
index d1026ca51abae..949c1d7a4dd41 100644
--- a/cpp/src/arrow/gpu/cuda_memory.cc
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <memory>
+#include <mutex>
 
 #include <cuda.h>
 
@@ -151,71 +152,139 @@ Status CudaBufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 // ----------------------------------------------------------------------
 // CudaBufferWriter
 
-CudaBufferWriter::CudaBufferWriter(const std::shared_ptr<CudaBuffer>& buffer)
-    : io::FixedSizeBufferWriter(buffer),
-      context_(buffer->context()),
-      buffer_size_(0),
-      buffer_position_(0) {}
-
-CudaBufferWriter::~CudaBufferWriter() {}
-
-Status CudaBufferWriter::Close() { return Flush(); }
+class CudaBufferWriter::CudaBufferWriterImpl {
+ public:
+  explicit CudaBufferWriterImpl(const std::shared_ptr<CudaBuffer>& buffer)
+      : context_(buffer->context()),
+        buffer_(buffer),
+        buffer_size_(0),
+        buffer_position_(0) {
+    buffer_ = buffer;
+    DCHECK(buffer->is_mutable()) << "Must pass mutable buffer";
+    mutable_data_ = buffer->mutable_data();
+    size_ = buffer->size();
+    position_ = 0;
+  }
 
-Status CudaBufferWriter::Flush() {
-  if (buffer_size_ > 0 && buffer_position_ > 0) {
-    // Only need to flush when the write has been buffered
-    RETURN_NOT_OK(context_->CopyHostToDevice(mutable_data_ + position_ - buffer_position_,
-                                             host_buffer_data_, buffer_position_));
-    buffer_position_ = 0;
+  Status Seek(int64_t position) {
+    if (position < 0 || position >= size_) {
+      return Status::IOError("position out of bounds");
+    }
+    position_ = position;
+    return Status::OK();
   }
-  return Status::OK();
-}
 
-Status CudaBufferWriter::Seek(int64_t position) {
-  if (buffer_position_ > 0) {
-    RETURN_NOT_OK(Flush());
+  Status Flush() {
+    if (buffer_size_ > 0 && buffer_position_ > 0) {
+      // Only need to flush when the write has been buffered
+      RETURN_NOT_OK(
+          context_->CopyHostToDevice(mutable_data_ + position_ - buffer_position_,
+                                     host_buffer_data_, buffer_position_));
+      buffer_position_ = 0;
+    }
+    return Status::OK();
   }
-  return io::FixedSizeBufferWriter::Seek(position);
-}
 
-Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
-  if (memcopy_num_threads_ > 1) {
-    return Status::Invalid("parallel CUDA memcpy not supported");
+  Status Tell(int64_t* position) const {
+    *position = position_;
+    return Status::OK();
   }
 
-  if (nbytes == 0) {
+  Status Write(const uint8_t* data, int64_t nbytes) {
+    if (nbytes == 0) {
+      return Status::OK();
+    }
+
+    if (buffer_size_ > 0) {
+      if (nbytes + buffer_position_ >= buffer_size_) {
+        // Reach end of buffer, write everything
+        RETURN_NOT_OK(Flush());
+        RETURN_NOT_OK(
+            context_->CopyHostToDevice(mutable_data_ + position_, data, nbytes));
+      } else {
+        // Write bytes to buffer
+        std::memcpy(host_buffer_data_ + buffer_position_, data, nbytes);
+        buffer_position_ += nbytes;
+      }
+    } else {
+      // Unbuffered write
+      RETURN_NOT_OK(context_->CopyHostToDevice(mutable_data_ + position_, data, nbytes));
+    }
+    position_ += nbytes;
     return Status::OK();
   }
 
-  if (buffer_size_ > 0) {
-    if (nbytes + buffer_position_ >= buffer_size_) {
-      // Reach end of buffer, write everything
+  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+    std::lock_guard<std::mutex> guard(lock_);
+    RETURN_NOT_OK(Seek(position));
+    return Write(data, nbytes);
+  }
+
+  Status SetBufferSize(const int64_t buffer_size) {
+    if (buffer_position_ > 0) {
+      // Flush any buffered data
       RETURN_NOT_OK(Flush());
-      RETURN_NOT_OK(context_->CopyHostToDevice(mutable_data_ + position_, data, nbytes));
-    } else {
-      // Write bytes to buffer
-      std::memcpy(host_buffer_data_ + buffer_position_, data, nbytes);
-      buffer_position_ += nbytes;
     }
-  } else {
-    // Unbuffered write
-    RETURN_NOT_OK(context_->CopyHostToDevice(mutable_data_ + position_, data, nbytes));
+    RETURN_NOT_OK(AllocateCudaHostBuffer(buffer_size, &host_buffer_));
+    host_buffer_data_ = host_buffer_->mutable_data();
+    buffer_size_ = buffer_size;
+    return Status::OK();
   }
-  position_ += nbytes;
-  return Status::OK();
+
+  int64_t buffer_size() const { return buffer_size_; }
+
+  int64_t buffer_position() const { return buffer_position_; }
+
+ private:
+  std::shared_ptr<CudaContext> context_;
+  std::shared_ptr<CudaBuffer> buffer_;
+  std::mutex lock_;
+  uint8_t* mutable_data_;
+  int64_t size_;
+  int64_t position_;
+
+  // Pinned host buffer for buffering writes on CPU before calling cudaMalloc
+  int64_t buffer_size_;
+  int64_t buffer_position_;
+  std::shared_ptr<CudaHostBuffer> host_buffer_;
+  uint8_t* host_buffer_data_;
+};
+
+CudaBufferWriter::CudaBufferWriter(const std::shared_ptr<CudaBuffer>& buffer) {
+  impl_.reset(new CudaBufferWriterImpl(buffer));
 }
 
-Status CudaBufferWriter::SetBufferSize(const int64_t buffer_size) {
-  if (buffer_position_ > 0) {
-    // Flush any buffered data
+CudaBufferWriter::~CudaBufferWriter() {}
+
+Status CudaBufferWriter::Close() { return Flush(); }
+
+Status CudaBufferWriter::Flush() { return impl_->Flush(); }
+
+Status CudaBufferWriter::Seek(int64_t position) {
+  if (impl_->buffer_position() > 0) {
     RETURN_NOT_OK(Flush());
   }
-  RETURN_NOT_OK(AllocateCudaHostBuffer(buffer_size, &host_buffer_));
-  host_buffer_data_ = host_buffer_->mutable_data();
-  buffer_size_ = buffer_size;
-  return Status::OK();
+  return impl_->Seek(position);
 }
 
+Status CudaBufferWriter::Tell(int64_t* position) const { return impl_->Tell(position); }
+
+Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
+  return impl_->Write(data, nbytes);
+}
+
+Status CudaBufferWriter::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+  return impl_->WriteAt(position, data, nbytes);
+}
+
+Status CudaBufferWriter::SetBufferSize(const int64_t buffer_size) {
+  return impl_->SetBufferSize(buffer_size);
+}
+
+int64_t CudaBufferWriter::buffer_size() const { return impl_->buffer_size(); }
+
+int64_t CudaBufferWriter::num_bytes_buffered() const { return impl_->buffer_position(); }
+
 // ----------------------------------------------------------------------
 
 Status AllocateCudaHostBuffer(const int64_t size, std::shared_ptr<CudaHostBuffer>* out) {
diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h
index d5407371f35f5..9ebd2ccf0c77e 100644
--- a/cpp/src/arrow/gpu/cuda_memory.h
+++ b/cpp/src/arrow/gpu/cuda_memory.h
@@ -145,7 +145,7 @@ class ARROW_EXPORT CudaBufferReader : public io::BufferReader {
 
 /// \class CudaBufferWriter
 /// \brief File interface for writing to CUDA buffers, with optional buffering
-class ARROW_EXPORT CudaBufferWriter : public io::FixedSizeBufferWriter {
+class ARROW_EXPORT CudaBufferWriter : public io::WriteableFile {
  public:
   explicit CudaBufferWriter(const std::shared_ptr<CudaBuffer>& buffer);
   ~CudaBufferWriter();
@@ -156,10 +156,14 @@ class ARROW_EXPORT CudaBufferWriter : public io::FixedSizeBufferWriter {
   /// \brief Flush buffered bytes to GPU
   Status Flush() override;
 
-  // Seek requires flushing if any bytes are buffered
   Status Seek(int64_t position) override;
+
   Status Write(const uint8_t* data, int64_t nbytes) override;
 
+  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+
+  Status Tell(int64_t* position) const override;
+
   /// \brief Set CPU buffer size to limit calls to cudaMemcpy
   /// \param[in] buffer_size the size of CPU buffer to allocate
   /// \return Status
@@ -168,19 +172,14 @@ class ARROW_EXPORT CudaBufferWriter : public io::FixedSizeBufferWriter {
   Status SetBufferSize(const int64_t buffer_size);
 
   /// \brief Returns size of host (CPU) buffer, 0 for unbuffered
-  int64_t buffer_size() const { return buffer_size_; }
+  int64_t buffer_size() const;
 
   /// \brief Returns number of bytes buffered on host
-  int64_t num_bytes_buffered() const { return buffer_position_; }
+  int64_t num_bytes_buffered() const;
 
  private:
-  std::shared_ptr<CudaContext> context_;
-
-  // Pinned host buffer for buffering writes on CPU before calling cudaMalloc
-  int64_t buffer_size_;
-  int64_t buffer_position_;
-  std::shared_ptr<CudaHostBuffer> host_buffer_;
-  uint8_t* host_buffer_data_;
+  class CudaBufferWriterImpl;
+  std::unique_ptr<CudaBufferWriterImpl> impl_;
 };
 
 /// \brief Allocate CUDA-accessible memory on CPU host
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index ca536321ba3d8..74c6c09e6219b 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -355,10 +355,15 @@ class OSFile {
   }
 
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-    std::lock_guard<std::mutex> guard(lock_);
     return FileRead(fd_, out, nbytes, bytes_read);
   }
 
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+    std::lock_guard<std::mutex> guard(lock_);
+    RETURN_NOT_OK(Seek(position));
+    return Read(nbytes, bytes_read, out);
+  }
+
   Status Seek(int64_t pos) {
     if (pos < 0) {
       return Status::Invalid("Invalid position");
@@ -384,6 +389,8 @@ class OSFile {
 
   FileMode::type mode() const { return mode_; }
 
+  std::mutex& lock() { return lock_; }
+
  protected:
   Status SetFileName(const std::string& file_name) {
 #if defined(_MSC_VER)
@@ -458,10 +465,24 @@ Status ReadableFile::Close() { return impl_->Close(); }
 Status ReadableFile::Tell(int64_t* pos) const { return impl_->Tell(pos); }
 
 Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  std::lock_guard<std::mutex> guard(impl_->lock());
   return impl_->Read(nbytes, bytes_read, out);
 }
 
+Status ReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                            uint8_t* out) {
+  return impl_->ReadAt(position, nbytes, bytes_read, out);
+}
+
+Status ReadableFile::ReadAt(int64_t position, int64_t nbytes,
+                            std::shared_ptr<Buffer>* out) {
+  std::lock_guard<std::mutex> guard(impl_->lock());
+  RETURN_NOT_OK(Seek(position));
+  return impl_->ReadBuffer(nbytes, out);
+}
+
 Status ReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
+  std::lock_guard<std::mutex> guard(impl_->lock());
   return impl_->ReadBuffer(nbytes, out);
 }
 
@@ -590,6 +611,8 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
 
   int fd() const { return file_->fd(); }
 
+  std::mutex& lock() { return file_->lock(); }
+
  private:
   std::unique_ptr<OSFile> file_;
   int64_t position_;
@@ -671,10 +694,24 @@ Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   return Status::OK();
 }
 
+Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                                uint8_t* out) {
+  std::lock_guard<std::mutex> guard(memory_map_->lock());
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, bytes_read, out);
+}
+
+Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes,
+                                std::shared_ptr<Buffer>* out) {
+  std::lock_guard<std::mutex> guard(memory_map_->lock());
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, out);
+}
+
 bool MemoryMappedFile::supports_zero_copy() const { return true; }
 
 Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
-  std::lock_guard<std::mutex> guard(lock_);
+  std::lock_guard<std::mutex> guard(memory_map_->lock());
 
   if (!memory_map_->opened() || !memory_map_->writable()) {
     return Status::IOError("Unable to write");
@@ -685,7 +722,7 @@ Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t
 }
 
 Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
-  std::lock_guard<std::mutex> guard(lock_);
+  std::lock_guard<std::mutex> guard(memory_map_->lock());
 
   if (!memory_map_->opened() || !memory_map_->writable()) {
     return Status::IOError("Unable to write");
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 4fb09634a2b78..7937fea74997d 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -96,6 +96,13 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile {
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
+  /// \brief Thread-safe implementation of ReadAt
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                uint8_t* out) override;
+
+  /// \brief Thread-safe implementation of ReadAt
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
   Status GetSize(int64_t* size) override;
   Status Seek(int64_t position) override;
 
@@ -139,6 +146,12 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   // Zero copy read. Not thread-safe
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                uint8_t* out) override;
+
+  /// Default implementation is thread-safe
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
   bool supports_zero_copy() const override;
 
   /// Write data at the current position in the file. Thread-safe
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 694575b5f06ab..499da6f744386 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -32,14 +32,12 @@ RandomAccessFile::RandomAccessFile() { set_mode(FileMode::READ); }
 
 Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
                                 uint8_t* out) {
-  std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
 }
 
 Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes,
                                 std::shared_ptr<Buffer>* out) {
-  std::lock_guard<std::mutex> guard(lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, out);
 }
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 2c5b351e2082a..eee4e956ee589 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -20,7 +20,6 @@
 
 #include <cstdint>
 #include <memory>
-#include <mutex>
 #include <string>
 #include <vector>
 
@@ -131,18 +130,14 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
   /// Read at position, provide default implementations using Read(...), but can
   /// be overridden
   ///
-  /// Default implementation is thread-safe
+  /// Default implementation is not thread-safe
   virtual Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
                         uint8_t* out);
 
-  /// Default implementation is thread-safe
+  /// Default implementation is not thread-safe
   virtual Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out);
 
-  std::mutex& lock() { return lock_; }
-
  protected:
-  std::mutex lock_;
-
   RandomAccessFile();
 };
 
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 0b91ab5186fef..370d3e9566a11 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -20,6 +20,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
+#include <mutex>
 
 #include "arrow/buffer.h"
 #include "arrow/status.h"
@@ -127,67 +128,109 @@ static constexpr int kMemcopyDefaultNumThreads = 1;
 static constexpr int64_t kMemcopyDefaultBlocksize = 64;
 static constexpr int64_t kMemcopyDefaultThreshold = 1024 * 1024;
 
-/// Input buffer must be mutable, will abort if not
-FixedSizeBufferWriter::FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer)
-    : memcopy_num_threads_(kMemcopyDefaultNumThreads),
-      memcopy_blocksize_(kMemcopyDefaultBlocksize),
-      memcopy_threshold_(kMemcopyDefaultThreshold) {
-  DCHECK(buffer) << "Buffer was nullptr";
-  buffer_ = buffer;
-  DCHECK(buffer->is_mutable()) << "Must pass mutable buffer";
-  mutable_data_ = buffer->mutable_data();
-  size_ = buffer->size();
-  position_ = 0;
-}
+class FixedSizeBufferWriter::FixedSizeBufferWriterImpl {
+ public:
+  /// Input buffer must be mutable, will abort if not
+
+  /// Input buffer must be mutable, will abort if not
+  explicit FixedSizeBufferWriterImpl(const std::shared_ptr<Buffer>& buffer)
+      : memcopy_num_threads_(kMemcopyDefaultNumThreads),
+        memcopy_blocksize_(kMemcopyDefaultBlocksize),
+        memcopy_threshold_(kMemcopyDefaultThreshold) {
+    buffer_ = buffer;
+    DCHECK(buffer->is_mutable()) << "Must pass mutable buffer";
+    mutable_data_ = buffer->mutable_data();
+    size_ = buffer->size();
+    position_ = 0;
+  }
 
-FixedSizeBufferWriter::~FixedSizeBufferWriter() {}
+  ~FixedSizeBufferWriterImpl() {}
 
-Status FixedSizeBufferWriter::Close() {
-  // no-op
-  return Status::OK();
-}
+  Status Close() {
+    // No-op
+    return Status::OK();
+  }
 
-Status FixedSizeBufferWriter::Seek(int64_t position) {
-  if (position < 0 || position >= size_) {
-    return Status::IOError("position out of bounds");
+  Status Seek(int64_t position) {
+    if (position < 0 || position >= size_) {
+      return Status::IOError("position out of bounds");
+    }
+    position_ = position;
+    return Status::OK();
   }
-  position_ = position;
-  return Status::OK();
-}
+
+  Status Tell(int64_t* position) {
+    *position = position_;
+    return Status::OK();
+  }
+
+  Status Write(const uint8_t* data, int64_t nbytes) {
+    if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) {
+      internal::parallel_memcopy(mutable_data_ + position_, data, nbytes,
+                                 memcopy_blocksize_, memcopy_num_threads_);
+    } else {
+      memcpy(mutable_data_ + position_, data, nbytes);
+    }
+    position_ += nbytes;
+    return Status::OK();
+  }
+
+  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+    std::lock_guard<std::mutex> guard(lock_);
+    RETURN_NOT_OK(Seek(position));
+    return Write(data, nbytes);
+  }
+
+  void set_memcopy_threads(int num_threads) { memcopy_num_threads_ = num_threads; }
+
+  void set_memcopy_blocksize(int64_t blocksize) { memcopy_blocksize_ = blocksize; }
+
+  void set_memcopy_threshold(int64_t threshold) { memcopy_threshold_ = threshold; }
+
+ private:
+  std::mutex lock_;
+  std::shared_ptr<Buffer> buffer_;
+  uint8_t* mutable_data_;
+  int64_t size_;
+  int64_t position_;
+
+  int memcopy_num_threads_;
+  int64_t memcopy_blocksize_;
+  int64_t memcopy_threshold_;
+};
+
+FixedSizeBufferWriter::~FixedSizeBufferWriter() {}
+
+FixedSizeBufferWriter::FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer)
+    : impl_(new FixedSizeBufferWriterImpl(buffer)) {}
+
+Status FixedSizeBufferWriter::Close() { return impl_->Close(); }
+
+Status FixedSizeBufferWriter::Seek(int64_t position) { return impl_->Seek(position); }
 
 Status FixedSizeBufferWriter::Tell(int64_t* position) const {
-  *position = position_;
-  return Status::OK();
+  return impl_->Tell(position);
 }
 
 Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
-  if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) {
-    internal::parallel_memcopy(mutable_data_ + position_, data, nbytes,
-                               memcopy_blocksize_, memcopy_num_threads_);
-  } else {
-    memcpy(mutable_data_ + position_, data, nbytes);
-  }
-  position_ += nbytes;
-  return Status::OK();
+  return impl_->Write(data, nbytes);
 }
 
 Status FixedSizeBufferWriter::WriteAt(int64_t position, const uint8_t* data,
                                       int64_t nbytes) {
-  std::lock_guard<std::mutex> guard(lock_);
-  RETURN_NOT_OK(Seek(position));
-  return Write(data, nbytes);
+  return impl_->WriteAt(position, data, nbytes);
 }
 
 void FixedSizeBufferWriter::set_memcopy_threads(int num_threads) {
-  memcopy_num_threads_ = num_threads;
+  impl_->set_memcopy_threads(num_threads);
 }
 
 void FixedSizeBufferWriter::set_memcopy_blocksize(int64_t blocksize) {
-  memcopy_blocksize_ = blocksize;
+  impl_->set_memcopy_blocksize(blocksize);
 }
 
 void FixedSizeBufferWriter::set_memcopy_threshold(int64_t threshold) {
-  memcopy_threshold_ = threshold;
+  impl_->set_memcopy_threshold(threshold);
 }
 
 // ----------------------------------------------------------------------
@@ -233,6 +276,18 @@ Status BufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   return Status::OK();
 }
 
+Status BufferReader::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                            uint8_t* out) {
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, bytes_read, out);
+}
+
+Status BufferReader::ReadAt(int64_t position, int64_t nbytes,
+                            std::shared_ptr<Buffer>* out) {
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, out);
+}
+
 Status BufferReader::GetSize(int64_t* size) {
   *size = size_;
   return Status::OK();
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 563000f77b4fb..978c198c2dd93 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -22,7 +22,6 @@
 
 #include <cstdint>
 #include <memory>
-#include <mutex>
 
 #include "arrow/io/interfaces.h"
 #include "arrow/util/visibility.h"
@@ -99,15 +98,8 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
   void set_memcopy_threshold(int64_t threshold);
 
  protected:
-  std::mutex lock_;
-  std::shared_ptr<Buffer> buffer_;
-  uint8_t* mutable_data_;
-  int64_t size_;
-  int64_t position_;
-
-  int memcopy_num_threads_;
-  int64_t memcopy_blocksize_;
-  int64_t memcopy_threshold_;
+  class FixedSizeBufferWriterImpl;
+  std::unique_ptr<FixedSizeBufferWriterImpl> impl_;
 };
 
 /// \class BufferReader
@@ -125,6 +117,12 @@ class ARROW_EXPORT BufferReader : public RandomAccessFile {
   // Zero copy read
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                uint8_t* out) override;
+
+  /// Default implementation is thread-safe
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
   Status GetSize(int64_t* size) override;
   Status Seek(int64_t position) override;
 
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 552c79b5ae78d..0a4785d5229be 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -59,39 +59,36 @@ TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
 }
 
 TEST(DefaultMemoryPoolDeathTest, MaxMemory) {
-  DefaultMemoryPool pool;
-
-  ASSERT_EQ(0, pool.max_memory());
+  MemoryPool* pool = default_memory_pool();
 
   uint8_t* data;
-  ASSERT_OK(pool.Allocate(100, &data));
+  ASSERT_OK(pool->Allocate(100, &data));
 
   uint8_t* data2;
-  ASSERT_OK(pool.Allocate(100, &data2));
+  ASSERT_OK(pool->Allocate(100, &data2));
 
-  pool.Free(data, 100);
-  pool.Free(data2, 100);
+  pool->Free(data, 100);
+  pool->Free(data2, 100);
 
-  ASSERT_EQ(200, pool.max_memory());
+  ASSERT_EQ(200, pool->max_memory());
 }
 
 #endif  // ARROW_VALGRIND
 
 TEST(LoggingMemoryPool, Logging) {
-  DefaultMemoryPool pool;
-  LoggingMemoryPool lp(&pool);
+  MemoryPool* pool = default_memory_pool();
 
-  ASSERT_EQ(0, lp.max_memory());
+  LoggingMemoryPool lp(pool);
 
   uint8_t* data;
-  ASSERT_OK(pool.Allocate(100, &data));
+  ASSERT_OK(pool->Allocate(100, &data));
 
   uint8_t* data2;
-  ASSERT_OK(pool.Allocate(100, &data2));
+  ASSERT_OK(pool->Allocate(100, &data2));
 
-  pool.Free(data, 100);
-  pool.Free(data2, 100);
+  pool->Free(data, 100);
+  pool->Free(data2, 100);
 
-  ASSERT_EQ(200, pool.max_memory());
+  ASSERT_EQ(200, pool->max_memory());
 }
 }  // namespace arrow
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 851065b521118..3496636a40587 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -85,74 +85,82 @@ MemoryPool::~MemoryPool() {}
 
 int64_t MemoryPool::max_memory() const { return -1; }
 
-DefaultMemoryPool::DefaultMemoryPool() : bytes_allocated_(0) { max_memory_ = 0; }
+class DefaultMemoryPool : public MemoryPool {
+ public:
+  DefaultMemoryPool() : bytes_allocated_(0) { max_memory_ = 0; }
 
-Status DefaultMemoryPool::Allocate(int64_t size, uint8_t** out) {
-  RETURN_NOT_OK(AllocateAligned(size, out));
-  bytes_allocated_ += size;
+  ~DefaultMemoryPool() {}
 
-  {
-    std::lock_guard<std::mutex> guard(lock_);
-    if (bytes_allocated_ > max_memory_) {
-      max_memory_ = bytes_allocated_.load();
+  Status Allocate(int64_t size, uint8_t** out) override {
+    RETURN_NOT_OK(AllocateAligned(size, out));
+    bytes_allocated_ += size;
+
+    {
+      std::lock_guard<std::mutex> guard(lock_);
+      if (bytes_allocated_ > max_memory_) {
+        max_memory_ = bytes_allocated_.load();
+      }
     }
+    return Status::OK();
   }
-  return Status::OK();
-}
 
-Status DefaultMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
+  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override {
 #ifdef ARROW_JEMALLOC
-  *ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
-  if (*ptr == NULL) {
-    std::stringstream ss;
-    ss << "realloc of size " << new_size << " failed";
-    return Status::OutOfMemory(ss.str());
-  }
+    *ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
+    if (*ptr == NULL) {
+      std::stringstream ss;
+      ss << "realloc of size " << new_size << " failed";
+      return Status::OutOfMemory(ss.str());
+    }
 #else
-  // Note: We cannot use realloc() here as it doesn't guarantee alignment.
-
-  // Allocate new chunk
-  uint8_t* out = nullptr;
-  RETURN_NOT_OK(AllocateAligned(new_size, &out));
-  DCHECK(out);
-  // Copy contents and release old memory chunk
-  memcpy(out, *ptr, static_cast<size_t>(std::min(new_size, old_size)));
+    // Note: We cannot use realloc() here as it doesn't guarantee alignment.
+
+    // Allocate new chunk
+    uint8_t* out = nullptr;
+    RETURN_NOT_OK(AllocateAligned(new_size, &out));
+    DCHECK(out);
+    // Copy contents and release old memory chunk
+    memcpy(out, *ptr, static_cast<size_t>(std::min(new_size, old_size)));
 #ifdef _MSC_VER
-  _aligned_free(*ptr);
+    _aligned_free(*ptr);
 #else
-  std::free(*ptr);
+    std::free(*ptr);
 #endif  // defined(_MSC_VER)
-  *ptr = out;
+    *ptr = out;
 #endif  // defined(ARROW_JEMALLOC)
 
-  bytes_allocated_ += new_size - old_size;
-  {
-    std::lock_guard<std::mutex> guard(lock_);
-    if (bytes_allocated_ > max_memory_) {
-      max_memory_ = bytes_allocated_.load();
+    bytes_allocated_ += new_size - old_size;
+    {
+      std::lock_guard<std::mutex> guard(lock_);
+      if (bytes_allocated_ > max_memory_) {
+        max_memory_ = bytes_allocated_.load();
+      }
     }
-  }
 
-  return Status::OK();
-}
+    return Status::OK();
+  }
 
-int64_t DefaultMemoryPool::bytes_allocated() const { return bytes_allocated_.load(); }
+  int64_t bytes_allocated() const override { return bytes_allocated_.load(); }
 
-void DefaultMemoryPool::Free(uint8_t* buffer, int64_t size) {
-  DCHECK_GE(bytes_allocated_, size);
+  void Free(uint8_t* buffer, int64_t size) override {
+    DCHECK_GE(bytes_allocated_, size);
 #ifdef _MSC_VER
-  _aligned_free(buffer);
+    _aligned_free(buffer);
 #elif defined(ARROW_JEMALLOC)
-  dallocx(buffer, MALLOCX_ALIGN(kAlignment));
+    dallocx(buffer, MALLOCX_ALIGN(kAlignment));
 #else
-  std::free(buffer);
+    std::free(buffer);
 #endif
-  bytes_allocated_ -= size;
-}
+    bytes_allocated_ -= size;
+  }
 
-int64_t DefaultMemoryPool::max_memory() const { return max_memory_.load(); }
+  int64_t max_memory() const override { return max_memory_.load(); }
 
-DefaultMemoryPool::~DefaultMemoryPool() {}
+ private:
+  mutable std::mutex lock_;
+  std::atomic<int64_t> bytes_allocated_;
+  std::atomic<int64_t> max_memory_;
+};
 
 MemoryPool* default_memory_pool() {
   static DefaultMemoryPool default_memory_pool_;
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 5bb2b5669ed31..52ec67fee8dc9 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -20,7 +20,6 @@
 
 #include <atomic>
 #include <cstdint>
-#include <mutex>
 
 #include "arrow/util/visibility.h"
 
@@ -69,26 +68,6 @@ class ARROW_EXPORT MemoryPool {
   MemoryPool();
 };
 
-class ARROW_EXPORT DefaultMemoryPool : public MemoryPool {
- public:
-  DefaultMemoryPool();
-  virtual ~DefaultMemoryPool();
-
-  Status Allocate(int64_t size, uint8_t** out) override;
-  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
-
-  void Free(uint8_t* buffer, int64_t size) override;
-
-  int64_t bytes_allocated() const override;
-
-  int64_t max_memory() const override;
-
- private:
-  mutable std::mutex lock_;
-  std::atomic<int64_t> bytes_allocated_;
-  std::atomic<int64_t> max_memory_;
-};
-
 class ARROW_EXPORT LoggingMemoryPool : public MemoryPool {
  public:
   explicit LoggingMemoryPool(MemoryPool* pool);
diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index 99b99f7469f9e..b01358ab00b4b 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -19,6 +19,7 @@
 
 #include <cstdint>
 #include <cstdlib>
+#include <mutex>
 #include <string>
 
 #include "arrow/io/memory.h"
@@ -33,10 +34,6 @@ namespace py {
 // ----------------------------------------------------------------------
 // Python file
 
-PythonFile::PythonFile(PyObject* file) : file_(file) { Py_INCREF(file_); }
-
-PythonFile::~PythonFile() { Py_DECREF(file_); }
-
 // This is annoying: because C++11 does not allow implicit conversion of string
 // literals to non-const char*, we need to go through some gymnastics to use
 // PyObject_CallMethod without a lot of pain (its arguments are non-const
@@ -48,53 +45,68 @@ static inline PyObject* cpp_PyObject_CallMethod(PyObject* obj, const char* metho
                              const_cast<char*>(argspec), args...);
 }
 
-Status PythonFile::Close() {
-  // whence: 0 for relative to start of file, 2 for end of file
-  PyObject* result = cpp_PyObject_CallMethod(file_, "close", "()");
-  Py_XDECREF(result);
-  PY_RETURN_IF_ERROR(StatusCode::IOError);
-  return Status::OK();
-}
-
-Status PythonFile::Seek(int64_t position, int whence) {
-  // whence: 0 for relative to start of file, 2 for end of file
-  PyObject* result = cpp_PyObject_CallMethod(file_, "seek", "(ii)", position, whence);
-  Py_XDECREF(result);
-  PY_RETURN_IF_ERROR(StatusCode::IOError);
-  return Status::OK();
-}
-
-Status PythonFile::Read(int64_t nbytes, PyObject** out) {
-  PyObject* result = cpp_PyObject_CallMethod(file_, "read", "(i)", nbytes);
-  PY_RETURN_IF_ERROR(StatusCode::IOError);
-  *out = result;
-  return Status::OK();
-}
-
-Status PythonFile::Write(const uint8_t* data, int64_t nbytes) {
-  PyObject* py_data =
-      PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), nbytes);
-  PY_RETURN_IF_ERROR(StatusCode::IOError);
-
-  PyObject* result = cpp_PyObject_CallMethod(file_, "write", "(O)", py_data);
-  Py_XDECREF(py_data);
-  Py_XDECREF(result);
-  PY_RETURN_IF_ERROR(StatusCode::IOError);
-  return Status::OK();
-}
-
-Status PythonFile::Tell(int64_t* position) {
-  PyObject* result = cpp_PyObject_CallMethod(file_, "tell", "()");
-  PY_RETURN_IF_ERROR(StatusCode::IOError);
-
-  *position = PyLong_AsLongLong(result);
-  Py_DECREF(result);
-
-  // PyLong_AsLongLong can raise OverflowError
-  PY_RETURN_IF_ERROR(StatusCode::IOError);
-
-  return Status::OK();
-}
+// A common interface to a Python file-like object. Must acquire GIL before
+// calling any methods
+class PythonFile {
+ public:
+  explicit PythonFile(PyObject* file) : file_(file) { Py_INCREF(file_); }
+
+  ~PythonFile() { Py_DECREF(file_); }
+
+  Status Close() {
+    // whence: 0 for relative to start of file, 2 for end of file
+    PyObject* result = cpp_PyObject_CallMethod(file_, "close", "()");
+    Py_XDECREF(result);
+    PY_RETURN_IF_ERROR(StatusCode::IOError);
+    return Status::OK();
+  }
+
+  Status Seek(int64_t position, int whence) {
+    // whence: 0 for relative to start of file, 2 for end of file
+    PyObject* result = cpp_PyObject_CallMethod(file_, "seek", "(ii)", position, whence);
+    Py_XDECREF(result);
+    PY_RETURN_IF_ERROR(StatusCode::IOError);
+    return Status::OK();
+  }
+
+  Status Read(int64_t nbytes, PyObject** out) {
+    PyObject* result = cpp_PyObject_CallMethod(file_, "read", "(i)", nbytes);
+    PY_RETURN_IF_ERROR(StatusCode::IOError);
+    *out = result;
+    return Status::OK();
+  }
+
+  Status Write(const uint8_t* data, int64_t nbytes) {
+    PyObject* py_data =
+        PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), nbytes);
+    PY_RETURN_IF_ERROR(StatusCode::IOError);
+
+    PyObject* result = cpp_PyObject_CallMethod(file_, "write", "(O)", py_data);
+    Py_XDECREF(py_data);
+    Py_XDECREF(result);
+    PY_RETURN_IF_ERROR(StatusCode::IOError);
+    return Status::OK();
+  }
+
+  Status Tell(int64_t* position) {
+    PyObject* result = cpp_PyObject_CallMethod(file_, "tell", "()");
+    PY_RETURN_IF_ERROR(StatusCode::IOError);
+
+    *position = PyLong_AsLongLong(result);
+    Py_DECREF(result);
+
+    // PyLong_AsLongLong can raise OverflowError
+    PY_RETURN_IF_ERROR(StatusCode::IOError);
+
+    return Status::OK();
+  }
+
+  std::mutex& lock() { return lock_; }
+
+ private:
+  std::mutex lock_;
+  PyObject* file_;
+};
 
 // ----------------------------------------------------------------------
 // Seekable input stream
@@ -142,6 +154,20 @@ Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   return Status::OK();
 }
 
+Status PyReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                              uint8_t* out) {
+  std::lock_guard<std::mutex> guard(file_->lock());
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, bytes_read, out);
+}
+
+Status PyReadableFile::ReadAt(int64_t position, int64_t nbytes,
+                              std::shared_ptr<Buffer>* out) {
+  std::lock_guard<std::mutex> guard(file_->lock());
+  RETURN_NOT_OK(Seek(position));
+  return Read(nbytes, out);
+}
+
 Status PyReadableFile::GetSize(int64_t* size) {
   PyAcquireGIL lock;
 
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index eda7346741d0b..bf5db5313a9db 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -32,22 +32,7 @@ class MemoryPool;
 
 namespace py {
 
-// A common interface to a Python file-like object. Must acquire GIL before
-// calling any methods
-class ARROW_EXPORT PythonFile {
- public:
-  explicit PythonFile(PyObject* file);
-  ~PythonFile();
-
-  Status Close();
-  Status Seek(int64_t position, int whence);
-  Status Read(int64_t nbytes, PyObject** out);
-  Status Tell(int64_t* position);
-  Status Write(const uint8_t* data, int64_t nbytes);
-
- private:
-  PyObject* file_;
-};
+class ARROW_NO_EXPORT PythonFile;
 
 class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile {
  public:
@@ -59,6 +44,13 @@ class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile {
   Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
+  // Thread-safe version
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
+                uint8_t* out) override;
+
+  // Thread-safe version
+  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
+
   Status GetSize(int64_t* size) override;
 
   Status Seek(int64_t position) override;

From eaeb5d4820ce36484aaa78a15fe5bf957a53651e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 7 Oct 2017 15:49:10 -0400
Subject: [PATCH 1133/1644] ARROW-1250: [Python] Add pyarrow.types module with
 useful type checking functions

cc @TomAugspurger @cpcloud @xhochy for review

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1183 from wesm/ARROW-1250 and squashes the following commits:

e8ee7460 [Wes McKinney] Add is_dictionary
6fe40301 [Wes McKinney] Add test_types.py
1de24190 [Wes McKinney] Do not use isinstance
2f09c00c [Wes McKinney] Remove unused imports
3074efb5 [Wes McKinney] Add pyarrow.types module with useful type checking functions
---
 cpp/src/arrow/type.h                        |  77 +++++----
 python/doc/source/api.rst                   |  33 ++++
 python/pyarrow/__init__.py                  |  20 +--
 python/pyarrow/includes/libarrow.pxd        |   2 +
 python/pyarrow/lib.pyx                      |   2 +
 python/pyarrow/public-api.pxi               |   4 +
 python/pyarrow/tests/test_convert_pandas.py |   3 +-
 python/pyarrow/tests/test_types.py          | 134 +++++++++++++++
 python/pyarrow/types.pxi                    |  12 ++
 python/pyarrow/types.py                     | 180 ++++++++++++++++++++
 10 files changed, 423 insertions(+), 44 deletions(-)
 create mode 100644 python/pyarrow/tests/test_types.py
 create mode 100644 python/pyarrow/types.py

diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index c1da6e2704389..443828423e7dc 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -35,82 +35,101 @@
 
 namespace arrow {
 
-// Data types in this library are all *logical*. They can be expressed as
-// either a primitive physical type (bytes or bits of some fixed size), a
-// nested type consisting of other data types, or another data type (e.g. a
-// timestamp encoded as an int64)
+/// \brief Main data type enumeration
+///
+/// Data types in this library are all *logical*. They can be expressed as
+/// either a primitive physical type (bytes or bits of some fixed size), a
+/// nested type consisting of other data types, or another data type (e.g. a
+/// timestamp encoded as an int64)
 struct Type {
   enum type {
-    // A degenerate NULL type represented as 0 bytes/bits
+    /// A NULL type having no physical storage
     NA,
 
-    // A boolean value represented as 1 bit
+    /// Boolean as 1 bit, LSB bit-packed ordering
     BOOL,
 
-    // Little-endian integer types
+    /// Unsigned 8-bit little-endian integer
     UINT8,
+
+    /// Signed 8-bit little-endian integer
     INT8,
+
+    /// Unsigned 16-bit little-endian integer
     UINT16,
+
+    /// Signed 16-bit little-endian integer
     INT16,
+
+    /// Unsigned 32-bit little-endian integer
     UINT32,
+
+    /// Signed 32-bit little-endian integer
     INT32,
+
+    /// Unsigned 64-bit little-endian integer
     UINT64,
+
+    /// Signed 64-bit little-endian integer
     INT64,
 
-    // 2-byte floating point value
+    /// 2-byte floating point value
     HALF_FLOAT,
 
-    // 4-byte floating point value
+    /// 4-byte floating point value
     FLOAT,
 
-    // 8-byte floating point value
+    /// 8-byte floating point value
     DOUBLE,
 
-    // UTF8 variable-length string as List<Char>
+    /// UTF8 variable-length string as List<Char>
     STRING,
 
-    // Variable-length bytes (no guarantee of UTF8-ness)
+    /// Variable-length bytes (no guarantee of UTF8-ness)
     BINARY,
 
-    // Fixed-size binary. Each value occupies the same number of bytes
+    /// Fixed-size binary. Each value occupies the same number of bytes
     FIXED_SIZE_BINARY,
 
-    // int32_t days since the UNIX epoch
+    /// int32_t days since the UNIX epoch
     DATE32,
 
-    // int64_t milliseconds since the UNIX epoch
+    /// int64_t milliseconds since the UNIX epoch
     DATE64,
 
-    // Exact timestamp encoded with int64 since UNIX epoch
-    // Default unit millisecond
+    /// Exact timestamp encoded with int64 since UNIX epoch
+    /// Default unit millisecond
     TIMESTAMP,
 
-    // Time as signed 32-bit integer, representing either seconds or
-    // milliseconds since midnight
+    /// Time as signed 32-bit integer, representing either seconds or
+    /// milliseconds since midnight
     TIME32,
 
-    // Time as signed 64-bit integer, representing either microseconds or
-    // nanoseconds since midnight
+    /// Time as signed 64-bit integer, representing either microseconds or
+    /// nanoseconds since midnight
     TIME64,
 
-    // YEAR_MONTH or DAY_TIME interval in SQL style
+    /// YEAR_MONTH or DAY_TIME interval in SQL style
     INTERVAL,
 
-    // Precision- and scale-based decimal type. Storage type depends on the
-    // parameters.
+    /// Precision- and scale-based decimal type. Storage type depends on the
+    /// parameters.
     DECIMAL,
 
-    // A list of some logical data type
+    /// A list of some logical data type
     LIST,
 
-    // Struct of logical types
+    /// Struct of logical types
     STRUCT,
 
-    // Unions of logical types
+    /// Unions of logical types
     UNION,
 
-    // Dictionary aka Category type
-    DICTIONARY
+    /// Dictionary aka Category type
+    DICTIONARY,
+
+    /// Map, a repeated struct logical type
+    MAP
   };
 };
 
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index fa07d203df397..26ccb98edbbe3 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -58,6 +58,39 @@ Type and Schema Factory Functions
    schema
    from_numpy_dtype
 
+.. currentmodule:: pyarrow.types
+.. _api.types.checking:
+
+Type checking functions
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   is_boolean
+   is_integer
+   is_signed_integer
+   is_unsigned_integer
+   is_floating
+   is_decimal
+   is_list
+   is_struct
+   is_union
+   is_nested
+   is_temporal
+   is_timestamp
+   is_date
+   is_time
+   is_null
+   is_binary
+   is_unicode
+   is_string
+   is_fixed_size_binary
+   is_map
+   is_dictionary
+
+.. currentmodule:: pyarrow
+
 .. _api.value:
 
 Scalar Value Types
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index ac06948227454..f956347ac2947 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -36,8 +36,8 @@
                          time32, time64, timestamp, date32, date64,
                          float16, float32, float64,
                          binary, string, decimal,
-                         list_, struct, dictionary, field, type_for_alias,
-                         DataType, NAType,
+                         list_, struct, dictionary, field,
+                         type_for_alias, DataType, NAType,
                          Field,
                          Schema,
                          schema,
@@ -65,8 +65,7 @@
                          FloatValue, DoubleValue, ListValue,
                          BinaryValue, StringValue, FixedSizeBinaryValue,
                          DecimalValue,
-                         Date32Value, Date64Value, TimestampValue,
-                         TimestampType)
+                         Date32Value, Date64Value, TimestampValue)
 
 from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
                          FixedSizeBufferWriter,
@@ -117,6 +116,8 @@
 
 localfs = LocalFileSystem.get_instance()
 
+import pyarrow.types as types
+
 # Entry point for starting the plasma store
 
 def _plasma_store_entry_point():
@@ -138,13 +139,4 @@ def _plasma_store_entry_point():
 # ----------------------------------------------------------------------
 # Deprecations
 
-from pyarrow.util import _deprecate_class
-
-FixedSizeBufferOutputStream = (
-    _deprecate_class('FixedSizeBufferOutputStream',
-                     'FixedSizeBufferWriter',
-                     FixedSizeBufferWriter, '0.7.0'))
-
-# Backwards compatibility with pyarrow < 0.6.0
-HdfsClient = _deprecate_class('HdfsClient', 'pyarrow.hdfs.connect',
-                              hdfs.connect, '0.6.0')
+from pyarrow.util import _deprecate_class  # noqa
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index fc17d1c06ae0a..60aa4d6940a80 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -63,7 +63,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         _Type_LIST" arrow::Type::LIST"
         _Type_STRUCT" arrow::Type::STRUCT"
+        _Type_UNION" arrow::Type::UNION"
         _Type_DICTIONARY" arrow::Type::DICTIONARY"
+        _Type_MAP" arrow::Type::MAP"
 
     enum TimeUnit" arrow::TimeUnit::type":
         TimeUnit_SECOND" arrow::TimeUnit::SECOND"
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 4ea327ef9264c..6f4451e3f5a41 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -88,7 +88,9 @@ Type_STRING = _Type_STRING
 Type_FIXED_SIZE_BINARY = _Type_FIXED_SIZE_BINARY
 Type_LIST = _Type_LIST
 Type_STRUCT = _Type_STRUCT
+Type_UNION = _Type_UNION
 Type_DICTIONARY = _Type_DICTIONARY
+Type_MAP = _Type_MAP
 
 
 # Exception types
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 7e08f632e872e..9f1051228047a 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -69,6 +69,10 @@ cdef public api object pyarrow_wrap_data_type(
         out = DictionaryType()
     elif type.get().id() == _Type_LIST:
         out = ListType()
+    elif type.get().id() == _Type_STRUCT:
+        out = StructType()
+    elif type.get().id() == _Type_UNION:
+        out = StructType()
     elif type.get().id() == _Type_TIMESTAMP:
         out = TimestampType()
     elif type.get().id() == _Type_FIXED_SIZE_BINARY:
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 182f3afc7e63e..ae4ed5556b1e0 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -33,6 +33,7 @@
 
 from pyarrow.compat import u
 import pyarrow as pa
+import pyarrow.types as patypes
 
 from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
@@ -85,7 +86,7 @@ def _check_series_roundtrip(self, s, type_=None):
         arr = pa.array(s, from_pandas=True, type=type_)
 
         result = pd.Series(arr.to_pandas(), name=s.name)
-        if isinstance(arr.type, pa.TimestampType) and arr.type.tz is not None:
+        if patypes.is_timestamp(arr.type) and arr.type.tz is not None:
             result = (result.dt.tz_localize('utc')
                       .dt.tz_convert(arr.type.tz))
 
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
new file mode 100644
index 0000000000000..99cd4d0d5fe4e
--- /dev/null
+++ b/python/pyarrow/tests/test_types.py
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import pyarrow as pa
+import pyarrow.types as types
+
+
+def test_is_boolean():
+    assert types.is_boolean(pa.bool_())
+    assert not types.is_boolean(pa.int8())
+
+
+def test_is_integer():
+    signed_ints = [pa.int8(), pa.int16(), pa.int32(), pa.int64()]
+    unsigned_ints = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()]
+
+    for t in signed_ints + unsigned_ints:
+        assert types.is_integer(t)
+
+    for t in signed_ints:
+        assert types.is_signed_integer(t)
+        assert not types.is_unsigned_integer(t)
+
+    for t in unsigned_ints:
+        assert types.is_unsigned_integer(t)
+        assert not types.is_signed_integer(t)
+
+    assert not types.is_integer(pa.float32())
+    assert not types.is_signed_integer(pa.float32())
+
+
+def test_is_floating():
+    for t in [pa.float16(), pa.float32(), pa.float64()]:
+        assert types.is_floating(t)
+
+    assert not types.is_floating(pa.int32())
+
+
+def test_is_null():
+    assert types.is_null(pa.null())
+    assert not types.is_null(pa.list_(pa.int32()))
+
+
+def test_is_decimal():
+    assert types.is_decimal(pa.decimal(19, 4))
+    assert not types.is_decimal(pa.int32())
+
+
+def test_is_list():
+    assert types.is_list(pa.list_(pa.int32()))
+    assert not types.is_list(pa.int32())
+
+
+def test_is_dictionary():
+    assert types.is_dictionary(
+        pa.dictionary(pa.int32(),
+                      pa.array(['a', 'b', 'c'])))
+    assert not types.is_dictionary(pa.int32())
+
+
+def test_is_nested_or_struct():
+    struct_ex = pa.struct([pa.field('a', pa.int32()),
+                           pa.field('b', pa.int8()),
+                           pa.field('c', pa.string())])
+
+    assert types.is_struct(struct_ex)
+    assert not types.is_struct(pa.list_(pa.int32()))
+
+    assert types.is_nested(struct_ex)
+    assert types.is_nested(pa.list_(pa.int32()))
+    assert not types.is_nested(pa.int32())
+
+
+# TODO(wesm): Union types not yet implemented in pyarrow
+
+# def test_is_union():
+#     assert types.is_union(pa.union([pa.field('a', pa.int32()),
+#                                     pa.field('b', pa.int8()),
+#                                     pa.field('c', pa.string())]))
+#     assert not types.is_union(pa.list_(pa.int32()))
+
+# TODO(wesm): is_map, once implemented
+
+def test_is_binary_string():
+    assert types.is_binary(pa.binary())
+    assert not types.is_binary(pa.string())
+
+    assert types.is_string(pa.string())
+    assert types.is_unicode(pa.string())
+    assert not types.is_string(pa.binary())
+
+    assert types.is_fixed_size_binary(pa.binary(5))
+    assert not types.is_fixed_size_binary(pa.binary())
+
+
+def test_is_temporal_date_time_timestamp():
+    date_types = [pa.date32(), pa.date64()]
+    time_types = [pa.time32('s'), pa.time64('ns')]
+    timestamp_types = [pa.timestamp('ms')]
+
+    for case in date_types + time_types + timestamp_types:
+        assert types.is_temporal(case)
+
+    for case in date_types:
+        assert types.is_date(case)
+        assert not types.is_time(case)
+        assert not types.is_timestamp(case)
+
+    for case in time_types:
+        assert types.is_time(case)
+        assert not types.is_date(case)
+        assert not types.is_timestamp(case)
+
+    for case in timestamp_types:
+        assert types.is_timestamp(case)
+        assert not types.is_date(case)
+        assert not types.is_time(case)
+
+    assert not types.is_temporal(pa.int32())
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 316e09a6efda7..0bef1aa608a4f 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -123,6 +123,18 @@ cdef class ListType(DataType):
             return pyarrow_wrap_data_type(self.list_type.value_type())
 
 
+cdef class StructType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+
+
+cdef class UnionType(DataType):
+
+    cdef void init(self, const shared_ptr[CDataType]& type):
+        DataType.init(self, type)
+
+
 cdef class TimestampType(DataType):
 
     cdef void init(self, const shared_ptr[CDataType]& type):
diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py
new file mode 100644
index 0000000000000..440d7eb09daa9
--- /dev/null
+++ b/python/pyarrow/types.py
@@ -0,0 +1,180 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Tools for dealing with Arrow type metadata in Python
+
+import pyarrow.lib as lib
+
+
+_SIGNED_INTEGER_TYPES = {lib.Type_INT8, lib.Type_INT16, lib.Type_INT32,
+                         lib.Type_INT64}
+_UNSIGNED_INTEGER_TYPES = {lib.Type_UINT8, lib.Type_UINT16, lib.Type_UINT32,
+                           lib.Type_UINT64}
+_INTEGER_TYPES = _SIGNED_INTEGER_TYPES | _UNSIGNED_INTEGER_TYPES
+_FLOATING_TYPES = {lib.Type_HALF_FLOAT, lib.Type_FLOAT, lib.Type_DOUBLE}
+_DATE_TYPES = {lib.Type_DATE32, lib.Type_DATE64}
+_TIME_TYPES = {lib.Type_TIME32, lib.Type_TIME64}
+_TEMPORAL_TYPES = {lib.Type_TIMESTAMP} | _TIME_TYPES | _DATE_TYPES
+_NESTED_TYPES = {lib.Type_LIST, lib.Type_STRUCT, lib.Type_UNION, lib.Type_MAP}
+
+
+def is_boolean(t):
+    """
+    Return True if value is an instance of a boolean type
+    """
+    return t.id == lib.Type_BOOL
+
+
+def is_integer(t):
+    """
+    Return True if value is an instance of an integer type
+    """
+    return t.id in _INTEGER_TYPES
+
+
+def is_signed_integer(t):
+    """
+    Return True if value is an instance of a signed integer type
+    """
+    return t.id in _SIGNED_INTEGER_TYPES
+
+
+def is_unsigned_integer(t):
+    """
+    Return True if value is an instance of an unsigned integer type
+    """
+    return t.id in _UNSIGNED_INTEGER_TYPES
+
+
+def is_floating(t):
+    """
+    Return True if value is an instance of a floating point numeric type
+    """
+    return t.id in _FLOATING_TYPES
+
+
+def is_list(t):
+    """
+    Return True if value is an instance of a list type
+    """
+    return t.id == lib.Type_LIST
+
+
+def is_struct(t):
+    """
+    Return True if value is an instance of a struct type
+    """
+    return t.id == lib.Type_STRUCT
+
+
+def is_union(t):
+    """
+    Return True if value is an instance of a union type
+    """
+    return t.id == lib.Type_UNION
+
+
+def is_nested(t):
+    """
+    Return True if value is an instance of a nested type
+    """
+    return t.id in _NESTED_TYPES
+
+
+def is_temporal(t):
+    """
+    Return True if value is an instance of a temporal (date, time, timestamp)
+    type
+    """
+    return t.id in _TEMPORAL_TYPES
+
+
+def is_timestamp(t):
+    """
+    Return True if value is an instance of a timestamp type
+    """
+    return t.id == lib.Type_TIMESTAMP
+
+
+def is_time(t):
+    """
+    Return True if value is an instance of a time type
+    """
+    return t.id in _TIME_TYPES
+
+
+def is_null(t):
+    """
+    Return True if value is an instance of a null type
+    """
+    return t.id == lib.Type_NA
+
+
+def is_binary(t):
+    """
+    Return True if value is an instance of a variable-length binary type
+    """
+    return t.id == lib.Type_BINARY
+
+
+def is_unicode(t):
+    """
+    Alias for is_string
+    """
+    return is_string(t)
+
+
+def is_string(t):
+    """
+    Return True if value is an instance of string (utf8 unicode) type
+    """
+    return t.id == lib.Type_STRING
+
+
+def is_fixed_size_binary(t):
+    """
+    Return True if value is an instance of a fixed size binary type
+    """
+    return t.id == lib.Type_FIXED_SIZE_BINARY
+
+
+def is_date(t):
+    """
+    Return True if value is an instance of a date type
+    """
+    return t.id in _DATE_TYPES
+
+
+def is_map(t):
+    """
+    Return True if value is an instance of a map logical type
+    """
+    return t.id == lib.Type_MAP
+
+
+def is_decimal(t):
+    """
+    Return True if value is an instance of a decimal type
+    """
+    return t.id == lib.Type_DECIMAL
+
+
+def is_dictionary(t):
+    """
+    Return True if value is an instance of a dictionary-encoded type
+    """
+    return t.id == lib.Type_DICTIONARY

From e31c2e376fb5df1d9143377b76b9a0d3f79ebbd4 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sat, 7 Oct 2017 16:28:44 -0400
Subject: [PATCH 1134/1644] ARROW-1585/ARROW-1586: [PYTHON] serialize_pandas
 roundtrip loses columns name

This PR also adds support for MultiIndex columns.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1161 from cpcloud/ARROW-1586 and squashes the following commits:

f5f687d0 [Phillip Cloud] ARROW-1586: [PYTHON] serialize_pandas roundtrip loses columns name
---
 python/pyarrow/compat.py                    |   2 +
 python/pyarrow/pandas_compat.py             | 191 +++++++++++++++++---
 python/pyarrow/tests/test_convert_pandas.py |  68 +++++++
 python/pyarrow/tests/test_ipc.py            |   4 -
 python/pyarrow/tests/test_parquet.py        |  22 +++
 5 files changed, 256 insertions(+), 31 deletions(-)

diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index df5e4faadd443..f9c148b14e368 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -80,6 +80,7 @@ class Categorical(ClassPlaceholder):
     unicode_type = unicode
     lzip = zip
     zip = itertools.izip
+    zip_longest = itertools.izip_longest
 
     def dict_values(x):
         return x.values()
@@ -108,6 +109,7 @@ def lzip(*x):
         return list(zip(*x))
     long = int
     zip = zip
+    zip_longest = itertools.zip_longest
     def dict_values(x):
         return list(x.values())
     from decimal import Decimal
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index be48aeb442d38..141b33f119c8a 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -15,15 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import re
+import ast
+import collections
 import json
+import re
+
 import numpy as np
 import pandas as pd
 
 import six
 
 import pyarrow as pa
-from pyarrow.compat import PY2  # noqa
+from pyarrow.compat import PY2, zip_longest  # noqa
 
 
 INDEX_LEVEL_NAME_REGEX = re.compile(r'^__index_level_\d+__$')
@@ -89,6 +92,52 @@ def get_logical_type(arrow_type):
         raise NotImplementedError(str(arrow_type))
 
 
+_numpy_logical_type_map = {
+    np.bool_: 'bool',
+    np.int8: 'int8',
+    np.int16: 'int16',
+    np.int32: 'int32',
+    np.int64: 'int64',
+    np.uint8: 'uint8',
+    np.uint16: 'uint16',
+    np.uint32: 'uint32',
+    np.uint64: 'uint64',
+    np.float32: 'float32',
+    np.float64: 'float64',
+    'datetime64[D]': 'date',
+    np.str_: 'unicode',
+    np.bytes_: 'bytes',
+}
+
+
+def get_logical_type_from_numpy(pandas_collection):
+    try:
+        return _numpy_logical_type_map[pandas_collection.dtype.type]
+    except KeyError:
+        if hasattr(pandas_collection.dtype, 'tz'):
+            return 'datetimetz'
+        return infer_dtype(pandas_collection)
+
+
+def get_extension_dtype_info(column):
+    dtype = column.dtype
+    if str(dtype) == 'category':
+        cats = getattr(column, 'cat', column)
+        assert cats is not None
+        metadata = {
+            'num_categories': len(cats.categories),
+            'ordered': cats.ordered,
+        }
+        physical_dtype = 'object'
+    elif hasattr(dtype, 'tz'):
+        metadata = {'timezone': str(dtype.tz)}
+        physical_dtype = 'datetime64[ns]'
+    else:
+        metadata = None
+        physical_dtype = str(dtype)
+    return physical_dtype, metadata
+
+
 def get_column_metadata(column, name, arrow_type):
     """Construct the metadata for a given column
 
@@ -102,25 +151,15 @@ def get_column_metadata(column, name, arrow_type):
     -------
     dict
     """
-    dtype = column.dtype
     logical_type = get_logical_type(arrow_type)
 
-    if hasattr(dtype, 'categories'):
-        assert logical_type == 'categorical'
-        extra_metadata = {
-            'num_categories': len(column.cat.categories),
-            'ordered': column.cat.ordered,
-        }
-    elif hasattr(dtype, 'tz'):
-        assert logical_type == 'datetimetz'
-        extra_metadata = {'timezone': str(dtype.tz)}
-    elif logical_type == 'decimal':
+    string_dtype, extra_metadata = get_extension_dtype_info(column)
+    if logical_type == 'decimal':
         extra_metadata = {
             'precision': arrow_type.precision,
             'scale': arrow_type.scale,
         }
-    else:
-        extra_metadata = None
+        string_dtype = 'object'
 
     if not isinstance(name, six.string_types):
         raise TypeError(
@@ -132,7 +171,7 @@ def get_column_metadata(column, name, arrow_type):
     return {
         'name': name,
         'pandas_type': logical_type,
-        'numpy_type': str(dtype),
+        'numpy_type': string_dtype,
         'metadata': extra_metadata,
     }
 
@@ -188,21 +227,69 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
         index_column_metadata = [
             get_column_metadata(level, name=index_level_name(level, i),
                                 arrow_type=arrow_type)
-            for i, (level, arrow_type) in enumerate(zip(index_levels,
-                                                        index_types))
+            for i, (level, arrow_type) in enumerate(
+                zip(index_levels, index_types)
+            )
         ]
+
+        column_indexes = []
+
+        for level in getattr(df.columns, 'levels', [df.columns]):
+            string_dtype, extra_metadata = get_extension_dtype_info(level)
+            column_index = {
+                'name': level.name,
+                'pandas_type': get_logical_type_from_numpy(level),
+                'numpy_type': string_dtype,
+                'metadata': extra_metadata,
+            }
+            column_indexes.append(column_index)
     else:
-        index_column_names = index_column_metadata = []
+        index_column_names = index_column_metadata = column_indexes = []
 
     return {
         b'pandas': json.dumps({
             'index_columns': index_column_names,
+            'column_indexes': column_indexes,
             'columns': column_metadata + index_column_metadata,
             'pandas_version': pd.__version__
         }).encode('utf8')
     }
 
 
+def _column_name_to_strings(name):
+    """Convert a column name (or level) to either a string or a recursive
+    collection of strings.
+
+    Parameters
+    ----------
+    name : str or tuple
+
+    Returns
+    -------
+    value : str or tuple
+
+    Examples
+    --------
+    >>> name = 'foo'
+    >>> _column_name_to_strings(name)
+    'foo'
+    >>> name = ('foo', 'bar')
+    >>> _column_name_to_strings(name)
+    ('foo', 'bar')
+    >>> import pandas as pd
+    >>> name = (1, pd.Timestamp('2017-02-01 00:00:00'))
+    >>> _column_name_to_strings(name)
+    ('1', '2017-02-01 00:00:00')
+    """
+    if isinstance(name, six.string_types):
+        return name
+    elif isinstance(name, tuple):
+        return tuple(map(_column_name_to_strings, name))
+    elif isinstance(name, collections.Sequence):
+        raise TypeError("Unsupported type for MultiIndex level")
+    return str(name)
+
+
 def dataframe_to_arrays(df, schema, preserve_index):
     names = []
     arrays = []
@@ -217,7 +304,7 @@ def dataframe_to_arrays(df, schema, preserve_index):
     for name in df.columns:
         col = df[name]
         if not isinstance(name, six.string_types):
-            name = str(name)
+            name = str(_column_name_to_strings(name))
 
         if schema is not None:
             field = schema.field_by_name(name)
@@ -267,25 +354,30 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     import pyarrow.lib as lib
 
     index_columns = []
+    column_indexes = []
     index_arrays = []
     index_names = []
     schema = table.schema
     row_count = table.num_rows
     metadata = schema.metadata
 
-    if metadata is not None and b'pandas' in metadata:
+    has_pandas_metadata = metadata is not None and b'pandas' in metadata
+
+    if has_pandas_metadata:
         pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
         index_columns = pandas_metadata['index_columns']
+        column_indexes = pandas_metadata.get('column_indexes', [])
         table = _add_any_metadata(table, pandas_metadata)
 
     block_table = table
 
+    # Build up a list of index columns and names while removing those columns
+    # from the original table
     for name in index_columns:
         i = schema.get_field_index(name)
         if i != -1:
             col = table.column(i)
-            index_name = (None if is_unnamed_index_level(name)
-                          else name)
+            index_name = None if is_unnamed_index_level(name) else name
             col_pandas = col.to_pandas()
             values = col_pandas.values
             if not values.flags.writeable:
@@ -299,8 +391,12 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
                 block_table.schema.get_field_index(name)
             )
 
+    # Convert an arrow table to Block from the internal pandas API
     result = lib.table_to_blocks(options, block_table, nthreads, memory_pool)
 
+    # Construct the individual blocks converting dictionary types to pandas
+    # categorical types and Timestamps-with-timezones types to the proper
+    # pandas Blocks
     blocks = []
     for item in result:
         block_arr = item['block']
@@ -321,6 +417,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
             block = _int.make_block(block_arr, placement=placement)
         blocks.append(block)
 
+    # Construct the row index
     if len(index_arrays) > 1:
         index = pd.MultiIndex.from_arrays(index_arrays, names=index_names)
     elif len(index_arrays) == 1:
@@ -328,11 +425,51 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     else:
         index = pd.RangeIndex(row_count)
 
-    axes = [
-        [column.name for column in block_table.itercolumns()],
-        index
-    ]
+    column_strings = [x.name for x in block_table.itercolumns()]
+
+    # If we're passed multiple column indexes then evaluate with
+    # ast.literal_eval, since the column index values show up as a list of
+    # tuples
+    to_pair = ast.literal_eval if len(column_indexes) > 1 else lambda x: (x,)
+
+    # Create the column index
 
+    # Construct the base index
+    if not column_strings:
+        columns = pd.Index(column_strings)
+    else:
+        columns = pd.MultiIndex.from_tuples(
+            list(map(to_pair, column_strings)),
+            names=[col_index['name'] for col_index in column_indexes] or None,
+        )
+
+    # if we're reconstructing the index
+    if has_pandas_metadata:
+
+        # Get levels and labels, and provide sane defaults if the index has a
+        # single level to avoid if/else spaghetti.
+        levels = getattr(columns, 'levels', None) or [columns]
+        labels = getattr(columns, 'labels', None) or [
+            pd.RangeIndex(len(level)) for level in levels
+        ]
+
+        # Convert each level to the dtype provided in the metadata
+        levels_dtypes = [
+            (level, col_index.get('numpy_type', level.dtype))
+            for level, col_index in zip_longest(
+                levels, column_indexes, fillvalue={}
+            )
+        ]
+        new_levels = [
+            level if level.dtype == dtype else level.astype(dtype)
+            for level, dtype in levels_dtypes
+        ]
+        columns = pd.MultiIndex(
+            levels=new_levels,
+            labels=labels,
+            names=columns.names
+        )
+    axes = [columns, index]
     return _int.BlockManager(blocks, axes)
 
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index ae4ed5556b1e0..d42aa0e0aa4d2 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -125,6 +125,74 @@ def test_non_string_columns(self):
         table = pa.Table.from_pandas(df)
         assert table.column(0).name == '0'
 
+    def test_column_index_names_are_preserved(self):
+        df = pd.DataFrame({'data': [1, 2, 3]})
+        df.columns.names = ['a']
+        self._check_pandas_roundtrip(df, check_index=True)
+
+    def test_multiindex_columns(self):
+        columns = pd.MultiIndex.from_arrays([
+            ['one', 'two'], ['X', 'Y']
+        ])
+        df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
+        self._check_pandas_roundtrip(df, check_index=True)
+
+    def test_multiindex_columns_with_dtypes(self):
+        columns = pd.MultiIndex.from_arrays(
+            [
+                ['one', 'two'],
+                pd.DatetimeIndex(['2017-08-01', '2017-08-02']),
+            ],
+            names=['level_1', 'level_2'],
+        )
+        df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
+        self._check_pandas_roundtrip(df, check_index=True)
+
+    def test_integer_index_column(self):
+        df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')])
+        self._check_pandas_roundtrip(df, check_index=True)
+
+    def test_categorical_column_index(self):
+        # I *really* hope no one uses category dtypes for single level column
+        # indexes
+        df = pd.DataFrame(
+            [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],
+            columns=pd.Index(list('def'), dtype='category')
+        )
+        t = pa.Table.from_pandas(df, preserve_index=True)
+        raw_metadata = t.schema.metadata
+        js = json.loads(raw_metadata[b'pandas'].decode('utf8'))
+
+        column_indexes, = js['column_indexes']
+        assert column_indexes['name'] is None
+        assert column_indexes['pandas_type'] == 'categorical'
+        assert column_indexes['numpy_type'] == 'object'
+
+        md = column_indexes['metadata']
+        assert md['num_categories'] == 3
+        assert md['ordered'] is False
+
+    def test_datetimetz_column_index(self):
+        # I *really* hope no one uses category dtypes for single level column
+        # indexes
+        df = pd.DataFrame(
+            [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],
+            columns=pd.date_range(
+                start='2017-01-01', periods=3, tz='America/New_York'
+            )
+        )
+        t = pa.Table.from_pandas(df, preserve_index=True)
+        raw_metadata = t.schema.metadata
+        js = json.loads(raw_metadata[b'pandas'].decode('utf8'))
+
+        column_indexes, = js['column_indexes']
+        assert column_indexes['name'] is None
+        assert column_indexes['pandas_type'] == 'datetimetz'
+        assert column_indexes['numpy_type'] == 'datetime64[ns]'
+
+        md = column_indexes['metadata']
+        assert md['timezone'] == 'America/New_York'
+
     def test_float_no_nulls(self):
         data = {}
         fields = []
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 7a771654e4d4d..6802c43de4031 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -409,10 +409,6 @@ def test_serialize_pandas_empty_dataframe():
     _check_serialize_pandas_round_trip(df)
 
 
-@pytest.mark.xfail(
-    raises=AssertionError,
-    reason='Non string columns are not supported',
-)
 def test_pandas_serialize_round_trip_not_string_columns():
     df = pd.DataFrame(list(zip([1.5, 1.6, 1.7], 'abc')))
     buf = pa.serialize_pandas(df)
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index d51b85d8ee4ed..deb4b3f35a677 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -161,6 +161,28 @@ def test_pandas_parquet_custom_metadata(tmpdir):
     assert js['index_columns'] == ['__index_level_0__']
 
 
+@parquet
+def test_pandas_parquet_column_multiindex(tmpdir):
+    import pyarrow.parquet as pq
+
+    df = alltypes_sample(size=10)
+    df.columns = pd.MultiIndex.from_tuples(
+        list(zip(df.columns, df.columns[::-1])),
+        names=['level_1', 'level_2']
+    )
+
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = pa.Table.from_pandas(df)
+    assert b'pandas' in arrow_table.schema.metadata
+
+    _write_table(arrow_table, filename.strpath, version="2.0",
+                 coerce_timestamps='ms')
+
+    table_read = pq.read_pandas(filename.strpath)
+    df_read = table_read.to_pandas()
+    tm.assert_frame_equal(df, df_read)
+
+
 @parquet
 def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tmpdir):
     import pyarrow.parquet as pq

From 208e79812b5d98f9cd31f4f9ed9e74e6f76f24fd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 8 Oct 2017 11:45:06 -0400
Subject: [PATCH 1135/1644] ARROW-1594: [Python] Multithreaded conversions to
 Arrow in from_pandas

This results in nice speedups when column conversions do not require GIL to be held:

```python
In [5]: import numpy as np

In [6]: import pandas as pd

In [7]: import pyarrow as pa

In [8]: NROWS = 1000000

In [9]: NCOLS = 50

In [10]: arr = np.random.randn(NCOLS, NROWS).T

In [11]: arr[::5] = np.nan

In [12]: df = pd.DataFrame(arr)

In [13]: %timeit rb = pa.RecordBatch.from_pandas(df, nthreads=1)
10 loops, best of 3: 179 ms per loop

In [14]: %timeit rb = pa.RecordBatch.from_pandas(df, nthreads=4)
10 loops, best of 3: 59.7 ms per loop
```

This introduces a dependency on the `futures` Python 2.7 backport of concurrent.futures (PSF license)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1186 from wesm/multithreaded-from-pandas and squashes the following commits:

a3072f0e [Wes McKinney] Only install futures on py2
c30e4735 [Wes McKinney] Add heuristic to use threadpool conversion only if nrows > ncols * 100
5a692085 [Wes McKinney] Only install concurrent.futures backport on py2, test serialize_pandas with nthreads
0afab342 [Wes McKinney] Add nthreads argument to serialize_pandas, make default for serialize/deserialize consistent
15841d13 [Wes McKinney] Default to cpu_count() for nthreads in from_pandas to conform with to_pandas default
6a58c038 [Wes McKinney] Add nthreads argument to RecordBatch/Table.from_pandas. Use concurrent.futures for parallel processing
---
 ci/travis_script_python.sh                  |  4 ++
 python/pyarrow/ipc.py                       | 13 ++++---
 python/pyarrow/pandas_compat.py             | 42 ++++++++++++++++-----
 python/pyarrow/table.pxi                    | 17 +++++++--
 python/pyarrow/tests/test_convert_pandas.py | 11 ++++--
 python/pyarrow/tests/test_ipc.py            |  2 +-
 python/setup.py                             | 12 ++++--
 7 files changed, 76 insertions(+), 25 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 694154335f179..97bde1a5ede95 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -68,6 +68,10 @@ popd
 # Other stuff pip install
 pushd $ARROW_PYTHON_DIR
 
+if [ "$PYTHON_VERSION" == "2.7" ]; then
+  pip install futures
+fi
+
 pip install -r requirements.txt
 python setup.py build_ext --with-parquet --with-plasma \
        install --single-version-externally-managed --record=record.text
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index 6eb4979f1a2b7..8cb6cdd335915 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -142,19 +142,21 @@ def open_file(source, footer_offset=None):
     return RecordBatchFileReader(source, footer_offset=footer_offset)
 
 
-def serialize_pandas(df):
+def serialize_pandas(df, nthreads=None):
     """Serialize a pandas DataFrame into a buffer protocol compatible object.
 
     Parameters
     ----------
     df : pandas.DataFrame
+    nthreads : int, default None
+        Number of threads to use for conversion to Arrow, default all CPUs
 
     Returns
     -------
     buf : buffer
         An object compatible with the buffer protocol
     """
-    batch = pa.RecordBatch.from_pandas(df)
+    batch = pa.RecordBatch.from_pandas(df, nthreads=nthreads)
     sink = pa.BufferOutputStream()
     writer = pa.RecordBatchStreamWriter(sink, batch.schema)
     writer.write_batch(batch)
@@ -162,15 +164,16 @@ def serialize_pandas(df):
     return sink.get_result()
 
 
-def deserialize_pandas(buf, nthreads=1):
+def deserialize_pandas(buf, nthreads=None):
     """Deserialize a buffer protocol compatible object into a pandas DataFrame.
 
     Parameters
     ----------
     buf : buffer
         An object compatible with the buffer protocol
-    nthreads : int, optional
-        The number of threads to use to convert the buffer to a DataFrame.
+    nthreads : int, defualt None
+        The number of threads to use to convert the buffer to a DataFrame,
+        default all CPUs
 
     Returns
     -------
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 141b33f119c8a..a071e5664796a 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -290,17 +290,17 @@ def _column_name_to_strings(name):
     return str(name)
 
 
-def dataframe_to_arrays(df, schema, preserve_index):
+def dataframe_to_arrays(df, schema, preserve_index, nthreads=1):
     names = []
-    arrays = []
     index_columns = []
-    types = []
     type = None
 
     if preserve_index:
         n = len(getattr(df.index, 'levels', [df.index]))
         index_columns.extend(df.index.get_level_values(i) for i in range(n))
 
+    columns_to_convert = []
+    convert_types = []
     for name in df.columns:
         col = df[name]
         if not isinstance(name, six.string_types):
@@ -310,16 +310,40 @@ def dataframe_to_arrays(df, schema, preserve_index):
             field = schema.field_by_name(name)
             type = getattr(field, "type", None)
 
-        array = pa.array(col, from_pandas=True, type=type)
-        arrays.append(array)
+        columns_to_convert.append(col)
+        convert_types.append(type)
         names.append(name)
-        types.append(array.type)
 
     for i, column in enumerate(index_columns):
-        array = pa.array(column)
-        arrays.append(array)
+        columns_to_convert.append(column)
+        convert_types.append(None)
         names.append(index_level_name(column, i))
-        types.append(array.type)
+
+    # NOTE(wesm): If nthreads=None, then we use a heuristic to decide whether
+    # using a thread pool is worth it. Currently the heuristic is whether the
+    # nrows > 100 * ncols.
+    if nthreads is None:
+        nrows, ncols = len(df), len(df.columns)
+        if nrows > ncols * 100:
+            nthreads = pa.cpu_count()
+        else:
+            nthreads = 1
+
+    def convert_column(col, ty):
+        return pa.array(col, from_pandas=True, type=ty)
+
+    if nthreads == 1:
+        arrays = [convert_column(c, t)
+                  for c, t in zip(columns_to_convert,
+                                  convert_types)]
+    else:
+        from concurrent import futures
+        with futures.ThreadPoolExecutor(nthreads) as executor:
+            arrays = list(executor.map(convert_column,
+                                       columns_to_convert,
+                                       convert_types))
+
+    types = [x.type for x in arrays]
 
     metadata = construct_metadata(
         df, names, index_columns, preserve_index, types
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index e5422a5beca3b..dd42cf231e7e2 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -556,7 +556,8 @@ cdef class RecordBatch:
         return Table.from_batches([self]).to_pandas(nthreads=nthreads)
 
     @classmethod
-    def from_pandas(cls, df, Schema schema=None, bint preserve_index=True):
+    def from_pandas(cls, df, Schema schema=None, bint preserve_index=True,
+                    nthreads=None):
         """
         Convert pandas.DataFrame to an Arrow RecordBatch
 
@@ -569,13 +570,16 @@ cdef class RecordBatch:
         preserve_index : bool, optional
             Whether to store the index as an additional column in the resulting
             ``RecordBatch``.
+        nthreads : int, default None (may use up to system CPU count threads)
+            If greater than 1, convert columns to Arrow in parallel using
+            indicated number of threads
 
         Returns
         -------
         pyarrow.RecordBatch
         """
         names, arrays, metadata = pdcompat.dataframe_to_arrays(
-            df, schema, preserve_index
+            df, schema, preserve_index, nthreads=nthreads
         )
         return cls.from_arrays(arrays, names, metadata)
 
@@ -714,7 +718,8 @@ cdef class Table:
         return result
 
     @classmethod
-    def from_pandas(cls, df, Schema schema=None, bint preserve_index=True):
+    def from_pandas(cls, df, Schema schema=None, bint preserve_index=True,
+                    nthreads=None):
         """
         Convert pandas.DataFrame to an Arrow Table
 
@@ -727,6 +732,9 @@ cdef class Table:
         preserve_index : bool, optional
             Whether to store the index as an additional column in the resulting
             ``Table``.
+        nthreads : int, default None (may use up to system CPU count threads)
+            If greater than 1, convert columns to Arrow in parallel using
+            indicated number of threads
 
         Returns
         -------
@@ -747,7 +755,8 @@ cdef class Table:
         names, arrays, metadata = pdcompat.dataframe_to_arrays(
             df,
             schema=schema,
-            preserve_index=preserve_index
+            preserve_index=preserve_index,
+            nthreads=nthreads
         )
         return cls.from_arrays(arrays, names=names, metadata=metadata)
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index d42aa0e0aa4d2..41ad20102112a 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -72,9 +72,13 @@ def tearDown(self):
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
                                 expected_schema=None,
                                 check_dtype=True, schema=None,
-                                check_index=False):
-        table = pa.Table.from_pandas(df,
-                                     schema=schema, preserve_index=check_index)
+                                check_index=False,
+                                as_batch=False):
+        klass = pa.RecordBatch if as_batch else pa.Table
+        table = klass.from_pandas(df, schema=schema,
+                                  preserve_index=check_index,
+                                  nthreads=nthreads)
+
         result = table.to_pandas(nthreads=nthreads)
         if expected_schema:
             assert table.schema.equals(expected_schema)
@@ -663,6 +667,7 @@ def test_nested_lists_all_none(self):
     def test_threaded_conversion(self):
         df = _alltypes_example()
         self._check_pandas_roundtrip(df, nthreads=2)
+        self._check_pandas_roundtrip(df, nthreads=2, as_batch=True)
 
     def test_category(self):
         repeats = 5
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 6802c43de4031..0d5b67395a11d 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -365,7 +365,7 @@ def test_get_record_batch_size():
 
 
 def _check_serialize_pandas_round_trip(df, nthreads=1):
-    buf = pa.serialize_pandas(df)
+    buf = pa.serialize_pandas(df, nthreads=nthreads)
     result = pa.deserialize_pandas(buf, nthreads=nthreads)
     assert_frame_equal(result, df)
 
diff --git a/python/setup.py b/python/setup.py
index 8966fb873a630..edcf397f3fb51 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -381,6 +381,13 @@ class BinaryDistribution(Distribution):
     def has_ext_modules(foo):
         return True
 
+
+install_requires = ['numpy >= 1.10', 'six >= 1.0.0']
+
+if sys.version_info.major == 2:
+    install_requires.append('futures')
+
+
 setup(
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
@@ -390,19 +397,18 @@ def has_ext_modules(foo):
     distclass=BinaryDistribution,
     # Dummy extension to trigger build_ext
     ext_modules=[Extension('__dummy__', sources=[])],
-
     cmdclass={
         'clean': clean,
         'build_ext': build_ext
     },
-    entry_points = {
+    entry_points={
         'console_scripts': [
             'plasma_store = pyarrow:_plasma_store_entry_point'
         ]
     },
     use_scm_version={"root": "..", "relative_to": __file__},
     setup_requires=['setuptools_scm', 'cython >= 0.23'],
-    install_requires=['numpy >= 1.10', 'six >= 1.0.0'],
+    install_requires=install_requires,
     tests_require=['pytest'],
     description="Python library for Apache Arrow",
     long_description=long_description,

From 33d446dd4d69cfc6cbbf91cd7af51c6c7d6eee8f Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 8 Oct 2017 13:04:02 -0400
Subject: [PATCH 1136/1644] ARROW-1656: [C++] Endianness Macro is Incorrect on
 Windows And Mac

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1184 from cpcloud/ARROW-1656 and squashes the following commits:

04a92621 [Phillip Cloud] ARROW-1656: [C++] Endianness Macro is Incorrect on Windows And Mac
---
 cpp/src/arrow/util/bit-util.h | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index dd1024596974e..2509de21ffb35 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -18,13 +18,29 @@
 #ifndef ARROW_UTIL_BIT_UTIL_H
 #define ARROW_UTIL_BIT_UTIL_H
 
-#if defined(__APPLE__)
+#ifdef _WIN32
+#define ARROW_LITTLE_ENDIAN 1
+#else
+#ifdef __APPLE__
 #include <machine/endian.h>
-#elif defined(_WIN32)
-#define __LITTLE_ENDIAN 1
 #else
 #include <endian.h>
 #endif
+#
+#ifndef __BYTE_ORDER__
+#error "__BYTE_ORDER__ not defined"
+#endif
+#
+#ifndef __ORDER_LITTLE_ENDIAN__
+#error "__ORDER_LITTLE_ENDIAN__ not defined"
+#endif
+#
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define ARROW_LITTLE_ENDIAN 1
+#else
+#define ARROW_LITTLE_ENDIAN 0
+#endif
+#endif
 
 #if defined(_MSC_VER)
 #define ARROW_BYTE_SWAP64 _byteswap_uint64
@@ -324,7 +340,7 @@ static inline void ByteSwap(void* dst, const void* src, int len) {
 
 /// Converts to big endian format (if not already in big endian) from the
 /// machine's native endian format.
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+#if ARROW_LITTLE_ENDIAN
 static inline int64_t ToBigEndian(int64_t value) { return ByteSwap(value); }
 static inline uint64_t ToBigEndian(uint64_t value) { return ByteSwap(value); }
 static inline int32_t ToBigEndian(int32_t value) { return ByteSwap(value); }
@@ -341,7 +357,7 @@ static inline uint16_t ToBigEndian(uint16_t val) { return val; }
 #endif
 
 /// Converts from big endian format to the machine's native endian format.
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+#if ARROW_LITTLE_ENDIAN
 static inline int64_t FromBigEndian(int64_t value) { return ByteSwap(value); }
 static inline uint64_t FromBigEndian(uint64_t value) { return ByteSwap(value); }
 static inline int32_t FromBigEndian(int32_t value) { return ByteSwap(value); }

From 81a0e67390b35c5696e6faa9605c50230c6cec4c Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 8 Oct 2017 14:32:40 -0400
Subject: [PATCH 1137/1644] ARROW-1657: [C++] Multithreaded Read Test Failing
 on Arch Linux

This makes `RandomAccessFile` thread-safe (and therefore `BufferReader` as well) using pimpl. JIRA is down for maintenance at the moment.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1187 from cpcloud/thread-safe-random-access-file and squashes the following commits:

60b90348 [Phillip Cloud] Fix GLib bindings
e0bcf22a [Phillip Cloud] ARROW-1657: [C++] Multithreaded Read Test Failing on Arch Linux
---
 c_glib/arrow-glib/input-stream.cpp | 11 +++++++
 cpp/src/arrow/io/interfaces.cc     | 13 +++++++--
 cpp/src/arrow/io/interfaces.h      | 47 +++++++++++++++++++++---------
 cpp/src/arrow/io/memory.cc         | 14 +++------
 cpp/src/arrow/io/memory.h          |  6 ++--
 cpp/src/arrow/status.h             |  8 ++---
 6 files changed, 65 insertions(+), 34 deletions(-)

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 11c63f265cde4..d628baeeeae5b 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -436,6 +436,17 @@ namespace garrow {
       }
     }
 
+    arrow::Status ReadAt(int64_t position, int64_t n_bytes,
+			 int64_t *n_read_bytes, uint8_t* out) {
+	return arrow::io::RandomAccessFile::ReadAt(
+	    position, n_bytes, n_read_bytes, out);
+    }
+
+    arrow::Status ReadAt(int64_t position, int64_t n_bytes,
+			 std::shared_ptr<arrow::Buffer>* out) {
+	return arrow::io::RandomAccessFile::ReadAt(position, n_bytes, out);
+    }
+
     arrow::Status Read(int64_t n_bytes,
                        std::shared_ptr<arrow::Buffer> *out) override {
       arrow::MemoryPool *pool = arrow::default_memory_pool();
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 499da6f744386..582cc2026d86d 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -26,18 +26,27 @@
 namespace arrow {
 namespace io {
 
-FileInterface::~FileInterface() {}
+FileInterface::~FileInterface() = default;
 
-RandomAccessFile::RandomAccessFile() { set_mode(FileMode::READ); }
+struct RandomAccessFile::RandomAccessFileImpl {
+  std::mutex lock_;
+};
+
+RandomAccessFile::~RandomAccessFile() = default;
+
+RandomAccessFile::RandomAccessFile()
+    : impl_(new RandomAccessFile::RandomAccessFileImpl()) {}
 
 Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
                                 uint8_t* out) {
+  std::lock_guard<std::mutex> lock(impl_->lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
 }
 
 Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes,
                                 std::shared_ptr<Buffer>* out) {
+  std::lock_guard<std::mutex> lock(impl_->lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, out);
 }
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index eee4e956ee589..82af875e7c07e 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -45,14 +45,11 @@ struct ARROW_EXPORT FileStatistics {
   /// Size of file, -1 if finding length is unsupported
   int64_t size;
   ObjectType::type kind;
-
-  FileStatistics() {}
-  FileStatistics(int64_t size, ObjectType::type kind) : size(size), kind(kind) {}
 };
 
 class ARROW_EXPORT FileSystem {
  public:
-  virtual ~FileSystem() {}
+  virtual ~FileSystem() = default;
 
   virtual Status MakeDirectory(const std::string& path) = 0;
 
@@ -75,7 +72,7 @@ class ARROW_EXPORT FileInterface {
   FileMode::type mode() const { return mode_; }
 
  protected:
-  FileInterface() {}
+  FileInterface() : mode_(FileMode::READ) {}
   FileMode::type mode_;
   void set_mode(FileMode::type mode) { mode_ = mode; }
 
@@ -113,32 +110,54 @@ class ARROW_EXPORT Readable {
 
 class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writeable {
  protected:
-  OutputStream() {}
+  OutputStream() = default;
 };
 
 class ARROW_EXPORT InputStream : virtual public FileInterface, public Readable {
  protected:
-  InputStream() {}
+  InputStream() = default;
 };
 
 class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
  public:
+  /// Necessary because we hold a std::unique_ptr
+  ~RandomAccessFile() override;
+
   virtual Status GetSize(int64_t* size) = 0;
 
   virtual bool supports_zero_copy() const = 0;
 
-  /// Read at position, provide default implementations using Read(...), but can
-  /// be overridden
+  /// \brief Read nbytes at position, provide default implementations using Read(...), but
+  /// can be overridden. Default implementation is thread-safe.
   ///
-  /// Default implementation is not thread-safe
+  /// \note Child classes must explicitly call this implementation or provide their own.
+  ///
+  /// \param[in] position Where to read bytes from
+  /// \param[in] nbytes The number of bytes to read
+  /// \param[out] bytes_read The number of bytes read
+  /// \param[out] out The buffer to read bytes into
+  /// \return Status
   virtual Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                        uint8_t* out);
+                        uint8_t* out) = 0;
 
-  /// Default implementation is not thread-safe
-  virtual Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out);
+  /// \brief Read nbytes at position, provide default implementations using Read(...), but
+  /// can be overridden. Default implementation is thread-safe.
+  ///
+  /// \note Child classes must explicitly call this implementation or provide their own.
+  ///
+  /// \param[in] position Where to read bytes from
+  /// \param[in] nbytes The number of bytes to read
+  /// \param[out] out The buffer to read bytes into. The number of bytes read can be
+  /// retrieved by calling Buffer::size().
+  virtual Status ReadAt(int64_t position, int64_t nbytes,
+                        std::shared_ptr<Buffer>* out) = 0;
 
  protected:
   RandomAccessFile();
+
+ private:
+  struct ARROW_NO_EXPORT RandomAccessFileImpl;
+  std::unique_ptr<RandomAccessFileImpl> impl_;
 };
 
 class ARROW_EXPORT WriteableFile : public OutputStream, public Seekable {
@@ -146,7 +165,7 @@ class ARROW_EXPORT WriteableFile : public OutputStream, public Seekable {
   virtual Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) = 0;
 
  protected:
-  WriteableFile() { set_mode(FileMode::READ); }
+  WriteableFile() = default;
 };
 
 class ARROW_EXPORT ReadWriteFileInterface : public RandomAccessFile,
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 370d3e9566a11..d9c84b495d21a 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -144,8 +144,6 @@ class FixedSizeBufferWriter::FixedSizeBufferWriterImpl {
     position_ = 0;
   }
 
-  ~FixedSizeBufferWriterImpl() {}
-
   Status Close() {
     // No-op
     return Status::OK();
@@ -199,11 +197,11 @@ class FixedSizeBufferWriter::FixedSizeBufferWriterImpl {
   int64_t memcopy_threshold_;
 };
 
-FixedSizeBufferWriter::~FixedSizeBufferWriter() {}
-
 FixedSizeBufferWriter::FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer)
     : impl_(new FixedSizeBufferWriterImpl(buffer)) {}
 
+FixedSizeBufferWriter::~FixedSizeBufferWriter() = default;
+
 Status FixedSizeBufferWriter::Close() { return impl_->Close(); }
 
 Status FixedSizeBufferWriter::Seek(int64_t position) { return impl_->Seek(position); }
@@ -242,8 +240,6 @@ BufferReader::BufferReader(const std::shared_ptr<Buffer>& buffer)
 BufferReader::BufferReader(const uint8_t* data, int64_t size)
     : buffer_(nullptr), data_(data), size_(size), position_(0) {}
 
-BufferReader::~BufferReader() {}
-
 Status BufferReader::Close() {
   // no-op
   return Status::OK();
@@ -278,14 +274,12 @@ Status BufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 
 Status BufferReader::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
                             uint8_t* out) {
-  RETURN_NOT_OK(Seek(position));
-  return Read(nbytes, bytes_read, out);
+  return RandomAccessFile::ReadAt(position, nbytes, bytes_read, out);
 }
 
 Status BufferReader::ReadAt(int64_t position, int64_t nbytes,
                             std::shared_ptr<Buffer>* out) {
-  RETURN_NOT_OK(Seek(position));
-  return Read(nbytes, out);
+  return RandomAccessFile::ReadAt(position, nbytes, out);
 }
 
 Status BufferReader::GetSize(int64_t* size) {
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 978c198c2dd93..3aec91f7237e1 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -43,7 +43,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   static Status Create(int64_t initial_capacity, MemoryPool* pool,
                        std::shared_ptr<BufferOutputStream>* out);
 
-  ~BufferOutputStream();
+  ~BufferOutputStream() override;
 
   // Implement the OutputStream interface
   Status Close() override;
@@ -85,7 +85,7 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
  public:
   /// Input buffer must be mutable, will abort if not
   explicit FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer);
-  ~FixedSizeBufferWriter();
+  ~FixedSizeBufferWriter() override;
 
   Status Close() override;
   Status Seek(int64_t position) override;
@@ -108,7 +108,6 @@ class ARROW_EXPORT BufferReader : public RandomAccessFile {
  public:
   explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
   BufferReader(const uint8_t* data, int64_t size);
-  virtual ~BufferReader();
 
   Status Close() override;
   Status Tell(int64_t* position) const override;
@@ -116,7 +115,6 @@ class ARROW_EXPORT BufferReader : public RandomAccessFile {
 
   // Zero copy read
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
-
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
                 uint8_t* out) override;
 
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 80285dac2b6f0..84f55e415cd34 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -33,7 +33,7 @@
     if (ARROW_PREDICT_FALSE(!_s.ok())) { \
       return _s;                         \
     }                                    \
-  } while (0)
+  } while (false)
 
 // If 'to_call' returns a bad status, CHECK immediately with a logged message
 // of 'msg' followed by the status.
@@ -41,7 +41,7 @@
   do {                                                      \
     ::arrow::Status _s = (to_call);                         \
     ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString(); \
-  } while (0)
+  } while (false)
 
 // If the status is bad, CHECK immediately, appending the status to the
 // logged message.
@@ -69,7 +69,7 @@ namespace arrow {
     if (ARROW_PREDICT_FALSE(!_s.ok())) { \
       return _s;                         \
     }                                    \
-  } while (0)
+  } while (false)
 
 #endif  // ARROW_EXTRA_ERROR_CONTEXT
 
@@ -80,7 +80,7 @@ namespace arrow {
       else_;                         \
       return _s;                     \
     }                                \
-  } while (0)
+  } while (false)
 
 enum class StatusCode : char {
   OK = 0,

From a0555c04dd5c43230a1c50d0d0a94e06d8ad9ff0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 8 Oct 2017 21:09:03 -0400
Subject: [PATCH 1138/1644] ARROW-1535: [Python] Enable sdist tarballs to be
 installed

The tarballs produced by `setup.py sdist` have always been broken. This creates a tarball that can be installed assuming that the Arrow/Parquet C++ libraries are installed already on the target system.

I wanted to see if I could bundle the `cpp/cmake_modules` files without using a symlink but I couldn't figure it out; it might require subclassing the sdist cmdclass to make this work. I'm not sure having the symlink is so bad

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1188 from wesm/ARROW-1535 and squashes the following commits:

dcc2e47d [Wes McKinney] Fix RAT excludes given changes to cmake files
310af1b7 [Wes McKinney] Move all cmake files to cpp/cmake_modules, fix up sdist tarball so it can be built/installed
---
 {python => cpp}/cmake_modules/FindArrow.cmake   |  0
 {python => cpp}/cmake_modules/FindCython.cmake  |  0
 {python => cpp}/cmake_modules/FindParquet.cmake |  0
 {python => cpp}/cmake_modules/FindPlasma.cmake  |  0
 {python => cpp}/cmake_modules/UseCython.cmake   |  0
 dev/release/rat_exclude_files.txt               |  6 +-----
 python/CMakeLists.txt                           | 12 +++++++++++-
 python/MANIFEST.in                              |  3 +--
 python/cmake_modules                            |  1 +
 python/cmake_modules/BuildUtils.cmake           |  1 -
 python/cmake_modules/CompilerInfo.cmake         |  1 -
 python/cmake_modules/FindNumPy.cmake            |  1 -
 python/cmake_modules/FindPythonLibsNew.cmake    |  1 -
 python/cmake_modules/SetupCxxFlags.cmake        |  1 -
 14 files changed, 14 insertions(+), 13 deletions(-)
 rename {python => cpp}/cmake_modules/FindArrow.cmake (100%)
 rename {python => cpp}/cmake_modules/FindCython.cmake (100%)
 rename {python => cpp}/cmake_modules/FindParquet.cmake (100%)
 rename {python => cpp}/cmake_modules/FindPlasma.cmake (100%)
 rename {python => cpp}/cmake_modules/UseCython.cmake (100%)
 create mode 120000 python/cmake_modules
 delete mode 120000 python/cmake_modules/BuildUtils.cmake
 delete mode 120000 python/cmake_modules/CompilerInfo.cmake
 delete mode 120000 python/cmake_modules/FindNumPy.cmake
 delete mode 120000 python/cmake_modules/FindPythonLibsNew.cmake
 delete mode 120000 python/cmake_modules/SetupCxxFlags.cmake

diff --git a/python/cmake_modules/FindArrow.cmake b/cpp/cmake_modules/FindArrow.cmake
similarity index 100%
rename from python/cmake_modules/FindArrow.cmake
rename to cpp/cmake_modules/FindArrow.cmake
diff --git a/python/cmake_modules/FindCython.cmake b/cpp/cmake_modules/FindCython.cmake
similarity index 100%
rename from python/cmake_modules/FindCython.cmake
rename to cpp/cmake_modules/FindCython.cmake
diff --git a/python/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
similarity index 100%
rename from python/cmake_modules/FindParquet.cmake
rename to cpp/cmake_modules/FindParquet.cmake
diff --git a/python/cmake_modules/FindPlasma.cmake b/cpp/cmake_modules/FindPlasma.cmake
similarity index 100%
rename from python/cmake_modules/FindPlasma.cmake
rename to cpp/cmake_modules/FindPlasma.cmake
diff --git a/python/cmake_modules/UseCython.cmake b/cpp/cmake_modules/UseCython.cmake
similarity index 100%
rename from python/cmake_modules/UseCython.cmake
rename to cpp/cmake_modules/UseCython.cmake
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 83c9f194791ee..fa7113cc4b54a 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -34,11 +34,7 @@ cpp/src/plasma/thirdparty/xxhash.h
 dev/release/rat_exclude_files.txt
 js/.npmignore
 js/closure-compiler-scripts/*
-python/cmake_modules/BuildUtils.cmake
-python/cmake_modules/FindPythonLibsNew.cmake
-python/cmake_modules/FindNumPy.cmake
-python/cmake_modules/SetupCxxFlags.cmake
-python/cmake_modules/CompilerInfo.cmake
+python/cmake_modules
 python/doc/requirements.txt
 python/MANIFEST.in
 python/pyarrow/includes/__init__.pxd
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index d148d1105b64e..169e7ad02efa9 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -21,7 +21,17 @@
 cmake_minimum_required(VERSION 2.7)
 project(pyarrow)
 
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
+# Running from a Python sdist tarball
+set(LOCAL_CMAKE_MODULES "${CMAKE_SOURCE_DIR}/cmake_modules")
+if (EXISTS "${LOCAL_CMAKE_MODULES}")
+  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${LOCAL_CMAKE_MODULES})
+endif()
+
+# Running from a git source tree
+set(CPP_CMAKE_MODULES "${CMAKE_SOURCE_DIR}/../cpp/cmake_modules")
+if (EXISTS "${CPP_CMAKE_MODULES}")
+  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CPP_CMAKE_MODULES})
+endif()
 
 include(CMakeParseArguments)
 
diff --git a/python/MANIFEST.in b/python/MANIFEST.in
index 756879a0bb033..c6c9cd4920e86 100644
--- a/python/MANIFEST.in
+++ b/python/MANIFEST.in
@@ -2,9 +2,8 @@ include README.md
 include LICENSE.txt
 
 global-include CMakeLists.txt
+graft pyarrow
 graft cmake_modules
-recursive-include src/pyarrow *.cc *.h
-recursive-include pyarrow *.pxd
 
 global-exclude *.so
 global-exclude *.pyc
diff --git a/python/cmake_modules b/python/cmake_modules
new file mode 120000
index 0000000000000..76e2a8d12c5ee
--- /dev/null
+++ b/python/cmake_modules
@@ -0,0 +1 @@
+../cpp/cmake_modules
\ No newline at end of file
diff --git a/python/cmake_modules/BuildUtils.cmake b/python/cmake_modules/BuildUtils.cmake
deleted file mode 120000
index e3c98c8fc53e5..0000000000000
--- a/python/cmake_modules/BuildUtils.cmake
+++ /dev/null
@@ -1 +0,0 @@
-../../cpp/cmake_modules/BuildUtils.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/CompilerInfo.cmake b/python/cmake_modules/CompilerInfo.cmake
deleted file mode 120000
index 559ac3240c393..0000000000000
--- a/python/cmake_modules/CompilerInfo.cmake
+++ /dev/null
@@ -1 +0,0 @@
-../../cpp/cmake_modules/CompilerInfo.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/FindNumPy.cmake b/python/cmake_modules/FindNumPy.cmake
deleted file mode 120000
index 767b320ec7e45..0000000000000
--- a/python/cmake_modules/FindNumPy.cmake
+++ /dev/null
@@ -1 +0,0 @@
-../../cpp/cmake_modules/FindNumPy.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/FindPythonLibsNew.cmake b/python/cmake_modules/FindPythonLibsNew.cmake
deleted file mode 120000
index 7087123cb2376..0000000000000
--- a/python/cmake_modules/FindPythonLibsNew.cmake
+++ /dev/null
@@ -1 +0,0 @@
-../../cpp/cmake_modules/FindPythonLibsNew.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/SetupCxxFlags.cmake b/python/cmake_modules/SetupCxxFlags.cmake
deleted file mode 120000
index 3121a89926d2d..0000000000000
--- a/python/cmake_modules/SetupCxxFlags.cmake
+++ /dev/null
@@ -1 +0,0 @@
-../../cpp/cmake_modules/SetupCxxFlags.cmake
\ No newline at end of file

From bf2e3ab2979ce59002c616369c62822696b7948f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 9 Oct 2017 21:02:48 -0400
Subject: [PATCH 1139/1644] ARROW-1593: [Python] Pass through preserve_index to
 RecordBatch.from_pandas in serialize_pandas

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1190 from wesm/ARROW-1593 and squashes the following commits:

4cfde4b4 [Wes McKinney] Also test passing preserve_index=True
04dc0171 [Wes McKinney] Pass through preserve_index to RecordBatch.from_pandas in serialize_pandas
---
 python/pyarrow/ipc.py            |  8 ++++++--
 python/pyarrow/pandas_compat.py  |  4 ++--
 python/pyarrow/tests/test_ipc.py | 13 +++++++++++++
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index 8cb6cdd335915..f264f089c4071 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -142,7 +142,7 @@ def open_file(source, footer_offset=None):
     return RecordBatchFileReader(source, footer_offset=footer_offset)
 
 
-def serialize_pandas(df, nthreads=None):
+def serialize_pandas(df, nthreads=None, preserve_index=True):
     """Serialize a pandas DataFrame into a buffer protocol compatible object.
 
     Parameters
@@ -150,13 +150,17 @@ def serialize_pandas(df, nthreads=None):
     df : pandas.DataFrame
     nthreads : int, default None
         Number of threads to use for conversion to Arrow, default all CPUs
+    preserve_index : boolean, default True
+        If True, preserve the pandas index data, otherwise the result will have
+        a default RangeIndex
 
     Returns
     -------
     buf : buffer
         An object compatible with the buffer protocol
     """
-    batch = pa.RecordBatch.from_pandas(df, nthreads=nthreads)
+    batch = pa.RecordBatch.from_pandas(df, nthreads=nthreads,
+                                       preserve_index=preserve_index)
     sink = pa.BufferOutputStream()
     writer = pa.RecordBatchStreamWriter(sink, batch.schema)
     writer.write_batch(batch)
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index a071e5664796a..5592d8dd93f8a 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -485,8 +485,8 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
             )
         ]
         new_levels = [
-            level if level.dtype == dtype else level.astype(dtype)
-            for level, dtype in levels_dtypes
+            _level if _level.dtype == _dtype else _level.astype(_dtype)
+            for _level, _dtype in levels_dtypes
         ]
         columns = pd.MultiIndex(
             levels=new_levels,
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 0d5b67395a11d..fcde5822205ec 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -416,6 +416,19 @@ def test_pandas_serialize_round_trip_not_string_columns():
     assert_frame_equal(result, df)
 
 
+def test_serialize_pandas_no_preserve_index():
+    df = pd.DataFrame({'a': [1, 2, 3]}, index=[1, 2, 3])
+    expected = pd.DataFrame({'a': [1, 2, 3]})
+
+    buf = pa.serialize_pandas(df, preserve_index=False)
+    result = pa.deserialize_pandas(buf)
+    assert_frame_equal(result, expected)
+
+    buf = pa.serialize_pandas(df, preserve_index=True)
+    result = pa.deserialize_pandas(buf)
+    assert_frame_equal(result, df)
+
+
 def test_schema_batch_serialize_methods():
     nrows = 5
     df = pd.DataFrame({

From 166f0a87148faf9c2717c11757fbec4501609011 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 9 Oct 2017 21:05:14 -0400
Subject: [PATCH 1140/1644] ARROW-1635: Add release management guide

I tried to write down all of the details that I've accumulated in my head (and probably missed some things). There are a lot of small details that are omitted, but it's clear there's plenty of opportunities to make things easier on the release manager or those working on post-source release binary package builds.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1185 from wesm/ARROW-1635 and squashes the following commits:

3dc2edb5 [Wes McKinney] Write up release management procedure from my perspective
9b7423f9 [Wes McKinney] Start release management markdown doc
---
 dev/release/RELEASE_MANAGEMENT.md | 160 ++++++++++++++++++++++++++++++
 1 file changed, 160 insertions(+)
 create mode 100644 dev/release/RELEASE_MANAGEMENT.md

diff --git a/dev/release/RELEASE_MANAGEMENT.md b/dev/release/RELEASE_MANAGEMENT.md
new file mode 100644
index 0000000000000..6e1146ccd8b7b
--- /dev/null
+++ b/dev/release/RELEASE_MANAGEMENT.md
@@ -0,0 +1,160 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Guide for Arrow Release Managers
+
+This document is intended to provide a comprehensive checklist of tasks before,
+during, and after an Arrow release.
+
+## Preparing for the release
+
+### JIRA tidying
+
+Before creating a source release, the release manager must ensure that any
+resolved JIRAs have the appropriate Fix Version set so that the changelog is
+generated properly.
+
+To do this, search for the Arrow project and issues with no fix version. Click
+the "Tools" dropdown menu in the top right of the page and select "Bulk
+Change". Indicate that you wish to edit the issues, then set the correct Fix
+Version and apply the change. Remember to uncheck the box about "send e-mail
+notifications" to avoid excess spam to issues@arrow.apache.org.
+
+## Main source release and vote
+
+### Source release and vote
+
+Follow the instructions in [dev/release/README][1] to produce the source
+release artifacts. PMC karma is required to upload these files to the dist
+system; they are uploaded automatically by the source release script.
+
+Start the vote thread on dev@arrow.apache.org and supply intructions for
+verifying the integrity of the release. Approval requires a net of 3 +1 votes
+from PMC members. A release cannot be vetoed.
+
+## Post-release tasks
+
+After the release vote, we must undertake many tasks to update source
+artifacts, binary builds, and the Arrow website.
+
+### Updating the Arrow website
+
+The website is a Jekyll project in the `site/` directory in the main Arrow
+repository. As part of updating the website, we must perform various subtasks.
+
+First, create a new entry to add to http://arrow.apache.org/release/; these are
+in the `_release` subdirectory. The new contents of the new entry will go into
+a new Markdown file of the form `X.Y.Z.md`. You can start by copying one of the
+other release entries.
+
+Generate a web-friendly changelog by running
+
+```
+dev/release/changelog.py $VERSION 1
+```
+
+Copy and paste the result.
+
+Update `index.html` as appropriate for the new release. Then update
+`install.md` to include links for the new release.
+
+Finally, if appropriate, write a short blog post summarizing the new release
+highlights. [Here is an example.][8]
+
+### Uploading release artifacts to SVN
+
+A PMC must commit the source release artifacts to SVN at:
+
+```
+https://dist.apache.org/repos/dist/release/arrow
+```
+
+Create a new directory in SVN of the form `arrow-X.Y.Z`. If possible, remove
+any old releases to reduce load on the ASF mirror system.
+
+### Announcing release
+
+Write a release announcement ([see example][9]) and send to announce@apache.org
+and dev@arrow.apache.org. The announcement to announce@apache.org must be send
+from your apache.org e-mail address to be accepted.
+
+### Updating website with new API documentation
+
+Currently, this is manual, but is in the process of being automated (see
+ARROW-905). See instructions in
+https://github.com/apache/arrow/blob/master/site/README.md
+
+### Updating C++ and Python packages
+
+We have been making Arrow available to C++ and Python users on the 3 major
+platforms (Linux, macOS, and Windows) via two package managers: pip and conda.
+
+#### Updating pip packages
+
+The pip binary packages (called "wheels") are generated from the
+[arrow-dist][2] repository. This is a multi-step process:
+
+* Unfortunately, we are unable to upload to the Apache Arrow BinTray account,
+  so if you are the release manager, make sure to enable arrow-dist on your
+  Appveyor account and create a BinTray project under your personal BinTray
+  account where the artifacts can be written.
+* Update `.travis.yml` to reference the new Arrow release tag
+* Update `appveyor.yml` to reference the correct Arrow git commit and
+  corresponding version of [parquet-cpp][3]. At the end of this file there are
+  instructions for deploying the Windows packages to BinTray.
+* Push arrow-dist updates to **both** apache/arrow-dist and your fork of
+  arrow-dist.
+* Wait for builds to complete
+* Download all wheel files from the new BinTray package version ([example][4])
+
+Now, you can finally upload the wheels to PyPI using the `twine` CLI tool. You
+must be permissioned on PyPI to upload here; ask Wes McKinney or Uwe Korn if
+you need help with this.
+
+#### Updating conda packages
+
+We have been building conda packages using [conda-forge][6]. The three
+"feedstocks" that must be updated in-order are:
+
+* https://github.com/conda-forge/arrow-cpp-feedstock
+* https://github.com/conda-forge/parquet-cpp-feedstock
+* https://github.com/conda-forge/pyarrow-feedstock
+
+To update a feedstock, open a pull request updating `meta.yaml` as
+appropriate. Once you are confident that the build is good and the metadata is
+updated properly, merge the pull request. You must wait until the results of
+each of the feedstocks land in [anaconda.org][7] before moving on to the next
+package.
+
+Unfortunately, you cannot open pull requests to all three repositories at the
+same time because they are interdependent.
+
+### Updating Java Maven artifacts in Maven central
+
+See instructions at end of https://github.com/apache/arrow/blob/master/dev/release/README
+
+[1]: https://github.com/apache/arrow/blob/master/dev/release/README
+[2]: https://github.com/apache/arrow-dist
+[3]: https://github.com/apache/parquet-cpp
+[4]: https://bintray.com/wesm/apache-arrow-test/pyarrow/0.7.1#files
+[5]: https://pypi.python.org/pypi/pyarrow
+[6]: https://conda-forge.org/
+[7]: https://anaconda.org
+[8]: http://arrow.apache.org/blog/2017/09/19/0.7.0-release/
+[9]: http://mail-archives.apache.org/mod_mbox/www-announce/201709.mbox/%3CCAJPUwMC+VDRQ+Qj25_pqoq+bvs0bsk2Vx614OUpYwTHteFOVGw@mail.gmail.com%3E

From ee78cdcb1c475a05df9cd9de63358e80ba280a63 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 10 Oct 2017 12:21:55 -0700
Subject: [PATCH 1141/1644] ARROW-1503: [Python] Add default serialization
 context, callbacks for pandas.Series/DataFrame

The performance is a bit slower than it could be because we do not have native handling of pyarrow.Buffer (per ARROW-1522). That would allow us to skip the `to_pybytes` copy portion

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1192 from wesm/ARROW-1503 and squashes the following commits:

a36f97d [Wes McKinney] Add default serialization context and add serialization callbacks for pandas Series, DataFrame
5ff10f4 [Wes McKinney] stubs for handling Series, DataFrame more efficiently by default in serialization code paths
---
 python/pyarrow/ipc.py            | 47 ++++++++++++++++++++++++++++++++
 python/pyarrow/serialization.pxi | 17 ++++++++++--
 python/pyarrow/tests/test_ipc.py | 17 +++++++++++-
 3 files changed, 78 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index f264f089c4071..122367346fc1e 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -187,3 +187,50 @@ def deserialize_pandas(buf, nthreads=None):
     reader = pa.RecordBatchStreamReader(buffer_reader)
     table = reader.read_all()
     return table.to_pandas(nthreads=nthreads)
+
+
+# ----------------------------------------------------------------------
+# Set up default serialization context
+
+def _serialize_pandas_series(s):
+    import pandas as pd
+    # TODO: serializing Series without extra copy
+    serialized = serialize_pandas(pd.DataFrame({s.name: s}))
+    return {
+        'type': 'Series',
+        'data': serialized.to_pybytes()
+    }
+
+
+def _serialize_pandas_dataframe(df):
+    return {
+        'type': 'DataFrame',
+        'data': serialize_pandas(df).to_pybytes()
+    }
+
+
+def _deserialize_callback_pandas(data):
+    deserialized = deserialize_pandas(data['data'])
+    type_ = data['type']
+    if type_ == 'Series':
+        return deserialized[deserialized.columns[0]]
+    elif type_ == 'DataFrame':
+        return deserialized
+    else:
+        raise ValueError(type_)
+
+
+try:
+    import pandas as pd
+    lib._default_serialization_context.register_type(
+        pd.Series, 'pandas.Series',
+        custom_serializer=_serialize_pandas_series,
+        custom_deserializer=_deserialize_callback_pandas)
+
+    lib._default_serialization_context.register_type(
+        pd.DataFrame, 'pandas.DataFrame',
+        custom_serializer=_serialize_pandas_dataframe,
+        custom_deserializer=_deserialize_callback_pandas)
+except ImportError:
+    # no pandas
+    pass
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index aa1a6a4bf5289..4e9ab8eb3b374 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -137,6 +137,10 @@ cdef class SerializationContext:
                     obj.__dict__.update(serialized_obj)
         return obj
 
+
+_default_serialization_context = SerializationContext()
+
+
 cdef class SerializedPyObject:
     """
     Arrow-serialized representation of Python object
@@ -174,6 +178,9 @@ cdef class SerializedPyObject:
         """
         cdef PyObject* result
 
+        if context is None:
+            context = _default_serialization_context
+
         with nogil:
             check_status(DeserializeObject(context, self.data,
                                            <PyObject*> self.base, &result))
@@ -202,7 +209,8 @@ def serialize(object value, SerializationContext context=None):
     value: object
         Python object for the sequence that is to be serialized.
     context : SerializationContext
-        Custom serialization and deserialization context
+        Custom serialization and deserialization context, uses a default
+        context with some standard type handlers if not specified
 
     Returns
     -------
@@ -210,6 +218,10 @@ def serialize(object value, SerializationContext context=None):
     """
     cdef SerializedPyObject serialized = SerializedPyObject()
     wrapped_value = [value]
+
+    if context is None:
+        context = _default_serialization_context
+
     with nogil:
         check_status(SerializeObject(context, wrapped_value, &serialized.data))
     return serialized
@@ -225,7 +237,8 @@ def serialize_to(object value, sink, SerializationContext context=None):
     sink: NativeFile or file-like
         File the sequence will be written to.
     context : SerializationContext
-        Custom serialization and deserialization context
+        Custom serialization and deserialization context, uses a default
+        context with some standard type handlers if not specified
     """
     serialized = serialize(value, context)
     serialized.write_to(sink)
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index fcde5822205ec..68c0c80aa6187 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -22,7 +22,8 @@
 
 import numpy as np
 
-from pandas.util.testing import assert_frame_equal
+from pandas.util.testing import (assert_frame_equal,
+                                 assert_series_equal)
 import pandas as pd
 
 from pyarrow.compat import unittest
@@ -429,6 +430,20 @@ def test_serialize_pandas_no_preserve_index():
     assert_frame_equal(result, df)
 
 
+def test_serialize_with_pandas_objects():
+    df = pd.DataFrame({'a': [1, 2, 3]}, index=[1, 2, 3])
+
+    data = {
+        'a_series': df['a'],
+        'a_frame': df
+    }
+
+    serialized = pa.serialize(data).to_buffer()
+    deserialized = pa.deserialize(serialized)
+    assert_frame_equal(deserialized['a_frame'], df)
+    assert_series_equal(deserialized['a_series'], df['a'])
+
+
 def test_schema_batch_serialize_methods():
     nrows = 5
     df = pd.DataFrame({

From 4cb3e97e9ad137313eadc18345e28e4e7642ed40 Mon Sep 17 00:00:00 2001
From: Stephen G <stephen@egroat.com>
Date: Tue, 10 Oct 2017 23:47:29 -0400
Subject: [PATCH 1142/1644] ARROW-1662: Move to using Homebrew/bundle and
 Brewfile

improves osx dependency management

Author: Stephen G <stephen@egroat.com>
Author: Stephen <stephengroat@users.noreply.github.com>
Author: Stephen Groat <stephengroat@Stephens-MacBook-Pro.local>

Closes #1143 from stephengroat/osx-brew and squashes the following commits:

81d4ac15 [Stephen G] Merge branch 'osx-brew' of https://github.com/stephengroat/arrow into osx-brew
62a79291 [Stephen G] Add license headers
cee779b1 [Stephen] increase to max travis_wait
583ec0a7 [Stephen G] Move to individual brewfiles
65148557 [Stephen] add travis wait to get around timeouts
6faef0ce [Stephen Groat] Move to using Homebrew/bundle and Brewfile
---
 .travis.yml                       |  2 +-
 c_glib/Brewfile                   | 25 +++++++++++++++++++++++++
 ci/travis_before_script_c_glib.sh |  6 +-----
 ci/travis_before_script_cpp.sh    |  4 +---
 cpp/Brewfile                      | 21 +++++++++++++++++++++
 cpp/README.md                     |  2 +-
 python/Brewfile                   | 21 +++++++++++++++++++++
 python/doc/source/development.rst |  2 +-
 python/testing/setup_toolchain.sh |  4 +---
 9 files changed, 73 insertions(+), 14 deletions(-)
 create mode 100644 c_glib/Brewfile
 create mode 100644 cpp/Brewfile
 create mode 100644 python/Brewfile

diff --git a/.travis.yml b/.travis.yml
index 4691eeb0656e0..e722c27f71917 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -72,7 +72,7 @@ matrix:
     before_script:
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_PLASMA=1
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
diff --git a/c_glib/Brewfile b/c_glib/Brewfile
new file mode 100644
index 0000000000000..80d3c81dd6f82
--- /dev/null
+++ b/c_glib/Brewfile
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+brew "gtk-doc"
+brew "autoconf-archive"
+brew "gobject-introspection"
+brew "git"
+brew "cmake"
+brew "wget"
+brew "libtool"
+brew "lua"
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 7100946d76ae9..14e4f9fa11393 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -22,10 +22,7 @@ set -ex
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
-  brew install gtk-doc autoconf-archive gobject-introspection
-  brew upgrade git cmake
-  brew outdated || brew upgrade wget
-  brew outdated || brew upgrade libtool
+  brew update && brew bundle --file=c_glib/Brewfile
 
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
 fi
@@ -33,7 +30,6 @@ fi
 gem install test-unit gobject-introspection
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
-  brew install lua
   sudo env PKG_CONFIG_PATH=$PKG_CONFIG_PATH luarocks install lgi
 else
   git clone \
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 5dd8237f909b5..dbdcd33ed0d5b 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -54,9 +54,7 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
 fi
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
-  brew update > /dev/null
-  brew install jemalloc
-  brew install ccache
+  brew update && brew bundle --file=cpp/Brewfile
 fi
 
 mkdir $ARROW_CPP_BUILD_DIR
diff --git a/cpp/Brewfile b/cpp/Brewfile
new file mode 100644
index 0000000000000..5f82cacc55991
--- /dev/null
+++ b/cpp/Brewfile
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+brew "jemalloc"
+brew "ccache"
+brew "boost"
+brew "cmake"
diff --git a/cpp/README.md b/cpp/README.md
index a94699b6e1a5a..9c26842359c70 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -42,7 +42,7 @@ sudo apt-get install cmake \
 On OS X, you can use [Homebrew][1]:
 
 ```shell
-brew install boost cmake
+brew update && brew bundle --file=cpp/Brewfile
 ```
 
 If you are developing on Windows, see the [Windows developer guide][2].
diff --git a/python/Brewfile b/python/Brewfile
new file mode 100644
index 0000000000000..dae4544e43892
--- /dev/null
+++ b/python/Brewfile
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+brew "ccache"
+brew "jemalloc"
+brew "boost"
+brew "thrift"
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 2d7132303a314..093c6c038694a 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -108,7 +108,7 @@ building Arrow C++:
 
 .. code-block:: shell
 
-   brew install ccache jemalloc boost thrift
+   brew update && brew bundle --file=python/Brewfile
 
 On Debian/Ubuntu, you need the following minimal set of dependencies. All other
 dependencies will be automatically built by Arrow' thrid-party toolchain.
diff --git a/python/testing/setup_toolchain.sh b/python/testing/setup_toolchain.sh
index c3837b45cbcd4..fa5df55975e8f 100644
--- a/python/testing/setup_toolchain.sh
+++ b/python/testing/setup_toolchain.sh
@@ -59,7 +59,5 @@ conda create -y -q -p $CPP_TOOLCHAIN python=3.6 \
     ninja
 
 if [ $BUILD_OS_NAME == "osx" ]; then
-  brew update > /dev/null
-  brew install jemalloc
-  brew install ccache
+  brew update && brew bundle --file=python/Brewfile
 fi

From 60cb1c372a7c4cf98aeeb5b9deb1ab6c67a5956a Mon Sep 17 00:00:00 2001
From: Heimir Sverrisson <heimir.sverrisson@gmail.com>
Date: Thu, 12 Oct 2017 00:49:23 -0400
Subject: [PATCH 1143/1644] ARROW-905 [Docs] Dockerize document generation

This PR has two parts. First part is to use Docker to generate the API documentation for C, C++, Python and Java. This part is now complete.
The second part is to be able to run the site locally, which is still in progress.

Author: Heimir Sverrisson <heimir.sverrisson@gmail.com>

Closes #1162 from heimir-sverrisson/hs/dockerize_api_docs and squashes the following commits:

f758fdc3 [Heimir Sverrisson] ARROW-905 [Docs] Add Dockerfile for reproducible documentation generation
---
 c_glib/.gitignore                   |   1 +
 dev/README.md                       |  32 +++++++-
 dev/docker-compose.yml              |  30 ++++++++
 dev/gen_apidocs.sh                  |  21 ++++++
 dev/gen_apidocs/Dockerfile          |  83 +++++++++++++++++++++
 dev/gen_apidocs/create_documents.sh | 110 ++++++++++++++++++++++++++++
 dev/run_docker_compose.sh           |  40 ++++++++++
 dev/run_site.sh                     |  21 ++++++
 dev/run_site/Dockerfile             |  34 +++++++++
 dev/run_site/run_site.sh            |  24 ++++++
 java/.gitignore                     |   1 +
 site/.gitignore                     |   2 +
 site/_config.yml                    |   3 +
 13 files changed, 401 insertions(+), 1 deletion(-)
 create mode 100644 dev/docker-compose.yml
 create mode 100755 dev/gen_apidocs.sh
 create mode 100644 dev/gen_apidocs/Dockerfile
 create mode 100755 dev/gen_apidocs/create_documents.sh
 create mode 100755 dev/run_docker_compose.sh
 create mode 100755 dev/run_site.sh
 create mode 100644 dev/run_site/Dockerfile
 create mode 100755 dev/run_site/run_site.sh

diff --git a/c_glib/.gitignore b/c_glib/.gitignore
index ff3615a3e4920..373ae73d5d2d1 100644
--- a/c_glib/.gitignore
+++ b/c_glib/.gitignore
@@ -30,6 +30,7 @@ Makefile.in
 /doc/reference/*.stamp
 /doc/reference/html/
 /doc/reference/xml/
+/doc/reference/tmpl/
 /libtool
 /m4/
 /stamp-h1
diff --git a/dev/README.md b/dev/README.md
index 91d688dfee734..05a68fc69f0e8 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -109,4 +109,34 @@ Studio 2015):
 
 ```
 dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
-```
\ No newline at end of file
+```
+
+## Creating API documentation
+
+The generation of API documentation for `C++`, `C Glib`, `Python` 
+and `Java` has been Dockerized. To generate the API documentation
+run the following command:
+
+```shell
+bash dev/gen_apidocs.sh
+```
+
+This script assumes that the `parquet-cpp` Git repository 
+https://github.com/apache/parquet-cpp has been cloned
+besides the Arrow repository and a `dist` directory can be created
+at the same level by the current user. Please note that most of the
+software must be built in order to create the documentation, so this
+step may take some time to run, especially the first time around as the
+Docker container will also have to be built.
+
+After successfully creating the API documentation the website can be
+run locally to browse the API documentation from the top level
+`Documentation` menu. To run the website issue the command:
+
+```shell
+bash dev/run_site.sh
+```
+
+The local URL for the website running inside the docker container
+will be shown as `Server address:` in the output of the command.
+To stop the server press `Ctrl-C` in that window.
\ No newline at end of file
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
new file mode 100644
index 0000000000000..7bd2cd4412cec
--- /dev/null
+++ b/dev/docker-compose.yml
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '3'
+services:
+  gen_apidocs:
+    build: 
+      context: gen_apidocs
+    volumes:
+     - ../..:/apache-arrow
+  run_site:
+    build:
+      context: run_site
+    ports:
+    - "4000:4000"
+    volumes:
+     - ../..:/apache-arrow
diff --git a/dev/gen_apidocs.sh b/dev/gen_apidocs.sh
new file mode 100755
index 0000000000000..4bba226fb511a
--- /dev/null
+++ b/dev/gen_apidocs.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Pass the service name to run_docker_compse.sh
+# Which validates environment and runs the service
+exec "$(dirname ${BASH_SOURCE})"/run_docker_compose.sh gen_apidocs
diff --git a/dev/gen_apidocs/Dockerfile b/dev/gen_apidocs/Dockerfile
new file mode 100644
index 0000000000000..0b2844cc8454b
--- /dev/null
+++ b/dev/gen_apidocs/Dockerfile
@@ -0,0 +1,83 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM ubuntu:14.04
+ADD . /apache-arrow
+WORKDIR /apache-arrow
+# Prerequsites for apt-add-repository
+RUN apt-get update && apt-get install -y \
+    software-properties-common python-software-properties
+# Basic OS dependencies
+RUN apt-add-repository -y ppa:ubuntu-toolchain-r/test && \
+    apt-get update && apt-get install -y \
+        wget \
+        rsync \
+        git \
+        gcc-4.9 \
+        g++-4.9 \
+        build-essential 
+# This will install conda in /home/ubuntu/miniconda
+RUN wget -O /tmp/miniconda.sh \
+    https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
+    rm /tmp/miniconda.sh
+# C++ dependencies
+RUN /home/ubuntu/miniconda/bin/conda install -c conda-forge \
+    boost-cpp \
+    doxygen \
+    maven \
+    cmake \
+    zlib \
+    thrift-cpp
+# C_Glib dependencies
+RUN apt-get install -y \
+    libgtk2.0-dev \
+    gtk-doc-tools \
+    gobject-introspection \
+    libgirepository1.0-dev \
+    autogen \
+    autoconf-archive
+# Python dependencies
+RUN apt-get install -y \
+    pkg-config
+# Create Conda environment
+RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \
+        # Python
+        python=3.6 \
+        numpy \
+        pandas \
+        pytest \
+        cython \
+        ipython \
+        matplotlib \
+        numpydoc \
+        sphinx \
+        sphinx_bootstrap_theme \
+        six \
+        setuptools \
+        # C++
+        cmake \
+        flatbuffers \
+        rapidjson \
+        thrift-cpp \
+        snappy \
+        zlib \
+        brotli \
+        jemalloc \
+        lz4-c \
+        zstd \
+        -c conda-forge
+CMD arrow/dev/gen_apidocs/create_documents.sh
diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh
new file mode 100755
index 0000000000000..573a32820e75d
--- /dev/null
+++ b/dev/gen_apidocs/create_documents.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set up environment and output directory for C++ libraries
+cd /apache-arrow
+rm -rf dist
+mkdir dist
+export ARROW_BUILD_TYPE=release
+export ARROW_HOME=$(pwd)/dist
+export PARQUET_HOME=$(pwd)/dist
+CONDA_BASE=/home/ubuntu/miniconda
+export LD_LIBRARY_PATH=$(pwd)/dist/lib:${CONDA_BASE}/lib:${LD_LIBRARY_PATH}
+export THRIFT_HOME=${CONDA_BASE}
+export BOOST_ROOT=${CONDA_BASE}
+export PATH=${CONDA_BASE}/bin:${PATH}
+
+# Prepare the asf-site before copying api docs
+pushd arrow/site
+rm -rf asf-site
+export GIT_COMMITTER_NAME="Nobody"
+export GIT_COMMITTER_EMAIL="nobody@nowhere.com"
+git clone --branch=asf-site \
+    https://git-wip-us.apache.org/repos/asf/arrow-site.git asf-site
+popd
+
+# Make Python documentation (Depends on C++ )
+# Build Arrow C++
+source activate pyarrow-dev
+rm -rf arrow/cpp/build
+mkdir arrow/cpp/build
+pushd arrow/cpp/build
+cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+      -DARROW_PYTHON=on \
+      -DARROW_PLASMA=on \
+      -DARROW_BUILD_TESTS=OFF \
+      ..
+make -j4
+make install
+popd
+
+# Build Parquet C++
+rm -rf parquet-cpp/build
+mkdir parquet-cpp/build
+pushd parquet-cpp/build
+cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+      -DPARQUET_BUILD_BENCHMARKS=off \
+      -DPARQUET_BUILD_EXECUTABLES=off \
+      -DPARQUET_BUILD_TESTS=off \
+      ..
+make -j4
+make install
+popd
+
+# Now Python documentation can be built
+pushd arrow/python
+rm -rf build/*
+rm -rf doc/_build
+python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
+    --with-plasma --with-parquet --inplace
+python setup.py build_sphinx -s doc/source
+mkdir -p ../site/asf-site/docs/python
+rsync -r doc/_build/html/ ../site/asf-site/docs/python
+popd
+
+# Build c_glib documentation
+pushd arrow/c_glib
+rm -rf doc/reference/html/*
+./autogen.sh
+./configure \
+    --with-arrow-cpp-build-dir=$(pwd)/../cpp/build \
+    --with-arrow-cpp-build-type=$ARROW_BUILD_TYPE \
+    --enable-gtk-doc
+LD_LIBRARY_PATH=$(pwd)/../cpp/build/$ARROW_BUILD_TYPE make GTK_DOC_V_XREF=": "
+mkdir -p ../site/asf-site/docs/c_glib
+rsync -r doc/reference/html/ ../site/asf-site/docs/c_glib
+popd
+
+# Make C++ documentation
+pushd arrow/cpp/apidoc
+rm -rf html/*
+doxygen Doxyfile
+mkdir -p ../../site/asf-site/docs/cpp
+rsync -r html/ ../../site/asf-site/docs/cpp
+popd
+
+# Make Java documentation
+pushd arrow/java
+rm -rf target/site/apidocs/*
+mvn -Drat.skip=true install
+mvn -Drat.skip=true site
+mkdir -p ../site/asf-site/docs/java/
+rsync -r target/site/apidocs ../site/asf-site/docs/java/
+popd
diff --git a/dev/run_docker_compose.sh b/dev/run_docker_compose.sh
new file mode 100755
index 0000000000000..f46879ed1e436
--- /dev/null
+++ b/dev/run_docker_compose.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ $# -lt 1 ]; then
+    echo "This script takes one argument - the docker service to run" >&2
+    exit 1
+fi
+
+# Make sure this is always run in the directory above Arrow root
+cd $(dirname "${BASH_SOURCE}")/../..
+
+if [ ! -d arrow ]; then
+    echo "Please make sure that the top level Arrow directory" >&2
+    echo "is named 'arrow'"
+    exit 1
+fi
+
+if [ ! -d parquet-cpp ]; then
+    echo "Please clone the Parquet repo next to the Arrow repo" >&2
+    exit 1
+fi
+
+GID=$(id -g ${USERNAME})
+docker-compose -f arrow/dev/docker-compose.yml run \
+               -u "${UID}:${GID}" "${1}"
diff --git a/dev/run_site.sh b/dev/run_site.sh
new file mode 100755
index 0000000000000..1e64dada3167e
--- /dev/null
+++ b/dev/run_site.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Pass the service name to run_docker_compse.sh
+# Which validates environment and runs the service
+exec "$(dirname ${BASH_SOURCE})"/run_docker_compose.sh run_site
diff --git a/dev/run_site/Dockerfile b/dev/run_site/Dockerfile
new file mode 100644
index 0000000000000..cea4ceb72e476
--- /dev/null
+++ b/dev/run_site/Dockerfile
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM ubuntu:14.04
+ADD . /apache-arrow
+WORKDIR /apache-arrow
+# Prerequsites for apt-add-repository
+RUN apt-get update && apt-get install -y \
+    software-properties-common python-software-properties
+# Set up Ruby repository
+RUN apt-add-repository ppa:brightbox/ruby-ng
+# The publication tools
+RUN apt-get update; apt-get install -y \
+    apt-transport-https \
+    ruby2.2-dev \
+    ruby2.2 \
+    zlib1g-dev \ 
+    make \
+    gcc
+RUN gem install jekyll bundler
+CMD arrow/dev/run_site/run_site.sh
diff --git a/dev/run_site/run_site.sh b/dev/run_site/run_site.sh
new file mode 100755
index 0000000000000..8527b2b16d222
--- /dev/null
+++ b/dev/run_site/run_site.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+cd /apache-arrow/arrow/site
+export GEM_HOME=$(pwd)
+export BUNDLE_PATH=$(pwd)
+export HOME=$(pwd)
+bundle install --path .
+HOST_IP=$(grep $(hostname) /etc/hosts | cut -f 1)
+bundle exec jekyll serve -c _config.yml -s asf-site --host ${HOST_IP} --port 4000
diff --git a/java/.gitignore b/java/.gitignore
index 73c1be4912297..03f5bf76e60d2 100644
--- a/java/.gitignore
+++ b/java/.gitignore
@@ -20,3 +20,4 @@ CMakeFiles
 Makefile
 cmake_install.cmake
 install_manifest.txt
+?/
diff --git a/site/.gitignore b/site/.gitignore
index 46bc466d3028e..d3c112a452ab8 100644
--- a/site/.gitignore
+++ b/site/.gitignore
@@ -4,3 +4,5 @@ _site
 Gemfile.lock
 asf-site
 build/
+.bundle/
+ruby/
diff --git a/site/_config.yml b/site/_config.yml
index fcb76a374fb51..a6c5575d0680f 100644
--- a/site/_config.yml
+++ b/site/_config.yml
@@ -23,6 +23,9 @@ kramdown:
   input: GFM
   syntax_highlighter: rouge
 
+include:
+  - _static
+
 exclude:
   - Gemfile
   - Gemfile.lock

From 0d1e69c2391f90d1eec60cd8304b956b855fe1c1 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Thu, 12 Oct 2017 12:12:35 -0400
Subject: [PATCH 1144/1644] ARROW-1630: [Serialization] Support Python datetime
 objects

An additional pair of eyes would be helpful, somewhat strangely the tests are passing for some datetime objects and not for others.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1153 from pcmoritz/serialize-datetime and squashes the following commits:

f3696ae4 [Philipp Moritz] add numpy to LICENSE.txt
a94bca7d [Philipp Moritz] put PyDateTime_IMPORT higher up
0ae645e9 [Philipp Moritz] windows fixes
cbd1b222 [Philipp Moritz] get rid of gmtime_r
f3ea6699 [Philipp Moritz] use numpy datetime code to implement time conversions
e644f4f5 [Philipp Moritz] linting
f38cbd46 [Philipp Moritz] fixes
6e549c47 [Philipp Moritz] serialize datetime
---
 LICENSE.txt                                |  39 ++++
 NOTICE.txt                                 |   5 -
 cpp/src/arrow/python/arrow_to_python.cc    |   8 +
 cpp/src/arrow/python/python_to_arrow.cc    |  14 ++
 cpp/src/arrow/python/util/datetime.h       | 255 ++++++++++++++++-----
 python/pyarrow/tests/test_serialization.py |  24 ++
 6 files changed, 286 insertions(+), 59 deletions(-)

diff --git a/LICENSE.txt b/LICENSE.txt
index 34879f61dc80f..00cb9ece232b0 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -359,3 +359,42 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the NumPy project.
+
+https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
+
+https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c
+
+Copyright (c) 2005-2017, NumPy Developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+       copyright notice, this list of conditions and the following
+       disclaimer in the documentation and/or other materials provided
+       with the distribution.
+
+    * Neither the name of the NumPy Developers nor the names of any
+       contributors may be used to endorse or promote products derived
+       from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/NOTICE.txt b/NOTICE.txt
index c02e75f91d966..875ad74b79a8b 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -8,11 +8,6 @@ This product includes software from the SFrame project (BSD, 3-clause).
 * Copyright (C) 2015 Dato, Inc.
 * Copyright (c) 2009 Carnegie Mellon University.
 
-This product includes software from the Numpy project (BSD-new)
- https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
- * Copyright (c) 1995, 1996, 1997 Jim Hugunin, hugunin@mit.edu
- * Copyright (c) 2005 Travis E. Oliphant oliphant@ee.byu.edu Brigham Young University
-
 This product includes software from the Feather project (Apache 2.0)
 https://github.com/wesm/feather
 
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index b4f4a41f44068..de05a230cab4f 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -28,6 +28,7 @@
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/python_to_arrow.h"
+#include "arrow/python/util/datetime.h"
 #include "arrow/table.h"
 #include "arrow/util/logging.h"
 
@@ -126,6 +127,12 @@ Status GetValue(PyObject* context, const Array& arr, int64_t index, int32_t type
     case Type::DOUBLE:
       *result = PyFloat_FromDouble(static_cast<const DoubleArray&>(arr).Value(index));
       return Status::OK();
+    case Type::DATE64: {
+      RETURN_NOT_OK(PyDateTime_from_int(static_cast<const Date64Array&>(arr).Value(index),
+                                        TimeUnit::MICRO, result));
+      RETURN_IF_PYERROR();
+      return Status::OK();
+    }
     case Type::STRUCT: {
       const auto& s = static_cast<const StructArray&>(arr);
       const auto& l = static_cast<const ListArray&>(*s.field(0));
@@ -248,6 +255,7 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out)
 Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObject* base,
                          PyObject** out) {
   PyAcquireGIL lock;
+  PyDateTime_IMPORT;
   return DeserializeList(context, *obj.batch->column(0), 0, obj.batch->num_rows(), base,
                          obj.tensors, out);
 }
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index a693a081f787a..e2d7452140eb6 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -36,6 +36,7 @@
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/platform.h"
+#include "arrow/python/util/datetime.h"
 #include "arrow/tensor.h"
 #include "arrow/util/logging.h"
 
@@ -59,6 +60,7 @@ class SequenceBuilder {
         strings_(pool),
         floats_(::arrow::float32(), pool),
         doubles_(::arrow::float64(), pool),
+        date64s_(::arrow::date64(), pool),
         tensor_indices_(::arrow::int32(), pool),
         list_offsets_({0}),
         tuple_offsets_({0}),
@@ -125,6 +127,11 @@ class SequenceBuilder {
     return AppendPrimitive(data, &double_tag_, &doubles_);
   }
 
+  /// Appending a Date64 timestamp to the sequence
+  Status AppendDate64(const int64_t timestamp) {
+    return AppendPrimitive(timestamp, &date64_tag_, &date64s_);
+  }
+
   /// Appending a tensor to the sequence
   ///
   /// \param tensor_index Index of the tensor in the object.
@@ -217,6 +224,7 @@ class SequenceBuilder {
     RETURN_NOT_OK(AddElement(bytes_tag_, &bytes_));
     RETURN_NOT_OK(AddElement(float_tag_, &floats_));
     RETURN_NOT_OK(AddElement(double_tag_, &doubles_));
+    RETURN_NOT_OK(AddElement(date64_tag_, &date64s_));
     RETURN_NOT_OK(AddElement(tensor_tag_, &tensor_indices_));
 
     RETURN_NOT_OK(AddSubsequence(list_tag_, list_data, list_offsets_, "list"));
@@ -244,6 +252,7 @@ class SequenceBuilder {
   StringBuilder strings_;
   FloatBuilder floats_;
   DoubleBuilder doubles_;
+  Date64Builder date64s_;
 
   // We use an Int32Builder here to distinguish the tensor indices from
   // the ints_ above (see the case Type::INT32 in get_value in python.cc).
@@ -267,6 +276,7 @@ class SequenceBuilder {
   int8_t bytes_tag_ = -1;
   int8_t float_tag_ = -1;
   int8_t double_tag_ = -1;
+  int8_t date64_tag_ = -1;
 
   int8_t tensor_tag_ = -1;
   int8_t list_tag_ = -1;
@@ -485,6 +495,9 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
                                  subdicts, tensors_out));
   } else if (elem == Py_None) {
     RETURN_NOT_OK(builder->AppendNone());
+  } else if (PyDateTime_CheckExact(elem)) {
+    PyDateTime_DateTime* datetime = reinterpret_cast<PyDateTime_DateTime*>(elem);
+    RETURN_NOT_OK(builder->AppendDate64(PyDateTime_to_us(datetime)));
   } else {
     // Attempt to serialize the object using the custom callback.
     PyObject* serialized_object;
@@ -656,6 +669,7 @@ std::shared_ptr<RecordBatch> MakeBatch(std::shared_ptr<Array> data) {
 
 Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out) {
   PyAcquireGIL lock;
+  PyDateTime_IMPORT;
   std::vector<PyObject*> sequences = {sequence};
   std::shared_ptr<Array> array;
   std::vector<PyObject*> py_tensors;
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index 4ebef720f7a19..01fbc18e6ed26 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -18,12 +18,149 @@
 #ifndef PYARROW_UTIL_DATETIME_H
 #define PYARROW_UTIL_DATETIME_H
 
+#include <algorithm>
+#include <sstream>
+
 #include <datetime.h>
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
 #include "arrow/python/platform.h"
 
 namespace arrow {
 namespace py {
 
+// The following code is adapted from
+// https://github.com/numpy/numpy/blob/master/numpy/core/src/multiarray/datetime.c
+
+// Days per month, regular year and leap year
+static int64_t _days_per_month_table[2][12] = {
+    { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 },
+    { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }
+};
+
+static bool is_leapyear(int64_t year) {
+    return (year & 0x3) == 0 && // year % 4 == 0
+           ((year % 100) != 0 ||
+            (year % 400) == 0);
+}
+
+// Calculates the days offset from the 1970 epoch.
+static int64_t get_days_from_date(int64_t date_year,
+                                  int64_t date_month,
+                                  int64_t date_day) {
+    int64_t i, month;
+    int64_t year, days = 0;
+    int64_t *month_lengths;
+
+    year = date_year - 1970;
+    days = year * 365;
+
+    // Adjust for leap years
+    if (days >= 0) {
+        // 1968 is the closest leap year before 1970.
+        // Exclude the current year, so add 1.
+        year += 1;
+        // Add one day for each 4 years
+        days += year / 4;
+        // 1900 is the closest previous year divisible by 100
+        year += 68;
+        // Subtract one day for each 100 years
+        days -= year / 100;
+        // 1600 is the closest previous year divisible by 400
+        year += 300;
+        // Add one day for each 400 years
+        days += year / 400;
+    } else {
+        // 1972 is the closest later year after 1970.
+        // Include the current year, so subtract 2.
+        year -= 2;
+        // Subtract one day for each 4 years
+        days += year / 4;
+        // 2000 is the closest later year divisible by 100
+        year -= 28;
+        // Add one day for each 100 years
+        days -= year / 100;
+        // 2000 is also the closest later year divisible by 400
+        // Subtract one day for each 400 years
+        days += year / 400;
+    }
+
+    month_lengths = _days_per_month_table[is_leapyear(date_year)];
+    month = date_month - 1;
+
+    // Add the months
+    for (i = 0; i < month; ++i) {
+        days += month_lengths[i];
+    }
+
+    // Add the days
+    days += date_day - 1;
+
+    return days;
+}
+
+// Modifies '*days_' to be the day offset within the year,
+// and returns the year.
+static int64_t days_to_yearsdays(int64_t* days_) {
+    const int64_t days_per_400years = (400*365 + 100 - 4 + 1);
+    // Adjust so it's relative to the year 2000 (divisible by 400)
+    int64_t days = (*days_) - (365*30 + 7);
+    int64_t year;
+
+    // Break down the 400 year cycle to get the year and day within the year
+    if (days >= 0) {
+        year = 400 * (days / days_per_400years);
+        days = days % days_per_400years;
+    } else {
+        year = 400 * ((days - (days_per_400years - 1)) / days_per_400years);
+        days = days % days_per_400years;
+        if (days < 0) {
+            days += days_per_400years;
+        }
+    }
+
+    // Work out the year/day within the 400 year cycle
+    if (days >= 366) {
+        year += 100 * ((days-1) / (100*365 + 25 - 1));
+        days = (days-1) % (100*365 + 25 - 1);
+        if (days >= 365) {
+            year += 4 * ((days+1) / (4*365 + 1));
+            days = (days+1) % (4*365 + 1);
+            if (days >= 366) {
+                year += (days-1) / 365;
+                days = (days-1) % 365;
+            }
+        }
+    }
+
+    *days_ = days;
+    return year + 2000;
+}
+
+// Extracts the month and year and day number from a number of days
+static void get_date_from_days(int64_t days,
+                               int64_t* date_year,
+                               int64_t* date_month,
+                               int64_t* date_day) {
+    int64_t *month_lengths, i;
+
+    *date_year = days_to_yearsdays(&days);
+    month_lengths = _days_per_month_table[is_leapyear(*date_year)];
+
+    for (i = 0; i < 12; ++i) {
+        if (days < month_lengths[i]) {
+            *date_month = i + 1;
+            *date_day = days + 1;
+            return;
+        } else {
+            days -= month_lengths[i];
+        }
+    }
+
+    // Should never get here
+    return;
+}
+
 static inline int64_t PyTime_to_us(PyObject* pytime) {
   return (static_cast<int64_t>(PyDateTime_TIME_GET_HOUR(pytime)) * 3600000000LL +
           static_cast<int64_t>(PyDateTime_TIME_GET_MINUTE(pytime)) * 60000000LL +
@@ -31,9 +168,28 @@ static inline int64_t PyTime_to_us(PyObject* pytime) {
           PyDateTime_TIME_GET_MICROSECOND(pytime));
 }
 
-static inline Status PyTime_from_int(int64_t val, const TimeUnit::type unit,
-                                     PyObject** out) {
-  int64_t hour = 0, minute = 0, second = 0, microsecond = 0;
+
+// Splitting time quantities, for example splitting total seconds into
+// minutes and remaining seconds. After we run
+// int64_t remaining = split_time(total, quotient, &next)
+// we have
+// total = next * quotient + remaining. Handles negative values by propagating
+// them: If total is negative, next will be negative and remaining will
+// always be non-negative.
+static inline int64_t split_time(int64_t total, int64_t quotient, int64_t* next) {
+  int64_t r = total % quotient;
+  if (r < 0) {
+    *next = total / quotient - 1;
+    return r + quotient;
+  } else {
+    *next = total / quotient;
+    return r;
+  }
+}
+
+static inline Status PyTime_convert_int(int64_t val, const TimeUnit::type unit,
+                                        int64_t *hour, int64_t *minute,
+                                        int64_t *second, int64_t *microsecond) {
   switch (unit) {
     case TimeUnit::NANO:
       if (val % 1000 != 0) {
@@ -44,75 +200,66 @@ static inline Status PyTime_from_int(int64_t val, const TimeUnit::type unit,
       val /= 1000;
     // fall through
     case TimeUnit::MICRO:
-      microsecond = val - (val / 1000000LL) * 1000000LL;
-      val /= 1000000LL;
-      second = val - (val / 60) * 60;
-      val /= 60;
-      minute = val - (val / 60) * 60;
-      hour = val / 60;
+      *microsecond = split_time(val, 1000000LL, &val);
+      *second = split_time(val, 60, &val);
+      *minute = split_time(val, 60, hour);
       break;
     case TimeUnit::MILLI:
-      microsecond = (val - (val / 1000) * 1000) * 1000;
-      val /= 1000;
+      *microsecond = split_time(val, 1000, &val) * 1000;
     // fall through
     case TimeUnit::SECOND:
-      second = val - (val / 60) * 60;
-      val /= 60;
-      minute = val - (val / 60) * 60;
-      hour = val / 60;
+      *second = split_time(val, 60, &val);
+      *minute = split_time(val, 60, hour);
       break;
     default:
       break;
   }
+  return Status::OK();
+}
+
+static inline Status PyTime_from_int(int64_t val, const TimeUnit::type unit,
+                                     PyObject** out) {
+  int64_t hour = 0, minute = 0, second = 0, microsecond = 0;
+  RETURN_NOT_OK(PyTime_convert_int(val, unit, &hour, &minute, &second, &microsecond));
   *out = PyTime_FromTime(static_cast<int32_t>(hour), static_cast<int32_t>(minute),
                          static_cast<int32_t>(second), static_cast<int32_t>(microsecond));
   return Status::OK();
 }
 
+static inline Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit,
+                                         PyObject** out) {
+  int64_t hour = 0, minute = 0, second = 0, microsecond = 0;
+  RETURN_NOT_OK(PyTime_convert_int(val, unit, &hour, &minute, &second, &microsecond));
+  int64_t total_days = 0;
+  hour = split_time(hour, 24, &total_days);
+  int64_t year = 0, month = 0, day = 0;
+  get_date_from_days(total_days, &year, &month, &day);
+  *out = PyDateTime_FromDateAndTime(static_cast<int32_t>(year),
+                                    static_cast<int32_t>(month),
+                                    static_cast<int32_t>(day),
+                                    static_cast<int32_t>(hour),
+                                    static_cast<int32_t>(minute),
+                                    static_cast<int32_t>(second),
+                                    static_cast<int32_t>(microsecond));
+  return Status::OK();
+}
+
 static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
-  struct tm date;
-  memset(&date, 0, sizeof(struct tm));
-  date.tm_year = PyDateTime_GET_YEAR(pydate) - 1900;
-  date.tm_mon = PyDateTime_GET_MONTH(pydate) - 1;
-  date.tm_mday = PyDateTime_GET_DAY(pydate);
-  struct tm epoch;
-  memset(&epoch, 0, sizeof(struct tm));
-
-  epoch.tm_year = 70;
-  epoch.tm_mday = 1;
-#ifdef _MSC_VER
-  // Milliseconds since the epoch
-  const int64_t current_timestamp = static_cast<int64_t>(_mkgmtime64(&date));
-  const int64_t epoch_timestamp = static_cast<int64_t>(_mkgmtime64(&epoch));
-  return (current_timestamp - epoch_timestamp) * 1000LL;
-#else
-  return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
-#endif
+  int64_t total_seconds = 0;
+  total_seconds += PyDateTime_DATE_GET_SECOND(pydate);
+  total_seconds += PyDateTime_DATE_GET_MINUTE(pydate) * 60;
+  total_seconds += PyDateTime_DATE_GET_HOUR(pydate) * 3600;
+  int64_t days = get_days_from_date(PyDateTime_GET_YEAR(pydate),
+                                    PyDateTime_GET_MONTH(pydate),
+                                    PyDateTime_GET_DAY(pydate));
+  total_seconds += days * 24 * 3600;
+  return total_seconds * 1000;
 }
 
 static inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) {
-  struct tm datetime;
-  memset(&datetime, 0, sizeof(struct tm));
-  datetime.tm_year = PyDateTime_GET_YEAR(pydatetime) - 1900;
-  datetime.tm_mon = PyDateTime_GET_MONTH(pydatetime) - 1;
-  datetime.tm_mday = PyDateTime_GET_DAY(pydatetime);
-  datetime.tm_hour = PyDateTime_DATE_GET_HOUR(pydatetime);
-  datetime.tm_min = PyDateTime_DATE_GET_MINUTE(pydatetime);
-  datetime.tm_sec = PyDateTime_DATE_GET_SECOND(pydatetime);
+  int64_t ms = PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(pydatetime));
   int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
-  struct tm epoch;
-  memset(&epoch, 0, sizeof(struct tm));
-  epoch.tm_year = 70;
-  epoch.tm_mday = 1;
-#ifdef _MSC_VER
-  // Microseconds since the epoch
-  const int64_t current_timestamp = static_cast<int64_t>(_mkgmtime64(&datetime));
-  const int64_t epoch_timestamp = static_cast<int64_t>(_mkgmtime64(&epoch));
-  return (current_timestamp - epoch_timestamp) * 1000000L + us;
-#else
-  return static_cast<int64_t>(
-      lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000 + us);
-#endif
+  return ms * 1000 + us;
 }
 
 static inline int32_t PyDate_to_days(PyDateTime_Date* pydate) {
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 09fc4aaa2aa36..5441b9fb4ffcb 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -20,6 +20,7 @@
 import pytest
 
 from collections import namedtuple, OrderedDict, defaultdict
+import datetime
 import string
 import sys
 
@@ -311,6 +312,29 @@ def test_numpy_serialization(large_memory_map):
             serialization_roundtrip(obj, mmap)
 
 
+def test_datetime_serialization(large_memory_map):
+    data = [# Principia Mathematica published
+            datetime.datetime(year=1687, month=7, day=5),
+            # Some random date
+            datetime.datetime(year=1911, month=6, day=3, hour=4,
+                              minute=55, second=44),
+            # End of WWI
+            datetime.datetime(year=1918, month=11, day=11),
+            # Beginning of UNIX time
+            datetime.datetime(year=1970, month=1, day=1),
+            # The Berlin wall falls
+            datetime.datetime(year=1989, month=11, day=9),
+            # Another random date
+            datetime.datetime(year=2011, month=6, day=3, hour=4,
+                              minute=0, second=3),
+            # Another random date
+            datetime.datetime(year=1970, month=1, day=3, hour=4,
+                              minute=0, second=0)]
+    with pa.memory_map(large_memory_map, mode="r+") as mmap:
+        for d in data:
+            serialization_roundtrip(d, mmap)
+
+
 def test_numpy_immutable(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         obj = np.zeros([10])

From 434df8af0527604303f98a23d826dad0ccc7ce12 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 12 Oct 2017 12:15:44 -0400
Subject: [PATCH 1145/1644] ARROW-1488: [C++] Implement ArrayBuilder::Finish in
 terms of FinishInternal based on ArrayData

I don't have strong feelings about the function names here, but this was backwards compatible and will prevent extra box-then-unbox steps in analytics code that utilizes builders for accumulating outputs

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1191 from wesm/ARROW-1488 and squashes the following commits:

0c23ff53 [Wes McKinney] Rename Finish with ArrayData to FinishInternal
5313cab2 [Wes McKinney] Initial hack at refactoring Finish methods to return ArrayData
---
 cpp/src/arrow/builder.cc | 126 +++++++++++++++++++--------------------
 cpp/src/arrow/builder.h  |  40 +++++++------
 2 files changed, 85 insertions(+), 81 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 55e7873e8308c..7152c7aa75940 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -99,6 +99,12 @@ Status ArrayBuilder::Advance(int64_t elements) {
   return Status::OK();
 }
 
+Status ArrayBuilder::Finish(std::shared_ptr<Array>* out) {
+  std::shared_ptr<ArrayData> internal_data;
+  RETURN_NOT_OK(FinishInternal(&internal_data));
+  return MakeArray(internal_data, out);
+}
+
 Status ArrayBuilder::Reserve(int64_t elements) {
   if (length_ + elements > capacity_) {
     // TODO(emkornfield) power of 2 growth is potentially suboptimal
@@ -213,8 +219,9 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
 // ----------------------------------------------------------------------
 // Null builder
 
-Status NullBuilder::Finish(std::shared_ptr<Array>* out) {
-  *out = std::make_shared<NullArray>(length_);
+Status NullBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  BufferVector buffers = {nullptr};
+  *out = std::make_shared<ArrayData>(null(), length_, std::move(buffers), length_);
   length_ = null_count_ = 0;
   return Status::OK();
 }
@@ -302,14 +309,14 @@ Status PrimitiveBuilder<T>::Append(const std::vector<value_type>& values) {
 }
 
 template <typename T>
-Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
+Status PrimitiveBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
   const int64_t bytes_required = TypeTraits<T>::bytes_required(length_);
   if (bytes_required > 0 && bytes_required < data_->size()) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
-  *out = std::make_shared<typename TypeTraits<T>::ArrayType>(type_, length_, data_,
-                                                             null_bitmap_, null_count_);
+  BufferVector buffers = {null_bitmap_, data_};
+  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -372,34 +379,36 @@ Status AdaptiveIntBuilderBase::Resize(int64_t capacity) {
 
 AdaptiveIntBuilder::AdaptiveIntBuilder(MemoryPool* pool) : AdaptiveIntBuilderBase(pool) {}
 
-Status AdaptiveIntBuilder::Finish(std::shared_ptr<Array>* out) {
+Status AdaptiveIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   const int64_t bytes_required = length_ * int_size_;
   if (bytes_required > 0 && bytes_required < data_->size()) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
+
+  std::shared_ptr<DataType> output_type;
   switch (int_size_) {
     case 1:
-      *out =
-          std::make_shared<Int8Array>(int8(), length_, data_, null_bitmap_, null_count_);
+      output_type = int8();
       break;
     case 2:
-      *out = std::make_shared<Int16Array>(int16(), length_, data_, null_bitmap_,
-                                          null_count_);
+      output_type = int16();
       break;
     case 4:
-      *out = std::make_shared<Int32Array>(int32(), length_, data_, null_bitmap_,
-                                          null_count_);
+      output_type = int32();
       break;
     case 8:
-      *out = std::make_shared<Int64Array>(int64(), length_, data_, null_bitmap_,
-                                          null_count_);
+      output_type = int64();
       break;
     default:
       DCHECK(false);
       return Status::NotImplemented("Only ints of size 1,2,4,8 are supported");
   }
 
+  BufferVector buffers = {null_bitmap_, data_};
+  *out =
+      std::make_shared<ArrayData>(output_type, length_, std::move(buffers), null_count_);
+
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
   return Status::OK();
@@ -529,34 +538,35 @@ Status AdaptiveIntBuilder::ExpandIntSize(uint8_t new_int_size) {
 AdaptiveUIntBuilder::AdaptiveUIntBuilder(MemoryPool* pool)
     : AdaptiveIntBuilderBase(pool) {}
 
-Status AdaptiveUIntBuilder::Finish(std::shared_ptr<Array>* out) {
+Status AdaptiveUIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   const int64_t bytes_required = length_ * int_size_;
   if (bytes_required > 0 && bytes_required < data_->size()) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
+  std::shared_ptr<DataType> output_type;
   switch (int_size_) {
     case 1:
-      *out = std::make_shared<UInt8Array>(uint8(), length_, data_, null_bitmap_,
-                                          null_count_);
+      output_type = uint8();
       break;
     case 2:
-      *out = std::make_shared<UInt16Array>(uint16(), length_, data_, null_bitmap_,
-                                           null_count_);
+      output_type = uint16();
       break;
     case 4:
-      *out = std::make_shared<UInt32Array>(uint32(), length_, data_, null_bitmap_,
-                                           null_count_);
+      output_type = uint32();
       break;
     case 8:
-      *out = std::make_shared<UInt64Array>(uint64(), length_, data_, null_bitmap_,
-                                           null_count_);
+      output_type = uint64();
       break;
     default:
       DCHECK(false);
       return Status::NotImplemented("Only ints of size 1,2,4,8 are supported");
   }
 
+  BufferVector buffers = {null_bitmap_, data_};
+  *out =
+      std::make_shared<ArrayData>(output_type, length_, std::move(buffers), null_count_);
+
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
   return Status::OK();
@@ -725,14 +735,15 @@ Status BooleanBuilder::Resize(int64_t capacity) {
   return Status::OK();
 }
 
-Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
+Status BooleanBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   const int64_t bytes_required = BitUtil::BytesForBits(length_);
 
   if (bytes_required > 0 && bytes_required < data_->size()) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
-  *out = std::make_shared<BooleanArray>(type_, length_, data_, null_bitmap_, null_count_);
+  BufferVector buffers = {null_bitmap_, data_};
+  *out = std::make_shared<ArrayData>(boolean(), length_, std::move(buffers), null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -861,15 +872,12 @@ Status DictionaryBuilder<T>::Resize(int64_t capacity) {
 }
 
 template <typename T>
-Status DictionaryBuilder<T>::Finish(std::shared_ptr<Array>* out) {
+Status DictionaryBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Array> dictionary;
   RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
 
-  std::shared_ptr<Array> values;
-  RETURN_NOT_OK(values_builder_.Finish(&values));
-
-  auto type = std::make_shared<DictionaryType>(values->type(), dictionary);
-  *out = std::make_shared<DictionaryArray>(type, values);
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
   return Status::OK();
 }
 
@@ -1104,10 +1112,12 @@ Status DecimalBuilder::Append(const Decimal128& value) {
   return FixedSizeBinaryBuilder::Append(value.ToBytes());
 }
 
-Status DecimalBuilder::Finish(std::shared_ptr<Array>* out) {
+Status DecimalBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
-  *out = std::make_shared<DecimalArray>(type_, length_, data, null_bitmap_, null_count_);
+
+  BufferVector buffers = {null_bitmap_, data};
+  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
   return Status::OK();
 }
 
@@ -1161,20 +1171,22 @@ Status ListBuilder::Resize(int64_t capacity) {
   return ArrayBuilder::Resize(capacity);
 }
 
-Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
+Status ListBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   RETURN_NOT_OK(AppendNextOffset());
 
   std::shared_ptr<Buffer> offsets;
   RETURN_NOT_OK(offsets_builder_.Finish(&offsets));
 
-  std::shared_ptr<Array> items = values_;
-  if (!items) {
-    RETURN_NOT_OK(value_builder_->Finish(&items));
+  std::shared_ptr<ArrayData> items;
+  if (values_) {
+    items = values_->data();
+  } else {
+    RETURN_NOT_OK(value_builder_->FinishInternal(&items));
   }
 
-  *out = std::make_shared<ListArray>(type_, length_, offsets, items, null_bitmap_,
-                                     null_count_);
-
+  BufferVector buffers = {null_bitmap_, offsets};
+  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  (*out)->child_data.emplace_back(std::move(items));
   Reset();
   return Status::OK();
 }
@@ -1247,13 +1259,6 @@ Status BinaryBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
 
   BufferVector buffers = {null_bitmap_, offsets, value_data};
   *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_, 0);
-  return Status::OK();
-}
-
-Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<ArrayData> data;
-  RETURN_NOT_OK(FinishInternal(&data));
-  *out = std::make_shared<BinaryArray>(data);
   Reset();
   return Status::OK();
 }
@@ -1277,14 +1282,6 @@ const uint8_t* BinaryBuilder::GetValue(int64_t i, int32_t* out_length) const {
 
 StringBuilder::StringBuilder(MemoryPool* pool) : BinaryBuilder(utf8(), pool) {}
 
-Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::shared_ptr<ArrayData> data;
-  RETURN_NOT_OK(FinishInternal(&data));
-  *out = std::make_shared<StringArray>(data);
-  Reset();
-  return Status::OK();
-}
-
 // ----------------------------------------------------------------------
 // Fixed width binary
 
@@ -1327,11 +1324,12 @@ Status FixedSizeBinaryBuilder::Resize(int64_t capacity) {
   return ArrayBuilder::Resize(capacity);
 }
 
-Status FixedSizeBinaryBuilder::Finish(std::shared_ptr<Array>* out) {
+Status FixedSizeBinaryBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
-  *out = std::make_shared<FixedSizeBinaryArray>(type_, length_, data, null_bitmap_,
-                                                null_count_);
+
+  BufferVector buffers = {null_bitmap_, data};
+  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
   return Status::OK();
 }
 
@@ -1349,17 +1347,17 @@ StructBuilder::StructBuilder(const std::shared_ptr<DataType>& type, MemoryPool*
   field_builders_ = std::move(field_builders);
 }
 
-Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
-  std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
+Status StructBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  BufferVector buffers = {null_bitmap_};
+  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+
+  (*out)->child_data.resize(field_builders_.size());
   for (size_t i = 0; i < field_builders_.size(); ++i) {
-    RETURN_NOT_OK(field_builders_[i]->Finish(&fields[i]));
+    RETURN_NOT_OK(field_builders_[i]->FinishInternal(&(*out)->child_data[i]));
   }
 
-  *out = std::make_shared<StructArray>(type_, length_, fields, null_bitmap_, null_count_);
-
   null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
-
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 1ddcf27143b53..54d11cf0fb1f2 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -108,9 +108,18 @@ class ARROW_EXPORT ArrayBuilder {
 
   std::shared_ptr<PoolBuffer> null_bitmap() const { return null_bitmap_; }
 
-  /// Creates new Array object to hold the contents of the builder and transfers
-  /// ownership of the data.  This resets all variables on the builder.
-  virtual Status Finish(std::shared_ptr<Array>* out) = 0;
+  /// \brief Return result of builder as an internal generic ArrayData
+  /// object. Resets builder
+  ///
+  /// \param[out] out the finalized ArrayData object
+  /// \return Status
+  virtual Status FinishInternal(std::shared_ptr<ArrayData>* out) = 0;
+
+  /// \brief Return result of builder as an Array object. Resets builder
+  ///
+  /// \param[out] out the finalized Array object
+  /// \return Status
+  Status Finish(std::shared_ptr<Array>* out);
 
   std::shared_ptr<DataType> type() const { return type_; }
 
@@ -170,7 +179,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
     return Status::OK();
   }
 
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 };
 
 template <typename Type>
@@ -228,7 +237,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   /// \return Status
   Status Append(const std::vector<value_type>& values);
 
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
   Status Init(int64_t capacity) override;
 
   /// Increase the capacity of the builder to accommodate at least the indicated
@@ -423,7 +432,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
   Status Append(const uint64_t* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
 
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
  protected:
   Status ExpandIntSize(uint8_t new_int_size);
@@ -485,7 +494,7 @@ class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase
   Status Append(const int64_t* values, int64_t length,
                 const uint8_t* valid_bytes = nullptr);
 
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
  protected:
   Status ExpandIntSize(uint8_t new_int_size);
@@ -582,7 +591,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   /// \return Status
   Status Append(const std::vector<bool>& values);
 
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
   Status Init(int64_t capacity) override;
 
   /// Increase the capacity of the builder to accommodate at least the indicated
@@ -619,7 +628,7 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
 
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
   /// \brief Vector append
   ///
@@ -673,7 +682,7 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
 
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
   /// \return size of values buffer so far
   int64_t value_data_length() const { return value_data_builder_.length(); }
@@ -690,7 +699,6 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
   static constexpr int64_t kMaximumCapacity = std::numeric_limits<int32_t>::max() - 1;
 
   Status AppendNextOffset();
-  Status FinishInternal(std::shared_ptr<ArrayData>* out);
   void Reset();
 };
 
@@ -703,8 +711,6 @@ class ARROW_EXPORT StringBuilder : public BinaryBuilder {
 
   using BinaryBuilder::Append;
 
-  Status Finish(std::shared_ptr<Array>* out) override;
-
   Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
 };
 
@@ -732,7 +738,7 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
   /// \return size of values buffer so far
   int64_t value_data_length() const { return byte_builder_.length(); }
@@ -756,7 +762,7 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
 
   Status Append(const Decimal128& val);
 
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 };
 
 // ----------------------------------------------------------------------
@@ -772,7 +778,7 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
   StructBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool,
                 std::vector<std::unique_ptr<ArrayBuilder>>&& field_builders);
 
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
   /// Null bitmap is of equal length to every child field, and any zero byte
   /// will be considered as a null for that field, but users must using app-
@@ -875,7 +881,7 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
 
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
-  Status Finish(std::shared_ptr<Array>* out) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
  protected:
   Status DoubleTableSize();

From 47e6ff6cf19a9d84d15f24715f4ddb87aa226d50 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Thu, 12 Oct 2017 18:16:34 -0400
Subject: [PATCH 1146/1644] ARROW-1665: [Serialization] Support more custom
 datatypes in the default serialization context

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1194 from pcmoritz/default-serialization-context and squashes the following commits:

43a4add8 [Wes McKinney] NumPy is hard requirement
7db591b8 [Philipp Moritz] update
20d75baf [Philipp Moritz] make custom serialization handlers accessible outside of the tests
---
 python/pyarrow/__init__.py                 |   2 +
 python/pyarrow/ipc.py                      |  47 --------
 python/pyarrow/serialization.py            | 126 +++++++++++++++++++++
 python/pyarrow/tests/test_serialization.py |  73 ++----------
 4 files changed, 140 insertions(+), 108 deletions(-)
 create mode 100644 python/pyarrow/serialization.py

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index f956347ac2947..e37c123d24d38 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -116,6 +116,8 @@
 
 localfs = LocalFileSystem.get_instance()
 
+from pyarrow.serialization import _default_serialization_context
+
 import pyarrow.types as types
 
 # Entry point for starting the plasma store
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index 122367346fc1e..f264f089c4071 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -187,50 +187,3 @@ def deserialize_pandas(buf, nthreads=None):
     reader = pa.RecordBatchStreamReader(buffer_reader)
     table = reader.read_all()
     return table.to_pandas(nthreads=nthreads)
-
-
-# ----------------------------------------------------------------------
-# Set up default serialization context
-
-def _serialize_pandas_series(s):
-    import pandas as pd
-    # TODO: serializing Series without extra copy
-    serialized = serialize_pandas(pd.DataFrame({s.name: s}))
-    return {
-        'type': 'Series',
-        'data': serialized.to_pybytes()
-    }
-
-
-def _serialize_pandas_dataframe(df):
-    return {
-        'type': 'DataFrame',
-        'data': serialize_pandas(df).to_pybytes()
-    }
-
-
-def _deserialize_callback_pandas(data):
-    deserialized = deserialize_pandas(data['data'])
-    type_ = data['type']
-    if type_ == 'Series':
-        return deserialized[deserialized.columns[0]]
-    elif type_ == 'DataFrame':
-        return deserialized
-    else:
-        raise ValueError(type_)
-
-
-try:
-    import pandas as pd
-    lib._default_serialization_context.register_type(
-        pd.Series, 'pandas.Series',
-        custom_serializer=_serialize_pandas_series,
-        custom_deserializer=_deserialize_callback_pandas)
-
-    lib._default_serialization_context.register_type(
-        pd.DataFrame, 'pandas.DataFrame',
-        custom_serializer=_serialize_pandas_dataframe,
-        custom_deserializer=_deserialize_callback_pandas)
-except ImportError:
-    # no pandas
-    pass
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
new file mode 100644
index 0000000000000..d08ae89a5930b
--- /dev/null
+++ b/python/pyarrow/serialization.py
@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections import OrderedDict, defaultdict
+import sys
+
+import numpy as np
+
+from pyarrow import serialize_pandas, deserialize_pandas
+from pyarrow.lib import _default_serialization_context
+
+# ----------------------------------------------------------------------
+# Set up serialization for primitive datatypes
+
+# TODO(pcm): This is currently a workaround until arrow supports
+# arbitrary precision integers. This is only called on long integers,
+# see the associated case in the append method in python_to_arrow.cc
+_default_serialization_context.register_type(
+    int, "int",
+    custom_serializer=lambda obj: str(obj),
+    custom_deserializer=lambda data: int(data))
+
+if (sys.version_info < (3, 0)):
+    _default_serialization_context.register_type(
+        long, "long",  # noqa: F821
+        custom_serializer=lambda obj: str(obj),
+        custom_deserializer=lambda data: long(data))  # noqa: F821
+
+
+def _serialize_ordered_dict(obj):
+    return list(obj.keys()), list(obj.values())
+
+
+def _deserialize_ordered_dict(data):
+    return OrderedDict(zip(data[0], data[1]))
+
+
+_default_serialization_context.register_type(
+    OrderedDict, "OrderedDict",
+    custom_serializer=_serialize_ordered_dict,
+    custom_deserializer=_deserialize_ordered_dict)
+
+
+def _serialize_default_dict(obj):
+    return list(obj.keys()), list(obj.values()), obj.default_factory
+
+
+def _deserialize_default_dict(data):
+    return defaultdict(data[2], zip(data[0], data[1]))
+
+
+_default_serialization_context.register_type(
+     defaultdict, "defaultdict",
+     custom_serializer=_serialize_default_dict,
+     custom_deserializer=_deserialize_default_dict)
+
+
+_default_serialization_context.register_type(
+     type(lambda: 0), "function",
+     pickle=True)
+
+# ----------------------------------------------------------------------
+# Set up serialization for numpy with dtype object (primitive types are
+# handled efficiently with Arrow's Tensor facilities, see python_to_arrow.cc)
+
+
+def _serialize_numpy_array(obj):
+    return obj.tolist(), obj.dtype.str
+
+
+def _deserialize_numpy_array(data):
+    return np.array(data[0], dtype=np.dtype(data[1]))
+
+
+_default_serialization_context.register_type(
+    np.ndarray, 'np.array',
+    custom_serializer=_serialize_numpy_array,
+    custom_deserializer=_deserialize_numpy_array)
+
+
+# ----------------------------------------------------------------------
+# Set up serialization for pandas Series and DataFrame
+
+try:
+    import pandas as pd
+
+    def _serialize_pandas_series(obj):
+        # TODO: serializing Series without extra copy
+        return serialize_pandas(pd.DataFrame({obj.name: obj})).to_pybytes()
+
+    def _deserialize_pandas_series(data):
+        deserialized = deserialize_pandas(data)
+        return deserialized[deserialized.columns[0]]
+
+    def _serialize_pandas_dataframe(obj):
+        return serialize_pandas(obj).to_pybytes()
+
+    def _deserialize_pandas_dataframe(data):
+        return deserialize_pandas(data)
+
+    _default_serialization_context.register_type(
+        pd.Series, 'pd.Series',
+        custom_serializer=_serialize_pandas_series,
+        custom_deserializer=_deserialize_pandas_series)
+
+    _default_serialization_context.register_type(
+        pd.DataFrame, 'pd.DataFrame',
+        custom_serializer=_serialize_pandas_dataframe,
+        custom_deserializer=_deserialize_pandas_dataframe)
+except ImportError:
+    # no pandas
+    pass
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 5441b9fb4ffcb..a9fd10243a611 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -181,67 +181,18 @@ class CustomError(Exception):
 
 def make_serialization_context():
 
-    def array_custom_serializer(obj):
-        return obj.tolist(), obj.dtype.str
-
-    def array_custom_deserializer(serialized_obj):
-        return np.array(serialized_obj[0], dtype=np.dtype(serialized_obj[1]))
-
-    context = pa.SerializationContext()
-
-    # This is for numpy arrays of "object" only; primitive types are handled
-    # efficiently with Arrow's Tensor facilities (see python_to_arrow.cc)
-    context.register_type(np.ndarray, 20 * b"\x00",
-                          custom_serializer=array_custom_serializer,
-                          custom_deserializer=array_custom_deserializer)
-
-    context.register_type(Foo, 20 * b"\x01")
-    context.register_type(Bar, 20 * b"\x02")
-    context.register_type(Baz, 20 * b"\x03")
-    context.register_type(Qux, 20 * b"\x04")
-    context.register_type(SubQux, 20 * b"\x05")
-    context.register_type(SubQuxPickle, 20 * b"\x05", pickle=True)
-    context.register_type(Exception, 20 * b"\x06")
-    context.register_type(CustomError, 20 * b"\x07")
-    context.register_type(Point, 20 * b"\x08")
-    context.register_type(NamedTupleExample, 20 * b"\x09")
-
-    # TODO(pcm): This is currently a workaround until arrow supports
-    # arbitrary precision integers. This is only called on long integers,
-    # see the associated case in the append method in python_to_arrow.cc
-    context.register_type(int, 20 * b"\x10", pickle=False,
-                          custom_serializer=lambda obj: str(obj),
-                          custom_deserializer=(
-                              lambda serialized_obj: int(serialized_obj)))
-
-    if (sys.version_info < (3, 0)):
-        deserializer = (
-            lambda serialized_obj: long(serialized_obj))  # noqa: E501,F821
-        context.register_type(long, 20 * b"\x11", pickle=False,  # noqa: E501,F821
-                              custom_serializer=lambda obj: str(obj),
-                              custom_deserializer=deserializer)
-
-    def ordered_dict_custom_serializer(obj):
-        return list(obj.keys()), list(obj.values())
-
-    def ordered_dict_custom_deserializer(obj):
-        return OrderedDict(zip(obj[0], obj[1]))
-
-    context.register_type(OrderedDict, 20 * b"\x12", pickle=False,
-                          custom_serializer=ordered_dict_custom_serializer,
-                          custom_deserializer=ordered_dict_custom_deserializer)
-
-    def default_dict_custom_serializer(obj):
-        return list(obj.keys()), list(obj.values()), obj.default_factory
-
-    def default_dict_custom_deserializer(obj):
-        return defaultdict(obj[2], zip(obj[0], obj[1]))
-
-    context.register_type(defaultdict, 20 * b"\x13", pickle=False,
-                          custom_serializer=default_dict_custom_serializer,
-                          custom_deserializer=default_dict_custom_deserializer)
-
-    context.register_type(type(lambda: 0), 20 * b"\x14", pickle=True)
+    context = pa._default_serialization_context
+
+    context.register_type(Foo, "Foo")
+    context.register_type(Bar, "Bar")
+    context.register_type(Baz, "Baz")
+    context.register_type(Qux, "Quz")
+    context.register_type(SubQux, "SubQux")
+    context.register_type(SubQuxPickle, "SubQuxPickle", pickle=True)
+    context.register_type(Exception, "Exception")
+    context.register_type(CustomError, "CustomError")
+    context.register_type(Point, "Point")
+    context.register_type(NamedTupleExample, "NamedTupleExample")
 
     return context
 

From dc533211a88c9b43d250841d5cba4261af01035e Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Fri, 13 Oct 2017 15:34:19 -0400
Subject: [PATCH 1147/1644] ARROW-1648: C++: Add cast from Dictionary[NullType]
 to NullType

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1189 from xhochy/ARROW-1648 and squashes the following commits:

21c2e33e [Wes McKinney] Resolve rebase conflicts
8d531c82 [Korn, Uwe] Mark single argument constructor explicit
68f180f4 [Korn, Uwe] ARROW-1648: C++: Add cast from Dictionary[NullType] to NullType
---
 cpp/src/arrow/builder.cc              |  46 ++++++
 cpp/src/arrow/builder.h               |  22 +++
 cpp/src/arrow/compute/cast.cc         |  35 ++--
 cpp/src/arrow/compute/compute-test.cc |   2 +-
 cpp/src/arrow/python/util/datetime.h  | 219 ++++++++++++--------------
 cpp/src/arrow/test-common.h           |   6 +
 6 files changed, 202 insertions(+), 128 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 7152c7aa75940..076c156b0af5d 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -830,6 +830,16 @@ DictionaryBuilder<T>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
   }
 }
 
+DictionaryBuilder<NullType>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
+                                               MemoryPool* pool)
+    : ArrayBuilder(type, pool), values_builder_(pool) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
+
+DictionaryBuilder<NullType>::~DictionaryBuilder() {}
+
 template <>
 DictionaryBuilder<FixedSizeBinaryType>::DictionaryBuilder(
     const std::shared_ptr<DataType>& type, MemoryPool* pool)
@@ -858,6 +868,11 @@ Status DictionaryBuilder<T>::Init(int64_t elements) {
   return values_builder_.Init(elements);
 }
 
+Status DictionaryBuilder<NullType>::Init(int64_t elements) {
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+  return values_builder_.Init(elements);
+}
+
 template <typename T>
 Status DictionaryBuilder<T>::Resize(int64_t capacity) {
   if (capacity < kMinBuilderCapacity) {
@@ -871,6 +886,18 @@ Status DictionaryBuilder<T>::Resize(int64_t capacity) {
   }
 }
 
+Status DictionaryBuilder<NullType>::Resize(int64_t capacity) {
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
+
+  if (capacity_ == 0) {
+    return Init(capacity);
+  } else {
+    return ArrayBuilder::Resize(capacity);
+  }
+}
+
 template <typename T>
 Status DictionaryBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Array> dictionary;
@@ -881,6 +908,14 @@ Status DictionaryBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
   return Status::OK();
 }
 
+Status DictionaryBuilder<NullType>::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  std::shared_ptr<Array> dictionary = std::make_shared<NullArray>(0);
+
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
+  return Status::OK();
+}
+
 template <typename T>
 Status DictionaryBuilder<T>::Append(const Scalar& value) {
   RETURN_NOT_OK(Reserve(1));
@@ -928,6 +963,13 @@ Status DictionaryBuilder<T>::AppendArray(const Array& array) {
   return Status::OK();
 }
 
+Status DictionaryBuilder<NullType>::AppendArray(const Array& array) {
+  for (int64_t i = 0; i < array.length(); i++) {
+    RETURN_NOT_OK(AppendNull());
+  }
+  return Status::OK();
+}
+
 template <>
 Status DictionaryBuilder<FixedSizeBinaryType>::AppendArray(const Array& array) {
   if (!type_->Equals(*array.type())) {
@@ -950,6 +992,8 @@ Status DictionaryBuilder<T>::AppendNull() {
   return values_builder_.AppendNull();
 }
 
+Status DictionaryBuilder<NullType>::AppendNull() { return values_builder_.AppendNull(); }
+
 template <typename T>
 Status DictionaryBuilder<T>::DoubleTableSize() {
   int new_size = hash_table_size_ * 2;
@@ -1438,6 +1482,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
 Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
                              std::shared_ptr<ArrayBuilder>* out) {
   switch (type->id()) {
+    DICTIONARY_BUILDER_CASE(NA, DictionaryBuilder<NullType>);
     DICTIONARY_BUILDER_CASE(UINT8, DictionaryBuilder<UInt8Type>);
     DICTIONARY_BUILDER_CASE(INT8, DictionaryBuilder<Int8Type>);
     DICTIONARY_BUILDER_CASE(UINT16, DictionaryBuilder<UInt16Type>);
@@ -1474,6 +1519,7 @@ Status EncodeArrayToDictionary(const Array& input, MemoryPool* pool,
   const std::shared_ptr<DataType>& type = input.data()->type;
   std::shared_ptr<ArrayBuilder> builder;
   switch (type->id()) {
+    DICTIONARY_ARRAY_CASE(NA, DictionaryBuilder<NullType>);
     DICTIONARY_ARRAY_CASE(UINT8, DictionaryBuilder<UInt8Type>);
     DICTIONARY_ARRAY_CASE(INT8, DictionaryBuilder<Int8Type>);
     DICTIONARY_ARRAY_CASE(UINT16, DictionaryBuilder<UInt16Type>);
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 54d11cf0fb1f2..1720c00145c4e 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -905,6 +905,28 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
   int32_t byte_width_;
 };
 
+template <>
+class ARROW_EXPORT DictionaryBuilder<NullType> : public ArrayBuilder {
+ public:
+  ~DictionaryBuilder();
+
+  DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
+  explicit DictionaryBuilder(MemoryPool* pool);
+
+  /// \brief Append a scalar null value
+  Status AppendNull();
+
+  /// \brief Append a whole dense array to the builder
+  Status AppendArray(const Array& array);
+
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
+
+ protected:
+  AdaptiveIntBuilder values_builder_;
+};
+
 class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
  public:
   using DictionaryBuilder::Append;
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index 149cc36b61597..2381e1ea3a4e8 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -123,6 +123,12 @@ struct CastFunctor<T, NullType, typename std::enable_if<
   }
 };
 
+template <>
+struct CastFunctor<NullType, DictionaryType> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {}
+};
+
 // ----------------------------------------------------------------------
 // Boolean to other things
 
@@ -499,23 +505,25 @@ static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input
       return Status::NotImplemented(ss.str());
     }
 
-    const auto& fw_type = static_cast<const FixedWidthType&>(*out->type);
+    if (type_id != Type::NA) {
+      const auto& fw_type = static_cast<const FixedWidthType&>(*out->type);
 
-    int bit_width = fw_type.bit_width();
-    int64_t buffer_size = 0;
+      int bit_width = fw_type.bit_width();
+      int64_t buffer_size = 0;
 
-    if (bit_width == 1) {
-      buffer_size = BitUtil::BytesForBits(length);
-    } else if (bit_width % 8 == 0) {
-      buffer_size = length * fw_type.bit_width() / 8;
-    } else {
-      DCHECK(false);
-    }
+      if (bit_width == 1) {
+        buffer_size = BitUtil::BytesForBits(length);
+      } else if (bit_width % 8 == 0) {
+        buffer_size = length * fw_type.bit_width() / 8;
+      } else {
+        DCHECK(false);
+      }
 
-    RETURN_NOT_OK(ctx->Allocate(buffer_size, &out_data));
-    memset(out_data->mutable_data(), 0, buffer_size);
+      RETURN_NOT_OK(ctx->Allocate(buffer_size, &out_data));
+      memset(out_data->mutable_data(), 0, buffer_size);
 
-    out->buffers.push_back(out_data);
+      out->buffers.push_back(out_data);
+    }
   }
 
   return Status::OK();
@@ -601,6 +609,7 @@ class CastKernel : public UnaryKernel {
 #define TIMESTAMP_CASES(FN, IN_TYPE) FN(TimestampType, TimestampType);
 
 #define DICTIONARY_CASES(FN, IN_TYPE) \
+  FN(IN_TYPE, NullType);              \
   FN(IN_TYPE, Time32Type);            \
   FN(IN_TYPE, Date32Type);            \
   FN(IN_TYPE, TimestampType);         \
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index a4b502d2ae613..602acff2088d6 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -379,7 +379,7 @@ TEST_F(TestCast, PreallocatedMemory) {
 template <typename TestType>
 class TestDictionaryCast : public TestCast {};
 
-typedef ::testing::Types<UInt8Type, Int8Type, UInt16Type, Int16Type, Int32Type,
+typedef ::testing::Types<NullType, UInt8Type, Int8Type, UInt16Type, Int16Type, Int32Type,
                          UInt32Type, UInt64Type, Int64Type, FloatType, DoubleType,
                          Date32Type, Date64Type, FixedSizeBinaryType, BinaryType>
     TestTypes;
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index 01fbc18e6ed26..782960f62518e 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -22,9 +22,9 @@
 #include <sstream>
 
 #include <datetime.h>
+#include "arrow/python/platform.h"
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
-#include "arrow/python/platform.h"
 
 namespace arrow {
 namespace py {
@@ -34,131 +34,126 @@ namespace py {
 
 // Days per month, regular year and leap year
 static int64_t _days_per_month_table[2][12] = {
-    { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 },
-    { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }
-};
+    {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31},
+    {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}};
 
 static bool is_leapyear(int64_t year) {
-    return (year & 0x3) == 0 && // year % 4 == 0
-           ((year % 100) != 0 ||
-            (year % 400) == 0);
+  return (year & 0x3) == 0 &&  // year % 4 == 0
+         ((year % 100) != 0 || (year % 400) == 0);
 }
 
 // Calculates the days offset from the 1970 epoch.
-static int64_t get_days_from_date(int64_t date_year,
-                                  int64_t date_month,
+static int64_t get_days_from_date(int64_t date_year, int64_t date_month,
                                   int64_t date_day) {
-    int64_t i, month;
-    int64_t year, days = 0;
-    int64_t *month_lengths;
-
-    year = date_year - 1970;
-    days = year * 365;
-
-    // Adjust for leap years
-    if (days >= 0) {
-        // 1968 is the closest leap year before 1970.
-        // Exclude the current year, so add 1.
-        year += 1;
-        // Add one day for each 4 years
-        days += year / 4;
-        // 1900 is the closest previous year divisible by 100
-        year += 68;
-        // Subtract one day for each 100 years
-        days -= year / 100;
-        // 1600 is the closest previous year divisible by 400
-        year += 300;
-        // Add one day for each 400 years
-        days += year / 400;
-    } else {
-        // 1972 is the closest later year after 1970.
-        // Include the current year, so subtract 2.
-        year -= 2;
-        // Subtract one day for each 4 years
-        days += year / 4;
-        // 2000 is the closest later year divisible by 100
-        year -= 28;
-        // Add one day for each 100 years
-        days -= year / 100;
-        // 2000 is also the closest later year divisible by 400
-        // Subtract one day for each 400 years
-        days += year / 400;
-    }
+  int64_t i, month;
+  int64_t year, days = 0;
+  int64_t* month_lengths;
+
+  year = date_year - 1970;
+  days = year * 365;
+
+  // Adjust for leap years
+  if (days >= 0) {
+    // 1968 is the closest leap year before 1970.
+    // Exclude the current year, so add 1.
+    year += 1;
+    // Add one day for each 4 years
+    days += year / 4;
+    // 1900 is the closest previous year divisible by 100
+    year += 68;
+    // Subtract one day for each 100 years
+    days -= year / 100;
+    // 1600 is the closest previous year divisible by 400
+    year += 300;
+    // Add one day for each 400 years
+    days += year / 400;
+  } else {
+    // 1972 is the closest later year after 1970.
+    // Include the current year, so subtract 2.
+    year -= 2;
+    // Subtract one day for each 4 years
+    days += year / 4;
+    // 2000 is the closest later year divisible by 100
+    year -= 28;
+    // Add one day for each 100 years
+    days -= year / 100;
+    // 2000 is also the closest later year divisible by 400
+    // Subtract one day for each 400 years
+    days += year / 400;
+  }
 
-    month_lengths = _days_per_month_table[is_leapyear(date_year)];
-    month = date_month - 1;
+  month_lengths = _days_per_month_table[is_leapyear(date_year)];
+  month = date_month - 1;
 
-    // Add the months
-    for (i = 0; i < month; ++i) {
-        days += month_lengths[i];
-    }
+  // Add the months
+  for (i = 0; i < month; ++i) {
+    days += month_lengths[i];
+  }
 
-    // Add the days
-    days += date_day - 1;
+  // Add the days
+  days += date_day - 1;
 
-    return days;
+  return days;
 }
 
 // Modifies '*days_' to be the day offset within the year,
 // and returns the year.
 static int64_t days_to_yearsdays(int64_t* days_) {
-    const int64_t days_per_400years = (400*365 + 100 - 4 + 1);
-    // Adjust so it's relative to the year 2000 (divisible by 400)
-    int64_t days = (*days_) - (365*30 + 7);
-    int64_t year;
-
-    // Break down the 400 year cycle to get the year and day within the year
-    if (days >= 0) {
-        year = 400 * (days / days_per_400years);
-        days = days % days_per_400years;
-    } else {
-        year = 400 * ((days - (days_per_400years - 1)) / days_per_400years);
-        days = days % days_per_400years;
-        if (days < 0) {
-            days += days_per_400years;
-        }
+  const int64_t days_per_400years = (400 * 365 + 100 - 4 + 1);
+  // Adjust so it's relative to the year 2000 (divisible by 400)
+  int64_t days = (*days_) - (365 * 30 + 7);
+  int64_t year;
+
+  // Break down the 400 year cycle to get the year and day within the year
+  if (days >= 0) {
+    year = 400 * (days / days_per_400years);
+    days = days % days_per_400years;
+  } else {
+    year = 400 * ((days - (days_per_400years - 1)) / days_per_400years);
+    days = days % days_per_400years;
+    if (days < 0) {
+      days += days_per_400years;
     }
+  }
 
-    // Work out the year/day within the 400 year cycle
-    if (days >= 366) {
-        year += 100 * ((days-1) / (100*365 + 25 - 1));
-        days = (days-1) % (100*365 + 25 - 1);
-        if (days >= 365) {
-            year += 4 * ((days+1) / (4*365 + 1));
-            days = (days+1) % (4*365 + 1);
-            if (days >= 366) {
-                year += (days-1) / 365;
-                days = (days-1) % 365;
-            }
-        }
+  // Work out the year/day within the 400 year cycle
+  if (days >= 366) {
+    year += 100 * ((days - 1) / (100 * 365 + 25 - 1));
+    days = (days - 1) % (100 * 365 + 25 - 1);
+    if (days >= 365) {
+      year += 4 * ((days + 1) / (4 * 365 + 1));
+      days = (days + 1) % (4 * 365 + 1);
+      if (days >= 366) {
+        year += (days - 1) / 365;
+        days = (days - 1) % 365;
+      }
     }
+  }
 
-    *days_ = days;
-    return year + 2000;
+  *days_ = days;
+  return year + 2000;
 }
 
 // Extracts the month and year and day number from a number of days
-static void get_date_from_days(int64_t days,
-                               int64_t* date_year,
-                               int64_t* date_month,
+static void get_date_from_days(int64_t days, int64_t* date_year, int64_t* date_month,
                                int64_t* date_day) {
-    int64_t *month_lengths, i;
-
-    *date_year = days_to_yearsdays(&days);
-    month_lengths = _days_per_month_table[is_leapyear(*date_year)];
-
-    for (i = 0; i < 12; ++i) {
-        if (days < month_lengths[i]) {
-            *date_month = i + 1;
-            *date_day = days + 1;
-            return;
-        } else {
-            days -= month_lengths[i];
-        }
+  int64_t *month_lengths, i;
+
+  *date_year = days_to_yearsdays(&days);
+  month_lengths = _days_per_month_table[is_leapyear(*date_year)];
+
+  for (i = 0; i < 12; ++i) {
+    if (days < month_lengths[i]) {
+      *date_month = i + 1;
+      *date_day = days + 1;
+      return;
+    } else {
+      days -= month_lengths[i];
     }
+  }
 
-    // Should never get here
-    return;
+  // Should never get here
+  return;
 }
 
 static inline int64_t PyTime_to_us(PyObject* pytime) {
@@ -168,7 +163,6 @@ static inline int64_t PyTime_to_us(PyObject* pytime) {
           PyDateTime_TIME_GET_MICROSECOND(pytime));
 }
 
-
 // Splitting time quantities, for example splitting total seconds into
 // minutes and remaining seconds. After we run
 // int64_t remaining = split_time(total, quotient, &next)
@@ -188,8 +182,8 @@ static inline int64_t split_time(int64_t total, int64_t quotient, int64_t* next)
 }
 
 static inline Status PyTime_convert_int(int64_t val, const TimeUnit::type unit,
-                                        int64_t *hour, int64_t *minute,
-                                        int64_t *second, int64_t *microsecond) {
+                                        int64_t* hour, int64_t* minute, int64_t* second,
+                                        int64_t* microsecond) {
   switch (unit) {
     case TimeUnit::NANO:
       if (val % 1000 != 0) {
@@ -234,13 +228,10 @@ static inline Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit,
   hour = split_time(hour, 24, &total_days);
   int64_t year = 0, month = 0, day = 0;
   get_date_from_days(total_days, &year, &month, &day);
-  *out = PyDateTime_FromDateAndTime(static_cast<int32_t>(year),
-                                    static_cast<int32_t>(month),
-                                    static_cast<int32_t>(day),
-                                    static_cast<int32_t>(hour),
-                                    static_cast<int32_t>(minute),
-                                    static_cast<int32_t>(second),
-                                    static_cast<int32_t>(microsecond));
+  *out = PyDateTime_FromDateAndTime(
+      static_cast<int32_t>(year), static_cast<int32_t>(month), static_cast<int32_t>(day),
+      static_cast<int32_t>(hour), static_cast<int32_t>(minute),
+      static_cast<int32_t>(second), static_cast<int32_t>(microsecond));
   return Status::OK();
 }
 
@@ -249,9 +240,9 @@ static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   total_seconds += PyDateTime_DATE_GET_SECOND(pydate);
   total_seconds += PyDateTime_DATE_GET_MINUTE(pydate) * 60;
   total_seconds += PyDateTime_DATE_GET_HOUR(pydate) * 3600;
-  int64_t days = get_days_from_date(PyDateTime_GET_YEAR(pydate),
-                                    PyDateTime_GET_MONTH(pydate),
-                                    PyDateTime_GET_DAY(pydate));
+  int64_t days =
+      get_days_from_date(PyDateTime_GET_YEAR(pydate), PyDateTime_GET_MONTH(pydate),
+                         PyDateTime_GET_DAY(pydate));
   total_seconds += days * 24 * 3600;
   return total_seconds * 1000;
 }
diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h
index 3dc39fc3d07ca..a4c4fddff7348 100644
--- a/cpp/src/arrow/test-common.h
+++ b/cpp/src/arrow/test-common.h
@@ -74,6 +74,12 @@ std::shared_ptr<Array> TestBase::MakeRandomArray(int64_t length, int64_t null_co
   return std::make_shared<ArrayType>(length, data, null_bitmap, null_count);
 }
 
+template <>
+std::shared_ptr<Array> TestBase::MakeRandomArray<NullArray>(int64_t length,
+                                                            int64_t null_count) {
+  return std::make_shared<NullArray>(length);
+}
+
 template <>
 std::shared_ptr<Array> TestBase::MakeRandomArray<FixedSizeBinaryArray>(
     int64_t length, int64_t null_count) {

From 894f7400977693b4e0e8f4b9845fd89481f6bf29 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 14 Oct 2017 11:46:36 -0400
Subject: [PATCH 1148/1644] ARROW-1670: [Serialization] Speed up
 deserialization by getting rid of smart pointer overhead

This optimization makes our deserialization codepath faster than pickle pretty much across the board.

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1197 from pcmoritz/deserialization-speedup and squashes the following commits:

88cf3ea1 [Wes McKinney] clang-format
c1ddbcd5 [Philipp Moritz] introduce unsafe child function for UnionArray
ae0a7202 [Philipp Moritz] speed up deserialization by getting rid of smart pointer overhead
---
 cpp/src/arrow/array.cc                  |  8 ++++
 cpp/src/arrow/array.h                   |  3 ++
 cpp/src/arrow/python/arrow_to_python.cc | 52 ++++++++++++-------------
 3 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index cd1721f07fe9e..a7930a139c75f 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -375,6 +375,14 @@ std::shared_ptr<Array> UnionArray::child(int i) const {
   return boxed_fields_[i];
 }
 
+const Array* UnionArray::UnsafeChild(int i) const {
+  if (!boxed_fields_[i]) {
+    DCHECK(MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
+  }
+  DCHECK(boxed_fields_[i]);
+  return boxed_fields_[i].get();
+}
+
 // ----------------------------------------------------------------------
 // DictionaryArray
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 36bceeb8f4aa0..0805cad3d286c 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -610,6 +610,9 @@ class ARROW_EXPORT UnionArray : public Array {
 
   std::shared_ptr<Array> child(int pos) const;
 
+  /// Only use this while the UnionArray is in scope
+  const Array* UnsafeChild(int pos) const;
+
  protected:
   void SetData(const std::shared_ptr<ArrayData>& data);
 
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index de05a230cab4f..ac459d4228710 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -163,26 +163,25 @@ Status GetValue(PyObject* context, const Array& arr, int64_t index, int32_t type
   return Status::OK();
 }
 
-#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                                  \
-  const auto& data = static_cast<const UnionArray&>(array);                           \
-  int64_t size = array.length();                                                      \
-  ScopedRef result(CREATE_FN(stop_idx - start_idx));                                  \
-  auto types = std::make_shared<Int8Array>(size, data.type_ids());                    \
-  auto offsets = std::make_shared<Int32Array>(size, data.value_offsets());            \
-  for (int64_t i = start_idx; i < stop_idx; ++i) {                                    \
-    if (data.IsNull(i)) {                                                             \
-      Py_INCREF(Py_None);                                                             \
-      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                              \
-    } else {                                                                          \
-      int64_t offset = offsets->Value(i);                                             \
-      int8_t type = types->Value(i);                                                  \
-      PyObject* value;                                                                \
-      RETURN_NOT_OK(                                                                  \
-          GetValue(context, *data.child(type), offset, type, base, tensors, &value)); \
-      SET_ITEM_FN(result.get(), i - start_idx, value);                                \
-    }                                                                                 \
-  }                                                                                   \
-  *out = result.release();                                                            \
+#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                               \
+  const auto& data = static_cast<const UnionArray&>(array);                        \
+  ScopedRef result(CREATE_FN(stop_idx - start_idx));                               \
+  const uint8_t* type_ids = data.raw_type_ids();                                   \
+  const int32_t* value_offsets = data.raw_value_offsets();                         \
+  for (int64_t i = start_idx; i < stop_idx; ++i) {                                 \
+    if (data.IsNull(i)) {                                                          \
+      Py_INCREF(Py_None);                                                          \
+      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                           \
+    } else {                                                                       \
+      int64_t offset = value_offsets[i];                                           \
+      uint8_t type = type_ids[i];                                                  \
+      PyObject* value;                                                             \
+      RETURN_NOT_OK(GetValue(context, *data.UnsafeChild(type), offset, type, base, \
+                             tensors, &value));                                    \
+      SET_ITEM_FN(result.get(), i - start_idx, value);                             \
+    }                                                                              \
+  }                                                                                \
+  *out = result.release();                                                         \
   return Status::OK()
 
 Status DeserializeList(PyObject* context, const Array& array, int64_t start_idx,
@@ -204,10 +203,9 @@ Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
                       const std::vector<std::shared_ptr<Tensor>>& tensors,
                       PyObject** out) {
   const auto& data = static_cast<const UnionArray&>(array);
-  int64_t size = array.length();
   ScopedRef result(PySet_New(nullptr));
-  auto types = std::make_shared<Int8Array>(size, data.type_ids());
-  auto offsets = std::make_shared<Int32Array>(size, data.value_offsets());
+  const uint8_t* type_ids = data.raw_type_ids();
+  const int32_t* value_offsets = data.raw_value_offsets();
   for (int64_t i = start_idx; i < stop_idx; ++i) {
     if (data.IsNull(i)) {
       Py_INCREF(Py_None);
@@ -215,11 +213,11 @@ Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
         RETURN_IF_PYERROR();
       }
     } else {
-      int64_t offset = offsets->Value(i);
-      int8_t type = types->Value(i);
+      int32_t offset = value_offsets[i];
+      int8_t type = type_ids[i];
       PyObject* value;
-      RETURN_NOT_OK(
-          GetValue(context, *data.child(type), offset, type, base, tensors, &value));
+      RETURN_NOT_OK(GetValue(context, *data.UnsafeChild(type), offset, type, base,
+                             tensors, &value));
       if (PySet_Add(result.get(), value) < 0) {
         RETURN_IF_PYERROR();
       }

From a6a97a9d4c07873266a71d8c87069dc4d168e4d2 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sat, 14 Oct 2017 11:47:58 -0400
Subject: [PATCH 1149/1644] ARROW-1631 [C++] Add GRPC to ThirdpartyToolchain

- Building of GRPC and linking to GRPC's libs is turned on by default ( building of GRPC takes too much CPU time; It makes sense to find the way to decrease build time of GRPC; Maybe, some patch to adjust their CMake builds script to not build not relevant targets like various languages support plugins )
- Searching for pre-installed GRPC package with `find_package(gRPC CONFIG REQUIRED) ` doesn't work correctly yet.

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #1182 from MaxRis/ARROW-1631 and squashes the following commits:

25b651bf [Max Risuhin] ARROW-1631: [C++] Add GRPC to ThirdpartyToolchain
---
 cpp/CMakeLists.txt                          | 12 +++++
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 51 +++++++++++++++++++++
 2 files changed, 63 insertions(+)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 12d48549e3b6c..a159b1e5674b4 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -190,6 +190,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build with zstd compression"
     ON)
 
+  option(ARROW_WITH_GRPC
+    "Build with GRPC"
+    OFF)
+
   option(ARROW_VERBOSE_THIRDPARTY_BUILD
     "If off, output from ExternalProjects will be logged to files rather than shown"
     ON)
@@ -510,6 +514,14 @@ if (ARROW_WITH_ZSTD)
   SET(ARROW_STATIC_LINK_LIBS zstd_static ${ARROW_STATIC_LINK_LIBS})
 endif()
 
+if (ARROW_WITH_GRPC)
+  SET(ARROW_STATIC_LINK_LIBS
+    grpc_grp
+    grpc_grpc
+    grpc_grpcpp
+    ${ARROW_STATIC_LINK_LIBS})
+endif()
+
 if (ARROW_STATIC_LINK_LIBS)
   add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
 endif()
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 98186d01415bc..42d7eddc9c9d9 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -29,6 +29,7 @@ set(SNAPPY_VERSION "1.1.3")
 set(BROTLI_VERSION "v0.6.0")
 set(LZ4_VERSION "1.7.5")
 set(ZSTD_VERSION "1.2.0")
+set(GRPC_VERSION "94582910ad7f82ad447ecc72e6548cb669e4f7a9") # v1.6.5
 
 string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
 
@@ -105,6 +106,10 @@ if (DEFINED ENV{ZSTD_HOME})
   set(ZSTD_HOME "$ENV{ZSTD_HOME}")
 endif()
 
+if (DEFINED ENV{GRPC_HOME})
+  set(GRPC_HOME "$ENV{GRPC_HOME}")
+endif()
+
 # Ensure that a default make is set
 if ("${MAKE}" STREQUAL "")
     if (NOT MSVC)
@@ -782,3 +787,49 @@ if (ARROW_WITH_ZSTD)
     add_dependencies(zstd_static zstd_ep)
   endif()
 endif()
+
+if (ARROW_WITH_GRPC)
+# ----------------------------------------------------------------------
+# GRPC
+  if ("${GRPC_HOME}" STREQUAL "")
+    set(GRPC_VENDORED 1)
+    set(GRPC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep-build")
+    set(GRPC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep/src/grpc_ep-install")
+    set(GRPC_HOME "${GRPC_PREFIX}")
+    set(GRPC_INCLUDE_DIR "${GRPC_PREFIX}/include")
+    set(GRPC_STATIC_LIBRARY_GPR "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gpr${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GRPC_STATIC_LIBRARY_GRPC "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpc${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GRPC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                          "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
+                          "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                          -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX}
+                          -DBUILD_SHARED_LIBS=OFF)
+
+    ExternalProject_Add(grpc_ep
+      GIT_REPOSITORY "https://github.com/grpc/grpc"
+      GIT_TAG ${GRPC_VERSION}
+      BUILD_BYPRODUCTS "${GRPC_STATIC_LIBRARY_GPR}" "${GRPC_STATIC_LIBRARY_GRPC}" "${GRPC_STATIC_LIBRARY_GRPCPP}"
+      ${GRPC_BUILD_BYPRODUCTS}
+      ${EP_LOG_OPTIONS}
+      CMAKE_ARGS ${GRPC_CMAKE_ARGS})
+  else()
+    find_package(gRPC CONFIG REQUIRED)
+    set(GRPC_VENDORED 0)
+  endif()
+
+  include_directories(SYSTEM ${GRPC_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(grpc_grp
+    STATIC_LIB ${GRPC_STATIC_LIBRARY_GPR})
+  ADD_THIRDPARTY_LIB(grpc_grpc
+    STATIC_LIB ${GRPC_STATIC_LIBRARY_GRPC})
+  ADD_THIRDPARTY_LIB(grpc_grpcpp
+    STATIC_LIB ${GRPC_STATIC_LIBRARY_GRPCPP})
+
+  if (GRPC_VENDORED)
+    add_dependencies(grpc_grp grpc_ep)
+    add_dependencies(grpc_grpc grpc_ep)
+    add_dependencies(grpc_grpcpp grpc_ep)
+  endif()
+
+endif()

From e39b479707b80fa8762cea1b69e23259e1c3f507 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 16 Oct 2017 22:51:54 +0900
Subject: [PATCH 1150/1644] ARROW-1667: [GLib] Support Meson

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1196 from kou/c-glib-support-meson and squashes the following commits:

409cd7bc [Kouhei Sutou] [GLib] Use pkgconfig module
1925be09 [Kouhei Sutou] [GLib] Use correct LD_LIBRARY_PATH for Meson build
c5702d99 [Kouhei Sutou] [GLib] Add missing license information
029d5b3f [Kouhei Sutou] [GLib] Fix build failure on Travis CI
37d41161 [Kouhei Sutou] [GLib] Use installed files in test
d601ad4c [Kouhei Sutou] [GLib] Disable Torch build test
f9636976 [Kouhei Sutou] [GLib] Fix "string equal" operator in Travis CI script
93cdd855 [Kouhei Sutou] [GLib] Fix build dependencies
cd5e9d38 [Kouhei Sutou] [GLib] Use correct file in test
06777d3c [Kouhei Sutou] [GLib] Fix generated files dependency
9db9fefe [Kouhei Sutou] [GLib] Add Travis CI configuration for building Arrow GLib with Meson
9b76f76a [Kouhei Sutou] [GLib] Support running test in no libtool case
25876b44 [Kouhei Sutou] [GLib] Support "meson test"
2d422afc [Kouhei Sutou] [GLib] Fix include path
30c2e85e [Kouhei Sutou] [GLib] Support building examples
6bc0f81b [Kouhei Sutou] [GLib] Add missing Makefile.am
12383deb [Kouhei Sutou] [GLib] Support Meson
---
 .travis.yml                                   |  15 +-
 c_glib/.gitignore                             |   5 +-
 c_glib/Makefile.am                            |   5 +-
 c_glib/arrow-glib/Makefile.am                 |   3 +-
 c_glib/arrow-glib/meson.build                 | 197 ++++++++++++++++++
 c_glib/configure.ac                           |   2 +
 c_glib/doc/reference/Makefile.am              |   9 +-
 c_glib/doc/reference/meson.build              |  58 ++++++
 c_glib/doc/reference/xml/Makefile.am          |  20 ++
 .../doc/reference/xml/gtkdocentities.ent.in   |  24 +++
 c_glib/doc/reference/xml/meson.build          |  31 +++
 c_glib/example/Makefile.am                    |   3 +
 c_glib/example/meson.build                    |  25 +++
 c_glib/meson.build                            |  61 ++++++
 c_glib/meson_options.txt                      |  23 ++
 c_glib/test/run-test.rb                       |   6 -
 c_glib/test/run-test.sh                       |  25 ++-
 c_glib/tool/Makefile.am                       |  19 ++
 c_glib/tool/get-version.py                    |  29 +++
 ci/travis_before_script_c_glib.sh             | 108 ++++++----
 ci/travis_script_c_glib.sh                    |  38 +++-
 21 files changed, 638 insertions(+), 68 deletions(-)
 create mode 100644 c_glib/arrow-glib/meson.build
 create mode 100644 c_glib/doc/reference/meson.build
 create mode 100644 c_glib/doc/reference/xml/Makefile.am
 create mode 100644 c_glib/doc/reference/xml/gtkdocentities.ent.in
 create mode 100644 c_glib/doc/reference/xml/meson.build
 create mode 100644 c_glib/example/meson.build
 create mode 100644 c_glib/meson.build
 create mode 100644 c_glib/meson_options.txt
 create mode 100644 c_glib/tool/Makefile.am
 create mode 100755 c_glib/tool/get-version.py

diff --git a/.travis.yml b/.travis.yml
index e722c27f71917..c682a9d9db89f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -120,7 +120,19 @@ matrix:
   - compiler: gcc
     language: cpp
     os: linux
-    group: deprecated
+    env: BUILD_SYSTEM=autotools BUILD_TORCH_EXAMPLE=no
+    before_script:
+    - export CC="gcc-4.9"
+    - export CXX="g++-4.9"
+    - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+  - compiler: gcc
+    language: cpp
+    os: linux
+    env: BUILD_SYSTEM=meson BUILD_TORCH_EXAMPLE=no
     before_script:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
@@ -135,6 +147,7 @@ matrix:
     cache:
     addons:
     rvm: 2.2
+    env: BUILD_SYSTEM=autotools
     before_script:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
diff --git a/c_glib/.gitignore b/c_glib/.gitignore
index 373ae73d5d2d1..03bb0fe61892e 100644
--- a/c_glib/.gitignore
+++ b/c_glib/.gitignore
@@ -29,7 +29,10 @@ Makefile.in
 /doc/reference/gtk-doc.make
 /doc/reference/*.stamp
 /doc/reference/html/
-/doc/reference/xml/
+/doc/reference/xml/*
+!/doc/reference/xml/Makefile.am
+!/doc/reference/xml/meson.build
+!/doc/reference/xml/gtkdocentities.ent.in
 /doc/reference/tmpl/
 /libtool
 /m4/
diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am
index d059d122ba412..577b749fb38bc 100644
--- a/c_glib/Makefile.am
+++ b/c_glib/Makefile.am
@@ -20,10 +20,13 @@ ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
 SUBDIRS =					\
 	arrow-glib				\
 	doc					\
-	example
+	example					\
+	tool
 
 EXTRA_DIST =					\
 	README.md				\
+	meson.build				\
+	meson_options.txt			\
 	test
 
 arrow_glib_docdir = ${datarootdir}/doc/arrow-glib
diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 4c22d11c76ca9..2066fa77cef84 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -17,7 +17,8 @@
 
 CLEANFILES =
 
-EXTRA_DIST =
+EXTRA_DIST =					\
+	meson.build
 
 AM_CPPFLAGS =					\
 	-I$(top_builddir)			\
diff --git a/c_glib/arrow-glib/meson.build b/c_glib/arrow-glib/meson.build
new file mode 100644
index 0000000000000..226c696bdcf37
--- /dev/null
+++ b/c_glib/arrow-glib/meson.build
@@ -0,0 +1,197 @@
+# -*- indent-tabs-mode: nil -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+sources = files(
+  'array.cpp',
+  'array-builder.cpp',
+  'basic-data-type.cpp',
+  'buffer.cpp',
+  'chunked-array.cpp',
+  'column.cpp',
+  'composite-data-type.cpp',
+  'error.cpp',
+  'field.cpp',
+  'record-batch.cpp',
+  'schema.cpp',
+  'table.cpp',
+  'tensor.cpp',
+  'type.cpp',
+)
+
+sources += files(
+  'file.cpp',
+  'file-mode.cpp',
+  'input-stream.cpp',
+  'output-stream.cpp',
+  'readable.cpp',
+  'writeable.cpp',
+  'writeable-file.cpp',
+)
+
+sources += files(
+  'metadata-version.cpp',
+  'reader.cpp',
+  'writer.cpp',
+)
+
+sources += files(
+  'compute.cpp',
+)
+
+c_headers = files(
+  'array.h',
+  'array-builder.h',
+  'arrow-glib.h',
+  'basic-data-type.h',
+  'buffer.h',
+  'chunked-array.h',
+  'column.h',
+  'composite-data-type.h',
+  'data-type.h',
+  'error.h',
+  'field.h',
+  'gobject-type.h',
+  'record-batch.h',
+  'schema.h',
+  'table.h',
+  'tensor.h',
+  'type.h',
+)
+
+
+c_headers += files(
+  'file.h',
+  'file-mode.h',
+  'input-stream.h',
+  'output-stream.h',
+  'readable.h',
+  'writeable.h',
+  'writeable-file.h',
+)
+
+c_headers += files(
+  'metadata-version.h',
+  'reader.h',
+  'writer.h',
+)
+
+c_headers += files(
+  'compute.h',
+)
+
+
+cpp_headers = files(
+  'array.hpp',
+  'array-builder.hpp',
+  'arrow-glib.hpp',
+  'basic-data-type.hpp',
+  'buffer.hpp',
+  'chunked-array.hpp',
+  'column.hpp',
+  'data-type.hpp',
+  'error.hpp',
+  'field.hpp',
+  'record-batch.hpp',
+  'schema.hpp',
+  'table.hpp',
+  'tensor.hpp',
+  'type.hpp',
+)
+
+cpp_headers += files(
+  'file.hpp',
+  'file-mode.hpp',
+  'input-stream.hpp',
+  'output-stream.hpp',
+  'readable.hpp',
+  'writeable.hpp',
+  'writeable-file.hpp',
+)
+
+cpp_headers += files(
+  'metadata-version.hpp',
+  'reader.hpp',
+  'writer.hpp',
+)
+
+cpp_headers += files(
+  'compute.hpp',
+)
+
+
+enums = gnome.mkenums('enums',
+                      sources: c_headers,
+                      identifier_prefix: 'GArrow',
+                      symbol_prefix: 'garrow',
+                      c_template: 'enums.c.template',
+                      h_template: 'enums.h.template',
+                      install_dir: join_paths(include_dir, meson.project_name()),
+                      install_header: true)
+enums_source = enums[0]
+enums_header = enums[1]
+
+
+headers = c_headers + cpp_headers
+install_headers(headers, subdir: meson.project_name())
+
+
+dependencies = [
+  dependency('arrow'),
+  dependency('gobject-2.0'),
+  dependency('gio-2.0'),
+]
+libarrow_glib = library('arrow-glib',
+                        sources: sources + enums,
+                        install: true,
+                        dependencies: dependencies,
+                        include_directories: [
+                          root_inc,
+                        ],
+                       soversion: so_version,
+                       version: library_version)
+libarrow_glib_dependency = declare_dependency(link_with: libarrow_glib,
+                                              include_directories: [
+                                                root_inc,
+                                              ],
+                                              dependencies: dependencies,
+                                              sources: enums_header)
+
+pkgconfig.generate(filebase: meson.project_name(),
+                   name: 'Apache Arrow GLib',
+                   description: 'C API for Apache Arrow based on GLib',
+                   version: version,
+                   requires: ['gobject-2.0', 'arrow'],
+                   libraries: [libarrow_glib],
+                   subdirs: ['arrow-glib'])
+
+gnome.generate_gir(libarrow_glib,
+                   sources: sources + c_headers + enums,
+                   namespace: 'Arrow',
+                   nsversion: api_version,
+                   identifier_prefix: 'GArrow',
+                   symbol_prefix: 'garrow',
+                   export_packages: 'arrow-glib',
+                   includes: [
+                     'GObject-2.0',
+                     'Gio-2.0',
+                   ],
+                   install: true,
+                   extra_args: [
+                     '--warn-all',
+                   ])
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index 938064c16a7dc..5db435275a300 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -100,8 +100,10 @@ AC_CONFIG_FILES([
   arrow-glib/arrow-glib.pc
   doc/Makefile
   doc/reference/Makefile
+  doc/reference/xml/Makefile
   example/Makefile
   example/lua/Makefile
+  tool/Makefile
 ])
 
 AC_OUTPUT
diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am
index d3389dc2ae81e..45b11f035183e 100644
--- a/c_glib/doc/reference/Makefile.am
+++ b/c_glib/doc/reference/Makefile.am
@@ -15,6 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+SUBDIRS =                                       \
+	xml
+
 DOC_MODULE = arrow-glib
 
 DOC_MAIN_SGML_FILE = $(DOC_MODULE)-docs.sgml
@@ -32,8 +35,7 @@ MKDB_OPTIONS =					\
 HFILE_GLOB =					\
 	$(top_srcdir)/arrow-glib/*.h
 
-IGNORE_HFILES =					\
-	enums.h
+IGNORE_HFILES =
 
 CFILE_GLOB =					\
 	$(top_srcdir)/arrow-glib/*.cpp
@@ -57,3 +59,6 @@ CLEANFILES +=					\
 	$(DOC_MODULE)-overrides.txt		\
 	$(DOC_MODULE)-sections.txt		\
 	$(DOC_MODULE).types
+
+EXTRA_DIST +=					\
+	meson.build
diff --git a/c_glib/doc/reference/meson.build b/c_glib/doc/reference/meson.build
new file mode 100644
index 0000000000000..08936daf87288
--- /dev/null
+++ b/c_glib/doc/reference/meson.build
@@ -0,0 +1,58 @@
+# -*- indent-tabs-mode: nil -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+subdir('xml')
+
+private_headers = [
+]
+
+content_files = [
+]
+
+html_images = [
+]
+
+glib_prefix = dependency('glib-2.0').get_pkgconfig_variable('prefix')
+glib_doc_path = join_paths(glib_prefix, 'share', 'gtk-doc', 'html')
+doc_path = join_paths(data_dir, meson.project_name(), 'gtk-doc', 'html')
+
+gnome.gtkdoc(meson.project_name(),
+             main_xml: meson.project_name() + '-docs.sgml',
+             src_dir: [
+               join_paths(meson.source_root(), 'arrow-glib'),
+               join_paths(meson.build_root(), 'arrow-glib'),
+             ],
+             dependencies: libarrow_glib_dependency,
+             gobject_typesfile: meson.project_name() + '.types',
+             scan_args: [
+               '--rebuild-types',
+               '--deprecated-guards=GARROW_DISABLE_DEPRECATED',
+             ],
+             mkdb_args: [
+               '--output-format=xml',
+               '--name-space=garrow',
+               '--source-suffixes=c,cpp,h',
+             ],
+             fixxref_args: [
+               '--html-dir=' + doc_path,
+               '--extra-dir=' + join_paths(glib_doc_path, 'glib'),
+               '--extra-dir=' + join_paths(glib_doc_path, 'gobject'),
+             ],
+             html_assets: html_images,
+             install: true)
diff --git a/c_glib/doc/reference/xml/Makefile.am b/c_glib/doc/reference/xml/Makefile.am
new file mode 100644
index 0000000000000..833cfddc69078
--- /dev/null
+++ b/c_glib/doc/reference/xml/Makefile.am
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+EXTRA_DIST =					\
+	gtkdocentities.ent.in			\
+	meson.build
diff --git a/c_glib/doc/reference/xml/gtkdocentities.ent.in b/c_glib/doc/reference/xml/gtkdocentities.ent.in
new file mode 100644
index 0000000000000..dc0cf1a0d8d4a
--- /dev/null
+++ b/c_glib/doc/reference/xml/gtkdocentities.ent.in
@@ -0,0 +1,24 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<!ENTITY package "@package@">
+<!ENTITY package_bugreport "@package_bugreport@">
+<!ENTITY package_name "@package_name@">
+<!ENTITY package_string "@package_string@">
+<!ENTITY package_url "@package_url@">
+<!ENTITY package_version "@package_version@">
diff --git a/c_glib/doc/reference/xml/meson.build b/c_glib/doc/reference/xml/meson.build
new file mode 100644
index 0000000000000..5b65042764fee
--- /dev/null
+++ b/c_glib/doc/reference/xml/meson.build
@@ -0,0 +1,31 @@
+# -*- indent-tabs-mode: nil -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+entities_conf = configuration_data()
+entities_conf.set('package', meson.project_name())
+entities_conf.set('package_bugreport',
+                  'https://issues.apache.org/jira/browse/ARROW')
+entities_conf.set('package_name', meson.project_name())
+entities_conf.set('package_string',
+                  ' '.join([meson.project_name(), version]))
+entities_conf.set('package_url', 'https://arrow.apache.org/')
+entities_conf.set('package_version', version)
+configure_file(input: 'gtkdocentities.ent.in',
+               output: 'gtkdocentities.ent',
+               configuration: entities_conf)
diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am
index 6c177954c6cb9..3eaf808bc5aa6 100644
--- a/c_glib/example/Makefile.am
+++ b/c_glib/example/Makefile.am
@@ -18,6 +18,9 @@
 SUBDIRS	=					\
 	lua
 
+EXTRA_DIST =					\
+	meson.build
+
 AM_CPPFLAGS =					\
 	-I$(top_builddir)			\
 	-I$(top_srcdir)				\
diff --git a/c_glib/example/meson.build b/c_glib/example/meson.build
new file mode 100644
index 0000000000000..338ac360f0258
--- /dev/null
+++ b/c_glib/example/meson.build
@@ -0,0 +1,25 @@
+# -*- indent-tabs-mode: nil -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+executable('build', 'build.c',
+	   dependencies: [libarrow_glib_dependency])
+executable('read-batch', 'read-batch.c',
+	   dependencies: [libarrow_glib_dependency])
+executable('read-stream', 'read-stream.c',
+	   dependencies: [libarrow_glib_dependency])
diff --git a/c_glib/meson.build b/c_glib/meson.build
new file mode 100644
index 0000000000000..1fa64ba19c406
--- /dev/null
+++ b/c_glib/meson.build
@@ -0,0 +1,61 @@
+# -*- indent-tabs-mode: nil -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+project('arrow-glib', 'c', 'cpp',
+        license: 'Apache-2.0',
+        default_options: [
+          'cpp_std=c++11',
+        ])
+
+version = run_command('python', 'tool/get-version.py').stdout().strip()
+if version.endswith('-SNAPSHOT')
+  version_numbers = version.split('-')[0].split('.')
+  version_tag = version.split('-')[1]
+else
+  version_numbers = version.split('.')
+  version_tag = ''
+endif
+version_major = version_numbers[0].to_int()
+version_minor = version_numbers[1].to_int()
+version_micro = version_numbers[2].to_int()
+
+api_version = '1.0'
+so_version = 0
+library_version = '@0@.@1@.@2@'.format(so_version, 0, 0)
+
+prefix = get_option('prefix')
+include_dir = join_paths(prefix, get_option('includedir'))
+data_dir = join_paths(prefix, get_option('datadir'))
+
+gnome = import('gnome')
+pkgconfig = import('pkgconfig')
+
+root_inc = include_directories('.')
+
+subdir('arrow-glib')
+subdir('example')
+
+if get_option('enable_gtk_doc')
+  subdir('doc/reference')
+endif
+
+run_test = find_program('test/run-test.sh')
+test('unit test',
+     run_test,
+     env: ['ARROW_GLIB_TYPELIB_DIR=@0@/arrow-glib'.format(meson.build_root())])
diff --git a/c_glib/meson_options.txt b/c_glib/meson_options.txt
new file mode 100644
index 0000000000000..2988e1af0acc9
--- /dev/null
+++ b/c_glib/meson_options.txt
@@ -0,0 +1,23 @@
+# -*- indent-tabs-mode: nil -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+option('enable_gtk_doc',
+       type: 'boolean',
+       value: false,
+       description: 'Build document by GTK-Doc')
diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb
index 75ff34ff47281..3451bd29fde1b 100755
--- a/c_glib/test/run-test.rb
+++ b/c_glib/test/run-test.rb
@@ -21,14 +21,8 @@
 require "test-unit"
 
 base_dir = Pathname(__dir__).parent
-typelib_dir = base_dir + "arrow-glib"
 test_dir = base_dir + "test"
 
-ENV["GI_TYPELIB_PATH"] = [
-  typelib_dir.to_s,
-  ENV["GI_TYPELIB_PATH"],
-].compact.join(File::PATH_SEPARATOR)
-
 require "gi"
 
 Gio = GI.load("Gio")
diff --git a/c_glib/test/run-test.sh b/c_glib/test/run-test.sh
index efa2829d74a29..19ccf077833a5 100755
--- a/c_glib/test/run-test.sh
+++ b/c_glib/test/run-test.sh
@@ -17,13 +17,30 @@
 # specific language governing permissions and limitations
 # under the License.
 
-base_dir="$(cd .; pwd)"
-lib_dir="${base_dir}/arrow-glib/.libs"
+test_dir="$(cd $(dirname $0); pwd)"
+build_dir="$(cd .; pwd)"
 
-LD_LIBRARY_PATH="${lib_dir}:${LD_LIBRARY_PATH}"
+arrow_glib_build_dir="${build_dir}/arrow-glib/"
+libtool_dir="${arrow_glib_build_dir}/.libs"
+if [ -d "${libtool_dir}" ]; then
+  LD_LIBRARY_PATH="${libtool_dir}:${LD_LIBRARY_PATH}"
+else
+  if [ -d "${arrow_glib_build_dir}" ]; then
+    LD_LIBRARY_PATH="${arrow_glib_build_dir}:${LD_LIBRARY_PATH}"
+  fi
+fi
 
 if [ -f "Makefile" -a "${NO_MAKE}" != "yes" ]; then
   make -j8 > /dev/null || exit $?
 fi
 
-${GDB} ruby ${base_dir}/test/run-test.rb "$@"
+arrow_glib_typelib_dir="${ARROW_GLIB_TYPELIB_DIR}"
+if [ -z "${arrow_glib_typelib_dir}" ]; then
+  arrow_glib_typelib_dir="${build_dir}/arrow-glib"
+fi
+
+if [ -d "${arrow_glib_typelib_dir}" ]; then
+  GI_TYPELIB_PATH="${arrow_glib_typelib_dir}:${GI_TYPELIB_PATH}"
+fi
+
+${GDB} ruby ${test_dir}/run-test.rb "$@"
diff --git a/c_glib/tool/Makefile.am b/c_glib/tool/Makefile.am
new file mode 100644
index 0000000000000..5d7498b957520
--- /dev/null
+++ b/c_glib/tool/Makefile.am
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+EXTRA_DIST =					\
+	get-version.py
diff --git a/c_glib/tool/get-version.py b/c_glib/tool/get-version.py
new file mode 100755
index 0000000000000..aacea6da3e865
--- /dev/null
+++ b/c_glib/tool/get-version.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+
+root = os.environ.get("MESON_SOURCE_ROOT", ".")
+pom_xml = os.path.join(root, "..", "java", "pom.xml")
+with open(pom_xml) as pom:
+    version_tag = re.search('^  <version>(.+)</version>',
+                            pom.read(),
+                            re.MULTILINE)
+    print(version_tag.group(1))
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 14e4f9fa11393..d60f6296cc46c 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -21,65 +21,91 @@ set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-if [ $TRAVIS_OS_NAME == "osx" ]; then
+if [ $TRAVIS_OS_NAME = "osx" ]; then
   brew update && brew bundle --file=c_glib/Brewfile
 
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
 fi
 
+if [ $BUILD_SYSTEM = "meson" ]; then
+  source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+  pip install meson ninja
+fi
+
 gem install test-unit gobject-introspection
 
-if [ $TRAVIS_OS_NAME == "osx" ]; then
+if [ $TRAVIS_OS_NAME = "osx" ]; then
   sudo env PKG_CONFIG_PATH=$PKG_CONFIG_PATH luarocks install lgi
-else
-  git clone \
-    --quiet \
-    --depth 1 \
-    --recursive \
-    https://github.com/torch/distro.git ~/torch
-  pushd ~/torch
-  ./install-deps > /dev/null
-  echo "yes" | ./install.sh > /dev/null
-  . ~/torch/install/bin/torch-activate
-  popd
-  luarocks install lgi
+elif [ $BUILD_SYSTEM = "autotools" ]; then
+  if [ $BUILD_TORCH_EXAMPLE = "yes" ]; then
+    git clone \
+      --quiet \
+      --depth 1 \
+      --recursive \
+      https://github.com/torch/distro.git ~/torch
+    pushd ~/torch
+    ./install-deps > /dev/null
+    echo "yes" | ./install.sh > /dev/null
+    . ~/torch/install/bin/torch-activate
+    popd
+    luarocks install lgi
+  else
+    sudo apt install -y -qq luarocks
+    sudo luarocks install lgi
+  fi
 fi
 
-go get github.com/linuxdeepin/go-gir-generator || :
-pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
-rm lib.in/gio-2.0/gdk_workaround.go
-mv lib.in/gio-2.0/config.json{,.orig}
-sed \
-  -e 's/\("Settings",\)/\/\/ \1/g' \
-  -e 's/\("SettingsBackend",\)/\/\/ \1/g' \
-  lib.in/gio-2.0/config.json.orig > lib.in/gio-2.0/config.json
-mv Makefile{,.orig}
-sed -e 's/ gudev-1.0//' Makefile.orig > Makefile
-mkdir -p out/src/gir/gudev-1.0
-make build copyfile
-mkdir -p $GOPATH/bin/
-cp -a out/gir-generator $GOPATH/bin/
-cp -a out/src/gir/ $GOPATH/src/gir/
-popd
+if [ $BUILD_SYSTEM = "autotools" ]; then
+  go get github.com/linuxdeepin/go-gir-generator || :
+  pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
+  rm lib.in/gio-2.0/gdk_workaround.go
+  mv lib.in/gio-2.0/config.json{,.orig}
+  sed \
+    -e 's/\("Settings",\)/\/\/ \1/g' \
+    -e 's/\("SettingsBackend",\)/\/\/ \1/g' \
+    lib.in/gio-2.0/config.json.orig > lib.in/gio-2.0/config.json
+  mv Makefile{,.orig}
+  sed -e 's/ gudev-1.0//' Makefile.orig > Makefile
+  mkdir -p out/src/gir/gudev-1.0
+  make build copyfile
+  mkdir -p $GOPATH/bin/
+  cp -a out/gir-generator $GOPATH/bin/
+  cp -a out/src/gir/ $GOPATH/src/gir/
+  popd
+fi
 
 pushd $ARROW_C_GLIB_DIR
 
-./autogen.sh
-
 export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
 
-CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"
-if [ $TRAVIS_OS_NAME != "osx" ]; then
-  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"
-fi
+if [ $BUILD_SYSTEM = "autotools" ]; then
+  ./autogen.sh
+
+  CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"
+  if [ $TRAVIS_OS_NAME != "osx" ]; then
+    CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"
+  fi
 
-CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CFLAGS=-DARROW_NO_DEPRECATED_API"
-CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API"
+  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CFLAGS=-DARROW_NO_DEPRECATED_API"
+  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API"
 
-./configure $CONFIGURE_OPTIONS
+  ./configure $CONFIGURE_OPTIONS
 
-make -j4
-make install
+  make -j4
+  make install
+else
+  MESON_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"
+  MESON_OPTIONS="$MESON_OPTIONS -Denable_gtk_doc=true"
+  mkdir -p build
+  env \
+    CFLAGS="-DARROW_NO_DEPRECATED_API" \
+    CXXFLAGS="-DARROW_NO_DEPRECATED_API" \
+    meson build $MESON_OPTIONS
+  pushd build
+  ninja
+  ninja install
+  popd
+fi
 
 popd
diff --git a/ci/travis_script_c_glib.sh b/ci/travis_script_c_glib.sh
index d392abdfbbc36..3833bb1701d9c 100755
--- a/ci/travis_script_c_glib.sh
+++ b/ci/travis_script_c_glib.sh
@@ -24,26 +24,42 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 pushd $ARROW_C_GLIB_DIR
 
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
-NO_MAKE=yes test/run-test.sh
+if [ $BUILD_SYSTEM = "autotools" ]; then
+  arrow_c_glib_lib_dir=$ARROW_C_GLIB_INSTALL/lib
+else
+  arrow_c_glib_lib_dir=$ARROW_C_GLIB_INSTALL/lib/$(arch)-linux-gnu
+fi
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$arrow_c_glib_lib_dir
+export GI_TYPELIB_PATH=$arrow_c_glib_lib_dir/girepository-1.0
+test/run-test.rb
+
+if [ $BUILD_SYSTEM = "meson" ]; then
+  exit
+fi
 
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_C_GLIB_INSTALL/lib
-export GI_TYPELIB_PATH=$ARROW_C_GLIB_INSTALL/lib/girepository-1.0
 export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
-export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_C_GLIB_INSTALL/lib/pkgconfig
+export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$arrow_c_glib_lib_dir/pkgconfig
 
 pushd example/lua
-if [ $TRAVIS_OS_NAME == "osx" ]; then
+if [ $TRAVIS_OS_NAME = "osx" ]; then
   lua write-batch.lua
   lua read-batch.lua
   lua write-stream.lua
   lua read-stream.lua
 else
-  . ~/torch/install/bin/torch-activate
-  luajit write-batch.lua
-  luajit read-batch.lua
-  luajit write-stream.lua
-  luajit read-stream.lua
-  luajit stream-to-torch-tensor.lua
+  if [ $BUILD_TORCH_EXAMPLE = "yes" ]; then
+    . ~/torch/install/bin/torch-activate
+    luajit write-batch.lua
+    luajit read-batch.lua
+    luajit write-stream.lua
+    luajit read-stream.lua
+    luajit stream-to-torch-tensor.lua
+  else
+    lua write-batch.lua
+    lua read-batch.lua
+    lua write-stream.lua
+    lua read-stream.lua
+  fi
 fi
 popd
 

From 1571fb4367655d86db515be1bd936f4cbc564f1c Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 16 Oct 2017 10:08:23 -0400
Subject: [PATCH 1151/1644] ARROW-1677: [Blog] Post on ray and arrow
 serialization

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1202 from pcmoritz/serialization-blog-post and squashes the following commits:

832395a9 [Philipp Moritz] add blog post on ray and arrow serialization
---
 ...python-serialization-with-ray-and-arrow.md | 308 ++++++++++++++++++
 .../arrow_object.png                          | Bin 0 -> 10539 bytes
 .../python_object.png                         | Bin 0 -> 24165 bytes
 .../speedups0.png                             | Bin 0 -> 4050 bytes
 .../speedups1.png                             | Bin 0 -> 4502 bytes
 .../speedups2.png                             | Bin 0 -> 4276 bytes
 .../speedups3.png                             | Bin 0 -> 13637 bytes
 7 files changed, 308 insertions(+)
 create mode 100644 site/_posts/2017-10-15-fast-python-serialization-with-ray-and-arrow.md
 create mode 100644 site/assets/fast_python_serialization_with_ray_and_arrow/arrow_object.png
 create mode 100644 site/assets/fast_python_serialization_with_ray_and_arrow/python_object.png
 create mode 100644 site/assets/fast_python_serialization_with_ray_and_arrow/speedups0.png
 create mode 100644 site/assets/fast_python_serialization_with_ray_and_arrow/speedups1.png
 create mode 100644 site/assets/fast_python_serialization_with_ray_and_arrow/speedups2.png
 create mode 100644 site/assets/fast_python_serialization_with_ray_and_arrow/speedups3.png

diff --git a/site/_posts/2017-10-15-fast-python-serialization-with-ray-and-arrow.md b/site/_posts/2017-10-15-fast-python-serialization-with-ray-and-arrow.md
new file mode 100644
index 0000000000000..f5f3ccad37d74
--- /dev/null
+++ b/site/_posts/2017-10-15-fast-python-serialization-with-ray-and-arrow.md
@@ -0,0 +1,308 @@
+---
+layout: post
+title: "Fast Python Serialization with Ray and Apache Arrow"
+excerpt: "This post describes how serialization works in Ray."
+date: 2017-10-15 14:00:00
+author: Philipp Moritz, Robert Nishihara
+categories: [application]
+---
+
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+*This was originally posted on the [Ray blog][11]. [Philipp Moritz][12] and [Robert Nishihara][13] are graduate students at UC Berkeley.*
+
+This post elaborates on the integration between [Ray][1] and [Apache Arrow][2].
+The main problem this addresses is [data serialization][3].
+
+From [Wikipedia][3], **serialization** is
+
+> ... the process of translating data structures or object state into a format
+> that can be stored ... or transmitted ... and reconstructed later (possibly
+> in a different computer environment).
+
+Why is any translation necessary? Well, when you create a Python object, it may
+have pointers to other Python objects, and these objects are all allocated in
+different regions of memory, and all of this has to make sense when unpacked by
+another process on another machine.
+
+Serialization and deserialization are **bottlenecks in parallel and distributed
+computing**, especially in machine learning applications with large objects and
+large quantities of data.
+
+## Design Goals
+
+As Ray is optimized for machine learning and AI applications, we have focused a
+lot on serialization and data handling, with the following design goals:
+
+1. It should be very efficient with **large numerical data** (this includes
+NumPy arrays and Pandas DataFrames, as well as objects that recursively contain
+Numpy arrays and Pandas DataFrames).
+2. It should be about as fast as Pickle for **general Python types**.
+3. It should be compatible with **shared memory**, allowing multiple processes
+to use the same data without copying it.
+4. **Deserialization** should be extremely fast (when possible, it should not
+require reading the entire serialized object).
+5. It should be **language independent** (eventually we'd like to enable Python
+workers to use objects created by workers in Java or other languages and vice
+versa).
+
+## Our Approach and Alternatives
+
+The go-to serialization approach in Python is the **pickle** module. Pickle is
+very general, especially if you use variants like [cloudpickle][4]. However, it
+does not satisfy requirements 1, 3, 4, or 5. Alternatives like **json** satisfy
+5, but not 1-4.
+
+**Our Approach:** To satisfy requirements 1-5, we chose to use the
+[Apache Arrow][2] format as our underlying data representation. In collaboration
+with the Apache Arrow team, we built [libraries][7] for mapping general Python
+objects to and from the Arrow format. Some properties of this approach:
+
+- The data layout is language independent (requirement 5).
+- Offsets into a serialized data blob can be computed in constant time without
+reading the full object (requirements 1 and 4).
+- Arrow supports **zero-copy reads**, so objects can naturally be stored in
+shared memory and used by multiple processes (requirements 1 and 3).
+- We can naturally fall back to pickle for anything we can’t handle well
+(requirement 2).
+
+**Alternatives to Arrow:** We could have built on top of
+[**Protocol Buffers**][5], but protocol buffers really isn't designed for
+numerical data, and that approach wouldn’t satisfy 1, 3, or 4. Building on top
+of [**Flatbuffers**][6] actually could be made to work, but it would have
+required implementing a lot of the facilities that Arrow already has and we
+preferred a columnar data layout more optimized for big data.
+
+## Speedups
+
+Here we show some performance improvements over Python’s pickle module. The
+experiments were done using `pickle.HIGHEST_PROTOCOL`. Code for generating these
+plots is included at the end of the post.
+
+**With NumPy arrays:** In machine learning and AI applications, data (e.g.,
+images, neural network weights, text documents) are typically represented as
+data structures containing NumPy arrays. When using NumPy arrays, the speedups
+are impressive.
+
+The fact that the Ray bars for deserialization are barely visible is not a
+mistake. This is a consequence of the support for zero-copy reads (the savings
+largely come from the lack of memory movement).
+
+<div align="center">
+<img src="{{ site.base-url }}/assets/fast_python_serialization_with_ray_and_arrow/speedups0.png" width="365" height="255">
+<img src="{{ site.base-url }}/assets/fast_python_serialization_with_ray_and_arrow/speedups1.png" width="365" height="255">
+</div>
+
+Note that the biggest wins are with deserialization. The speedups here are
+multiple orders of magnitude and get better as the NumPy arrays get larger
+(thanks to design goals 1, 3, and 4). Making **deserialization** fast is
+important for two reasons. First, an object may be serialized once and then
+deserialized many times (e.g., an object that is broadcast to all workers).
+Second, a common pattern is for many objects to be serialized in parallel and
+then aggregated and deserialized one at a time on a single worker making
+deserialization the bottleneck.
+
+**Without NumPy arrays:** When using regular Python objects, for which we
+cannot take advantage of shared memory, the results are comparable to pickle.
+
+<div align="center">
+<img src="{{ site.base-url }}/assets/fast_python_serialization_with_ray_and_arrow/speedups2.png" width="365" height="255">
+<img src="{{ site.base-url }}/assets/fast_python_serialization_with_ray_and_arrow/speedups3.png" width="365" height="255">
+</div>
+
+These are just a few examples of interesting Python objects. The most important
+case is the case where NumPy arrays are nested within other objects. Note that
+our serialization library works with very general Python types including custom
+Python classes and deeply nested objects.
+
+## The API
+
+The serialization library can be used directly through pyarrow as follows. More
+documentation is available [here][7].
+
+```python
+x = [(1, 2), 'hello', 3, 4, np.array([5.0, 6.0])]
+serialized_x = pyarrow.serialize(x).to_buffer()
+deserialized_x = pyarrow.deserialize(serialized_x)
+```
+
+It can be used directly through the Ray API as follows.
+
+```python
+x = [(1, 2), 'hello', 3, 4, np.array([5.0, 6.0])]
+x_id = ray.put(x)
+deserialized_x = ray.get(x_id)
+```
+
+## Data Representation
+
+We use Apache Arrow as the underlying language-independent data layout. Objects
+are stored in two parts: a **schema** and a **data blob**. At a high level, the
+data blob is roughly a flattened concatenation of all of the data values
+recursively contained in the object, and the schema defines the types and
+nesting structure of the data blob.
+
+**Technical Details:** Python sequences (e.g., dictionaries, lists, tuples,
+sets) are encoded as Arrow [UnionArrays][8] of other types (e.g., bools, ints,
+strings, bytes, floats, doubles, date64s, tensors (i.e., NumPy arrays), lists,
+tuples, dicts and sets). Nested sequences are encoded using Arrow
+[ListArrays][9]. All tensors are collected and appended to the end of the
+serialized object, and the UnionArray contains references to these tensors.
+
+To give a concrete example, consider the following object.
+
+```python
+[(1, 2), 'hello', 3, 4, np.array([5.0, 6.0])]
+```
+
+It would be represented in Arrow with the following structure.
+
+```
+UnionArray(type_ids=[tuple, string, int, int, ndarray],
+           tuples=ListArray(offsets=[0, 2],
+                            UnionArray(type_ids=[int, int],
+                                       ints=[1, 2])),
+           strings=['hello'],
+           ints=[3, 4],
+           ndarrays=[<offset of numpy array>])
+```
+
+Arrow uses Flatbuffers to encode serialized schemas. **Using only the schema, we
+can compute the offsets of each value in the data blob without scanning through
+the data blob** (unlike Pickle, this is what enables fast deserialization). This
+means that we can avoid copying or otherwise converting large arrays and other
+values during deserialization. Tensors are appended at the end of the UnionArray
+and can be efficiently shared and accessed using shared memory.
+
+Note that the actual object would be laid out in memory as shown below.
+
+<div align="center">
+<img src="{{ site.base-url }}/assets/fast_python_serialization_with_ray_and_arrow/python_object.png" width="600">
+</div>
+<div><i>The layout of a Python object in the heap. Each box is allocated in a
+different memory region, and arrows between boxes represent pointers.</i></div>
+<br />
+
+The Arrow serialized representation would be as follows.
+
+<div align="center">
+<img src="{{ site.base-url }}/assets/fast_python_serialization_with_ray_and_arrow/arrow_object.png" width="400">
+</div>
+<div><i>The memory layout of the Arrow-serialized object.</i></div>
+<br />
+
+## Getting Involved
+
+We welcome contributions, especially in the following areas.
+
+- Use the C++ and Java implementations of Arrow to implement versions of this
+for C++ and Java.
+- Implement support for more Python types and better test coverage.
+
+## Reproducing the Figures Above
+
+For reference, the figures can be reproduced with the following code.
+Benchmarking `ray.put` and `ray.get` instead of `pyarrow.serialize` and
+`pyarrow.deserialize` gives similar figures. The plots were generated at this
+[commit][10].
+
+```python
+import pickle
+import pyarrow
+import matplotlib.pyplot as plt
+import numpy as np
+import timeit
+
+
+def benchmark_object(obj, number=10):
+    # Time serialization and deserialization for pickle.
+    pickle_serialize = timeit.timeit(
+        lambda: pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL),
+        number=number)
+    serialized_obj = pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
+    pickle_deserialize = timeit.timeit(lambda: pickle.loads(serialized_obj),
+                                       number=number)
+
+    # Time serialization and deserialization for Ray.
+    ray_serialize = timeit.timeit(
+        lambda: pyarrow.serialize(obj).to_buffer(), number=number)
+    serialized_obj = pyarrow.serialize(obj).to_buffer()
+    ray_deserialize = timeit.timeit(
+        lambda: pyarrow.deserialize(serialized_obj), number=number)
+
+    return [[pickle_serialize, pickle_deserialize],
+            [ray_serialize, ray_deserialize]]
+
+
+def plot(pickle_times, ray_times, title, i):
+    fig, ax = plt.subplots()
+    fig.set_size_inches(3.8, 2.7)
+
+    bar_width = 0.35
+    index = np.arange(2)
+    opacity = 0.6
+
+    plt.bar(index, pickle_times, bar_width,
+            alpha=opacity, color='r', label='Pickle')
+
+    plt.bar(index + bar_width, ray_times, bar_width,
+            alpha=opacity, color='c', label='Ray')
+
+    plt.title(title, fontweight='bold')
+    plt.ylabel('Time (seconds)', fontsize=10)
+    labels = ['serialization', 'deserialization']
+    plt.xticks(index + bar_width / 2, labels, fontsize=10)
+    plt.legend(fontsize=10, bbox_to_anchor=(1, 1))
+    plt.tight_layout()
+    plt.yticks(fontsize=10)
+    plt.savefig('plot-' + str(i) + '.png', format='png')
+
+
+test_objects = [
+    [np.random.randn(50000) for i in range(100)],
+    {'weight-' + str(i): np.random.randn(50000) for i in range(100)},
+    {i: set(['string1' + str(i), 'string2' + str(i)]) for i in range(100000)},
+    [str(i) for i in range(200000)]
+]
+
+titles = [
+    'List of large numpy arrays',
+    'Dictionary of large numpy arrays',
+    'Large dictionary of small sets',
+    'Large list of strings'
+]
+
+for i in range(len(test_objects)):
+    plot(*benchmark_object(test_objects[i]), titles[i], i)
+```
+
+[1]: http://ray.readthedocs.io/en/latest/index.html
+[2]: https://arrow.apache.org/
+[3]: https://en.wikipedia.org/wiki/Serialization
+[4]: https://github.com/cloudpipe/cloudpickle/
+[5]: https://developers.google.com/protocol-buffers/
+[6]: https://google.github.io/flatbuffers/
+[7]: https://arrow.apache.org/docs/python/ipc.html#arbitrary-object-serialization
+[8]: http://arrow.apache.org/docs/memory_layout.html#dense-union-type
+[9]: http://arrow.apache.org/docs/memory_layout.html#list-type
+[10]: https://github.com/apache/arrow/tree/894f7400977693b4e0e8f4b9845fd89481f6bf29
+[11]: https://ray-project.github.io/
+[12]: https://people.eecs.berkeley.edu/~pcmoritz/
+[13]: http://www.robertnishihara.com
diff --git a/site/assets/fast_python_serialization_with_ray_and_arrow/arrow_object.png b/site/assets/fast_python_serialization_with_ray_and_arrow/arrow_object.png
new file mode 100644
index 0000000000000000000000000000000000000000..e24dcccb454a583846d7917799b61f8cf2c34cb0
GIT binary patch
literal 10539
zcmZ{K2UJr}(|70{5ov<7&;+D64NYnix-=n3QHqA%t8@VofrQYj!Ud!wP3a&A(jjz^
zVgU&q3?QNS@qeE4eDC{}_ntj>cV=gPJG(PGXV1MGZ)5<aqh_ZDfk1R{n5HoZM1%!_
zh;LGoTyX?~40f+7(x+<rY9P@2WSaBGWLGqik1<poR6TkNcST^0^h~v&o_bf!|LK{>
z;_$=6^P7`n-+%u-96nzgxsv3(#m>mcD1Q$kCMHTrN$KG5*vQDT<KwWlmPS#2jjL-w
zNp1`n%ww((i%WU2GF2BD@NDN>M_hD@o1b&f$K1h%&nkDNJ4TudtMgLg{c|fm+93>$
zEevPBRMouybh`Z|$k{$CBI3)ZS^toj>9w(j&r_4NwXGEy6N9VW*zZA6e)ZkeWgmdZ
z#EhY(ozyod6sjOCr>q){UhnKYTKV#@l(_QBN@`zY0}~Jy{^?m#6^+Qcl^{qI1QPlN
z*HkkNT-aWEZ1Nq#O0n<9y{DxpeV!xerYmM61}@a?7nURj57Msu6;-~6R5L2uZLb(d
zb#%!;Ub-3flV3=AHJMvAO!M~H^$W@Vg-GZ5Yiybt&M>+%UO6Ck3dL^x{~@Z5l?*SQ
z<J%?{T-Jlz3p0E-+b=0%eW5T;hhJQh79k_9ceNyNG7<T_V$5Z~K59vnkwkD{y-^0U
z$cpH+KP}rFAli?XO~1wlTA(BvEuw%zlkz^P1&mW(zm6}1Xrf5T3hy{EL*^8b9UIal
zV^8VuSD#d#Gl-a>`wjbozdEMDRRLJ!BLho<l20E&28XzTrMkT=Z2^)RZ%X!DUN#c-
zc@GkUX`;+GZ*oCz3D@SPQ0|yifoa!2!p7Gqp(GTUWgTkEP}Sh+e%DAQ;)!i9Pm)zH
zf0@-fuT3I8IM;vz*=lxItC=yo#cSDk(_m7j&={r80Z<Hrl0s83CrG=T*oP~wq6q1?
ziuKqcqG}e3#aqEi>{@gPAHUdcv-XL^wa+LNo*o@Y8Pfog8dgdH1U5+!n<*4e?$z<k
z+7+gdB5vN&skLCmvKjQs7lk%f9qIULR@A`G!yamO?0nVVG9L^re@lq<O(mPuTD3mb
zVvy&IVu5V|%5vTz<!<C3ncmXjKBSC-j_<;h5S1Og#aYMiA=DdQ#d}Wge^Mc7t1;!0
zN`9t`sYH}|HiV_c<56(za+^jr<$VWn(a_~5n8)`A1|c|FyL`HLr_&=oTpENbz$9b1
z%cFb`Wlli^VljAlrcFFCboR|DuYsL9;+O6((myG{6e`Tq#|`qad0y#TVoLX3*_Uvm
z(0O_MaO^4ztaBrip8UBZ#~doXpn)2cG~YBEpBf#+r%`?biSwxD<YYH)hlx_eHf#eA
zvQ+Q7!B#?xW(WQ4ZXoY1=b?Eol!Pm#Z%O_;^620us;aTErZ=!_iA{4TXLa^Dc@o#f
zFnNG5T<q7DXBih3_PsO#l*f)uLDQ4{yssY#&<ioy$;`9KbfH_QD7tQFAgUj1c4J~!
zeZ`dM(O*QrL2Bd7NhN%{XpzmZEe4GtwdCrp^v}hfnhAp9+!2A#K0BqRyNNd*<PvwZ
zm3(9S1SgW9z4xe5eEo5`3=VgCsui}Nth!g-nZzn#*mCCvl2Rhpz8ysX6)jlLvbWHJ
zbmauw$ykz3x-J%m<{r<FHG`t0(F6v1EksZrc8c913hW7%NXbg;r9B8k*(@9{Sooq<
zt4Kgg(gITSJh3d+zb&r?HYx1`qx-oauWU);WeV(U|4vkDG*N&fI8iOZhPv%KC@9<8
zK}Z>N-)?8}c6%prm!V6eDA62UwteJpN>_u@rqJM+jqiiIU!#_0A-5PFp5P4?Wlt84
zm*nA-cdxzcKy2z5hs@DDb?*Omk9K+-A96adJY1XssweYA)Dx|W&~z>NI6r8SXBbe~
z-0=s27?zsp^0hY3I6vNyxfN`=E(|~M)YzB9i4_LzfWQB09J+5a^jPsJV^fY8ss%J^
z$Q<ya!NeuP7G+=RUm7;Tylwrld0qT?mkz1@O1rcbl9A7tHC6b_uii@({`pZ=jD)k#
ze0X(~&(3$6r(aiiL;a&%x0bJk#HJEUL~K0o;HQf)D4jHVv~)`>q28O&{)4#d9vUiZ
z)7pB!B{|>;J<TC@DE5oc#OnqcP!!i(TM2m)*ev>p-Y!vbTJW9(iKoeQEho0<2Ql{a
z^ZDi1l9|`|-QwXLJTp$UtLn+_yB*4h<MF&p#K<+f(A50p3O>Ge;sVkB@OLgM1&c;F
z8m*tJ%(#mu&;9FcNxj~O0LT^!ApD|TC1V|`EfTrCc)!X+xKg_`qIJ&men6h@C=WYP
zMC&kXVwJlb|K9H5z5Gi+HCdDlXrw2J=*vB3epXH@p0OAeP_vFTA%8h)CLgn4x!IzO
zJvlgzXz)I-|Fgc(780ICHh#Lc*7hgHCX{@P4?l`>;#F|b4^4#sz1$Mr&9E6B%eWq#
zz#P@_w1Sg-<&aoN6vm5vL2_p%4Zr{9kQtzBl=!#=vEhOZ2ne@59^BzYm%Kjf;c^O2
z;o{^Dg`4$ocI~+|R#7ao8ZZ3R=Pz>InXon)@@n9>evUF7Qcv?U_WS!fW@?klkSiAn
z7x|qL%6nuEvwTO-2^S8ZC&6aIgoo*`n^aWvJxzHALktKSyTKhR>0%6BY!O`3I1vFX
zsdq%lopF@ungk)Gr8cxcDvr!KsF&El>_paI!!volBMgZ9Tk%XE@;am}?fBEYB-dcq
zRva(y=sOVss-D1#f(Pg9*doWdrMP6;kVQSqj(Fdl1W(#BdLZH4(GXT(zXK2FG93<(
zd#2R_L9-ik!bJZn%C5XCWX0)R)<FRcTU+eI#}9q5j@{HdN6tFs)HmfZyXznJwf<)N
zS}v`|l~n^09Fcn|-yK-5ozobH7t4M|B#mpK2jvsYE+ui_)Ep&XA?j%*4rHQViN<zw
z$UzlD@p7O-1_Y`j3nnL^&tRqUE2x_~H!z*GlAmhnf*x?86@%ISl2}MlncR`>%wJI`
zUPyK9)}a+OD{%bXz+=5gIhEzh7TJ2FFin>P*586jFCJpJI3FIrm}2iEinh1hLlb1u
z#y9R0<ijV@Ws9#b3^^08qzL*de#Jfx!vBj|GLC>Q6-K2g0O>s@qL0^vLWROlBO3fd
zFHi*r8<nBT$TSD>wL6rWDCWvGfg`uaG^Kc8Z%rZwaH9Yf3PG-c8b(2{S2~t2>IdlE
z<$l^`tCNZPQP(4t8a7Ss_e|0TYcF$SD}Mqxe+$RRvGOE=T@f-t^s-|HcXR8H7iSl{
zkHmMmm^n9dGB|C#u>rp`z*vx8-$EE@0@6(0xCn^$c(oSvuWk2^!mM`8kuc?Nv7;Zt
zMd!H%FM=MODq~r^??)co1l<KiV1&qdP{CCoJAFp7TnwDF3z_j^>J>@<62kAefOiSi
z$%L{Y*-@${pS|Vz2+HHVp#E`$G7+Of5pc6lh>q8gK<jSOlp13ittiEDm_dw{`B~bz
zn%hRY>{7cGc2*X8NIC<_ig1klKzA_Zp{&e?-2*SBUV*9*RXsPb7I2fgfA_~F{Dy!1
zJn&WzxPZxhpv5Vm-G^<dVcej!4kMVgHJ9=^qw}Gw`uGXcBE~gns>2afcv+3w{kd{}
z>H~0}Yhmd`mnn0}t#4#jqK>boNJTd)rJs8d7se9k_<Nv-%{FcHTm`pzB6v;1RslP5
zs(Zc6p&`ef;*VdQdN8>>QdzBGf%wO<7}(jrpgOX2MsxM?Ls(DR(KdY==LIh|>b%(V
z1Xe6++|a4B;PI~SJwlb}NuhuGTzp0L5@(KXoXDj!6d{}oOE?a+G8|dAaFni+8GKMM
zwLH=2Y8w4s*}eS8$mMHl`;8#E`MfIfUh`*h#Dz~kgVd{F35Tqqdl`(vf!`_i<Hp{V
zLeL%Jy~{$zD|L)hel%@R?ilMlxNz1)5<dF;Gb%m#-&GR+m)d6(!cy%G7f_H1qrqhQ
z+tL_w>t*n-FR5=EVvfU!wks=v_XVNP+}KPk{Vir+;D4MKznpT!T(V(OQZ|Ex)_ooa
z+(6nIt+9EqVV8KtL>&H*;xy@uGHzZ`7oL3kMdh(g!P^{IYhGogcjr(ntA$1j4N#B=
zqk&lsK{Z-PfPyP0iIO6pCuzZd8DCJFsnlQp^MF$#jqBpOfWUOM(b~qxLtFUYvE`*>
z!9iayP-pc>p07mFz)FxhPo)IE=6816mx6^X2WR2$LPjI3-CC>6f0kudzOv7856|n#
z&ElK<^?Z6|z)b_bX#URI@AB|EE-C)rjS6icsU2a{W=Qy{XHo{Wk+bx`*$$0a^_Xpo
zdYThz8P(|<Ba+C|UV?gSMeM<Z6Fb#~E|6lm|0??dRp559a2&N6`~6_6%R|ezI^)z>
zC)V<>nVruy**!s>Igsh`Z1ps&-x<lUn7Jx?W4Y7@F;OXnUlQEraxbjSG>(9O+Eu+T
zn0(T<VHN!C%HgN3y&SbcPy{b-@}U91ZL48a`-ZOyg{L_k5`w>evxV^k($F@RFGH2D
z%tv-|0!7-@z31mo<}O5WHis?4vS0Nf4eT&N<M~?Ab%`H-r>IE<feXatArX!R1d`3b
zGr1kvYquz+(TU@{?6aXoEh(3OPRi`C7k69b{F7!$KAKxmc0I|KJp#@7PP?*tN5jBN
z7L0af%_#qxU-Vm=grgfsmxoX098z_rt)x{D3$?*~@7A26;*W1&;@y)Ai=P;H@LPEB
zArYMo%@-~(58>WRaX@Fq%b@ks2AgQrvbvje6V-<I#v3E0w+T-t74OsxLq55{I&(W4
zg4?HDx2pAI<#KTM(BD0O-%j|F3pbf{B4nDI4lR;gQe#^`xevjsZs5LP9_i1~;Cg+>
z-|mP&j;FK@HZ=$_>alwo?C@6~K5Yua`5pxs=p1CqQ)2tYbH3<@!a5<>|A@h$rfEA}
zn^|Iba4tr?%#Cj&ko57iQ2gmiry(cK7TyT{kTt5ynb;RoJ4R9?=9VyDB3(9oaa)D2
z{kP_VIhLw9t1678S`h+Sm)g09Y&tTWS=L>8Hj%@C%sK;wst4O%9;T;GdrNp8@29bS
zn~B9I#dih3IlM%3rx;x=S+NEG1kMRYq;*(l<r;i%#1?{GsOtTi)9DNN@wF?4<6th@
z!Qq!!d2Ddf?8@tvzdFHCUJWaFPN2s7tI$J;{wMUr_^)KQG!d|?0jDa~B0inb1D}6N
z?CCAe{_eb>gV7BB1C3%yyFzZf+L8ZAt0OEfvFMo>__D4W$W~apwjO1$2RvcOc0n64
zCY__;sNezH6YbrFZ9xh3bG3_Ta|6V)e>5T+#?OoSs)%sO;RO(^Kbj>JyaN*z)Zgz6
z9|W&`$+zkvd_R#V{kt*SNnX1Fs}V@tHw`gnjo5_M@LNlsn<yg*&(Akz$3V3JU;8Ur
zF0Kl2;?sJ5gC)KPBTj<Vg8jF%v$zu$^6Q(rS1Bo1=U><@p|ok73nr|7$iBi_{B_)9
zt*c4sR4gkdH$#s$Fp}_mH(6Dd2EF<5yQ#Y?_O*Z}OH>^7YKt<lgS3n-Zvn-1&tWj0
z@Bpw^?Q9SE+!h@kc=Rghul#er;?mU#pp5k)TqK4uA_s$|X#(NLWcdu%vYH~-8#Z)V
zPA$_0o;qg?>#$`SGkP;<bjGdEo%Rq*sOqCTL(m2HY;3z^!#e{mS!>zrDKORZPmBbF
z4ZnOEg_F%?$GR*?xme{iRZ@)B5v9<~!E-nGScJT8%_p?EET9JL&~d+@N7#G&A8ZI*
z!Y3CijLVj)bliTNHIrkxFP|9+BYoMou}csQUA7lgZB7=Nc?Owpugjdcv)DbA;9Cu3
zp(40LojrMfK<Sgc(Or)cw<PnAOMiq)9-l}6tvmAP8k>`%B@b)pJ5=cjk;Huj3Gi3K
zB2EKlt#UhKiuiki5AGh)pmHgW)oqIJ-ml@d)Pg&PdwE`<6dPT(%1DY3BVyc?5^du&
zTNKg0k1;nWtZEc~)Ke421BxKy9nJ|SilF7b53%<;1-4w<qW{q0z=}Tfq-_cETaI}T
zb1}we#0+BzPk(1e<vLn_G&P=kOMDzF`$B13*;NAeYn8wA+RY|QiKWOG#iXDE01|#P
zRgc~AK-$SJy4wY-sm>oe0P#o=P|9$RB8O!F?n&^I-VohG1cUHosPX-q*ygZpZk}b-
zarB$c8-%-O<P~pJ<s?}3oh#Jl0T<9$vte38TMQ$es`l%jqYL@J*>nx3M6oXyZlf)8
zmP03*Ow<4+)nu$Hz5Ffd+7E!exc--s4z*`8pwAeU1i;t$Nq5k0kXp7UC_+zSp;vfX
zkKe)@-jWVlbv!seZHiwl94%%AzVcVI0$iry3E%uxRnf615qyG>+^H@Oh>rQLF_yGZ
zgLJc$H32GbcTOVf#9kn58$PG6<$r2&6iZ5H{)^<k<C+YOcPwTawPiI^PVv<VSbx;A
zMWNw(rde*c)m8CH2HRB{K*@g8LAt!+%Wf8f5&RxxM1~{rK&rF_E8$+L{n5xm*ENG^
zXmO^W%xM>g%BbjEdN>yGt`u`W#D1DHa|fm=KVwFI7oobDp|PMUd4Ht7OOc{2G{5%+
z7th7-c|PIWF-t)a`@95E`gai|dac#e0wcdOSdNUTctqR;oYFk@VLe5w&#`upP;O_4
zayTPX2OY%|d5(tHvUO9H=bszB-hcZ|Z)`<(l`}?yk)bdxESg0fk;CaP#o-HxSrPx@
zL@vU#rqE5V@ncgD$^e@@mBs+Yvd*oz%OF9MiiILmQ*=h50qbIG9@CGG@R_Im{_TSU
zcc#%{k@)9HcB5Nb!H7FMFc0w3g9-SC$H~Q|P2dlu=qx8cq3*kE9?_nx?g1szYqgt(
zBUVvfS0QWHH#3bE&pmH)wOeh>8bNlye*av?ocny1;c+-M$D9}apU5`~@n$?o`tTuh
zN7Q-@bQ;w$WoVEMZRyEJMP+nxjAMtKpSc)cCk+)3sOT!C8@oox>wKnc_Rd?U{sxzs
z_9lXb;?9tFHWc#td3ccc5?Mq4y;-p-fqSYWLvfU6y#DjtYQodUrFtdVK4l~hiyz$U
z&PJZi`(R!9x*u$2=2gk}4|tihcD1J20R(l3S(F;1uvpmdAo*J?#+yHwO+p29&Oo0h
z5=4a@nL?mlUm{mQ&wSKqK$8~=jvxm_E5DPUk;XPb2wTRqD7nfCsDyF*uUoulUQ9p9
zVjC*YHI3Z5Te)|uE8$Mp1l0Arx_JEJ1Vv6Tdbi9#VSLOGuo}NT(8|MkSs3FDS?-1!
zaUs7Kn_<Kw91ZQTJ`vXwbMA<hDV8L@qp%otdeG@pMs03Y{*^OUt*NWz_w>p~9%k^y
zp7OX*!Hb5oH8ed->VT<^PA=b<M3oZG!AJW7Ja@UVebFHPfK8Ubep)+-DqS(}R?RwS
zUBxO7y?b+&Gt;B@T5%Yg)OYx2;Ci5@qIdyjA>GlEa4TnlwyRJa2roCSb51qeKQWt}
z)^!}rtkMxE)+<3^i(kCSYv8ocAhQY0EE#g#-0T{r9<_e3tE0z|ib}N!a|j%9noc=(
z&Ah~c?r?Ak%P2(^y%&25-kR0*Xr4D^K#1d3?^b5wDR27>Z`bo{Zwk>57071#2HY-9
zS^NIfd>ZAhUG%&|LD34rcFZDSLI;&OZS@k286z$#)?NLj>iuZhFL~KA-tsPSVFqUA
zxq7V>AYhZX&1p>Lpy(6;Xr-`(N-gIE3-9E*X?G+|qFJ;Nc!B!G7Sd&YZ0RQr4=6$k
z*}Z7!3&`<^or8x8(B?*O56!S8?r<x7@wh6Va>1I_LJ<Tbf8cZ}qdGJvw0>c;`Gzl)
z^h)d1b8W~1KD}u9N#Tu5{a4aG`wCz&c}z2SzhyX`*4%@*YX`=yhWN_T^DyPp!bYnW
zYaq8a;+dciGcJXCAU6r`1!y@*KSUoxuuu$3=72A7#3Lq&xjJB|BfgUFeL(xehkw9m
zh8)~^t&K%YY_War@H4~rS;KxNbbp`j(TAAyzn26BIPJR1ahSHt8#Gv=$9Z7Sm_@!&
zlTo2iDS!Xw%=i=2wKv><8!>Var>^c^JY>RscGv$=`{~Kw%d-LwK{>y<Fq6^Lf$_Y|
zMP6kSK@C7bGdN!_7UuC1uarfLUS`BuO5X&x3uAp8q&0)BfTv#^?qt7vLWa93eKSF@
zd`>MmSkQh#lUWyLIQp+1D%?QUYpok*?7+~2)}4UTY2($Sf!&SYF=q*Ak&9%Gb-Eh0
zVCR2y(gLvJAW&F^TwW_j`gKA<*n|WK#N%4MWBcmT3}!g|58R=_pcI1kY}}BhH`aIF
z`neWl&mTHx?D0bfD0P%(St9mOdZ49xchsZi$+dm+sxZ~#wr@Ip(~mnj#-O1Wremki
zxW>*7vP~yKg8%jwE9f{*?MgQk;|f(%SZz{G<x7(_TeR3ClJSrWUeIeq1SocTHxLZs
zDXFq(CvU6nmBSHRG?mvXm&*q^ixGjk@r8hpqqKR7xHagRgw-T-Edqvpqv$cs!Bujg
z(_L(r@R80HZ4l|MYbTdhSryZ@QM}f??2CI6WQpzX46}YYmC7%;O9Qg_P#m1GCx@(D
zSjb3UwhugrY-sToi544oo_53e&aDMD-@yt0YZbGBO`^G5m$CS@yaqG3XNbJB`w-Ko
z867mN79LUezE9isJHidWIJoA%Gj?NK+T{Bam0Cf#dwzd`?0aU9mAE`We5pGuZb-P$
z>+4gLVjRqAitWy5lzs8CG%VWp-mU7u#-p=_meGemu&C7uHLx=3P}$a;PApFGD4vLP
z*#H|u^RZG5apd}AhGAcszPF0Woq>XZi^QB&=)&K}SpetD1#-7KGCmX67VDd`N*=M&
z?fa?N4UA)-eiPgu{9V`JcR9bdYHe#%N81&57`Js~`S%dG)_DoW+7CK+V0j5x4nZz8
z!ie7bRcJ@dvVQzieIJSnlff^GA-72)EWS$PPNcJzfb(=N<wKJb-5+a1%uQv=ukhdY
zI>)qha;%praX+}j8OyAJ6{JBtz6suW0OAydw+A*m`-Ly|_hZV|-$~soZm0aEU`X)v
z6}0EWCCoBTa<-<|g;gx(zUj5?-8hCP)3?i~_-hS6JcC=|o}iaia1)qDx34s93A}_9
z-*1WdJKcdQhb(m1k2T4xMKjMHgK+OnZmHaK_N=xw;B;p5;v%5Xe%&#@D!fjzeQ~eC
ztX7y5-iG*|JGdpl-y8kzL5NM{mxo^aA%}dEyfnv#<;yWifNL>F$7NoN>tU@#Fyqu$
zZYR{3>33=bKN7T;LbigfSEzSp0EweJMICAYRNt$%z_&RIT`Oq4*&N}6fhAM5A7c!x
z8<m%JLmpuIs>=&1Xk5=iwXycnH?L+9IvghvnOWTm%~#~E4s3N~!d=gL-45h0GiDFO
z`iy>yt&0|2KeXukszPz{qz86w%wb!g(P8fL&$=%+l5Mo9W1}tXNZk@!>FXb+1N;(F
z4t&+4V$9=`5?>qEzgb41-z)8)N$>&UoTU5Fv4KKIuUtP}msJ_L52(U}R~>dVU<q(^
z$hV3gOvF!&XMk71<V<zDgSV9i^1f1JNH{XZ+8F$UDv7!G!k2i3i~+(6`blRD?eg(v
z1qEgi-8Ux(MchQ^7vO94)+Mo<apiGg-(=x8s5lSC2ktrOlS>>%#n+i)HG{*j(b8Ex
zu$b4?<ze!R*zBp<rIOb}Km4o^EX2I%OAt;iYxprR@Dv~}Gl{+BsQ}z4NqUKg!mbTC
z81RXuIZ$@q#>gRG+>fyXU);o!NN4R~w1ZT1yL3;RJO@Sgz55T$WU?7>hUncfha|p4
z`DmIWqiCz!s_SH36(*oNIbBDj)oRY0=?&2I_N-wQ<34JkCyckB-U;x2FUb5_Dt+Tz
z#;~jXWwA5=r;)*Nzl*5&Axw6M;c)X+{*2w(i}AI+O1gdM-64mUQ%YFntDJlFZNTq}
zCzukJHNa3757;_MYdZ7e9%c;_0nJcA0ZRBx;(Lr)C*s5A@Lhm0^STI01{_^VSpL8^
zUmh?1C7uPYI^uAQUT>C>dNGAhEE-E2xHYaU9fN0S!9mYI7zkdtnvj6yGVqwIOiw4c
zly@~wH_$3`6D*hI`}AsTZOrbV13gb7DSG)1E=jMi<f!-r)>3p&z_<q<9v+dD1~?<^
zi&D4vl&c=R;7*@*bi!Pw4_Lrk+HW_la?ji=`uIJuq#kq061*3I?_rEokH{De;f%on
z*TUVOO3$P?!PBV(qg{KuG3$kMj}lm^?8Ck>5q$+bogQ~-I^D+oI4&WdNiksfNX(!(
zd#}`hh?4&2H00M;Ee6+Fw`iEEt8?vNmcZb2N5VrMKOZ5I+@*vkGPWIbGebL0Z*4Bo
z>^dz=xx0LMJh_w~B+>W;ztK(2{NhBv(V%bdWhzHn7;)x;^>GcFKnKZVFphOAS6PTP
zV)Y$dm)kC~8yL~2J4UUCRacbj1NV&X5+jAfrvR>$uLpX9ZZDRjTc9Pg#+{Eav6d`S
z%qeN^Hp+<+8O6!t!Z8Ct;=n>2%IUVAU?RJ+jkQH}#wY3wK5r+GjV)1k(9!Md8t|3U
zySMIVe-rVkH{>z9rgG;KOqp2XPM);>`72Y^KjV`-NSr`eNlknT;Aw+#NQo;O0CF5l
z&)`BWahg5H9}HTM1}7=8?&}O?RKS*=(_W9&QkDkbgqktAb&O(RxMB8k3WV*4at5EE
z3@omQ7YKx-v}{GUpz%LmHUmU3?=ilGd#TI3WrGft{B%o|`$1vJ|K#LawCrrrZ+G7e
z;M(81y7bX0N#%mCLb#%;r6MKYJ(+#P;*Y#l$O*f9M(PSS+NDzi0g2&YL`LJAxE>s|
zxA{h;z}$F8lw-1Zg^XQbevCgTm;!gG+TJP-oL3B2^I{8}b!AkWCyCT5TA}o4M&bVy
z+yL;Q`7^T+b`Qh;;3eJ$uB8e1qf0XXv6k$YrZY2gKt>xdM-$#^T)myzeA&<l&Fl)H
zuCEd}RmrOJ`V86Ymg!!qt&{O0+8KthBNF9qO9P+@wdvg%nE{KRVluAbF&={L1<d8c
z;#2?cQll}L_;IT%9`9*<gXTXr{cq!h(F+>vY*JGU47lqgQHFv{><s+IV1fU!h-m5n
zn$GgL-##_LH&u`F&2HAHqT}MiPNqBgv5P#8u=0#<6ABRMZxx|8PAR}59t3h<aLK;~
zB>}h73q$sRcstPS(az0?07+p(VtP`26LN4ni}0QsY%N|Pw$>@CRj~Y%S50ZOs|Q6{
z`KAo(1HSaxwCA%&nuH#pT)G3J(oW3yczFC<U|K&XKULJzTB`kX77!)5VSIJBx^1ms
zs1EPXwOjYGA;b|sar&mSH*f-QWwShXz@sI2WHm`y3ovLM4k2fF=><|{JoKf|>tDCQ
z;F$wAn_)taqJ$#$^NINh!Qgg&Tvr^M0<1s|`4W$Ov*I^3_Fn0XAj=hC(|;)Y(3?c~
zraVc_C0YAuMV)?mmzC`vBhC?CVBHp0S**;%PHXChDGeg43i%ZMC4l*Xh$>^t(?)#}
z=5|sSEFqZTO4PB~`&1DLjhTFSbtqJXLHb0D=dW`Kt-dWapzX@<?zaNa=c`$f#ki7m
z!s@pdFbM+)J*X019EGQ3;II1_xCG8C+da5nK)LFXlKZNmUlQ$cp}ksFBZSq(luNXO
z^(RxyK14%?W4H2+^cNz}j<>g9x4wnm1```jqC`6>3l%6uu<+83{3~Qa>i1Cm%aKiA
z&Eo7XJJEe|5F_gtL9<pln3OksI;W!AXoHFef23|}2xNeFeDQxGgM)V8N&g8C38>u?
zq>mx8sNe2`G)bli*x%G;NpZAv>OWJRx8q<;G0$_>@gQGUP54UcJ^8AJ&FB2#fS-#Z
zQT+^Zsjbug;<|PotJ~HW#a-@{K?(6H^s}Cgt<n}r@&1!&d}ll3k4HR^&Ii}9V`yja
zD~m_1H;~eT4yfgLCf!vYGGUlnFt}_|*U6^Twxy)$!<yTJsrfx)(Z6BMOu$n#MD1sA
z;b}<cL8vZ|wA)ch1*RCi7fK=ZVIC7&T@;Pih5tUd?3ES_PkoS8av~PJ@C~u@79G5o
zk;P>1s4LW?`elt4TMzfy!295*9EBR`{0%`Z3dXaAqR$mQ#2Wja@GTjvwjB2Tx_fI<
zT$QrbLJEP(hdYPx1`i$Hi5DCKU-tqRvfmS?Tal(V@pT0+uxlg1Dfarg6Y0t{Jt$jP
z{cVjpp*@ac;N5T88`CN(bn!=iZU$Mm?%uqHt%tE{S-Y_h13sI9{*jf9OqE4aihR-u
zpJXJ!K9dcf2mhqj*#uZEe-KM=kYHUQ5jI)dmfa#zXsoZne~;hlDjBWoW!7y-U3rug
zPJ_CXFI%d&u^g3~xH{W<1dZm7xznUI_jhq@mRTCq{pzZi*Sn07tsb*(MkW1ZtGL$j
z+?0OJaC^4h+To7t(8kO-B~}JzEfM7Ds%piyS1^l>53G@~f_-M2rf$NpB*`a2Qq~`T
z*?Wq7^XAB+c93hP&o2Fw>rVm%*5A@x^+yj-f{it&w{$t;MRk-5l~myMNl}o}GH%^@
z3gk~Wv7<C!vYcdK4Y2?py_ccH4d?Tdn)1o?E$q$Ai`l+WUHoe#u-+V4+~1@pshOM>
z`xAT#31<s*tnR4^5pa|#6Dq5=|KM(zL4xEAT~wthB2?2~c0cY5ISfVW=y~;97^d$4
z9V0ff93_5+Oi5)(id#ds;O`Z%O5+C3(c98nRyn*hemv7_yi_*7Pum{Vuzt<BMDL<G
zUzFia`@38K=_iB3G+c^hF2s^<zl#Q5j<#c(-pr>X9=I2OO*ajxGsG&reD*^4X)ueb
zn^+`Xfg{Q?`R%ZS+eFy}qiT|_+cnn^Z<>k**;mz(=?x!Z7U{MMKTN&yILL7OFfH8J
zFvQ|dRjq7BPS+h$)oallfj79it3mH}W`%Z|R75Siu73W`4twygc&9c-MKJ;Yt!b+!
zB7PtjObm1yypm`oz1kB0j|h)eB!0#(v3hrGlY`;GZ6@j%^|wa<u2d|-L+j*ZSO5Ht
z3?4fUL(XJ)o~ykg{x6g()o???#HWLP%m0)8Uq%$(|F52-o7kV6uo^vQt3$is2zpJ2
zTd*44a`@-QPsdoV_j73Q6$wdE-laOCSvi}%`lrQI#wZa|Lcp4Ut}<ftVJP?ErSVKc
zIUfnOA_bDtD?sNTK3`ophGT@)Xm29x?oezVpGv?aRQFqgI}5)=<87*9?*}j1C#l{K
zRx{rEO@lLnu4>A_YIGn9x{WI@wY`&DO;9!|vJZYyKX$m2*wH4RWhv!=vN=Yu!wg9D
zlce<RcLWdMI?1uCF{M!C>vnOW;_0&G8N8R0pD;PVoptb}{a;h125$d*<#CR%`tU?j
zFmIqAmgnWhd@)pgpPWu@4a3V%>Y<VSIdv8~ie_k|DRWP75$5F$WW{FXP7<2;qh748
z!TFGCnu2`)FyVuCCIA;z_aaJu*Z_iBc3M5!3GWNM!cd2Fps&z_jBQDv!zc~)eetED
zYR!IG#wZ1_er`*(pz0MomGBU6U&M>mJlK1ie>uFL`#%Q$%Xv)yEe@!GcN(E*KR>k6
h+zMx+_NyRp?6tG)-`J)){O3v|T+2YSTHP+{e*raJDz^Xt

literal 0
HcmV?d00001

diff --git a/site/assets/fast_python_serialization_with_ray_and_arrow/python_object.png b/site/assets/fast_python_serialization_with_ray_and_arrow/python_object.png
new file mode 100644
index 0000000000000000000000000000000000000000..b1f1c72f5e5685d0d5954f1180e5c9f7c4b7007f
GIT binary patch
literal 24165
zcmbrlcRU>57eBmq_1^0$(Fr1Yx6wNxM2Q}(E_$~T(Yqk}>LNt%o#<^vCy2hHvxxrq
ze1E^!^WXE=GxM5x&73p$eeOMT@4V04GZUqwrA&ZJiwgh%2vk)RbO8V$;%Nh8gPtVF
zLVHU90L@cJ<Bg)So5qvm|Nnv-8+$w(o7^~G{5$=GhJlXmZ)?j)M<XR5(43k1PE&KX
zwl*^~6xq{51E#~p#bIG&pr#-vB_R>u<^J8=tRpW^NI=jz-f}WL%+AVUVWQbMFsG$&
zf~c(Q&d)E6jfMHy9t;dXqy?cL{6pi?ZT0nU4u(4GzYZ@S=o)HAxx2fWnZcZ#qP~1C
z$P5*c5PqwrtSl*cw$qdKt+8wB&yVEfKeKh~?d?1q>_Xzw)0^Xt=7!E*4h2=Y@e%IT
zrExwVV}=&`21-g6Cn_u2M_!4Dqy_~IcfmbfOjVSmrRC(3;T3;Z+eW^9)6i0k_x9!!
z5J~d)S5VWgPfIJT?Xu9(iA;rmE&5_;39ap}%t{Fiitx`54}WKERMt>v4t4tw5HK}R
z_TI^|uO%B~vyArSBWh1wEj@rUYtr&gC06H|c3z-7KMjIJ!XkX%XgI(9%NqdTU8JhQ
zt2e$&2ifB;8){_XUHvWCVXKV<n0KFE8Q$TLu;Llw_eR1O;gK1NI^=SsKdp+!wnI!;
z8m&`442_Y9vu-AnW*#-l9}I3ceIL0y4#}`dGqU~P?(y^erv#hF_gGu^@%_p|Te4~E
zTUe}DmiRQCG>HGNHBX}XTpQC96J>YPl!TV^f7kmxSSJ)`Uz>>LEPdYAX3AAJV8!9u
zGUU5g(uk>G&HP>Ynt8Fy&h<6LcxEmQVx^#ZDq*1KrORDei-VUA+`G+?C+vQ6gY%-j
zzE3Ii(uh;(QrbCYEAb!yIX-d?&`L3K5=5FG5q)5*{!l`sPywtI%5IAJX?9N4t+-^{
zHX<a80gDj?8RHN|8fby7l<{sD#rH7a<b+Tiy$LnHpS`v?0JX`OT&uqs8u(E}g?5lf
z2E_85H^?FfHjVka^dSvrV*~PnQdX}_kOZ{U*{lKxs$L}gx^o}1Z3L9&3Z9})bZd{}
zwjlVT4EmRhe)HWxUHo|h1RrU+7v8o^Ytgxrp8_#YRiZ(35eU_^N}a$>ERIj-x#U;U
zA-BQ0$x>a;BeraWo}a@8)!%iLv<vWILpz_KH6027tl(qfvgFY@eOf9=dP-6Mszj}k
z27vb#C3Gz1Lkf}9g6j*wAF+|_)rN1mi(VMq_1l&CS_NtkU4&j>x<&P<ZAZZ>09M|s
zt}myY45+^z+)yB_>Si%vJyl<d2@F#`FeOTX9(ru8v_Edx-eMFn9VN%`0s3e^1&vBN
zlq>zwN3(_ui4KTZ#EyuM17?hA5t-Dq<>m5SOSFHync-a~Ft04M`(4N}tkTY!4133`
z=o{mZ*E!z+4@?@}$`^-Pq?_i#bQQhY2?)5chfLdc#@+OyH`dDot_3lT?W*0$_~Vh%
zI^3Z2&`S#)4-8lrfEtswwY!@KvU$^r$&aalOa{~o!S!Dj>2v`N0T%BtAPL*+|EJMG
zUD{^e%B#()y1w$9n4M|wL{FPe&n1;I<r0V84*deE*$H^hA27YTjC$CK8Mz?G0GWsS
z#efIW1>s2wWI+MGlSSg~v!XMWL1Qer;tF7x(LJ}1b-q~|#Ju;{HY(tcdeut+t`-3z
zrwPru(SAe$c*w5-rP7#$J)Cn68u)J71S`gNpl7DV&RgU%>2X^mu($9JCmb6n3wZ+S
zm-GDXs`867&fiV$P>VYKc<EU~84J=>A9(5Z0ybAHS?SJ-1&CPSGQBXHN2tQv+NII<
zap1>*A2K7ok58RY29c^}Q?oKfN0k-GIL3j>-TKF^gI%Pai)2e=%Idf3@lBD|3nYh5
zVgMRVbHov8<pvym3?YJm=WApQggON5Gp~sX!Giu%Md&YuHR~+Tnx;B~K~QO@X@1u?
z*Q-c|B((boTH_^Bs9AEA8Cq7^_dHCY&l@Lnklp*dbh+muld>bvDnVZ`tD6NH!?M#%
zX)Q1BsUVKVheV++gP{znaAD_cwEJV&CC#O-X>-xHZ-YPs^b<Zl@wDzwLl;vDay}hF
zXKR4^rjgq=UanLkpy!0*IO$(2fK;?_KIU&yTM*6Yb8NUP>}CJbmV(xwz_SJqe5h!4
zrAi2+YJ7eoIL?o9`(G18?;njw)NkSLOQPq|6AcP~`kZ#=@#mAr*m-fq=Ponbz1Pk~
zEznWyA<djlrr78GwgIhToU+Y@`8khhFa}^ZL5(pM`;~ta?pJA&ZO*MF%D>VroI?2$
zEp)b=TMdpj#^kGHh;2Vw#2R$m$2uZLPTFU(SJ9HLOxV4t+U^(|Ww^7I+8YzN>t<D+
zS%CHJ-?c|_XVyp|nMmifjD6i@556>5k{7YhiPeceuY{)!FttlHTyP<J!^wZReb1!0
zI=L%4iG#vMc9&SyrJ^-1^1lf*4*A6>69C`kiJhm2{S#8NByb3fk&ZZQh2_7bc4pRJ
zXacK7==gP^XUci->4D$`Bov3}mThPj`e;p8@sNm&An=<K@(MWS2HzXS*IIh2-z~mT
zSeN4su_bVsvVsKvt5B{Eb#KW$KCA`x9(uD*%3qq~{m&lcv>$^@h`I1(-fTVrkYmmZ
zsNOGWr#czkJP0i*JL5m$`f?e5@5T2S5|J<y@tRk8p4DHn;A0<Pf#vP*R-9&gh-fsv
z6F(j-w*}_M$ZnwE5<Z*#`xa=*GslNEMw(ks*vw)Bd=|6u)KR!TD}_@dY`>vMk*>_%
z=H)p6<fpNE4AHa_v;Mjj0sF(x2$8JSR}!}^B!K(xvc%3idYky6GS#$(o{vt*9I<yE
zIlyNv4uHzS2%pVdK4me&oBVLCt&!0w;i8DWGwdRZ6X{SwnNg<eM{9^zgsIX?pHtP{
z;WJqd@4%gi3(kJGJrisqO%?-0aW*wlzmB3mPtU0(t~RvF2yYJMYg#6|k|T;}4d$q&
zuUa^G&f@i9OszXXPTWT48so4BixYp121<{xEmxapDlGRQgpWupp<#G)TA2|fqsrB;
zO4~S9kI88)R<^+u{4?u#@av<c!04X0liVOZ<x(tv5PYLcEe(w(R}8m)$ceG0j)zg2
zNfUedtrO7VjvY?q+UAOOLqDnvf*-@ICW+_EfMP2udTS~B1|Mm$3(26!UDlw*FT%v~
z^Z<A}AX6@@8sO<+iE;QXbX72*&3~n|b<DIAtviprNJhd@v2M-|-}9Rbv~FHChk9;)
z-~-%_+@Mpf&~!R`CW*?r6lDDR3oXkZhMpbrmTM)d4O2ey$4r&$FI@eyNsz&06wBEk
z8>)+1!WlW?RCHr!J!0g`(zljLQm?gs<ku-zKP)L%7jQ=KTs`Er71;c_Dm#%xZ)K8?
zCqVP$QFV<jRw!N8nnqx>v{UvB`}}$e5kf=}<>d~-B=6XFD|B4y$)h3&Q|&M|piCZ_
zYEr6??>bTmJ-NmbZ6S7$u9R@XS{{pd)y^CC=%NJsN@BphFuFuq(IjjFCSwk#4`<$%
z7=F7MfvoxVhqVmv4wGH@$!b+^QG=ZG*4;ehg~{rN0)xtkE==U@#`uSdk^WyCn8xU?
z72$!9h&T!7=m`S!&=UEW-pv0{83CK&dY@mj+tQu0K$bgFcyUe72^21BZX96@-Qw~3
z&or2Vwcu^92?N1=lwGM&-K{FWM(rHG_$CK46b%ic>gCA~)X`e8E!>&m(%e+i4kkPI
z!jQL>q87p$aoi!e@PNjO;R3p}0l?QLf>$L=`LvO8hNC3vzxIUzBBnbl{$E@)&|0md
zBBr%~DDu&Es-40@rLTL0dB=?+r@L)r=G^Ctu%nm3vILuDlX1V_5p9m!1V1=vyl=|c
z6{?=TML;;>Ktb0$RZDMAotiX$P4@kZV4yE>|GIblSvwa4(p4Y+BTr5KMNP2P8|)Dp
z=NkCNP(UqO+l$VjZVY2h?ms72nAJ>)w^^wH|Hg&#Fp-1<O_bEdkN(jFZxm&=v@pJw
z<v3}>%B|1NSN@#8SDn)6$cR*&+6A<(@afK8IeMD}=!Mruj>;`nR+w_;0+GC9hyHI%
zkIAbKwo$!I=Is`@7ydDM*LRuD-jehEe8+7AlY0{a;@eqqvrDppx^P!ZhvG#x)H`|W
zLb>^R`ZKnMqx(ot^X1wjTe9?#dq?WuljX5%naTsAnp{D&@b7?`x2|FYnDBE-__Vr$
zww_F8Ris0-;{cb4LBWnkra9S&oawFCTgv{*Gv3!fUv5JKG-M{u#6urGT7AMkI>~#i
z^YNCp_cQ@lIa4@KZcgt?1uZ^v#ACrf2Wor;XkaH2|B(ZtM~a5QsfhKW)tdS986OWC
zMDPnRjGr^qYgnnRVCNj=#XOo-4M9oODkiuThmXn3vE9b7c6n6T)wadS;VrvA@?Fiv
zU6fMPA4$-%_GQoU-pO%XfaVwoJ@qMx{t_IgR>T}O@5#~Ub9assT0R@4+XeZBk44%M
znw{h$N;Y~Nb*9r|xv2H3%FFk8FiMkeM5c$U`g^VPgjnvAVZjq@hSDQcm2$n3n1W&H
z0ExA~OV0@ORBtJBNaIQnOKI;WA|q1M@#?u*%m{L3A?NG;7znNUbwQhxbQx%&R2n*8
z`u%~G<>>$=vJ}%;;_KAxOIU2X#I+1#efV3J8uGvDf-;ot30r5WqNiIj?HuSg$y)Nr
zznm6>H0=%**mBIzUvXAj!|nEhcOY;0)~e4-QR~j3X1Jbj3q7@ajsV!xmV=UCj2;MT
zSVLXV4El7T#T>D9D<M6>NMMFVY!><Zm&v-;X!Z@-{kmUr7sK^-hJKE`;T?CE>cH@q
zhl$)yH>-wkoh2^0y-|scWJ?C?lU1iE#k3}|hs82<Sip-vwLp>p_&#jHPJN-t%oeX=
z;R9xvJviGW@(DF=pE>~WE{4aSZKd&gbOfI7dZWUd%W3*XBCpQnU?|C)Sgtk!LJ8>=
zfuHvLMK;fA#(yLiZf*me!%Qps+Tr9HEZuKLeQ2<0GZSh==2PDo^-b`<=H%X$;qCu*
zcf$s#T$VjOQg*AEf)L?oMO~y4bPvAS9!JZhSJ%m9{J<YB;xAlH#Wy%bURgT1?<)b{
z|AiLufsB$11o-Sl9}Ax;VB&nVMU$0iJoy*7P5*$0-=EzDfb0LG!84r`dFMVxl=}5^
z_X@R-7bQX0#R-urW2JUq>r2tEqrd8)&9r4*U*8c*5R`(BG+RqQ8tMBe*o>WxKE@mM
zxLSM)UEPs?Y`I2MppOV+f$oKqII$QMZD$KGEOcYrN<dv&8!d9<_5ho$cBJ-S$0#o6
z(BKoV>MoCeSiqo@_3^fOk0yGI`=2-tB1%`|t|Ki5w&ZtR6{%AxdneUoGtO>?rt@ke
zaiKvuw3eG&^&J>D^Py(D7dSHa7um4jq8P?V$u9kB7ajOo)%x3HcjKk8{EA~C5z8i8
z?h-TzEXS>WLpans)5ES%7`5|c(cjq$^e?^7&L;m#I>CJze%R4z>cvJT9e46mb$ICr
zUM1qS#Z=mESqqR&vKwM4>Bd<G-ZMQhYis0AK!~g$LC53iOPAZe-RtGF56qrt*;a!-
zc|^R8#@zcd1cK)r`b4%F6x~IMvZ>_UCU8b<)ar8AJP5Tmd8G4i@~BPtw|7$JkNH2@
zsjtFO3CVky36w6n;x()tbR@4n5Nd6^M=LfgP$IE__x5(C3bL{<2G>jH6GE2l=7d~y
z(?*u;>pU{HU1Qje(*4j33Dh%+8^~?yoxq3`qqWVpYme(^@iz}zN61{sy|zrlnHEQb
ztXZ7lzZEj!PTWxw)?XZZdX6THqjC=q-ZNd<pSbvkyIbA%DlUy}IsNka@Op88m#BeP
zTT48)X#Sg~@7C3&|8C6gRA}Qw$%hn{zvje6B4A`s^DD0v{^^;E4nEF7{^Z`O4BY;4
zkiQ6qBOjF16}Cu^tQC5`*_asr!a+Lo>qLH~S^qQhZ~g(-I#tv8?r5$B*UOM&`U8KN
z&ih>)m^YUDvd7G~*MA;Ts(HqBZ9gRi;l<9p4Z10#JfBT=E-4%AzhT4s6SbdpnHA7+
zwNv9N$+a%je~2!$xg#)m|FWceQSodjxsa^TSy?nF_FK635AX+5HswJ~pk`ysbe<!~
znBW69C)i=3J_OUVv$lJo^;4>e#V3MALhA51G6IZcvwuNwAMdV%cK^C#KV0};thsD>
z3wm5xY-p30xZ^m6L_;^;{#6Sfu&;hy%+`HnkRZ(fX!z$WE8x)Y_Ik>7`9qD?9>ADH
zHa}hrBK{3$lfDg;-vM*Snyt^!>>Kk|XdK2;z~^(l-;P4gWTv6v=n^vXwoQNCrN5Eo
zY2qgM*t?i`;pcykn#u#xcS$k?&fNu4AR^iV+H!?SgNUGMb1qr0?Us<-RwvkKrg@@&
zj^?xqdif9<yzbHz*7gzSS*qQ>xSm0nVSz37BrUWmFV^;zlc!>WYnZ36-iExn;7gm0
z8{hs4q>r$bQpoT*MO?MpYGJ6nt+2CzS~rsZc!sQdwf0t1!*Im>GXzgl_ar=U=(E<|
zSKjO+X?)Wt#rPQBJFdWA76`ojpTn8mr}Po6U_hskuCH5l)44;C!WvfQHSF4kAWe2;
zf@yC@JsllUEC4$iY8ZweGrI_Q-OH#8FqX!zqAXn~@MEbW5=I|{V*cSS9Me%b5&-I`
z`$E^CC|cF}SUcbzd1kk}`<~h3ZBLyNJnw@HiX53=T@%FnBIVwdg1~lNa$-Z{jd2=H
zI;{jqlLv<?K_YXiJT~8CTuxT9rnc{RwWhmK@Z@sfjr(&-=Nhr_S5@;0qiaGH8jW~C
zg{g;q2$N8kDgonMYl5oFwIgh@dk0s|rv^y$234p88)#h4|K1U#(y_&L3mKofE}lW)
z&VNc*3=Y?ixKp_<p>fa={gm>_`u3|haLr$cG;&T=9?Oe3MtBu_&Ye0_47e#~7#6ON
zuk>v2je6H>wam`Is<o~);Jr0G;?%eOmt0MD@jKnNS4IymjI`Nx8St<trZ(;WVqZa4
z<&V`a|AM58O?>Bt1bM+05^=1;fQOmMIe$(YFA+QaeVRD`r#t+y?vX*W-p+Nh&IPUw
z3Wcaau2R|1ygcLQS(<K&5!sqq6L*$4^>J~8hG^i^i#W55yR@yZT}YkCWER_hZ`>cb
zfl!U#Qic3=Q4O&R3^ksVMy>O?M2{|^W*ghp!~&~|XKp+@^l-@(c3<rJpJpdH7iULr
z(5lc|*+vIX6WC|{Kg*FFu6%T{(KqI9zu6z{37Jx;ibCF+5gurD_u6I_kC%0y5F)nn
zq{Fd+IbW`ANnSnDDAN9-s+on9ZgKQrkZ0gOXG|$S-6Yq(S2FFGk7$c9frp@@s5z|F
zib#9^#{Ycl$`0`7r-V#9kYOY7sSWR)A?z}JzHA5_c;*K(<5l*KHayQx9Jp8AbD{cO
zi%0rQq%cVk9TKYZj&O0%!|m%QBeA*xL$7WpEJdYf<mkp|>H{C|$9CIxw++!fb(9hb
zG7l?>5uwAA6ll#|<VFqid29ZSkQXzEX&E0sYZyy1onXkI4%0yn$Y1xt^ja9cQgA8>
ziD{x21>9V3?-iQMbjm7_-H+6DU9m>UNo<{S#$>*%7$|Rhh~JQgj|xzN?}XtTiklPC
zaHHC-T?=%B!;7pAUc_kVW%vz6GUlf}pYLg@c{ll9SuhvP64+p|0tvItj1fUSPdl5R
z`@EbUPYqtjHYDWE_s=sj-pJtdH;_6qhV@d`#L~43c{Qd$J~n0gP#5)K5_R1CSII1C
z9tNNwF7ubX!G8%6HSaTDUO-<{yEH!CSV<FjIw&U;P5eOOH`C&~!$K2g`Q%pSpIAWp
z^Z`#+>;re!ApEzhiXC6Dp{ppZHeRpw+|L8k#oueFK=&>ok2yk9`%L~Mh;-uO6IbJ1
za`mWf#Is7oiQJVA6p??TL~K}af^JK48%F^t<&_|(<m{Eg@{E$3bY3j5-!toa@cajb
zP^l9QR@K+j)==L)qS5C~MA5s;Ky8%S?}b6eCCS2Z<(|uN4}+_Vk3}^HmVq^#(Hkbx
zhXaBjrtYIN?`NDpVK|DJb4=gnJV<}MN`@LYIJjo*e-pjmKb{!t#D9LfQOi_ud#^~o
zNWQb#G&wPLjxVNk?i8sz6ymD?I!~+gyQwMMkFa<e=1NuR9M*t|#B=kSD#zg>8?I?Q
z@lSfjn`z{c4V8v3Mi%Azpu2fzV4pK(DnR-YL{piUknyk|fCrhmKVjNXqkjT(WMQmX
zAPx%Cr}yQBz5ph@DB&s6T1+Gl2dmW~zLYTI{-4A)G)_2+ypxc?yK9Yws8n}cD2WKy
zoBEje6Ux^tLAP>DuUe_0?ue_#?1mUdFDBIiI&4n@X`_%o02uUM;d{O#=k0y7qiv5d
z0WW$H<#@%l=m5H>q1N9=&t#>RwK!mIMyHaUN0SZV$*VOxXfktr?8fWU*W%J*G&CSn
zO2lg^98@P!?Ewox9HrsVtmdG<b#QMepHPo)=-f=|#Ot+Wf_JOgMGPoj$)0^B4#eE>
z%XK#CdEalI&?BzE?JS~^Cx@KgDxuex^y;jLP|1$}`rapm_4n0UG}&)AWz#ud9C5ly
zW4itByW~0u$1H}6m$<o4vE*~cmCCCG>2X-yE_$n@`^**Z#9|Ob%0ZU+YmXt`CHsO4
zC04GjSXn>cllCk1=XLvpK>Cy`dVS_?FuGsScf9rXe|5HDcTuS<nrGX|nycHzo1EnE
z=O((t%Z`0=%nyMOpU!Q9RHyrO--DJyZiHZ+BOY3@qNE1O0&d-cErb#hb>AGLrx%;c
zU!)j>x%8=21p#B)oPY`i9gYkuo4K%ek|d#L6A}#CYX?zlwif?NlI0z*a1oY%$CZJM
zoZct9=(J`u17|+d*AaHHFHa^TGv)24nE!OX1_fm%{H6;&l1f*X5k!prB3?ETf}H!!
zgui&Dhc2N98U5<x1Wu-*r$Mrl-hD>V1gPy$ko*Z3{d{USi%7{Ht@&p>10Kt+{cJga
z<GLbbTz%t6O|wFTq7umO9JN_MZ=u4pV_tPOXtC+fYP>|W$1i;JIlb&<A%eC+ZYh&?
z3=OCP=~?|VKiL}C%1;1-<bfZJlBUo}2$;CJ<HbnG&UiOo`KOP6Xk$i1b(B3N)I2q6
z=fkz7;p3rZmxAZ6)V5|THap-cy`1cN7Zar`&`KJN?-JB8O@xvzV}1SGp%(k!J3-1g
zYI`UVw2-s<bLzYYK3T!#7S9WGX?R59)IDU>@t>jl5X78p(?Sc)U-YpiLasJz%dm+h
z6evNH&V79cF?Y&}*y7guhJ{z>1GQ~!Sn(o8wuq{DAb5?PwDt>u{&u6kf#qYAR}~6V
zww0zc1BT1O;|?ykQ=}ZvnZYHn3FaoAHjw_~9|o%JnsO!8DRZW;Is~S|c}aRJ%UmAI
zj(g?oyyOy?^~V%*V9yxuXe1g7sURcvY)6Mpe*bPGkLk>WVb%2~Er2PS@-K&WT%lz^
zhLYI$H%R($QWlPz&gve{?;@b71AV5|1v%WL`vZLbElrX5jd8ua0ugo6`>~nn3c?}^
zcn=cR=$JXnR{<9eY=0w%Hoj`dv<9)oY>6h8UHt^Wo+oe<w1muiQ5G?p)G(s0P=)_!
z$p)e<hq>UOcT6xLW%C%@k9hkNKvP82OYH&8(2`!n?9ho*EhBQpx90<#CXaWThr0S}
zWrm?PX`+W40xQ`@`ORhM5ksN0O4HDnpf<l$W-oQgwp2>KmL>|g$=hU;+kn59V3;S?
z3wC$FLs^N4Bv7SP0q(eQF)bW%mcJ`s5I@%lWk*61S-iBKojIRhLA{!Kj+9k|p>|Cp
zf*)L;LsGKcNN%0Bf{B2>5s6ABi4)vO?&82^6>-eVV<s2}#^B&mT5RLV;|P~a_2S)-
zZdW!&>ja+gd*lymf|XZk$ykXJMcFG392`f^7)VNuS!1G;W<IOu+4B-y^@TLfq=(qV
zPTq{&-kO1!(UaKnctm^myw3cti91NwDJ^1xGT!^~1?dEz(}+dfOg(4!Y7OzW*>a1k
zMj(nK)Nz1s8d&R%cW3h=T1N$@QJlOE;mHC_DTrWNoVKPXHJyly%{A8ARcIW&|6^)x
z4y)M%w8r?*&HiW|(X{Ls^0K#U`mrT8k`i~lrsy<zI#A4*+|>3Q$g3Xjx{z+@g^5H+
zeG-l+QIv$#OVn~U=dKL@{_ZXsfnu!^Yx80o#4*!*{z)3Weqki#T@o*LAgT7^!299Q
z#tV+yNB(RYN68YS=Mau7Kh7<!#XZ0cF>Zsre_CNM{uH6@;^Mt26?9l9l2D&rnR>3~
z6P@{17A~rPXU7)DNfIPfK;uZL8DksRM_L(wK;m?S8g~zydONaMiC5oU_v@!qHhQRC
zqAXZ@PUH=|hKP-nrpIu3DEC7?#&-}}b9ov$yy?S0&fkMMj8?Y1`yif4o5%$hvMWo9
zCmb<Mp}JNrp@>FB!9JxF-MvCT05gXT29hB>#oc=!$eVXhutj%UR=eR{2&8AAL5S4a
zy~9S?e)E_>J2C*as>O1Y*DV8()@RQfUIOnORDXe&D5ZI(W`@|w7%>1p*e;iRO+&rC
z_d(<8bTka06$U(nw&-{(F;r;S;P{VFMp}iRD3@ImAHsAvXrWRMqHZvM{u+%w+d6W`
zccvhM)tKHWkY`QP>?w|NzvFPP`p?&)6DcW%o2s}-fko>@>i+{sgrqf~j$j_dMpAzH
z^^((0RR6j&OO9cUJebO)71t9-(;u}$E`f&{0kYV~EAwX3aLdq^jRF|ip2aSV;(q*+
zQ6|y$O#%tJAzULLunY%gZeyH!4G;Okg557;Z(%=5o3-|VpRFPx>0<wwa0q4>H7UHG
z2821&s80kLAs4*=`#Ud&GcY8D;q7eeG(_;hM%6>gWlCN#MK@yp)4ib+QR#5lBJ)4a
z|CODkUub(98o%43gs&{73u&?}THF#x7b5R9SHef>)!Dt{6i48!GU@k;e&+Q%H&6BU
z)}wUcR1RDJoElTGHCF;#JplJu*_pHl--BT$gP~@q;mpksF%02Q2TL|Av_F@4)X-~C
z)UOGbhLMk8l5i@6&TLf|XI!WV<#yP;I=UzI$APj?!uE>}RF>Rp&UDX;g^Qje=bPfp
zd&Y>ZTaWmgftC(1)WJb(4h%9z5i4BfY%nmki{#K(v*z-C<-ttpg0ZW9AC3>M$YU=w
z#i>v6irKwZI|*dH%|A^G7NYAYnJzdSR8DYue@d#B|8<Ri-Vn=fUJ8D(UtRuYu(xVH
z#;xr<amPg&L+F!%4y(yJjM7;$bLM79q*u-8bnqzjlTtIM(W?Q0Iql)x3=A^k-XWNb
z!QZUnwa3@!ToK+EMCi0QNYVF0m<`Z21rsa@D*ch@kDNmB7B>~r13SX+FULIRJdLZ1
zG#_ery4&vQnf+UN4ey1+?B_uK2ap%HG>RT!23R6xEmVXkw?#n)#3n}Os}*t1;h)#9
z23Sm{ihcn7%MNIWkS{@TOliV{-IUJHWtL1KSUaL-)Cl0$ADP9iM6}-(?LS(3!9{13
z5^@WOwutat?G7<UkoJ`b6Qx;j&uq`;-kR@v-mja{l&NWB-{%n(;-1F!o2OILZ#$dN
z5>ofdnx3h7AnTzC(B(xzv$!^qG|ut-`2y*JL)%GT;iF!j=b7?Vijs7U(s4F|0R!J_
zni$#{jpx9vyh~I6kb7UIzURi~MjM-!2uHZYTt?=ape!=N{kIE+BZ6asai<=*vc6uK
z`00ht&QISo-oL<s*7lfBfp2(ka89TGOzv?z-hS46>TY$6{^gD~Y5BqtuVNY*H2-SB
zpQKW7j)FTuLywBDP0&$=Hgc%#hY67XSxi}+V%}#;=TvAD&^cLp6HS`t{B_KIW?z}2
zNo<1ma|C;d5fL(~j3$t%!hx`l(wR4NrU4@Os_>4U2szC8v9??s-=v<cU<I$<@-h|+
z6;?r=SVl9u>a<W$NqlGpkE{L9vVy{9&ihbVgNg<y-suz{W@g4z*NaP7ZmcoKU3q3-
zkz$r|N-GVNy(sH{HDI&j`1U_3>7fHWgFt$^;t6;rP^D1(>nuV?p^Xl@^54zG`BqF&
zxr3dFVL>yf{QnBMRGoA(`|1_xo*=={j{h!<BAG&~-0^kJd<|8o$3=y#uHW20)P?6y
z9(PgdWrPIh56Z4}+_F91g&YU(yI;My?s?%kp0|PbI9wEMzUG*F*aa^ME=Yf%C57f3
z(BL9>D1V$3o@1xPfn-PtHmyCMo0zB-oy-|1rYo~uR7L!VGHWcnj%&Wl#uEC`v~N64
zauN~FWQ=k1!cYAuBDmo3{?*O3<mqk*=)^2|1EjJC|936=g|Sn)>h5;(QO-TCYC~A&
z@gn4M;z-olCT7DvYxQ@z{-kt=eF9q*4)I6Xf9}43pYjYUAHfU-OjNRBx-1xbtmWmn
z&v2v9=GPVlMHVbL5V|WnGc3Yr13HS=em3K)0hiU+#p^`xsQCb%!Q{xwg;l|4@VcJK
zAX3HWyG=ZnGY~JLbHRfo-*BG}flW_F0MdEtw#e;0x(62`d0bRIck@G3Fg(Z;9r>!w
zS&)gjT?>bw<GzuwzVJ@-APb#*8G?;$i{E|u=2-HcnLmEaou={7vbWSXr%e+;zZ}4h
zi+Tty#$x)#Eo_eAz!6{yz<Ld@Tl`d%yMT@63<mRn?r~^>zk}cC!tER(8IuG7@f55Q
zEyDKb&~zza!Bc)gjbRZzZU+MqPl4h3FjH#<Yy3H%e|PZ8ckaB`C!XTYrut)gr6!=`
zg3or^i5#lvaPqn+fImd5lw63W90LcXh+_J5Hf9dKQ3f;Ndm03n5qYXJ#3w2aJWlGu
zgF5^g8j#`!Nl1l)beMr~w_v@<GlR`FwT~)i3#?Oo{GFDZ?)mzzLRn0_Xg6Voo+_ai
z0m=<tkTx8U<}sU^WZ!H4fU1>PnxoQ8kvF`W`@i=s-aE|XV=ioL6QI%02iVder4pql
z->U@DWsL0#B#NUXn>M4dozh^&^7$TC$nLXAc8gWq>MI(HV1p4F$Rzj$+PLhGi5FAa
z*CsTEJFFNIQjIqHMNs_4Rs{~y4)pqd_%{&$8owlD@~uv$8`=XwuLL%cw2eh3YDE-a
zUDT85wqXhH;O@rUObY2kIMK0#B;?L`yTb-xofeF=8lh+5u=qXcI((F!MOX1=%FpWG
z*?xatu&x9Etl8nBfkZ*LNX64K3xEV{Qwm_=ih~4ObfY7lzVEu_LaeW=pKJX)M&gp@
z%*aRpJ=@mu$U|`~7oNFxgYmGlYcYutvq=&Huf13i($KW_-^DhCf@mdk1jGxNIQ0@U
zXaM14Q}Ja?wW9?bnC%Y<E6?K@xc29+MJ7auONcKqvZJE|7nC{jmfwW|KmG#k<Ft@U
zYgD)Ht7}Y8HiS*M%l-*APgQo}M=FOh$}<R?c_c_e@9jirm0Bly%7H4bC$?GM=)CJ(
z3Nmc2=;iD9%u)%(chl)2FK6R{L5yYMXwEqL`s&nWWZi|SzlV%25!vGb`D14?p{T`a
zrjhkEVYcf~vc2D;*eLhojAqV38hWCU_4Xp0KNN1n7!n+Fo(DRBdzwGSrhs_y2hT*8
zLL==ngm`PTLF~$m{9x>EaKwsA3{sy}qVs9IK=r28K1mAHlJO6+IY`5|%ykrnBi-@$
zvUqt_#v2cr@gK-fU&mXFL%p2ff5$?-<seu0V(>@C<}3_<$C@h*a-;@(N@e69fv9r^
z!%*@iQ6d63se*Vsl5RZF%C64Hokd6*56K45yiy7`Qw=p^2v&st^r&E+s-f*@=lpU_
z3>OR?ND}O6|HCGU16}U)BSwBO8MzbY&jeKtfTSOAAg&$3_mL5obvza}pXzU1;pFXM
z(*R@w8iIoq>2xWCaibg&GNDGCWF}v*xunbisGM`F#PNA~Dji`>=5RDdqJa^@@+W@7
z^-RwAfq;G52JTwa1wT#z9JVDql!TfgrBl#7Gb|bZG1t>>pmyC!%3?Sp>cPL{8SzFx
z_&$RO|C>J_Q5DCyxqa}z5xC_$5I0Ccew`~)g(RTC3P=qosGH{!KN`eGx;L*Y=A(`4
zDQg_3IG6|2b9#QxIic+JwjHEhiJ=zFyJ%;Hb7?Fv#?e$8!KwTwS<xyz(Y`W2;6J78
zatb_@KSByMVtS*|TM6F~rkapX4u@wmq^TqDejcp%vp}d!_6pqAMeFD<81_|A0#mAx
z0G`NsO|OutqB!)BOzdh5%eH6mW&E4S+`feQ6;SB;rL@pz6aYrB{dZag-Jko%cV{rN
z@-bZ7M+3c8d^9u<Gp1h~gSomqrh28Au~8DJ(K=`{@;Q%LR{zVYRA?>$B?EK8_4Gs^
zKV;6eR*W)ZmUv?HHtUd=%J#109JyPr{-8I5b;j|Qambu#Tta%{VUkgR0|`8*6BEEj
zviAecmKpJPW-hL?%@59!WlI+7LJ`_^5AaMw<98v#$jDG4;&*?=JGgF7VnLxsx$_(l
zO+ExQu-h9zhtsk5Yv{sRSj47zH^dE6k^PrbQQL*{JytsJi%Shy=XsN7*cgpn)5^Uo
zn1Y;*)L#=m(A0ft+-9BIBPh*%=hd2pDe*giy(aJrXeBG1boNEx;!!Q<ZZq}<<2BPx
z1^L-EQ;%u9FY+tMyTAD9d2za<n7wPLUqEG?G5~TmvVQG-+U>@{GlVH4Bw;~6Z&LWD
z1rwj%5wPRuVpj->2$*C&c~f-^Y9gZKSu+T0^L08XOdn;V*xtU>gUef|UBvswr?N??
zc}gLI08Tedz95yn6HY{=amL`lg3pf>&D4bOf%Yp6pf(#IWA2mkS3y9WK1_>`L~~S`
z>cO*fDe4^jT-`Ga=?j<Cm#!%n1Q?5QOl$@ca;Zog+i9x}&gt;PDcbD0JMSgj_u7d_
zgqE@xRLn4<3!7UH^x#TrYjLUE(|dHWARe?96(?(26(W|EAE%ovc%eP!t_-J$J<5v5
zzJ(yNeQ|?yq>kgSH9Tac>>Z;N#?4#Wy13QXlP|xU7)K!2@R9H<4O~=}&;z~(>f`Rh
zVsM+tR2Lm0_&9?cY3^{h>s+yGs9d0Ui_GCfxCDHj&6+-vvWI`W8q*?2F5gqAI39@V
z!b9eKN#0|xDiUcf`CWSx6qp~m94o+byZy7v>=Pk@1rQIq`<;6&bZB8C&_3W5@Fj--
z`DUL+6cu)GW4MCs+FVnfB8<=ed4E)?HXX9kbRhS0D4zwz=cB*hw*T30CR_<tuHKtP
z1z)K}Jh*e9*Z2L<_ww8KO5D5jv0vlokf`*ZnAU~T`!ZtQ;JKt4Ad29ruDr*8`KZW?
zGgCA6<(h4t+891?dG0$|6R#i}Qn`6A7&iWMWdXenPsA-ftdPMG;25KX34(9YNXuxx
zn0r^F7$AiGH!(L_PWDkWfCBS|yrl}CovnP70@L7`EgpwHoYVdHF6P3#MCuepyeC{e
zJMaCzQvKm8TvS4=AK51iRkKmWW2t~^QcFS0g&a$ww7&}ntMqfx3~W7B5l7OqUWO&&
z41^AEjzT^~N%FiTyz6jTzMq%|`aA9)nnOQJx>4JWgL3HhT8-|q+kUmZi>f==8ud&6
zE0I|-BgO9k<)#V`&b+mq`xg+vc7CvFgNrH|s9Ov#SXntvgXSAxUr84Hnv^MVF#n<x
z>sIQo@r85ox9Ku^b>Zo^c+QQJg)nXz4($o~7^n?rk8||ji3+Vp<hIE>Zq1sihR;F8
z?+!oo{{Z=I$0kMRdDNT43hNPd^qdCn2(9w@#y7A^&^`s3{!=1SrQF}o$1S>=>}F=T
z*pqLPjHlmUmsOUx?n!CF<rVX55z{<e;zm^dGzV8Z31uu1NMc*cR4K_6$06z9ZKGDs
z-^j0d2aR7fjPwe2A(FLd%9A<~2w~b-kaYWzz2YCG!`n7sh9EFyx=}*TJm@b(R1p-9
zH*uu=l}1LT2CvkVrBKf%c$oYxMcc^BXrUYu6Wph8k&gcHCUHxA?h7<5GbmoMY*UH8
zaD~dcZ1t1L`=-9qrL&RDalfv#>Gv|fIfzhE)r`B%x~(%G!j(E^ty4E$<5*<a3h8XL
zPDx}&_QG|ii1gs1js@nde(}O={)DKUM2nHESVcR<S^Dy{H-|WRtt4;b`Cmr}Gzid3
z{0$S=W<YAsGP(Zx++FYY?cFf<BW<1)5NVBYF|=%-q>HwCKP`r`RW9SPpYo{SnDT&E
zVPt<1{h;{xG0DZF{nmZ6i3MI_hDo32t}O^|n9<9VLd<HvT79pLdLc&aEXMiqH14-o
z)xrkQv>&7NE_zAnF`>ibR`p#>Z}aB6>=;3Y-LYaEM|m`L6Bv*P68Hx#j9-eHpclk$
zEx<1Ea)7uGf**3f*w$9eTi<ZOskd=s1~%><6Ysh{^E&RBV;230j)jfdbl6|{@pcE>
zZi*6-k)4#Zx5>~#7hC7>a&B7Sla(Zk9p_q+)Ysh6_R#?+$O|od8T$%p1O=g8T5VR^
ztn9oOb6ObdU3=M2;l|GkAA9XJhuP6z>2BA(&*n_-M1(?Qf9your3)%^#^i#ex4u(5
zt+9$oVZqG(QxsFZ4Db3|z2k(L?!yP%FWJy6Y&i9xRnmw6iEQwGIgp95n(eO`8q(ms
z!)h@d;6le{C5c-ckep1tWsiQ7)NtP~=n+;6P1r0vS36p54_=;2`fY&hcu4A7{Q3L$
zby|h=QK9E>EAP5dd})$1d%x}uw)NEHkyO1b!k6LZ_%o7sa*Sl4fCP#zL3{JCDCrSa
z4MnF7jSqL~=T^^P2~%W2V0#L^r;EDVyBPSyOB<>LFMQ?!PRkXSkzM#-G3Hvf-4`#?
zdHu<rbo0*@x=x0{jbzhAfF<Mix#My`Q4|ku-lcV+x6_t^QSf|Z|599jA>#saqT$m5
z8Cc6f8r+rUBfPPvdNt^Z6H3SzF!^zNjAW)@1(s`MswkL|2%+k66n#fSIND*jb{J@e
zweZR@VAzjwiC{h@FoDQ(BLpl&DxK4b<M3>-^??QPs=>Z)`*7PHvdqYU?>PYUmk8p4
z$7Jj=`OuLg*Q<K?-@Y@4Yv(_&DV)PVmuPlkXm80D_wI5D$7r>4f@VRMmy(Gpb9c-L
zp#LH|Lc<3DHxh_BH@8hkX9(TxzM_rqca#B6{d;FO_45?kT*(zb56>n$DxwZv&3Gn6
z&&7bq{9fK9Q9ab8U+oj%%kqpG&TwpWhBsDiKcUU-nR^ictR2_u+OYD{c635O@xhv!
z&J!E5GeL-|u!J)FaHV5OW0N?L9Q$~tu}tVWC3PwEbwH`9W)>|+Xp(-GbQXvfxBXZ5
zUE#bj4g+0z6y@WoNurC8HMk#)G#*KOJ*>N!LX52TKm8@@fnEp2L#qE`tRLk+Y^dAz
zuECC8LLzBeQ=X-G=aqFV@UvWD3Cc;>#^Fiwu8ewHH7VE=4d8mM6$G8|q=SZJ63;IU
zZ6dzgxyjIa{rmTf5T(O*-(#5S`nF}H2PVTBRo6$RyGnJ*^eoqu>i~Ndhn(<=^rw_j
zDH62!7iAOwv&s>}z(5R3!sWq+zN7HDf&%>0XmEHR+y5#inlbCMwzDMVP991g7ttFZ
ztm0Yn_EQ3i79eQ-99Pm(%~czP<5mIgzRgFV-{?N#2bG%XtnMeSf{{2?A{;sTph~D;
z$L}`9(23NAg>kxPnynv$>t&@~*It=Q-lptqg2>#ae6F=el30QCa&yl@=Eq2e3V7iO
z!=_XIoIGbF&%w|%&*i6tCC_p7Ovbco+-3T2LCPDYP#6P^>TNuge~teOGU(^3Z`plr
z4G{B_+?TiR@xYatmK)X6xzP?ic7&g<U4q}t$19`kA;vYVde*aJIryi)hXHH<$5JUR
z#8w~&v{DV5dmH{OV1#~`TO~=uHz3}yCL|-J2MyKi3PnSPqM^d#C@0^)Y0d&U8-}cv
z2Cb$5f-0$k@qjj-2@#mY_8Tha{FXAz-gUQfuU1PT$FqtQ-GrXMIIH4J#_fLL+HVY~
zaV|dd9tFSRSz2BGv7&Ra9+nGn#Dfe@)Jb>vZAK<eyq9tV9H!=L!s+<eyNMno5D}ON
z8x8~o2O<Q{o<#Sz-9`tuvgqr{I6_tb$GVoKSkOb~dY1S{Lm{!ZJDA1;Sjg--4+X-x
z0eGLpa_V-08L10aD|7?|WpS?-mLfyO_A+X}r5D%o+v40y>O;SYrXr-=fgC%^PAoXQ
zi%Ia3G;oHf+sp4rnClua&S>8lft)3`j&sF<hC)0Ck>Kw#kc10N%pgY@OKr*di23tZ
zLxNw?hY;+DCaGd`UCBlU+zXhd`}bhZH~N?x6hECIyG9KaI?oL5ElepXr1k5xrl`;Z
z1w5Z}2)=17fsqf$yeH`T$~d$!%uhJ`7CLFSdT-$YgGDc0h8jb&$U3Qig?>2oW}coz
z$I4_56ffvH`R$G@;7-6g#HXpCMbd)jlX(pg&S<<s&*o$8Z19lWj4dr#D-PQlsrRtX
zQ0q><I7q})`eDG+$4`{b%Z8R&v}dv5(ITNy&k;52tnw0ibO=3{x_&Ug(`7Wn3>zLG
zNC2z0jF=lx8w6u_8pA(lS~;aIz$hQKB7bRwdNCAgpk(K9VE(gVj;`=J+Y%*sA98b<
z;Bo9jm!f1-XWY0;)xC3P+lq(v<twj09ozY9S3#30^o*w5$TYDbjP!9Ct{kddl}S3J
zX+Y%aUF!uPhR};4lgN^$&m2rw-+GQH^?&iJ{qntO7ctUC+-g9Uk<nr*AJ)m1vDo}-
zq+=IQpU5IQ4oYPuLCy|)53PM#x#f{~*FS<vR~xzS8an3=O(P^Ppwz-Z_xHN7eL?Rz
z4!$xHO0%%UF$k+apLqCtJ9t7F2Vp_H{`Fk4^2jUf;s2gtJ=N#KIQajbCQpA^3&wYz
z!WIJ3vpIJzHH&&2bgsUFlOTns=5qoBBW@X(fpHM!*K=Y#$740OUReK;3adg_yR3&(
zdtlU)f&FL8|I?jMOc&@Iz6c7+esroTyt&fLlSG(Ysu@*{t=mm;zA1WN_2ps=EJGP5
zr>M4U4t!5o51nZp`Y^O6=mA$OoMTsT#TWuZr^>vN&SO_5+IXMT68qbYlk&{QLJlja
zRhohhfrtpjuIRrX51!x2-fA@5y0ED1oK!=93&<;K;(dDmq;GrT$FiZ%NUHesM}PVd
z?}jJxq!7NpwSD%{>GOX~N+Hba-Eckv{_*6FLgv!XNnyDeedABW`DW`qW5jm4_-~#k
zPF2*3QJ;KL^#4zRvbQAb74&E}6avOY=VnLT1qlq+{$~g~9C?K%IJnXCG=EQ_$a~3k
zG69yX@_7QA*`|)z&Js`iADFgD3?RTRe!Qv;TZsFwF;Bbk0zuP$Un3SmSf-K7>^X0(
z{&O&gyI0VG8Cz<mPrYj~q11<L?Rd4?;|fFquU7sS)=dUrobj*p!|n|0o~MWAZl8|g
zE-IU4sySF*ReQ;{INC|9I^g*BU*N9^hTHc7IK)2fg`TOJbXUaCnPf_Cn{PHDc+e?A
zc-uF*cz{@J6DtsU;TsJo%7)}cVh~ZgR(!A}U)nJX#z9g-jLQz_nGl?!yyCUmJ8s7H
zASfo4jdsl^FN{P<D@%EsD=#?Pu8!dHsMMqN2%4wyqb?cpuJd&C*IoKE1Sb`*)(UDE
ztQBJhG*$tCxo|}8THk2h5xaLsG~)@w7-PHFFo4{3dmm+LYq^0#96ZHejJaXnOgIJj
ziJUy39F>_&BPLnabqGjD5uJ9sh4#v7y*JCXhsPfWDb%lz1m;JYfwENrgG!f7iNu_t
z0%d@c<spPV4Xo4Fx+dhL%#@fE0N4;f0C3CD;S)UIg;Emem$uH1mgA2Rijpv`{!I>U
z&`|<Isw?N5y%HzY_7{>e{m(Hx&|5GPTt@NW?f;$Wa2~CEmTwz1Ljct!JEP^4`%HRY
z8VxCI-vUQ8tk()1T2AqO2ahCGSG|r^H_CkZiw7zOlle30IWK{*XNdPe9g^E*!q4qy
zgR1Ze5XEm>57=#?U%JekF(<*^G2VbmcGAHW%F#YACeq?VogvUxtSA(o@f{p-5HZ9O
z$sEn>VLDZ?K!*I{_Y}lk@At(JXY|=ljS_usE&sAWreiN)a3MDSA}Y39kP5C?1A`PY
z=&A1}-3dgD7f8b>M|5;9JYQoqPvzl+)N=H&sxN#*yg#;qm;;0sckn`3Y*1wZI*Rbu
zk0yC(5(L*agrfx=Fc%=88mp}xUq_{cND*L@N1~D}XPhHT&Yd`fN2L%WIM1h2EHTRU
z#b^&H1OHA6j60hPImj7zT^gYsIREfV#D2Pj@ye5O(&AH%leLpLUTnJsh^H4w5=BqC
zn9mtqJNj6)PjhZVP1nbDWtDT)1jQB^E|Aq^m~0n3HaKo<AFH7)s0(8_?GH4aHTG18
zs)Fdnz{rWx!zEa;N%e-K9svARV6IQV&kp;Z5Wp&wG(HE*ei~u&o@t0N_1-W;7L*tu
zL`qFS4v7}*V>CK7_w+;C2%E<knwcPtqy6ai0DitE)gKnp%%QD2_q?$haDm{cvh>Mz
zvNxifc7Iz<uqE(tL&j)~kV;gk(%=k0GiKb=iYWJQr%@ziexb0IbC@DdDPM1lEIIw6
zL2D;?@F9uM`!&|;_mUgqx+S*!D14D6%fk_H%rj$NPYeK&hbBD$NkuB5XUxsW=Dx66
zT>AkK=N@T*XWG;(IVP&V8_co9i_+21)xPXfl6%}&ZR$(T*qmORc$V2{LBcU|c)Y^J
zDy(20#F`(aM>xfJH3A-X5XaHX=I(O68U<TLb}~IFY&=R2evEy%T)(^XH$^%)H<}35
z8F1uBJz|J1Z=UY{9{b?o73Y;N2|c2$Y&C>DVQrq>w?=M0?>sGnTSe`Y6IW)5duYg!
zg`~xK8%hceYCmW5{Mpf8;#3pc^lnu!-EqDkY+~~)eh1n>%8-co`Ep^2tL?eRopU7~
zeZk`fOg6?iRt<#Mb~)ytt5)bI5tJhAu#cf_x3e=#6q;c-yIBB{X8zOTj23klRKCOM
zWJd`)UD-qXEJ7y+XB!V=srRiB4DstPZN(<e-EKHX?iqFH2u)uLES+PyY)6H$$M@Ju
zR;-+Z?L-~r=Ko;C>J!(>{M1mw3!2-iRZyM(X?zu11L-6CJ{niUe#~D|C{L3qM){U1
zz&3uScckOD)IUtYvy{Jg;hg4_Xp8mElvX^e0o`GX@fzblCcbVfunK+PA%FffKRuX<
zuyqFv7LJEk3!zm-r6`+@qe-zVxx8i{|3vBQZe&*%$Fi>i-Ow%&YYBO~V~BIl&GEJW
zjq=-UF&64t29(-;1A}=vo6l-n1;LA!yNSPL$%vm*3#(jhf*uymAH}n)vOJ%uD}UFw
zWi7m<eio4Eqs4FQ+&-k5G#}~JtSUz~C_lyRt^5R>qrva9O;RX5U^x2Ah@v38YmX&a
zw+^Bl<l?7S2ilSO)r<RKj5w%p2)q`JbEqQD`19-SP}{nCpZ&|}`SO7_TYD>~#_#Lh
z$}umAI;wvdJ&XBAhocEjdJ%nz%A3T)kyc~>v(>8Vfos7;iM=diL4;*>61@H=HS+}n
zqcmYN*?^o>65=Px^#`9uyPY9UM*8s`_2nriqw4n)2H)4~*d+y6h4NRavhu{s7&66d
zWLAzc#Gf4VWs&b}E$M$pE&4;Zr?P7*(1Ps1CYhu3|Aa-0P7$fXPALclE*<jgU0XuZ
zOfmZ}96~$q<5LmZ4O2ar4T2`#b+*Nqm;ZG$%fL3ie`U>Vp1hmCmzm!Pd_R)Q+vh_?
zhOrY`2b4Ow?1(iY8rQ52Xo8+_@NL?MCcq|gSOfo09oHQW$M^0ni5jbSB6<YttG8&2
zAcW{GNTRLIYSD#NqZ6WAl1TI}R>W8IvaupT^ltUGdb=a{cb|LzxzDpR&zX7OdE5EC
z?>VzGbDYF2zp>gG#b!TB7r@0Izt1zcA*-c?TCI#zqSicj+v7z(`Un{Rd>&5jsucQr
zV!v^2#v;VO5rm!3##Bt?b|tgEgNqG@0eTjpdP@aqGC!e7tXBS8Qsv_?Y4c;pIIxY1
z+t?&3@@yf9XAf_@n`WcKnIdqC#29MAumwAmkV*iB0@Z*(Z!;a;$LffbgIDz<beST4
z+X{NHc5ioi625MslubV;mUpX-DlZICd1C}`e5nQo%gDzgc&XD{!$rl^&0%~pV)l4x
zJp9T;&ODQ?8)V2VS26qjXj8bz5QEQCJAY`7u<#iu&`}e{Ie(#_B?E~gmLEdN*S~XV
z=S5WhXrhEfZgbk|QgRec9Bj*sMoK>L(pOI>l(h#C&p5eNZM$-!uK{z1)Ym|8Ua5%;
ziIV>6`42m~VRr&{K2w=zA<Z_g#Q@kuoy~<$ws1nFwoOuOIO<b2(dKGkNf;xi3gR8%
z&npQem(7nLbuFK>$55K<_$^;knJg|LTn_EX9K?bI{Y<e@3~mB#5(#TJwS;7<`dUTh
zbK6{{a}s&}0~i4p1w8FIw*KyyU@{g7ZXVg0u@FE;L}}9!0!S|VvDaVAPAc!sYqS$a
zv#SAnnF0wZpc`UkArlZ{3lrFUMuPtSdAUu#3Sn>4JLbbaCohtaDdhcYK5VwcQEq`9
z0kb1ATfV^go?>CO&Bzvs(8@_GRuc1yX(M%eX}Kj;Z|ZE*-E-r^?d41;)P2N*$)+9m
zsw5-@qY7zDU%&by_Y`}VaJn+^tJfnS&IR$VGyQ{lO4zy66pMg7AW5#Hcg7qd#fV(n
zIduwJs(=<ypA|A!|I_Htt6d*dhSHQI6|or`uK@L$dUs51_~sPPT*pvRRs@3g@xQ05
z|J?{HWy=_3wbq0c#oOjigr2?uh)hN9zp`wcm%e?Eg{xi1c9I&&3Hq$ui=i%rR2bNS
z!|#Kb)UM(Nl_~A!Mk>xO+wOZ&+AWP$UC!{ur15TSR#m0T#E2HTM5|>*+!reKHH{m5
zQLz3{{E~-5|5*rsx(w_?@_iyH_8)P!KtA3lnyn;5hbCiGHG2cjv>a|ukz`rk*Z&A)
zt>(!oQsc&jRF4#iy|?|W+~Rulm5^G|9twbsz2DbV)azL)M)g#>ZLpu!1fVMX+Nzcm
z1tOAu+efZE;l}F^^CM42Z}rjv=aPi8x-QD}zrMw6WzSXooLP?%EpN`A$>|LdXqc##
zf7&)W@N3xg>!&#Oi8#g48JAXSbQRDq%7D=K>zR{KlZ+?NN6?0YYZsR~_vE@DN&90+
zyIw}4lLb3Ayi%ik)5Dyv;PIytfN{XYBJN^LRQ)kl-O^7cnfVm~+paTR5`nuTvImvP
zvB*-*i~I7@g=CuO&S>c|05gh^+3i7Aeq`h`qkpX!Hg0>U6*pyYRcaxNy%(Dr+|el)
z`dTEez3eDT6w_d67BKMwv<y}W1H)$qMw{SKHm`Xb;c<4)v%yK`rq@I-=DZR*s5<IP
z*5YfP5?$nL@GH-Pm|J7k=nMTPq0Z(llXz8G<*NjVI#8PPlwq*5ycb~1lE*lnap$<e
zt}XuY!Dc(M=D~9$XwEJlsY2#~C;xmCSdApf5zqEui5v;C2x;ta!s{&2guP_{W*s{9
zRz@E>_TX(kSM9zWhF<So4q>%MKl8miI2B3ff0L+@mg4pcN2o7LS$gEe9iXRqpAaiM
zgS#*4jLBb705Wv9zis!k#+iWwJ2|akoB!HG5i|261N+3L%V;MwzSoc?`GoP>pjR=2
z{g{8#MI<R1_*H14J^a53c)dsXilg(7)b4JF<rqEh%?^`mLxMs@21IDBaopkKsLD5b
zeyCb$1MI{CCs19!&$9pUwN2cLhH4W3>hZuA(H_bP(#6Z;N4UZ8^VR^kae1W#_wPUS
zvL!}u&&x0l)G)K1YD?n*PiPieD!-Tz%M+kpq99p8iEM1-0W?6hrw_HXZMhQB#X|66
zh~zr(zn1Qm5}0#4$dnFxWKcwA$4rUMVCy93)Rfhi(wU!Uw;=N0af{aoGu_n0FK8N6
z-|L?DsU7%ucsg3lSfxeyvkYhx$olvN`W^A`RO*3`jbNHz6vhWH7r}ePQ+$)N1qTRw
zg2J4yPX4T6znOTLIA|A+G8<TZi2e<}5AR_iI*G_Ce=yMu&AGPQx$|s9W8$BXS<4;g
z$(F@0JNL_XbxewDE6a+Y@eoxGk|L<~z32Ck5|QsY((w`a8rP@pGg(G{=tRvx-yF1j
z8#MLHq|!#!h@ZnuRu31oIwf-uk29F%^7QbaV6;BTSW_^gUe2U>>nVlRhY#x{!lv-P
z)fFlFp&#BK^b}jx5s``WrID!wLGn@3<!WduVQ(H`HP*^46NMnIU-kR)v@r^+aeHGA
zn&pYfztzTsG<;i$VaR4MQ+2NRtimb#DjD<0aae)$8?ymx#*%KO%t9oDc%H3N>agNd
z@Pip)99UPs>Dv`Eg0SY@LN()$7wz<d$q({#&eML;5+exL7P{D-ox#N9Kf6%lWelMY
zqcQg^FZRAaOT0MRv@^v{jsCtY@@XWcWPaGqc{Y?e6!M`O(_+|cT|X5BqJl=Shx3Cs
zN(w?kHK~yD$DpDuCWk~0agA{65UDYYa0hfu&F33`PUfEMn~JFcp4}olBcDLrFdU3B
z1r*6xW#wD|{|eBf)QF@Yu^Xp^+fD0Fi=h#{dRq5SOAB2-bjrVy;^9onR0VI<tp_m(
zAqX{}$9JS&=pqo`K6dXN>S!942N-XLOz}6Xe+~3FIrYf&V_dQHb6?#OVV4psY@bK2
zYRn%ytGHjYAqYuoA$LxXlQ9;C6<#drTIPd^R>S#5RHbTtb^aCSkHOQD)26o@M2<8z
z;S@NC6D(P<TUwBp+gk;jXV|6l#BVjTDSX<fOTW*F%DuOi(A)m)Gb4#WvjO-N7(v*#
ze;mrr?XAcDujjK`a35L}O4=&cJ0Yp}dm$xf^1WJcPX9Vw$k}q3We^^QI6eInff%J3
zWYcRnI_YFu!A$U(akA4deGt;`fOn*QGdWxyGJXxGWadr~0Sj>A@|nXeeoo7ueY!Jb
zMknu7(5pr8(}5}1RIc=Uqj_^;gFxK>uzDMLhx?=@l_rl@?uoEYby$DaQ`ba0Y|xi~
zmDT$EGIKq{FFukkS^D~PnFHGL_{Zz}=7%<(!9E7QO%+8ptNU_|i~AVM{)Q-ti52t%
z_e#>nky_aW8U$giO=1TrN$`8VxnC$v9<6p%zfQuZda<SU^GC_!v%t4fHg9pL$8RV&
zA~E+~40@V&6J9^wzHhqaCIo#NSE1G(lwM{!%S22b|Mk^j1GJc(`Qb}|i<0?0r57tb
zg3wX0DB<rR)!#$@PS)t#NM%xyo)^lnWGKL?g`9O|3$6>6>+W@v$ZTyPK(;>;>3*U2
zsZH@34Q(@<^l3U5dh(HyxxAe*`fV5{`WCW13SgL~lWm*oY#qn968DMq-9J1>D8QpB
znOzudSV!Y9Z-ta!_TH{(^(*vhUZ%9*49KbT;dth5$WTzJHP!ixl%%zluKJ-Y${ZDr
zc|HQ9P!W{9`DN&8VfT4+pAW^%XZdP&wKYqQji(;B#`(7IHGtQ;Wee9=^MdC`Goh<1
zp7noDlUk`N9wG>Hx@n{7=mIjizEz0LtXFUvuU9&ok!QB;uRxm&M-&UB3>5Oo+#oWI
zc~U?;SbA=_Cxku=NB=Ztt4ZZ?wvc+vumZK<@i)-BV+%qb1RYxP*XdzVe$S#Y&kz2Z
zMx?BKUhn#uSdw10q~*kd3`z0HuGL@?>~;RzM)?I&ATsP8W+5T@kOZ)%hcdc>(_4a4
zmYx30+@!|~DX`Glk8;{^aebv96OSpAsGp;ucvxF^6RXCUkV8D=z}Y(#2OZ#*i5A$|
z)YxX5u667FxeC;8AJm>m?-NG!-%<e|wb<;E8S)!wC-$9EqM?Z@mHV{T%JOWZS{qI<
z`MQKu0vA>lR%1i#SA&qI%Vuu)3#7$&`7a9e^^_<pV>&gfbol8r<6by3G~&HT?owNA
zHGG9iFJkK%sjH@AUPbE%%qYp{Dl!Sqnw&Hg6qYQ#=V|Kg*LLir6@6tj<XkQoRFD^3
zRmaRB{w4ZE8hBSzCG%A9=h@_r-#P<=a8@aZb(8=xQ!;;Ob#$I5Mv822+BMnK`9nl(
zIqRIfOA16kDQ;X^(4`&Ukw+3t292nvX1PJ5d2B*QIObkMXV(oV%@-D`?}=*&dH#iM
z#usTppp0s1-++0Ihxg_(95qe!VfJ1AwuLSH;~EH<EbQ0cJ+1~IhyC9@E&cF{v>={-
zj0`}ah4X>S_PAEF3+Bl|*YR?mrs=1zpG#0)sBCxMD0VVlyiqJs_E&NC?;FMXXs5r5
zfd#Sv#ol$)H};~;VsGr7F8_NOyy!1~YfQ*(WIGKf^9@ITZ1;`C|B&)DSV#4-BfMV2
z+tYj7^=ncWZ<s>;XIw(E+8#PuF&eP}th6b-*B>xj{*$e<38*!qD(l{{K-J6B+XXHW
zpQe2YfF6tKe4&Oo3Yw)+-HORo(5v`!S6Y=%4@uPMPTAmS$Oer_po#5>BS!y@GL#hk
zIz#QBxqmRolT8R!CyU$OpByHJMs$+&yqKnvwV|<`I3mdjaFG(bn^KxD7F@(jChsU~
zd(6Ov`^ERbOCohm$Az<L8hrvL3B*GxLlHR_JH+zzwhlpX@Ae=$&i(M$rcb%Jtl>+-
z{v4l>CE&KA_3e+4hMWuC@Lpg9;`r`qN`(|YOC!s4URD_9XIYQ$Rz0X0<$Z~C9vQ6H
zUaq8{;PUIX(<~4se>4pm_CCv>nz;jwKoIx5=o*BJL&Pkjv$NdHLKqK{S*XsSv*&6g
z@}o(~`Sn)_tUh=*O_=K!=AIe!MvLWS<K>Zt{HV<I-LZR%c6(2YCxS8-(-mtl7sPAl
zAqmV()>hquL?={;h3ahu+6Uh#^Q@Am?Go7JeMIaUY!Hyi_#q!D1fe3LEwn`!s~wTu
zSPIET^*%iqrlEhgs>u5=vIjbJq9&ij=g#UNn#m5a2gU@g```g?93m!$ee(xfQM>ol
zLF`f(x{kGOaQvhB?o*h7O?p-=+~UcGPYd6LK{odzE@E<O8>VPG>Hxu&1D_I|-D+%p
zS=Pf$7xjdJ$FR?yf2vaIbE7><K0bDHY)AuYy-0?qDKAoN$48{y8Nq&wN(Qafn%FDA
zuM;>tFx(XTNn{yS%pcRK1V)Pm+BE}nhMj&<@W8l&>^;wB)8YI>dt~B=J;UELch~ez
zzDniKO8qtwon9+H{}dS#rD9S*J0M;D3Mt$ZTy*Sgx7pAYVNAeYf+-8#RGs|3$3D_+
z*8QTZEsa5y`=j$LT|vwQB{NhJOz&yBw_|Ia?3dlF4l9uf$55VmR>yRlWxBP8V|3%y
zfK-XX=oZI>fRP+HnwNKhRN%m+J^yClYd3bh>|f_!#u9&Wpt^Ibt@X~d_c`WxD<f1G
zxFq(Uqz?17=(}EZ-OXnfs|wiDD0c_98uh8Jy?=gzGkPB2C~lv*=C@gO6HmJ(1sqHp
zcp-yl-6>OMHsrZTUC+Noj~qq!2#7zzmOP@2rlT^Ka!$rZBaEF$;lIBGLBAt6*F8lo
z<?WMpbR1M*#Cf#sI)9{*eQauxE9<Yv%uo^mvv+)7PaeJ3)Ny}K!J7zk2r%QU<;&7j
z**QMiItF~_(*Wdhc^gaYsUsAA4@~J}t{A2lgsU=RJ3r{!akv(c7G*O}`WU3ST|{Fj
zWktfJ%NddG^4p6s#M2`dcE8sM8$$%Si|ltYc6yH{rW+R%#l+MLraa3+#u=e1Sngp@
zQxWKKN%&W{c+Bd$jfEmT;Ru_8d-u;nZ@>P@&K5~U!;D{PJV31akc-s|h!r-zG$W+0
zA;pYloAA!S1LS4lnvRs3^+}?v-J=;Eh=X|mF*!0#1;{5-`P$}tU=sOx*`f^yufBsI
z?j8(9HgVv@&U@{?6BNR#Rw<qTr2db#acr$7ubyU=>TEwFNnb8y^g<LRku3=cm=%xY
zlwvOGVo11()WybUXnPap*mQjU*1ntEPHXY(>E-(G^Z6eeCzKP0o_OSiO=+&thYv8?
z3qqI)uB?L}m~=sps+?}ECzjDx;@7T1TfKEsAn~x7rytoyH<aEa!U$y(7db=~y;WE4
z=$*SLRK1oY?EUJ)s_|nj7LQrU-BI=W(zVc8&D#t2#ELhdne{X6Lwj;@cK(n3ymoeR
zz3uI!VKe4{Fczf)U53K9CQdf$LymLg?qp+1W9pn{gCD)Bxg9_-yjG;PTzOz&XkGUK
z4@cQMSDG7R(@FNN$ChIbrG|_%Onzp3`n*W5M1Upl+N$2}P;c{Z)F!VvneR{s;b`I_
zE~>=LYPGf}UsrOoxr8|muoD-8aP|wnrW)<|7+o&6=-a^Y0oMAO5QBFXi_6@(Qe_5&
zISgN4T`z8FcekGiOu?1$0VbOXOo2k*WR~Ek>*YVv!mE*hql&p)0bBQvLY_FJr!oVk
zu^WV5d6U&=l;lXx{OS5Kn(33%5>TRF<^JXlAe3l0sYyx75S&=SU#t>sYl|<M%9S!J
zT7RmkKelsX8aPhXy8GBt#s`JdxndMM?-(Pfu&(RGO*0u*f^aH=!%f|Y)wdnWM^N?J
zn6$jKy)MXS(jPi~)4^G)cZrUPXglBb&}Epn1v(BS-NLQ5T^$T4jeXoHsuM@hUuyom
zx@zmCXmYA*>ok{;LV_?mW!F=Kwn}_x4mLC{I&&@Maf%i&t?Q!V)GwZ+9arY&m8~}s
zQct2kv$@>nDcZR{?_db4?0v)AKvqU%WV~ishad8#<W0MeoTVW$UOX(sZ+dXa6KwyZ
z-kt08@_A$O;rxAOQG?u{_rZ<vsSG5@Dfc~_HUmc)wSt}L9Vu&-FNv53Ph#_QDl%r3
z5_u~e?wSo{$&!SH+$$8zIb7dT)dpATrQG>41@rcZ(H}P7c~t8CB+zRzsZ(QKeUfc>
zwJrEKso+~R)bIRv9>XkTvZ>n1?pvwsTBgHXP%CL6`z`Faq)h&O2tRSc%i*6|a)!&^
z0u5XgDW9;m?tCu`Pl`d8Bf_qv(-8LlGG~3J9D%0gBFN;b>e8MMhvJ{xvtA$^0mzkY
zFvv~cTsy~f$(BAZ<I!+d3RgpsD19TNjS%T)#gEtPu7!bc9x9(YG0_6@WuvqfMCk~c
zk?N0{JSX0zl2So>eobNmhu(Qgb(!yW8vX4AC^m-rg;4j*AIEwBg+IWH{*)UJ3vpm9
z%+936ijE=E*=yVnuaKPt*7I-QD(~5+r~r>?b6cmerMJbbR;X6{O<D&s-`euDJSRj(
z*9d5u#Q++35Vd1es50^A^1#J|O%TI>6U0Dgo@}o)Sx!!`O8rySmeR$p7DMjIuqKV*
z$EsExjZ3@UO4jcxRw`;i<_0KHnTd_c2v_piD=rX@t9i>farD*9zYb^~eBn2BhoVQh
zHLORy?jZA%6raI{rWdiJ%{0Oa`C)@nUefqj_1kAjEEH^v47t)m3id4xjM&B+H~XZV
z)UHg2og-R27~bWpobNhyvodyy2fbD19JP5J3~2&~H0^)dhg36m1!e~zN%&!e7z#T$
ztv9i(EbgVkf-XPlv%X2iRn)w6uN%sA@8htD77ns&Y)k?$Ce`wn>*$%(>IH$dZL*MW
z@VziUOUs7!9;Ni{tPw=)$t0juZ`6LWN;Ih(qFq1&wE&bCb(67Fm{vRtI~7(|I`ooz
zFXyZfN=hFVB3)>6MjlCe^5k@kFbyAuO?mwzmFgyC@lkE~d9A4&0chNhQN3dGs)Fzk
zJNVH}jo5OKbPnDob$;oFZ+8Io@xP~UiSF`McL$qhThaKB^<H&zt)DPT`#t8!6~#F!
zVxcJpo@dznUM)PJk_G^4SsHifBy#fl2oA6bH{brCNuA86s^P{=aUT3*{>n1->oeme
zJ>X94b5|7a7{8XuBP*r{iaCT$zvo(0#_<c0v+nl4V@)P)IA1QnW^;8o4S0W*7h56H
zv!{xo@hfi?AZs;jnPUH>T+_qsvV(TC@%;sTFGx<2w=j4Ci&@j3U(_Jcj+J7_cUiga
zi4VzK6E}w?-es339OV8J5c?2#sN?l9qm{l-wC5wGQd`HMaa=E%I$?lm82zf`AuUtk
zVQxBAGJj`nfotN_f}&Lzc(BWZ5VaQz-nyx<D`Jxrd}^X*tn_!hABT+b`bN?-6k6oq
z<Y(~n+CcTaQNQ@Y`#^<K$8cLH_A#4VSR(&#-m}ctl3?(VC8@XVAL`8zbGGs+P+koa
z%{eI$ZC8iU75JOQZ9;52BYT!}7<l!`3l}0^aY)YhDW&TNm{xRGAP8`6BP+pHCQ+zi
zfz(^YDy`>kNkBCB01D1D6BFn8b?NlYqr0UwM(*Vp{)R=yZz;LA{s!_qF~$(-?uyl?
z`kWy!fG|9h{B2?TnlrU)Sy03`W@<?&jq9zdXzxwO1=*i_c~Mx6ijPjS>gP)w9<Dph
zjRzsaeqKHY1^}ttKTWjySK4A^g}``8h|#^}O?`L+I||IrfCID$U;rBj-etpC{)}Eh
z*tD-wTh_ki?Kcc4itRY8?=ez+%TllWt`;3m2cU4USk>z6A1eEn7uQ-hY59)IXGGvB
zsj%P1=1=976bXZ#;4ud<6uQ?oiCvOnju!Ci8~G-JHQnA;Y`(dT4I&3VIDo<A3Wv0g
z8Zk&M+Pyf|50dDdMfK>I`^*f50!h6h!ylrLD4!Yh&^5<`WcOlhf5mijEQ=RW1Qi9B
zfyC>{F$wUMO;<L(>~wIZqNKm+%9I@UAEV`&At4NjzlzU7Aqq8+?QIq)N1FOnGujJW
z%;ET<>>&B;KV#~2yeg_3VNs2siCgQ=5Z&j23%JTM6D2?1?~4B@Ib5YM=`72BUv!bB
z;8`M`_do~gc?6It;=Og<XqIdQiQ~>e!CNfwqL7+w_t%m+&><-$3x%rSLi*6WCpItA
zIt-uvyxlf%B=L*)^tO@;3n&;plW%b$I@zR@(<NOP{~F@x4sUau{<LeW2zkj8aOcVQ
z6a0FeI5%~m%b&ci<fl?!Wq1wF^qt$lM@`dVlMM~IGwD!@7!Gn9A|(|I;LA5=I-a6B
z*W<^f_C&c)&hNTwCw7i*WBeS#x^h{zW3r1}_N>G(k-;5zZ`89{);4L`Fm2@Z^M*7~
zPKYQSkSaWG;UD}^LSD6Du-gyjR3HDCK4B4-IH*FMMo@@!4Rc%KxNFhp)>3?@NBU*C
z#rnd736?@5Gn2&9ib+VW+piA~XY$%qMRN%?f%PV054=h~QA1^T_(H}`t26PKCx~Hs
zGO9YgL;@ePnw&3OE*=PxV{(sLoiifnu3}N=)m&l<&sk9Wv9=y7JaXI*>S(BBG?is;
z`okRSeB|qUcQrFO=(Vxbq7&}4;XZ2MRWSPK$CprF_OY1du*O?w6tooNz!dC18!<kH
z@)iKx8}R<!l<}Bv0R6ukKt?K-DzrIR{@K~PRH3}3{Qvw~+d>G>KaUG9w2(>NtQ4=Q
LrlVR3eIEW_k~0i>

literal 0
HcmV?d00001

diff --git a/site/assets/fast_python_serialization_with_ray_and_arrow/speedups0.png b/site/assets/fast_python_serialization_with_ray_and_arrow/speedups0.png
new file mode 100644
index 0000000000000000000000000000000000000000..31ab26be1b48a1d059111f90f379ef016da98922
GIT binary patch
literal 4050
zcmb_fcTkgC(?3rFA@qbIp-2-U)C3R%NC^;;PH3V?i6Dpwh*Cu_JbD3xN>dOK>EfkG
zQ7IyL34&5YX;P&ZQIH~nh9+P9{(5KLf8Uwiv*(=I*_}N*XZN@9RGVYGTw+`R0A6!5
ziah`j%wk?Nl69j_GQPzU5mYP3qkDUM92^`p8cj+{>ihTaOeS++U|`S3$7g7WrNYn8
z9~~W?o0~f|HAU;}{7$188X7h>HU<O)P^r|EloT8eS5Z+xBoav^()|4V?(XjB=xA40
z*W0&mi;9Zycs!j>udS^O3kx$fHC<j_K6&zFMn=ZQ#)d^G?8~w*Y)&z9j2KxOTl6f^
z+lQJYvE4rj`Do|77VX|{Y~F;FT(+&lYj0i&f7R#`fW0iIOc8|mZZuB?X-oX4_??ew
zC;avv<?6U6g_Ql9f9yg_odiQ1ZofFXz1N4qBK0J5%S)H^B~zmJqJ3oN*K2uJwc*UG
ze3OD-`?_*K>VTx}=NF3whL1Uw>W`y!Bxx9I08f=ntQb=I#4@o47=2Vq>Fj@IG?*ep
z(@52LT<G?E+KRr@o6k-$&EXbB66pxW!Al^g=z-I8YFgvyR{@<s<q@A@B7DE-RP7~P
zAtpduwIaTQ8WOecb^oC890!@ubz0W%nu$C`BW#6dQsoAzk-eyYZF@?MTstc7&ycx8
z5jvrRwKJ82&buy49R_#k^eukYig#ShQnRz9P<RjZypZs_FxK(1e?WI{X0zwkTzP1w
z<fK4oP`dcta8#RF-8FI3ZeF`EHEwT9p$)#2)lS_5etv(2qFT;%hg06lYX2%Xz!aMt
zh?nce&Yc?(=obMudo%D`Ao<XKZ1p)aAxI4yyK9wtVl|%2YWkA>%Rs4hlpT^udHr?f
zM?t1dABVMNhYO#+tZV|kMaCDkmjzw2SeR)Qqhb&4!(iVy6J85XemUd0B&I{z+*-SR
zEy}6zaryrI)isa!*z5Rh5Cnwm;CMXJHD3aM&<>8no0^1ZJ1d%S$>w8L2m2P+znHbg
z(CJDN!rqlaa||n4;zR>Vf=+*QdIB7Jw>goOy~Ind%KmOCJ^(S+B1!Rb(gXG98g`u7
zQYmX*yg)9cZaJ}DVu;=rTZnJA@^m42TIkls{f-RFlGNAN_XIH%r|3u>OnY7A;vzim
z%WkK!%LN~M#Ag(Q2y%uOkmRVb@DUJ_`O`&UiVqm)TV!FU1gfp~=YaLCkkRS_pY;th
zAt2m?5fBD`2Lvg`jll#oSyW&{Dl>PjPrgY0CcMQ(lhgHoAiYwO^MV~6?@HSdVi;-}
z717s17f-nJs&{9B234}<nSB3v1Q0bhKQN#mBKT&7y_>I@$kVrW`@l^1F9SQSyJm&e
z;fk++*vG5@Axg<f?&Yr-3A(^9`<PRtpsaycNQ_{6#A90s|1F^qA#*_Xbl``MCrNMI
z;EQErLMr-$bDxR2>wm3>0n3It(F(Wv7T0Vw9jTXjzx)_xlvEFBbTO`1A2!Egt8)-%
z()Vu1`x>mg%y^O%$&o7~D=>^8a=IwBj#Nk0%75XK&MFjWe*%56a|5{vl%l7tZg9Jq
zjfmSnCU85|Z-Al-YkWvcE;v69k&`(^>vIs=*2)78X9Xx<y(a=Z4%!5XfE&kWo9x};
z!bWa3(^7o=cnR8&a1g97I_w3GzPALhZO7L5@dr7ijx#N3=^YbS7f!*wJT03R)6S-J
zy_}EV=OLzL-{JHm1p43MtClZ$q;?-ktWoe2yJa)cNa<&i6Oh?M0alIpq3IAQKVSMi
zo2sfeeN}&a2C|xVn0%)IGP(e<ZvXg?6Y2m-1UDeb{^FYKlsnMB5lrrF9YqTi(}L@Y
zuE7+WwoTY~4LJHQ%?lS1Lw$E^^o9>2clJ*Fd_I0cXJjApDmx=``JN~Y{Wb>9Zc+Et
z0f=e?6Da1IyBNh-ckU?-&H`^=A8~Dz=>i0HI=Ed|3Le!_dk{fXXa`dnN126?dZN-D
zuI%fxLrxrRxU7<NO-37XtV}9o(+xgxv%z5Rc#eFE26^~alM~q>=RvN_({%Z^0t7=_
z50Q&44=8dd%b0Bgme_s9xMR-HYj!}kC>#{)zKb!YZ(_k^bZUF?NM~X*OJl&jF2tmt
zlXiTJt2?=@df*Kbs94ylN&*t03k<aW(usEAJpp!t1szKRp-ROFM?z7q7$X@8LOf@9
zDz9!mgfOl*gUY50(Onw-p@W9F+)jRw1JwdE&6Fo&-Q*Ta(#MqPL#HM$T6c$JNVb_^
z3F6!%LQ}mp0Z05vGc`P`S78TI2oa>>_m<tek0i#Gi+H}(eVPhIXg`2(@PE+c3WE8|
za5hqfZr0Z136HTJ(XyWb>LHaCPNa$QBd?|cF0zUxDq=sJH48gjIoAt$p{0YFJP{{(
zG3ml_FL``ZV;7dK98nQAyxnQ1f$tiH&Q^H`mDi;;nPyELmZUm3pF0NOOfBnpR(oYf
z<yvhJL?Mtly?vgf&gcUjk1UTCcohO^5t(ok4{lgZhtJ~uQv^u&Lk$Jvrx#wmzJM4?
zB;4)vRY5W52J^EdqC|4VVhvkvd`K>Nj+-3xq<0Nqc68y}wwhXtbzbmbQk+Z9PA4Ep
zCjQ>K?VYW#6T8!^H&qV#Sd`>&tjcT7c_+AUyk-6X<7djyQ!<BWrBdis$<jhOt&z`X
z5e==8MtKrx@$%a#pzHl0c%t?MW-`DLXlL|kZa6R_oQCzm;nHs!3>#}|&c@^6`A-DR
zq0ZEb;DaOgp=@;b0KinGpQ_QED}DRZ^&$<PhtT3c>0;en<YPV9*N1gxdO<=nb2Gw&
znsbmNXAWe`)?JotUzJV}dig++Uv7HmJt=yavJ_s&hu_v-CS6z>1z#3&#Q^yItdMJb
z!}yztAlpB4nv?OMiH3_iy<>t6lN)5&)lNNTCv;S*KJRa+rFej2j>z}{?j+>|j&`Od
zoyvP=(hj<i)UEKt9iWkysLfNUOfK3X@-!zyKVJ&CH-nW+`x$`1v8sm};?E5cO4(XV
z7t&-jUomRWE15n+gI}q&*aI^`)DS0m2++7$Qkox<?i8gdz~rj9o)Q@Pxe&L{L`a7{
z++@@E+YQ7EUT{ba`Hig`#n_m?j+?9=qw)j2<+x>b=}lw*`8>2}s<-rIye9#imzQa)
z#L;R4i^d*}hm$%0KdAgBz*8yC=#lP4ANk{|(=RV&O{IP8Wu8`Q+;N`D!Pxt8y+t@0
z0T{;hqK~mua*yYskJ+g++{dT^vrDemps;#AW>$g*;{wuAIN21*fAkxB@;}|4ft$^B
zS0H>sAVpP_W(ceQ8)f42+|fu^e!))^iHcx`!c$xcv|yBfx)QqHh5wU*5>l@LUE2Y@
ze(%R7<zrr>XTI`vjJnv%y+yQrk*bc=<D489he(=p*8&3)Bl3UFKqEg&zHaR#GE~V2
zY>wVfSBlUkCl}WR_nF+k)zmP-=9Xxf19cTaq4yP2DBsu3v4ND1L5Lv9J)-8+O@D#s
zx(m7M;%ZNpFcf5E|AyST7$#UA24ImPJNZcPwejCaps=yV3Y%W>SW^*oh7GvPQ)s8a
zQz^(;YkJZc8rLZWl0Q<#iqEjBBC3)4@*W>64K&f~Z0`@UHX!1!g2s9~)Fk&)!~MDb
zuQStvMJEeE9`3?GViY^0u`#0&L*BS@$cQH>u}YNY48jdG7}=oJuKMIX?>|l+#w5;Y
zvrKB?oP&9XgmcSqG-o(!9Y-5n1V$E$g*aM8;1@9z)9&w!Je9vWY2|<ZOcBGt<4Y&7
zye`lrh8ikFX&)<x>JikYKVk7yO2N9`!81O^*F6Ij&%g?Z7O9GuT1(-Gyi9e*t<2oa
zLawW*2vx?CJ-Z0Z@j=iB?t%i(c6b0uZHH{gplcpnMK!QZ{*b4iXPl2Ks6(&tRJxS_
zr?mHJ?*ZYjKM8Kqa8VV7C$L@sMf+b66M41`fa_Lf>8u2?oRM!eB^IBBoe}<L&G}T}
z%V%H_gzJ03pC+Ij0caUTOBtCvzcM$wNM64^!^R9(AkDc?d}P~W05*B(aqCyYj7Jn7
zE#tAyxgSuSPSUs<|A=A5Re!<tst?~jc~9(ax5{>cuCB1+>-=L>RD@BDrSPabU0?p5
z*DEbX{fgB8!*!2${oXORAq&Gz%L9?0qg)BQp$eaNFFlJM%7Pyoa!#HY1wvBwI7uha
zdCnPLiqaY09s6zqj@$h<YTeuNeaBpuQe{=WJ?O=_t-|)>=?~{z^4=ccEA#bY6ucEF
zF9rJ#%B~QsyEV$^^Sjc-w8)g%W;#%|qPQa=_KdbAj;xqCGhM?NEjd|!K(NnWg02X)
z>p@XCC8<=zpKg+sDV=mrKl3xCikAicFGJSk5w2k<IV*2=$~rMx#<Sw4HOV}*#5F?B
z!R-ZC5noMMOgM^oEB{MO!l8br)U+Z+ygUwzk$)9X#0kw^TbfTAuAriv9vR~<=N`#Z
zPfW8xTD8-h3;3CSl>t;P?do*-G0|L=oB8&94duP+{lwq`d<QJ=I6G7x1QmOZrR++A
zehr{34s7_6rUS5nGXdE0qt8e6QeGW4GL0$l1<GGBRIt9DtZeUah#_a3#06&HP6%ip
z07?GL;Tm)}i%(sys$tGl9yWrk-+|$7#3fY6thLRxOO8FKb?>m?0h?sMGOSZ&(On6x
z3UI*Lg#sTvz`oaQWbzx)b@hr{D&NxgXI{kAIY22GWAwt5xB@J18)H@jg3lug`;=rp
z^O8RUv!k5DkU|d$Q<SAr+}`hU{*~l$XDH7~IV)BK_S%;QM${_<ELIR!{2ab*11^h*
zRlXFux~a?gvZV9p3Wc;n)51QY7<OXhe?~2mdfZ@U767T{54Au2n@|c5Kvtm;zMzwh
zXa_!~=D+A;)90J~$TnQH`K(B$x%=Lye|ZbP$N?{8@=fqg8R0Ds&9usY$pQ=IYhpc8
z6o2Qy28jNw<mO?JSaK_0`<~x`k&U87kML(}Y?K5r79C3AQh(<-=$2GD=wwh}mhbY*
z^fF||YW$&4WvqkHn;7aqO`a^H?bX-E=te<UUp_zKn!I3#EeT{3&V2I#I}W5O;ge*=
zpkx-9=4T_$SBo#0rBH;86$QKyA&)5Hfq;nDY83&$KJq3wPUEn&`RN+*tw3wce~+<$
ehT7*Izd35|gx+i<50Y3DI50nILn$|=UH?D6a`HL=

literal 0
HcmV?d00001

diff --git a/site/assets/fast_python_serialization_with_ray_and_arrow/speedups1.png b/site/assets/fast_python_serialization_with_ray_and_arrow/speedups1.png
new file mode 100644
index 0000000000000000000000000000000000000000..cac4e25ce962ab8bbd9da6e8912b8c3322b06357
GIT binary patch
literal 4502
zcmb7Ic{tSF+dtp$m`TPSMq_D|eNRFOBQnNbkA$q*cP;ktO+~WAL_(HQVHjBoqiCL_
ztPj~IYaz;Bwh*u9_x|zz{(aAVo$EgLeVyw%*ZJI^bIyJ4cndQU2OFLZ0Kh>u)ISFR
zgzR+=R@9#7sQT%IJr!kPVtHnFcUM$Y6obKxj*c!bFK=vY3=9nHy1Tm%4(?5fj*ez$
zXXoYRb#!!WXlTgG%d4oU(9zLZc5-rRZ#OqL|M201nwr|5KYxmfiaI+xm6ViHQc?l}
z0%m4rCMG7Nq@?2G;|&Z9f`fw>78WjCxR8;Nar5R)H#awLZ|~LB)yxcORsgtr$ojgL
zp`Yh-e4?q!8BMr&l`6y9jzE$=pWmMy$3O?<uGU&+u2n+e$ZUwi%kBf0c9<C~AuRfb
zQw_y=<NsT27EY$rIB=2|{nwjswx_)Qg6sZ0rMHl3`Fy@08=QL4pb2P(P#_u+&y)vg
zUg1GO!g-RW10k;`a!q1XalixE<7s@E!?m0A<r6y}OZKU3@A?Nma*_TyB^<JDc!;R4
zqVYQaaCPf4t{_N53imXU#8d>_DN!jhg&2!9nusGOIbe*cbr3a}+V@>8!LP)my7R_g
z*$05n+sf7tp9TXhWw_|mMYV}!4#JO8@A?V8>}<iU%2Qz^(+>zKHuv}+|7kAV@q>jG
zSKbK!y`<gNrjOn1mo8L_(`Du2y!mTHy`sQ!RwPwZ6$|*8gqhMxruQ-{m}$;90HOuO
z|6qvTv198*;>k%s6O6P5w;J1=P0DVhoJtH~zG@s>QHP$oV876vx#TFSoW-5~Dh>`n
z(3moM2-s5QDwm%~is(_4nnkjX1QHS|Qaie+m!W`KpTLH#!4V$Wa!JivZ{dtMu+;3!
zVv#az;9NK$<JI$0c+_o_5;>XborJjZuDWui1@W;&G;y+$gU_bMPGV-1A|oYAq+hz;
z=#{GRI)TntKg)gVfdY3*`(5cka`;)&xg!!e%7)DyXfQNG{Hy~4{}SxJ710f6x#C0&
zA53(4s7mRv{g7r=n`~P;v?YDbJcJ}-pxc`&qsoLLRP;^DV!ozmVHml-B(R^JlWea2
zrj7E_m>}lE^3?u`m>l87F}+-a#uiAJLr$Kf>VC*8*ZsY&D*qI3%VKU-NRSn|Q{2@}
zI#~u~&bgfN?@`UtP^Cz8Q4(rEI6<s_50egfrUljS8WTI$dHF*LJJ||R-)R3e@?|ag
za0`B>2!G8IJJ(VdX3750mT3HZ;HZkY7N4s+dxspX0$u6;dtBa-m$fCMCs7GiDVptN
zh$2V~-9Kv{V3B8QeMHbdCd0Ym!c&8gx>89+R-(y|8sN>4Dp-CZn&0=7>zd_jt%Esu
za_Cy@sHc75>)B7tZWLRkgR~^kh5WfJlpWdNMQLLghm@4~`y>tLS)Htv(k9o^efNA~
zHM+K`o4reZz|JAa2hM-A;giSi4@oyio{sQ#G0lqPegtkEL4Nrrb}M{`jN3{D566vu
zc#uK(SuQ_A!@gspPyB8|=Kxh#OlcSz@QZjnG?l_lI?(gwz+C~2&oT%H)7;J?G)L47
z=uct;Lvaiy+Sx0!_|xe)W(xRj3Y1(~-movapZTClV)w6lq*0++hI05A#E@Yv@w2>!
zFLzC?Q@^>ipIBD3c6JRz)N(%_jrk%3Y_A-{4EJ!?igRTUd}s3q-|uh$iDI?fa2(?V
z>%9gQMicCN_N6)`+^nck^kpP%a_I`sz}fK;2jvkTiR(GB$x+>JGh=hXc)IwI5AIgZ
z;l73I8CUUHV{o|`dB_YJHqr~j0u_qy7&_2Mk5puqTq4Ll#!{{}ahywVftw==xqb_K
z*mQ{W+f_+AG3m-Cx@%<(1`B<s=Ty1d(p@O8u5y4st0r{$_x_-nEBPSK-UBqAF2#Q}
z?6sMCFBFMI7SinLi}0N`Yv#Q2+(CNXHLvN5vy1*PF{?N@I$S-X`x{uHR;pgU5q<g#
z&clckXdZjBYQDIY^nJ>$@>!mzn`y(2kEY4TAd{_U3h5nc_Jv24bbqx0fjYOXbwBx#
zq0Y<MTl9BfLSKz5ibqB`7*ZoO<__KTXjWum%F$cQP}q&yIH1-r3Ie^aW03Dccl4hq
zX~$i$REwn`lWy6>6$M$#nTA1kmT@)HW0ybu4j0>e*q@i=%alve%;sS%+|$3Xq8s^>
zxk2FpItL+*>TLIF?A`s>yTC@vOWzJYh8hagZmF#HZJ2ofPIr&a5eR_*(C||TpD2+2
zqzBp22~Mz{{P;y+hjs?UjNt+sXsO0)rhnfL!;F%)`|*d+1V?TC-vcD7sKBKtc3)e!
z382Hn=#EVJu)V8$Wc4cWjI>j{*EW%}D-z=WD+%n(-Vev1LVHa<)9GM?6(L2It<5N+
z86+ToXs8lwbe-bIZWPKBR5S9-JKMjeTQjDozmv9R?o(HPy8ME`B`De%I)v#u9^Top
zB2j?^yh2!r8MiUSG?{F6uqH`-13$R<(e;Kk1EoNKQY;@shr|8p&?CH1HmAT61sz^s
zSAmdhiK0b*o=B+<J)btgsp#2p9PyVESkl1epKaBN#aM^|knFN5h&qgSN&L|Si+Vsg
z4%2#gZ*ve{br9ma_Nu>*2OH|kE(qRx;L@@>03*kLFado^9*YMo8H7h!;jHiK4}4$Y
zjz?dQLlda~czHVg5aU=Co)Uj%jm)82{EMuSe+6)3_hfHZmmruZzbD`h`cxHqdp)|V
z&w;!6(fm32@zGx^jiAn5By(QfhVj&dpBR23dLDO?v1B4Rxc7!V=#qze&30I^4<8i6
znuR6r;cXUaKaB7>5hh1hJzVrA&YBT)+hj59613#i`+Gat6vG!B|8uP(-|-bPi?2{_
z>jeJ2cmCuG<{R&#iK#XP^5rklR?B*q7(toTpv#!R0;8YmO~@Oz)Sni0`%q4O3VzEe
z@}6vj&NalEF~iLkF{}X`*g?jt6WZ_L=Zc;oCuVIb36uXwp1WN}gAkK?ZBG&9{qMqT
z>ZW`yib`N%BuUGk_T2{szC^^`7g<b?^*2ZV)X~6h#&Gu8TfHm8X;%MC@CuN0qw3%o
zxb>LiXCq_I5_Z9|qy1Y+i-R2>H;)J?Z;JO#hm-@4=#!k%^IUJSjH`38ietZR<i}?f
zcF%L~Y{X~lG{d=1%|ibIS~)*!pwKL%FsU<6H!~J$pypTF+F#4l-fGL8rd5RB@QD^e
z3Qs6Bkv|CpcDz6#+dOg4<nKjCICPctFwu8^tV5yY+H6jAn{;A54jh9vY^&gLAMhlB
zg&_<ir{~y^Ogjq;bi$bwH0&e^{Mdl59B_y8jevAIc(|W9e4N<R1UrcWdYR=A)e4Ky
z`=Kz40bNdrfUR(3BMdfdG%R9~Oc2^m5hwFP$_IeZU2GN(5dc;2pePHY6j-T6+0(4#
zAmst0&@_+|#CU-&kNTApabLDYwTenZGf>cEu2_e2CGIGVB9FZg=dJ`qAEKF&HVgAl
zF~9FE@)%!-mNbAF3rM{h_2phbw`Mf&4yo^s76+Juk%%zDuWzr#MVcDJ2EOh)PuAu@
ze9_<qGXh+r>!iEfMW2F+PPStKrb`XjcaPHrml&$o>P>kAvhM8g+|nX}>(i%wZ6)aU
z?9^3rL8vfN4v<TAGFruy(u*$CNJSvJFP{Vaw(EN2RfGWwdD!r*+YiCUHyW!{^DhQ-
z4?iFuA$ulT2DlsRjU4XI^RoiO5Tm<847j5LSI9{NNqaalWQ~&y#LrWzo;{XHj8%i3
z8g4)QB#ma+fv<RjiB|{qB4gN5VjJ|FZ*SDRYTUmj!NTyJMP8`Nfd=+``5V&AR;%}R
zpc1xSrCrwuQ3M7qJ!0weGfYrB<6Sy+quqBF=vd<SZVzAGS{2cy&sfB!r|X%kZWU;L
z`?(Zs;E}cK-gR4lOTqAX8Hj0L%YHKWLZ!-{6*+N<xM^!MCBABAmokUm51^$zW}K&-
z_2}rk5e1JggXiZdC8j_8&&z;@1B^pxJgs-%bty#0DD8?zY3~DkMI7jI7$aQieEU|W
z8>LqGqB>o+g#=hF3su!xUW2<=p*(BG5#V|}F;_b!Y$Aq@pN!u$1O|;WC2W}`gl+e2
zG-Cu}KV<sIs@bGO9Q!ID%Ux^_b|-#ygTjAShCWH1ryM1p<D3m8AX5oXp5#TB*+7AD
zL5!2Kc1$d#qjrpqpyUv}&_cf&yx6E~9kg>#OLNl^I29b4i`hYg<UK;8xz|XLvJ~Rt
zL8Ve<_gIRIHiYL+hyBZW$#g+wIi#9zkFa-6Dgbx;ebRUz8b7%D>~R4L!vsXFxDWTB
ztU<zA*ohPPE)~4(`u!bZIT5pGKM))jk-)UGAswf{&em04029HsGFTIbl(~@b-1Kt5
zjS~kf2ng-&s3oVn_SD$zM;6h_G)iwtIWHJvA2&^Rl?M$t#*l0a7SZg#vmaSstu|bQ
z$CJ-f;PGSNJ?tK=%Zn0c-KDBtn?*^tl`7doOEhc+IbGb4k_<MiKqbG}ze{ifm<U(_
zL;x)RZ)52yFrkX`SW-FAj2gI>N>P=oaVPv6@0UcEA$Ud;T1o<CMrmL9k*>rG0tro7
zvrYQDzsP?Ur(bRucQET$?eDI9Mj>}d03m%E^Nb7tZB3UddJ;5el0>w!7kcW0q5^O$
z<*sW?)vN1eF@dX+tXSlukF7*;NB_t%ZZJmmN_Vx8k$STw@?i95CUh!n{XZY8&HOSe
z9y5-mdL{Ff`=qsr&8}3TthKh19la2EVMhXd{KkQN>*Uj`VksZ&U~~Gc^rCJgClu|(
zoZRi&aJ*Oj*VuLe#Bf>N1+@!>7a}QN+ht|lULRSbd9NexJSEDih*@b%Oak_&Aw}%T
z;GjNA{9vOwGSy1eK*N~K)?v+qj|1i;9h~VwCIajuUPtoh5?6%?QT9y1M8vN|I3O%o
zH%qTGxW0f2FOkpf8-~u5JmLV;<3b+0;uB*<wi#PzzUZH<6jRMHRDTS$Mt%?Xj>MFE
zX1Ti<@ue!>mnF64GygV6(htxZM#P*1YA*-pMb)3w&3wr3P)Q6(OL!_5#OLHu<Iij&
z`c{`=4H#m5sUwe|Nz<OR{>JdXeq_lAFpYaAp%Ce?Ks|mqd1RJjWExnULlG|m4cD&H
zLfu+gp{~aHCc~tunV4-B(s^L)aT9^Kmlfr@Z_6S%wlq4X%kOL$K=k%Gnc^-V=XjUx
zi@%j&qY}RDs#%whympTE9fFg&CfCR&ghi9LV<N?gM_W%H6M^KbX-9|6Kq+k>V_w8X
z&>pa}9ykOE<!{eD*c=M0z*x}!lZ8}V$h-j8t9%eNxZ`LB?zZFI3c+SDG#S^5?=uIM
zqKM&Cpc*57<^Z>0ed}}im;JW)8u9q?rg$TJnj+Xc1}M+hjG!^7T=QaYJ@`)w1NCp|
z_s#IqUiE)G`cma|tKN6fb{59p-ats`!AY_|T$HyTAj8EujjA>KIV?nw+jP);-y3rJ
z2n-K8M>e8IxESLx?X5Zfv;K0vL)Wv0l;3@TmQ)~%BG8yf9F-)lZ^mK6CBgT{b~xG3
zC7%@<!(l>kt4bzVGWo39akQp^Q^laU@3GCB{qNRMft>%FGCA|lsLWPWM^aHbZ^0eQ
z$378|1^$IsmWY+vkA{aVp)D&zgl!Wr{13HSwDOZi7U-(DvbKySo_hXRw}guDl!xpX
z>MgQ5T{hCwBAp<|4_Mhm@2ibJKkp5Li$0s1FyYj*$MF@pdBFeq=<~;&-(vo|_V9m|
ch$jbEqYGkh7tg1mhQMAUpE1)fJM9$nKQ^k@MgRZ+

literal 0
HcmV?d00001

diff --git a/site/assets/fast_python_serialization_with_ray_and_arrow/speedups2.png b/site/assets/fast_python_serialization_with_ray_and_arrow/speedups2.png
new file mode 100644
index 0000000000000000000000000000000000000000..0d68299d135c3215422a2a6b7dd31ad1ae40b7e5
GIT binary patch
literal 4276
zcmb7Hc{J5s_dnnJz3w&NdkuwLQ|dwHd3JHh?DAyD93c%N^H9FkjVm(cp-5d9nKBhZ
z;roOnGChjO&|@qqiS&xRp7&k9wSK=pe`l|=_Bv<n^V#d{{aO2*olLVk%EKwb2>{@+
zw4gcy03mymhaKMAQPDC<*;5fTn`36XySw7z;#^!@)6>&mzkdDk<HzXe=x#tjz}VQH
z34$Q}{QNi^&fD8tUS9s*y?aJRM(yqG3kwS=DJedEeP4ZiYHDh_y1MM`?KL$ugM))3
zBO`ZqcCxdxU%h%&R#vvMveMYtc=YH|S6A0P)ZyXbjg1Y;VpAyqge^;|@v-Rfm4dgv
zMY?>636`2_*XUybV=7OkcS(zwv$LI(zonehx7;aJ+ES+r#E(cBb0A&5b*}RJQvR2E
z;QiaZ7i<`*2tPlIxydgsWQ=Y2k3Rb*s-XnKYjdl4x|RAE>a3270(!RVLLjAKcL3a1
zpRC!9&{_1%)7^Yn&zZcYY+wk`9K|&)(47u94akz)SMx6s5pAOR_?otRv8pVhti|%`
z_qg$F=|sfQ8cjvXvgHQS?X{pcvRJcg%BrV;3~PpH;pt47JAN)aL%vY1?tX@}9CNtX
z{#Si^Rkr37?5nfzK=v>52?vPk%UOJG(ACcryq72*>ImmT>jccN%R__vo-G-&^}k{D
zmynE~3Ie?eQXjx(Qcg1%@|j$s>LgOn`g+mw4K)yDXz~Mx<RxTTQAk0S)H(RW!(0NL
zdWGSUF2iQR`Wn7-wX1AL1Bui*$xaNirOxq@diF6EZHJp7H2*OVl9wKs#f8i(`Kl3C
zpD2GwcBBw{nkqB-*$S+wigMP+B2_Z0a(2R4m)>f_+QgwT4O5|#JQ>@+*eLb_(shzn
zg$x-BPL?PwlrOREFhr+jYguZ0Z)M5ozokm&nG?ITv#DM!GhY~q%E?!W5-fdThRiaC
zCBnWe&F#nfiUYP%AL463tW-_qp%1ff$5U;{n*OzdNDwiRQHH~25+tNl6_kn1+Qhf&
zw%@|Y52v(Y8AjrAnFwFZsYS9i3v>MiwM;q)TvDFy)}w!G<#HX4bLD&8bwrPO@ZqQq
z^Pt@8r~D)xZDJ2ugoV*=apohv%?F=O5P(9a-Cyblt@`!2r8_1gbtpYuf}Rwj8E};1
zVjjLQMD#SlO`W|1ty5p|fZjA72ZF|~C$YekEz`te1f~e`AL4ZgRUIHUP^i*1kSH6e
z2dx28H|Cojb@+$#9{K{ZsnkYB|JN_!EnH4gDx#X08NPW-x4ZHczu1|-#E_Z#<HSEs
zoZb%}31${bdI|niL%x$ieK0dw2FJkW%qCLB1Tr6VtZD-Fs~XhC79gXfS6uW5)$ttc
zBmEHxy=YxODk;-cA-F4xbxL424g`tnAtNpzBeTyXXw_b_%q?Csfgo|{C9<xGaH5wa
zF1We<{q9miW20Mq?Il&<p5uSKQPE+#Ie^EfAn=Iw+#%1Eplv@s+y$0yfwN^(<$M->
zRE!v9;0asgl&u<actaLpk2%gr{S>26i}TIj;WE-M-Ew8Yu`I(6+YXhF8qA!dJk14+
zgi$`mqA1Y=YH~>QvOBC!rT%R$+|oHd6Cf<&F%K@1uh4>|T2eKf{g*V5uABps>^m34
z9s~J?2UT|dGMT#WdWsvB<2W$vare1X;LVPAjF!9SIW;(oR)oBGI{%J>RSIN%w61d@
zT5ugd@={Rs)|E3rW-;~hYf9f(^lpH(=pW6Rsx_4dfSdi9npPfkfUfSF3hGNVhGQ)w
z>fSuc3XepWPbseBSaSpJUmmRXNX|5rY*BR-GcP&E#HS`UV**}*C3UZaEKG}yX)Q8&
zs)RH}UxWr{Gi&tr>Wx8lV`}7sbh!$#ke%d>Py^=SDbf=(iHt<1gW6f`r}GWIfP1DJ
zW>>p-+lZ@*WC%ygA=NhhAY1-UUdugB=D3l`>&=?i!jvG0duISkGwb)OAKBdmL;(>0
zFExm27xzy5%Wi1|>j^|uLDCj<p6?W$p12{328oByy(t&Npq^9oElFM7?j&-|ZU%9^
zL*M7iC-JzGcF)9H?7auz$RxN6e{C4fmQOe<yrn^Kb&;0x>Pr3@L$RPz#h%Ut5;_-t
zu6rj8z5XngNeYfg{UwZUhM_gLu8$?~QN+u)wyJXRG3a?{^*G@LSZ`K>q??D{a-+GD
zS1@C_CJT~rC~P1BXUw-@buSdi;;5;vrH}BbF=f89wRrtuxMDE74ZHSU;EI*OD@h9n
ztc1*4SZooJzVS7!4Lj$U!(RA03)anYv{KtM?}uZLI)w>W*aMuf5)jENeLn_^<--Bs
zPaSND>Rm`%;DX>fUG5Z)=#>>BH><-zka7rmSBKZ{fHTIzghgJ)Dw=wp2-mE`ZzIjj
z>3N8tuL}DR_5^%_K0(}w!Ja_1_*%R&_>SyfM-q0;T%N>H&9mdaBYM-N;`qU%a+!zX
z&;YRv=!HCuQ}&&K02y?$d<oH6_>*XH7;5Xp@?(lauDJ#?K{}MYZ^L?_yy~&(9k2Ym
z_bzr5zSDC6Qg6;NhvO!h?AmyJHT<zRN3A~p8F~m|O)oEyoM;L*m)!iC>Kx+~;?OMm
zfs3hZhyx}>CZ`)8P4SBl93cUT61NgfKY}YbCS_++bXozD(3(B?9EyRC^~ROf0UFnL
zgdcGgw^m4U#UQ&$N+$6G@b~PQZC=Xh7CFv+{qH2dnOE=pvKR}63#&#w&L+#S(H5-r
zvl1Wr&!#Vveu^QJ#XDcp=!DB0FUQjj3TJl|Kw4U9f9xknYwmX8RyCfnb!?{iYTOZQ
zj4%@3@R?^IaVkRb)H&mbBOHMp18l2S1Ms-J|EUj9Te(x-qES6(Y_j3|CJ4v(*U!th
z_Jgvj5!3a9&{K=jrxrsWf<GlpAM1ST+_|&S(;$^<qVjC0ors=!ki+_Y441Zj4cINS
zk<agL&KuoxS1Kd6P8HS}@uTf+Wly%((mg9LdWa%pf#3iXeeLQmFVJJg?ZX8sog#%%
zb4i)LCzI8^dbnB=$j4A#I1M_3J}kI-V$Yg=i=DBBKxeUNojJQ78CUzJy}hi)qq3rM
zxf%nHaRNf8CXSd>v=}1>kFg`xF?&VuIncx~FY+2sU})Svzk`xqpvgvCV#-`+`TT>p
zPX&nKx7~n4CuoFe%~=*Lg78=c=kSL5>z0$i{RYejEU^E8g#z@rQO0qt4=dK88*E8h
z_{s3Loj~w{3&=PtQEKwnGv*7ee#fn9gTnf|f&S4q=0?N^d%o95GNAZ&?i(-VkGNc0
za!RrxWt(L8#;t3)WecG@cwZL--_{R**+rM$UXHSj<(`=U+c+SyYY!qRYm4^6s>pem
zC>gThH5hU8E||&a(3uJKFT();;NRZ6RqG>=ekVM|)A`0WR!>E{P^g(RcIFtJy;4qK
z{y#|ziIu&65RMi_b|}v?HivSK<3e^-M}k_?jrIqCO?+*XoHFtXh#E*;lL`Rt2H69@
zmuI-*YMq4hOe&N%In?&NXK*^6@CEs;IgX}s0tS1LpvWEleSHlDjj=~eZZwk*E1K>a
zdH4B(X<W!HMEXdKobRKgHI8gy>Yt$Ti0Q0YxesuMLIy`&2q%Tmux{w?c~SZ&$Q`=%
zApEkg56I6cjH|w)3DGWYr3#^*(Ac+%5B)-F$Rf-hX3%N-_x@)2C~lt=V)j1g1DORb
z4uL>_e<u(Eun*uyplvU(eAqzZP%jYUp!{J<58kj@tFR<gEeWH3`RlPo#2V*5w@o~_
zx~z&M8In>YUvFLRb4udgeyR)4>?`SRz10va@om90xPSiY#dpP<O)mT{WYsgk$*ZRA
z)(f%KjNGuR7P{pj=M$Nc$0rQuL|2=5Zx3Zi*0kKFR|e(XAF|s!^dc1bm{<8n(Wq@K
znxEdNiY)SUKHW>r?Brm;_Vbs`+%rEd$JVDtN7Dkpw6{v-&k<o2<ViYwM*+N>$0p!F
zg0?E60Exy0RW;=7vD;^`Xt|}j7c}OjTI=A?a-w$sFxRt22)#cGOdZcu38Q<js}ydv
z(#!e;=v736N$?m3nEjZ-7YL(TKVdm~iV%8O2Q|3BJ0<|k{)y9Rpa75ULkyFlvoLcZ
zQ+W~K(7h<yKUYC=Kas3s6~3iG`hsZ7(FY8IT@Ww#%xzE|*9ly6A$<}WkuHQ-jd0H-
zgX-8$AYs_>V5-h68HyLBg#GHDBj>LRp-HB+g9B#7I232uq{5L7Ys#5g6HeKX&4$vT
zv#$^CBa$+^LA2ZzNcq2x|C_%5_rmDCX>g&&vp|rq@j>>uwnhZEPvtLU7$X$B$J!(t
z8d#lw=>K>^_Il)zoz0Z3f&C3Xi!~dRT1?Y#F`)2pENJT7y2JX38T%6X#Chd2b7?%?
z%P#7nQpM*3SHztA+U_(qe5tX#1?hY3(dtm|vT8P@Z@f%sZY47&kH$sq4g;@A!~b+8
zhW9H^&>4``uiDaFgW)vhk0kP&$mZ`#3-o(lA<+|AQn9c^thcX>E*NXb$GA?+k54RM
z*^r&l#3pyo-O62erEA}LQNIlWs9p(Q*)JRX;JFA{^u4>^8HxkH&~z^Hvc9jwnmuoE
zBR>4{NGpcHz(fnzD%ErFQY=m=hAkQfkPXUbnP{7UDj_iJEZ(Md#ET1SRtVED*(RJ(
z3B1PF1A51eEhc~%DU-9DjqePVo6>aGL+N8<xfFk8`ILsPL-O}R3u(EvL&~pj&Wc3f
zYExJ;Is93jIwu>SU4B+}Ef>1BBx?VdU6xQF(Vp&E#9nH%Pv{%1>$&l?eFfLh$6=jy
zi1=bSl)^D{UvB8DFh$U8$oL84@2eHEU5esGWt`<*mp!_34JE=?I_I9^5Lgxvd4N_D
zE4F}suGQ{YdC)>aaFClxM(+DlB0fsKbO8)`(x2L*#xwZR<ezVa{W7J+0AYw?BMa!W
z@dfnFm`3V$n}-R_VnyHB*^LrSDe>-PtDx{+ij$TtQ5U&B%!XFiJ32OFnDtz!Fg<(3
z6WA1klT#i+@|5<<e=!@E0y=O%4oCgwqwHv9Nxt$G%j{`TkLB^H)ni_$$_a3VY88>r
zV`0vZKBsgt<sz!=%0orF9eVvY4k=$hdEsm*6jFbj4gFrI=CCFWcy!WcwMop6eIrY1
z-Jzn3O~uYu8Ch{59?h7U(`*uNe$B1FU4{WI*|lkT{CV!9%X%>o%|8;)EFoIR8iyam
zP7+XAtWN!w>#-n*@zb6Yv@5Pa$OZ86`?ejVC37!LCo%V9XykWSPunc^j^EynW|Mf7
zB)eBl{YA`%>W1&~FJv>bg6rdM(!bzm?$Gg>W$K@4;U6;yjP(#&fr}@QzVFb`25uD}
zFmp&TDejk1^XYT?d6=U;3$NX(+2~a<bdrt2<~Z^v8>Ke8uuwy@7`xi_M0Gt4M3$<K
z2$~1-bDpRl#n|01QE&#d(>k5|!LM@eSH+R7+#whK)yRFZfw-=7@BM9&|I?@cPuKp&
aV8ckkHON3bB3}aR4NEgSYPE^am45?-7Ep5l

literal 0
HcmV?d00001

diff --git a/site/assets/fast_python_serialization_with_ray_and_arrow/speedups3.png b/site/assets/fast_python_serialization_with_ray_and_arrow/speedups3.png
new file mode 100644
index 0000000000000000000000000000000000000000..8f1879750431ac474013c22807f5946752d8fa17
GIT binary patch
literal 13637
zcmb_@2{e{_`|e}NJP(nKFCmExNkTG2hR9G98A?hdl_8lkMJZ)Y8A6644V27dp^}J7
znHnU8%tM^(*?WKA`p#P4`k%AbIeV@4w)1-5=lMPNecjh^yI{6kpOr~~i698pT?V@5
z1VI&yKdcON_#3+K_n+WDw5PRqSu)@se}=<R_<zQe1~#V&g58n)LzTDv_A!DGC3flV
zu=Gv)`SGl+$@~iac(?nLyF7br_DjDrbT&%#xcE%5Nsew^Ymd{_XL7BZl8srv9^cgB
zo56p?--Ys;?|kaTmYGK{bQLX?lPzxz-|Q1P6KKbqqqcE&+F$+Gkmswd>62T>PH+1-
ztC9ZdTbejG?GA#IGhgsg0b6iXQ~@0~!5J6rL@i1&-(yVZ>dHhj21gm_DNu26=1WTw
z6bf%6t(I<fRx%M4Wy$kDzu0^$x1@2fvE+GU$@+YY+YR%pCS^BnER-$nU|?d>F5G)J
zBPVB7iZLBMePjU}<MZdw2fuYcaCue07HPQaRsl`Mh?uA-v0?8Wg7Ee8qbMssbfqb9
zyr~%wosl6*1n1<elQd2?TmONPnYlh~>uH&sU%@!=;LJ=he}8`&=e3rVo2WP{y}!1t
zA%aUwy;Hga7N+u(wd`zdWj0bREls`}Ty$<t%rs5cZ13(azEaLSSa$E8^l9p&oq2I{
z^K1C|Ee+Hy=aY#%!^18vI;N(@1@&efmFH#)*amJ0tNV&prEhYiPThQbZ}{tqtJkj4
z*x7v)n3#`KoS`yH30R*0HvH~^RH}+QlaP?m@W@D($#oU$`-d4%ojMiYvVZ^nX!rW{
z>(}#Z`m3_Aus9q)z9DgIz=E=x*9fh<ySufGjpO9k7YwYdyxkj)rb?gu_EN|=MRC>Y
z)vMb(IzmfI6nZ~?^!&3hed^~v&j5lcoH{*VW{sP>d*-cMH2C@C>(>wU*M!W?eDI*+
z`9skRSU8jNzJ6tHh~4+>+1j=?mQzmzckbNDzsZ5fx^=5j@%HW;p?FBouSqqYeQ|wD
zq_O0=Z||D6^!cBU722k4Z!dxiKH8DZXlrLz68OiDC{7JLte9fG{OjXWJ<XUcE5ZlL
zPUy8JZp_Tjzwl>qHmQf+%&g2<Qpdo6Av849!QGv8<Hn6WA3h9Kul(_d^Q~T4KG*a9
zz4n0vYfDQ@d#Zz~%{9WWOUc`nht-~^ukY#MPfbnTq^KBr{ya^t$@Pf%_;{bd85vVi
zE?UR=-(RTFS^eL?&%eFD@jxk0Yt{TnS7DO54Kp*drwvZ6aTdoED$1bk=5_}aW|b{`
z?%cVFKfk}UHZBYf4Xx7Asdp;7TU;#P(AZdc+FnCLqtR>kP4)FJgOpTuaD*jn*q8XY
z_+z=(2J^n_@_c+rWldasd{UjL;2tk$#gy^sZhL0Dy_D%qS<kf|i{}DnKl-dI>dxN9
zy@MluWyYh5Ni*ikl`Bpwu|(LFD;xtaZ)|J)b$Xz@&z3#d?DlS<fr+7Jr}^{r>`aO&
zqj#HeNKv8;p;zMK#FX87?ltsW7TLZ$LlZ))LbWFaMZ?M}pd1_*$9cxfD=sDF!rcR9
zgU{l`RnPvGaB^~Le{{T?@+0T=?L$LNm+n1!)Ocf?4?$E_RXuz0Vihqp(#4Fg4Gau8
zKHOsFrlq=h^QP~@)Oy0<SMQUPr%&rRIZ;+`+#h!5jx6e80LN%peYACGe4w5<=ymbL
zr;0VXxw*dM{Z#lQTptx-UE!;sxiqeGQE)34(cj-6jO(NFJ$LT0D8u;g-*sqSf)L+%
zF*qrSFYwRwKL43_E@sj^;}a9Lks^T%`uh6lQ@RI7Ueq<l@KF);Y=Xi0`Rk31jX$la
zj)<UB@%VHRmm|7yBP$~#V^mBG4Pl(7%2`xYH2nG&JyFxy$%7vu%>09%{rpfi@bzoN
z_cyo2<>iYzZ{?oot6~ib3Oe)SEvr$ITw{(=()j2*rO}U1`IKB<>IMdC0uw|Ts0oM9
z&*GBNx2Wrg;;MxLgMgBf65pv|PQsxrSs_`~lXCyTgNbI5p#IzGfuZH)e))su$E?zV
z^HQZ-*VJ?s?hAi?OP?9H&8xh5bIj6jCN{R!t*bR@dP?1|n`?ak{(Vppr7k8eu4Aj>
zH#JOgAue3FFsM#XPe0I>qBPk0?wyUT_ds2ES(AODQm~v|c~o`cp+kqlM@L7Ue#n03
z8~-37ArbW>IW?8(?Afz}_Xp}DV_K`0ej9mK#l*y@uArF4dUw`-%PTD0kf!RXJ-0Yk
zZ6tf3cbQMxpU%P2QLMsuLi^a+V=oChc466<?99yE-xlZQ7(&NK-14$t-Rd6;)^;J{
z;^GD?<|n@~1A;=eGkw3gXs=ujAS@+)#y;tF7uj*HULCQ0Wr<t<V1=}KZa?3`bvjmo
zY~_H(8TX0GgaqD;7cVv(b8-1b`O#PIW#k_;F{n2+{94Dqv8gG#wdDh!Sx@$^YXmVn
z)))EUfzs6Uw4=NGB{Yys<D@22abH!yxudSG(Qi<!a;3I5xX~Sw*Tkfx=<%pGZQ2x^
zlCq}h(%ML7KDmm0H-K{IBX+)9%Wi9H+b|ox@^)3T(ZJBai&wWE4`u3K7C!nqYZc*u
zS8dv|B{C$0%5P@$0?I=t(=2}LPOExaE$92kyX6iaIYR9_Z8|@6dHWz9MDq5aD(l?r
zpo+51R+g66SjSz%_UNUkpH++%)`&)b*1vtb8W_22*DgHRwfJO1WA86kd(!oB?Vi7S
zw~}BS8ynl+)zz3Dw0uG1+)PblquzI;vWK5$eEhHbO^;j{W0v9=@VaK|nDZ{k&dx4v
zn-ANDeMJrQBL0Mhg@yXL8J63o=}l7U{uJ+17SAzejDPi2%U9VyJj#hjcM;G!G*-P5
z_^sSy+2VkW&4saRSFetLe$MS#J}$pNt)ikbxOh|JoHj=orFeAI&G?4eBjd#>l~B<P
z`|7~CUr)zQere%7?CgBu(C0TD9UR=;G=!PO;!03X$MSF4itVUM-}&D;s&?&~D@UHp
zUg&%GZq0{McO7kQYP@SiO^p^&9N~c(5$(*ptT`54O}Xaf7=Agjp+Wcaw{J0T8im;L
zQI7Qy*5zl`V}`_~rbd4#^N8uz6I45sXFhnxq=(U1uc5j5P;KZcy|0Uli}iJN+G)!p
zg~g&tt(*W<whOW`5v*&oj&~PDUARy?`R$d0-_+5zn;L7;{Fn(raldC~W=Kp!*LjX~
zt_KNlYKY=&@9fk)bxQGN`u58rfpa_7$HjL?af*|`iV~zjXEwGb$tR@lW>UMyyi#6y
z>I+lg(uBV9(RSkN*RQA%`O*{U#)l_93Un9S>$|$jG{p+VFmg+Z%E_^9UmB+&wB(Ww
z&5aL?39XF2dqCb7#%B#?rhtIJ;Hm@RL-IY5&6h=nFsXVTxUkF0$~pi>RUTND&iwpP
zkJruq`dANi?;jWlDLK?U*i&+74kIHb{xa?o@2KEEck<!U*VKUZnlG(MjhWBFXD=C=
zIajh<-mP$#7G?5M`&4;uA4+SZx544s#kzVGW*=%kDHWByS6aX5jo`fON~SB_`G-H0
zT3Fktg`AuHWjE*z6b(Z`MLTahGi;V^JkS$k@!m0GRm*!u^<y2`TtnTjUvpl)dbQzx
z__mwO2UpQhGvQ6kOmn`|$_-cW(tdHg8%{42uguc<@@0{MhRe&evp+oVdH;Ox6R24b
zYocW-iPMXUj@JJ6@&-vVHf-47|7W^8S^ezqb6wpzMtp+Xyjg^{d577`MS_`O>TmKK
z)GfJ)j-IOmZans#ppc{d>x=6;a!K{=?Hri=$tv!t9gP;ej!hTWob>UD1hN1tGBYzn
zBu!EsMte$B&&}NF*z^iNpNXnAbfFpRcyihWRaO7y4Yx_EatNjdrdDL&p9L`)8KyV4
zP38A39cqpZzkOS3f2kWI4Gj%|k|Y#55iGSSN-pgcXHS=m;vy7WJEh5SYx_a=GOE~d
zW%<w1H@U2zPi@^_cH$CPUKXIxDJ68MwyBA3d3kxbBZnDBoROJH6%Y_`W~7txK&e|u
zb#;){&z+aVxWk<M9HJyB4B92`Ew}V9@AUOm4c_PeL?@ChK~1@UdYurvUHKW#MLjMq
zuKwX+X|JP?xL$!+eylvl2fow{l=lR-+CKGW1>faZBu2W6+4t@1mWjQZeF_Bj;u<Bo
z=2)Tnk57F(zjv%_OWSI(^xNaz{dJ>1e-^5?yg%<kSmuC(8dYSb*vjeS(<4~cP&_Ry
zt#;o~3qM%t$NTgSz9OGrlAN7HY{85<yYJ-g1G-%178W`?chZ5CjxWuhCbZt<nke{=
zJA{RWN!zLGKC-W!xN_shrQTlKy!`x3n+Hemn3@4ijxH|yy2RefsMC-_Vf3$m|1B5x
zQ72v1&a6QLpe1h@KIOR?HEc(_qXyid{jukvoC8-X*d>F_{lF^P+S&@#SSS>t2It-W
z@YtJEU3(63*CcK{aL=)xE;}cu_R}Zf>niT%<>4nBU0&S`udd#HqW5t~TAHA>wY9@Y
zXMVDh%kKQLedv9nSaYh}>&M<_njD-7R!5FV;>{bzvVZ5rF^a0I3xWmpfYGI;rE&1^
z&~DkXrMKcNcT-c7UdxZ~-)ld7So`z+qw_a!3ctLm5ra9UZEDJ%?*DsNeyL6Ss72Wc
zJ1C!mQ@pQv?8GBO)V6NTw79*yr{b(q@V=juM~)mxh6pI}DUp4N>(01+I}(p6slXo>
z0ct-!+ZUun#jz(L|JhiUz!oNHd3pAUi3!6pdNT`)^M!@SOJ-T*cLPDdv3%p8#2yNC
z)5gceg)A*CQB2n<Dk#)mTD#>_`3XFkiQ(7c75?*FR8&+RmFJyLoG9w})4+T)R8QCP
zK<8sxGhwvtj=g(VliW+z`X2H2?OT4OBgQ4pfQ%U44SNj)#rC|2OHa2SRm#lw2Z6~f
zDw2C%>S{}Uv^|5Wv#Tra=FKAI?HU?F;C&u;W&OE_yuG~*OilH*w6rcTI)Jx~PfzpS
zX7>AZ!SaSEgMpDz;lpoY+zv;NGFVu=F|7WY*R+m!VjIgQq$aASw!gwCTRT(h%o*j1
z%1R@<%ODy*#D^X}d`PnLU%nSyuZPM9f&4%n{jPh)B*8#Umf`q-4xx2x=Or1hbDQO;
zh+q(Pbq$TzC5QCS{OVn^W5<qXFJDH0U<U82{xsHCeR-^Ek*%iiI9=O|7q1JfHV`%P
z)k~rBRddv+xS}P84dYZ_x?XPSv-F~sC|O$apJ?rS_AIZUK0o2inKO{xBAQ_(C7wAd
z^j*(r`+xn41*33$ci$Peb{^`3A}-z-JJ8u_nVp>-^@W#d)oMB0sBJ#Kv~+Z6_U_$l
zU3Nk!_w84Cmn=iG_OJGH3O6h#m^AzN<PTc9Qs>2WQ(1=33ZFZ0{Tj<F!ty0XoF{~w
z^6|GU@87?-654%ty7ZMzPBFSCE4mi4T1H-3nI|G5B1g{S*s%*485vwjv!n00H8nMP
zIrrxj#03z#mM;uGcFqgGxD07sQ)Ks)s1P{rapD9UJ3G7V{)Y^|e*J=|)6&zo`3cD$
z*D~B)ETXNgedYS~7{HR#*R~YQ5bJvmA*u^I6YeoW!$%}0t~NF?addPHIrb*EwyTRb
zl3hfulyMmRR~sFiarf>ekUxjv_RMu}pB`#v;};O1sHv6re0lQZ$@t`?&NWFB)I7bE
zlvJ9UR{_Q6Cn>w*S~}jo-Ffif!MJ%&PELNcGtzV{{J|g(Y3gUW0EO#IW2b*~liYyB
zq_IzxF&l?|{=C%FV;vSAJ^-o4zx9;F!t{^GzMz##?b+LRdT8NA=yTBO>gx|-qLLjK
z_k|R_J1-00J+pn&wrx)-UzYy-DLFTLB26<u!2Lt%M(+clw+xJoI}8n(1h$-DA;`-;
zd{_bpa_`8CRHpZZ_YcA7NSK&>wOt1y7i>2e&;{~;5l(`j>PfBqUI-Nh=a%@gx4iuP
zQQ%p&k55uTA<LKaLML8HPiJ4dc5Ti)j>0N%^Jdr~`4@q<1AQyYTQEw<4`sbHSWRv8
zMy&UiKD(~2t^)>*M*xQvx)lA!DI#M3x!_<xz@Pg+wO0LgF_}2a<HvvaBx24Bk>Y$Y
zhWGaAZ!gDZW_DHw{vij@_U+Y9ty|U9DBHJ-pni2P3u}DZHq%mr3Yv-lrMQLBb?^TD
zI;f_J-(QRbx1OTK1T*|n?)@`bTt7A=Xl1#*y<K`{P#zb5QE1z0VPRng7M8GEx5R}t
z{KAHZk3Kx!%|=7d7VlaX-7k&m+%knSg!ZZlTAmb9RaNaj&#qYm{SdS9L*+Ri^0uB|
zUDxyQm_>;*4wqy;PfmZ!g1f@6c$gt@@#hv868pZMsNnXX+h<s@##X^sNj0JVdhM?B
zTh5D9vJC(4J*9seP<RK5Oj-UG55w|GkNu^xlWO52OvRuMU>QyjdDc%(<+OZy8w&f%
za%Zj1fPIB;$lRPSNTjypHAl#ME1~@f4&1P{E`~<dT~A4Ie0D`__|0whSdr~=a<j9u
zk(Vy9LUX?cW018g=Rt80goKooj)@5?pPa2(fs(W|1B?df+pE8f^c=`SIx{_DMe_Vp
zPi)i3p%fMoG5G$`aRxTFCUC=vFG=>)1RSt|k&$Q^137*;t4)q|w0Zem6~%8-`4l)`
zpOf*GyjNPP0y#(mVcYdBrnXgwpL3o0`JOj1F_B;SC<o@ky`Ysq&-u^Kua$W8X@36t
zm6`yr0N+n2)7*MT?XNO=UARx)w!)Xw)6;W&YAOtP8G*Kz15D_bo%rbB>dFKfBWrV?
z7A-RPTTCZH%g#>dFZwGA465{-4$aBQ>3nJ~2BXiwnMa+w2eTN!r)z5~SoLS-0HNjP
zCQCABDjK@}Z|OlhNh?AA;Khmxb<cn&wj>^bLx^J7fu1%tT;MKr^=Y|u3tr;7{QP{!
z*I9a)B#mUNB`5Qf(|lsE(JAGr&)8}kkKdo4leE9vKIl@CyuDM;JxB8hyG&m?0OWBz
zS+jlnn!yp;v-FsO%A&HeERbn)5Ec!&H|t3D231|t*C%qQ>0(H1>~3D&g9?A$#Id7C
zYcT15bQkLba1>l#3c|UludSt0P*8x1IacxmjFJ&zbaVMUeZS6t(Ixh9c{&u;sX4Q`
zeS7cj4@XP%1Dg`et1a2Xld{?Sflv%^A3-cnf`otm{FzilC(oP-yMBEw&YKNZz<O;l
zil2v#jg7(X-Ay2#CD4WiY`1%59x+ObiBW+c<Mj>i9=HsA{!A+(B64T{BPMt`=S3Mv
zvjqrp(d)<C3x|$hh$;~G@bEY@+)f3xd1vCqcCB^kR4gKzg0Q^mK{D9kYjL$gfm2jf
z`9nfNtREiT;~b%@JM~FhM~72DV1JjRWwN0OfyW=3oBMTJ>(*`C>S0Gze52#!w5l?6
zr}pbkS^<S_wY(WsKp!iVZ)+Y<b(NLI4nwZ4zP|DMo?AO>pFLy1Db>PX$!0nq)soq-
zzkm1c-MsqdPkH^T68_w>zP}*^pAd~}BnY%p>`?RNApj8s-FDo4WL}<hzP4D_Y<5u*
zn~si-y8j#pkx^L4oMODUaZi_UsHiAx0CFh9#sxrb@H%&OUGT$)%E>7yRK8yNRk(&s
zkG`sV4<CkIyLQdekz>!+l+;w};AGmo!8HN`(dnn1ugX$&7Vcw*3_A}aft=t95AnX+
zx7!zNiponnthh#Cvnwrl2U~9*EsVy5<<{!p!;B87v$?sU%P39eB?z|RzUu9SsDc6~
zSd@AA4qaYdsQCNxG6jOTB5({Y^y6cnG2yqx_Uz~-LJLyzNL#X)6+g~A^)REBg#~wD
zV4&XDS1C&R1sqXPc~@}yEAy{{4DDuaUDhmUFo=p8Hc3b&z}2&qH(q-lFLzQ}-T$Ic
zt*&lIK__!94^#MDbaXW7f|9lb)ZHKqFV$0@8Qzz9Tz>ZKSqsbgsHkW$8<F74m)T2k
z>R{$ElfAvYIMEnW&3RfV+$0$u3MDkGEY!fzP-Z4JgN}|4m6#|4-tA{{pSC^ULTYYd
zfuf;dmzH$%X6)I?ubG%O<PGZ<?@A=RfAwAf{Q-X0zke@)qkw{LhGd!eS=N{MqzVRX
zO-Bb8=*-Der|K{fjAYLZw~I}Ee?!l@Rz(*uJ1{zW5sECq#V0DtRsHz_ap&$`&-vfZ
zg4+WHx18uD2vUio3!JM`ETLQc0|H7w<5jnA6$8GWIC0|f*-6QZmoCM`uK+&VZ<~tT
zv3WoTj)b{ho3Z0O2Ol53zJ4>)yX};$)ZCq#JfEIFXS|{2Rfl4ust|C8?-K#p;s~aZ
zQ*aBcfD|zIom&m9ZvNffrV>vCX3gfPU%!4G6&u@BAIY9+CLN#nv-imZORvvgzKCgR
z3d4xv`3Be`*%9o*oC26j{b<=dh|&wrr%s<vxVa}SCB>$o{6dr~6Yg>E9qSONUhpJ=
zHEY)R{{DQSqwt}kq9REcxVgD=<|QR1!7w*i3EBHkth#^y{wCMXJg(L|c28I1`3x5J
z?|rMQ`zSn{J&ci+wQ+7@DChm*l9^v0NfO@=d$w_gkH`A)Nim#O<M*~UV@WDXft7?2
z0q`2ez<KiH*x!6UA>tZ!aYTI84=N@QJcxBgHbGSa!m=!}3pqk!;@&4i>nH_9633}i
z@lAZffnR<gE^23ngFyp4Z~jf+({9`_vPIVgkR~tgYtza8np4d?bS>ZP@}?toquYm4
zR;k@-T2H|+)JJ`RUpYms-Q9ffn}>T!#MOL$T|{Ps7^S|omAR(y>-ELiv7x79RW#!T
z@0uayYiny!KU|AU;N%Vt4rV*f8R%L*Fi0fi?5kMd))1ugnvj?<tw#9x_^A6$F(7Qv
z*}q>vP*9L#&6@SzEVpxWDeKnh$yU36P$4A}T;B%xe2w$C1jHx-uQVu>f_JW;y$Gw2
zZYrMh-BcXAiYUc2SMIQ9+#{{dpyhcNE4S3BsDsz;FdAf<IZTgqffVV2s1X^^&dKm9
zG2mD>KazQ_r~AmrU1R9;yCXSQWT|8iR`B-)E^gFZSvpPn&&0`@Q5#xX+J3ad+~|W>
z*-YV4QKF92J&%s_d5`vFPL1}#SIOLUO)^>G&|i65gT!&4vrBUd`yaVl9S%e=Ha<6T
zD8CNGs~Zm$F4*A5ZOk8_HzD_P0K0!(<55_i{HiBKcc-a}9V`Kpi@~HFKs4}A?|Req
z4-ST2-{Ro}>Q1`;WO9b_G=R<kAV&f_IbR>Ys>z;#D;XU{8AwWOqaoWWgoZx4ZTrc7
z0++e&UvTCCg{&t$NKHUBTR%8*>7O587aSkY19ad^@o;p^JTs(1p<InP%vb}<1+!aD
zait^22uaS>f8ouhBW?Vo$A|QUbmf0fPELR<^Q8watOt|d_|K=^3MR+x=g)_yetldk
zAW*DQXeJF-S*uA<vSdVUs3Bq)*x8#4tO`%U4jCREu6g;A4P?wbTuWCs+`vzWAguTA
z*XE!_kaod^T31&>16>Jqx#?Jk1R2Hw2nL^}0~iLt{geK6araIRXxo;bVI<=)*i))~
z==XtvfqxG4E!`yCV+z~^2$(&74F2#-<Kp8b{<&rs0z4eioM+tpyLayl8AGC?vc+tu
z$@-^IC=K63<^RARiz<thq)@&ayrq43<b}Z0Sf2>Ac|9CUPC>zop!{Nzl5_z2f8n!I
zFzEviHpalLT?bOg!OOb}ldGYrDGT1xKNqIP(bv+#1ot)R)HPjQOX}w@|KWqYo(ljA
zl7o{e52)F{YYP5FBa$S7DD&uxc=~i3h^}XU4OLtEUqdo#c=L0v8hjrS57FOcQ3C_~
zp3m4plF#<np8xl7pxptzL=ea!sA=d}2m(WzOg=DX@eaF+X{;;#HhYhKx&%@|#*suY
z2qe#9o&j3X7DE(}fUtw)WHEg55sw~iCi;<P=;J{CDRMXdqd=7Som9M-%`~6`fB$`9
zC#@($-L|nwi@BE>c(ymdU0k8}>gwt~ZGE`{F4T+xG;d2#D2!GkYighf`p3pDuTeUZ
zfv}f+AfzxuObYXomOOqy`2><l_D`QaC2I!wIY;Q|Ztm>7zNJtt%y81AHPNc{Cq_3+
zs-|#eKC-*RA|nsIx~cK_+^igeUJ=q<Mj9|aK3)TDx~KSHWeW>>{*&k~De%INuc49F
ztLW+K=19ETWQLnR_}|lW!w%Xj@$q5nj1nE%Chp%*CJ3Gn<Ssz6g74h87}1ZeokO!o
zIfG%)GIG~|48VtreEe7yo;w35CJ~J6i%dxW{e)J_S_cC<hcH6WGlNU)nsY?;@<QRi
zyBCoJDed*X0zf81HtFhGY2BxVpA*&D*-4qita^=H^WS9y4<Q03Tt`QT=(Xn=X3{4S
zh0M***Vfnn^UzT?NW|OW!;rPGVFR=K`-ipF%hU3tfH6vxjr*t7q(Yy6Ipx+@B}9gG
zxWKrCgjI+K;frhkMU^<7b7@hXK7AU<VpM4Zs&f9YiB_m6iim~;&7BwdLjYy#D_dMK
z2uLyr%>;Ar<nq!yS1Zh`L3C;a1RU<7e`v_!rkxn2fEQQ_D}{f1pgQ7D8gMp#etzl7
z7P6z^4_a6GZ-tC<@bF-VKR7WmxaG^&uUYsMGPMWrZgBdW!K4P0C5`TLtO=(2Qgj@{
zE?Lo;siFe<kc`3cY)I0yig`89deX+}?{^?-5Zkp@22irzYu#5O;7Tide5&Z3{)b+G
z{S2ya-n{v7c{xQzrL=dNn=_vYO-xNqeb1?pX|(qVx0;&Tn4Kw%S^%b_lT(&*Uqb^e
z=##WZau&>4Xu#+<DJdzudwtyZ91!WWOJM&ldG#ttZD-z|o-z*+Fet}k#~4jaOme;;
zUNH|hmjT!`M)PhU9oQ^GaT{vLL|uP>bp&tpc<j#(-=8hGz2iI`)Rw-VUwOyU)YKWI
zl6!UYh+uR<U3d4sw3lsdVW2(XU8cI0T{vA>J*wWL{N;d(qW_=~ENc{L2~-P*kPxef
z$5=E2FF|g8Y}|O4sqk`CUU{4>6{;ok#K&@8^Q8Y~_pR1nwbAx52jawLpS}R8qSMG5
z2`x-!(yjn@Xv-ztA|qKYZq7X>xE0LphNbyQl1O3<Bo@m3&!pj)5K3<%05=Z;ge?MQ
z$o!-Yt3fCY@6&EN(W?q!R|AACFE7_q3lZ5KXq&r0I+||3!Ok`+nsS6Gk|*#1bQ!rv
z_In;q&V<lxXx*s1g}hGMD=8`Ky_c$fXct6}h)PnjI`TVd&593dWasAAVN8O{Hpc$p
zqoq<zVVR$wPgujm$yu3c&Rr9<vS{B~Y%fBbgdlF|CEs;VjwHdy$9Ko}2^XZrNl(xG
zHWN<HJ!`BMK5Fd)=e%?0PRl2jzg{Rs#8yjD3T{fA6|PzQG<PW~O6IaeH&sthPlXa=
z{IsD1g(4%ei>n4D(6ZEP{}&`n1S8x=T4EhdzUz7`KM~@?3WTRrEHwo-f5Io)Y%^(@
zbKDR(!V+uf40Uz8*#GAhpnuyn{~huEAC`;$|G#rr5A8kY*3~3&z+wtit!mKn(t*}=
zH7_RQ$&v6$AzdJ(WlS)1x-2gl6W|7<ivdd@?#dMcTPBa6JQ+Zi)0kL&`WqV^n_v`a
z24Z@X8xXK-JV$#JE{Nl)Xu2Mz?s@Dfo@<;!EiW&R=z|W>5e_#V&|%{?kI|b2azgUY
zA8+ptz!~Kf-`U?PUmdS{ijp;=gY78kLd<hAUxg|EpXkx!<#gx}9Hs*0S1(?$0`JzH
zklM0^4-Vzv6wFjs;#pgp?C~1s4ff|hJ3Cu>F(hQQ1r3d4CklH!nzp8C39R6a=cCQ+
z{ZQn|C@L;agOc2&ps**^!^tUh?UoZfPj_P%sN{>1lG4Ph?JHXP`nR8{{R1V@ZCjL-
zHt0`1GT!)8pYiSpk(amM?+(>iPzBro76_X!@4YV>5#~1kD3@d5+960%>)|OVrrco?
zFBDc(QgVb~1Cb@sz)Fa%%SruhJY#4u(w|Jvp{6fIg-z>%VMmj06_zawC*Qnz<LK(T
ze^d>i-<Vj8Q!J@orqVJpV#Z|}PO`AFwsM~RX^9AD+2X2%@0~x@zn6c@n+>UjeEDxO
zC&sQBmPo!j0eM={@LN~6AOnd8K|W<TcqLdQu&oXh0-w?M_EsSwzrwX@VtkyV)f9_5
z;8IP19b6|XnT`Hsd9<ZiSUuI!wfy30%gsSNe)@C>EoNQnwgxj4>?-)#GlP;_TQ+az
zg%9U64_8`@LZLqWwQa}Albf(S77fZ+^Ww$b0`JTw#Q1e6wHW1VoL0)h#}o7=EgP&s
zEekwd55vdgOc43sdQnCK8*^>lil}Fbw6sB5;`76ESZvv#ro4{K4z7sl$at+SD8k8l
zRPsjiwD`%cqNS~=t=*B9IH<bSYlH_z2_(b*{15pun@{m@REhvaS@<_)*m?oEvdB=8
z%M?L8=6wW={hXjt+z-=}SBzF3Uo;YB7F54u{|)*qK5(2I{*ceS-UC)%^|Tq3gB8A1
zLP9sViv8v8UM(suGI%tde(B)z*4Fd3a`Lk!5{;hboSu7K+^7qhs>oHbckg!S>(gTy
z;iQ+B9uA&xKp1-TWBKNXj~<bk4|&>yepn#9^87G#L1at}Grm)TTcen0!5rjQ_uWK>
zx5dSmq2vbOMtWSzA3t_ZON%(HXcM=uw%!{y5lPb!Zq%V1u~;BRhUMkBIXZ$g^svkZ
z!bioU@d$0zh#23|&5fjxXU0AWRsWf}{<xxFSp#yPjKI;xq}2}TRn>g^;36q|>+5M2
z78bBs$Vp8@laToB>({zhuh_{18=)gfT(IBZIgi{>;b^-k>0bHy`5gtNjTPKVk*oHb
zmRXn`BVDHC<Yb4Xxe0hb9DhYaV64CvjA_<}A0D6FsDJ-A-&)$Unn^@zRrAvI;?P4Q
z#nsDo;85-0bx=Ze5KfCTKP?J=fHK3VUfpf7dp8;XfajAJfV9rx*fE>WllY<mr$MIr
zScRj<HUKtbK0IAEwhCp;NHsJuX~dh7-z6Ia0;&#KC>E|}M&F%B4lwZGm1aE>pAv{3
z(oF}gH!mRLpSh_wrsN`{le}MROG{la6%AOzCDhZ@k8XC{mU(&9FLzkpS5i}Ds;cG&
znMw5qKZ!UPFah6cur8b#woOE8sz3yr5L37|6)|Yh3rQRT(~8_g>G=swBSOzE+<>qQ
zTY1Fs<Ht#q#*UBS7t*lCrVE+rV7UX!UrzZmw@UHcgm<TFUV;@Qj->+J3w-*}$Gy-g
zFn>tD7a5Dfsm})FC04I2ZzCL#t6|};kxU**LJivsE2Z>=dcXpYi2pd1ppcN~K;0_c
zojc*P2_f4wk9LIa03%oR`6USBZogZa*GT7%qerE2l@VZ!pvIM*uSPd1DOsqk1|4%e
zd^jxV&kuVd_}i=WTOglNQBkBPGVxq0oh*n!1%5$678e&6EGFsQx%GJ4N)tu{$REFk
zpK_7ylTZX&IQNIoxaq(?$uLzyQql=)7o^^qUfMx1d&DCmB2RE&i6BPy!TtMOphAVp
z9$f`ip${HBz<wgRf=U&P^gP>AQbIx(Yg)cPyrF<MxV?Y4E3Ne2J&wP&)C7XZ>)E&S
z*vMmDr5#IDFpiDxECTbWiGh}c4Thd@zGM<6r>1zw{WOM9QLHF2!i25E)=|S3_*dHC
z)8VL;e^xAc2nHBS9IeQh7>F>KG~ezn9{7NHVvN(PTmG}ycsA(IbW#SgEB}S>x9iYC
ztGv9t2JjujAJg!&V3TFy@*P}Uq~OW$99V&A5C#uR<^{rrn}inp#aw(JUI|52O>MPJ
zS@xm)8tf&7A;XS(17Xew?%(*mu6dW^%F^WJ{?DH!{O1NDwZXqGOtTBBn!y+<QvHb^
zCqLiNnJM-zybatKgGkU)329j7D8ixVe*f^8`0fMN)(p5m?%TXf;SLB;mKOU~<P;JM
z>NJC0TM)27Vn*D!Ap{4vz0AY@BEJIl-%Du8%E!3C6)1^8a8Bb?UWeKbu6y3nq74AU
z06=d<!)zJECffSxHCiEXWJt=5D>STL3S9Z)BYU8f1wjyr8bR~7v$Jd7zAd%%l3KTJ
z0P*JV+q>6QO<`7J8?OlmpU_fzynDBQU)~;Zq6R(Sc;dvt+&}pyz{4v^No3CJc@Pcz
z`o6OfQS%=m=rnv#lqoHdtZ_~y*EBsARzDPt6IQ7PT>@a!Ao?{|7xz8=Vb!$vaQncG
zl8DI2p~B)SZF6%jSfJyWZL{Tz-+;~&SluN3VOQ5;fvr$HBx;QIRf|9X=pQ^Nj1S5z
zEIg1~UTj|-i#4nbFF=ykf_jtoDjq)3(wv2@^Zfi{7cSqqYiLY##g~_>kzoX~CS5%}
zN32qi;Y#4bcPXL<b3I`>pZV>2V^h<yyC0Ad*8?0T8u{#N;CMrL@hdpd{%#q!%t!p|
zKlNJwq_=kyN;x4B8(d@p*i#j-jWiShJ_inbQ0%f_jd~tj40TcAkr0wO_#a8{-1<55
zboi`zzL?h`I3UvTMOK42do}3-1N`qCC@V7fxM~-_?`#P_Og*fSIYFmkTlYMc8)0!o
zKP;|p#nKZ=CeyYD2?IDdzj^fBqk@Twz4}np&(zcuRIm8+5{3#j;liPkXS8OQo*z~u
z%}GF0J?w96iiW`ECpp#QfJGGqc?XIXUftLhiA6Zl4*<NbAytrHzzz;l0ZH>Pjee>;
z@O{TtbQ>QwgUJ<3xB<dMFxD(&?#d^%QdKN??d8H=Sjo?xx;pBbLQWG5#sdkL4;3>)
zmt<fy98SfA@i4Z`AA9{+4NnQHP*GUPKnP^Ew2ikJyK{Il^<cxK5l6ZMSQXaBbn+bU
z-$BxHouNO~Se0=iuO5{fGx-bGl&i)jtR96z9E3lD*xP(RIP_K)VRe3V8qN4fWG&cq
zd#=gal8d%-K|e|u2>gE&^6BVK_d|y)BtqWOqCzZ@Ya(rbjkd$7i(?lMZk{&ksssw0
zG%P`1$z=-E9tq`ej&#4TKQh*Vu!uBYFsaA|dYGN9QybRd*A_@O+|rT<5B19Q!jrrB
z>AZc#!q}U=2ZLJKwUYsSkKDuPSb(3}{`&O}>?M%DEa2|`lk&nE(;vXBMh0ex80&j`
z1<BNM_ii@q#aWJ6TP0LgY2wQ;q%~wBM_RHKz9l`%p#JAkFaTW|R23O4;u(<x6;7@r
z%oCC;U;)7qF9S&RLz0jzDtMnz1Kh)Aa)1Xd`;(e8Xla5TwRR7a);wr^Db^LqRSITi
z9ZW;AbU~0ZI~UI)@AmceCBs%2s(SF_gZJ5w07LX+R&N~tGutN)HxoX3Jxm31wJ4rU
zzp?un31@fs$B$cB+tVvm^7&;;?$g2c4uunmTPQg@c_e3#?`B4QDk}@5pMT9&<jgf#
z#e;{m!@)rmbm%hb2$v@W$2tTxk8<LGHe)N4y^DP*lAa<7*#N0PZhJjMc+vSI9eW{g
z$;iJ)0w__1W6I3UjktGj6S+}{VFlh@3+Ru3+uhz?v(G*t+@orq4(}${QM$h$MsYLT
zRE5ALU%Z_#c|uqs-|NVi-Q~Sj#Z3$KLiR5pFAMs`(EYcVm>B5@_C7gHVkL>lBvJHj
z0!m#*Vw{CLBP+_y&8<Pc0w*+tS3Z07id-><tw)b67;H3pSFW<=!mv2940@2glP6DV
z85^@C%GvE7nU=SG9E@GKI|nQGf43G?apNb*HH@QuRT|^~dEjCSzh?E5=pHc5k&zK{
zy-`5L&Dzc08ZA|V6#}f<)Z(t;pw$D<6ddcq%$xV$824@`f9=PG_<#H5pZ|kCA6X%|
Yvp?l16c4?_??Dl}^mgkOX*-1eFO%Z7M*si-

literal 0
HcmV?d00001


From 2f2a0c139a96984086359946a67082f6fb8e5efb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 16 Oct 2017 15:35:43 -0400
Subject: [PATCH 1152/1644] ARROW-1676: [C++] Only pad null bitmap up to a
 factor of 8 bytes in Feather format

cc @rvernica

Due to Arrow's buffer padding, the validity bitmap may be larger than the Feather format expects it to be. This can result in off-by-one errors when reading files.

It's disappointing that this correctness issue has existed for so long. I am looking to determine if this might have affected users of the Python API

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1204 from wesm/ARROW-1676 and squashes the following commits:

60a2ed55 [Wes McKinney] Fuzz test more array lengths
540f79d3 [Wes McKinney] Ensure proper padding
fa677cc2 [Wes McKinney] Only write necessary bytes from null bitmap in Feather writer
---
 cpp/src/arrow/ipc/feather.cc         |  6 +++---
 python/pyarrow/tests/test_feather.py | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 9d244f1154416..97ed601adb58c 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -554,12 +554,12 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     if (values.null_count() > 0) {
       // We assume there is one bit for each value in values.nulls, aligned on a
       // byte boundary, and we write this much data into the stream
+      int64_t null_bitmap_size = GetOutputLength(BitUtil::BytesForBits(values.length()));
       if (values.null_bitmap()) {
         RETURN_NOT_OK(WritePadded(stream_.get(), values.null_bitmap()->data(),
-                                  values.null_bitmap()->size(), &bytes_written));
+                                  null_bitmap_size, &bytes_written));
       } else {
-        RETURN_NOT_OK(WritePaddedBlank(
-            stream_.get(), BitUtil::BytesForBits(values.length()), &bytes_written));
+        RETURN_NOT_OK(WritePaddedBlank(stream_.get(), null_bitmap_size, &bytes_written));
       }
       meta->total_bytes += bytes_written;
     }
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index a7013ba59986d..76f0844fa4843 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -249,6 +249,26 @@ def test_boolean_nulls(self):
         result = read_feather(path)
         assert_frame_equal(result, ex_frame)
 
+    def test_buffer_bounds_error(self):
+        # ARROW-1676
+        path = random_path()
+        self.test_files.append(path)
+
+        for i in range(16, 256):
+            values = pa.array([None] + list(range(i)), type=pa.float64())
+
+            writer = FeatherWriter()
+            writer.open(path)
+
+            writer.write_array('arr', values)
+            writer.close()
+
+            result = read_feather(path)
+            expected = pd.DataFrame({'arr': values.to_pandas()})
+            assert_frame_equal(result, expected)
+
+            self._check_pandas_roundtrip(expected, null_counts=[1])
+
     def test_boolean_object_nulls(self):
         repeats = 100
         arr = np.array([False, None, True] * repeats, dtype=object)

From 1926bdc9786af8f72111b5b6520388f362bd8809 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 16 Oct 2017 15:53:58 -0400
Subject: [PATCH 1153/1644] ARROW-1613: [Java] Alternative ArrowReader close to
 free resources but leave ReadChannel open

This adds an alternative `ArrowReader.close(boolean closeReadChannel)` that if called with `false` will close reader resources such as vectors/dictionaries but leave the `ReadChannel` open. The behavior of the default `ArrowReader.close()` is unchanged.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1138 from BryanCutler/java-ArrowRead-not-close-input-ARROW-1613 and squashes the following commits:

028f2cd5 [Bryan Cutler] Added docs to ArrowReader
a9125dda [Bryan Cutler] revert test that manually closed ReadChannel
103a4192 [Bryan Cutler] changed to alternate close
b735135f [Bryan Cutler] Removed closing of ReadChannel from ArrowReader.close() and updated usage in tests to close things correctly
---
 .../org/apache/arrow/tools/EchoServer.java    |  4 +-
 .../org/apache/arrow/tools/FileRoundtrip.java |  5 +-
 .../apache/arrow/tools/EchoServerTest.java    |  4 +-
 .../apache/arrow/vector/file/ArrowReader.java | 51 +++++++++++++++++--
 .../vector/file/TestArrowReaderWriter.java    |  4 +-
 .../vector/file/TestArrowStreamPipe.java      |  1 +
 6 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
index c53f0ea86935e..3091bc4dab123 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
@@ -95,9 +95,9 @@ public ClientConnection(Socket socket) {
     }
 
     public void run() throws IOException {
-      BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
       // Read the entire input stream and write it back
-      try (ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
+      try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+           ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
         VectorSchemaRoot root = reader.getVectorSchemaRoot();
         // load the first batch before instantiating the writer so that we have any dictionaries
         reader.loadNextBatch();
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
index 7d71b0b8f9d3f..ab8fa6e45cecd 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
@@ -79,8 +79,9 @@ int run(String[] args) {
 
       File inFile = validateFile("input", inFileName);
       File outFile = validateFile("output", outFileName);
-      BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); // TODO: close
-      try (FileInputStream fileInputStream = new FileInputStream(inFile);
+
+      try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+           FileInputStream fileInputStream = new FileInputStream(inFile);
            ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(),
                allocator)) {
 
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 467965aff95d9..ecac6d6e53067 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -95,9 +95,9 @@ private void testEchoServer(int serverPort,
                               NullableTinyIntVector vector,
                               int batches)
       throws UnknownHostException, IOException {
-    BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
     VectorSchemaRoot root = new VectorSchemaRoot(asList(field), asList((FieldVector) vector), 0);
-    try (Socket socket = new Socket("localhost", serverPort);
+    try (BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
+         Socket socket = new Socket("localhost", serverPort);
          ArrowStreamWriter writer = new ArrowStreamWriter(root, null, socket.getOutputStream());
          ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), alloc)) {
       writer.start();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index 646d6feeef086..21fb2207eb019 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -41,6 +41,11 @@
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.DictionaryUtility;
 
+/**
+ * Abstract class to read ArrowRecordBatches from a ReadChannel.
+ *
+ * @param <T> Type of ReadChannel to use
+ */
 public abstract class ArrowReader<T extends ReadChannel> implements DictionaryProvider, AutoCloseable {
 
   private final T in;
@@ -58,7 +63,7 @@ protected ArrowReader(T in, BufferAllocator allocator) {
   }
 
   /**
-   * Returns the vector schema root. This will be loaded with new values on every call to loadNextBatch
+   * Returns the vector schema root. This will be loaded with new values on every call to loadNextBatch.
    *
    * @return the vector schema root
    * @throws IOException if reading of schema fails
@@ -69,9 +74,9 @@ public VectorSchemaRoot getVectorSchemaRoot() throws IOException {
   }
 
   /**
-   * Returns any dictionaries
+   * Returns any dictionaries that were loaded along with ArrowRecordBatches.
    *
-   * @return dictionaries, if any
+   * @return Map of dictionaries to dictionary id, empty if no dictionaries loaded
    * @throws IOException if reading of schema fails
    */
   public Map<Long, Dictionary> getDictionaryVectors() throws IOException {
@@ -79,6 +84,12 @@ public Map<Long, Dictionary> getDictionaryVectors() throws IOException {
     return dictionaries;
   }
 
+  /**
+   * Lookup a dictionary that has been loaded using the dictionary id.
+   *
+   * @param id Unique identifier for a dictionary
+   * @return the requested dictionary or null if not found
+   */
   @Override
   public Dictionary lookup(long id) {
     if (!initialized) {
@@ -88,7 +99,12 @@ public Dictionary lookup(long id) {
     return dictionaries.get(id);
   }
 
-  // Returns true if a batch was read, false on EOS
+  /**
+   * Load the next ArrowRecordBatch to the vector schema root if available.
+   *
+   * @return true if a batch was read, false on EOS
+   * @throws IOException
+   */
   public boolean loadNextBatch() throws IOException {
     ensureInitialized();
     // read in all dictionary batches, then stop after our first record batch
@@ -129,19 +145,44 @@ public Boolean visit(ArrowRecordBatch message) {
     return readBatch;
   }
 
+  /**
+   * Return the number of bytes read from the ReadChannel.
+   *
+   * @return number of bytes read
+   */
   public long bytesRead() {
     return in.bytesRead();
   }
 
+  /**
+   * Close resources, including vector schema root and dictionary vectors, and the
+   * underlying ReadChannel.
+   *
+   * @throws IOException
+   */
   @Override
   public void close() throws IOException {
+    close(true);
+  }
+
+  /**
+   * Close resources, including vector schema root and dictionary vectors. If the flag
+   * closeReadChannel is true then close the underlying ReadChannel, otherwise leave it open.
+   *
+   * @param closeReadChannel Flag to control if closing the underlying ReadChannel
+   * @throws IOException
+   */
+  public void close(boolean closeReadChannel) throws IOException {
     if (initialized) {
       root.close();
       for (Dictionary dictionary : dictionaries.values()) {
         dictionary.getVector().close();
       }
     }
-    in.close();
+
+    if (closeReadChannel) {
+      in.close();
+    }
   }
 
   protected abstract Schema readSchema(T in) throws IOException;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index 65332aa2c7de2..3ce01a26835d5 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -92,8 +92,8 @@ public void test() throws IOException {
 
     byte[] byteArray = out.toByteArray();
 
-    SeekableReadChannel channel = new SeekableReadChannel(new ByteArrayReadableSeekableByteChannel(byteArray));
-    try (ArrowFileReader reader = new ArrowFileReader(channel, allocator)) {
+    try (SeekableReadChannel channel = new SeekableReadChannel(new ByteArrayReadableSeekableByteChannel(byteArray));
+         ArrowFileReader reader = new ArrowFileReader(channel, allocator)) {
       Schema readSchema = reader.getVectorSchemaRoot().getSchema();
       assertEquals(schema, readSchema);
       assertTrue(readSchema.getFields().get(0).getTypeLayout().getVectorTypes().toString(), readSchema.getFields().get(0).getTypeLayout().getVectors().size() > 0);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
index a19c3795fd5bb..40716942f02a2 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
@@ -141,6 +141,7 @@ public void run() {
         while (!done) {
           assertTrue(reader.loadNextBatch());
         }
+        reader.close();
       } catch (IOException e) {
         e.printStackTrace();
         Assert.fail(e.toString()); // have to explicitly fail since we're in a separate thread

From 8eb2b0ee67ee6697ce01acb3dda881d2acc83912 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 17 Oct 2017 09:28:03 -0400
Subject: [PATCH 1154/1644] ARROW-1679: [GLib] Add
 garrow_record_batch_reader_read_next()

garrow_record_batch_reader_read_next_record_batch() is deprecated.
It's for following C++ API change.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1206 from kou/glib-record-batch-reader-follow-api-change and squashes the following commits:

a2126120 [Kouhei Sutou] [GLib] Omit tests with old GObject Introspection that doesn't support nullable
0b759082 [Kouhei Sutou] [GLib] Add garrow_record_batch_reader_read_next()
---
 c_glib/arrow-glib/reader.cpp       | 26 +++++++++++++++++++++++---
 c_glib/arrow-glib/reader.h         |  8 +++++++-
 c_glib/example/go/read-stream.go   |  4 ++--
 c_glib/example/lua/read-stream.lua |  2 +-
 c_glib/example/read-stream.c       |  5 ++---
 c_glib/test/test-array-builder.rb  |  2 ++
 c_glib/test/test-stream-writer.rb  |  4 ++--
 7 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index 1941f6435ced6..2b3424411fc08 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -168,13 +168,13 @@ garrow_record_batch_reader_get_schema(GArrowRecordBatchReader *reader)
  * Since: 0.4.0
  *
  * Deprecated: 0.5.0:
- *   Use garrow_record_batch_reader_read_next_record_batch() instead.
+ *   Use garrow_record_batch_reader_read_next() instead.
  */
 GArrowRecordBatch *
 garrow_record_batch_reader_get_next_record_batch(GArrowRecordBatchReader *reader,
                                                  GError **error)
 {
-  return garrow_record_batch_reader_read_next_record_batch(reader, error);
+  return garrow_record_batch_reader_read_next(reader, error);
 }
 
 /**
@@ -186,10 +186,30 @@ garrow_record_batch_reader_get_next_record_batch(GArrowRecordBatchReader *reader
  *   The next record batch in the stream or %NULL on end of stream.
  *
  * Since: 0.5.0
+ *
+ * Deprecated: 0.8.0:
+ *   Use garrow_record_batch_reader_read_next() instead.
  */
 GArrowRecordBatch *
 garrow_record_batch_reader_read_next_record_batch(GArrowRecordBatchReader *reader,
                                                   GError **error)
+{
+  return garrow_record_batch_reader_read_next(reader, error);
+}
+
+/**
+ * garrow_record_batch_reader_read_next:
+ * @reader: A #GArrowRecordBatchReader.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   The next record batch in the stream or %NULL on end of stream.
+ *
+ * Since: 0.8.0
+ */
+GArrowRecordBatch *
+garrow_record_batch_reader_read_next(GArrowRecordBatchReader *reader,
+                                     GError **error)
 {
   auto arrow_reader = garrow_record_batch_reader_get_raw(reader);
   std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
@@ -197,7 +217,7 @@ garrow_record_batch_reader_read_next_record_batch(GArrowRecordBatchReader *reade
 
   if (garrow_error_check(error,
                          status,
-                         "[record-batch-reader][read-next-record-batch]")) {
+                         "[record-batch-reader][read-next]")) {
     if (arrow_record_batch == nullptr) {
       return NULL;
     } else {
diff --git a/c_glib/arrow-glib/reader.h b/c_glib/arrow-glib/reader.h
index f5195e4a434f6..6b746d1856339 100644
--- a/c_glib/arrow-glib/reader.h
+++ b/c_glib/arrow-glib/reader.h
@@ -77,14 +77,20 @@ GType garrow_record_batch_reader_get_type(void) G_GNUC_CONST;
 GArrowSchema *garrow_record_batch_reader_get_schema(
   GArrowRecordBatchReader *reader);
 #ifndef GARROW_DISABLE_DEPRECATED
-G_GNUC_DEPRECATED_FOR(garrow_record_batch_reader_read_next_record_batch)
+G_GNUC_DEPRECATED_FOR(garrow_record_batch_reader_read_next)
 GArrowRecordBatch *garrow_record_batch_reader_get_next_record_batch(
   GArrowRecordBatchReader *reader,
   GError **error);
 #endif
+#ifndef GARROW_DISABLE_DEPRECATED
+G_GNUC_DEPRECATED_FOR(garrow_record_batch_reader_read_next)
 GArrowRecordBatch *garrow_record_batch_reader_read_next_record_batch(
   GArrowRecordBatchReader *reader,
   GError **error);
+#endif
+GArrowRecordBatch *garrow_record_batch_reader_read_next(
+  GArrowRecordBatchReader *reader,
+  GError **error);
 
 
 #define GARROW_TYPE_RECORD_BATCH_STREAM_READER          \
diff --git a/c_glib/example/go/read-stream.go b/c_glib/example/go/read-stream.go
index fdb7ba1b164d3..7bd076473f667 100644
--- a/c_glib/example/go/read-stream.go
+++ b/c_glib/example/go/read-stream.go
@@ -87,9 +87,9 @@ func main() {
 		log.Fatalf("Failed to parse data: %v", err)
 	}
 	for i := 0; true; i++ {
-		recordBatch, err := reader.ReadNextRecordBatch()
+		recordBatch, err := reader.ReadNext()
 		if err != nil {
-			log.Fatalf("Failed to read next record batch: %v", err)
+			log.Fatalf("Failed to read the next record batch: %v", err)
 		}
 		if recordBatch == nil {
 			break
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index 7dc9617d05a61..e67acf506ff9e 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -25,7 +25,7 @@ local reader = Arrow.RecordBatchStreamReader.new(input)
 
 local i = 0
 while true do
-   local record_batch = reader:read_next_record_batch()
+   local record_batch = reader:read_next()
    if not record_batch then
       break
    end
diff --git a/c_glib/example/read-stream.c b/c_glib/example/read-stream.c
index a624f72949118..92411bcc78078 100644
--- a/c_glib/example/read-stream.c
+++ b/c_glib/example/read-stream.c
@@ -117,10 +117,9 @@ main(int argc, char **argv)
     while (TRUE) {
       GArrowRecordBatch *record_batch;
 
-      record_batch =
-        garrow_record_batch_reader_read_next_record_batch(reader, &error);
+      record_batch = garrow_record_batch_reader_read_next(reader, &error);
       if (error) {
-        g_print("failed to get record batch: %s\n", error->message);
+        g_print("failed to read the next record batch: %s\n", error->message);
         g_error_free(error);
         g_object_unref(reader);
         g_object_unref(input);
diff --git a/c_glib/test/test-array-builder.rb b/c_glib/test/test-array-builder.rb
index 0c47b0a75f488..92976a424ccad 100644
--- a/c_glib/test/test-array-builder.rb
+++ b/c_glib/test/test-array-builder.rb
@@ -17,6 +17,7 @@
 
 module ArrayBuilderAppendValuesTests
   def test_empty
+    require_gi(1, 42, 0)
     builder = create_builder
     builder.append_values([])
     assert_equal(build_array([]),
@@ -24,6 +25,7 @@ def test_empty
   end
 
   def test_values_only
+    require_gi(1, 42, 0)
     builder = create_builder
     builder.append_values(sample_values)
     assert_equal(build_array(sample_values),
diff --git a/c_glib/test/test-stream-writer.rb b/c_glib/test/test-stream-writer.rb
index 833e08b145a52..c3d0e1490cef6 100644
--- a/c_glib/test/test-stream-writer.rb
+++ b/c_glib/test/test-stream-writer.rb
@@ -44,8 +44,8 @@ def test_write_record_batch
       assert_equal(["enabled"],
                    stream_reader.schema.fields.collect(&:name))
       assert_equal(true,
-                   stream_reader.read_next_record_batch.get_column(0).get_value(0))
-      assert_nil(stream_reader.read_next_record_batch)
+                   stream_reader.read_next.get_column(0).get_value(0))
+      assert_nil(stream_reader.read_next)
     ensure
       input.close
     end

From a04301833f585a66fa371394c9ee33a4a59c2ed8 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Tue, 17 Oct 2017 19:21:05 -0400
Subject: [PATCH 1155/1644] ARROW-1678: [Python] Implement numpy.float16 SerDe

This is the patch for [ARROW-1678](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1678).

Author: Licht-T <licht-t@outlook.jp>

Closes #1205 from Licht-T/feature-halffloat-python-serde and squashes the following commits:

e397cc90 [Licht-T] Add NPY_HALF case into SerializeArray method
22c37d76 [Licht-T] Use npy_half as type of the numpy float16 method return value
319abb7c [Licht-T] Add RETURN_IF_PYERROR after PyArrayScalar_New
eec4e0a2 [Licht-T] Change headers include order
5a985b8b [Licht-T] Fix lint issue
f19e5431 [Licht-T] TST: Add tests for numpy.float16 SerDe
66d5f15a [Licht-T] ENH: Set VISIT_INLINE for HalfFloat
834357e7 [Licht-T] ENH: Add numpy.float16 deserializer
02dc77b7 [Licht-T] ENH: Add numpy.float16 serializer
9d19e086 [Licht-T] ENH: Add HalfFloat type case for numpy
dda98b9e [Licht-T] ENH: Add HalfFloat type traits
0be305a3 [Licht-T] ENH: Add the IPC internal metadata for HalfFloat
7348cc67 [Licht-T] ENH: Add HalfFloat BUILDER_CASE
---
 cpp/src/arrow/builder.cc                   |  1 +
 cpp/src/arrow/ipc/metadata-internal.cc     |  4 ++++
 cpp/src/arrow/python/arrow_to_python.cc    | 13 +++++++++++++
 cpp/src/arrow/python/numpy_convert.cc      |  1 +
 cpp/src/arrow/python/numpy_to_arrow.cc     |  3 +++
 cpp/src/arrow/python/python_to_arrow.cc    | 15 +++++++++++++++
 cpp/src/arrow/python/type_traits.h         | 21 +++++++++++++++++++++
 cpp/src/arrow/visitor_inline.h             |  2 ++
 python/pyarrow/tests/test_serialization.py |  6 +++---
 9 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 076c156b0af5d..331de2d36e130 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -1438,6 +1438,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
       BUILDER_CASE(TIME64, Time64Builder);
       BUILDER_CASE(TIMESTAMP, TimestampBuilder);
       BUILDER_CASE(BOOL, BooleanBuilder);
+      BUILDER_CASE(HALF_FLOAT, HalfFloatBuilder);
       BUILDER_CASE(FLOAT, FloatBuilder);
       BUILDER_CASE(DOUBLE, DoubleBuilder);
       BUILDER_CASE(STRING, StringBuilder);
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 162afb94bfc02..ad00cfb6c09be 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -381,6 +381,10 @@ static Status TypeToFlatbuffer(FBB& fbb, const DataType& type,
       INT_TO_FB_CASE(64, false);
     case Type::INT64:
       INT_TO_FB_CASE(64, true);
+    case Type::HALF_FLOAT:
+      *out_type = flatbuf::Type_FloatingPoint;
+      *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_HALF);
+      break;
     case Type::FLOAT:
       *out_type = flatbuf::Type_FloatingPoint;
       *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_SINGLE);
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index ac459d4228710..761d2903aada9 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -17,10 +17,15 @@
 
 #include "arrow/python/arrow_to_python.h"
 
+#include "arrow/python/numpy_interop.h"
+
 #include <cstdint>
 #include <memory>
 #include <vector>
 
+#include <numpy/arrayobject.h>
+#include <numpy/arrayscalars.h>
+
 #include "arrow/array.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/ipc/reader.h"
@@ -121,6 +126,14 @@ Status GetValue(PyObject* context, const Array& arr, int64_t index, int32_t type
       *result = PyUnicode_FromStringAndSize(reinterpret_cast<const char*>(str), nchars);
       return CheckPyError();
     }
+    case Type::HALF_FLOAT: {
+      *result = PyArrayScalar_New(Half);
+      RETURN_IF_PYERROR();
+
+      npy_half halffloat = static_cast<const HalfFloatArray&>(arr).Value(index);
+      PyArrayScalar_ASSIGN(*result, Half, halffloat);
+      return Status::OK();
+    }
     case Type::FLOAT:
       *result = PyFloat_FromDouble(static_cast<const FloatArray&>(arr).Value(index));
       return Status::OK();
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index bef80542ab0ba..9ed2d73d42b57 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -165,6 +165,7 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
 #if (NPY_UINT64 != NPY_ULONGLONG)
     TO_ARROW_CASE(ULONGLONG);
 #endif
+    TO_ARROW_TYPE_CASE(FLOAT16, float16);
     TO_ARROW_TYPE_CASE(FLOAT32, float32);
     TO_ARROW_TYPE_CASE(FLOAT64, float64);
     TO_ARROW_TYPE_CASE(STRING, binary);
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index e935b45b5e8d2..27ee2302e99bd 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -284,6 +284,8 @@ class NumPyConverter {
     return VisitNative<T>();
   }
 
+  Status Visit(const HalfFloatType& type) { return VisitNative<UInt16Type>(); }
+
   Status Visit(const Date32Type& type) { return VisitNative<Date32Type>(); }
   Status Visit(const Date64Type& type) { return VisitNative<Int64Type>(); }
   Status Visit(const TimestampType& type) { return VisitNative<TimestampType>(); }
@@ -1183,6 +1185,7 @@ Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type,
     LIST_CASE(UINT64, NPY_UINT64, UInt64Type)
     LIST_CASE(INT64, NPY_INT64, Int64Type)
     LIST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
+    LIST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
     LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
     LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
     LIST_CASE(STRING, NPY_OBJECT, StringType)
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index e2d7452140eb6..ab444f280f6ec 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -58,6 +58,7 @@ class SequenceBuilder {
         ints_(::arrow::int64(), pool),
         bytes_(::arrow::binary(), pool),
         strings_(pool),
+        half_floats_(::arrow::float16(), pool),
         floats_(::arrow::float32(), pool),
         doubles_(::arrow::float64(), pool),
         date64s_(::arrow::date64(), pool),
@@ -117,6 +118,11 @@ class SequenceBuilder {
     return strings_.Append(data, length);
   }
 
+  /// Appending a half_float to the sequence
+  Status AppendHalfFloat(const npy_half data) {
+    return AppendPrimitive(data, &half_float_tag_, &half_floats_);
+  }
+
   /// Appending a float to the sequence
   Status AppendFloat(const float data) {
     return AppendPrimitive(data, &float_tag_, &floats_);
@@ -222,6 +228,7 @@ class SequenceBuilder {
     RETURN_NOT_OK(AddElement(int_tag_, &ints_));
     RETURN_NOT_OK(AddElement(string_tag_, &strings_));
     RETURN_NOT_OK(AddElement(bytes_tag_, &bytes_));
+    RETURN_NOT_OK(AddElement(half_float_tag_, &half_floats_));
     RETURN_NOT_OK(AddElement(float_tag_, &floats_));
     RETURN_NOT_OK(AddElement(double_tag_, &doubles_));
     RETURN_NOT_OK(AddElement(date64_tag_, &date64s_));
@@ -250,6 +257,7 @@ class SequenceBuilder {
   Int64Builder ints_;
   BinaryBuilder bytes_;
   StringBuilder strings_;
+  HalfFloatBuilder half_floats_;
   FloatBuilder floats_;
   DoubleBuilder doubles_;
   Date64Builder date64s_;
@@ -274,6 +282,7 @@ class SequenceBuilder {
   int8_t int_tag_ = -1;
   int8_t string_tag_ = -1;
   int8_t bytes_tag_ = -1;
+  int8_t half_float_tag_ = -1;
   int8_t float_tag_ = -1;
   int8_t double_tag_ = -1;
   int8_t date64_tag_ = -1;
@@ -394,6 +403,8 @@ Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
 Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
   if (PyArray_IsScalar(obj, Bool)) {
     return builder->AppendBool(reinterpret_cast<PyBoolScalarObject*>(obj)->obval != 0);
+  } else if (PyArray_IsScalar(obj, Half)) {
+    return builder->AppendHalfFloat(reinterpret_cast<PyHalfScalarObject*>(obj)->obval);
   } else if (PyArray_IsScalar(obj, Float)) {
     return builder->AppendFloat(reinterpret_cast<PyFloatScalarObject*>(obj)->obval);
   } else if (PyArray_IsScalar(obj, Double)) {
@@ -437,6 +448,9 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
   // The bool case must precede the int case (PyInt_Check passes for bools)
   if (PyBool_Check(elem)) {
     RETURN_NOT_OK(builder->AppendBool(elem == Py_True));
+  } else if (PyArray_DescrFromScalar(elem)->type_num == NPY_HALF) {
+    npy_half halffloat = reinterpret_cast<PyHalfScalarObject *>(elem)->obval;
+    RETURN_NOT_OK(builder->AppendHalfFloat(halffloat));
   } else if (PyFloat_Check(elem)) {
     RETURN_NOT_OK(builder->AppendDouble(PyFloat_AS_DOUBLE(elem)));
   } else if (PyLong_Check(elem)) {
@@ -523,6 +537,7 @@ Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder*
     case NPY_INT32:
     case NPY_UINT64:
     case NPY_INT64:
+    case NPY_HALF:
     case NPY_FLOAT:
     case NPY_DOUBLE: {
       RETURN_NOT_OK(builder->AppendTensor(static_cast<int32_t>(tensors_out->size())));
diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h
index 2cbbdf4cf1519..587b27c9a6039 100644
--- a/cpp/src/arrow/python/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -24,6 +24,8 @@
 
 #include "arrow/python/numpy_interop.h"
 
+#include <numpy/halffloat.h>
+
 #include "arrow/builder.h"
 #include "arrow/type.h"
 #include "arrow/util/logging.h"
@@ -71,6 +73,17 @@ NPY_INT_DECL(LONGLONG, Int64, int64_t);
 NPY_INT_DECL(ULONGLONG, UInt64, uint64_t);
 #endif
 
+template <>
+struct npy_traits<NPY_FLOAT16> {
+  typedef npy_half value_type;
+  using TypeClass = HalfFloatType;
+  using BuilderClass = HalfFloatBuilder;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(npy_half v) { return v == NPY_HALF_NAN; }
+};
+
 template <>
 struct npy_traits<NPY_FLOAT32> {
   typedef float value_type;
@@ -143,6 +156,14 @@ INT_DECL(UINT16);
 INT_DECL(UINT32);
 INT_DECL(UINT64);
 
+template <>
+struct arrow_traits<Type::HALF_FLOAT> {
+  static constexpr int npy_type = NPY_FLOAT16;
+  static constexpr bool supports_nulls = true;
+  static constexpr uint16_t na_value = NPY_HALF_NAN;
+  typedef typename npy_traits<NPY_FLOAT16>::value_type T;
+};
+
 template <>
 struct arrow_traits<Type::FLOAT> {
   static constexpr int npy_type = NPY_FLOAT32;
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index 54f9e880b834d..5ecabd2a4fbb1 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -44,6 +44,7 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
     TYPE_VISIT_INLINE(UInt32Type);
     TYPE_VISIT_INLINE(Int64Type);
     TYPE_VISIT_INLINE(UInt64Type);
+    TYPE_VISIT_INLINE(HalfFloatType);
     TYPE_VISIT_INLINE(FloatType);
     TYPE_VISIT_INLINE(DoubleType);
     TYPE_VISIT_INLINE(StringType);
@@ -85,6 +86,7 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
     ARRAY_VISIT_INLINE(UInt32Type);
     ARRAY_VISIT_INLINE(Int64Type);
     ARRAY_VISIT_INLINE(UInt64Type);
+    ARRAY_VISIT_INLINE(HalfFloatType);
     ARRAY_VISIT_INLINE(FloatType);
     ARRAY_VISIT_INLINE(DoubleType);
     ARRAY_VISIT_INLINE(StringType);
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index a9fd10243a611..7e8060b71d45e 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -99,7 +99,7 @@ def assert_equal(obj1, obj2):
     {True: "hello", False: "world"}, {"hello": "world", 1: 42, 2.5: 45},
     {"hello": set([2, 3]), "world": set([42.0]), "this": None},
     np.int8(3), np.int32(4), np.int64(5),
-    np.uint8(3), np.uint32(4), np.uint64(5), np.float32(1.9),
+    np.uint8(3), np.uint32(4), np.uint64(5), np.float16(1.9), np.float32(1.9),
     np.float64(1.9), np.zeros([100, 100]),
     np.random.normal(size=[100, 100]), np.array(["hi", 3]),
     np.array(["hi", 3], dtype=object),
@@ -257,8 +257,8 @@ def test_default_dict_serialization(large_memory_map):
 
 def test_numpy_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        for t in ["int8", "uint8", "int16", "uint16",
-                  "int32", "uint32", "float32", "float64"]:
+        for t in ["int8", "uint8", "int16", "uint16", "int32", "uint32",
+                  "float16", "float32", "float64"]:
             obj = np.random.randint(0, 10, size=(100, 100)).astype(t)
             serialization_roundtrip(obj, mmap)
 

From a4813bdd1d0f52b5c4d1769b914ee9ad24d2223a Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 18 Oct 2017 17:03:26 -0400
Subject: [PATCH 1156/1644] ARROW-1685: [GLib] Add GArrowTableBatchReader

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1210 from kou/glib-add-table-batch-reader and squashes the following commits:

249afb27 [Kouhei Sutou] [GLib] Add GArrowTableBatchReader
---
 c_glib/arrow-glib/reader.cpp           | 43 ++++++++++++++++++
 c_glib/arrow-glib/reader.h             | 63 ++++++++------------------
 c_glib/arrow-glib/reader.hpp           |  2 +
 c_glib/test/helper/buildable.rb        | 20 ++++++++
 c_glib/test/test-table-batch-reader.rb | 42 +++++++++++++++++
 5 files changed, 125 insertions(+), 45 deletions(-)
 create mode 100644 c_glib/test/test-table-batch-reader.rb

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
index 2b3424411fc08..f08e8f8bc25cb 100644
--- a/c_glib/arrow-glib/reader.cpp
+++ b/c_glib/arrow-glib/reader.cpp
@@ -25,6 +25,7 @@
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
+#include <arrow-glib/table.hpp>
 
 #include <arrow-glib/input-stream.hpp>
 
@@ -229,6 +230,38 @@ garrow_record_batch_reader_read_next(GArrowRecordBatchReader *reader,
 }
 
 
+G_DEFINE_TYPE(GArrowTableBatchReader,
+              garrow_table_batch_reader,
+              GARROW_TYPE_RECORD_BATCH_READER);
+
+static void
+garrow_table_batch_reader_init(GArrowTableBatchReader *object)
+{
+}
+
+static void
+garrow_table_batch_reader_class_init(GArrowTableBatchReaderClass *klass)
+{
+}
+
+/**
+ * garrow_table_batch_reader_new:
+ * @table: The table to be read.
+ *
+ * Returns: A newly created #GArrowTableBatchReader.
+ *
+ * Since: 0.8.0
+ */
+GArrowTableBatchReader *
+garrow_table_batch_reader_new(GArrowTable *table)
+{
+  auto arrow_table = garrow_table_get_raw(table);
+  auto arrow_table_batch_reader =
+    std::make_shared<arrow::TableBatchReader>(*arrow_table);
+  return garrow_table_batch_reader_new_raw(&arrow_table_batch_reader);
+}
+
+
 G_DEFINE_TYPE(GArrowRecordBatchStreamReader,
               garrow_record_batch_stream_reader,
               GARROW_TYPE_RECORD_BATCH_READER);
@@ -790,6 +823,16 @@ garrow_record_batch_reader_get_raw(GArrowRecordBatchReader *reader)
   return priv->record_batch_reader;
 }
 
+GArrowTableBatchReader *
+garrow_table_batch_reader_new_raw(std::shared_ptr<arrow::TableBatchReader> *arrow_reader)
+{
+  auto reader =
+    GARROW_TABLE_BATCH_READER(g_object_new(GARROW_TYPE_TABLE_BATCH_READER,
+                                           "record-batch-reader", arrow_reader,
+                                           NULL));
+  return reader;
+}
+
 GArrowRecordBatchStreamReader *
 garrow_record_batch_stream_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *arrow_reader)
 {
diff --git a/c_glib/arrow-glib/reader.h b/c_glib/arrow-glib/reader.h
index 6b746d1856339..964146ab8d721 100644
--- a/c_glib/arrow-glib/reader.h
+++ b/c_glib/arrow-glib/reader.h
@@ -19,8 +19,10 @@
 
 #pragma once
 
+#include <arrow-glib/gobject-type.h>
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
+#include <arrow-glib/table.h>
 
 #include <arrow-glib/input-stream.h>
 
@@ -28,51 +30,12 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_RECORD_BATCH_READER         \
-  (garrow_record_batch_reader_get_type())
-#define GARROW_RECORD_BATCH_READER(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_RECORD_BATCH_READER,  \
-                              GArrowRecordBatchReader))
-#define GARROW_RECORD_BATCH_READER_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_RECORD_BATCH_READER,     \
-                           GArrowRecordBatchReaderClass))
-#define GARROW_IS_RECORD_BATCH_READER(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_RECORD_BATCH_READER))
-#define GARROW_IS_RECORD_BATCH_READER_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_RECORD_BATCH_READER))
-#define GARROW_RECORD_BATCH_READER_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_RECORD_BATCH_READER,   \
-                             GArrowRecordBatchReaderClass))
-
-typedef struct _GArrowRecordBatchReader      GArrowRecordBatchReader;
-#ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowRecordBatchReaderClass GArrowRecordBatchReaderClass;
-#endif
-
-/**
- * GArrowRecordBatchReader:
- *
- * It wraps `arrow::ipc::RecordBatchReader`.
- */
-struct _GArrowRecordBatchReader
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-#ifndef __GTK_DOC_IGNORE__
-struct _GArrowRecordBatchReaderClass
-{
-  GObjectClass parent_class;
-};
-#endif
-
-GType garrow_record_batch_reader_get_type(void) G_GNUC_CONST;
+#define GARROW_TYPE_RECORD_BATCH_READER (garrow_record_batch_reader_get_type())
+GARROW_DECLARE_TYPE(GArrowRecordBatchReader,
+                    garrow_record_batch_reader,
+                    GARROW,
+                    RECORD_BATCH_READER,
+                    GObject)
 
 GArrowSchema *garrow_record_batch_reader_get_schema(
   GArrowRecordBatchReader *reader);
@@ -93,6 +56,16 @@ GArrowRecordBatch *garrow_record_batch_reader_read_next(
   GError **error);
 
 
+#define GARROW_TYPE_TABLE_BATCH_READER (garrow_table_batch_reader_get_type())
+GARROW_DECLARE_TYPE(GArrowTableBatchReader,
+                    garrow_table_batch_reader,
+                    GARROW,
+                    TABLE_BATCH_READER,
+                    GArrowRecordBatchReader)
+
+GArrowTableBatchReader *garrow_table_batch_reader_new(GArrowTable *table);
+
+
 #define GARROW_TYPE_RECORD_BATCH_STREAM_READER          \
   (garrow_record_batch_stream_reader_get_type())
 #define GARROW_RECORD_BATCH_STREAM_READER(obj)                          \
diff --git a/c_glib/arrow-glib/reader.hpp b/c_glib/arrow-glib/reader.hpp
index 7970e731deb01..3e1135e137715 100644
--- a/c_glib/arrow-glib/reader.hpp
+++ b/c_glib/arrow-glib/reader.hpp
@@ -28,6 +28,8 @@
 GArrowRecordBatchReader *garrow_record_batch_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchReader> *arrow_reader);
 std::shared_ptr<arrow::ipc::RecordBatchReader> garrow_record_batch_reader_get_raw(GArrowRecordBatchReader *reader);
 
+GArrowTableBatchReader *garrow_table_batch_reader_new_raw(std::shared_ptr<arrow::TableBatchReader> *arrow_reader);
+
 GArrowRecordBatchStreamReader *garrow_record_batch_stream_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchStreamReader> *arrow_reader);
 
 GArrowRecordBatchFileReader *garrow_record_batch_file_reader_new_raw(std::shared_ptr<arrow::ipc::RecordBatchFileReader> *arrow_reader);
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index bdb7e11caa380..0f56fecd0fa51 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -145,6 +145,26 @@ def build_struct_array(fields, structs)
       builder.finish
     end
 
+    def build_table(arrays)
+      fields = arrays.collect do |name, array|
+        Arrow::Field.new(name, array.value_data_type)
+      end
+      schema = Arrow::Schema.new(fields)
+      columns = arrays.collect.with_index do |(_name, array), i|
+        Arrow::Column.new(fields[i], array)
+      end
+      Arrow::Table.new(schema, columns)
+    end
+
+    def build_record_batch(arrays)
+      n_rows = arrays.collect {|_, array| array.length}.min || 0
+      fields = arrays.collect do |name, array|
+        Arrow::Field.new(name, array.value_data_type)
+      end
+      schema = Arrow::Schema.new(fields)
+      Arrow::RecordBatch.new(schema, n_rows, arrays.values)
+    end
+
     private
     def build_array(builder, values)
       values.each do |value|
diff --git a/c_glib/test/test-table-batch-reader.rb b/c_glib/test/test-table-batch-reader.rb
new file mode 100644
index 0000000000000..b161c8a558589
--- /dev/null
+++ b/c_glib/test/test-table-batch-reader.rb
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestTableBatchReader < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def test_empty
+    table = build_table("visible" => build_boolean_array([]))
+    reader = Arrow::TableBatchReader.new(table)
+    assert_nil(reader.read_next)
+  end
+
+  def test_have_record
+    array = build_boolean_array([true])
+    table = build_table("visible" => array)
+    reader = Arrow::TableBatchReader.new(table)
+    assert_equal(build_record_batch("visible" => array),
+                 reader.read_next)
+    assert_nil(reader.read_next)
+  end
+
+  def test_schema
+    array = build_boolean_array([])
+    table = build_table("visible" => array)
+    reader = Arrow::TableBatchReader.new(table)
+    assert_equal(table.schema, reader.schema)
+  end
+end

From 298e343240173e0a326495154f196565adc8d301 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Wed, 18 Oct 2017 17:07:41 -0400
Subject: [PATCH 1157/1644] ARROW-1673: [Python] Add support for numpy 'bool'
 type

This is currently a workaround until the Arrow tensor supports zero copy of byte-length booleans.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1199 from pcmoritz/ndarray-bool and squashes the following commits:

14943a09 [Philipp Moritz] deploy workaround
8fce724c [Philipp Moritz] update
ad4c6b97 [Philipp Moritz] change bool width to 1 byte
11c7ed3b [Philipp Moritz] add support for numpy 'bool' type
---
 cpp/src/arrow/python/python_to_arrow.cc    | 1 -
 python/pyarrow/tests/test_serialization.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index ab444f280f6ec..47d48d7a11cf5 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -528,7 +528,6 @@ Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder*
                       std::vector<PyObject*>* tensors_out) {
   int dtype = PyArray_TYPE(array);
   switch (dtype) {
-    case NPY_BOOL:
     case NPY_UINT8:
     case NPY_INT8:
     case NPY_UINT16:
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 7e8060b71d45e..460a11ba2f853 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -257,7 +257,7 @@ def test_default_dict_serialization(large_memory_map):
 
 def test_numpy_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        for t in ["int8", "uint8", "int16", "uint16", "int32", "uint32",
+        for t in ["bool", "int8", "uint8", "int16", "uint16", "int32", "uint32",
                   "float16", "float32", "float64"]:
             obj = np.random.randint(0, 10, size=(100, 100)).astype(t)
             serialization_roundtrip(obj, mmap)

From d7bf5f2807ba1ffed249928c3d94e2627e7fb8de Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 19 Oct 2017 15:04:01 -0400
Subject: [PATCH 1158/1644] ARROW-1690: [GLib] Add garrow_array_is_valid()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1212 from kou/glib-add-is-valid and squashes the following commits:

7a4b460e [Kouhei Sutou] [GLib] Add garrow_array_is_valid()
---
 c_glib/arrow-glib/array.cpp | 16 ++++++++++++++++
 c_glib/arrow-glib/array.h   |  2 ++
 c_glib/test/test-array.rb   |  9 +++++++++
 3 files changed, 27 insertions(+)

diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/array.cpp
index 1bdbb0c0da9c9..4945ca3cfe903 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/array.cpp
@@ -367,6 +367,22 @@ garrow_array_is_null(GArrowArray *array, gint64 i)
   return arrow_array->IsNull(i);
 }
 
+/**
+ * garrow_array_is_valid:
+ * @array: A #GArrowArray.
+ * @i: The index of the target value.
+ *
+ * Returns: Whether the i-th value is valid (not null) or not.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_array_is_valid(GArrowArray *array, gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  return arrow_array->IsValid(i);
+}
+
 /**
  * garrow_array_get_length:
  * @array: A #GArrowArray.
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index 8a611225112e5..e988a8aae931d 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -71,6 +71,8 @@ gboolean       garrow_array_equal_range (GArrowArray *array,
 
 gboolean       garrow_array_is_null     (GArrowArray *array,
                                          gint64 i);
+gboolean       garrow_array_is_valid    (GArrowArray *array,
+                                         gint64 i);
 gint64         garrow_array_get_length  (GArrowArray *array);
 gint64         garrow_array_get_offset  (GArrowArray *array);
 gint64         garrow_array_get_n_nulls (GArrowArray *array);
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
index bd0c7d6ecda09..d0ade2f1e7ea2 100644
--- a/c_glib/test/test-array.rb
+++ b/c_glib/test/test-array.rb
@@ -48,6 +48,15 @@ def test_is_null
                  array.length.times.collect {|i| array.null?(i)})
   end
 
+  def test_is_valid
+    builder = Arrow::BooleanArrayBuilder.new
+    builder.append_null
+    builder.append(true)
+    array = builder.finish
+    assert_equal([false, true],
+                 array.length.times.collect {|i| array.valid?(i)})
+  end
+
   def test_length
     builder = Arrow::BooleanArrayBuilder.new
     builder.append(true)

From a8f518588fda471b2e3cc8e0f0064e7c4bb99899 Mon Sep 17 00:00:00 2001
From: Wataru Shimizu <waruzilla@gmail.com>
Date: Fri, 20 Oct 2017 09:42:08 -0400
Subject: [PATCH 1159/1644] ARROW-1666: [GLib] Enable gtk-doc on Travis CI Mac
 environment

The current build script for Travis CI (`ci/travis_before_script_c_glib.sh`) does not use `--enable-gtk-doc` option on Mac. Now it can be used also on Mac, so it should be enabled to check build status.

Author: Wataru Shimizu <waruzilla@gmail.com>

Closes #1195 from wagavulin/ci-mac-enable-gtk-doc and squashes the following commits:

1b93aa7f [Wataru Shimizu] Add environment variable for xsltproc on macOS
675c9101 [Wataru Shimizu] Enable gtk-doc on Travis CI Mac environment
---
 ci/travis_before_script_c_glib.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index d60f6296cc46c..01ef9e8e76e7f 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -25,6 +25,7 @@ if [ $TRAVIS_OS_NAME = "osx" ]; then
   brew update && brew bundle --file=c_glib/Brewfile
 
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
+  export XML_CATALOG_FILES=/usr/local/etc/xml/catalog
 fi
 
 if [ $BUILD_SYSTEM = "meson" ]; then
@@ -83,10 +84,7 @@ if [ $BUILD_SYSTEM = "autotools" ]; then
   ./autogen.sh
 
   CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"
-  if [ $TRAVIS_OS_NAME != "osx" ]; then
-    CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"
-  fi
-
+  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"
   CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CFLAGS=-DARROW_NO_DEPRECATED_API"
   CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API"
 

From 971e99dde3ebabcd6791f4e936c0273938c45893 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 20 Oct 2017 14:27:38 -0700
Subject: [PATCH 1160/1644] ARROW-1695: [Serialization] Fix reference counting
 of numpy arrays created in custom serializer

This uses the NumPyBuffer built into Arrow's Tensor facility to protect the numpy arrays holding the Tensors to be serialized. See also the problem description in https://issues.apache.org/jira/browse/ARROW-1695.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1220 from pcmoritz/fix-serialize-tensors and squashes the following commits:

7e23bb5 [Philipp Moritz] fix linting
dce92ad [Philipp Moritz] fix handling of numpy arrays generated in the custom serializer methods
---
 cpp/src/arrow/python/python_to_arrow.cc    | 27 ++++++++++------------
 python/pyarrow/tests/test_serialization.py | 21 +++++++++++++++++
 2 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 47d48d7a11cf5..a46d10d446084 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -390,15 +390,15 @@ Status CallDeserializeCallback(PyObject* context, PyObject* value,
 
 Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
                      int32_t recursion_depth, std::shared_ptr<Array>* out,
-                     std::vector<PyObject*>* tensors_out);
+                     std::vector<std::shared_ptr<Tensor>>* tensors_out);
 
 Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder,
                       std::vector<PyObject*>* subdicts,
-                      std::vector<PyObject*>* tensors_out);
+                      std::vector<std::shared_ptr<Tensor>>* tensors_out);
 
 Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
                           int32_t recursion_depth, std::shared_ptr<Array>* out,
-                          std::vector<PyObject*>* tensors_out);
+                          std::vector<std::shared_ptr<Tensor>>* tensors_out);
 
 Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
   if (PyArray_IsScalar(obj, Bool)) {
@@ -444,7 +444,7 @@ Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
 Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
               std::vector<PyObject*>* sublists, std::vector<PyObject*>* subtuples,
               std::vector<PyObject*>* subdicts, std::vector<PyObject*>* subsets,
-              std::vector<PyObject*>* tensors_out) {
+              std::vector<std::shared_ptr<Tensor>>* tensors_out) {
   // The bool case must precede the int case (PyInt_Check passes for bools)
   if (PyBool_Check(elem)) {
     RETURN_NOT_OK(builder->AppendBool(elem == Py_True));
@@ -525,7 +525,7 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
 
 Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder,
                       std::vector<PyObject*>* subdicts,
-                      std::vector<PyObject*>* tensors_out) {
+                      std::vector<std::shared_ptr<Tensor>>* tensors_out) {
   int dtype = PyArray_TYPE(array);
   switch (dtype) {
     case NPY_UINT8:
@@ -540,7 +540,10 @@ Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder*
     case NPY_FLOAT:
     case NPY_DOUBLE: {
       RETURN_NOT_OK(builder->AppendTensor(static_cast<int32_t>(tensors_out->size())));
-      tensors_out->push_back(reinterpret_cast<PyObject*>(array));
+      std::shared_ptr<Tensor> tensor;
+      RETURN_NOT_OK(NdarrayToTensor(default_memory_pool(),
+                                    reinterpret_cast<PyObject*>(array), &tensor));
+      tensors_out->push_back(tensor);
     } break;
     default: {
       PyObject* serialized_object;
@@ -556,7 +559,7 @@ Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder*
 
 Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
                           int32_t recursion_depth, std::shared_ptr<Array>* out,
-                          std::vector<PyObject*>* tensors_out) {
+                          std::vector<std::shared_ptr<Tensor>>* tensors_out) {
   DCHECK(out);
   if (recursion_depth >= kMaxRecursionDepth) {
     return Status::NotImplemented(
@@ -603,7 +606,7 @@ Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
 
 Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
                      int32_t recursion_depth, std::shared_ptr<Array>* out,
-                     std::vector<PyObject*>* tensors_out) {
+                     std::vector<std::shared_ptr<Tensor>>* tensors_out) {
   DictBuilder result;
   if (recursion_depth >= kMaxRecursionDepth) {
     return Status::NotImplemented(
@@ -686,14 +689,8 @@ Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject
   PyDateTime_IMPORT;
   std::vector<PyObject*> sequences = {sequence};
   std::shared_ptr<Array> array;
-  std::vector<PyObject*> py_tensors;
-  RETURN_NOT_OK(SerializeSequences(context, sequences, 0, &array, &py_tensors));
+  RETURN_NOT_OK(SerializeSequences(context, sequences, 0, &array, &out->tensors));
   out->batch = MakeBatch(array);
-  for (const auto& py_tensor : py_tensors) {
-    std::shared_ptr<Tensor> arrow_tensor;
-    RETURN_NOT_OK(NdarrayToTensor(default_memory_pool(), py_tensor, &arrow_tensor));
-    out->tensors.push_back(arrow_tensor);
-  }
   return Status::OK();
 }
 
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 460a11ba2f853..fea7ceab64aa6 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -297,6 +297,27 @@ def test_numpy_immutable(large_memory_map):
             result[0] = 1.0
 
 
+# see https://issues.apache.org/jira/browse/ARROW-1695
+def test_serialization_callback_numpy():
+
+    class DummyClass(object):
+        pass
+
+    def serialize_dummy_class(obj):
+        x = np.zeros(4)
+        return x
+
+    def deserialize_dummy_class(serialized_obj):
+        return serialized_obj
+
+    pa._default_serialization_context.register_type(
+        DummyClass, "DummyClass", pickle=False,
+        custom_serializer=serialize_dummy_class,
+        custom_deserializer=deserialize_dummy_class)
+
+    pa.serialize(DummyClass())
+
+
 @pytest.mark.skip(reason="extensive memory requirements")
 def test_arrow_limits(self):
     def huge_memory_map(temp_dir):

From deaa0cf784bd1152b5e5b8f13fe43879a1b88cae Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 20 Oct 2017 20:55:27 -0400
Subject: [PATCH 1161/1644] ARROW-1686: [Docs] rsync contents of apidocs
 directory into site java directory

Omitting the trailing slash results in a `java/apidocs` subdirectory being created

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1221 from wesm/ARROW-1686 and squashes the following commits:

565681e7 [Wes McKinney] Move API docs generation readme to RELEASE_MANAGEMENT.md
2fcf5fcf [Wes McKinney] rsync contents of apidocs directory into site java directory
---
 dev/README.md                       | 30 -----------------------------
 dev/gen_apidocs/create_documents.sh |  2 +-
 dev/release/README                  |  4 ++--
 dev/release/RELEASE_MANAGEMENT.md   | 30 ++++++++++++++++++++++++++---
 4 files changed, 30 insertions(+), 36 deletions(-)

diff --git a/dev/README.md b/dev/README.md
index 05a68fc69f0e8..82f66f6ace2e2 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -110,33 +110,3 @@ Studio 2015):
 ```
 dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
 ```
-
-## Creating API documentation
-
-The generation of API documentation for `C++`, `C Glib`, `Python` 
-and `Java` has been Dockerized. To generate the API documentation
-run the following command:
-
-```shell
-bash dev/gen_apidocs.sh
-```
-
-This script assumes that the `parquet-cpp` Git repository 
-https://github.com/apache/parquet-cpp has been cloned
-besides the Arrow repository and a `dist` directory can be created
-at the same level by the current user. Please note that most of the
-software must be built in order to create the documentation, so this
-step may take some time to run, especially the first time around as the
-Docker container will also have to be built.
-
-After successfully creating the API documentation the website can be
-run locally to browse the API documentation from the top level
-`Documentation` menu. To run the website issue the command:
-
-```shell
-bash dev/run_site.sh
-```
-
-The local URL for the website running inside the docker container
-will be shown as `Server address:` in the output of the command.
-To stop the server press `Ctrl-C` in that window.
\ No newline at end of file
diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh
index 573a32820e75d..afbe041506d85 100755
--- a/dev/gen_apidocs/create_documents.sh
+++ b/dev/gen_apidocs/create_documents.sh
@@ -106,5 +106,5 @@ rm -rf target/site/apidocs/*
 mvn -Drat.skip=true install
 mvn -Drat.skip=true site
 mkdir -p ../site/asf-site/docs/java/
-rsync -r target/site/apidocs ../site/asf-site/docs/java/
+rsync -r target/site/apidocs/ ../site/asf-site/docs/java/
 popd
diff --git a/dev/release/README b/dev/release/README
index c53d1d8598135..dd2f512a0bb1b 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -1,7 +1,7 @@
 requirements:
 - being a committer to be able to push to dist and maven repository
 - a gpg key to sign the artifacts
-- use java 7. check your JAVA_HOME environment variable (at least for now. See ARROW-930) 
+- use java 7. check your JAVA_HOME environment variable (at least for now. See ARROW-930)
 - have the build requirements for cpp and c_glibg installed (see their README)
 
 to release, run the following (replace 0.1.0 with version to release):
@@ -28,7 +28,7 @@ mvn versions:set -DnewVersion=0.1-SNAPSHOT
 git reset --hard
 - setup gpg agent
 eval $(gpg-agent --daemon --allow-preset-passphrase)
-gpg --use-agent -s LICENSE.txt 
+gpg --use-agent -s LICENSE.txt
 - delete tag localy
 git tag -d apache-arrow-0.1.0
 
diff --git a/dev/release/RELEASE_MANAGEMENT.md b/dev/release/RELEASE_MANAGEMENT.md
index 6e1146ccd8b7b..ce7835d5f5d85 100644
--- a/dev/release/RELEASE_MANAGEMENT.md
+++ b/dev/release/RELEASE_MANAGEMENT.md
@@ -96,9 +96,33 @@ from your apache.org e-mail address to be accepted.
 
 ### Updating website with new API documentation
 
-Currently, this is manual, but is in the process of being automated (see
-ARROW-905). See instructions in
-https://github.com/apache/arrow/blob/master/site/README.md
+The API documentation for `C++`, `C Glib`, `Python` and `Java` can be generated
+via a Docker-based setup. To generate the API documentation run the following
+command:
+
+```shell
+bash dev/gen_apidocs.sh
+```
+
+This script assumes that the `parquet-cpp` Git repository
+https://github.com/apache/parquet-cpp has been cloned
+besides the Arrow repository and a `dist` directory can be created
+at the same level by the current user. Please note that most of the
+software must be built in order to create the documentation, so this
+step may take some time to run, especially the first time around as the
+Docker container will also have to be built.
+
+After successfully creating the API documentation the website can be
+run locally to browse the API documentation from the top level
+`Documentation` menu. To run the website issue the command:
+
+```shell
+bash dev/run_site.sh
+```
+
+The local URL for the website running inside the docker container
+will be shown as `Server address:` in the output of the command.
+To stop the server press `Ctrl-C` in that window.
 
 ### Updating C++ and Python packages
 

From 989aba6153ea48666a378336b7a2baf85e61c48a Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Sat, 21 Oct 2017 07:52:19 -0400
Subject: [PATCH 1162/1644] ARROW-1698: [JS] File reader attempts to load the
 same dictionary batch more than once

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1222 from TheNeuralBit/ARROW-1698 and squashes the following commits:

aec2f6db [Brian Hulette] readDictionary now returns a single vector or null
f7b251b0 [Brian Hulette] only load a dictionary once
---
 js/src/reader/arrow.ts      | 10 +++++++---
 js/src/reader/dictionary.ts | 18 +++++++-----------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/js/src/reader/arrow.ts b/js/src/reader/arrow.ts
index 9716c7fb60150..dbb6acd0e79e8 100644
--- a/js/src/reader/arrow.ts
+++ b/js/src/reader/arrow.ts
@@ -23,7 +23,7 @@ import { readFile } from './file';
 import { readStream } from './stream';
 import { readVector } from './vector';
 import { Vector } from '../vector/vector';
-import { readDictionaries } from './dictionary';
+import { readDictionary } from './dictionary';
 
 import ByteBuffer = flatbuffers.ByteBuffer;
 export import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
@@ -51,9 +51,13 @@ export function* readBuffers(...bytes: Array<Uint8Array | Buffer | string>) {
         let state = { nodeIndex: 0, bufferIndex: 0 };
         let index = -1, fieldsLength = schema.fieldsLength();
         if (batch.id) {
+            // A dictionary batch only contain a single vector. Traverse each
+            // field and its children until we find one that uses this dictionary
             while (++index < fieldsLength) {
-                for (let [id, vector] of readDictionaries(schema.fields(index), batch, state, dictionaries)) {
-                    dictionaries[id] = dictionaries[id] && dictionaries[id].concat(vector) || vector;
+                let vector = readDictionary(schema.fields(index), batch, state, dictionaries);
+                if (vector) {
+                    dictionaries[batch.id] = dictionaries[batch.id] && dictionaries[batch.id].concat(vector) || vector;
+                    break;
                 }
             }
         } else {
diff --git a/js/src/reader/dictionary.ts b/js/src/reader/dictionary.ts
index abf7ac3dfb976..61698e80c0054 100644
--- a/js/src/reader/dictionary.ts
+++ b/js/src/reader/dictionary.ts
@@ -21,20 +21,16 @@ import * as Schema_ from '../format/Schema_generated';
 import { IteratorState, Dictionaries } from './arrow';
 import Field = Schema_.org.apache.arrow.flatbuf.Field;
 
-export function* readDictionaries(field: Field | null,
-                                  batch: MessageBatch,
-                                  iterator: IteratorState,
-                                  dictionaries: Dictionaries) {
+export function readDictionary(field: Field | null,
+                               batch: MessageBatch,
+                               iterator: IteratorState,
+                               dictionaries: Dictionaries) {
     let id: string, encoding = field && field.dictionary();
     if (encoding && batch.id === (id = encoding.id().toFloat64().toString())) {
-        yield [id, readVector(field, batch, iterator, null)];
-        return;
+        return readVector(field, batch, iterator, null);
     }
     for (let i = -1, n = field && field.childrenLength() || 0; ++i < n;) {
-        // Since a dictionary batch can only contain a single vector, return early after we find it
-        for (let result of readDictionaries(field.children(i), batch, iterator, dictionaries)) {
-            yield result;
-            return;
-        }
+        let vector = readDictionary(field.children(i), batch, iterator, dictionaries);
+        if (vector) return vector;
     }
 }

From 2ee900c8ff1c3453594e1167999e49a9ee68d2e4 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 21 Oct 2017 12:08:28 -0400
Subject: [PATCH 1163/1644] ARROW-1702: Update jemalloc in manylinux1 build

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1230 from xhochy/update-manylinux1-jemalloc and squashes the following commits:

6a29fa8f [Uwe L. Korn] Update jemalloc in manylinux1 build
---
 python/manylinux1/scripts/build_jemalloc.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/python/manylinux1/scripts/build_jemalloc.sh b/python/manylinux1/scripts/build_jemalloc.sh
index 5bd4aeb1529fc..370bdcd4b4bcc 100755
--- a/python/manylinux1/scripts/build_jemalloc.sh
+++ b/python/manylinux1/scripts/build_jemalloc.sh
@@ -16,11 +16,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
-wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2
-tar xf jemalloc-4.4.0.tar.bz2
-pushd /jemalloc-4.4.0
+wget https://github.com/jemalloc/jemalloc/archive/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz -O jemalloc.tar.gz
+tar xf jemalloc.tar.gz
+mv jemalloc-* jemalloc
+pushd /jemalloc
+./autogen.sh
 ./configure "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
+# Skip doc generation
+touch doc/jemalloc.html
+touch doc/jemalloc.3
 make -j5
 make install
 popd
-rm -rf jemalloc-4.4.0.tar.bz2 jemalloc-4.4.0
+rm -rf jemalloc.tar.gz jemalloc

From 3549fa892ec65d016dbef6645b9daebd392c990d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 21 Oct 2017 12:09:19 -0400
Subject: [PATCH 1164/1644] ARROW-1683: [Python] Restore TimestampType to
 pyarrow namespace

This is being used in PySpark. After 0.8.0 we should migrate Spark to use the `pyarrow.types` APIs and then remove this from the top level namespace again

cc @BryanCutler

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1229 from wesm/ARROW-1683 and squashes the following commits:

7378d9d9 [Wes McKinney] Restore TimestampType to pyarrow namespace
---
 python/pyarrow/__init__.py         | 6 +++++-
 python/pyarrow/tests/test_types.py | 5 +++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index e37c123d24d38..0a1575f0edcb6 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -37,7 +37,8 @@
                          float16, float32, float64,
                          binary, string, decimal,
                          list_, struct, dictionary, field,
-                         type_for_alias, DataType, NAType,
+                         type_for_alias,
+                         DataType, NAType,
                          Field,
                          Schema,
                          schema,
@@ -67,6 +68,9 @@
                          DecimalValue,
                          Date32Value, Date64Value, TimestampValue)
 
+# ARROW-1683: Remove after 0.8.0?
+from pyarrow.lib import TimestampType
+
 from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
                          FixedSizeBufferWriter,
                          Buffer, BufferReader, BufferOutputStream,
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index 99cd4d0d5fe4e..d8eea622ccc87 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -132,3 +132,8 @@ def test_is_temporal_date_time_timestamp():
         assert not types.is_time(case)
 
     assert not types.is_temporal(pa.int32())
+
+
+def test_timestamp_type():
+    # See ARROW-1683
+    assert isinstance(pa.timestamp('ns'), pa.TimestampType)

From 9d12c7c92e4f1e8798f965925d44c1fed73cd29d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 21 Oct 2017 12:11:24 -0400
Subject: [PATCH 1165/1644] ARROW-1697: [GitHub] Add ISSUE_TEMPLATE.md

This will save us repeating ourselves when users report bugs or request features on GitHub

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1226 from wesm/ARROW-1697 and squashes the following commits:

8a2ed582 [Wes McKinney] Add note about mailing list
b11e497a [Wes McKinney] Add ISSUE_TEMPLATE.md
---
 .github/ISSUE_TEMPLATE.md         | 15 +++++++++++++++
 dev/release/rat_exclude_files.txt |  1 +
 2 files changed, 16 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE.md

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
new file mode 100644
index 0000000000000..fb8d249507c94
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1,15 @@
+## GitHub Issues for Apache Arrow
+
+We support GitHub issues as a lightweight way to ask questions and engage with
+the Arrow developer community. We use the Apache Software Foundation's JIRA
+instance for maintaining a queue of development work and as the public record
+for work on the project. So, feel free to delete this text and open an issue
+here if you have a question or aren't sure whether something is a bug or
+feature request. Any bugs, suggestions, or feature requests will eventually
+need to end up in JIRA, either before or after completing a pull request.
+
+See the Arrow JIRA project here: https://issues.apache.org/jira/browse/ARROW.
+
+For more in-depth development discussions, please use the public mailing list
+dev@arrow.apache.org (first subscribe by sending an e-mail to
+dev-subscribe@arrow.apache.org).
\ No newline at end of file
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index fa7113cc4b54a..db3b3aa5fac9c 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -5,6 +5,7 @@
 *_generated.ts
 *.json
 *.snap
+.github/ISSUE_TEMPLATE.md
 cpp/src/arrow/io/mman.h
 cpp/src/arrow/util/random.h
 cpp/src/arrow/status.cc

From 05788d035f4aa918d80c9db7a1bf74fe38309c60 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sat, 21 Oct 2017 12:26:08 -0400
Subject: [PATCH 1166/1644] ARROW-1701: [Serialization] Support zero copy
 PyTorch Tensor serialization

This also restructures the code such that it is easier to reset the default serialization context back to the initial state after more handlers have been registered.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1223 from pcmoritz/torch-tensor and squashes the following commits:

db09afce [Philipp Moritz] fix test
ba0856c1 [Philipp Moritz] don't run pytorch test on python 2.7
264d1992 [Philipp Moritz] remove import that is not required
882d9a56 [Philipp Moritz] small restructuring and support all PyTorch tensor types
c6dac9e6 [Philipp Moritz] add -q flag
23de67b6 [Philipp Moritz] add -y to torch installation
9814897f [Philipp Moritz] test torch tensor conversion
375bbfa5 [Philipp Moritz] support serializing torch tensors
---
 ci/travis_script_python.sh                 |   6 +
 python/pyarrow/__init__.py                 |   3 +-
 python/pyarrow/serialization.py            | 165 ++++++++++++---------
 python/pyarrow/tests/test_serialization.py |  19 +++
 4 files changed, 124 insertions(+), 69 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 97bde1a5ede95..603201bcc166b 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -46,6 +46,12 @@ conda install -y -q pip \
       sphinx \
       sphinx_bootstrap_theme
 
+if [ "$PYTHON_VERSION" != "2.7" ] || [ $TRAVIS_OS_NAME != "osx" ]; then
+  # Install pytorch for torch tensor conversion tests
+  # PyTorch seems to be broken on Python 2.7 on macOS so we skip it
+  conda install -y -q pytorch torchvision -c soumith
+fi
+
 # Build C++ libraries
 pushd $ARROW_CPP_BUILD_DIR
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 0a1575f0edcb6..ffc833a8e0bf6 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -120,7 +120,8 @@
 
 localfs = LocalFileSystem.get_instance()
 
-from pyarrow.serialization import _default_serialization_context
+from pyarrow.serialization import (_default_serialization_context,
+                                   register_default_serialization_handlers)
 
 import pyarrow.types as types
 
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index d08ae89a5930b..248b51c8c657f 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -20,107 +20,136 @@
 
 import numpy as np
 
+import pyarrow as pa
 from pyarrow import serialize_pandas, deserialize_pandas
 from pyarrow.lib import _default_serialization_context
 
-# ----------------------------------------------------------------------
-# Set up serialization for primitive datatypes
+def register_default_serialization_handlers(serialization_context):
 
-# TODO(pcm): This is currently a workaround until arrow supports
-# arbitrary precision integers. This is only called on long integers,
-# see the associated case in the append method in python_to_arrow.cc
-_default_serialization_context.register_type(
-    int, "int",
-    custom_serializer=lambda obj: str(obj),
-    custom_deserializer=lambda data: int(data))
+    # ----------------------------------------------------------------------
+    # Set up serialization for primitive datatypes
 
-if (sys.version_info < (3, 0)):
-    _default_serialization_context.register_type(
-        long, "long",  # noqa: F821
+    # TODO(pcm): This is currently a workaround until arrow supports
+    # arbitrary precision integers. This is only called on long integers,
+    # see the associated case in the append method in python_to_arrow.cc
+    serialization_context.register_type(
+        int, "int",
         custom_serializer=lambda obj: str(obj),
-        custom_deserializer=lambda data: long(data))  # noqa: F821
+        custom_deserializer=lambda data: int(data))
 
+    if (sys.version_info < (3, 0)):
+        serialization_context.register_type(
+            long, "long",  # noqa: F821
+            custom_serializer=lambda obj: str(obj),
+            custom_deserializer=lambda data: long(data))  # noqa: F821
 
-def _serialize_ordered_dict(obj):
-    return list(obj.keys()), list(obj.values())
 
+    def _serialize_ordered_dict(obj):
+        return list(obj.keys()), list(obj.values())
 
-def _deserialize_ordered_dict(data):
-    return OrderedDict(zip(data[0], data[1]))
 
+    def _deserialize_ordered_dict(data):
+        return OrderedDict(zip(data[0], data[1]))
 
-_default_serialization_context.register_type(
-    OrderedDict, "OrderedDict",
-    custom_serializer=_serialize_ordered_dict,
-    custom_deserializer=_deserialize_ordered_dict)
 
+    serialization_context.register_type(
+        OrderedDict, "OrderedDict",
+        custom_serializer=_serialize_ordered_dict,
+        custom_deserializer=_deserialize_ordered_dict)
 
-def _serialize_default_dict(obj):
-    return list(obj.keys()), list(obj.values()), obj.default_factory
 
+    def _serialize_default_dict(obj):
+        return list(obj.keys()), list(obj.values()), obj.default_factory
 
-def _deserialize_default_dict(data):
-    return defaultdict(data[2], zip(data[0], data[1]))
 
+    def _deserialize_default_dict(data):
+        return defaultdict(data[2], zip(data[0], data[1]))
 
-_default_serialization_context.register_type(
-     defaultdict, "defaultdict",
-     custom_serializer=_serialize_default_dict,
-     custom_deserializer=_deserialize_default_dict)
 
+    serialization_context.register_type(
+        defaultdict, "defaultdict",
+        custom_serializer=_serialize_default_dict,
+        custom_deserializer=_deserialize_default_dict)
 
-_default_serialization_context.register_type(
-     type(lambda: 0), "function",
-     pickle=True)
 
-# ----------------------------------------------------------------------
-# Set up serialization for numpy with dtype object (primitive types are
-# handled efficiently with Arrow's Tensor facilities, see python_to_arrow.cc)
+    serialization_context.register_type(
+        type(lambda: 0), "function",
+        pickle=True)
 
+    # ----------------------------------------------------------------------
+    # Set up serialization for numpy with dtype object (primitive types are
+    # handled efficiently with Arrow's Tensor facilities, see python_to_arrow.cc)
 
-def _serialize_numpy_array(obj):
-    return obj.tolist(), obj.dtype.str
 
+    def _serialize_numpy_array(obj):
+        return obj.tolist(), obj.dtype.str
 
-def _deserialize_numpy_array(data):
-    return np.array(data[0], dtype=np.dtype(data[1]))
 
+    def _deserialize_numpy_array(data):
+        return np.array(data[0], dtype=np.dtype(data[1]))
 
-_default_serialization_context.register_type(
-    np.ndarray, 'np.array',
-    custom_serializer=_serialize_numpy_array,
-    custom_deserializer=_deserialize_numpy_array)
 
+    serialization_context.register_type(
+        np.ndarray, 'np.array',
+        custom_serializer=_serialize_numpy_array,
+        custom_deserializer=_deserialize_numpy_array)
 
-# ----------------------------------------------------------------------
-# Set up serialization for pandas Series and DataFrame
 
-try:
-    import pandas as pd
+    # ----------------------------------------------------------------------
+    # Set up serialization for pandas Series and DataFrame
 
-    def _serialize_pandas_series(obj):
-        # TODO: serializing Series without extra copy
-        return serialize_pandas(pd.DataFrame({obj.name: obj})).to_pybytes()
+    try:
+        import pandas as pd
 
-    def _deserialize_pandas_series(data):
-        deserialized = deserialize_pandas(data)
-        return deserialized[deserialized.columns[0]]
+        def _serialize_pandas_series(obj):
+            # TODO: serializing Series without extra copy
+            return serialize_pandas(pd.DataFrame({obj.name: obj})).to_pybytes()
 
-    def _serialize_pandas_dataframe(obj):
-        return serialize_pandas(obj).to_pybytes()
+        def _deserialize_pandas_series(data):
+            deserialized = deserialize_pandas(data)
+            return deserialized[deserialized.columns[0]]
 
-    def _deserialize_pandas_dataframe(data):
-        return deserialize_pandas(data)
+        def _serialize_pandas_dataframe(obj):
+            return serialize_pandas(obj).to_pybytes()
 
-    _default_serialization_context.register_type(
-        pd.Series, 'pd.Series',
-        custom_serializer=_serialize_pandas_series,
-        custom_deserializer=_deserialize_pandas_series)
+        def _deserialize_pandas_dataframe(data):
+            return deserialize_pandas(data)
 
-    _default_serialization_context.register_type(
-        pd.DataFrame, 'pd.DataFrame',
-        custom_serializer=_serialize_pandas_dataframe,
-        custom_deserializer=_deserialize_pandas_dataframe)
-except ImportError:
-    # no pandas
-    pass
+        serialization_context.register_type(
+            pd.Series, 'pd.Series',
+            custom_serializer=_serialize_pandas_series,
+            custom_deserializer=_deserialize_pandas_series)
+
+        serialization_context.register_type(
+            pd.DataFrame, 'pd.DataFrame',
+            custom_serializer=_serialize_pandas_dataframe,
+            custom_deserializer=_deserialize_pandas_dataframe)
+    except ImportError:
+        # no pandas
+        pass
+
+    # ----------------------------------------------------------------------
+    # Set up serialization for pytorch tensors
+
+    try:
+        import torch
+
+        def _serialize_torch_tensor(obj):
+            return obj.numpy()
+
+        def _deserialize_torch_tensor(data):
+            return torch.from_numpy(data)
+
+        for t in [torch.FloatTensor, torch.DoubleTensor, torch.HalfTensor,
+                  torch.ByteTensor, torch.CharTensor, torch.ShortTensor,
+                  torch.IntTensor, torch.LongTensor]:
+            serialization_context.register_type(
+                t, "torch." + t.__name__,
+                custom_serializer=_serialize_torch_tensor,
+                custom_deserializer=_deserialize_torch_tensor)
+    except ImportError:
+        # no torch
+        pass
+
+
+register_default_serialization_handlers(_default_serialization_context)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index fea7ceab64aa6..39329481588da 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -29,6 +29,13 @@
 
 
 def assert_equal(obj1, obj2):
+    try:
+        import torch
+        if torch.is_tensor(obj1) and torch.is_tensor(obj2):
+            assert torch.equal(obj1, obj2)
+            return
+    except ImportError:
+        pass
     module_numpy = (type(obj1).__module__ == np.__name__ or
                     type(obj2).__module__ == np.__name__)
     if module_numpy:
@@ -57,6 +64,8 @@ def assert_equal(obj1, obj2):
                 return
         except:
             pass
+        if obj1.__dict__ == {}:
+            print("WARNING: Empty dict in ", obj1)
         for key in obj1.__dict__.keys():
             if key not in special_keys:
                 assert_equal(obj1.__dict__[key], obj2.__dict__[key])
@@ -285,6 +294,16 @@ def test_datetime_serialization(large_memory_map):
         for d in data:
             serialization_roundtrip(d, mmap)
 
+def test_torch_serialization(large_memory_map):
+    pytest.importorskip("torch")
+    import torch
+    with pa.memory_map(large_memory_map, mode="r+") as mmap:
+        # These are the only types that are supported for the
+        # PyTorch to NumPy conversion
+        for t in ["float32", "float64",
+                  "uint8", "int16", "int32", "int64"]:
+            obj = torch.from_numpy(np.random.randn(1000).astype(t))
+            serialization_roundtrip(obj, mmap)
 
 def test_numpy_immutable(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:

From fbeaeea59e4b855b8f27cf03c8dd244556cd5517 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@cozmixng.org>
Date: Sun, 22 Oct 2017 20:31:56 +0900
Subject: [PATCH 1167/1644] ARROW-1704: [GLib] Fix Go example failure

It's caused by GObject Introspection package in Ubuntu 14.04 is old.
Recent GObject Introspection doesn't have this problem.
---
 ci/travis_before_script_c_glib.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 01ef9e8e76e7f..a63945e1745d0 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -59,12 +59,22 @@ fi
 if [ $BUILD_SYSTEM = "autotools" ]; then
   go get github.com/linuxdeepin/go-gir-generator || :
   pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
+
+  # For old GObject Introspection.
+  # We can remove this when we use more later Ubuntu.
+  mv lib.in/glib-2.0/config.json{,.orig}
+  sed \
+    -e 's/\("unref_to_array"\)/"get_data", \1/g' \
+    lib.in/glib-2.0/config.json.orig > lib.in/glib-2.0/config.json
+
+  # Workaround. TODO: We should send a patch to go-gir-generator.
   rm lib.in/gio-2.0/gdk_workaround.go
   mv lib.in/gio-2.0/config.json{,.orig}
   sed \
     -e 's/\("Settings",\)/\/\/ \1/g' \
     -e 's/\("SettingsBackend",\)/\/\/ \1/g' \
     lib.in/gio-2.0/config.json.orig > lib.in/gio-2.0/config.json
+
   mv Makefile{,.orig}
   sed -e 's/ gudev-1.0//' Makefile.orig > Makefile
   mkdir -p out/src/gir/gudev-1.0

From 9ee5508ab3ec010b256c88181ba3215d5e966b04 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Sun, 22 Oct 2017 13:48:55 +0200
Subject: [PATCH 1168/1644] ARROW-1522: [Python] Zero copy buffer
 deserialization

This PR makes it possible to add serialization handlers that allow to deserialize python objects using zero copy. If the serialization handler returns an arrow buffer, it will be appended to the serialized objects and the object can be reconstructed from there without copying the buffer.

TODO before merge:

- [x] Add pandas zero copy buffer read
- [x] See if fixing the TODO about union tags makes the code cleaner

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1231 from pcmoritz/buffer-serialization and squashes the following commits:

10a01c0 [Philipp Moritz] run clang-format
529366d [Philipp Moritz] fix linting
7d88a67 [Philipp Moritz] fixes and make pandas deserialization zero-copy
2b75a0d [Philipp Moritz] fix linting
e078cb0 [Philipp Moritz] use union array fields to distinguish between ints and tensor/buffer references
dbd2364 [Philipp Moritz] add read path
0cc6d45 [Philipp Moritz] add read path
9f0173d [Philipp Moritz] buffer serialization write path
---
 cpp/src/arrow/python/arrow_to_python.cc    | 127 ++++++++++++---------
 cpp/src/arrow/python/python_to_arrow.cc    |  94 +++++++++------
 cpp/src/arrow/python/python_to_arrow.h     |   3 +
 python/pyarrow/serialization.py            |   5 +-
 python/pyarrow/tests/test_serialization.py |  19 +++
 5 files changed, 153 insertions(+), 95 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index 761d2903aada9..9686050b9676f 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -21,6 +21,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <string>
 #include <vector>
 
 #include <numpy/arrayobject.h>
@@ -32,6 +33,7 @@
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
+#include "arrow/python/pyarrow.h"
 #include "arrow/python/python_to_arrow.h"
 #include "arrow/python/util/datetime.h"
 #include "arrow/table.h"
@@ -44,23 +46,19 @@ Status CallDeserializeCallback(PyObject* context, PyObject* value,
                                PyObject** deserialized_object);
 
 Status DeserializeTuple(PyObject* context, const Array& array, int64_t start_idx,
-                        int64_t stop_idx, PyObject* base,
-                        const std::vector<std::shared_ptr<Tensor>>& tensors,
+                        int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                         PyObject** out);
 
 Status DeserializeList(PyObject* context, const Array& array, int64_t start_idx,
-                       int64_t stop_idx, PyObject* base,
-                       const std::vector<std::shared_ptr<Tensor>>& tensors,
+                       int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                        PyObject** out);
 
 Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
-                      int64_t stop_idx, PyObject* base,
-                      const std::vector<std::shared_ptr<Tensor>>& tensors,
+                      int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                       PyObject** out);
 
 Status DeserializeDict(PyObject* context, const Array& array, int64_t start_idx,
-                       int64_t stop_idx, PyObject* base,
-                       const std::vector<std::shared_ptr<Tensor>>& tensors,
+                       int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                        PyObject** out) {
   const auto& data = static_cast<const StructArray&>(array);
   ScopedRef keys, vals;
@@ -69,10 +67,10 @@ Status DeserializeDict(PyObject* context, const Array& array, int64_t start_idx,
 
   DCHECK_EQ(2, data.num_fields());
 
-  RETURN_NOT_OK(DeserializeList(context, *data.field(0), start_idx, stop_idx, base,
-                                tensors, keys.ref()));
-  RETURN_NOT_OK(DeserializeList(context, *data.field(1), start_idx, stop_idx, base,
-                                tensors, vals.ref()));
+  RETURN_NOT_OK(DeserializeList(context, *data.field(0), start_idx, stop_idx, base, blobs,
+                                keys.ref()));
+  RETURN_NOT_OK(DeserializeList(context, *data.field(1), start_idx, stop_idx, base, blobs,
+                                vals.ref()));
   for (int64_t i = start_idx; i < stop_idx; ++i) {
     // PyDict_SetItem behaves differently from PyList_SetItem and PyTuple_SetItem.
     // The latter two steal references whereas PyDict_SetItem does not. So we need
@@ -91,10 +89,9 @@ Status DeserializeDict(PyObject* context, const Array& array, int64_t start_idx,
 }
 
 Status DeserializeArray(const Array& array, int64_t offset, PyObject* base,
-                        const std::vector<std::shared_ptr<arrow::Tensor>>& tensors,
-                        PyObject** out) {
+                        const SerializedPyObject& blobs, PyObject** out) {
   int32_t index = static_cast<const Int32Array&>(array).Value(offset);
-  RETURN_NOT_OK(py::TensorToNdarray(*tensors[index], base, out));
+  RETURN_NOT_OK(py::TensorToNdarray(*blobs.tensors[index], base, out));
   // Mark the array as immutable
   ScopedRef flags(PyObject_GetAttrString(*out, "flags"));
   DCHECK(flags.get() != NULL) << "Could not mark Numpy array immutable";
@@ -104,9 +101,9 @@ Status DeserializeArray(const Array& array, int64_t offset, PyObject* base,
   return Status::OK();
 }
 
-Status GetValue(PyObject* context, const Array& arr, int64_t index, int32_t type,
-                PyObject* base, const std::vector<std::shared_ptr<Tensor>>& tensors,
-                PyObject** result) {
+Status GetValue(PyObject* context, const UnionArray& parent, const Array& arr,
+                int64_t index, int32_t type, PyObject* base,
+                const SerializedPyObject& blobs, PyObject** result) {
   switch (arr.type()->id()) {
     case Type::BOOL:
       *result = PyBool_FromLong(static_cast<const BooleanArray&>(arr).Value(index));
@@ -151,69 +148,72 @@ Status GetValue(PyObject* context, const Array& arr, int64_t index, int32_t type
       const auto& l = static_cast<const ListArray&>(*s.field(0));
       if (s.type()->child(0)->name() == "list") {
         return DeserializeList(context, *l.values(), l.value_offset(index),
-                               l.value_offset(index + 1), base, tensors, result);
+                               l.value_offset(index + 1), base, blobs, result);
       } else if (s.type()->child(0)->name() == "tuple") {
         return DeserializeTuple(context, *l.values(), l.value_offset(index),
-                                l.value_offset(index + 1), base, tensors, result);
+                                l.value_offset(index + 1), base, blobs, result);
       } else if (s.type()->child(0)->name() == "dict") {
         return DeserializeDict(context, *l.values(), l.value_offset(index),
-                               l.value_offset(index + 1), base, tensors, result);
+                               l.value_offset(index + 1), base, blobs, result);
       } else if (s.type()->child(0)->name() == "set") {
         return DeserializeSet(context, *l.values(), l.value_offset(index),
-                              l.value_offset(index + 1), base, tensors, result);
+                              l.value_offset(index + 1), base, blobs, result);
       } else {
         DCHECK(false) << "unexpected StructArray type " << s.type()->child(0)->name();
       }
     }
-    // We use an Int32Builder here to distinguish the tensor indices from
-    // the Type::INT64 above (see tensor_indices_ in SequenceBuilder).
-    case Type::INT32: {
-      return DeserializeArray(arr, index, base, tensors, result);
+    default: {
+      const std::string& child_name = parent.type()->child(type)->name();
+      if (child_name == "tensor") {
+        return DeserializeArray(arr, index, base, blobs, result);
+      } else if (child_name == "buffer") {
+        int32_t ref = static_cast<const Int32Array&>(arr).Value(index);
+        *result = wrap_buffer(blobs.buffers[ref]);
+        return Status::OK();
+      } else {
+        DCHECK(false) << "union tag " << type << " with child name '" << child_name
+                      << "' not recognized";
+      }
     }
-    default:
-      DCHECK(false) << "union tag " << type << " not recognized";
   }
   return Status::OK();
 }
 
-#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                               \
-  const auto& data = static_cast<const UnionArray&>(array);                        \
-  ScopedRef result(CREATE_FN(stop_idx - start_idx));                               \
-  const uint8_t* type_ids = data.raw_type_ids();                                   \
-  const int32_t* value_offsets = data.raw_value_offsets();                         \
-  for (int64_t i = start_idx; i < stop_idx; ++i) {                                 \
-    if (data.IsNull(i)) {                                                          \
-      Py_INCREF(Py_None);                                                          \
-      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                           \
-    } else {                                                                       \
-      int64_t offset = value_offsets[i];                                           \
-      uint8_t type = type_ids[i];                                                  \
-      PyObject* value;                                                             \
-      RETURN_NOT_OK(GetValue(context, *data.UnsafeChild(type), offset, type, base, \
-                             tensors, &value));                                    \
-      SET_ITEM_FN(result.get(), i - start_idx, value);                             \
-    }                                                                              \
-  }                                                                                \
-  *out = result.release();                                                         \
+#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                                     \
+  const auto& data = static_cast<const UnionArray&>(array);                              \
+  ScopedRef result(CREATE_FN(stop_idx - start_idx));                                     \
+  const uint8_t* type_ids = data.raw_type_ids();                                         \
+  const int32_t* value_offsets = data.raw_value_offsets();                               \
+  for (int64_t i = start_idx; i < stop_idx; ++i) {                                       \
+    if (data.IsNull(i)) {                                                                \
+      Py_INCREF(Py_None);                                                                \
+      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                                 \
+    } else {                                                                             \
+      int64_t offset = value_offsets[i];                                                 \
+      uint8_t type = type_ids[i];                                                        \
+      PyObject* value;                                                                   \
+      RETURN_NOT_OK(GetValue(context, data, *data.UnsafeChild(type), offset, type, base, \
+                             blobs, &value));                                            \
+      SET_ITEM_FN(result.get(), i - start_idx, value);                                   \
+    }                                                                                    \
+  }                                                                                      \
+  *out = result.release();                                                               \
   return Status::OK()
 
 Status DeserializeList(PyObject* context, const Array& array, int64_t start_idx,
-                       int64_t stop_idx, PyObject* base,
-                       const std::vector<std::shared_ptr<Tensor>>& tensors,
+                       int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                        PyObject** out) {
   DESERIALIZE_SEQUENCE(PyList_New, PyList_SET_ITEM);
 }
 
 Status DeserializeTuple(PyObject* context, const Array& array, int64_t start_idx,
-                        int64_t stop_idx, PyObject* base,
-                        const std::vector<std::shared_ptr<Tensor>>& tensors,
+                        int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                         PyObject** out) {
   DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SET_ITEM);
 }
 
 Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
-                      int64_t stop_idx, PyObject* base,
-                      const std::vector<std::shared_ptr<Tensor>>& tensors,
+                      int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                       PyObject** out) {
   const auto& data = static_cast<const UnionArray&>(array);
   ScopedRef result(PySet_New(nullptr));
@@ -229,8 +229,8 @@ Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
       int32_t offset = value_offsets[i];
       int8_t type = type_ids[i];
       PyObject* value;
-      RETURN_NOT_OK(GetValue(context, *data.UnsafeChild(type), offset, type, base,
-                             tensors, &value));
+      RETURN_NOT_OK(GetValue(context, data, *data.UnsafeChild(type), offset, type, base,
+                             blobs, &value));
       if (PySet_Add(result.get(), value) < 0) {
         RETURN_IF_PYERROR();
       }
@@ -244,9 +244,12 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out)
   int64_t offset;
   int64_t bytes_read;
   int32_t num_tensors;
+  int32_t num_buffers;
   // Read number of tensors
   RETURN_NOT_OK(
       src->Read(sizeof(int32_t), &bytes_read, reinterpret_cast<uint8_t*>(&num_tensors)));
+  RETURN_NOT_OK(
+      src->Read(sizeof(int32_t), &bytes_read, reinterpret_cast<uint8_t*>(&num_buffers)));
 
   std::shared_ptr<RecordBatchReader> reader;
   RETURN_NOT_OK(ipc::RecordBatchStreamReader::Open(src, &reader));
@@ -260,6 +263,18 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out)
     out->tensors.push_back(tensor);
     RETURN_NOT_OK(src->Tell(&offset));
   }
+
+  for (int i = 0; i < num_buffers; ++i) {
+    int64_t size;
+    RETURN_NOT_OK(src->ReadAt(offset, sizeof(int64_t), &bytes_read,
+                              reinterpret_cast<uint8_t*>(&size)));
+    RETURN_NOT_OK(src->Tell(&offset));
+    std::shared_ptr<Buffer> buffer;
+    RETURN_NOT_OK(src->ReadAt(offset, size, &buffer));
+    out->buffers.push_back(buffer);
+    RETURN_NOT_OK(src->Tell(&offset));
+  }
+
   return Status::OK();
 }
 
@@ -268,7 +283,7 @@ Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObj
   PyAcquireGIL lock;
   PyDateTime_IMPORT;
   return DeserializeList(context, *obj.batch->column(0), 0, obj.batch->num_rows(), base,
-                         obj.tensors, out);
+                         obj, out);
 }
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index a46d10d446084..b0c6287f088a1 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -63,6 +63,7 @@ class SequenceBuilder {
         doubles_(::arrow::float64(), pool),
         date64s_(::arrow::date64(), pool),
         tensor_indices_(::arrow::int32(), pool),
+        buffer_indices_(::arrow::int32(), pool),
         list_offsets_({0}),
         tuple_offsets_({0}),
         dict_offsets_({0}),
@@ -146,6 +147,14 @@ class SequenceBuilder {
     return tensor_indices_.Append(tensor_index);
   }
 
+  /// Appending a buffer to the sequence
+  ///
+  /// \param buffer_index Indes of the buffer in the object.
+  Status AppendBuffer(const int32_t buffer_index) {
+    RETURN_NOT_OK(Update(buffer_indices_.length(), &buffer_tag_));
+    return buffer_indices_.Append(buffer_index);
+  }
+
   /// Add a sublist to the sequence. The data contained in the sublist will be
   /// specified in the "Finish" method.
   ///
@@ -184,9 +193,9 @@ class SequenceBuilder {
   }
 
   template <typename BuilderType>
-  Status AddElement(const int8_t tag, BuilderType* out) {
+  Status AddElement(const int8_t tag, BuilderType* out, const std::string& name = "") {
     if (tag != -1) {
-      fields_[tag] = ::arrow::field("", out->type());
+      fields_[tag] = ::arrow::field(name, out->type());
       RETURN_NOT_OK(out->Finish(&children_[tag]));
       RETURN_NOT_OK(nones_.AppendToBitmap(true));
       type_ids_.push_back(tag);
@@ -232,7 +241,8 @@ class SequenceBuilder {
     RETURN_NOT_OK(AddElement(float_tag_, &floats_));
     RETURN_NOT_OK(AddElement(double_tag_, &doubles_));
     RETURN_NOT_OK(AddElement(date64_tag_, &date64s_));
-    RETURN_NOT_OK(AddElement(tensor_tag_, &tensor_indices_));
+    RETURN_NOT_OK(AddElement(tensor_tag_, &tensor_indices_, "tensor"));
+    RETURN_NOT_OK(AddElement(buffer_tag_, &buffer_indices_, "buffer"));
 
     RETURN_NOT_OK(AddSubsequence(list_tag_, list_data, list_offsets_, "list"));
     RETURN_NOT_OK(AddSubsequence(tuple_tag_, tuple_data, tuple_offsets_, "tuple"));
@@ -262,11 +272,8 @@ class SequenceBuilder {
   DoubleBuilder doubles_;
   Date64Builder date64s_;
 
-  // We use an Int32Builder here to distinguish the tensor indices from
-  // the ints_ above (see the case Type::INT32 in get_value in python.cc).
-  // TODO(pcm): Replace this by using the union tags to distinguish between
-  // these two cases.
   Int32Builder tensor_indices_;
+  Int32Builder buffer_indices_;
 
   std::vector<int32_t> list_offsets_;
   std::vector<int32_t> tuple_offsets_;
@@ -288,6 +295,7 @@ class SequenceBuilder {
   int8_t date64_tag_ = -1;
 
   int8_t tensor_tag_ = -1;
+  int8_t buffer_tag_ = -1;
   int8_t list_tag_ = -1;
   int8_t tuple_tag_ = -1;
   int8_t dict_tag_ = -1;
@@ -390,15 +398,14 @@ Status CallDeserializeCallback(PyObject* context, PyObject* value,
 
 Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
                      int32_t recursion_depth, std::shared_ptr<Array>* out,
-                     std::vector<std::shared_ptr<Tensor>>* tensors_out);
+                     SerializedPyObject* blobs_out);
 
 Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder,
-                      std::vector<PyObject*>* subdicts,
-                      std::vector<std::shared_ptr<Tensor>>* tensors_out);
+                      std::vector<PyObject*>* subdicts, SerializedPyObject* blobs_out);
 
 Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
                           int32_t recursion_depth, std::shared_ptr<Array>* out,
-                          std::vector<std::shared_ptr<Tensor>>* tensors_out);
+                          SerializedPyObject* blobs_out);
 
 Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
   if (PyArray_IsScalar(obj, Bool)) {
@@ -444,12 +451,12 @@ Status AppendScalar(PyObject* obj, SequenceBuilder* builder) {
 Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
               std::vector<PyObject*>* sublists, std::vector<PyObject*>* subtuples,
               std::vector<PyObject*>* subdicts, std::vector<PyObject*>* subsets,
-              std::vector<std::shared_ptr<Tensor>>* tensors_out) {
+              SerializedPyObject* blobs_out) {
   // The bool case must precede the int case (PyInt_Check passes for bools)
   if (PyBool_Check(elem)) {
     RETURN_NOT_OK(builder->AppendBool(elem == Py_True));
   } else if (PyArray_DescrFromScalar(elem)->type_num == NPY_HALF) {
-    npy_half halffloat = reinterpret_cast<PyHalfScalarObject *>(elem)->obval;
+    npy_half halffloat = reinterpret_cast<PyHalfScalarObject*>(elem)->obval;
     RETURN_NOT_OK(builder->AppendHalfFloat(halffloat));
   } else if (PyFloat_Check(elem)) {
     RETURN_NOT_OK(builder->AppendDouble(PyFloat_AS_DOUBLE(elem)));
@@ -506,12 +513,17 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
     RETURN_NOT_OK(AppendScalar(elem, builder));
   } else if (PyArray_Check(elem)) {
     RETURN_NOT_OK(SerializeArray(context, reinterpret_cast<PyArrayObject*>(elem), builder,
-                                 subdicts, tensors_out));
+                                 subdicts, blobs_out));
   } else if (elem == Py_None) {
     RETURN_NOT_OK(builder->AppendNone());
   } else if (PyDateTime_CheckExact(elem)) {
     PyDateTime_DateTime* datetime = reinterpret_cast<PyDateTime_DateTime*>(elem);
     RETURN_NOT_OK(builder->AppendDate64(PyDateTime_to_us(datetime)));
+  } else if (is_buffer(elem)) {
+    RETURN_NOT_OK(builder->AppendBuffer(static_cast<int32_t>(blobs_out->buffers.size())));
+    std::shared_ptr<Buffer> buffer;
+    RETURN_NOT_OK(unwrap_buffer(elem, &buffer));
+    blobs_out->buffers.push_back(buffer);
   } else {
     // Attempt to serialize the object using the custom callback.
     PyObject* serialized_object;
@@ -524,8 +536,7 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
 }
 
 Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder,
-                      std::vector<PyObject*>* subdicts,
-                      std::vector<std::shared_ptr<Tensor>>* tensors_out) {
+                      std::vector<PyObject*>* subdicts, SerializedPyObject* blobs_out) {
   int dtype = PyArray_TYPE(array);
   switch (dtype) {
     case NPY_UINT8:
@@ -539,11 +550,12 @@ Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder*
     case NPY_HALF:
     case NPY_FLOAT:
     case NPY_DOUBLE: {
-      RETURN_NOT_OK(builder->AppendTensor(static_cast<int32_t>(tensors_out->size())));
+      RETURN_NOT_OK(
+          builder->AppendTensor(static_cast<int32_t>(blobs_out->tensors.size())));
       std::shared_ptr<Tensor> tensor;
       RETURN_NOT_OK(NdarrayToTensor(default_memory_pool(),
                                     reinterpret_cast<PyObject*>(array), &tensor));
-      tensors_out->push_back(tensor);
+      blobs_out->tensors.push_back(tensor);
     } break;
     default: {
       PyObject* serialized_object;
@@ -559,7 +571,7 @@ Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder*
 
 Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
                           int32_t recursion_depth, std::shared_ptr<Array>* out,
-                          std::vector<std::shared_ptr<Tensor>>* tensors_out) {
+                          SerializedPyObject* blobs_out) {
   DCHECK(out);
   if (recursion_depth >= kMaxRecursionDepth) {
     return Status::NotImplemented(
@@ -578,35 +590,35 @@ Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
         break;
       }
       RETURN_NOT_OK(Append(context, item.get(), &builder, &sublists, &subtuples,
-                           &subdicts, &subsets, tensors_out));
+                           &subdicts, &subsets, blobs_out));
     }
   }
   std::shared_ptr<Array> list;
   if (sublists.size() > 0) {
     RETURN_NOT_OK(
-        SerializeSequences(context, sublists, recursion_depth + 1, &list, tensors_out));
+        SerializeSequences(context, sublists, recursion_depth + 1, &list, blobs_out));
   }
   std::shared_ptr<Array> tuple;
   if (subtuples.size() > 0) {
     RETURN_NOT_OK(
-        SerializeSequences(context, subtuples, recursion_depth + 1, &tuple, tensors_out));
+        SerializeSequences(context, subtuples, recursion_depth + 1, &tuple, blobs_out));
   }
   std::shared_ptr<Array> dict;
   if (subdicts.size() > 0) {
     RETURN_NOT_OK(
-        SerializeDict(context, subdicts, recursion_depth + 1, &dict, tensors_out));
+        SerializeDict(context, subdicts, recursion_depth + 1, &dict, blobs_out));
   }
   std::shared_ptr<Array> set;
   if (subsets.size() > 0) {
     RETURN_NOT_OK(
-        SerializeSequences(context, subsets, recursion_depth + 1, &set, tensors_out));
+        SerializeSequences(context, subsets, recursion_depth + 1, &set, blobs_out));
   }
   return builder.Finish(list.get(), tuple.get(), dict.get(), set.get(), out);
 }
 
 Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
                      int32_t recursion_depth, std::shared_ptr<Array>* out,
-                     std::vector<std::shared_ptr<Tensor>>* tensors_out) {
+                     SerializedPyObject* blobs_out) {
   DictBuilder result;
   if (recursion_depth >= kMaxRecursionDepth) {
     return Status::NotImplemented(
@@ -616,45 +628,46 @@ Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
   std::vector<PyObject*> key_tuples, key_dicts, val_lists, val_tuples, val_dicts,
       val_sets, dummy;
   for (const auto& dict : dicts) {
-    PyObject *key, *value;
+    PyObject* key;
+    PyObject* value;
     Py_ssize_t pos = 0;
     while (PyDict_Next(dict, &pos, &key, &value)) {
       RETURN_NOT_OK(Append(context, key, &result.keys(), &dummy, &key_tuples, &key_dicts,
-                           &dummy, tensors_out));
+                           &dummy, blobs_out));
       DCHECK_EQ(dummy.size(), 0);
       RETURN_NOT_OK(Append(context, value, &result.vals(), &val_lists, &val_tuples,
-                           &val_dicts, &val_sets, tensors_out));
+                           &val_dicts, &val_sets, blobs_out));
     }
   }
   std::shared_ptr<Array> key_tuples_arr;
   if (key_tuples.size() > 0) {
     RETURN_NOT_OK(SerializeSequences(context, key_tuples, recursion_depth + 1,
-                                     &key_tuples_arr, tensors_out));
+                                     &key_tuples_arr, blobs_out));
   }
   std::shared_ptr<Array> key_dicts_arr;
   if (key_dicts.size() > 0) {
     RETURN_NOT_OK(SerializeDict(context, key_dicts, recursion_depth + 1, &key_dicts_arr,
-                                tensors_out));
+                                blobs_out));
   }
   std::shared_ptr<Array> val_list_arr;
   if (val_lists.size() > 0) {
     RETURN_NOT_OK(SerializeSequences(context, val_lists, recursion_depth + 1,
-                                     &val_list_arr, tensors_out));
+                                     &val_list_arr, blobs_out));
   }
   std::shared_ptr<Array> val_tuples_arr;
   if (val_tuples.size() > 0) {
     RETURN_NOT_OK(SerializeSequences(context, val_tuples, recursion_depth + 1,
-                                     &val_tuples_arr, tensors_out));
+                                     &val_tuples_arr, blobs_out));
   }
   std::shared_ptr<Array> val_dict_arr;
   if (val_dicts.size() > 0) {
-    RETURN_NOT_OK(SerializeDict(context, val_dicts, recursion_depth + 1, &val_dict_arr,
-                                tensors_out));
+    RETURN_NOT_OK(
+        SerializeDict(context, val_dicts, recursion_depth + 1, &val_dict_arr, blobs_out));
   }
   std::shared_ptr<Array> val_set_arr;
   if (val_sets.size() > 0) {
     RETURN_NOT_OK(SerializeSequences(context, val_sets, recursion_depth + 1, &val_set_arr,
-                                     tensors_out));
+                                     blobs_out));
   }
   RETURN_NOT_OK(result.Finish(key_tuples_arr.get(), key_dicts_arr.get(),
                               val_list_arr.get(), val_tuples_arr.get(),
@@ -687,16 +700,19 @@ std::shared_ptr<RecordBatch> MakeBatch(std::shared_ptr<Array> data) {
 Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out) {
   PyAcquireGIL lock;
   PyDateTime_IMPORT;
+  import_pyarrow();
   std::vector<PyObject*> sequences = {sequence};
   std::shared_ptr<Array> array;
-  RETURN_NOT_OK(SerializeSequences(context, sequences, 0, &array, &out->tensors));
+  RETURN_NOT_OK(SerializeSequences(context, sequences, 0, &array, out));
   out->batch = MakeBatch(array);
   return Status::OK();
 }
 
 Status WriteSerializedObject(const SerializedPyObject& obj, io::OutputStream* dst) {
   int32_t num_tensors = static_cast<int32_t>(obj.tensors.size());
+  int32_t num_buffers = static_cast<int32_t>(obj.buffers.size());
   RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&num_tensors), sizeof(int32_t)));
+  RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&num_buffers), sizeof(int32_t)));
   RETURN_NOT_OK(ipc::WriteRecordBatchStream({obj.batch}, dst));
 
   int32_t metadata_length;
@@ -705,6 +721,12 @@ Status WriteSerializedObject(const SerializedPyObject& obj, io::OutputStream* ds
     RETURN_NOT_OK(ipc::WriteTensor(*tensor, dst, &metadata_length, &body_length));
   }
 
+  for (const auto& buffer : obj.buffers) {
+    int64_t size = buffer->size();
+    RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&size), sizeof(int64_t)));
+    RETURN_NOT_OK(dst->Write(buffer->data(), size));
+  }
+
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/python/python_to_arrow.h b/cpp/src/arrow/python/python_to_arrow.h
index 2c80e5d6bf891..c5b6396145b7f 100644
--- a/cpp/src/arrow/python/python_to_arrow.h
+++ b/cpp/src/arrow/python/python_to_arrow.h
@@ -23,6 +23,8 @@
 #include <memory>
 #include <vector>
 
+#include "arrow/python/common.h"
+#include "arrow/python/pyarrow.h"
 #include "arrow/status.h"
 #include "arrow/util/visibility.h"
 
@@ -42,6 +44,7 @@ namespace py {
 struct ARROW_EXPORT SerializedPyObject {
   std::shared_ptr<RecordBatch> batch;
   std::vector<std::shared_ptr<Tensor>> tensors;
+  std::vector<std::shared_ptr<Buffer>> buffers;
 };
 
 /// \brief Serialize Python sequence as a RecordBatch plus
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 248b51c8c657f..eed6aae83736f 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -102,15 +102,14 @@ def _deserialize_numpy_array(data):
         import pandas as pd
 
         def _serialize_pandas_series(obj):
-            # TODO: serializing Series without extra copy
-            return serialize_pandas(pd.DataFrame({obj.name: obj})).to_pybytes()
+            return serialize_pandas(pd.DataFrame({obj.name: obj}))
 
         def _deserialize_pandas_series(data):
             deserialized = deserialize_pandas(data)
             return deserialized[deserialized.columns[0]]
 
         def _serialize_pandas_dataframe(obj):
-            return serialize_pandas(obj).to_pybytes()
+            return serialize_pandas(obj)
 
         def _deserialize_pandas_dataframe(data):
             return deserialize_pandas(data)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 39329481588da..9321ebc34abbe 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -336,6 +336,25 @@ def deserialize_dummy_class(serialized_obj):
 
     pa.serialize(DummyClass())
 
+def test_buffer_serialization():
+
+    class BufferClass(object):
+        pass
+
+    def serialize_buffer_class(obj):
+        return pa.frombuffer(b"hello")
+
+    def deserialize_buffer_class(serialized_obj):
+        return serialized_obj
+
+    pa._default_serialization_context.register_type(
+        BufferClass, "BufferClass", pickle=False,
+        custom_serializer=serialize_buffer_class,
+        custom_deserializer=deserialize_buffer_class)
+
+    b = pa.serialize(BufferClass()).to_buffer()
+    assert pa.deserialize(b).to_pybytes() == b"hello"
+
 
 @pytest.mark.skip(reason="extensive memory requirements")
 def test_arrow_limits(self):

From 61d8a76c4c4fd13a1fbf11e0bc9d23f7e4fdcd23 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 22 Oct 2017 14:19:00 +0200
Subject: [PATCH 1169/1644] ARROW-641: [C++] Do not build io-hdfs-test if
 ARROW_HDFS is off

Initially I was looking to see if we could gracefully skip the tests if we have the requisite libraries available but are unable to connect to a running HDFS cluster, but I think this would mask errors that we want to see. So this just makes it so that developers who don't want to see the test failure can skip building the unit test executable

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1232 from wesm/ARROW-641 and squashes the following commits:

eed9ea4 [Wes McKinney] Do not build io-hdfs-test if ARROW_HDFS is off
---
 cpp/src/arrow/io/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt
index 98e3859910d43..a7e569870b6e3 100644
--- a/cpp/src/arrow/io/CMakeLists.txt
+++ b/cpp/src/arrow/io/CMakeLists.txt
@@ -19,9 +19,11 @@
 # arrow_io : Arrow IO interfaces
 
 ADD_ARROW_TEST(io-file-test)
-if (NOT ARROW_BOOST_HEADER_ONLY)
+
+if (ARROW_HDFS AND NOT ARROW_BOOST_HEADER_ONLY)
   ADD_ARROW_TEST(io-hdfs-test NO_VALGRIND)
 endif()
+
 ADD_ARROW_TEST(io-memory-test)
 
 ADD_ARROW_BENCHMARK(io-memory-benchmark)

From 53dd0c85718112b274ecd266b6b524f627d16474 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 22 Oct 2017 21:37:49 -0400
Subject: [PATCH 1170/1644] ARROW-1087: [Python] Add pyarrow.get_include
 function. Bundle includes in all builds

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1219 from wesm/ARROW-1087 and squashes the following commits:

5444b82e [Wes McKinney] Use more stable include
6cc855de [Wes McKinney] Add stub rst file about C extensions with pyarrow
---
 python/.gitignore                 |  4 ++++
 python/CMakeLists.txt             |  4 +++-
 python/doc/source/extending.rst   | 33 +++++++++++++++++++++++++++++++
 python/doc/source/index.rst       |  1 +
 python/pyarrow/__init__.py        | 12 +++++++++++
 python/pyarrow/tests/test_misc.py | 25 +++++++++++++++++++++++
 python/setup.py                   | 12 ++++++-----
 7 files changed, 85 insertions(+), 6 deletions(-)
 create mode 100644 python/doc/source/extending.rst
 create mode 100644 python/pyarrow/tests/test_misc.py

diff --git a/python/.gitignore b/python/.gitignore
index 1bf20c4ca5298..6a63e5a2a7c2a 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -17,6 +17,10 @@ Testing/
 *.cpp
 pyarrow/version.py
 pyarrow/*_api.h
+
+# Bundled headers
+pyarrow/include
+
 # Python files
 
 # setup.py working directory
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 169e7ad02efa9..8c7348298db5f 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -226,9 +226,11 @@ function(bundle_arrow_implib library_path)
       COPYONLY)
 endfunction(bundle_arrow_implib)
 
+# Always bundle includes
+file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
+
 if (PYARROW_BUNDLE_ARROW_CPP)
   # arrow
-  file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
   bundle_arrow_lib(ARROW_SHARED_LIB
     ABI_VERSION ${ARROW_ABI_VERSION}
     SO_VERSION ${ARROW_SO_VERSION})
diff --git a/python/doc/source/extending.rst b/python/doc/source/extending.rst
new file mode 100644
index 0000000000000..2d6caed268ef2
--- /dev/null
+++ b/python/doc/source/extending.rst
@@ -0,0 +1,33 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+.. _extending:
+
+Building C++ and Cython Extensions using pyarrow
+================================================
+
+pyarrow features both a Cython and C++ API. We intend to fully document the
+details of how to do this.
+
+The Arrow C++ header files are bundled with a pyarrow installation. To get the
+absolute path to this directory (like ``numpy.get_include()``), use:
+
+.. code-block:: python
+
+   import pyarrow as pa
+   pa.get_include()
diff --git a/python/doc/source/index.rst b/python/doc/source/index.rst
index c2ae769b23e83..b933d2359f720 100644
--- a/python/doc/source/index.rst
+++ b/python/doc/source/index.rst
@@ -43,5 +43,6 @@ structures.
    plasma
    pandas
    parquet
+   extending
    api
    getting_involved
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index ffc833a8e0bf6..1215c822d2e47 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -147,3 +147,15 @@ def _plasma_store_entry_point():
 # Deprecations
 
 from pyarrow.util import _deprecate_class  # noqa
+
+# ----------------------------------------------------------------------
+# Returning absolute path to the pyarrow include directory (if bundled, e.g. in
+# wheels)
+
+def get_include():
+    """
+    Return absolute path to directory containing Arrow C++ include
+    headers. Similar to numpy.get_include
+    """
+    import os
+    return os.path.join(os.path.dirname(__file__), 'include')
diff --git a/python/pyarrow/tests/test_misc.py b/python/pyarrow/tests/test_misc.py
new file mode 100644
index 0000000000000..55787f1359437
--- /dev/null
+++ b/python/pyarrow/tests/test_misc.py
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import pyarrow as pa
+
+
+def test_get_include():
+    include_dir = pa.get_include()
+    assert os.path.exists(os.path.join(include_dir, 'arrow', 'api.h'))
diff --git a/python/setup.py b/python/setup.py
index edcf397f3fb51..ccab8fb6581f2 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -220,11 +220,7 @@ def _run_cmake(self):
             build_prefix = self.build_type
 
         if self.bundle_arrow_cpp:
-            print(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
-            if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
-                shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
-            shutil.move(pjoin(build_prefix, 'include'),
-                        pjoin(build_lib, 'pyarrow'))
+            print(pjoin(build_lib, 'pyarrow'))
             move_shared_libs(build_prefix, build_lib, "arrow")
             move_shared_libs(build_prefix, build_lib, "arrow_python")
             if self.with_plasma:
@@ -232,6 +228,12 @@ def _run_cmake(self):
             if self.with_parquet:
                 move_shared_libs(build_prefix, build_lib, "parquet")
 
+        print('Bundling includes: ' + pjoin(build_prefix, 'include'))
+        if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
+            shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
+        shutil.move(pjoin(build_prefix, 'include'),
+                    pjoin(build_lib, 'pyarrow'))
+
         # Move the built C-extension to the place expected by the Python build
         self._found_names = []
         for name in self.CYTHON_MODULE_NAMES:

From f40618d19152b039f343b7341dd4b0a03c221323 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 22 Oct 2017 21:41:44 -0400
Subject: [PATCH 1171/1644] ARROW-1671: [C++] Deprecate arrow::MakeArray that
 returns Status, refactor existing code to new variant

We should be very strict to not return Status from functions that cannot fail

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1227 from wesm/ARROW-1671 and squashes the following commits:

8a15a068 [Wes McKinney] Deprecate arrow::MakeArray that returns Status, refactor existing code to use new MakeArray returning std::shared_ptr<Array>
---
 cpp/src/arrow/array.cc                 | 27 ++++++++++++++++++--------
 cpp/src/arrow/array.h                  | 12 ++++++++++++
 cpp/src/arrow/builder.cc               |  3 ++-
 cpp/src/arrow/compute/cast.cc          |  3 ++-
 cpp/src/arrow/compute/compute-test.cc  |  4 ++--
 cpp/src/arrow/ipc/feather.cc           |  6 ++++--
 cpp/src/arrow/python/numpy_to_arrow.cc |  8 +++-----
 cpp/src/arrow/table.cc                 |  2 +-
 8 files changed, 45 insertions(+), 20 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index a7930a139c75f..eaac187a38dff 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -97,9 +97,7 @@ static inline std::shared_ptr<ArrayData> SliceData(const ArrayData& data, int64_
 }
 
 std::shared_ptr<Array> Array::Slice(int64_t offset, int64_t length) const {
-  std::shared_ptr<Array> result;
-  DCHECK(MakeArray(SliceData(*data_, offset, length), &result).ok());
-  return result;
+  return MakeArray(SliceData(*data_, offset, length));
 }
 
 std::shared_ptr<Array> Array::Slice(int64_t offset) const {
@@ -210,7 +208,7 @@ void ListArray::SetData(const std::shared_ptr<ArrayData>& data) {
   raw_value_offsets_ = value_offsets == nullptr
                            ? nullptr
                            : reinterpret_cast<const int32_t*>(value_offsets->data());
-  DCHECK(MakeArray(data_->child_data[0], &values_).ok());
+  values_ = MakeArray(data_->child_data[0]);
 }
 
 std::shared_ptr<DataType> ListArray::value_type() const {
@@ -323,7 +321,7 @@ StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
 
 std::shared_ptr<Array> StructArray::field(int i) const {
   if (!boxed_fields_[i]) {
-    DCHECK(MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
+    boxed_fields_[i] = MakeArray(data_->child_data[i]);
   }
   DCHECK(boxed_fields_[i]);
   return boxed_fields_[i];
@@ -369,7 +367,7 @@ UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
 
 std::shared_ptr<Array> UnionArray::child(int i) const {
   if (!boxed_fields_[i]) {
-    DCHECK(MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
+    boxed_fields_[i] = MakeArray(data_->child_data[i]);
   }
   DCHECK(boxed_fields_[i]);
   return boxed_fields_[i];
@@ -377,7 +375,7 @@ std::shared_ptr<Array> UnionArray::child(int i) const {
 
 const Array* UnionArray::UnsafeChild(int i) const {
   if (!boxed_fields_[i]) {
-    DCHECK(MakeArray(data_->child_data[i], &boxed_fields_[i]).ok());
+    boxed_fields_[i] = MakeArray(data_->child_data[i]);
   }
   DCHECK(boxed_fields_[i]);
   return boxed_fields_[i].get();
@@ -407,7 +405,7 @@ void DictionaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
   auto indices_data = data_->ShallowCopy();
   indices_data->type = dict_type_->index_type();
   std::shared_ptr<Array> result;
-  DCHECK(MakeArray(indices_data, &indices_).ok());
+  indices_ = MakeArray(indices_data);
 }
 
 std::shared_ptr<Array> DictionaryArray::indices() const { return indices_; }
@@ -589,6 +587,8 @@ class ArrayDataWrapper {
 
 }  // namespace internal
 
+#ifndef ARROW_NO_DEPRECATED_API
+
 Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out) {
   internal::ArrayDataWrapper wrapper_visitor(data, out);
   RETURN_NOT_OK(VisitTypeInline(*data->type, &wrapper_visitor));
@@ -596,6 +596,17 @@ Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>*
   return Status::OK();
 }
 
+#endif
+
+std::shared_ptr<Array> MakeArray(const std::shared_ptr<ArrayData>& data) {
+  std::shared_ptr<Array> out;
+  internal::ArrayDataWrapper wrapper_visitor(data, &out);
+  Status s = VisitTypeInline(*data->type, &wrapper_visitor);
+  DCHECK(s.ok());
+  DCHECK(out);
+  return out;
+}
+
 // ----------------------------------------------------------------------
 // Instantiate templates
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 0805cad3d286c..75dda4a754c87 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -144,13 +144,25 @@ struct ARROW_EXPORT ArrayData {
   std::vector<std::shared_ptr<ArrayData>> child_data;
 };
 
+#ifndef ARROW_NO_DEPRECATED_API
+
 /// \brief Create a strongly-typed Array instance from generic ArrayData
 /// \param[in] data the array contents
 /// \param[out] out the resulting Array instance
 /// \return Status
+///
+/// \note Deprecated since 0.8.0
 ARROW_EXPORT
 Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out);
 
+#endif
+
+/// \brief Create a strongly-typed Array instance from generic ArrayData
+/// \param[in] data the array contents
+/// \return the resulting Array instance
+ARROW_EXPORT
+std::shared_ptr<Array> MakeArray(const std::shared_ptr<ArrayData>& data);
+
 // ----------------------------------------------------------------------
 // User array accessor types
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 331de2d36e130..c910170dd5c87 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -102,7 +102,8 @@ Status ArrayBuilder::Advance(int64_t elements) {
 Status ArrayBuilder::Finish(std::shared_ptr<Array>* out) {
   std::shared_ptr<ArrayData> internal_data;
   RETURN_NOT_OK(FinishInternal(&internal_data));
-  return MakeArray(internal_data, out);
+  *out = MakeArray(internal_data);
+  return Status::OK();
 }
 
 Status ArrayBuilder::Reserve(int64_t elements) {
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index 2381e1ea3a4e8..e8bbfd3470670 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -717,7 +717,8 @@ Status Cast(FunctionContext* ctx, const Array& array,
   auto out_data = std::make_shared<ArrayData>(out_type, array.length());
 
   RETURN_NOT_OK(func->Call(ctx, array, out_data.get()));
-  return MakeArray(out_data, out);
+  *out = MakeArray(out_data);
+  return Status::OK();
 }
 
 }  // namespace compute
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 602acff2088d6..8a595178d057a 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -369,8 +369,8 @@ TEST_F(TestCast, PreallocatedMemory) {
   // Buffer address unchanged
   ASSERT_EQ(out_values.get(), out_data->buffers[1].get());
 
-  std::shared_ptr<Array> result, expected;
-  ASSERT_OK(MakeArray(out_data, &result));
+  std::shared_ptr<Array> result = MakeArray(out_data);
+  std::shared_ptr<Array> expected;
   ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, e1, &expected);
 
   AssertArraysEqual(*expected, *result);
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 97ed601adb58c..cea720bd01bc7 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -371,7 +371,8 @@ class TableReader::TableReaderImpl {
 
     auto arr_data =
         std::make_shared<ArrayData>(type, meta->length(), buffers, meta->null_count());
-    return MakeArray(arr_data, out);
+    *out = MakeArray(arr_data);
+    return Status::OK();
   }
 
   bool HasDescription() const { return metadata_->HasDescription(); }
@@ -490,7 +491,8 @@ static Status SanitizeUnsupportedTypes(const Array& values, std::shared_ptr<Arra
                                          values.null_bitmap(), values.null_count());
     return Status::OK();
   } else {
-    return MakeArray(values.data(), out);
+    *out = MakeArray(values.data());
+    return Status::OK();
   }
 }
 
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 27ee2302e99bd..2c89a9f613ab7 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -364,9 +364,7 @@ class NumPyConverter {
   }
 
   Status PushArray(const std::shared_ptr<ArrayData>& data) {
-    std::shared_ptr<Array> result;
-    RETURN_NOT_OK(MakeArray(data, &result));
-    out_arrays_.emplace_back(std::move(result));
+    out_arrays_.emplace_back(MakeArray(data));
     return Status::OK();
   }
 
@@ -495,8 +493,8 @@ static Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t len
   std::vector<std::shared_ptr<Buffer>> buffers = {nullptr, input};
   auto tmp_data = std::make_shared<ArrayData>(in_type, length, buffers, 0);
 
-  std::shared_ptr<Array> tmp_array, casted_array;
-  RETURN_NOT_OK(MakeArray(tmp_data, &tmp_array));
+  std::shared_ptr<Array> tmp_array = MakeArray(tmp_data);
+  std::shared_ptr<Array> casted_array;
 
   compute::FunctionContext context(pool);
   compute::CastOptions cast_options;
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index aac5dad1fbe1f..fe19bf4ce0b3f 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -192,7 +192,7 @@ RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows
 
 std::shared_ptr<Array> RecordBatch::column(int i) const {
   if (!boxed_columns_[i]) {
-    DCHECK(MakeArray(columns_[i], &boxed_columns_[i]).ok());
+    boxed_columns_[i] = MakeArray(columns_[i]);
   }
   DCHECK(boxed_columns_[i]);
   return boxed_columns_[i];

From 6209489004184d82d568b7505baf77c93ab21069 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 22 Oct 2017 21:42:47 -0400
Subject: [PATCH 1172/1644] ARROW-1708: [JS] Fix linter error

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1236 from xhochy/ARROW-1708 and squashes the following commits:

0695745e [Uwe L. Korn] ARROW-1708: [JS] Fix linter error
---
 js/src/reader/dictionary.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/js/src/reader/dictionary.ts b/js/src/reader/dictionary.ts
index 61698e80c0054..93a9ba76bba3a 100644
--- a/js/src/reader/dictionary.ts
+++ b/js/src/reader/dictionary.ts
@@ -31,6 +31,8 @@ export function readDictionary(field: Field | null,
     }
     for (let i = -1, n = field && field.childrenLength() || 0; ++i < n;) {
         let vector = readDictionary(field.children(i), batch, iterator, dictionaries);
-        if (vector) return vector;
+        if (vector) {
+            return vector;
+        }
     }
 }

From 8eb2a1b66afde615c223c95b944fb6d8cc685b22 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 22 Oct 2017 22:08:54 -0400
Subject: [PATCH 1173/1644] ARROW-1707: Update dev README after movement to
 GitBox

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1235 from xhochy/ARROW-1707 and squashes the following commits:

21a6734c [Uwe L. Korn] Add link to GitBox for account setup
d484bb06 [Uwe L. Korn] ARROW-1707: Update dev README after movement to GitBox
---
 dev/README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/dev/README.md b/dev/README.md
index 82f66f6ace2e2..f3e963cf491bb 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -22,13 +22,16 @@
 This directory contains scripts useful to developers when packaging,
 testing, or committing to Arrow.
 
-Merging a pull request requires being a committer on the project.
+Merging a pull request requires being a committer on the project. In addition
+you need to have linked your GitHub and ASF accounts on
+https://gitbox.apache.org/setup/ to be able to push to GitHub as the main
+remote.
 
 * How to merge a Pull request:
 have an apache and apache-github remote setup
 ```
 git remote add apache-github https://github.com/apache/arrow.git
-git remote add apache https://git-wip-us.apache.org/repos/asf/arrow.git
+git remote add apache git@github.com:apache/arrow.git
 ```
 run the following command
 ```

From 4eb38a227d100165c59ed784c1aad09eea41236f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Oct 2017 09:23:55 -0400
Subject: [PATCH 1174/1644] ARROW-571: [Python] Add unit test for incremental
 Parquet file building, improve docs

This was actually already documented. I improved the docstring for `ParquetWriter` and added a unit test to validate the user API. Added ParquetWriter to the API listing also

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1218 from wesm/ARROW-571 and squashes the following commits:

e4f44b47 [Wes McKinney] Use py27-compatible class __doc__ setting
bb16d9b5 [Wes McKinney] Add ParquetWriter to API listing
4cda8ab2 [Wes McKinney] Add unit test for incremental Parquet file building
---
 python/doc/source/api.rst                  |   1 +
 python/pyarrow/parquet.py                  | 138 ++++++++++++---------
 python/pyarrow/tests/test_parquet.py       |  29 +++++
 python/pyarrow/tests/test_serialization.py |  44 ++++---
 4 files changed, 134 insertions(+), 78 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 26ccb98edbbe3..6bceba3c650b6 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -312,6 +312,7 @@ Apache Parquet
 
    ParquetDataset
    ParquetFile
+   ParquetWriter
    read_table
    read_metadata
    read_pandas
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 1584b849ad7c4..b6a7b1244e0a2 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -202,17 +202,47 @@ def _sanitize_table(table, new_schema, flavor):
         return table
 
 
+_parquet_writer_arg_docs = """version : {"1.0", "2.0"}, default "1.0"
+    The Parquet format version, defaults to 1.0
+use_dictionary : bool or list
+    Specify if we should use dictionary encoding in general or only for
+    some columns.
+use_deprecated_int96_timestamps : boolean, default None
+    Write nanosecond resolution timestamps to INT96 Parquet
+    format. Defaults to False unless enabled by flavor argument
+coerce_timestamps : string, default None
+    Cast timestamps a particular resolution.
+    Valid values: {None, 'ms', 'us'}
+compression : str or dict
+    Specify the compression codec, either on a general basis or per-column.
+flavor : {'spark'}, default None
+    Sanitize schema or set other compatibility options for compatibility"""
+
+
 class ParquetWriter(object):
-    """
 
-    Parameters
-    ----------
-    where
-    schema
-    flavor : {'spark', ...}
-        Set options for compatibility with a particular reader
-    """
-    def __init__(self, where, schema, flavor=None, **options):
+    __doc__ = """
+Class for incrementally building a Parquet file for Arrow tables
+
+Parameters
+----------
+where : path or file-like object
+schema : arrow Schema
+{0}
+""".format(_parquet_writer_arg_docs)
+
+    def __init__(self, where, schema, flavor=None,
+                 version='1.0',
+                 use_dictionary=True,
+                 compression='snappy',
+                 use_deprecated_int96_timestamps=None, **options):
+        if use_deprecated_int96_timestamps is None:
+            # Use int96 timestamps for Spark
+            if flavor is not None and 'spark' in flavor:
+                use_deprecated_int96_timestamps = True
+            else:
+                use_deprecated_int96_timestamps = False
+
         self.flavor = flavor
         if flavor is not None:
             schema, self.schema_changed = _sanitize_schema(schema, flavor)
@@ -220,15 +250,29 @@ def __init__(self, where, schema, flavor=None, **options):
             self.schema_changed = False
 
         self.schema = schema
-        self.writer = _parquet.ParquetWriter(where, schema, **options)
+        self.writer = _parquet.ParquetWriter(
+            where, schema,
+            version=version,
+            compression=compression,
+            use_dictionary=use_dictionary,
+            use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
+            **options)
+        self.is_open = True
+
+    def __del__(self):
+        if self.is_open:
+            self.close()
 
     def write_table(self, table, row_group_size=None):
         if self.schema_changed:
             table = _sanitize_table(table, self.schema, self.flavor)
+        assert self.is_open
         self.writer.write_table(table, row_group_size=row_group_size)
 
     def close(self):
-        self.writer.close()
+        if self.is_open:
+            self.writer.close()
+            self.is_open = False
 
 
 def _get_pandas_index_columns(keyvalues):
@@ -857,52 +901,19 @@ def write_table(table, where, row_group_size=None, version='1.0',
                 use_deprecated_int96_timestamps=None,
                 coerce_timestamps=None,
                 flavor=None, **kwargs):
-    """
-    Write a Table to Parquet format
-
-    Parameters
-    ----------
-    table : pyarrow.Table
-    where: string or pyarrow.io.NativeFile
-    row_group_size : int, default None
-        The maximum number of rows in each Parquet RowGroup. As a default,
-        we will write a single RowGroup per file.
-    version : {"1.0", "2.0"}, default "1.0"
-        The Parquet format version, defaults to 1.0
-    use_dictionary : bool or list
-        Specify if we should use dictionary encoding in general or only for
-        some columns.
-    use_deprecated_int96_timestamps : boolean, default None
-        Write nanosecond resolution timestamps to INT96 Parquet
-        format. Defaults to False unless enabled by flavor argument
-    coerce_timestamps : string, default None
-        Cast timestamps a particular resolution.
-        Valid values: {None, 'ms', 'us'}
-    compression : str or dict
-        Specify the compression codec, either on a general basis or per-column.
-    flavor : {'spark'}, default None
-        Sanitize schema or set other compatibility options for compatibility
-    """
-    row_group_size = kwargs.get('chunk_size', row_group_size)
-
-    if use_deprecated_int96_timestamps is None:
-        # Use int96 timestamps for Spark
-        if flavor is not None and 'spark' in flavor:
-            use_deprecated_int96_timestamps = True
-        else:
-            use_deprecated_int96_timestamps = False
-
-    options = dict(
-        use_dictionary=use_dictionary,
-        compression=compression,
-        version=version,
-        use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
-        coerce_timestamps=coerce_timestamps)
+    row_group_size = kwargs.pop('chunk_size', row_group_size)
 
     writer = None
     try:
-        writer = ParquetWriter(where, table.schema, flavor=flavor,
-                               **options)
+        writer = ParquetWriter(
+            where, table.schema,
+            version=version,
+            flavor=flavor,
+            use_dictionary=use_dictionary,
+            coerce_timestamps=coerce_timestamps,
+            compression=compression,
+            use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
+            **kwargs)
         writer.write_table(table, row_group_size=row_group_size)
     except:
         if writer is not None:
@@ -917,6 +928,17 @@ def write_table(table, where, row_group_size=None, version='1.0',
         writer.close()
 
 
+write_table.__doc__ = """
+Write a Table to Parquet format
+
+Parameters
+----------
+table : pyarrow.Table
+where: string or pyarrow.io.NativeFile
+{0}
+""".format(_parquet_writer_arg_docs)
+
+
 def write_to_dataset(table, root_path, partition_cols=None,
                      filesystem=None, preserve_index=True, **kwargs):
     """
@@ -1013,12 +1035,10 @@ def write_metadata(schema, where, version='1.0',
         Cast timestamps a particular resolution.
         Valid values: {None, 'ms', 'us'}
     """
-    options = dict(
-        version=version,
+    writer = ParquetWriter(
+        where, schema, version=version,
         use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
-        coerce_timestamps=coerce_timestamps
-    )
-    writer = ParquetWriter(where, schema, **options)
+        coerce_timestamps=coerce_timestamps)
     writer.close()
 
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index deb4b3f35a677..09184cc056588 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -300,6 +300,35 @@ def test_pandas_parquet_native_file_roundtrip(tmpdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@parquet
+def test_parquet_incremental_file_build(tmpdir):
+    import pyarrow.parquet as pq
+
+    df = _test_dataframe(100)
+    df['unique_id'] = 0
+
+    arrow_table = pa.Table.from_pandas(df, preserve_index=False)
+    out = pa.BufferOutputStream()
+
+    writer = pq.ParquetWriter(out, arrow_table.schema, version='2.0')
+
+    frames = []
+    for i in range(10):
+        df['unique_id'] = i
+        arrow_table = pa.Table.from_pandas(df, preserve_index=False)
+        writer.write_table(arrow_table)
+
+        frames.append(df.copy())
+
+    writer.close()
+
+    buf = out.get_result()
+    result = _read_table(pa.BufferReader(buf))
+
+    expected = pd.concat(frames, ignore_index=True)
+    tm.assert_frame_equal(result.to_pandas(), expected)
+
+
 @parquet
 def test_read_pandas_column_subset(tmpdir):
     import pyarrow.parquet as pq
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 9321ebc34abbe..67798ac31dc38 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -266,30 +266,36 @@ def test_default_dict_serialization(large_memory_map):
 
 def test_numpy_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        for t in ["bool", "int8", "uint8", "int16", "uint16", "int32", "uint32",
-                  "float16", "float32", "float64"]:
+        for t in ["bool", "int8", "uint8", "int16", "uint16", "int32",
+                  "uint32", "float16", "float32", "float64"]:
             obj = np.random.randint(0, 10, size=(100, 100)).astype(t)
             serialization_roundtrip(obj, mmap)
 
 
 def test_datetime_serialization(large_memory_map):
-    data = [# Principia Mathematica published
-            datetime.datetime(year=1687, month=7, day=5),
-            # Some random date
-            datetime.datetime(year=1911, month=6, day=3, hour=4,
-                              minute=55, second=44),
-            # End of WWI
-            datetime.datetime(year=1918, month=11, day=11),
-            # Beginning of UNIX time
-            datetime.datetime(year=1970, month=1, day=1),
-            # The Berlin wall falls
-            datetime.datetime(year=1989, month=11, day=9),
-            # Another random date
-            datetime.datetime(year=2011, month=6, day=3, hour=4,
-                              minute=0, second=3),
-            # Another random date
-            datetime.datetime(year=1970, month=1, day=3, hour=4,
-                              minute=0, second=0)]
+    data = [
+        #  Principia Mathematica published
+        datetime.datetime(year=1687, month=7, day=5),
+
+        # Some random date
+        datetime.datetime(year=1911, month=6, day=3, hour=4,
+                          minute=55, second=44),
+        # End of WWI
+        datetime.datetime(year=1918, month=11, day=11),
+
+        # Beginning of UNIX time
+        datetime.datetime(year=1970, month=1, day=1),
+
+        # The Berlin wall falls
+        datetime.datetime(year=1989, month=11, day=9),
+
+        # Another random date
+        datetime.datetime(year=2011, month=6, day=3, hour=4,
+                          minute=0, second=3),
+        # Another random date
+        datetime.datetime(year=1970, month=1, day=3, hour=4,
+                          minute=0, second=0)
+    ]
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for d in data:
             serialization_roundtrip(d, mmap)

From 2b77b7ce7a9f46b8c0bd9d4bf751e7686cec0f95 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Oct 2017 17:57:13 -0400
Subject: [PATCH 1175/1644] ARROW-507: [C++] Complete ListArray::FromArrays
 implementation, add unit tests

In the event that the offsets array has nulls, this will backward-fill the offsets to compute the correct value sizes.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1224 from wesm/ARROW-507 and squashes the following commits:

9027c140 [Wes McKinney] Clean valid bits to remove trailing set bit
8d2cb512 [Wes McKinney] Implement / add tests for ListArray.from_arrays in Python
1c6a8702 [Wes McKinney] Complete C++ implementation, unit test for ListArray::FromArrays, handling of offsets with nulls
---
 cpp/src/arrow/array-test.cc        | 73 ++++++++++++++++++++++++++----
 cpp/src/arrow/array.cc             | 50 +++++++++++++++-----
 cpp/src/arrow/array.h              |  9 ++--
 python/pyarrow/array.pxi           | 37 +++++++++++++--
 python/pyarrow/tests/test_array.py | 16 +++++++
 python/pyarrow/types.pxi           | 27 ++++++++---
 6 files changed, 180 insertions(+), 32 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index ae9e9fd0091bd..168ef10573e77 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1946,7 +1946,7 @@ TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
 // ----------------------------------------------------------------------
 // List tests
 
-class TestListBuilder : public TestBuilder {
+class TestListArray : public TestBuilder {
  public:
   void SetUp() {
     TestBuilder::SetUp();
@@ -1973,7 +1973,7 @@ class TestListBuilder : public TestBuilder {
   std::shared_ptr<ListArray> result_;
 };
 
-TEST_F(TestListBuilder, Equality) {
+TEST_F(TestListArray, Equality) {
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder());
 
   std::shared_ptr<Array> array, equal_array, unequal_array;
@@ -2032,9 +2032,66 @@ TEST_F(TestListBuilder, Equality) {
   ASSERT_TRUE(array->RangeEquals(1, 5, 0, slice));
 }
 
-TEST_F(TestListBuilder, TestResize) {}
+TEST_F(TestListArray, TestResize) {}
 
-TEST_F(TestListBuilder, TestAppendNull) {
+TEST_F(TestListArray, TestFromArrays) {
+  std::shared_ptr<Array> offsets1, offsets2, offsets3, offsets4, values;
+
+  std::vector<bool> offsets_is_valid3 = {true, false, true, true};
+  std::vector<bool> offsets_is_valid4 = {true, true, false, true};
+
+  std::vector<bool> values_is_valid = {true, false, true, true, true, true};
+
+  std::vector<int32_t> offset1_values = {0, 2, 2, 6};
+  std::vector<int32_t> offset2_values = {0, 2, 6, 6};
+
+  std::vector<int8_t> values_values = {0, 1, 2, 3, 4, 5};
+  const int length = 3;
+
+  ArrayFromVector<Int32Type, int32_t>(offset1_values, &offsets1);
+  ArrayFromVector<Int32Type, int32_t>(offset2_values, &offsets2);
+
+  ArrayFromVector<Int32Type, int32_t>(offsets_is_valid3, offset1_values, &offsets3);
+  ArrayFromVector<Int32Type, int32_t>(offsets_is_valid4, offset2_values, &offsets4);
+
+  ArrayFromVector<Int8Type, int8_t>(values_is_valid, values_values, &values);
+
+  auto list_type = list(int8());
+
+  std::shared_ptr<Array> list1, list3, list4;
+  ASSERT_OK(ListArray::FromArrays(*offsets1, *values, pool_, &list1));
+  ASSERT_OK(ListArray::FromArrays(*offsets3, *values, pool_, &list3));
+  ASSERT_OK(ListArray::FromArrays(*offsets4, *values, pool_, &list4));
+
+  ListArray expected1(list_type, length, offsets1->data()->buffers[1], values,
+                      offsets1->data()->buffers[0], 0);
+  AssertArraysEqual(expected1, *list1);
+
+  // Use null bitmap from offsets3, but clean offsets from non-null version
+  ListArray expected3(list_type, length, offsets1->data()->buffers[1], values,
+                      offsets3->data()->buffers[0], 1);
+  AssertArraysEqual(expected3, *list3);
+
+  // Check that the last offset bit is zero
+  ASSERT_TRUE(BitUtil::BitNotSet(list3->null_bitmap()->data(), length + 1));
+
+  ListArray expected4(list_type, length, offsets2->data()->buffers[1], values,
+                      offsets4->data()->buffers[0], 1);
+  AssertArraysEqual(expected4, *list4);
+
+  // Test failure modes
+
+  std::shared_ptr<Array> tmp;
+
+  // Zero-length offsets
+  ASSERT_RAISES(Invalid,
+                ListArray::FromArrays(*offsets1->Slice(0, 0), *values, pool_, &tmp));
+
+  // Offsets not int32
+  ASSERT_RAISES(Invalid, ListArray::FromArrays(*values, *offsets1, pool_, &tmp));
+}
+
+TEST_F(TestListArray, TestAppendNull) {
   ASSERT_OK(builder_->AppendNull());
   ASSERT_OK(builder_->AppendNull());
 
@@ -2076,7 +2133,7 @@ void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& valu
   }
 }
 
-TEST_F(TestListBuilder, TestBasics) {
+TEST_F(TestListArray, TestBasics) {
   vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
   vector<int> lengths = {3, 0, 4};
   vector<uint8_t> is_valid = {1, 0, 1};
@@ -2098,7 +2155,7 @@ TEST_F(TestListBuilder, TestBasics) {
   ValidateBasicListArray(result_.get(), values, is_valid);
 }
 
-TEST_F(TestListBuilder, BulkAppend) {
+TEST_F(TestListArray, BulkAppend) {
   vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
   vector<int> lengths = {3, 0, 4};
   vector<uint8_t> is_valid = {1, 0, 1};
@@ -2115,7 +2172,7 @@ TEST_F(TestListBuilder, BulkAppend) {
   ValidateBasicListArray(result_.get(), values, is_valid);
 }
 
-TEST_F(TestListBuilder, BulkAppendInvalid) {
+TEST_F(TestListArray, BulkAppendInvalid) {
   vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
   vector<int> lengths = {3, 0, 4};
   vector<uint8_t> is_null = {0, 1, 0};
@@ -2135,7 +2192,7 @@ TEST_F(TestListBuilder, BulkAppendInvalid) {
   ASSERT_RAISES(Invalid, ValidateArray(*result_));
 }
 
-TEST_F(TestListBuilder, TestZeroLength) {
+TEST_F(TestListArray, TestZeroLength) {
   // All buffers are null
   Done();
   ASSERT_OK(ValidateArray(*result_));
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index eaac187a38dff..fc4b96e1b2bec 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -172,27 +172,55 @@ ListArray::ListArray(const std::shared_ptr<DataType>& type, int64_t length,
   SetData(internal_data);
 }
 
-Status ListArray::FromArrays(const Array& offsets, const Array& values,
-                             MemoryPool* ARROW_ARG_UNUSED(pool),
+Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPool* pool,
                              std::shared_ptr<Array>* out) {
-  if (ARROW_PREDICT_FALSE(offsets.length() == 0)) {
+  if (offsets.length() == 0) {
     return Status::Invalid("List offsets must have non-zero length");
   }
 
-  if (ARROW_PREDICT_FALSE(offsets.null_count() > 0)) {
-    return Status::Invalid("Null offsets in ListArray::FromArrays not yet implemented");
-  }
-
-  if (ARROW_PREDICT_FALSE(offsets.type_id() != Type::INT32)) {
+  if (offsets.type_id() != Type::INT32) {
     return Status::Invalid("List offsets must be signed int32");
   }
 
-  BufferVector buffers = {offsets.null_bitmap(),
-                          static_cast<const Int32Array&>(offsets).values()};
+  BufferVector buffers = {};
+
+  const auto& typed_offsets = static_cast<const Int32Array&>(offsets);
+
+  const int64_t num_offsets = offsets.length();
+
+  if (offsets.null_count() > 0) {
+    std::shared_ptr<Buffer> clean_offsets, clean_valid_bits;
+
+    RETURN_NOT_OK(AllocateBuffer(pool, num_offsets * sizeof(int32_t), &clean_offsets));
+
+    // Copy valid bits, zero out the bit for the final offset
+    RETURN_NOT_OK(offsets.null_bitmap()->Copy(0, BitUtil::BytesForBits(num_offsets - 1),
+                                              &clean_valid_bits));
+    BitUtil::ClearBit(clean_valid_bits->mutable_data(), num_offsets);
+    buffers.emplace_back(std::move(clean_valid_bits));
+
+    const int32_t* raw_offsets = typed_offsets.raw_values();
+    auto clean_raw_offsets = reinterpret_cast<int32_t*>(clean_offsets->mutable_data());
+
+    // Must work backwards so we can tell how many values were in the last non-null value
+    DCHECK(offsets.IsValid(num_offsets - 1));
+    int32_t current_offset = raw_offsets[num_offsets - 1];
+    for (int64_t i = num_offsets - 1; i >= 0; --i) {
+      if (offsets.IsValid(i)) {
+        current_offset = raw_offsets[i];
+      }
+      clean_raw_offsets[i] = current_offset;
+    }
+
+    buffers.emplace_back(std::move(clean_offsets));
+  } else {
+    buffers.emplace_back(offsets.null_bitmap());
+    buffers.emplace_back(typed_offsets.values());
+  }
 
   auto list_type = list(values.type());
   auto internal_data =
-      std::make_shared<ArrayData>(list_type, offsets.length() - 1, std::move(buffers),
+      std::make_shared<ArrayData>(list_type, num_offsets - 1, std::move(buffers),
                                   offsets.null_count(), offsets.offset());
   internal_data->child_data.push_back(values.data());
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 75dda4a754c87..b5d2530998f10 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -400,10 +400,13 @@ class ARROW_EXPORT ListArray : public Array {
 
   /// \brief Construct ListArray from array of offsets and child value array
   ///
-  /// Note: does not validate input beyond sanity checks. Use
-  /// arrow::ValidateArray if you need stronger validation of inputs
+  /// This function does the bare minimum of validation of the offsets and
+  /// input types, and will allocate a new offsets array if necessary (i.e. if
+  /// the offsets contain any nulls). If the offsets do not have nulls, they
+  /// are assumed to be well-formed
   ///
-  /// \param[in] offsets Array containing n + 1 offsets encoding length and size
+  /// \param[in] offsets Array containing n + 1 offsets encoding length and
+  /// size. Must be of int32 type
   /// \param[in] values Array containing
   /// \param[in] pool MemoryPool in case new offsets array needs to be
   /// allocated because of null values
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index f402defc9b06f..c5f28a9519bcc 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -173,6 +173,29 @@ def array(object obj, type=None, mask=None,
         return _sequence_to_array(obj, size, type, pool)
 
 
+def asarray(values, type=None):
+    """
+    Convert to pyarrow.Array, inferring type if not provided. Attempt to cast
+    if indicated type is different
+
+    Parameters
+    ----------
+    values : array-like (sequence, numpy.ndarray, pyarrow.Array)
+    type : string or DataType
+
+    Returns
+    -------
+    arr : Array
+    """
+    if isinstance(values, Array):
+        if type is not None and not values.type.equals(type):
+            values = values.cast(type)
+
+        return values
+    else:
+        return array(values, type=type)
+
+
 def _normalize_slice(object arrow_obj, slice key):
     cdef Py_ssize_t n = len(arrow_obj)
 
@@ -574,7 +597,7 @@ cdef class DecimalArray(FixedSizeBinaryArray):
 cdef class ListArray(Array):
 
     @staticmethod
-    def from_arrays(Array offsets, Array values, MemoryPool pool=None):
+    def from_arrays(offsets, values, MemoryPool pool=None):
         """
         Construct ListArray from arrays of int32 offsets and values
 
@@ -587,11 +610,17 @@ cdef class ListArray(Array):
         -------
         list_array : ListArray
         """
-        cdef shared_ptr[CArray] out
+        cdef:
+            Array _offsets, _values
+            shared_ptr[CArray] out
         cdef CMemoryPool* cpool = maybe_unbox_memory_pool(pool)
+
+        _offsets = asarray(offsets, type='int32')
+        _values = asarray(values)
+
         with nogil:
-            check_status(CListArray.FromArrays(
-                deref(offsets.ap), deref(values.ap), cpool, &out))
+            check_status(CListArray.FromArrays(_offsets.ap[0], _values.ap[0],
+                                               cpool, &out))
         return pyarrow_wrap_array(out)
 
 
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 414a268ce1bb8..418076f819684 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -218,6 +218,22 @@ def test_list_from_arrays():
 
     assert result.equals(expected)
 
+    # With nulls
+    offsets = [0, None, 2, 6]
+
+    values = ['a', 'b', 'c', 'd', 'e', 'f']
+
+    result = pa.ListArray.from_arrays(offsets, values)
+    expected = pa.array([values[:2], None, values[2:]])
+
+    assert result.equals(expected)
+
+    # Another edge case
+    offsets2 = [0, 2, None, 6]
+    result = pa.ListArray.from_arrays(offsets2, values)
+    expected = pa.array([values[:2], values[2:], None])
+    assert result.equals(expected)
+
 
 def _check_cast_case(case, safe=True):
     in_data, in_type, out_data, out_type = case
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 0bef1aa608a4f..3d838ba39fbb8 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -73,7 +73,27 @@ cdef class DataType:
         return '{0.__class__.__name__}({0})'.format(self)
 
     def __richcmp__(DataType self, object other, int op):
+        if op == cp.Py_EQ:
+            return self.equals(other)
+        elif op == cp.Py_NE:
+            return not self.equals(other)
+        else:
+            raise TypeError('Invalid comparison')
+
+    def equals(self, other):
+        """
+        Return true if type is equivalent to passed value
+
+        Parameters
+        ----------
+        other : DataType or string convertible to DataType
+
+        Returns
+        -------
+        is_equal : boolean
+        """
         cdef DataType other_type
+
         if not isinstance(other, DataType):
             if not isinstance(other, six.string_types):
                 raise TypeError(other)
@@ -81,12 +101,7 @@ cdef class DataType:
         else:
             other_type = other
 
-        if op == cp.Py_EQ:
-            return self.type.Equals(deref(other_type.type))
-        elif op == cp.Py_NE:
-            return not self.type.Equals(deref(other_type.type))
-        else:
-            raise TypeError('Invalid comparison')
+        return self.type.Equals(deref(other_type.type))
 
     def to_pandas_dtype(self):
         """

From 8e00ee993a158ba444cf5a4b75e7ad24756a7fbb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Oct 2017 17:58:32 -0400
Subject: [PATCH 1176/1644] ARROW-1114: [C++] Add simple RecordBatchBuilder
 class

This patch includes ARROW-1172 which I put up separately. Putting this up for comments on the API before I go too far down the rabbit hole. The idea is to make it simpler for users to construct record batches given a known schema. For example, this could be used in turbodbc or another database driver

closes #810 incidentally

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #802 from wesm/ARROW-1114 and squashes the following commits:

5f104c4e [Wes McKinney] Rename Create to Make, remove const method versions
a8b6c5cc [Wes McKinney] Update doxygen comments, change FlushAndReset to instead have Flush with a flag for resetting
2573ae7e [Wes McKinney] Test invalid field length
256419fc [Wes McKinney] Basic test passing
be7e2325 [Wes McKinney] Start table_builder-test.cc
9660502a [Wes McKinney] Draft RecordBatchBuilder, no tests or benchmarks yet
---
 cpp/src/arrow/CMakeLists.txt               |   3 +
 cpp/src/arrow/api.h                        |   1 +
 cpp/src/arrow/ipc/json-integration-test.cc |   4 +-
 cpp/src/arrow/table_builder-test.cc        | 146 +++++++++++++++++++++
 cpp/src/arrow/table_builder.cc             | 101 ++++++++++++++
 cpp/src/arrow/table_builder.h              | 111 ++++++++++++++++
 cpp/src/arrow/test-util.h                  |  14 ++
 7 files changed, 378 insertions(+), 2 deletions(-)
 create mode 100644 cpp/src/arrow/table_builder-test.cc
 create mode 100644 cpp/src/arrow/table_builder.cc
 create mode 100644 cpp/src/arrow/table_builder.h

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 9a676510e4bf9..22b475146da7c 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -24,6 +24,7 @@ set(ARROW_SRCS
   pretty_print.cc
   status.cc
   table.cc
+  table_builder.cc
   tensor.cc
   type.cc
   visitor.cc
@@ -143,6 +144,7 @@ install(FILES
   pretty_print.h
   status.h
   table.h
+  table_builder.h
   tensor.h
   type.h
   type_fwd.h
@@ -172,6 +174,7 @@ ADD_ARROW_TEST(pretty_print-test)
 ADD_ARROW_TEST(status-test)
 ADD_ARROW_TEST(type-test)
 ADD_ARROW_TEST(table-test)
+ADD_ARROW_TEST(table_builder-test)
 ADD_ARROW_TEST(tensor-test)
 
 ADD_ARROW_BENCHMARK(builder-benchmark)
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 4d731bd32bf1b..5d2e859f3a4be 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -28,6 +28,7 @@
 #include "arrow/pretty_print.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
+#include "arrow/table_builder.h"
 #include "arrow/tensor.h"
 #include "arrow/type.h"
 #include "arrow/visitor.h"
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 4a3b2b399741e..c7530a467b398 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -182,10 +182,10 @@ static Status ValidateArrowVsJson(const std::string& arrow_path,
       ss << "Record batch " << i << " did not match";
 
       ss << "\nJSON:\n";
-      RETURN_NOT_OK(PrettyPrint(*json_batch.get(), 0, &ss));
+      RETURN_NOT_OK(PrettyPrint(*json_batch, 0, &ss));
 
       ss << "\nArrow:\n";
-      RETURN_NOT_OK(PrettyPrint(*arrow_batch.get(), 0, &ss));
+      RETURN_NOT_OK(PrettyPrint(*arrow_batch, 0, &ss));
       return Status::Invalid(ss.str());
     }
   }
diff --git a/cpp/src/arrow/table_builder-test.cc b/cpp/src/arrow/table_builder-test.cc
new file mode 100644
index 0000000000000..07d9b6b2d6568
--- /dev/null
+++ b/cpp/src/arrow/table_builder-test.cc
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/status.h"
+#include "arrow/table.h"
+#include "arrow/table_builder.h"
+#include "arrow/test-common.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+class TestRecordBatchBuilder : public TestBase {
+ public:
+};
+
+std::shared_ptr<Schema> ExampleSchema1() {
+  auto f0 = field("f0", int32());
+  auto f1 = field("f1", utf8());
+  auto f2 = field("f1", list(int8()));
+  return ::arrow::schema({f0, f1, f2});
+}
+
+template <typename BuilderType, typename T>
+void AppendValues(BuilderType* builder, const std::vector<T>& values,
+                  const std::vector<bool>& is_valid) {
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid.size() == 0 || is_valid[i]) {
+      ASSERT_OK(builder->Append(values[i]));
+    } else {
+      ASSERT_OK(builder->AppendNull());
+    }
+  }
+}
+
+template <typename ValueType, typename T>
+void AppendList(ListBuilder* builder, const std::vector<std::vector<T>>& values,
+                const std::vector<bool>& is_valid) {
+  auto values_builder = static_cast<ValueType*>(builder->value_builder());
+
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid.size() == 0 || is_valid[i]) {
+      ASSERT_OK(builder->Append());
+      AppendValues<ValueType, T>(values_builder, values[i], {});
+    } else {
+      ASSERT_OK(builder->AppendNull());
+    }
+  }
+}
+
+TEST_F(TestRecordBatchBuilder, Basics) {
+  auto schema = ExampleSchema1();
+
+  std::unique_ptr<RecordBatchBuilder> builder;
+  ASSERT_OK(RecordBatchBuilder::Make(schema, pool_, &builder));
+
+  std::vector<bool> is_valid = {false, true, true, true};
+  std::vector<int32_t> f0_values = {0, 1, 2, 3};
+  std::vector<std::string> f1_values = {"a", "bb", "ccc", "dddd"};
+  std::vector<std::vector<int8_t>> f2_values = {{}, {0, 1}, {}, {2}};
+
+  std::shared_ptr<Array> a0, a1, a2;
+
+  // Make the expected record batch
+  auto AppendData = [&](Int32Builder* b0, StringBuilder* b1, ListBuilder* b2) {
+    AppendValues<Int32Builder, int32_t>(b0, f0_values, is_valid);
+    AppendValues<StringBuilder, std::string>(b1, f1_values, is_valid);
+    AppendList<Int8Builder, int8_t>(b2, f2_values, is_valid);
+  };
+
+  Int32Builder ex_b0;
+  StringBuilder ex_b1;
+  ListBuilder ex_b2(pool_, std::unique_ptr<Int8Builder>(new Int8Builder(pool_)));
+
+  AppendData(&ex_b0, &ex_b1, &ex_b2);
+  ASSERT_OK(ex_b0.Finish(&a0));
+  ASSERT_OK(ex_b1.Finish(&a1));
+  ASSERT_OK(ex_b2.Finish(&a2));
+
+  RecordBatch expected(schema, 4, {a0, a1, a2});
+
+  // Builder attributes
+  ASSERT_EQ(3, builder->num_fields());
+  ASSERT_EQ(schema.get(), builder->schema().get());
+
+  const int kIter = 3;
+  for (int i = 0; i < kIter; ++i) {
+    AppendData(builder->GetFieldAs<Int32Builder>(0),
+               static_cast<StringBuilder*>(builder->GetField(1)),
+               builder->GetFieldAs<ListBuilder>(2));
+
+    std::shared_ptr<RecordBatch> batch;
+
+    if (i == kIter - 1) {
+      // Do not flush in last iteration
+      ASSERT_OK(builder->Flush(false, &batch));
+    } else {
+      ASSERT_OK(builder->Flush(&batch));
+    }
+
+    ASSERT_BATCHES_EQUAL(expected, *batch);
+  }
+
+  // Test setting initial capacity
+  builder->SetInitialCapacity(4096);
+  ASSERT_EQ(4096, builder->initial_capacity());
+}
+
+TEST_F(TestRecordBatchBuilder, InvalidFieldLength) {
+  auto schema = ExampleSchema1();
+
+  std::unique_ptr<RecordBatchBuilder> builder;
+  ASSERT_OK(RecordBatchBuilder::Make(schema, pool_, &builder));
+
+  std::vector<bool> is_valid = {false, true, true, true};
+  std::vector<int32_t> f0_values = {0, 1, 2, 3};
+
+  AppendValues<Int32Builder, int32_t>(builder->GetFieldAs<Int32Builder>(0), f0_values,
+                                      is_valid);
+
+  std::shared_ptr<RecordBatch> dummy;
+  ASSERT_RAISES(Invalid, builder->Flush(&dummy));
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/table_builder.cc b/cpp/src/arrow/table_builder.cc
new file mode 100644
index 0000000000000..a1bd95940a6db
--- /dev/null
+++ b/cpp/src/arrow/table_builder.cc
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/table_builder.h"
+
+#include <algorithm>
+#include <cstdlib>
+#include <memory>
+#include <sstream>
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/status.h"
+#include "arrow/table.h"
+#include "arrow/type.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+// ----------------------------------------------------------------------
+// RecordBatchBuilder
+
+RecordBatchBuilder::RecordBatchBuilder(const std::shared_ptr<Schema>& schema,
+                                       MemoryPool* pool, int64_t initial_capacity)
+    : schema_(schema), initial_capacity_(initial_capacity), pool_(pool) {}
+
+Status RecordBatchBuilder::Make(const std::shared_ptr<Schema>& schema, MemoryPool* pool,
+                                std::unique_ptr<RecordBatchBuilder>* builder) {
+  return Make(schema, pool, kMinBuilderCapacity, builder);
+}
+
+Status RecordBatchBuilder::Make(const std::shared_ptr<Schema>& schema, MemoryPool* pool,
+                                int64_t initial_capacity,
+                                std::unique_ptr<RecordBatchBuilder>* builder) {
+  builder->reset(new RecordBatchBuilder(schema, pool, initial_capacity));
+  RETURN_NOT_OK((*builder)->CreateBuilders());
+  return (*builder)->InitBuilders();
+}
+
+Status RecordBatchBuilder::Flush(bool reset_builders,
+                                 std::shared_ptr<RecordBatch>* batch) {
+  std::vector<std::shared_ptr<Array>> fields;
+  fields.resize(this->num_fields());
+
+  int64_t length = 0;
+  for (int i = 0; i < this->num_fields(); ++i) {
+    RETURN_NOT_OK(raw_field_builders_[i]->Finish(&fields[i]));
+    if (i > 0 && fields[i]->length() != length) {
+      return Status::Invalid("All fields must be same length when calling Flush");
+    }
+    length = fields[i]->length();
+  }
+  *batch = std::make_shared<RecordBatch>(schema_, length, std::move(fields));
+  if (reset_builders) {
+    return InitBuilders();
+  } else {
+    return Status::OK();
+  }
+}
+
+Status RecordBatchBuilder::Flush(std::shared_ptr<RecordBatch>* batch) {
+  return Flush(true, batch);
+}
+
+void RecordBatchBuilder::SetInitialCapacity(int64_t capacity) {
+  DCHECK_GT(capacity, 0) << "Initial capacity must be positive";
+  initial_capacity_ = capacity;
+}
+
+Status RecordBatchBuilder::CreateBuilders() {
+  field_builders_.resize(this->num_fields());
+  raw_field_builders_.resize(this->num_fields());
+  for (int i = 0; i < this->num_fields(); ++i) {
+    RETURN_NOT_OK(MakeBuilder(pool_, schema_->field(i)->type(), &field_builders_[i]));
+    raw_field_builders_[i] = field_builders_[i].get();
+  }
+  return Status::OK();
+}
+
+Status RecordBatchBuilder::InitBuilders() {
+  for (int i = 0; i < this->num_fields(); ++i) {
+    RETURN_NOT_OK(raw_field_builders_[i]->Init(initial_capacity_));
+  }
+  return Status::OK();
+}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/table_builder.h b/cpp/src/arrow/table_builder.h
new file mode 100644
index 0000000000000..582389b70fe53
--- /dev/null
+++ b/cpp/src/arrow/table_builder.h
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_TABLE_BUILDER_H
+#define ARROW_TABLE_BUILDER_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class ArrayBuilder;
+class MemoryPool;
+class RecordBatch;
+class Schema;
+
+/// \class RecordBatchBuilder
+/// \brief Helper class for creating record batches iteratively given a known
+/// schema
+class RecordBatchBuilder {
+ public:
+  /// \brief Create an initialize a RecordBatchBuilder
+  /// \param[in] schema The schema for the record batch
+  /// \param[in] pool A MemoryPool to use for allocations
+  /// \param[in] builder the created builder instance
+  static Status Make(const std::shared_ptr<Schema>& schema, MemoryPool* pool,
+                     std::unique_ptr<RecordBatchBuilder>* builder);
+
+  /// \brief Create an initialize a RecordBatchBuilder
+  /// \param[in] schema The schema for the record batch
+  /// \param[in] pool A MemoryPool to use for allocations
+  /// \param[in] initial_capacity The initial capacity for the builders
+  /// \param[in] builder the created builder instance
+  static Status Make(const std::shared_ptr<Schema>& schema, MemoryPool* pool,
+                     int64_t initial_capacity,
+                     std::unique_ptr<RecordBatchBuilder>* builder);
+
+  /// \brief Get base pointer to field builder
+  /// \param i the field index
+  /// \return pointer to ArrayBuilder
+  ArrayBuilder* GetField(int i) { return raw_field_builders_[i]; }
+
+  /// \brief Return field builder casted to indicated specific builder type
+  /// \param i the field index
+  /// \return pointer to template type
+  template <typename T>
+  T* GetFieldAs(int i) {
+    return static_cast<T*>(raw_field_builders_[i]);
+  }
+
+  /// \brief Finish current batch and optionally reset
+  /// \param[in] reset_builders the resulting RecordBatch
+  /// \param[out] batch the resulting RecordBatch
+  /// \return Status
+  Status Flush(bool reset_builders, std::shared_ptr<RecordBatch>* batch);
+
+  /// \brief Finish current batch and reset
+  /// \param[out] batch the resulting RecordBatch
+  /// \return Status
+  Status Flush(std::shared_ptr<RecordBatch>* batch);
+
+  /// \brief Set the initial capacity for new builders
+  void SetInitialCapacity(int64_t capacity);
+
+  /// \brief The initial capacity for builders
+  int64_t initial_capacity() const { return initial_capacity_; }
+
+  /// \brief The number of fields in the schema
+  int num_fields() const { return schema_->num_fields(); }
+
+  /// \brief The number of fields in the schema
+  std::shared_ptr<Schema> schema() const { return schema_; }
+
+ private:
+  RecordBatchBuilder(const std::shared_ptr<Schema>& schema, MemoryPool* pool,
+                     int64_t initial_capacity);
+
+  Status CreateBuilders();
+  Status InitBuilders();
+
+  std::shared_ptr<Schema> schema_;
+  int64_t initial_capacity_;
+  MemoryPool* pool_;
+
+  std::vector<std::unique_ptr<ArrayBuilder>> field_builders_;
+  std::vector<ArrayBuilder*> raw_field_builders_;
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_TABLE_BUILDER_H
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 80e4feb6c329a..83ebdea4a8543 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -22,6 +22,7 @@
 #include <limits>
 #include <memory>
 #include <random>
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -291,6 +292,19 @@ void AssertArraysEqual(const Array& expected, const Array& actual) {
   }
 }
 
+#define ASSERT_BATCHES_EQUAL(LEFT, RIGHT)    \
+  do {                                       \
+    if (!LEFT.ApproxEquals(RIGHT)) {         \
+      std::stringstream ss;                  \
+      ss << "Left:\n";                       \
+      ASSERT_OK(PrettyPrint(LEFT, 0, &ss));  \
+                                             \
+      ss << "\nRight:\n";                    \
+      ASSERT_OK(PrettyPrint(RIGHT, 0, &ss)); \
+      FAIL() << ss.str();                    \
+    }                                        \
+  } while (false)
+
 }  // namespace arrow
 
 #endif  // ARROW_TEST_UTIL_H_

From 935a3cf2a4b94d9599204e0be600e407eeb88ca7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 23 Oct 2017 18:18:46 -0400
Subject: [PATCH 1177/1644] ARROW-1654: [Python] Implement pickling for
 DataType, Field, Schema

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1238 from wesm/ARROW-1654 and squashes the following commits:

2e6f9e32 [Wes McKinney] Add pickling test cases for timestamp, decimal
1827b231 [Wes McKinney] Fix pickling on py27, implement for Schema. Also pickle field/schema metadata
1395583b [Wes McKinney] Implement pickling for list, struct, add __richcmp__ for Field
366f428f [Wes McKinney] Start implementing pickling for DataType, Field
---
 python/pyarrow/includes/libarrow.pxd |   1 +
 python/pyarrow/tests/test_schema.py  |  39 +++++++++
 python/pyarrow/tests/test_types.py   |   1 -
 python/pyarrow/types.pxi             | 120 +++++++++++++++++++++++++--
 4 files changed, 155 insertions(+), 6 deletions(-)

diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 60aa4d6940a80..0e5d4a8eddcb1 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -182,6 +182,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CListType(const shared_ptr[CDataType]& value_type)
         CListType(const shared_ptr[CField]& field)
         shared_ptr[CDataType] value_type()
+        shared_ptr[CField] value_field()
 
     cdef cppclass CStringType" arrow::StringType"(CDataType):
         pass
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index c77be98054c6f..d6b2655b7c6a0 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import pickle
+
 import pytest
 import numpy as np
 
@@ -304,3 +306,40 @@ def test_schema_repr_with_dictionaries():
 two: int32""")
 
     assert repr(sch) == expected
+
+
+def test_type_schema_pickling():
+    cases = [
+        pa.int8(),
+        pa.string(),
+        pa.binary(),
+        pa.binary(10),
+        pa.list_(pa.string()),
+        pa.struct([
+            pa.field('a', 'int8'),
+            pa.field('b', 'string')
+        ]),
+        pa.time32('s'),
+        pa.time64('us'),
+        pa.date32(),
+        pa.date64(),
+        pa.timestamp('ms'),
+        pa.timestamp('ns'),
+        pa.decimal(12, 2),
+        pa.field('a', 'string', metadata={b'foo': b'bar'})
+    ]
+
+    for val in cases:
+        roundtripped = pickle.loads(pickle.dumps(val))
+        assert val == roundtripped
+
+    fields = []
+    for i, f in enumerate(cases):
+        if isinstance(f, pa.Field):
+            fields.append(f)
+        else:
+            fields.append(pa.field('_f{}'.format(i), f))
+
+    schema = pa.schema(fields, metadata={b'foo': b'bar'})
+    roundtripped = pickle.loads(pickle.dumps(schema))
+    assert schema == roundtripped
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index d8eea622ccc87..e6ff5b1560c1d 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-
 import pyarrow as pa
 import pyarrow.types as types
 
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 3d838ba39fbb8..686e56ead342d 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -69,6 +69,16 @@ cdef class DataType:
             )
         return frombytes(self.type.ToString())
 
+    def __reduce__(self):
+        return self.__class__, (), self.__getstate__()
+
+    def __getstate__(self):
+        return str(self),
+
+    def __setstate__(self, state):
+        cdef DataType reconstituted = type_for_alias(state[0])
+        self.init(reconstituted.sp_type)
+
     def __repr__(self):
         return '{0.__class__.__name__}({0})'.format(self)
 
@@ -132,6 +142,15 @@ cdef class ListType(DataType):
         DataType.init(self, type)
         self.list_type = <const CListType*> type.get()
 
+    def __getstate__(self):
+        cdef CField* field = self.list_type.value_field().get()
+        name = field.name()
+        return name, self.value_type
+
+    def __setstate__(self, state):
+        cdef DataType reconstituted = list_(field(state[0], state[1]))
+        self.init(reconstituted.sp_type)
+
     property value_type:
 
         def __get__(self):
@@ -143,6 +162,25 @@ cdef class StructType(DataType):
     cdef void init(self, const shared_ptr[CDataType]& type):
         DataType.init(self, type)
 
+    def __getitem__(self, i):
+        if i < 0 or i >= self.num_children:
+            raise IndexError(i)
+
+        return pyarrow_wrap_field(self.type.child(i))
+
+    property num_children:
+
+        def __get__(self):
+            return self.type.num_children()
+
+    def __getstate__(self):
+        cdef CStructType* type = <CStructType*> self.sp_type.get()
+        return [self[i] for i in range(self.num_children)]
+
+    def __setstate__(self, state):
+        cdef DataType reconstituted = struct(state)
+        self.init(reconstituted.sp_type)
+
 
 cdef class UnionType(DataType):
 
@@ -211,6 +249,13 @@ cdef class FixedSizeBinaryType(DataType):
         self.fixed_size_binary_type = (
             <const CFixedSizeBinaryType*> type.get())
 
+    def __getstate__(self):
+        return self.byte_width
+
+    def __setstate__(self, state):
+        cdef DataType reconstituted = binary(state)
+        self.init(reconstituted.sp_type)
+
     property byte_width:
 
         def __get__(self):
@@ -223,6 +268,13 @@ cdef class DecimalType(FixedSizeBinaryType):
         DataType.init(self, type)
         self.decimal_type = <const CDecimalType*> type.get()
 
+    def __getstate__(self):
+        return (self.precision, self.scale)
+
+    def __setstate__(self, state):
+        cdef DataType reconstituted = decimal(*state)
+        self.init(reconstituted.sp_type)
+
     property precision:
 
         def __get__(self):
@@ -257,6 +309,24 @@ cdef class Field:
         """
         return self.field.Equals(deref(other.field))
 
+    def __richcmp__(Field self, Field other, int op):
+        if op == cp.Py_EQ:
+            return self.equals(other)
+        elif op == cp.Py_NE:
+            return not self.equals(other)
+        else:
+            raise TypeError('Invalid comparison')
+
+    def __reduce__(self):
+        return Field, (), self.__getstate__()
+
+    def __getstate__(self):
+        return (self.name, self.type, self.metadata)
+
+    def __setstate__(self, state):
+        cdef Field reconstituted = field(state[0], state[1], metadata=state[2])
+        self.init(reconstituted.sp_field)
+
     def __str__(self):
         self._check_null()
         return 'pyarrow.Field<{0}>'.format(frombytes(self.field.ToString()))
@@ -369,6 +439,16 @@ cdef class Schema:
         self.schema = schema.get()
         self.sp_schema = schema
 
+    def __reduce__(self):
+        return Schema, (), self.__getstate__()
+
+    def __getstate__(self):
+        return ([self[i] for i in range(len(self))], self.metadata)
+
+    def __setstate__(self, state):
+        cdef Schema reconstituted = schema(state[0], metadata=state[1])
+        self.init_schema(reconstituted.sp_schema)
+
     property names:
 
         def __get__(self):
@@ -387,6 +467,14 @@ cdef class Schema:
                 self.schema.metadata())
             return box_metadata(metadata.get())
 
+    def __richcmp__(self, other, int op):
+        if op == cp.Py_EQ:
+            return self.equals(other)
+        elif op == cp.Py_NE:
+            return not self.equals(other)
+        else:
+            raise TypeError('Invalid comparison')
+
     def equals(self, other):
         """
         Test if this schema is equal to the other
@@ -533,7 +621,7 @@ cdef int convert_metadata(dict metadata,
     return 0
 
 
-def field(name, DataType type, bint nullable=True, dict metadata=None):
+def field(name, type, bint nullable=True, dict metadata=None):
     """
     Create a pyarrow.Field instance
 
@@ -552,17 +640,29 @@ def field(name, DataType type, bint nullable=True, dict metadata=None):
     cdef:
         shared_ptr[CKeyValueMetadata] c_meta
         Field result = Field()
+        DataType _type
 
     if metadata is not None:
         convert_metadata(metadata, &c_meta)
 
-    result.sp_field.reset(new CField(tobytes(name), type.sp_type,
+    _type = _as_type(type)
+
+    result.sp_field.reset(new CField(tobytes(name), _type.sp_type,
                                      nullable == 1, c_meta))
     result.field = result.sp_field.get()
-    result.type = type
+    result.type = _type
     return result
 
 
+cdef _as_type(type):
+    if isinstance(type, DataType):
+        return type
+    if not isinstance(type, six.string_types):
+        raise TypeError(type)
+    return type_for_alias(type)
+
+
+
 cdef set PRIMITIVE_TYPES = set([
     _Type_NA, _Type_BOOL,
     _Type_UINT8, _Type_INT8,
@@ -985,6 +1085,8 @@ cdef dict _type_aliases = {
     'binary': binary,
     'date32': date32,
     'date64': date64,
+    'date32[day]': date32,
+    'date64[ms]': date64,
     'time32[s]': time32('s'),
     'time32[ms]': time32('ms'),
     'time64[us]': time64('us'),
@@ -1015,19 +1117,23 @@ def type_for_alias(name):
     return alias()
 
 
-def schema(fields):
+def schema(fields, dict metadata=None):
     """
     Construct pyarrow.Schema from collection of fields
 
     Parameters
     ----------
     field : list or iterable
+    metadata : dict, default None
+        Keys and values must be coercible to bytes
 
     Returns
     -------
     schema : pyarrow.Schema
     """
     cdef:
+        shared_ptr[CKeyValueMetadata] c_meta
+        shared_ptr[CSchema] c_schema
         Schema result
         Field field
         vector[shared_ptr[CField]] c_fields
@@ -1035,8 +1141,12 @@ def schema(fields):
     for i, field in enumerate(fields):
         c_fields.push_back(field.sp_field)
 
+    if metadata is not None:
+        convert_metadata(metadata, &c_meta)
+
+    c_schema.reset(new CSchema(c_fields, c_meta))
     result = Schema()
-    result.init(c_fields)
+    result.init_schema(c_schema)
     return result
 
 

From e876e175dc4d7655aaefd32ed144b6bbca54c2da Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Tue, 24 Oct 2017 12:41:47 -0400
Subject: [PATCH 1178/1644] ARROW-1720: [Python] Implement bounds check in
 chunk getter

This closes [ARROW-1720](https://issues.apache.org/jira/browse/ARROW-1720).

Author: Licht-T <licht-t@outlook.jp>

Closes #1243 from Licht-T/fix-unbound-chunk and squashes the following commits:

cabdd437 [Licht-T] TST: Add bounds check tests for chunk getter
bda7f4ce [Licht-T] BUG: Implement bounds check in chunk getter
---
 python/pyarrow/table.pxi           | 4 ++++
 python/pyarrow/tests/test_table.py | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index dd42cf231e7e2..694fe919044c1 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -102,6 +102,10 @@ cdef class ChunkedArray:
         pyarrow.Array
         """
         self._check_nullptr()
+
+        if i >= self.num_chunks or i < 0:
+            raise IndexError('Chunk index out of range.')
+
         return pyarrow_wrap_array(self.chunked_array.chunk(i))
 
     def iterchunks(self):
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 4a2868a3c1de3..50190f597b972 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -211,6 +211,12 @@ def test_table_basics():
         for chunk in col.data.iterchunks():
             assert chunk is not None
 
+        with pytest.raises(IndexError):
+            col.data.chunk(-1)
+
+        with pytest.raises(IndexError):
+            col.data.chunk(col.data.num_chunks)
+
 
 def test_table_from_arrays_invalid_names():
     data = [

From b08f7e326b8fc1a000a277eef3bc84a0106d6c3c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 24 Oct 2017 12:57:23 -0400
Subject: [PATCH 1179/1644] ARROW-1711: [Python] Fix flake8 calls to lint the
 right directories

This got messed up during one of the patches in which these files were refactored. Once the build fails, I will fix the lint errors

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1242 from wesm/ARROW-1711 and squashes the following commits:

cd4b655d [Wes McKinney] Fix more flake8 warnings
2eb8bf49 [Wes McKinney] Fix flake8 issues
cef7a7c9 [Wes McKinney] Fix flake8 calls to lint the right directories
---
 ci/travis_lint.sh                          |  6 ++--
 python/pyarrow/array.pxi                   |  2 +-
 python/pyarrow/feather.py                  |  2 +-
 python/pyarrow/io-hdfs.pxi                 |  4 +--
 python/pyarrow/parquet.py                  |  2 +-
 python/pyarrow/serialization.py            | 16 ++-------
 python/pyarrow/tests/test_feather.py       |  2 +-
 python/pyarrow/tests/test_serialization.py | 41 ++++++++++++----------
 8 files changed, 34 insertions(+), 41 deletions(-)

diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
index 9b7b474524fb4..8c956646cb39e 100755
--- a/ci/travis_lint.sh
+++ b/ci/travis_lint.sh
@@ -31,10 +31,10 @@ popd
 # Fail fast on style checks
 sudo pip install flake8
 
-PYARROW_DIR=$TRAVIS_BUILD_DIR/python
+PYARROW_DIR=$TRAVIS_BUILD_DIR/python/pyarrow
 
-flake8 --count $PYTHON_DIR/pyarrow
+flake8 --count $PYARROW_DIR
 
 # Check Cython files with some checks turned off
 flake8 --count --config=$PYTHON_DIR/.flake8.cython \
-       $PYTHON_DIR/pyarrow
+       $PYARROW_DIR
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index c5f28a9519bcc..c596d2ad8e7c5 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -47,7 +47,7 @@ cdef _is_array_like(obj):
     try:
         import pandas
         return isinstance(obj, (np.ndarray, pd.Series, pd.Index, Categorical))
-    except:
+    except ImportError:
         return isinstance(obj, np.ndarray)
 
 
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index aba76a008a072..2091c9154fdd5 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -116,7 +116,7 @@ def write_feather(df, dest):
     writer = FeatherWriter(dest)
     try:
         writer.write(df)
-    except:
+    except Exception:
         # Try to make sure the resource is closed
         import gc
         writer = None
diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi
index e6285e465d2be..e653813235862 100644
--- a/python/pyarrow/io-hdfs.pxi
+++ b/python/pyarrow/io-hdfs.pxi
@@ -32,7 +32,7 @@ def have_libhdfs():
         with nogil:
             check_status(HaveLibHdfs())
         return True
-    except:
+    except Exception:
         return False
 
 
@@ -41,7 +41,7 @@ def have_libhdfs3():
         with nogil:
             check_status(HaveLibHdfs3())
         return True
-    except:
+    except Exception:
         return False
 
 
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index b6a7b1244e0a2..0a40f5fb7fdce 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -915,7 +915,7 @@ def write_table(table, where, row_group_size=None, version='1.0',
             use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
             **kwargs)
         writer.write_table(table, row_group_size=row_group_size)
-    except:
+    except Exception:
         if writer is not None:
             writer.close()
         if isinstance(where, six.string_types):
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index eed6aae83736f..9dc8ee6dee9ad 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -20,10 +20,10 @@
 
 import numpy as np
 
-import pyarrow as pa
 from pyarrow import serialize_pandas, deserialize_pandas
 from pyarrow.lib import _default_serialization_context
 
+
 def register_default_serialization_handlers(serialization_context):
 
     # ----------------------------------------------------------------------
@@ -43,58 +43,48 @@ def register_default_serialization_handlers(serialization_context):
             custom_serializer=lambda obj: str(obj),
             custom_deserializer=lambda data: long(data))  # noqa: F821
 
-
     def _serialize_ordered_dict(obj):
         return list(obj.keys()), list(obj.values())
 
-
     def _deserialize_ordered_dict(data):
         return OrderedDict(zip(data[0], data[1]))
 
-
     serialization_context.register_type(
         OrderedDict, "OrderedDict",
         custom_serializer=_serialize_ordered_dict,
         custom_deserializer=_deserialize_ordered_dict)
 
-
     def _serialize_default_dict(obj):
         return list(obj.keys()), list(obj.values()), obj.default_factory
 
-
     def _deserialize_default_dict(data):
         return defaultdict(data[2], zip(data[0], data[1]))
 
-
     serialization_context.register_type(
         defaultdict, "defaultdict",
         custom_serializer=_serialize_default_dict,
         custom_deserializer=_deserialize_default_dict)
 
-
     serialization_context.register_type(
         type(lambda: 0), "function",
         pickle=True)
 
     # ----------------------------------------------------------------------
     # Set up serialization for numpy with dtype object (primitive types are
-    # handled efficiently with Arrow's Tensor facilities, see python_to_arrow.cc)
-
+    # handled efficiently with Arrow's Tensor facilities, see
+    # python_to_arrow.cc)
 
     def _serialize_numpy_array(obj):
         return obj.tolist(), obj.dtype.str
 
-
     def _deserialize_numpy_array(data):
         return np.array(data[0], dtype=np.dtype(data[1]))
 
-
     serialization_context.register_type(
         np.ndarray, 'np.array',
         custom_serializer=_serialize_numpy_array,
         custom_deserializer=_deserialize_numpy_array)
 
-
     # ----------------------------------------------------------------------
     # Set up serialization for pandas Series and DataFrame
 
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 76f0844fa4843..810ee3c8cf8ac 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -289,7 +289,7 @@ def test_delete_partial_file_on_error(self):
         path = random_path()
         try:
             write_feather(df, path)
-        except:
+        except Exception:
             pass
 
         assert not os.path.exists(path)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 67798ac31dc38..7878a09228d06 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -62,7 +62,7 @@ def assert_equal(obj1, obj2):
             # Workaround to make comparison of OrderedDicts work on Python 2.7
             if obj1 == obj2:
                 return
-        except:
+        except Exception:
             pass
         if obj1.__dict__ == {}:
             print("WARNING: Empty dict in ", obj1)
@@ -300,6 +300,7 @@ def test_datetime_serialization(large_memory_map):
         for d in data:
             serialization_roundtrip(d, mmap)
 
+
 def test_torch_serialization(large_memory_map):
     pytest.importorskip("torch")
     import torch
@@ -311,6 +312,7 @@ def test_torch_serialization(large_memory_map):
             obj = torch.from_numpy(np.random.randn(1000).astype(t))
             serialization_roundtrip(obj, mmap)
 
+
 def test_numpy_immutable(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         obj = np.zeros([10])
@@ -342,6 +344,7 @@ def deserialize_dummy_class(serialized_obj):
 
     pa.serialize(DummyClass())
 
+
 def test_buffer_serialization():
 
     class BufferClass(object):
@@ -371,24 +374,24 @@ def huge_memory_map(temp_dir):
         # Test that objects that are too large for Arrow throw a Python
         # exception. These tests give out of memory errors on Travis and need
         # to be run on a machine with lots of RAM.
-        l = 2 ** 29 * [1.0]
-        serialization_roundtrip(l, mmap)
-        del l
-        l = 2 ** 29 * ["s"]
-        serialization_roundtrip(l, mmap)
-        del l
-        l = 2 ** 29 * [["1"], 2, 3, [{"s": 4}]]
-        serialization_roundtrip(l, mmap)
-        del l
-        l = 2 ** 29 * [{"s": 1}] + 2 ** 29 * [1.0]
-        serialization_roundtrip(l, mmap)
-        del l
-        l = np.zeros(2 ** 25)
-        serialization_roundtrip(l, mmap)
-        del l
-        l = [np.zeros(2 ** 18) for _ in range(2 ** 7)]
-        serialization_roundtrip(l, mmap)
-        del l
+        x = 2 ** 29 * [1.0]
+        serialization_roundtrip(x, mmap)
+        del x
+        x = 2 ** 29 * ["s"]
+        serialization_roundtrip(x, mmap)
+        del x
+        x = 2 ** 29 * [["1"], 2, 3, [{"s": 4}]]
+        serialization_roundtrip(x, mmap)
+        del x
+        x = 2 ** 29 * [{"s": 1}] + 2 ** 29 * [1.0]
+        serialization_roundtrip(x, mmap)
+        del x
+        x = np.zeros(2 ** 25)
+        serialization_roundtrip(x, mmap)
+        del x
+        x = [np.zeros(2 ** 18) for _ in range(2 ** 7)]
+        serialization_roundtrip(x, mmap)
+        del x
 
 
 def test_serialization_callback_error():

From ecb7605a640dc34452dd696240a74df9a9f7f39b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 25 Oct 2017 09:08:38 -0400
Subject: [PATCH 1180/1644] ARROW-1134: [C++] Support for C++/CLI compilation,
 add NULLPTR define to avoid using nullptr in public headers

cc @tobyshaw. Can you test this?

Close #1098

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1228 from wesm/ARROW-1134 and squashes the following commits:

bf181584 [Wes McKinney] Only define NULLPTR if not already defined
a51dd885 [Wes McKinney] Add NULLPTR macro to avoid using nullptr in public headers for C++/CLI users
---
 cpp/src/arrow/allocator.h             |  5 ++--
 cpp/src/arrow/array.h                 | 36 +++++++++++++--------------
 cpp/src/arrow/buffer.h                | 12 ++++-----
 cpp/src/arrow/builder.h               | 20 +++++++--------
 cpp/src/arrow/io/hdfs.h               |  6 ++---
 cpp/src/arrow/ipc/message.h           |  6 ++---
 cpp/src/arrow/python/common.h         | 10 ++++----
 cpp/src/arrow/python/helpers.h        |  5 ++--
 cpp/src/arrow/python/numpy_to_arrow.h |  2 +-
 cpp/src/arrow/table.h                 |  4 +--
 cpp/src/arrow/type.h                  | 22 ++++++++--------
 cpp/src/arrow/util/decimal.h          |  3 ++-
 cpp/src/arrow/util/io-util.h          |  2 +-
 cpp/src/arrow/util/macros.h           | 15 +++++++++++
 14 files changed, 83 insertions(+), 65 deletions(-)

diff --git a/cpp/src/arrow/allocator.h b/cpp/src/arrow/allocator.h
index e00023dc460fb..c7780f19e1141 100644
--- a/cpp/src/arrow/allocator.h
+++ b/cpp/src/arrow/allocator.h
@@ -24,6 +24,7 @@
 
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
+#include "arrow/util/macros.h"
 
 namespace arrow {
 
@@ -49,13 +50,13 @@ class stl_allocator {
   template <class U>
   stl_allocator(const stl_allocator<U>& rhs) noexcept : pool_(rhs.pool_) {}
 
-  ~stl_allocator() { pool_ = nullptr; }
+  ~stl_allocator() { pool_ = NULLPTR; }
 
   pointer address(reference r) const noexcept { return std::addressof(r); }
 
   const_pointer address(const_reference r) const noexcept { return std::addressof(r); }
 
-  pointer allocate(size_type n, const void* /*hint*/ = nullptr) {
+  pointer allocate(size_type n, const void* /*hint*/ = NULLPTR) {
     uint8_t* data;
     Status s = pool_->Allocate(n * sizeof(T), &data);
     if (!s.ok()) throw std::bad_alloc();
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index b5d2530998f10..afbd780dd3ad5 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -183,14 +183,14 @@ class ARROW_EXPORT Array {
 
   /// \brief Return true if value at index is null. Does not boundscheck
   bool IsNull(int64_t i) const {
-    return null_bitmap_data_ != nullptr &&
+    return null_bitmap_data_ != NULLPTR &&
            BitUtil::BitNotSet(null_bitmap_data_, i + data_->offset);
   }
 
   /// \brief Return true if value at index is valid (not null). Does not
   /// boundscheck
   bool IsValid(int64_t i) const {
-    return null_bitmap_data_ != nullptr &&
+    return null_bitmap_data_ != NULLPTR &&
            BitUtil::GetBit(null_bitmap_data_, i + data_->offset);
   }
 
@@ -212,13 +212,13 @@ class ARROW_EXPORT Array {
 
   /// Buffer for the null bitmap.
   ///
-  /// Note that for `null_count == 0`, this can be a `nullptr`.
+  /// Note that for `null_count == 0`, this can be null.
   /// This buffer does not account for any slice offset
   std::shared_ptr<Buffer> null_bitmap() const { return data_->buffers[0]; }
 
   /// Raw pointer to the null bitmap.
   ///
-  /// Note that for `null_count == 0`, this can be a `nullptr`.
+  /// Note that for `null_count == 0`, this can be null.
   /// This buffer does not account for any slice offset
   const uint8_t* null_bitmap_data() const { return null_bitmap_data_; }
 
@@ -270,7 +270,7 @@ class ARROW_EXPORT Array {
     if (data->buffers.size() > 0 && data->buffers[0]) {
       null_bitmap_data_ = data->buffers[0]->data();
     } else {
-      null_bitmap_data_ = nullptr;
+      null_bitmap_data_ = NULLPTR;
     }
     data_ = data;
   }
@@ -299,7 +299,7 @@ class ARROW_EXPORT NullArray : public FlatArray {
 
  private:
   inline void SetData(const std::shared_ptr<ArrayData>& data) {
-    null_bitmap_data_ = nullptr;
+    null_bitmap_data_ = NULLPTR;
     data->null_count = data->length;
     data_ = data;
   }
@@ -310,7 +310,7 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray {
  public:
   PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length,
                  const std::shared_ptr<Buffer>& data,
-                 const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+                 const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
                  int64_t null_count = 0, int64_t offset = 0);
 
   /// Does not account for any slice offset
@@ -325,7 +325,7 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray {
   inline void SetData(const std::shared_ptr<ArrayData>& data) {
     auto values = data->buffers[1];
     this->Array::SetData(data);
-    raw_values_ = values == nullptr ? nullptr : values->data();
+    raw_values_ = values == NULLPTR ? NULLPTR : values->data();
   }
 
   explicit inline PrimitiveArray(const std::shared_ptr<ArrayData>& data) {
@@ -349,7 +349,7 @@ class ARROW_EXPORT NumericArray : public PrimitiveArray {
   NumericArray(
       typename std::enable_if<TypeTraits<T1>::is_parameter_free, int64_t>::type length,
       const std::shared_ptr<Buffer>& data,
-      const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+      const std::shared_ptr<Buffer>& null_bitmap = NULLPTR, int64_t null_count = 0,
       int64_t offset = 0)
       : PrimitiveArray(TypeTraits<T1>::type_singleton(), length, data, null_bitmap,
                        null_count, offset) {}
@@ -371,7 +371,7 @@ class ARROW_EXPORT BooleanArray : public PrimitiveArray {
   explicit BooleanArray(const std::shared_ptr<ArrayData>& data);
 
   BooleanArray(int64_t length, const std::shared_ptr<Buffer>& data,
-               const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+               const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
                int64_t null_count = 0, int64_t offset = 0);
 
   bool Value(int64_t i) const {
@@ -395,7 +395,7 @@ class ARROW_EXPORT ListArray : public Array {
   ListArray(const std::shared_ptr<DataType>& type, int64_t length,
             const std::shared_ptr<Buffer>& value_offsets,
             const std::shared_ptr<Array>& values,
-            const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+            const std::shared_ptr<Buffer>& null_bitmap = NULLPTR, int64_t null_count = 0,
             int64_t offset = 0);
 
   /// \brief Construct ListArray from array of offsets and child value array
@@ -451,7 +451,7 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
 
   BinaryArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
               const std::shared_ptr<Buffer>& data,
-              const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+              const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
               int64_t null_count = 0, int64_t offset = 0);
 
   // Return the pointer to the given elements bytes
@@ -503,7 +503,7 @@ class ARROW_EXPORT BinaryArray : public FlatArray {
   BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
               const std::shared_ptr<Buffer>& value_offsets,
               const std::shared_ptr<Buffer>& data,
-              const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+              const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
               int64_t null_count = 0, int64_t offset = 0);
 
   const int32_t* raw_value_offsets_;
@@ -518,7 +518,7 @@ class ARROW_EXPORT StringArray : public BinaryArray {
 
   StringArray(int64_t length, const std::shared_ptr<Buffer>& value_offsets,
               const std::shared_ptr<Buffer>& data,
-              const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+              const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
               int64_t null_count = 0, int64_t offset = 0);
 
   // Construct a std::string
@@ -541,7 +541,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 
   FixedSizeBinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
                        const std::shared_ptr<Buffer>& data,
-                       const std::shared_ptr<Buffer>& null_bitmap = nullptr,
+                       const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
                        int64_t null_count = 0, int64_t offset = 0);
 
   const uint8_t* GetValue(int64_t i) const;
@@ -583,7 +583,7 @@ class ARROW_EXPORT StructArray : public Array {
 
   StructArray(const std::shared_ptr<DataType>& type, int64_t length,
               const std::vector<std::shared_ptr<Array>>& children,
-              std::shared_ptr<Buffer> null_bitmap = nullptr, int64_t null_count = 0,
+              std::shared_ptr<Buffer> null_bitmap = NULLPTR, int64_t null_count = 0,
               int64_t offset = 0);
 
   // Return a shared pointer in case the requestor desires to share ownership
@@ -608,8 +608,8 @@ class ARROW_EXPORT UnionArray : public Array {
   UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
              const std::vector<std::shared_ptr<Array>>& children,
              const std::shared_ptr<Buffer>& type_ids,
-             const std::shared_ptr<Buffer>& value_offsets = nullptr,
-             const std::shared_ptr<Buffer>& null_bitmap = nullptr, int64_t null_count = 0,
+             const std::shared_ptr<Buffer>& value_offsets = NULLPTR,
+             const std::shared_ptr<Buffer>& null_bitmap = NULLPTR, int64_t null_count = 0,
              int64_t offset = 0);
 
   /// Note that this buffer does not account for any slice offset
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index b745812b10366..8e989064be4e1 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -112,7 +112,7 @@ class ARROW_EXPORT Buffer {
   int64_t size_;
   int64_t capacity_;
 
-  // nullptr by default, but may be set
+  // null by default, but may be set
   std::shared_ptr<Buffer> parent_;
 
  private:
@@ -145,7 +145,7 @@ class ARROW_EXPORT MutableBuffer : public Buffer {
                 const int64_t size);
 
  protected:
-  MutableBuffer() : Buffer(nullptr, 0) {}
+  MutableBuffer() : Buffer(NULLPTR, 0) {}
 };
 
 class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
@@ -180,7 +180,7 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
 /// A Buffer whose lifetime is tied to a particular MemoryPool
 class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
  public:
-  explicit PoolBuffer(MemoryPool* pool = nullptr);
+  explicit PoolBuffer(MemoryPool* pool = NULLPTR);
   virtual ~PoolBuffer();
 
   Status Resize(const int64_t new_size, bool shrink_to_fit = true) override;
@@ -193,7 +193,7 @@ class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
 class ARROW_EXPORT BufferBuilder {
  public:
   explicit BufferBuilder(MemoryPool* pool)
-      : pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
+      : pool_(pool), data_(NULLPTR), capacity_(0), size_(0) {}
 
   /// Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
   Status Resize(const int64_t elements) {
@@ -201,7 +201,7 @@ class ARROW_EXPORT BufferBuilder {
     if (elements == 0) {
       return Status::OK();
     }
-    if (buffer_ == nullptr) {
+    if (buffer_ == NULLPTR) {
       buffer_ = std::make_shared<PoolBuffer>(pool_);
     }
     int64_t old_capacity = capacity_;
@@ -264,7 +264,7 @@ class ARROW_EXPORT BufferBuilder {
   }
 
   void Reset() {
-    buffer_ = nullptr;
+    buffer_ = NULLPTR;
     capacity_ = size_ = 0;
   }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 1720c00145c4e..c580eeb3b35fb 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -59,9 +59,9 @@ class ARROW_EXPORT ArrayBuilder {
   explicit ArrayBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
       : type_(type),
         pool_(pool),
-        null_bitmap_(nullptr),
+        null_bitmap_(NULLPTR),
         null_count_(0),
-        null_bitmap_data_(nullptr),
+        null_bitmap_data_(NULLPTR),
         length_(0),
         capacity_(0) {}
 
@@ -188,7 +188,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   using value_type = typename Type::c_type;
 
   explicit PrimitiveBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
-      : ArrayBuilder(type, pool), data_(nullptr), raw_data_(nullptr) {}
+      : ArrayBuilder(type, pool), data_(NULLPTR), raw_data_(NULLPTR) {}
 
   using ArrayBuilder::Advance;
 
@@ -214,7 +214,7 @@ class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
   /// indicates a valid (non-null) value
   /// \return Status
   Status Append(const value_type* values, int64_t length,
-                const uint8_t* valid_bytes = nullptr);
+                const uint8_t* valid_bytes = NULLPTR);
 
   /// \brief Append a sequence of elements in one shot
   /// \param[in] values a contiguous C array of values
@@ -430,7 +430,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
   /// indicates a valid (non-null) value
   /// \return Status
   Status Append(const uint64_t* values, int64_t length,
-                const uint8_t* valid_bytes = nullptr);
+                const uint8_t* valid_bytes = NULLPTR);
 
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
@@ -492,7 +492,7 @@ class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase
   /// indicates a valid (non-null) value
   /// \return Status
   Status Append(const int64_t* values, int64_t length,
-                const uint8_t* valid_bytes = nullptr);
+                const uint8_t* valid_bytes = NULLPTR);
 
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
@@ -557,7 +557,7 @@ class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
   /// indicates a valid (non-null) value
   /// \return Status
   Status Append(const uint8_t* values, int64_t length,
-                const uint8_t* valid_bytes = nullptr);
+                const uint8_t* valid_bytes = NULLPTR);
 
   /// \brief Append a sequence of elements in one shot
   /// \param[in] values a contiguous C array of values
@@ -624,7 +624,7 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   /// Use this constructor to incrementally build the value array along with offsets and
   /// null bitmap.
   ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
-              const std::shared_ptr<DataType>& type = nullptr);
+              const std::shared_ptr<DataType>& type = NULLPTR);
 
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
@@ -635,7 +635,7 @@ class ARROW_EXPORT ListBuilder : public ArrayBuilder {
   /// If passed, valid_bytes is of equal length to values, and any zero byte
   /// will be considered as a null for that slot
   Status Append(const int32_t* offsets, int64_t length,
-                const uint8_t* valid_bytes = nullptr);
+                const uint8_t* valid_bytes = NULLPTR);
 
   /// \brief Start a new variable-length list slot
   ///
@@ -732,7 +732,7 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
   }
 
   Status Append(const uint8_t* data, int64_t length,
-                const uint8_t* valid_bytes = nullptr);
+                const uint8_t* valid_bytes = NULLPTR);
   Status Append(const std::string& value);
   Status AppendNull();
 
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index aaaafc819f217..0708b11cca550 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -121,8 +121,8 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
   /// Change
   ///
   /// @param path file path to change
-  /// @param owner pass nullptr for no change
-  /// @param group pass nullptr for no change
+  /// @param owner pass null for no change
+  /// @param group pass null for no change
   Status Chown(const std::string& path, const char* owner, const char* group);
 
   /// Change path permissions
@@ -199,7 +199,7 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
   void set_memory_pool(MemoryPool* pool);
 
  private:
-  explicit HdfsReadableFile(MemoryPool* pool = nullptr);
+  explicit HdfsReadableFile(MemoryPool* pool = NULLPTR);
 
   class ARROW_NO_EXPORT HdfsReadableFileImpl;
   std::unique_ptr<HdfsReadableFileImpl> impl_;
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index 67a95c7d24761..a1b6c07a43d0e 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -69,7 +69,7 @@ class ARROW_EXPORT Message {
   /// \brief Create and validate a Message instance from two buffers
   ///
   /// \param[in] metadata a buffer containing the Flatbuffer metadata
-  /// \param[in] body a buffer containing the message body, which may be nullptr
+  /// \param[in] body a buffer containing the message body, which may be null
   /// \param[out] out the created message
   /// \return Status
   static Status Open(const std::shared_ptr<Buffer>& metadata,
@@ -98,7 +98,7 @@ class ARROW_EXPORT Message {
 
   /// \brief the Message body, if any
   ///
-  /// \return buffer is nullptr if no body
+  /// \return buffer is null if no body
   std::shared_ptr<Buffer> body() const;
 
   /// \brief The Message type
@@ -179,7 +179,7 @@ Status ReadMessage(const int64_t offset, const int32_t metadata_length,
 
 /// \brief Read encapulated RPC message (metadata and body) from InputStream
 ///
-/// Read length-prefixed message with as-yet unknown length. Returns nullptr if
+/// Read length-prefixed message with as-yet unknown length. Returns null if
 /// there are not enough bytes available or the message length is 0 (e.g. EOS
 /// in a stream)
 ARROW_EXPORT
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index e3fe2ef429a19..146864ffdc26e 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -63,7 +63,7 @@ class ARROW_EXPORT PyAcquireGIL {
 
 class ARROW_EXPORT OwnedRef {
  public:
-  OwnedRef() : obj_(nullptr) {}
+  OwnedRef() : obj_(NULLPTR) {}
 
   explicit OwnedRef(PyObject* obj) : obj_(obj) {}
 
@@ -82,7 +82,7 @@ class ARROW_EXPORT OwnedRef {
 
   void release() {
     Py_XDECREF(obj_);
-    obj_ = nullptr;
+    obj_ = NULLPTR;
   }
 
   PyObject* obj() const { return obj_; }
@@ -96,7 +96,7 @@ class ARROW_EXPORT OwnedRef {
 // reference count when release is called.
 class ARROW_EXPORT ScopedRef {
  public:
-  ScopedRef() : obj_(nullptr) {}
+  ScopedRef() : obj_(NULLPTR) {}
 
   explicit ScopedRef(PyObject* obj) : obj_(obj) {}
 
@@ -109,7 +109,7 @@ class ARROW_EXPORT ScopedRef {
 
   PyObject* release() {
     PyObject* result = obj_;
-    obj_ = nullptr;
+    obj_ = NULLPTR;
     return result;
   }
 
@@ -137,7 +137,7 @@ struct ARROW_EXPORT PyObjectStringify {
       bytes = PyBytes_AsString(obj);
       size = PyBytes_GET_SIZE(obj);
     } else {
-      bytes = nullptr;
+      bytes = NULLPTR;
       size = -1;
     }
   }
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index 01ab91657d9fc..719ed796e08a1 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -25,6 +25,7 @@
 #include <utility>
 
 #include "arrow/type.h"
+#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -43,8 +44,8 @@ Status ImportFromModule(const OwnedRef& module, const std::string& module_name,
 
 Status PythonDecimalToString(PyObject* python_decimal, std::string* out);
 
-Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision = nullptr,
-                                     int* scale = nullptr);
+Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision = NULLPTR,
+                                     int* scale = NULLPTR);
 
 PyObject* DecimalFromString(PyObject* decimal_constructor,
                             const std::string& decimal_string);
diff --git a/cpp/src/arrow/python/numpy_to_arrow.h b/cpp/src/arrow/python/numpy_to_arrow.h
index 4a870fff9fcf3..5bcbea32548cf 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.h
+++ b/cpp/src/arrow/python/numpy_to_arrow.h
@@ -37,7 +37,7 @@ class Status;
 namespace py {
 
 /// Convert NumPy arrays to Arrow. If target data type is not known, pass a
-/// type with nullptr
+/// type with null
 ///
 /// \param[in] pool Memory pool for any memory allocations
 /// \param[in] ao an ndarray with the array data
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index d40bdb8569307..d3145ff107ae0 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -288,10 +288,10 @@ class ARROW_EXPORT RecordBatchReader {
   /// \return the shared schema of the record batches in the stream
   virtual std::shared_ptr<Schema> schema() const = 0;
 
-  /// Read the next record batch in the stream. Return nullptr for batch when
+  /// Read the next record batch in the stream. Return null for batch when
   /// reaching end of stream
   ///
-  /// \param[out] batch the next loaded batch, nullptr at end of stream
+  /// \param[out] batch the next loaded batch, null at end of stream
   /// \return Status
   virtual Status ReadNext(std::shared_ptr<RecordBatch>* batch) = 0;
 };
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 443828423e7dc..2030f371d4faf 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -241,7 +241,7 @@ class ARROW_EXPORT Field {
  public:
   Field(const std::string& name, const std::shared_ptr<DataType>& type,
         bool nullable = true,
-        const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr)
+        const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR)
       : name_(name), type_(type), nullable_(nullable), metadata_(metadata) {}
 
   std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
@@ -737,10 +737,10 @@ class ARROW_EXPORT DictionaryType : public FixedWidthType {
 class ARROW_EXPORT Schema {
  public:
   explicit Schema(const std::vector<std::shared_ptr<Field>>& fields,
-                  const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+                  const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
 
   explicit Schema(std::vector<std::shared_ptr<Field>>&& fields,
-                  const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+                  const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
 
   virtual ~Schema() = default;
 
@@ -750,7 +750,7 @@ class ARROW_EXPORT Schema {
   /// Return the ith schema element. Does not boundscheck
   std::shared_ptr<Field> field(int i) const { return fields_[i]; }
 
-  /// Returns nullptr if name not found
+  /// Returns null if name not found
   std::shared_ptr<Field> GetFieldByName(const std::string& name) const;
 
   /// Returns -1 if name not found
@@ -760,7 +760,7 @@ class ARROW_EXPORT Schema {
 
   /// \brief The custom key-value metadata, if any
   ///
-  /// \return metadata may be nullptr
+  /// \return metadata may be null
   std::shared_ptr<const KeyValueMetadata> metadata() const;
 
   /// \brief Render a string representation of the schema suitable for debugging
@@ -850,30 +850,30 @@ dictionary(const std::shared_ptr<DataType>& index_type,
 /// \param name the field name
 /// \param type the field value type
 /// \param nullable whether the values are nullable, default true
-/// \param metadata any custom key-value metadata, default nullptr
+/// \param metadata any custom key-value metadata, default null
 std::shared_ptr<Field> ARROW_EXPORT field(
     const std::string& name, const std::shared_ptr<DataType>& type, bool nullable = true,
-    const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+    const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
 
 /// \brief Create a Schema instance
 ///
 /// \param fields the schema's fields
-/// \param metadata any custom key-value metadata, default nullptr
+/// \param metadata any custom key-value metadata, default null
 /// \return schema shared_ptr to Schema
 ARROW_EXPORT
 std::shared_ptr<Schema> schema(
     const std::vector<std::shared_ptr<Field>>& fields,
-    const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+    const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
 
 /// \brief Create a Schema instance
 ///
 /// \param fields the schema's fields (rvalue reference)
-/// \param metadata any custom key-value metadata, default nullptr
+/// \param metadata any custom key-value metadata, default null
 /// \return schema shared_ptr to Schema
 ARROW_EXPORT
 std::shared_ptr<Schema> schema(
     std::vector<std::shared_ptr<Field>>&& fields,
-    const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr);
+    const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
 
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 58496a874f17d..6f8d5a46cd8d3 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -24,6 +24,7 @@
 #include <type_traits>
 
 #include "arrow/status.h"
+#include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -114,7 +115,7 @@ class ARROW_EXPORT Decimal128 {
   /// \brief Convert a decimal string to an Decimal128 value, optionally including
   /// precision and scale if they're passed in and not null.
   static Status FromString(const std::string& s, Decimal128* out,
-                           int* precision = nullptr, int* scale = nullptr);
+                           int* precision = NULLPTR, int* scale = NULLPTR);
 
  private:
   int64_t high_bits_;
diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
index 6fe3a5c1739d7..dbca0d8be394e 100644
--- a/cpp/src/arrow/util/io-util.h
+++ b/cpp/src/arrow/util/io-util.h
@@ -75,7 +75,7 @@ class StdinStream : public InputStream {
   }
 
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override {
-    auto buffer = std::make_shared<PoolBuffer>(nullptr);
+    auto buffer = std::make_shared<PoolBuffer>(NULLPTR);
     RETURN_NOT_OK(buffer->Resize(nbytes));
     int64_t bytes_read;
     RETURN_NOT_OK(Read(nbytes, &bytes_read, buffer->mutable_data()));
diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h
index a5f6e57079a43..8b1125d024191 100644
--- a/cpp/src/arrow/util/macros.h
+++ b/cpp/src/arrow/util/macros.h
@@ -58,6 +58,21 @@
 #define ARROW_MUST_USE_RESULT
 #endif
 
+// ----------------------------------------------------------------------
+// C++/CLI support macros (see ARROW-1134)
+
+#ifndef NULLPTR
+
+#ifdef __cplusplus_cli
+#define NULLPTR __nullptr
+#else
+#define NULLPTR nullptr
+#endif
+
+#endif  // ifndef NULLPTR
+
+// ----------------------------------------------------------------------
+
 // macros to disable padding
 // these macros are portable across different compilers and platforms
 //[https://github.com/google/flatbuffers/blob/master/include/flatbuffers/flatbuffers.h#L1355]

From b2596f66ac2a8aa66da81c1f75c36fa30c40d0df Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Wed, 25 Oct 2017 09:10:40 -0400
Subject: [PATCH 1181/1644] ARROW-1588: [C++/Format] Harden Decimal Format

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1211 from cpcloud/ARROW-1588 and squashes the following commits:

ae0d5624 [Phillip Cloud] ARROW-1588: [C++/Format] Harden Decimal Format
---
 cpp/src/arrow/util/CMakeLists.txt   |  1 +
 cpp/src/arrow/util/bit-util-test.cc | 33 ++++++++++++-
 cpp/src/arrow/util/bit-util.h       | 76 +++++++++++++++++++----------
 cpp/src/arrow/util/decimal.cc       |  9 ++--
 cpp/src/arrow/util/decimal.h        |  5 +-
 cpp/src/arrow/util/type_traits.h    | 41 ++++++++++++++++
 format/Layout.md                    |  5 +-
 format/Metadata.md                  |  3 +-
 8 files changed, 136 insertions(+), 37 deletions(-)
 create mode 100644 cpp/src/arrow/util/type_traits.h

diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 1178c658c4800..5df5e748f39e5 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -42,6 +42,7 @@ install(FILES
   rle-encoding.h
   sse-util.h
   stl.h
+  type_traits.h
   visibility.h
   DESTINATION include/arrow/util)
 
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index 5a66d7e85ad5a..92bdcb5fc0846 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -28,7 +28,6 @@
 
 #include "arrow/buffer.h"
 #include "arrow/memory_pool.h"
-#include "arrow/status.h"
 #include "arrow/test-util.h"
 #include "arrow/util/bit-stream-utils.h"
 #include "arrow/util/bit-util.h"
@@ -334,4 +333,36 @@ TEST(BitStreamUtil, ZigZag) {
   TestZigZag(-std::numeric_limits<int32_t>::max());
 }
 
+TEST(BitUtil, RoundTripLittleEndianTest) {
+  uint64_t value = 0xFF;
+
+#if ARROW_LITTLE_ENDIAN
+  uint64_t expected = value;
+#else
+  uint64_t expected = std::numeric_limits<uint64_t>::max() << 56;
+#endif
+
+  uint64_t little_endian_result = BitUtil::ToLittleEndian(value);
+  ASSERT_EQ(expected, little_endian_result);
+
+  uint64_t from_little_endian = BitUtil::FromLittleEndian(little_endian_result);
+  ASSERT_EQ(value, from_little_endian);
+}
+
+TEST(BitUtil, RoundTripBigEndianTest) {
+  uint64_t value = 0xFF;
+
+#if ARROW_LITTLE_ENDIAN
+  uint64_t expected = std::numeric_limits<uint64_t>::max() << 56;
+#else
+  uint64_t expected = value;
+#endif
+
+  uint64_t big_endian_result = BitUtil::ToBigEndian(value);
+  ASSERT_EQ(expected, big_endian_result);
+
+  uint64_t from_big_endian = BitUtil::FromBigEndian(big_endian_result);
+  ASSERT_EQ(value, from_big_endian);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 2509de21ffb35..8043f90ccdf6a 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -56,6 +56,7 @@
 #include <vector>
 
 #include "arrow/util/macros.h"
+#include "arrow/util/type_traits.h"
 #include "arrow/util/visibility.h"
 
 #ifdef ARROW_USE_SSE
@@ -305,7 +306,7 @@ static inline uint32_t ByteSwap(uint32_t value) {
   return static_cast<uint32_t>(ARROW_BYTE_SWAP32(value));
 }
 static inline int16_t ByteSwap(int16_t value) {
-  constexpr int16_t m = static_cast<int16_t>(0xff);
+  constexpr auto m = static_cast<int16_t>(0xff);
   return static_cast<int16_t>(((value >> 8) & m) | ((value & m) << 8));
 }
 static inline uint16_t ByteSwap(uint16_t value) {
@@ -331,8 +332,8 @@ static inline void ByteSwap(void* dst, const void* src, int len) {
       break;
   }
 
-  uint8_t* d = reinterpret_cast<uint8_t*>(dst);
-  const uint8_t* s = reinterpret_cast<const uint8_t*>(src);
+  auto d = reinterpret_cast<uint8_t*>(dst);
+  auto s = reinterpret_cast<const uint8_t*>(src);
   for (int i = 0; i < len; ++i) {
     d[i] = s[len - i - 1];
   }
@@ -341,36 +342,57 @@ static inline void ByteSwap(void* dst, const void* src, int len) {
 /// Converts to big endian format (if not already in big endian) from the
 /// machine's native endian format.
 #if ARROW_LITTLE_ENDIAN
-static inline int64_t ToBigEndian(int64_t value) { return ByteSwap(value); }
-static inline uint64_t ToBigEndian(uint64_t value) { return ByteSwap(value); }
-static inline int32_t ToBigEndian(int32_t value) { return ByteSwap(value); }
-static inline uint32_t ToBigEndian(uint32_t value) { return ByteSwap(value); }
-static inline int16_t ToBigEndian(int16_t value) { return ByteSwap(value); }
-static inline uint16_t ToBigEndian(uint16_t value) { return ByteSwap(value); }
+template <typename T,
+          typename =
+              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+static inline T ToBigEndian(T value) {
+  return ByteSwap(value);
+}
+
+template <typename T,
+          typename =
+              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+static inline T ToLittleEndian(T value) {
+  return value;
+}
 #else
-static inline int64_t ToBigEndian(int64_t val) { return val; }
-static inline uint64_t ToBigEndian(uint64_t val) { return val; }
-static inline int32_t ToBigEndian(int32_t val) { return val; }
-static inline uint32_t ToBigEndian(uint32_t val) { return val; }
-static inline int16_t ToBigEndian(int16_t val) { return val; }
-static inline uint16_t ToBigEndian(uint16_t val) { return val; }
+template <typename T,
+          typename =
+              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+static inline T ToBigEndian(T value) {
+  return value;
+}
 #endif
 
 /// Converts from big endian format to the machine's native endian format.
 #if ARROW_LITTLE_ENDIAN
-static inline int64_t FromBigEndian(int64_t value) { return ByteSwap(value); }
-static inline uint64_t FromBigEndian(uint64_t value) { return ByteSwap(value); }
-static inline int32_t FromBigEndian(int32_t value) { return ByteSwap(value); }
-static inline uint32_t FromBigEndian(uint32_t value) { return ByteSwap(value); }
-static inline int16_t FromBigEndian(int16_t value) { return ByteSwap(value); }
-static inline uint16_t FromBigEndian(uint16_t value) { return ByteSwap(value); }
+template <typename T,
+          typename =
+              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+static inline T FromBigEndian(T value) {
+  return ByteSwap(value);
+}
+
+template <typename T,
+          typename =
+              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+static inline T FromLittleEndian(T value) {
+  return value;
+}
 #else
-static inline int64_t FromBigEndian(int64_t val) { return val; }
-static inline uint64_t FromBigEndian(uint64_t val) { return val; }
-static inline int32_t FromBigEndian(int32_t val) { return val; }
-static inline uint32_t FromBigEndian(uint32_t val) { return val; }
-static inline int16_t FromBigEndian(int16_t val) { return val; }
-static inline uint16_t FromBigEndian(uint16_t val) { return val; }
+template <typename T,
+          typename =
+              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+static inline T FromBigEndian(T value) {
+  return value;
+}
+
+template <typename T,
+          typename =
+              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+static inline T FromLittleEndian(T value) {
+  return ByteSwap(value);
+}
 #endif
 
 // Logical right shift for signed integer types
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 49d5c024950f3..7196b252c5b60 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -27,6 +27,7 @@
 #pragma intrinsic(_BitScanReverse)
 #endif
 
+#include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 
@@ -41,11 +42,13 @@ Decimal128::Decimal128(const std::string& str) : Decimal128() {
 }
 
 Decimal128::Decimal128(const uint8_t* bytes)
-    : Decimal128(reinterpret_cast<const int64_t*>(bytes)[0],
-                 reinterpret_cast<const uint64_t*>(bytes)[1]) {}
+    : Decimal128(BitUtil::FromLittleEndian(reinterpret_cast<const int64_t*>(bytes)[1]),
+                 BitUtil::FromLittleEndian(reinterpret_cast<const uint64_t*>(bytes)[0])) {
+}
 
 std::array<uint8_t, 16> Decimal128::ToBytes() const {
-  const uint64_t raw[] = {static_cast<uint64_t>(high_bits_), low_bits_};
+  const uint64_t raw[] = {BitUtil::ToLittleEndian(low_bits_),
+                          BitUtil::ToLittleEndian(static_cast<uint64_t>(high_bits_))};
   const auto* raw_data = reinterpret_cast<const uint8_t*>(raw);
   std::array<uint8_t, 16> out{{0}};
   std::copy(raw_data, raw_data + out.size(), out.begin());
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 6f8d5a46cd8d3..72da5547907db 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -54,7 +54,8 @@ class ARROW_EXPORT Decimal128 {
   /// \brief Parse the number from a base 10 string representation.
   explicit Decimal128(const std::string& value);
 
-  /// \brief Create an Decimal128 from an array of bytes
+  /// \brief Create an Decimal128 from an array of bytes. Bytes are assumed to be in
+  /// little endian byte order.
   explicit Decimal128(const uint8_t* bytes);
 
   /// \brief Negate the current value
@@ -105,7 +106,7 @@ class ARROW_EXPORT Decimal128 {
   /// \brief Get the low bits of the two's complement representation of the number.
   uint64_t low_bits() const { return low_bits_; }
 
-  /// \brief Return the raw bytes of the value.
+  /// \brief Return the raw bytes of the value in little-endian byte order.
   std::array<uint8_t, 16> ToBytes() const;
 
   /// \brief Convert the Decimal128 value to a base 10 decimal string with the given
diff --git a/cpp/src/arrow/util/type_traits.h b/cpp/src/arrow/util/type_traits.h
new file mode 100644
index 0000000000000..c05309af826bd
--- /dev/null
+++ b/cpp/src/arrow/util/type_traits.h
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_TYPE_TRAITS_H
+#define ARROW_UTIL_TYPE_TRAITS_H
+
+#include <type_traits>
+
+namespace arrow {
+
+/// \brief Metafunction to allow checking if a type matches any of another set of types
+template <typename...>
+struct IsOneOf : std::false_type {};  /// Base case: nothing has matched
+
+template <typename T, typename U, typename... Args>
+struct IsOneOf<T, U, Args...> {
+  /// Recursive case: T == U or T matches any other types provided (not including U).
+  static constexpr bool value = std::is_same<T, U>::value || IsOneOf<T, Args...>::value;
+};
+
+/// \brief Shorthand for using IsOneOf + std::enable_if
+template <typename T, typename... Args>
+using EnableIfIsOneOf = typename std::enable_if<IsOneOf<T, Args...>::value, T>::type;
+
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_TYPE_TRAITS_H
diff --git a/format/Layout.md b/format/Layout.md
index 3c21dbc0dc2b1..ebf93821aab24 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -41,9 +41,8 @@ concepts, here is a small glossary to help disambiguate.
   or a fully-specified nested type. When we say slot we mean a relative type
   value, not necessarily any physical storage region.
 * Logical type: A data type that is implemented using some relative (physical)
-  type. For example, a Decimal value stored in 16 bytes could be stored in a
-  primitive array with slot size 16 bytes. Similarly, strings can be stored as
-  `List<1-byte>`.
+  type. For example, Decimal values are stored as 16 bytes in a fixed byte
+  size array. Similarly, strings can be stored as `List<1-byte>`.
 * Parent and child arrays: names to express relationships between physical
   value arrays in a nested type structure. For example, a `List<T>`-type parent
   array has a T-type array as its child (see more on lists below).
diff --git a/format/Metadata.md b/format/Metadata.md
index 80ca08ae13fb1..893b0a4742f89 100644
--- a/format/Metadata.md
+++ b/format/Metadata.md
@@ -391,7 +391,8 @@ logical type, which have no children) and 3 buffers:
 
 ### Decimal
 
-TBD
+Decimals are represented as a 2's complement 128-bit (16 byte) signed integer
+in little-endian byte order.
 
 ### Timestamp
 

From 8148b6d82d6bfacbeb01709f29f3abbddee2c2b3 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 25 Oct 2017 12:16:12 -0400
Subject: [PATCH 1182/1644] ARROW-1726: [GLib] Add setup description to verify
 C GLib build

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1247 from kou/c-glib-release-verify and squashes the following commits:

e9f23074 [Kouhei Sutou] [GLib] Add setup description to verify C GLib build
---
 dev/release/VERIFY.md | 53 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 52 insertions(+), 1 deletion(-)

diff --git a/dev/release/VERIFY.md b/dev/release/VERIFY.md
index 3f073e4086538..5b441ac13f1ca 100644
--- a/dev/release/VERIFY.md
+++ b/dev/release/VERIFY.md
@@ -22,4 +22,55 @@
 ## Windows
 
 We've provided a convenience script for verifying the C++ and Python builds on
-Windows. Read the comments in `verify-release-candidate.bat` for instructions
\ No newline at end of file
+Windows. Read the comments in `verify-release-candidate.bat` for instructions.
+
+## Linux and macOS
+
+We've provided a convenience script for verifying the C++, Python, C
+GLib, Java and JavaScript builds on Linux and macOS. Read the comments in
+`verify-release-candidate.sh` for instructions.
+
+### C GLib
+
+You need the followings to verify C GLib build:
+
+  * GLib
+  * GObject Introspection
+  * Ruby (not EOL-ed version is required)
+  * gobject-introspection gem
+  * test-unit gem
+
+You can install them by the followings on Debian GNU/Linux and Ubuntu:
+
+```console
+% sudo apt install -y -V libgirepository1.0-dev ruby-dev
+% sudo gem install gobject-introspection test-unit
+```
+
+You can install them by the followings on CentOS:
+
+```console
+% sudo yum install -y gobject-introspection-devel
+% git clone https://github.com/sstephenson/rbenv.git ~/.rbenv
+% git clone https://github.com/sstephenson/ruby-build.git ~/.rbenv/plugins/ruby-build
+% echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile
+% echo 'eval "$(rbenv init -)"' >> ~/.bash_profile
+% exec ${SHELL} --login
+% sudo yum install -y gcc make patch openssl-devel readline-devel zlib-devel
+% rbenv install 2.4.2
+% rbenv global 2.4.2
+% gem install gobject-introspection test-unit
+```
+
+You can install them by the followings on macOS:
+
+```console
+% brew install -y gobject-introspection
+% gem install gobject-introspection test-unit
+```
+
+You need to set `PKG_CONFIG_PATH` to find libffi on macOS:
+
+```console
+% export PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig:$PKG_CONFIG_PATH
+```

From 54d5c81af0a9cbc6ea551922c795728cd43bd86c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 25 Oct 2017 22:05:05 -0400
Subject: [PATCH 1183/1644] ARROW-1484: [C++/Python] Implement casts between
 date, time, timestamp units

Several JIRAs here that made sense to tackle together:

* ARROW-1680
* ARROW-1482
* ARROW-1484
* ARROW-1524

This also fixes bugs relating to ignoring the offset in sliced arrays in some of the cast kernel implementations.

cc @BryanCutler @xhochy @cpcloud

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1245 from wesm/ARROW-1484 and squashes the following commits:

6f6107fc [Wes McKinney] Test that timestamp truncation raises for safe casts
1d28ce27 [Wes McKinney] Fix typo
a820ed90 [Wes McKinney] Add test case for ARROW-1680
ac9845e3 [Wes McKinney] Implement date32<->date64 casts, unit tests
4911b09f [Wes McKinney] Add time casts
6441a954 [Wes McKinney] Write unit tests for timestamp to timestamp casts
1bde233e [Wes McKinney] Cast table for timestamps
569dfb81 [Wes McKinney] Fix existing cast tests with sliced arrays
884f6d25 [Wes McKinney] Baby steps
a0401363 [Wes McKinney] Placeholders for time32/64/timestamp cast functions
fddf466b [Wes McKinney] Stub out unit test for timestamp unit change
---
 cpp/src/arrow/compute/cast.cc         | 213 ++++++++++++++++++++++----
 cpp/src/arrow/compute/cast.h          |   3 +-
 cpp/src/arrow/compute/compute-test.cc | 210 ++++++++++++++++++++++++-
 cpp/src/arrow/test-util.h             |  21 ++-
 cpp/src/arrow/type.h                  |  12 +-
 python/pyarrow/array.pxi              |   4 +-
 python/pyarrow/includes/libarrow.pxd  |   1 +
 python/pyarrow/tests/test_array.py    |  26 ++++
 8 files changed, 440 insertions(+), 50 deletions(-)

diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index e8bbfd3470670..68a2b12379e34 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -25,6 +25,7 @@
 #include <sstream>
 #include <string>
 #include <type_traits>
+#include <utility>
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
@@ -68,6 +69,24 @@
 namespace arrow {
 namespace compute {
 
+template <typename T>
+inline const T* GetValuesAs(const ArrayData& data, int i) {
+  return reinterpret_cast<const T*>(data.buffers[i]->data()) + data.offset;
+}
+
+namespace {
+
+void CopyData(const Array& input, ArrayData* output) {
+  auto in_data = input.data();
+  output->length = in_data->length;
+  output->null_count = input.null_count();
+  output->buffers = in_data->buffers;
+  output->offset = in_data->offset;
+  output->child_data = in_data->child_data;
+}
+
+}  // namespace
+
 // ----------------------------------------------------------------------
 // Zero copy casts
 
@@ -77,7 +96,9 @@ struct is_zero_copy_cast {
 };
 
 template <typename O, typename I>
-struct is_zero_copy_cast<O, I, typename std::enable_if<std::is_same<I, O>::value>::type> {
+struct is_zero_copy_cast<
+    O, I, typename std::enable_if<std::is_same<I, O>::value &&
+                                  !std::is_base_of<ParametricType, O>::value>::type> {
   static constexpr bool value = true;
 };
 
@@ -102,10 +123,7 @@ template <typename O, typename I>
 struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
                   ArrayData* output) {
-    auto in_data = input.data();
-    output->null_count = input.null_count();
-    output->buffers = in_data->buffers;
-    output->child_data = in_data->child_data;
+    CopyData(input, output);
   }
 };
 
@@ -119,6 +137,7 @@ struct CastFunctor<T, NullType, typename std::enable_if<
                   ArrayData* output) {
     // Simply initialize data to 0
     auto buf = output->buffers[1];
+    DCHECK_EQ(output->offset, 0);
     memset(buf->mutable_data(), 0, buf->size());
   }
 };
@@ -139,12 +158,16 @@ struct CastFunctor<T, BooleanType,
   void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
                   ArrayData* output) {
     using c_type = typename T::c_type;
-    const uint8_t* data = input.data()->buffers[1]->data();
-    auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
     constexpr auto kOne = static_cast<c_type>(1);
     constexpr auto kZero = static_cast<c_type>(0);
+
+    auto in_data = input.data();
+    internal::BitmapReader bit_reader(in_data->buffers[1]->data(), in_data->offset,
+                                      in_data->length);
+    auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
     for (int64_t i = 0; i < input.length(); ++i) {
-      *out++ = BitUtil::GetBit(data, i) ? kOne : kZero;
+      *out++ = bit_reader.IsSet() ? kOne : kZero;
+      bit_reader.Next();
     }
   }
 };
@@ -189,7 +212,9 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::v
   void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
                   ArrayData* output) {
     using in_type = typename I::c_type;
-    auto in_data = reinterpret_cast<const in_type*>(input.data()->buffers[1]->data());
+    DCHECK_EQ(output->offset, 0);
+
+    const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
     uint8_t* out_data = reinterpret_cast<uint8_t*>(output->buffers[1]->mutable_data());
     for (int64_t i = 0; i < input.length(); ++i) {
       BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
@@ -204,12 +229,11 @@ struct CastFunctor<O, I,
                   ArrayData* output) {
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
+    DCHECK_EQ(output->offset, 0);
 
     auto in_offset = input.offset();
 
-    const auto& input_buffers = input.data()->buffers;
-
-    auto in_data = reinterpret_cast<const in_type*>(input_buffers[1]->data()) + in_offset;
+    const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
     auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
 
     if (!options.allow_int_overflow) {
@@ -217,14 +241,15 @@ struct CastFunctor<O, I,
       constexpr in_type kMin = static_cast<in_type>(std::numeric_limits<out_type>::min());
 
       if (input.null_count() > 0) {
-        const uint8_t* is_valid = input_buffers[0]->data();
-        int64_t is_valid_offset = in_offset;
+        internal::BitmapReader is_valid_reader(input.data()->buffers[0]->data(),
+                                               in_offset, input.length());
         for (int64_t i = 0; i < input.length(); ++i) {
-          if (ARROW_PREDICT_FALSE(BitUtil::GetBit(is_valid, is_valid_offset++) &&
+          if (ARROW_PREDICT_FALSE(is_valid_reader.IsSet() &&
                                   (*in_data > kMax || *in_data < kMin))) {
             ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
           }
           *out_data++ = static_cast<out_type>(*in_data++);
+          is_valid_reader.Next();
         }
       } else {
         for (int64_t i = 0; i < input.length(); ++i) {
@@ -251,7 +276,7 @@ struct CastFunctor<O, I,
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
 
-    auto in_data = reinterpret_cast<const in_type*>(input.data()->buffers[1]->data());
+    const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
     auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
     for (int64_t i = 0; i < input.length(); ++i) {
       *out_data++ = static_cast<out_type>(*in_data++);
@@ -259,6 +284,125 @@ struct CastFunctor<O, I,
   }
 };
 
+// ----------------------------------------------------------------------
+// From one timestamp to another
+
+template <typename in_type, typename out_type>
+inline void ShiftTime(FunctionContext* ctx, const CastOptions& options,
+                      const bool is_multiply, const int64_t factor, const Array& input,
+                      ArrayData* output) {
+  const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
+  auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
+
+  if (is_multiply) {
+    for (int64_t i = 0; i < input.length(); i++) {
+      out_data[i] = static_cast<out_type>(in_data[i] * factor);
+    }
+  } else {
+    if (options.allow_time_truncate) {
+      for (int64_t i = 0; i < input.length(); i++) {
+        out_data[i] = static_cast<out_type>(in_data[i] / factor);
+      }
+    } else {
+      for (int64_t i = 0; i < input.length(); i++) {
+        out_data[i] = static_cast<out_type>(in_data[i] / factor);
+        if (input.IsValid(i) && (out_data[i] * factor != in_data[i])) {
+          std::stringstream ss;
+          ss << "Casting from " << input.type()->ToString() << " to "
+             << output->type->ToString() << " would lose data: " << in_data[i];
+          ctx->SetStatus(Status::Invalid(ss.str()));
+          break;
+        }
+      }
+    }
+  }
+}
+
+namespace {
+
+// {is_multiply, factor}
+const std::pair<bool, int64_t> kTimeConversionTable[4][4] = {
+    {{true, 1}, {true, 1000}, {true, 1000000}, {true, 1000000000L}},     // SECOND
+    {{false, 1000}, {true, 1}, {true, 1000}, {true, 1000000}},           // MILLI
+    {{false, 1000000}, {false, 1000}, {true, 1}, {true, 1000}},          // MICRO
+    {{false, 1000000000L}, {false, 1000000}, {false, 1000}, {true, 1}},  // NANO
+};
+
+}  // namespace
+
+template <>
+struct CastFunctor<TimestampType, TimestampType> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    // If units are the same, zero copy, otherwise convert
+    const auto& in_type = static_cast<const TimestampType&>(*input.type());
+    const auto& out_type = static_cast<const TimestampType&>(*output->type);
+
+    if (in_type.unit() == out_type.unit()) {
+      CopyData(input, output);
+      return;
+    }
+
+    std::pair<bool, int64_t> conversion =
+        kTimeConversionTable[static_cast<int>(in_type.unit())]
+                            [static_cast<int>(out_type.unit())];
+
+    ShiftTime<int64_t, int64_t>(ctx, options, conversion.first, conversion.second, input,
+                                output);
+  }
+};
+
+// ----------------------------------------------------------------------
+// From one time32 or time64 to another
+
+template <typename O, typename I>
+struct CastFunctor<O, I,
+                   typename std::enable_if<std::is_base_of<TimeType, I>::value &&
+                                           std::is_base_of<TimeType, O>::value>::type> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    using in_t = typename I::c_type;
+    using out_t = typename O::c_type;
+
+    // If units are the same, zero copy, otherwise convert
+    const auto& in_type = static_cast<const I&>(*input.type());
+    const auto& out_type = static_cast<const O&>(*output->type);
+
+    if (in_type.unit() == out_type.unit()) {
+      CopyData(input, output);
+      return;
+    }
+
+    std::pair<bool, int64_t> conversion =
+        kTimeConversionTable[static_cast<int>(in_type.unit())]
+                            [static_cast<int>(out_type.unit())];
+
+    ShiftTime<in_t, out_t>(ctx, options, conversion.first, conversion.second, input,
+                           output);
+  }
+};
+
+// ----------------------------------------------------------------------
+// Between date32 and date64
+
+constexpr int64_t kMillisecondsInDay = 86400000;
+
+template <>
+struct CastFunctor<Date64Type, Date32Type> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    ShiftTime<int32_t, int64_t>(ctx, options, true, kMillisecondsInDay, input, output);
+  }
+};
+
+template <>
+struct CastFunctor<Date32Type, Date64Type> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    ShiftTime<int64_t, int32_t>(ctx, options, false, kMillisecondsInDay, input, output);
+  }
+};
+
 // ----------------------------------------------------------------------
 // Dictionary to other things
 
@@ -271,9 +415,8 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices,
   internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
                                            indices.length());
 
-  const index_c_type* in =
-      reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
-      indices.offset();
+  const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
+
   uint8_t* out = output->buffers[1]->mutable_data();
   int32_t byte_width =
       static_cast<const FixedSizeBinaryType&>(*output->type).byte_width();
@@ -336,9 +479,7 @@ Status UnpackBinaryDictionary(FunctionContext* ctx, const Array& indices,
   internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
                                            indices.length());
 
-  const index_c_type* in =
-      reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
-      indices.offset();
+  const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
   for (int64_t i = 0; i < indices.length(); ++i) {
     if (valid_bits_reader.IsSet()) {
       int32_t length;
@@ -409,9 +550,7 @@ void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
   internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
                                            indices.length());
 
-  const index_c_type* in =
-      reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
-      indices.offset();
+  const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
   for (int64_t i = 0; i < indices.length(); ++i) {
     if (valid_bits_reader.IsSet()) {
       out[i] = dictionary[in[i]];
@@ -436,9 +575,8 @@ struct CastFunctor<T, DictionaryType,
     DCHECK(values_type.Equals(*output->type))
         << "Dictionary type: " << values_type << " target type: " << (*output->type);
 
-    auto dictionary =
-        reinterpret_cast<const c_type*>(type.dictionary()->data()->buffers[1]->data()) +
-        type.dictionary()->offset();
+    const c_type* dictionary = GetValuesAs<c_type>(*type.dictionary()->data(), 1);
+
     auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
     const Array& indices = *dict_array.indices();
     switch (indices.type()->id()) {
@@ -481,6 +619,9 @@ static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input
     int64_t bitmap_size = BitUtil::BytesForBits(length);
     RETURN_NOT_OK(ctx->Allocate(bitmap_size, &validity_bitmap));
     memset(validity_bitmap->mutable_data(), 0, bitmap_size);
+  } else if (input.offset() != 0) {
+    RETURN_NOT_OK(CopyBitmap(ctx->memory_pool(), validity_bitmap->data(), input.offset(),
+                             length, &validity_bitmap));
   }
 
   if (out->buffers.size() == 2) {
@@ -598,13 +739,21 @@ class CastKernel : public UnaryKernel {
   FN(Int64Type, Time64Type);     \
   FN(Int64Type, Date64Type);
 
-#define DATE32_CASES(FN, IN_TYPE) FN(Date32Type, Date32Type);
+#define DATE32_CASES(FN, IN_TYPE) \
+  FN(Date32Type, Date32Type);     \
+  FN(Date32Type, Date64Type);
 
-#define DATE64_CASES(FN, IN_TYPE) FN(Date64Type, Date64Type);
+#define DATE64_CASES(FN, IN_TYPE) \
+  FN(Date64Type, Date64Type);     \
+  FN(Date64Type, Date32Type);
 
-#define TIME32_CASES(FN, IN_TYPE) FN(Time32Type, Time32Type);
+#define TIME32_CASES(FN, IN_TYPE) \
+  FN(Time32Type, Time32Type);     \
+  FN(Time32Type, Time64Type);
 
-#define TIME64_CASES(FN, IN_TYPE) FN(Time64Type, Time64Type);
+#define TIME64_CASES(FN, IN_TYPE) \
+  FN(Time64Type, Time32Type);     \
+  FN(Time64Type, Time64Type);
 
 #define TIMESTAMP_CASES(FN, IN_TYPE) FN(TimestampType, TimestampType);
 
diff --git a/cpp/src/arrow/compute/cast.h b/cpp/src/arrow/compute/cast.h
index 7a07512b2adcf..d7bde20d607db 100644
--- a/cpp/src/arrow/compute/cast.h
+++ b/cpp/src/arrow/compute/cast.h
@@ -34,9 +34,10 @@ class FunctionContext;
 class UnaryKernel;
 
 struct CastOptions {
-  CastOptions() : allow_int_overflow(false) {}
+  CastOptions() : allow_int_overflow(false), allow_time_truncate(false) {}
 
   bool allow_int_overflow;
+  bool allow_time_truncate;
 };
 
 /// \since 0.7.0
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 8a595178d057a..8a7ef923b4719 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -68,7 +68,7 @@ class TestCast : public ComputeFixture, public TestBase {
                  const std::shared_ptr<DataType>& out_type, const CastOptions& options) {
     std::shared_ptr<Array> result;
     ASSERT_OK(Cast(&ctx_, input, out_type, options, &result));
-    AssertArraysEqual(expected, *result);
+    ASSERT_ARRAYS_EQUAL(expected, *result);
   }
 
   template <typename InType, typename I_TYPE>
@@ -105,6 +105,11 @@ class TestCast : public ComputeFixture, public TestBase {
       ArrayFromVector<OutType, O_TYPE>(out_type, out_values, &expected);
     }
     CheckPass(*input, *expected, out_type, options);
+
+    // Check a sliced variant
+    if (input->length() > 1) {
+      CheckPass(*input->Slice(1), *expected->Slice(1), out_type, options);
+    }
   }
 };
 
@@ -270,6 +275,205 @@ TEST_F(TestCast, ToIntDowncastUnsafe) {
                                                     options);
 }
 
+TEST_F(TestCast, TimestampToTimestamp) {
+  CastOptions options;
+
+  auto CheckTimestampCast = [this](
+      const CastOptions& options, TimeUnit::type from_unit, TimeUnit::type to_unit,
+      const std::vector<int64_t>& from_values, const std::vector<int64_t>& to_values,
+      const std::vector<bool>& is_valid) {
+    CheckCase<TimestampType, int64_t, TimestampType, int64_t>(
+        timestamp(from_unit), from_values, is_valid, timestamp(to_unit), to_values,
+        options);
+  };
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  // Multiply promotions
+  vector<int64_t> v1 = {0, 100, 200, 1, 2};
+  vector<int64_t> e1 = {0, 100000, 200000, 1000, 2000};
+  CheckTimestampCast(options, TimeUnit::SECOND, TimeUnit::MILLI, v1, e1, is_valid);
+
+  vector<int64_t> v2 = {0, 100, 200, 1, 2};
+  vector<int64_t> e2 = {0, 100000000L, 200000000L, 1000000, 2000000};
+  CheckTimestampCast(options, TimeUnit::SECOND, TimeUnit::MICRO, v2, e2, is_valid);
+
+  vector<int64_t> v3 = {0, 100, 200, 1, 2};
+  vector<int64_t> e3 = {0, 100000000000L, 200000000000L, 1000000000L, 2000000000L};
+  CheckTimestampCast(options, TimeUnit::SECOND, TimeUnit::NANO, v3, e3, is_valid);
+
+  vector<int64_t> v4 = {0, 100, 200, 1, 2};
+  vector<int64_t> e4 = {0, 100000, 200000, 1000, 2000};
+  CheckTimestampCast(options, TimeUnit::MILLI, TimeUnit::MICRO, v4, e4, is_valid);
+
+  vector<int64_t> v5 = {0, 100, 200, 1, 2};
+  vector<int64_t> e5 = {0, 100000000L, 200000000L, 1000000, 2000000};
+  CheckTimestampCast(options, TimeUnit::MILLI, TimeUnit::NANO, v5, e5, is_valid);
+
+  vector<int64_t> v6 = {0, 100, 200, 1, 2};
+  vector<int64_t> e6 = {0, 100000, 200000, 1000, 2000};
+  CheckTimestampCast(options, TimeUnit::MICRO, TimeUnit::NANO, v6, e6, is_valid);
+
+  // Zero copy
+  std::shared_ptr<Array> arr;
+  vector<int64_t> v7 = {0, 70000, 2000, 1000, 0};
+  ArrayFromVector<TimestampType, int64_t>(timestamp(TimeUnit::SECOND), is_valid, v7,
+                                          &arr);
+  CheckZeroCopy(*arr, timestamp(TimeUnit::SECOND));
+
+  // Divide, truncate
+  vector<int64_t> v8 = {0, 100123, 200456, 1123, 2456};
+  vector<int64_t> e8 = {0, 100, 200, 1, 2};
+
+  options.allow_time_truncate = true;
+  CheckTimestampCast(options, TimeUnit::MILLI, TimeUnit::SECOND, v8, e8, is_valid);
+  CheckTimestampCast(options, TimeUnit::MICRO, TimeUnit::MILLI, v8, e8, is_valid);
+  CheckTimestampCast(options, TimeUnit::NANO, TimeUnit::MICRO, v8, e8, is_valid);
+
+  vector<int64_t> v9 = {0, 100123000, 200456000, 1123000, 2456000};
+  vector<int64_t> e9 = {0, 100, 200, 1, 2};
+  CheckTimestampCast(options, TimeUnit::MICRO, TimeUnit::SECOND, v9, e9, is_valid);
+  CheckTimestampCast(options, TimeUnit::NANO, TimeUnit::MILLI, v9, e9, is_valid);
+
+  vector<int64_t> v10 = {0, 100123000000L, 200456000000L, 1123000000L, 2456000000};
+  vector<int64_t> e10 = {0, 100, 200, 1, 2};
+  CheckTimestampCast(options, TimeUnit::NANO, TimeUnit::SECOND, v10, e10, is_valid);
+
+  // Disallow truncate, failures
+  options.allow_time_truncate = false;
+  CheckFails<TimestampType>(timestamp(TimeUnit::MILLI), v8, is_valid,
+                            timestamp(TimeUnit::SECOND), options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::MICRO), v8, is_valid,
+                            timestamp(TimeUnit::MILLI), options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::NANO), v8, is_valid,
+                            timestamp(TimeUnit::MICRO), options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::MICRO), v9, is_valid,
+                            timestamp(TimeUnit::SECOND), options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::NANO), v9, is_valid,
+                            timestamp(TimeUnit::MILLI), options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::NANO), v10, is_valid,
+                            timestamp(TimeUnit::SECOND), options);
+}
+
+TEST_F(TestCast, TimeToTime) {
+  CastOptions options;
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  // Multiply promotions
+  vector<int32_t> v1 = {0, 100, 200, 1, 2};
+  vector<int32_t> e1 = {0, 100000, 200000, 1000, 2000};
+  CheckCase<Time32Type, int32_t, Time32Type, int32_t>(
+      time32(TimeUnit::SECOND), v1, is_valid, time32(TimeUnit::MILLI), e1, options);
+
+  vector<int32_t> v2 = {0, 100, 200, 1, 2};
+  vector<int64_t> e2 = {0, 100000000L, 200000000L, 1000000, 2000000};
+  CheckCase<Time32Type, int32_t, Time64Type, int64_t>(
+      time32(TimeUnit::SECOND), v2, is_valid, time64(TimeUnit::MICRO), e2, options);
+
+  vector<int32_t> v3 = {0, 100, 200, 1, 2};
+  vector<int64_t> e3 = {0, 100000000000L, 200000000000L, 1000000000L, 2000000000L};
+  CheckCase<Time32Type, int32_t, Time64Type, int64_t>(
+      time32(TimeUnit::SECOND), v3, is_valid, time64(TimeUnit::NANO), e3, options);
+
+  vector<int32_t> v4 = {0, 100, 200, 1, 2};
+  vector<int64_t> e4 = {0, 100000, 200000, 1000, 2000};
+  CheckCase<Time32Type, int32_t, Time64Type, int64_t>(
+      time32(TimeUnit::MILLI), v4, is_valid, time64(TimeUnit::MICRO), e4, options);
+
+  vector<int32_t> v5 = {0, 100, 200, 1, 2};
+  vector<int64_t> e5 = {0, 100000000L, 200000000L, 1000000, 2000000};
+  CheckCase<Time32Type, int32_t, Time64Type, int64_t>(
+      time32(TimeUnit::MILLI), v5, is_valid, time64(TimeUnit::NANO), e5, options);
+
+  vector<int64_t> v6 = {0, 100, 200, 1, 2};
+  vector<int64_t> e6 = {0, 100000, 200000, 1000, 2000};
+  CheckCase<Time64Type, int64_t, Time64Type, int64_t>(
+      time64(TimeUnit::MICRO), v6, is_valid, time64(TimeUnit::NANO), e6, options);
+
+  // Zero copy
+  std::shared_ptr<Array> arr;
+  vector<int64_t> v7 = {0, 70000, 2000, 1000, 0};
+  ArrayFromVector<Time64Type, int64_t>(time64(TimeUnit::MICRO), is_valid, v7, &arr);
+  CheckZeroCopy(*arr, time64(TimeUnit::MICRO));
+
+  // Divide, truncate
+  vector<int32_t> v8 = {0, 100123, 200456, 1123, 2456};
+  vector<int32_t> e8 = {0, 100, 200, 1, 2};
+
+  options.allow_time_truncate = true;
+  CheckCase<Time32Type, int32_t, Time32Type, int32_t>(
+      time32(TimeUnit::MILLI), v8, is_valid, time32(TimeUnit::SECOND), e8, options);
+  CheckCase<Time64Type, int32_t, Time32Type, int32_t>(
+      time64(TimeUnit::MICRO), v8, is_valid, time32(TimeUnit::MILLI), e8, options);
+  CheckCase<Time64Type, int32_t, Time64Type, int32_t>(
+      time64(TimeUnit::NANO), v8, is_valid, time64(TimeUnit::MICRO), e8, options);
+
+  vector<int64_t> v9 = {0, 100123000, 200456000, 1123000, 2456000};
+  vector<int32_t> e9 = {0, 100, 200, 1, 2};
+  CheckCase<Time64Type, int64_t, Time32Type, int32_t>(
+      time64(TimeUnit::MICRO), v9, is_valid, time32(TimeUnit::SECOND), e9, options);
+  CheckCase<Time64Type, int64_t, Time32Type, int32_t>(
+      time64(TimeUnit::NANO), v9, is_valid, time32(TimeUnit::MILLI), e9, options);
+
+  vector<int64_t> v10 = {0, 100123000000L, 200456000000L, 1123000000L, 2456000000};
+  vector<int32_t> e10 = {0, 100, 200, 1, 2};
+  CheckCase<Time64Type, int64_t, Time32Type, int32_t>(
+      time64(TimeUnit::NANO), v10, is_valid, time32(TimeUnit::SECOND), e10, options);
+
+  // Disallow truncate, failures
+
+  options.allow_time_truncate = false;
+  CheckFails<Time32Type>(time32(TimeUnit::MILLI), v8, is_valid, time32(TimeUnit::SECOND),
+                         options);
+  CheckFails<Time64Type>(time64(TimeUnit::MICRO), v8, is_valid, time32(TimeUnit::MILLI),
+                         options);
+  CheckFails<Time64Type>(time64(TimeUnit::NANO), v8, is_valid, time64(TimeUnit::MICRO),
+                         options);
+  CheckFails<Time64Type>(time64(TimeUnit::MICRO), v9, is_valid, time32(TimeUnit::SECOND),
+                         options);
+  CheckFails<Time64Type>(time64(TimeUnit::NANO), v9, is_valid, time32(TimeUnit::MILLI),
+                         options);
+  CheckFails<Time64Type>(time64(TimeUnit::NANO), v10, is_valid, time32(TimeUnit::SECOND),
+                         options);
+}
+
+TEST_F(TestCast, DateToDate) {
+  CastOptions options;
+
+  vector<bool> is_valid = {true, false, true, true, true};
+
+  constexpr int64_t F = 86400000;
+
+  // Multiply promotion
+  vector<int32_t> v1 = {0, 100, 200, 1, 2};
+  vector<int64_t> e1 = {0, 100 * F, 200 * F, F, 2 * F};
+  CheckCase<Date32Type, int32_t, Date64Type, int64_t>(date32(), v1, is_valid, date64(),
+                                                      e1, options);
+
+  // Zero copy
+  std::shared_ptr<Array> arr;
+  vector<int32_t> v2 = {0, 70000, 2000, 1000, 0};
+  vector<int64_t> v3 = {0, 70000, 2000, 1000, 0};
+  ArrayFromVector<Date32Type, int32_t>(date32(), is_valid, v2, &arr);
+  CheckZeroCopy(*arr, date32());
+
+  ArrayFromVector<Date64Type, int64_t>(date64(), is_valid, v3, &arr);
+  CheckZeroCopy(*arr, date64());
+
+  // Divide, truncate
+  vector<int64_t> v8 = {0, 100 * F + 123, 200 * F + 456, F + 123, 2 * F + 456};
+  vector<int32_t> e8 = {0, 100, 200, 1, 2};
+
+  options.allow_time_truncate = true;
+  CheckCase<Date64Type, int64_t, Date32Type, int32_t>(date64(), v8, is_valid, date32(),
+                                                      e8, options);
+
+  // Disallow truncate, failures
+  options.allow_time_truncate = false;
+  CheckFails<Date64Type>(date64(), v8, is_valid, date32(), options);
+}
+
 TEST_F(TestCast, ToDouble) {
   CastOptions options;
   vector<bool> is_valid = {true, false, true, true, true};
@@ -335,7 +539,7 @@ TEST_F(TestCast, FromNull) {
   ASSERT_EQ(length, result->null_count());
 
   // OK to look at bitmaps
-  AssertArraysEqual(*result, *result);
+  ASSERT_ARRAYS_EQUAL(*result, *result);
 }
 
 TEST_F(TestCast, PreallocatedMemory) {
@@ -373,7 +577,7 @@ TEST_F(TestCast, PreallocatedMemory) {
   std::shared_ptr<Array> expected;
   ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, e1, &expected);
 
-  AssertArraysEqual(*expected, *result);
+  ASSERT_ARRAYS_EQUAL(*expected, *result);
 }
 
 template <typename TestType>
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 83ebdea4a8543..044fb9476ca73 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -281,15 +281,20 @@ Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>&
   return builder->Finish(out);
 }
 
-void AssertArraysEqual(const Array& expected, const Array& actual) {
-  if (!actual.Equals(expected)) {
-    std::stringstream pp_result;
-    std::stringstream pp_expected;
+#define ASSERT_ARRAYS_EQUAL(LEFT, RIGHT)                                               \
+  do {                                                                                 \
+    if (!(LEFT).Equals((RIGHT))) {                                                     \
+      std::stringstream pp_result;                                                     \
+      std::stringstream pp_expected;                                                   \
+                                                                                       \
+      EXPECT_OK(PrettyPrint(RIGHT, 0, &pp_result));                                    \
+      EXPECT_OK(PrettyPrint(LEFT, 0, &pp_expected));                                   \
+      FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str(); \
+    }                                                                                  \
+  } while (false)
 
-    EXPECT_OK(PrettyPrint(actual, 0, &pp_result));
-    EXPECT_OK(PrettyPrint(expected, 0, &pp_expected));
-    FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str();
-  }
+void AssertArraysEqual(const Array& expected, const Array& actual) {
+  ASSERT_ARRAYS_EQUAL(expected, actual);
 }
 
 #define ASSERT_BATCHES_EQUAL(LEFT, RIGHT)    \
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 2030f371d4faf..446f4d3a0b33f 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -228,7 +228,11 @@ class ARROW_EXPORT FloatingPoint : public Number {
   virtual Precision precision() const = 0;
 };
 
-class ARROW_EXPORT NestedType : public DataType {
+/// \class ParametricType
+/// \brief A superclass for types having additional metadata
+class ParametricType {};
+
+class ARROW_EXPORT NestedType : public DataType, public ParametricType {
  public:
   using DataType::DataType;
 };
@@ -444,7 +448,7 @@ class ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
 };
 
 // BinaryType type is represents lists of 1-byte values.
-class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType {
+class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType, public ParametricType {
  public:
   static constexpr Type::type type_id = Type::FIXED_SIZE_BINARY;
 
@@ -611,7 +615,7 @@ static inline std::ostream& operator<<(std::ostream& os, TimeUnit::type unit) {
   return os;
 }
 
-class ARROW_EXPORT TimeType : public FixedWidthType {
+class ARROW_EXPORT TimeType : public FixedWidthType, public ParametricType {
  public:
   TimeUnit::type unit() const { return unit_; }
 
@@ -650,7 +654,7 @@ class ARROW_EXPORT Time64Type : public TimeType {
   std::string name() const override { return "time64"; }
 };
 
-class ARROW_EXPORT TimestampType : public FixedWidthType {
+class ARROW_EXPORT TimestampType : public FixedWidthType, public ParametricType {
  public:
   using Unit = TimeUnit;
 
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index c596d2ad8e7c5..72262f0c981a0 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -260,8 +260,8 @@ cdef class Array:
 
         type = _ensure_type(target_type)
 
-        if not safe:
-            options.allow_int_overflow = 1
+        options.allow_int_overflow = not safe
+        options.allow_time_truncate = not safe
 
         with nogil:
             check_status(Cast(_context(), self.ap[0], type.sp_type,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 0e5d4a8eddcb1..809bb96b7a465 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -747,6 +747,7 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
 
     cdef cppclass CCastOptions" arrow::compute::CastOptions":
         c_bool allow_int_overflow
+        c_bool allow_time_truncate
 
     CStatus Cast(CFunctionContext* context, const CArray& array,
                  const shared_ptr[CDataType]& to_type,
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 418076f819684..e3a4c97567ee6 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -290,6 +290,32 @@ def test_cast_integers_unsafe():
         _check_cast_case(case, safe=False)
 
 
+def test_cast_timestamp_unit():
+    # ARROW-1680
+    val = datetime.datetime.now()
+    s = pd.Series([val])
+    s_nyc = s.dt.tz_localize('tzlocal()').dt.tz_convert('America/New_York')
+
+    us_with_tz = pa.timestamp('us', tz='America/New_York')
+    arr = pa.Array.from_pandas(s_nyc, type=us_with_tz)
+
+    arr2 = pa.Array.from_pandas(s, type=pa.timestamp('us'))
+
+    assert arr[0].as_py() == s_nyc[0]
+    assert arr2[0].as_py() == s[0]
+
+    # Disallow truncation
+    arr = pa.array([123123], type='int64').cast(pa.timestamp('ms'))
+    expected = pa.array([123], type='int64').cast(pa.timestamp('s'))
+
+    target = pa.timestamp('s')
+    with pytest.raises(ValueError):
+        arr.cast(target)
+
+    result = arr.cast(target, safe=False)
+    assert result.equals(expected)
+
+
 def test_cast_signed_to_unsigned():
     safe_cases = [
         (np.array([0, 1, 2, 3], dtype='i1'), pa.uint8(),

From 48a6ff856cf4de939f5ced42a09b1b39866efc1e Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Wed, 25 Oct 2017 22:19:51 -0400
Subject: [PATCH 1184/1644] ARROW-1721: [Python] Implement null-mask check in
 places where it isn't supported in numpy_to_arrow.cc

This closes [ARROW-1721](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1721).

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1246 from Licht-T/feature-object-from_pandas-mask and squashes the following commits:

41a1229d [Wes McKinney] Fix flake8 issues
d7545334 [Licht-T] Fix lint issues by clang-format-4.0
7ef7f784 [Licht-T] Revert "Fix lint issues"
5c6c1822 [Licht-T] Fix lint issues
78d3c3fc [Licht-T] TST: Add tests of null-mask check for object types
72030bfe [Licht-T] ENH: Implement null-mask check for object types
---
 cpp/src/arrow/python/numpy_to_arrow.cc      | 86 ++++++++++++++++-----
 python/pyarrow/tests/test_convert_pandas.py | 48 ++++++++++--
 2 files changed, 110 insertions(+), 24 deletions(-)

diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 2c89a9f613ab7..ead3a04810121 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -622,8 +622,12 @@ Status NumPyConverter::ConvertDates() {
 
   Ndarray1DIndexer<PyObject*> objects(arr_);
 
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
   if (mask_ != nullptr) {
-    return Status::NotImplemented("mask not supported in object conversions yet");
+    mask_values.Init(mask_);
+    have_mask = true;
   }
 
   BuilderType builder(pool_);
@@ -636,10 +640,10 @@ Status NumPyConverter::ConvertDates() {
   PyObject* obj;
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
-    if (PyDate_CheckExact(obj)) {
-      RETURN_NOT_OK(builder.Append(UnboxDate<ArrowType>::Unbox(obj)));
-    } else if (PandasObjectIsNull(obj)) {
+    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder.AppendNull());
+    } else if (PyDate_CheckExact(obj)) {
+      RETURN_NOT_OK(builder.Append(UnboxDate<ArrowType>::Unbox(obj)));
     } else {
       std::stringstream ss;
       ss << "Error converting from Python objects to Date: ";
@@ -1029,6 +1033,41 @@ Status LoopPySequence(PyObject* sequence, T func) {
   return Status::OK();
 }
 
+template <typename T>
+Status LoopPySequenceWithMasks(PyObject* sequence,
+                               const Ndarray1DIndexer<uint8_t>& mask_values,
+                               bool have_mask, T func) {
+  if (PySequence_Check(sequence)) {
+    OwnedRef ref;
+    Py_ssize_t size = PySequence_Size(sequence);
+    if (PyArray_Check(sequence)) {
+      auto array = reinterpret_cast<PyArrayObject*>(sequence);
+      Ndarray1DIndexer<PyObject*> objects(array);
+      for (int64_t i = 0; i < size; ++i) {
+        RETURN_NOT_OK(func(objects[i], have_mask && mask_values[i]));
+      }
+    } else {
+      for (int64_t i = 0; i < size; ++i) {
+        ref.reset(PySequence_GetItem(sequence, i));
+        RETURN_NOT_OK(func(ref.obj(), have_mask && mask_values[i]));
+      }
+    }
+  } else if (PyObject_HasAttrString(sequence, "__iter__")) {
+    OwnedRef iter = OwnedRef(PyObject_GetIter(sequence));
+    PyObject* item;
+    int64_t i = 0;
+    while ((item = PyIter_Next(iter.obj()))) {
+      OwnedRef ref = OwnedRef(item);
+      RETURN_NOT_OK(func(ref.obj(), have_mask && mask_values[i]));
+      i++;
+    }
+  } else {
+    return Status::TypeError("Object is not a sequence or iterable");
+  }
+
+  return Status::OK();
+}
+
 template <int ITEM_TYPE, typename ArrowType>
 inline Status NumPyConverter::ConvertTypedLists(const std::shared_ptr<DataType>& type,
                                                 ListBuilder* builder, PyObject* list) {
@@ -1037,15 +1076,18 @@ inline Status NumPyConverter::ConvertTypedLists(const std::shared_ptr<DataType>&
 
   PyAcquireGIL lock;
 
-  // TODO: mask not supported here
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
   if (mask_ != nullptr) {
-    return Status::NotImplemented("mask not supported in object conversions yet");
+    mask_values.Init(mask_);
+    have_mask = true;
   }
 
   BuilderT* value_builder = static_cast<BuilderT*>(builder->value_builder());
 
-  auto foreach_item = [&](PyObject* object) {
-    if (PandasObjectIsNull(object)) {
+  auto foreach_item = [&](PyObject* object, bool mask) {
+    if (mask || PandasObjectIsNull(object)) {
       return builder->AppendNull();
     } else if (PyArray_Check(object)) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
@@ -1071,7 +1113,7 @@ inline Status NumPyConverter::ConvertTypedLists(const std::shared_ptr<DataType>&
     }
   };
 
-  return LoopPySequence(list, foreach_item);
+  return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
 }
 
 template <>
@@ -1079,15 +1121,18 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
     const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
   PyAcquireGIL lock;
 
-  // TODO: mask not supported here
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
   if (mask_ != nullptr) {
-    return Status::NotImplemented("mask not supported in object conversions yet");
+    mask_values.Init(mask_);
+    have_mask = true;
   }
 
   auto value_builder = static_cast<NullBuilder*>(builder->value_builder());
 
-  auto foreach_item = [&](PyObject* object) {
-    if (PandasObjectIsNull(object)) {
+  auto foreach_item = [&](PyObject* object, bool mask) {
+    if (mask || PandasObjectIsNull(object)) {
       return builder->AppendNull();
     } else if (PyArray_Check(object)) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
@@ -1112,7 +1157,7 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
     }
   };
 
-  return LoopPySequence(list, foreach_item);
+  return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
 }
 
 template <>
@@ -1122,15 +1167,18 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
   // TODO: If there are bytes involed, convert to Binary representation
   bool have_bytes = false;
 
-  // TODO: mask not supported here
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
   if (mask_ != nullptr) {
-    return Status::NotImplemented("mask not supported in object conversions yet");
+    mask_values.Init(mask_);
+    have_mask = true;
   }
 
   auto value_builder = static_cast<StringBuilder*>(builder->value_builder());
 
-  auto foreach_item = [&](PyObject* object) {
-    if (PandasObjectIsNull(object)) {
+  auto foreach_item = [&](PyObject* object, bool mask) {
+    if (mask || PandasObjectIsNull(object)) {
       return builder->AppendNull();
     } else if (PyArray_Check(object)) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
@@ -1162,7 +1210,7 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     }
   };
 
-  return LoopPySequence(list, foreach_item);
+  return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
 }
 
 #define LIST_CASE(TYPE, NUMPY_TYPE, ArrowType)                            \
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 41ad20102112a..527466e6eef64 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -19,7 +19,6 @@
 from collections import OrderedDict
 
 from datetime import date, time
-import unittest
 import decimal
 import json
 
@@ -61,7 +60,7 @@ def _alltypes_example(size=100):
     })
 
 
-class TestPandasConversion(unittest.TestCase):
+class TestPandasConversion(object):
 
     def setUp(self):
         pass
@@ -420,7 +419,7 @@ def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
         values = [b'foo', None, b'ba', None, None, b'hey']
         df = pd.DataFrame({'strings': values})
         schema = pa.schema([pa.field('strings', pa.binary(3))])
-        with self.assertRaises(pa.ArrowInvalid):
+        with pytest.raises(pa.ArrowInvalid):
             pa.Table.from_pandas(df, schema=schema)
 
     def test_timestamps_notimezone_no_nulls(self):
@@ -697,11 +696,11 @@ def test_category(self):
 
     def test_mixed_types_fails(self):
         data = pd.DataFrame({'a': ['a', 1, 2.0]})
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             pa.Table.from_pandas(data)
 
         data = pd.DataFrame({'a': [1, True]})
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             pa.Table.from_pandas(data)
 
     def test_strided_data_import(self):
@@ -1096,6 +1095,45 @@ def test_table_str_to_categorical(self):
         expected = pd.DataFrame({'strings': pd.Categorical(values)})
         tm.assert_frame_equal(result, expected, check_dtype=True)
 
+    def test_array_from_pandas_date_with_mask(self):
+        m = np.array([True, False, True])
+        data = pd.Series([
+            date(1990, 1, 1),
+            date(1991, 1, 1),
+            date(1992, 1, 1)
+        ])
+
+        result = pa.Array.from_pandas(data, mask=m)
+
+        expected = pd.Series([None, date(1991, 1, 1), None])
+        assert pa.Array.from_pandas(expected).equals(result)
+
+    @pytest.mark.parametrize('t,data,expected', [
+        (
+            pa.int64,
+            [[1, 2], [3], None],
+            [None, [3], None]
+        ),
+        (
+            pa.string,
+            [[u'aaa', u'bb'], [u'c'], None],
+            [None, [u'c'], None]
+        ),
+        (
+            pa.null,
+            [[None, None], [None], None],
+            [None, [None], None]
+        )
+    ])
+    def test_array_from_pandas_typed_array_with_mask(self, t, data, expected):
+        m = np.array([True, False, True])
+
+        s = pd.Series(data)
+        result = pa.Array.from_pandas(s, mask=m, type=pa.list_(t()))
+
+        assert pa.Array.from_pandas(expected,
+                                    type=pa.list_(t())).equals(result)
+
 
 def _pytime_from_micros(val):
     microseconds = val % 1000000

From 238881fae8530a1ae994eb0e283e4783d3dd2855 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 25 Oct 2017 22:24:21 -0400
Subject: [PATCH 1185/1644] ARROW-1675: [Python] Use RecordBatch.from_pandas in
 Feather write path

This also makes Feather writes more robust to columns having a mix of unicode and bytes (these gets coerced to binary)

Also resolves ARROW-1672

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1250 from wesm/ARROW-1675 and squashes the following commits:

9d55886b [Wes McKinney] Use RecordBatch.from_pandas in Feather write path. Fix test cases
---
 python/pyarrow/feather.py            | 32 ++++++----------------------
 python/pyarrow/tests/test_feather.py | 12 ++++++++---
 python/pyarrow/types.pxi             |  1 -
 3 files changed, 16 insertions(+), 29 deletions(-)

diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 2091c9154fdd5..3ba9d652c3e7a 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -23,7 +23,7 @@
 
 from pyarrow.compat import pdapi
 from pyarrow.lib import FeatherError  # noqa
-from pyarrow.lib import Table
+from pyarrow.lib import RecordBatch, Table
 import pyarrow.lib as ext
 
 try:
@@ -75,30 +75,12 @@ def write(self, df):
         if not df.columns.is_unique:
             raise ValueError("cannot serialize duplicate column names")
 
-        # TODO(wesm): pipeline conversion to Arrow memory layout
-        for i, name in enumerate(df.columns):
-            col = df.iloc[:, i]
-
-            if pdapi.is_object_dtype(col):
-                inferred_type = infer_dtype(col)
-                msg = ("cannot serialize column {n} "
-                       "named {name} with dtype {dtype}".format(
-                           n=i, name=name, dtype=inferred_type))
-
-                if inferred_type in ['mixed']:
-
-                    # allow columns with nulls + an inferable type
-                    inferred_type = infer_dtype(col[col.notnull()])
-                    if inferred_type in ['mixed']:
-                        raise ValueError(msg)
-
-                elif inferred_type not in ['unicode', 'string']:
-                    raise ValueError(msg)
-
-            if not isinstance(name, six.string_types):
-                name = str(name)
-
-            self.writer.write_array(name, col)
+        # TODO(wesm): Remove this length check, see ARROW-1732
+        if len(df.columns) > 0:
+            batch = RecordBatch.from_pandas(df, preserve_index=False)
+            for i, name in enumerate(batch.schema.names):
+                col = batch[i]
+                self.writer.write_array(name, col)
 
         self.writer.close()
 
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 810ee3c8cf8ac..9e7fc8863e759 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -279,11 +279,14 @@ def test_delete_partial_file_on_error(self):
         if sys.platform == 'win32':
             pytest.skip('Windows hangs on to file handle for some reason')
 
+        class CustomClass(object):
+            pass
+
         # strings will fail
         df = pd.DataFrame(
             {
                 'numbers': range(5),
-                'strings': [b'foo', None, u'bar', 'qux', np.nan]},
+                'strings': [b'foo', None, u'bar', CustomClass(), np.nan]},
             columns=['numbers', 'strings'])
 
         path = random_path()
@@ -297,10 +300,13 @@ def test_delete_partial_file_on_error(self):
     def test_strings(self):
         repeats = 1000
 
-        # we hvae mixed bytes, unicode, strings
+        # Mixed bytes, unicode, strings coerced to binary
         values = [b'foo', None, u'bar', 'qux', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
-        self._assert_error_on_write(df, ValueError)
+
+        ex_values = [b'foo', None, b'bar', b'qux', np.nan]
+        expected = pd.DataFrame({'strings': ex_values * repeats})
+        self._check_pandas_roundtrip(df, expected, null_counts=[2 * repeats])
 
         # embedded nulls are ok
         values = ['foo', None, 'bar', 'qux', None]
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 686e56ead342d..c9a490960ec38 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -662,7 +662,6 @@ cdef _as_type(type):
     return type_for_alias(type)
 
 
-
 cdef set PRIMITIVE_TYPES = set([
     _Type_NA, _Type_BOOL,
     _Type_UINT8, _Type_INT8,

From c30a7e30af2469dde1a00f74d8ba9631887825c4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 26 Oct 2017 14:13:50 +0200
Subject: [PATCH 1186/1644] ARROW-1732: [Python] Permit creating record batches
 with no columns, test pandas roundtrips

I ran into this rough edge today, invariably serialization code paths will need to send across a DataFrame with no columns, this will need to work even if `preserve_index=False`

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1252 from wesm/ARROW-1732 and squashes the following commits:

a240c05 [Wes McKinney] Permit creating record batches with no columns, test pandas roundtrips
---
 python/pyarrow/table.pxi                    | 20 ++++++++---------
 python/pyarrow/tests/test_convert_pandas.py | 25 +++++++++++++++------
 python/pyarrow/tests/test_table.py          |  8 +++++++
 3 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 694fe919044c1..eb1911592f4c4 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -308,8 +308,8 @@ cdef shared_ptr[const CKeyValueMetadata] unbox_metadata(dict metadata):
             make_shared[CKeyValueMetadata](unordered_metadata))
 
 
-cdef int _schema_from_arrays(
-        arrays, names, dict metadata, shared_ptr[CSchema]* schema) except -1:
+cdef _schema_from_arrays(arrays, names, dict metadata,
+                         shared_ptr[CSchema]* schema):
     cdef:
         Column col
         c_string c_name
@@ -317,10 +317,11 @@ cdef int _schema_from_arrays(
         shared_ptr[CDataType] type_
         Py_ssize_t K = len(arrays)
 
-    fields.resize(K)
+    if K == 0:
+        schema.reset(new CSchema(fields, unbox_metadata(metadata)))
+        return
 
-    if not K:
-        raise ValueError('Must pass at least one array')
+    fields.resize(K)
 
     if isinstance(arrays[0], Column):
         for i in range(K):
@@ -346,7 +347,6 @@ cdef int _schema_from_arrays(
             fields[i].reset(new CField(c_name, type_, True))
 
     schema.reset(new CSchema(fields, unbox_metadata(metadata)))
-    return 0
 
 
 cdef class RecordBatch:
@@ -613,10 +613,10 @@ cdef class RecordBatch:
             int64_t i
             int64_t number_of_arrays = len(arrays)
 
-        if not number_of_arrays:
-            raise ValueError('Record batch cannot contain no arrays (for now)')
-
-        num_rows = len(arrays[0])
+        if len(arrays) > 0:
+            num_rows = len(arrays[0])
+        else:
+            num_rows = 0
         _schema_from_arrays(arrays, names, metadata, &schema)
 
         c_arrays.reserve(len(arrays))
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 527466e6eef64..6d146f9778c05 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -71,11 +71,11 @@ def tearDown(self):
     def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
                                 expected_schema=None,
                                 check_dtype=True, schema=None,
-                                check_index=False,
+                                preserve_index=False,
                                 as_batch=False):
         klass = pa.RecordBatch if as_batch else pa.Table
         table = klass.from_pandas(df, schema=schema,
-                                  preserve_index=check_index,
+                                  preserve_index=preserve_index,
                                   nthreads=nthreads)
 
         result = table.to_pandas(nthreads=nthreads)
@@ -83,7 +83,9 @@ def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
             assert table.schema.equals(expected_schema)
         if expected is None:
             expected = df
-        tm.assert_frame_equal(result, expected, check_dtype=check_dtype)
+        tm.assert_frame_equal(result, expected, check_dtype=check_dtype,
+                              check_index_type=('equiv' if preserve_index
+                                                else False))
 
     def _check_series_roundtrip(self, s, type_=None):
         arr = pa.array(s, from_pandas=True, type=type_)
@@ -131,14 +133,14 @@ def test_non_string_columns(self):
     def test_column_index_names_are_preserved(self):
         df = pd.DataFrame({'data': [1, 2, 3]})
         df.columns.names = ['a']
-        self._check_pandas_roundtrip(df, check_index=True)
+        self._check_pandas_roundtrip(df, preserve_index=True)
 
     def test_multiindex_columns(self):
         columns = pd.MultiIndex.from_arrays([
             ['one', 'two'], ['X', 'Y']
         ])
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
-        self._check_pandas_roundtrip(df, check_index=True)
+        self._check_pandas_roundtrip(df, preserve_index=True)
 
     def test_multiindex_columns_with_dtypes(self):
         columns = pd.MultiIndex.from_arrays(
@@ -149,11 +151,11 @@ def test_multiindex_columns_with_dtypes(self):
             names=['level_1', 'level_2'],
         )
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
-        self._check_pandas_roundtrip(df, check_index=True)
+        self._check_pandas_roundtrip(df, preserve_index=True)
 
     def test_integer_index_column(self):
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')])
-        self._check_pandas_roundtrip(df, check_index=True)
+        self._check_pandas_roundtrip(df, preserve_index=True)
 
     def test_categorical_column_index(self):
         # I *really* hope no one uses category dtypes for single level column
@@ -1095,6 +1097,15 @@ def test_table_str_to_categorical(self):
         expected = pd.DataFrame({'strings': pd.Categorical(values)})
         tm.assert_frame_equal(result, expected, check_dtype=True)
 
+    def test_table_batch_empty_dataframe(self):
+        df = pd.DataFrame({})
+        self._check_pandas_roundtrip(df)
+        self._check_pandas_roundtrip(df, as_batch=True)
+
+        df2 = pd.DataFrame({}, index=[0, 1, 2])
+        self._check_pandas_roundtrip(df2, preserve_index=True)
+        self._check_pandas_roundtrip(df2, as_batch=True, preserve_index=True)
+
     def test_array_from_pandas_date_with_mask(self):
         m = np.array([True, False, True])
         data = pd.Series([
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 50190f597b972..4282224660a1a 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -82,6 +82,14 @@ def test_recordbatch_basics():
         batch[2]
 
 
+def test_recordbatch_no_fields():
+    batch = pa.RecordBatch.from_arrays([], [])
+
+    assert len(batch) == 0
+    assert batch.num_rows == 0
+    assert batch.num_columns == 0
+
+
 def test_recordbatch_from_arrays_invalid_names():
     data = [
         pa.array(range(5)),

From 6b16cca3e081fc05652d75f2d4f854b958d9f6d9 Mon Sep 17 00:00:00 2001
From: Nick White <n.j.white@gmail.com>
Date: Thu, 26 Oct 2017 09:58:48 -0400
Subject: [PATCH 1187/1644] ARROW-1689: [Python] Allow user to request no data
 copies

This makes performance debugging much easier, as it allows you to track down what (Arrow) data is causing unexpected delays in loading. It also makes testing features like ARROW-1689 easier as you can prove (via unit tests) that copies are not being made.

Author: Nick White <n.j.white@gmail.com>

Closes #1233 from njwhite/feature/zerocopycategories and squashes the following commits:

b06f50d3 [Nick White] ARROW-1689 Don't Deserialize the Dictionary Twice
a968b0b8 [Nick White] ARROW-1689 Allow User To Request No Data Copies
---
 cpp/src/arrow/python/arrow_to_pandas.cc     | 43 ++++++++++++++-------
 cpp/src/arrow/python/arrow_to_pandas.h      |  3 ++
 python/pyarrow/array.pxi                    |  9 ++++-
 python/pyarrow/includes/libarrow.pxd        |  1 +
 python/pyarrow/table.pxi                    | 15 +++++--
 python/pyarrow/tests/test_convert_pandas.py | 40 +++++++++++++++++++
 6 files changed, 92 insertions(+), 19 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 88b594cac9428..0c2e0ad85bfa4 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -1040,6 +1040,8 @@ class CategoricalBlock : public PandasBlock {
     return Status::OK();
   }
 
+  PyObject* dictionary() const { return dictionary_.obj(); }
+
  protected:
   MemoryPool* pool_;
   OwnedRef dictionary_;
@@ -1399,6 +1401,11 @@ class ArrowDeserializer {
 
     if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
       return ConvertValuesZeroCopy<TYPE>(options_, npy_type, data_.chunk(0));
+    } else if (options_.zero_copy_only) {
+      std::stringstream ss;
+      ss << "Needed to copy " << data_.num_chunks() << " chunks with "
+         << data_.null_count() << " nulls, but zero_copy_only was True";
+      return Status::Invalid(ss.str());
     }
 
     RETURN_NOT_OK(AllocateOutput(npy_type));
@@ -1413,6 +1420,10 @@ class ArrowDeserializer {
                               std::is_base_of<TimestampType, Type>::value,
                           Status>::type
   Visit(const Type& type) {
+    if (options_.zero_copy_only) {
+      return Status::Invalid("Copy Needed, but zero_copy_only was True");
+    }
+
     constexpr int TYPE = Type::type_id;
     using traits = internal::arrow_traits<TYPE>;
     using c_type = typename Type::c_type;
@@ -1453,6 +1464,11 @@ class ArrowDeserializer {
 
     if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
       return ConvertValuesZeroCopy<TYPE>(options_, traits::npy_type, data_.chunk(0));
+    } else if (options_.zero_copy_only) {
+      std::stringstream ss;
+      ss << "Needed to copy " << data_.num_chunks() << " chunks with "
+         << data_.null_count() << " nulls, but zero_copy_only was True";
+      return Status::Invalid(ss.str());
     }
 
     if (data_.null_count() > 0) {
@@ -1470,6 +1486,9 @@ class ArrowDeserializer {
 
   template <typename FUNCTOR>
   inline Status VisitObjects(FUNCTOR func) {
+    if (options_.zero_copy_only) {
+      return Status::Invalid("Object types need copies, but zero_copy_only was True");
+    }
     RETURN_NOT_OK(AllocateOutput(NPY_OBJECT));
     auto out_values = reinterpret_cast<PyObject**>(PyArray_DATA(arr_));
     return func(options_, data_, out_values);
@@ -1499,7 +1518,9 @@ class ArrowDeserializer {
 
   // Boolean specialization
   Status Visit(const BooleanType& type) {
-    if (data_.null_count() > 0) {
+    if (options_.zero_copy_only) {
+      return Status::Invalid("BooleanType needs copies, but zero_copy_only was True");
+    } else if (data_.null_count() > 0) {
       return VisitObjects(ConvertBooleanWithNulls);
     } else {
       RETURN_NOT_OK(AllocateOutput(internal::arrow_traits<Type::BOOL>::npy_type));
@@ -1510,6 +1531,9 @@ class ArrowDeserializer {
   }
 
   Status Visit(const ListType& type) {
+    if (options_.zero_copy_only) {
+      return Status::Invalid("ListType needs copies, but zero_copy_only was True");
+    }
 #define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \
   case Type::ArrowEnum:                                    \
     return ConvertListsLike<ArrowType>(options_, col_, out_values);
@@ -1542,26 +1566,19 @@ class ArrowDeserializer {
   }
 
   Status Visit(const DictionaryType& type) {
+    if (options_.zero_copy_only) {
+      return Status::Invalid("DictionaryType needs copies, but zero_copy_only was True");
+    }
+
     auto block = std::make_shared<CategoricalBlock>(options_, nullptr, col_->length());
     RETURN_NOT_OK(block->Write(col_, 0, 0));
 
-    auto dict_type = static_cast<const DictionaryType*>(col_->type().get());
-
     PyAcquireGIL lock;
     result_ = PyDict_New();
     RETURN_IF_PYERROR();
 
-    PyObject* dictionary;
-
-    // Release GIL before calling ConvertArrayToPandas, will be reacquired
-    // there if needed
-    lock.release();
-    RETURN_NOT_OK(
-        ConvertArrayToPandas(options_, dict_type->dictionary(), nullptr, &dictionary));
-    lock.acquire();
-
     PyDict_SetItemString(result_, "indices", block->block_arr());
-    PyDict_SetItemString(result_, "dictionary", dictionary);
+    PyDict_SetItemString(result_, "dictionary", block->dictionary());
 
     return Status::OK();
   }
diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h
index 1d716a5c94fa6..1e486463708ac 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.h
+++ b/cpp/src/arrow/python/arrow_to_pandas.h
@@ -41,6 +41,9 @@ namespace py {
 
 struct PandasOptions {
   bool strings_to_categorical;
+  bool zero_copy_only;
+
+  PandasOptions() : strings_to_categorical(false), zero_copy_only(false) {}
 };
 
 ARROW_EXPORT
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 72262f0c981a0..c57eda1da759f 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -373,7 +373,7 @@ cdef class Array:
 
         return pyarrow_wrap_array(result)
 
-    def to_pandas(self, c_bool strings_to_categorical=False):
+    def to_pandas(self, c_bool strings_to_categorical=False, zero_copy_only=False):
         """
         Convert to an array object suitable for use in pandas
 
@@ -381,6 +381,9 @@ cdef class Array:
         ----------
         strings_to_categorical : boolean, default False
             Encode string (UTF8) and binary types to pandas.Categorical
+        zero_copy_only : boolean, default False
+            Raise an ArrowException if this function call would require copying
+            the underlying data
 
         See also
         --------
@@ -392,7 +395,9 @@ cdef class Array:
             PyObject* out
             PandasOptions options
 
-        options = PandasOptions(strings_to_categorical=strings_to_categorical)
+        options = PandasOptions(
+            strings_to_categorical=strings_to_categorical,
+            zero_copy_only=zero_copy_only)
         with nogil:
             check_status(ConvertArrayToPandas(options, self.sp_array,
                                               self, &out))
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 809bb96b7a465..731ef94971da0 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -814,6 +814,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
 
     cdef struct PandasOptions:
         c_bool strings_to_categorical
+        c_bool zero_copy_only
 
 cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index eb1911592f4c4..6165a6622b836 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -163,7 +163,7 @@ cdef class Column:
         sp_column.reset(new CColumn(boxed_field.sp_field, arr.sp_array))
         return pyarrow_wrap_column(sp_column)
 
-    def to_pandas(self, strings_to_categorical=False):
+    def to_pandas(self, strings_to_categorical=False, zero_copy_only=False):
         """
         Convert the arrow::Column to a pandas.Series
 
@@ -175,7 +175,9 @@ cdef class Column:
             PyObject* out
             PandasOptions options
 
-        options = PandasOptions(strings_to_categorical=strings_to_categorical)
+        options = PandasOptions(
+            strings_to_categorical=strings_to_categorical,
+            zero_copy_only=zero_copy_only)
 
         with nogil:
             check_status(libarrow.ConvertColumnToPandas(options,
@@ -857,7 +859,7 @@ cdef class Table:
         return pyarrow_wrap_table(c_table)
 
     def to_pandas(self, nthreads=None, strings_to_categorical=False,
-                  memory_pool=None):
+                  memory_pool=None, zero_copy_only=False):
         """
         Convert the arrow::Table to a pandas DataFrame
 
@@ -871,6 +873,9 @@ cdef class Table:
             Encode string (UTF8) and binary types to pandas.Categorical
         memory_pool: MemoryPool, optional
             Specific memory pool to use to allocate casted columns
+        zero_copy_only : boolean, default False
+            Raise an ArrowException if this function call would require copying
+            the underlying data
 
         Returns
         -------
@@ -878,7 +883,9 @@ cdef class Table:
         """
         cdef:
             PandasOptions options
-        options = PandasOptions(strings_to_categorical=strings_to_categorical)
+        options = PandasOptions(
+            strings_to_categorical=strings_to_categorical,
+            zero_copy_only=zero_copy_only)
         self._check_nullptr()
         if nthreads is None:
             nthreads = cpu_count()
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 6d146f9778c05..ddb7eb41c1305 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -213,6 +213,46 @@ def test_float_no_nulls(self):
         schema = pa.schema(fields)
         self._check_pandas_roundtrip(df, expected_schema=schema)
 
+    def test_zero_copy_success(self):
+        result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True)
+        npt.assert_array_equal(result, [0, 1, 2])
+
+    def test_zero_copy_failure_on_object_types(self):
+        with self.assertRaises(pa.ArrowException):
+            pa.array(['A', 'B', 'C']).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_with_int_when_nulls(self):
+        with self.assertRaises(pa.ArrowException):
+            pa.array([0, 1, None]).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_with_float_when_nulls(self):
+        with self.assertRaises(pa.ArrowException):
+            pa.array([0.0, 1.0, None]).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_on_bool_types(self):
+        with self.assertRaises(pa.ArrowException):
+            pa.array([True, False]).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_on_list_types(self):
+        arr = np.array([[1, 2], [8, 9]], dtype=object)
+
+        with self.assertRaises(pa.ArrowException):
+            pa.array(arr).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_on_timestamp_types(self):
+        arr = np.array(['2007-07-13'], dtype='datetime64[ns]')
+
+        with self.assertRaises(pa.ArrowException):
+            pa.array(arr).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_dictionaries(self):
+        arr = pa.DictionaryArray.from_arrays(
+            np.array([0, 0]),
+            np.array(['A']))
+
+        with self.assertRaises(pa.ArrowException):
+            arr.to_pandas(zero_copy_only=True)
+
     def test_float_nulls(self):
         num_values = 100
 

From 7abaa006e021ffa96bad4b0c7fc99358e289e250 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 26 Oct 2017 11:14:32 -0400
Subject: [PATCH 1188/1644] ARROW-587: Add fix version to PR merge tool

This was ported from parquet-mr/parquet-cpp and modified to work in Python 2 and 3

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1248 from wesm/ARROW-587 and squashes the following commits:

da49b1c1 [Wes McKinney] Another buglet
d65a1b5c [Wes McKinney] Fix a python2 only statement
938b6dc6 [Wes McKinney] Fix script to pass right JIRA object
bbbce868 [Wes McKinney] Add fix version to PR merge tool (ported from apache/parquet-mr)
---
 dev/merge_arrow_pr.py | 42 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index 99ccc43394f27..6c0e663769e16 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -257,11 +257,47 @@ def resolve_jira(title, merge_branches, comment):
     print("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n"
           % (cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))
 
+    jira_fix_versions = _get_fix_version(asf_jira, merge_branches)
+
     resolve = [x for x in asf_jira.transitions(jira_id)
                if x['name'] == "Resolve Issue"][0]
-    asf_jira.transition_issue(jira_id, resolve["id"], comment=comment)
-
-    print("Succesfully resolved %s!" % (jira_id))
+    asf_jira.transition_issue(jira_id, resolve["id"], comment=comment,
+                              fixVersions=jira_fix_versions)
+
+    print("Successfully resolved %s!" % (jira_id))
+
+
+def _get_fix_version(asf_jira, merge_branches):
+    versions = asf_jira.project_versions("ARROW")
+    versions = sorted(versions, key=lambda x: x.name, reverse=True)
+    versions = [x for x in versions if not x.raw['released']]
+
+    default_fix_versions = [fix_version_from_branch(x, versions).name
+                            for x in merge_branches]
+    for v in default_fix_versions:
+        # Handles the case where we have forked a release branch but not yet
+        # made the release.  In this case, if the PR is committed to the master
+        # branch and the release branch, we only consider the release branch to
+        # be the fix version. E.g. it is not valid to have both 1.1.0 and 1.0.0
+        # as fix versions.
+        (major, minor, patch) = v.split(".")
+        if patch == "0":
+            previous = "%s.%s.%s" % (major, int(minor) - 1, 0)
+            if previous in default_fix_versions:
+                default_fix_versions = [x for x in default_fix_versions
+                                        if x != v]
+    default_fix_versions = ",".join(default_fix_versions)
+
+    fix_versions = input("Enter comma-separated fix version(s) [%s]: "
+                         % default_fix_versions)
+    if fix_versions == "":
+        fix_versions = default_fix_versions
+    fix_versions = fix_versions.replace(" ", "").split(",")
+
+    def get_version_json(version_str):
+        return [x for x in versions if x.name == version_str][0].raw
+
+    return [get_version_json(v) for v in fix_versions]
 
 
 if not JIRA_USERNAME:

From a385e2b963c1141b3febc3e27c01b62ce35272cb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 26 Oct 2017 13:39:05 -0400
Subject: [PATCH 1189/1644] ARROW-1739: [Python] Fix broken build due to using
 unittest.TestCase methods

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1255 from wesm/ARROW-1739 and squashes the following commits:

f8d2457f [Wes McKinney] Fix flake
101f236c [Wes McKinney] Use pytest.raises instead of self.assertRaises
---
 python/pyarrow/array.pxi                    |  3 ++-
 python/pyarrow/tests/test_convert_pandas.py | 14 +++++++-------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index c57eda1da759f..7da5c3caffdc2 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -373,7 +373,8 @@ cdef class Array:
 
         return pyarrow_wrap_array(result)
 
-    def to_pandas(self, c_bool strings_to_categorical=False, zero_copy_only=False):
+    def to_pandas(self, c_bool strings_to_categorical=False,
+                  zero_copy_only=False):
         """
         Convert to an array object suitable for use in pandas
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index ddb7eb41c1305..8360dae54d613 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -218,31 +218,31 @@ def test_zero_copy_success(self):
         npt.assert_array_equal(result, [0, 1, 2])
 
     def test_zero_copy_failure_on_object_types(self):
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             pa.array(['A', 'B', 'C']).to_pandas(zero_copy_only=True)
 
     def test_zero_copy_failure_with_int_when_nulls(self):
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             pa.array([0, 1, None]).to_pandas(zero_copy_only=True)
 
     def test_zero_copy_failure_with_float_when_nulls(self):
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             pa.array([0.0, 1.0, None]).to_pandas(zero_copy_only=True)
 
     def test_zero_copy_failure_on_bool_types(self):
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             pa.array([True, False]).to_pandas(zero_copy_only=True)
 
     def test_zero_copy_failure_on_list_types(self):
         arr = np.array([[1, 2], [8, 9]], dtype=object)
 
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             pa.array(arr).to_pandas(zero_copy_only=True)
 
     def test_zero_copy_failure_on_timestamp_types(self):
         arr = np.array(['2007-07-13'], dtype='datetime64[ns]')
 
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             pa.array(arr).to_pandas(zero_copy_only=True)
 
     def test_zero_copy_dictionaries(self):
@@ -250,7 +250,7 @@ def test_zero_copy_dictionaries(self):
             np.array([0, 0]),
             np.array(['A']))
 
-        with self.assertRaises(pa.ArrowException):
+        with pytest.raises(pa.ArrowException):
             arr.to_pandas(zero_copy_only=True)
 
     def test_float_nulls(self):

From 3596a43f8be224194368bf0da0cd25b7bcd178ac Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 26 Oct 2017 13:51:19 -0400
Subject: [PATCH 1190/1644] ARROW-1737: [GLib] Use G_DECLARE_DERIVABLE_TYPE

Because GTK-Doc scans G_DECLARE_DERIVABLE_TYPE to find class definition.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1254 from kou/glib-use-g-declare-derivable-type and squashes the following commits:

0bf4ad3a [Kouhei Sutou] [GLib] Use G_DECLARE_DERIVABLE_TYPE
---
 c_glib/arrow-glib/array-builder.h | 28 +++++++++++++++++----------
 c_glib/arrow-glib/compute.h       | 14 +++++++++-----
 c_glib/arrow-glib/gobject-type.h  | 32 ++++++-------------------------
 c_glib/arrow-glib/reader.h        | 28 +++++++++++++++++----------
 4 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index 7a53bff11de1f..d9e91212e1663 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -25,11 +25,15 @@
 G_BEGIN_DECLS
 
 #define GARROW_TYPE_ARRAY_BUILDER (garrow_array_builder_get_type())
-GARROW_DECLARE_TYPE(GArrowArrayBuilder,
-                    garrow_array_builder,
-                    GARROW,
-                    ARRAY_BUILDER,
-                    GObject)
+G_DECLARE_DERIVABLE_TYPE(GArrowArrayBuilder,
+                         garrow_array_builder,
+                         GARROW,
+                         ARRAY_BUILDER,
+                         GObject)
+struct _GArrowArrayBuilderClass
+{
+  GObjectClass parent_class;
+};
 
 GArrowArray        *garrow_array_builder_finish   (GArrowArrayBuilder *builder,
                                                    GError **error);
@@ -156,11 +160,15 @@ gboolean garrow_int_array_builder_append_nulls(GArrowIntArrayBuilder *builder,
 
 
 #define GARROW_TYPE_UINT_ARRAY_BUILDER (garrow_uint_array_builder_get_type())
-GARROW_DECLARE_TYPE(GArrowUIntArrayBuilder,
-                    garrow_uint_array_builder,
-                    GARROW,
-                    UINT_ARRAY_BUILDER,
-                    GArrowArrayBuilder)
+G_DECLARE_DERIVABLE_TYPE(GArrowUIntArrayBuilder,
+                         garrow_uint_array_builder,
+                         GARROW,
+                         UINT_ARRAY_BUILDER,
+                         GArrowArrayBuilder)
+struct _GArrowUIntArrayBuilderClass
+{
+  GArrowArrayBuilderClass parent_class;
+};
 
 GArrowUIntArrayBuilder *garrow_uint_array_builder_new(void);
 
diff --git a/c_glib/arrow-glib/compute.h b/c_glib/arrow-glib/compute.h
index cdae5790e8346..cd9dd2b63f07c 100644
--- a/c_glib/arrow-glib/compute.h
+++ b/c_glib/arrow-glib/compute.h
@@ -24,11 +24,15 @@
 G_BEGIN_DECLS
 
 #define GARROW_TYPE_CAST_OPTIONS (garrow_cast_options_get_type())
-GARROW_DECLARE_TYPE(GArrowCastOptions,
-                    garrow_cast_options,
-                    GARROW,
-                    CAST_OPTIONS,
-                    GObject)
+G_DECLARE_DERIVABLE_TYPE(GArrowCastOptions,
+                         garrow_cast_options,
+                         GARROW,
+                         CAST_OPTIONS,
+                         GObject)
+struct _GArrowCastOptionsClass
+{
+  GObjectClass parent_class;
+};
 
 GArrowCastOptions *garrow_cast_options_new(void);
 
diff --git a/c_glib/arrow-glib/gobject-type.h b/c_glib/arrow-glib/gobject-type.h
index a2f8397b941a9..41e3a2953d436 100644
--- a/c_glib/arrow-glib/gobject-type.h
+++ b/c_glib/arrow-glib/gobject-type.h
@@ -21,27 +21,12 @@
 
 #include <glib-object.h>
 
-#ifdef G_DECLARE_DERIVABLE_TYPE
-#  define GARROW_DECLARE_TYPE(ObjectName,       \
-                              object_name,      \
-                              MODULE_NAME,      \
-                              OBJECT_NAME,      \
-                              ParentName)       \
-  G_DECLARE_DERIVABLE_TYPE(ObjectName,          \
-                           object_name,         \
-                           MODULE_NAME,         \
-                           OBJECT_NAME,         \
-                           ParentName)          \
-  struct _ ## ObjectName ## Class               \
-  {                                             \
-    ParentName ## Class parent_class;           \
-  };
-#else
-#  define GARROW_DECLARE_TYPE(ObjectName,                               \
-                              object_name,                              \
-                              MODULE_NAME,                              \
-                              OBJECT_NAME,                              \
-                              ParentName)                               \
+#ifndef G_DECLARE_DERIVABLE_TYPE
+#  define G_DECLARE_DERIVABLE_TYPE(ObjectName,                          \
+                                   object_name,                         \
+                                   MODULE_NAME,                         \
+                                   OBJECT_NAME,                         \
+                                   ParentName)                          \
   typedef struct _ ## ObjectName ObjectName;                            \
   typedef struct _ ## ObjectName ## Class ObjectName ## Class;          \
                                                                         \
@@ -50,11 +35,6 @@
     ParentName parent_instance;                                         \
   };                                                                    \
                                                                         \
-  struct _ ## ObjectName ## Class                                       \
-  {                                                                     \
-    ParentName ## Class parent_class;                                   \
-  };                                                                    \
-                                                                        \
   GType object_name ## _get_type(void) G_GNUC_CONST;                    \
                                                                         \
   static inline ObjectName *                                            \
diff --git a/c_glib/arrow-glib/reader.h b/c_glib/arrow-glib/reader.h
index 964146ab8d721..daec676d6b4b2 100644
--- a/c_glib/arrow-glib/reader.h
+++ b/c_glib/arrow-glib/reader.h
@@ -31,11 +31,15 @@
 G_BEGIN_DECLS
 
 #define GARROW_TYPE_RECORD_BATCH_READER (garrow_record_batch_reader_get_type())
-GARROW_DECLARE_TYPE(GArrowRecordBatchReader,
-                    garrow_record_batch_reader,
-                    GARROW,
-                    RECORD_BATCH_READER,
-                    GObject)
+G_DECLARE_DERIVABLE_TYPE(GArrowRecordBatchReader,
+                         garrow_record_batch_reader,
+                         GARROW,
+                         RECORD_BATCH_READER,
+                         GObject)
+struct _GArrowRecordBatchReaderClass
+{
+  GObjectClass parent_class;
+};
 
 GArrowSchema *garrow_record_batch_reader_get_schema(
   GArrowRecordBatchReader *reader);
@@ -57,11 +61,15 @@ GArrowRecordBatch *garrow_record_batch_reader_read_next(
 
 
 #define GARROW_TYPE_TABLE_BATCH_READER (garrow_table_batch_reader_get_type())
-GARROW_DECLARE_TYPE(GArrowTableBatchReader,
-                    garrow_table_batch_reader,
-                    GARROW,
-                    TABLE_BATCH_READER,
-                    GArrowRecordBatchReader)
+G_DECLARE_DERIVABLE_TYPE(GArrowTableBatchReader,
+                         garrow_table_batch_reader,
+                         GARROW,
+                         TABLE_BATCH_READER,
+                         GArrowRecordBatchReader)
+struct _GArrowTableBatchReaderClass
+{
+  GArrowRecordBatchReaderClass parent_class;
+};
 
 GArrowTableBatchReader *garrow_table_batch_reader_new(GArrowTable *table);
 

From 2e040896d59dfeea7b05bfc27915efbea59f7171 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 26 Oct 2017 13:52:23 -0400
Subject: [PATCH 1191/1644] ARROW-1736: [GLib] Add
 GArrowCastOptions:allow-time-truncate

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1253 from kou/glib-support-allow-time-truncate-cast-option and squashes the following commits:

609eaafb [Kouhei Sutou] [GLib] Add GArrowCastOptions:allow-time-truncate
---
 c_glib/arrow-glib/compute.cpp | 32 ++++++++++++++++++++++++++++++--
 c_glib/test/test-cast.rb      | 23 +++++++++++++++++++++++
 2 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/c_glib/arrow-glib/compute.cpp b/c_glib/arrow-glib/compute.cpp
index 9134aa675c50f..ce427e616744f 100644
--- a/c_glib/arrow-glib/compute.cpp
+++ b/c_glib/arrow-glib/compute.cpp
@@ -40,7 +40,8 @@ typedef struct GArrowCastOptionsPrivate_ {
 
 enum {
   PROP_0,
-  PROP_ALLOW_INT_OVERFLOW
+  PROP_ALLOW_INT_OVERFLOW,
+  PROP_ALLOW_TIME_TRUNCATE
 };
 
 G_DEFINE_TYPE_WITH_PRIVATE(GArrowCastOptions,
@@ -64,6 +65,9 @@ garrow_cast_options_set_property(GObject *object,
   case PROP_ALLOW_INT_OVERFLOW:
     priv->options.allow_int_overflow = g_value_get_boolean(value);
     break;
+  case PROP_ALLOW_TIME_TRUNCATE:
+    priv->options.allow_time_truncate = g_value_get_boolean(value);
+    break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
     break;
@@ -82,6 +86,9 @@ garrow_cast_options_get_property(GObject *object,
   case PROP_ALLOW_INT_OVERFLOW:
     g_value_set_boolean(value, priv->options.allow_int_overflow);
     break;
+  case PROP_ALLOW_TIME_TRUNCATE:
+    g_value_set_boolean(value, priv->options.allow_time_truncate);
+    break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
     break;
@@ -103,12 +110,33 @@ garrow_cast_options_class_init(GArrowCastOptionsClass *klass)
   gobject_class->set_property = garrow_cast_options_set_property;
   gobject_class->get_property = garrow_cast_options_get_property;
 
-  spec = g_param_spec_boolean("allow_int_overflow",
+  /**
+   * GArrowCastOptions:allow-int-overflow:
+   *
+   * Whether integer overflow is allowed or not.
+   *
+   * Since: 0.7.0
+   */
+  spec = g_param_spec_boolean("allow-int-overflow",
                               "Allow int overflow",
                               "Whether integer overflow is allowed or not",
                               FALSE,
                               static_cast<GParamFlags>(G_PARAM_READWRITE));
   g_object_class_install_property(gobject_class, PROP_ALLOW_INT_OVERFLOW, spec);
+
+  /**
+   * GArrowCastOptions:allow-time-truncate:
+   *
+   * Whether truncating time value is allowed or not.
+   *
+   * Since: 0.8.0
+   */
+  spec = g_param_spec_boolean("allow-time-truncate",
+                              "Allow time truncate",
+                              "Whether truncating time value is allowed or not",
+                              FALSE,
+                              static_cast<GParamFlags>(G_PARAM_READWRITE));
+  g_object_class_install_property(gobject_class, PROP_ALLOW_TIME_TRUNCATE, spec);
 }
 
 /**
diff --git a/c_glib/test/test-cast.rb b/c_glib/test/test-cast.rb
index f32cd83cc9d8a..6c29e85f155c2 100644
--- a/c_glib/test/test-cast.rb
+++ b/c_glib/test/test-cast.rb
@@ -42,4 +42,27 @@ def test_true
                                                  options))
     end
   end
+
+  sub_test_case("allow-time-truncate") do
+    def test_default
+      require_gi(1, 42, 0)
+      after_epoch = 1504953190854 # 2017-09-09T10:33:10.854Z
+      second_timestamp = Arrow::TimestampDataType.new(:second)
+      assert_raise(Arrow::Error::Invalid) do
+        build_timestamp_array(:milli, [after_epoch]).cast(second_timestamp)
+      end
+    end
+
+    def test_true
+      options = Arrow::CastOptions.new
+      options.allow_time_truncate = true
+      after_epoch_in_milli = 1504953190854 # 2017-09-09T10:33:10.854Z
+      second_array = build_timestamp_array(:second,
+                                           [after_epoch_in_milli / 1000])
+      milli_array  = build_timestamp_array(:milli, [after_epoch_in_milli])
+      second_timestamp = Arrow::TimestampDataType.new(:second)
+      assert_equal(second_array,
+                   milli_array.cast(second_timestamp, options))
+    end
+  end
 end

From 59030fe1d319d36157554fb7880e9b574c60a3d1 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Thu, 26 Oct 2017 22:44:46 -0400
Subject: [PATCH 1192/1644] ARROW-1730, ARROW-1738: [Python] Fix wrong datetime
 conversion

This closes [ARROW-1730](https://issues.apache.org/jira/browse/ARROW-1730) and [ARROW-1738](https://issues.apache.org/jira/browse/ARROW-1738).

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1256 from Licht-T/fix-pylong-force-cast-to-date-type and squashes the following commits:

b4334641 [Wes McKinney] Code review comments, Add test cases for invalid values
23ad1a52 [Licht-T] Fix C++ lint issues
d9cd15b0 [Licht-T] TST: Add tests for datetime converter
560a97e4 [Licht-T] BUG: Implement time unit resolution converter
c1063eb2 [Licht-T] BUG: Fix PyLong force casting to PyDateTime/PyDate type
---
 cpp/src/arrow/python/builtin_convert.cc      | 44 +++++++++++--
 cpp/src/arrow/python/util/datetime.h         | 14 ++++
 python/pyarrow/tests/test_convert_builtin.py | 69 ++++++++++++++++++++
 3 files changed, 122 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index f7862d1516230..d52627ebfee12 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -522,18 +522,51 @@ class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Conver
 class DateConverter : public TypedConverterVisitor<Date64Builder, DateConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
-    auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
-    return typed_builder_->Append(PyDate_to_ms(pydate));
+    int64_t t;
+    if (PyDate_Check(item.obj())) {
+      auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
+      t = PyDate_to_ms(pydate);
+    } else {
+      t = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      RETURN_IF_PYERROR();
+    }
+    return typed_builder_->Append(t);
   }
 };
 
 class TimestampConverter
     : public TypedConverterVisitor<Date64Builder, TimestampConverter> {
  public:
+  explicit TimestampConverter(TimeUnit::type unit) : unit_(unit) {}
+
   inline Status AppendItem(const OwnedRef& item) {
-    auto pydatetime = reinterpret_cast<PyDateTime_DateTime*>(item.obj());
-    return typed_builder_->Append(PyDateTime_to_us(pydatetime));
+    int64_t t;
+    if (PyDateTime_Check(item.obj())) {
+      auto pydatetime = reinterpret_cast<PyDateTime_DateTime*>(item.obj());
+
+      switch (unit_) {
+        case TimeUnit::SECOND:
+          t = PyDateTime_to_s(pydatetime);
+          break;
+        case TimeUnit::MILLI:
+          t = PyDateTime_to_ms(pydatetime);
+          break;
+        case TimeUnit::MICRO:
+          t = PyDateTime_to_us(pydatetime);
+          break;
+        case TimeUnit::NANO:
+          t = PyDateTime_to_ns(pydatetime);
+          break;
+      }
+    } else {
+      t = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      RETURN_IF_PYERROR();
+    }
+    return typed_builder_->Append(t);
   }
+
+ private:
+  TimeUnit::type unit_;
 };
 
 class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConverter> {
@@ -687,7 +720,8 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
     case Type::DATE64:
       return std::make_shared<DateConverter>();
     case Type::TIMESTAMP:
-      return std::make_shared<TimestampConverter>();
+      return std::make_shared<TimestampConverter>(
+          static_cast<const TimestampType&>(*type).unit());
     case Type::DOUBLE:
       return std::make_shared<DoubleConverter>();
     case Type::BINARY:
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index 782960f62518e..c110bc64a2a2f 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -247,12 +247,26 @@ static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   return total_seconds * 1000;
 }
 
+static inline int64_t PyDateTime_to_s(PyDateTime_DateTime* pydatetime) {
+  return PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(pydatetime)) / 1000LL;
+}
+
+static inline int64_t PyDateTime_to_ms(PyDateTime_DateTime* pydatetime) {
+  int64_t date_ms = PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(pydatetime));
+  int ms = PyDateTime_DATE_GET_MICROSECOND(pydatetime) / 1000;
+  return date_ms + ms;
+}
+
 static inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) {
   int64_t ms = PyDate_to_ms(reinterpret_cast<PyDateTime_Date*>(pydatetime));
   int us = PyDateTime_DATE_GET_MICROSECOND(pydatetime);
   return ms * 1000 + us;
 }
 
+static inline int64_t PyDateTime_to_ns(PyDateTime_DateTime* pydatetime) {
+  return PyDateTime_to_us(pydatetime) * 1000;
+}
+
 static inline int32_t PyDate_to_days(PyDateTime_Date* pydate) {
   return static_cast<int32_t>(PyDate_to_ms(pydate) / 86400000LL);
 }
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index d18ed9506bbbb..414266ddb14ed 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -197,6 +197,75 @@ def test_timestamp(self):
         assert arr[3].as_py() == datetime.datetime(2010, 8, 13, 5,
                                                    46, 57, 437699)
 
+    def test_timestamp_with_unit(self):
+        data = [
+            datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
+        ]
+
+        s = pa.timestamp('s')
+        ms = pa.timestamp('ms')
+        us = pa.timestamp('us')
+        ns = pa.timestamp('ns')
+
+        arr_s = pa.array(data, type=s)
+        assert len(arr_s) == 1
+        assert arr_s.type == s
+        assert arr_s[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                     23, 34, 0)
+
+        arr_ms = pa.array(data, type=ms)
+        assert len(arr_ms) == 1
+        assert arr_ms.type == ms
+        assert arr_ms[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                      23, 34, 123000)
+
+        arr_us = pa.array(data, type=us)
+        assert len(arr_us) == 1
+        assert arr_us.type == us
+        assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                      23, 34, 123456)
+
+        arr_ns = pa.array(data, type=ns)
+        assert len(arr_ns) == 1
+        assert arr_ns.type == ns
+        assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                      23, 34, 123456)
+
+    def test_timestamp_from_int_with_unit(self):
+        data = [1]
+
+        s = pa.timestamp('s')
+        ms = pa.timestamp('ms')
+        us = pa.timestamp('us')
+        ns = pa.timestamp('ns')
+
+        arr_s = pa.array(data, type=s)
+        assert len(arr_s) == 1
+        assert arr_s.type == s
+        assert str(arr_s[0]) == "Timestamp('1970-01-01 00:00:01')"
+
+        arr_ms = pa.array(data, type=ms)
+        assert len(arr_ms) == 1
+        assert arr_ms.type == ms
+        assert str(arr_ms[0]) == "Timestamp('1970-01-01 00:00:00.001000')"
+
+        arr_us = pa.array(data, type=us)
+        assert len(arr_us) == 1
+        assert arr_us.type == us
+        assert str(arr_us[0]) == "Timestamp('1970-01-01 00:00:00.000001')"
+
+        arr_ns = pa.array(data, type=ns)
+        assert len(arr_ns) == 1
+        assert arr_ns.type == ns
+        assert str(arr_ns[0]) == "Timestamp('1970-01-01 00:00:00.000000001')"
+
+        with pytest.raises(pa.ArrowException):
+            class CustomClass():
+                pass
+            pa.array([1, CustomClass()], type=ns)
+            pa.array([1, CustomClass()], type=pa.date32())
+            pa.array([1, CustomClass()], type=pa.date64())
+
     def test_mixed_nesting_levels(self):
         pa.array([1, 2, None])
         pa.array([[1], [2], None])

From 2ed886ee707822b398cdb22f4d4c10116fa1d9f3 Mon Sep 17 00:00:00 2001
From: John Jenkins <jjenkins@wolve.com>
Date: Thu, 26 Oct 2017 22:45:55 -0400
Subject: [PATCH 1193/1644] ARROW-1723: [C++] add ARROW_STATIC to mark static
 libs on Windows

Add a preprocessor macro ARROW_STATIC when doing static library builds on Windows. Clients developing/building off the static library will also need to define this - please let me know how this should be documented, if this is an acceptable approach.

Author: John Jenkins <jjenkins@wolve.com>

Closes #1244 from JohnPJenkins/windows-static-lib and squashes the following commits:

46b1a215 [John Jenkins] [ARROW-1723] rework static lib generation on Windows
---
 cpp/README.md                      |  8 +++++
 cpp/cmake_modules/BuildUtils.cmake | 49 +++++++++++++++++++-----------
 cpp/src/arrow/util/visibility.h    |  4 ++-
 3 files changed, 42 insertions(+), 19 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 9c26842359c70..60383535b1596 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -69,6 +69,14 @@ Simple release build:
 
 Detailed unit test logs will be placed in the build directory under `build/test-logs`.
 
+### Statically linking to Arrow on Windows
+
+The Arrow headers on Windows static library builds (enabled by the CMake
+option `ARROW_BUILD_STATIC`) use the preprocessor macro `ARROW_STATIC` to
+suppress dllimport/dllexport marking of symbols. Projects that statically link
+against Arrow on Windows additionally need this definition. The Unix builds do
+not use the macro.
+
 ### Building/Running benchmarks
 
 Follow the directions for simple build except run cmake
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index fb09e4e95c6ae..207bb9aede0c5 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -98,21 +98,28 @@ function(ADD_ARROW_LIB LIB_NAME)
     message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
   endif()
 
-  add_library(${LIB_NAME}_objlib OBJECT
-    ${ARG_SOURCES}
-  )
-
-  if (ARG_DEPENDENCIES)
-    add_dependencies(${LIB_NAME}_objlib ${ARG_DEPENDENCIES})
+  if(MSVC)
+    set(LIB_DEPS ${ARG_SOURCES})
+    set(EXTRA_DEPS ${ARG_DEPENDENCIES})
+  else()
+    add_library(${LIB_NAME}_objlib OBJECT
+      ${ARG_SOURCES})
+    # Necessary to make static linking into other shared libraries work properly
+    set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
+    if (ARG_DEPENDENCIES)
+      add_dependencies(${LIB_NAME}_objlib ${ARG_DEPENDENCIES})
+    endif()
+    set(LIB_DEPS $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
+    set(EXTRA_DEPS)
   endif()
 
-  # Necessary to make static linking into other shared libraries work properly
-  set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
-
   set(RUNTIME_INSTALL_DIR bin)
 
   if (ARROW_BUILD_SHARED)
-    add_library(${LIB_NAME}_shared SHARED $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
+    add_library(${LIB_NAME}_shared SHARED ${LIB_DEPS})
+    if (EXTRA_DEPS)
+      add_dependencies(${LIB_NAME}_shared ${EXTRA_DEPS})
+    endif()
 
     if(APPLE)
       # On OS X, you can avoid linking at library load time and instead
@@ -154,22 +161,28 @@ function(ADD_ARROW_LIB LIB_NAME)
   endif()
 
   if (ARROW_BUILD_STATIC)
-      if (MSVC)
-        set(LIB_NAME_STATIC ${LIB_NAME}_static)
-      else()
-        set(LIB_NAME_STATIC ${LIB_NAME})
-      endif()
-      add_library(${LIB_NAME}_static STATIC $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
+    add_library(${LIB_NAME}_static STATIC ${LIB_DEPS})
+    if(EXTRA_DEPS)
+      add_dependencies(${LIB_NAME}_static ${EXTRA_DEPS})
+    endif()
+
+    if (MSVC)
+      set(LIB_NAME_STATIC ${LIB_NAME}_static)
+      target_compile_definitions(${LIB_NAME}_static PUBLIC ARROW_STATIC)
+    else()
+      set(LIB_NAME_STATIC ${LIB_NAME})
+    endif()
+
     set_target_properties(${LIB_NAME}_static
       PROPERTIES
       LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
       OUTPUT_NAME ${LIB_NAME_STATIC})
 
-  target_link_libraries(${LIB_NAME}_static
+    target_link_libraries(${LIB_NAME}_static
       LINK_PUBLIC ${ARG_STATIC_LINK_LIBS}
       LINK_PRIVATE ${ARG_STATIC_PRIVATE_LINK_LIBS})
 
-  install(TARGETS ${LIB_NAME}_static
+    install(TARGETS ${LIB_NAME}_static
       RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
       LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
       ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
index ea78e57204efc..119c55df3b619 100644
--- a/cpp/src/arrow/util/visibility.h
+++ b/cpp/src/arrow/util/visibility.h
@@ -25,7 +25,9 @@
 #pragma GCC diagnostic ignored "-Wattributes"
 #endif
 
-#ifdef ARROW_EXPORTING
+#ifdef ARROW_STATIC
+#define ARROW_EXPORT
+#elif defined(ARROW_EXPORTING)
 #define ARROW_EXPORT __declspec(dllexport)
 #else
 #define ARROW_EXPORT __declspec(dllimport)

From 4db0046af18bfe90ecec5510e1962e81b8eccf6b Mon Sep 17 00:00:00 2001
From: Benjamin Goldberg <ben@spothero.com>
Date: Thu, 26 Oct 2017 22:55:18 -0400
Subject: [PATCH 1194/1644] ARROW-1555 [Python] Implement Dask exists function

Hi Maintainers,

This is a tiny patch to add the `exists` implementation for the Dask filesystem closing [ARROW-1555](https://issues.apache.org/jira/browse/ARROW-1555).

I ran into an issue attempting to use the new `S3FSWrapper` with `s3fs` to upload a parquet file to S3. The following pseudo-code raises a `NotImplemented` error because the parquet implementation makes a call to check if the directory exists on the fs first. Because there is no implementation for `DaskFilesystem`, it defaults to the base classes' `exists` which raises a `NotImplemented` error.

```
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq

df = pd.DataFrame(<some_data>)
pa_table = pa.Table.from_pandas(df)
dst_path = s3://<bucket>/<path>.parq.snappy
s3_fs = pa.filesystem.S3FSWrapper(fs=s3fs.S3FileSystem())
pq.write_to_dataset(pa_table, dst_path, filesystem=s3_fs)
```

Author: Benjamin Goldberg <ben@spothero.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1240 from benjigoldberg/feature/dask-fs-exists-impl and squashes the following commits:

cf9946c9 [Wes McKinney] Fix flakes
bd9f2473 [Benjamin Goldberg] make filestore check private
5d9a1614 [Benjamin Goldberg] Reverse order of fs type checks
d285a84f [Benjamin Goldberg] Add an attribute on FileSystem to indicate whether its a file store or object store
51d2cbfa [Benjamin Goldberg] Reference the fs instead of os
649f572a [Benjamin Goldberg] Implement Dask exists function
---
 python/pyarrow/filesystem.py         | 23 +++++++++++++++++++++++
 python/pyarrow/parquet.py            |  4 ++--
 python/pyarrow/tests/test_parquet.py |  1 +
 3 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 8d2d8fcd342f7..926df0e30e565 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -135,6 +135,13 @@ def isfile(self, path):
         """
         raise NotImplementedError
 
+    def _isfilestore(self):
+        """
+        Returns True if this FileSystem is a unix-style file store with
+        directories.
+        """
+        raise NotImplementedError
+
     def read_parquet(self, path, columns=None, metadata=None, schema=None,
                      nthreads=1, use_pandas_metadata=False):
         """
@@ -209,6 +216,10 @@ def isdir(self, path):
     def isfile(self, path):
         return os.path.isfile(path)
 
+    @implements(FileSystem._isfilestore)
+    def _isfilestore(self):
+        return True
+
     @implements(FileSystem.exists)
     def exists(self, path):
         return os.path.exists(path)
@@ -247,10 +258,22 @@ def isdir(self, path):
     def isfile(self, path):
         raise NotImplementedError("Unsupported file system API")
 
+    @implements(FileSystem._isfilestore)
+    def _isfilestore(self):
+        """
+        Object Stores like S3 and GCSFS are based on key lookups, not true
+        file-paths
+        """
+        return False
+
     @implements(FileSystem.delete)
     def delete(self, path, recursive=False):
         return self.fs.rm(path, recursive=recursive)
 
+    @implements(FileSystem.exists)
+    def exists(self, path):
+        return self.fs.exists(path)
+
     @implements(FileSystem.mkdir)
     def mkdir(self, path):
         return self.fs.mkdir(path)
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 0a40f5fb7fdce..9dcc30c8af479 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -985,7 +985,7 @@ def write_to_dataset(table, root_path, partition_cols=None,
     else:
         fs = _ensure_filesystem(filesystem)
 
-    if not fs.exists(root_path):
+    if fs._isfilestore() and not fs.exists(root_path):
         fs.mkdir(root_path)
 
     if partition_cols is not None and len(partition_cols) > 0:
@@ -1004,7 +1004,7 @@ def write_to_dataset(table, root_path, partition_cols=None,
             subtable = Table.from_pandas(subgroup,
                                          preserve_index=preserve_index)
             prefix = "/".join([root_path, subdir])
-            if not fs.exists(prefix):
+            if fs._isfilestore() and not fs.exists(prefix):
                 fs.mkdir(prefix)
             outfile = compat.guid() + ".parquet"
             full_path = "/".join([prefix, outfile])
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 09184cc056588..a7fe98ce71cd1 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -978,6 +978,7 @@ def _visit_level(base_dir, level, part_keys):
                 part_table = pa.Table.from_pandas(filtered_df)
                 with fs.open(file_path, 'wb') as f:
                     _write_table(part_table, f)
+                assert fs.exists(file_path)
             else:
                 _visit_level(level_dir, level + 1, this_part_keys)
 

From 2eb78b023c9957cb8dfe61022a1ce0cd11919a11 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 27 Oct 2017 10:37:38 -0400
Subject: [PATCH 1195/1644] ARROW-1728: [C++] Run clang-format checks in Travis
 CI

While this will result in more build failures, it will help keep master clean to avoid unrelated noise diffs in later patches (which has been frequently happening)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1251 from wesm/ARROW-1728 and squashes the following commits:

8a170fa2 [Wes McKinney] Revert flake
c57d5855 [Wes McKinney] Make travis_install_clang_tools.sh executable
6b5651e6 [Wes McKinney] Implement check-format in run_clang_format.py, run in Travis CI
---
 .travis.yml                           |  2 ++
 ci/travis_install_clang_tools.sh      |  0
 ci/travis_lint.sh                     |  4 +++
 ci/travis_script_cpp.sh               |  8 ------
 cpp/CMakeLists.txt                    |  8 +++---
 cpp/build-support/run_clang_format.py | 40 +++++++++++++++++----------
 6 files changed, 36 insertions(+), 26 deletions(-)
 mode change 100644 => 100755 ci/travis_install_clang_tools.sh

diff --git a/.travis.yml b/.travis.yml
index c682a9d9db89f..039ae95208b74 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -56,6 +56,8 @@ matrix:
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_VALGRIND=1
     - export ARROW_TRAVIS_PLASMA=1
+    - export ARROW_TRAVIS_CLANG_FORMAT=1
+    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
old mode 100644
new mode 100755
diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
index 8c956646cb39e..e234b7b015b8d 100755
--- a/ci/travis_lint.sh
+++ b/ci/travis_lint.sh
@@ -26,6 +26,10 @@ pushd $TRAVIS_BUILD_DIR/cpp/lint
 cmake ..
 make lint
 
+if [ "$ARROW_TRAVIS_CLANG_FORMAT" == "1" ]; then
+  make check-format
+fi
+
 popd
 
 # Fail fast on style checks
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index a2079036c4549..3d61bc5b89b71 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -27,14 +27,6 @@ git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
 
 pushd $CPP_BUILD_DIR
 
-# ARROW-209: checks depending on the LLVM toolchain are disabled temporarily
-# until we are able to install the full LLVM toolchain in Travis CI again
-
-# if [ $TRAVIS_OS_NAME == "linux" ]; then
-#   make check-format
-#   make check-clang-tidy
-# fi
-
 ctest -VV -L unittest
 
 popd
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index a159b1e5674b4..d8dc5df88b4a4 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -446,10 +446,10 @@ add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py
 # runs clang format and exits with a non-zero exit code if any files need to be reformatted
 
 # TODO(wesm): Make this work in run_clang_format.py
-# add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py
-#   ${CLANG_FORMAT_VERSION}
-#   ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
-#   ${CMAKE_CURRENT_SOURCE_DIR}/src 1)
+add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py
+   ${CLANG_FORMAT_VERSION}
+   ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
+   ${CMAKE_CURRENT_SOURCE_DIR}/src 1)
 
 ############################################################
 # "make clang-tidy" and "make check-clang-tidy" targets
diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py
index f1a448f536101..fcf39ecc6a5f9 100755
--- a/cpp/build-support/run_clang_format.py
+++ b/cpp/build-support/run_clang_format.py
@@ -31,6 +31,12 @@
 EXCLUDE_GLOBS_FILENAME = sys.argv[2]
 SOURCE_DIR = sys.argv[3]
 
+if len(sys.argv) > 4:
+    CHECK_FORMAT = int(sys.argv[4]) == 1
+else:
+    CHECK_FORMAT = False
+
+
 exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")]
 
 files_to_format = []
@@ -49,18 +55,24 @@
         if not excluded:
             files_to_format.append(name)
 
-# TODO(wesm): Port this to work with Python, for check-format
-# NUM_CORRECTIONS=`$CLANG_FORMAT -output-replacements-xml  $@ |
-# grep offset | wc -l`
-# if [ "$NUM_CORRECTIONS" -gt "0" ]; then
-#   echo "clang-format suggested changes, please run 'make format'!!!!"
-#   exit 1
-# fi
+if CHECK_FORMAT:
+    output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml']
+                                     + files_to_format,
+                                     stderr=subprocess.STDOUT).decode('utf8')
+
+    to_fix = []
+    for line in output.split('\n'):
+        if 'offset' in line:
+            to_fix.append(line)
 
-try:
-    cmd = [CLANG_FORMAT, '-i'] + files_to_format
-    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-except Exception as e:
-    print(e)
-    print(' '.join(cmd))
-    raise
+    if len(to_fix) > 0:
+        print("clang-format checks failed, run 'make format' to fix")
+        sys.exit(-1)
+else:
+    try:
+        cmd = [CLANG_FORMAT, '-i'] + files_to_format
+        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except Exception as e:
+        print(e)
+        print(' '.join(cmd))
+        raise

From cc03a457e3a2e7f56f5325fcac780762ecef0522 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Sat, 28 Oct 2017 13:52:43 +0200
Subject: [PATCH 1196/1644] ARROW-1745: [Plasma] Include gtest after
 plasma/compat.h in tests.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1261 from robertnishihara/fixosxproblem and squashes the following commits:

50d1675 [Robert Nishihara] Include gtest after plasma/compat.h in tests.
---
 cpp/src/plasma/test/client_tests.cc        | 4 ++--
 cpp/src/plasma/test/serialization_tests.cc | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index 02b3832145186..0f19da5f72342 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -15,8 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "gtest/gtest.h"
-
 #include <assert.h>
 #include <signal.h>
 #include <stdlib.h>
@@ -29,6 +27,8 @@
 #include "plasma/plasma.h"
 #include "plasma/protocol.h"
 
+#include "gtest/gtest.h"
+
 namespace plasma {
 
 std::string test_executable;  // NOLINT
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
index 7c9d90133a6d4..b593b6ae94890 100644
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -15,8 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "gtest/gtest.h"
-
 #include <sys/types.h>
 #include <unistd.h>
 
@@ -25,6 +23,8 @@
 #include "plasma/plasma.h"
 #include "plasma/protocol.h"
 
+#include "gtest/gtest.h"
+
 namespace plasma {
 
 /**

From 74a934ae044fea47ea71a761cf71f60c1397108e Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Sat, 28 Oct 2017 13:54:11 +0200
Subject: [PATCH 1197/1644] ARROW-1689: [Python] Implement zero-copy
 conversions for DictionaryArray

This PR closes [ARROW-1689](https://issues.apache.org/jira/browse/ARROW-1689).
I want to add the zero-copy option after https://github.com/apache/arrow/pull/1233 merged.

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1237 from Licht-T/feature-categorical-index-zerocopy and squashes the following commits:

53342e8 [Wes McKinney] Use the PyCapsule API to preserve base references to C++ objects when no PyObject* is available to set as zero-copy ndarray base
0b847d1 [Wes McKinney] Fix flakes
4270b5d [Licht-T] Fix C++ lint issues
ddc6b84 [Licht-T] TST: Add test_zero_copy_dictionaries
e0561dc [Licht-T] ENH: Add zero_copy_only option check
de4ed3e [Licht-T] ENH: Implement Categorical Block Zero-Copy
---
 cpp/src/arrow/python/arrow_to_pandas.cc     | 120 ++++++++++++++++----
 python/pyarrow/tests/test_convert_pandas.py |  19 ++--
 2 files changed, 110 insertions(+), 29 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 0c2e0ad85bfa4..7f1591213cec6 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -109,6 +109,20 @@ static inline bool ListTypeSupported(const DataType& type) {
   }
   return false;
 }
+// ----------------------------------------------------------------------
+// PyCapsule code for setting ndarray base to reference C++ object
+
+struct ArrowCapsule {
+  std::shared_ptr<Array> array;
+};
+
+namespace {
+
+void ArrowCapsule_Destructor(PyObject* capsule) {
+  delete reinterpret_cast<ArrowCapsule*>(PyCapsule_GetPointer(capsule, "arrow"));
+}
+
+}  // namespace
 
 // ----------------------------------------------------------------------
 // pandas 0.x DataFrame conversion internals
@@ -957,23 +971,40 @@ class CategoricalBlock : public PandasBlock {
     using TRAITS = internal::arrow_traits<ARROW_INDEX_TYPE>;
     using T = typename TRAITS::T;
     constexpr int npy_type = TRAITS::npy_type;
-    RETURN_NOT_OK(AllocateNDArray(npy_type, 1));
-
-    // No relative placement offset because a single column
-    T* out_values = reinterpret_cast<T*>(block_data_);
 
     const ChunkedArray& data = *col->data().get();
 
-    for (int c = 0; c < data.num_chunks(); c++) {
-      const std::shared_ptr<Array> arr = data.chunk(c);
-      const auto& dict_arr = static_cast<const DictionaryArray&>(*arr);
+    // Sniff the first chunk
+    const std::shared_ptr<Array> arr_first = data.chunk(0);
+    const auto& dict_arr_first = static_cast<const DictionaryArray&>(*arr_first);
+    const auto& indices_first =
+        static_cast<const PrimitiveArray&>(*dict_arr_first.indices());
+
+    if (data.num_chunks() == 1 && indices_first.null_count() == 0) {
+      RETURN_NOT_OK(AllocateNDArrayFromIndices<T>(npy_type, indices_first));
+    } else {
+      if (options_.zero_copy_only) {
+        std::stringstream ss;
+        ss << "Needed to copy " << data.num_chunks() << " chunks with "
+           << indices_first.null_count() << " indices nulls, but zero_copy_only was True";
+        return Status::Invalid(ss.str());
+      }
+      RETURN_NOT_OK(AllocateNDArray(npy_type, 1));
+
+      // No relative placement offset because a single column
+      T* out_values = reinterpret_cast<T*>(block_data_);
 
-      const auto& indices = static_cast<const PrimitiveArray&>(*dict_arr.indices());
-      auto in_values = reinterpret_cast<const T*>(indices.raw_values());
+      for (int c = 0; c < data.num_chunks(); c++) {
+        const std::shared_ptr<Array> arr = data.chunk(c);
+        const auto& dict_arr = static_cast<const DictionaryArray&>(*arr);
 
-      // Null is -1 in CategoricalBlock
-      for (int i = 0; i < arr->length(); ++i) {
-        *out_values++ = indices.IsNull(i) ? -1 : in_values[i];
+        const auto& indices = static_cast<const PrimitiveArray&>(*dict_arr.indices());
+        auto in_values = reinterpret_cast<const T*>(indices.raw_values());
+
+        // Null is -1 in CategoricalBlock
+        for (int i = 0; i < arr->length(); ++i) {
+          *out_values++ = indices.IsNull(i) ? -1 : in_values[i];
+        }
       }
     }
 
@@ -1043,6 +1074,43 @@ class CategoricalBlock : public PandasBlock {
   PyObject* dictionary() const { return dictionary_.obj(); }
 
  protected:
+  template <typename T>
+  Status AllocateNDArrayFromIndices(int npy_type, const PrimitiveArray& indices) {
+    npy_intp block_dims[1] = {num_rows_};
+
+    auto in_values = reinterpret_cast<const T*>(indices.raw_values());
+    void* data = const_cast<T*>(in_values);
+
+    PyAcquireGIL lock;
+
+    PyArray_Descr* descr = GetSafeNumPyDtype(npy_type);
+    if (descr == nullptr) {
+      // Error occurred, trust error state is set
+      return Status::OK();
+    }
+
+    PyObject* block_arr = PyArray_NewFromDescr(&PyArray_Type, descr, 1, block_dims,
+                                               nullptr, data, NPY_ARRAY_CARRAY, nullptr);
+
+    npy_intp placement_dims[1] = {num_columns_};
+    PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
+    if (placement_arr == NULL) {
+      // TODO(wesm): propagating Python exception
+      return Status::OK();
+    }
+
+    block_arr_.reset(block_arr);
+    placement_arr_.reset(placement_arr);
+
+    block_data_ = reinterpret_cast<uint8_t*>(
+        PyArray_DATA(reinterpret_cast<PyArrayObject*>(block_arr)));
+
+    placement_data_ = reinterpret_cast<int64_t*>(
+        PyArray_DATA(reinterpret_cast<PyArrayObject*>(placement_arr)));
+
+    return Status::OK();
+  }
+
   MemoryPool* pool_;
   OwnedRef dictionary_;
   bool ordered_;
@@ -1369,12 +1437,26 @@ class ArrowDeserializer {
       return Status::OK();
     }
 
-    if (PyArray_SetBaseObject(arr_, py_ref_) == -1) {
+    PyObject* base;
+    if (py_ref_ == nullptr) {
+      ArrowCapsule* capsule = new ArrowCapsule;
+      capsule->array = arr;
+      base = PyCapsule_New(reinterpret_cast<void*>(capsule), "arrow",
+                           &ArrowCapsule_Destructor);
+      if (base == nullptr) {
+        delete capsule;
+        RETURN_IF_PYERROR();
+      }
+    } else {
+      base = py_ref_;
+    }
+
+    if (PyArray_SetBaseObject(arr_, base) == -1) {
       // Error occurred, trust that SetBaseObject set the error state
       return Status::OK();
     } else {
-      // PyArray_SetBaseObject steals our reference to py_ref_
-      Py_INCREF(py_ref_);
+      // PyArray_SetBaseObject steals our reference to base
+      Py_INCREF(base);
     }
 
     // Arrow data is immutable.
@@ -1399,7 +1481,7 @@ class ArrowDeserializer {
     typedef typename traits::T T;
     int npy_type = traits::npy_type;
 
-    if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
+    if (data_.num_chunks() == 1 && data_.null_count() == 0) {
       return ConvertValuesZeroCopy<TYPE>(options_, npy_type, data_.chunk(0));
     } else if (options_.zero_copy_only) {
       std::stringstream ss;
@@ -1462,7 +1544,7 @@ class ArrowDeserializer {
 
     typedef typename traits::T T;
 
-    if (data_.num_chunks() == 1 && data_.null_count() == 0 && py_ref_ != nullptr) {
+    if (data_.num_chunks() == 1 && data_.null_count() == 0) {
       return ConvertValuesZeroCopy<TYPE>(options_, traits::npy_type, data_.chunk(0));
     } else if (options_.zero_copy_only) {
       std::stringstream ss;
@@ -1566,10 +1648,6 @@ class ArrowDeserializer {
   }
 
   Status Visit(const DictionaryType& type) {
-    if (options_.zero_copy_only) {
-      return Status::Invalid("DictionaryType needs copies, but zero_copy_only was True");
-    }
-
     auto block = std::make_shared<CategoricalBlock>(options_, nullptr, col_->length());
     RETURN_NOT_OK(block->Write(col_, 0, 0));
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 8360dae54d613..d00bf1b28eddc 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -217,6 +217,17 @@ def test_zero_copy_success(self):
         result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True)
         npt.assert_array_equal(result, [0, 1, 2])
 
+    def test_zero_copy_dictionaries(self):
+        arr = pa.DictionaryArray.from_arrays(
+            np.array([0, 0]),
+            np.array([5]))
+
+        result = arr.to_pandas(zero_copy_only=True)
+        values = pd.Categorical([5, 5])
+
+        tm.assert_series_equal(pd.Series(result), pd.Series(values),
+                               check_names=False)
+
     def test_zero_copy_failure_on_object_types(self):
         with pytest.raises(pa.ArrowException):
             pa.array(['A', 'B', 'C']).to_pandas(zero_copy_only=True)
@@ -245,14 +256,6 @@ def test_zero_copy_failure_on_timestamp_types(self):
         with pytest.raises(pa.ArrowException):
             pa.array(arr).to_pandas(zero_copy_only=True)
 
-    def test_zero_copy_dictionaries(self):
-        arr = pa.DictionaryArray.from_arrays(
-            np.array([0, 0]),
-            np.array(['A']))
-
-        with pytest.raises(pa.ArrowException):
-            arr.to_pandas(zero_copy_only=True)
-
     def test_float_nulls(self):
         num_values = 100
 

From b221a2c7a08f4f709c3a4eb4b2d68dae16c82da8 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 29 Oct 2017 20:32:58 -0400
Subject: [PATCH 1198/1644] ARROW-1751: [Python] Pandas 0.21.0 introduces a
 breaking API change for MultiIndex construction

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1268 from cpcloud/ARROW-1751 and squashes the following commits:

e38eb665 [Phillip Cloud] ARROW-1751: [Python] Pandas 0.21.0 introduces a breaking API change for MultiIndex construction
---
 python/pyarrow/pandas_compat.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 5592d8dd93f8a..d6c844c8490f5 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -493,6 +493,16 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
             labels=labels,
             names=columns.names
         )
+
+    # flatten a single level column MultiIndex for pandas 0.21.0 :(
+    if isinstance(columns, pd.MultiIndex) and columns.nlevels == 1:
+        levels, = columns.levels
+        labels, = columns.labels
+
+        # Cheaply check that we do not somehow have duplicate column names
+        assert len(levels) == len(labels), 'Found non-unique column index'
+        columns = levels[labels]
+
     axes = [columns, index]
     return _int.BlockManager(blocks, axes)
 

From f257b0016b71b804d79e2359983bc2067c9788a4 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sun, 29 Oct 2017 20:35:19 -0400
Subject: [PATCH 1199/1644] ARROW-1746: [Python] Add build dependencies for
 Arch Linux

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1262 from xhochy/ARROW-1746 and squashes the following commits:

d5381cc2 [Korn, Uwe] ARROW-1746: [Python] Add build dependencies for Arch Linux
---
 python/doc/source/development.rst | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 093c6c038694a..3ca460d463a06 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -111,7 +111,7 @@ building Arrow C++:
    brew update && brew bundle --file=python/Brewfile
 
 On Debian/Ubuntu, you need the following minimal set of dependencies. All other
-dependencies will be automatically built by Arrow' thrid-party toolchain.
+dependencies will be automatically built by Arrow's third-party toolchain.
 
 .. code-block:: shell
 
@@ -119,6 +119,12 @@ dependencies will be automatically built by Arrow' thrid-party toolchain.
                           libboost-filesystem-dev \
                           libboost-system-dev
 
+On Arch Linux, you can get these dependencies via pacman.
+
+.. code-block:: shell
+
+   $ sudo pacman -S jemalloc boost
+
 Now, let's create a Python virtualenv with all Python dependencies in the same
 folder as the repositories and a target installation folder:
 

From ec22228a72915c03691be7808829676be60901e1 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sun, 29 Oct 2017 20:37:14 -0400
Subject: [PATCH 1200/1644] ARROW-1747: [C++] Don't export symbols of
 statically linked libraries

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1264 from xhochy/ARROW-1747 and squashes the following commits:

85d17255 [Korn, Uwe] ARROW-1747: [C++] Don't export symbols of statically linked libraries
---
 cpp/src/arrow/symbols.map                     | 38 +++++++++++++++++++
 python/manylinux1/Dockerfile-x86_64           |  3 ++
 .../scripts/check_arrow_visibility.sh         | 26 +++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100755 python/manylinux1/scripts/check_arrow_visibility.sh

diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map
index 49511c6a74934..f216d865001d2 100644
--- a/cpp/src/arrow/symbols.map
+++ b/cpp/src/arrow/symbols.map
@@ -22,11 +22,49 @@
     # devtoolset / static-libstdc++ symbols
     __cxa_*;
 
+
+    # Static libraries that are linked in e.g. the manylinux1 build
+    # Brotli compression library
+    Brotli*;
+    # zlib
+    adler32;
+    adler32_combine;
+    adler32_combine64;
+    crc32;
+    crc32_combine;
+    crc32_combine64;
+    deflate*;
+    inflate*;
+    get_crc_table;
+    zcalloc;
+    zcfree;
+    zError;
+    zlibCompileFlags;
+    zlibVersion;
+    _tr_*;
+    # lz4
+    LZ4_*;
+    # zstandard
+    ZSTD_*;
+    ZSTDv*;
+    HUF_*;
+    HUFv*;
+    FSE_*;
+    FSEv*;
+    ZBUFFv*;
+    ERR_getErrorString;
+    # jemalloc
+    je_arrow_*;
+
     extern "C++" {
       # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
       # links c++11 symbols into binaries so that the result may be executed on
       # a system with an older libstdc++ which doesn't include the necessary
       # c++11 symbols.
       std::*;
+
+      # Statically linked C++ dependencies
+      boost::*;
+      snappy::*;
     };
 };
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 6d72ec7538c0f..69d8d3dbf8cef 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -23,6 +23,9 @@ WORKDIR /arrow/cpp/build-plain
 RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF ..
 RUN ninja install
 
+ADD scripts/check_arrow_visibility.sh /
+RUN /check_arrow_visibility.sh
+
 WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
diff --git a/python/manylinux1/scripts/check_arrow_visibility.sh b/python/manylinux1/scripts/check_arrow_visibility.sh
new file mode 100755
index 0000000000000..27a30f7479bb6
--- /dev/null
+++ b/python/manylinux1/scripts/check_arrow_visibility.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -ex
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+nm -D -C /arrow-dist/lib64/libarrow.so > nm_arrow.log
+
+if [[ `grep ' T ' nm_arrow.log | grep -v arrow | wc -l` -eq 2 ]]
+then
+    exit 0
+fi
+
+exit 1

From 1d36dd2543e7dc5d1084d8f9552e851438b1c80a Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 29 Oct 2017 22:18:48 -0400
Subject: [PATCH 1201/1644] ARROW-1748: [GLib] Add GArrowRecordBatchBuilder

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1265 from kou/glib-add-record-batch-builder and squashes the following commits:

f1387278 [Kouhei Sutou] [GLib] Add GArrowRecordBatchBuilder
---
 c_glib/arrow-glib/Makefile.am             |   3 +
 c_glib/arrow-glib/array-builder.cpp       |  28 +-
 c_glib/arrow-glib/array-builder.h         |   2 +
 c_glib/arrow-glib/arrow-glib.h            |   1 +
 c_glib/arrow-glib/arrow-glib.hpp          |   1 +
 c_glib/arrow-glib/meson.build             |   3 +
 c_glib/arrow-glib/table-builder.cpp       | 303 ++++++++++++++++++++++
 c_glib/arrow-glib/table-builder.h         |  56 ++++
 c_glib/arrow-glib/table-builder.hpp       |  27 ++
 c_glib/doc/reference/arrow-glib-docs.sgml |   4 +
 c_glib/test/test-record-batch-builder.rb  |  84 ++++++
 11 files changed, 510 insertions(+), 2 deletions(-)
 create mode 100644 c_glib/arrow-glib/table-builder.cpp
 create mode 100644 c_glib/arrow-glib/table-builder.h
 create mode 100644 c_glib/arrow-glib/table-builder.hpp
 create mode 100644 c_glib/test/test-record-batch-builder.rb

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 2066fa77cef84..bf68ec4910e77 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -57,6 +57,7 @@ libarrow_glib_la_headers =			\
 	record-batch.h				\
 	schema.h				\
 	table.h					\
+	table-builder.h				\
 	tensor.h				\
 	type.h
 
@@ -97,6 +98,7 @@ libarrow_glib_la_sources =			\
 	record-batch.cpp			\
 	schema.cpp				\
 	table.cpp				\
+	table-builder.cpp			\
 	tensor.cpp				\
 	type.cpp				\
 	$(libarrow_glib_la_headers)		\
@@ -133,6 +135,7 @@ libarrow_glib_la_cpp_headers =			\
 	record-batch.hpp			\
 	schema.hpp				\
 	table.hpp				\
+	table-builder.hpp			\
 	tensor.hpp				\
 	type.hpp
 
diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index bea7e8342c797..86e7f985be4a4 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -202,6 +202,7 @@ G_BEGIN_DECLS
 
 typedef struct GArrowArrayBuilderPrivate_ {
   arrow::ArrayBuilder *array_builder;
+  gboolean have_ownership;
 } GArrowArrayBuilderPrivate;
 
 enum {
@@ -225,7 +226,9 @@ garrow_array_builder_finalize(GObject *object)
 
   priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(object);
 
-  delete priv->array_builder;
+  if (priv->have_ownership) {
+    delete priv->array_builder;
+  }
 
   G_OBJECT_CLASS(garrow_array_builder_parent_class)->finalize(object);
 }
@@ -267,6 +270,10 @@ garrow_array_builder_get_property(GObject *object,
 static void
 garrow_array_builder_init(GArrowArrayBuilder *builder)
 {
+  GArrowArrayBuilderPrivate *priv;
+
+  priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(builder);
+  priv->have_ownership = TRUE;
 }
 
 static void
@@ -301,7 +308,24 @@ garrow_array_builder_new(const std::shared_ptr<arrow::DataType> &type,
     return NULL;
   }
   return garrow_array_builder_new_raw(arrow_builder.release());
-};
+}
+
+/**
+ * garrow_array_builder_release_ownership: (skip)
+ * @builder: A #GArrowArrayBuilder.
+ *
+ * Release ownership of `arrow::ArrayBuilder` in `builder`.
+ *
+ * Since: 0.8.8
+ */
+void
+garrow_array_builder_release_ownership(GArrowArrayBuilder *builder)
+{
+  GArrowArrayBuilderPrivate *priv;
+
+  priv = GARROW_ARRAY_BUILDER_GET_PRIVATE(builder);
+  priv->have_ownership = FALSE;
+}
 
 /**
  * garrow_array_builder_finish:
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index d9e91212e1663..19dadb30999bd 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -35,6 +35,8 @@ struct _GArrowArrayBuilderClass
   GObjectClass parent_class;
 };
 
+void garrow_array_builder_release_ownership(GArrowArrayBuilder *builder);
+
 GArrowArray        *garrow_array_builder_finish   (GArrowArrayBuilder *builder,
                                                    GError **error);
 
diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h
index 47f324dc04640..fb1b37e2c688a 100644
--- a/c_glib/arrow-glib/arrow-glib.h
+++ b/c_glib/arrow-glib/arrow-glib.h
@@ -33,6 +33,7 @@
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
 #include <arrow-glib/table.h>
+#include <arrow-glib/table-builder.h>
 #include <arrow-glib/tensor.h>
 #include <arrow-glib/type.h>
 
diff --git a/c_glib/arrow-glib/arrow-glib.hpp b/c_glib/arrow-glib/arrow-glib.hpp
index 7fc6c4828d8c5..0c411346ad430 100644
--- a/c_glib/arrow-glib/arrow-glib.hpp
+++ b/c_glib/arrow-glib/arrow-glib.hpp
@@ -32,6 +32,7 @@
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 #include <arrow-glib/table.hpp>
+#include <arrow-glib/table-builder.hpp>
 #include <arrow-glib/tensor.hpp>
 #include <arrow-glib/type.hpp>
 
diff --git a/c_glib/arrow-glib/meson.build b/c_glib/arrow-glib/meson.build
index 226c696bdcf37..464a002e78b0c 100644
--- a/c_glib/arrow-glib/meson.build
+++ b/c_glib/arrow-glib/meson.build
@@ -30,6 +30,7 @@ sources = files(
   'record-batch.cpp',
   'schema.cpp',
   'table.cpp',
+  'table-builder.cpp',
   'tensor.cpp',
   'type.cpp',
 )
@@ -70,6 +71,7 @@ c_headers = files(
   'record-batch.h',
   'schema.h',
   'table.h',
+  'table-builder.h',
   'tensor.h',
   'type.h',
 )
@@ -110,6 +112,7 @@ cpp_headers = files(
   'record-batch.hpp',
   'schema.hpp',
   'table.hpp',
+  'table-builder.hpp',
   'tensor.hpp',
   'type.hpp',
 )
diff --git a/c_glib/arrow-glib/table-builder.cpp b/c_glib/arrow-glib/table-builder.cpp
new file mode 100644
index 0000000000000..e87314bf52b9f
--- /dev/null
+++ b/c_glib/arrow-glib/table-builder.cpp
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array-builder.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/record-batch.hpp>
+#include <arrow-glib/schema.hpp>
+#include <arrow-glib/table-builder.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: table-builder
+ * @section_id: table-builder-classes
+ * @title: Table builder classes
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowRecordBatchBuilder is a class to create
+ * new #GArrowRecordBatch.
+ */
+
+typedef struct GArrowRecordBatchBuilderPrivate_ {
+  arrow::RecordBatchBuilder *record_batch_builder;
+  GPtrArray *fields;
+} GArrowRecordBatchBuilderPrivate;
+
+enum {
+  PROP_0,
+  PROP_RECORD_BATCH_BUILDER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowRecordBatchBuilder,
+                           garrow_record_batch_builder,
+                           G_TYPE_OBJECT)
+
+#define GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(object)            \
+  static_cast<GArrowRecordBatchBuilderPrivate *>(                  \
+    garrow_record_batch_builder_get_instance_private(              \
+      GARROW_RECORD_BATCH_BUILDER(object)))
+
+static void
+garrow_record_batch_builder_constructed(GObject *object)
+{
+  auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(object);
+  auto arrow_builder = priv->record_batch_builder;
+  auto n_fields = arrow_builder->num_fields();
+  priv->fields = g_ptr_array_new_full(n_fields, g_object_unref);
+  for (int i = 0; i < n_fields; ++i) {
+    auto arrow_array_builder = arrow_builder->GetField(i);
+    auto array_builder = garrow_array_builder_new_raw(arrow_array_builder);
+    garrow_array_builder_release_ownership(array_builder);
+    g_ptr_array_add(priv->fields, array_builder);
+  }
+
+  G_OBJECT_CLASS(garrow_record_batch_builder_parent_class)->constructed(object);
+}
+
+static void
+garrow_record_batch_builder_finalize(GObject *object)
+{
+  auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(object);
+
+  g_ptr_array_free(priv->fields, TRUE);
+  delete priv->record_batch_builder;
+
+  G_OBJECT_CLASS(garrow_record_batch_builder_parent_class)->finalize(object);
+}
+
+static void
+garrow_record_batch_builder_set_property(GObject *object,
+                                         guint prop_id,
+                                         const GValue *value,
+                                         GParamSpec *pspec)
+{
+  auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_RECORD_BATCH_BUILDER:
+    priv->record_batch_builder =
+      static_cast<arrow::RecordBatchBuilder *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_record_batch_builder_get_property(GObject *object,
+                                         guint prop_id,
+                                         GValue *value,
+                                         GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_record_batch_builder_init(GArrowRecordBatchBuilder *builder)
+{
+}
+
+static void
+garrow_record_batch_builder_class_init(GArrowRecordBatchBuilderClass *klass)
+{
+  auto gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->constructed  = garrow_record_batch_builder_constructed;
+  gobject_class->finalize     = garrow_record_batch_builder_finalize;
+  gobject_class->set_property = garrow_record_batch_builder_set_property;
+  gobject_class->get_property = garrow_record_batch_builder_get_property;
+
+  GParamSpec *spec;
+  spec = g_param_spec_pointer("record-batch-builder",
+                              "RecordBatch builder",
+                              "The raw arrow::RecordBatchBuilder *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class,
+                                  PROP_RECORD_BATCH_BUILDER,
+                                  spec);
+}
+
+/**
+ * garrow_record_batch_builder_new:
+ * @schema: A #GArrowSchema.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: A newly created #GArrowRecordBatchBuilder on success,
+ *   %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowRecordBatchBuilder *
+garrow_record_batch_builder_new(GArrowSchema *schema, GError **error)
+{
+  auto arrow_schema = garrow_schema_get_raw(schema);
+  auto memory_pool = arrow::default_memory_pool();
+  std::unique_ptr<arrow::RecordBatchBuilder> arrow_builder;
+  auto status = arrow::RecordBatchBuilder::Make(arrow_schema,
+                                                memory_pool,
+                                                &arrow_builder);
+  if (garrow_error_check(error, status, "[record-batch-builder][new]")) {
+    return garrow_record_batch_builder_new_raw(arrow_builder.release());
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_record_batch_builder_get_initial_capacity:
+ * @builder: A #GArrowRecordBatchBuilder.
+ *
+ * Returns: The initial capacity for array builders.
+ *
+ * Since: 0.8.0
+ */
+gint64
+garrow_record_batch_builder_get_initial_capacity(GArrowRecordBatchBuilder *builder)
+{
+  auto arrow_builder = garrow_record_batch_builder_get_raw(builder);
+  return arrow_builder->initial_capacity();
+}
+
+/**
+ * garrow_record_batch_builder_set_initial_capacity:
+ * @builder: A #GArrowRecordBatchBuilder.
+ * @capacity: The new initial capacity for array builders.
+ *
+ * Since: 0.8.0
+ */
+void
+garrow_record_batch_builder_set_initial_capacity(GArrowRecordBatchBuilder *builder,
+                                                 gint64 capacity)
+{
+  auto arrow_builder = garrow_record_batch_builder_get_raw(builder);
+  arrow_builder->SetInitialCapacity(capacity);
+}
+
+/**
+ * garrow_record_batch_builder_get_schema:
+ * @builder: A #GArrowRecordBatchBuilder.
+ *
+ * Returns: (transfer full): The #GArrowSchema of the record batch builder.
+ *
+ * Since: 0.8.0
+ */
+GArrowSchema *
+garrow_record_batch_builder_get_schema(GArrowRecordBatchBuilder *builder)
+{
+  auto arrow_builder = garrow_record_batch_builder_get_raw(builder);
+  auto arrow_schema = arrow_builder->schema();
+  return garrow_schema_new_raw(&arrow_schema);
+}
+
+/**
+ * garrow_record_batch_builder_get_n_fields:
+ * @builder: A #GArrowRecordBatchBuilder.
+ *
+ * Returns: The number of fields.
+ *
+ * Since: 0.8.0
+ */
+gint
+garrow_record_batch_builder_get_n_fields(GArrowRecordBatchBuilder *builder)
+{
+  auto arrow_builder = garrow_record_batch_builder_get_raw(builder);
+  return arrow_builder->num_fields();
+}
+
+/**
+ * garrow_record_batch_builder_get_field:
+ * @builder: A #GArrowRecordBatchBuilder.
+ * @i: The field index. If it's negative, index is counted backward
+ *   from the end of the fields. `-1` means the last field.
+ *
+ * Returns: (transfer none) (nullable): The #GArrowArrayBuilder for
+ *   the `i`-th field on success, %NULL on out of index.
+ *
+ * Since: 0.8.0
+ */
+GArrowArrayBuilder *
+garrow_record_batch_builder_get_field(GArrowRecordBatchBuilder *builder,
+                                      gint i)
+{
+  auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(builder);
+  if (i < 0) {
+    i += priv->fields->len;
+  }
+  if (i < 0) {
+    return NULL;
+  }
+  if (static_cast<guint>(i) >= priv->fields->len) {
+    return NULL;
+  }
+
+  return GARROW_ARRAY_BUILDER(g_ptr_array_index(priv->fields, i));
+}
+
+/**
+ * garrow_record_batch_builder_flush:
+ * @builder: A #GArrowRecordBatchBuilder.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): The built #GArrowRecordBatch on success,
+ *   %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowRecordBatch *
+garrow_record_batch_builder_flush(GArrowRecordBatchBuilder *builder,
+                                  GError **error)
+{
+  auto arrow_builder = garrow_record_batch_builder_get_raw(builder);
+  std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
+  auto status = arrow_builder->Flush(&arrow_record_batch);
+  if (garrow_error_check(error, status, "[record-batch-builder][flush]")) {
+    return garrow_record_batch_new_raw(&arrow_record_batch);
+  } else {
+    return NULL;
+  }
+}
+
+G_END_DECLS
+
+GArrowRecordBatchBuilder *
+garrow_record_batch_builder_new_raw(arrow::RecordBatchBuilder *arrow_builder)
+{
+  auto builder = g_object_new(GARROW_TYPE_RECORD_BATCH_BUILDER,
+                              "record-batch-builder", arrow_builder,
+                              NULL);
+  return GARROW_RECORD_BATCH_BUILDER(builder);
+}
+
+arrow::RecordBatchBuilder *
+garrow_record_batch_builder_get_raw(GArrowRecordBatchBuilder *builder)
+{
+  auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(builder);
+  return priv->record_batch_builder;
+}
diff --git a/c_glib/arrow-glib/table-builder.h b/c_glib/arrow-glib/table-builder.h
new file mode 100644
index 0000000000000..d05525e54f52e
--- /dev/null
+++ b/c_glib/arrow-glib/table-builder.h
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/array-builder.h>
+#include <arrow-glib/gobject-type.h>
+#include <arrow-glib/record-batch.h>
+#include <arrow-glib/schema.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_RECORD_BATCH_BUILDER (garrow_record_batch_builder_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowRecordBatchBuilder,
+                         garrow_record_batch_builder,
+                         GARROW,
+                         RECORD_BATCH_BUILDER,
+                         GObject)
+struct _GArrowRecordBatchBuilderClass
+{
+  GObjectClass parent_class;
+};
+
+GArrowRecordBatchBuilder *garrow_record_batch_builder_new(GArrowSchema *schema,
+                                                          GError **error);
+
+gint64 garrow_record_batch_builder_get_initial_capacity(GArrowRecordBatchBuilder *builder);
+void garrow_record_batch_builder_set_initial_capacity(GArrowRecordBatchBuilder *builder,
+                                                      gint64 capacity);
+GArrowSchema *garrow_record_batch_builder_get_schema(GArrowRecordBatchBuilder *builder);
+
+gint garrow_record_batch_builder_get_n_fields(GArrowRecordBatchBuilder *builder);
+GArrowArrayBuilder *garrow_record_batch_builder_get_field(GArrowRecordBatchBuilder *builder,
+                                                          gint i);
+
+GArrowRecordBatch *garrow_record_batch_builder_flush(GArrowRecordBatchBuilder *builder,
+                                                     GError **error);
+
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/table-builder.hpp b/c_glib/arrow-glib/table-builder.hpp
new file mode 100644
index 0000000000000..cf93ded9b4b65
--- /dev/null
+++ b/c_glib/arrow-glib/table-builder.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/table-builder.h>
+
+GArrowRecordBatchBuilder *garrow_record_batch_builder_new_raw(arrow::RecordBatchBuilder *arrow_builder);
+arrow::RecordBatchBuilder *garrow_record_batch_builder_get_raw(GArrowRecordBatchBuilder *builder);
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index fc161a5864c24..a504ef1148383 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -72,6 +72,10 @@
       <xi:include href="xml/column.xml"/>
       <xi:include href="xml/chunked-array.xml"/>
     </chapter>
+    <chapter id="table-builder">
+      <title>Table builder</title>
+      <xi:include href="xml/table-builder.xml"/>
+    </chapter>
     <chapter id="buffer">
       <title>Buffer</title>
       <xi:include href="xml/buffer.xml"/>
diff --git a/c_glib/test/test-record-batch-builder.rb b/c_glib/test/test-record-batch-builder.rb
new file mode 100644
index 0000000000000..1bb72820a5860
--- /dev/null
+++ b/c_glib/test/test-record-batch-builder.rb
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestRecordBatchBuilder < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def setup
+    @fields = [
+      Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+      Arrow::Field.new("point", Arrow::Int32DataType.new),
+    ]
+    @schema = Arrow::Schema.new(@fields)
+    @builder = Arrow::RecordBatchBuilder.new(@schema)
+  end
+
+  def test_initial_capacity
+    @builder.initial_capacity = 128
+    assert_equal(128, @builder.initial_capacity)
+  end
+
+  def test_schema
+    assert_equal(@schema, @builder.schema)
+  end
+
+  def test_n_fields
+    assert_equal(@fields.size, @builder.n_fields)
+  end
+
+  sub_test_case("#get_field") do
+    def test_valid
+      assert_equal(Arrow::BooleanArrayBuilder,
+                   @builder.get_field(0).class)
+    end
+
+    def test_negative
+      assert_equal(Arrow::Int32ArrayBuilder,
+                   @builder.get_field(-1).class)
+    end
+
+    def test_too_negative
+      assert_nil(@builder.get_field(-@fields.size - 1))
+    end
+
+    def test_too_large
+      assert_nil(@builder.get_field(@fields.size))
+    end
+  end
+
+  def test_flush
+    arrays = {
+      "visible" => build_boolean_array([true, false, true]),
+      "point"   => build_int32_array([1, -1, 0]),
+    }
+    arrays.each_with_index do |(_, array), i|
+      @builder.get_field(i).append_values(array.values, [])
+    end
+    assert_equal(build_record_batch(arrays),
+                 @builder.flush)
+
+    arrays = {
+      "visible" => build_boolean_array([false, true]),
+      "point"   => build_int32_array([10, -10]),
+    }
+    arrays.each_with_index do |(_, array), i|
+      @builder.get_field(i).append_values(array.values, [])
+    end
+    assert_equal(build_record_batch(arrays),
+                 @builder.flush)
+  end
+end

From 30158ad5a068208237f3c30e6c9eb60454bae402 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 30 Oct 2017 08:50:28 +0100
Subject: [PATCH 1202/1644] ARROW-1718: [C++/Python] Implement casts from
 timestamp to date32/64, properly handle NumPy datetime64[D] -> date32

This was sort of a can of worms. cc @xhochy @cpcloud @BryanCutler

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1258 from wesm/ARROW-1718 and squashes the following commits:

c693889 [Wes McKinney] Use explicit static_cast to convert int64->int32
586face [Wes McKinney] Use correct syntax for int64 literals in MSVC. remove incorrect comments
d61b7a4 [Wes McKinney] Downcast datetime64[D] as int64_t to int32_t
e6f8b62 [Wes McKinney] Build with clang in Travis CI
b187a09 [Wes McKinney] Remove now unneeded template specialization for Date32Type, but another test fails
284b9ba [Wes McKinney] Complete unit tests for timestamp->date32/64
6ca361f [Wes McKinney] tweak test case
68f3a32 [Wes McKinney] Test case, c++ unittest placeholder
383f730 [Wes McKinney] Implement builtin converter for date32, test datetime.date, ints, overflows
d52bd77 [Wes McKinney] One failing test case
---
 .travis.yml                                  |   4 +-
 cpp/src/arrow/compute/cast.cc                |  99 +++++++++---
 cpp/src/arrow/compute/compute-test.cc        |  64 ++++++++
 cpp/src/arrow/python/builtin_convert.cc      |  27 +++-
 cpp/src/arrow/python/numpy_to_arrow.cc       | 153 +++++++++----------
 cpp/src/arrow/python/util/datetime.h         |   5 +
 python/pyarrow/tests/test_convert_builtin.py |  19 +++
 python/pyarrow/tests/test_convert_pandas.py  |  11 ++
 8 files changed, 278 insertions(+), 104 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 039ae95208b74..6419548a622f3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -51,12 +51,12 @@ matrix:
     os: linux
     group: deprecated
     before_script:
-    - export CC="gcc-4.9"
-    - export CXX="g++-4.9"
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_VALGRIND=1
     - export ARROW_TRAVIS_PLASMA=1
     - export ARROW_TRAVIS_CLANG_FORMAT=1
+    - export CC="clang-4.0"
+    - export CXX="clang++-4.0"
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index 68a2b12379e34..114ab9af0d0d0 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -69,11 +69,18 @@
 namespace arrow {
 namespace compute {
 
+constexpr int64_t kMillisecondsInDay = 86400000;
+
 template <typename T>
-inline const T* GetValuesAs(const ArrayData& data, int i) {
+inline const T* GetValues(const ArrayData& data, int i) {
   return reinterpret_cast<const T*>(data.buffers[i]->data()) + data.offset;
 }
 
+template <typename T>
+inline T* GetMutableValues(const ArrayData* data, int i) {
+  return reinterpret_cast<T*>(data->buffers[i]->mutable_data()) + data->offset;
+}
+
 namespace {
 
 void CopyData(const Array& input, ArrayData* output) {
@@ -164,7 +171,7 @@ struct CastFunctor<T, BooleanType,
     auto in_data = input.data();
     internal::BitmapReader bit_reader(in_data->buffers[1]->data(), in_data->offset,
                                       in_data->length);
-    auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
+    auto out = GetMutableValues<c_type>(output, 1);
     for (int64_t i = 0; i < input.length(); ++i) {
       *out++ = bit_reader.IsSet() ? kOne : kZero;
       bit_reader.Next();
@@ -214,8 +221,8 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::v
     using in_type = typename I::c_type;
     DCHECK_EQ(output->offset, 0);
 
-    const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
-    uint8_t* out_data = reinterpret_cast<uint8_t*>(output->buffers[1]->mutable_data());
+    const in_type* in_data = GetValues<in_type>(*input.data(), 1);
+    uint8_t* out_data = GetMutableValues<uint8_t>(output, 1);
     for (int64_t i = 0; i < input.length(); ++i) {
       BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
     }
@@ -233,8 +240,8 @@ struct CastFunctor<O, I,
 
     auto in_offset = input.offset();
 
-    const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
-    auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
+    const in_type* in_data = GetValues<in_type>(*input.data(), 1);
+    auto out_data = GetMutableValues<out_type>(output, 1);
 
     if (!options.allow_int_overflow) {
       constexpr in_type kMax = static_cast<in_type>(std::numeric_limits<out_type>::max());
@@ -276,8 +283,8 @@ struct CastFunctor<O, I,
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
 
-    const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
-    auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
+    const in_type* in_data = GetValues<in_type>(*input.data(), 1);
+    auto out_data = GetMutableValues<out_type>(output, 1);
     for (int64_t i = 0; i < input.length(); ++i) {
       *out_data++ = static_cast<out_type>(*in_data++);
     }
@@ -288,13 +295,16 @@ struct CastFunctor<O, I,
 // From one timestamp to another
 
 template <typename in_type, typename out_type>
-inline void ShiftTime(FunctionContext* ctx, const CastOptions& options,
-                      const bool is_multiply, const int64_t factor, const Array& input,
-                      ArrayData* output) {
-  const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
-  auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
+void ShiftTime(FunctionContext* ctx, const CastOptions& options, const bool is_multiply,
+               const int64_t factor, const Array& input, ArrayData* output) {
+  const in_type* in_data = GetValues<in_type>(*input.data(), 1);
+  auto out_data = GetMutableValues<out_type>(output, 1);
 
-  if (is_multiply) {
+  if (factor == 1) {
+    for (int64_t i = 0; i < input.length(); i++) {
+      out_data[i] = static_cast<out_type>(in_data[i]);
+    }
+  } else if (is_multiply) {
     for (int64_t i = 0; i < input.length(); i++) {
       out_data[i] = static_cast<out_type>(in_data[i] * factor);
     }
@@ -352,6 +362,52 @@ struct CastFunctor<TimestampType, TimestampType> {
   }
 };
 
+template <>
+struct CastFunctor<Date32Type, TimestampType> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    const auto& in_type = static_cast<const TimestampType&>(*input.type());
+
+    static const int64_t kTimestampToDateFactors[4] = {
+        86400LL,                             // SECOND
+        86400LL * 1000LL,                    // MILLI
+        86400LL * 1000LL * 1000LL,           // MICRO
+        86400LL * 1000LL * 1000LL * 1000LL,  // NANO
+    };
+
+    const int64_t factor = kTimestampToDateFactors[static_cast<int>(in_type.unit())];
+    ShiftTime<int64_t, int32_t>(ctx, options, false, factor, input, output);
+  }
+};
+
+template <>
+struct CastFunctor<Date64Type, TimestampType> {
+  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+                  ArrayData* output) {
+    const auto& in_type = static_cast<const TimestampType&>(*input.type());
+
+    std::pair<bool, int64_t> conversion =
+        kTimeConversionTable[static_cast<int>(in_type.unit())]
+                            [static_cast<int>(TimeUnit::MILLI)];
+
+    ShiftTime<int64_t, int64_t>(ctx, options, conversion.first, conversion.second, input,
+                                output);
+
+    // Ensure that intraday milliseconds have been zeroed out
+    auto out_data = GetMutableValues<int64_t>(output, 1);
+    for (int64_t i = 0; i < input.length(); ++i) {
+      const int64_t remainder = out_data[i] % kMillisecondsInDay;
+      if (ARROW_PREDICT_FALSE(!options.allow_time_truncate && input.IsValid(i) &&
+                              remainder > 0)) {
+        ctx->SetStatus(
+            Status::Invalid("Timestamp value had non-zero intraday milliseconds"));
+        break;
+      }
+      out_data[i] -= remainder;
+    }
+  }
+};
+
 // ----------------------------------------------------------------------
 // From one time32 or time64 to another
 
@@ -385,8 +441,6 @@ struct CastFunctor<O, I,
 // ----------------------------------------------------------------------
 // Between date32 and date64
 
-constexpr int64_t kMillisecondsInDay = 86400000;
-
 template <>
 struct CastFunctor<Date64Type, Date32Type> {
   void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
@@ -415,7 +469,7 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices,
   internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
                                            indices.length());
 
-  const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
+  const index_c_type* in = GetValues<index_c_type>(*indices.data(), 1);
 
   uint8_t* out = output->buffers[1]->mutable_data();
   int32_t byte_width =
@@ -479,7 +533,7 @@ Status UnpackBinaryDictionary(FunctionContext* ctx, const Array& indices,
   internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
                                            indices.length());
 
-  const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
+  const index_c_type* in = GetValues<index_c_type>(*indices.data(), 1);
   for (int64_t i = 0; i < indices.length(); ++i) {
     if (valid_bits_reader.IsSet()) {
       int32_t length;
@@ -550,7 +604,7 @@ void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
   internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
                                            indices.length());
 
-  const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
+  const index_c_type* in = GetValues<index_c_type>(*indices.data(), 1);
   for (int64_t i = 0; i < indices.length(); ++i) {
     if (valid_bits_reader.IsSet()) {
       out[i] = dictionary[in[i]];
@@ -575,7 +629,7 @@ struct CastFunctor<T, DictionaryType,
     DCHECK(values_type.Equals(*output->type))
         << "Dictionary type: " << values_type << " target type: " << (*output->type);
 
-    const c_type* dictionary = GetValuesAs<c_type>(*type.dictionary()->data(), 1);
+    const c_type* dictionary = GetValues<c_type>(*type.dictionary()->data(), 1);
 
     auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
     const Array& indices = *dict_array.indices();
@@ -755,7 +809,10 @@ class CastKernel : public UnaryKernel {
   FN(Time64Type, Time32Type);     \
   FN(Time64Type, Time64Type);
 
-#define TIMESTAMP_CASES(FN, IN_TYPE) FN(TimestampType, TimestampType);
+#define TIMESTAMP_CASES(FN, IN_TYPE) \
+  FN(TimestampType, TimestampType);  \
+  FN(TimestampType, Date32Type);     \
+  FN(TimestampType, Date64Type);
 
 #define DICTIONARY_CASES(FN, IN_TYPE) \
   FN(IN_TYPE, NullType);              \
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 8a7ef923b4719..61d53c4d50b33 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -355,6 +355,70 @@ TEST_F(TestCast, TimestampToTimestamp) {
                             timestamp(TimeUnit::SECOND), options);
 }
 
+TEST_F(TestCast, TimestampToDate32_Date64) {
+  CastOptions options;
+
+  vector<bool> is_valid = {true, true, false};
+
+  // 2000-01-01, 2000-01-02, null
+  vector<int64_t> v_nano = {946684800000000000, 946771200000000000, 0};
+  vector<int64_t> v_micro = {946684800000000, 946771200000000, 0};
+  vector<int64_t> v_milli = {946684800000, 946771200000, 0};
+  vector<int64_t> v_second = {946684800, 946771200, 0};
+  vector<int32_t> v_day = {10957, 10958, 0};
+
+  // Simple conversions
+  CheckCase<TimestampType, int64_t, Date64Type, int64_t>(
+      timestamp(TimeUnit::NANO), v_nano, is_valid, date64(), v_milli, options);
+  CheckCase<TimestampType, int64_t, Date64Type, int64_t>(
+      timestamp(TimeUnit::MICRO), v_micro, is_valid, date64(), v_milli, options);
+  CheckCase<TimestampType, int64_t, Date64Type, int64_t>(
+      timestamp(TimeUnit::MILLI), v_milli, is_valid, date64(), v_milli, options);
+  CheckCase<TimestampType, int64_t, Date64Type, int64_t>(
+      timestamp(TimeUnit::SECOND), v_second, is_valid, date64(), v_milli, options);
+
+  CheckCase<TimestampType, int64_t, Date32Type, int32_t>(
+      timestamp(TimeUnit::NANO), v_nano, is_valid, date32(), v_day, options);
+  CheckCase<TimestampType, int64_t, Date32Type, int32_t>(
+      timestamp(TimeUnit::MICRO), v_micro, is_valid, date32(), v_day, options);
+  CheckCase<TimestampType, int64_t, Date32Type, int32_t>(
+      timestamp(TimeUnit::MILLI), v_milli, is_valid, date32(), v_day, options);
+  CheckCase<TimestampType, int64_t, Date32Type, int32_t>(
+      timestamp(TimeUnit::SECOND), v_second, is_valid, date32(), v_day, options);
+
+  // Disallow truncate, failures
+  vector<int64_t> v_nano_fail = {946684800000000001, 946771200000000001, 0};
+  vector<int64_t> v_micro_fail = {946684800000001, 946771200000001, 0};
+  vector<int64_t> v_milli_fail = {946684800001, 946771200001, 0};
+  vector<int64_t> v_second_fail = {946684801, 946771201, 0};
+
+  options.allow_time_truncate = false;
+  CheckFails<TimestampType>(timestamp(TimeUnit::NANO), v_nano_fail, is_valid, date64(),
+                            options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::MICRO), v_micro_fail, is_valid, date64(),
+                            options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::MILLI), v_milli_fail, is_valid, date64(),
+                            options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::SECOND), v_second_fail, is_valid,
+                            date64(), options);
+
+  CheckFails<TimestampType>(timestamp(TimeUnit::NANO), v_nano_fail, is_valid, date32(),
+                            options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::MICRO), v_micro_fail, is_valid, date32(),
+                            options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::MILLI), v_milli_fail, is_valid, date32(),
+                            options);
+  CheckFails<TimestampType>(timestamp(TimeUnit::SECOND), v_second_fail, is_valid,
+                            date32(), options);
+
+  // Make sure that nulls are excluded from the truncation checks
+  vector<int64_t> v_second_nofail = {946684800, 946771200, 1};
+  CheckCase<TimestampType, int64_t, Date64Type, int64_t>(
+      timestamp(TimeUnit::SECOND), v_second_nofail, is_valid, date64(), v_milli, options);
+  CheckCase<TimestampType, int64_t, Date32Type, int32_t>(
+      timestamp(TimeUnit::SECOND), v_second_nofail, is_valid, date32(), v_day, options);
+}
+
 TEST_F(TestCast, TimeToTime) {
   CastOptions options;
 
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index d52627ebfee12..0e775a0fb0e99 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -519,7 +519,26 @@ class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Conver
   }
 };
 
-class DateConverter : public TypedConverterVisitor<Date64Builder, DateConverter> {
+class Date32Converter : public TypedConverterVisitor<Date32Builder, Date32Converter> {
+ public:
+  inline Status AppendItem(const OwnedRef& item) {
+    int32_t t;
+    if (PyDate_Check(item.obj())) {
+      auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
+      t = static_cast<int32_t>(PyDate_to_s(pydate));
+    } else {
+      int64_t casted_val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      RETURN_IF_PYERROR();
+      if (casted_val > std::numeric_limits<int32_t>::max()) {
+        return Status::Invalid("Integer as date32 larger than INT32_MAX");
+      }
+      t = static_cast<int32_t>(casted_val);
+    }
+    return typed_builder_->Append(t);
+  }
+};
+
+class Date64Converter : public TypedConverterVisitor<Date64Builder, Date64Converter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     int64_t t;
@@ -535,7 +554,7 @@ class DateConverter : public TypedConverterVisitor<Date64Builder, DateConverter>
 };
 
 class TimestampConverter
-    : public TypedConverterVisitor<Date64Builder, TimestampConverter> {
+    : public TypedConverterVisitor<TimestampBuilder, TimestampConverter> {
  public:
   explicit TimestampConverter(TimeUnit::type unit) : unit_(unit) {}
 
@@ -717,8 +736,10 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
       return std::make_shared<UInt32Converter>();
     case Type::UINT64:
       return std::make_shared<UInt64Converter>();
+    case Type::DATE32:
+      return std::make_shared<Date32Converter>();
     case Type::DATE64:
-      return std::make_shared<DateConverter>();
+      return std::make_shared<Date64Converter>();
     case Type::TIMESTAMP:
       return std::make_shared<TimestampConverter>(
           static_cast<const TimestampType&>(*type).unit());
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index ead3a04810121..c5aff2e4f2e3a 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -260,6 +260,7 @@ class NumPyConverter {
       : pool_(pool),
         type_(type),
         arr_(reinterpret_cast<PyArrayObject*>(ao)),
+        dtype_(PyArray_DESCR(arr_)),
         mask_(nullptr),
         use_pandas_null_sentinels_(use_pandas_null_sentinels) {
     if (mo != nullptr && mo != Py_None) {
@@ -431,6 +432,7 @@ class NumPyConverter {
   MemoryPool* pool_;
   std::shared_ptr<DataType> type_;
   PyArrayObject* arr_;
+  PyArray_Descr* dtype_;
   PyArrayObject* mask_;
   int64_t length_;
   int64_t stride_;
@@ -450,7 +452,7 @@ Status NumPyConverter::Convert() {
     return Status::Invalid("only handle 1-dimensional arrays");
   }
 
-  if (PyArray_DESCR(arr_)->type_num == NPY_OBJECT) {
+  if (dtype_->type_num == NPY_OBJECT) {
     return ConvertObjects();
   }
 
@@ -462,33 +464,12 @@ Status NumPyConverter::Convert() {
   return VisitTypeInline(*type_, this);
 }
 
-template <typename T, typename T2>
-void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) {
-  // Passing input_data as non-const is a concession to PyObject*
-  int64_t j = 0;
-  for (int64_t i = 0; i < length; ++i) {
-    output_data[i] = static_cast<T2>(input_data[j]);
-    j += stride;
-  }
-}
-
-template <>
-void CopyStrided<PyObject*, PyObject*>(PyObject** input_data, int64_t length,
-                                       int64_t stride, PyObject** output_data) {
-  int64_t j = 0;
-  for (int64_t i = 0; i < length; ++i) {
-    output_data[i] = input_data[j];
-    if (output_data[i] != nullptr) {
-      Py_INCREF(output_data[i]);
-    }
-    j += stride;
-  }
-}
+namespace {
 
-static Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t length,
-                         const std::shared_ptr<DataType>& in_type,
-                         const std::shared_ptr<DataType>& out_type, MemoryPool* pool,
-                         std::shared_ptr<Buffer>* out) {
+Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t length,
+                  const std::shared_ptr<DataType>& in_type,
+                  const std::shared_ptr<DataType>& out_type, MemoryPool* pool,
+                  std::shared_ptr<Buffer>* out) {
   // Must cast
   std::vector<std::shared_ptr<Buffer>> buffers = {nullptr, input};
   auto tmp_data = std::make_shared<ArrayData>(in_type, length, buffers, 0);
@@ -499,6 +480,7 @@ static Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t len
   compute::FunctionContext context(pool);
   compute::CastOptions cast_options;
   cast_options.allow_int_overflow = false;
+  cast_options.allow_time_truncate = false;
 
   RETURN_NOT_OK(
       compute::Cast(&context, *tmp_array, out_type, cast_options, &casted_array));
@@ -506,29 +488,47 @@ static Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t len
   return Status::OK();
 }
 
+template <typename T, typename T2>
+void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) {
+  // Passing input_data as non-const is a concession to PyObject*
+  int64_t j = 0;
+  for (int64_t i = 0; i < length; ++i) {
+    output_data[i] = static_cast<T2>(input_data[j]);
+    j += stride;
+  }
+}
+
 template <typename ArrowType>
-inline Status NumPyConverter::ConvertData(std::shared_ptr<Buffer>* data) {
+Status CopyStridedArray(PyArrayObject* arr, const int64_t length, MemoryPool* pool,
+                        std::shared_ptr<Buffer>* out) {
   using traits = internal::arrow_traits<ArrowType::type_id>;
   using T = typename traits::T;
 
+  // Strided, must copy into new contiguous memory
+  const int64_t stride = PyArray_STRIDES(arr)[0];
+  const int64_t stride_elements = stride / sizeof(T);
+
+  auto new_buffer = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length));
+  CopyStrided(reinterpret_cast<T*>(PyArray_DATA(arr)), length, stride_elements,
+              reinterpret_cast<T*>(new_buffer->mutable_data()));
+  *out = new_buffer;
+  return Status::OK();
+}
+
+}  // namespace
+
+template <typename ArrowType>
+inline Status NumPyConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   if (is_strided()) {
-    // Strided, must copy into new contiguous memory
-    const int64_t stride = PyArray_STRIDES(arr_)[0];
-    const int64_t stride_elements = stride / sizeof(T);
-
-    auto new_buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length_));
-    CopyStrided(reinterpret_cast<T*>(PyArray_DATA(arr_)), length_, stride_elements,
-                reinterpret_cast<T*>(new_buffer->mutable_data()));
-    *data = new_buffer;
+    RETURN_NOT_OK(CopyStridedArray<ArrowType>(arr_, length_, pool_, data));
   } else {
     // Can zero-copy
     *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
   }
 
   std::shared_ptr<DataType> input_type;
-  RETURN_NOT_OK(
-      NumPyDtypeToArrow(reinterpret_cast<PyObject*>(PyArray_DESCR(arr_)), &input_type));
+  RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_), &input_type));
 
   if (!input_type->Equals(*type_)) {
     RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data));
@@ -537,45 +537,6 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   return Status::OK();
 }
 
-template <>
-inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* data) {
-  // Handle LONGLONG->INT64 and other fun things
-  int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num);
-  int type_size = NumPyTypeSize(type_num_compat);
-
-  if (type_size == 4) {
-    // Source and target are INT32, so can refer to the main implementation.
-    return ConvertData<Int32Type>(data);
-  } else if (type_size == 8) {
-    // We need to scale down from int64 to int32
-    auto new_buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(new_buffer->Resize(sizeof(int32_t) * length_));
-
-    auto input = reinterpret_cast<const int64_t*>(PyArray_DATA(arr_));
-    auto output = reinterpret_cast<int32_t*>(new_buffer->mutable_data());
-
-    if (is_strided()) {
-      // Strided, must copy into new contiguous memory
-      const int64_t stride = PyArray_STRIDES(arr_)[0];
-      const int64_t stride_elements = stride / sizeof(int64_t);
-      CopyStrided(input, length_, stride_elements, output);
-    } else {
-      // TODO(wesm): int32 overflow checks
-      for (int64_t i = 0; i < length_; ++i) {
-        *output++ = static_cast<int32_t>(*input++);
-      }
-    }
-    *data = new_buffer;
-  } else {
-    std::stringstream ss;
-    ss << "Cannot convert NumPy array of element size ";
-    ss << type_size << " to a Date32 array";
-    return Status::NotImplemented(ss.str());
-  }
-
-  return Status::OK();
-}
-
 template <>
 inline Status NumPyConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>* data) {
   int64_t nbytes = BitUtil::BytesForBits(length_);
@@ -597,6 +558,42 @@ inline Status NumPyConverter::ConvertData<BooleanType>(std::shared_ptr<Buffer>*
   return Status::OK();
 }
 
+template <>
+inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* data) {
+  if (is_strided()) {
+    RETURN_NOT_OK(CopyStridedArray<Date32Type>(arr_, length_, pool_, data));
+  } else {
+    // Can zero-copy
+    *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
+  }
+
+  // If we have inbound datetime64[D] data, this needs to be downcasted
+  // separately here from int64_t to int32_t, because this data is not
+  // supported in compute::Cast
+  auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(dtype_->c_metadata);
+  if (dtype_->type_num == NPY_DATETIME && date_dtype->meta.base == NPY_FR_D) {
+    auto date32_buffer = std::make_shared<PoolBuffer>(pool_);
+    RETURN_NOT_OK(date32_buffer->Resize(sizeof(int32_t) * length_));
+
+    auto datetime64_values = reinterpret_cast<const int64_t*>((*data)->data());
+    auto date32_values = reinterpret_cast<int32_t*>(date32_buffer->mutable_data());
+    for (int64_t i = 0; i < length_; ++i) {
+      // TODO(wesm): How pedantic do we really want to be about checking for int32
+      // overflow here?
+      *date32_values++ = static_cast<int32_t>(*datetime64_values++);
+    }
+    *data = date32_buffer;
+  } else {
+    std::shared_ptr<DataType> input_type;
+    RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_), &input_type));
+    if (!input_type->Equals(*type_)) {
+      RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data));
+    }
+  }
+
+  return Status::OK();
+}
+
 template <typename T>
 struct UnboxDate {};
 
diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h
index c110bc64a2a2f..e76c2e0db4aea 100644
--- a/cpp/src/arrow/python/util/datetime.h
+++ b/cpp/src/arrow/python/util/datetime.h
@@ -235,6 +235,11 @@ static inline Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit,
   return Status::OK();
 }
 
+static inline int64_t PyDate_to_s(PyDateTime_Date* pydate) {
+  return get_days_from_date(PyDateTime_GET_YEAR(pydate), PyDateTime_GET_MONTH(pydate),
+                            PyDateTime_GET_DAY(pydate));
+}
+
 static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   int64_t total_seconds = 0;
   total_seconds += PyDateTime_DATE_GET_SECOND(pydate);
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 414266ddb14ed..c7a0d49b40db1 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -178,6 +178,25 @@ def test_date(self):
         assert arr[2].as_py() == datetime.date(1970, 1, 1)
         assert arr[3].as_py() == datetime.date(2040, 2, 26)
 
+    def test_date32(self):
+        data = [datetime.date(2000, 1, 1), None]
+        arr = pa.array(data, type=pa.date32())
+
+        data2 = [10957, None]
+        arr2 = pa.array(data2, type=pa.date32())
+
+        for x in [arr, arr2]:
+            assert len(x) == 2
+            assert x.type == pa.date32()
+            assert x.null_count == 1
+            assert x[0].as_py() == datetime.date(2000, 1, 1)
+            assert x[1] is pa.NA
+
+        # Overflow
+        data3 = [2**32, None]
+        with pytest.raises(pa.ArrowException):
+            pa.array(data3, type=pa.date32())
+
     def test_timestamp(self):
         data = [
             datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index d00bf1b28eddc..07ecf3010a32e 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -525,6 +525,16 @@ def test_timestamps_with_timezone(self):
 
         self._check_pandas_roundtrip(df)
 
+    def test_datetime64_to_date32(self):
+        # ARROW-1718
+        arr = pa.array([date(2017, 10, 23), None])
+        c = pa.Column.from_array("d", arr)
+        s = c.to_pandas()
+
+        arr2 = pa.Array.from_pandas(s, type=pa.date32())
+
+        assert arr2.equals(arr.cast('date32'))
+
     def test_date_infer(self):
         df = pd.DataFrame({
             'date': [date(2000, 1, 1),
@@ -984,6 +994,7 @@ def test_numpy_datetime64_columns(self):
                 dtype='datetime64[s]')
         self._check_array_from_pandas_roundtrip(datetime64_s)
 
+    def test_numpy_datetime64_day_unit(self):
         datetime64_d = np.array([
                 '2007-07-13',
                 None,

From 39243ffaf5eb1d1f2a748ea1ec2b36658ba7f3d7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 30 Oct 2017 14:59:47 -0400
Subject: [PATCH 1203/1644] ARROW-1409: [Format] Remove page id from Buffer
 metadata, increment metadata version number

This is a breaking metadata change per discussion on the mailing list. I expect this kind of truly breaking changes to be exceedingly rare going forward, and when we make a 1.0.0 release we should document expectations around metadata / memory format stability.

This could be made backwards compatible with some effort (we would have to add `RecordBatchV3` and `BufferV3` types).

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1225 from wesm/ARROW-1409 and squashes the following commits:

582fad90 [Wes McKinney] Disable JS in Travis CI for now
845f290f [Wes McKinney] Bump metadata version in Java, add check for V4
e2150c19 [Wes McKinney] Remove page id from Buffer metadata, increment metadata version number
---
 .travis.yml                                   | 15 +++++------
 cpp/src/arrow/ipc/ipc-read-write-test.cc      |  2 +-
 cpp/src/arrow/ipc/message.cc                  | 15 +----------
 cpp/src/arrow/ipc/message.h                   | 14 ++++++++++-
 cpp/src/arrow/ipc/metadata-internal.cc        | 25 +++++++++++++++++--
 cpp/src/arrow/ipc/metadata-internal.h         | 10 ++++----
 cpp/src/arrow/ipc/reader.cc                   | 15 +----------
 cpp/src/arrow/ipc/writer.cc                   | 12 +--------
 format/Schema.fbs                             | 14 +++++++----
 .../arrow/vector/schema/ArrowBuffer.java      | 16 +++---------
 .../arrow/vector/schema/ArrowRecordBatch.java |  2 +-
 .../vector/stream/MessageSerializer.java      |  6 ++++-
 js/src/format/Schema_generated.ts             | 20 +++------------
 13 files changed, 75 insertions(+), 91 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6419548a622f3..52d7a5f800505 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -112,13 +112,14 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
-  - language: node_js
-    os: linux
-    node_js: node
-    before_script:
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
+  # TODO(wesm): Re-enable after issues in ARROW-1409 resolved
+  # - language: node_js
+  #   os: linux
+  #   node_js: node
+  #   before_script:
+  #   - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+  #   script:
+  #   - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
   - compiler: gcc
     language: cpp
     os: linux
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index adf34a9eb5422..6f2f5cf856055 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -243,7 +243,7 @@ TEST_F(TestIpcRoundTrip, MetadataVersion) {
   std::unique_ptr<Message> message;
   ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
 
-  ASSERT_EQ(MetadataVersion::V3, message->metadata_version());
+  ASSERT_EQ(MetadataVersion::V4, message->metadata_version());
 }
 
 TEST_P(TestIpcRoundTrip, SliceRoundTrip) {
diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc
index 0dd5c72e51980..21d6a69a28603 100644
--- a/cpp/src/arrow/ipc/message.cc
+++ b/cpp/src/arrow/ipc/message.cc
@@ -67,20 +67,7 @@ class Message::MessageImpl {
   }
 
   MetadataVersion version() const {
-    switch (message_->version()) {
-      case flatbuf::MetadataVersion_V1:
-        // Arrow 0.1
-        return MetadataVersion::V1;
-      case flatbuf::MetadataVersion_V2:
-        // Arrow 0.2
-        return MetadataVersion::V2;
-      case flatbuf::MetadataVersion_V3:
-        // Arrow >= 0.3
-        return MetadataVersion::V3;
-      // Add cases as other versions become available
-      default:
-        return MetadataVersion::V3;
-    }
+    return internal::GetMetadataVersion(message_->version());
   }
 
   const void* header() const { return message_->header(); }
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index a1b6c07a43d0e..495474e505157 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -42,7 +42,19 @@ class RandomAccessFile;
 
 namespace ipc {
 
-enum class MetadataVersion : char { V1, V2, V3 };
+enum class MetadataVersion : char {
+  /// 0.1.0
+  V1,
+
+  /// 0.2.0
+  V2,
+
+  /// 0.3.0 to 0.7.1
+  V3,
+
+  /// >= 0.8.0
+  V4
+};
 
 // ARROW-109: We set this number arbitrarily to help catch user mistakes. For
 // deeply nested schemas, it is expected the user will indicate explicitly the
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index ad00cfb6c09be..f04e9b05a01b8 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -33,6 +33,7 @@
 #include "arrow/ipc/Message_generated.h"
 #include "arrow/ipc/Tensor_generated.h"
 #include "arrow/ipc/dictionary.h"
+#include "arrow/ipc/message.h"
 #include "arrow/ipc/util.h"
 #include "arrow/status.h"
 #include "arrow/tensor.h"
@@ -57,6 +58,26 @@ using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
 using FBString = flatbuffers::Offset<flatbuffers::String>;
 
+MetadataVersion GetMetadataVersion(flatbuf::MetadataVersion version) {
+  switch (version) {
+    case flatbuf::MetadataVersion_V1:
+      // Arrow 0.1
+      return MetadataVersion::V1;
+    case flatbuf::MetadataVersion_V2:
+      // Arrow 0.2
+      return MetadataVersion::V2;
+    case flatbuf::MetadataVersion_V3:
+      // Arrow 0.3 to 0.7.1
+      return MetadataVersion::V4;
+    case flatbuf::MetadataVersion_V4:
+      // Arrow >= 0.8
+      return MetadataVersion::V4;
+      // Add cases as other versions become available
+    default:
+      return MetadataVersion::V4;
+  }
+}
+
 static Status IntFromFlatbuffer(const flatbuf::Int* int_data,
                                 std::shared_ptr<DataType>* out) {
   if (int_data->bitWidth() > 64) {
@@ -700,7 +721,7 @@ static Status WriteBuffers(FBB& fbb, const std::vector<BufferMetadata>& buffers,
 
   for (size_t i = 0; i < buffers.size(); ++i) {
     const BufferMetadata& buffer = buffers[i];
-    fb_buffers.emplace_back(buffer.page, buffer.offset, buffer.length);
+    fb_buffers.emplace_back(buffer.offset, buffer.length);
   }
   *out = fbb.CreateVectorOfStructs(fb_buffers);
   return Status::OK();
@@ -751,7 +772,7 @@ Status WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset,
   auto fb_shape = fbb.CreateVector(dims);
   auto fb_strides = fbb.CreateVector(tensor.strides());
   int64_t body_length = tensor.data()->size();
-  flatbuf::Buffer buffer(-1, buffer_start_offset, body_length);
+  flatbuf::Buffer buffer(buffer_start_offset, body_length);
 
   TensorOffset fb_tensor =
       flatbuf::CreateTensor(fbb, fb_type_type, fb_type, fb_shape, fb_strides, &buffer);
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
index 309e7587a754c..380f3c9eb1013 100644
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ b/cpp/src/arrow/ipc/metadata-internal.h
@@ -27,6 +27,7 @@
 
 #include "arrow/ipc/Schema_generated.h"
 #include "arrow/ipc/dictionary.h"
+#include "arrow/ipc/message.h"
 
 namespace arrow {
 
@@ -48,10 +49,12 @@ namespace ipc {
 namespace internal {
 
 static constexpr flatbuf::MetadataVersion kCurrentMetadataVersion =
-    flatbuf::MetadataVersion_V3;
+    flatbuf::MetadataVersion_V4;
 
 static constexpr flatbuf::MetadataVersion kMinMetadataVersion =
-    flatbuf::MetadataVersion_V3;
+    flatbuf::MetadataVersion_V4;
+
+MetadataVersion GetMetadataVersion(flatbuf::MetadataVersion version);
 
 static constexpr const char* kArrowMagicBytes = "ARROW1";
 
@@ -62,9 +65,6 @@ struct FieldMetadata {
 };
 
 struct BufferMetadata {
-  /// The shared memory page id where to find this. Set to -1 if unused
-  int32_t page;
-
   /// The relative offset into the memory page to the starting byte of the buffer
   int64_t offset;
 
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 50eb9039c6ab6..8e10d7d66f907 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -550,20 +550,7 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl {
   int num_record_batches() const { return footer_->recordBatches()->size(); }
 
   MetadataVersion version() const {
-    switch (footer_->version()) {
-      case flatbuf::MetadataVersion_V1:
-        // Arrow 0.1
-        return MetadataVersion::V1;
-      case flatbuf::MetadataVersion_V2:
-        // Arrow 0.2
-        return MetadataVersion::V2;
-      case flatbuf::MetadataVersion_V3:
-        // Arrow 0.3
-        return MetadataVersion::V3;
-      // Add cases as other versions become available
-      default:
-        return MetadataVersion::V3;
-    }
+    return internal::GetMetadataVersion(footer_->version());
   }
 
   FileBlock record_batch(int i) const {
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 279a69544faf2..5598cc68296f7 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -149,8 +149,6 @@ class RecordBatchSerializer : public ArrayVisitor {
 
     buffer_meta_.reserve(buffers_.size());
 
-    const int32_t kNoPageId = -1;
-
     // Construct the buffer metadata for the record batch header
     for (size_t i = 0; i < buffers_.size(); ++i) {
       const Buffer* buffer = buffers_[i].get();
@@ -163,15 +161,7 @@ class RecordBatchSerializer : public ArrayVisitor {
         padding = BitUtil::RoundUpToMultipleOf8(size) - size;
       }
 
-      // TODO(wesm): We currently have no notion of shared memory page id's,
-      // but we've included it in the metadata IDL for when we have it in the
-      // future. Use page = -1 for now
-      //
-      // Note that page ids are a bespoke notion for Arrow and not a feature we
-      // are using from any OS-level shared memory. The thought is that systems
-      // may (in the future) associate integer page id's with physical memory
-      // pages (according to whatever is the desired shared memory mechanism)
-      buffer_meta_.push_back({kNoPageId, offset, size + padding});
+      buffer_meta_.push_back({offset, size + padding});
       offset += size + padding;
     }
 
diff --git a/format/Schema.fbs b/format/Schema.fbs
index 186f8e362bde2..6021e92b847e7 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -20,9 +20,17 @@
 namespace org.apache.arrow.flatbuf;
 
 enum MetadataVersion:short {
+  /// 0.1.0
   V1,
+
+  /// 0.2.0
   V2,
-  V3
+
+  /// 0.3.0 -> 0.7.1
+  V3,
+
+  /// >= 0.8.0
+  V4
 }
 
 /// These are stored in the flatbuffer in the Type union below
@@ -293,10 +301,6 @@ enum Endianness:short { Little, Big }
 /// ----------------------------------------------------------------------
 /// A Buffer represents a single contiguous memory segment
 struct Buffer {
-  /// The shared memory page id where this buffer is located. Currently this is
-  /// not used
-  page: int;
-
   /// The relative offset into the shared memory page where the bytes for this
   /// buffer starts
   offset: long;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
index d8c9e3001d0a5..4e0187e791b5a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
@@ -24,21 +24,15 @@
 
 public class ArrowBuffer implements FBSerializable {
 
-  private int page;
   private long offset;
   private long size;
 
-  public ArrowBuffer(int page, long offset, long size) {
+  public ArrowBuffer(long offset, long size) {
     super();
-    this.page = page;
     this.offset = offset;
     this.size = size;
   }
 
-  public int getPage() {
-    return page;
-  }
-
   public long getOffset() {
     return offset;
   }
@@ -52,7 +46,6 @@ public int hashCode() {
     final int prime = 31;
     int result = 1;
     result = prime * result + (int) (offset ^ (offset >>> 32));
-    result = prime * result + page;
     result = prime * result + (int) (size ^ (size >>> 32));
     return result;
   }
@@ -72,9 +65,6 @@ public boolean equals(Object obj) {
     if (offset != other.offset) {
       return false;
     }
-    if (page != other.page) {
-      return false;
-    }
     if (size != other.size) {
       return false;
     }
@@ -83,12 +73,12 @@ public boolean equals(Object obj) {
 
   @Override
   public int writeTo(FlatBufferBuilder builder) {
-    return Buffer.createBuffer(builder, page, offset, size);
+    return Buffer.createBuffer(builder, offset, size);
   }
 
   @Override
   public String toString() {
-    return "ArrowBuffer [page=" + page + ", offset=" + offset + ", size=" + size + "]";
+    return "ArrowBuffer [offset=" + offset + ", size=" + size + "]";
   }
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index c842d4c3f9a74..bf0967a2797fe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -72,7 +72,7 @@ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> b
     for (ArrowBuf arrowBuf : buffers) {
       arrowBuf.retain();
       long size = arrowBuf.readableBytes();
-      arrowBuffers.add(new ArrowBuffer(0, offset, size));
+      arrowBuffers.add(new ArrowBuffer(offset, size));
       LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", offset, size));
       offset += size;
       if (alignBuffers && offset % 8 != 0) { // align on 8 byte boundaries
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index f69aa41e7f6bd..c397cec72f0ed 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -385,6 +385,10 @@ public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocat
       throw new IOException("Cannot currently deserialize record batches over 2GB");
     }
 
+    if (message.version() != MetadataVersion.V4) {
+      throw new IOException("Received metadata with an incompatible version number");
+    }
+
     switch (message.headerType()) {
       case MessageHeader.RecordBatch:
         return deserializeRecordBatch(in, message, alloc);
@@ -409,7 +413,7 @@ public static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte header
     Message.startMessage(builder);
     Message.addHeaderType(builder, headerType);
     Message.addHeader(builder, headerOffset);
-    Message.addVersion(builder, MetadataVersion.V3);
+    Message.addVersion(builder, MetadataVersion.V4);
     Message.addBodyLength(builder, bodyLength);
     builder.finish(Message.endMessage(builder));
     return builder.dataBuffer();
diff --git a/js/src/format/Schema_generated.ts b/js/src/format/Schema_generated.ts
index 65493b7f685ec..c5b3e5011d790 100644
--- a/js/src/format/Schema_generated.ts
+++ b/js/src/format/Schema_generated.ts
@@ -2027,16 +2027,6 @@ export namespace org.apache.arrow.flatbuf {
       return this;
     }
 
-    /**
-     * The shared memory page id where this buffer is located. Currently this is
-     * not used
-     *
-     * @returns {number}
-     */
-    page(): number {
-      return this.bb.readInt32(this.bb_pos);
-    }
-
     /**
      * The relative offset into the shared memory page where the bytes for this
      * buffer starts
@@ -2044,7 +2034,7 @@ export namespace org.apache.arrow.flatbuf {
      * @returns {flatbuffers.Long}
      */
     offset(): flatbuffers.Long {
-      return this.bb.readInt64(this.bb_pos + 8);
+      return this.bb.readInt64(this.bb_pos);
     }
 
     /**
@@ -2054,7 +2044,7 @@ export namespace org.apache.arrow.flatbuf {
      * @returns {flatbuffers.Long}
      */
     length(): flatbuffers.Long {
-      return this.bb.readInt64(this.bb_pos + 16);
+      return this.bb.readInt64(this.bb_pos + 8);
     }
 
     /**
@@ -2064,12 +2054,10 @@ export namespace org.apache.arrow.flatbuf {
      * @param {flatbuffers.Long} length
      * @returns {flatbuffers.Offset}
      */
-    static createBuffer(builder: flatbuffers.Builder, page: number, offset: flatbuffers.Long, length: flatbuffers.Long): flatbuffers.Offset {
-      builder.prep(8, 24);
+    static createBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Long, length: flatbuffers.Long): flatbuffers.Offset {
+      builder.prep(8, 16);
       builder.writeInt64(length);
       builder.writeInt64(offset);
-      builder.pad(4);
-      builder.writeInt32(page);
       return builder.offset();
     }
 

From 72b50bc597693f098b67489bd8da40862c9770a2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 31 Oct 2017 11:36:28 -0400
Subject: [PATCH 1204/1644] [C++] Fix clang-format failure from ARROW-1409

Change-Id: Ia45874945e050de0aa6294cbbe01ec63e9b14235
---
 cpp/src/arrow/ipc/metadata-internal.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index f04e9b05a01b8..f0f0f675853b1 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -72,7 +72,7 @@ MetadataVersion GetMetadataVersion(flatbuf::MetadataVersion version) {
     case flatbuf::MetadataVersion_V4:
       // Arrow >= 0.8
       return MetadataVersion::V4;
-      // Add cases as other versions become available
+    // Add cases as other versions become available
     default:
       return MetadataVersion::V4;
   }

From 088055019837f3a8215425959fadb037e01d0b02 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Tue, 31 Oct 2017 12:51:06 -0400
Subject: [PATCH 1205/1644] ARROW-1754: [Python] Fix buggy Parquet roundtrip
 when an index name is the same as a column name

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1271 from cpcloud/ARROW-1754 and squashes the following commits:

3ffbe541 [Phillip Cloud] ARROW-1754: [Python] Fix buggy Parquet roundtrip when an index name is the same as a column name
---
 python/pyarrow/pandas_compat.py      | 52 ++++++++--------------------
 python/pyarrow/tests/test_parquet.py | 43 +++++++++++++++++------
 2 files changed, 48 insertions(+), 47 deletions(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index d6c844c8490f5..1984598ff3533 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -18,7 +18,6 @@
 import ast
 import collections
 import json
-import re
 
 import numpy as np
 import pandas as pd
@@ -29,13 +28,6 @@
 from pyarrow.compat import PY2, zip_longest  # noqa
 
 
-INDEX_LEVEL_NAME_REGEX = re.compile(r'^__index_level_\d+__$')
-
-
-def is_unnamed_index_level(name):
-    return INDEX_LEVEL_NAME_REGEX.match(name) is not None
-
-
 def infer_dtype(column):
     try:
         return pd.api.types.infer_dtype(column)
@@ -143,7 +135,7 @@ def get_column_metadata(column, name, arrow_type):
 
     Parameters
     ----------
-    column : pandas.Series
+    column : pandas.Series or pandas.Index
     name : str
     arrow_type : pyarrow.DataType
 
@@ -161,7 +153,7 @@ def get_column_metadata(column, name, arrow_type):
         }
         string_dtype = 'object'
 
-    if not isinstance(name, six.string_types):
+    if name is not None and not isinstance(name, six.string_types):
         raise TypeError(
             'Column name must be a string. Got column {} of type {}'.format(
                 name, type(name).__name__
@@ -176,23 +168,7 @@ def get_column_metadata(column, name, arrow_type):
     }
 
 
-def index_level_name(index, i):
-    """Return the name of an index level or a default name if `index.name` is
-    None.
-
-    Parameters
-    ----------
-    index : pandas.Index
-    i : int
-
-    Returns
-    -------
-    name : str
-    """
-    if index.name is not None:
-        return index.name
-    else:
-        return '__index_level_{:d}__'.format(i)
+index_level_name = '__index_level_{:d}__'.format
 
 
 def construct_metadata(df, column_names, index_levels, preserve_index, types):
@@ -222,11 +198,11 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
     ]
 
     if preserve_index:
-        index_column_names = [index_level_name(level, i)
-                              for i, level in enumerate(index_levels)]
+        index_column_names = list(map(
+            index_level_name, range(len(index_levels))
+        ))
         index_column_metadata = [
-            get_column_metadata(level, name=index_level_name(level, i),
-                                arrow_type=arrow_type)
+            get_column_metadata(level, name=level.name, arrow_type=arrow_type)
             for i, (level, arrow_type) in enumerate(
                 zip(index_levels, index_types)
             )
@@ -317,7 +293,7 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1):
     for i, column in enumerate(index_columns):
         columns_to_convert.append(column)
         convert_types.append(None)
-        names.append(index_level_name(column, i))
+        names.append(index_level_name(i))
 
     # NOTE(wesm): If nthreads=None, then we use a heuristic to decide whether
     # using a thread pool is worth it. Currently the heuristic is whether the
@@ -378,6 +354,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     import pyarrow.lib as lib
 
     index_columns = []
+    columns = []
     column_indexes = []
     index_arrays = []
     index_names = []
@@ -390,6 +367,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     if has_pandas_metadata:
         pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
         index_columns = pandas_metadata['index_columns']
+        columns = pandas_metadata['columns']
         column_indexes = pandas_metadata.get('column_indexes', [])
         table = _add_any_metadata(table, pandas_metadata)
 
@@ -397,11 +375,11 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
 
     # Build up a list of index columns and names while removing those columns
     # from the original table
-    for name in index_columns:
-        i = schema.get_field_index(name)
+    logical_index_names = [c['name'] for c in columns[-len(index_columns):]]
+    for raw_name, logical_name in zip(index_columns, logical_index_names):
+        i = schema.get_field_index(raw_name)
         if i != -1:
             col = table.column(i)
-            index_name = None if is_unnamed_index_level(name) else name
             col_pandas = col.to_pandas()
             values = col_pandas.values
             if not values.flags.writeable:
@@ -410,9 +388,9 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
                 values = values.copy()
 
             index_arrays.append(pd.Series(values, dtype=col_pandas.dtype))
-            index_names.append(index_name)
+            index_names.append(logical_name)
             block_table = block_table.remove_column(
-                block_table.schema.get_field_index(name)
+                block_table.schema.get_field_index(raw_name)
             )
 
     # Convert an arrow table to Block from the internal pandas API
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index a7fe98ce71cd1..95dd6a471b6b3 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1171,7 +1171,8 @@ def test_dataset_read_pandas(tmpdir):
 
 
 @parquet
-def test_dataset_read_pandas_common_metadata(tmpdir):
+@pytest.mark.parametrize('preserve_index', [True, False])
+def test_dataset_read_pandas_common_metadata(tmpdir, preserve_index):
     # ARROW-1103
     import pyarrow.parquet as pq
 
@@ -1186,15 +1187,11 @@ def test_dataset_read_pandas_common_metadata(tmpdir):
     paths = []
     for i in range(nfiles):
         df = _test_dataframe(size, seed=i)
-        df.index = pd.Index(np.arange(i * size, (i + 1) * size))
-        df.index.name = 'index'
+        df.index = pd.Index(np.arange(i * size, (i + 1) * size), name='index')
 
-        path = pjoin(dirpath, '{0}.parquet'.format(i))
+        path = pjoin(dirpath, '{:d}.parquet'.format(i))
 
-        df_ex_index = df.reset_index(drop=True)
-        df_ex_index['index'] = df.index
-        table = pa.Table.from_pandas(df_ex_index,
-                                     preserve_index=False)
+        table = pa.Table.from_pandas(df, preserve_index=preserve_index)
 
         # Obliterate metadata
         table = table.replace_schema_metadata(None)
@@ -1206,7 +1203,9 @@ def test_dataset_read_pandas_common_metadata(tmpdir):
         paths.append(path)
 
     # Write _metadata common file
-    table_for_metadata = pa.Table.from_pandas(df)
+    table_for_metadata = pa.Table.from_pandas(
+        df, preserve_index=preserve_index
+    )
     pq.write_metadata(table_for_metadata.schema,
                       pjoin(dirpath, '_metadata'))
 
@@ -1214,7 +1213,7 @@ def test_dataset_read_pandas_common_metadata(tmpdir):
     columns = ['uint8', 'strings']
     result = dataset.read_pandas(columns=columns).to_pandas()
     expected = pd.concat([x[columns] for x in frames])
-
+    expected.index.name = df.index.name if preserve_index else None
     tm.assert_frame_equal(result, expected)
 
 
@@ -1387,3 +1386,27 @@ def test_large_table_int32_overflow():
     table = pa.Table.from_arrays([parr], names=['one'])
     f = io.BytesIO()
     _write_table(table, f)
+
+
+def test_index_column_name_duplicate(tmpdir):
+    data = {
+        'close': {
+            pd.Timestamp('2017-06-30 01:31:00'): 154.99958999999998,
+            pd.Timestamp('2017-06-30 01:32:00'): 154.99958999999998,
+        },
+        'time': {
+            pd.Timestamp('2017-06-30 01:31:00'): pd.Timestamp(
+                '2017-06-30 01:31:00'
+            ),
+            pd.Timestamp('2017-06-30 01:32:00'): pd.Timestamp(
+                '2017-06-30 01:32:00'
+            ),
+        }
+    }
+    path = str(tmpdir / 'data.parquet')
+    dfx = pd.DataFrame(data).set_index('time', drop=False)
+    tdfx = pa.Table.from_pandas(dfx)
+    _write_table(tdfx, path)
+    arrow_table = _read_table(path)
+    result_df = arrow_table.to_pandas()
+    tm.assert_frame_equal(result_df, dfx)

From eca992471b1b2230259509f74d7bc4af97922788 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 31 Oct 2017 12:54:06 -0400
Subject: [PATCH 1206/1644] ARROW-1658: [Python] Add boundschecking of
 dictionary indices when creating CategoricalBlock

We should probably do this bounds-checking earlier and in the main Arrow C++ library when ingesting "untrusted" arrays. I will create a JIRA, but this is a stopgap in the meantime

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1270 from wesm/ARROW-1658 and squashes the following commits:

234a5685 [Wes McKinney] Add boundschecking of dictionary indices when creating CategoricalBlock as workaround for segfaults from invalid codes making their way into pandas
---
 cpp/src/arrow/python/arrow_to_pandas.cc     | 38 ++++++++++++++-------
 python/pyarrow/array.pxi                    | 37 +++++++++++---------
 python/pyarrow/tests/test_convert_pandas.py | 15 ++++++++
 3 files changed, 61 insertions(+), 29 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 7f1591213cec6..c92faede1347b 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -966,9 +966,10 @@ class CategoricalBlock : public PandasBlock {
         "CategoricalBlock allocation happens when calling Write");
   }
 
-  template <int ARROW_INDEX_TYPE>
+  template <typename ArrowType>
   Status WriteIndices(const std::shared_ptr<Column>& col) {
-    using TRAITS = internal::arrow_traits<ARROW_INDEX_TYPE>;
+    using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
+    using TRAITS = internal::arrow_traits<ArrowType::type_id>;
     using T = typename TRAITS::T;
     constexpr int npy_type = TRAITS::npy_type;
 
@@ -977,10 +978,22 @@ class CategoricalBlock : public PandasBlock {
     // Sniff the first chunk
     const std::shared_ptr<Array> arr_first = data.chunk(0);
     const auto& dict_arr_first = static_cast<const DictionaryArray&>(*arr_first);
-    const auto& indices_first =
-        static_cast<const PrimitiveArray&>(*dict_arr_first.indices());
+    const auto& indices_first = static_cast<const ArrayType&>(*dict_arr_first.indices());
+
+    auto CheckIndices = [](const ArrayType& arr, int64_t dict_length) {
+      const T* values = arr.raw_values();
+      for (int64_t i = 0; i < arr.length(); ++i) {
+        if (arr.IsValid(i) && (values[i] < 0 || values[i] >= dict_length)) {
+          std::stringstream ss;
+          ss << "Out of bounds dictionary index: " << static_cast<int64_t>(values[i]);
+          return Status::Invalid(ss.str());
+        }
+      }
+      return Status::OK();
+    };
 
     if (data.num_chunks() == 1 && indices_first.null_count() == 0) {
+      RETURN_NOT_OK(CheckIndices(indices_first, dict_arr_first.dictionary()->length()));
       RETURN_NOT_OK(AllocateNDArrayFromIndices<T>(npy_type, indices_first));
     } else {
       if (options_.zero_copy_only) {
@@ -998,9 +1011,10 @@ class CategoricalBlock : public PandasBlock {
         const std::shared_ptr<Array> arr = data.chunk(c);
         const auto& dict_arr = static_cast<const DictionaryArray&>(*arr);
 
-        const auto& indices = static_cast<const PrimitiveArray&>(*dict_arr.indices());
+        const auto& indices = static_cast<const ArrayType&>(*dict_arr.indices());
         auto in_values = reinterpret_cast<const T*>(indices.raw_values());
 
+        RETURN_NOT_OK(CheckIndices(indices, dict_arr.dictionary()->length()));
         // Null is -1 in CategoricalBlock
         for (int i = 0; i < arr->length(); ++i) {
           *out_values++ = indices.IsNull(i) ? -1 : in_values[i];
@@ -1026,16 +1040,16 @@ class CategoricalBlock : public PandasBlock {
 
     switch (dict_type.index_type()->id()) {
       case Type::INT8:
-        RETURN_NOT_OK(WriteIndices<Type::INT8>(converted_col));
+        RETURN_NOT_OK(WriteIndices<Int8Type>(converted_col));
         break;
       case Type::INT16:
-        RETURN_NOT_OK(WriteIndices<Type::INT16>(converted_col));
+        RETURN_NOT_OK(WriteIndices<Int16Type>(converted_col));
         break;
       case Type::INT32:
-        RETURN_NOT_OK(WriteIndices<Type::INT32>(converted_col));
+        RETURN_NOT_OK(WriteIndices<Int32Type>(converted_col));
         break;
       case Type::INT64:
-        RETURN_NOT_OK(WriteIndices<Type::INT64>(converted_col));
+        RETURN_NOT_OK(WriteIndices<Int64Type>(converted_col));
         break;
       default: {
         std::stringstream ss;
@@ -1091,13 +1105,11 @@ class CategoricalBlock : public PandasBlock {
 
     PyObject* block_arr = PyArray_NewFromDescr(&PyArray_Type, descr, 1, block_dims,
                                                nullptr, data, NPY_ARRAY_CARRAY, nullptr);
+    RETURN_IF_PYERROR();
 
     npy_intp placement_dims[1] = {num_columns_};
     PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
-    if (placement_arr == NULL) {
-      // TODO(wesm): propagating Python exception
-      return Status::OK();
-    }
+    RETURN_IF_PYERROR();
 
     block_arr_.reset(block_arr);
     placement_arr_.reset(placement_arr);
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 7da5c3caffdc2..7752d062a774c 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -162,6 +162,7 @@ def array(object obj, type=None, mask=None,
             return DictionaryArray.from_arrays(
                 values.codes, values.categories.values,
                 mask=mask, ordered=values.ordered,
+                from_pandas=from_pandas,
                 memory_pool=memory_pool)
         else:
             values, type = pdcompat.get_datetimetz_type(values, obj.dtype,
@@ -671,7 +672,7 @@ cdef class DictionaryArray(Array):
 
     @staticmethod
     def from_arrays(indices, dictionary, mask=None, ordered=False,
-                    MemoryPool memory_pool=None):
+                    from_pandas=False, MemoryPool memory_pool=None):
         """
         Construct Arrow DictionaryArray from array of indices (must be
         non-negative integers) and corresponding array of dictionary values
@@ -682,15 +683,20 @@ cdef class DictionaryArray(Array):
         dictionary : ndarray or pandas.Series
         mask : ndarray or pandas.Series, boolean type
             True values indicate that indices are actually null
+        from_pandas : boolean, default False
+            If True, the indices should be treated as though they originated in
+            a pandas.Categorical (null encoded as -1)
         ordered : boolean, default False
             Set to True if the category values are ordered
+        memory_pool : MemoryPool, default None
+            For memory allocations, if required, otherwise uses default pool
 
         Returns
         -------
         dict_array : DictionaryArray
         """
         cdef:
-            Array arrow_indices, arrow_dictionary
+            Array _indices, _dictionary
             DictionaryArray result
             shared_ptr[CDataType] c_type
             shared_ptr[CArray] c_result
@@ -699,29 +705,28 @@ cdef class DictionaryArray(Array):
             if mask is not None:
                 raise NotImplementedError(
                     "mask not implemented with Arrow array inputs yet")
-            arrow_indices = indices
+            _indices = indices
         else:
-            if mask is None:
-                mask = indices == -1
-            else:
-                mask = mask | (indices == -1)
-            arrow_indices = Array.from_pandas(indices, mask=mask,
-                                              memory_pool=memory_pool)
+            if from_pandas:
+                if mask is None:
+                    mask = indices == -1
+                else:
+                    mask = mask | (indices == -1)
+            _indices = array(indices, mask=mask, memory_pool=memory_pool)
 
         if isinstance(dictionary, Array):
-            arrow_dictionary = dictionary
+            _dictionary = dictionary
         else:
-            arrow_dictionary = Array.from_pandas(dictionary,
-                                                 memory_pool=memory_pool)
+            _dictionary = array(dictionary, memory_pool=memory_pool)
 
-        if not isinstance(arrow_indices, IntegerArray):
+        if not isinstance(_indices, IntegerArray):
             raise ValueError('Indices must be integer type')
 
         cdef c_bool c_ordered = ordered
 
-        c_type.reset(new CDictionaryType(arrow_indices.type.sp_type,
-                                         arrow_dictionary.sp_array, c_ordered))
-        c_result.reset(new CDictionaryArray(c_type, arrow_indices.sp_array))
+        c_type.reset(new CDictionaryType(_indices.type.sp_type,
+                                         _dictionary.sp_array, c_ordered))
+        c_result.reset(new CDictionaryArray(c_type, _indices.sp_array))
 
         result = DictionaryArray()
         result.init(c_result)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 07ecf3010a32e..dabccac37c3d8 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -217,6 +217,21 @@ def test_zero_copy_success(self):
         result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True)
         npt.assert_array_equal(result, [0, 1, 2])
 
+    def test_dictionary_indices_boundscheck(self):
+        # ARROW-1658. No validation of indices leads to segfaults in pandas
+        indices = [[0, 1], [0, -1]]
+
+        for inds in indices:
+            arr = pa.DictionaryArray.from_arrays(inds, ['a'])
+            batch = pa.RecordBatch.from_arrays([arr], ['foo'])
+            table = pa.Table.from_batches([batch, batch, batch])
+
+            with pytest.raises(pa.ArrowException):
+                arr.to_pandas()
+
+            with pytest.raises(pa.ArrowException):
+                table.to_pandas()
+
     def test_zero_copy_dictionaries(self):
         arr = pa.DictionaryArray.from_arrays(
             np.array([0, 0]),

From 9dc4c58d57159edf24bcbe86b6220a6a3bcb09ef Mon Sep 17 00:00:00 2001
From: dhirschf <david.hirschfeld@stanwell.com>
Date: Tue, 31 Oct 2017 14:11:19 -0400
Subject: [PATCH 1207/1644] ARROW-1753: [Python] Provide for matching
 subclasses with register_type in serialization context

https://issues.apache.org/jira/browse/ARROW-1753

Author: dhirschf <david.hirschfeld@stanwell.com>
Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1272 from dhirschfeld/ARROW-1753 and squashes the following commits:

bb7f041a [Philipp Moritz] fix subclass serialization tests
cbc5e09c [dhirschf] Added a test that register_type will work for subclasses of the registered type
46b60f44 [dhirschf] Allow register_type to match subclasses in serialization context
---
 python/pyarrow/serialization.pxi           | 17 ++++--
 python/pyarrow/serialization.py            |  2 +
 python/pyarrow/tests/test_serialization.py | 66 ++++++++++++++++++++++
 python/requirements.txt                    |  2 +-
 4 files changed, 82 insertions(+), 5 deletions(-)

diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index 4e9ab8eb3b374..6b7227797a836 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -88,17 +88,26 @@ cdef class SerializationContext:
             self.custom_deserializers[type_id] = custom_deserializer
 
     def _serialize_callback(self, obj):
-        if type(obj) not in self.type_to_type_id:
+        found = False
+        for type_ in type(obj).__mro__:
+            if type_ in self.type_to_type_id:
+                found = True
+                break
+
+        if not found:
             raise SerializationCallbackError(
                 "pyarrow does not know how to "
-                "serialize objects of type {}.".format(type(obj)), obj)
-        type_id = self.type_to_type_id[type(obj)]
+                "serialize objects of type {}.".format(type(obj)), obj
+            )
+
+        # use the closest match to type(obj)
+        type_id = self.type_to_type_id[type_]
         if type_id in self.types_to_pickle:
             serialized_obj = {"data": pickle.dumps(obj), "pickle": True}
         elif type_id in self.custom_serializers:
             serialized_obj = {"data": self.custom_serializers[type_id](obj)}
         else:
-            if is_named_tuple(type(obj)):
+            if is_named_tuple(type_):
                 serialized_obj = {}
                 serialized_obj["_pa_getnewargs_"] = obj.__getnewargs__()
             elif hasattr(obj, "__dict__"):
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 9dc8ee6dee9ad..2b47513fd1c85 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -69,6 +69,8 @@ def _deserialize_default_dict(data):
         type(lambda: 0), "function",
         pickle=True)
 
+    serialization_context.register_type(type, "type", pickle=True)
+
     # ----------------------------------------------------------------------
     # Set up serialization for numpy with dtype object (primitive types are
     # handled efficiently with Arrow's Tensor facilities, see
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 7878a09228d06..b0c5bc49e6a58 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -416,3 +416,69 @@ class TempClass(object):
     with pytest.raises(pa.DeserializationCallbackError) as err:
         serialized_object.deserialize(deserialization_context)
     assert err.value.type_id == 20*b"\x00"
+
+
+def test_fallback_to_subclasses():
+
+    class SubFoo(Foo):
+        def __init__(self):
+            Foo.__init__(self)
+
+    # should be able to serialize/deserialize an instance
+    # if a base class has been registered
+    serialization_context = pa.SerializationContext()
+    serialization_context.register_type(Foo, "Foo")
+
+    subfoo = SubFoo()
+    # should fallbact to Foo serializer
+    serialized_object = pa.serialize(subfoo, serialization_context)
+
+    reconstructed_object = serialized_object.deserialize(
+        serialization_context
+    )
+    assert type(reconstructed_object) == Foo
+
+
+class Serializable(object):
+    pass
+
+
+def serialize_serializable(obj):
+    return {"type": type(obj), "data": obj.__dict__}
+
+
+def deserialize_serializable(obj):
+    val = obj["type"].__new__(obj["type"])
+    val.__dict__.update(obj["data"])
+    return val
+
+
+class SerializableClass(Serializable):
+    def __init__(self):
+        self.value = 3
+
+
+def test_serialize_subclasses():
+
+    # This test shows how subclasses can be handled in an idiomatic way
+    # by having only a serializer for the base class
+
+    # This technique should however be used with care, since pickling
+    # type(obj) with couldpickle will include the full class definition
+    # in the serialized representation.
+    # This means the class definition is part of every instance of the
+    # object, which in general is not desirable; registering all subclasses
+    # with register_type will result in faster and more memory
+    # efficient serialization.
+
+    serialization_context.register_type(
+        Serializable, "Serializable",
+        custom_serializer=serialize_serializable,
+        custom_deserializer=deserialize_serializable)
+
+    a = SerializableClass()
+    serialized = pa.serialize(a)
+
+    deserialized = serialized.deserialize()
+    assert type(deserialized).__name__ == SerializableClass.__name__
+    assert deserialized.value == 3
diff --git a/python/requirements.txt b/python/requirements.txt
index d2e28a7747ba8..8d0c33afa69a6 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,4 +1,4 @@
 pytest
-cloudpickle
+cloudpickle>=0.4.0
 numpy>=1.10.0
 six

From 142e6ee69bd6a4dc316d00d9efd6d86d119df075 Mon Sep 17 00:00:00 2001
From: Heimir Sverrisson <heimir.sverrisson@gmail.com>
Date: Tue, 31 Oct 2017 22:54:09 -0400
Subject: [PATCH 1208/1644] ARROW-1455 [Python] Add Dockerfile for validating
 Dask integration

A Docker container is created with all the dependencies needed to pull down the Dask code from Github and install it locally, together with Arrow, to run an integration test.

Author: Heimir Sverrisson <heimir.sverrisson@gmail.com>

Closes #1249 from heimir-sverrisson/hs/dockerize_dask and squashes the following commits:

d146185b [Heimir Sverrisson] ARROW-1455 [Python] Add Dockerfile for validating Dask integration
---
 dev/dask_integration.sh                       | 21 +++++
 dev/dask_integration/Dockerfile               | 88 +++++++++++++++++++
 dev/dask_integration/dask_integration.sh      | 49 +++++++++++
 dev/docker-compose.yml                        |  5 ++
 dev/run_docker_compose.sh                     |  2 +-
 python/testing/README.md                      | 24 ++++-
 .../dask_tests/test_dask_integration.py       | 51 +++++++++++
 7 files changed, 238 insertions(+), 2 deletions(-)
 create mode 100755 dev/dask_integration.sh
 create mode 100644 dev/dask_integration/Dockerfile
 create mode 100755 dev/dask_integration/dask_integration.sh
 create mode 100644 python/testing/dask_tests/test_dask_integration.py

diff --git a/dev/dask_integration.sh b/dev/dask_integration.sh
new file mode 100755
index 0000000000000..d344328b6af1e
--- /dev/null
+++ b/dev/dask_integration.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Pass the service name to run_docker_compose.sh
+# Which validates environment and runs the service
+exec "$(dirname ${BASH_SOURCE})"/run_docker_compose.sh dask_integration
diff --git a/dev/dask_integration/Dockerfile b/dev/dask_integration/Dockerfile
new file mode 100644
index 0000000000000..f72ef8ca0daab
--- /dev/null
+++ b/dev/dask_integration/Dockerfile
@@ -0,0 +1,88 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM ubuntu:14.04
+ADD . /apache-arrow
+WORKDIR /apache-arrow
+# Basic OS utilities
+RUN apt-get update && apt-get install -y \
+        wget \
+        git \
+        gcc \
+        g++
+# This will install conda in /home/ubuntu/miniconda
+RUN wget -O /tmp/miniconda.sh \
+    https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
+    rm /tmp/miniconda.sh
+# Create Conda environment
+ENV PATH="/home/ubuntu/miniconda/bin:${PATH}"
+RUN conda create -y -q -n test-environment \
+    python=3.6
+# Install dependencies
+RUN conda install -c conda-forge \
+    numpy \
+    pandas \
+    bcolz \
+    blosc \
+    bokeh \
+    boto3 \
+    chest \
+    cloudpickle \
+    coverage \
+    cytoolz \
+    distributed \
+    graphviz \
+    h5py \
+    ipython \
+    partd \
+    psutil \
+    "pytest<=3.1.1" \
+    scikit-image \
+    scikit-learn \
+    scipy \
+    sqlalchemy \
+    toolz
+# install pytables from defaults for now
+RUN conda install pytables
+
+RUN pip install -q git+https://github.com/dask/partd --upgrade --no-deps
+RUN pip install -q git+https://github.com/dask/zict --upgrade --no-deps
+RUN pip install -q git+https://github.com/dask/distributed --upgrade --no-deps
+RUN pip install -q git+https://github.com/mrocklin/sparse --upgrade --no-deps
+RUN pip install -q git+https://github.com/dask/s3fs --upgrade --no-deps
+
+RUN conda install -q -c conda-forge numba cython
+RUN pip install -q git+https://github.com/dask/fastparquet
+
+RUN pip install -q \
+    cachey \
+    graphviz \
+    moto \
+    pyarrow \
+    --upgrade --no-deps
+
+RUN pip install -q \
+    cityhash \
+    flake8 \
+    mmh3 \
+    pandas_datareader \
+    pytest-xdist \
+    xxhash \
+    pycodestyle
+
+CMD arrow/dev/dask_integration/dask_integration.sh
+
diff --git a/dev/dask_integration/dask_integration.sh b/dev/dask_integration/dask_integration.sh
new file mode 100755
index 0000000000000..f5a24e462b742
--- /dev/null
+++ b/dev/dask_integration/dask_integration.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set up environment and working directory
+cd /apache-arrow
+
+export ARROW_BUILD_TYPE=release
+export ARROW_HOME=$(pwd)/dist
+export PARQUET_HOME=$(pwd)/dist
+CONDA_BASE=/home/ubuntu/miniconda
+export LD_LIBRARY_PATH=$(pwd)/dist/lib:${CONDA_BASE}/lib:${LD_LIBRARY_PATH}
+
+# Allow for --user Python installation inside Docker
+export HOME=$(pwd)
+
+# Clean up and get the dask master branch from github
+rm -rf dask .local
+export GIT_COMMITTER_NAME="Nobody"
+export GIT_COMMITTER_EMAIL="nobody@nowhere.com"
+git clone https://github.com/dask/dask.git
+pushd dask
+pip install --user -e .[complete]
+# Verify integrity of the installed dask dataframe code
+py.test dask/dataframe/tests/test_dataframe.py
+popd
+
+# Run the integration test
+pushd arrow/python/testing
+py.test dask_tests
+popd
+
+pushd dask/dask/dataframe/io
+py.test tests/test_parquet.py
+popd
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
index 7bd2cd4412cec..4b9014894003b 100644
--- a/dev/docker-compose.yml
+++ b/dev/docker-compose.yml
@@ -28,3 +28,8 @@ services:
     - "4000:4000"
     volumes:
      - ../..:/apache-arrow
+  dask_integration:
+    build: 
+      context: dask_integration
+    volumes:
+     - ../..:/apache-arrow
diff --git a/dev/run_docker_compose.sh b/dev/run_docker_compose.sh
index f46879ed1e436..681a3a75ffe20 100755
--- a/dev/run_docker_compose.sh
+++ b/dev/run_docker_compose.sh
@@ -37,4 +37,4 @@ fi
 
 GID=$(id -g ${USERNAME})
 docker-compose -f arrow/dev/docker-compose.yml run \
-               -u "${UID}:${GID}" "${1}"
+               --rm -u "${UID}:${GID}" "${1}"
diff --git a/python/testing/README.md b/python/testing/README.md
index 07970a231b54b..0ebeec4a1c3e7 100644
--- a/python/testing/README.md
+++ b/python/testing/README.md
@@ -23,4 +23,26 @@
 
 ```shell
 ./test_hdfs.sh
-```
\ No newline at end of file
+```
+
+## Testing Dask integration
+
+Initial integration testing with Dask has been Dockerized.
+To invoke the test run the following command in the `arrow`
+root-directory:
+
+```shell
+bash dev/dask_integration.sh
+```
+
+This script will create a `dask` directory on the same level as
+`arrow`. It will clone the Dask project from Github into `dask`
+and do a Python `--user` install. The Docker code will use the parent
+directory of `arrow` as `$HOME` and that's where Python will
+install `dask` into a `.local` directory.
+
+The output of the Docker session will contain the results of tests
+of the Dask dataframe followed by the single integration test that
+now exists for Arrow. That test creates a set of `csv`-files and then
+does parallel reading of `csv`-files into a Dask dataframe. The code
+for this test resides here in the `dask_test` directory.
diff --git a/python/testing/dask_tests/test_dask_integration.py b/python/testing/dask_tests/test_dask_integration.py
new file mode 100644
index 0000000000000..e678348780cd8
--- /dev/null
+++ b/python/testing/dask_tests/test_dask_integration.py
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import date, timedelta
+import csv
+from random import randint
+import dask.dataframe as dd
+import pyarrow as pa
+
+def make_datafiles(tmpdir, prefix='data', num_files=20):
+    rowcount = 5000
+    fieldnames = ['date', 'temperature', 'dewpoint']
+    start_date = date(1900, 1, 1)
+    for i in range(num_files):
+        filename = '{0}/{1}-{2}.csv'.format(tmpdir, prefix, i)
+        with open(filename, 'w') as outcsv:
+            writer = csv.DictWriter(outcsv, fieldnames)
+            writer.writeheader()
+            the_date = start_date
+            for _ in range(rowcount):
+                temperature = randint(-10, 35)
+                dewpoint = temperature - randint(0, 10)
+                writer.writerow({'date': the_date, 'temperature': temperature,
+                                 'dewpoint': dewpoint})
+                the_date += timedelta(days=1)
+
+def test_dask_file_read(tmpdir):
+    prefix = 'data'
+    make_datafiles(tmpdir, prefix)
+    # Read all datafiles in parallel
+    datafiles = '{0}/{1}-*.csv'.format(tmpdir, prefix)
+    dask_df = dd.read_csv(datafiles)
+    # Convert Dask dataframe to Arrow table
+    table = pa.Table.from_pandas(dask_df.compute())
+    # Second column (1) is temperature
+    dask_temp = int(1000 * dask_df['temperature'].mean().compute())
+    arrow_temp = int(1000 * table[1].to_pandas().mean())
+    assert dask_temp == arrow_temp

From 0373541e2e9cd3510d6e4c8ac5b580f11eb675ec Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 3 Nov 2017 17:10:30 +0100
Subject: [PATCH 1209/1644] ARROW-1766: [GLib] Fix failing builds on OSX

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1279 from cpcloud/ARROW-1766 and squashes the following commits:

ed3b22b [Phillip Cloud] Only set ARCHFLAGS on osx
d9c1604 [Phillip Cloud] [GLib] Fix failing builds on OSX
---
 ci/travis_before_script_c_glib.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index a63945e1745d0..a22ecd3753fd8 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -100,7 +100,12 @@ if [ $BUILD_SYSTEM = "autotools" ]; then
 
   ./configure $CONFIGURE_OPTIONS
 
-  make -j4
+  if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+    ARCHFLAGS="-arch x86_64" make -j4
+  else
+    make -j4
+  fi
+
   make install
 else
   MESON_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"

From 527af63cde4ac1df53e82d6f8ced064738f5c1a3 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Fri, 3 Nov 2017 17:44:52 -0400
Subject: [PATCH 1210/1644] ARROW-1652: [JS] housekeeping, vector cleanup

This PR addresses the first few issues in the [JS roadmap doc](https://docs.google.com/document/d/142dek89oM2TVI2Yql106Zo8IB1Ff_9zDg_EG6jPWS0M) I sent out a week or so ago. Sorry for the big PR, the housekeeping and vector cleanup work were pretty co-dependent.

JIRA issues addressed by this PR:
[ARROW-1032](https://issues.apache.org/jira/browse/ARROW-1032) - Support custom_metadata
[ARROW-1651](https://issues.apache.org/jira/browse/ARROW-1651) - Lazy row accessor in Table
[ARROW-1652](https://issues.apache.org/jira/browse/ARROW-1652) - Separate Vector into BatchVector and CompositeVector

Tasks from the roadmap (some not in JIRA):

##### Housekeeping
1. Enable the strict-mode tsc compiler settings in the build
2. Compile mjs files for node 8.x ESModules
3. Compile ES6 UMD target with native iterators/generators

##### Vector
1. Refactor Vector types to primitive forms representing the portion of a column in a single RecordBatch
2. Add Column Vector that represents primitive Vectors across RecordBatches as an entire column
3. Refactor linear column-to-batch-index lookup in `Vector.get(i)`
4. Simplify inheritance hierarchy/generic types with Traits (e.g. Nullable, Iterable, and Typed numeric variants)

##### Table
1. Implement lazy row accessor
2. Share API/row logic with StructVector

cc: @wesm @TheNeuralBit

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1273 from trxcllnt/vector-cleanup and squashes the following commits:

c53d6de3 [Paul Taylor] refactor: rename vector mixins
2c83c823 [Paul Taylor] update to typescript@2.6.1
48c6ca48 [Paul Taylor] refactor: StructVector/Table#get always take numeric index, Table extends StructVector
18671edc [Paul Taylor] fix lint
04e9941d [Paul Taylor] refactor: use new compilation targets in perf tests
bd7a8373 [Paul Taylor] refactor: update test's Arrow imports for new types
37b7f615 [Paul Taylor] refactor: update vector tests for new types
15ab8d4c [Paul Taylor] refactor: update table tests for new types
db04a0b0 [Paul Taylor] refactor: export new Arrow types
84233dec [Paul Taylor] refactor reader to use new arrow types, fix strict TS compilation errors
af4845d6 [Paul Taylor] refactor: add Arrow vector mixins
54fa2fd3 [Paul Taylor] refactor: break out virtual vector, move to types folder
2121bf1a [Paul Taylor] refactor: break out table, add Row type, move to types folder
abc93310 [Paul Taylor] refactor: move struct to types folder
2a4127ce [Paul Taylor] refactor: move dictionary to types folder
607be424 [Paul Taylor] refactor: break out list/fixedsizelist/utf8, move to types folder
b8a68665 [Paul Taylor] refactor: break out Typed vectors, move into types folder
0f8de75a [Paul Taylor] refactor: rename vector folder to types, move vector base class
d2def198 [Paul Taylor] clean up build scripts, add ES2015 UMD and mjs targets
84b2c505 [Paul Taylor] use strict typescript compiler settings
---
 js/.gitignore                                 |   9 +-
 js/closure-compiler-scripts/text-encoding.js  |   2 +-
 js/gulp/argv.js                               |  36 ++
 js/gulp/arrow-task.js                         |  57 +++
 js/gulp/build-task.js                         |  35 ++
 js/gulp/clean-task.js                         |  31 ++
 js/gulp/closure-task.js                       |  91 +++++
 js/gulp/memoize-task.js                       |  30 ++
 js/gulp/package-task.js                       |  75 ++++
 js/gulp/test-task.js                          |  44 +++
 js/gulp/typescript-task.js                    |  43 +++
 js/gulp/uglify-task.js                        | 113 ++++++
 js/gulp/util.js                               | 152 ++++++++
 js/gulpfile.js                                | 350 ++++--------------
 js/package.json                               |  39 +-
 js/perf/index.js                              |  10 +-
 js/src/Arrow.externs.ts                       |  45 ++-
 js/src/Arrow.ts                               |  73 ++--
 js/src/reader/arrow.ts                        |  28 +-
 js/src/reader/dictionary.ts                   |  16 +-
 js/src/reader/file.ts                         |   9 +-
 js/src/reader/message.ts                      |   8 +-
 js/src/reader/stream.ts                       |   4 +-
 js/src/reader/vector.ts                       | 312 +++++++++-------
 js/src/table.ts                               | 143 -------
 js/src/text-encoding-utf-8.d.ts               |   4 +
 js/src/types/arrow.ts                         |  88 +++++
 js/src/types/dictionary.ts                    |  58 +++
 .../struct.ts => types/fixedsizelist.ts}      |  26 +-
 js/src/types/list.ts                          |  35 ++
 js/src/types/table/from.ts                    |  34 ++
 js/src/types/table/row.ts                     |  61 +++
 js/src/types/table/struct.ts                  |  63 ++++
 js/src/types/table/table.ts                   |  30 ++
 js/src/types/table/toString.ts                |  40 ++
 js/src/types/types.ts                         |  98 +++++
 .../{vector/dictionary.ts => types/utf8.ts}   |  41 +-
 js/src/types/vector/bool.ts                   |  55 +++
 js/src/types/vector/date.ts                   |  29 ++
 js/src/types/vector/long.ts                   |  35 ++
 js/src/types/vector/traits.ts                 |  69 ++++
 js/src/types/vector/typed.ts                  |  57 +++
 js/src/types/vector/virtual.ts                | 129 +++++++
 js/src/vector/list.ts                         | 108 ------
 js/src/vector/typed.ts                        | 326 ----------------
 js/src/vector/vector.ts                       |  91 -----
 js/test/Arrow.ts                              |  30 +-
 js/test/__snapshots__/table-tests.ts.snap     | 200 +++++-----
 js/test/table-tests.ts                        |  17 +-
 js/test/test-config.ts                        |   2 +-
 js/test/vector-tests.ts                       |  75 ++--
 js/tsconfig/tsconfig.base.json                |  30 +-
 js/tsconfig/tsconfig.es2015.cls.json          |   3 +-
 js/tsconfig/tsconfig.es5.cls.json             |   1 +
 js/tsconfig/tsconfig.esnext.cls.json          |   3 +-
 55 files changed, 2195 insertions(+), 1398 deletions(-)
 create mode 100644 js/gulp/argv.js
 create mode 100644 js/gulp/arrow-task.js
 create mode 100644 js/gulp/build-task.js
 create mode 100644 js/gulp/clean-task.js
 create mode 100644 js/gulp/closure-task.js
 create mode 100644 js/gulp/memoize-task.js
 create mode 100644 js/gulp/package-task.js
 create mode 100644 js/gulp/test-task.js
 create mode 100644 js/gulp/typescript-task.js
 create mode 100644 js/gulp/uglify-task.js
 create mode 100644 js/gulp/util.js
 delete mode 100644 js/src/table.ts
 create mode 100644 js/src/text-encoding-utf-8.d.ts
 create mode 100644 js/src/types/arrow.ts
 create mode 100644 js/src/types/dictionary.ts
 rename js/src/{vector/struct.ts => types/fixedsizelist.ts} (53%)
 create mode 100644 js/src/types/list.ts
 create mode 100644 js/src/types/table/from.ts
 create mode 100644 js/src/types/table/row.ts
 create mode 100644 js/src/types/table/struct.ts
 create mode 100644 js/src/types/table/table.ts
 create mode 100644 js/src/types/table/toString.ts
 create mode 100644 js/src/types/types.ts
 rename js/src/{vector/dictionary.ts => types/utf8.ts} (50%)
 create mode 100644 js/src/types/vector/bool.ts
 create mode 100644 js/src/types/vector/date.ts
 create mode 100644 js/src/types/vector/long.ts
 create mode 100644 js/src/types/vector/traits.ts
 create mode 100644 js/src/types/vector/typed.ts
 create mode 100644 js/src/types/vector/virtual.ts
 delete mode 100644 js/src/vector/list.ts
 delete mode 100644 js/src/vector/typed.ts
 delete mode 100644 js/src/vector/vector.ts

diff --git a/js/.gitignore b/js/.gitignore
index 6d0f88d191cb0..88c612d8faf37 100644
--- a/js/.gitignore
+++ b/js/.gitignore
@@ -18,6 +18,7 @@
 # Logs
 logs
 *.log
+.esm-cache
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
@@ -57,10 +58,6 @@ build/Release
 node_modules/
 jspm_packages/
 
-# Typescript declaration files
-types/
-typings/
-
 # Optional npm cache directory
 .npm
 
@@ -85,6 +82,4 @@ package-lock.json
 
 # compilation targets
 dist
-targets/es5
-targets/es2015
-targets/esnext
+targets
diff --git a/js/closure-compiler-scripts/text-encoding.js b/js/closure-compiler-scripts/text-encoding.js
index ca9154f88ecba..398883ab9b4be 100644
--- a/js/closure-compiler-scripts/text-encoding.js
+++ b/js/closure-compiler-scripts/text-encoding.js
@@ -11,7 +11,7 @@
 // Utilities
 //
 
-goog.module("module$text_encoding");
+goog.module("module$text_encoding_utf_8");
 goog.module.declareLegacyNamespace();
 /**
  * @param {number} a The number to test.
diff --git a/js/gulp/argv.js b/js/gulp/argv.js
new file mode 100644
index 0000000000000..33553704eec25
--- /dev/null
+++ b/js/gulp/argv.js
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const argv = require(`command-line-args`)([
+    { name: `all`, alias: `a`, type: Boolean },
+    { name: 'update', alias: 'u', type: Boolean },
+    { name: 'verbose', alias: 'v', type: Boolean },
+    { name: `target`, type: String, defaultValue: `` },
+    { name: `module`, type: String, defaultValue: `` },
+    { name: `coverage`, type: Boolean, defaultValue: false },
+    { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
+    { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }
+]);
+
+const { targets, modules } = argv;
+
+argv.target && !targets.length && targets.push(argv.target);
+argv.module && !modules.length && modules.push(argv.module);
+(argv.all || !targets.length) && targets.push(`all`);
+(argv.all || !modules.length) && modules.push(`all`);
+
+module.exports = { argv, targets, modules };
\ No newline at end of file
diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js
new file mode 100644
index 0000000000000..d160ecb0e5de4
--- /dev/null
+++ b/js/gulp/arrow-task.js
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const {
+    mainExport, gCCLanguageNames,
+    targetDir, observableFromStreams
+} = require('./util');
+
+const gulp = require('gulp');
+const path = require('path');
+const gulpRename = require(`gulp-rename`);
+const { memoizeTask } = require('./memoize-task');
+const { Observable, ReplaySubject } = require('rxjs');
+
+const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, format) {
+    const out = targetDir(target), srcGlob = `src/**/*`;
+    const es5Glob = `${targetDir(`es5`, `cjs`)}/**/*.js`;
+    const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`;
+    const es5UmdGlob = `${targetDir(`es5`, `umd`)}/**/*.js`;
+    const es5UmdMaps = `${targetDir(`es5`, `umd`)}/**/*.map`;
+    const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/**/*.js`;
+    const es2015UmdMaps = `${targetDir(`es2015`, `umd`)}/**/*.map`;
+    const ch_ext = (ext) => gulpRename((p) => { p.extname = ext; });
+    const append = (ap) => gulpRename((p) => { p.basename += ap; });
+    return Observable.forkJoin(
+      observableFromStreams(gulp.src(srcGlob), gulp.dest(out)), // copy src ts files
+      observableFromStreams(gulp.src(es5Glob), gulp.dest(out)), // copy es5 cjs files
+      observableFromStreams(gulp.src(esmGlob), ch_ext(`.mjs`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs`
+      observableFromStreams(gulp.src(es5UmdGlob), append(`.es5.min`), gulp.dest(out)), // copy es5 umd files and add `.min`
+      observableFromStreams(gulp.src(es5UmdMaps),                     gulp.dest(out)), // copy es5 umd sourcemap files, but don't rename
+      observableFromStreams(gulp.src(es2015UmdGlob), append(`.es2015.min`), gulp.dest(out)), // copy es2015 umd files and add `.es6.min`
+      observableFromStreams(gulp.src(es2015UmdMaps),                        gulp.dest(out)), // copy es2015 umd sourcemap files, but don't rename
+    ).publish(new ReplaySubject()).refCount();
+}))({});
+
+const arrowTSTask = ((cache) => memoizeTask(cache, function copyTS(target, format) {
+    return observableFromStreams(gulp.src(`src/**/*`), gulp.dest(targetDir(target, format)));
+}))({});
+  
+  
+module.exports = arrowTask;
+module.exports.arrowTask = arrowTask;
+module.exports.arrowTSTask = arrowTSTask;
\ No newline at end of file
diff --git a/js/gulp/build-task.js b/js/gulp/build-task.js
new file mode 100644
index 0000000000000..01152e662fcec
--- /dev/null
+++ b/js/gulp/build-task.js
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const { npmPkgName } = require('./util');
+const { memoizeTask } = require('./memoize-task');
+
+const uglifyTask = require('./uglify-task');
+const closureTask = require('./closure-task');
+const typescriptTask = require('./typescript-task');
+const { arrowTask, arrowTSTask } = require('./arrow-task');
+
+const buildTask = ((cache) => memoizeTask(cache, function build(target, format, ...args) {
+    return target === npmPkgName               ? arrowTask(target, format, ...args)()
+         : target === `ts`                     ? arrowTSTask(target, format, ...args)()
+         : format === `umd` ? target === `es5` ? closureTask(target, format, ...args)()
+                                               : uglifyTask(target, format, ...args)()
+                                               : typescriptTask(target, format, ...args)();
+}))({});
+
+module.exports = buildTask;
+module.exports.buildTask = buildTask;
\ No newline at end of file
diff --git a/js/gulp/clean-task.js b/js/gulp/clean-task.js
new file mode 100644
index 0000000000000..d6c90f4637c8b
--- /dev/null
+++ b/js/gulp/clean-task.js
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const del = require('del');
+const { targetDir } = require('./util');
+const { memoizeTask } = require('./memoize-task');
+const { Observable, ReplaySubject } = require('rxjs');
+
+const cleanTask = ((cache) => memoizeTask(cache, function clean(target, format) {
+    return Observable
+        .from(del(`${targetDir(target, format)}/**`))
+        .catch((e) => Observable.empty())
+        .multicast(new ReplaySubject()).refCount();
+}))({});
+
+module.exports = cleanTask;
+module.exports.cleanTask = cleanTask;
\ No newline at end of file
diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js
new file mode 100644
index 0000000000000..950bf40e22a39
--- /dev/null
+++ b/js/gulp/closure-task.js
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const {
+    targetDir,
+    mainExport,
+    gCCLanguageNames,
+    UMDSourceTargets,
+    observableFromStreams
+} = require('./util');
+
+const gulp = require('gulp');
+const path = require('path');
+const sourcemaps = require('gulp-sourcemaps');
+const { memoizeTask } = require('./memoize-task');
+const { Observable, ReplaySubject } = require('rxjs');
+const closureCompiler = require('google-closure-compiler').gulp();
+
+const closureTask = ((cache) => memoizeTask(cache, function closure(target, format) {
+    const src = targetDir(target, `cls`);
+    const out = targetDir(target, format);
+    const entry = path.join(src, mainExport);
+    const externs = path.join(src, `${mainExport}.externs`);
+    return observableFromStreams(
+        gulp.src([
+/*   external libs first --> */ `closure-compiler-scripts/*.js`,
+/*    then sources glob --> */ `${src}/**/*.js`,
+/* and exclusions last --> */ `!${src}/format/*.js`,
+                              `!${src}/Arrow.externs.js`,
+        ], { base: `./` }),
+        sourcemaps.init(),
+        closureCompiler(createClosureArgs(entry, externs)),
+        // rename the sourcemaps from *.js.map files to *.min.js.map
+        sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }),
+        gulp.dest(out)
+    ).publish(new ReplaySubject()).refCount();
+}))({});
+
+const createClosureArgs = (entry, externs) => ({
+    third_party: true,
+    warning_level: `QUIET`,
+    dependency_mode: `LOOSE`,
+    rewrite_polyfills: false,
+    externs: `${externs}.js`,
+    entry_point: `${entry}.js`,
+    // formatting: `PRETTY_PRINT`,
+    compilation_level: `ADVANCED`,
+    assume_function_wrapper: true,
+    js_output_file: `${mainExport}.js`,
+    language_in: gCCLanguageNames[`es2015`],
+    language_out: gCCLanguageNames[`es5`],
+    output_wrapper:
+`// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+(function (global, factory) {
+    typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :
+    typeof define === 'function' && define.amd ? define(['exports'], factory) :
+    (factory(global.Arrow = global.Arrow || {}));
+}(this, (function (exports) {%output%}.bind(this))));`
+});
+
+module.exports = closureTask;
+module.exports.closureTask = closureTask;
diff --git a/js/gulp/memoize-task.js b/js/gulp/memoize-task.js
new file mode 100644
index 0000000000000..0b0fc843c451a
--- /dev/null
+++ b/js/gulp/memoize-task.js
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const { taskName } = require('./util');
+
+const memoizeTask = ((cache, taskFn) => ((target, format, ...args) => {
+    // Give the memoized fn a displayName so gulp's output is easier to follow.
+    const fn = () => (
+      cache[taskName(target, format)] || (
+      cache[taskName(target, format)] = taskFn(target, format, ...args)));
+    fn.displayName = `${taskFn.name || ``}:${taskName(target, format, ...args)}:task`;
+    return fn;
+}));
+
+module.exports = memoizeTask;
+module.exports.memoizeTask = memoizeTask;
\ No newline at end of file
diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
new file mode 100644
index 0000000000000..7b4b15a33e6ef
--- /dev/null
+++ b/js/gulp/package-task.js
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const {
+    metadataFiles, packageJSONFields,
+    mainExport, npmPkgName, npmOrgName,
+    targetDir, packageName, observableFromStreams
+} = require('./util');
+
+const gulp = require('gulp');
+const { memoizeTask } = require('./memoize-task');
+const { Observable, ReplaySubject } = require('rxjs');
+const gulpJsonTransform = require('gulp-json-transform');
+
+const packageTask = ((cache) => memoizeTask(cache, function bundle(target, format) {
+    const out = targetDir(target, format);
+    const jsonTransform = gulpJsonTransform(target === npmPkgName ? createMainPackageJson(target, format) :
+                                            target === `ts`       ? createTypeScriptPackageJson(target, format)
+                                                                  : createScopedPackageJSON(target, format),
+                                            2);
+    return Observable.forkJoin(
+      observableFromStreams(gulp.src(metadataFiles), gulp.dest(out)), // copy metadata files
+      observableFromStreams(gulp.src(`package.json`), jsonTransform, gulp.dest(out)) // write packageJSONs
+    ).publish(new ReplaySubject()).refCount();
+}))({});
+
+module.exports = packageTask;
+module.exports.packageTask = packageTask;
+
+const createMainPackageJson = (target, format) => (orig) => ({
+    ...createTypeScriptPackageJson(target, format)(orig),
+    name: npmPkgName,
+    main: mainExport,
+    module: `${mainExport}.mjs`,
+    browser: `${mainExport}.es5.min.js`,
+    [`browser:es2015`]: `${mainExport}.es2015.min.js`,
+    [`@std/esm`]: { esm: `mjs` },
+});
+  
+const createTypeScriptPackageJson = (target, format) => (orig) => ({
+    ...createScopedPackageJSON(target, format)(orig),
+    main: `${mainExport}.ts`, types: `${mainExport}.ts`
+});
+  
+const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
+    conditionallyAddStandardESMEntry(target, format)(
+      packageJSONFields.reduce(
+        (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }),
+        { name: `${npmOrgName}/${packageName(target, format)}`,
+          version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`,
+          browser: undefined, [`browser:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined }
+      )
+    )
+);
+  
+const conditionallyAddStandardESMEntry = (target, format) => (packageJSON) => (
+    format !== `esm`
+      ? packageJSON
+      : { ...packageJSON, [`@std/esm`]: { esm: `js` } }
+);
+  
\ No newline at end of file
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
new file mode 100644
index 0000000000000..b46b2bb14edcb
--- /dev/null
+++ b/js/gulp/test-task.js
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const path = require('path');
+const child_process = require(`child_process`);
+const { argv } = require('./argv');
+const { memoizeTask } = require('./memoize-task');
+
+const jestArgv = [];
+argv.update && jestArgv.push(`-u`);
+argv.verbose && jestArgv.push(`--verbose`);
+argv.coverage && jestArgv.push(`--coverage`);
+
+const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`];
+const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`));
+
+const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) {
+    const opts = Object.assign({}, testOptions);
+    const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv];
+    opts.env = Object.assign({}, opts.env, { TEST_TARGET: target, TEST_MODULE: format });
+    return !debug ?
+        child_process.spawn(jest, args, opts) :
+        child_process.exec(`node --inspect-brk ${jest} ${args.join(` `)}`, opts);
+}))({}, jestArgv, {
+    env: Object.assign({}, process.env),
+    stdio: [`ignore`, `inherit`, `inherit`],
+});
+
+module.exports = testTask;
+module.exports.testTask = testTask;
diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js
new file mode 100644
index 0000000000000..2c66846667cd3
--- /dev/null
+++ b/js/gulp/typescript-task.js
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const {
+    targetDir, tsconfigName, observableFromStreams
+} = require('./util');
+
+const gulp = require('gulp');
+const path = require('path');
+const ts = require(`gulp-typescript`);
+const sourcemaps = require('gulp-sourcemaps');
+const { memoizeTask } = require('./memoize-task');
+const { Observable, ReplaySubject } = require('rxjs');
+
+const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target, format) {
+    const out = targetDir(target, format);
+    const tsconfigFile = `tsconfig.${tsconfigName(target, format)}.json`;
+    const tsProject = ts.createProject(path.join(`tsconfig`, tsconfigFile), { typescript: require(`typescript`) });
+    const { stream: { js, dts } } = observableFromStreams(
+      tsProject.src(), sourcemaps.init(),
+      tsProject(ts.reporter.fullReporter(true))
+    );
+    const writeDTypes = observableFromStreams(dts, gulp.dest(out));
+    const writeJS = observableFromStreams(js, sourcemaps.write(), gulp.dest(out));
+    return Observable.forkJoin(writeDTypes, writeJS).publish(new ReplaySubject()).refCount();
+}))({});
+  
+module.exports = typescriptTask;
+module.exports.typescriptTask = typescriptTask;
\ No newline at end of file
diff --git a/js/gulp/uglify-task.js b/js/gulp/uglify-task.js
new file mode 100644
index 0000000000000..804d450453644
--- /dev/null
+++ b/js/gulp/uglify-task.js
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const {
+    targetDir,
+    mainExport,
+    ESKeywords,
+    UMDSourceTargets,
+    uglifyLanguageNames,
+    observableFromStreams
+} = require('./util');
+
+const path = require('path');
+const webpack = require(`webpack`);
+const { memoizeTask } = require('./memoize-task');
+const { Observable, ReplaySubject } = require('rxjs');
+const UglifyJSPlugin = require(`uglifyjs-webpack-plugin`);
+const esmRequire = require(`@std/esm`)(module, { cjs: true, esm: `js` });
+
+const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJS(target, format) {
+
+    const sourceTarget = UMDSourceTargets[target];
+    const PublicNames = reservePublicNames(sourceTarget, `cls`);
+    const out = targetDir(target, format), src = targetDir(sourceTarget, `cls`);
+
+    const targetConfig = { ...commonConfig,
+        output: { ...commonConfig.output,
+             path: path.resolve(`./${out}`) } };
+
+    const webpackConfigs = [
+        [mainExport, PublicNames]
+    ].map(([entry, reserved]) => ({
+        ...targetConfig,
+        name: entry,
+        entry: { [entry]: path.resolve(`${src}/${entry}.js`) },
+        plugins: [
+            ...(targetConfig.plugins || []),
+            new webpack.SourceMapDevToolPlugin({
+                filename: `[name].${target}.min.js.map`,
+                moduleFilenameTemplate: ({ resourcePath }) =>
+                    resourcePath
+                        .replace(/\s/, `_`)
+                        .replace(/\.\/node_modules\//, ``)
+            }),
+            new UglifyJSPlugin({
+                sourceMap: true,
+                uglifyOptions: {
+                    ecma: uglifyLanguageNames[target],
+                    compress: { unsafe: true },
+                    output: { comments: false, beautify: false },
+                    mangle: { eval: true, safari10: true, // <-- Works around a Safari 10 bug: // https://github.com/mishoo/UglifyJS2/issues/1753
+                        properties: { reserved, keep_quoted: true }
+                    }
+                },
+            })
+        ]
+    }));
+
+    const compilers = webpack(webpackConfigs);
+    return Observable
+            .bindNodeCallback(compilers.run.bind(compilers))()
+            .multicast(new ReplaySubject()).refCount();
+}))({}, {
+    resolve: { mainFields: [`module`, `main`] },
+    module: { rules: [{ test: /\.js$/, enforce: `pre`, use: [`source-map-loader`] }] },
+    output: { filename: '[name].js', library: mainExport, libraryTarget: `umd`, umdNamedDefine: true },
+});
+
+module.exports = uglifyTask;
+module.exports.uglifyTask = uglifyTask;
+
+const reservePublicNames = ((ESKeywords) => function reservePublicNames(target, format) {
+    const publicModulePath = `../${targetDir(target, format)}/${mainExport}.js`;
+    return [
+        ...ESKeywords,
+        ...reserveExportedNames(esmRequire(publicModulePath))
+    ];
+})(ESKeywords);
+
+// Reflect on the Arrow modules to come up with a list of keys to save from Uglify's
+// mangler. Assume all the non-inherited static and prototype members of the Arrow
+// module and its direct exports are public, and should be preserved through minification.
+const reserveExportedNames = (entryModule) => (
+    Object
+        .getOwnPropertyNames(entryModule)
+        .filter((name) => (
+            typeof entryModule[name] === `object` ||
+            typeof entryModule[name] === `function`
+        ))
+        .map((name) => [name, entryModule[name]])
+        .reduce((reserved, [name, value]) => {
+            const fn = function() {};
+            const ownKeys = value && Object.getOwnPropertyNames(value) || [];
+            const protoKeys = typeof value === `function` && Object.getOwnPropertyNames(value.prototype) || [];
+            const publicNames = [...ownKeys, ...protoKeys].filter((x) => x !== `default` && x !== `undefined` && !(x in fn));
+            return [...reserved, name, ...publicNames];
+        }, []
+    )
+);
diff --git a/js/gulp/util.js b/js/gulp/util.js
new file mode 100644
index 0000000000000..21ffc3127339c
--- /dev/null
+++ b/js/gulp/util.js
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const path = require(`path`);
+const pump = require(`pump`);
+const { Observable, ReplaySubject } = require('rxjs');
+
+const mainExport = `Arrow`;
+const npmPkgName = `apache-arrow`;
+const npmOrgName = `@${npmPkgName}`;
+
+const releasesRootDir = `targets`;
+const knownTargets = [`es5`, `es2015`, `esnext`];
+const knownModules = [`cjs`, `esm`, `cls`, `umd`];
+const moduleFormatsToSkipCombosOf = { cls: true };
+const metadataFiles = [`LICENSE`, `README.md`];
+const packageJSONFields = [
+  `version`, `license`, `description`,
+  `author`, `homepage`, `repository`,
+  `bugs`, `keywords`,  `dependencies`
+];
+
+// see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988
+const gCCLanguageNames = {
+    es5: `ECMASCRIPT5`,
+ es2015: `ECMASCRIPT_2015`,
+ es2016: `ECMASCRIPT_2016`,
+ es2017: `ECMASCRIPT_2017`,
+ esnext: `ECMASCRIPT_NEXT`
+};
+
+const UMDSourceTargets = {
+    es5: `es5`,
+ es2015: `es2015`,
+ es2016: `es2015`,
+ es2017: `es2015`,
+ esnext: `es2015`
+};
+
+const uglifyLanguageNames = {
+    es5: 5, es2015: 6,
+ es2016: 7, es2017: 8,
+ esnext: 8 // <--- ?
+};
+
+// ES7+ keywords Uglify shouldn't mangle
+// Hardcoded here since some are from ES7+, others are
+// only defined in interfaces, so difficult to get by reflection.
+const ESKeywords = [
+    // PropertyDescriptors
+    `configurable`, `enumerable`,
+    // IteratorResult, Symbol.asyncIterator
+    `done`, `value`, `Symbol.asyncIterator`, `asyncIterator`,
+    // AsyncObserver
+    `values`, `hasError`, `hasCompleted`,`errorValue`, `closed`,
+    // Observable/Subscription/Scheduler
+    `next`, `error`, `complete`, `subscribe`, `unsubscribe`, `isUnsubscribed`,
+    // EventTarget
+    `addListener`, `removeListener`, `addEventListener`, `removeEventListener`,
+    // Arrow properties
+    `low`, `high`, `data`, `index`, `field`, `validity`, `columns`, `fieldNode`, `subarray`,
+];
+
+function taskName(target, format) {
+    return !format ? target : `${target}:${format}`;
+}
+
+function packageName(target, format) {
+    return !format ? target : `${target}-${format}`;
+}
+
+function tsconfigName(target, format) {
+    return !format ? target : `${target}.${format}`;
+}
+
+function targetDir(target, format) {
+    return path.join(releasesRootDir, ...(!format ? [target] : [target, format]));
+}
+
+function logAndDie(e) {
+    if (e) {
+        console.error(e);
+        process.exit(1);
+    }
+}
+
+function observableFromStreams(...streams) {
+    const pumped = streams.length <= 1 ? streams[0]
+        : pump(...streams, logAndDie);
+    const fromEvent = Observable.fromEvent.bind(null, pumped);
+    const streamObs = fromEvent(`data`)
+               .merge(fromEvent(`error`).flatMap((e) => Observable.throw(e)))
+           .takeUntil(fromEvent(`end`).merge(fromEvent(`close`)))
+           .defaultIfEmpty(`empty stream`)
+           .multicast(new ReplaySubject()).refCount();
+    streamObs.stream = pumped;
+    streamObs.observable = streamObs;
+    return streamObs;
+}
+
+function* combinations(_targets, _modules) {
+
+    const targets = known(knownTargets, _targets || [`all`]);
+    const modules = known(knownModules, _modules || [`all`]);
+
+    if (_targets[0] === `all` && _modules[0] === `all`) {
+        yield [`ts`, ``];
+        yield [npmPkgName, ``];
+    }        
+    
+    for (const format of modules) {
+        for (const target of targets) {
+            yield [target, format];
+        }
+    }
+
+    function known(known, values) {
+        return ~values.indexOf(`all`)
+            ? known
+            : Object.keys(
+                values.reduce((map, arg) => ((
+                    (known.indexOf(arg) !== -1) &&
+                    (map[arg.toLowerCase()] = true)
+                    || true) && map
+                ), {})
+            ).sort((a, b) => known.indexOf(a) - known.indexOf(b));
+    }
+}
+    
+module.exports = {
+
+    mainExport, npmPkgName, npmOrgName, metadataFiles, packageJSONFields,
+
+    knownTargets, knownModules, moduleFormatsToSkipCombosOf,
+    ESKeywords, gCCLanguageNames, UMDSourceTargets, uglifyLanguageNames,
+
+    taskName, packageName, tsconfigName, targetDir, combinations, observableFromStreams,
+};
\ No newline at end of file
diff --git a/js/gulpfile.js b/js/gulpfile.js
index 9f8e564bd9e3a..4cf0342c3be78 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -15,278 +15,92 @@
 // specific language governing permissions and limitations
 // under the License.
 
-const del = require(`del`);
-const gulp = require(`gulp`);
-const path = require(`path`);
-const pump = require(`pump`);
-const ts = require(`gulp-typescript`);
-const streamMerge = require(`merge2`);
-const sourcemaps = require(`gulp-sourcemaps`);
-const child_process = require(`child_process`);
-const gulpJsonTransform = require(`gulp-json-transform`);
-const closureCompiler = require(`google-closure-compiler`).gulp();
-
-const knownTargets = [`es5`, `es2015`, `esnext`];
-const knownModules = [`cjs`, `esm`, `cls`, `umd`];
-
-// see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988
-const gCCTargets = {
-    es5: `ECMASCRIPT5`,
-    es2015: `ECMASCRIPT_2015`,
-    es2016: `ECMASCRIPT_2016`,
-    es2017: `ECMASCRIPT_2017`,
-    esnext: `ECMASCRIPT_NEXT`
-};
-
-const tsProjects = [];
-const argv = require(`command-line-args`)([
-    { name: `all`, alias: `a`, type: Boolean },
-    { name: 'update', alias: 'u', type: Boolean },
-    { name: 'verbose', alias: 'v', type: Boolean },
-    { name: `target`, type: String, defaultValue: `` },
-    { name: `module`, type: String, defaultValue: `` },
-    { name: `coverage`, type: Boolean, defaultValue: false },
-    { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
-    { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }
-]);
-
-const { targets, modules } = argv;
-
-argv.target && !targets.length && targets.push(argv.target);
-argv.module && !modules.length && modules.push(argv.module);
-(argv.all || !targets.length) && targets.push(`all`);
-(argv.all || !modules.length) && modules.push(`all`);
-
-for (const [target, format] of combinations([`all`, `all`])) {
-    const combo = `${target}:${format}`;
-    gulp.task(`test:${combo}`, gulp.series(testTask(target, format, combo, `targets/${target}/${format}`)));
-    gulp.task(`clean:${combo}`, gulp.series(cleanTask(target, format, combo, `targets/${target}/${format}`)));
-    gulp.task(`build:${combo}`, gulp.series(buildTask(target, format, combo, `targets/${target}/${format}`)));
-    gulp.task(`bundle:${combo}`, gulp.series(bundleTask(target, format, combo, `targets/${target}/${format}`)));
-    gulp.task(`package:${combo}`, gulp.series(packageTask(target, format, combo, `targets/${target}/${format}`)));
-    gulp.task(`test:debug:${combo}`, gulp.series(testTask(target, format, combo, `targets/${target}/${format}`, true)));
+const del = require('del');
+const gulp = require('gulp');
+const path = require('path');
+const { Observable } = require('rxjs');
+const testsTask = require('./gulp/test-task');
+const buildTask = require('./gulp/build-task');
+const cleanTask = require('./gulp/clean-task');
+const packageTask = require('./gulp/package-task');
+const { targets, modules } = require('./gulp/argv');
+const {
+    targetDir,
+    taskName, combinations,
+    knownTargets, knownModules,
+    npmPkgName, UMDSourceTargets,
+    moduleFormatsToSkipCombosOf
+} = require('./gulp/util');
+
+for (const [target, format] of combinations([`all`], [`all`])) {
+    const task = taskName(target, format);
+    gulp.task(`clean:${task}`, cleanTask(target, format));
+    gulp.task( `test:${task}`, testsTask(target, format));
+    gulp.task(`debug:${task}`, testsTask(target, format, true));
+    gulp.task(`build:${task}`, gulp.series(`clean:${task}`,
+                                            buildTask(target, format),
+                                            packageTask(target, format)));
 }
 
-gulp.task(`test`, gulp.series(runTaskCombos(`test`)));
-gulp.task(`clean`, gulp.parallel(runTaskCombos(`clean`)));
-gulp.task(`build`, gulp.parallel(runTaskCombos(`build`)));
-gulp.task(`bundle`, gulp.parallel(runTaskCombos(`bundle`)));
-gulp.task(`package`, gulp.parallel(runTaskCombos(`package`)));
-gulp.task(`test:debug`, gulp.series(runTaskCombos(`test:debug`)));
-gulp.task(`default`, gulp.task(`package`));
-
-function runTaskCombos(name) {
-    const combos = [];
+// The UMD bundles build temporary es5/6/next targets via TS,
+// then run the TS source through either closure-compiler or
+// uglify, so we special case that here.
+knownTargets.forEach((target) =>
+    gulp.task(`build:${target}:umd`,
+        gulp.series(
+            gulp.parallel(
+                cleanTask(target, `umd`),
+                cleanTask(UMDSourceTargets[target], `cls`),
+            ),
+            buildTask(UMDSourceTargets[target], `cls`),
+            buildTask(target, `umd`), packageTask(target, `umd`)
+        )
+    )
+);
+
+// The main "apache-arrow" module builds the es5/cjs, es5/umd,
+// es2015/esm, es2015/umd, and ts targets, then copies and
+// renames the compiled output into the apache-arrow folder
+gulp.task(`build:${npmPkgName}`,
+    gulp.series(
+        cleanTask(npmPkgName),
+        gulp.parallel(
+            `build:${taskName(`es5`, `cjs`)}`,
+            `build:${taskName(`es5`, `umd`)}`,
+            `build:${taskName(`es2015`, `esm`)}`,
+            `build:${taskName(`es2015`, `umd`)}`
+        ),
+        buildTask(npmPkgName), packageTask(npmPkgName)
+    )
+);
+
+
+function gulpConcurrent(tasks) {
+    return () => Observable.bindCallback((tasks, cb) => gulp.parallel(tasks)(cb))(tasks);
+}
+  
+const buildConcurrent = (tasks) => () =>
+    gulpConcurrent(tasks)()
+        .concat(Observable
+            .defer(() => Observable
+            .merge(...knownTargets.map((target) =>
+                del(`${targetDir(target, `cls`)}/**`)))));
+  
+gulp.task( `test`, gulp.series(getTasks(`test`)));
+gulp.task(`debug`, gulp.series(getTasks(`debug`)));
+gulp.task(`clean`, gulp.parallel(getTasks(`clean`)));
+gulp.task(`build`, buildConcurrent(getTasks(`build`)));
+gulp.task(`default`,  gulp.series(`build`, `test`));
+  
+function getTasks(name) {
+    const tasks = [];
+    if (targets.indexOf(`ts`) !== -1) tasks.push(`${name}:ts`);
+    if (targets.indexOf(npmPkgName) !== -1) tasks.push(`${name}:${npmPkgName}`);
     for (const [target, format] of combinations(targets, modules)) {
-        if (format === `cls`) {
+        if (moduleFormatsToSkipCombosOf[format] && name === `test`) {
             continue;
         }
-        combos.push(`${name}:${target}:${format}`);
-    }
-    return combos;
-}
-
-function cleanTask(target, format, taskName, outDir) {
-    return function cleanTask() {
-        const globs = [`${outDir}/**`];
-        if (target === `es5` && format === `cjs`) {
-            globs.push(`types`, `typings`);
-        }
-        return del(globs);
-    };
-}
-
-function buildTask(target, format, taskName, outDir) {
-    return format === `umd`
-        ? closureTask(target, format, taskName, outDir)
-        : typescriptTask(target, format, taskName, outDir);
-}
-
-function bundleTask(target, format, taskName, outDir) {
-    return function bundleTask() {
-        return streamMerge([
-            pump(gulp.src([`LICENSE`, `README.md`]), gulp.dest(outDir), onError),
-            pump(
-                gulp.src(`package.json`),
-                gulpJsonTransform((orig) => [
-                    `version`, `description`, `keywords`,
-                    `repository`, `author`, `homepage`, `bugs`, `license`,
-                    `dependencies`, `peerDependencies`
-                ].reduce((copy, key) => (
-                    (copy[key] = orig[key]) && copy || copy
-                ), {
-                    main: `Arrow.js`,
-                    types: `Arrow.d.ts`,
-                    typings: `Arrow.d.ts`,
-                    name: `@apache-arrow/${target}-${format}`
-                }), 2),
-                gulp.dest(outDir),
-                onError
-            )
-        ]);
-    }
-}
-
-function packageTask(target, format, taskName, outDir) {
-    return [`build:${taskName}`, `bundle:${taskName}`];
-}
-
-function testTask(target, format, taskName, outDir, debug) {
-    const jestOptions = !debug ? [] : [
-        `--runInBand`, `--env`, `jest-environment-node-debug`];
-    argv.update && jestOptions.unshift(`-u`);
-    argv.verbose && jestOptions.unshift(`--verbose`);
-    argv.coverage && jestOptions.unshift(`--coverage`);
-    const jestPath = `./node_modules/.bin/jest`;
-    const debugOpts = jestOptions.join(' ');
-    const spawnOptions = {
-        stdio: [`ignore`, `inherit`, `inherit`],
-        env: Object.assign({}, process.env, {
-            TEST_TARGET: target, TEST_MODULE: format
-        })
-    };
-    return function testTask() {
-        return !debug ?
-            child_process.spawn(jestPath, jestOptions, spawnOptions) :
-            child_process.exec(`node --inspect-brk ${jestPath} ${debugOpts}`, spawnOptions);
-    }
-}
-
-function closureTask(target, format, taskName, outDir) {
-    const clsTarget = `es5`;
-    const googleRoot = `targets/${clsTarget}/cls`;
-    const languageIn = clsTarget === `es5` ? `es2015` : clsTarget;
-    return [
-        [`clean:${taskName}`, `build:${clsTarget}:cls`],
-        function closureTask() {
-            return closureStream(
-                closureSrcs(),
-                closureCompiler(closureArgs())
-            ).on('end', () => del([`targets/${target}/cls/**`]));
-        }
-    ];
-    function closureSrcs() {
-        return gulp.src([
-            `closure-compiler-scripts/*.js`,
-            `${googleRoot}/**/*.js`,
-            `!${googleRoot}/format/*.js`,
-            `!${googleRoot}/Arrow.externs.js`,
-        ], { base: `./` });
-    }
-    function closureStream(sources, compiler) {
-        const streams = [
-            sources,
-            sourcemaps.init(),
-            compiler,
-            sourcemaps.write('.'),
-            gulp.dest(outDir)
-        ];
-        // copy the ES5 UMD bundle to dist
-        if (target === `es5`) {
-            streams.push(gulp.dest(`dist`));
-        }
-        return pump(...streams, onError);
-    }
-    function closureArgs() {
-        return {
-            third_party: true,
-            externs: `${googleRoot}/Arrow.externs.js`,
-            warning_level: `QUIET`,
-            dependency_mode: `LOOSE`,
-            rewrite_polyfills: false,
-            // formatting: `PRETTY_PRINT`,
-            compilation_level: `ADVANCED`,
-            assume_function_wrapper: true,
-            js_output_file: `Arrow.js`,
-            language_in: gCCTargets[languageIn],
-            language_out: gCCTargets[clsTarget],
-            entry_point: `${googleRoot}/Arrow.js`,
-            output_wrapper:
-`// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-(function (global, factory) {
-    typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :
-    typeof define === 'function' && define.amd ? define(['exports'], factory) :
-    (factory(global.Arrow = global.Arrow || {}));
-}(this, (function (exports) {%output%}.bind(this))));`
-        };
-    }
-}
-
-function typescriptTask(target, format, taskName, outDir) {
-    return [
-        [`clean:${taskName}`],
-        function typescriptTask() {
-            const tsconfigPath = `tsconfig/tsconfig.${target}.${format}.json`;
-            let { js, dts } = tsProjects.find((p) => p.target === target && p.format === format) || {};
-            if (!js || !dts) {
-                let tsProject = ts.createProject(tsconfigPath);
-                ({ js, dts } = pump(
-                    tsProject.src(),
-                    sourcemaps.init(),
-                    tsProject(ts.reporter.fullReporter(true)),
-                    onError
-                ));
-                dts = [dts, gulp.dest(outDir)];
-                js = [js, sourcemaps.write(), gulp.dest(outDir)];
-                // copy types to the root
-                if (target === `es5` && format === `cjs`) {
-                    dts.push(gulp.dest(`types`));
-                }
-                tsProjects.push({
-                    target, format,
-                    js: js = pump(...js, onError),
-                    dts: dts = pump(...dts, onError)
-                });
-            }
-            return streamMerge([ dts, js ]);
-        }
-    ];
-}
-
-function* combinations(_targets, _modules) {
-
-    const targets = known(knownTargets, _targets || [`all`]);
-    const modules = known(knownModules, _modules || [`all`]);
-
-    for (const format of modules) {
-        for (const target of targets) {
-            yield [target, format];
-        }
-    }
-
-    function known(known, values) {
-        return ~values.indexOf(`all`)
-            ? known
-            : Object.keys(
-                values.reduce((map, arg) => ((
-                    (known.indexOf(arg) !== -1) &&
-                    (map[arg.toLowerCase()] = true)
-                    || true) && map
-                ), {})
-            ).sort((a, b) => known.indexOf(a) - known.indexOf(b));
+        tasks.push(`${name}:${taskName(target, format)}`);
     }
+    return tasks.length && tasks || [(done) => done()];
 }
-
-function onError(err) {
-    if (typeof err === 'number') {
-        process.exit(err);
-    } else if (err) {
-        console.error(err.stack || err.toString());
-        process.exit(1);
-    }
-}
\ No newline at end of file
diff --git a/js/package.json b/js/package.json
index 03687a8b25ca2..ba93a34686288 100644
--- a/js/package.json
+++ b/js/package.json
@@ -14,18 +14,18 @@
     "test": "gulp test",
     "build": "gulp build",
     "clean": "gulp clean",
+    "debug": "gulp debug",
     "bundle": "gulp bundle",
     "package": "gulp package",
     "perf": "node ./perf/index.js",
-    "test:debug": "gulp test:debug",
     "test:coverage": "gulp test -t esnext -m esm --coverage",
     "validate": "npm-run-all clean lint build test bundle",
     "lerna:publish": "lerna exec --bail=false npm publish",
     "prepublishOnly": "sh ./prepublish.sh",
     "doc": "shx rm -rf ./doc && esdoc",
     "lint": "npm-run-all -p lint:*",
-    "lint:src": "tslint --fix --type-check -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
-    "lint:test": "tslint --fix --type-check -p test/tsconfig.json -c tslint.json \"test/**/*.ts\""
+    "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
+    "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\""
   },
   "repository": {
     "type": "git",
@@ -50,7 +50,6 @@
     "README.md"
   ],
   "peerDependencies": {
-    "tslib": "~1.7.1",
     "command-line-usage": "4.0.1"
   },
   "dependencies": {
@@ -58,38 +57,44 @@
     "text-encoding": "0.6.4"
   },
   "devDependencies": {
-    "@types/flatbuffers": "1.6.4",
-    "@types/jest": "20.0.8",
-    "@types/node": "^8.0.24",
+    "@std/esm": "0.12.5",
+    "@types/flatbuffers": "1.6.5",
+    "@types/jest": "21.1.5",
+    "@types/node": "8.0.47",
     "@types/text-encoding": "0.0.32",
     "benchmark": "2.1.4",
-    "coveralls": "2.13.1",
     "command-line-args": "4.0.7",
+    "coveralls": "3.0.0",
     "del": "3.0.0",
     "esdoc": "1.0.3",
     "esdoc-standard-plugin": "1.0.0",
     "google-closure-compiler": "20170910.0.0",
     "gulp": "github:gulpjs/gulp#4.0",
-    "gulp-json-transform": "0.4.2",
+    "gulp-json-transform": "0.4.5",
+    "gulp-rename": "1.2.2",
     "gulp-sourcemaps": "2.6.1",
-    "gulp-typescript": "3.2.2",
-    "jest": "21.1.0",
+    "gulp-typescript": "3.2.3",
+    "jest": "21.2.1",
     "jest-environment-node-debug": "2.0.0",
     "json": "9.0.6",
-    "lerna": "2.2.0",
-    "lint-staged": "4.2.1",
+    "lerna": "2.5.0",
+    "lint-staged": "4.3.0",
     "merge2": "1.2.0",
     "mkdirp": "0.5.1",
     "npm-run-all": "4.1.1",
     "pump": "1.0.2",
     "rimraf": "2.6.2",
+    "rxjs": "5.5.2",
     "shx": "0.2.2",
+    "source-map-loader": "0.2.3",
     "text-encoding-utf-8": "1.0.1",
     "trash": "4.1.0",
-    "ts-jest": "21.0.1",
-    "tslib": "1.7.1",
-    "tslint": "5.7.0",
-    "typescript": "2.5.2"
+    "ts-jest": "21.1.4",
+    "tslib": "1.8.0",
+    "tslint": "5.8.0",
+    "typescript": "2.6.1",
+    "uglifyjs-webpack-plugin": "1.0.1",
+    "webpack": "3.8.1"
   },
   "lint-staged": {
     "*.@(ts)": [
diff --git a/js/perf/index.js b/js/perf/index.js
index 669f690122d10..3a2ed96772330 100644
--- a/js/perf/index.js
+++ b/js/perf/index.js
@@ -16,12 +16,10 @@
 // under the License.
 
 // Use the ES5 UMD target as perf baseline
-// ES6/7 iterators are faster in turbofan, but something about the
-// ES5 transpilation (rewriting let and const to var?) JITs better
-const { Table, readBuffers } = require('../dist/Arrow');
+// const { Table, readBuffers } = require('../targets/es5/umd');
 // const { Table, readBuffers } = require('../targets/es5/cjs');
+const { Table, readBuffers } = require('../targets/es2015/umd');
 // const { Table, readBuffers } = require('../targets/es2015/cjs');
-// const { Table, readBuffers } = require('../targets/esnext/cjs');
 
 const Benchmark = require('benchmark');
 const arrowTestConfigurations = require('./config');
@@ -35,12 +33,12 @@ for (let [name, ...buffers] of arrowTestConfigurations) {
     const getByIndexSuite = new Benchmark.Suite(`Get ${name} values by index`, { async: true });
     parseSuite.add(createFromTableTest(name, buffers));
     parseSuite.add(createReadBuffersTest(name, buffers));
-    for (const vector of Table.from(...buffers).cols()) {
+    for (const vector of Table.from(...buffers).columns) {
         sliceSuite.add(createSliceTest(vector));
         iterateSuite.add(createIterateTest(vector));
         getByIndexSuite.add(createGetByIndexTest(vector));
     }
-    suites.push(parseSuite, sliceSuite, getByIndexSuite, iterateSuite);
+    suites.push(getByIndexSuite, iterateSuite, sliceSuite, parseSuite);
 }
 
 console.log('Running apache-arrow performance tests...\n');
diff --git a/js/src/Arrow.externs.ts b/js/src/Arrow.externs.ts
index 7289d6d2732b6..c23930271183d 100644
--- a/js/src/Arrow.externs.ts
+++ b/js/src/Arrow.externs.ts
@@ -24,23 +24,30 @@
 Symbol.iterator;
 /** @type {symbol} */
 Symbol.asyncIterator;
-let Table = function() {};
+
+let RowVector = function() {};
 /** @type {?} */
-Table.prototype.length;
+RowVector.prototype.toJSON;
 /** @type {?} */
-Table.prototype.rows;
+RowVector.prototype.toArray;
 /** @type {?} */
-Table.prototype.cols;
+RowVector.prototype.toObject;
 /** @type {?} */
-Table.prototype.getRow;
+RowVector.prototype.toString;
+
+let Table = function() {};
+/** @type {?} */
+(<any> Table).from;
 /** @type {?} */
-Table.prototype.getCell;
+Table.prototype.columns;
 /** @type {?} */
-Table.prototype.getCellAt;
+Table.prototype.length;
 /** @type {?} */
-Table.prototype.getColumn;
+Table.prototype.col;
 /** @type {?} */
-Table.prototype.getColumnAt;
+Table.prototype.key;
+/** @type {?} */
+Table.prototype.select;
 /** @type {?} */
 Table.prototype.toString;
 
@@ -52,24 +59,26 @@ Vector.prototype.name;
 /** @type {?} */
 Vector.prototype.type;
 /** @type {?} */
-Vector.prototype.props;
-/** @type {?} */
 Vector.prototype.get;
 /** @type {?} */
 Vector.prototype.concat;
 /** @type {?} */
 Vector.prototype.slice;
-
-let TypedVector = function() {};
 /** @type {?} */
-TypedVector.prototype.arrayType;
+Vector.prototype.metadata;
+/** @type {?} */
+Vector.prototype.nullable;
+/** @type {?} */
+Vector.prototype.nullCount;
 
-let ValidityVector = function() {};
+let BoolVector = function() {};
+/** @type {?} */
+(<any> BoolVector).pack;
 /** @type {?} */
-(<any> ValidityVector).pack;
+BoolVector.prototype.set;
 
 let DictionaryVector = function() {};
 /** @type {?} */
-DictionaryVector.prototype.index;
+DictionaryVector.prototype.getKey;
 /** @type {?} */
-DictionaryVector.prototype.value;
+DictionaryVector.prototype.getValue;
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index ea8a5c3e1d9bb..3196550884dbf 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -15,44 +15,45 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Table } from './table';
 import { readBuffers } from './reader/arrow';
-import { Vector } from './vector/vector';
-import { StructVector } from './vector/struct';
-import { DictionaryVector } from './vector/dictionary';
-import { ListVector, Utf8Vector, FixedSizeListVector } from './vector/list';
-import {
-    TypedVector, BitVector,
-    DateVector, IndexVector,
-    Int8Vector, Int16Vector,
-    Int32Vector, Int64Vector,
-    Uint8Vector, Uint16Vector,
-    Uint32Vector, Uint64Vector,
-    Float32Vector, Float64Vector,
-} from './vector/typed';
 
-export {
-    Table, readBuffers,
-    Vector,
-    BitVector,
-    ListVector,
-    Utf8Vector,
-    DateVector,
-    IndexVector,
+import { Vector } from './types/types';
+import { ListVector } from './types/list';
+import { Utf8Vector } from './types/utf8';
+import { BoolVector } from './types/vector/bool';
+import { DateVector } from './types/vector/date';
+import { RowVector } from './types/table/row';
+import { TableVector } from './types/table/table';
+import { StructVector } from './types/table/struct';
+import { DictionaryVector } from './types/dictionary';
+import { FixedSizeListVector } from './types/fixedsizelist';
+import { LongVector, Int64Vector, Uint64Vector, } from './types/vector/long';
+import {
     TypedVector,
     Int8Vector,
     Int16Vector,
     Int32Vector,
-    Int64Vector,
     Uint8Vector,
     Uint16Vector,
     Uint32Vector,
-    Uint64Vector,
     Float32Vector,
-    Float64Vector,
-    StructVector,
+    Float64Vector
+} from './types/vector/typed';
+
+import './types/table/from';
+
+export {
+    Vector,
+    readBuffers,
     DictionaryVector,
-    FixedSizeListVector,
+    RowVector as Row,
+    TableVector as Table,
+    StructVector, Utf8Vector,
+    ListVector, FixedSizeListVector,
+    BoolVector, TypedVector, LongVector,
+    DateVector, Float32Vector, Float64Vector,
+    Int8Vector, Int16Vector, Int32Vector, Int64Vector,
+    Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector,
 };
 
 /* These exports are needed for the closure umd targets */
@@ -60,15 +61,18 @@ try {
     const Arrow = eval('exports');
     if (typeof Arrow === 'object') {
         // string indexers tell closure compiler not to rename these properties
-        Arrow['Table'] = Table;
-        Arrow['readBuffers'] = readBuffers;
         Arrow['Vector'] = Vector;
-        Arrow['BitVector'] = BitVector;
-        Arrow['ListVector'] = ListVector;
+        Arrow['Table'] = TableVector;
+        Arrow['readBuffers'] = readBuffers;
+        Arrow['BoolVector'] = BoolVector;
         Arrow['Utf8Vector'] = Utf8Vector;
-        Arrow['DateVector'] = DateVector;
-        Arrow['IndexVector'] = IndexVector;
+        Arrow['ListVector'] = ListVector;
+        Arrow['StructVector'] = StructVector;
+        Arrow['DictionaryVector'] = DictionaryVector;
+        Arrow['FixedSizeListVector'] = FixedSizeListVector;
+        Arrow['LongVector'] = LongVector;
         Arrow['TypedVector'] = TypedVector;
+        Arrow['DateVector'] = DateVector;
         Arrow['Int8Vector'] = Int8Vector;
         Arrow['Int16Vector'] = Int16Vector;
         Arrow['Int32Vector'] = Int32Vector;
@@ -79,9 +83,6 @@ try {
         Arrow['Uint64Vector'] = Uint64Vector;
         Arrow['Float32Vector'] = Float32Vector;
         Arrow['Float64Vector'] = Float64Vector;
-        Arrow['StructVector'] = StructVector;
-        Arrow['DictionaryVector'] = DictionaryVector;
-        Arrow['FixedSizeListVector'] = FixedSizeListVector;
     }
 } catch (e) { /* not the UMD bundle */ }
 /* end closure exports */
diff --git a/js/src/reader/arrow.ts b/js/src/reader/arrow.ts
index dbb6acd0e79e8..033bfecae61dd 100644
--- a/js/src/reader/arrow.ts
+++ b/js/src/reader/arrow.ts
@@ -18,17 +18,18 @@
 import { flatbuffers } from 'flatbuffers';
 import * as Schema_ from '../format/Schema_generated';
 import * as Message_ from '../format/Message_generated';
+export import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
 
 import { readFile } from './file';
 import { readStream } from './stream';
 import { readVector } from './vector';
-import { Vector } from '../vector/vector';
 import { readDictionary } from './dictionary';
+import { Vector, Column } from '../types/types';
 
 import ByteBuffer = flatbuffers.ByteBuffer;
-export import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
-export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
-export type Dictionaries = { [k: string]: Vector<any> };
+import Field = Schema_.org.apache.arrow.flatbuf.Field;
+export type Dictionaries = { [k: string]: Vector<any> } | null;
 export type IteratorState = { nodeIndex: number; bufferIndex: number };
 
 export function* readRecords(...bytes: ByteBuffer[]) {
@@ -47,22 +48,27 @@ export function* readBuffers(...bytes: Array<Uint8Array | Buffer | string>) {
     const dictionaries: Dictionaries = {};
     const byteBuffers = bytes.map(toByteBuffer);
     for (let { schema, batch } of readRecords(...byteBuffers)) {
-        let vectors: Vector<any>[] = [];
+        let vectors: Column<any>[] = [];
         let state = { nodeIndex: 0, bufferIndex: 0 };
-        let index = -1, fieldsLength = schema.fieldsLength();
+        let fieldsLength = schema.fieldsLength();
+        let index = -1, field: Field, vector: Vector<any>;
         if (batch.id) {
             // A dictionary batch only contain a single vector. Traverse each
             // field and its children until we find one that uses this dictionary
             while (++index < fieldsLength) {
-                let vector = readDictionary(schema.fields(index), batch, state, dictionaries);
-                if (vector) {
-                    dictionaries[batch.id] = dictionaries[batch.id] && dictionaries[batch.id].concat(vector) || vector;
-                    break;
+                if (field = schema.fields(index)!) {
+                    if (vector = readDictionary<any>(field, batch, state, dictionaries)!) {
+                        dictionaries[batch.id] = dictionaries[batch.id] && dictionaries[batch.id].concat(vector) || vector;
+                        break;
+                    }
                 }
             }
         } else {
             while (++index < fieldsLength) {
-                vectors[index] = readVector(schema.fields(index), batch, state, dictionaries);
+                if ((field = schema.fields(index)!) &&
+                    (vector = readVector<any>(field, batch, state, dictionaries)!)) {
+                    vectors[index] = vector as Column<any>;
+                }
             }
             yield vectors;
         }
diff --git a/js/src/reader/dictionary.ts b/js/src/reader/dictionary.ts
index 93a9ba76bba3a..0c58ace3b0dca 100644
--- a/js/src/reader/dictionary.ts
+++ b/js/src/reader/dictionary.ts
@@ -17,22 +17,20 @@
 
 import { readVector } from './vector';
 import { MessageBatch } from './message';
+import { DictionaryVector } from '../types/dictionary';
 import * as Schema_ from '../format/Schema_generated';
 import { IteratorState, Dictionaries } from './arrow';
 import Field = Schema_.org.apache.arrow.flatbuf.Field;
 
-export function readDictionary(field: Field | null,
-                               batch: MessageBatch,
-                               iterator: IteratorState,
-                               dictionaries: Dictionaries) {
-    let id: string, encoding = field && field.dictionary();
+export function readDictionary<T>(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): DictionaryVector<T> | null {
+    let vector: DictionaryVector<T> | null, id, encoding = field.dictionary();
     if (encoding && batch.id === (id = encoding.id().toFloat64().toString())) {
-        return readVector(field, batch, iterator, null);
+        return readVector<T>(field, batch, iterator, null) as DictionaryVector<T>;
     }
-    for (let i = -1, n = field && field.childrenLength() || 0; ++i < n;) {
-        let vector = readDictionary(field.children(i), batch, iterator, dictionaries);
-        if (vector) {
+    for (let i = -1, n = field.childrenLength() | 0; ++i < n;) {
+        if (vector = readDictionary<T>(field.children(i)!, batch, iterator, dictionaries)) {
             return vector;
         }
     }
+    return null;
 }
diff --git a/js/src/reader/file.ts b/js/src/reader/file.ts
index b05b99a5e6dcf..bd60b476312a0 100644
--- a/js/src/reader/file.ts
+++ b/js/src/reader/file.ts
@@ -58,9 +58,12 @@ export function* readFile(...bbs: ByteBuffer[]) {
             throw new Error('Invalid file');
         }
         bb.setPosition(footerOffset - footerLength);
-        let footer = Footer.getRootAsFooter(bb), schema = footer.schema();
+        let schema, footer = Footer.getRootAsFooter(bb);
+        if (!(schema = footer.schema()!)) {
+            return;
+        }
         for (let i = -1, n = footer.dictionariesLength(); ++i < n;) {
-            let block = footer.dictionaries(i);
+            let block = footer.dictionaries(i)!;
             bb.setPosition(block.offset().low);
             for (let batch of readMessageBatches(bb)) {
                 yield { schema, batch };
@@ -68,7 +71,7 @@ export function* readFile(...bbs: ByteBuffer[]) {
             }
         }
         for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) {
-            const block = footer.recordBatches(i);
+            const block = footer.recordBatches(i)!;
             bb.setPosition(block.offset().low);
             for (let batch of readMessageBatches(bb)) {
                 yield { schema, batch };
diff --git a/js/src/reader/message.ts b/js/src/reader/message.ts
index 5472f10833878..6c8a969021f90 100644
--- a/js/src/reader/message.ts
+++ b/js/src/reader/message.ts
@@ -46,13 +46,13 @@ export function* readMessageBatches(bb: ByteBuffer) {
     let bytes = bb.bytes();
     for (let message of readMessages(bb)) {
         let type = message.headerType();
-        let id: string, data: RecordBatch;
+        let id: string | void, data: RecordBatch;
         if (type === MessageHeader.RecordBatch) {
-            data = message.header(new RecordBatch());
+            data = message.header(new RecordBatch())!;
         } else if (type === MessageHeader.DictionaryBatch) {
-            let header = message.header(new DictionaryBatch());
+            let header = message.header(new DictionaryBatch())!;
             id = header.id().toFloat64().toString();
-            data = header.data();
+            data = header.data()!;
         } else {
             continue;
         }
diff --git a/js/src/reader/stream.ts b/js/src/reader/stream.ts
index 9869f633d08f4..2062b1a8c4c10 100644
--- a/js/src/reader/stream.ts
+++ b/js/src/reader/stream.ts
@@ -30,8 +30,8 @@ export function* readStream(...bbs: ByteBuffer[]) {
         throw new Error('Invalid Arrow Stream');
     }
     for (const message of readMessages(bbs[0])) {
-        if (message.headerType() === MessageHeader.Schema) {
-            const schema = message.header(new Schema());
+        let schema: Schema;
+        if (message.headerType() === MessageHeader.Schema && (schema = message.header(new Schema())!)) {
             for (const bb of bbs) {
                 for (const batch of readMessageBatches(bb)) {
                     yield { schema, batch };
diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts
index 3b6663be89bdc..4d3321833209d 100644
--- a/js/src/reader/vector.ts
+++ b/js/src/reader/vector.ts
@@ -17,22 +17,28 @@
 
 import { flatbuffers } from 'flatbuffers';
 import { MessageBatch } from './message';
-import { Vector } from '../vector/vector';
 import * as Schema_ from '../format/Schema_generated';
-import { StructVector } from '../vector/struct';
+import * as Message_ from '../format/Message_generated';
 import { IteratorState, Dictionaries } from './arrow';
-import { DictionaryVector } from '../vector/dictionary';
-import { Utf8Vector, ListVector, FixedSizeListVector } from '../vector/list';
 import {
-    TypedArray, TypedArrayCtor, IntArray, FloatArray,
+    Vector, Column,
+    IntArray, FloatArray,
+    TypedArray, TypedArrayConstructor,
+} from '../types/types';
+
+import {
+    DictionaryVector,
+    Utf8Vector, StructVector,
+    ListVector, FixedSizeListVector,
+    DateVector, Float32Vector, Float64Vector,
     Int8Vector, Int16Vector, Int32Vector, Int64Vector,
     Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector,
-    Float32Vector, Float64Vector, IndexVector, DateVector,
-} from '../vector/typed';
+} from '../types/arrow';
 
 import Int = Schema_.org.apache.arrow.flatbuf.Int;
 import Type = Schema_.org.apache.arrow.flatbuf.Type;
 import Field = Schema_.org.apache.arrow.flatbuf.Field;
+import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
 import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
 import VectorType = Schema_.org.apache.arrow.flatbuf.VectorType;
 import VectorLayout = Schema_.org.apache.arrow.flatbuf.VectorLayout;
@@ -40,31 +46,33 @@ import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
 import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
 import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
 
-export function readVector(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries) {
-    return readDictionaryVector(field, batch, state, dictionaries) ||
-                readTypedVector(field, batch, state, dictionaries);
+export function readVector<T>(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries): Column<T> | DictionaryVector<T> | null {
+    return readDictionaryVector<T>(field, batch, state, dictionaries) ||
+                readTypedVector<T>(field, batch, state, dictionaries);
 }
 
-function readTypedVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) {
+function readTypedVector<T>(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): Column<T> | DictionaryVector<T> | null {
     let typeType = field.typeType(), readTyped = typedVectorReaders[typeType];
     if (!readTyped) {
         throw new Error('Unrecognized vector name "' + Type[typeType] + '" type "' + typeType + '"');
     }
-    return readTyped(field, batch, iterator, dictionaries);
+    return readTyped(field, batch, iterator, dictionaries) as Column<T>;
 }
 
-function readDictionaryVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) {
-    let encoding: DictionaryEncoding | null;
-    if (dictionaries && (encoding = field.dictionary())) {
-        let id = encoding.id().toFloat64().toString();
-        let fieldType =  encoding.indexType() ||
+function readDictionaryVector<T>(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): DictionaryVector<T> | null {
+    let data: Vector<any>, encoding: DictionaryEncoding;
+    if (dictionaries &&
+        (encoding = field.dictionary()!) &&
+        (data = dictionaries[encoding.id().toFloat64().toString()])) {
+        let indexType =  encoding.indexType() ||
             /* a dictionary index defaults to signed 32 bit int if unspecified */
             { bitWidth: () => 32, isSigned: () => true };
         // workaround for https://issues.apache.org/jira/browse/ARROW-1363
-        let indexField = createSyntheticDictionaryIndexField(field, fieldType);
-        let index = readIntVector(indexField, batch, iterator, null, fieldType);
-        return DictionaryVector.create(field, index.length, index, dictionaries[id]);
+        let indexField = createSyntheticDictionaryIndexField(field, indexType);
+        let keys = readIntVector(indexField, batch, iterator, null, indexType)!;
+        return new DictionaryVector<T>({ data, keys: keys! });
     }
+    return null;
 }
 
 const IntViews    = [Int8Array,    Int16Array,   Int32Array,   Int32Array  ];
@@ -74,39 +82,54 @@ const Uint8Views  = [Uint8Array,   Uint8Array,   Uint8Array,   Uint8Array  ];
 const Uint32Views = [Uint32Array,  Uint32Array,  Uint32Array,  Uint32Array ];
 const FloatViews  = [Int8Array,    Int16Array,   Float32Array, Float64Array];
 
-const createIntDataViews = createDataView.bind(null, IntViews, null);
-const createUintDataViews = createDataView.bind(null, UintViews, null);
-const createDateDataViews = createDataView.bind(null, Uint32Views, null);
-const createFloatDataViews = createDataView.bind(null, FloatViews, null);
-const createNestedDataViews = createDataView.bind(null, Uint32Views, null);
-const createValidityDataViews = createDataView.bind(null, Uint8Views, null);
-const createUtf8DataViews = createDataView.bind(null, Uint8Views, Int32Views);
+const createIntDataViews = createTypedArray.bind(null, IntViews, null);
+const createUintDataViews = createTypedArray.bind(null, UintViews, null);
+const createDateDataViews = createTypedArray.bind(null, Uint32Views, null);
+const createFloatDataViews = createTypedArray.bind(null, FloatViews, null);
+const createNestedDataViews = createTypedArray.bind(null, Uint32Views, null);
+const createValidityDataViews = createTypedArray.bind(null, Uint8Views, null);
+const createUtf8DataViews = createTypedArray.bind(null, Uint8Views, Int32Views);
 
+// Define as computed properties for closure-compiler
 const floatVectors = {
+    [Precision.HALF]: Float32Vector,
     [Precision.SINGLE]: Float32Vector,
-    [Precision.DOUBLE]: Float64Vector
-};
+    [Precision.DOUBLE]: Float64Vector,
+} as { [k: number]: any };
+
+// and again as string-indexed keys for Uglify...
+floatVectors[Precision['HALF']] = Float32Vector;
+floatVectors[Precision['SINGLE']] = Float32Vector;
+floatVectors[Precision['DOUBLE']] = Float64Vector;
+
 const intVectors = [
     [/* unsigned */ Uint8Vector,   /* signed */ Int8Vector ],
     [/* unsigned */ Uint16Vector,  /* signed */ Int16Vector],
     [/* unsigned */ Uint32Vector,  /* signed */ Int32Vector],
     [/* unsigned */ Uint64Vector,  /* signed */ Int64Vector]
-];
+] as any[][];
 
-function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, fieldType?: FieldType) {
-    let type = (fieldType || field.type(new Int()));
+function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) {
+    let type = (primitiveType || field.type(new Int())!);
     return type.isSigned() ?
         read_IntVector(field, batch, iterator, dictionaries, type) :
         readUintVector(field, batch, iterator, dictionaries, type);
 }
 
-const read_IntVector = readVectorLayout<number, IntArray>(createIntDataViews, createIntVector);
-const readUintVector = readVectorLayout<number, IntArray>(createUintDataViews, createIntVector);
-function createIntVector(field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) {
-    let type = fieldType || field.type(new Int()), bitWidth = type.bitWidth();
-    let Vector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()];
-    return Vector.create(field, length, validity, data || offsets);
-    // ----------------------------------------------- 👆:
+function read_IntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) {
+    return readVectorLayout(createIntDataViews, createIntVector, field, batch, iterator, dictionaries, primitiveType);
+}
+
+function readUintVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) {
+    return readVectorLayout(createUintDataViews, createIntVector, field, batch, iterator, dictionaries, primitiveType);
+}
+
+function createIntVector(argv: VectorFactoryArgv<IntArray>) {
+    let { field, fieldNode, data, validity, offsets, primitiveType } = argv;
+    let type = primitiveType || field.type(new Int())!, bitWidth = type.bitWidth();
+    let IntVector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()];
+    return new IntVector({ fieldNode, field, validity, data: data! || offsets! });
+    // ---------------------------------------------------- 👆:
     // Workaround for https://issues.apache.org/jira/browse/ARROW-1363
     // This bug causes dictionary encoded vector indicies' IntVector data
     // buffers to be tagged as VectorType.OFFSET (0) in the field metadata
@@ -118,68 +141,58 @@ function createIntVector(field, length, data, validity, offsets, fieldType, batc
     // the offset buffer is the data, because IntVectors don't have offsets.
 }
 
-const readFloatVector = readVectorLayout<number, FloatArray>(
-    createFloatDataViews,
-    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
-        let type = field.type(new FloatingPoint());
-        let Vector = floatVectors[type.precision()];
-        return Vector.create(field, length, validity, data);
-    }
-);
+function bindVectorReader<T extends TypedArray, V>(createBufferView: BufferViewFactory<T>, createVector: VectorFactory<T, V>) {
+    return function readVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) {
+        return readVectorLayout(createBufferView, createVector, field, batch, iterator, dictionaries, primitiveType);
+    };
+}
 
-const readDateVector = readVectorLayout<Date, Uint32Array>(
-    createDateDataViews,
-    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
-        return DateVector.create(field, length, validity, data);
-    }
-);
+const readFloatVector = bindVectorReader(createFloatDataViews, ({ field, fieldNode, data, validity }: VectorFactoryArgv<FloatArray>) => {
+    const type = field.type(new FloatingPoint())!;
+    const FloatVector = floatVectors[type.precision()];
+    return new FloatVector({ field, fieldNode, validity, data: data! });
+});
 
-const readUtf8Vector = readVectorLayout<string, Uint8Array>(
-    createUtf8DataViews,
-    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
-        let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1);
-        return Utf8Vector.create(
-            field, length, validity,
-            Uint8Vector.create(field, data.length, null, data),
-            IndexVector.create(field, length + 1, null, offsetsAdjusted)
-        );
-    }
-);
+const readDateVector = bindVectorReader(createDateDataViews, ({ field, fieldNode, data, validity }: VectorFactoryArgv<Uint32Array>) => {
+    return new DateVector({ field, fieldNode, validity, data: data! });
+});
 
-const readListVector = readVectorLayout<any[], Uint32Array>(
-    createNestedDataViews,
-    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
-        let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1);
-        return ListVector.create(
-            field, length, validity,
-            readVector(field.children(0), batch, iterator, dictionaries),
-            IndexVector.create(field, length + 1, null, offsetsAdjusted)
-        );
-    }
-);
+const readUtf8Vector = bindVectorReader(createUtf8DataViews, ({ field, fieldNode, data, offsets, validity }: VectorFactoryArgv<Uint8Array>) => {
+    return new Utf8Vector({
+        field, fieldNode,
+        values: new ListVector({
+            validity,
+            offsets: offsets as Int32Array,
+            values: new Uint8Vector({ data: data! })
+        }) as any as Vector<Uint8Array | null>
+    });
+});
 
-const readFixedSizeListVector = readVectorLayout<any[], Uint32Array>(
-    createNestedDataViews,
-    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
-        let size = field.type(new FixedSizeList()).listSize();
-        return FixedSizeListVector.create(
-            field, length, size, validity,
-            readVector(field.children(0), batch, iterator, dictionaries)
-        );
-    }
-);
+const readListVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, offsets, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv<TypedArray>) => {
+    return new ListVector({
+        field, fieldNode, validity,
+        offsets: offsets! as Int32Array,
+        values: readVector(field.children(0)!, messageBatch, iterator, dictionaries)!
+    });
+});
 
-const readStructVector = readVectorLayout<any[], ArrayLike<any>>(
-    createNestedDataViews,
-    (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
-        let vectors: Vector<any>[] = [];
-        for (let i = -1, n = field.childrenLength(); ++i < n;) {
-            vectors[i] = readVector(field.children(i), batch, iterator, dictionaries);
-        }
-        return StructVector.create(field, length, validity, ...vectors);
+const readFixedSizeListVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv<Uint32Array>) => {
+    return new FixedSizeListVector({
+        field, fieldNode, validity,
+        listSize: field.type(new FixedSizeList())!.listSize(),
+        values: readVector(field.children(0)!, messageBatch, iterator, dictionaries)!
+    });
+});
+
+const readStructVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv<ArrayLike<any>>) => {
+    let columns: Column<any>[] = [];
+    for (let i = -1, n = field.childrenLength(); ++i < n;) {
+        columns[i] = readVector<any>(field.children(i)!, messageBatch, iterator, dictionaries) as Column<any>;
     }
-);
+    return new StructVector({ field, fieldNode, validity, columns });
+});
 
+// Define as computed properties for closure-compiler
 const typedVectorReaders = {
     [Type.Int]: readIntVector,
     [Type.Date]: readDateVector,
@@ -188,60 +201,73 @@ const typedVectorReaders = {
     [Type.Struct_]: readStructVector,
     [Type.FloatingPoint]: readFloatVector,
     [Type.FixedSizeList]: readFixedSizeListVector,
-};
+} as { [k: number]: (...args: any[]) => Vector | null };
+
+// and again as string-indexed keys for Uglify...
+typedVectorReaders[Type['Int']] = readIntVector;
+typedVectorReaders[Type['Date']] = readDateVector;
+typedVectorReaders[Type['List']] = readListVector;
+typedVectorReaders[Type['Utf8']] = readUtf8Vector;
+typedVectorReaders[Type['Struct_']] = readStructVector;
+typedVectorReaders[Type['FloatingPoint']] = readFloatVector;
+typedVectorReaders[Type['FixedSizeList']] = readFixedSizeListVector;
+
+type VectorFactory<T, V> = (argv: VectorFactoryArgv<T>) => V;
+type PrimitiveType = { bitWidth(): number; isSigned(): boolean };
+type BufferViewFactory<T extends TypedArray> = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => T;
 
-type FieldType = { bitWidth(): number; isSigned(): boolean };
-type dataViewFactory<V = TypedArray> = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => V;
-type vectorFactory<TList, V = Vector<any>> = (field: Field,
-                                              length: number,
-                                              data: TList,
-                                              nulls: Uint8Array,
-                                              offsets: TypedArray,
-                                              fieldType: FieldType,
-                                              chunk: MessageBatch,
-                                              iterable: IteratorState,
-                                              dictionaries: Dictionaries) => V;
+interface VectorFactoryArgv<T> {
+    field: Field;
+    fieldNode: FieldNode;
+    iterator: IteratorState;
+    dictionaries: Dictionaries;
+    messageBatch: MessageBatch;
+    data?: T;
+    offsets?: TypedArray;
+    validity?: Uint8Array;
+    primitiveType?: PrimitiveType;
+}
 
-function readVectorLayout<T, TList>(createDataView: dataViewFactory<TList>, createVector: vectorFactory<TList, Vector<T>>) {
-    return function readLayout(
-            field: Field,
-            chunk: MessageBatch,
-            iterator: IteratorState,
-            dictionaries: Dictionaries,
-            integerFieldType?: FieldType
-    ) {
-        let batch = chunk.data;
-        let layoutLength = field.layoutLength();
-        let node = batch.nodes(iterator.nodeIndex++);
-        let data: TList, offsets: any, validity: Uint8Array;
-        let type, bitWidth, bufferLength, nodeLength = node.length().low;
-        for (let i = -1; ++i < layoutLength;) {
-            let layout = field.layout(i);
-            let buffer = batch.buffers(iterator.bufferIndex++);
-            if ((type = layout.type()) === VectorType.TYPE ||
-                (bufferLength = buffer.length().low) <= 0  ||
-                (bitWidth = layout.bitWidth()) <= 0) {
-                continue;
-            } else if (type === VectorType.DATA) {
-                data = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength);
-            } else if (type === VectorType.OFFSET) {
-                offsets = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength);
-            } else if (node.nullCount().low > 0) {
-                validity = createValidityDataViews(chunk, type, bitWidth, buffer.offset().low, nodeLength);
-            }
+function readVectorLayout<T extends TypedArray, V>(
+    createBufferView: BufferViewFactory<T>, createVector: VectorFactory<T, V>,
+    field: Field, messageBatch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType
+) {
+    let fieldNode: FieldNode, recordBatch = messageBatch.data;
+    if (!(fieldNode = recordBatch.nodes(iterator.nodeIndex)!)) {
+        return null;
+    }
+    iterator.nodeIndex += 1;
+    let type, bitWidth, layout, buffer, bufferLength;
+    let data: T | undefined, offsets: TypedArray | undefined, validity: Uint8Array | undefined;
+    for (let i = -1, n = field.layoutLength(); ++i < n;) {
+        if (!(layout = field.layout(i)!) ||
+            !(buffer = recordBatch.buffers(iterator.bufferIndex)!)) {
+            continue;
         }
-        return createVector(field, nodeLength, data, validity, offsets, integerFieldType, chunk, iterator, dictionaries);
-    };
+        iterator.bufferIndex += 1;
+        if ((type = layout.type()) === VectorType.TYPE ||
+            (bufferLength = buffer.length().low) <= 0  ||
+            (bitWidth = layout.bitWidth()) <= 0) {
+            continue;
+        } else if (type === VectorType.DATA) {
+            data = createBufferView(messageBatch, type, bitWidth, buffer.offset().low, bufferLength);
+        } else if (type === VectorType.OFFSET) {
+            offsets = createBufferView(messageBatch, type, bitWidth, buffer.offset().low, bufferLength);
+        } else if (fieldNode.nullCount().low > 0) {
+            validity = createValidityDataViews(messageBatch, type, bitWidth, buffer.offset().low, fieldNode.length().low);
+        }
+    }
+    return createVector({ data, offsets, validity, field, fieldNode, iterator, messageBatch, dictionaries, primitiveType });
 }
 
-function createDataView(
-    dataViews: TypedArrayCtor<any>[], offsetViews: TypedArrayCtor<any>[] | null,
+function createTypedArray(
+    bufferViews: TypedArrayConstructor[], offsetViews: TypedArrayConstructor[] | null,
     batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number
 ) {
     const buffer = batch.bytes.buffer;
     const byteLength = buffer.byteLength;
     const byteOffset = batch.offset + offset;
-    const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || dataViews);
+    const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || bufferViews);
     const dataViewLength = ((byteOffset + length) <= byteLength
         ? length
         : byteLength - byteOffset
@@ -249,12 +275,12 @@ function createDataView(
     return new DataViewType(buffer, byteOffset, dataViewLength);
 }
 
-function valueForBitWidth(bitWidth: number, values: any[]) {
+function valueForBitWidth<T>(bitWidth: number, values: T[]) {
     return values[bitWidth >> 4] || values[3];
 }
 
-function createSyntheticDictionaryIndexField(field: Field, type: FieldType) {
-    let layouts = [];
+function createSyntheticDictionaryIndexField(field: Field, type: PrimitiveType) {
+    let layouts = [] as VectorLayout[];
     let builder = new flatbuffers.Builder();
     if (field.nullable()) {
         VectorLayout.startVectorLayout(builder);
@@ -270,7 +296,7 @@ function createSyntheticDictionaryIndexField(field: Field, type: FieldType) {
     builder.finish(VectorLayout.endVectorLayout(builder));
     layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer()));
     return Object.create(field, {
-        layout: { value(i) { return layouts[i]; } },
+        layout: { value(i: number) { return layouts[i]; } },
         layoutLength: { value() { return layouts.length; } }
     });
-}
\ No newline at end of file
+}
diff --git a/js/src/table.ts b/js/src/table.ts
deleted file mode 100644
index 5e781054daf31..0000000000000
--- a/js/src/table.ts
+++ /dev/null
@@ -1,143 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { readBuffers } from './reader/arrow';
-import { StructVector } from './vector/struct';
-import { Vector, sliceToRangeArgs } from './vector/vector';
-
-export type RowObject = { [k: string]: any };
-
-export class Table implements Iterable<Map<string, any>> {
-    public length: number;
-    protected _columns: Vector<any>[];
-    protected _columnsMap: { [k: string]: Vector<any> };
-    static from(...bytes: Array<Uint8Array | Buffer | string>) {
-        let columns: Vector<any>[];
-        for (let vectors of readBuffers(...bytes)) {
-            columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i]));
-        }
-        return new Table(columns);
-    }
-    static fromStruct(vector: StructVector) {
-        return new Table((<any> vector).vectors);
-    }
-    constructor(columns: Vector<any>[]) {
-        this._columns = columns || [];
-        this.length = Math.max(...this._columns.map((v) => v.length));
-        this._columnsMap = this._columns.reduce((map, vec) => {
-            return (map[vec.name] = vec) && map || map;
-        }, <any> {});
-    }
-    *[Symbol.iterator]() {
-        for (let cols = this._columns, i = -1, n = this.length; ++i < n;) {
-            yield rowAsMap(i, cols);
-        }
-    }
-    *rows(startRow?: number | boolean, endRow?: number | boolean, compact?: boolean) {
-        let start = startRow as number, end = endRow as number;
-        if (typeof startRow === 'boolean') {
-            compact = startRow;
-            start = end;
-            end = undefined;
-        } else if (typeof endRow === 'boolean') {
-            compact = endRow;
-            end = undefined;
-        }
-        let rowIndex = -1, { length } = this;
-        const [rowOffset, rowsTotal] = sliceToRangeArgs(length, start, end);
-        while (++rowIndex < rowsTotal) {
-            yield this.getRow((rowIndex + rowOffset) % length, compact);
-        }
-    }
-    *cols(startCol?: number, endCol?: number) {
-        for (const column of this._columns.slice(startCol, endCol)) {
-            yield column;
-        }
-    }
-    getRow(rowIndex: number): RowObject;
-    getRow(rowIndex: number, compact: boolean): Array<any>;
-    getRow(rowIndex: number, compact?: boolean) {
-        return (compact && rowAsArray || rowAsObject)(rowIndex, this._columns);
-    }
-    getCell<T extends any>(columnName: string, rowIndex: number) {
-        return this.getColumn<Vector<T>>(columnName).get(rowIndex);
-    }
-    getCellAt<T extends any>(columnIndex: number, rowIndex: number) {
-        return this.getColumnAt<Vector<T>>(columnIndex).get(rowIndex);
-    }
-    getColumn<T extends Vector<any>>(columnName: string) {
-        return this._columnsMap[columnName] as T;
-    }
-    getColumnAt<T extends Vector<any>>(columnIndex: number) {
-        return this._columns[columnIndex] as T;
-    }
-    toString(): string;
-    toString(index: boolean): string;
-    toString(options: { index: boolean }): string;
-    toString(options?: any) {
-        const index = typeof options === 'object' ? options && !!options.index
-                    : typeof options === 'boolean' ? !!options
-                    : false;
-        const { length } = this;
-        if (length <= 0) { return ''; }
-        const maxColumnWidths = [];
-        const rows = new Array(length + 1);
-        rows[0] = this._columns.map((c) => c.name);
-        index && rows[0].unshift('Index');
-        for (let i = -1, n = rows.length - 1; ++i < n;) {
-            rows[i + 1] = this.getRow(i, true);
-            index && rows[i + 1].unshift(i);
-        }
-        // Pass one to convert to strings and count max column widths
-        for (let i = -1, n = rows.length; ++i < n;) {
-            const row = rows[i];
-            for (let j = -1, k = row.length; ++j < k;) {
-                const val = row[j] = `${row[j]}`;
-                maxColumnWidths[j] = !maxColumnWidths[j]
-                    ? val.length
-                    : Math.max(maxColumnWidths[j], val.length);
-            }
-        }
-        // Pass two to pad each one to max column width
-        for (let i = -1, n = rows.length; ++i < n;) {
-            const row = rows[i];
-            for (let j = -1, k = row.length; ++j < k;) {
-                row[j] = leftPad(row[j], ' ', maxColumnWidths[j]);
-            }
-            rows[i] = row.join(', ');
-        }
-        return rows.join('\n');
-    }
-}
-
-Table.prototype.length = 0;
-
-function leftPad(str, fill, n) {
-    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
-}
-
-function rowAsMap(row: number, columns: Vector<any>[]) {
-    return columns.reduce((map, vector) => map.set(vector.name, vector.get(row)), new Map());
-}
-
-function rowAsObject(rowIndex: number, columns: Vector<any>[]) {
-    return columns.reduce((row, vector) => (row[vector.name] = vector.get(rowIndex)) && row || row, Object.create(null));
-}
-
-function rowAsArray(rowIndex: number, columns: Vector<any>[]) {
-    return columns.reduce((row, vector, columnIndex) => (row[columnIndex] = vector.get(rowIndex)) && row || row, new Array(columns.length));
-}
diff --git a/js/src/text-encoding-utf-8.d.ts b/js/src/text-encoding-utf-8.d.ts
new file mode 100644
index 0000000000000..68ba4dfd9a346
--- /dev/null
+++ b/js/src/text-encoding-utf-8.d.ts
@@ -0,0 +1,4 @@
+declare module 'text-encoding-utf-8' {
+    import * as TextEncoding from 'text-encoding';
+    export = TextEncoding;
+}
diff --git a/js/src/types/arrow.ts b/js/src/types/arrow.ts
new file mode 100644
index 0000000000000..e18f5da4f1fd6
--- /dev/null
+++ b/js/src/types/arrow.ts
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Schema_ from '../format/Schema_generated';
+import * as Message_ from '../format/Message_generated';
+import Field = Schema_.org.apache.arrow.flatbuf.Field;
+import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+
+import { BoolVector } from './vector/bool';
+import { DictionaryVector } from './dictionary';
+import { nullableMixin, fieldMixin } from './vector/traits';
+import { ListVector as ListVectorBase } from './list';
+import { Utf8Vector as Utf8VectorBase } from './utf8';
+import { Vector, Column, TypedArray } from './types';
+import { DateVector as DateVectorBase } from './vector/date';
+import { TableVector as TableVectorBase } from './table/table';
+import { StructVector as StructVectorBase } from './table/struct';
+import { FixedSizeListVector as FixedSizeListVectorBase } from './fixedsizelist';
+import {
+    LongVector as LongVectorBase,
+    Int64Vector as Int64VectorBase,
+    Uint64Vector as Uint64VectorBase,
+} from './vector/long';
+import {
+    TypedVector,
+    Int8Vector as Int8VectorBase,
+    Int16Vector as Int16VectorBase,
+    Int32Vector as Int32VectorBase,
+    Uint8Vector as Uint8VectorBase,
+    Uint16Vector as Uint16VectorBase,
+    Uint32Vector as Uint32VectorBase,
+    Float32Vector as Float32VectorBase,
+    Float64Vector as Float64VectorBase,
+} from './vector/typed';
+
+export { TypedArray, TypedVector };
+export { Column, BoolVector, DictionaryVector };
+export class ListVector extends MixinArrowTraits(ListVectorBase) {}
+export class Utf8Vector extends MixinArrowTraits(Utf8VectorBase) {}
+export class TableVector extends MixinArrowTraits(TableVectorBase) {}
+export class StructVector extends MixinArrowTraits(StructVectorBase) {}
+export class FixedSizeListVector extends MixinArrowTraits(FixedSizeListVectorBase) {}
+export class DateVector extends MixinArrowTraits(DateVectorBase) {}
+export class LongVector extends MixinArrowTraits(LongVectorBase) {}
+export class Int8Vector extends MixinArrowTraits(Int8VectorBase) {}
+export class Int16Vector extends MixinArrowTraits(Int16VectorBase) {}
+export class Int32Vector extends MixinArrowTraits(Int32VectorBase) {}
+export class Int64Vector extends MixinArrowTraits(Int64VectorBase) {}
+export class Uint8Vector extends MixinArrowTraits(Uint8VectorBase) {}
+export class Uint16Vector extends MixinArrowTraits(Uint16VectorBase) {}
+export class Uint32Vector extends MixinArrowTraits(Uint32VectorBase) {}
+export class Uint64Vector extends MixinArrowTraits(Uint64VectorBase) {}
+export class Float32Vector extends MixinArrowTraits(Float32VectorBase) {}
+export class Float64Vector extends MixinArrowTraits(Float64VectorBase) {}
+
+export function MixinArrowTraits<T extends Vector<any>, TArgv>(BaseVector: new (argv: TArgv) => T) {
+    const FieldVector = fieldMixin(BaseVector);
+    const NullableVector = nullableMixin(BaseVector);
+    const NullableFieldVector = nullableMixin(FieldVector);
+    return function(this: any, argv: TArgv & (object | { validity: Uint8Array } | { field: Field, fieldNode: FieldNode })) {
+        return new ((!isFieldArgv(argv) ? !isNullableArgv(argv) ?
+            BaseVector : NullableVector : !isNullableArgv(argv) ?
+            FieldVector : NullableFieldVector
+        ) as any)(argv);
+    } as any as { new (argv: TArgv & (object | { validity: Uint8Array } | { field: Field, fieldNode: FieldNode })): T };
+}
+
+function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } {
+    return x && x.field instanceof Field && x.fieldNode instanceof FieldNode;
+}
+
+function isNullableArgv(x: any): x is { validity: Uint8Array } {
+    return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array;
+}
diff --git a/js/src/types/dictionary.ts b/js/src/types/dictionary.ts
new file mode 100644
index 0000000000000..cafa753311d3b
--- /dev/null
+++ b/js/src/types/dictionary.ts
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector, Column } from './types';
+import { VirtualVector } from './vector/virtual';
+
+export interface DictionaryVector<T> extends Vector<T> {
+    getValue(key: number): T;
+    getKey(index: number): number;
+}
+
+export class DictionaryVector<T> extends Vector<T> implements Column<T>, DictionaryVector<T> {
+    readonly data: Vector<T>;
+    readonly keys: Column<number>;
+    constructor(argv: { data: Vector<T>, keys: Vector<number> }) {
+        super();
+        this.data = argv.data;
+        this.keys = argv.keys as Column<number>;
+    }
+    get name () { return this.keys.name; }
+    get type () { return this.keys.type; }
+    get length () { return this.keys.length; }
+    get metadata () { return this.keys.metadata; }
+    get nullable () { return this.keys.nullable; }
+    get nullCount () { return this.keys.nullCount; }
+    get(index: number) {
+        return this.getValue(this.getKey(index)!);
+    }
+    getKey(index: number) {
+        return this.keys.get(index);
+    }
+    getValue(key: number) {
+        return this.data.get(key);
+    }
+    concat(...vectors: Vector<T>[]): Vector<T> {
+        return new VirtualVector(Array, this, ...vectors);
+    }
+    *[Symbol.iterator]() {
+        const { data, keys } = this;
+        for (let i = -1, n = keys.length; ++i < n;) {
+            yield data.get(keys.get(i)!);
+        }
+    }
+}
diff --git a/js/src/vector/struct.ts b/js/src/types/fixedsizelist.ts
similarity index 53%
rename from js/src/vector/struct.ts
rename to js/src/types/fixedsizelist.ts
index e59ac91e9cd08..6311d891d5058 100644
--- a/js/src/vector/struct.ts
+++ b/js/src/types/fixedsizelist.ts
@@ -15,25 +15,21 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Vector } from './vector';
-import { BitVector, ValidityArgs } from './typed';
+import { List, Vector } from './types';
+import { VirtualVector } from './vector/virtual';
 
-export class StructVector extends Vector<any[]> {
-    protected vectors: Vector<any>[];
-    constructor(validity: ValidityArgs, ...vectors: Vector<any>[]) {
+export class FixedSizeListVector<T, TArray extends List<T>> extends Vector<TArray> {
+    readonly listSize: number;
+    readonly values: Vector<T>;
+    constructor(argv: { listSize: number, values: Vector<T> }) {
         super();
-        this.vectors = vectors;
-        this.length = Math.max(0, ...vectors.map((v) => v.length));
-        validity && (this.validity = BitVector.from(validity));
+        this.values = argv.values;
+        this.listSize = argv.listSize;
     }
     get(index: number) {
-        return this.validity.get(index) ? this.vectors.map((v) => v.get(index)) : null;
+        return this.values.slice<TArray>(this.listSize * index, this.listSize * (index + 1));
     }
-    concat(vector: StructVector) {
-        return StructVector.from(this,
-            this.length + vector.length,
-            this.validity.concat(vector.validity),
-            ...this.vectors.map((v, i) => v.concat(vector.vectors[i]))
-        );
+    concat(...vectors: Vector<TArray>[]): Vector<TArray> {
+        return new VirtualVector(Array, this, ...vectors);
     }
 }
diff --git a/js/src/types/list.ts b/js/src/types/list.ts
new file mode 100644
index 0000000000000..ca9170b5908d5
--- /dev/null
+++ b/js/src/types/list.ts
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { List, Vector } from './types';
+import { VirtualVector } from './vector/virtual';
+
+export class ListVector<T, TArray extends List<T>> extends Vector<TArray> {
+    readonly offsets: Int32Array;
+    readonly values: Vector<T>;
+    constructor(argv: { offsets: Int32Array, values: Vector<T> }) {
+        super();
+        this.values = argv.values;
+        this.offsets = argv.offsets;
+    }
+    get(index: number) {
+        return this.values.slice<TArray>(this.offsets[index], this.offsets[index + 1]);
+    }
+    concat(...vectors: Vector<TArray>[]): Vector<TArray> {
+        return new VirtualVector(Array, this, ...vectors);
+    }
+}
\ No newline at end of file
diff --git a/js/src/types/table/from.ts b/js/src/types/table/from.ts
new file mode 100644
index 0000000000000..ae0755961eb7d
--- /dev/null
+++ b/js/src/types/table/from.ts
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Column } from '../types';
+import { TableVector } from './table';
+import { readBuffers } from '../../reader/arrow';
+
+export function fromBuffers(...bytes: Array<Uint8Array | Buffer | string>) {
+    let columns: Column<any>[] = null as any;
+    for (let vectors of readBuffers(...bytes)) {
+        columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i]) as Column<any>);
+    }
+    return new TableVector({ columns });
+}
+
+TableVector.from = fromBuffers;
+
+declare module './table' {
+    namespace TableVector { export let from: typeof fromBuffers; }
+}
\ No newline at end of file
diff --git a/js/src/types/table/row.ts b/js/src/types/table/row.ts
new file mode 100644
index 0000000000000..432cfd7364ba3
--- /dev/null
+++ b/js/src/types/table/row.ts
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Row, Vector, Struct } from '../types';
+import { VirtualVector } from '../vector/virtual';
+
+export class RowVector<T = any> extends Vector<T> implements Row<T> {
+    readonly row: number;
+    readonly length: number;
+    readonly table: Struct<T>;
+    [Symbol.toStringTag]() { return 'Row'; }
+    constructor(table: Struct<T>, row: number) {
+        super();
+        this.row = row;
+        this.table = table;
+        this.length = table.columns.length;
+    }
+    get(index: number) {
+        const col = this.table.columns[index];
+        return col ? col.get(this.row) as T : null;
+    }
+    col(key: string) {
+        const col = this.table.col(key);
+        return col ? col.get(this.row) as T : null;
+    }
+    *[Symbol.iterator]() {
+        const { row } = this;
+        for (const col of this.table.columns) {
+            yield col ? col.get(row) : null;
+        }
+    }
+    concat(...rows: Vector<T>[]): Vector<T> {
+        return new VirtualVector(Array, this, ...rows as any[]);
+    }
+    toArray() { return [...this]; }
+    toJSON() { return this.toArray(); }
+    toString() { return `Row [${this.length})` }
+    toObject(): Record<string, any> {
+        const { row } = this, map = Object.create(null);
+        for (const col of this.table.columns) {
+            if (col && col.name) {
+                map[col.name] = col.get(row);
+            }
+        }
+        return map;
+    }
+}
diff --git a/js/src/types/table/struct.ts b/js/src/types/table/struct.ts
new file mode 100644
index 0000000000000..de6a3a05692aa
--- /dev/null
+++ b/js/src/types/table/struct.ts
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { RowVector } from './row';
+import { toString } from './toString';
+import { VirtualVector } from '../vector/virtual';
+import { Row, Vector, Column, Struct } from '../types';
+
+export interface StructVector {
+    toString(): string;
+    toString(index: boolean): string;
+    toString(options: { index: boolean }): string;
+}
+
+export class StructVector<T = any> extends Vector<Row<T>> implements Struct<T> {
+    readonly length: number;
+    readonly columns: Column[];
+    constructor(argv: { columns: Column[] }) {
+        super();
+        this.columns = argv.columns || [];
+        if (!this.length) {
+            this.length = Math.max(...this.columns.map((col) => col.length)) | 0;
+        }
+    }
+    get(index: number): StructRow<T> {
+        return new StructRow(this, index);
+    }
+    col(name: string) {
+        return this.columns.find((col) => col.name === name) || null;
+    }
+    key(index: number) {
+        return this.columns[index] ? this.columns[index].name : null;
+    }
+    select(...columns: string[]) {
+        return new StructVector({ columns: columns.map((name) => this.col(name)!) });
+    }
+    concat(...structs: Vector<Row<T>>[]): Vector<Row<T>> {
+        return new VirtualVector(Array, this, ...structs as any[]);
+    }
+    toString(x?: any) {
+        return toString(this, x);
+    }
+}
+
+export class StructRow<T> extends RowVector<T> {
+    toString() {
+        return JSON.stringify(this);
+    }
+}
\ No newline at end of file
diff --git a/js/src/types/table/table.ts b/js/src/types/table/table.ts
new file mode 100644
index 0000000000000..d9074dec2d382
--- /dev/null
+++ b/js/src/types/table/table.ts
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { StructVector, StructRow } from './struct';
+
+export class TableVector<T> extends StructVector<T> {
+    get(index: number): TableRow<T> {
+        return new TableRow(this, index);
+    }
+}
+
+export class TableRow<T> extends StructRow<T> {
+    toString() {
+        return this.toArray().map((x) => JSON.stringify(x)).join(', ');
+    }
+}
\ No newline at end of file
diff --git a/js/src/types/table/toString.ts b/js/src/types/table/toString.ts
new file mode 100644
index 0000000000000..85b23ca246058
--- /dev/null
+++ b/js/src/types/table/toString.ts
@@ -0,0 +1,40 @@
+import { Struct } from '../types';
+
+export function toString<T>(source: Struct<T>, options?: any) {
+    const index = typeof options === 'object' ? options && !!options.index
+                : typeof options === 'boolean' ? !!options
+                : false;
+    const { length } = source;
+    if (length <= 0) { return ''; }
+    const rows = new Array(length + 1);
+    const maxColumnWidths = [] as number[];
+    rows[0] = source.columns.map((_, i) => source.key(i));
+    index && rows[0].unshift('Index');
+    for (let i = -1, n = rows.length - 1; ++i < n;) {
+        rows[i + 1] = [...source.get(i)!];
+        index && rows[i + 1].unshift(i);
+    }
+    // Pass one to convert to strings and count max column widths
+    for (let i = -1, n = rows.length; ++i < n;) {
+        const row = rows[i];
+        for (let j = -1, k = row.length; ++j < k;) {
+            const val = row[j] = `${row[j]}`;
+            maxColumnWidths[j] = !maxColumnWidths[j]
+                ? val.length
+                : Math.max(maxColumnWidths[j], val.length);
+        }
+    }
+    // Pass two to pad each one to max column width
+    for (let i = -1, n = rows.length; ++i < n;) {
+        const row = rows[i];
+        for (let j = -1, k = row.length; ++j < k;) {
+            row[j] = leftPad(row[j], ' ', maxColumnWidths[j]);
+        }
+        rows[i] = row.join(', ');
+    }
+    return rows.join('\n');
+}
+
+function leftPad(str: string, fill: string, n: number) {
+    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
+}
diff --git a/js/src/types/types.ts b/js/src/types/types.ts
new file mode 100644
index 0000000000000..f732bc0971f35
--- /dev/null
+++ b/js/src/types/types.ts
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Schema_ from '../format/Schema_generated';
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+
+export interface TypedArrayConstructor<T extends TypedArray = TypedArray> {
+    readonly prototype: T;
+    readonly BYTES_PER_ELEMENT: number;
+    new (length: number): T;
+    new (elements: Iterable<number>): T;
+    new (arrayOrArrayBuffer: ArrayLike<number> | ArrayBufferLike): T;
+    new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T;
+}
+
+export interface TypedArray extends Iterable<number> {
+    [index: number]: number;
+    readonly length: number;
+    readonly byteLength: number;
+    readonly byteOffset: number;
+    readonly buffer: ArrayBufferLike;
+    readonly BYTES_PER_ELEMENT: number;
+    [Symbol.iterator](): IterableIterator<number>;
+    slice(start?: number, end?: number): TypedArray;
+    subarray(begin: number, end?: number): TypedArray;
+    set(array: ArrayLike<number>, offset?: number): void;
+}
+
+export type FloatArray = Float32Array | Float64Array;
+export type IntArray = Int8Array | Int16Array | Int32Array;
+export type UintArray = Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array;
+
+export type List<T> = T[] | TypedArray;
+
+export interface Vector<T = any> extends Iterable<T | null> {
+    readonly length: number;
+    get(index: number): T | null;
+    concat(...vectors: Vector<T>[]): Vector<T>;
+    slice<R = T[]>(start?: number, end?: number): R;
+}
+
+export interface Row<T = any> extends Vector<T> {
+    col(key: string): T | null;
+}
+
+export interface Column<T = any> extends Vector<T> {
+    readonly name: string;
+    readonly type: string;
+    readonly nullable: boolean;
+    readonly nullCount: number;
+    readonly metadata: Map<string, string>;
+}
+
+export interface Struct<T = any> extends Vector<Row<T>> {
+    readonly columns: Column[];
+    key(key: number): string | null;
+    col(key: string): Column | null;
+    select(...columns: string[]): Struct<T>;
+    concat(...structs: Vector<Row<T>>[]): Vector<Row<T>>;
+}
+
+export class Vector<T = any> implements Vector<T> {
+    slice<R = T[]>(start?: number, end?: number): R {
+        let { length } = this, from = start! | 0;
+        let to = end === undefined ? length : Math.max(end | 0, from);
+        let result = new Array<T | null>(to - Math.min(from, to));
+        for (let i = -1, n = result.length; ++i < n;) {
+            result[i] = this.get(i + from);
+        }
+        return result as any;
+    }
+    *[Symbol.iterator]() {
+        for (let i = -1, n = this.length; ++i < n;) {
+            yield this.get(i);
+        }
+    }
+}
+
+(Vector.prototype as any).name = '';
+(Vector.prototype as any).type = Type[0];
+(Vector.prototype as any).stride = 1;
+(Vector.prototype as any).nullable = !1;
+(Vector.prototype as any).nullCount = 0;
+(Vector.prototype as any).metadata = new Map();
diff --git a/js/src/vector/dictionary.ts b/js/src/types/utf8.ts
similarity index 50%
rename from js/src/vector/dictionary.ts
rename to js/src/types/utf8.ts
index de811eaf5b050..178704f6161f5 100644
--- a/js/src/vector/dictionary.ts
+++ b/js/src/types/utf8.ts
@@ -15,37 +15,26 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Vector } from './vector';
+import { Vector } from './types';
+import { TextDecoder } from 'text-encoding-utf-8';
+import { VirtualVector } from './vector/virtual';
 
-export class DictionaryVector<T> extends Vector<T> {
-    protected data: Vector<T>;
-    protected keys: Vector<number>;
-    constructor(index: Vector<number>, dictionary: Vector<T>) {
+const decoder = new TextDecoder('utf-8');
+
+export class Utf8Vector extends Vector<string> {
+    readonly values: Vector<Uint8Array | null>;
+    constructor(argv: { values: Vector<Uint8Array | null> }) {
         super();
-        this.keys = index;
-        this.data = dictionary;
-        this.length = index && index.length || 0;
-    }
-    index(index: number) {
-        return this.keys.get(index);
-    }
-    value(index: number) {
-        return this.data.get(index);
+        this.values = argv.values;
     }
     get(index: number) {
-        return this.value(this.index(index));
+        const chars = this.getCodePoints(index);
+        return chars ? decoder.decode(chars) : null;
     }
-    concat(vector: DictionaryVector<T>) {
-        return DictionaryVector.from(this,
-            this.length + vector.length,
-            this.keys.concat(vector.keys),
-            this.data
-        );
+    getCodePoints(index: number) {
+        return this.values.get(index);
     }
-    *[Symbol.iterator]() {
-        let { data } = this;
-        for (const loc of this.keys) {
-            yield data.get(loc);
-        }
+    concat(...vectors: Vector<string>[]): Vector<string> {
+        return new VirtualVector(Array, this, ...vectors);
     }
 }
diff --git a/js/src/types/vector/bool.ts b/js/src/types/vector/bool.ts
new file mode 100644
index 0000000000000..b2eea81f87f05
--- /dev/null
+++ b/js/src/types/vector/bool.ts
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { TypedVector } from './typed';
+
+export interface BoolVector extends TypedVector<boolean, Uint8Array> {
+    set(index: number, value: boolean): void;
+}
+
+export class BoolVector extends TypedVector<boolean, Uint8Array> {
+    static pack = pack;
+    get(index: number) {
+        return (this.data[index >> 3] & 1 << index % 8) !== 0;
+    }
+    set(index: number, value: boolean) {
+        if (index > -1 === false) {
+            return;
+        } else if (value) {
+            this.data[index >> 3] |=  (1 << (index % 8));
+        } else {
+            this.data[index >> 3] &= ~(1 << (index % 8));
+        }
+    }
+}
+
+export function pack(values: Iterable<any>) {
+    let xs = [], n, i = 0;
+    let bit = 0, byte = 0;
+    for (const value of values) {
+        value && (byte |= 1 << bit);
+        if (++bit === 8) {
+            xs[i++] = byte;
+            byte = bit = 0;
+        }
+    }
+    if (i === 0 || bit > 0) { xs[i++] = byte; }
+    if (i % 8 && (n = i + 8 - i % 8)) {
+        do { xs[i] = 0; } while (++i < n);
+    }
+    return new Uint8Array(xs);
+}
diff --git a/js/src/types/vector/date.ts b/js/src/types/vector/date.ts
new file mode 100644
index 0000000000000..82dc82e64021f
--- /dev/null
+++ b/js/src/types/vector/date.ts
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { TypedVector } from './typed';
+
+export class DateVector extends TypedVector<Date, Uint32Array> {
+    get(index: number): Date {
+        return new Date(4294967296   * /* 2^32 */
+            (super.get(index * 2 + 1) as any) + /* high */
+            (super.get(index * 2) as any)       /*  low */
+        );
+    }
+}
+
+(DateVector.prototype as any).stride = 2;
diff --git a/js/src/types/vector/long.ts b/js/src/types/vector/long.ts
new file mode 100644
index 0000000000000..de8eb0c13710b
--- /dev/null
+++ b/js/src/types/vector/long.ts
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { TypedVector } from './typed';
+import { TypedArray } from '../types';
+import { flatbuffers } from 'flatbuffers';
+import Long = flatbuffers.Long;
+
+export class LongVector<T extends TypedArray> extends TypedVector<Long, T> {
+    get(index: number) {
+        return new Long(
+            super.get(index * 2) as any,    /*  low */
+            super.get(index * 2 + 1) as any /* high */
+        );
+    }
+}
+
+(LongVector.prototype as any).stride = 2;
+
+export class Int64Vector extends LongVector<Int32Array> {}
+export class Uint64Vector extends LongVector<Uint32Array> {}
\ No newline at end of file
diff --git a/js/src/types/vector/traits.ts b/js/src/types/vector/traits.ts
new file mode 100644
index 0000000000000..872c40b64d559
--- /dev/null
+++ b/js/src/types/vector/traits.ts
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { BoolVector } from './bool';
+import { Vector, Column } from '../types';
+import * as Schema_ from '../../format/Schema_generated';
+import * as Message_ from '../../format/Message_generated';
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+import Field = Schema_.org.apache.arrow.flatbuf.Field;
+import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+
+type Ctor<TArgv> = new (argv: TArgv) => Vector;
+
+export const nullableMixin = <T extends Vector, TArgv>(superclass: new (argv: TArgv) => T) =>
+    class extends (superclass as Ctor<TArgv>) {
+        readonly validity: Vector<boolean>;
+        constructor(argv: TArgv & { validity: Uint8Array }) {
+            super(argv);
+            this.validity = new BoolVector({ data: argv.validity });
+        }
+        get(index: number) {
+            return this.validity.get(index) ? super.get(index) : null;
+        }
+    };
+
+export const fieldMixin = <T extends Vector, TArgv>(superclass: new (argv: TArgv) => T) =>
+    class extends (superclass as Ctor<TArgv>) implements Column {
+        readonly field: Field;
+        readonly type: string;
+        readonly length: number;
+        readonly stride: number;
+        readonly nullable: boolean;
+        readonly nullCount: number;
+        readonly fieldNode: FieldNode;
+        constructor(argv: TArgv & { field: Field, fieldNode: FieldNode }) {
+            super(argv);
+            const { field, fieldNode } = argv;
+            this.field = field;
+            this.fieldNode = fieldNode;
+            this.nullable = field.nullable();
+            this.type = Type[field.typeType()];
+            this.nullCount = fieldNode.nullCount().low;
+            this.length = (fieldNode.length().low / this.stride) | 0;
+        }
+        get name() { return this.field.name()!; }
+        get metadata()  {
+            const { field } = this, data = new Map<string, string>();
+            for (let entry, key, i = -1, n = field && field.customMetadataLength() | 0; ++i < n;) {
+                if ((entry = field.customMetadata(i)) && (key = entry.key()) != null) {
+                    data.set(key, entry.value()!);
+                }
+            }
+            return data;
+        }
+    };
diff --git a/js/src/types/vector/typed.ts b/js/src/types/vector/typed.ts
new file mode 100644
index 0000000000000..fc093f2cb5f2a
--- /dev/null
+++ b/js/src/types/vector/typed.ts
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from '../types';
+import { VirtualVector } from './virtual';
+import { TypedArray, TypedArrayConstructor } from '../types';
+
+export interface TypedVector<T, TArray extends TypedArray> {
+    slice(start?: number, end?: number): TArray;
+}
+
+export class TypedVector<T, TArray extends TypedArray> extends Vector<T> {
+    readonly data: TArray;
+    readonly stride: number;
+    readonly length: number;
+    constructor(argv: { data: TArray } | TArray) {
+        super();
+        const data = ArrayBuffer.isView(argv) ? argv : argv.data;
+        this.length = ((this.data = data).length / this.stride) | 0;
+    }
+    get(index: number): T | null {
+        return this.data[index] as any;
+    }
+    concat(...vectors: Vector<T>[]): Vector<T> {
+        return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors);
+    }
+    slice(start?: number, end?: number) {
+        const { data, stride } = this, from = start! | 0;
+        const to = end === undefined ? data.length : Math.max(end | 0, from);
+        return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0);
+    }
+}
+
+(TypedVector.prototype as any).stride = 1;
+
+export class Int8Vector extends TypedVector<number, Int8Array> {}
+export class Int16Vector extends TypedVector<number, Int16Array> {}
+export class Int32Vector extends TypedVector<number, Int32Array> {}
+export class Uint8Vector extends TypedVector<number, Uint8Array> {}
+export class Uint16Vector extends TypedVector<number, Uint16Array> {}
+export class Uint32Vector extends TypedVector<number, Uint32Array> {}
+export class Float32Vector extends TypedVector<number, Float32Array> {}
+export class Float64Vector extends TypedVector<number, Float64Array> {}
diff --git a/js/src/types/vector/virtual.ts b/js/src/types/vector/virtual.ts
new file mode 100644
index 0000000000000..7f56012dc7d4b
--- /dev/null
+++ b/js/src/types/vector/virtual.ts
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { TypedVector } from './typed';
+import { Vector, Column, TypedArray, TypedArrayConstructor } from '../types';
+
+export class VirtualVector<T> implements Column<T> {
+    readonly name: string;
+    readonly type: string;
+    readonly length: number;
+    readonly vectors: Vector<T>[];
+    readonly offsets: Uint32Array;
+    readonly ArrayType: ArrayConstructor | TypedArrayConstructor;
+    constructor(ArrayType: ArrayConstructor | TypedArrayConstructor, ...vectors: Vector<T>[]) {
+        this.vectors = vectors;
+        this.ArrayType = ArrayType;
+        this.name = (vectors[0] as any).name;
+        this.type = (vectors[0] as any).type;
+        this.length = vectors.reduce((sum, vec) => sum + vec.length, 0);
+        this.offsets = Uint32Array.from(vectors.reduce((sums, vector, index) => [...sums, vector.length + sums[index]], [0]));
+    }
+    *[Symbol.iterator]() {
+        for (const vector of this.vectors) {
+            yield* vector;
+        }
+    }
+    get nullable() {
+        return (this.vectors as Column<T>[]).some((vec) => vec.nullable);
+    }
+    get nullCount() {
+        return (this.vectors as Column<T>[]).reduce((sum, v) => sum + v.nullCount | 0, 0);
+    }
+    get metadata() {
+        return new Map<string, string>(
+            (this.vectors as Column<T>[]).reduce((entries, v) => [
+                ...entries, ...v.metadata.entries()
+            ], [] as [string, string][])
+        );
+    }
+    get(index: number) {
+        return findIndex(this.offsets, index) ? this.vectors[_vector].get(_offset) : null;
+    }
+    concat(...vectors: Vector<T>[]) {
+        return new VirtualVector(this.ArrayType, ...this.vectors, ...vectors);
+    }
+    slice(begin?: number, end?: number) {
+        const ArrayType = this.ArrayType as any;
+        // clamp begin and end values between the virtual length
+        clampRange(this.length, begin!, end);
+        const from = _from, total = _total;
+        // find the start vector index and adjusted value index offset
+        if (!findIndex(this.offsets, from)) { return new ArrayType(0); }
+        const set = ArrayType === Array ? arraySet : typedArraySet as any;
+        let index = _vector, vectors = this.vectors as TypedVector<T, TypedArray>[];
+        let vector = vectors[index], source = vector.slice(_offset, _offset + total), target = source;
+        // Perf optimization: if the first slice contains all the values we're looking for,
+        // we don't have to copy values to a target Array. If we're slicing a TypedArray,
+        // this is a significant improvement as we avoid the memcpy 🎉
+        if ((source.length / vector.stride | 0) < total) {
+            let vectorsLength = vectors.length;
+            let count = 0, length = 0, sources = [];
+            do {
+                sources.push(source);
+                length += source.length;
+                count += (source.length / vector.stride | 0);
+            } while (
+                (count  < total) &&
+                (vector = vectors[index = (++index % vectorsLength)]) &&
+                (source = vector.slice(0, Math.min(vector.length, total - count)))
+            );
+            target = new ArrayType(length);
+            for (let i = -1, j = 0, n = sources.length; ++i < n;) {
+                j = set(sources[i], target, j);
+            }
+        }
+        return target;
+    }
+}
+
+let _from = -1, _total = -1;
+function clampRange(length: number, start: number, end?: number) {
+    let total = length, from = start || 0;
+    let to = end === end && typeof end == 'number' ? end : total;
+    if (to < 0) { to = total + to; }
+    if (from < 0) { from = total - (from * -1) % total; }
+    if (to < from) { from = to; to = start; }
+    _from = from;
+    _total = !isFinite(total = (to - from)) || total < 0 ? 0 : total;
+}
+
+let _offset = -1, _vector = -1;
+function findIndex(offsets: Uint32Array, index: number) {
+    let offset = 0, left = 0, middle = 0, right = offsets.length - 1;
+    while (index < offsets[right] && index >= (offset = offsets[left])) {
+        if (left + 1 === right) {
+            _vector = left;
+            _offset = index - offset;
+            return true;
+        }
+        middle = left + ((right - left) / 2) | 0;
+        index >= offsets[middle] ? (left = middle) : (right = middle);
+    }
+    return false;
+}
+
+function arraySet<T>(source: T[], target: T[], index: number) {
+    for (let i = 0, n = source.length; i < n;) {
+        target[index++] = source[i++];
+    }
+    return index;
+}
+
+function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
+    return target.set(source, index) || index + source.length;
+}
diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts
deleted file mode 100644
index 7360d968b0250..0000000000000
--- a/js/src/vector/list.ts
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from './vector';
-import { TextDecoder } from 'text-encoding';
-import { IndexVector, BitVector, ValidityArgs } from './typed';
-
-export class ListVectorBase<T> extends Vector<T> {
-    protected values: Vector<T>;
-    protected offsets: IndexVector;
-    constructor(validity: ValidityArgs, values: Vector<any>, offsets: IndexVector) {
-        super();
-        this.values = values;
-        this.offsets = offsets;
-        validity && (this.validity = BitVector.from(validity));
-    }
-    get(index: number) {
-        let batch, from, to, { offsets } = this;
-        if (!this.validity.get(index) ||
-            /* return null if `to` is null */
-            ((to = offsets.get(index + 1)) === null) || !(
-            /*
-            return null if `batch` is less than than 0. this check is placed
-            second to avoid creating the [from, batch] tuple if `to` is null
-            */
-            ([from, batch] = offsets.get(index, true) as number[]) && batch > -1)) {
-            return null;
-        }
-        return this.values.slice(from, to, batch) as any;
-    }
-    concat(vector: ListVectorBase<T>) {
-        return (this.constructor as typeof ListVectorBase).from(this,
-            this.length + vector.length,
-            this.validity.concat(vector.validity),
-            this.values.concat(vector.values),
-            this.offsets.concat(vector.offsets)
-        );
-    }
-    *[Symbol.iterator]() {
-        let v, r1, r2, { values } = this;
-        let it = this.offsets[Symbol.iterator]();
-        let iv = this.validity[Symbol.iterator]();
-        while (!(v = iv.next()).done && !(r1 = it.next()).done && !(r2 = it.next()).done) {
-            yield !v.value ? null : values.slice(r1.value[0], r2.value, r1.value[1]) as any;
-        }
-    }
-}
-
-export class ListVector<T> extends ListVectorBase<T[]> {}
-export class Utf8Vector extends ListVectorBase<string> {
-    protected static decoder = new TextDecoder(`utf-8`);
-    get(index: number) {
-        let chars = super.get(index) as any;
-        return chars ? Utf8Vector.decoder.decode(chars) : null;
-    }
-    *[Symbol.iterator]() {
-        let decoder = Utf8Vector.decoder;
-        for (const chars of super[Symbol.iterator]()) {
-            yield !chars ? null : decoder.decode(chars);
-        }
-    }
-}
-
-export class FixedSizeListVector<T> extends Vector<T[]> {
-    protected size: number;
-    protected values: Vector<T>;
-    constructor(size: number, validity: ValidityArgs, values: Vector<T>) {
-        super();
-        this.values = values;
-        this.size = Math.abs(size | 0) || 1;
-        validity && (this.validity = BitVector.from(validity));
-    }
-    get(index: number) {
-        return !this.validity.get(index) ? null : this.values.slice(
-            this.size * index, this.size * (index + 1)
-        ) as T[];
-    }
-    concat(vector: FixedSizeListVector<T>) {
-        return FixedSizeListVector.from(this,
-            this.length + vector.length,
-            this.size,
-            this.validity.concat(vector.validity),
-            this.values.concat(vector.values)
-        );
-    }
-    *[Symbol.iterator]() {
-        let v, i = -1;
-        let { size, length, values } = this;
-        let iv = this.validity[Symbol.iterator]();
-        while (!(v = iv.next()).done && ++i < length) {
-            yield !v.value ? null : values.slice(size * i, size * (i + 1)) as T[];
-        }
-    }
-}
diff --git a/js/src/vector/typed.ts b/js/src/vector/typed.ts
deleted file mode 100644
index b38812e07d065..0000000000000
--- a/js/src/vector/typed.ts
+++ /dev/null
@@ -1,326 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from './vector';
-import { flatbuffers } from 'flatbuffers';
-
-import Long = flatbuffers.Long;
-
-export type VArray<T = any> = {
-    [k: number]: T; length: number;
-    constructor: VArrayCtor<VArray<T>>;
-};
-
-export type VArrayCtor<VArray> = {
-    readonly prototype: VArray;
-    BYTES_PER_ELEMENT?: number;
-    new(...args: any[]): VArray;
-};
-
-export class VirtualVector<T, TArrayType = VArray<T>> extends Vector<T> {
-    protected lists: TArrayType[];
-    protected _arrayType: VArrayCtor<TArrayType>;
-    public get arrayType() { return this._arrayType; }
-    constructor(...lists: TArrayType[]) {
-        super();
-        this.lists = lists.filter(Boolean);
-    }
-    get(index: number): T {
-        /* inlined `findVirtual` impl */
-        let rows, length, lists = this.lists;
-        for (let batch = -1;
-            (rows = lists[++batch]) &&
-            (length = rows.length) <= index &&
-            0 <= (index -= length);) {}
-        return rows && -1 < index ? rows[index] : null;
-    }
-    protected range(from: number, total: number, batch?: number) {
-        /* inlined `findVirtual` impl */
-        let rows, local = from, length;
-        let { lists, _arrayType } = this;
-        for (batch = (batch || 0) - 1;
-            (rows = lists[++batch]) &&
-            (length = rows.length) <= local &&
-            0 <= (local -= length);) {}
-        if (rows && local > -1) {
-            let index = 0, listsLength = lists.length;
-            let set: any = Array.isArray(rows) ? arraySet : typedArraySet;
-            let slice = _arrayType['prototype']['subarray'] || _arrayType['prototype']['slice'];
-            let source = slice.call(rows, local, local + total), target = source;
-            // Perf optimization: if the first slice contains all the values we're looking for,
-            // we don't have to copy values to a target Array. If we're slicing a TypedArray,
-            // this is a significant improvement as we avoid the memcpy 🎉
-            if (source.length < total) {
-                target = new _arrayType(total);
-                while ((index = set(source, target, index)) < total) {
-                    rows = lists[batch = ((batch + 1) % listsLength)];
-                    source = slice.call(rows, 0, Math.min(rows.length, total - index));
-                }
-            }
-            return target as any;
-        }
-        return new _arrayType(0);
-    }
-    *[Symbol.iterator]() {
-        let index = -1, { lists, length } = this;
-        for (let outer = -1, n = lists.length; ++outer < n;) {
-            let list = lists[outer] as any;
-            for (let inner = -1, k = list.length; ++index < length && ++inner < k;) {
-                yield list[inner];
-            }
-        }
-    }
-}
-
-export type ValidityArgs = Vector<boolean> | Uint8Array;
-export class BitVector extends VirtualVector<boolean, Uint8Array> {
-    static constant: Vector<boolean> = new (class ValidVector extends Vector<boolean> {
-        get() { return true; }
-        *[Symbol.iterator]() {
-            do { yield true; } while (true);
-        }
-    })();
-    static from(src: any) {
-        return src instanceof BitVector   ? src
-             : src === BitVector.constant ? src
-             : src instanceof Uint8Array       ? new BitVector(src)
-             : src instanceof Array            ? new BitVector(BitVector.pack(src))
-             : src instanceof Vector           ? new BitVector(BitVector.pack(src))
-                                               : BitVector.constant as Vector<any>;
-    }
-    static pack(values: Iterable<any>) {
-        let xs = [], n, i = 0;
-        let bit = 0, byte = 0;
-        for (const value of values) {
-            value && (byte |= 1 << bit);
-            if (++bit === 8) {
-                xs[i++] = byte;
-                byte = bit = 0;
-            }
-        }
-        if (i === 0 || bit > 0) { xs[i++] = byte; }
-        if (i % 8 && (n = n = i + 8 - i % 8)) {
-            do { xs[i] = 0; } while (++i < n);
-        }
-        return new Uint8Array(xs);
-    }
-    constructor(...lists: Uint8Array[]) {
-        super(...lists);
-        this.length = this.lists.reduce((l, xs) => l + xs['length'], 0);
-    }
-    get(index: number) {
-        /* inlined `findVirtual` impl */
-        let rows, length, lists = this.lists;
-        for (let batch = -1;
-            (rows = lists[++batch]) &&
-            (length = rows.length) <= index &&
-            0 <= (index -= length);) {}
-        return !(!rows || index < 0 || (rows[index >> 3 | 0] & 1 << index % 8) === 0);
-    }
-    set(index: number, value: boolean) {
-        /* inlined `findVirtual` impl */
-        let rows, length, lists = this.lists;
-        for (let batch = -1;
-            (rows = lists[++batch]) &&
-            (length = rows.length) <= index &&
-            0 <= (index -= length);) {}
-        if (rows && index > -1) {
-            value
-                ? (rows[index >> 3 | 0] |=  (1 << (index % 8)))
-                : (rows[index >> 3 | 0] &= ~(1 << (index % 8)));
-        }
-    }
-    concat(vector: BitVector) {
-        return new BitVector(...this.lists, ...vector.lists);
-    }
-    *[Symbol.iterator]() {
-        for (const byte of super[Symbol.iterator]()) {
-            for (let i = -1; ++i < 8;) {
-                yield (byte & 1 << i) !== 0;
-            }
-        }
-    }
-}
-
-export class TypedVector<T, TArrayType> extends VirtualVector<T, TArrayType> {
-    constructor(validity: ValidityArgs, ...lists: TArrayType[]) {
-        super(...lists);
-        validity && (this.validity = BitVector.from(validity));
-    }
-    concat(vector: TypedVector<T, TArrayType>) {
-        return (this.constructor as typeof TypedVector).from(this,
-            this.length + vector.length,
-            this.validity.concat(vector.validity),
-            ...this.lists, ...vector.lists
-        );
-    }
-}
-
-export class DateVector extends TypedVector<Date, Uint32Array> {
-    get(index: number) {
-        return !this.validity.get(index) ? null : new Date(
-            Math.pow(2, 32) *
-                <any> super.get(2 * index + 1) +
-                <any> super.get(2 * index)
-        );
-    }
-    *[Symbol.iterator]() {
-        let v, low, high;
-        let it = super[Symbol.iterator]();
-        let iv = this.validity[Symbol.iterator]();
-        while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) {
-            yield !v.value ? null : new Date(Math.pow(2, 32) * high.value + low.value);
-        }
-    }
-}
-
-export class IndexVector extends TypedVector<number | number[], Int32Array> {
-    get(index: number, returnWithBatchIndex = false) {
-        /* inlined `findVirtual` impl */
-        let rows, length, batch = -1, lists = this.lists;
-        for (;
-            (rows = lists[++batch]) &&
-            (length = rows.length) <= index &&
-            0 <= (index -= length);) {}
-        return !returnWithBatchIndex
-            ? (rows && -1 < index ? rows[index + batch] : null) as number
-            : (rows && -1 < index ? [rows[index + batch], batch] : [0, -1]) as number[];
-    }
-    *[Symbol.iterator]() {
-        // Alternate between iterating a tuple of [from, batch], and to. The from
-        // and to values are relative to the record batch they're defined in, so
-        // `ListVectorBase` needs to know the right batch to read.
-        let xs = new Int32Array(2), { lists } = this;
-        for (let i = -1, n = lists.length; ++i < n;) {
-            let list = lists[i] as any;
-            for (let j = -1, k = list.length - 1; ++j < k;) {
-                xs[1] = i;
-                xs[0] = list[j];
-                yield xs;
-                yield list[j + 1];
-            }
-        }
-    }
-}
-
-export class ByteVector<TList> extends TypedVector<number, TList> {
-    get(index: number) {
-        return this.validity.get(index) ? super.get(index) : null;
-    }
-    *[Symbol.iterator]() {
-        let v, r, { validity } = this;
-        let it = super[Symbol.iterator]();
-        // fast path the case of no nulls
-        if (validity === BitVector.constant) {
-            yield* it;
-        } else {
-            let iv = validity[Symbol.iterator]();
-            while (!(v = iv.next()).done && !(r = it.next()).done) {
-                yield !v.value ? null : r.value;
-            }
-        }
-    }
-}
-
-export class LongVector<TList> extends TypedVector<Long, TList> {
-    get(index: number) {
-        return !this.validity.get(index) ? null : new Long(
-            <any> super.get(index * 2),     /* low */
-            <any> super.get(index * 2 + 1) /* high */
-        );
-    }
-    *[Symbol.iterator]() {
-        let v, low, high;
-        let it = super[Symbol.iterator]();
-        let iv = this.validity[Symbol.iterator]();
-        while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) {
-            yield !v.value ? null : new Long(low.value, high.value);
-        }
-    }
-}
-
-export class Int8Vector    extends ByteVector<Int8Array>    {}
-export class Int16Vector   extends ByteVector<Int16Array>   {}
-export class Int32Vector   extends ByteVector<Int32Array>   {}
-export class Int64Vector   extends LongVector<Int32Array>   {}
-export class Uint8Vector   extends ByteVector<Uint8Array>   {}
-export class Uint16Vector  extends ByteVector<Uint16Array>  {}
-export class Uint32Vector  extends ByteVector<Uint32Array>  {}
-export class Uint64Vector  extends LongVector<Uint32Array>  {}
-export class Float32Vector extends ByteVector<Float32Array> {}
-export class Float64Vector extends ByteVector<Float64Array> {}
-
-LongVector.prototype.stride = 2;
-(Vector.prototype as any).lists = [];
-(Vector.prototype as any).validity = BitVector.constant;
-(VirtualVector.prototype as any)._arrayType = Array;
-(BitVector.prototype as any)._arrayType = Uint8Array;
-(Int8Vector.prototype as any)._arrayType = Int8Array;
-(Int16Vector.prototype as any)._arrayType = Int16Array;
-(Int32Vector.prototype as any)._arrayType = Int32Array;
-(Int64Vector.prototype as any)._arrayType = Int32Array;
-(Uint8Vector.prototype as any)._arrayType = Uint8Array;
-(Uint16Vector.prototype as any)._arrayType = Uint16Array;
-(Uint32Vector.prototype as any)._arrayType = Uint32Array;
-(Uint64Vector.prototype as any)._arrayType = Uint32Array;
-(DateVector.prototype as any)._arrayType = Uint32Array;
-(IndexVector.prototype as any)._arrayType = Int32Array;
-(Float32Vector.prototype as any)._arrayType = Float32Array;
-(Float64Vector.prototype as any)._arrayType = Float64Array;
-
-function arraySet<T>(source: Array<T>, target: Array<T>, index: number) {
-    for (let i = 0, n = source.length; i < n;) {
-        target[index++] = source[i++];
-    }
-    return index;
-}
-
-function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
-    return target.set(source, index) || index + source.length;
-}
-
-// Rather than eat the iterator cost, we've inlined this function into the relevant functions
-// function* findVirtual<TList>(index: number, lists: TList[], batch?: number) {
-//     let rows, length;
-//     for (batch = (batch || 0) - 1;
-//         (rows = lists[++batch]) &&
-//         (length = rows.length) <= index &&
-//         0 <= (index -= length);) {}
-//     return rows && -1 < index ? yield [rows, index, batch] : null;
-// }
-
-export type TypedArrayCtor<T extends TypedArray> = {
-    readonly prototype: T;
-    readonly BYTES_PER_ELEMENT: number;
-    new(length: number): T;
-    new(array: ArrayLike<number>): T;
-    new(buffer: ArrayBufferLike, byteOffset?: number, length?: number): T;
-};
-
-export type FloatArray = Float32Array | Float64Array;
-export type IntArray = Int8Array | Int16Array | Int32Array | Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array;
-
-export type TypedArray = (
-            Int8Array        |
-            Uint8Array       |
-            Int16Array       |
-            Int32Array       |
-            Uint16Array      |
-            Uint32Array      |
-            Float32Array     |
-            Float64Array     |
-            Uint8ClampedArray);
diff --git a/js/src/vector/vector.ts b/js/src/vector/vector.ts
deleted file mode 100644
index 1f39f87cbefc8..0000000000000
--- a/js/src/vector/vector.ts
+++ /dev/null
@@ -1,91 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import * as Schema_ from '../format/Schema_generated';
-export import Type = Schema_.org.apache.arrow.flatbuf.Type;
-export import Field = Schema_.org.apache.arrow.flatbuf.Field;
-
-export function sliceToRangeArgs(length: number, start: number, end?: number) {
-    let total = length, from = start || 0;
-    let to = end === end && typeof end == 'number' ? end : total;
-    if (to < 0) { to = total + to; }
-    if (from < 0) { from = total - (from * -1) % total; }
-    if (to < from) { from = to; to = start; }
-    total = !isFinite(total = (to - from)) || total < 0 ? 0 : total;
-    return [from, total];
-}
-
-export class Vector<T> implements Iterable<T> {
-    static defaultName = '';
-    static defaultProps = new Map();
-    static defaultType = Type[Type.NONE];
-    static create<T = any>(field: Field, length: number, ...args: any[]) {
-        let vector = new this<T>(...args), m;
-        vector.length = length;
-        vector.name = field.name();
-        vector.type = Type[field.typeType()];
-        if ((m = field.customMetadataLength()) > 0) {
-            let entry, i = 0, data = vector.props = new Map();
-            do {
-                entry = field.customMetadata(i);
-                data[entry.key()] = entry.value();
-            } while (++i < m);
-        }
-        return vector;
-    }
-    static from<T = any>(source: Vector<T>, length: number, ...args: any[]) {
-        let vector = new this<T>(...args);
-        vector.length = length;
-        source.name !== Vector.defaultName && (vector.name = source.name);
-        source.type !== Vector.defaultType && (vector.type = source.type);
-        source.props !== Vector.defaultProps && (vector.props = source.props);
-        return vector;
-    }
-    public name: string;
-    public type: string;
-    public length: number;
-    public stride: number;
-    public props: Map<PropertyKey, any>;
-    protected validity: Vector<boolean>;
-    get(index: number): T { return null; }
-    concat(vector: Vector<T>) { return vector; }
-    slice<R = T>(start?: number, end?: number, batch?: number) {
-        const { stride } = this;
-        const [offset, length] = sliceToRangeArgs(
-            stride * this.length, stride * (start || 0), stride * end
-        );
-        return this.range<R>(offset, length, batch);
-    }
-    protected range<R = T>(index: number, length: number, batch?: number) {
-        const result = new Array<R>(length);
-        for (let i = -1, n = this.length; ++i < length;) {
-            result[i] = this.get((i + index) % n) as any;
-        }
-        return result as Iterable<R>;
-    }
-    *[Symbol.iterator]() {
-        for (let i = -1, n = this.length; ++i < n;) {
-            yield this.get(i);
-        }
-    }
-}
-
-Vector.prototype.length = 0;
-Vector.prototype.stride = 1;
-Vector.prototype.name = Vector.defaultName;
-Vector.prototype.type = Vector.defaultType;
-Vector.prototype.props = Vector.defaultProps;
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
index 3f29c5409ab26..722781db658dc 100644
--- a/js/test/Arrow.ts
+++ b/js/test/Arrow.ts
@@ -18,9 +18,9 @@
 /* tslint:disable */
 // Dynamically load an Ix target build based on command line arguments
 
-const target = process.env.TEST_TARGET;
-const format = process.env.TEST_MODULE;
-const resolve = require('path').resolve;
+const path = require('path');
+const target = process.env.TEST_TARGET!;
+const format = process.env.TEST_MODULE!;
 
 // these are duplicated in the gulpfile :<
 const targets = [`es5`, `es2015`, `esnext`];
@@ -30,21 +30,24 @@ function throwInvalidImportError(name: string, value: string, values: string[])
     throw new Error('Unrecognized ' + name + ' \'' + value + '\'. Please run tests with \'--' + name + ' <any of ' + values.join(', ') + '>\'');
 }
 
-if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets);
-if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats);
+let modulePath = ``;
 
-let Arrow: any = require(resolve(`./targets/${target}/${format}/Arrow.js`));
+if (target === `ts` || target === `apache-arrow`) modulePath = target;
+else if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets);
+else if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats);
+else modulePath = path.join(target, format);
+
+let Arrow: any = require(path.resolve(`./targets`, modulePath, `Arrow`));
 
 import {
     Table as Table_,
-    readBuffers as readBuffers_,
     Vector as Vector_,
-    BitVector as BitVector_,
+    readBuffers as readBuffers_,
+    BoolVector as BoolVector_,
+    TypedVector as TypedVector_,
     ListVector as ListVector_,
     Utf8Vector as Utf8Vector_,
     DateVector as DateVector_,
-    IndexVector as IndexVector_,
-    TypedVector as TypedVector_,
     Int8Vector as Int8Vector_,
     Int16Vector as Int16Vector_,
     Int32Vector as Int32Vector_,
@@ -61,14 +64,13 @@ import {
 } from '../src/Arrow';
 
 export let Table = Arrow.Table as typeof Table_;
-export let readBuffers = Arrow.readBuffers as typeof readBuffers_;
 export let Vector = Arrow.Vector as typeof Vector_;
-export let BitVector = Arrow.BitVector as typeof BitVector_;
+export let readBuffers = Arrow.readBuffers as typeof readBuffers_;
+export let BoolVector = Arrow.BoolVector as typeof BoolVector_;
+export let TypedVector = Arrow.TypedVector as typeof TypedVector_;
 export let ListVector = Arrow.ListVector as typeof ListVector_;
 export let Utf8Vector = Arrow.Utf8Vector as typeof Utf8Vector_;
 export let DateVector = Arrow.DateVector as typeof DateVector_;
-export let IndexVector = Arrow.IndexVector as typeof IndexVector_;
-export let TypedVector = Arrow.TypedVector as typeof TypedVector_;
 export let Int8Vector = Arrow.Int8Vector as typeof Int8Vector_;
 export let Int16Vector = Arrow.Int16Vector as typeof Int16Vector_;
 export let Int32Vector = Arrow.Int32Vector as typeof Int32Vector_;
diff --git a/js/test/__snapshots__/table-tests.ts.snap b/js/test/__snapshots__/table-tests.ts.snap
index a7fb9c5a64c24..401b992d9d7b5 100644
--- a/js/test/__snapshots__/table-tests.ts.snap
+++ b/js/test/__snapshots__/table-tests.ts.snap
@@ -118,19 +118,19 @@ Array [
 `;
 
 exports[`dictionary file Arrow Table toString() prints a pretty Table 1`] = `
-"                                       example-csv
-Hermione,25,-53.235599517822266,40.231998443603516
-                   Severus,30,-62.22999954223633,3
-                   Harry,20,23,-100.23652648925781"
+"                                                     example-csv
+[\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}]
+                   [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}]
+                   [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]"
 `;
 
 exports[`dictionary file Arrow Table toString() prints an empty Table 1`] = `""`;
 
 exports[`dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,                                        example-csv
-    0, Hermione,25,-53.235599517822266,40.231998443603516
-    1,                    Severus,30,-62.22999954223633,3
-    2,                    Harry,20,23,-100.23652648925781"
+"Index,                                                      example-csv
+    0, [\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}]
+    1,                    [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}]
+    2,                    [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]"
 `;
 
 exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 1`] = `"example-csv"`;
@@ -251,19 +251,19 @@ Array [
 `;
 
 exports[`dictionary stream Arrow Table toString() prints a pretty Table 1`] = `
-"                                       example-csv
-Hermione,25,-53.235599517822266,40.231998443603516
-                   Severus,30,-62.22999954223633,3
-                   Harry,20,23,-100.23652648925781"
+"                                                     example-csv
+[\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}]
+                   [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}]
+                   [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]"
 `;
 
 exports[`dictionary stream Arrow Table toString() prints an empty Table 1`] = `""`;
 
 exports[`dictionary stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,                                        example-csv
-    0, Hermione,25,-53.235599517822266,40.231998443603516
-    1,                    Severus,30,-62.22999954223633,3
-    2,                    Harry,20,23,-100.23652648925781"
+"Index,                                                      example-csv
+    0, [\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}]
+    1,                    [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}]
+    2,                    [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]"
 `;
 
 exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`;
@@ -353,17 +353,17 @@ Array [
 `;
 
 exports[`dictionary2 file Arrow Table toString() prints a pretty Table 1`] = `
-"                                                                                      struct
- a0fb47f9-f8fb-4403-a64a-786d7611f8ef,Airbus,1502880750,32.45663833618164,1.8712350130081177
-50fb46f4-fefa-42c1-919c-0121974cdd00,Boeing,1502880750,38.766666412353516,-4.181231498718262"
+"                                                                                                      struct
+ [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}]
+[\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]"
 `;
 
 exports[`dictionary2 file Arrow Table toString() prints an empty Table 1`] = `""`;
 
 exports[`dictionary2 file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,                                                                                       struct
-    0,  a0fb47f9-f8fb-4403-a64a-786d7611f8ef,Airbus,1502880750,32.45663833618164,1.8712350130081177
-    1, 50fb46f4-fefa-42c1-919c-0121974cdd00,Boeing,1502880750,38.766666412353516,-4.181231498718262"
+"Index,                                                                                                       struct
+    0,  [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}]
+    1, [\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]"
 `;
 
 exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`;
@@ -459,17 +459,17 @@ Array [
 `;
 
 exports[`multi_dictionary file Arrow Table toString() prints a pretty Table 1`] = `
-"                                                                                            struct
- a0fb47f9-f8fb-4403-a64a-786d7611f8ef,12345,Airbus,1502880750,32.45663833618164,1.8712350130081177
-50fb46f4-fefa-42c1-919c-0121974cdd00,67890,Boeing,1502880750,38.766666412353516,-4.181231498718262"
+"                                                                                                              struct
+ [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"12345\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}]
+[\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"67890\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]"
 `;
 
 exports[`multi_dictionary file Arrow Table toString() prints an empty Table 1`] = `""`;
 
 exports[`multi_dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,                                                                                             struct
-    0,  a0fb47f9-f8fb-4403-a64a-786d7611f8ef,12345,Airbus,1502880750,32.45663833618164,1.8712350130081177
-    1, 50fb46f4-fefa-42c1-919c-0121974cdd00,67890,Boeing,1502880750,38.766666412353516,-4.181231498718262"
+"Index,                                                                                                               struct
+    0,  [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"12345\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}]
+    1, [\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"67890\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]"
 `;
 
 exports[`multipart count Arrow Table creates a Table from Arrow buffers 1`] = `"row_count"`;
@@ -1373,47 +1373,47 @@ Array [
 `;
 
 exports[`struct file Arrow Table toString() prints a pretty Table 1`] = `
-"  struct_nullable
-             null
-         ,MhRNxD4
-137773603,3F9HBxK
-410361374,aVd88fp
-             null
-         ,3loZrRf
-             null
-             null
-                ,
-                ,
-             null
-         ,78SLiRw
-             null
-             null
-         ,0ilsf82
-         ,LjS9MbU
-                ,"
+"      struct_nullable
+                 null
+     [null,\\"MhRNxD4\\"]
+[137773603,\\"3F9HBxK\\"]
+[410361374,\\"aVd88fp\\"]
+                 null
+     [null,\\"3loZrRf\\"]
+                 null
+                 null
+          [null,null]
+          [null,null]
+                 null
+     [null,\\"78SLiRw\\"]
+                 null
+                 null
+     [null,\\"0ilsf82\\"]
+     [null,\\"LjS9MbU\\"]
+          [null,null]"
 `;
 
 exports[`struct file Arrow Table toString() prints an empty Table 1`] = `""`;
 
 exports[`struct file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,   struct_nullable
-    0,              null
-    1,          ,MhRNxD4
-    2, 137773603,3F9HBxK
-    3, 410361374,aVd88fp
-    4,              null
-    5,          ,3loZrRf
-    6,              null
-    7,              null
-    8,                 ,
-    9,                 ,
-   10,              null
-   11,          ,78SLiRw
-   12,              null
-   13,              null
-   14,          ,0ilsf82
-   15,          ,LjS9MbU
-   16,                 ,"
+"Index,       struct_nullable
+    0,                  null
+    1,      [null,\\"MhRNxD4\\"]
+    2, [137773603,\\"3F9HBxK\\"]
+    3, [410361374,\\"aVd88fp\\"]
+    4,                  null
+    5,      [null,\\"3loZrRf\\"]
+    6,                  null
+    7,                  null
+    8,           [null,null]
+    9,           [null,null]
+   10,                  null
+   11,      [null,\\"78SLiRw\\"]
+   12,                  null
+   13,                  null
+   14,      [null,\\"0ilsf82\\"]
+   15,      [null,\\"LjS9MbU\\"]
+   16,           [null,null]"
 `;
 
 exports[`struct stream Arrow Table creates a Table from Arrow buffers 1`] = `"struct_nullable"`;
@@ -1771,45 +1771,45 @@ Array [
 `;
 
 exports[`struct stream Arrow Table toString() prints a pretty Table 1`] = `
-"  struct_nullable
-             null
-         ,MhRNxD4
-137773603,3F9HBxK
-410361374,aVd88fp
-             null
-         ,3loZrRf
-             null
-             null
-                ,
-                ,
-             null
-         ,78SLiRw
-             null
-             null
-         ,0ilsf82
-         ,LjS9MbU
-                ,"
+"      struct_nullable
+                 null
+     [null,\\"MhRNxD4\\"]
+[137773603,\\"3F9HBxK\\"]
+[410361374,\\"aVd88fp\\"]
+                 null
+     [null,\\"3loZrRf\\"]
+                 null
+                 null
+          [null,null]
+          [null,null]
+                 null
+     [null,\\"78SLiRw\\"]
+                 null
+                 null
+     [null,\\"0ilsf82\\"]
+     [null,\\"LjS9MbU\\"]
+          [null,null]"
 `;
 
 exports[`struct stream Arrow Table toString() prints an empty Table 1`] = `""`;
 
 exports[`struct stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,   struct_nullable
-    0,              null
-    1,          ,MhRNxD4
-    2, 137773603,3F9HBxK
-    3, 410361374,aVd88fp
-    4,              null
-    5,          ,3loZrRf
-    6,              null
-    7,              null
-    8,                 ,
-    9,                 ,
-   10,              null
-   11,          ,78SLiRw
-   12,              null
-   13,              null
-   14,          ,0ilsf82
-   15,          ,LjS9MbU
-   16,                 ,"
+"Index,       struct_nullable
+    0,                  null
+    1,      [null,\\"MhRNxD4\\"]
+    2, [137773603,\\"3F9HBxK\\"]
+    3, [410361374,\\"aVd88fp\\"]
+    4,                  null
+    5,      [null,\\"3loZrRf\\"]
+    6,                  null
+    7,                  null
+    8,           [null,null]
+    9,           [null,null]
+   10,                  null
+   11,      [null,\\"78SLiRw\\"]
+   12,                  null
+   13,                  null
+   14,      [null,\\"0ilsf82\\"]
+   15,      [null,\\"LjS9MbU\\"]
+   16,           [null,null]"
 `;
diff --git a/js/test/table-tests.ts b/js/test/table-tests.ts
index c840299155af4..d0d70059e1561 100644
--- a/js/test/table-tests.ts
+++ b/js/test/table-tests.ts
@@ -23,7 +23,7 @@ for (let [name, ...buffers] of arrowTestConfigurations) {
         test(`creates a Table from Arrow buffers`, () => {
             expect.hasAssertions();
             const table = Table.from(...buffers);
-            for (const vector of table.cols()) {
+            for (const vector of table.columns) {
                 expect(vector.name).toMatchSnapshot();
                 expect(vector.type).toMatchSnapshot();
                 expect(vector.length).toMatchSnapshot();
@@ -35,7 +35,7 @@ for (let [name, ...buffers] of arrowTestConfigurations) {
         test(`vector iterators report the same values as get`, () => {
             expect.hasAssertions();
             const table = Table.from(...buffers);
-            for (const vector of table.cols()) {
+            for (const vector of table.columns) {
                 let i = -1, n = vector.length;
                 for (let v of vector) {
                     expect(++i).toBeLessThan(n);
@@ -46,12 +46,13 @@ for (let [name, ...buffers] of arrowTestConfigurations) {
         });
         test(`batch and Table Vectors report the same values`, () => {
             expect.hasAssertions();
-            let rowsTotal = 0, table = Table.from(...buffers);
+            let rowsTotal = 0;
+            let table = Table.from(...buffers);
             for (let vectors of readBuffers(...buffers)) {
                 let rowsNow = Math.max(...vectors.map((v) => v.length));
                 for (let vi = -1, vn = vectors.length; ++vi < vn;) {
                     let v1 = vectors[vi];
-                    let v2 = table.getColumnAt(vi);
+                    let v2 = table.columns[vi];
                     expect(v1.name).toEqual(v2.name);
                     expect(v1.type).toEqual(v2.type);
                     for (let i = -1, n = v1.length; ++i < n;) {
@@ -64,15 +65,15 @@ for (let [name, ...buffers] of arrowTestConfigurations) {
         test(`enumerates Table rows`, () => {
             expect.hasAssertions();
             const table = Table.from(...buffers);
-            for (const row of table.rows()) {
-                expect(row).toMatchSnapshot();
+            for (const row of table) {
+                expect(row!.toObject()).toMatchSnapshot();
             }
         });
         test(`enumerates Table rows compact`, () => {
             expect.hasAssertions();
             const table = Table.from(...buffers);
-            for (const row of table.rows(true)) {
-                expect(row).toMatchSnapshot();
+            for (const row of table) {
+                expect(row!.toArray()).toMatchSnapshot();
             }
         });
         test(`toString() prints an empty Table`, () => {
diff --git a/js/test/test-config.ts b/js/test/test-config.ts
index b31ff11ad4173..89de1cc6c70c9 100644
--- a/js/test/test-config.ts
+++ b/js/test/test-config.ts
@@ -20,7 +20,7 @@ import * as path from 'path';
 const arrowFormats = ['file', 'stream'];
 const arrowFileNames = ['simple', 'struct', 'dictionary', 'dictionary2', 'multi_dictionary'];
 const multipartArrows = ['count', 'latlong', 'origins'];
-export let arrowTestConfigurations = [];
+export let arrowTestConfigurations = [] as (string | Buffer)[][];
 
 arrowTestConfigurations = arrowFormats.reduce((configs, format) => {
     return arrowFileNames.reduce((configs, name) => {
diff --git a/js/test/vector-tests.ts b/js/test/vector-tests.ts
index 0c9ef4404ed6a..0eca2327bff43 100644
--- a/js/test/vector-tests.ts
+++ b/js/test/vector-tests.ts
@@ -18,7 +18,7 @@
 import { flatbuffers } from 'flatbuffers';
 import Long = flatbuffers.Long;
 import {
-    BitVector,
+    BoolVector,
     TypedVector,
     Int64Vector,
     Uint64Vector,
@@ -32,22 +32,34 @@ import {
     Float64Vector,
 } from './Arrow';
 
-const LongVectors = { Int64Vector, Uint64Vector };
-const ByteVectors = { Int8Vector, Int16Vector, Int32Vector, Uint8Vector, Uint16Vector, Uint32Vector, Float32Vector, Float64Vector };
+const LongVectors = {
+    Int64Vector: [Int64Vector, Int32Array],
+    Uint64Vector: [Uint64Vector, Uint32Array]
+};
 
-const longVectors = toMap<typeof TypedVector>(LongVectors, Object.keys(LongVectors));
-const byteVectors = toMap<typeof TypedVector>(ByteVectors, Object.keys(ByteVectors));
+const TypedVectors = {
+    Int8Vector: [Int8Vector, Int8Array],
+    Int16Vector: [Int16Vector, Int16Array],
+    Int32Vector: [Int32Vector, Int32Array],
+    Uint8Vector: [Uint8Vector, Uint8Array],
+    Uint16Vector: [Uint16Vector, Uint16Array],
+    Uint32Vector: [Uint32Vector, Uint32Array],
+    Float32Vector: [Float32Vector, Float32Array],
+    Float64Vector: [Float64Vector, Float64Array]
+};
+
+const longVectors = toMap<[typeof TypedVector, any]>(LongVectors, Object.keys(LongVectors));
+const byteVectors = toMap<[typeof TypedVector, any]>(TypedVectors, Object.keys(TypedVectors));
 const bytes = Array.from(
     { length: 5 },
     () => Uint8Array.from(
         { length: 64 },
         () => Math.random() * 255 | 0));
 
-describe(`BitVector`, () => {
-    const vector = new BitVector(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]));
+describe(`BoolVector`, () => {
+    const vector = new BoolVector({ data: new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]) });
     const values = [true, true, false, true, true, false, false, false];
     const n = values.length;
-    vector.length = 1;
     test(`gets expected values`, () => {
         let i = -1;
         while (++i < n) {
@@ -62,11 +74,11 @@ describe(`BitVector`, () => {
         }
     });
     test(`can set values to true and false`, () => {
-        const v = new BitVector(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]));
+        const v = new BoolVector({ data: new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]) });
         const expected1 = [true, true, false, true, true, false, false, false];
         const expected2 = [true, true,  true, true, true, false, false, false];
         const expected3 = [true, true, false, false, false, false, true, true];
-        function validate(expected) {
+        function validate(expected: boolean[]) {
             for (let i = -1; ++i < n;) {
                 expect(v.get(i)).toEqual(expected[i]);
             }
@@ -88,40 +100,42 @@ describe(`BitVector`, () => {
         validate(expected1);
     });
     test(`packs 0 values`, () => {
-        expect(BitVector.pack([])).toEqual(
+        expect(BoolVector.pack([])).toEqual(
             new Uint8Array([0, 0, 0, 0, 0, 0, 0, 0]));
     });
     test(`packs 3 values`, () => {
-        expect(BitVector.pack([
+        expect(BoolVector.pack([
             true, false, true
         ])).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
     });
     test(`packs 8 values`, () => {
-        expect(BitVector.pack([
+        expect(BoolVector.pack([
             true, true, false, true, true, false, false, false
         ])).toEqual(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]));
     });
     test(`packs 25 values`, () => {
-        expect(BitVector.pack([
+        expect(BoolVector.pack([
             true, true, false, true, true, false, false, false,
             false, false, false, true, true, false, true, true,
             false
         ])).toEqual(new Uint8Array([27, 216, 0, 0, 0, 0, 0, 0]));
     });
     test(`from with boolean Array packs values`, () => {
-        expect(BitVector.from([
-            true, false, true
-        ]).slice()).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
+        expect(new BoolVector({
+            data: BoolVector.pack([true, false, true])
+        }).slice()).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
     });
 });
 
-for (const [VectorName, VectorType] of longVectors) {
-    const ArrayType = VectorType.prototype.arrayType;
+for (const [VectorName, [VectorType, ArrayType]] of longVectors) {
     describe(`${VectorName}`, () => {
         const values = concatTyped(ArrayType, ...bytes);
-        const bLists = bytes.map((b) => new ArrayType(b.buffer));
-        const vector = new VectorType(null, ...bLists);
-        const n = vector.length = values.length * 0.5;
+        const vector = bytes
+            .map((b) => new VectorType<Long, any>({
+                data: new ArrayType(b.buffer)
+            }))
+            .reduce((v: any, v2) => v.concat(v2));
+        const n = values.length * 0.5;
         test(`gets expected values`, () => {
             let i = -1;
             while (++i < n) {
@@ -161,13 +175,16 @@ for (const [VectorName, VectorType] of longVectors) {
     });
 }
 
-for (const [VectorName, VectorType] of byteVectors) {
-    const ArrayType = VectorType.prototype.arrayType;
+for (const [VectorName, [VectorType, ArrayType]] of byteVectors) {
     describe(`${VectorName}`, () => {
         const values = concatTyped(ArrayType, ...bytes);
-        const bLists = bytes.map((b) => new ArrayType(b.buffer));
-        const vector = new VectorType(null, ...bLists);
-        const n = vector.length = values.length;
+        const vector = bytes
+            .map((b) => new VectorType<number, any>({
+                data: new ArrayType(b.buffer)
+            }))
+            .reduce((v: any, v2) => v.concat(v2));
+
+        const n = values.length;
         test(`gets expected values`, () => {
             let i = -1;
             while (++i < n) {
@@ -212,9 +229,9 @@ function toMap<T>(entries: any, keys: string[]) {
 }
 
 function concatTyped(ArrayType: any, ...bytes: any[]) {
-    const BPM = ArrayType.BYTES_PER_ELEMENT;
+    const BPE = ArrayType.BYTES_PER_ELEMENT;
     return bytes.reduce((v, bytes) => {
-        const l = bytes.byteLength / BPM;
+        const l = bytes.byteLength / BPE;
         const a = new ArrayType(v.length + l);
         const b = new ArrayType(bytes.buffer);
         a.set(v);
diff --git a/js/tsconfig/tsconfig.base.json b/js/tsconfig/tsconfig.base.json
index 4a46ed1f0af9b..8b8210198960a 100644
--- a/js/tsconfig/tsconfig.base.json
+++ b/js/tsconfig/tsconfig.base.json
@@ -3,23 +3,33 @@
   "include": ["../src/**/*.ts"],
   "compileOnSave": false,
   "compilerOptions": {
-    "lib": ["dom", "esnext", "esnext.asynciterable"],
+
+    /* Basic stuff */
     "moduleResolution": "node",
+    "lib": ["dom", "esnext", "esnext.asynciterable"],
+
+    /* Control what is emitted */
+    "declaration": true,
+    "noEmitOnError": true,
+    "removeComments": false,
+    "downlevelIteration": true,
+
+    /* Create inline sourcemaps with sources */
     "sourceMap": false,
     "inlineSources": true,
     "inlineSourceMap": true,
-    "declaration": true,
-    "skipLibCheck": true,
+
+    /* The most restrictive settings possible */
+    "strict": true,
+    "skipLibCheck": false,
     "importHelpers": true,
     "noEmitHelpers": true,
-    "noImplicitAny": false,
-    "noEmitOnError": false,
-    "noImplicitThis": true,
+    "noImplicitAny": true,
     "noUnusedLocals": true,
-    "removeComments": false,
-    "downlevelIteration": true,
-    "noImplicitUseStrict": true,
-    "preserveConstEnums": false,
+    "noImplicitReturns": true,
+    "allowUnusedLabels": false,
+    "noUnusedParameters": true,
+    "allowUnreachableCode": false,
     "noFallthroughCasesInSwitch": true,
     "forceConsistentCasingInFileNames": true
   }
diff --git a/js/tsconfig/tsconfig.es2015.cls.json b/js/tsconfig/tsconfig.es2015.cls.json
index 11ccc04d58375..fccacb349d023 100644
--- a/js/tsconfig/tsconfig.es2015.cls.json
+++ b/js/tsconfig/tsconfig.es2015.cls.json
@@ -4,7 +4,8 @@
   "compilerOptions": {
     "target": "ES2015",
     "module": "es2015",
+    "declaration": false,
     "noEmitHelpers": true,
-    "importHelpers": false
+    "importHelpers": true
   }
 }
diff --git a/js/tsconfig/tsconfig.es5.cls.json b/js/tsconfig/tsconfig.es5.cls.json
index 55f7ea52cf362..6e6f213b4d6e5 100644
--- a/js/tsconfig/tsconfig.es5.cls.json
+++ b/js/tsconfig/tsconfig.es5.cls.json
@@ -4,6 +4,7 @@
   "compilerOptions": {
     "target": "ES5",
     "module": "es2015",
+    "declaration": false,
     "noEmitHelpers": true,
     "importHelpers": false
   }
diff --git a/js/tsconfig/tsconfig.esnext.cls.json b/js/tsconfig/tsconfig.esnext.cls.json
index 009a5ac10d644..03206c9d77d38 100644
--- a/js/tsconfig/tsconfig.esnext.cls.json
+++ b/js/tsconfig/tsconfig.esnext.cls.json
@@ -4,7 +4,8 @@
   "compilerOptions": {
     "target": "ESNEXT",
     "module": "es2015",
+    "declaration": false,
     "noEmitHelpers": true,
-    "importHelpers": false
+    "importHelpers": true
   }
 }

From 82cd6e5158169eb3c782dba8991473e88a113181 Mon Sep 17 00:00:00 2001
From: dhirschf <david.hirschfeld@stanwell.com>
Date: Fri, 3 Nov 2017 17:51:52 -0400
Subject: [PATCH 1211/1644] ARROW-1764: [Python] Add -c conda-forge for Windows
 dev installation instructions

Author: dhirschf <david.hirschfeld@stanwell.com>
Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>
Author: Uwe L. Korn <xhochy@users.noreply.github.com>

Closes #1277 from xhochy/ARROW-1764 and squashes the following commits:

63143bba [Uwe L. Korn] Merge pull request #4 from dhirschfeld/gflags
bdb6b3f1 [dhirschf] Clarification of gflags channel
27f08961 [Korn, Uwe] ARROW-1764: [Python] Add -c conda-forge for Windows dev installation instructions
---
 cpp/apidoc/Windows.md             | 6 +++++-
 python/doc/source/development.rst | 3 ++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index 774482ea1c4f3..e7e83f1946b56 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -41,9 +41,13 @@ conda config --add channels conda-forge
 Now, you can bootstrap a build environment
 
 ```shell
-conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy zlib brotli gflags lz4-c zstd
+conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy zlib brotli gflags lz4-c zstd -c conda-forge
 ```
 
+***Note:***
+> *Make sure to get the `conda-forge` build of `gflags` as the
+  naming of the library differs from that in the `defaults` channel*
+
 Activate just created conda environment with pre-installed packages from
 previous step:
 
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 3ca460d463a06..7ef6a722be1dc 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -257,7 +257,8 @@ First, starting from fresh clones of Apache Arrow and parquet-cpp:
 .. code-block:: shell
 
    conda create -n arrow-dev cmake git boost-cpp ^
-         flatbuffers snappy zlib brotli thrift-cpp rapidjson
+         flatbuffers snappy zlib brotli thrift-cpp rapidjson ^
+         -c conda-forge
    activate arrow-dev
 
 As one git housekeeping item, we must run this command in our Arrow clone:

From 5d665762cd8c6ebbe94ce39b435a63ca4cf15967 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Fri, 3 Nov 2017 20:55:27 -0400
Subject: [PATCH 1212/1644] ARROW-1727: [Format] Expand Arrow streaming format
 to permit deltas / additions to existing dictionaries

Add an `isDelta` flag to the `DictionaryBatch` to allow for dictionary modifications mid-stream, update documentation.

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1257 from TheNeuralBit/ARROW-1727 and squashes the following commits:

c69a5539 [Brian Hulette] Documentation tweaks
3dff0a9c [Brian Hulette] Add isDelta flag to DictionaryBatch, update documentation
---
 format/IPC.md      | 45 ++++++++++++++++++++++++++++++++++++++++++++-
 format/Layout.md   |  6 +++---
 format/Message.fbs | 10 +++++++---
 3 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/format/IPC.md b/format/IPC.md
index 2f79031443b17..f3b48854c2072 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -67,7 +67,9 @@ We provide a streaming format for record batches. It is presented as a sequence
 of encapsulated messages, each of which follows the format above. The schema
 comes first in the stream, and it is the same for all of the record batches
 that follow. If any fields in the schema are dictionary-encoded, one or more
-`DictionaryBatch` messages will follow the schema.
+`DictionaryBatch` messages will be included. `DictionaryBatch` and
+`RecordBatch` messages may be interleaved, but before any dictionary key is used
+in a `RecordBatch` it should be defined in a `DictionaryBatch`.
 
 ```
 <SCHEMA>
@@ -76,6 +78,10 @@ that follow. If any fields in the schema are dictionary-encoded, one or more
 <DICTIONARY k - 1>
 <RECORD BATCH 0>
 ...
+<DICTIONARY x DELTA>
+...
+<DICTIONARY y DELTA>
+...
 <RECORD BATCH n - 1>
 <EOS [optional]: int32>
 ```
@@ -109,6 +115,10 @@ Schematically we have:
 <magic number "ARROW1">
 ```
 
+In the file format, there is no requirement that dictionary keys should be
+defined in a `DictionaryBatch` before they are used in a `RecordBatch`, as long
+as the keys are defined somewhere in the file.
+
 ### RecordBatch body structure
 
 The `RecordBatch` metadata contains a depth-first (pre-order) flattened set of
@@ -181,6 +191,7 @@ the dictionaries can be properly interpreted.
 table DictionaryBatch {
   id: long;
   data: RecordBatch;
+  isDelta: boolean = false;
 }
 ```
 
@@ -189,6 +200,38 @@ in the schema, so that dictionaries can even be used for multiple fields. See
 the [Physical Layout][4] document for more about the semantics of
 dictionary-encoded data.
 
+The dictionary `isDelta` flag allows dictionary batches to be modified
+mid-stream.  A dictionary batch with `isDelta` set indicates that its vector
+should be concatenated with those of any previous batches with the same `id`. A
+stream which encodes one column, the list of strings
+`["A", "B", "C", "B", "D", "C", "E", "A"]`, with a delta dictionary batch could
+take the form:
+
+```
+<SCHEMA>
+<DICTIONARY 0>
+(0) "A"
+(1) "B"
+(2) "C"
+
+<RECORD BATCH 0>
+0
+1
+2
+1
+
+<DICTIONARY 0 DELTA>
+(3) "D"
+(4) "E"
+
+<RECORD BATCH 1>
+3
+2
+4
+0
+EOS
+```
+
 ### Tensor (Multi-dimensional Array) Message Format
 
 The `Tensor` message types provides a way to write a multidimensional array of
diff --git a/format/Layout.md b/format/Layout.md
index ebf93821aab24..963202f9fb77a 100644
--- a/format/Layout.md
+++ b/format/Layout.md
@@ -615,9 +615,9 @@ the the types array indicates that a slot contains a different type at the index
 ## Dictionary encoding
 
 When a field is dictionary encoded, the values are represented by an array of Int32 representing the index of the value in the dictionary.
-The Dictionary is received as a DictionaryBatch whose id is referenced by a dictionary attribute defined in the metadata ([Message.fbs][7]) in the Field table.
-The dictionary has the same layout as the type of the field would dictate. Each entry in the dictionary can be accessed by its index in the DictionaryBatch.
-When a Schema references a Dictionary id, it must send a DictionaryBatch for this id before any RecordBatch.
+The Dictionary is received as one or more DictionaryBatches with the id referenced by a dictionary attribute defined in the metadata ([Message.fbs][7]) in the Field table.
+The dictionary has the same layout as the type of the field would dictate. Each entry in the dictionary can be accessed by its index in the DictionaryBatches.
+When a Schema references a Dictionary id, it must send at least one DictionaryBatch for this id.
 
 As an example, you could have the following data:
 ```
diff --git a/format/Message.fbs b/format/Message.fbs
index f4a95713cea93..830718139d88c 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -61,16 +61,20 @@ table RecordBatch {
   buffers: [Buffer];
 }
 
-/// ----------------------------------------------------------------------
 /// For sending dictionary encoding information. Any Field can be
 /// dictionary-encoded, but in this case none of its children may be
 /// dictionary-encoded.
-/// There is one vector / column per dictionary
-///
+/// There is one vector / column per dictionary, but that vector / column
+/// may be spread across multiple dictionary batches by using the isDelta
+/// flag
 
 table DictionaryBatch {
   id: long;
   data: RecordBatch;
+
+  /// If isDelta is true the values in the dictionary are to be appended to a
+  /// dictionary with the indicated id
+  isDelta: bool = false;
 }
 
 /// ----------------------------------------------------------------------

From b9a2ce9b277c7938775d51b919b4a6464be7a66f Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sat, 4 Nov 2017 14:24:12 +0100
Subject: [PATCH 1213/1644] ARROW-1765: [Doc] Use dependencies from conda in
 C++ docker build

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1278 from xhochy/ARROW-1765 and squashes the following commits:

f29ffaf [Korn, Uwe] ARROW-1765: [Doc] Use dependencies from conda in C++ docker build
---
 dev/gen_apidocs/Dockerfile          | 15 +++++----------
 dev/gen_apidocs/create_documents.sh |  8 ++++++--
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/dev/gen_apidocs/Dockerfile b/dev/gen_apidocs/Dockerfile
index 0b2844cc8454b..eaeb548184d61 100644
--- a/dev/gen_apidocs/Dockerfile
+++ b/dev/gen_apidocs/Dockerfile
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 FROM ubuntu:14.04
-ADD . /apache-arrow
-WORKDIR /apache-arrow
 # Prerequsites for apt-add-repository
 RUN apt-get update && apt-get install -y \
     software-properties-common python-software-properties
@@ -34,14 +32,6 @@ RUN wget -O /tmp/miniconda.sh \
     https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
     bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
     rm /tmp/miniconda.sh
-# C++ dependencies
-RUN /home/ubuntu/miniconda/bin/conda install -c conda-forge \
-    boost-cpp \
-    doxygen \
-    maven \
-    cmake \
-    zlib \
-    thrift-cpp
 # C_Glib dependencies
 RUN apt-get install -y \
     libgtk2.0-dev \
@@ -69,6 +59,7 @@ RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \
         six \
         setuptools \
         # C++
+        boost-cpp \
         cmake \
         flatbuffers \
         rapidjson \
@@ -79,5 +70,9 @@ RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \
         jemalloc \
         lz4-c \
         zstd \
+        doxygen \
+        maven \
         -c conda-forge
+ADD . /apache-arrow
+WORKDIR /apache-arrow
 CMD arrow/dev/gen_apidocs/create_documents.sh
diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh
index afbe041506d85..762b85222bd4d 100755
--- a/dev/gen_apidocs/create_documents.sh
+++ b/dev/gen_apidocs/create_documents.sh
@@ -16,6 +16,8 @@
 # limitations under the License.
 #
 
+set -ex
+
 # Set up environment and output directory for C++ libraries
 cd /apache-arrow
 rm -rf dist
@@ -25,8 +27,6 @@ export ARROW_HOME=$(pwd)/dist
 export PARQUET_HOME=$(pwd)/dist
 CONDA_BASE=/home/ubuntu/miniconda
 export LD_LIBRARY_PATH=$(pwd)/dist/lib:${CONDA_BASE}/lib:${LD_LIBRARY_PATH}
-export THRIFT_HOME=${CONDA_BASE}
-export BOOST_ROOT=${CONDA_BASE}
 export PATH=${CONDA_BASE}/bin:${PATH}
 
 # Prepare the asf-site before copying api docs
@@ -41,6 +41,10 @@ popd
 # Make Python documentation (Depends on C++ )
 # Build Arrow C++
 source activate pyarrow-dev
+
+export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
+export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX
+
 rm -rf arrow/cpp/build
 mkdir arrow/cpp/build
 pushd arrow/cpp/build

From fc7104fc4f42d77525fe15bb221bf80c181fc1d4 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sat, 4 Nov 2017 15:14:20 -0400
Subject: [PATCH 1214/1644] ARROW-1742: C++: clang-format is not detected
 correct on OSX anymore

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1281 from xhochy/ARROW-1742 and squashes the following commits:

89e9a767 [Korn, Uwe] ARROW-1742: C++: clang-format is not detected correct on OSX anymore
---
 cpp/CMakeLists.txt                     |  7 ++--
 cpp/build-support/run_clang_format.py  |  2 +-
 cpp/cmake_modules/FindClangTools.cmake | 51 +++++++++++++++++++++-----
 3 files changed, 45 insertions(+), 15 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d8dc5df88b4a4..5f0c431d54aca 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -48,6 +48,7 @@ set(ARROW_ABI_VERSION "${ARROW_SO_VERSION}.0.0")
 
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 
+set(CLANG_FORMAT_VERSION "4.0")
 find_package(ClangTools)
 if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
   # Generate a Clang compile_commands.json "compilation database" file for use
@@ -435,11 +436,9 @@ endif (UNIX)
 # "make format" and "make check-format" targets
 ############################################################
 
-set(CLANG_FORMAT_VERSION 4.0)
-
 # runs clang format and updates files in place.
 add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py
-  ${CLANG_FORMAT_VERSION}
+  ${CLANG_FORMAT_BIN}
   ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
   ${CMAKE_CURRENT_SOURCE_DIR}/src)
 
@@ -447,7 +446,7 @@ add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py
 
 # TODO(wesm): Make this work in run_clang_format.py
 add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py
-   ${CLANG_FORMAT_VERSION}
+   ${CLANG_FORMAT_BIN}
    ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
    ${CMAKE_CURRENT_SOURCE_DIR}/src 1)
 
diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py
index fcf39ecc6a5f9..6dec34bd09afe 100755
--- a/cpp/build-support/run_clang_format.py
+++ b/cpp/build-support/run_clang_format.py
@@ -27,7 +27,7 @@
                      sys.argv[0])
     sys.exit(1)
 
-CLANG_FORMAT = 'clang-format-{0}'.format(sys.argv[1])
+CLANG_FORMAT = sys.argv[1]
 EXCLUDE_GLOBS_FILENAME = sys.argv[2]
 SOURCE_DIR = sys.argv[3]
 
diff --git a/cpp/cmake_modules/FindClangTools.cmake b/cpp/cmake_modules/FindClangTools.cmake
index 0e9430ba29195..e9221ff22dc1f 100644
--- a/cpp/cmake_modules/FindClangTools.cmake
+++ b/cpp/cmake_modules/FindClangTools.cmake
@@ -49,16 +49,47 @@ else()
   message("clang-tidy found at ${CLANG_TIDY_BIN}")
 endif()
 
-find_program(CLANG_FORMAT_BIN
-  NAMES clang-format-4.0
-  clang-format-3.9
-  clang-format-3.8
-  clang-format-3.7
-  clang-format-3.6
-  clang-format
-  PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
-        NO_DEFAULT_PATH
-)
+if (CLANG_FORMAT_VERSION)
+    find_program(CLANG_FORMAT_BIN
+      NAMES clang-format-${CLANG_FORMAT_VERSION}
+      PATHS
+            ${ClangTools_PATH}
+            $ENV{CLANG_TOOLS_PATH}
+            /usr/local/bin /usr/bin
+            NO_DEFAULT_PATH
+    )
+
+    # If not found yet, search alternative locations
+    if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE)
+        # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/
+        STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}")
+        STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}")
+        if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0")
+            find_program(CLANG_FORMAT_BIN
+              NAMES clang-format
+              PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin
+                    NO_DEFAULT_PATH
+            )
+        else()
+            find_program(CLANG_FORMAT_BIN
+              NAMES clang-format
+              PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin
+                    NO_DEFAULT_PATH
+            )
+        endif()
+    endif()
+else()
+    find_program(CLANG_FORMAT_BIN
+      NAMES clang-format-4.0
+      clang-format-3.9
+      clang-format-3.8
+      clang-format-3.7
+      clang-format-3.6
+      clang-format
+      PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
+            NO_DEFAULT_PATH
+    )
+endif()
 
 if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
   set(CLANG_FORMAT_FOUND 0)

From 62190d7a5201cad6ae0b26d790942ffc8861eee9 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Sat, 4 Nov 2017 16:56:27 -0400
Subject: [PATCH 1215/1644] ARROW-1756: [Python] Fix large file read/write
 error

This is the part of [ARROW-1756](https://issues.apache.org/jira/browse/ARROW-1756).

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1276 from Licht-T/fix-large-file-read-write-error and squashes the following commits:

e21964a3 [Wes McKinney] Break in read IO loop when reaching EOF
8a68756b [Wes McKinney] Minor code tweaks, fix clang documentation warnings
81c19721 [Licht-T] TST: Add test for the large file read/write
9b71afee [Licht-T] ENH: Convert errno to string error message
fbb7eea6 [Licht-T] BUG: Fix large file read/write error
---
 cpp/src/arrow/buffer.h               |  2 +-
 cpp/src/arrow/compare.h              | 10 ++---
 cpp/src/arrow/io/file.cc             | 66 +++++++++++++++++++++++-----
 python/pyarrow/tests/conftest.py     | 14 +++++-
 python/pyarrow/tests/test_feather.py |  9 +++-
 5 files changed, 81 insertions(+), 20 deletions(-)

diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 8e989064be4e1..7c5f6174fa6e9 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -340,7 +340,7 @@ Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
 #ifndef ARROW_NO_DEPRECATED_API
 
 /// \brief Create Buffer referencing std::string memory
-/// \deprecated Since 0.8.0
+/// \note Deprecated since 0.8.0
 ///
 /// Warning: string instance must stay alive
 ///
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index 27176ed864cdd..df3386e4bfc19 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -33,27 +33,27 @@ class Tensor;
 
 #ifndef ARROW_NO_DEPRECATED_API
 /// Returns true if the arrays are exactly equal
-/// \deprecated Since 0.8.0
+/// \note Deprecated since 0.8.0
 Status ARROW_EXPORT ArrayEquals(const Array& left, const Array& right, bool* are_equal);
 
-/// \deprecated Since 0.8.0
+/// \note Deprecated since 0.8.0
 Status ARROW_EXPORT TensorEquals(const Tensor& left, const Tensor& right,
                                  bool* are_equal);
 
 /// Returns true if the arrays are approximately equal. For non-floating point
 /// types, this is equivalent to ArrayEquals(left, right)
-/// \deprecated Since 0.8.0
+/// \note Deprecated since 0.8.0
 Status ARROW_EXPORT ArrayApproxEquals(const Array& left, const Array& right,
                                       bool* are_equal);
 
 /// Returns true if indicated equal-length segment of arrays is exactly equal
-/// \deprecated Since 0.8.0
+/// \note Deprecated since 0.8.0
 Status ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right,
                                      int64_t start_idx, int64_t end_idx,
                                      int64_t other_start_idx, bool* are_equal);
 
 /// Returns true if the type metadata are exactly equal
-/// \deprecated Since 0.8.0
+/// \note Deprecated since 0.8.0
 Status ARROW_EXPORT TypeEquals(const DataType& left, const DataType& right,
                                bool* are_equal);
 #endif
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 74c6c09e6219b..057cad1111685 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -22,6 +22,21 @@
 
 #define _FILE_OFFSET_BITS 64
 
+// define max read/write count
+#if defined(_MSC_VER)
+#define ARROW_MAX_IO_CHUNKSIZE INT32_MAX
+#else
+
+#ifdef __APPLE__
+// due to macOS bug, we need to set read/write max
+#define ARROW_MAX_IO_CHUNKSIZE INT32_MAX
+#else
+// see notes on Linux read/write manpage
+#define ARROW_MAX_IO_CHUNKSIZE 0x7ffff000
+#endif
+
+#endif
+
 #include "arrow/io/file.h"
 
 #if _WIN32 || _WIN64
@@ -238,39 +253,68 @@ static inline Status FileSeek(int fd, int64_t pos) {
   return Status::OK();
 }
 
-static inline Status FileRead(int fd, uint8_t* buffer, int64_t nbytes,
+static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t nbytes,
                               int64_t* bytes_read) {
 #if defined(_MSC_VER)
-  if (nbytes > INT32_MAX) {
+  if (nbytes > ARROW_MAX_IO_CHUNKSIZE) {
     return Status::IOError("Unable to read > 2GB blocks yet");
   }
   *bytes_read = static_cast<int64_t>(_read(fd, buffer, static_cast<uint32_t>(nbytes)));
 #else
-  *bytes_read = static_cast<int64_t>(read(fd, buffer, static_cast<size_t>(nbytes)));
+  *bytes_read = 0;
+
+  while (*bytes_read != -1 && *bytes_read < nbytes) {
+    int64_t chunksize =
+        std::min(static_cast<int64_t>(ARROW_MAX_IO_CHUNKSIZE), nbytes - *bytes_read);
+    int64_t ret = static_cast<int64_t>(
+        read(fd, buffer + *bytes_read, static_cast<size_t>(chunksize)));
+
+    if (ret != -1) {
+      *bytes_read += ret;
+      if (ret < chunksize) {
+        // EOF
+        break;
+      }
+    } else {
+      *bytes_read = ret;
+    }
+  }
 #endif
 
   if (*bytes_read == -1) {
-    // TODO(wesm): errno to string
-    return Status::IOError("Error reading bytes from file");
+    return Status::IOError(std::string("Error reading bytes from file: ") +
+                           std::string(strerror(errno)));
   }
 
   return Status::OK();
 }
 
-static inline Status FileWrite(int fd, const uint8_t* buffer, int64_t nbytes) {
-  int ret;
+static inline Status FileWrite(const int fd, const uint8_t* buffer,
+                               const int64_t nbytes) {
+  int ret = 0;
 #if defined(_MSC_VER)
-  if (nbytes > INT32_MAX) {
+  if (nbytes > ARROW_MAX_IO_CHUNKSIZE) {
     return Status::IOError("Unable to write > 2GB blocks to file yet");
   }
   ret = static_cast<int>(_write(fd, buffer, static_cast<uint32_t>(nbytes)));
 #else
-  ret = static_cast<int>(write(fd, buffer, static_cast<size_t>(nbytes)));
+  int64_t bytes_written = 0;
+
+  while (ret != -1 && bytes_written < nbytes) {
+    int64_t chunksize =
+        std::min(static_cast<int64_t>(ARROW_MAX_IO_CHUNKSIZE), nbytes - bytes_written);
+    ret = static_cast<int>(
+        write(fd, buffer + bytes_written, static_cast<size_t>(chunksize)));
+
+    if (ret != -1) {
+      bytes_written += ret;
+    }
+  }
 #endif
 
   if (ret == -1) {
-    // TODO(wesm): errno to string
-    return Status::IOError("Error writing bytes to file");
+    return Status::IOError(std::string("Error writing bytes from file: ") +
+                           std::string(strerror(errno)));
   }
   return Status::OK();
 }
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index c6bd6c9b3a2d7..e27682232a22d 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pytest import skip
+from pytest import skip, mark
 
 
 groups = [
@@ -70,6 +70,18 @@ def pytest_addoption(parser):
                          default=False,
                          help=('Run only the {0} test group'.format(group)))
 
+    parser.addoption('--runslow', action='store_true',
+                     default=False, help='run slow tests')
+
+
+def pytest_collection_modifyitems(config, items):
+    if not config.getoption('--runslow'):
+        skip_slow = mark.skip(reason='need --runslow option to run')
+
+        for item in items:
+            if 'slow' in item.keywords:
+                item.add_marker(skip_slow)
+
 
 def pytest_runtest_setup(item):
     only_set = False
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index 9e7fc8863e759..b0764fdec1768 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -50,7 +50,7 @@ def tearDown(self):
                 pass
 
     def test_file_not_exist(self):
-        with self.assertRaises(pa.ArrowIOError):
+        with pytest.raises(pa.ArrowIOError):
             FeatherReader('test_invalid_file')
 
     def _get_null_counts(self, path, columns=None):
@@ -98,7 +98,7 @@ def _assert_error_on_write(self, df, exc, path=None):
         def f():
             write_feather(df, path)
 
-        self.assertRaises(exc, f)
+        pytest.raises(exc, f)
 
     def test_num_rows_attr(self):
         df = pd.DataFrame({'foo': [1, 2, 3, 4, 5]})
@@ -466,3 +466,8 @@ def test_unsupported(self):
         # non-strings
         df = pd.DataFrame({'a': ['a', 1, 2.0]})
         self._assert_error_on_write(df, ValueError)
+
+    @pytest.mark.slow
+    def test_large_dataframe(self):
+        df = pd.DataFrame({'A': np.arange(400000000)})
+        self._check_pandas_roundtrip(df)

From b513c8d2047c6ba7dea0530f436ca80f74809530 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 5 Nov 2017 12:52:38 +0100
Subject: [PATCH 1216/1644] ARROW-1762: [C++] Add note to readme about need to
 set LC_ALL on some Linux systems

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1284 from wesm/ARROW-1762 and squashes the following commits:

b71cf40 [Wes McKinney] Add note to readme about need to set LC_ALL on some Linux systems
---
 cpp/README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cpp/README.md b/cpp/README.md
index 60383535b1596..2034d29fae324 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -69,6 +69,14 @@ Simple release build:
 
 Detailed unit test logs will be placed in the build directory under `build/test-logs`.
 
+On some Linux distributions, running the test suite might require setting an
+explicit locale. If you see any locale-related errors, try setting the
+environment variable (which requires the `locales` package or equivalent):
+
+```
+export LC_ALL="en_US.UTF-8"
+```
+
 ### Statically linking to Arrow on Windows
 
 The Arrow headers on Windows static library builds (enabled by the CMake

From ea4a8f5a01c0c028bbec1e199ca70efaffbf068b Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Sun, 5 Nov 2017 13:32:13 +0100
Subject: [PATCH 1217/1644] ARROW-1714: [Python] Fix invalid
 serialization/deserialization None name Series

This closes [ARROW-1714](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1714).

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1263 from Licht-T/fix-invaid-conversion-none-column-name and squashes the following commits:

3afa60d [Wes McKinney] Be robust to pandas 0.21 conventions for null column labels
b353260 [Wes McKinney] Don't use locals()
fe6a075 [Licht-T] TST: Add test for None name Series serialization
3535dc4 [Licht-T] BUG: Fix invalid deserialization of None column name
---
 python/pyarrow/pandas_compat.py  | 52 ++++++++++++++++++++++++--------
 python/pyarrow/table.pxi         |  5 ++-
 python/pyarrow/tests/test_ipc.py |  9 +++++-
 3 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 1984598ff3533..87b47b8a6bc13 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -263,6 +263,8 @@ def _column_name_to_strings(name):
         return tuple(map(_column_name_to_strings, name))
     elif isinstance(name, collections.Sequence):
         raise TypeError("Unsupported type for MultiIndex level")
+    elif name is None:
+        return None
     return str(name)
 
 
@@ -280,7 +282,9 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1):
     for name in df.columns:
         col = df[name]
         if not isinstance(name, six.string_types):
-            name = str(_column_name_to_strings(name))
+            name = _column_name_to_strings(name)
+            if name is not None:
+                name = str(name)
 
         if schema is not None:
             field = schema.field_by_name(name)
@@ -361,6 +365,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     schema = table.schema
     row_count = table.num_rows
     metadata = schema.metadata
+    columns_metadata = None
 
     has_pandas_metadata = metadata is not None and b'pandas' in metadata
 
@@ -370,6 +375,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
         columns = pandas_metadata['columns']
         column_indexes = pandas_metadata.get('column_indexes', [])
         table = _add_any_metadata(table, pandas_metadata)
+        columns_metadata = pandas_metadata.get('columns', None)
 
     block_table = table
 
@@ -428,6 +434,18 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
         index = pd.RangeIndex(row_count)
 
     column_strings = [x.name for x in block_table.itercolumns()]
+    if columns_metadata is not None:
+        columns_name_dict = dict(
+            (str(x['name']), x['name'])
+            for x in columns_metadata
+        )
+        columns_values = [
+            columns_name_dict[y]
+            if y in columns_name_dict.keys() else y
+            for y in column_strings
+        ]
+    else:
+        columns_values = column_strings
 
     # If we're passed multiple column indexes then evaluate with
     # ast.literal_eval, since the column index values show up as a list of
@@ -437,11 +455,11 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     # Create the column index
 
     # Construct the base index
-    if not column_strings:
-        columns = pd.Index(column_strings)
+    if not columns_values:
+        columns = pd.Index(columns_values)
     else:
         columns = pd.MultiIndex.from_tuples(
-            list(map(to_pair, column_strings)),
+            list(map(to_pair, columns_values)),
             names=[col_index['name'] for col_index in column_indexes] or None,
         )
 
@@ -466,25 +484,35 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
             _level if _level.dtype == _dtype else _level.astype(_dtype)
             for _level, _dtype in levels_dtypes
         ]
+
         columns = pd.MultiIndex(
             levels=new_levels,
             labels=labels,
             names=columns.names
         )
 
-    # flatten a single level column MultiIndex for pandas 0.21.0 :(
-    if isinstance(columns, pd.MultiIndex) and columns.nlevels == 1:
-        levels, = columns.levels
-        labels, = columns.labels
-
-        # Cheaply check that we do not somehow have duplicate column names
-        assert len(levels) == len(labels), 'Found non-unique column index'
-        columns = levels[labels]
+    # ARROW-1751: flatten a single level column MultiIndex for pandas 0.21.0
+    columns = _flatten_single_level_multiindex(columns)
 
     axes = [columns, index]
     return _int.BlockManager(blocks, axes)
 
 
+def _flatten_single_level_multiindex(index):
+    if isinstance(index, pd.MultiIndex) and index.nlevels == 1:
+        levels, = index.levels
+        labels, = index.labels
+
+        # Cheaply check that we do not somehow have duplicate column names
+        if not index.is_unique:
+            raise ValueError('Found non-unique column index')
+
+        return pd.Index([levels[_label] if _label != -1 else None
+                         for _label in labels],
+                        name=index.names[0])
+    return index
+
+
 def _add_any_metadata(table, pandas_metadata):
     modified_columns = {}
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 6165a6622b836..5ba5f83d22f90 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -345,7 +345,10 @@ cdef _schema_from_arrays(arrays, names, dict metadata,
             else:
                 raise TypeError(type(val))
 
-            c_name = tobytes(names[i])
+            if names[i] is None:
+                c_name = tobytes(u'None')
+            else:
+                c_name = tobytes(names[i])
             fields[i].reset(new CField(c_name, type_, True))
 
     schema.reset(new CSchema(fields, unbox_metadata(metadata)))
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 68c0c80aa6187..5033ea95783ab 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -432,16 +432,23 @@ def test_serialize_pandas_no_preserve_index():
 
 def test_serialize_with_pandas_objects():
     df = pd.DataFrame({'a': [1, 2, 3]}, index=[1, 2, 3])
+    s = pd.Series([1, 2, 3, 4])
 
     data = {
         'a_series': df['a'],
-        'a_frame': df
+        'a_frame': df,
+        's_series': s
     }
 
     serialized = pa.serialize(data).to_buffer()
     deserialized = pa.deserialize(serialized)
     assert_frame_equal(deserialized['a_frame'], df)
+
     assert_series_equal(deserialized['a_series'], df['a'])
+    assert deserialized['a_series'].name == 'a'
+
+    assert_series_equal(deserialized['s_series'], s)
+    assert deserialized['s_series'].name is None
 
 
 def test_schema_batch_serialize_methods():

From 1ee73ef4bedc5f63b909421b1ad247e086689a74 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 5 Nov 2017 18:26:06 -0500
Subject: [PATCH 1218/1644] ARROW-1770: [GLib] Fix GLib compiler warning

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1287 from cpcloud/ARROW-1770 and squashes the following commits:

bb406961 [Phillip Cloud] ARROW-1770: [GLib] Fix GLib compiler warning
---
 c_glib/arrow-glib/input-stream.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index d628baeeeae5b..a7a894b9d4f3e 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -437,13 +437,13 @@ namespace garrow {
     }
 
     arrow::Status ReadAt(int64_t position, int64_t n_bytes,
-			 int64_t *n_read_bytes, uint8_t* out) {
+			 int64_t *n_read_bytes, uint8_t* out) override {
 	return arrow::io::RandomAccessFile::ReadAt(
 	    position, n_bytes, n_read_bytes, out);
     }
 
     arrow::Status ReadAt(int64_t position, int64_t n_bytes,
-			 std::shared_ptr<arrow::Buffer>* out) {
+			 std::shared_ptr<arrow::Buffer>* out) override {
 	return arrow::io::RandomAccessFile::ReadAt(position, n_bytes, out);
     }
 

From d7f1398a050b07dde3949e03792e5c477e4abfe4 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 5 Nov 2017 18:50:50 -0500
Subject: [PATCH 1219/1644] ARROW-1749: [C++] Handle range of Decimal128 values
 that require 39 digits to be displayed

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1282 from cpcloud/ARROW-1749 and squashes the following commits:

858a6f1d [Phillip Cloud] Move _BitScanReverse to the correct file
71b0d88a [Phillip Cloud] Move bit utility to bitutil
439fd2e9 [Phillip Cloud] Fold the constant
2084065d [Phillip Cloud] Remove unused include
0c3df97c [Phillip Cloud] Try things
f577612c [Phillip Cloud] Variable name
0eef9579 [Phillip Cloud] ARROW-1749: [C++] Handle range of Decimal128 values that require 39 digits to be displayed
---
 cpp/src/arrow/array.cc                  |   4 +-
 cpp/src/arrow/python/arrow_to_pandas.cc |   8 +-
 cpp/src/arrow/util/bit-util.h           |  22 +++
 cpp/src/arrow/util/decimal-test.cc      |  39 ++++-
 cpp/src/arrow/util/decimal.cc           | 213 ++++++++++++++----------
 cpp/src/arrow/util/decimal.h            |  24 ++-
 6 files changed, 204 insertions(+), 106 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index fc4b96e1b2bec..b523876bf0e4e 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -321,8 +321,8 @@ DecimalArray::DecimalArray(const std::shared_ptr<ArrayData>& data)
 
 std::string DecimalArray::FormatValue(int64_t i) const {
   const auto& type_ = static_cast<const DecimalType&>(*type());
-  Decimal128 value(GetValue(i));
-  return value.ToString(type_.precision(), type_.scale());
+  const Decimal128 value(GetValue(i));
+  return value.ToString(type_.scale());
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index c92faede1347b..3894772daa467 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -616,11 +616,10 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
   return Status::OK();
 }
 
-static Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
-                                 std::string* result) {
+static Status RawDecimalToString(const uint8_t* bytes, int scale, std::string* result) {
   DCHECK_NE(result, nullptr);
   Decimal128 decimal(bytes);
-  *result = decimal.ToString(precision, scale);
+  *result = decimal.ToString(scale);
   return Status::OK();
 }
 
@@ -636,7 +635,6 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
   for (int c = 0; c < data.num_chunks(); c++) {
     auto* arr(static_cast<arrow::DecimalArray*>(data.chunk(c).get()));
     auto type(std::dynamic_pointer_cast<arrow::DecimalType>(arr->type()));
-    const int precision = type->precision();
     const int scale = type->scale();
 
     for (int64_t i = 0; i < arr->length(); ++i) {
@@ -646,7 +644,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
       } else {
         const uint8_t* raw_value = arr->GetValue(i);
         std::string decimal_string;
-        RETURN_NOT_OK(RawDecimalToString(raw_value, precision, scale, &decimal_string));
+        RETURN_NOT_OK(RawDecimalToString(raw_value, scale, &decimal_string));
         *out_values++ = internal::DecimalFromString(Decimal, decimal_string);
         RETURN_IF_PYERROR();
       }
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 8043f90ccdf6a..d6415f3c75b7b 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -43,6 +43,8 @@
 #endif
 
 #if defined(_MSC_VER)
+#include <intrin.h>
+#pragma intrinsic(_BitScanReverse)
 #define ARROW_BYTE_SWAP64 _byteswap_uint64
 #define ARROW_BYTE_SWAP32 _byteswap_ulong
 #else
@@ -55,6 +57,7 @@
 #include <memory>
 #include <vector>
 
+#include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/type_traits.h"
 #include "arrow/util/visibility.h"
@@ -296,6 +299,25 @@ static inline int Log2(uint64_t x) {
   return result;
 }
 
+/// \brief Count the number of leading zeros in a 32 bit integer.
+static inline int64_t CountLeadingZeros(uint32_t value) {
+  DCHECK_NE(value, 0);
+#if defined(__clang__) || defined(__GNUC__)
+  return static_cast<int64_t>(__builtin_clz(value));
+#elif defined(_MSC_VER)
+  unsigned long index;                                         // NOLINT
+  _BitScanReverse(&index, static_cast<unsigned long>(value));  // NOLINT
+  return 31LL - static_cast<int64_t>(index);
+#else
+  int64_t bitpos = 0;
+  while (value != 0) {
+    value >>= 1;
+    ++bitpos;
+  }
+  return 32LL - bitpos;
+#endif
+}
+
 /// Swaps the byte order (i.e. endianess)
 static inline int64_t ByteSwap(int64_t value) { return ARROW_BYTE_SWAP64(value); }
 static inline uint64_t ByteSwap(uint64_t value) {
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 565a1bbb939b6..b0271fff15ccf 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -36,9 +36,8 @@ class DecimalTestFixture : public ::testing::Test {
 
 TEST_F(DecimalTestFixture, TestToString) {
   Decimal128 decimal(this->integer_value_);
-  int precision = 8;
   int scale = 5;
-  std::string result = decimal.ToString(precision, scale);
+  std::string result = decimal.ToString(scale);
   ASSERT_EQ(result, this->string_value_);
 }
 
@@ -256,4 +255,40 @@ TEST(Decimal128TestFalse, ConstructibleFromBool) {
   ASSERT_EQ(0, value.low_bits());
 }
 
+TEST(Decimal128Test, Division) {
+  const std::string expected_string_value("-23923094039234029");
+  const Decimal128 value(expected_string_value);
+  const Decimal128 result(value / 3);
+  const Decimal128 expected_value("-7974364679744676");
+  ASSERT_EQ(expected_value, result);
+}
+
+TEST(Decimal128Test, PrintLargePositiveValue) {
+  const std::string string_value("99999999999999999999999999999999999999");
+  const Decimal128 value(string_value);
+  const std::string printed_value = value.ToIntegerString();
+  ASSERT_EQ(string_value, printed_value);
+}
+
+TEST(Decimal128Test, PrintLargeNegativeValue) {
+  const std::string string_value("-99999999999999999999999999999999999999");
+  const Decimal128 value(string_value);
+  const std::string printed_value = value.ToIntegerString();
+  ASSERT_EQ(string_value, printed_value);
+}
+
+TEST(Decimal128Test, PrintMaxValue) {
+  const std::string string_value("170141183460469231731687303715884105727");
+  const Decimal128 value(string_value);
+  const std::string printed_value = value.ToIntegerString();
+  ASSERT_EQ(string_value, printed_value);
+}
+
+TEST(Decimal128Test, PrintMinValue) {
+  const std::string string_value("-170141183460469231731687303715884105728");
+  const Decimal128 value(string_value);
+  const std::string printed_value = value.ToIntegerString();
+  ASSERT_EQ(string_value, printed_value);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 7196b252c5b60..9d94bef847fa1 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -19,14 +19,10 @@
 #include <cctype>
 #include <cmath>
 #include <cstring>
+#include <iomanip>
 #include <limits>
 #include <sstream>
 
-#ifdef _MSC_VER
-#include <intrin.h>
-#pragma intrinsic(_BitScanReverse)
-#endif
-
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
@@ -55,62 +51,116 @@ std::array<uint8_t, 16> Decimal128::ToBytes() const {
   return out;
 }
 
-std::string Decimal128::ToString(int precision, int scale) const {
-  using std::size_t;
+static constexpr Decimal128 kTenTo36(static_cast<int64_t>(0xC097CE7BC90715),
+                                     0xB34B9F1000000000);
+static constexpr Decimal128 kTenTo18(0xDE0B6B3A7640000);
 
-  const bool is_negative = *this < 0;
-
-  // Decimal values are sent to clients as strings so in the interest of
-  // speed the string will be created without the using stringstream with the
-  // whole/fractional_part().
-  size_t last_char_idx = precision + (scale > 0)  // Add a space for decimal place
-                         + (scale == precision)   // Add a space for leading 0
-                         + is_negative;           // Add a space for negative sign
-
-  std::string str(last_char_idx, '0');
-
-  // Start filling in the values in reverse order by taking the last digit
-  // of the value. Use a positive value and worry about the sign later. At this
-  // point the last_char_idx points to the string terminator.
-  Decimal128 remaining_value(*this);
-
-  const auto first_digit_idx = static_cast<size_t>(is_negative);
-  if (is_negative) {
-    remaining_value.Negate();
-  }
-
-  if (scale > 0) {
-    int remaining_scale = scale;
-    do {
-      str[--last_char_idx] =
-          static_cast<char>(remaining_value % 10 + '0');  // Ascii offset
-      remaining_value /= 10;
-    } while (--remaining_scale > 0);
-    str[--last_char_idx] = '.';
-    DCHECK_GT(last_char_idx, first_digit_idx) << "Not enough space remaining";
-  }
-
-  do {
-    str[--last_char_idx] = static_cast<char>(remaining_value % 10 + '0');  // Ascii offset
-    remaining_value /= 10;
-    if (remaining_value == 0) {
-      // Trim any extra leading 0's.
-      if (last_char_idx > first_digit_idx) {
-        str.erase(0, last_char_idx - first_digit_idx);
-      }
+std::string Decimal128::ToIntegerString() const {
+  Decimal128 remainder;
+  std::stringstream buf;
+  bool need_fill = false;
 
-      break;
+  // get anything above 10 ** 36 and print it
+  Decimal128 top;
+  Status s = Divide(kTenTo36, &top, &remainder);
+  DCHECK(s.ok()) << s.message();
+
+  if (top != 0) {
+    buf << static_cast<int64_t>(top);
+    remainder.Abs();
+    need_fill = true;
+  }
+
+  // now get anything above 10 ** 18 and print it
+  Decimal128 tail;
+  s = remainder.Divide(kTenTo18, &top, &tail);
+
+  if (need_fill || top != 0) {
+    if (need_fill) {
+      buf << std::setw(18) << std::setfill('0');
+    } else {
+      need_fill = true;
+      tail.Abs();
+    }
+
+    buf << static_cast<int64_t>(top);
+  }
+
+  // finally print the tail, which is less than 10**18
+  if (need_fill) {
+    buf << std::setw(18) << std::setfill('0');
+  }
+  buf << static_cast<int64_t>(tail);
+  return buf.str();
+}
+
+Decimal128::operator int64_t() const {
+  DCHECK(high_bits_ == 0 || high_bits_ == -1)
+      << "Trying to cast an Decimal128 greater than the value range of a "
+         "int64_t. high_bits_ must be equal to 0 or -1, got: "
+      << high_bits_;
+  return static_cast<int64_t>(low_bits_);
+}
+
+std::string Decimal128::ToString(int32_t scale) const {
+  const std::string str(ToIntegerString());
+
+  if (scale == 0) {
+    return str;
+  }
+
+  if (*this < 0) {
+    const auto len = static_cast<int32_t>(str.size());
+
+    if (len - 1 > scale) {
+      const auto n = static_cast<size_t>(len - scale);
+      return str.substr(0, n) + "." + str.substr(n, static_cast<size_t>(scale));
+    }
+
+    if (len - 1 == scale) {
+      return "-0." + str.substr(1, std::string::npos);
     }
-    // For safety, enforce string length independent of remaining_value.
-  } while (last_char_idx > first_digit_idx);
 
-  if (is_negative) {
-    str[0] = '-';
+    std::string result("-0." + std::string(static_cast<size_t>(scale - len + 1), '0'));
+    return result + str.substr(1, std::string::npos);
+  }
+
+  const auto len = static_cast<int32_t>(str.size());
+
+  if (len > scale) {
+    const auto n = static_cast<size_t>(len - scale);
+    return str.substr(0, n) + "." + str.substr(n, static_cast<size_t>(scale));
+  }
+
+  if (len == scale) {
+    return "0." + str;
   }
 
-  return str;
+  return "0." + std::string(static_cast<size_t>(scale - len), '0') + str;
 }
 
+static constexpr auto kInt64DecimalDigits =
+    static_cast<size_t>(std::numeric_limits<int64_t>::digits10);
+static constexpr int64_t kPowersOfTen[kInt64DecimalDigits + 1] = {1LL,
+                                                                  10LL,
+                                                                  100LL,
+                                                                  1000LL,
+                                                                  10000LL,
+                                                                  100000LL,
+                                                                  1000000LL,
+                                                                  10000000LL,
+                                                                  100000000LL,
+                                                                  1000000000LL,
+                                                                  10000000000LL,
+                                                                  100000000000LL,
+                                                                  1000000000000LL,
+                                                                  10000000000000LL,
+                                                                  100000000000000LL,
+                                                                  1000000000000000LL,
+                                                                  10000000000000000LL,
+                                                                  100000000000000000LL,
+                                                                  1000000000000000000LL};
+
 static void StringToInteger(const std::string& str, Decimal128* out) {
   using std::size_t;
 
@@ -122,13 +172,10 @@ static void StringToInteger(const std::string& str, Decimal128* out) {
 
   DCHECK_GT(length, 0) << "length of parsed decimal string should be greater than 0";
 
-  size_t posn = 0;
-
-  while (posn < length) {
-    const size_t group = std::min(static_cast<size_t>(18), length - posn);
-    const auto chunk = static_cast<int64_t>(std::stoll(str.substr(posn, group)));
-    const auto multiple =
-        static_cast<int64_t>(std::pow(10.0, static_cast<double>(group)));
+  for (size_t posn = 0; posn < length;) {
+    const size_t group = std::min(kInt64DecimalDigits, length - posn);
+    const int64_t chunk = std::stoll(str.substr(posn, group));
+    const int64_t multiple = kPowersOfTen[group];
 
     *out *= multiple;
     *out += chunk;
@@ -266,6 +313,8 @@ Decimal128& Decimal128::Negate() {
   return *this;
 }
 
+Decimal128& Decimal128::Abs() { return *this < 0 ? Negate() : *this; }
+
 Decimal128& Decimal128::operator+=(const Decimal128& right) {
   const uint64_t sum = low_bits_ + right.low_bits_;
   high_bits_ += right.high_bits_;
@@ -288,20 +337,11 @@ Decimal128& Decimal128::operator-=(const Decimal128& right) {
 
 Decimal128& Decimal128::operator/=(const Decimal128& right) {
   Decimal128 remainder;
-  DCHECK(Divide(right, this, &remainder).ok());
+  Status s = Divide(right, this, &remainder);
+  DCHECK(s.ok());
   return *this;
 }
 
-Decimal128::operator char() const {
-  DCHECK(high_bits_ == 0 || high_bits_ == -1)
-      << "Trying to cast an Decimal128 greater than the value range of a "
-         "char. high_bits_ must be equal to 0 or -1, got: "
-      << high_bits_;
-  DCHECK_LE(low_bits_, std::numeric_limits<char>::max())
-      << "low_bits_ too large for C type char, got: " << low_bits_;
-  return static_cast<char>(low_bits_);
-}
-
 Decimal128& Decimal128::operator|=(const Decimal128& right) {
   low_bits_ |= right.low_bits_;
   high_bits_ |= right.high_bits_;
@@ -440,18 +480,6 @@ static int64_t FillInArray(const Decimal128& value, uint32_t* array, bool& was_n
   return 1;
 }
 
-/// \brief Find last set bit in a 32 bit integer. Bit 1 is the LSB and bit 32 is the MSB.
-static int64_t FindLastSetBit(uint32_t value) {
-#if defined(__clang__) || defined(__GNUC__)
-  // Count leading zeros
-  return __builtin_clz(value) + 1;
-#elif defined(_MSC_VER)
-  unsigned long index;                                         // NOLINT
-  _BitScanReverse(&index, static_cast<unsigned long>(value));  // NOLINT
-  return static_cast<int64_t>(index + 1UL);
-#endif
-}
-
 /// Shift the number in the array left by bits positions.
 /// \param array the number to shift, must have length elements
 /// \param length the number of entries in the array
@@ -581,7 +609,7 @@ Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result,
   // Normalize by shifting both by a multiple of 2 so that
   // the digit guessing is better. The requirement is that
   // divisor_array[0] is greater than 2**31.
-  int64_t normalize_bits = 32 - FindLastSetBit(divisor_array[0]);
+  int64_t normalize_bits = BitUtil::CountLeadingZeros(divisor_array[0]);
   ShiftArrayLeft(divisor_array, divisor_length, normalize_bits);
   ShiftArrayLeft(dividend_array, dividend_length, normalize_bits);
 
@@ -589,7 +617,7 @@ Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result,
   for (int64_t j = 0; j < result_length; ++j) {
     // Guess the next digit. At worst it is two too large
     uint32_t guess = std::numeric_limits<uint32_t>::max();
-    auto high_dividend =
+    const auto high_dividend =
         static_cast<uint64_t>(dividend_array[j]) << 32 | dividend_array[j + 1];
     if (dividend_array[j] != divisor_array[0]) {
       guess = static_cast<uint32_t>(high_dividend / divisor_array[0]);
@@ -625,10 +653,9 @@ Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result,
     // if guess was too big, we add back divisor
     if (dividend_array[j] > prev) {
       --guess;
-
       uint32_t carry = 0;
       for (int64_t i = divisor_length - 1; i >= 0; --i) {
-        uint64_t sum =
+        const auto sum =
             static_cast<uint64_t>(divisor_array[i]) + dividend_array[j + i + 1] + carry;
         dividend_array[j + i + 1] = static_cast<uint32_t>(sum);
         carry = static_cast<uint32_t>(sum >> 32);
@@ -645,6 +672,7 @@ Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result,
   // return result and remainder
   RETURN_NOT_OK(BuildFromArray(result, result_array, result_length));
   RETURN_NOT_OK(BuildFromArray(remainder, dividend_array, dividend_length));
+
   FixDivisionSigns(result, remainder, dividend_was_negative, divisor_was_negative);
   return Status::OK();
 }
@@ -679,6 +707,11 @@ Decimal128 operator-(const Decimal128& operand) {
   return result.Negate();
 }
 
+Decimal128 operator~(const Decimal128& operand) {
+  Decimal128 result(~operand.high_bits(), ~operand.low_bits());
+  return result;
+}
+
 Decimal128 operator+(const Decimal128& left, const Decimal128& right) {
   Decimal128 result(left.high_bits(), left.low_bits());
   result += right;
@@ -700,14 +733,16 @@ Decimal128 operator*(const Decimal128& left, const Decimal128& right) {
 Decimal128 operator/(const Decimal128& left, const Decimal128& right) {
   Decimal128 remainder;
   Decimal128 result;
-  DCHECK(left.Divide(right, &result, &remainder).ok());
+  Status s = left.Divide(right, &result, &remainder);
+  DCHECK(s.ok());
   return result;
 }
 
 Decimal128 operator%(const Decimal128& left, const Decimal128& right) {
   Decimal128 remainder;
   Decimal128 result;
-  DCHECK(left.Divide(right, &result, &remainder).ok());
+  Status s = left.Divide(right, &result, &remainder);
+  DCHECK(s.ok());
   return remainder;
 }
 
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 72da5547907db..487f222580201 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -39,15 +39,16 @@ namespace arrow {
 class ARROW_EXPORT Decimal128 {
  public:
   /// \brief Create an Decimal128 from the two's complement representation.
-  constexpr Decimal128(int64_t high, uint64_t low) : high_bits_(high), low_bits_(low) {}
+  constexpr Decimal128(int64_t high, uint64_t low) noexcept
+      : high_bits_(high), low_bits_(low) {}
 
   /// \brief Empty constructor creates an Decimal128 with a value of 0.
-  constexpr Decimal128() : Decimal128(0, 0) {}
+  constexpr Decimal128() noexcept : Decimal128(0, 0) {}
 
   /// \brief Convert any integer value into an Decimal128.
   template <typename T,
             typename = typename std::enable_if<std::is_integral<T>::value, T>::type>
-  constexpr Decimal128(T value)
+  constexpr Decimal128(T value) noexcept
       : Decimal128(static_cast<int64_t>(value) >= 0 ? 0 : -1,
                    static_cast<uint64_t>(value)) {}
 
@@ -61,6 +62,9 @@ class ARROW_EXPORT Decimal128 {
   /// \brief Negate the current value
   Decimal128& Negate();
 
+  /// \brief Absolute value
+  Decimal128& Abs();
+
   /// \brief Add a number to this one. The result is truncated to 128 bits.
   Decimal128& operator+=(const Decimal128& right);
 
@@ -85,9 +89,6 @@ class ARROW_EXPORT Decimal128 {
   /// \brief In-place division.
   Decimal128& operator/=(const Decimal128& right);
 
-  /// \brief Cast the value to char. This is used when converting the value a string.
-  explicit operator char() const;
-
   /// \brief Bitwise or between two Decimal128.
   Decimal128& operator|=(const Decimal128& right);
 
@@ -110,8 +111,14 @@ class ARROW_EXPORT Decimal128 {
   std::array<uint8_t, 16> ToBytes() const;
 
   /// \brief Convert the Decimal128 value to a base 10 decimal string with the given
-  /// precision and scale.
-  std::string ToString(int precision, int scale) const;
+  /// scale.
+  std::string ToString(int32_t scale) const;
+
+  /// \brief Convert the value to an integer string
+  std::string ToIntegerString() const;
+
+  /// \brief Cast this value to an int64_t.
+  explicit operator int64_t() const;
 
   /// \brief Convert a decimal string to an Decimal128 value, optionally including
   /// precision and scale if they're passed in and not null.
@@ -131,6 +138,7 @@ ARROW_EXPORT bool operator>(const Decimal128& left, const Decimal128& right);
 ARROW_EXPORT bool operator>=(const Decimal128& left, const Decimal128& right);
 
 ARROW_EXPORT Decimal128 operator-(const Decimal128& operand);
+ARROW_EXPORT Decimal128 operator~(const Decimal128& operand);
 ARROW_EXPORT Decimal128 operator+(const Decimal128& left, const Decimal128& right);
 ARROW_EXPORT Decimal128 operator-(const Decimal128& left, const Decimal128& right);
 ARROW_EXPORT Decimal128 operator*(const Decimal128& left, const Decimal128& right);

From b25b2433d67d797e9cd461377eb3798e60de2727 Mon Sep 17 00:00:00 2001
From: Yuliya Feldman <yuliya@dremio.com>
Date: Sun, 5 Nov 2017 18:57:05 -0500
Subject: [PATCH 1220/1644] =?UTF-8?q?ARROW-1663:=20[Java]=20use=20consiste?=
 =?UTF-8?q?nt=20name=20for=20null=20and=20not-null=20in=20FixedSizeLis?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…t, add backward compatibility while deserializing schema that was generated before this JIRA checkin

Author: Yuliya Feldman <yuliya@dremio.com>

Closes #1193 from yufeldman/ARROW-1663 and squashes the following commits:

7f9bd34f [Yuliya Feldman] ARROW-1663: Addressing code review comments
600d379b [Yuliya Feldman] ARROW-1663: Addressing code review comments
93f527b0 [Yuliya Feldman] ARROW-1663: Addressing code review comments
7cfa22b0 [Yuliya Feldman] ARROW-1663: use consistent name for null and not-null in FixedSizeListVector and ListVector, add backward compatibility while deserializing schema that was generated before this JIRA checkin
58d6e9c9 [Yuliya Feldman] ARROW-1663: use consistent name for null and not-null in FixedSizeList, add backward compatibility while deserializing schema that was generated before this JIRA checkin
---
 .../org/apache/arrow/vector/ZeroVector.java   |  6 +--
 .../arrow/vector/complex/ListVector.java      |  5 ---
 .../apache/arrow/vector/types/pojo/Field.java | 27 ++++++++++++-
 .../arrow/vector/TestFixedSizeListVector.java | 14 +++++++
 .../apache/arrow/vector/TestListVector.java   |  7 +---
 .../apache/arrow/vector/pojo/TestConvert.java | 40 +++++++++++++++++++
 6 files changed, 85 insertions(+), 14 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index b267b2087d05c..5ac00375f8317 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -18,6 +18,8 @@
 
 package org.apache.arrow.vector;
 
+import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME;
+
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -39,8 +41,6 @@
 public class ZeroVector implements FieldVector {
   public final static ZeroVector INSTANCE = new ZeroVector();
 
-  private final String name = "[DEFAULT]";
-
   private final TransferPair defaultPair = new TransferPair() {
     @Override
     public void transfer() {
@@ -109,7 +109,7 @@ public void clear() {
 
   @Override
   public Field getField() {
-    return new Field(name, FieldType.nullable(new Null()), null);
+    return new Field(DATA_VECTOR_NAME, FieldType.nullable(new Null()), null);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 6511efcb7d513..ea28a60619209 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -39,7 +39,6 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
@@ -49,7 +48,6 @@
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -324,9 +322,6 @@ public int getBufferSize() {
 
   @Override
   public Field getField() {
-    if (getDataVector() instanceof ZeroVector) {
-      return new Field(name, fieldType, ImmutableList.of(new Field(DATA_VECTOR_NAME, FieldType.nullable(Null.INSTANCE), null)));
-    }
     return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 48e71a976c0e8..eba149bf79f65 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -20,6 +20,7 @@
 
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME;
 import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
 
 import java.util.Iterator;
@@ -39,6 +40,7 @@
 import com.google.flatbuffers.FlatBufferBuilder;
 
 import org.apache.arrow.flatbuf.KeyValue;
+import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.schema.TypeLayout;
@@ -121,7 +123,9 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
     }
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     for (int i = 0; i < field.childrenLength(); i++) {
-      childrenBuilder.add(convertField(field.children(i)));
+      Field childField = convertField(field.children(i));
+      childField = mutateOriginalNameIfNeeded(field, childField);
+      childrenBuilder.add(childField);
     }
     List<Field> children = childrenBuilder.build();
     ImmutableMap.Builder<String, String> metadataBuilder = ImmutableMap.builder();
@@ -134,6 +138,27 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
     return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()), metadata);
   }
 
+  /**
+   * Helper method to ensure backward compatibility with schemas generated prior to ARROW-1347, ARROW-1663
+   * @param field
+   * @param originalChildField original field which name might be mutated
+   * @return original or mutated field
+   */
+  private static Field mutateOriginalNameIfNeeded(org.apache.arrow.flatbuf.Field field, Field originalChildField) {
+    if ((field.typeType() == Type.List || field.typeType() == Type.FixedSizeList)
+        && originalChildField.getName().equals("[DEFAULT]")) {
+      return
+        new Field(DATA_VECTOR_NAME,
+          originalChildField.isNullable(),
+          originalChildField.getType(),
+          originalChildField.getDictionary(),
+          originalChildField.getChildren(),
+          originalChildField.getTypeLayout(),
+          originalChildField.getMetadata());
+    }
+    return originalChildField;
+  }
+
   public void validate() {
     TypeLayout expectedLayout = TypeLayout.getTypeLayout(getType());
     if (!expectedLayout.equals(typeLayout)) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index 5677f2566797a..43d9387b106a4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -26,6 +26,7 @@
 import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -220,4 +221,17 @@ public void testTransferPair() {
       }
     }
   }
+
+  @Test
+  public void testConsistentChildName() throws Exception {
+    try (FixedSizeListVector listVector = FixedSizeListVector.empty("sourceVector", 2, allocator)) {
+      String emptyListStr = listVector.getField().toString();
+      Assert.assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME));
+
+      listVector.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType()));
+      String emptyVectorStr = listVector.getField().toString();
+      Assert.assertTrue(emptyVectorStr.contains(ListVector.DATA_VECTOR_NAME));
+    }
+  }
+
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 1c9b574998018..59e1646e86e56 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -26,12 +26,9 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
-import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.holders.NullableBigIntHolder;
 import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.*;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.TransferPair;
 import org.junit.After;
@@ -635,7 +632,7 @@ public void testConsistentChildName() throws Exception {
       String emptyListStr = listVector.getField().toString();
       assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME));
 
-      listVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+      listVector.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType()));
       String emptyVectorStr = listVector.getField().toString();
       assertTrue(emptyVectorStr.contains(ListVector.DATA_VECTOR_NAME));
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index f98aeac8c8196..f6f1ad221f3d1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -21,13 +21,19 @@
 import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE;
 import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
+import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.Map;
 
 import com.google.common.collect.ImmutableList;
 import com.google.flatbuffers.FlatBufferBuilder;
 
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.UnionMode;
@@ -64,6 +70,40 @@ public void complex() {
     run(initialField);
   }
 
+  @Test
+  public void list() throws Exception {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+         ListVector writeVector = ListVector.empty("list", allocator);
+         FixedSizeListVector writeFixedVector = FixedSizeListVector.empty("fixedlist", 5, allocator)) {
+      Field listVectorField = writeVector.getField();
+      childrenBuilder.add(listVectorField);
+      Field listFixedVectorField = writeFixedVector.getField();
+      childrenBuilder.add(listFixedVectorField);
+    }
+
+    Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), childrenBuilder.build());
+    ImmutableList.Builder<Field> parentBuilder = ImmutableList.builder();
+    parentBuilder.add(initialField);
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    builder.finish(initialField.getField(builder));
+    org.apache.arrow.flatbuf.Field flatBufField = org.apache.arrow.flatbuf.Field.getRootAsField(builder.dataBuffer());
+    Field finalField = Field.convertField(flatBufField);
+    assertEquals(initialField, finalField);
+    assertFalse(finalField.toString().contains("[DEFAULT]"));
+
+    Schema initialSchema = new Schema(parentBuilder.build());
+    String jsonSchema = initialSchema.toJson();
+    String modifiedSchema = jsonSchema.replace("$data$", "[DEFAULT]");
+
+    Schema tempSchema = Schema.fromJSON(modifiedSchema);
+    FlatBufferBuilder schemaBuilder = new FlatBufferBuilder();
+    org.apache.arrow.vector.types.pojo.Schema schema = new org.apache.arrow.vector.types.pojo.Schema(tempSchema.getFields());
+    schemaBuilder.finish(schema.getSchema(schemaBuilder));
+    Schema finalSchema = Schema.deserialize(ByteBuffer.wrap(schemaBuilder.sizedByteArray()));
+    assertFalse(finalSchema.toString().contains("[DEFAULT]"));
+  }
+
   @Test
   public void schema() {
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();

From 972193035802c47fa9ee8cb0962cda14c3f77847 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Mon, 6 Nov 2017 00:02:31 -0500
Subject: [PATCH 1221/1644] ARROW-480: [Python] Implement
 RowGroupMetaData.ColumnChunk

This is the patch for [ARROW-480](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-480).

Author: Licht-T <licht-t@outlook.jp>

Closes #1215 from Licht-T/feature-column-metadata and squashes the following commits:

b89c98c1 [Licht-T] Fix string format to get Python 3 compatible
2de9ca8a [Licht-T] Merge remote-tracking branch 'arrow/master' into feature-column-metadata
367761b7 [Licht-T] TST: Check every row-groups and columns
d4dc4d1a [Licht-T] TST: Add test for column statistics
06397f4b [Licht-T] Refactoring
c5dbef39 [Licht-T] Fix RowGroupStatistics.__repr__
e3b0c6a4 [Licht-T] Convert std::string to Python str by frombytes
5ab032ae [Licht-T] Change EncodeMin/EncodeMax to decoded min/max
c29fb4e4 [Licht-T] Rename ColumnChunk to column
6a71795b [Licht-T] Rename HasMinMax to snake case
8f605cfb [Licht-T] Use bint as C++ boolean type for existing checks
9cc55cb9 [Licht-T] Remove unnecessary class and methods
33c087a4 [Licht-T] TST: Fix Parquet MetaData test
bdc0bdca [Licht-T] ENH: Implement RowGroupMetaData.ColumnChunk
---
 python/pyarrow/_parquet.pxd          |  49 ++++++
 python/pyarrow/_parquet.pyx          | 224 +++++++++++++++++++++++++++
 python/pyarrow/compat.py             |   6 +
 python/pyarrow/tests/test_parquet.py |  54 ++++++-
 4 files changed, 330 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 5094232bdc7b4..04a5b1368ce45 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -105,6 +105,11 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
         ParquetVersion_V1" parquet::ParquetVersion::PARQUET_1_0"
         ParquetVersion_V2" parquet::ParquetVersion::PARQUET_2_0"
 
+    enum ParquetSortOrder" parquet::SortOrder::type":
+        ParquetSortOrder_SIGNED" parquet::SortOrder::SIGNED"
+        ParquetSortOrder_UNSIGNED" parquet::SortOrder::UNSIGNED"
+        ParquetSortOrder_UNKNOWN" parquet::SortOrder::UNKNOWN"
+
     cdef cppclass ColumnDescriptor:
         c_bool Equals(const ColumnDescriptor& other)
 
@@ -126,6 +131,8 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
         c_bool Equals(const SchemaDescriptor& other)
         int num_columns()
 
+    cdef c_string FormatStatValue(ParquetType parquet_type, const char* val)
+
 
 cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
     cdef cppclass ColumnReader:
@@ -155,10 +162,52 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
     cdef cppclass RowGroupReader:
         pass
 
+    cdef cppclass CEncodedStatistics" parquet::EncodedStatistics":
+        const c_string& max() const
+        const c_string& min() const
+        int64_t null_count
+        int64_t distinct_count
+        bint has_min
+        bint has_max
+        bint has_null_count
+        bint has_distinct_count
+
+    cdef cppclass CRowGroupStatistics" parquet::RowGroupStatistics":
+        int64_t null_count() const
+        int64_t distinct_count() const
+        int64_t num_values() const
+        bint HasMinMax()
+        void Reset()
+        c_string EncodeMin()
+        c_string EncodeMax()
+        CEncodedStatistics Encode()
+        void SetComparator()
+        ParquetType physical_type() const
+
+    cdef cppclass CColumnChunkMetaData" parquet::ColumnChunkMetaData":
+        int64_t file_offset() const
+        const c_string& file_path() const
+
+        ParquetType type() const
+        int64_t num_values() const
+        shared_ptr[ColumnPath] path_in_schema() const
+        bint is_stats_set() const
+        shared_ptr[CRowGroupStatistics] statistics() const;
+        ParquetCompression compression() const
+        const vector[ParquetEncoding]& encodings() const
+
+        bint has_dictionary_page() const
+        int64_t dictionary_page_offset() const
+        int64_t data_page_offset() const
+        int64_t index_page_offset() const
+        int64_t total_compressed_size() const
+        int64_t total_uncompressed_size() const
+
     cdef cppclass CRowGroupMetaData" parquet::RowGroupMetaData":
         int num_columns()
         int64_t num_rows()
         int64_t total_byte_size()
+        unique_ptr[CColumnChunkMetaData] ColumnChunk(int i) const
 
     cdef cppclass CFileMetaData" parquet::FileMetaData":
         uint32_t size()
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index b096fa1b4d337..eca6b201b3ad0 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -35,6 +35,212 @@ from pyarrow.lib import ArrowException, NativeFile
 
 import six
 
+try:
+    from textwrap import indent
+except ImportError:
+    def indent(text, prefix):
+        lines = [prefix + line for line in text.splitlines(True)]
+        return ''.join(lines)
+
+
+cdef class RowGroupStatistics:
+    cdef:
+        shared_ptr[CRowGroupStatistics] statistics
+
+    def __cinit__(self):
+        pass
+
+    cdef init(self, const shared_ptr[CRowGroupStatistics]& statistics):
+        self.statistics = statistics
+
+    def __repr__(self):
+        return """{0}
+  has_min_max: {1}
+  min: {2}
+  max: {3}
+  null_count: {4}
+  distinct_count: {5}
+  num_values: {6}
+  physical_type: {7}""".format(object.__repr__(self),
+                               self.has_min_max,
+                               self.min,
+                               self.max,
+                               self.null_count,
+                               self.distinct_count,
+                               self.num_values,
+                               self.physical_type)
+
+    property has_min_max:
+
+        def __get__(self):
+            return self.statistics.get().HasMinMax()
+
+    property min:
+
+        def __get__(self):
+            raw_physical_type = self.statistics.get().physical_type()
+            encode_min = self.statistics.get().EncodeMin()
+
+            min_value = FormatStatValue(raw_physical_type, encode_min.c_str())
+            return frombytes(min_value)
+
+    property max:
+
+        def __get__(self):
+            raw_physical_type = self.statistics.get().physical_type()
+            encode_max = self.statistics.get().EncodeMax()
+
+            max_value = FormatStatValue(raw_physical_type, encode_max.c_str())
+            return frombytes(max_value)
+
+    property null_count:
+
+        def __get__(self):
+            return self.statistics.get().null_count()
+
+    property distinct_count:
+
+        def __get__(self):
+            return self.statistics.get().distinct_count()
+
+    property num_values:
+
+        def __get__(self):
+            return self.statistics.get().num_values()
+
+    property physical_type:
+
+        def __get__(self):
+            physical_type = self.statistics.get().physical_type()
+            return physical_type_name_from_enum(physical_type)
+
+
+cdef class ColumnChunkMetaData:
+    cdef:
+        unique_ptr[CColumnChunkMetaData] up_metadata
+        CColumnChunkMetaData* metadata
+
+    def __cinit__(self):
+        pass
+
+    cdef init(self, const CRowGroupMetaData& row_group_metadata, int i):
+        self.up_metadata = row_group_metadata.ColumnChunk(i)
+        self.metadata = self.up_metadata.get()
+
+    def __repr__(self):
+        statistics = indent(repr(self.statistics), 4 * ' ')
+        return """{0}
+  file_offset: {1}
+  file_path: {2}
+  type: {3}
+  num_values: {4}
+  path_in_schema: {5}
+  is_stats_set: {6}
+  statistics:
+{7}
+  compression: {8}
+  encodings: {9}
+  has_dictionary_page: {10}
+  dictionary_page_offset: {11}
+  data_page_offset: {12}
+  index_page_offset: {13}
+  total_compressed_size: {14}
+  total_uncompressed_size: {15}""".format(object.__repr__(self),
+                                          self.file_offset,
+                                          self.file_path,
+                                          self.type,
+                                          self.num_values,
+                                          self.path_in_schema,
+                                          self.is_stats_set,
+                                          statistics,
+                                          self.compression,
+                                          self.encodings,
+                                          self.has_dictionary_page,
+                                          self.dictionary_page_offset,
+                                          self.data_page_offset,
+                                          self.index_page_offset,
+                                          self.total_compressed_size,
+                                          self.total_uncompressed_size)
+
+    property file_offset:
+
+        def __get__(self):
+            return self.metadata.file_offset()
+
+    property file_path:
+
+        def __get__(self):
+            return frombytes(self.metadata.file_path())
+
+    property type:
+
+        def __get__(self):
+            return physical_type_name_from_enum(self.metadata.type())
+
+    property num_values:
+
+        def __get__(self):
+            return self.metadata.num_values()
+
+    property path_in_schema:
+
+        def __get__(self):
+            path = self.metadata.path_in_schema().get().ToDotString()
+            return frombytes(path)
+
+    property is_stats_set:
+
+        def __get__(self):
+            return self.metadata.is_stats_set()
+
+    property statistics:
+
+        def __get__(self):
+            statistics = RowGroupStatistics()
+            statistics.init(self.metadata.statistics())
+            return statistics
+
+    property compression:
+
+        def __get__(self):
+            return self.metadata.compression()
+
+    property encodings:
+
+        def __get__(self):
+            return map(encoding_name_from_enum,
+                       self.metadata.encodings())
+
+    property has_dictionary_page:
+
+        def __get__(self):
+            return self.metadata.has_dictionary_page()
+
+    property dictionary_page_offset:
+
+        def __get__(self):
+            return self.metadata.dictionary_page_offset()
+
+    property data_page_offset:
+
+        def __get__(self):
+            return self.metadata.data_page_offset()
+
+    property index_page_offset:
+
+        def __get__(self):
+            return self.metadata.index_page_offset()
+
+    property total_compressed_size:
+
+        def __get__(self):
+            return self.metadata.total_compressed_size()
+
+    property total_uncompressed_size:
+
+        def __get__(self):
+            return self.metadata.total_uncompressed_size()
+
 
 cdef class RowGroupMetaData:
     cdef:
@@ -52,6 +258,11 @@ cdef class RowGroupMetaData:
         self.metadata = self.up_metadata.get()
         self.parent = parent
 
+    def column(self, int i):
+        chunk = ColumnChunkMetaData()
+        chunk.init(deref(self.metadata), i)
+        return chunk
+
     def __repr__(self):
         return """{0}
   num_columns: {1}
@@ -371,6 +582,19 @@ cdef logical_type_name_from_enum(ParquetLogicalType type_):
     }.get(type_, 'UNKNOWN')
 
 
+cdef encoding_name_from_enum (ParquetEncoding encoding_):
+    return {
+        ParquetEncoding_PLAIN: "PLAIN",
+        ParquetEncoding_PLAIN_DICTIONARY: "PLAIN_DICTIONARY",
+        ParquetEncoding_RLE: "RLE",
+        ParquetEncoding_BIT_PACKED: "BIT_PACKED",
+        ParquetEncoding_DELTA_BINARY_PACKED: "DELTA_BINARY_PACKED",
+        ParquetEncoding_DELTA_LENGTH_BYTE_ARRAY: "DELTA_LENGTH_BYTE_ARRAY",
+        ParquetEncoding_DELTA_BYTE_ARRAY: "DELTA_BYTE_ARRAY",
+        ParquetEncoding_RLE_DICTIONARY: "RLE_DICTIONARY",
+    }.get(encoding_, 'UNKNOWN')
+
+
 cdef class ParquetReader:
     cdef:
         object source
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index f9c148b14e368..866cbdd96d063 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -103,6 +103,9 @@ def tobytes(o):
 
     def frombytes(o):
         return o
+
+    def unichar(s):
+        return unichr(s)
 else:
     unicode_type = str
     def lzip(*x):
@@ -131,6 +134,9 @@ def tobytes(o):
     def frombytes(o):
         return o.decode('utf8')
 
+    def unichar(s):
+        return chr(s)
+
 try:
     import cloudpickle as pickle
 except ImportError:
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 95dd6a471b6b3..e2e6863c4748f 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -22,7 +22,7 @@
 import json
 import pytest
 
-from pyarrow.compat import guid, u, BytesIO
+from pyarrow.compat import guid, u, BytesIO, unichar, frombytes
 from pyarrow.filesystem import LocalFileSystem
 import pyarrow as pa
 from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
@@ -469,13 +469,61 @@ def test_parquet_metadata_api():
         schema[-1]
 
     # Row group
-    rg_meta = meta.row_group(0)
-    repr(rg_meta)
+    for rg in range(meta.num_row_groups):
+        rg_meta = meta.row_group(rg)
+        repr(rg_meta)
+
+        for col in range(rg_meta.num_columns):
+            col_meta = rg_meta.column(col)
+            repr(col_meta)
 
     assert rg_meta.num_rows == len(df)
     assert rg_meta.num_columns == ncols + 1  # +1 for index
 
 
+@parquet
+@pytest.mark.parametrize(
+    'data, dtype, min_value, max_value, null_count, num_values',
+    [
+        ([1, 2, 2, None, 4], np.uint8, u'1', u'4', 1, 4),
+        ([1, 2, 2, None, 4], np.uint16, u'1', u'4', 1, 4),
+        ([1, 2, 2, None, 4], np.uint32, u'1', u'4', 1, 4),
+        ([1, 2, 2, None, 4], np.uint64, u'1', u'4', 1, 4),
+        ([-1, 2, 2, None, 4], np.int16, u'-1', u'4', 1, 4),
+        ([-1, 2, 2, None, 4], np.int32, u'-1', u'4', 1, 4),
+        ([-1, 2, 2, None, 4], np.int64, u'-1', u'4', 1, 4),
+        ([-1.1, 2.2, 2.3, None, 4.4], np.float32, u'-1.1', u'4.4', 1, 4),
+        ([-1.1, 2.2, 2.3, None, 4.4], np.float64, u'-1.1', u'4.4', 1, 4),
+        (
+            [u'', u'b', unichar(1000), None, u'aaa'],
+            str, u' ', frombytes((unichar(1000) + u' ').encode('utf-8')), 1, 4
+        ),
+        ([True, False, False, True, True], np.bool, u'0', u'1', 0, 5),
+    ]
+)
+def test_parquet_column_statistics_api(
+        data,
+        dtype,
+        min_value,
+        max_value,
+        null_count,
+        num_values):
+    df = pd.DataFrame({'data': data}, dtype=dtype)
+
+    fileh = make_sample_file(df)
+
+    meta = fileh.metadata
+
+    rg_meta = meta.row_group(0)
+    col_meta = rg_meta.column(0)
+
+    stat = col_meta.statistics
+    assert stat.min == min_value
+    assert stat.max == max_value
+    assert stat.null_count == null_count
+    assert stat.num_values == num_values
+
+
 @parquet
 def test_compare_schemas():
     df = alltypes_sample(size=10000)

From 0106f531c04477b1c8bd088d097624ff43b44658 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 6 Nov 2017 12:55:49 -0500
Subject: [PATCH 1222/1644] ARROW-1750: [C++] Remove the need for
 arrow/util/random.h

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1283 from cpcloud/ARROW-1750 and squashes the following commits:

3f6af737 [Phillip Cloud] ARROW-1750: [C++] Remove the need for arrow/util/random.h
---
 cpp/src/arrow/array-test.cc        |  93 ++++++++-----
 cpp/src/arrow/ipc/ipc-json-test.cc |   4 +-
 cpp/src/arrow/test-util.h          | 214 ++++++++++++++++++++++-------
 cpp/src/arrow/util/CMakeLists.txt  |   1 -
 cpp/src/arrow/util/decimal.cc      |  11 +-
 cpp/src/arrow/util/decimal.h       |   5 +-
 cpp/src/arrow/util/random.h        | 126 -----------------
 7 files changed, 236 insertions(+), 218 deletions(-)
 delete mode 100644 cpp/src/arrow/util/random.h

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 168ef10573e77..9f248cdbbb754 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -263,6 +263,8 @@ class TestPrimitiveBuilder : public TestBuilder {
     ASSERT_TRUE(result->Equals(*expected));
   }
 
+  int64_t FlipValue(int64_t value) const { return ~value; }
+
  protected:
   std::shared_ptr<DataType> type_;
   std::unique_ptr<BuilderType> builder_;
@@ -272,44 +274,64 @@ class TestPrimitiveBuilder : public TestBuilder {
   vector<uint8_t> valid_bytes_;
 };
 
-#define PTYPE_DECL(CapType, c_type)               \
-  typedef CapType##Array ArrayType;               \
-  typedef CapType##Builder BuilderType;           \
-  typedef CapType##Type Type;                     \
-  typedef c_type T;                               \
-                                                  \
-  static std::shared_ptr<DataType> type() {       \
-    return std::shared_ptr<DataType>(new Type()); \
-  }
+/// \brief uint8_t isn't a valid template parameter to uniform_int_distribution, so
+/// we use SampleType to determine which kind of integer to use to sample.
+template <typename T,
+          typename = typename std::enable_if<std::is_integral<T>::value, T>::type>
+struct UniformIntSampleType {
+  using type = T;
+};
+
+template <>
+struct UniformIntSampleType<uint8_t> {
+  using type = uint16_t;
+};
+
+template <>
+struct UniformIntSampleType<int8_t> {
+  using type = int16_t;
+};
 
-#define PINT_DECL(CapType, c_type, LOWER, UPPER)    \
+#define PTYPE_DECL(CapType, c_type)     \
+  typedef CapType##Array ArrayType;     \
+  typedef CapType##Builder BuilderType; \
+  typedef CapType##Type Type;           \
+  typedef c_type T;                     \
+                                        \
+  static std::shared_ptr<DataType> type() { return std::make_shared<Type>(); }
+
+#define PINT_DECL(CapType, c_type)                                                       \
+  struct P##CapType {                                                                    \
+    PTYPE_DECL(CapType, c_type)                                                          \
+    static void draw(int64_t N, vector<T>* draws) {                                      \
+      using sample_type = typename UniformIntSampleType<c_type>::type;                   \
+      const T lower = std::numeric_limits<T>::min();                                     \
+      const T upper = std::numeric_limits<T>::max();                                     \
+      test::randint(N, static_cast<sample_type>(lower), static_cast<sample_type>(upper), \
+                    draws);                                                              \
+    }                                                                                    \
+  }
+
+#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)  \
   struct P##CapType {                               \
     PTYPE_DECL(CapType, c_type)                     \
     static void draw(int64_t N, vector<T>* draws) { \
-      test::randint<T>(N, LOWER, UPPER, draws);     \
+      test::random_real(N, 0, LOWER, UPPER, draws); \
     }                                               \
   }
 
-#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER)     \
-  struct P##CapType {                                  \
-    PTYPE_DECL(CapType, c_type)                        \
-    static void draw(int64_t N, vector<T>* draws) {    \
-      test::random_real<T>(N, 0, LOWER, UPPER, draws); \
-    }                                                  \
-  }
-
-PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX);
-PINT_DECL(UInt16, uint16_t, 0, UINT16_MAX);
-PINT_DECL(UInt32, uint32_t, 0, UINT32_MAX);
-PINT_DECL(UInt64, uint64_t, 0, UINT64_MAX);
+PINT_DECL(UInt8, uint8_t);
+PINT_DECL(UInt16, uint16_t);
+PINT_DECL(UInt32, uint32_t);
+PINT_DECL(UInt64, uint64_t);
 
-PINT_DECL(Int8, int8_t, INT8_MIN, INT8_MAX);
-PINT_DECL(Int16, int16_t, INT16_MIN, INT16_MAX);
-PINT_DECL(Int32, int32_t, INT32_MIN, INT32_MAX);
-PINT_DECL(Int64, int64_t, INT64_MIN, INT64_MAX);
+PINT_DECL(Int8, int8_t);
+PINT_DECL(Int16, int16_t);
+PINT_DECL(Int32, int32_t);
+PINT_DECL(Int64, int64_t);
 
-PFLOAT_DECL(Float, float, -1000, 1000);
-PFLOAT_DECL(Double, double, -1000, 1000);
+PFLOAT_DECL(Float, float, -1000.0f, 1000.0f);
+PFLOAT_DECL(Double, double, -1000.0, 1000.0);
 
 struct PBoolean {
   PTYPE_DECL(Boolean, uint8_t)
@@ -324,6 +346,11 @@ void TestPrimitiveBuilder<PBoolean>::RandomData(int64_t N, double pct_null) {
   test::random_null_bytes(N, pct_null, valid_bytes_.data());
 }
 
+template <>
+int64_t TestPrimitiveBuilder<PBoolean>::FlipValue(int64_t value) const {
+  return !value;
+}
+
 template <>
 void TestPrimitiveBuilder<PBoolean>::Check(const std::unique_ptr<BooleanBuilder>& builder,
                                            bool nullable) {
@@ -454,8 +481,8 @@ TYPED_TEST(TestPrimitiveBuilder, Equality) {
   const int64_t first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
   // This should be true with a very high probability, but might introduce flakiness
   ASSERT_LT(first_valid_idx, size - 1);
-  draws[first_valid_idx] =
-      static_cast<T>(~*reinterpret_cast<int64_t*>(&draws[first_valid_idx]));
+  draws[first_valid_idx] = static_cast<T>(
+      this->FlipValue(*reinterpret_cast<int64_t*>(&draws[first_valid_idx])));
   ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &unequal_array));
 
   // test normal equality
@@ -724,8 +751,8 @@ void CheckSliceApproxEquals() {
   vector<T> draws2;
 
   const uint32_t kSeed = 0;
-  test::random_real<T>(kSize, kSeed, 0, 100, &draws1);
-  test::random_real<T>(kSize, kSeed + 1, 0, 100, &draws2);
+  test::random_real(kSize, kSeed, 0.0, 100.0, &draws1);
+  test::random_real(kSize, kSeed + 1, 0.0, 100.0, &draws2);
 
   // Make the draws equal in the sliced segment, but unequal elsewhere (to
   // catch not using the slice offset)
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index f2dd9e74e335d..a560f09d6fdb1 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -222,8 +222,8 @@ void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
   std::vector<int8_t> v1_values;
   std::vector<int32_t> v2_values;
 
-  test::randint<int8_t>(num_rows, 0, 100, &v1_values);
-  test::randint<int32_t>(num_rows, 0, 100, &v2_values);
+  test::randint(num_rows, 0, 100, &v1_values);
+  test::randint(num_rows, 0, 100, &v2_values);
 
   std::shared_ptr<Array> v1;
   ArrayFromVector<Int8Type, int8_t>(is_valid, v1_values, &v1);
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 044fb9476ca73..7306f577a36e0 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_TEST_UTIL_H_
 #define ARROW_TEST_UTIL_H_
 
+#include <algorithm>
 #include <cstdint>
 #include <limits>
 #include <memory>
@@ -38,8 +39,8 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
-#include "arrow/util/random.h"
 
 #define ASSERT_RAISES(ENUM, expr) \
   do {                            \
@@ -47,7 +48,7 @@
     if (!s.Is##ENUM()) {          \
       FAIL() << s.ToString();     \
     }                             \
-  } while (0)
+  } while (false)
 
 #define ASSERT_OK(expr)         \
   do {                          \
@@ -55,7 +56,7 @@
     if (!s.ok()) {              \
       FAIL() << s.ToString();   \
     }                           \
-  } while (0)
+  } while (false)
 
 #define ASSERT_OK_NO_THROW(expr) ASSERT_NO_THROW(ASSERT_OK(expr))
 
@@ -63,15 +64,15 @@
   do {                          \
     ::arrow::Status s = (expr); \
     EXPECT_TRUE(s.ok());        \
-  } while (0)
+  } while (false)
 
 #define ABORT_NOT_OK(s)                  \
   do {                                   \
     ::arrow::Status _s = (s);            \
     if (ARROW_PREDICT_FALSE(!_s.ok())) { \
-      exit(-1);                          \
+      exit(EXIT_FAILURE);                \
     }                                    \
-  } while (0);
+  } while (false);
 
 namespace arrow {
 
@@ -79,27 +80,22 @@ using ArrayVector = std::vector<std::shared_ptr<Array>>;
 
 namespace test {
 
-template <typename T>
-void randint(int64_t N, T lower, T upper, std::vector<T>* out) {
-  Random rng(random_seed());
-  uint64_t draw;
-  uint64_t span = upper - lower;
-  T val;
-  for (int64_t i = 0; i < N; ++i) {
-    draw = rng.Uniform64(span);
-    val = static_cast<T>(draw + lower);
-    out->push_back(val);
-  }
+template <typename T, typename U>
+void randint(int64_t N, T lower, T upper, std::vector<U>* out) {
+  const int random_seed = 0;
+  std::mt19937 gen(random_seed);
+  std::uniform_int_distribution<T> d(lower, upper);
+  out->resize(N, static_cast<T>(0));
+  std::generate(out->begin(), out->end(), [&d, &gen] { return static_cast<U>(d(gen)); });
 }
 
-template <typename T>
+template <typename T, typename U>
 void random_real(int64_t n, uint32_t seed, T min_value, T max_value,
-                 std::vector<T>* out) {
+                 std::vector<U>* out) {
   std::mt19937 gen(seed);
   std::uniform_real_distribution<T> d(min_value, max_value);
-  for (int64_t i = 0; i < n; ++i) {
-    out->push_back(d(gen));
-  }
+  out->resize(n, static_cast<T>(0));
+  std::generate(out->begin(), out->end(), [&d, &gen] { return static_cast<U>(d(gen)); });
 }
 
 template <typename T>
@@ -115,7 +111,8 @@ inline Status CopyBufferFromVector(const std::vector<T>& values, MemoryPool* poo
 
   auto buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(buffer->Resize(nbytes));
-  memcpy(buffer->mutable_data(), values.data(), nbytes);
+  auto immutable_data = reinterpret_cast<const uint8_t*>(values.data());
+  std::copy(immutable_data, immutable_data + nbytes, buffer->mutable_data());
 
   *result = buffer;
   return Status::OK();
@@ -143,56 +140,173 @@ static inline Status GetBitmapFromVector(const std::vector<T>& is_valid,
 // Sets approximately pct_null of the first n bytes in null_bytes to zero
 // and the rest to non-zero (true) values.
 static inline void random_null_bytes(int64_t n, double pct_null, uint8_t* null_bytes) {
-  Random rng(random_seed());
-  for (int64_t i = 0; i < n; ++i) {
-    null_bytes[i] = rng.NextDoubleFraction() > pct_null;
-  }
+  const int random_seed = 0;
+  std::mt19937 gen(random_seed);
+  std::uniform_real_distribution<double> d(0.0, 1.0);
+  std::generate(null_bytes, null_bytes + n,
+                [&d, &gen, &pct_null] { return d(gen) > pct_null; });
 }
 
 static inline void random_is_valid(int64_t n, double pct_null,
                                    std::vector<bool>* is_valid) {
-  Random rng(random_seed());
-  for (int64_t i = 0; i < n; ++i) {
-    is_valid->push_back(rng.NextDoubleFraction() > pct_null);
-  }
+  const int random_seed = 0;
+  std::mt19937 gen(random_seed);
+  std::uniform_real_distribution<double> d(0.0, 1.0);
+  is_valid->resize(n, false);
+  std::generate(is_valid->begin(), is_valid->end(),
+                [&d, &gen, &pct_null] { return d(gen) > pct_null; });
 }
 
 static inline void random_bytes(int64_t n, uint32_t seed, uint8_t* out) {
   std::mt19937 gen(seed);
-  std::uniform_int_distribution<int> d(0, 255);
+  std::uniform_int_distribution<int> d(0, std::numeric_limits<uint8_t>::max());
+  std::generate(out, out + n, [&d, &gen] { return static_cast<uint8_t>(d(gen) & 0xFF); });
+}
 
-  for (int64_t i = 0; i < n; ++i) {
-    out[i] = static_cast<uint8_t>(d(gen) & 0xFF);
+static void DecimalRange(int32_t precision, Decimal128* min_decimal,
+                         Decimal128* max_decimal) {
+  DCHECK_GE(precision, 1) << "decimal precision must be greater than or equal to 1, got "
+                          << precision;
+  DCHECK_LE(precision, 38) << "decimal precision must be less than or equal to 38, got "
+                           << precision;
+
+  switch (precision) {
+    case 1:
+    case 2:
+      *max_decimal = std::numeric_limits<int8_t>::max();
+      break;
+    case 3:
+    case 4:
+      *max_decimal = std::numeric_limits<int16_t>::max();
+      break;
+    case 5:
+    case 6:
+      *max_decimal = 8388607;
+      break;
+    case 7:
+    case 8:
+    case 9:
+      *max_decimal = std::numeric_limits<int32_t>::max();
+      break;
+    case 10:
+    case 11:
+      *max_decimal = 549755813887;
+      break;
+    case 12:
+    case 13:
+    case 14:
+      *max_decimal = 140737488355327;
+      break;
+    case 15:
+    case 16:
+      *max_decimal = 36028797018963967;
+      break;
+    case 17:
+    case 18:
+      *max_decimal = std::numeric_limits<int64_t>::max();
+      break;
+    case 19:
+    case 20:
+    case 21:
+      *max_decimal = Decimal128("2361183241434822606847");
+      break;
+    case 22:
+    case 23:
+      *max_decimal = Decimal128("604462909807314587353087");
+      break;
+    case 24:
+    case 25:
+    case 26:
+      *max_decimal = Decimal128("154742504910672534362390527");
+      break;
+    case 27:
+    case 28:
+      *max_decimal = Decimal128("39614081257132168796771975167");
+      break;
+    case 29:
+    case 30:
+    case 31:
+      *max_decimal = Decimal128("10141204801825835211973625643007");
+      break;
+    case 32:
+    case 33:
+      *max_decimal = Decimal128("2596148429267413814265248164610047");
+      break;
+    case 34:
+    case 35:
+      *max_decimal = Decimal128("664613997892457936451903530140172287");
+      break;
+    case 36:
+    case 37:
+    case 38:
+      *max_decimal = Decimal128("170141183460469231731687303715884105727");
+      break;
+    default:
+      DCHECK(false);
+      break;
   }
+
+  *min_decimal = ~(*max_decimal);
 }
 
-static inline void random_ascii(int64_t n, uint32_t seed, uint8_t* out) {
+class UniformDecimalDistribution {
+ public:
+  explicit UniformDecimalDistribution(int32_t precision) {
+    Decimal128 max_decimal;
+    Decimal128 min_decimal;
+    DecimalRange(precision, &min_decimal, &max_decimal);
+
+    const auto min_low = static_cast<int64_t>(min_decimal.low_bits());
+    const auto max_low = static_cast<int64_t>(max_decimal.low_bits());
+
+    const int64_t min_high = min_decimal.high_bits();
+    const int64_t max_high = max_decimal.high_bits();
+
+    using param_type = std::uniform_int_distribution<int64_t>::param_type;
+
+    lower_dist_.param(param_type(min_low, max_low));
+    upper_dist_.param(param_type(min_high, max_high));
+  }
+
+  template <typename Generator>
+  Decimal128 operator()(Generator& gen) {
+    return Decimal128(upper_dist_(gen), static_cast<uint64_t>(lower_dist_(gen)));
+  }
+
+ private:
+  // The lower bits distribution is intentionally int64_t.
+  // If it were uint64_t then the size of the interval [min_high, max_high] would be 0
+  // because min_high > max_high due to 2's complement.
+  // So, we generate the same range of bits using int64_t and then cast to uint64_t.
+  std::uniform_int_distribution<int64_t> lower_dist_;
+  std::uniform_int_distribution<int64_t> upper_dist_;
+};
+
+static inline void random_decimals(int64_t n, uint32_t seed, int32_t precision,
+                                   uint8_t* out) {
   std::mt19937 gen(seed);
-  std::uniform_int_distribution<int> d(65, 122);
+  UniformDecimalDistribution dist(precision);
 
-  for (int64_t i = 0; i < n; ++i) {
-    out[i] = static_cast<uint8_t>(d(gen) & 0xFF);
+  for (int64_t i = 0; i < n; ++i, out += 16) {
+    const Decimal128 value(dist(gen));
+    value.ToBytes(out);
   }
 }
 
-template <typename T>
-void rand_uniform_int(int64_t n, uint32_t seed, T min_value, T max_value, T* out) {
+template <typename T, typename U>
+void rand_uniform_int(int64_t n, uint32_t seed, T min_value, T max_value, U* out) {
   DCHECK(out || (n == 0));
   std::mt19937 gen(seed);
   std::uniform_int_distribution<T> d(min_value, max_value);
-  for (int64_t i = 0; i < n; ++i) {
-    out[i] = static_cast<T>(d(gen));
-  }
+  std::generate(out, out + n, [&d, &gen] { return static_cast<U>(d(gen)); });
+}
+
+static inline void random_ascii(int64_t n, uint32_t seed, uint8_t* out) {
+  rand_uniform_int(n, seed, static_cast<int32_t>('A'), static_cast<int32_t>('z'), out);
 }
 
 static inline int64_t null_count(const std::vector<uint8_t>& valid_bytes) {
-  int64_t result = 0;
-  for (size_t i = 0; i < valid_bytes.size(); ++i) {
-    if (valid_bytes[i] == 0) {
-      ++result;
-    }
-  }
-  return result;
+  return static_cast<int64_t>(std::count(valid_bytes.cbegin(), valid_bytes.cend(), '\0'));
 }
 
 Status MakeRandomInt32PoolBuffer(int64_t length, MemoryPool* pool,
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 5df5e748f39e5..7810a3be46da5 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -38,7 +38,6 @@ install(FILES
   logging.h
   macros.h
   parallel.h
-  random.h
   rle-encoding.h
   sse-util.h
   stl.h
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 9d94bef847fa1..cc180258aa4df 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -43,14 +43,17 @@ Decimal128::Decimal128(const uint8_t* bytes)
 }
 
 std::array<uint8_t, 16> Decimal128::ToBytes() const {
-  const uint64_t raw[] = {BitUtil::ToLittleEndian(low_bits_),
-                          BitUtil::ToLittleEndian(static_cast<uint64_t>(high_bits_))};
-  const auto* raw_data = reinterpret_cast<const uint8_t*>(raw);
   std::array<uint8_t, 16> out{{0}};
-  std::copy(raw_data, raw_data + out.size(), out.begin());
+  ToBytes(out.data());
   return out;
 }
 
+void Decimal128::ToBytes(uint8_t* out) const {
+  DCHECK_NE(out, NULLPTR);
+  reinterpret_cast<uint64_t*>(out)[0] = BitUtil::ToLittleEndian(low_bits_);
+  reinterpret_cast<int64_t*>(out)[1] = BitUtil::ToLittleEndian(high_bits_);
+}
+
 static constexpr Decimal128 kTenTo36(static_cast<int64_t>(0xC097CE7BC90715),
                                      0xB34B9F1000000000);
 static constexpr Decimal128 kTenTo18(0xDE0B6B3A7640000);
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 487f222580201..a0423e9fce49c 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -102,13 +102,14 @@ class ARROW_EXPORT Decimal128 {
   Decimal128& operator>>=(uint32_t bits);
 
   /// \brief Get the high bits of the two's complement representation of the number.
-  int64_t high_bits() const { return high_bits_; }
+  inline int64_t high_bits() const { return high_bits_; }
 
   /// \brief Get the low bits of the two's complement representation of the number.
-  uint64_t low_bits() const { return low_bits_; }
+  inline uint64_t low_bits() const { return low_bits_; }
 
   /// \brief Return the raw bytes of the value in little-endian byte order.
   std::array<uint8_t, 16> ToBytes() const;
+  void ToBytes(uint8_t* out) const;
 
   /// \brief Convert the Decimal128 value to a base 10 decimal string with the given
   /// scale.
diff --git a/cpp/src/arrow/util/random.h b/cpp/src/arrow/util/random.h
deleted file mode 100644
index 2e05a73033d0f..0000000000000
--- a/cpp/src/arrow/util/random.h
+++ /dev/null
@@ -1,126 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-// Moved from Kudu http://github.com/cloudera/kudu
-
-#ifndef ARROW_UTIL_RANDOM_H_
-#define ARROW_UTIL_RANDOM_H_
-
-#include <stdint.h>
-
-#include <cmath>
-
-namespace arrow {
-namespace internal {
-namespace random {
-
-static const uint32_t M = 2147483647L;  // 2^31-1
-const double kTwoPi = 6.283185307179586476925286;
-
-}  // namespace random
-}  // namespace internal
-
-// A very simple random number generator.  Not especially good at
-// generating truly random bits, but good enough for our needs in this
-// package. This implementation is not thread-safe.
-class Random {
- public:
-  explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) {
-    // Avoid bad seeds.
-    if (seed_ == 0 || seed_ == internal::random::M) {
-      seed_ = 1;
-    }
-  }
-
-  // Next pseudo-random 32-bit unsigned integer.
-  // FIXME: This currently only generates 31 bits of randomness.
-  // The MSB will always be zero.
-  uint32_t Next() {
-    static const uint64_t A = 16807;  // bits 14, 8, 7, 5, 2, 1, 0
-    // We are computing
-    //       seed_ = (seed_ * A) % M,    where M = 2^31-1
-    //
-    // seed_ must not be zero or M, or else all subsequent computed values
-    // will be zero or M respectively.  For all other values, seed_ will end
-    // up cycling through every number in [1,M-1]
-    uint64_t product = seed_ * A;
-
-    // Compute (product % M) using the fact that ((x << 31) % M) == x.
-    seed_ = static_cast<uint32_t>((product >> 31) + (product & internal::random::M));
-    // The first reduction may overflow by 1 bit, so we may need to
-    // repeat.  mod == M is not possible; using > allows the faster
-    // sign-bit-based test.
-    if (seed_ > internal::random::M) {
-      seed_ -= internal::random::M;
-    }
-    return seed_;
-  }
-
-  // Alias for consistency with Next64
-  uint32_t Next32() { return Next(); }
-
-  // Next pseudo-random 64-bit unsigned integer.
-  // FIXME: This currently only generates 62 bits of randomness due to Next()
-  // only giving 31 bits of randomness. The 2 most significant bits will always
-  // be zero.
-  uint64_t Next64() {
-    uint64_t large = Next();
-    // Only shift by 31 bits so we end up with zeros in MSB and not scattered
-    // throughout the 64-bit word. This is due to the weakness in Next() noted
-    // above.
-    large <<= 31;
-    large |= Next();
-    return large;
-  }
-
-  // Returns a uniformly distributed value in the range [0..n-1]
-  // REQUIRES: n > 0
-  uint32_t Uniform(uint32_t n) { return Next() % n; }
-
-  // Alias for consistency with Uniform64
-  uint32_t Uniform32(uint32_t n) { return Uniform(n); }
-
-  // Returns a uniformly distributed 64-bit value in the range [0..n-1]
-  // REQUIRES: n > 0
-  uint64_t Uniform64(uint64_t n) { return Next64() % n; }
-
-  // Randomly returns true ~"1/n" of the time, and false otherwise.
-  // REQUIRES: n > 0
-  bool OneIn(int n) { return (Next() % n) == 0; }
-
-  // Skewed: pick "base" uniformly from range [0,max_log] and then
-  // return "base" random bits.  The effect is to pick a number in the
-  // range [0,2^max_log-1] with exponential bias towards smaller numbers.
-  uint32_t Skewed(int max_log) { return Uniform(1 << Uniform(max_log + 1)); }
-
-  // Creates a normal distribution variable using the
-  // Box-Muller transform. See:
-  // http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform
-  // Adapted from WebRTC source code at:
-  // webrtc/trunk/modules/video_coding/main/test/test_util.cc
-  double Normal(double mean, double std_dev) {
-    double uniform1 = (Next() + 1.0) / (internal::random::M + 1.0);
-    double uniform2 = (Next() + 1.0) / (internal::random::M + 1.0);
-    return (mean +
-            std_dev * sqrt(-2 * ::log(uniform1)) *
-                cos(internal::random::kTwoPi * uniform2));
-  }
-
-  // Return a random number between 0.0 and 1.0 inclusive.
-  double NextDoubleFraction() {
-    return Next() / static_cast<double>(internal::random::M + 1.0);
-  }
-
- private:
-  uint32_t seed_;
-};
-
-uint32_t random_seed() {
-  // TODO(wesm): use system time to get a reasonably random seed
-  return 0;
-}
-
-}  // namespace arrow
-
-#endif  // ARROW_UTIL_RANDOM_H_

From 99ea353dbaf15b8db1ad7d8d4419643abe99189d Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 6 Nov 2017 22:24:19 -0500
Subject: [PATCH 1223/1644] ARROW-1771: [C++] ARROW-1749 Breaks Public API test
 in parquet-cpp

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1288 from cpcloud/ARROW-1771 and squashes the following commits:

dc002568 [Phillip Cloud] Formatting again
ac2367b4 [Phillip Cloud] Fix formatting
35e67e50 [Phillip Cloud] Add public API test
659e5165 [Phillip Cloud] Fix formatting
de4632ac [Phillip Cloud] ARROW-1771: [C++] ARROW-1749 Breaks Public API test in parquet-cpp
---
 cpp/src/arrow/CMakeLists.txt     |  1 +
 cpp/src/arrow/public-api-test.cc | 26 ++++++++++++++++++++++++++
 cpp/src/arrow/util/bit-util.h    |  3 +--
 3 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 cpp/src/arrow/public-api-test.cc

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 22b475146da7c..69d5052330c3a 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -171,6 +171,7 @@ ADD_ARROW_TEST(array-test)
 ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(memory_pool-test)
 ADD_ARROW_TEST(pretty_print-test)
+ADD_ARROW_TEST(public-api-test)
 ADD_ARROW_TEST(status-test)
 ADD_ARROW_TEST(type-test)
 ADD_ARROW_TEST(table-test)
diff --git a/cpp/src/arrow/public-api-test.cc b/cpp/src/arrow/public-api-test.cc
new file mode 100644
index 0000000000000..8298d748fe8fd
--- /dev/null
+++ b/cpp/src/arrow/public-api-test.cc
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/api.h"
+
+#ifdef DCHECK
+#error "DCHECK should not be visible from Arrow public headers."
+#endif
+
+#include <gtest/gtest.h>
+
+TEST(_, _) {}
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index d6415f3c75b7b..cab3c9ee703c9 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -57,7 +57,6 @@
 #include <memory>
 #include <vector>
 
-#include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/type_traits.h"
 #include "arrow/util/visibility.h"
@@ -301,7 +300,7 @@ static inline int Log2(uint64_t x) {
 
 /// \brief Count the number of leading zeros in a 32 bit integer.
 static inline int64_t CountLeadingZeros(uint32_t value) {
-  DCHECK_NE(value, 0);
+// DCHECK_NE(value, 0);
 #if defined(__clang__) || defined(__GNUC__)
   return static_cast<int64_t>(__builtin_clz(value));
 #elif defined(_MSC_VER)

From 3995eb3c1d94ab8fd151c32a5d1994a6b085deec Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 6 Nov 2017 22:24:51 -0500
Subject: [PATCH 1224/1644] ARROW-1768: [Python] Fix suppressed exception in
 ParquetWriter.__del__

This closes [ARROW-1768](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1768).

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Licht-T <licht-t@outlook.jp>

Closes #1286 from Licht-T/fix-suppressed-exception-in-parquetwriter-del and squashes the following commits:

ccd7344d [Wes McKinney] Use getattr to be robust to is_open attribute not existing
4a42683d [Licht-T] Fix suppressed exception in ParquetWriter.__del__
---
 python/pyarrow/parquet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 9dcc30c8af479..9e0749bb35c9e 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -260,7 +260,7 @@ def __init__(self, where, schema, flavor=None,
         self.is_open = True
 
     def __del__(self):
-        if self.is_open:
+        if getattr(self, 'is_open', False):
             self.close()
 
     def write_table(self, table, row_group_size=None):

From e631119d853a3182fa4e2d8cd980440f59c7a679 Mon Sep 17 00:00:00 2001
From: Victor Uriarte <vmuriart@gmail.com>
Date: Tue, 7 Nov 2017 11:57:43 -0700
Subject: [PATCH 1225/1644] [Format] Fix link to Flatbuffers project in IPC.md

Close #1291

Change-Id: Ibae55c472b08cedcf22bd783c7a7fce1449051fe
---
 format/IPC.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/format/IPC.md b/format/IPC.md
index f3b48854c2072..5a5d3aef62be7 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -252,5 +252,5 @@ shared memory region) to be a multiple of 8:
 
 [1]: https://github.com/apache/arrow/blob/master/format/File.fbs
 [2]: https://github.com/apache/arrow/blob/master/format/Message.fbs
-[3]: https://github.com/google]/flatbuffers
+[3]: https://github.com/google/flatbuffers
 [4]: https://github.com/apache/arrow/blob/master/format/Layout.md

From 3188d70202795d8e0a8092ec5685d859b02e366d Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Tue, 7 Nov 2017 16:22:20 -0500
Subject: [PATCH 1226/1644] ARROW-1716: [Format/JSON] Use string integer value
 for Decimals in JSON

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1267 from cpcloud/ARROW-1716 and squashes the following commits:

b4f3aed2 [Phillip Cloud] Add cases for every valid precision
f8d4391f [Phillip Cloud] Use the full range of decimal values in integration tests
6fef5f71 [Phillip Cloud] ARROW-1716: [Format/JSON] Use string integer value for Decimals in JSON
---
 cpp/src/arrow/ipc/ipc-read-write-test.cc      |   4 +-
 cpp/src/arrow/ipc/json-internal.cc            |  69 +++++++++---
 cpp/src/arrow/ipc/test-common.h               |  13 ++-
 integration/integration_test.py               | 105 ++++++++++--------
 .../vector/file/json/JsonFileReader.java      |   9 +-
 .../vector/file/json/JsonFileWriter.java      |   8 +-
 .../arrow/vector/util/DecimalUtility.java     |  32 +++++-
 7 files changed, 165 insertions(+), 75 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 6f2f5cf856055..40cd3f0eef0b8 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -727,7 +727,7 @@ TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
   int64_t size = 24;
 
   std::vector<int64_t> values;
-  test::randint<int64_t>(size, 0, 100, &values);
+  test::randint(size, 0, 100, &values);
 
   auto data = test::GetBufferFromVector(values);
 
@@ -748,7 +748,7 @@ TEST_F(TestTensorRoundTrip, NonContiguous) {
   ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_));
 
   std::vector<int64_t> values;
-  test::randint<int64_t>(24, 0, 100, &values);
+  test::randint(24, 0, 100, &values);
 
   auto data = test::GetBufferFromVector(values);
   Tensor tensor(int64(), data, {4, 3}, {48, 16});
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 025f6c276541e..c1c0661d6ad35 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -33,6 +33,7 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/string.h"
 #include "arrow/visitor_inline.h"
@@ -448,7 +449,8 @@ class ArrayWriter {
   }
 
   void WriteDataValues(const FixedSizeBinaryArray& arr) {
-    int32_t width = arr.byte_width();
+    const int32_t width = arr.byte_width();
+
     for (int64_t i = 0; i < arr.length(); ++i) {
       const uint8_t* buf = arr.GetValue(i);
       std::string encoded = HexEncode(buf, width);
@@ -456,6 +458,13 @@ class ArrayWriter {
     }
   }
 
+  void WriteDataValues(const DecimalArray& arr) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      const Decimal128 value(arr.GetValue(i));
+      writer_->String(value.ToIntegerString());
+    }
+  }
+
   void WriteDataValues(const BooleanArray& arr) {
     for (int i = 0; i < arr.length(); ++i) {
       writer_->Bool(arr.Value(i));
@@ -1053,7 +1062,9 @@ class ArrayReader {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value, Status>::type
+  typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value &&
+                              !std::is_base_of<DecimalType, T>::value,
+                          Status>::type
   Visit(const T& type) {
     typename TypeTraits<T>::BuilderType builder(type_, pool_);
 
@@ -1073,22 +1084,52 @@ class ArrayReader {
     for (int i = 0; i < length_; ++i) {
       if (!is_valid_[i]) {
         RETURN_NOT_OK(builder.AppendNull());
-        continue;
-      }
+      } else {
+        const rj::Value& val = json_data_arr[i];
+        DCHECK(val.IsString())
+            << "Found non-string JSON value when parsing FixedSizeBinary value";
+        std::string hex_string = val.GetString();
+        if (static_cast<int32_t>(hex_string.size()) != byte_width * 2) {
+          DCHECK(false) << "Expected size: " << byte_width * 2
+                        << " got: " << hex_string.size();
+        }
+        const char* hex_data = hex_string.c_str();
 
-      const rj::Value& val = json_data_arr[i];
-      DCHECK(val.IsString());
-      std::string hex_string = val.GetString();
-      if (static_cast<int32_t>(hex_string.size()) != byte_width * 2) {
-        DCHECK(false) << "Expected size: " << byte_width * 2
-                      << " got: " << hex_string.size();
+        for (int32_t j = 0; j < byte_width; ++j) {
+          RETURN_NOT_OK(ParseHexValue(hex_data + j * 2, &byte_buffer_data[j]));
+        }
+        RETURN_NOT_OK(builder.Append(byte_buffer_data));
       }
-      const char* hex_data = hex_string.c_str();
+    }
+    return builder.Finish(&result_);
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<DecimalType, T>::value, Status>::type Visit(
+      const T& type) {
+    typename TypeTraits<T>::BuilderType builder(type_, pool_);
+
+    const auto& json_data = obj_->FindMember("DATA");
+    RETURN_NOT_ARRAY("DATA", json_data, *obj_);
 
-      for (int32_t j = 0; j < byte_width; ++j) {
-        RETURN_NOT_OK(ParseHexValue(hex_data + j * 2, &byte_buffer_data[j]));
+    const auto& json_data_arr = json_data->value.GetArray();
+
+    DCHECK_EQ(static_cast<int32_t>(json_data_arr.Size()), length_);
+
+    for (int i = 0; i < length_; ++i) {
+      if (!is_valid_[i]) {
+        RETURN_NOT_OK(builder.AppendNull());
+      } else {
+        const rj::Value& val = json_data_arr[i];
+        DCHECK(val.IsString())
+            << "Found non-string JSON value when parsing Decimal128 value";
+        DCHECK_GT(val.GetStringLength(), 0)
+            << "Empty string found when parsing Decimal128 value";
+
+        Decimal128 value;
+        RETURN_NOT_OK(Decimal128::FromString(val.GetString(), &value));
+        RETURN_NOT_OK(builder.Append(value));
       }
-      RETURN_NOT_OK(builder.Append(byte_buffer_data));
     }
     return builder.Finish(&result_);
   }
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index b2137b7dbef6a..91023db489852 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -671,8 +671,11 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
 }
 
 Status MakeDecimal(std::shared_ptr<RecordBatch>* out) {
-  auto f0 = field("f0", decimal(19, 4));
-  auto schema = ::arrow::schema({f0, f0});
+  constexpr int kDecimalPrecision = 38;
+  auto type = decimal(kDecimalPrecision, 4);
+  auto f0 = field("f0", type);
+  auto f1 = field("f1", type);
+  auto schema = ::arrow::schema({f0, f1});
 
   constexpr int kDecimalSize = 16;
   constexpr int length = 10;
@@ -682,7 +685,7 @@ Status MakeDecimal(std::shared_ptr<RecordBatch>* out) {
 
   RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), kDecimalSize * length, &data));
 
-  test::random_bytes(kDecimalSize * length, 0, data->mutable_data());
+  test::random_decimals(length, 1, kDecimalPrecision, data->mutable_data());
   test::random_null_bytes(length, 0.1, is_valid_bytes.data());
 
   RETURN_NOT_OK(BitUtil::BytesToBits(is_valid_bytes, default_memory_pool(), &is_valid));
@@ -690,10 +693,10 @@ Status MakeDecimal(std::shared_ptr<RecordBatch>* out) {
   auto a1 = std::make_shared<DecimalArray>(f0->type(), length, data, is_valid,
                                            kUnknownNullCount);
 
-  auto a2 = std::make_shared<DecimalArray>(f0->type(), length, data);
+  auto a2 = std::make_shared<DecimalArray>(f1->type(), length, data);
 
   ArrayVector arrays = {a1, a2};
-  *out = std::make_shared<RecordBatch>(schema, a1->length(), arrays);
+  *out = std::make_shared<RecordBatch>(schema, length, arrays);
   return Status::OK();
 }
 
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 59a1de5a4639d..205176eccc11a 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -65,24 +65,16 @@ def rands(nchars):
     return ''.join(np.random.choice(RANDS_CHARS, nchars))
 
 
-if six.PY2:
-    def frombytes(o):
-        return o
+def tobytes(o):
+    if isinstance(o, six.text_type):
+        return o.encode('utf8')
+    return o
 
-    def tobytes(o):
-        if isinstance(o, unicode):
-            return o.encode('utf8')
-        else:
-            return o
-else:
-    def tobytes(o):
-        if isinstance(o, str):
-            return o.encode('utf8')
-        else:
-            return o
 
-    def frombytes(o):
+def frombytes(o):
+    if isinstance(o, six.binary_type):
         return o.decode('utf8')
+    return o
 
 
 # from the merge_arrow_pr.py script
@@ -177,7 +169,7 @@ def _get_type_layout(self):
 class PrimitiveColumn(Column):
 
     def __init__(self, name, count, is_valid, values):
-        Column.__init__(self, name, count)
+        super(PrimitiveColumn, self).__init__(name, count)
         self.is_valid = is_valid
         self.values = values
 
@@ -191,15 +183,16 @@ def _get_buffers(self):
         ]
 
 
-TEST_INT_MIN = - 2**31 + 1
-TEST_INT_MAX = 2**31 - 1
+TEST_INT_MAX = 2 ** 31 - 1
+TEST_INT_MIN = ~TEST_INT_MAX
+
 
 class IntegerType(PrimitiveType):
 
     def __init__(self, name, is_signed, bit_width, nullable=True,
                  min_value=TEST_INT_MIN,
                  max_value=TEST_INT_MAX):
-        PrimitiveType.__init__(self, name, nullable=nullable)
+        super(IntegerType, self).__init__(name, nullable=nullable)
         self.is_signed = is_signed
         self.bit_width = bit_width
         self.min_value = min_value
@@ -239,9 +232,11 @@ class DateType(IntegerType):
     MILLISECOND = 1
 
     def __init__(self, name, unit, nullable=True):
-        self.unit = unit
         bit_width = 32 if unit == self.DAY else 64
-        IntegerType.__init__(self, name, True, bit_width, nullable=nullable)
+        super(DateType, self).__init__(
+            name, True, bit_width, nullable=nullable
+        )
+        self.unit = unit
 
     def _get_type(self):
         return OrderedDict([
@@ -268,9 +263,10 @@ class TimeType(IntegerType):
     }
 
     def __init__(self, name, unit='s', nullable=True):
+        super(TimeType, self).__init__(
+            name, True, self.BIT_WIDTHS[unit], nullable=nullable
+        )
         self.unit = unit
-        IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit],
-                             nullable=nullable)
 
     def _get_type(self):
         return OrderedDict([
@@ -283,9 +279,9 @@ def _get_type(self):
 class TimestampType(IntegerType):
 
     def __init__(self, name, unit='s', tz=None, nullable=True):
+        super(TimestampType, self).__init__(name, True, 64, nullable=nullable)
         self.unit = unit
         self.tz = tz
-        IntegerType.__init__(self, name, True, 64, nullable=nullable)
 
     def _get_type(self):
         fields = [
@@ -302,7 +298,7 @@ def _get_type(self):
 class FloatingPointType(PrimitiveType):
 
     def __init__(self, name, bit_width, nullable=True):
-        PrimitiveType.__init__(self, name, nullable=nullable)
+        super(FloatingPointType, self).__init__(name, nullable=nullable)
 
         self.bit_width = bit_width
         self.precision = {
@@ -331,13 +327,30 @@ def generate_column(self, size, name=None):
         return PrimitiveColumn(name, size, is_valid, values)
 
 
-class DecimalType(PrimitiveType):
-    def __init__(self, name, bit_width, precision, scale, nullable=True):
-        PrimitiveType.__init__(self, name, nullable=True)
+DECIMAL_PRECISION_TO_VALUE = {
+    key: (1 << (8 * i - 1)) - 1 for i, key in enumerate(
+        [1, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24, 27, 29, 32, 34, 36],
+        start=1,
+    )
+}
 
-        self.bit_width = bit_width
+
+def decimal_range_from_precision(precision):
+    assert 1 <= precision <= 38
+    try:
+        max_value = DECIMAL_PRECISION_TO_VALUE[precision]
+    except KeyError:
+        return decimal_range_from_precision(precision - 1)
+    else:
+        return ~max_value, max_value
+
+
+class DecimalType(PrimitiveType):
+    def __init__(self, name, precision, scale, bit_width=128, nullable=True):
+        super(DecimalType, self).__init__(name, nullable=True)
         self.precision = precision
         self.scale = scale
+        self.bit_width = bit_width
 
     @property
     def numpy_type(self):
@@ -359,7 +372,8 @@ def _get_type_layout(self):
                            ('typeBitWidth', self.bit_width)])])])
 
     def generate_column(self, size, name=None):
-        values = [random.randint(0, 2**self.bit_width - 1) for x in range(size)]
+        min_value, max_value = decimal_range_from_precision(self.precision)
+        values = [random.randint(min_value, max_value) for _ in range(size)]
 
         is_valid = self._make_is_valid(size)
         if name is None:
@@ -369,14 +383,12 @@ def generate_column(self, size, name=None):
 
 class DecimalColumn(PrimitiveColumn):
 
-    def __init__(self, name, count, is_valid, values, bit_width):
-        PrimitiveColumn.__init__(self, name, count, is_valid, values)
+    def __init__(self, name, count, is_valid, values, bit_width=128):
+        super(DecimalColumn, self).__init__(name, count, is_valid, values)
         self.bit_width = bit_width
-        self.hex_width = bit_width / 4
 
     def _encode_value(self, x):
-        hex_format_str = '%%0%dx' % self.hex_width
-        return (hex_format_str % x).upper()
+        return str(x)
 
 
 class BooleanType(PrimitiveType):
@@ -510,7 +522,7 @@ def _encode_value(self, x):
 class ListType(DataType):
 
     def __init__(self, name, value_type, nullable=True):
-        DataType.__init__(self, name, nullable=nullable)
+        super(ListType, self).__init__(name, nullable=nullable)
         self.value_type = value_type
 
     def _get_type(self):
@@ -553,7 +565,7 @@ def generate_column(self, size, name=None):
 class ListColumn(Column):
 
     def __init__(self, name, count, is_valid, offsets, values):
-        Column.__init__(self, name, count)
+        super(ListColumn, self).__init__(name, count)
         self.is_valid = is_valid
         self.offsets = offsets
         self.values = values
@@ -571,7 +583,7 @@ def _get_children(self):
 class StructType(DataType):
 
     def __init__(self, name, field_types, nullable=True):
-        DataType.__init__(self, name, nullable=nullable)
+        super(StructType, self).__init__(name, nullable=nullable)
         self.field_types = field_types
 
     def _get_type(self):
@@ -620,7 +632,7 @@ def get_json(self):
 class DictionaryType(DataType):
 
     def __init__(self, name, index_type, dictionary, nullable=True):
-        DataType.__init__(self, name, nullable=nullable)
+        super(DictionaryType, self).__init__(name, nullable=nullable)
         assert isinstance(index_type, IntegerType)
         assert isinstance(dictionary, Dictionary)
 
@@ -655,7 +667,7 @@ def generate_column(self, size, name=None):
 class StructColumn(Column):
 
     def __init__(self, name, count, is_valid, field_values):
-        Column.__init__(self, name, count)
+        super(StructColumn, self).__init__(name, count)
         self.is_valid = is_valid
         self.field_values = field_values
 
@@ -758,11 +770,12 @@ def generate_primitive_case(batch_sizes):
 
 def generate_decimal_case():
     fields = [
-        DecimalType('f1', 128, 24, 10, True),
-        DecimalType('f2', 128, 32, -10, True)
+        DecimalType(name='f{}'.format(i), precision=precision, scale=2)
+        for i, precision in enumerate(range(3, 39))
     ]
 
-    batch_sizes = [7, 10]
+    possible_batch_sizes = 7, 10
+    batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))]
 
     return _generate_file('decimal', fields, batch_sizes)
 
@@ -867,8 +880,9 @@ def run(self):
 
     def _compare_implementations(self, producer, consumer):
         print('##########################################################')
-        print('{0} producing, {1} consuming'.format(producer.name,
-                                                       consumer.name))
+        print(
+            '{0} producing, {1} consuming'.format(producer.name, consumer.name)
+        )
         print('##########################################################')
 
         for json_path in self.json_files:
@@ -1033,6 +1047,7 @@ def run_all_tests(debug=False):
     runner.run()
     print('-- All tests passed!')
 
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Arrow integration test CLI')
     parser.add_argument('--debug', dest='debug', action='store_true',
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index c6ebd61aa07b9..e1c7c909f10be 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -27,6 +27,8 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -332,9 +334,10 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti
         ((Float8Vector) valueVector).getMutator().set(i, parser.readValueAs(Double.class));
         break;
       case DECIMAL: {
-          DecimalVector decimalVector = ((DecimalVector) valueVector);
-          byte[] value = decodeHexSafe(parser.readValueAs(String.class));
-          DecimalUtility.writeByteArrayToArrowBuf(value, decimalVector.getBuffer(), i);
+          DecimalVector decimalVector = (DecimalVector) valueVector;
+          // Here we assume the decimal value is the unscaled integer value as a string
+          BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class));
+          DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, decimalVector.getBuffer(), i);
         }
         break;
       case VARBINARY:
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index 04e44379e5dfa..05341bec44ea3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -20,6 +20,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -48,6 +49,7 @@
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 
@@ -242,9 +244,9 @@ private void writeValueToGenerator(ValueVector valueVector, int i) throws IOExce
         }
         break;
       case DECIMAL: {
-          ArrowBuf bytebuf = valueVector.getDataBuffer();
-          String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(bytebuf, i));
-          generator.writeString(hexString);
+          BigDecimal decimalValue = ((DecimalVector) valueVector).getAccessor().getObject(i);
+          // We write the unscaled value, because the scale is stored in the type metadata.
+          generator.writeString(decimalValue.unscaledValue().toString());
         }
         break;
       default:
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index 033ae6c09914d..acf7c58a1337f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -142,8 +142,18 @@ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
    */
   public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale) {
     byte[] value = new byte[DECIMAL_BYTE_LENGTH];
+    byte temp;
     final int startIndex = index * DECIMAL_BYTE_LENGTH;
+
+    // Decimal stored as little endian, need to swap bytes to make BigDecimal
     bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH);
+    int stop = DECIMAL_BYTE_LENGTH / 2;
+    for (int i = 0, j; i < stop; i++) {
+      temp = value[i];
+      j = (DECIMAL_BYTE_LENGTH - 1) - i;
+      value[i] = value[j];
+      value[j] = temp;
+    }
     BigInteger unscaledValue = new BigInteger(value);
     return new BigDecimal(unscaledValue, scale);
   }
@@ -212,10 +222,26 @@ private static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int
     if (bytes.length > DECIMAL_BYTE_LENGTH) {
       throw new UnsupportedOperationException("Decimal size greater than 16 bytes");
     }
-    final int padLength = DECIMAL_BYTE_LENGTH - bytes.length;
-    for (int i = 0; i < padLength; i++) {
+
+    // Decimal stored as little endian, need to swap data bytes before writing to ArrowBuf
+    byte[] bytesLE = new byte[bytes.length];
+    int stop = bytes.length / 2;
+    for (int i = 0, j; i < stop; i++) {
+      j = (bytes.length - 1) - i;
+      bytesLE[i] = bytes[j];
+      bytesLE[j] = bytes[i];
+    }
+    if (bytes.length % 2 != 0) {
+      int i = (bytes.length / 2);
+      bytesLE[i] = bytes[i];
+    }
+
+    // Write LE data
+    bytebuf.setBytes(startIndex, bytesLE, 0, bytes.length);
+
+    // Write padding after data
+    for (int i = bytes.length; i < DECIMAL_BYTE_LENGTH; i++) {
       bytebuf.setByte(startIndex + i, padValue);
     }
-    bytebuf.setBytes(startIndex + padLength, bytes, 0, bytes.length);
   }
 }

From bfc0f24fcc58f3885c2175c64864cc12af95f938 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 8 Nov 2017 14:44:26 +0100
Subject: [PATCH 1227/1644] ARROW-1776: [C++] Define
 arrow::gpu::CudaContext::bytes_allocated()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1293 from kou/cpp-gpu-cuda-context-bytes-allocated and squashes the following commits:

5221887 [Kouhei Sutou] [C++] Define arrow::gpu::CudaContext::bytes_allocated()
---
 cpp/src/arrow/gpu/cuda-test.cc    | 1 +
 cpp/src/arrow/gpu/cuda_context.cc | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/cpp/src/arrow/gpu/cuda-test.cc b/cpp/src/arrow/gpu/cuda-test.cc
index afdc3020e8581..7595f8bec7912 100644
--- a/cpp/src/arrow/gpu/cuda-test.cc
+++ b/cpp/src/arrow/gpu/cuda-test.cc
@@ -55,6 +55,7 @@ TEST_F(TestCudaBuffer, Allocate) {
   std::shared_ptr<CudaBuffer> buffer;
   ASSERT_OK(context_->Allocate(kSize, &buffer));
   ASSERT_EQ(kSize, buffer->size());
+  ASSERT_EQ(kSize, context_->bytes_allocated());
 }
 
 void AssertCudaBufferEquals(const CudaBuffer& buffer, const uint8_t* host_data,
diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc
index 42315cd509c3c..fff8ece6c1cff 100644
--- a/cpp/src/arrow/gpu/cuda_context.cc
+++ b/cpp/src/arrow/gpu/cuda_context.cc
@@ -64,6 +64,7 @@ class CudaContext::CudaContextImpl {
 
     CUdeviceptr data;
     CU_RETURN_NOT_OK(cuMemAlloc(&data, static_cast<size_t>(nbytes)));
+    bytes_allocated_ += nbytes;
     *out = reinterpret_cast<uint8_t*>(data);
     return Status::OK();
   }
@@ -85,6 +86,7 @@ class CudaContext::CudaContextImpl {
 
   Status Free(uint8_t* device_ptr, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuMemFree(reinterpret_cast<CUdeviceptr>(device_ptr)));
+    bytes_allocated_ -= nbytes;
     return Status::OK();
   }
 
@@ -273,5 +275,7 @@ Status CudaContext::OpenIpcBuffer(const CudaIpcMemHandle& ipc_handle,
   return Status::OK();
 }
 
+int64_t CudaContext::bytes_allocated() const { return impl_->bytes_allocated(); }
+
 }  // namespace gpu
 }  // namespace arrow

From 252a2a55fc8b80a6412922987b42d64fe41119ef Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 9 Nov 2017 00:24:59 +0900
Subject: [PATCH 1228/1644] [GLib] Fix a typo in document

---
 c_glib/arrow-glib/array-builder.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index 86e7f985be4a4..a5df681421c25 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -316,7 +316,7 @@ garrow_array_builder_new(const std::shared_ptr<arrow::DataType> &type,
  *
  * Release ownership of `arrow::ArrayBuilder` in `builder`.
  *
- * Since: 0.8.8
+ * Since: 0.8.0
  */
 void
 garrow_array_builder_release_ownership(GArrowArrayBuilder *builder)

From 78872a1be263e61d7901eb36663a184c2b04effb Mon Sep 17 00:00:00 2001
From: Stephanie <swang@cs.berkeley.edu>
Date: Wed, 8 Nov 2017 14:13:28 -0800
Subject: [PATCH 1229/1644] ARROW-1775: Ability to abort created but unsealed
 Plasma objects

Author: Stephanie <swang@cs.berkeley.edu>
Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1289 from stephanie-wang/abort-objects and squashes the following commits:

38c42b9 [Stephanie] TODO for PascalCase
08d4040 [Stephanie] Move documentation
dd5b29e [Stephanie] Fix memory error
e6934ac [Philipp Moritz] fix linting
2b8e385 [Stephanie] Return status code when unmapping object
fe20b3b [Stephanie] Add test case for PlasmaClient::Abort
646190c [Stephanie] Abort objects that were not sealed when client disconnects
5fc44c5 [Stephanie] Implement PlasmaClient::Abort
---
 cpp/src/plasma/client.cc            | 97 +++++++++++++++++++++++------
 cpp/src/plasma/client.h             | 19 ++++++
 cpp/src/plasma/format/plasma.fbs    | 12 ++++
 cpp/src/plasma/protocol.cc          | 28 +++++++++
 cpp/src/plasma/protocol.h           |  8 +++
 cpp/src/plasma/store.cc             | 23 ++++++-
 cpp/src/plasma/store.h              |  3 +
 cpp/src/plasma/test/client_tests.cc | 44 +++++++++++++
 8 files changed, 213 insertions(+), 21 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index e57a2a6f3008c..dd32bdc8149a3 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -278,6 +278,39 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
   return Status::OK();
 }
 
+Status PlasmaClient::UnmapObject(const ObjectID& object_id) {
+  auto object_entry = objects_in_use_.find(object_id);
+  ARROW_CHECK(object_entry != objects_in_use_.end());
+  ARROW_CHECK(object_entry->second->count == 0);
+
+  // Decrement the count of the number of objects in this memory-mapped file
+  // that the client is using. The corresponding increment should have
+  // happened in plasma_get.
+  int fd = object_entry->second->object.handle.store_fd;
+  auto entry = mmap_table_.find(fd);
+  ARROW_CHECK(entry != mmap_table_.end());
+  ARROW_CHECK(entry->second.count >= 1);
+  if (entry->second.count == 1) {
+    // If no other objects are being used, then unmap the file.
+    int err = munmap(entry->second.pointer, entry->second.length);
+    if (err == -1) {
+      return Status::IOError("Error during munmap");
+    }
+    // Remove the corresponding entry from the hash table.
+    mmap_table_.erase(fd);
+  } else {
+    // If there are other objects being used, decrement the reference count.
+    entry->second.count -= 1;
+  }
+  // Update the in_use_object_bytes_.
+  in_use_object_bytes_ -= (object_entry->second->object.data_size +
+                           object_entry->second->object.metadata_size);
+  DCHECK_GE(in_use_object_bytes_, 0);
+  // Remove the entry from the hash table of objects currently in use.
+  objects_in_use_.erase(object_id);
+  return Status::OK();
+}
+
 /// This is a helper method for implementing plasma_release. We maintain a
 /// buffer
 /// of release calls and only perform them once the buffer becomes full (as
@@ -297,28 +330,9 @@ Status PlasmaClient::PerformRelease(const ObjectID& object_id) {
   ARROW_CHECK(object_entry->second->count >= 0);
   // Check if the client is no longer using this object.
   if (object_entry->second->count == 0) {
-    // Decrement the count of the number of objects in this memory-mapped file
-    // that the client is using. The corresponding increment should have
-    // happened in plasma_get.
-    int fd = object_entry->second->object.handle.store_fd;
-    auto entry = mmap_table_.find(fd);
-    ARROW_CHECK(entry != mmap_table_.end());
-    entry->second.count -= 1;
-    ARROW_CHECK(entry->second.count >= 0);
-    // If none are being used then unmap the file.
-    if (entry->second.count == 0) {
-      munmap(entry->second.pointer, entry->second.length);
-      // Remove the corresponding entry from the hash table.
-      mmap_table_.erase(fd);
-    }
     // Tell the store that the client no longer needs the object.
+    RETURN_NOT_OK(UnmapObject(object_id));
     RETURN_NOT_OK(SendReleaseRequest(store_conn_, object_id));
-    // Update the in_use_object_bytes_.
-    in_use_object_bytes_ -= (object_entry->second->object.data_size +
-                             object_entry->second->object.metadata_size);
-    DCHECK_GE(in_use_object_bytes_, 0);
-    // Remove the entry from the hash table of objects currently in use.
-    objects_in_use_.erase(object_id);
   }
   return Status::OK();
 }
@@ -344,6 +358,20 @@ Status PlasmaClient::Release(const ObjectID& object_id) {
   return Status::OK();
 }
 
+Status PlasmaClient::FlushReleaseHistory() {
+  // If the client is already disconnected, ignore the flush.
+  if (store_conn_ < 0) {
+    return Status::OK();
+  }
+  while (release_history_.size() > 0) {
+    // Perform a release for the object ID for the first pending release.
+    RETURN_NOT_OK(PerformRelease(release_history_.back()));
+    // Remove the last entry from the release history.
+    release_history_.pop_back();
+  }
+  return Status::OK();
+}
+
 // This method is used to query whether the plasma store contains an object.
 Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) {
   // Check if we already have a reference to the object.
@@ -443,6 +471,35 @@ Status PlasmaClient::Seal(const ObjectID& object_id) {
   return Release(object_id);
 }
 
+Status PlasmaClient::Abort(const ObjectID& object_id) {
+  auto object_entry = objects_in_use_.find(object_id);
+  ARROW_CHECK(object_entry != objects_in_use_.end())
+      << "Plasma client called abort on an object without a reference to it";
+  ARROW_CHECK(!object_entry->second->is_sealed)
+      << "Plasma client called abort on a sealed object";
+
+  // Flush the release history.
+  RETURN_NOT_OK(FlushReleaseHistory());
+  // Make sure that the Plasma client only has one reference to the object. If
+  // it has more, then the client needs to release the buffer before calling
+  // abort.
+  if (object_entry->second->count > 1) {
+    return Status::Invalid("Plasma client cannot have a reference to the buffer.");
+  }
+
+  // Send the abort request.
+  RETURN_NOT_OK(SendAbortRequest(store_conn_, object_id));
+  // Decrease the reference count to zero, then remove the object.
+  object_entry->second->count--;
+  RETURN_NOT_OK(UnmapObject(object_id));
+
+  std::vector<uint8_t> buffer;
+  ObjectID id;
+  int64_t type;
+  RETURN_NOT_OK(ReadMessage(store_conn_, &type, &buffer));
+  return ReadAbortReply(buffer.data(), buffer.size(), &id);
+}
+
 Status PlasmaClient::Delete(const ObjectID& object_id) {
   // TODO(rkn): In the future, we can use this method to give hints to the
   // eviction policy about when an object will no longer be needed.
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index 145942441c9f1..89df2b0b0039f 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -152,6 +152,15 @@ class ARROW_EXPORT PlasmaClient {
   /// \return The return status.
   Status Contains(const ObjectID& object_id, bool* has_object);
 
+  /// Abort an unsealed object in the object store. If the abort succeeds, then
+  /// it will be as if the object was never created at all. The unsealed object
+  /// must have only a single reference (the one that would have been removed by
+  /// calling Seal).
+  ///
+  /// \param object_id The ID of the object to abort.
+  /// \return The return status.
+  Status Abort(const ObjectID& object_id);
+
   /// Seal an object in the object store. The object will be immutable after
   /// this
   /// call.
@@ -307,6 +316,16 @@ class ARROW_EXPORT PlasmaClient {
   int get_manager_fd();
 
  private:
+  /// This is a helper method for unmapping objects for which all references have
+  /// gone out of scope, either by calling Release or Abort.
+  ///
+  /// @param object_id The object ID whose data we should unmap.
+  Status UnmapObject(const ObjectID& object_id);
+
+  /// This is a helper method that flushes all pending release calls to the
+  /// store.
+  Status FlushReleaseHistory();
+
   Status PerformRelease(const ObjectID& object_id);
 
   uint8_t* lookup_or_mmap(int fd, int store_fd_val, int64_t map_size);
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
index 23782ade539d4..b6d03b8a3c10d 100644
--- a/cpp/src/plasma/format/plasma.fbs
+++ b/cpp/src/plasma/format/plasma.fbs
@@ -21,6 +21,8 @@ enum MessageType:int {
   // Create a new object.
   PlasmaCreateRequest = 1,
   PlasmaCreateReply,
+  PlasmaAbortRequest,
+  PlasmaAbortReply,
   // Seal an object.
   PlasmaSealRequest,
   PlasmaSealReply,
@@ -113,6 +115,16 @@ table PlasmaCreateReply {
   error: PlasmaError;
 }
 
+table PlasmaAbortRequest {
+  // ID of the object to be aborted.
+  object_id: string;
+}
+
+table PlasmaAbortReply {
+  // ID of the object that was aborted.
+  object_id: string;
+}
+
 table PlasmaSealRequest {
   // ID of the object to be sealed.
   object_id: string;
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 2261b6a624a8c..c0ebb88fe5019 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -100,6 +100,34 @@ Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id,
   return plasma_error_status(message->error());
 }
 
+Status SendAbortRequest(int sock, ObjectID object_id) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaAbortRequest(fbb, fbb.CreateString(object_id.binary()));
+  return PlasmaSend(sock, MessageType_PlasmaAbortRequest, &fbb, message);
+}
+
+Status ReadAbortRequest(uint8_t* data, size_t size, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaAbortRequest>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
+Status SendAbortReply(int sock, ObjectID object_id) {
+  flatbuffers::FlatBufferBuilder fbb;
+  auto message = CreatePlasmaAbortReply(fbb, fbb.CreateString(object_id.binary()));
+  return PlasmaSend(sock, MessageType_PlasmaAbortReply, &fbb, message);
+}
+
+Status ReadAbortReply(uint8_t* data, size_t size, ObjectID* object_id) {
+  DCHECK(data);
+  auto message = flatbuffers::GetRoot<PlasmaAbortReply>(data);
+  DCHECK(verify_flatbuffer(message, data, size));
+  *object_id = ObjectID::from_binary(message->object_id()->str());
+  return Status::OK();
+}
+
 // Seal messages.
 
 Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest) {
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
index af4b13978c697..e8c334f9181fc 100644
--- a/cpp/src/plasma/protocol.h
+++ b/cpp/src/plasma/protocol.h
@@ -51,6 +51,14 @@ Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int e
 Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id,
                        PlasmaObject* object);
 
+Status SendAbortRequest(int sock, ObjectID object_id);
+
+Status ReadAbortRequest(uint8_t* data, size_t size, ObjectID* object_id);
+
+Status SendAbortReply(int sock, ObjectID object_id);
+
+Status ReadAbortReply(uint8_t* data, size_t size, ObjectID* object_id);
+
 /* Plasma Seal message functions. */
 
 Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest);
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 210cce16238f8..5dbdebc237ce6 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -393,6 +393,18 @@ void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[])
   update_object_get_requests(object_id);
 }
 
+void PlasmaStore::abort_object(const ObjectID& object_id) {
+  auto entry = get_object_table_entry(&store_info_, object_id);
+  ARROW_CHECK(entry != NULL) << "To abort an object it must be in the object table.";
+  ARROW_CHECK(entry->state != PLASMA_SEALED)
+      << "To abort an object it must not have been sealed.";
+  ARROW_CHECK(entry->clients.size() == 1)
+      << "To abort an object, the only client currently using it is the creator.";
+
+  dlfree(entry->pointer);
+  store_info_.objects.erase(object_id);
+}
+
 void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
   for (const auto& object_id : object_ids) {
     ARROW_LOG(DEBUG) << "deleting object " << object_id.hex();
@@ -443,7 +455,11 @@ void PlasmaStore::disconnect_client(int client_fd) {
   // If this client was using any objects, remove it from the appropriate
   // lists.
   for (const auto& entry : store_info_.objects) {
-    remove_client_from_object_clients(entry.second.get(), it->second.get());
+    if (entry.second->state == PLASMA_SEALED) {
+      remove_client_from_object_clients(entry.second.get(), it->second.get());
+    } else {
+      abort_object(entry.first);
+    }
   }
 
   // Note, the store may still attempt to send a message to the disconnected
@@ -582,6 +598,11 @@ Status PlasmaStore::process_message(Client* client) {
         warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
       }
     } break;
+    case MessageType_PlasmaAbortRequest: {
+      RETURN_NOT_OK(ReadAbortRequest(input, input_size, &object_id));
+      abort_object(object_id);
+      HANDLE_SIGPIPE(SendAbortReply(client->fd, object_id), client->fd);
+    } break;
     case MessageType_PlasmaGetRequest: {
       std::vector<ObjectID> object_ids_to_get;
       int64_t timeout_ms;
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index d03d11f4ef0c4..0d08d8a67ffaa 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -48,6 +48,7 @@ struct Client {
 
 class PlasmaStore {
  public:
+  // TODO: PascalCase PlasmaStore methods.
   PlasmaStore(EventLoop* loop, int64_t system_memory, std::string directory,
               bool hugetlbfs_enabled);
 
@@ -73,6 +74,8 @@ class PlasmaStore {
   int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
                     Client* client, PlasmaObject* result);
 
+  void abort_object(const ObjectID& object_id);
+
   /// Delete objects that have been created in the hash table. This should only
   /// be called on objects that are returned by the eviction policy to evict.
   ///
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index 0f19da5f72342..5c0cee4c071ad 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -127,6 +127,50 @@ TEST_F(TestPlasmaStore, MultipleGetTest) {
   ASSERT_EQ(object_buffer[1].data[0], 2);
 }
 
+TEST_F(TestPlasmaStore, AbortTest) {
+  ObjectID object_id = ObjectID::from_random();
+  ObjectBuffer object_buffer;
+
+  // Test for object non-existence.
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer));
+  ASSERT_EQ(object_buffer.data_size, -1);
+
+  // Test object abort.
+  // First create object.
+  int64_t data_size = 4;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  uint8_t* data;
+  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  // Write some data.
+  for (int64_t i = 0; i < data_size / 2; i++) {
+    data[i] = static_cast<uint8_t>(i % 4);
+  }
+  // Attempt to abort. Test that this fails before the first release.
+  Status status = client_.Abort(object_id);
+  ASSERT_TRUE(status.IsInvalid());
+  // Release, then abort.
+  ARROW_CHECK_OK(client_.Release(object_id));
+  ARROW_CHECK_OK(client_.Abort(object_id));
+
+  // Test for object non-existence after the abort.
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer));
+  ASSERT_EQ(object_buffer.data_size, -1);
+
+  // Create the object successfully this time.
+  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  for (int64_t i = 0; i < data_size; i++) {
+    data[i] = static_cast<uint8_t>(i % 4);
+  }
+  ARROW_CHECK_OK(client_.Seal(object_id));
+
+  // Test that we can get the object.
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  for (int64_t i = 0; i < data_size; i++) {
+    ASSERT_EQ(data[i], object_buffer.data[i]);
+  }
+}
+
 }  // namespace plasma
 
 int main(int argc, char** argv) {

From dffa486c86d1d09c16e4c52ad0ff78bbee22c4e1 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Wed, 8 Nov 2017 20:50:51 -0500
Subject: [PATCH 1230/1644] ARROW-1709: [C++] Decimal.ToString is incorrect for
 negative scale

This is on top of ARROW-1716. Will rebase when that's merged.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1292 from cpcloud/ARROW-1709 and squashes the following commits:

13ef9a32 [Phillip Cloud] Fix formatting
e87e88f0 [Phillip Cloud] Cleanup
3431b667 [Phillip Cloud] Skip leading negative sign in dcheck
9874d84d [Phillip Cloud] Use lambda
3a5e3f6f [Phillip Cloud] DCHECK and format
2dde6d2d [Phillip Cloud] ARROW-1709: [C++] Decimal.ToString is incorrect for negative scale
---
 cpp/src/arrow/util/decimal-test.cc |  75 +++++++++++++++++++++
 cpp/src/arrow/util/decimal.cc      | 104 +++++++++++++++++++++++------
 2 files changed, 159 insertions(+), 20 deletions(-)

diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index b0271fff15ccf..0d0c08cc4551f 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -18,6 +18,7 @@
 
 #include <cstdint>
 #include <string>
+#include <tuple>
 
 #include <gtest/gtest.h>
 
@@ -291,4 +292,78 @@ TEST(Decimal128Test, PrintMinValue) {
   ASSERT_EQ(string_value, printed_value);
 }
 
+class Decimal128PrintingTest
+    : public ::testing::TestWithParam<std::tuple<int32_t, int32_t, std::string>> {};
+
+TEST_P(Decimal128PrintingTest, Print) {
+  int32_t test_value;
+  int32_t scale;
+  std::string expected_string;
+  std::tie(test_value, scale, expected_string) = GetParam();
+  const Decimal128 value(test_value);
+  const std::string printed_value = value.ToString(scale);
+  ASSERT_EQ(expected_string, printed_value);
+}
+
+INSTANTIATE_TEST_CASE_P(Decimal128PrintingTest, Decimal128PrintingTest,
+                        ::testing::Values(std::make_tuple(123, 1, "12.3"),
+                                          std::make_tuple(123, 5, "0.00123"),
+                                          std::make_tuple(123, 10, "1.23E-8"),
+                                          std::make_tuple(123, -1, "1.23E+3"),
+                                          std::make_tuple(-123, -1, "-1.23E+3"),
+                                          std::make_tuple(123, -3, "1.23E+5"),
+                                          std::make_tuple(-123, -3, "-1.23E+5"),
+                                          std::make_tuple(12345, -3, "1.2345E+7")));
+
+class Decimal128ParsingTest
+    : public ::testing::TestWithParam<std::tuple<std::string, uint64_t, int32_t>> {};
+
+TEST_P(Decimal128ParsingTest, Parse) {
+  std::string test_string;
+  uint64_t expected_low_bits;
+  int32_t expected_scale;
+  std::tie(test_string, expected_low_bits, expected_scale) = GetParam();
+  Decimal128 value;
+  int32_t scale;
+  ASSERT_OK(Decimal128::FromString(test_string, &value, NULLPTR, &scale));
+  ASSERT_EQ(value.low_bits(), expected_low_bits);
+  ASSERT_EQ(expected_scale, scale);
+}
+
+INSTANTIATE_TEST_CASE_P(Decimal128ParsingTest, Decimal128ParsingTest,
+                        ::testing::Values(std::make_tuple("12.3", 123ULL, 1),
+                                          std::make_tuple("0.00123", 123ULL, 5),
+                                          std::make_tuple("1.23E-8", 123ULL, 10),
+                                          std::make_tuple("-1.23E-8", -123LL, 10),
+                                          std::make_tuple("1.23E+3", 123ULL, -1),
+                                          std::make_tuple("-1.23E+3", -123LL, -1),
+                                          std::make_tuple("1.23E+5", 123ULL, -3),
+                                          std::make_tuple("1.2345E+7", 12345ULL, -3),
+                                          std::make_tuple("1.23e-8", 123ULL, 10),
+                                          std::make_tuple("-1.23e-8", -123LL, 10),
+                                          std::make_tuple("1.23e+3", 123ULL, -1),
+                                          std::make_tuple("-1.23e+3", -123LL, -1),
+                                          std::make_tuple("1.23e+5", 123ULL, -3),
+                                          std::make_tuple("1.2345e+7", 12345ULL, -3)));
+
+class Decimal128ParsingTestInvalid : public ::testing::TestWithParam<std::string> {};
+
+TEST_P(Decimal128ParsingTestInvalid, Parse) {
+  std::string test_string = GetParam();
+  Decimal128 value;
+  ASSERT_RAISES(Invalid, Decimal128::FromString(test_string, &value));
+}
+
+INSTANTIATE_TEST_CASE_P(Decimal128ParsingTestInvalid, Decimal128ParsingTestInvalid,
+                        ::testing::Values("0.00123D/3", "1.23eA8", "1.23E+3A",
+                                          "-1.23E--5", "1.2345E+++07"));
+
+TEST(Decimal128ParseTest, WithExponentAndNullptrScale) {
+  Decimal128 value;
+  ASSERT_OK(Decimal128::FromString("1.23E-8", &value));
+
+  const Decimal128 expected_value(123);
+  ASSERT_EQ(expected_value, value);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index cc180258aa4df..447cae5c54654 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -105,6 +105,22 @@ Decimal128::operator int64_t() const {
   return static_cast<int64_t>(low_bits_);
 }
 
+static std::string ToStringNegativeScale(const std::string& str,
+                                         int32_t adjusted_exponent, bool is_negative) {
+  std::stringstream buf;
+
+  size_t offset = 0;
+  buf << str[offset++];
+
+  if (is_negative) {
+    buf << str[offset++];
+  }
+
+  buf << '.' << str.substr(offset, std::string::npos) << 'E' << std::showpos
+      << adjusted_exponent;
+  return buf.str();
+}
+
 std::string Decimal128::ToString(int32_t scale) const {
   const std::string str(ToIntegerString());
 
@@ -112,9 +128,18 @@ std::string Decimal128::ToString(int32_t scale) const {
     return str;
   }
 
-  if (*this < 0) {
-    const auto len = static_cast<int32_t>(str.size());
+  const bool is_negative = *this < 0;
 
+  const auto len = static_cast<int32_t>(str.size());
+  const auto is_negative_offset = static_cast<int32_t>(is_negative);
+  const int32_t adjusted_exponent = -scale + (len - 1 - is_negative_offset);
+
+  /// Note that the -6 is taken from the Java BigDecimal documentation.
+  if (scale < 0 || adjusted_exponent < -6) {
+    return ToStringNegativeScale(str, adjusted_exponent, is_negative);
+  }
+
+  if (is_negative) {
     if (len - 1 > scale) {
       const auto n = static_cast<size_t>(len - scale);
       return str.substr(0, n) + "." + str.substr(n, static_cast<size_t>(scale));
@@ -128,8 +153,6 @@ std::string Decimal128::ToString(int32_t scale) const {
     return result + str.substr(1, std::string::npos);
   }
 
-  const auto len = static_cast<int32_t>(str.size());
-
   if (len > scale) {
     const auto n = static_cast<size_t>(len - scale);
     return str.substr(0, n) + "." + str.substr(n, static_cast<size_t>(scale));
@@ -164,10 +187,12 @@ static constexpr int64_t kPowersOfTen[kInt64DecimalDigits + 1] = {1LL,
                                                                   100000000000000000LL,
                                                                   1000000000000000000LL};
 
+static inline bool isdigit(char value) { return std::isdigit(value) != 0; }
+
 static void StringToInteger(const std::string& str, Decimal128* out) {
   using std::size_t;
 
-  DCHECK_NE(out, nullptr) << "Decimal128 output variable cannot be nullptr";
+  DCHECK_NE(out, NULLPTR) << "Decimal128 output variable cannot be NULLPTR";
   DCHECK_EQ(*out, 0)
       << "When converting a string to Decimal128 the initial output must be 0";
 
@@ -189,7 +214,7 @@ static void StringToInteger(const std::string& str, Decimal128* out) {
 
 Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precision,
                               int* scale) {
-  // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?";
+  // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?((E|e)(\\+|-)?\\d+)?";
   if (s.empty()) {
     return Status::Invalid("Empty string cannot be converted to decimal");
   }
@@ -215,21 +240,21 @@ Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precis
   DCHECK_LT(charp, end);
 
   // skip leading zeros
-  charp = std::find_if_not(charp, end, [](char c) { return c == '0'; });
+  charp = std::find_if_not(charp, end, [](char value) { return value == '0'; });
 
   // all zeros and no decimal point
   if (charp == end) {
-    if (out != nullptr) {
+    if (out != NULLPTR) {
       *out = 0;
     }
 
     // Not sure what other libraries assign precision to for this case (this case of
     // a string consisting only of one or more zeros)
-    if (precision != nullptr) {
+    if (precision != NULLPTR) {
       *precision = static_cast<int>(charp - numeric_string_start);
     }
 
-    if (scale != nullptr) {
+    if (scale != NULLPTR) {
       *scale = 0;
     }
 
@@ -238,7 +263,7 @@ Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precis
 
   std::string::const_iterator whole_part_start = charp;
 
-  charp = std::find_if_not(charp, end, [](char c) { return std::isdigit(c) != 0; });
+  charp = std::find_if_not(charp, end, isdigit);
 
   std::string::const_iterator whole_part_end = charp;
   std::string whole_part(whole_part_start, whole_part_end);
@@ -269,14 +294,13 @@ Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precis
 
   std::string::const_iterator fractional_part_start = charp;
 
-  // The rest must be digits, because if we have a decimal point it must be followed by
-  // digits
+  // The rest must be digits or an exponent
   if (charp != end) {
-    charp = std::find_if_not(charp, end, [](char c) { return std::isdigit(c) != 0; });
+    charp = std::find_if_not(charp, end, isdigit);
 
     // The while loop has ended before the end of the string which means we've hit a
-    // character that isn't a base ten digit
-    if (charp != end) {
+    // character that isn't a base ten digit or "E" for exponent
+    if (charp != end && *charp != 'E' && *charp != 'e') {
       std::stringstream ss;
       ss << "Found non base ten digit character '" << *charp
          << "' before the end of the string";
@@ -287,15 +311,55 @@ Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precis
   std::string::const_iterator fractional_part_end = charp;
   std::string fractional_part(fractional_part_start, fractional_part_end);
 
-  if (precision != nullptr) {
+  if (precision != NULLPTR) {
     *precision = static_cast<int>(whole_part.size() + fractional_part.size());
   }
 
-  if (scale != nullptr) {
-    *scale = static_cast<int>(fractional_part.size());
+  if (charp != end) {
+    // we must have an exponent, if this aborts then we have somehow not caught this and
+    // raised a proper error
+    DCHECK(*charp == 'E' || *charp == 'e');
+
+    ++charp;
+
+    const char value = *charp;
+    const bool starts_with_plus_or_minus = value == '+' || value == '-';
+
+    // we use this to construct the adjusted exponent integer later
+    std::string::const_iterator digit_start = charp;
+
+    // skip plus or minus
+    charp += starts_with_plus_or_minus;
+
+    // confirm that the rest of the characters are digits
+    charp = std::find_if_not(charp, end, isdigit);
+
+    if (charp != end) {
+      // we have something other than digits here
+      std::stringstream ss;
+      ss << "Found non decimal digit exponent value '" << *charp << "'";
+      return Status::Invalid(ss.str());
+    }
+
+    if (scale != NULLPTR) {
+      // compute the scale from the adjusted exponent
+      std::string adjusted_exponent_string(digit_start, end);
+      DCHECK(std::all_of(adjusted_exponent_string.cbegin() + starts_with_plus_or_minus,
+                         adjusted_exponent_string.cend(), isdigit))
+          << "Non decimal digit character found in " << adjusted_exponent_string;
+      const auto adjusted_exponent =
+          static_cast<int32_t>(std::stol(adjusted_exponent_string));
+      const auto len = static_cast<int32_t>(whole_part.size() + fractional_part.size());
+
+      *scale = -adjusted_exponent + len - 1;
+    }
+  } else {
+    if (scale != NULLPTR) {
+      *scale = static_cast<int>(fractional_part.size());
+    }
   }
 
-  if (out != nullptr) {
+  if (out != NULLPTR) {
     // zero out in case we've passed in a previously used value
     *out = 0;
     StringToInteger(whole_part + fractional_part, out);

From 65a9055c705e5f09c949c12365d12839ace063f5 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Thu, 9 Nov 2017 11:31:36 -0500
Subject: [PATCH 1231/1644] ARROW-972: UnionArray in pyarrow

This is taking a stab at exposing UnionArray to pyarrow. Tasks to be done:

- [x] Support UnionType::SPARSE
- [x] Add doc strings

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1216 from pcmoritz/pyarrow-union-array and squashes the following commits:

7f3ca313 [Wes McKinney] Fix flakes
9f33076b [Wes McKinney] Change UnionMode to scoped enumeration
9e602a8d [Philipp Moritz] wrap UnionType in pyarrow
eeef7226 [Philipp Moritz] linting
502c335a [Philipp Moritz] fixes
c6c85491 [Philipp Moritz] add doc strings
9068bbb5 [Philipp Moritz] linting
d8da0170 [Philipp Moritz] implement dense and sparse UnionArrays
cbdedc7a [Philipp Moritz] make fields in UnionArray unique to be compatiable with Java
b796ce64 [Philipp Moritz] Implement UnionArray in pyarrow
---
 cpp/src/arrow/array.cc                 | 56 ++++++++++++++++++++++++++
 cpp/src/arrow/array.h                  | 33 ++++++++++++++-
 cpp/src/arrow/compare.cc               |  2 +-
 cpp/src/arrow/ipc/json-internal.cc     |  2 +-
 cpp/src/arrow/ipc/metadata-internal.cc |  5 ++-
 cpp/src/arrow/type.cc                  | 18 ++++++++-
 cpp/src/arrow/type.h                   | 18 ++++++---
 python/pyarrow/__init__.py             |  4 +-
 python/pyarrow/_parquet.pxd            |  2 +-
 python/pyarrow/array.pxi               | 53 ++++++++++++++++++++++++
 python/pyarrow/includes/libarrow.pxd   | 25 ++++++++++++
 python/pyarrow/lib.pxd                 | 16 ++++++++
 python/pyarrow/lib.pyx                 |  2 +
 python/pyarrow/public-api.pxi          |  2 +-
 python/pyarrow/scalar.pxi              | 19 +++++++++
 python/pyarrow/tests/test_array.py     | 22 ++++++++++
 python/pyarrow/tests/test_schema.py    |  8 ++++
 python/pyarrow/tests/test_types.py     | 13 +++---
 python/pyarrow/types.pxi               | 49 ++++++++++++++++++++++
 19 files changed, 327 insertions(+), 22 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index b523876bf0e4e..9c91d619cc7e8 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -393,6 +393,62 @@ UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
   SetData(internal_data);
 }
 
+Status UnionArray::MakeDense(const Array& type_ids, const Array& value_offsets,
+                             const std::vector<std::shared_ptr<Array>>& children,
+                             std::shared_ptr<Array>* out) {
+  if (value_offsets.length() == 0) {
+    return Status::Invalid("UnionArray offsets must have non-zero length");
+  }
+
+  if (value_offsets.type_id() != Type::INT32) {
+    return Status::Invalid("UnionArray offsets must be signed int32");
+  }
+
+  if (type_ids.type_id() != Type::INT8) {
+    return Status::Invalid("UnionArray type_ids must be signed int8");
+  }
+
+  if (value_offsets.null_count() != 0) {
+    return Status::Invalid("MakeDense does not allow NAs in value_offsets");
+  }
+
+  BufferVector buffers = {type_ids.null_bitmap(),
+                          static_cast<const UInt8Array&>(type_ids).values(),
+                          static_cast<const Int32Array&>(value_offsets).values()};
+  auto union_type = union_(children, UnionMode::DENSE);
+  auto internal_data =
+      std::make_shared<ArrayData>(union_type, type_ids.length(), std::move(buffers),
+                                  type_ids.null_count(), type_ids.offset());
+  for (const auto& child : children) {
+    internal_data->child_data.push_back(child->data());
+  }
+  *out = std::make_shared<UnionArray>(internal_data);
+  return Status::OK();
+}
+
+Status UnionArray::MakeSparse(const Array& type_ids,
+                              const std::vector<std::shared_ptr<Array>>& children,
+                              std::shared_ptr<Array>* out) {
+  if (type_ids.type_id() != Type::INT8) {
+    return Status::Invalid("UnionArray type_ids must be signed int8");
+  }
+  BufferVector buffers = {type_ids.null_bitmap(),
+                          static_cast<const UInt8Array&>(type_ids).values(), nullptr};
+  auto union_type = union_(children, UnionMode::SPARSE);
+  auto internal_data =
+      std::make_shared<ArrayData>(union_type, type_ids.length(), std::move(buffers),
+                                  type_ids.null_count(), type_ids.offset());
+  for (const auto& child : children) {
+    internal_data->child_data.push_back(child->data());
+    if (child->length() != type_ids.length()) {
+      return Status::Invalid(
+          "Sparse UnionArray must have len(child) == len(type_ids) for all children");
+    }
+  }
+  *out = std::make_shared<UnionArray>(internal_data);
+  return Status::OK();
+}
+
 std::shared_ptr<Array> UnionArray::child(int i) const {
   if (!boxed_fields_[i]) {
     boxed_fields_[i] = MakeArray(data_->child_data[i]);
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index afbd780dd3ad5..f7762ce104398 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -612,16 +612,47 @@ class ARROW_EXPORT UnionArray : public Array {
              const std::shared_ptr<Buffer>& null_bitmap = NULLPTR, int64_t null_count = 0,
              int64_t offset = 0);
 
+  /// \brief Construct Dense UnionArray from types_ids, value_offsets and children
+  ///
+  /// This function does the bare minimum of validation of the offsets and
+  /// input types. The value_offsets are assumed to be well-formed.
+  ///
+  /// \param[in] type_ids An array of 8-bit signed integers, enumerated from
+  /// 0 corresponding to each type.
+  /// \param[in] value_offsets An array of signed int32 values indicating the
+  /// relative offset into the respective child array for the type in a given slot.
+  /// The respective offsets for each child value array must be in order / increasing.
+  /// \param[in] children Vector of children Arrays containing the data for each type.
+  /// \param[out] out Will have length equal to value_offsets.length()
+  static Status MakeDense(const Array& type_ids, const Array& value_offsets,
+                          const std::vector<std::shared_ptr<Array>>& children,
+                          std::shared_ptr<Array>* out);
+
+  /// \brief Construct Sparse UnionArray from type_ids and children
+  ///
+  /// This function does the bare minimum of validation of the offsets and
+  /// input types.
+  ///
+  /// \param[in] type_ids An array of 8-bit signed integers, enumerated from
+  /// 0 corresponding to each type.
+  /// \param[in] children Vector of children Arrays containing the data for each type.
+  /// \param[out] out Will have length equal to type_ids.length()
+  static Status MakeSparse(const Array& type_ids,
+                           const std::vector<std::shared_ptr<Array>>& children,
+                           std::shared_ptr<Array>* out);
+
   /// Note that this buffer does not account for any slice offset
   std::shared_ptr<Buffer> type_ids() const { return data_->buffers[1]; }
 
   /// Note that this buffer does not account for any slice offset
   std::shared_ptr<Buffer> value_offsets() const { return data_->buffers[2]; }
 
+  int32_t value_offset(int64_t i) const { return raw_value_offsets_[i + data_->offset]; }
+
   const type_id_t* raw_type_ids() const { return raw_type_ids_ + data_->offset; }
   const int32_t* raw_value_offsets() const { return raw_value_offsets_ + data_->offset; }
 
-  UnionMode mode() const { return static_cast<const UnionType&>(*type()).mode(); }
+  UnionMode::type mode() const { return static_cast<const UnionType&>(*type()).mode(); }
 
   std::shared_ptr<Array> child(int pos) const;
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 2ec86c3695aa5..a2d4de7b73afb 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -152,7 +152,7 @@ class RangeEqualsVisitor {
   bool CompareUnions(const UnionArray& left) const {
     const auto& right = static_cast<const UnionArray&>(right_);
 
-    const UnionMode union_mode = left.mode();
+    const UnionMode::type union_mode = left.mode();
     if (union_mode != right.mode()) {
       return false;
     }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index c1c0661d6ad35..1b9baee7dafef 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -774,7 +774,7 @@ static Status GetUnion(const RjObject& json_type,
   RETURN_NOT_STRING("mode", it_mode, json_type);
 
   std::string mode_str = it_mode->value.GetString();
-  UnionMode mode;
+  UnionMode::type mode;
 
   if (mode_str == "SPARSE") {
     mode = UnionMode::SPARSE;
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index f0f0f675853b1..63ef8a549f236 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -163,8 +163,9 @@ static Status StructToFlatbuffer(FBB& fbb, const DataType& type,
 static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
                                   const std::vector<std::shared_ptr<Field>>& children,
                                   std::shared_ptr<DataType>* out) {
-  UnionMode mode = union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE
-                                                                   : UnionMode::DENSE;
+  UnionMode::type mode =
+      (union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE
+                                                       : UnionMode::DENSE);
 
   std::vector<uint8_t> type_codes;
 
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index a9bf591918558..0d1985fb2d914 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -190,7 +190,7 @@ std::string TimestampType::ToString() const {
 // Union type
 
 UnionType::UnionType(const std::vector<std::shared_ptr<Field>>& fields,
-                     const std::vector<uint8_t>& type_codes, UnionMode mode)
+                     const std::vector<uint8_t>& type_codes, UnionMode::type mode)
     : NestedType(Type::UNION), mode_(mode), type_codes_(type_codes) {
   children_ = fields;
 }
@@ -440,10 +440,24 @@ std::shared_ptr<DataType> struct_(const std::vector<std::shared_ptr<Field>>& fie
 }
 
 std::shared_ptr<DataType> union_(const std::vector<std::shared_ptr<Field>>& child_fields,
-                                 const std::vector<uint8_t>& type_codes, UnionMode mode) {
+                                 const std::vector<uint8_t>& type_codes,
+                                 UnionMode::type mode) {
   return std::make_shared<UnionType>(child_fields, type_codes, mode);
 }
 
+std::shared_ptr<DataType> union_(const std::vector<std::shared_ptr<Array>>& children,
+                                 UnionMode::type mode) {
+  std::vector<std::shared_ptr<Field>> types;
+  std::vector<uint8_t> type_codes;
+  uint8_t counter = 0;
+  for (const auto& child : children) {
+    types.push_back(field(std::to_string(counter), child->type()));
+    type_codes.push_back(counter);
+    counter++;
+  }
+  return union_(types, type_codes, mode);
+}
+
 std::shared_ptr<DataType> dictionary(const std::shared_ptr<DataType>& index_type,
                                      const std::shared_ptr<Array>& dict_values,
                                      bool ordered) {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 446f4d3a0b33f..9e11a034420e5 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -517,14 +517,17 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
   int32_t scale_;
 };
 
-enum class UnionMode : char { SPARSE, DENSE };
+struct UnionMode {
+  enum type { SPARSE, DENSE };
+};
 
 class ARROW_EXPORT UnionType : public NestedType {
  public:
   static constexpr Type::type type_id = Type::UNION;
 
   UnionType(const std::vector<std::shared_ptr<Field>>& fields,
-            const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
+            const std::vector<uint8_t>& type_codes,
+            UnionMode::type mode = UnionMode::SPARSE);
 
   std::string ToString() const override;
   std::string name() const override { return "union"; }
@@ -534,10 +537,10 @@ class ARROW_EXPORT UnionType : public NestedType {
 
   const std::vector<uint8_t>& type_codes() const { return type_codes_; }
 
-  UnionMode mode() const { return mode_; }
+  UnionMode::type mode() const { return mode_; }
 
  private:
-  UnionMode mode_;
+  UnionMode::type mode_;
 
   // The type id used in the data to indicate each data type in the union. For
   // example, the first type in the union might be denoted by the id 5 (instead
@@ -842,7 +845,12 @@ struct_(const std::vector<std::shared_ptr<Field>>& fields);
 /// \brief Create an instance of Union type
 std::shared_ptr<DataType> ARROW_EXPORT
 union_(const std::vector<std::shared_ptr<Field>>& child_fields,
-       const std::vector<uint8_t>& type_codes, UnionMode mode = UnionMode::SPARSE);
+       const std::vector<uint8_t>& type_codes, UnionMode::type mode = UnionMode::SPARSE);
+
+/// \brief Create and instance of Union type
+std::shared_ptr<DataType> ARROW_EXPORT
+union_(const std::vector<std::shared_ptr<Array>>& children,
+       UnionMode::type mode = UnionMode::SPARSE);
 
 /// \brief Create an instance of Dictionary type
 std::shared_ptr<DataType> ARROW_EXPORT
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 1215c822d2e47..2d7d7288b3835 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -36,7 +36,7 @@
                          time32, time64, timestamp, date32, date64,
                          float16, float32, float64,
                          binary, string, decimal,
-                         list_, struct, dictionary, field,
+                         list_, struct, union, dictionary, field,
                          type_for_alias,
                          DataType, NAType,
                          Field,
@@ -52,7 +52,7 @@
                          Int16Array, UInt16Array,
                          Int32Array, UInt32Array,
                          Int64Array, UInt64Array,
-                         ListArray,
+                         ListArray, UnionArray,
                          BinaryArray, StringArray,
                          FixedSizeBinaryArray,
                          DictionaryArray,
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 04a5b1368ce45..7e5e575096ddd 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -192,7 +192,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
         int64_t num_values() const
         shared_ptr[ColumnPath] path_in_schema() const
         bint is_stats_set() const
-        shared_ptr[CRowGroupStatistics] statistics() const;
+        shared_ptr[CRowGroupStatistics] statistics() const
         ParquetCompression compression() const
         const vector[ParquetEncoding]& encodings() const
 
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 7752d062a774c..9991411e55dfe 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -631,6 +631,58 @@ cdef class ListArray(Array):
         return pyarrow_wrap_array(out)
 
 
+cdef class UnionArray(Array):
+
+    @staticmethod
+    def from_dense(Array types, Array value_offsets, list children):
+        """
+        Construct dense UnionArray from arrays of int8 types, int32 offsets and
+        children arrays
+
+        Parameters
+        ----------
+        types : Array (int8 type)
+        value_offsets : Array (int32 type)
+        children : list
+
+        Returns
+        -------
+        union_array : UnionArray
+        """
+        cdef shared_ptr[CArray] out
+        cdef vector[shared_ptr[CArray]] c
+        cdef Array child
+        for child in children:
+            c.push_back(child.sp_array)
+        with nogil:
+            check_status(CUnionArray.MakeDense(
+                deref(types.ap), deref(value_offsets.ap), c, &out))
+        return pyarrow_wrap_array(out)
+
+    @staticmethod
+    def from_sparse(Array types, list children):
+        """
+        Construct sparse UnionArray from arrays of int8 types and children
+        arrays
+
+        Parameters
+        ----------
+        types : Array (int8 type)
+        children : list
+
+        Returns
+        -------
+        union_array : UnionArray
+        """
+        cdef shared_ptr[CArray] out
+        cdef vector[shared_ptr[CArray]] c
+        cdef Array child
+        for child in children:
+            c.push_back(child.sp_array)
+        with nogil:
+            check_status(CUnionArray.MakeSparse(deref(types.ap), c, &out))
+        return pyarrow_wrap_array(out)
+
 cdef class StringArray(Array):
     pass
 
@@ -789,6 +841,7 @@ cdef dict _array_classes = {
     _Type_FLOAT: FloatArray,
     _Type_DOUBLE: DoubleArray,
     _Type_LIST: ListArray,
+    _Type_UNION: UnionArray,
     _Type_BINARY: BinaryArray,
     _Type_STRING: StringArray,
     _Type_DICTIONARY: DictionaryArray,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 731ef94971da0..dfafd371b2857 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -67,6 +67,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         _Type_DICTIONARY" arrow::Type::DICTIONARY"
         _Type_MAP" arrow::Type::MAP"
 
+    enum UnionMode" arrow::UnionMode::type":
+        _UnionMode_SPARSE" arrow::UnionMode::SPARSE"
+        _UnionMode_DENSE" arrow::UnionMode::DENSE"
+
     enum TimeUnit" arrow::TimeUnit::type":
         TimeUnit_SECOND" arrow::TimeUnit::SECOND"
         TimeUnit_MILLI" arrow::TimeUnit::MILLI"
@@ -222,6 +226,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CStructType" arrow::StructType"(CDataType):
         CStructType(const vector[shared_ptr[CField]]& fields)
 
+    cdef cppclass CUnionType" arrow::UnionType"(CDataType):
+        CUnionType(const vector[shared_ptr[CField]]& fields,
+                   const vector[uint8_t]& type_codes, UnionMode mode)
+        UnionMode mode()
+
     cdef cppclass CSchema" arrow::Schema":
         CSchema(const vector[shared_ptr[CField]]& fields)
         CSchema(const vector[shared_ptr[CField]]& fields,
@@ -317,6 +326,22 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CArray] values()
         shared_ptr[CDataType] value_type()
 
+    cdef cppclass CUnionArray" arrow::UnionArray"(CArray):
+        @staticmethod
+        CStatus MakeSparse(const CArray& type_ids,
+                           const vector[shared_ptr[CArray]]& children,
+                           shared_ptr[CArray]* out)
+
+        @staticmethod
+        CStatus MakeDense(const CArray& type_ids, const CArray& value_offsets,
+                          const vector[shared_ptr[CArray]]& children,
+                          shared_ptr[CArray]* out)
+        uint8_t* raw_type_ids()
+        int32_t value_offset(int i)
+        shared_ptr[CArray] child(int pos)
+        const CArray* UnsafeChild(int pos)
+        UnionMode mode()
+
     cdef cppclass CBinaryArray" arrow::BinaryArray"(CListArray):
         const uint8_t* GetValue(int i, int32_t* length)
 
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 8fdcf553c13fc..531489490754e 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -56,6 +56,11 @@ cdef class DictionaryType(DataType):
         const CDictionaryType* dict_type
 
 
+cdef class UnionType(DataType):
+    cdef:
+        list child_types
+
+
 cdef class TimestampType(DataType):
     cdef:
         const CTimestampType* ts_type
@@ -139,6 +144,13 @@ cdef class ListValue(ArrayValue):
     cdef getitem(self, int64_t i)
 
 
+cdef class UnionValue(ArrayValue):
+    cdef:
+        CUnionArray* ap
+        list value_types
+
+    cdef getitem(self, int64_t i)
+
 cdef class StringValue(ArrayValue):
     pass
 
@@ -242,6 +254,10 @@ cdef class ListArray(Array):
     pass
 
 
+cdef class UnionArray(Array):
+    pass
+
+
 cdef class StringArray(Array):
     pass
 
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 6f4451e3f5a41..b4ca49cafe160 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -92,6 +92,8 @@ Type_UNION = _Type_UNION
 Type_DICTIONARY = _Type_DICTIONARY
 Type_MAP = _Type_MAP
 
+UnionMode_SPARSE = _UnionMode_SPARSE
+UnionMode_DENSE = _UnionMode_DENSE
 
 # Exception types
 include "error.pxi"
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 9f1051228047a..90aff9e936d95 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -72,7 +72,7 @@ cdef public api object pyarrow_wrap_data_type(
     elif type.get().id() == _Type_STRUCT:
         out = StructType()
     elif type.get().id() == _Type_UNION:
-        out = StructType()
+        out = UnionType()
     elif type.get().id() == _Type_TIMESTAMP:
         out = TimestampType()
     elif type.get().id() == _Type_FIXED_SIZE_BINARY:
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index c37ed3b200ea3..a396fa763c8c8 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -315,6 +315,24 @@ cdef class ListValue(ArrayValue):
         return result
 
 
+cdef class UnionValue(ArrayValue):
+
+    cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
+        self.sp_array = sp_array
+        self.ap = <CUnionArray*> sp_array.get()
+
+    cdef getitem(self, int64_t i):
+        cdef int8_t type_id = self.ap.raw_type_ids()[i]
+        cdef shared_ptr[CArray] child = self.ap.child(type_id)
+        if self.ap.mode() == _UnionMode_SPARSE:
+            return box_scalar(self.type[type_id], child, i)
+        else:
+            return box_scalar(self.type[type_id], child,
+                              self.ap.value_offset(i))
+
+    def as_py(self):
+        return self.getitem(self.index).as_py()
+
 cdef class FixedSizeBinaryValue(ArrayValue):
 
     def as_py(self):
@@ -364,6 +382,7 @@ cdef dict _scalar_classes = {
     _Type_FLOAT: FloatValue,
     _Type_DOUBLE: DoubleValue,
     _Type_LIST: ListValue,
+    _Type_UNION: UnionValue,
     _Type_BINARY: BinaryValue,
     _Type_STRING: StringValue,
     _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index e3a4c97567ee6..7dc93c28ea7a4 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -235,6 +235,28 @@ def test_list_from_arrays():
     assert result.equals(expected)
 
 
+def test_union_from_dense():
+    binary = pa.array([b'a', b'b', b'c', b'd'], type='binary')
+    int64 = pa.array([1, 2, 3], type='int64')
+    types = pa.array([0, 1, 0, 0, 1, 1, 0], type='int8')
+    value_offsets = pa.array([0, 0, 2, 1, 1, 2, 3], type='int32')
+
+    result = pa.UnionArray.from_dense(types, value_offsets, [binary, int64])
+
+    assert result.to_pylist() == [b'a', 1, b'c', b'b', 2, 3, b'd']
+
+
+def test_union_from_sparse():
+    binary = pa.array([b'a', b' ', b'b', b'c', b' ', b' ', b'd'],
+                      type='binary')
+    int64 = pa.array([0, 1, 0, 0, 2, 3, 0], type='int64')
+    types = pa.array([0, 1, 0, 0, 1, 1, 0], type='int8')
+
+    result = pa.UnionArray.from_sparse(types, [binary, int64])
+
+    assert result.to_pylist() == [b'a', 1, b'b', b'c', 2, 3, b'd']
+
+
 def _check_cast_case(case, safe=True):
     in_data, in_type, out_data, out_type = case
 
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index d6b2655b7c6a0..116f3978333a8 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -319,6 +319,14 @@ def test_type_schema_pickling():
             pa.field('a', 'int8'),
             pa.field('b', 'string')
         ]),
+        pa.union([
+            pa.field('a', pa.int8()),
+            pa.field('b', pa.int16())
+        ], pa.lib.UnionMode_SPARSE),
+        pa.union([
+            pa.field('a', pa.int8()),
+            pa.field('b', pa.int16())
+        ], pa.lib.UnionMode_DENSE),
         pa.time32('s'),
         pa.time64('us'),
         pa.date32(),
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index e6ff5b1560c1d..0e3ea1fd40bf5 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -85,16 +85,17 @@ def test_is_nested_or_struct():
     assert not types.is_nested(pa.int32())
 
 
-# TODO(wesm): Union types not yet implemented in pyarrow
+def test_is_union():
+    assert types.is_union(pa.union([pa.field('a', pa.int32()),
+                                    pa.field('b', pa.int8()),
+                                    pa.field('c', pa.string())],
+                                   pa.lib.UnionMode_SPARSE))
+    assert not types.is_union(pa.list_(pa.int32()))
 
-# def test_is_union():
-#     assert types.is_union(pa.union([pa.field('a', pa.int32()),
-#                                     pa.field('b', pa.int8()),
-#                                     pa.field('c', pa.string())]))
-#     assert not types.is_union(pa.list_(pa.int32()))
 
 # TODO(wesm): is_map, once implemented
 
+
 def test_is_binary_string():
     assert types.is_binary(pa.binary())
     assert not types.is_binary(pa.string())
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index c9a490960ec38..d2e68ff79a524 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -186,7 +186,32 @@ cdef class UnionType(DataType):
 
     cdef void init(self, const shared_ptr[CDataType]& type):
         DataType.init(self, type)
+        self.child_types = [
+            pyarrow_wrap_data_type(type.get().child(i).get().type())
+            for i in range(self.num_children)]
 
+    property num_children:
+
+        def __get__(self):
+            return self.type.num_children()
+
+    property mode:
+
+        def __get__(self):
+            cdef CUnionType* type = <CUnionType*> self.sp_type.get()
+            return type.mode()
+
+    def __getitem__(self, i):
+        return self.child_types[i]
+
+    def __getstate__(self):
+        children = [pyarrow_wrap_field(self.type.child(i))
+                    for i in range(self.num_children)]
+        return children, self.mode
+
+    def __setstate__(self, state):
+        cdef DataType reconstituted = union(*state)
+        self.init(reconstituted.sp_type)
 
 cdef class TimestampType(DataType):
 
@@ -1056,6 +1081,30 @@ def struct(fields):
     return pyarrow_wrap_data_type(struct_type)
 
 
+def union(children_fields, mode):
+    """
+    Create UnionType from children fields.
+    """
+    cdef:
+        Field child_field
+        vector[shared_ptr[CField]] c_fields
+        vector[uint8_t] type_codes
+        shared_ptr[CDataType] union_type
+        int i
+
+    for i, child_field in enumerate(children_fields):
+        type_codes.push_back(i)
+        c_fields.push_back(child_field.sp_field)
+
+        if mode == UnionMode_SPARSE:
+            union_type.reset(new CUnionType(c_fields, type_codes,
+                                            _UnionMode_SPARSE))
+        else:
+            union_type.reset(new CUnionType(c_fields, type_codes,
+                                            _UnionMode_DENSE))
+
+    return pyarrow_wrap_data_type(union_type)
+
 cdef dict _type_aliases = {
     'null': null,
     'i1': int8,

From ed8aef2c66169bca214f3a77cf6b6d797e172791 Mon Sep 17 00:00:00 2001
From: Lu Qi <qi1.lu@intel.com>
Date: Fri, 10 Nov 2017 14:23:42 +0100
Subject: [PATCH 1232/1644] ARROW-1793: fix a typo for README.md

fix a typo for integration test README.md

Author: Lu Qi <qi1.lu@intel.com>

Closes #1301 from luchy0120/master and squashes the following commits:

2cc86fe [Lu Qi] fix a typo
---
 integration/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/README.md b/integration/README.md
index 6e5a6c15641ec..8d6df8e5772de 100644
--- a/integration/README.md
+++ b/integration/README.md
@@ -77,7 +77,7 @@ export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug
 
 Here `$ARROW_HOME` is the location of your Arrow git clone. The
 `$CPP_BUILD_DIR` may be different depending on how you built with CMake
-(in-source of out-of-source).
+(in-source or out-of-source).
 
 Once this is done, run the integration tests with (optionally adding `--debug`
 for additional output)
@@ -88,4 +88,4 @@ python integration_test.py
 python integration_test.py --debug  # additional output
 ```
 
-[1]: https://conda.io/miniconda.html
\ No newline at end of file
+[1]: https://conda.io/miniconda.html

From 2d34f34dc81966f3e186055dc0b962699c98b236 Mon Sep 17 00:00:00 2001
From: Stephanie <swang@cs.berkeley.edu>
Date: Fri, 10 Nov 2017 09:36:12 -0500
Subject: [PATCH 1233/1644] ARROW-1788 Fix Plasma store abort bug on client
 disconnection

Author: Stephanie <swang@cs.berkeley.edu>

Closes #1299 from stephanie-wang/plasma-client-disconnect-bug and squashes the following commits:

295144bd [Stephanie] Revert disconnect client check
8e24affd [Stephanie] Refactor abort_object to match remove_client_from_object_clients
b41591d5 [Stephanie] When disconnecting a plasma client, only abort the objects that the client created
50932e53 [Stephanie] Add Plasma test for multiple clients
---
 cpp/src/plasma/store.cc             | 27 +++++++++++++-------
 cpp/src/plasma/store.h              |  9 ++++++-
 cpp/src/plasma/test/client_tests.cc | 38 +++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 5dbdebc237ce6..31033ccbb3202 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -393,16 +393,22 @@ void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[])
   update_object_get_requests(object_id);
 }
 
-void PlasmaStore::abort_object(const ObjectID& object_id) {
+int PlasmaStore::abort_object(const ObjectID& object_id, Client* client) {
   auto entry = get_object_table_entry(&store_info_, object_id);
   ARROW_CHECK(entry != NULL) << "To abort an object it must be in the object table.";
   ARROW_CHECK(entry->state != PLASMA_SEALED)
       << "To abort an object it must not have been sealed.";
-  ARROW_CHECK(entry->clients.size() == 1)
-      << "To abort an object, the only client currently using it is the creator.";
-
-  dlfree(entry->pointer);
-  store_info_.objects.erase(object_id);
+  auto it = entry->clients.find(client);
+  if (it == entry->clients.end()) {
+    // If the client requesting the abort is not the creator, do not
+    // perform the abort.
+    return 0;
+  } else {
+    // The client requesting the abort is the creator. Free the object.
+    dlfree(entry->pointer);
+    store_info_.objects.erase(object_id);
+    return 1;
+  }
 }
 
 void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
@@ -454,11 +460,12 @@ void PlasmaStore::disconnect_client(int client_fd) {
   ARROW_LOG(INFO) << "Disconnecting client on fd " << client_fd;
   // If this client was using any objects, remove it from the appropriate
   // lists.
+  auto client = it->second.get();
   for (const auto& entry : store_info_.objects) {
     if (entry.second->state == PLASMA_SEALED) {
-      remove_client_from_object_clients(entry.second.get(), it->second.get());
+      remove_client_from_object_clients(entry.second.get(), client);
     } else {
-      abort_object(entry.first);
+      abort_object(entry.first, client);
     }
   }
 
@@ -600,7 +607,9 @@ Status PlasmaStore::process_message(Client* client) {
     } break;
     case MessageType_PlasmaAbortRequest: {
       RETURN_NOT_OK(ReadAbortRequest(input, input_size, &object_id));
-      abort_object(object_id);
+      ARROW_CHECK(abort_object(object_id, client) == 1) << "To abort an object, the only "
+                                                           "client currently using it "
+                                                           "must be the creator.";
       HANDLE_SIGPIPE(SendAbortReply(client->fd, object_id), client->fd);
     } break;
     case MessageType_PlasmaGetRequest: {
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index 0d08d8a67ffaa..a72c6259a9cea 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -74,7 +74,14 @@ class PlasmaStore {
   int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
                     Client* client, PlasmaObject* result);
 
-  void abort_object(const ObjectID& object_id);
+  /// Abort a created but unsealed object. If the client is not the
+  /// creator, then the abort will fail.
+  ///
+  /// @param object_id Object ID of the object to be aborted.
+  /// @param client The client who created the object. If this does not
+  ///   match the creator of the object, then the abort will fail.
+  /// @return 1 if the abort succeeds, else 0.
+  int abort_object(const ObjectID& object_id, Client* client);
 
   /// Delete objects that have been created in the hash table. This should only
   /// be called on objects that are returned by the eviction policy to evict.
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index 5c0cee4c071ad..d4285f8988404 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -45,14 +45,17 @@ class TestPlasmaStore : public ::testing::Test {
         "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &";
     system(plasma_command.c_str());
     ARROW_CHECK_OK(client_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
+    ARROW_CHECK_OK(client2_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
   }
   virtual void Finish() {
     ARROW_CHECK_OK(client_.Disconnect());
+    ARROW_CHECK_OK(client2_.Disconnect());
     system("killall plasma_store &");
   }
 
  protected:
   PlasmaClient client_;
+  PlasmaClient client2_;
 };
 
 TEST_F(TestPlasmaStore, ContainsTest) {
@@ -171,6 +174,41 @@ TEST_F(TestPlasmaStore, AbortTest) {
   }
 }
 
+TEST_F(TestPlasmaStore, MultipleClientTest) {
+  ObjectID object_id = ObjectID::from_random();
+
+  // Test for object non-existence on the first client.
+  bool has_object;
+  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, false);
+
+  // Test for the object being in local Plasma store.
+  // First create and seal object on the second client.
+  int64_t data_size = 100;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  uint8_t* data;
+  ARROW_CHECK_OK(client2_.Create(object_id, data_size, metadata, metadata_size, &data));
+  ARROW_CHECK_OK(client2_.Seal(object_id));
+  // Test that the first client can get the object.
+  ObjectBuffer object_buffer;
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, true);
+
+  // Test that one client disconnecting does not interfere with the other.
+  // First create object on the second client.
+  object_id = ObjectID::from_random();
+  ARROW_CHECK_OK(client2_.Create(object_id, data_size, metadata, metadata_size, &data));
+  // Disconnect the first client.
+  ARROW_CHECK_OK(client_.Disconnect());
+  // Test that the second client can seal and get the created object.
+  ARROW_CHECK_OK(client2_.Seal(object_id));
+  ARROW_CHECK_OK(client2_.Get(&object_id, 1, -1, &object_buffer));
+  ARROW_CHECK_OK(client2_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, true);
+}
+
 }  // namespace plasma
 
 int main(int argc, char** argv) {

From 7c205b0337cd0364a7f9e1e19a9a6d5423abfe30 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sat, 11 Nov 2017 17:09:50 -0500
Subject: [PATCH 1234/1644] ARROW-1787: [Python] Support reading parquet files
 into DataFrames in a backward compatible way

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1298 from cpcloud/ARROW-1787 and squashes the following commits:

6f5fbd55 [Phillip Cloud] Add more index naming tests
8ba06256 [Phillip Cloud] Add test data
56e7fe58 [Phillip Cloud] Use BytesIO
5f50da38 [Phillip Cloud] Implement
d61c43e6 [Phillip Cloud] Add test
9abad95f [Phillip Cloud] Add test data
---
 python/pyarrow/pandas_compat.py               |  13 +++-
 .../tests/data/v0.7.1.all-named-index.parquet | Bin 0 -> 3948 bytes
 python/pyarrow/tests/data/v0.7.1.parquet      | Bin 0 -> 4372 bytes
 .../data/v0.7.1.some-named-index.parquet      | Bin 0 -> 4008 bytes
 python/pyarrow/tests/test_parquet.py          |  73 ++++++++++++++++++
 5 files changed, 85 insertions(+), 1 deletion(-)
 create mode 100644 python/pyarrow/tests/data/v0.7.1.all-named-index.parquet
 create mode 100644 python/pyarrow/tests/data/v0.7.1.parquet
 create mode 100644 python/pyarrow/tests/data/v0.7.1.some-named-index.parquet

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 87b47b8a6bc13..db28ee09e1e73 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -18,6 +18,7 @@
 import ast
 import collections
 import json
+import re
 
 import numpy as np
 import pandas as pd
@@ -353,6 +354,14 @@ def make_datetimetz(tz):
     return DatetimeTZDtype('ns', tz=tz)
 
 
+def backwards_compatible_index_name(raw_name, logical_name):
+    pattern = r'^__index_level_\d+__$'
+    if raw_name == logical_name and re.match(pattern, raw_name) is not None:
+        return None
+    else:
+        return logical_name
+
+
 def table_to_blockmanager(options, table, memory_pool, nthreads=1):
     import pandas.core.internals as _int
     import pyarrow.lib as lib
@@ -394,7 +403,9 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
                 values = values.copy()
 
             index_arrays.append(pd.Series(values, dtype=col_pandas.dtype))
-            index_names.append(logical_name)
+            index_names.append(
+                backwards_compatible_index_name(raw_name, logical_name)
+            )
             block_table = block_table.remove_column(
                 block_table.schema.get_field_index(raw_name)
             )
diff --git a/python/pyarrow/tests/data/v0.7.1.all-named-index.parquet b/python/pyarrow/tests/data/v0.7.1.all-named-index.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..e9efd9b390ed4ceaa53c20ca1ac182101593a6bb
GIT binary patch
literal 3948
zcmcInU5FD`6uvi^$xhOkjN9Dlq^33mJ4_)iCc7@Bxa11D?$)>)U0DjX$e7I5l>C`Z
z#9eDysM5t3tq;Xo5Yf^nDTS6IMT)egNLh>62SG|vD8AIEwifZJ=iZq~ve{9nYz8u!
zIrq$Wzwh4jok?<A_oRqGYNVbeAtLl6^xAu`?mPAULwzZ8Xe4m^Qtd_tPH^>W=H7hd
ziyilV&fxXgeWz|^K4Z~KH#1k(pZIh4m0vOhkr;_5Nd)kG#1~T~=Rs9+9#s$=p&<{P
zBiJi0riX_S(iS~25{PaVkT0r=d82MLh)0PikFm1g(LTlsQluxK=`M2sHZZt=$@^Kx
z1tjIr493CH938}1n$1!imS;YnqzGf-+Js*g?(g!)g~3U`B>X)Ar?22v*Cy!<KPD*%
zi<1qV!uSB-snQ@j0Fn!nw^+giQBLnYd=`jk--#X43TCZwkW2g0qnz<**9DMv*$dj-
z0CktBr7cIcWl4Zb(-EkU7A7d}lMu}Td$h|h=?gn4q$0GCW$e4Qg^y6i-pbi&vG3Rx
z-<l1;{Tr!+d&4+TGCB7<Sh<C@ANzp&A8eQ1;B=QL<nc<&wJJ^8jgg4ajx`dRgwH5^
zCe}z>_FxW)a`yL11x|)4joLZgg*AMU71p=U#fab(R;^ylTMD4Ce#hGN04Vs)4+`Ai
zbeE{5;4~&DahCM+^&gA>t24WV=%9!Nnc`4H-?x^Y`vl`;n7+)Mu|7<1EMZ)b=@_?x
zOf7jb<)bIUE(OLN2so<bz`GRK2Y{n3gxi~hkMIq;=us|sw38u_cQ**~+`x2~XiMH=
zTOQ>-U?kMyJ_9n>QmhOGZilI5{SbYy8wVtr9uiaAWXdfuC{ySa&@kN8S;k`O9P}rt
zyV~EOZq8A6-lP4r7Sx@W9qY_22eYlSEZS1nv3Wq8N$}F6th3E~D3s#{AGI2vmttRV
zBp^~Td~igt_tHy$c*r)~OReVSb=f?a2@W5~b6{7<+iHFYDcWPOrqlTFydw`b?>GT@
zcb<3L)dgm|YgrWFz+DGO!RZXCm;kX+qgcn%=^fQ-L4@1!f@zfc;38Ky%f)8d50}$s
zeJ;)tR`|@{8d3!Y#AUSG#Msv#u{Lo$8Y4m5WZrz!>fQIT@&IdRRgiJ^h1f9CyRl$m
z;H#5lmPp)5ku0<Fc6NG$oyOT|3^=JI*3o<M8!Fz;i3F2YxAlnfBOlVvtZUN{^VL$d
z&e!t{R?cDV+KWKsQIrwk000fMze%7M1^Mhr)eEudskFZjrEs4xH7<t6rp8lj4X3B4
zobU9CH4<Pnt$HD~YjqZfl1W&dYcI8F_2o-Oz1Xno_$F4Cuy!{F#Qs^wh_#UzJ5TiL
zW~BON^9RkcK_Z9*Uhz0DM;PXJ*89Y{vpO^r#xkvLXOBT+aIWTBSM%&D2)SOiX~?;=
z*RFNnBF!R7BpQoeUi}w#58jhKczyNYeX`2^TU*_RxylXd*;~c`cK5UTPWh`}iJ)jq
z_0^0@!I)J)m5wHgm4bO>KlGGlxiXu0J^oe#9Fs`K6THpv3(MGvx2+)L?dK1fJpE{*
zVw6pIu+l7*Aj5**-<YehU}CmWFIL`RFqLMxHrI+*54>yU`8;K_VHAvpVM82iPqQ39
z!GAn6q^>DuN>!sVJhXC-j)Cq;lJOSr1n)FGz~w!w2e`cZx!@KnuB*@rKWz<;xZrgx
z-}1fd7<d}yTn{kkdw^-zdskI8E5&@ZV6yp^tABFkb{193J*y0BdvT=;o`ElOnoaMp
zS)VOdE6lN*2GW}bQi)?og~9rvrrFq-uhrsqMm#kzIFQ~rHL*1}H8!~yEx`}29Q@Dz
E52CGv`2YX_

literal 0
HcmV?d00001

diff --git a/python/pyarrow/tests/data/v0.7.1.parquet b/python/pyarrow/tests/data/v0.7.1.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..44670bcd19afac98f01f49d5d2f9576a9925f63f
GIT binary patch
literal 4372
zcmcIoU5FD`6ux&dlT5bBZg$N~CnePw>@bCOYj)Q~ic79k-Cd2l(Uql8OFAZ#nvy^B
zBkqbI1=~dgr4Pl5w4&mJ4<h!(hdzk)LCRWWTWpac_~JtqAKF?<pL*`xOnyh9vKg4n
z%(-X2`+eu0bMEZMx9o{Xyi}AbF{xML2N2ry+rjakUoIB!MjtKsuTDJrA$sc8b8X+L
z<tP|o<frJ3)yJRgzVT!95`DaQFZ%XA>ai4+kQ9_cgOVQrosu)Ch}Ij5Xkp0+j^0rY
ze*M@Xh@;n?2&pHypwtuC$RlS!5t4dIuSlHim;c4`r<{5Q%SaYqfTr2Z0ocG`zm$$i
zJ*K3{O#y6EDMQb?;3i%&a+zw5hueulA?1eaoKcz&(FbbWu4hUeTHJLPg{KZ}LKZLN
zf*|Ca5R^PMA&z8~$#{(AXIQ<)gN&<=;26@{v0zRhtVx40Ni+%jV;F4P=+6}W*+G9M
zfs-Nyo8pUiD0rJm#52eu$_eCw2dP(F4H`nSkS&zVcz%K9vsnGr2}DjiDG?I@pkem+
zrC3bzn6nQm4zSHm4ZB?^f?fRFtk*?`X=LW+X02<aBSJhKRUETZk@na)$5ugXes(uV
zxsqAEl&Q>{wC-W~4p#pXf!0Zp(rO?vZs##gPe?0B&p1LDdu~pXG+Z9<A|n{{<(OOK
zM`qj#y6hEE+AH$vj5~z>^ls9&$H@pizTHKDa7bF$6h1Lc5AaY#)2AFGCUf6r7EE67
z4evd41~{loo~8s%8O6#0lle7HUgXsKuL1MhJ&>VxfSQe!$%;!`tqj$uD)HD(f?Xmy
zHVtw}+%0No7Xep<j>RbZw&F5J2xZTFb~5&j;#wRVfZSGZ_cd{^==-yV$^8MAm$ACs
z3*7(k0(Uz&&Bh9TkS;|jUT+xUc8mm!CLs3HaL&LP_llc$)1-xCPwm>Nz#35~f_b!M
z4jH|^5ft1R2ui%w9*U(*k|}`p@DQu_`$56)K2Tr>r`cFxCQxu1ON%%r4VYOq$)EiS
zW37*@p{;Fo9~lv_$4BliVZ6>q%>K5op9pIX9PcBOW-s%Rzo9AKdfrPIbDvFqS??*L
z<=dHGAAJt*t0(2g8sWp{`dr}TuQ~Ng3DiAX1?ud;G#jg(x)f`~eT22{-nxdTyYYbN
zBWY0EPp(ejz`8!d+P{woE^WM*RnLf>Iy2eR{g8j=egVEOQg<iNq>j}*Q@5;ux+||+
zmD2`RtDF{VUEPVs+HGcx9pnhzuNF9hdwt_Q0)eF5>Lg6MgM3dnv0DxzyfZn@lc%uC
z$DcTA>+P^vJhrS(kY~ZR%Uk=NVJ_ZhZ#KzWuomw;r=AXiyvJ`?#f4hIYQ@!JW%8ik
zrJDMk7b6Tc#lh)-lZR7)(+Q^w>WtMWLUbRaM`xe2MruV4KJ8MhK6ftEsI>k>B9l)U
zhZ9-jkdaMnNF=PD_YYPtuK{V7R)aJ<aLvX_nQ?)ojYx5plLw5P4w)_Wyv11?J`8){
z$w7Bb(7>K*n^*n0zHgWo^KI6R!QXsh{SBUYip+XNZ4Bzlx|^@xEzmpaYoivz<PQV`
z->v-T$*X@S{O_&(Rkz(PY<XSSiMp^gtZ*+@SGM`7E8A?;?iRfK&@cN@AgDNtdOoF>
zmFMD-Fl#FCRa(vE%i(uJAA~y^P58sXP?&kJ9%Pt~N5XkMXTU4@YBmcV3^Y-hFH&Q;
zTq$MpABSNeU(FTg>-NI_PmE-R+H*!lPw5rC26418O)Gxb{2PS6(KbaoThJ?Gqn&d!
z4YW@(7^?FQLvyo1t>!byLdxhweKGSEQ2K%21z7fpzVg6k$F_lOiec*lHVwQixMuI~
z0<O7>*x+XJwy97zzl`f)8@wi&ucUvz3z!RC!0asd&O)mPu{9EPb9-!QCD&^8kWngU
z3VGTQHVh4K7>b0CA_WFZ2dhS9eX>{#)jUFxp^>5C^|RAk;<J-8d(k)WB^8JNhx`Z9
Cv&vln

literal 0
HcmV?d00001

diff --git a/python/pyarrow/tests/data/v0.7.1.some-named-index.parquet b/python/pyarrow/tests/data/v0.7.1.some-named-index.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..34097ca12c7e281607df62d8ea47c7c50a9c5cfb
GIT binary patch
literal 4008
zcmcInUx*V&9G=;1a+@?Jxoc*-X;T}5UA7PplRJ-6JTiiwcN*`aCr6={u$S!Bl>9T7
zh<Da<P^E_yv_2H;gNRC>q!cVgiWF%}k)lQHgGEZIP<*LREtW(2(BI5vlU%a;U@r^V
z%>HKfo9~<X{k}=kTldC^NGhb7B0eGxAoTisukAni?IZ1G^5CfF?#0TjB;26&bMpRT
z@bew_e@x=_sr@JKBrmb(#oNiN>yQ7r=ju;Mf=HM|Vk8K7F5(I+lJ%k@S+B|nF5j>K
z_aJsSmlGo+2&v0Ln0P{)MdS)8&a7V5YebNP@?$J73F;?UMzZt-G|grXzy=2AF?m15
zxPYWIn#0&TmZn1(OY<p;{eij9rYOQ#ygum;h!1wVBjV7MTN3}CgxeK()YK_D$FC_0
z!XjityD;7Xc#1T{E`a310$WUBqBD@#cjycdQNMAvNy{0P+5s-@3qd|Ds5eBAcG&^i
z>;N^JsHrVYwxx)NOVbdjkd`JX?voHr1ADaFEon=;C=5ktDaF`#Yzyz9jJ-K#m&v|u
zTU_fl0Qaw?HtscjwqS7X_pp2itKatl_dnP!+reozQ5eTFE#1sCX%9wDjCQP(&=h>e
z;4`^S+OijONR)1TcPp?)DAHJCOlM+^Twsay^>bk&T8UMu=Ch^(NUYzmdcy+>eszNa
zJ2=fIYAQH|$qAey1AP6*BmZcuTzquMiNyfLzM!_hpPu~`<Cvem!YX6EpWeEOaV|i|
zc^L$#DK9{I?{U9Pg7FFj97VF=Z4&GQz)_d{txdv*`37AO<nw}h!Uytx_kuh-FwG{~
zl(*cHN4W=#`kLJ5KxRM1^04QwpPHrbqYw9Bj})K>o$+k}%1dG>K*1}(FznQsg~ila
z=uc92ZJ<rvf~D@9p#IPg>dpnM(wSQg)+n7-(Wbh#%>&|0f`cAm&bBB}NXHE>Y8sz|
zVwZQ+<D^difl;xsmk#>fBevlVYMPtFX7gYsxLhF5g6)#G)%XyGsE?sEE#t$BmOR+J
zV+7>gd)}(94zNabt%?F1RM#xYS<Vm-ix3;r^HnUJ+EFg&obWi2GxS0qG}2Y0n6DSz
z(3~-<3lTP;tIpibE=6QOTt>?##=rcS*~GC>n0OmS=E&Af?|y*ghgdzMfQ&OQgh!Cp
zi3Jk_Uya04MB<ecOtDhl&Tc!|ZGzp#fs;bQZQhG_DtJ35;*FVZD>wryE~K7*sYRnd
zlgXEI#^FrCIAjzu@l1xV?P)BZ#p?AJfz(Qfk>X%54KR6=fG>GDlbGUw*vxdo-G}11
zPn@1``o^ax;%q%<W~Qz0MAv$W(3?;kFtlT(mWN|8SgBuLZc*&Z7W8VqX6E#5EZ@ZH
zN*IWJ6lTO)NQ|8jy0#f9uKDZ%qo|W0BA!<T>!5^g{$!pl)|2VhOc)z!I$xs?Ou}j{
zG+T?T<%O{hwt>icvRTJwUsC{!$iYxJbb0L$@*bR@Jvi2SaL%l;ud8coq-$(k&u$_A
ztLmS%9_7CV<RA)#6<0+s<@9;wigYAu`c1Z6s25A~(KjM*MWK?Su}HMbide8w{L!~r
zo4;ckmcd^=5-sUP13*gkLIFmY(`0H36&8%n*Q)u_I}D~&FIE<s@$&4uMwZW6G-`TI
zujvhlqpfLH<45_AZHCk_#ayAR*G7iB=V<HboFo=$@{V#h>H#kIwI1MdhqS@Xmuy#|
z8GhOt9Jax0TfWtM*Vgeg%!M9cF7^Pkk^3E4RWIeU<($FhTdn@dncJRJww1KDQ&@G`
zT@lZ~omE|f-XWtppD&kK^==wWY#NM5k0J#+s|V{wZDY1li8Mw;;)6qjiH*~fThr6y
PQ~S`r@MA3v|Iz;gF;Sai

literal 0
HcmV?d00001

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index e2e6863c4748f..6ba4fd2fad8ea 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1458,3 +1458,76 @@ def test_index_column_name_duplicate(tmpdir):
     arrow_table = _read_table(path)
     result_df = arrow_table.to_pandas()
     tm.assert_frame_equal(result_df, dfx)
+
+
+def test_backwards_compatible_index_naming():
+    expected_string = b"""\
+carat        cut  color  clarity  depth  table  price     x     y     z
+ 0.23      Ideal      E      SI2   61.5   55.0    326  3.95  3.98  2.43
+ 0.21    Premium      E      SI1   59.8   61.0    326  3.89  3.84  2.31
+ 0.23       Good      E      VS1   56.9   65.0    327  4.05  4.07  2.31
+ 0.29    Premium      I      VS2   62.4   58.0    334  4.20  4.23  2.63
+ 0.31       Good      J      SI2   63.3   58.0    335  4.34  4.35  2.75
+ 0.24  Very Good      J     VVS2   62.8   57.0    336  3.94  3.96  2.48
+ 0.24  Very Good      I     VVS1   62.3   57.0    336  3.95  3.98  2.47
+ 0.26  Very Good      H      SI1   61.9   55.0    337  4.07  4.11  2.53
+ 0.22       Fair      E      VS2   65.1   61.0    337  3.87  3.78  2.49
+ 0.23  Very Good      H      VS1   59.4   61.0    338  4.00  4.05  2.39"""
+    expected = pd.read_csv(
+        io.BytesIO(expected_string), sep=r'\s{2,}', index_col=None, header=0
+    )
+    path = os.path.join(os.path.dirname(__file__), 'data', 'v0.7.1.parquet')
+    t = _read_table(path)
+    result = t.to_pandas()
+    tm.assert_frame_equal(result, expected)
+
+
+def test_backwards_compatible_index_multi_level_named():
+    expected_string = b"""\
+carat        cut  color  clarity  depth  table  price     x     y     z
+ 0.23      Ideal      E      SI2   61.5   55.0    326  3.95  3.98  2.43
+ 0.21    Premium      E      SI1   59.8   61.0    326  3.89  3.84  2.31
+ 0.23       Good      E      VS1   56.9   65.0    327  4.05  4.07  2.31
+ 0.29    Premium      I      VS2   62.4   58.0    334  4.20  4.23  2.63
+ 0.31       Good      J      SI2   63.3   58.0    335  4.34  4.35  2.75
+ 0.24  Very Good      J     VVS2   62.8   57.0    336  3.94  3.96  2.48
+ 0.24  Very Good      I     VVS1   62.3   57.0    336  3.95  3.98  2.47
+ 0.26  Very Good      H      SI1   61.9   55.0    337  4.07  4.11  2.53
+ 0.22       Fair      E      VS2   65.1   61.0    337  3.87  3.78  2.49
+ 0.23  Very Good      H      VS1   59.4   61.0    338  4.00  4.05  2.39"""
+    expected = pd.read_csv(
+        io.BytesIO(expected_string),
+        sep=r'\s{2,}', index_col=['cut', 'color', 'clarity'], header=0
+    ).sort_index()
+    path = os.path.join(
+        os.path.dirname(__file__), 'data', 'v0.7.1.all-named-index.parquet'
+    )
+    t = _read_table(path)
+    result = t.to_pandas()
+    tm.assert_frame_equal(result, expected)
+
+
+def test_backwards_compatible_index_multi_level_some_named():
+    expected_string = b"""\
+carat        cut  color  clarity  depth  table  price     x     y     z
+ 0.23      Ideal      E      SI2   61.5   55.0    326  3.95  3.98  2.43
+ 0.21    Premium      E      SI1   59.8   61.0    326  3.89  3.84  2.31
+ 0.23       Good      E      VS1   56.9   65.0    327  4.05  4.07  2.31
+ 0.29    Premium      I      VS2   62.4   58.0    334  4.20  4.23  2.63
+ 0.31       Good      J      SI2   63.3   58.0    335  4.34  4.35  2.75
+ 0.24  Very Good      J     VVS2   62.8   57.0    336  3.94  3.96  2.48
+ 0.24  Very Good      I     VVS1   62.3   57.0    336  3.95  3.98  2.47
+ 0.26  Very Good      H      SI1   61.9   55.0    337  4.07  4.11  2.53
+ 0.22       Fair      E      VS2   65.1   61.0    337  3.87  3.78  2.49
+ 0.23  Very Good      H      VS1   59.4   61.0    338  4.00  4.05  2.39"""
+    expected = pd.read_csv(
+        io.BytesIO(expected_string),
+        sep=r'\s{2,}', index_col=['cut', 'color', 'clarity'], header=0
+    ).sort_index()
+    expected.index = expected.index.set_names(['cut', None, 'clarity'])
+    path = os.path.join(
+        os.path.dirname(__file__), 'data', 'v0.7.1.some-named-index.parquet'
+    )
+    t = _read_table(path)
+    result = t.to_pandas()
+    tm.assert_frame_equal(result, expected)

From 21112f85faa6e1328b3d59a54aa24becc50df4c1 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sat, 11 Nov 2017 18:58:01 -0500
Subject: [PATCH 1235/1644] ARROW-1800: [C++] Fix and simplify random_decimals

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1306 from cpcloud/ARROW-1800 and squashes the following commits:

d5b08ff0 [Phillip Cloud] ARROW-1800: [C++] Fix and simplify random_decimals
---
 cpp/src/arrow/test-util.h | 110 ++++++++++++--------------------------
 1 file changed, 34 insertions(+), 76 deletions(-)

diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 7306f577a36e0..9b875ce116a29 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -159,12 +159,11 @@ static inline void random_is_valid(int64_t n, double pct_null,
 
 static inline void random_bytes(int64_t n, uint32_t seed, uint8_t* out) {
   std::mt19937 gen(seed);
-  std::uniform_int_distribution<int> d(0, std::numeric_limits<uint8_t>::max());
-  std::generate(out, out + n, [&d, &gen] { return static_cast<uint8_t>(d(gen) & 0xFF); });
+  std::uniform_int_distribution<uint32_t> d(0, std::numeric_limits<uint8_t>::max());
+  std::generate(out, out + n, [&d, &gen] { return static_cast<uint8_t>(d(gen)); });
 }
 
-static void DecimalRange(int32_t precision, Decimal128* min_decimal,
-                         Decimal128* max_decimal) {
+static int32_t DecimalSize(int32_t precision) {
   DCHECK_GE(precision, 1) << "decimal precision must be greater than or equal to 1, got "
                           << precision;
   DCHECK_LE(precision, 38) << "decimal precision must be less than or equal to 38, got "
@@ -173,123 +172,82 @@ static void DecimalRange(int32_t precision, Decimal128* min_decimal,
   switch (precision) {
     case 1:
     case 2:
-      *max_decimal = std::numeric_limits<int8_t>::max();
-      break;
+      return 1;  // 127
     case 3:
     case 4:
-      *max_decimal = std::numeric_limits<int16_t>::max();
-      break;
+      return 2;  // 32,767
     case 5:
     case 6:
-      *max_decimal = 8388607;
-      break;
+      return 3;  // 8,388,607
     case 7:
     case 8:
     case 9:
-      *max_decimal = std::numeric_limits<int32_t>::max();
-      break;
+      return 4;  // 2,147,483,427
     case 10:
     case 11:
-      *max_decimal = 549755813887;
-      break;
+      return 5;  // 549,755,813,887
     case 12:
     case 13:
     case 14:
-      *max_decimal = 140737488355327;
-      break;
+      return 6;  // 140,737,488,355,327
     case 15:
     case 16:
-      *max_decimal = 36028797018963967;
-      break;
+      return 7;  // 36,028,797,018,963,967
     case 17:
     case 18:
-      *max_decimal = std::numeric_limits<int64_t>::max();
-      break;
+      return 8;  // 9,223,372,036,854,775,807
     case 19:
     case 20:
     case 21:
-      *max_decimal = Decimal128("2361183241434822606847");
-      break;
+      return 9;  // 2,361,183,241,434,822,606,847
     case 22:
     case 23:
-      *max_decimal = Decimal128("604462909807314587353087");
-      break;
+      return 10;  // 604,462,909,807,314,587,353,087
     case 24:
     case 25:
     case 26:
-      *max_decimal = Decimal128("154742504910672534362390527");
-      break;
+      return 11;  // 154,742,504,910,672,534,362,390,527
     case 27:
     case 28:
-      *max_decimal = Decimal128("39614081257132168796771975167");
-      break;
+      return 12;  // 39,614,081,257,132,168,796,771,975,167
     case 29:
     case 30:
     case 31:
-      *max_decimal = Decimal128("10141204801825835211973625643007");
-      break;
+      return 13;  // 10,141,204,801,825,835,211,973,625,643,007
     case 32:
     case 33:
-      *max_decimal = Decimal128("2596148429267413814265248164610047");
-      break;
+      return 14;  // 2,596,148,429,267,413,814,265,248,164,610,047
     case 34:
     case 35:
-      *max_decimal = Decimal128("664613997892457936451903530140172287");
-      break;
+      return 15;  // 664,613,997,892,457,936,451,903,530,140,172,287
     case 36:
     case 37:
     case 38:
-      *max_decimal = Decimal128("170141183460469231731687303715884105727");
-      break;
+      return 16;  // 170,141,183,460,469,231,731,687,303,715,884,105,727
     default:
       DCHECK(false);
       break;
   }
-
-  *min_decimal = ~(*max_decimal);
+  return -1;
 }
 
-class UniformDecimalDistribution {
- public:
-  explicit UniformDecimalDistribution(int32_t precision) {
-    Decimal128 max_decimal;
-    Decimal128 min_decimal;
-    DecimalRange(precision, &min_decimal, &max_decimal);
-
-    const auto min_low = static_cast<int64_t>(min_decimal.low_bits());
-    const auto max_low = static_cast<int64_t>(max_decimal.low_bits());
-
-    const int64_t min_high = min_decimal.high_bits();
-    const int64_t max_high = max_decimal.high_bits();
-
-    using param_type = std::uniform_int_distribution<int64_t>::param_type;
-
-    lower_dist_.param(param_type(min_low, max_low));
-    upper_dist_.param(param_type(min_high, max_high));
-  }
-
-  template <typename Generator>
-  Decimal128 operator()(Generator& gen) {
-    return Decimal128(upper_dist_(gen), static_cast<uint64_t>(lower_dist_(gen)));
-  }
-
- private:
-  // The lower bits distribution is intentionally int64_t.
-  // If it were uint64_t then the size of the interval [min_high, max_high] would be 0
-  // because min_high > max_high due to 2's complement.
-  // So, we generate the same range of bits using int64_t and then cast to uint64_t.
-  std::uniform_int_distribution<int64_t> lower_dist_;
-  std::uniform_int_distribution<int64_t> upper_dist_;
-};
-
 static inline void random_decimals(int64_t n, uint32_t seed, int32_t precision,
                                    uint8_t* out) {
   std::mt19937 gen(seed);
-  UniformDecimalDistribution dist(precision);
-
-  for (int64_t i = 0; i < n; ++i, out += 16) {
-    const Decimal128 value(dist(gen));
-    value.ToBytes(out);
+  std::uniform_int_distribution<uint32_t> d(0, std::numeric_limits<uint8_t>::max());
+  const int32_t required_bytes = DecimalSize(precision);
+  constexpr int32_t byte_width = 16;
+  std::fill(out, out + byte_width * n, '\0');
+
+  for (int64_t i = 0; i < n; ++i, out += byte_width) {
+    std::generate(out, out + required_bytes,
+                  [&d, &gen] { return static_cast<uint8_t>(d(gen)); });
+
+    // sign extend if the sign bit is set for the last byte generated
+    // 0b10000000 == 0x80 == 128
+    if ((out[required_bytes - 1] & '\x80') != 0) {
+      std::fill(out + required_bytes, out + byte_width, '\xFF');
+    }
   }
 }
 

From 357eedcbf4b06d4ab1a78005fe34244319d6b9b0 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sat, 11 Nov 2017 19:02:53 -0500
Subject: [PATCH 1236/1644] ARROW-1781: Don't use brew when using the toolchain

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1295 from xhochy/ARROW-1781 and squashes the following commits:

7de96d8a [Uwe L. Korn] Use a single Brewfile for c_glib
9fde4e91 [Korn, Uwe] Use gtest from conda-forge
8cc6898f [Korn, Uwe] ARROW-1781: Don't use brew when using the toolchain
---
 .travis.yml                       |  1 +
 c_glib/Brewfile                   | 11 +++++++----
 ci/travis_before_script_c_glib.sh |  2 --
 ci/travis_before_script_cpp.sh    |  5 ++---
 cpp/Brewfile                      | 21 ---------------------
 5 files changed, 10 insertions(+), 30 deletions(-)
 delete mode 100644 cpp/Brewfile

diff --git a/.travis.yml b/.travis.yml
index 52d7a5f800505..9917a261451c2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -152,6 +152,7 @@ matrix:
     rvm: 2.2
     env: BUILD_SYSTEM=autotools
     before_script:
+    - brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
diff --git a/c_glib/Brewfile b/c_glib/Brewfile
index 80d3c81dd6f82..9fe5c3b616317 100644
--- a/c_glib/Brewfile
+++ b/c_glib/Brewfile
@@ -15,11 +15,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
-brew "gtk-doc"
 brew "autoconf-archive"
-brew "gobject-introspection"
-brew "git"
+brew "boost"
+brew "ccache"
 brew "cmake"
-brew "wget"
+brew "git"
+brew "gobject-introspection"
+brew "gtk-doc"
+brew "jemalloc"
 brew "libtool"
 brew "lua"
+brew "wget"
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index a22ecd3753fd8..99d05397a9f23 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -22,8 +22,6 @@ set -ex
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 if [ $TRAVIS_OS_NAME = "osx" ]; then
-  brew update && brew bundle --file=c_glib/Brewfile
-
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
   export XML_CATALOG_FILES=/usr/local/etc/xml/catalog
 fi
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index dbdcd33ed0d5b..4998f190f9891 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -38,8 +38,10 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
         rapidjson \
         flatbuffers \
         gflags \
+        gtest \
         lz4-c \
         snappy \
+        ccache \
         zstd \
         brotli \
         zlib \
@@ -53,9 +55,6 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
   conda update -y -p $CPP_TOOLCHAIN ca-certificates -c defaults
 fi
 
-if [ $TRAVIS_OS_NAME == "osx" ]; then
-  brew update && brew bundle --file=cpp/Brewfile
-fi
 
 mkdir $ARROW_CPP_BUILD_DIR
 pushd $ARROW_CPP_BUILD_DIR
diff --git a/cpp/Brewfile b/cpp/Brewfile
deleted file mode 100644
index 5f82cacc55991..0000000000000
--- a/cpp/Brewfile
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-brew "jemalloc"
-brew "ccache"
-brew "boost"
-brew "cmake"

From 550a39ff23054ee8cce99380b1ee1f29035f9f67 Mon Sep 17 00:00:00 2001
From: rvernica <rvernica@gmail.com>
Date: Sun, 12 Nov 2017 22:16:29 +0900
Subject: [PATCH 1237/1644] ARROW-1801: [Docs] Update install instructions to
 use red-data-tools repos

* Update package install instructions to use red-data-tools.org repositories.
* Instructions taken from https://github.com/red-data-tools/packages.red-data-tools.org/blob/master/README.md
* Dropped support for Debian jessie and Ubuntu 16.10
* Add support for Debian stretch, Ubuntu 14.04, and CentOS 6

Author: rvernica <rvernica@gmail.com>

Closes #1307 from rvernica/patch-3 and squashes the following commits:

f15dac08 [rvernica] ARROW-1801 [Docs] Update install instructions to use red-data-tools repos
---
 site/install.md | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/site/install.md b/site/install.md
index 1513fe819789c..0ef2008db9061 100644
--- a/site/install.md
+++ b/site/install.md
@@ -75,23 +75,25 @@ with the wheel.
 We have provided APT and Yum repositories for Apache Arrow C++ and
 Apache Arrow GLib (C). Here are supported platforms:
 
-* Debian GNU/Linux Jessie
+* Debian GNU/Linux stretch
+* Ubuntu 14.04 LTS
 * Ubuntu 16.04 LTS
-* Ubuntu 16.10
 * Ubuntu 17.04
+* Ubuntu 17.10
+* CentOS 6
 * CentOS 7
 
-Debian GNU/Linux Jessie:
+Debian GNU/Linux:
 
 ```shell
-sudo apt update
 sudo apt install -y -V apt-transport-https
-cat <<APT_LINE | sudo tee /etc/apt/sources.list.d/groonga.list
-deb https://packages.groonga.org/debian/ jessie main
-deb-src https://packages.groonga.org/debian/ jessie main
+sudo apt install -y -V lsb-release
+cat <<APT_LINE | sudo tee /etc/apt/sources.list.d/red-data-tools.list
+deb https://packages.red-data-tools.org/debian/ $(lsb_release --codename --short) main
+deb-src https://packages.red-data-tools.org/debian/ $(lsb_release --codename --short) main
 APT_LINE
-sudo apt update
-sudo apt install -y -V --allow-unauthenticated groonga-keyring
+sudo apt update --allow-insecure-repositories
+sudo apt install -y -V --allow-unauthenticated red-data-tools-keyring
 sudo apt update
 sudo apt install -y -V libarrow-dev # For C++
 sudo apt install -y -V libarrow-glib-dev # For GLib (C)
@@ -100,8 +102,14 @@ sudo apt install -y -V libarrow-glib-dev # For GLib (C)
 Ubuntu:
 
 ```shell
-sudo apt install -y software-properties-common
-sudo add-apt-repository -y ppa:groonga/ppa
+sudo apt install -y -V apt-transport-https
+sudo apt install -y -V lsb-release
+cat <<APT_LINE | sudo tee /etc/apt/sources.list.d/red-data-tools.list
+deb https://packages.red-data-tools.org/ubuntu/ $(lsb_release --codename --short) universe
+deb-src https://packages.red-data-tools.org/ubuntu/ $(lsb_release --codename --short) universe
+APT_LINE
+sudo apt update --allow-insecure-repositories || sudo apt update
+sudo apt install -y -V --allow-unauthenticated red-data-tools-keyring
 sudo apt update
 sudo apt install -y -V libarrow-dev # For C++
 sudo apt install -y -V libarrow-glib-dev # For GLib (C)
@@ -110,7 +118,7 @@ sudo apt install -y -V libarrow-glib-dev # For GLib (C)
 CentOS:
 
 ```shell
-sudo yum install -y https://packages.groonga.org/centos/groonga-release-1.3.0-1.noarch.rpm
+sudo yum install -y https://packages.red-data-tools.org/centos/red-data-tools-release-1.0.0-1.noarch.rpm
 sudo yum install -y --enablerepo=epel arrow-devel # For C++
 sudo yum install -y --enablerepo=epel arrow-glib-devel # For GLib (C)
 ```

From 7adadd8e0c2f693d7c862e36162e1342dff20926 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 12 Nov 2017 15:16:35 +0100
Subject: [PATCH 1238/1644] ARROW-1763: [Python] Implement __hash__ for
 DataType

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1308 from wesm/ARROW-1763 and squashes the following commits:

4e379b9 [Wes McKinney] Implement __hash__ for DataType
---
 python/pyarrow/tests/test_types.py | 24 ++++++++++++++++++++++++
 python/pyarrow/types.pxi           |  3 +++
 2 files changed, 27 insertions(+)

diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index 0e3ea1fd40bf5..9eefa33b66187 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -137,3 +137,27 @@ def test_is_temporal_date_time_timestamp():
 def test_timestamp_type():
     # See ARROW-1683
     assert isinstance(pa.timestamp('ns'), pa.TimestampType)
+
+
+def test_types_hashable():
+    types = [
+        pa.null(),
+        pa.int32(),
+        pa.time32('s'),
+        pa.time64('us'),
+        pa.date32(),
+        pa.timestamp('us'),
+        pa.string(),
+        pa.binary(),
+        pa.binary(10),
+        pa.list_(pa.int32()),
+        pa.struct([pa.field('a', pa.int32()),
+                   pa.field('b', pa.int8()),
+                   pa.field('c', pa.string())])
+    ]
+
+    in_dict = {}
+    for i, type_ in enumerate(types):
+        assert hash(type_) == hash(type_)
+        in_dict[type_] = i
+        assert in_dict[type_] == i
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index d2e68ff79a524..edf0d8a305a02 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -69,6 +69,9 @@ cdef class DataType:
             )
         return frombytes(self.type.ToString())
 
+    def __hash__(self):
+        return hash(str(self))
+
     def __reduce__(self):
         return self.__class__, (), self.__getstate__()
 

From e8331f46f8b324271e694557789ea53b082fdc05 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 12 Nov 2017 23:47:47 -0500
Subject: [PATCH 1239/1644] ARROW-1794: [C++/Python] Rename DecimalArray to
 Decimal128Array

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1312 from cpcloud/ARROW-1794 and squashes the following commits:

0b8ba5e0 [Phillip Cloud] Backward compat
4eb2a3ba [Phillip Cloud] ARROW-1794: [C++/Python] Rename DecimalArray to Decimal128Array
---
 cpp/src/arrow/array-test.cc             |  2 +-
 cpp/src/arrow/array.cc                  |  6 +++---
 cpp/src/arrow/array.h                   | 11 +++++++----
 cpp/src/arrow/compare.cc                |  2 +-
 cpp/src/arrow/ipc/json-internal.cc      |  2 +-
 cpp/src/arrow/ipc/test-common.h         |  6 +++---
 cpp/src/arrow/ipc/writer.cc             |  2 +-
 cpp/src/arrow/pretty_print.cc           |  2 +-
 cpp/src/arrow/python/arrow_to_pandas.cc |  2 +-
 cpp/src/arrow/type_fwd.h                |  2 +-
 cpp/src/arrow/type_traits.h             |  2 +-
 cpp/src/arrow/visitor.cc                |  2 +-
 cpp/src/arrow/visitor.h                 |  2 +-
 python/doc/source/api.rst               |  2 +-
 python/pyarrow/__init__.py              |  2 +-
 python/pyarrow/array.pxi                |  4 ++--
 python/pyarrow/includes/libarrow.pxd    |  4 +++-
 python/pyarrow/lib.pxd                  |  2 +-
 python/pyarrow/scalar.pxi               |  2 +-
 19 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 9f248cdbbb754..15c75534e53ef 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -2799,7 +2799,7 @@ class DecimalTest : public ::testing::TestWithParam<int> {
         BitUtil::BytesToBits(valid_bytes, default_memory_pool(), &expected_null_bitmap));
 
     int64_t expected_null_count = test::null_count(valid_bytes);
-    auto expected = std::make_shared<DecimalArray>(
+    auto expected = std::make_shared<Decimal128Array>(
         type, size, expected_data, expected_null_bitmap, expected_null_count);
 
     std::shared_ptr<Array> lhs = out->Slice(offset);
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 9c91d619cc7e8..651fa26ba9026 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -314,12 +314,12 @@ const uint8_t* FixedSizeBinaryArray::GetValue(int64_t i) const {
 // ----------------------------------------------------------------------
 // Decimal
 
-DecimalArray::DecimalArray(const std::shared_ptr<ArrayData>& data)
+Decimal128Array::Decimal128Array(const std::shared_ptr<ArrayData>& data)
     : FixedSizeBinaryArray(data) {
   DCHECK_EQ(data->type->id(), Type::DECIMAL);
 }
 
-std::string DecimalArray::FormatValue(int64_t i) const {
+std::string Decimal128Array::FormatValue(int64_t i) const {
   const auto& type_ = static_cast<const DecimalType&>(*type());
   const Decimal128 value(GetValue(i));
   return value.ToString(type_.scale());
@@ -515,7 +515,7 @@ struct ValidateVisitor {
 
   Status Visit(const PrimitiveArray&) { return Status::OK(); }
 
-  Status Visit(const DecimalArray&) { return Status::OK(); }
+  Status Visit(const Decimal128Array&) { return Status::OK(); }
 
   Status Visit(const BinaryArray&) {
     // TODO(wesm): what to do here?
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index f7762ce104398..3337e4b158267 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -559,19 +559,22 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 };
 
 // ----------------------------------------------------------------------
-// DecimalArray
-class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray {
+// Decimal128Array
+class ARROW_EXPORT Decimal128Array : public FixedSizeBinaryArray {
  public:
   using TypeClass = DecimalType;
 
   using FixedSizeBinaryArray::FixedSizeBinaryArray;
 
-  /// \brief Construct DecimalArray from ArrayData instance
-  explicit DecimalArray(const std::shared_ptr<ArrayData>& data);
+  /// \brief Construct Decimal128Array from ArrayData instance
+  explicit Decimal128Array(const std::shared_ptr<ArrayData>& data);
 
   std::string FormatValue(int64_t i) const;
 };
 
+// Backward compatibility
+using DecimalArray = Decimal128Array;
+
 // ----------------------------------------------------------------------
 // Struct
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index a2d4de7b73afb..253c2e1feb801 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -255,7 +255,7 @@ class RangeEqualsVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DecimalArray& left) {
+  Status Visit(const Decimal128Array& left) {
     return Visit(static_cast<const FixedSizeBinaryArray&>(left));
   }
 
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 1b9baee7dafef..976f9660a1b78 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -458,7 +458,7 @@ class ArrayWriter {
     }
   }
 
-  void WriteDataValues(const DecimalArray& arr) {
+  void WriteDataValues(const Decimal128Array& arr) {
     for (int64_t i = 0; i < arr.length(); ++i) {
       const Decimal128 value(arr.GetValue(i));
       writer_->String(value.ToIntegerString());
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 91023db489852..7fc139381052c 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -690,10 +690,10 @@ Status MakeDecimal(std::shared_ptr<RecordBatch>* out) {
 
   RETURN_NOT_OK(BitUtil::BytesToBits(is_valid_bytes, default_memory_pool(), &is_valid));
 
-  auto a1 = std::make_shared<DecimalArray>(f0->type(), length, data, is_valid,
-                                           kUnknownNullCount);
+  auto a1 = std::make_shared<Decimal128Array>(f0->type(), length, data, is_valid,
+                                              kUnknownNullCount);
 
-  auto a2 = std::make_shared<DecimalArray>(f1->type(), length, data);
+  auto a2 = std::make_shared<Decimal128Array>(f1->type(), length, data);
 
   ArrayVector arrays = {a1, a2};
   *out = std::make_shared<RecordBatch>(schema, length, arrays);
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 5598cc68296f7..323116f589b73 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -339,7 +339,7 @@ class RecordBatchSerializer : public ArrayVisitor {
   VISIT_FIXED_WIDTH(Time32Array)
   VISIT_FIXED_WIDTH(Time64Array)
   VISIT_FIXED_WIDTH(FixedSizeBinaryArray)
-  VISIT_FIXED_WIDTH(DecimalArray)
+  VISIT_FIXED_WIDTH(Decimal128Array)
 
 #undef VISIT_FIXED_WIDTH
 
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index aaea34a51388c..cfbc30315fcd7 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -170,7 +170,7 @@ class ArrayPrinter : public PrettyPrinter {
   }
 
   template <typename T>
-  inline typename std::enable_if<std::is_same<DecimalArray, T>::value, void>::type
+  inline typename std::enable_if<std::is_same<Decimal128Array, T>::value, void>::type
   WriteDataValues(const T& array) {
     for (int i = 0; i < array.length(); ++i) {
       if (i > 0) {
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 3894772daa467..f966b2987b93f 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -633,7 +633,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
   PyObject* Decimal = Decimal_ref.obj();
 
   for (int c = 0; c < data.num_chunks(); c++) {
-    auto* arr(static_cast<arrow::DecimalArray*>(data.chunk(c).get()));
+    auto* arr(static_cast<arrow::Decimal128Array*>(data.chunk(c).get()));
     auto type(std::dynamic_pointer_cast<arrow::DecimalType>(arr->type()));
     const int scale = type->scale();
 
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 0d06b6f6cb86e..343487055c6fc 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -69,7 +69,7 @@ class StructArray;
 class StructBuilder;
 
 class DecimalType;
-class DecimalArray;
+class Decimal128Array;
 class DecimalBuilder;
 
 class UnionType;
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index fbd78398f4579..bc600372eb01f 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -231,7 +231,7 @@ struct TypeTraits<DoubleType> {
 
 template <>
 struct TypeTraits<DecimalType> {
-  using ArrayType = DecimalArray;
+  using ArrayType = Decimal128Array;
   using BuilderType = DecimalBuilder;
   constexpr static bool is_parameter_free = false;
 };
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index a7b01b0f6315a..3739e89f3d019 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -56,7 +56,7 @@ ARRAY_VISITOR_DEFAULT(ListArray);
 ARRAY_VISITOR_DEFAULT(StructArray);
 ARRAY_VISITOR_DEFAULT(UnionArray);
 ARRAY_VISITOR_DEFAULT(DictionaryArray);
-ARRAY_VISITOR_DEFAULT(DecimalArray);
+ARRAY_VISITOR_DEFAULT(Decimal128Array);
 
 #undef ARRAY_VISITOR_DEFAULT
 
diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h
index 6c36e465ec436..34679eb950d4b 100644
--- a/cpp/src/arrow/visitor.h
+++ b/cpp/src/arrow/visitor.h
@@ -50,7 +50,7 @@ class ARROW_EXPORT ArrayVisitor {
   virtual Status Visit(const Time64Array& array);
   virtual Status Visit(const TimestampArray& array);
   virtual Status Visit(const IntervalArray& array);
-  virtual Status Visit(const DecimalArray& array);
+  virtual Status Visit(const Decimal128Array& array);
   virtual Status Visit(const ListArray& array);
   virtual Status Visit(const StructArray& array);
   virtual Status Visit(const UnionArray& array);
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 6bceba3c650b6..c52024044a62d 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -155,7 +155,7 @@ Array Types
    Date32Array
    Date64Array
    TimestampArray
-   DecimalArray
+   Decimal128Array
    ListArray
 
 .. _api.table:
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 2d7d7288b3835..09bf6b35f396a 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -58,7 +58,7 @@
                          DictionaryArray,
                          Date32Array, Date64Array,
                          TimestampArray, Time32Array, Time64Array,
-                         DecimalArray, StructArray,
+                         Decimal128Array, StructArray,
                          ArrayValue, Scalar, NA,
                          BooleanValue,
                          Int8Value, Int16Value, Int32Value, Int64Value,
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 9991411e55dfe..2ef592ff758fc 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -597,7 +597,7 @@ cdef class FixedSizeBinaryArray(Array):
     pass
 
 
-cdef class DecimalArray(FixedSizeBinaryArray):
+cdef class Decimal128Array(FixedSizeBinaryArray):
     pass
 
 
@@ -846,7 +846,7 @@ cdef dict _array_classes = {
     _Type_STRING: StringArray,
     _Type_DICTIONARY: DictionaryArray,
     _Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
-    _Type_DECIMAL: DecimalArray,
+    _Type_DECIMAL: Decimal128Array,
     _Type_STRUCT: StructArray,
 }
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index dfafd371b2857..11cc6b3ff2664 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -312,7 +312,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CFixedSizeBinaryArray" arrow::FixedSizeBinaryArray"(CArray):
         const uint8_t* GetValue(int i)
 
-    cdef cppclass CDecimalArray" arrow::DecimalArray"(CFixedSizeBinaryArray):
+    cdef cppclass CDecimal128Array" arrow::Decimal128Array"(
+        CFixedSizeBinaryArray
+    ):
         c_string FormatValue(int i)
 
     cdef cppclass CListArray" arrow::ListArray"(CArray):
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 531489490754e..6413b838f0595 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -246,7 +246,7 @@ cdef class FixedSizeBinaryArray(Array):
     pass
 
 
-cdef class DecimalArray(FixedSizeBinaryArray):
+cdef class Decimal128Array(FixedSizeBinaryArray):
     pass
 
 
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index a396fa763c8c8..1bc5ed7a372a8 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -258,7 +258,7 @@ cdef class DecimalValue(ArrayValue):
 
     def as_py(self):
         cdef:
-            CDecimalArray* ap = <CDecimalArray*> self.sp_array.get()
+            CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
             c_string s = ap.FormatValue(self.index)
         return _pydecimal.Decimal(s.decode('utf8'))
 

From 4a33bad5f28830812fe4d47dcfdfb184d5ee43c0 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Sun, 12 Nov 2017 23:49:12 -0500
Subject: [PATCH 1240/1644] ARROW-1767: [C++] Support file reads and writes
 over 2GB on Windows

This closes [ARROW-1767](https://issues.apache.org/jira/browse/ARROW-1767).

Author: Licht-T <licht-t@outlook.jp>

Closes #1311 from Licht-T/feature-large-file-io-windows and squashes the following commits:

690d2801 [Licht-T] ENH: Support large file io on Windows
---
 cpp/src/arrow/io/file.cc | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 057cad1111685..1ec5e23e587d5 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -255,19 +255,18 @@ static inline Status FileSeek(int fd, int64_t pos) {
 
 static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t nbytes,
                               int64_t* bytes_read) {
-#if defined(_MSC_VER)
-  if (nbytes > ARROW_MAX_IO_CHUNKSIZE) {
-    return Status::IOError("Unable to read > 2GB blocks yet");
-  }
-  *bytes_read = static_cast<int64_t>(_read(fd, buffer, static_cast<uint32_t>(nbytes)));
-#else
   *bytes_read = 0;
 
   while (*bytes_read != -1 && *bytes_read < nbytes) {
     int64_t chunksize =
         std::min(static_cast<int64_t>(ARROW_MAX_IO_CHUNKSIZE), nbytes - *bytes_read);
+#if defined(_MSC_VER)
+    int64_t ret = static_cast<int64_t>(
+        _read(fd, buffer + *bytes_read, static_cast<uint32_t>(chunksize)));
+#else
     int64_t ret = static_cast<int64_t>(
         read(fd, buffer + *bytes_read, static_cast<size_t>(chunksize)));
+#endif
 
     if (ret != -1) {
       *bytes_read += ret;
@@ -279,7 +278,6 @@ static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t nbyte
       *bytes_read = ret;
     }
   }
-#endif
 
   if (*bytes_read == -1) {
     return Status::IOError(std::string("Error reading bytes from file: ") +
@@ -292,25 +290,23 @@ static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t nbyte
 static inline Status FileWrite(const int fd, const uint8_t* buffer,
                                const int64_t nbytes) {
   int ret = 0;
-#if defined(_MSC_VER)
-  if (nbytes > ARROW_MAX_IO_CHUNKSIZE) {
-    return Status::IOError("Unable to write > 2GB blocks to file yet");
-  }
-  ret = static_cast<int>(_write(fd, buffer, static_cast<uint32_t>(nbytes)));
-#else
   int64_t bytes_written = 0;
 
   while (ret != -1 && bytes_written < nbytes) {
     int64_t chunksize =
         std::min(static_cast<int64_t>(ARROW_MAX_IO_CHUNKSIZE), nbytes - bytes_written);
+#if defined(_MSC_VER)
+    ret = static_cast<int>(
+        _write(fd, buffer + bytes_written, static_cast<uint32_t>(chunksize)));
+#else
     ret = static_cast<int>(
         write(fd, buffer + bytes_written, static_cast<size_t>(chunksize)));
+#endif
 
     if (ret != -1) {
       bytes_written += ret;
     }
   }
-#endif
 
   if (ret == -1) {
     return Status::IOError(std::string("Error writing bytes from file: ") +

From 6f8e2873c3931ff279b05a5e517a87637eb13d05 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Mon, 13 Nov 2017 22:47:22 -0500
Subject: [PATCH 1241/1644] ARROW-1743: [Python] Avoid non-array writeable-flag
 check

This closes [ARROW-1743](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1743).

Author: Licht-T <licht-t@outlook.jp>

Closes #1260 from Licht-T/fix-non-array-writable-check and squashes the following commits:

8a104f7c [Licht-T] TST: Add test_categorical_row_index
555acdeb [Licht-T] BUG: Avoid the writeable-flag check for non-array index
---
 python/pyarrow/pandas_compat.py             | 2 +-
 python/pyarrow/tests/test_convert_pandas.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index db28ee09e1e73..41eaf0bac4525 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -397,7 +397,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1):
             col = table.column(i)
             col_pandas = col.to_pandas()
             values = col_pandas.values
-            if not values.flags.writeable:
+            if hasattr(values, 'flags') and not values.flags.writeable:
                 # ARROW-1054: in pandas 0.19.2, factorize will reject
                 # non-writeable arrays when calling MultiIndex.from_arrays
                 values = values.copy()
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index dabccac37c3d8..b9c3a12213243 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -198,6 +198,13 @@ def test_datetimetz_column_index(self):
         md = column_indexes['metadata']
         assert md['timezone'] == 'America/New_York'
 
+    def test_categorical_row_index(self):
+        df = pd.DataFrame({'a': [1, 2, 3], 'b': [1, 2, 3]})
+        df['a'] = df.a.astype('category')
+        df = df.set_index('a')
+
+        self._check_pandas_roundtrip(df, preserve_index=True)
+
     def test_float_no_nulls(self):
         data = {}
         fields = []

From 8f2d15256d6b3a9da569797f363518abf50d23f5 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 13 Nov 2017 22:53:58 -0500
Subject: [PATCH 1242/1644] ARROW-1802: [GLib] Support arrow-gpu

arrow-gpu isn't required. If `arrow-gpu.pc` isn't installed, GPU support is just ignored.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1313 from kou/glib-gpu and squashes the following commits:

b36d491a [Kouhei Sutou] [GLib] Support arrow-gpu
---
 c_glib/.gitignore                          |   1 +
 c_glib/Makefile.am                         |   1 +
 c_glib/arrow-glib/Makefile.am              |   8 +-
 c_glib/arrow-glib/buffer.h                 |  82 +-
 c_glib/arrow-glib/input-stream.h           | 136 +--
 c_glib/arrow-glib/meson.build              |  33 +-
 c_glib/arrow-glib/output-stream.h          |  47 +-
 c_glib/arrow-glib/readable.cpp             |   4 +-
 c_glib/arrow-glib/readable.hpp             |   1 +
 c_glib/arrow-gpu-glib/Makefile.am          | 109 +++
 c_glib/arrow-gpu-glib/arrow-gpu-glib.h     |  24 +
 c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp   |  24 +
 c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in |  28 +
 c_glib/arrow-gpu-glib/cuda.cpp             | 941 +++++++++++++++++++++
 c_glib/arrow-gpu-glib/cuda.h               | 181 ++++
 c_glib/arrow-gpu-glib/cuda.hpp             |  54 ++
 c_glib/arrow-gpu-glib/meson.build          |  80 ++
 c_glib/configure.ac                        |  26 +-
 c_glib/doc/Makefile.am                     |   2 +-
 c_glib/doc/reference/Makefile.am           |  11 +
 c_glib/doc/reference/arrow-glib-docs.sgml  |  10 +
 c_glib/doc/reference/meson.build           |  23 +-
 c_glib/meson.build                         |   9 +-
 c_glib/test/run-test.rb                    |   6 +
 c_glib/test/run-test.sh                    |  37 +-
 c_glib/test/test-gpu-cuda.rb               | 144 ++++
 26 files changed, 1744 insertions(+), 278 deletions(-)
 create mode 100644 c_glib/arrow-gpu-glib/Makefile.am
 create mode 100644 c_glib/arrow-gpu-glib/arrow-gpu-glib.h
 create mode 100644 c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp
 create mode 100644 c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in
 create mode 100644 c_glib/arrow-gpu-glib/cuda.cpp
 create mode 100644 c_glib/arrow-gpu-glib/cuda.h
 create mode 100644 c_glib/arrow-gpu-glib/cuda.hpp
 create mode 100644 c_glib/arrow-gpu-glib/meson.build
 create mode 100644 c_glib/test/test-gpu-cuda.rb

diff --git a/c_glib/.gitignore b/c_glib/.gitignore
index 03bb0fe61892e..2719147405f71 100644
--- a/c_glib/.gitignore
+++ b/c_glib/.gitignore
@@ -41,6 +41,7 @@ Makefile.in
 /arrow-glib/enums.h
 /arrow-glib/stamp-*
 /arrow-glib/*.pc
+/arrow-gpu-glib/*.pc
 /example/build
 /example/read-batch
 /example/read-stream
diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am
index 577b749fb38bc..4cc70e5a08870 100644
--- a/c_glib/Makefile.am
+++ b/c_glib/Makefile.am
@@ -19,6 +19,7 @@ ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
 
 SUBDIRS =					\
 	arrow-glib				\
+	arrow-gpu-glib				\
 	doc					\
 	example					\
 	tool
diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index bf68ec4910e77..5ecb1a6614268 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -203,20 +203,18 @@ pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA =				\
 	arrow-glib.pc
 
-# GObject Introspection
+if HAVE_INTROSPECTION
 -include $(INTROSPECTION_MAKEFILE)
 INTROSPECTION_GIRS =
 INTROSPECTION_SCANNER_ARGS =
 INTROSPECTION_COMPILER_ARGS =
 
-if HAVE_INTROSPECTION
 Arrow-1.0.gir: libarrow-glib.la
 Arrow_1_0_gir_PACKAGES =			\
-	gobject-2.0				\
 	gio-2.0
-Arrow_1_0_gir_EXPORT_PACKAGES = arrow
+Arrow_1_0_gir_EXPORT_PACKAGES =			\
+	arrow-glib
 Arrow_1_0_gir_INCLUDES =			\
-	GObject-2.0				\
 	Gio-2.0
 Arrow_1_0_gir_CFLAGS =				\
 	$(AM_CPPFLAGS)
diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h
index b3f3a2cdc5e9b..300bb4f4ea3ca 100644
--- a/c_glib/arrow-glib/buffer.h
+++ b/c_glib/arrow-glib/buffer.h
@@ -19,44 +19,21 @@
 
 #pragma once
 
-#include <glib-object.h>
+#include <arrow-glib/gobject-type.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_BUFFER \
-  (garrow_buffer_get_type())
-#define GARROW_BUFFER(obj) \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj), GARROW_TYPE_BUFFER, GArrowBuffer))
-#define GARROW_BUFFER_CLASS(klass) \
-  (G_TYPE_CHECK_CLASS_CAST((klass), GARROW_TYPE_BUFFER, GArrowBufferClass))
-#define GARROW_IS_BUFFER(obj) \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_BUFFER))
-#define GARROW_IS_BUFFER_CLASS(klass) \
-  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_BUFFER))
-#define GARROW_BUFFER_GET_CLASS(obj) \
-  (G_TYPE_INSTANCE_GET_CLASS((obj), GARROW_TYPE_BUFFER, GArrowBufferClass))
-
-typedef struct _GArrowBuffer         GArrowBuffer;
-typedef struct _GArrowBufferClass    GArrowBufferClass;
-
-/**
- * GArrowBuffer:
- *
- * It wraps `arrow::Buffer`.
- */
-struct _GArrowBuffer
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
+#define GARROW_TYPE_BUFFER (garrow_buffer_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowBuffer,
+                         garrow_buffer,
+                         GARROW,
+                         BUFFER,
+                         GObject)
 struct _GArrowBufferClass
 {
   GObjectClass parent_class;
 };
 
-GType          garrow_buffer_get_type     (void) G_GNUC_CONST;
-
 GArrowBuffer  *garrow_buffer_new          (const guint8 *data,
                                            gint64 size);
 gboolean       garrow_buffer_equal        (GArrowBuffer *buffer,
@@ -80,49 +57,16 @@ GArrowBuffer  *garrow_buffer_slice        (GArrowBuffer *buffer,
                                            gint64 size);
 
 
-#define GARROW_TYPE_MUTABLE_BUFFER              \
-  (garrow_mutable_buffer_get_type())
-#define GARROW_MUTABLE_BUFFER(obj)                              \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_MUTABLE_BUFFER,       \
-                              GArrowMutableBuffer))
-#define GARROW_MUTABLE_BUFFER_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_MUTABLE_BUFFER,  \
-                           GArrowMutableBufferClass))
-#define GARROW_IS_MUTABLE_BUFFER(obj)                                   \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_MUTABLE_BUFFER))
-#define GARROW_IS_MUTABLE_BUFFER_CLASS(klass)                           \
-  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_MUTABLE_BUFFER))
-#define GARROW_MUTABLE_BUFFER_GET_CLASS(obj)                    \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_MUTABLE_BUFFER,        \
-                             GArrowMutableBufferClass))
-
-typedef struct _GArrowMutableBuffer         GArrowMutableBuffer;
-#ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowMutableBufferClass    GArrowMutableBufferClass;
-#endif
-
-/**
- * GArrowMutableBuffer:
- *
- * It wraps `arrow::MutableBuffer`.
- */
-struct _GArrowMutableBuffer
-{
-  /*< private >*/
-  GArrowBuffer parent_instance;
-};
-
-#ifndef __GTK_DOC_IGNORE__
+#define GARROW_TYPE_MUTABLE_BUFFER (garrow_mutable_buffer_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowMutableBuffer,
+                         garrow_mutable_buffer,
+                         GARROW,
+                         MUTABLE_BUFFER,
+                         GArrowBuffer)
 struct _GArrowMutableBufferClass
 {
   GArrowBufferClass parent_class;
 };
-#endif
-
-GType garrow_mutable_buffer_get_type(void) G_GNUC_CONST;
 
 GArrowMutableBuffer *garrow_mutable_buffer_new  (guint8 *data,
                                                  gint64 size);
diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h
index 12c7ae700f79d..c2068d6ac0e41 100644
--- a/c_glib/arrow-glib/input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -26,98 +26,28 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_INPUT_STREAM                \
-  (garrow_input_stream_get_type())
-#define GARROW_INPUT_STREAM(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INPUT_STREAM, \
-                              GArrowInputStream))
-#define GARROW_INPUT_STREAM_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INPUT_STREAM,    \
-                           GArrowInputStreamClass))
-#define GARROW_IS_INPUT_STREAM(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INPUT_STREAM))
-#define GARROW_IS_INPUT_STREAM_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INPUT_STREAM))
-#define GARROW_INPUT_STREAM_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INPUT_STREAM,  \
-                             GArrowInputStreamClass))
-
-typedef struct _GArrowInputStream         GArrowInputStream;
-#ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowInputStreamClass    GArrowInputStreamClass;
-#endif
-
-/**
- * GArrowInputStream:
- *
- * It wraps `arrow::io::InputStream`.
- */
-struct _GArrowInputStream
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-#ifndef __GTK_DOC_IGNORE__
+#define GARROW_TYPE_INPUT_STREAM (garrow_input_stream_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInputStream,
+                         garrow_input_stream,
+                         GARROW,
+                         INPUT_STREAM,
+                         GObject)
 struct _GArrowInputStreamClass
 {
   GObjectClass parent_class;
 };
-#endif
-
-GType garrow_input_stream_get_type(void) G_GNUC_CONST;
-
 
 #define GARROW_TYPE_SEEKABLE_INPUT_STREAM       \
   (garrow_seekable_input_stream_get_type())
-#define GARROW_SEEKABLE_INPUT_STREAM(obj)                               \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                                    \
-                              GARROW_TYPE_SEEKABLE_INPUT_STREAM,        \
-                              GArrowSeekableInputStream))
-#define GARROW_SEEKABLE_INPUT_STREAM_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_SEEKABLE_INPUT_STREAM,   \
-                           GArrowSeekableInputStreamClass))
-#define GARROW_IS_SEEKABLE_INPUT_STREAM(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                                    \
-                              GARROW_TYPE_SEEKABLE_INPUT_STREAM))
-#define GARROW_IS_SEEKABLE_INPUT_STREAM_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_SEEKABLE_INPUT_STREAM))
-#define GARROW_SEEKABLE_INPUT_STREAM_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_SEEKABLE_INPUT_STREAM, \
-                             GArrowSeekableInputStreamClass))
-
-typedef struct _GArrowSeekableInputStream         GArrowSeekableInputStream;
-#ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowSeekableInputStreamClass    GArrowSeekableInputStreamClass;
-#endif
-
-/**
- * GArrowSeekableInputStream:
- *
- * It wraps `arrow::io::RandomAccessFile`.
- */
-struct _GArrowSeekableInputStream
-{
-  /*< private >*/
-  GArrowInputStream parent_instance;
-};
-
-#ifndef __GTK_DOC_IGNORE__
+G_DECLARE_DERIVABLE_TYPE(GArrowSeekableInputStream,
+                         garrow_seekable_input_stream,
+                         GARROW,
+                         SEEKABLE_INPUT_STREAM,
+                         GArrowInputStream)
 struct _GArrowSeekableInputStreamClass
 {
   GArrowInputStreamClass parent_class;
 };
-#endif
-
-GType garrow_seekable_input_stream_get_type(void) G_GNUC_CONST;
 
 guint64 garrow_seekable_input_stream_get_size(GArrowSeekableInputStream *input_stream,
                                               GError **error);
@@ -133,49 +63,15 @@ GArrowTensor *garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream
 
 #define GARROW_TYPE_BUFFER_INPUT_STREAM         \
   (garrow_buffer_input_stream_get_type())
-#define GARROW_BUFFER_INPUT_STREAM(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BUFFER_INPUT_STREAM,  \
-                              GArrowBufferInputStream))
-#define GARROW_BUFFER_INPUT_STREAM_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BUFFER_INPUT_STREAM,     \
-                           GArrowBufferInputStreamClass))
-#define GARROW_IS_BUFFER_INPUT_STREAM(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BUFFER_INPUT_STREAM))
-#define GARROW_IS_BUFFER_INPUT_STREAM_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BUFFER_INPUT_STREAM))
-#define GARROW_BUFFER_INPUT_STREAM_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BUFFER_INPUT_STREAM,   \
-                             GArrowBufferInputStreamClass))
-
-typedef struct _GArrowBufferInputStream         GArrowBufferInputStream;
-#ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowBufferInputStreamClass    GArrowBufferInputStreamClass;
-#endif
-
-/**
- * GArrowBufferInputStream:
- *
- * It wraps `arrow::io::BufferReader`.
- */
-struct _GArrowBufferInputStream
-{
-  /*< private >*/
-  GArrowSeekableInputStream parent_instance;
-};
-
-#ifndef __GTK_DOC_IGNORE__
+G_DECLARE_DERIVABLE_TYPE(GArrowBufferInputStream,
+                         garrow_buffer_input_stream,
+                         GARROW,
+                         BUFFER_INPUT_STREAM,
+                         GArrowSeekableInputStream)
 struct _GArrowBufferInputStreamClass
 {
   GArrowSeekableInputStreamClass parent_class;
 };
-#endif
-
-GType garrow_buffer_input_stream_get_type(void) G_GNUC_CONST;
 
 GArrowBufferInputStream *garrow_buffer_input_stream_new(GArrowBuffer *buffer);
 
diff --git a/c_glib/arrow-glib/meson.build b/c_glib/arrow-glib/meson.build
index 464a002e78b0c..aeec4172d6284 100644
--- a/c_glib/arrow-glib/meson.build
+++ b/c_glib/arrow-glib/meson.build
@@ -179,22 +179,23 @@ pkgconfig.generate(filebase: meson.project_name(),
                    name: 'Apache Arrow GLib',
                    description: 'C API for Apache Arrow based on GLib',
                    version: version,
-                   requires: ['gobject-2.0', 'arrow'],
+                   requires: ['gio-2.0', 'arrow'],
                    libraries: [libarrow_glib],
                    subdirs: ['arrow-glib'])
 
-gnome.generate_gir(libarrow_glib,
-                   sources: sources + c_headers + enums,
-                   namespace: 'Arrow',
-                   nsversion: api_version,
-                   identifier_prefix: 'GArrow',
-                   symbol_prefix: 'garrow',
-                   export_packages: 'arrow-glib',
-                   includes: [
-                     'GObject-2.0',
-                     'Gio-2.0',
-                   ],
-                   install: true,
-                   extra_args: [
-                     '--warn-all',
-                   ])
+arrow_glib_gir = gnome.generate_gir(libarrow_glib,
+                                    sources: sources + c_headers + enums,
+                                    namespace: 'Arrow',
+                                    nsversion: api_version,
+                                    identifier_prefix: 'GArrow',
+                                    symbol_prefix: 'garrow',
+                                    export_packages: 'arrow-glib',
+                                    includes: [
+                                      'GObject-2.0',
+                                      'Gio-2.0',
+                                    ],
+                                    install: true,
+                                    extra_args: [
+                                      '--warn-all',
+                                    ])
+arrow_glib_gir_dependency = declare_dependency(sources: arrow_glib_gir)
diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h
index e42ebcde47d6b..195a97ac9f053 100644
--- a/c_glib/arrow-glib/output-stream.h
+++ b/c_glib/arrow-glib/output-stream.h
@@ -26,51 +26,16 @@
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_OUTPUT_STREAM               \
-  (garrow_output_stream_get_type())
-#define GARROW_OUTPUT_STREAM(obj)                               \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_OUTPUT_STREAM,        \
-                              GArrowOutputStream))
-#define GARROW_OUTPUT_STREAM_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_OUTPUT_STREAM,   \
-                           GArrowOutputStreamClass))
-#define GARROW_IS_OUTPUT_STREAM(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_OUTPUT_STREAM))
-#define GARROW_IS_OUTPUT_STREAM_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_OUTPUT_STREAM))
-#define GARROW_OUTPUT_STREAM_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_OUTPUT_STREAM, \
-                             GArrowOutputStreamClass))
-
-typedef struct _GArrowOutputStream          GArrowOutputStream;
-#ifndef __GTK_DOC_IGNORE__
-typedef struct _GArrowOutputStreamClass     GArrowOutputStreamClass;
-#endif
-
-/**
- * GArrowOutputStream:
- *
- * It wraps `arrow::io::OutputStream`.
- */
-struct _GArrowOutputStream
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-#ifndef __GTK_DOC_IGNORE__
+#define GARROW_TYPE_OUTPUT_STREAM (garrow_output_stream_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowOutputStream,
+                         garrow_output_stream,
+                         GARROW,
+                         OUTPUT_STREAM,
+                         GObject)
 struct _GArrowOutputStreamClass
 {
   GObjectClass parent_class;
 };
-#endif
-
-GType garrow_output_stream_get_type(void) G_GNUC_CONST;
 
 gint64 garrow_output_stream_write_tensor(GArrowOutputStream *stream,
                                          GArrowTensor *tensor,
diff --git a/c_glib/arrow-glib/readable.cpp b/c_glib/arrow-glib/readable.cpp
index 6a9023e6cddf0..33f98d98c88a4 100644
--- a/c_glib/arrow-glib/readable.cpp
+++ b/c_glib/arrow-glib/readable.cpp
@@ -45,6 +45,7 @@ G_DEFINE_INTERFACE(GArrowReadable,
 static void
 garrow_readable_default_init (GArrowReadableInterface *iface)
 {
+  iface->new_raw = garrow_buffer_new_raw;
 }
 
 /**
@@ -66,7 +67,8 @@ garrow_readable_read(GArrowReadable *readable,
   std::shared_ptr<arrow::Buffer> arrow_buffer;
   auto status = arrow_readable->Read(n_bytes, &arrow_buffer);
   if (garrow_error_check(error, status, "[io][readable][read]")) {
-    return garrow_buffer_new_raw(&arrow_buffer);
+    auto *iface = GARROW_READABLE_GET_IFACE(readable);
+    return iface->new_raw(&arrow_buffer);
   } else {
     return NULL;
   }
diff --git a/c_glib/arrow-glib/readable.hpp b/c_glib/arrow-glib/readable.hpp
index c241c77aa0329..ce7770103aa1a 100644
--- a/c_glib/arrow-glib/readable.hpp
+++ b/c_glib/arrow-glib/readable.hpp
@@ -32,6 +32,7 @@ struct _GArrowReadableInterface
 {
   GTypeInterface parent_iface;
 
+  GArrowBuffer *(*new_raw)(std::shared_ptr<arrow::Buffer> *arrow_buffer);
   std::shared_ptr<arrow::io::Readable> (*get_raw)(GArrowReadable *file);
 };
 
diff --git a/c_glib/arrow-gpu-glib/Makefile.am b/c_glib/arrow-gpu-glib/Makefile.am
new file mode 100644
index 0000000000000..ec9615987ee5e
--- /dev/null
+++ b/c_glib/arrow-gpu-glib/Makefile.am
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+CLEANFILES =
+
+EXTRA_DIST =					\
+	meson.build
+
+AM_CPPFLAGS =					\
+	-I$(top_builddir)			\
+	-I$(top_srcdir)
+
+if HAVE_ARROW_GPU
+lib_LTLIBRARIES =				\
+	libarrow-gpu-glib.la
+
+libarrow_gpu_glib_la_CXXFLAGS =			\
+	$(GLIB_CFLAGS)				\
+	$(ARROW_CFLAGS)				\
+	$(ARROW_GPU_CFLAGS)			\
+	$(GARROW_CXXFLAGS)
+
+libarrow_gpu_glib_la_LIBADD =			\
+	$(GLIB_LIBS)				\
+	$(ARROW_LIBS)				\
+	$(ARROW_GPU_LIBS)			\
+	../arrow-glib/libarrow-glib.la
+
+libarrow_gpu_glib_la_headers =			\
+	arrow-gpu-glib.h			\
+	cuda.h
+
+libarrow_gpu_glib_la_sources =			\
+	cuda.cpp				\
+	$(libarrow_gpu_glib_la_headers)
+
+libarrow_gpu_glib_la_cpp_headers =		\
+	arrow-gpu-glib.hpp			\
+	cuda.hpp
+
+libarrow_gpu_glib_la_SOURCES =			\
+	$(libarrow_gpu_glib_la_sources)		\
+	$(libarrow_gpu_glib_la_cpp_headers)
+
+arrow_gpu_glib_includedir =			\
+	$(includedir)/arrow-gpu-glib
+arrow_gpu_glib_include_HEADERS =		\
+	$(libarrow_gpu_glib_la_headers)		\
+	$(libarrow_gpu_glib_la_cpp_headers)
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA =				\
+	arrow-gpu-glib.pc
+
+if HAVE_INTROSPECTION
+-include $(INTROSPECTION_MAKEFILE)
+INTROSPECTION_GIRS =
+INTROSPECTION_SCANNER_ARGS =
+INTROSPECTION_SCANNER_ENV =			\
+	PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$${PKG_CONFIG_PATH}
+INTROSPECTION_COMPILER_ARGS =			\
+	--includedir=$(abs_builddir)/../arrow-glib
+
+ArrowGPU-1.0.gir: libarrow-gpu-glib.la
+ArrowGPU_1_0_gir_PACKAGES =			\
+	arrow-glib
+ArrowGPU_1_0_gir_EXPORT_PACKAGES =		\
+	arrow-gpu-glib
+ArrowGPU_1_0_gir_INCLUDES =			\
+	Arrow-1.0
+ArrowGPU_1_0_gir_CFLAGS =			\
+	$(AM_CPPFLAGS)
+ArrowGPU_1_0_gir_LIBS =					\
+	$(abs_builddir)/../arrow-glib/libarrow-glib.la	\
+	libarrow-gpu-glib.la
+ArrowGPU_1_0_gir_FILES =			\
+	$(libarrow_gpu_glib_la_sources)
+ArrowGPU_1_0_gir_SCANNERFLAGS =					\
+	--warn-all						\
+	--add-include-path=$(abs_builddir)/../arrow-glib	\
+	--identifier-prefix=GArrowGPU				\
+	--symbol-prefix=garrow_gpu
+INTROSPECTION_GIRS += ArrowGPU-1.0.gir
+
+girdir = $(datadir)/gir-1.0
+gir_DATA = $(INTROSPECTION_GIRS)
+
+typelibdir = $(libdir)/girepository-1.0
+typelib_DATA = $(INTROSPECTION_GIRS:.gir=.typelib)
+
+CLEANFILES +=					\
+	$(gir_DATA)				\
+	$(typelib_DATA)
+endif
+endif
diff --git a/c_glib/arrow-gpu-glib/arrow-gpu-glib.h b/c_glib/arrow-gpu-glib/arrow-gpu-glib.h
new file mode 100644
index 0000000000000..1538c9a1865ac
--- /dev/null
+++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.h
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/arrow-glib.h>
+
+#include <arrow-gpu-glib/cuda.h>
diff --git a/c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp b/c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp
new file mode 100644
index 0000000000000..92017d8b67aab
--- /dev/null
+++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/arrow-glib.hpp>
+
+#include <arrow-gpu-glib/cuda.hpp>
diff --git a/c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in b/c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in
new file mode 100644
index 0000000000000..38a6bae1a1298
--- /dev/null
+++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Apache Arrow GPU GLib
+Description: C API for Apache Arrow GPU based on GLib
+Version: @VERSION@
+Libs: -L${libdir} -larrow-gpu-glib
+Cflags: -I${includedir}
+Requires: arrow-glib
diff --git a/c_glib/arrow-gpu-glib/cuda.cpp b/c_glib/arrow-gpu-glib/cuda.cpp
new file mode 100644
index 0000000000000..c2a9af54dda94
--- /dev/null
+++ b/c_glib/arrow-gpu-glib/cuda.cpp
@@ -0,0 +1,941 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/buffer.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/input-stream.hpp>
+#include <arrow-glib/output-stream.hpp>
+#include <arrow-glib/readable.hpp>
+#include <arrow-glib/record-batch.hpp>
+#include <arrow-glib/schema.hpp>
+
+#include <arrow-gpu-glib/cuda.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: cuda
+ * @section_id: cuda-classes
+ * @title: CUDA related classes
+ * @include: arrow-gpu-glib/arrow-gpu-glib.h
+ *
+ * The following classes provide CUDA support for Apache Arrow data.
+ *
+ * #GArrowGPUCUDADeviceManager is the starting point. You need at
+ * least one #GArrowGPUCUDAContext to process Apache Arrow data on
+ * NVIDIA GPU.
+ *
+ * #GArrowGPUCUDAContext is a class to keep context for one GPU. You
+ * need to create #GArrowGPUCUDAContext for each GPU that you want to
+ * use. You can create #GArrowGPUCUDAContext by
+ * garrow_gpu_cuda_device_manager_get_context().
+ *
+ * #GArrowGPUCUDABuffer is a class for data on GPU. You can copy data
+ * on GPU to/from CPU by garrow_gpu_cuda_buffer_copy_to_host() and
+ * garrow_gpu_cuda_buffer_copy_from_host(). You can share data on GPU
+ * with other processes by garrow_gpu_cuda_buffer_export() and
+ * garrow_gpu_cuda_buffer_new_ipc().
+ *
+ * #GArrowGPUCUDAHostBuffer is a class for data on CPU that is
+ * directly accessible from GPU.
+ *
+ * #GArrowGPUCUDAIPCMemoryHandle is a class to share data on GPU with
+ * other processes. You can export your data on GPU to other processes
+ * by garrow_gpu_cuda_buffer_export() and
+ * garrow_gpu_cuda_ipc_memory_handle_new(). You can import other
+ * process data on GPU by garrow_gpu_cuda_ipc_memory_handle_new() and
+ * garrow_gpu_cuda_buffer_new_ipc().
+ *
+ * #GArrowGPUCUDABufferInputStream is a class to read data in
+ * #GArrowGPUCUDABuffer.
+ *
+ * #GArrowGPUCUDABufferOutputStream is a class to write data into
+ * #GArrowGPUCUDABuffer.
+ */
+
+G_DEFINE_TYPE(GArrowGPUCUDADeviceManager,
+              garrow_gpu_cuda_device_manager,
+              G_TYPE_OBJECT)
+
+static void
+garrow_gpu_cuda_device_manager_init(GArrowGPUCUDADeviceManager *object)
+{
+}
+
+static void
+garrow_gpu_cuda_device_manager_class_init(GArrowGPUCUDADeviceManagerClass *klass)
+{
+}
+
+/**
+ * garrow_gpu_cuda_device_manager_new:
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: A newly created #GArrowGPUCUDADeviceManager on success,
+ *   %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDADeviceManager *
+garrow_gpu_cuda_device_manager_new(GError **error)
+{
+  arrow::gpu::CudaDeviceManager *manager;
+  auto status = arrow::gpu::CudaDeviceManager::GetInstance(&manager);
+  if (garrow_error_check(error, status, "[gpu][cuda][device-manager][new]")) {
+    auto manager = g_object_new(GARROW_GPU_TYPE_CUDA_DEVICE_MANAGER,
+                                NULL);
+    return GARROW_GPU_CUDA_DEVICE_MANAGER(manager);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_gpu_cuda_device_manager_get_context:
+ * @manager: A #GArrowGPUCUDADeviceManager.
+ * @gpu_number: A GPU device number for the target context.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A newly created #GArrowGPUCUDAContext on
+ *   success, %NULL on error. Contexts for the same GPU device number
+ *   share the same data internally.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDAContext *
+garrow_gpu_cuda_device_manager_get_context(GArrowGPUCUDADeviceManager *manager,
+                                           gint gpu_number,
+                                           GError **error)
+{
+  arrow::gpu::CudaDeviceManager *arrow_manager;
+  arrow::gpu::CudaDeviceManager::GetInstance(&arrow_manager);
+  std::shared_ptr<arrow::gpu::CudaContext> context;
+  auto status = arrow_manager->GetContext(gpu_number, &context);
+  if (garrow_error_check(error, status,
+                         "[gpu][cuda][device-manager][get-context]]")) {
+    return garrow_gpu_cuda_context_new_raw(&context);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_gpu_cuda_device_manager_get_n_devices:
+ * @manager: A #GArrowGPUCUDADeviceManager.
+ *
+ * Returns: The number of GPU devices.
+ *
+ * Since: 0.8.0
+ */
+gsize
+garrow_gpu_cuda_device_manager_get_n_devices(GArrowGPUCUDADeviceManager *manager)
+{
+  arrow::gpu::CudaDeviceManager *arrow_manager;
+  arrow::gpu::CudaDeviceManager::GetInstance(&arrow_manager);
+  return arrow_manager->num_devices();
+}
+
+
+typedef struct GArrowGPUCUDAContextPrivate_ {
+  std::shared_ptr<arrow::gpu::CudaContext> context;
+} GArrowGPUCUDAContextPrivate;
+
+enum {
+  PROP_CONTEXT = 1
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowGPUCUDAContext,
+                           garrow_gpu_cuda_context,
+                           G_TYPE_OBJECT)
+
+#define GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object)     \
+  static_cast<GArrowGPUCUDAContextPrivate *>(           \
+    garrow_gpu_cuda_context_get_instance_private(       \
+      GARROW_GPU_CUDA_CONTEXT(object)))
+
+static void
+garrow_gpu_cuda_context_finalize(GObject *object)
+{
+  auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object);
+
+  priv->context = nullptr;
+
+  G_OBJECT_CLASS(garrow_gpu_cuda_context_parent_class)->finalize(object);
+}
+
+static void
+garrow_gpu_cuda_context_set_property(GObject *object,
+                                     guint prop_id,
+                                     const GValue *value,
+                                     GParamSpec *pspec)
+{
+  auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_CONTEXT:
+    priv->context =
+      *static_cast<std::shared_ptr<arrow::gpu::CudaContext> *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_gpu_cuda_context_get_property(GObject *object,
+                                     guint prop_id,
+                                     GValue *value,
+                                     GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_gpu_cuda_context_init(GArrowGPUCUDAContext *object)
+{
+}
+
+static void
+garrow_gpu_cuda_context_class_init(GArrowGPUCUDAContextClass *klass)
+{
+  GParamSpec *spec;
+
+  auto gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_gpu_cuda_context_finalize;
+  gobject_class->set_property = garrow_gpu_cuda_context_set_property;
+  gobject_class->get_property = garrow_gpu_cuda_context_get_property;
+
+  /**
+   * GArrowGPUCUDAContext:context:
+   *
+   * Since: 0.8.0
+   */
+  spec = g_param_spec_pointer("context",
+                              "Context",
+                              "The raw std::shared_ptr<arrow::gpu::CudaContext> *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_CONTEXT, spec);
+}
+
+/**
+ * garrow_gpu_cuda_context_get_allocated_size:
+ * @context: A #GArrowGPUCUDAContext.
+ *
+ * Returns: The allocated memory by this context in bytes.
+ *
+ * Since: 0.8.0
+ */
+gint64
+garrow_gpu_cuda_context_get_allocated_size(GArrowGPUCUDAContext *context)
+{
+  auto arrow_context = garrow_gpu_cuda_context_get_raw(context);
+  return arrow_context->bytes_allocated();
+}
+
+
+G_DEFINE_TYPE(GArrowGPUCUDABuffer,
+              garrow_gpu_cuda_buffer,
+              GARROW_TYPE_BUFFER)
+
+static void
+garrow_gpu_cuda_buffer_init(GArrowGPUCUDABuffer *object)
+{
+}
+
+static void
+garrow_gpu_cuda_buffer_class_init(GArrowGPUCUDABufferClass *klass)
+{
+}
+
+/**
+ * garrow_gpu_cuda_buffer_new:
+ * @context: A #GArrowGPUCUDAContext.
+ * @size: The number of bytes to be allocated on GPU device for this context.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on
+ *   success, %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDABuffer *
+garrow_gpu_cuda_buffer_new(GArrowGPUCUDAContext *context,
+                           gint64 size,
+                           GError **error)
+{
+  auto arrow_context = garrow_gpu_cuda_context_get_raw(context);
+  std::shared_ptr<arrow::gpu::CudaBuffer> arrow_buffer;
+  auto status = arrow_context->Allocate(size, &arrow_buffer);
+  if (garrow_error_check(error, status, "[gpu][cuda][buffer][new]")) {
+    return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_gpu_cuda_buffer_new_ipc:
+ * @context: A #GArrowGPUCUDAContext.
+ * @handle: A #GArrowGPUCUDAIPCMemoryHandle to be communicated.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on
+ *   success, %NULL on error. The buffer has data from the IPC target.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDABuffer *
+garrow_gpu_cuda_buffer_new_ipc(GArrowGPUCUDAContext *context,
+                               GArrowGPUCUDAIPCMemoryHandle *handle,
+                               GError **error)
+{
+  auto arrow_context = garrow_gpu_cuda_context_get_raw(context);
+  auto arrow_handle = garrow_gpu_cuda_ipc_memory_handle_get_raw(handle);
+  std::shared_ptr<arrow::gpu::CudaBuffer> arrow_buffer;
+  auto status = arrow_context->OpenIpcBuffer(*arrow_handle, &arrow_buffer);
+  if (garrow_error_check(error, status,
+                         "[gpu][cuda][buffer][new-ipc]")) {
+    return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_gpu_cuda_buffer_new_record_batch:
+ * @context: A #GArrowGPUCUDAContext.
+ * @record_batch: A #GArrowRecordBatch to be serialized.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on
+ *   success, %NULL on error. The buffer has serialized record batch
+ *   data.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDABuffer *
+garrow_gpu_cuda_buffer_new_record_batch(GArrowGPUCUDAContext *context,
+                                        GArrowRecordBatch *record_batch,
+                                        GError **error)
+{
+  auto arrow_context = garrow_gpu_cuda_context_get_raw(context);
+  auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  std::shared_ptr<arrow::gpu::CudaBuffer> arrow_buffer;
+  auto status = arrow::gpu::SerializeRecordBatch(*arrow_record_batch,
+                                                 arrow_context.get(),
+                                                 &arrow_buffer);
+  if (garrow_error_check(error, status,
+                         "[gpu][cuda][buffer][new-record-batch]")) {
+    return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_gpu_cuda_buffer_copy_to_host:
+ * @buffer: A #GArrowGPUCUDABuffer.
+ * @position: The offset of memory on GPU device to be copied.
+ * @size: The size of memory on GPU device to be copied in bytes.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A #GBytes that have copied memory on CPU
+ *   host on success, %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GBytes *
+garrow_gpu_cuda_buffer_copy_to_host(GArrowGPUCUDABuffer *buffer,
+                                    gint64 position,
+                                    gint64 size,
+                                    GError **error)
+{
+  auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer);
+  auto data = static_cast<uint8_t *>(g_malloc(size));
+  auto status = arrow_buffer->CopyToHost(position, size, data);
+  if (garrow_error_check(error, status, "[gpu][cuda][buffer][copy-to-host]")) {
+    return g_bytes_new_take(data, size);
+  } else {
+    g_free(data);
+    return NULL;
+  }
+}
+
+/**
+ * garrow_gpu_cuda_buffer_copy_from_host:
+ * @buffer: A #GArrowGPUCUDABuffer.
+ * @data: (array length=size): Data on CPU host to be copied.
+ * @size: The size of data on CPU host to be copied in bytes.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_gpu_cuda_buffer_copy_from_host(GArrowGPUCUDABuffer *buffer,
+                                      const guint8 *data,
+                                      gint64 size,
+                                      GError **error)
+{
+  auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer);
+  auto status = arrow_buffer->CopyFromHost(0, data, size);
+  return garrow_error_check(error,
+                            status,
+                            "[gpu][cuda][buffer][copy-from-host]");
+}
+
+/**
+ * garrow_gpu_cuda_buffer_export:
+ * @buffer: A #GArrowGPUCUDABuffer.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A newly created
+ *   #GArrowGPUCUDAIPCMemoryHandle to handle the exported buffer on
+ *   success, %NULL on error
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDAIPCMemoryHandle *
+garrow_gpu_cuda_buffer_export(GArrowGPUCUDABuffer *buffer, GError **error)
+{
+  auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer);
+  std::unique_ptr<arrow::gpu::CudaIpcMemHandle> arrow_handle;
+  auto status = arrow_buffer->ExportForIpc(&arrow_handle);
+  if (garrow_error_check(error, status, "[gpu][cuda][buffer][export-for-ipc]")) {
+    return garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow_handle.release());
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_gpu_cuda_buffer_get_context:
+ * @buffer: A #GArrowGPUCUDABuffer.
+ *
+ * Returns: (transfer full): A newly created #GArrowGPUCUDAContext for the
+ *   buffer. Contexts for the same buffer share the same data internally.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDAContext *
+garrow_gpu_cuda_buffer_get_context(GArrowGPUCUDABuffer *buffer)
+{
+  auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer);
+  auto arrow_context = arrow_buffer->context();
+  return garrow_gpu_cuda_context_new_raw(&arrow_context);
+}
+
+/**
+ * garrow_gpu_cuda_buffer_read_record_batch:
+ * @buffer: A #GArrowGPUCUDABuffer.
+ * @schema: A #GArrowSchema for record batch.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A newly created #GArrowRecordBatch on
+ *   success, %NULL on error. The record batch data is located on GPU.
+ *
+ * Since: 0.8.0
+ */
+GArrowRecordBatch *
+garrow_gpu_cuda_buffer_read_record_batch(GArrowGPUCUDABuffer *buffer,
+                                         GArrowSchema *schema,
+                                         GError **error)
+{
+  auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer);
+  auto arrow_schema = garrow_schema_get_raw(schema);
+  auto pool = arrow::default_memory_pool();
+  std::shared_ptr<arrow::RecordBatch> arrow_record_batch;
+  auto status = arrow::gpu::ReadRecordBatch(arrow_schema,
+                                            arrow_buffer,
+                                            pool,
+                                            &arrow_record_batch);
+  if (garrow_error_check(error, status,
+                         "[gpu][cuda][buffer][read-record-batch]")) {
+    return garrow_record_batch_new_raw(&arrow_record_batch);
+  } else {
+    return NULL;
+  }
+}
+
+
+G_DEFINE_TYPE(GArrowGPUCUDAHostBuffer,
+              garrow_gpu_cuda_host_buffer,
+              GARROW_TYPE_MUTABLE_BUFFER)
+
+static void
+garrow_gpu_cuda_host_buffer_init(GArrowGPUCUDAHostBuffer *object)
+{
+}
+
+static void
+garrow_gpu_cuda_host_buffer_class_init(GArrowGPUCUDAHostBufferClass *klass)
+{
+}
+
+/**
+ * garrow_gpu_cuda_host_buffer_new:
+ * @size: The number of bytes to be allocated on CPU host.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: A newly created #GArrowGPUCUDAHostBuffer on success,
+ *   %NULL on error. The allocated memory is accessible from GPU
+ *   device for the @context.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDAHostBuffer *
+garrow_gpu_cuda_host_buffer_new(gint64 size, GError **error)
+{
+  arrow::gpu::CudaDeviceManager *manager;
+  auto status = arrow::gpu::CudaDeviceManager::GetInstance(&manager);
+  std::shared_ptr<arrow::gpu::CudaHostBuffer> arrow_buffer;
+  status = manager->AllocateHost(size, &arrow_buffer);
+  if (garrow_error_check(error, status, "[gpu][cuda][host-buffer][new]")) {
+    return garrow_gpu_cuda_host_buffer_new_raw(&arrow_buffer);
+  } else {
+    return NULL;
+  }
+}
+
+
+typedef struct GArrowGPUCUDAIPCMemoryHandlePrivate_ {
+  arrow::gpu::CudaIpcMemHandle *ipc_memory_handle;
+} GArrowGPUCUDAIPCMemoryHandlePrivate;
+
+enum {
+  PROP_IPC_MEMORY_HANDLE = 1
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowGPUCUDAIPCMemoryHandle,
+                           garrow_gpu_cuda_ipc_memory_handle,
+                           G_TYPE_OBJECT)
+
+#define GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object)   \
+  static_cast<GArrowGPUCUDAIPCMemoryHandlePrivate *>(           \
+    garrow_gpu_cuda_ipc_memory_handle_get_instance_private(     \
+      GARROW_GPU_CUDA_IPC_MEMORY_HANDLE(object)))
+
+static void
+garrow_gpu_cuda_ipc_memory_handle_finalize(GObject *object)
+{
+  auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object);
+
+  delete priv->ipc_memory_handle;
+
+  G_OBJECT_CLASS(garrow_gpu_cuda_ipc_memory_handle_parent_class)->finalize(object);
+}
+
+static void
+garrow_gpu_cuda_ipc_memory_handle_set_property(GObject *object,
+                                               guint prop_id,
+                                               const GValue *value,
+                                               GParamSpec *pspec)
+{
+  auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_IPC_MEMORY_HANDLE:
+    priv->ipc_memory_handle =
+      static_cast<arrow::gpu::CudaIpcMemHandle *>(g_value_get_pointer(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_gpu_cuda_ipc_memory_handle_get_property(GObject *object,
+                                               guint prop_id,
+                                               GValue *value,
+                                               GParamSpec *pspec)
+{
+  switch (prop_id) {
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_gpu_cuda_ipc_memory_handle_init(GArrowGPUCUDAIPCMemoryHandle *object)
+{
+}
+
+static void
+garrow_gpu_cuda_ipc_memory_handle_class_init(GArrowGPUCUDAIPCMemoryHandleClass *klass)
+{
+  GParamSpec *spec;
+
+  auto gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->finalize     = garrow_gpu_cuda_ipc_memory_handle_finalize;
+  gobject_class->set_property = garrow_gpu_cuda_ipc_memory_handle_set_property;
+  gobject_class->get_property = garrow_gpu_cuda_ipc_memory_handle_get_property;
+
+  /**
+   * GArrowGPUCUDAIPCMemoryHandle:ipc-memory-handle:
+   *
+   * Since: 0.8.0
+   */
+  spec = g_param_spec_pointer("ipc-memory-handle",
+                              "IPC Memory Handle",
+                              "The raw arrow::gpu::CudaIpcMemHandle *",
+                              static_cast<GParamFlags>(G_PARAM_WRITABLE |
+                                                       G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_IPC_MEMORY_HANDLE, spec);
+}
+
+/**
+ * garrow_gpu_cuda_ipc_memory_handle_new:
+ * @data: (array length=size): A serialized #GArrowGPUCUDAIPCMemoryHandle.
+ * @size: The size of data.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A newly created #GArrowGPUCUDAIPCMemoryHandle
+ *   on success, %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDAIPCMemoryHandle *
+garrow_gpu_cuda_ipc_memory_handle_new(const guint8 *data,
+                                      gsize size,
+                                      GError **error)
+{
+  std::unique_ptr<arrow::gpu::CudaIpcMemHandle> arrow_handle;
+  auto status = arrow::gpu::CudaIpcMemHandle::FromBuffer(data, &arrow_handle);
+  if (garrow_error_check(error, status,
+                         "[gpu][cuda][ipc-memory-handle][new]")) {
+    return garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow_handle.release());
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_gpu_cuda_ipc_memory_handle_serialize:
+ * @handle: A #GArrowGPUCUDAIPCMemoryHandle.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A newly created #GArrowBuffer on success,
+ *   %NULL on error. The buffer has serialized @handle. The serialized
+ *   @handle can be deserialized by garrow_gpu_cuda_ipc_memory_handle_new()
+ *   in other process.
+ *
+ * Since: 0.8.0
+ */
+GArrowBuffer *
+garrow_gpu_cuda_ipc_memory_handle_serialize(GArrowGPUCUDAIPCMemoryHandle *handle,
+                                            GError **error)
+{
+  auto arrow_handle = garrow_gpu_cuda_ipc_memory_handle_get_raw(handle);
+  std::shared_ptr<arrow::Buffer> arrow_buffer;
+  auto status = arrow_handle->Serialize(arrow::default_memory_pool(),
+                                        &arrow_buffer);
+  if (garrow_error_check(error, status,
+                         "[gpu][cuda][ipc-memory-handle][serialize]")) {
+    return garrow_buffer_new_raw(&arrow_buffer);
+  } else {
+    return NULL;
+  }
+}
+
+GArrowBuffer *
+garrow_gpu_cuda_buffer_input_stream_new_raw_readable_interface(std::shared_ptr<arrow::Buffer> *arrow_buffer)
+{
+  auto buffer = GARROW_BUFFER(g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER,
+                                           "buffer", arrow_buffer,
+                                           NULL));
+  return buffer;
+}
+
+static std::shared_ptr<arrow::io::Readable>
+garrow_gpu_cuda_buffer_input_stream_get_raw_readable_interface(GArrowReadable *readable)
+{
+  auto input_stream = GARROW_INPUT_STREAM(readable);
+  auto arrow_input_stream = garrow_input_stream_get_raw(input_stream);
+  return arrow_input_stream;
+}
+
+static void
+garrow_gpu_cuda_buffer_input_stream_readable_interface_init(GArrowReadableInterface *iface)
+{
+  iface->new_raw =
+    garrow_gpu_cuda_buffer_input_stream_new_raw_readable_interface;
+  iface->get_raw =
+    garrow_gpu_cuda_buffer_input_stream_get_raw_readable_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(
+  GArrowGPUCUDABufferInputStream,
+  garrow_gpu_cuda_buffer_input_stream,
+  GARROW_TYPE_BUFFER_INPUT_STREAM,
+  G_IMPLEMENT_INTERFACE(
+    GARROW_TYPE_READABLE,
+    garrow_gpu_cuda_buffer_input_stream_readable_interface_init))
+
+static void
+garrow_gpu_cuda_buffer_input_stream_init(GArrowGPUCUDABufferInputStream *object)
+{
+}
+
+static void
+garrow_gpu_cuda_buffer_input_stream_class_init(GArrowGPUCUDABufferInputStreamClass *klass)
+{
+}
+
+/**
+ * garrow_gpu_cuda_buffer_input_stream_new:
+ * @buffer: A #GArrowGPUCUDABuffer.
+ *
+ * Returns: (transfer full): A newly created
+ *   #GArrowGPUCUDABufferInputStream.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDABufferInputStream *
+garrow_gpu_cuda_buffer_input_stream_new(GArrowGPUCUDABuffer *buffer)
+{
+  auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer);
+  auto arrow_reader =
+    std::make_shared<arrow::gpu::CudaBufferReader>(arrow_buffer);
+  return garrow_gpu_cuda_buffer_input_stream_new_raw(&arrow_reader);
+}
+
+
+G_DEFINE_TYPE(GArrowGPUCUDABufferOutputStream,
+              garrow_gpu_cuda_buffer_output_stream,
+              GARROW_TYPE_OUTPUT_STREAM)
+
+static void
+garrow_gpu_cuda_buffer_output_stream_init(GArrowGPUCUDABufferOutputStream *object)
+{
+}
+
+static void
+garrow_gpu_cuda_buffer_output_stream_class_init(GArrowGPUCUDABufferOutputStreamClass *klass)
+{
+}
+
+/**
+ * garrow_gpu_cuda_buffer_output_stream_new:
+ * @buffer: A #GArrowGPUCUDABuffer.
+ *
+ * Returns: (transfer full): A newly created
+ *   #GArrowGPUCUDABufferOutputStream.
+ *
+ * Since: 0.8.0
+ */
+GArrowGPUCUDABufferOutputStream *
+garrow_gpu_cuda_buffer_output_stream_new(GArrowGPUCUDABuffer *buffer)
+{
+  auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer);
+  auto arrow_writer =
+    std::make_shared<arrow::gpu::CudaBufferWriter>(arrow_buffer);
+  return garrow_gpu_cuda_buffer_output_stream_new_raw(&arrow_writer);
+}
+
+/**
+ * garrow_gpu_cuda_buffer_output_stream_set_buffer_size:
+ * @stream: A #GArrowGPUCUDABufferOutputStream.
+ * @size: A size of CPU buffer in bytes.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Sets CPU buffer size. to limit `cudaMemcpy()` calls. If CPU buffer
+ * size is `0`, buffering is disabled.
+ *
+ * The default is `0`.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_gpu_cuda_buffer_output_stream_set_buffer_size(GArrowGPUCUDABufferOutputStream *stream,
+                                                     gint64 size,
+                                                     GError **error)
+{
+  auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream);
+  auto status = arrow_stream->SetBufferSize(size);
+  return garrow_error_check(error,
+                            status,
+                            "[gpu][cuda][buffer-output-stream][set-buffer-size]");
+}
+
+/**
+ * garrow_gpu_cuda_buffer_output_stream_get_buffer_size:
+ * @stream: A #GArrowGPUCUDABufferOutputStream.
+ *
+ * Returns: The CPU buffer size in bytes.
+ *
+ * See garrow_gpu_cuda_buffer_output_stream_set_buffer_size() for CPU
+ * buffer size details.
+ *
+ * Since: 0.8.0
+ */
+gint64
+garrow_gpu_cuda_buffer_output_stream_get_buffer_size(GArrowGPUCUDABufferOutputStream *stream)
+{
+  auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream);
+  return arrow_stream->buffer_size();
+}
+
+/**
+ * garrow_gpu_cuda_buffer_output_stream_get_buffered_size:
+ * @stream: A #GArrowGPUCUDABufferOutputStream.
+ *
+ * Returns: The size of buffered data in bytes.
+ *
+ * Since: 0.8.0
+ */
+gint64
+garrow_gpu_cuda_buffer_output_stream_get_buffered_size(GArrowGPUCUDABufferOutputStream *stream)
+{
+  auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream);
+  return arrow_stream->num_bytes_buffered();
+}
+
+
+G_END_DECLS
+
+GArrowGPUCUDAContext *
+garrow_gpu_cuda_context_new_raw(std::shared_ptr<arrow::gpu::CudaContext> *arrow_context)
+{
+  return GARROW_GPU_CUDA_CONTEXT(g_object_new(GARROW_GPU_TYPE_CUDA_CONTEXT,
+                                              "context", arrow_context,
+                                              NULL));
+}
+
+std::shared_ptr<arrow::gpu::CudaContext>
+garrow_gpu_cuda_context_get_raw(GArrowGPUCUDAContext *context)
+{
+  if (!context)
+    return nullptr;
+
+  auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(context);
+  return priv->context;
+}
+
+GArrowGPUCUDAIPCMemoryHandle *
+garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_handle)
+{
+  auto handle = g_object_new(GARROW_GPU_TYPE_CUDA_IPC_MEMORY_HANDLE,
+                             "ipc-memory-handle", arrow_handle,
+                             NULL);
+  return GARROW_GPU_CUDA_IPC_MEMORY_HANDLE(handle);
+}
+
+arrow::gpu::CudaIpcMemHandle *
+garrow_gpu_cuda_ipc_memory_handle_get_raw(GArrowGPUCUDAIPCMemoryHandle *handle)
+{
+  if (!handle)
+    return nullptr;
+
+  auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(handle);
+  return priv->ipc_memory_handle;
+}
+
+GArrowGPUCUDABuffer *
+garrow_gpu_cuda_buffer_new_raw(std::shared_ptr<arrow::gpu::CudaBuffer> *arrow_buffer)
+{
+  return GARROW_GPU_CUDA_BUFFER(g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER,
+                                             "buffer", arrow_buffer,
+                                             NULL));
+}
+
+std::shared_ptr<arrow::gpu::CudaBuffer>
+garrow_gpu_cuda_buffer_get_raw(GArrowGPUCUDABuffer *buffer)
+{
+  if (!buffer)
+    return nullptr;
+
+  auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer));
+  return std::static_pointer_cast<arrow::gpu::CudaBuffer>(arrow_buffer);
+}
+
+GArrowGPUCUDAHostBuffer *
+garrow_gpu_cuda_host_buffer_new_raw(std::shared_ptr<arrow::gpu::CudaHostBuffer> *arrow_buffer)
+{
+  auto buffer = g_object_new(GARROW_GPU_TYPE_CUDA_HOST_BUFFER,
+                             "buffer", arrow_buffer,
+                             NULL);
+  return GARROW_GPU_CUDA_HOST_BUFFER(buffer);
+}
+
+std::shared_ptr<arrow::gpu::CudaHostBuffer>
+garrow_gpu_cuda_host_buffer_get_raw(GArrowGPUCUDAHostBuffer *buffer)
+{
+  if (!buffer)
+    return nullptr;
+
+  auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer));
+  return std::static_pointer_cast<arrow::gpu::CudaHostBuffer>(arrow_buffer);
+}
+
+GArrowGPUCUDABufferInputStream *
+garrow_gpu_cuda_buffer_input_stream_new_raw(std::shared_ptr<arrow::gpu::CudaBufferReader> *arrow_reader)
+{
+  auto input_stream = g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER_INPUT_STREAM,
+                                   "input-stream", arrow_reader,
+                                   NULL);
+  return GARROW_GPU_CUDA_BUFFER_INPUT_STREAM(input_stream);
+}
+
+std::shared_ptr<arrow::gpu::CudaBufferReader>
+garrow_gpu_cuda_buffer_input_stream_get_raw(GArrowGPUCUDABufferInputStream *input_stream)
+{
+  if (!input_stream)
+    return nullptr;
+
+  auto arrow_reader =
+    garrow_input_stream_get_raw(GARROW_INPUT_STREAM(input_stream));
+  return std::static_pointer_cast<arrow::gpu::CudaBufferReader>(arrow_reader);
+}
+
+GArrowGPUCUDABufferOutputStream *
+garrow_gpu_cuda_buffer_output_stream_new_raw(std::shared_ptr<arrow::gpu::CudaBufferWriter> *arrow_writer)
+{
+  auto output_stream = g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER_OUTPUT_STREAM,
+                                    "output-stream", arrow_writer,
+                                    NULL);
+  return GARROW_GPU_CUDA_BUFFER_OUTPUT_STREAM(output_stream);
+}
+
+std::shared_ptr<arrow::gpu::CudaBufferWriter>
+garrow_gpu_cuda_buffer_output_stream_get_raw(GArrowGPUCUDABufferOutputStream *output_stream)
+{
+  if (!output_stream)
+    return nullptr;
+
+  auto arrow_writer =
+    garrow_output_stream_get_raw(GARROW_OUTPUT_STREAM(output_stream));
+  return std::static_pointer_cast<arrow::gpu::CudaBufferWriter>(arrow_writer);
+}
diff --git a/c_glib/arrow-gpu-glib/cuda.h b/c_glib/arrow-gpu-glib/cuda.h
new file mode 100644
index 0000000000000..7c615a144e739
--- /dev/null
+++ b/c_glib/arrow-gpu-glib/cuda.h
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/arrow-glib.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_GPU_TYPE_CUDA_DEVICE_MANAGER     \
+  (garrow_gpu_cuda_device_manager_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDADeviceManager,
+                         garrow_gpu_cuda_device_manager,
+                         GARROW_GPU,
+                         CUDA_DEVICE_MANAGER,
+                         GObject)
+struct _GArrowGPUCUDADeviceManagerClass
+{
+  GObjectClass parent_class;
+};
+
+#define GARROW_GPU_TYPE_CUDA_CONTEXT (garrow_gpu_cuda_context_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAContext,
+                         garrow_gpu_cuda_context,
+                         GARROW_GPU,
+                         CUDA_CONTEXT,
+                         GObject)
+struct _GArrowGPUCUDAContextClass
+{
+  GObjectClass parent_class;
+};
+
+#define GARROW_GPU_TYPE_CUDA_BUFFER (garrow_gpu_cuda_buffer_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABuffer,
+                         garrow_gpu_cuda_buffer,
+                         GARROW_GPU,
+                         CUDA_BUFFER,
+                         GArrowBuffer)
+struct _GArrowGPUCUDABufferClass
+{
+  GArrowBufferClass parent_class;
+};
+
+#define GARROW_GPU_TYPE_CUDA_HOST_BUFFER (garrow_gpu_cuda_host_buffer_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAHostBuffer,
+                         garrow_gpu_cuda_host_buffer,
+                         GARROW_GPU,
+                         CUDA_HOST_BUFFER,
+                         GArrowMutableBuffer)
+struct _GArrowGPUCUDAHostBufferClass
+{
+  GArrowMutableBufferClass parent_class;
+};
+
+#define GARROW_GPU_TYPE_CUDA_IPC_MEMORY_HANDLE          \
+  (garrow_gpu_cuda_ipc_memory_handle_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAIPCMemoryHandle,
+                         garrow_gpu_cuda_ipc_memory_handle,
+                         GARROW_GPU,
+                         CUDA_IPC_MEMORY_HANDLE,
+                         GObject)
+struct _GArrowGPUCUDAIPCMemoryHandleClass
+{
+  GObjectClass parent_class;
+};
+
+#define GARROW_GPU_TYPE_CUDA_BUFFER_INPUT_STREAM        \
+  (garrow_gpu_cuda_buffer_input_stream_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABufferInputStream,
+                         garrow_gpu_cuda_buffer_input_stream,
+                         GARROW_GPU,
+                         CUDA_BUFFER_INPUT_STREAM,
+                         GArrowBufferInputStream)
+struct _GArrowGPUCUDABufferInputStreamClass
+{
+  GArrowBufferInputStreamClass parent_class;
+};
+
+#define GARROW_GPU_TYPE_CUDA_BUFFER_OUTPUT_STREAM               \
+  (garrow_gpu_cuda_buffer_output_stream_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABufferOutputStream,
+                         garrow_gpu_cuda_buffer_output_stream,
+                         GARROW_GPU,
+                         CUDA_BUFFER_OUTPUT_STREAM,
+                         GArrowOutputStream)
+struct _GArrowGPUCUDABufferOutputStreamClass
+{
+  GArrowOutputStreamClass parent_class;
+};
+
+GArrowGPUCUDADeviceManager *
+garrow_gpu_cuda_device_manager_new(GError **error);
+
+GArrowGPUCUDAContext *
+garrow_gpu_cuda_device_manager_get_context(GArrowGPUCUDADeviceManager *manager,
+                                           gint gpu_number,
+                                           GError **error);
+gsize
+garrow_gpu_cuda_device_manager_get_n_devices(GArrowGPUCUDADeviceManager *manager);
+
+gint64
+garrow_gpu_cuda_context_get_allocated_size(GArrowGPUCUDAContext *context);
+
+
+GArrowGPUCUDABuffer *
+garrow_gpu_cuda_buffer_new(GArrowGPUCUDAContext *context,
+                           gint64 size,
+                           GError **error);
+GArrowGPUCUDABuffer *
+garrow_gpu_cuda_buffer_new_ipc(GArrowGPUCUDAContext *context,
+                               GArrowGPUCUDAIPCMemoryHandle *handle,
+                               GError **error);
+GArrowGPUCUDABuffer *
+garrow_gpu_cuda_buffer_new_record_batch(GArrowGPUCUDAContext *context,
+                                        GArrowRecordBatch *record_batch,
+                                        GError **error);
+GBytes *
+garrow_gpu_cuda_buffer_copy_to_host(GArrowGPUCUDABuffer *buffer,
+                                    gint64 position,
+                                    gint64 size,
+                                    GError **error);
+gboolean
+garrow_gpu_cuda_buffer_copy_from_host(GArrowGPUCUDABuffer *buffer,
+                                      const guint8 *data,
+                                      gint64 size,
+                                      GError **error);
+GArrowGPUCUDAIPCMemoryHandle *
+garrow_gpu_cuda_buffer_export(GArrowGPUCUDABuffer *buffer,
+                              GError **error);
+GArrowGPUCUDAContext *
+garrow_gpu_cuda_buffer_get_context(GArrowGPUCUDABuffer *buffer);
+GArrowRecordBatch *
+garrow_gpu_cuda_buffer_read_record_batch(GArrowGPUCUDABuffer *buffer,
+                                         GArrowSchema *schema,
+                                         GError **error);
+
+
+GArrowGPUCUDAHostBuffer *
+garrow_gpu_cuda_host_buffer_new(gint64 size, GError **error);
+
+GArrowGPUCUDAIPCMemoryHandle *
+garrow_gpu_cuda_ipc_memory_handle_new(const guint8 *data,
+                                      gsize size,
+                                      GError **error);
+
+GArrowBuffer *
+garrow_gpu_cuda_ipc_memory_handle_serialize(GArrowGPUCUDAIPCMemoryHandle *handle,
+                                            GError **error);
+
+GArrowGPUCUDABufferInputStream *
+garrow_gpu_cuda_buffer_input_stream_new(GArrowGPUCUDABuffer *buffer);
+
+GArrowGPUCUDABufferOutputStream *
+garrow_gpu_cuda_buffer_output_stream_new(GArrowGPUCUDABuffer *buffer);
+
+gboolean
+garrow_gpu_cuda_buffer_output_stream_set_buffer_size(GArrowGPUCUDABufferOutputStream *stream,
+                                                     gint64 size,
+                                                     GError **error);
+gint64
+garrow_gpu_cuda_buffer_output_stream_get_buffer_size(GArrowGPUCUDABufferOutputStream *stream);
+gint64
+garrow_gpu_cuda_buffer_output_stream_get_buffered_size(GArrowGPUCUDABufferOutputStream *stream);
+
+G_END_DECLS
diff --git a/c_glib/arrow-gpu-glib/cuda.hpp b/c_glib/arrow-gpu-glib/cuda.hpp
new file mode 100644
index 0000000000000..3eeff8b6f18ed
--- /dev/null
+++ b/c_glib/arrow-gpu-glib/cuda.hpp
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/gpu/cuda_api.h>
+
+#include <arrow-gpu-glib/cuda.h>
+
+GArrowGPUCUDAContext *
+garrow_gpu_cuda_context_new_raw(std::shared_ptr<arrow::gpu::CudaContext> *arrow_context);
+std::shared_ptr<arrow::gpu::CudaContext>
+garrow_gpu_cuda_context_get_raw(GArrowGPUCUDAContext *context);
+
+GArrowGPUCUDAIPCMemoryHandle *
+garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_handle);
+arrow::gpu::CudaIpcMemHandle *
+garrow_gpu_cuda_ipc_memory_handle_get_raw(GArrowGPUCUDAIPCMemoryHandle *handle);
+
+GArrowGPUCUDABuffer *
+garrow_gpu_cuda_buffer_new_raw(std::shared_ptr<arrow::gpu::CudaBuffer> *arrow_buffer);
+std::shared_ptr<arrow::gpu::CudaBuffer>
+garrow_gpu_cuda_buffer_get_raw(GArrowGPUCUDABuffer *buffer);
+
+GArrowGPUCUDAHostBuffer *
+garrow_gpu_cuda_host_buffer_new_raw(std::shared_ptr<arrow::gpu::CudaHostBuffer> *arrow_buffer);
+std::shared_ptr<arrow::gpu::CudaHostBuffer>
+garrow_gpu_cuda_host_buffer_get_raw(GArrowGPUCUDAHostBuffer *buffer);
+
+GArrowGPUCUDABufferInputStream *
+garrow_gpu_cuda_buffer_input_stream_new_raw(std::shared_ptr<arrow::gpu::CudaBufferReader> *arrow_reader);
+std::shared_ptr<arrow::gpu::CudaBufferReader>
+garrow_gpu_cuda_buffer_input_stream_get_raw(GArrowGPUCUDABufferInputStream *input_stream);
+
+GArrowGPUCUDABufferOutputStream *
+garrow_gpu_cuda_buffer_output_stream_new_raw(std::shared_ptr<arrow::gpu::CudaBufferWriter> *arrow_writer);
+std::shared_ptr<arrow::gpu::CudaBufferWriter>
+garrow_gpu_cuda_buffer_output_stream_get_raw(GArrowGPUCUDABufferOutputStream *output_stream);
diff --git a/c_glib/arrow-gpu-glib/meson.build b/c_glib/arrow-gpu-glib/meson.build
new file mode 100644
index 0000000000000..00c7f079d6485
--- /dev/null
+++ b/c_glib/arrow-gpu-glib/meson.build
@@ -0,0 +1,80 @@
+# -*- indent-tabs-mode: nil -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+sources = files(
+  'cuda.cpp',
+)
+
+c_headers = files(
+  'arrow-gpu-glib.h',
+  'cuda.h',
+)
+
+cpp_headers = files(
+  'arrow-gpu-glib.hpp',
+  'cuda.hpp',
+)
+
+headers = c_headers + cpp_headers
+install_headers(headers, subdir: 'arrow-gpu-glib')
+
+
+dependencies = [
+  arrow_gpu_dependency,
+  libarrow_glib_dependency,
+]
+libarrow_gpu_glib = library('arrow-gpu-glib',
+                            sources: sources,
+                            install: true,
+                            dependencies: dependencies,
+                            include_directories: [
+                              root_inc,
+                            ],
+                            soversion: so_version,
+                            version: library_version)
+libarrow_gpu_glib_dependency = declare_dependency(link_with: libarrow_gpu_glib,
+                                                  include_directories: [
+                                                    root_inc,
+                                                  ],
+                                                  dependencies: dependencies)
+
+pkgconfig.generate(filebase: 'arrow-gpu-glib',
+                   name: 'Apache Arrow GPU GLib',
+                   description: 'C API for Apache Arrow GPU based on GLib',
+                   version: version,
+                   requires: ['arrow-glib', 'arrow-gpu'],
+                   libraries: [libarrow_gpu_glib],
+                   subdirs: ['arrow-gpu-glib'])
+
+gnome.generate_gir(libarrow_gpu_glib,
+                   dependencies: arrow_glib_gir_dependency,
+                   sources: sources + c_headers,
+                   namespace: 'ArrowGPU',
+                   nsversion: api_version,
+                   identifier_prefix: 'GArrowGPU',
+                   symbol_prefix: 'garrow_gpu',
+                   export_packages: 'arrow-gpu-glib',
+                   includes: [
+                     'Arrow-1.0',
+                   ],
+                   install: true,
+                   extra_args: [
+                     '--warn-all',
+                     '--include-uninstalled=./arrow-glib/Arrow-1.0.gir',
+                   ])
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index 5db435275a300..c6fa0192c944d 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -77,18 +77,34 @@ AC_ARG_WITH(arrow-cpp-build-dir,
   [GARROW_ARROW_CPP_BUILD_DIR=""])
 if test "x$GARROW_ARROW_CPP_BUILD_DIR" = "x"; then
   PKG_CHECK_MODULES([ARROW], [arrow arrow-compute])
+  PKG_CHECK_MODULES([ARROW_GPU],
+                    [arrow-gpu],
+                    [HAVE_ARROW_GPU=yes],
+                    [HAVE_ARROW_GPU=no])
 else
   ARROW_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src"
   ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}"
 
   ARROW_CFLAGS="-I${ARROW_INCLUDE_DIR}"
-
   ARROW_LIBS="-L${ARROW_LIB_DIR} -larrow"
-
-  AC_SUBST(ARROW_LIB_DIR)
-
   AC_SUBST(ARROW_CFLAGS)
   AC_SUBST(ARROW_LIBS)
+
+  ARROW_GPU_CFLAGS=""
+  if test -f "${GARROW_ARROW_CPP_BUILD_DIR}/src/arrow/gpu/arrow-gpu.pc"; then
+    HAVE_ARROW_GPU=yes
+    ARROW_GPU_LIBS="-larrow_gpu"
+  else
+    HAVE_ARROW_GPU=no
+    ARROW_GPU_LIBS=""
+  fi
+  AC_SUBST(ARROW_GPU_CFLAGS)
+  AC_SUBST(ARROW_GPU_LIBS)
+fi
+
+AM_CONDITIONAL([HAVE_ARROW_GPU], [test "$HAVE_ARROW_GPU" = "yes"])
+if test "$HAVE_ARROW_GPU" = "yes"; then
+  AC_DEFINE(HAVE_ARROW_GPU, [1], [Define to 1 if Apache Arrow supports GPU.])
 fi
 
 exampledir="\$(datadir)/arrow-glib/example"
@@ -98,6 +114,8 @@ AC_CONFIG_FILES([
   Makefile
   arrow-glib/Makefile
   arrow-glib/arrow-glib.pc
+  arrow-gpu-glib/Makefile
+  arrow-gpu-glib/arrow-gpu-glib.pc
   doc/Makefile
   doc/reference/Makefile
   doc/reference/xml/Makefile
diff --git a/c_glib/doc/Makefile.am b/c_glib/doc/Makefile.am
index 85c1d5126097c..1d491ab09110e 100644
--- a/c_glib/doc/Makefile.am
+++ b/c_glib/doc/Makefile.am
@@ -16,4 +16,4 @@
 # under the License.
 
 SUBDIRS =                                       \
-        reference
+	reference
diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am
index 45b11f035183e..896aff544d454 100644
--- a/c_glib/doc/reference/Makefile.am
+++ b/c_glib/doc/reference/Makefile.am
@@ -51,6 +51,17 @@ AM_CFLAGS =					\
 GTKDOC_LIBS =						\
 	$(top_builddir)/arrow-glib/libarrow-glib.la
 
+if HAVE_ARROW_GPU
+DOC_SOURCE_DIR +=				\
+	$(top_srcdir)/arrow-gpu-glib
+HFILE_GLOB +=					\
+	$(top_srcdir)/arrow-gpu-glib/*.h
+CFILE_GLOB +=					\
+	$(top_srcdir)/arrow-gpu-glib/*.cpp
+GTKDOC_LIBS +=							\
+	$(top_builddir)/arrow-gpu-glib/libarrow-gpu-glib.la
+endif
+
 include $(srcdir)/gtk-doc.make
 
 CLEANFILES +=					\
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml
index a504ef1148383..e267ea2f9d356 100644
--- a/c_glib/doc/reference/arrow-glib-docs.sgml
+++ b/c_glib/doc/reference/arrow-glib-docs.sgml
@@ -125,6 +125,16 @@
     </chapter>
   </part>
 
+  <!-- TODO
+  <part id="gpu">
+    <title>GPU</title>
+    <chapter id="cuda">
+      <title>CUDA</title>
+      <xi:include href="xml/cuda.xml"/>
+    </chapter>
+  </part>
+  -->
+
   <chapter id="object-tree">
     <title>Object Hierarchy</title>
     <xi:include href="xml/tree_index.sgml"/>
diff --git a/c_glib/doc/reference/meson.build b/c_glib/doc/reference/meson.build
index 08936daf87288..4c9552e83c303 100644
--- a/c_glib/doc/reference/meson.build
+++ b/c_glib/doc/reference/meson.build
@@ -32,13 +32,26 @@ glib_prefix = dependency('glib-2.0').get_pkgconfig_variable('prefix')
 glib_doc_path = join_paths(glib_prefix, 'share', 'gtk-doc', 'html')
 doc_path = join_paths(data_dir, meson.project_name(), 'gtk-doc', 'html')
 
+source_directories = [
+  join_paths(meson.source_root(), 'arrow-glib'),
+  join_paths(meson.build_root(), 'arrow-glib'),
+]
+dependencies = [
+  libarrow_glib_dependency,
+]
+if arrow_gpu_dependency.found()
+  source_directories += [
+    join_paths(meson.source_root(), 'arrow-gpu-glib'),
+    join_paths(meson.build_root(), 'arrow-gpu-glib'),
+  ]
+  dependencies += [
+    libarrow_gpu_glib_dependency,
+  ]
+endif
 gnome.gtkdoc(meson.project_name(),
              main_xml: meson.project_name() + '-docs.sgml',
-             src_dir: [
-               join_paths(meson.source_root(), 'arrow-glib'),
-               join_paths(meson.build_root(), 'arrow-glib'),
-             ],
-             dependencies: libarrow_glib_dependency,
+             src_dir: source_directories,
+             dependencies: dependencies,
              gobject_typesfile: meson.project_name() + '.types',
              scan_args: [
                '--rebuild-types',
diff --git a/c_glib/meson.build b/c_glib/meson.build
index 1fa64ba19c406..9fe1b8cbd7179 100644
--- a/c_glib/meson.build
+++ b/c_glib/meson.build
@@ -49,6 +49,10 @@ pkgconfig = import('pkgconfig')
 root_inc = include_directories('.')
 
 subdir('arrow-glib')
+arrow_gpu_dependency = dependency('arrow-gpu', required: false)
+if arrow_gpu_dependency.found()
+  subdir('arrow-gpu-glib')
+endif
 subdir('example')
 
 if get_option('enable_gtk_doc')
@@ -58,4 +62,7 @@ endif
 run_test = find_program('test/run-test.sh')
 test('unit test',
      run_test,
-     env: ['ARROW_GLIB_TYPELIB_DIR=@0@/arrow-glib'.format(meson.build_root())])
+     env: [
+       'ARROW_GLIB_TYPELIB_DIR=@0@/arrow-glib'.format(meson.build_root()),
+       'ARROW_GPU_GLIB_TYPELIB_DIR=@0@/arrow-gpu-glib'.format(meson.build_root()),
+     ])
diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb
index 3451bd29fde1b..392c56f33ae51 100755
--- a/c_glib/test/run-test.rb
+++ b/c_glib/test/run-test.rb
@@ -37,6 +37,12 @@ def initialize(data)
   end
 end
 
+begin
+  ArrowGPU = GI.load("ArrowGPU")
+rescue GObjectIntrospection::RepositoryError::TypelibNotFound
+end
+
+require "rbconfig"
 require "tempfile"
 require_relative "helper/buildable"
 require_relative "helper/omittable"
diff --git a/c_glib/test/run-test.sh b/c_glib/test/run-test.sh
index 19ccf077833a5..d563e8586ce59 100755
--- a/c_glib/test/run-test.sh
+++ b/c_glib/test/run-test.sh
@@ -20,27 +20,34 @@
 test_dir="$(cd $(dirname $0); pwd)"
 build_dir="$(cd .; pwd)"
 
-arrow_glib_build_dir="${build_dir}/arrow-glib/"
-libtool_dir="${arrow_glib_build_dir}/.libs"
-if [ -d "${libtool_dir}" ]; then
-  LD_LIBRARY_PATH="${libtool_dir}:${LD_LIBRARY_PATH}"
-else
-  if [ -d "${arrow_glib_build_dir}" ]; then
-    LD_LIBRARY_PATH="${arrow_glib_build_dir}:${LD_LIBRARY_PATH}"
+modules="arrow-glib arrow-gpu-glib"
+
+for module in ${modules}; do
+  module_build_dir="${build_dir}/${module}"
+  libtool_dir="${module_build_dir}/.libs"
+  if [ -d "${libtool_dir}" ]; then
+    LD_LIBRARY_PATH="${libtool_dir}:${LD_LIBRARY_PATH}"
+  else
+    if [ -d "${module_build_dir}" ]; then
+      LD_LIBRARY_PATH="${module_build_dir}:${LD_LIBRARY_PATH}"
+    fi
   fi
-fi
+done
 
 if [ -f "Makefile" -a "${NO_MAKE}" != "yes" ]; then
   make -j8 > /dev/null || exit $?
 fi
 
-arrow_glib_typelib_dir="${ARROW_GLIB_TYPELIB_DIR}"
-if [ -z "${arrow_glib_typelib_dir}" ]; then
-  arrow_glib_typelib_dir="${build_dir}/arrow-glib"
-fi
+for module in ${modules}; do
+  MODULE_TYPELIB_DIR_VAR_NAME="$(echo ${module} | tr a-z- A-Z_)_TYPELIB_DIR"
+  module_typelib_dir=$(eval "echo \${${MODULE_TYPELIB_DIR_VAR_NAME}}")
+  if [ -z "${module_typelib_dir}" ]; then
+    module_typelib_dir="${build_dir}/${module}"
+  fi
 
-if [ -d "${arrow_glib_typelib_dir}" ]; then
-  GI_TYPELIB_PATH="${arrow_glib_typelib_dir}:${GI_TYPELIB_PATH}"
-fi
+  if [ -d "${module_typelib_dir}" ]; then
+    GI_TYPELIB_PATH="${module_typelib_dir}:${GI_TYPELIB_PATH}"
+  fi
+done
 
 ${GDB} ruby ${test_dir}/run-test.rb "$@"
diff --git a/c_glib/test/test-gpu-cuda.rb b/c_glib/test/test-gpu-cuda.rb
new file mode 100644
index 0000000000000..c710ef2264976
--- /dev/null
+++ b/c_glib/test/test-gpu-cuda.rb
@@ -0,0 +1,144 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestGPUCUDA < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def setup
+    omit("Arrow GPU is required") unless defined?(::ArrowGPU)
+    @manager = ArrowGPU::CUDADeviceManager.new
+    omit("At least one GPU is required") if @manager.n_devices.zero?
+    @context = @manager.get_context(0)
+  end
+
+  sub_test_case("Context") do
+    def test_allocated_size
+      allocated_size_before = @context.allocated_size
+      size = 128
+      buffer = ArrowGPU::CUDABuffer.new(@context, size)
+      assert_equal(size,
+                   @context.allocated_size - allocated_size_before)
+    end
+  end
+
+  sub_test_case("Buffer") do
+    def setup
+      super
+      @buffer = ArrowGPU::CUDABuffer.new(@context, 128)
+    end
+
+    def test_copy
+      @buffer.copy_from_host("Hello World")
+      assert_equal("llo W", @buffer.copy_to_host(2, 5).to_s)
+    end
+
+    def test_export
+      @buffer.copy_from_host("Hello World")
+      handle = @buffer.export
+      serialized_handle = handle.serialize.data
+      Tempfile.open("arrow-gpu-cuda-export") do |output|
+        pid = spawn(RbConfig.ruby, "-e", <<-SCRIPT)
+require "gi"
+
+Gio = GI.load("Gio")
+Arrow = GI.load("Arrow")
+ArrowGPU = GI.load("ArrowGPU")
+
+manager = ArrowGPU::CUDADeviceManager.new
+context = manager.get_context(0)
+serialized_handle = #{serialized_handle.to_s.dump}
+handle = ArrowGPU::CUDAIPCMemoryHandle.new(serialized_handle)
+buffer = ArrowGPU::CUDABuffer.new(context, handle)
+File.open(#{output.path.dump}, "w") do |output|
+  output.print(buffer.copy_to_host(0, 6).to_s)
+end
+        SCRIPT
+        Process.waitpid(pid)
+        assert_equal("Hello ", output.read)
+      end
+    end
+
+    def test_context
+      assert_equal(@context.allocated_size,
+                   @buffer.context.allocated_size)
+    end
+
+    def test_record_batch
+      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+      schema = Arrow::Schema.new([field])
+      columns = [
+        build_boolean_array([true]),
+      ]
+      cpu_record_batch = Arrow::RecordBatch.new(schema, 1, columns)
+
+      buffer = ArrowGPU::CUDABuffer.new(@context, cpu_record_batch)
+      gpu_record_batch = buffer.read_record_batch(schema)
+      assert_equal(cpu_record_batch.n_rows,
+                   gpu_record_batch.n_rows)
+    end
+  end
+
+  sub_test_case("HostBuffer") do
+    def test_new
+      buffer = ArrowGPU::CUDAHostBuffer.new(128)
+      assert_equal(128, buffer.size)
+    end
+  end
+
+  sub_test_case("BufferInputStream") do
+    def test_new
+      buffer = ArrowGPU::CUDABuffer.new(@context, 128)
+      buffer.copy_from_host("Hello World")
+      stream = ArrowGPU::CUDABufferInputStream.new(buffer)
+      begin
+        assert_equal("Hello Worl", stream.read(5).copy_to_host(0, 10).to_s)
+      ensure
+        stream.close
+      end
+    end
+  end
+
+  sub_test_case("BufferOutputStream") do
+    def setup
+      super
+      @buffer = ArrowGPU::CUDABuffer.new(@context, 128)
+      @buffer.copy_from_host("\x00" * @buffer.size)
+      @stream = ArrowGPU::CUDABufferOutputStream.new(@buffer)
+    end
+
+    def cleanup
+      super
+      @stream.close
+    end
+
+    def test_new
+      @stream.write("Hello World")
+      assert_equal("Hello World", @buffer.copy_to_host(0, 11).to_s)
+    end
+
+    def test_buffer
+      assert_equal(0, @stream.buffer_size)
+      @stream.buffer_size = 5
+      assert_equal(5, @stream.buffer_size)
+      @stream.write("Hell")
+      assert_equal(4, @stream.buffered_size)
+      assert_equal("\x00" * 5, @buffer.copy_to_host(0, 5).to_s)
+      @stream.write("o")
+      assert_equal("Hello", @buffer.copy_to_host(0, 5).to_s)
+    end
+  end
+end

From b18bbeba0ed8b76f6575bd0501de3beb1e6e60f5 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Mon, 13 Nov 2017 22:56:13 -0500
Subject: [PATCH 1243/1644] ARROW-1371: [Website] Add "Powered By" page to the
 website

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1280 from xhochy/ARROW-1371 and squashes the following commits:

84a76e94 [Uwe L. Korn] Add Red Data Tools
57338daa [Wes McKinney] Formatting
522a3592 [Wes McKinney] Add a few more projects
ad511ed2 [Korn, Uwe] Add GeoMesa and Dremio
0e4f755c [Korn, Uwe] Add Ray and Spark
ecabb167 [Korn, Uwe] ARROW-1371: [Website] Add "Powered By" page to the website
---
 site/_config.yml           |   1 +
 site/_includes/header.html |   1 +
 site/powered_by.md         | 126 +++++++++++++++++++++++++++++++++++++
 3 files changed, 128 insertions(+)
 create mode 100644 site/powered_by.md

diff --git a/site/_config.yml b/site/_config.yml
index a6c5575d0680f..cbcf97dd3b0d1 100644
--- a/site/_config.yml
+++ b/site/_config.yml
@@ -30,6 +30,7 @@ exclude:
   - Gemfile
   - Gemfile.lock
   - _docs/format/*
+  - ruby
   - asf-site
   - scripts
   - README.md
diff --git a/site/_includes/header.html b/site/_includes/header.html
index 6c0ec30f39ca7..03b3c8750cdb8 100644
--- a/site/_includes/header.html
+++ b/site/_includes/header.html
@@ -27,6 +27,7 @@
             <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Mailing List</a></li>
             <li><a href="https://apachearrowslackin.herokuapp.com">Slack Channel</a></li>
             <li><a href="{{ site.baseurl }}/committers/">Committers</a></li>
+            <li><a href="{{ site.baseurl }}/powered_by/">Powered By</a></li>
           </ul>
         </li>
         <li class="dropdown">
diff --git a/site/powered_by.md b/site/powered_by.md
new file mode 100644
index 0000000000000..38320c0ec91b0
--- /dev/null
+++ b/site/powered_by.md
@@ -0,0 +1,126 @@
+---
+layout: default
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+## Powered By
+
+### Project and Product names using "Apache Arrow"
+
+Organizations creating products and projects for use with Apache Arrow, along
+with associated marketing materials, should take care to respect the trademark
+in "Apache Arrow" and its logo. Please refer to [ASF Trademarks Guidance][1]
+and associated [FAQ][2] for comprehensive and authoritative guidance on proper
+usage of ASF trademarks.
+
+Names that do not include "Apache Arrow" at all have no potential trademark
+issue with the Apache Arrow project. This is recommended.
+
+Names like "Apache Arrow BigCoProduct" are not OK, as are names including
+"Apache Arrow" in general. The above links, however, describe some exceptions,
+like for names such as "BigCoProduct, powered by Apache Arrow" or
+"BigCoProduct for Apache Arrow".
+
+It is common practice to create software identifiers (Maven coordinates, module
+names, etc.) like "arrow-foo". These are permitted. Nominative use of trademarks
+in descriptions is also always allowed, as in "BigCoProduct is a widget for
+Apache Arrow".
+
+### Open Source Projects
+
+To add yourself to the list, please email dev@arrow.apache.org with your
+organization name, URL, a list of which Arrow components you are using, and a
+short description of your use case.
+
+* **[Apache Parquet][3]:** A columnar storage format available to any project
+  in the Hadoop ecosystem, regardless of the choice of data processing
+  framework, data model or programming language. The C++ and Java
+  implementation provide vectorized reads and write to/from Arrow data
+  structures.
+* **[Apache Spark][7]:** Apache Spark™ is a fast and general engine for
+  large-scale data processing. Spark uses Apache Arrow to
+  1. improve performance of conversion between Spark DataFrame and pandas DataFrame
+  2. enable a set of vectorized user-defined functions (`pandas_udf`) in PySpark.
+* **[Dask][15]:** Python library for parallel and distributed execution of
+  dynamic task graphs. Dask supports using pyarrow for accessing Parquet
+  files
+* **[GeoMesa][8]:** A suite of tools that enables large-scale geospatial query
+  and analytics on distributed computing systems. GeoMesa supports query
+  results in the Arrow IPC format, which can then be used for in-browser
+  visualizations and/or further analytics.
+* **[libgdf][14]:** A C library of CUDA-based analytics functions and GPU IPC
+  support for structured data. Uses the Arrow IPC format and targets the Arrow
+  memory layout in its analytic functions. This work is part of the [GPU Open
+  Analytics Initiative][11]
+* **[MapD][10]:** in-memory columnar SQL engine designed to run on GPUs. MapD
+  supports Arrow for data ingest and data interchange via CUDA IPC
+  handles. This work is part of the [GPU Open Analytics Initiative][11]
+* **[pandas][12]:** data analysis toolkit for Python programmers. pandas
+  supports reading and writing Parquet files using pyarrow. Several pandas
+  core developers are also contributors to Apache Arrow.
+* **[Ray][5]:** A flexible, high-performance distributed execution framework
+  with a focus on machine learning and AI applications. Uses Arrow to
+  efficiently store Python data structures containing large arrays of numerical
+  data. Data can be accessed with zero-copy by multiple processes using the
+  [Plasma shared memory object store][6] which originated from Ray and is part
+  of Arrow now.
+* **[Red Data Tools][16]:** A project that provides data processing
+  tools for Ruby. It provides [Red Arrow][17] that is a Ruby bindings
+  of Apache Arrow based on Apache Arrow GLib. Red Arrow is a core
+  library for it. It also provides many Ruby libraries to integrate
+  existing Ruby libraries with Apache Arrow. They use Red Arrow.
+* **[Turbodbc][4]:** Python module to access relational databases via the Open
+  Database Connectivity (ODBC) interface. It provides the ability to return
+  Arrow Tables and RecordBatches in addition to the Python Database API
+  Specification 2.0.
+
+### Companies and Organizations
+
+To add yourself to the list, please email dev@arrow.apache.org with your
+organization name, URL, a list of which Arrow components you are using, and a
+short description of your use case.
+
+* **[Dremio][9]:** A self-service data platform. Dremio makes it easy for
+  users to discover, curate, accelerate, and share data from any source.
+  It includes a distributed SQL execution engine based on Apache Arrow.
+  Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow
+  buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI,
+  Python, R, and more (all backed by Apache Arrow).
+* **[Quilt Data][13]:** Quilt is a data package manager, designed to make
+  managing data as easy as managing code. It supports Parquet format via
+  pyarrow for data access.
+
+[1]: https://www.apache.org/foundation/marks/
+[2]: https://www.apache.org/foundation/marks/faq/
+[3]: https://parquet.apache.org/
+[4]: https://github.com/blue-yonder/turbodbc
+[5]: https://github.com/ray-project/ray
+[6]: https://ray-project.github.io/2017/08/08/plasma-in-memory-object-store.html
+[7]: https://spark.apache.org/
+[8]: https://github.com/locationtech/geomesa
+[9]: https://www.dremio.com/
+[10]: https://github.com/mapd/mapd-core
+[11]: https://gpuopenanalytics.com/
+[12]: https://pandas.pydata.org
+[13]: https://quiltdata.com/
+[14]: https://github.com/gpuopenanalytics/libgdf
+[15]: https://github.com/dask/dask
+[16]: https://red-data-tools.github.io/
+[17]: https://github.com/red-data-tools/red-arrow/

From e3db5da648bb74b3d325042e595ae856f1ad8f7f Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 14 Nov 2017 11:17:59 -0500
Subject: [PATCH 1244/1644] ARROW-1806: [GLib] Add
 garrow_record_batch_writer_write_table()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1315 from kou/glib-add-recored-batch-writer-write-table and squashes the following commits:

15743615 [Kouhei Sutou] [GLib] Add garrow_record_batch_writer_write_table()
---
 c_glib/arrow-glib/writer.cpp    | 25 +++++++++++++++++++++++++
 c_glib/arrow-glib/writer.h      |  4 ++++
 c_glib/test/test-file-writer.rb | 33 +++++++++++++++++++++++++++++++++
 3 files changed, 62 insertions(+)

diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp
index 7d3b59457fd33..9bcda2dbbfd2c 100644
--- a/c_glib/arrow-glib/writer.cpp
+++ b/c_glib/arrow-glib/writer.cpp
@@ -25,6 +25,7 @@
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
+#include <arrow-glib/table.hpp>
 
 #include <arrow-glib/output-stream.hpp>
 
@@ -165,6 +166,30 @@ garrow_record_batch_writer_write_record_batch(GArrowRecordBatchWriter *writer,
                             "[record-batch-writer][write-record-batch]");
 }
 
+/**
+ * garrow_record_batch_writer_write_table:
+ * @writer: A #GArrowRecordBatchWriter.
+ * @table: The table to be written.
+ * @error: (nullable): Return locatipcn for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 0.8.0
+ */
+gboolean
+garrow_record_batch_writer_write_table(GArrowRecordBatchWriter *writer,
+                                       GArrowTable *table,
+                                       GError **error)
+{
+  auto arrow_writer = garrow_record_batch_writer_get_raw(writer);
+  auto arrow_table = garrow_table_get_raw(table);
+
+  auto status = arrow_writer->WriteTable(*arrow_table);
+  return garrow_error_check(error,
+                            status,
+                            "[record-batch-writer][write-table]");
+}
+
 /**
  * garrow_record_batch_writer_close:
  * @writer: A #GArrowRecordBatchWriter.
diff --git a/c_glib/arrow-glib/writer.h b/c_glib/arrow-glib/writer.h
index 3853c2bd72c50..41b5f723d10eb 100644
--- a/c_glib/arrow-glib/writer.h
+++ b/c_glib/arrow-glib/writer.h
@@ -77,6 +77,10 @@ gboolean garrow_record_batch_writer_write_record_batch(
   GArrowRecordBatchWriter *writer,
   GArrowRecordBatch *record_batch,
   GError **error);
+gboolean garrow_record_batch_writer_write_table(
+  GArrowRecordBatchWriter *writer,
+  GArrowTable *table,
+  GError **error);
 gboolean garrow_record_batch_writer_close(
   GArrowRecordBatchWriter *writer,
   GError **error);
diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb
index 6ba5c7aebca44..3de8e5cf34b51 100644
--- a/c_glib/test/test-file-writer.rb
+++ b/c_glib/test/test-file-writer.rb
@@ -16,6 +16,8 @@
 # under the License.
 
 class TestFileWriter < Test::Unit::TestCase
+  include Helper::Buildable
+
   def test_write_record_batch
     tempfile = Tempfile.open("arrow-ipc-file-writer")
     output = Arrow::FileOutputStream.new(tempfile.path, false)
@@ -42,4 +44,35 @@ def test_write_record_batch
       input.close
     end
   end
+
+  def test_write_table
+    tempfile = Tempfile.open("arrow-ipc-file-writer")
+    output = Arrow::FileOutputStream.new(tempfile.path, false)
+
+    array = build_boolean_array([true, false, true])
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+    column = Arrow::Column.new(field, array)
+
+    begin
+      file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
+      begin
+        table = Arrow::Table.new(schema, [column])
+        file_writer.write_table(table)
+      ensure
+        file_writer.close
+      end
+    ensure
+      output.close
+    end
+
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      file_reader = Arrow::RecordBatchFileReader.new(input)
+      assert_equal(Arrow::RecordBatch.new(schema, array.length, [array]),
+                   file_reader.read_record_batch(0))
+    ensure
+      input.close
+    end
+  end
 end

From 9fb806ce2ca0ccdee1b89c510dcfae16996cf243 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Wed, 15 Nov 2017 13:23:51 +0100
Subject: [PATCH 1245/1644] ARROW-1811: [C++/Python] Rename all Decimal based
 APIs to Decimal128

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1321 from cpcloud/ARROW-1811 and squashes the following commits:

1b0a3b8 [Phillip Cloud] ARROW-1811: [C++/Python] Rename all Decimal based APIs to Decimal128
---
 cpp/src/arrow/array.cc                  | 2 +-
 cpp/src/arrow/array.h                   | 2 +-
 cpp/src/arrow/builder.cc                | 9 +++++----
 cpp/src/arrow/builder.h                 | 8 +++++---
 cpp/src/arrow/compare.cc                | 4 ++--
 cpp/src/arrow/ipc/json-internal.cc      | 8 ++++----
 cpp/src/arrow/ipc/metadata-internal.cc  | 2 +-
 cpp/src/arrow/python/arrow_to_pandas.cc | 4 ++--
 cpp/src/arrow/python/builtin_convert.cc | 2 +-
 cpp/src/arrow/python/numpy_to_arrow.cc  | 6 +++---
 cpp/src/arrow/type.cc                   | 6 +++---
 cpp/src/arrow/type.h                    | 6 ++++--
 cpp/src/arrow/type_fwd.h                | 4 ++--
 cpp/src/arrow/type_traits.h             | 4 ++--
 cpp/src/arrow/visitor.h                 | 2 +-
 cpp/src/arrow/visitor_inline.h          | 2 +-
 16 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 651fa26ba9026..f2dd75335df8b 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -320,7 +320,7 @@ Decimal128Array::Decimal128Array(const std::shared_ptr<ArrayData>& data)
 }
 
 std::string Decimal128Array::FormatValue(int64_t i) const {
-  const auto& type_ = static_cast<const DecimalType&>(*type());
+  const auto& type_ = static_cast<const Decimal128Type&>(*type());
   const Decimal128 value(GetValue(i));
   return value.ToString(type_.scale());
 }
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 3337e4b158267..28756a6abdafe 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -562,7 +562,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 // Decimal128Array
 class ARROW_EXPORT Decimal128Array : public FixedSizeBinaryArray {
  public:
-  using TypeClass = DecimalType;
+  using TypeClass = Decimal128Type;
 
   using FixedSizeBinaryArray::FixedSizeBinaryArray;
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index c910170dd5c87..d2d3dbdf7d13e 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -1147,17 +1147,18 @@ template class DictionaryBuilder<BinaryType>;
 template class DictionaryBuilder<StringType>;
 
 // ----------------------------------------------------------------------
-// DecimalBuilder
+// Decimal128Builder
 
-DecimalBuilder::DecimalBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+Decimal128Builder::Decimal128Builder(const std::shared_ptr<DataType>& type,
+                                     MemoryPool* pool)
     : FixedSizeBinaryBuilder(type, pool) {}
 
-Status DecimalBuilder::Append(const Decimal128& value) {
+Status Decimal128Builder::Append(const Decimal128& value) {
   RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
   return FixedSizeBinaryBuilder::Append(value.ToBytes());
 }
 
-Status DecimalBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
+Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index c580eeb3b35fb..bc25d0d211934 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -753,10 +753,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
   BufferBuilder byte_builder_;
 };
 
-class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
+class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
  public:
-  explicit DecimalBuilder(const std::shared_ptr<DataType>& type,
-                          MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
+  explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
+                             MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
 
   using FixedSizeBinaryBuilder::Append;
 
@@ -765,6 +765,8 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 };
 
+using DecimalBuilder = Decimal128Builder;
+
 // ----------------------------------------------------------------------
 // Struct
 
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 253c2e1feb801..72ef122e3481e 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -615,8 +615,8 @@ class TypeEqualsVisitor {
     return Status::OK();
   }
 
-  Status Visit(const DecimalType& left) {
-    const auto& right = static_cast<const DecimalType&>(right_);
+  Status Visit(const Decimal128Type& left) {
+    const auto& right = static_cast<const Decimal128Type&>(right_);
     result_ = left.precision() == right.precision() && left.scale() == right.scale();
     return Status::OK();
   }
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 976f9660a1b78..bdf1ef52b40eb 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -253,7 +253,7 @@ class SchemaWriter {
     writer_->Int(type.byte_width());
   }
 
-  void WriteTypeMetadata(const DecimalType& type) {
+  void WriteTypeMetadata(const Decimal128Type& type) {
     writer_->Key("precision");
     writer_->Int(type.precision());
     writer_->Key("scale");
@@ -347,7 +347,7 @@ class SchemaWriter {
     return WritePrimitive("fixedsizebinary", type);
   }
 
-  Status Visit(const DecimalType& type) { return WritePrimitive("decimal", type); }
+  Status Visit(const Decimal128Type& type) { return WritePrimitive("decimal", type); }
   Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); }
   Status Visit(const IntervalType& type) { return WritePrimitive("interval", type); }
 
@@ -1063,7 +1063,7 @@ class ArrayReader {
 
   template <typename T>
   typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value &&
-                              !std::is_base_of<DecimalType, T>::value,
+                              !std::is_base_of<Decimal128Type, T>::value,
                           Status>::type
   Visit(const T& type) {
     typename TypeTraits<T>::BuilderType builder(type_, pool_);
@@ -1105,7 +1105,7 @@ class ArrayReader {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<DecimalType, T>::value, Status>::type Visit(
+  typename std::enable_if<std::is_base_of<Decimal128Type, T>::value, Status>::type Visit(
       const T& type) {
     typename TypeTraits<T>::BuilderType builder(type_, pool_);
 
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 63ef8a549f236..87b4708bf6346 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -458,7 +458,7 @@ static Status TypeToFlatbuffer(FBB& fbb, const DataType& type,
       *offset = flatbuf::CreateTimestamp(fbb, fb_unit, fb_timezone).Union();
     } break;
     case Type::DECIMAL: {
-      const auto& dec_type = static_cast<const DecimalType&>(*value_type);
+      const auto& dec_type = static_cast<const Decimal128Type&>(*value_type);
       *out_type = flatbuf::Type_Decimal;
       *offset =
           flatbuf::CreateDecimal(fbb, dec_type.precision(), dec_type.scale()).Union();
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index f966b2987b93f..2b0f964391cb1 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -634,7 +634,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
 
   for (int c = 0; c < data.num_chunks(); c++) {
     auto* arr(static_cast<arrow::Decimal128Array*>(data.chunk(c).get()));
-    auto type(std::dynamic_pointer_cast<arrow::DecimalType>(arr->type()));
+    auto type(std::dynamic_pointer_cast<arrow::Decimal128Type>(arr->type()));
     const int scale = type->scale();
 
     for (int64_t i = 0; i < arr->length(); ++i) {
@@ -1600,7 +1600,7 @@ class ArrowDeserializer {
     return VisitObjects(ConvertFixedSizeBinary);
   }
 
-  Status Visit(const DecimalType& type) { return VisitObjects(ConvertDecimals); }
+  Status Visit(const Decimal128Type& type) { return VisitObjects(ConvertDecimals); }
 
   Status Visit(const Time32Type& type) { return VisitObjects(ConvertTimes<Time32Type>); }
 
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 0e775a0fb0e99..fa0098bdf63e9 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -700,7 +700,7 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
 };
 
 class DecimalConverter
-    : public TypedConverterVisitor<arrow::DecimalBuilder, DecimalConverter> {
+    : public TypedConverterVisitor<arrow::Decimal128Builder, DecimalConverter> {
  public:
   inline Status AppendItem(const OwnedRef& item) {
     /// TODO(phillipc): Check for nan?
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index c5aff2e4f2e3a..a6c28af9b7443 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -305,7 +305,7 @@ class NumPyConverter {
     return TypeNotImplemented(type.ToString());
   }
 
-  Status Visit(const DecimalType& type) { return TypeNotImplemented(type.ToString()); }
+  Status Visit(const Decimal128Type& type) { return TypeNotImplemented(type.ToString()); }
 
   Status Visit(const DictionaryType& type) { return TypeNotImplemented(type.ToString()); }
 
@@ -669,9 +669,9 @@ Status NumPyConverter::ConvertDecimals() {
 
   RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(object, &precision, &scale));
 
-  type_ = std::make_shared<DecimalType>(precision, scale);
+  type_ = std::make_shared<Decimal128Type>(precision, scale);
 
-  DecimalBuilder builder(type_, pool_);
+  Decimal128Builder builder(type_, pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
   for (int64_t i = 0; i < length_; ++i) {
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 0d1985fb2d914..ae17118323450 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -373,7 +373,7 @@ ACCEPT_VISITOR(FixedSizeBinaryType);
 ACCEPT_VISITOR(StringType);
 ACCEPT_VISITOR(ListType);
 ACCEPT_VISITOR(StructType);
-ACCEPT_VISITOR(DecimalType);
+ACCEPT_VISITOR(Decimal128Type);
 ACCEPT_VISITOR(UnionType);
 ACCEPT_VISITOR(Date32Type);
 ACCEPT_VISITOR(Date64Type);
@@ -471,7 +471,7 @@ std::shared_ptr<Field> field(const std::string& name,
 }
 
 std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale) {
-  return std::make_shared<DecimalType>(precision, scale);
+  return std::make_shared<Decimal128Type>(precision, scale);
 }
 
 static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1);
@@ -511,7 +511,7 @@ std::vector<BufferDescr> UnionType::GetBufferLayout() const {
   }
 }
 
-std::string DecimalType::ToString() const {
+std::string Decimal128Type::ToString() const {
   std::stringstream s;
   s << "decimal(" << precision_ << ", " << scale_ << ")";
   return s.str();
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 9e11a034420e5..d86e7ef57bec6 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -498,11 +498,11 @@ class ARROW_EXPORT StructType : public NestedType {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
+class ARROW_EXPORT Decimal128Type : public FixedSizeBinaryType {
  public:
   static constexpr Type::type type_id = Type::DECIMAL;
 
-  explicit DecimalType(int32_t precision, int32_t scale)
+  explicit Decimal128Type(int32_t precision, int32_t scale)
       : FixedSizeBinaryType(16, Type::DECIMAL), precision_(precision), scale_(scale) {}
 
   Status Accept(TypeVisitor* visitor) const override;
@@ -517,6 +517,8 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
   int32_t scale_;
 };
 
+using DecimalType = Decimal128Type;
+
 struct UnionMode {
   enum type { SPARSE, DENSE };
 };
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index 343487055c6fc..b8b3c5aa5adb7 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -68,9 +68,9 @@ class StructType;
 class StructArray;
 class StructBuilder;
 
-class DecimalType;
+class Decimal128Type;
 class Decimal128Array;
-class DecimalBuilder;
+class Decimal128Builder;
 
 class UnionType;
 class UnionArray;
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index bc600372eb01f..6707f3759581d 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -230,9 +230,9 @@ struct TypeTraits<DoubleType> {
 };
 
 template <>
-struct TypeTraits<DecimalType> {
+struct TypeTraits<Decimal128Type> {
   using ArrayType = Decimal128Array;
-  using BuilderType = DecimalBuilder;
+  using BuilderType = Decimal128Builder;
   constexpr static bool is_parameter_free = false;
 };
 
diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h
index 34679eb950d4b..030ffc85744e4 100644
--- a/cpp/src/arrow/visitor.h
+++ b/cpp/src/arrow/visitor.h
@@ -83,7 +83,7 @@ class ARROW_EXPORT TypeVisitor {
   virtual Status Visit(const Time64Type& type);
   virtual Status Visit(const TimestampType& type);
   virtual Status Visit(const IntervalType& type);
-  virtual Status Visit(const DecimalType& type);
+  virtual Status Visit(const Decimal128Type& type);
   virtual Status Visit(const ListType& type);
   virtual Status Visit(const StructType& type);
   virtual Status Visit(const UnionType& type);
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index 5ecabd2a4fbb1..72c82a3bb871b 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -55,7 +55,7 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
     TYPE_VISIT_INLINE(TimestampType);
     TYPE_VISIT_INLINE(Time32Type);
     TYPE_VISIT_INLINE(Time64Type);
-    TYPE_VISIT_INLINE(DecimalType);
+    TYPE_VISIT_INLINE(Decimal128Type);
     TYPE_VISIT_INLINE(ListType);
     TYPE_VISIT_INLINE(StructType);
     TYPE_VISIT_INLINE(UnionType);

From 725546059134c1895742aca131f486e909b58ce8 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Wed, 15 Nov 2017 13:25:17 +0100
Subject: [PATCH 1246/1644] ARROW-1810: [Plasma] Remove unused Plasma test
 shell scripts

These tests are run via CMake, so it is confusing to have the scripts in the repo.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1320 from pcmoritz/remove-test-sh and squashes the following commits:

ed6fec4 [Philipp Moritz] removed unused test shell scripts
---
 cpp/src/plasma/test/run_tests.sh    | 61 -----------------------------
 cpp/src/plasma/test/run_valgrind.sh | 27 -------------
 2 files changed, 88 deletions(-)
 delete mode 100644 cpp/src/plasma/test/run_tests.sh
 delete mode 100644 cpp/src/plasma/test/run_valgrind.sh

diff --git a/cpp/src/plasma/test/run_tests.sh b/cpp/src/plasma/test/run_tests.sh
deleted file mode 100644
index 958bd08398e23..0000000000000
--- a/cpp/src/plasma/test/run_tests.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Cause the script to exit if a single command fails.
-set -e
-
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
-sleep 1
-./src/plasma/manager_tests
-killall plasma_store
-./src/plasma/serialization_tests
-
-# Start the Redis shards.
-./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
-redis_pid1=$!
-./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
-redis_pid2=$!
-sleep 1
-
-# Flush the redis server
-./src/common/thirdparty/redis/src/redis-cli flushall
-# Register the shard location with the primary shard.
-./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
-./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
-sleep 1
-./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
-plasma1_pid=$!
-./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
-plasma2_pid=$!
-./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 &
-plasma3_pid=$!
-./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
-plasma4_pid=$!
-sleep 1
-
-./src/plasma/client_tests
-
-kill $plasma4_pid
-kill $plasma3_pid
-kill $plasma2_pid
-kill $plasma1_pid
-kill $redis_pid1
-wait $redis_pid1
-kill $redis_pid2
-wait $redis_pid2
diff --git a/cpp/src/plasma/test/run_valgrind.sh b/cpp/src/plasma/test/run_valgrind.sh
deleted file mode 100644
index 0472194128679..0000000000000
--- a/cpp/src/plasma/test/run_valgrind.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Cause the script to exit if a single command fails.
-set -e
-
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
-sleep 1
-valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests
-killall plasma_store
-valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests

From 1d951b5d2799b08b9ce81902e525390054b05fe4 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 15 Nov 2017 13:26:28 +0100
Subject: [PATCH 1247/1644] ARROW-1809: [GLib] Use .xml instead of .sgml for
 GTK-Doc main file

.sgml is old extension:

https://developer.gnome.org/gtk-doc-manual/unstable/settingup_firstrun.html.en

> `<package>-docs.xml` (in the past `.sgml`)

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1318 from kou/glib-doc-use-xml and squashes the following commits:

138c199 [Kouhei Sutou] [GLib] Use .xml instead of .sgml for GTK-Doc main file
---
 c_glib/doc/reference/Makefile.am                                | 2 +-
 .../doc/reference/{arrow-glib-docs.sgml => arrow-glib-docs.xml} | 0
 c_glib/doc/reference/meson.build                                | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename c_glib/doc/reference/{arrow-glib-docs.sgml => arrow-glib-docs.xml} (100%)

diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am
index 896aff544d454..4c005c237b300 100644
--- a/c_glib/doc/reference/Makefile.am
+++ b/c_glib/doc/reference/Makefile.am
@@ -20,7 +20,7 @@ SUBDIRS =                                       \
 
 DOC_MODULE = arrow-glib
 
-DOC_MAIN_SGML_FILE = $(DOC_MODULE)-docs.sgml
+DOC_MAIN_SGML_FILE = $(DOC_MODULE)-docs.xml
 
 DOC_SOURCE_DIR =				\
 	$(top_srcdir)/arrow-glib
diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.xml
similarity index 100%
rename from c_glib/doc/reference/arrow-glib-docs.sgml
rename to c_glib/doc/reference/arrow-glib-docs.xml
diff --git a/c_glib/doc/reference/meson.build b/c_glib/doc/reference/meson.build
index 4c9552e83c303..3374fbde5b9ed 100644
--- a/c_glib/doc/reference/meson.build
+++ b/c_glib/doc/reference/meson.build
@@ -49,7 +49,7 @@ if arrow_gpu_dependency.found()
   ]
 endif
 gnome.gtkdoc(meson.project_name(),
-             main_xml: meson.project_name() + '-docs.sgml',
+             main_xml: meson.project_name() + '-docs.xml',
              src_dir: source_directories,
              dependencies: dependencies,
              gobject_typesfile: meson.project_name() + '.types',

From 9812aea13e810626f4f5b74f1ca4ecbb843578e9 Mon Sep 17 00:00:00 2001
From: Stephanie <swang@cs.berkeley.edu>
Date: Wed, 15 Nov 2017 10:51:45 -0500
Subject: [PATCH 1248/1644] ARROW-1812: [C++] Plasma store modifies hash table
 while iterating during client disconnect

Author: Stephanie <swang@cs.berkeley.edu>

Closes #1322 from stephanie-wang/plasma-disconnect-bug and squashes the following commits:

cf1872d8 [Stephanie] Fix disconnect client memory error
8929b6bc [Stephanie] Introduce more complicated test for multiple Plasma clients
---
 cpp/src/plasma/store.cc             | 10 +++++-
 cpp/src/plasma/test/client_tests.cc | 53 +++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 31033ccbb3202..7094aed6f8e86 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -460,14 +460,22 @@ void PlasmaStore::disconnect_client(int client_fd) {
   ARROW_LOG(INFO) << "Disconnecting client on fd " << client_fd;
   // If this client was using any objects, remove it from the appropriate
   // lists.
+  // TODO(swang): Avoid iteration through the object table.
   auto client = it->second.get();
+  std::vector<ObjectID> unsealed_objects;
   for (const auto& entry : store_info_.objects) {
     if (entry.second->state == PLASMA_SEALED) {
       remove_client_from_object_clients(entry.second.get(), client);
     } else {
-      abort_object(entry.first, client);
+      // Add unsealed objects to a temporary list of object IDs. Do not perform
+      // the abort here, since it potentially modifies the object table.
+      unsealed_objects.push_back(entry.first);
     }
   }
+  // If the client was creating any objects, abort them.
+  for (const auto& entry : unsealed_objects) {
+    abort_object(entry, client);
+  }
 
   // Note, the store may still attempt to send a message to the disconnected
   // client (for example, when an object ID that the client was waiting for
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index d4285f8988404..f44ed2510c582 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -209,6 +209,59 @@ TEST_F(TestPlasmaStore, MultipleClientTest) {
   ASSERT_EQ(has_object, true);
 }
 
+TEST_F(TestPlasmaStore, ManyObjectTest) {
+  // Create many objects on the first client. Seal one third, abort one third,
+  // and leave the last third unsealed.
+  std::vector<ObjectID> object_ids;
+  for (int i = 0; i < 100; i++) {
+    ObjectID object_id = ObjectID::from_random();
+    object_ids.push_back(object_id);
+
+    // Test for object non-existence on the first client.
+    bool has_object;
+    ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+    ASSERT_EQ(has_object, false);
+
+    // Test for the object being in local Plasma store.
+    // First create and seal object on the first client.
+    int64_t data_size = 100;
+    uint8_t metadata[] = {5};
+    int64_t metadata_size = sizeof(metadata);
+    uint8_t* data;
+    ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+
+    if (i % 3 == 0) {
+      // Seal one third of the objects.
+      ARROW_CHECK_OK(client_.Seal(object_id));
+      // Test that the first client can get the object.
+      ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+      ASSERT_EQ(has_object, true);
+    } else if (i % 3 == 1) {
+      // Abort one third of the objects.
+      ARROW_CHECK_OK(client_.Release(object_id));
+      ARROW_CHECK_OK(client_.Abort(object_id));
+    }
+  }
+  // Disconnect the first client. All unsealed objects should be aborted.
+  ARROW_CHECK_OK(client_.Disconnect());
+
+  // Check that the second client can query the object store for the first
+  // client's objects.
+  int i = 0;
+  for (auto const& object_id : object_ids) {
+    bool has_object;
+    ARROW_CHECK_OK(client2_.Contains(object_id, &has_object));
+    if (i % 3 == 0) {
+      // The first third should be sealed.
+      ASSERT_EQ(has_object, true);
+    } else {
+      // The rest were aborted, so the object is not in the store.
+      ASSERT_EQ(has_object, false);
+    }
+    i++;
+  }
+}
+
 }  // namespace plasma
 
 int main(int argc, char** argv) {

From 42353ba993314e6bba963aae50861566150f96eb Mon Sep 17 00:00:00 2001
From: Siddharth Teotia <siddharth@dremio.com>
Date: Sat, 14 Oct 2017 11:38:30 -0400
Subject: [PATCH 1249/1644] ARROW-1473: ValueVector new hierarchy prototype
 (implementation phase 1)

Close #1164
Close #1198

Change-Id: If18e42d2edfdfef83e83621334a5b65a390e9db9
---
 .../codegen/templates/ComplexReaders.java     |  24 +-
 .../codegen/templates/ComplexWriters.java     |  45 +-
 .../templates/NullableValueVectors.java       | 542 ++++++++++--
 .../main/codegen/templates/UnionVector.java   |  18 +-
 .../vector/BaseNullableFixedWidthVector.java  | 701 ++++++++++++++++
 .../BaseNullableVariableWidthVector.java      | 764 +++++++++++++++++
 .../apache/arrow/vector/BaseValueVector.java  |  20 +
 .../apache/arrow/vector/BitVectorHelper.java  |  60 ++
 .../arrow/vector/NullableIntVector.java       | 299 +++++++
 .../arrow/vector/NullableVarCharVector.java   | 451 ++++++++++
 .../org/apache/arrow/vector/ValueVector.java  |   6 +
 .../apache/arrow/vector/VectorUnloader.java   |  15 +-
 .../org/apache/arrow/vector/ZeroVector.java   |   9 +
 .../vector/complex/FixedSizeListVector.java   |  27 +-
 .../arrow/vector/complex/ListVector.java      |  23 +-
 .../arrow/vector/complex/MapVector.java       |  33 +-
 .../vector/file/json/JsonFileReader.java      |   2 +-
 .../vector/TestBufferOwnershipTransfer.java   |   6 +-
 .../arrow/vector/TestDictionaryVector.java    |  59 +-
 .../arrow/vector/TestFixedSizeListVector.java |  13 +-
 .../arrow/vector/TestSplitAndTransfer.java    |  15 +-
 .../apache/arrow/vector/TestValueVector.java  | 788 +++++++++++++-----
 .../arrow/vector/TestVectorReAlloc.java       |  11 +-
 .../arrow/vector/TestVectorUnloadLoad.java    | 104 ++-
 .../arrow/vector/file/BaseFileTest.java       |  70 +-
 .../arrow/vector/file/TestArrowFile.java      |   2 +-
 26 files changed, 3658 insertions(+), 449 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java

diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 38cd1bfdeb3c5..791064905156d 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -70,7 +70,11 @@ public Field getField(){
   
   public boolean isSet(){
     <#if nullMode == "Nullable">
-    return !vector.getAccessor().isNull(idx());
+      <#if minor.class != "Int" && minor.class != "VarChar">
+        return !vector.getAccessor().isNull(idx());
+      <#else>
+        return !vector.isNull(idx());
+      </#if>
     <#else>
     return true;
     </#if>
@@ -93,11 +97,19 @@ public void read(${minor.class?cap_first}Holder h){
   </#if>
 
   public void read(Nullable${minor.class?cap_first}Holder h){
-    vector.getAccessor().get(idx(), h);
+    <#if minor.class != "Int" && minor.class != "VarChar">
+      vector.getAccessor().get(idx(), h);
+    <#else>
+      vector.get(idx(), h);
+    </#if>
   }
   
   public ${friendlyType} read${safeType}(){
-    return vector.getAccessor().getObject(idx());
+    <#if minor.class == "Int" || minor.class == "VarChar">
+      return vector.getObject(idx());
+    <#else>
+      return vector.getAccessor().getObject(idx());
+    </#if>
   }
 
   <#if minor.class == "TimeStampSec" ||
@@ -115,7 +127,11 @@ public void copyValue(FieldWriter w){
   }
   
   public Object readObject(){
-    return vector.getAccessor().getObject(idx());
+    <#if minor.class == "Int" || minor.class == "VarChar">
+      return (Object)vector.getObject(idx());
+    <#else>
+      return vector.getAccessor().getObject(idx());
+    </#if>
   }
 }
 </#if>
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index fe099bede3568..77f6594a07804 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -39,11 +39,16 @@
 @SuppressWarnings("unused")
 public class ${eName}WriterImpl extends AbstractFieldWriter {
 
-  private final Nullable${name}Vector.Mutator mutator;
+  <#if minor.class != "Int" && minor.class != "VarChar">
+    private final Nullable${name}Vector.Mutator mutator;
+  </#if>
+
   final Nullable${name}Vector vector;
 
   public ${eName}WriterImpl(Nullable${name}Vector vector) {
-    this.mutator = vector.getMutator();
+    <#if minor.class != "Int" && minor.class != "VarChar">
+      this.mutator = vector.getMutator();
+    </#if>
     this.vector = vector;
   }
 
@@ -103,18 +108,33 @@ public void setPosition(int idx) {
   <#else>
 
   public void write(${minor.class}Holder h) {
-    mutator.setSafe(idx(), h);
-    vector.getMutator().setValueCount(idx()+1);
+    <#if minor.class != "Int" && minor.class != "VarChar">
+      mutator.setSafe(idx(), h);
+      vector.getMutator().setValueCount(idx()+1);
+    <#else>
+        vector.setSafe(idx(), h);
+        vector.setValueCount(idx()+1);
+    </#if>
   }
 
   public void write(Nullable${minor.class}Holder h) {
-    mutator.setSafe(idx(), h);
-    vector.getMutator().setValueCount(idx()+1);
+    <#if minor.class != "Int" && minor.class != "VarChar">
+      mutator.setSafe(idx(), h);
+      vector.getMutator().setValueCount(idx()+1);
+    <#else>
+      vector.setSafe(idx(), h);
+      vector.setValueCount(idx()+1);
+    </#if>
   }
 
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-    mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
-    vector.getMutator().setValueCount(idx()+1);
+    <#if minor.class != "Int" && minor.class != "VarChar">
+      mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
+      vector.getMutator().setValueCount(idx()+1);
+    <#else>
+      vector.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
+      vector.setValueCount(idx()+1);
+    </#if>
   }
   <#if minor.class == "Decimal">
 
@@ -126,8 +146,13 @@ public void write(Nullable${minor.class}Holder h) {
   <#if mode == "Nullable">
 
   public void writeNull() {
-    mutator.setNull(idx());
-    vector.getMutator().setValueCount(idx()+1);
+    <#if minor.class != "Int" && minor.class != "VarChar">
+        mutator.setNull(idx());
+        vector.getMutator().setValueCount(idx()+1);
+    <#else>
+        vector.setNull(idx());
+        vector.setValueCount(idx()+1);
+    </#if>
   }
   </#if>
   </#if>
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 122cd23648286..5d1f5a33b90f5 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -19,8 +19,14 @@
 <#list vv.types as type>
 <#list type.minor as minor>
 
+<#if minor.class == "Int" || minor.class == "VarChar">
+<#assign className = "LegacyNullable${minor.class}Vector" />
+<#assign valuesName = "Nullable${minor.class}Vector" />
+<#else>
 <#assign className = "Nullable${minor.class}Vector" />
 <#assign valuesName = "${minor.class}Vector" />
+</#if>
+
 <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
 
 <@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" />
@@ -44,15 +50,24 @@
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
 @SuppressWarnings("unused")
-public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector, FieldVector {
+<#if minor.class == "Int" || minor.class == "VarChar">
+@Deprecated
+</#if>
+public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, FieldVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
 protected final static byte[] emptyByteArray = new byte[]{};
+
+  <#if minor.class != "Int" && minor.class != "VarChar">
   private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this);
+  </#if>
 
   private final String bitsField = "$bits$";
   private final String valuesField = "$values$";
+
+  <#if minor.class != "Int" && minor.class != "VarChar">
   private final Field field;
+  </#if>
 
   final BitVector bits = new BitVector(bitsField, allocator);
   final ${valuesName} values;
@@ -60,7 +75,9 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
   private final Mutator mutator;
   private final Accessor accessor;
 
+  <#if minor.class != "Int" && minor.class != "VarChar">
   private final List<BufferBacked> innerVectors;
+  </#if>
 
   <#if minor.typeParams??>
     <#assign typeParams = minor.typeParams?reverse>
@@ -105,6 +122,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
     </#if>
     this.mutator = new Mutator();
     this.accessor = new Accessor();
+    <#if minor.class != "Int" && minor.class != "VarChar">
     this.field = new Field(name, fieldType, null);
     innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
         bits,
@@ -113,16 +131,24 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
         </#if>
         values
     ));
+    </#if>
   }
 
-  @Override
+  <#if minor.class != "Int" && minor.class != "VarChar">
+  /* not needed for new vectors */
   public BitVector getValidityVector() {
     return bits;
   }
+  </#if>
 
   @Override
   public List<BufferBacked> getFieldInnerVectors() {
-    return innerVectors;
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getFieldInnerVectors();
+    <#else>
+        return innerVectors;
+    </#if>
   }
 
   @Override
@@ -139,6 +165,10 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        values.loadFieldBuffers(fieldNode, ownBuffers);
+    <#else>
     <#if type.major = "VarLen">
     // variable width values: truncate offset vector buffer to size (#1)
     org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1,
@@ -151,34 +181,64 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
     </#if>
     org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
     bits.valueCount = fieldNode.getLength();
+    </#if>
   }
 
   public List<ArrowBuf> getFieldBuffers() {
-    return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors());
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getFieldBuffers();
+    <#else>
+        return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors());
+    </#if>
   }
 
   @Override
   public Field getField() {
-    return field;
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getField();
+    <#else>
+      return field;
+    </#if>
   }
 
   @Override
   public MinorType getMinorType() {
-    return MinorType.${minor.class?upper_case};
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getMinorType();
+    <#else>
+        return MinorType.${minor.class?upper_case};
+    </#if>
   }
 
   @Override
   public FieldReader getReader(){
-    return reader;
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getReader();
+    <#else>
+        return reader;
+    </#if>
   }
 
   @Override
   public int getValueCapacity(){
-    return Math.min(bits.getValueCapacity(), values.getValueCapacity());
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getValueCapacity();
+    <#else>
+        return Math.min(bits.getValueCapacity(), values.getValueCapacity());
+    </#if>
   }
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getBuffers(clear);
+    <#else>
     final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), ArrowBuf.class);
     if (clear) {
       for (final ArrowBuf buffer:buffers) {
@@ -187,25 +247,41 @@ public ArrowBuf[] getBuffers(boolean clear) {
       clear();
     }
     return buffers;
+    </#if>
   }
 
   @Override
   public void close() {
-    bits.close();
-    values.close();
-    super.close();
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        values.close();
+    <#else>
+        bits.close();
+        values.close();
+        super.close();
+    </#if>
   }
 
   @Override
   public void clear() {
-    bits.clear();
-    values.clear();
-    super.clear();
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        values.clear();
+    <#else>
+        bits.clear();
+        values.clear();
+        super.clear();
+    </#if>
   }
 
   @Override
   public int getBufferSize(){
-    return values.getBufferSize() + bits.getBufferSize();
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getBufferSize();
+    <#else>
+        return values.getBufferSize() + bits.getBufferSize();
+    </#if>
   }
 
   @Override
@@ -214,34 +290,52 @@ public int getBufferSizeFor(final int valueCount) {
       return 0;
     }
 
-    return values.getBufferSizeFor(valueCount)
-        + bits.getBufferSizeFor(valueCount);
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getBufferSizeFor(valueCount);
+    <#else>
+        return values.getBufferSizeFor(valueCount)
+          + bits.getBufferSizeFor(valueCount);
+    </#if>
   }
 
   public ArrowBuf getBuffer() {
     return values.getDataBuffer();
   }
 
-  @Override
   public ${valuesName} getValuesVector() {
     return values;
   }
 
   @Override
   public void setInitialCapacity(int numRecords) {
-    bits.setInitialCapacity(numRecords);
-    values.setInitialCapacity(numRecords);
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        values.setInitialCapacity(numRecords);
+    <#else>
+        bits.setInitialCapacity(numRecords);
+        values.setInitialCapacity(numRecords);
+    </#if>
   }
 
   @Override
   public void allocateNew() {
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        values.allocateNew();
+    <#else>
     if(!allocateNewSafe()){
       throw new OutOfMemoryException("Failure while allocating buffer.");
     }
+    </#if>
   }
 
   @Override
   public boolean allocateNewSafe() {
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.allocateNewSafe();
+    <#else>
     /* Boolean to keep track if all the memory allocations were successful
      * Used in the case of composite vectors when we need to allocate multiple
      * buffers for multiple vectors. If one of the allocations failed we need to
@@ -259,23 +353,38 @@ public boolean allocateNewSafe() {
     mutator.reset();
     accessor.reset();
     return success;
+    </#if>
   }
 
   @Override
   public void reAlloc() {
-    bits.reAlloc();
-    values.reAlloc();
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        values.reAlloc();
+    <#else>
+        bits.reAlloc();
+        values.reAlloc();
+    </#if>
   }
 
   public void reset() {
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        values.reset();
+    <#else>
     bits.zeroVector();
     mutator.reset();
     accessor.reset();
+    </#if>
   }
 
   <#if type.major == "VarLen">
   @Override
   public void allocateNew(int totalBytes, int valueCount) {
+    <#if minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        values.allocateNew(totalBytes, valueCount);
+    <#else>
     try {
       values.allocateNew(totalBytes, valueCount);
       bits.allocateNew(valueCount);
@@ -286,6 +395,7 @@ public void allocateNew(int totalBytes, int valueCount) {
     bits.zeroVector();
     mutator.reset();
     accessor.reset();
+    </#if>
   }
 
   @Override
@@ -301,6 +411,10 @@ public int getCurrentSizeInBytes(){
   <#else>
   @Override
   public void allocateNew(int valueCount) {
+    <#if minor.class == "Int">
+        /* DELEGATE TO NEW VECTOR */
+        values.allocateNew(valueCount);
+    <#else>
     try {
       values.allocateNew(valueCount);
       bits.allocateNew(valueCount);
@@ -311,6 +425,7 @@ public void allocateNew(int valueCount) {
     bits.zeroVector();
     mutator.reset();
     accessor.reset();
+    </#if>
   }
 
   /**
@@ -318,32 +433,86 @@ public void allocateNew(int valueCount) {
    */
   @Override
   public void zeroVector() {
-    bits.zeroVector();
-    values.zeroVector();
+    <#if minor.class == "Int">
+        /* DELEGATE TO NEW VECTOR */
+        values.zeroVector();
+    <#else>
+        bits.zeroVector();
+        values.zeroVector();
+    </#if>
   }
   </#if>
 
+
+
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getTransferPair(ref, allocator, callBack);
+    <#else>
         return getTransferPair(ref, allocator);
+    </#if>
   }
 
+
+
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator){
-    return new TransferImpl(name, allocator);
-
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getTransferPair(allocator);
+    <#else>
+        return new TransferImpl(name, allocator);
+    </#if>
   }
 
+
+
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    return new TransferImpl(ref, allocator);
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getTransferPair(ref, allocator);
+    <#else>
+        return new TransferImpl(ref, allocator);
+    </#if>
   }
 
+
+
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((${className}) to);
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.makeTransferPair(to);
+    <#else>
+        return new TransferImpl((${className}) to);
+    </#if>
+  }
+
+
+
+  <#if minor.class == "Int" || minor.class == "VarChar">
+  public void transferTo(${valuesName} target) {
+    /* DELEGATE TO NEW VECTOR */
+    <#if type.major == "VarLen">
+        values.transferTo((BaseNullableVariableWidthVector) target);
+    <#else>
+        values.transferTo((BaseNullableFixedWidthVector) target);
+    </#if>
+  }
+
+  public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) {
+    /* DELEGATE TO NEW VECTOR */
+    <#if type.major == "VarLen">
+        values.splitAndTransferTo(startIndex, length, (BaseNullableVariableWidthVector) target);
+    <#else>
+        values.splitAndTransferTo(startIndex, length, (BaseNullableFixedWidthVector) target);
+    </#if>
   }
 
+  <#else>
   public void transferTo(${className} target){
     bits.transferTo(target.bits);
     values.transferTo(target.values);
@@ -360,7 +529,11 @@ public void splitAndTransferTo(int startIndex, int length, ${className} target)
     target.mutator.lastSet = length - 1;
     </#if>
   }
+  </#if>
 
+
+
+  <#if minor.class != "Int" && minor.class != "VarChar">
   private class TransferImpl implements TransferPair {
     ${className} to;
 
@@ -392,6 +565,9 @@ public void copyValueSafe(int fromIndex, int toIndex) {
       to.copyFromSafe(fromIndex, toIndex, ${className}.this);
     }
   }
+  </#if>
+
+
 
   @Override
   public Accessor getAccessor(){
@@ -403,7 +579,20 @@ public Mutator getMutator(){
     return mutator;
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, ${className} from){
+
+
+  <#if minor.class == "Int" || minor.class == "VarChar">
+  public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) {
+    /* DELEGATE TO NEW VECTOR */
+    values.copyFrom(fromIndex, thisIndex, from);
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) {
+    /* DELEGATE TO NEW VECTOR */
+    values.copyFromSafe(fromIndex, thisIndex, from);
+  }
+  <#else>
+  public void copyFrom(int fromIndex, int thisIndex, ${className} from) {
     final Accessor fromAccessor = from.getAccessor();
     if (!fromAccessor.isNull(fromIndex)) {
       mutator.set(thisIndex, fromAccessor.get(fromIndex));
@@ -428,17 +617,28 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
     values.copyFromSafe(fromIndex, thisIndex, from.values);
     <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
   }
+  </#if>
 
   @Override
   public long getValidityBufferAddress() {
     /* address of the databuffer associated with the bitVector */
-    return (bits.getDataBuffer().memoryAddress());
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getValidityBufferAddress();
+    <#else>
+        return (bits.getDataBuffer().memoryAddress());
+    </#if>
   }
 
   @Override
   public long getDataBufferAddress() {
     /* address of the dataBuffer associated with the valueVector */
-    return (values.getDataBuffer().memoryAddress());
+    <#if minor.class == "Int" || minor.class == "VarChar">
+          /* DELEGATE TO NEW VECTOR */
+          return values.getDataBufferAddress();
+    <#else>
+          return (bits.getDataBuffer().memoryAddress());
+    </#if>
   }
 
   @Override
@@ -446,17 +646,26 @@ public long getOffsetBufferAddress() {
     /* address of the dataBuffer associated with the offsetVector
      * this operation is not supported for fixed-width vector types.
      */
-    <#if type.major != "VarLen">
-        throw new UnsupportedOperationException();
+    <#if minor.class == "Int" || minor.class == "VarChar">
+          /* DELEGATE TO NEW VECTOR */
+          return values.getOffsetBufferAddress();
     <#else>
-        return (values.getOffsetAddr());
+        <#if type.major != "VarLen">
+          throw new UnsupportedOperationException();
+        <#else>
+          return (values.getOffsetAddr());
+        </#if>
     </#if>
   }
 
   @Override
   public ArrowBuf getValidityBuffer() {
-    /* dataBuffer associated with the bitVector */
-    return (bits.getDataBuffer());
+    <#if minor.class == "Int" || minor.class == "VarChar">
+          /* DELEGATE TO NEW VECTOR */
+          return values.getValidityBuffer();
+    <#else>
+          return (bits.getDataBuffer());
+    </#if>
   }
 
   @Override
@@ -468,10 +677,15 @@ public ArrowBuf getDataBuffer() {
   @Override
   public ArrowBuf getOffsetBuffer() {
     /* dataBuffer associated with the offsetVector of the valueVector */
-    <#if type.major != "VarLen">
-        throw new UnsupportedOperationException();
+    <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getOffsetBuffer();
     <#else>
-        return (values.getOffsetBuffer());
+        <#if type.major != "VarLen">
+          throw new UnsupportedOperationException();
+        <#else>
+          return (values.getOffsetBuffer());
+        </#if>
     </#if>
   }
 
@@ -485,38 +699,80 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m
      * @param  index   position of the value
      * @return value of the element, if not null
      */
-    public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
-      if (isNull(index)) {
+    <#if minor.class == "Int" || minor.class == "VarChar">
+      public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+        /* DELEGATE TO NEW VECTOR */
+        return values.get(index);
+      }
+    <#else>
+
+      public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+        if (isNull(index)) {
           throw new IllegalStateException("Can't get a null value");
+        }
+        return vAccessor.get(index);
       }
-      return vAccessor.get(index);
-    }
+    </#if>
 
     @Override
     public boolean isNull(int index) {
-      return isSet(index) == 0;
+      <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.isNull(index);
+      <#else>
+        return isSet(index) == 0;
+      </#if>
     }
 
     public int isSet(int index){
-      return bAccessor.get(index);
+      <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.isSet(index);
+      <#else>
+        return bAccessor.get(index);
+      </#if>
     }
 
     <#if type.major == "VarLen">
     public long getStartEnd(int index){
-      return vAccessor.getStartEnd(index);
+      <#if minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getStartEnd(index);
+      <#else>
+        return vAccessor.getStartEnd(index);
+      </#if>
     }
 
     @Override
     public int getValueLength(int index) {
-      return values.getAccessor().getValueLength(index);
+      <#if minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getValueLength(index);
+      <#else>
+        return values.getAccessor().getValueLength(index);
+      </#if>
     }
     </#if>
 
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    public void get(int index, Nullable${minor.class}Holder holder){
+        /* DELEGATE TO NEW VECTOR */
+        values.get(index, holder);
+    }
+    <#else>
     public void get(int index, Nullable${minor.class}Holder holder){
       vAccessor.get(index, holder);
       holder.isSet = bAccessor.get(index);
     }
+    </#if>
 
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      /* DELEGATE TO NEW VECTOR */
+      return values.getObject(index);
+    }
+    <#else>
     @Override
     public ${friendlyType} getObject(int index) {
       if (isNull(index)) {
@@ -525,6 +781,7 @@ public void get(int index, Nullable${minor.class}Holder holder){
         return vAccessor.getObject(index);
       }
     }
+    </#if>
 
     <#if minor.class == "IntervalYear" || minor.class == "IntervalDay">
     public StringBuilder getAsStringBuilder(int index) {
@@ -538,7 +795,12 @@ public StringBuilder getAsStringBuilder(int index) {
 
     @Override
     public int getValueCount(){
-      return bits.getAccessor().getValueCount();
+      <#if minor.class == "Int" || minor.class == "VarChar">
+        /* DELEGATE TO NEW VECTOR */
+        return values.getValueCount();
+      <#else>
+        return bits.getAccessor().getValueCount();
+      </#if>
     }
 
     public void reset(){}
@@ -551,21 +813,35 @@ public final class Mutator extends BaseDataValueVector.BaseMutator implements Nu
     private Mutator(){
     }
 
-    public ${valuesName} getVectorWithValues(){
+    public ${valuesName} getVectorWithValues() {
       return values;
     }
 
+
     @Override
     public void setIndexDefined(int index){
+      <#if minor.class == "Int" || minor.class == "VarChar">
+      values.setIndexDefined(index);
+      <#else>
       bits.getMutator().setToOne(index);
+      </#if>
     }
 
+
+
     /**
      * Set the variable length element at the specified index to the supplied byte array.
      *
      * @param index   position of the bit to set
      * @param value   array of bytes (or int if smaller than 4 bytes) to write
      */
+
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+       /* DELEGATE TO NEW VECTOR */
+       values.set(index, value);
+    }
+    <#else>
     public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
       setCount++;
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
@@ -579,9 +855,24 @@ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.widt
       valuesMutator.set(index, value);
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
+    </#if>
+
+
 
     <#if type.major == "VarLen">
+    <#if minor.class == "VarChar">
+    public void fillEmpties(int index) {
+      /* DELEGATE TO NEW VECTOR */
+      values.fillEmpties(index);
+    }
 
+    @Override
+    public void setValueLengthSafe(int index, int length) {
+      /* DELEGATE TO NEW VECTOR */
+      values.setValueLengthSafe(index, length);
+    }
+
+    <#else>
     public void fillEmpties(int index){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       for (int i = lastSet + 1; i < index; i++) {
@@ -599,7 +890,16 @@ public void setValueLengthSafe(int index, int length) {
       lastSet = index;
     }
     </#if>
+    </#if>
 
+
+
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    public void setSafe(int index, byte[] value, int start, int length) {
+       /* DELEGATE TO NEW VECTOR */
+      values.setSafe(index, value, start, length);
+    }
+    <#else>
     public void setSafe(int index, byte[] value, int start, int length) {
       <#if type.major != "VarLen">
       throw new UnsupportedOperationException();
@@ -612,7 +912,16 @@ public void setSafe(int index, byte[] value, int start, int length) {
       <#if type.major == "VarLen">lastSet = index;</#if>
       </#if>
     }
+    </#if>
+
+
 
+    <#if minor.class == "VarChar">
+    public void setSafe(int index, ByteBuffer value, int start, int length) {
+       /* DELEGATE TO NEW VECTOR */
+       values.setSafe(index, value, start, length);
+    }
+    <#else>
     public void setSafe(int index, ByteBuffer value, int start, int length) {
       <#if type.major != "VarLen">
       throw new UnsupportedOperationException();
@@ -625,11 +934,25 @@ public void setSafe(int index, ByteBuffer value, int start, int length) {
       <#if type.major == "VarLen">lastSet = index;</#if>
       </#if>
     }
+    </#if>
 
+
+
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    public void setNull(int index) {
+       /* DELEGATE TO NEW VECTOR */
+       values.setNull(index);
+    }
+    <#else>
     public void setNull(int index){
       bits.getMutator().setSafe(index, 0);
     }
+    </#if>
+
+
 
+    <#if minor.class != "Int" && minor.class != "VarChar">
+    /* these methods are probably not needed */
     public void setSkipNull(int index, ${minor.class}Holder holder){
       values.getMutator().set(index, holder);
     }
@@ -637,8 +960,17 @@ public void setSkipNull(int index, ${minor.class}Holder holder){
     public void setSkipNull(int index, Nullable${minor.class}Holder holder){
       values.getMutator().set(index, holder);
     }
+    </#if>
+
 
-    public void set(int index, Nullable${minor.class}Holder holder){
+
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    public void set(int index, Nullable${minor.class}Holder holder) {
+      /* DELEGATE TO NEW VECTOR */
+      values.set(index, holder);
+    }
+    <#else>
+    public void set(int index, Nullable${minor.class}Holder holder) {
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -649,8 +981,17 @@ public void set(int index, Nullable${minor.class}Holder holder){
       valuesMutator.set(index, holder);
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
+    </#if>
+
+
 
-    public void set(int index, ${minor.class}Holder holder){
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    public void set(int index, ${minor.class}Holder holder) {
+        /* DELEGATE TO NEW VECTOR */
+        values.set(index, holder);
+    }
+    <#else>
+    public void set(int index, ${minor.class}Holder holder) {
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -661,11 +1002,45 @@ public void set(int index, ${minor.class}Holder holder){
       valuesMutator.set(index, holder);
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
+    </#if>
 
+
+
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    public boolean isSafe(int outIndex) {
+       /* DELEGATE TO NEW VECTOR */
+       return values.isSafe(outIndex);
+    }
+    <#else>
     public boolean isSafe(int outIndex) {
       return outIndex < ${className}.this.getValueCapacity();
     }
+    </#if>
+
+
 
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    <#if minor.class == "Int">
+    public void set(int index, int isSet, int valueField) {
+      /* DELEGATE TO NEW VECTOR */
+      values.set(index, isSet, valueField);
+    }
+    public void setSafe(int index, int isSet, int valueField) {
+      /* DELEGATE TO NEW VECTOR */
+      values.setSafe(index, isSet, valueField);
+    }
+    </#if>
+    <#if minor.class == "VarChar">
+    public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) {
+      /* DELEGATE TO NEW VECTOR */
+      values.set(index, isSet, startField, endField, bufferField);
+    }
+    public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) {
+        /* DELEGATE TO NEW VECTOR */
+        values.setSafe(index, isSet, startField, endField, bufferField);
+    }
+    </#if>
+    <#else>
     <#assign fields = minor.fields!type.fields />
     public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field</#list> ){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
@@ -688,8 +1063,21 @@ public void setSafe(int index, int isSet<#list fields as field><#if field.includ
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
+    </#if>
+
 
 
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    public void setSafe(int index, Nullable${minor.class}Holder value) {
+      /* DELEGATE TO NEW VECTOR */
+      values.setSafe(index, value);
+    }
+
+    public void setSafe(int index, ${minor.class}Holder value) {
+      /* DELEGATE TO NEW VECTOR */
+      values.setSafe(index, value);
+    }
+    <#else>
     public void setSafe(int index, Nullable${minor.class}Holder value) {
       <#if type.major == "VarLen">
       fillEmpties(index);
@@ -709,15 +1097,25 @@ public void setSafe(int index, ${minor.class}Holder value) {
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
+    </#if>
+
+
 
     <#if !(type.major == "VarLen" || minor.class == "IntervalDay")>
     public void setSafe(int index, ${minor.javaType!type.javaType} value) {
+      <#if minor.class == "Int">
+        /* DELEGATE TO NEW VECTOR */
+        values.setSafe(index, value);
+      <#else>
       bits.getMutator().setSafeToOne(index);
       values.getMutator().setSafe(index, value);
       setCount++;
+      </#if>
     }
-
     </#if>
+
+
+
     <#if minor.class == "Decimal">
     public void set(int index, ${friendlyType} value) {
       bits.getMutator().setToOne(index);
@@ -729,8 +1127,17 @@ public void setSafe(int index, ${friendlyType} value) {
       values.getMutator().setSafe(index, value);
       setCount++;
     }
-
     </#if>
+
+
+
+    <#if minor.class == "Int" || minor.class == "VarChar">
+    @Override
+    public void setValueCount(int valueCount) {
+      /* DELEGATE TO NEW VECTOR */
+      values.setValueCount(valueCount);
+    }
+    <#else>
     @Override
     public void setValueCount(int valueCount) {
       assert valueCount >= 0;
@@ -740,7 +1147,12 @@ public void setValueCount(int valueCount) {
       values.getMutator().setValueCount(valueCount);
       bits.getMutator().setValueCount(valueCount);
     }
+    </#if>
+
 
+
+    <#if minor.class != "Int" && minor.class != "VarChar">
+    /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */
     @Override
     public void generateTestData(int valueCount){
       bits.getMutator().generateTestDataAlt(valueCount);
@@ -748,13 +1160,27 @@ public void generateTestData(int valueCount){
       <#if type.major = "VarLen">lastSet = valueCount;</#if>
       setValueCount(valueCount);
     }
+    </#if>
+
 
+
+    <#if minor.class != "Int" && minor.class != "VarChar">
+    /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */
     @Override
     public void reset(){
       setCount = 0;
       <#if type.major = "VarLen">lastSet = -1;</#if>
     }
+    </#if>
+
+
 
+    <#if minor.class == "VarChar">
+    public void setLastSet(int value) {
+      /* DELEGATE TO NEW VECTOR */
+      values.setLastSet(value);
+    }
+    <#else>
     public void setLastSet(int value) {
       <#if type.major = "VarLen">
         lastSet = value;
@@ -762,7 +1188,16 @@ public void setLastSet(int value) {
         throw new UnsupportedOperationException();
       </#if>
     }
+    </#if>
+
 
+
+    <#if minor.class == "VarChar">
+    public int getLastSet() {
+      /* DELEGATE TO NEW VECTOR */
+      return values.getLastSet();
+    }
+    <#else>
     public int getLastSet() {
       <#if type.major != "VarLen">
         throw new UnsupportedOperationException();
@@ -770,6 +1205,7 @@ public int getLastSet() {
         return lastSet;
       </#if>
     }
+    </#if>
   }
 }
 </#list>
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index fe24a8674bdc5..3c7ed0190196a 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -436,7 +436,11 @@ public Object getObject(int index) {
           <#assign uncappedName = name?uncap_first/>
           <#if !minor.typeParams?? >
       case ${name?upper_case}:
-        return get${name}Vector().getAccessor().getObject(index);
+        <#if minor.class != "Int" && minor.class != "VarChar">
+          return get${name}Vector().getAccessor().getObject(index);
+        <#else>
+          return get${name}Vector().getObject(index);
+        </#if>
           </#if>
         </#list>
       </#list>
@@ -530,7 +534,11 @@ public void setSafe(int index, UnionHolder holder) {
         <#if !minor.typeParams?? >
     public void setSafe(int index, Nullable${name}Holder holder) {
       setType(index, MinorType.${name?upper_case});
-      get${name}Vector().getMutator().setSafe(index, holder);
+      <#if minor.class != "Int" && minor.class != "VarChar">
+        get${name}Vector().getMutator().setSafe(index, holder);
+      <#else>
+        get${name}Vector().setSafe(index, holder);
+      </#if>
     }
 
         </#if>
@@ -547,4 +555,10 @@ public void reset() { }
     @Override
     public void generateTestData(int values) { }
   }
+
+  public int getValueCount() { return 0; }
+
+  public void setValueCount(int valueCount) { }
+
+  public Object getObject(int index) { return null; }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
new file mode 100644
index 0000000000000..c5f781000c9a9
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
@@ -0,0 +1,701 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.TransferPair;
+
+public abstract class BaseNullableFixedWidthVector extends BaseValueVector
+        implements FixedWidthVector, FieldVector {
+   private final byte typeWidth;
+
+   private int valueAllocationSizeInBytes;
+   private int validityAllocationSizeInBytes;
+
+   protected final Field field;
+   private int allocationMonitor;
+   protected ArrowBuf validityBuffer;
+   protected ArrowBuf valueBuffer;
+   protected int valueCount;
+
+   public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator,
+                                       FieldType fieldType, final byte typeWidth) {
+      super(name, allocator);
+      this.typeWidth = typeWidth;
+      valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth;
+      validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION);
+      field = new Field(name, fieldType, null);
+      valueCount = 0;
+      allocationMonitor = 0;
+      validityBuffer = allocator.getEmpty();
+      valueBuffer = allocator.getEmpty();
+   }
+
+   /* TODO:
+    * Determine how writerIndex and readerIndex need to be used. Right now we
+    * are setting the writerIndex and readerIndex in the call to getFieldBuffers
+    * using the valueCount -- this assumes that the caller of getFieldBuffers
+    * on the vector has already invoked setValueCount.
+    *
+    * Do we need to set them during vector transfer and splitAndTransfer?
+    */
+
+   /* TODO:
+    *
+    * see if getNullCount() can be made faster -- O(1)
+    */
+
+   /* TODO:
+    * Once the entire hierarchy has been refactored, move common functions
+    * like getNullCount(), splitAndTransferValidityBuffer to top level
+    * base class BaseValueVector.
+    *
+    * Along with this, some class members (validityBuffer) can also be
+    * abstracted out to top level base class.
+    *
+    * Right now BaseValueVector is the top level base class for other
+    * vector types in ValueVector hierarchy and those vectors have not
+    * yet been refactored so moving things to the top class as of now
+    * is not a good idea.
+    */
+
+   /* TODO:
+    * See if we need logger -- unnecessary object probably
+    */
+
+   protected abstract org.slf4j.Logger getLogger();
+
+   @Override
+   public Mutator getMutator() {
+      throw new  UnsupportedOperationException("Mutator is not needed to write into vector");
+   }
+
+   @Override
+   public Accessor getAccessor() {
+      throw new UnsupportedOperationException("Accessor is not needed to read from vector");
+   }
+
+   @Override
+   public long getValidityBufferAddress() {
+      return (validityBuffer.memoryAddress());
+   }
+
+   @Override
+   public long getDataBufferAddress() {
+      return (valueBuffer.memoryAddress());
+   }
+
+   @Override
+   public long getOffsetBufferAddress() {
+      throw new UnsupportedOperationException("not supported for fixed-width vectors");
+   }
+
+   @Override
+   public ArrowBuf getValidityBuffer() {
+      return validityBuffer;
+   }
+
+   @Override
+   public ArrowBuf getDataBuffer() {
+      return valueBuffer;
+   }
+
+   @Override
+   public ArrowBuf getOffsetBuffer() {
+      throw new UnsupportedOperationException("not supported for fixed-width vectors");
+   }
+
+   @Override
+   public void setInitialCapacity(int valueCount) {
+      final long size = (long)valueCount * typeWidth;
+      if (size > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+      }
+      valueAllocationSizeInBytes = (int)size;
+      validityAllocationSizeInBytes = getSizeFromCount(valueCount);
+   }
+
+   @Override
+   public int getValueCapacity(){
+      return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity());
+   }
+
+   /* for test purposes */
+   private int getValueBufferValueCapacity() {
+      return (int)((valueBuffer.capacity() * 1.0)/typeWidth);
+   }
+
+   /* for test purposes */
+   private int getValidityBufferValueCapacity() {
+      return (int)(validityBuffer.capacity() * 8L);
+   }
+
+   /* number of bytes for the validity buffer for the given valueCount */
+   protected int getSizeFromCount(int valueCount) {
+      return (int) Math.ceil(valueCount / 8.0);
+   }
+
+   @Override
+   public void zeroVector() {
+      initValidityBuffer();
+      initValueBuffer();
+   }
+
+   private void initValidityBuffer() {
+      validityBuffer.setZero(0, validityBuffer.capacity());
+   }
+
+   private void initValueBuffer() {
+      valueBuffer.setZero(0, valueBuffer.capacity());
+   }
+
+   public void reset() {
+      zeroVector();
+   }
+
+   @Override
+   public void close() { clear(); }
+
+   @Override
+   public void clear() {
+      validityBuffer = releaseBuffer(validityBuffer);
+      valueBuffer = releaseBuffer(valueBuffer);
+   }
+
+   /* used to step down the memory allocation */
+   protected void incrementAllocationMonitor() {
+      if (allocationMonitor < 0) {
+         allocationMonitor = 0;
+      }
+      allocationMonitor++;
+   }
+
+   /* used to step up the memory allocation */
+   protected void decrementAllocationMonitor() {
+      if (allocationMonitor > 0) {
+         allocationMonitor = 0;
+      }
+      allocationMonitor--;
+   }
+
+   @Override
+   public void allocateNew() {
+      if(!allocateNewSafe()){
+         throw new OutOfMemoryException("Failure while allocating memory.");
+      }
+   }
+
+   public boolean allocateNewSafe() {
+      long curAllocationSizeValue = valueAllocationSizeInBytes;
+      long curAllocationSizeValidity = validityAllocationSizeInBytes;
+
+      if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Requested amount of memory exceeds limit");
+      }
+
+      /* we are doing a new allocation -- release the current buffers */
+      clear();
+
+      try{
+         allocateBytes(curAllocationSizeValue, curAllocationSizeValidity);
+      } catch (Exception e) {
+         getLogger().error("ERROR: Failure in allocateNewSafe");
+         getLogger().error(e.getMessage());
+         clear();
+         return false;
+      }
+
+      return true;
+   }
+
+   public void allocateNew(int valueCount) {
+      long valueBufferSize = valueCount * typeWidth;
+      long validityBufferSize = getSizeFromCount(valueCount);
+
+      if (allocationMonitor > 10) {
+         /* step down the default memory allocation since we have observed
+          * multiple times that provisioned value capacity was much larger than
+          * actually needed. see setValueCount for more details.
+          */
+         valueBufferSize = Math.max(8, valueBufferSize / 2);
+         validityBufferSize = Math.max(8, validityBufferSize / 2);
+         allocationMonitor = 0;
+      } else if (allocationMonitor < -2) {
+         valueBufferSize = valueBufferSize * 2L;
+         validityBufferSize = validityBufferSize * 2L;
+         allocationMonitor = 0;
+      }
+
+      if (valueBufferSize > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+      }
+
+      /* we are doing a new allocation -- release the current buffers */
+      clear();
+
+      try {
+         allocateBytes(valueBufferSize, validityBufferSize);
+      } catch(Exception e) {
+         getLogger().error("ERROR: Failure in allocateNew");
+         getLogger().error(e.getMessage());
+         clear();
+         throw e;
+      }
+   }
+
+   /**
+    * Actual memory allocation is done by this function. All the calculations
+    * and knowledge about what size to allocate is upto the callers of this
+    * method.
+    * Callers appropriately handle errors if memory allocation fails here.
+    * Callers should also take care of determining that desired size is
+    * within the bounds of max allocation allowed and any other error
+    * conditions.
+    */
+   private void allocateBytes(final long valueBufferSize, final long validityBufferSize) {
+      /* allocate data buffer */
+      int curSize = (int)valueBufferSize;
+      valueBuffer = allocator.buffer(curSize);
+      valueBuffer.readerIndex(0);
+      valueAllocationSizeInBytes = curSize;
+
+      /* allocate validity buffer */
+      allocateValidityBuffer((int)validityBufferSize);
+      initValidityBuffer();
+   }
+
+   /*
+    * during splitAndTransfer, if we splitting from a random position within a byte,
+    * we can't just slice the source buffer so we have to explicitly allocate the
+    * validityBuffer of the target vector. This is unlike the databuffer which we can
+    * always slice for the target vector.
+    */
+   private void allocateValidityBuffer(final int validityBufferSize) {
+      validityBuffer = allocator.buffer(validityBufferSize);
+      validityBuffer.readerIndex(0);
+      validityAllocationSizeInBytes = validityBufferSize;
+      initValidityBuffer();
+   }
+
+   @Override
+   public int getBufferSizeFor(final int count) {
+      if (count == 0) { return 0; }
+      return (count * typeWidth) + getSizeFromCount(count);
+   }
+
+   @Override
+   public int getBufferSize() {
+      if (valueCount == 0) { return 0; }
+      return (valueCount * typeWidth) + getSizeFromCount(valueCount);
+   }
+
+   @Override
+   public Field getField() {
+      return field;
+   }
+
+   @Override
+   public ArrowBuf[] getBuffers(boolean clear) {
+      final ArrowBuf[] buffers = new ArrowBuf[2];
+      buffers[0] = validityBuffer;
+      buffers[1] = valueBuffer;
+      if (clear) {
+         for (final ArrowBuf buffer:buffers) {
+            buffer.retain(1);
+         }
+         clear();
+      }
+      return buffers;
+   }
+
+   @Override
+   public void reAlloc() {
+      valueBuffer = reallocBufferHelper(valueBuffer, true);
+      validityBuffer = reallocBufferHelper(validityBuffer, false);
+   }
+
+   private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) {
+      final int currentBufferCapacity = buffer.capacity();
+      long baseSize  = (dataBuffer ? valueAllocationSizeInBytes
+                                   : validityAllocationSizeInBytes);
+
+      if (baseSize < (long)currentBufferCapacity) {
+         baseSize = (long)currentBufferCapacity;
+      }
+
+      long newAllocationSize = baseSize * 2L;
+      newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+      if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Unable to expand the buffer");
+      }
+
+      getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]",
+              name, (dataBuffer ? valueAllocationSizeInBytes : validityAllocationSizeInBytes),
+              newAllocationSize);
+
+      final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+      newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
+      final int halfNewCapacity = newBuf.capacity() / 2;
+      newBuf.setZero(halfNewCapacity, halfNewCapacity);
+      buffer.release(1);
+      buffer = newBuf;
+      if (dataBuffer) {
+         valueAllocationSizeInBytes = (int)newAllocationSize;
+      }
+      else {
+         validityAllocationSizeInBytes = (int)newAllocationSize;
+      }
+
+      return buffer;
+   }
+
+   @Override
+   public List<BufferBacked> getFieldInnerVectors() { throw new UnsupportedOperationException(); }
+
+   @Override
+   public void initializeChildrenFromFields(List<Field> children) {
+      if (!children.isEmpty()) {
+         throw new IllegalArgumentException("primitive type vector can not have children");
+      }
+   }
+
+   @Override
+   public List<FieldVector> getChildrenFromFields() {
+      return Collections.emptyList();
+   }
+
+   @Override
+   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+      if (ownBuffers.size() != 2) {
+         throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size());
+      }
+
+      ArrowBuf bitBuffer = ownBuffers.get(0);
+      ArrowBuf dataBuffer = ownBuffers.get(1);
+
+      validityBuffer.release();
+      validityBuffer = bitBuffer.retain(allocator);
+      valueBuffer.release();
+      valueBuffer = dataBuffer.retain(allocator);
+
+      valueCount = fieldNode.getLength();
+
+      valueAllocationSizeInBytes = valueBuffer.capacity();
+      validityAllocationSizeInBytes = validityBuffer.capacity();
+   }
+
+   public List<ArrowBuf> getFieldBuffers() {
+      List<ArrowBuf> result = new ArrayList<>(2);
+
+      validityBuffer.readerIndex(0);
+      validityBuffer.writerIndex(getSizeFromCount(valueCount));
+      valueBuffer.readerIndex(0);
+      valueBuffer.writerIndex(valueCount * typeWidth);
+
+      result.add(validityBuffer);
+      result.add(valueBuffer);
+
+      return result;
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+      return getTransferPair(ref, allocator);
+   }
+
+   @Override
+   public TransferPair getTransferPair(BufferAllocator allocator){
+      return getTransferPair(name, allocator);
+   }
+
+   public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);
+
+   public void transferTo(BaseNullableFixedWidthVector target){
+      compareTypes(target, "transferTo");
+      target.clear();
+      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
+      target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer;
+      target.valueCount = valueCount;
+      clear();
+   }
+
+   public void splitAndTransferTo(int startIndex, int length,
+                                  BaseNullableFixedWidthVector target) {
+      compareTypes(target, "splitAndTransferTo");
+      target.clear();
+      splitAndTransferValidityBuffer(startIndex, length, target);
+      splitAndTransferValueBuffer(startIndex, length, target);
+      target.setValueCount(length);
+   }
+
+   private void splitAndTransferValueBuffer(int startIndex, int length,
+                                            BaseNullableFixedWidthVector target) {
+      final int startPoint = startIndex * typeWidth;
+      final int sliceLength = length * typeWidth;
+      target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
+   }
+
+   private void splitAndTransferValidityBuffer(int startIndex, int length,
+                                               BaseNullableFixedWidthVector target) {
+      assert startIndex + length <= valueCount;
+      int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+      int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+      int byteSizeTarget = getSizeFromCount(length);
+      int offset = startIndex % 8;
+
+      if (length > 0) {
+         if (offset == 0) {
+            // slice
+            if (target.validityBuffer != null) {
+               target.validityBuffer.release();
+            }
+            target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
+            target.validityBuffer.retain(1);
+         }
+         else {
+            /* Copy data
+             * When the first bit starts from the middle of a byte (offset != 0),
+             * copy data from src BitVector.
+             * Each byte in the target is composed by a part in i-th byte,
+             * another part in (i+1)-th byte.
+             */
+            target.allocateValidityBuffer(byteSizeTarget);
+
+            for (int i = 0; i < byteSizeTarget - 1; i++) {
+               byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset);
+               byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset);
+
+               target.validityBuffer.setByte(i, (b1 + b2));
+            }
+
+            /* Copying the last piece is done in the following manner:
+             * if the source vector has 1 or more bytes remaining, we copy
+             * the last piece as a byte formed by shifting data
+             * from the current byte and the next byte.
+             *
+             * if the source vector has no more bytes remaining
+             * (we are at the last byte), we copy the last piece as a byte
+             * by shifting data from the current byte.
+             */
+            if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+               byte b1 = getBitsFromCurrentByte(this.validityBuffer,
+                       firstByteSource + byteSizeTarget - 1, offset);
+               byte b2 = getBitsFromNextByte(this.validityBuffer,
+                       firstByteSource + byteSizeTarget, offset);
+
+               target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+            }
+            else {
+               byte b1 = getBitsFromCurrentByte(this.validityBuffer,
+                       firstByteSource + byteSizeTarget - 1, offset);
+               target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+            }
+         }
+      }
+   }
+
+   private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) {
+      return (byte)((data.getByte(index) & 0xFF) >>> offset);
+   }
+
+   private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
+      return (byte)((data.getByte(index) << (8 - offset)));
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          common getters and setters                            *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the number of elements that are null in the vector
+    *
+    * @return the number of null elements.
+    */
+   public int getNullCount() {
+      int count = 0;
+      final int sizeInBytes = getSizeFromCount(valueCount);
+
+      for (int i = 0; i < sizeInBytes; ++i) {
+         final byte byteValue = validityBuffer.getByte(i);
+         /* Java uses two's complement binary representation, hence 11111111_b which is -1
+          * when converted to Int will have 32bits set to 1. Masking the MSB and then
+          * adding it back solves the issue.
+          */
+         count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
+      }
+      int nullCount = (sizeInBytes * 8) - count;
+      /* if the valueCount is not a multiple of 8,
+       * the bits on the right were counted as null bits.
+       */
+      int remainder = valueCount % 8;
+      nullCount -= remainder == 0 ? 0 : 8 - remainder;
+      return nullCount;
+   }
+
+
+   /**
+    * Get the value count of vector. This will always be zero unless
+    * {@link #setValueCount(int)} has been called prior to calling this.
+    *
+    * @return valueCount for the vector
+    */
+   public int getValueCount(){
+      return valueCount;
+   }
+
+
+   /**
+    * Set value count for the vector.
+    *
+    * @param valueCount  value count to set
+    */
+   public void setValueCount(int valueCount) {
+      this.valueCount = valueCount;
+      final int currentValueCapacity = getValueCapacity();
+      while (valueCount > getValueCapacity()) {
+         reAlloc();
+      }
+      /*
+       * We are trying to understand the pattern of memory allocation.
+       * If initially, the user did vector.allocateNew(), we would have
+       * allocated memory of default size (4096 * type width).
+       * Later on user invokes setValueCount(count).
+       *
+       * If the existing value capacity is twice as large as the
+       * valueCount, we know that we over-provisioned memory in the
+       * first place when default memory allocation was done because user
+       * really needs a much less value count in the vector.
+       *
+       * We record this by bumping up the allocationMonitor. If this pattern
+       * happens for certain number of times and allocationMonitor
+       * reaches the threshold (internal hardcoded) value, subsequent
+       * call to allocateNew() will take care of stepping down the
+       * default memory allocation size.
+       *
+       * Another case would be under-provisioning the initial memory and
+       * thus going through a lot of realloc(). Here the goal is to
+       * see if we can minimize the number of reallocations. Again the
+       * state is recorded in allocationMonitor by decrementing it
+       * (negative value). If a threshold is hit, realloc will try to
+       * allocate more memory in order to possibly avoid a future realloc.
+       * This case is also applicable to setSafe() methods which can trigger
+       * a realloc() and thus we record the state there as well.
+       */
+      if (valueCount > 0) {
+         if (currentValueCapacity >= (valueCount * 2)) {
+            incrementAllocationMonitor();
+         } else if (currentValueCapacity <= (valueCount/2)) {
+            decrementAllocationMonitor();
+         }
+      }
+   }
+
+
+   /**
+    * Check if the given index is within the current value capacity
+    * of the vector
+    *
+    * @param index  position to check
+    * @return true if index is within the current value capacity
+    */
+   public boolean isSafe(int index) {
+      return index < getValueCapacity();
+   }
+
+
+   /**
+    * Check if element at given index is null.
+    *
+    * @param index  position of element
+    * @return true if element at given index is null, false otherwise
+    */
+   public boolean isNull(int index) {
+      return (isSet(index) == 0);
+   }
+
+
+   /**
+    * Same as {@link #isNull(int)}.
+    *
+    * @param index  position of element
+    * @return 1 if element at given index is not null, 0 otherwise
+    */
+   public int isSet(int index) {
+      final int byteIndex = index >> 3;
+      final byte b = validityBuffer.getByte(byteIndex);
+      final int bitIndex = index & 7;
+      return Long.bitCount(b & (1L << bitIndex));
+   }
+
+   public void setIndexDefined(int index) {
+      handleSafe(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+   }
+
+   public void set(int index, byte[] value, int start, int length) {
+      throw new UnsupportedOperationException();
+   }
+
+   public void setSafe(int index, byte[] value, int start, int length) {
+      throw new UnsupportedOperationException();
+   }
+
+   public void set(int index, ByteBuffer value, int start, int length) {
+      throw new UnsupportedOperationException();
+   }
+
+   public void setSafe(int index, ByteBuffer value, int start, int length) {
+      throw new UnsupportedOperationException();
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                helper methods for setters                      *
+    *                                                                *
+    ******************************************************************/
+
+
+
+   protected void handleSafe(int index) {
+      while (index >= getValueCapacity()) {
+         decrementAllocationMonitor();
+         reAlloc();
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
new file mode 100644
index 0000000000000..a79709da4dd86
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
@@ -0,0 +1,764 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.TransferPair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public abstract class BaseNullableVariableWidthVector extends BaseValueVector
+        implements VariableWidthVector, FieldVector {
+   private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
+   private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT;
+
+   private int valueAllocationSizeInBytes;
+   private int validityAllocationSizeInBytes;
+   private int offsetAllocationSizeInBytes;
+
+   /* protected members */
+   protected static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */
+   protected static final byte[] emptyByteArray = new byte[]{};
+   protected ArrowBuf validityBuffer;
+   protected ArrowBuf valueBuffer;
+   protected ArrowBuf offsetBuffer;
+   protected int valueCount;
+   protected int lastSet;
+   protected final Field field;
+   private boolean cleared;
+
+   public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator,
+                                          FieldType fieldType) {
+      super(name, allocator);
+      valueAllocationSizeInBytes = INITIAL_BYTE_COUNT;
+      validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION);
+      offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH;
+      field = new Field(name, fieldType, null);
+      valueCount = 0;
+      lastSet = -1;
+      offsetBuffer = allocator.getEmpty();
+      validityBuffer = allocator.getEmpty();
+      valueBuffer = allocator.getEmpty();
+      cleared = false;
+   }
+
+  /* TODO:
+    * Determine how writerIndex and readerIndex need to be used. Right now we
+    * are setting the writerIndex and readerIndex in the call to getFieldBuffers
+    * using the valueCount -- this assumes that the caller of getFieldBuffers
+    * on the vector has already invoked setValueCount.
+    *
+    * Do we need to set them during vector transfer and splitAndTransfer?
+    */
+
+   /* TODO:
+    *
+    * see if getNullCount() can be made faster -- O(1)
+    */
+
+   /* TODO:
+    * Once the entire hierarchy has been refactored, move common functions
+    * like getNullCount(), splitAndTransferValidityBuffer to top level
+    * base class BaseValueVector.
+    *
+    * Along with this, some class members (validityBuffer) can also be
+    * abstracted out to top level base class.
+    *
+    * Right now BaseValueVector is the top level base class for other
+    * vector types in ValueVector hierarchy and those vectors have not
+    * yet been refactored so moving things to the top class as of now
+    * is not a good idea.
+    */
+
+   /* TODO:
+    * See if we need logger -- unnecessary object probably
+    */
+
+   /* TODO:
+    * Implement getBufferSize(), getCurrentSizeInBytes().
+    */
+
+   protected abstract org.slf4j.Logger getLogger();
+
+   public VariableWidthMutator getMutator() {
+      throw new  UnsupportedOperationException("Mutator is not needed to write into vector");
+   }
+
+   public VariableWidthAccessor getAccessor() {
+      throw new UnsupportedOperationException("Accessor is not needed to read from vector");
+   }
+
+   @Override
+   public ArrowBuf getValidityBuffer() {
+      return validityBuffer;
+   }
+
+   @Override
+   public ArrowBuf getDataBuffer() {
+      return valueBuffer;
+   }
+
+   @Override
+   public ArrowBuf getOffsetBuffer() {
+      return offsetBuffer;
+   }
+
+   @Override
+   public long getOffsetBufferAddress() {
+      return offsetBuffer.memoryAddress();
+   }
+
+   @Override
+   public long getValidityBufferAddress() {
+      return validityBuffer.memoryAddress();
+   }
+
+   @Override
+   public long getDataBufferAddress() {
+      return valueBuffer.memoryAddress();
+   }
+
+   @Override
+   public void setInitialCapacity(int valueCount) {
+      final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT;
+      if (size > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+      }
+      valueAllocationSizeInBytes = (int)size;
+      validityAllocationSizeInBytes = getSizeFromCount(valueCount);
+      /* to track the end offset of last data element in vector, we need
+       * an additional slot in offset buffer.
+       */
+      offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH;
+   }
+
+   @Override
+   public int getValueCapacity(){
+      final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0);
+      return Math.min(offsetValueCapacity, getValidityBufferValueCapacity());
+   }
+
+   /* for test purposes */
+   private int getValidityBufferValueCapacity() {
+      return (int)(validityBuffer.capacity() * 8L);
+   }
+
+   /* for test purposes */
+   private int getOffsetBufferValueCapacity() {
+      return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH);
+   }
+
+   /* number of bytes for the validity buffer for a given valueCount */
+   protected int getSizeFromCount(int valueCount) {
+      return (int) Math.ceil(valueCount / 8.0);
+   }
+
+   public void zeroVector() {
+      initValidityBuffer();
+      initOffsetBuffer();
+   }
+
+   private void initValidityBuffer() {
+      validityBuffer.setZero(0, validityBuffer.capacity());
+   }
+
+   private void initOffsetBuffer() {
+      offsetBuffer.setZero(0, offsetBuffer.capacity());
+   }
+
+   public void reset() {
+      zeroVector();
+      lastSet = -1;
+   }
+
+   @Override
+   public void close() {
+      clear();
+   }
+
+   @Override
+   public void clear() {
+      validityBuffer = releaseBuffer(validityBuffer);
+      valueBuffer = releaseBuffer(valueBuffer);
+      offsetBuffer = releaseBuffer(offsetBuffer);
+      cleared = true;
+      lastSet = -1;
+      valueCount = 0;
+   }
+
+   @Override
+   public List<BufferBacked> getFieldInnerVectors() { throw new UnsupportedOperationException(); }
+
+   @Override
+   public void initializeChildrenFromFields(List<Field> children) {
+      if (!children.isEmpty()) {
+         throw new IllegalArgumentException("primitive type vector can not have children");
+      }
+   }
+
+   @Override
+   public List<FieldVector> getChildrenFromFields() {
+      return Collections.emptyList();
+   }
+
+   @Override
+   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+      ArrowBuf bitBuffer = ownBuffers.get(0);
+      ArrowBuf offBuffer = ownBuffers.get(1);
+      ArrowBuf dataBuffer = ownBuffers.get(2);
+
+      validityBuffer.release();
+      validityBuffer = bitBuffer.retain(allocator);
+      offsetBuffer.release();
+      offsetBuffer = offBuffer.retain(allocator);
+      valueBuffer.release();
+      valueBuffer = dataBuffer.retain(allocator);
+
+      lastSet = fieldNode.getLength() - 1;
+      valueCount = fieldNode.getLength();
+   }
+
+   public List<ArrowBuf> getFieldBuffers() {
+      List<ArrowBuf> result = new ArrayList<>(3);
+      final int lastDataOffset = getstartOffset(valueCount);
+      validityBuffer.readerIndex(0);
+      validityBuffer.writerIndex(getSizeFromCount(valueCount));
+      offsetBuffer.readerIndex(0);
+      offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
+      valueBuffer.readerIndex(0);
+      valueBuffer.writerIndex(lastDataOffset);
+
+      result.add(validityBuffer);
+      result.add(offsetBuffer);
+      result.add(valueBuffer);
+
+      return result;
+   }
+
+   @Override
+   public void allocateNew() {
+      if(!allocateNewSafe()){
+         throw new OutOfMemoryException("Failure while allocating memory.");
+      }
+   }
+
+   @Override
+   public boolean allocateNewSafe() {
+      long curAllocationSizeValue = valueAllocationSizeInBytes;
+      long curAllocationSizeValidity = validityAllocationSizeInBytes;
+      long curAllocationSizeOffset = offsetAllocationSizeInBytes;
+
+      if (curAllocationSizeValue > MAX_ALLOCATION_SIZE ||
+              curAllocationSizeOffset > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Requested amount of memory exceeds limit");
+      }
+
+      /* we are doing a new allocation -- release the current buffers */
+      clear();
+
+      try {
+         allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset);
+      } catch (Exception e) {
+         getLogger().error("ERROR: Failure in allocateNewSafe");
+         getLogger().error(e.getMessage());
+         clear();
+         return false;
+      }
+
+      return true;
+   }
+
+   @Override
+   public void allocateNew(int totalBytes, int valueCount) {
+      assert totalBytes >= 0;
+      final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
+      final int validityBufferSize = getSizeFromCount(valueCount);
+
+      if (totalBytes > MAX_ALLOCATION_SIZE ||
+              offsetBufferSize > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Requested amount of memory exceeds limit");
+      }
+
+      /* we are doing a new allocation -- release the current buffers */
+      clear();
+
+      try {
+         allocateBytes(totalBytes, validityBufferSize, offsetBufferSize);
+      } catch (Exception e) {
+         getLogger().error("ERROR: Failure in allocateNewSafe");
+         getLogger().error(e.getMessage());
+         clear();
+      }
+   }
+
+   private void allocateBytes(final long valueBufferSize, final long validityBufferSize,
+                              final long offsetBufferSize) {
+      /* allocate data buffer */
+      int curSize = (int)valueBufferSize;
+      valueBuffer = allocator.buffer(curSize);
+      valueBuffer.readerIndex(0);
+      valueAllocationSizeInBytes = curSize;
+      allocateValidityBuffer(validityBufferSize);
+      allocateOffsetBuffer(offsetBufferSize);
+   }
+
+   private void allocateOffsetBuffer(final long size) {
+      final int curSize = (int)size;
+      offsetBuffer = allocator.buffer(curSize);
+      offsetBuffer.readerIndex(0);
+      offsetAllocationSizeInBytes = curSize;
+      initOffsetBuffer();
+   }
+
+   private void allocateValidityBuffer(final long size) {
+      final int curSize = (int)size;
+      validityBuffer = allocator.buffer(curSize);
+      validityBuffer.readerIndex(0);
+      validityAllocationSizeInBytes = curSize;
+      initValidityBuffer();
+   }
+
+   public void reAlloc() {
+      reallocValueBuffer();
+      reallocValidityAndOffsetBuffers();
+   }
+
+   protected void reallocValueBuffer() {
+      long baseSize = valueAllocationSizeInBytes;
+      final int currentBufferCapacity = valueBuffer.capacity();
+
+      if (baseSize < (long)currentBufferCapacity) {
+         baseSize = (long)currentBufferCapacity;
+      }
+
+      long newAllocationSize = baseSize * 2L;
+      newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+      if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
+         throw new OversizedAllocationException("Unable to expand the buffer");
+      }
+
+      final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+      newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity);
+      valueBuffer.release();
+      valueBuffer = newBuf;
+      valueAllocationSizeInBytes = (int)newAllocationSize;
+   }
+
+   protected void reallocValidityAndOffsetBuffers() {
+      offsetBuffer = reallocBufferHelper(offsetBuffer, true);
+      validityBuffer = reallocBufferHelper(validityBuffer, false);
+   }
+
+   /* need to refactor this to keep the logic in an single place and make callers
+    * more intelligent. see handleSafe() for more comments on realloc
+    */
+
+   private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) {
+      final int currentBufferCapacity = buffer.capacity();
+      long baseSize  = (offsetBuffer ? offsetAllocationSizeInBytes
+              : validityAllocationSizeInBytes);
+
+      if (baseSize < (long)currentBufferCapacity) {
+         baseSize = (long)currentBufferCapacity;
+      }
+
+      long newAllocationSize = baseSize * 2L;
+      newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+      if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Unable to expand the buffer");
+      }
+
+      getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]",
+              name, (offsetBuffer ? offsetAllocationSizeInBytes : validityAllocationSizeInBytes),
+              newAllocationSize);
+
+      final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+      newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
+      final int halfNewCapacity = newBuf.capacity() / 2;
+      newBuf.setZero(halfNewCapacity, halfNewCapacity);
+      buffer.release(1);
+      buffer = newBuf;
+      if (offsetBuffer) {
+         offsetAllocationSizeInBytes = (int)newAllocationSize;
+      }
+      else {
+         validityAllocationSizeInBytes = (int)newAllocationSize;
+      }
+
+      return buffer;
+   }
+
+   @Override
+   public int getByteCapacity(){
+      return valueBuffer.capacity();
+   }
+
+   @Override
+   public int getCurrentSizeInBytes(){
+      /* TODO */
+      return 0;
+   }
+
+   @Override
+   public int getBufferSize() {
+      /* TODO */
+      return 0;
+   }
+
+   @Override
+   public int getBufferSizeFor(final int valueCount) {
+      if (valueCount == 0) {
+         return 0;
+      }
+
+      final int validityBufferSize = getSizeFromCount(valueCount);
+      final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
+      /* get the end offset for this valueCount */
+      final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
+      return validityBufferSize + offsetBufferSize + dataBufferSize;
+   }
+
+   @Override
+   public Field getField() {
+      return field;
+   }
+
+   @Override
+   public ArrowBuf[] getBuffers(boolean clear) {
+      final ArrowBuf[] buffers = new ArrowBuf[3];
+      buffers[0] = validityBuffer;
+      buffers[1] = offsetBuffer;
+      buffers[1] = valueBuffer;
+      if (clear) {
+         for (final ArrowBuf buffer:buffers) {
+            buffer.retain(1);
+         }
+         clear();
+      }
+      return buffers;
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+      return getTransferPair(ref, allocator);
+   }
+
+   @Override
+   public TransferPair getTransferPair(BufferAllocator allocator){
+      return getTransferPair(name, allocator);
+   }
+
+   public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);
+
+   public void transferTo(BaseNullableVariableWidthVector target){
+      compareTypes(target, "transferTo");
+      target.clear();
+      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
+      target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer;
+      target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer;
+      target.valueCount = valueCount;
+      target.setLastSet(lastSet);
+      clear();
+   }
+
+   public void splitAndTransferTo(int startIndex, int length,
+                                  BaseNullableVariableWidthVector target) {
+      compareTypes(target, "splitAndTransferTo");
+      target.clear();
+      splitAndTransferValidityBuffer(startIndex, length, target);
+      splitAndTransferOffsetBuffer(startIndex, length, target);
+      target.setLastSet(length - 1);
+      target.setValueCount(length);
+   }
+
+   /*
+    * transfer the offsets along with data
+    */
+   private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) {
+      final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH);
+      final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH);
+      final int dataLength = end - start;
+      target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH);
+      for (int i = 0; i < length + 1; i++) {
+         final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start;
+         target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset);
+      }
+      target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer;
+   }
+
+   /*
+    * transfer the validity.
+    */
+   private void splitAndTransferValidityBuffer(int startIndex, int length,
+                                               BaseNullableVariableWidthVector target) {
+      assert startIndex + length <= valueCount;
+      int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+      int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+      int byteSizeTarget = getSizeFromCount(length);
+      int offset = startIndex % 8;
+
+      if (length > 0) {
+         if (offset == 0) {
+            // slice
+            if (target.validityBuffer != null) {
+               target.validityBuffer.release();
+            }
+            target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
+            target.validityBuffer.retain(1);
+         }
+         else {
+            /* Copy data
+             * When the first bit starts from the middle of a byte (offset != 0),
+             * copy data from src BitVector.
+             * Each byte in the target is composed by a part in i-th byte,
+             * another part in (i+1)-th byte.
+             */
+            target.allocateValidityBuffer(byteSizeTarget);
+
+            for (int i = 0; i < byteSizeTarget - 1; i++) {
+               byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset);
+               byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset);
+
+               target.validityBuffer.setByte(i, (b1 + b2));
+            }
+
+            /* Copying the last piece is done in the following manner:
+             * if the source vector has 1 or more bytes remaining, we copy
+             * the last piece as a byte formed by shifting data
+             * from the current byte and the next byte.
+             *
+             * if the source vector has no more bytes remaining
+             * (we are at the last byte), we copy the last piece as a byte
+             * by shifting data from the current byte.
+             */
+            if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+               byte b1 = getBitsFromCurrentByte(this.validityBuffer,
+                       firstByteSource + byteSizeTarget - 1, offset);
+               byte b2 = getBitsFromNextByte(this.validityBuffer,
+                       firstByteSource + byteSizeTarget, offset);
+
+               target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+            }
+            else {
+               byte b1 = getBitsFromCurrentByte(this.validityBuffer,
+                       firstByteSource + byteSizeTarget - 1, offset);
+               target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+            }
+         }
+      }
+   }
+
+   private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) {
+      return (byte)((data.getByte(index) & 0xFF) >>> offset);
+   }
+
+   private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
+      return (byte)((data.getByte(index) << (8 - offset)));
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                common getters and setters                      *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the number of elements that are null in the vector
+    *
+    * @return the number of null elements.
+    */
+   public int getNullCount() {
+      int count = 0;
+      final int sizeInBytes = getSizeFromCount(valueCount);
+
+      for (int i = 0; i < sizeInBytes; ++i) {
+         final byte byteValue = validityBuffer.getByte(i);
+         /* Java uses two's complement binary representation, hence 11111111_b which is -1
+          * when converted to Int will have 32bits set to 1. Masking the MSB and then
+          * adding it back solves the issue.
+          */
+         count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
+      }
+      int nullCount = (sizeInBytes * 8) - count;
+      /* if the valueCount is not a multiple of 8,
+       * the bits on the right were counted as null bits.
+       */
+      int remainder = valueCount % 8;
+      nullCount -= remainder == 0 ? 0 : 8 - remainder;
+      return nullCount;
+   }
+
+   /**
+    * Check if the given index is within the current value capacity
+    * of the vector
+    *
+    * @param index  position to check
+    * @return true if index is within the current value capacity
+    */
+   public boolean isSafe(int index) {
+      return index < getValueCapacity();
+   }
+
+   /**
+    * Check if element at given index is null.
+    *
+    * @param index  position of element
+    * @return true if element at given index is null
+    */
+   public boolean isNull(int index) {
+      return (isSet(index) == 0);
+   }
+
+   /**
+    * Same as {@link #isNull(int)}.
+    *
+    * @param index  position of element
+    * @return 1 if element at given index is not null, 0 otherwise
+    */
+   public int isSet(int index) {
+      final int byteIndex = index >> 3;
+      final byte b = validityBuffer.getByte(byteIndex);
+      final int bitIndex = index & 7;
+      return Long.bitCount(b & (1L << bitIndex));
+   }
+
+   /**
+    * Get the value count of vector. This will always be zero unless
+    * setValueCount(int) has been called prior to calling this.
+    *
+    * @return valueCount for the vector
+    */
+   public int getValueCount(){
+      return valueCount;
+   }
+
+   /**
+    * Sets the value count for the vector
+    *
+    * @param valueCount   value count
+    */
+   public void setValueCount(int valueCount) {
+      assert valueCount >= 0;
+      this.valueCount = valueCount;
+      while (valueCount > getValueCapacity()) {
+         reallocValidityAndOffsetBuffers();
+      }
+      fillHoles(valueCount);
+      lastSet = valueCount - 1;
+   }
+
+   public void fillEmpties(int index) {
+      handleSafe(index, emptyByteArray.length);
+      fillHoles(index);
+      lastSet = index - 1;
+   }
+
+   public void setLastSet(int value) {
+      lastSet = value;
+   }
+
+   public int getLastSet() {
+      return lastSet;
+   }
+
+   public long getStartEnd(int index) {
+      return (long)offsetBuffer.getInt(index * OFFSET_WIDTH);
+   }
+
+   public void setIndexDefined(int index) {
+      handleSafe(index, 0);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                helper methods for setters                      *
+    *                                                                *
+    ******************************************************************/
+
+
+   protected final void fillHoles(int index) {
+      for (int i = lastSet + 1; i < index; i++) {
+         setBytes(i, emptyByteArray, 0, emptyByteArray.length);
+      }
+      lastSet = index - 1;
+   }
+
+   protected final void setBytes(int index, byte[] value, int start, int length) {
+      /* end offset of current last element in the vector. this will
+       * be the start offset of new element we are trying to store.
+       */
+      final int startOffset = getstartOffset(index);
+      /* set new end offset */
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+      /* store the var length data in value buffer */
+      valueBuffer.setBytes(startOffset, value, start, length);
+   }
+
+   protected final int getstartOffset(int index) {
+      return offsetBuffer.getInt(index * OFFSET_WIDTH);
+   }
+
+   protected final void handleSafe(int index, int dataLength) {
+      /*
+       * IMPORTANT:
+       * value buffer for variable length vectors moves independent
+       * of the companion validity and offset buffers. This is in
+       * contrast to what we have for fixed width vectors.
+       *
+       * Here there is no concept of getValueCapacity() in the
+       * data stream. getValueCapacity() is applicable only to validity
+       * and offset buffers.
+       *
+       * So even though we may have setup an initial capacity of 1024
+       * elements in the vector, it is quite possible
+       * that we need to reAlloc() the data buffer when we are setting
+       * the 5th element in the vector simply because previous
+       * variable length elements have exhausted the buffer capacity.
+       * However, we really don't need to reAlloc() validity and
+       * offset buffers until we try to set the 1025th element
+       * This is why we do a separate check for safe methods to
+       * determine which buffer needs reallocation.
+       */
+      while (index >= getValueCapacity()) {
+         reallocValidityAndOffsetBuffers();
+      }
+      final int startOffset = getstartOffset(index);
+      while (valueBuffer.capacity() < (startOffset + dataLength)) {
+         reallocValueBuffer();
+      }
+   }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 598e578e55a6d..1db96246485d3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -18,8 +18,10 @@
 
 package org.apache.arrow.vector;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.List;
 
 import com.google.flatbuffers.FlatBufferBuilder;
 import org.apache.arrow.memory.BufferAllocator;
@@ -122,5 +124,23 @@ public static boolean checkBufRefs(final ValueVector vv) {
   public BufferAllocator getAllocator() {
     return allocator;
   }
+
+  protected void compareTypes(BaseValueVector target, String caller) {
+    if (this.getMinorType() != target.getMinorType()) {
+      throw new UnsupportedOperationException(caller + " should have vectors of exact same type");
+    }
+  }
+
+  protected ArrowBuf releaseBuffer(ArrowBuf buffer) {
+    buffer.release();
+    buffer = allocator.getEmpty();
+    return buffer;
+  }
+
+  public int getValueCount() { return 0; }
+
+  public void setValueCount(int valueCount) { }
+
+  public Object getObject(int index) { return null; }
 }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
new file mode 100644
index 0000000000000..2439bd26eab56
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+class BitVectorHelper {
+
+   /**
+    * Get the index of byte corresponding to bit index in validity buffer
+    */
+   protected static int byteIndex(int absoluteBitIndex) {
+      return absoluteBitIndex >> 3;
+   }
+
+   /**
+    * Get the relative index of bit within the byte in validity buffer
+    */
+   private static int bitIndex(int absoluteBitIndex) {
+      return absoluteBitIndex & 7;
+   }
+
+   protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) {
+      final int byteIndex = byteIndex(index);
+      final int bitIndex = bitIndex(index);
+      byte currentByte = validityBuffer.getByte(byteIndex);
+      final byte bitMask = (byte) (1L << bitIndex);
+      currentByte |= bitMask;
+      validityBuffer.setByte(byteIndex, currentByte);
+   }
+
+   protected static void setValidityBit(ArrowBuf validityBuffer, int index, int value) {
+      final int byteIndex = byteIndex(index);
+      final int bitIndex = bitIndex(index);
+      byte currentByte = validityBuffer.getByte(byteIndex);
+      final byte bitMask = (byte) (1L << bitIndex);
+      if (value != 0) {
+         currentByte |= bitMask;
+      } else {
+         currentByte -= (bitMask & currentByte);
+      }
+      validityBuffer.setByte(byteIndex, currentByte);
+   }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
new file mode 100644
index 0000000000000..26b19fa7bd562
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
@@ -0,0 +1,299 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.IntReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.IntHolder;
+import org.apache.arrow.vector.holders.NullableIntHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.slf4j.Logger;
+
+/**
+ * NullableIntVector implements a fixed width vector of values which could
+ * be null. A validity buffer (bit vector) is maintained to track which
+ * elements in the vector are null.
+ */
+public class NullableIntVector extends BaseNullableFixedWidthVector {
+   private static final org.slf4j.Logger logger =
+           org.slf4j.LoggerFactory.getLogger(NullableIntVector.class);
+   private static final byte TYPE_WIDTH = 4;
+   private final FieldReader reader;
+
+   public NullableIntVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()),
+              allocator);
+   }
+
+   public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new IntReaderImpl(NullableIntVector.this);
+   }
+
+   @Override
+   protected org.slf4j.Logger getLogger() {
+      return logger;
+   }
+
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.INT;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public int get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableIntHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Integer getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setInt(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableIntHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, IntHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableIntHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, IntHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, IntHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   public void set(int index, int isSet, int valueField ) {
+      if (isSet > 0) {
+         set(index, valueField);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   public void setSafe(int index, int isSet, int valueField ) {
+      handleSafe(index);
+      set(index, isSet, valueField);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableIntVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableIntVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableIntVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableIntVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableIntVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
new file mode 100644
index 0000000000000..b9021544b0e10
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
@@ -0,0 +1,451 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.VarCharReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.VarCharHolder;
+import org.apache.arrow.vector.holders.NullableVarCharHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.Text;
+import org.apache.arrow.vector.util.TransferPair;
+
+import java.nio.ByteBuffer;
+
+public class NullableVarCharVector extends BaseNullableVariableWidthVector {
+   private static final org.slf4j.Logger logger =
+           org.slf4j.LoggerFactory.getLogger(NullableIntVector.class);
+   private final FieldReader reader;
+
+   public NullableVarCharVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator);
+   }
+
+   public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType);
+      reader = new VarCharReaderImpl(NullableVarCharVector.this);
+   }
+
+   @Override
+   protected org.slf4j.Logger getLogger() {
+      return logger;
+   }
+
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.VARCHAR;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value getter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the variable length element at specified index as byte array.
+    *
+    * @param index   position of element to get
+    * @return array of bytes for non-null element, null otherwise
+    */
+   public byte[] get(int index) {
+      assert index >= 0;
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      final int startOffset = getstartOffset(index);
+      final int dataLength =
+              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+      final byte[] result = new byte[dataLength];
+      valueBuffer.getBytes(startOffset, result, 0, dataLength);
+      return result;
+   }
+
+   /**
+    * Get the variable length element at specified index as Text.
+    *
+    * @param index   position of element to get
+    * @return Text object for non-null element, null otherwise
+    */
+   public Text getObject(int index) {
+      Text result = new Text();
+      byte[] b;
+      try {
+         b = get(index);
+      } catch (IllegalStateException e) {
+         return null;
+      }
+      result.set(b);
+      return result;
+   }
+
+   /**
+    * Get the variable length element at specified index as Text.
+    *
+    * @param index   position of element to get
+    * @return greater than 0 length for non-null element, 0 otherwise
+    */
+   public int getValueLength(int index) {
+      assert index >= 0;
+      if(isSet(index) == 0) {
+         return 0;
+      }
+      final int startOffset = getstartOffset(index);
+      final int dataLength =
+              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+      return dataLength;
+   }
+
+   /**
+    * Get the variable length element at specified index and sets the state
+    * in provided holder.
+    *
+    * @param index   position of element to get
+    * @param holder  data holder to be populated by this function
+    */
+   public void get(int index, NullableVarCharHolder holder){
+      assert index >= 0;
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      final int startOffset = getstartOffset(index);
+      final int dataLength =
+              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+      holder.isSet = 1;
+      holder.start = startOffset;
+      holder.end = dataLength;
+      holder.buffer = valueBuffer;
+   }
+
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+
+   public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) {
+      fillHoles(thisIndex);
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+         lastSet = thisIndex;
+      }
+   }
+
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) {
+      fillEmpties(thisIndex);
+      if (from.isSet(fromIndex) != 0) {
+         setSafe(thisIndex, from.get(fromIndex));
+         lastSet = thisIndex;
+      }
+   }
+
+
+   /**
+    * Set the variable length element at the specified index to the supplied
+    * byte array. This is same as using {@link #set(int, byte[], int, int)}
+    * with start as 0 and length as value.length
+    *
+    * @param index   position of the element to set
+    * @param value   array of bytes to write
+    */
+   public void set(int index, byte[] value) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setBytes(index, value, 0, value.length);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, byte[])} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param value   array of bytes to write
+    */
+   public void setSafe(int index, byte[] value) {
+      assert index >= 0;
+      fillEmpties(index);
+      handleSafe(index, value.length);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setBytes(index, value, 0, value.length);
+      lastSet = index;
+   }
+
+   /**
+    * Set the variable length element at the specified index to the supplied
+    * byte array.
+    *
+    * @param index   position of the element to set
+    * @param value   array of bytes to write
+    * @param start   start index in array of bytes
+    * @param length  length of data in array of bytes
+    */
+   public void set(int index, byte[] value, int start, int length) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setBytes(index, value, start, length);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, byte[], int, int)} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param value   array of bytes to write
+    * @param start   start index in array of bytes
+    * @param length  length of data in array of bytes
+    */
+   public void setSafe(int index, byte[] value, int start, int length) {
+      assert index >= 0;
+      fillEmpties(index);
+      handleSafe(index, length);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setBytes(index, value, start, length);
+      lastSet = index;
+   }
+
+   /**
+    * Set the variable length element at the specified index to the
+    * content in supplied ByteBuffer
+    *
+    * @param index   position of the element to set
+    * @param value   ByteBuffer with data
+    * @param start   start index in ByteBuffer
+    * @param length  length of data in ByteBuffer
+    */
+   public void set(int index, ByteBuffer value, int start, int length) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+      valueBuffer.setBytes(startOffset, value, start, length);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param value   ByteBuffer with data
+    * @param start   start index in ByteBuffer
+    * @param length  length of data in ByteBuffer
+    */
+   public void setSafe(int index, ByteBuffer value, int start, int length) {
+      assert index >= 0;
+      fillEmpties(index);
+      handleSafe(index, length);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+      valueBuffer.setBytes(startOffset, value, start, length);
+      lastSet = index;
+   }
+
+   /**
+    * Set the variable length element at the specified index to the data
+    * buffer supplied in the holder
+    *
+    * @param index   position of the element to set
+    * @param holder  holder that carries data buffer.
+    */
+   public void set(int index, VarCharHolder holder) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      final int dataLength = holder.end - holder.start;
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, VarCharHolder)} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param holder  holder that carries data buffer.
+    */
+   public void setSafe(int index, VarCharHolder holder) {
+      assert index >= 0;
+      final int dataLength = holder.end - holder.start;
+      fillEmpties(index);
+      handleSafe(index, dataLength);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+      lastSet = index;
+   }
+
+   /**
+    * Set the variable length element at the specified index to the data
+    * buffer supplied in the holder
+    *
+    * @param index   position of the element to set
+    * @param holder  holder that carries data buffer.
+    */
+   public void set(int index, NullableVarCharHolder holder) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
+      final int dataLength = holder.end - holder.start;
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param holder  holder that carries data buffer.
+    */
+   public void setSafe(int index, NullableVarCharHolder holder) {
+      assert index >= 0;
+      final int dataLength = holder.end - holder.start;
+      fillEmpties(index);
+      handleSafe(index, dataLength);
+      BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+      lastSet = index;
+   }
+
+   /**
+    * Sets the value length for an element.
+    *
+    * @param index   position of the element to set
+    * @param length  length of the element
+    */
+   public void setValueLengthSafe(int index, int length) {
+      assert index >= 0;
+      handleSafe(index, length);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index, 0);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBit(validityBuffer, index, isSet);
+      final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + endField);
+      final ArrowBuf bb = bufferField.slice(startField, endField);
+      valueBuffer.setBytes(startOffset, bb);
+      lastSet = index;
+   }
+
+   public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) {
+      assert index >= 0;
+      handleSafe(index, endField);
+      set(index, isSet, startField, endField, bufferField);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableVarCharVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableVarCharVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableVarCharVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableVarCharVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableVarCharVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this);
+      }
+   }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index fb7286f852c06..e6048b4b517c9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -255,4 +255,10 @@ interface Mutator {
    * @return buffer
    */
   public ArrowBuf getOffsetBuffer();
+
+  public int getValueCount();
+
+  public void setValueCount(int valueCount);
+
+  public Object getObject(int index);
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index f8385a7262a21..96243eb5876b0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -27,6 +27,8 @@
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.schema.ArrowVectorType;
 
+import javax.annotation.Nullable;
+
 public class VectorUnloader {
 
   private final VectorSchemaRoot root;
@@ -53,8 +55,17 @@ public ArrowRecordBatch getRecordBatch() {
   }
 
   private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
-    Accessor accessor = vector.getAccessor();
-    nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1));
+    Accessor accessor = null;
+    if (vector instanceof NullableIntVector) {
+      nodes.add(new ArrowFieldNode(((NullableIntVector)vector).getValueCount(),
+                includeNullCount ? ((NullableIntVector)vector).getNullCount() : -1));
+    } else if (vector instanceof NullableVarCharVector) {
+      nodes.add(new ArrowFieldNode(((NullableVarCharVector)vector).getValueCount(),
+                includeNullCount ? ((NullableVarCharVector)vector).getNullCount() : -1));
+    } else {
+      accessor = vector.getAccessor();
+      nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1));
+    }
     List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
     List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
     if (fieldBuffers.size() != expectedBuffers.size()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 5ac00375f8317..fcef02f4a1ba8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -259,4 +259,13 @@ public ArrowBuf getDataBuffer() {
   public ArrowBuf getOffsetBuffer() {
     throw new UnsupportedOperationException();
   }
+
+  @Override
+  public int getValueCount() { return 0; }
+
+  @Override
+  public void setValueCount(int valueCount) { }
+
+  @Override
+  public Object getObject(int index) { return null; }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index b3be37541c172..e95442acf6d41 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -34,14 +34,7 @@
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.BaseDataValueVector;
-import org.apache.arrow.vector.BaseValueVector;
-import org.apache.arrow.vector.BitVector;
-import org.apache.arrow.vector.BufferBacked;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -331,9 +324,15 @@ public Object getObject(int index) {
         return null;
       }
       final List<Object> vals = new JsonStringArrayList<>(listSize);
-      final ValueVector.Accessor valuesAccessor = vector.getAccessor();
-      for (int i = 0; i < listSize; i++) {
-        vals.add(valuesAccessor.getObject(index * listSize + i));
+      if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) {
+        for (int i = 0; i < listSize; i++) {
+          vals.add(vector.getObject(index * listSize + i));
+        }
+      } else {
+        final ValueVector.Accessor valuesAccessor = vector.getAccessor();
+        for (int i = 0; i < listSize; i++) {
+          vals.add(valuesAccessor.getObject(index * listSize + i));
+        }
       }
       return vals;
     }
@@ -367,7 +366,11 @@ public void setNotNull(int index) {
     @Override
     public void setValueCount(int valueCount) {
       bits.getMutator().setValueCount(valueCount);
-      vector.getMutator().setValueCount(valueCount * listSize);
+      if (vector instanceof  NullableIntVector || vector instanceof NullableVarCharVector) {
+        vector.setValueCount(valueCount * listSize);
+      } else {
+        vector.getMutator().setValueCount(valueCount * listSize);
+      }
     }
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index ea28a60619209..4b2c9139219f3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -40,6 +40,8 @@
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.NullableIntVector;
+import org.apache.arrow.vector.NullableVarCharVector;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
@@ -376,10 +378,18 @@ public Object getObject(int index) {
       final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
       final int start = offsetsAccessor.get(index);
       final int end = offsetsAccessor.get(index + 1);
-      final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor();
-      for (int i = start; i < end; i++) {
-        vals.add(valuesAccessor.getObject(i));
+      final ValueVector vv = getDataVector();
+      if (vv instanceof  NullableIntVector || vv instanceof NullableVarCharVector) {
+        for (int i = start; i < end; i++) {
+          vals.add(vv.getObject(i));
+        }
+      } else {
+        final ValueVector.Accessor valuesAccessor = vv.getAccessor();
+        for (int i = start; i < end; i++) {
+          vals.add(valuesAccessor.getObject(i));
+        }
       }
+
       return vals;
     }
 
@@ -432,7 +442,12 @@ public void setValueCount(int valueCount) {
         offsets.getMutator().setValueCount(valueCount + 1);
       }
       final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
-      vector.getMutator().setValueCount(childValueCount);
+      if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) {
+        vector.setValueCount(childValueCount);
+      } else {
+        vector.getMutator().setValueCount(childValueCount);
+      }
+
       bits.getMutator().setValueCount(valueCount);
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index f46635ad76021..95efa60871806 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -35,9 +35,7 @@
 import io.netty.buffer.ArrowBuf;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.BaseValueVector;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
@@ -272,10 +270,19 @@ public Object getObject(int index) {
       Map<String, Object> vv = new JsonStringHashMap<>();
       for (String child : getChildFieldNames()) {
         ValueVector v = getChild(child);
-        if (v != null && index < v.getAccessor().getValueCount()) {
-          Object value = v.getAccessor().getObject(index);
-          if (value != null) {
-            vv.put(child, value);
+        if (v instanceof  NullableVarCharVector || v instanceof  NullableIntVector) {
+          if (v != null && index < v.getValueCount()) {
+            Object value = v.getObject(index);
+            if (value != null) {
+              vv.put(child, value);
+            }
+          }
+        } else {
+          if (v != null && index < v.getAccessor().getValueCount()) {
+            Object value = v.getAccessor().getObject(index);
+            if (value != null) {
+              vv.put(child, value);
+            }
           }
         }
       }
@@ -302,7 +309,11 @@ public class Mutator extends BaseValueVector.BaseMutator {
     @Override
     public void setValueCount(int valueCount) {
       for (final ValueVector v : getChildren()) {
-        v.getMutator().setValueCount(valueCount);
+        if (v instanceof NullableIntVector || v instanceof NullableVarCharVector) {
+          v.setValueCount(valueCount);
+        } else {
+          v.getMutator().setValueCount(valueCount);
+        }
       }
       MapVector.this.valueCount = valueCount;
     }
@@ -362,4 +373,10 @@ public List<FieldVector> getChildrenFromFields() {
     return getChildren();
   }
 
+  public int getValueCount() { return 0; }
+
+  public void setValueCount(int valueCount) { }
+
+  public Object getObject(int index) { return null; }
+
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index e1c7c909f10be..0de804406f7b3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -265,7 +265,7 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
           ((NullableVarBinaryVector) vector).getMutator().setLastSet(count - 1);
           break;
         case VARCHAR:
-          ((NullableVarCharVector) vector).getMutator().setLastSet(count - 1);
+          ((NullableVarCharVector) vector).setLastSet(count - 1);
           break;
       }
       vector.getMutator().setValueCount(count);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
index 1a801a63ec688..4fe2861450a34 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
@@ -40,7 +40,7 @@ public void testTransferFixedWidth() {
 
     NullableIntVector v1 = new NullableIntVector("v1", childAllocator1);
     v1.allocateNew();
-    v1.getMutator().setValueCount(4095);
+    v1.setValueCount(4095);
 
     NullableIntVector v2 = new NullableIntVector("v2", childAllocator2);
 
@@ -60,8 +60,8 @@ public void testTransferVariableidth() {
 
     NullableVarCharVector v1 = new NullableVarCharVector("v1", childAllocator1);
     v1.allocateNew();
-    v1.getMutator().setSafe(4094, "hello world".getBytes(), 0, 11);
-    v1.getMutator().setValueCount(4001);
+    v1.setSafe(4094, "hello world".getBytes(), 0, 11);
+    v1.setValueCount(4001);
 
     NullableVarCharVector v2 = new NullableVarCharVector("v2", childAllocator2);
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
index f8c16e7fc8176..1185246f88cde 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -56,24 +56,22 @@ public void testEncodeStrings() {
     // Create a new value vector
     try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator);
          final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) {
-      final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(512, 5);
 
       // set some values
-      m.setSafe(0, zero, 0, zero.length);
-      m.setSafe(1, one, 0, one.length);
-      m.setSafe(2, one, 0, one.length);
-      m.setSafe(3, two, 0, two.length);
-      m.setSafe(4, zero, 0, zero.length);
-      m.setValueCount(5);
+      vector.setSafe(0, zero, 0, zero.length);
+      vector.setSafe(1, one, 0, one.length);
+      vector.setSafe(2, one, 0, one.length);
+      vector.setSafe(3, two, 0, two.length);
+      vector.setSafe(4, zero, 0, zero.length);
+      vector.setValueCount(5);
 
       // set some dictionary values
-      final NullableVarCharVector.Mutator m2 = dictionaryVector.getMutator();
       dictionaryVector.allocateNew(512, 3);
-      m2.setSafe(0, zero, 0, zero.length);
-      m2.setSafe(1, one, 0, one.length);
-      m2.setSafe(2, two, 0, two.length);
-      m2.setValueCount(3);
+      dictionaryVector.setSafe(0, zero, 0, zero.length);
+      dictionaryVector.setSafe(1, one, 0, one.length);
+      dictionaryVector.setSafe(2, two, 0, two.length);
+      dictionaryVector.setValueCount(3);
 
       Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null));
 
@@ -81,20 +79,20 @@ public void testEncodeStrings() {
         // verify indices
         assertEquals(NullableIntVector.class, encoded.getClass());
 
-        NullableIntVector.Accessor indexAccessor = ((NullableIntVector) encoded).getAccessor();
-        assertEquals(5, indexAccessor.getValueCount());
-        assertEquals(0, indexAccessor.get(0));
-        assertEquals(1, indexAccessor.get(1));
-        assertEquals(1, indexAccessor.get(2));
-        assertEquals(2, indexAccessor.get(3));
-        assertEquals(0, indexAccessor.get(4));
+        NullableIntVector index = ((NullableIntVector)encoded);
+        assertEquals(5, index.getValueCount());
+        assertEquals(0, index.get(0));
+        assertEquals(1, index.get(1));
+        assertEquals(1, index.get(2));
+        assertEquals(2, index.get(3));
+        assertEquals(0, index.get(4));
 
         // now run through the decoder and verify we get the original back
         try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) {
           assertEquals(vector.getClass(), decoded.getClass());
-          assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount());
+          assertEquals(vector.getValueCount(), ((NullableVarCharVector)decoded).getValueCount());
           for (int i = 0; i < 5; i++) {
-            assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i));
+            assertEquals(vector.getObject(i), ((NullableVarCharVector)decoded).getObject(i));
           }
         }
       }
@@ -106,21 +104,20 @@ public void testEncodeLargeVector() {
     // Create a new value vector
     try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator);
          final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) {
-      final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew();
 
       int count = 10000;
 
       for (int i = 0; i < 10000; ++i) {
-        vector.getMutator().setSafe(i, data[i % 3], 0, data[i % 3].length);
+        vector.setSafe(i, data[i % 3], 0, data[i % 3].length);
       }
-      vector.getMutator().setValueCount(count);
+      vector.setValueCount(count);
 
       dictionaryVector.allocateNew(512, 3);
-      dictionaryVector.getMutator().setSafe(0, zero, 0, zero.length);
-      dictionaryVector.getMutator().setSafe(1, one, 0, one.length);
-      dictionaryVector.getMutator().setSafe(2, two, 0, two.length);
-      dictionaryVector.getMutator().setValueCount(3);
+      dictionaryVector.setSafe(0, zero, 0, zero.length);
+      dictionaryVector.setSafe(1, one, 0, one.length);
+      dictionaryVector.setSafe(2, two, 0, two.length);
+      dictionaryVector.setValueCount(3);
 
       Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null));
 
@@ -129,10 +126,10 @@ public void testEncodeLargeVector() {
         // verify indices
         assertEquals(NullableIntVector.class, encoded.getClass());
 
-        NullableIntVector.Accessor indexAccessor = ((NullableIntVector) encoded).getAccessor();
-        assertEquals(count, indexAccessor.getValueCount());
+        NullableIntVector index = ((NullableIntVector) encoded);
+        assertEquals(count, index.getValueCount());
         for (int i = 0; i < count; ++i) {
-          assertEquals(i % 3, indexAccessor.get(i));
+          assertEquals(i % 3, index.get(i));
         }
 
         // now run through the decoder and verify we get the original back
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index 43d9387b106a4..168deacd6a640 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -54,15 +54,14 @@ public void terminate() throws Exception {
   public void testIntType() {
     try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) {
       NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
-      NullableIntVector.Mutator mutator = nested.getMutator();
       vector.allocateNew();
 
       for (int i = 0; i < 10; i++) {
         vector.getMutator().setNotNull(i);
-        mutator.set(i * 2, i);
-        mutator.set(i * 2 + 1, i + 10);
+        nested.set(i * 2, i);
+        nested.set(i * 2 + 1, i + 10);
       }
-      vector.getMutator().setValueCount(10);
+      vector.setValueCount(10);
 
       UnionFixedSizeListReader reader = vector.getReader();
       for (int i = 0; i < 10; i++) {
@@ -119,7 +118,7 @@ public void testNestedInList() {
       ListVector.Mutator mutator = vector.getMutator();
       FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector();
       FixedSizeListVector.Mutator tupleMutator = tuples.getMutator();
-      NullableIntVector.Mutator innerMutator = (NullableIntVector.Mutator) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector().getMutator();
+      NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
       vector.allocateNew();
 
       for (int i = 0; i < 10; i++) {
@@ -127,8 +126,8 @@ public void testNestedInList() {
           int position = mutator.startNewValue(i);
           for (int j = 0; j < i % 7; j++) {
             tupleMutator.setNotNull(position + j);
-            innerMutator.set((position + j) * 2, j);
-            innerMutator.set((position + j) * 2 + 1, j + 1);
+            innerVector.set((position + j) * 2, j);
+            innerVector.set((position + j) * 2 + 1, j + 1);
           }
           mutator.endValue(i, i % 7);
         }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
index 66e5375e3bd0d..7de3bcb33195f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
@@ -25,7 +25,6 @@
 import org.apache.arrow.memory.RootAllocator;
 
 import org.apache.arrow.vector.NullableVarCharVector;
-import org.apache.arrow.vector.NullableVarCharVector.Accessor;
 import org.apache.arrow.vector.util.TransferPair;
 
 import org.junit.After;
@@ -54,32 +53,30 @@ public void test() throws Exception {
             final int valueCount = 500;
             final String[] compareArray = new String[valueCount];
 
-            final NullableVarCharVector.Mutator mutator = varCharVector.getMutator();
             for (int i = 0; i < valueCount; i += 3) {
                 final String s = String.format("%010d", i);
-                mutator.set(i, s.getBytes());
+                varCharVector.set(i, s.getBytes());
                 compareArray[i] = s;
             }
-            mutator.setValueCount(valueCount);
+            varCharVector.setValueCount(valueCount);
 
             final TransferPair tp = varCharVector.getTransferPair(allocator);
             final NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo();
-            final Accessor accessor = newVarCharVector.getAccessor();
             final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}};
 
             for (final int[] startLength : startLengths) {
                 final int start = startLength[0];
                 final int length = startLength[1];
                 tp.splitAndTransfer(start, length);
-                newVarCharVector.getMutator().setValueCount(length);
+                newVarCharVector.setValueCount(length);
                 for (int i = 0; i < length; i++) {
                     final boolean expectedSet = ((start + i) % 3) == 0;
                     if (expectedSet) {
                         final byte[] expectedValue = compareArray[start + i].getBytes();
-                        assertFalse(accessor.isNull(i));
-                        assertArrayEquals(expectedValue, accessor.get(i));
+                        assertFalse(newVarCharVector.isNull(i));
+                        assertArrayEquals(expectedValue, newVarCharVector.get(i));
                     } else {
-                        assertTrue(accessor.isNull(i));
+                        assertTrue(newVarCharVector.isNull(i));
                     }
                 }
                 newVarCharVector.clear();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index a239861d9b32f..b7f88c3babf1b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -694,8 +694,6 @@ public void testNullableFixedType2() {
   public void testNullableFixedType3() {
     // Create a new value vector for 1024 integers
     try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) {
-      final NullableIntVector.Mutator mutator = vector.getMutator();
-      final NullableIntVector.Accessor accessor = vector.getAccessor();
       boolean error = false;
       int initialCapacity = 1024;
 
@@ -706,26 +704,26 @@ public void testNullableFixedType3() {
       /* underlying buffer should be able to store 16 values */
       assertEquals(initialCapacity, vector.getValueCapacity());
 
-      mutator.set(0, 1);
-      mutator.set(1, 2);
-      mutator.set(100, 3);
-      mutator.set(1022, 4);
-      mutator.set(1023, 5);
+      vector.set(0, 1);
+      vector.set(1, 2);
+      vector.set(100, 3);
+      vector.set(1022, 4);
+      vector.set(1023, 5);
 
       /* check vector contents */
       int j = 1;
       for(int i = 0; i <= 1023; i++) {
         if((i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) {
-          assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+          assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
         }
         else {
-          assertFalse("null data not expected at index: " + i, accessor.isNull(i));
-          assertEquals("unexpected value at index: " + i, j, accessor.get(i));
+          assertFalse("null data not expected at index: " + i, vector.isNull(i));
+          assertEquals("unexpected value at index: " + i, j, vector.get(i));
           j++;
         }
       }
 
-      mutator.setValueCount(1024);
+      vector.setValueCount(1024);
       Field field = vector.getField();
       TypeLayout typeLayout = field.getTypeLayout();
 
@@ -749,7 +747,7 @@ public void testNullableFixedType3() {
       assertEquals(-64, validityVectorBuf.getByte(127)); // 1022nd and 1023rd bit defined
 
       /* this should trigger a realloc() */
-      mutator.setSafe(1024, 6);
+      vector.setSafe(1024, 6);
 
       /* underlying buffer should now be able to store double the number of values */
       assertEquals(initialCapacity * 2, vector.getValueCapacity());
@@ -758,11 +756,11 @@ public void testNullableFixedType3() {
       j = 1;
       for(int i = 0; i < (initialCapacity * 2); i++) {
         if((i > 1024) || (i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) {
-          assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+          assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
         }
         else {
-          assertFalse("null data not expected at index: " + i, accessor.isNull(i));
-          assertEquals("unexpected value at index: " + i, j, accessor.get(i));
+          assertFalse("null data not expected at index: " + i, vector.isNull(i));
+          assertEquals("unexpected value at index: " + i, j, vector.get(i));
           j++;
         }
       }
@@ -775,13 +773,101 @@ public void testNullableFixedType3() {
 
       /* vector data should have been zeroed out */
       for(int i = 0; i < (initialCapacity * 2); i++) {
-        assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+        assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
       }
 
       vector.allocateNew(4096);
       // vector has been erased
       for(int i = 0; i < 4096; i++) {
-        assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+        assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
+      }
+    }
+  }
+
+  @Test /* NullableIntVector */
+  public void testNullableFixedType4() {
+    try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) {
+
+      /* no memory allocation has happened yet */
+      assertEquals(0, vector.getValueCapacity());
+
+      vector.allocateNew();
+      int valueCapacity = vector.getValueCapacity();
+      assertEquals(vector.INITIAL_VALUE_ALLOCATION, valueCapacity);
+
+      int baseValue = 20000;
+
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          vector.set(i, baseValue + i);
+        }
+      }
+
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          assertFalse("unexpected null value at index: " + i, vector.isNull(i));
+          assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i));
+        } else {
+          assertTrue("unexpected non-null value at index: " + i, vector.isNull(i));
+        }
+      }
+
+      vector.setSafe(valueCapacity, 20000000);
+      assertEquals(valueCapacity * 2, vector.getValueCapacity());
+
+      for (int i = 0; i < vector.getValueCapacity(); i++) {
+        if (i == valueCapacity) {
+          assertFalse("unexpected null value at index: " + i, vector.isNull(i));
+          assertEquals("unexpected value at index: " + i, 20000000, vector.get(i));
+        } else if (i < valueCapacity) {
+          if ((i & 1) == 1) {
+            assertFalse("unexpected null value at index: " + i, vector.isNull(i));
+            assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i));
+          }
+        } else {
+          assertTrue("unexpected non-null value at index: " + i, vector.isNull(i));
+        }
+      }
+
+      vector.zeroVector();
+
+      for (int i = 0; i < vector.getValueCapacity(); i+=2) {
+          vector.set(i, baseValue + i);
+      }
+
+      for (int i = 0; i < vector.getValueCapacity(); i++) {
+        if (i%2 == 0) {
+          assertFalse("unexpected null value at index: " + i, vector.isNull(i));
+          assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i));
+        } else {
+          assertTrue("unexpected non-null value at index: " + i, vector.isNull(i));
+        }
+      }
+
+      vector.setSafe((valueCapacity *  2) + 1000, 400000000);
+      assertEquals(valueCapacity * 4, vector.getValueCapacity());
+
+      for (int i = 0; i < vector.getValueCapacity(); i++) {
+        if (i == (valueCapacity*2 + 1000)) {
+          assertFalse("unexpected null value at index: " + i, vector.isNull(i));
+          assertEquals("unexpected value at index: " + i, 400000000, vector.get(i));
+        } else if (i < valueCapacity*2 && (i%2) == 0) {
+          assertFalse("unexpected null value at index: " + i, vector.isNull(i));
+          assertEquals("unexpected value at index: " + i, baseValue + i, vector.get(i));
+        } else {
+          assertTrue("unexpected non-null value at index: " + i, vector.isNull(i));
+        }
+      }
+
+      /* reset the vector */
+      vector.reset();
+
+       /* capacity shouldn't change after reset */
+      assertEquals(valueCapacity * 4, vector.getValueCapacity());
+
+      /* vector data should be zeroed out */
+      for(int i = 0; i < (valueCapacity * 4); i++) {
+        assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
       }
     }
   }
@@ -805,32 +891,30 @@ public void testNullableVarType1() {
 
     // Create a new value vector for 1024 integers.
     try (final NullableVarCharVector vector = newNullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
-      final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024 * 10, 1024);
 
-      m.set(0, STR1);
-      m.set(1, STR2);
-      m.set(2, STR3);
-      m.setSafe(3, STR3, 1, STR3.length - 1);
-      m.setSafe(4, STR3, 2, STR3.length - 2);
+      vector.set(0, STR1);
+      vector.set(1, STR2);
+      vector.set(2, STR3);
+      vector.setSafe(3, STR3, 1, STR3.length - 1);
+      vector.setSafe(4, STR3, 2, STR3.length - 2);
       ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3);
-      m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1);
-      m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2);
+      vector.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1);
+      vector.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2);
 
       // Check the sample strings.
-      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
-      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3));
-      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4));
-      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5));
-      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6));
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(1));
+      assertArrayEquals(STR3, vector.get(2));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(3));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(4));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(5));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(6));
 
       // Ensure null value throws.
       boolean b = false;
       try {
-        vector.getAccessor().get(7);
+        vector.get(7);
       } catch (IllegalStateException e) {
         b = true;
       } finally {
@@ -1070,43 +1154,42 @@ public void testReallocAfterVectorTransfer2() {
   @Test /* NullableVarCharVector */
   public void testReallocAfterVectorTransfer3() {
     try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
-      final NullableVarCharVector.Mutator mutator = vector.getMutator();
-      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
-
       /* 4096 values with 10 byte per record */
       vector.allocateNew(4096 * 10, 4096);
       int valueCapacity = vector.getValueCapacity();
+      assertEquals(4096, valueCapacity);
 
       /* populate the vector */
       for (int i = 0; i < valueCapacity; i++) {
         if ((i & 1) == 1) {
-          mutator.set(i, STR1);
+          vector.set(i, STR1);
         }
         else {
-          mutator.set(i, STR2);
+          vector.set(i, STR2);
         }
       }
 
       /* Check the vector output */
       for (int i = 0; i < valueCapacity; i++) {
         if ((i & 1) == 1) {
-          assertArrayEquals(STR1, accessor.get(i));
+          assertArrayEquals(STR1, vector.get(i));
         }
         else {
-          assertArrayEquals(STR2, accessor.get(i));
+          assertArrayEquals(STR2, vector.get(i));
         }
       }
 
       /* trigger first realloc */
-      mutator.setSafe(valueCapacity, STR2, 0, STR2.length);
+      vector.setSafe(valueCapacity, STR2, 0, STR2.length);
+      assertEquals(valueCapacity * 2, vector.getValueCapacity());
 
       /* populate the remaining vector */
       for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
         if ((i & 1) == 1) {
-          mutator.set(i, STR1);
+          vector.set(i, STR1);
         }
         else {
-          mutator.set(i, STR2);
+          vector.set(i, STR2);
         }
       }
 
@@ -1114,23 +1197,24 @@ public void testReallocAfterVectorTransfer3() {
       valueCapacity = vector.getValueCapacity();
       for (int i = 0; i < valueCapacity; i++) {
         if ((i & 1) == 1) {
-          assertArrayEquals(STR1, accessor.get(i));
+          assertArrayEquals(STR1, vector.get(i));
         }
         else {
-          assertArrayEquals(STR2, accessor.get(i));
+          assertArrayEquals(STR2, vector.get(i));
         }
       }
 
       /* trigger second realloc */
-      mutator.setSafe(valueCapacity + 10, STR2, 0, STR2.length);
+      vector.setSafe(valueCapacity + 10, STR2, 0, STR2.length);
+      assertEquals(valueCapacity * 2, vector.getValueCapacity());
 
       /* populate the remaining vector */
       for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
         if ((i & 1) == 1) {
-          mutator.set(i, STR1);
+          vector.set(i, STR1);
         }
         else {
-          mutator.set(i, STR2);
+          vector.set(i, STR2);
         }
       }
 
@@ -1138,10 +1222,10 @@ public void testReallocAfterVectorTransfer3() {
       valueCapacity = vector.getValueCapacity();
       for (int i = 0; i < valueCapacity; i++) {
         if ((i & 1) == 1) {
-          assertArrayEquals(STR1, accessor.get(i));
+          assertArrayEquals(STR1, vector.get(i));
         }
         else {
-          assertArrayEquals(STR2, accessor.get(i));
+          assertArrayEquals(STR2, vector.get(i));
         }
       }
 
@@ -1152,13 +1236,112 @@ public void testReallocAfterVectorTransfer3() {
       TransferPair transferPair = vector.getTransferPair(allocator);
       transferPair.transfer();
       NullableVarCharVector toVector = (NullableVarCharVector)transferPair.getTo();
-      NullableVarCharVector.Mutator toMutator = toVector.getMutator();
-      NullableVarCharVector.Accessor toAccessor = toVector.getAccessor();
-
       valueCapacity = toVector.getValueCapacity();
 
-      /* trigger a realloc of this toVector */
-      toMutator.setSafe(valueCapacity + 10, STR2, 0, STR2.length);
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 1) {
+          assertArrayEquals(STR1, toVector.get(i));
+        }
+        else {
+          assertArrayEquals(STR2, toVector.get(i));
+        }
+      }
+
+      toVector.close();
+    }
+  }
+
+  @Test /* NullableIntVector */
+  public void testReallocAfterVectorTransfer4() {
+    try (final NullableIntVector vector = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      /* 4096 values  */
+      vector.allocateNew(4096);
+      int valueCapacity = vector.getValueCapacity();
+      assertEquals(4096, valueCapacity);
+
+      /* populate the vector */
+      int baseValue = 1000;
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 0) {
+          vector.set(i, 1000 + i);
+        }
+      }
+
+      /* Check the vector output */
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 0) {
+          assertEquals(1000 + i, vector.get(i));
+        }
+        else {
+          assertTrue(vector.isNull(i));
+        }
+      }
+
+      /* trigger first realloc */
+      vector.setSafe(valueCapacity, 10000000);
+      assertEquals(valueCapacity * 2, vector.getValueCapacity());
+
+      /* populate the remaining vector */
+      for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
+        if ((i & 1) == 0) {
+          vector.set(i, 1000 + i);
+        }
+      }
+
+      /* Check the vector output */
+      valueCapacity = vector.getValueCapacity();
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 0) {
+          assertEquals(1000 + i, vector.get(i));
+        }
+        else {
+          assertTrue(vector.isNull(i));
+        }
+      }
+
+      /* trigger second realloc */
+      vector.setSafe(valueCapacity, 10000000);
+      assertEquals(valueCapacity * 2, vector.getValueCapacity());
+
+      /* populate the remaining vector */
+      for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
+        if ((i & 1) == 0) {
+          vector.set(i, 1000 + i);
+        }
+      }
+
+      /* Check the vector output */
+      valueCapacity = vector.getValueCapacity();
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 0) {
+          assertEquals(1000 + i, vector.get(i));
+        }
+        else {
+          assertTrue(vector.isNull(i));
+        }
+      }
+
+      /* we are potentially working with 4x the size of vector buffer
+       * that we initially started with. Now let's transfer the vector.
+       */
+
+      TransferPair transferPair = vector.getTransferPair(allocator);
+      transferPair.transfer();
+      NullableIntVector toVector = (NullableIntVector)transferPair.getTo();
+      /* value capacity of source and target vectors should be same after
+       * the transfer.
+       */
+      assertEquals(valueCapacity, toVector.getValueCapacity());
+
+      for (int i = 0; i < valueCapacity; i++) {
+        if ((i & 1) == 0) {
+          assertEquals(1000 + i, toVector.get(i));
+        }
+        else {
+          assertTrue(toVector.isNull(i));
+        }
+      }
 
       toVector.close();
     }
@@ -1199,31 +1382,33 @@ public void testReAllocNullableFixedWidthVector() {
 
   @Test
   public void testReAllocNullableVariableWidthVector() {
-    // Create a new value vector for 1024 integers
     try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
-      final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew();
 
       int initialCapacity = vector.getValueCapacity();
+      assertEquals(4095, initialCapacity);
 
-      // Put values in indexes that fall within the initial allocation
-      m.setSafe(0, STR1, 0, STR1.length);
-      m.setSafe(initialCapacity - 1, STR2, 0, STR2.length);
+      /* Put values in indexes that fall within the initial allocation */
+      vector.setSafe(0, STR1, 0, STR1.length);
+      vector.setSafe(initialCapacity - 1, STR2, 0, STR2.length);
 
-      // Now try to put values in space that falls beyond the initial allocation
-      m.setSafe(initialCapacity + 200, STR3, 0, STR3.length);
+      /* the above set calls should NOT have triggered a realloc */
+      initialCapacity = vector.getValueCapacity();
+      assertEquals(4095, initialCapacity);
 
-      // Check valueCapacity is more than initial allocation
-      assertEquals((initialCapacity + 1) * 2 - 1, vector.getValueCapacity());
+      /* Now try to put values in space that falls beyond the initial allocation */
+      vector.setSafe(initialCapacity + 200, STR3, 0, STR3.length);
 
-      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(initialCapacity - 1));
-      assertArrayEquals(STR3, accessor.get(initialCapacity + 200));
+      /* Check valueCapacity is more than initial allocation */
+      assertEquals(((initialCapacity + 1) * 2) - 1, vector.getValueCapacity());
+
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(initialCapacity - 1));
+      assertArrayEquals(STR3, vector.get(initialCapacity + 200));
 
       // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors
       // as we don't call setSafe for null values, but we do call setValueCount when the current batch is processed.
-      m.setValueCount(vector.getValueCapacity() + 200);
+      vector.setValueCount(vector.getValueCapacity() + 200);
     }
   }
 
@@ -1232,10 +1417,18 @@ public void testFillEmptiesNotOverfill() {
     try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
       vector.allocateNew();
 
-      vector.getMutator().setSafe(4094, "hello".getBytes(), 0, 5);
-      vector.getMutator().setValueCount(4095);
+      int initialCapacity = vector.getValueCapacity();
+      assertEquals(4095, initialCapacity);
+
+      vector.setSafe(4094, "hello".getBytes(), 0, 5);
+      /* the above set method should NOT have trigerred a realloc */
+      initialCapacity = vector.getValueCapacity();
+      assertEquals(4095, initialCapacity);
 
-      assertEquals(4096 * 4, vector.getFieldBuffers().get(1).capacity());
+      vector.setValueCount(4095);
+      assertEquals(4096 * vector.OFFSET_WIDTH, vector.getFieldBuffers().get(1).capacity());
+      initialCapacity = vector.getValueCapacity();
+      assertEquals(4095, initialCapacity);
     }
   }
 
@@ -1243,42 +1436,129 @@ public void testFillEmptiesNotOverfill() {
   public void testCopyFromWithNulls() {
     try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
          final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+
       vector.allocateNew();
+      int capacity = vector.getValueCapacity();
+      assertEquals(4095, capacity);
 
       for (int i = 0; i < 4095; i++) {
         if (i % 3 == 0) {
           continue;
         }
         byte[] b = Integer.toString(i).getBytes();
-        vector.getMutator().setSafe(i, b, 0, b.length);
+        vector.setSafe(i, b, 0, b.length);
       }
 
-      vector.getMutator().setValueCount(4095);
+      /* NO reAlloc() should have happened in setSafe() */
+      capacity = vector.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      vector.setValueCount(4095);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          assertNull(vector.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString());
+        }
+      }
 
       vector2.allocateNew();
+      capacity = vector2.getValueCapacity();
+      assertEquals(4095, capacity);
 
       for (int i = 0; i < 4095; i++) {
         vector2.copyFromSafe(i, i, vector);
+        if (i % 3 == 0) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+        }
       }
 
-      vector2.getMutator().setValueCount(4095);
+      /* NO reAlloc() should have happened in copyFrom */
+      capacity = vector2.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      vector2.setValueCount(4095);
 
       for (int i = 0; i < 4095; i++) {
         if (i % 3 == 0) {
-          assertNull(vector2.getAccessor().getObject(i));
+          assertNull(vector2.getObject(i));
         } else {
-          assertEquals(Integer.toString(i), vector2.getAccessor().getObject(i).toString());
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
         }
       }
     }
   }
 
   @Test
-  public void testSetLastSetUsage() {
-    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
+  public void testCopyFromWithNulls1() {
+    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
+         final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
 
-      final NullableVarCharVector.Mutator mutator = vector.getMutator();
+      vector.allocateNew();
+      int capacity = vector.getValueCapacity();
+      assertEquals(4095, capacity);
 
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          continue;
+        }
+        byte[] b = Integer.toString(i).getBytes();
+        vector.setSafe(i, b, 0, b.length);
+      }
+
+      /* NO reAlloc() should have happened in setSafe() */
+      capacity = vector.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      vector.setValueCount(4095);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          assertNull(vector.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString());
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024 * 10, 1024);
+
+      capacity = vector2.getValueCapacity();
+      assertEquals(1024, capacity);
+
+      for (int i = 0; i < 4095; i++) {
+        vector2.copyFromSafe(i, i, vector);
+        if (i % 3 == 0) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+        }
+      }
+
+      /* 2 reAllocs should have happened in copyFromSafe() */
+      capacity = vector2.getValueCapacity();
+      assertEquals(4096, capacity);
+
+      vector2.setValueCount(4095);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testSetLastSetUsage() {
+    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
       vector.allocateNew(1024 * 10, 1024);
 
       setBytes(0, STR1, vector);
@@ -1289,32 +1569,102 @@ public void testSetLastSetUsage() {
       setBytes(5, STR6, vector);
 
       /* Check current lastSet */
-      assertEquals(Integer.toString(-1), Integer.toString(mutator.getLastSet()));
+      assertEquals(Integer.toString(-1), Integer.toString(vector.getLastSet()));
 
       /* Check the vector output */
-      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
-      assertArrayEquals(STR4, accessor.get(3));
-      assertArrayEquals(STR5, accessor.get(4));
-      assertArrayEquals(STR6, accessor.get(5));
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(1));
+      assertArrayEquals(STR3, vector.get(2));
+      assertArrayEquals(STR4, vector.get(3));
+      assertArrayEquals(STR5, vector.get(4));
+      assertArrayEquals(STR6, vector.get(5));
 
       /*
        * If we don't do setLastSe(5) before setValueCount(), then the latter will corrupt
        * the value vector by filling in all positions [0,valuecount-1] will empty byte arrays.
        * Run the test by commenting out next line and we should see incorrect vector output.
        */
-      mutator.setLastSet(5);
-      mutator.setValueCount(20);
+      vector.setLastSet(5);
+      vector.setValueCount(20);
+
+      /* Check current lastSet */
+      assertEquals(Integer.toString(19), Integer.toString(vector.getLastSet()));
 
       /* Check the vector output again */
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
-      assertArrayEquals(STR4, accessor.get(3));
-      assertArrayEquals(STR5, accessor.get(4));
-      assertArrayEquals(STR6, accessor.get(5));
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(1));
+      assertArrayEquals(STR3, vector.get(2));
+      assertArrayEquals(STR4, vector.get(3));
+      assertArrayEquals(STR5, vector.get(4));
+      assertArrayEquals(STR6, vector.get(5));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(10)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(11)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(12)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(13)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(14)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(15)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(16)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(17)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(18)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(19)));
+
+      /* Check offsets */
+      assertEquals(Integer.toString(0),
+              Integer.toString(vector.offsetBuffer.getInt(0 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(6),
+              Integer.toString(vector.offsetBuffer.getInt(1 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(16),
+              Integer.toString(vector.offsetBuffer.getInt(2 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(21),
+              Integer.toString(vector.offsetBuffer.getInt(3 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(30),
+              Integer.toString(vector.offsetBuffer.getInt(4 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(34),
+              Integer.toString(vector.offsetBuffer.getInt(5 * vector.OFFSET_WIDTH)));
+
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(6 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(7 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(8 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(9 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(10 * vector.OFFSET_WIDTH)));
+
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(11 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(12 * vector.OFFSET_WIDTH)));
+
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(13 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(14 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(15 * vector.OFFSET_WIDTH)));
+
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(16 * vector.OFFSET_WIDTH)));
+
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(17 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(18 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(19 * vector.OFFSET_WIDTH)));
+
+      vector.set(19, STR6);
+      assertArrayEquals(STR6, vector.get(19));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(19 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(46),
+              Integer.toString(vector.offsetBuffer.getInt(20 * vector.OFFSET_WIDTH)));
     }
   }
 
@@ -1322,29 +1672,25 @@ public void testSetLastSetUsage() {
   public void testVectorLoadUnload() {
 
     try (final NullableVarCharVector vector1 = new NullableVarCharVector("myvector", allocator)) {
-
-      final NullableVarCharVector.Mutator mutator1 = vector1.getMutator();
-
       vector1.allocateNew(1024 * 10, 1024);
 
-      mutator1.set(0, STR1);
-      mutator1.set(1, STR2);
-      mutator1.set(2, STR3);
-      mutator1.set(3, STR4);
-      mutator1.set(4, STR5);
-      mutator1.set(5, STR6);
-      assertEquals(Integer.toString(5), Integer.toString(mutator1.getLastSet()));
-      mutator1.setValueCount(15);
-      assertEquals(Integer.toString(14), Integer.toString(mutator1.getLastSet()));
+      vector1.set(0, STR1);
+      vector1.set(1, STR2);
+      vector1.set(2, STR3);
+      vector1.set(3, STR4);
+      vector1.set(4, STR5);
+      vector1.set(5, STR6);
+      assertEquals(Integer.toString(5), Integer.toString(vector1.getLastSet()));
+      vector1.setValueCount(15);
+      assertEquals(Integer.toString(14), Integer.toString(vector1.getLastSet()));
 
       /* Check the vector output */
-      final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
-      assertArrayEquals(STR1, accessor1.get(0));
-      assertArrayEquals(STR2, accessor1.get(1));
-      assertArrayEquals(STR3, accessor1.get(2));
-      assertArrayEquals(STR4, accessor1.get(3));
-      assertArrayEquals(STR5, accessor1.get(4));
-      assertArrayEquals(STR6, accessor1.get(5));
+      assertArrayEquals(STR1, vector1.get(0));
+      assertArrayEquals(STR2, vector1.get(1));
+      assertArrayEquals(STR3, vector1.get(2));
+      assertArrayEquals(STR4, vector1.get(3));
+      assertArrayEquals(STR5, vector1.get(4));
+      assertArrayEquals(STR6, vector1.get(5));
 
       Field field = vector1.getField();
       String fieldName = field.getName();
@@ -1357,7 +1703,7 @@ public void testVectorLoadUnload() {
 
       Schema schema = new Schema(fields);
 
-      VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, accessor1.getValueCount());
+      VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, vector1.getValueCount());
       VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1);
 
       try (
@@ -1370,24 +1716,21 @@ public void testVectorLoadUnload() {
         vectorLoader.load(recordBatch);
 
         NullableVarCharVector vector2 = (NullableVarCharVector) schemaRoot2.getVector(fieldName);
-        NullableVarCharVector.Mutator mutator2 = vector2.getMutator();
-
         /*
          * lastSet would have internally been set by VectorLoader.load() when it invokes
          * loadFieldBuffers.
          */
-        assertEquals(Integer.toString(14), Integer.toString(mutator2.getLastSet()));
-        mutator2.setValueCount(25);
-        assertEquals(Integer.toString(24), Integer.toString(mutator2.getLastSet()));
+        assertEquals(Integer.toString(14), Integer.toString(vector2.getLastSet()));
+        vector2.setValueCount(25);
+        assertEquals(Integer.toString(24), Integer.toString(vector2.getLastSet()));
 
         /* Check the vector output */
-        final NullableVarCharVector.Accessor accessor2 = vector2.getAccessor();
-        assertArrayEquals(STR1, accessor2.get(0));
-        assertArrayEquals(STR2, accessor2.get(1));
-        assertArrayEquals(STR3, accessor2.get(2));
-        assertArrayEquals(STR4, accessor2.get(3));
-        assertArrayEquals(STR5, accessor2.get(4));
-        assertArrayEquals(STR6, accessor2.get(5));
+        assertArrayEquals(STR1, vector2.get(0));
+        assertArrayEquals(STR2, vector2.get(1));
+        assertArrayEquals(STR3, vector2.get(2));
+        assertArrayEquals(STR4, vector2.get(3));
+        assertArrayEquals(STR5, vector2.get(4));
+        assertArrayEquals(STR6, vector2.get(5));
       }
     }
   }
@@ -1396,8 +1739,6 @@ public void testVectorLoadUnload() {
   public void testFillEmptiesUsage() {
     try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
 
-      final NullableVarCharVector.Mutator mutator = vector.getMutator();
-
       vector.allocateNew(1024 * 10, 1024);
 
       setBytes(0, STR1, vector);
@@ -1408,84 +1749,98 @@ public void testFillEmptiesUsage() {
       setBytes(5, STR6, vector);
 
       /* Check current lastSet */
-      assertEquals(Integer.toString(-1), Integer.toString(mutator.getLastSet()));
+      assertEquals(Integer.toString(-1), Integer.toString(vector.getLastSet()));
 
       /* Check the vector output */
-      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
-      assertArrayEquals(STR4, accessor.get(3));
-      assertArrayEquals(STR5, accessor.get(4));
-      assertArrayEquals(STR6, accessor.get(5));
-
-      mutator.setLastSet(5);
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(1));
+      assertArrayEquals(STR3, vector.get(2));
+      assertArrayEquals(STR4, vector.get(3));
+      assertArrayEquals(STR5, vector.get(4));
+      assertArrayEquals(STR6, vector.get(5));
+
+      vector.setLastSet(5);
       /* fill empty byte arrays from index [6, 9] */
-      mutator.fillEmpties(10);
+      vector.fillEmpties(10);
 
       /* Check current lastSet */
-      assertEquals(Integer.toString(9), Integer.toString(mutator.getLastSet()));
+      assertEquals(Integer.toString(9), Integer.toString(vector.getLastSet()));
 
       /* Check the vector output */
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
-      assertArrayEquals(STR4, accessor.get(3));
-      assertArrayEquals(STR5, accessor.get(4));
-      assertArrayEquals(STR6, accessor.get(5));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(6)));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(7)));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(8)));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(9)));
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(1));
+      assertArrayEquals(STR3, vector.get(2));
+      assertArrayEquals(STR4, vector.get(3));
+      assertArrayEquals(STR5, vector.get(4));
+      assertArrayEquals(STR6, vector.get(5));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9)));
 
       setBytes(10, STR1, vector);
       setBytes(11, STR2, vector);
 
-      mutator.setLastSet(11);
+      vector.setLastSet(11);
       /* fill empty byte arrays from index [12, 14] */
-      mutator.setValueCount(15);
+      vector.setValueCount(15);
 
       /* Check current lastSet */
-      assertEquals(Integer.toString(14), Integer.toString(mutator.getLastSet()));
+      assertEquals(Integer.toString(14), Integer.toString(vector.getLastSet()));
 
       /* Check the vector output */
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
-      assertArrayEquals(STR4, accessor.get(3));
-      assertArrayEquals(STR5, accessor.get(4));
-      assertArrayEquals(STR6, accessor.get(5));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(6)));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(7)));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(8)));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(9)));
-      assertArrayEquals(STR1, accessor.get(10));
-      assertArrayEquals(STR2, accessor.get(11));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(12)));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(13)));
-      assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(14)));
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(1));
+      assertArrayEquals(STR3, vector.get(2));
+      assertArrayEquals(STR4, vector.get(3));
+      assertArrayEquals(STR5, vector.get(4));
+      assertArrayEquals(STR6, vector.get(5));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9)));
+      assertArrayEquals(STR1, vector.get(10));
+      assertArrayEquals(STR2, vector.get(11));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(12)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(13)));
+      assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(14)));
 
       /* Check offsets */
-      final UInt4Vector.Accessor offsetAccessor = vector.values.offsetVector.getAccessor();
-      assertEquals(Integer.toString(0), Integer.toString(offsetAccessor.get(0)));
-      assertEquals(Integer.toString(6), Integer.toString(offsetAccessor.get(1)));
-      assertEquals(Integer.toString(16), Integer.toString(offsetAccessor.get(2)));
-      assertEquals(Integer.toString(21), Integer.toString(offsetAccessor.get(3)));
-      assertEquals(Integer.toString(30), Integer.toString(offsetAccessor.get(4)));
-      assertEquals(Integer.toString(34), Integer.toString(offsetAccessor.get(5)));
-
-      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(6)));
-      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(7)));
-      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(8)));
-      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(9)));
-      assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(10)));
-
-      assertEquals(Integer.toString(46), Integer.toString(offsetAccessor.get(11)));
-      assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(12)));
-
-      assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(13)));
-      assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(14)));
-      assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(15)));
+      assertEquals(Integer.toString(0),
+              Integer.toString(vector.offsetBuffer.getInt(0 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(6),
+              Integer.toString(vector.offsetBuffer.getInt(1 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(16),
+              Integer.toString(vector.offsetBuffer.getInt(2 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(21),
+              Integer.toString(vector.offsetBuffer.getInt(3 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(30),
+              Integer.toString(vector.offsetBuffer.getInt(4 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(34),
+              Integer.toString(vector.offsetBuffer.getInt(5 * vector.OFFSET_WIDTH)));
+
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(6 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(7 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(8 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(9 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(40),
+              Integer.toString(vector.offsetBuffer.getInt(10 * vector.OFFSET_WIDTH)));
+
+      assertEquals(Integer.toString(46),
+              Integer.toString(vector.offsetBuffer.getInt(11 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(56),
+              Integer.toString(vector.offsetBuffer.getInt(12 * vector.OFFSET_WIDTH)));
+
+      assertEquals(Integer.toString(56),
+              Integer.toString(vector.offsetBuffer.getInt(13 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(56),
+              Integer.toString(vector.offsetBuffer.getInt(14 * vector.OFFSET_WIDTH)));
+      assertEquals(Integer.toString(56),
+              Integer.toString(vector.offsetBuffer.getInt(15 * vector.OFFSET_WIDTH)));
     }
   }
 
@@ -1493,29 +1848,25 @@ public void testFillEmptiesUsage() {
   public void testGetBufferAddress1() {
 
     try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
-
-      final NullableVarCharVector.Mutator mutator = vector.getMutator();
-      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
-
       vector.allocateNew(1024 * 10, 1024);
 
       /* populate the vector */
-      mutator.set(0, STR1);
-      mutator.set(1, STR2);
-      mutator.set(2, STR3);
-      mutator.set(3, STR4);
-      mutator.set(4, STR5);
-      mutator.set(5, STR6);
+      vector.set(0, STR1);
+      vector.set(1, STR2);
+      vector.set(2, STR3);
+      vector.set(3, STR4);
+      vector.set(4, STR5);
+      vector.set(5, STR6);
 
-      mutator.setValueCount(15);
+      vector.setValueCount(15);
 
       /* check the vector output */
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
-      assertArrayEquals(STR4, accessor.get(3));
-      assertArrayEquals(STR5, accessor.get(4));
-      assertArrayEquals(STR6, accessor.get(5));
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(1));
+      assertArrayEquals(STR3, vector.get(2));
+      assertArrayEquals(STR4, vector.get(3));
+      assertArrayEquals(STR5, vector.get(4));
+      assertArrayEquals(STR6, vector.get(5));
 
       List<ArrowBuf> buffers = vector.getFieldBuffers();
       long bitAddress = vector.getValidityBufferAddress();
@@ -1531,23 +1882,18 @@ public void testGetBufferAddress1() {
 
   @Test /* NullableIntVector */
   public void testGetBufferAddress2() {
-
     try (final NullableIntVector vector = new NullableIntVector("myvector", allocator)) {
-
-      final NullableIntVector.Mutator mutator = vector.getMutator();
-      final NullableIntVector.Accessor accessor = vector.getAccessor();
       boolean error = false;
-
       vector.allocateNew(16);
 
       /* populate the vector */
       for(int i = 0; i < 16; i += 2) {
-        mutator.set(i, i+10);
+        vector.set(i, i+10);
       }
 
       /* check the vector output */
       for(int i = 0; i < 16; i += 2) {
-        assertEquals(i+10, accessor.get(i));
+        assertEquals(i+10, vector.get(i));
       }
 
       List<ArrowBuf> buffers = vector.getFieldBuffers();
@@ -1580,11 +1926,15 @@ public void testMultipleClose() {
     vectorAllocator.close();
   }
 
+  /* this method is used by the tests to bypass the vector set methods that manipulate
+   * lastSet. The method is to test the lastSet property and that's why we load the vector
+   * in a way that lastSet is not set automatically.
+   */
   public static void setBytes(int index, byte[] bytes, NullableVarCharVector vector) {
-    final int currentOffset = vector.values.offsetVector.getAccessor().get(index);
+    final int currentOffset = vector.offsetBuffer.getInt(index * vector.OFFSET_WIDTH);
 
-    vector.bits.getMutator().setToOne(index);
-    vector.values.offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
-    vector.values.data.setBytes(currentOffset, bytes, 0, bytes.length);
+    BitVectorHelper.setValidityBitToOne(vector.validityBuffer, index);
+    vector.offsetBuffer.setInt((index + 1) * vector.OFFSET_WIDTH, currentOffset + bytes.length);
+    vector.valueBuffer.setBytes(currentOffset, bytes, 0, bytes.length);
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index 4ac7536c017db..531a46c2734ef 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -76,24 +76,23 @@ public void testFixedType() {
   @Test
   public void testNullableType() {
     try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
-      final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.setInitialCapacity(512);
       vector.allocateNew();
 
       assertEquals(512, vector.getValueCapacity());
 
       try {
-        m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+        vector.set(512, "foo".getBytes(StandardCharsets.UTF_8));
         Assert.fail("Expected out of bounds exception");
       } catch (Exception e) {
         // ok
       }
 
       vector.reAlloc();
-      assertEquals(1023, vector.getValueCapacity());
+      assertEquals(1024, vector.getValueCapacity());
 
-      m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
-      assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+      vector.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+      assertEquals("foo", new String(vector.get(512), StandardCharsets.UTF_8));
     }
   }
 
@@ -105,7 +104,7 @@ public void testListType() {
       vector.setInitialCapacity(512);
       vector.allocateNew();
 
-      assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+      assertEquals(1023, vector.getValueCapacity());
 
       try {
         vector.getOffsetVector().getAccessor().get(2014);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 7facf73f511da..0b7928dca2c39 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -45,13 +45,21 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.*;
 
 public class TestVectorUnloadLoad {
 
-  static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
 
   @Test
   public void testUnloadLoad() throws IOException {
@@ -183,24 +191,40 @@ public void testUnloadLoadAddPadding() throws IOException {
    * @throws IOException
    */
   @Test
-  public void testLoadEmptyValidityBuffer() throws IOException {
+  public void testLoadValidityBuffer() throws IOException {
     Schema schema = new Schema(asList(
         new Field("intDefined", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList()),
         new Field("intNull", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList())
     ));
     int count = 10;
-    ArrowBuf validity = allocator.buffer(10).slice(0, 0);
-    ArrowBuf[] values = new ArrowBuf[2];
-    for (int i = 0; i < values.length; i++) {
-      ArrowBuf arrowBuf = allocator.buffer(count * 4); // integers
-      values[i] = arrowBuf;
+    ArrowBuf[] values = new ArrowBuf[4];
+    for (int i = 0; i < 4; i+=2) {
+      ArrowBuf buf1 = allocator.buffer((int)Math.ceil(count / 8.0));
+      ArrowBuf buf2 = allocator.buffer(count * 4); // integers
+      values[i] = buf1;
+      values[i+1] = buf2;
       for (int j = 0; j < count; j++) {
-        arrowBuf.setInt(j * 4, j);
+        if (i == 2) {
+          BitVectorHelper.setValidityBit(buf1, j, 0);
+        } else {
+          BitVectorHelper.setValidityBitToOne(buf1, j);
+        }
+
+        buf2.setInt(j * 4, j);
       }
-      arrowBuf.writerIndex(count * 4);
+      buf1.writerIndex((int)Math.ceil(count / 8));
+      buf2.writerIndex(count * 4);
     }
+
+    /*
+     * values[0] - validity buffer for first vector
+     * values[1] - data buffer for first vector
+     * values[2] - validity buffer for second vector
+     * values[3] - data buffer for second vector
+     */
+
     try (
-        ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(validity, values[0], validity, values[1]));
+        ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(values[0], values[1], values[2], values[3]));
         BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE);
         VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator);
     ) {
@@ -213,32 +237,31 @@ public void testLoadEmptyValidityBuffer() throws IOException {
       NullableIntVector intDefinedVector = (NullableIntVector) newRoot.getVector("intDefined");
       NullableIntVector intNullVector = (NullableIntVector) newRoot.getVector("intNull");
       for (int i = 0; i < count; i++) {
-        assertFalse("#" + i, intDefinedVector.getAccessor().isNull(i));
-        assertEquals("#" + i, i, intDefinedVector.getAccessor().get(i));
-        assertTrue("#" + i, intNullVector.getAccessor().isNull(i));
+        assertFalse("#" + i, intDefinedVector.isNull(i));
+        assertEquals("#" + i, i, intDefinedVector.get(i));
+        assertTrue("#" + i, intNullVector.isNull(i));
       }
-      intDefinedVector.getMutator().setSafe(count + 10, 1234);
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 1));
+      intDefinedVector.setSafe(count + 10, 1234);
+      assertTrue(intDefinedVector.isNull(count + 1));
       // empty slots should still default to unset
-      intDefinedVector.getMutator().setSafe(count + 1, 789);
-      assertFalse(intDefinedVector.getAccessor().isNull(count + 1));
-      assertEquals(789, intDefinedVector.getAccessor().get(count + 1));
-      assertTrue(intDefinedVector.getAccessor().isNull(count));
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 2));
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 3));
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 4));
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 5));
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 6));
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 7));
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 8));
-      assertTrue(intDefinedVector.getAccessor().isNull(count + 9));
-      assertFalse(intDefinedVector.getAccessor().isNull(count + 10));
-      assertEquals(1234, intDefinedVector.getAccessor().get(count + 10));
+      intDefinedVector.setSafe(count + 1, 789);
+      assertFalse(intDefinedVector.isNull(count + 1));
+      assertEquals(789, intDefinedVector.get(count + 1));
+      assertTrue(intDefinedVector.isNull(count));
+      assertTrue(intDefinedVector.isNull(count + 2));
+      assertTrue(intDefinedVector.isNull(count + 3));
+      assertTrue(intDefinedVector.isNull(count + 4));
+      assertTrue(intDefinedVector.isNull(count + 5));
+      assertTrue(intDefinedVector.isNull(count + 6));
+      assertTrue(intDefinedVector.isNull(count + 7));
+      assertTrue(intDefinedVector.isNull(count + 8));
+      assertTrue(intDefinedVector.isNull(count + 9));
+      assertFalse(intDefinedVector.isNull(count + 10));
+      assertEquals(1234, intDefinedVector.get(count + 10));
     } finally {
       for (ArrowBuf arrowBuf : values) {
         arrowBuf.release();
       }
-      validity.release();
     }
   }
 
@@ -258,11 +281,11 @@ public void testUnloadLoadDuplicates() throws IOException {
         FieldVector vector = field.createVector(originalVectorsAllocator);
         vector.allocateNew();
         sources.add(vector);
-        NullableIntVector.Mutator mutator = (NullableIntVector.Mutator) vector.getMutator();
+        NullableIntVector intVector = (NullableIntVector)vector;
         for (int i = 0; i < count; i++) {
-          mutator.set(i, i);
+          intVector.set(i, i);
         }
-        mutator.setValueCount(count);
+        intVector.setValueCount(count);
       }
 
       try (VectorSchemaRoot root = new VectorSchemaRoot(schema.getFields(), sources, count)) {
@@ -277,8 +300,8 @@ public void testUnloadLoadDuplicates() throws IOException {
           List<FieldVector> targets = newRoot.getFieldVectors();
           Assert.assertEquals(sources.size(), targets.size());
           for (int k = 0; k < sources.size(); k++) {
-            NullableIntVector.Accessor src = (NullableIntVector.Accessor) sources.get(k).getAccessor();
-            NullableIntVector.Accessor tgt = (NullableIntVector.Accessor) targets.get(k).getAccessor();
+            NullableIntVector src = (NullableIntVector) sources.get(k);
+            NullableIntVector tgt = (NullableIntVector) targets.get(k);
             Assert.assertEquals(src.getValueCount(), tgt.getValueCount());
             for (int i = 0; i < count; i++) {
               Assert.assertEquals(src.get(i), tgt.get(i));
@@ -296,9 +319,4 @@ public static VectorUnloader newVectorUnloader(FieldVector root) {
     VectorSchemaRoot vsr = new VectorSchemaRoot(schema.getFields(), fields, valueCount);
     return new VectorUnloader(vsr);
   }
-
-  @AfterClass
-  public static void afterClass() {
-    allocator.close();
-  }
 }
\ No newline at end of file
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index ba62de0a6d93a..60009b0a46396 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -97,8 +97,14 @@ public void tearDown() {
 
   protected void validateContent(int count, VectorSchemaRoot root) {
     for (int i = 0; i < count; i++) {
-      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
-      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
+      FieldVector fv = root.getVector("int");
+      if (fv instanceof NullableIntVector) {
+        Assert.assertEquals(i, fv.getObject(i));
+        Assert.assertEquals(Integer.valueOf(i), fv.getObject(i));
+      } else {
+        Assert.assertEquals(i, fv.getAccessor().getObject(i));
+        Assert.assertEquals(Long.valueOf(i), fv.getAccessor().getObject(i));
+      }
     }
   }
 
@@ -152,6 +158,7 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) {
     Assert.assertEquals(count, root.getRowCount());
     printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
+
       Object intVal = root.getVector("int").getAccessor().getObject(i);
       if (i % 5 != 3) {
         Assert.assertEquals(i, intVal);
@@ -220,22 +227,20 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat
     // Define dictionaries and add to provider
     NullableVarCharVector dictionary1Vector = newNullableVarCharVector("D1", bufferAllocator);
     dictionary1Vector.allocateNewSafe();
-    NullableVarCharVector.Mutator mutator = dictionary1Vector.getMutator();
-    mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
-    mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
-    mutator.set(2, "baz".getBytes(StandardCharsets.UTF_8));
-    mutator.setValueCount(3);
+    dictionary1Vector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+    dictionary1Vector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+    dictionary1Vector.set(2, "baz".getBytes(StandardCharsets.UTF_8));
+    dictionary1Vector.setValueCount(3);
 
     Dictionary dictionary1 = new Dictionary(dictionary1Vector, new DictionaryEncoding(1L, false, null));
     provider.put(dictionary1);
 
     NullableVarCharVector dictionary2Vector = newNullableVarCharVector("D2", bufferAllocator);
     dictionary2Vector.allocateNewSafe();
-    mutator = dictionary2Vector.getMutator();
-    mutator.set(0, "micro".getBytes(StandardCharsets.UTF_8));
-    mutator.set(1, "small".getBytes(StandardCharsets.UTF_8));
-    mutator.set(2, "large".getBytes(StandardCharsets.UTF_8));
-    mutator.setValueCount(3);
+    dictionary2Vector.set(0, "micro".getBytes(StandardCharsets.UTF_8));
+    dictionary2Vector.set(1, "small".getBytes(StandardCharsets.UTF_8));
+    dictionary2Vector.set(2, "large".getBytes(StandardCharsets.UTF_8));
+    dictionary2Vector.setValueCount(3);
 
     Dictionary dictionary2 = new Dictionary(dictionary2Vector, new DictionaryEncoding(2L, false, null));
     provider.put(dictionary2);
@@ -243,13 +248,12 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat
     // Populate the vectors
     NullableVarCharVector vector1A = newNullableVarCharVector("varcharA", bufferAllocator);
     vector1A.allocateNewSafe();
-    mutator = vector1A.getMutator();
-    mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
-    mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
-    mutator.set(3, "baz".getBytes(StandardCharsets.UTF_8));
-    mutator.set(4, "bar".getBytes(StandardCharsets.UTF_8));
-    mutator.set(5, "baz".getBytes(StandardCharsets.UTF_8));
-    mutator.setValueCount(6);
+    vector1A.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+    vector1A.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+    vector1A.set(3, "baz".getBytes(StandardCharsets.UTF_8));
+    vector1A.set(4, "bar".getBytes(StandardCharsets.UTF_8));
+    vector1A.set(5, "baz".getBytes(StandardCharsets.UTF_8));
+    vector1A.setValueCount(6);
 
     FieldVector encodedVector1A = (FieldVector) DictionaryEncoder.encode(vector1A, dictionary1);
     vector1A.close();  // Done with this vector after encoding
@@ -257,22 +261,20 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat
     // Write this vector using indices instead of encoding
     NullableIntVector encodedVector1B = new NullableIntVector("varcharB", bufferAllocator);
     encodedVector1B.allocateNewSafe();
-    NullableIntVector.Mutator mutator1B = encodedVector1B.getMutator();
-    mutator1B.set(0, 2);  // "baz"
-    mutator1B.set(1, 1);  // "bar"
-    mutator1B.set(2, 2);  // "baz"
-    mutator1B.set(4, 1);  // "bar"
-    mutator1B.set(5, 0);  // "foo"
-    mutator1B.setValueCount(6);
+    encodedVector1B.set(0, 2);  // "baz"
+    encodedVector1B.set(1, 1);  // "bar"
+    encodedVector1B.set(2, 2);  // "baz"
+    encodedVector1B.set(4, 1);  // "bar"
+    encodedVector1B.set(5, 0);  // "foo"
+    encodedVector1B.setValueCount(6);
 
     NullableVarCharVector vector2 = newNullableVarCharVector("sizes", bufferAllocator);
     vector2.allocateNewSafe();
-    mutator = vector2.getMutator();
-    mutator.set(1, "large".getBytes(StandardCharsets.UTF_8));
-    mutator.set(2, "small".getBytes(StandardCharsets.UTF_8));
-    mutator.set(3, "small".getBytes(StandardCharsets.UTF_8));
-    mutator.set(4, "large".getBytes(StandardCharsets.UTF_8));
-    mutator.setValueCount(6);
+    vector2.set(1, "large".getBytes(StandardCharsets.UTF_8));
+    vector2.set(2, "small".getBytes(StandardCharsets.UTF_8));
+    vector2.set(3, "small".getBytes(StandardCharsets.UTF_8));
+    vector2.set(4, "large".getBytes(StandardCharsets.UTF_8));
+    vector2.setValueCount(6);
 
     FieldVector encodedVector2 = (FieldVector) DictionaryEncoder.encode(vector2, dictionary2);
     vector2.close();  // Done with this vector after encoding
@@ -355,8 +357,8 @@ protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAlloc
     // Define the dictionary and add to the provider
     NullableVarCharVector dictionaryVector = newNullableVarCharVector("D2", bufferAllocator);
     dictionaryVector.allocateNewSafe();
-    dictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8));
-    dictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
+    dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+    dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
     dictionaryVector.getMutator().setValueCount(2);
 
     Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(2L, false, null));
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 81e58989fccc4..feae08ea223e1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -581,7 +581,7 @@ public void testWriteReadFixedSizeList() throws IOException {
         tuples.getMutator().setNotNull(i);
         floats.getMutator().set(i * 2, i + 0.1f);
         floats.getMutator().set(i * 2 + 1, i + 10.1f);
-        ints.getMutator().set(i, i);
+        ints.set(i, i);
       }
 
       parent.getMutator().setValueCount(10);

From 9ee838ac9538666ae6d7b0b1e680e80410503f7e Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 16 Oct 2017 00:39:30 -0700
Subject: [PATCH 1250/1644] ARROW-1474:[JAVA] ValueVector hierarchy
 (Implementation Phase 2)

Implementation of all scalar types and complex types with corresponding legacy
versions.

Closes #1203
---
 .../arrow/tools/ArrowFileTestFixtures.java    |   4 +-
 .../apache/arrow/tools/EchoServerTest.java    |  84 +-
 .../codegen/templates/ComplexReaders.java     |  24 +-
 .../codegen/templates/ComplexWriters.java     |  51 +-
 .../codegen/templates/LegacyUnionVector.java  | 356 ++++++++
 .../main/codegen/templates/MapWriters.java    |   4 +-
 .../templates/NullableValueVectors.java       | 787 ++----------------
 .../codegen/templates/UnionListWriter.java    |  11 +-
 .../main/codegen/templates/UnionVector.java   | 212 +++--
 .../main/codegen/templates/UnionWriter.java   |  26 +-
 .../vector/BaseNullableFixedWidthVector.java  | 315 +++++--
 .../BaseNullableVariableWidthVector.java      | 412 +++++++--
 .../apache/arrow/vector/BaseValueVector.java  |  33 +-
 .../apache/arrow/vector/BitVectorHelper.java  | 115 ++-
 .../arrow/vector/NullableBigIntVector.java    | 403 +++++++++
 .../arrow/vector/NullableBitVector.java       | 502 +++++++++++
 .../arrow/vector/NullableDateDayVector.java   | 404 +++++++++
 .../arrow/vector/NullableDateMilliVector.java | 408 +++++++++
 .../arrow/vector/NullableDecimalVector.java   | 454 ++++++++++
 .../arrow/vector/NullableFloat4Vector.java    | 404 +++++++++
 .../arrow/vector/NullableFloat8Vector.java    | 404 +++++++++
 .../arrow/vector/NullableIntVector.java       | 140 +++-
 .../vector/NullableIntervalDayVector.java     | 429 ++++++++++
 .../vector/NullableIntervalYearVector.java    | 387 +++++++++
 .../arrow/vector/NullableSmallIntVector.java  | 431 ++++++++++
 .../arrow/vector/NullableTimeMicroVector.java | 404 +++++++++
 .../arrow/vector/NullableTimeMilliVector.java | 405 +++++++++
 .../arrow/vector/NullableTimeNanoVector.java  | 402 +++++++++
 .../arrow/vector/NullableTimeSecVector.java   | 403 +++++++++
 .../NullableTimeStampMicroTZVector.java       | 219 +++++
 .../vector/NullableTimeStampMicroVector.java  | 220 +++++
 .../NullableTimeStampMilliTZVector.java       | 218 +++++
 .../vector/NullableTimeStampMilliVector.java  | 218 +++++
 .../vector/NullableTimeStampNanoTZVector.java | 219 +++++
 .../vector/NullableTimeStampNanoVector.java   | 219 +++++
 .../vector/NullableTimeStampSecTZVector.java  | 217 +++++
 .../vector/NullableTimeStampSecVector.java    | 219 +++++
 .../arrow/vector/NullableTimeStampVector.java | 253 ++++++
 .../arrow/vector/NullableTinyIntVector.java   | 432 ++++++++++
 .../arrow/vector/NullableUInt1Vector.java     | 318 +++++++
 .../arrow/vector/NullableUInt2Vector.java     | 319 +++++++
 .../arrow/vector/NullableUInt4Vector.java     | 291 +++++++
 .../arrow/vector/NullableUInt8Vector.java     | 291 +++++++
 .../arrow/vector/NullableVarBinaryVector.java | 514 ++++++++++++
 .../arrow/vector/NullableVarCharVector.java   | 100 ++-
 .../org/apache/arrow/vector/ValueVector.java  |   8 +
 .../apache/arrow/vector/VectorSchemaRoot.java |   2 +-
 .../apache/arrow/vector/VectorUnloader.java   |  12 +-
 .../org/apache/arrow/vector/ZeroVector.java   |   6 +
 .../vector/complex/AbstractMapVector.java     |   4 +-
 .../complex/BaseRepeatedValueVector.java      | 174 ++--
 .../vector/complex/FixedSizeListVector.java   | 238 ++++--
 .../LegacyBaseRepeatedValueVector.java        | 240 ++++++
 .../complex/LegacyFixedSizeListVector.java    | 298 +++++++
 .../vector/complex/LegacyListVector.java      | 288 +++++++
 .../arrow/vector/complex/LegacyMapVector.java | 229 +++++
 .../arrow/vector/complex/ListVector.java      | 424 ++++++----
 .../arrow/vector/complex/MapVector.java       |  96 +--
 .../vector/complex/NullableMapVector.java     | 299 +++++--
 .../complex/impl/NullableMapReaderImpl.java   |   2 +-
 .../vector/complex/impl/PromotableWriter.java |   2 +-
 .../complex/impl/SingleMapReaderImpl.java     |   2 +-
 .../impl/UnionFixedSizeListReader.java        |   4 +-
 .../vector/complex/impl/UnionListReader.java  |  12 +-
 .../vector/dictionary/DictionaryEncoder.java  |  28 +-
 .../apache/arrow/vector/file/ArrowWriter.java |   2 +-
 .../vector/file/json/JsonFileReader.java      | 341 ++++----
 .../vector/file/json/JsonFileWriter.java      | 198 +++--
 .../apache/arrow/vector/util/Validator.java   |  10 +-
 .../apache/arrow/vector/TestBitVector.java    |  35 +-
 .../arrow/vector/TestDecimalVector.java       |  12 +-
 .../arrow/vector/TestDictionaryVector.java    |   4 +-
 .../arrow/vector/TestFixedSizeListVector.java |  33 +-
 .../apache/arrow/vector/TestListVector.java   | 196 ++---
 .../apache/arrow/vector/TestUnionVector.java  | 269 +++---
 .../apache/arrow/vector/TestValueVector.java  | 222 +++--
 .../arrow/vector/TestVectorReAlloc.java       |   5 +-
 .../arrow/vector/TestVectorUnloadLoad.java    |   6 +-
 .../complex/impl/TestPromotableWriter.java    |  21 +-
 .../complex/writer/TestComplexWriter.java     |   4 +-
 .../arrow/vector/file/BaseFileTest.java       | 139 ++--
 .../arrow/vector/file/TestArrowFile.java      |  28 +-
 .../arrow/vector/file/TestArrowStream.java    |   6 +-
 .../vector/file/TestArrowStreamPipe.java      |  14 +-
 .../arrow/vector/file/json/TestJSONFile.java  |   2 +-
 85 files changed, 14169 insertions(+), 2427 deletions(-)
 create mode 100644 java/vector/src/main/codegen/templates/LegacyUnionVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java

diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
index 6d9a6c1323e76..c56a5a3303b75 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -78,8 +78,8 @@ static void validateOutput(File testOutFile, BufferAllocator allocator) throws E
   static void validateContent(int count, VectorSchemaRoot root) {
     Assert.assertEquals(count, root.getRowCount());
     for (int i = 0; i < count; i++) {
-      Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
-      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
+      Assert.assertEquals(i, root.getVector("int").getObject(i));
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i));
     }
   }
 
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index ecac6d6e53067..89714e4479180 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -104,10 +104,10 @@ private void testEchoServer(int serverPort,
       for (int i = 0; i < batches; i++) {
         vector.allocateNew(16);
         for (int j = 0; j < 8; j++) {
-          vector.getMutator().set(j, j + i);
-          vector.getMutator().set(j + 8, 0, (byte) (j + i));
+          vector.set(j, j + i);
+          vector.set(j + 8, 0, (byte) (j + i));
         }
-        vector.getMutator().setValueCount(16);
+        vector.setValueCount(16);
         root.setRowCount(16);
         writer.writeBatch();
       }
@@ -120,10 +120,10 @@ private void testEchoServer(int serverPort,
       for (int i = 0; i < batches; i++) {
         Assert.assertTrue(reader.loadNextBatch());
         assertEquals(16, reader.getVectorSchemaRoot().getRowCount());
-        assertEquals(16, readVector.getAccessor().getValueCount());
+        assertEquals(16, readVector.getValueCount());
         for (int j = 0; j < 8; j++) {
-          assertEquals(j + i, readVector.getAccessor().get(j));
-          assertTrue(readVector.getAccessor().isNull(j + 8));
+          assertEquals(j + i, readVector.get(j));
+          assertTrue(readVector.isNull(j + 8));
         }
       }
       Assert.assertFalse(reader.loadNextBatch());
@@ -169,20 +169,18 @@ public void testFlatDictionary() throws IOException {
                  FieldType.nullable(VARCHAR.getType()),
                  allocator)) {
       writeVector.allocateNewSafe();
-      NullableIntVector.Mutator mutator = writeVector.getMutator();
-      mutator.set(0, 0);
-      mutator.set(1, 1);
-      mutator.set(3, 2);
-      mutator.set(4, 1);
-      mutator.set(5, 2);
-      mutator.setValueCount(6);
+      writeVector.set(0, 0);
+      writeVector.set(1, 1);
+      writeVector.set(3, 2);
+      writeVector.set(4, 1);
+      writeVector.set(5, 2);
+      writeVector.setValueCount(6);
 
       writeDictionaryVector.allocateNewSafe();
-      NullableVarCharVector.Mutator dictionaryMutator = writeDictionaryVector.getMutator();
-      dictionaryMutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
-      dictionaryMutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
-      dictionaryMutator.set(2, "baz".getBytes(StandardCharsets.UTF_8));
-      dictionaryMutator.setValueCount(3);
+      writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+      writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+      writeDictionaryVector.set(2, "baz".getBytes(StandardCharsets.UTF_8));
+      writeDictionaryVector.setValueCount(3);
 
       List<Field> fields = ImmutableList.of(writeVector.getField());
       List<FieldVector> vectors = ImmutableList.of((FieldVector) writeVector);
@@ -210,23 +208,21 @@ public void testFlatDictionary() throws IOException {
         Assert.assertNotNull(readEncoding);
         Assert.assertEquals(1L, readEncoding.getId());
 
-        FieldVector.Accessor accessor = readVector.getAccessor();
-        Assert.assertEquals(6, accessor.getValueCount());
-        Assert.assertEquals(0, accessor.getObject(0));
-        Assert.assertEquals(1, accessor.getObject(1));
-        Assert.assertEquals(null, accessor.getObject(2));
-        Assert.assertEquals(2, accessor.getObject(3));
-        Assert.assertEquals(1, accessor.getObject(4));
-        Assert.assertEquals(2, accessor.getObject(5));
+        Assert.assertEquals(6, readVector.getValueCount());
+        Assert.assertEquals(0, readVector.getObject(0));
+        Assert.assertEquals(1, readVector.getObject(1));
+        Assert.assertEquals(null, readVector.getObject(2));
+        Assert.assertEquals(2, readVector.getObject(3));
+        Assert.assertEquals(1, readVector.getObject(4));
+        Assert.assertEquals(2, readVector.getObject(5));
 
         Dictionary dictionary = reader.lookup(1L);
         Assert.assertNotNull(dictionary);
-        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary
-            .getVector()).getAccessor();
-        Assert.assertEquals(3, dictionaryAccessor.getValueCount());
-        Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
-        Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
-        Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2));
+        NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector());
+        Assert.assertEquals(3, dictionaryVector.getValueCount());
+        Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
+        Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
+        Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2));
       }
     }
   }
@@ -243,9 +239,9 @@ public void testNestedDictionary() throws IOException {
       // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
 
       writeDictionaryVector.allocateNew();
-      writeDictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8));
-      writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
-      writeDictionaryVector.getMutator().setValueCount(2);
+      writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
+      writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
+      writeDictionaryVector.setValueCount(2);
 
       writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
       writeVector.allocateNew();
@@ -297,19 +293,17 @@ public void testNestedDictionary() throws IOException {
         Assert.assertEquals(2L, encoding.getId());
         Assert.assertEquals(new Int(32, true), encoding.getIndexType());
 
-        ListVector.Accessor accessor = readVector.getAccessor();
-        Assert.assertEquals(3, accessor.getValueCount());
-        Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
-        Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
-        Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
+        Assert.assertEquals(3, readVector.getValueCount());
+        Assert.assertEquals(Arrays.asList(0, 1), readVector.getObject(0));
+        Assert.assertEquals(Arrays.asList(0), readVector.getObject(1));
+        Assert.assertEquals(Arrays.asList(1), readVector.getObject(2));
 
         Dictionary readDictionary = reader.lookup(2L);
         Assert.assertNotNull(readDictionary);
-        NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector)
-            readDictionary.getVector()).getAccessor();
-        Assert.assertEquals(2, dictionaryAccessor.getValueCount());
-        Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
-        Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+        NullableVarCharVector dictionaryVector = ((NullableVarCharVector) readDictionary.getVector());
+        Assert.assertEquals(2, dictionaryVector.getValueCount());
+        Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
+        Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
       }
     }
   }
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 791064905156d..60347a2fcb8e2 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -70,11 +70,7 @@ public Field getField(){
   
   public boolean isSet(){
     <#if nullMode == "Nullable">
-      <#if minor.class != "Int" && minor.class != "VarChar">
-        return !vector.getAccessor().isNull(idx());
-      <#else>
         return !vector.isNull(idx());
-      </#if>
     <#else>
     return true;
     </#if>
@@ -97,19 +93,11 @@ public void read(${minor.class?cap_first}Holder h){
   </#if>
 
   public void read(Nullable${minor.class?cap_first}Holder h){
-    <#if minor.class != "Int" && minor.class != "VarChar">
-      vector.getAccessor().get(idx(), h);
-    <#else>
-      vector.get(idx(), h);
-    </#if>
+    vector.get(idx(), h);
   }
   
   public ${friendlyType} read${safeType}(){
-    <#if minor.class == "Int" || minor.class == "VarChar">
-      return vector.getObject(idx());
-    <#else>
-      return vector.getAccessor().getObject(idx());
-    </#if>
+    return vector.getObject(idx());
   }
 
   <#if minor.class == "TimeStampSec" ||
@@ -118,7 +106,7 @@ public void read(Nullable${minor.class?cap_first}Holder h){
        minor.class == "TimeStampNano">
   @Override
   public ${minor.boxedType} read${minor.boxedType}(){
-    return vector.getAccessor().get(idx());
+    return vector.get(idx());
   }
   </#if>
   
@@ -127,11 +115,7 @@ public void copyValue(FieldWriter w){
   }
   
   public Object readObject(){
-    <#if minor.class == "Int" || minor.class == "VarChar">
-      return (Object)vector.getObject(idx());
-    <#else>
-      return vector.getAccessor().getObject(idx());
-    </#if>
+    return (Object)vector.getObject(idx());
   }
 }
 </#if>
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 77f6594a07804..406bbb39c7f4a 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -39,16 +39,9 @@
 @SuppressWarnings("unused")
 public class ${eName}WriterImpl extends AbstractFieldWriter {
 
-  <#if minor.class != "Int" && minor.class != "VarChar">
-    private final Nullable${name}Vector.Mutator mutator;
-  </#if>
-
   final Nullable${name}Vector vector;
 
   public ${eName}WriterImpl(Nullable${name}Vector vector) {
-    <#if minor.class != "Int" && minor.class != "VarChar">
-      this.mutator = vector.getMutator();
-    </#if>
     this.vector = vector;
   }
 
@@ -108,51 +101,31 @@ public void setPosition(int idx) {
   <#else>
 
   public void write(${minor.class}Holder h) {
-    <#if minor.class != "Int" && minor.class != "VarChar">
-      mutator.setSafe(idx(), h);
-      vector.getMutator().setValueCount(idx()+1);
-    <#else>
-        vector.setSafe(idx(), h);
-        vector.setValueCount(idx()+1);
-    </#if>
+    vector.setSafe(idx(), h);
+    vector.setValueCount(idx()+1);
   }
 
   public void write(Nullable${minor.class}Holder h) {
-    <#if minor.class != "Int" && minor.class != "VarChar">
-      mutator.setSafe(idx(), h);
-      vector.getMutator().setValueCount(idx()+1);
-    <#else>
-      vector.setSafe(idx(), h);
-      vector.setValueCount(idx()+1);
-    </#if>
+    vector.setSafe(idx(), h);
+    vector.setValueCount(idx()+1);
   }
 
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-    <#if minor.class != "Int" && minor.class != "VarChar">
-      mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
-      vector.getMutator().setValueCount(idx()+1);
-    <#else>
-      vector.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
-      vector.setValueCount(idx()+1);
-    </#if>
+    vector.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
+    vector.setValueCount(idx()+1);
   }
-  <#if minor.class == "Decimal">
 
+  <#if minor.class == "Decimal">
   public void write${minor.class}(${friendlyType} value) {
-    mutator.setSafe(idx(), value);
-    vector.getMutator().setValueCount(idx()+1);
+    vector.setSafe(idx(), value);
+    vector.setValueCount(idx()+1);
   }
   </#if>
-  <#if mode == "Nullable">
 
+  <#if mode == "Nullable">
   public void writeNull() {
-    <#if minor.class != "Int" && minor.class != "VarChar">
-        mutator.setNull(idx());
-        vector.getMutator().setValueCount(idx()+1);
-    <#else>
-        vector.setNull(idx());
-        vector.setValueCount(idx()+1);
-    </#if>
+    vector.setNull(idx());
+    vector.setValueCount(idx()+1);
   }
   </#if>
   </#if>
diff --git a/java/vector/src/main/codegen/templates/LegacyUnionVector.java b/java/vector/src/main/codegen/templates/LegacyUnionVector.java
new file mode 100644
index 0000000000000..b9cb6dfdc2482
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/LegacyUnionVector.java
@@ -0,0 +1,356 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/LegacyUnionVector.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+        package org.apache.arrow.vector.complex;
+
+<#include "/@includes/vv_imports.ftl" />
+        import com.google.common.collect.ImmutableList;
+        import java.util.ArrayList;
+        import java.util.Collections;
+        import java.util.Iterator;
+        import org.apache.arrow.vector.BaseDataValueVector;
+        import org.apache.arrow.vector.complex.impl.ComplexCopier;
+        import org.apache.arrow.vector.util.CallBack;
+        import org.apache.arrow.vector.schema.ArrowFieldNode;
+
+        import static org.apache.arrow.vector.types.UnionMode.Sparse;
+
+
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+
+
+/**
+ * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each
+ * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods,
+ * as well as the addOrGet method.
+ *
+ * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
+ * each time the vector is accessed.
+ * Source code generated using FreeMarker template ${.template_name}
+ */
+public class LegacyUnionVector implements FieldVector {
+
+   private Accessor accessor = new Accessor();
+   private Mutator mutator = new Mutator();
+   private final UnionVector unionVector;
+
+   public LegacyUnionVector(String name, BufferAllocator allocator, CallBack callBack) {
+     unionVector = new UnionVector(name, allocator, callBack);
+   }
+
+   public BufferAllocator getAllocator() {
+      return unionVector.getAllocator();
+   }
+
+   @Override
+   public MinorType getMinorType() {
+      return MinorType.UNION;
+   }
+
+   @Override
+   public void initializeChildrenFromFields(List<Field> children) {
+      unionVector.initializeChildrenFromFields(children);
+   }
+
+   @Override
+   public List<FieldVector> getChildrenFromFields() {
+      return unionVector.getChildrenFromFields();
+   }
+
+   @Override
+   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+      unionVector.loadFieldBuffers(fieldNode, ownBuffers);
+   }
+
+   @Override
+   public List<ArrowBuf> getFieldBuffers() {
+      return unionVector.getFieldBuffers();
+   }
+
+   @Override
+   public List<BufferBacked> getFieldInnerVectors() {
+      return unionVector.getFieldInnerVectors();
+   }
+
+   @Override
+   public long getValidityBufferAddress() {
+      return unionVector.getValidityBufferAddress();
+   }
+
+   @Override
+   public long getDataBufferAddress() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public long getOffsetBufferAddress() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public ArrowBuf getValidityBuffer() {
+      return unionVector.getValidityBuffer();
+   }
+
+   @Override
+   public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); }
+
+   @Override
+   public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); }
+
+   public NullableMapVector getMap() {
+      return unionVector.getMap();
+   }
+  <#list vv.types as type>
+    <#list type.minor as minor>
+      <#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#assign lowerCaseName = name?lower_case/>
+      <#if !minor.typeParams?? >
+
+   private Nullable${name}Vector ${uncappedName}Vector;
+
+   public Nullable${name}Vector get${name}Vector() {
+      return unionVector.get${name}Vector();
+   }
+      </#if>
+    </#list>
+  </#list>
+
+   public ListVector getList() {
+      return unionVector.getList();
+   }
+
+   public int getTypeValue(int index) {
+      return unionVector.getTypeValue(index);
+   }
+
+   @Override
+   public void allocateNew() throws OutOfMemoryException {
+     unionVector.allocateNew();
+   }
+
+   @Override
+   public boolean allocateNewSafe() {
+     return unionVector.allocateNewSafe();
+   }
+
+   @Override
+   public void reAlloc() {
+      unionVector.reAlloc();
+   }
+
+   @Override
+   public void setInitialCapacity(int numRecords) {
+   }
+
+   @Override
+   public int getValueCapacity() {
+      return unionVector.getValueCapacity();
+   }
+
+   @Override
+   public void close() {
+     unionVector.close();
+   }
+
+   @Override
+   public void clear() {
+      unionVector.clear();
+   }
+
+   @Override
+   public Field getField() {
+      return unionVector.getField();
+   }
+
+   @Override
+   public TransferPair getTransferPair(BufferAllocator allocator) {
+      return unionVector.getTransferPair(allocator);
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      return unionVector.getTransferPair(ref, allocator);
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+      return unionVector.getTransferPair(ref, allocator, callBack);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector target) {
+      return unionVector.makeTransferPair(((LegacyUnionVector)target).unionVector);
+   }
+
+   public void copyFrom(int inIndex, int outIndex, UnionVector from) {
+      unionVector.copyFrom(inIndex, outIndex, from);
+   }
+
+   public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
+      unionVector.copyFromSafe(inIndex, outIndex, from);
+   }
+
+   public FieldVector addVector(FieldVector v) {
+     return unionVector.addVector(v);
+   }
+
+   @Override
+   public Accessor getAccessor() {
+      return accessor;
+   }
+
+   @Override
+   public Mutator getMutator() {
+      return mutator;
+   }
+
+   @Override
+   public FieldReader getReader() {
+      return unionVector.getReader();
+   }
+
+   public FieldWriter getWriter() {
+      return unionVector.getWriter();
+   }
+
+   @Override
+   public int getBufferSize() {
+      return unionVector.getBufferSize();
+   }
+
+   @Override
+   public int getBufferSizeFor(final int valueCount) {
+     return unionVector.getBufferSizeFor(valueCount);
+   }
+
+   @Override
+   public ArrowBuf[] getBuffers(boolean clear) {
+     return unionVector.getBuffers(clear);
+   }
+
+   @Override
+   public Iterator<ValueVector> iterator() {
+      return unionVector.iterator();
+   }
+
+   public class Accessor extends BaseValueVector.BaseAccessor {
+
+      @Override
+      public Object getObject(int index) {
+        return unionVector.getObject(index);
+      }
+
+      public byte[] get(int index) {
+         return unionVector.get(index);
+      }
+
+      public void get(int index, ComplexHolder holder) {
+      }
+
+      public void get(int index, UnionHolder holder) {
+         unionVector.get(index, holder);
+      }
+
+      public int getNullCount() {
+         return unionVector.getNullCount();
+      }
+
+      @Override
+      public int getValueCount() {
+         return unionVector.getValueCount();
+      }
+
+      @Override
+      public boolean isNull(int index) {
+         return unionVector.isNull(index);
+      }
+
+      public int isSet(int index) {
+         return unionVector.isSet(index);
+      }
+   }
+
+   public class Mutator extends BaseValueVector.BaseMutator {
+
+      UnionWriter writer;
+
+      @Override
+      public void setValueCount(int valueCount) {
+         unionVector.setValueCount(valueCount);
+      }
+
+      public void setSafe(int index, UnionHolder holder) {
+        unionVector.setSafe(index, holder);
+      }
+    <#list vv.types as type>
+      <#list type.minor as minor>
+        <#assign name = minor.class?cap_first />
+        <#assign fields = minor.fields!type.fields />
+        <#assign uncappedName = name?uncap_first/>
+        <#if !minor.typeParams?? >
+      public void setSafe(int index, Nullable${name}Holder holder) {
+         unionVector.setSafe(index, holder);
+      }
+
+        </#if>
+      </#list>
+    </#list>
+
+      public void setType(int index, MinorType type) {
+         unionVector.setType(index, type);
+      }
+
+      @Override
+      public void reset() { }
+
+      @Override
+      public void generateTestData(int values) { }
+   }
+
+   @Override
+   @Deprecated
+   public int getValueCount() { return getAccessor().getValueCount(); }
+
+   @Override
+   @Deprecated
+   public void setValueCount(int valueCount) { getMutator().setValueCount(valueCount);}
+
+   @Override
+   @Deprecated
+   public Object getObject(int index) { return getAccessor().getObject(index); }
+
+   @Override
+   @Deprecated
+   public int getNullCount() { return getAccessor().getNullCount(); }
+
+   @Override
+   @Deprecated
+   public boolean isNull(int index) { return getAccessor().isNull(index); }
+}
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index b89f91457e8b2..6afd508e68837 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -184,7 +184,7 @@ public ListWriter list(String name) {
   }
 
   public void setValueCount(int count) {
-    container.getMutator().setValueCount(count);
+    container.setValueCount(count);
   }
 
   @Override
@@ -199,7 +199,7 @@ public void setPosition(int index) {
   public void start() {
     <#if mode == "Single">
     <#else>
-    container.getMutator().setIndexDefined(idx());
+    container.setIndexDefined(idx());
     </#if>
   }
 
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 5d1f5a33b90f5..8e5b10b649b13 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -19,14 +19,8 @@
 <#list vv.types as type>
 <#list type.minor as minor>
 
-<#if minor.class == "Int" || minor.class == "VarChar">
 <#assign className = "LegacyNullable${minor.class}Vector" />
 <#assign valuesName = "Nullable${minor.class}Vector" />
-<#else>
-<#assign className = "Nullable${minor.class}Vector" />
-<#assign valuesName = "${minor.class}Vector" />
-</#if>
-
 <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
 
 <@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" />
@@ -50,35 +44,21 @@
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
 @SuppressWarnings("unused")
-<#if minor.class == "Int" || minor.class == "VarChar">
 @Deprecated
-</#if>
 public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, FieldVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
 protected final static byte[] emptyByteArray = new byte[]{};
 
-  <#if minor.class != "Int" && minor.class != "VarChar">
-  private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this);
-  </#if>
-
   private final String bitsField = "$bits$";
   private final String valuesField = "$values$";
 
-  <#if minor.class != "Int" && minor.class != "VarChar">
-  private final Field field;
-  </#if>
-
   final BitVector bits = new BitVector(bitsField, allocator);
   final ${valuesName} values;
 
   private final Mutator mutator;
   private final Accessor accessor;
 
-  <#if minor.class != "Int" && minor.class != "VarChar">
-  private final List<BufferBacked> innerVectors;
-  </#if>
-
   <#if minor.typeParams??>
     <#assign typeParams = minor.typeParams?reverse>
     <#list typeParams as typeParam>
@@ -122,33 +102,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
     </#if>
     this.mutator = new Mutator();
     this.accessor = new Accessor();
-    <#if minor.class != "Int" && minor.class != "VarChar">
-    this.field = new Field(name, fieldType, null);
-    innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(
-        bits,
-        <#if type.major = "VarLen">
-        values.offsetVector,
-        </#if>
-        values
-    ));
-    </#if>
   }
 
-  <#if minor.class != "Int" && minor.class != "VarChar">
-  /* not needed for new vectors */
-  public BitVector getValidityVector() {
-    return bits;
-  }
-  </#if>
-
   @Override
   public List<BufferBacked> getFieldInnerVectors() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getFieldInnerVectors();
-    <#else>
-        return innerVectors;
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getFieldInnerVectors();
   }
 
   @Override
@@ -165,123 +124,61 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        values.loadFieldBuffers(fieldNode, ownBuffers);
-    <#else>
-    <#if type.major = "VarLen">
-    // variable width values: truncate offset vector buffer to size (#1)
-    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1,
-        values.offsetVector.getBufferSizeFor(
-        fieldNode.getLength() == 0? 0 : fieldNode.getLength() + 1));
-    mutator.lastSet = fieldNode.getLength() - 1;
-    <#else>
-    // fixed width values truncate value vector to size (#1)
-    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.getBufferSizeFor(fieldNode.getLength()));
-    </#if>
-    org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
-    bits.valueCount = fieldNode.getLength();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.loadFieldBuffers(fieldNode, ownBuffers);
   }
 
   public List<ArrowBuf> getFieldBuffers() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getFieldBuffers();
-    <#else>
-        return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors());
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getFieldBuffers();
   }
 
   @Override
   public Field getField() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getField();
-    <#else>
-      return field;
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getField();
   }
 
   @Override
   public MinorType getMinorType() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getMinorType();
-    <#else>
-        return MinorType.${minor.class?upper_case};
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getMinorType();
   }
 
   @Override
   public FieldReader getReader(){
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getReader();
-    <#else>
-        return reader;
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getReader();
   }
 
   @Override
   public int getValueCapacity(){
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getValueCapacity();
-    <#else>
-        return Math.min(bits.getValueCapacity(), values.getValueCapacity());
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getValueCapacity();
   }
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getBuffers(clear);
-    <#else>
-    final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), ArrowBuf.class);
-    if (clear) {
-      for (final ArrowBuf buffer:buffers) {
-        buffer.retain(1);
-      }
-      clear();
-    }
-    return buffers;
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getBuffers(clear);
   }
 
   @Override
   public void close() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        values.close();
-    <#else>
-        bits.close();
-        values.close();
-        super.close();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.close();
   }
 
   @Override
   public void clear() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        values.clear();
-    <#else>
-        bits.clear();
-        values.clear();
-        super.clear();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.clear();
   }
 
   @Override
   public int getBufferSize(){
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getBufferSize();
-    <#else>
-        return values.getBufferSize() + bits.getBufferSize();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getBufferSize();
   }
 
   @Override
@@ -289,14 +186,8 @@ public int getBufferSizeFor(final int valueCount) {
     if (valueCount == 0) {
       return 0;
     }
-
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getBufferSizeFor(valueCount);
-    <#else>
-        return values.getBufferSizeFor(valueCount)
-          + bits.getBufferSizeFor(valueCount);
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getBufferSizeFor(valueCount);
   }
 
   public ArrowBuf getBuffer() {
@@ -309,93 +200,38 @@ public ArrowBuf getBuffer() {
 
   @Override
   public void setInitialCapacity(int numRecords) {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        values.setInitialCapacity(numRecords);
-    <#else>
-        bits.setInitialCapacity(numRecords);
-        values.setInitialCapacity(numRecords);
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.setInitialCapacity(numRecords);
   }
 
   @Override
   public void allocateNew() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        values.allocateNew();
-    <#else>
-    if(!allocateNewSafe()){
-      throw new OutOfMemoryException("Failure while allocating buffer.");
-    }
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.allocateNew();
   }
 
   @Override
   public boolean allocateNewSafe() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.allocateNewSafe();
-    <#else>
-    /* Boolean to keep track if all the memory allocations were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      success = values.allocateNewSafe() && bits.allocateNewSafe();
-    } finally {
-      if (!success) {
-        clear();
-      }
-    }
-    bits.zeroVector();
-    mutator.reset();
-    accessor.reset();
-    return success;
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.allocateNewSafe();
   }
 
   @Override
   public void reAlloc() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        values.reAlloc();
-    <#else>
-        bits.reAlloc();
-        values.reAlloc();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.reAlloc();
   }
 
   public void reset() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        values.reset();
-    <#else>
-    bits.zeroVector();
-    mutator.reset();
-    accessor.reset();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.reset();
   }
 
   <#if type.major == "VarLen">
   @Override
   public void allocateNew(int totalBytes, int valueCount) {
-    <#if minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        values.allocateNew(totalBytes, valueCount);
-    <#else>
-    try {
-      values.allocateNew(totalBytes, valueCount);
-      bits.allocateNew(valueCount);
-    } catch(RuntimeException e) {
-      clear();
-      throw e;
-    }
-    bits.zeroVector();
-    mutator.reset();
-    accessor.reset();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.allocateNew(totalBytes, valueCount);
   }
 
   @Override
@@ -411,21 +247,8 @@ public int getCurrentSizeInBytes(){
   <#else>
   @Override
   public void allocateNew(int valueCount) {
-    <#if minor.class == "Int">
-        /* DELEGATE TO NEW VECTOR */
-        values.allocateNew(valueCount);
-    <#else>
-    try {
-      values.allocateNew(valueCount);
-      bits.allocateNew(valueCount);
-    } catch(OutOfMemoryException e) {
-      clear();
-      throw e;
-    }
-    bits.zeroVector();
-    mutator.reset();
-    accessor.reset();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.allocateNew(valueCount);
   }
 
   /**
@@ -433,13 +256,8 @@ public void allocateNew(int valueCount) {
    */
   @Override
   public void zeroVector() {
-    <#if minor.class == "Int">
-        /* DELEGATE TO NEW VECTOR */
-        values.zeroVector();
-    <#else>
-        bits.zeroVector();
-        values.zeroVector();
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    values.zeroVector();
   }
   </#if>
 
@@ -447,53 +265,36 @@ public void zeroVector() {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getTransferPair(ref, allocator, callBack);
-    <#else>
-        return getTransferPair(ref, allocator);
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getTransferPair(ref, allocator, callBack);
   }
 
 
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator){
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getTransferPair(allocator);
-    <#else>
-        return new TransferImpl(name, allocator);
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getTransferPair(allocator);
   }
 
 
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getTransferPair(ref, allocator);
-    <#else>
-        return new TransferImpl(ref, allocator);
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getTransferPair(ref, allocator);
   }
 
 
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.makeTransferPair(to);
-    <#else>
-        return new TransferImpl((${className}) to);
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.makeTransferPair(to);
   }
 
 
 
-  <#if minor.class == "Int" || minor.class == "VarChar">
   public void transferTo(${valuesName} target) {
     /* DELEGATE TO NEW VECTOR */
     <#if type.major == "VarLen">
@@ -512,61 +313,6 @@ public void splitAndTransferTo(int startIndex, int length, ${valuesName} target)
     </#if>
   }
 
-  <#else>
-  public void transferTo(${className} target){
-    bits.transferTo(target.bits);
-    values.transferTo(target.values);
-    <#if type.major == "VarLen">
-    target.mutator.lastSet = mutator.lastSet;
-    </#if>
-    clear();
-  }
-
-  public void splitAndTransferTo(int startIndex, int length, ${className} target) {
-    bits.splitAndTransferTo(startIndex, length, target.bits);
-    values.splitAndTransferTo(startIndex, length, target.values);
-    <#if type.major == "VarLen">
-    target.mutator.lastSet = length - 1;
-    </#if>
-  }
-  </#if>
-
-
-
-  <#if minor.class != "Int" && minor.class != "VarChar">
-  private class TransferImpl implements TransferPair {
-    ${className} to;
-
-    public TransferImpl(String ref, BufferAllocator allocator){
-      to = new ${className}(ref, field.getFieldType(), allocator);
-    }
-
-    public TransferImpl(${className} to){
-      this.to = to;
-    }
-
-    @Override
-    public ${className} getTo(){
-      return to;
-    }
-
-    @Override
-    public void transfer(){
-      transferTo(to);
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      splitAndTransferTo(startIndex, length, to);
-    }
-
-    @Override
-    public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, ${className}.this);
-    }
-  }
-  </#if>
-
 
 
   @Override
@@ -580,8 +326,6 @@ public Mutator getMutator(){
   }
 
 
-
-  <#if minor.class == "Int" || minor.class == "VarChar">
   public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) {
     /* DELEGATE TO NEW VECTOR */
     values.copyFrom(fromIndex, thisIndex, from);
@@ -591,107 +335,43 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) {
     /* DELEGATE TO NEW VECTOR */
     values.copyFromSafe(fromIndex, thisIndex, from);
   }
-  <#else>
-  public void copyFrom(int fromIndex, int thisIndex, ${className} from) {
-    final Accessor fromAccessor = from.getAccessor();
-    if (!fromAccessor.isNull(fromIndex)) {
-      mutator.set(thisIndex, fromAccessor.get(fromIndex));
-    }
-    <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
-  }
-
-  public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from){
-    <#if type.major == "VarLen">
-    mutator.fillEmpties(thisIndex);
-    </#if>
-    values.copyFromSafe(fromIndex, thisIndex, from);
-    bits.getMutator().setSafeToOne(thisIndex);
-    <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
-  }
-
-  public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
-    <#if type.major == "VarLen">
-    mutator.fillEmpties(thisIndex);
-    </#if>
-    bits.copyFromSafe(fromIndex, thisIndex, from.bits);
-    values.copyFromSafe(fromIndex, thisIndex, from.values);
-    <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
-  }
-  </#if>
 
   @Override
   public long getValidityBufferAddress() {
-    /* address of the databuffer associated with the bitVector */
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getValidityBufferAddress();
-    <#else>
-        return (bits.getDataBuffer().memoryAddress());
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getValidityBufferAddress();
   }
 
   @Override
   public long getDataBufferAddress() {
-    /* address of the dataBuffer associated with the valueVector */
-    <#if minor.class == "Int" || minor.class == "VarChar">
-          /* DELEGATE TO NEW VECTOR */
-          return values.getDataBufferAddress();
-    <#else>
-          return (bits.getDataBuffer().memoryAddress());
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getDataBufferAddress();
   }
 
   @Override
   public long getOffsetBufferAddress() {
-    /* address of the dataBuffer associated with the offsetVector
-     * this operation is not supported for fixed-width vector types.
-     */
-    <#if minor.class == "Int" || minor.class == "VarChar">
-          /* DELEGATE TO NEW VECTOR */
-          return values.getOffsetBufferAddress();
-    <#else>
-        <#if type.major != "VarLen">
-          throw new UnsupportedOperationException();
-        <#else>
-          return (values.getOffsetAddr());
-        </#if>
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getOffsetBufferAddress();
   }
 
   @Override
   public ArrowBuf getValidityBuffer() {
-    <#if minor.class == "Int" || minor.class == "VarChar">
-          /* DELEGATE TO NEW VECTOR */
-          return values.getValidityBuffer();
-    <#else>
-          return (bits.getDataBuffer());
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getValidityBuffer();
   }
 
   @Override
   public ArrowBuf getDataBuffer() {
-    /* dataBuffer associated with the valueVector */
     return (values.getDataBuffer());
   }
 
   @Override
   public ArrowBuf getOffsetBuffer() {
-    /* dataBuffer associated with the offsetVector of the valueVector */
-    <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getOffsetBuffer();
-    <#else>
-        <#if type.major != "VarLen">
-          throw new UnsupportedOperationException();
-        <#else>
-          return (values.getOffsetBuffer());
-        </#if>
-    </#if>
+    /* DELEGATE TO NEW VECTOR */
+    return values.getOffsetBuffer();
   }
 
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
-    final BitVector.Accessor bAccessor = bits.getAccessor();
-    final ${valuesName}.Accessor vAccessor = values.getAccessor();
 
     /**
      * Get the element at the specified position.
@@ -699,119 +379,67 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m
      * @param  index   position of the value
      * @return value of the element, if not null
      */
-    <#if minor.class == "Int" || minor.class == "VarChar">
-      public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
-        /* DELEGATE TO NEW VECTOR */
-        return values.get(index);
-      }
-    <#else>
-
-      public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
-        if (isNull(index)) {
-          throw new IllegalStateException("Can't get a null value");
-        }
-        return vAccessor.get(index);
-      }
-    </#if>
+    public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+      /* DELEGATE TO NEW VECTOR */
+      return values.get(index);
+    }
 
     @Override
     public boolean isNull(int index) {
-      <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.isNull(index);
-      <#else>
-        return isSet(index) == 0;
-      </#if>
+      /* DELEGATE TO NEW VECTOR */
+      return values.isNull(index);
     }
 
     public int isSet(int index){
-      <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.isSet(index);
-      <#else>
-        return bAccessor.get(index);
-      </#if>
+      /* DELEGATE TO NEW VECTOR */
+      return values.isSet(index);
     }
 
     <#if type.major == "VarLen">
     public long getStartEnd(int index){
-      <#if minor.class == "VarChar">
         /* DELEGATE TO NEW VECTOR */
         return values.getStartEnd(index);
-      <#else>
-        return vAccessor.getStartEnd(index);
-      </#if>
     }
 
     @Override
     public int getValueLength(int index) {
-      <#if minor.class == "VarChar">
         /* DELEGATE TO NEW VECTOR */
         return values.getValueLength(index);
-      <#else>
-        return values.getAccessor().getValueLength(index);
-      </#if>
     }
     </#if>
 
-    <#if minor.class == "Int" || minor.class == "VarChar">
     public void get(int index, Nullable${minor.class}Holder holder){
         /* DELEGATE TO NEW VECTOR */
         values.get(index, holder);
     }
-    <#else>
-    public void get(int index, Nullable${minor.class}Holder holder){
-      vAccessor.get(index, holder);
-      holder.isSet = bAccessor.get(index);
-    }
-    </#if>
 
-    <#if minor.class == "Int" || minor.class == "VarChar">
     @Override
     public ${friendlyType} getObject(int index) {
       /* DELEGATE TO NEW VECTOR */
       return values.getObject(index);
     }
-    <#else>
-    @Override
-    public ${friendlyType} getObject(int index) {
-      if (isNull(index)) {
-          return null;
-      }else{
-        return vAccessor.getObject(index);
-      }
-    }
-    </#if>
 
     <#if minor.class == "IntervalYear" || minor.class == "IntervalDay">
     public StringBuilder getAsStringBuilder(int index) {
-      if (isNull(index)) {
-          return null;
-      }else{
-        return vAccessor.getAsStringBuilder(index);
-      }
+       /* DELEGATE TO NEW VECTOR */
+       return values.getAsStringBuilder(index);
     }
     </#if>
 
     @Override
     public int getValueCount(){
-      <#if minor.class == "Int" || minor.class == "VarChar">
-        /* DELEGATE TO NEW VECTOR */
-        return values.getValueCount();
-      <#else>
-        return bits.getAccessor().getValueCount();
-      </#if>
+      /* DELEGATE TO NEW VECTOR */
+      return values.getValueCount();
     }
 
-    public void reset(){}
+    public void reset() { }
   }
 
   public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> {
     private int setCount;
     <#if type.major = "VarLen"> private int lastSet = -1;</#if>
 
-    private Mutator(){
-    }
+    private Mutator() { }
 
     public ${valuesName} getVectorWithValues() {
       return values;
@@ -819,12 +447,9 @@ private Mutator(){
 
 
     @Override
-    public void setIndexDefined(int index){
-      <#if minor.class == "Int" || minor.class == "VarChar">
+    public void setIndexDefined(int index) {
+      /* DELEGATE TO NEW VECTOR */
       values.setIndexDefined(index);
-      <#else>
-      bits.getMutator().setToOne(index);
-      </#if>
     }
 
 
@@ -835,32 +460,14 @@ public void setIndexDefined(int index){
      * @param index   position of the bit to set
      * @param value   array of bytes (or int if smaller than 4 bytes) to write
      */
-
-    <#if minor.class == "Int" || minor.class == "VarChar">
     public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
        /* DELEGATE TO NEW VECTOR */
        values.set(index, value);
     }
-    <#else>
-    public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
-      setCount++;
-      final ${valuesName}.Mutator valuesMutator = values.getMutator();
-      final BitVector.Mutator bitsMutator = bits.getMutator();
-      <#if type.major == "VarLen">
-      for (int i = lastSet + 1; i < index; i++) {
-        valuesMutator.set(i, emptyByteArray);
-      }
-      </#if>
-      bitsMutator.setToOne(index);
-      valuesMutator.set(index, value);
-      <#if type.major == "VarLen">lastSet = index;</#if>
-    }
-    </#if>
 
 
 
     <#if type.major == "VarLen">
-    <#if minor.class == "VarChar">
     public void fillEmpties(int index) {
       /* DELEGATE TO NEW VECTOR */
       values.fillEmpties(index);
@@ -871,203 +478,55 @@ public void setValueLengthSafe(int index, int length) {
       /* DELEGATE TO NEW VECTOR */
       values.setValueLengthSafe(index, length);
     }
-
-    <#else>
-    public void fillEmpties(int index){
-      final ${valuesName}.Mutator valuesMutator = values.getMutator();
-      for (int i = lastSet + 1; i < index; i++) {
-        valuesMutator.setSafe(i, emptyByteArray);
-      }
-      while(index > bits.getValueCapacity()) {
-        bits.reAlloc();
-      }
-      lastSet = index - 1;
-    }
-
-    @Override
-    public void setValueLengthSafe(int index, int length) {
-      values.getMutator().setValueLengthSafe(index, length);
-      lastSet = index;
-    }
-    </#if>
     </#if>
 
 
-
-    <#if minor.class == "Int" || minor.class == "VarChar">
     public void setSafe(int index, byte[] value, int start, int length) {
        /* DELEGATE TO NEW VECTOR */
       values.setSafe(index, value, start, length);
     }
-    <#else>
-    public void setSafe(int index, byte[] value, int start, int length) {
-      <#if type.major != "VarLen">
-      throw new UnsupportedOperationException();
-      <#else>
-      fillEmpties(index);
-
-      bits.getMutator().setSafeToOne(index);
-      values.getMutator().setSafe(index, value, start, length);
-      setCount++;
-      <#if type.major == "VarLen">lastSet = index;</#if>
-      </#if>
-    }
-    </#if>
 
 
-
-    <#if minor.class == "VarChar">
     public void setSafe(int index, ByteBuffer value, int start, int length) {
        /* DELEGATE TO NEW VECTOR */
        values.setSafe(index, value, start, length);
     }
-    <#else>
-    public void setSafe(int index, ByteBuffer value, int start, int length) {
-      <#if type.major != "VarLen">
-      throw new UnsupportedOperationException();
-      <#else>
-      fillEmpties(index);
-
-      bits.getMutator().setSafeToOne(index);
-      values.getMutator().setSafe(index, value, start, length);
-      setCount++;
-      <#if type.major == "VarLen">lastSet = index;</#if>
-      </#if>
-    }
-    </#if>
-
 
 
-    <#if minor.class == "Int" || minor.class == "VarChar">
     public void setNull(int index) {
        /* DELEGATE TO NEW VECTOR */
        values.setNull(index);
     }
-    <#else>
-    public void setNull(int index){
-      bits.getMutator().setSafe(index, 0);
-    }
-    </#if>
-
-
-
-    <#if minor.class != "Int" && minor.class != "VarChar">
-    /* these methods are probably not needed */
-    public void setSkipNull(int index, ${minor.class}Holder holder){
-      values.getMutator().set(index, holder);
-    }
-
-    public void setSkipNull(int index, Nullable${minor.class}Holder holder){
-      values.getMutator().set(index, holder);
-    }
-    </#if>
 
 
-
-    <#if minor.class == "Int" || minor.class == "VarChar">
     public void set(int index, Nullable${minor.class}Holder holder) {
       /* DELEGATE TO NEW VECTOR */
       values.set(index, holder);
     }
-    <#else>
-    public void set(int index, Nullable${minor.class}Holder holder) {
-      final ${valuesName}.Mutator valuesMutator = values.getMutator();
-      <#if type.major == "VarLen">
-      for (int i = lastSet + 1; i < index; i++) {
-        valuesMutator.set(i, emptyByteArray);
-      }
-      </#if>
-      bits.getMutator().set(index, holder.isSet);
-      valuesMutator.set(index, holder);
-      <#if type.major == "VarLen">lastSet = index;</#if>
-    }
-    </#if>
 
 
-
-    <#if minor.class == "Int" || minor.class == "VarChar">
     public void set(int index, ${minor.class}Holder holder) {
         /* DELEGATE TO NEW VECTOR */
         values.set(index, holder);
     }
-    <#else>
-    public void set(int index, ${minor.class}Holder holder) {
-      final ${valuesName}.Mutator valuesMutator = values.getMutator();
-      <#if type.major == "VarLen">
-      for (int i = lastSet + 1; i < index; i++) {
-        valuesMutator.set(i, emptyByteArray);
-      }
-      </#if>
-      bits.getMutator().setToOne(index);
-      valuesMutator.set(index, holder);
-      <#if type.major == "VarLen">lastSet = index;</#if>
-    }
-    </#if>
-
 
 
-    <#if minor.class == "Int" || minor.class == "VarChar">
     public boolean isSafe(int outIndex) {
        /* DELEGATE TO NEW VECTOR */
        return values.isSafe(outIndex);
     }
-    <#else>
-    public boolean isSafe(int outIndex) {
-      return outIndex < ${className}.this.getValueCapacity();
-    }
-    </#if>
-
 
 
-    <#if minor.class == "Int" || minor.class == "VarChar">
-    <#if minor.class == "Int">
-    public void set(int index, int isSet, int valueField) {
-      /* DELEGATE TO NEW VECTOR */
-      values.set(index, isSet, valueField);
-    }
-    public void setSafe(int index, int isSet, int valueField) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setSafe(index, isSet, valueField);
-    }
-    </#if>
-    <#if minor.class == "VarChar">
-    public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) {
-      /* DELEGATE TO NEW VECTOR */
-      values.set(index, isSet, startField, endField, bufferField);
-    }
-    public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) {
-        /* DELEGATE TO NEW VECTOR */
-        values.setSafe(index, isSet, startField, endField, bufferField);
-    }
-    </#if>
-    <#else>
     <#assign fields = minor.fields!type.fields />
     public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field</#list> ){
-      final ${valuesName}.Mutator valuesMutator = values.getMutator();
-      <#if type.major == "VarLen">
-      for (int i = lastSet + 1; i < index; i++) {
-        valuesMutator.set(i, emptyByteArray);
-      }
-      </#if>
-      bits.getMutator().set(index, isSet);
-      valuesMutator.set(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
-      <#if type.major == "VarLen">lastSet = index;</#if>
+      values.set(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
     }
 
     public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
-      <#if type.major == "VarLen">
-      fillEmpties(index);
-      </#if>
-      bits.getMutator().setSafe(index, isSet);
-      values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
-      setCount++;
-      <#if type.major == "VarLen">lastSet = index;</#if>
+      values.setSafe(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
     }
-    </#if>
 
 
-
-    <#if minor.class == "Int" || minor.class == "VarChar">
     public void setSafe(int index, Nullable${minor.class}Holder value) {
       /* DELEGATE TO NEW VECTOR */
       values.setSafe(index, value);
@@ -1077,40 +536,12 @@ public void setSafe(int index, ${minor.class}Holder value) {
       /* DELEGATE TO NEW VECTOR */
       values.setSafe(index, value);
     }
-    <#else>
-    public void setSafe(int index, Nullable${minor.class}Holder value) {
-      <#if type.major == "VarLen">
-      fillEmpties(index);
-      </#if>
-      bits.getMutator().setSafe(index, value.isSet);
-      values.getMutator().setSafe(index, value);
-      setCount++;
-      <#if type.major == "VarLen">lastSet = index;</#if>
-    }
-
-    public void setSafe(int index, ${minor.class}Holder value) {
-      <#if type.major == "VarLen">
-      fillEmpties(index);
-      </#if>
-      bits.getMutator().setSafeToOne(index);
-      values.getMutator().setSafe(index, value);
-      setCount++;
-      <#if type.major == "VarLen">lastSet = index;</#if>
-    }
-    </#if>
-
 
 
     <#if !(type.major == "VarLen" || minor.class == "IntervalDay")>
     public void setSafe(int index, ${minor.javaType!type.javaType} value) {
-      <#if minor.class == "Int">
-        /* DELEGATE TO NEW VECTOR */
-        values.setSafe(index, value);
-      <#else>
-      bits.getMutator().setSafeToOne(index);
-      values.getMutator().setSafe(index, value);
-      setCount++;
-      </#if>
+      /* DELEGATE TO NEW VECTOR */
+      values.setSafe(index, value);
     }
     </#if>
 
@@ -1118,93 +549,45 @@ public void setSafe(int index, ${minor.javaType!type.javaType} value) {
 
     <#if minor.class == "Decimal">
     public void set(int index, ${friendlyType} value) {
-      bits.getMutator().setToOne(index);
-      values.getMutator().set(index, value);
+      /* DELEGATE TO NEW VECTOR */
+      values.set(index, value);
     }
 
     public void setSafe(int index, ${friendlyType} value) {
-      bits.getMutator().setSafeToOne(index);
-      values.getMutator().setSafe(index, value);
-      setCount++;
+      /* DELEGATE TO NEW VECTOR */
+      values.setSafe(index, value);
     }
     </#if>
 
 
-
-    <#if minor.class == "Int" || minor.class == "VarChar">
     @Override
     public void setValueCount(int valueCount) {
       /* DELEGATE TO NEW VECTOR */
       values.setValueCount(valueCount);
     }
-    <#else>
-    @Override
-    public void setValueCount(int valueCount) {
-      assert valueCount >= 0;
-      <#if type.major == "VarLen">
-      fillEmpties(valueCount);
-      </#if>
-      values.getMutator().setValueCount(valueCount);
-      bits.getMutator().setValueCount(valueCount);
-    }
-    </#if>
-
 
 
-    <#if minor.class != "Int" && minor.class != "VarChar">
     /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */
     @Override
-    public void generateTestData(int valueCount){
-      bits.getMutator().generateTestDataAlt(valueCount);
-      values.getMutator().generateTestData(valueCount);
-      <#if type.major = "VarLen">lastSet = valueCount;</#if>
-      setValueCount(valueCount);
-    }
-    </#if>
+    public void generateTestData(int valueCount) { }
 
 
-
-    <#if minor.class != "Int" && minor.class != "VarChar">
     /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */
     @Override
-    public void reset(){
-      setCount = 0;
-      <#if type.major = "VarLen">lastSet = -1;</#if>
-    }
-    </#if>
-
+    public void reset() { }
 
 
-    <#if minor.class == "VarChar">
+    <#if type.major == "VarLen">
     public void setLastSet(int value) {
       /* DELEGATE TO NEW VECTOR */
       values.setLastSet(value);
     }
-    <#else>
-    public void setLastSet(int value) {
-      <#if type.major = "VarLen">
-        lastSet = value;
-      <#else>
-        throw new UnsupportedOperationException();
-      </#if>
-    }
-    </#if>
 
 
-
-    <#if minor.class == "VarChar">
     public int getLastSet() {
       /* DELEGATE TO NEW VECTOR */
       return values.getLastSet();
     }
-    <#else>
-    public int getLastSet() {
-      <#if type.major != "VarLen">
-        throw new UnsupportedOperationException();
-      <#else>
-        return lastSet;
-      </#if>
-    }
     </#if>
   }
 }
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index 9fe41d0d96335..8ac23fe46f7d2 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -36,11 +36,11 @@
 public class UnionListWriter extends AbstractFieldWriter {
 
   private ListVector vector;
-  private UInt4Vector offsets;
   private PromotableWriter writer;
   private boolean inMap = false;
   private String mapName;
   private int lastIndex = 0;
+  private static final int OFFSET_WIDTH = 4;
 
   public UnionListWriter(ListVector vector) {
     this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
@@ -49,7 +49,6 @@ public UnionListWriter(ListVector vector) {
   public UnionListWriter(ListVector vector, NullableMapWriterFactory nullableMapWriterFactory) {
     this.vector = vector;
     this.writer = new PromotableWriter(vector.getDataVector(), vector, nullableMapWriterFactory);
-    this.offsets = vector.getOffsetVector();
   }
 
   public UnionListWriter(ListVector vector, AbstractFieldWriter parent) {
@@ -72,7 +71,7 @@ public Field getField() {
   }
 
   public void setValueCount(int count) {
-    vector.getMutator().setValueCount(count);
+    vector.setValueCount(count);
   }
 
   @Override
@@ -133,13 +132,13 @@ public MapWriter map(String name) {
 
   @Override
   public void startList() {
-    vector.getMutator().startNewValue(idx());
-    writer.setPosition(offsets.getAccessor().get(idx() + 1));
+    vector.startNewValue(idx());
+    writer.setPosition(vector.getOffsetBuffer().getInt((idx() + 1) * OFFSET_WIDTH));
   }
 
   @Override
   public void endList() {
-    offsets.getMutator().set(idx() + 1, writer.idx());
+    vector.getOffsetBuffer().setInt((idx() + 1) * OFFSET_WIDTH, writer.idx());
     setPosition(idx() + 1);
   }
 
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 3c7ed0190196a..3da383fe6f3c5 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -25,6 +25,7 @@
 
 <#include "/@includes/vv_imports.ftl" />
 import com.google.common.collect.ImmutableList;
+import io.netty.buffer.ArrowBuf;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -32,6 +33,9 @@
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.util.OversizedAllocationException;
 
 import static org.apache.arrow.vector.types.UnionMode.Sparse;
 
@@ -56,12 +60,10 @@ public class UnionVector implements FieldVector {
 
   private String name;
   private BufferAllocator allocator;
-  private Accessor accessor = new Accessor();
-  private Mutator mutator = new Mutator();
   int valueCount;
 
   MapVector internalMap;
-  UInt1Vector typeVector;
+  ArrowBuf typeBuffer;
 
   private NullableMapVector mapVector;
   private ListVector listVector;
@@ -71,16 +73,17 @@ public class UnionVector implements FieldVector {
   private int singleType = 0;
   private ValueVector singleVector;
 
+  private static final byte TYPE_WIDTH = 1;
   private final CallBack callBack;
-  private final List<BufferBacked> innerVectors;
+  private int typeBufferAllocationSizeInBytes;
 
   public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
     this.name = name;
     this.allocator = allocator;
     this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
-    this.typeVector = new UInt1Vector("types", allocator);
+    this.typeBuffer = allocator.getEmpty();
     this.callBack = callBack;
-    this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
+    this.typeBufferAllocationSizeInBytes = BaseValueVector.INITIAL_VALUE_ALLOCATION * TYPE_WIDTH;
   }
 
   public BufferAllocator getAllocator() {
@@ -104,20 +107,33 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    // truncate types vector buffer to size (#0)
-    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 0, typeVector.getBufferSizeFor(fieldNode.getLength()));
-    BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
+    if (ownBuffers.size() != 1) {
+      throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size());
+    }
+
+    ArrowBuf buffer = ownBuffers.get(0);
+    typeBuffer.release();
+    typeBuffer = buffer.retain(allocator);
+    typeBufferAllocationSizeInBytes = typeBuffer.capacity();
     this.valueCount = fieldNode.getLength();
   }
 
   @Override
   public List<ArrowBuf> getFieldBuffers() {
-    return BaseDataValueVector.unload(getFieldInnerVectors());
+    List<ArrowBuf> result = new ArrayList<>(1);
+
+    typeBuffer.readerIndex(0);
+    typeBuffer.writerIndex(valueCount * TYPE_WIDTH);
+
+    result.add(typeBuffer);
+
+    return result;
   }
 
   @Override
+  @Deprecated
   public List<BufferBacked> getFieldInnerVectors() {
-     return this.innerVectors;
+     throw new UnsupportedOperationException("There are no inner vectors. Use geFieldBuffers");
   }
 
   private String fieldName(MinorType type) {
@@ -134,7 +150,7 @@ private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
 
   @Override
   public long getValidityBufferAddress() {
-    return typeVector.getDataBuffer().memoryAddress();
+    return typeBuffer.memoryAddress();
   }
 
   @Override
@@ -148,7 +164,7 @@ public long getOffsetBufferAddress() {
   }
 
   @Override
-  public ArrowBuf getValidityBuffer() { return typeVector.getDataBuffer(); }
+  public ArrowBuf getValidityBuffer() { return typeBuffer; }
 
   @Override
   public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); }
@@ -211,47 +227,76 @@ public ListVector getList() {
   }
 
   public int getTypeValue(int index) {
-    return typeVector.getAccessor().get(index);
-  }
-
-  public UInt1Vector getTypeVector() {
-    return typeVector;
+    return typeBuffer.getByte(index * TYPE_WIDTH);
   }
 
   @Override
   public void allocateNew() throws OutOfMemoryException {
     internalMap.allocateNew();
-    typeVector.allocateNew();
-    if (typeVector != null) {
-      typeVector.zeroVector();
+    try {
+      allocateTypeBuffer();
+    } catch (Exception e) {
+      clear();
+      throw e;
     }
   }
 
   @Override
   public boolean allocateNewSafe() {
     boolean safe = internalMap.allocateNewSafe();
-    safe = safe && typeVector.allocateNewSafe();
-    if (safe) {
-      if (typeVector != null) {
-        typeVector.zeroVector();
-      }
+    if (!safe) { return false; }
+    try {
+      allocateTypeBuffer();
+    } catch (Exception e) {
+      clear();
+      return  false;
     }
-    return safe;
+
+    return true;
+  }
+
+  private void allocateTypeBuffer() {
+    typeBuffer = allocator.buffer(typeBufferAllocationSizeInBytes);
+    typeBuffer.readerIndex(0);
+    typeBuffer.setZero(0, typeBuffer.capacity());
   }
 
   @Override
   public void reAlloc() {
     internalMap.reAlloc();
-    typeVector.reAlloc();
+    reallocTypeBuffer();
   }
 
-  @Override
-  public void setInitialCapacity(int numRecords) {
+  private void reallocTypeBuffer() {
+    final int currentBufferCapacity = typeBuffer.capacity();
+    long baseSize  = typeBufferAllocationSizeInBytes;
+
+    if (baseSize < (long)currentBufferCapacity) {
+      baseSize = (long)currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    newBuf.setBytes(0, typeBuffer, 0, currentBufferCapacity);
+    final int halfNewCapacity = newBuf.capacity() / 2;
+    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    typeBuffer.release(1);
+    typeBuffer = newBuf;
+    typeBufferAllocationSizeInBytes = (int)newAllocationSize;
   }
 
+  @Override
+  public void setInitialCapacity(int numRecords) { }
+
   @Override
   public int getValueCapacity() {
-    return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity());
+    return Math.min(getTypeBufferValueCapacity(), internalMap.getValueCapacity());
   }
 
   @Override
@@ -261,7 +306,9 @@ public void close() {
 
   @Override
   public void clear() {
-    typeVector.clear();
+    valueCount = 0;
+    typeBuffer.release();
+    typeBuffer = allocator.getEmpty();
     internalMap.clear();
   }
 
@@ -300,7 +347,7 @@ public TransferPair makeTransferPair(ValueVector target) {
   public void copyFrom(int inIndex, int outIndex, UnionVector from) {
     from.getReader().setPosition(inIndex);
     getWriter().setPosition(outIndex);
-    ComplexCopier.copy(from.reader, mutator.writer);
+    ComplexCopier.copy(from.reader, writer);
   }
 
   public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
@@ -321,33 +368,34 @@ public FieldVector addVector(FieldVector v) {
 
   private class TransferImpl implements TransferPair {
     private final TransferPair internalMapVectorTransferPair;
-    private final TransferPair typeVectorTransferPair;
     private final UnionVector to;
 
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
       to = new UnionVector(name, allocator, callBack);
       internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap);
-      typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector);
     }
 
     public TransferImpl(UnionVector to) {
       this.to = to;
       internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap);
-      typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector);
     }
 
     @Override
     public void transfer() {
+      to.clear();
+      to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer;
       internalMapVectorTransferPair.transfer();
-      typeVectorTransferPair.transfer();
       to.valueCount = valueCount;
     }
 
     @Override
     public void splitAndTransfer(int startIndex, int length) {
+      to.clear();
       internalMapVectorTransferPair.splitAndTransfer(startIndex, length);
-      typeVectorTransferPair.splitAndTransfer(startIndex, length);
-      to.getMutator().setValueCount(length);
+      final int startPoint = startIndex * TYPE_WIDTH;
+      final int sliceLength = length * TYPE_WIDTH;
+      to.typeBuffer = typeBuffer.slice(startPoint, sliceLength).transferOwnership(to.allocator).buffer;
+      to.setValueCount(length);
     }
 
     @Override
@@ -362,13 +410,15 @@ public void copyValueSafe(int from, int to) {
   }
 
   @Override
+  @Deprecated
   public Accessor getAccessor() {
-    return accessor;
+    throw new UnsupportedOperationException("Accessor is not supported for reading from UNION");
   }
 
   @Override
+  @Deprecated
   public Mutator getMutator() {
-    return mutator;
+    throw new UnsupportedOperationException("Mutator is not supported for writing to UNION");
   }
 
   @Override
@@ -380,15 +430,17 @@ public FieldReader getReader() {
   }
 
   public FieldWriter getWriter() {
-    if (mutator.writer == null) {
-      mutator.writer = new UnionWriter(this);
+    if (writer == null) {
+      writer = new UnionWriter(this);
     }
-    return mutator.writer;
+    return writer;
   }
 
   @Override
   public int getBufferSize() {
-    return typeVector.getBufferSize() + internalMap.getBufferSize();
+    if (valueCount == 0) { return 0; }
+
+    return (valueCount * TYPE_WIDTH) + internalMap.getBufferSize();
   }
 
   @Override
@@ -402,13 +454,13 @@ public int getBufferSizeFor(final int valueCount) {
       bufferSize += v.getBufferSizeFor(valueCount);
     }
 
-    return (int) bufferSize;
+    return (int) bufferSize + (valueCount * TYPE_WIDTH);
   }
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
     ImmutableList.Builder<ArrowBuf> builder = ImmutableList.builder();
-    builder.add(typeVector.getBuffers(clear));
+    builder.add(typeBuffer);
     builder.add(internalMap.getBuffers(clear));
     List<ArrowBuf> list = builder.build();
     return list.toArray(new ArrowBuf[list.size()]);
@@ -417,15 +469,12 @@ public ArrowBuf[] getBuffers(boolean clear) {
   @Override
   public Iterator<ValueVector> iterator() {
     List<ValueVector> vectors = Lists.newArrayList(internalMap.iterator());
-    vectors.add(typeVector);
     return vectors.iterator();
   }
 
-  public class Accessor extends BaseValueVector.BaseAccessor {
 
-    @Override
     public Object getObject(int index) {
-      int type = typeVector.getAccessor().get(index);
+      int type = typeBuffer.getByte(index * TYPE_WIDTH);
       switch (MinorType.values()[type]) {
       case NULL:
         return null;
@@ -436,18 +485,14 @@ public Object getObject(int index) {
           <#assign uncappedName = name?uncap_first/>
           <#if !minor.typeParams?? >
       case ${name?upper_case}:
-        <#if minor.class != "Int" && minor.class != "VarChar">
-          return get${name}Vector().getAccessor().getObject(index);
-        <#else>
           return get${name}Vector().getObject(index);
-        </#if>
           </#if>
         </#list>
       </#list>
       case MAP:
-        return getMap().getAccessor().getObject(index);
+        return getMap().getObject(index);
       case LIST:
-        return getList().getAccessor().getObject(index);
+        return getList().getObject(index);
       default:
         throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[type]);
       }
@@ -466,30 +511,37 @@ public void get(int index, UnionHolder holder) {
       holder.reader = reader;
     }
 
-    @Override
     public int getValueCount() {
       return valueCount;
     }
 
-    @Override
     public boolean isNull(int index) {
-      return typeVector.getAccessor().get(index) == 0;
+      return (typeBuffer.getByte(index * TYPE_WIDTH) == 0);
+    }
+
+    @Override
+    public int getNullCount() {
+      int nullCount = 0;
+      for (int i = 0; i < getValueCount(); i++) {
+        if (isNull(i)) {
+          nullCount++;
+        }
+      }
+      return nullCount;
     }
 
     public int isSet(int index) {
       return isNull(index) ? 0 : 1;
     }
-  }
-
-  public class Mutator extends BaseValueVector.BaseMutator {
 
     UnionWriter writer;
 
-    @Override
     public void setValueCount(int valueCount) {
-      UnionVector.this.valueCount = valueCount;
-      typeVector.getMutator().setValueCount(valueCount);
-      internalMap.getMutator().setValueCount(valueCount);
+      this.valueCount = valueCount;
+      while (valueCount > getTypeBufferValueCapacity()) {
+        reallocTypeBuffer();
+      }
+      internalMap.setValueCount(valueCount);
     }
 
     public void setSafe(int index, UnionHolder holder) {
@@ -534,11 +586,7 @@ public void setSafe(int index, UnionHolder holder) {
         <#if !minor.typeParams?? >
     public void setSafe(int index, Nullable${name}Holder holder) {
       setType(index, MinorType.${name?upper_case});
-      <#if minor.class != "Int" && minor.class != "VarChar">
-        get${name}Vector().getMutator().setSafe(index, holder);
-      <#else>
-        get${name}Vector().setSafe(index, holder);
-      </#if>
+      get${name}Vector().setSafe(index, holder);
     }
 
         </#if>
@@ -546,19 +594,13 @@ public void setSafe(int index, Nullable${name}Holder holder) {
     </#list>
 
     public void setType(int index, MinorType type) {
-      typeVector.getMutator().setSafe(index, (byte) type.ordinal());
+      while (index >= getTypeBufferValueCapacity()) {
+        reallocTypeBuffer();
+      }
+      typeBuffer.setByte(index * TYPE_WIDTH , (byte) type.ordinal());
     }
 
-    @Override
-    public void reset() { }
-
-    @Override
-    public void generateTestData(int values) { }
-  }
-
-  public int getValueCount() { return 0; }
-
-  public void setValueCount(int valueCount) { }
-
-  public Object getObject(int index) { return null; }
+    private int getTypeBufferValueCapacity() {
+      return (int) ((typeBuffer.capacity() * 1.0) / TYPE_WIDTH);
+    }
 }
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index f892bac913392..526708a4c328b 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -62,7 +62,7 @@ public void setPosition(int index) {
 
   @Override
   public void start() {
-    data.getMutator().setType(idx(), MinorType.MAP);
+    data.setType(idx(), MinorType.MAP);
     getMapWriter().start();
   }
 
@@ -74,7 +74,7 @@ public void end() {
   @Override
   public void startList() {
     getListWriter().startList();
-    data.getMutator().setType(idx(), MinorType.LIST);
+    data.setType(idx(), MinorType.LIST);
   }
 
   @Override
@@ -92,7 +92,7 @@ private MapWriter getMapWriter() {
   }
 
   public MapWriter asMap() {
-    data.getMutator().setType(idx(), MinorType.MAP);
+    data.setType(idx(), MinorType.MAP);
     return getMapWriter();
   }
 
@@ -106,7 +106,7 @@ private ListWriter getListWriter() {
   }
 
   public ListWriter asList() {
-    data.getMutator().setType(idx(), MinorType.LIST);
+    data.setType(idx(), MinorType.LIST);
     return getListWriter();
   }
 
@@ -150,19 +150,19 @@ BaseWriter getWriter(MinorType minorType) {
   }
 
   public ${name}Writer as${name}() {
-    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    data.setType(idx(), MinorType.${name?upper_case});
     return get${name}Writer();
   }
 
   @Override
   public void write(${name}Holder holder) {
-    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    data.setType(idx(), MinorType.${name?upper_case});
     get${name}Writer().setPosition(idx());
     get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
   }
 
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    data.setType(idx(), MinorType.${name?upper_case});
     get${name}Writer().setPosition(idx());
     get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
   }
@@ -175,28 +175,28 @@ public void writeNull() {
 
   @Override
   public MapWriter map() {
-    data.getMutator().setType(idx(), MinorType.LIST);
+    data.setType(idx(), MinorType.LIST);
     getListWriter().setPosition(idx());
     return getListWriter().map();
   }
 
   @Override
   public ListWriter list() {
-    data.getMutator().setType(idx(), MinorType.LIST);
+    data.setType(idx(), MinorType.LIST);
     getListWriter().setPosition(idx());
     return getListWriter().list();
   }
 
   @Override
   public ListWriter list(String name) {
-    data.getMutator().setType(idx(), MinorType.MAP);
+    data.setType(idx(), MinorType.MAP);
     getMapWriter().setPosition(idx());
     return getMapWriter().list(name);
   }
 
   @Override
   public MapWriter map(String name) {
-    data.getMutator().setType(idx(), MinorType.MAP);
+    data.setType(idx(), MinorType.MAP);
     getMapWriter().setPosition(idx());
     return getMapWriter().map(name);
   }
@@ -209,14 +209,14 @@ public MapWriter map(String name) {
   <#if !minor.typeParams?? >
   @Override
   public ${capName}Writer ${lowerName}(String name) {
-    data.getMutator().setType(idx(), MinorType.MAP);
+    data.setType(idx(), MinorType.MAP);
     getMapWriter().setPosition(idx());
     return getMapWriter().${lowerName}(name);
   }
 
   @Override
   public ${capName}Writer ${lowerName}() {
-    data.getMutator().setType(idx(), MinorType.LIST);
+    data.setType(idx(), MinorType.LIST);
     getListWriter().setPosition(idx());
     return getListWriter().${lowerName}();
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
index c5f781000c9a9..83dc409a74f14 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
@@ -35,12 +35,17 @@
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
+/**
+ * BaseNullableFixedWidthVector provides an abstract interface for
+ * implementing vectors of fixed width values. The vectors are nullable
+ * implying that zero or more elements in the vector could be NULL.
+ */
 public abstract class BaseNullableFixedWidthVector extends BaseValueVector
         implements FixedWidthVector, FieldVector {
    private final byte typeWidth;
 
-   private int valueAllocationSizeInBytes;
-   private int validityAllocationSizeInBytes;
+   protected int valueAllocationSizeInBytes;
+   protected int validityAllocationSizeInBytes;
 
    protected final Field field;
    private int allocationMonitor;
@@ -52,23 +57,21 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all
                                        FieldType fieldType, final byte typeWidth) {
       super(name, allocator);
       this.typeWidth = typeWidth;
-      valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth;
-      validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION);
       field = new Field(name, fieldType, null);
       valueCount = 0;
       allocationMonitor = 0;
       validityBuffer = allocator.getEmpty();
       valueBuffer = allocator.getEmpty();
+      if (typeWidth > 0) {
+         valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth;
+         validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
+      } else {
+         /* specialized handling for NullableBitVector */
+         valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
+         validityAllocationSizeInBytes = valueAllocationSizeInBytes;
+      }
    }
 
-   /* TODO:
-    * Determine how writerIndex and readerIndex need to be used. Right now we
-    * are setting the writerIndex and readerIndex in the call to getFieldBuffers
-    * using the valueCount -- this assumes that the caller of getFieldBuffers
-    * on the vector has already invoked setValueCount.
-    *
-    * Do we need to set them during vector transfer and splitAndTransfer?
-    */
 
    /* TODO:
     *
@@ -84,57 +87,91 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all
     * abstracted out to top level base class.
     *
     * Right now BaseValueVector is the top level base class for other
-    * vector types in ValueVector hierarchy and those vectors have not
-    * yet been refactored so moving things to the top class as of now
-    * is not a good idea.
+    * vector types in ValueVector hierarchy (non-nullable) and those
+    * vectors have not yet been refactored/removed so moving things to
+    * the top class as of now is not a good idea.
     */
 
-   /* TODO:
-    * See if we need logger -- unnecessary object probably
-    */
-
-   protected abstract org.slf4j.Logger getLogger();
 
    @Override
+   @Deprecated
    public Mutator getMutator() {
-      throw new  UnsupportedOperationException("Mutator is not needed to write into vector");
+      throw new UnsupportedOperationException("Mutator is not supported for writing to vector");
    }
 
    @Override
+   @Deprecated
    public Accessor getAccessor() {
-      throw new UnsupportedOperationException("Accessor is not needed to read from vector");
+      throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
    }
 
+   /**
+    * Get the memory address of buffer that manages the validity
+    * (NULL or NON-NULL nature) of elements in the vector.
+    * @return starting address of the buffer
+    */
    @Override
    public long getValidityBufferAddress() {
       return (validityBuffer.memoryAddress());
    }
 
+   /**
+    * Get the memory address of buffer that stores the data for elements
+    * in the vector.
+    * @return starting address of the buffer
+    */
    @Override
    public long getDataBufferAddress() {
       return (valueBuffer.memoryAddress());
    }
 
+   /**
+    * Get the memory address of buffer that stores the offsets for elements
+    * in the vector. This operation is not supported for fixed-width vectors.
+    * @return starting address of the buffer
+    * @throws UnsupportedOperationException for fixed width vectors
+    */
    @Override
    public long getOffsetBufferAddress() {
       throw new UnsupportedOperationException("not supported for fixed-width vectors");
    }
 
+   /**
+    * Get buffer that manages the validity (NULL or NON-NULL nature) of
+    * elements in the vector. Consider it as a buffer for internal bit vector
+    * data structure.
+    * @return buffer
+    */
    @Override
    public ArrowBuf getValidityBuffer() {
       return validityBuffer;
    }
 
+   /**
+    * Get the buffer that stores the data for elements in the vector.
+    * @return buffer
+    */
    @Override
    public ArrowBuf getDataBuffer() {
       return valueBuffer;
    }
 
+   /**
+    * buffer that stores the offsets for elements
+    * in the vector. This operation is not supported for fixed-width vectors.
+    * @return buffer
+    * @throws UnsupportedOperationException for fixed width vectors
+    */
    @Override
    public ArrowBuf getOffsetBuffer() {
       throw new UnsupportedOperationException("not supported for fixed-width vectors");
    }
 
+   /**
+    * Sets the desired value capacity for the vector. This function doesn't
+    * allocate any memory for the vector.
+    * @param valueCount desired number of elements in the vector
+    */
    @Override
    public void setInitialCapacity(int valueCount) {
       final long size = (long)valueCount * typeWidth;
@@ -142,52 +179,65 @@ public void setInitialCapacity(int valueCount) {
          throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
       }
       valueAllocationSizeInBytes = (int)size;
-      validityAllocationSizeInBytes = getSizeFromCount(valueCount);
+      validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
    }
 
+   /**
+    * Get the current value capacity for the vector
+    * @return number of elements that vector can hold.
+    */
    @Override
    public int getValueCapacity(){
       return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity());
    }
 
-   /* for test purposes */
    private int getValueBufferValueCapacity() {
       return (int)((valueBuffer.capacity() * 1.0)/typeWidth);
    }
 
-   /* for test purposes */
    private int getValidityBufferValueCapacity() {
       return (int)(validityBuffer.capacity() * 8L);
    }
 
-   /* number of bytes for the validity buffer for the given valueCount */
-   protected int getSizeFromCount(int valueCount) {
-      return (int) Math.ceil(valueCount / 8.0);
-   }
-
+   /**
+    * zero out the vector and the data in associated buffers.
+    */
    @Override
    public void zeroVector() {
       initValidityBuffer();
       initValueBuffer();
    }
 
+   /* zero out the validity buffer */
    private void initValidityBuffer() {
       validityBuffer.setZero(0, validityBuffer.capacity());
    }
 
+   /* zero out the data buffer */
    private void initValueBuffer() {
       valueBuffer.setZero(0, valueBuffer.capacity());
    }
 
+   /**
+    * Reset the vector to initial state. Same as {@link #zeroVector()}.
+    * Note that this method doesn't release any memory.
+    */
    public void reset() {
       zeroVector();
    }
 
+   /**
+    * Close the vector and release the associated buffers.
+    */
    @Override
    public void close() { clear(); }
 
+   /**
+    * Same as {@link #close()}
+    */
    @Override
    public void clear() {
+      valueCount = 0;
       validityBuffer = releaseBuffer(validityBuffer);
       valueBuffer = releaseBuffer(valueBuffer);
    }
@@ -208,6 +258,9 @@ protected void decrementAllocationMonitor() {
       allocationMonitor--;
    }
 
+   /**
+    * Same as {@link #allocateNewSafe()}.
+    */
    @Override
    public void allocateNew() {
       if(!allocateNewSafe()){
@@ -215,6 +268,15 @@ public void allocateNew() {
       }
    }
 
+   /**
+    * Allocate memory for the vector. We internally use a default value count
+    * of 4096 to allocate memory for at least these many elements in the
+    * vector. See {@link #allocateNew(int)} for allocating memory for specific
+    * number of elements in the vector.
+    *
+    * @return false if memory allocation fails, true otherwise.
+    */
+   @Override
    public boolean allocateNewSafe() {
       long curAllocationSizeValue = valueAllocationSizeInBytes;
       long curAllocationSizeValidity = validityAllocationSizeInBytes;
@@ -229,8 +291,7 @@ public boolean allocateNewSafe() {
       try{
          allocateBytes(curAllocationSizeValue, curAllocationSizeValidity);
       } catch (Exception e) {
-         getLogger().error("ERROR: Failure in allocateNewSafe");
-         getLogger().error(e.getMessage());
+         e.printStackTrace();
          clear();
          return false;
       }
@@ -238,9 +299,20 @@ public boolean allocateNewSafe() {
       return true;
    }
 
+   /**
+    * Allocate memory for the vector to support storing at least the provided number of
+    * elements in the vector. This method must be called prior to using the ValueVector.
+    *
+    * @param valueCount the desired number of elements in the vector
+    * @throws org.apache.arrow.memory.OutOfMemoryException
+    */
    public void allocateNew(int valueCount) {
       long valueBufferSize = valueCount * typeWidth;
-      long validityBufferSize = getSizeFromCount(valueCount);
+      long validityBufferSize = getValidityBufferSizeFromCount(valueCount);
+      if (typeWidth == 0) {
+         /* specialized handling for NullableBitVector */
+         valueBufferSize = validityBufferSize;
+      }
 
       if (allocationMonitor > 10) {
          /* step down the default memory allocation since we have observed
@@ -266,8 +338,7 @@ public void allocateNew(int valueCount) {
       try {
          allocateBytes(valueBufferSize, validityBufferSize);
       } catch(Exception e) {
-         getLogger().error("ERROR: Failure in allocateNew");
-         getLogger().error(e.getMessage());
+         e.printStackTrace();
          clear();
          throw e;
       }
@@ -294,8 +365,8 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer
       initValidityBuffer();
    }
 
-   /*
-    * during splitAndTransfer, if we splitting from a random position within a byte,
+   /**
+    * During splitAndTransfer, if we splitting from a random position within a byte,
     * we can't just slice the source buffer so we have to explicitly allocate the
     * validityBuffer of the target vector. This is unlike the databuffer which we can
     * always slice for the target vector.
@@ -307,23 +378,49 @@ private void allocateValidityBuffer(final int validityBufferSize) {
       initValidityBuffer();
    }
 
+   /**
+    * Get the potential buffer size for a particular number of records.
+    * @param count desired number of elements in the vector
+    * @return estimated size of underlying buffers if the vector holds
+    *         a given number of elements
+    */
    @Override
    public int getBufferSizeFor(final int count) {
       if (count == 0) { return 0; }
-      return (count * typeWidth) + getSizeFromCount(count);
+      return (count * typeWidth) + getValidityBufferSizeFromCount(count);
    }
 
+   /**
+    * Get the size (number of bytes) of underlying buffers used by this
+    * vector
+    * @return size of underlying buffers.
+    */
    @Override
    public int getBufferSize() {
       if (valueCount == 0) { return 0; }
-      return (valueCount * typeWidth) + getSizeFromCount(valueCount);
+      return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount);
    }
 
+   /**
+    * Get information about how this field is materialized.
+    * @return the field corresponding to this vector
+    */
    @Override
    public Field getField() {
       return field;
    }
 
+   /**
+    * Return the underlying buffers associated with this vector. Note that this doesn't
+    * impact the reference counts for this buffer so it only should be used for in-context
+    * access. Also note that this buffer changes regularly thus
+    * external classes shouldn't hold a reference to it (unless they change it).
+    *
+    * @param clear Whether to clear vector before returning; the buffers will still be refcounted
+    *              but the returned array will be the only reference to them
+    * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
+    *         vector instance.
+    */
    @Override
    public ArrowBuf[] getBuffers(boolean clear) {
       final ArrowBuf[] buffers = new ArrowBuf[2];
@@ -338,12 +435,20 @@ public ArrowBuf[] getBuffers(boolean clear) {
       return buffers;
    }
 
+   /**
+    * Resize the vector to increase the capacity. The internal behavior is to
+    * double the current value capacity.
+    */
    @Override
    public void reAlloc() {
       valueBuffer = reallocBufferHelper(valueBuffer, true);
       validityBuffer = reallocBufferHelper(validityBuffer, false);
    }
 
+   /**
+    * Helper method for reallocating a particular internal buffer
+    * Returns the new buffer.
+    */
    private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) {
       final int currentBufferCapacity = buffer.capacity();
       long baseSize  = (dataBuffer ? valueAllocationSizeInBytes
@@ -360,10 +465,6 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer)
          throw new OversizedAllocationException("Unable to expand the buffer");
       }
 
-      getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]",
-              name, (dataBuffer ? valueAllocationSizeInBytes : validityAllocationSizeInBytes),
-              newAllocationSize);
-
       final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
       newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
       final int halfNewCapacity = newBuf.capacity() / 2;
@@ -381,8 +482,17 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer)
    }
 
    @Override
-   public List<BufferBacked> getFieldInnerVectors() { throw new UnsupportedOperationException(); }
+   @Deprecated
+   public List<BufferBacked> getFieldInnerVectors() {
+      throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
+   }
 
+   /**
+    * Initialize the children in schema for this Field. This operation is a
+    * NO-OP for scalar types since they don't have any children.
+    * @param children the schema
+    * @throws IllegalArgumentException if children is a non-empty list for scalar types.
+    */
    @Override
    public void initializeChildrenFromFields(List<Field> children) {
       if (!children.isEmpty()) {
@@ -390,11 +500,23 @@ public void initializeChildrenFromFields(List<Field> children) {
       }
    }
 
+   /**
+    * Get the inner child vectors.
+    * @return list of child vectors for complex types, empty list for scalar vector
+    * types
+    */
    @Override
    public List<FieldVector> getChildrenFromFields() {
       return Collections.emptyList();
    }
 
+   /**
+    * Load the buffers of this vector with provided source buffers.
+    * The caller manages the source buffers and populates them before invoking
+    * this method.
+    * @param fieldNode  the fieldNode indicating the value count
+    * @param ownBuffers the buffers for this Field (own buffers only, children not included)
+    */
    @Override
    public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
       if (ownBuffers.size() != 2) {
@@ -415,13 +537,22 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
       validityAllocationSizeInBytes = validityBuffer.capacity();
    }
 
+   /**
+    * Get the buffers belonging to this vector
+    * @return the inner buffers.
+    */
    public List<ArrowBuf> getFieldBuffers() {
       List<ArrowBuf> result = new ArrayList<>(2);
 
       validityBuffer.readerIndex(0);
-      validityBuffer.writerIndex(getSizeFromCount(valueCount));
+      validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
       valueBuffer.readerIndex(0);
-      valueBuffer.writerIndex(valueCount * typeWidth);
+      if (typeWidth == 0) {
+         /* specialized handling for NullableBitVector */
+         valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
+      } else {
+         valueBuffer.writerIndex(valueCount * typeWidth);
+      }
 
       result.add(validityBuffer);
       result.add(valueBuffer);
@@ -429,18 +560,42 @@ public List<ArrowBuf> getFieldBuffers() {
       return result;
    }
 
+   /**
+    * Construct a transfer pair of this vector and another vector of same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @param callBack
+    * @return TransferPair
+    */
    @Override
    public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
       return getTransferPair(ref, allocator);
    }
 
+   /**
+    * Construct a transfer pair of this vector and another vector of same type.
+    * @param allocator allocator for the target vector
+    * @return TransferPair
+    */
    @Override
    public TransferPair getTransferPair(BufferAllocator allocator){
       return getTransferPair(name, allocator);
    }
 
+   /**
+    * Construct a transfer pair of this vector and another vector of same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return TransferPair
+    */
    public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);
 
+   /**
+    * Transfer this vector'data to another vector. The memory associated
+    * with this vector is transferred to the allocator of target vector
+    * for accounting and management purposes.
+    * @param target destination vector for transfer
+    */
    public void transferTo(BaseNullableFixedWidthVector target){
       compareTypes(target, "transferTo");
       target.clear();
@@ -450,6 +605,13 @@ public void transferTo(BaseNullableFixedWidthVector target){
       clear();
    }
 
+   /**
+    * Slice this vector at desired index and length and transfer the
+    * corresponding data to the target vector.
+    * @param startIndex start position of the split in source vector.
+    * @param length length of the split.
+    * @param target destination vector
+    */
    public void splitAndTransferTo(int startIndex, int length,
                                   BaseNullableFixedWidthVector target) {
       compareTypes(target, "splitAndTransferTo");
@@ -459,6 +621,9 @@ public void splitAndTransferTo(int startIndex, int length,
       target.setValueCount(length);
    }
 
+   /**
+    * Data buffer can always be split and transferred using slicing.
+    */
    private void splitAndTransferValueBuffer(int startIndex, int length,
                                             BaseNullableFixedWidthVector target) {
       final int startPoint = startIndex * typeWidth;
@@ -466,17 +631,21 @@ private void splitAndTransferValueBuffer(int startIndex, int length,
       target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
    }
 
+   /**
+    * Validity buffer has multiple cases of split and transfer depending on
+    * the starting position of the source index.
+    */
    private void splitAndTransferValidityBuffer(int startIndex, int length,
                                                BaseNullableFixedWidthVector target) {
       assert startIndex + length <= valueCount;
       int firstByteSource = BitVectorHelper.byteIndex(startIndex);
       int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
-      int byteSizeTarget = getSizeFromCount(length);
+      int byteSizeTarget = getValidityBufferSizeFromCount(length);
       int offset = startIndex % 8;
 
       if (length > 0) {
          if (offset == 0) {
-            // slice
+            /* slice */
             if (target.validityBuffer != null) {
                target.validityBuffer.release();
             }
@@ -493,8 +662,10 @@ private void splitAndTransferValidityBuffer(int startIndex, int length,
             target.allocateValidityBuffer(byteSizeTarget);
 
             for (int i = 0; i < byteSizeTarget - 1; i++) {
-               byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset);
-               byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset);
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
+                       firstByteSource + i, offset);
+               byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
+                       firstByteSource + i + 1, offset);
 
                target.validityBuffer.setByte(i, (b1 + b2));
             }
@@ -509,15 +680,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length,
              * by shifting data from the current byte.
              */
             if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-               byte b1 = getBitsFromCurrentByte(this.validityBuffer,
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
                        firstByteSource + byteSizeTarget - 1, offset);
-               byte b2 = getBitsFromNextByte(this.validityBuffer,
+               byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
                        firstByteSource + byteSizeTarget, offset);
 
                target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
             }
             else {
-               byte b1 = getBitsFromCurrentByte(this.validityBuffer,
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
                        firstByteSource + byteSizeTarget - 1, offset);
                target.validityBuffer.setByte(byteSizeTarget - 1, b1);
             }
@@ -525,14 +696,6 @@ private void splitAndTransferValidityBuffer(int startIndex, int length,
       }
    }
 
-   private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) {
-      return (byte)((data.getByte(index) & 0xFF) >>> offset);
-   }
-
-   private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
-      return (byte)((data.getByte(index) << (8 - offset)));
-   }
-
 
    /******************************************************************
     *                                                                *
@@ -547,27 +710,9 @@ private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
     * @return the number of null elements.
     */
    public int getNullCount() {
-      int count = 0;
-      final int sizeInBytes = getSizeFromCount(valueCount);
-
-      for (int i = 0; i < sizeInBytes; ++i) {
-         final byte byteValue = validityBuffer.getByte(i);
-         /* Java uses two's complement binary representation, hence 11111111_b which is -1
-          * when converted to Int will have 32bits set to 1. Masking the MSB and then
-          * adding it back solves the issue.
-          */
-         count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
-      }
-      int nullCount = (sizeInBytes * 8) - count;
-      /* if the valueCount is not a multiple of 8,
-       * the bits on the right were counted as null bits.
-       */
-      int remainder = valueCount % 8;
-      nullCount -= remainder == 0 ? 0 : 8 - remainder;
-      return nullCount;
+      return BitVectorHelper.getNullCount(validityBuffer, valueCount);
    }
 
-
    /**
     * Get the value count of vector. This will always be zero unless
     * {@link #setValueCount(int)} has been called prior to calling this.
@@ -578,7 +723,6 @@ public int getValueCount(){
       return valueCount;
    }
 
-
    /**
     * Set value count for the vector.
     *
@@ -625,7 +769,6 @@ public void setValueCount(int valueCount) {
       }
    }
 
-
    /**
     * Check if the given index is within the current value capacity
     * of the vector
@@ -637,7 +780,6 @@ public boolean isSafe(int index) {
       return index < getValueCapacity();
    }
 
-
    /**
     * Check if element at given index is null.
     *
@@ -648,7 +790,6 @@ public boolean isNull(int index) {
       return (isSet(index) == 0);
    }
 
-
    /**
     * Same as {@link #isNull(int)}.
     *
@@ -662,6 +803,11 @@ public int isSet(int index) {
       return Long.bitCount(b & (1L << bitIndex));
    }
 
+   /**
+    * Mark the particular position in the vector as non-null.
+    *
+    * @param index position of the element.
+    */
    public void setIndexDefined(int index) {
       handleSafe(index);
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
@@ -691,7 +837,6 @@ public void setSafe(int index, ByteBuffer value, int start, int length) {
     ******************************************************************/
 
 
-
    protected void handleSafe(int index) {
       while (index >= getValueCapacity()) {
          decrementAllocationMonitor();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
index a79709da4dd86..6587cde633ea9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
@@ -45,7 +45,7 @@ public abstract class BaseNullableVariableWidthVector extends BaseValueVector
    private int offsetAllocationSizeInBytes;
 
    /* protected members */
-   protected static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */
+   public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */
    protected static final byte[] emptyByteArray = new byte[]{};
    protected ArrowBuf validityBuffer;
    protected ArrowBuf valueBuffer;
@@ -59,7 +59,7 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator
                                           FieldType fieldType) {
       super(name, allocator);
       valueAllocationSizeInBytes = INITIAL_BYTE_COUNT;
-      validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION);
+      validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
       offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH;
       field = new Field(name, fieldType, null);
       valueCount = 0;
@@ -70,21 +70,12 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator
       cleared = false;
    }
 
-  /* TODO:
-    * Determine how writerIndex and readerIndex need to be used. Right now we
-    * are setting the writerIndex and readerIndex in the call to getFieldBuffers
-    * using the valueCount -- this assumes that the caller of getFieldBuffers
-    * on the vector has already invoked setValueCount.
-    *
-    * Do we need to set them during vector transfer and splitAndTransfer?
-    */
-
    /* TODO:
     *
     * see if getNullCount() can be made faster -- O(1)
     */
 
-   /* TODO:
+  /* TODO:
     * Once the entire hierarchy has been refactored, move common functions
     * like getNullCount(), splitAndTransferValidityBuffer to top level
     * base class BaseValueVector.
@@ -93,59 +84,89 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator
     * abstracted out to top level base class.
     *
     * Right now BaseValueVector is the top level base class for other
-    * vector types in ValueVector hierarchy and those vectors have not
-    * yet been refactored so moving things to the top class as of now
-    * is not a good idea.
-    */
-
-   /* TODO:
-    * See if we need logger -- unnecessary object probably
+    * vector types in ValueVector hierarchy (non-nullable) and those
+    * vectors have not yet been refactored/removed so moving things to
+    * the top class as of now is not a good idea.
     */
 
-   /* TODO:
-    * Implement getBufferSize(), getCurrentSizeInBytes().
-    */
-
-   protected abstract org.slf4j.Logger getLogger();
 
+   @Override
+   @Deprecated
    public VariableWidthMutator getMutator() {
-      throw new  UnsupportedOperationException("Mutator is not needed to write into vector");
+      throw new  UnsupportedOperationException("Mutator is not supported for writing into vector");
    }
 
+   @Override
+   @Deprecated
    public VariableWidthAccessor getAccessor() {
-      throw new UnsupportedOperationException("Accessor is not needed to read from vector");
+      throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
    }
 
+   /**
+    * Get buffer that manages the validity (NULL or NON-NULL nature) of
+    * elements in the vector. Consider it as a buffer for internal bit vector
+    * data structure.
+    * @return buffer
+    */
    @Override
    public ArrowBuf getValidityBuffer() {
       return validityBuffer;
    }
 
+   /**
+    * Get the buffer that stores the data for elements in the vector.
+    * @return buffer
+    */
    @Override
    public ArrowBuf getDataBuffer() {
       return valueBuffer;
    }
 
+   /**
+    * buffer that stores the offsets for elements
+    * in the vector. This operation is not supported for fixed-width vectors.
+    * @return buffer
+    */
    @Override
    public ArrowBuf getOffsetBuffer() {
       return offsetBuffer;
    }
 
+   /**
+    * Get the memory address of buffer that stores the offsets for elements
+    * in the vector.
+    * @return starting address of the buffer
+    */
    @Override
    public long getOffsetBufferAddress() {
       return offsetBuffer.memoryAddress();
    }
 
+   /**
+    * Get the memory address of buffer that manages the validity
+    * (NULL or NON-NULL nature) of elements in the vector.
+    * @return starting address of the buffer
+    */
    @Override
    public long getValidityBufferAddress() {
       return validityBuffer.memoryAddress();
    }
 
+   /**
+    * Get the memory address of buffer that stores the data for elements
+    * in the vector.
+    * @return starting address of the buffer
+    */
    @Override
    public long getDataBufferAddress() {
       return valueBuffer.memoryAddress();
    }
 
+   /**
+    * Sets the desired value capacity for the vector. This function doesn't
+    * allocate any memory for the vector.
+    * @param valueCount desired number of elements in the vector
+    */
    @Override
    public void setInitialCapacity(int valueCount) {
       final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT;
@@ -153,57 +174,69 @@ public void setInitialCapacity(int valueCount) {
          throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
       }
       valueAllocationSizeInBytes = (int)size;
-      validityAllocationSizeInBytes = getSizeFromCount(valueCount);
+      validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
       /* to track the end offset of last data element in vector, we need
        * an additional slot in offset buffer.
        */
       offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH;
    }
 
+   /**
+    * Get the current value capacity for the vector
+    * @return number of elements that vector can hold.
+    */
    @Override
    public int getValueCapacity(){
       final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0);
       return Math.min(offsetValueCapacity, getValidityBufferValueCapacity());
    }
 
-   /* for test purposes */
    private int getValidityBufferValueCapacity() {
       return (int)(validityBuffer.capacity() * 8L);
    }
 
-   /* for test purposes */
    private int getOffsetBufferValueCapacity() {
       return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH);
    }
 
-   /* number of bytes for the validity buffer for a given valueCount */
-   protected int getSizeFromCount(int valueCount) {
-      return (int) Math.ceil(valueCount / 8.0);
-   }
-
+   /**
+    * zero out the vector and the data in associated buffers.
+    */
    public void zeroVector() {
       initValidityBuffer();
       initOffsetBuffer();
    }
 
+   /* zero out the validity buffer */
    private void initValidityBuffer() {
       validityBuffer.setZero(0, validityBuffer.capacity());
    }
 
+   /* zero out the offset buffer */
    private void initOffsetBuffer() {
       offsetBuffer.setZero(0, offsetBuffer.capacity());
    }
 
+   /**
+    * Reset the vector to initial state. Same as {@link #zeroVector()}.
+    * Note that this method doesn't release any memory.
+    */
    public void reset() {
       zeroVector();
       lastSet = -1;
    }
 
+   /**
+    * Close the vector and release the associated buffers.
+    */
    @Override
    public void close() {
       clear();
    }
 
+   /**
+    * Same as {@link #close()}
+    */
    @Override
    public void clear() {
       validityBuffer = releaseBuffer(validityBuffer);
@@ -215,8 +248,17 @@ public void clear() {
    }
 
    @Override
-   public List<BufferBacked> getFieldInnerVectors() { throw new UnsupportedOperationException(); }
+   @Deprecated
+   public List<BufferBacked> getFieldInnerVectors() {
+      throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
+   }
 
+   /**
+    * Initialize the children in schema for this Field. This operation is a
+    * NO-OP for scalar types since they don't have any children.
+    * @param children the schema
+    * @throws IllegalArgumentException if children is a non-empty list for scalar types.
+    */
    @Override
    public void initializeChildrenFromFields(List<Field> children) {
       if (!children.isEmpty()) {
@@ -224,11 +266,24 @@ public void initializeChildrenFromFields(List<Field> children) {
       }
    }
 
+   /**
+    * Get the inner child vectors.
+    * @return list of child vectors for complex types, empty list for scalar vector
+    * types
+    */
    @Override
    public List<FieldVector> getChildrenFromFields() {
       return Collections.emptyList();
    }
 
+
+   /**
+    * Load the buffers of this vector with provided source buffers.
+    * The caller manages the source buffers and populates them before invoking
+    * this method.
+    * @param fieldNode  the fieldNode indicating the value count
+    * @param ownBuffers the buffers for this Field (own buffers only, children not included)
+    */
    @Override
    public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
       ArrowBuf bitBuffer = ownBuffers.get(0);
@@ -246,11 +301,15 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
       valueCount = fieldNode.getLength();
    }
 
+   /**
+    * Get the buffers belonging to this vector
+    * @return the inner buffers.
+    */
    public List<ArrowBuf> getFieldBuffers() {
       List<ArrowBuf> result = new ArrayList<>(3);
       final int lastDataOffset = getstartOffset(valueCount);
       validityBuffer.readerIndex(0);
-      validityBuffer.writerIndex(getSizeFromCount(valueCount));
+      validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
       offsetBuffer.readerIndex(0);
       offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
       valueBuffer.readerIndex(0);
@@ -263,6 +322,9 @@ public List<ArrowBuf> getFieldBuffers() {
       return result;
    }
 
+   /**
+    * Same as {@link #allocateNewSafe()}.
+    */
    @Override
    public void allocateNew() {
       if(!allocateNewSafe()){
@@ -270,6 +332,14 @@ public void allocateNew() {
       }
    }
 
+   /**
+    * Allocate memory for the vector. We internally use a default value count
+    * of 4096 to allocate memory for at least these many elements in the
+    * vector. See {@link #allocateNew(int, int)} for allocating memory for specific
+    * number of elements in the vector.
+    *
+    * @return false if memory allocation fails, true otherwise.
+    */
    @Override
    public boolean allocateNewSafe() {
       long curAllocationSizeValue = valueAllocationSizeInBytes;
@@ -287,8 +357,7 @@ public boolean allocateNewSafe() {
       try {
          allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset);
       } catch (Exception e) {
-         getLogger().error("ERROR: Failure in allocateNewSafe");
-         getLogger().error(e.getMessage());
+         e.printStackTrace();
          clear();
          return false;
       }
@@ -296,11 +365,19 @@ public boolean allocateNewSafe() {
       return true;
    }
 
+   /**
+    * Allocate memory for the vector to support storing at least the provided number of
+    * elements in the vector. This method must be called prior to using the ValueVector.
+    *
+    * @param totalBytes desired total memory capacity
+    * @param valueCount the desired number of elements in the vector
+    * @throws org.apache.arrow.memory.OutOfMemoryException
+    */
    @Override
    public void allocateNew(int totalBytes, int valueCount) {
       assert totalBytes >= 0;
       final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
-      final int validityBufferSize = getSizeFromCount(valueCount);
+      final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
 
       if (totalBytes > MAX_ALLOCATION_SIZE ||
               offsetBufferSize > MAX_ALLOCATION_SIZE) {
@@ -313,12 +390,12 @@ public void allocateNew(int totalBytes, int valueCount) {
       try {
          allocateBytes(totalBytes, validityBufferSize, offsetBufferSize);
       } catch (Exception e) {
-         getLogger().error("ERROR: Failure in allocateNewSafe");
-         getLogger().error(e.getMessage());
+         e.printStackTrace();
          clear();
       }
    }
 
+   /* allocate the inner buffers */
    private void allocateBytes(final long valueBufferSize, final long validityBufferSize,
                               final long offsetBufferSize) {
       /* allocate data buffer */
@@ -330,6 +407,7 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer
       allocateOffsetBuffer(offsetBufferSize);
    }
 
+   /* allocate offset buffer */
    private void allocateOffsetBuffer(final long size) {
       final int curSize = (int)size;
       offsetBuffer = allocator.buffer(curSize);
@@ -338,6 +416,7 @@ private void allocateOffsetBuffer(final long size) {
       initOffsetBuffer();
    }
 
+   /* allocate validity buffer */
    private void allocateValidityBuffer(final long size) {
       final int curSize = (int)size;
       validityBuffer = allocator.buffer(curSize);
@@ -346,6 +425,10 @@ private void allocateValidityBuffer(final long size) {
       initValidityBuffer();
    }
 
+   /**
+    * Resize the vector to increase the capacity. The internal behavior is to
+    * double the current value capacity.
+    */
    public void reAlloc() {
       reallocValueBuffer();
       reallocValidityAndOffsetBuffers();
@@ -378,10 +461,7 @@ protected void reallocValidityAndOffsetBuffers() {
       validityBuffer = reallocBufferHelper(validityBuffer, false);
    }
 
-   /* need to refactor this to keep the logic in an single place and make callers
-    * more intelligent. see handleSafe() for more comments on realloc
-    */
-
+   /* helper method to realloc a particular buffer. returns the allocated buffer */
    private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) {
       final int currentBufferCapacity = buffer.capacity();
       long baseSize  = (offsetBuffer ? offsetAllocationSizeInBytes
@@ -398,10 +478,6 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer
          throw new OversizedAllocationException("Unable to expand the buffer");
       }
 
-      getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]",
-              name, (offsetBuffer ? offsetAllocationSizeInBytes : validityAllocationSizeInBytes),
-              newAllocationSize);
-
       final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
       newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
       final int halfNewCapacity = newBuf.capacity() / 2;
@@ -418,6 +494,10 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer
       return buffer;
    }
 
+   /**
+    * Get the size (number of bytes) of underlying data buffer.
+    * @return
+    */
    @Override
    public int getByteCapacity(){
       return valueBuffer.capacity();
@@ -429,30 +509,55 @@ public int getCurrentSizeInBytes(){
       return 0;
    }
 
+   /**
+    * Get the size (number of bytes) of underlying buffers used by this
+    * vector
+    * @return size of underlying buffers.
+    */
    @Override
    public int getBufferSize() {
-      /* TODO */
-      return 0;
+      return getBufferSizeFor(this.valueCount);
    }
 
+   /**
+    * Get the potential buffer size for a particular number of records.
+    * @param valueCount desired number of elements in the vector
+    * @return estimated size of underlying buffers if the vector holds
+    *         a given number of elements
+    */
    @Override
    public int getBufferSizeFor(final int valueCount) {
       if (valueCount == 0) {
          return 0;
       }
 
-      final int validityBufferSize = getSizeFromCount(valueCount);
+      final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
       final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
       /* get the end offset for this valueCount */
       final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
       return validityBufferSize + offsetBufferSize + dataBufferSize;
    }
 
+   /**
+    * Get information about how this field is materialized.
+    * @return the field corresponding to this vector
+    */
    @Override
    public Field getField() {
       return field;
    }
 
+   /**
+    * Return the underlying buffers associated with this vector. Note that this doesn't
+    * impact the reference counts for this buffer so it only should be used for in-context
+    * access. Also note that this buffer changes regularly thus
+    * external classes shouldn't hold a reference to it (unless they change it).
+    *
+    * @param clear Whether to clear vector before returning; the buffers will still be refcounted
+    *              but the returned array will be the only reference to them
+    * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
+    *         vector instance.
+    */
    @Override
    public ArrowBuf[] getBuffers(boolean clear) {
       final ArrowBuf[] buffers = new ArrowBuf[3];
@@ -468,18 +573,42 @@ public ArrowBuf[] getBuffers(boolean clear) {
       return buffers;
    }
 
+   /**
+    * Construct a transfer pair of this vector and another vector of same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @param callBack
+    * @return TransferPair
+    */
    @Override
    public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
       return getTransferPair(ref, allocator);
    }
 
+   /**
+    * Construct a transfer pair of this vector and another vector of same type.
+    * @param allocator allocator for the target vector
+    * @return TransferPair
+    */
    @Override
    public TransferPair getTransferPair(BufferAllocator allocator){
       return getTransferPair(name, allocator);
    }
 
+   /**
+    * Construct a transfer pair of this vector and another vector of same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return TransferPair
+    */
    public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);
 
+   /**
+    * Transfer this vector'data to another vector. The memory associated
+    * with this vector is transferred to the allocator of target vector
+    * for accounting and management purposes.
+    * @param target destination vector for transfer
+    */
    public void transferTo(BaseNullableVariableWidthVector target){
       compareTypes(target, "transferTo");
       target.clear();
@@ -491,6 +620,13 @@ public void transferTo(BaseNullableVariableWidthVector target){
       clear();
    }
 
+   /**
+    * Slice this vector at desired index and length and transfer the
+    * corresponding data to the target vector.
+    * @param startIndex start position of the split in source vector.
+    * @param length length of the split.
+    * @param target destination vector
+    */
    public void splitAndTransferTo(int startIndex, int length,
                                   BaseNullableVariableWidthVector target) {
       compareTypes(target, "splitAndTransferTo");
@@ -502,7 +638,12 @@ public void splitAndTransferTo(int startIndex, int length,
    }
 
    /*
-    * transfer the offsets along with data
+    * Transfer the offsets along with data. Unlike the data buffer, we cannot simply
+    * slice the offset buffer for split and transfer. The reason is that offsets
+    * in the target vector have to be adjusted and made relative to the staring
+    * offset in source vector from the start index of split. This is why, we
+    * need to explicitly allocate the offset buffer and set the adjusted offsets
+    * in the target vector.
     */
    private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) {
       final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH);
@@ -517,14 +658,14 @@ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullab
    }
 
    /*
-    * transfer the validity.
+    * Transfer the validity.
     */
    private void splitAndTransferValidityBuffer(int startIndex, int length,
                                                BaseNullableVariableWidthVector target) {
       assert startIndex + length <= valueCount;
       int firstByteSource = BitVectorHelper.byteIndex(startIndex);
       int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
-      int byteSizeTarget = getSizeFromCount(length);
+      int byteSizeTarget = getValidityBufferSizeFromCount(length);
       int offset = startIndex % 8;
 
       if (length > 0) {
@@ -546,8 +687,8 @@ private void splitAndTransferValidityBuffer(int startIndex, int length,
             target.allocateValidityBuffer(byteSizeTarget);
 
             for (int i = 0; i < byteSizeTarget - 1; i++) {
-               byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset);
-               byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset);
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset);
+               byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset);
 
                target.validityBuffer.setByte(i, (b1 + b2));
             }
@@ -562,15 +703,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length,
              * by shifting data from the current byte.
              */
             if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-               byte b1 = getBitsFromCurrentByte(this.validityBuffer,
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
                        firstByteSource + byteSizeTarget - 1, offset);
-               byte b2 = getBitsFromNextByte(this.validityBuffer,
+               byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
                        firstByteSource + byteSizeTarget, offset);
 
                target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
             }
             else {
-               byte b1 = getBitsFromCurrentByte(this.validityBuffer,
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
                        firstByteSource + byteSizeTarget - 1, offset);
                target.validityBuffer.setByte(byteSizeTarget - 1, b1);
             }
@@ -578,14 +719,6 @@ private void splitAndTransferValidityBuffer(int startIndex, int length,
       }
    }
 
-   private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) {
-      return (byte)((data.getByte(index) & 0xFF) >>> offset);
-   }
-
-   private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
-      return (byte)((data.getByte(index) << (8 - offset)));
-   }
-
 
    /******************************************************************
     *                                                                *
@@ -600,24 +733,7 @@ private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
     * @return the number of null elements.
     */
    public int getNullCount() {
-      int count = 0;
-      final int sizeInBytes = getSizeFromCount(valueCount);
-
-      for (int i = 0; i < sizeInBytes; ++i) {
-         final byte byteValue = validityBuffer.getByte(i);
-         /* Java uses two's complement binary representation, hence 11111111_b which is -1
-          * when converted to Int will have 32bits set to 1. Masking the MSB and then
-          * adding it back solves the issue.
-          */
-         count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
-      }
-      int nullCount = (sizeInBytes * 8) - count;
-      /* if the valueCount is not a multiple of 8,
-       * the bits on the right were counted as null bits.
-       */
-      int remainder = valueCount % 8;
-      nullCount -= remainder == 0 ? 0 : 8 - remainder;
-      return nullCount;
+      return BitVectorHelper.getNullCount(validityBuffer, valueCount);
    }
 
    /**
@@ -679,24 +795,55 @@ public void setValueCount(int valueCount) {
       lastSet = valueCount - 1;
    }
 
+   /**
+    * Create holes in the vector upto the given index (exclusive).
+    * Holes will be created from the current last set position in
+    * the vector.
+    *
+    * @param index target index
+    */
    public void fillEmpties(int index) {
       handleSafe(index, emptyByteArray.length);
       fillHoles(index);
       lastSet = index - 1;
    }
 
+   /**
+    * Set the index of last non-null element in the vector.
+    * It is important to call this method with appropriate value
+    * before calling {@link #setValueCount(int)}.
+    *
+    * @param value desired index of last non-null element.
+    */
    public void setLastSet(int value) {
       lastSet = value;
    }
 
+   /**
+    * Get the index of last non-null element in the vector.
+    *
+    * @return index of the last non-null element
+    */
    public int getLastSet() {
       return lastSet;
    }
 
+   /**
+    * Get the starting position (offset) in the data stream for a given
+    * element in the vector.
+    *
+    * @param index position of the element in the vector
+    * @return starting offset for the element
+    */
    public long getStartEnd(int index) {
       return (long)offsetBuffer.getInt(index * OFFSET_WIDTH);
    }
 
+   /**
+    * Mark the particular position in the vector as non-null.
+    *
+    * @param index position of the element.
+    */
    public void setIndexDefined(int index) {
       handleSafe(index, 0);
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
@@ -761,4 +908,97 @@ protected final void handleSafe(int index, int dataLength) {
          reallocValueBuffer();
       }
    }
+
+
+   /******************************************************************
+    *                                                                *
+    *                helper methods currently                        *
+    *                used by JsonFileReader and                      *
+    *                JsonFileWriter                                  *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Method used by Json Reader to explicitly set the data of the variable
+    * width vector elements. The method takes care of allocating the memory
+    * for the vector if caller hasn't done so.
+    *
+    * This method should not be used externally.
+    *
+    * @param data ArrowBuf for storing variable width elements in the vector
+    * @param offset offset of the element
+    * @param allocator memory allocator
+    * @param index position of the element in the vector
+    * @param value array of bytes for the element
+    * @param valueCount number of elements in the vector
+    * @return buffer holding the variable width data.
+    */
+   public static ArrowBuf set(ArrowBuf data, ArrowBuf offset,
+                              BufferAllocator allocator, int index, byte[] value,
+                              int valueCount) {
+      if (data == null) {
+         data = allocator.buffer(INITIAL_BYTE_COUNT);
+      }
+      final int currentBufferCapacity = data.capacity();
+      final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH);
+      while (currentBufferCapacity < currentStartOffset + value.length) {
+         final ArrowBuf newBuf = allocator.buffer(currentBufferCapacity * 2);
+         newBuf.setBytes(0, data, 0, currentBufferCapacity);
+         data.release();
+         data = newBuf;
+      }
+      data.setBytes(currentStartOffset, value, 0, value.length);
+      if (index == (valueCount - 1)) {
+         data.writerIndex(offset.getInt(valueCount * OFFSET_WIDTH));
+      }
+      return data;
+   }
+
+   /**
+    * Method used by Json Writer to read a variable width element from
+    * the variable width vector and write to Json.
+    *
+    * This method should not be used externally.
+    *
+    * @param data buffer storing the variable width vector elements
+    * @param offset buffer storing the offsets of variable width vector elements
+    * @param index position of the element in the vector
+    * @return array of bytes
+    */
+   public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) {
+      final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH);
+      final int dataLength =
+              offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset;
+      final byte[] result = new byte[dataLength];
+      data.getBytes(currentStartOffset, result, 0, dataLength);
+      return result;
+   }
+
+   /**
+    * Method used by Json Reader to explicitly set the offsets of the variable
+    * width vector data. The method takes care of allocating the memory for
+    * offsets if the caller hasn't done so.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer ArrowBuf to store offsets for variable width elements
+    * @param allocator memory allocator
+    * @param valueCount number of elements
+    * @param index position of the element
+    * @param value offset of the element
+    * @return buffer holding the offsets
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, int value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * OFFSET_WIDTH);
+      }
+      buffer.setInt(index * OFFSET_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * OFFSET_WIDTH);
+      }
+
+      return buffer;
+   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 1db96246485d3..1b96782681f26 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -56,7 +56,6 @@ public String toString() {
 
   @Override
   public void clear() {
-    getMutator().reset();
   }
 
   @Override
@@ -137,10 +136,36 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) {
     return buffer;
   }
 
-  public int getValueCount() { return 0; }
+  @Override
+  public int getValueCount() {
+    return getAccessor().getValueCount();
+  }
+
+  @Override
+  public void setValueCount(int valueCount) {
+    getMutator().setValueCount(valueCount);
+  }
+
+  @Override
+
+  public Object getObject(int index) {
+    return getAccessor().getObject(index);
+  }
+
+  @Override
+
+  public int getNullCount() {
+    return getAccessor().getNullCount();
+  }
 
-  public void setValueCount(int valueCount) { }
+  @Override
+  public boolean isNull(int index) {
+    return getAccessor().isNull(index);
+  }
 
-  public Object getObject(int index) { return null; }
+  /* number of bytes for the validity buffer for the given valueCount */
+  protected static int getValidityBufferSizeFromCount(final int valueCount) {
+    return (int) Math.ceil(valueCount / 8.0);
+  }
 }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
index 2439bd26eab56..69325b5ea06b6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
@@ -19,24 +19,35 @@
 package org.apache.arrow.vector;
 
 import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
 
-class BitVectorHelper {
+/**
+ * Helper class for performing generic operations on a bit vector buffer.
+ * External use of this class is not recommended.
+ */
+public class BitVectorHelper {
 
    /**
     * Get the index of byte corresponding to bit index in validity buffer
     */
-   protected static int byteIndex(int absoluteBitIndex) {
+   public static int byteIndex(int absoluteBitIndex) {
       return absoluteBitIndex >> 3;
    }
 
    /**
     * Get the relative index of bit within the byte in validity buffer
     */
-   private static int bitIndex(int absoluteBitIndex) {
+   public static int bitIndex(int absoluteBitIndex) {
       return absoluteBitIndex & 7;
    }
 
-   protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) {
+   /**
+    * Set the bit at provided index to 1.
+    *
+    * @param validityBuffer
+    * @param index
+    */
+   public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) {
       final int byteIndex = byteIndex(index);
       final int bitIndex = bitIndex(index);
       byte currentByte = validityBuffer.getByte(byteIndex);
@@ -45,7 +56,14 @@ protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) {
       validityBuffer.setByte(byteIndex, currentByte);
    }
 
-   protected static void setValidityBit(ArrowBuf validityBuffer, int index, int value) {
+   /**
+    * Set the bit at a given index to provided value (1 or 0)
+    *
+    * @param validityBuffer
+    * @param index
+    * @param value
+    */
+   public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) {
       final int byteIndex = byteIndex(index);
       final int bitIndex = bitIndex(index);
       byte currentByte = validityBuffer.getByte(byteIndex);
@@ -57,4 +75,91 @@ protected static void setValidityBit(ArrowBuf validityBuffer, int index, int val
       }
       validityBuffer.setByte(byteIndex, currentByte);
    }
+
+   /**
+    * Set the bit at a given index to provided value (1 or 0). Internally
+    * takes care of allocating the buffer if the caller didn't do so.
+    *
+    * @param validityBuffer
+    * @param allocator
+    * @param valueCount
+    * @param index
+    * @param value
+    * @return ArrowBuf
+    */
+   public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator,
+                                         int valueCount, int index, int value) {
+      if (validityBuffer == null) {
+         validityBuffer = allocator.buffer(getValidityBufferSize(valueCount));
+      }
+      setValidityBit(validityBuffer, index, value);
+      if (index == (valueCount - 1)) {
+         validityBuffer.writerIndex(getValidityBufferSize(valueCount));
+      }
+
+      return validityBuffer;
+   }
+
+   /**
+    * Check if a bit at a given index is set or not.
+    *
+    * @param buffer
+    * @param index
+    * @return 1 if bit is set, 0 otherwise.
+    */
+   public static int get(final ArrowBuf buffer, int index) {
+      final int byteIndex = index >> 3;
+      final byte b = buffer.getByte(byteIndex);
+      final int bitIndex = index & 7;
+      return Long.bitCount(b & (1L << bitIndex));
+   }
+
+   /**
+    * Compute the size of validity buffer required to manage a given number
+    * of elements in a vector.
+    *
+    * @param valueCount
+    * @return buffer size
+    */
+   public static int getValidityBufferSize(int valueCount) {
+      return ((int) Math.ceil(valueCount / 8.0));
+   }
+
+   /**
+    * Given a validity buffer, find the number of bits that are not set.
+    * This is used to compute the number of null elements in a nullable vector.
+    *
+    * @param validityBuffer
+    * @param valueCount
+    * @return number of bits not set.
+    */
+   public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) {
+      if (valueCount == 0) { return 0; }
+      int count = 0;
+      final int sizeInBytes = getValidityBufferSize(valueCount);
+
+      for (int i = 0; i < sizeInBytes; ++i) {
+         final byte byteValue = validityBuffer.getByte(i);
+         /* Java uses two's complement binary representation, hence 11111111_b which is -1
+          * when converted to Int will have 32bits set to 1. Masking the MSB and then
+          * adding it back solves the issue.
+          */
+         count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
+      }
+      int nullCount = (sizeInBytes * 8) - count;
+      /* if the valueCount is not a multiple of 8,
+       * the bits on the right were counted as null bits.
+       */
+      int remainder = valueCount % 8;
+      nullCount -= remainder == 0 ? 0 : 8 - remainder;
+      return nullCount;
+   }
+
+   public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) {
+      return (byte)((data.getByte(index) & 0xFF) >>> offset);
+   }
+
+   public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
+      return (byte)((data.getByte(index) << (8 - offset)));
+   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
new file mode 100644
index 0000000000000..ee40d708cc199
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.BigIntReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.BigIntHolder;
+import org.apache.arrow.vector.holders.NullableBigIntHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableBigIntVector implements a fixed width vector (8 bytes) of
+ * integer values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableBigIntVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 8;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableBigIntVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableBigIntVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableBigIntVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new BigIntReaderImpl(NullableBigIntVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.BIGINT;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public long get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableBigIntHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Long getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, long value) {
+      valueBuffer.setLong(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, long value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, BigIntHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, long)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, long value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, BigIntHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, BigIntHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, long value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, long)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, long value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, long value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setLong(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static long get(final ArrowBuf buffer, final int index) {
+      return buffer.getLong(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableBigIntVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableBigIntVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableBigIntVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableBigIntVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableBigIntVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
new file mode 100644
index 0000000000000..eceaa61b57f41
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
@@ -0,0 +1,502 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.BitReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.BitHolder;
+import org.apache.arrow.vector.holders.NullableBitHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.TransferPair;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * NullableBitVector implements a fixed width (1 bit) vector of
+ * boolean values which could be null. Each value in the vector corresponds
+ * to a single bit in the underlying data stream backing the vector.
+ */
+public class NullableBitVector extends BaseNullableFixedWidthVector {
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableBitVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableBitVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.BIT.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableBitVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, (byte)0);
+      reader = new BitReaderImpl(NullableBitVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.BIT;
+   }
+
+   /**
+    * Sets the desired value capacity for the vector. This function doesn't
+    * allocate any memory for the vector.
+    * @param valueCount desired number of elements in the vector
+    */
+   @Override
+   public void setInitialCapacity(int valueCount) {
+      final int size = getValidityBufferSizeFromCount(valueCount);
+      if (size > MAX_ALLOCATION_SIZE) {
+         throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+      }
+      valueAllocationSizeInBytes = size;
+      validityAllocationSizeInBytes = size;
+   }
+
+   /**
+    * Get the current value capacity for the vector
+    * @return number of elements that vector can hold.
+    */
+   @Override
+   public int getValueCapacity(){
+      return (int)(validityBuffer.capacity() * 8L);
+   }
+
+   /**
+    * Get the potential buffer size for a particular number of records.
+    * @param count desired number of elements in the vector
+    * @return estimated size of underlying buffers if the vector holds
+    *         a given number of elements
+    */
+   @Override
+   public int getBufferSizeFor(final int count) {
+      if (count == 0) { return 0; }
+      return 2 * getValidityBufferSizeFromCount(count);
+   }
+
+   /**
+    * Get the size (number of bytes) of underlying buffers used by this
+    * vector
+    * @return size of underlying buffers.
+    */
+   @Override
+   public int getBufferSize() {
+     return getBufferSizeFor(valueCount);
+   }
+
+   /**
+    * Slice this vector at desired index and length and transfer the
+    * corresponding data to the target vector.
+    * @param startIndex start position of the split in source vector.
+    * @param length length of the split.
+    * @param target destination vector
+    */
+   public void splitAndTransferTo(int startIndex, int length,
+                                  BaseNullableFixedWidthVector target) {
+      compareTypes(target, "splitAndTransferTo");
+      target.clear();
+      target.validityBuffer = splitAndTransferBuffer(startIndex, length, target,
+                                 validityBuffer, target.validityBuffer);
+      target.valueBuffer = splitAndTransferBuffer(startIndex, length, target,
+                                 valueBuffer, target.valueBuffer);
+
+      target.setValueCount(length);
+   }
+
+   private ArrowBuf splitAndTransferBuffer(int startIndex, int length,
+                                           BaseNullableFixedWidthVector target,
+                                           ArrowBuf sourceBuffer, ArrowBuf destBuffer) {
+      assert startIndex + length <= valueCount;
+      int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+      int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+      int byteSizeTarget = getValidityBufferSizeFromCount(length);
+      int offset = startIndex % 8;
+
+      if (length > 0) {
+         if (offset == 0) {
+            /* slice */
+            if (destBuffer != null) {
+               destBuffer.release();
+            }
+            destBuffer = destBuffer.slice(firstByteSource, byteSizeTarget);
+            destBuffer.retain(1);
+         }
+         else {
+            /* Copy data
+             * When the first bit starts from the middle of a byte (offset != 0),
+             * copy data from src BitVector.
+             * Each byte in the target is composed by a part in i-th byte,
+             * another part in (i+1)-th byte.
+             */
+            destBuffer = allocator.buffer(byteSizeTarget);
+            destBuffer.readerIndex(0);
+            destBuffer.setZero(0, destBuffer.capacity());
+
+            for (int i = 0; i < byteSizeTarget - 1; i++) {
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset);
+               byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset);
+
+               destBuffer.setByte(i, (b1 + b2));
+            }
+
+            /* Copying the last piece is done in the following manner:
+             * if the source vector has 1 or more bytes remaining, we copy
+             * the last piece as a byte formed by shifting data
+             * from the current byte and the next byte.
+             *
+             * if the source vector has no more bytes remaining
+             * (we are at the last byte), we copy the last piece as a byte
+             * by shifting data from the current byte.
+             */
+            if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
+                       firstByteSource + byteSizeTarget - 1, offset);
+               byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer,
+                       firstByteSource + byteSizeTarget, offset);
+
+               destBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+            }
+            else {
+               byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
+                       firstByteSource + byteSizeTarget - 1, offset);
+               destBuffer.setByte(byteSizeTarget - 1, b1);
+            }
+         }
+      }
+
+      return destBuffer;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+   private int getBit(int index) {
+      final int byteIndex = index >> 3;
+      final byte b = valueBuffer.getByte(byteIndex);
+      final int bitIndex = index & 7;
+      return Long.bitCount(b & (1L << bitIndex));
+   }
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public int get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return getBit(index);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableBitHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = getBit(index);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Boolean getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return new Boolean (getBit(index) != 0);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableBitVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      if (value != 0) {
+         BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+      } else {
+         BitVectorHelper.setValidityBit(valueBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableBitHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         if (holder.value != 0) {
+            BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+         } else {
+            BitVectorHelper.setValidityBit(valueBuffer, index, 0);
+         }
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, BitHolder holder) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      if (holder.value != 0) {
+         BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+      } else {
+         BitVectorHelper.setValidityBit(valueBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableBitHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, BitHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, BitHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, int value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, int)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, int value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableBitVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableBitVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableBitVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableBitVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableBitVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
new file mode 100644
index 0000000000000..949287ecafe45
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.DateDayReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.DateDayHolder;
+import org.apache.arrow.vector.holders.NullableDateDayHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.slf4j.Logger;
+
+/**
+ * NullableDateDayVector implements a fixed width (4 bytes) vector of
+ * date values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableDateDayVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 4;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableDateDayVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableDateDayVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableDateDayVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new DateDayReaderImpl(NullableDateDayVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.DATEDAY;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public int get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableDateDayHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Integer getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setInt(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, DateDayHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, DateDayHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, DateDayHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, int value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, int)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, int value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader                                *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, int value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setInt(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static int get(final ArrowBuf buffer, final int index) {
+      return buffer.getInt(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableDateDayVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableDateDayVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableDateDayVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableDateDayVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableDateDayVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
new file mode 100644
index 0000000000000..a0bdccedea405
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
@@ -0,0 +1,408 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.DateMilliReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.DateMilliHolder;
+import org.apache.arrow.vector.holders.NullableDateMilliHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.joda.time.LocalDateTime;
+import org.joda.time.LocalDateTimes;
+import org.slf4j.Logger;
+
+/**
+ * NullableDateMilliVector implements a fixed width vector (8 bytes) of
+ * date values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableDateMilliVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 8;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableDateMilliVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new DateMilliReaderImpl(NullableDateMilliVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.DATEMILLI;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public long get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableDateMilliHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public LocalDateTime getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         final long millis = get(index);
+         final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
+         return localDateTime;
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, long value) {
+      valueBuffer.setLong(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, long value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, DateMilliHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, long)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, long value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, DateMilliHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, DateMilliHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, long value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, long)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, long value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader                                *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, long value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setLong(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static long get(final ArrowBuf buffer, final int index) {
+      return buffer.getLong(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableDateMilliVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableDateMilliVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableDateMilliVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableDateMilliVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableDateMilliVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
new file mode 100644
index 0000000000000..8320f90830a5b
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
@@ -0,0 +1,454 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.DecimalReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.DecimalHolder;
+import org.apache.arrow.vector.holders.NullableDecimalHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.DecimalUtility;
+import org.apache.arrow.vector.util.TransferPair;
+
+import java.math.BigDecimal;
+
+/**
+ * NullableDecimalVector implements a fixed width vector (16 bytes) of
+ * decimal values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableDecimalVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 16;
+   private final FieldReader reader;
+
+   private final int precision;
+   private final int scale;
+
+   /**
+    * Instantiate a NullableDecimalVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableDecimalVector(String name, BufferAllocator allocator,
+                                int precision, int scale) {
+      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableDecimalVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal)fieldType.getType();
+      reader = new DecimalReaderImpl(NullableDecimalVector.this);
+      this.precision = arrowType.getPrecision();
+      this.scale = arrowType.getScale();
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.DECIMAL;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public ArrowBuf get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableDecimalHolder holder) {
+      if (isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.buffer = valueBuffer;
+      holder.precision = precision;
+      holder.scale = scale;
+      holder.start = index * TYPE_WIDTH;
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public BigDecimal getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer,
+                 thisIndex * TYPE_WIDTH, TYPE_WIDTH);
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index    position of element
+    * @param buffer   ArrowBuf containing decimal value.
+    */
+   public void set(int index, ArrowBuf buffer) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index    position of element
+    * @param start    start index of data in the buffer
+    * @param buffer   ArrowBuf containing decimal value.
+    */
+   public void set(int index, int start, ArrowBuf buffer) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   BigDecimal containing decimal value.
+    */
+   public void set(int index, BigDecimal value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      DecimalUtility.checkPrecisionAndScale(value, precision, scale);
+      DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, DecimalHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #set(int, ArrowBuf)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param buffer  ArrowBuf containing decimal value.
+    */
+   public void setSafe(int index, ArrowBuf buffer) {
+      handleSafe(index);
+      set(index, buffer);
+   }
+
+   /**
+    * Same as {@link #set(int, int, ArrowBuf)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index    position of element
+    * @param start    start index of data in the buffer
+    * @param buffer   ArrowBuf containing decimal value.
+    */
+   public void setSafe(int index, int start, ArrowBuf buffer) {
+      handleSafe(index);
+      set(index, start, buffer);
+   }
+
+   /**
+    * Same as {@link #set(int, BigDecimal)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   BigDecimal containing decimal value.
+    */
+   public void setSafe(int index, BigDecimal value){
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, DecimalHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, DecimalHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param start start position of the value in the buffer
+    * @param buffer buffer containing the value to be stored in the vector
+    */
+   public void set(int index, int isSet, int start, ArrowBuf buffer) {
+      if (isSet > 0) {
+         set(index, start, buffer);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles
+    * the case when the position of new value is beyond the current value
+    * capacity of the vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param start start position of the value in the buffer
+    * @param buffer buffer containing the value to be stored in the vector
+    */
+   public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
+      handleSafe(index);
+      set(index, isSet, start, buffer);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader                                *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value as array of bytes
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, byte[] value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      DecimalUtility.writeByteArrayToArrowBuf(value, buffer, index);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableDecimalVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableDecimalVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision,
+                 NullableDecimalVector.this.scale);
+      }
+
+      public TransferImpl(NullableDecimalVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableDecimalVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
new file mode 100644
index 0000000000000..5b28065c402c4
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.Float4ReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.Float4Holder;
+import org.apache.arrow.vector.holders.NullableFloat4Holder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableFloat4Vector implements a fixed width vector (4 bytes) of
+ * float values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableFloat4Vector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 4;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableFloat4Vector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new Float4ReaderImpl(NullableFloat4Vector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.FLOAT4;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public float get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getFloat(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableFloat4Holder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getFloat(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Float getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, float value) {
+      valueBuffer.setFloat(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, float value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, Float4Holder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, float)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, float value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, Float4Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, Float4Holder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, float value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, float)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, float value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, float value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setFloat(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static float get(final ArrowBuf buffer, final int index) {
+      return buffer.getFloat(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableFloat4Vector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableFloat4Vector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableFloat4Vector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableFloat4Vector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableFloat4Vector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
new file mode 100644
index 0000000000000..624abf2f272cd
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.Float8ReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.Float8Holder;
+import org.apache.arrow.vector.holders.NullableFloat8Holder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableFloat8Vector implements a fixed width vector (8 bytes) of
+ * double values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableFloat8Vector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 8;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableFloat8Vector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new Float8ReaderImpl(NullableFloat8Vector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.FLOAT8;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public double get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getDouble(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableFloat8Holder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getDouble(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Double getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, double value) {
+      valueBuffer.setDouble(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, double value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, Float8Holder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, double)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, double value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, Float8Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, Float8Holder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, double value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, double)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, double value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, double value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setDouble(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static double get(final ArrowBuf buffer, final int index) {
+      return buffer.getDouble(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableFloat8Vector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableFloat8Vector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableFloat8Vector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableFloat8Vector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableFloat8Vector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
index 26b19fa7bd562..6311daf4f5cc1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
@@ -1,5 +1,4 @@
-/*******************************************************************************
-
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,10 +14,11 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- ******************************************************************************/
+ */
 
 package org.apache.arrow.vector;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.impl.IntReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
@@ -27,39 +27,53 @@
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.TransferPair;
-import org.slf4j.Logger;
 
 /**
- * NullableIntVector implements a fixed width vector of values which could
- * be null. A validity buffer (bit vector) is maintained to track which
- * elements in the vector are null.
+ * NullableIntVector implements a fixed width (4 bytes) vector of
+ * integer values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
  */
 public class NullableIntVector extends BaseNullableFixedWidthVector {
-   private static final org.slf4j.Logger logger =
-           org.slf4j.LoggerFactory.getLogger(NullableIntVector.class);
    private static final byte TYPE_WIDTH = 4;
    private final FieldReader reader;
 
+   /**
+    * Instantiate a NullableIntVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
    public NullableIntVector(String name, BufferAllocator allocator) {
       this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()),
               allocator);
    }
 
+   /**
+    * Instantiate a NullableIntVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
    public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
       super(name, allocator, fieldType, TYPE_WIDTH);
       reader = new IntReaderImpl(NullableIntVector.this);
    }
 
-   @Override
-   protected org.slf4j.Logger getLogger() {
-      return logger;
-   }
-
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
    @Override
    public FieldReader getReader(){
       return reader;
    }
 
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
    @Override
    public Types.MinorType getMinorType() {
       return Types.MinorType.INT;
@@ -116,12 +130,27 @@ public Integer getObject(int index) {
       }
    }
 
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
    public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) {
       if (from.isSet(fromIndex) != 0) {
          set(thisIndex, from.get(fromIndex));
       }
    }
 
+   /**
+    * Same as {@link #copyFrom(int, int, NullableIntVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
    public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) {
       handleSafe(thisIndex);
       copyFrom(fromIndex, thisIndex, from);
@@ -234,17 +263,80 @@ public void setNull(int index){
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
    }
 
-   public void set(int index, int isSet, int valueField ) {
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, int value) {
       if (isSet > 0) {
-         set(index, valueField);
+         set(index, value);
       } else {
          BitVectorHelper.setValidityBit(validityBuffer, index, 0);
       }
    }
 
-   public void setSafe(int index, int isSet, int valueField ) {
+   /**
+    * Same as {@link #set(int, int, int)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, int value) {
       handleSafe(index);
-      set(index, isSet, valueField);
+      set(index, isSet, value);
+   }
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, int value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setInt(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static int get(final ArrowBuf buffer, final int index) {
+      return buffer.getInt(index * TYPE_WIDTH);
    }
 
 
@@ -255,11 +347,23 @@ public void setSafe(int index, int isSet, int valueField ) {
     ******************************************************************/
 
 
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
    @Override
    public TransferPair getTransferPair(String ref, BufferAllocator allocator){
       return new TransferImpl(ref, allocator);
    }
 
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
    @Override
    public TransferPair makeTransferPair(ValueVector to) {
       return new TransferImpl((NullableIntVector)to);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
new file mode 100644
index 0000000000000..174b722b04316
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
@@ -0,0 +1,429 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.IntervalDayReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.IntervalDayHolder;
+import org.apache.arrow.vector.holders.NullableIntervalDayHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.joda.time.Period;
+
+/**
+ * NullableIntervalDayVector implements a fixed width vector (8 bytes) of
+ * interval (days and milliseconds) values which could be null.
+ * A validity buffer (bit vector) is maintained to track which elements in the
+ * vector are null.
+ */
+public class NullableIntervalDayVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 8;
+   private static final byte MILLISECOND_OFFSET = 4;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableIntervalDayVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.INTERVALDAY;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public ArrowBuf get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         return null;
+      }
+      return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableIntervalDayHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      final int startIndex = index * TYPE_WIDTH;
+      holder.isSet = 1;
+      holder.days = valueBuffer.getInt(startIndex);
+      holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Period getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         final int startIndex = index * TYPE_WIDTH;
+         final int days = valueBuffer.getInt(startIndex);
+         final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
+         final Period p = new Period();
+         return p.plusDays(days).plusMillis(milliseconds);
+      }
+   }
+
+   /**
+    * Get the Interval value at a given index as a {@link StringBuilder} object
+    * @param index position of the element
+    * @return String Builder object with Interval value as
+    *         [days, hours, minutes, seconds, millis]
+    */
+   public StringBuilder getAsStringBuilder(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      }else{
+         return getAsStringBuilderHelper(index);
+      }
+   }
+
+   private StringBuilder getAsStringBuilderHelper(int index) {
+      final int startIndex = index * TYPE_WIDTH;
+
+      final int  days = valueBuffer.getInt(startIndex);
+      int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
+
+      final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
+      millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
+
+      final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
+      millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
+
+      final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
+      millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
+
+      final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
+
+      return(new StringBuilder().
+              append(days).append(dayString).
+              append(hours).append(":").
+              append(minutes).append(":").
+              append(seconds).append(".").
+              append(millis));
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex);
+         from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer,
+                 thisIndex * TYPE_WIDTH, TYPE_WIDTH);
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, ArrowBuf value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index          position of element
+    * @param days           days for the interval
+    * @param milliseconds   milliseconds for the interval
+    */
+   public void set(int index, int days, int milliseconds){
+      final int offsetIndex = index * TYPE_WIDTH;
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      valueBuffer.setInt(offsetIndex, days);
+      valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         set(index, holder.days, holder.milliseconds);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, IntervalDayHolder holder){
+      set(index, holder.days, holder.milliseconds);
+   }
+
+   /**
+    * Same as {@link #set(int, ArrowBuf)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, ArrowBuf value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index          position of element
+    * @param days           days for the interval
+    * @param milliseconds   milliseconds for the interval
+    */
+   public void setSafe(int index, int days, int milliseconds) {
+      handleSafe(index);
+      set(index, days, milliseconds);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, IntervalDayHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, IntervalDayHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param days days component of interval
+    * @param milliseconds millisecond component of interval
+    */
+   public void set(int index, int isSet, int days, int milliseconds) {
+      if (isSet > 0) {
+         set(index, days, milliseconds);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, int, int)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param days days component of interval
+    * @param milliseconds millisecond component of interval
+    */
+   public void setSafe(int index, int isSet, int days, int milliseconds) {
+      handleSafe(index);
+      set(index, isSet, days, milliseconds);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableIntervalDayVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableIntervalDayVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableIntervalDayVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableIntervalDayVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
new file mode 100644
index 0000000000000..303286d0ed30c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
@@ -0,0 +1,387 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.IntervalYearReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.IntervalYearHolder;
+import org.apache.arrow.vector.holders.NullableIntervalYearHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.joda.time.Period;
+
+/**
+ * NullableIntervalYearVector implements a fixed width (4 bytes) vector of
+ * interval (years and months) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableIntervalYearVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 4;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableIntervalYearVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.INTERVALYEAR;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public int get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableIntervalYearHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Period getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         final int interval = get(index);
+         final int years  = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+         final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+         final Period p = new Period();
+         return p.plusYears(years).plusMonths(months);
+      }
+   }
+
+   /**
+    * Get the Interval value at a given index as a {@link StringBuilder} object
+    * @param index position of the element
+    * @return String Builder object with Interval value as
+    *         [years, months]
+    */
+   public StringBuilder getAsStringBuilder(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      }else{
+         return getAsStringBuilderHelper(index);
+      }
+   }
+
+   private StringBuilder getAsStringBuilderHelper(int index) {
+      int value  = valueBuffer.getInt(index * TYPE_WIDTH);
+
+      final int years  = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+      final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+
+      final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
+      final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
+
+      return(new StringBuilder().
+              append(years).append(yearString).
+              append(months).append(monthString));
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setInt(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, IntervalYearHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, IntervalYearHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, IntervalYearHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, int value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, int)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, int value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableIntervalYearVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableIntervalYearVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableIntervalYearVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableIntervalYearVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
new file mode 100644
index 0000000000000..c45a8d5f5bcba
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
@@ -0,0 +1,431 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.SmallIntReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.SmallIntHolder;
+import org.apache.arrow.vector.holders.NullableSmallIntHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableSmallIntVector implements a fixed width (2 bytes) vector of
+ * short values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableSmallIntVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 2;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableSmallIntVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new SmallIntReaderImpl(NullableSmallIntVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.SMALLINT;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public short get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getShort(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableSmallIntHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getShort(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Short getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setShort(index * TYPE_WIDTH, value);
+   }
+
+   private void setValue(int index, short value) {
+      valueBuffer.setShort(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, short value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, SmallIntHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, short)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, short value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, SmallIntHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, SmallIntHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, short value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, short)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, short value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, short value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setShort(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static short get(final ArrowBuf buffer, final int index) {
+      return buffer.getShort(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableSmallIntVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableSmallIntVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableSmallIntVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableSmallIntVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableSmallIntVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
new file mode 100644
index 0000000000000..454a4ac41a6d9
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeMicroReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeMicroHolder;
+import org.apache.arrow.vector.holders.NullableTimeMicroHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.slf4j.Logger;
+
+/**
+ * NullableTimeMicroVector implements a fixed width vector (8 bytes) of
+ * time (microsecond resolution) values which could be null.
+ * A validity buffer (bit vector) is maintained to track which elements in the
+ * vector are null.
+ */
+public class NullableTimeMicroVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 8;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeMicroVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMEMICRO;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public long get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeMicroHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Long getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, long value) {
+      valueBuffer.setLong(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, long value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeMicroHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, long)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, long value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeMicroHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeMicroHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, long value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, long)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, long value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, long value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setLong(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static long get(final ArrowBuf buffer, int index) {
+      return buffer.getLong(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeMicroVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableTimeMicroVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableTimeMicroVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableTimeMicroVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
new file mode 100644
index 0000000000000..8540d16924210
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
@@ -0,0 +1,405 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeMilliReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeMilliHolder;
+import org.apache.arrow.vector.holders.NullableTimeMilliHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.joda.time.LocalDateTime;
+import org.slf4j.Logger;
+
+/**
+ * NullableTimeMilliVector implements a fixed width (4 bytes) vector of
+ * time (millisecond resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeMilliVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 4;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeMilliVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMEMILLI;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public int get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeMilliHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public LocalDateTime getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      }
+      org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index),
+              org.joda.time.DateTimeZone.UTC);
+      return ldt;
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setInt(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeMilliHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeMilliHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeMilliHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, int value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, int)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, int value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, int value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setInt(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static int get(final ArrowBuf buffer, final int index) {
+      return buffer.getInt(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeMilliVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableTimeMilliVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableTimeMilliVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableTimeMilliVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
new file mode 100644
index 0000000000000..015226da22318
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
@@ -0,0 +1,402 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeNanoReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeNanoHolder;
+import org.apache.arrow.vector.holders.NullableTimeNanoHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableTimeNanoVector implements a fixed width vector (8 bytes) of
+ * time (nanosecond resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeNanoVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 8;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeNanoVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMENANO;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public long get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeNanoHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Long getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, long value) {
+      valueBuffer.setLong(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, long value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeNanoHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, long)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, long value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeNanoHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeNanoHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, long value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, long)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, long value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, long value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setLong(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static long get(final ArrowBuf buffer,final int index) {
+      return buffer.getLong(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeNanoVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableTimeNanoVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableTimeNanoVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableTimeNanoVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
new file mode 100644
index 0000000000000..2b2375e92f1f4
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeSecReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeSecHolder;
+import org.apache.arrow.vector.holders.NullableTimeSecHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableTimeSecVector implements a fixed width (4 bytes) vector of
+ * time (seconds resolution) values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableTimeSecVector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 4;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeSecVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new TimeSecReaderImpl(NullableTimeSecVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESEC;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public int get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeSecHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Integer getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setInt(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeSecHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeSecHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeSecHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, int value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, int)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, int value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, int value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setInt(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static int get(final ArrowBuf buffer, final int index) {
+      return buffer.getInt(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeSecVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableTimeSecVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableTimeSecVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableTimeSecVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableTimeSecVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java
new file mode 100644
index 0000000000000..0d25eff4e02b7
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeStampMicroTZReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeStampMicroTZHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampMicroTZHolder;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableTimeStampMicroTZVector implements a fixed width vector (8 bytes) of
+ * timestamp (microsecond resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeStampMicroTZVector extends NullableTimeStampVector {
+   private final FieldReader reader;
+   private final String timeZone;
+
+   /**
+    * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) {
+      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, fieldType, allocator);
+      org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType();
+      timeZone = arrowType.getTimezone();
+      reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESTAMPMICROTZ;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeStampMicroTZHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Long getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeStampMicroTZHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeStampMicroTZHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref,
+              field.getFieldType(), allocator);
+      return new TransferImpl(to);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeStampMicroTZVector)to);
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java
new file mode 100644
index 0000000000000..41f7a1112f2df
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeStampMicroReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeStampMicroHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampMicroHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.joda.time.LocalDateTime;
+
+/**
+ * NullableTimeStampMicroVector implements a fixed width vector (8 bytes) of
+ * timestamp (microsecond resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeStampMicroVector extends NullableTimeStampVector {
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampMicroVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, fieldType, allocator);
+      reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESTAMPMICRO;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeStampMicroHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public LocalDateTime getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */
+         final long micros = get(index);
+         final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros);
+         final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
+         return localDateTime;
+      }
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeStampMicroHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeStampMicroHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref,
+              field.getFieldType(), allocator);
+      return new TransferImpl(to);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeStampMicroVector )to);
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java
new file mode 100644
index 0000000000000..6376b085e64e1
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeStampMilliTZReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeStampMilliTZHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableTimeStampMilliTZVector implements a fixed width vector (8 bytes) of
+ * timestamp (millisecond resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeStampMilliTZVector extends NullableTimeStampVector {
+   private final FieldReader reader;
+   private final String timeZone;
+
+   /**
+    * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) {
+      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, fieldType, allocator);
+      org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType();
+      timeZone = arrowType.getTimezone();
+      reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESTAMPMILLITZ;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeStampMilliTZHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Long getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeStampMilliTZHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeStampMilliTZHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref,
+              field.getFieldType(), allocator);
+      return new TransferImpl(to);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeStampMilliTZVector)to);
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java
new file mode 100644
index 0000000000000..ff581d2ea2f03
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeStampMilliReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeStampMilliHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.joda.time.LocalDateTime;
+
+/**
+ * NullableTimeStampMilliVector implements a fixed width vector (8 bytes) of
+ * timestamp (millisecond resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeStampMilliVector extends NullableTimeStampVector {
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampMilliVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, fieldType, allocator);
+      reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESTAMPMILLI;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeStampMilliHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public LocalDateTime getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         final long millis = get(index);
+         final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
+         return localDateTime;
+      }
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeStampMilliHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeStampMilliHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref,
+              field.getFieldType(), allocator);
+      return new TransferImpl(to);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeStampMilliVector)to);
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java
new file mode 100644
index 0000000000000..6c4445d500b39
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeStampNanoTZReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeStampNanoTZHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableTimeStampNanoTZVector implements a fixed width vector (8 bytes) of
+ * timestamp (nanosecond resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeStampNanoTZVector extends NullableTimeStampVector {
+   private final FieldReader reader;
+   private final String timeZone;
+
+   /**
+    * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) {
+      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, fieldType, allocator);
+      org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType();
+      timeZone = arrowType.getTimezone();
+      reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESTAMPNANOTZ;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeStampNanoTZHolder holder) {
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Long getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeStampNanoTZHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeStampNanoTZHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref,
+              field.getFieldType(), allocator);
+      return new TransferImpl(to);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeStampNanoTZVector)to);
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java
new file mode 100644
index 0000000000000..096913ebf1cf0
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeStampNanoReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeStampNanoHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampNanoHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.joda.time.LocalDateTime;
+
+/**
+ * NullableTimeStampNanoVector implements a fixed width vector (8 bytes) of
+ * timestamp (nanosecond resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeStampNanoVector extends NullableTimeStampVector {
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampNanoVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, fieldType, allocator);
+      reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESTAMPNANO;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeStampNanoHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public LocalDateTime getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         final long nanos = get(index);
+         final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos);
+         final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
+         return localDateTime;
+      }
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeStampNanoHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeStampNanoHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref,
+              field.getFieldType(), allocator);
+      return new TransferImpl(to);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeStampNanoVector)to);
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java
new file mode 100644
index 0000000000000..8e5ba72ae1fe2
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeStampSecTZReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeStampSecTZHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampSecTZHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableTimeStampSecTZVector implements a fixed width vector (8 bytes) of
+ * timestamp (seconds resolution) values which could be null. A validity buffer
+ * (bit vector) is maintained to track which elements in the vector are null.
+ */
+public class NullableTimeStampSecTZVector extends NullableTimeStampVector {
+   private final FieldReader reader;
+   private final String timeZone;
+
+   /**
+    * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) {
+      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, fieldType, allocator);
+      org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType();
+      timeZone = arrowType.getTimezone();
+      reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESTAMPSECTZ;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeStampSecTZHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Long getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeStampSecTZHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeStampSecTZHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref,
+              field.getFieldType(), allocator);
+      return new TransferImpl(to);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeStampSecTZVector)to);
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java
new file mode 100644
index 0000000000000..bc800e0e4f04f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TimeStampSecReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TimeStampSecHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampSecHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.joda.time.LocalDateTime;
+
+/**
+ * NullableTimeStampSecVector implements a fixed width vector (8 bytes) of
+ * timestamp (seconds resolution) values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableTimeStampSecVector extends NullableTimeStampVector {
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampSecVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, fieldType, allocator);
+      reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader() {
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TIMESTAMPSEC;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTimeStampSecHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public LocalDateTime getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         final long secs = get(index);
+         final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs);
+         final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
+         return localDateTime;
+      }
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TimeStampSecHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TimeStampSecHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref,
+              field.getFieldType(), allocator);
+      return new TransferImpl(to);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTimeStampSecVector)to);
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
new file mode 100644
index 0000000000000..b2a58bd4568f9
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableTimeStampVector is an abstract interface for fixed width vector (8 bytes)
+ * of timestamp values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector {
+   protected static final byte TYPE_WIDTH = 8;
+
+   /**
+    * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public long get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   protected void setValue(int index, long value) {
+      valueBuffer.setLong(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, long value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, long)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, long value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, long value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, long)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, long value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and                            *
+    *          JsonFileWriter                                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, long value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setLong(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static long get(final ArrowBuf buffer, final int index) {
+      return buffer.getLong(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   public class TransferImpl implements TransferPair {
+      NullableTimeStampVector to;
+
+      public TransferImpl(NullableTimeStampVector to) {
+         this.to = to;
+      }
+
+      @Override
+      public NullableTimeStampVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer() {
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
new file mode 100644
index 0000000000000..15100530d0ec5
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
@@ -0,0 +1,432 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.TinyIntReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.TinyIntHolder;
+import org.apache.arrow.vector.holders.NullableTinyIntHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableTinyIntVector implements a fixed width (1 bytes) vector of
+ * byte values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableTinyIntVector extends BaseNullableFixedWidthVector {
+   public static final byte TYPE_WIDTH = 1;
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTinyIntVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()),
+              allocator);
+   }
+
+   /**
+    * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new TinyIntReaderImpl(NullableTinyIntVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.TINYINT;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public byte get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getByte(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableTinyIntHolder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getByte(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Byte getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+   private void setValue(int index, int value) {
+      valueBuffer.setByte(index * TYPE_WIDTH, value);
+   }
+
+   private void setValue(int index, byte value) {
+      valueBuffer.setByte(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, byte value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, TinyIntHolder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, byte)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, byte value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, TinyIntHolder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, TinyIntHolder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void set(int index, int isSet, byte value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Same as {@link #set(int, int, byte)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param value element value
+    */
+   public void setSafe(int index, int isSet, byte value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+
+   /******************************************************************
+    *                                                                *
+    *          helper routines currently                             *
+    *          used in JsonFileReader and JsonFileWriter             *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Given a data buffer, this method sets the element value at a particular
+    * position. Reallocates the buffer if needed.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param allocator allocator
+    * @param valueCount number of elements in the vector
+    * @param index position of the new element
+    * @param value element value
+    * @return data buffer
+    */
+   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                              int valueCount, int index, byte value) {
+      if (buffer == null) {
+         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
+      }
+      buffer.setByte(index * TYPE_WIDTH, value);
+      if (index == (valueCount - 1)) {
+         buffer.writerIndex(valueCount * TYPE_WIDTH);
+      }
+
+      return buffer;
+   }
+
+   /**
+    * Given a data buffer, get the value stored at a particular position
+    * in the vector.
+    *
+    * This method should not be used externally.
+    *
+    * @param buffer data buffer
+    * @param index position of the element.
+    * @return value stored at the index.
+    */
+   public static byte get(final ArrowBuf buffer, final int index) {
+      return buffer.getByte(index * TYPE_WIDTH);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableTinyIntVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableTinyIntVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableTinyIntVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableTinyIntVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableTinyIntVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
new file mode 100644
index 0000000000000..4d20f800ba1fd
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
@@ -0,0 +1,318 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.UInt1ReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.UInt1Holder;
+import org.apache.arrow.vector.holders.NullableUInt1Holder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * NullableUInt1Vector implements a fixed width (1 bytes) vector of
+ * integer values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableUInt1Vector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 1;
+   private final FieldReader reader;
+
+   public NullableUInt1Vector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()),
+              allocator);
+   }
+
+   public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new UInt1ReaderImpl(NullableUInt1Vector.this);
+   }
+
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.UINT1;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public byte get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getByte(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableUInt1Holder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getByte(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Byte getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setByte(index * TYPE_WIDTH, value);
+   }
+
+   private void setValue(int index, byte value) {
+      valueBuffer.setByte(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, byte value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, UInt1Holder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, byte)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, byte value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, UInt1Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, UInt1Holder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   public void set(int index, int isSet, byte value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   public void setSafe(int index, int isSet, byte value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableUInt1Vector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableUInt1Vector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableUInt1Vector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableUInt1Vector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableUInt1Vector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
new file mode 100644
index 0000000000000..b140b9f5cbc67
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
@@ -0,0 +1,319 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.UInt2ReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.UInt2Holder;
+import org.apache.arrow.vector.holders.NullableUInt2Holder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.slf4j.Logger;
+
+/**
+ * NullableUInt2Vector implements a fixed width (2 bytes) vector of
+ * integer values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableUInt2Vector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 2;
+   private final FieldReader reader;
+
+   public NullableUInt2Vector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()),
+              allocator);
+   }
+
+   public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new UInt2ReaderImpl(NullableUInt2Vector.this);
+   }
+
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.UINT2;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public char get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getChar(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableUInt2Holder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getChar(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Character getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setChar(index * TYPE_WIDTH, value);
+   }
+
+   private void setValue(int index, char value) {
+      valueBuffer.setChar(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, char value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, UInt2Holder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, char)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, char value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, UInt2Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, UInt2Holder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   public void set(int index, int isSet, char value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   public void setSafe(int index, int isSet, char value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableUInt2Vector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableUInt2Vector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableUInt2Vector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableUInt2Vector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableUInt2Vector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
new file mode 100644
index 0000000000000..fc8cb639581fb
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.UInt4ReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.UInt4Holder;
+import org.apache.arrow.vector.holders.NullableUInt4Holder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.slf4j.Logger;
+
+/**
+ * NullableUInt4Vector implements a fixed width (4 bytes) vector of
+ * integer values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableUInt4Vector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 4;
+   private final FieldReader reader;
+
+   public NullableUInt4Vector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()),
+              allocator);
+   }
+
+   public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new UInt4ReaderImpl(NullableUInt4Vector.this);
+   }
+
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.UINT4;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public int get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableUInt4Holder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Integer getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, int value) {
+      valueBuffer.setInt(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, int value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, UInt4Holder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, int)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, int value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, UInt4Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, UInt4Holder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   public void set(int index, int isSet, int value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   public void setSafe(int index, int isSet, int value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableUInt4Vector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableUInt4Vector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableUInt4Vector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableUInt4Vector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableUInt4Vector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
new file mode 100644
index 0000000000000..2110cf03b9f05
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.UInt8ReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.UInt8Holder;
+import org.apache.arrow.vector.holders.NullableUInt8Holder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+import org.slf4j.Logger;
+
+/**
+ * NullableUInt8Vector implements a fixed width vector (8 bytes) of
+ * integer values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class NullableUInt8Vector extends BaseNullableFixedWidthVector {
+   private static final byte TYPE_WIDTH = 8;
+   private final FieldReader reader;
+
+   public NullableUInt8Vector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.UINT8.getType()),
+              allocator);
+   }
+
+   public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType, TYPE_WIDTH);
+      reader = new UInt8ReaderImpl(NullableUInt8Vector.this);
+   }
+
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.UINT8;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value retrieval methods                        *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the element at the given index from the vector.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public long get(int index) throws IllegalStateException {
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Get the element at the given index from the vector and
+    * sets the state in holder. If element at given index
+    * is null, holder.isSet will be zero.
+    *
+    * @param index   position of element
+    */
+   public void get(int index, NullableUInt8Holder holder){
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      holder.isSet = 1;
+      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+   }
+
+   /**
+    * Same as {@link #get(int)}.
+    *
+    * @param index   position of element
+    * @return element at given index
+    */
+   public Long getObject(int index) {
+      if (isSet(index) == 0) {
+         return null;
+      } else {
+         return get(index);
+      }
+   }
+
+   public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) {
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+      }
+   }
+
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) {
+      handleSafe(thisIndex);
+      copyFrom(fromIndex, thisIndex, from);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   private void setValue(int index, long value) {
+      valueBuffer.setLong(index * TYPE_WIDTH, value);
+   }
+
+   /**
+    * Set the element at the given index to the given value.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void set(int index, long value) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, value);
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    * If the value in holder is not indicated as set, element in the
+    * at the given index will be null.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException {
+      if(holder.isSet < 0) {
+         throw new IllegalArgumentException();
+      }
+      else if(holder.isSet > 0) {
+         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+         setValue(index, holder.value);
+      }
+      else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   /**
+    * Set the element at the given index to the value set in data holder.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void set(int index, UInt8Holder holder){
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setValue(index, holder.value);
+   }
+
+   /**
+    * Same as {@link #set(int, long)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param value   value of element
+    */
+   public void setSafe(int index, long value) {
+      handleSafe(index);
+      set(index, value);
+   }
+
+   /**
+    * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  nullable data holder for value of element
+    */
+   public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException {
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Same as {@link #set(int, UInt8Holder)} except that it handles the
+    * case when index is greater than or equal to existing
+    * value capacity {@link #getValueCapacity()}.
+    *
+    * @param index   position of element
+    * @param holder  data holder for value of element
+    */
+   public void setSafe(int index, UInt8Holder holder){
+      handleSafe(index);
+      set(index, holder);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   public void set(int index, int isSet, long value) {
+      if (isSet > 0) {
+         set(index, value);
+      } else {
+         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+      }
+   }
+
+   public void setSafe(int index, int isSet, long value) {
+      handleSafe(index);
+      set(index, isSet, value);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableUInt8Vector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableUInt8Vector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableUInt8Vector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableUInt8Vector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableUInt8Vector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this);
+      }
+   }
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java
new file mode 100644
index 0000000000000..330e06710fcdd
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java
@@ -0,0 +1,514 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.VarBinaryReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.VarBinaryHolder;
+import org.apache.arrow.vector.holders.NullableVarBinaryHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+import java.nio.ByteBuffer;
+
+/**
+ * NullableVarBinaryVector implements a variable width vector of binary
+ * values which could be NULL. A validity buffer (bit vector) is maintained
+ * to track which elements in the vector are null.
+ */
+public class NullableVarBinaryVector extends BaseNullableVariableWidthVector {
+   private final FieldReader reader;
+
+   /**
+    * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableVarBinaryVector(String name, BufferAllocator allocator) {
+      this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator);
+   }
+
+   /**
+    * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
+   public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) {
+      super(name, allocator, fieldType);
+      reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this);
+   }
+
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
+   @Override
+   public FieldReader getReader(){
+      return reader;
+   }
+
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
+   @Override
+   public Types.MinorType getMinorType() {
+      return Types.MinorType.VARBINARY;
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value getter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Get the variable length element at specified index as byte array.
+    *
+    * @param index   position of element to get
+    * @return array of bytes for non-null element, null otherwise
+    */
+   public byte[] get(int index) {
+      assert index >= 0;
+      if(isSet(index) == 0) {
+         throw new IllegalStateException("Value at index is null");
+      }
+      final int startOffset = getstartOffset(index);
+      final int dataLength =
+              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+      final byte[] result = new byte[dataLength];
+      valueBuffer.getBytes(startOffset, result, 0, dataLength);
+      return result;
+   }
+
+   /**
+    * Get the variable length element at specified index as Text.
+    *
+    * @param index   position of element to get
+    * @return byte array for non-null element, null otherwise
+    */
+   public byte[] getObject(int index) {
+      byte[] b;
+      try {
+         b = get(index);
+      } catch (IllegalStateException e) {
+         return null;
+      }
+      return b;
+   }
+
+   /**
+    * Get the variable length element at specified index as Text.
+    *
+    * @param index   position of element to get
+    * @return greater than 0 length for non-null element, 0 otherwise
+    */
+   public int getValueLength(int index) {
+      assert index >= 0;
+      if(isSet(index) == 0) {
+         return 0;
+      }
+      final int startOffset = getstartOffset(index);
+      final int dataLength =
+              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+      return dataLength;
+   }
+
+   /**
+    * Get the variable length element at specified index and sets the state
+    * in provided holder.
+    *
+    * @param index   position of element to get
+    * @param holder  data holder to be populated by this function
+    */
+   public void get(int index, NullableVarBinaryHolder holder){
+      assert index >= 0;
+      if(isSet(index) == 0) {
+         holder.isSet = 0;
+         return;
+      }
+      final int startOffset = getstartOffset(index);
+      final int dataLength =
+              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+      holder.isSet = 1;
+      holder.start = startOffset;
+      holder.end = dataLength;
+      holder.buffer = valueBuffer;
+   }
+
+
+
+   /******************************************************************
+    *                                                                *
+    *          vector value setter methods                           *
+    *                                                                *
+    ******************************************************************/
+
+
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) {
+      fillHoles(thisIndex);
+      if (from.isSet(fromIndex) != 0) {
+         set(thisIndex, from.get(fromIndex));
+         lastSet = thisIndex;
+      }
+   }
+
+   /**
+    * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
+   public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) {
+      fillEmpties(thisIndex);
+      if (from.isSet(fromIndex) != 0) {
+         setSafe(thisIndex, from.get(fromIndex));
+         lastSet = thisIndex;
+      }
+   }
+
+
+   /**
+    * Set the variable length element at the specified index to the supplied
+    * byte array. This is same as using {@link #set(int, byte[], int, int)}
+    * with start as 0 and length as value.length
+    *
+    * @param index   position of the element to set
+    * @param value   array of bytes to write
+    */
+   public void set(int index, byte[] value) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setBytes(index, value, 0, value.length);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, byte[])} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param value   array of bytes to write
+    */
+   public void setSafe(int index, byte[] value) {
+      assert index >= 0;
+      fillEmpties(index);
+      handleSafe(index, value.length);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setBytes(index, value, 0, value.length);
+      lastSet = index;
+   }
+
+   /**
+    * Set the variable length element at the specified index to the supplied
+    * byte array.
+    *
+    * @param index   position of the element to set
+    * @param value   array of bytes to write
+    * @param start   start index in array of bytes
+    * @param length  length of data in array of bytes
+    */
+   public void set(int index, byte[] value, int start, int length) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setBytes(index, value, start, length);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, byte[], int, int)} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param value   array of bytes to write
+    * @param start   start index in array of bytes
+    * @param length  length of data in array of bytes
+    */
+   public void setSafe(int index, byte[] value, int start, int length) {
+      assert index >= 0;
+      fillEmpties(index);
+      handleSafe(index, length);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      setBytes(index, value, start, length);
+      lastSet = index;
+   }
+
+   /**
+    * Set the variable length element at the specified index to the
+    * content in supplied ByteBuffer
+    *
+    * @param index   position of the element to set
+    * @param value   ByteBuffer with data
+    * @param start   start index in ByteBuffer
+    * @param length  length of data in ByteBuffer
+    */
+   public void set(int index, ByteBuffer value, int start, int length) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+      valueBuffer.setBytes(startOffset, value, start, length);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param value   ByteBuffer with data
+    * @param start   start index in ByteBuffer
+    * @param length  length of data in ByteBuffer
+    */
+   public void setSafe(int index, ByteBuffer value, int start, int length) {
+      assert index >= 0;
+      fillEmpties(index);
+      handleSafe(index, length);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+      valueBuffer.setBytes(startOffset, value, start, length);
+      lastSet = index;
+   }
+
+   /**
+    * Set the variable length element at the specified index to the data
+    * buffer supplied in the holder
+    *
+    * @param index   position of the element to set
+    * @param holder  holder that carries data buffer.
+    */
+   public void set(int index, VarBinaryHolder holder) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      final int dataLength = holder.end - holder.start;
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, VarBinaryHolder)} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param holder  holder that carries data buffer.
+    */
+   public void setSafe(int index, VarBinaryHolder holder) {
+      assert index >= 0;
+      final int dataLength = holder.end - holder.start;
+      fillEmpties(index);
+      handleSafe(index, dataLength);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+      lastSet = index;
+   }
+
+   /**
+    * Set the variable length element at the specified index to the data
+    * buffer supplied in the holder
+    *
+    * @param index   position of the element to set
+    * @param holder  holder that carries data buffer.
+    */
+   public void set(int index, NullableVarBinaryHolder holder) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
+      final int dataLength = holder.end - holder.start;
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the
+    * case where index and length of new element are beyond the existing
+    * capacity of the vector.
+    *
+    * @param index   position of the element to set
+    * @param holder  holder that carries data buffer.
+    */
+   public void setSafe(int index, NullableVarBinaryHolder holder) {
+      assert index >= 0;
+      final int dataLength = holder.end - holder.start;
+      fillEmpties(index);
+      handleSafe(index, dataLength);
+      BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+      lastSet = index;
+   }
+
+   /**
+    * Sets the value length for an element.
+    *
+    * @param index   position of the element to set
+    * @param length  length of the element
+    */
+   public void setValueLengthSafe(int index, int length) {
+      assert index >= 0;
+      handleSafe(index, length);
+      final int startOffset = getstartOffset(index);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+   }
+
+   /**
+    * Set the element at the given index to null.
+    *
+    * @param index   position of element
+    */
+   public void setNull(int index){
+      handleSafe(index, 0);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+   }
+
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param start start position of data in buffer
+    * @param end end position of data in buffer
+    * @param buffer data buffer containing the variable width element to be stored
+    *               in the vector
+    */
+   public void set(int index, int isSet, int start, int end, ArrowBuf buffer) {
+      assert index >= 0;
+      fillHoles(index);
+      BitVectorHelper.setValidityBit(validityBuffer, index, isSet);
+      final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end);
+      final ArrowBuf bb = buffer.slice(start, end);
+      valueBuffer.setBytes(startOffset, bb);
+      lastSet = index;
+   }
+
+   /**
+    * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param start start position of data in buffer
+    * @param end end position of data in buffer
+    * @param buffer data buffer containing the variable width element to be stored
+    *               in the vector
+    */
+   public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) {
+      assert index >= 0;
+      handleSafe(index, end);
+      set(index, isSet, start, end, buffer);
+   }
+
+
+   /******************************************************************
+    *                                                                *
+    *                      vector transfer                           *
+    *                                                                *
+    ******************************************************************/
+
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+      return new TransferImpl(ref, allocator);
+   }
+
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return new TransferImpl((NullableVarBinaryVector)to);
+   }
+
+   private class TransferImpl implements TransferPair {
+      NullableVarBinaryVector to;
+
+      public TransferImpl(String ref, BufferAllocator allocator){
+         to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator);
+      }
+
+      public TransferImpl(NullableVarBinaryVector to){
+         this.to = to;
+      }
+
+      @Override
+      public NullableVarBinaryVector getTo(){
+         return to;
+      }
+
+      @Override
+      public void transfer(){
+         transferTo(to);
+      }
+
+      @Override
+      public void splitAndTransfer(int startIndex, int length) {
+         splitAndTransferTo(startIndex, length, to);
+      }
+
+      @Override
+      public void copyValueSafe(int fromIndex, int toIndex) {
+         to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this);
+      }
+   }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
index b9021544b0e10..0b03164f0f385 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
@@ -1,5 +1,4 @@
-/*******************************************************************************
-
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +14,7 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- ******************************************************************************/
+ */
 
 package org.apache.arrow.vector;
 
@@ -32,30 +31,50 @@
 
 import java.nio.ByteBuffer;
 
+/**
+ * NullableVarCharVector implements a variable width vector of VARCHAR
+ * values which could be NULL. A validity buffer (bit vector) is maintained
+ * to track which elements in the vector are null.
+ */
 public class NullableVarCharVector extends BaseNullableVariableWidthVector {
-   private static final org.slf4j.Logger logger =
-           org.slf4j.LoggerFactory.getLogger(NullableIntVector.class);
    private final FieldReader reader;
 
+   /**
+    * Instantiate a NullableVarCharVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param allocator allocator for memory management.
+    */
    public NullableVarCharVector(String name, BufferAllocator allocator) {
       this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator);
    }
 
+   /**
+    * Instantiate a NullableVarCharVector. This doesn't allocate any memory for
+    * the data in vector.
+    * @param name name of the vector
+    * @param fieldType type of Field materialized by this vector
+    * @param allocator allocator for memory management.
+    */
    public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) {
       super(name, allocator, fieldType);
       reader = new VarCharReaderImpl(NullableVarCharVector.this);
    }
 
-   @Override
-   protected org.slf4j.Logger getLogger() {
-      return logger;
-   }
-
+   /**
+    * Get a reader that supports reading values from this vector
+    * @return Field Reader for this vector
+    */
    @Override
    public FieldReader getReader(){
       return reader;
    }
 
+   /**
+    * Get minor type for this vector. The vector holds values belonging
+    * to a particular type.
+    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+    */
    @Override
    public Types.MinorType getMinorType() {
       return Types.MinorType.VARCHAR;
@@ -154,7 +173,13 @@ public void get(int index, NullableVarCharHolder holder){
     ******************************************************************/
 
 
-
+   /**
+    * Copy a cell value from a particular index in source vector to a particular
+    * position in this vector
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
    public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) {
       fillHoles(thisIndex);
       if (from.isSet(fromIndex) != 0) {
@@ -163,6 +188,14 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) {
       }
    }
 
+   /**
+    * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that
+    * it handles the case when the capacity of the vector needs to be expanded
+    * before copy.
+    * @param fromIndex position to copy from in source vector
+    * @param thisIndex position to copy to in this vector
+    * @param from source vector
+    */
    public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) {
       fillEmpties(thisIndex);
       if (from.isSet(fromIndex) != 0) {
@@ -383,21 +416,42 @@ public void setNull(int index){
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
    }
 
-   public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) {
+   /**
+    * Store the given value at a particular position in the vector. isSet indicates
+    * whether the value is NULL or not.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param start start position of data in buffer
+    * @param end end position of data in buffer
+    * @param buffer data buffer containing the variable width element to be stored
+    *               in the vector
+    */
+   public void set(int index, int isSet, int start, int end, ArrowBuf buffer) {
       assert index >= 0;
       fillHoles(index);
       BitVectorHelper.setValidityBit(validityBuffer, index, isSet);
       final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + endField);
-      final ArrowBuf bb = bufferField.slice(startField, endField);
+      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end);
+      final ArrowBuf bb = buffer.slice(start, end);
       valueBuffer.setBytes(startOffset, bb);
       lastSet = index;
    }
 
-   public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) {
+   /**
+    * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case
+    * when index is greater than or equal to current value capacity of the
+    * vector.
+    * @param index position of the new value
+    * @param isSet 0 for NULL value, 1 otherwise
+    * @param start start position of data in buffer
+    * @param end end position of data in buffer
+    * @param buffer data buffer containing the variable width element to be stored
+    *               in the vector
+    */
+   public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) {
       assert index >= 0;
-      handleSafe(index, endField);
-      set(index, isSet, startField, endField, bufferField);
+      handleSafe(index, end);
+      set(index, isSet, start, end, buffer);
    }
 
 
@@ -407,11 +461,23 @@ public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf
     *                                                                *
     ******************************************************************/
 
+   /**
+    * Construct a TransferPair comprising of this and and a target vector of
+    * the same type.
+    * @param ref name of the target vector
+    * @param allocator allocator for the target vector
+    * @return {@link TransferPair}
+    */
    @Override
    public TransferPair getTransferPair(String ref, BufferAllocator allocator){
       return new TransferImpl(ref, allocator);
    }
 
+   /**
+    * Construct a TransferPair with a desired target vector of the same type.
+    * @param to target vector
+    * @return {@link TransferPair}
+    */
    @Override
    public TransferPair makeTransferPair(ValueVector to) {
       return new TransferImpl((NullableVarCharVector)to);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index e6048b4b517c9..dc74afaa2e49c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -256,9 +256,17 @@ interface Mutator {
    */
   public ArrowBuf getOffsetBuffer();
 
+  /* temporarily add these methods here until we remove other vectors
+   * (non-nullable) which are under ValueVector hierarchy and still
+   * use the mutator/accessor interfaces.
+   */
   public int getValueCount();
 
   public void setValueCount(int valueCount);
 
   public Object getObject(int index);
+
+  public int getNullCount();
+
+  public boolean isNull(int index);
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
index 0c8868cad55b5..196393fb9584a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -38,7 +38,7 @@ public class VectorSchemaRoot implements AutoCloseable {
   private final Map<String, FieldVector> fieldVectorsMap = new HashMap<>();
 
   public VectorSchemaRoot(FieldVector parent) {
-    this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getAccessor().getValueCount());
+    this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getValueCount());
   }
 
   public VectorSchemaRoot(List<Field> fields, List<FieldVector> fieldVectors, int rowCount) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 96243eb5876b0..288afc6bbec69 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -55,17 +55,7 @@ public ArrowRecordBatch getRecordBatch() {
   }
 
   private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
-    Accessor accessor = null;
-    if (vector instanceof NullableIntVector) {
-      nodes.add(new ArrowFieldNode(((NullableIntVector)vector).getValueCount(),
-                includeNullCount ? ((NullableIntVector)vector).getNullCount() : -1));
-    } else if (vector instanceof NullableVarCharVector) {
-      nodes.add(new ArrowFieldNode(((NullableVarCharVector)vector).getValueCount(),
-                includeNullCount ? ((NullableVarCharVector)vector).getNullCount() : -1));
-    } else {
-      accessor = vector.getAccessor();
-      nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1));
-    }
+    nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ? vector.getNullCount() : -1));
     List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
     List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
     if (fieldBuffers.size() != expectedBuffers.size()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index fcef02f4a1ba8..3cc93a2a34f43 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -268,4 +268,10 @@ public void setValueCount(int valueCount) { }
 
   @Override
   public Object getObject(int index) { return null; }
+
+  @Override
+  public int getNullCount() { return 0; }
+
+  @Override
+  public boolean isNull(int index) { return false; }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index 26b0f90581ffc..1b20b09d81b0b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -141,8 +141,8 @@ public <T extends FieldVector> T addOrGet(String childName, FieldType fieldType,
   }
 
   private boolean nullFilled(ValueVector vector) {
-    for (int r = 0; r < vector.getAccessor().getValueCount(); r++) {
-      if (!vector.getAccessor().isNull(r)) {
+    for (int r = 0; r < vector.getValueCount(); r++) {
+      if (!vector.isNull(r)) {
         return false;
       }
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 8e2877f892a64..3eab30a32d224 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -21,6 +21,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,6 +32,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
 import com.google.common.base.Preconditions;
@@ -41,12 +43,14 @@
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
   public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
-  public final static String OFFSETS_VECTOR_NAME = "$offsets$";
   public final static String DATA_VECTOR_NAME = "$data$";
 
-  protected final UInt4Vector offsets;
+  public final static byte OFFSET_WIDTH = 4;
+  protected ArrowBuf offsetBuffer;
   protected FieldVector vector;
   protected final CallBack callBack;
+  protected int valueCount;
+  protected int offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH;
 
   protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) {
     this(name, allocator, DEFAULT_DATA_VECTOR, callBack);
@@ -54,42 +58,73 @@ protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBa
 
   protected BaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) {
     super(name, allocator);
-    this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
+    this.offsetBuffer = allocator.getEmpty();
     this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
     this.callBack = callBack;
+    this.valueCount = 0;
   }
 
   @Override
   public boolean allocateNewSafe() {
-    /* boolean to keep track if all the memory allocation were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
+    boolean dataAlloc = false;
     try {
-      if (!offsets.allocateNewSafe()) {
-        return false;
-      }
-      success = vector.allocateNewSafe();
-    } finally {
-      if (!success) {
+      allocateOffsetBuffer(offsetAllocationSizeInBytes);
+      dataAlloc = vector.allocateNewSafe();
+    } catch (Exception e) {
+      e.printStackTrace();
+      clear();
+      return false;
+    }
+    finally {
+      if (!dataAlloc) {
         clear();
       }
     }
-    offsets.zeroVector();
-    return success;
+    return dataAlloc;
+  }
+
+  protected void allocateOffsetBuffer(final long size) {
+    final int curSize = (int)size;
+    offsetBuffer = allocator.buffer(curSize);
+    offsetBuffer.readerIndex(0);
+    offsetAllocationSizeInBytes = curSize;
+    offsetBuffer.setZero(0, offsetBuffer.capacity());
   }
 
   @Override
   public void reAlloc() {
-    offsets.reAlloc();
+    reallocOffsetBuffer();
     vector.reAlloc();
   }
 
+  protected void reallocOffsetBuffer() {
+    final int currentBufferCapacity = offsetBuffer.capacity();
+    long baseSize = offsetAllocationSizeInBytes;
+
+    if (baseSize < (long)currentBufferCapacity) {
+      baseSize = (long)currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity);
+    final int halfNewCapacity = newBuf.capacity() / 2;
+    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    offsetBuffer.release(1);
+    offsetBuffer = newBuf;
+    offsetAllocationSizeInBytes = (int)newAllocationSize;
+  }
+
   @Override
+  @Deprecated
   public UInt4Vector getOffsetVector() {
-    return offsets;
+    throw new UnsupportedOperationException("There is no inner offset vector");
   }
 
   @Override
@@ -99,25 +134,29 @@ public FieldVector getDataVector() {
 
   @Override
   public void setInitialCapacity(int numRecords) {
-    offsets.setInitialCapacity(numRecords + 1);
+    offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH;
     vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
   }
 
   @Override
   public int getValueCapacity() {
-    final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
+    final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0);
     if (vector == DEFAULT_DATA_VECTOR) {
       return offsetValueCapacity;
     }
     return Math.min(vector.getValueCapacity(), offsetValueCapacity);
   }
 
+  private int getOffsetBufferValueCapacity() {
+    return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH);
+  }
+
   @Override
   public int getBufferSize() {
-    if (getAccessor().getValueCount() == 0) {
+    if (getValueCount() == 0) {
       return 0;
     }
-    return offsets.getBufferSize() + vector.getBufferSize();
+    return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSize();
   }
 
   @Override
@@ -126,7 +165,7 @@ public int getBufferSizeFor(int valueCount) {
       return 0;
     }
 
-    return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount);
+    return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSizeFor(valueCount);
   }
 
   @Override
@@ -136,14 +175,16 @@ public Iterator<ValueVector> iterator() {
 
   @Override
   public void clear() {
-    offsets.clear();
+    offsetBuffer = releaseBuffer(offsetBuffer);
     vector.clear();
+    valueCount = 0;
     super.clear();
   }
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
+    final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer},
+            vector.getBuffers(false), ArrowBuf.class);
     if (clear) {
       for (ArrowBuf buffer : buffers) {
         buffer.retain();
@@ -187,54 +228,61 @@ protected void replaceDataVector(FieldVector v) {
     vector = v;
   }
 
-  public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
 
-    @Override
-    public int getValueCount() {
-      return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
-    }
+  @Override
+  public int getValueCount() {
+    return valueCount;
+  }
 
-    @Override
-    public int getInnerValueCount() {
-      return vector.getAccessor().getValueCount();
-    }
+  /* returns the value count for inner data vector for this list vector */
+  public int getInnerValueCount() {
+    return vector.getValueCount();
+  }
 
-    @Override
-    public int getInnerValueCountAt(int index) {
-      return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index);
-    }
 
-    @Override
-    public boolean isNull(int index) {
-      return false;
-    }
+  /* returns the value count for inner data vector at a particular index */
+  public int getInnerValueCountAt(int index) {
+    return offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) -
+            offsetBuffer.getInt(index * OFFSET_WIDTH);
+  }
 
-    @Override
-    public boolean isEmpty(int index) {
-      return false;
-    }
+  public boolean isNull(int index) {
+    return false;
   }
 
-  public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
+  public boolean isEmpty(int index) {
+    return false;
+  }
 
-    @Override
-    public int startNewValue(int index) {
-      while (offsets.getValueCapacity() <= index) {
-        offsets.reAlloc();
-      }
-      int offset = offsets.getAccessor().get(index);
-      offsets.getMutator().setSafe(index + 1, offset);
-      setValueCount(index + 1);
-      return offset;
+  public int startNewValue(int index) {
+    while (index >= getOffsetBufferValueCapacity()) {
+      reallocOffsetBuffer();
     }
+    int offset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, offset);
+    setValueCount(index + 1);
+    return offset;
+  }
 
-    @Override
-    public void setValueCount(int valueCount) {
-      // TODO: populate offset end points
-      offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1);
-      final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
-      vector.getMutator().setValueCount(childValueCount);
+  public void setValueCount(int valueCount) {
+    this.valueCount = valueCount;
+    while (valueCount > getOffsetBufferValueCapacity()) {
+      reallocOffsetBuffer();
     }
+    final int childValueCount = valueCount == 0 ? 0 :
+            offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
+    vector.setValueCount(childValueCount);
+  }
+
+  @Override
+  @Deprecated
+  public RepeatedAccessor getAccessor() {
+    throw new UnsupportedOperationException("Accessor is not supported for reading from LIST.");
   }
 
+  @Override
+  @Deprecated
+  public RepeatedMutator getMutator() {
+    throw new UnsupportedOperationException("Mutator is not supported for writing to LIST");
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index e95442acf6d41..bb091f68c1807 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -22,16 +22,14 @@
 import static java.util.Collections.singletonList;
 import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME;
 
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ObjectArrays;
 
 import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.*;
@@ -42,10 +40,7 @@
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
-import org.apache.arrow.vector.util.TransferPair;
+import org.apache.arrow.vector.util.*;
 
 public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {
 
@@ -55,15 +50,13 @@ public static FixedSizeListVector empty(String name, int size, BufferAllocator a
   }
 
   private FieldVector vector;
-  private final BitVector bits;
+  private ArrowBuf validityBuffer;
   private final int listSize;
   private final FieldType fieldType;
-  private final List<BufferBacked> innerVectors;
 
   private UnionFixedSizeListReader reader;
-
-  private Mutator mutator = new Mutator();
-  private Accessor accessor = new Accessor();
+  private int valueCount;
+  private int validityAllocationSizeInBytes;
 
   // deprecated, use FieldType or static constructor instead
   @Deprecated
@@ -80,13 +73,14 @@ public FixedSizeListVector(String name,
                              FieldType fieldType,
                              CallBack schemaChangeCallback) {
     super(name, allocator);
-    this.bits = new BitVector("$bits$", allocator);
+    this.validityBuffer = allocator.getEmpty();
     this.vector = ZeroVector.INSTANCE;
     this.fieldType = fieldType;
     this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
     Preconditions.checkArgument(listSize > 0, "list size must be positive");
-    this.innerVectors = Collections.singletonList((BufferBacked) bits);
     this.reader = new UnionFixedSizeListReader(this);
+    this.valueCount = 0;
+    this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
   }
 
   @Override
@@ -124,27 +118,47 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    BaseDataValueVector.load(fieldNode, innerVectors, ownBuffers);
+    if (ownBuffers.size() != 1) {
+      throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size());
+    }
+
+    ArrowBuf bitBuffer = ownBuffers.get(0);
+
+    validityBuffer.release();
+    validityBuffer = bitBuffer.retain(allocator);
+    valueCount = fieldNode.getLength();
+
+    validityAllocationSizeInBytes = validityBuffer.capacity();
   }
 
   @Override
   public List<ArrowBuf> getFieldBuffers() {
-    return BaseDataValueVector.unload(innerVectors);
+    List<ArrowBuf> result = new ArrayList<>(1);
+
+    validityBuffer.readerIndex(0);
+    validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
+
+    result.add(validityBuffer);
+
+    return result;
   }
 
   @Override
+  @Deprecated
   public List<BufferBacked> getFieldInnerVectors() {
-    return innerVectors;
+    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
   }
 
   @Override
+  @Deprecated
   public Accessor getAccessor() {
-    return accessor;
+    throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
   }
 
   @Override
+  @Deprecated
   public Mutator getMutator() {
-    return mutator;
+    throw new UnsupportedOperationException("Mutator is not supported for writing to vector");
   }
 
   @Override
@@ -154,7 +168,9 @@ public UnionFixedSizeListReader getReader() {
 
   @Override
   public void allocateNew() throws OutOfMemoryException {
-    allocateNewSafe();
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryException("Failure while allocating memory");
+    }
   }
 
   @Override
@@ -166,31 +182,63 @@ public boolean allocateNewSafe() {
      */
     boolean success = false;
     try {
-      success = bits.allocateNewSafe() && vector.allocateNewSafe();
+      /* allocate validity buffer */
+      allocateValidityBuffer(validityAllocationSizeInBytes);
+      success = vector.allocateNewSafe();
     } finally {
       if (!success) {
         clear();
+        return false;
       }
     }
-    if (success) {
-      bits.zeroVector();
-    }
-    return success;
+
+    return true;
+  }
+
+  private void allocateValidityBuffer(final long size) {
+    final int curSize = (int)size;
+    validityBuffer = allocator.buffer(curSize);
+    validityBuffer.readerIndex(0);
+    validityAllocationSizeInBytes = curSize;
+    validityBuffer.setZero(0, validityBuffer.capacity());
   }
 
   @Override
   public void reAlloc() {
-    bits.reAlloc();
+    reallocValidityBuffer();
     vector.reAlloc();
   }
 
+  private void reallocValidityBuffer() {
+    final int currentBufferCapacity = validityBuffer.capacity();
+    long baseSize = validityAllocationSizeInBytes;
+
+    if (baseSize < (long)currentBufferCapacity) {
+      baseSize = (long)currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    newBuf.setZero(0, newBuf.capacity());
+    newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+    validityBuffer.release(1);
+    validityBuffer = newBuf;
+    validityAllocationSizeInBytes = (int)newAllocationSize;
+  }
+
   public FieldVector getDataVector() {
     return vector;
   }
 
   @Override
   public void setInitialCapacity(int numRecords) {
-    bits.setInitialCapacity(numRecords);
+    validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
     vector.setInitialCapacity(numRecords * listSize);
   }
 
@@ -199,15 +247,15 @@ public int getValueCapacity() {
     if (vector == ZeroVector.INSTANCE) {
       return 0;
     }
-    return vector.getValueCapacity() / listSize;
+    return Math.min(vector.getValueCapacity() / listSize, getValidityBufferValueCapacity());
   }
 
   @Override
   public int getBufferSize() {
-    if (accessor.getValueCount() == 0) {
+    if (getValueCount() == 0) {
       return 0;
     }
-    return bits.getBufferSize() + vector.getBufferSize();
+    return getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize();
   }
 
   @Override
@@ -215,7 +263,8 @@ public int getBufferSizeFor(int valueCount) {
     if (valueCount == 0) {
       return 0;
     }
-    return bits.getBufferSizeFor(valueCount) + vector.getBufferSizeFor(valueCount * listSize);
+    return getValidityBufferSizeFromCount(valueCount) +
+            vector.getBufferSizeFor(valueCount * listSize);
   }
 
   @Override
@@ -225,14 +274,15 @@ public Iterator<ValueVector> iterator() {
 
   @Override
   public void clear() {
-    bits.clear();
+    validityBuffer = releaseBuffer(validityBuffer);
     vector.clear();
+    valueCount = 0;
     super.clear();
   }
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
+    final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class);
     if (clear) {
       for (ArrowBuf buffer : buffers) {
         buffer.retain();
@@ -288,7 +338,7 @@ public UnionVector promoteToUnion() {
 
   @Override
   public long getValidityBufferAddress() {
-    return (bits.getDataBuffer().memoryAddress());
+    return validityBuffer.memoryAddress();
   }
 
   @Override
@@ -303,7 +353,7 @@ public long getOffsetBufferAddress() {
 
   @Override
   public ArrowBuf getValidityBuffer() {
-    return (bits.getDataBuffer());
+    return validityBuffer;
   }
 
   @Override
@@ -316,62 +366,64 @@ public ArrowBuf getOffsetBuffer() {
     throw new UnsupportedOperationException();
   }
 
-  public class Accessor extends BaseValueVector.BaseAccessor {
-
-    @Override
-    public Object getObject(int index) {
-      if (isNull(index)) {
-        return null;
-      }
-      final List<Object> vals = new JsonStringArrayList<>(listSize);
-      if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) {
-        for (int i = 0; i < listSize; i++) {
-          vals.add(vector.getObject(index * listSize + i));
-        }
-      } else {
-        final ValueVector.Accessor valuesAccessor = vector.getAccessor();
-        for (int i = 0; i < listSize; i++) {
-          vals.add(valuesAccessor.getObject(index * listSize + i));
-        }
-      }
-      return vals;
+  @Override
+  public Object getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
     }
-
-    @Override
-    public boolean isNull(int index) {
-      return bits.getAccessor().get(index) == 0;
+    final List<Object> vals = new JsonStringArrayList<>(listSize);
+    for (int i = 0; i < listSize; i++) {
+      vals.add(vector.getObject(index * listSize + i));
     }
+    return vals;
+  }
 
-    @Override
-    public int getNullCount() {
-      return bits.getAccessor().getNullCount();
-    }
+  public boolean isNull(int index) {
+    return (isSet(index) == 0);
+  }
 
-    @Override
-    public int getValueCount() {
-      return bits.getAccessor().getValueCount();
-    }
+  public int isSet(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = validityBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
   }
 
-  public class Mutator extends BaseValueVector.BaseMutator {
+  @Override
+  public int getNullCount() {
+    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+  }
 
-    public void setNull(int index) {
-      bits.getMutator().setSafe(index, 0);
+  @Override
+  public int getValueCount() {
+    return valueCount;
+  }
+
+  private int getValidityBufferValueCapacity() {
+    return (int)(validityBuffer.capacity() * 8L);
+  }
+
+  public void setNull(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      reallocValidityBuffer();
     }
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
 
-    public void setNotNull(int index) {
-      bits.getMutator().setSafe(index, 1);
+  public void setNotNull(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      reallocValidityBuffer();
     }
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+  }
 
-    @Override
-    public void setValueCount(int valueCount) {
-      bits.getMutator().setValueCount(valueCount);
-      if (vector instanceof  NullableIntVector || vector instanceof NullableVarCharVector) {
-        vector.setValueCount(valueCount * listSize);
-      } else {
-        vector.getMutator().setValueCount(valueCount * listSize);
-      }
+  @Override
+  public void setValueCount(int valueCount) {
+    this.valueCount = valueCount;
+    while (valueCount > getValidityBufferValueCapacity()) {
+      reallocValidityBuffer();
     }
+    vector.setValueCount(valueCount * listSize);
   }
 
   @Override
@@ -392,7 +444,7 @@ public TransferPair makeTransferPair(ValueVector target) {
   private class TransferImpl implements TransferPair {
 
     FixedSizeListVector to;
-    TransferPair pairs[] = new TransferPair[2];
+    TransferPair dataPair;
 
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
       this(new FixedSizeListVector(name, allocator, fieldType, callBack));
@@ -401,19 +453,20 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
     public TransferImpl(FixedSizeListVector to) {
       this.to = to;
       to.addOrGetVector(vector.getField().getFieldType());
-      pairs[0] = bits.makeTransferPair(to.bits);
-      pairs[1] = vector.makeTransferPair(to.vector);
+      dataPair = vector.makeTransferPair(to.vector);
     }
 
     @Override
     public void transfer() {
-      for (TransferPair pair : pairs) {
-        pair.transfer();
-      }
+      to.clear();
+      dataPair.transfer();
+      to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer;
+      to.setValueCount(valueCount);
     }
 
     @Override
     public void splitAndTransfer(int startIndex, int length) {
+      to.clear();
       to.allocateNew();
       for (int i = 0; i < length; i++) {
         copyValueSafe(startIndex + i, i);
@@ -426,12 +479,15 @@ public ValueVector getTo() {
     }
 
     @Override
-    public void copyValueSafe(int from, int to) {
-      pairs[0].copyValueSafe(from, to);
-      int fromOffset = from * listSize;
-      int toOffset = to * listSize;
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      while (toIndex >= to.getValueCapacity()) {
+        to.reAlloc();
+      }
+      BitVectorHelper.setValidityBit(to.validityBuffer, toIndex, isSet(fromIndex));
+      int fromOffset = fromIndex * listSize;
+      int toOffset = toIndex * listSize;
       for (int i = 0; i < listSize; i++) {
-        pairs[1].copyValueSafe(fromOffset + i, toOffset + i);
+        dataPair.copyValueSafe(fromOffset + i, toOffset + i);
       }
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java
new file mode 100644
index 0000000000000..435d3d8ccf2e6
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.complex;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+
+public abstract class LegacyBaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
+
+   public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
+   public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+   public final static String DATA_VECTOR_NAME = "$data$";
+
+   protected final UInt4Vector offsets;
+   protected FieldVector vector;
+   protected final CallBack callBack;
+
+   protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) {
+      this(name, allocator, DEFAULT_DATA_VECTOR, callBack);
+   }
+
+   protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) {
+      super(name, allocator);
+      this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
+      this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
+      this.callBack = callBack;
+   }
+
+   @Override
+   public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+      boolean success = false;
+      try {
+         if (!offsets.allocateNewSafe()) {
+            return false;
+         }
+         success = vector.allocateNewSafe();
+      } finally {
+         if (!success) {
+            clear();
+         }
+      }
+      offsets.zeroVector();
+      return success;
+   }
+
+   @Override
+   public void reAlloc() {
+      offsets.reAlloc();
+      vector.reAlloc();
+   }
+
+   @Override
+   public UInt4Vector getOffsetVector() {
+      return offsets;
+   }
+
+   @Override
+   public FieldVector getDataVector() {
+      return vector;
+   }
+
+   @Override
+   public void setInitialCapacity(int numRecords) {
+      offsets.setInitialCapacity(numRecords + 1);
+      vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+   }
+
+   @Override
+   public int getValueCapacity() {
+      final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
+      if (vector == DEFAULT_DATA_VECTOR) {
+         return offsetValueCapacity;
+      }
+      return Math.min(vector.getValueCapacity(), offsetValueCapacity);
+   }
+
+   @Override
+   public int getBufferSize() {
+      if (getAccessor().getValueCount() == 0) {
+         return 0;
+      }
+      return offsets.getBufferSize() + vector.getBufferSize();
+   }
+
+   @Override
+   public int getBufferSizeFor(int valueCount) {
+      if (valueCount == 0) {
+         return 0;
+      }
+
+      return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount);
+   }
+
+   @Override
+   public Iterator<ValueVector> iterator() {
+      return Collections.<ValueVector>singleton(getDataVector()).iterator();
+   }
+
+   @Override
+   public void clear() {
+      offsets.clear();
+      vector.clear();
+      super.clear();
+   }
+
+   @Override
+   public ArrowBuf[] getBuffers(boolean clear) {
+      final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
+      if (clear) {
+         for (ArrowBuf buffer : buffers) {
+            buffer.retain();
+         }
+         clear();
+      }
+      return buffers;
+   }
+
+   /**
+    * @return 1 if inner vector is explicitly set via #addOrGetVector else 0
+    */
+   public int size() {
+      return vector == DEFAULT_DATA_VECTOR ? 0 : 1;
+   }
+
+   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) {
+      boolean created = false;
+      if (vector instanceof ZeroVector) {
+         vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack);
+         // returned vector must have the same field
+         created = true;
+         if (callBack != null &&
+                 // not a schema change if changing from ZeroVector to ZeroVector
+                 (fieldType.getType().getTypeID() != ArrowTypeID.Null)) {
+            callBack.doWork();
+         }
+      }
+
+      if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) {
+         final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
+                 fieldType.getType().getTypeID(), vector.getField().getType().getTypeID());
+         throw new SchemaChangeRuntimeException(msg);
+      }
+
+      return new AddOrGetResult<>((T) vector, created);
+   }
+
+   protected void replaceDataVector(FieldVector v) {
+      vector.clear();
+      vector = v;
+   }
+
+   public abstract class LegacyBaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
+
+      @Override
+      public int getValueCount() {
+         return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
+      }
+
+      @Override
+      public int getInnerValueCount() {
+         return vector.getAccessor().getValueCount();
+      }
+
+      @Override
+      public int getInnerValueCountAt(int index) {
+         return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index);
+      }
+
+      @Override
+      public boolean isNull(int index) {
+         return false;
+      }
+
+      @Override
+      public boolean isEmpty(int index) {
+         return false;
+      }
+   }
+
+   public abstract class LegacyBaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
+
+      @Override
+      public int startNewValue(int index) {
+         while (offsets.getValueCapacity() <= index) {
+            offsets.reAlloc();
+         }
+         int offset = offsets.getAccessor().get(index);
+         offsets.getMutator().setSafe(index + 1, offset);
+         setValueCount(index + 1);
+         return offset;
+      }
+
+      @Override
+      public void setValueCount(int valueCount) {
+         // TODO: populate offset end points
+         offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1);
+         final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+         vector.getMutator().setValueCount(childValueCount);
+      }
+   }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java
new file mode 100644
index 0000000000000..0843907d9df82
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java
@@ -0,0 +1,298 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.apache.arrow.vector.complex;
+
+import static java.util.Collections.singletonList;
+import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class LegacyFixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {
+
+   public static LegacyFixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
+      FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
+      return new LegacyFixedSizeListVector(name, allocator, fieldType, null);
+   }
+
+   private Mutator mutator = new Mutator();
+   private Accessor accessor = new Accessor();
+   /* delegate */
+   private final FixedSizeListVector fixedSizeListVector;
+
+   // deprecated, use FieldType or static constructor instead
+   @Deprecated
+   public LegacyFixedSizeListVector(String name,
+                              BufferAllocator allocator,
+                              int listSize,
+                              DictionaryEncoding dictionary,
+                              CallBack schemaChangeCallback) {
+      super(name, allocator);
+      fixedSizeListVector = new FixedSizeListVector(name, allocator, listSize, dictionary, schemaChangeCallback);
+   }
+
+   public LegacyFixedSizeListVector(String name,
+                              BufferAllocator allocator,
+                              FieldType fieldType,
+                              CallBack schemaChangeCallback) {
+      super(name, allocator);
+      fixedSizeListVector = new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback);
+   }
+
+   @Override
+   public Field getField() {
+      return fixedSizeListVector.getField();
+   }
+
+   @Override
+   public MinorType getMinorType() {
+      return MinorType.FIXED_SIZE_LIST;
+   }
+
+   public int getListSize() {
+      return fixedSizeListVector.getListSize();
+   }
+
+   @Override
+   public void initializeChildrenFromFields(List<Field> children) {
+      fixedSizeListVector.initializeChildrenFromFields(children);
+   }
+
+   @Override
+   public List<FieldVector> getChildrenFromFields() {
+      return fixedSizeListVector.getChildrenFromFields();
+   }
+
+   @Override
+   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+      fixedSizeListVector.loadFieldBuffers(fieldNode, ownBuffers);
+   }
+
+   @Override
+   public List<ArrowBuf> getFieldBuffers() {
+      return fixedSizeListVector.getFieldBuffers();
+   }
+
+   @Override
+   public List<BufferBacked> getFieldInnerVectors() {
+      return fixedSizeListVector.getFieldInnerVectors();
+   }
+
+   @Override
+   public Accessor getAccessor() {
+      return accessor;
+   }
+
+   @Override
+   public Mutator getMutator() {
+      return mutator;
+   }
+
+   @Override
+   public UnionFixedSizeListReader getReader() {
+      return fixedSizeListVector.getReader();
+   }
+
+   @Override
+   public void allocateNew() throws OutOfMemoryException {
+     fixedSizeListVector.allocateNew();
+   }
+
+   @Override
+   public boolean allocateNewSafe() {
+    return fixedSizeListVector.allocateNewSafe();
+   }
+
+   @Override
+   public void reAlloc() {
+      fixedSizeListVector.reAlloc();
+   }
+
+   public FieldVector getDataVector() {
+      return fixedSizeListVector.getDataVector();
+   }
+
+   @Override
+   public void setInitialCapacity(int numRecords) {
+      fixedSizeListVector.setInitialCapacity(numRecords);
+   }
+
+   @Override
+   public int getValueCapacity() {
+     return fixedSizeListVector.getValueCapacity();
+   }
+
+   @Override
+   public int getBufferSize() {
+     return fixedSizeListVector.getBufferSize();
+   }
+
+   @Override
+   public int getBufferSizeFor(int valueCount) {
+    return fixedSizeListVector.getBufferSizeFor(valueCount);
+   }
+
+   @Override
+   public Iterator<ValueVector> iterator() {
+      return fixedSizeListVector.iterator();
+   }
+
+   @Override
+   public void clear() {
+     fixedSizeListVector.clear();
+   }
+
+   @Override
+   public ArrowBuf[] getBuffers(boolean clear) {
+      return fixedSizeListVector.getBuffers(clear);
+   }
+
+   /**
+    * @return 1 if inner vector is explicitly set via #addOrGetVector else 0
+    */
+   public int size() {
+      return fixedSizeListVector.size();
+   }
+
+   @Override
+   @SuppressWarnings("unchecked")
+   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType type) {
+      return fixedSizeListVector.addOrGetVector(type);
+   }
+
+   public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) {
+      fixedSizeListVector.copyFromSafe(inIndex, outIndex, from);
+   }
+
+   public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) {
+     fixedSizeListVector.copyFrom(fromIndex, thisIndex, from);
+   }
+
+   @Override
+   public UnionVector promoteToUnion() {
+      return fixedSizeListVector.promoteToUnion();
+   }
+
+   @Override
+   public long getValidityBufferAddress() {
+      return fixedSizeListVector.getValidityBufferAddress();
+   }
+
+   @Override
+   public long getDataBufferAddress() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public long getOffsetBufferAddress() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public ArrowBuf getValidityBuffer() {
+      return fixedSizeListVector.getValidityBuffer();
+   }
+
+   @Override
+   public ArrowBuf getDataBuffer() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public ArrowBuf getOffsetBuffer() {
+      throw new UnsupportedOperationException();
+   }
+
+   public class Accessor extends BaseValueVector.BaseAccessor {
+
+      @Override
+      public Object getObject(int index) {
+         return fixedSizeListVector.getObject(index);
+      }
+
+      @Override
+      public boolean isNull(int index) {
+         return fixedSizeListVector.isNull(index);
+      }
+
+      @Override
+      public int getNullCount() {
+         return fixedSizeListVector.getNullCount();
+      }
+
+      @Override
+      public int getValueCount() {
+         return fixedSizeListVector.getValueCount();
+      }
+   }
+
+   public class Mutator extends BaseValueVector.BaseMutator {
+
+      public void setNull(int index) {
+         fixedSizeListVector.setNull(index);
+      }
+
+      public void setNotNull(int index) {
+         fixedSizeListVector.setNotNull(index);
+      }
+
+      @Override
+      public void setValueCount(int valueCount) {
+         fixedSizeListVector.setValueCount(valueCount);
+      }
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      return fixedSizeListVector.getTransferPair(ref, allocator);
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+      return fixedSizeListVector.getTransferPair(ref, allocator, callBack);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector target) {
+      return fixedSizeListVector.makeTransferPair(((LegacyFixedSizeListVector)target).fixedSizeListVector);
+   }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java
new file mode 100644
index 0000000000000..732699ada7a81
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java
@@ -0,0 +1,288 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.apache.arrow.vector.complex;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Collections.list;
+import static java.util.Collections.singletonList;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.complex.impl.ComplexCopier;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class LegacyListVector extends LegacyBaseRepeatedValueVector implements FieldVector, PromotableVector {
+
+   public static LegacyListVector empty(String name, BufferAllocator allocator) {
+      return new LegacyListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
+   }
+
+   private Mutator mutator = new Mutator();
+   private Accessor accessor = new Accessor();
+
+   private final ListVector listVector;
+
+   // deprecated, use FieldType or static constructor instead
+   @Deprecated
+   public LegacyListVector(String name, BufferAllocator allocator, CallBack callBack) {
+      super(name, allocator, callBack);
+      listVector = new ListVector(name, allocator, callBack);
+   }
+
+   // deprecated, use FieldType or static constructor instead
+   @Deprecated
+   public LegacyListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
+      super(name, allocator, callBack);
+      listVector = new ListVector(name, allocator, dictionary, callBack);
+   }
+
+   public LegacyListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+      super(name, allocator, callBack);
+      listVector = new ListVector(name, allocator, fieldType, callBack);
+   }
+
+   @Override
+   public void initializeChildrenFromFields(List<Field> children) {
+      listVector.initializeChildrenFromFields(children);
+   }
+
+   @Override
+   public List<FieldVector> getChildrenFromFields() {
+      return listVector.getChildrenFromFields();
+   }
+
+   @Override
+   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+      listVector.loadFieldBuffers(fieldNode, ownBuffers);
+   }
+
+   @Override
+   public List<ArrowBuf> getFieldBuffers() {
+      return listVector.getFieldBuffers();
+   }
+
+   @Override
+   public List<BufferBacked> getFieldInnerVectors() {
+      return listVector.getFieldInnerVectors();
+   }
+
+   public UnionListWriter getWriter() {
+      return listVector.getWriter();
+   }
+
+   @Override
+   public void allocateNew() throws OutOfMemoryException {
+      listVector.allocateNew();
+   }
+
+   @Override
+   public void reAlloc() {
+      listVector.reAlloc();
+   }
+
+   public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
+      listVector.copyFrom(inIndex, outIndex, from);
+   }
+
+   public void copyFrom(int inIndex, int outIndex, ListVector from) {
+      listVector.copyFrom(inIndex, outIndex, from);
+   }
+
+   @Override
+   public FieldVector getDataVector() {
+      return listVector.getDataVector();
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      return listVector.getTransferPair(ref, allocator);
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+      return listVector.getTransferPair(ref, allocator, callBack);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector target) {
+      return listVector.makeTransferPair(((LegacyListVector)target).listVector);
+   }
+
+   @Override
+   public long getValidityBufferAddress() {
+      return listVector.getValidityBufferAddress();
+   }
+
+   @Override
+   public long getDataBufferAddress() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public long getOffsetBufferAddress() {
+      return listVector.getOffsetBufferAddress();
+   }
+
+   @Override
+   public ArrowBuf getValidityBuffer() {
+      return listVector.getValidityBuffer();
+   }
+
+   @Override
+   public ArrowBuf getDataBuffer() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public ArrowBuf getOffsetBuffer() {
+      return listVector.getOffsetBuffer();
+   }
+
+   @Override
+   public Accessor getAccessor() {
+      return accessor;
+   }
+
+   @Override
+   public Mutator getMutator() {
+      return mutator;
+   }
+
+   @Override
+   public UnionListReader getReader() {
+      return listVector.getReader();
+   }
+
+   @Override
+   public boolean allocateNewSafe() {
+      return listVector.allocateNewSafe();
+   }
+
+   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) {
+      return listVector.addOrGetVector(fieldType);
+   }
+
+   @Override
+   public int getBufferSize() {
+      return listVector.getBufferSize();
+   }
+
+   @Override
+   public Field getField() {
+     return listVector.getField();
+   }
+
+   @Override
+   public MinorType getMinorType() {
+      return MinorType.LIST;
+   }
+
+   @Override
+   public void clear() {
+      listVector.clear();
+   }
+
+   @Override
+   public ArrowBuf[] getBuffers(boolean clear) {
+     return listVector.getBuffers(clear);
+   }
+
+   @Override
+   public UnionVector promoteToUnion() {
+     return listVector.promoteToUnion();
+   }
+
+   private int lastSet = 0;
+
+   public class Accessor extends LegacyBaseRepeatedAccessor {
+
+      @Override
+      public Object getObject(int index) {
+        return listVector.getObject(index);
+      }
+
+      @Override
+      public boolean isNull(int index) {
+         return listVector.isNull(index);
+      }
+
+      @Override
+      public int getNullCount() {
+         return listVector.getNullCount();
+      }
+   }
+
+   public class Mutator extends LegacyBaseRepeatedMutator {
+      public void setNotNull(int index) {
+        listVector.setNotNull(index);
+      }
+
+      @Override
+      public int startNewValue(int index) {
+         return listVector.startNewValue(index);
+      }
+
+      /**
+       * End the current value
+       *
+       * @param index index of the value to end
+       * @param size  number of elements in the list that was written
+       */
+      public void endValue(int index, int size) {
+         listVector.endValue(index, size);
+      }
+
+      @Override
+      public void setValueCount(int valueCount) {
+        listVector.setValueCount(valueCount);
+      }
+
+      public void setLastSet(int value) {
+         listVector.setLastSet(value);
+      }
+
+      public int getLastSet() {
+         return listVector.getLastSet();
+      }
+   }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java
new file mode 100644
index 0000000000000..77556ab774f02
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.complex;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringHashMap;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class LegacyMapVector extends AbstractMapVector {
+
+   /* delegate */
+   private final MapVector mapVector;
+
+   public static LegacyMapVector empty(String name, BufferAllocator allocator) {
+      FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
+      return new LegacyMapVector(name, allocator, fieldType, null);
+   }
+
+   private final Accessor accessor = new Accessor();
+   private final Mutator mutator = new Mutator();
+
+   @Deprecated
+   public LegacyMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+      super(name, allocator, callBack);
+      mapVector = new MapVector(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
+   }
+
+   public LegacyMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+      super(name, allocator, callBack);
+      mapVector = new MapVector(name, allocator, fieldType, callBack);
+   }
+
+   @Override
+   public FieldReader getReader() {
+      return mapVector.getReader();
+   }
+
+   public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
+     mapVector.copyFromSafe(fromIndex, thisIndex, from);
+   }
+
+   @Override
+   protected boolean supportsDirectRead() {
+      return true;
+   }
+
+   public Iterator<String> fieldNameIterator() {
+      return mapVector.fieldNameIterator();
+   }
+
+   @Override
+   public void setInitialCapacity(int numRecords) {
+     mapVector.setInitialCapacity(numRecords);
+   }
+
+   @Override
+   public int getBufferSize() {
+     return mapVector.getBufferSize();
+   }
+
+   @Override
+   public int getBufferSizeFor(final int valueCount) {
+      return (int) mapVector.getBufferSizeFor(valueCount);
+   }
+
+   @Override
+   public ArrowBuf getValidityBuffer() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public ArrowBuf getDataBuffer() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public ArrowBuf getOffsetBuffer() {
+      throw new UnsupportedOperationException();
+   }
+
+   @Override
+   public TransferPair getTransferPair(BufferAllocator allocator) {
+      return getTransferPair(name, allocator, null);
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+      return mapVector.getTransferPair(ref, allocator, callBack);
+   }
+
+   @Override
+   public TransferPair makeTransferPair(ValueVector to) {
+      return mapVector.makeTransferPair(((LegacyMapVector)to).mapVector);
+   }
+
+   @Override
+   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+      return mapVector.getTransferPair(ref, allocator);
+   }
+
+   @Override
+   public int getValueCapacity() {
+      return mapVector.getValueCapacity();
+   }
+
+   @Override
+   public Accessor getAccessor() {
+      return accessor;
+   }
+
+   @Override
+   public Mutator getMutator() {
+      return mutator;
+   }
+
+   public class Accessor extends BaseValueVector.BaseAccessor {
+
+      @Override
+      public Object getObject(int index) {
+         return mapVector.getObject(index);
+      }
+
+      public void get(int index, ComplexHolder holder) {
+        mapVector.get(index, holder);
+      }
+
+      @Override
+      public int getValueCount() {
+         return mapVector.getValueCount();
+      }
+   }
+
+   public ValueVector getVectorById(int id) {
+      return mapVector.getVectorById(id);
+   }
+
+   public class Mutator extends BaseValueVector.BaseMutator {
+
+      @Override
+      public void setValueCount(int valueCount) {
+        mapVector.setValueCount(valueCount);
+      }
+
+      @Override
+      public void reset() { }
+
+      @Override
+      public void generateTestData(int values) { }
+   }
+
+   @Override
+   public void clear() {
+     mapVector.clear();
+   }
+
+   @Override
+   public Field getField() {
+     return mapVector.getField();
+   }
+
+   @Override
+   public MinorType getMinorType() {
+      return MinorType.MAP;
+   }
+
+   @Override
+   public void close() {
+      mapVector.close();
+   }
+
+   public void initializeChildrenFromFields(List<Field> children) {
+      mapVector.initializeChildrenFromFields(children);
+   }
+
+   public List<FieldVector> getChildrenFromFields() {
+      return mapVector.getChildren();
+   }
+
+   public boolean isNull(int index) { return false; }
+
+   public int getNullCount() { return  0; }
+
+   public int getValueCount() { return 0; }
+
+   public void setValueCount(int valueCount) { }
+
+   public Object getObject(int index) { return null; }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 4b2c9139219f3..ea577208cc238 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -22,26 +22,22 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Collections.singletonList;
 
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.List;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ObjectArrays;
 
 import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.BaseDataValueVector;
-import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.BufferBacked;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.NullableIntVector;
-import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.BitVectorHelper;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
@@ -55,6 +51,7 @@
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
 public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {
@@ -63,14 +60,12 @@ public static ListVector empty(String name, BufferAllocator allocator) {
     return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
   }
 
-  final UInt4Vector offsets;
-  final BitVector bits;
-  private final List<BufferBacked> innerVectors;
-  private Mutator mutator = new Mutator();
-  private Accessor accessor = new Accessor();
+  private ArrowBuf validityBuffer;
   private UnionListReader reader;
   private CallBack callBack;
   private final FieldType fieldType;
+  private int validityAllocationSizeInBytes;
+  private int lastSet;
 
   // deprecated, use FieldType or static constructor instead
   @Deprecated
@@ -86,12 +81,12 @@ public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dic
 
   public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
     super(name, allocator, callBack);
-    this.bits = new BitVector("$bits$", allocator);
-    this.offsets = getOffsetVector();
-    this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
+    this.validityBuffer = allocator.getEmpty();
     this.reader = new UnionListReader(this);
     this.fieldType = checkNotNull(fieldType);
     this.callBack = callBack;
+    this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
+    this.lastSet = 0;
   }
 
   @Override
@@ -115,20 +110,43 @@ public List<FieldVector> getChildrenFromFields() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    // variable width values: truncate offset vector buffer to size (#1)
-    org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, offsets.getBufferSizeFor(fieldNode.getLength() + 1));
-    BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
+    if (ownBuffers.size() != 2) {
+      throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size());
+    }
+
+    ArrowBuf bitBuffer = ownBuffers.get(0);
+    ArrowBuf offBuffer = ownBuffers.get(1);
+
+    validityBuffer.release();
+    validityBuffer = bitBuffer.retain(allocator);
+    offsetBuffer.release();
+    offsetBuffer = offBuffer.retain(allocator);
+
+    validityAllocationSizeInBytes = validityBuffer.capacity();
+    offsetAllocationSizeInBytes = offsetBuffer.capacity();
+
     lastSet = fieldNode.getLength();
+    valueCount = fieldNode.getLength();
   }
 
   @Override
   public List<ArrowBuf> getFieldBuffers() {
-    return BaseDataValueVector.unload(getFieldInnerVectors());
+    List<ArrowBuf> result = new ArrayList<>(2);
+    validityBuffer.readerIndex(0);
+    validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
+    offsetBuffer.readerIndex(0);
+    offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
+
+    result.add(validityBuffer);
+    result.add(offsetBuffer);
+
+    return result;
   }
 
   @Override
+  @Deprecated
   public List<BufferBacked> getFieldInnerVectors() {
-    return innerVectors;
+    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
   }
 
   public UnionListWriter getWriter() {
@@ -137,14 +155,70 @@ public UnionListWriter getWriter() {
 
   @Override
   public void allocateNew() throws OutOfMemoryException {
-    super.allocateNewSafe();
-    bits.allocateNewSafe();
+   if (!allocateNewSafe()) {
+     throw new OutOfMemoryException("Failure while allocating memory");
+   }
+  }
+
+  public boolean allocateNewSafe() {
+    boolean success = false;
+    try {
+      /* allocate validity buffer */
+      allocateValidityBuffer(validityAllocationSizeInBytes);
+      /* allocate offset and data buffer */
+      success = super.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private void allocateValidityBuffer(final long size) {
+    final int curSize = (int)size;
+    validityBuffer = allocator.buffer(curSize);
+    validityBuffer.readerIndex(0);
+    validityAllocationSizeInBytes = curSize;
+    validityBuffer.setZero(0, validityBuffer.capacity());
   }
 
   @Override
   public void reAlloc() {
+    /* reallocate the validity buffer */
+    reallocValidityBuffer();
+    /* reallocate the offset and data */
     super.reAlloc();
-    bits.reAlloc();
+  }
+
+  private void reallocValidityAndOffsetBuffers() {
+    reallocOffsetBuffer();
+    reallocValidityBuffer();
+  }
+
+  private void reallocValidityBuffer() {
+    final int currentBufferCapacity = validityBuffer.capacity();
+    long baseSize = validityAllocationSizeInBytes;
+
+    if (baseSize < (long)currentBufferCapacity) {
+      baseSize = (long)currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+    final int halfNewCapacity = newBuf.capacity() / 2;
+    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    validityBuffer.release(1);
+    validityBuffer = newBuf;
+    validityAllocationSizeInBytes = (int)newAllocationSize;
   }
 
   public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
@@ -181,7 +255,7 @@ public TransferPair makeTransferPair(ValueVector target) {
 
   @Override
   public long getValidityBufferAddress() {
-    return (bits.getDataBuffer().memoryAddress());
+    return (validityBuffer.memoryAddress());
   }
 
   @Override
@@ -191,11 +265,11 @@ public long getDataBufferAddress() {
 
   @Override
   public long getOffsetBufferAddress() {
-    return (offsets.getDataBuffer().memoryAddress());
+    return (offsetBuffer.memoryAddress());
   }
 
   @Override
-  public ArrowBuf getValidityBuffer() { return bits.getDataBuffer(); }
+  public ArrowBuf getValidityBuffer() { return validityBuffer; }
 
   @Override
   public ArrowBuf getDataBuffer() {
@@ -203,17 +277,13 @@ public ArrowBuf getDataBuffer() {
   }
 
   @Override
-  public ArrowBuf getOffsetBuffer() { return offsets.getDataBuffer(); }
+  public ArrowBuf getOffsetBuffer() { return offsetBuffer; }
 
   private class TransferImpl implements TransferPair {
 
     ListVector to;
-    TransferPair bitsTransferPair;
-    TransferPair offsetsTransferPair;
     TransferPair dataTransferPair;
 
-    TransferPair[] pairs;
-
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
       this(new ListVector(name, allocator, fieldType, callBack));
     }
@@ -221,39 +291,99 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
     public TransferImpl(ListVector to) {
       this.to = to;
       to.addOrGetVector(vector.getField().getFieldType());
-      offsetsTransferPair = offsets.makeTransferPair(to.offsets);
-      bitsTransferPair = bits.makeTransferPair(to.bits);
       if (to.getDataVector() instanceof ZeroVector) {
         to.addOrGetVector(vector.getField().getFieldType());
       }
       dataTransferPair = getDataVector().makeTransferPair(to.getDataVector());
-      pairs = new TransferPair[] {bitsTransferPair, offsetsTransferPair, dataTransferPair};
     }
 
     @Override
     public void transfer() {
-      for (TransferPair pair : pairs) {
-        pair.transfer();
-      }
+      dataTransferPair.transfer();
+      to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer;
+      to.offsetBuffer = offsetBuffer.transferOwnership(to.allocator).buffer;
       to.lastSet = lastSet;
+      to.setValueCount(valueCount);
     }
 
     @Override
     public void splitAndTransfer(int startIndex, int length) {
-      UInt4Vector.Accessor offsetVectorAccessor = ListVector.this.offsets.getAccessor();
-      final int startPoint = offsetVectorAccessor.get(startIndex);
-      final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint;
+      final int startPoint = offsetBuffer.getInt(startIndex * OFFSET_WIDTH);
+      final int sliceLength = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH) - startPoint;
       to.clear();
-      to.offsets.allocateNew(length + 1);
-      offsetVectorAccessor = ListVector.this.offsets.getAccessor();
-      final UInt4Vector.Mutator targetOffsetVectorMutator = to.offsets.getMutator();
+      to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH);
+      /* splitAndTransfer offset buffer */
       for (int i = 0; i < length + 1; i++) {
-        targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
+        final int relativeOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - startPoint;
+        to.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeOffset);
       }
-      bitsTransferPair.splitAndTransfer(startIndex, length);
+      /* splitAndTransfer validity buffer */
+      splitAndTransferValidityBuffer(startIndex, length, to);
+      /* splitAndTransfer data buffer */
       dataTransferPair.splitAndTransfer(startPoint, sliceLength);
       to.lastSet = length;
-      to.mutator.setValueCount(length);
+      to.setValueCount(length);
+    }
+
+    /*
+     * transfer the validity.
+     */
+    private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) {
+      assert startIndex + length <= valueCount;
+      int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+      int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+      int byteSizeTarget = getValidityBufferSizeFromCount(length);
+      int offset = startIndex % 8;
+
+      if (length > 0) {
+        if (offset == 0) {
+          // slice
+          if (target.validityBuffer != null) {
+            target.validityBuffer.release();
+          }
+          target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
+          target.validityBuffer.retain(1);
+        }
+        else {
+          /* Copy data
+           * When the first bit starts from the middle of a byte (offset != 0),
+           * copy data from src BitVector.
+           * Each byte in the target is composed by a part in i-th byte,
+           * another part in (i+1)-th byte.
+           */
+          target.allocateValidityBuffer(byteSizeTarget);
+
+          for (int i = 0; i < byteSizeTarget - 1; i++) {
+            byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset);
+            byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset);
+
+            target.validityBuffer.setByte(i, (b1 + b2));
+          }
+
+          /* Copying the last piece is done in the following manner:
+           * if the source vector has 1 or more bytes remaining, we copy
+           * the last piece as a byte formed by shifting data
+           * from the current byte and the next byte.
+           *
+           * if the source vector has no more bytes remaining
+           * (we are at the last byte), we copy the last piece as a byte
+           * by shifting data from the current byte.
+           */
+          if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+            byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+                    firstByteSource + byteSizeTarget - 1, offset);
+            byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
+                    firstByteSource + byteSizeTarget, offset);
+
+            target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+          }
+          else {
+            byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+                    firstByteSource + byteSizeTarget - 1, offset);
+            target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+          }
+        }
+      }
     }
 
     @Override
@@ -267,47 +397,11 @@ public void copyValueSafe(int from, int to) {
     }
   }
 
-  @Override
-  public Accessor getAccessor() {
-    return accessor;
-  }
-
-  @Override
-  public Mutator getMutator() {
-    return mutator;
-  }
-
   @Override
   public UnionListReader getReader() {
     return reader;
   }
 
-  @Override
-  public boolean allocateNewSafe() {
-    /* boolean to keep track if all the memory allocation were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      if (!offsets.allocateNewSafe()) {
-        return false;
-      }
-      success = vector.allocateNewSafe();
-      success = success && bits.allocateNewSafe();
-    } finally {
-      if (!success) {
-        clear();
-      }
-    }
-    if (success) {
-      offsets.zeroVector();
-      bits.zeroVector();
-    }
-    return success;
-  }
-
   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) {
     AddOrGetResult<T> result = super.addOrGetVector(fieldType);
     reader = new UnionListReader(this);
@@ -316,10 +410,12 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldT
 
   @Override
   public int getBufferSize() {
-    if (getAccessor().getValueCount() == 0) {
+    if (getValueCount() == 0) {
       return 0;
     }
-    return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize();
+    final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
+    final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
+    return offsetBufferSize + validityBufferSize + vector.getBufferSize();
   }
 
   @Override
@@ -334,16 +430,14 @@ public MinorType getMinorType() {
 
   @Override
   public void clear() {
-    offsets.clear();
-    vector.clear();
-    bits.clear();
-    lastSet = 0;
     super.clear();
+    validityBuffer = releaseBuffer(validityBuffer);
+    lastSet = 0;
   }
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false),
+    final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, ObjectArrays.concat(new ArrowBuf[] {validityBuffer},
         vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
     if (clear) {
       for (ArrowBuf buffer : buffers) {
@@ -365,99 +459,97 @@ public UnionVector promoteToUnion() {
     return vector;
   }
 
-  private int lastSet = 0;
+  @Override
+  public Object getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    }
+    final List<Object> vals = new JsonStringArrayList<>();
+    final int start = offsetBuffer.getInt(index * OFFSET_WIDTH);
+    final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH);
+    final ValueVector vv = getDataVector();
+    for (int i = start; i < end; i++) {
+      vals.add(vv.getObject(i));
+    }
 
-  public class Accessor extends BaseRepeatedAccessor {
+    return vals;
+  }
 
-    @Override
-    public Object getObject(int index) {
-      if (isNull(index)) {
-        return null;
-      }
-      final List<Object> vals = new JsonStringArrayList<>();
-      final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
-      final int start = offsetsAccessor.get(index);
-      final int end = offsetsAccessor.get(index + 1);
-      final ValueVector vv = getDataVector();
-      if (vv instanceof  NullableIntVector || vv instanceof NullableVarCharVector) {
-        for (int i = start; i < end; i++) {
-          vals.add(vv.getObject(i));
-        }
-      } else {
-        final ValueVector.Accessor valuesAccessor = vv.getAccessor();
-        for (int i = start; i < end; i++) {
-          vals.add(valuesAccessor.getObject(i));
-        }
-      }
+  @Override
+  public boolean isNull(int index) {
+    return (isSet(index) == 0);
+  }
 
-      return vals;
-    }
+  public int isSet(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = validityBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
+  }
 
-    @Override
-    public boolean isNull(int index) {
-      return bits.getAccessor().get(index) == 0;
-    }
+  @Override
+  public int getNullCount() {
+    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+  }
 
-    @Override
-    public int getNullCount() {
-      return bits.getAccessor().getNullCount();
+  public void setNotNull(int index) {
+    if (index >= getValueCapacity()) {
+      reallocValidityAndOffsetBuffers();
     }
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    lastSet = index + 1;
   }
 
-  public class Mutator extends BaseRepeatedMutator {
-    public void setNotNull(int index) {
-      bits.getMutator().setSafe(index, 1);
-      lastSet = index + 1;
+  @Override
+  public int startNewValue(int index) {
+    if (index >= getValueCapacity()) {
+      reallocValidityAndOffsetBuffers();
     }
-
-    @Override
-    public int startNewValue(int index) {
-      for (int i = lastSet; i <= index; i++) {
-        offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
-      }
-      setNotNull(index);
-      lastSet = index + 1;
-      return offsets.getAccessor().get(lastSet);
+    for (int i = lastSet; i <= index; i++) {
+      final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH);
+      offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset);
     }
+    setNotNull(index);
+    lastSet = index + 1;
+    return offsetBuffer.getInt(lastSet * OFFSET_WIDTH);
+  }
 
-    /**
-     * End the current value
-     *
-     * @param index index of the value to end
-     * @param size  number of elements in the list that was written
-     */
-    public void endValue(int index, int size) {
-      offsets.getMutator().set(index + 1, offsets.getAccessor().get(index + 1) + size);
-    }
+  /**
+   * End the current value
+   *
+   * @param index index of the value to end
+   * @param size  number of elements in the list that was written
+   */
+  public void endValue(int index, int size) {
+    final int currentOffset = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, currentOffset + size);
+  }
 
-    @Override
-    public void setValueCount(int valueCount) {
-      // TODO: populate offset end points
-      if (valueCount == 0) {
-        offsets.getMutator().setValueCount(0);
-      } else {
-        for (int i = lastSet; i < valueCount; i++) {
-          offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
-        }
-        offsets.getMutator().setValueCount(valueCount + 1);
+  @Override
+  public void setValueCount(int valueCount) {
+    this.valueCount = valueCount;
+    if (valueCount > 0) {
+      while (valueCount > getValueCapacity()) {
+        /* realloc the inner buffers if needed */
+        reallocValidityAndOffsetBuffers();
       }
-      final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
-      if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) {
-        vector.setValueCount(childValueCount);
-      } else {
-        vector.getMutator().setValueCount(childValueCount);
+      for (int i = lastSet; i < valueCount; i++) {
+        /* fill the holes with offsets */
+        final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH);
+        offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset);
       }
-
-      bits.getMutator().setValueCount(valueCount);
-    }
-
-    public void setLastSet(int value) {
-      lastSet = value;
     }
+    /* valueCount for the data vector is the current end offset */
+    final int childValueCount = (valueCount == 0) ? 0 :
+            offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
+    vector.setValueCount(childValueCount);
+  }
 
-    public int getLastSet() {
-      return lastSet;
-    }
+  public void setLastSet(int value) {
+    lastSet = value;
   }
 
+  public int getLastSet() {
+    return lastSet;
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 95efa60871806..6089a67924fe0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -56,8 +56,6 @@ public static MapVector empty(String name, BufferAllocator allocator) {
   }
 
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
-  private final Accessor accessor = new Accessor();
-  private final Mutator mutator = new Mutator();
   protected final FieldType fieldType;
   public int valueCount;
 
@@ -70,6 +68,7 @@ public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
   public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
     super(name, allocator, callBack);
     this.fieldType = checkNotNull(fieldType);
+    this.valueCount = 0;
   }
 
   @Override
@@ -230,7 +229,7 @@ public void splitAndTransfer(int startIndex, int length) {
       for (TransferPair p : pairs) {
         p.splitAndTransfer(startIndex, length);
       }
-      to.getMutator().setValueCount(length);
+      to.setValueCount(length);
     }
   }
 
@@ -255,76 +254,54 @@ public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
 
   @Override
   public Accessor getAccessor() {
-    return accessor;
+    throw new UnsupportedOperationException("accessor is not needed for MAP");
   }
 
   @Override
   public Mutator getMutator() {
-    return mutator;
+    throw new UnsupportedOperationException("mutator is not needed for MAP");
   }
 
-  public class Accessor extends BaseValueVector.BaseAccessor {
-
-    @Override
-    public Object getObject(int index) {
-      Map<String, Object> vv = new JsonStringHashMap<>();
-      for (String child : getChildFieldNames()) {
-        ValueVector v = getChild(child);
-        if (v instanceof  NullableVarCharVector || v instanceof  NullableIntVector) {
-          if (v != null && index < v.getValueCount()) {
-            Object value = v.getObject(index);
-            if (value != null) {
-              vv.put(child, value);
-            }
-          }
-        } else {
-          if (v != null && index < v.getAccessor().getValueCount()) {
-            Object value = v.getAccessor().getObject(index);
-            if (value != null) {
-              vv.put(child, value);
-            }
-          }
+  @Override
+  public Object getObject(int index) {
+    Map<String, Object> vv = new JsonStringHashMap<>();
+    for (String child : getChildFieldNames()) {
+      ValueVector v = getChild(child);
+      if (v != null && index < v.getValueCount()) {
+        Object value = v.getObject(index);
+        if (value != null) {
+          vv.put(child, value);
         }
       }
-      return vv;
     }
+    return vv;
+  }
 
-    public void get(int index, ComplexHolder holder) {
-      reader.setPosition(index);
-      holder.reader = reader;
-    }
+  @Override
+  public boolean isNull(int index) { return false; }
+  @Override
+  public int getNullCount() { return 0; }
 
-    @Override
-    public int getValueCount() {
-      return valueCount;
-    }
+  public void get(int index, ComplexHolder holder) {
+    reader.setPosition(index);
+    holder.reader = reader;
   }
 
-  public ValueVector getVectorById(int id) {
-    return getChildByOrdinal(id);
+  @Override
+  public int getValueCount() {
+    return valueCount;
   }
 
-  public class Mutator extends BaseValueVector.BaseMutator {
-
-    @Override
-    public void setValueCount(int valueCount) {
-      for (final ValueVector v : getChildren()) {
-        if (v instanceof NullableIntVector || v instanceof NullableVarCharVector) {
-          v.setValueCount(valueCount);
-        } else {
-          v.getMutator().setValueCount(valueCount);
-        }
-      }
-      MapVector.this.valueCount = valueCount;
-    }
-
-    @Override
-    public void reset() {
-    }
+  public ValueVector getVectorById(int id) {
+  return getChildByOrdinal(id);
+}
 
-    @Override
-    public void generateTestData(int values) {
+  @Override
+  public void setValueCount(int valueCount) {
+    for (final ValueVector v : getChildren()) {
+      v.setValueCount(valueCount);
     }
+    MapVector.this.valueCount = valueCount;
   }
 
   @Override
@@ -372,11 +349,4 @@ public void initializeChildrenFromFields(List<Field> children) {
   public List<FieldVector> getChildrenFromFields() {
     return getChildren();
   }
-
-  public int getValueCount() { return 0; }
-
-  public void setValueCount(int valueCount) { }
-
-  public Object getObject(int index) { return null; }
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 1c9d5aa265963..a1478c5a7213d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -20,6 +20,7 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -27,13 +28,9 @@
 import com.google.common.collect.ObjectArrays;
 
 import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.BaseDataValueVector;
-import org.apache.arrow.vector.BitVector;
-import org.apache.arrow.vector.BufferBacked;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.NullableVectorDefinitionSetter;
-import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
 import org.apache.arrow.vector.complex.impl.NullableMapWriter;
 import org.apache.arrow.vector.holders.ComplexHolder;
@@ -44,6 +41,7 @@
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
 public class NullableMapVector extends MapVector implements FieldVector {
@@ -56,12 +54,8 @@ public static NullableMapVector empty(String name, BufferAllocator allocator) {
   private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
   private final NullableMapWriter writer = new NullableMapWriter(this);
 
-  protected final BitVector bits;
-
-  private final List<BufferBacked> innerVectors;
-
-  private final Accessor accessor;
-  private final Mutator mutator;
+  private ArrowBuf validityBuffer;
+  private int validityAllocationSizeInBytes;
 
   // deprecated, use FieldType or static constructor instead
   @Deprecated
@@ -77,10 +71,8 @@ public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncod
 
   public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
     super(name, checkNotNull(allocator), fieldType, callBack);
-    this.bits = new BitVector("$bits$", allocator);
-    this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits));
-    this.accessor = new Accessor();
-    this.mutator = new Mutator();
+    this.validityBuffer = allocator.getEmpty();
+    this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION);
   }
 
   @Override
@@ -92,18 +84,33 @@ public Field getField() {
 
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
-    this.valueCount = fieldNode.getLength();
+    if (ownBuffers.size() != 1) {
+      throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size());
+    }
+
+    ArrowBuf bitBuffer = ownBuffers.get(0);
+
+    validityBuffer.release();
+    validityBuffer = bitBuffer.retain(allocator);
+    valueCount = fieldNode.getLength();
+    validityAllocationSizeInBytes = validityBuffer.capacity();
   }
 
   @Override
   public List<ArrowBuf> getFieldBuffers() {
-    return BaseDataValueVector.unload(getFieldInnerVectors());
+    List<ArrowBuf> result = new ArrayList<>(1);
+
+    validityBuffer.readerIndex(0);
+    validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
+    result.add(validityBuffer);
+
+    return result;
   }
 
   @Override
+  @Deprecated
   public List<BufferBacked> getFieldInnerVectors() {
-    return innerVectors;
+    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
   }
 
   @Override
@@ -146,49 +153,129 @@ protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to,
 
     @Override
     public void transfer() {
-      bits.transferTo(target.bits);
+      target.clear();
+      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
       super.transfer();
     }
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      target.bits.copyFromSafe(fromIndex, toIndex, bits);
+      while (toIndex >= target.getValidityBufferValueCapacity()) {
+        target.reallocValidityBuffer();
+      }
+      BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex));
       super.copyValueSafe(fromIndex, toIndex);
     }
 
     @Override
     public void splitAndTransfer(int startIndex, int length) {
-      bits.splitAndTransferTo(startIndex, length, target.bits);
+      target.clear();
+      splitAndTransferValidityBuffer(startIndex, length, target);
       super.splitAndTransfer(startIndex, length);
     }
   }
 
+  /*
+   * transfer the validity.
+   */
+  private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) {
+    assert startIndex + length <= valueCount;
+    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+    int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length);
+    int offset = startIndex % 8;
+
+    if (length > 0) {
+      if (offset == 0) {
+        // slice
+        if (target.validityBuffer != null) {
+          target.validityBuffer.release();
+        }
+        target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
+        target.validityBuffer.retain(1);
+      }
+      else {
+        /* Copy data
+         * When the first bit starts from the middle of a byte (offset != 0),
+         * copy data from src BitVector.
+         * Each byte in the target is composed by a part in i-th byte,
+         * another part in (i+1)-th byte.
+         */
+        target.allocateValidityBuffer(byteSizeTarget);
+
+        for (int i = 0; i < byteSizeTarget - 1; i++) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset);
+
+          target.validityBuffer.setByte(i, (b1 + b2));
+        }
+
+        /* Copying the last piece is done in the following manner:
+         * if the source vector has 1 or more bytes remaining, we copy
+         * the last piece as a byte formed by shifting data
+         * from the current byte and the next byte.
+         *
+         * if the source vector has no more bytes remaining
+         * (we are at the last byte), we copy the last piece as a byte
+         * by shifting data from the current byte.
+         */
+        if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
+                  firstByteSource + byteSizeTarget, offset);
+
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+        }
+        else {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+        }
+      }
+    }
+  }
+
+  private int getValidityBufferValueCapacity() {
+    return (int)(validityBuffer.capacity() * 8L);
+  }
+
   @Override
   public int getValueCapacity() {
-    return Math.min(bits.getValueCapacity(), super.getValueCapacity());
+    return Math.min(getValidityBufferValueCapacity(),
+            super.getValueCapacity());
   }
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    return ObjectArrays.concat(bits.getBuffers(clear), super.getBuffers(clear), ArrowBuf.class);
+    if (clear) {
+      validityBuffer.retain(1);
+    }
+    return ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(clear), ArrowBuf.class);
   }
 
   @Override
   public void close() {
-    bits.close();
+    clearValidityBuffer();
     super.close();
   }
 
   @Override
   public void clear() {
-    bits.clear();
+    clearValidityBuffer();
     super.clear();
   }
 
+  private void clearValidityBuffer() {
+    validityBuffer.release();
+    validityBuffer = allocator.getEmpty();
+  }
 
   @Override
   public int getBufferSize() {
-    return super.getBufferSize() + bits.getBufferSize();
+    if (valueCount == 0) { return 0; }
+    return super.getBufferSize() +
+            BitVectorHelper.getValidityBufferSize(valueCount);
   }
 
   @Override
@@ -197,12 +284,12 @@ public int getBufferSizeFor(final int valueCount) {
       return 0;
     }
     return super.getBufferSizeFor(valueCount)
-        + bits.getBufferSizeFor(valueCount);
+        + BitVectorHelper.getValidityBufferSize(valueCount);
   }
 
   @Override
   public void setInitialCapacity(int numRecords) {
-    bits.setInitialCapacity(numRecords);
+    validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords);
     super.setInitialCapacity(numRecords);
   }
 
@@ -215,25 +302,59 @@ public boolean allocateNewSafe() {
      */
     boolean success = false;
     try {
-      success = super.allocateNewSafe() && bits.allocateNewSafe();
+      clearValidityBuffer();
+      allocateValidityBuffer(validityAllocationSizeInBytes);
+      success = super.allocateNewSafe();
     } finally {
       if (!success) {
         clear();
+        return false;
       }
     }
-    bits.zeroVector();
-    return success;
+    return true;
+  }
+
+  private void allocateValidityBuffer(final long size) {
+    final int curSize = (int)size;
+    validityBuffer = allocator.buffer(curSize);
+    validityBuffer.readerIndex(0);
+    validityAllocationSizeInBytes = curSize;
+    validityBuffer.setZero(0, validityBuffer.capacity());
   }
 
   @Override
   public void reAlloc() {
-    bits.reAlloc();
+    /* reallocate the validity buffer */
+    reallocValidityBuffer();
     super.reAlloc();
   }
 
+  private void reallocValidityBuffer() {
+    final int currentBufferCapacity = validityBuffer.capacity();
+    long baseSize = validityAllocationSizeInBytes;
+
+    if (baseSize < (long)currentBufferCapacity) {
+      baseSize = (long)currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    newBuf.setZero(0, newBuf.capacity());
+    newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+    validityBuffer.release(1);
+    validityBuffer = newBuf;
+    validityAllocationSizeInBytes = (int)newAllocationSize;
+  }
+
   @Override
   public long getValidityBufferAddress() {
-    return bits.getBuffer().memoryAddress();
+    return validityBuffer.memoryAddress();
   }
 
   @Override
@@ -248,7 +369,7 @@ public long getOffsetBufferAddress() {
 
   @Override
   public ArrowBuf getValidityBuffer() {
-    return bits.getDataBuffer();
+    return validityBuffer;
   }
 
   @Override
@@ -261,82 +382,76 @@ public ArrowBuf getOffsetBuffer() {
     throw new UnsupportedOperationException();
   }
 
-  public final class Accessor extends MapVector.Accessor {
-    final BitVector.Accessor bAccessor = bits.getAccessor();
-
-    @Override
-    public Object getObject(int index) {
-      if (isNull(index)) {
-        return null;
-      } else {
-        return super.getObject(index);
-      }
-    }
-
-    @Override
-    public void get(int index, ComplexHolder holder) {
-      holder.isSet = isSet(index);
-      super.get(index, holder);
-    }
-
-    @Override
-    public int getNullCount() {
-      return bits.getAccessor().getNullCount();
-    }
-
-    @Override
-    public boolean isNull(int index) {
-      return isSet(index) == 0;
-    }
-
-    public int isSet(int index) {
-      return bAccessor.get(index);
+  @Override
+  public Object getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return super.getObject(index);
     }
-
   }
 
-  public final class Mutator extends MapVector.Mutator implements NullableVectorDefinitionSetter {
+  @Override
+  public void get(int index, ComplexHolder holder) {
+    holder.isSet = isSet(index);
+    super.get(index, holder);
+  }
 
-    private Mutator() {
-    }
+  public int getNullCount() {
+    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+  }
 
-    @Override
-    public void setIndexDefined(int index) {
-      bits.getMutator().setSafe(index, 1);
-    }
+  public boolean isNull(int index) {
+    return isSet(index) == 0;
+  }
 
-    public void setNull(int index) {
-      bits.getMutator().setSafe(index, 0);
-    }
+  public int isSet(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = validityBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
+  }
 
-    @Override
-    public void setValueCount(int valueCount) {
-      assert valueCount >= 0;
-      super.setValueCount(valueCount);
-      bits.getMutator().setValueCount(valueCount);
+  public void setIndexDefined(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
     }
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+  }
 
-    @Override
-    public void generateTestData(int valueCount) {
-      super.generateTestData(valueCount);
-      bits.getMutator().generateTestDataAlt(valueCount);
+  public void setNull(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
     }
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
 
-    @Override
-    public void reset() {
-      bits.getMutator().setValueCount(0);
+  @Override
+  public void setValueCount(int valueCount) {
+    assert valueCount >= 0;
+    while (valueCount > getValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
     }
+    super.setValueCount(valueCount);
+    this.valueCount = valueCount;
+  }
 
+  public void reset() {
+    valueCount = 0;
   }
 
   @Override
+  @Deprecated
   public Accessor getAccessor() {
-    return accessor;
+    throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP");
   }
 
   @Override
+  @Deprecated
   public Mutator getMutator() {
-    return mutator;
+    throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP");
   }
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
index 614c266acf147..06b0f4d43272f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
@@ -52,6 +52,6 @@ public void copyAsField(String name, MapWriter writer) {
 
   @Override
   public boolean isSet() {
-    return !nullableMapVector.getAccessor().isNull(idx());
+    return !nullableMapVector.isNull(idx());
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 9722196ed7cd2..5bd439cac4269 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -162,7 +162,7 @@ private FieldWriter promoteToUnion() {
     writer = new UnionWriter(unionVector, nullableMapWriterFactory);
     writer.setPosition(idx());
     for (int i = 0; i <= idx(); i++) {
-      unionVector.getMutator().setType(i, vector.getMinorType());
+      unionVector.setType(i, vector.getMinorType());
     }
     vector = null;
     state = State.UNION;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index 3ebd0cd7dd959..c77ca4e8f23ba 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -78,7 +78,7 @@ public void setPosition(int index) {
 
   @Override
   public Object readObject() {
-    return vector.getAccessor().getObject(idx());
+    return vector.getObject(idx());
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
index f3e9b8773f25e..4ad2f6f5fad7a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
@@ -46,7 +46,7 @@ public UnionFixedSizeListReader(FixedSizeListVector vector) {
 
   @Override
   public boolean isSet() {
-    return !vector.getAccessor().isNull(idx());
+    return !vector.isNull(idx());
   }
 
   @Override
@@ -56,7 +56,7 @@ public FieldReader reader() {
 
   @Override
   public Object readObject() {
-    return vector.getAccessor().getObject(idx());
+    return vector.getObject(idx());
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
index b98c36d2bf721..6243a2833c055 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
@@ -19,6 +19,7 @@
 
 package org.apache.arrow.vector.complex.impl;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.ListVector;
@@ -33,12 +34,11 @@ public class UnionListReader extends AbstractFieldReader {
 
   private ListVector vector;
   private ValueVector data;
-  private UInt4Vector offsets;
+  private static final int OFFSET_WIDTH = 4;
 
   public UnionListReader(ListVector vector) {
     this.vector = vector;
     this.data = vector.getDataVector();
-    this.offsets = vector.getOffsetVector();
   }
 
   @Override
@@ -48,7 +48,7 @@ public Field getField() {
 
   @Override
   public boolean isSet() {
-    return !vector.getAccessor().isNull(idx());
+    return !vector.isNull(idx());
   }
 
   private int currentOffset;
@@ -57,8 +57,8 @@ public boolean isSet() {
   @Override
   public void setPosition(int index) {
     super.setPosition(index);
-    currentOffset = offsets.getAccessor().get(index) - 1;
-    maxOffset = offsets.getAccessor().get(index + 1);
+    currentOffset = vector.getOffsetBuffer().getInt(index * OFFSET_WIDTH) - 1;
+    maxOffset = vector.getOffsetBuffer().getInt((index + 1) * OFFSET_WIDTH);
   }
 
   @Override
@@ -68,7 +68,7 @@ public FieldReader reader() {
 
   @Override
   public Object readObject() {
-    return vector.getAccessor().getObject(idx());
+    return vector.getObject(idx());
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
index 3b7dc4a56e1ac..762a442c983fe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -47,11 +47,10 @@ public class DictionaryEncoder {
   public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
     validateType(vector.getMinorType());
     // load dictionary values into a hashmap for lookup
-    ValueVector.Accessor dictionaryAccessor = dictionary.getVector().getAccessor();
-    Map<Object, Integer> lookUps = new HashMap<>(dictionaryAccessor.getValueCount());
-    for (int i = 0; i < dictionaryAccessor.getValueCount(); i++) {
+    Map<Object, Integer> lookUps = new HashMap<>(dictionary.getVector().getValueCount());
+    for (int i = 0; i < dictionary.getVector().getValueCount(); i++) {
       // for primitive array types we need a wrapper that implements equals and hashcode appropriately
-      lookUps.put(dictionaryAccessor.getObject(i), i);
+      lookUps.put(dictionary.getVector().getObject(i), i);
     }
 
     Field valueField = vector.getField();
@@ -61,14 +60,13 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
 
     // vector to hold our indices (dictionary encoded values)
     FieldVector indices = indexField.createVector(vector.getAllocator());
-    ValueVector.Mutator mutator = indices.getMutator();
 
     // use reflection to pull out the set method
     // TODO implement a common interface for int vectors
     Method setter = null;
     for (Class<?> c : ImmutableList.of(int.class, long.class)) {
       try {
-        setter = mutator.getClass().getMethod("setSafe", int.class, c);
+        setter = indices.getClass().getMethod("setSafe", int.class, c);
         break;
       } catch (NoSuchMethodException e) {
         // ignore
@@ -78,21 +76,20 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
       throw new IllegalArgumentException("Dictionary encoding does not have a valid int type:" + indices.getClass());
     }
 
-    ValueVector.Accessor accessor = vector.getAccessor();
-    int count = accessor.getValueCount();
+    int count = vector.getValueCount();
 
     indices.allocateNew();
 
     try {
       for (int i = 0; i < count; i++) {
-        Object value = accessor.getObject(i);
+        Object value = vector.getObject(i);
         if (value != null) { // if it's null leave it null
           // note: this may fail if value was not included in the dictionary
           Object encoded = lookUps.get(value);
           if (encoded == null) {
             throw new IllegalArgumentException("Dictionary encoding not defined for value:" + value);
           }
-          setter.invoke(mutator, i, encoded);
+          setter.invoke(indices, i, encoded);
         }
       }
     } catch (IllegalAccessException e) {
@@ -101,7 +98,7 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
       throw new RuntimeException("InvocationTargetException invoking vector mutator set():", e.getCause());
     }
 
-    mutator.setValueCount(count);
+    indices.setValueCount(count);
 
     return indices;
   }
@@ -114,15 +111,14 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) {
    * @return vector with values restored from dictionary
    */
   public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
-    ValueVector.Accessor accessor = indices.getAccessor();
-    int count = accessor.getValueCount();
+    int count = indices.getValueCount();
     ValueVector dictionaryVector = dictionary.getVector();
-    int dictionaryCount = dictionaryVector.getAccessor().getValueCount();
+    int dictionaryCount = dictionaryVector.getValueCount();
     // copy the dictionary values into the decoded vector
     TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator());
     transfer.getTo().allocateNewSafe();
     for (int i = 0; i < count; i++) {
-      Object index = accessor.getObject(i);
+      Object index = indices.getObject(i);
       if (index != null) {
         int indexAsInt = ((Number) index).intValue();
         if (indexAsInt > dictionaryCount) {
@@ -133,7 +129,7 @@ public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
     }
     // TODO do we need to worry about the field?
     ValueVector decoded = transfer.getTo();
-    decoded.getMutator().setValueCount(count);
+    decoded.setValueCount(count);
     return decoded;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index b35aba5426e4a..7dc10b5e6299f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -82,7 +82,7 @@ protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, Writab
     for (long id : dictionaryIdsUsed) {
       Dictionary dictionary = provider.lookup(id);
       FieldVector vector = dictionary.getVector();
-      int count = vector.getAccessor().getValueCount();
+      int count = vector.getValueCount();
       VectorSchemaRoot dictRoot = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), count);
       VectorUnloader unloader = new VectorUnloader(dictRoot);
       ArrowRecordBatch batch = unloader.getRecordBatch();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 0de804406f7b3..b21fcaac391e8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -23,60 +23,28 @@
 import static com.fasterxml.jackson.core.JsonToken.START_ARRAY;
 import static com.fasterxml.jackson.core.JsonToken.START_OBJECT;
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET;
+import static org.apache.arrow.vector.schema.ArrowVectorType.*;
 
 import java.io.File;
 import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.*;
 
 import com.google.common.collect.ImmutableList;
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.BigIntVector;
-import org.apache.arrow.vector.BitVector;
-import org.apache.arrow.vector.BufferBacked;
-import org.apache.arrow.vector.DateDayVector;
-import org.apache.arrow.vector.DateMilliVector;
-import org.apache.arrow.vector.DecimalVector;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.Float4Vector;
-import org.apache.arrow.vector.Float8Vector;
-import org.apache.arrow.vector.IntVector;
-import org.apache.arrow.vector.NullableVarBinaryVector;
-import org.apache.arrow.vector.NullableVarCharVector;
-import org.apache.arrow.vector.SmallIntVector;
-import org.apache.arrow.vector.TimeMicroVector;
-import org.apache.arrow.vector.TimeMilliVector;
-import org.apache.arrow.vector.TimeNanoVector;
-import org.apache.arrow.vector.TimeSecVector;
-import org.apache.arrow.vector.TimeStampMicroTZVector;
-import org.apache.arrow.vector.TimeStampMicroVector;
-import org.apache.arrow.vector.TimeStampMilliTZVector;
-import org.apache.arrow.vector.TimeStampMilliVector;
-import org.apache.arrow.vector.TimeStampNanoTZVector;
-import org.apache.arrow.vector.TimeStampNanoVector;
-import org.apache.arrow.vector.TimeStampSecTZVector;
-import org.apache.arrow.vector.TimeStampSecVector;
-import org.apache.arrow.vector.TinyIntVector;
-import org.apache.arrow.vector.UInt1Vector;
-import org.apache.arrow.vector.UInt2Vector;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.UInt8Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VarBinaryVector;
-import org.apache.arrow.vector.VarCharVector;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
-import org.apache.arrow.vector.util.DecimalUtility;
 import org.apache.arrow.vector.util.DictionaryUtility;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
@@ -153,7 +121,7 @@ private void readDictionaryBatches() throws JsonParseException, IOException {
       FieldVector vector = dict.getVector();
       List<Field> fields = ImmutableList.of(vector.getField());
       List<FieldVector> vectors = ImmutableList.of(vector);
-      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount());
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount());
       read(root);
 
       readToken(END_OBJECT);
@@ -177,7 +145,7 @@ public boolean read(VectorSchemaRoot root) throws IOException {
         {
           for (Field field : root.getSchema().getFields()) {
             FieldVector vector = root.getVector(field.getName());
-            readVector(field, vector);
+            readFromJsonIntoVector(field, vector);
           }
         }
         readToken(END_ARRAY);
@@ -204,7 +172,7 @@ public VectorSchemaRoot read() throws IOException {
         {
           for (Field field : schema.getFields()) {
             FieldVector vector = recordBatch.getVector(field.getName());
-            readVector(field, vector);
+            readFromJsonIntoVector(field, vector);
           }
         }
         readToken(END_ARRAY);
@@ -218,16 +186,24 @@ public VectorSchemaRoot read() throws IOException {
     }
   }
 
-  /**
-   * TODO: A better way of implementing this function is to use `loadFieldBuffers` methods in
-   * FieldVector to set the inner-vector data as done in `ArrowFileReader`.
-   */
-  private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException {
+  private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException {
     List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
-    List<BufferBacked> fieldInnerVectors = vector.getFieldInnerVectors();
-    if (vectorTypes.size() != fieldInnerVectors.size()) {
-      throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size());
-    }
+    ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()];
+    /*
+     * The order of inner buffers is :
+     * Fixed width vector:
+     *    -- validity buffer
+     *    -- data buffer
+     * Variable width vector:
+     *    -- validity buffer
+     *    -- offset buffer
+     *    -- data buffer
+     *
+     * This is similar to what getFieldInnerVectors() used to give but now that we don't have
+     * inner vectors anymore, we will work directly at the buffer level -- populate buffers
+     * locally as we read from Json parser and do loadFieldBuffers on the vector followed by
+     * releasing the local buffers.
+     */
     readToken(START_OBJECT);
     {
       // If currently reading dictionaries, field name is not important so don't check
@@ -236,39 +212,41 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name);
       }
 
-      // Initialize the vector with required capacity
-      int count = readNextField("count", Integer.class);
-      vector.setInitialCapacity(count);
-      vector.allocateNew();
+      /* Initialize the vector with required capacity but don't allocate since we would
+       * be doing loadFieldBuffers.
+       */
+      int valueCount = readNextField("count", Integer.class);
+      vector.setInitialCapacity(valueCount);
 
-      // Read inner vectors
       for (int v = 0; v < vectorTypes.size(); v++) {
         ArrowVectorType vectorType = vectorTypes.get(v);
-        ValueVector valueVector = (ValueVector) fieldInnerVectors.get(v);
         nextFieldIs(vectorType.getName());
         readToken(START_ARRAY);
-        int innerVectorCount = vectorType.equals(OFFSET) ? count + 1 : count;
-        for (int i = 0; i < innerVectorCount; i++) {
+        int innerBufferValueCount = valueCount;
+        if (vectorType.equals(OFFSET)) {
+          /* offset buffer has 1 additional value capacity */
+          innerBufferValueCount = valueCount + 1;
+        }
+        for (int i = 0; i < innerBufferValueCount; i++) {
+          /* write data to the buffer */
           parser.nextToken();
-          setValueFromParser(valueVector, i);
+          /* for variable width vectors, value count doesn't help pre-determining the capacity of
+           * the underlying data buffer. So we need to pass down the offset buffer (which was already
+           * populated in the previous iteration of this loop).
+           */
+          if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR
+                  || vector.getMinorType() == Types.MinorType.VARBINARY)) {
+            vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v],
+                    vectorBuffers[v-1], i, innerBufferValueCount);
+          } else {
+            vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v],
+                    null, i, innerBufferValueCount);
+          }
         }
         readToken(END_ARRAY);
       }
 
-      // Set lastSet before valueCount to prevent setValueCount from filling empty values
-      switch (vector.getMinorType()) {
-        case LIST:
-          // ListVector starts lastSet from index 0, so lastSet value is always last index written + 1
-          ((ListVector) vector).getMutator().setLastSet(count);
-          break;
-        case VARBINARY:
-          ((NullableVarBinaryVector) vector).getMutator().setLastSet(count - 1);
-          break;
-        case VARCHAR:
-          ((NullableVarCharVector) vector).setLastSet(count - 1);
-          break;
-      }
-      vector.getMutator().setValueCount(count);
+      vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers));
 
       // read child vectors, if any
       List<Field> fields = field.getChildren();
@@ -282,12 +260,16 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
         for (int i = 0; i < fields.size(); i++) {
           Field childField = fields.get(i);
           FieldVector childVector = vectorChildren.get(i);
-          readVector(childField, childVector);
+          readFromJsonIntoVector(childField, childVector);
         }
         readToken(END_ARRAY);
       }
     }
     readToken(END_OBJECT);
+
+    for (ArrowBuf buffer: vectorBuffers) {
+      buffer.release();
+    }
   }
 
   private byte[] decodeHexSafe(String hexString) throws IOException {
@@ -298,99 +280,124 @@ private byte[] decodeHexSafe(String hexString) throws IOException {
     }
   }
 
-  private void setValueFromParser(ValueVector valueVector, int i) throws IOException {
-    switch (valueVector.getMinorType()) {
-      case BIT:
-        ((BitVector) valueVector).getMutator().set(i, parser.readValueAs(Boolean.class) ? 1 : 0);
-        break;
-      case TINYINT:
-        ((TinyIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case SMALLINT:
-        ((SmallIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case INT:
-        ((IntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case BIGINT:
-        ((BigIntVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case UINT1:
-        ((UInt1Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case UINT2:
-        ((UInt2Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case UINT4:
-        ((UInt4Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case UINT8:
-        ((UInt8Vector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case FLOAT4:
-        ((Float4Vector) valueVector).getMutator().set(i, parser.readValueAs(Float.class));
-        break;
-      case FLOAT8:
-        ((Float8Vector) valueVector).getMutator().set(i, parser.readValueAs(Double.class));
-        break;
-      case DECIMAL: {
-          DecimalVector decimalVector = (DecimalVector) valueVector;
-          // Here we assume the decimal value is the unscaled integer value as a string
-          BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class));
-          DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, decimalVector.getBuffer(), i);
-        }
-        break;
-      case VARBINARY:
-        ((VarBinaryVector) valueVector).getMutator().setSafe(i, decodeHexSafe(parser.readValueAs(String.class)));
-        break;
-      case VARCHAR:
-        ((VarCharVector) valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8));
-        break;
-      case DATEDAY:
-        ((DateDayVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case DATEMILLI:
-        ((DateMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESEC:
-        ((TimeSecVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case TIMEMILLI:
-        ((TimeMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class));
-        break;
-      case TIMEMICRO:
-        ((TimeMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMENANO:
-        ((TimeNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESTAMPSEC:
-        ((TimeStampSecVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESTAMPMILLI:
-        ((TimeStampMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESTAMPMICRO:
-        ((TimeStampMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESTAMPNANO:
-        ((TimeStampNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESTAMPSECTZ:
-        ((TimeStampSecTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESTAMPMILLITZ:
-        ((TimeStampMilliTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESTAMPMICROTZ:
-        ((TimeStampMicroTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      case TIMESTAMPNANOTZ:
-        ((TimeStampNanoTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-        break;
-      default:
-        throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
+  private ArrowBuf setValueFromParser(ArrowVectorType bufferType, FieldVector vector,
+                                      ArrowBuf buffer, ArrowBuf offsetBuffer, int index,
+                                      int valueCount) throws IOException {
+    if (bufferType.equals(TYPE)) {
+      buffer = NullableTinyIntVector.set(buffer, allocator,
+              valueCount, index, parser.readValueAs(Byte.class));
+    } else if (bufferType.equals(OFFSET)) {
+      buffer = BaseNullableVariableWidthVector.set(buffer, allocator,
+              valueCount, index, parser.readValueAs(Integer.class));
+    } else if (bufferType.equals(VALIDITY)) {
+      buffer = BitVectorHelper.setValidityBit(buffer, allocator,
+              valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0);
+    } else if (bufferType.equals(DATA)) {
+      switch (vector.getMinorType()) {
+        case BIT:
+          buffer = BitVectorHelper.setValidityBit(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0);
+          break;
+        case TINYINT:
+          buffer = NullableTinyIntVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Byte.class));
+          break;
+        case SMALLINT:
+          buffer = NullableSmallIntVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Short.class));
+          break;
+        case INT:
+          buffer = NullableIntVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Integer.class));
+          break;
+        case BIGINT:
+          buffer = NullableBigIntVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case FLOAT4:
+          buffer = NullableFloat4Vector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Float.class));
+          break;
+        case FLOAT8:
+          buffer = NullableFloat8Vector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Double.class));
+          break;
+        case DECIMAL:
+          buffer = NullableDecimalVector.set(buffer, allocator,
+                  valueCount, index, decodeHexSafe(parser.readValueAs(String.class)));
+          break;
+        case VARBINARY:
+          assert (offsetBuffer != null);
+          buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index,
+                  decodeHexSafe(parser.readValueAs(String.class)), valueCount);
+          break;
+        case VARCHAR:
+          assert (offsetBuffer != null);
+          buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index,
+                  parser.readValueAs(String.class).getBytes(UTF_8), valueCount);
+          break;
+        case DATEDAY:
+          buffer = NullableDateDayVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Integer.class));
+          break;
+        case DATEMILLI:
+          buffer = NullableDateMilliVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESEC:
+          buffer = NullableTimeSecVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Integer.class));
+          break;
+        case TIMEMILLI:
+          buffer = NullableTimeMilliVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Integer.class));
+          break;
+        case TIMEMICRO:
+          buffer = NullableTimeMicroVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMENANO:
+          buffer = NullableTimeNanoVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESTAMPSEC:
+          buffer = NullableTimeStampSecVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESTAMPMILLI:
+          buffer = NullableTimeStampMilliVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESTAMPMICRO:
+          buffer = NullableTimeStampMicroVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESTAMPNANO:
+          buffer = NullableTimeStampNanoVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESTAMPSECTZ:
+          buffer = NullableTimeStampSecTZVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESTAMPMILLITZ:
+          buffer = NullableTimeStampMilliTZVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESTAMPMICROTZ:
+          buffer = NullableTimeStampMicroTZVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        case TIMESTAMPNANOTZ:
+          buffer = NullableTimeStampNanoTZVector.set(buffer, allocator,
+                  valueCount, index, parser.readValueAs(Long.class));
+          break;
+        default:
+          throw new UnsupportedOperationException("minor type: " + vector.getMinorType());
+      }
     }
+
+    return buffer;
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index 05341bec44ea3..7921cd34c773c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -18,9 +18,10 @@
 
 package org.apache.arrow.vector.file.json;
 
+import static org.apache.arrow.vector.schema.ArrowVectorType.*;
+
 import java.io.File;
 import java.io.IOException;
-import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -28,28 +29,11 @@
 
 import com.google.common.collect.ImmutableList;
 import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.vector.BitVector;
-import org.apache.arrow.vector.BufferBacked;
-import org.apache.arrow.vector.DateDayVector;
-import org.apache.arrow.vector.DateMilliVector;
-import org.apache.arrow.vector.DecimalVector;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.TimeMicroVector;
-import org.apache.arrow.vector.TimeMilliVector;
-import org.apache.arrow.vector.TimeNanoVector;
-import org.apache.arrow.vector.TimeSecVector;
-import org.apache.arrow.vector.TimeStampMicroVector;
-import org.apache.arrow.vector.TimeStampMilliVector;
-import org.apache.arrow.vector.TimeStampNanoVector;
-import org.apache.arrow.vector.TimeStampSecVector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.ValueVector.Accessor;
-import org.apache.arrow.vector.VarBinaryVector;
-import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.schema.ArrowVectorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 
@@ -135,7 +119,7 @@ private void writeDictionaryBatches(JsonGenerator generator, Set<Long> dictionar
       FieldVector vector = dictionary.getVector();
       List<Field> fields = ImmutableList.of(vector.getField());
       List<FieldVector> vectors = ImmutableList.of(vector);
-      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount());
+      VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount());
       writeBatch(root);
 
       generator.writeEndObject();
@@ -157,31 +141,36 @@ private void writeBatch(VectorSchemaRoot recordBatch) throws IOException {
       generator.writeArrayFieldStart("columns");
       for (Field field : recordBatch.getSchema().getFields()) {
         FieldVector vector = recordBatch.getVector(field.getName());
-        writeVector(field, vector);
+        writeFromVectorIntoJson(field, vector);
       }
       generator.writeEndArray();
     }
     generator.writeEndObject();
   }
 
-  private void writeVector(Field field, FieldVector vector) throws IOException {
+  private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOException {
     List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
-    List<BufferBacked> fieldInnerVectors = vector.getFieldInnerVectors();
-    if (vectorTypes.size() != fieldInnerVectors.size()) {
-      throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size());
+    List<ArrowBuf> vectorBuffers = vector.getFieldBuffers();
+    if (vectorTypes.size() != vectorBuffers.size()) {
+      throw new IllegalArgumentException("vector types and inner vector buffers are not the same size: " + vectorTypes.size() + " != " + vectorBuffers.size());
     }
     generator.writeStartObject();
     {
       generator.writeObjectField("name", field.getName());
-      int valueCount = vector.getAccessor().getValueCount();
+      int valueCount = vector.getValueCount();
       generator.writeObjectField("count", valueCount);
       for (int v = 0; v < vectorTypes.size(); v++) {
         ArrowVectorType vectorType = vectorTypes.get(v);
-        BufferBacked innerVector = fieldInnerVectors.get(v);
+        ArrowBuf vectorBuffer = vectorBuffers.get(v);
         generator.writeArrayFieldStart(vectorType.getName());
-        ValueVector valueVector = (ValueVector) innerVector;
-        for (int i = 0; i < valueVector.getAccessor().getValueCount(); i++) {
-          writeValueToGenerator(valueVector, i);
+        final int bufferValueCount = (vectorType.equals(OFFSET)) ? valueCount + 1 : valueCount;
+        for (int i = 0; i < bufferValueCount; i++) {
+          if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR ||
+                  vector.getMinorType() == Types.MinorType.VARBINARY)) {
+            writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i);
+          } else {
+            writeValueToGenerator(vectorType, vectorBuffer, null, vector, i);
+          }
         }
         generator.writeEndArray();
       }
@@ -195,7 +184,7 @@ private void writeVector(Field field, FieldVector vector) throws IOException {
         for (int i = 0; i < fields.size(); i++) {
           Field childField = fields.get(i);
           FieldVector childVector = children.get(i);
-          writeVector(childField, childVector);
+          writeFromVectorIntoJson(childField, childVector);
         }
         generator.writeEndArray();
       }
@@ -203,62 +192,101 @@ private void writeVector(Field field, FieldVector vector) throws IOException {
     generator.writeEndObject();
   }
 
-  private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException {
-    switch (valueVector.getMinorType()) {
-      case DATEDAY:
-        generator.writeNumber(((DateDayVector) valueVector).getAccessor().get(i));
-        break;
-      case DATEMILLI:
-        generator.writeNumber(((DateMilliVector) valueVector).getAccessor().get(i));
-        break;
-      case TIMESEC:
-        generator.writeNumber(((TimeSecVector) valueVector).getAccessor().get(i));
-        break;
-      case TIMEMILLI:
-        generator.writeNumber(((TimeMilliVector) valueVector).getAccessor().get(i));
-        break;
-      case TIMEMICRO:
-        generator.writeNumber(((TimeMicroVector) valueVector).getAccessor().get(i));
-        break;
-      case TIMENANO:
-        generator.writeNumber(((TimeNanoVector) valueVector).getAccessor().get(i));
-        break;
-      case TIMESTAMPSEC:
-        generator.writeNumber(((TimeStampSecVector) valueVector).getAccessor().get(i));
-        break;
-      case TIMESTAMPMILLI:
-        generator.writeNumber(((TimeStampMilliVector) valueVector).getAccessor().get(i));
-        break;
-      case TIMESTAMPMICRO:
-        generator.writeNumber(((TimeStampMicroVector) valueVector).getAccessor().get(i));
-        break;
-      case TIMESTAMPNANO:
-        generator.writeNumber(((TimeStampNanoVector) valueVector).getAccessor().get(i));
-        break;
-      case BIT:
-        generator.writeNumber(((BitVector) valueVector).getAccessor().get(i));
-        break;
-      case VARBINARY: {
-          String hexString = Hex.encodeHexString(((VarBinaryVector) valueVector).getAccessor().get(i));
-          generator.writeString(hexString);
+  private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer,
+                                     ArrowBuf offsetBuffer, FieldVector vector, int index) throws IOException {
+    if (bufferType.equals(TYPE)) {
+      generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH));
+    } else if (bufferType.equals(OFFSET)) {
+      generator.writeNumber(buffer.getInt(index * BaseNullableVariableWidthVector.OFFSET_WIDTH));
+    } else if(bufferType.equals(VALIDITY)) {
+      generator.writeNumber(vector.isNull(index) ? 0 : 1);
+    } else if (bufferType.equals(DATA)) {
+      switch (vector.getMinorType()) {
+        case TINYINT:
+          generator.writeNumber(NullableTinyIntVector.get(buffer, index));
+          break;
+        case SMALLINT:
+          generator.writeNumber(NullableSmallIntVector.get(buffer, index));
+          break;
+        case INT:
+          generator.writeNumber(NullableIntVector.get(buffer, index));
+          break;
+        case BIGINT:
+          generator.writeNumber(NullableBigIntVector.get(buffer, index));
+          break;
+        case FLOAT4:
+          generator.writeNumber(NullableFloat4Vector.get(buffer, index));
+          break;
+        case FLOAT8:
+          generator.writeNumber(NullableFloat8Vector.get(buffer, index));
+          break;
+        case DATEDAY:
+          generator.writeNumber(NullableDateDayVector.get(buffer, index));
+          break;
+        case DATEMILLI:
+          generator.writeNumber(NullableDateMilliVector.get(buffer, index));
+          break;
+        case TIMESEC:
+          generator.writeNumber(NullableTimeSecVector.get(buffer, index));
+          break;
+        case TIMEMILLI:
+          generator.writeNumber(NullableTimeMilliVector.get(buffer, index));
+          break;
+        case TIMEMICRO:
+          generator.writeNumber(NullableTimeMicroVector.get(buffer, index));
+          break;
+        case TIMENANO:
+          generator.writeNumber(NullableTimeNanoVector.get(buffer, index));
+          break;
+        case TIMESTAMPSEC:
+          generator.writeNumber(NullableTimeStampSecVector.get(buffer, index));
+          break;
+        case TIMESTAMPMILLI:
+          generator.writeNumber(NullableTimeStampMilliVector.get(buffer, index));
+          break;
+        case TIMESTAMPMICRO:
+          generator.writeNumber(NullableTimeStampMicroVector.get(buffer, index));
+          break;
+        case TIMESTAMPNANO:
+          generator.writeNumber(NullableTimeStampNanoVector.get(buffer, index));
+          break;
+        case TIMESTAMPSECTZ:
+          generator.writeNumber(NullableTimeStampSecTZVector.get(buffer, index));
+          break;
+        case TIMESTAMPMILLITZ:
+          generator.writeNumber(NullableTimeStampMilliTZVector.get(buffer, index));
+          break;
+        case TIMESTAMPMICROTZ:
+          generator.writeNumber(NullableTimeStampMicroTZVector.get(buffer, index));
+          break;
+        case TIMESTAMPNANOTZ:
+          generator.writeNumber(NullableTimeStampNanoTZVector.get(buffer, index));
+          break;
+        case BIT:
+          generator.writeNumber(BitVectorHelper.get(buffer, index));
+          break;
+        case VARBINARY: {
+          assert offsetBuffer != null;
+          String hexString = Hex.encodeHexString(BaseNullableVariableWidthVector.get(buffer,
+                  offsetBuffer, index));
+          generator.writeObject(hexString);
+          break;
         }
-        break;
-      case DECIMAL: {
-          BigDecimal decimalValue = ((DecimalVector) valueVector).getAccessor().getObject(i);
-          // We write the unscaled value, because the scale is stored in the type metadata.
-          generator.writeString(decimalValue.unscaledValue().toString());
+        case VARCHAR: {
+          assert offsetBuffer != null;
+          byte[] b = (BaseNullableVariableWidthVector.get(buffer, offsetBuffer, index));
+          generator.writeString(new String(b, "UTF-8"));
+          break;
         }
-        break;
-      default:
-        // TODO: each type
-        Accessor accessor = valueVector.getAccessor();
-        Object value = accessor.getObject(i);
-        if (value instanceof Number || value instanceof Boolean) {
-          generator.writeObject(value);
-        } else {
-          generator.writeObject(value.toString());
+        case DECIMAL: {
+          String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(buffer,
+                  index));
+          generator.writeString(hexString);
+          break;
         }
-        break;
+        default:
+          throw new UnsupportedOperationException("minor type: " + vector.getMinorType());
+      }
     }
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
index 5851bd5fa5d97..c27e5e5c85c30 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
@@ -114,13 +114,13 @@ public static void compareFieldVectors(FieldVector vector1, FieldVector vector2)
     if (!field1.equals(vector2.getField())) {
       throw new IllegalArgumentException("Different Fields:\n" + field1 + "\n!=\n" + vector2.getField());
     }
-    int valueCount = vector1.getAccessor().getValueCount();
-    if (valueCount != vector2.getAccessor().getValueCount()) {
-      throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getAccessor().getValueCount());
+    int valueCount = vector1.getValueCount();
+    if (valueCount != vector2.getValueCount()) {
+      throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getValueCount());
     }
     for (int j = 0; j < valueCount; j++) {
-      Object obj1 = vector1.getAccessor().getObject(j);
-      Object obj2 = vector2.getAccessor().getObject(j);
+      Object obj1 = vector1.getObject(j);
+      Object obj2 = vector2.getObject(j);
       if (!equals(field1.getType(), obj1, obj2)) {
         throw new IllegalArgumentException(
             "Different values in column:\n" + field1 + " at index " + j + ": " + obj1 + " != " + obj2);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
index 17fcf05fcd9bd..ada341497878e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -340,59 +340,56 @@ public void testReallocAfterVectorTransfer2() {
       int valueCapacity = vector.getValueCapacity();
       assertEquals(4096, valueCapacity);
 
-      final NullableBitVector.Mutator mutator = vector.getMutator();
-      final NullableBitVector.Accessor accessor = vector.getAccessor();
-
       for (int i = 0; i < valueCapacity; i++) {
         if ((i & 1) == 1) {
-          mutator.set(i, 1);
+          vector.set(i, 1);
         }
       }
 
       for (int i = 0; i < valueCapacity; i++) {
         if ((i & 1) == 1) {
-          assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i));
+          assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i));
         }
         else {
-          assertTrue("unexpected set bit at index: " + i, accessor.isNull(i));
+          assertTrue("unexpected set bit at index: " + i, vector.isNull(i));
         }
       }
 
       /* trigger first realloc */
-      mutator.setSafe(valueCapacity, 1, 1);
+      vector.setSafe(valueCapacity, 1, 1);
       assertEquals(valueCapacity * 2, vector.getValueCapacity());
 
       for (int i = valueCapacity; i < valueCapacity*2; i++) {
         if ((i & 1) == 1) {
-          mutator.set(i, 1);
+          vector.set(i, 1);
         }
       }
 
       for (int i = 0; i < valueCapacity*2; i++) {
         if (((i & 1) == 1) || (i == valueCapacity)) {
-          assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i));
+          assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i));
         }
         else {
-          assertTrue("unexpected set bit at index: " + i, accessor.isNull(i));
+          assertTrue("unexpected set bit at index: " + i, vector.isNull(i));
         }
       }
 
       /* trigger second realloc */
-      mutator.setSafe(valueCapacity*2, 1, 1);
+      vector.setSafe(valueCapacity*2, 1, 1);
       assertEquals(valueCapacity * 4, vector.getValueCapacity());
 
       for (int i = valueCapacity*2; i < valueCapacity*4; i++) {
         if ((i & 1) == 1) {
-          mutator.set(i, 1);
+          vector.set(i, 1);
         }
       }
 
       for (int i = 0; i < valueCapacity*4; i++) {
         if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2)) {
-          assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i));
+          assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i));
         }
         else {
-          assertTrue("unexpected set bit at index: " + i, accessor.isNull(i));
+          assertTrue("unexpected set bit at index: " + i, vector.isNull(i));
         }
       }
 
@@ -400,26 +397,24 @@ public void testReallocAfterVectorTransfer2() {
       TransferPair transferPair = vector.getTransferPair(allocator);
       transferPair.transfer();
       final NullableBitVector toVector = (NullableBitVector)transferPair.getTo();
-      final NullableBitVector.Accessor toAccessor = toVector.getAccessor();
-      final NullableBitVector.Mutator toMutator = toVector.getMutator();
 
       assertEquals(valueCapacity * 4, toVector.getValueCapacity());
 
       /* realloc the toVector */
-      toMutator.setSafe(valueCapacity * 4, 1, 1);
+      toVector.setSafe(valueCapacity * 4, 1, 1);
 
       for (int i = 0; i < toVector.getValueCapacity(); i++) {
         if (i <= valueCapacity * 4) {
           if (((i & 1) == 1) || (i == valueCapacity) ||
                   (i == valueCapacity*2) || (i == valueCapacity*4)) {
-            assertFalse("unexpected cleared bit at index: " + i, toAccessor.isNull(i));
+            assertFalse("unexpected cleared bit at index: " + i, toVector.isNull(i));
           }
           else {
-            assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i));
+            assertTrue("unexpected set bit at index: " + i, toVector.isNull(i));
           }
         }
         else {
-          assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i));
+          assertTrue("unexpected set bit at index: " + i, toVector.isNull(i));
         }
       }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index 56d2293276404..4d844d6d3ca0f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -71,14 +71,14 @@ public void testValuesWriteRead() {
       for (int i = 0; i < intValues.length; i++) {
         BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale);
         values[i] = decimal;
-        decimalVector.getMutator().setSafe(i, decimal);
+        decimalVector.setSafe(i, decimal);
       }
 
-      decimalVector.getMutator().setValueCount(intValues.length);
+      decimalVector.setValueCount(intValues.length);
 
       for (int i = 0; i < intValues.length; i++) {
-        BigDecimal value = decimalVector.getAccessor().getObject(i);
-        assertEquals(values[i], value);
+        BigDecimal value = decimalVector.getObject(i);
+        assertEquals("unexpected data at index: " + i, values[i], value);
       }
     }
   }
@@ -92,7 +92,7 @@ public void testBigDecimalDifferentScaleAndPrecision() {
       boolean hasError = false;
       try {
         BigDecimal decimal = new BigDecimal(BigInteger.valueOf(0), 3);
-        decimalVector.getMutator().setSafe(0, decimal);
+        decimalVector.setSafe(0, decimal);
       } catch (UnsupportedOperationException ue) {
         hasError = true;
       } finally {
@@ -103,7 +103,7 @@ public void testBigDecimalDifferentScaleAndPrecision() {
       hasError = false;
       try {
         BigDecimal decimal = new BigDecimal(BigInteger.valueOf(12345), 2);
-        decimalVector.getMutator().setSafe(0, decimal);
+        decimalVector.setSafe(0, decimal);
       } catch (UnsupportedOperationException ue) {
         hasError = true;
       } finally {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
index 1185246f88cde..46a2bafa00208 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -135,9 +135,9 @@ public void testEncodeLargeVector() {
         // now run through the decoder and verify we get the original back
         try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) {
           assertEquals(vector.getClass(), decoded.getClass());
-          assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount());
+          assertEquals(vector.getValueCount(), decoded.getValueCount());
           for (int i = 0; i < count; ++i) {
-            assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i));
+            assertEquals(vector.getObject(i), decoded.getObject(i));
           }
         }
       }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index 168deacd6a640..2af6cd53aa0c1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -57,7 +57,7 @@ public void testIntType() {
       vector.allocateNew();
 
       for (int i = 0; i < 10; i++) {
-        vector.getMutator().setNotNull(i);
+        vector.setNotNull(i);
         nested.set(i * 2, i);
         nested.set(i * 2 + 1, i + 10);
       }
@@ -81,17 +81,16 @@ public void testIntType() {
   public void testFloatTypeNullable() {
     try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) {
       NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
-      NullableFloat4Vector.Mutator mutator = nested.getMutator();
       vector.allocateNew();
 
       for (int i = 0; i < 10; i++) {
         if (i % 2 == 0) {
-          vector.getMutator().setNotNull(i);
-          mutator.set(i * 2, i + 0.1f);
-          mutator.set(i * 2 + 1, i + 10.1f);
+          vector.setNotNull(i);
+          nested.set(i * 2, i + 0.1f);
+          nested.set(i * 2 + 1, i + 10.1f);
         }
       }
-      vector.getMutator().setValueCount(10);
+      vector.setValueCount(10);
 
       UnionFixedSizeListReader reader = vector.getReader();
       for (int i = 0; i < 10; i++) {
@@ -115,24 +114,22 @@ public void testFloatTypeNullable() {
   @Test
   public void testNestedInList() {
     try (ListVector vector = ListVector.empty("list", allocator)) {
-      ListVector.Mutator mutator = vector.getMutator();
       FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector();
-      FixedSizeListVector.Mutator tupleMutator = tuples.getMutator();
       NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
       vector.allocateNew();
 
       for (int i = 0; i < 10; i++) {
         if (i % 2 == 0) {
-          int position = mutator.startNewValue(i);
+          int position = vector.startNewValue(i);
           for (int j = 0; j < i % 7; j++) {
-            tupleMutator.setNotNull(position + j);
+            tuples.setNotNull(position + j);
             innerVector.set((position + j) * 2, j);
             innerVector.set((position + j) * 2 + 1, j + 1);
           }
-          mutator.endValue(i, i % 7);
+          vector.endValue(i, i % 7);
         }
       }
-      mutator.setValueCount(10);
+      vector.setValueCount(10);
 
       UnionListReader reader = vector.getReader();
       for (int i = 0; i < 10; i++) {
@@ -161,24 +158,24 @@ public void testTransferPair() {
     try (FixedSizeListVector from = new FixedSizeListVector("from", allocator, 2, null, null);
          FixedSizeListVector to = new FixedSizeListVector("to", allocator, 2, null, null)) {
       NullableFloat4Vector nested = (NullableFloat4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
-      NullableFloat4Vector.Mutator mutator = nested.getMutator();
       from.allocateNew();
 
       for (int i = 0; i < 10; i++) {
         if (i % 2 == 0) {
-          from.getMutator().setNotNull(i);
-          mutator.set(i * 2, i + 0.1f);
-          mutator.set(i * 2 + 1, i + 10.1f);
+          from.setNotNull(i);
+          nested.set(i * 2, i + 0.1f);
+          nested.set(i * 2 + 1, i + 10.1f);
         }
       }
-      from.getMutator().setValueCount(10);
+      from.setValueCount(10);
 
       TransferPair pair = from.makeTransferPair(to);
 
       pair.copyValueSafe(0, 1);
       pair.copyValueSafe(2, 2);
       to.copyFromSafe(4, 3, from);
-      to.getMutator().setValueCount(10);
+
+      to.setValueCount(10);
 
       UnionFixedSizeListReader reader = to.getReader();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 59e1646e86e56..f6aa86a300876 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -85,7 +85,7 @@ public void testCopyFrom() throws Exception {
       for (int i = 0; i < 3; i++) {
         outVector.copyFrom(i, i, inVector);
       }
-      outVector.getMutator().setValueCount(3);
+      outVector.setValueCount(3);
 
       // assert the output vector is correct
       FieldReader reader = outVector.getReader();
@@ -95,11 +95,9 @@ public void testCopyFrom() throws Exception {
       reader.setPosition(2);
       Assert.assertTrue("shouldn't be null", reader.isSet());
 
-      /* check the exact contents of vector */
-      final ListVector.Accessor accessor = outVector.getAccessor();
 
       /* index 0 */
-      Object result = accessor.getObject(0);
+      Object result = outVector.getObject(0);
       ArrayList<Long> resultSet = (ArrayList<Long>) result;
       assertEquals(3, resultSet.size());
       assertEquals(new Long(1), (Long) resultSet.get(0));
@@ -107,11 +105,11 @@ public void testCopyFrom() throws Exception {
       assertEquals(new Long(3), (Long) resultSet.get(2));
 
       /* index 1 */
-      result = accessor.getObject(1);
+      result = outVector.getObject(1);
       assertNull(result);
 
       /* index 2 */
-      result = accessor.getObject(2);
+      result = outVector.getObject(2);
       resultSet = (ArrayList<Long>) result;
       assertEquals(0, resultSet.size());
     }
@@ -128,46 +126,46 @@ public void testSetLastSetUsage() throws Exception {
       /* allocate memory */
       listVector.allocateNew();
 
-      /* get inner vectors; bitVector and offsetVector */
-      List<BufferBacked> innerVectors = listVector.getFieldInnerVectors();
-      BitVector bitVector = (BitVector) innerVectors.get(0);
-      UInt4Vector offsetVector = (UInt4Vector) innerVectors.get(1);
+      /* get inner buffers; validityBuffer and offsetBuffer */
+
+      ArrowBuf validityBuffer = listVector.getValidityBuffer();
+      ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
 
       /* get the underlying data vector -- NullableBigIntVector */
       NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector();
 
       /* check current lastSet */
-      assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet()));
+      assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet()));
 
       int index = 0;
       int offset = 0;
 
-      /* write [10, 11, 12] to the list vector at index */
-      bitVector.getMutator().setSafe(index, 1);
-      dataVector.getMutator().setSafe(0, 1, 10);
-      dataVector.getMutator().setSafe(1, 1, 11);
-      dataVector.getMutator().setSafe(2, 1, 12);
-      offsetVector.getMutator().setSafe(index + 1, 3);
+      /* write [10, 11, 12] to the list vector at index 0 */
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      dataVector.setSafe(0, 1, 10);
+      dataVector.setSafe(1, 1, 11);
+      dataVector.setSafe(2, 1, 12);
+      offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 3);
 
       index += 1;
 
       /* write [13, 14] to the list vector at index 1 */
-      bitVector.getMutator().setSafe(index, 1);
-      dataVector.getMutator().setSafe(3, 1, 13);
-      dataVector.getMutator().setSafe(4, 1, 14);
-      offsetVector.getMutator().setSafe(index + 1, 5);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      dataVector.setSafe(3, 1, 13);
+      dataVector.setSafe(4, 1, 14);
+      offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 5);
 
       index += 1;
 
       /* write [15, 16, 17] to the list vector at index 2 */
-      bitVector.getMutator().setSafe(index, 1);
-      dataVector.getMutator().setSafe(5, 1, 15);
-      dataVector.getMutator().setSafe(6, 1, 16);
-      dataVector.getMutator().setSafe(7, 1, 17);
-      offsetVector.getMutator().setSafe(index + 1, 8);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      dataVector.setSafe(5, 1, 15);
+      dataVector.setSafe(6, 1, 16);
+      dataVector.setSafe(7, 1, 17);
+      offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 8);
 
       /* check current lastSet */
-      assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet()));
+      assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet()));
 
       /* set lastset and arbitrary valuecount for list vector.
        *
@@ -208,54 +206,50 @@ public void testSetLastSetUsage() throws Exception {
        *                [15, 16, 17]
        *              }
        */
-      listVector.getMutator().setLastSet(3);
-      listVector.getMutator().setValueCount(10);
-
-      /* check the vector output */
-      final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
-      final ValueVector.Accessor valueAccessor = dataVector.getAccessor();
+      listVector.setLastSet(3);
+      listVector.setValueCount(10);
 
       index = 0;
-      offset = offsetAccessor.get(index);
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(0), Integer.toString(offset));
 
-      Object actual = valueAccessor.getObject(offset);
+      Object actual = dataVector.getObject(offset);
       assertEquals(new Long(10), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(11), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(12), (Long) actual);
 
       index++;
-      offset = offsetAccessor.get(index);
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(3), Integer.toString(offset));
 
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(13), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(14), (Long) actual);
 
       index++;
-      offset = offsetAccessor.get(index);
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(5), Integer.toString(offset));
 
-      actual = valueAccessor.getObject(offsetAccessor.get(index));
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(15), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(16), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(17), (Long) actual);
 
       index++;
-      offset = offsetAccessor.get(index);
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(8), Integer.toString(offset));
 
-      actual = valueAccessor.getObject(offsetAccessor.get(index));
+      actual = dataVector.getObject(offset);
       assertNull(actual);
     }
   }
@@ -308,99 +302,97 @@ public void testSplitAndTransfer() throws Exception {
       listWriter.bigInt().writeBigInt(23);
       listWriter.endList();
 
-      listVector.getMutator().setValueCount(5);
+      listVector.setValueCount(5);
 
-      assertEquals(5, listVector.getMutator().getLastSet());
+      assertEquals(5, listVector.getLastSet());
 
-      /* get offsetVector */
-      UInt4Vector offsetVector = (UInt4Vector) listVector.getOffsetVector();
+      /* get offset buffer */
+      final ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
 
       /* get dataVector */
       NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector();
 
       /* check the vector output */
-      final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
-      final ValueVector.Accessor valueAccessor = dataVector.getAccessor();
 
       int index = 0;
       int offset = 0;
       Object actual = null;
 
       /* index 0 */
-      assertFalse(listVector.getAccessor().isNull(index));
-      offset = offsetAccessor.get(index);
+      assertFalse(listVector.isNull(index));
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(0), Integer.toString(offset));
 
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(10), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(11), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(12), (Long) actual);
 
       /* index 1 */
       index++;
-      assertFalse(listVector.getAccessor().isNull(index));
-      offset = offsetAccessor.get(index);
+      assertFalse(listVector.isNull(index));
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(3), Integer.toString(offset));
 
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(13), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(14), (Long) actual);
 
       /* index 2 */
       index++;
-      assertFalse(listVector.getAccessor().isNull(index));
-      offset = offsetAccessor.get(index);
+      assertFalse(listVector.isNull(index));
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(5), Integer.toString(offset));
 
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(15), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(16), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(17), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(18), (Long) actual);
 
       /* index 3 */
       index++;
-      assertFalse(listVector.getAccessor().isNull(index));
-      offset = offsetAccessor.get(index);
+      assertFalse(listVector.isNull(index));
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(9), Integer.toString(offset));
 
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(19), (Long) actual);
 
       /* index 4 */
       index++;
-      assertFalse(listVector.getAccessor().isNull(index));
-      offset = offsetAccessor.get(index);
+      assertFalse(listVector.isNull(index));
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(10), Integer.toString(offset));
 
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(20), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(21), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(22), (Long) actual);
       offset++;
-      actual = valueAccessor.getObject(offset);
+      actual = dataVector.getObject(offset);
       assertEquals(new Long(23), (Long) actual);
 
       /* index 5 */
       index++;
-      assertTrue(listVector.getAccessor().isNull(index));
-      offset = offsetAccessor.get(index);
+      assertTrue(listVector.isNull(index));
+      offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(14), Integer.toString(offset));
 
       /* do split and transfer */
@@ -423,26 +415,26 @@ public void testSplitAndTransfer() throws Exception {
           transferPair.splitAndTransfer(start, splitLength);
 
           /* get offsetVector of toVector */
-          UInt4Vector offsetVector1 = (UInt4Vector) toVector.getOffsetVector();
-          UInt4Vector.Accessor offsetAccessor1 = offsetVector1.getAccessor();
+          final ArrowBuf toOffsetBuffer = toVector.getOffsetBuffer();
 
           /* get dataVector of toVector */
           NullableBigIntVector dataVector1 = (NullableBigIntVector) toVector.getDataVector();
-          NullableBigIntVector.Accessor valueAccessor1 = dataVector1.getAccessor();
 
           for (int i = 0; i < splitLength; i++) {
-            dataLength1 = offsetAccessor.get(start + i + 1) - offsetAccessor.get(start + i);
-            dataLength2 = offsetAccessor1.get(i + 1) - offsetAccessor1.get(i);
+            dataLength1 = offsetBuffer.getInt((start + i + 1) * ListVector.OFFSET_WIDTH) -
+                    offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH);
+            dataLength2 = toOffsetBuffer.getInt((i + 1) * ListVector.OFFSET_WIDTH)
+                    - toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH);
 
             assertEquals("Different data lengths at index: " + i + " and start: " + start,
                     dataLength1, dataLength2);
 
-            offset1 = offsetAccessor.get(start + i);
-            offset2 = offsetAccessor1.get(i);
+            offset1 = offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH);
+            offset2 = toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH);
 
             for (int j = 0; j < dataLength1; j++) {
               assertEquals("Different data at indexes: " + offset1 + " and " + offset2,
-                      valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
+                      dataVector.getObject(offset1), dataVector1.getObject(offset2));
 
               offset1++;
               offset2++;
@@ -506,15 +498,14 @@ public void testNestedListVector() throws Exception {
 
       listWriter.endList();
 
-      assertEquals(2, listVector.getMutator().getLastSet());
+      assertEquals(2, listVector.getLastSet());
 
-      listVector.getMutator().setValueCount(2);
+      listVector.setValueCount(2);
 
-      final ListVector.Accessor accessor = listVector.getAccessor();
-      assertEquals(2, accessor.getValueCount());
+      assertEquals(2, listVector.getValueCount());
 
       /* get listVector value at index 0 -- the value itself is a listvector */
-      Object result = accessor.getObject(0);
+      Object result = listVector.getObject(0);
       ArrayList<ArrayList<Long>> resultSet = (ArrayList<ArrayList<Long>>) result;
       ArrayList<Long> list;
 
@@ -534,7 +525,7 @@ public void testNestedListVector() throws Exception {
       assertEquals(new Long(175), list.get(3));
 
        /* get listVector value at index 1 -- the value itself is a listvector */
-      result = accessor.getObject(1);
+      result = listVector.getObject(1);
       resultSet = (ArrayList<ArrayList<Long>>) result;
 
       assertEquals(3, resultSet.size());              /* 3 inner lists at index 1 */
@@ -555,17 +546,16 @@ public void testNestedListVector() throws Exception {
       assertEquals(new Long(35), list.get(2));
 
       /* check underlying bitVector */
-      assertFalse(accessor.isNull(0));
-      assertFalse(accessor.isNull(1));
+      assertFalse(listVector.isNull(0));
+      assertFalse(listVector.isNull(1));
 
-      /* check underlying offsetVector */
-      UInt4Vector offsetVector = listVector.getOffsetVector();
-      final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor();
+      /* check underlying offsets */
+      final ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
 
       /* listVector has 2 lists at index 0 and 3 lists at index 1 */
-      assertEquals(0, offsetAccessor.get(0));
-      assertEquals(2, offsetAccessor.get(1));
-      assertEquals(5, offsetAccessor.get(2));
+      assertEquals(0, offsetBuffer.getInt(0 * ListVector.OFFSET_WIDTH));
+      assertEquals(2, offsetBuffer.getInt(1 * ListVector.OFFSET_WIDTH));
+      assertEquals(5, offsetBuffer.getInt(2 * ListVector.OFFSET_WIDTH));
     }
   }
 
@@ -591,17 +581,15 @@ public void testGetBufferAddress() throws Exception {
       listWriter.bigInt().writeBigInt(300);
       listWriter.endList();
 
-      final ListVector.Accessor accessor = listVector.getAccessor();
-
       /* check listVector contents */
-      Object result = accessor.getObject(0);
+      Object result = listVector.getObject(0);
       ArrayList<Long> resultSet = (ArrayList<Long>) result;
       assertEquals(3, resultSet.size());
       assertEquals(new Long(50), resultSet.get(0));
       assertEquals(new Long(100), resultSet.get(1));
       assertEquals(new Long(200), resultSet.get(2));
 
-      result = accessor.getObject(1);
+      result = listVector.getObject(1);
       resultSet = (ArrayList<Long>) result;
       assertEquals(2, resultSet.size());
       assertEquals(new Long(250), resultSet.get(0));
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index 86f0bf337f9d4..ede4f992291b4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -64,26 +64,24 @@ public void testUnionVector() throws Exception {
       unionVector.allocateNew();
 
       // write some data
-      final UnionVector.Mutator mutator = unionVector.getMutator();
-      mutator.setType(0, Types.MinorType.UINT4);
-      mutator.setSafe(0, uInt4Holder);
-      mutator.setType(2, Types.MinorType.UINT4);
-      mutator.setSafe(2, uInt4Holder);
-      mutator.setValueCount(4);
+      unionVector.setType(0, Types.MinorType.UINT4);
+      unionVector.setSafe(0, uInt4Holder);
+      unionVector.setType(2, Types.MinorType.UINT4);
+      unionVector.setSafe(2, uInt4Holder);
+      unionVector.setValueCount(4);
 
       // check that what we wrote is correct
-      final UnionVector.Accessor accessor = unionVector.getAccessor();
-      assertEquals(4, accessor.getValueCount());
+      assertEquals(4, unionVector.getValueCount());
 
-      assertEquals(false, accessor.isNull(0));
-      assertEquals(100, accessor.getObject(0));
+      assertEquals(false, unionVector.isNull(0));
+      assertEquals(100, unionVector.getObject(0));
 
-      assertEquals(true, accessor.isNull(1));
+      assertEquals(true, unionVector.isNull(1));
 
-      assertEquals(false, accessor.isNull(2));
-      assertEquals(100, accessor.getObject(2));
+      assertEquals(false, unionVector.isNull(2));
+      assertEquals(100, unionVector.getObject(2));
 
-      assertEquals(true, accessor.isNull(3));
+      assertEquals(true, unionVector.isNull(3));
     }
   }
 
@@ -93,16 +91,15 @@ public void testTransfer() throws Exception {
       srcVector.allocateNew();
 
       // write some data
-      final UnionVector.Mutator mutator = srcVector.getMutator();
-      mutator.setType(0, MinorType.INT);
-      mutator.setSafe(0, newIntHolder(5));
-      mutator.setType(1, MinorType.BIT);
-      mutator.setSafe(1, newBitHolder(false));
-      mutator.setType(3, MinorType.INT);
-      mutator.setSafe(3, newIntHolder(10));
-      mutator.setType(5, MinorType.BIT);
-      mutator.setSafe(5, newBitHolder(false));
-      mutator.setValueCount(6);
+      srcVector.setType(0, MinorType.INT);
+      srcVector.setSafe(0, newIntHolder(5));
+      srcVector.setType(1, MinorType.BIT);
+      srcVector.setSafe(1, newBitHolder(false));
+      srcVector.setType(3, MinorType.INT);
+      srcVector.setSafe(3, newIntHolder(10));
+      srcVector.setType(5, MinorType.BIT);
+      srcVector.setSafe(5, newBitHolder(false));
+      srcVector.setValueCount(6);
 
       try (UnionVector destVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
         TransferPair pair = srcVector.makeTransferPair(destVector);
@@ -116,9 +113,9 @@ public void testTransfer() throws Exception {
         assertEquals(srcVector.getField(), destVector.getField());
 
         // now check the values are transferred
-        assertEquals(srcVector.getAccessor().getValueCount(), destVector.getAccessor().getValueCount());
-        for (int i = 0; i < srcVector.getAccessor().getValueCount(); i++) {
-          assertEquals("Different values at index " + i, srcVector.getAccessor().get(i), destVector.getAccessor().get(i));
+        assertEquals(srcVector.getValueCount(), destVector.getValueCount());
+        for (int i = 0; i < srcVector.getValueCount(); i++) {
+          assertEquals("Different values at index " + i, srcVector.get(i), destVector.get(i));
         }
       }
     }
@@ -127,61 +124,58 @@ public void testTransfer() throws Exception {
   @Test
   public void testSplitAndTransfer() throws Exception {
     try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
-      final UnionVector.Mutator sourceMutator = sourceVector.getMutator();
-      final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor();
 
       sourceVector.allocateNew();
 
       /* populate the UnionVector */
-      sourceMutator.setType(0, MinorType.INT);
-      sourceMutator.setSafe(0, newIntHolder(5));
-      sourceMutator.setType(1, MinorType.INT);
-      sourceMutator.setSafe(1, newIntHolder(10));
-      sourceMutator.setType(2, MinorType.INT);
-      sourceMutator.setSafe(2, newIntHolder(15));
-      sourceMutator.setType(3, MinorType.INT);
-      sourceMutator.setSafe(3, newIntHolder(20));
-      sourceMutator.setType(4, MinorType.INT);
-      sourceMutator.setSafe(4, newIntHolder(25));
-      sourceMutator.setType(5, MinorType.INT);
-      sourceMutator.setSafe(5, newIntHolder(30));
-      sourceMutator.setType(6, MinorType.INT);
-      sourceMutator.setSafe(6, newIntHolder(35));
-      sourceMutator.setType(7, MinorType.INT);
-      sourceMutator.setSafe(7, newIntHolder(40));
-      sourceMutator.setType(8, MinorType.INT);
-      sourceMutator.setSafe(8, newIntHolder(45));
-      sourceMutator.setType(9, MinorType.INT);
-      sourceMutator.setSafe(9, newIntHolder(50));
-      sourceMutator.setValueCount(10);
+      sourceVector.setType(0, MinorType.INT);
+      sourceVector.setSafe(0, newIntHolder(5));
+      sourceVector.setType(1, MinorType.INT);
+      sourceVector.setSafe(1, newIntHolder(10));
+      sourceVector.setType(2, MinorType.INT);
+      sourceVector.setSafe(2, newIntHolder(15));
+      sourceVector.setType(3, MinorType.INT);
+      sourceVector.setSafe(3, newIntHolder(20));
+      sourceVector.setType(4, MinorType.INT);
+      sourceVector.setSafe(4, newIntHolder(25));
+      sourceVector.setType(5, MinorType.INT);
+      sourceVector.setSafe(5, newIntHolder(30));
+      sourceVector.setType(6, MinorType.INT);
+      sourceVector.setSafe(6, newIntHolder(35));
+      sourceVector.setType(7, MinorType.INT);
+      sourceVector.setSafe(7, newIntHolder(40));
+      sourceVector.setType(8, MinorType.INT);
+      sourceVector.setSafe(8, newIntHolder(45));
+      sourceVector.setType(9, MinorType.INT);
+      sourceVector.setSafe(9, newIntHolder(50));
+      sourceVector.setValueCount(10);
 
       /* check the vector output */
-      assertEquals(10, sourceAccessor.getValueCount());
-      assertEquals(false, sourceAccessor.isNull(0));
-      assertEquals(5, sourceAccessor.getObject(0));
-      assertEquals(false, sourceAccessor.isNull(1));
-      assertEquals(10, sourceAccessor.getObject(1));
-      assertEquals(false, sourceAccessor.isNull(2));
-      assertEquals(15, sourceAccessor.getObject(2));
-      assertEquals(false, sourceAccessor.isNull(3));
-      assertEquals(20, sourceAccessor.getObject(3));
-      assertEquals(false, sourceAccessor.isNull(4));
-      assertEquals(25, sourceAccessor.getObject(4));
-      assertEquals(false, sourceAccessor.isNull(5));
-      assertEquals(30, sourceAccessor.getObject(5));
-      assertEquals(false, sourceAccessor.isNull(6));
-      assertEquals(35, sourceAccessor.getObject(6));
-      assertEquals(false, sourceAccessor.isNull(7));
-      assertEquals(40, sourceAccessor.getObject(7));
-      assertEquals(false, sourceAccessor.isNull(8));
-      assertEquals(45, sourceAccessor.getObject(8));
-      assertEquals(false, sourceAccessor.isNull(9));
-      assertEquals(50, sourceAccessor.getObject(9));
+      assertEquals(10, sourceVector.getValueCount());
+      assertEquals(false, sourceVector.isNull(0));
+      assertEquals(5, sourceVector.getObject(0));
+      assertEquals(false, sourceVector.isNull(1));
+      assertEquals(10, sourceVector.getObject(1));
+      assertEquals(false, sourceVector.isNull(2));
+      assertEquals(15, sourceVector.getObject(2));
+      assertEquals(false, sourceVector.isNull(3));
+      assertEquals(20, sourceVector.getObject(3));
+      assertEquals(false, sourceVector.isNull(4));
+      assertEquals(25, sourceVector.getObject(4));
+      assertEquals(false, sourceVector.isNull(5));
+      assertEquals(30, sourceVector.getObject(5));
+      assertEquals(false, sourceVector.isNull(6));
+      assertEquals(35, sourceVector.getObject(6));
+      assertEquals(false, sourceVector.isNull(7));
+      assertEquals(40, sourceVector.getObject(7));
+      assertEquals(false, sourceVector.isNull(8));
+      assertEquals(45, sourceVector.getObject(8));
+      assertEquals(false, sourceVector.isNull(9));
+      assertEquals(50, sourceVector.getObject(9));
 
       try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
 
         final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
-        final UnionVector.Accessor toAccessor = toVector.getAccessor();
 
         final int[][] transferLengths = {{0, 3},
             {3, 1},
@@ -199,8 +193,8 @@ public void testSplitAndTransfer() throws Exception {
 
           /* check the toVector output after doing the splitAndTransfer */
           for (int i = 0; i < length; i++) {
-            assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceAccessor.getObject(start + i),
-                toAccessor.getObject(i));
+            assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceVector.getObject(start + i),
+                toVector.getObject(i));
           }
         }
       }
@@ -210,70 +204,67 @@ public void testSplitAndTransfer() throws Exception {
   @Test
   public void testSplitAndTransferWithMixedVectors() throws Exception {
     try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
-      final UnionVector.Mutator sourceMutator = sourceVector.getMutator();
-      final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor();
 
       sourceVector.allocateNew();
 
       /* populate the UnionVector */
-      sourceMutator.setType(0, MinorType.INT);
-      sourceMutator.setSafe(0, newIntHolder(5));
+      sourceVector.setType(0, MinorType.INT);
+      sourceVector.setSafe(0, newIntHolder(5));
 
-      sourceMutator.setType(1, MinorType.FLOAT4);
-      sourceMutator.setSafe(1, newFloat4Holder(5.5f));
+      sourceVector.setType(1, MinorType.FLOAT4);
+      sourceVector.setSafe(1, newFloat4Holder(5.5f));
 
-      sourceMutator.setType(2, MinorType.INT);
-      sourceMutator.setSafe(2, newIntHolder(10));
+      sourceVector.setType(2, MinorType.INT);
+      sourceVector.setSafe(2, newIntHolder(10));
 
-      sourceMutator.setType(3, MinorType.FLOAT4);
-      sourceMutator.setSafe(3, newFloat4Holder(10.5f));
+      sourceVector.setType(3, MinorType.FLOAT4);
+      sourceVector.setSafe(3, newFloat4Holder(10.5f));
 
-      sourceMutator.setType(4, MinorType.INT);
-      sourceMutator.setSafe(4, newIntHolder(15));
+      sourceVector.setType(4, MinorType.INT);
+      sourceVector.setSafe(4, newIntHolder(15));
 
-      sourceMutator.setType(5, MinorType.FLOAT4);
-      sourceMutator.setSafe(5, newFloat4Holder(15.5f));
+      sourceVector.setType(5, MinorType.FLOAT4);
+      sourceVector.setSafe(5, newFloat4Holder(15.5f));
 
-      sourceMutator.setType(6, MinorType.INT);
-      sourceMutator.setSafe(6, newIntHolder(20));
+      sourceVector.setType(6, MinorType.INT);
+      sourceVector.setSafe(6, newIntHolder(20));
 
-      sourceMutator.setType(7, MinorType.FLOAT4);
-      sourceMutator.setSafe(7, newFloat4Holder(20.5f));
+      sourceVector.setType(7, MinorType.FLOAT4);
+      sourceVector.setSafe(7, newFloat4Holder(20.5f));
 
-      sourceMutator.setType(8, MinorType.INT);
-      sourceMutator.setSafe(8, newIntHolder(30));
+      sourceVector.setType(8, MinorType.INT);
+      sourceVector.setSafe(8, newIntHolder(30));
 
-      sourceMutator.setType(9, MinorType.FLOAT4);
-      sourceMutator.setSafe(9, newFloat4Holder(30.5f));
-      sourceMutator.setValueCount(10);
+      sourceVector.setType(9, MinorType.FLOAT4);
+      sourceVector.setSafe(9, newFloat4Holder(30.5f));
+      sourceVector.setValueCount(10);
 
       /* check the vector output */
-      assertEquals(10, sourceAccessor.getValueCount());
-      assertEquals(false, sourceAccessor.isNull(0));
-      assertEquals(5, sourceAccessor.getObject(0));
-      assertEquals(false, sourceAccessor.isNull(1));
-      assertEquals(5.5f, sourceAccessor.getObject(1));
-      assertEquals(false, sourceAccessor.isNull(2));
-      assertEquals(10, sourceAccessor.getObject(2));
-      assertEquals(false, sourceAccessor.isNull(3));
-      assertEquals(10.5f, sourceAccessor.getObject(3));
-      assertEquals(false, sourceAccessor.isNull(4));
-      assertEquals(15, sourceAccessor.getObject(4));
-      assertEquals(false, sourceAccessor.isNull(5));
-      assertEquals(15.5f, sourceAccessor.getObject(5));
-      assertEquals(false, sourceAccessor.isNull(6));
-      assertEquals(20, sourceAccessor.getObject(6));
-      assertEquals(false, sourceAccessor.isNull(7));
-      assertEquals(20.5f, sourceAccessor.getObject(7));
-      assertEquals(false, sourceAccessor.isNull(8));
-      assertEquals(30, sourceAccessor.getObject(8));
-      assertEquals(false, sourceAccessor.isNull(9));
-      assertEquals(30.5f, sourceAccessor.getObject(9));
+      assertEquals(10, sourceVector.getValueCount());
+      assertEquals(false, sourceVector.isNull(0));
+      assertEquals(5, sourceVector.getObject(0));
+      assertEquals(false, sourceVector.isNull(1));
+      assertEquals(5.5f, sourceVector.getObject(1));
+      assertEquals(false, sourceVector.isNull(2));
+      assertEquals(10, sourceVector.getObject(2));
+      assertEquals(false, sourceVector.isNull(3));
+      assertEquals(10.5f, sourceVector.getObject(3));
+      assertEquals(false, sourceVector.isNull(4));
+      assertEquals(15, sourceVector.getObject(4));
+      assertEquals(false, sourceVector.isNull(5));
+      assertEquals(15.5f, sourceVector.getObject(5));
+      assertEquals(false, sourceVector.isNull(6));
+      assertEquals(20, sourceVector.getObject(6));
+      assertEquals(false, sourceVector.isNull(7));
+      assertEquals(20.5f, sourceVector.getObject(7));
+      assertEquals(false, sourceVector.isNull(8));
+      assertEquals(30, sourceVector.getObject(8));
+      assertEquals(false, sourceVector.isNull(9));
+      assertEquals(30.5f, sourceVector.getObject(9));
 
       try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
 
         final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
-        final UnionVector.Accessor toAccessor = toVector.getAccessor();
 
         final int[][] transferLengths = {{0, 2},
             {2, 1},
@@ -290,7 +281,7 @@ public void testSplitAndTransferWithMixedVectors() throws Exception {
 
           /* check the toVector output after doing the splitAndTransfer */
           for (int i = 0; i < length; i++) {
-            assertEquals("Different values at index: " + i, sourceAccessor.getObject(start + i), toAccessor.getObject(i));
+            assertEquals("Different values at index: " + i, sourceVector.getObject(start + i), toVector.getObject(i));
           }
         }
       }
@@ -300,37 +291,35 @@ public void testSplitAndTransferWithMixedVectors() throws Exception {
   @Test
   public void testGetBufferAddress() throws Exception {
     try (UnionVector vector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
-      final UnionVector.Mutator mutator = vector.getMutator();
-      final UnionVector.Accessor accessor = vector.getAccessor();
       boolean error = false;
 
       vector.allocateNew();
 
       /* populate the UnionVector */
-      mutator.setType(0, MinorType.INT);
-      mutator.setSafe(0, newIntHolder(5));
+      vector.setType(0, MinorType.INT);
+      vector.setSafe(0, newIntHolder(5));
 
-      mutator.setType(1, MinorType.FLOAT4);
-      mutator.setSafe(1, newFloat4Holder(5.5f));
+      vector.setType(1, MinorType.FLOAT4);
+      vector.setSafe(1, newFloat4Holder(5.5f));
 
-      mutator.setType(2, MinorType.INT);
-      mutator.setSafe(2, newIntHolder(10));
+      vector.setType(2, MinorType.INT);
+      vector.setSafe(2, newIntHolder(10));
 
-      mutator.setType(3, MinorType.FLOAT4);
-      mutator.setSafe(3, newFloat4Holder(10.5f));
+      vector.setType(3, MinorType.FLOAT4);
+      vector.setSafe(3, newFloat4Holder(10.5f));
 
-      mutator.setValueCount(10);
+      vector.setValueCount(10);
 
       /* check the vector output */
-      assertEquals(10, accessor.getValueCount());
-      assertEquals(false, accessor.isNull(0));
-      assertEquals(5, accessor.getObject(0));
-      assertEquals(false, accessor.isNull(1));
-      assertEquals(5.5f, accessor.getObject(1));
-      assertEquals(false, accessor.isNull(2));
-      assertEquals(10, accessor.getObject(2));
-      assertEquals(false, accessor.isNull(3));
-      assertEquals(10.5f, accessor.getObject(3));
+      assertEquals(10, vector.getValueCount());
+      assertEquals(false, vector.isNull(0));
+      assertEquals(5, vector.getObject(0));
+      assertEquals(false, vector.isNull(1));
+      assertEquals(5.5f, vector.getObject(1));
+      assertEquals(false, vector.isNull(2));
+      assertEquals(10, vector.getObject(2));
+      assertEquals(false, vector.isNull(3));
+      assertEquals(10.5f, vector.getObject(3));
 
       List<ArrowBuf> buffers = vector.getFieldBuffers();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index b7f88c3babf1b..c7ee202f94661 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -487,8 +487,6 @@ public void testNullableFixedType1() {
 
     // Create a new value vector for 1024 integers.
     try (final NullableUInt4Vector vector = newVector(NullableUInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) {
-      final NullableUInt4Vector.Mutator mutator = vector.getMutator();
-      final NullableUInt4Vector.Accessor accessor = vector.getAccessor();
       boolean error = false;
       int initialCapacity = 1024;
 
@@ -500,33 +498,33 @@ public void testNullableFixedType1() {
       assertEquals(initialCapacity, vector.getValueCapacity());
 
       // Put and set a few values
-      mutator.set(0, 100);
-      mutator.set(1, 101);
-      mutator.set(100, 102);
-      mutator.set(1022, 103);
-      mutator.set(1023, 104);
+      vector.set(0, 100);
+      vector.set(1, 101);
+      vector.set(100, 102);
+      vector.set(1022, 103);
+      vector.set(1023, 104);
 
       /* check vector contents */
-      assertEquals(100, accessor.get(0));
-      assertEquals(101, accessor.get(1));
-      assertEquals(102, accessor.get(100));
-      assertEquals(103, accessor.get(1022));
-      assertEquals(104, accessor.get(1023));
+      assertEquals(100, vector.get(0));
+      assertEquals(101, vector.get(1));
+      assertEquals(102, vector.get(100));
+      assertEquals(103, vector.get(1022));
+      assertEquals(104, vector.get(1023));
 
       int val = 0;
 
       /* check unset bits/null values */
       for (int i = 2, j = 101; i <= 99 || j <= 1021; i++, j++) {
         if (i <= 99) {
-          assertTrue(accessor.isNull(i));
+          assertTrue(vector.isNull(i));
         }
         if(j <= 1021) {
-          assertTrue(accessor.isNull(j));
+          assertTrue(vector.isNull(j));
         }
       }
 
       try {
-        mutator.set(1024, 10000);
+        vector.set(1024, 10000);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -537,7 +535,7 @@ public void testNullableFixedType1() {
       }
 
       try {
-        accessor.get(1024);
+        vector.get(1024);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -548,28 +546,28 @@ public void testNullableFixedType1() {
       }
 
       /* should trigger a realloc of the underlying bitvector and valuevector */
-      mutator.setSafe(1024, 10000);
+      vector.setSafe(1024, 10000);
 
       /* check new capacity */
       assertEquals(initialCapacity * 2, vector.getValueCapacity());
 
       /* vector contents should still be intact after realloc */
-      assertEquals(100, accessor.get(0));
-      assertEquals(101, accessor.get(1));
-      assertEquals(102, accessor.get(100));
-      assertEquals(103, accessor.get(1022));
-      assertEquals(104, accessor.get(1023));
-      assertEquals(10000, accessor.get(1024));
+      assertEquals(100, vector.get(0));
+      assertEquals(101, vector.get(1));
+      assertEquals(102, vector.get(100));
+      assertEquals(103, vector.get(1022));
+      assertEquals(104, vector.get(1023));
+      assertEquals(10000, vector.get(1024));
 
       val = 0;
 
       /* check unset bits/null values */
       for (int i = 2, j = 101; i < 99 || j < 1021; i++, j++) {
         if (i <= 99) {
-          assertTrue(accessor.isNull(i));
+          assertTrue(vector.isNull(i));
         }
         if(j <= 1021) {
-          assertTrue(accessor.isNull(j));
+          assertTrue(vector.isNull(j));
         }
       }
 
@@ -581,7 +579,7 @@ public void testNullableFixedType1() {
 
       /* vector data should be zeroed out */
       for(int i = 0; i < (initialCapacity * 2); i++) {
-        assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+        assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
       }
     }
   }
@@ -590,8 +588,6 @@ public void testNullableFixedType1() {
   public void testNullableFixedType2() {
     // Create a new value vector for 1024 integers
     try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) {
-      final NullableFloat4Vector.Mutator mutator = vector.getMutator();
-      final NullableFloat4Vector.Accessor accessor = vector.getAccessor();
       boolean error = false;
       int initialCapacity = 16;
 
@@ -603,17 +599,17 @@ public void testNullableFixedType2() {
       assertEquals(initialCapacity, vector.getValueCapacity());
 
       /* populate the vector */
-      mutator.set(0, 100.5f);
-      mutator.set(2, 201.5f);
-      mutator.set(4, 300.3f);
-      mutator.set(6, 423.8f);
-      mutator.set(8, 555.6f);
-      mutator.set(10, 66.6f);
-      mutator.set(12, 78.8f);
-      mutator.set(14, 89.5f);
+      vector.set(0, 100.5f);
+      vector.set(2, 201.5f);
+      vector.set(4, 300.3f);
+      vector.set(6, 423.8f);
+      vector.set(8, 555.6f);
+      vector.set(10, 66.6f);
+      vector.set(12, 78.8f);
+      vector.set(14, 89.5f);
 
       try {
-        mutator.set(16, 90.5f);
+        vector.set(16, 90.5f);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -624,25 +620,25 @@ public void testNullableFixedType2() {
       }
 
       /* check vector contents */
-      assertEquals(100.5f, accessor.get(0), 0);
-      assertTrue(accessor.isNull(1));
-      assertEquals(201.5f, accessor.get(2), 0);
-      assertTrue(accessor.isNull(3));
-      assertEquals(300.3f, accessor.get(4), 0);
-      assertTrue(accessor.isNull(5));
-      assertEquals(423.8f, accessor.get(6), 0);
-      assertTrue(accessor.isNull(7));
-      assertEquals(555.6f, accessor.get(8), 0);
-      assertTrue(accessor.isNull(9));
-      assertEquals(66.6f, accessor.get(10), 0);
-      assertTrue(accessor.isNull(11));
-      assertEquals(78.8f, accessor.get(12), 0);
-      assertTrue(accessor.isNull(13));
-      assertEquals(89.5f, accessor.get(14), 0);
-      assertTrue(accessor.isNull(15));
+      assertEquals(100.5f, vector.get(0), 0);
+      assertTrue(vector.isNull(1));
+      assertEquals(201.5f, vector.get(2), 0);
+      assertTrue(vector.isNull(3));
+      assertEquals(300.3f, vector.get(4), 0);
+      assertTrue(vector.isNull(5));
+      assertEquals(423.8f, vector.get(6), 0);
+      assertTrue(vector.isNull(7));
+      assertEquals(555.6f, vector.get(8), 0);
+      assertTrue(vector.isNull(9));
+      assertEquals(66.6f, vector.get(10), 0);
+      assertTrue(vector.isNull(11));
+      assertEquals(78.8f, vector.get(12), 0);
+      assertTrue(vector.isNull(13));
+      assertEquals(89.5f, vector.get(14), 0);
+      assertTrue(vector.isNull(15));
 
       try {
-        accessor.get(16);
+        vector.get(16);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -653,29 +649,28 @@ public void testNullableFixedType2() {
       }
 
       /* this should trigger a realloc() */
-      mutator.setSafe(16, 90.5f);
+      vector.setSafe(16, 90.5f);
 
       /* underlying buffer should now be able to store double the number of values */
       assertEquals(initialCapacity * 2, vector.getValueCapacity());
 
       /* vector data should still be intact after realloc */
-      assertEquals(100.5f, accessor.get(0), 0);
-      assertTrue(accessor.isNull(1));
-      assertEquals(201.5f, accessor.get(2), 0);
-      assertTrue(accessor.isNull(3));
-      assertEquals(300.3f, accessor.get(4), 0);
-      assertTrue(accessor.isNull(5));
-      assertEquals(423.8f, accessor.get(6), 0);
-      assertTrue(accessor.isNull(7));
-      assertEquals(555.6f, accessor.get(8), 0);
-      assertTrue(accessor.isNull(9));
-      assertEquals(66.6f, accessor.get(10), 0);
-      assertTrue(accessor.isNull(11));
-      assertEquals(78.8f, accessor.get(12), 0);
-      assertTrue(accessor.isNull(13));
-      assertEquals(89.5f, accessor.get(14), 0);
-      assertTrue(accessor.isNull(15));
-      assertEquals(90.5f, accessor.get(16), 0);
+      assertEquals(100.5f, vector.get(0), 0);
+      assertTrue(vector.isNull(1));
+      assertEquals(201.5f, vector.get(2), 0);
+      assertTrue(vector.isNull(3));
+      assertEquals(300.3f, vector.get(4), 0);
+      assertTrue(vector.isNull(5));
+      assertEquals(423.8f, vector.get(6), 0);
+      assertTrue(vector.isNull(7));
+      assertEquals(555.6f, vector.get(8), 0);
+      assertTrue(vector.isNull(9));
+      assertEquals(66.6f, vector.get(10), 0);
+      assertTrue(vector.isNull(11));
+      assertEquals(78.8f, vector.get(12), 0);
+      assertTrue(vector.isNull(13));
+      assertEquals(89.5f, vector.get(14), 0);
+      assertTrue(vector.isNull(15));
 
       /* reset the vector */
       vector.reset();
@@ -685,7 +680,7 @@ public void testNullableFixedType2() {
 
       /* vector data should be zeroed out */
       for(int i = 0; i < (initialCapacity * 2); i++) {
-        assertTrue("non-null data not expected at index: " + i, accessor.isNull(i));
+        assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
       }
     }
   }
@@ -928,32 +923,30 @@ public void testNullableVarType2() {
 
     // Create a new value vector for 1024 integers.
     try (final NullableVarBinaryVector vector = newNullableVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) {
-      final NullableVarBinaryVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024 * 10, 1024);
 
-      m.set(0, STR1);
-      m.set(1, STR2);
-      m.set(2, STR3);
-      m.setSafe(3, STR3, 1, STR3.length - 1);
-      m.setSafe(4, STR3, 2, STR3.length - 2);
+      vector.set(0, STR1);
+      vector.set(1, STR2);
+      vector.set(2, STR3);
+      vector.setSafe(3, STR3, 1, STR3.length - 1);
+      vector.setSafe(4, STR3, 2, STR3.length - 2);
       ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3);
-      m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1);
-      m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2);
+      vector.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1);
+      vector.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2);
 
       // Check the sample strings.
-      final NullableVarBinaryVector.Accessor accessor = vector.getAccessor();
-      assertArrayEquals(STR1, accessor.get(0));
-      assertArrayEquals(STR2, accessor.get(1));
-      assertArrayEquals(STR3, accessor.get(2));
-      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3));
-      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4));
-      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5));
-      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6));
+      assertArrayEquals(STR1, vector.get(0));
+      assertArrayEquals(STR2, vector.get(1));
+      assertArrayEquals(STR3, vector.get(2));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(3));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(4));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(5));
+      assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(6));
 
       // Ensure null value throws.
       boolean b = false;
       try {
-        vector.getAccessor().get(7);
+        vector.get(7);
       } catch (IllegalStateException e) {
         b = true;
       } finally {
@@ -1064,8 +1057,6 @@ public void testReallocAfterVectorTransfer1() {
   @Test /* NullableFloat8Vector */
   public void testReallocAfterVectorTransfer2() {
     try (final NullableFloat8Vector vector = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) {
-      final NullableFloat8Vector.Mutator mutator = vector.getMutator();
-      final NullableFloat8Vector.Accessor accessor = vector.getAccessor();
       final int initialDefaultCapacity = 4096;
       boolean error = false;
 
@@ -1076,7 +1067,7 @@ public void testReallocAfterVectorTransfer2() {
       double baseValue = 100.375;
 
       for (int i = 0; i < initialDefaultCapacity; i++) {
-        mutator.setSafe(i, baseValue + (double)i);
+        vector.setSafe(i, baseValue + (double)i);
       }
 
       /* the above setSafe calls should not have triggered a realloc as
@@ -1085,33 +1076,33 @@ public void testReallocAfterVectorTransfer2() {
       assertEquals(initialDefaultCapacity, vector.getValueCapacity());
 
       for (int i = 0; i < initialDefaultCapacity; i++) {
-        double value = accessor.get(i);
+        double value = vector.get(i);
         assertEquals(baseValue + (double)i, value, 0);
       }
 
       /* this should trigger a realloc */
-      mutator.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity);
+      vector.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity);
       assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity());
 
       for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) {
-        mutator.setSafe(i, baseValue + (double)i);
+        vector.setSafe(i, baseValue + (double)i);
       }
 
       for (int i = 0; i < (initialDefaultCapacity * 2); i++) {
-        double value = accessor.get(i);
+        double value = vector.get(i);
         assertEquals(baseValue + (double)i, value, 0);
       }
 
       /* this should trigger a realloc */
-      mutator.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2));
+      vector.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2));
       assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity());
 
       for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) {
-        mutator.setSafe(i, baseValue + (double)i);
+        vector.setSafe(i, baseValue + (double)i);
       }
 
       for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
-        double value = accessor.get(i);
+        double value = vector.get(i);
         assertEquals(baseValue + (double)i, value, 0);
       }
 
@@ -1123,12 +1114,11 @@ public void testReallocAfterVectorTransfer2() {
       transferPair.transfer();
 
       NullableFloat8Vector toVector = (NullableFloat8Vector)transferPair.getTo();
-      final NullableFloat8Vector.Accessor toAccessor = toVector.getAccessor();
 
       /* check toVector contents before realloc */
       for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
-        assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i));
-        double value = toAccessor.get(i);
+        assertFalse("unexpected null value at index: " + i, toVector.isNull(i));
+        double value = toVector.get(i);
         assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0);
       }
 
@@ -1138,12 +1128,12 @@ public void testReallocAfterVectorTransfer2() {
 
       for (int i = 0; i < (initialDefaultCapacity * 8); i++) {
         if (i < (initialDefaultCapacity * 4)) {
-          assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i));
-          double value = toAccessor.get(i);
+          assertFalse("unexpected null value at index: " + i, toVector.isNull(i));
+          double value = toVector.get(i);
           assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0);
         }
         else {
-          assertTrue("unexpected non-null value at index: " + i, toAccessor.isNull(i));
+          assertTrue("unexpected non-null value at index: " + i, toVector.isNull(i));
         }
       }
 
@@ -1351,32 +1341,30 @@ public void testReallocAfterVectorTransfer4() {
   public void testReAllocNullableFixedWidthVector() {
     // Create a new value vector for 1024 integers
     try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) {
-      final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
       assertEquals(1024, vector.getValueCapacity());
 
       // Put values in indexes that fall within the initial allocation
-      m.setSafe(0, 100.1f);
-      m.setSafe(100, 102.3f);
-      m.setSafe(1023, 104.5f);
+      vector.setSafe(0, 100.1f);
+      vector.setSafe(100, 102.3f);
+      vector.setSafe(1023, 104.5f);
 
       // Now try to put values in space that falls beyond the initial allocation
-      m.setSafe(2000, 105.5f);
+      vector.setSafe(2000, 105.5f);
 
       // Check valueCapacity is more than initial allocation
       assertEquals(1024 * 2, vector.getValueCapacity());
 
-      final NullableFloat4Vector.Accessor accessor = vector.getAccessor();
-      assertEquals(100.1f, accessor.get(0), 0);
-      assertEquals(102.3f, accessor.get(100), 0);
-      assertEquals(104.5f, accessor.get(1023), 0);
-      assertEquals(105.5f, accessor.get(2000), 0);
+      assertEquals(100.1f, vector.get(0), 0);
+      assertEquals(102.3f, vector.get(100), 0);
+      assertEquals(104.5f, vector.get(1023), 0);
+      assertEquals(105.5f, vector.get(2000), 0);
 
       // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors
       // as we don't call setSafe for null values, but we do call setValueCount when all values are inserted into the
       // vector
-      m.setValueCount(vector.getValueCapacity() + 200);
+      vector.setValueCount(vector.getValueCapacity() + 200);
     }
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index 531a46c2734ef..293ffbfe19233 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -28,6 +28,7 @@
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.junit.After;
 import org.junit.Assert;
@@ -115,7 +116,7 @@ public void testListType() {
 
       vector.reAlloc();
       assertEquals(2047, vector.getValueCapacity()); // note: size - 1
-      assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+      assertEquals(0, vector.getOffsetBuffer().getInt(2014 * ListVector.OFFSET_WIDTH));
     }
   }
 
@@ -138,7 +139,7 @@ public void testMapType() {
 
       vector.reAlloc();
       assertEquals(1024, vector.getValueCapacity());
-      assertNull(vector.getAccessor().getObject(513));
+      assertNull(vector.getObject(513));
     }
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 0b7928dca2c39..3853eecac0552 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -199,8 +199,10 @@ public void testLoadValidityBuffer() throws IOException {
     int count = 10;
     ArrowBuf[] values = new ArrowBuf[4];
     for (int i = 0; i < 4; i+=2) {
-      ArrowBuf buf1 = allocator.buffer((int)Math.ceil(count / 8.0));
+      ArrowBuf buf1 = allocator.buffer(BitVectorHelper.getValidityBufferSize(count));
       ArrowBuf buf2 = allocator.buffer(count * 4); // integers
+      buf1.setZero(0, buf1.capacity());
+      buf2.setZero(0, buf2.capacity());
       values[i] = buf1;
       values[i+1] = buf2;
       for (int j = 0; j < count; j++) {
@@ -314,7 +316,7 @@ public void testUnloadLoadDuplicates() throws IOException {
 
   public static VectorUnloader newVectorUnloader(FieldVector root) {
     Schema schema = new Schema(root.getField().getChildren());
-    int valueCount = root.getAccessor().getValueCount();
+    int valueCount = root.getValueCount();
     List<FieldVector> fields = root.getChildrenFromFields();
     VectorSchemaRoot vsr = new VectorSchemaRoot(schema.getFields(), fields, valueCount);
     return new VectorUnloader(vsr);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 97efb7d5a6d30..b0d6cf555e98d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -80,24 +80,23 @@ public void testPromoteToUnion() throws Exception {
 
       writer.end();
 
-      container.getMutator().setValueCount(5);
+      container.setValueCount(5);
 
       final UnionVector uv = v.getChild("A", UnionVector.class);
-      final UnionVector.Accessor accessor = uv.getAccessor();
 
-      assertFalse("0 shouldn't be null", accessor.isNull(0));
-      assertEquals(false, accessor.getObject(0));
+      assertFalse("0 shouldn't be null", uv.isNull(0));
+      assertEquals(false, uv.getObject(0));
 
-      assertFalse("1 shouldn't be null", accessor.isNull(1));
-      assertEquals(true, accessor.getObject(1));
+      assertFalse("1 shouldn't be null", uv.isNull(1));
+      assertEquals(true, uv.getObject(1));
 
-      assertFalse("2 shouldn't be null", accessor.isNull(2));
-      assertEquals(10, accessor.getObject(2));
+      assertFalse("2 shouldn't be null", uv.isNull(2));
+      assertEquals(10, uv.getObject(2));
 
-      assertTrue("3 should be null", accessor.isNull(3));
+      assertTrue("3 should be null", uv.isNull(3));
 
-      assertFalse("4 shouldn't be null", accessor.isNull(4));
-      assertEquals(100, accessor.getObject(4));
+      assertFalse("4 shouldn't be null", uv.isNull(4));
+      assertEquals(100, uv.getObject(4));
 
       container.clear();
       container.allocateNew();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 856d60724b085..52defdc2e1b00 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -456,7 +456,7 @@ public void simpleUnion() {
         unionWriter.writeFloat4((float) i);
       }
     }
-    vector.getMutator().setValueCount(COUNT);
+    vector.setValueCount(COUNT);
     UnionReader unionReader = new UnionReader(vector);
     for (int i = 0; i < COUNT; i++) {
       unionReader.setPosition(i);
@@ -834,7 +834,7 @@ public void complexCopierWithList() {
     TransferPair tp = mapVector.getTransferPair(allocator);
     tp.splitAndTransfer(0, 1);
     MapVector toMapVector = (MapVector) tp.getTo();
-    JsonStringHashMap<?, ?> toMapValue = (JsonStringHashMap<?, ?>) toMapVector.getAccessor().getObject(0);
+    JsonStringHashMap<?, ?> toMapValue = (JsonStringHashMap<?, ?>) toMapVector.getObject(0);
     JsonStringArrayList<?> object = (JsonStringArrayList<?>) toMapValue.get("list");
     assertEquals(1, object.get(0));
     assertEquals(2, object.get(1));
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 60009b0a46396..874ba99e20f29 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -97,14 +97,8 @@ public void tearDown() {
 
   protected void validateContent(int count, VectorSchemaRoot root) {
     for (int i = 0; i < count; i++) {
-      FieldVector fv = root.getVector("int");
-      if (fv instanceof NullableIntVector) {
-        Assert.assertEquals(i, fv.getObject(i));
-        Assert.assertEquals(Integer.valueOf(i), fv.getObject(i));
-      } else {
-        Assert.assertEquals(i, fv.getAccessor().getObject(i));
-        Assert.assertEquals(Long.valueOf(i), fv.getAccessor().getObject(i));
-      }
+      Assert.assertEquals(i, root.getVector("int").getObject(i));
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i));
     }
   }
 
@@ -146,10 +140,9 @@ protected void writeComplexData(int count, MapVector parent) {
   public void printVectors(List<FieldVector> vectors) {
     for (FieldVector vector : vectors) {
       LOGGER.debug(vector.getField().getName());
-      Accessor accessor = vector.getAccessor();
-      int valueCount = accessor.getValueCount();
+      int valueCount = vector.getValueCount();
       for (int i = 0; i < valueCount; i++) {
-        LOGGER.debug(String.valueOf(accessor.getObject(i)));
+        LOGGER.debug(String.valueOf(vector.getObject(i)));
       }
     }
   }
@@ -159,14 +152,14 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) {
     printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
 
-      Object intVal = root.getVector("int").getAccessor().getObject(i);
+      Object intVal = root.getVector("int").getObject(i);
       if (i % 5 != 3) {
         Assert.assertEquals(i, intVal);
       } else {
         Assert.assertNull(intVal);
       }
-      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
-      Assert.assertEquals(i % 3, ((List<?>) root.getVector("list").getAccessor().getObject(i)).size());
+      Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i));
+      Assert.assertEquals(i % 3, ((List<?>) root.getVector("list").getObject(i)).size());
       NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
       FieldReader mapReader = root.getVector("map").getReader();
       mapReader.setPosition(i);
@@ -209,15 +202,15 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
     Assert.assertEquals(count, root.getRowCount());
     printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
-      long dateVal = ((NullableDateMilliVector) root.getVector("date")).getAccessor().get(i);
+      long dateVal = ((NullableDateMilliVector) root.getVector("date")).get(i);
       LocalDateTime dt = makeDateTimeFromCount(i);
       LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
       Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal);
-      long timeVal = ((NullableTimeMilliVector) root.getVector("time")).getAccessor().get(i);
+      long timeVal = ((NullableTimeMilliVector) root.getVector("time")).get(i);
       Assert.assertEquals(dt.getMillisOfDay(), timeVal);
-      Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
+      Object timestampMilliVal = root.getVector("timestamp-milli").getObject(i);
       Assert.assertEquals(dt, timestampMilliVal);
-      Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getAccessor().getObject(i);
+      Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getObject(i);
       Assert.assertEquals(DateUtility.toMillis(dt), timestampMilliTZVal);
     }
   }
@@ -282,7 +275,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat
     List<Field> fields = ImmutableList.of(encodedVector1A.getField(), encodedVector1B.getField(), encodedVector2.getField());
     List<FieldVector> vectors = ImmutableList.of(encodedVector1A, encodedVector1B, encodedVector2);
 
-    return new VectorSchemaRoot(fields, vectors, encodedVector1A.getAccessor().getValueCount());
+    return new VectorSchemaRoot(fields, vectors, encodedVector1A.getValueCount());
   }
 
   protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider provider) {
@@ -293,14 +286,13 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider
     Assert.assertNotNull(encoding1A);
     Assert.assertEquals(1L, encoding1A.getId());
 
-    FieldVector.Accessor accessor = vector1A.getAccessor();
-    Assert.assertEquals(6, accessor.getValueCount());
-    Assert.assertEquals(0, accessor.getObject(0));
-    Assert.assertEquals(1, accessor.getObject(1));
-    Assert.assertEquals(null, accessor.getObject(2));
-    Assert.assertEquals(2, accessor.getObject(3));
-    Assert.assertEquals(1, accessor.getObject(4));
-    Assert.assertEquals(2, accessor.getObject(5));
+    Assert.assertEquals(6, vector1A.getValueCount());
+    Assert.assertEquals(0, vector1A.getObject(0));
+    Assert.assertEquals(1, vector1A.getObject(1));
+    Assert.assertEquals(null, vector1A.getObject(2));
+    Assert.assertEquals(2, vector1A.getObject(3));
+    Assert.assertEquals(1, vector1A.getObject(4));
+    Assert.assertEquals(2, vector1A.getObject(5));
 
     FieldVector vector1B = root.getVector("varcharB");
     Assert.assertNotNull(vector1B);
@@ -310,14 +302,13 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider
     Assert.assertTrue(encoding1A.equals(encoding1B));
     Assert.assertEquals(1L, encoding1B.getId());
 
-    accessor = vector1B.getAccessor();
-    Assert.assertEquals(6, accessor.getValueCount());
-    Assert.assertEquals(2, accessor.getObject(0));
-    Assert.assertEquals(1, accessor.getObject(1));
-    Assert.assertEquals(2, accessor.getObject(2));
-    Assert.assertEquals(null, accessor.getObject(3));
-    Assert.assertEquals(1, accessor.getObject(4));
-    Assert.assertEquals(0, accessor.getObject(5));
+    Assert.assertEquals(6, vector1B.getValueCount());
+    Assert.assertEquals(2, vector1B.getObject(0));
+    Assert.assertEquals(1, vector1B.getObject(1));
+    Assert.assertEquals(2, vector1B.getObject(2));
+    Assert.assertEquals(null, vector1B.getObject(3));
+    Assert.assertEquals(1, vector1B.getObject(4));
+    Assert.assertEquals(0, vector1B.getObject(5));
 
     FieldVector vector2 = root.getVector("sizes");
     Assert.assertNotNull(vector2);
@@ -326,30 +317,29 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider
     Assert.assertNotNull(encoding2);
     Assert.assertEquals(2L, encoding2.getId());
 
-    accessor = vector2.getAccessor();
-    Assert.assertEquals(6, accessor.getValueCount());
-    Assert.assertEquals(null, accessor.getObject(0));
-    Assert.assertEquals(2, accessor.getObject(1));
-    Assert.assertEquals(1, accessor.getObject(2));
-    Assert.assertEquals(1, accessor.getObject(3));
-    Assert.assertEquals(2, accessor.getObject(4));
-    Assert.assertEquals(null, accessor.getObject(5));
+    Assert.assertEquals(6, vector2.getValueCount());
+    Assert.assertEquals(null, vector2.getObject(0));
+    Assert.assertEquals(2, vector2.getObject(1));
+    Assert.assertEquals(1, vector2.getObject(2));
+    Assert.assertEquals(1, vector2.getObject(3));
+    Assert.assertEquals(2, vector2.getObject(4));
+    Assert.assertEquals(null, vector2.getObject(5));
 
     Dictionary dictionary1 = provider.lookup(1L);
     Assert.assertNotNull(dictionary1);
-    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary1.getVector()).getAccessor();
-    Assert.assertEquals(3, dictionaryAccessor.getValueCount());
-    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
-    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
-    Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2));
+    NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary1.getVector());
+    Assert.assertEquals(3, dictionaryVector.getValueCount());
+    Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
+    Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
+    Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2));
 
     Dictionary dictionary2 = provider.lookup(2L);
     Assert.assertNotNull(dictionary2);
-    dictionaryAccessor = ((NullableVarCharVector) dictionary2.getVector()).getAccessor();
-    Assert.assertEquals(3, dictionaryAccessor.getValueCount());
-    Assert.assertEquals(new Text("micro"), dictionaryAccessor.getObject(0));
-    Assert.assertEquals(new Text("small"), dictionaryAccessor.getObject(1));
-    Assert.assertEquals(new Text("large"), dictionaryAccessor.getObject(2));
+    dictionaryVector = ((NullableVarCharVector) dictionary2.getVector());
+    Assert.assertEquals(3, dictionaryVector.getValueCount());
+    Assert.assertEquals(new Text("micro"), dictionaryVector.getObject(0));
+    Assert.assertEquals(new Text("small"), dictionaryVector.getObject(1));
+    Assert.assertEquals(new Text("large"), dictionaryVector.getObject(2));
   }
 
   protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) {
@@ -359,7 +349,7 @@ protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAlloc
     dictionaryVector.allocateNewSafe();
     dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
     dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
-    dictionaryVector.getMutator().setValueCount(2);
+    dictionaryVector.setValueCount(2);
 
     Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(2L, false, null));
     provider.put(dictionary);
@@ -398,18 +388,17 @@ protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvide
     Assert.assertEquals(2L, encoding.getId());
     Assert.assertEquals(new ArrowType.Int(32, true), encoding.getIndexType());
 
-    FieldVector.Accessor accessor = vector.getAccessor();
-    Assert.assertEquals(3, accessor.getValueCount());
-    Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
-    Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
-    Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
+    Assert.assertEquals(3, vector.getValueCount());
+    Assert.assertEquals(Arrays.asList(0, 1), vector.getObject(0));
+    Assert.assertEquals(Arrays.asList(0), vector.getObject(1));
+    Assert.assertEquals(Arrays.asList(1), vector.getObject(2));
 
     Dictionary dictionary = provider.lookup(2L);
     Assert.assertNotNull(dictionary);
-    NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor();
-    Assert.assertEquals(2, dictionaryAccessor.getValueCount());
-    Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
-    Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
+    NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector());
+    Assert.assertEquals(2, dictionaryVector.getValueCount());
+    Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
+    Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
   }
 
   protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) {
@@ -423,14 +412,14 @@ protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) {
     decimalVector3.allocateNew(count);
 
     for (int i = 0; i < count; i++) {
-      decimalVector1.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3));
-      decimalVector2.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2));
-      decimalVector3.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8));
+      decimalVector1.setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3));
+      decimalVector2.setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2));
+      decimalVector3.setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8));
     }
 
-    decimalVector1.getMutator().setValueCount(count);
-    decimalVector2.getMutator().setValueCount(count);
-    decimalVector3.getMutator().setValueCount(count);
+    decimalVector1.setValueCount(count);
+    decimalVector2.setValueCount(count);
+    decimalVector3.setValueCount(count);
 
     List<Field> fields = ImmutableList.of(decimalVector1.getField(), decimalVector2.getField(), decimalVector3.getField());
     List<FieldVector> vectors = ImmutableList.<FieldVector>of(decimalVector1, decimalVector2, decimalVector3);
@@ -446,19 +435,19 @@ protected void validateDecimalData(VectorSchemaRoot root) {
 
     for (int i = 0; i < count; i++) {
       // Verify decimal 1 vector
-      BigDecimal readValue = decimalVector1.getAccessor().getObject(i);
+      BigDecimal readValue = decimalVector1.getObject(i);
       ArrowType.Decimal type = (ArrowType.Decimal) decimalVector1.getField().getType();
       BigDecimal genValue = new BigDecimal(BigInteger.valueOf(i), type.getScale());
       Assert.assertEquals(genValue, readValue);
 
       // Verify decimal 2 vector
-      readValue = decimalVector2.getAccessor().getObject(i);
+      readValue = decimalVector2.getObject(i);
       type = (ArrowType.Decimal) decimalVector2.getField().getType();
       genValue = new BigDecimal(BigInteger.valueOf(i * (1 << 10)), type.getScale());
       Assert.assertEquals(genValue, readValue);
 
       // Verify decimal 3 vector
-      readValue = decimalVector3.getAccessor().getObject(i);
+      readValue = decimalVector3.getObject(i);
       type = (ArrowType.Decimal) decimalVector3.getField().getType();
       genValue = new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), type.getScale());
       Assert.assertEquals(genValue, readValue);
@@ -571,7 +560,7 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) {
     int numVarBinaryValues = 0;
     for (int i = 0; i < count; i++) {
       expectedArray[i] = (byte) i;
-      Object obj = listVector.getAccessor().getObject(i);
+      Object obj = listVector.getObject(i);
       List<?> objList = (List) obj;
       if (i % 3 == 0) {
         Assert.assertTrue(objList.isEmpty());
@@ -586,10 +575,10 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) {
     }
 
     // ListVector lastSet should be the index of last value + 1
-    Assert.assertEquals(listVector.getMutator().getLastSet(), count);
+    Assert.assertEquals(listVector.getLastSet(), count);
 
     // NullableVarBinaryVector lastSet should be the index of last value
     NullableVarBinaryVector binaryVector = (NullableVarBinaryVector) listVector.getChildrenFromFields().get(0);
-    Assert.assertEquals(binaryVector.getMutator().getLastSet(), numVarBinaryValues - 1);
+    Assert.assertEquals(binaryVector.getLastSet(), numVarBinaryValues - 1);
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index feae08ea223e1..8559969a20424 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -317,11 +317,11 @@ public void testWriteReadTiny() throws IOException {
 
     try (VectorSchemaRoot root = VectorSchemaRoot.create(MessageSerializerTest.testSchema(), allocator)) {
       root.getFieldVectors().get(0).allocateNew();
-      NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator();
+      NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
       for (int i = 0; i < 16; i++) {
-        mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
+        vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
       }
-      mutator.setValueCount(16);
+      vector.setValueCount(16);
       root.setRowCount(16);
 
       // write file
@@ -368,9 +368,9 @@ private void validateTinyData(VectorSchemaRoot root) {
     NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
     for (int i = 0; i < 16; i++) {
       if (i < 8) {
-        Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i));
+        Assert.assertEquals((byte) (i + 1), vector.get(i));
       } else {
-        Assert.assertTrue(vector.getAccessor().isNull(i));
+        Assert.assertTrue(vector.isNull(i));
       }
     }
   }
@@ -397,7 +397,7 @@ public void testWriteReadMetadata() throws IOException {
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
          NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) {
       vector.allocateNewSafe();
-      vector.getMutator().setValueCount(0);
+      vector.setValueCount(0);
 
       List<FieldVector> vectors = ImmutableList.<FieldVector>of(vector);
       VectorSchemaRoot root = new VectorSchemaRoot(originalSchema, vectors, 0);
@@ -578,13 +578,13 @@ public void testWriteReadFixedSizeList() throws IOException {
       parent.allocateNew();
 
       for (int i = 0; i < 10; i++) {
-        tuples.getMutator().setNotNull(i);
-        floats.getMutator().set(i * 2, i + 0.1f);
-        floats.getMutator().set(i * 2 + 1, i + 10.1f);
+        tuples.setNotNull(i);
+        floats.set(i * 2, i + 0.1f);
+        floats.set(i * 2 + 1, i + 10.1f);
         ints.set(i, i);
       }
 
-      parent.getMutator().setValueCount(10);
+      parent.setValueCount(10);
       write(parent, file, stream);
     }
 
@@ -600,8 +600,8 @@ public void testWriteReadFixedSizeList() throws IOException {
         arrowReader.loadRecordBatch(rbBlock);
         Assert.assertEquals(count, root.getRowCount());
         for (int i = 0; i < 10; i++) {
-          Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i));
-          Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i));
+          Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i));
+          Assert.assertEquals(i, root.getVector("ints").getObject(i));
         }
       }
     }
@@ -616,8 +616,8 @@ public void testWriteReadFixedSizeList() throws IOException {
       arrowReader.loadNextBatch();
       Assert.assertEquals(count, root.getRowCount());
       for (int i = 0; i < 10; i++) {
-        Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i));
-        Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i));
+        Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i));
+        Assert.assertEquals(i, root.getVector("ints").getObject(i));
       }
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
index e2efabef0095b..c7e34191a3f1a 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
@@ -70,11 +70,11 @@ public void testReadWrite() throws IOException {
       int numBatches = 1;
 
       root.getFieldVectors().get(0).allocateNew();
-      NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator();
+      NullableTinyIntVector vector = (NullableTinyIntVector)root.getFieldVectors().get(0);
       for (int i = 0; i < 16; i++) {
-        mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
+        vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
       }
-      mutator.setValueCount(16);
+      vector.setValueCount(16);
       root.setRowCount(16);
 
       ByteArrayOutputStream out = new ByteArrayOutputStream();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
index 40716942f02a2..f393733cc9021 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
@@ -62,13 +62,13 @@ public void run() {
         writer.start();
         for (int j = 0; j < numBatches; j++) {
           root.getFieldVectors().get(0).allocateNew();
-          NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator();
+          NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
           // Send a changing batch id first
-          mutator.set(0, j);
+          vector.set(0, j);
           for (int i = 1; i < 16; i++) {
-            mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
+            vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
           }
-          mutator.setValueCount(16);
+          vector.setValueCount(16);
           root.setRowCount(16);
 
           writer.writeBatch();
@@ -117,12 +117,12 @@ public boolean loadNextBatch() throws IOException {
             VectorSchemaRoot root = getVectorSchemaRoot();
             Assert.assertEquals(16, root.getRowCount());
             NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
-            Assert.assertEquals((byte) (batchesRead - 1), vector.getAccessor().get(0));
+            Assert.assertEquals((byte) (batchesRead - 1), vector.get(0));
             for (int i = 1; i < 16; i++) {
               if (i < 8) {
-                Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i));
+                Assert.assertEquals((byte) (i + 1), vector.get(i));
               } else {
-                Assert.assertTrue(vector.getAccessor().isNull(i));
+                Assert.assertTrue(vector.isNull(i));
               }
             }
           }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index ee90d340d7cc8..5c4c48cd26b60 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -280,7 +280,7 @@ public void testSetStructLength() throws IOException {
       // initialize vectors
       try (VectorSchemaRoot root = reader.read();) {
         FieldVector vector = root.getVector("struct_nullable");
-        Assert.assertEquals(7, vector.getAccessor().getValueCount());
+        Assert.assertEquals(7, vector.getValueCount());
       }
     }
   }

From 5bea9836ffa044e07cf376a5c0bfd88afc3560d4 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Tue, 7 Nov 2017 11:12:57 -0500
Subject: [PATCH 1251/1644] ARROW-1717: [Java] Refactor JsonReader for new
 class hierarchy and fix integration tests

---
 .../BaseNullableVariableWidthVector.java      |  46 --
 .../arrow/vector/NullableBigIntVector.java    |  37 +-
 .../arrow/vector/NullableDateDayVector.java   |  34 --
 .../arrow/vector/NullableDateMilliVector.java |  34 --
 .../arrow/vector/NullableDecimalVector.java   |  37 +-
 .../arrow/vector/NullableFloat4Vector.java    |  37 +-
 .../arrow/vector/NullableFloat8Vector.java    |  37 +-
 .../arrow/vector/NullableIntVector.java       |  36 +-
 .../arrow/vector/NullableSmallIntVector.java  |  37 +-
 .../arrow/vector/NullableTimeMicroVector.java |  34 --
 .../arrow/vector/NullableTimeMilliVector.java |  34 --
 .../arrow/vector/NullableTimeNanoVector.java  |  34 --
 .../arrow/vector/NullableTimeSecVector.java   |  34 --
 .../arrow/vector/NullableTimeStampVector.java |  36 --
 .../arrow/vector/NullableTinyIntVector.java   |  36 --
 .../vector/file/json/JsonFileReader.java      | 424 ++++++++++++------
 16 files changed, 290 insertions(+), 677 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
index 6587cde633ea9..47daeee7f202c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
@@ -909,52 +909,6 @@ protected final void handleSafe(int index, int dataLength) {
       }
    }
 
-
-   /******************************************************************
-    *                                                                *
-    *                helper methods currently                        *
-    *                used by JsonFileReader and                      *
-    *                JsonFileWriter                                  *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Method used by Json Reader to explicitly set the data of the variable
-    * width vector elements. The method takes care of allocating the memory
-    * for the vector if caller hasn't done so.
-    *
-    * This method should not be used externally.
-    *
-    * @param data ArrowBuf for storing variable width elements in the vector
-    * @param offset offset of the element
-    * @param allocator memory allocator
-    * @param index position of the element in the vector
-    * @param value array of bytes for the element
-    * @param valueCount number of elements in the vector
-    * @return buffer holding the variable width data.
-    */
-   public static ArrowBuf set(ArrowBuf data, ArrowBuf offset,
-                              BufferAllocator allocator, int index, byte[] value,
-                              int valueCount) {
-      if (data == null) {
-         data = allocator.buffer(INITIAL_BYTE_COUNT);
-      }
-      final int currentBufferCapacity = data.capacity();
-      final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH);
-      while (currentBufferCapacity < currentStartOffset + value.length) {
-         final ArrowBuf newBuf = allocator.buffer(currentBufferCapacity * 2);
-         newBuf.setBytes(0, data, 0, currentBufferCapacity);
-         data.release();
-         data = newBuf;
-      }
-      data.setBytes(currentStartOffset, value, 0, value.length);
-      if (index == (valueCount - 1)) {
-         data.writerIndex(offset.getInt(valueCount * OFFSET_WIDTH));
-      }
-      return data;
-   }
-
    /**
     * Method used by Json Writer to read a variable width element from
     * the variable width vector and write to Json.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
index ee40d708cc199..253427333a2e0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
@@ -34,7 +34,7 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableBigIntVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 8;
+   public static final byte TYPE_WIDTH = 8;
    private final FieldReader reader;
 
    /**
@@ -290,41 +290,6 @@ public void setSafe(int index, int isSet, long value) {
       set(index, isSet, value);
    }
 
-
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, long value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setLong(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
index 949287ecafe45..e6b5b590f3f7e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
@@ -292,40 +292,6 @@ public void setSafe(int index, int isSet, int value) {
       set(index, isSet, value);
    }
 
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader                                *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, int value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setInt(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
index a0bdccedea405..8e15100f70818 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
@@ -296,40 +296,6 @@ public void setSafe(int index, int isSet, long value) {
       set(index, isSet, value);
    }
 
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader                                *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, long value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setLong(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
index 8320f90830a5b..2a611c6a2d5ba 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
@@ -37,7 +37,7 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableDecimalVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 16;
+   public static final byte TYPE_WIDTH = 16;
    private final FieldReader reader;
 
    private final int precision;
@@ -355,41 +355,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
    }
 
 
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader                                *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value as array of bytes
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, byte[] value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      DecimalUtility.writeByteArrayToArrowBuf(value, buffer, index);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
-
    /******************************************************************
     *                                                                *
     *                      vector transfer                           *
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
index 5b28065c402c4..3ba5cfcea5251 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
@@ -34,7 +34,7 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableFloat4Vector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 4;
+   public static final byte TYPE_WIDTH = 4;
    private final FieldReader reader;
 
    /**
@@ -291,41 +291,6 @@ public void setSafe(int index, int isSet, float value) {
       set(index, isSet, value);
    }
 
-
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, float value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setFloat(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
index 624abf2f272cd..2fb96a44fe78b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
@@ -34,7 +34,7 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableFloat8Vector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 8;
+   public static final byte TYPE_WIDTH = 8;
    private final FieldReader reader;
 
    /**
@@ -291,41 +291,6 @@ public void setSafe(int index, int isSet, double value) {
       set(index, isSet, value);
    }
 
-
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, double value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setDouble(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
index 6311daf4f5cc1..93deacbdfdc74 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
@@ -34,7 +34,7 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableIntVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 4;
+   public static final byte TYPE_WIDTH = 4;
    private final FieldReader reader;
 
    /**
@@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, int value) {
       set(index, isSet, value);
    }
 
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, int value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setInt(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
index c45a8d5f5bcba..ed337188b5b0b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
@@ -34,7 +34,7 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableSmallIntVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 2;
+   public static final byte TYPE_WIDTH = 2;
    private final FieldReader reader;
 
    /**
@@ -319,41 +319,6 @@ public void setSafe(int index, int isSet, short value) {
       set(index, isSet, value);
    }
 
-
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, short value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setShort(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
index 454a4ac41a6d9..d7ea322239741 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
@@ -292,40 +292,6 @@ public void setSafe(int index, int isSet, long value) {
       set(index, isSet, value);
    }
 
-
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, long value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setLong(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
index 8540d16924210..07d8abba4e57b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
@@ -294,40 +294,6 @@ public void setSafe(int index, int isSet, int value) {
    }
 
 
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, int value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setInt(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
index 015226da22318..947b2392f7a15 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
@@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, long value) {
       set(index, isSet, value);
    }
 
-
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, long value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setLong(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
index 2b2375e92f1f4..0a3cfaac047eb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
@@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, int value) {
       set(index, isSet, value);
    }
 
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, int value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setInt(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
index b2a58bd4568f9..1bf2abc384d92 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
@@ -165,42 +165,6 @@ public void setSafe(int index, int isSet, long value) {
       set(index, isSet, value);
    }
 
-
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and                            *
-    *          JsonFileWriter                                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, long value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setLong(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
index 15100530d0ec5..ccbfa32b85f0c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
@@ -318,42 +318,6 @@ public void setSafe(int index, int isSet, byte value) {
       set(index, isSet, value);
    }
 
-
-
-   /******************************************************************
-    *                                                                *
-    *          helper routines currently                             *
-    *          used in JsonFileReader and JsonFileWriter             *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Given a data buffer, this method sets the element value at a particular
-    * position. Reallocates the buffer if needed.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param allocator allocator
-    * @param valueCount number of elements in the vector
-    * @param index position of the new element
-    * @param value element value
-    * @return data buffer
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, byte value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * TYPE_WIDTH);
-      }
-      buffer.setByte(index * TYPE_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * TYPE_WIDTH);
-      }
-
-      return buffer;
-   }
-
    /**
     * Given a data buffer, get the value stored at a particular position
     * in the vector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index b21fcaac391e8..d3c8b990394d8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -35,16 +35,19 @@
 import java.util.*;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.file.InvalidArrowFileException;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.DecimalUtility;
 import org.apache.arrow.vector.util.DictionaryUtility;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
@@ -186,6 +189,282 @@ public VectorSchemaRoot read() throws IOException {
     }
   }
 
+  private abstract class BufferReader {
+    abstract protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException;
+
+    final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException {
+      readToken(START_ARRAY);
+      ArrowBuf buf = read(allocator, count);
+      readToken(END_ARRAY);
+      return buf;
+    }
+  }
+
+  private class BufferHelper {
+     BufferReader BIT = new BufferReader() {
+
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        final int bufferSize = BitVectorHelper.getValidityBufferSize(count);
+        ArrowBuf buf = allocator.buffer(bufferSize);
+
+        // C++ integration test fails without this.
+        buf.setZero(0, bufferSize);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          BitVectorHelper.setValidityBit(buf, i, parser.readValueAs(Boolean.class) ? 1 : 0);
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader INT1 = new BufferReader() {
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrowBuf buf = allocator.buffer(count * NullableTinyIntVector.TYPE_WIDTH);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          buf.writeByte(parser.getByteValue());
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader INT2 = new BufferReader() {
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrowBuf buf = allocator.buffer(count * NullableSmallIntVector.TYPE_WIDTH);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          buf.writeShort(parser.getShortValue());
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader INT4 = new BufferReader() {
+
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrowBuf buf = allocator.buffer(count * NullableIntVector.TYPE_WIDTH);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          buf.writeInt(parser.getIntValue());
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader INT8 = new BufferReader() {
+
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrowBuf buf = allocator.buffer(count * NullableBigIntVector.TYPE_WIDTH);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          buf.writeLong(parser.getLongValue());
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader FLOAT4 = new BufferReader() {
+
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrowBuf buf = allocator.buffer(count * NullableFloat4Vector.TYPE_WIDTH);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          buf.writeFloat(parser.getFloatValue());
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader FLOAT8 = new BufferReader() {
+
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrowBuf buf = allocator.buffer(count * NullableFloat8Vector.TYPE_WIDTH);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          buf.writeDouble(parser.getDoubleValue());
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader DECIMAL = new BufferReader() {
+
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrowBuf buf = allocator.buffer(count * NullableDecimalVector.TYPE_WIDTH);
+
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          final byte[] value = decodeHexSafe(parser.getValueAsString());
+          DecimalUtility.writeByteArrayToArrowBuf(value, buf, i);
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader VARCHAR = new BufferReader() {
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrayList<byte[]> values = Lists.newArrayList();
+        int bufferSize = 0;
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          final byte[] value = parser.getValueAsString().getBytes(UTF_8);
+          values.add(value);
+          bufferSize += value.length;
+
+        }
+
+        ArrowBuf buf = allocator.buffer(bufferSize);
+
+        for (byte[] value : values) {
+          buf.writeBytes(value);
+        }
+
+        return buf;
+      }
+    };
+
+    BufferReader VARBINARY = new BufferReader() {
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrayList<byte[]> values = Lists.newArrayList();
+        int bufferSize = 0;
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          final byte[] value = decodeHexSafe(parser.readValueAs(String.class));
+          values.add(value);
+          bufferSize += value.length;
+
+        }
+
+        ArrowBuf buf = allocator.buffer(bufferSize);
+
+        for (byte[] value : values) {
+          buf.writeBytes(value);
+        }
+
+        return buf;
+      }
+    };
+
+  }
+
+  private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferType, Types.MinorType type, int count) throws IOException {
+    ArrowBuf buf;
+
+    BufferHelper helper = new BufferHelper();
+
+    BufferReader reader = null;
+
+    if (bufferType.equals(VALIDITY)) {
+      reader = helper.BIT;
+    } else if (bufferType.equals(OFFSET)) {
+      reader = helper.INT4;
+    } else if (bufferType.equals(TYPE)) {
+      reader = helper.INT1;
+    } else if (bufferType.equals(DATA)) {
+      switch (type) {
+        case BIT:
+          reader = helper.BIT;
+          break;
+        case TINYINT:
+          reader = helper.INT1;
+          break;
+        case SMALLINT:
+          reader = helper.INT2;
+          break;
+        case INT:
+          reader = helper.INT4;
+          break;
+        case BIGINT:
+          reader = helper.INT8;
+          break;
+        case UINT1:
+          reader = helper.INT1;
+          break;
+        case UINT2:
+          reader = helper.INT2;
+          break;
+        case UINT4:
+          reader = helper.INT4;
+          break;
+        case UINT8:
+          reader = helper.INT8;
+          break;
+        case FLOAT4:
+          reader = helper.FLOAT4;
+          break;
+        case FLOAT8:
+          reader = helper.FLOAT8;
+          break;
+        case DECIMAL:
+          reader = helper.DECIMAL;
+          break;
+        case VARCHAR:
+          reader = helper.VARCHAR;
+          break;
+        case VARBINARY:
+          reader = helper.VARBINARY;
+          break;
+        case DATEDAY:
+          reader = helper.INT4;
+          break;
+        case DATEMILLI:
+          reader = helper.INT8;
+          break;
+        case TIMESEC:
+        case TIMEMILLI:
+          reader = helper.INT4;
+          break;
+        case TIMEMICRO:
+        case TIMENANO:
+          reader = helper.INT8;
+          break;
+        case TIMESTAMPNANO:
+        case TIMESTAMPMICRO:
+        case TIMESTAMPMILLI:
+        case TIMESTAMPSEC:
+        case TIMESTAMPNANOTZ:
+        case TIMESTAMPMICROTZ:
+        case TIMESTAMPMILLITZ:
+        case TIMESTAMPSECTZ:
+          reader = helper.INT8;
+          break;
+        default:
+          throw new UnsupportedOperationException("Cannot read array of type " + type);
+      }
+    } else {
+      throw new InvalidArrowFileException("Unrecognized buffer type " + bufferType);
+    }
+
+    buf = reader.readBuffer(allocator, count);
+    assert buf != null;
+    return buf;
+  }
+
   private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException {
     List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
     ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()];
@@ -212,7 +491,7 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json
         throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name);
       }
 
-      /* Initialize the vector with required capacity but don't allocate since we would
+      /* Initialize the vector with required capacity but don't allocateNew since we would
        * be doing loadFieldBuffers.
        */
       int valueCount = readNextField("count", Integer.class);
@@ -221,29 +500,13 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json
       for (int v = 0; v < vectorTypes.size(); v++) {
         ArrowVectorType vectorType = vectorTypes.get(v);
         nextFieldIs(vectorType.getName());
-        readToken(START_ARRAY);
         int innerBufferValueCount = valueCount;
         if (vectorType.equals(OFFSET)) {
           /* offset buffer has 1 additional value capacity */
           innerBufferValueCount = valueCount + 1;
         }
-        for (int i = 0; i < innerBufferValueCount; i++) {
-          /* write data to the buffer */
-          parser.nextToken();
-          /* for variable width vectors, value count doesn't help pre-determining the capacity of
-           * the underlying data buffer. So we need to pass down the offset buffer (which was already
-           * populated in the previous iteration of this loop).
-           */
-          if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR
-                  || vector.getMinorType() == Types.MinorType.VARBINARY)) {
-            vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v],
-                    vectorBuffers[v-1], i, innerBufferValueCount);
-          } else {
-            vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v],
-                    null, i, innerBufferValueCount);
-          }
-        }
-        readToken(END_ARRAY);
+
+        vectorBuffers[v] = readBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount);
       }
 
       vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers));
@@ -253,7 +516,8 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json
       if (!fields.isEmpty()) {
         List<FieldVector> vectorChildren = vector.getChildrenFromFields();
         if (fields.size() != vectorChildren.size()) {
-          throw new IllegalArgumentException("fields and children are not the same size: " + fields.size() + " != " + vectorChildren.size());
+          throw new IllegalArgumentException(
+                  "fields and children are not the same size: " + fields.size() + " != " + vectorChildren.size());
         }
         nextFieldIs("children");
         readToken(START_ARRAY);
@@ -280,126 +544,6 @@ private byte[] decodeHexSafe(String hexString) throws IOException {
     }
   }
 
-  private ArrowBuf setValueFromParser(ArrowVectorType bufferType, FieldVector vector,
-                                      ArrowBuf buffer, ArrowBuf offsetBuffer, int index,
-                                      int valueCount) throws IOException {
-    if (bufferType.equals(TYPE)) {
-      buffer = NullableTinyIntVector.set(buffer, allocator,
-              valueCount, index, parser.readValueAs(Byte.class));
-    } else if (bufferType.equals(OFFSET)) {
-      buffer = BaseNullableVariableWidthVector.set(buffer, allocator,
-              valueCount, index, parser.readValueAs(Integer.class));
-    } else if (bufferType.equals(VALIDITY)) {
-      buffer = BitVectorHelper.setValidityBit(buffer, allocator,
-              valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0);
-    } else if (bufferType.equals(DATA)) {
-      switch (vector.getMinorType()) {
-        case BIT:
-          buffer = BitVectorHelper.setValidityBit(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0);
-          break;
-        case TINYINT:
-          buffer = NullableTinyIntVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Byte.class));
-          break;
-        case SMALLINT:
-          buffer = NullableSmallIntVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Short.class));
-          break;
-        case INT:
-          buffer = NullableIntVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Integer.class));
-          break;
-        case BIGINT:
-          buffer = NullableBigIntVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case FLOAT4:
-          buffer = NullableFloat4Vector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Float.class));
-          break;
-        case FLOAT8:
-          buffer = NullableFloat8Vector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Double.class));
-          break;
-        case DECIMAL:
-          buffer = NullableDecimalVector.set(buffer, allocator,
-                  valueCount, index, decodeHexSafe(parser.readValueAs(String.class)));
-          break;
-        case VARBINARY:
-          assert (offsetBuffer != null);
-          buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index,
-                  decodeHexSafe(parser.readValueAs(String.class)), valueCount);
-          break;
-        case VARCHAR:
-          assert (offsetBuffer != null);
-          buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index,
-                  parser.readValueAs(String.class).getBytes(UTF_8), valueCount);
-          break;
-        case DATEDAY:
-          buffer = NullableDateDayVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Integer.class));
-          break;
-        case DATEMILLI:
-          buffer = NullableDateMilliVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESEC:
-          buffer = NullableTimeSecVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Integer.class));
-          break;
-        case TIMEMILLI:
-          buffer = NullableTimeMilliVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Integer.class));
-          break;
-        case TIMEMICRO:
-          buffer = NullableTimeMicroVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMENANO:
-          buffer = NullableTimeNanoVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESTAMPSEC:
-          buffer = NullableTimeStampSecVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESTAMPMILLI:
-          buffer = NullableTimeStampMilliVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESTAMPMICRO:
-          buffer = NullableTimeStampMicroVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESTAMPNANO:
-          buffer = NullableTimeStampNanoVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESTAMPSECTZ:
-          buffer = NullableTimeStampSecTZVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESTAMPMILLITZ:
-          buffer = NullableTimeStampMilliTZVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESTAMPMICROTZ:
-          buffer = NullableTimeStampMicroTZVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        case TIMESTAMPNANOTZ:
-          buffer = NullableTimeStampNanoTZVector.set(buffer, allocator,
-                  valueCount, index, parser.readValueAs(Long.class));
-          break;
-        default:
-          throw new UnsupportedOperationException("minor type: " + vector.getMinorType());
-      }
-    }
-
-    return buffer;
-  }
-
   @Override
   public void close() throws IOException {
     parser.close();

From 837150e245823c6f0cd9e16dba89b6d1a0396aa7 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 13 Nov 2017 17:20:56 -0800
Subject: [PATCH 1252/1644] ARROW-1476: [JAVA] Implement Final ValueVector
 Updates

indentation and line length

rebase on master, fix tests

address review comments

integration test fix, some checkstyle fixes

review comments

indentation and javadocs
---
 .../codegen/templates/LegacyUnionVector.java  |  356 ---
 .../templates/NullableValueVectors.java       |  595 -----
 .../main/codegen/templates/UnionReader.java   |    2 +-
 .../main/codegen/templates/UnionVector.java   |   30 +-
 .../arrow/vector/BaseDataValueVector.java     |    6 +-
 .../vector/BaseNullableFixedWidthVector.java  | 1582 ++++++-------
 .../BaseNullableVariableWidthVector.java      | 2100 ++++++++++-------
 .../apache/arrow/vector/BaseValueVector.java  |    4 +-
 .../org/apache/arrow/vector/BitVector.java    |    4 +-
 .../apache/arrow/vector/BitVectorHelper.java  |  298 ++-
 .../arrow/vector/GenerateSampleData.java      |  336 +++
 .../arrow/vector/NullableBigIntVector.java    |  652 ++---
 .../arrow/vector/NullableBitVector.java       |  888 +++----
 .../arrow/vector/NullableDateDayVector.java   |  654 ++---
 .../arrow/vector/NullableDateMilliVector.java |  661 +++---
 .../arrow/vector/NullableDecimalVector.java   |  758 +++---
 .../arrow/vector/NullableFloat4Vector.java    |  654 ++---
 .../arrow/vector/NullableFloat8Vector.java    |  654 ++---
 .../arrow/vector/NullableIntVector.java       |  664 +++---
 .../vector/NullableIntervalDayVector.java     |  772 +++---
 .../vector/NullableIntervalYearVector.java    |  692 +++---
 .../arrow/vector/NullableSmallIntVector.java  |  708 +++---
 .../arrow/vector/NullableTimeMicroVector.java |  652 ++---
 .../arrow/vector/NullableTimeMilliVector.java |  656 ++---
 .../arrow/vector/NullableTimeNanoVector.java  |  652 ++---
 .../arrow/vector/NullableTimeSecVector.java   |  654 ++---
 .../NullableTimeStampMicroTZVector.java       |  362 ++-
 .../vector/NullableTimeStampMicroVector.java  |  363 ++-
 .../NullableTimeStampMilliTZVector.java       |  360 ++-
 .../vector/NullableTimeStampMilliVector.java  |  361 ++-
 .../vector/NullableTimeStampNanoTZVector.java |  362 ++-
 .../vector/NullableTimeStampNanoVector.java   |  363 ++-
 .../vector/NullableTimeStampSecTZVector.java  |  360 ++-
 .../vector/NullableTimeStampSecVector.java    |  363 ++-
 .../arrow/vector/NullableTimeStampVector.java |  368 +--
 .../arrow/vector/NullableTinyIntVector.java   |  708 +++---
 .../arrow/vector/NullableUInt1Vector.java     |  554 ++---
 .../arrow/vector/NullableUInt2Vector.java     |  554 ++---
 .../arrow/vector/NullableUInt4Vector.java     |  498 ++--
 .../arrow/vector/NullableUInt8Vector.java     |  498 ++--
 .../arrow/vector/NullableVarBinaryVector.java |  768 +++---
 .../arrow/vector/NullableVarCharVector.java   |  772 +++---
 .../org/apache/arrow/vector/ValueVector.java  |    4 +-
 .../apache/arrow/vector/VectorUnloader.java   |    3 -
 .../complex/BaseRepeatedValueVector.java      |   36 +-
 .../vector/complex/FixedSizeListVector.java   |   47 +-
 .../LegacyBaseRepeatedValueVector.java        |  240 --
 .../complex/LegacyFixedSizeListVector.java    |  298 ---
 .../vector/complex/LegacyListVector.java      |  288 ---
 .../arrow/vector/complex/LegacyMapVector.java |  229 --
 .../arrow/vector/complex/ListVector.java      |  219 +-
 .../vector/complex/NullableMapVector.java     |  103 +-
 .../vector/file/json/JsonFileReader.java      |   53 +-
 .../vector/file/json/JsonFileWriter.java      |   16 +-
 .../apache/arrow/vector/TestUnionVector.java  |   22 +-
 55 files changed, 11713 insertions(+), 13143 deletions(-)
 delete mode 100644 java/vector/src/main/codegen/templates/LegacyUnionVector.java
 delete mode 100644 java/vector/src/main/codegen/templates/NullableValueVectors.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java

diff --git a/java/vector/src/main/codegen/templates/LegacyUnionVector.java b/java/vector/src/main/codegen/templates/LegacyUnionVector.java
deleted file mode 100644
index b9cb6dfdc2482..0000000000000
--- a/java/vector/src/main/codegen/templates/LegacyUnionVector.java
+++ /dev/null
@@ -1,356 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/LegacyUnionVector.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-        package org.apache.arrow.vector.complex;
-
-<#include "/@includes/vv_imports.ftl" />
-        import com.google.common.collect.ImmutableList;
-        import java.util.ArrayList;
-        import java.util.Collections;
-        import java.util.Iterator;
-        import org.apache.arrow.vector.BaseDataValueVector;
-        import org.apache.arrow.vector.complex.impl.ComplexCopier;
-        import org.apache.arrow.vector.util.CallBack;
-        import org.apache.arrow.vector.schema.ArrowFieldNode;
-
-        import static org.apache.arrow.vector.types.UnionMode.Sparse;
-
-
-
-/*
- * This class is generated using freemarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-
-
-/**
- * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each
- * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods,
- * as well as the addOrGet method.
- *
- * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
- * each time the vector is accessed.
- * Source code generated using FreeMarker template ${.template_name}
- */
-public class LegacyUnionVector implements FieldVector {
-
-   private Accessor accessor = new Accessor();
-   private Mutator mutator = new Mutator();
-   private final UnionVector unionVector;
-
-   public LegacyUnionVector(String name, BufferAllocator allocator, CallBack callBack) {
-     unionVector = new UnionVector(name, allocator, callBack);
-   }
-
-   public BufferAllocator getAllocator() {
-      return unionVector.getAllocator();
-   }
-
-   @Override
-   public MinorType getMinorType() {
-      return MinorType.UNION;
-   }
-
-   @Override
-   public void initializeChildrenFromFields(List<Field> children) {
-      unionVector.initializeChildrenFromFields(children);
-   }
-
-   @Override
-   public List<FieldVector> getChildrenFromFields() {
-      return unionVector.getChildrenFromFields();
-   }
-
-   @Override
-   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-      unionVector.loadFieldBuffers(fieldNode, ownBuffers);
-   }
-
-   @Override
-   public List<ArrowBuf> getFieldBuffers() {
-      return unionVector.getFieldBuffers();
-   }
-
-   @Override
-   public List<BufferBacked> getFieldInnerVectors() {
-      return unionVector.getFieldInnerVectors();
-   }
-
-   @Override
-   public long getValidityBufferAddress() {
-      return unionVector.getValidityBufferAddress();
-   }
-
-   @Override
-   public long getDataBufferAddress() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public long getOffsetBufferAddress() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public ArrowBuf getValidityBuffer() {
-      return unionVector.getValidityBuffer();
-   }
-
-   @Override
-   public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); }
-
-   @Override
-   public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); }
-
-   public NullableMapVector getMap() {
-      return unionVector.getMap();
-   }
-  <#list vv.types as type>
-    <#list type.minor as minor>
-      <#assign name = minor.class?cap_first />
-      <#assign fields = minor.fields!type.fields />
-      <#assign uncappedName = name?uncap_first/>
-      <#assign lowerCaseName = name?lower_case/>
-      <#if !minor.typeParams?? >
-
-   private Nullable${name}Vector ${uncappedName}Vector;
-
-   public Nullable${name}Vector get${name}Vector() {
-      return unionVector.get${name}Vector();
-   }
-      </#if>
-    </#list>
-  </#list>
-
-   public ListVector getList() {
-      return unionVector.getList();
-   }
-
-   public int getTypeValue(int index) {
-      return unionVector.getTypeValue(index);
-   }
-
-   @Override
-   public void allocateNew() throws OutOfMemoryException {
-     unionVector.allocateNew();
-   }
-
-   @Override
-   public boolean allocateNewSafe() {
-     return unionVector.allocateNewSafe();
-   }
-
-   @Override
-   public void reAlloc() {
-      unionVector.reAlloc();
-   }
-
-   @Override
-   public void setInitialCapacity(int numRecords) {
-   }
-
-   @Override
-   public int getValueCapacity() {
-      return unionVector.getValueCapacity();
-   }
-
-   @Override
-   public void close() {
-     unionVector.close();
-   }
-
-   @Override
-   public void clear() {
-      unionVector.clear();
-   }
-
-   @Override
-   public Field getField() {
-      return unionVector.getField();
-   }
-
-   @Override
-   public TransferPair getTransferPair(BufferAllocator allocator) {
-      return unionVector.getTransferPair(allocator);
-   }
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      return unionVector.getTransferPair(ref, allocator);
-   }
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-      return unionVector.getTransferPair(ref, allocator, callBack);
-   }
-
-   @Override
-   public TransferPair makeTransferPair(ValueVector target) {
-      return unionVector.makeTransferPair(((LegacyUnionVector)target).unionVector);
-   }
-
-   public void copyFrom(int inIndex, int outIndex, UnionVector from) {
-      unionVector.copyFrom(inIndex, outIndex, from);
-   }
-
-   public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
-      unionVector.copyFromSafe(inIndex, outIndex, from);
-   }
-
-   public FieldVector addVector(FieldVector v) {
-     return unionVector.addVector(v);
-   }
-
-   @Override
-   public Accessor getAccessor() {
-      return accessor;
-   }
-
-   @Override
-   public Mutator getMutator() {
-      return mutator;
-   }
-
-   @Override
-   public FieldReader getReader() {
-      return unionVector.getReader();
-   }
-
-   public FieldWriter getWriter() {
-      return unionVector.getWriter();
-   }
-
-   @Override
-   public int getBufferSize() {
-      return unionVector.getBufferSize();
-   }
-
-   @Override
-   public int getBufferSizeFor(final int valueCount) {
-     return unionVector.getBufferSizeFor(valueCount);
-   }
-
-   @Override
-   public ArrowBuf[] getBuffers(boolean clear) {
-     return unionVector.getBuffers(clear);
-   }
-
-   @Override
-   public Iterator<ValueVector> iterator() {
-      return unionVector.iterator();
-   }
-
-   public class Accessor extends BaseValueVector.BaseAccessor {
-
-      @Override
-      public Object getObject(int index) {
-        return unionVector.getObject(index);
-      }
-
-      public byte[] get(int index) {
-         return unionVector.get(index);
-      }
-
-      public void get(int index, ComplexHolder holder) {
-      }
-
-      public void get(int index, UnionHolder holder) {
-         unionVector.get(index, holder);
-      }
-
-      public int getNullCount() {
-         return unionVector.getNullCount();
-      }
-
-      @Override
-      public int getValueCount() {
-         return unionVector.getValueCount();
-      }
-
-      @Override
-      public boolean isNull(int index) {
-         return unionVector.isNull(index);
-      }
-
-      public int isSet(int index) {
-         return unionVector.isSet(index);
-      }
-   }
-
-   public class Mutator extends BaseValueVector.BaseMutator {
-
-      UnionWriter writer;
-
-      @Override
-      public void setValueCount(int valueCount) {
-         unionVector.setValueCount(valueCount);
-      }
-
-      public void setSafe(int index, UnionHolder holder) {
-        unionVector.setSafe(index, holder);
-      }
-    <#list vv.types as type>
-      <#list type.minor as minor>
-        <#assign name = minor.class?cap_first />
-        <#assign fields = minor.fields!type.fields />
-        <#assign uncappedName = name?uncap_first/>
-        <#if !minor.typeParams?? >
-      public void setSafe(int index, Nullable${name}Holder holder) {
-         unionVector.setSafe(index, holder);
-      }
-
-        </#if>
-      </#list>
-    </#list>
-
-      public void setType(int index, MinorType type) {
-         unionVector.setType(index, type);
-      }
-
-      @Override
-      public void reset() { }
-
-      @Override
-      public void generateTestData(int values) { }
-   }
-
-   @Override
-   @Deprecated
-   public int getValueCount() { return getAccessor().getValueCount(); }
-
-   @Override
-   @Deprecated
-   public void setValueCount(int valueCount) { getMutator().setValueCount(valueCount);}
-
-   @Override
-   @Deprecated
-   public Object getObject(int index) { return getAccessor().getObject(index); }
-
-   @Override
-   @Deprecated
-   public int getNullCount() { return getAccessor().getNullCount(); }
-
-   @Override
-   @Deprecated
-   public boolean isNull(int index) { return getAccessor().isNull(index); }
-}
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
deleted file mode 100644
index 8e5b10b649b13..0000000000000
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ /dev/null
@@ -1,595 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-
-<#assign className = "LegacyNullable${minor.class}Vector" />
-<#assign valuesName = "Nullable${minor.class}Vector" />
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-
-<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector;
-
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import java.util.Collections;
-
-<#include "/@includes/vv_imports.ftl" />
-
-import org.apache.arrow.flatbuf.Precision;
-
-/**
- * ${className} implements a vector of values which could be null.  Elements in the vector
- * are first checked against a fixed length vector of boolean values.  Then the element is retrieved
- * from the base class (if not null).
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-@SuppressWarnings("unused")
-@Deprecated
-public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, FieldVector {
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
-
-protected final static byte[] emptyByteArray = new byte[]{};
-
-  private final String bitsField = "$bits$";
-  private final String valuesField = "$values$";
-
-  final BitVector bits = new BitVector(bitsField, allocator);
-  final ${valuesName} values;
-
-  private final Mutator mutator;
-  private final Accessor accessor;
-
-  <#if minor.typeParams??>
-    <#assign typeParams = minor.typeParams?reverse>
-    <#list typeParams as typeParam>
-  private final ${typeParam.type} ${typeParam.name};
-    </#list>
-
-  /**
-   * Assumes the type is nullable and not dictionary encoded
-   * @param name name of the field
-   * @param allocator allocator to use to resize the vector<#list typeParams as typeParam>
-   * @param ${typeParam.name} type parameter ${typeParam.name}</#list>
-   */
-  public ${className}(String name, BufferAllocator allocator<#list typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
-    <#if minor.arrowTypeConstructorParams??>
-       <#assign constructorParams = minor.arrowTypeConstructorParams />
-    <#else>
-       <#assign constructorParams = [] />
-       <#list typeParams as typeParam>
-         <#assign constructorParams = constructorParams + [ typeParam.name ] />
-      </#list>
-    </#if>
-    this(name, FieldType.nullable(new ${minor.arrowType}(${constructorParams?join(", ")})), allocator);
-  }
-  <#else>
-  public ${className}(String name, BufferAllocator allocator) {
-    this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
-  }
-  </#if>
-
-  public ${className}(String name, FieldType fieldType, BufferAllocator allocator) {
-    super(name, allocator);
-    <#if minor.typeParams??>
-    <#assign typeParams = minor.typeParams?reverse>
-    ${minor.arrowType} arrowType = (${minor.arrowType})fieldType.getType();
-    <#list typeParams as typeParam>
-    this.${typeParam.name} = arrowType.get${typeParam.name?cap_first}();
-    </#list>
-    this.values = new ${valuesName}(valuesField, allocator<#list typeParams as typeParam>, ${typeParam.name}</#list>);
-    <#else>
-    this.values = new ${valuesName}(valuesField, allocator);
-    </#if>
-    this.mutator = new Mutator();
-    this.accessor = new Accessor();
-  }
-
-  @Override
-  public List<BufferBacked> getFieldInnerVectors() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getFieldInnerVectors();
-  }
-
-  @Override
-  public void initializeChildrenFromFields(List<Field> children) {
-    if (!children.isEmpty()) {
-      throw new IllegalArgumentException("primitive type vector ${className} can not have children: " + children);
-    }
-  }
-
-  @Override
-  public List<FieldVector> getChildrenFromFields() {
-    return Collections.emptyList();
-  }
-
-  @Override
-  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    /* DELEGATE TO NEW VECTOR */
-    values.loadFieldBuffers(fieldNode, ownBuffers);
-  }
-
-  public List<ArrowBuf> getFieldBuffers() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getFieldBuffers();
-  }
-
-  @Override
-  public Field getField() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getField();
-  }
-
-  @Override
-  public MinorType getMinorType() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getMinorType();
-  }
-
-  @Override
-  public FieldReader getReader(){
-    /* DELEGATE TO NEW VECTOR */
-    return values.getReader();
-  }
-
-  @Override
-  public int getValueCapacity(){
-    /* DELEGATE TO NEW VECTOR */
-    return values.getValueCapacity();
-  }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getBuffers(clear);
-  }
-
-  @Override
-  public void close() {
-    /* DELEGATE TO NEW VECTOR */
-    values.close();
-  }
-
-  @Override
-  public void clear() {
-    /* DELEGATE TO NEW VECTOR */
-    values.clear();
-  }
-
-  @Override
-  public int getBufferSize(){
-    /* DELEGATE TO NEW VECTOR */
-    return values.getBufferSize();
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
-    /* DELEGATE TO NEW VECTOR */
-    return values.getBufferSizeFor(valueCount);
-  }
-
-  public ArrowBuf getBuffer() {
-    return values.getDataBuffer();
-  }
-
-  public ${valuesName} getValuesVector() {
-    return values;
-  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    /* DELEGATE TO NEW VECTOR */
-    values.setInitialCapacity(numRecords);
-  }
-
-  @Override
-  public void allocateNew() {
-    /* DELEGATE TO NEW VECTOR */
-    values.allocateNew();
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.allocateNewSafe();
-  }
-
-  @Override
-  public void reAlloc() {
-    /* DELEGATE TO NEW VECTOR */
-    values.reAlloc();
-  }
-
-  public void reset() {
-    /* DELEGATE TO NEW VECTOR */
-    values.reset();
-  }
-
-  <#if type.major == "VarLen">
-  @Override
-  public void allocateNew(int totalBytes, int valueCount) {
-    /* DELEGATE TO NEW VECTOR */
-    values.allocateNew(totalBytes, valueCount);
-  }
-
-  @Override
-  public int getByteCapacity(){
-    return values.getByteCapacity();
-  }
-
-  @Override
-  public int getCurrentSizeInBytes(){
-    return values.getCurrentSizeInBytes();
-  }
-
-  <#else>
-  @Override
-  public void allocateNew(int valueCount) {
-    /* DELEGATE TO NEW VECTOR */
-    values.allocateNew(valueCount);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void zeroVector() {
-    /* DELEGATE TO NEW VECTOR */
-    values.zeroVector();
-  }
-  </#if>
-
-
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getTransferPair(ref, allocator, callBack);
-  }
-
-
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator){
-    /* DELEGATE TO NEW VECTOR */
-    return values.getTransferPair(allocator);
-  }
-
-
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    /* DELEGATE TO NEW VECTOR */
-    return values.getTransferPair(ref, allocator);
-  }
-
-
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    /* DELEGATE TO NEW VECTOR */
-    return values.makeTransferPair(to);
-  }
-
-
-
-  public void transferTo(${valuesName} target) {
-    /* DELEGATE TO NEW VECTOR */
-    <#if type.major == "VarLen">
-        values.transferTo((BaseNullableVariableWidthVector) target);
-    <#else>
-        values.transferTo((BaseNullableFixedWidthVector) target);
-    </#if>
-  }
-
-  public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) {
-    /* DELEGATE TO NEW VECTOR */
-    <#if type.major == "VarLen">
-        values.splitAndTransferTo(startIndex, length, (BaseNullableVariableWidthVector) target);
-    <#else>
-        values.splitAndTransferTo(startIndex, length, (BaseNullableFixedWidthVector) target);
-    </#if>
-  }
-
-
-
-  @Override
-  public Accessor getAccessor(){
-    return accessor;
-  }
-
-  @Override
-  public Mutator getMutator(){
-    return mutator;
-  }
-
-
-  public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) {
-    /* DELEGATE TO NEW VECTOR */
-    values.copyFrom(fromIndex, thisIndex, from);
-  }
-
-  public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) {
-    /* DELEGATE TO NEW VECTOR */
-    values.copyFromSafe(fromIndex, thisIndex, from);
-  }
-
-  @Override
-  public long getValidityBufferAddress() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getValidityBufferAddress();
-  }
-
-  @Override
-  public long getDataBufferAddress() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getDataBufferAddress();
-  }
-
-  @Override
-  public long getOffsetBufferAddress() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getOffsetBufferAddress();
-  }
-
-  @Override
-  public ArrowBuf getValidityBuffer() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getValidityBuffer();
-  }
-
-  @Override
-  public ArrowBuf getDataBuffer() {
-    return (values.getDataBuffer());
-  }
-
-  @Override
-  public ArrowBuf getOffsetBuffer() {
-    /* DELEGATE TO NEW VECTOR */
-    return values.getOffsetBuffer();
-  }
-
-  public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
-
-    /**
-     * Get the element at the specified position.
-     *
-     * @param  index   position of the value
-     * @return value of the element, if not null
-     */
-    public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
-      /* DELEGATE TO NEW VECTOR */
-      return values.get(index);
-    }
-
-    @Override
-    public boolean isNull(int index) {
-      /* DELEGATE TO NEW VECTOR */
-      return values.isNull(index);
-    }
-
-    public int isSet(int index){
-      /* DELEGATE TO NEW VECTOR */
-      return values.isSet(index);
-    }
-
-    <#if type.major == "VarLen">
-    public long getStartEnd(int index){
-        /* DELEGATE TO NEW VECTOR */
-        return values.getStartEnd(index);
-    }
-
-    @Override
-    public int getValueLength(int index) {
-        /* DELEGATE TO NEW VECTOR */
-        return values.getValueLength(index);
-    }
-    </#if>
-
-    public void get(int index, Nullable${minor.class}Holder holder){
-        /* DELEGATE TO NEW VECTOR */
-        values.get(index, holder);
-    }
-
-    @Override
-    public ${friendlyType} getObject(int index) {
-      /* DELEGATE TO NEW VECTOR */
-      return values.getObject(index);
-    }
-
-    <#if minor.class == "IntervalYear" || minor.class == "IntervalDay">
-    public StringBuilder getAsStringBuilder(int index) {
-       /* DELEGATE TO NEW VECTOR */
-       return values.getAsStringBuilder(index);
-    }
-    </#if>
-
-    @Override
-    public int getValueCount(){
-      /* DELEGATE TO NEW VECTOR */
-      return values.getValueCount();
-    }
-
-    public void reset() { }
-  }
-
-  public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> {
-    private int setCount;
-    <#if type.major = "VarLen"> private int lastSet = -1;</#if>
-
-    private Mutator() { }
-
-    public ${valuesName} getVectorWithValues() {
-      return values;
-    }
-
-
-    @Override
-    public void setIndexDefined(int index) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setIndexDefined(index);
-    }
-
-
-
-    /**
-     * Set the variable length element at the specified index to the supplied byte array.
-     *
-     * @param index   position of the bit to set
-     * @param value   array of bytes (or int if smaller than 4 bytes) to write
-     */
-    public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
-       /* DELEGATE TO NEW VECTOR */
-       values.set(index, value);
-    }
-
-
-
-    <#if type.major == "VarLen">
-    public void fillEmpties(int index) {
-      /* DELEGATE TO NEW VECTOR */
-      values.fillEmpties(index);
-    }
-
-    @Override
-    public void setValueLengthSafe(int index, int length) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setValueLengthSafe(index, length);
-    }
-    </#if>
-
-
-    public void setSafe(int index, byte[] value, int start, int length) {
-       /* DELEGATE TO NEW VECTOR */
-      values.setSafe(index, value, start, length);
-    }
-
-
-    public void setSafe(int index, ByteBuffer value, int start, int length) {
-       /* DELEGATE TO NEW VECTOR */
-       values.setSafe(index, value, start, length);
-    }
-
-
-    public void setNull(int index) {
-       /* DELEGATE TO NEW VECTOR */
-       values.setNull(index);
-    }
-
-
-    public void set(int index, Nullable${minor.class}Holder holder) {
-      /* DELEGATE TO NEW VECTOR */
-      values.set(index, holder);
-    }
-
-
-    public void set(int index, ${minor.class}Holder holder) {
-        /* DELEGATE TO NEW VECTOR */
-        values.set(index, holder);
-    }
-
-
-    public boolean isSafe(int outIndex) {
-       /* DELEGATE TO NEW VECTOR */
-       return values.isSafe(outIndex);
-    }
-
-
-    <#assign fields = minor.fields!type.fields />
-    public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field</#list> ){
-      values.set(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
-    }
-
-    public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
-      values.setSafe(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
-    }
-
-
-    public void setSafe(int index, Nullable${minor.class}Holder value) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setSafe(index, value);
-    }
-
-    public void setSafe(int index, ${minor.class}Holder value) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setSafe(index, value);
-    }
-
-
-    <#if !(type.major == "VarLen" || minor.class == "IntervalDay")>
-    public void setSafe(int index, ${minor.javaType!type.javaType} value) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setSafe(index, value);
-    }
-    </#if>
-
-
-
-    <#if minor.class == "Decimal">
-    public void set(int index, ${friendlyType} value) {
-      /* DELEGATE TO NEW VECTOR */
-      values.set(index, value);
-    }
-
-    public void setSafe(int index, ${friendlyType} value) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setSafe(index, value);
-    }
-    </#if>
-
-
-    @Override
-    public void setValueCount(int valueCount) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setValueCount(valueCount);
-    }
-
-
-    /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */
-    @Override
-    public void generateTestData(int valueCount) { }
-
-
-    /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */
-    @Override
-    public void reset() { }
-
-
-    <#if type.major == "VarLen">
-    public void setLastSet(int value) {
-      /* DELEGATE TO NEW VECTOR */
-      values.setLastSet(value);
-    }
-
-
-    public int getLastSet() {
-      /* DELEGATE TO NEW VECTOR */
-      return values.getLastSet();
-    }
-    </#if>
-  }
-}
-</#list>
-</#list>
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index fd3a766fc2cb4..98bb7c1f53d84 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -60,7 +60,7 @@ public Field getField() {
   }
 
   public boolean isSet(){
-    return !data.getAccessor().isNull(idx());
+    return !data.isNull(idx());
   }
 
   public void read(UnionHolder holder) {
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 3da383fe6f3c5..e44edbd47b67a 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -63,7 +63,7 @@ public class UnionVector implements FieldVector {
   int valueCount;
 
   MapVector internalMap;
-  ArrowBuf typeBuffer;
+  protected ArrowBuf typeBuffer;
 
   private NullableMapVector mapVector;
   private ListVector listVector;
@@ -121,15 +121,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
   @Override
   public List<ArrowBuf> getFieldBuffers() {
     List<ArrowBuf> result = new ArrayList<>(1);
-
-    typeBuffer.readerIndex(0);
-    typeBuffer.writerIndex(valueCount * TYPE_WIDTH);
-
+    setReaderAndWriterIndex();
     result.add(typeBuffer);
 
     return result;
   }
 
+  private void setReaderAndWriterIndex() {
+    typeBuffer.readerIndex(0);
+    typeBuffer.writerIndex(valueCount * TYPE_WIDTH);
+  }
+
   @Override
   @Deprecated
   public List<BufferBacked> getFieldInnerVectors() {
@@ -232,6 +234,8 @@ public int getTypeValue(int index) {
 
   @Override
   public void allocateNew() throws OutOfMemoryException {
+    /* new allocation -- clear the current buffers */
+    clear();
     internalMap.allocateNew();
     try {
       allocateTypeBuffer();
@@ -243,6 +247,8 @@ public void allocateNew() throws OutOfMemoryException {
 
   @Override
   public boolean allocateNewSafe() {
+    /* new allocation -- clear the current buffers */
+    clear();
     boolean safe = internalMap.allocateNewSafe();
     if (!safe) { return false; }
     try {
@@ -386,6 +392,7 @@ public void transfer() {
       to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer;
       internalMapVectorTransferPair.transfer();
       to.valueCount = valueCount;
+      clear();
     }
 
     @Override
@@ -460,8 +467,17 @@ public int getBufferSizeFor(final int valueCount) {
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
     ImmutableList.Builder<ArrowBuf> builder = ImmutableList.builder();
-    builder.add(typeBuffer);
-    builder.add(internalMap.getBuffers(clear));
+    setReaderAndWriterIndex();
+    if (getBufferSize() != 0) {
+      builder.add(typeBuffer);
+      builder.add(internalMap.getBuffers(clear));
+    }
+    if (clear) {
+      valueCount = 0;
+      typeBuffer.retain();
+      typeBuffer.release();
+      typeBuffer = allocator.getEmpty();
+    }
     List<ArrowBuf> list = builder.build();
     return list.toArray(new ArrowBuf[list.size()]);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 01340f66c4095..38524ff8adf82 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -85,7 +85,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
     if (getBufferSize() == 0) {
       out = new ArrowBuf[0];
     } else {
-      out = new ArrowBuf[] {data};
+      out = new ArrowBuf[]{data};
       data.readerIndex(0);
       if (clear) {
         data.retain(1);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
index 83dc409a74f14..209758e4ece9e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -41,806 +41,810 @@
  * implying that zero or more elements in the vector could be NULL.
  */
 public abstract class BaseNullableFixedWidthVector extends BaseValueVector
-        implements FixedWidthVector, FieldVector {
-   private final byte typeWidth;
-
-   protected int valueAllocationSizeInBytes;
-   protected int validityAllocationSizeInBytes;
-
-   protected final Field field;
-   private int allocationMonitor;
-   protected ArrowBuf validityBuffer;
-   protected ArrowBuf valueBuffer;
-   protected int valueCount;
-
-   public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator,
-                                       FieldType fieldType, final byte typeWidth) {
-      super(name, allocator);
-      this.typeWidth = typeWidth;
-      field = new Field(name, fieldType, null);
-      valueCount = 0;
+        implements FixedWidthVector, FieldVector, NullableVectorDefinitionSetter {
+  private final byte typeWidth;
+
+  protected int valueAllocationSizeInBytes;
+  protected int validityAllocationSizeInBytes;
+
+  protected final Field field;
+  private int allocationMonitor;
+  protected ArrowBuf validityBuffer;
+  protected ArrowBuf valueBuffer;
+  protected int valueCount;
+
+  public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator,
+                                      FieldType fieldType, final byte typeWidth) {
+    super(name, allocator);
+    this.typeWidth = typeWidth;
+    field = new Field(name, fieldType, null);
+    valueCount = 0;
+    allocationMonitor = 0;
+    validityBuffer = allocator.getEmpty();
+    valueBuffer = allocator.getEmpty();
+    if (typeWidth > 0) {
+      valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth;
+      validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
+    } else {
+      /* specialized handling for NullableBitVector */
+      valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
+      validityAllocationSizeInBytes = valueAllocationSizeInBytes;
+    }
+  }
+
+
+  /* TODO:
+   * see if getNullCount() can be made faster -- O(1)
+   */
+
+  /* TODO:
+   * Once the entire hierarchy has been refactored, move common functions
+   * like getNullCount(), splitAndTransferValidityBuffer to top level
+   * base class BaseValueVector.
+   *
+   * Along with this, some class members (validityBuffer) can also be
+   * abstracted out to top level base class.
+   *
+   * Right now BaseValueVector is the top level base class for other
+   * vector types in ValueVector hierarchy (non-nullable) and those
+   * vectors have not yet been refactored/removed so moving things to
+   * the top class as of now is not a good idea.
+   */
+
+
+  @Override
+  @Deprecated
+  public Mutator getMutator() {
+    throw new UnsupportedOperationException("Mutator is not supported for writing to vector");
+  }
+
+  @Override
+  @Deprecated
+  public Accessor getAccessor() {
+    throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
+  }
+
+  /**
+   * Get the memory address of buffer that manages the validity
+   * (NULL or NON-NULL nature) of elements in the vector.
+   * @return starting address of the buffer
+   */
+  @Override
+  public long getValidityBufferAddress() {
+    return (validityBuffer.memoryAddress());
+  }
+
+  /**
+   * Get the memory address of buffer that stores the data for elements
+   * in the vector.
+   * @return starting address of the buffer
+   */
+  @Override
+  public long getDataBufferAddress() {
+    return (valueBuffer.memoryAddress());
+  }
+
+  /**
+   * Get the memory address of buffer that stores the offsets for elements
+   * in the vector. This operation is not supported for fixed-width vectors.
+   * @return starting address of the buffer
+   * @throws UnsupportedOperationException for fixed width vectors
+   */
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException("not supported for fixed-width vectors");
+  }
+
+  /**
+   * Get buffer that manages the validity (NULL or NON-NULL nature) of
+   * elements in the vector. Consider it as a buffer for internal bit vector
+   * data structure.
+   * @return buffer
+   */
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    return validityBuffer;
+  }
+
+  /**
+   * Get the buffer that stores the data for elements in the vector.
+   * @return buffer
+   */
+  @Override
+  public ArrowBuf getDataBuffer() {
+    return valueBuffer;
+  }
+
+  /**
+   * buffer that stores the offsets for elements
+   * in the vector. This operation is not supported for fixed-width vectors.
+   * @return buffer
+   * @throws UnsupportedOperationException for fixed width vectors
+   */
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException("not supported for fixed-width vectors");
+  }
+
+  /**
+   * Sets the desired value capacity for the vector. This function doesn't
+   * allocate any memory for the vector.
+   * @param valueCount desired number of elements in the vector
+   */
+  @Override
+  public void setInitialCapacity(int valueCount) {
+    final long size = (long) valueCount * typeWidth;
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+    }
+    valueAllocationSizeInBytes = (int) size;
+    validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
+  }
+
+  /**
+   * Get the current value capacity for the vector
+   * @return number of elements that vector can hold.
+   */
+  @Override
+  public int getValueCapacity() {
+    return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity());
+  }
+
+  private int getValueBufferValueCapacity() {
+    return (int) ((valueBuffer.capacity() * 1.0) / typeWidth);
+  }
+
+  private int getValidityBufferValueCapacity() {
+    return (int) (validityBuffer.capacity() * 8L);
+  }
+
+  /**
+   * zero out the vector and the data in associated buffers.
+   */
+  @Override
+  public void zeroVector() {
+    initValidityBuffer();
+    initValueBuffer();
+  }
+
+  /* zero out the validity buffer */
+  private void initValidityBuffer() {
+    validityBuffer.setZero(0, validityBuffer.capacity());
+  }
+
+  /* zero out the data buffer */
+  private void initValueBuffer() {
+    valueBuffer.setZero(0, valueBuffer.capacity());
+  }
+
+  /**
+   * Reset the vector to initial state. Same as {@link #zeroVector()}.
+   * Note that this method doesn't release any memory.
+   */
+  public void reset() {
+    zeroVector();
+  }
+
+  /**
+   * Close the vector and release the associated buffers.
+   */
+  @Override
+  public void close() {
+    clear();
+  }
+
+  /**
+   * Same as {@link #close()}
+   */
+  @Override
+  public void clear() {
+    valueCount = 0;
+    validityBuffer = releaseBuffer(validityBuffer);
+    valueBuffer = releaseBuffer(valueBuffer);
+  }
+
+  /* used to step down the memory allocation */
+  protected void incrementAllocationMonitor() {
+    if (allocationMonitor < 0) {
       allocationMonitor = 0;
-      validityBuffer = allocator.getEmpty();
-      valueBuffer = allocator.getEmpty();
-      if (typeWidth > 0) {
-         valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth;
-         validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
-      } else {
-         /* specialized handling for NullableBitVector */
-         valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
-         validityAllocationSizeInBytes = valueAllocationSizeInBytes;
-      }
-   }
-
-
-   /* TODO:
-    *
-    * see if getNullCount() can be made faster -- O(1)
-    */
-
-   /* TODO:
-    * Once the entire hierarchy has been refactored, move common functions
-    * like getNullCount(), splitAndTransferValidityBuffer to top level
-    * base class BaseValueVector.
-    *
-    * Along with this, some class members (validityBuffer) can also be
-    * abstracted out to top level base class.
-    *
-    * Right now BaseValueVector is the top level base class for other
-    * vector types in ValueVector hierarchy (non-nullable) and those
-    * vectors have not yet been refactored/removed so moving things to
-    * the top class as of now is not a good idea.
-    */
-
-
-   @Override
-   @Deprecated
-   public Mutator getMutator() {
-      throw new UnsupportedOperationException("Mutator is not supported for writing to vector");
-   }
-
-   @Override
-   @Deprecated
-   public Accessor getAccessor() {
-      throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
-   }
-
-   /**
-    * Get the memory address of buffer that manages the validity
-    * (NULL or NON-NULL nature) of elements in the vector.
-    * @return starting address of the buffer
-    */
-   @Override
-   public long getValidityBufferAddress() {
-      return (validityBuffer.memoryAddress());
-   }
-
-   /**
-    * Get the memory address of buffer that stores the data for elements
-    * in the vector.
-    * @return starting address of the buffer
-    */
-   @Override
-   public long getDataBufferAddress() {
-      return (valueBuffer.memoryAddress());
-   }
-
-   /**
-    * Get the memory address of buffer that stores the offsets for elements
-    * in the vector. This operation is not supported for fixed-width vectors.
-    * @return starting address of the buffer
-    * @throws UnsupportedOperationException for fixed width vectors
-    */
-   @Override
-   public long getOffsetBufferAddress() {
-      throw new UnsupportedOperationException("not supported for fixed-width vectors");
-   }
-
-   /**
-    * Get buffer that manages the validity (NULL or NON-NULL nature) of
-    * elements in the vector. Consider it as a buffer for internal bit vector
-    * data structure.
-    * @return buffer
-    */
-   @Override
-   public ArrowBuf getValidityBuffer() {
-      return validityBuffer;
-   }
-
-   /**
-    * Get the buffer that stores the data for elements in the vector.
-    * @return buffer
-    */
-   @Override
-   public ArrowBuf getDataBuffer() {
-      return valueBuffer;
-   }
-
-   /**
-    * buffer that stores the offsets for elements
-    * in the vector. This operation is not supported for fixed-width vectors.
-    * @return buffer
-    * @throws UnsupportedOperationException for fixed width vectors
-    */
-   @Override
-   public ArrowBuf getOffsetBuffer() {
-      throw new UnsupportedOperationException("not supported for fixed-width vectors");
-   }
-
-   /**
-    * Sets the desired value capacity for the vector. This function doesn't
-    * allocate any memory for the vector.
-    * @param valueCount desired number of elements in the vector
-    */
-   @Override
-   public void setInitialCapacity(int valueCount) {
-      final long size = (long)valueCount * typeWidth;
-      if (size > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
-      }
-      valueAllocationSizeInBytes = (int)size;
-      validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
-   }
-
-   /**
-    * Get the current value capacity for the vector
-    * @return number of elements that vector can hold.
-    */
-   @Override
-   public int getValueCapacity(){
-      return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity());
-   }
-
-   private int getValueBufferValueCapacity() {
-      return (int)((valueBuffer.capacity() * 1.0)/typeWidth);
-   }
-
-   private int getValidityBufferValueCapacity() {
-      return (int)(validityBuffer.capacity() * 8L);
-   }
-
-   /**
-    * zero out the vector and the data in associated buffers.
-    */
-   @Override
-   public void zeroVector() {
-      initValidityBuffer();
-      initValueBuffer();
-   }
-
-   /* zero out the validity buffer */
-   private void initValidityBuffer() {
-      validityBuffer.setZero(0, validityBuffer.capacity());
-   }
-
-   /* zero out the data buffer */
-   private void initValueBuffer() {
-      valueBuffer.setZero(0, valueBuffer.capacity());
-   }
-
-   /**
-    * Reset the vector to initial state. Same as {@link #zeroVector()}.
-    * Note that this method doesn't release any memory.
-    */
-   public void reset() {
-      zeroVector();
-   }
-
-   /**
-    * Close the vector and release the associated buffers.
-    */
-   @Override
-   public void close() { clear(); }
-
-   /**
-    * Same as {@link #close()}
-    */
-   @Override
-   public void clear() {
-      valueCount = 0;
-      validityBuffer = releaseBuffer(validityBuffer);
-      valueBuffer = releaseBuffer(valueBuffer);
-   }
-
-   /* used to step down the memory allocation */
-   protected void incrementAllocationMonitor() {
-      if (allocationMonitor < 0) {
-         allocationMonitor = 0;
-      }
-      allocationMonitor++;
-   }
+    }
+    allocationMonitor++;
+  }
 
-   /* used to step up the memory allocation */
-   protected void decrementAllocationMonitor() {
-      if (allocationMonitor > 0) {
-         allocationMonitor = 0;
-      }
-      allocationMonitor--;
-   }
-
-   /**
-    * Same as {@link #allocateNewSafe()}.
-    */
-   @Override
-   public void allocateNew() {
-      if(!allocateNewSafe()){
-         throw new OutOfMemoryException("Failure while allocating memory.");
-      }
-   }
-
-   /**
-    * Allocate memory for the vector. We internally use a default value count
-    * of 4096 to allocate memory for at least these many elements in the
-    * vector. See {@link #allocateNew(int)} for allocating memory for specific
-    * number of elements in the vector.
-    *
-    * @return false if memory allocation fails, true otherwise.
-    */
-   @Override
-   public boolean allocateNewSafe() {
-      long curAllocationSizeValue = valueAllocationSizeInBytes;
-      long curAllocationSizeValidity = validityAllocationSizeInBytes;
-
-      if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Requested amount of memory exceeds limit");
-      }
+  /* used to step up the memory allocation */
+  protected void decrementAllocationMonitor() {
+    if (allocationMonitor > 0) {
+      allocationMonitor = 0;
+    }
+    allocationMonitor--;
+  }
+
+  /**
+   * Same as {@link #allocateNewSafe()}.
+   */
+  @Override
+  public void allocateNew() {
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryException("Failure while allocating memory.");
+    }
+  }
+
+  /**
+   * Allocate memory for the vector. We internally use a default value count
+   * of 4096 to allocate memory for at least these many elements in the
+   * vector. See {@link #allocateNew(int)} for allocating memory for specific
+   * number of elements in the vector.
+   *
+   * @return false if memory allocation fails, true otherwise.
+   */
+  @Override
+  public boolean allocateNewSafe() {
+    long curAllocationSizeValue = valueAllocationSizeInBytes;
+    long curAllocationSizeValidity = validityAllocationSizeInBytes;
+
+    if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory exceeds limit");
+    }
 
       /* we are doing a new allocation -- release the current buffers */
-      clear();
-
-      try{
-         allocateBytes(curAllocationSizeValue, curAllocationSizeValidity);
-      } catch (Exception e) {
-         e.printStackTrace();
-         clear();
-         return false;
-      }
+    clear();
 
-      return true;
-   }
-
-   /**
-    * Allocate memory for the vector to support storing at least the provided number of
-    * elements in the vector. This method must be called prior to using the ValueVector.
-    *
-    * @param valueCount the desired number of elements in the vector
-    * @throws org.apache.arrow.memory.OutOfMemoryException
-    */
-   public void allocateNew(int valueCount) {
-      long valueBufferSize = valueCount * typeWidth;
-      long validityBufferSize = getValidityBufferSizeFromCount(valueCount);
-      if (typeWidth == 0) {
-         /* specialized handling for NullableBitVector */
-         valueBufferSize = validityBufferSize;
-      }
-
-      if (allocationMonitor > 10) {
-         /* step down the default memory allocation since we have observed
-          * multiple times that provisioned value capacity was much larger than
-          * actually needed. see setValueCount for more details.
-          */
-         valueBufferSize = Math.max(8, valueBufferSize / 2);
-         validityBufferSize = Math.max(8, validityBufferSize / 2);
-         allocationMonitor = 0;
-      } else if (allocationMonitor < -2) {
-         valueBufferSize = valueBufferSize * 2L;
-         validityBufferSize = validityBufferSize * 2L;
-         allocationMonitor = 0;
-      }
-
-      if (valueBufferSize > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
-      }
-
-      /* we are doing a new allocation -- release the current buffers */
+    try {
+      allocateBytes(curAllocationSizeValue, curAllocationSizeValidity);
+    } catch (Exception e) {
+      e.printStackTrace();
       clear();
-
-      try {
-         allocateBytes(valueBufferSize, validityBufferSize);
-      } catch(Exception e) {
-         e.printStackTrace();
-         clear();
-         throw e;
-      }
-   }
-
-   /**
-    * Actual memory allocation is done by this function. All the calculations
-    * and knowledge about what size to allocate is upto the callers of this
-    * method.
-    * Callers appropriately handle errors if memory allocation fails here.
-    * Callers should also take care of determining that desired size is
-    * within the bounds of max allocation allowed and any other error
-    * conditions.
-    */
-   private void allocateBytes(final long valueBufferSize, final long validityBufferSize) {
-      /* allocate data buffer */
-      int curSize = (int)valueBufferSize;
-      valueBuffer = allocator.buffer(curSize);
-      valueBuffer.readerIndex(0);
-      valueAllocationSizeInBytes = curSize;
-
-      /* allocate validity buffer */
-      allocateValidityBuffer((int)validityBufferSize);
-      initValidityBuffer();
-   }
-
-   /**
-    * During splitAndTransfer, if we splitting from a random position within a byte,
-    * we can't just slice the source buffer so we have to explicitly allocate the
-    * validityBuffer of the target vector. This is unlike the databuffer which we can
-    * always slice for the target vector.
-    */
-   private void allocateValidityBuffer(final int validityBufferSize) {
-      validityBuffer = allocator.buffer(validityBufferSize);
-      validityBuffer.readerIndex(0);
-      validityAllocationSizeInBytes = validityBufferSize;
-      initValidityBuffer();
-   }
-
-   /**
-    * Get the potential buffer size for a particular number of records.
-    * @param count desired number of elements in the vector
-    * @return estimated size of underlying buffers if the vector holds
-    *         a given number of elements
-    */
-   @Override
-   public int getBufferSizeFor(final int count) {
-      if (count == 0) { return 0; }
-      return (count * typeWidth) + getValidityBufferSizeFromCount(count);
-   }
-
-   /**
-    * Get the size (number of bytes) of underlying buffers used by this
-    * vector
-    * @return size of underlying buffers.
-    */
-   @Override
-   public int getBufferSize() {
-      if (valueCount == 0) { return 0; }
-      return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount);
-   }
-
-   /**
-    * Get information about how this field is materialized.
-    * @return the field corresponding to this vector
-    */
-   @Override
-   public Field getField() {
-      return field;
-   }
-
-   /**
-    * Return the underlying buffers associated with this vector. Note that this doesn't
-    * impact the reference counts for this buffer so it only should be used for in-context
-    * access. Also note that this buffer changes regularly thus
-    * external classes shouldn't hold a reference to it (unless they change it).
-    *
-    * @param clear Whether to clear vector before returning; the buffers will still be refcounted
-    *              but the returned array will be the only reference to them
-    * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
-    *         vector instance.
-    */
-   @Override
-   public ArrowBuf[] getBuffers(boolean clear) {
-      final ArrowBuf[] buffers = new ArrowBuf[2];
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * Allocate memory for the vector to support storing at least the provided number of
+   * elements in the vector. This method must be called prior to using the ValueVector.
+   *
+   * @param valueCount the desired number of elements in the vector
+   * @throws org.apache.arrow.memory.OutOfMemoryException
+   */
+  public void allocateNew(int valueCount) {
+    long valueBufferSize = valueCount * typeWidth;
+    long validityBufferSize = getValidityBufferSizeFromCount(valueCount);
+    if (typeWidth == 0) {
+      /* specialized handling for NullableBitVector */
+      valueBufferSize = validityBufferSize;
+    }
+
+    if (valueBufferSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+    }
+
+    /* we are doing a new allocation -- release the current buffers */
+    clear();
+
+    try {
+      allocateBytes(valueBufferSize, validityBufferSize);
+    } catch (Exception e) {
+      e.printStackTrace();
+      clear();
+      throw e;
+    }
+  }
+
+  /**
+   * Actual memory allocation is done by this function. All the calculations
+   * and knowledge about what size to allocate is upto the callers of this
+   * method.
+   * Callers appropriately handle errors if memory allocation fails here.
+   * Callers should also take care of determining that desired size is
+   * within the bounds of max allocation allowed and any other error
+   * conditions.
+   */
+  private void allocateBytes(final long valueBufferSize, final long validityBufferSize) {
+    /* allocate data buffer */
+    int curSize = (int) valueBufferSize;
+    valueBuffer = allocator.buffer(curSize);
+    valueBuffer.readerIndex(0);
+    valueAllocationSizeInBytes = curSize;
+    /* allocate validity buffer */
+    allocateValidityBuffer((int) validityBufferSize);
+    zeroVector();
+  }
+
+  /**
+   * During splitAndTransfer, if we splitting from a random position within a byte,
+   * we can't just slice the source buffer so we have to explicitly allocate the
+   * validityBuffer of the target vector. This is unlike the databuffer which we can
+   * always slice for the target vector.
+   */
+  private void allocateValidityBuffer(final int validityBufferSize) {
+    validityBuffer = allocator.buffer(validityBufferSize);
+    validityBuffer.readerIndex(0);
+    validityAllocationSizeInBytes = validityBufferSize;
+  }
+
+  /**
+   * Get the potential buffer size for a particular number of records.
+   * @param count desired number of elements in the vector
+   * @return estimated size of underlying buffers if the vector holds
+   *         a given number of elements
+   */
+  @Override
+  public int getBufferSizeFor(final int count) {
+    if (count == 0) {
+      return 0;
+    }
+    return (count * typeWidth) + getValidityBufferSizeFromCount(count);
+  }
+
+  /**
+   * Get the size (number of bytes) of underlying buffers used by this
+   * vector
+   * @return size of underlying buffers.
+   */
+  @Override
+  public int getBufferSize() {
+    if (valueCount == 0) {
+      return 0;
+    }
+    return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount);
+  }
+
+  /**
+   * Get information about how this field is materialized.
+   * @return the field corresponding to this vector
+   */
+  @Override
+  public Field getField() {
+    return field;
+  }
+
+  /**
+   * Return the underlying buffers associated with this vector. Note that this doesn't
+   * impact the reference counts for this buffer so it only should be used for in-context
+   * access. Also note that this buffer changes regularly thus
+   * external classes shouldn't hold a reference to it (unless they change it).
+   *
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
+   *              but the returned array will be the only reference to them
+   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
+   *         vector instance.
+   */
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers;
+    setReaderAndWriterIndex();
+    if (getBufferSize() == 0) {
+      buffers = new ArrowBuf[0];
+    } else {
+      buffers = new ArrowBuf[2];
       buffers[0] = validityBuffer;
       buffers[1] = valueBuffer;
-      if (clear) {
-         for (final ArrowBuf buffer:buffers) {
-            buffer.retain(1);
-         }
-         clear();
+    }
+    if (clear) {
+      for (final ArrowBuf buffer : buffers) {
+        buffer.retain(1);
       }
-      return buffers;
-   }
-
-   /**
-    * Resize the vector to increase the capacity. The internal behavior is to
-    * double the current value capacity.
-    */
-   @Override
-   public void reAlloc() {
-      valueBuffer = reallocBufferHelper(valueBuffer, true);
-      validityBuffer = reallocBufferHelper(validityBuffer, false);
-   }
-
-   /**
-    * Helper method for reallocating a particular internal buffer
-    * Returns the new buffer.
-    */
-   private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) {
-      final int currentBufferCapacity = buffer.capacity();
-      long baseSize  = (dataBuffer ? valueAllocationSizeInBytes
-                                   : validityAllocationSizeInBytes);
-
-      if (baseSize < (long)currentBufferCapacity) {
-         baseSize = (long)currentBufferCapacity;
-      }
-
-      long newAllocationSize = baseSize * 2L;
-      newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
-
-      if (newAllocationSize > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Unable to expand the buffer");
-      }
-
-      final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
-      newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
-      final int halfNewCapacity = newBuf.capacity() / 2;
-      newBuf.setZero(halfNewCapacity, halfNewCapacity);
-      buffer.release(1);
-      buffer = newBuf;
-      if (dataBuffer) {
-         valueAllocationSizeInBytes = (int)newAllocationSize;
-      }
-      else {
-         validityAllocationSizeInBytes = (int)newAllocationSize;
-      }
-
-      return buffer;
-   }
-
-   @Override
-   @Deprecated
-   public List<BufferBacked> getFieldInnerVectors() {
-      throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
-   }
-
-   /**
-    * Initialize the children in schema for this Field. This operation is a
-    * NO-OP for scalar types since they don't have any children.
-    * @param children the schema
-    * @throws IllegalArgumentException if children is a non-empty list for scalar types.
-    */
-   @Override
-   public void initializeChildrenFromFields(List<Field> children) {
-      if (!children.isEmpty()) {
-         throw new IllegalArgumentException("primitive type vector can not have children");
-      }
-   }
-
-   /**
-    * Get the inner child vectors.
-    * @return list of child vectors for complex types, empty list for scalar vector
-    * types
-    */
-   @Override
-   public List<FieldVector> getChildrenFromFields() {
-      return Collections.emptyList();
-   }
-
-   /**
-    * Load the buffers of this vector with provided source buffers.
-    * The caller manages the source buffers and populates them before invoking
-    * this method.
-    * @param fieldNode  the fieldNode indicating the value count
-    * @param ownBuffers the buffers for this Field (own buffers only, children not included)
-    */
-   @Override
-   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-      if (ownBuffers.size() != 2) {
-         throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size());
-      }
-
-      ArrowBuf bitBuffer = ownBuffers.get(0);
-      ArrowBuf dataBuffer = ownBuffers.get(1);
-
-      validityBuffer.release();
-      validityBuffer = bitBuffer.retain(allocator);
-      valueBuffer.release();
-      valueBuffer = dataBuffer.retain(allocator);
-
-      valueCount = fieldNode.getLength();
-
-      valueAllocationSizeInBytes = valueBuffer.capacity();
-      validityAllocationSizeInBytes = validityBuffer.capacity();
-   }
-
-   /**
-    * Get the buffers belonging to this vector
-    * @return the inner buffers.
-    */
-   public List<ArrowBuf> getFieldBuffers() {
-      List<ArrowBuf> result = new ArrayList<>(2);
-
-      validityBuffer.readerIndex(0);
+      clear();
+    }
+    return buffers;
+  }
+
+  /**
+   * Resize the vector to increase the capacity. The internal behavior is to
+   * double the current value capacity.
+   */
+  @Override
+  public void reAlloc() {
+    valueBuffer = reallocBufferHelper(valueBuffer, true);
+    validityBuffer = reallocBufferHelper(validityBuffer, false);
+  }
+
+  /**
+   * Helper method for reallocating a particular internal buffer
+   * Returns the new buffer.
+   */
+  private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) {
+    final int currentBufferCapacity = buffer.capacity();
+    long baseSize = (dataBuffer ? valueAllocationSizeInBytes
+            : validityAllocationSizeInBytes);
+
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
+    newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
+    final int halfNewCapacity = newBuf.capacity() / 2;
+    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    buffer.release(1);
+    buffer = newBuf;
+    if (dataBuffer) {
+      valueAllocationSizeInBytes = (int) newAllocationSize;
+    } else {
+      validityAllocationSizeInBytes = (int) newAllocationSize;
+    }
+
+    return buffer;
+  }
+
+  @Override
+  @Deprecated
+  public List<BufferBacked> getFieldInnerVectors() {
+    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
+  }
+
+  /**
+   * Initialize the children in schema for this Field. This operation is a
+   * NO-OP for scalar types since they don't have any children.
+   * @param children the schema
+   * @throws IllegalArgumentException if children is a non-empty list for scalar types.
+   */
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    if (!children.isEmpty()) {
+      throw new IllegalArgumentException("primitive type vector can not have children");
+    }
+  }
+
+  /**
+   * Get the inner child vectors.
+   * @return list of child vectors for complex types, empty list for scalar vector
+   * types
+   */
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return Collections.emptyList();
+  }
+
+  /**
+   * Load the buffers of this vector with provided source buffers.
+   * The caller manages the source buffers and populates them before invoking
+   * this method.
+   * @param fieldNode  the fieldNode indicating the value count
+   * @param ownBuffers the buffers for this Field (own buffers only, children not included)
+   */
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    if (ownBuffers.size() != 2) {
+      throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size());
+    }
+
+    ArrowBuf bitBuffer = ownBuffers.get(0);
+    ArrowBuf dataBuffer = ownBuffers.get(1);
+
+    validityBuffer.release();
+    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
+    valueBuffer.release();
+    valueBuffer = dataBuffer.retain(allocator);
+
+    valueCount = fieldNode.getLength();
+
+    valueAllocationSizeInBytes = valueBuffer.capacity();
+    validityAllocationSizeInBytes = validityBuffer.capacity();
+  }
+
+  /**
+   * Get the buffers belonging to this vector
+   * @return the inner buffers.
+   */
+  public List<ArrowBuf> getFieldBuffers() {
+    List<ArrowBuf> result = new ArrayList<>(2);
+    setReaderAndWriterIndex();
+    result.add(validityBuffer);
+    result.add(valueBuffer);
+
+    return result;
+  }
+
+  /**
+   * Set the reader and writer indexes for the inner buffers.
+   */
+  private void setReaderAndWriterIndex() {
+    validityBuffer.readerIndex(0);
+    valueBuffer.readerIndex(0);
+    if (valueCount == 0) {
+      validityBuffer.writerIndex(0);
+      valueBuffer.writerIndex(0);
+    } else {
       validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
-      valueBuffer.readerIndex(0);
       if (typeWidth == 0) {
-         /* specialized handling for NullableBitVector */
-         valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
+        /* specialized handling for NullableBitVector */
+        valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
       } else {
-         valueBuffer.writerIndex(valueCount * typeWidth);
+        valueBuffer.writerIndex(valueCount * typeWidth);
       }
-
-      result.add(validityBuffer);
-      result.add(valueBuffer);
-
-      return result;
-   }
-
-   /**
-    * Construct a transfer pair of this vector and another vector of same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @param callBack
-    * @return TransferPair
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-      return getTransferPair(ref, allocator);
-   }
-
-   /**
-    * Construct a transfer pair of this vector and another vector of same type.
-    * @param allocator allocator for the target vector
-    * @return TransferPair
-    */
-   @Override
-   public TransferPair getTransferPair(BufferAllocator allocator){
-      return getTransferPair(name, allocator);
-   }
-
-   /**
-    * Construct a transfer pair of this vector and another vector of same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return TransferPair
-    */
-   public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);
-
-   /**
-    * Transfer this vector'data to another vector. The memory associated
-    * with this vector is transferred to the allocator of target vector
-    * for accounting and management purposes.
-    * @param target destination vector for transfer
-    */
-   public void transferTo(BaseNullableFixedWidthVector target){
-      compareTypes(target, "transferTo");
-      target.clear();
-      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
-      target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer;
-      target.valueCount = valueCount;
-      clear();
-   }
-
-   /**
-    * Slice this vector at desired index and length and transfer the
-    * corresponding data to the target vector.
-    * @param startIndex start position of the split in source vector.
-    * @param length length of the split.
-    * @param target destination vector
-    */
-   public void splitAndTransferTo(int startIndex, int length,
-                                  BaseNullableFixedWidthVector target) {
-      compareTypes(target, "splitAndTransferTo");
-      target.clear();
-      splitAndTransferValidityBuffer(startIndex, length, target);
-      splitAndTransferValueBuffer(startIndex, length, target);
-      target.setValueCount(length);
-   }
-
-   /**
-    * Data buffer can always be split and transferred using slicing.
-    */
-   private void splitAndTransferValueBuffer(int startIndex, int length,
-                                            BaseNullableFixedWidthVector target) {
-      final int startPoint = startIndex * typeWidth;
-      final int sliceLength = length * typeWidth;
-      target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
-   }
-
-   /**
-    * Validity buffer has multiple cases of split and transfer depending on
-    * the starting position of the source index.
-    */
-   private void splitAndTransferValidityBuffer(int startIndex, int length,
-                                               BaseNullableFixedWidthVector target) {
-      assert startIndex + length <= valueCount;
-      int firstByteSource = BitVectorHelper.byteIndex(startIndex);
-      int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
-      int byteSizeTarget = getValidityBufferSizeFromCount(length);
-      int offset = startIndex % 8;
-
-      if (length > 0) {
-         if (offset == 0) {
-            /* slice */
-            if (target.validityBuffer != null) {
-               target.validityBuffer.release();
-            }
-            target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
-            target.validityBuffer.retain(1);
-         }
-         else {
-            /* Copy data
-             * When the first bit starts from the middle of a byte (offset != 0),
-             * copy data from src BitVector.
-             * Each byte in the target is composed by a part in i-th byte,
-             * another part in (i+1)-th byte.
-             */
-            target.allocateValidityBuffer(byteSizeTarget);
-
-            for (int i = 0; i < byteSizeTarget - 1; i++) {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
-                       firstByteSource + i, offset);
-               byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
-                       firstByteSource + i + 1, offset);
-
-               target.validityBuffer.setByte(i, (b1 + b2));
-            }
-
-            /* Copying the last piece is done in the following manner:
-             * if the source vector has 1 or more bytes remaining, we copy
-             * the last piece as a byte formed by shifting data
-             * from the current byte and the next byte.
-             *
-             * if the source vector has no more bytes remaining
-             * (we are at the last byte), we copy the last piece as a byte
-             * by shifting data from the current byte.
-             */
-            if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
-                       firstByteSource + byteSizeTarget - 1, offset);
-               byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
-                       firstByteSource + byteSizeTarget, offset);
-
-               target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
-            }
-            else {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
-                       firstByteSource + byteSizeTarget - 1, offset);
-               target.validityBuffer.setByte(byteSizeTarget - 1, b1);
-            }
-         }
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          common getters and setters                            *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the number of elements that are null in the vector
-    *
-    * @return the number of null elements.
-    */
-   public int getNullCount() {
-      return BitVectorHelper.getNullCount(validityBuffer, valueCount);
-   }
-
-   /**
-    * Get the value count of vector. This will always be zero unless
-    * {@link #setValueCount(int)} has been called prior to calling this.
-    *
-    * @return valueCount for the vector
-    */
-   public int getValueCount(){
-      return valueCount;
-   }
-
-   /**
-    * Set value count for the vector.
-    *
-    * @param valueCount  value count to set
-    */
-   public void setValueCount(int valueCount) {
-      this.valueCount = valueCount;
-      final int currentValueCapacity = getValueCapacity();
-      while (valueCount > getValueCapacity()) {
-         reAlloc();
-      }
-      /*
-       * We are trying to understand the pattern of memory allocation.
-       * If initially, the user did vector.allocateNew(), we would have
-       * allocated memory of default size (4096 * type width).
-       * Later on user invokes setValueCount(count).
-       *
-       * If the existing value capacity is twice as large as the
-       * valueCount, we know that we over-provisioned memory in the
-       * first place when default memory allocation was done because user
-       * really needs a much less value count in the vector.
-       *
-       * We record this by bumping up the allocationMonitor. If this pattern
-       * happens for certain number of times and allocationMonitor
-       * reaches the threshold (internal hardcoded) value, subsequent
-       * call to allocateNew() will take care of stepping down the
-       * default memory allocation size.
-       *
-       * Another case would be under-provisioning the initial memory and
-       * thus going through a lot of realloc(). Here the goal is to
-       * see if we can minimize the number of reallocations. Again the
-       * state is recorded in allocationMonitor by decrementing it
-       * (negative value). If a threshold is hit, realloc will try to
-       * allocate more memory in order to possibly avoid a future realloc.
-       * This case is also applicable to setSafe() methods which can trigger
-       * a realloc() and thus we record the state there as well.
-       */
-      if (valueCount > 0) {
-         if (currentValueCapacity >= (valueCount * 2)) {
-            incrementAllocationMonitor();
-         } else if (currentValueCapacity <= (valueCount/2)) {
-            decrementAllocationMonitor();
-         }
+    }
+  }
+
+  /**
+   * Construct a transfer pair of this vector and another vector of same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @param callBack
+   * @return TransferPair
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return getTransferPair(ref, allocator);
+  }
+
+  /**
+   * Construct a transfer pair of this vector and another vector of same type.
+   * @param allocator allocator for the target vector
+   * @return TransferPair
+   */
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return getTransferPair(name, allocator);
+  }
+
+  /**
+   * Construct a transfer pair of this vector and another vector of same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return TransferPair
+   */
+  public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);
+
+  /**
+   * Transfer this vector'data to another vector. The memory associated
+   * with this vector is transferred to the allocator of target vector
+   * for accounting and management purposes.
+   * @param target destination vector for transfer
+   */
+  public void transferTo(BaseNullableFixedWidthVector target) {
+    compareTypes(target, "transferTo");
+    target.clear();
+    target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
+    target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer;
+    target.valueCount = valueCount;
+    clear();
+  }
+
+  /**
+   * Slice this vector at desired index and length and transfer the
+   * corresponding data to the target vector.
+   * @param startIndex start position of the split in source vector.
+   * @param length length of the split.
+   * @param target destination vector
+   */
+  public void splitAndTransferTo(int startIndex, int length,
+                                 BaseNullableFixedWidthVector target) {
+    compareTypes(target, "splitAndTransferTo");
+    target.clear();
+    splitAndTransferValidityBuffer(startIndex, length, target);
+    splitAndTransferValueBuffer(startIndex, length, target);
+    target.setValueCount(length);
+  }
+
+  /**
+   * Data buffer can always be split and transferred using slicing.
+   */
+  private void splitAndTransferValueBuffer(int startIndex, int length,
+                                           BaseNullableFixedWidthVector target) {
+    final int startPoint = startIndex * typeWidth;
+    final int sliceLength = length * typeWidth;
+    target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
+  }
+
+  /**
+   * Validity buffer has multiple cases of split and transfer depending on
+   * the starting position of the source index.
+   */
+  private void splitAndTransferValidityBuffer(int startIndex, int length,
+                                              BaseNullableFixedWidthVector target) {
+    assert startIndex + length <= valueCount;
+    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+    int byteSizeTarget = getValidityBufferSizeFromCount(length);
+    int offset = startIndex % 8;
+
+    if (length > 0) {
+      if (offset == 0) {
+        /* slice */
+        if (target.validityBuffer != null) {
+          target.validityBuffer.release();
+        }
+        target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
+        target.validityBuffer.retain(1);
+      } else {
+        /* Copy data
+         * When the first bit starts from the middle of a byte (offset != 0),
+         * copy data from src BitVector.
+         * Each byte in the target is composed by a part in i-th byte,
+         * another part in (i+1)-th byte.
+         */
+        target.allocateValidityBuffer(byteSizeTarget);
+
+        for (int i = 0; i < byteSizeTarget - 1; i++) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
+                  firstByteSource + i, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
+                  firstByteSource + i + 1, offset);
+
+          target.validityBuffer.setByte(i, (b1 + b2));
+        }
+
+        /* Copying the last piece is done in the following manner:
+         * if the source vector has 1 or more bytes remaining, we copy
+         * the last piece as a byte formed by shifting data
+         * from the current byte and the next byte.
+         *
+         * if the source vector has no more bytes remaining
+         * (we are at the last byte), we copy the last piece as a byte
+         * by shifting data from the current byte.
+         */
+        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
+                  firstByteSource + byteSizeTarget, offset);
+
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+        } else {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+        }
       }
-   }
-
-   /**
-    * Check if the given index is within the current value capacity
-    * of the vector
-    *
-    * @param index  position to check
-    * @return true if index is within the current value capacity
-    */
-   public boolean isSafe(int index) {
-      return index < getValueCapacity();
-   }
-
-   /**
-    * Check if element at given index is null.
-    *
-    * @param index  position of element
-    * @return true if element at given index is null, false otherwise
-    */
-   public boolean isNull(int index) {
-      return (isSet(index) == 0);
-   }
-
-   /**
-    * Same as {@link #isNull(int)}.
-    *
-    * @param index  position of element
-    * @return 1 if element at given index is not null, 0 otherwise
-    */
-   public int isSet(int index) {
-      final int byteIndex = index >> 3;
-      final byte b = validityBuffer.getByte(byteIndex);
-      final int bitIndex = index & 7;
-      return Long.bitCount(b & (1L << bitIndex));
-   }
-
-   /**
-    * Mark the particular position in the vector as non-null.
-    *
-    * @param index position of the element.
-    */
-   public void setIndexDefined(int index) {
-      handleSafe(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-   }
-
-   public void set(int index, byte[] value, int start, int length) {
-      throw new UnsupportedOperationException();
-   }
-
-   public void setSafe(int index, byte[] value, int start, int length) {
-      throw new UnsupportedOperationException();
-   }
-
-   public void set(int index, ByteBuffer value, int start, int length) {
-      throw new UnsupportedOperationException();
-   }
-
-   public void setSafe(int index, ByteBuffer value, int start, int length) {
-      throw new UnsupportedOperationException();
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                helper methods for setters                      *
-    *                                                                *
-    ******************************************************************/
-
-
-   protected void handleSafe(int index) {
-      while (index >= getValueCapacity()) {
-         decrementAllocationMonitor();
-         reAlloc();
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          common getters and setters                            *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the number of elements that are null in the vector
+   *
+   * @return the number of null elements.
+   */
+  public int getNullCount() {
+    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+  }
+
+  /**
+   * Get the value count of vector. This will always be zero unless
+   * {@link #setValueCount(int)} has been called prior to calling this.
+   *
+   * @return valueCount for the vector
+   */
+  public int getValueCount() {
+    return valueCount;
+  }
+
+  /**
+   * Set value count for the vector.
+   *
+   * @param valueCount  value count to set
+   */
+  public void setValueCount(int valueCount) {
+    this.valueCount = valueCount;
+    final int currentValueCapacity = getValueCapacity();
+    while (valueCount > getValueCapacity()) {
+      reAlloc();
+    }
+    /*
+     * We are trying to understand the pattern of memory allocation.
+     * If initially, the user did vector.allocateNew(), we would have
+     * allocated memory of default size (4096 * type width).
+     * Later on user invokes setValueCount(count).
+     *
+     * If the existing value capacity is twice as large as the
+     * valueCount, we know that we over-provisioned memory in the
+     * first place when default memory allocation was done because user
+     * really needs a much less value count in the vector.
+     *
+     * We record this by bumping up the allocationMonitor. If this pattern
+     * happens for certain number of times and allocationMonitor
+     * reaches the threshold (internal hardcoded) value, subsequent
+     * call to allocateNew() will take care of stepping down the
+     * default memory allocation size.
+     *
+     * Another case would be under-provisioning the initial memory and
+     * thus going through a lot of realloc(). Here the goal is to
+     * see if we can minimize the number of reallocations. Again the
+     * state is recorded in allocationMonitor by decrementing it
+     * (negative value). If a threshold is hit, realloc will try to
+     * allocate more memory in order to possibly avoid a future realloc.
+     * This case is also applicable to setSafe() methods which can trigger
+     * a realloc() and thus we record the state there as well.
+     */
+    if (valueCount > 0) {
+      if (currentValueCapacity >= (valueCount * 2)) {
+        incrementAllocationMonitor();
+      } else if (currentValueCapacity <= (valueCount / 2)) {
+        decrementAllocationMonitor();
       }
-   }
+    }
+    setReaderAndWriterIndex();
+  }
+
+  /**
+   * Check if the given index is within the current value capacity
+   * of the vector
+   *
+   * @param index  position to check
+   * @return true if index is within the current value capacity
+   */
+  public boolean isSafe(int index) {
+    return index < getValueCapacity();
+  }
+
+  /**
+   * Check if element at given index is null.
+   *
+   * @param index  position of element
+   * @return true if element at given index is null, false otherwise
+   */
+  public boolean isNull(int index) {
+    return (isSet(index) == 0);
+  }
+
+  /**
+   * Same as {@link #isNull(int)}.
+   *
+   * @param index  position of element
+   * @return 1 if element at given index is not null, 0 otherwise
+   */
+  public int isSet(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = validityBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
+  }
+
+  /**
+   * Mark the particular position in the vector as non-null.
+   *
+   * @param index position of the element.
+   */
+  @Override
+  public void setIndexDefined(int index) {
+    handleSafe(index);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+  }
+
+  public void set(int index, byte[] value, int start, int length) {
+    throw new UnsupportedOperationException();
+  }
+
+  public void setSafe(int index, byte[] value, int start, int length) {
+    throw new UnsupportedOperationException();
+  }
+
+  public void set(int index, ByteBuffer value, int start, int length) {
+    throw new UnsupportedOperationException();
+  }
+
+  public void setSafe(int index, ByteBuffer value, int start, int length) {
+    throw new UnsupportedOperationException();
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                helper methods for setters                      *
+   *                                                                *
+   ******************************************************************/
+
+
+  protected void handleSafe(int index) {
+    while (index >= getValueCapacity()) {
+      decrementAllocationMonitor();
+      reAlloc();
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
index 47daeee7f202c..edf4987de5756 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -24,6 +24,7 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -31,928 +32,1219 @@
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
 public abstract class BaseNullableVariableWidthVector extends BaseValueVector
-        implements VariableWidthVector, FieldVector {
-   private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
-   private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT;
-
-   private int valueAllocationSizeInBytes;
-   private int validityAllocationSizeInBytes;
-   private int offsetAllocationSizeInBytes;
-
-   /* protected members */
-   public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */
-   protected static final byte[] emptyByteArray = new byte[]{};
-   protected ArrowBuf validityBuffer;
-   protected ArrowBuf valueBuffer;
-   protected ArrowBuf offsetBuffer;
-   protected int valueCount;
-   protected int lastSet;
-   protected final Field field;
-   private boolean cleared;
-
-   public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator,
-                                          FieldType fieldType) {
-      super(name, allocator);
-      valueAllocationSizeInBytes = INITIAL_BYTE_COUNT;
-      validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
-      offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH;
-      field = new Field(name, fieldType, null);
-      valueCount = 0;
-      lastSet = -1;
-      offsetBuffer = allocator.getEmpty();
-      validityBuffer = allocator.getEmpty();
-      valueBuffer = allocator.getEmpty();
-      cleared = false;
-   }
-
-   /* TODO:
-    *
-    * see if getNullCount() can be made faster -- O(1)
-    */
+        implements VariableWidthVector, FieldVector, NullableVectorDefinitionSetter {
+  private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
+  private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT;
+
+  private int valueAllocationSizeInBytes;
+  private int validityAllocationSizeInBytes;
+  private int offsetAllocationSizeInBytes;
+
+  /* protected members */
+  public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */
+  protected static final byte[] emptyByteArray = new byte[]{};
+  protected ArrowBuf validityBuffer;
+  protected ArrowBuf valueBuffer;
+  protected ArrowBuf offsetBuffer;
+  protected int valueCount;
+  protected int lastSet;
+  protected final Field field;
+  private boolean cleared;
+
+  public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator,
+                                         FieldType fieldType) {
+    super(name, allocator);
+    valueAllocationSizeInBytes = INITIAL_BYTE_COUNT;
+    validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
+    offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH;
+    field = new Field(name, fieldType, null);
+    valueCount = 0;
+    lastSet = -1;
+    offsetBuffer = allocator.getEmpty();
+    validityBuffer = allocator.getEmpty();
+    valueBuffer = allocator.getEmpty();
+    cleared = false;
+  }
 
   /* TODO:
-    * Once the entire hierarchy has been refactored, move common functions
-    * like getNullCount(), splitAndTransferValidityBuffer to top level
-    * base class BaseValueVector.
-    *
-    * Along with this, some class members (validityBuffer) can also be
-    * abstracted out to top level base class.
-    *
-    * Right now BaseValueVector is the top level base class for other
-    * vector types in ValueVector hierarchy (non-nullable) and those
-    * vectors have not yet been refactored/removed so moving things to
-    * the top class as of now is not a good idea.
-    */
-
-
-   @Override
-   @Deprecated
-   public VariableWidthMutator getMutator() {
-      throw new  UnsupportedOperationException("Mutator is not supported for writing into vector");
-   }
-
-   @Override
-   @Deprecated
-   public VariableWidthAccessor getAccessor() {
-      throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
-   }
-
-   /**
-    * Get buffer that manages the validity (NULL or NON-NULL nature) of
-    * elements in the vector. Consider it as a buffer for internal bit vector
-    * data structure.
-    * @return buffer
-    */
-   @Override
-   public ArrowBuf getValidityBuffer() {
-      return validityBuffer;
-   }
-
-   /**
-    * Get the buffer that stores the data for elements in the vector.
-    * @return buffer
-    */
-   @Override
-   public ArrowBuf getDataBuffer() {
-      return valueBuffer;
-   }
-
-   /**
-    * buffer that stores the offsets for elements
-    * in the vector. This operation is not supported for fixed-width vectors.
-    * @return buffer
-    */
-   @Override
-   public ArrowBuf getOffsetBuffer() {
-      return offsetBuffer;
-   }
-
-   /**
-    * Get the memory address of buffer that stores the offsets for elements
-    * in the vector.
-    * @return starting address of the buffer
-    */
-   @Override
-   public long getOffsetBufferAddress() {
-      return offsetBuffer.memoryAddress();
-   }
-
-   /**
-    * Get the memory address of buffer that manages the validity
-    * (NULL or NON-NULL nature) of elements in the vector.
-    * @return starting address of the buffer
-    */
-   @Override
-   public long getValidityBufferAddress() {
-      return validityBuffer.memoryAddress();
-   }
-
-   /**
-    * Get the memory address of buffer that stores the data for elements
-    * in the vector.
-    * @return starting address of the buffer
-    */
-   @Override
-   public long getDataBufferAddress() {
-      return valueBuffer.memoryAddress();
-   }
-
-   /**
-    * Sets the desired value capacity for the vector. This function doesn't
-    * allocate any memory for the vector.
-    * @param valueCount desired number of elements in the vector
-    */
-   @Override
-   public void setInitialCapacity(int valueCount) {
-      final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT;
-      if (size > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
-      }
-      valueAllocationSizeInBytes = (int)size;
-      validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
-      /* to track the end offset of last data element in vector, we need
-       * an additional slot in offset buffer.
-       */
-      offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH;
-   }
-
-   /**
-    * Get the current value capacity for the vector
-    * @return number of elements that vector can hold.
-    */
-   @Override
-   public int getValueCapacity(){
-      final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0);
-      return Math.min(offsetValueCapacity, getValidityBufferValueCapacity());
-   }
-
-   private int getValidityBufferValueCapacity() {
-      return (int)(validityBuffer.capacity() * 8L);
-   }
-
-   private int getOffsetBufferValueCapacity() {
-      return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH);
-   }
-
-   /**
-    * zero out the vector and the data in associated buffers.
-    */
-   public void zeroVector() {
-      initValidityBuffer();
-      initOffsetBuffer();
-   }
-
-   /* zero out the validity buffer */
-   private void initValidityBuffer() {
-      validityBuffer.setZero(0, validityBuffer.capacity());
-   }
-
-   /* zero out the offset buffer */
-   private void initOffsetBuffer() {
-      offsetBuffer.setZero(0, offsetBuffer.capacity());
-   }
-
-   /**
-    * Reset the vector to initial state. Same as {@link #zeroVector()}.
-    * Note that this method doesn't release any memory.
-    */
-   public void reset() {
-      zeroVector();
-      lastSet = -1;
-   }
-
-   /**
-    * Close the vector and release the associated buffers.
-    */
-   @Override
-   public void close() {
-      clear();
-   }
-
-   /**
-    * Same as {@link #close()}
-    */
-   @Override
-   public void clear() {
-      validityBuffer = releaseBuffer(validityBuffer);
-      valueBuffer = releaseBuffer(valueBuffer);
-      offsetBuffer = releaseBuffer(offsetBuffer);
-      cleared = true;
-      lastSet = -1;
-      valueCount = 0;
-   }
-
-   @Override
-   @Deprecated
-   public List<BufferBacked> getFieldInnerVectors() {
-      throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
-   }
-
-   /**
-    * Initialize the children in schema for this Field. This operation is a
-    * NO-OP for scalar types since they don't have any children.
-    * @param children the schema
-    * @throws IllegalArgumentException if children is a non-empty list for scalar types.
-    */
-   @Override
-   public void initializeChildrenFromFields(List<Field> children) {
-      if (!children.isEmpty()) {
-         throw new IllegalArgumentException("primitive type vector can not have children");
-      }
-   }
-
-   /**
-    * Get the inner child vectors.
-    * @return list of child vectors for complex types, empty list for scalar vector
-    * types
-    */
-   @Override
-   public List<FieldVector> getChildrenFromFields() {
-      return Collections.emptyList();
-   }
-
-
-   /**
-    * Load the buffers of this vector with provided source buffers.
-    * The caller manages the source buffers and populates them before invoking
-    * this method.
-    * @param fieldNode  the fieldNode indicating the value count
-    * @param ownBuffers the buffers for this Field (own buffers only, children not included)
-    */
-   @Override
-   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-      ArrowBuf bitBuffer = ownBuffers.get(0);
-      ArrowBuf offBuffer = ownBuffers.get(1);
-      ArrowBuf dataBuffer = ownBuffers.get(2);
-
-      validityBuffer.release();
-      validityBuffer = bitBuffer.retain(allocator);
-      offsetBuffer.release();
-      offsetBuffer = offBuffer.retain(allocator);
-      valueBuffer.release();
-      valueBuffer = dataBuffer.retain(allocator);
-
-      lastSet = fieldNode.getLength() - 1;
-      valueCount = fieldNode.getLength();
-   }
-
-   /**
-    * Get the buffers belonging to this vector
-    * @return the inner buffers.
-    */
-   public List<ArrowBuf> getFieldBuffers() {
-      List<ArrowBuf> result = new ArrayList<>(3);
+   * see if getNullCount() can be made faster -- O(1)
+   */
+
+  /* TODO:
+   * Once the entire hierarchy has been refactored, move common functions
+   * like getNullCount(), splitAndTransferValidityBuffer to top level
+   * base class BaseValueVector.
+   *
+   * Along with this, some class members (validityBuffer) can also be
+   * abstracted out to top level base class.
+   *
+   * Right now BaseValueVector is the top level base class for other
+   * vector types in ValueVector hierarchy (non-nullable) and those
+   * vectors have not yet been refactored/removed so moving things to
+   * the top class as of now is not a good idea.
+   */
+
+  @Override
+  @Deprecated
+  public VariableWidthMutator getMutator() {
+    throw new UnsupportedOperationException("Mutator is not supported for writing into vector");
+  }
+
+  @Override
+  @Deprecated
+  public VariableWidthAccessor getAccessor() {
+    throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
+  }
+
+  /**
+   * Get buffer that manages the validity (NULL or NON-NULL nature) of
+   * elements in the vector. Consider it as a buffer for internal bit vector
+   * data structure.
+   * @return buffer
+   */
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    return validityBuffer;
+  }
+
+  /**
+   * Get the buffer that stores the data for elements in the vector.
+   * @return buffer
+   */
+  @Override
+  public ArrowBuf getDataBuffer() {
+    return valueBuffer;
+  }
+
+  /**
+   * buffer that stores the offsets for elements
+   * in the vector. This operation is not supported for fixed-width vectors.
+   * @return buffer
+   */
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    return offsetBuffer;
+  }
+
+  /**
+   * Get the memory address of buffer that stores the offsets for elements
+   * in the vector.
+   * @return starting address of the buffer
+   */
+  @Override
+  public long getOffsetBufferAddress() {
+    return offsetBuffer.memoryAddress();
+  }
+
+  /**
+   * Get the memory address of buffer that manages the validity
+   * (NULL or NON-NULL nature) of elements in the vector.
+   * @return starting address of the buffer
+   */
+  @Override
+  public long getValidityBufferAddress() {
+    return validityBuffer.memoryAddress();
+  }
+
+  /**
+   * Get the memory address of buffer that stores the data for elements
+   * in the vector.
+   * @return starting address of the buffer
+   */
+  @Override
+  public long getDataBufferAddress() {
+    return valueBuffer.memoryAddress();
+  }
+
+  /**
+   * Sets the desired value capacity for the vector. This function doesn't
+   * allocate any memory for the vector.
+   * @param valueCount desired number of elements in the vector
+   */
+  @Override
+  public void setInitialCapacity(int valueCount) {
+    final long size = (long) valueCount * DEFAULT_RECORD_BYTE_COUNT;
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+    }
+    valueAllocationSizeInBytes = (int) size;
+    validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
+    /* to track the end offset of last data element in vector, we need
+     * an additional slot in offset buffer.
+     */
+    offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH;
+  }
+
+  /**
+   * Get the current value capacity for the vector
+   * @return number of elements that vector can hold.
+   */
+  @Override
+  public int getValueCapacity() {
+    final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0);
+    return Math.min(offsetValueCapacity, getValidityBufferValueCapacity());
+  }
+
+  private int getValidityBufferValueCapacity() {
+    return (int) (validityBuffer.capacity() * 8L);
+  }
+
+  private int getOffsetBufferValueCapacity() {
+    return (int) ((offsetBuffer.capacity() * 1.0) / OFFSET_WIDTH);
+  }
+
+  /**
+   * zero out the vector and the data in associated buffers.
+   */
+  public void zeroVector() {
+    initValidityBuffer();
+    initOffsetBuffer();
+  }
+
+  /* zero out the validity buffer */
+  private void initValidityBuffer() {
+    validityBuffer.setZero(0, validityBuffer.capacity());
+  }
+
+  /* zero out the offset buffer */
+  private void initOffsetBuffer() {
+    offsetBuffer.setZero(0, offsetBuffer.capacity());
+  }
+
+  /**
+   * Reset the vector to initial state. Same as {@link #zeroVector()}.
+   * Note that this method doesn't release any memory.
+   */
+  public void reset() {
+    zeroVector();
+    lastSet = -1;
+  }
+
+  /**
+   * Close the vector and release the associated buffers.
+   */
+  @Override
+  public void close() {
+    clear();
+  }
+
+  /**
+   * Same as {@link #close()}
+   */
+  @Override
+  public void clear() {
+    validityBuffer = releaseBuffer(validityBuffer);
+    valueBuffer = releaseBuffer(valueBuffer);
+    offsetBuffer = releaseBuffer(offsetBuffer);
+    cleared = true;
+    lastSet = -1;
+    valueCount = 0;
+  }
+
+  @Override
+  @Deprecated
+  public List<BufferBacked> getFieldInnerVectors() {
+    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
+  }
+
+  /**
+   * Initialize the children in schema for this Field. This operation is a
+   * NO-OP for scalar types since they don't have any children.
+   * @param children the schema
+   * @throws IllegalArgumentException if children is a non-empty list for scalar types.
+   */
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    if (!children.isEmpty()) {
+      throw new IllegalArgumentException("primitive type vector can not have children");
+    }
+  }
+
+  /**
+   * Get the inner child vectors.
+   * @return list of child vectors for complex types, empty list for scalar vector
+   * types
+   */
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return Collections.emptyList();
+  }
+
+
+  /**
+   * Load the buffers of this vector with provided source buffers.
+   * The caller manages the source buffers and populates them before invoking
+   * this method.
+   * @param fieldNode  the fieldNode indicating the value count
+   * @param ownBuffers the buffers for this Field (own buffers only, children not included)
+   */
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    ArrowBuf bitBuffer = ownBuffers.get(0);
+    ArrowBuf offBuffer = ownBuffers.get(1);
+    ArrowBuf dataBuffer = ownBuffers.get(2);
+
+    validityBuffer.release();
+    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
+    offsetBuffer.release();
+    offsetBuffer = offBuffer.retain(allocator);
+    valueBuffer.release();
+    valueBuffer = dataBuffer.retain(allocator);
+
+    lastSet = fieldNode.getLength() - 1;
+    valueCount = fieldNode.getLength();
+  }
+
+  /**
+   * Get the buffers belonging to this vector
+   * @return the inner buffers.
+   */
+  public List<ArrowBuf> getFieldBuffers() {
+    List<ArrowBuf> result = new ArrayList<>(3);
+    setReaderAndWriterIndex();
+    result.add(validityBuffer);
+    result.add(offsetBuffer);
+    result.add(valueBuffer);
+
+    return result;
+  }
+
+  /**
+   * Set the reader and writer indexes for the inner buffers.
+   */
+  private void setReaderAndWriterIndex() {
+    validityBuffer.readerIndex(0);
+    offsetBuffer.readerIndex(0);
+    valueBuffer.readerIndex(0);
+    if (valueCount == 0) {
+      validityBuffer.writerIndex(0);
+      offsetBuffer.writerIndex(0);
+      valueBuffer.writerIndex(0);
+    } else {
       final int lastDataOffset = getstartOffset(valueCount);
-      validityBuffer.readerIndex(0);
       validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
-      offsetBuffer.readerIndex(0);
       offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
-      valueBuffer.readerIndex(0);
       valueBuffer.writerIndex(lastDataOffset);
-
-      result.add(validityBuffer);
-      result.add(offsetBuffer);
-      result.add(valueBuffer);
-
-      return result;
-   }
-
-   /**
-    * Same as {@link #allocateNewSafe()}.
-    */
-   @Override
-   public void allocateNew() {
-      if(!allocateNewSafe()){
-         throw new OutOfMemoryException("Failure while allocating memory.");
-      }
-   }
-
-   /**
-    * Allocate memory for the vector. We internally use a default value count
-    * of 4096 to allocate memory for at least these many elements in the
-    * vector. See {@link #allocateNew(int, int)} for allocating memory for specific
-    * number of elements in the vector.
-    *
-    * @return false if memory allocation fails, true otherwise.
-    */
-   @Override
-   public boolean allocateNewSafe() {
-      long curAllocationSizeValue = valueAllocationSizeInBytes;
-      long curAllocationSizeValidity = validityAllocationSizeInBytes;
-      long curAllocationSizeOffset = offsetAllocationSizeInBytes;
-
-      if (curAllocationSizeValue > MAX_ALLOCATION_SIZE ||
-              curAllocationSizeOffset > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Requested amount of memory exceeds limit");
-      }
-
-      /* we are doing a new allocation -- release the current buffers */
+    }
+  }
+
+  /**
+   * Same as {@link #allocateNewSafe()}.
+   */
+  @Override
+  public void allocateNew() {
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryException("Failure while allocating memory.");
+    }
+  }
+
+  /**
+   * Allocate memory for the vector. We internally use a default value count
+   * of 4096 to allocate memory for at least these many elements in the
+   * vector. See {@link #allocateNew(int, int)} for allocating memory for specific
+   * number of elements in the vector.
+   *
+   * @return false if memory allocation fails, true otherwise.
+   */
+  @Override
+  public boolean allocateNewSafe() {
+    long curAllocationSizeValue = valueAllocationSizeInBytes;
+    long curAllocationSizeValidity = validityAllocationSizeInBytes;
+    long curAllocationSizeOffset = offsetAllocationSizeInBytes;
+
+    if (curAllocationSizeValue > MAX_ALLOCATION_SIZE ||
+            curAllocationSizeOffset > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory exceeds limit");
+    }
+
+    /* we are doing a new allocation -- release the current buffers */
+    clear();
+
+    try {
+      allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset);
+    } catch (Exception e) {
+      e.printStackTrace();
       clear();
-
-      try {
-         allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset);
-      } catch (Exception e) {
-         e.printStackTrace();
-         clear();
-         return false;
-      }
-
-      return true;
-   }
-
-   /**
-    * Allocate memory for the vector to support storing at least the provided number of
-    * elements in the vector. This method must be called prior to using the ValueVector.
-    *
-    * @param totalBytes desired total memory capacity
-    * @param valueCount the desired number of elements in the vector
-    * @throws org.apache.arrow.memory.OutOfMemoryException
-    */
-   @Override
-   public void allocateNew(int totalBytes, int valueCount) {
-      assert totalBytes >= 0;
-      final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
-      final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
-
-      if (totalBytes > MAX_ALLOCATION_SIZE ||
-              offsetBufferSize > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Requested amount of memory exceeds limit");
-      }
-
-      /* we are doing a new allocation -- release the current buffers */
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * Allocate memory for the vector to support storing at least the provided number of
+   * elements in the vector. This method must be called prior to using the ValueVector.
+   *
+   * @param totalBytes desired total memory capacity
+   * @param valueCount the desired number of elements in the vector
+   * @throws org.apache.arrow.memory.OutOfMemoryException
+   */
+  @Override
+  public void allocateNew(int totalBytes, int valueCount) {
+    assert totalBytes >= 0;
+    final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
+    final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
+
+    if (totalBytes > MAX_ALLOCATION_SIZE ||
+            offsetBufferSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory exceeds limit");
+    }
+
+    /* we are doing a new allocation -- release the current buffers */
+    clear();
+
+    try {
+      allocateBytes(totalBytes, validityBufferSize, offsetBufferSize);
+    } catch (Exception e) {
+      e.printStackTrace();
       clear();
-
-      try {
-         allocateBytes(totalBytes, validityBufferSize, offsetBufferSize);
-      } catch (Exception e) {
-         e.printStackTrace();
-         clear();
-      }
-   }
-
-   /* allocate the inner buffers */
-   private void allocateBytes(final long valueBufferSize, final long validityBufferSize,
-                              final long offsetBufferSize) {
-      /* allocate data buffer */
-      int curSize = (int)valueBufferSize;
-      valueBuffer = allocator.buffer(curSize);
-      valueBuffer.readerIndex(0);
-      valueAllocationSizeInBytes = curSize;
-      allocateValidityBuffer(validityBufferSize);
-      allocateOffsetBuffer(offsetBufferSize);
-   }
-
-   /* allocate offset buffer */
-   private void allocateOffsetBuffer(final long size) {
-      final int curSize = (int)size;
-      offsetBuffer = allocator.buffer(curSize);
-      offsetBuffer.readerIndex(0);
-      offsetAllocationSizeInBytes = curSize;
-      initOffsetBuffer();
-   }
-
-   /* allocate validity buffer */
-   private void allocateValidityBuffer(final long size) {
-      final int curSize = (int)size;
-      validityBuffer = allocator.buffer(curSize);
-      validityBuffer.readerIndex(0);
-      validityAllocationSizeInBytes = curSize;
-      initValidityBuffer();
-   }
-
-   /**
-    * Resize the vector to increase the capacity. The internal behavior is to
-    * double the current value capacity.
-    */
-   public void reAlloc() {
-      reallocValueBuffer();
-      reallocValidityAndOffsetBuffers();
-   }
-
-   protected void reallocValueBuffer() {
-      long baseSize = valueAllocationSizeInBytes;
-      final int currentBufferCapacity = valueBuffer.capacity();
-
-      if (baseSize < (long)currentBufferCapacity) {
-         baseSize = (long)currentBufferCapacity;
-      }
-
-      long newAllocationSize = baseSize * 2L;
-      newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
-
-      if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
-         throw new OversizedAllocationException("Unable to expand the buffer");
-      }
-
-      final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
-      newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity);
-      valueBuffer.release();
-      valueBuffer = newBuf;
-      valueAllocationSizeInBytes = (int)newAllocationSize;
-   }
-
-   protected void reallocValidityAndOffsetBuffers() {
-      offsetBuffer = reallocBufferHelper(offsetBuffer, true);
-      validityBuffer = reallocBufferHelper(validityBuffer, false);
-   }
-
-   /* helper method to realloc a particular buffer. returns the allocated buffer */
-   private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) {
-      final int currentBufferCapacity = buffer.capacity();
-      long baseSize  = (offsetBuffer ? offsetAllocationSizeInBytes
-              : validityAllocationSizeInBytes);
-
-      if (baseSize < (long)currentBufferCapacity) {
-         baseSize = (long)currentBufferCapacity;
-      }
-
-      long newAllocationSize = baseSize * 2L;
-      newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
-
-      if (newAllocationSize > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Unable to expand the buffer");
-      }
-
-      final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
-      newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
-      final int halfNewCapacity = newBuf.capacity() / 2;
-      newBuf.setZero(halfNewCapacity, halfNewCapacity);
-      buffer.release(1);
-      buffer = newBuf;
-      if (offsetBuffer) {
-         offsetAllocationSizeInBytes = (int)newAllocationSize;
-      }
-      else {
-         validityAllocationSizeInBytes = (int)newAllocationSize;
-      }
-
-      return buffer;
-   }
-
-   /**
-    * Get the size (number of bytes) of underlying data buffer.
-    * @return
-    */
-   @Override
-   public int getByteCapacity(){
-      return valueBuffer.capacity();
-   }
-
-   @Override
-   public int getCurrentSizeInBytes(){
+    }
+  }
+
+  /* allocate the inner buffers */
+  private void allocateBytes(final long valueBufferSize, final long validityBufferSize,
+                             final long offsetBufferSize) {
+    /* allocate data buffer */
+    int curSize = (int) valueBufferSize;
+    valueBuffer = allocator.buffer(curSize);
+    valueBuffer.readerIndex(0);
+    valueAllocationSizeInBytes = curSize;
+    allocateValidityBuffer(validityBufferSize);
+    allocateOffsetBuffer(offsetBufferSize);
+  }
+
+  /* allocate offset buffer */
+  private void allocateOffsetBuffer(final long size) {
+    final int curSize = (int) size;
+    offsetBuffer = allocator.buffer(curSize);
+    offsetBuffer.readerIndex(0);
+    offsetAllocationSizeInBytes = curSize;
+    initOffsetBuffer();
+  }
+
+  /* allocate validity buffer */
+  private void allocateValidityBuffer(final long size) {
+    final int curSize = (int) size;
+    validityBuffer = allocator.buffer(curSize);
+    validityBuffer.readerIndex(0);
+    validityAllocationSizeInBytes = curSize;
+    initValidityBuffer();
+  }
+
+  /**
+   * Resize the vector to increase the capacity. The internal behavior is to
+   * double the current value capacity.
+   */
+  public void reAlloc() {
+    reallocDataBuffer();
+    reallocValidityAndOffsetBuffers();
+  }
+
+  /**
+   * Reallocate the data buffer. Data Buffer stores the actual data for
+   * VARCHAR or VARBINARY elements in the vector. The behavior is to double
+   * the size of buffer.
+   * @throws OversizedAllocationException if the desired new size is more than
+   *                                      max allowed
+   * @throws OutOfMemoryException if the internal memory allocation fails
+   */
+  public void reallocDataBuffer() {
+    long baseSize = valueAllocationSizeInBytes;
+    final int currentBufferCapacity = valueBuffer.capacity();
+
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
+    newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity);
+    valueBuffer.release();
+    valueBuffer = newBuf;
+    valueAllocationSizeInBytes = (int) newAllocationSize;
+  }
+
+  /**
+   * Reallocate the validity and offset buffers for this vector. Validity
+   * buffer is used to track the NULL or NON-NULL nature of elements in
+   * the vector and offset buffer is used to store the lengths of variable
+   * width elements in the vector.
+   *
+   * Note that data buffer for variable length vectors moves independent
+   * of the companion validity and offset buffers. This is in
+   * contrast to what we have for fixed width vectors.
+   *
+   * So even though we may have setup an initial capacity of 1024
+   * elements in the vector, it is quite possible
+   * that we need to reAlloc() the data buffer when we are setting
+   * the 5th element in the vector simply because previous
+   * variable length elements have exhausted the buffer capacity.
+   * However, we really don't need to reAlloc() validity and
+   * offset buffers until we try to set the 1025th element
+   * This is why we do a separate check for safe methods to
+   * determine which buffer needs reallocation.
+   * @throws OversizedAllocationException if the desired new size is more than
+   *                                      max allowed
+   * @throws OutOfMemoryException if the internal memory allocation fails
+   */
+  public void reallocValidityAndOffsetBuffers() {
+    offsetBuffer = reallocBufferHelper(offsetBuffer, true);
+    validityBuffer = reallocBufferHelper(validityBuffer, false);
+  }
+
+  /* helper method to realloc a particular buffer. returns the allocated buffer */
+  private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) {
+    final int currentBufferCapacity = buffer.capacity();
+    long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes
+            : validityAllocationSizeInBytes);
+
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
+    newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
+    final int halfNewCapacity = newBuf.capacity() / 2;
+    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    buffer.release(1);
+    buffer = newBuf;
+    if (offsetBuffer) {
+      offsetAllocationSizeInBytes = (int) newAllocationSize;
+    } else {
+      validityAllocationSizeInBytes = (int) newAllocationSize;
+    }
+
+    return buffer;
+  }
+
+  /**
+   * Get the size (number of bytes) of underlying data buffer.
+   * @return
+   */
+  @Override
+  public int getByteCapacity() {
+    return valueBuffer.capacity();
+  }
+
+  @Override
+  public int getCurrentSizeInBytes() {
       /* TODO */
+    return 0;
+  }
+
+  /**
+   * Get the size (number of bytes) of underlying buffers used by this
+   * vector
+   * @return size of underlying buffers.
+   */
+  @Override
+  public int getBufferSize() {
+    return getBufferSizeFor(this.valueCount);
+  }
+
+  /**
+   * Get the potential buffer size for a particular number of records.
+   * @param valueCount desired number of elements in the vector
+   * @return estimated size of underlying buffers if the vector holds
+   *         a given number of elements
+   */
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
       return 0;
-   }
-
-   /**
-    * Get the size (number of bytes) of underlying buffers used by this
-    * vector
-    * @return size of underlying buffers.
-    */
-   @Override
-   public int getBufferSize() {
-      return getBufferSizeFor(this.valueCount);
-   }
-
-   /**
-    * Get the potential buffer size for a particular number of records.
-    * @param valueCount desired number of elements in the vector
-    * @return estimated size of underlying buffers if the vector holds
-    *         a given number of elements
-    */
-   @Override
-   public int getBufferSizeFor(final int valueCount) {
-      if (valueCount == 0) {
-         return 0;
-      }
+    }
 
-      final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
-      final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
+    final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
+    final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
       /* get the end offset for this valueCount */
-      final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
-      return validityBufferSize + offsetBufferSize + dataBufferSize;
-   }
-
-   /**
-    * Get information about how this field is materialized.
-    * @return the field corresponding to this vector
-    */
-   @Override
-   public Field getField() {
-      return field;
-   }
-
-   /**
-    * Return the underlying buffers associated with this vector. Note that this doesn't
-    * impact the reference counts for this buffer so it only should be used for in-context
-    * access. Also note that this buffer changes regularly thus
-    * external classes shouldn't hold a reference to it (unless they change it).
-    *
-    * @param clear Whether to clear vector before returning; the buffers will still be refcounted
-    *              but the returned array will be the only reference to them
-    * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
-    *         vector instance.
-    */
-   @Override
-   public ArrowBuf[] getBuffers(boolean clear) {
-      final ArrowBuf[] buffers = new ArrowBuf[3];
+    final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
+    return validityBufferSize + offsetBufferSize + dataBufferSize;
+  }
+
+  /**
+   * Get information about how this field is materialized.
+   * @return the field corresponding to this vector
+   */
+  @Override
+  public Field getField() {
+    return field;
+  }
+
+  /**
+   * Return the underlying buffers associated with this vector. Note that this doesn't
+   * impact the reference counts for this buffer so it only should be used for in-context
+   * access. Also note that this buffer changes regularly thus
+   * external classes shouldn't hold a reference to it (unless they change it).
+   *
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
+   *              but the returned array will be the only reference to them
+   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
+   *         vector instance.
+   */
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers;
+    setReaderAndWriterIndex();
+    if (getBufferSize() == 0) {
+      buffers = new ArrowBuf[0];
+    } else {
+      buffers = new ArrowBuf[3];
       buffers[0] = validityBuffer;
       buffers[1] = offsetBuffer;
-      buffers[1] = valueBuffer;
-      if (clear) {
-         for (final ArrowBuf buffer:buffers) {
-            buffer.retain(1);
-         }
-         clear();
+      buffers[2] = valueBuffer;
+    }
+    if (clear) {
+      for (final ArrowBuf buffer : buffers) {
+        buffer.retain(1);
       }
-      return buffers;
-   }
-
-   /**
-    * Construct a transfer pair of this vector and another vector of same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @param callBack
-    * @return TransferPair
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-      return getTransferPair(ref, allocator);
-   }
-
-   /**
-    * Construct a transfer pair of this vector and another vector of same type.
-    * @param allocator allocator for the target vector
-    * @return TransferPair
-    */
-   @Override
-   public TransferPair getTransferPair(BufferAllocator allocator){
-      return getTransferPair(name, allocator);
-   }
-
-   /**
-    * Construct a transfer pair of this vector and another vector of same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return TransferPair
-    */
-   public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);
-
-   /**
-    * Transfer this vector'data to another vector. The memory associated
-    * with this vector is transferred to the allocator of target vector
-    * for accounting and management purposes.
-    * @param target destination vector for transfer
-    */
-   public void transferTo(BaseNullableVariableWidthVector target){
-      compareTypes(target, "transferTo");
-      target.clear();
-      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
-      target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer;
-      target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer;
-      target.valueCount = valueCount;
-      target.setLastSet(lastSet);
       clear();
-   }
-
-   /**
-    * Slice this vector at desired index and length and transfer the
-    * corresponding data to the target vector.
-    * @param startIndex start position of the split in source vector.
-    * @param length length of the split.
-    * @param target destination vector
-    */
-   public void splitAndTransferTo(int startIndex, int length,
-                                  BaseNullableVariableWidthVector target) {
-      compareTypes(target, "splitAndTransferTo");
-      target.clear();
-      splitAndTransferValidityBuffer(startIndex, length, target);
-      splitAndTransferOffsetBuffer(startIndex, length, target);
-      target.setLastSet(length - 1);
-      target.setValueCount(length);
-   }
-
-   /*
-    * Transfer the offsets along with data. Unlike the data buffer, we cannot simply
-    * slice the offset buffer for split and transfer. The reason is that offsets
-    * in the target vector have to be adjusted and made relative to the staring
-    * offset in source vector from the start index of split. This is why, we
-    * need to explicitly allocate the offset buffer and set the adjusted offsets
-    * in the target vector.
-    */
-   private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) {
-      final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH);
-      final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH);
-      final int dataLength = end - start;
-      target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH);
-      for (int i = 0; i < length + 1; i++) {
-         final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start;
-         target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset);
+    }
+    return buffers;
+  }
+
+  /**
+   * Construct a transfer pair of this vector and another vector of same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @param callBack
+   * @return TransferPair
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return getTransferPair(ref, allocator);
+  }
+
+  /**
+   * Construct a transfer pair of this vector and another vector of same type.
+   * @param allocator allocator for the target vector
+   * @return TransferPair
+   */
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return getTransferPair(name, allocator);
+  }
+
+  /**
+   * Construct a transfer pair of this vector and another vector of same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return TransferPair
+   */
+  public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);
+
+  /**
+   * Transfer this vector'data to another vector. The memory associated
+   * with this vector is transferred to the allocator of target vector
+   * for accounting and management purposes.
+   * @param target destination vector for transfer
+   */
+  public void transferTo(BaseNullableVariableWidthVector target) {
+    compareTypes(target, "transferTo");
+    target.clear();
+    target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
+    target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer;
+    target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer;
+    target.setLastSet(this.lastSet);
+    if (this.valueCount > 0) {
+      target.setValueCount(this.valueCount);
+    }
+    clear();
+  }
+
+  /**
+   * Slice this vector at desired index and length and transfer the
+   * corresponding data to the target vector.
+   * @param startIndex start position of the split in source vector.
+   * @param length length of the split.
+   * @param target destination vector
+   */
+  public void splitAndTransferTo(int startIndex, int length,
+                                 BaseNullableVariableWidthVector target) {
+    compareTypes(target, "splitAndTransferTo");
+    target.clear();
+    splitAndTransferValidityBuffer(startIndex, length, target);
+    splitAndTransferOffsetBuffer(startIndex, length, target);
+    target.setLastSet(length - 1);
+    if (this.valueCount > 0) {
+      target.setValueCount(this.valueCount);
+    }
+  }
+
+  /*
+   * Transfer the offsets along with data. Unlike the data buffer, we cannot simply
+   * slice the offset buffer for split and transfer. The reason is that offsets
+   * in the target vector have to be adjusted and made relative to the staring
+   * offset in source vector from the start index of split. This is why, we
+   * need to explicitly allocate the offset buffer and set the adjusted offsets
+   * in the target vector.
+   */
+  private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) {
+    final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH);
+    final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH);
+    final int dataLength = end - start;
+    target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH);
+    for (int i = 0; i < length + 1; i++) {
+      final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start;
+      target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset);
+    }
+    target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer;
+  }
+
+  /*
+   * Transfer the validity.
+   */
+  private void splitAndTransferValidityBuffer(int startIndex, int length,
+                                              BaseNullableVariableWidthVector target) {
+    assert startIndex + length <= valueCount;
+    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+    int byteSizeTarget = getValidityBufferSizeFromCount(length);
+    int offset = startIndex % 8;
+
+    if (length > 0) {
+      if (offset == 0) {
+        // slice
+        if (target.validityBuffer != null) {
+          target.validityBuffer.release();
+        }
+        target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
+        target.validityBuffer.retain(1);
+      } else {
+        /* Copy data
+         * When the first bit starts from the middle of a byte (offset != 0),
+         * copy data from src BitVector.
+         * Each byte in the target is composed by a part in i-th byte,
+         * another part in (i+1)-th byte.
+         */
+        target.allocateValidityBuffer(byteSizeTarget);
+
+        for (int i = 0; i < byteSizeTarget - 1; i++) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset);
+
+          target.validityBuffer.setByte(i, (b1 + b2));
+        }
+        /* Copying the last piece is done in the following manner:
+         * if the source vector has 1 or more bytes remaining, we copy
+         * the last piece as a byte formed by shifting data
+         * from the current byte and the next byte.
+         *
+         * if the source vector has no more bytes remaining
+         * (we are at the last byte), we copy the last piece as a byte
+         * by shifting data from the current byte.
+         */
+        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
+                  firstByteSource + byteSizeTarget, offset);
+
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+        } else {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+        }
       }
-      target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer;
-   }
-
-   /*
-    * Transfer the validity.
-    */
-   private void splitAndTransferValidityBuffer(int startIndex, int length,
-                                               BaseNullableVariableWidthVector target) {
-      assert startIndex + length <= valueCount;
-      int firstByteSource = BitVectorHelper.byteIndex(startIndex);
-      int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
-      int byteSizeTarget = getValidityBufferSizeFromCount(length);
-      int offset = startIndex % 8;
-
-      if (length > 0) {
-         if (offset == 0) {
-            // slice
-            if (target.validityBuffer != null) {
-               target.validityBuffer.release();
-            }
-            target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
-            target.validityBuffer.retain(1);
-         }
-         else {
-            /* Copy data
-             * When the first bit starts from the middle of a byte (offset != 0),
-             * copy data from src BitVector.
-             * Each byte in the target is composed by a part in i-th byte,
-             * another part in (i+1)-th byte.
-             */
-            target.allocateValidityBuffer(byteSizeTarget);
-
-            for (int i = 0; i < byteSizeTarget - 1; i++) {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset);
-               byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset);
-
-               target.validityBuffer.setByte(i, (b1 + b2));
-            }
-
-            /* Copying the last piece is done in the following manner:
-             * if the source vector has 1 or more bytes remaining, we copy
-             * the last piece as a byte formed by shifting data
-             * from the current byte and the next byte.
-             *
-             * if the source vector has no more bytes remaining
-             * (we are at the last byte), we copy the last piece as a byte
-             * by shifting data from the current byte.
-             */
-            if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
-                       firstByteSource + byteSizeTarget - 1, offset);
-               byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
-                       firstByteSource + byteSizeTarget, offset);
-
-               target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
-            }
-            else {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
-                       firstByteSource + byteSizeTarget - 1, offset);
-               target.validityBuffer.setByte(byteSizeTarget - 1, b1);
-            }
-         }
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                common getters and setters                      *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the number of elements that are null in the vector
-    *
-    * @return the number of null elements.
-    */
-   public int getNullCount() {
-      return BitVectorHelper.getNullCount(validityBuffer, valueCount);
-   }
-
-   /**
-    * Check if the given index is within the current value capacity
-    * of the vector
-    *
-    * @param index  position to check
-    * @return true if index is within the current value capacity
-    */
-   public boolean isSafe(int index) {
-      return index < getValueCapacity();
-   }
-
-   /**
-    * Check if element at given index is null.
-    *
-    * @param index  position of element
-    * @return true if element at given index is null
-    */
-   public boolean isNull(int index) {
-      return (isSet(index) == 0);
-   }
-
-   /**
-    * Same as {@link #isNull(int)}.
-    *
-    * @param index  position of element
-    * @return 1 if element at given index is not null, 0 otherwise
-    */
-   public int isSet(int index) {
-      final int byteIndex = index >> 3;
-      final byte b = validityBuffer.getByte(byteIndex);
-      final int bitIndex = index & 7;
-      return Long.bitCount(b & (1L << bitIndex));
-   }
-
-   /**
-    * Get the value count of vector. This will always be zero unless
-    * setValueCount(int) has been called prior to calling this.
-    *
-    * @return valueCount for the vector
-    */
-   public int getValueCount(){
-      return valueCount;
-   }
-
-   /**
-    * Sets the value count for the vector
-    *
-    * @param valueCount   value count
-    */
-   public void setValueCount(int valueCount) {
-      assert valueCount >= 0;
-      this.valueCount = valueCount;
-      while (valueCount > getValueCapacity()) {
-         reallocValidityAndOffsetBuffers();
-      }
-      fillHoles(valueCount);
-      lastSet = valueCount - 1;
-   }
-
-   /**
-    * Create holes in the vector upto the given index (exclusive).
-    * Holes will be created from the current last set position in
-    * the vector.
-    *
-    * @param index target index
-    */
-   public void fillEmpties(int index) {
-      handleSafe(index, emptyByteArray.length);
-      fillHoles(index);
-      lastSet = index - 1;
-   }
-
-   /**
-    * Set the index of last non-null element in the vector.
-    * It is important to call this method with appropriate value
-    * before calling {@link #setValueCount(int)}.
-    *
-    * @param value desired index of last non-null element.
-    */
-   public void setLastSet(int value) {
-      lastSet = value;
-   }
-
-   /**
-    * Get the index of last non-null element in the vector.
-    *
-    * @return index of the last non-null element
-    */
-   public int getLastSet() {
-      return lastSet;
-   }
-
-   /**
-    * Get the starting position (offset) in the data stream for a given
-    * element in the vector.
-    *
-    * @param index position of the element in the vector
-    * @return starting offset for the element
-    */
-   public long getStartEnd(int index) {
-      return (long)offsetBuffer.getInt(index * OFFSET_WIDTH);
-   }
-
-   /**
-    * Mark the particular position in the vector as non-null.
-    *
-    * @param index position of the element.
-    */
-   public void setIndexDefined(int index) {
-      handleSafe(index, 0);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                helper methods for setters                      *
-    *                                                                *
-    ******************************************************************/
-
-
-   protected final void fillHoles(int index) {
-      for (int i = lastSet + 1; i < index; i++) {
-         setBytes(i, emptyByteArray, 0, emptyByteArray.length);
-      }
-      lastSet = index - 1;
-   }
-
-   protected final void setBytes(int index, byte[] value, int start, int length) {
-      /* end offset of current last element in the vector. this will
-       * be the start offset of new element we are trying to store.
-       */
-      final int startOffset = getstartOffset(index);
-      /* set new end offset */
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
-      /* store the var length data in value buffer */
-      valueBuffer.setBytes(startOffset, value, start, length);
-   }
-
-   protected final int getstartOffset(int index) {
-      return offsetBuffer.getInt(index * OFFSET_WIDTH);
-   }
-
-   protected final void handleSafe(int index, int dataLength) {
-      /*
-       * IMPORTANT:
-       * value buffer for variable length vectors moves independent
-       * of the companion validity and offset buffers. This is in
-       * contrast to what we have for fixed width vectors.
-       *
-       * Here there is no concept of getValueCapacity() in the
-       * data stream. getValueCapacity() is applicable only to validity
-       * and offset buffers.
-       *
-       * So even though we may have setup an initial capacity of 1024
-       * elements in the vector, it is quite possible
-       * that we need to reAlloc() the data buffer when we are setting
-       * the 5th element in the vector simply because previous
-       * variable length elements have exhausted the buffer capacity.
-       * However, we really don't need to reAlloc() validity and
-       * offset buffers until we try to set the 1025th element
-       * This is why we do a separate check for safe methods to
-       * determine which buffer needs reallocation.
-       */
-      while (index >= getValueCapacity()) {
-         reallocValidityAndOffsetBuffers();
-      }
-      final int startOffset = getstartOffset(index);
-      while (valueBuffer.capacity() < (startOffset + dataLength)) {
-         reallocValueBuffer();
-      }
-   }
-
-   /**
-    * Method used by Json Writer to read a variable width element from
-    * the variable width vector and write to Json.
-    *
-    * This method should not be used externally.
-    *
-    * @param data buffer storing the variable width vector elements
-    * @param offset buffer storing the offsets of variable width vector elements
-    * @param index position of the element in the vector
-    * @return array of bytes
-    */
-   public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) {
-      final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH);
-      final int dataLength =
-              offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset;
-      final byte[] result = new byte[dataLength];
-      data.getBytes(currentStartOffset, result, 0, dataLength);
-      return result;
-   }
-
-   /**
-    * Method used by Json Reader to explicitly set the offsets of the variable
-    * width vector data. The method takes care of allocating the memory for
-    * offsets if the caller hasn't done so.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer ArrowBuf to store offsets for variable width elements
-    * @param allocator memory allocator
-    * @param valueCount number of elements
-    * @param index position of the element
-    * @param value offset of the element
-    * @return buffer holding the offsets
-    */
-   public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
-                              int valueCount, int index, int value) {
-      if (buffer == null) {
-         buffer = allocator.buffer(valueCount * OFFSET_WIDTH);
-      }
-      buffer.setInt(index * OFFSET_WIDTH, value);
-      if (index == (valueCount - 1)) {
-         buffer.writerIndex(valueCount * OFFSET_WIDTH);
-      }
-
-      return buffer;
-   }
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                common getters and setters                      *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the number of elements that are null in the vector
+   *
+   * @return the number of null elements.
+   */
+  public int getNullCount() {
+    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+  }
+
+  /**
+   * Check if the given index is within the current value capacity
+   * of the vector
+   *
+   * @param index  position to check
+   * @return true if index is within the current value capacity
+   */
+  public boolean isSafe(int index) {
+    return index < getValueCapacity();
+  }
+
+  /**
+   * Check if element at given index is null.
+   *
+   * @param index  position of element
+   * @return true if element at given index is null
+   */
+  public boolean isNull(int index) {
+    return (isSet(index) == 0);
+  }
+
+  /**
+   * Same as {@link #isNull(int)}.
+   *
+   * @param index  position of element
+   * @return 1 if element at given index is not null, 0 otherwise
+   */
+  public int isSet(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = validityBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
+  }
+
+  /**
+   * Get the value count of vector. This will always be zero unless
+   * setValueCount(int) has been called prior to calling this.
+   *
+   * @return valueCount for the vector
+   */
+  public int getValueCount() {
+    return valueCount;
+  }
+
+  /**
+   * Sets the value count for the vector
+   *
+   * @param valueCount   value count
+   */
+  public void setValueCount(int valueCount) {
+    assert valueCount >= 0;
+    this.valueCount = valueCount;
+    while (valueCount > getValueCapacity()) {
+      reallocValidityAndOffsetBuffers();
+    }
+    fillHoles(valueCount);
+    lastSet = valueCount - 1;
+    setReaderAndWriterIndex();
+  }
+
+  /**
+   * Create holes in the vector upto the given index (exclusive).
+   * Holes will be created from the current last set position in
+   * the vector.
+   *
+   * @param index target index
+   */
+  public void fillEmpties(int index) {
+    handleSafe(index, emptyByteArray.length);
+    fillHoles(index);
+    lastSet = index - 1;
+  }
+
+  /**
+   * Set the index of last non-null element in the vector.
+   * It is important to call this method with appropriate value
+   * before calling {@link #setValueCount(int)}.
+   *
+   * @param value desired index of last non-null element.
+   */
+  public void setLastSet(int value) {
+    lastSet = value;
+  }
+
+  /**
+   * Get the index of last non-null element in the vector.
+   *
+   * @return index of the last non-null element
+   */
+  public int getLastSet() {
+    return lastSet;
+  }
+
+  /**
+   * Get the starting position (offset) in the data stream for a given
+   * element in the vector.
+   *
+   * @param index position of the element in the vector
+   * @return starting offset for the element
+   */
+  public long getStartEnd(int index) {
+    return offsetBuffer.getLong(index * OFFSET_WIDTH);
+  }
+
+  /**
+   * Mark the particular position in the vector as non-null.
+   *
+   * @param index position of the element.
+   */
+  @Override
+  public void setIndexDefined(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      validityBuffer = reallocBufferHelper(validityBuffer, false);
+    }
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+  }
+
+  /**
+   * Sets the value length for an element.
+   *
+   * @param index   position of the element to set
+   * @param length  length of the element
+   */
+  public void setValueLengthSafe(int index, int length) {
+    assert index >= 0;
+    handleSafe(index, length);
+    fillHoles(index);
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+    lastSet = index;
+  }
+
+  /**
+   * Get the variable length element at specified index as Text.
+   *
+   * @param index   position of element to get
+   * @return greater than 0 length for non-null element, 0 otherwise
+   */
+  public int getValueLength(int index) {
+    assert index >= 0;
+    if (isSet(index) == 0) {
+      return 0;
+    }
+    final int startOffset = getstartOffset(index);
+    final int dataLength =
+            offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+    return dataLength;
+  }
+
+  /**
+   * Set the variable length element at the specified index to the supplied
+   * byte array. This is same as using {@link #set(int, byte[], int, int)}
+   * with start as 0 and length as value.length
+   *
+   * @param index   position of the element to set
+   * @param value   array of bytes to write
+   */
+  public void set(int index, byte[] value) {
+    assert index >= 0;
+    fillHoles(index);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setBytes(index, value, 0, value.length);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, byte[])} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param value   array of bytes to write
+   */
+  public void setSafe(int index, byte[] value) {
+    assert index >= 0;
+    fillEmpties(index);
+    handleSafe(index, value.length);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setBytes(index, value, 0, value.length);
+    lastSet = index;
+  }
+
+  /**
+   * Set the variable length element at the specified index to the supplied
+   * byte array.
+   *
+   * @param index   position of the element to set
+   * @param value   array of bytes to write
+   * @param start   start index in array of bytes
+   * @param length  length of data in array of bytes
+   */
+  public void set(int index, byte[] value, int start, int length) {
+    assert index >= 0;
+    fillHoles(index);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setBytes(index, value, start, length);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, byte[], int, int)} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param value   array of bytes to write
+   * @param start   start index in array of bytes
+   * @param length  length of data in array of bytes
+   */
+  public void setSafe(int index, byte[] value, int start, int length) {
+    assert index >= 0;
+    fillEmpties(index);
+    handleSafe(index, length);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setBytes(index, value, start, length);
+    lastSet = index;
+  }
+
+  /**
+   * Set the variable length element at the specified index to the
+   * content in supplied ByteBuffer
+   *
+   * @param index   position of the element to set
+   * @param value   ByteBuffer with data
+   * @param start   start index in ByteBuffer
+   * @param length  length of data in ByteBuffer
+   */
+  public void set(int index, ByteBuffer value, int start, int length) {
+    assert index >= 0;
+    fillHoles(index);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+    valueBuffer.setBytes(startOffset, value, start, length);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param value   ByteBuffer with data
+   * @param start   start index in ByteBuffer
+   * @param length  length of data in ByteBuffer
+   */
+  public void setSafe(int index, ByteBuffer value, int start, int length) {
+    assert index >= 0;
+    fillEmpties(index);
+    handleSafe(index, length);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+    valueBuffer.setBytes(startOffset, value, start, length);
+    lastSet = index;
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      validityBuffer = reallocBufferHelper(validityBuffer, false);
+    }
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param start start position of data in buffer
+   * @param end end position of data in buffer
+   * @param buffer data buffer containing the variable width element to be stored
+   *               in the vector
+   */
+  public void set(int index, int isSet, int start, int end, ArrowBuf buffer) {
+    assert index >= 0;
+    final int dataLength = end - start;
+    fillHoles(index);
+    BitVectorHelper.setValidityBit(validityBuffer, index, isSet);
+    final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, buffer, start, dataLength);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param start start position of data in buffer
+   * @param end end position of data in buffer
+   * @param buffer data buffer containing the variable width element to be stored
+   *               in the vector
+   */
+  public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) {
+    assert index >= 0;
+    final int dataLength = end - start;
+    fillEmpties(index);
+    handleSafe(index, end);
+    BitVectorHelper.setValidityBit(validityBuffer, index, isSet);
+    final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, buffer, start, dataLength);
+    lastSet = index;
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param start start position of data in buffer
+   * @param length length of data in buffer
+   * @param buffer data buffer containing the variable width element to be stored
+   *               in the vector
+   */
+  public void set(int index, int start, int length, ArrowBuf buffer) {
+    assert index >= 0;
+    fillHoles(index);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+    final ArrowBuf bb = buffer.slice(start, length);
+    valueBuffer.setBytes(startOffset, bb);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param start start position of data in buffer
+   * @param length length of data in buffer
+   * @param buffer data buffer containing the variable width element to be stored
+   *               in the vector
+   */
+  public void setSafe(int index, int start, int length, ArrowBuf buffer) {
+    assert index >= 0;
+    fillEmpties(index);
+    handleSafe(index, length);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+    final ArrowBuf bb = buffer.slice(start, length);
+    valueBuffer.setBytes(startOffset, bb);
+    lastSet = index;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                helper methods for setters                      *
+   *                                                                *
+   ******************************************************************/
+
+
+  protected final void fillHoles(int index) {
+    for (int i = lastSet + 1; i < index; i++) {
+      setBytes(i, emptyByteArray, 0, emptyByteArray.length);
+    }
+    lastSet = index - 1;
+  }
+
+  protected final void setBytes(int index, byte[] value, int start, int length) {
+    /* end offset of current last element in the vector. this will
+     * be the start offset of new element we are trying to store.
+     */
+    final int startOffset = getstartOffset(index);
+    /* set new end offset */
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
+    /* store the var length data in value buffer */
+    valueBuffer.setBytes(startOffset, value, start, length);
+  }
+
+  protected final int getstartOffset(int index) {
+    return offsetBuffer.getInt(index * OFFSET_WIDTH);
+  }
+
+  protected final void handleSafe(int index, int dataLength) {
+    /*
+     * IMPORTANT:
+     * value buffer for variable length vectors moves independent
+     * of the companion validity and offset buffers. This is in
+     * contrast to what we have for fixed width vectors.
+     *
+     * Here there is no concept of getValueCapacity() in the
+     * data stream. getValueCapacity() is applicable only to validity
+     * and offset buffers.
+     *
+     * So even though we may have setup an initial capacity of 1024
+     * elements in the vector, it is quite possible
+     * that we need to reAlloc() the data buffer when we are setting
+     * the 5th element in the vector simply because previous
+     * variable length elements have exhausted the buffer capacity.
+     * However, we really don't need to reAlloc() validity and
+     * offset buffers until we try to set the 1025th element
+     * This is why we do a separate check for safe methods to
+     * determine which buffer needs reallocation.
+     */
+    while (index >= getValueCapacity()) {
+      reallocValidityAndOffsetBuffers();
+    }
+    final int startOffset = getstartOffset(index);
+    while (valueBuffer.capacity() < (startOffset + dataLength)) {
+      reallocDataBuffer();
+    }
+  }
+
+  /**
+   * Method used by Json Writer to read a variable width element from
+   * the variable width vector and write to Json.
+   *
+   * This method should not be used externally.
+   *
+   * @param data buffer storing the variable width vector elements
+   * @param offset buffer storing the offsets of variable width vector elements
+   * @param index position of the element in the vector
+   * @return array of bytes
+   */
+  public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) {
+    final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH);
+    final int dataLength =
+            offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset;
+    final byte[] result = new byte[dataLength];
+    data.getBytes(currentStartOffset, result, 0, dataLength);
+    return result;
+  }
+
+  /**
+   * Method used by Json Reader to explicitly set the offsets of the variable
+   * width vector data. The method takes care of allocating the memory for
+   * offsets if the caller hasn't done so.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer ArrowBuf to store offsets for variable width elements
+   * @param allocator memory allocator
+   * @param valueCount number of elements
+   * @param index position of the element
+   * @param value offset of the element
+   * @return buffer holding the offsets
+   */
+  public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator,
+                             int valueCount, int index, int value) {
+    if (buffer == null) {
+      buffer = allocator.buffer(valueCount * OFFSET_WIDTH);
+    }
+    buffer.setInt(index * OFFSET_WIDTH, value);
+    if (index == (valueCount - 1)) {
+      buffer.writerIndex(valueCount * OFFSET_WIDTH);
+    }
+
+    return buffer;
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 1b96782681f26..fc0ab3e5915cc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 591d13c4bd5b4..c6d404e15430c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -251,12 +251,12 @@ public void copyFromSafe(int inIndex, int outIndex, BitVector from) {
 
   @Override
   public Mutator getMutator() {
-    return new Mutator();
+    return mutator;
   }
 
   @Override
   public Accessor getAccessor() {
-    return new Accessor();
+    return accessor;
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
index 69325b5ea06b6..23252ca697b8d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,6 +20,7 @@
 
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
 
 /**
  * Helper class for performing generic operations on a bit vector buffer.
@@ -27,139 +28,174 @@
  */
 public class BitVectorHelper {
 
-   /**
-    * Get the index of byte corresponding to bit index in validity buffer
-    */
-   public static int byteIndex(int absoluteBitIndex) {
-      return absoluteBitIndex >> 3;
-   }
-
-   /**
-    * Get the relative index of bit within the byte in validity buffer
-    */
-   public static int bitIndex(int absoluteBitIndex) {
-      return absoluteBitIndex & 7;
-   }
-
-   /**
-    * Set the bit at provided index to 1.
-    *
-    * @param validityBuffer
-    * @param index
-    */
-   public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) {
-      final int byteIndex = byteIndex(index);
-      final int bitIndex = bitIndex(index);
-      byte currentByte = validityBuffer.getByte(byteIndex);
-      final byte bitMask = (byte) (1L << bitIndex);
+  /**
+   * Get the index of byte corresponding to bit index in validity buffer
+   */
+  public static int byteIndex(int absoluteBitIndex) {
+    return absoluteBitIndex >> 3;
+  }
+
+  /**
+   * Get the relative index of bit within the byte in validity buffer
+   */
+  public static int bitIndex(int absoluteBitIndex) {
+    return absoluteBitIndex & 7;
+  }
+
+  /**
+   * Set the bit at provided index to 1.
+   *
+   * @param validityBuffer
+   * @param index
+   */
+  public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) {
+    final int byteIndex = byteIndex(index);
+    final int bitIndex = bitIndex(index);
+    byte currentByte = validityBuffer.getByte(byteIndex);
+    final byte bitMask = (byte) (1L << bitIndex);
+    currentByte |= bitMask;
+    validityBuffer.setByte(byteIndex, currentByte);
+  }
+
+  /**
+   * Set the bit at a given index to provided value (1 or 0)
+   *
+   * @param validityBuffer
+   * @param index
+   * @param value
+   */
+  public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) {
+    final int byteIndex = byteIndex(index);
+    final int bitIndex = bitIndex(index);
+    byte currentByte = validityBuffer.getByte(byteIndex);
+    final byte bitMask = (byte) (1L << bitIndex);
+    if (value != 0) {
       currentByte |= bitMask;
-      validityBuffer.setByte(byteIndex, currentByte);
-   }
-
-   /**
-    * Set the bit at a given index to provided value (1 or 0)
-    *
-    * @param validityBuffer
-    * @param index
-    * @param value
-    */
-   public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) {
-      final int byteIndex = byteIndex(index);
-      final int bitIndex = bitIndex(index);
-      byte currentByte = validityBuffer.getByte(byteIndex);
-      final byte bitMask = (byte) (1L << bitIndex);
-      if (value != 0) {
-         currentByte |= bitMask;
-      } else {
-         currentByte -= (bitMask & currentByte);
-      }
-      validityBuffer.setByte(byteIndex, currentByte);
-   }
-
-   /**
-    * Set the bit at a given index to provided value (1 or 0). Internally
-    * takes care of allocating the buffer if the caller didn't do so.
-    *
-    * @param validityBuffer
-    * @param allocator
-    * @param valueCount
-    * @param index
-    * @param value
-    * @return ArrowBuf
-    */
-   public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator,
-                                         int valueCount, int index, int value) {
-      if (validityBuffer == null) {
-         validityBuffer = allocator.buffer(getValidityBufferSize(valueCount));
+    } else {
+      currentByte -= (bitMask & currentByte);
+    }
+    validityBuffer.setByte(byteIndex, currentByte);
+  }
+
+  /**
+   * Set the bit at a given index to provided value (1 or 0). Internally
+   * takes care of allocating the buffer if the caller didn't do so.
+   *
+   * @param validityBuffer
+   * @param allocator
+   * @param valueCount
+   * @param index
+   * @param value
+   * @return ArrowBuf
+   */
+  public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator,
+                                        int valueCount, int index, int value) {
+    if (validityBuffer == null) {
+      validityBuffer = allocator.buffer(getValidityBufferSize(valueCount));
+    }
+    setValidityBit(validityBuffer, index, value);
+    if (index == (valueCount - 1)) {
+      validityBuffer.writerIndex(getValidityBufferSize(valueCount));
+    }
+
+    return validityBuffer;
+  }
+
+  /**
+   * Check if a bit at a given index is set or not.
+   *
+   * @param buffer
+   * @param index
+   * @return 1 if bit is set, 0 otherwise.
+   */
+  public static int get(final ArrowBuf buffer, int index) {
+    final int byteIndex = index >> 3;
+    final byte b = buffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
+  }
+
+  /**
+   * Compute the size of validity buffer required to manage a given number
+   * of elements in a vector.
+   *
+   * @param valueCount
+   * @return buffer size
+   */
+  public static int getValidityBufferSize(int valueCount) {
+    return ((int) Math.ceil(valueCount / 8.0));
+  }
+
+  /**
+   * Given a validity buffer, find the number of bits that are not set.
+   * This is used to compute the number of null elements in a nullable vector.
+   *
+   * @param validityBuffer
+   * @param valueCount
+   * @return number of bits not set.
+   */
+  public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+    int count = 0;
+    final int sizeInBytes = getValidityBufferSize(valueCount);
+
+    for (int i = 0; i < sizeInBytes; ++i) {
+      final byte byteValue = validityBuffer.getByte(i);
+      /* Java uses two's complement binary representation, hence 11111111_b which is -1
+       * when converted to Int will have 32bits set to 1. Masking the MSB and then
+       * adding it back solves the issue.
+       */
+      count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
+    }
+    int nullCount = (sizeInBytes * 8) - count;
+    /* if the valueCount is not a multiple of 8,
+     * the bits on the right were counted as null bits.
+     */
+    int remainder = valueCount % 8;
+    nullCount -= remainder == 0 ? 0 : 8 - remainder;
+    return nullCount;
+  }
+
+  public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) {
+    return (byte) ((data.getByte(index) & 0xFF) >>> offset);
+  }
+
+  public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
+    return (byte) ((data.getByte(index) << (8 - offset)));
+  }
+
+  public static ArrowBuf loadValidityBuffer(final ArrowFieldNode fieldNode,
+                                            final ArrowBuf sourceValidityBuffer,
+                                            final BufferAllocator allocator) {
+    final int valueCount = fieldNode.getLength();
+    ArrowBuf newBuffer = null;
+    /* either all NULLs or all non-NULLs */
+    if (fieldNode.getNullCount() == 0 || fieldNode.getNullCount() == valueCount) {
+      newBuffer = allocator.buffer(getValidityBufferSize(valueCount));
+      newBuffer.setZero(0, newBuffer.capacity());
+      if (fieldNode.getNullCount() != 0) {
+        /* all NULLs */
+        return newBuffer;
       }
-      setValidityBit(validityBuffer, index, value);
-      if (index == (valueCount - 1)) {
-         validityBuffer.writerIndex(getValidityBufferSize(valueCount));
+      /* all non-NULLs */
+      int fullBytesCount = valueCount / 8;
+      for (int i = 0; i < fullBytesCount; ++i) {
+        newBuffer.setByte(i, 0xFF);
       }
-
-      return validityBuffer;
-   }
-
-   /**
-    * Check if a bit at a given index is set or not.
-    *
-    * @param buffer
-    * @param index
-    * @return 1 if bit is set, 0 otherwise.
-    */
-   public static int get(final ArrowBuf buffer, int index) {
-      final int byteIndex = index >> 3;
-      final byte b = buffer.getByte(byteIndex);
-      final int bitIndex = index & 7;
-      return Long.bitCount(b & (1L << bitIndex));
-   }
-
-   /**
-    * Compute the size of validity buffer required to manage a given number
-    * of elements in a vector.
-    *
-    * @param valueCount
-    * @return buffer size
-    */
-   public static int getValidityBufferSize(int valueCount) {
-      return ((int) Math.ceil(valueCount / 8.0));
-   }
-
-   /**
-    * Given a validity buffer, find the number of bits that are not set.
-    * This is used to compute the number of null elements in a nullable vector.
-    *
-    * @param validityBuffer
-    * @param valueCount
-    * @return number of bits not set.
-    */
-   public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) {
-      if (valueCount == 0) { return 0; }
-      int count = 0;
-      final int sizeInBytes = getValidityBufferSize(valueCount);
-
-      for (int i = 0; i < sizeInBytes; ++i) {
-         final byte byteValue = validityBuffer.getByte(i);
-         /* Java uses two's complement binary representation, hence 11111111_b which is -1
-          * when converted to Int will have 32bits set to 1. Masking the MSB and then
-          * adding it back solves the issue.
-          */
-         count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
+      int remainder = valueCount % 8;
+      if (remainder > 0) {
+        byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7));
+        newBuffer.setByte(fullBytesCount, bitMask);
       }
-      int nullCount = (sizeInBytes * 8) - count;
-      /* if the valueCount is not a multiple of 8,
-       * the bits on the right were counted as null bits.
+    } else {
+      /* mixed byte pattern -- create another ArrowBuf associated with the
+       * target allocator
        */
-      int remainder = valueCount % 8;
-      nullCount -= remainder == 0 ? 0 : 8 - remainder;
-      return nullCount;
-   }
-
-   public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) {
-      return (byte)((data.getByte(index) & 0xFF) >>> offset);
-   }
+      newBuffer = sourceValidityBuffer.retain(allocator);
+    }
 
-   public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
-      return (byte)((data.getByte(index) << (8 - offset)));
-   }
+    return newBuffer;
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java
new file mode 100644
index 0000000000000..806beb5f8fedc
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import java.math.BigDecimal;
+import java.nio.charset.Charset;
+
+/**
+ * Helper class to generate test data for Nullable fixed and variable
+ * width scalar vectors. Previous implementations of java vector classes
+ * provided generateTestData(now deprecated) API to populate the vector
+ * with sample data. This class should be used for that purpose.
+ */
+public class GenerateSampleData {
+
+  public static void generateTestData(final ValueVector vector, final int valueCount) {
+    if (vector instanceof NullableIntVector) {
+      writeIntData((NullableIntVector) vector, valueCount);
+    } else if (vector instanceof NullableDecimalVector) {
+      writeDecimalData((NullableDecimalVector) vector, valueCount);
+    } else if (vector instanceof NullableBitVector) {
+      writeBooleanData((NullableBitVector) vector, valueCount);
+    } else if (vector instanceof NullableVarCharVector) {
+      writeVarCharData((NullableVarCharVector) vector, valueCount);
+    } else if (vector instanceof NullableVarBinaryVector) {
+      writeVarBinaryData((NullableVarBinaryVector) vector, valueCount);
+    } else if (vector instanceof NullableBigIntVector) {
+      writeBigIntData((NullableBigIntVector) vector, valueCount);
+    } else if (vector instanceof NullableFloat4Vector) {
+      writeFloatData((NullableFloat4Vector) vector, valueCount);
+    } else if (vector instanceof NullableFloat8Vector) {
+      writeDoubleData((NullableFloat8Vector) vector, valueCount);
+    } else if (vector instanceof NullableDateDayVector) {
+      writeDateDayData((NullableDateDayVector) vector, valueCount);
+    } else if (vector instanceof NullableDateMilliVector) {
+      writeDateMilliData((NullableDateMilliVector) vector, valueCount);
+    } else if (vector instanceof NullableIntervalDayVector) {
+      writeIntervalDayData((NullableIntervalDayVector) vector, valueCount);
+    } else if (vector instanceof NullableIntervalYearVector) {
+      writeIntervalYearData((NullableIntervalYearVector) vector, valueCount);
+    } else if (vector instanceof NullableSmallIntVector) {
+      writeSmallIntData((NullableSmallIntVector) vector, valueCount);
+    } else if (vector instanceof NullableTinyIntVector) {
+      writeTinyIntData((NullableTinyIntVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeMicroVector) {
+      writeTimeMicroData((NullableTimeMicroVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeMilliVector) {
+      writeTimeMilliData((NullableTimeMilliVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeNanoVector) {
+      writeTimeNanoData((NullableTimeNanoVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeSecVector) {
+      writeTimeSecData((NullableTimeSecVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeStampSecVector) {
+      writeTimeStampData((NullableTimeStampSecVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeStampMicroVector) {
+      writeTimeStampData((NullableTimeStampMicroVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeStampMilliVector) {
+      writeTimeStampData((NullableTimeStampMilliVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeStampNanoVector) {
+      writeTimeStampData((NullableTimeStampNanoVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeStampSecTZVector) {
+      writeTimeStampData((NullableTimeStampSecTZVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeStampMicroTZVector) {
+      writeTimeStampData((NullableTimeStampMicroTZVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeStampMilliTZVector) {
+      writeTimeStampData((NullableTimeStampMilliTZVector) vector, valueCount);
+    } else if (vector instanceof NullableTimeStampNanoTZVector) {
+      writeTimeStampData((NullableTimeStampNanoTZVector) vector, valueCount);
+    }
+  }
+
+  private static void writeTimeStampData(NullableTimeStampVector vector, int valueCount) {
+    final long even = 100000;
+    final long odd = 200000;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeDecimalData(NullableDecimalVector vector, int valueCount) {
+    final BigDecimal even = new BigDecimal(0.0543278923);
+    final BigDecimal odd = new BigDecimal(2.0543278923);
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeIntData(NullableIntVector vector, int valueCount) {
+    final int even = 1000;
+    final int odd = 2000;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeBooleanData(NullableBitVector vector, int valueCount) {
+    final int even = 0;
+    final int odd = 1;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeIntervalYearData(NullableIntervalYearVector vector, int valueCount) {
+    final int even = 1;
+    final int odd = 2;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeIntervalDayData(NullableIntervalDayVector vector, int valueCount) {
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, 1, 50);
+      } else {
+        vector.setSafe(i, 2, 100);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeTimeSecData(NullableTimeSecVector vector, int valueCount) {
+    final int even = 500;
+    final int odd = 900;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeTimeMilliData(NullableTimeMilliVector vector, int valueCount) {
+    final int even = 1000;
+    final int odd = 2000;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeTimeMicroData(NullableTimeMicroVector vector, int valueCount) {
+    final long even = 1000000000;
+    final long odd = 2000000000;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+
+  }
+
+  private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCount) {
+    final long even = 1000000000;
+    final long odd = 2000000000;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeDateDayData(NullableDateDayVector vector, int valueCount) {
+    final int even = 1000;
+    final int odd = 2000;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeDateMilliData(NullableDateMilliVector vector, int valueCount) {
+    final long even = 1000000000;
+    final long odd = 2000000000;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeSmallIntData(NullableSmallIntVector vector, int valueCount) {
+    final short even = 10;
+    final short odd = 20;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeTinyIntData(NullableTinyIntVector vector, int valueCount) {
+    final byte even = 1;
+    final byte odd = 2;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeBigIntData(NullableBigIntVector vector, int valueCount) {
+    final long even = 1000000000;
+    final long odd = 2000000000;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeFloatData(NullableFloat4Vector vector, int valueCount) {
+    final float even = 20.3f;
+    final float odd = 40.2f;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeDoubleData(NullableFloat8Vector vector, int valueCount) {
+    final double even = 20.2373;
+    final double odd = 40.2378;
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeVarBinaryData(NullableVarBinaryVector vector, int valueCount) {
+    Charset utf8Charset = Charset.forName("UTF-8");
+    final byte[] even = "AAAAA1".getBytes(utf8Charset);
+    final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset);
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+
+  private static void writeVarCharData(NullableVarCharVector vector, int valueCount) {
+    Charset utf8Charset = Charset.forName("UTF-8");
+    final byte[] even = "AAAAA1".getBytes(utf8Charset);
+    final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset);
+    for (int i = 0; i < valueCount; i++) {
+      if (i % 2 == 0) {
+        vector.setSafe(i, even);
+      } else {
+        vector.setSafe(i, odd);
+      }
+    }
+    vector.setValueCount(valueCount);
+  }
+}
+
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
index 253427333a2e0..eca6592c54ad2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,335 +34,335 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableBigIntVector extends BaseNullableFixedWidthVector {
-   public static final byte TYPE_WIDTH = 8;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableBigIntVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableBigIntVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableBigIntVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new BigIntReaderImpl(NullableBigIntVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.BIGINT;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public long get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  public static final byte TYPE_WIDTH = 8;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableBigIntVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableBigIntVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableBigIntVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new BigIntReaderImpl(NullableBigIntVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.BIGINT;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public long get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableBigIntHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Long getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableBigIntHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Long getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, long value) {
-      valueBuffer.setLong(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, long value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, BigIntHolder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, long value) {
+    valueBuffer.setLong(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, long value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, long)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, long value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, BigIntHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, BigIntHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, BigIntHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, long)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, long value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, BigIntHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, BigIntHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, long value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, long value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, long)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, long value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static long get(final ArrowBuf buffer, final int index) {
-      return buffer.getLong(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableBigIntVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableBigIntVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableBigIntVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableBigIntVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableBigIntVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, long)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, long value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static long get(final ArrowBuf buffer, final int index) {
+    return buffer.getLong(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableBigIntVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableBigIntVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableBigIntVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableBigIntVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableBigIntVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
index eceaa61b57f41..bd363f5b7f1f3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
@@ -1,5 +1,4 @@
-/*******************************************************************************
-
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -7,15 +6,15 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- ******************************************************************************/
+ */
 
 package org.apache.arrow.vector;
 
@@ -39,150 +38,160 @@
  * to a single bit in the underlying data stream backing the vector.
  */
 public class NullableBitVector extends BaseNullableFixedWidthVector {
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableBitVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableBitVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.BIT.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableBitVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, (byte)0);
-      reader = new BitReaderImpl(NullableBitVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.BIT;
-   }
-
-   /**
-    * Sets the desired value capacity for the vector. This function doesn't
-    * allocate any memory for the vector.
-    * @param valueCount desired number of elements in the vector
-    */
-   @Override
-   public void setInitialCapacity(int valueCount) {
-      final int size = getValidityBufferSizeFromCount(valueCount);
-      if (size > MAX_ALLOCATION_SIZE) {
-         throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
-      }
-      valueAllocationSizeInBytes = size;
-      validityAllocationSizeInBytes = size;
-   }
-
-   /**
-    * Get the current value capacity for the vector
-    * @return number of elements that vector can hold.
-    */
-   @Override
-   public int getValueCapacity(){
-      return (int)(validityBuffer.capacity() * 8L);
-   }
-
-   /**
-    * Get the potential buffer size for a particular number of records.
-    * @param count desired number of elements in the vector
-    * @return estimated size of underlying buffers if the vector holds
-    *         a given number of elements
-    */
-   @Override
-   public int getBufferSizeFor(final int count) {
-      if (count == 0) { return 0; }
-      return 2 * getValidityBufferSizeFromCount(count);
-   }
-
-   /**
-    * Get the size (number of bytes) of underlying buffers used by this
-    * vector
-    * @return size of underlying buffers.
-    */
-   @Override
-   public int getBufferSize() {
-     return getBufferSizeFor(valueCount);
-   }
-
-   /**
-    * Slice this vector at desired index and length and transfer the
-    * corresponding data to the target vector.
-    * @param startIndex start position of the split in source vector.
-    * @param length length of the split.
-    * @param target destination vector
-    */
-   public void splitAndTransferTo(int startIndex, int length,
-                                  BaseNullableFixedWidthVector target) {
-      compareTypes(target, "splitAndTransferTo");
-      target.clear();
-      target.validityBuffer = splitAndTransferBuffer(startIndex, length, target,
-                                 validityBuffer, target.validityBuffer);
-      target.valueBuffer = splitAndTransferBuffer(startIndex, length, target,
-                                 valueBuffer, target.valueBuffer);
-
-      target.setValueCount(length);
-   }
-
-   private ArrowBuf splitAndTransferBuffer(int startIndex, int length,
-                                           BaseNullableFixedWidthVector target,
-                                           ArrowBuf sourceBuffer, ArrowBuf destBuffer) {
-      assert startIndex + length <= valueCount;
-      int firstByteSource = BitVectorHelper.byteIndex(startIndex);
-      int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
-      int byteSizeTarget = getValidityBufferSizeFromCount(length);
-      int offset = startIndex % 8;
-
-      if (length > 0) {
-         if (offset == 0) {
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableBitVector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name      name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableBitVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.BIT.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableBitVector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name      name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, (byte) 0);
+    reader = new BitReaderImpl(NullableBitVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   *
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   *
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.BIT;
+  }
+
+  /**
+   * Sets the desired value capacity for the vector. This function doesn't
+   * allocate any memory for the vector.
+   *
+   * @param valueCount desired number of elements in the vector
+   */
+  @Override
+  public void setInitialCapacity(int valueCount) {
+    final int size = getValidityBufferSizeFromCount(valueCount);
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+    }
+    valueAllocationSizeInBytes = size;
+    validityAllocationSizeInBytes = size;
+  }
+
+  /**
+   * Get the current value capacity for the vector
+   *
+   * @return number of elements that vector can hold.
+   */
+  @Override
+  public int getValueCapacity() {
+    return (int) (validityBuffer.capacity() * 8L);
+  }
+
+  /**
+   * Get the potential buffer size for a particular number of records.
+   *
+   * @param count desired number of elements in the vector
+   * @return estimated size of underlying buffers if the vector holds
+   * a given number of elements
+   */
+  @Override
+  public int getBufferSizeFor(final int count) {
+    if (count == 0) {
+      return 0;
+    }
+    return 2 * getValidityBufferSizeFromCount(count);
+  }
+
+  /**
+   * Get the size (number of bytes) of underlying buffers used by this
+   * vector
+   *
+   * @return size of underlying buffers.
+   */
+  @Override
+  public int getBufferSize() {
+    return getBufferSizeFor(valueCount);
+  }
+
+  /**
+   * Slice this vector at desired index and length and transfer the
+   * corresponding data to the target vector.
+   *
+   * @param startIndex start position of the split in source vector.
+   * @param length     length of the split.
+   * @param target     destination vector
+   */
+  public void splitAndTransferTo(int startIndex, int length,
+                                 BaseNullableFixedWidthVector target) {
+    compareTypes(target, "splitAndTransferTo");
+    target.clear();
+    target.validityBuffer = splitAndTransferBuffer(startIndex, length, target,
+            validityBuffer, target.validityBuffer);
+    target.valueBuffer = splitAndTransferBuffer(startIndex, length, target,
+            valueBuffer, target.valueBuffer);
+
+    target.setValueCount(length);
+  }
+
+  private ArrowBuf splitAndTransferBuffer(int startIndex, int length,
+                                          BaseNullableFixedWidthVector target,
+                                          ArrowBuf sourceBuffer, ArrowBuf destBuffer) {
+    assert startIndex + length <= valueCount;
+    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+    int byteSizeTarget = getValidityBufferSizeFromCount(length);
+    int offset = startIndex % 8;
+
+    if (length > 0) {
+      if (offset == 0) {
             /* slice */
-            if (destBuffer != null) {
-               destBuffer.release();
-            }
-            destBuffer = destBuffer.slice(firstByteSource, byteSizeTarget);
-            destBuffer.retain(1);
-         }
-         else {
+        if (destBuffer != null) {
+          destBuffer.release();
+        }
+        destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget);
+        destBuffer.retain(1);
+      } else {
             /* Copy data
              * When the first bit starts from the middle of a byte (offset != 0),
              * copy data from src BitVector.
              * Each byte in the target is composed by a part in i-th byte,
              * another part in (i+1)-th byte.
              */
-            destBuffer = allocator.buffer(byteSizeTarget);
-            destBuffer.readerIndex(0);
-            destBuffer.setZero(0, destBuffer.capacity());
+        destBuffer = allocator.buffer(byteSizeTarget);
+        destBuffer.readerIndex(0);
+        destBuffer.setZero(0, destBuffer.capacity());
 
-            for (int i = 0; i < byteSizeTarget - 1; i++) {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset);
-               byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset);
+        for (int i = 0; i < byteSizeTarget - 1; i++) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset);
 
-               destBuffer.setByte(i, (b1 + b2));
-            }
+          destBuffer.setByte(i, (b1 + b2));
+        }
 
             /* Copying the last piece is done in the following manner:
              * if the source vector has 1 or more bytes remaining, we copy
@@ -193,310 +202,315 @@ private ArrowBuf splitAndTransferBuffer(int startIndex, int length,
              * (we are at the last byte), we copy the last piece as a byte
              * by shifting data from the current byte.
              */
-            if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
-                       firstByteSource + byteSizeTarget - 1, offset);
-               byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer,
-                       firstByteSource + byteSizeTarget, offset);
-
-               destBuffer.setByte(byteSizeTarget - 1, b1 + b2);
-            }
-            else {
-               byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
-                       firstByteSource + byteSizeTarget - 1, offset);
-               destBuffer.setByte(byteSizeTarget - 1, b1);
-            }
-         }
-      }
-
-      return destBuffer;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-   private int getBit(int index) {
-      final int byteIndex = index >> 3;
-      final byte b = valueBuffer.getByte(byteIndex);
-      final int bitIndex = index & 7;
-      return Long.bitCount(b & (1L << bitIndex));
-   }
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public int get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
-      return getBit(index);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableBitHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = getBit(index);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Boolean getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return new Boolean (getBit(index) != 0);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableBitVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      if (value != 0) {
-         BitVectorHelper.setValidityBitToOne(valueBuffer, index);
-      } else {
-         BitVectorHelper.setValidityBit(valueBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableBitHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         if (holder.value != 0) {
-            BitVectorHelper.setValidityBitToOne(valueBuffer, index);
-         } else {
-            BitVectorHelper.setValidityBit(valueBuffer, index, 0);
-         }
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer,
+                  firstByteSource + byteSizeTarget, offset);
+
+          destBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+        } else {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          destBuffer.setByte(byteSizeTarget - 1, b1);
+        }
       }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, BitHolder holder) {
+    }
+
+    return destBuffer;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+  private int getBit(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = valueBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
+  }
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index position of element
+   * @return element at given index
+   */
+  public int get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return getBit(index);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index position of element
+   */
+  public void get(int index, NullableBitHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = getBit(index);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index position of element
+   * @return element at given index
+   */
+  public Boolean getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return new Boolean(getBit(index) != 0);
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   *
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from      source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableBitVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   *
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from      source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index position of element
+   * @param value value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    if (value != 0) {
+      BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+    } else {
+      BitVectorHelper.setValidityBit(valueBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index  position of element
+   * @param holder nullable data holder for value of element
+   */
+  public void set(int index, NullableBitHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       if (holder.value != 0) {
-         BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+        BitVectorHelper.setValidityBitToOne(valueBuffer, index);
       } else {
-         BitVectorHelper.setValidityBit(valueBuffer, index, 0);
+        BitVectorHelper.setValidityBit(valueBuffer, index, 0);
       }
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableBitHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, BitHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, BitHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index  position of element
+   * @param holder data holder for value of element
+   */
+  public void set(int index, BitHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    if (holder.value != 0) {
+      BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+    } else {
+      BitVectorHelper.setValidityBit(valueBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index position of element
+   * @param value value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableBitHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index  position of element
+   * @param holder nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, BitHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index  position of element
+   * @param holder data holder for value of element
+   */
+  public void setSafe(int index, BitHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   *
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, int value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, int value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, int)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, int value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableBitVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableBitVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableBitVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableBitVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableBitVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, int)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   *
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, int value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   *
+   * @param ref       name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   *
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableBitVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableBitVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableBitVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableBitVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableBitVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
index e6b5b590f3f7e..c5ddc9e87be6a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -35,336 +35,336 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableDateDayVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 4;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableDateDayVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableDateDayVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableDateDayVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new DateDayReaderImpl(NullableDateDayVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.DATEDAY;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public int get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  private static final byte TYPE_WIDTH = 4;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableDateDayVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableDateDayVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableDateDayVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new DateDayReaderImpl(NullableDateDayVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.DATEDAY;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public int get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableDateDayHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Integer getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableDateDayHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Integer getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setInt(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, DateDayHolder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setInt(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, DateDayHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, DateDayHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, DateDayHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, DateDayHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, DateDayHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, int value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, int value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, int)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, int value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static int get(final ArrowBuf buffer, final int index) {
-      return buffer.getInt(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableDateDayVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableDateDayVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableDateDayVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableDateDayVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableDateDayVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, int)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, int value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static int get(final ArrowBuf buffer, final int index) {
+    return buffer.getInt(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableDateDayVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableDateDayVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableDateDayVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableDateDayVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableDateDayVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
index 8e15100f70818..7e37055dab188 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,338 +37,339 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableDateMilliVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 8;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableDateMilliVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new DateMilliReaderImpl(NullableDateMilliVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.DATEMILLI;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public long get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
-      return valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableDateMilliHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public LocalDateTime getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         final long millis = get(index);
-         final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
-         return localDateTime;
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, long value) {
-      valueBuffer.setLong(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, long value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, DateMilliHolder holder){
+  private static final byte TYPE_WIDTH = 8;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableDateMilliVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new DateMilliReaderImpl(NullableDateMilliVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.DATEMILLI;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public long get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableDateMilliHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public LocalDateTime getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      final long millis = valueBuffer.getLong(index * TYPE_WIDTH);
+      final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis,
+              org.joda.time.DateTimeZone.UTC);
+      return localDateTime;
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, long value) {
+    valueBuffer.setLong(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, long value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, long)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, long value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, DateMilliHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, DateMilliHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, DateMilliHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, long)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, long value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, DateMilliHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, DateMilliHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, long value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, long value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, long)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, long value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static long get(final ArrowBuf buffer, final int index) {
-      return buffer.getLong(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableDateMilliVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableDateMilliVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableDateMilliVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableDateMilliVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableDateMilliVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, long)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, long value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static long get(final ArrowBuf buffer, final int index) {
+    return buffer.getLong(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableDateMilliVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableDateMilliVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableDateMilliVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableDateMilliVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableDateMilliVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
index 2a611c6a2d5ba..dcc551094ae28 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,383 +37,391 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableDecimalVector extends BaseNullableFixedWidthVector {
-   public static final byte TYPE_WIDTH = 16;
-   private final FieldReader reader;
-
-   private final int precision;
-   private final int scale;
-
-   /**
-    * Instantiate a NullableDecimalVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableDecimalVector(String name, BufferAllocator allocator,
-                                int precision, int scale) {
-      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableDecimalVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal)fieldType.getType();
-      reader = new DecimalReaderImpl(NullableDecimalVector.this);
-      this.precision = arrowType.getPrecision();
-      this.scale = arrowType.getScale();
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.DECIMAL;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public ArrowBuf get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
-      return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableDecimalHolder holder) {
-      if (isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.buffer = valueBuffer;
-      holder.precision = precision;
-      holder.scale = scale;
-      holder.start = index * TYPE_WIDTH;
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public BigDecimal getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer,
-                 thisIndex * TYPE_WIDTH, TYPE_WIDTH);
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index    position of element
-    * @param buffer   ArrowBuf containing decimal value.
-    */
-   public void set(int index, ArrowBuf buffer) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index    position of element
-    * @param start    start index of data in the buffer
-    * @param buffer   ArrowBuf containing decimal value.
-    */
-   public void set(int index, int start, ArrowBuf buffer) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   BigDecimal containing decimal value.
-    */
-   public void set(int index, BigDecimal value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      DecimalUtility.checkPrecisionAndScale(value, precision, scale);
-      DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, DecimalHolder holder){
+  public static final byte TYPE_WIDTH = 16;
+  private final FieldReader reader;
+
+  private final int precision;
+  private final int scale;
+
+  /**
+   * Instantiate a NullableDecimalVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableDecimalVector(String name, BufferAllocator allocator,
+                               int precision, int scale) {
+    this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableDecimalVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal) fieldType.getType();
+    reader = new DecimalReaderImpl(NullableDecimalVector.this);
+    this.precision = arrowType.getPrecision();
+    this.scale = arrowType.getScale();
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.DECIMAL;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public ArrowBuf get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableDecimalHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.buffer = valueBuffer;
+    holder.precision = precision;
+    holder.scale = scale;
+    holder.start = index * TYPE_WIDTH;
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public BigDecimal getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale);
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer,
+              thisIndex * TYPE_WIDTH, TYPE_WIDTH);
+      BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+  /**
+   * Return scale for the decimal value
+   */
+  public int getScale() {
+    return scale;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index    position of element
+   * @param buffer   ArrowBuf containing decimal value.
+   */
+  public void set(int index, ArrowBuf buffer) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index    position of element
+   * @param start    start index of data in the buffer
+   * @param buffer   ArrowBuf containing decimal value.
+   */
+  public void set(int index, int start, ArrowBuf buffer) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   BigDecimal containing decimal value.
+   */
+  public void set(int index, BigDecimal value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    DecimalUtility.checkPrecisionAndScale(value, precision, scale);
+    DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #set(int, ArrowBuf)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param buffer  ArrowBuf containing decimal value.
-    */
-   public void setSafe(int index, ArrowBuf buffer) {
-      handleSafe(index);
-      set(index, buffer);
-   }
-
-   /**
-    * Same as {@link #set(int, int, ArrowBuf)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index    position of element
-    * @param start    start index of data in the buffer
-    * @param buffer   ArrowBuf containing decimal value.
-    */
-   public void setSafe(int index, int start, ArrowBuf buffer) {
-      handleSafe(index);
-      set(index, start, buffer);
-   }
-
-   /**
-    * Same as {@link #set(int, BigDecimal)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   BigDecimal containing decimal value.
-    */
-   public void setSafe(int index, BigDecimal value){
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, DecimalHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, DecimalHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, DecimalHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #set(int, ArrowBuf)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param buffer  ArrowBuf containing decimal value.
+   */
+  public void setSafe(int index, ArrowBuf buffer) {
+    handleSafe(index);
+    set(index, buffer);
+  }
+
+  /**
+   * Same as {@link #set(int, int, ArrowBuf)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index    position of element
+   * @param start    start index of data in the buffer
+   * @param buffer   ArrowBuf containing decimal value.
+   */
+  public void setSafe(int index, int start, ArrowBuf buffer) {
+    handleSafe(index);
+    set(index, start, buffer);
+  }
+
+  /**
+   * Same as {@link #set(int, BigDecimal)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   BigDecimal containing decimal value.
+   */
+  public void setSafe(int index, BigDecimal value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, DecimalHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, DecimalHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param start start position of the value in the buffer
+   * @param buffer buffer containing the value to be stored in the vector
+   */
+  public void set(int index, int isSet, int start, ArrowBuf buffer) {
+    if (isSet > 0) {
+      set(index, start, buffer);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param start start position of the value in the buffer
-    * @param buffer buffer containing the value to be stored in the vector
-    */
-   public void set(int index, int isSet, int start, ArrowBuf buffer) {
-      if (isSet > 0) {
-         set(index, start, buffer);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles
-    * the case when the position of new value is beyond the current value
-    * capacity of the vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param start start position of the value in the buffer
-    * @param buffer buffer containing the value to be stored in the vector
-    */
-   public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
-      handleSafe(index);
-      set(index, isSet, start, buffer);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableDecimalVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableDecimalVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision,
-                 NullableDecimalVector.this.scale);
-      }
-
-      public TransferImpl(NullableDecimalVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableDecimalVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles
+   * the case when the position of new value is beyond the current value
+   * capacity of the vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param start start position of the value in the buffer
+   * @param buffer buffer containing the value to be stored in the vector
+   */
+  public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
+    handleSafe(index);
+    set(index, isSet, start, buffer);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableDecimalVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableDecimalVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision,
+              NullableDecimalVector.this.scale);
+    }
+
+    public TransferImpl(NullableDecimalVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableDecimalVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
index 3ba5cfcea5251..834f16c211cdb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,336 +34,336 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableFloat4Vector extends BaseNullableFixedWidthVector {
-   public static final byte TYPE_WIDTH = 4;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableFloat4Vector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new Float4ReaderImpl(NullableFloat4Vector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.FLOAT4;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public float get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  public static final byte TYPE_WIDTH = 4;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableFloat4Vector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new Float4ReaderImpl(NullableFloat4Vector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.FLOAT4;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public float get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getFloat(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableFloat4Holder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getFloat(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Float getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getFloat(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableFloat4Holder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getFloat(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Float getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, float value) {
-      valueBuffer.setFloat(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, float value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, Float4Holder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, float value) {
+    valueBuffer.setFloat(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, float value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, float)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, float value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, Float4Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, Float4Holder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, Float4Holder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, float)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, float value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, Float4Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, Float4Holder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, float value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, float value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, float)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, float value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static float get(final ArrowBuf buffer, final int index) {
-      return buffer.getFloat(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableFloat4Vector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableFloat4Vector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableFloat4Vector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableFloat4Vector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableFloat4Vector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, float)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, float value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static float get(final ArrowBuf buffer, final int index) {
+    return buffer.getFloat(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableFloat4Vector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableFloat4Vector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableFloat4Vector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableFloat4Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableFloat4Vector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
index 2fb96a44fe78b..b054d9b05c06a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,336 +34,336 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableFloat8Vector extends BaseNullableFixedWidthVector {
-   public static final byte TYPE_WIDTH = 8;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableFloat8Vector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new Float8ReaderImpl(NullableFloat8Vector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.FLOAT8;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public double get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  public static final byte TYPE_WIDTH = 8;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableFloat8Vector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new Float8ReaderImpl(NullableFloat8Vector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.FLOAT8;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public double get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getDouble(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableFloat8Holder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getDouble(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Double getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getDouble(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableFloat8Holder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getDouble(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Double getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, double value) {
-      valueBuffer.setDouble(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, double value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, Float8Holder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, double value) {
+    valueBuffer.setDouble(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, double value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, double)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, double value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, Float8Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, Float8Holder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, Float8Holder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, double)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, double value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, Float8Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, Float8Holder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, double value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, double value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, double)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, double value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static double get(final ArrowBuf buffer, final int index) {
-      return buffer.getDouble(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableFloat8Vector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableFloat8Vector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableFloat8Vector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableFloat8Vector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableFloat8Vector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, double)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, double value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static double get(final ArrowBuf buffer, final int index) {
+    return buffer.getDouble(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableFloat8Vector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableFloat8Vector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableFloat8Vector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableFloat8Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableFloat8Vector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
index 93deacbdfdc74..e5dfbcedcf90e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,336 +34,346 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableIntVector extends BaseNullableFixedWidthVector {
-   public static final byte TYPE_WIDTH = 4;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableIntVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableIntVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableIntVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new IntReaderImpl(NullableIntVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.INT;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public int get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  public static final byte TYPE_WIDTH = 4;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableIntVector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name      name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableIntVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableIntVector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name      name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new IntReaderImpl(NullableIntVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   *
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   *
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.INT;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index position of element
+   * @return element at given index
+   */
+  public int get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index position of element
+   */
+  public void get(int index, NullableIntHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index position of element
+   * @return element at given index
+   */
+  public Integer getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableIntHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Integer getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableIntVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setInt(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableIntHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, IntHolder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   *
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from      source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableIntVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   *
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from      source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setInt(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index position of element
+   * @param value value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index  position of element
+   * @param holder nullable data holder for value of element
+   */
+  public void set(int index, NullableIntHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableIntHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, IntHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, IntHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index  position of element
+   * @param holder data holder for value of element
+   */
+  public void set(int index, IntHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index position of element
+   * @param value value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableIntHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index  position of element
+   * @param holder nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, IntHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index  position of element
+   * @param holder data holder for value of element
+   */
+  public void setSafe(int index, IntHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   *
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, int value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, int value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, int)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, int value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static int get(final ArrowBuf buffer, final int index) {
-      return buffer.getInt(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableIntVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableIntVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableIntVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableIntVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableIntVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, int)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   *
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, int value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   * <p>
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index  position of the element.
+   * @return value stored at the index.
+   */
+  public static int get(final ArrowBuf buffer, final int index) {
+    return buffer.getInt(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   *
+   * @param ref       name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   *
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableIntVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableIntVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableIntVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableIntVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableIntVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
index 174b722b04316..8d2fb5661d9eb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -36,394 +36,394 @@
  * vector are null.
  */
 public class NullableIntervalDayVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 8;
-   private static final byte MILLISECOND_OFFSET = 4;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableIntervalDayVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.INTERVALDAY;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public ArrowBuf get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         return null;
-      }
-      return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableIntervalDayHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
+  private static final byte TYPE_WIDTH = 8;
+  private static final byte MILLISECOND_OFFSET = 4;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableIntervalDayVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.INTERVALDAY;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public ArrowBuf get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      return null;
+    }
+    return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableIntervalDayHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    final int startIndex = index * TYPE_WIDTH;
+    holder.isSet = 1;
+    holder.days = valueBuffer.getInt(startIndex);
+    holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Period getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       final int startIndex = index * TYPE_WIDTH;
-      holder.isSet = 1;
-      holder.days = valueBuffer.getInt(startIndex);
-      holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Period getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         final int startIndex = index * TYPE_WIDTH;
-         final int days = valueBuffer.getInt(startIndex);
-         final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
-         final Period p = new Period();
-         return p.plusDays(days).plusMillis(milliseconds);
-      }
-   }
-
-   /**
-    * Get the Interval value at a given index as a {@link StringBuilder} object
-    * @param index position of the element
-    * @return String Builder object with Interval value as
-    *         [days, hours, minutes, seconds, millis]
-    */
-   public StringBuilder getAsStringBuilder(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      }else{
-         return getAsStringBuilderHelper(index);
-      }
-   }
-
-   private StringBuilder getAsStringBuilderHelper(int index) {
-      final int startIndex = index * TYPE_WIDTH;
-
-      final int  days = valueBuffer.getInt(startIndex);
-      int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
-
-      final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
-      millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
-
-      final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
-      millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
-
-      final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
-      millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
-
-      final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
-
-      return(new StringBuilder().
-              append(days).append(dayString).
-              append(hours).append(":").
-              append(minutes).append(":").
-              append(seconds).append(".").
-              append(millis));
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex);
-         from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer,
-                 thisIndex * TYPE_WIDTH, TYPE_WIDTH);
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, ArrowBuf value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index          position of element
-    * @param days           days for the interval
-    * @param milliseconds   milliseconds for the interval
-    */
-   public void set(int index, int days, int milliseconds){
-      final int offsetIndex = index * TYPE_WIDTH;
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      valueBuffer.setInt(offsetIndex, days);
-      valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         set(index, holder.days, holder.milliseconds);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, IntervalDayHolder holder){
+      final int days = valueBuffer.getInt(startIndex);
+      final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
+      final Period p = new Period();
+      return p.plusDays(days).plusMillis(milliseconds);
+    }
+  }
+
+  /**
+   * Get the Interval value at a given index as a {@link StringBuilder} object
+   * @param index position of the element
+   * @return String Builder object with Interval value as
+   *         [days, hours, minutes, seconds, millis]
+   */
+  public StringBuilder getAsStringBuilder(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return getAsStringBuilderHelper(index);
+    }
+  }
+
+  private StringBuilder getAsStringBuilderHelper(int index) {
+    final int startIndex = index * TYPE_WIDTH;
+
+    final int days = valueBuffer.getInt(startIndex);
+    int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET);
+
+    final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
+    millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
+
+    final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
+    millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
+
+    final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
+    millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
+
+    final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
+
+    return (new StringBuilder().
+            append(days).append(dayString).
+            append(hours).append(":").
+            append(minutes).append(":").
+            append(seconds).append(".").
+            append(millis));
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex);
+      from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer,
+              thisIndex * TYPE_WIDTH, TYPE_WIDTH);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, ArrowBuf value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index          position of element
+   * @param days           days for the interval
+   * @param milliseconds   milliseconds for the interval
+   */
+  public void set(int index, int days, int milliseconds) {
+    final int offsetIndex = index * TYPE_WIDTH;
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    valueBuffer.setInt(offsetIndex, days);
+    valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       set(index, holder.days, holder.milliseconds);
-   }
-
-   /**
-    * Same as {@link #set(int, ArrowBuf)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, ArrowBuf value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index          position of element
-    * @param days           days for the interval
-    * @param milliseconds   milliseconds for the interval
-    */
-   public void setSafe(int index, int days, int milliseconds) {
-      handleSafe(index);
-      set(index, days, milliseconds);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, IntervalDayHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, IntervalDayHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, IntervalDayHolder holder) {
+    set(index, holder.days, holder.milliseconds);
+  }
+
+  /**
+   * Same as {@link #set(int, ArrowBuf)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, ArrowBuf value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index          position of element
+   * @param days           days for the interval
+   * @param milliseconds   milliseconds for the interval
+   */
+  public void setSafe(int index, int days, int milliseconds) {
+    handleSafe(index);
+    set(index, days, milliseconds);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, IntervalDayHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, IntervalDayHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param days days component of interval
+   * @param milliseconds millisecond component of interval
+   */
+  public void set(int index, int isSet, int days, int milliseconds) {
+    if (isSet > 0) {
+      set(index, days, milliseconds);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param days days component of interval
-    * @param milliseconds millisecond component of interval
-    */
-   public void set(int index, int isSet, int days, int milliseconds) {
-      if (isSet > 0) {
-         set(index, days, milliseconds);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, int, int)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param days days component of interval
-    * @param milliseconds millisecond component of interval
-    */
-   public void setSafe(int index, int isSet, int days, int milliseconds) {
-      handleSafe(index);
-      set(index, isSet, days, milliseconds);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableIntervalDayVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableIntervalDayVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableIntervalDayVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableIntervalDayVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, int, int)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param days days component of interval
+   * @param milliseconds millisecond component of interval
+   */
+  public void setSafe(int index, int isSet, int days, int milliseconds) {
+    handleSafe(index);
+    set(index, isSet, days, milliseconds);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableIntervalDayVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableIntervalDayVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableIntervalDayVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableIntervalDayVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
index 303286d0ed30c..ac121d2f078ff 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,354 +34,354 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableIntervalYearVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 4;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableIntervalYearVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.INTERVALYEAR;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public int get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
-      return valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableIntervalYearHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Period getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         final int interval = get(index);
-         final int years  = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-         final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-         final Period p = new Period();
-         return p.plusYears(years).plusMonths(months);
-      }
-   }
-
-   /**
-    * Get the Interval value at a given index as a {@link StringBuilder} object
-    * @param index position of the element
-    * @return String Builder object with Interval value as
-    *         [years, months]
-    */
-   public StringBuilder getAsStringBuilder(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      }else{
-         return getAsStringBuilderHelper(index);
-      }
-   }
-
-   private StringBuilder getAsStringBuilderHelper(int index) {
-      int value  = valueBuffer.getInt(index * TYPE_WIDTH);
-
-      final int years  = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-      final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-
-      final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
-      final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
-
-      return(new StringBuilder().
-              append(years).append(yearString).
-              append(months).append(monthString));
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setInt(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, IntervalYearHolder holder){
+  private static final byte TYPE_WIDTH = 4;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableIntervalYearVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.INTERVALYEAR;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public int get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableIntervalYearHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Period getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      final int interval = valueBuffer.getInt(index * TYPE_WIDTH);
+      final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+      final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+      final Period p = new Period();
+      return p.plusYears(years).plusMonths(months);
+    }
+  }
+
+  /**
+   * Get the Interval value at a given index as a {@link StringBuilder} object
+   * @param index position of the element
+   * @return String Builder object with Interval value as
+   *         [years, months]
+   */
+  public StringBuilder getAsStringBuilder(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return getAsStringBuilderHelper(index);
+    }
+  }
+
+  private StringBuilder getAsStringBuilderHelper(int index) {
+    int value = valueBuffer.getInt(index * TYPE_WIDTH);
+
+    final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+    final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
+
+    final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
+    final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
+
+    return (new StringBuilder().
+            append(years).append(yearString).
+            append(months).append(monthString));
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setInt(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, IntervalYearHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, IntervalYearHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, IntervalYearHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, IntervalYearHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, IntervalYearHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, int value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, int value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, int)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, int value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableIntervalYearVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableIntervalYearVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableIntervalYearVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableIntervalYearVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, int)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, int value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableIntervalYearVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableIntervalYearVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableIntervalYearVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableIntervalYearVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
index ed337188b5b0b..5dd1492a3e095 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,363 +34,363 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableSmallIntVector extends BaseNullableFixedWidthVector {
-   public static final byte TYPE_WIDTH = 2;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableSmallIntVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new SmallIntReaderImpl(NullableSmallIntVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.SMALLINT;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public short get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  public static final byte TYPE_WIDTH = 2;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableSmallIntVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new SmallIntReaderImpl(NullableSmallIntVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.SMALLINT;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public short get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getShort(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableSmallIntHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getShort(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Short getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getShort(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableSmallIntHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getShort(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Short getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setShort(index * TYPE_WIDTH, value);
-   }
-
-   private void setValue(int index, short value) {
-      valueBuffer.setShort(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, short value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, SmallIntHolder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setShort(index * TYPE_WIDTH, value);
+  }
+
+  private void setValue(int index, short value) {
+    valueBuffer.setShort(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, short value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, short)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, short value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, SmallIntHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, SmallIntHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, SmallIntHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, short)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, short value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, SmallIntHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, SmallIntHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, short value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, short value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, short)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, short value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static short get(final ArrowBuf buffer, final int index) {
-      return buffer.getShort(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableSmallIntVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableSmallIntVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableSmallIntVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableSmallIntVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableSmallIntVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, short)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, short value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static short get(final ArrowBuf buffer, final int index) {
+    return buffer.getShort(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableSmallIntVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableSmallIntVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableSmallIntVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableSmallIntVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableSmallIntVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
index d7ea322239741..25da4f68d506c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -36,335 +36,335 @@
  * vector are null.
  */
 public class NullableTimeMicroVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 8;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeMicroVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMEMICRO;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public long get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  private static final byte TYPE_WIDTH = 8;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeMicroVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMEMICRO;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public long get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeMicroHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Long getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeMicroHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Long getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, long value) {
-      valueBuffer.setLong(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, long value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeMicroHolder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, long value) {
+    valueBuffer.setLong(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, long value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, long)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, long value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeMicroHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeMicroHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeMicroHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, long)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, long value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeMicroHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeMicroHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, long value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, long value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, long)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, long value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static long get(final ArrowBuf buffer, int index) {
-      return buffer.getLong(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeMicroVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableTimeMicroVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableTimeMicroVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableTimeMicroVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, long)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, long value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static long get(final ArrowBuf buffer, int index) {
+    return buffer.getLong(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeMicroVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableTimeMicroVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableTimeMicroVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableTimeMicroVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
index 07d8abba4e57b..7d2d5d1f10fe2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -36,336 +36,336 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeMilliVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 4;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeMilliVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMEMILLI;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public int get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
-      return valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeMilliHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public LocalDateTime getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      }
-      org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index),
-              org.joda.time.DateTimeZone.UTC);
-      return ldt;
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setInt(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeMilliHolder holder){
+  private static final byte TYPE_WIDTH = 4;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeMilliVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMEMILLI;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public int get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeMilliHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public LocalDateTime getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    }
+    org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index),
+            org.joda.time.DateTimeZone.UTC);
+    return ldt;
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setInt(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeMilliHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeMilliHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeMilliHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeMilliHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeMilliHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, int value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, int value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, int)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, int value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static int get(final ArrowBuf buffer, final int index) {
-      return buffer.getInt(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeMilliVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableTimeMilliVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableTimeMilliVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableTimeMilliVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, int)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, int value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static int get(final ArrowBuf buffer, final int index) {
+    return buffer.getInt(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeMilliVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableTimeMilliVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableTimeMilliVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableTimeMilliVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
index 947b2392f7a15..e93a63fe2a1f9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,335 +34,335 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeNanoVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 8;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeNanoVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMENANO;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public long get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  private static final byte TYPE_WIDTH = 8;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeNanoVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMENANO;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public long get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeNanoHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Long getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeNanoHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Long getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, long value) {
-      valueBuffer.setLong(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, long value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeNanoHolder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, long value) {
+    valueBuffer.setLong(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, long value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, long)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, long value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeNanoHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeNanoHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeNanoHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, long)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, long value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeNanoHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeNanoHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, long value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, long value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, long)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, long value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static long get(final ArrowBuf buffer,final int index) {
-      return buffer.getLong(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeNanoVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableTimeNanoVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableTimeNanoVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableTimeNanoVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, long)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, long value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static long get(final ArrowBuf buffer, final int index) {
+    return buffer.getLong(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeNanoVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableTimeNanoVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableTimeNanoVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableTimeNanoVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
index 0a3cfaac047eb..8329fa7813987 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,336 +34,336 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableTimeSecVector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 4;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeSecVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new TimeSecReaderImpl(NullableTimeSecVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESEC;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public int get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  private static final byte TYPE_WIDTH = 4;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeSecVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new TimeSecReaderImpl(NullableTimeSecVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESEC;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public int get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeSecHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Integer getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeSecHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Integer getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setInt(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeSecHolder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setInt(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeSecHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeSecHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeSecHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeSecHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeSecHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, int value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, int value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, int)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, int value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static int get(final ArrowBuf buffer, final int index) {
-      return buffer.getInt(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeSecVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableTimeSecVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableTimeSecVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableTimeSecVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableTimeSecVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, int)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, int value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static int get(final ArrowBuf buffer, final int index) {
+    return buffer.getInt(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeSecVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableTimeSecVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableTimeSecVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableTimeSecVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableTimeSecVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java
index 0d25eff4e02b7..c7ad141549bcb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,186 +34,184 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeStampMicroTZVector extends NullableTimeStampVector {
-   private final FieldReader reader;
-   private final String timeZone;
-
-   /**
-    * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) {
-      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, fieldType, allocator);
-      org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType();
-      timeZone = arrowType.getTimezone();
-      reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESTAMPMICROTZ;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeStampMicroTZHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Long getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeStampMicroTZHolder holder){
+  private final FieldReader reader;
+  private final String timeZone;
+
+  /**
+   * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) {
+    this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, fieldType, allocator);
+    org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType();
+    timeZone = arrowType.getTimezone();
+    reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESTAMPMICROTZ;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeStampMicroTZHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Long getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeStampMicroTZHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref,
-              field.getFieldType(), allocator);
-      return new TransferImpl(to);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeStampMicroTZVector)to);
-   }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeStampMicroTZHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeStampMicroTZHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref,
+            field.getFieldType(), allocator);
+    return new TransferImpl(to);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeStampMicroTZVector) to);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java
index 41f7a1112f2df..5a34b194ce77f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,187 +34,186 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeStampMicroVector extends NullableTimeStampVector {
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampMicroVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, fieldType, allocator);
-      reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESTAMPMICRO;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeStampMicroHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public LocalDateTime getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampMicroVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, fieldType, allocator);
+    reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESTAMPMICRO;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeStampMicroHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public LocalDateTime getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
          /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */
-         final long micros = get(index);
-         final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros);
-         final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
-         return localDateTime;
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeStampMicroHolder holder){
+      final long micros = valueBuffer.getLong(index * TYPE_WIDTH);
+      final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros);
+      final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis,
+              org.joda.time.DateTimeZone.UTC);
+      return localDateTime;
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeStampMicroHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref,
-              field.getFieldType(), allocator);
-      return new TransferImpl(to);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeStampMicroVector )to);
-   }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeStampMicroHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeStampMicroHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref,
+            field.getFieldType(), allocator);
+    return new TransferImpl(to);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeStampMicroVector) to);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java
index 6376b085e64e1..f9736aed7be62 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,185 +34,183 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeStampMilliTZVector extends NullableTimeStampVector {
-   private final FieldReader reader;
-   private final String timeZone;
-
-   /**
-    * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) {
-      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, fieldType, allocator);
-      org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType();
-      timeZone = arrowType.getTimezone();
-      reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESTAMPMILLITZ;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeStampMilliTZHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Long getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeStampMilliTZHolder holder){
+  private final FieldReader reader;
+  private final String timeZone;
+
+  /**
+   * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) {
+    this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, fieldType, allocator);
+    org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType();
+    timeZone = arrowType.getTimezone();
+    reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESTAMPMILLITZ;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeStampMilliTZHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Long getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeStampMilliTZHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref,
-              field.getFieldType(), allocator);
-      return new TransferImpl(to);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeStampMilliTZVector)to);
-   }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeStampMilliTZHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeStampMilliTZHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref,
+            field.getFieldType(), allocator);
+    return new TransferImpl(to);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeStampMilliTZVector) to);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java
index ff581d2ea2f03..fc38a6bd507de 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,185 +34,184 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeStampMilliVector extends NullableTimeStampVector {
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampMilliVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, fieldType, allocator);
-      reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESTAMPMILLI;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeStampMilliHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public LocalDateTime getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         final long millis = get(index);
-         final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
-         return localDateTime;
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeStampMilliHolder holder){
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampMilliVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, fieldType, allocator);
+    reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESTAMPMILLI;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeStampMilliHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public LocalDateTime getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      final long millis = valueBuffer.getLong(index * TYPE_WIDTH);
+      final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis,
+              org.joda.time.DateTimeZone.UTC);
+      return localDateTime;
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeStampMilliHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref,
-              field.getFieldType(), allocator);
-      return new TransferImpl(to);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeStampMilliVector)to);
-   }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeStampMilliHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeStampMilliHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref,
+            field.getFieldType(), allocator);
+    return new TransferImpl(to);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeStampMilliVector) to);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java
index 6c4445d500b39..8eb48462c994b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,186 +34,184 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeStampNanoTZVector extends NullableTimeStampVector {
-   private final FieldReader reader;
-   private final String timeZone;
-
-   /**
-    * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) {
-      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, fieldType, allocator);
-      org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType();
-      timeZone = arrowType.getTimezone();
-      reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESTAMPNANOTZ;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeStampNanoTZHolder holder) {
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Long getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeStampNanoTZHolder holder){
+  private final FieldReader reader;
+  private final String timeZone;
+
+  /**
+   * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) {
+    this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, fieldType, allocator);
+    org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType();
+    timeZone = arrowType.getTimezone();
+    reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESTAMPNANOTZ;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeStampNanoTZHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Long getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeStampNanoTZHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref,
-              field.getFieldType(), allocator);
-      return new TransferImpl(to);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeStampNanoTZVector)to);
-   }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeStampNanoTZHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeStampNanoTZHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref,
+            field.getFieldType(), allocator);
+    return new TransferImpl(to);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeStampNanoTZVector) to);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java
index 096913ebf1cf0..6cc5c905783d7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,186 +34,185 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeStampNanoVector extends NullableTimeStampVector {
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampNanoVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, fieldType, allocator);
-      reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESTAMPNANO;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeStampNanoHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public LocalDateTime getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         final long nanos = get(index);
-         final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos);
-         final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
-         return localDateTime;
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeStampNanoHolder holder){
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampNanoVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, fieldType, allocator);
+    reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESTAMPNANO;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeStampNanoHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public LocalDateTime getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      final long nanos = valueBuffer.getLong(index * TYPE_WIDTH);
+      final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos);
+      final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis,
+              org.joda.time.DateTimeZone.UTC);
+      return localDateTime;
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeStampNanoHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref,
-              field.getFieldType(), allocator);
-      return new TransferImpl(to);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeStampNanoVector)to);
-   }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeStampNanoHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeStampNanoHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref,
+            field.getFieldType(), allocator);
+    return new TransferImpl(to);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeStampNanoVector) to);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java
index 8e5ba72ae1fe2..0593ac76dce57 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -33,185 +33,183 @@
  * (bit vector) is maintained to track which elements in the vector are null.
  */
 public class NullableTimeStampSecTZVector extends NullableTimeStampVector {
-   private final FieldReader reader;
-   private final String timeZone;
-
-   /**
-    * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) {
-      this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, fieldType, allocator);
-      org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType();
-      timeZone = arrowType.getTimezone();
-      reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESTAMPSECTZ;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeStampSecTZHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Long getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeStampSecTZHolder holder){
+  private final FieldReader reader;
+  private final String timeZone;
+
+  /**
+   * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) {
+    this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, fieldType, allocator);
+    org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType();
+    timeZone = arrowType.getTimezone();
+    reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESTAMPSECTZ;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeStampSecTZHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Long getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return valueBuffer.getLong(index * TYPE_WIDTH);
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeStampSecTZHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref,
-              field.getFieldType(), allocator);
-      return new TransferImpl(to);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeStampSecTZVector)to);
-   }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeStampSecTZHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeStampSecTZHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref,
+            field.getFieldType(), allocator);
+    return new TransferImpl(to);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeStampSecTZVector) to);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java
index bc800e0e4f04f..1afdb30d0453e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,186 +34,185 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableTimeStampSecVector extends NullableTimeStampVector {
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampSecVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, fieldType, allocator);
-      reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader() {
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TIMESTAMPSEC;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTimeStampSecHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public LocalDateTime getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         final long secs = get(index);
-         final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs);
-         final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
-         return localDateTime;
-      }
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TimeStampSecHolder holder){
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampSecVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, fieldType, allocator);
+    reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TIMESTAMPSEC;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTimeStampSecHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public LocalDateTime getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      final long secs = valueBuffer.getLong(index * TYPE_WIDTH);
+      final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs);
+      final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis,
+              org.joda.time.DateTimeZone.UTC);
+      return localDateTime;
+    }
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TimeStampSecHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref,
-              field.getFieldType(), allocator);
-      return new TransferImpl(to);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTimeStampSecVector)to);
-   }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TimeStampSecHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TimeStampSecHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref,
+            field.getFieldType(), allocator);
+    return new TransferImpl(to);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTimeStampSecVector) to);
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
index 1bf2abc384d92..d65cb83e15123 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -29,189 +29,191 @@
  * maintained to track which elements in the vector are null.
  */
 public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector {
-   protected static final byte TYPE_WIDTH = 8;
-
-   /**
-    * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public long get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
-      return valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   protected void setValue(int index, long value) {
-      valueBuffer.setLong(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, long value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, long)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, long value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+  protected static final byte TYPE_WIDTH = 8;
+
+  /**
+   * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public long get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  protected void setValue(int index, long value) {
+    valueBuffer.setLong(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, long value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, long)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, long value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, long value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, long value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, long)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, long value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static long get(final ArrowBuf buffer, final int index) {
-      return buffer.getLong(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   public class TransferImpl implements TransferPair {
-      NullableTimeStampVector to;
-
-      public TransferImpl(NullableTimeStampVector to) {
-         this.to = to;
-      }
-
-      @Override
-      public NullableTimeStampVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer() {
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, long)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, long value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static long get(final ArrowBuf buffer, final int index) {
+    return buffer.getLong(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  public class TransferImpl implements TransferPair {
+    NullableTimeStampVector to;
+
+    public TransferImpl(NullableTimeStampVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableTimeStampVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
index ccbfa32b85f0c..106359ddd9666 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,363 +34,363 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableTinyIntVector extends BaseNullableFixedWidthVector {
-   public static final byte TYPE_WIDTH = 1;
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTinyIntVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()),
-              allocator);
-   }
-
-   /**
-    * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new TinyIntReaderImpl(NullableTinyIntVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.TINYINT;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public byte get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  public static final byte TYPE_WIDTH = 1;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTinyIntVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()),
+            allocator);
+  }
+
+  /**
+   * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new TinyIntReaderImpl(NullableTinyIntVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.TINYINT;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public byte get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getByte(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableTinyIntHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getByte(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Byte getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getByte(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableTinyIntHolder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getByte(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Byte getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-   private void setValue(int index, int value) {
-      valueBuffer.setByte(index * TYPE_WIDTH, value);
-   }
-
-   private void setValue(int index, byte value) {
-      valueBuffer.setByte(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, byte value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, TinyIntHolder holder){
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+  private void setValue(int index, int value) {
+    valueBuffer.setByte(index * TYPE_WIDTH, value);
+  }
+
+  private void setValue(int index, byte value) {
+    valueBuffer.setByte(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, byte value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, byte)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, byte value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, TinyIntHolder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, TinyIntHolder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, TinyIntHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, byte)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, byte value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, TinyIntHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, TinyIntHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, byte value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void set(int index, int isSet, byte value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Same as {@link #set(int, int, byte)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param value element value
-    */
-   public void setSafe(int index, int isSet, byte value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-   /**
-    * Given a data buffer, get the value stored at a particular position
-    * in the vector.
-    *
-    * This method should not be used externally.
-    *
-    * @param buffer data buffer
-    * @param index position of the element.
-    * @return value stored at the index.
-    */
-   public static byte get(final ArrowBuf buffer, final int index) {
-      return buffer.getByte(index * TYPE_WIDTH);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableTinyIntVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableTinyIntVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableTinyIntVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableTinyIntVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableTinyIntVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this);
-      }
-   }
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, int, byte)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, byte value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static byte get(final ArrowBuf buffer, final int index) {
+    return buffer.getByte(index * TYPE_WIDTH);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableTinyIntVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableTinyIntVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableTinyIntVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableTinyIntVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableTinyIntVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
index 4d20f800ba1fd..52f6c151bdff9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -33,286 +33,286 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableUInt1Vector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 1;
-   private final FieldReader reader;
-
-   public NullableUInt1Vector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()),
-              allocator);
-   }
-
-   public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new UInt1ReaderImpl(NullableUInt1Vector.this);
-   }
-
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.UINT1;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public byte get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  private static final byte TYPE_WIDTH = 1;
+  private final FieldReader reader;
+
+  public NullableUInt1Vector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()),
+            allocator);
+  }
+
+  public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new UInt1ReaderImpl(NullableUInt1Vector.this);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.UINT1;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public byte get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getByte(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableUInt1Holder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getByte(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Byte getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getByte(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableUInt1Holder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getByte(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Byte getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setByte(index * TYPE_WIDTH, value);
-   }
-
-   private void setValue(int index, byte value) {
-      valueBuffer.setByte(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, byte value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, UInt1Holder holder){
+    }
+  }
+
+  public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setByte(index * TYPE_WIDTH, value);
+  }
+
+  private void setValue(int index, byte value) {
+    valueBuffer.setByte(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, byte value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, byte)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, byte value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, UInt1Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, UInt1Holder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, UInt1Holder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, byte)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, byte value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, UInt1Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, UInt1Holder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  public void set(int index, int isSet, byte value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   public void set(int index, int isSet, byte value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   public void setSafe(int index, int isSet, byte value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableUInt1Vector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableUInt1Vector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableUInt1Vector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableUInt1Vector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableUInt1Vector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this);
-      }
-   }
+    }
+  }
+
+  public void setSafe(int index, int isSet, byte value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableUInt1Vector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableUInt1Vector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableUInt1Vector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableUInt1Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableUInt1Vector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
index b140b9f5cbc67..42ba6e139bf71 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,286 +34,286 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableUInt2Vector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 2;
-   private final FieldReader reader;
-
-   public NullableUInt2Vector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()),
-              allocator);
-   }
-
-   public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new UInt2ReaderImpl(NullableUInt2Vector.this);
-   }
-
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.UINT2;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public char get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  private static final byte TYPE_WIDTH = 2;
+  private final FieldReader reader;
+
+  public NullableUInt2Vector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()),
+            allocator);
+  }
+
+  public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new UInt2ReaderImpl(NullableUInt2Vector.this);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.UINT2;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public char get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getChar(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableUInt2Holder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getChar(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Character getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getChar(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableUInt2Holder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getChar(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Character getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setChar(index * TYPE_WIDTH, value);
-   }
-
-   private void setValue(int index, char value) {
-      valueBuffer.setChar(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, char value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, UInt2Holder holder){
+    }
+  }
+
+  public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setChar(index * TYPE_WIDTH, value);
+  }
+
+  private void setValue(int index, char value) {
+    valueBuffer.setChar(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, char value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, char)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, char value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, UInt2Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, UInt2Holder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, UInt2Holder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, char)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, char value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, UInt2Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, UInt2Holder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  public void set(int index, int isSet, char value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   public void set(int index, int isSet, char value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   public void setSafe(int index, int isSet, char value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableUInt2Vector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableUInt2Vector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableUInt2Vector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableUInt2Vector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableUInt2Vector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this);
-      }
-   }
+    }
+  }
+
+  public void setSafe(int index, int isSet, char value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableUInt2Vector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableUInt2Vector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableUInt2Vector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableUInt2Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableUInt2Vector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
index fc8cb639581fb..1ec21490d8447 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,258 +34,258 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableUInt4Vector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 4;
-   private final FieldReader reader;
-
-   public NullableUInt4Vector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()),
-              allocator);
-   }
-
-   public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new UInt4ReaderImpl(NullableUInt4Vector.this);
-   }
-
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.UINT4;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public int get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  private static final byte TYPE_WIDTH = 4;
+  private final FieldReader reader;
+
+  public NullableUInt4Vector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()),
+            allocator);
+  }
+
+  public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new UInt4ReaderImpl(NullableUInt4Vector.this);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.UINT4;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public int get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableUInt4Holder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Integer getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableUInt4Holder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getInt(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Integer getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, int value) {
-      valueBuffer.setInt(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, int value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, UInt4Holder holder){
+    }
+  }
+
+  public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, int value) {
+    valueBuffer.setInt(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, int)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, int value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, UInt4Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, UInt4Holder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, UInt4Holder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, UInt4Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, UInt4Holder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  public void set(int index, int isSet, int value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   public void set(int index, int isSet, int value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   public void setSafe(int index, int isSet, int value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableUInt4Vector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableUInt4Vector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableUInt4Vector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableUInt4Vector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableUInt4Vector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this);
-      }
-   }
+    }
+  }
+
+  public void setSafe(int index, int isSet, int value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableUInt4Vector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableUInt4Vector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableUInt4Vector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableUInt4Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableUInt4Vector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
index 2110cf03b9f05..408a810cfa0af 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,258 +34,258 @@
  * maintained to track which elements in the vector are null.
  */
 public class NullableUInt8Vector extends BaseNullableFixedWidthVector {
-   private static final byte TYPE_WIDTH = 8;
-   private final FieldReader reader;
-
-   public NullableUInt8Vector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.UINT8.getType()),
-              allocator);
-   }
-
-   public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType, TYPE_WIDTH);
-      reader = new UInt8ReaderImpl(NullableUInt8Vector.this);
-   }
-
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.UINT8;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value retrieval methods                        *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the element at the given index from the vector.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public long get(int index) throws IllegalStateException {
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
+  private static final byte TYPE_WIDTH = 8;
+  private final FieldReader reader;
+
+  public NullableUInt8Vector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.UINT8.getType()),
+            allocator);
+  }
+
+  public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, TYPE_WIDTH);
+    reader = new UInt8ReaderImpl(NullableUInt8Vector.this);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.UINT8;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public long get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    return valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index   position of element
+   */
+  public void get(int index, NullableUInt8Holder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index   position of element
+   * @return element at given index
+   */
+  public Long getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
       return valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Get the element at the given index from the vector and
-    * sets the state in holder. If element at given index
-    * is null, holder.isSet will be zero.
-    *
-    * @param index   position of element
-    */
-   public void get(int index, NullableUInt8Holder holder){
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      holder.isSet = 1;
-      holder.value = valueBuffer.getLong(index * TYPE_WIDTH);
-   }
-
-   /**
-    * Same as {@link #get(int)}.
-    *
-    * @param index   position of element
-    * @return element at given index
-    */
-   public Long getObject(int index) {
-      if (isSet(index) == 0) {
-         return null;
-      } else {
-         return get(index);
-      }
-   }
-
-   public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) {
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-      }
-   }
-
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) {
-      handleSafe(thisIndex);
-      copyFrom(fromIndex, thisIndex, from);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   private void setValue(int index, long value) {
-      valueBuffer.setLong(index * TYPE_WIDTH, value);
-   }
-
-   /**
-    * Set the element at the given index to the given value.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void set(int index, long value) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setValue(index, value);
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    * If the value in holder is not indicated as set, element in the
-    * at the given index will be null.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException {
-      if(holder.isSet < 0) {
-         throw new IllegalArgumentException();
-      }
-      else if(holder.isSet > 0) {
-         BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-         setValue(index, holder.value);
-      }
-      else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   /**
-    * Set the element at the given index to the value set in data holder.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void set(int index, UInt8Holder holder){
+    }
+  }
+
+  public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) {
+    if (from.isSet(fromIndex) != 0) {
+      set(thisIndex, from.get(fromIndex));
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
+    }
+  }
+
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  private void setValue(int index, long value) {
+    valueBuffer.setLong(index * TYPE_WIDTH, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void set(int index, long value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, value);
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
       BitVectorHelper.setValidityBitToOne(validityBuffer, index);
       setValue(index, holder.value);
-   }
-
-   /**
-    * Same as {@link #set(int, long)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param value   value of element
-    */
-   public void setSafe(int index, long value) {
-      handleSafe(index);
-      set(index, value);
-   }
-
-   /**
-    * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  nullable data holder for value of element
-    */
-   public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException {
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Same as {@link #set(int, UInt8Holder)} except that it handles the
-    * case when index is greater than or equal to existing
-    * value capacity {@link #getValueCapacity()}.
-    *
-    * @param index   position of element
-    * @param holder  data holder for value of element
-    */
-   public void setSafe(int index, UInt8Holder holder){
-      handleSafe(index);
-      set(index, holder);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void set(int index, UInt8Holder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    setValue(index, holder.value);
+  }
+
+  /**
+   * Same as {@link #set(int, long)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param value   value of element
+   */
+  public void setSafe(int index, long value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  /**
+   * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Same as {@link #set(int, UInt8Holder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index   position of element
+   * @param holder  data holder for value of element
+   */
+  public void setSafe(int index, UInt8Holder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index   position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
       /* not really needed to set the bit to 0 as long as
        * the buffer always starts from 0.
        */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  public void set(int index, int isSet, long value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
       BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   public void set(int index, int isSet, long value) {
-      if (isSet > 0) {
-         set(index, value);
-      } else {
-         BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-      }
-   }
-
-   public void setSafe(int index, int isSet, long value) {
-      handleSafe(index);
-      set(index, isSet, value);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableUInt8Vector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableUInt8Vector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableUInt8Vector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableUInt8Vector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableUInt8Vector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this);
-      }
-   }
+    }
+  }
+
+  public void setSafe(int index, int isSet, long value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableUInt8Vector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableUInt8Vector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableUInt8Vector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableUInt8Vector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableUInt8Vector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this);
+    }
+  }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java
index 330e06710fcdd..1dfe8912c52f9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -36,479 +36,293 @@
  * to track which elements in the vector are null.
  */
 public class NullableVarBinaryVector extends BaseNullableVariableWidthVector {
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableVarBinaryVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator);
-   }
-
-   /**
-    * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType);
-      reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.VARBINARY;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value getter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the variable length element at specified index as byte array.
-    *
-    * @param index   position of element to get
-    * @return array of bytes for non-null element, null otherwise
-    */
-   public byte[] get(int index) {
-      assert index >= 0;
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
-      final int startOffset = getstartOffset(index);
-      final int dataLength =
-              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
-      final byte[] result = new byte[dataLength];
-      valueBuffer.getBytes(startOffset, result, 0, dataLength);
-      return result;
-   }
-
-   /**
-    * Get the variable length element at specified index as Text.
-    *
-    * @param index   position of element to get
-    * @return byte array for non-null element, null otherwise
-    */
-   public byte[] getObject(int index) {
-      byte[] b;
-      try {
-         b = get(index);
-      } catch (IllegalStateException e) {
-         return null;
-      }
-      return b;
-   }
-
-   /**
-    * Get the variable length element at specified index as Text.
-    *
-    * @param index   position of element to get
-    * @return greater than 0 length for non-null element, 0 otherwise
-    */
-   public int getValueLength(int index) {
-      assert index >= 0;
-      if(isSet(index) == 0) {
-         return 0;
-      }
-      final int startOffset = getstartOffset(index);
-      final int dataLength =
-              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
-      return dataLength;
-   }
-
-   /**
-    * Get the variable length element at specified index and sets the state
-    * in provided holder.
-    *
-    * @param index   position of element to get
-    * @param holder  data holder to be populated by this function
-    */
-   public void get(int index, NullableVarBinaryHolder holder){
-      assert index >= 0;
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      final int startOffset = getstartOffset(index);
-      final int dataLength =
-              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
-      holder.isSet = 1;
-      holder.start = startOffset;
-      holder.end = dataLength;
-      holder.buffer = valueBuffer;
-   }
-
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) {
-      fillHoles(thisIndex);
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-         lastSet = thisIndex;
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) {
-      fillEmpties(thisIndex);
-      if (from.isSet(fromIndex) != 0) {
-         setSafe(thisIndex, from.get(fromIndex));
-         lastSet = thisIndex;
-      }
-   }
-
-
-   /**
-    * Set the variable length element at the specified index to the supplied
-    * byte array. This is same as using {@link #set(int, byte[], int, int)}
-    * with start as 0 and length as value.length
-    *
-    * @param index   position of the element to set
-    * @param value   array of bytes to write
-    */
-   public void set(int index, byte[] value) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setBytes(index, value, 0, value.length);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, byte[])} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param value   array of bytes to write
-    */
-   public void setSafe(int index, byte[] value) {
-      assert index >= 0;
-      fillEmpties(index);
-      handleSafe(index, value.length);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setBytes(index, value, 0, value.length);
-      lastSet = index;
-   }
-
-   /**
-    * Set the variable length element at the specified index to the supplied
-    * byte array.
-    *
-    * @param index   position of the element to set
-    * @param value   array of bytes to write
-    * @param start   start index in array of bytes
-    * @param length  length of data in array of bytes
-    */
-   public void set(int index, byte[] value, int start, int length) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setBytes(index, value, start, length);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, byte[], int, int)} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param value   array of bytes to write
-    * @param start   start index in array of bytes
-    * @param length  length of data in array of bytes
-    */
-   public void setSafe(int index, byte[] value, int start, int length) {
-      assert index >= 0;
-      fillEmpties(index);
-      handleSafe(index, length);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setBytes(index, value, start, length);
-      lastSet = index;
-   }
-
-   /**
-    * Set the variable length element at the specified index to the
-    * content in supplied ByteBuffer
-    *
-    * @param index   position of the element to set
-    * @param value   ByteBuffer with data
-    * @param start   start index in ByteBuffer
-    * @param length  length of data in ByteBuffer
-    */
-   public void set(int index, ByteBuffer value, int start, int length) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
-      valueBuffer.setBytes(startOffset, value, start, length);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param value   ByteBuffer with data
-    * @param start   start index in ByteBuffer
-    * @param length  length of data in ByteBuffer
-    */
-   public void setSafe(int index, ByteBuffer value, int start, int length) {
-      assert index >= 0;
-      fillEmpties(index);
-      handleSafe(index, length);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
-      valueBuffer.setBytes(startOffset, value, start, length);
-      lastSet = index;
-   }
-
-   /**
-    * Set the variable length element at the specified index to the data
-    * buffer supplied in the holder
-    *
-    * @param index   position of the element to set
-    * @param holder  holder that carries data buffer.
-    */
-   public void set(int index, VarBinaryHolder holder) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      final int dataLength = holder.end - holder.start;
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
-      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, VarBinaryHolder)} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param holder  holder that carries data buffer.
-    */
-   public void setSafe(int index, VarBinaryHolder holder) {
-      assert index >= 0;
-      final int dataLength = holder.end - holder.start;
-      fillEmpties(index);
-      handleSafe(index, dataLength);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
-      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
-      lastSet = index;
-   }
-
-   /**
-    * Set the variable length element at the specified index to the data
-    * buffer supplied in the holder
-    *
-    * @param index   position of the element to set
-    * @param holder  holder that carries data buffer.
-    */
-   public void set(int index, NullableVarBinaryHolder holder) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
-      final int dataLength = holder.end - holder.start;
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
-      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param holder  holder that carries data buffer.
-    */
-   public void setSafe(int index, NullableVarBinaryHolder holder) {
-      assert index >= 0;
-      final int dataLength = holder.end - holder.start;
-      fillEmpties(index);
-      handleSafe(index, dataLength);
-      BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
-      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
-      lastSet = index;
-   }
-
-   /**
-    * Sets the value length for an element.
-    *
-    * @param index   position of the element to set
-    * @param length  length of the element
-    */
-   public void setValueLengthSafe(int index, int length) {
-      assert index >= 0;
-      handleSafe(index, length);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index, 0);
-      /* not really needed to set the bit to 0 as long as
-       * the buffer always starts from 0.
-       */
-      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param start start position of data in buffer
-    * @param end end position of data in buffer
-    * @param buffer data buffer containing the variable width element to be stored
-    *               in the vector
-    */
-   public void set(int index, int isSet, int start, int end, ArrowBuf buffer) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBit(validityBuffer, index, isSet);
-      final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end);
-      final ArrowBuf bb = buffer.slice(start, end);
-      valueBuffer.setBytes(startOffset, bb);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param start start position of data in buffer
-    * @param end end position of data in buffer
-    * @param buffer data buffer containing the variable width element to be stored
-    *               in the vector
-    */
-   public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) {
-      assert index >= 0;
-      handleSafe(index, end);
-      set(index, isSet, start, end, buffer);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableVarBinaryVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableVarBinaryVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableVarBinaryVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableVarBinaryVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this);
-      }
-   }
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableVarBinaryVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator);
+  }
+
+  /**
+   * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType);
+    reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.VARBINARY;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value getter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the variable length element at specified index as byte array.
+   *
+   * @param index   position of element to get
+   * @return array of bytes for non-null element, null otherwise
+   */
+  public byte[] get(int index) {
+    assert index >= 0;
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    final int startOffset = getstartOffset(index);
+    final int dataLength =
+            offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+    final byte[] result = new byte[dataLength];
+    valueBuffer.getBytes(startOffset, result, 0, dataLength);
+    return result;
+  }
+
+  /**
+   * Get the variable length element at specified index as Text.
+   *
+   * @param index   position of element to get
+   * @return byte array for non-null element, null otherwise
+   */
+  public byte[] getObject(int index) {
+    byte[] b;
+    try {
+      b = get(index);
+    } catch (IllegalStateException e) {
+      return null;
+    }
+    return b;
+  }
+
+  /**
+   * Get the variable length element at specified index and sets the state
+   * in provided holder.
+   *
+   * @param index   position of element to get
+   * @param holder  data holder to be populated by this function
+   */
+  public void get(int index, NullableVarBinaryHolder holder) {
+    assert index >= 0;
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.start = getstartOffset(index);
+    holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH);
+    holder.buffer = valueBuffer;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) {
+    final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH);
+    final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH);
+    final int length = end - start;
+    fillHoles(thisIndex);
+    BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH);
+    from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length);
+    offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length);
+    lastSet = thisIndex;
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) {
+    final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH);
+    final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH);
+    final int length = end - start;
+    handleSafe(thisIndex, length);
+    fillHoles(thisIndex);
+    BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH);
+    from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length);
+    offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length);
+    lastSet = thisIndex;
+  }
+
+  /**
+   * Set the variable length element at the specified index to the data
+   * buffer supplied in the holder
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void set(int index, VarBinaryHolder holder) {
+    assert index >= 0;
+    fillHoles(index);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int dataLength = holder.end - holder.start;
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, VarBinaryHolder)} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void setSafe(int index, VarBinaryHolder holder) {
+    assert index >= 0;
+    final int dataLength = holder.end - holder.start;
+    fillEmpties(index);
+    handleSafe(index, dataLength);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+    lastSet = index;
+  }
+
+  /**
+   * Set the variable length element at the specified index to the data
+   * buffer supplied in the holder
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void set(int index, NullableVarBinaryHolder holder) {
+    assert index >= 0;
+    fillHoles(index);
+    BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
+    final int dataLength = holder.end - holder.start;
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void setSafe(int index, NullableVarBinaryHolder holder) {
+    assert index >= 0;
+    final int dataLength = holder.end - holder.start;
+    fillEmpties(index);
+    handleSafe(index, dataLength);
+    BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+    lastSet = index;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableVarBinaryVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableVarBinaryVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableVarBinaryVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableVarBinaryVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this);
+    }
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
index 0b03164f0f385..31e409cc6263f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,481 +37,295 @@
  * to track which elements in the vector are null.
  */
 public class NullableVarCharVector extends BaseNullableVariableWidthVector {
-   private final FieldReader reader;
-
-   /**
-    * Instantiate a NullableVarCharVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableVarCharVector(String name, BufferAllocator allocator) {
-      this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator);
-   }
-
-   /**
-    * Instantiate a NullableVarCharVector. This doesn't allocate any memory for
-    * the data in vector.
-    * @param name name of the vector
-    * @param fieldType type of Field materialized by this vector
-    * @param allocator allocator for memory management.
-    */
-   public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) {
-      super(name, allocator, fieldType);
-      reader = new VarCharReaderImpl(NullableVarCharVector.this);
-   }
-
-   /**
-    * Get a reader that supports reading values from this vector
-    * @return Field Reader for this vector
-    */
-   @Override
-   public FieldReader getReader(){
-      return reader;
-   }
-
-   /**
-    * Get minor type for this vector. The vector holds values belonging
-    * to a particular type.
-    * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-    */
-   @Override
-   public Types.MinorType getMinorType() {
-      return Types.MinorType.VARCHAR;
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value getter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Get the variable length element at specified index as byte array.
-    *
-    * @param index   position of element to get
-    * @return array of bytes for non-null element, null otherwise
-    */
-   public byte[] get(int index) {
-      assert index >= 0;
-      if(isSet(index) == 0) {
-         throw new IllegalStateException("Value at index is null");
-      }
-      final int startOffset = getstartOffset(index);
-      final int dataLength =
-              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
-      final byte[] result = new byte[dataLength];
-      valueBuffer.getBytes(startOffset, result, 0, dataLength);
-      return result;
-   }
-
-   /**
-    * Get the variable length element at specified index as Text.
-    *
-    * @param index   position of element to get
-    * @return Text object for non-null element, null otherwise
-    */
-   public Text getObject(int index) {
-      Text result = new Text();
-      byte[] b;
-      try {
-         b = get(index);
-      } catch (IllegalStateException e) {
-         return null;
-      }
-      result.set(b);
-      return result;
-   }
-
-   /**
-    * Get the variable length element at specified index as Text.
-    *
-    * @param index   position of element to get
-    * @return greater than 0 length for non-null element, 0 otherwise
-    */
-   public int getValueLength(int index) {
-      assert index >= 0;
-      if(isSet(index) == 0) {
-         return 0;
-      }
-      final int startOffset = getstartOffset(index);
-      final int dataLength =
-              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
-      return dataLength;
-   }
-
-   /**
-    * Get the variable length element at specified index and sets the state
-    * in provided holder.
-    *
-    * @param index   position of element to get
-    * @param holder  data holder to be populated by this function
-    */
-   public void get(int index, NullableVarCharHolder holder){
-      assert index >= 0;
-      if(isSet(index) == 0) {
-         holder.isSet = 0;
-         return;
-      }
-      final int startOffset = getstartOffset(index);
-      final int dataLength =
-              offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
-      holder.isSet = 1;
-      holder.start = startOffset;
-      holder.end = dataLength;
-      holder.buffer = valueBuffer;
-   }
-
-
-
-   /******************************************************************
-    *                                                                *
-    *          vector value setter methods                           *
-    *                                                                *
-    ******************************************************************/
-
-
-   /**
-    * Copy a cell value from a particular index in source vector to a particular
-    * position in this vector
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) {
-      fillHoles(thisIndex);
-      if (from.isSet(fromIndex) != 0) {
-         set(thisIndex, from.get(fromIndex));
-         lastSet = thisIndex;
-      }
-   }
-
-   /**
-    * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that
-    * it handles the case when the capacity of the vector needs to be expanded
-    * before copy.
-    * @param fromIndex position to copy from in source vector
-    * @param thisIndex position to copy to in this vector
-    * @param from source vector
-    */
-   public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) {
-      fillEmpties(thisIndex);
-      if (from.isSet(fromIndex) != 0) {
-         setSafe(thisIndex, from.get(fromIndex));
-         lastSet = thisIndex;
-      }
-   }
-
-
-   /**
-    * Set the variable length element at the specified index to the supplied
-    * byte array. This is same as using {@link #set(int, byte[], int, int)}
-    * with start as 0 and length as value.length
-    *
-    * @param index   position of the element to set
-    * @param value   array of bytes to write
-    */
-   public void set(int index, byte[] value) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setBytes(index, value, 0, value.length);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, byte[])} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param value   array of bytes to write
-    */
-   public void setSafe(int index, byte[] value) {
-      assert index >= 0;
-      fillEmpties(index);
-      handleSafe(index, value.length);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setBytes(index, value, 0, value.length);
-      lastSet = index;
-   }
-
-   /**
-    * Set the variable length element at the specified index to the supplied
-    * byte array.
-    *
-    * @param index   position of the element to set
-    * @param value   array of bytes to write
-    * @param start   start index in array of bytes
-    * @param length  length of data in array of bytes
-    */
-   public void set(int index, byte[] value, int start, int length) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setBytes(index, value, start, length);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, byte[], int, int)} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param value   array of bytes to write
-    * @param start   start index in array of bytes
-    * @param length  length of data in array of bytes
-    */
-   public void setSafe(int index, byte[] value, int start, int length) {
-      assert index >= 0;
-      fillEmpties(index);
-      handleSafe(index, length);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      setBytes(index, value, start, length);
-      lastSet = index;
-   }
-
-   /**
-    * Set the variable length element at the specified index to the
-    * content in supplied ByteBuffer
-    *
-    * @param index   position of the element to set
-    * @param value   ByteBuffer with data
-    * @param start   start index in ByteBuffer
-    * @param length  length of data in ByteBuffer
-    */
-   public void set(int index, ByteBuffer value, int start, int length) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
-      valueBuffer.setBytes(startOffset, value, start, length);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param value   ByteBuffer with data
-    * @param start   start index in ByteBuffer
-    * @param length  length of data in ByteBuffer
-    */
-   public void setSafe(int index, ByteBuffer value, int start, int length) {
-      assert index >= 0;
-      fillEmpties(index);
-      handleSafe(index, length);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
-      valueBuffer.setBytes(startOffset, value, start, length);
-      lastSet = index;
-   }
-
-   /**
-    * Set the variable length element at the specified index to the data
-    * buffer supplied in the holder
-    *
-    * @param index   position of the element to set
-    * @param holder  holder that carries data buffer.
-    */
-   public void set(int index, VarCharHolder holder) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      final int dataLength = holder.end - holder.start;
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
-      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, VarCharHolder)} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param holder  holder that carries data buffer.
-    */
-   public void setSafe(int index, VarCharHolder holder) {
-      assert index >= 0;
-      final int dataLength = holder.end - holder.start;
-      fillEmpties(index);
-      handleSafe(index, dataLength);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
-      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
-      lastSet = index;
-   }
-
-   /**
-    * Set the variable length element at the specified index to the data
-    * buffer supplied in the holder
-    *
-    * @param index   position of the element to set
-    * @param holder  holder that carries data buffer.
-    */
-   public void set(int index, NullableVarCharHolder holder) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
-      final int dataLength = holder.end - holder.start;
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
-      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the
-    * case where index and length of new element are beyond the existing
-    * capacity of the vector.
-    *
-    * @param index   position of the element to set
-    * @param holder  holder that carries data buffer.
-    */
-   public void setSafe(int index, NullableVarCharHolder holder) {
-      assert index >= 0;
-      final int dataLength = holder.end - holder.start;
-      fillEmpties(index);
-      handleSafe(index, dataLength);
-      BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
-      valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
-      lastSet = index;
-   }
-
-   /**
-    * Sets the value length for an element.
-    *
-    * @param index   position of the element to set
-    * @param length  length of the element
-    */
-   public void setValueLengthSafe(int index, int length) {
-      assert index >= 0;
-      handleSafe(index, length);
-      final int startOffset = getstartOffset(index);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length);
-   }
-
-   /**
-    * Set the element at the given index to null.
-    *
-    * @param index   position of element
-    */
-   public void setNull(int index){
-      handleSafe(index, 0);
-      /* not really needed to set the bit to 0 as long as
-       * the buffer always starts from 0.
-       */
-      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-   }
-
-   /**
-    * Store the given value at a particular position in the vector. isSet indicates
-    * whether the value is NULL or not.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param start start position of data in buffer
-    * @param end end position of data in buffer
-    * @param buffer data buffer containing the variable width element to be stored
-    *               in the vector
-    */
-   public void set(int index, int isSet, int start, int end, ArrowBuf buffer) {
-      assert index >= 0;
-      fillHoles(index);
-      BitVectorHelper.setValidityBit(validityBuffer, index, isSet);
-      final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
-      offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end);
-      final ArrowBuf bb = buffer.slice(start, end);
-      valueBuffer.setBytes(startOffset, bb);
-      lastSet = index;
-   }
-
-   /**
-    * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case
-    * when index is greater than or equal to current value capacity of the
-    * vector.
-    * @param index position of the new value
-    * @param isSet 0 for NULL value, 1 otherwise
-    * @param start start position of data in buffer
-    * @param end end position of data in buffer
-    * @param buffer data buffer containing the variable width element to be stored
-    *               in the vector
-    */
-   public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) {
-      assert index >= 0;
-      handleSafe(index, end);
-      set(index, isSet, start, end, buffer);
-   }
-
-
-   /******************************************************************
-    *                                                                *
-    *                      vector transfer                           *
-    *                                                                *
-    ******************************************************************/
-
-   /**
-    * Construct a TransferPair comprising of this and and a target vector of
-    * the same type.
-    * @param ref name of the target vector
-    * @param allocator allocator for the target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-      return new TransferImpl(ref, allocator);
-   }
-
-   /**
-    * Construct a TransferPair with a desired target vector of the same type.
-    * @param to target vector
-    * @return {@link TransferPair}
-    */
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return new TransferImpl((NullableVarCharVector)to);
-   }
-
-   private class TransferImpl implements TransferPair {
-      NullableVarCharVector to;
-
-      public TransferImpl(String ref, BufferAllocator allocator){
-         to = new NullableVarCharVector(ref, field.getFieldType(), allocator);
-      }
-
-      public TransferImpl(NullableVarCharVector to){
-         this.to = to;
-      }
-
-      @Override
-      public NullableVarCharVector getTo(){
-         return to;
-      }
-
-      @Override
-      public void transfer(){
-         transferTo(to);
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-         splitAndTransferTo(startIndex, length, to);
-      }
-
-      @Override
-      public void copyValueSafe(int fromIndex, int toIndex) {
-         to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this);
-      }
-   }
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a NullableVarCharVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableVarCharVector(String name, BufferAllocator allocator) {
+    this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator);
+  }
+
+  /**
+   * Instantiate a NullableVarCharVector. This doesn't allocate any memory for
+   * the data in vector.
+   * @param name name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType);
+    reader = new VarCharReaderImpl(NullableVarCharVector.this);
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.VARCHAR;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value getter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Get the variable length element at specified index as byte array.
+   *
+   * @param index   position of element to get
+   * @return array of bytes for non-null element, null otherwise
+   */
+  public byte[] get(int index) {
+    assert index >= 0;
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    final int startOffset = getstartOffset(index);
+    final int dataLength =
+            offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset;
+    final byte[] result = new byte[dataLength];
+    valueBuffer.getBytes(startOffset, result, 0, dataLength);
+    return result;
+  }
+
+  /**
+   * Get the variable length element at specified index as Text.
+   *
+   * @param index   position of element to get
+   * @return Text object for non-null element, null otherwise
+   */
+  public Text getObject(int index) {
+    Text result = new Text();
+    byte[] b;
+    try {
+      b = get(index);
+    } catch (IllegalStateException e) {
+      return null;
+    }
+    result.set(b);
+    return result;
+  }
+
+  /**
+   * Get the variable length element at specified index and sets the state
+   * in provided holder.
+   *
+   * @param index   position of element to get
+   * @param holder  data holder to be populated by this function
+   */
+  public void get(int index, NullableVarCharHolder holder) {
+    assert index >= 0;
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.start = getstartOffset(index);
+    holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH);
+    holder.buffer = valueBuffer;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) {
+    final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH);
+    final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH);
+    final int length = end - start;
+    fillHoles(thisIndex);
+    BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH);
+    from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length);
+    offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length);
+    lastSet = thisIndex;
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) {
+    final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH);
+    final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH);
+    final int length = end - start;
+    handleSafe(thisIndex, length);
+    fillHoles(thisIndex);
+    BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH);
+    from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length);
+    offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length);
+    lastSet = thisIndex;
+  }
+
+  /**
+   * Set the variable length element at the specified index to the data
+   * buffer supplied in the holder
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void set(int index, VarCharHolder holder) {
+    assert index >= 0;
+    fillHoles(index);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int dataLength = holder.end - holder.start;
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, VarCharHolder)} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void setSafe(int index, VarCharHolder holder) {
+    assert index >= 0;
+    final int dataLength = holder.end - holder.start;
+    fillEmpties(index);
+    handleSafe(index, dataLength);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+    lastSet = index;
+  }
+
+  /**
+   * Set the variable length element at the specified index to the data
+   * buffer supplied in the holder
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void set(int index, NullableVarCharHolder holder) {
+    assert index >= 0;
+    fillHoles(index);
+    BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
+    final int dataLength = holder.end - holder.start;
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+    lastSet = index;
+  }
+
+  /**
+   * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void setSafe(int index, NullableVarCharHolder holder) {
+    assert index >= 0;
+    final int dataLength = holder.end - holder.start;
+    fillEmpties(index);
+    handleSafe(index, dataLength);
+    BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet);
+    final int startOffset = getstartOffset(index);
+    offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
+    valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength);
+    lastSet = index;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   * @param ref name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((NullableVarCharVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    NullableVarCharVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new NullableVarCharVector(ref, field.getFieldType(), allocator);
+    }
+
+    public TransferImpl(NullableVarCharVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public NullableVarCharVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this);
+    }
+  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index dc74afaa2e49c..0d3acf1646f53 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 288afc6bbec69..fd9677312ed33 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -22,13 +22,10 @@
 import java.util.List;
 
 import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.schema.ArrowRecordBatch;
 import org.apache.arrow.vector.schema.ArrowVectorType;
 
-import javax.annotation.Nullable;
-
 public class VectorUnloader {
 
   private final VectorSchemaRoot root;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 3eab30a32d224..4648d078949a7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -74,8 +74,7 @@ public boolean allocateNewSafe() {
       e.printStackTrace();
       clear();
       return false;
-    }
-    finally {
+    } finally {
       if (!dataAlloc) {
         clear();
       }
@@ -84,7 +83,7 @@ public boolean allocateNewSafe() {
   }
 
   protected void allocateOffsetBuffer(final long size) {
-    final int curSize = (int)size;
+    final int curSize = (int) size;
     offsetBuffer = allocator.buffer(curSize);
     offsetBuffer.readerIndex(0);
     offsetAllocationSizeInBytes = curSize;
@@ -101,8 +100,8 @@ protected void reallocOffsetBuffer() {
     final int currentBufferCapacity = offsetBuffer.capacity();
     long baseSize = offsetAllocationSizeInBytes;
 
-    if (baseSize < (long)currentBufferCapacity) {
-      baseSize = (long)currentBufferCapacity;
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
     }
 
     long newAllocationSize = baseSize * 2L;
@@ -112,13 +111,13 @@ protected void reallocOffsetBuffer() {
       throw new OversizedAllocationException("Unable to expand the buffer");
     }
 
-    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
     newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity);
     final int halfNewCapacity = newBuf.capacity() / 2;
     newBuf.setZero(halfNewCapacity, halfNewCapacity);
     offsetBuffer.release(1);
     offsetBuffer = newBuf;
-    offsetAllocationSizeInBytes = (int)newAllocationSize;
+    offsetAllocationSizeInBytes = (int) newAllocationSize;
   }
 
   @Override
@@ -147,8 +146,8 @@ public int getValueCapacity() {
     return Math.min(vector.getValueCapacity(), offsetValueCapacity);
   }
 
-  private int getOffsetBufferValueCapacity() {
-    return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH);
+  protected int getOffsetBufferValueCapacity() {
+    return (int) ((offsetBuffer.capacity() * 1.0) / OFFSET_WIDTH);
   }
 
   @Override
@@ -183,8 +182,13 @@ public void clear() {
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer},
-            vector.getBuffers(false), ArrowBuf.class);
+    final ArrowBuf[] buffers;
+    if (getBufferSize() == 0) {
+      buffers = new ArrowBuf[0];
+    } else {
+      buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, vector.getBuffers(false),
+              ArrowBuf.class);
+    }
     if (clear) {
       for (ArrowBuf buffer : buffers) {
         buffer.retain();
@@ -208,15 +212,15 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldT
       // returned vector must have the same field
       created = true;
       if (callBack != null &&
-          // not a schema change if changing from ZeroVector to ZeroVector
-          (fieldType.getType().getTypeID() != ArrowTypeID.Null)) {
+              // not a schema change if changing from ZeroVector to ZeroVector
+              (fieldType.getType().getTypeID() != ArrowTypeID.Null)) {
         callBack.doWork();
       }
     }
 
     if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) {
       final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
-          fieldType.getType().getTypeID(), vector.getField().getType().getTypeID());
+              fieldType.getType().getTypeID(), vector.getField().getType().getTypeID());
       throw new SchemaChangeRuntimeException(msg);
     }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index bb091f68c1807..6713b1c7871b5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -1,5 +1,4 @@
-/*******************************************************************************
-
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -7,15 +6,15 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- ******************************************************************************/
+ */
 
 package org.apache.arrow.vector.complex;
 
@@ -125,7 +124,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
     ArrowBuf bitBuffer = ownBuffers.get(0);
 
     validityBuffer.release();
-    validityBuffer = bitBuffer.retain(allocator);
+    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
     valueCount = fieldNode.getLength();
 
     validityAllocationSizeInBytes = validityBuffer.capacity();
@@ -134,15 +133,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
   @Override
   public List<ArrowBuf> getFieldBuffers() {
     List<ArrowBuf> result = new ArrayList<>(1);
-
-    validityBuffer.readerIndex(0);
-    validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
-
+    setReaderAndWriterIndex();
     result.add(validityBuffer);
 
     return result;
   }
 
+  private void setReaderAndWriterIndex() {
+    validityBuffer.readerIndex(0);
+    validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
+  }
+
   @Override
   @Deprecated
   public List<BufferBacked> getFieldInnerVectors() {
@@ -182,6 +183,8 @@ public boolean allocateNewSafe() {
      */
     boolean success = false;
     try {
+      /* we are doing a new allocation -- release the current buffers */
+      clear();
       /* allocate validity buffer */
       allocateValidityBuffer(validityAllocationSizeInBytes);
       success = vector.allocateNewSafe();
@@ -196,7 +199,7 @@ public boolean allocateNewSafe() {
   }
 
   private void allocateValidityBuffer(final long size) {
-    final int curSize = (int)size;
+    final int curSize = (int) size;
     validityBuffer = allocator.buffer(curSize);
     validityBuffer.readerIndex(0);
     validityAllocationSizeInBytes = curSize;
@@ -213,8 +216,8 @@ private void reallocValidityBuffer() {
     final int currentBufferCapacity = validityBuffer.capacity();
     long baseSize = validityAllocationSizeInBytes;
 
-    if (baseSize < (long)currentBufferCapacity) {
-      baseSize = (long)currentBufferCapacity;
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
     }
 
     long newAllocationSize = baseSize * 2L;
@@ -224,12 +227,12 @@ private void reallocValidityBuffer() {
       throw new OversizedAllocationException("Unable to expand the buffer");
     }
 
-    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
     newBuf.setZero(0, newBuf.capacity());
     newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
     validityBuffer.release(1);
     validityBuffer = newBuf;
-    validityAllocationSizeInBytes = (int)newAllocationSize;
+    validityAllocationSizeInBytes = (int) newAllocationSize;
   }
 
   public FieldVector getDataVector() {
@@ -282,7 +285,14 @@ public void clear() {
 
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class);
+    setReaderAndWriterIndex();
+    final ArrowBuf[] buffers;
+    if (getBufferSize() == 0) {
+      buffers = new ArrowBuf[0];
+    } else {
+      buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false),
+              ArrowBuf.class);
+    }
     if (clear) {
       for (ArrowBuf buffer : buffers) {
         buffer.retain();
@@ -311,7 +321,7 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType type)
     // returned vector must have the same field
     if (!Objects.equals(vector.getField().getType(), type.getType())) {
       final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
-          type.getType(), vector.getField().getType());
+              type.getType(), vector.getField().getType());
       throw new SchemaChangeRuntimeException(msg);
     }
 
@@ -400,7 +410,7 @@ public int getValueCount() {
   }
 
   private int getValidityBufferValueCapacity() {
-    return (int)(validityBuffer.capacity() * 8L);
+    return (int) (validityBuffer.capacity() * 8L);
   }
 
   public void setNull(int index) {
@@ -462,6 +472,7 @@ public void transfer() {
       dataPair.transfer();
       to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer;
       to.setValueCount(valueCount);
+      clear();
     }
 
     @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java
deleted file mode 100644
index 435d3d8ccf2e6..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex;
-
-import java.util.Collections;
-import java.util.Iterator;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.BaseValueVector;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
-public abstract class LegacyBaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
-
-   public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
-   public final static String OFFSETS_VECTOR_NAME = "$offsets$";
-   public final static String DATA_VECTOR_NAME = "$data$";
-
-   protected final UInt4Vector offsets;
-   protected FieldVector vector;
-   protected final CallBack callBack;
-
-   protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) {
-      this(name, allocator, DEFAULT_DATA_VECTOR, callBack);
-   }
-
-   protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) {
-      super(name, allocator);
-      this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
-      this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
-      this.callBack = callBack;
-   }
-
-   @Override
-   public boolean allocateNewSafe() {
-    /* boolean to keep track if all the memory allocation were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-      boolean success = false;
-      try {
-         if (!offsets.allocateNewSafe()) {
-            return false;
-         }
-         success = vector.allocateNewSafe();
-      } finally {
-         if (!success) {
-            clear();
-         }
-      }
-      offsets.zeroVector();
-      return success;
-   }
-
-   @Override
-   public void reAlloc() {
-      offsets.reAlloc();
-      vector.reAlloc();
-   }
-
-   @Override
-   public UInt4Vector getOffsetVector() {
-      return offsets;
-   }
-
-   @Override
-   public FieldVector getDataVector() {
-      return vector;
-   }
-
-   @Override
-   public void setInitialCapacity(int numRecords) {
-      offsets.setInitialCapacity(numRecords + 1);
-      vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
-   }
-
-   @Override
-   public int getValueCapacity() {
-      final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
-      if (vector == DEFAULT_DATA_VECTOR) {
-         return offsetValueCapacity;
-      }
-      return Math.min(vector.getValueCapacity(), offsetValueCapacity);
-   }
-
-   @Override
-   public int getBufferSize() {
-      if (getAccessor().getValueCount() == 0) {
-         return 0;
-      }
-      return offsets.getBufferSize() + vector.getBufferSize();
-   }
-
-   @Override
-   public int getBufferSizeFor(int valueCount) {
-      if (valueCount == 0) {
-         return 0;
-      }
-
-      return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount);
-   }
-
-   @Override
-   public Iterator<ValueVector> iterator() {
-      return Collections.<ValueVector>singleton(getDataVector()).iterator();
-   }
-
-   @Override
-   public void clear() {
-      offsets.clear();
-      vector.clear();
-      super.clear();
-   }
-
-   @Override
-   public ArrowBuf[] getBuffers(boolean clear) {
-      final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
-      if (clear) {
-         for (ArrowBuf buffer : buffers) {
-            buffer.retain();
-         }
-         clear();
-      }
-      return buffers;
-   }
-
-   /**
-    * @return 1 if inner vector is explicitly set via #addOrGetVector else 0
-    */
-   public int size() {
-      return vector == DEFAULT_DATA_VECTOR ? 0 : 1;
-   }
-
-   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) {
-      boolean created = false;
-      if (vector instanceof ZeroVector) {
-         vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack);
-         // returned vector must have the same field
-         created = true;
-         if (callBack != null &&
-                 // not a schema change if changing from ZeroVector to ZeroVector
-                 (fieldType.getType().getTypeID() != ArrowTypeID.Null)) {
-            callBack.doWork();
-         }
-      }
-
-      if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) {
-         final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
-                 fieldType.getType().getTypeID(), vector.getField().getType().getTypeID());
-         throw new SchemaChangeRuntimeException(msg);
-      }
-
-      return new AddOrGetResult<>((T) vector, created);
-   }
-
-   protected void replaceDataVector(FieldVector v) {
-      vector.clear();
-      vector = v;
-   }
-
-   public abstract class LegacyBaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
-
-      @Override
-      public int getValueCount() {
-         return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
-      }
-
-      @Override
-      public int getInnerValueCount() {
-         return vector.getAccessor().getValueCount();
-      }
-
-      @Override
-      public int getInnerValueCountAt(int index) {
-         return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index);
-      }
-
-      @Override
-      public boolean isNull(int index) {
-         return false;
-      }
-
-      @Override
-      public boolean isEmpty(int index) {
-         return false;
-      }
-   }
-
-   public abstract class LegacyBaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
-
-      @Override
-      public int startNewValue(int index) {
-         while (offsets.getValueCapacity() <= index) {
-            offsets.reAlloc();
-         }
-         int offset = offsets.getAccessor().get(index);
-         offsets.getMutator().setSafe(index + 1, offset);
-         setValueCount(index + 1);
-         return offset;
-      }
-
-      @Override
-      public void setValueCount(int valueCount) {
-         // TODO: populate offset end points
-         offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1);
-         final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
-         vector.getMutator().setValueCount(childValueCount);
-      }
-   }
-
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java
deleted file mode 100644
index 0843907d9df82..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-
-package org.apache.arrow.vector.complex;
-
-import static java.util.Collections.singletonList;
-import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
-import org.apache.arrow.vector.util.TransferPair;
-
-public class LegacyFixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {
-
-   public static LegacyFixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
-      FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
-      return new LegacyFixedSizeListVector(name, allocator, fieldType, null);
-   }
-
-   private Mutator mutator = new Mutator();
-   private Accessor accessor = new Accessor();
-   /* delegate */
-   private final FixedSizeListVector fixedSizeListVector;
-
-   // deprecated, use FieldType or static constructor instead
-   @Deprecated
-   public LegacyFixedSizeListVector(String name,
-                              BufferAllocator allocator,
-                              int listSize,
-                              DictionaryEncoding dictionary,
-                              CallBack schemaChangeCallback) {
-      super(name, allocator);
-      fixedSizeListVector = new FixedSizeListVector(name, allocator, listSize, dictionary, schemaChangeCallback);
-   }
-
-   public LegacyFixedSizeListVector(String name,
-                              BufferAllocator allocator,
-                              FieldType fieldType,
-                              CallBack schemaChangeCallback) {
-      super(name, allocator);
-      fixedSizeListVector = new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback);
-   }
-
-   @Override
-   public Field getField() {
-      return fixedSizeListVector.getField();
-   }
-
-   @Override
-   public MinorType getMinorType() {
-      return MinorType.FIXED_SIZE_LIST;
-   }
-
-   public int getListSize() {
-      return fixedSizeListVector.getListSize();
-   }
-
-   @Override
-   public void initializeChildrenFromFields(List<Field> children) {
-      fixedSizeListVector.initializeChildrenFromFields(children);
-   }
-
-   @Override
-   public List<FieldVector> getChildrenFromFields() {
-      return fixedSizeListVector.getChildrenFromFields();
-   }
-
-   @Override
-   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-      fixedSizeListVector.loadFieldBuffers(fieldNode, ownBuffers);
-   }
-
-   @Override
-   public List<ArrowBuf> getFieldBuffers() {
-      return fixedSizeListVector.getFieldBuffers();
-   }
-
-   @Override
-   public List<BufferBacked> getFieldInnerVectors() {
-      return fixedSizeListVector.getFieldInnerVectors();
-   }
-
-   @Override
-   public Accessor getAccessor() {
-      return accessor;
-   }
-
-   @Override
-   public Mutator getMutator() {
-      return mutator;
-   }
-
-   @Override
-   public UnionFixedSizeListReader getReader() {
-      return fixedSizeListVector.getReader();
-   }
-
-   @Override
-   public void allocateNew() throws OutOfMemoryException {
-     fixedSizeListVector.allocateNew();
-   }
-
-   @Override
-   public boolean allocateNewSafe() {
-    return fixedSizeListVector.allocateNewSafe();
-   }
-
-   @Override
-   public void reAlloc() {
-      fixedSizeListVector.reAlloc();
-   }
-
-   public FieldVector getDataVector() {
-      return fixedSizeListVector.getDataVector();
-   }
-
-   @Override
-   public void setInitialCapacity(int numRecords) {
-      fixedSizeListVector.setInitialCapacity(numRecords);
-   }
-
-   @Override
-   public int getValueCapacity() {
-     return fixedSizeListVector.getValueCapacity();
-   }
-
-   @Override
-   public int getBufferSize() {
-     return fixedSizeListVector.getBufferSize();
-   }
-
-   @Override
-   public int getBufferSizeFor(int valueCount) {
-    return fixedSizeListVector.getBufferSizeFor(valueCount);
-   }
-
-   @Override
-   public Iterator<ValueVector> iterator() {
-      return fixedSizeListVector.iterator();
-   }
-
-   @Override
-   public void clear() {
-     fixedSizeListVector.clear();
-   }
-
-   @Override
-   public ArrowBuf[] getBuffers(boolean clear) {
-      return fixedSizeListVector.getBuffers(clear);
-   }
-
-   /**
-    * @return 1 if inner vector is explicitly set via #addOrGetVector else 0
-    */
-   public int size() {
-      return fixedSizeListVector.size();
-   }
-
-   @Override
-   @SuppressWarnings("unchecked")
-   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType type) {
-      return fixedSizeListVector.addOrGetVector(type);
-   }
-
-   public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) {
-      fixedSizeListVector.copyFromSafe(inIndex, outIndex, from);
-   }
-
-   public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) {
-     fixedSizeListVector.copyFrom(fromIndex, thisIndex, from);
-   }
-
-   @Override
-   public UnionVector promoteToUnion() {
-      return fixedSizeListVector.promoteToUnion();
-   }
-
-   @Override
-   public long getValidityBufferAddress() {
-      return fixedSizeListVector.getValidityBufferAddress();
-   }
-
-   @Override
-   public long getDataBufferAddress() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public long getOffsetBufferAddress() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public ArrowBuf getValidityBuffer() {
-      return fixedSizeListVector.getValidityBuffer();
-   }
-
-   @Override
-   public ArrowBuf getDataBuffer() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public ArrowBuf getOffsetBuffer() {
-      throw new UnsupportedOperationException();
-   }
-
-   public class Accessor extends BaseValueVector.BaseAccessor {
-
-      @Override
-      public Object getObject(int index) {
-         return fixedSizeListVector.getObject(index);
-      }
-
-      @Override
-      public boolean isNull(int index) {
-         return fixedSizeListVector.isNull(index);
-      }
-
-      @Override
-      public int getNullCount() {
-         return fixedSizeListVector.getNullCount();
-      }
-
-      @Override
-      public int getValueCount() {
-         return fixedSizeListVector.getValueCount();
-      }
-   }
-
-   public class Mutator extends BaseValueVector.BaseMutator {
-
-      public void setNull(int index) {
-         fixedSizeListVector.setNull(index);
-      }
-
-      public void setNotNull(int index) {
-         fixedSizeListVector.setNotNull(index);
-      }
-
-      @Override
-      public void setValueCount(int valueCount) {
-         fixedSizeListVector.setValueCount(valueCount);
-      }
-   }
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      return fixedSizeListVector.getTransferPair(ref, allocator);
-   }
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-      return fixedSizeListVector.getTransferPair(ref, allocator, callBack);
-   }
-
-   @Override
-   public TransferPair makeTransferPair(ValueVector target) {
-      return fixedSizeListVector.makeTransferPair(((LegacyFixedSizeListVector)target).fixedSizeListVector);
-   }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java
deleted file mode 100644
index 732699ada7a81..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-
-package org.apache.arrow.vector.complex;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.util.Collections.list;
-import static java.util.Collections.singletonList;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.ComplexCopier;
-import org.apache.arrow.vector.complex.impl.UnionListReader;
-import org.apache.arrow.vector.complex.impl.UnionListWriter;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Null;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-
-public class LegacyListVector extends LegacyBaseRepeatedValueVector implements FieldVector, PromotableVector {
-
-   public static LegacyListVector empty(String name, BufferAllocator allocator) {
-      return new LegacyListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
-   }
-
-   private Mutator mutator = new Mutator();
-   private Accessor accessor = new Accessor();
-
-   private final ListVector listVector;
-
-   // deprecated, use FieldType or static constructor instead
-   @Deprecated
-   public LegacyListVector(String name, BufferAllocator allocator, CallBack callBack) {
-      super(name, allocator, callBack);
-      listVector = new ListVector(name, allocator, callBack);
-   }
-
-   // deprecated, use FieldType or static constructor instead
-   @Deprecated
-   public LegacyListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
-      super(name, allocator, callBack);
-      listVector = new ListVector(name, allocator, dictionary, callBack);
-   }
-
-   public LegacyListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
-      super(name, allocator, callBack);
-      listVector = new ListVector(name, allocator, fieldType, callBack);
-   }
-
-   @Override
-   public void initializeChildrenFromFields(List<Field> children) {
-      listVector.initializeChildrenFromFields(children);
-   }
-
-   @Override
-   public List<FieldVector> getChildrenFromFields() {
-      return listVector.getChildrenFromFields();
-   }
-
-   @Override
-   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-      listVector.loadFieldBuffers(fieldNode, ownBuffers);
-   }
-
-   @Override
-   public List<ArrowBuf> getFieldBuffers() {
-      return listVector.getFieldBuffers();
-   }
-
-   @Override
-   public List<BufferBacked> getFieldInnerVectors() {
-      return listVector.getFieldInnerVectors();
-   }
-
-   public UnionListWriter getWriter() {
-      return listVector.getWriter();
-   }
-
-   @Override
-   public void allocateNew() throws OutOfMemoryException {
-      listVector.allocateNew();
-   }
-
-   @Override
-   public void reAlloc() {
-      listVector.reAlloc();
-   }
-
-   public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
-      listVector.copyFrom(inIndex, outIndex, from);
-   }
-
-   public void copyFrom(int inIndex, int outIndex, ListVector from) {
-      listVector.copyFrom(inIndex, outIndex, from);
-   }
-
-   @Override
-   public FieldVector getDataVector() {
-      return listVector.getDataVector();
-   }
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      return listVector.getTransferPair(ref, allocator);
-   }
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-      return listVector.getTransferPair(ref, allocator, callBack);
-   }
-
-   @Override
-   public TransferPair makeTransferPair(ValueVector target) {
-      return listVector.makeTransferPair(((LegacyListVector)target).listVector);
-   }
-
-   @Override
-   public long getValidityBufferAddress() {
-      return listVector.getValidityBufferAddress();
-   }
-
-   @Override
-   public long getDataBufferAddress() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public long getOffsetBufferAddress() {
-      return listVector.getOffsetBufferAddress();
-   }
-
-   @Override
-   public ArrowBuf getValidityBuffer() {
-      return listVector.getValidityBuffer();
-   }
-
-   @Override
-   public ArrowBuf getDataBuffer() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public ArrowBuf getOffsetBuffer() {
-      return listVector.getOffsetBuffer();
-   }
-
-   @Override
-   public Accessor getAccessor() {
-      return accessor;
-   }
-
-   @Override
-   public Mutator getMutator() {
-      return mutator;
-   }
-
-   @Override
-   public UnionListReader getReader() {
-      return listVector.getReader();
-   }
-
-   @Override
-   public boolean allocateNewSafe() {
-      return listVector.allocateNewSafe();
-   }
-
-   public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) {
-      return listVector.addOrGetVector(fieldType);
-   }
-
-   @Override
-   public int getBufferSize() {
-      return listVector.getBufferSize();
-   }
-
-   @Override
-   public Field getField() {
-     return listVector.getField();
-   }
-
-   @Override
-   public MinorType getMinorType() {
-      return MinorType.LIST;
-   }
-
-   @Override
-   public void clear() {
-      listVector.clear();
-   }
-
-   @Override
-   public ArrowBuf[] getBuffers(boolean clear) {
-     return listVector.getBuffers(clear);
-   }
-
-   @Override
-   public UnionVector promoteToUnion() {
-     return listVector.promoteToUnion();
-   }
-
-   private int lastSet = 0;
-
-   public class Accessor extends LegacyBaseRepeatedAccessor {
-
-      @Override
-      public Object getObject(int index) {
-        return listVector.getObject(index);
-      }
-
-      @Override
-      public boolean isNull(int index) {
-         return listVector.isNull(index);
-      }
-
-      @Override
-      public int getNullCount() {
-         return listVector.getNullCount();
-      }
-   }
-
-   public class Mutator extends LegacyBaseRepeatedMutator {
-      public void setNotNull(int index) {
-        listVector.setNotNull(index);
-      }
-
-      @Override
-      public int startNewValue(int index) {
-         return listVector.startNewValue(index);
-      }
-
-      /**
-       * End the current value
-       *
-       * @param index index of the value to end
-       * @param size  number of elements in the list that was written
-       */
-      public void endValue(int index, int size) {
-         listVector.endValue(index, size);
-      }
-
-      @Override
-      public void setValueCount(int valueCount) {
-        listVector.setValueCount(valueCount);
-      }
-
-      public void setLastSet(int value) {
-         listVector.setLastSet(value);
-      }
-
-      public int getLastSet() {
-         return listVector.getLastSet();
-      }
-   }
-
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java
deleted file mode 100644
index 77556ab774f02..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
-
-import io.netty.buffer.ArrowBuf;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringHashMap;
-import org.apache.arrow.vector.util.TransferPair;
-
-public class LegacyMapVector extends AbstractMapVector {
-
-   /* delegate */
-   private final MapVector mapVector;
-
-   public static LegacyMapVector empty(String name, BufferAllocator allocator) {
-      FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
-      return new LegacyMapVector(name, allocator, fieldType, null);
-   }
-
-   private final Accessor accessor = new Accessor();
-   private final Mutator mutator = new Mutator();
-
-   @Deprecated
-   public LegacyMapVector(String name, BufferAllocator allocator, CallBack callBack) {
-      super(name, allocator, callBack);
-      mapVector = new MapVector(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
-   }
-
-   public LegacyMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
-      super(name, allocator, callBack);
-      mapVector = new MapVector(name, allocator, fieldType, callBack);
-   }
-
-   @Override
-   public FieldReader getReader() {
-      return mapVector.getReader();
-   }
-
-   public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
-     mapVector.copyFromSafe(fromIndex, thisIndex, from);
-   }
-
-   @Override
-   protected boolean supportsDirectRead() {
-      return true;
-   }
-
-   public Iterator<String> fieldNameIterator() {
-      return mapVector.fieldNameIterator();
-   }
-
-   @Override
-   public void setInitialCapacity(int numRecords) {
-     mapVector.setInitialCapacity(numRecords);
-   }
-
-   @Override
-   public int getBufferSize() {
-     return mapVector.getBufferSize();
-   }
-
-   @Override
-   public int getBufferSizeFor(final int valueCount) {
-      return (int) mapVector.getBufferSizeFor(valueCount);
-   }
-
-   @Override
-   public ArrowBuf getValidityBuffer() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public ArrowBuf getDataBuffer() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public ArrowBuf getOffsetBuffer() {
-      throw new UnsupportedOperationException();
-   }
-
-   @Override
-   public TransferPair getTransferPair(BufferAllocator allocator) {
-      return getTransferPair(name, allocator, null);
-   }
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-      return mapVector.getTransferPair(ref, allocator, callBack);
-   }
-
-   @Override
-   public TransferPair makeTransferPair(ValueVector to) {
-      return mapVector.makeTransferPair(((LegacyMapVector)to).mapVector);
-   }
-
-   @Override
-   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      return mapVector.getTransferPair(ref, allocator);
-   }
-
-   @Override
-   public int getValueCapacity() {
-      return mapVector.getValueCapacity();
-   }
-
-   @Override
-   public Accessor getAccessor() {
-      return accessor;
-   }
-
-   @Override
-   public Mutator getMutator() {
-      return mutator;
-   }
-
-   public class Accessor extends BaseValueVector.BaseAccessor {
-
-      @Override
-      public Object getObject(int index) {
-         return mapVector.getObject(index);
-      }
-
-      public void get(int index, ComplexHolder holder) {
-        mapVector.get(index, holder);
-      }
-
-      @Override
-      public int getValueCount() {
-         return mapVector.getValueCount();
-      }
-   }
-
-   public ValueVector getVectorById(int id) {
-      return mapVector.getVectorById(id);
-   }
-
-   public class Mutator extends BaseValueVector.BaseMutator {
-
-      @Override
-      public void setValueCount(int valueCount) {
-        mapVector.setValueCount(valueCount);
-      }
-
-      @Override
-      public void reset() { }
-
-      @Override
-      public void generateTestData(int values) { }
-   }
-
-   @Override
-   public void clear() {
-     mapVector.clear();
-   }
-
-   @Override
-   public Field getField() {
-     return mapVector.getField();
-   }
-
-   @Override
-   public MinorType getMinorType() {
-      return MinorType.MAP;
-   }
-
-   @Override
-   public void close() {
-      mapVector.close();
-   }
-
-   public void initializeChildrenFromFields(List<Field> children) {
-      mapVector.initializeChildrenFromFields(children);
-   }
-
-   public List<FieldVector> getChildrenFromFields() {
-      return mapVector.getChildren();
-   }
-
-   public boolean isNull(int index) { return false; }
-
-   public int getNullCount() { return  0; }
-
-   public int getValueCount() { return 0; }
-
-   public void setValueCount(int valueCount) { }
-
-   public Object getObject(int index) { return null; }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index ea577208cc238..afe86a692c3e1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -1,5 +1,4 @@
-/*******************************************************************************
-
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -7,15 +6,15 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- ******************************************************************************/
+ */
 
 package org.apache.arrow.vector.complex;
 
@@ -60,7 +59,7 @@ public static ListVector empty(String name, BufferAllocator allocator) {
     return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
   }
 
-  private ArrowBuf validityBuffer;
+  protected ArrowBuf validityBuffer;
   private UnionListReader reader;
   private CallBack callBack;
   private final FieldType fieldType;
@@ -108,6 +107,13 @@ public List<FieldVector> getChildrenFromFields() {
     return singletonList(getDataVector());
   }
 
+  /**
+   * Load the buffers of this vector with provided source buffers.
+   * The caller manages the source buffers and populates them before invoking
+   * this method.
+   * @param fieldNode  the fieldNode indicating the value count
+   * @param ownBuffers the buffers for this Field (own buffers only, children not included)
+   */
   @Override
   public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
     if (ownBuffers.size() != 2) {
@@ -129,20 +135,35 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
     valueCount = fieldNode.getLength();
   }
 
+  /**
+   * Get the buffers belonging to this vector
+   * @return the inner buffers.
+   */
   @Override
   public List<ArrowBuf> getFieldBuffers() {
     List<ArrowBuf> result = new ArrayList<>(2);
-    validityBuffer.readerIndex(0);
-    validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
-    offsetBuffer.readerIndex(0);
-    offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
-
+    setReaderAndWriterIndex();
     result.add(validityBuffer);
     result.add(offsetBuffer);
 
     return result;
   }
 
+  /**
+   * Set the reader and writer indexes for the inner buffers.
+   */
+  private void setReaderAndWriterIndex() {
+    validityBuffer.readerIndex(0);
+    offsetBuffer.readerIndex(0);
+    if (valueCount == 0) {
+      validityBuffer.writerIndex(0);
+      offsetBuffer.writerIndex(0);
+    } else {
+      validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
+      offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
+    }
+  }
+
   @Override
   @Deprecated
   public List<BufferBacked> getFieldInnerVectors() {
@@ -153,16 +174,28 @@ public UnionListWriter getWriter() {
     return new UnionListWriter(this);
   }
 
+  /**
+   * Same as {@link #allocateNewSafe()}.
+   */
   @Override
   public void allocateNew() throws OutOfMemoryException {
-   if (!allocateNewSafe()) {
-     throw new OutOfMemoryException("Failure while allocating memory");
-   }
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryException("Failure while allocating memory");
+    }
   }
 
+  /**
+   * Allocate memory for the vector. We internally use a default value count
+   * of 4096 to allocate memory for at least these many elements in the
+   * vector.
+   *
+   * @return false if memory allocation fails, true otherwise.
+   */
   public boolean allocateNewSafe() {
     boolean success = false;
     try {
+      /* we are doing a new allocation -- release the current buffers */
+      clear();
       /* allocate validity buffer */
       allocateValidityBuffer(validityAllocationSizeInBytes);
       /* allocate offset and data buffer */
@@ -177,13 +210,17 @@ public boolean allocateNewSafe() {
   }
 
   private void allocateValidityBuffer(final long size) {
-    final int curSize = (int)size;
+    final int curSize = (int) size;
     validityBuffer = allocator.buffer(curSize);
     validityBuffer.readerIndex(0);
     validityAllocationSizeInBytes = curSize;
     validityBuffer.setZero(0, validityBuffer.capacity());
   }
 
+  /**
+   * Resize the vector to increase the capacity. The internal behavior is to
+   * double the current value capacity.
+   */
   @Override
   public void reAlloc() {
     /* reallocate the validity buffer */
@@ -201,8 +238,8 @@ private void reallocValidityBuffer() {
     final int currentBufferCapacity = validityBuffer.capacity();
     long baseSize = validityAllocationSizeInBytes;
 
-    if (baseSize < (long)currentBufferCapacity) {
-      baseSize = (long)currentBufferCapacity;
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
     }
 
     long newAllocationSize = baseSize * 2L;
@@ -212,19 +249,34 @@ private void reallocValidityBuffer() {
       throw new OversizedAllocationException("Unable to expand the buffer");
     }
 
-    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
     newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
     final int halfNewCapacity = newBuf.capacity() / 2;
     newBuf.setZero(halfNewCapacity, halfNewCapacity);
     validityBuffer.release(1);
     validityBuffer = newBuf;
-    validityAllocationSizeInBytes = (int)newAllocationSize;
+    validityAllocationSizeInBytes = (int) newAllocationSize;
   }
 
+  /**
+   * Same as {@link #copyFrom(int, int, ListVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   * @param inIndex position to copy from in source vector
+   * @param outIndex position to copy to in this vector
+   * @param from source vector
+   */
   public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
     copyFrom(inIndex, outIndex, from);
   }
 
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   * @param inIndex position to copy from in source vector
+   * @param outIndex position to copy to in this vector
+   * @param from source vector
+   */
   public void copyFrom(int inIndex, int outIndex, ListVector from) {
     FieldReader in = from.getReader();
     in.setPosition(inIndex);
@@ -233,6 +285,10 @@ public void copyFrom(int inIndex, int outIndex, ListVector from) {
     ComplexCopier.copy(in, out);
   }
 
+  /**
+   * Get the inner data vector for this list vector
+   * @return data vector
+   */
   @Override
   public FieldVector getDataVector() {
     return vector;
@@ -269,7 +325,9 @@ public long getOffsetBufferAddress() {
   }
 
   @Override
-  public ArrowBuf getValidityBuffer() { return validityBuffer; }
+  public ArrowBuf getValidityBuffer() {
+    return validityBuffer;
+  }
 
   @Override
   public ArrowBuf getDataBuffer() {
@@ -277,7 +335,9 @@ public ArrowBuf getDataBuffer() {
   }
 
   @Override
-  public ArrowBuf getOffsetBuffer() { return offsetBuffer; }
+  public ArrowBuf getOffsetBuffer() {
+    return offsetBuffer;
+  }
 
   private class TransferImpl implements TransferPair {
 
@@ -297,15 +357,30 @@ public TransferImpl(ListVector to) {
       dataTransferPair = getDataVector().makeTransferPair(to.getDataVector());
     }
 
+    /**
+     * Transfer this vector'data to another vector. The memory associated
+     * with this vector is transferred to the allocator of target vector
+     * for accounting and management purposes.
+     */
     @Override
     public void transfer() {
+      to.clear();
       dataTransferPair.transfer();
       to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer;
       to.offsetBuffer = offsetBuffer.transferOwnership(to.allocator).buffer;
       to.lastSet = lastSet;
-      to.setValueCount(valueCount);
+      if (valueCount > 0) {
+        to.setValueCount(valueCount);
+      }
+      clear();
     }
 
+    /**
+     * Slice this vector at desired index and length and transfer the
+     * corresponding data to the target vector.
+     * @param startIndex start position of the split in source vector.
+     * @param length length of the split.
+     */
     @Override
     public void splitAndTransfer(int startIndex, int length) {
       final int startPoint = offsetBuffer.getInt(startIndex * OFFSET_WIDTH);
@@ -343,8 +418,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListVect
           }
           target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
           target.validityBuffer.retain(1);
-        }
-        else {
+        } else {
           /* Copy data
            * When the first bit starts from the middle of a byte (offset != 0),
            * copy data from src BitVector.
@@ -369,15 +443,14 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListVect
            * (we are at the last byte), we copy the last piece as a byte
            * by shifting data from the current byte.
            */
-          if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
             byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
                     firstByteSource + byteSizeTarget - 1, offset);
             byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
                     firstByteSource + byteSizeTarget, offset);
 
             target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
-          }
-          else {
+          } else {
             byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
                     firstByteSource + byteSizeTarget - 1, offset);
             target.validityBuffer.setByte(byteSizeTarget - 1, b1);
@@ -408,9 +481,14 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldT
     return result;
   }
 
+  /**
+   * Get the size (number of bytes) of underlying buffers used by this
+   * vector
+   * @return size of underlying buffers.
+   */
   @Override
   public int getBufferSize() {
-    if (getValueCount() == 0) {
+    if (valueCount == 0) {
       return 0;
     }
     final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
@@ -435,10 +513,28 @@ public void clear() {
     lastSet = 0;
   }
 
+  /**
+   * Return the underlying buffers associated with this vector. Note that this doesn't
+   * impact the reference counts for this buffer so it only should be used for in-context
+   * access. Also note that this buffer changes regularly thus
+   * external classes shouldn't hold a reference to it (unless they change it).
+   *
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
+   *              but the returned array will be the only reference to them
+   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
+   *         vector instance.
+   */
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
-    final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, ObjectArrays.concat(new ArrowBuf[] {validityBuffer},
-        vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
+    setReaderAndWriterIndex();
+    final ArrowBuf[] buffers;
+    if (getBufferSize() == 0) {
+      buffers = new ArrowBuf[0];
+    } else {
+      buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer},
+              ObjectArrays.concat(new ArrowBuf[]{validityBuffer},
+                      vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
+    }
     if (clear) {
       for (ArrowBuf buffer : buffers) {
         buffer.retain();
@@ -459,6 +555,11 @@ public UnionVector promoteToUnion() {
     return vector;
   }
 
+  /**
+   * Get the element in the list vector at a particular index
+   * @param index position of the element
+   * @return Object at given position
+   */
   @Override
   public Object getObject(int index) {
     if (isSet(index) == 0) {
@@ -475,11 +576,23 @@ public Object getObject(int index) {
     return vals;
   }
 
+  /**
+   * Check if element at given index is null.
+   *
+   * @param index  position of element
+   * @return true if element at given index is null, false otherwise
+   */
   @Override
   public boolean isNull(int index) {
     return (isSet(index) == 0);
   }
 
+  /**
+   * Same as {@link #isNull(int)}.
+   *
+   * @param index  position of element
+   * @return 1 if element at given index is not null, 0 otherwise
+   */
   public int isSet(int index) {
     final int byteIndex = index >> 3;
     final byte b = validityBuffer.getByte(byteIndex);
@@ -487,29 +600,57 @@ public int isSet(int index) {
     return Long.bitCount(b & (1L << bitIndex));
   }
 
+  /**
+   * Get the number of elements that are null in the vector
+   *
+   * @return the number of null elements.
+   */
   @Override
   public int getNullCount() {
     return BitVectorHelper.getNullCount(validityBuffer, valueCount);
   }
 
+  /**
+   * Get the current value capacity for the vector
+   * @return number of elements that vector can hold.
+   */
+  @Override
+  public int getValueCapacity() {
+    return Math.min(getValidityBufferValueCapacity(), super.getValueCapacity());
+  }
+
+  private int getValidityAndOffsetValueCapacity() {
+    final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0);
+    return Math.min(offsetValueCapacity, getValidityBufferValueCapacity());
+  }
+
+  private int getValidityBufferValueCapacity() {
+    return (int) (validityBuffer.capacity() * 8L);
+  }
+
   public void setNotNull(int index) {
-    if (index >= getValueCapacity()) {
+    while (index >= getValidityAndOffsetValueCapacity()) {
       reallocValidityAndOffsetBuffers();
     }
     BitVectorHelper.setValidityBitToOne(validityBuffer, index);
     lastSet = index + 1;
   }
 
+  /**
+   * Start a new value in the list vector
+   *
+   * @param index index of the value to start
+   */
   @Override
   public int startNewValue(int index) {
-    if (index >= getValueCapacity()) {
+    while (index >= getValidityAndOffsetValueCapacity()) {
       reallocValidityAndOffsetBuffers();
     }
     for (int i = lastSet; i <= index; i++) {
       final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH);
       offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset);
     }
-    setNotNull(index);
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
     lastSet = index + 1;
     return offsetBuffer.getInt(lastSet * OFFSET_WIDTH);
   }
@@ -525,12 +666,17 @@ public void endValue(int index, int size) {
     offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, currentOffset + size);
   }
 
+  /**
+   * Sets the value count for the vector
+   *
+   * @param valueCount   value count
+   */
   @Override
   public void setValueCount(int valueCount) {
     this.valueCount = valueCount;
     if (valueCount > 0) {
-      while (valueCount > getValueCapacity()) {
-        /* realloc the inner buffers if needed */
+      while (valueCount > getValidityAndOffsetValueCapacity()) {
+        /* check if validity and offset buffers need to be re-allocated */
         reallocValidityAndOffsetBuffers();
       }
       for (int i = lastSet; i < valueCount; i++) {
@@ -542,6 +688,9 @@ public void setValueCount(int valueCount) {
     /* valueCount for the data vector is the current end offset */
     final int childValueCount = (valueCount == 0) ? 0 :
             offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
+    /* set the value count of data vector and this will take care of
+     * checking whether data buffer needs to be reallocated.
+     */
     vector.setValueCount(childValueCount);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index a1478c5a7213d..f95302f55f887 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -54,7 +54,7 @@ public static NullableMapVector empty(String name, BufferAllocator allocator) {
   private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
   private final NullableMapWriter writer = new NullableMapWriter(this);
 
-  private ArrowBuf validityBuffer;
+  protected ArrowBuf validityBuffer;
   private int validityAllocationSizeInBytes;
 
   // deprecated, use FieldType or static constructor instead
@@ -91,7 +91,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
     ArrowBuf bitBuffer = ownBuffers.get(0);
 
     validityBuffer.release();
-    validityBuffer = bitBuffer.retain(allocator);
+    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
     valueCount = fieldNode.getLength();
     validityAllocationSizeInBytes = validityBuffer.capacity();
   }
@@ -99,14 +99,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
   @Override
   public List<ArrowBuf> getFieldBuffers() {
     List<ArrowBuf> result = new ArrayList<>(1);
-
-    validityBuffer.readerIndex(0);
-    validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
+    setReaderAndWriterIndex();
     result.add(validityBuffer);
 
     return result;
   }
 
+  private void setReaderAndWriterIndex() {
+    validityBuffer.readerIndex(0);
+    validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
+  }
+
   @Override
   @Deprecated
   public List<BufferBacked> getFieldInnerVectors() {
@@ -156,6 +159,7 @@ public void transfer() {
       target.clear();
       target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
       super.transfer();
+      clear();
     }
 
     @Override
@@ -193,8 +197,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable
         }
         target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
         target.validityBuffer.retain(1);
-      }
-      else {
+      } else {
         /* Copy data
          * When the first bit starts from the middle of a byte (offset != 0),
          * copy data from src BitVector.
@@ -219,15 +222,14 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable
          * (we are at the last byte), we copy the last piece as a byte
          * by shifting data from the current byte.
          */
-        if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
           byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
                   firstByteSource + byteSizeTarget - 1, offset);
           byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
                   firstByteSource + byteSizeTarget, offset);
 
           target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
-        }
-        else {
+        } else {
           byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
                   firstByteSource + byteSizeTarget - 1, offset);
           target.validityBuffer.setByte(byteSizeTarget - 1, b1);
@@ -236,55 +238,108 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable
     }
   }
 
+  /**
+   * Get the value capacity of the internal validity buffer.
+   * @return number of elements that validity buffer can hold
+   */
   private int getValidityBufferValueCapacity() {
-    return (int)(validityBuffer.capacity() * 8L);
+    return (int) (validityBuffer.capacity() * 8L);
   }
 
+  /**
+   * Get the current value capacity for the vector
+   * @return number of elements that vector can hold.
+   */
   @Override
   public int getValueCapacity() {
     return Math.min(getValidityBufferValueCapacity(),
             super.getValueCapacity());
   }
 
+  /**
+   * Return the underlying buffers associated with this vector. Note that this doesn't
+   * impact the reference counts for this buffer so it only should be used for in-context
+   * access. Also note that this buffer changes regularly thus
+   * external classes shouldn't hold a reference to it (unless they change it).
+   *
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
+   *              but the returned array will be the only reference to them
+   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
+   *         vector instance.
+   */
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
+    setReaderAndWriterIndex();
+    final ArrowBuf[] buffers;
+    if (getBufferSize() == 0) {
+      buffers = new ArrowBuf[0];
+    } else {
+      buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false),
+              ArrowBuf.class);
+    }
     if (clear) {
-      validityBuffer.retain(1);
+      for (ArrowBuf buffer : buffers) {
+        buffer.retain();
+      }
+      clear();
     }
-    return ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(clear), ArrowBuf.class);
+
+    return buffers;
   }
 
+  /**
+   * Close the vector and release the associated buffers.
+   */
   @Override
   public void close() {
     clearValidityBuffer();
     super.close();
   }
 
+  /**
+   * Same as {@link #close()}
+   */
   @Override
   public void clear() {
     clearValidityBuffer();
     super.clear();
   }
 
+  /**
+   * Release the validity buffer
+   */
   private void clearValidityBuffer() {
     validityBuffer.release();
     validityBuffer = allocator.getEmpty();
   }
 
+  /**
+   * Get the size (number of bytes) of underlying buffers used by this
+   * vector
+   * @return size of underlying buffers.
+   */
   @Override
   public int getBufferSize() {
-    if (valueCount == 0) { return 0; }
+    if (valueCount == 0) {
+      return 0;
+    }
     return super.getBufferSize() +
             BitVectorHelper.getValidityBufferSize(valueCount);
   }
 
+  /**
+   * Get the potential buffer size for a particular number of records.
+   * @param valueCount desired number of elements in the vector
+   * @return estimated size of underlying buffers if the vector holds
+   *         a given number of elements
+   */
   @Override
   public int getBufferSizeFor(final int valueCount) {
     if (valueCount == 0) {
       return 0;
     }
     return super.getBufferSizeFor(valueCount)
-        + BitVectorHelper.getValidityBufferSize(valueCount);
+            + BitVectorHelper.getValidityBufferSize(valueCount);
   }
 
   @Override
@@ -302,7 +357,7 @@ public boolean allocateNewSafe() {
      */
     boolean success = false;
     try {
-      clearValidityBuffer();
+      clear();
       allocateValidityBuffer(validityAllocationSizeInBytes);
       success = super.allocateNewSafe();
     } finally {
@@ -315,7 +370,7 @@ public boolean allocateNewSafe() {
   }
 
   private void allocateValidityBuffer(final long size) {
-    final int curSize = (int)size;
+    final int curSize = (int) size;
     validityBuffer = allocator.buffer(curSize);
     validityBuffer.readerIndex(0);
     validityAllocationSizeInBytes = curSize;
@@ -333,8 +388,8 @@ private void reallocValidityBuffer() {
     final int currentBufferCapacity = validityBuffer.capacity();
     long baseSize = validityAllocationSizeInBytes;
 
-    if (baseSize < (long)currentBufferCapacity) {
-      baseSize = (long)currentBufferCapacity;
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
     }
 
     long newAllocationSize = baseSize * 2L;
@@ -344,12 +399,12 @@ private void reallocValidityBuffer() {
       throw new OversizedAllocationException("Unable to expand the buffer");
     }
 
-    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
     newBuf.setZero(0, newBuf.capacity());
     newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
     validityBuffer.release(1);
     validityBuffer = newBuf;
-    validityAllocationSizeInBytes = (int)newAllocationSize;
+    validityAllocationSizeInBytes = (int) newAllocationSize;
   }
 
   @Override
@@ -431,7 +486,7 @@ public void setNull(int index) {
   @Override
   public void setValueCount(int valueCount) {
     assert valueCount >= 0;
-    while (valueCount > getValueCapacity()) {
+    while (valueCount > getValidityBufferValueCapacity()) {
       /* realloc the inner buffers if needed */
       reallocValidityBuffer();
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index d3c8b990394d8..8017b385d148a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -27,7 +27,8 @@
 
 import java.io.File;
 import java.io.IOException;
-
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -192,7 +193,7 @@ public VectorSchemaRoot read() throws IOException {
   private abstract class BufferReader {
     abstract protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException;
 
-    final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException {
+    ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException {
       readToken(START_ARRAY);
       ArrowBuf buf = read(allocator, count);
       readToken(END_ARRAY);
@@ -201,8 +202,7 @@ final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOExcepti
   }
 
   private class BufferHelper {
-     BufferReader BIT = new BufferReader() {
-
+    BufferReader BIT = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
         final int bufferSize = BitVectorHelper.getValidityBufferSize(count);
@@ -216,6 +216,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
           BitVectorHelper.setValidityBit(buf, i, parser.readValueAs(Boolean.class) ? 1 : 0);
         }
 
+        buf.writerIndex(bufferSize);
         return buf;
       }
     };
@@ -223,7 +224,8 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader INT1 = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        ArrowBuf buf = allocator.buffer(count * NullableTinyIntVector.TYPE_WIDTH);
+        final int size = count * NullableTinyIntVector.TYPE_WIDTH;
+        ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
           parser.nextToken();
@@ -237,7 +239,8 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader INT2 = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        ArrowBuf buf = allocator.buffer(count * NullableSmallIntVector.TYPE_WIDTH);
+        final int size = count * NullableSmallIntVector.TYPE_WIDTH;
+        ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
           parser.nextToken();
@@ -249,10 +252,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     };
 
     BufferReader INT4 = new BufferReader() {
-
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        ArrowBuf buf = allocator.buffer(count * NullableIntVector.TYPE_WIDTH);
+        final int size = count * NullableIntVector.TYPE_WIDTH;
+        ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
           parser.nextToken();
@@ -264,10 +267,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     };
 
     BufferReader INT8 = new BufferReader() {
-
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        ArrowBuf buf = allocator.buffer(count * NullableBigIntVector.TYPE_WIDTH);
+        final int size = count * NullableBigIntVector.TYPE_WIDTH;
+        ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
           parser.nextToken();
@@ -279,10 +282,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     };
 
     BufferReader FLOAT4 = new BufferReader() {
-
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        ArrowBuf buf = allocator.buffer(count * NullableFloat4Vector.TYPE_WIDTH);
+        final int size = count * NullableFloat4Vector.TYPE_WIDTH;
+        ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
           parser.nextToken();
@@ -294,10 +297,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     };
 
     BufferReader FLOAT8 = new BufferReader() {
-
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        ArrowBuf buf = allocator.buffer(count * NullableFloat8Vector.TYPE_WIDTH);
+        final int size = count * NullableFloat8Vector.TYPE_WIDTH;
+        ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
           parser.nextToken();
@@ -309,17 +312,18 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     };
 
     BufferReader DECIMAL = new BufferReader() {
-
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        ArrowBuf buf = allocator.buffer(count * NullableDecimalVector.TYPE_WIDTH);
+        final int size = count * NullableDecimalVector.TYPE_WIDTH;
+        ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
           parser.nextToken();
-          final byte[] value = decodeHexSafe(parser.getValueAsString());
-          DecimalUtility.writeByteArrayToArrowBuf(value, buf, i);
+          BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class));
+          DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i);
         }
 
+        buf.writerIndex(size);
         return buf;
       }
     };
@@ -369,10 +373,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
         return buf;
       }
     };
-
   }
 
-  private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferType, Types.MinorType type, int count) throws IOException {
+  private ArrowBuf readIntoBuffer(BufferAllocator allocator, ArrowVectorType bufferType,
+                                  Types.MinorType type, int count) throws IOException {
     ArrowBuf buf;
 
     BufferHelper helper = new BufferHelper();
@@ -461,6 +465,7 @@ private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferTyp
     }
 
     buf = reader.readBuffer(allocator, count);
+
     assert buf != null;
     return buf;
   }
@@ -506,12 +511,14 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json
           innerBufferValueCount = valueCount + 1;
         }
 
-        vectorBuffers[v] = readBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount);
+        vectorBuffers[v] = readIntoBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount);
       }
 
-      vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers));
+      final int nullCount = BitVectorHelper.getNullCount(vectorBuffers[0], valueCount);
+      final ArrowFieldNode fieldNode = new ArrowFieldNode(valueCount, nullCount);
+      vector.loadFieldBuffers(fieldNode, Arrays.asList(vectorBuffers));
 
-      // read child vectors, if any
+      /* read child vectors (if any) */
       List<Field> fields = field.getChildren();
       if (!fields.isEmpty()) {
         List<FieldVector> vectorChildren = vector.getChildrenFromFields();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index 7921cd34c773c..0c8507b51f598 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -22,6 +22,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -159,6 +160,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
       generator.writeObjectField("name", field.getName());
       int valueCount = vector.getValueCount();
       generator.writeObjectField("count", valueCount);
+      final int scale = (vector instanceof NullableDecimalVector) ?
+                            ((NullableDecimalVector) vector).getScale() : 0;
       for (int v = 0; v < vectorTypes.size(); v++) {
         ArrowVectorType vectorType = vectorTypes.get(v);
         ArrowBuf vectorBuffer = vectorBuffers.get(v);
@@ -167,9 +170,9 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
         for (int i = 0; i < bufferValueCount; i++) {
           if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR ||
                   vector.getMinorType() == Types.MinorType.VARBINARY)) {
-            writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i);
+            writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i, scale);
           } else {
-            writeValueToGenerator(vectorType, vectorBuffer, null, vector, i);
+            writeValueToGenerator(vectorType, vectorBuffer, null, vector, i, scale);
           }
         }
         generator.writeEndArray();
@@ -193,7 +196,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
   }
 
   private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer,
-                                     ArrowBuf offsetBuffer, FieldVector vector, int index) throws IOException {
+                                     ArrowBuf offsetBuffer, FieldVector vector,
+                                     final int index, final int scale) throws IOException {
     if (bufferType.equals(TYPE)) {
       generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH));
     } else if (bufferType.equals(OFFSET)) {
@@ -279,9 +283,9 @@ private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer,
           break;
         }
         case DECIMAL: {
-          String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(buffer,
-                  index));
-          generator.writeString(hexString);
+          BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale);
+          // We write the unscaled value, because the scale is stored in the type metadata.
+          generator.writeString(decimalValue.unscaledValue().toString());
           break;
         }
         default:
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index ede4f992291b4..aec7d0f327389 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -20,6 +20,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import java.util.List;
 
@@ -113,10 +114,23 @@ public void testTransfer() throws Exception {
         assertEquals(srcVector.getField(), destVector.getField());
 
         // now check the values are transferred
-        assertEquals(srcVector.getValueCount(), destVector.getValueCount());
-        for (int i = 0; i < srcVector.getValueCount(); i++) {
-          assertEquals("Different values at index " + i, srcVector.get(i), destVector.get(i));
-        }
+        assertEquals(6, destVector.getValueCount());
+
+        assertFalse(destVector.isNull(0));
+        assertEquals(5, destVector.getObject(0));
+
+        assertFalse(destVector.isNull(1));
+        assertEquals(false, destVector.getObject(1));
+
+        assertTrue(destVector.isNull(2));
+
+        assertFalse(destVector.isNull(3));
+        assertEquals(10, destVector.getObject(3));
+
+        assertTrue(destVector.isNull(4));
+
+        assertFalse(destVector.isNull(5));
+        assertEquals(false, destVector.getObject(5));
       }
     }
   }

From ca3acdc138b1ac27c9111b236d33593988689a20 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 16 Nov 2017 00:07:13 -0500
Subject: [PATCH 1253/1644] ARROW-1821: [INTEGRATION] Add integration test case
 for when Field has zero null count and optional validity buffer

Currently when a Field has null count = 0, C++ will omit the validity buffer as it is optional in this case.  Testing for #1316 was failing because Java was not handling this properly.  This PR adds an explicit test to ensure this is being tested and easier to locate possible issues.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1323 from BryanCutler/integration-test-optional-validity-buffer-ARROW-1821 and squashes the following commits:

4295660d [Bryan Cutler] expanded simple.json static test file to include field with null count = 0
---
 integration/data/simple.json | 48 ++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/integration/data/simple.json b/integration/data/simple.json
index fb903e7ac4b63..bc8949912a48a 100644
--- a/integration/data/simple.json
+++ b/integration/data/simple.json
@@ -61,6 +61,54 @@
           "DATA": ["aa", "", "", "bbb", "cccc"]
         }
       ]
+    },
+    {
+      "count": 5,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 5,
+          "VALIDITY": [1, 1, 1, 1, 1],
+          "DATA": [1, 2, 3, 4, 5]
+        },
+        {
+          "name": "bar",
+          "count": 5,
+          "VALIDITY": [1, 1, 1, 1, 1],
+          "DATA": [1.0, 2.0, 3.0, 4.0, 5.0]
+        },
+        {
+          "name": "baz",
+          "count": 5,
+          "VALIDITY": [1, 1, 1, 1, 1],
+          "OFFSET": [0, 2, 3, 4, 7, 11],
+          "DATA": ["aa", "b", "c", "ddd", "eeee"]
+        }
+      ]
+    },
+    {
+      "count": 5,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 5,
+          "VALIDITY": [0, 0, 0, 0, 0],
+          "DATA": [1, 2, 3, 4, 5]
+        },
+        {
+          "name": "bar",
+          "count": 5,
+          "VALIDITY": [0, 0, 0, 0, 0],
+          "DATA": [1.0, 2.0, 3.0, 4.0, 5.0]
+        },
+        {
+          "name": "baz",
+          "count": 5,
+          "VALIDITY": [0, 0, 0, 0, 0],
+          "OFFSET": [0, 0, 0, 0, 0, 0],
+          "DATA": ["", "", "", "", ""]
+        }
+      ]
     }
   ]
 }

From ac26eb72c02d4ea7e8788ee3a8a3eeec6403b9bb Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Thu, 16 Nov 2017 18:15:06 -0800
Subject: [PATCH 1254/1644] ARROW-1829: [Plasma] Fixes to eviction policy.

Some fixes to the bookkeeping for the eviction policy.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1317 from robertnishihara/evictionpolicy and squashes the following commits:

db34d8e [Robert Nishihara] Add sanity checks.
0d3a6c8 [Robert Nishihara] Fixes to eviction policy.
---
 cpp/src/plasma/eviction_policy.cc | 34 +++++++++++++------------------
 cpp/src/plasma/eviction_policy.h  |  3 +--
 2 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
index 6c2309f1709d2..a7758fd2c0e74 100644
--- a/cpp/src/plasma/eviction_policy.cc
+++ b/cpp/src/plasma/eviction_policy.cc
@@ -61,38 +61,32 @@ int64_t EvictionPolicy::choose_objects_to_evict(int64_t num_bytes_required,
   }
   /* Update the number of bytes used. */
   memory_used_ -= bytes_evicted;
+  ARROW_CHECK(memory_used_ >= 0);
   return bytes_evicted;
 }
 
 void EvictionPolicy::object_created(const ObjectID& object_id) {
   auto entry = store_info_->objects[object_id].get();
   cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
+  int64_t size = entry->info.data_size + entry->info.metadata_size;
+  memory_used_ += size;
+  ARROW_CHECK(memory_used_ <= store_info_->memory_capacity);
 }
 
 bool EvictionPolicy::require_space(int64_t size,
                                    std::vector<ObjectID>* objects_to_evict) {
   /* Check if there is enough space to create the object. */
   int64_t required_space = memory_used_ + size - store_info_->memory_capacity;
-  int64_t num_bytes_evicted;
-  if (required_space > 0) {
-    /* Try to free up at least as much space as we need right now but ideally
-     * up to 20% of the total capacity. */
-    int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5);
-    ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects";
-    /* Choose some objects to evict, and update the return pointers. */
-    num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict);
-    ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting "
-                    << objects_to_evict->size() << " objects to free up "
-                    << num_bytes_evicted << " bytes.";
-  } else {
-    num_bytes_evicted = 0;
-  }
-  if (num_bytes_evicted >= required_space) {
-    /* We only increment the space used if there is enough space to create the
-     * object. */
-    memory_used_ += size;
-  }
-  return num_bytes_evicted >= required_space;
+  /* Try to free up at least as much space as we need right now but ideally
+   * up to 20% of the total capacity. */
+  int64_t space_to_free = std::max(required_space, store_info_->memory_capacity / 5);
+  ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects";
+  /* Choose some objects to evict, and update the return pointers. */
+  int64_t num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict);
+  ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting "
+                  << objects_to_evict->size() << " objects to free up "
+                  << num_bytes_evicted << " bytes.";
+  return num_bytes_evicted >= required_space && num_bytes_evicted > 0;
 }
 
 void EvictionPolicy::begin_object_access(const ObjectID& object_id,
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
index de33dabcbafb3..cebf35b1c1b08 100644
--- a/cpp/src/plasma/eviction_policy.h
+++ b/cpp/src/plasma/eviction_policy.h
@@ -73,8 +73,7 @@ class EvictionPolicy {
   void object_created(const ObjectID& object_id);
 
   /// This method will be called when the Plasma store needs more space, perhaps
-  /// to create a new object. If the required amount of space cannot be freed up,
-  /// then a fatal error will be thrown. When this method is called, the eviction
+  /// to create a new object. When this method is called, the eviction
   /// policy will assume that the objects chosen to be evicted will in fact be
   /// evicted from the Plasma store by the caller.
   ///

From cacbacd439919742a0b6fbec27ee73b5af29347f Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Thu, 16 Nov 2017 21:56:03 -0800
Subject: [PATCH 1255/1644] ARROW-1795: [Plasma] Create flag to make Plasma
 store use a single memory-mapped file.

This adds the `-f` flag which tells the plasma store to use a single memory-mapped file. This is accomplished by simply calling `dlmemalign`/`dlfree` on the entire space at startup.

Question: Why does the test pass? Given that `plasma_client` is constructed with a release delay of 64, shouldn't the store by unable to evict some objects? Yet they all seem to get evicted just fine.

cc @pcmoritz @atumanov

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1327 from robertnishihara/allocateupfront and squashes the following commits:

6f3b953 [Robert Nishihara] Augment test.
0daeae3 [Robert Nishihara] Remove hard-coded values and update test.
a446fbd [Robert Nishihara] Add a test.
c2374b6 [Robert Nishihara] Add flag to tell the plasma store to use a single memory-mapped file.
---
 cpp/src/plasma/store.cc             | 27 ++++++++++++++++++++++-----
 python/pyarrow/tests/test_plasma.py | 29 +++++++++++++++++++++++++++--
 2 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 7094aed6f8e86..c6a19a5471883 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -676,12 +676,22 @@ class PlasmaStoreRunner {
   PlasmaStoreRunner() {}
 
   void Start(char* socket_name, int64_t system_memory, std::string directory,
-             bool hugepages_enabled) {
+             bool hugepages_enabled, bool use_one_memory_mapped_file) {
     // Create the event loop.
     loop_.reset(new EventLoop);
     store_.reset(
         new PlasmaStore(loop_.get(), system_memory, directory, hugepages_enabled));
     plasma_config = store_->get_plasma_store_info();
+
+    // If the store is configured to use a single memory-mapped file, then we
+    // achieve that by mallocing and freeing a single large amount of space.
+    // that maximum allowed size up front.
+    if (use_one_memory_mapped_file) {
+      void* pointer = plasma::dlmemalign(BLOCK_SIZE, system_memory);
+      ARROW_CHECK(pointer != NULL);
+      plasma::dlfree(pointer);
+    }
+
     int socket = bind_ipc_sock(socket_name, true);
     // TODO(pcm): Check return value.
     ARROW_CHECK(socket >= 0);
@@ -716,14 +726,15 @@ void HandleSignal(int signal) {
 }
 
 void start_server(char* socket_name, int64_t system_memory, std::string plasma_directory,
-                  bool hugepages_enabled) {
+                  bool hugepages_enabled, bool use_one_memory_mapped_file) {
   // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
   // to a client that has already died, the store could die.
   signal(SIGPIPE, SIG_IGN);
 
   g_runner.reset(new PlasmaStoreRunner());
   signal(SIGTERM, HandleSignal);
-  g_runner->Start(socket_name, system_memory, plasma_directory, hugepages_enabled);
+  g_runner->Start(socket_name, system_memory, plasma_directory, hugepages_enabled,
+                  use_one_memory_mapped_file);
 }
 
 }  // namespace plasma
@@ -733,9 +744,11 @@ int main(int argc, char* argv[]) {
   // Directory where plasma memory mapped files are stored.
   std::string plasma_directory;
   bool hugepages_enabled = false;
+  // True if a single large memory-mapped file should be created at startup.
+  bool use_one_memory_mapped_file = false;
   int64_t system_memory = -1;
   int c;
-  while ((c = getopt(argc, argv, "s:m:d:h")) != -1) {
+  while ((c = getopt(argc, argv, "s:m:d:hf")) != -1) {
     switch (c) {
       case 'd':
         plasma_directory = std::string(optarg);
@@ -755,6 +768,9 @@ int main(int argc, char* argv[]) {
                         << "GB of memory.";
         break;
       }
+      case 'f':
+        use_one_memory_mapped_file = true;
+        break;
       default:
         exit(-1);
     }
@@ -808,5 +824,6 @@ int main(int argc, char* argv[]) {
   // available.
   plasma::dlmalloc_set_footprint_limit((size_t)system_memory);
   ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
-  plasma::start_server(socket_name, system_memory, plasma_directory, hugepages_enabled);
+  plasma::start_server(socket_name, system_memory, plasma_directory, hugepages_enabled,
+                       use_one_memory_mapped_file);
 }
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index b73d92d14fd43..b28bd60c48ad4 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -102,7 +102,8 @@ def assert_get_object_equal(unit_test, client1, client2, object_id,
 
 def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
                        use_valgrind=False, use_profiler=False,
-                       stdout_file=None, stderr_file=None):
+                       stdout_file=None, stderr_file=None,
+                       use_one_memory_mapped_file=False):
     """Start a plasma store process.
     Args:
         use_valgrind (bool): True if the plasma store should be started inside
@@ -113,6 +114,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
             no redirection should happen, then this should be None.
         stderr_file: A file handle opened for writing to redirect stderr to. If
             no redirection should happen, then this should be None.
+        use_one_memory_mapped_file: If True, then the store will use only a
+            single memory-mapped file.
     Return:
         A tuple of the name of the plasma store socket and the process ID of
             the plasma store process.
@@ -124,6 +127,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
     command = [plasma_store_executable,
                "-s", plasma_store_name,
                "-m", str(plasma_store_memory)]
+    if use_one_memory_mapped_file:
+        command += ["-f"]
     if use_valgrind:
         pid = subprocess.Popen(["valgrind",
                                 "--track-origins=yes",
@@ -147,10 +152,14 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
 class TestPlasmaClient(object):
 
     def setup_method(self, test_method):
+        use_one_memory_mapped_file = (test_method ==
+                                      self.test_use_one_memory_mapped_file)
+
         import pyarrow.plasma as plasma
         # Start Plasma store.
         plasma_store_name, self.p = start_plasma_store(
-            use_valgrind=os.getenv("PLASMA_VALGRIND") == "1")
+            use_valgrind=os.getenv("PLASMA_VALGRIND") == "1",
+            use_one_memory_mapped_file=use_one_memory_mapped_file)
         # Connect to Plasma.
         self.plasma_client = plasma.connect(plasma_store_name, "", 64)
         # For the eviction test
@@ -720,3 +729,19 @@ def test_subscribe_deletions(self):
             assert object_ids[i] == recv_objid
             assert -1 == recv_dsize
             assert -1 == recv_msize
+
+    def test_use_one_memory_mapped_file(self):
+        # Fill the object store up with a large number of small objects and let
+        # them go out of scope.
+        for _ in range(100):
+            create_object(
+                self.plasma_client,
+                np.random.randint(1, DEFAULT_PLASMA_STORE_MEMORY // 20), 0)
+        # Create large objects that require the full object store size, and
+        # verify that they fit.
+        for _ in range(2):
+            create_object(self.plasma_client, DEFAULT_PLASMA_STORE_MEMORY, 0)
+        # Verify that an object that is too large does not fit.
+        with pytest.raises(pa.lib.PlasmaStoreFull):
+            create_object(self.plasma_client, DEFAULT_PLASMA_STORE_MEMORY + 1,
+                          0)

From f2806fa518583907a129b2ecb0b7ec8758b69e17 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 17 Nov 2017 18:29:49 -0500
Subject: [PATCH 1256/1644] ARROW-1559: [C++] Add Unique kernel and refactor
 DictionaryBuilder to be a stateful kernel

Only intended to implement selective categorical conversion in `to_pandas()` but it seems that there is a lot missing to do this in a clean fashion.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1266 from xhochy/ARROW-1559 and squashes the following commits:

50249652 [Wes McKinney] Fix MSVC linker issue
b6cb1ece [Wes McKinney] Export CastOptions
4ea3ce61 [Wes McKinney] Return NONE Datum in else branch of functions
4f969c6b [Wes McKinney] Move deprecation suppression after flag munging
7f557cc0 [Wes McKinney] Code review comments, disable C4996 warning (equivalent to -Wno-deprecated) in MSVC builds
84717461 [Wes McKinney] Do not compute hash table threshold on each iteration
ae8f2339 [Wes McKinney] Fix double to int64_t conversion warning
c1444a26 [Wes McKinney] Fix doxygen warnings
2de85961 [Wes McKinney] Add test cases for unique, dictionary_encode
383b46fd [Wes McKinney] Add Array methods for Unique, DictionaryEncode
0962f06b [Wes McKinney] Add cast method for Column, chunked_array and column factory functions
62c3cefd [Wes McKinney] Datum stubs
27151c47 [Wes McKinney] Implement Cast for chunked arrays, fix kernel implementation. Change kernel API to write to a single Datum
1bf2e2f4 [Wes McKinney] Fix bug with column using wrong type
eaadc3e5 [Wes McKinney] Use macros to reduce code duplication in DoubleTableSize
6b4f8f3c [Wes McKinney] Fix datetime64->date32 casting error raised by refactor
2c77a19e [Wes McKinney] Some Decimal->Decimal128 renaming. Add DecimalType base class
c07f91b3 [Wes McKinney] ARROW-1559: Add unique kernel
---
 LICENSE.txt                                   |   59 +
 cpp/CMakeLists.txt                            |    1 +
 cpp/build-support/clang_format_exclusions.txt |    2 +
 cpp/cmake_modules/SetupCxxFlags.cmake         |    9 +
 cpp/src/arrow/CMakeLists.txt                  |    4 +-
 cpp/src/arrow/array-test.cc                   |  356 +-----
 cpp/src/arrow/array.cc                        |    2 +-
 cpp/src/arrow/buffer.h                        |    3 +-
 cpp/src/arrow/builder-benchmark.cc            |   43 -
 cpp/src/arrow/builder.cc                      |  456 +------
 cpp/src/arrow/builder.h                       |  203 +--
 cpp/src/arrow/compute/CMakeLists.txt          |    4 +-
 cpp/src/arrow/compute/api.h                   |    4 +-
 cpp/src/arrow/compute/compute-benchmark.cc    |   88 ++
 cpp/src/arrow/compute/compute-test.cc         |  369 +++++-
 cpp/src/arrow/compute/context.cc              |    7 +-
 cpp/src/arrow/compute/kernel.h                |   97 +-
 cpp/src/arrow/compute/kernels/CMakeLists.txt  |   21 +
 cpp/src/arrow/compute/{ => kernels}/cast.cc   |  317 ++---
 cpp/src/arrow/compute/{ => kernels}/cast.h    |   33 +-
 cpp/src/arrow/compute/kernels/hash.cc         |  822 ++++++++++++
 cpp/src/arrow/compute/kernels/hash.h          |  106 ++
 .../arrow/compute/kernels/util-internal.cc    |   85 ++
 cpp/src/arrow/compute/kernels/util-internal.h |  105 ++
 cpp/src/arrow/pretty_print-test.cc            |    5 +-
 cpp/src/arrow/python/arrow_to_pandas.cc       |   15 +-
 cpp/src/arrow/python/numpy_to_arrow.cc        |   59 +-
 cpp/src/arrow/table.h                         |    3 +
 cpp/src/arrow/test-util.h                     |    4 +
 cpp/src/arrow/type-test.cc                    |    6 +-
 cpp/src/arrow/type.h                          |   27 +-
 cpp/src/arrow/type_fwd.h                      |    6 +
 cpp/src/arrow/type_traits.h                   |    4 +
 cpp/src/arrow/util/CMakeLists.txt             |    3 +
 cpp/src/arrow/util/variant.h                  | 1127 +++++++++++++++++
 cpp/src/arrow/util/variant/CMakeLists.txt     |   28 +
 cpp/src/arrow/util/variant/optional.h         |  100 ++
 .../arrow/util/variant/recursive_wrapper.h    |  122 ++
 cpp/src/arrow/util/variant/variant_cast.h     |  112 ++
 cpp/src/arrow/util/variant/variant_io.h       |   72 ++
 cpp/src/arrow/util/variant/variant_visitor.h  |   69 +
 cpp/src/arrow/visitor.cc                      |    2 +-
 cpp/src/arrow/visitor_inline.h                |    2 +-
 dev/release/rat_exclude_files.txt             |    6 +
 python/doc/source/api.rst                     |    3 +
 python/doc/source/development.rst             |   20 +-
 python/pyarrow/__init__.py                    |    2 +-
 python/pyarrow/array.pxi                      |   37 +-
 python/pyarrow/includes/libarrow.pxd          |   49 +
 python/pyarrow/pandas_compat.py               |    3 +-
 python/pyarrow/table.pxi                      |  129 +-
 python/pyarrow/tests/test_array.py            |   39 +
 python/pyarrow/tests/test_parquet.py          |    4 +
 python/pyarrow/types.pxi                      |    1 +
 54 files changed, 3947 insertions(+), 1308 deletions(-)
 create mode 100644 cpp/src/arrow/compute/compute-benchmark.cc
 create mode 100644 cpp/src/arrow/compute/kernels/CMakeLists.txt
 rename cpp/src/arrow/compute/{ => kernels}/cast.cc (82%)
 rename cpp/src/arrow/compute/{ => kernels}/cast.h (69%)
 create mode 100644 cpp/src/arrow/compute/kernels/hash.cc
 create mode 100644 cpp/src/arrow/compute/kernels/hash.h
 create mode 100644 cpp/src/arrow/compute/kernels/util-internal.cc
 create mode 100644 cpp/src/arrow/compute/kernels/util-internal.h
 create mode 100644 cpp/src/arrow/util/variant.h
 create mode 100644 cpp/src/arrow/util/variant/CMakeLists.txt
 create mode 100644 cpp/src/arrow/util/variant/optional.h
 create mode 100644 cpp/src/arrow/util/variant/recursive_wrapper.h
 create mode 100644 cpp/src/arrow/util/variant/variant_cast.h
 create mode 100644 cpp/src/arrow/util/variant/variant_io.h
 create mode 100644 cpp/src/arrow/util/variant/variant_visitor.h

diff --git a/LICENSE.txt b/LICENSE.txt
index 00cb9ece232b0..038518a5d9b68 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -398,3 +398,62 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the Boost project
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the mapbox/variant project, BSD 3-clause
+license
+
+Copyright (c) MapBox
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+- Neither the name "MapBox" nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5f0c431d54aca..f4b7b29b9d3f3 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -414,6 +414,7 @@ if (UNIX)
             (item MATCHES "xxhash.h") OR
             (item MATCHES "xxhash.cc") OR
             (item MATCHES "config.h") OR
+            (item MATCHES "util/variant") OR
             (item MATCHES "zmalloc.h") OR
             (item MATCHES "ae.h")))
       LIST(APPEND FILTERED_LINT_FILES ${item})
diff --git a/cpp/build-support/clang_format_exclusions.txt b/cpp/build-support/clang_format_exclusions.txt
index 2d5d86d4e4cde..d31d8a00d2ab7 100644
--- a/cpp/build-support/clang_format_exclusions.txt
+++ b/cpp/build-support/clang_format_exclusions.txt
@@ -3,6 +3,8 @@
 *pyarrow_api.h
 *python/config.h
 *python/platform.h
+*util/variant.h
+*util/variant/*
 *thirdparty/ae/*
 *xxhash.cc
 *xxhash.h
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 4b1950f7a17bd..6b0974b3d4c19 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -68,6 +68,9 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
+
+    # MSVC version of -Wno-deprecated
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
 -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
@@ -115,6 +118,9 @@ elseif ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "EVERYTHING")
     # /wdnnnn disables a warning where "nnnn" is a warning number
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}  /WX")
+
+    # MSVC version of -Wno-deprecated
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic")
     # Treat all compiler warnings as errors
@@ -134,6 +140,9 @@ else()
     # /wdnnnn disables a warning where "nnnn" is a warning number
     string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
+
+    # MSVC version of -Wno-deprecated
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
   elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 69d5052330c3a..496e0da9d621e 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -50,8 +50,10 @@ endif()
 if (ARROW_COMPUTE)
   add_subdirectory(compute)
   set(ARROW_SRCS ${ARROW_SRCS}
-    compute/cast.cc
     compute/context.cc
+    compute/kernels/cast.cc
+    compute/kernels/hash.cc
+    compute/kernels/util-internal.cc
   )
 endif()
 
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 15c75534e53ef..d894df1311e9e 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -402,10 +402,6 @@ typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PIn
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
-#define DECL_T() typedef typename TestFixture::T T;
-
-#define DECL_TYPE() typedef typename TestFixture::Type Type;
-
 TYPED_TEST(TestPrimitiveBuilder, TestInit) {
   DECL_TYPE();
 
@@ -1623,353 +1619,6 @@ TEST_F(TestAdaptiveUIntBuilder, TestAppendVector) {
   ASSERT_TRUE(expected_->Equals(result_));
 }
 
-// ----------------------------------------------------------------------
-// Dictionary tests
-
-template <typename Type>
-class TestDictionaryBuilder : public TestBuilder {};
-
-typedef ::testing::Types<Int8Type, UInt8Type, Int16Type, UInt16Type, Int32Type,
-                         UInt32Type, Int64Type, UInt64Type, FloatType, DoubleType>
-    PrimitiveDictionaries;
-
-TYPED_TEST_CASE(TestDictionaryBuilder, PrimitiveDictionaries);
-
-TYPED_TEST(TestDictionaryBuilder, Basic) {
-  DictionaryBuilder<TypeParam> builder(default_memory_pool());
-  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
-  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
-  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
-
-  std::shared_ptr<Array> result;
-  ASSERT_OK(builder.Finish(&result));
-
-  // Build expected data
-  NumericBuilder<TypeParam> dict_builder;
-  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
-  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
-  std::shared_ptr<Array> dict_array;
-  ASSERT_OK(dict_builder.Finish(&dict_array));
-  auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
-
-  Int8Builder int_builder;
-  ASSERT_OK(int_builder.Append(0));
-  ASSERT_OK(int_builder.Append(1));
-  ASSERT_OK(int_builder.Append(0));
-  std::shared_ptr<Array> int_array;
-  ASSERT_OK(int_builder.Finish(&int_array));
-
-  DictionaryArray expected(dtype, int_array);
-  ASSERT_TRUE(expected.Equals(result));
-}
-
-TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
-  NumericBuilder<TypeParam> builder;
-  // DictionaryBuilder<TypeParam> builder;
-  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
-  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
-  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
-
-  std::shared_ptr<Array> intermediate_result;
-  ASSERT_OK(builder.Finish(&intermediate_result));
-  DictionaryBuilder<TypeParam> dictionary_builder(default_memory_pool());
-  ASSERT_OK(dictionary_builder.AppendArray(*intermediate_result));
-  std::shared_ptr<Array> result;
-  ASSERT_OK(dictionary_builder.Finish(&result));
-
-  // Build expected data
-  NumericBuilder<TypeParam> dict_builder;
-  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
-  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
-  std::shared_ptr<Array> dict_array;
-  ASSERT_OK(dict_builder.Finish(&dict_array));
-  auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
-
-  Int8Builder int_builder;
-  ASSERT_OK(int_builder.Append(0));
-  ASSERT_OK(int_builder.Append(1));
-  ASSERT_OK(int_builder.Append(0));
-  std::shared_ptr<Array> int_array;
-  ASSERT_OK(int_builder.Finish(&int_array));
-
-  DictionaryArray expected(dtype, int_array);
-  ASSERT_TRUE(expected.Equals(result));
-}
-
-TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
-  using Scalar = typename TypeParam::c_type;
-  // Skip this test for (u)int8
-  if (sizeof(Scalar) > 1) {
-    // Build the dictionary Array
-    DictionaryBuilder<TypeParam> builder(default_memory_pool());
-    // Build expected data
-    NumericBuilder<TypeParam> dict_builder;
-    Int16Builder int_builder;
-
-    // Fill with 1024 different values
-    for (int64_t i = 0; i < 1024; i++) {
-      ASSERT_OK(builder.Append(static_cast<Scalar>(i)));
-      ASSERT_OK(dict_builder.Append(static_cast<Scalar>(i)));
-      ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
-    }
-    // Fill with an already existing value
-    for (int64_t i = 0; i < 1024; i++) {
-      ASSERT_OK(builder.Append(static_cast<Scalar>(1)));
-      ASSERT_OK(int_builder.Append(1));
-    }
-
-    // Finalize result
-    std::shared_ptr<Array> result;
-    ASSERT_OK(builder.Finish(&result));
-
-    // Finalize expected data
-    std::shared_ptr<Array> dict_array;
-    ASSERT_OK(dict_builder.Finish(&dict_array));
-    auto dtype = std::make_shared<DictionaryType>(int16(), dict_array);
-    std::shared_ptr<Array> int_array;
-    ASSERT_OK(int_builder.Finish(&int_array));
-
-    DictionaryArray expected(dtype, int_array);
-    ASSERT_TRUE(expected.Equals(result));
-  }
-}
-
-TEST(TestStringDictionaryBuilder, Basic) {
-  // Build the dictionary Array
-  StringDictionaryBuilder builder(default_memory_pool());
-  ASSERT_OK(builder.Append("test"));
-  ASSERT_OK(builder.Append("test2"));
-  ASSERT_OK(builder.Append("test"));
-
-  std::shared_ptr<Array> result;
-  ASSERT_OK(builder.Finish(&result));
-
-  // Build expected data
-  StringBuilder str_builder;
-  ASSERT_OK(str_builder.Append("test"));
-  ASSERT_OK(str_builder.Append("test2"));
-  std::shared_ptr<Array> str_array;
-  ASSERT_OK(str_builder.Finish(&str_array));
-  auto dtype = std::make_shared<DictionaryType>(int8(), str_array);
-
-  Int8Builder int_builder;
-  ASSERT_OK(int_builder.Append(0));
-  ASSERT_OK(int_builder.Append(1));
-  ASSERT_OK(int_builder.Append(0));
-  std::shared_ptr<Array> int_array;
-  ASSERT_OK(int_builder.Finish(&int_array));
-
-  DictionaryArray expected(dtype, int_array);
-  ASSERT_TRUE(expected.Equals(result));
-}
-
-TEST(TestStringDictionaryBuilder, DoubleTableSize) {
-  // Build the dictionary Array
-  StringDictionaryBuilder builder(default_memory_pool());
-  // Build expected data
-  StringBuilder str_builder;
-  Int16Builder int_builder;
-
-  // Fill with 1024 different values
-  for (int64_t i = 0; i < 1024; i++) {
-    std::stringstream ss;
-    ss << "test" << i;
-    ASSERT_OK(builder.Append(ss.str()));
-    ASSERT_OK(str_builder.Append(ss.str()));
-    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
-  }
-  // Fill with an already existing value
-  for (int64_t i = 0; i < 1024; i++) {
-    ASSERT_OK(builder.Append("test1"));
-    ASSERT_OK(int_builder.Append(1));
-  }
-
-  // Finalize result
-  std::shared_ptr<Array> result;
-  ASSERT_OK(builder.Finish(&result));
-
-  // Finalize expected data
-  std::shared_ptr<Array> str_array;
-  ASSERT_OK(str_builder.Finish(&str_array));
-  auto dtype = std::make_shared<DictionaryType>(int16(), str_array);
-  std::shared_ptr<Array> int_array;
-  ASSERT_OK(int_builder.Finish(&int_array));
-
-  DictionaryArray expected(dtype, int_array);
-  ASSERT_TRUE(expected.Equals(result));
-}
-
-TEST(TestFixedSizeBinaryDictionaryBuilder, Basic) {
-  // Build the dictionary Array
-  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
-                                                 default_memory_pool());
-  std::vector<uint8_t> test{12, 12, 11, 12};
-  std::vector<uint8_t> test2{12, 12, 11, 11};
-  ASSERT_OK(builder.Append(test.data()));
-  ASSERT_OK(builder.Append(test2.data()));
-  ASSERT_OK(builder.Append(test.data()));
-
-  std::shared_ptr<Array> result;
-  ASSERT_OK(builder.Finish(&result));
-
-  // Build expected data
-  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4));
-  ASSERT_OK(fsb_builder.Append(test.data()));
-  ASSERT_OK(fsb_builder.Append(test2.data()));
-  std::shared_ptr<Array> fsb_array;
-  ASSERT_OK(fsb_builder.Finish(&fsb_array));
-  auto dtype = std::make_shared<DictionaryType>(int8(), fsb_array);
-
-  Int8Builder int_builder;
-  ASSERT_OK(int_builder.Append(0));
-  ASSERT_OK(int_builder.Append(1));
-  ASSERT_OK(int_builder.Append(0));
-  std::shared_ptr<Array> int_array;
-  ASSERT_OK(int_builder.Finish(&int_array));
-
-  DictionaryArray expected(dtype, int_array);
-  ASSERT_TRUE(expected.Equals(result));
-}
-
-TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) {
-  // Build the dictionary Array
-  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
-                                                 default_memory_pool());
-  // Build expected data
-  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4));
-  Int16Builder int_builder;
-
-  // Fill with 1024 different values
-  for (int64_t i = 0; i < 1024; i++) {
-    std::vector<uint8_t> value{12, 12, static_cast<uint8_t>(i / 128),
-                               static_cast<uint8_t>(i % 128)};
-    ASSERT_OK(builder.Append(value.data()));
-    ASSERT_OK(fsb_builder.Append(value.data()));
-    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
-  }
-  // Fill with an already existing value
-  std::vector<uint8_t> known_value{12, 12, 0, 1};
-  for (int64_t i = 0; i < 1024; i++) {
-    ASSERT_OK(builder.Append(known_value.data()));
-    ASSERT_OK(int_builder.Append(1));
-  }
-
-  // Finalize result
-  std::shared_ptr<Array> result;
-  ASSERT_OK(builder.Finish(&result));
-
-  // Finalize expected data
-  std::shared_ptr<Array> fsb_array;
-  ASSERT_OK(fsb_builder.Finish(&fsb_array));
-  auto dtype = std::make_shared<DictionaryType>(int16(), fsb_array);
-  std::shared_ptr<Array> int_array;
-  ASSERT_OK(int_builder.Finish(&int_array));
-
-  DictionaryArray expected(dtype, int_array);
-  ASSERT_TRUE(expected.Equals(result));
-}
-
-TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) {
-  // Build the dictionary Array
-  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
-                                                 default_memory_pool());
-  // Build an array with different byte width
-  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(5));
-  std::vector<uint8_t> value{100, 1, 1, 1, 1};
-  ASSERT_OK(fsb_builder.Append(value.data()));
-  std::shared_ptr<Array> fsb_array;
-  ASSERT_OK(fsb_builder.Finish(&fsb_array));
-
-  ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array));
-}
-
-TEST(TestDecimalDictionaryBuilder, Basic) {
-  // Build the dictionary Array
-  const auto& decimal_type = arrow::decimal(2, 0);
-  DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type, default_memory_pool());
-
-  // Test data
-  std::vector<Decimal128> test{12, 12, 11, 12};
-  for (const auto& value : test) {
-    ASSERT_OK(builder.Append(value.ToBytes().data()));
-  }
-
-  std::shared_ptr<Array> result;
-  ASSERT_OK(builder.Finish(&result));
-
-  // Build expected data
-  FixedSizeBinaryBuilder decimal_builder(decimal_type);
-  ASSERT_OK(decimal_builder.Append(Decimal128(12).ToBytes()));
-  ASSERT_OK(decimal_builder.Append(Decimal128(11).ToBytes()));
-
-  std::shared_ptr<Array> decimal_array;
-  ASSERT_OK(decimal_builder.Finish(&decimal_array));
-  auto dtype = arrow::dictionary(int8(), decimal_array);
-
-  Int8Builder int_builder;
-  ASSERT_OK(int_builder.Append({0, 0, 1, 0}));
-  std::shared_ptr<Array> int_array;
-  ASSERT_OK(int_builder.Finish(&int_array));
-
-  DictionaryArray expected(dtype, int_array);
-  ASSERT_TRUE(expected.Equals(result));
-}
-
-TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
-  const auto& decimal_type = arrow::decimal(21, 0);
-
-  // Build the dictionary Array
-  DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type, default_memory_pool());
-
-  // Build expected data
-  FixedSizeBinaryBuilder fsb_builder(decimal_type);
-  Int16Builder int_builder;
-
-  // Fill with 1024 different values
-  for (int64_t i = 0; i < 1024; i++) {
-    const uint8_t bytes[] = {0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             0,
-                             12,
-                             12,
-                             static_cast<uint8_t>(i / 128),
-                             static_cast<uint8_t>(i % 128)};
-    ASSERT_OK(builder.Append(bytes));
-    ASSERT_OK(fsb_builder.Append(bytes));
-    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
-  }
-  // Fill with an already existing value
-  const uint8_t known_value[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1};
-  for (int64_t i = 0; i < 1024; i++) {
-    ASSERT_OK(builder.Append(known_value));
-    ASSERT_OK(int_builder.Append(1));
-  }
-
-  // Finalize result
-  std::shared_ptr<Array> result;
-  ASSERT_OK(builder.Finish(&result));
-
-  // Finalize expected data
-  std::shared_ptr<Array> fsb_array;
-  ASSERT_OK(fsb_builder.Finish(&fsb_array));
-
-  auto dtype = std::make_shared<DictionaryType>(int16(), fsb_array);
-  std::shared_ptr<Array> int_array;
-  ASSERT_OK(int_builder.Finish(&int_array));
-
-  DictionaryArray expected(dtype, int_array);
-  ASSERT_TRUE(expected.Equals(result));
-}
-
 // ----------------------------------------------------------------------
 // List tests
 
@@ -2766,9 +2415,8 @@ class DecimalTest : public ::testing::TestWithParam<int> {
   template <size_t BYTE_WIDTH = 16>
   void TestCreate(int32_t precision, const DecimalVector& draw,
                   const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
-    auto type = std::make_shared<DecimalType>(precision, 4);
-
-    auto builder = std::make_shared<DecimalBuilder>(type);
+    auto type = std::make_shared<Decimal128Type>(precision, 4);
+    auto builder = std::make_shared<Decimal128Builder>(type);
 
     size_t null_count = 0;
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index f2dd75335df8b..4ceb071ac3338 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -92,7 +92,7 @@ static inline std::shared_ptr<ArrayData> SliceData(const ArrayData& data, int64_
   auto new_data = data.ShallowCopy();
   new_data->length = length;
   new_data->offset = offset;
-  new_data->null_count = kUnknownNullCount;
+  new_data->null_count = data.null_count != 0 ? kUnknownNullCount : 0;
   return new_data;
 }
 
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 7c5f6174fa6e9..450a4c78b5bbb 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -340,12 +340,13 @@ Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
 #ifndef ARROW_NO_DEPRECATED_API
 
 /// \brief Create Buffer referencing std::string memory
-/// \note Deprecated since 0.8.0
 ///
 /// Warning: string instance must stay alive
 ///
 /// \param str std::string instance
 /// \return std::shared_ptr<Buffer>
+///
+/// \note Deprecated Since 0.8.0
 static inline std::shared_ptr<Buffer> GetBufferFromString(const std::string& str) {
   return std::make_shared<Buffer>(str);
 }
diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc
index 7ac7fe3bed533..12dfbe8170846 100644
--- a/cpp/src/arrow/builder-benchmark.cc
+++ b/cpp/src/arrow/builder-benchmark.cc
@@ -115,47 +115,6 @@ static void BM_BuildAdaptiveUIntNoNulls(
   state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t));
 }
 
-static void BM_BuildDictionary(benchmark::State& state) {  // NOLINT non-const reference
-  const int64_t iterations = 1024;
-  while (state.KeepRunning()) {
-    DictionaryBuilder<Int64Type> builder(default_memory_pool());
-    for (int64_t i = 0; i < iterations; i++) {
-      for (int64_t j = 0; j < i; j++) {
-        ABORT_NOT_OK(builder.Append(j));
-      }
-    }
-    std::shared_ptr<Array> out;
-    ABORT_NOT_OK(builder.Finish(&out));
-  }
-  state.SetBytesProcessed(state.iterations() * iterations * (iterations + 1) / 2 *
-                          sizeof(int64_t));
-}
-
-static void BM_BuildStringDictionary(
-    benchmark::State& state) {  // NOLINT non-const reference
-  const int64_t iterations = 1024;
-  // Pre-render strings
-  std::vector<std::string> data;
-  for (int64_t i = 0; i < iterations; i++) {
-    std::stringstream ss;
-    ss << i;
-    data.push_back(ss.str());
-  }
-  while (state.KeepRunning()) {
-    StringDictionaryBuilder builder(default_memory_pool());
-    for (int64_t i = 0; i < iterations; i++) {
-      for (int64_t j = 0; j < i; j++) {
-        ABORT_NOT_OK(builder.Append(data[j]));
-      }
-    }
-    std::shared_ptr<Array> out;
-    ABORT_NOT_OK(builder.Finish(&out));
-  }
-  // Assuming a string here needs on average 2 bytes
-  state.SetBytesProcessed(state.iterations() * iterations * (iterations + 1) / 2 *
-                          sizeof(int32_t));
-}
-
 static void BM_BuildBinaryArray(benchmark::State& state) {  // NOLINT non-const reference
   const int64_t iterations = 1 << 20;
 
@@ -179,8 +138,6 @@ BENCHMARK(BM_BuildAdaptiveIntNoNullsScalarAppend)
     ->Repetitions(3)
     ->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildAdaptiveUIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 
 BENCHMARK(BM_BuildBinaryArray)->Repetitions(3)->Unit(benchmark::kMicrosecond);
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index d2d3dbdf7d13e..3e213fcd5ca70 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -40,7 +40,6 @@
 namespace arrow {
 
 using internal::AdaptiveIntBuilderBase;
-using internal::WrappedBinary;
 
 Status ArrayBuilder::AppendToBitmap(bool is_valid) {
   if (length_ == capacity_) {
@@ -814,338 +813,6 @@ Status BooleanBuilder::Append(const std::vector<bool>& values) {
   return Status::OK();
 }
 
-// ----------------------------------------------------------------------
-// DictionaryBuilder
-
-template <typename T>
-DictionaryBuilder<T>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
-                                        MemoryPool* pool)
-    : ArrayBuilder(type, pool),
-      hash_table_(new PoolBuffer(pool)),
-      hash_slots_(nullptr),
-      dict_builder_(type, pool),
-      values_builder_(pool),
-      byte_width_(-1) {
-  if (!::arrow::CpuInfo::initialized()) {
-    ::arrow::CpuInfo::Init();
-  }
-}
-
-DictionaryBuilder<NullType>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
-                                               MemoryPool* pool)
-    : ArrayBuilder(type, pool), values_builder_(pool) {
-  if (!::arrow::CpuInfo::initialized()) {
-    ::arrow::CpuInfo::Init();
-  }
-}
-
-DictionaryBuilder<NullType>::~DictionaryBuilder() {}
-
-template <>
-DictionaryBuilder<FixedSizeBinaryType>::DictionaryBuilder(
-    const std::shared_ptr<DataType>& type, MemoryPool* pool)
-    : ArrayBuilder(type, pool),
-      hash_table_(new PoolBuffer(pool)),
-      hash_slots_(nullptr),
-      dict_builder_(type, pool),
-      values_builder_(pool),
-      byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {
-  if (!::arrow::CpuInfo::initialized()) {
-    ::arrow::CpuInfo::Init();
-  }
-}
-
-template <typename T>
-Status DictionaryBuilder<T>::Init(int64_t elements) {
-  RETURN_NOT_OK(ArrayBuilder::Init(elements));
-
-  // Fill the initial hash table
-  RETURN_NOT_OK(hash_table_->Resize(sizeof(hash_slot_t) * kInitialHashTableSize));
-  hash_slots_ = reinterpret_cast<int32_t*>(hash_table_->mutable_data());
-  std::fill(hash_slots_, hash_slots_ + kInitialHashTableSize, kHashSlotEmpty);
-  hash_table_size_ = kInitialHashTableSize;
-  mod_bitmask_ = kInitialHashTableSize - 1;
-
-  return values_builder_.Init(elements);
-}
-
-Status DictionaryBuilder<NullType>::Init(int64_t elements) {
-  RETURN_NOT_OK(ArrayBuilder::Init(elements));
-  return values_builder_.Init(elements);
-}
-
-template <typename T>
-Status DictionaryBuilder<T>::Resize(int64_t capacity) {
-  if (capacity < kMinBuilderCapacity) {
-    capacity = kMinBuilderCapacity;
-  }
-
-  if (capacity_ == 0) {
-    return Init(capacity);
-  } else {
-    return ArrayBuilder::Resize(capacity);
-  }
-}
-
-Status DictionaryBuilder<NullType>::Resize(int64_t capacity) {
-  if (capacity < kMinBuilderCapacity) {
-    capacity = kMinBuilderCapacity;
-  }
-
-  if (capacity_ == 0) {
-    return Init(capacity);
-  } else {
-    return ArrayBuilder::Resize(capacity);
-  }
-}
-
-template <typename T>
-Status DictionaryBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
-  std::shared_ptr<Array> dictionary;
-  RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
-
-  RETURN_NOT_OK(values_builder_.FinishInternal(out));
-  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
-  return Status::OK();
-}
-
-Status DictionaryBuilder<NullType>::FinishInternal(std::shared_ptr<ArrayData>* out) {
-  std::shared_ptr<Array> dictionary = std::make_shared<NullArray>(0);
-
-  RETURN_NOT_OK(values_builder_.FinishInternal(out));
-  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
-  return Status::OK();
-}
-
-template <typename T>
-Status DictionaryBuilder<T>::Append(const Scalar& value) {
-  RETURN_NOT_OK(Reserve(1));
-  // Based on DictEncoder<DType>::Put
-  int j = HashValue(value) & mod_bitmask_;
-  hash_slot_t index = hash_slots_[j];
-
-  // Find an empty slot
-  while (kHashSlotEmpty != index && SlotDifferent(index, value)) {
-    // Linear probing
-    ++j;
-    if (j == hash_table_size_) {
-      j = 0;
-    }
-    index = hash_slots_[j];
-  }
-
-  if (index == kHashSlotEmpty) {
-    // Not in the hash table, so we insert it now
-    index = static_cast<hash_slot_t>(dict_builder_.length());
-    hash_slots_[j] = index;
-    RETURN_NOT_OK(AppendDictionary(value));
-
-    if (ARROW_PREDICT_FALSE(static_cast<int32_t>(dict_builder_.length()) >
-                            hash_table_size_ * kMaxHashTableLoad)) {
-      RETURN_NOT_OK(DoubleTableSize());
-    }
-  }
-
-  RETURN_NOT_OK(values_builder_.Append(index));
-
-  return Status::OK();
-}
-
-template <typename T>
-Status DictionaryBuilder<T>::AppendArray(const Array& array) {
-  const auto& numeric_array = static_cast<const NumericArray<T>&>(array);
-  for (int64_t i = 0; i < array.length(); i++) {
-    if (array.IsNull(i)) {
-      RETURN_NOT_OK(AppendNull());
-    } else {
-      RETURN_NOT_OK(Append(numeric_array.Value(i)));
-    }
-  }
-  return Status::OK();
-}
-
-Status DictionaryBuilder<NullType>::AppendArray(const Array& array) {
-  for (int64_t i = 0; i < array.length(); i++) {
-    RETURN_NOT_OK(AppendNull());
-  }
-  return Status::OK();
-}
-
-template <>
-Status DictionaryBuilder<FixedSizeBinaryType>::AppendArray(const Array& array) {
-  if (!type_->Equals(*array.type())) {
-    return Status::Invalid("Cannot append FixedSizeBinary array with non-matching type");
-  }
-
-  const auto& numeric_array = static_cast<const FixedSizeBinaryArray&>(array);
-  for (int64_t i = 0; i < array.length(); i++) {
-    if (array.IsNull(i)) {
-      RETURN_NOT_OK(AppendNull());
-    } else {
-      RETURN_NOT_OK(Append(numeric_array.Value(i)));
-    }
-  }
-  return Status::OK();
-}
-
-template <typename T>
-Status DictionaryBuilder<T>::AppendNull() {
-  return values_builder_.AppendNull();
-}
-
-Status DictionaryBuilder<NullType>::AppendNull() { return values_builder_.AppendNull(); }
-
-template <typename T>
-Status DictionaryBuilder<T>::DoubleTableSize() {
-  int new_size = hash_table_size_ * 2;
-  auto new_hash_table = std::make_shared<PoolBuffer>(pool_);
-
-  RETURN_NOT_OK(new_hash_table->Resize(sizeof(hash_slot_t) * new_size));
-  int32_t* new_hash_slots = reinterpret_cast<int32_t*>(new_hash_table->mutable_data());
-  std::fill(new_hash_slots, new_hash_slots + new_size, kHashSlotEmpty);
-  int new_mod_bitmask = new_size - 1;
-
-  for (int i = 0; i < hash_table_size_; ++i) {
-    hash_slot_t index = hash_slots_[i];
-
-    if (index == kHashSlotEmpty) {
-      continue;
-    }
-
-    // Compute the hash value mod the new table size to start looking for an
-    // empty slot
-    Scalar value = GetDictionaryValue(static_cast<int64_t>(index));
-
-    // Find an empty slot in the new hash table
-    int j = HashValue(value) & new_mod_bitmask;
-    hash_slot_t slot = new_hash_slots[j];
-
-    while (kHashSlotEmpty != slot && SlotDifferent(slot, value)) {
-      ++j;
-      if (j == new_size) {
-        j = 0;
-      }
-      slot = new_hash_slots[j];
-    }
-
-    // Copy the old slot index to the new hash table
-    new_hash_slots[j] = index;
-  }
-
-  hash_table_ = new_hash_table;
-  hash_slots_ = reinterpret_cast<int32_t*>(hash_table_->mutable_data());
-  hash_table_size_ = new_size;
-  mod_bitmask_ = new_size - 1;
-
-  return Status::OK();
-}
-
-template <typename T>
-typename DictionaryBuilder<T>::Scalar DictionaryBuilder<T>::GetDictionaryValue(
-    int64_t index) {
-  const Scalar* data = reinterpret_cast<const Scalar*>(dict_builder_.data()->data());
-  return data[index];
-}
-
-template <>
-const uint8_t* DictionaryBuilder<FixedSizeBinaryType>::GetDictionaryValue(int64_t index) {
-  return dict_builder_.GetValue(index);
-}
-
-template <typename T>
-int DictionaryBuilder<T>::HashValue(const Scalar& value) {
-  return HashUtil::Hash(&value, sizeof(Scalar), 0);
-}
-
-template <>
-int DictionaryBuilder<FixedSizeBinaryType>::HashValue(const Scalar& value) {
-  return HashUtil::Hash(value, byte_width_, 0);
-}
-
-template <typename T>
-bool DictionaryBuilder<T>::SlotDifferent(hash_slot_t index, const Scalar& value) {
-  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
-  return other != value;
-}
-
-template <>
-bool DictionaryBuilder<FixedSizeBinaryType>::SlotDifferent(hash_slot_t index,
-                                                           const Scalar& value) {
-  int32_t width = static_cast<const FixedSizeBinaryType&>(*type_).byte_width();
-  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
-  return memcmp(other, value, width) != 0;
-}
-
-template <typename T>
-Status DictionaryBuilder<T>::AppendDictionary(const Scalar& value) {
-  return dict_builder_.Append(value);
-}
-
-#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                     \
-  template <>                                                                       \
-  WrappedBinary DictionaryBuilder<Type>::GetDictionaryValue(int64_t index) {        \
-    int32_t v_len;                                                                  \
-    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len); \
-    return WrappedBinary(v, v_len);                                                 \
-  }                                                                                 \
-                                                                                    \
-  template <>                                                                       \
-  Status DictionaryBuilder<Type>::AppendDictionary(const WrappedBinary& value) {    \
-    return dict_builder_.Append(value.ptr_, value.length_);                         \
-  }                                                                                 \
-                                                                                    \
-  template <>                                                                       \
-  Status DictionaryBuilder<Type>::AppendArray(const Array& array) {                 \
-    const BinaryArray& binary_array = static_cast<const BinaryArray&>(array);       \
-    WrappedBinary value(nullptr, 0);                                                \
-    for (int64_t i = 0; i < array.length(); i++) {                                  \
-      if (array.IsNull(i)) {                                                        \
-        RETURN_NOT_OK(AppendNull());                                                \
-      } else {                                                                      \
-        value.ptr_ = binary_array.GetValue(i, &value.length_);                      \
-        RETURN_NOT_OK(Append(value));                                               \
-      }                                                                             \
-    }                                                                               \
-    return Status::OK();                                                            \
-  }                                                                                 \
-                                                                                    \
-  template <>                                                                       \
-  int DictionaryBuilder<Type>::HashValue(const WrappedBinary& value) {              \
-    return HashUtil::Hash(value.ptr_, value.length_, 0);                            \
-  }                                                                                 \
-                                                                                    \
-  template <>                                                                       \
-  bool DictionaryBuilder<Type>::SlotDifferent(hash_slot_t index,                    \
-                                              const WrappedBinary& value) {         \
-    int32_t other_length;                                                           \
-    const uint8_t* other_value =                                                    \
-        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);         \
-    return !(other_length == value.length_ &&                                       \
-             0 == memcmp(other_value, value.ptr_, value.length_));                  \
-  }
-
-BINARY_DICTIONARY_SPECIALIZATIONS(StringType);
-BINARY_DICTIONARY_SPECIALIZATIONS(BinaryType);
-
-template class DictionaryBuilder<UInt8Type>;
-template class DictionaryBuilder<UInt16Type>;
-template class DictionaryBuilder<UInt32Type>;
-template class DictionaryBuilder<UInt64Type>;
-template class DictionaryBuilder<Int8Type>;
-template class DictionaryBuilder<Int16Type>;
-template class DictionaryBuilder<Int32Type>;
-template class DictionaryBuilder<Int64Type>;
-template class DictionaryBuilder<Date32Type>;
-template class DictionaryBuilder<Date64Type>;
-template class DictionaryBuilder<Time32Type>;
-template class DictionaryBuilder<Time64Type>;
-template class DictionaryBuilder<TimestampType>;
-template class DictionaryBuilder<FloatType>;
-template class DictionaryBuilder<DoubleType>;
-template class DictionaryBuilder<FixedSizeBinaryType>;
-template class DictionaryBuilder<BinaryType>;
-template class DictionaryBuilder<StringType>;
-
 // ----------------------------------------------------------------------
 // Decimal128Builder
 
@@ -1446,7 +1113,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
       BUILDER_CASE(STRING, StringBuilder);
       BUILDER_CASE(BINARY, BinaryBuilder);
       BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder);
-      BUILDER_CASE(DECIMAL, DecimalBuilder);
+      BUILDER_CASE(DECIMAL, Decimal128Builder);
     case Type::LIST: {
       std::unique_ptr<ArrayBuilder> value_builder;
       std::shared_ptr<DataType> value_type =
@@ -1477,125 +1144,4 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
   }
 }
 
-#define DICTIONARY_BUILDER_CASE(ENUM, BuilderType) \
-  case Type::ENUM:                                 \
-    out->reset(new BuilderType(type, pool));       \
-    return Status::OK();
-
-Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
-                             std::shared_ptr<ArrayBuilder>* out) {
-  switch (type->id()) {
-    DICTIONARY_BUILDER_CASE(NA, DictionaryBuilder<NullType>);
-    DICTIONARY_BUILDER_CASE(UINT8, DictionaryBuilder<UInt8Type>);
-    DICTIONARY_BUILDER_CASE(INT8, DictionaryBuilder<Int8Type>);
-    DICTIONARY_BUILDER_CASE(UINT16, DictionaryBuilder<UInt16Type>);
-    DICTIONARY_BUILDER_CASE(INT16, DictionaryBuilder<Int16Type>);
-    DICTIONARY_BUILDER_CASE(UINT32, DictionaryBuilder<UInt32Type>);
-    DICTIONARY_BUILDER_CASE(INT32, DictionaryBuilder<Int32Type>);
-    DICTIONARY_BUILDER_CASE(UINT64, DictionaryBuilder<UInt64Type>);
-    DICTIONARY_BUILDER_CASE(INT64, DictionaryBuilder<Int64Type>);
-    DICTIONARY_BUILDER_CASE(DATE32, DictionaryBuilder<Date32Type>);
-    DICTIONARY_BUILDER_CASE(DATE64, DictionaryBuilder<Date64Type>);
-    DICTIONARY_BUILDER_CASE(TIME32, DictionaryBuilder<Time32Type>);
-    DICTIONARY_BUILDER_CASE(TIME64, DictionaryBuilder<Time64Type>);
-    DICTIONARY_BUILDER_CASE(TIMESTAMP, DictionaryBuilder<TimestampType>);
-    DICTIONARY_BUILDER_CASE(FLOAT, DictionaryBuilder<FloatType>);
-    DICTIONARY_BUILDER_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
-    DICTIONARY_BUILDER_CASE(STRING, StringDictionaryBuilder);
-    DICTIONARY_BUILDER_CASE(BINARY, BinaryDictionaryBuilder);
-    DICTIONARY_BUILDER_CASE(FIXED_SIZE_BINARY, DictionaryBuilder<FixedSizeBinaryType>);
-    DICTIONARY_BUILDER_CASE(DECIMAL, DictionaryBuilder<FixedSizeBinaryType>);
-    default:
-      return Status::NotImplemented(type->ToString());
-  }
-}
-
-#define DICTIONARY_ARRAY_CASE(ENUM, BuilderType)                           \
-  case Type::ENUM:                                                         \
-    builder = std::make_shared<BuilderType>(type, pool);                   \
-    RETURN_NOT_OK(static_cast<BuilderType&>(*builder).AppendArray(input)); \
-    RETURN_NOT_OK(builder->Finish(out));                                   \
-    return Status::OK();
-
-Status EncodeArrayToDictionary(const Array& input, MemoryPool* pool,
-                               std::shared_ptr<Array>* out) {
-  const std::shared_ptr<DataType>& type = input.data()->type;
-  std::shared_ptr<ArrayBuilder> builder;
-  switch (type->id()) {
-    DICTIONARY_ARRAY_CASE(NA, DictionaryBuilder<NullType>);
-    DICTIONARY_ARRAY_CASE(UINT8, DictionaryBuilder<UInt8Type>);
-    DICTIONARY_ARRAY_CASE(INT8, DictionaryBuilder<Int8Type>);
-    DICTIONARY_ARRAY_CASE(UINT16, DictionaryBuilder<UInt16Type>);
-    DICTIONARY_ARRAY_CASE(INT16, DictionaryBuilder<Int16Type>);
-    DICTIONARY_ARRAY_CASE(UINT32, DictionaryBuilder<UInt32Type>);
-    DICTIONARY_ARRAY_CASE(INT32, DictionaryBuilder<Int32Type>);
-    DICTIONARY_ARRAY_CASE(UINT64, DictionaryBuilder<UInt64Type>);
-    DICTIONARY_ARRAY_CASE(INT64, DictionaryBuilder<Int64Type>);
-    DICTIONARY_ARRAY_CASE(DATE32, DictionaryBuilder<Date32Type>);
-    DICTIONARY_ARRAY_CASE(DATE64, DictionaryBuilder<Date64Type>);
-    DICTIONARY_ARRAY_CASE(TIME32, DictionaryBuilder<Time32Type>);
-    DICTIONARY_ARRAY_CASE(TIME64, DictionaryBuilder<Time64Type>);
-    DICTIONARY_ARRAY_CASE(TIMESTAMP, DictionaryBuilder<TimestampType>);
-    DICTIONARY_ARRAY_CASE(FLOAT, DictionaryBuilder<FloatType>);
-    DICTIONARY_ARRAY_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
-    DICTIONARY_ARRAY_CASE(STRING, StringDictionaryBuilder);
-    DICTIONARY_ARRAY_CASE(BINARY, BinaryDictionaryBuilder);
-    DICTIONARY_ARRAY_CASE(FIXED_SIZE_BINARY, DictionaryBuilder<FixedSizeBinaryType>);
-    DICTIONARY_ARRAY_CASE(DECIMAL, DictionaryBuilder<FixedSizeBinaryType>);
-    default:
-      std::stringstream ss;
-      ss << "Cannot encode array of type " << type->ToString();
-      ss << " to dictionary";
-      return Status::NotImplemented(ss.str());
-  }
-}
-#define DICTIONARY_COLUMN_CASE(ENUM, BuilderType)                             \
-  case Type::ENUM:                                                            \
-    builder = std::make_shared<BuilderType>(type, pool);                      \
-    chunks = input.data();                                                    \
-    for (auto chunk : chunks->chunks()) {                                     \
-      RETURN_NOT_OK(static_cast<BuilderType&>(*builder).AppendArray(*chunk)); \
-    }                                                                         \
-    RETURN_NOT_OK(builder->Finish(&arr));                                     \
-    *out = std::make_shared<Column>(input.name(), arr);                       \
-    return Status::OK();
-
-/// \brief Encodes a column to a suitable dictionary type
-/// \param input Column to be encoded
-/// \param pool MemoryPool to allocate the dictionary
-/// \param out The new column
-/// \return Status
-Status EncodeColumnToDictionary(const Column& input, MemoryPool* pool,
-                                std::shared_ptr<Column>* out) {
-  const std::shared_ptr<DataType>& type = input.type();
-  std::shared_ptr<ArrayBuilder> builder;
-  std::shared_ptr<Array> arr;
-  std::shared_ptr<ChunkedArray> chunks;
-  switch (type->id()) {
-    DICTIONARY_COLUMN_CASE(UINT8, DictionaryBuilder<UInt8Type>);
-    DICTIONARY_COLUMN_CASE(INT8, DictionaryBuilder<Int8Type>);
-    DICTIONARY_COLUMN_CASE(UINT16, DictionaryBuilder<UInt16Type>);
-    DICTIONARY_COLUMN_CASE(INT16, DictionaryBuilder<Int16Type>);
-    DICTIONARY_COLUMN_CASE(UINT32, DictionaryBuilder<UInt32Type>);
-    DICTIONARY_COLUMN_CASE(INT32, DictionaryBuilder<Int32Type>);
-    DICTIONARY_COLUMN_CASE(UINT64, DictionaryBuilder<UInt64Type>);
-    DICTIONARY_COLUMN_CASE(INT64, DictionaryBuilder<Int64Type>);
-    DICTIONARY_COLUMN_CASE(DATE32, DictionaryBuilder<Date32Type>);
-    DICTIONARY_COLUMN_CASE(DATE64, DictionaryBuilder<Date64Type>);
-    DICTIONARY_COLUMN_CASE(TIME32, DictionaryBuilder<Time32Type>);
-    DICTIONARY_COLUMN_CASE(TIME64, DictionaryBuilder<Time64Type>);
-    DICTIONARY_COLUMN_CASE(TIMESTAMP, DictionaryBuilder<TimestampType>);
-    DICTIONARY_COLUMN_CASE(FLOAT, DictionaryBuilder<FloatType>);
-    DICTIONARY_COLUMN_CASE(DOUBLE, DictionaryBuilder<DoubleType>);
-    DICTIONARY_COLUMN_CASE(STRING, StringDictionaryBuilder);
-    DICTIONARY_COLUMN_CASE(BINARY, BinaryDictionaryBuilder);
-    DICTIONARY_COLUMN_CASE(FIXED_SIZE_BINARY, DictionaryBuilder<FixedSizeBinaryType>);
-    default:
-      std::stringstream ss;
-      ss << "Cannot encode column of type " << type->ToString();
-      ss << " to dictionary";
-      return Status::NotImplemented(ss.str());
-  }
-}
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index bc25d0d211934..32741b53ac438 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -123,6 +123,18 @@ class ARROW_EXPORT ArrayBuilder {
 
   std::shared_ptr<DataType> type() const { return type_; }
 
+  // Unsafe operations (don't check capacity/don't resize)
+
+  // Append to null bitmap.
+  void UnsafeAppendToBitmap(bool is_valid) {
+    if (is_valid) {
+      BitUtil::SetBit(null_bitmap_data_, length_);
+    } else {
+      ++null_count_;
+    }
+    ++length_;
+  }
+
  protected:
   ArrayBuilder() {}
 
@@ -143,18 +155,6 @@ class ARROW_EXPORT ArrayBuilder {
 
   void Reset();
 
-  // Unsafe operations (don't check capacity/don't resize)
-
-  // Append to null bitmap.
-  void UnsafeAppendToBitmap(bool is_valid) {
-    if (is_valid) {
-      BitUtil::SetBit(null_bitmap_data_, length_);
-    } else {
-      ++null_count_;
-    }
-    ++length_;
-  }
-
   // Vector append. Treat each zero byte as a nullzero. If valid_bytes is null
   // assume all of length bits are valid.
   void UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t length);
@@ -810,191 +810,12 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
   std::vector<std::unique_ptr<ArrayBuilder>> field_builders_;
 };
 
-// ----------------------------------------------------------------------
-// Dictionary builder
-
-// Based on Apache Parquet-cpp's DictEncoder
-
-// Initially 1024 elements
-static constexpr int kInitialHashTableSize = 1 << 10;
-
-typedef int32_t hash_slot_t;
-static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits<int32_t>::max();
-
-// The maximum load factor for the hash table before resizing.
-static constexpr double kMaxHashTableLoad = 0.7;
-
-namespace internal {
-
-// TODO(ARROW-1176): Use Tensorflow's StringPiece instead of this here.
-struct WrappedBinary {
-  WrappedBinary(const uint8_t* ptr, int32_t length) : ptr_(ptr), length_(length) {}
-
-  const uint8_t* ptr_;
-  int32_t length_;
-};
-
-template <typename T>
-struct DictionaryScalar {
-  using type = typename T::c_type;
-};
-
-template <>
-struct DictionaryScalar<BinaryType> {
-  using type = WrappedBinary;
-};
-
-template <>
-struct DictionaryScalar<StringType> {
-  using type = WrappedBinary;
-};
-
-template <>
-struct DictionaryScalar<FixedSizeBinaryType> {
-  using type = uint8_t const*;
-};
-
-}  // namespace internal
-
-/// \brief Array builder for created encoded DictionaryArray from dense array
-/// data
-template <typename T>
-class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
- public:
-  using Scalar = typename internal::DictionaryScalar<T>::type;
-
-  ~DictionaryBuilder() {}
-
-  DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
-
-  template <typename T1 = T>
-  explicit DictionaryBuilder(
-      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool)
-      : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
-
-  /// \brief Append a scalar value
-  Status Append(const Scalar& value);
-
-  /// \brief Append a scalar null value
-  Status AppendNull();
-
-  /// \brief Append a whole dense array to the builder
-  Status AppendArray(const Array& array);
-
-  Status Init(int64_t elements) override;
-  Status Resize(int64_t capacity) override;
-  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
-
- protected:
-  Status DoubleTableSize();
-  Scalar GetDictionaryValue(int64_t index);
-  int HashValue(const Scalar& value);
-  bool SlotDifferent(hash_slot_t slot, const Scalar& value);
-  Status AppendDictionary(const Scalar& value);
-
-  std::shared_ptr<PoolBuffer> hash_table_;
-  int32_t* hash_slots_;
-
-  /// Size of the table. Must be a power of 2.
-  int hash_table_size_;
-
-  // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j %
-  // hash_table_size_, but uses far fewer CPU cycles
-  int mod_bitmask_;
-
-  typename TypeTraits<T>::BuilderType dict_builder_;
-  AdaptiveIntBuilder values_builder_;
-  int32_t byte_width_;
-};
-
-template <>
-class ARROW_EXPORT DictionaryBuilder<NullType> : public ArrayBuilder {
- public:
-  ~DictionaryBuilder();
-
-  DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
-  explicit DictionaryBuilder(MemoryPool* pool);
-
-  /// \brief Append a scalar null value
-  Status AppendNull();
-
-  /// \brief Append a whole dense array to the builder
-  Status AppendArray(const Array& array);
-
-  Status Init(int64_t elements) override;
-  Status Resize(int64_t capacity) override;
-  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
-
- protected:
-  AdaptiveIntBuilder values_builder_;
-};
-
-class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
- public:
-  using DictionaryBuilder::Append;
-  using DictionaryBuilder::DictionaryBuilder;
-
-  Status Append(const uint8_t* value, int32_t length) {
-    return Append(internal::WrappedBinary(value, length));
-  }
-
-  Status Append(const char* value, int32_t length) {
-    return Append(
-        internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
-  }
-
-  Status Append(const std::string& value) {
-    return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
-                                          static_cast<int32_t>(value.size())));
-  }
-};
-
-/// \brief Dictionary array builder with convenience methods for strings
-class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
- public:
-  using DictionaryBuilder::Append;
-  using DictionaryBuilder::DictionaryBuilder;
-
-  Status Append(const uint8_t* value, int32_t length) {
-    return Append(internal::WrappedBinary(value, length));
-  }
-
-  Status Append(const char* value, int32_t length) {
-    return Append(
-        internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
-  }
-
-  Status Append(const std::string& value) {
-    return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
-                                          static_cast<int32_t>(value.size())));
-  }
-};
-
 // ----------------------------------------------------------------------
 // Helper functions
 
 Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
                                 std::unique_ptr<ArrayBuilder>* out);
 
-Status ARROW_EXPORT MakeDictionaryBuilder(MemoryPool* pool,
-                                          const std::shared_ptr<DataType>& type,
-                                          std::shared_ptr<ArrayBuilder>* out);
-
-/// \brief Convert Array to encoded DictionaryArray form
-///
-/// \param[in] input The Array to be encoded
-/// \param[in] pool MemoryPool to allocate memory for the hash table
-/// \param[out] out Array encoded to DictionaryArray
-Status ARROW_EXPORT EncodeArrayToDictionary(const Array& input, MemoryPool* pool,
-                                            std::shared_ptr<Array>* out);
-
-/// \brief Convert a Column's data internally to DictionaryArray
-///
-/// \param[in] input The ChunkedArray to be encoded
-/// \param[in] pool MemoryPool to allocate memory for the hash table
-/// \param[out] out Column with data converted to DictionaryArray
-Status ARROW_EXPORT EncodeColumnToDictionary(const Column& input, MemoryPool* pool,
-                                             std::shared_ptr<Column>* out);
 }  // namespace arrow
 
 #endif  // ARROW_BUILDER_H_
diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt
index 4589afb9574d3..d4369ed27b7c4 100644
--- a/cpp/src/arrow/compute/CMakeLists.txt
+++ b/cpp/src/arrow/compute/CMakeLists.txt
@@ -18,7 +18,6 @@
 # Headers: top level
 install(FILES
   api.h
-  cast.h
   context.h
   kernel.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute")
@@ -36,3 +35,6 @@ install(
 #######################################
 
 ADD_ARROW_TEST(compute-test)
+ADD_ARROW_BENCHMARK(compute-benchmark)
+
+add_subdirectory(kernels)
diff --git a/cpp/src/arrow/compute/api.h b/cpp/src/arrow/compute/api.h
index da7df1cbbfafc..b3700b4c5813c 100644
--- a/cpp/src/arrow/compute/api.h
+++ b/cpp/src/arrow/compute/api.h
@@ -18,8 +18,10 @@
 #ifndef ARROW_COMPUTE_API_H
 #define ARROW_COMPUTE_API_H
 
-#include "arrow/compute/cast.h"
 #include "arrow/compute/context.h"
 #include "arrow/compute/kernel.h"
 
+#include "arrow/compute/kernels/cast.h"
+#include "arrow/compute/kernels/hash.h"
+
 #endif  // ARROW_COMPUTE_API_H
diff --git a/cpp/src/arrow/compute/compute-benchmark.cc b/cpp/src/arrow/compute/compute-benchmark.cc
new file mode 100644
index 0000000000000..974fffcd6d99d
--- /dev/null
+++ b/cpp/src/arrow/compute/compute-benchmark.cc
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include <vector>
+
+#include "arrow/builder.h"
+#include "arrow/memory_pool.h"
+#include "arrow/test-util.h"
+
+#include "arrow/compute/context.h"
+#include "arrow/compute/kernels/hash.h"
+
+namespace arrow {
+namespace compute {
+
+static void BM_BuildDictionary(benchmark::State& state) {  // NOLINT non-const reference
+  const int64_t iterations = 1024;
+
+  std::vector<int64_t> values;
+  std::vector<bool> is_valid;
+  for (int64_t i = 0; i < iterations; i++) {
+    for (int64_t j = 0; j < i; j++) {
+      is_valid.push_back((i + j) % 9 == 0);
+      values.push_back(j);
+    }
+  }
+
+  std::shared_ptr<Array> arr;
+  ArrayFromVector<Int64Type, int64_t>(is_valid, values, &arr);
+
+  FunctionContext ctx;
+
+  while (state.KeepRunning()) {
+    Datum out;
+    ABORT_NOT_OK(DictionaryEncode(&ctx, Datum(arr), &out));
+  }
+  state.SetBytesProcessed(state.iterations() * values.size() * sizeof(int64_t));
+}
+
+static void BM_BuildStringDictionary(
+    benchmark::State& state) {  // NOLINT non-const reference
+  const int64_t iterations = 1024 * 64;
+  // Pre-render strings
+  std::vector<std::string> data;
+
+  int64_t total_bytes = 0;
+  for (int64_t i = 0; i < iterations; i++) {
+    std::stringstream ss;
+    ss << i;
+    auto val = ss.str();
+    data.push_back(val);
+    total_bytes += static_cast<int64_t>(val.size());
+  }
+
+  std::shared_ptr<Array> arr;
+  ArrayFromVector<StringType, std::string>(data, &arr);
+
+  FunctionContext ctx;
+
+  while (state.KeepRunning()) {
+    Datum out;
+    ABORT_NOT_OK(DictionaryEncode(&ctx, Datum(arr), &out));
+  }
+  // Assuming a string here needs on average 2 bytes
+  state.SetBytesProcessed(state.iterations() * total_bytes);
+}
+
+BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 61d53c4d50b33..5eada398d52fc 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -37,10 +37,12 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 
-#include "arrow/compute/cast.h"
 #include "arrow/compute/context.h"
 #include "arrow/compute/kernel.h"
+#include "arrow/compute/kernels/cast.h"
+#include "arrow/compute/kernels/hash.h"
 
+using std::shared_ptr;
 using std::vector;
 
 namespace arrow {
@@ -54,6 +56,18 @@ class ComputeFixture {
   FunctionContext ctx_;
 };
 
+template <typename Type, typename T>
+shared_ptr<Array> _MakeArray(const shared_ptr<DataType>& type, const vector<T>& values,
+                             const vector<bool>& is_valid) {
+  shared_ptr<Array> result;
+  if (is_valid.size() > 0) {
+    ArrayFromVector<Type, T>(type, is_valid, values, &result);
+  } else {
+    ArrayFromVector<Type, T>(type, values, &result);
+  }
+  return result;
+}
+
 // ----------------------------------------------------------------------
 // Cast
 
@@ -65,17 +79,17 @@ static void AssertBufferSame(const Array& left, const Array& right, int buffer_i
 class TestCast : public ComputeFixture, public TestBase {
  public:
   void CheckPass(const Array& input, const Array& expected,
-                 const std::shared_ptr<DataType>& out_type, const CastOptions& options) {
-    std::shared_ptr<Array> result;
+                 const shared_ptr<DataType>& out_type, const CastOptions& options) {
+    shared_ptr<Array> result;
     ASSERT_OK(Cast(&ctx_, input, out_type, options, &result));
     ASSERT_ARRAYS_EQUAL(expected, *result);
   }
 
   template <typename InType, typename I_TYPE>
-  void CheckFails(const std::shared_ptr<DataType>& in_type,
-                  const std::vector<I_TYPE>& in_values, const std::vector<bool>& is_valid,
-                  const std::shared_ptr<DataType>& out_type, const CastOptions& options) {
-    std::shared_ptr<Array> input, result;
+  void CheckFails(const shared_ptr<DataType>& in_type, const vector<I_TYPE>& in_values,
+                  const vector<bool>& is_valid, const shared_ptr<DataType>& out_type,
+                  const CastOptions& options) {
+    shared_ptr<Array> input, result;
     if (is_valid.size() > 0) {
       ArrayFromVector<InType, I_TYPE>(in_type, is_valid, in_values, &input);
     } else {
@@ -84,19 +98,18 @@ class TestCast : public ComputeFixture, public TestBase {
     ASSERT_RAISES(Invalid, Cast(&ctx_, *input, out_type, options, &result));
   }
 
-  void CheckZeroCopy(const Array& input, const std::shared_ptr<DataType>& out_type) {
-    std::shared_ptr<Array> result;
+  void CheckZeroCopy(const Array& input, const shared_ptr<DataType>& out_type) {
+    shared_ptr<Array> result;
     ASSERT_OK(Cast(&ctx_, input, out_type, {}, &result));
     AssertBufferSame(input, *result, 0);
     AssertBufferSame(input, *result, 1);
   }
 
   template <typename InType, typename I_TYPE, typename OutType, typename O_TYPE>
-  void CheckCase(const std::shared_ptr<DataType>& in_type,
-                 const std::vector<I_TYPE>& in_values, const std::vector<bool>& is_valid,
-                 const std::shared_ptr<DataType>& out_type,
-                 const std::vector<O_TYPE>& out_values, const CastOptions& options) {
-    std::shared_ptr<Array> input, expected;
+  void CheckCase(const shared_ptr<DataType>& in_type, const vector<I_TYPE>& in_values,
+                 const vector<bool>& is_valid, const shared_ptr<DataType>& out_type,
+                 const vector<O_TYPE>& out_values, const CastOptions& options) {
+    shared_ptr<Array> input, expected;
     if (is_valid.size() > 0) {
       ArrayFromVector<InType, I_TYPE>(in_type, is_valid, in_values, &input);
       ArrayFromVector<OutType, O_TYPE>(out_type, is_valid, out_values, &expected);
@@ -117,10 +130,10 @@ TEST_F(TestCast, SameTypeZeroCopy) {
   vector<bool> is_valid = {true, false, true, true, true};
   vector<int32_t> v1 = {0, 1, 2, 3, 4};
 
-  std::shared_ptr<Array> arr;
+  shared_ptr<Array> arr;
   ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
 
-  std::shared_ptr<Array> result;
+  shared_ptr<Array> result;
   ASSERT_OK(Cast(&this->ctx_, *arr, int32(), {}, &result));
 
   AssertBufferSame(*arr, *result, 0);
@@ -185,7 +198,7 @@ TEST_F(TestCast, OverflowInNullSlot) {
   vector<int32_t> v11 = {0, 70000, 2000, 1000, 0};
   vector<int16_t> e11 = {0, 0, 2000, 1000, 0};
 
-  std::shared_ptr<Array> expected;
+  shared_ptr<Array> expected;
   ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, e11, &expected);
 
   auto buf = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(v11.data()),
@@ -280,8 +293,8 @@ TEST_F(TestCast, TimestampToTimestamp) {
 
   auto CheckTimestampCast = [this](
       const CastOptions& options, TimeUnit::type from_unit, TimeUnit::type to_unit,
-      const std::vector<int64_t>& from_values, const std::vector<int64_t>& to_values,
-      const std::vector<bool>& is_valid) {
+      const vector<int64_t>& from_values, const vector<int64_t>& to_values,
+      const vector<bool>& is_valid) {
     CheckCase<TimestampType, int64_t, TimestampType, int64_t>(
         timestamp(from_unit), from_values, is_valid, timestamp(to_unit), to_values,
         options);
@@ -315,8 +328,8 @@ TEST_F(TestCast, TimestampToTimestamp) {
   CheckTimestampCast(options, TimeUnit::MICRO, TimeUnit::NANO, v6, e6, is_valid);
 
   // Zero copy
-  std::shared_ptr<Array> arr;
   vector<int64_t> v7 = {0, 70000, 2000, 1000, 0};
+  shared_ptr<Array> arr;
   ArrayFromVector<TimestampType, int64_t>(timestamp(TimeUnit::SECOND), is_valid, v7,
                                           &arr);
   CheckZeroCopy(*arr, timestamp(TimeUnit::SECOND));
@@ -456,8 +469,8 @@ TEST_F(TestCast, TimeToTime) {
       time64(TimeUnit::MICRO), v6, is_valid, time64(TimeUnit::NANO), e6, options);
 
   // Zero copy
-  std::shared_ptr<Array> arr;
   vector<int64_t> v7 = {0, 70000, 2000, 1000, 0};
+  shared_ptr<Array> arr;
   ArrayFromVector<Time64Type, int64_t>(time64(TimeUnit::MICRO), is_valid, v7, &arr);
   CheckZeroCopy(*arr, time64(TimeUnit::MICRO));
 
@@ -516,9 +529,9 @@ TEST_F(TestCast, DateToDate) {
                                                       e1, options);
 
   // Zero copy
-  std::shared_ptr<Array> arr;
   vector<int32_t> v2 = {0, 70000, 2000, 1000, 0};
   vector<int64_t> v3 = {0, 70000, 2000, 1000, 0};
+  shared_ptr<Array> arr;
   ArrayFromVector<Date32Type, int32_t>(date32(), is_valid, v2, &arr);
   CheckZeroCopy(*arr, date32());
 
@@ -561,22 +574,54 @@ TEST_F(TestCast, ToDouble) {
                                                    options);
 }
 
+TEST_F(TestCast, ChunkedArray) {
+  vector<int16_t> values1 = {0, 1, 2};
+  vector<int16_t> values2 = {3, 4, 5};
+
+  auto type = int16();
+  auto out_type = int64();
+
+  auto a1 = _MakeArray<Int16Type, int16_t>(type, values1, {});
+  auto a2 = _MakeArray<Int16Type, int16_t>(type, values2, {});
+
+  ArrayVector arrays = {a1, a2};
+  auto carr = std::make_shared<ChunkedArray>(arrays);
+
+  CastOptions options;
+
+  Datum out;
+  ASSERT_OK(Cast(&this->ctx_, Datum(carr), out_type, options, &out));
+  ASSERT_EQ(Datum::CHUNKED_ARRAY, out.kind());
+
+  auto out_carr = out.chunked_array();
+
+  vector<int64_t> ex_values1 = {0, 1, 2};
+  vector<int64_t> ex_values2 = {3, 4, 5};
+  auto a3 = _MakeArray<Int64Type, int64_t>(out_type, ex_values1, {});
+  auto a4 = _MakeArray<Int64Type, int64_t>(out_type, ex_values2, {});
+
+  ArrayVector ex_arrays = {a3, a4};
+  auto ex_carr = std::make_shared<ChunkedArray>(ex_arrays);
+
+  ASSERT_TRUE(out.chunked_array()->Equals(*ex_carr));
+}
+
 TEST_F(TestCast, UnsupportedTarget) {
   vector<bool> is_valid = {true, false, true, true, true};
   vector<int32_t> v1 = {0, 1, 2, 3, 4};
 
-  std::shared_ptr<Array> arr;
+  shared_ptr<Array> arr;
   ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
 
-  std::shared_ptr<Array> result;
+  shared_ptr<Array> result;
   ASSERT_RAISES(NotImplemented, Cast(&this->ctx_, *arr, utf8(), {}, &result));
 }
 
 TEST_F(TestCast, DateTimeZeroCopy) {
   vector<bool> is_valid = {true, false, true, true, true};
 
-  std::shared_ptr<Array> arr;
   vector<int32_t> v1 = {0, 70000, 2000, 1000, 0};
+  shared_ptr<Array> arr;
   ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
 
   CheckZeroCopy(*arr, time32(TimeUnit::SECOND));
@@ -596,7 +641,7 @@ TEST_F(TestCast, FromNull) {
 
   NullArray arr(length);
 
-  std::shared_ptr<Array> result;
+  shared_ptr<Array> result;
   ASSERT_OK(Cast(&ctx_, arr, int32(), {}, &result));
 
   ASSERT_EQ(length, result->length());
@@ -614,7 +659,7 @@ TEST_F(TestCast, PreallocatedMemory) {
 
   const int64_t length = 5;
 
-  std::shared_ptr<Array> arr;
+  shared_ptr<Array> arr;
   vector<int32_t> v1 = {0, 70000, 2000, 1000, 0};
   vector<int64_t> e1 = {0, 70000, 2000, 1000, 0};
   ArrayFromVector<Int32Type, int32_t>(int32(), is_valid, v1, &arr);
@@ -626,19 +671,20 @@ TEST_F(TestCast, PreallocatedMemory) {
 
   auto out_data = std::make_shared<ArrayData>(out_type, length);
 
-  std::shared_ptr<Buffer> out_values;
+  shared_ptr<Buffer> out_values;
   ASSERT_OK(this->ctx_.Allocate(length * sizeof(int64_t), &out_values));
 
   out_data->buffers.push_back(nullptr);
   out_data->buffers.push_back(out_values);
 
-  ASSERT_OK(kernel->Call(&this->ctx_, *arr, out_data.get()));
+  Datum out(out_data);
+  ASSERT_OK(kernel->Call(&this->ctx_, *arr->data(), &out));
 
   // Buffer address unchanged
   ASSERT_EQ(out_values.get(), out_data->buffers[1].get());
 
-  std::shared_ptr<Array> result = MakeArray(out_data);
-  std::shared_ptr<Array> expected;
+  shared_ptr<Array> result = MakeArray(out_data);
+  shared_ptr<Array> expected;
   ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, e1, &expected);
 
   ASSERT_ARRAYS_EQUAL(*expected, *result);
@@ -656,13 +702,268 @@ TYPED_TEST_CASE(TestDictionaryCast, TestTypes);
 
 TYPED_TEST(TestDictionaryCast, Basic) {
   CastOptions options;
-  std::shared_ptr<Array> plain_array =
+  shared_ptr<Array> plain_array =
+      TestBase::MakeRandomArray<typename TypeTraits<TypeParam>::ArrayType>(10, 2);
+
+  Datum out;
+  ASSERT_OK(DictionaryEncode(&this->ctx_, Datum(plain_array->data()), &out));
+
+  this->CheckPass(*MakeArray(out.array()), *plain_array, plain_array->type(), options);
+}
+
+/*TYPED_TEST(TestDictionaryCast, Reverse) {
+  CastOptions options;
+  shared_ptr<Array> plain_array =
       TestBase::MakeRandomArray<typename TypeTraits<TypeParam>::ArrayType>(10, 2);
 
-  std::shared_ptr<Array> dict_array;
+  shared_ptr<Array> dict_array;
   ASSERT_OK(EncodeArrayToDictionary(*plain_array, this->pool_, &dict_array));
 
-  this->CheckPass(*dict_array, *plain_array, plain_array->type(), options);
+  this->CheckPass(*plain_array, *dict_array, dict_array->type(), options);
+}*/
+
+// ----------------------------------------------------------------------
+// Dictionary tests
+
+template <typename Type, typename T>
+void CheckUnique(FunctionContext* ctx, const shared_ptr<DataType>& type,
+                 const vector<T>& in_values, const vector<bool>& in_is_valid,
+                 const vector<T>& out_values, const vector<bool>& out_is_valid) {
+  shared_ptr<Array> input = _MakeArray<Type, T>(type, in_values, in_is_valid);
+  shared_ptr<Array> expected = _MakeArray<Type, T>(type, out_values, out_is_valid);
+
+  shared_ptr<Array> result;
+  ASSERT_OK(Unique(ctx, Datum(input), &result));
+  ASSERT_ARRAYS_EQUAL(*expected, *result);
+}
+
+template <typename Type, typename T>
+void CheckDictEncode(FunctionContext* ctx, const shared_ptr<DataType>& type,
+                     const vector<T>& in_values, const vector<bool>& in_is_valid,
+                     const vector<T>& out_values, const vector<bool>& out_is_valid,
+                     const vector<int32_t>& out_indices) {
+  shared_ptr<Array> input = _MakeArray<Type, T>(type, in_values, in_is_valid);
+  shared_ptr<Array> ex_dict = _MakeArray<Type, T>(type, out_values, out_is_valid);
+  shared_ptr<Array> ex_indices =
+      _MakeArray<Int32Type, int32_t>(int32(), out_indices, in_is_valid);
+
+  DictionaryArray expected(dictionary(int32(), ex_dict), ex_indices);
+
+  Datum datum_out;
+  ASSERT_OK(DictionaryEncode(ctx, Datum(input), &datum_out));
+  shared_ptr<Array> result = MakeArray(datum_out.array());
+
+  ASSERT_ARRAYS_EQUAL(expected, *result);
+}
+
+class TestHashKernel : public ComputeFixture, public TestBase {};
+
+template <typename Type>
+class TestHashKernelPrimitive : public ComputeFixture, public TestBase {};
+
+typedef ::testing::Types<Int8Type, UInt8Type, Int16Type, UInt16Type, Int32Type,
+                         UInt32Type, Int64Type, UInt64Type, FloatType, DoubleType,
+                         Date32Type, Date64Type>
+    PrimitiveDictionaries;
+
+TYPED_TEST_CASE(TestHashKernelPrimitive, PrimitiveDictionaries);
+
+TYPED_TEST(TestHashKernelPrimitive, Unique) {
+  using T = typename TypeParam::c_type;
+  auto type = TypeTraits<TypeParam>::type_singleton();
+  CheckUnique<TypeParam, T>(&this->ctx_, type, {2, 1, 2, 1}, {true, false, true, true},
+                            {2, 1}, {});
+}
+
+TYPED_TEST(TestHashKernelPrimitive, DictEncode) {
+  using T = typename TypeParam::c_type;
+  auto type = TypeTraits<TypeParam>::type_singleton();
+  CheckDictEncode<TypeParam, T>(&this->ctx_, type, {2, 1, 2, 1, 2, 3},
+                                {true, false, true, true, true, true}, {2, 1, 3}, {},
+                                {0, 0, 0, 1, 0, 2});
+}
+
+TYPED_TEST(TestHashKernelPrimitive, PrimitiveResizeTable) {
+  using T = typename TypeParam::c_type;
+  // Skip this test for (u)int8
+  if (sizeof(Scalar) == 1) {
+    return;
+  }
+
+  const int64_t kTotalValues = 10000;
+  const int64_t kRepeats = 10;
+
+  vector<T> values;
+  vector<T> uniques;
+  vector<int32_t> indices;
+  for (int64_t i = 0; i < kTotalValues * kRepeats; i++) {
+    const auto val = static_cast<T>(i % kTotalValues);
+    values.push_back(val);
+
+    if (i < kTotalValues) {
+      uniques.push_back(val);
+    }
+    indices.push_back(static_cast<int32_t>(i % kTotalValues));
+  }
+
+  auto type = TypeTraits<TypeParam>::type_singleton();
+  CheckUnique<TypeParam, T>(&this->ctx_, type, values, {}, uniques, {});
+
+  CheckDictEncode<TypeParam, T>(&this->ctx_, type, values, {}, uniques, {}, indices);
+}
+
+TEST_F(TestHashKernel, UniqueTimeTimestamp) {
+  CheckUnique<Time32Type, int32_t>(&this->ctx_, time32(TimeUnit::SECOND), {2, 1, 2, 1},
+                                   {true, false, true, true}, {2, 1}, {});
+
+  CheckUnique<Time64Type, int64_t>(&this->ctx_, time64(TimeUnit::NANO), {2, 1, 2, 1},
+                                   {true, false, true, true}, {2, 1}, {});
+
+  CheckUnique<TimestampType, int64_t>(&this->ctx_, timestamp(TimeUnit::NANO),
+                                      {2, 1, 2, 1}, {true, false, true, true}, {2, 1},
+                                      {});
+}
+
+TEST_F(TestHashKernel, UniqueBinary) {
+  CheckUnique<BinaryType, std::string>(&this->ctx_, binary(),
+                                       {"test", "", "test2", "test"},
+                                       {true, false, true, true}, {"test", "test2"}, {});
+
+  CheckUnique<StringType, std::string>(&this->ctx_, utf8(), {"test", "", "test2", "test"},
+                                       {true, false, true, true}, {"test", "test2"}, {});
+}
+
+TEST_F(TestHashKernel, DictEncodeBinary) {
+  CheckDictEncode<BinaryType, std::string>(
+      &this->ctx_, binary(), {"test", "", "test2", "test", "baz"},
+      {true, false, true, true, true}, {"test", "test2", "baz"}, {}, {0, 0, 1, 0, 2});
+
+  CheckDictEncode<StringType, std::string>(
+      &this->ctx_, utf8(), {"test", "", "test2", "test", "baz"},
+      {true, false, true, true, true}, {"test", "test2", "baz"}, {}, {0, 0, 1, 0, 2});
+}
+
+TEST_F(TestHashKernel, BinaryResizeTable) {
+  const int64_t kTotalValues = 10000;
+  const int64_t kRepeats = 10;
+
+  vector<std::string> values;
+  vector<std::string> uniques;
+  vector<int32_t> indices;
+  for (int64_t i = 0; i < kTotalValues * kRepeats; i++) {
+    int64_t index = i % kTotalValues;
+    std::stringstream ss;
+    ss << "test" << index;
+    std::string val = ss.str();
+
+    values.push_back(val);
+
+    if (i < kTotalValues) {
+      uniques.push_back(val);
+    }
+    indices.push_back(static_cast<int32_t>(i % kTotalValues));
+  }
+
+  CheckUnique<BinaryType, std::string>(&this->ctx_, binary(), values, {}, uniques, {});
+  CheckDictEncode<BinaryType, std::string>(&this->ctx_, binary(), values, {}, uniques, {},
+                                           indices);
+
+  CheckUnique<StringType, std::string>(&this->ctx_, utf8(), values, {}, uniques, {});
+  CheckDictEncode<StringType, std::string>(&this->ctx_, utf8(), values, {}, uniques, {},
+                                           indices);
+}
+
+TEST_F(TestHashKernel, UniqueFixedSizeBinary) {
+  CheckUnique<FixedSizeBinaryType, std::string>(
+      &this->ctx_, fixed_size_binary(5), {"aaaaa", "", "bbbbb", "aaaaa"},
+      {true, false, true, true}, {"aaaaa", "bbbbb"}, {});
+}
+
+TEST_F(TestHashKernel, DictEncodeFixedSizeBinary) {
+  CheckDictEncode<FixedSizeBinaryType, std::string>(
+      &this->ctx_, fixed_size_binary(5), {"bbbbb", "", "bbbbb", "aaaaa", "ccccc"},
+      {true, false, true, true, true}, {"bbbbb", "aaaaa", "ccccc"}, {}, {0, 0, 0, 1, 2});
+}
+
+TEST_F(TestHashKernel, FixedSizeBinaryResizeTable) {
+  const int64_t kTotalValues = 10000;
+  const int64_t kRepeats = 10;
+
+  vector<std::string> values;
+  vector<std::string> uniques;
+  vector<int32_t> indices;
+  for (int64_t i = 0; i < kTotalValues * kRepeats; i++) {
+    int64_t index = i % kTotalValues;
+    std::stringstream ss;
+    ss << "test" << static_cast<char>(index / 128) << static_cast<char>(index % 128);
+    std::string val = ss.str();
+
+    values.push_back(val);
+
+    if (i < kTotalValues) {
+      uniques.push_back(val);
+    }
+    indices.push_back(static_cast<int32_t>(i % kTotalValues));
+  }
+
+  auto type = fixed_size_binary(6);
+  CheckUnique<FixedSizeBinaryType, std::string>(&this->ctx_, type, values, {}, uniques,
+                                                {});
+  CheckDictEncode<FixedSizeBinaryType, std::string>(&this->ctx_, type, values, {},
+                                                    uniques, {}, indices);
+}
+
+TEST_F(TestHashKernel, UniqueDecimal) {
+  vector<Decimal128> values{12, 12, 11, 12};
+  vector<Decimal128> expected{12, 11};
+
+  CheckUnique<Decimal128Type, Decimal128>(&this->ctx_, decimal(2, 0), values,
+                                          {true, false, true, true}, expected, {});
+}
+
+TEST_F(TestHashKernel, DictEncodeDecimal) {
+  vector<Decimal128> values{12, 12, 11, 12, 13};
+  vector<Decimal128> expected{12, 11, 13};
+
+  CheckDictEncode<Decimal128Type, Decimal128>(&this->ctx_, decimal(2, 0), values,
+                                              {true, false, true, true, true}, expected,
+                                              {}, {0, 0, 1, 0, 2});
+}
+
+TEST_F(TestHashKernel, ChunkedArrayInvoke) {
+  vector<std::string> values1 = {"foo", "bar", "foo"};
+  vector<std::string> values2 = {"bar", "baz", "quuux", "foo"};
+
+  auto type = utf8();
+  auto a1 = _MakeArray<StringType, std::string>(type, values1, {});
+  auto a2 = _MakeArray<StringType, std::string>(type, values2, {});
+
+  vector<std::string> dict_values = {"foo", "bar", "baz", "quuux"};
+  auto ex_dict = _MakeArray<StringType, std::string>(type, dict_values, {});
+
+  ArrayVector arrays = {a1, a2};
+  auto carr = std::make_shared<ChunkedArray>(arrays);
+
+  // Unique
+  shared_ptr<Array> result;
+  ASSERT_OK(Unique(&this->ctx_, Datum(carr), &result));
+  ASSERT_ARRAYS_EQUAL(*ex_dict, *result);
+
+  // Dictionary encode
+  auto dict_type = dictionary(int32(), ex_dict);
+
+  auto i1 = _MakeArray<Int32Type, int32_t>(int32(), {0, 1, 0}, {});
+  auto i2 = _MakeArray<Int32Type, int32_t>(int32(), {1, 2, 3, 0}, {});
+
+  ArrayVector dict_arrays = {std::make_shared<DictionaryArray>(dict_type, i1),
+                             std::make_shared<DictionaryArray>(dict_type, i2)};
+  auto dict_carr = std::make_shared<ChunkedArray>(dict_arrays);
+
+  Datum encoded_out;
+  ASSERT_OK(DictionaryEncode(&this->ctx_, Datum(carr), &encoded_out));
+  ASSERT_EQ(Datum::CHUNKED_ARRAY, encoded_out.kind());
+
+  ASSERT_TRUE(encoded_out.chunked_array()->Equals(*dict_carr));
 }
 
 }  // namespace compute
diff --git a/cpp/src/arrow/compute/context.cc b/cpp/src/arrow/compute/context.cc
index 792dc4f386939..63aa341a3d718 100644
--- a/cpp/src/arrow/compute/context.cc
+++ b/cpp/src/arrow/compute/context.cc
@@ -20,11 +20,16 @@
 #include <memory>
 
 #include "arrow/buffer.h"
+#include "arrow/util/cpu-info.h"
 
 namespace arrow {
 namespace compute {
 
-FunctionContext::FunctionContext(MemoryPool* pool) : pool_(pool) {}
+FunctionContext::FunctionContext(MemoryPool* pool) : pool_(pool) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
 
 MemoryPool* FunctionContext::memory_pool() const { return pool_; }
 
diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h
index 4e072a7c143ed..0037245d610b4 100644
--- a/cpp/src/arrow/compute/kernel.h
+++ b/cpp/src/arrow/compute/kernel.h
@@ -18,7 +18,14 @@
 #ifndef ARROW_COMPUTE_KERNEL_H
 #define ARROW_COMPUTE_KERNEL_H
 
+#include <memory>
+#include <vector>
+
 #include "arrow/array.h"
+#include "arrow/table.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/variant.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 namespace compute {
@@ -32,11 +39,99 @@ class ARROW_EXPORT OpKernel {
   virtual ~OpKernel() = default;
 };
 
+/// \brief Placeholder for Scalar values until we implement these
+struct ARROW_EXPORT Scalar {
+  ~Scalar() {}
+
+  ARROW_DISALLOW_COPY_AND_ASSIGN(Scalar);
+};
+
+/// \class Datum
+/// \brief Variant type for various Arrow C++ data structures
+struct ARROW_EXPORT Datum {
+  enum type { NONE, SCALAR, ARRAY, CHUNKED_ARRAY, RECORD_BATCH, TABLE, COLLECTION };
+
+  util::variant<decltype(NULLPTR), std::shared_ptr<Scalar>, std::shared_ptr<ArrayData>,
+                std::shared_ptr<ChunkedArray>, std::shared_ptr<RecordBatch>,
+                std::shared_ptr<Table>, std::vector<Datum>>
+      value;
+
+  /// \brief Empty datum, to be populated elsewhere
+  Datum() : value(nullptr) {}
+
+  explicit Datum(const std::shared_ptr<Scalar>& value) : value(value) {}
+
+  explicit Datum(const std::shared_ptr<ArrayData>& value) : value(value) {}
+
+  explicit Datum(const std::shared_ptr<Array>& value) : Datum(value->data()) {}
+
+  explicit Datum(const std::shared_ptr<ChunkedArray>& value) : value(value) {}
+
+  explicit Datum(const std::shared_ptr<RecordBatch>& value) : value(value) {}
+
+  explicit Datum(const std::shared_ptr<Table>& value) : value(value) {}
+
+  explicit Datum(const std::vector<Datum>& value) : value(value) {}
+
+  ~Datum() {}
+
+  Datum(const Datum& other) noexcept { this->value = other.value; }
+
+  Datum::type kind() const {
+    switch (this->value.which()) {
+      case 0:
+        return Datum::NONE;
+      case 1:
+        return Datum::SCALAR;
+      case 2:
+        return Datum::ARRAY;
+      case 3:
+        return Datum::CHUNKED_ARRAY;
+      case 4:
+        return Datum::RECORD_BATCH;
+      case 5:
+        return Datum::TABLE;
+      case 6:
+        return Datum::COLLECTION;
+      default:
+        return Datum::NONE;
+    }
+  }
+
+  std::shared_ptr<ArrayData> array() const {
+    return util::get<std::shared_ptr<ArrayData>>(this->value);
+  }
+
+  std::shared_ptr<ChunkedArray> chunked_array() const {
+    return util::get<std::shared_ptr<ChunkedArray>>(this->value);
+  }
+
+  const std::vector<Datum> collection() const {
+    return util::get<std::vector<Datum>>(this->value);
+  }
+
+  bool is_arraylike() const {
+    return this->kind() == Datum::ARRAY || this->kind() == Datum::CHUNKED_ARRAY;
+  }
+
+  /// \brief The value type of the variant, if any
+  ///
+  /// \return nullptr if no type
+  std::shared_ptr<DataType> type() const {
+    if (this->kind() == Datum::ARRAY) {
+      return util::get<std::shared_ptr<ArrayData>>(this->value)->type;
+    } else if (this->kind() == Datum::CHUNKED_ARRAY) {
+      return util::get<std::shared_ptr<ChunkedArray>>(this->value)->type();
+    }
+    return nullptr;
+  }
+};
+
 /// \class UnaryKernel
 /// \brief An array-valued function of a single input argument
 class ARROW_EXPORT UnaryKernel : public OpKernel {
  public:
-  virtual Status Call(FunctionContext* ctx, const Array& input, ArrayData* out) = 0;
+  virtual Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) = 0;
 };
 
 }  // namespace compute
diff --git a/cpp/src/arrow/compute/kernels/CMakeLists.txt b/cpp/src/arrow/compute/kernels/CMakeLists.txt
new file mode 100644
index 0000000000000..715e6c6612fe4
--- /dev/null
+++ b/cpp/src/arrow/compute/kernels/CMakeLists.txt
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+install(FILES
+  cast.h
+  hash.h
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute/kernels")
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc
similarity index 82%
rename from cpp/src/arrow/compute/cast.cc
rename to cpp/src/arrow/compute/kernels/cast.cc
index 114ab9af0d0d0..6a42ec8b2b807 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/kernels/cast.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/compute/cast.h"
+#include "arrow/compute/kernels/cast.h"
 
 #include <cstdint>
 #include <cstring>
@@ -26,6 +26,7 @@
 #include <string>
 #include <type_traits>
 #include <utility>
+#include <vector>
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
@@ -39,6 +40,7 @@
 
 #include "arrow/compute/context.h"
 #include "arrow/compute/kernel.h"
+#include "arrow/compute/kernels/util-internal.h"
 
 #ifdef ARROW_EXTRA_ERROR_CONTEXT
 
@@ -71,29 +73,6 @@ namespace compute {
 
 constexpr int64_t kMillisecondsInDay = 86400000;
 
-template <typename T>
-inline const T* GetValues(const ArrayData& data, int i) {
-  return reinterpret_cast<const T*>(data.buffers[i]->data()) + data.offset;
-}
-
-template <typename T>
-inline T* GetMutableValues(const ArrayData* data, int i) {
-  return reinterpret_cast<T*>(data->buffers[i]->mutable_data()) + data->offset;
-}
-
-namespace {
-
-void CopyData(const Array& input, ArrayData* output) {
-  auto in_data = input.data();
-  output->length = in_data->length;
-  output->null_count = input.null_count();
-  output->buffers = in_data->buffers;
-  output->offset = in_data->offset;
-  output->child_data = in_data->child_data;
-}
-
-}  // namespace
-
 // ----------------------------------------------------------------------
 // Zero copy casts
 
@@ -128,8 +107,8 @@ struct CastFunctor {};
 // Indicated no computation required
 template <typename O, typename I>
 struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     CopyData(input, output);
   }
 };
@@ -140,8 +119,8 @@ struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>
 template <typename T>
 struct CastFunctor<T, NullType, typename std::enable_if<
                                     std::is_base_of<FixedWidthType, T>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     // Simply initialize data to 0
     auto buf = output->buffers[1];
     DCHECK_EQ(output->offset, 0);
@@ -151,8 +130,8 @@ struct CastFunctor<T, NullType, typename std::enable_if<
 
 template <>
 struct CastFunctor<NullType, DictionaryType> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {}
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {}
 };
 
 // ----------------------------------------------------------------------
@@ -160,19 +139,17 @@ struct CastFunctor<NullType, DictionaryType> {
 
 // Cast from Boolean to other numbers
 template <typename T>
-struct CastFunctor<T, BooleanType,
-                   typename std::enable_if<std::is_base_of<Number, T>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+struct CastFunctor<T, BooleanType, enable_if_number<T>> {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     using c_type = typename T::c_type;
     constexpr auto kOne = static_cast<c_type>(1);
     constexpr auto kZero = static_cast<c_type>(0);
 
-    auto in_data = input.data();
-    internal::BitmapReader bit_reader(in_data->buffers[1]->data(), in_data->offset,
-                                      in_data->length);
+    internal::BitmapReader bit_reader(input.buffers[1]->data(), input.offset,
+                                      input.length);
     auto out = GetMutableValues<c_type>(output, 1);
-    for (int64_t i = 0; i < input.length(); ++i) {
+    for (int64_t i = 0; i < input.length; ++i) {
       *out++ = bit_reader.IsSet() ? kOne : kZero;
       bit_reader.Next();
     }
@@ -216,14 +193,14 @@ template <typename O, typename I>
 struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::value &&
                                                  std::is_base_of<Number, I>::value &&
                                                  !std::is_same<O, I>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     using in_type = typename I::c_type;
     DCHECK_EQ(output->offset, 0);
 
-    const in_type* in_data = GetValues<in_type>(*input.data(), 1);
+    const in_type* in_data = GetValues<in_type>(input, 1);
     uint8_t* out_data = GetMutableValues<uint8_t>(output, 1);
-    for (int64_t i = 0; i < input.length(); ++i) {
+    for (int64_t i = 0; i < input.length; ++i) {
       BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
     }
   }
@@ -232,25 +209,26 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::v
 template <typename O, typename I>
 struct CastFunctor<O, I,
                    typename std::enable_if<is_integer_downcast<O, I>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
     DCHECK_EQ(output->offset, 0);
 
-    auto in_offset = input.offset();
+    auto in_offset = input.offset;
 
-    const in_type* in_data = GetValues<in_type>(*input.data(), 1);
+    const in_type* in_data = GetValues<in_type>(input, 1);
     auto out_data = GetMutableValues<out_type>(output, 1);
 
     if (!options.allow_int_overflow) {
       constexpr in_type kMax = static_cast<in_type>(std::numeric_limits<out_type>::max());
       constexpr in_type kMin = static_cast<in_type>(std::numeric_limits<out_type>::min());
 
-      if (input.null_count() > 0) {
-        internal::BitmapReader is_valid_reader(input.data()->buffers[0]->data(),
-                                               in_offset, input.length());
-        for (int64_t i = 0; i < input.length(); ++i) {
+      // Null count may be -1 if the input array had been sliced
+      if (input.null_count != 0) {
+        internal::BitmapReader is_valid_reader(input.buffers[0]->data(), in_offset,
+                                               input.length);
+        for (int64_t i = 0; i < input.length; ++i) {
           if (ARROW_PREDICT_FALSE(is_valid_reader.IsSet() &&
                                   (*in_data > kMax || *in_data < kMin))) {
             ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
@@ -259,7 +237,7 @@ struct CastFunctor<O, I,
           is_valid_reader.Next();
         }
       } else {
-        for (int64_t i = 0; i < input.length(); ++i) {
+        for (int64_t i = 0; i < input.length; ++i) {
           if (ARROW_PREDICT_FALSE(*in_data > kMax || *in_data < kMin)) {
             ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
           }
@@ -267,7 +245,7 @@ struct CastFunctor<O, I,
         }
       }
     } else {
-      for (int64_t i = 0; i < input.length(); ++i) {
+      for (int64_t i = 0; i < input.length; ++i) {
         *out_data++ = static_cast<out_type>(*in_data++);
       }
     }
@@ -278,14 +256,14 @@ template <typename O, typename I>
 struct CastFunctor<O, I,
                    typename std::enable_if<is_numeric_cast<O, I>::value &&
                                            !is_integer_downcast<O, I>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
 
-    const in_type* in_data = GetValues<in_type>(*input.data(), 1);
+    const in_type* in_data = GetValues<in_type>(input, 1);
     auto out_data = GetMutableValues<out_type>(output, 1);
-    for (int64_t i = 0; i < input.length(); ++i) {
+    for (int64_t i = 0; i < input.length; ++i) {
       *out_data++ = static_cast<out_type>(*in_data++);
     }
   }
@@ -296,34 +274,52 @@ struct CastFunctor<O, I,
 
 template <typename in_type, typename out_type>
 void ShiftTime(FunctionContext* ctx, const CastOptions& options, const bool is_multiply,
-               const int64_t factor, const Array& input, ArrayData* output) {
-  const in_type* in_data = GetValues<in_type>(*input.data(), 1);
+               const int64_t factor, const ArrayData& input, ArrayData* output) {
+  const in_type* in_data = GetValues<in_type>(input, 1);
   auto out_data = GetMutableValues<out_type>(output, 1);
 
   if (factor == 1) {
-    for (int64_t i = 0; i < input.length(); i++) {
+    for (int64_t i = 0; i < input.length; i++) {
       out_data[i] = static_cast<out_type>(in_data[i]);
     }
   } else if (is_multiply) {
-    for (int64_t i = 0; i < input.length(); i++) {
+    for (int64_t i = 0; i < input.length; i++) {
       out_data[i] = static_cast<out_type>(in_data[i] * factor);
     }
   } else {
     if (options.allow_time_truncate) {
-      for (int64_t i = 0; i < input.length(); i++) {
+      for (int64_t i = 0; i < input.length; i++) {
         out_data[i] = static_cast<out_type>(in_data[i] / factor);
       }
     } else {
-      for (int64_t i = 0; i < input.length(); i++) {
-        out_data[i] = static_cast<out_type>(in_data[i] / factor);
-        if (input.IsValid(i) && (out_data[i] * factor != in_data[i])) {
-          std::stringstream ss;
-          ss << "Casting from " << input.type()->ToString() << " to "
-             << output->type->ToString() << " would lose data: " << in_data[i];
-          ctx->SetStatus(Status::Invalid(ss.str()));
-          break;
+#define RAISE_INVALID_CAST(VAL)                                                         \
+  std::stringstream ss;                                                                 \
+  ss << "Casting from " << input.type->ToString() << " to " << output->type->ToString() \
+     << " would lose data: " << VAL;                                                    \
+  ctx->SetStatus(Status::Invalid(ss.str()));
+
+      if (input.null_count != 0) {
+        internal::BitmapReader bit_reader(input.buffers[0]->data(), input.offset,
+                                          input.length);
+        for (int64_t i = 0; i < input.length; i++) {
+          out_data[i] = static_cast<out_type>(in_data[i] / factor);
+          if (bit_reader.IsSet() && (out_data[i] * factor != in_data[i])) {
+            RAISE_INVALID_CAST(in_data[i]);
+            break;
+          }
+          bit_reader.Next();
+        }
+      } else {
+        for (int64_t i = 0; i < input.length; i++) {
+          out_data[i] = static_cast<out_type>(in_data[i] / factor);
+          if (out_data[i] * factor != in_data[i]) {
+            RAISE_INVALID_CAST(in_data[i]);
+            break;
+          }
         }
       }
+
+#undef RAISE_INVALID_CAST
     }
   }
 }
@@ -342,10 +338,10 @@ const std::pair<bool, int64_t> kTimeConversionTable[4][4] = {
 
 template <>
 struct CastFunctor<TimestampType, TimestampType> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     // If units are the same, zero copy, otherwise convert
-    const auto& in_type = static_cast<const TimestampType&>(*input.type());
+    const auto& in_type = static_cast<const TimestampType&>(*input.type);
     const auto& out_type = static_cast<const TimestampType&>(*output->type);
 
     if (in_type.unit() == out_type.unit()) {
@@ -364,9 +360,9 @@ struct CastFunctor<TimestampType, TimestampType> {
 
 template <>
 struct CastFunctor<Date32Type, TimestampType> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
-    const auto& in_type = static_cast<const TimestampType&>(*input.type());
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
+    const auto& in_type = static_cast<const TimestampType&>(*input.type);
 
     static const int64_t kTimestampToDateFactors[4] = {
         86400LL,                             // SECOND
@@ -382,9 +378,9 @@ struct CastFunctor<Date32Type, TimestampType> {
 
 template <>
 struct CastFunctor<Date64Type, TimestampType> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
-    const auto& in_type = static_cast<const TimestampType&>(*input.type());
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
+    const auto& in_type = static_cast<const TimestampType&>(*input.type);
 
     std::pair<bool, int64_t> conversion =
         kTimeConversionTable[static_cast<int>(in_type.unit())]
@@ -393,17 +389,21 @@ struct CastFunctor<Date64Type, TimestampType> {
     ShiftTime<int64_t, int64_t>(ctx, options, conversion.first, conversion.second, input,
                                 output);
 
+    internal::BitmapReader bit_reader(input.buffers[0]->data(), input.offset,
+                                      input.length);
+
     // Ensure that intraday milliseconds have been zeroed out
     auto out_data = GetMutableValues<int64_t>(output, 1);
-    for (int64_t i = 0; i < input.length(); ++i) {
+    for (int64_t i = 0; i < input.length; ++i) {
       const int64_t remainder = out_data[i] % kMillisecondsInDay;
-      if (ARROW_PREDICT_FALSE(!options.allow_time_truncate && input.IsValid(i) &&
+      if (ARROW_PREDICT_FALSE(!options.allow_time_truncate && bit_reader.IsSet() &&
                               remainder > 0)) {
         ctx->SetStatus(
             Status::Invalid("Timestamp value had non-zero intraday milliseconds"));
         break;
       }
       out_data[i] -= remainder;
+      bit_reader.Next();
     }
   }
 };
@@ -415,13 +415,13 @@ template <typename O, typename I>
 struct CastFunctor<O, I,
                    typename std::enable_if<std::is_base_of<TimeType, I>::value &&
                                            std::is_base_of<TimeType, O>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     using in_t = typename I::c_type;
     using out_t = typename O::c_type;
 
     // If units are the same, zero copy, otherwise convert
-    const auto& in_type = static_cast<const I&>(*input.type());
+    const auto& in_type = static_cast<const I&>(*input.type);
     const auto& out_type = static_cast<const O&>(*output->type);
 
     if (in_type.unit() == out_type.unit()) {
@@ -443,16 +443,16 @@ struct CastFunctor<O, I,
 
 template <>
 struct CastFunctor<Date64Type, Date32Type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     ShiftTime<int32_t, int64_t>(ctx, options, true, kMillisecondsInDay, input, output);
   }
 };
 
 template <>
 struct CastFunctor<Date32Type, Date64Type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     ShiftTime<int64_t, int32_t>(ctx, options, false, kMillisecondsInDay, input, output);
   }
 };
@@ -487,10 +487,11 @@ template <typename T>
 struct CastFunctor<
     T, DictionaryType,
     typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
-    const DictionaryArray& dict_array = static_cast<const DictionaryArray&>(input);
-    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type());
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
+    DictionaryArray dict_array(input.ShallowCopy());
+
+    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
     const FixedSizeBinaryArray& dictionary =
         static_cast<const FixedSizeBinaryArray&>(*type.dictionary());
@@ -558,10 +559,11 @@ Status UnpackBinaryDictionary(FunctionContext* ctx, const Array& indices,
 template <typename T>
 struct CastFunctor<T, DictionaryType,
                    typename std::enable_if<std::is_base_of<BinaryType, T>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
-    const DictionaryArray& dict_array = static_cast<const DictionaryArray&>(input);
-    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type());
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
+    DictionaryArray dict_array(input.ShallowCopy());
+
+    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
     const BinaryArray& dictionary = static_cast<const BinaryArray&>(*type.dictionary());
 
@@ -617,12 +619,13 @@ void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
 template <typename T>
 struct CastFunctor<T, DictionaryType,
                    typename std::enable_if<IsNumeric<T>::value>::type> {
-  void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
-                  ArrayData* output) {
+  void operator()(FunctionContext* ctx, const CastOptions& options,
+                  const ArrayData& input, ArrayData* output) {
     using c_type = typename T::c_type;
 
-    const DictionaryArray& dict_array = static_cast<const DictionaryArray&>(input);
-    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type());
+    DictionaryArray dict_array(input.ShallowCopy());
+
+    const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
 
     // Check if values and output type match
@@ -657,24 +660,23 @@ struct CastFunctor<T, DictionaryType,
 
 // ----------------------------------------------------------------------
 
-typedef std::function<void(FunctionContext*, const CastOptions& options, const Array&,
+typedef std::function<void(FunctionContext*, const CastOptions& options, const ArrayData&,
                            ArrayData*)>
     CastFunction;
 
-static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input,
+static Status AllocateIfNotPreallocated(FunctionContext* ctx, const ArrayData& input,
                                         bool can_pre_allocate_values, ArrayData* out) {
-  const int64_t length = input.length();
-
-  out->null_count = input.null_count();
+  const int64_t length = input.length;
+  out->null_count = input.null_count;
 
   // Propagate bitmap unless we are null type
-  std::shared_ptr<Buffer> validity_bitmap = input.data()->buffers[0];
-  if (input.type_id() == Type::NA) {
+  std::shared_ptr<Buffer> validity_bitmap = input.buffers[0];
+  if (input.type->id() == Type::NA) {
     int64_t bitmap_size = BitUtil::BytesForBits(length);
     RETURN_NOT_OK(ctx->Allocate(bitmap_size, &validity_bitmap));
     memset(validity_bitmap->mutable_data(), 0, bitmap_size);
-  } else if (input.offset() != 0) {
-    RETURN_NOT_OK(CopyBitmap(ctx->memory_pool(), validity_bitmap->data(), input.offset(),
+  } else if (input.offset != 0) {
+    RETURN_NOT_OK(CopyBitmap(ctx->memory_pool(), validity_bitmap->data(), input.offset,
                              length, &validity_bitmap));
   }
 
@@ -727,17 +729,28 @@ static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input
 class CastKernel : public UnaryKernel {
  public:
   CastKernel(const CastOptions& options, const CastFunction& func, bool is_zero_copy,
-             bool can_pre_allocate_values)
+             bool can_pre_allocate_values, const std::shared_ptr<DataType>& out_type)
       : options_(options),
         func_(func),
         is_zero_copy_(is_zero_copy),
-        can_pre_allocate_values_(can_pre_allocate_values) {}
+        can_pre_allocate_values_(can_pre_allocate_values),
+        out_type_(out_type) {}
+
+  Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) override {
+    ArrayData* result;
+
+    if (out->kind() == Datum::NONE) {
+      out->value = std::make_shared<ArrayData>(out_type_, input.length);
+    }
+
+    result = out->array().get();
 
-  Status Call(FunctionContext* ctx, const Array& input, ArrayData* out) override {
     if (!is_zero_copy_) {
-      RETURN_NOT_OK(AllocateIfNotPreallocated(ctx, input, can_pre_allocate_values_, out));
+      RETURN_NOT_OK(
+          AllocateIfNotPreallocated(ctx, input, can_pre_allocate_values_, result));
     }
-    func_(ctx, options_, input, out);
+    func_(ctx, options_, input, result);
+
     RETURN_IF_ERROR(ctx);
     return Status::OK();
   }
@@ -747,18 +760,19 @@ class CastKernel : public UnaryKernel {
   CastFunction func_;
   bool is_zero_copy_;
   bool can_pre_allocate_values_;
+  std::shared_ptr<DataType> out_type_;
 };
 
-#define CAST_CASE(InType, OutType)                                                  \
-  case OutType::type_id:                                                            \
-    is_zero_copy = is_zero_copy_cast<OutType, InType>::value;                       \
-    can_pre_allocate_values =                                                       \
-        !(!is_binary_like(InType::type_id) && is_binary_like(OutType::type_id));    \
-    func = [](FunctionContext* ctx, const CastOptions& options, const Array& input, \
-              ArrayData* out) {                                                     \
-      CastFunctor<OutType, InType> func;                                            \
-      func(ctx, options, input, out);                                               \
-    };                                                                              \
+#define CAST_CASE(InType, OutType)                                                      \
+  case OutType::type_id:                                                                \
+    is_zero_copy = is_zero_copy_cast<OutType, InType>::value;                           \
+    can_pre_allocate_values =                                                           \
+        !(!is_binary_like(InType::type_id) && is_binary_like(OutType::type_id));        \
+    func = [](FunctionContext* ctx, const CastOptions& options, const ArrayData& input, \
+              ArrayData* out) {                                                         \
+      CastFunctor<OutType, InType> func;                                                \
+      func(ctx, options, input, out);                                                   \
+    };                                                                                  \
     break;
 
 #define NUMERIC_CASES(FN, IN_TYPE) \
@@ -832,26 +846,26 @@ class CastKernel : public UnaryKernel {
   FN(IN_TYPE, FloatType);             \
   FN(IN_TYPE, DoubleType);            \
   FN(IN_TYPE, FixedSizeBinaryType);   \
-  FN(IN_TYPE, DecimalType);           \
+  FN(IN_TYPE, Decimal128Type);        \
   FN(IN_TYPE, BinaryType);            \
   FN(IN_TYPE, StringType);
 
-#define GET_CAST_FUNCTION(CASE_GENERATOR, InType)                                \
-  static std::unique_ptr<UnaryKernel> Get##InType##CastFunc(                     \
-      const std::shared_ptr<DataType>& out_type, const CastOptions& options) {   \
-    CastFunction func;                                                           \
-    bool is_zero_copy = false;                                                   \
-    bool can_pre_allocate_values = true;                                         \
-    switch (out_type->id()) {                                                    \
-      CASE_GENERATOR(CAST_CASE, InType);                                         \
-      default:                                                                   \
-        break;                                                                   \
-    }                                                                            \
-    if (func != nullptr) {                                                       \
-      return std::unique_ptr<UnaryKernel>(                                       \
-          new CastKernel(options, func, is_zero_copy, can_pre_allocate_values)); \
-    }                                                                            \
-    return nullptr;                                                              \
+#define GET_CAST_FUNCTION(CASE_GENERATOR, InType)                              \
+  static std::unique_ptr<UnaryKernel> Get##InType##CastFunc(                   \
+      const std::shared_ptr<DataType>& out_type, const CastOptions& options) { \
+    CastFunction func;                                                         \
+    bool is_zero_copy = false;                                                 \
+    bool can_pre_allocate_values = true;                                       \
+    switch (out_type->id()) {                                                  \
+      CASE_GENERATOR(CAST_CASE, InType);                                       \
+      default:                                                                 \
+        break;                                                                 \
+    }                                                                          \
+    if (func != nullptr) {                                                     \
+      return std::unique_ptr<UnaryKernel>(new CastKernel(                      \
+          options, func, is_zero_copy, can_pre_allocate_values, out_type));    \
+    }                                                                          \
+    return nullptr;                                                            \
   }
 
 GET_CAST_FUNCTION(NULL_CASES, NullType);
@@ -912,18 +926,27 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>&
   return Status::OK();
 }
 
-Status Cast(FunctionContext* ctx, const Array& array,
+Status Cast(FunctionContext* ctx, const Datum& value,
             const std::shared_ptr<DataType>& out_type, const CastOptions& options,
-            std::shared_ptr<Array>* out) {
+            Datum* out) {
   // Dynamic dispatch to obtain right cast function
   std::unique_ptr<UnaryKernel> func;
-  RETURN_NOT_OK(GetCastFunction(*array.type(), out_type, options, &func));
+  RETURN_NOT_OK(GetCastFunction(*value.type(), out_type, options, &func));
 
-  // Data structure for output
-  auto out_data = std::make_shared<ArrayData>(out_type, array.length());
+  std::vector<Datum> result;
+  RETURN_NOT_OK(detail::InvokeUnaryArrayKernel(ctx, func.get(), value, &result));
 
-  RETURN_NOT_OK(func->Call(ctx, array, out_data.get()));
-  *out = MakeArray(out_data);
+  *out = detail::WrapDatumsLike(value, result);
+  return Status::OK();
+}
+
+Status Cast(FunctionContext* ctx, const Array& array,
+            const std::shared_ptr<DataType>& out_type, const CastOptions& options,
+            std::shared_ptr<Array>* out) {
+  Datum datum_out;
+  RETURN_NOT_OK(Cast(ctx, Datum(array.data()), out_type, options, &datum_out));
+  DCHECK_EQ(Datum::ARRAY, datum_out.kind());
+  *out = MakeArray(datum_out.array());
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/compute/cast.h b/cpp/src/arrow/compute/kernels/cast.h
similarity index 69%
rename from cpp/src/arrow/compute/cast.h
rename to cpp/src/arrow/compute/kernels/cast.h
index d7bde20d607db..b75bb7b6c15f4 100644
--- a/cpp/src/arrow/compute/cast.h
+++ b/cpp/src/arrow/compute/kernels/cast.h
@@ -15,25 +15,26 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef ARROW_COMPUTE_CAST_H
-#define ARROW_COMPUTE_CAST_H
+#ifndef ARROW_COMPUTE_KERNELS_CAST_H
+#define ARROW_COMPUTE_KERNELS_CAST_H
 
 #include <memory>
 
 #include "arrow/status.h"
 #include "arrow/util/visibility.h"
 
+#include "arrow/compute/kernel.h"
+
 namespace arrow {
 
 class Array;
+class ChunkedArray;
+class Column;
 class DataType;
 
 namespace compute {
 
-class FunctionContext;
-class UnaryKernel;
-
-struct CastOptions {
+struct ARROW_EXPORT CastOptions {
   CastOptions() : allow_int_overflow(false), allow_time_truncate(false) {}
 
   bool allow_int_overflow;
@@ -48,7 +49,7 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>&
 
 /// \brief Cast from one array type to another
 /// \param[in] context the FunctionContext
-/// \param[in] array array to cast
+/// \param[in] value array to cast
 /// \param[in] to_type type to cast to
 /// \param[in] options casting options
 /// \param[out] out resulting array
@@ -56,11 +57,25 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>&
 /// \since 0.7.0
 /// \note API not yet finalized
 ARROW_EXPORT
-Status Cast(FunctionContext* context, const Array& array,
+Status Cast(FunctionContext* context, const Array& value,
             const std::shared_ptr<DataType>& to_type, const CastOptions& options,
             std::shared_ptr<Array>* out);
 
+/// \brief Cast from one value to another
+/// \param[in] context the FunctionContext
+/// \param[in] value datum to cast
+/// \param[in] to_type type to cast to
+/// \param[in] options casting options
+/// \param[out] out resulting datum
+///
+/// \since 0.8.0
+/// \note API not yet finalized
+ARROW_EXPORT
+Status Cast(FunctionContext* context, const Datum& value,
+            const std::shared_ptr<DataType>& to_type, const CastOptions& options,
+            Datum* out);
+
 }  // namespace compute
 }  // namespace arrow
 
-#endif  // ARROW_COMPUTE_CAST_H
+#endif  // ARROW_COMPUTE_KERNELS_CAST_H
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
new file mode 100644
index 0000000000000..3af41609fef28
--- /dev/null
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -0,0 +1,822 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/compute/kernels/hash.h"
+
+#include <exception>
+#include <limits>
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "arrow/builder.h"
+#include "arrow/compute/context.h"
+#include "arrow/compute/kernel.h"
+#include "arrow/compute/kernels/util-internal.h"
+#include "arrow/util/hash-util.h"
+
+namespace arrow {
+namespace compute {
+
+namespace {
+
+// Initially 1024 elements
+static constexpr int64_t kInitialHashTableSize = 1 << 10;
+
+typedef int32_t hash_slot_t;
+static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits<int32_t>::max();
+
+// The maximum load factor for the hash table before resizing.
+static constexpr double kMaxHashTableLoad = 0.7;
+
+enum class SIMDMode : char { NOSIMD, SSE4, AVX2 };
+
+#define CHECK_IMPLEMENTED(KERNEL, FUNCNAME, TYPE)                  \
+  if (!KERNEL) {                                                   \
+    std::stringstream ss;                                          \
+    ss << FUNCNAME << " not implemented for " << type->ToString(); \
+    return Status::NotImplemented(ss.str());                       \
+  }
+
+Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr<Buffer>* out) {
+  auto hash_table = std::make_shared<PoolBuffer>(pool);
+
+  RETURN_NOT_OK(hash_table->Resize(sizeof(hash_slot_t) * size));
+  int32_t* slots = reinterpret_cast<hash_slot_t*>(hash_table->mutable_data());
+  std::fill(slots, slots + size, kHashSlotEmpty);
+
+  *out = hash_table;
+  return Status::OK();
+}
+
+// This is a slight design concession -- some hash actions have the possibility
+// of failure. Rather than introduce extra error checking into all actions, we
+// will raise an internal exception so that only the actions where errors can
+// occur will experience the extra overhead
+class HashException : public std::exception {
+ public:
+  explicit HashException(const std::string& msg, StatusCode code = StatusCode::Invalid)
+      : msg_(msg), code_(code) {}
+
+  ~HashException() throw() {}
+
+  const char* what() const throw() override;
+
+  StatusCode code() const { return code_; }
+
+ private:
+  std::string msg_;
+  StatusCode code_;
+};
+
+const char* HashException::what() const throw() { return msg_.c_str(); }
+
+class HashTable {
+ public:
+  HashTable(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : type_(type),
+        pool_(pool),
+        initialized_(false),
+        hash_table_(nullptr),
+        hash_slots_(nullptr),
+        hash_table_size_(0),
+        mod_bitmask_(0) {}
+
+  virtual ~HashTable() {}
+
+  virtual Status Append(const ArrayData& input) = 0;
+  virtual Status Flush(Datum* out) = 0;
+  virtual Status GetDictionary(std::shared_ptr<ArrayData>* out) = 0;
+
+ protected:
+  Status Init(int64_t elements);
+
+  std::shared_ptr<DataType> type_;
+  MemoryPool* pool_;
+  bool initialized_;
+
+  // The hash table contains integer indices that reference the set of observed
+  // distinct values
+  std::shared_ptr<Buffer> hash_table_;
+  hash_slot_t* hash_slots_;
+
+  /// Size of the table. Must be a power of 2.
+  int64_t hash_table_size_;
+
+  /// Size at which we decide to resize
+  int64_t hash_table_load_threshold_;
+
+  // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j %
+  // hash_table_size_, but uses far fewer CPU cycles
+  int64_t mod_bitmask_;
+};
+
+Status HashTable::Init(int64_t elements) {
+  DCHECK_EQ(elements, BitUtil::NextPower2(elements));
+  RETURN_NOT_OK(NewHashTable(elements, pool_, &hash_table_));
+  hash_slots_ = reinterpret_cast<hash_slot_t*>(hash_table_->mutable_data());
+  hash_table_size_ = elements;
+  hash_table_load_threshold_ =
+      static_cast<int64_t>(static_cast<double>(elements) * kMaxHashTableLoad);
+  mod_bitmask_ = elements - 1;
+  initialized_ = true;
+  return Status::OK();
+}
+
+template <typename Type, typename Action, typename Enable = void>
+class HashTableKernel : public HashTable {};
+
+// Types of hash actions
+//
+// unique: append to dictionary when not found, no-op with slot
+// dictionary-encode: append to dictionary when not found, append slot #
+// match: raise or set null when not found, otherwise append slot #
+// isin: set false when not found, otherwise true
+// value counts: append to dictionary when not found, increment count for slot
+
+template <typename Type, typename Enable = void>
+class HashDictionary {};
+
+// ----------------------------------------------------------------------
+// Hash table pass for nulls
+
+template <typename Type, typename Action>
+class HashTableKernel<Type, Action, enable_if_null<Type>> : public HashTable {
+ public:
+  using HashTable::HashTable;
+
+  Status Init() {
+    // No-op, do not even need to initialize hash table
+    return Status::OK();
+  }
+
+  Status Append(const ArrayData& arr) override {
+    if (!initialized_) {
+      RETURN_NOT_OK(Init());
+    }
+    auto action = static_cast<Action*>(this);
+    RETURN_NOT_OK(action->Reserve(arr.length));
+    for (int64_t i = 0; i < arr.length; ++i) {
+      action->ObserveNull();
+    }
+    return Status::OK();
+  }
+
+  Status GetDictionary(std::shared_ptr<ArrayData>* out) override {
+    // TODO(wesm): handle null being a valid dictionary value
+    auto null_array = std::make_shared<NullArray>(0);
+    *out = null_array->data();
+    return Status::OK();
+  }
+};
+
+// ----------------------------------------------------------------------
+// Hash table pass for primitive types
+
+template <typename Type>
+struct HashDictionary<Type, enable_if_has_c_type<Type>> {
+  using T = typename Type::c_type;
+
+  explicit HashDictionary(MemoryPool* pool)
+      : pool(pool), buffer(std::make_shared<PoolBuffer>(pool)), size(0), capacity(0) {}
+
+  Status Init() {
+    this->size = 0;
+    return Resize(kInitialHashTableSize);
+  }
+
+  Status DoubleSize() { return Resize(this->size * 2); }
+
+  Status Resize(const int64_t elements) {
+    RETURN_NOT_OK(this->buffer->Resize(elements * sizeof(T)));
+
+    this->capacity = elements;
+    this->values = reinterpret_cast<T*>(this->buffer->mutable_data());
+    return Status::OK();
+  }
+
+  MemoryPool* pool;
+  std::shared_ptr<ResizableBuffer> buffer;
+  T* values;
+  int64_t size;
+  int64_t capacity;
+};
+
+#define GENERIC_HASH_PASS(HASH_INNER_LOOP)                                               \
+  if (arr.null_count != 0) {                                                             \
+    internal::BitmapReader valid_reader(arr.buffers[0]->data(), arr.offset, arr.length); \
+    for (int64_t i = 0; i < arr.length; ++i) {                                           \
+      const bool is_null = valid_reader.IsNotSet();                                      \
+      valid_reader.Next();                                                               \
+                                                                                         \
+      if (is_null) {                                                                     \
+        action->ObserveNull();                                                           \
+        continue;                                                                        \
+      }                                                                                  \
+                                                                                         \
+      HASH_INNER_LOOP();                                                                 \
+    }                                                                                    \
+  } else {                                                                               \
+    for (int64_t i = 0; i < arr.length; ++i) {                                           \
+      HASH_INNER_LOOP();                                                                 \
+    }                                                                                    \
+  }
+
+#define DOUBLE_TABLE_SIZE(SETUP_CODE, COMPUTE_HASH)                              \
+  do {                                                                           \
+    int64_t new_size = hash_table_size_ * 2;                                     \
+                                                                                 \
+    std::shared_ptr<Buffer> new_hash_table;                                      \
+    RETURN_NOT_OK(NewHashTable(new_size, pool_, &new_hash_table));               \
+    int32_t* new_hash_slots =                                                    \
+        reinterpret_cast<hash_slot_t*>(new_hash_table->mutable_data());          \
+    int64_t new_mod_bitmask = new_size - 1;                                      \
+                                                                                 \
+    SETUP_CODE;                                                                  \
+                                                                                 \
+    for (int i = 0; i < hash_table_size_; ++i) {                                 \
+      hash_slot_t index = hash_slots_[i];                                        \
+                                                                                 \
+      if (index == kHashSlotEmpty) {                                             \
+        continue;                                                                \
+      }                                                                          \
+                                                                                 \
+      COMPUTE_HASH;                                                              \
+      while (kHashSlotEmpty != new_hash_slots[j]) {                              \
+        ++j;                                                                     \
+        if (ARROW_PREDICT_FALSE(j == hash_table_size_)) {                        \
+          j = 0;                                                                 \
+        }                                                                        \
+      }                                                                          \
+                                                                                 \
+      new_hash_slots[j] = index;                                                 \
+    }                                                                            \
+                                                                                 \
+    hash_table_ = new_hash_table;                                                \
+    hash_slots_ = reinterpret_cast<hash_slot_t*>(hash_table_->mutable_data());   \
+    hash_table_size_ = new_size;                                                 \
+    hash_table_load_threshold_ =                                                 \
+        static_cast<int64_t>(static_cast<double>(new_size) * kMaxHashTableLoad); \
+    mod_bitmask_ = new_size - 1;                                                 \
+  } while (false)
+
+template <typename Type, typename Action>
+class HashTableKernel<Type, Action, enable_if_has_c_type<Type>> : public HashTable {
+ public:
+  using T = typename Type::c_type;
+
+  HashTableKernel(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : HashTable(type, pool), dict_(pool) {}
+
+  Status Init() {
+    RETURN_NOT_OK(dict_.Init());
+    return HashTable::Init(kInitialHashTableSize);
+  }
+
+  Status Append(const ArrayData& arr) override {
+    if (!initialized_) {
+      RETURN_NOT_OK(Init());
+    }
+
+    const T* values = GetValues<T>(arr, 1);
+    auto action = static_cast<Action*>(this);
+
+    RETURN_NOT_OK(action->Reserve(arr.length));
+
+#define HASH_INNER_LOOP()                                               \
+  const T value = values[i];                                            \
+  int64_t j = HashValue(value) & mod_bitmask_;                          \
+  hash_slot_t slot = hash_slots_[j];                                    \
+                                                                        \
+  while (kHashSlotEmpty != slot && dict_.values[slot] != value) {       \
+    ++j;                                                                \
+    if (ARROW_PREDICT_FALSE(j == hash_table_size_)) {                   \
+      j = 0;                                                            \
+    }                                                                   \
+    slot = hash_slots_[j];                                              \
+  }                                                                     \
+                                                                        \
+  if (slot == kHashSlotEmpty) {                                         \
+    if (!Action::allow_expand) {                                        \
+      throw HashException("Encountered new dictionary value");          \
+    }                                                                   \
+                                                                        \
+    slot = static_cast<hash_slot_t>(dict_.size);                        \
+    hash_slots_[j] = slot;                                              \
+    dict_.values[dict_.size++] = value;                                 \
+                                                                        \
+    action->ObserveNotFound(slot);                                      \
+                                                                        \
+    if (ARROW_PREDICT_FALSE(dict_.size > hash_table_load_threshold_)) { \
+      RETURN_NOT_OK(action->DoubleSize());                              \
+    }                                                                   \
+  } else {                                                              \
+    action->ObserveFound(slot);                                         \
+  }
+
+    GENERIC_HASH_PASS(HASH_INNER_LOOP);
+
+#undef HASH_INNER_LOOP
+
+    return Status::OK();
+  }
+
+  Status GetDictionary(std::shared_ptr<ArrayData>* out) override {
+    // TODO(wesm): handle null being in the dictionary
+    auto dict_data = dict_.buffer;
+    RETURN_NOT_OK(dict_data->Resize(dict_.size * sizeof(T), false));
+
+    BufferVector buffers = {nullptr, dict_data};
+    *out = std::make_shared<ArrayData>(type_, dict_.size, std::move(buffers), 0);
+    return Status::OK();
+  }
+
+ protected:
+  int64_t HashValue(const T& value) const {
+    // TODO(wesm): Use faster hash function for C types
+    return HashUtil::Hash(&value, sizeof(T), 0);
+  }
+
+  Status DoubleTableSize() {
+#define PRIMITIVE_INNER_LOOP           \
+  const T value = dict_.values[index]; \
+  int64_t j = HashValue(value) & new_mod_bitmask;
+
+    DOUBLE_TABLE_SIZE(, PRIMITIVE_INNER_LOOP);
+
+#undef PRIMITIVE_INNER_LOOP
+
+    return dict_.Resize(hash_table_size_);
+  }
+
+  HashDictionary<Type> dict_;
+};
+
+// ----------------------------------------------------------------------
+// Hash table pass for variable-length binary types
+
+template <typename Type, typename Action>
+class HashTableKernel<Type, Action, enable_if_binary<Type>> : public HashTable {
+ public:
+  HashTableKernel(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : HashTable(type, pool), dict_offsets_(pool), dict_data_(pool), dict_size_(0) {}
+
+  Status Init() {
+    RETURN_NOT_OK(dict_offsets_.Resize(kInitialHashTableSize));
+
+    // We append the end offset after each append to the dictionary, so this
+    // sets the initial condition for the length-0 case
+    //
+    // initial offsets (dict size == 0): 0
+    // after 1st dict entry of length 3: 0 3
+    // after 2nd dict entry of length 4: 0 3 7
+    RETURN_NOT_OK(dict_offsets_.Append(0));
+    return HashTable::Init(kInitialHashTableSize);
+  }
+
+  Status Append(const ArrayData& arr) override {
+    if (!initialized_) {
+      RETURN_NOT_OK(Init());
+    }
+
+    const int32_t* offsets = GetValues<int32_t>(arr, 1);
+    const uint8_t* data = GetValues<uint8_t>(arr, 2);
+
+    auto action = static_cast<Action*>(this);
+    RETURN_NOT_OK(action->Reserve(arr.length));
+
+#define HASH_INNER_LOOP()                                                           \
+  const int32_t position = offsets[i];                                              \
+  const int32_t length = offsets[i + 1] - position;                                 \
+  const uint8_t* value = data + position;                                           \
+                                                                                    \
+  int64_t j = HashValue(value, length) & mod_bitmask_;                              \
+  hash_slot_t slot = hash_slots_[j];                                                \
+                                                                                    \
+  const int32_t* dict_offsets = dict_offsets_.data();                               \
+  const uint8_t* dict_data = dict_data_.data();                                     \
+  while (kHashSlotEmpty != slot &&                                                  \
+         !((dict_offsets[slot + 1] - dict_offsets[slot]) == length &&               \
+           0 == memcmp(value, dict_data + dict_offsets[slot], length))) {           \
+    ++j;                                                                            \
+    if (ARROW_PREDICT_FALSE(j == hash_table_size_)) {                               \
+      j = 0;                                                                        \
+    }                                                                               \
+    slot = hash_slots_[j];                                                          \
+  }                                                                                 \
+                                                                                    \
+  if (slot == kHashSlotEmpty) {                                                     \
+    if (!Action::allow_expand) {                                                    \
+      throw HashException("Encountered new dictionary value");                      \
+    }                                                                               \
+                                                                                    \
+    slot = dict_size_++;                                                            \
+    hash_slots_[j] = slot;                                                          \
+                                                                                    \
+    RETURN_NOT_OK(dict_data_.Append(value, length));                                \
+    RETURN_NOT_OK(dict_offsets_.Append(static_cast<int32_t>(dict_data_.length()))); \
+                                                                                    \
+    action->ObserveNotFound(slot);                                                  \
+                                                                                    \
+    if (ARROW_PREDICT_FALSE(dict_size_ > hash_table_load_threshold_)) {             \
+      RETURN_NOT_OK(action->DoubleSize());                                          \
+    }                                                                               \
+  } else {                                                                          \
+    action->ObserveFound(slot);                                                     \
+  }
+
+    GENERIC_HASH_PASS(HASH_INNER_LOOP);
+
+#undef HASH_INNER_LOOP
+
+    return Status::OK();
+  }
+
+  Status GetDictionary(std::shared_ptr<ArrayData>* out) override {
+    // TODO(wesm): handle null being in the dictionary
+    BufferVector buffers = {nullptr, nullptr, nullptr};
+
+    RETURN_NOT_OK(dict_offsets_.Finish(&buffers[1]));
+    RETURN_NOT_OK(dict_data_.Finish(&buffers[2]));
+
+    *out = std::make_shared<ArrayData>(type_, dict_size_, std::move(buffers), 0);
+    return Status::OK();
+  }
+
+ protected:
+  int64_t HashValue(const uint8_t* data, int32_t length) const {
+    return HashUtil::Hash(data, length, 0);
+  }
+
+  Status DoubleTableSize() {
+#define VARBYTES_SETUP                                \
+  const int32_t* dict_offsets = dict_offsets_.data(); \
+  const uint8_t* dict_data = dict_data_.data()
+
+#define VARBYTES_COMPUTE_HASH                                           \
+  const int32_t length = dict_offsets[index + 1] - dict_offsets[index]; \
+  const uint8_t* value = dict_data + dict_offsets[index];               \
+  int64_t j = HashValue(value, length) & new_mod_bitmask
+
+    DOUBLE_TABLE_SIZE(VARBYTES_SETUP, VARBYTES_COMPUTE_HASH);
+
+#undef VARBYTES_SETUP
+#undef VARBYTES_COMPUTE_HASH
+
+    return Status::OK();
+  }
+
+  TypedBufferBuilder<int32_t> dict_offsets_;
+  TypedBufferBuilder<uint8_t> dict_data_;
+  int32_t dict_size_;
+};
+
+// ----------------------------------------------------------------------
+// Hash table pass for fixed size binary types
+
+template <typename Type, typename Action>
+class HashTableKernel<Type, Action, enable_if_fixed_size_binary<Type>>
+    : public HashTable {
+ public:
+  HashTableKernel(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : HashTable(type, pool), dict_data_(pool), dict_size_(0) {
+    const auto& fw_type = static_cast<const FixedSizeBinaryType&>(*type);
+    byte_width_ = fw_type.bit_width() / 8;
+  }
+
+  Status Init() {
+    RETURN_NOT_OK(dict_data_.Resize(kInitialHashTableSize * byte_width_));
+    return HashTable::Init(kInitialHashTableSize);
+  }
+
+  Status Append(const ArrayData& arr) override {
+    if (!initialized_) {
+      RETURN_NOT_OK(Init());
+    }
+
+    const uint8_t* data = GetValues<uint8_t>(arr, 1);
+
+    auto action = static_cast<Action*>(this);
+    RETURN_NOT_OK(action->Reserve(arr.length));
+
+#define HASH_INNER_LOOP()                                                      \
+  const uint8_t* value = data + i * byte_width_;                               \
+  int64_t j = HashValue(value) & mod_bitmask_;                                 \
+  hash_slot_t slot = hash_slots_[j];                                           \
+                                                                               \
+  const uint8_t* dict_data = dict_data_.data();                                \
+  while (kHashSlotEmpty != slot &&                                             \
+         !(0 == memcmp(value, dict_data + slot * byte_width_, byte_width_))) { \
+    ++j;                                                                       \
+    if (ARROW_PREDICT_FALSE(j == hash_table_size_)) {                          \
+      j = 0;                                                                   \
+    }                                                                          \
+    slot = hash_slots_[j];                                                     \
+  }                                                                            \
+                                                                               \
+  if (slot == kHashSlotEmpty) {                                                \
+    if (!Action::allow_expand) {                                               \
+      throw HashException("Encountered new dictionary value");                 \
+    }                                                                          \
+                                                                               \
+    slot = dict_size_++;                                                       \
+    hash_slots_[j] = slot;                                                     \
+                                                                               \
+    RETURN_NOT_OK(dict_data_.Append(value, byte_width_));                      \
+                                                                               \
+    action->ObserveNotFound(slot);                                             \
+                                                                               \
+    if (ARROW_PREDICT_FALSE(dict_size_ > hash_table_load_threshold_)) {        \
+      RETURN_NOT_OK(action->DoubleSize());                                     \
+    }                                                                          \
+  } else {                                                                     \
+    action->ObserveFound(slot);                                                \
+  }
+
+    GENERIC_HASH_PASS(HASH_INNER_LOOP);
+
+#undef HASH_INNER_LOOP
+
+    return Status::OK();
+  }
+
+  Status GetDictionary(std::shared_ptr<ArrayData>* out) override {
+    // TODO(wesm): handle null being in the dictionary
+    BufferVector buffers = {nullptr, nullptr};
+    RETURN_NOT_OK(dict_data_.Finish(&buffers[1]));
+
+    *out = std::make_shared<ArrayData>(type_, dict_size_, std::move(buffers), 0);
+    return Status::OK();
+  }
+
+ protected:
+  int64_t HashValue(const uint8_t* data) const {
+    return HashUtil::Hash(data, byte_width_, 0);
+  }
+
+  Status DoubleTableSize() {
+#define FIXED_BYTES_SETUP const uint8_t* dict_data = dict_data_.data()
+
+#define FIXED_BYTES_COMPUTE_HASH \
+  int64_t j = HashValue(dict_data + index * byte_width_) & new_mod_bitmask
+
+    DOUBLE_TABLE_SIZE(FIXED_BYTES_SETUP, FIXED_BYTES_COMPUTE_HASH);
+
+#undef FIXED_BYTES_SETUP
+#undef FIXED_BYTES_COMPUTE_HASH
+
+    return Status::OK();
+  }
+
+  int32_t byte_width_;
+  TypedBufferBuilder<uint8_t> dict_data_;
+  int32_t dict_size_;
+};
+
+// ----------------------------------------------------------------------
+// Unique implementation
+
+template <typename Type>
+class UniqueImpl : public HashTableKernel<Type, UniqueImpl<Type>> {
+ public:
+  static constexpr bool allow_expand = true;
+  using Base = HashTableKernel<Type, UniqueImpl<Type>>;
+  using Base::Base;
+
+  Status Reserve(const int64_t length) { return Status::OK(); }
+
+  void ObserveFound(const hash_slot_t slot) {}
+  void ObserveNull() {}
+  void ObserveNotFound(const hash_slot_t slot) {}
+
+  Status DoubleSize() { return Base::DoubleTableSize(); }
+
+  Status Append(const ArrayData& input) override { return Base::Append(input); }
+
+  Status Flush(Datum* out) override {
+    // No-op
+    return Status::OK();
+  }
+};
+
+// ----------------------------------------------------------------------
+// Dictionary encode implementation
+
+template <typename Type>
+class DictEncodeImpl : public HashTableKernel<Type, DictEncodeImpl<Type>> {
+ public:
+  static constexpr bool allow_expand = true;
+  using Base = HashTableKernel<Type, DictEncodeImpl>;
+
+  DictEncodeImpl(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : Base(type, pool), indices_builder_(pool) {}
+
+  Status Reserve(const int64_t length) { return indices_builder_.Reserve(length); }
+
+  void ObserveNull() { indices_builder_.UnsafeAppendToBitmap(false); }
+
+  void ObserveFound(const hash_slot_t slot) { indices_builder_.UnsafeAppend(slot); }
+
+  void ObserveNotFound(const hash_slot_t slot) { return ObserveFound(slot); }
+
+  Status DoubleSize() { return Base::DoubleTableSize(); }
+
+  Status Flush(Datum* out) override {
+    std::shared_ptr<ArrayData> result;
+    RETURN_NOT_OK(indices_builder_.FinishInternal(&result));
+    out->value = std::move(result);
+    return Status::OK();
+  }
+
+  using Base::Append;
+
+ private:
+  Int32Builder indices_builder_;
+};
+
+// ----------------------------------------------------------------------
+// Kernel wrapper for generic hash table kernels
+
+class HashKernelImpl : public HashKernel {
+ public:
+  explicit HashKernelImpl(std::unique_ptr<HashTable> hasher)
+      : hasher_(std::move(hasher)) {}
+
+  Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) override {
+    RETURN_NOT_OK(Append(ctx, input));
+    return Flush(out);
+  }
+
+  Status Append(FunctionContext* ctx, const ArrayData& input) override {
+    std::lock_guard<std::mutex> guard(lock_);
+    try {
+      RETURN_NOT_OK(hasher_->Append(input));
+    } catch (const HashException& e) {
+      return Status(e.code(), e.what());
+    }
+    return Status::OK();
+  }
+
+  Status Flush(Datum* out) override { return hasher_->Flush(out); }
+
+  Status GetDictionary(std::shared_ptr<ArrayData>* out) override {
+    return hasher_->GetDictionary(out);
+  }
+
+ private:
+  std::mutex lock_;
+  std::unique_ptr<HashTable> hasher_;
+};
+
+}  // namespace
+
+Status GetUniqueKernel(FunctionContext* ctx, const std::shared_ptr<DataType>& type,
+                       std::unique_ptr<HashKernel>* out) {
+  std::unique_ptr<HashTable> hasher;
+
+#define UNIQUE_CASE(InType)                                         \
+  case InType::type_id:                                             \
+    hasher.reset(new UniqueImpl<InType>(type, ctx->memory_pool())); \
+    break
+
+  switch (type->id()) {
+    UNIQUE_CASE(NullType);
+    // UNIQUE_CASE(BooleanType);
+    UNIQUE_CASE(UInt8Type);
+    UNIQUE_CASE(Int8Type);
+    UNIQUE_CASE(UInt16Type);
+    UNIQUE_CASE(Int16Type);
+    UNIQUE_CASE(UInt32Type);
+    UNIQUE_CASE(Int32Type);
+    UNIQUE_CASE(UInt64Type);
+    UNIQUE_CASE(Int64Type);
+    UNIQUE_CASE(FloatType);
+    UNIQUE_CASE(DoubleType);
+    UNIQUE_CASE(Date32Type);
+    UNIQUE_CASE(Date64Type);
+    UNIQUE_CASE(Time32Type);
+    UNIQUE_CASE(Time64Type);
+    UNIQUE_CASE(TimestampType);
+    UNIQUE_CASE(BinaryType);
+    UNIQUE_CASE(StringType);
+    UNIQUE_CASE(FixedSizeBinaryType);
+    UNIQUE_CASE(Decimal128Type);
+    default:
+      break;
+  }
+
+#undef UNIQUE_CASE
+
+  CHECK_IMPLEMENTED(hasher, "unique", type);
+  out->reset(new HashKernelImpl(std::move(hasher)));
+  return Status::OK();
+}
+
+Status GetDictionaryEncodeKernel(FunctionContext* ctx,
+                                 const std::shared_ptr<DataType>& type,
+                                 std::unique_ptr<HashKernel>* out) {
+  std::unique_ptr<HashTable> hasher;
+
+#define DICTIONARY_ENCODE_CASE(InType)                                  \
+  case InType::type_id:                                                 \
+    hasher.reset(new DictEncodeImpl<InType>(type, ctx->memory_pool())); \
+    break
+
+  switch (type->id()) {
+    DICTIONARY_ENCODE_CASE(NullType);
+    // DICTIONARY_ENCODE_CASE(BooleanType);
+    DICTIONARY_ENCODE_CASE(UInt8Type);
+    DICTIONARY_ENCODE_CASE(Int8Type);
+    DICTIONARY_ENCODE_CASE(UInt16Type);
+    DICTIONARY_ENCODE_CASE(Int16Type);
+    DICTIONARY_ENCODE_CASE(UInt32Type);
+    DICTIONARY_ENCODE_CASE(Int32Type);
+    DICTIONARY_ENCODE_CASE(UInt64Type);
+    DICTIONARY_ENCODE_CASE(Int64Type);
+    DICTIONARY_ENCODE_CASE(FloatType);
+    DICTIONARY_ENCODE_CASE(DoubleType);
+    DICTIONARY_ENCODE_CASE(Date32Type);
+    DICTIONARY_ENCODE_CASE(Date64Type);
+    DICTIONARY_ENCODE_CASE(Time32Type);
+    DICTIONARY_ENCODE_CASE(Time64Type);
+    DICTIONARY_ENCODE_CASE(TimestampType);
+    DICTIONARY_ENCODE_CASE(BinaryType);
+    DICTIONARY_ENCODE_CASE(StringType);
+    DICTIONARY_ENCODE_CASE(FixedSizeBinaryType);
+    DICTIONARY_ENCODE_CASE(Decimal128Type);
+    default:
+      break;
+  }
+
+#undef DICTIONARY_ENCODE_CASE
+
+  CHECK_IMPLEMENTED(hasher, "dictionary-encode", type);
+  out->reset(new HashKernelImpl(std::move(hasher)));
+  return Status::OK();
+}
+
+namespace {
+
+Status InvokeHash(FunctionContext* ctx, HashKernel* func, const Datum& value,
+                  std::vector<Datum>* kernel_outputs,
+                  std::shared_ptr<Array>* dictionary) {
+  RETURN_NOT_OK(detail::InvokeUnaryArrayKernel(ctx, func, value, kernel_outputs));
+
+  std::shared_ptr<ArrayData> dict_data;
+  RETURN_NOT_OK(func->GetDictionary(&dict_data));
+  *dictionary = MakeArray(dict_data);
+  return Status::OK();
+}
+
+}  // namespace
+
+Status Unique(FunctionContext* ctx, const Datum& value, std::shared_ptr<Array>* out) {
+  std::unique_ptr<HashKernel> func;
+  RETURN_NOT_OK(GetUniqueKernel(ctx, value.type(), &func));
+
+  std::vector<Datum> dummy_outputs;
+  return InvokeHash(ctx, func.get(), value, &dummy_outputs, out);
+}
+
+Status DictionaryEncode(FunctionContext* ctx, const Datum& value, Datum* out) {
+  std::unique_ptr<HashKernel> func;
+  RETURN_NOT_OK(GetDictionaryEncodeKernel(ctx, value.type(), &func));
+
+  std::shared_ptr<Array> dictionary;
+  std::vector<Datum> indices_outputs;
+  RETURN_NOT_OK(InvokeHash(ctx, func.get(), value, &indices_outputs, &dictionary));
+
+  // Create the dictionary type
+  DCHECK_EQ(indices_outputs[0].kind(), Datum::ARRAY);
+  std::shared_ptr<DataType> dict_type =
+      ::arrow::dictionary(indices_outputs[0].array()->type, dictionary);
+
+  // Create DictionaryArray for each piece yielded by the kernel invocations
+  std::vector<std::shared_ptr<Array>> dict_chunks;
+  for (const Datum& datum : indices_outputs) {
+    dict_chunks.emplace_back(
+        std::make_shared<DictionaryArray>(dict_type, MakeArray(datum.array())));
+  }
+
+  *out = detail::WrapArraysLike(value, dict_chunks);
+  return Status::OK();
+}
+
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/hash.h b/cpp/src/arrow/compute/kernels/hash.h
new file mode 100644
index 0000000000000..05f242949893f
--- /dev/null
+++ b/cpp/src/arrow/compute/kernels/hash.h
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_COMPUTE_KERNELS_HASH_H
+#define ARROW_COMPUTE_KERNELS_HASH_H
+
+#include <memory>
+#include <vector>
+
+#include "arrow/compute/kernel.h"
+#include "arrow/status.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+namespace compute {
+
+class FunctionContext;
+
+/// \brief Invoke hash table kernel on input array, returning any output
+/// values. Implementations should be thread-safe
+class ARROW_EXPORT HashKernel : public UnaryKernel {
+ public:
+  virtual Status Append(FunctionContext* ctx, const ArrayData& input) = 0;
+  virtual Status Flush(Datum* out) = 0;
+  virtual Status GetDictionary(std::shared_ptr<ArrayData>* out) = 0;
+};
+
+/// \since 0.8.0
+/// \note API not yet finalized
+ARROW_EXPORT
+Status GetUniqueKernel(FunctionContext* ctx, const std::shared_ptr<DataType>& type,
+                       std::unique_ptr<HashKernel>* kernel);
+
+ARROW_EXPORT
+Status GetDictionaryEncodeKernel(FunctionContext* ctx,
+                                 const std::shared_ptr<DataType>& type,
+                                 std::unique_ptr<HashKernel>* kernel);
+
+/// \brief Compute unique elements from an array-like object
+/// \param[in] context the FunctionContext
+/// \param[in] datum array-like input
+/// \param[out] out result as Array
+///
+/// \since 0.8.0
+/// \note API not yet finalized
+ARROW_EXPORT
+Status Unique(FunctionContext* context, const Datum& datum, std::shared_ptr<Array>* out);
+
+/// \brief Dictionary-encode values in an array-like object
+/// \param[in] context the FunctionContext
+/// \param[in] data array-like input
+/// \param[out] out result with same shape and type as input
+///
+/// \since 0.8.0
+/// \note API not yet finalized
+ARROW_EXPORT
+Status DictionaryEncode(FunctionContext* context, const Datum& data, Datum* out);
+
+// TODO(wesm): Define API for incremental dictionary encoding
+
+// TODO(wesm): Define API for regularizing DictionaryArray objects with
+// different dictionaries
+
+// class DictionaryEncoder {
+//  public:
+//   virtual Encode(const Datum& data, Datum* out) = 0;
+// };
+
+//
+// ARROW_EXPORT
+// Status DictionaryEncode(FunctionContext* context, const Datum& data,
+//                         const Array& prior_dictionary, Datum* out);
+
+// TODO(wesm): Implement these next
+// ARROW_EXPORT
+// Status Match(FunctionContext* context, const Datum& values, const Datum& member_set,
+//              Datum* out);
+
+// ARROW_EXPORT
+// Status IsIn(FunctionContext* context, const Datum& values, const Datum& member_set,
+//             Datum* out);
+
+// ARROW_EXPORT
+// Status CountValues(FunctionContext* context, const Datum& values,
+//                    std::shared_ptr<Array>* out_uniques,
+//                    std::shared_ptr<Array>* out_counts);
+
+}  // namespace compute
+}  // namespace arrow
+
+#endif  // ARROW_COMPUTE_KERNELS_HASH_H
diff --git a/cpp/src/arrow/compute/kernels/util-internal.cc b/cpp/src/arrow/compute/kernels/util-internal.cc
new file mode 100644
index 0000000000000..df68637e088f7
--- /dev/null
+++ b/cpp/src/arrow/compute/kernels/util-internal.cc
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/compute/kernels/util-internal.h"
+
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+#include "arrow/compute/context.h"
+#include "arrow/compute/kernel.h"
+
+namespace arrow {
+namespace compute {
+namespace detail {
+
+Status InvokeUnaryArrayKernel(FunctionContext* ctx, UnaryKernel* kernel,
+                              const Datum& value, std::vector<Datum>* outputs) {
+  if (value.kind() == Datum::ARRAY) {
+    Datum output;
+    RETURN_NOT_OK(kernel->Call(ctx, *value.array(), &output));
+    outputs->push_back(output);
+  } else if (value.kind() == Datum::CHUNKED_ARRAY) {
+    const ChunkedArray& array = *value.chunked_array();
+    for (int i = 0; i < array.num_chunks(); i++) {
+      Datum output;
+      RETURN_NOT_OK(kernel->Call(ctx, *(array.chunk(i)->data()), &output));
+      outputs->push_back(output);
+    }
+  } else {
+    return Status::Invalid("Input Datum was not array-like");
+  }
+  return Status::OK();
+}
+
+Datum WrapArraysLike(const Datum& value,
+                     const std::vector<std::shared_ptr<Array>>& arrays) {
+  // Create right kind of datum
+  if (value.kind() == Datum::ARRAY) {
+    return Datum(arrays[0]->data());
+  } else if (value.kind() == Datum::CHUNKED_ARRAY) {
+    return Datum(std::make_shared<ChunkedArray>(arrays));
+  } else {
+    DCHECK(false) << "unhandled datum kind";
+    return Datum();
+  }
+}
+
+Datum WrapDatumsLike(const Datum& value, const std::vector<Datum>& datums) {
+  // Create right kind of datum
+  if (value.kind() == Datum::ARRAY) {
+    DCHECK_EQ(1, datums.size());
+    return Datum(datums[0].array());
+  } else if (value.kind() == Datum::CHUNKED_ARRAY) {
+    std::vector<std::shared_ptr<Array>> arrays;
+    for (const Datum& datum : datums) {
+      DCHECK_EQ(Datum::ARRAY, datum.kind());
+      arrays.emplace_back(MakeArray(datum.array()));
+    }
+    return Datum(std::make_shared<ChunkedArray>(arrays));
+  } else {
+    DCHECK(false) << "unhandled datum kind";
+    return Datum();
+  }
+}
+
+}  // namespace detail
+}  // namespace compute
+}  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/util-internal.h b/cpp/src/arrow/compute/kernels/util-internal.h
new file mode 100644
index 0000000000000..70c506286ede6
--- /dev/null
+++ b/cpp/src/arrow/compute/kernels/util-internal.h
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_COMPUTE_KERNELS_UTIL_INTERNAL_H
+#define ARROW_COMPUTE_KERNELS_UTIL_INTERNAL_H
+
+#include <vector>
+
+#include "arrow/compute/kernel.h"
+#include "arrow/type_fwd.h"
+
+namespace arrow {
+namespace compute {
+
+class FunctionContext;
+
+template <typename T>
+using is_number = std::is_base_of<Number, T>;
+
+template <typename T>
+using enable_if_primitive_ctype =
+    typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value>::type;
+
+template <typename T>
+using enable_if_date = typename std::enable_if<std::is_base_of<DateType, T>::value>::type;
+
+template <typename T>
+using enable_if_time = typename std::enable_if<std::is_base_of<TimeType, T>::value>::type;
+
+template <typename T>
+using enable_if_timestamp =
+    typename std::enable_if<std::is_base_of<TimestampType, T>::value>::type;
+
+template <typename T>
+using enable_if_has_c_type =
+    typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
+                            std::is_base_of<DateType, T>::value ||
+                            std::is_base_of<TimeType, T>::value ||
+                            std::is_base_of<TimestampType, T>::value>::type;
+
+template <typename T>
+using enable_if_null = typename std::enable_if<std::is_same<NullType, T>::value>::type;
+
+template <typename T>
+using enable_if_binary =
+    typename std::enable_if<std::is_base_of<BinaryType, T>::value>::type;
+
+template <typename T>
+using enable_if_fixed_size_binary =
+    typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value>::type;
+
+template <typename T>
+using enable_if_list = typename std::enable_if<std::is_base_of<ListType, T>::value>::type;
+
+template <typename T>
+using enable_if_number = typename std::enable_if<is_number<T>::value>::type;
+
+template <typename T>
+inline const T* GetValues(const ArrayData& data, int i) {
+  return reinterpret_cast<const T*>(data.buffers[i]->data()) + data.offset;
+}
+
+template <typename T>
+inline T* GetMutableValues(const ArrayData* data, int i) {
+  return reinterpret_cast<T*>(data->buffers[i]->mutable_data()) + data->offset;
+}
+
+static inline void CopyData(const ArrayData& input, ArrayData* output) {
+  output->length = input.length;
+  output->null_count = input.null_count;
+  output->buffers = input.buffers;
+  output->offset = input.offset;
+  output->child_data = input.child_data;
+}
+
+namespace detail {
+
+Status InvokeUnaryArrayKernel(FunctionContext* ctx, UnaryKernel* kernel,
+                              const Datum& value, std::vector<Datum>* outputs);
+
+Datum WrapArraysLike(const Datum& value,
+                     const std::vector<std::shared_ptr<Array>>& arrays);
+
+Datum WrapDatumsLike(const Datum& value, const std::vector<Datum>& datums);
+
+}  // namespace detail
+
+}  // namespace compute
+}  // namespace arrow
+
+#endif  // ARROW_COMPUTE_KERNELS_UTIL_INTERNAL_H
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index 8b9a24fecd332..bf29d6a03cd93 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -107,14 +107,13 @@ TEST_F(TestPrettyPrint, FixedSizeBinaryType) {
   CheckArray(*array, 0, ex);
 }
 
-TEST_F(TestPrettyPrint, DecimalType) {
+TEST_F(TestPrettyPrint, Decimal128Type) {
   int32_t p = 19;
   int32_t s = 4;
 
   auto type = decimal(p, s);
 
-  DecimalBuilder builder(type);
-
+  Decimal128Builder builder(type);
   Decimal128 val;
 
   ASSERT_OK(Decimal128::FromString("123.4567", &val));
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 2b0f964391cb1..8814fc190ab3d 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -42,6 +42,8 @@
 #include "arrow/util/parallel.h"
 #include "arrow/visitor_inline.h"
 
+#include "arrow/compute/api.h"
+
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
 #include "arrow/python/config.h"
@@ -57,6 +59,8 @@ namespace py {
 using internal::kPandasTimestampNull;
 using internal::kNanosecondsInDay;
 
+using compute::Datum;
+
 // ----------------------------------------------------------------------
 // Utility code
 
@@ -1028,8 +1032,13 @@ class CategoricalBlock : public PandasBlock {
     std::shared_ptr<Column> converted_col;
     if (options_.strings_to_categorical &&
         (col->type()->id() == Type::STRING || col->type()->id() == Type::BINARY)) {
-      RETURN_NOT_OK(EncodeColumnToDictionary(static_cast<const Column&>(*col), pool_,
-                                             &converted_col));
+      compute::FunctionContext ctx(pool_);
+
+      Datum out;
+      RETURN_NOT_OK(compute::DictionaryEncode(&ctx, Datum(col->data()), &out));
+      DCHECK_EQ(out.kind(), Datum::CHUNKED_ARRAY);
+      converted_col =
+          std::make_shared<Column>(field(col->name(), out.type()), out.chunked_array());
     } else {
       converted_col = col;
     }
@@ -1646,7 +1655,7 @@ class ArrowDeserializer {
       CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT)
       CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE)
       CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING)
-      CONVERTVALUES_LISTSLIKE_CASE(DecimalType, DECIMAL)
+      CONVERTVALUES_LISTSLIKE_CASE(Decimal128Type, DECIMAL)
       CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST)
       default: {
         std::stringstream ss;
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index a6c28af9b7443..0d2df93151913 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -42,8 +42,8 @@
 #include "arrow/util/macros.h"
 #include "arrow/visitor_inline.h"
 
-#include "arrow/compute/cast.h"
 #include "arrow/compute/context.h"
+#include "arrow/compute/kernels/cast.h"
 
 #include "arrow/python/builtin_convert.h"
 #include "arrow/python/common.h"
@@ -466,13 +466,14 @@ Status NumPyConverter::Convert() {
 
 namespace {
 
-Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t length,
-                  const std::shared_ptr<DataType>& in_type,
+Status CastBuffer(const std::shared_ptr<DataType>& in_type,
+                  const std::shared_ptr<Buffer>& input, const int64_t length,
+                  const std::shared_ptr<Buffer>& valid_bitmap, const int64_t null_count,
                   const std::shared_ptr<DataType>& out_type, MemoryPool* pool,
                   std::shared_ptr<Buffer>* out) {
   // Must cast
-  std::vector<std::shared_ptr<Buffer>> buffers = {nullptr, input};
-  auto tmp_data = std::make_shared<ArrayData>(in_type, length, buffers, 0);
+  std::vector<std::shared_ptr<Buffer>> buffers = {valid_bitmap, input};
+  auto tmp_data = std::make_shared<ArrayData>(in_type, length, buffers, null_count);
 
   std::shared_ptr<Array> tmp_array = MakeArray(tmp_data);
   std::shared_ptr<Array> casted_array;
@@ -488,6 +489,21 @@ Status CastBuffer(const std::shared_ptr<Buffer>& input, const int64_t length,
   return Status::OK();
 }
 
+template <typename FromType, typename ToType>
+Status StaticCastBuffer(const Buffer& input, const int64_t length, MemoryPool* pool,
+                        std::shared_ptr<Buffer>* out) {
+  auto result = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(result->Resize(sizeof(ToType) * length));
+
+  auto in_values = reinterpret_cast<const FromType*>(input.data());
+  auto out_values = reinterpret_cast<ToType*>(result->mutable_data());
+  for (int64_t i = 0; i < length; ++i) {
+    *out_values++ = static_cast<ToType>(*in_values++);
+  }
+  *out = result;
+  return Status::OK();
+}
+
 template <typename T, typename T2>
 void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) {
   // Passing input_data as non-const is a concession to PyObject*
@@ -531,7 +547,7 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr<Buffer>* data) {
   RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_), &input_type));
 
   if (!input_type->Equals(*type_)) {
-    RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data));
+    RETURN_NOT_OK(CastBuffer(input_type, *data, length_, nullptr, 0, type_, pool_, data));
   }
 
   return Status::OK();
@@ -567,27 +583,32 @@ inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* d
     *data = std::make_shared<NumPyBuffer>(reinterpret_cast<PyObject*>(arr_));
   }
 
-  // If we have inbound datetime64[D] data, this needs to be downcasted
-  // separately here from int64_t to int32_t, because this data is not
-  // supported in compute::Cast
+  std::shared_ptr<DataType> input_type;
+
   auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(dtype_->c_metadata);
-  if (dtype_->type_num == NPY_DATETIME && date_dtype->meta.base == NPY_FR_D) {
-    auto date32_buffer = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(date32_buffer->Resize(sizeof(int32_t) * length_));
+  if (dtype_->type_num == NPY_DATETIME) {
+    const int64_t null_count = ValuesToBitmap<NPY_DATETIME>(arr_, null_bitmap_data_);
 
-    auto datetime64_values = reinterpret_cast<const int64_t*>((*data)->data());
-    auto date32_values = reinterpret_cast<int32_t*>(date32_buffer->mutable_data());
-    for (int64_t i = 0; i < length_; ++i) {
+    // If we have inbound datetime64[D] data, this needs to be downcasted
+    // separately here from int64_t to int32_t, because this data is not
+    // supported in compute::Cast
+    if (date_dtype->meta.base == NPY_FR_D) {
       // TODO(wesm): How pedantic do we really want to be about checking for int32
       // overflow here?
-      *date32_values++ = static_cast<int32_t>(*datetime64_values++);
+      Status s = StaticCastBuffer<int64_t, int32_t>(**data, length_, pool_, data);
+      RETURN_NOT_OK(s);
+    } else {
+      RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_), &input_type));
+      if (!input_type->Equals(*type_)) {
+        RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count,
+                                 type_, pool_, data));
+      }
     }
-    *data = date32_buffer;
   } else {
-    std::shared_ptr<DataType> input_type;
     RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_), &input_type));
     if (!input_type->Equals(*type_)) {
-      RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data));
+      RETURN_NOT_OK(
+          CastBuffer(input_type, *data, length_, nullptr, 0, type_, pool_, data));
     }
   }
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index d3145ff107ae0..2cff32f74ef14 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -63,6 +63,9 @@ class ARROW_EXPORT ChunkedArray {
   ArrayVector chunks_;
   int64_t length_;
   int64_t null_count_;
+
+ private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(ChunkedArray);
 };
 
 /// \brief An immutable column data structure consisting of a field (type
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 9b875ce116a29..77f489ab17797 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -365,6 +365,10 @@ Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>&
     }                                                                                  \
   } while (false)
 
+#define DECL_T() typedef typename TestFixture::T T;
+
+#define DECL_TYPE() typedef typename TestFixture::Type Type;
+
 void AssertArraysEqual(const Array& expected, const Array& actual) {
   ASSERT_ARRAYS_EQUAL(expected, actual);
 }
diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc
index 3242fadd50dd6..48982cad42435 100644
--- a/cpp/src/arrow/type-test.cc
+++ b/cpp/src/arrow/type-test.cc
@@ -400,7 +400,7 @@ TEST(TestStructType, Basics) {
 }
 
 TEST(TypesTest, TestDecimal128Small) {
-  DecimalType t1(8, 4);
+  Decimal128Type t1(8, 4);
 
   ASSERT_EQ(t1.id(), Type::DECIMAL);
   ASSERT_EQ(t1.precision(), 8);
@@ -414,7 +414,7 @@ TEST(TypesTest, TestDecimal128Small) {
 }
 
 TEST(TypesTest, TestDecimal128Medium) {
-  DecimalType t1(12, 5);
+  Decimal128Type t1(12, 5);
 
   ASSERT_EQ(t1.id(), Type::DECIMAL);
   ASSERT_EQ(t1.precision(), 12);
@@ -428,7 +428,7 @@ TEST(TypesTest, TestDecimal128Medium) {
 }
 
 TEST(TypesTest, TestDecimal128Large) {
-  DecimalType t1(27, 7);
+  Decimal128Type t1(27, 7);
 
   ASSERT_EQ(t1.id(), Type::DECIMAL);
   ASSERT_EQ(t1.precision(), 27);
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index d86e7ef57bec6..70f275c0fa495 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -498,25 +498,34 @@ class ARROW_EXPORT StructType : public NestedType {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-class ARROW_EXPORT Decimal128Type : public FixedSizeBinaryType {
+class ARROW_EXPORT DecimalBaseType : public FixedSizeBinaryType {
+ public:
+  explicit DecimalBaseType(int32_t byte_width, int32_t precision, int32_t scale)
+      : FixedSizeBinaryType(byte_width, Type::DECIMAL),
+        precision_(precision),
+        scale_(scale) {}
+
+  int32_t precision() const { return precision_; }
+  int32_t scale() const { return scale_; }
+
+ protected:
+  int32_t precision_;
+  int32_t scale_;
+};
+
+class ARROW_EXPORT Decimal128Type : public DecimalBaseType {
  public:
   static constexpr Type::type type_id = Type::DECIMAL;
 
   explicit Decimal128Type(int32_t precision, int32_t scale)
-      : FixedSizeBinaryType(16, Type::DECIMAL), precision_(precision), scale_(scale) {}
+      : DecimalBaseType(16, precision, scale) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   std::string name() const override { return "decimal"; }
-
-  int32_t precision() const { return precision_; }
-  int32_t scale() const { return scale_; }
-
- private:
-  int32_t precision_;
-  int32_t scale_;
 };
 
+// TODO(wesm): Remove this
 using DecimalType = Decimal128Type;
 
 struct UnionMode {
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index b8b3c5aa5adb7..9d8a23ce0714f 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -28,10 +28,16 @@ class Status;
 
 class DataType;
 class Array;
+struct ArrayData;
 class ArrayBuilder;
 class Field;
 class Tensor;
 
+class ChunkedArray;
+class Column;
+class RecordBatch;
+class Table;
+
 class Buffer;
 class MemoryPool;
 class RecordBatch;
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 6707f3759581d..4bfce9b5f0c53 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -430,6 +430,10 @@ static inline bool is_binary_like(Type::type type_id) {
   return false;
 }
 
+static inline bool is_dictionary(Type::type type_id) {
+  return type_id == Type::DICTIONARY;
+}
+
 }  // namespace arrow
 
 #endif  // ARROW_TYPE_TRAITS_H
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 7810a3be46da5..29b18a9357ad0 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -42,6 +42,7 @@ install(FILES
   sse-util.h
   stl.h
   type_traits.h
+  variant.h
   visibility.h
   DESTINATION include/arrow/util)
 
@@ -72,3 +73,5 @@ ADD_ARROW_TEST(decimal-test)
 ADD_ARROW_TEST(key-value-metadata-test)
 ADD_ARROW_TEST(rle-encoding-test)
 ADD_ARROW_TEST(stl-util-test)
+
+add_subdirectory(variant)
diff --git a/cpp/src/arrow/util/variant.h b/cpp/src/arrow/util/variant.h
new file mode 100644
index 0000000000000..923a8685f0cdb
--- /dev/null
+++ b/cpp/src/arrow/util/variant.h
@@ -0,0 +1,1127 @@
+// Copyright (c) MapBox
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// - Redistributions of source code must retain the above copyright notice, this
+//   list of conditions and the following disclaimer.
+// - Redistributions in binary form must reproduce the above copyright notice, this
+//   list of conditions and the following disclaimer in the documentation and/or
+//   other materials provided with the distribution.
+// - Neither the name "MapBox" nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef ARROW_UTIL_VARIANT_H
+#define ARROW_UTIL_VARIANT_H
+
+#include <cassert>
+#include <cstddef>   // size_t
+#include <new>       // operator new
+#include <stdexcept> // runtime_error
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <typeinfo>
+#include <utility>
+#include <functional>
+#include <limits>
+
+#include <arrow/util/variant/recursive_wrapper.h>
+#include <arrow/util/variant/variant_visitor.h>
+
+// clang-format off
+// [[deprecated]] is only available in C++14, use this for the time being
+#if __cplusplus <= 201103L
+# ifdef __GNUC__
+#  define ARROW_VARIANT_DEPRECATED __attribute__((deprecated))
+# elif defined(_MSC_VER)
+#  define ARROW_VARIANT_DEPRECATED __declspec(deprecated)
+# else
+#  define ARROW_VARIANT_DEPRECATED
+# endif
+#else
+#  define ARROW_VARIANT_DEPRECATED [[deprecated]]
+#endif
+
+
+#ifdef _MSC_VER
+// https://msdn.microsoft.com/en-us/library/bw1hbe6y.aspx
+# ifdef NDEBUG
+#  define VARIANT_INLINE __forceinline
+# else
+#  define VARIANT_INLINE //__declspec(noinline)
+# endif
+#else
+# ifdef NDEBUG
+#  define VARIANT_INLINE //inline __attribute__((always_inline))
+# else
+#  define VARIANT_INLINE __attribute__((noinline))
+# endif
+#endif
+// clang-format on
+
+// Exceptions
+#if defined( __EXCEPTIONS) || defined( _MSC_VER)
+#define HAS_EXCEPTIONS
+#endif
+
+#define VARIANT_MAJOR_VERSION 1
+#define VARIANT_MINOR_VERSION 1
+#define VARIANT_PATCH_VERSION 0
+
+#define VARIANT_VERSION (VARIANT_MAJOR_VERSION * 100000) + (VARIANT_MINOR_VERSION * 100) + (VARIANT_PATCH_VERSION)
+
+namespace arrow {
+namespace util {
+
+// XXX This should derive from std::logic_error instead of std::runtime_error.
+//     See https://github.com/mapbox/variant/issues/48 for details.
+class bad_variant_access : public std::runtime_error
+{
+
+public:
+    explicit bad_variant_access(const std::string& what_arg)
+        : runtime_error(what_arg) {}
+
+    explicit bad_variant_access(const char* what_arg)
+        : runtime_error(what_arg) {}
+
+}; // class bad_variant_access
+
+template <typename R = void>
+struct ARROW_VARIANT_DEPRECATED static_visitor
+{
+    using result_type = R;
+
+protected:
+    static_visitor() {}
+    ~static_visitor() {}
+};
+
+#if !defined(ARROW_VARIANT_MINIMIZE_SIZE)
+using type_index_t = unsigned int;
+#else
+#if defined(ARROW_VARIANT_OPTIMIZE_FOR_SPEED)
+using type_index_t = std::uint_fast8_t;
+#else
+using type_index_t = std::uint_least8_t;
+#endif
+#endif
+
+namespace detail {
+
+static constexpr type_index_t invalid_value = type_index_t(-1);
+
+template <typename T, typename... Types>
+struct direct_type;
+
+template <typename T, typename First, typename... Types>
+struct direct_type<T, First, Types...>
+{
+    static constexpr type_index_t index = std::is_same<T, First>::value
+        ? sizeof...(Types)
+        : direct_type<T, Types...>::index;
+};
+
+template <typename T>
+struct direct_type<T>
+{
+    static constexpr type_index_t index = invalid_value;
+};
+
+#if __cpp_lib_logical_traits >= 201510L
+
+using std::conjunction;
+using std::disjunction;
+
+#else
+
+template <typename...>
+struct conjunction : std::true_type {};
+
+template <typename B1>
+struct conjunction<B1> : B1 {};
+
+template <typename B1, typename B2>
+struct conjunction<B1, B2> : std::conditional<B1::value, B2, B1>::type {};
+
+template <typename B1, typename... Bs>
+struct conjunction<B1, Bs...> : std::conditional<B1::value, conjunction<Bs...>, B1>::type {};
+
+template <typename...>
+struct disjunction : std::false_type {};
+
+template <typename B1>
+struct disjunction<B1> : B1 {};
+
+template <typename B1, typename B2>
+struct disjunction<B1, B2> : std::conditional<B1::value, B1, B2>::type {};
+
+template <typename B1, typename... Bs>
+struct disjunction<B1, Bs...> : std::conditional<B1::value, B1, disjunction<Bs...>>::type {};
+
+#endif
+
+template <typename T, typename... Types>
+struct convertible_type;
+
+template <typename T, typename First, typename... Types>
+struct convertible_type<T, First, Types...>
+{
+    static constexpr type_index_t index = std::is_convertible<T, First>::value
+        ? disjunction<std::is_convertible<T, Types>...>::value ? invalid_value : sizeof...(Types)
+        : convertible_type<T, Types...>::index;
+};
+
+template <typename T>
+struct convertible_type<T>
+{
+    static constexpr type_index_t index = invalid_value;
+};
+
+template <typename T, typename... Types>
+struct value_traits
+{
+    using value_type = typename std::remove_const<typename std::remove_reference<T>::type>::type;
+    using value_type_wrapper = recursive_wrapper<value_type>;
+    static constexpr type_index_t direct_index = direct_type<value_type, Types...>::index;
+    static constexpr bool is_direct = direct_index != invalid_value;
+    static constexpr type_index_t index_direct_or_wrapper = is_direct ? direct_index : direct_type<value_type_wrapper, Types...>::index;
+    static constexpr bool is_direct_or_wrapper = index_direct_or_wrapper != invalid_value;
+    static constexpr type_index_t index = is_direct_or_wrapper ? index_direct_or_wrapper : convertible_type<value_type, Types...>::index;
+    static constexpr bool is_valid = index != invalid_value;
+    static constexpr type_index_t tindex = is_valid ? sizeof...(Types)-index : 0;
+    using target_type = typename std::tuple_element<tindex, std::tuple<void, Types...>>::type;
+};
+
+template <typename T, typename R = void>
+struct enable_if_type
+{
+    using type = R;
+};
+
+template <typename F, typename V, typename Enable = void>
+struct result_of_unary_visit
+{
+    using type = typename std::result_of<F(V&)>::type;
+};
+
+template <typename F, typename V>
+struct result_of_unary_visit<F, V, typename enable_if_type<typename F::result_type>::type>
+{
+    using type = typename F::result_type;
+};
+
+template <typename F, typename V, typename Enable = void>
+struct result_of_binary_visit
+{
+    using type = typename std::result_of<F(V&, V&)>::type;
+};
+
+template <typename F, typename V>
+struct result_of_binary_visit<F, V, typename enable_if_type<typename F::result_type>::type>
+{
+    using type = typename F::result_type;
+};
+
+template <type_index_t arg1, type_index_t... others>
+struct static_max;
+
+template <type_index_t arg>
+struct static_max<arg>
+{
+    static const type_index_t value = arg;
+};
+
+template <type_index_t arg1, type_index_t arg2, type_index_t... others>
+struct static_max<arg1, arg2, others...>
+{
+    static const type_index_t value = arg1 >= arg2 ? static_max<arg1, others...>::value : static_max<arg2, others...>::value;
+};
+
+template <typename... Types>
+struct variant_helper;
+
+template <typename T, typename... Types>
+struct variant_helper<T, Types...>
+{
+    VARIANT_INLINE static void destroy(const type_index_t type_index, void* data)
+    {
+        if (type_index == sizeof...(Types))
+        {
+            reinterpret_cast<T*>(data)->~T();
+        }
+        else
+        {
+            variant_helper<Types...>::destroy(type_index, data);
+        }
+    }
+
+    VARIANT_INLINE static void move(const type_index_t old_type_index, void* old_value, void* new_value)
+    {
+        if (old_type_index == sizeof...(Types))
+        {
+            new (new_value) T(std::move(*reinterpret_cast<T*>(old_value)));
+        }
+        else
+        {
+            variant_helper<Types...>::move(old_type_index, old_value, new_value);
+        }
+    }
+
+    VARIANT_INLINE static void copy(const type_index_t old_type_index, const void* old_value, void* new_value)
+    {
+        if (old_type_index == sizeof...(Types))
+        {
+            new (new_value) T(*reinterpret_cast<const T*>(old_value));
+        }
+        else
+        {
+            variant_helper<Types...>::copy(old_type_index, old_value, new_value);
+        }
+    }
+};
+
+template <>
+struct variant_helper<>
+{
+    VARIANT_INLINE static void destroy(const type_index_t, void*) {}
+    VARIANT_INLINE static void move(const type_index_t, void*, void*) {}
+    VARIANT_INLINE static void copy(const type_index_t, const void*, void*) {}
+};
+
+template <typename T>
+struct unwrapper
+{
+    static T const& apply_const(T const& obj) { return obj; }
+    static T& apply(T& obj) { return obj; }
+};
+
+template <typename T>
+struct unwrapper<recursive_wrapper<T>>
+{
+    static auto apply_const(recursive_wrapper<T> const& obj)
+        -> typename recursive_wrapper<T>::type const&
+    {
+        return obj.get();
+    }
+    static auto apply(recursive_wrapper<T>& obj)
+        -> typename recursive_wrapper<T>::type&
+    {
+        return obj.get();
+    }
+};
+
+template <typename T>
+struct unwrapper<std::reference_wrapper<T>>
+{
+    static auto apply_const(std::reference_wrapper<T> const& obj)
+        -> typename std::reference_wrapper<T>::type const&
+    {
+        return obj.get();
+    }
+    static auto apply(std::reference_wrapper<T>& obj)
+        -> typename std::reference_wrapper<T>::type&
+    {
+        return obj.get();
+    }
+};
+
+template <typename F, typename V, typename R, typename... Types>
+struct dispatcher;
+
+template <typename F, typename V, typename R, typename T, typename... Types>
+struct dispatcher<F, V, R, T, Types...>
+{
+    VARIANT_INLINE static R apply_const(V const& v, F&& f)
+    {
+        if (v.template is<T>())
+        {
+            return f(unwrapper<T>::apply_const(v.template get_unchecked<T>()));
+        }
+        else
+        {
+            return dispatcher<F, V, R, Types...>::apply_const(v, std::forward<F>(f));
+        }
+    }
+
+    VARIANT_INLINE static R apply(V& v, F&& f)
+    {
+        if (v.template is<T>())
+        {
+            return f(unwrapper<T>::apply(v.template get_unchecked<T>()));
+        }
+        else
+        {
+            return dispatcher<F, V, R, Types...>::apply(v, std::forward<F>(f));
+        }
+    }
+};
+
+template <typename F, typename V, typename R, typename T>
+struct dispatcher<F, V, R, T>
+{
+    VARIANT_INLINE static R apply_const(V const& v, F&& f)
+    {
+        return f(unwrapper<T>::apply_const(v.template get_unchecked<T>()));
+    }
+
+    VARIANT_INLINE static R apply(V& v, F&& f)
+    {
+        return f(unwrapper<T>::apply(v.template get_unchecked<T>()));
+    }
+};
+
+template <typename F, typename V, typename R, typename T, typename... Types>
+struct binary_dispatcher_rhs;
+
+template <typename F, typename V, typename R, typename T0, typename T1, typename... Types>
+struct binary_dispatcher_rhs<F, V, R, T0, T1, Types...>
+{
+    VARIANT_INLINE static R apply_const(V const& lhs, V const& rhs, F&& f)
+    {
+        if (rhs.template is<T1>()) // call binary functor
+        {
+            return f(unwrapper<T0>::apply_const(lhs.template get_unchecked<T0>()),
+                     unwrapper<T1>::apply_const(rhs.template get_unchecked<T1>()));
+        }
+        else
+        {
+            return binary_dispatcher_rhs<F, V, R, T0, Types...>::apply_const(lhs, rhs, std::forward<F>(f));
+        }
+    }
+
+    VARIANT_INLINE static R apply(V& lhs, V& rhs, F&& f)
+    {
+        if (rhs.template is<T1>()) // call binary functor
+        {
+            return f(unwrapper<T0>::apply(lhs.template get_unchecked<T0>()),
+                     unwrapper<T1>::apply(rhs.template get_unchecked<T1>()));
+        }
+        else
+        {
+            return binary_dispatcher_rhs<F, V, R, T0, Types...>::apply(lhs, rhs, std::forward<F>(f));
+        }
+    }
+};
+
+template <typename F, typename V, typename R, typename T0, typename T1>
+struct binary_dispatcher_rhs<F, V, R, T0, T1>
+{
+    VARIANT_INLINE static R apply_const(V const& lhs, V const& rhs, F&& f)
+    {
+        return f(unwrapper<T0>::apply_const(lhs.template get_unchecked<T0>()),
+                 unwrapper<T1>::apply_const(rhs.template get_unchecked<T1>()));
+    }
+
+    VARIANT_INLINE static R apply(V& lhs, V& rhs, F&& f)
+    {
+        return f(unwrapper<T0>::apply(lhs.template get_unchecked<T0>()),
+                 unwrapper<T1>::apply(rhs.template get_unchecked<T1>()));
+    }
+};
+
+template <typename F, typename V, typename R, typename T, typename... Types>
+struct binary_dispatcher_lhs;
+
+template <typename F, typename V, typename R, typename T0, typename T1, typename... Types>
+struct binary_dispatcher_lhs<F, V, R, T0, T1, Types...>
+{
+    VARIANT_INLINE static R apply_const(V const& lhs, V const& rhs, F&& f)
+    {
+        if (lhs.template is<T1>()) // call binary functor
+        {
+            return f(unwrapper<T1>::apply_const(lhs.template get_unchecked<T1>()),
+                     unwrapper<T0>::apply_const(rhs.template get_unchecked<T0>()));
+        }
+        else
+        {
+            return binary_dispatcher_lhs<F, V, R, T0, Types...>::apply_const(lhs, rhs, std::forward<F>(f));
+        }
+    }
+
+    VARIANT_INLINE static R apply(V& lhs, V& rhs, F&& f)
+    {
+        if (lhs.template is<T1>()) // call binary functor
+        {
+            return f(unwrapper<T1>::apply(lhs.template get_unchecked<T1>()),
+                     unwrapper<T0>::apply(rhs.template get_unchecked<T0>()));
+        }
+        else
+        {
+            return binary_dispatcher_lhs<F, V, R, T0, Types...>::apply(lhs, rhs, std::forward<F>(f));
+        }
+    }
+};
+
+template <typename F, typename V, typename R, typename T0, typename T1>
+struct binary_dispatcher_lhs<F, V, R, T0, T1>
+{
+    VARIANT_INLINE static R apply_const(V const& lhs, V const& rhs, F&& f)
+    {
+        return f(unwrapper<T1>::apply_const(lhs.template get_unchecked<T1>()),
+                 unwrapper<T0>::apply_const(rhs.template get_unchecked<T0>()));
+    }
+
+    VARIANT_INLINE static R apply(V& lhs, V& rhs, F&& f)
+    {
+        return f(unwrapper<T1>::apply(lhs.template get_unchecked<T1>()),
+                 unwrapper<T0>::apply(rhs.template get_unchecked<T0>()));
+    }
+};
+
+template <typename F, typename V, typename R, typename... Types>
+struct binary_dispatcher;
+
+template <typename F, typename V, typename R, typename T, typename... Types>
+struct binary_dispatcher<F, V, R, T, Types...>
+{
+    VARIANT_INLINE static R apply_const(V const& v0, V const& v1, F&& f)
+    {
+        if (v0.template is<T>())
+        {
+            if (v1.template is<T>())
+            {
+                return f(unwrapper<T>::apply_const(v0.template get_unchecked<T>()),
+                         unwrapper<T>::apply_const(v1.template get_unchecked<T>())); // call binary functor
+            }
+            else
+            {
+                return binary_dispatcher_rhs<F, V, R, T, Types...>::apply_const(v0, v1, std::forward<F>(f));
+            }
+        }
+        else if (v1.template is<T>())
+        {
+            return binary_dispatcher_lhs<F, V, R, T, Types...>::apply_const(v0, v1, std::forward<F>(f));
+        }
+        return binary_dispatcher<F, V, R, Types...>::apply_const(v0, v1, std::forward<F>(f));
+    }
+
+    VARIANT_INLINE static R apply(V& v0, V& v1, F&& f)
+    {
+        if (v0.template is<T>())
+        {
+            if (v1.template is<T>())
+            {
+                return f(unwrapper<T>::apply(v0.template get_unchecked<T>()),
+                         unwrapper<T>::apply(v1.template get_unchecked<T>())); // call binary functor
+            }
+            else
+            {
+                return binary_dispatcher_rhs<F, V, R, T, Types...>::apply(v0, v1, std::forward<F>(f));
+            }
+        }
+        else if (v1.template is<T>())
+        {
+            return binary_dispatcher_lhs<F, V, R, T, Types...>::apply(v0, v1, std::forward<F>(f));
+        }
+        return binary_dispatcher<F, V, R, Types...>::apply(v0, v1, std::forward<F>(f));
+    }
+};
+
+template <typename F, typename V, typename R, typename T>
+struct binary_dispatcher<F, V, R, T>
+{
+    VARIANT_INLINE static R apply_const(V const& v0, V const& v1, F&& f)
+    {
+        return f(unwrapper<T>::apply_const(v0.template get_unchecked<T>()),
+                 unwrapper<T>::apply_const(v1.template get_unchecked<T>())); // call binary functor
+    }
+
+    VARIANT_INLINE static R apply(V& v0, V& v1, F&& f)
+    {
+        return f(unwrapper<T>::apply(v0.template get_unchecked<T>()),
+                 unwrapper<T>::apply(v1.template get_unchecked<T>())); // call binary functor
+    }
+};
+
+// comparator functors
+struct equal_comp
+{
+    template <typename T>
+    bool operator()(T const& lhs, T const& rhs) const
+    {
+        return lhs == rhs;
+    }
+};
+
+struct less_comp
+{
+    template <typename T>
+    bool operator()(T const& lhs, T const& rhs) const
+    {
+        return lhs < rhs;
+    }
+};
+
+template <typename Variant, typename Comp>
+class comparer
+{
+public:
+    explicit comparer(Variant const& lhs) noexcept
+        : lhs_(lhs) {}
+    comparer& operator=(comparer const&) = delete;
+    // visitor
+    template <typename T>
+    bool operator()(T const& rhs_content) const
+    {
+        T const& lhs_content = lhs_.template get_unchecked<T>();
+        return Comp()(lhs_content, rhs_content);
+    }
+
+private:
+    Variant const& lhs_;
+};
+
+// hashing visitor
+struct hasher
+{
+    template <typename T>
+    std::size_t operator()(const T& hashable) const
+    {
+        return std::hash<T>{}(hashable);
+    }
+};
+
+} // namespace detail
+
+struct no_init {};
+
+template <typename... Types>
+class variant
+{
+    static_assert(sizeof...(Types) > 0, "Template parameter type list of variant can not be empty.");
+    static_assert(!detail::disjunction<std::is_reference<Types>...>::value, "Variant can not hold reference types. Maybe use std::reference_wrapper?");
+    static_assert(!detail::disjunction<std::is_array<Types>...>::value, "Variant can not hold array types.");
+    static_assert(sizeof...(Types) < std::numeric_limits<type_index_t>::max(), "Internal index type must be able to accommodate all alternatives.");
+private:
+    static const std::size_t data_size = detail::static_max<sizeof(Types)...>::value;
+    static const std::size_t data_align = detail::static_max<alignof(Types)...>::value;
+public:
+    struct adapted_variant_tag;
+    using types = std::tuple<Types...>;
+private:
+    using first_type = typename std::tuple_element<0, types>::type;
+    using data_type = typename std::aligned_storage<data_size, data_align>::type;
+    using helper_type = detail::variant_helper<Types...>;
+
+    type_index_t type_index;
+    data_type data;
+
+public:
+    VARIANT_INLINE variant() noexcept(std::is_nothrow_default_constructible<first_type>::value)
+        : type_index(sizeof...(Types)-1)
+    {
+        static_assert(std::is_default_constructible<first_type>::value, "First type in variant must be default constructible to allow default construction of variant.");
+        new (&data) first_type();
+    }
+
+    VARIANT_INLINE variant(no_init) noexcept
+        : type_index(detail::invalid_value) {}
+
+    // http://isocpp.org/blog/2012/11/universal-references-in-c11-scott-meyers
+    template <typename T, typename Traits = detail::value_traits<T, Types...>,
+              typename Enable = typename std::enable_if<Traits::is_valid && !std::is_same<variant<Types...>, typename Traits::value_type>::value>::type >
+    VARIANT_INLINE variant(T&& val) noexcept(std::is_nothrow_constructible<typename Traits::target_type, T&&>::value)
+        : type_index(Traits::index)
+    {
+        new (&data) typename Traits::target_type(std::forward<T>(val));
+    }
+
+    VARIANT_INLINE variant(variant<Types...> const& old)
+        : type_index(old.type_index)
+    {
+        helper_type::copy(old.type_index, &old.data, &data);
+    }
+
+    VARIANT_INLINE variant(variant<Types...>&& old)
+        noexcept(detail::conjunction<std::is_nothrow_move_constructible<Types>...>::value)
+        : type_index(old.type_index)
+    {
+        helper_type::move(old.type_index, &old.data, &data);
+    }
+
+private:
+    VARIANT_INLINE void copy_assign(variant<Types...> const& rhs)
+    {
+        helper_type::destroy(type_index, &data);
+        type_index = detail::invalid_value;
+        helper_type::copy(rhs.type_index, &rhs.data, &data);
+        type_index = rhs.type_index;
+    }
+
+    VARIANT_INLINE void move_assign(variant<Types...>&& rhs)
+    {
+        helper_type::destroy(type_index, &data);
+        type_index = detail::invalid_value;
+        helper_type::move(rhs.type_index, &rhs.data, &data);
+        type_index = rhs.type_index;
+    }
+
+public:
+    VARIANT_INLINE variant<Types...>& operator=(variant<Types...>&& other)
+    {
+        move_assign(std::move(other));
+        return *this;
+    }
+
+    VARIANT_INLINE variant<Types...>& operator=(variant<Types...> const& other)
+    {
+        copy_assign(other);
+        return *this;
+    }
+
+    // conversions
+    // move-assign
+    template <typename T>
+    VARIANT_INLINE variant<Types...>& operator=(T&& rhs) noexcept
+    {
+        variant<Types...> temp(std::forward<T>(rhs));
+        move_assign(std::move(temp));
+        return *this;
+    }
+
+    // copy-assign
+    template <typename T>
+    VARIANT_INLINE variant<Types...>& operator=(T const& rhs)
+    {
+        variant<Types...> temp(rhs);
+        copy_assign(temp);
+        return *this;
+    }
+
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<T, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE bool is() const
+    {
+        return type_index == detail::direct_type<T, Types...>::index;
+    }
+
+    template <typename T,typename std::enable_if<
+                         (detail::direct_type<recursive_wrapper<T>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE bool is() const
+    {
+        return type_index == detail::direct_type<recursive_wrapper<T>, Types...>::index;
+    }
+
+    VARIANT_INLINE bool valid() const
+    {
+        return type_index != detail::invalid_value;
+    }
+
+    template <typename T, typename... Args>
+    VARIANT_INLINE void set(Args&&... args)
+    {
+        helper_type::destroy(type_index, &data);
+        type_index = detail::invalid_value;
+        new (&data) T(std::forward<Args>(args)...);
+        type_index = detail::direct_type<T, Types...>::index;
+    }
+
+    // get_unchecked<T>()
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<T, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T& get_unchecked()
+    {
+        return *reinterpret_cast<T*>(&data);
+    }
+
+#ifdef HAS_EXCEPTIONS
+    // get<T>()
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<T, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T& get()
+    {
+        if (type_index == detail::direct_type<T, Types...>::index)
+        {
+            return *reinterpret_cast<T*>(&data);
+        }
+        else
+        {
+            throw bad_variant_access("in get<T>()");
+        }
+    }
+#endif
+
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<T, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T const& get_unchecked() const
+    {
+        return *reinterpret_cast<T const*>(&data);
+    }
+
+#ifdef HAS_EXCEPTIONS
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<T, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T const& get() const
+    {
+        if (type_index == detail::direct_type<T, Types...>::index)
+        {
+            return *reinterpret_cast<T const*>(&data);
+        }
+        else
+        {
+            throw bad_variant_access("in get<T>()");
+        }
+    }
+#endif
+
+    // get_unchecked<T>() - T stored as recursive_wrapper<T>
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<recursive_wrapper<T>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T& get_unchecked()
+    {
+        return (*reinterpret_cast<recursive_wrapper<T>*>(&data)).get();
+    }
+
+#ifdef HAS_EXCEPTIONS
+    // get<T>() - T stored as recursive_wrapper<T>
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<recursive_wrapper<T>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T& get()
+    {
+        if (type_index == detail::direct_type<recursive_wrapper<T>, Types...>::index)
+        {
+            return (*reinterpret_cast<recursive_wrapper<T>*>(&data)).get();
+        }
+        else
+        {
+            throw bad_variant_access("in get<T>()");
+        }
+    }
+#endif
+
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<recursive_wrapper<T>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T const& get_unchecked() const
+    {
+        return (*reinterpret_cast<recursive_wrapper<T> const*>(&data)).get();
+    }
+
+#ifdef HAS_EXCEPTIONS
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<recursive_wrapper<T>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T const& get() const
+    {
+        if (type_index == detail::direct_type<recursive_wrapper<T>, Types...>::index)
+        {
+            return (*reinterpret_cast<recursive_wrapper<T> const*>(&data)).get();
+        }
+        else
+        {
+            throw bad_variant_access("in get<T>()");
+        }
+    }
+#endif
+
+    // get_unchecked<T>() - T stored as std::reference_wrapper<T>
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<std::reference_wrapper<T>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T& get_unchecked()
+    {
+        return (*reinterpret_cast<std::reference_wrapper<T>*>(&data)).get();
+    }
+
+#ifdef HAS_EXCEPTIONS
+    // get<T>() - T stored as std::reference_wrapper<T>
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<std::reference_wrapper<T>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T& get()
+    {
+        if (type_index == detail::direct_type<std::reference_wrapper<T>, Types...>::index)
+        {
+            return (*reinterpret_cast<std::reference_wrapper<T>*>(&data)).get();
+        }
+        else
+        {
+            throw bad_variant_access("in get<T>()");
+        }
+    }
+#endif
+
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<std::reference_wrapper<T const>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T const& get_unchecked() const
+    {
+        return (*reinterpret_cast<std::reference_wrapper<T const> const*>(&data)).get();
+    }
+
+#ifdef HAS_EXCEPTIONS
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<std::reference_wrapper<T const>, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE T const& get() const
+    {
+        if (type_index == detail::direct_type<std::reference_wrapper<T const>, Types...>::index)
+        {
+            return (*reinterpret_cast<std::reference_wrapper<T const> const*>(&data)).get();
+        }
+        else
+        {
+            throw bad_variant_access("in get<T>()");
+        }
+    }
+#endif
+
+    // This function is deprecated because it returns an internal index field.
+    // Use which() instead.
+    ARROW_VARIANT_DEPRECATED VARIANT_INLINE type_index_t get_type_index() const
+    {
+        return type_index;
+    }
+
+    VARIANT_INLINE int which() const noexcept
+    {
+        return static_cast<int>(sizeof...(Types) - type_index - 1);
+    }
+
+    template <typename T, typename std::enable_if<
+                          (detail::direct_type<T, Types...>::index != detail::invalid_value)>::type* = nullptr>
+    VARIANT_INLINE static constexpr int which() noexcept
+    {
+        return static_cast<int>(sizeof...(Types)-detail::direct_type<T, Types...>::index - 1);
+    }
+
+    // visitor
+    // unary
+    template <typename F, typename V, typename R = typename detail::result_of_unary_visit<F, first_type>::type>
+    auto VARIANT_INLINE static visit(V const& v, F&& f)
+        -> decltype(detail::dispatcher<F, V, R, Types...>::apply_const(v, std::forward<F>(f)))
+    {
+        return detail::dispatcher<F, V, R, Types...>::apply_const(v, std::forward<F>(f));
+    }
+    // non-const
+    template <typename F, typename V, typename R = typename detail::result_of_unary_visit<F, first_type>::type>
+    auto VARIANT_INLINE static visit(V& v, F&& f)
+        -> decltype(detail::dispatcher<F, V, R, Types...>::apply(v, std::forward<F>(f)))
+    {
+        return detail::dispatcher<F, V, R, Types...>::apply(v, std::forward<F>(f));
+    }
+
+    // binary
+    // const
+    template <typename F, typename V, typename R = typename detail::result_of_binary_visit<F, first_type>::type>
+    auto VARIANT_INLINE static binary_visit(V const& v0, V const& v1, F&& f)
+        -> decltype(detail::binary_dispatcher<F, V, R, Types...>::apply_const(v0, v1, std::forward<F>(f)))
+    {
+        return detail::binary_dispatcher<F, V, R, Types...>::apply_const(v0, v1, std::forward<F>(f));
+    }
+    // non-const
+    template <typename F, typename V, typename R = typename detail::result_of_binary_visit<F, first_type>::type>
+    auto VARIANT_INLINE static binary_visit(V& v0, V& v1, F&& f)
+        -> decltype(detail::binary_dispatcher<F, V, R, Types...>::apply(v0, v1, std::forward<F>(f)))
+    {
+        return detail::binary_dispatcher<F, V, R, Types...>::apply(v0, v1, std::forward<F>(f));
+    }
+
+    // match
+    // unary
+    template <typename... Fs>
+    auto VARIANT_INLINE match(Fs&&... fs) const
+        -> decltype(variant::visit(*this, ::arrow::util::make_visitor(std::forward<Fs>(fs)...)))
+    {
+        return variant::visit(*this, ::arrow::util::make_visitor(std::forward<Fs>(fs)...));
+    }
+    // non-const
+    template <typename... Fs>
+    auto VARIANT_INLINE match(Fs&&... fs)
+        -> decltype(variant::visit(*this, ::arrow::util::make_visitor(std::forward<Fs>(fs)...)))
+    {
+        return variant::visit(*this, ::arrow::util::make_visitor(std::forward<Fs>(fs)...));
+    }
+
+    ~variant() noexcept // no-throw destructor
+    {
+        helper_type::destroy(type_index, &data);
+    }
+
+    // comparison operators
+    // equality
+    VARIANT_INLINE bool operator==(variant const& rhs) const
+    {
+        assert(valid() && rhs.valid());
+        if (this->which() != rhs.which())
+        {
+            return false;
+        }
+        detail::comparer<variant, detail::equal_comp> visitor(*this);
+        return visit(rhs, visitor);
+    }
+
+    VARIANT_INLINE bool operator!=(variant const& rhs) const
+    {
+        return !(*this == rhs);
+    }
+
+    // less than
+    VARIANT_INLINE bool operator<(variant const& rhs) const
+    {
+        assert(valid() && rhs.valid());
+        if (this->which() != rhs.which())
+        {
+            return this->which() < rhs.which();
+        }
+        detail::comparer<variant, detail::less_comp> visitor(*this);
+        return visit(rhs, visitor);
+    }
+    VARIANT_INLINE bool operator>(variant const& rhs) const
+    {
+        return rhs < *this;
+    }
+    VARIANT_INLINE bool operator<=(variant const& rhs) const
+    {
+        return !(*this > rhs);
+    }
+    VARIANT_INLINE bool operator>=(variant const& rhs) const
+    {
+        return !(*this < rhs);
+    }
+};
+
+// unary visitor interface
+// const
+template <typename F, typename V>
+auto VARIANT_INLINE apply_visitor(F&& f, V const& v) -> decltype(V::visit(v, std::forward<F>(f)))
+{
+    return V::visit(v, std::forward<F>(f));
+}
+
+// non-const
+template <typename F, typename V>
+auto VARIANT_INLINE apply_visitor(F&& f, V& v) -> decltype(V::visit(v, std::forward<F>(f)))
+{
+    return V::visit(v, std::forward<F>(f));
+}
+
+// binary visitor interface
+// const
+template <typename F, typename V>
+auto VARIANT_INLINE apply_visitor(F&& f, V const& v0, V const& v1) -> decltype(V::binary_visit(v0, v1, std::forward<F>(f)))
+{
+    return V::binary_visit(v0, v1, std::forward<F>(f));
+}
+
+// non-const
+template <typename F, typename V>
+auto VARIANT_INLINE apply_visitor(F&& f, V& v0, V& v1) -> decltype(V::binary_visit(v0, v1, std::forward<F>(f)))
+{
+    return V::binary_visit(v0, v1, std::forward<F>(f));
+}
+
+// getter interface
+
+#ifdef HAS_EXCEPTIONS
+template <typename ResultType, typename T>
+auto get(T& var)->decltype(var.template get<ResultType>())
+{
+    return var.template get<ResultType>();
+}
+#endif
+
+template <typename ResultType, typename T>
+ResultType& get_unchecked(T& var)
+{
+    return var.template get_unchecked<ResultType>();
+}
+
+#ifdef HAS_EXCEPTIONS
+template <typename ResultType, typename T>
+auto get(T const& var)->decltype(var.template get<ResultType>())
+{
+    return var.template get<ResultType>();
+}
+#endif
+
+template <typename ResultType, typename T>
+ResultType const& get_unchecked(T const& var)
+{
+    return var.template get_unchecked<ResultType>();
+}
+// variant_size
+template <typename T>
+struct variant_size;
+
+//variable templates is c++14
+//template <typename T>
+//constexpr std::size_t variant_size_v = variant_size<T>::value;
+
+template <typename T>
+struct variant_size<const T>
+    : variant_size<T> {};
+
+template <typename T>
+struct variant_size<volatile T>
+    : variant_size<T> {};
+
+template <typename T>
+struct variant_size<const volatile T>
+    : variant_size<T> {};
+
+template <typename... Types>
+struct variant_size<variant<Types...>>
+    : std::integral_constant<std::size_t, sizeof...(Types)> {};
+
+// variant_alternative
+template <std::size_t Index, typename T>
+struct variant_alternative;
+
+#if defined(__clang__)
+#if __has_builtin(__type_pack_element)
+#define has_type_pack_element
+#endif
+#endif
+
+#if defined(has_type_pack_element)
+template <std::size_t Index, typename ...Types>
+struct variant_alternative<Index, variant<Types...>>
+{
+    static_assert(sizeof...(Types) > Index , "Index out of range");
+    using type = __type_pack_element<Index, Types...>;
+};
+#else
+template <std::size_t Index, typename First, typename...Types>
+struct variant_alternative<Index, variant<First, Types...>>
+    : variant_alternative<Index - 1, variant<Types...>>
+{
+    static_assert(sizeof...(Types) > Index -1 , "Index out of range");
+};
+
+template <typename First, typename...Types>
+struct variant_alternative<0, variant<First, Types...>>
+{
+    using type = First;
+};
+
+#endif
+
+template <size_t Index, typename T>
+using variant_alternative_t = typename variant_alternative<Index, T>::type;
+
+template <size_t Index, typename T>
+struct variant_alternative<Index, const T>
+    : std::add_const<variant_alternative<Index, T>> {};
+
+template <size_t Index, typename T>
+struct variant_alternative<Index, volatile T>
+    : std::add_volatile<variant_alternative<Index, T>> {};
+
+template <size_t Index, typename T>
+struct variant_alternative<Index, const volatile T>
+    : std::add_cv<variant_alternative<Index, T>> {};
+
+} // namespace util
+} // namespace arrow
+
+#endif // ARROW_UTIL_VARIANT_H
diff --git a/cpp/src/arrow/util/variant/CMakeLists.txt b/cpp/src/arrow/util/variant/CMakeLists.txt
new file mode 100644
index 0000000000000..0ebb2516246ed
--- /dev/null
+++ b/cpp/src/arrow/util/variant/CMakeLists.txt
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######################################
+# arrow_util_variant
+#######################################
+
+install(FILES
+  optional.h
+  recursive_wrapper.h
+  variant_cast.h
+  variant_io.h
+  variant_visitor.h
+  DESTINATION include/arrow/util/variant)
diff --git a/cpp/src/arrow/util/variant/optional.h b/cpp/src/arrow/util/variant/optional.h
new file mode 100644
index 0000000000000..4c6671061fe80
--- /dev/null
+++ b/cpp/src/arrow/util/variant/optional.h
@@ -0,0 +1,100 @@
+// Copyright (c) MapBox
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// - Redistributions of source code must retain the above copyright notice, this
+//   list of conditions and the following disclaimer.
+// - Redistributions in binary form must reproduce the above copyright notice, this
+//   list of conditions and the following disclaimer in the documentation and/or
+//   other materials provided with the distribution.
+// - Neither the name "MapBox" nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef ARROW_UTIL_VARIANT_OPTIONAL_H
+#define ARROW_UTIL_VARIANT_OPTIONAL_H
+
+#pragma message("This implementation of optional is deprecated. See https://github.com/mapbox/variant/issues/64.")
+
+#include <type_traits>
+#include <utility>
+
+#include <arrow/util/variant.h>
+
+namespace arrow {
+namespace util {
+
+template <typename T>
+class optional
+{
+    static_assert(!std::is_reference<T>::value, "optional doesn't support references");
+
+    struct none_type
+    {
+    };
+
+    variant<none_type, T> variant_;
+
+public:
+    optional() = default;
+
+    optional(optional const& rhs)
+    {
+        if (this != &rhs)
+        { // protect against invalid self-assignment
+            variant_ = rhs.variant_;
+        }
+    }
+
+    optional(T const& v) { variant_ = v; }
+
+    explicit operator bool() const noexcept { return variant_.template is<T>(); }
+
+    T const& get() const { return variant_.template get<T>(); }
+    T& get() { return variant_.template get<T>(); }
+
+    T const& operator*() const { return this->get(); }
+    T operator*() { return this->get(); }
+
+    optional& operator=(T const& v)
+    {
+        variant_ = v;
+        return *this;
+    }
+
+    optional& operator=(optional const& rhs)
+    {
+        if (this != &rhs)
+        {
+            variant_ = rhs.variant_;
+        }
+        return *this;
+    }
+
+    template <typename... Args>
+    void emplace(Args&&... args)
+    {
+        variant_ = T{std::forward<Args>(args)...};
+    }
+
+    void reset() { variant_ = none_type{}; }
+
+}; // class optional
+
+} // namespace util
+} // namespace arrow
+
+#endif // ARROW_UTIL_VARIANT_OPTIONAL_H
diff --git a/cpp/src/arrow/util/variant/recursive_wrapper.h b/cpp/src/arrow/util/variant/recursive_wrapper.h
new file mode 100644
index 0000000000000..c9d9385394b38
--- /dev/null
+++ b/cpp/src/arrow/util/variant/recursive_wrapper.h
@@ -0,0 +1,122 @@
+#ifndef ARROW_UTIL_VARIANT_RECURSIVE_WRAPPER_H
+#define ARROW_UTIL_VARIANT_RECURSIVE_WRAPPER_H
+
+// Based on variant/recursive_wrapper.h from boost.
+//
+// Original license:
+//
+// Copyright (c) 2002-2003
+// Eric Friedman, Itay Maman
+//
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#include <cassert>
+#include <utility>
+
+namespace arrow {
+namespace util {
+
+template <typename T>
+class recursive_wrapper
+{
+
+    T* p_;
+
+    void assign(T const& rhs)
+    {
+        this->get() = rhs;
+    }
+
+public:
+    using type = T;
+
+    /**
+     * Default constructor default initializes the internally stored value.
+     * For POD types this means nothing is done and the storage is
+     * uninitialized.
+     *
+     * @throws std::bad_alloc if there is insufficient memory for an object
+     *         of type T.
+     * @throws any exception thrown by the default constructur of T.
+     */
+    recursive_wrapper()
+        : p_(new T){}
+
+    ~recursive_wrapper() noexcept { delete p_; }
+
+    recursive_wrapper(recursive_wrapper const& operand)
+        : p_(new T(operand.get())) {}
+
+    recursive_wrapper(T const& operand)
+        : p_(new T(operand)) {}
+
+    recursive_wrapper(recursive_wrapper&& operand)
+        : p_(new T(std::move(operand.get()))) {}
+
+    recursive_wrapper(T&& operand)
+        : p_(new T(std::move(operand))) {}
+
+    inline recursive_wrapper& operator=(recursive_wrapper const& rhs)
+    {
+        assign(rhs.get());
+        return *this;
+    }
+
+    inline recursive_wrapper& operator=(T const& rhs)
+    {
+        assign(rhs);
+        return *this;
+    }
+
+    inline void swap(recursive_wrapper& operand) noexcept
+    {
+        T* temp = operand.p_;
+        operand.p_ = p_;
+        p_ = temp;
+    }
+
+    recursive_wrapper& operator=(recursive_wrapper&& rhs) noexcept
+    {
+        swap(rhs);
+        return *this;
+    }
+
+    recursive_wrapper& operator=(T&& rhs)
+    {
+        get() = std::move(rhs);
+        return *this;
+    }
+
+    T& get()
+    {
+        assert(p_);
+        return *get_pointer();
+    }
+
+    T const& get() const
+    {
+        assert(p_);
+        return *get_pointer();
+    }
+
+    T* get_pointer() { return p_; }
+
+    const T* get_pointer() const { return p_; }
+
+    operator T const&() const { return this->get(); }
+
+    operator T&() { return this->get(); }
+
+}; // class recursive_wrapper
+
+template <typename T>
+inline void swap(recursive_wrapper<T>& lhs, recursive_wrapper<T>& rhs) noexcept
+{
+    lhs.swap(rhs);
+}
+} // namespace util
+} // namespace arrow
+
+#endif // ARROW_UTIL_VARIANT_RECURSIVE_WRAPPER_H
diff --git a/cpp/src/arrow/util/variant/variant_cast.h b/cpp/src/arrow/util/variant/variant_cast.h
new file mode 100644
index 0000000000000..558f1d9a60c20
--- /dev/null
+++ b/cpp/src/arrow/util/variant/variant_cast.h
@@ -0,0 +1,112 @@
+// Copyright (c) MapBox
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// - Redistributions of source code must retain the above copyright notice, this
+//   list of conditions and the following disclaimer.
+// - Redistributions in binary form must reproduce the above copyright notice, this
+//   list of conditions and the following disclaimer in the documentation and/or
+//   other materials provided with the distribution.
+// - Neither the name "MapBox" nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef ARROW_UTIL_VARIANT_CAST_H
+#define ARROW_UTIL_VARIANT_CAST_H
+
+#include <type_traits>
+
+namespace arrow {
+namespace util {
+
+namespace detail {
+
+template <class T>
+class static_caster
+{
+public:
+    template <class V>
+    T& operator()(V& v) const
+    {
+        return static_cast<T&>(v);
+    }
+};
+
+template <class T>
+class dynamic_caster
+{
+public:
+    using result_type = T&;
+    template <class V>
+    T& operator()(V& v, typename std::enable_if<!std::is_polymorphic<V>::value>::type* = nullptr) const
+    {
+        throw std::bad_cast();
+    }
+    template <class V>
+    T& operator()(V& v, typename std::enable_if<std::is_polymorphic<V>::value>::type* = nullptr) const
+    {
+        return dynamic_cast<T&>(v);
+    }
+};
+
+template <class T>
+class dynamic_caster<T*>
+{
+public:
+    using result_type = T*;
+    template <class V>
+    T* operator()(V& v, typename std::enable_if<!std::is_polymorphic<V>::value>::type* = nullptr) const
+    {
+        return nullptr;
+    }
+    template <class V>
+    T* operator()(V& v, typename std::enable_if<std::is_polymorphic<V>::value>::type* = nullptr) const
+    {
+        return dynamic_cast<T*>(&v);
+    }
+};
+}
+
+template <class T, class V>
+typename detail::dynamic_caster<T>::result_type
+dynamic_variant_cast(V& v)
+{
+    return arrow::util::apply_visitor(detail::dynamic_caster<T>(), v);
+}
+
+template <class T, class V>
+typename detail::dynamic_caster<const T>::result_type
+dynamic_variant_cast(const V& v)
+{
+    return arrow::util::apply_visitor(detail::dynamic_caster<const T>(), v);
+}
+
+template <class T, class V>
+T& static_variant_cast(V& v)
+{
+    return arrow::util::apply_visitor(detail::static_caster<T>(), v);
+}
+
+template <class T, class V>
+const T& static_variant_cast(const V& v)
+{
+    return arrow::util::apply_visitor(detail::static_caster<const T>(), v);
+}
+
+}  // namespace util
+}  // namespace arrow
+
+#endif // ARROW_UTIL_VARIANT_CAST_H
diff --git a/cpp/src/arrow/util/variant/variant_io.h b/cpp/src/arrow/util/variant/variant_io.h
new file mode 100644
index 0000000000000..5541a81f7035f
--- /dev/null
+++ b/cpp/src/arrow/util/variant/variant_io.h
@@ -0,0 +1,72 @@
+// Copyright (c) MapBox
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// - Redistributions of source code must retain the above copyright notice, this
+//   list of conditions and the following disclaimer.
+// - Redistributions in binary form must reproduce the above copyright notice, this
+//   list of conditions and the following disclaimer in the documentation and/or
+//   other materials provided with the distribution.
+// - Neither the name "MapBox" nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef ARROW_UTIL_VARIANT_IO_H
+#define ARROW_UTIL_VARIANT_IO_H
+
+#include <iosfwd>
+
+#include <arrow/util/variant.h>
+
+namespace arrow {
+namespace util {
+
+namespace detail {
+// operator<< helper
+template <typename Out>
+class printer
+{
+public:
+    explicit printer(Out& out)
+        : out_(out) {}
+    printer& operator=(printer const&) = delete;
+
+    // visitor
+    template <typename T>
+    void operator()(T const& operand) const
+    {
+        out_ << operand;
+    }
+
+private:
+    Out& out_;
+};
+}
+
+// operator<<
+template <typename CharT, typename Traits, typename... Types>
+VARIANT_INLINE std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& out, variant<Types...> const& rhs)
+{
+    detail::printer<std::basic_ostream<CharT, Traits>> visitor(out);
+    apply_visitor(visitor, rhs);
+    return out;
+}
+
+} // namespace util
+} // namespace arrow
+
+#endif // ARROW_UTIL_VARIANT_IO_H
diff --git a/cpp/src/arrow/util/variant/variant_visitor.h b/cpp/src/arrow/util/variant/variant_visitor.h
new file mode 100644
index 0000000000000..66b1dfea3d7c9
--- /dev/null
+++ b/cpp/src/arrow/util/variant/variant_visitor.h
@@ -0,0 +1,69 @@
+// Copyright (c) MapBox
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// - Redistributions of source code must retain the above copyright notice, this
+//   list of conditions and the following disclaimer.
+// - Redistributions in binary form must reproduce the above copyright notice, this
+//   list of conditions and the following disclaimer in the documentation and/or
+//   other materials provided with the distribution.
+// - Neither the name "MapBox" nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef ARROW_UTIL_VARIANT_VISITOR_HPP
+#define ARROW_UTIL_VARIANT_VISITOR_HPP
+
+#include <utility>
+
+namespace arrow {
+namespace util {
+
+template <typename... Fns>
+struct visitor;
+
+template <typename Fn>
+struct visitor<Fn> : Fn
+{
+    using Fn::operator();
+
+    template<typename T>
+    visitor(T&& fn) : Fn(std::forward<T>(fn)) {}
+};
+
+template <typename Fn, typename... Fns>
+struct visitor<Fn, Fns...> : Fn, visitor<Fns...>
+{
+    using Fn::operator();
+    using visitor<Fns...>::operator();
+
+    template<typename T, typename... Ts>
+    visitor(T&& fn, Ts&&... fns)
+        : Fn(std::forward<T>(fn))
+        , visitor<Fns...>(std::forward<Ts>(fns)...) {}
+};
+
+template <typename... Fns>
+visitor<typename std::decay<Fns>::type...> make_visitor(Fns&&... fns)
+{
+    return visitor<typename std::decay<Fns>::type...>
+        (std::forward<Fns>(fns)...);
+}
+
+} // namespace util
+} // namespace arrow
+
+#endif // ARROW_UTIL_VARIANT_VISITOR_HPP
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index 3739e89f3d019..47dba6cd8ddf2 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -90,7 +90,7 @@ TYPE_VISITOR_DEFAULT(Time32Type);
 TYPE_VISITOR_DEFAULT(Time64Type);
 TYPE_VISITOR_DEFAULT(TimestampType);
 TYPE_VISITOR_DEFAULT(IntervalType);
-TYPE_VISITOR_DEFAULT(DecimalType);
+TYPE_VISITOR_DEFAULT(Decimal128Type);
 TYPE_VISITOR_DEFAULT(ListType);
 TYPE_VISITOR_DEFAULT(StructType);
 TYPE_VISITOR_DEFAULT(UnionType);
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index 72c82a3bb871b..41b0108aeb116 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -97,7 +97,7 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
     ARRAY_VISIT_INLINE(TimestampType);
     ARRAY_VISIT_INLINE(Time32Type);
     ARRAY_VISIT_INLINE(Time64Type);
-    ARRAY_VISIT_INLINE(DecimalType);
+    ARRAY_VISIT_INLINE(Decimal128Type);
     ARRAY_VISIT_INLINE(ListType);
     ARRAY_VISIT_INLINE(StructType);
     ARRAY_VISIT_INLINE(UnionType);
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index db3b3aa5fac9c..bf962bcd49e42 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -10,6 +10,12 @@ cpp/src/arrow/io/mman.h
 cpp/src/arrow/util/random.h
 cpp/src/arrow/status.cc
 cpp/src/arrow/status.h
+cpp/src/arrow/util/variant.h
+cpp/src/arrow/util/variant/optional.h
+cpp/src/arrow/util/variant/recursive_wrapper.h
+cpp/src/arrow/util/variant/variant_cast.h
+cpp/src/arrow/util/variant/variant_io.h
+cpp/src/arrow/util/variant/variant_visitor.h
 cpp/build-support/asan_symbolize.py
 cpp/build-support/cpplint.py
 cpp/build-support/clang_format_exclusions.txt
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index c52024044a62d..8f2f23d9fd470 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -132,6 +132,7 @@ Array Types
 .. autosummary::
    :toctree: generated/
 
+   array
    Array
    BooleanArray
    DictionaryArray
@@ -168,6 +169,8 @@ Tables and Record Batches
 .. autosummary::
    :toctree: generated/
 
+   column
+   chunked_array
    ChunkedArray
    Column
    RecordBatch
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 7ef6a722be1dc..885613725a10a 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -84,7 +84,7 @@ from conda-forge:
    conda create -y -q -n pyarrow-dev \
          python=3.6 numpy six setuptools cython pandas pytest \
          cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
-         brotli jemalloc lz4-c zstd -c conda-forge
+         gflags brotli jemalloc lz4-c zstd -c conda-forge
    source activate pyarrow-dev
 
 
@@ -256,17 +256,11 @@ First, starting from fresh clones of Apache Arrow and parquet-cpp:
 
 .. code-block:: shell
 
-   conda create -n arrow-dev cmake git boost-cpp ^
-         flatbuffers snappy zlib brotli thrift-cpp rapidjson ^
-         -c conda-forge
-   activate arrow-dev
-
-As one git housekeeping item, we must run this command in our Arrow clone:
-
-.. code-block:: shell
-
-   cd arrow
-   git config core.symlinks true
+   conda create -y -q -n pyarrow-dev ^
+         python=3.6 numpy six setuptools cython pandas pytest ^
+         cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib ^
+         gflags brotli lz4-c zstd -c conda-forge
+   activate pyarrow-dev
 
 Now, we build and install Arrow C++ libraries
 
@@ -280,7 +274,7 @@ Now, we build and install Arrow C++ libraries
          -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
          -DCMAKE_BUILD_TYPE=Release ^
          -DARROW_BUILD_TESTS=on ^
-         -DARROW_CXXFLAGS="/WX" ^
+         -DARROW_CXXFLAGS="/WX /MP" ^
          -DARROW_PYTHON=on ..
    cmake --build . --target INSTALL --config Release
    cd ..\..
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 09bf6b35f396a..c8ded2d3cb88d 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -43,7 +43,7 @@
                          Schema,
                          schema,
                          Array, Tensor,
-                         array,
+                         array, chunked_array, column,
                          from_numpy_dtype,
                          NullArray,
                          NumericArray, IntegerArray, FloatingPointArray,
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 2ef592ff758fc..cca9425881b00 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -77,7 +77,7 @@ cdef _ndarray_to_array(object values, object mask, DataType type,
         return pyarrow_wrap_array(chunked_out.get().chunk(0))
 
 
-cdef DataType _ensure_type(object type):
+cdef inline DataType _ensure_type(object type):
     if type is None:
         return None
     elif not isinstance(type, DataType):
@@ -228,6 +228,15 @@ cdef CFunctionContext* _context() nogil:
     return _global_ctx.ctx.get()
 
 
+cdef wrap_datum(const CDatum& datum):
+    if datum.kind() == DatumType_ARRAY:
+        return pyarrow_wrap_array(MakeArray(datum.array()))
+    elif datum.kind() == DatumType_CHUNKED_ARRAY:
+        return pyarrow_wrap_chunked_array(datum.chunked_array())
+    else:
+        raise ValueError("Unable to wrap Datum in a Python object")
+
+
 cdef class Array:
 
     cdef void init(self, const shared_ptr[CArray]& sp_array):
@@ -270,6 +279,29 @@ cdef class Array:
 
         return pyarrow_wrap_array(result)
 
+    def unique(self):
+        """
+        Compute distinct elements in array
+        """
+        cdef shared_ptr[CArray] result
+
+        with nogil:
+            check_status(Unique(_context(), CDatum(self.sp_array), &result))
+
+        return pyarrow_wrap_array(result)
+
+    def dictionary_encode(self):
+        """
+        Compute dictionary-encoded representation of array
+        """
+        cdef CDatum out
+
+        with nogil:
+            check_status(DictionaryEncode(_context(), CDatum(self.sp_array),
+                                          &out))
+
+        return wrap_datum(out)
+
     @staticmethod
     def from_pandas(obj, mask=None, type=None, MemoryPool memory_pool=None):
         """
@@ -702,6 +734,9 @@ cdef class DictionaryArray(Array):
             return box_scalar(dictionary.type, dictionary.sp_array,
                               index.as_py())
 
+    def dictionary_encode(self):
+        return self
+
     property dictionary:
 
         def __get__(self):
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 11cc6b3ff2664..dbfd89cc378cb 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -90,6 +90,14 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_string ToString()
 
+    cdef cppclass CArrayData" arrow::ArrayData":
+        shared_ptr[CDataType] type
+        int64_t length
+        int64_t null_count
+        int64_t offset
+        vector[shared_ptr[CBuffer]] buffers
+        vector[shared_ptr[CArrayData]] child_data
+
     cdef cppclass CArray" arrow::Array":
         shared_ptr[CDataType] type()
 
@@ -102,9 +110,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool Equals(const CArray& arr)
         c_bool IsNull(int i)
 
+        shared_ptr[CArrayData] data()
+
         shared_ptr[CArray] Slice(int64_t offset)
         shared_ptr[CArray] Slice(int64_t offset, int64_t length)
 
+    shared_ptr[CArray] MakeArray(const shared_ptr[CArrayData]& data)
+
     CStatus DebugPrint(const CArray& arr, int indent)
 
     cdef cppclass CFixedWidthType" arrow::FixedWidthType"(CDataType):
@@ -363,6 +375,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     CStatus ValidateArray(const CArray& array)
 
     cdef cppclass CChunkedArray" arrow::ChunkedArray":
+        CChunkedArray(const vector[shared_ptr[CArray]]& arrays)
         int64_t length()
         int64_t null_count()
         int num_chunks()
@@ -376,8 +389,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CColumn(const shared_ptr[CField]& field,
                 const vector[shared_ptr[CArray]]& chunks)
 
+        CColumn(const shared_ptr[CField]& field,
+                const shared_ptr[CChunkedArray]& data)
+
         c_bool Equals(const CColumn& other)
 
+        shared_ptr[CField] field()
+
         int64_t length()
         int64_t null_count()
         const c_string& name()
@@ -776,11 +794,42 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
         c_bool allow_int_overflow
         c_bool allow_time_truncate
 
+    enum DatumType" arrow::compute::Datum::type":
+        DatumType_NONE" arrow::compute::Datum::NONE"
+        DatumType_SCALAR" arrow::compute::Datum::SCALAR"
+        DatumType_ARRAY" arrow::compute::Datum::ARRAY"
+        DatumType_CHUNKED_ARRAY" arrow::compute::Datum::CHUNKED_ARRAY"
+        DatumType_RECORD_BATCH" arrow::compute::Datum::RECORD_BATCH"
+        DatumType_TABLE" arrow::compute::Datum::TABLE"
+        DatumType_COLLECTION" arrow::compute::Datum::COLLECTION"
+
+    cdef cppclass CDatum" arrow::compute::Datum":
+        CDatum()
+        CDatum(const shared_ptr[CArray]& value)
+        CDatum(const shared_ptr[CChunkedArray]& value)
+        CDatum(const shared_ptr[CRecordBatch]& value)
+        CDatum(const shared_ptr[CTable]& value)
+
+        DatumType kind()
+
+        shared_ptr[CArrayData] array()
+        shared_ptr[CChunkedArray] chunked_array()
+
     CStatus Cast(CFunctionContext* context, const CArray& array,
                  const shared_ptr[CDataType]& to_type,
                  const CCastOptions& options,
                  shared_ptr[CArray]* out)
 
+    CStatus Cast(CFunctionContext* context, const CDatum& value,
+                 const shared_ptr[CDataType]& to_type,
+                 const CCastOptions& options, CDatum* out)
+
+    CStatus Unique(CFunctionContext* context, const CDatum& value,
+                   shared_ptr[CArray]* out)
+
+    CStatus DictionaryEncode(CFunctionContext* context, const CDatum& value,
+                             CDatum* out)
+
 
 cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     shared_ptr[CDataType] GetPrimitiveType(Type type)
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 41eaf0bac4525..0aab9a41b8025 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -362,7 +362,8 @@ def backwards_compatible_index_name(raw_name, logical_name):
         return logical_name
 
 
-def table_to_blockmanager(options, table, memory_pool, nthreads=1):
+def table_to_blockmanager(options, table, memory_pool, nthreads=1,
+                          categoricals=None):
     import pandas.core.internals as _int
     import pyarrow.lib as lib
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 5ba5f83d22f90..1a9d23db4ebdc 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -108,6 +108,15 @@ cdef class ChunkedArray:
 
         return pyarrow_wrap_array(self.chunked_array.chunk(i))
 
+    property chunks:
+
+        def __get__(self):
+            cdef int i
+            chunks = []
+            for i in range(self.num_chunks):
+                chunks.append(self.chunk(i))
+            return chunks
+
     def iterchunks(self):
         for i in range(self.num_chunks):
             yield self.chunk(i)
@@ -122,6 +131,74 @@ cdef class ChunkedArray:
         return result
 
 
+def chunked_array(arrays, type=None):
+    """
+    Construct chunked array from list of array-like objects
+
+    Parameters
+    ----------
+    arrays : list of Array or values coercible to arrays
+    type : DataType or string coercible to DataType
+
+    Returns
+    -------
+    ChunkedArray
+    """
+    cdef:
+        Array arr
+        vector[shared_ptr[CArray]] c_arrays
+        shared_ptr[CChunkedArray] sp_chunked_array
+
+    for x in arrays:
+        if isinstance(x, Array):
+            arr = x
+            if type is not None:
+                assert x.type == type
+        else:
+            arr = array(x, type=type)
+
+        c_arrays.push_back(arr.sp_array)
+
+    sp_chunked_array.reset(new CChunkedArray(c_arrays))
+    return pyarrow_wrap_chunked_array(sp_chunked_array)
+
+
+def column(object field_or_name, arr):
+    """
+    Create Column object from field/string and array-like data
+    """
+    cdef:
+        Field boxed_field
+        Array _arr
+        ChunkedArray _carr
+        shared_ptr[CColumn] sp_column
+
+    if isinstance(arr, list):
+        arr = chunked_array(arr)
+    elif not isinstance(arr, (Array, ChunkedArray)):
+        arr = array(arr)
+
+    if isinstance(field_or_name, Field):
+        boxed_field = field_or_name
+        if arr.type != boxed_field.type:
+            raise ValueError('Passed field type does not match array')
+    else:
+        boxed_field = field(field_or_name, arr.type)
+
+    if isinstance(arr, Array):
+        _arr = arr
+        sp_column.reset(new CColumn(boxed_field.sp_field, _arr.sp_array))
+    elif isinstance(arr, ChunkedArray):
+        _carr = arr
+        sp_column.reset(new CColumn(boxed_field.sp_field,
+                                    _carr.sp_chunked_array))
+    else:
+        raise ValueError("Unsupported type for column(...): {}"
+                         .format(type(arr)))
+
+    return pyarrow_wrap_column(sp_column)
+
+
 cdef class Column:
     """
     Named vector of elements of equal type.
@@ -143,25 +220,47 @@ cdef class Column:
         result = StringIO()
         result.write(object.__repr__(self))
         data = self.data
-        for i in range(len(data)):
-            result.write('\nchunk {0}: {1}'.format(i, repr(data.chunk(0))))
+        for i, chunk in enumerate(data.chunks):
+            result.write('\nchunk {0}: {1}'.format(i, repr(chunk)))
 
         return result.getvalue()
 
     @staticmethod
-    def from_array(object field_or_name, Array arr):
-        cdef Field boxed_field
+    def from_array(*args):
+        return column(*args)
 
-        if isinstance(field_or_name, Field):
-            boxed_field = field_or_name
-            if arr.type != boxed_field.type:
-                raise ValueError('Passed field type does not match array')
-        else:
-            boxed_field = field(field_or_name, arr.type)
+    def cast(self, object target_type, safe=True):
+        """
+        Cast column values to another data type
+
+        Parameters
+        ----------
+        target_type : DataType
+            Type to cast to
+        safe : boolean, default True
+            Check for overflows or other unsafe conversions
+
+        Returns
+        -------
+        casted : Column
+        """
+        cdef:
+            CCastOptions options
+            shared_ptr[CArray] result
+            DataType type
+            CDatum out
+
+        type = _ensure_type(target_type)
 
-        cdef shared_ptr[CColumn] sp_column
-        sp_column.reset(new CColumn(boxed_field.sp_field, arr.sp_array))
-        return pyarrow_wrap_column(sp_column)
+        options.allow_int_overflow = not safe
+        options.allow_time_truncate = not safe
+
+        with nogil:
+            check_status(Cast(_context(), CDatum(self.column.data()),
+                              type.sp_type, options, &out))
+
+        casted_data = pyarrow_wrap_chunked_array(out.chunked_array())
+        return column(self.name, casted_data)
 
     def to_pandas(self, strings_to_categorical=False, zero_copy_only=False):
         """
@@ -241,6 +340,10 @@ cdef class Column:
         self._check_nullptr()
         return self.column.length()
 
+    @property
+    def field(self):
+        return pyarrow_wrap_field(self.column.field())
+
     @property
     def shape(self):
         """
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 7dc93c28ea7a4..c061e6820753e 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -295,6 +295,18 @@ def test_cast_integers_safe():
             in_arr.cast(out_type)
 
 
+def test_cast_column():
+    arrays = [pa.array([1, 2, 3]), pa.array([4, 5, 6])]
+
+    col = pa.column('foo', arrays)
+
+    target = pa.float64()
+    casted = col.cast(target)
+
+    expected = pa.column('foo', [x.cast(target) for x in arrays])
+    assert casted.equals(expected)
+
+
 def test_cast_integers_unsafe():
     # We let NumPy do the unsafe casting
     unsafe_cases = [
@@ -350,6 +362,33 @@ def test_cast_signed_to_unsigned():
         _check_cast_case(case)
 
 
+def test_unique_simple():
+    cases = [
+        (pa.array([1, 2, 3, 1, 2, 3]), pa.array([1, 2, 3])),
+        (pa.array(['foo', None, 'bar', 'foo']),
+         pa.array(['foo', 'bar']))
+    ]
+    for arr, expected in cases:
+        result = arr.unique()
+        assert result.equals(expected)
+
+
+def test_dictionary_encode_simple():
+    cases = [
+        (pa.array([1, 2, 3, None, 1, 2, 3]),
+         pa.DictionaryArray.from_arrays(
+             pa.array([0, 1, 2, None, 0, 1, 2], type='int32'),
+             [1, 2, 3])),
+        (pa.array(['foo', None, 'bar', 'foo']),
+         pa.DictionaryArray.from_arrays(
+             pa.array([0, None, 1, 0], type='int32'),
+             ['foo', 'bar']))
+    ]
+    for arr, expected in cases:
+        result = arr.dictionary_encode()
+        assert result.equals(expected)
+
+
 def test_simple_type_construction():
     result = pa.lib.TimestampType()
     with pytest.raises(TypeError):
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 6ba4fd2fad8ea..1df80acc0d6cf 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1436,6 +1436,7 @@ def test_large_table_int32_overflow():
     _write_table(table, f)
 
 
+@parquet
 def test_index_column_name_duplicate(tmpdir):
     data = {
         'close': {
@@ -1460,6 +1461,7 @@ def test_index_column_name_duplicate(tmpdir):
     tm.assert_frame_equal(result_df, dfx)
 
 
+@parquet
 def test_backwards_compatible_index_naming():
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
@@ -1482,6 +1484,7 @@ def test_backwards_compatible_index_naming():
     tm.assert_frame_equal(result, expected)
 
 
+@parquet
 def test_backwards_compatible_index_multi_level_named():
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
@@ -1507,6 +1510,7 @@ def test_backwards_compatible_index_multi_level_named():
     tm.assert_frame_equal(result, expected)
 
 
+@parquet
 def test_backwards_compatible_index_multi_level_some_named():
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index edf0d8a305a02..abd678bac6e58 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -1108,6 +1108,7 @@ def union(children_fields, mode):
 
     return pyarrow_wrap_data_type(union_type)
 
+
 cdef dict _type_aliases = {
     'null': null,
     'i1': int8,

From eb7be48ef80fcecea11be29320b244df82f1cc06 Mon Sep 17 00:00:00 2001
From: Manuel <manuvaldes@gmail.com>
Date: Fri, 17 Nov 2017 21:14:16 -0500
Subject: [PATCH 1257/1644] ARROW-1805: [Python] Ignore special private files
 when traversing ParquetDataset

When exploring dataset, we should ignore _SUCCESS, _metadata and _common_metadata files to determine if the current directory is valid

Author: Manuel <manuvaldes@gmail.com>

Closes #1314 from manuvaldes/ARROW-1805 and squashes the following commits:

c11e2435 [Manuel] ignore files when exploring partitioned dataset
---
 python/pyarrow/parquet.py            |  2 +-
 python/pyarrow/tests/test_parquet.py | 11 +++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 9e0749bb35c9e..3023e1771e404 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -573,7 +573,7 @@ def _visit_level(self, level, base_path, part_keys):
         filtered_files.sort()
         filtered_directories.sort()
 
-        if len(files) > 0 and len(filtered_directories) > 0:
+        if len(filtered_files) > 0 and len(filtered_directories) > 0:
             raise ValueError('Found files in an intermediate '
                              'directory: {0}'.format(base_path))
         elif len(filtered_directories) > 0:
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 1df80acc0d6cf..522815fce16c8 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1027,8 +1027,11 @@ def _visit_level(base_dir, level, part_keys):
                 with fs.open(file_path, 'wb') as f:
                     _write_table(part_table, f)
                 assert fs.exists(file_path)
+
+                _touch(pjoin(level_dir, '_SUCCESS'))
             else:
                 _visit_level(level_dir, level + 1, this_part_keys)
+                _touch(pjoin(level_dir, '_SUCCESS'))
 
     _visit_level(base_dir, 0, [])
 
@@ -1101,6 +1104,11 @@ def _filter_partition(df, part_keys):
     return df[predicate].drop(to_drop, axis=1)
 
 
+def _touch(path):
+    with open(path, 'wb'):
+        pass
+
+
 @parquet
 def test_read_multiple_files(tmpdir):
     import pyarrow.parquet as pq
@@ -1128,8 +1136,7 @@ def test_read_multiple_files(tmpdir):
         paths.append(path)
 
     # Write a _SUCCESS.crc file
-    with open(pjoin(dirpath, '_SUCCESS.crc'), 'wb') as f:
-        f.write(b'0')
+    _touch(pjoin(dirpath, '_SUCCESS.crc'))
 
     def read_multiple_files(paths, columns=None, nthreads=None, **kwargs):
         dataset = pq.ParquetDataset(paths, **kwargs)

From 202e6503cd2e941a2df2ccacedf09611d8ad2e0f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 18 Nov 2017 00:41:01 -0500
Subject: [PATCH 1258/1644] ARROW-1791: Limit generated data range to physical
 limits for temporal types

cc @trxcllnt, do these time ranges seem reasonable?

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1328 from wesm/ARROW-1791 and squashes the following commits:

57a117aa [Wes McKinney] Add comment about nanosecond timestamp range
e20ad490 [Wes McKinney] Also restrict time and timestamp generated integer ranges
784e7a95 [Wes McKinney] Restrict generated date range in integration tests
---
 integration/integration_test.py | 41 +++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 205176eccc11a..46d010608118d 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -231,10 +231,19 @@ class DateType(IntegerType):
     DAY = 0
     MILLISECOND = 1
 
+    # 1/1/1 to 12/31/9999
+    _ranges = {
+        DAY: [-719162, 2932896],
+        MILLISECOND: [-62135596800000, 253402214400000]
+    }
+
     def __init__(self, name, unit, nullable=True):
         bit_width = 32 if unit == self.DAY else 64
+
+        min_value, max_value = self._ranges[unit]
         super(DateType, self).__init__(
-            name, True, bit_width, nullable=nullable
+            name, True, bit_width, nullable=nullable,
+            min_value=min_value, max_value=max_value
         )
         self.unit = unit
 
@@ -262,10 +271,19 @@ class TimeType(IntegerType):
         'ns': 64
     }
 
+    _ranges = {
+        's': [0, 86400],
+        'ms': [0, 86400000],
+        'us': [0, 86400000000],
+        'ns': [0, 86400000000000]
+    }
+
     def __init__(self, name, unit='s', nullable=True):
-        super(TimeType, self).__init__(
-            name, True, self.BIT_WIDTHS[unit], nullable=nullable
-        )
+        min_val, max_val = self._ranges[unit]
+        super(TimeType, self).__init__(name, True, self.BIT_WIDTHS[unit],
+                                       nullable=nullable,
+                                       min_value=min_val,
+                                       max_value=max_val)
         self.unit = unit
 
     def _get_type(self):
@@ -278,8 +296,21 @@ def _get_type(self):
 
 class TimestampType(IntegerType):
 
+    # 1/1/1 to 12/31/9999
+    _ranges = {
+        's': [-62135596800, 253402214400],
+        'ms': [-62135596800000, 253402214400000],
+        'us': [-62135596800000000, 253402214400000000],
+
+        # Physical range for int64, ~584 years and change
+        'ns': [np.iinfo('int64').min, np.iinfo('int64').max]
+    }
+
     def __init__(self, name, unit='s', tz=None, nullable=True):
-        super(TimestampType, self).__init__(name, True, 64, nullable=nullable)
+        min_val, max_val = self._ranges[unit]
+        super(TimestampType, self).__init__(name, True, 64, nullable=nullable,
+                                            min_value=min_val,
+                                            max_value=max_val)
         self.unit = unit
         self.tz = tz
 

From 952ec05b19fdaff8548746b6b43618ab044cd298 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Sat, 18 Nov 2017 00:42:12 -0500
Subject: [PATCH 1259/1644] ARROW-1773: [C++] Add casts from date/time types to
 compatible signed integers

This closes [ARROW-1773](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1773).

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1310 from Licht-T/feature-date-time-to-int and squashes the following commits:

1e6061a4 [Wes McKinney] Add zero-copy cast from timestamp to int64
039609a3 [Wes McKinney] Add C++ unit test, clean up Python tests
69a53991 [Licht-T] ENH: Add test for casting from/to Date or Time to/from int
9a64e61d [Licht-T] ENH: Add cast methods from Date/Time to int
---
 cpp/src/arrow/compute/compute-test.cc | 23 +++++++++++-
 cpp/src/arrow/compute/kernels/cast.cc | 27 +++++++++-----
 python/pyarrow/tests/test_array.py    | 53 +++++++++++++++++++++++++++
 3 files changed, 92 insertions(+), 11 deletions(-)

diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 5eada398d52fc..58a991c60f24b 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -334,6 +334,9 @@ TEST_F(TestCast, TimestampToTimestamp) {
                                           &arr);
   CheckZeroCopy(*arr, timestamp(TimeUnit::SECOND));
 
+  // ARROW-1773, cast to integer
+  CheckZeroCopy(*arr, int64());
+
   // Divide, truncate
   vector<int64_t> v8 = {0, 100123, 200456, 1123, 2456};
   vector<int64_t> e8 = {0, 100, 200, 1, 2};
@@ -432,7 +435,7 @@ TEST_F(TestCast, TimestampToDate32_Date64) {
       timestamp(TimeUnit::SECOND), v_second_nofail, is_valid, date32(), v_day, options);
 }
 
-TEST_F(TestCast, TimeToTime) {
+TEST_F(TestCast, TimeToCompatible) {
   CastOptions options;
 
   vector<bool> is_valid = {true, false, true, true, true};
@@ -474,6 +477,16 @@ TEST_F(TestCast, TimeToTime) {
   ArrayFromVector<Time64Type, int64_t>(time64(TimeUnit::MICRO), is_valid, v7, &arr);
   CheckZeroCopy(*arr, time64(TimeUnit::MICRO));
 
+  // ARROW-1773: cast to int64
+  CheckZeroCopy(*arr, int64());
+
+  vector<int32_t> v7_2 = {0, 70000, 2000, 1000, 0};
+  ArrayFromVector<Time32Type, int32_t>(time32(TimeUnit::SECOND), is_valid, v7_2, &arr);
+  CheckZeroCopy(*arr, time32(TimeUnit::SECOND));
+
+  // ARROW-1773: cast to int64
+  CheckZeroCopy(*arr, int32());
+
   // Divide, truncate
   vector<int32_t> v8 = {0, 100123, 200456, 1123, 2456};
   vector<int32_t> e8 = {0, 100, 200, 1, 2};
@@ -515,7 +528,7 @@ TEST_F(TestCast, TimeToTime) {
                          options);
 }
 
-TEST_F(TestCast, DateToDate) {
+TEST_F(TestCast, DateToCompatible) {
   CastOptions options;
 
   vector<bool> is_valid = {true, false, true, true, true};
@@ -535,9 +548,15 @@ TEST_F(TestCast, DateToDate) {
   ArrayFromVector<Date32Type, int32_t>(date32(), is_valid, v2, &arr);
   CheckZeroCopy(*arr, date32());
 
+  // ARROW-1773: zero copy cast to integer
+  CheckZeroCopy(*arr, int32());
+
   ArrayFromVector<Date64Type, int64_t>(date64(), is_valid, v3, &arr);
   CheckZeroCopy(*arr, date64());
 
+  // ARROW-1773: zero copy cast to integer
+  CheckZeroCopy(*arr, int64());
+
   // Divide, truncate
   vector<int64_t> v8 = {0, 100 * F + 123, 200 * F + 456, F + 123, 2 * F + 456};
   vector<int32_t> e8 = {0, 100, 200, 1, 2};
diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc
index 6a42ec8b2b807..c866054eadd8a 100644
--- a/cpp/src/arrow/compute/kernels/cast.cc
+++ b/cpp/src/arrow/compute/kernels/cast.cc
@@ -91,10 +91,14 @@ struct is_zero_copy_cast<
 // From integers to date/time types with zero copy
 template <typename O, typename I>
 struct is_zero_copy_cast<
-    O, I, typename std::enable_if<std::is_base_of<Integer, I>::value &&
-                                  (std::is_base_of<TimeType, O>::value ||
-                                   std::is_base_of<DateType, O>::value ||
-                                   std::is_base_of<TimestampType, O>::value)>::type> {
+    O, I,
+    typename std::enable_if<
+        (std::is_base_of<Integer, I>::value &&
+         (std::is_base_of<TimeType, O>::value || std::is_base_of<DateType, O>::value ||
+          std::is_base_of<TimestampType, O>::value)) ||
+        (std::is_base_of<Integer, O>::value &&
+         (std::is_base_of<TimeType, I>::value || std::is_base_of<DateType, I>::value ||
+          std::is_base_of<TimestampType, I>::value))>::type> {
   using O_T = typename O::c_type;
   using I_T = typename I::c_type;
 
@@ -809,24 +813,29 @@ class CastKernel : public UnaryKernel {
 
 #define DATE32_CASES(FN, IN_TYPE) \
   FN(Date32Type, Date32Type);     \
-  FN(Date32Type, Date64Type);
+  FN(Date32Type, Date64Type);     \
+  FN(Date32Type, Int32Type);
 
 #define DATE64_CASES(FN, IN_TYPE) \
   FN(Date64Type, Date64Type);     \
-  FN(Date64Type, Date32Type);
+  FN(Date64Type, Date32Type);     \
+  FN(Date64Type, Int64Type);
 
 #define TIME32_CASES(FN, IN_TYPE) \
   FN(Time32Type, Time32Type);     \
-  FN(Time32Type, Time64Type);
+  FN(Time32Type, Time64Type);     \
+  FN(Time32Type, Int32Type);
 
 #define TIME64_CASES(FN, IN_TYPE) \
   FN(Time64Type, Time32Type);     \
-  FN(Time64Type, Time64Type);
+  FN(Time64Type, Time64Type);     \
+  FN(Time64Type, Int64Type);
 
 #define TIMESTAMP_CASES(FN, IN_TYPE) \
   FN(TimestampType, TimestampType);  \
   FN(TimestampType, Date32Type);     \
-  FN(TimestampType, Date64Type);
+  FN(TimestampType, Date64Type);     \
+  FN(TimestampType, Int64Type);
 
 #define DICTIONARY_CASES(FN, IN_TYPE) \
   FN(IN_TYPE, NullType);              \
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index c061e6820753e..b7b0b18333677 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -389,6 +389,59 @@ def test_dictionary_encode_simple():
         assert result.equals(expected)
 
 
+def test_cast_time32_to_int():
+    arr = pa.array(np.array([0, 1, 2], dtype='int32'),
+                   type=pa.time32('s'))
+    expected = pa.array([0, 1, 2], type='i4')
+
+    result = arr.cast('i4')
+    assert result.equals(expected)
+
+
+def test_cast_time64_to_int():
+    arr = pa.array(np.array([0, 1, 2], dtype='int64'),
+                   type=pa.time64('us'))
+    expected = pa.array([0, 1, 2], type='i8')
+
+    result = arr.cast('i8')
+    assert result.equals(expected)
+
+
+def test_cast_timestamp_to_int():
+    arr = pa.array(np.array([0, 1, 2], dtype='int64'),
+                   type=pa.timestamp('us'))
+    expected = pa.array([0, 1, 2], type='i8')
+
+    result = arr.cast('i8')
+    assert result.equals(expected)
+
+
+def test_cast_date32_to_int():
+    arr = pa.array([0, 1, 2], type='i4')
+
+    result1 = arr.cast('date32')
+    result2 = result1.cast('i4')
+
+    expected1 = pa.array([
+        datetime.date(1970, 1, 1),
+        datetime.date(1970, 1, 2),
+        datetime.date(1970, 1, 3)
+    ]).cast('date32')
+
+    assert result1.equals(expected1)
+    assert result2.equals(arr)
+
+
+def test_cast_date64_to_int():
+    arr = pa.array(np.array([0, 1, 2], dtype='int64'),
+                   type=pa.date64())
+    expected = pa.array([0, 1, 2], type='i8')
+
+    result = arr.cast('i8')
+
+    assert result.equals(expected)
+
+
 def test_simple_type_construction():
     result = pa.lib.TimestampType()
     with pytest.raises(TypeError):

From 37214ef35cf5ecaf195c0796df50371e9c574928 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Sat, 18 Nov 2017 16:22:08 -0500
Subject: [PATCH 1260/1644] ARROW-1827: [Java] Add checkstyle file and license
 template

Author: Li Jin <ice.xelloss@gmail.com>

Closes #1326 from icexelloss/ARROW-1827 and squashes the following commits:

a6ea0321 [Li Jin] Make license consistent
28348721 [Li Jin] Minor edit
ab705a14 [Li Jin] Add newline
291dbf93 [Li Jin] Add checkstyle config and header
---
 java/.gitattributes                    |   3 +
 java/dev/checkstyle/checkstyle.license |  17 ++
 java/dev/checkstyle/checkstyle.xml     | 254 +++++++++++++++++++++++++
 java/dev/checkstyle/suppressions.xml   |  31 +++
 java/pom.xml                           |   4 +-
 5 files changed, 308 insertions(+), 1 deletion(-)
 create mode 100644 java/.gitattributes
 create mode 100644 java/dev/checkstyle/checkstyle.license
 create mode 100644 java/dev/checkstyle/checkstyle.xml
 create mode 100644 java/dev/checkstyle/suppressions.xml

diff --git a/java/.gitattributes b/java/.gitattributes
new file mode 100644
index 0000000000000..cb02d822677ed
--- /dev/null
+++ b/java/.gitattributes
@@ -0,0 +1,3 @@
+.gitattributes export-ignore
+.gitignore export-ignore
+/dev export-ignore
diff --git a/java/dev/checkstyle/checkstyle.license b/java/dev/checkstyle/checkstyle.license
new file mode 100644
index 0000000000000..c06c90cd287af
--- /dev/null
+++ b/java/dev/checkstyle/checkstyle.license
@@ -0,0 +1,17 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
diff --git a/java/dev/checkstyle/checkstyle.xml b/java/dev/checkstyle/checkstyle.xml
new file mode 100644
index 0000000000000..14dbede169f19
--- /dev/null
+++ b/java/dev/checkstyle/checkstyle.xml
@@ -0,0 +1,254 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE module PUBLIC
+          "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+          "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+
+<!--
+    Checkstyle configuration that checks the Google coding conventions from Google Java Style
+    that can be found at https://google.github.io/styleguide/javaguide.html.
+
+    Checkstyle is very configurable. Be sure to read the documentation at
+    http://checkstyle.sf.net (or in your downloaded distribution).
+
+    To completely disable a check, just comment it out or delete it from the file.
+
+    Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
+ -->
+
+<module name = "Checker">
+    <property name="charset" value="UTF-8"/>
+
+    <property name="severity" value="warning"/>
+
+    <property name="fileExtensions" value="java, properties, xml"/>
+
+    <!-- Arrow specific modules -->
+
+    <module name="Header">
+      <property name="headerFile" value="${checkstyle.header.file}"/>
+    </module>
+
+    <module name="SuppressionFilter">
+      <property name="file" value="${checkstyle.suppressions.file}"/>
+    </module>
+
+    <module name="NewlineAtEndOfFile"/>
+
+    <!--
+    If you wish to turn off checking for a section of code, you can put a comment in the source
+    before and after the section, with the following syntax:
+    // checkstyle:off no.XXX (such as checkstyle.off: NoFinalizer)
+    ...  // stuff that breaks the styles
+    // checkstyle:on
+    -->
+    <module name="SuppressionCommentFilter">
+      <property name="offCommentFormat" value="checkstyle.off\: ([\w\|]+)"/>
+      <property name="onCommentFormat" value="checkstyle.on\: ([\w\|]+)"/>
+      <property name="checkFormat" value="$1"/>
+    </module>
+
+    <!-- Google style modules -->
+
+    <!-- Checks for whitespace                               -->
+    <!-- See http://checkstyle.sf.net/config_whitespace.html -->
+    <module name="FileTabCharacter">
+      <property name="eachLine" value="true"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="OuterTypeFilename"/>
+        <module name="IllegalTokenText">
+            <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
+            <property name="format" value="\\u00(08|09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
+            <property name="message" value="Avoid using corresponding octal or Unicode escape."/>
+        </module>
+        <module name="AvoidEscapedUnicodeCharacters">
+            <property name="allowEscapesForControlCharacters" value="true"/>
+            <property name="allowByTailComment" value="true"/>
+            <property name="allowNonPrintableEscapes" value="true"/>
+        </module>
+        <module name="LineLength">
+            <property name="max" value="100"/>
+            <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
+        </module>
+        <module name="AvoidStarImport"/>
+        <module name="OneTopLevelClass"/>
+        <module name="NoLineWrap"/>
+        <module name="EmptyBlock">
+            <property name="option" value="TEXT"/>
+            <property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
+        </module>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="maxLineLength" value="100"/>
+        </module>
+        <module name="RightCurly"/>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+            <property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO, STATIC_INIT, INSTANCE_INIT"/>
+        </module>
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="allowEmptyTypes" value="true"/>
+            <property name="allowEmptyLoops" value="true"/>
+            <message key="ws.notFollowed"
+             value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
+             <message key="ws.notPreceded"
+             value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
+        </module>
+        <module name="OneStatementPerLine"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MissingSwitchDefault"/>
+        <module name="FallThrough"/>
+        <module name="UpperEll"/>
+        <module name="ModifierOrder"/>
+        <module name="EmptyLineSeparator">
+            <property name="allowNoEmptyLineBetweenFields" value="true"/>
+        </module>
+        <module name="SeparatorWrap">
+            <property name="tokens" value="DOT"/>
+            <property name="option" value="nl"/>
+        </module>
+        <module name="SeparatorWrap">
+            <property name="tokens" value="COMMA"/>
+            <property name="option" value="EOL"/>
+        </module>
+        <module name="PackageName">
+            <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
+            <message key="name.invalidPattern"
+             value="Package name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="TypeName">
+            <message key="name.invalidPattern"
+             value="Type name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="MemberName">
+            <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+            <message key="name.invalidPattern"
+             value="Member name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="ParameterName">
+            <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+            <message key="name.invalidPattern"
+             value="Parameter name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="CatchParameterName">
+            <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+            <message key="name.invalidPattern"
+             value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="LocalVariableName">
+            <property name="tokens" value="VARIABLE_DEF"/>
+            <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+            <property name="allowOneCharVarInForLoop" value="true"/>
+            <message key="name.invalidPattern"
+             value="Local variable name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+            <message key="name.invalidPattern"
+             value="Class type name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+            <message key="name.invalidPattern"
+             value="Method type name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="InterfaceTypeParameterName">
+            <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+            <message key="name.invalidPattern"
+             value="Interface type name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="NoFinalizer"/>
+        <module name="GenericWhitespace">
+            <message key="ws.followed"
+             value="GenericWhitespace ''{0}'' is followed by whitespace."/>
+             <message key="ws.preceded"
+             value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
+             <message key="ws.illegalFollow"
+             value="GenericWhitespace ''{0}'' should followed by whitespace."/>
+             <message key="ws.notPreceded"
+             value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
+        </module>
+        <module name="Indentation">
+            <property name="basicOffset" value="2"/>
+            <property name="braceAdjustment" value="0"/>
+            <property name="caseIndent" value="2"/>
+            <property name="throwsIndent" value="4"/>
+            <property name="lineWrappingIndentation" value="4"/>
+            <property name="arrayInitIndent" value="2"/>
+        </module>
+        <module name="AbbreviationAsWordInName">
+            <property name="ignoreFinal" value="false"/>
+            <property name="allowedAbbreviationLength" value="1"/>
+        </module>
+        <module name="OverloadMethodsDeclarationOrder"/>
+        <module name="VariableDeclarationUsageDistance"/>
+        <module name="CustomImportOrder">
+            <property name="specialImportsRegExp" value="com.google"/>
+            <property name="sortImportsInGroupAlphabetically" value="true"/>
+            <property name="customImportOrderRules" value="STATIC###SPECIAL_IMPORTS###THIRD_PARTY_PACKAGE###STANDARD_JAVA_PACKAGE"/>
+        </module>
+        <module name="MethodParamPad"/>
+        <module name="OperatorWrap">
+            <property name="option" value="NL"/>
+            <property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR "/>
+        </module>
+        <module name="AnnotationLocation">
+            <property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
+        </module>
+        <module name="AnnotationLocation">
+            <property name="tokens" value="VARIABLE_DEF"/>
+            <property name="allowSamelineMultipleAnnotations" value="true"/>
+        </module>
+        <module name="NonEmptyAtclauseDescription"/>
+        <module name="JavadocTagContinuationIndentation"/>
+        <module name="SummaryJavadoc">
+            <property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
+        </module>
+        <module name="JavadocParagraph"/>
+        <module name="AtclauseOrder">
+            <property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
+            <property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
+        </module>
+        <module name="JavadocMethod">
+            <property name="scope" value="public"/>
+            <property name="allowMissingParamTags" value="true"/>
+            <property name="allowMissingThrowsTags" value="true"/>
+            <property name="allowMissingReturnTag" value="true"/>
+            <property name="minLineCount" value="2"/>
+            <property name="allowedAnnotations" value="Override, Test"/>
+            <property name="allowThrowsTagsForSubclasses" value="true"/>
+        </module>
+        <module name="MethodName">
+            <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
+            <message key="name.invalidPattern"
+             value="Method name ''{0}'' must match pattern ''{1}''."/>
+        </module>
+        <module name="SingleLineJavadoc">
+            <property name="ignoreInlineTags" value="false"/>
+        </module>
+        <module name="EmptyCatchBlock">
+            <property name="exceptionVariableName" value="expected"/>
+        </module>
+        <module name="CommentsIndentation"/>
+    </module>
+</module>
diff --git a/java/dev/checkstyle/suppressions.xml b/java/dev/checkstyle/suppressions.xml
new file mode 100644
index 0000000000000..36697256d69d9
--- /dev/null
+++ b/java/dev/checkstyle/suppressions.xml
@@ -0,0 +1,31 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE suppressions PUBLIC
+"-//Puppy Crawl//DTD Suppressions 1.1//EN"
+"http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
+
+<suppressions>
+  <suppress checks="JavadocPackage" files=".*[\\/]src[\\/]test[\\/].*"/>
+  <suppress checks="JavadocPackage" files=".*[\\/]maven-archetypes[\\/].*"/>
+  <suppress checks="JavadocPackage" files=".*[\\/]examples[\\/].*"/>
+
+  <!-- suppress all checks in the generated directories -->
+  <suppress checks=".*" files=".+[\\/]generated[\\/].+\.java" />
+  <suppress checks=".*" files=".+[\\/]generated-sources[\\/].+\.java" />
+  <suppress checks=".*" files=".+[\\/]generated-test-sources[\\/].+\.java" />
+</suppressions>
diff --git a/java/pom.xml b/java/pom.xml
index 0a0f2e0ce8f65..c479d651f6b5f 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -304,7 +304,9 @@
           </execution>
         </executions>
         <configuration>
-          <configLocation>google_checks.xml</configLocation>
+          <configLocation>dev/checkstyle/checkstyle.xml</configLocation>
+          <headerLocation>dev/checkstyle/checkstyle.license</headerLocation>
+          <suppressionsLocation>dev/checkstyle/suppressions.xml</suppressionsLocation>
           <encoding>UTF-8</encoding>
           <consoleOutput>true</consoleOutput>
           <failsOnError>${checkstyle.failOnViolation}</failsOnError>

From 9f9dc5b7025744458e50713fa460803bf52f5d33 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 18 Nov 2017 16:25:56 -0500
Subject: [PATCH 1261/1644] ARROW-1575: [Python] Add tests for pyarrow.column
 factory function

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1329 from wesm/ARROW-1575 and squashes the following commits:

d9f14a12 [Wes McKinney] Add tests for pyarrow.column factory function
---
 python/pyarrow/table.pxi           |  9 ++++
 python/pyarrow/tests/test_table.py | 79 +++++++++++++++++-------------
 2 files changed, 55 insertions(+), 33 deletions(-)

diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 1a9d23db4ebdc..591f3297587a3 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -166,6 +166,15 @@ def chunked_array(arrays, type=None):
 def column(object field_or_name, arr):
     """
     Create Column object from field/string and array-like data
+
+    Parameters
+    ----------
+    field_or_name : string or Field
+    arr : Array, list of Arrays, or ChunkedArray
+
+    Returns
+    -------
+    column : Column
     """
     cdef:
         Field boxed_field
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 4282224660a1a..cd05fb8e1fce5 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -21,42 +21,55 @@
 import pandas as pd
 import pytest
 
-from pyarrow.compat import unittest
 import pyarrow as pa
 
 
-class TestColumn(unittest.TestCase):
-
-    def test_basics(self):
-        data = [
-            pa.array([-10, -5, 0, 5, 10])
-        ]
-        table = pa.Table.from_arrays(data, names=['a'])
-        column = table.column(0)
-        assert column.name == 'a'
-        assert column.length() == 5
-        assert len(column) == 5
-        assert column.shape == (5,)
-        assert column.to_pylist() == [-10, -5, 0, 5, 10]
-
-    def test_from_array(self):
-        arr = pa.array([0, 1, 2, 3, 4])
-
-        col1 = pa.Column.from_array('foo', arr)
-        col2 = pa.Column.from_array(pa.field('foo', arr.type), arr)
-
-        assert col1.equals(col2)
-
-    def test_pandas(self):
-        data = [
-            pa.array([-10, -5, 0, 5, 10])
-        ]
-        table = pa.Table.from_arrays(data, names=['a'])
-        column = table.column(0)
-        series = column.to_pandas()
-        assert series.name == 'a'
-        assert series.shape == (5,)
-        assert series.iloc[0] == -10
+def test_column_basics():
+    data = [
+        pa.array([-10, -5, 0, 5, 10])
+    ]
+    table = pa.Table.from_arrays(data, names=['a'])
+    column = table.column(0)
+    assert column.name == 'a'
+    assert column.length() == 5
+    assert len(column) == 5
+    assert column.shape == (5,)
+    assert column.to_pylist() == [-10, -5, 0, 5, 10]
+
+
+def test_column_factory_function():
+    # ARROW-1575
+    arr = pa.array([0, 1, 2, 3, 4])
+    arr2 = pa.array([5, 6, 7, 8])
+
+    col1 = pa.Column.from_array('foo', arr)
+    col2 = pa.Column.from_array(pa.field('foo', arr.type), arr)
+
+    assert col1.equals(col2)
+
+    col3 = pa.column('foo', [arr, arr2])
+    chunked_arr = pa.chunked_array([arr, arr2])
+    col4 = pa.column('foo', chunked_arr)
+    assert col3.equals(col4)
+
+    col5 = pa.column('foo', arr.to_pandas())
+    assert col5.equals(pa.column('foo', arr))
+
+    # Type mismatch
+    with pytest.raises(ValueError):
+        pa.Column.from_array(pa.field('foo', pa.string()), arr)
+
+
+def test_column_to_pandas():
+    data = [
+        pa.array([-10, -5, 0, 5, 10])
+    ]
+    table = pa.Table.from_arrays(data, names=['a'])
+    column = table.column(0)
+    series = column.to_pandas()
+    assert series.name == 'a'
+    assert series.shape == (5,)
+    assert series.iloc[0] == -10
 
 
 def test_recordbatch_basics():

From d92735e3956f61ba647f88409e1bf39b8b07ba72 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sun, 19 Nov 2017 20:02:17 -0500
Subject: [PATCH 1262/1644] =?UTF-8?q?ARROW-1834:=20[Doc]=C2=A0Build=20docu?=
 =?UTF-8?q?mentation=20in=20separate=20build=20folders?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1332 from xhochy/ARROW-1834 and squashes the following commits:

e9c02533 [Korn, Uwe] ARROW-1834: [Doc] Build documentation in separate build folders
---
 dev/gen_apidocs/Dockerfile          |  2 +-
 dev/gen_apidocs/create_documents.sh | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/dev/gen_apidocs/Dockerfile b/dev/gen_apidocs/Dockerfile
index eaeb548184d61..ca4718e637842 100644
--- a/dev/gen_apidocs/Dockerfile
+++ b/dev/gen_apidocs/Dockerfile
@@ -26,7 +26,7 @@ RUN apt-add-repository -y ppa:ubuntu-toolchain-r/test && \
         git \
         gcc-4.9 \
         g++-4.9 \
-        build-essential 
+        build-essential
 # This will install conda in /home/ubuntu/miniconda
 RUN wget -O /tmp/miniconda.sh \
     https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh
index 762b85222bd4d..566d9cee79c85 100755
--- a/dev/gen_apidocs/create_documents.sh
+++ b/dev/gen_apidocs/create_documents.sh
@@ -45,9 +45,9 @@ source activate pyarrow-dev
 export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
 export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX
 
-rm -rf arrow/cpp/build
-mkdir arrow/cpp/build
-pushd arrow/cpp/build
+rm -rf arrow/cpp/build_docs
+mkdir arrow/cpp/build_docs
+pushd arrow/cpp/build_docs
 cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
       -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
       -DARROW_PYTHON=on \
@@ -59,9 +59,9 @@ make install
 popd
 
 # Build Parquet C++
-rm -rf parquet-cpp/build
-mkdir parquet-cpp/build
-pushd parquet-cpp/build
+rm -rf parquet-cpp/build_docs
+mkdir parquet-cpp/build_docs
+pushd parquet-cpp/build_docs
 cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
       -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
       -DPARQUET_BUILD_BENCHMARKS=off \

From b3a3a743866e2cbeff6fa20c77fa378db13232f6 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 20 Nov 2017 09:22:54 -0500
Subject: [PATCH 1263/1644] ARROW-1693: [JS] Expand JavaScript implementation,
 build system, fix integration tests

This PR adds a workaround for reading the metadata layout for C++ dictionary-encoded vectors.

I added tests that validate against the C++/Java integration suite. In order to make the new tests pass, I had to update the generated flatbuffers format and add a few types the JS version didn't have yet (Bool, Date32, and Timestamp). It also uses the new `isDelta` flag on DictionaryBatches to determine whether the DictionaryBatch vector should replace or append to the existing dictionary.

I also added a script for generating test arrow files from the C++ and Java implementations, so we don't break the tests updating the format in the future. I saved the generated Arrow files in with the tests because I didn't see a way to pipe the JSON test data through the C++/Java json-to-arrow commands without writing to a file. If I missed something and we can do it all in-memory, I'd be happy to make that change!

This PR is marked WIP because I added an [integration test](https://github.com/apache/arrow/commit/6e98874d9f4bfae7758f8f731212ae7ceb3f1321#diff-18c6be12406c482092d4b1f7bd70a8e1R22) that validates the JS reader reads C++ and Java files the same way, but unfortunately it doesn't. Debugging, I noticed a number of other differences between the buffer layout metadata between the C++ and Java versions. If we go ahead with @jacques-n [comment in ARROW-1693](https://issues.apache.org/jira/browse/ARROW-1693?focusedCommentId=16244812&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16244812) and remove/ignore the metadata, this test should pass too.

cc @TheNeuralBit

Author: Paul Taylor <paul.e.taylor@me.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1294 from trxcllnt/generate-js-test-files and squashes the following commits:

f907d5a7 [Paul Taylor] fix aggressive closure-compiler mangling in the ES5 UMD bundle
57c7df45 [Paul Taylor] remove arrow files from perf tests
5972349c [Paul Taylor] update performance tests to use generated test data
14be77f4 [Paul Taylor] fix Date64Vector TypedArray, enable datetime integration tests
5660eb34 [Wes McKinney] Use openjdk8 for integration tests, jdk7 for main Java CI job
019e8e24 [Paul Taylor] update closure compiler with full support for ESModules, and remove closure-compiler-scripts
48111290 [Paul Taylor] Add support for reading Arrow buffers < MetadataVersion 4
c72134a5 [Paul Taylor] compile JS source in integration tests
c83a700d [Wes McKinney] Hack until ARROW-1837 resolved. Constrain unsigned integers max to signed max for bit width
fd3ed475 [Wes McKinney] Uppercase hex values
224e041c [Wes McKinney] Remove hard-coded file name to prevent primitive JSON file from being clobbered
0882d8e9 [Paul Taylor] separate JS unit tests from integration tests in CI
1f6a81b4 [Paul Taylor] add missing mkdirp for test json data
19136fbf [Paul Taylor] remove test data files in favor of auto-generating them in CI
9f195682 [Paul Taylor] Generate test files when the test run if they don't exist
0cdb74e0 [Paul Taylor] Add a cli arg to integration_test.py generate test JSON files for JS
cc744564 [Paul Taylor] resolve LICENSE.txt conflict
33916230 [Paul Taylor] move js license to top-level license.txt
d0b61f49 [Paul Taylor] add validate package script back in, make npm-release.sh suitable for ASF release process
7e3be574 [Paul Taylor] Copy license.txt and notice.txt into target dirs from arrow root.
c8125d2d [Paul Taylor] Update readme to reflect new Table.from signature
49ac3398 [Paul Taylor] allow unrecognized cli args in gulpfile
3c52587e [Paul Taylor] re-enable node_js job in travis
cb142f11 [Paul Taylor] add npm release script, remove unused package scripts
d51793dd [Paul Taylor] run tests on src folder for accurate jest coverage statistics
c087f482 [Paul Taylor] generate test data in build scripts
1d814d00 [Paul Taylor] excise test data csvs
14d48964 [Paul Taylor] stringify Struct Array cells
1f004968 [Paul Taylor] rename FixedWidthListVector to FixedWidthNumericVector
be73c918 [Paul Taylor] add BinaryVector, change ListVector to always return an Array
02fb3006 [Paul Taylor] compare iterator results in integration tests
e67a66a1 [Paul Taylor] remove/ignore test snapshots (getting too big)
de7d96a3 [Paul Taylor] regenerate test arrows from master
a6d3c83e [Paul Taylor] enable integration tests
44889fbe [Paul Taylor] report errors generating test arrows
fd68d510 [Paul Taylor] always increment validity buffer index while reading
562eba7d [Paul Taylor] update test snapshots
d4399a8a [Paul Taylor] update integration tests, add custom jest vector matcher
8d44dcd7 [Paul Taylor] update tests
6d2c03d4 [Paul Taylor] clean arrows folders before regenerating test data
4166a9ff [Paul Taylor] hard-code reader to Arrow spec and ignore field layout metadata
c60305d6 [Paul Taylor] refactor: flatten vector folder, add more types
ba984c61 [Paul Taylor] update dependencies
5eee3eaa [Paul Taylor] add integration tests to compare how JS reads cpp vs. java arrows
d4ff57aa [Paul Taylor] update test snapshots
407b9f5b [Paul Taylor] update reader/table tests for new generated arrows
85497069 [Paul Taylor] update cli args to execute partial test runs for debugging
eefc256d [Paul Taylor] remove old test arrows, add new generated test arrows
0cd31ab9 [Paul Taylor] add generate-arrows script to tests
3ff71384 [Paul Taylor] Add bool, date, time, timestamp, and ARROW-1693 workaround in reader
4a34247c [Paul Taylor] export Row type
141194e7 [Paul Taylor] use fieldNode.length as vector length
c45718e7 [Paul Taylor] support new DictionaryBatch isDelta flag
9d8fef97 [Paul Taylor] split DateVector into Date32 and Date64 types
8592ff3c [Paul Taylor] update generated format flatbuffers
---
 .travis.yml                                   |   21 +-
 LICENSE.txt                                   |   95 +
 ci/travis_script_integration.sh               |   19 +
 ci/travis_script_js.sh                        |   15 +-
 integration/integration_test.py               |   56 +-
 js/.gitignore                                 |    5 +
 js/DEVELOP.md                                 |  251 ++-
 js/LICENSE                                    |   39 -
 js/README.md                                  |   12 +-
 js/closure-compiler-scripts/flatbuffers.js    | 1204 -----------
 js/closure-compiler-scripts/text-encoding.js  |  648 ------
 js/closure-compiler-scripts/tslib.js          |  151 --
 js/gulp/argv.js                               |    7 +-
 js/gulp/arrow-task.js                         |    5 +-
 js/gulp/closure-task.js                       |   16 +-
 js/gulp/test-task.js                          |  130 +-
 js/gulp/typescript-task.js                    |   26 +-
 js/gulp/uglify-task.js                        |    2 +-
 js/gulp/util.js                               |    2 +-
 js/gulpfile.js                                |   10 +-
 js/lerna.json                                 |    4 +-
 js/{prepublish.sh => npm-release.sh}          |    8 +-
 js/package.json                               |   38 +-
 js/perf/arrows/file/dictionary.arrow          |  Bin 2522 -> 0 bytes
 js/perf/arrows/file/simple.arrow              |  Bin 1642 -> 0 bytes
 js/perf/arrows/file/struct.arrow              |  Bin 2354 -> 0 bytes
 js/perf/arrows/multi/count/records.arrow      |  Bin 224 -> 0 bytes
 js/perf/arrows/multi/count/schema.arrow       |  Bin 184 -> 0 bytes
 js/perf/arrows/multi/latlong/records.arrow    |  Bin 352 -> 0 bytes
 js/perf/arrows/multi/latlong/schema.arrow     |  Bin 264 -> 0 bytes
 js/perf/arrows/multi/origins/records.arrow    |  Bin 224 -> 0 bytes
 js/perf/arrows/multi/origins/schema.arrow     |  Bin 1604 -> 0 bytes
 js/perf/arrows/stream/dictionary.arrow        |  Bin 1776 -> 0 bytes
 js/perf/arrows/stream/simple.arrow            |  Bin 1188 -> 0 bytes
 js/perf/arrows/stream/struct.arrow            |  Bin 1884 -> 0 bytes
 js/perf/config.js                             |   24 +-
 js/perf/index.js                              |   24 +-
 js/src/Arrow.ts                               |   98 +-
 js/src/format/{File_generated.ts => File.ts}  |    2 +-
 .../format}/File_generated.js                 |   12 +-
 .../{Message_generated.ts => Message.ts}      |   33 +-
 .../format}/Message_generated.js              |   43 +-
 .../format/{Schema_generated.ts => Schema.ts} |   19 +-
 .../format}/Schema_generated.js               |   46 +-
 js/src/reader/arrow.ts                        |  197 +-
 js/src/reader/dictionary.ts                   |   36 -
 js/src/reader/file.ts                         |   82 -
 js/src/reader/format.ts                       |  112 +
 js/src/reader/message.ts                      |   63 -
 js/src/reader/stream.ts                       |   43 -
 js/src/reader/vector.ts                       |  423 ++--
 js/src/types/arrow.ts                         |   88 -
 js/src/types/fixedsizelist.ts                 |   35 -
 js/src/types/list.ts                          |   35 -
 js/src/types/table/from.ts                    |   34 -
 js/src/types/table/row.ts                     |   61 -
 js/src/types/table/struct.ts                  |   63 -
 js/src/types/table/table.ts                   |   30 -
 js/src/types/table/toString.ts                |   40 -
 js/src/types/vector/bool.ts                   |   55 -
 js/src/types/vector/date.ts                   |   29 -
 js/src/types/vector/long.ts                   |   35 -
 js/src/types/vector/typed.ts                  |   57 -
 js/src/types/vector/virtual.ts                |  129 --
 js/src/vector/arrow.ts                        |  245 +++
 js/src/{types => vector}/dictionary.ts        |   28 +-
 js/src/vector/list.ts                         |   74 +
 js/src/vector/numeric.ts                      |  168 ++
 js/src/vector/struct.ts                       |  127 ++
 js/src/vector/table.ts                        |   59 +
 js/src/{types => }/vector/traits.ts           |   12 +-
 js/src/vector/types.ts                        |   43 +
 js/src/{types => vector}/utf8.ts              |    4 +-
 js/src/{types/types.ts => vector/vector.ts}   |   56 +-
 js/src/vector/virtual.ts                      |  129 ++
 js/test/Arrow.ts                              |   57 +-
 js/test/__snapshots__/reader-tests.ts.snap    |  497 -----
 js/test/__snapshots__/table-tests.ts.snap     | 1815 -----------------
 js/test/arrows/file/dictionary.arrow          |  Bin 2522 -> 0 bytes
 js/test/arrows/file/dictionary2.arrow         |  Bin 2762 -> 0 bytes
 js/test/arrows/file/multi_dictionary.arrow    |  Bin 3482 -> 0 bytes
 js/test/arrows/file/simple.arrow              |  Bin 1642 -> 0 bytes
 js/test/arrows/file/struct.arrow              |  Bin 2354 -> 0 bytes
 js/test/arrows/multi/count/records.arrow      |  Bin 224 -> 0 bytes
 js/test/arrows/multi/count/schema.arrow       |  Bin 184 -> 0 bytes
 js/test/arrows/multi/latlong/records.arrow    |  Bin 352 -> 0 bytes
 js/test/arrows/multi/latlong/schema.arrow     |  Bin 264 -> 0 bytes
 js/test/arrows/multi/origins/records.arrow    |  Bin 224 -> 0 bytes
 js/test/arrows/multi/origins/schema.arrow     |  Bin 1604 -> 0 bytes
 js/test/arrows/stream/dictionary.arrow        |  Bin 1776 -> 0 bytes
 js/test/arrows/stream/simple.arrow            |  Bin 1188 -> 0 bytes
 js/test/arrows/stream/struct.arrow            |  Bin 1884 -> 0 bytes
 js/test/integration-tests.ts                  |  114 ++
 js/test/reader-tests.ts                       |   69 +-
 js/test/table-tests.ts                        |  175 +-
 js/test/test-config.ts                        |   50 +-
 js/test/tsconfig.json                         |    2 +-
 js/test/vector-tests.ts                       |  109 +-
 js/tsconfig/tsconfig.es5.cls.json             |    4 +-
 99 files changed, 2468 insertions(+), 6081 deletions(-)
 delete mode 100644 js/LICENSE
 delete mode 100644 js/closure-compiler-scripts/flatbuffers.js
 delete mode 100644 js/closure-compiler-scripts/text-encoding.js
 delete mode 100644 js/closure-compiler-scripts/tslib.js
 rename js/{prepublish.sh => npm-release.sh} (85%)
 delete mode 100644 js/perf/arrows/file/dictionary.arrow
 delete mode 100644 js/perf/arrows/file/simple.arrow
 delete mode 100644 js/perf/arrows/file/struct.arrow
 delete mode 100644 js/perf/arrows/multi/count/records.arrow
 delete mode 100644 js/perf/arrows/multi/count/schema.arrow
 delete mode 100644 js/perf/arrows/multi/latlong/records.arrow
 delete mode 100644 js/perf/arrows/multi/latlong/schema.arrow
 delete mode 100644 js/perf/arrows/multi/origins/records.arrow
 delete mode 100644 js/perf/arrows/multi/origins/schema.arrow
 delete mode 100644 js/perf/arrows/stream/dictionary.arrow
 delete mode 100644 js/perf/arrows/stream/simple.arrow
 delete mode 100644 js/perf/arrows/stream/struct.arrow
 rename js/src/format/{File_generated.ts => File.ts} (99%)
 rename js/{closure-compiler-scripts => src/format}/File_generated.js (95%)
 rename js/src/format/{Message_generated.ts => Message.ts} (94%)
 rename js/{closure-compiler-scripts => src/format}/Message_generated.js (93%)
 rename js/src/format/{Schema_generated.ts => Schema.ts} (99%)
 rename js/{closure-compiler-scripts => src/format}/Schema_generated.js (98%)
 delete mode 100644 js/src/reader/dictionary.ts
 delete mode 100644 js/src/reader/file.ts
 create mode 100644 js/src/reader/format.ts
 delete mode 100644 js/src/reader/message.ts
 delete mode 100644 js/src/reader/stream.ts
 delete mode 100644 js/src/types/arrow.ts
 delete mode 100644 js/src/types/fixedsizelist.ts
 delete mode 100644 js/src/types/list.ts
 delete mode 100644 js/src/types/table/from.ts
 delete mode 100644 js/src/types/table/row.ts
 delete mode 100644 js/src/types/table/struct.ts
 delete mode 100644 js/src/types/table/table.ts
 delete mode 100644 js/src/types/table/toString.ts
 delete mode 100644 js/src/types/vector/bool.ts
 delete mode 100644 js/src/types/vector/date.ts
 delete mode 100644 js/src/types/vector/long.ts
 delete mode 100644 js/src/types/vector/typed.ts
 delete mode 100644 js/src/types/vector/virtual.ts
 create mode 100644 js/src/vector/arrow.ts
 rename js/src/{types => vector}/dictionary.ts (62%)
 create mode 100644 js/src/vector/list.ts
 create mode 100644 js/src/vector/numeric.ts
 create mode 100644 js/src/vector/struct.ts
 create mode 100644 js/src/vector/table.ts
 rename js/src/{types => }/vector/traits.ts (88%)
 create mode 100644 js/src/vector/types.ts
 rename js/src/{types => vector}/utf8.ts (94%)
 rename js/src/{types/types.ts => vector/vector.ts} (56%)
 create mode 100644 js/src/vector/virtual.ts
 delete mode 100644 js/test/__snapshots__/reader-tests.ts.snap
 delete mode 100644 js/test/__snapshots__/table-tests.ts.snap
 delete mode 100644 js/test/arrows/file/dictionary.arrow
 delete mode 100644 js/test/arrows/file/dictionary2.arrow
 delete mode 100644 js/test/arrows/file/multi_dictionary.arrow
 delete mode 100644 js/test/arrows/file/simple.arrow
 delete mode 100644 js/test/arrows/file/struct.arrow
 delete mode 100644 js/test/arrows/multi/count/records.arrow
 delete mode 100644 js/test/arrows/multi/count/schema.arrow
 delete mode 100644 js/test/arrows/multi/latlong/records.arrow
 delete mode 100644 js/test/arrows/multi/latlong/schema.arrow
 delete mode 100644 js/test/arrows/multi/origins/records.arrow
 delete mode 100644 js/test/arrows/multi/origins/schema.arrow
 delete mode 100644 js/test/arrows/stream/dictionary.arrow
 delete mode 100644 js/test/arrows/stream/simple.arrow
 delete mode 100644 js/test/arrows/stream/struct.arrow
 create mode 100644 js/test/integration-tests.ts

diff --git a/.travis.yml b/.travis.yml
index 9917a261451c2..9c714a68948e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -87,7 +87,7 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
   - language: java
     os: linux
-    jdk: openjdk8
+    jdk: openjdk7
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
   - language: java
@@ -103,23 +103,24 @@ matrix:
   - language: java
     os: linux
     env: ARROW_TEST_GROUP=integration
-    jdk: openjdk7
+    jdk: openjdk8
     before_script:
     - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - export CC="clang-4.0"
     - export CXX="clang++-4.0"
+    - nvm install node
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
-  # TODO(wesm): Re-enable after issues in ARROW-1409 resolved
-  # - language: node_js
-  #   os: linux
-  #   node_js: node
-  #   before_script:
-  #   - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
-  #   script:
-  #   - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
+  - language: node_js
+    os: linux
+    node_js: node
+    before_script:
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
   - compiler: gcc
     language: cpp
     os: linux
diff --git a/LICENSE.txt b/LICENSE.txt
index 038518a5d9b68..84e6a4e2a2a97 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -457,3 +457,98 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the Boost project
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the mapbox/variant project, BSD 3-clause
+license
+
+Copyright (c) MapBox
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+- Neither the name "MapBox" nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the FlatBuffers project
+
+Copyright 2014 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the tslib project
+
+Copyright 2015 Microsoft Corporation. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index be025512f0b88..105a0cc6086df 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -44,3 +44,22 @@ conda install -y pip numpy six
 python integration_test.py --debug
 
 popd
+
+pushd $ARROW_JS_DIR
+
+# lint and compile JS source
+npm run lint
+npm run build
+# create initial test data
+npm run test:createTestData
+# run once to write the snapshots
+npm test -- -t ts -u --integration
+# run again to test all builds against the snapshots
+npm test -- --integration
+# run tests against source to generate coverage data
+npm run test:coverage -- --integration
+# Uncomment to upload to coveralls
+# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
+
+
+popd
\ No newline at end of file
diff --git a/ci/travis_script_js.sh b/ci/travis_script_js.sh
index 9f77dec8c53a7..1871b4265cd01 100755
--- a/ci/travis_script_js.sh
+++ b/ci/travis_script_js.sh
@@ -17,16 +17,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+set -ex
 
-JS_DIR=${TRAVIS_BUILD_DIR}/js
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-pushd $JS_DIR
+pushd $ARROW_JS_DIR
 
-npm run validate
-
-# Uncomment to use coveralls
-# npm run test:coverage
-# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
+npm run lint
+npm run build
+# run the non-snapshot unit tests
+npm test
 
 popd
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 46d010608118d..a063fb36f49d5 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -28,6 +28,7 @@
 import subprocess
 import tempfile
 import uuid
+import errno
 
 import numpy as np
 
@@ -198,9 +199,18 @@ def __init__(self, name, is_signed, bit_width, nullable=True,
         self.min_value = min_value
         self.max_value = max_value
 
-    @property
-    def numpy_type(self):
-        return ('int' if self.is_signed else 'uint') + str(self.bit_width)
+    def _get_generated_data_bounds(self):
+        signed_iinfo = np.iinfo('int' + str(self.bit_width))
+        if self.is_signed:
+            min_value, max_value = signed_iinfo.min, signed_iinfo.max
+        else:
+            # ARROW-1837 Remove this hack and restore full unsigned integer
+            # range
+            min_value, max_value = 0, signed_iinfo.max
+
+        lower_bound = max(min_value, self.min_value)
+        upper_bound = min(max_value, self.max_value)
+        return lower_bound, upper_bound
 
     def _get_type(self):
         return OrderedDict([
@@ -210,9 +220,7 @@ def _get_type(self):
         ])
 
     def generate_column(self, size, name=None):
-        iinfo = np.iinfo(self.numpy_type)
-        lower_bound = max(iinfo.min, self.min_value)
-        upper_bound = min(iinfo.max, self.max_value)
+        lower_bound, upper_bound = self._get_generated_data_bounds()
         return self.generate_range(size, lower_bound, upper_bound, name=name)
 
     def generate_range(self, size, lower, upper, name=None):
@@ -521,7 +529,7 @@ def get_json(self):
 class BinaryColumn(PrimitiveColumn):
 
     def _encode_value(self, x):
-        return frombytes(binascii.hexlify(x))
+        return frombytes(binascii.hexlify(x).upper())
 
     def _get_buffers(self):
         offset = 0
@@ -785,7 +793,7 @@ def _generate_file(name, fields, batch_sizes, dictionaries=None):
     return JsonFile(name, schema, batches, dictionaries)
 
 
-def generate_primitive_case(batch_sizes):
+def generate_primitive_case(batch_sizes, name='primitive'):
     types = ['bool', 'int8', 'int16', 'int32', 'int64',
              'uint8', 'uint16', 'uint32', 'uint64',
              'float32', 'float64', 'binary', 'utf8']
@@ -796,7 +804,7 @@ def generate_primitive_case(batch_sizes):
         fields.append(get_field(type_ + "_nullable", type_, True))
         fields.append(get_field(type_ + "_nonnullable", type_, False))
 
-    return _generate_file("primitive", fields, batch_sizes)
+    return _generate_file(name, fields, batch_sizes)
 
 
 def generate_decimal_case():
@@ -874,8 +882,8 @@ def _temp_path():
         return
 
     file_objs = [
-        generate_primitive_case([7, 10]),
-        generate_primitive_case([0, 0, 0]),
+        generate_primitive_case([17, 20], name='primitive'),
+        generate_primitive_case([0, 0, 0], name='primitive_zerolength'),
         generate_decimal_case(),
         generate_datetime_case(),
         generate_nested_case(),
@@ -1079,11 +1087,33 @@ def run_all_tests(debug=False):
     print('-- All tests passed!')
 
 
+def write_js_test_json(directory):
+    generate_nested_case().write(os.path.join(directory, 'nested.json'))
+    generate_decimal_case().write(os.path.join(directory, 'decimal.json'))
+    generate_datetime_case().write(os.path.join(directory, 'datetime.json'))
+    (generate_dictionary_case()
+     .write(os.path.join(directory, 'dictionary.json')))
+    (generate_primitive_case([7, 10])
+     .write(os.path.join(directory, 'primitive.json')))
+    (generate_primitive_case([0, 0, 0])
+     .write(os.path.join(directory, 'primitive-empty.json')))
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Arrow integration test CLI')
+    parser.add_argument('--write_generated_json', dest='generated_json_path',
+                        action='store', default=False,
+                        help='Generate test JSON')
     parser.add_argument('--debug', dest='debug', action='store_true',
                         default=False,
                         help='Run executables in debug mode as relevant')
-
     args = parser.parse_args()
-    run_all_tests(debug=args.debug)
+    if args.generated_json_path:
+        try:
+            os.makedirs(args.generated_json_path)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        write_js_test_json(args.generated_json_path)
+    else:
+        run_all_tests(debug=args.debug)
diff --git a/js/.gitignore b/js/.gitignore
index 88c612d8faf37..f705f2510f93f 100644
--- a/js/.gitignore
+++ b/js/.gitignore
@@ -83,3 +83,8 @@ package-lock.json
 # compilation targets
 dist
 targets
+
+# test data files
+test/data/
+# jest snapshots (too big)
+test/__snapshots__/
\ No newline at end of file
diff --git a/js/DEVELOP.md b/js/DEVELOP.md
index 5b4ac14ed3466..9f586e1b39bcb 100644
--- a/js/DEVELOP.md
+++ b/js/DEVELOP.md
@@ -50,91 +50,220 @@ Once generated, the flatbuffers format code needs to be adjusted for our TS and
 
 1. Generate the flatbuffers TypeScript source from the Arrow project root directory:
     ```sh
+    cd $ARROW_HOME
+
     flatc --ts -o ./js/src/format ./format/*.fbs
+
+    cd ./js/src/format
+
+    # Delete Tensor_generated.js (skip this when we support Tensors)
+    rm ./Tensor_generated.ts
+
+    # Remove "_generated" suffix from TS files
+    mv ./File_generated.ts .File.ts
+    mv ./Schema_generated.ts .Schema.ts
+    mv ./Message_generated.ts .Message.ts
     ```
-1. Change all the `flatbuffers` imports to
+1. Remove Tensor import from `Schema.ts`
+1. Fix all the `flatbuffers` imports
     ```ts
-    import { flatbuffers } from "flatbuffers"
+    import { flatbuffers } from "./flatbuffers" // <-- change
+    import { flatbuffers } from "flatbuffers" // <-- to this
     ```
-1. Delete `Tensor_generated.ts` (remove this step once we support Tensors)
-1. Remove Tensor import from `Schema_generated.ts`
-1. Add `/* tslint:disable:class-name */` to the top of `Schema_generated.ts`
+1. Remove `_generated` from the ES6 imports of the generated files
+    ```ts
+    import * as NS16187549871986683199 from "./Schema_generated"; // <-- change
+    import * as NS16187549871986683199 from "./Schema"; // <------- to this
+    ```
+1. Add `/* tslint:disable:class-name */` to the top of `Schema.ts`
+1. Execute `npm run lint` to fix all the linting errors
 
 ## JavaScript (for Google Closure Compiler builds)
 
 1. Generate the flatbuffers JS source from the Arrow project root directory
     ```sh
-    flatc --js -o ./js/closure-compiler-scripts ./format/*.fbs
-    ```
-1. Delete `Tensor_generated.js` (remove this step once we support Tensors)
-1. Add `goog.module` declarations to the top of each generated file
+    cd $ARROW_HOME
 
-    Each file starts with a header that looks like this:
-    ```js
-    // automatically generated by the FlatBuffers compiler, do not modify
+    flatc --js --no-js-exports -o ./js/src/format ./format/*.fbs
 
-    /**
-    * @const
-    * @namespace
-    */
-    var org = org || {};
-    ```
+    cd ./js/src/format
+
+    # Delete Tensor_generated.js (skip this when we support Tensors)
+    rm Tensor_generated.js
 
-    Update the header of each file to explicitly declare its module.
+    # append an ES6 export to Schema_generated.js
+    echo "$(cat Schema_generated.js)
+    export { org };
+    " > Schema_generated.js
 
-    `Schema_generated.js`:
+    # import Schema's "org" namespace and
+    # append an ES6 export to File_generated.js
+    echo "import { org } from './Schema';
+    $(cat File_generated.js)
+    export { org };
+    " > File_generated.js
+
+    # import Schema's "org" namespace and
+    # append an ES6 export to Message_generated.js
+    echo "import { org } from './Schema';
+    $(cat Message_generated.js)
+    export { org };
+    " > Message_generated.js
+    ```
+1. Fixup the generated JS enums with the reverse value-to-key mappings to match TypeScript
+    `Message_generated.js`
     ```js
-    // automatically generated by the FlatBuffers compiler, do not modify
-    goog.module("module$targets$es5$cls$format$Schema_generated");
-    goog.module.declareLegacyNamespace();
+    // Replace this
+    org.apache.arrow.flatbuf.MessageHeader = {
+      NONE: 0,
+      Schema: 1,
+      DictionaryBatch: 2,
+      RecordBatch: 3,
+      Tensor: 4
+    };
+    // With this
+    org.apache.arrow.flatbuf.MessageHeader = {
+      NONE: 0, 0: 'NONE',
+      Schema: 1, 1: 'Schema',
+      DictionaryBatch: 2, 2: 'DictionaryBatch',
+      RecordBatch: 3, 3: 'RecordBatch',
+      Tensor: 4, 4: 'Tensor'
+    };
+    ```
+    `Schema_generated.js`
+    ```js
+    /**
+     * @enum
+     */
+    org.apache.arrow.flatbuf.MetadataVersion = {
+      /**
+       * 0.1.0
+       */
+      V1: 0, 0: 'V1',
+
+      /**
+       * 0.2.0
+       */
+      V2: 1, 1: 'V2',
+
+      /**
+       * 0.3.0 -> 0.7.1
+       */
+      V3: 2, 2: 'V3',
+
+      /**
+       * >= 0.8.0
+       */
+      V4: 3, 3: 'V4'
+    };
 
     /**
-    * @const
-    * @namespace
-    */
-    var org = org || {};
-    ```
+     * @enum
+     */
+    org.apache.arrow.flatbuf.UnionMode = {
+      Sparse: 0, 0: 'Sparse',
+      Dense: 1, 1: 'Dense',
+    };
 
-    `File_generated.js`:
+    /**
+     * @enum
+     */
+    org.apache.arrow.flatbuf.Precision = {
+      HALF: 0, 0: 'HALF',
+      SINGLE: 1, 1: 'SINGLE',
+      DOUBLE: 2, 2: 'DOUBLE',
+    };
 
-    ```js
-    // automatically generated by the FlatBuffers compiler, do not modify
-    goog.module("module$targets$es5$cls$format$File_generated");
-    goog.module.declareLegacyNamespace();
-    var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
     /**
-    * @const
-    * @namespace
-    */
-    var org = Schema_.org;
-    ```
+     * @enum
+     */
+    org.apache.arrow.flatbuf.DateUnit = {
+      DAY: 0, 0: 'DAY',
+      MILLISECOND: 1, 1: 'MILLISECOND',
+    };
 
-    `Message_generated.js`:
+    /**
+     * @enum
+     */
+    org.apache.arrow.flatbuf.TimeUnit = {
+      SECOND: 0, 0: 'SECOND',
+      MILLISECOND: 1, 1: 'MILLISECOND',
+      MICROSECOND: 2, 2: 'MICROSECOND',
+      NANOSECOND: 3, 3: 'NANOSECOND',
+    };
 
-    ```js
-    // automatically generated by the FlatBuffers compiler, do not modify
-    goog.module("module$targets$es5$cls$format$Message_generated");
-    goog.module.declareLegacyNamespace();
-    var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
     /**
-    * @const
-    * @namespace
-    */
-    var org = Schema_.org;
-    ```
+     * @enum
+     */
+    org.apache.arrow.flatbuf.IntervalUnit = {
+      YEAR_MONTH: 0, 0: 'YEAR_MONTH',
+      DAY_TIME: 1, 1: 'DAY_TIME',
+    };
 
-1. Replace the last line's export declaration
+    /**
+     * ----------------------------------------------------------------------
+     * Top-level Type value, enabling extensible type-specific metadata. We can
+     * add new logical types to Type without breaking backwards compatibility
+     *
+     * @enum
+     */
+    org.apache.arrow.flatbuf.Type = {
+      NONE: 0, 0: 'NONE',
+      Null: 1, 1: 'Null',
+      Int: 2, 2: 'Int',
+      FloatingPoint: 3, 3: 'FloatingPoint',
+      Binary: 4, 4: 'Binary',
+      Utf8: 5, 5: 'Utf8',
+      Bool: 6, 6: 'Bool',
+      Decimal: 7, 7: 'Decimal',
+      Date: 8, 8: 'Date',
+      Time: 9, 9: 'Time',
+      Timestamp: 10, 10: 'Timestamp',
+      Interval: 11, 11: 'Interval',
+      List: 12, 12: 'List',
+      Struct_: 13, 13: 'Struct_',
+      Union: 14, 14: 'Union',
+      FixedSizeBinary: 15, 15: 'FixedSizeBinary',
+      FixedSizeList: 16, 16: 'FixedSizeList',
+      Map: 17, 17: 'Map'
+    };
 
-    The last line of each file is:
+    /**
+     * ----------------------------------------------------------------------
+     * The possible types of a vector
+     *
+     * @enum
+     */
+    org.apache.arrow.flatbuf.VectorType = {
+      /**
+       * used in List type, Dense Union and variable length primitive types (String, Binary)
+       */
+      OFFSET: 0, 0: 'OFFSET',
 
-    ```js
-    // Exports for Node.js and RequireJS
-    this.org = org;
-    ```
+      /**
+       * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
+       */
+      DATA: 1, 1: 'DATA',
 
-    This should instead read:
+      /**
+       * Bit vector indicating if each value is null
+       */
+      VALIDITY: 2, 2: 'VALIDITY',
 
-    ```js
-    // Exports for Node.js and RequireJS
-    exports.org = org;
-    ```
\ No newline at end of file
+      /**
+       * Type vector used in Union type
+       */
+      TYPE: 3, 3: 'TYPE'
+    };
+
+    /**
+     * ----------------------------------------------------------------------
+     * Endianness of the platform producing the data
+     *
+     * @enum
+     */
+    org.apache.arrow.flatbuf.Endianness = {
+      Little: 0, 0: 'Little',
+      Big: 1, 1: 'Big',
+    };
+    ```
diff --git a/js/LICENSE b/js/LICENSE
deleted file mode 100644
index 02e7948085055..0000000000000
--- a/js/LICENSE
+++ /dev/null
@@ -1,39 +0,0 @@
-## 3rd-party licenses for code that has been adapted for the Arrow JavaScript
-   library
-
---------------------------------------------------------------------------------
-
-This project includes code from the FlatBuffers project
-
-Copyright 2014 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-This project includes code from the tslib project
-
-Copyright 2015 Microsoft Corporation. All rights reserved. 
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
diff --git a/js/README.md b/js/README.md
index bee3a9c7d42db..e07be6bc0aaaf 100644
--- a/js/README.md
+++ b/js/README.md
@@ -22,7 +22,7 @@
 [![Build Status](https://travis-ci.org/apache/arrow.svg?branch=master)](https://travis-ci.org/apache/arrow)
 [![Coverage Status](https://coveralls.io/repos/github/apache/arrow/badge.svg)](https://coveralls.io/github/apache/arrow)
 
-Arrow is a set of technologies that enable big-data systems to process and move data fast.
+Arrow is a set of technologies that enable big-data systems to process and transfer data quickly.
 
 ## install [apache-arrow from npm](https://www.npmjs.com/package/apache-arrow)
 
@@ -50,7 +50,7 @@ import { readFileSync } from 'fs';
 import { Table } from 'apache-arrow';
 
 const arrow = readFileSync('simple.arrow');
-const table = Table.from(arrow);
+const table = Table.from([arrow]);
 
 console.log(table.toString());
 
@@ -70,7 +70,7 @@ null, null, null
 import { readFileSync } from 'fs';
 import { Table } from 'apache-arrow';
 
-const table = Table.from(...[
+const table = Table.from([
     'latlong/schema.arrow',
     'latlong/records.arrow'
 ].map((file) => readFileSync(file)));
@@ -93,12 +93,12 @@ console.log(table.toString());
 import { readFileSync } from 'fs';
 import { Table } from 'apache-arrow';
 
-const table = Table.from(...[
+const table = Table.from([
     'latlong/schema.arrow',
     'latlong/records.arrow'
 ].map(readFileSync));
 
-const column = table.getColumn('origin_lat');
+const column = table.col('origin_lat');
 const typed = column.slice();
 
 assert(typed instanceof Float32Array);
@@ -135,7 +135,7 @@ MapD.open(host, port)
     // Create Arrow Table from results
     Table.from(schema, records))
   .map((table) =>
-    // Stringify the table to CSV
+    // Stringify the table to CSV with row numbers
     table.toString({ index: true }))
   .subscribe((csvStr) =>
     console.log(csvStr));
diff --git a/js/closure-compiler-scripts/flatbuffers.js b/js/closure-compiler-scripts/flatbuffers.js
deleted file mode 100644
index e51a4a039536f..0000000000000
--- a/js/closure-compiler-scripts/flatbuffers.js
+++ /dev/null
@@ -1,1204 +0,0 @@
-/**
- * closure-compiler-friendly flatbuffers
- * copied from node_modules/flatbuffers/js/flatbuffers.js
- * update as needed
- */
-
- /// @file
-/// @addtogroup flatbuffers_javascript_api
-/// @{
-/// @cond FLATBUFFERS_INTERNAL
-
-goog.module("module$flatbuffers");
-goog.module.declareLegacyNamespace();
-/**
- * @fileoverview
- *
- * Need to suppress 'global this' error so the Node.js export line doesn't cause
- * closure compile to error out.
- * @suppress {globalThis}
- */
-
-/**
- * @const
- * @namespace
- */
-var flatbuffers = {};
-
-/**
- * @typedef {number}
- */
-flatbuffers.Offset;
-
-/**
- * @typedef {{
- *   bb: flatbuffers.ByteBuffer,
- *   bb_pos: number
- * }}
- */
-flatbuffers.Table;
-
-/**
- * @type {number}
- * @const
- */
-flatbuffers.SIZEOF_SHORT = 2;
-
-/**
- * @type {number}
- * @const
- */
-flatbuffers.SIZEOF_INT = 4;
-
-/**
- * @type {number}
- * @const
- */
-flatbuffers.FILE_IDENTIFIER_LENGTH = 4;
-
-/**
- * @enum {number}
- */
-flatbuffers.Encoding = {
-  UTF8_BYTES: 1,
-  UTF16_STRING: 2
-};
-
-/**
- * @type {Int32Array}
- * @const
- */
-flatbuffers.int32 = new Int32Array(2);
-
-/**
- * @type {Float32Array}
- * @const
- */
-flatbuffers.float32 = new Float32Array(flatbuffers.int32.buffer);
-
-/**
- * @type {Float64Array}
- * @const
- */
-flatbuffers.float64 = new Float64Array(flatbuffers.int32.buffer);
-
-/**
- * @type {boolean}
- * @const
- */
-flatbuffers.isLittleEndian = new Uint16Array(new Uint8Array([1, 0]).buffer)[0] === 1;
-
-////////////////////////////////////////////////////////////////////////////////
-
-/**
- * @constructor
- * @param {number} low
- * @param {number} high
- */
-flatbuffers.Long = function(low, high) {
-  /**
-   * @type {number}
-   * @const
-   */
-  this.low = low | 0;
-
-  /**
-   * @type {number}
-   * @const
-   */
-  this.high = high | 0;
-};
-
-/**
- * @param {number} low
- * @param {number} high
- * @returns {flatbuffers.Long}
- */
-flatbuffers.Long.create = function(low, high) {
-  // Special-case zero to avoid GC overhead for default values
-  return low == 0 && high == 0 ? flatbuffers.Long.ZERO : new flatbuffers.Long(low, high);
-};
-
-/**
- * @returns {number}
- */
-flatbuffers.Long.prototype.toFloat64 = function() {
-  return (this.low >>> 0) + this.high * 0x100000000;
-};
-
-/**
- * @param {flatbuffers.Long} other
- * @returns {boolean}
- */
-flatbuffers.Long.prototype.equals = function(other) {
-  return this.low == other.low && this.high == other.high;
-};
-
-/**
- * @type {flatbuffers.Long}
- * @const
- */
-flatbuffers.Long.ZERO = new flatbuffers.Long(0, 0);
-
-/// @endcond
-////////////////////////////////////////////////////////////////////////////////
-/**
- * Create a FlatBufferBuilder.
- *
- * @constructor
- * @param {number=} opt_initial_size
- */
-flatbuffers.Builder = function(opt_initial_size) {
-  if (!opt_initial_size) {
-    var initial_size = 1024;
-  } else {
-    var initial_size = opt_initial_size;
-  }
-
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   * @private
-   */
-  this.bb = flatbuffers.ByteBuffer.allocate(initial_size);
-
-  /**
-   * Remaining space in the ByteBuffer.
-   *
-   * @type {number}
-   * @private
-   */
-  this.space = initial_size;
-
-  /**
-   * Minimum alignment encountered so far.
-   *
-   * @type {number}
-   * @private
-   */
-  this.minalign = 1;
-
-  /**
-   * The vtable for the current table.
-   *
-   * @type {Array.<number>}
-   * @private
-   */
-  this.vtable = null;
-
-  /**
-   * The amount of fields we're actually using.
-   *
-   * @type {number}
-   * @private
-   */
-  this.vtable_in_use = 0;
-
-  /**
-   * Whether we are currently serializing a table.
-   *
-   * @type {boolean}
-   * @private
-   */
-  this.isNested = false;
-
-  /**
-   * Starting offset of the current struct/table.
-   *
-   * @type {number}
-   * @private
-   */
-  this.object_start = 0;
-
-  /**
-   * List of offsets of all vtables.
-   *
-   * @type {Array.<number>}
-   * @private
-   */
-  this.vtables = [];
-
-  /**
-   * For the current vector being built.
-   *
-   * @type {number}
-   * @private
-   */
-  this.vector_num_elems = 0;
-
-  /**
-   * False omits default values from the serialized data
-   *
-   * @type {boolean}
-   * @private
-   */
-  this.force_defaults = false;
-};
-
-/**
- * In order to save space, fields that are set to their default value
- * don't get serialized into the buffer. Forcing defaults provides a
- * way to manually disable this optimization.
- *
- * @param {boolean} forceDefaults true always serializes default values
- */
-flatbuffers.Builder.prototype.forceDefaults = function(forceDefaults) {
-  this.force_defaults = forceDefaults;
-};
-
-/**
- * Get the ByteBuffer representing the FlatBuffer. Only call this after you've
- * called finish(). The actual data starts at the ByteBuffer's current position,
- * not necessarily at 0.
- *
- * @returns {flatbuffers.ByteBuffer}
- */
-flatbuffers.Builder.prototype.dataBuffer = function() {
-  return this.bb;
-};
-
-/**
- * Get the bytes representing the FlatBuffer. Only call this after you've
- * called finish().
- *
- * @returns {Uint8Array}
- */
-flatbuffers.Builder.prototype.asUint8Array = function() {
-  return this.bb.bytes().subarray(this.bb.position(), this.bb.position() + this.offset());
-};
-
-/// @cond FLATBUFFERS_INTERNAL
-/**
- * Prepare to write an element of `size` after `additional_bytes` have been
- * written, e.g. if you write a string, you need to align such the int length
- * field is aligned to 4 bytes, and the string data follows it directly. If all
- * you need to do is alignment, `additional_bytes` will be 0.
- *
- * @param {number} size This is the of the new element to write
- * @param {number} additional_bytes The padding size
- */
-flatbuffers.Builder.prototype.prep = function(size, additional_bytes) {
-  // Track the biggest thing we've ever aligned to.
-  if (size > this.minalign) {
-    this.minalign = size;
-  }
-
-  // Find the amount of alignment needed such that `size` is properly
-  // aligned after `additional_bytes`
-  var align_size = ((~(this.bb.capacity() - this.space + additional_bytes)) + 1) & (size - 1);
-
-  // Reallocate the buffer if needed.
-  while (this.space < align_size + size + additional_bytes) {
-    var old_buf_size = this.bb.capacity();
-    this.bb = flatbuffers.Builder.growByteBuffer(this.bb);
-    this.space += this.bb.capacity() - old_buf_size;
-  }
-
-  this.pad(align_size);
-};
-
-/**
- * @param {number} byte_size
- */
-flatbuffers.Builder.prototype.pad = function(byte_size) {
-  for (var i = 0; i < byte_size; i++) {
-    this.bb.writeInt8(--this.space, 0);
-  }
-};
-
-/**
- * @param {number} value
- */
-flatbuffers.Builder.prototype.writeInt8 = function(value) {
-  this.bb.writeInt8(this.space -= 1, value);
-};
-
-/**
- * @param {number} value
- */
-flatbuffers.Builder.prototype.writeInt16 = function(value) {
-  this.bb.writeInt16(this.space -= 2, value);
-};
-
-/**
- * @param {number} value
- */
-flatbuffers.Builder.prototype.writeInt32 = function(value) {
-  this.bb.writeInt32(this.space -= 4, value);
-};
-
-/**
- * @param {flatbuffers.Long} value
- */
-flatbuffers.Builder.prototype.writeInt64 = function(value) {
-  this.bb.writeInt64(this.space -= 8, value);
-};
-
-/**
- * @param {number} value
- */
-flatbuffers.Builder.prototype.writeFloat32 = function(value) {
-  this.bb.writeFloat32(this.space -= 4, value);
-};
-
-/**
- * @param {number} value
- */
-flatbuffers.Builder.prototype.writeFloat64 = function(value) {
-  this.bb.writeFloat64(this.space -= 8, value);
-};
-/// @endcond
-
-/**
- * Add an `int8` to the buffer, properly aligned, and grows the buffer (if necessary).
- * @param {number} value The `int8` to add the the buffer.
- */
-flatbuffers.Builder.prototype.addInt8 = function(value) {
-  this.prep(1, 0);
-  this.writeInt8(value);
-};
-
-/**
- * Add an `int16` to the buffer, properly aligned, and grows the buffer (if necessary).
- * @param {number} value The `int16` to add the the buffer.
- */
-flatbuffers.Builder.prototype.addInt16 = function(value) {
-  this.prep(2, 0);
-  this.writeInt16(value);
-};
-
-/**
- * Add an `int32` to the buffer, properly aligned, and grows the buffer (if necessary).
- * @param {number} value The `int32` to add the the buffer.
- */
-flatbuffers.Builder.prototype.addInt32 = function(value) {
-  this.prep(4, 0);
-  this.writeInt32(value);
-};
-
-/**
- * Add an `int64` to the buffer, properly aligned, and grows the buffer (if necessary).
- * @param {flatbuffers.Long} value The `int64` to add the the buffer.
- */
-flatbuffers.Builder.prototype.addInt64 = function(value) {
-  this.prep(8, 0);
-  this.writeInt64(value);
-};
-
-/**
- * Add a `float32` to the buffer, properly aligned, and grows the buffer (if necessary).
- * @param {number} value The `float32` to add the the buffer.
- */
-flatbuffers.Builder.prototype.addFloat32 = function(value) {
-  this.prep(4, 0);
-  this.writeFloat32(value);
-};
-
-/**
- * Add a `float64` to the buffer, properly aligned, and grows the buffer (if necessary).
- * @param {number} value The `float64` to add the the buffer.
- */
-flatbuffers.Builder.prototype.addFloat64 = function(value) {
-  this.prep(8, 0);
-  this.writeFloat64(value);
-};
-
-/// @cond FLATBUFFERS_INTERNAL
-/**
- * @param {number} voffset
- * @param {number} value
- * @param {number} defaultValue
- */
-flatbuffers.Builder.prototype.addFieldInt8 = function(voffset, value, defaultValue) {
-  if (this.force_defaults || value != defaultValue) {
-    this.addInt8(value);
-    this.slot(voffset);
-  }
-};
-
-/**
- * @param {number} voffset
- * @param {number} value
- * @param {number} defaultValue
- */
-flatbuffers.Builder.prototype.addFieldInt16 = function(voffset, value, defaultValue) {
-  if (this.force_defaults || value != defaultValue) {
-    this.addInt16(value);
-    this.slot(voffset);
-  }
-};
-
-/**
- * @param {number} voffset
- * @param {number} value
- * @param {number} defaultValue
- */
-flatbuffers.Builder.prototype.addFieldInt32 = function(voffset, value, defaultValue) {
-  if (this.force_defaults || value != defaultValue) {
-    this.addInt32(value);
-    this.slot(voffset);
-  }
-};
-
-/**
- * @param {number} voffset
- * @param {flatbuffers.Long} value
- * @param {flatbuffers.Long} defaultValue
- */
-flatbuffers.Builder.prototype.addFieldInt64 = function(voffset, value, defaultValue) {
-  if (this.force_defaults || !value.equals(defaultValue)) {
-    this.addInt64(value);
-    this.slot(voffset);
-  }
-};
-
-/**
- * @param {number} voffset
- * @param {number} value
- * @param {number} defaultValue
- */
-flatbuffers.Builder.prototype.addFieldFloat32 = function(voffset, value, defaultValue) {
-  if (this.force_defaults || value != defaultValue) {
-    this.addFloat32(value);
-    this.slot(voffset);
-  }
-};
-
-/**
- * @param {number} voffset
- * @param {number} value
- * @param {number} defaultValue
- */
-flatbuffers.Builder.prototype.addFieldFloat64 = function(voffset, value, defaultValue) {
-  if (this.force_defaults || value != defaultValue) {
-    this.addFloat64(value);
-    this.slot(voffset);
-  }
-};
-
-/**
- * @param {number} voffset
- * @param {flatbuffers.Offset} value
- * @param {flatbuffers.Offset} defaultValue
- */
-flatbuffers.Builder.prototype.addFieldOffset = function(voffset, value, defaultValue) {
-  if (this.force_defaults || value != defaultValue) {
-    this.addOffset(value);
-    this.slot(voffset);
-  }
-};
-
-/**
- * Structs are stored inline, so nothing additional is being added. `d` is always 0.
- *
- * @param {number} voffset
- * @param {flatbuffers.Offset} value
- * @param {flatbuffers.Offset} defaultValue
- */
-flatbuffers.Builder.prototype.addFieldStruct = function(voffset, value, defaultValue) {
-  if (value != defaultValue) {
-    this.nested(value);
-    this.slot(voffset);
-  }
-};
-
-/**
- * Structures are always stored inline, they need to be created right
- * where they're used.  You'll get this assertion failure if you
- * created it elsewhere.
- *
- * @param {flatbuffers.Offset} obj The offset of the created object
- */
-flatbuffers.Builder.prototype.nested = function(obj) {
-  if (obj != this.offset()) {
-    throw new Error('FlatBuffers: struct must be serialized inline.');
-  }
-};
-
-/**
- * Should not be creating any other object, string or vector
- * while an object is being constructed
- */
-flatbuffers.Builder.prototype.notNested = function() {
-  if (this.isNested) {
-    throw new Error('FlatBuffers: object serialization must not be nested.');
-  }
-};
-
-/**
- * Set the current vtable at `voffset` to the current location in the buffer.
- *
- * @param {number} voffset
- */
-flatbuffers.Builder.prototype.slot = function(voffset) {
-  this.vtable[voffset] = this.offset();
-};
-
-/**
- * @returns {flatbuffers.Offset} Offset relative to the end of the buffer.
- */
-flatbuffers.Builder.prototype.offset = function() {
-  return this.bb.capacity() - this.space;
-};
-
-/**
- * Doubles the size of the backing ByteBuffer and copies the old data towards
- * the end of the new buffer (since we build the buffer backwards).
- *
- * @param {flatbuffers.ByteBuffer} bb The current buffer with the existing data
- * @returns {flatbuffers.ByteBuffer} A new byte buffer with the old data copied
- * to it. The data is located at the end of the buffer.
- *
- * uint8Array.set() formally takes {Array<number>|ArrayBufferView}, so to pass
- * it a uint8Array we need to suppress the type check:
- * @suppress {checkTypes}
- */
-flatbuffers.Builder.growByteBuffer = function(bb) {
-  var old_buf_size = bb.capacity();
-
-  // Ensure we don't grow beyond what fits in an int.
-  if (old_buf_size & 0xC0000000) {
-    throw new Error('FlatBuffers: cannot grow buffer beyond 2 gigabytes.');
-  }
-
-  var new_buf_size = old_buf_size << 1;
-  var nbb = flatbuffers.ByteBuffer.allocate(new_buf_size);
-  nbb.setPosition(new_buf_size - old_buf_size);
-  nbb.bytes().set(bb.bytes(), new_buf_size - old_buf_size);
-  return nbb;
-};
-/// @endcond
-
-/**
- * Adds on offset, relative to where it will be written.
- *
- * @param {flatbuffers.Offset} offset The offset to add.
- */
-flatbuffers.Builder.prototype.addOffset = function(offset) {
-  this.prep(flatbuffers.SIZEOF_INT, 0); // Ensure alignment is already done.
-  this.writeInt32(this.offset() - offset + flatbuffers.SIZEOF_INT);
-};
-
-/// @cond FLATBUFFERS_INTERNAL
-/**
- * Start encoding a new object in the buffer.  Users will not usually need to
- * call this directly. The FlatBuffers compiler will generate helper methods
- * that call this method internally.
- *
- * @param {number} numfields
- */
-flatbuffers.Builder.prototype.startObject = function(numfields) {
-  this.notNested();
-  if (this.vtable == null) {
-    this.vtable = [];
-  }
-  this.vtable_in_use = numfields;
-  for (var i = 0; i < numfields; i++) {
-    this.vtable[i] = 0; // This will push additional elements as needed
-  }
-  this.isNested = true;
-  this.object_start = this.offset();
-};
-
-/**
- * Finish off writing the object that is under construction.
- *
- * @returns {flatbuffers.Offset} The offset to the object inside `dataBuffer`
- */
-flatbuffers.Builder.prototype.endObject = function() {
-  if (this.vtable == null || !this.isNested) {
-    throw new Error('FlatBuffers: endObject called without startObject');
-  }
-
-  this.addInt32(0);
-  var vtableloc = this.offset();
-
-  // Write out the current vtable.
-  for (var i = this.vtable_in_use - 1; i >= 0; i--) {
-    // Offset relative to the start of the table.
-    this.addInt16(this.vtable[i] != 0 ? vtableloc - this.vtable[i] : 0);
-  }
-
-  var standard_fields = 2; // The fields below:
-  this.addInt16(vtableloc - this.object_start);
-  this.addInt16((this.vtable_in_use + standard_fields) * flatbuffers.SIZEOF_SHORT);
-
-  // Search for an existing vtable that matches the current one.
-  var existing_vtable = 0;
-outer_loop:
-  for (var i = 0; i < this.vtables.length; i++) {
-    var vt1 = this.bb.capacity() - this.vtables[i];
-    var vt2 = this.space;
-    var len = this.bb.readInt16(vt1);
-    if (len == this.bb.readInt16(vt2)) {
-      for (var j = flatbuffers.SIZEOF_SHORT; j < len; j += flatbuffers.SIZEOF_SHORT) {
-        if (this.bb.readInt16(vt1 + j) != this.bb.readInt16(vt2 + j)) {
-          continue outer_loop;
-        }
-      }
-      existing_vtable = this.vtables[i];
-      break;
-    }
-  }
-
-  if (existing_vtable) {
-    // Found a match:
-    // Remove the current vtable.
-    this.space = this.bb.capacity() - vtableloc;
-
-    // Point table to existing vtable.
-    this.bb.writeInt32(this.space, existing_vtable - vtableloc);
-  } else {
-    // No match:
-    // Add the location of the current vtable to the list of vtables.
-    this.vtables.push(this.offset());
-
-    // Point table to current vtable.
-    this.bb.writeInt32(this.bb.capacity() - vtableloc, this.offset() - vtableloc);
-  }
-
-  this.isNested = false;
-  return vtableloc;
-};
-/// @endcond
-
-/**
- * Finalize a buffer, poiting to the given `root_table`.
- *
- * @param {flatbuffers.Offset} root_table
- * @param {string=} opt_file_identifier
- */
-flatbuffers.Builder.prototype.finish = function(root_table, opt_file_identifier) {
-  if (opt_file_identifier) {
-    var file_identifier = opt_file_identifier;
-    this.prep(this.minalign, flatbuffers.SIZEOF_INT +
-      flatbuffers.FILE_IDENTIFIER_LENGTH);
-    if (file_identifier.length != flatbuffers.FILE_IDENTIFIER_LENGTH) {
-      throw new Error('FlatBuffers: file identifier must be length ' +
-        flatbuffers.FILE_IDENTIFIER_LENGTH);
-    }
-    for (var i = flatbuffers.FILE_IDENTIFIER_LENGTH - 1; i >= 0; i--) {
-      this.writeInt8(file_identifier.charCodeAt(i));
-    }
-  }
-  this.prep(this.minalign, flatbuffers.SIZEOF_INT);
-  this.addOffset(root_table);
-  this.bb.setPosition(this.space);
-};
-
-/// @cond FLATBUFFERS_INTERNAL
-/**
- * This checks a required field has been set in a given table that has
- * just been constructed.
- *
- * @param {flatbuffers.Offset} table
- * @param {number} field
- */
-flatbuffers.Builder.prototype.requiredField = function(table, field) {
-  var table_start = this.bb.capacity() - table;
-  var vtable_start = table_start - this.bb.readInt32(table_start);
-  var ok = this.bb.readInt16(vtable_start + field) != 0;
-
-  // If this fails, the caller will show what field needs to be set.
-  if (!ok) {
-    throw new Error('FlatBuffers: field ' + field + ' must be set');
-  }
-};
-
-/**
- * Start a new array/vector of objects.  Users usually will not call
- * this directly. The FlatBuffers compiler will create a start/end
- * method for vector types in generated code.
- *
- * @param {number} elem_size The size of each element in the array
- * @param {number} num_elems The number of elements in the array
- * @param {number} alignment The alignment of the array
- */
-flatbuffers.Builder.prototype.startVector = function(elem_size, num_elems, alignment) {
-  this.notNested();
-  this.vector_num_elems = num_elems;
-  this.prep(flatbuffers.SIZEOF_INT, elem_size * num_elems);
-  this.prep(alignment, elem_size * num_elems); // Just in case alignment > int.
-};
-
-/**
- * Finish off the creation of an array and all its elements. The array must be
- * created with `startVector`.
- *
- * @returns {flatbuffers.Offset} The offset at which the newly created array
- * starts.
- */
-flatbuffers.Builder.prototype.endVector = function() {
-  this.writeInt32(this.vector_num_elems);
-  return this.offset();
-};
-/// @endcond
-
-/**
- * Encode the string `s` in the buffer using UTF-8. If a Uint8Array is passed
- * instead of a string, it is assumed to contain valid UTF-8 encoded data.
- *
- * @param {string|Uint8Array} s The string to encode
- * @return {flatbuffers.Offset} The offset in the buffer where the encoded string starts
- */
-flatbuffers.Builder.prototype.createString = function(s) {
-  if (s instanceof Uint8Array) {
-    var utf8 = s;
-  } else {
-    var utf8 = [];
-    var i = 0;
-
-    while (i < s.length) {
-      var codePoint;
-
-      // Decode UTF-16
-      var a = s.charCodeAt(i++);
-      if (a < 0xD800 || a >= 0xDC00) {
-        codePoint = a;
-      } else {
-        var b = s.charCodeAt(i++);
-        codePoint = (a << 10) + b + (0x10000 - (0xD800 << 10) - 0xDC00);
-      }
-
-      // Encode UTF-8
-      if (codePoint < 0x80) {
-        utf8.push(codePoint);
-      } else {
-        if (codePoint < 0x800) {
-          utf8.push(((codePoint >> 6) & 0x1F) | 0xC0);
-        } else {
-          if (codePoint < 0x10000) {
-            utf8.push(((codePoint >> 12) & 0x0F) | 0xE0);
-          } else {
-            utf8.push(
-              ((codePoint >> 18) & 0x07) | 0xF0,
-              ((codePoint >> 12) & 0x3F) | 0x80);
-          }
-          utf8.push(((codePoint >> 6) & 0x3F) | 0x80);
-        }
-        utf8.push((codePoint & 0x3F) | 0x80);
-      }
-    }
-  }
-
-  this.addInt8(0);
-  this.startVector(1, utf8.length, 1);
-  this.bb.setPosition(this.space -= utf8.length);
-  for (var i = 0, offset = this.space, bytes = this.bb.bytes(); i < utf8.length; i++) {
-    bytes[offset++] = utf8[i];
-  }
-  return this.endVector();
-};
-
-/**
- * A helper function to avoid generated code depending on this file directly.
- *
- * @param {number} low
- * @param {number} high
- * @returns {flatbuffers.Long}
- */
-flatbuffers.Builder.prototype.createLong = function(low, high) {
-  return flatbuffers.Long.create(low, high);
-};
-////////////////////////////////////////////////////////////////////////////////
-/// @cond FLATBUFFERS_INTERNAL
-/**
- * Create a new ByteBuffer with a given array of bytes (`Uint8Array`).
- *
- * @constructor
- * @param {Uint8Array} bytes
- */
-flatbuffers.ByteBuffer = function(bytes) {
-  /**
-   * @type {Uint8Array}
-   * @private
-   */
-  this.bytes_ = bytes;
-
-  /**
-   * @type {number}
-   * @private
-   */
-  this.position_ = 0;
-};
-
-/**
- * Create and allocate a new ByteBuffer with a given size.
- *
- * @param {number} byte_size
- * @returns {flatbuffers.ByteBuffer}
- */
-flatbuffers.ByteBuffer.allocate = function(byte_size) {
-  return new flatbuffers.ByteBuffer(new Uint8Array(byte_size));
-};
-
-/**
- * Get the underlying `Uint8Array`.
- *
- * @returns {Uint8Array}
- */
-flatbuffers.ByteBuffer.prototype.bytes = function() {
-  return this.bytes_;
-};
-
-/**
- * Get the buffer's position.
- *
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.position = function() {
-  return this.position_;
-};
-
-/**
- * Set the buffer's position.
- *
- * @param {number} position
- */
-flatbuffers.ByteBuffer.prototype.setPosition = function(position) {
-  this.position_ = position;
-};
-
-/**
- * Get the buffer's capacity.
- *
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.capacity = function() {
-  return this.bytes_.length;
-};
-
-/**
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.readInt8 = function(offset) {
-  return this.readUint8(offset) << 24 >> 24;
-};
-
-/**
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.readUint8 = function(offset) {
-  return this.bytes_[offset];
-};
-
-/**
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.readInt16 = function(offset) {
-  return this.readUint16(offset) << 16 >> 16;
-};
-
-/**
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.readUint16 = function(offset) {
-  return this.bytes_[offset] | this.bytes_[offset + 1] << 8;
-};
-
-/**
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.readInt32 = function(offset) {
-  return this.bytes_[offset] | this.bytes_[offset + 1] << 8 | this.bytes_[offset + 2] << 16 | this.bytes_[offset + 3] << 24;
-};
-
-/**
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.readUint32 = function(offset) {
-  return this.readInt32(offset) >>> 0;
-};
-
-/**
- * @param {number} offset
- * @returns {flatbuffers.Long}
- */
-flatbuffers.ByteBuffer.prototype.readInt64 = function(offset) {
-  return new flatbuffers.Long(this.readInt32(offset), this.readInt32(offset + 4));
-};
-
-/**
- * @param {number} offset
- * @returns {flatbuffers.Long}
- */
-flatbuffers.ByteBuffer.prototype.readUint64 = function(offset) {
-  return new flatbuffers.Long(this.readUint32(offset), this.readUint32(offset + 4));
-};
-
-/**
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.readFloat32 = function(offset) {
-  flatbuffers.int32[0] = this.readInt32(offset);
-  return flatbuffers.float32[0];
-};
-
-/**
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.readFloat64 = function(offset) {
-  flatbuffers.int32[flatbuffers.isLittleEndian ? 0 : 1] = this.readInt32(offset);
-  flatbuffers.int32[flatbuffers.isLittleEndian ? 1 : 0] = this.readInt32(offset + 4);
-  return flatbuffers.float64[0];
-};
-
-/**
- * @param {number} offset
- * @param {number|boolean} value
- */
-flatbuffers.ByteBuffer.prototype.writeInt8 = function(offset, value) {
-  this.bytes_[offset] = /** @type {number} */(value);
-};
-
-/**
- * @param {number} offset
- * @param {number} value
- */
-flatbuffers.ByteBuffer.prototype.writeUint8 = function(offset, value) {
-  this.bytes_[offset] = value;
-};
-
-/**
- * @param {number} offset
- * @param {number} value
- */
-flatbuffers.ByteBuffer.prototype.writeInt16 = function(offset, value) {
-  this.bytes_[offset] = value;
-  this.bytes_[offset + 1] = value >> 8;
-};
-
-/**
- * @param {number} offset
- * @param {number} value
- */
-flatbuffers.ByteBuffer.prototype.writeUint16 = function(offset, value) {
-    this.bytes_[offset] = value;
-    this.bytes_[offset + 1] = value >> 8;
-};
-
-/**
- * @param {number} offset
- * @param {number} value
- */
-flatbuffers.ByteBuffer.prototype.writeInt32 = function(offset, value) {
-  this.bytes_[offset] = value;
-  this.bytes_[offset + 1] = value >> 8;
-  this.bytes_[offset + 2] = value >> 16;
-  this.bytes_[offset + 3] = value >> 24;
-};
-
-/**
- * @param {number} offset
- * @param {number} value
- */
-flatbuffers.ByteBuffer.prototype.writeUint32 = function(offset, value) {
-    this.bytes_[offset] = value;
-    this.bytes_[offset + 1] = value >> 8;
-    this.bytes_[offset + 2] = value >> 16;
-    this.bytes_[offset + 3] = value >> 24;
-};
-
-/**
- * @param {number} offset
- * @param {flatbuffers.Long} value
- */
-flatbuffers.ByteBuffer.prototype.writeInt64 = function(offset, value) {
-  this.writeInt32(offset, value.low);
-  this.writeInt32(offset + 4, value.high);
-};
-
-/**
- * @param {number} offset
- * @param {flatbuffers.Long} value
- */
-flatbuffers.ByteBuffer.prototype.writeUint64 = function(offset, value) {
-    this.writeUint32(offset, value.low);
-    this.writeUint32(offset + 4, value.high);
-};
-
-/**
- * @param {number} offset
- * @param {number} value
- */
-flatbuffers.ByteBuffer.prototype.writeFloat32 = function(offset, value) {
-  flatbuffers.float32[0] = value;
-  this.writeInt32(offset, flatbuffers.int32[0]);
-};
-
-/**
- * @param {number} offset
- * @param {number} value
- */
-flatbuffers.ByteBuffer.prototype.writeFloat64 = function(offset, value) {
-  flatbuffers.float64[0] = value;
-  this.writeInt32(offset, flatbuffers.int32[flatbuffers.isLittleEndian ? 0 : 1]);
-  this.writeInt32(offset + 4, flatbuffers.int32[flatbuffers.isLittleEndian ? 1 : 0]);
-};
-
-/**
- * Look up a field in the vtable, return an offset into the object, or 0 if the
- * field is not present.
- *
- * @param {number} bb_pos
- * @param {number} vtable_offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.__offset = function(bb_pos, vtable_offset) {
-  var vtable = bb_pos - this.readInt32(bb_pos);
-  return vtable_offset < this.readInt16(vtable) ? this.readInt16(vtable + vtable_offset) : 0;
-};
-
-/**
- * Initialize any Table-derived type to point to the union at the given offset.
- *
- * @param {flatbuffers.Table} t
- * @param {number} offset
- * @returns {flatbuffers.Table}
- */
-flatbuffers.ByteBuffer.prototype.__union = function(t, offset) {
-  t.bb_pos = offset + this.readInt32(offset);
-  t.bb = this;
-  return t;
-};
-
-/**
- * Create a JavaScript string from UTF-8 data stored inside the FlatBuffer.
- * This allocates a new string and converts to wide chars upon each access.
- *
- * To avoid the conversion to UTF-16, pass flatbuffers.Encoding.UTF8_BYTES as
- * the "optionalEncoding" argument. This is useful for avoiding conversion to
- * and from UTF-16 when the data will just be packaged back up in another
- * FlatBuffer later on.
- *
- * @param {number} offset
- * @param {flatbuffers.Encoding=} opt_encoding Defaults to UTF16_STRING
- * @returns {string|Uint8Array}
- */
-flatbuffers.ByteBuffer.prototype.__string = function(offset, opt_encoding) {
-  offset += this.readInt32(offset);
-
-  var length = this.readInt32(offset);
-  var result = '';
-  var i = 0;
-
-  offset += flatbuffers.SIZEOF_INT;
-
-  if (opt_encoding === flatbuffers.Encoding.UTF8_BYTES) {
-    return this.bytes_.subarray(offset, offset + length);
-  }
-
-  while (i < length) {
-    var codePoint;
-
-    // Decode UTF-8
-    var a = this.readUint8(offset + i++);
-    if (a < 0xC0) {
-      codePoint = a;
-    } else {
-      var b = this.readUint8(offset + i++);
-      if (a < 0xE0) {
-        codePoint =
-          ((a & 0x1F) << 6) |
-          (b & 0x3F);
-      } else {
-        var c = this.readUint8(offset + i++);
-        if (a < 0xF0) {
-          codePoint =
-            ((a & 0x0F) << 12) |
-            ((b & 0x3F) << 6) |
-            (c & 0x3F);
-        } else {
-          var d = this.readUint8(offset + i++);
-          codePoint =
-            ((a & 0x07) << 18) |
-            ((b & 0x3F) << 12) |
-            ((c & 0x3F) << 6) |
-            (d & 0x3F);
-        }
-      }
-    }
-
-    // Encode UTF-16
-    if (codePoint < 0x10000) {
-      result += String.fromCharCode(codePoint);
-    } else {
-      codePoint -= 0x10000;
-      result += String.fromCharCode(
-        (codePoint >> 10) + 0xD800,
-        (codePoint & ((1 << 10) - 1)) + 0xDC00);
-    }
-  }
-
-  return result;
-};
-
-/**
- * Retrieve the relative offset stored at "offset"
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.__indirect = function(offset) {
-  return offset + this.readInt32(offset);
-};
-
-/**
- * Get the start of data of a vector whose offset is stored at "offset" in this object.
- *
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.__vector = function(offset) {
-  return offset + this.readInt32(offset) + flatbuffers.SIZEOF_INT; // data starts after the length
-};
-
-/**
- * Get the length of a vector whose offset is stored at "offset" in this object.
- *
- * @param {number} offset
- * @returns {number}
- */
-flatbuffers.ByteBuffer.prototype.__vector_len = function(offset) {
-  return this.readInt32(offset + this.readInt32(offset));
-};
-
-/**
- * @param {string} ident
- * @returns {boolean}
- */
-flatbuffers.ByteBuffer.prototype.__has_identifier = function(ident) {
-  if (ident.length != flatbuffers.FILE_IDENTIFIER_LENGTH) {
-    throw new Error('FlatBuffers: file identifier must be length ' +
-                    flatbuffers.FILE_IDENTIFIER_LENGTH);
-  }
-  for (var i = 0; i < flatbuffers.FILE_IDENTIFIER_LENGTH; i++) {
-    if (ident.charCodeAt(i) != this.readInt8(this.position_ + flatbuffers.SIZEOF_INT + i)) {
-      return false;
-    }
-  }
-  return true;
-};
-
-/**
- * A helper function to avoid generated code depending on this file directly.
- *
- * @param {number} low
- * @param {number} high
- * @returns {flatbuffers.Long}
- */
-flatbuffers.ByteBuffer.prototype.createLong = function(low, high) {
-  return flatbuffers.Long.create(low, high);
-};
-
-// Exports for Node.js and RequireJS
-exports.flatbuffers = flatbuffers;
-
-/// @endcond
-/// @}
diff --git a/js/closure-compiler-scripts/text-encoding.js b/js/closure-compiler-scripts/text-encoding.js
deleted file mode 100644
index 398883ab9b4be..0000000000000
--- a/js/closure-compiler-scripts/text-encoding.js
+++ /dev/null
@@ -1,648 +0,0 @@
-/**
- * closure-compiler-friendly text-encoding-utf-8
- * copied from node_modules/text-encoding-utf-8/lib/encoding.cjs.js
- * update as needed
- */
-
- // This is free and unencumbered software released into the public domain.
-// See LICENSE.md for more information.
-
-//
-// Utilities
-//
-
-goog.module("module$text_encoding_utf_8");
-goog.module.declareLegacyNamespace();
-/**
- * @param {number} a The number to test.
- * @param {number} min The minimum value in the range, inclusive.
- * @param {number} max The maximum value in the range, inclusive.
- * @return {boolean} True if a >= min and a <= max.
- */
-function inRange(a, min, max) {
-  return min <= a && a <= max;
-}
-
-/**
- * @param {*} o
- * @return {Object}
- */
-function ToDictionary(o) {
-  if (o === undefined) return {};
-  if (o === Object(o)) return o;
-  throw TypeError('Could not convert argument to dictionary');
-}
-
-/**
- * @param {string} string Input string of UTF-16 code units.
- * @return {!Array.<number>} Code points.
- */
-function stringToCodePoints(string) {
-  // https://heycam.github.io/webidl/#dfn-obtain-unicode
-
-  // 1. Let S be the DOMString value.
-  var s = String(string);
-
-  // 2. Let n be the length of S.
-  var n = s.length;
-
-  // 3. Initialize i to 0.
-  var i = 0;
-
-  // 4. Initialize U to be an empty sequence of Unicode characters.
-  var u = [];
-
-  // 5. While i < n:
-  while (i < n) {
-
-    // 1. Let c be the code unit in S at index i.
-    var c = s.charCodeAt(i);
-
-    // 2. Depending on the value of c:
-
-    // c < 0xD800 or c > 0xDFFF
-    if (c < 0xD800 || c > 0xDFFF) {
-      // Append to U the Unicode character with code point c.
-      u.push(c);
-    }
-
-    // 0xDC00 ≤ c ≤ 0xDFFF
-    else if (0xDC00 <= c && c <= 0xDFFF) {
-      // Append to U a U+FFFD REPLACEMENT CHARACTER.
-      u.push(0xFFFD);
-    }
-
-    // 0xD800 ≤ c ≤ 0xDBFF
-    else if (0xD800 <= c && c <= 0xDBFF) {
-      // 1. If i = n−1, then append to U a U+FFFD REPLACEMENT
-      // CHARACTER.
-      if (i === n - 1) {
-        u.push(0xFFFD);
-      }
-      // 2. Otherwise, i < n−1:
-      else {
-        // 1. Let d be the code unit in S at index i+1.
-        var d = string.charCodeAt(i + 1);
-
-        // 2. If 0xDC00 ≤ d ≤ 0xDFFF, then:
-        if (0xDC00 <= d && d <= 0xDFFF) {
-          // 1. Let a be c & 0x3FF.
-          var a = c & 0x3FF;
-
-          // 2. Let b be d & 0x3FF.
-          var b = d & 0x3FF;
-
-          // 3. Append to U the Unicode character with code point
-          // 2^16+2^10*a+b.
-          u.push(0x10000 + (a << 10) + b);
-
-          // 4. Set i to i+1.
-          i += 1;
-        }
-
-        // 3. Otherwise, d < 0xDC00 or d > 0xDFFF. Append to U a
-        // U+FFFD REPLACEMENT CHARACTER.
-        else  {
-          u.push(0xFFFD);
-        }
-      }
-    }
-
-    // 3. Set i to i+1.
-    i += 1;
-  }
-
-  // 6. Return U.
-  return u;
-}
-
-/**
- * @param {!Array.<number>} code_points Array of code points.
- * @return {string} string String of UTF-16 code units.
- */
-function codePointsToString(code_points) {
-  var s = '';
-  for (var i = 0; i < code_points.length; ++i) {
-    var cp = code_points[i];
-    if (cp <= 0xFFFF) {
-      s += String.fromCharCode(cp);
-    } else {
-      cp -= 0x10000;
-      s += String.fromCharCode((cp >> 10) + 0xD800,
-                               (cp & 0x3FF) + 0xDC00);
-    }
-  }
-  return s;
-}
-
-
-//
-// Implementation of Encoding specification
-// https://encoding.spec.whatwg.org/
-//
-
-//
-// 3. Terminology
-//
-
-/**
- * End-of-stream is a special token that signifies no more tokens
- * are in the stream.
- * @const
- */ var end_of_stream = -1;
-
-/**
- * A stream represents an ordered sequence of tokens.
- *
- * @constructor
- * @param {!(Array.<number>|Uint8Array)} tokens Array of tokens that provide the
- * stream.
- */
-function Stream(tokens) {
-  /** @type {!Array.<number>} */
-  this.tokens = [].slice.call(tokens);
-}
-
-Stream.prototype = {
-  /**
-   * @return {boolean} True if end-of-stream has been hit.
-   */
-  endOfStream: function() {
-    return !this.tokens.length;
-  },
-
-  /**
-   * When a token is read from a stream, the first token in the
-   * stream must be returned and subsequently removed, and
-   * end-of-stream must be returned otherwise.
-   *
-   * @return {number} Get the next token from the stream, or
-   * end_of_stream.
-   */
-   read: function() {
-    if (!this.tokens.length)
-      return end_of_stream;
-     return this.tokens.shift();
-   },
-
-  /**
-   * When one or more tokens are prepended to a stream, those tokens
-   * must be inserted, in given order, before the first token in the
-   * stream.
-   *
-   * @param {(number|!Array.<number>)} token The token(s) to prepend to the stream.
-   */
-  prepend: function(token) {
-    if (Array.isArray(token)) {
-      var tokens = /**@type {!Array.<number>}*/(token);
-      while (tokens.length)
-        this.tokens.unshift(tokens.pop());
-    } else {
-      this.tokens.unshift(token);
-    }
-  },
-
-  /**
-   * When one or more tokens are pushed to a stream, those tokens
-   * must be inserted, in given order, after the last token in the
-   * stream.
-   *
-   * @param {(number|!Array.<number>)} token The tokens(s) to prepend to the stream.
-   */
-  push: function(token) {
-    if (Array.isArray(token)) {
-      var tokens = /**@type {!Array.<number>}*/(token);
-      while (tokens.length)
-        this.tokens.push(tokens.shift());
-    } else {
-      this.tokens.push(token);
-    }
-  }
-};
-
-//
-// 4. Encodings
-//
-
-// 4.1 Encoders and decoders
-
-/** @const */
-var finished = -1;
-
-/**
- * @param {boolean} fatal If true, decoding errors raise an exception.
- * @param {number=} opt_code_point Override the standard fallback code point.
- * @return {number} The code point to insert on a decoding error.
- */
-function decoderError(fatal, opt_code_point) {
-  if (fatal)
-    throw TypeError('Decoder error');
-  return opt_code_point || 0xFFFD;
-}
-
-//
-// 7. API
-//
-
-/** @const */ var DEFAULT_ENCODING = 'utf-8';
-
-// 7.1 Interface TextDecoder
-
-/**
- * @constructor
- * @param {string=} encoding The label of the encoding;
- *     defaults to 'utf-8'.
- * @param {Object=} options
- */
-function TextDecoder(encoding, options) {
-  if (!(this instanceof TextDecoder)) {
-    return new TextDecoder(encoding, options);
-  }
-  encoding = encoding !== undefined ? String(encoding).toLowerCase() : DEFAULT_ENCODING;
-  if (encoding !== DEFAULT_ENCODING) {
-    throw new Error('Encoding not supported. Only utf-8 is supported');
-  }
-  options = ToDictionary(options);
-
-  /** @private @type {boolean} */
-  this._streaming = false;
-  /** @private @type {boolean} */
-  this._BOMseen = false;
-  /** @private @type {?Decoder} */
-  this._decoder = null;
-  /** @private @type {boolean} */
-  this._fatal = Boolean(options['fatal']);
-  /** @private @type {boolean} */
-  this._ignoreBOM = Boolean(options['ignoreBOM']);
-
-  Object.defineProperty(this, 'encoding', {value: 'utf-8'});
-  Object.defineProperty(this, 'fatal', {value: this._fatal});
-  Object.defineProperty(this, 'ignoreBOM', {value: this._ignoreBOM});
-}
-
-TextDecoder.prototype = {
-  /**
-   * @param {ArrayBufferView=} input The buffer of bytes to decode.
-   * @param {Object=} options
-   * @return {string} The decoded string.
-   */
-  decode: function decode(input, options) {
-    var bytes;
-    if (typeof input === 'object' && input instanceof ArrayBuffer) {
-      bytes = new Uint8Array(input);
-    } else if (typeof input === 'object' && 'buffer' in input &&
-               input.buffer instanceof ArrayBuffer) {
-      bytes = new Uint8Array(input.buffer,
-                             input.byteOffset,
-                             input.byteLength);
-    } else {
-      bytes = new Uint8Array(0);
-    }
-
-    options = ToDictionary(options);
-
-    if (!this._streaming) {
-      this._decoder = new UTF8Decoder({fatal: this._fatal});
-      this._BOMseen = false;
-    }
-    this._streaming = Boolean(options['stream']);
-
-    var input_stream = new Stream(bytes);
-
-    var code_points = [];
-
-    /** @type {?(number|!Array.<number>)} */
-    var result;
-
-    while (!input_stream.endOfStream()) {
-      result = this._decoder.handler(input_stream, input_stream.read());
-      if (result === finished)
-        break;
-      if (result === null)
-        continue;
-      if (Array.isArray(result))
-        code_points.push.apply(code_points, /**@type {!Array.<number>}*/(result));
-      else
-        code_points.push(result);
-    }
-    if (!this._streaming) {
-      do {
-        result = this._decoder.handler(input_stream, input_stream.read());
-        if (result === finished)
-          break;
-        if (result === null)
-          continue;
-        if (Array.isArray(result))
-          code_points.push.apply(code_points, /**@type {!Array.<number>}*/(result));
-        else
-          code_points.push(result);
-      } while (!input_stream.endOfStream());
-      this._decoder = null;
-    }
-
-    if (code_points.length) {
-      // If encoding is one of utf-8, utf-16be, and utf-16le, and
-      // ignore BOM flag and BOM seen flag are unset, run these
-      // subsubsteps:
-      if (['utf-8'].indexOf(this.encoding) !== -1 &&
-          !this._ignoreBOM && !this._BOMseen) {
-        // If token is U+FEFF, set BOM seen flag.
-        if (code_points[0] === 0xFEFF) {
-          this._BOMseen = true;
-          code_points.shift();
-        } else {
-          // Otherwise, if token is not end-of-stream, set BOM seen
-          // flag and append token to output.
-          this._BOMseen = true;
-        }
-      }
-    }
-
-    return codePointsToString(code_points);
-  }
-};
-
-// 7.2 Interface TextEncoder
-
-/**
- * @constructor
- * @param {string=} encoding The label of the encoding;
- *     defaults to 'utf-8'.
- * @param {Object=} options
- */
-function TextEncoder(encoding, options) {
-  if (!(this instanceof TextEncoder))
-    return new TextEncoder(encoding, options);
-  encoding = encoding !== undefined ? String(encoding).toLowerCase() : DEFAULT_ENCODING;
-  if (encoding !== DEFAULT_ENCODING) {
-    throw new Error('Encoding not supported. Only utf-8 is supported');
-  }
-  options = ToDictionary(options);
-
-  /** @private @type {boolean} */
-  this._streaming = false;
-  /** @private @type {?Encoder} */
-  this._encoder = null;
-  /** @private @type {{fatal: boolean}} */
-  this._options = {fatal: Boolean(options['fatal'])};
-
-  Object.defineProperty(this, 'encoding', {value: 'utf-8'});
-}
-
-TextEncoder.prototype = {
-  /**
-   * @param {string=} opt_string The string to encode.
-   * @param {Object=} options
-   * @return {Uint8Array} Encoded bytes, as a Uint8Array.
-   */
-  encode: function encode(opt_string, options) {
-    opt_string = opt_string ? String(opt_string) : '';
-    options = ToDictionary(options);
-
-    // NOTE: This option is nonstandard. None of the encodings
-    // permitted for encoding (i.e. UTF-8, UTF-16) are stateful,
-    // so streaming is not necessary.
-    if (!this._streaming)
-      this._encoder = new UTF8Encoder(this._options);
-    this._streaming = Boolean(options['stream']);
-
-    var bytes = [];
-    var input_stream = new Stream(stringToCodePoints(opt_string));
-    /** @type {?(number|!Array.<number>)} */
-    var result;
-    while (!input_stream.endOfStream()) {
-      result = this._encoder.handler(input_stream, input_stream.read());
-      if (result === finished)
-        break;
-      if (Array.isArray(result))
-        bytes.push.apply(bytes, /**@type {!Array.<number>}*/(result));
-      else
-        bytes.push(result);
-    }
-    if (!this._streaming) {
-      while (true) {
-        result = this._encoder.handler(input_stream, input_stream.read());
-        if (result === finished)
-          break;
-        if (Array.isArray(result))
-          bytes.push.apply(bytes, /**@type {!Array.<number>}*/(result));
-        else
-          bytes.push(result);
-      }
-      this._encoder = null;
-    }
-    return new Uint8Array(bytes);
-  }
-};
-
-//
-// 8. The encoding
-//
-
-// 8.1 utf-8
-
-/**
- * @constructor
- * @implements {Decoder}
- * @param {{fatal: boolean}} options
- */
-function UTF8Decoder(options) {
-  var fatal = options.fatal;
-
-  // utf-8's decoder's has an associated utf-8 code point, utf-8
-  // bytes seen, and utf-8 bytes needed (all initially 0), a utf-8
-  // lower boundary (initially 0x80), and a utf-8 upper boundary
-  // (initially 0xBF).
-  var /** @type {number} */ utf8_code_point = 0,
-      /** @type {number} */ utf8_bytes_seen = 0,
-      /** @type {number} */ utf8_bytes_needed = 0,
-      /** @type {number} */ utf8_lower_boundary = 0x80,
-      /** @type {number} */ utf8_upper_boundary = 0xBF;
-
-  /**
-   * @param {Stream} stream The stream of bytes being decoded.
-   * @param {number} bite The next byte read from the stream.
-   * @return {?(number|!Array.<number>)} The next code point(s)
-   *     decoded, or null if not enough data exists in the input
-   *     stream to decode a complete code point.
-   */
-  this.handler = function(stream, bite) {
-    // 1. If byte is end-of-stream and utf-8 bytes needed is not 0,
-    // set utf-8 bytes needed to 0 and return error.
-    if (bite === end_of_stream && utf8_bytes_needed !== 0) {
-      utf8_bytes_needed = 0;
-      return decoderError(fatal);
-    }
-
-    // 2. If byte is end-of-stream, return finished.
-    if (bite === end_of_stream)
-      return finished;
-
-    // 3. If utf-8 bytes needed is 0, based on byte:
-    if (utf8_bytes_needed === 0) {
-
-      // 0x00 to 0x7F
-      if (inRange(bite, 0x00, 0x7F)) {
-        // Return a code point whose value is byte.
-        return bite;
-      }
-
-      // 0xC2 to 0xDF
-      if (inRange(bite, 0xC2, 0xDF)) {
-        // Set utf-8 bytes needed to 1 and utf-8 code point to byte
-        // − 0xC0.
-        utf8_bytes_needed = 1;
-        utf8_code_point = bite - 0xC0;
-      }
-
-      // 0xE0 to 0xEF
-      else if (inRange(bite, 0xE0, 0xEF)) {
-        // 1. If byte is 0xE0, set utf-8 lower boundary to 0xA0.
-        if (bite === 0xE0)
-          utf8_lower_boundary = 0xA0;
-        // 2. If byte is 0xED, set utf-8 upper boundary to 0x9F.
-        if (bite === 0xED)
-          utf8_upper_boundary = 0x9F;
-        // 3. Set utf-8 bytes needed to 2 and utf-8 code point to
-        // byte − 0xE0.
-        utf8_bytes_needed = 2;
-        utf8_code_point = bite - 0xE0;
-      }
-
-      // 0xF0 to 0xF4
-      else if (inRange(bite, 0xF0, 0xF4)) {
-        // 1. If byte is 0xF0, set utf-8 lower boundary to 0x90.
-        if (bite === 0xF0)
-          utf8_lower_boundary = 0x90;
-        // 2. If byte is 0xF4, set utf-8 upper boundary to 0x8F.
-        if (bite === 0xF4)
-          utf8_upper_boundary = 0x8F;
-        // 3. Set utf-8 bytes needed to 3 and utf-8 code point to
-        // byte − 0xF0.
-        utf8_bytes_needed = 3;
-        utf8_code_point = bite - 0xF0;
-      }
-
-      // Otherwise
-      else {
-        // Return error.
-        return decoderError(fatal);
-      }
-
-      // Then (byte is in the range 0xC2 to 0xF4) set utf-8 code
-      // point to utf-8 code point << (6 × utf-8 bytes needed) and
-      // return continue.
-      utf8_code_point = utf8_code_point << (6 * utf8_bytes_needed);
-      return null;
-    }
-
-    // 4. If byte is not in the range utf-8 lower boundary to utf-8
-    // upper boundary, run these substeps:
-    if (!inRange(bite, utf8_lower_boundary, utf8_upper_boundary)) {
-
-      // 1. Set utf-8 code point, utf-8 bytes needed, and utf-8
-      // bytes seen to 0, set utf-8 lower boundary to 0x80, and set
-      // utf-8 upper boundary to 0xBF.
-      utf8_code_point = utf8_bytes_needed = utf8_bytes_seen = 0;
-      utf8_lower_boundary = 0x80;
-      utf8_upper_boundary = 0xBF;
-
-      // 2. Prepend byte to stream.
-      stream.prepend(bite);
-
-      // 3. Return error.
-      return decoderError(fatal);
-    }
-
-    // 5. Set utf-8 lower boundary to 0x80 and utf-8 upper boundary
-    // to 0xBF.
-    utf8_lower_boundary = 0x80;
-    utf8_upper_boundary = 0xBF;
-
-    // 6. Increase utf-8 bytes seen by one and set utf-8 code point
-    // to utf-8 code point + (byte − 0x80) << (6 × (utf-8 bytes
-    // needed − utf-8 bytes seen)).
-    utf8_bytes_seen += 1;
-    utf8_code_point += (bite - 0x80) << (6 * (utf8_bytes_needed - utf8_bytes_seen));
-
-    // 7. If utf-8 bytes seen is not equal to utf-8 bytes needed,
-    // continue.
-    if (utf8_bytes_seen !== utf8_bytes_needed)
-      return null;
-
-    // 8. Let code point be utf-8 code point.
-    var code_point = utf8_code_point;
-
-    // 9. Set utf-8 code point, utf-8 bytes needed, and utf-8 bytes
-    // seen to 0.
-    utf8_code_point = utf8_bytes_needed = utf8_bytes_seen = 0;
-
-    // 10. Return a code point whose value is code point.
-    return code_point;
-  };
-}
-
-/**
- * @constructor
- * @implements {Encoder}
- * @param {{fatal: boolean}} options
- */
-function UTF8Encoder(options) {
-  var fatal = options.fatal;
-  /**
-   * @param {Stream} stream Input stream.
-   * @param {number} code_point Next code point read from the stream.
-   * @return {(number|!Array.<number>)} Byte(s) to emit.
-   */
-  this.handler = function(stream, code_point) {
-    // 1. If code point is end-of-stream, return finished.
-    if (code_point === end_of_stream)
-      return finished;
-
-    // 2. If code point is in the range U+0000 to U+007F, return a
-    // byte whose value is code point.
-    if (inRange(code_point, 0x0000, 0x007f))
-      return code_point;
-
-    // 3. Set count and offset based on the range code point is in:
-    var count, offset;
-    // U+0080 to U+07FF:    1 and 0xC0
-    if (inRange(code_point, 0x0080, 0x07FF)) {
-      count = 1;
-      offset = 0xC0;
-    }
-    // U+0800 to U+FFFF:    2 and 0xE0
-    else if (inRange(code_point, 0x0800, 0xFFFF)) {
-      count = 2;
-      offset = 0xE0;
-    }
-    // U+10000 to U+10FFFF: 3 and 0xF0
-    else if (inRange(code_point, 0x10000, 0x10FFFF)) {
-      count = 3;
-      offset = 0xF0;
-    }
-
-    // 4.Let bytes be a byte sequence whose first byte is (code
-    // point >> (6 × count)) + offset.
-    var bytes = [(code_point >> (6 * count)) + offset];
-
-    // 5. Run these substeps while count is greater than 0:
-    while (count > 0) {
-
-      // 1. Set temp to code point >> (6 × (count − 1)).
-      var temp = code_point >> (6 * (count - 1));
-
-      // 2. Append to bytes 0x80 | (temp & 0x3F).
-      bytes.push(0x80 | (temp & 0x3F));
-
-      // 3. Decrease count by one.
-      count -= 1;
-    }
-
-    // 6. Return bytes bytes, in order.
-    return bytes;
-  };
-}
-
-exports.TextEncoder = TextEncoder;
-exports.TextDecoder = TextDecoder;
diff --git a/js/closure-compiler-scripts/tslib.js b/js/closure-compiler-scripts/tslib.js
deleted file mode 100644
index b5a722a652c34..0000000000000
--- a/js/closure-compiler-scripts/tslib.js
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * closure-compiler-friendly tslib
- * copied from node_modules/tslib/tslib.js
- * update as needed
- */
-
-var extendStatics = Object.setPrototypeOf ||
-    ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
-    function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; };
-
-function __extends(d, b) {
-    extendStatics(d, b);
-    function __() { this.constructor = d; }
-    d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
-};
-
-var __assign = Object.assign || function (t) {
-    for (var s, i = 1, n = arguments.length; i < n; i++) {
-        s = arguments[i];
-        for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];
-    }
-    return t;
-};
-
-function __rest(s, e) {
-    var t = {};
-    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
-        t[p] = s[p];
-    if (s != null && typeof Object.getOwnPropertySymbols === "function")
-        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) if (e.indexOf(p[i]) < 0)
-            t[p[i]] = s[p[i]];
-    return t;
-};
-
-function __decorate(decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-
-function __param(paramIndex, decorator) {
-    return function (target, key) { decorator(target, key, paramIndex); }
-};
-
-function __metadata(metadataKey, metadataValue) {
-    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(metadataKey, metadataValue);
-};
-
-function __awaiter(thisArg, _arguments, P, generator) {
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-
-function __generator(thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (_) try {
-            if (f = 1, y && (t = y[op[0] & 2 ? "return" : op[0] ? "throw" : "next"]) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [0, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-
-function __exportStar(m, exports) {
-    for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p];
-};
-
-function __values(o) {
-    var m = typeof Symbol === "function" && o[Symbol.iterator], i = 0;
-    if (m) return m.call(o);
-    return {
-        next: function () {
-            if (o && i >= o.length) o = void 0;
-            return { value: o && o[i++], done: !o };
-        }
-    };
-};
-
-function __read(o, n) {
-    var m = typeof Symbol === "function" && o[Symbol.iterator];
-    if (!m) return o;
-    var i = m.call(o), r, ar = [], e;
-    try {
-        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
-    }
-    catch (error) { e = { error: error }; }
-    finally {
-        try {
-            if (r && !r.done && (m = i["return"])) m.call(i);
-        }
-        finally { if (e) throw e.error; }
-    }
-    return ar;
-};
-
-function __spread() {
-    for (var ar = [], i = 0; i < arguments.length; i++)
-        ar = ar.concat(__read(arguments[i]));
-    return ar;
-};
-
-function __await(v) {
-    return this instanceof __await ? (this.v = v, this) : new __await(v);
-};
-
-function __asyncGenerator(thisArg, _arguments, generator) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var g = generator.apply(thisArg, _arguments || []), i, q = [];
-    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
-    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
-    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
-    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r);  }
-    function fulfill(value) { resume("next", value); }
-    function reject(value) { resume("throw", value); }
-    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
-};
-
-function __asyncDelegator(o) {
-    var i, p;
-    return i = {}, verb("next"), verb("throw", function (e) { throw e; }), verb("return"), i[Symbol.iterator] = function () { return this; }, i;
-    function verb(n, f) { if (o[n]) i[n] = function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === "return" } : f ? f(v) : v; }; }
-};
-
-function __asyncValues(o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator];
-    return m ? m.call(o) : typeof __values === "function" ? __values(o) : o[Symbol.iterator]();
-};
diff --git a/js/gulp/argv.js b/js/gulp/argv.js
index 33553704eec25..253553c733c8a 100644
--- a/js/gulp/argv.js
+++ b/js/gulp/argv.js
@@ -22,9 +22,12 @@ const argv = require(`command-line-args`)([
     { name: `target`, type: String, defaultValue: `` },
     { name: `module`, type: String, defaultValue: `` },
     { name: `coverage`, type: Boolean, defaultValue: false },
+    { name: `integration`, alias: `i`, type: Boolean, defaultValue: false },
     { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
-    { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }
-]);
+    { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] },
+    { name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] },
+    { name: `formats`, alias: `f`, type: String, multiple: true, defaultValue: [`file`, `stream`] },
+], { partial: true });
 
 const { targets, modules } = argv;
 
diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js
index d160ecb0e5de4..cc33ee14497b6 100644
--- a/js/gulp/arrow-task.js
+++ b/js/gulp/arrow-task.js
@@ -27,7 +27,8 @@ const { memoizeTask } = require('./memoize-task');
 const { Observable, ReplaySubject } = require('rxjs');
 
 const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, format) {
-    const out = targetDir(target), srcGlob = `src/**/*`;
+    const out = targetDir(target);
+    const srcGlob = `src/**/*.ts`;
     const es5Glob = `${targetDir(`es5`, `cjs`)}/**/*.js`;
     const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`;
     const es5UmdGlob = `${targetDir(`es5`, `umd`)}/**/*.js`;
@@ -48,7 +49,7 @@ const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, forma
 }))({});
 
 const arrowTSTask = ((cache) => memoizeTask(cache, function copyTS(target, format) {
-    return observableFromStreams(gulp.src(`src/**/*`), gulp.dest(targetDir(target, format)));
+    return observableFromStreams(gulp.src(`src/**/*.ts`), gulp.dest(targetDir(target, format)));
 }))({});
   
   
diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js
index 950bf40e22a39..a1f0a9a698843 100644
--- a/js/gulp/closure-task.js
+++ b/js/gulp/closure-task.js
@@ -37,10 +37,14 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
     const externs = path.join(src, `${mainExport}.externs`);
     return observableFromStreams(
         gulp.src([
-/*   external libs first --> */ `closure-compiler-scripts/*.js`,
-/*    then sources glob --> */ `${src}/**/*.js`,
-/* and exclusions last --> */ `!${src}/format/*.js`,
-                              `!${src}/Arrow.externs.js`,
+/*   external libs first --> */ `node_modules/tslib/package.json`,
+                                `node_modules/tslib/tslib.es6.js`,
+                                `node_modules/flatbuffers/package.json`,
+                                `node_modules/flatbuffers/js/flatbuffers.mjs`,
+                                `node_modules/text-encoding-utf-8/package.json`,
+                                `node_modules/text-encoding-utf-8/src/encoding.js`,
+/*    then sources globs --> */ `${src}/**/*.js`,
+/* and exclusions last -->  */ `!${src}/Arrow.externs.js`,
         ], { base: `./` }),
         sourcemaps.init(),
         closureCompiler(createClosureArgs(entry, externs)),
@@ -53,12 +57,14 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
 const createClosureArgs = (entry, externs) => ({
     third_party: true,
     warning_level: `QUIET`,
-    dependency_mode: `LOOSE`,
+    dependency_mode: `STRICT`,
     rewrite_polyfills: false,
     externs: `${externs}.js`,
     entry_point: `${entry}.js`,
+    module_resolution: `NODE`,
     // formatting: `PRETTY_PRINT`,
     compilation_level: `ADVANCED`,
+    package_json_entry_names: `module,jsnext:main,main`,
     assume_function_wrapper: true,
     js_output_file: `${mainExport}.js`,
     language_in: gCCLanguageNames[`es2015`],
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
index b46b2bb14edcb..ae6b8071d2ab7 100644
--- a/js/gulp/test-task.js
+++ b/js/gulp/test-task.js
@@ -15,10 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
+const del = require('del');
 const path = require('path');
-const child_process = require(`child_process`);
 const { argv } = require('./argv');
+const { promisify } = require('util');
+const glob = promisify(require('glob'));
+const stat = promisify(require('fs').stat);
+const mkdirp = promisify(require('mkdirp'));
+const rimraf = promisify(require('rimraf'));
+const child_process = require(`child_process`);
 const { memoizeTask } = require('./memoize-task');
+const readFile = promisify(require('fs').readFile);
+const exec = promisify(require('child_process').exec);
+const parseXML = promisify(require('xml2js').parseString);
 
 const jestArgv = [];
 argv.update && jestArgv.push(`-u`);
@@ -29,16 +38,129 @@ const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`];
 const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`));
 
 const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) {
-    const opts = Object.assign({}, testOptions);
+    const opts = { ...testOptions };
     const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv];
-    opts.env = Object.assign({}, opts.env, { TEST_TARGET: target, TEST_MODULE: format });
+    if (!argv.integration) {
+        args.push('test/vector-tests.ts');
+    }
+    opts.env = { ...opts.env,
+        TEST_TARGET: target,
+        TEST_MODULE: format,
+        TEST_TS_SOURCE: !!argv.coverage,
+        TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]),
+        TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]),
+    };
     return !debug ?
         child_process.spawn(jest, args, opts) :
         child_process.exec(`node --inspect-brk ${jest} ${args.join(` `)}`, opts);
 }))({}, jestArgv, {
-    env: Object.assign({}, process.env),
+    env: { ...process.env },
     stdio: [`ignore`, `inherit`, `inherit`],
 });
 
 module.exports = testTask;
 module.exports.testTask = testTask;
+module.exports.cleanTestData = cleanTestData;
+module.exports.createTestData = createTestData;
+
+// Pull C++ and Java paths from environment vars first, otherwise sane defaults
+const ARROW_HOME = process.env.ARROW_HOME || path.resolve('../');
+const ARROW_JAVA_DIR = process.env.ARROW_JAVA_DIR || path.join(ARROW_HOME, 'java');
+const CPP_EXE_PATH = process.env.ARROW_CPP_EXE_PATH || path.join(ARROW_HOME, 'cpp/build/debug');
+const ARROW_INTEGRATION_DIR = process.env.ARROW_INTEGRATION_DIR || path.join(ARROW_HOME, 'integration');
+const CPP_JSON_TO_ARROW = path.join(CPP_EXE_PATH, 'json-integration-test');
+const CPP_STREAM_TO_FILE = path.join(CPP_EXE_PATH, 'stream-to-file');
+const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'file-to-stream');
+
+const testFilesDir = path.join(ARROW_HOME, 'js/test/data');
+const cppFilesDir = path.join(testFilesDir, 'cpp');
+const javaFilesDir = path.join(testFilesDir, 'java');
+const jsonFilesDir = path.join(testFilesDir, 'json');
+
+async function cleanTestData() {
+    return await del(`${testFilesDir}/**`);
+}
+
+async function createTestJSON() {
+    await mkdirp(jsonFilesDir);
+    await exec(`shx cp ${ARROW_INTEGRATION_DIR}/data/*.json ${jsonFilesDir}`);
+    await exec(`python ${ARROW_INTEGRATION_DIR}/integration_test.py --write_generated_json ${jsonFilesDir}`);
+}
+
+async function createTestData() {
+
+    let JAVA_TOOLS_JAR = process.env.ARROW_JAVA_INTEGRATION_JAR;
+    if (!JAVA_TOOLS_JAR) {
+        const pom_version = await
+            readFile(path.join(ARROW_JAVA_DIR, 'pom.xml'))
+                .then((pom) => parseXML(pom.toString()))
+                .then((pomXML) => pomXML.project.version[0]);
+        JAVA_TOOLS_JAR = path.join(ARROW_JAVA_DIR, `/tools/target/arrow-tools-${pom_version}-jar-with-dependencies.jar`);
+    }
+
+    await cleanTestData().then(createTestJSON);
+    await mkdirp(path.join(cppFilesDir, 'file'));
+    await mkdirp(path.join(javaFilesDir, 'file'));
+    await mkdirp(path.join(cppFilesDir, 'stream'));
+    await mkdirp(path.join(javaFilesDir, 'stream'));
+
+    const errors = [];
+    const names = await glob(path.join(jsonFilesDir, '*.json'));
+
+    for (let jsonPath of names) {
+        const name = path.parse(path.basename(jsonPath)).name;
+        const arrowCppFilePath = path.join(cppFilesDir, 'file', `${name}.arrow`);
+        const arrowJavaFilePath = path.join(javaFilesDir, 'file', `${name}.arrow`);
+        const arrowCppStreamPath = path.join(cppFilesDir, 'stream', `${name}.arrow`);
+        const arrowJavaStreamPath = path.join(javaFilesDir, 'stream', `${name}.arrow`);
+        try {
+            await generateCPPFile(path.resolve(jsonPath), arrowCppFilePath);
+            await generateCPPStream(arrowCppFilePath, arrowCppStreamPath);
+        } catch (e) { errors.push(`${e.stdout}\n${e.message}`); }
+        try {
+            await generateJavaFile(path.resolve(jsonPath), arrowJavaFilePath);
+            await generateJavaStream(arrowJavaFilePath, arrowJavaStreamPath);
+        } catch (e) { errors.push(`${e.stdout}\n${e.message}`); }
+    }
+    if (errors.length) {
+        console.error(errors.join(`\n`));
+        process.exit(1);
+    }
+
+    async function generateCPPFile(jsonPath, filePath) {
+        await rimraf(filePath);
+        return await exec(
+            `${CPP_JSON_TO_ARROW} ${
+            `--integration --mode=JSON_TO_ARROW`} ${
+            `--json=${jsonPath} --arrow=${filePath}`}`,
+            { maxBuffer: Math.pow(2, 53) - 1 }
+        );
+    }
+    
+    async function generateCPPStream(filePath, streamPath) {
+        await rimraf(streamPath);
+        return await exec(
+            `${CPP_FILE_TO_STREAM} ${filePath} > ${streamPath}`,
+            { maxBuffer: Math.pow(2, 53) - 1 }
+        );
+    }
+    
+    async function generateJavaFile(jsonPath, filePath) {
+        await rimraf(filePath);
+        return await exec(
+            `java -cp ${JAVA_TOOLS_JAR} ${
+            `org.apache.arrow.tools.Integration -c JSON_TO_ARROW`} ${
+            `-j ${path.resolve(jsonPath)} -a ${filePath}`}`,
+            { maxBuffer: Math.pow(2, 53) - 1 }
+        );
+    }
+    
+    async function generateJavaStream(filePath, streamPath) {
+        await rimraf(streamPath);
+        return await exec(
+            `java -cp ${JAVA_TOOLS_JAR} ${
+            `org.apache.arrow.tools.FileToStream`} ${filePath} ${streamPath}`,
+            { maxBuffer: Math.pow(2, 53) - 1 }
+        );
+    }
+}
diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js
index 2c66846667cd3..2fd9f1350a6f1 100644
--- a/js/gulp/typescript-task.js
+++ b/js/gulp/typescript-task.js
@@ -19,9 +19,11 @@ const {
     targetDir, tsconfigName, observableFromStreams
 } = require('./util');
 
+const del = require('del');
 const gulp = require('gulp');
 const path = require('path');
 const ts = require(`gulp-typescript`);
+const gulpRename = require(`gulp-rename`);
 const sourcemaps = require('gulp-sourcemaps');
 const { memoizeTask } = require('./memoize-task');
 const { Observable, ReplaySubject } = require('rxjs');
@@ -36,8 +38,26 @@ const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target
     );
     const writeDTypes = observableFromStreams(dts, gulp.dest(out));
     const writeJS = observableFromStreams(js, sourcemaps.write(), gulp.dest(out));
-    return Observable.forkJoin(writeDTypes, writeJS).publish(new ReplaySubject()).refCount();
+    return Observable
+        .forkJoin(writeDTypes, writeJS)
+        .concat(maybeCopyRawJSArrowFormatFiles(target, format))
+        .publish(new ReplaySubject()).refCount();
 }))({});
-  
+
 module.exports = typescriptTask;
-module.exports.typescriptTask = typescriptTask;
\ No newline at end of file
+module.exports.typescriptTask = typescriptTask;
+
+function maybeCopyRawJSArrowFormatFiles(target, format) {
+    if (target !== `es5` || format !== `cls`) {
+        return Observable.empty();
+    }
+    return Observable.defer(async () => {
+        const outFormatDir = path.join(targetDir(target, format), `format`);
+        await del(path.join(outFormatDir, '*.js'));
+        await observableFromStreams(
+            gulp.src(path.join(`src`, `format`, `*_generated.js`)),
+            gulpRename((p) => { p.basename = p.basename.replace(`_generated`, ``); }),
+            gulp.dest(outFormatDir)
+        ).toPromise();
+    });
+}
\ No newline at end of file
diff --git a/js/gulp/uglify-task.js b/js/gulp/uglify-task.js
index 804d450453644..5c605cb7882bd 100644
--- a/js/gulp/uglify-task.js
+++ b/js/gulp/uglify-task.js
@@ -39,7 +39,7 @@ const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJ
 
     const targetConfig = { ...commonConfig,
         output: { ...commonConfig.output,
-             path: path.resolve(`./${out}`) } };
+            path: path.resolve(`./${out}`) } };
 
     const webpackConfigs = [
         [mainExport, PublicNames]
diff --git a/js/gulp/util.js b/js/gulp/util.js
index 21ffc3127339c..e23fc398562d4 100644
--- a/js/gulp/util.js
+++ b/js/gulp/util.js
@@ -27,7 +27,7 @@ const releasesRootDir = `targets`;
 const knownTargets = [`es5`, `es2015`, `esnext`];
 const knownModules = [`cjs`, `esm`, `cls`, `umd`];
 const moduleFormatsToSkipCombosOf = { cls: true };
-const metadataFiles = [`LICENSE`, `README.md`];
+const metadataFiles = [`../LICENSE.txt`, `../NOTICE.txt`, `README.md`];
 const packageJSONFields = [
   `version`, `license`, `description`,
   `author`, `homepage`, `repository`,
diff --git a/js/gulpfile.js b/js/gulpfile.js
index 4cf0342c3be78..cd0ac70029aba 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -19,11 +19,11 @@ const del = require('del');
 const gulp = require('gulp');
 const path = require('path');
 const { Observable } = require('rxjs');
-const testsTask = require('./gulp/test-task');
 const buildTask = require('./gulp/build-task');
 const cleanTask = require('./gulp/clean-task');
 const packageTask = require('./gulp/package-task');
 const { targets, modules } = require('./gulp/argv');
+const { testTask, createTestData, cleanTestData } = require('./gulp/test-task');
 const {
     targetDir,
     taskName, combinations,
@@ -35,8 +35,8 @@ const {
 for (const [target, format] of combinations([`all`], [`all`])) {
     const task = taskName(target, format);
     gulp.task(`clean:${task}`, cleanTask(target, format));
-    gulp.task( `test:${task}`, testsTask(target, format));
-    gulp.task(`debug:${task}`, testsTask(target, format, true));
+    gulp.task( `test:${task}`,  testTask(target, format));
+    gulp.task(`debug:${task}`,  testTask(target, format, true));
     gulp.task(`build:${task}`, gulp.series(`clean:${task}`,
                                             buildTask(target, format),
                                             packageTask(target, format)));
@@ -50,7 +50,7 @@ knownTargets.forEach((target) =>
         gulp.series(
             gulp.parallel(
                 cleanTask(target, `umd`),
-                cleanTask(UMDSourceTargets[target], `cls`),
+                cleanTask(UMDSourceTargets[target], `cls`)
             ),
             buildTask(UMDSourceTargets[target], `cls`),
             buildTask(target, `umd`), packageTask(target, `umd`)
@@ -86,6 +86,8 @@ const buildConcurrent = (tasks) => () =>
             .merge(...knownTargets.map((target) =>
                 del(`${targetDir(target, `cls`)}/**`)))));
   
+gulp.task(`test:cleanTestData`, cleanTestData);
+gulp.task(`test:createTestData`, createTestData);
 gulp.task( `test`, gulp.series(getTasks(`test`)));
 gulp.task(`debug`, gulp.series(getTasks(`debug`)));
 gulp.task(`clean`, gulp.parallel(getTasks(`clean`)));
diff --git a/js/lerna.json b/js/lerna.json
index c8fb8c072c61e..0bf16fdfd57be 100644
--- a/js/lerna.json
+++ b/js/lerna.json
@@ -2,8 +2,10 @@
   "lerna": "2.0.0",
   "version": "0.1.1",
   "packages": [
+    "targets/ts",
     "targets/es5/*",
     "targets/es2015/*",
-    "targets/esnext/*"
+    "targets/esnext/*",
+    "targets/apache-arrow"
   ]
 }
diff --git a/js/prepublish.sh b/js/npm-release.sh
similarity index 85%
rename from js/prepublish.sh
rename to js/npm-release.sh
index b40504ae808cb..0bf70d9798538 100644
--- a/js/prepublish.sh
+++ b/js/npm-release.sh
@@ -17,10 +17,4 @@
 # specific language governing permissions and limitations
 # under the License.
 
-npm run clean
-npm run lint
-npm run build
-npm run test
-npm --no-git-tag-version version patch &>/dev/null
-npm run bundle
-npm run lerna:publish
\ No newline at end of file
+lerna publish --yes --skip-git --force-publish=*
\ No newline at end of file
diff --git a/js/package.json b/js/package.json
index ba93a34686288..24bc27f5b6325 100644
--- a/js/package.json
+++ b/js/package.json
@@ -15,17 +15,17 @@
     "build": "gulp build",
     "clean": "gulp clean",
     "debug": "gulp debug",
-    "bundle": "gulp bundle",
-    "package": "gulp package",
     "perf": "node ./perf/index.js",
-    "test:coverage": "gulp test -t esnext -m esm --coverage",
-    "validate": "npm-run-all clean lint build test bundle",
-    "lerna:publish": "lerna exec --bail=false npm publish",
-    "prepublishOnly": "sh ./prepublish.sh",
+    "release": "./npm-release.sh",
+    "validate": "run-s --silent lint build clean",
+    "test:coverage": "gulp test -t ts --coverage",
+    "test:cleanTestData": "gulp test:cleanTestData",
+    "test:createTestData": "gulp test:createTestData",
     "doc": "shx rm -rf ./doc && esdoc",
     "lint": "npm-run-all -p lint:*",
     "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
-    "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\""
+    "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"",
+    "prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1"
   },
   "repository": {
     "type": "git",
@@ -53,14 +53,15 @@
     "command-line-usage": "4.0.1"
   },
   "dependencies": {
-    "flatbuffers": "1.7.0",
+    "flatbuffers": "trxcllnt/flatbuffers-esm",
     "text-encoding": "0.6.4"
   },
   "devDependencies": {
-    "@std/esm": "0.12.5",
+    "@std/esm": "0.13.0",
     "@types/flatbuffers": "1.6.5",
-    "@types/jest": "21.1.5",
-    "@types/node": "8.0.47",
+    "@types/glob": "5.0.33",
+    "@types/jest": "21.1.6",
+    "@types/node": "8.0.51",
     "@types/text-encoding": "0.0.32",
     "benchmark": "2.1.4",
     "command-line-args": "4.0.7",
@@ -68,20 +69,22 @@
     "del": "3.0.0",
     "esdoc": "1.0.3",
     "esdoc-standard-plugin": "1.0.0",
-    "google-closure-compiler": "20170910.0.0",
+    "glob": "7.1.2",
+    "google-closure-compiler": "20171112.0.0",
     "gulp": "github:gulpjs/gulp#4.0",
     "gulp-json-transform": "0.4.5",
     "gulp-rename": "1.2.2",
     "gulp-sourcemaps": "2.6.1",
     "gulp-typescript": "3.2.3",
+    "ix": "2.3.1",
     "jest": "21.2.1",
     "jest-environment-node-debug": "2.0.0",
     "json": "9.0.6",
-    "lerna": "2.5.0",
-    "lint-staged": "4.3.0",
+    "lerna": "2.5.1",
+    "lint-staged": "5.0.0",
     "merge2": "1.2.0",
     "mkdirp": "0.5.1",
-    "npm-run-all": "4.1.1",
+    "npm-run-all": "4.1.2",
     "pump": "1.0.2",
     "rimraf": "2.6.2",
     "rxjs": "5.5.2",
@@ -89,12 +92,13 @@
     "source-map-loader": "0.2.3",
     "text-encoding-utf-8": "1.0.1",
     "trash": "4.1.0",
-    "ts-jest": "21.1.4",
+    "ts-jest": "21.2.1",
     "tslib": "1.8.0",
     "tslint": "5.8.0",
     "typescript": "2.6.1",
     "uglifyjs-webpack-plugin": "1.0.1",
-    "webpack": "3.8.1"
+    "webpack": "3.8.1",
+    "xml2js": "0.4.19"
   },
   "lint-staged": {
     "*.@(ts)": [
diff --git a/js/perf/arrows/file/dictionary.arrow b/js/perf/arrows/file/dictionary.arrow
deleted file mode 100644
index 34d41db1f200159406c459b8c076d5ce0c3d30ec..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2522
zcmeHJ&1(};5T9+*G{mHqHnv(ta_B)s!H*ykC?w(`f?#L`@#bQlX&@hEla^FG<l;rV
z_&@kZc+N?|lLzTZaumda;Mw^5y?s;GjipqtoiK0S%)I%2yPL|!#)C)KhchAzB9fJP
zNl8x9c8^77tX#Z|l_ep`07*UISzsP(9&`!Tf>YFyCl*NI8<1N7r49A1Uz3Rk-wEvR
zLio|iYm>=D?S_ycE00v|-a0$whwl6poGAiD5*{f-vL43ThfM(<#?BTZKnj_r@FRGZ
zuGRg%zeb+Aq?OUn8Q|rH>9$L?ZnM)b^@3)7y?FD+Ey%;#mt^rTwKFFovhl8Q1!_($
z<jtJ=pzH(TKK3A90m}-|Z<x}(PTw?4P-=GSX0ZOuZ}kjfUO>lJWEtL)h|5{AXHOih
zg1!KX%z6W?>@oA;tk{pQh?7R_wCw;VMkrXGRg>48IPE+nwBy|A<L8$TrndV0F}Bb!
z{+zV|>(O=$|I45qzimpJ{h--txcvLUj$sb`iJStOf#2S4nX6BGyYdM<>mX!N<D#4g
z&kD7VR)xjgNo>+M#q1dO&)hQLnOoB4t^Hvy6ry{T@m@tC>`gegsGP9~nlJaXo%T6L
zm4jK(9FSI3KM3}!CTKUi9kXe6O|aA3f(!MS#eC;+_re+Dyz_BTePX)8eVhsJQg~Ri
zKQP7SunOaH-c&-=A9<}je!f5@ARhBlD4vSTCxQ_7b2}8<C&g`reT(ODT%I>B3OXsJ
zo^z=DIUK$SRQ|LxuKaj7dh@+-=jFG-NTghDp@73?R(3W=%`Do3c>m}kswb~!;&Po?
zTz=Hd;=TFTnQ30kku!_uaa=w=WBOGeN+}~Kud`5I=O^;6vby(vr24GAVc{EsZx~`3
zxCLa~*UpghB~(KqWflh5cJi|2!iAz&F6qxsUX1AZ4rPz@ZAwZT&gbMSdh!)@&(D8-
MMaQ^*75ks=H#cC^v;Y7A

diff --git a/js/perf/arrows/file/simple.arrow b/js/perf/arrows/file/simple.arrow
deleted file mode 100644
index 838db6dc8eda5376aaaf977c5f96c50397582a5f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1642
zcmeHIJxc>Y5S{x-E`~!=1UW$=g@vVpg@xdyvk-(J*x1}DwgF44l$Ms3{sezQ3QK>C
zrQP}7+>FLVB_!H8VfN$A?7W@X-AjKkI62#uh2=aD>0os#tJ0K)v~-Ln2o)D;D;5K4
zL)k~rXXvHPPphiJg2plCThPpK`t2A-4oa*);8@d=0I^%(73C1=$r12CLggrn8+oi_
zM>%{Ly9?zWLYtGTmZnBxEL!idE=T<PPw}p?K2yI088dUnqcUON|0D<ZzoB;SF@+{j
zm7s_274dtPM~qBp^Zj29hq8zI*6~^6HLG!|>=K1qD@_LLSm6wo7*Sa3Cn)RIfVw0Y
zfD-loklcoj``mlSN9H$;kH5i@h)QmDkCPidn!?9Z{xtIr`2NlOKIa)y-0pK?n{1X4
z=ORDvi*w8ya(!OMAN^hqzczRB8t+;^>-8J_tfwpUUzw%(Go&cWvb=yS$iRD_;^pu~
prFaE=t<2^C?KYHW$}@hjgWmT8pS(n{|9`)h`+nV`u6_NFc>}#ma&iCw

diff --git a/js/perf/arrows/file/struct.arrow b/js/perf/arrows/file/struct.arrow
deleted file mode 100644
index 3d2c018e6c27c95057f2a18395a9dc6031e6b201..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2354
zcmeHJJ5L)y5MCPt7wZBk2y#J?B~nBw5RVv4cSZyvVqWKjR}<Ng3!qal*iI-Sg_1T>
z6s4k~Oa4I`{{(&lB_(a7NY3}|?Kp^s{6L5%V|}|jbMx4po7-{9b?24_Rns&dQtAP$
zZPlULRI9S|8bc_o<w`}hECNv!;0zNz7b}$txch&=#{v40=O<W)flmN^9z*vi&~HtH
ze+<L`>bC#`0A=(+is+wGdDvKpXyMleIejtO0puY>U<_|asV;b<MmtVIFI^Z39%KR|
zeUoQA+B^Yz0Fnh83zmUz6^cW43jcA$Jv$!<dFDPMh(le*@5Edv?v~byZ{C-DKb!O4
zsu&U!#AaT*O<rxiGu0B-=Pomenxf!s__TC>e?rC_v<DCM$a7;G$n`&R%u?jS%PjJJ
zqjynWzDABiY1%i>tA32!F>+3PZ?aFWNwxU?K&|QVQPY-z;pc@f8Ls-qN37n%QY`%I
zxY&rxo9<s?F;5-9Lj%^L)FYO+Ml&1k?B2w);mNTV<9jc&i|fgxmml`GS9e{no}fK-
z)VBV>w6m*goA-#&UDS5j1ee;DZ_$xWVP-wcK6z)5gufTCRbFbJ;&0~}qu0pfG@{&q
z?#j*&Ubi*rclBPXsd}vfYg-^S&f4w+o&s!FH)vU~zCB8165A>F`SjF!_TAd@Xfl)D
zbU%K`O?jhpD~ZxZ!h2O1*z^ltawxr(8Jo$yzOWVE-8OE2b_g3cIlB=%&N4;^kpA=4
j<eA?SNop`}iY+ehfz9d?{qM=u-d|@={*HX6^nazl^uV^z

diff --git a/js/perf/arrows/multi/count/records.arrow b/js/perf/arrows/multi/count/records.arrow
deleted file mode 100644
index 00d883762d36959727cd26f5c3e3e4dfafeba981..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 224
zcmbQkz`!5^#2~=KAjZJPz{<b@WbuFmn3)(PfRqD>4F+65J`V#6Pz4u|?*qgFK+Fi1
ggAhzW>OT-53Bj4@;tp^bh#AP{z@!A!VJz}!0H3Q0ZvX%Q

diff --git a/js/perf/arrows/multi/count/schema.arrow b/js/perf/arrows/multi/count/schema.arrow
deleted file mode 100644
index dfd24e9e0018c471a053b8621cb7ea8b46a5bd9e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 184
zcmXYqF%E<<3<Ng_?hHcc3W_TzD0l!5;SUt_AR!Rn;(6DFadeJ$?QAs8E$M@5HF~L#
zr5OuZ%B3o0IZ=hY2lPcPZcuM%-_oWG1D^2Ve|IKBBkGWs!5F0fM};2y#Ta|G8z*P4
RzXiyWzHisOZ0C9~dp%Qi3m5<Z

diff --git a/js/perf/arrows/multi/latlong/records.arrow b/js/perf/arrows/multi/latlong/records.arrow
deleted file mode 100644
index 563d12d175d4e39b10386f98c5614cb86a493424..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 352
zcmcb^z`!5^#2~=KAjZJPz{<b@WbuFmn3)(PfK&sB4F+65J`V#6Pz4u|UjxJfK+Fo3
zgAgo0>OT-53Bj4@;to(5m^h3N(||6HEYAc~glsQJ4>}(t);5XP35J<m-#CIWNSs(O
U+4%4w7;gV?<PZoGs~4mW0F=Zhr~m)}

diff --git a/js/perf/arrows/multi/latlong/schema.arrow b/js/perf/arrows/multi/latlong/schema.arrow
deleted file mode 100644
index 638b2ab622f8ed55ff85783a2c1ad54321b90021..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 264
zcmZ8cI|{-;6r7jDH3Sh0ixep=EImMu;4Q>T0wDqM7?xI^BFD2^XWnihKA86z-kaHl
z0XqO2Y!R?R?LCtf&I65e6{&_$Er>DqON^1rWzV@I_L<j?W@#(lX%A1<0}L-#qmne|
z`*0hk%Q#QEZ|N7X=8JSKJwYC4<|FazapK$(DT3by9}J)DRg<l#slWYuULWc53;*jL
ADgXcg

diff --git a/js/perf/arrows/multi/origins/records.arrow b/js/perf/arrows/multi/origins/records.arrow
deleted file mode 100644
index 49a8c407e176e107a9007cbed26b87e3c84608ed..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 224
zcmb7+Q3`+{5JV@-2odTdqT_e~kJw%7t)WPN4UGGCX7Q4=V^h?jSzZeiEf}3nMlvo6
l>`@aJfj#35`_jFSW8A2V8h-QC8fk$0e`;lU^$MH+=mDH!3Sj^M

diff --git a/js/perf/arrows/multi/origins/schema.arrow b/js/perf/arrows/multi/origins/schema.arrow
deleted file mode 100644
index 0d10fb0e2d1354e5a147f49b5283632d4fdde04e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1604
zcmeIxJ!lj`6bJCRGapA1Bua|d$Pxq#Q7lBQLUI>l5-+(k7firH-tE4-9d35!oY{?D
z7p!cw6C1I!5^U^j#6|==vD3yTSSZ+OBj<lMIh25nwR8M-zINWcnN7Y4A*R5W32|7A
zh+$F6+jta;V$x5Dv3zVOZwcY+0!+h{IF7jyF^bib=<SVRLKKGY;<*KWYzfYRk2Q%E
zgIIoVAIp~#Vw4cc?~hUl@Hhk);RNjE;`jGIAwKu}{SsD}@EbvYZ(Wf0hcM#%1yK~^
zsEw6vsc(cb?TI(&|3357{+TDRJ4&&b&+YgPP)gz`4CS%>IR?4<b4+7}pBck=1U|nx
z7!;p7QtS`<f7S=b=Zbr?gYiLsQ2SZG_ndDWD5L26^BjW!PQrPZg%!98*MUJn7kY39
z?!iNN3{T-Xyn?sz9zMb+*o80f9e%<vk{O3-n1NGp7B0XnT!uPqK?@XE=)n%$hX?Qo
zp1?DB0WaY-yn_$03t!;}^kEG5bOfeh22R8O^6xiLqmCjS)$oi>LO})pAhm>Is_B%h
zG;7ATjOuK%OM^P8Bq%e5aoyO|FRRB=QN*enORC3)wY*)IY3iDpZO3(Mwq?XR<aJUk
z+wnSUvzVR<R(ot+gLJSUxov!7UZ+M&Q#F}%+fpfRFr_If%q?2dp>$#X;)-z?RB2G9
z4(G`>D72I?0^sG85Q-X@_K`Ly>C#4~93G)FdR{fNn;Ba`lXP!B!4Z`*^O;Mf-jq6W
z#$s*Qab;-}soOCnYvkfQr?Qcb%UP>M%9JyU0G%>Yk%_TfQM}FmMk+>SNi%a5W4oCP
zDzTM_y+BHlTVTD-c7aoCWO_;AK3#qvRmvi^5iY6GH67BtWRQ^2OJo&tU|orEG$h(v
z){!JlU8AI1c1m_v2)P=)U*&05$pzQAO|EJjZlleLT{UYqajGi5jpAS(%~cusbgz(h
ict8K@)(@CUEJJFWRa|<EkIr_DZv{207;paN&+Hd!R}{7Y

diff --git a/js/perf/arrows/stream/dictionary.arrow b/js/perf/arrows/stream/dictionary.arrow
deleted file mode 100644
index 17ca48b3a97f51ad016faa004cc53a2a5e809eec..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1776
zcmbVLL2DCH5T0$)G{mHqHnv(t_Rxcff@l#56jJdJ!9$>4yt$ZX2qYWXY)dL0a`6ZF
zAN&!Xds6V=u_x(K5D$W9*YA7#rmU}Osk6+RH}l@id^7WQHYc(qB6V4miZrBZ_gG}k
z+9jx2TN#!bm@)ug1UIoZAy?onxWzN-<d71vL5(Fy`tW@CGs^<RE@A%w#*fb4&9Y4W
zX0QQOK&gIjouBG6cmD?NY=T8f0VPwi9_BiPPlyOd&KD+71xPFSNP(3by(o<~sPn9B
zZH#jPe7$eRqgHo3h||_28T7U`@7%r%dtUpBEdQZ)Vj=@gaEmKgF%5y6m{Lggf%FJ_
zh#<tW29Els^)OCN-z2R;+%way=h1Ltkn;*Uz9y@PRz_aViamSkWC!vUSY$q!Vr7qs
zgR^2kz93H(xvRDVoERyzI;*Cxm^kewEcD~t8RN(252m~R@(H#$Vg3c{12v%U5#p~x
z#?i>McG6@J_Z|K;*)_z#PvjifOrz0@p}F~NvL~OQvkp=nH7?6#=&VrtXpgYCH;GLZ
zw}_5;f5%pXPHbh1Tl>RaC>8f=<GU(S-kW@GMLTnm6fgJm^<ysZz|lNp15BSzlqCBd
zlZ*!A*zA}+lk8502%z!mnC>E8UOr*mcQKA@FP*IK5O?yIlwVft$FA7?tv=t*cS<V8
zqpo%P_d_0n{G8Wv^NGCtA4q<jd%4;ECvGe3n~&qHI^Ua5IxCfd3#$4hI9vs*e%=Mx
z{{Hyj&A0IW>#x%Tk#_rEb89=F)8<y}gWp4jD4tW-JHB1#=G#x3o8O=Rotxq!M$XO0
faaP@*GvjIum9&vm*MCTLouA0N?e+cln3>2g$Le<8

diff --git a/js/perf/arrows/stream/simple.arrow b/js/perf/arrows/stream/simple.arrow
deleted file mode 100644
index 2c68c0e44b0af2ed78a47edbd038ae8fa8ab574b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1188
zcmcIiJ5Iw;5M2La6IsYbgd7PZ7aV|so>x*(asiu&HWDROj*yaLaD)_;9EOt4%&d3$
zIDjGrHd;TwnSC>FH(o>{m57vbC=<z~(04qdSaFdlIFS_0)VLHYBSAhR7f52)ec$6?
zc7^v-WNkR>&J3d$Bh?>poGB!NdI5h?f00HmfkzUnN7vrV<Di`c)KGhdq-Sh<dJQpA
zC2Emf<6MCL!xz0foUgSm*X}CSqb3{o`+xS}{g0LB9ZO`6)QB1^uh4I-j})Dd^ZVZ}
zmvW9d5Bi+-k<GePcS&NMl{SNOoOun#XZ{3)bIV9s<XB;ed4EK(SI2$ttK%!H8{?ZF
za5SP3jCY-2_-cf2M)jom4fy?ye&6#yWH|48Vkdm;5$;8QelPB^tK@vw!e8rNF21L$
fFyJ#=x2yibpY!x&{|DPP|Aw?}*L63DJsjQuh?Gj^

diff --git a/js/perf/arrows/stream/struct.arrow b/js/perf/arrows/stream/struct.arrow
deleted file mode 100644
index 4e97b7084f6b7c62df23acfe9e200661946314e3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1884
zcmd5+O;1`u6us|>_(~{cQ4I?<B)Di?G*n8(9U7~~qNY$AbYnn$)#jys0P4b|F1zif
zOIPmnAGqNs@DsZ1vfCz2@SJ%w)HE8vrqN5z%-l2gW9D+-6fKdch=e5~LFpAotua`(
z;zdH(6A54^r1t`tDR2cZQf)Sy&>sKBIu0-%dG3D)Y~UDR>;`PleEZ%U^bsHi(7p#q
z0^}Hj5>P*pUHCYN=-?Lw&sfa%1-yj}%;AoSJVi8G^r=%vp_8Ln_B2Murp|oyc?Jvu
z6bC*IJpI^W#lihR{5bOdyq@O;Z5_l2DqH4%0^3S;zqVdo`&9G1QrX*-7z*I$W_<^A
zeZ%Tx(qpT8*I6VbA?RVm^y_>9jy34>H`T~{;~1#*AIa*a;f-!)!=GrozxnztJg%(d
zMDMraXg;fXPW<1@Ik~2^)1UihcOHlC?CCpxe*3k+RbuLuGkjW$*_VxKHzEJ^{3aLc
z6ao4**o&qgvHWkcu$5mtntqv@nV6e8dQ*D8k;%BbDQ{<WKks&ubg+ZoRv(xOc5`p@
zJuz}0y<OMIO>diTF>4Nmnf+|e$v1;y^hdth(oOG^{&ro_cHK<bZj8IIUD-S5bK6#a
zwce$+rneffw+&2>v$qq#D1c{F0n>w%$wGQ(GXHvhcBAxRy*QpJ<hJvN2jy9Jd}$?J
Z+e*9dD#>lH;%3Hj9}5!;<>ec0a0N6*h?f8W

diff --git a/js/perf/config.js b/js/perf/config.js
index 4fbcda3799587..cca1080154790 100644
--- a/js/perf/config.js
+++ b/js/perf/config.js
@@ -17,22 +17,14 @@
 
 const fs = require('fs');
 const path = require('path');
-const arrowFormats = ['file', 'stream'];
-const arrowFileNames = ['simple', 'struct', 'dictionary'];
-const multipartArrows = ['count', 'latlong', 'origins'];
-let arrowTestConfigurations = [];
+const glob = require('glob');
 
-arrowTestConfigurations = multipartArrows.reduce((configs, folder) => {
-    const schemaPath = path.resolve(__dirname, `./arrows/multi/${folder}/schema.arrow`);
-    const recordsPath = path.resolve(__dirname, `./arrows/multi/${folder}/records.arrow`);
-    return [...configs, [`multipart ${folder}`, fs.readFileSync(schemaPath), fs.readFileSync(recordsPath)]];
-}, arrowTestConfigurations);
+const config = [];
+const filenames = glob.sync(path.resolve(__dirname, `../test/data/cpp/stream`, `*.arrow`));
 
-arrowTestConfigurations = arrowFormats.reduce((configs, format) => {
-    return arrowFileNames.reduce((configs, name) => {
-        const arrowPath = path.resolve(__dirname, `./arrows/${format}/${name}.arrow`);
-        return [...configs, [`${name} ${format}`, fs.readFileSync(arrowPath)]];
-    }, configs);
-}, arrowTestConfigurations);
+for (const filename of filenames) {
+    const { name } = path.parse(filename);
+    config.push({ name, buffers: [fs.readFileSync(filename)] });
+}
 
-module.exports = arrowTestConfigurations;
+module.exports = config;
diff --git a/js/perf/index.js b/js/perf/index.js
index 3a2ed96772330..9eac40e64ac71 100644
--- a/js/perf/index.js
+++ b/js/perf/index.js
@@ -16,24 +16,24 @@
 // under the License.
 
 // Use the ES5 UMD target as perf baseline
-// const { Table, readBuffers } = require('../targets/es5/umd');
-// const { Table, readBuffers } = require('../targets/es5/cjs');
-const { Table, readBuffers } = require('../targets/es2015/umd');
-// const { Table, readBuffers } = require('../targets/es2015/cjs');
+// const { Table, readVectors } = require('../targets/es5/umd');
+// const { Table, readVectors } = require('../targets/es5/cjs');
+const { Table, readVectors } = require('../targets/es2015/umd');
+// const { Table, readVectors } = require('../targets/es2015/cjs');
 
+const config = require('./config');
 const Benchmark = require('benchmark');
-const arrowTestConfigurations = require('./config');
 
 const suites = [];
 
-for (let [name, ...buffers] of arrowTestConfigurations) {
+for (let { name, buffers} of config) {
     const parseSuite = new Benchmark.Suite(`Parse ${name}`, { async: true });
     const sliceSuite = new Benchmark.Suite(`Slice ${name} vectors`, { async: true });
     const iterateSuite = new Benchmark.Suite(`Iterate ${name} vectors`, { async: true });
     const getByIndexSuite = new Benchmark.Suite(`Get ${name} values by index`, { async: true });
     parseSuite.add(createFromTableTest(name, buffers));
-    parseSuite.add(createReadBuffersTest(name, buffers));
-    for (const vector of Table.from(...buffers).columns) {
+    parseSuite.add(createReadVectorsTest(name, buffers));
+    for (const vector of Table.from(buffers).columns) {
         sliceSuite.add(createSliceTest(vector));
         iterateSuite.add(createIterateTest(vector));
         getByIndexSuite.add(createGetByIndexTest(vector));
@@ -66,16 +66,16 @@ function createFromTableTest(name, buffers) {
     return {
         async: true,
         name: `Table.from`,
-        fn() { table = Table.from(...buffers); }
+        fn() { table = Table.from(buffers); }
     };
 }
 
-function createReadBuffersTest(name, buffers) {
+function createReadVectorsTest(name, buffers) {
     let vectors;
     return {
         async: true,
-        name: `readBuffers`,
-        fn() { for (vectors of readBuffers(...buffers)) {} }
+        name: `readVectors`,
+        fn() { for (vectors of readVectors(buffers)) {} }
     };
 }
 
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index 3196550884dbf..e6527b34e10dd 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -15,45 +15,64 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { readBuffers } from './reader/arrow';
+import { Table } from './vector/table';
+import { Vector } from './vector/vector';
+import { Utf8Vector } from './vector/utf8';
+import { DictionaryVector } from './vector/dictionary';
+import { StructVector, StructRow } from './vector/struct';
+import { readVectors, readVectorsAsync } from './reader/arrow';
+import { ListVector, BinaryVector, FixedSizeListVector } from './vector/list';
 
-import { Vector } from './types/types';
-import { ListVector } from './types/list';
-import { Utf8Vector } from './types/utf8';
-import { BoolVector } from './types/vector/bool';
-import { DateVector } from './types/vector/date';
-import { RowVector } from './types/table/row';
-import { TableVector } from './types/table/table';
-import { StructVector } from './types/table/struct';
-import { DictionaryVector } from './types/dictionary';
-import { FixedSizeListVector } from './types/fixedsizelist';
-import { LongVector, Int64Vector, Uint64Vector, } from './types/vector/long';
 import {
-    TypedVector,
+    BoolVector,
     Int8Vector,
     Int16Vector,
     Int32Vector,
+    Int64Vector,
     Uint8Vector,
     Uint16Vector,
     Uint32Vector,
+    Uint64Vector,
+    Float16Vector,
     Float32Vector,
-    Float64Vector
-} from './types/vector/typed';
-
-import './types/table/from';
+    Float64Vector,
+    Date32Vector,
+    Date64Vector,
+    Time32Vector,
+    Time64Vector,
+    DecimalVector,
+    TimestampVector,
+} from './vector/numeric';
 
+export { Table, Vector, StructRow };
+export { readVectors, readVectorsAsync };
+export { NumericVectorConstructor } from './vector/numeric';
+export { List, TypedArray, TypedArrayConstructor } from './vector/types';
 export {
-    Vector,
-    readBuffers,
+    BoolVector,
+    ListVector,
+    Utf8Vector,
+    Int8Vector,
+    Int16Vector,
+    Int32Vector,
+    Int64Vector,
+    Uint8Vector,
+    Uint16Vector,
+    Uint32Vector,
+    Uint64Vector,
+    Date32Vector,
+    Date64Vector,
+    Time32Vector,
+    Time64Vector,
+    BinaryVector,
+    StructVector,
+    Float16Vector,
+    Float32Vector,
+    Float64Vector,
+    DecimalVector,
+    TimestampVector,
     DictionaryVector,
-    RowVector as Row,
-    TableVector as Table,
-    StructVector, Utf8Vector,
-    ListVector, FixedSizeListVector,
-    BoolVector, TypedVector, LongVector,
-    DateVector, Float32Vector, Float64Vector,
-    Int8Vector, Int16Vector, Int32Vector, Int64Vector,
-    Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector,
+    FixedSizeListVector,
 };
 
 /* These exports are needed for the closure umd targets */
@@ -61,18 +80,14 @@ try {
     const Arrow = eval('exports');
     if (typeof Arrow === 'object') {
         // string indexers tell closure compiler not to rename these properties
+        Arrow['readVectors'] = readVectors;
+        Arrow['readVectorsAsync'] = readVectorsAsync;
+        Arrow['Table'] = Table;
         Arrow['Vector'] = Vector;
-        Arrow['Table'] = TableVector;
-        Arrow['readBuffers'] = readBuffers;
+        Arrow['StructRow'] = StructRow;
         Arrow['BoolVector'] = BoolVector;
-        Arrow['Utf8Vector'] = Utf8Vector;
         Arrow['ListVector'] = ListVector;
-        Arrow['StructVector'] = StructVector;
-        Arrow['DictionaryVector'] = DictionaryVector;
-        Arrow['FixedSizeListVector'] = FixedSizeListVector;
-        Arrow['LongVector'] = LongVector;
-        Arrow['TypedVector'] = TypedVector;
-        Arrow['DateVector'] = DateVector;
+        Arrow['Utf8Vector'] = Utf8Vector;
         Arrow['Int8Vector'] = Int8Vector;
         Arrow['Int16Vector'] = Int16Vector;
         Arrow['Int32Vector'] = Int32Vector;
@@ -81,8 +96,19 @@ try {
         Arrow['Uint16Vector'] = Uint16Vector;
         Arrow['Uint32Vector'] = Uint32Vector;
         Arrow['Uint64Vector'] = Uint64Vector;
+        Arrow['Date32Vector'] = Date32Vector;
+        Arrow['Date64Vector'] = Date64Vector;
+        Arrow['Time32Vector'] = Time32Vector;
+        Arrow['Time64Vector'] = Time64Vector;
+        Arrow['BinaryVector'] = BinaryVector;
+        Arrow['StructVector'] = StructVector;
+        Arrow['Float16Vector'] = Float16Vector;
         Arrow['Float32Vector'] = Float32Vector;
         Arrow['Float64Vector'] = Float64Vector;
+        Arrow['DecimalVector'] = DecimalVector;
+        Arrow['TimestampVector'] = TimestampVector;
+        Arrow['DictionaryVector'] = DictionaryVector;
+        Arrow['FixedSizeListVector'] = FixedSizeListVector;
     }
 } catch (e) { /* not the UMD bundle */ }
 /* end closure exports */
diff --git a/js/src/format/File_generated.ts b/js/src/format/File.ts
similarity index 99%
rename from js/src/format/File_generated.ts
rename to js/src/format/File.ts
index d0b774ae34095..56f50ed20e936 100644
--- a/js/src/format/File_generated.ts
+++ b/js/src/format/File.ts
@@ -1,7 +1,7 @@
 // automatically generated by the FlatBuffers compiler, do not modify
 
 import { flatbuffers } from 'flatbuffers';
-import * as NS16187549871986683199 from './Schema_generated';
+import * as NS16187549871986683199 from './Schema';
 /**
  * ----------------------------------------------------------------------
  * Arrow File metadata
diff --git a/js/closure-compiler-scripts/File_generated.js b/js/src/format/File_generated.js
similarity index 95%
rename from js/closure-compiler-scripts/File_generated.js
rename to js/src/format/File_generated.js
index bb82cc4ccc6e4..12aae293ea4eb 100644
--- a/js/closure-compiler-scripts/File_generated.js
+++ b/js/src/format/File_generated.js
@@ -1,12 +1,5 @@
+import { org } from './Schema';
 // automatically generated by the FlatBuffers compiler, do not modify
-goog.module("module$targets$es5$cls$format$File_generated");
-goog.module.declareLegacyNamespace();
-var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
-/**
- * @const
- * @namespace
- */
-var org = Schema_.org;
 
 /**
  * @const
@@ -259,6 +252,5 @@ org.apache.arrow.flatbuf.Block.createBlock = function(builder, offset, metaDataL
   builder.writeInt64(offset);
   return builder.offset();
 };
+export { org };
 
-// Exports for Node.js and RequireJS
-exports.org = org;
diff --git a/js/src/format/Message_generated.ts b/js/src/format/Message.ts
similarity index 94%
rename from js/src/format/Message_generated.ts
rename to js/src/format/Message.ts
index daa781f9b9290..4610fbef2e1c8 100644
--- a/js/src/format/Message_generated.ts
+++ b/js/src/format/Message.ts
@@ -1,7 +1,7 @@
 // automatically generated by the FlatBuffers compiler, do not modify
 
 import { flatbuffers } from 'flatbuffers';
-import * as NS16187549871986683199 from './Schema_generated';
+import * as NS16187549871986683199 from './Schema';
 export namespace org.apache.arrow.flatbuf {
   export import Schema = NS16187549871986683199.org.apache.arrow.flatbuf.Schema;
 }
@@ -181,7 +181,7 @@ export namespace org.apache.arrow.flatbuf {
      */
     buffers(index: number, obj?: NS16187549871986683199.org.apache.arrow.flatbuf.Buffer): NS16187549871986683199.org.apache.arrow.flatbuf.Buffer | null {
       let offset = this.bb.__offset(this.bb_pos, 8);
-      return offset ? (obj || new NS16187549871986683199.org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
+      return offset ? (obj || new NS16187549871986683199.org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 16, this.bb) : null;
     }
 
     /**
@@ -236,7 +236,7 @@ export namespace org.apache.arrow.flatbuf {
      * @param {number} numElems
      */
     static startBuffersVector(builder: flatbuffers.Builder, numElems: number) {
-      builder.startVector(24, numElems, 8);
+      builder.startVector(16, numElems, 8);
     }
 
     /**
@@ -251,12 +251,12 @@ export namespace org.apache.arrow.flatbuf {
   }
 }
 /**
- * ----------------------------------------------------------------------
  * For sending dictionary encoding information. Any Field can be
  * dictionary-encoded, but in this case none of its children may be
  * dictionary-encoded.
- * There is one vector / column per dictionary
- *
+ * There is one vector / column per dictionary, but that vector / column
+ * may be spread across multiple dictionary batches by using the isDelta
+ * flag
  *
  * @constructor
  */
@@ -308,11 +308,22 @@ export namespace org.apache.arrow.flatbuf {
       return offset ? (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
     }
 
+    /**
+     * If isDelta is true the values in the dictionary are to be appended to a
+     * dictionary with the indicated id
+     *
+     * @returns {boolean}
+     */
+    isDelta(): boolean {
+      let offset = this.bb.__offset(this.bb_pos, 8);
+      return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+    }
+
     /**
      * @param {flatbuffers.Builder} builder
      */
     static startDictionaryBatch(builder: flatbuffers.Builder) {
-      builder.startObject(2);
+      builder.startObject(3);
     }
 
     /**
@@ -331,6 +342,14 @@ export namespace org.apache.arrow.flatbuf {
       builder.addFieldOffset(1, dataOffset, 0);
     }
 
+    /**
+     * @param {flatbuffers.Builder} builder
+     * @param {boolean} isDelta
+     */
+    static addIsDelta(builder: flatbuffers.Builder, isDelta: boolean) {
+      builder.addFieldInt8(2, +isDelta, +false);
+    }
+
     /**
      * @param {flatbuffers.Builder} builder
      * @returns {flatbuffers.Offset}
diff --git a/js/closure-compiler-scripts/Message_generated.js b/js/src/format/Message_generated.js
similarity index 93%
rename from js/closure-compiler-scripts/Message_generated.js
rename to js/src/format/Message_generated.js
index 0c1a1a99d74ca..ef46c98057c9f 100644
--- a/js/closure-compiler-scripts/Message_generated.js
+++ b/js/src/format/Message_generated.js
@@ -1,12 +1,5 @@
+import { org } from './Schema';
 // automatically generated by the FlatBuffers compiler, do not modify
-goog.module("module$targets$es5$cls$format$Message_generated");
-goog.module.declareLegacyNamespace();
-var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
-/**
- * @const
- * @namespace
- */
-var org = Schema_.org;
 
 /**
  * @const
@@ -200,7 +193,7 @@ org.apache.arrow.flatbuf.RecordBatch.prototype.nodesLength = function() {
  */
 org.apache.arrow.flatbuf.RecordBatch.prototype.buffers = function(index, obj) {
   var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? (obj || new org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
+  return offset ? (obj || new org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 16, this.bb) : null;
 };
 
 /**
@@ -255,7 +248,7 @@ org.apache.arrow.flatbuf.RecordBatch.addBuffers = function(builder, buffersOffse
  * @param {number} numElems
  */
 org.apache.arrow.flatbuf.RecordBatch.startBuffersVector = function(builder, numElems) {
-  builder.startVector(24, numElems, 8);
+  builder.startVector(16, numElems, 8);
 };
 
 /**
@@ -268,12 +261,12 @@ org.apache.arrow.flatbuf.RecordBatch.endRecordBatch = function(builder) {
 };
 
 /**
- * ----------------------------------------------------------------------
  * For sending dictionary encoding information. Any Field can be
  * dictionary-encoded, but in this case none of its children may be
  * dictionary-encoded.
- * There is one vector / column per dictionary
- *
+ * There is one vector / column per dictionary, but that vector / column
+ * may be spread across multiple dictionary batches by using the isDelta
+ * flag
  *
  * @constructor
  */
@@ -326,11 +319,22 @@ org.apache.arrow.flatbuf.DictionaryBatch.prototype.data = function(obj) {
   return offset ? (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
 };
 
+/**
+ * If isDelta is true the values in the dictionary are to be appended to a
+ * dictionary with the indicated id
+ *
+ * @returns {boolean}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.prototype.isDelta = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
+};
+
 /**
  * @param {flatbuffers.Builder} builder
  */
 org.apache.arrow.flatbuf.DictionaryBatch.startDictionaryBatch = function(builder) {
-  builder.startObject(2);
+  builder.startObject(3);
 };
 
 /**
@@ -349,6 +353,14 @@ org.apache.arrow.flatbuf.DictionaryBatch.addData = function(builder, dataOffset)
   builder.addFieldOffset(1, dataOffset, 0);
 };
 
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {boolean} isDelta
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.addIsDelta = function(builder, isDelta) {
+  builder.addFieldInt8(2, +isDelta, +false);
+};
+
 /**
  * @param {flatbuffers.Builder} builder
  * @returns {flatbuffers.Offset}
@@ -481,6 +493,5 @@ org.apache.arrow.flatbuf.Message.endMessage = function(builder) {
 org.apache.arrow.flatbuf.Message.finishMessageBuffer = function(builder, offset) {
   builder.finish(offset);
 };
+export { org };
 
-// Exports for Node.js and RequireJS
-exports.org = org;
diff --git a/js/src/format/Schema_generated.ts b/js/src/format/Schema.ts
similarity index 99%
rename from js/src/format/Schema_generated.ts
rename to js/src/format/Schema.ts
index c5b3e5011d790..d9b45ed20089c 100644
--- a/js/src/format/Schema_generated.ts
+++ b/js/src/format/Schema.ts
@@ -7,9 +7,25 @@ import { flatbuffers } from 'flatbuffers';
  */
 export namespace org.apache.arrow.flatbuf {
   export enum MetadataVersion {
+    /**
+     * 0.1.0
+     */
     V1 = 0,
+
+    /**
+     * 0.2.0
+     */
     V2 = 1,
-    V3 = 2
+
+    /**
+     * 0.3.0 -> 0.7.1
+     */
+    V3 = 2,
+
+    /**
+     * >= 0.8.0
+     */
+    V4 = 3
   }
 }
 
@@ -2049,7 +2065,6 @@ export namespace org.apache.arrow.flatbuf {
 
     /**
      * @param {flatbuffers.Builder} builder
-     * @param {number} page
      * @param {flatbuffers.Long} offset
      * @param {flatbuffers.Long} length
      * @returns {flatbuffers.Offset}
diff --git a/js/closure-compiler-scripts/Schema_generated.js b/js/src/format/Schema_generated.js
similarity index 98%
rename from js/closure-compiler-scripts/Schema_generated.js
rename to js/src/format/Schema_generated.js
index 5b76443886543..f89cf2fa8fa0a 100644
--- a/js/closure-compiler-scripts/Schema_generated.js
+++ b/js/src/format/Schema_generated.js
@@ -1,6 +1,4 @@
 // automatically generated by the FlatBuffers compiler, do not modify
-goog.module("module$targets$es5$cls$format$Schema_generated");
-goog.module.declareLegacyNamespace();
 
 /**
  * @const
@@ -30,9 +28,25 @@ org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
  * @enum
  */
 org.apache.arrow.flatbuf.MetadataVersion = {
+  /**
+   * 0.1.0
+   */
   V1: 0, 0: 'V1',
+
+  /**
+   * 0.2.0
+   */
   V2: 1, 1: 'V2',
+
+  /**
+   * 0.3.0 -> 0.7.1
+   */
   V3: 2, 2: 'V3',
+
+  /**
+   * >= 0.8.0
+   */
+  V4: 3, 3: 'V4'
 };
 
 /**
@@ -103,7 +117,7 @@ org.apache.arrow.flatbuf.Type = {
   Union: 14, 14: 'Union',
   FixedSizeBinary: 15, 15: 'FixedSizeBinary',
   FixedSizeList: 16, 16: 'FixedSizeList',
-  Map: 17, 17: 'Map',
+  Map: 17, 17: 'Map'
 };
 
 /**
@@ -131,7 +145,7 @@ org.apache.arrow.flatbuf.VectorType = {
   /**
    * Type vector used in Union type
    */
-  TYPE: 3, 3: 'TYPE',
+  TYPE: 3, 3: 'TYPE'
 };
 
 /**
@@ -2005,16 +2019,6 @@ org.apache.arrow.flatbuf.Buffer.prototype.__init = function(i, bb) {
   return this;
 };
 
-/**
- * The shared memory page id where this buffer is located. Currently this is
- * not used
- *
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Buffer.prototype.page = function() {
-  return this.bb.readInt32(this.bb_pos);
-};
-
 /**
  * The relative offset into the shared memory page where the bytes for this
  * buffer starts
@@ -2022,7 +2026,7 @@ org.apache.arrow.flatbuf.Buffer.prototype.page = function() {
  * @returns {flatbuffers.Long}
  */
 org.apache.arrow.flatbuf.Buffer.prototype.offset = function() {
-  return this.bb.readInt64(this.bb_pos + 8);
+  return this.bb.readInt64(this.bb_pos);
 };
 
 /**
@@ -2032,22 +2036,19 @@ org.apache.arrow.flatbuf.Buffer.prototype.offset = function() {
  * @returns {flatbuffers.Long}
  */
 org.apache.arrow.flatbuf.Buffer.prototype.length = function() {
-  return this.bb.readInt64(this.bb_pos + 16);
+  return this.bb.readInt64(this.bb_pos + 8);
 };
 
 /**
  * @param {flatbuffers.Builder} builder
- * @param {number} page
  * @param {flatbuffers.Long} offset
  * @param {flatbuffers.Long} length
  * @returns {flatbuffers.Offset}
  */
-org.apache.arrow.flatbuf.Buffer.createBuffer = function(builder, page, offset, length) {
-  builder.prep(8, 24);
+org.apache.arrow.flatbuf.Buffer.createBuffer = function(builder, offset, length) {
+  builder.prep(8, 16);
   builder.writeInt64(length);
   builder.writeInt64(offset);
-  builder.pad(4);
-  builder.writeInt32(page);
   return builder.offset();
 };
 
@@ -2226,6 +2227,5 @@ org.apache.arrow.flatbuf.Schema.endSchema = function(builder) {
 org.apache.arrow.flatbuf.Schema.finishSchemaBuffer = function(builder, offset) {
   builder.finish(offset);
 };
+export { org };
 
-// Exports for Node.js and RequireJS
-exports.org = org;
diff --git a/js/src/reader/arrow.ts b/js/src/reader/arrow.ts
index 033bfecae61dd..8d3aafc722160 100644
--- a/js/src/reader/arrow.ts
+++ b/js/src/reader/arrow.ts
@@ -15,66 +15,143 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { Vector } from '../vector/vector';
 import { flatbuffers } from 'flatbuffers';
-import * as Schema_ from '../format/Schema_generated';
-import * as Message_ from '../format/Message_generated';
-export import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
-export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
+import { readVector, readValueVector } from './vector';
+import {
+    readFileFooter, readFileMessages,
+    readStreamSchema, readStreamMessages
+} from './format';
 
-import { readFile } from './file';
-import { readStream } from './stream';
-import { readVector } from './vector';
-import { readDictionary } from './dictionary';
-import { Vector, Column } from '../types/types';
+import * as File_ from '../format/File';
+import * as Schema_ from '../format/Schema';
+import * as Message_ from '../format/Message';
 
 import ByteBuffer = flatbuffers.ByteBuffer;
+import Footer = File_.org.apache.arrow.flatbuf.Footer;
 import Field = Schema_.org.apache.arrow.flatbuf.Field;
-export type Dictionaries = { [k: string]: Vector<any> } | null;
-export type IteratorState = { nodeIndex: number; bufferIndex: number };
-
-export function* readRecords(...bytes: ByteBuffer[]) {
-    try {
-        yield* readFile(...bytes);
-    } catch (e) {
-        try {
-            yield* readStream(...bytes);
-        } catch (e) {
-            throw new Error('Invalid Arrow buffer');
-        }
+import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+import Message = Message_.org.apache.arrow.flatbuf.Message;
+import ArrowBuffer = Schema_.org.apache.arrow.flatbuf.Buffer;
+import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
+import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
+import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
+import DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch;
+import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
+
+export type ArrowReaderContext = {
+    schema?: Schema;
+    footer?: Footer | null;
+    dictionaries: Map<string, Vector>;
+    dictionaryEncodedFields: Map<string, Field>;
+    readMessages: (bb: ByteBuffer, footer: Footer) => Iterable<Message>;
+};
+
+export interface VectorReaderContext {
+    offset: number;
+    bytes: Uint8Array;
+    batch: RecordBatch;
+    dictionaries: Map<string, Vector>;
+    readNextNode(): FieldNode;
+    readNextBuffer(): ArrowBuffer;
+}
+
+export function* readVectors(buffers: Iterable<Uint8Array | Buffer | string>, context?: ArrowReaderContext) {
+    const context_ = context || {} as ArrowReaderContext;
+    for (const buffer of buffers) {
+        yield* readBuffer(toByteBuffer(buffer), context_);
     }
 }
 
-export function* readBuffers(...bytes: Array<Uint8Array | Buffer | string>) {
-    const dictionaries: Dictionaries = {};
-    const byteBuffers = bytes.map(toByteBuffer);
-    for (let { schema, batch } of readRecords(...byteBuffers)) {
-        let vectors: Column<any>[] = [];
-        let state = { nodeIndex: 0, bufferIndex: 0 };
-        let fieldsLength = schema.fieldsLength();
-        let index = -1, field: Field, vector: Vector<any>;
-        if (batch.id) {
-            // A dictionary batch only contain a single vector. Traverse each
-            // field and its children until we find one that uses this dictionary
-            while (++index < fieldsLength) {
-                if (field = schema.fields(index)!) {
-                    if (vector = readDictionary<any>(field, batch, state, dictionaries)!) {
-                        dictionaries[batch.id] = dictionaries[batch.id] && dictionaries[batch.id].concat(vector) || vector;
-                        break;
-                    }
+export async function* readVectorsAsync(buffers: AsyncIterable<Uint8Array | Buffer | string>, context?: ArrowReaderContext) {
+    const context_ = context || {} as ArrowReaderContext;
+    for await (const buffer of buffers) {
+        yield* readBuffer(toByteBuffer(buffer), context_);
+    }
+}
+
+function* readBuffer(bb: ByteBuffer, readerContext: ArrowReaderContext) {
+
+    let { schema, footer, readMessages, dictionaryEncodedFields, dictionaries } = readerContext;
+
+    if (!schema) {
+        ({ schema, footer, readMessages, dictionaryEncodedFields } = readSchema(bb));
+        readerContext.schema = schema;
+        readerContext.readMessages = readMessages;
+        readerContext.dictionaryEncodedFields = dictionaryEncodedFields;
+        readerContext.dictionaries = dictionaries = new Map<string, Vector>();
+    }
+
+    const fieldsLength = schema.fieldsLength();
+    const context = new BufferReaderContext(bb.bytes(), dictionaries);
+
+    for (const message of readMessages(bb, footer!)) {
+
+        let id: string;
+        let field: Field;
+        let vector: Vector;
+        let vectors: Array<Vector>;
+
+        context.message = message;
+
+        if (message.headerType() === MessageHeader.DictionaryBatch) {
+            let batch: DictionaryBatch;
+            if (batch = message.header(new DictionaryBatch())!) {
+                context.batch = batch.data()!;
+                id = batch.id().toFloat64().toString();
+                field = dictionaryEncodedFields.get(id)!;
+                vector = readValueVector(field, context);
+                if (batch.isDelta() && dictionaries.has(id)) {
+                    vector = dictionaries.get(id)!.concat(vector);
                 }
+                dictionaries.set(id, vector);
             }
-        } else {
-            while (++index < fieldsLength) {
-                if ((field = schema.fields(index)!) &&
-                    (vector = readVector<any>(field, batch, state, dictionaries)!)) {
-                    vectors[index] = vector as Column<any>;
-                }
+            continue;
+        }
+
+        vectors = new Array<Vector>(fieldsLength);
+        context.batch = message.header(new RecordBatch())!;
+
+        for (let i = -1; ++i < fieldsLength;) {
+            if ((field = schema.fields(i)!) || (vectors[i] = null as any)) {
+                vectors[i] = readVector(field, context);
             }
-            yield vectors;
         }
+
+        yield vectors;
     }
 }
 
+function readSchema(bb: ByteBuffer) {
+    let schema: Schema, readMessages, footer = readFileFooter(bb);
+    if (footer) {
+        schema = footer.schema()!;
+        readMessages = readFileMessages;
+    } else if (schema = readStreamSchema(bb)!) {
+        readMessages = readStreamMessages;
+    } else {
+        throw new Error('Invalid Arrow buffer');
+    }
+    return { schema, footer, readMessages, dictionaryEncodedFields: readDictionaryEncodedFields(schema, new Map<string, Field>()) };
+}
+
+function readDictionaryEncodedFields(parent: Schema | Field, fields: Map<string, Field>) {
+    let field: Field, encoding: DictionaryEncoding, id: string;
+    let getField = parent instanceof Field ? parent.children : parent.fields;
+    let getFieldCount = parent instanceof Field ? parent.childrenLength : parent.fieldsLength;
+    for (let i = -1, n = getFieldCount.call(parent); ++i < n;) {
+        if (field = getField.call(parent, i)!) {
+            if ((encoding = field.dictionary()!) &&
+                (id = encoding.id().toFloat64().toString())) {
+                !fields.has(id) && fields.set(id, field);
+            }
+            readDictionaryEncodedFields(field, fields);
+        }
+    }
+    return fields;
+}
+
 function toByteBuffer(bytes?: Uint8Array | Buffer | string) {
     let arr: Uint8Array = bytes as any || new Uint8Array(0);
     if (typeof bytes === 'string') {
@@ -86,3 +163,33 @@ function toByteBuffer(bytes?: Uint8Array | Buffer | string) {
     }
     return new ByteBuffer(arr);
 }
+
+class BufferReaderContext implements VectorReaderContext {
+    public offset: number;
+    public batch: RecordBatch;
+    private nodeIndex: number;
+    private bufferIndex: number;
+    private metadataVersion: MetadataVersion;
+    constructor(public bytes: Uint8Array,
+                public dictionaries: Map<string, Vector>) {
+    }
+    set message(m: Message) {
+        this.nodeIndex = 0;
+        this.bufferIndex = 0;
+        this.offset = m.bb.position();
+        this.metadataVersion = m.version();
+    }
+    public readNextNode() {
+        return this.batch.nodes(this.nodeIndex++)!;
+    }
+    public readNextBuffer() {
+        const buffer = this.batch.buffers(this.bufferIndex++)!;
+        // If this Arrow buffer was written before version 4,
+        // advance the buffer's bb_pos 8 bytes to skip past
+        // the now-removed page id field.
+        if (this.metadataVersion < MetadataVersion[`V4`]) {
+            buffer.bb_pos += (8 * this.bufferIndex);
+        }
+        return buffer;
+    }
+}
\ No newline at end of file
diff --git a/js/src/reader/dictionary.ts b/js/src/reader/dictionary.ts
deleted file mode 100644
index 0c58ace3b0dca..0000000000000
--- a/js/src/reader/dictionary.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { readVector } from './vector';
-import { MessageBatch } from './message';
-import { DictionaryVector } from '../types/dictionary';
-import * as Schema_ from '../format/Schema_generated';
-import { IteratorState, Dictionaries } from './arrow';
-import Field = Schema_.org.apache.arrow.flatbuf.Field;
-
-export function readDictionary<T>(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): DictionaryVector<T> | null {
-    let vector: DictionaryVector<T> | null, id, encoding = field.dictionary();
-    if (encoding && batch.id === (id = encoding.id().toFloat64().toString())) {
-        return readVector<T>(field, batch, iterator, null) as DictionaryVector<T>;
-    }
-    for (let i = -1, n = field.childrenLength() | 0; ++i < n;) {
-        if (vector = readDictionary<T>(field.children(i)!, batch, iterator, dictionaries)) {
-            return vector;
-        }
-    }
-    return null;
-}
diff --git a/js/src/reader/file.ts b/js/src/reader/file.ts
deleted file mode 100644
index bd60b476312a0..0000000000000
--- a/js/src/reader/file.ts
+++ /dev/null
@@ -1,82 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { flatbuffers } from 'flatbuffers';
-import * as File_ from '../format/File_generated';
-import * as Schema_ from '../format/Schema_generated';
-import * as Message_ from '../format/Message_generated';
-import { PADDING, readMessageBatches } from './message';
-
-import ByteBuffer = flatbuffers.ByteBuffer;
-import Footer = File_.org.apache.arrow.flatbuf.Footer;
-export import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
-export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
-
-const MAGIC_STR = 'ARROW1';
-const MAGIC = new Uint8Array(MAGIC_STR.length);
-for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
-    MAGIC[i] = MAGIC_STR.charCodeAt(i);
-}
-
-export function _checkMagic(buffer: Uint8Array, index = 0) {
-    for (let i = -1, n = MAGIC.length; ++i < n;) {
-        if (MAGIC[i] !== buffer[index + i]) {
-            return false;
-        }
-    }
-    return true;
-}
-
-const magicLength = MAGIC.length;
-const magicAndPadding = magicLength + PADDING;
-const magicX2AndPadding = magicLength * 2 + PADDING;
-
-export function* readFile(...bbs: ByteBuffer[]) {
-    for (let bb of bbs) {
-        let fileLength = bb.capacity();
-        let footerLength: number, footerOffset: number;
-        if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
-            (!_checkMagic(bb.bytes(), 0) /*                        Missing magic start    */) ||
-            (!_checkMagic(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
-            (/*                                                    Invalid footer length  */
-            (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
-            (footerLength + magicX2AndPadding > fileLength))) {
-            throw new Error('Invalid file');
-        }
-        bb.setPosition(footerOffset - footerLength);
-        let schema, footer = Footer.getRootAsFooter(bb);
-        if (!(schema = footer.schema()!)) {
-            return;
-        }
-        for (let i = -1, n = footer.dictionariesLength(); ++i < n;) {
-            let block = footer.dictionaries(i)!;
-            bb.setPosition(block.offset().low);
-            for (let batch of readMessageBatches(bb)) {
-                yield { schema, batch };
-                break;
-            }
-        }
-        for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) {
-            const block = footer.recordBatches(i)!;
-            bb.setPosition(block.offset().low);
-            for (let batch of readMessageBatches(bb)) {
-                yield { schema, batch };
-                break;
-            }
-        }
-    }
-}
diff --git a/js/src/reader/format.ts b/js/src/reader/format.ts
new file mode 100644
index 0000000000000..fd8f1b40d918a
--- /dev/null
+++ b/js/src/reader/format.ts
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import * as File_ from '../format/File';
+import * as Schema_ from '../format/Schema';
+import * as Message_ from '../format/Message';
+import ByteBuffer = flatbuffers.ByteBuffer;
+import Footer = File_.org.apache.arrow.flatbuf.Footer;
+import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+import Message = Message_.org.apache.arrow.flatbuf.Message;
+import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
+
+const PADDING = 4;
+const MAGIC_STR = 'ARROW1';
+const MAGIC = new Uint8Array(MAGIC_STR.length);
+for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
+    MAGIC[i] = MAGIC_STR.charCodeAt(i);
+}
+
+export function _checkMagic(buffer: Uint8Array, index = 0) {
+    for (let i = -1, n = MAGIC.length; ++i < n;) {
+        if (MAGIC[i] !== buffer[index + i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+const magicLength = MAGIC.length;
+const magicAndPadding = magicLength + PADDING;
+const magicX2AndPadding = magicLength * 2 + PADDING;
+
+export function readStreamSchema(bb: ByteBuffer) {
+    if (!_checkMagic(bb.bytes(), 0)) {
+        for (const message of readMessages(bb)) {
+            if (message.headerType() === MessageHeader.Schema) {
+                return message.header(new Schema());
+            }
+        }
+    }
+    return null;
+}
+
+export function readFileFooter(bb: ByteBuffer) {
+    let fileLength = bb.capacity();
+    let footerLength: number, footerOffset: number;
+    if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
+        (!_checkMagic(bb.bytes(), 0) /*                        Missing magic start    */) ||
+        (!_checkMagic(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
+        (/*                                                    Invalid footer length  */
+        (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
+        (footerLength + magicX2AndPadding > fileLength))) {
+        return null;
+    }
+    bb.setPosition(footerOffset - footerLength);
+    return Footer.getRootAsFooter(bb);
+}
+
+export function* readFileMessages(bb: ByteBuffer, footer: Footer) {
+    for (let i = -1, n = footer.dictionariesLength(); ++i < n;) {
+        bb.setPosition(footer.dictionaries(i)!.offset().low);
+        yield readMessage(bb, bb.readInt32(bb.position()));
+    }
+    for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) {
+        bb.setPosition(footer.recordBatches(i)!.offset().low);
+        yield readMessage(bb, bb.readInt32(bb.position()));
+    }
+}
+
+export function readMessage(bb: ByteBuffer, length: number) {
+    bb.setPosition(bb.position() + PADDING);
+    const message = Message.getRootAsMessage(bb);
+    bb.setPosition(bb.position() + length);
+    return message;
+}
+
+export function* readMessages(bb: ByteBuffer) {
+    let length;
+    while (bb.position() < bb.capacity() &&
+          (length = bb.readInt32(bb.position())) > 0) {
+        yield readMessage(bb, length);
+    }
+}
+
+export function* readStreamMessages(bb: ByteBuffer) {
+    for (const message of readMessages(bb)) {
+        switch (message.headerType()) {
+            case MessageHeader.RecordBatch:
+            case MessageHeader.DictionaryBatch:
+                yield message;
+                break;
+            default: continue;
+        }
+        // position the buffer after the body to read the next message
+        bb.setPosition(bb.position() + message.bodyLength().low);
+    }
+}
diff --git a/js/src/reader/message.ts b/js/src/reader/message.ts
deleted file mode 100644
index 6c8a969021f90..0000000000000
--- a/js/src/reader/message.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { flatbuffers } from 'flatbuffers';
-import * as Message_ from '../format/Message_generated';
-import ByteBuffer = flatbuffers.ByteBuffer;
-import Message = Message_.org.apache.arrow.flatbuf.Message;
-import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
-import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
-import DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch;
-
-export const PADDING = 4;
-export type MessageBatch = {
-    id?: string;
-    offset: number;
-    bytes: Uint8Array;
-    data: RecordBatch;
-};
-
-export function* readMessages(bb: ByteBuffer) {
-    let message, length;
-    while (bb.position() < bb.capacity() &&
-          (length = bb.readInt32(bb.position())) > 0) {
-        bb.setPosition(bb.position() + PADDING);
-        message = Message.getRootAsMessage(bb);
-        bb.setPosition(bb.position() + length);
-        yield message;
-    }
-}
-
-export function* readMessageBatches(bb: ByteBuffer) {
-    let bytes = bb.bytes();
-    for (let message of readMessages(bb)) {
-        let type = message.headerType();
-        let id: string | void, data: RecordBatch;
-        if (type === MessageHeader.RecordBatch) {
-            data = message.header(new RecordBatch())!;
-        } else if (type === MessageHeader.DictionaryBatch) {
-            let header = message.header(new DictionaryBatch())!;
-            id = header.id().toFloat64().toString();
-            data = header.data()!;
-        } else {
-            continue;
-        }
-        yield <MessageBatch> { id, data, bytes, offset: bytes.byteOffset + bb.position() };
-        // position the buffer after the body to read the next message
-        bb.setPosition(bb.position() + message.bodyLength().low);
-    }
-}
diff --git a/js/src/reader/stream.ts b/js/src/reader/stream.ts
deleted file mode 100644
index 2062b1a8c4c10..0000000000000
--- a/js/src/reader/stream.ts
+++ /dev/null
@@ -1,43 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { _checkMagic } from './file';
-import { flatbuffers } from 'flatbuffers';
-import * as Schema_ from '../format/Schema_generated';
-import * as Message_ from '../format/Message_generated';
-import { readMessages, readMessageBatches } from './message';
-
-import ByteBuffer = flatbuffers.ByteBuffer;
-import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
-import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
-
-export function* readStream(...bbs: ByteBuffer[]) {
-    if (!bbs.length || _checkMagic(bbs[0].bytes(), 0)) {
-        throw new Error('Invalid Arrow Stream');
-    }
-    for (const message of readMessages(bbs[0])) {
-        let schema: Schema;
-        if (message.headerType() === MessageHeader.Schema && (schema = message.header(new Schema())!)) {
-            for (const bb of bbs) {
-                for (const batch of readMessageBatches(bb)) {
-                    yield { schema, batch };
-                }
-            }
-            break;
-        }
-    }
-}
diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts
index 4d3321833209d..0f95b769e141f 100644
--- a/js/src/reader/vector.ts
+++ b/js/src/reader/vector.ts
@@ -15,288 +15,257 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { flatbuffers } from 'flatbuffers';
-import { MessageBatch } from './message';
-import * as Schema_ from '../format/Schema_generated';
-import * as Message_ from '../format/Message_generated';
-import { IteratorState, Dictionaries } from './arrow';
+import { VectorReaderContext } from './arrow';
+import * as Schema_ from '../format/Schema';
+import * as Message_ from '../format/Message';
+import { TypedArray, TypedArrayConstructor } from '../vector/types';
 import {
-    Vector, Column,
-    IntArray, FloatArray,
-    TypedArray, TypedArrayConstructor,
-} from '../types/types';
-
-import {
-    DictionaryVector,
-    Utf8Vector, StructVector,
-    ListVector, FixedSizeListVector,
-    DateVector, Float32Vector, Float64Vector,
+    Vector, BoolVector, BinaryVector, DictionaryVector,
     Int8Vector, Int16Vector, Int32Vector, Int64Vector,
     Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector,
-} from '../types/arrow';
+    Utf8Vector, ListVector, FixedSizeListVector, StructVector,
+    Float16Vector, Float32Vector, Float64Vector, DecimalVector,
+    Date32Vector, Date64Vector, Time32Vector, Time64Vector, TimestampVector,
+} from '../vector/arrow';
 
 import Int = Schema_.org.apache.arrow.flatbuf.Int;
+import Date = Schema_.org.apache.arrow.flatbuf.Date;
+import Time = Schema_.org.apache.arrow.flatbuf.Time;
 import Type = Schema_.org.apache.arrow.flatbuf.Type;
 import Field = Schema_.org.apache.arrow.flatbuf.Field;
-import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+import Buffer = Schema_.org.apache.arrow.flatbuf.Buffer;
+import Decimal = Schema_.org.apache.arrow.flatbuf.Decimal;
+import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
+import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
+// import Interval = Schema_.org.apache.arrow.flatbuf.Interval;
+import Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp;
+// import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
 import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
-import VectorType = Schema_.org.apache.arrow.flatbuf.VectorType;
-import VectorLayout = Schema_.org.apache.arrow.flatbuf.VectorLayout;
+import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
 import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
 import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
-import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
+import FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary;
 
-export function readVector<T>(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries): Column<T> | DictionaryVector<T> | null {
-    return readDictionaryVector<T>(field, batch, state, dictionaries) ||
-                readTypedVector<T>(field, batch, state, dictionaries);
+export function readVector(field: Field, state: VectorReaderContext) {
+    return readDictionaryVector(field, state) || readValueVector(field, state);
 }
 
-function readTypedVector<T>(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): Column<T> | DictionaryVector<T> | null {
-    let typeType = field.typeType(), readTyped = typedVectorReaders[typeType];
-    if (!readTyped) {
-        throw new Error('Unrecognized vector name "' + Type[typeType] + '" type "' + typeType + '"');
-    }
-    return readTyped(field, batch, iterator, dictionaries) as Column<T>;
-}
+/* a dictionary index defaults to signed 32 bit int if unspecified */
+const defaultDictionaryIndexType = { bitWidth: () => 32, isSigned: () => true } as Int;
+const intVectors = [
+    [/* unsigned */ [Uint8Vector,  Uint8Array ],  /* signed */ [Int8Vector , Int8Array ]],
+    [/* unsigned */ [Uint16Vector, Uint16Array],  /* signed */ [Int16Vector, Int16Array]],
+    [/* unsigned */ [Uint32Vector, Uint32Array],  /* signed */ [Int32Vector, Int32Array]],,
+    [/* unsigned */ [Uint64Vector, Uint32Array],  /* signed */ [Int64Vector, Int32Array]]
+] as [any, TypedArrayConstructor][][];
 
-function readDictionaryVector<T>(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): DictionaryVector<T> | null {
-    let data: Vector<any>, encoding: DictionaryEncoding;
-    if (dictionaries &&
-        (encoding = field.dictionary()!) &&
-        (data = dictionaries[encoding.id().toFloat64().toString()])) {
-        let indexType =  encoding.indexType() ||
-            /* a dictionary index defaults to signed 32 bit int if unspecified */
-            { bitWidth: () => 32, isSigned: () => true };
-        // workaround for https://issues.apache.org/jira/browse/ARROW-1363
-        let indexField = createSyntheticDictionaryIndexField(field, indexType);
-        let keys = readIntVector(indexField, batch, iterator, null, indexType)!;
-        return new DictionaryVector<T>({ data, keys: keys! });
+export function readDictionaryVector(field: Field, state: VectorReaderContext) {
+    const encoding = field.dictionary()!;
+    if (encoding) {
+        const type = encoding.indexType() || defaultDictionaryIndexType;
+        const data = state.dictionaries.get(encoding.id().toFloat64().toString())!;
+        const [IntVector, IntArray] = intVectors[type.bitWidth() >>> 4]![+type.isSigned()];
+        const { fieldNode, validity, data: keys } = readNumericBuffers(field, state, IntArray);
+        return new DictionaryVector({
+            validity, data, field, fieldNode,
+            keys: new IntVector({ field, fieldNode, data: keys })
+        });
     }
     return null;
 }
 
-const IntViews    = [Int8Array,    Int16Array,   Int32Array,   Int32Array  ];
-const Int32Views  = [Int32Array,   Int32Array,   Int32Array,   Int32Array  ];
-const UintViews   = [Uint8Array,   Uint16Array,  Uint32Array,  Uint32Array ];
-const Uint8Views  = [Uint8Array,   Uint8Array,   Uint8Array,   Uint8Array  ];
-const Uint32Views = [Uint32Array,  Uint32Array,  Uint32Array,  Uint32Array ];
-const FloatViews  = [Int8Array,    Int16Array,   Float32Array, Float64Array];
-
-const createIntDataViews = createTypedArray.bind(null, IntViews, null);
-const createUintDataViews = createTypedArray.bind(null, UintViews, null);
-const createDateDataViews = createTypedArray.bind(null, Uint32Views, null);
-const createFloatDataViews = createTypedArray.bind(null, FloatViews, null);
-const createNestedDataViews = createTypedArray.bind(null, Uint32Views, null);
-const createValidityDataViews = createTypedArray.bind(null, Uint8Views, null);
-const createUtf8DataViews = createTypedArray.bind(null, Uint8Views, Int32Views);
-
-// Define as computed properties for closure-compiler
-const floatVectors = {
-    [Precision.HALF]: Float32Vector,
-    [Precision.SINGLE]: Float32Vector,
-    [Precision.DOUBLE]: Float64Vector,
-} as { [k: number]: any };
+export function readValueVector(field: Field, state: VectorReaderContext): Vector {
+    switch (field.typeType()) {
+        case Type.NONE: return readNullVector();
+        case Type.Null: return readNullVector();
+        // case Type.Map: return readMapVector(field, state);
+        case Type.Int: return readIntVector(field, state);
+        case Type.Bool: return readBoolVector(field, state);
+        case Type.Date: return readDateVector(field, state);
+        case Type.List: return readListVector(field, state);
+        case Type.Utf8: return readUtf8Vector(field, state);
+        case Type.Time: return readTimeVector(field, state);
+        // case Type.Union: return readUnionVector(field, state);
+        case Type.Binary: return readBinaryVector(field, state);
+        case Type.Decimal: return readDecimalVector(field, state);
+        case Type.Struct_: return readStructVector(field, state);
+        case Type.FloatingPoint: return readFloatVector(field, state);
+        case Type.Timestamp: return readTimestampVector(field, state);
+        case Type.FixedSizeList: return readFixedSizeListVector(field, state);
+        case Type.FixedSizeBinary: return readFixedSizeBinaryVector(field, state);
+    }
+    throw new Error(`Unrecognized Vector { name: ${Type[field.typeType()]}, type: ${field.typeType()} }`);
+}
 
-// and again as string-indexed keys for Uglify...
-floatVectors[Precision['HALF']] = Float32Vector;
-floatVectors[Precision['SINGLE']] = Float32Vector;
-floatVectors[Precision['DOUBLE']] = Float64Vector;
+export function readNullVector() {
+    return new Vector();
+}
 
-const intVectors = [
-    [/* unsigned */ Uint8Vector,   /* signed */ Int8Vector ],
-    [/* unsigned */ Uint16Vector,  /* signed */ Int16Vector],
-    [/* unsigned */ Uint32Vector,  /* signed */ Int32Vector],
-    [/* unsigned */ Uint64Vector,  /* signed */ Int64Vector]
-] as any[][];
+export function readBoolVector(field: Field, state: VectorReaderContext) {
+    return new BoolVector(readNumericBuffers(field, state, Uint8Array));
+}
 
-function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) {
-    let type = (primitiveType || field.type(new Int())!);
-    return type.isSigned() ?
-        read_IntVector(field, batch, iterator, dictionaries, type) :
-        readUintVector(field, batch, iterator, dictionaries, type);
+export function readDateVector(field: Field, state: VectorReaderContext) {
+    const type = field.type(new Date())!;
+    switch (type.unit()) {
+        case DateUnit.DAY: return new Date32Vector({ ...readNumericBuffers(field, state, Int32Array), unit: DateUnit[type.unit()] });
+        case DateUnit.MILLISECOND: return new Date64Vector({ ...readNumericBuffers(field, state, Int32Array), unit: DateUnit[type.unit()] });
+    }
+    throw new Error(`Unrecognized Date { unit: ${type.unit()} }`);
 }
 
-function read_IntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) {
-    return readVectorLayout(createIntDataViews, createIntVector, field, batch, iterator, dictionaries, primitiveType);
+export function readTimeVector(field: Field, state: VectorReaderContext) {
+    const type = field.type(new Time())!;
+    switch (type.bitWidth()) {
+        case 32: return new Time32Vector({ ...readNumericBuffers(field, state, Int32Array), unit: TimeUnit[type.unit()] });
+        case 64: return new Time64Vector({ ...readNumericBuffers(field, state, Uint32Array), unit: TimeUnit[type.unit()] });
+    }
+    throw new Error(`Unrecognized Time { unit: ${type.unit()}, bitWidth: ${type.bitWidth()} }`);
 }
 
-function readUintVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) {
-    return readVectorLayout(createUintDataViews, createIntVector, field, batch, iterator, dictionaries, primitiveType);
+export function readTimestampVector(field: Field, state: VectorReaderContext) {
+    const type = field.type(new Timestamp())!;
+    const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint32Array);
+    return new TimestampVector({
+        field, fieldNode, validity, data,
+        timezone: type.timezone()!,
+        unit: TimeUnit[type.unit()],
+    });
 }
 
-function createIntVector(argv: VectorFactoryArgv<IntArray>) {
-    let { field, fieldNode, data, validity, offsets, primitiveType } = argv;
-    let type = primitiveType || field.type(new Int())!, bitWidth = type.bitWidth();
-    let IntVector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()];
-    return new IntVector({ fieldNode, field, validity, data: data! || offsets! });
-    // ---------------------------------------------------- 👆:
-    // Workaround for https://issues.apache.org/jira/browse/ARROW-1363
-    // This bug causes dictionary encoded vector indicies' IntVector data
-    // buffers to be tagged as VectorType.OFFSET (0) in the field metadata
-    // instead of VectorType.DATA. The `readVectorLayout` routine strictly
-    // obeys the types in the field metadata, so if we're parsing an Arrow
-    // file written by a version of the library published before ARROW-1363
-    // was fixed, the IntVector's data buffer will be null, and the offset
-    // buffer will be the actual data. If data is null, it's safe to assume
-    // the offset buffer is the data, because IntVectors don't have offsets.
+export function readListVector(field: Field, state: VectorReaderContext) {
+    const { fieldNode, validity, offsets } = readListBuffers(field, state);
+    return new ListVector({
+        field, fieldNode, validity, offsets,
+        values: readVector(field.children(0)!, state)
+    });
 }
 
-function bindVectorReader<T extends TypedArray, V>(createBufferView: BufferViewFactory<T>, createVector: VectorFactory<T, V>) {
-    return function readVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) {
-        return readVectorLayout(createBufferView, createVector, field, batch, iterator, dictionaries, primitiveType);
-    };
+export function readStructVector(field: Field, state: VectorReaderContext) {
+    const n = field.childrenLength();
+    const columns = new Array<Vector>(n);
+    const fieldNode = state.readNextNode();
+    const validity = readValidityBuffer(field, fieldNode, state);
+    for (let i = -1, child: Field; ++i < n;) {
+        if (child = field.children(i)!) {
+            columns[i] = readVector(child, state);
+        }
+    }
+    return new StructVector({ field, fieldNode, validity, columns });
 }
 
-const readFloatVector = bindVectorReader(createFloatDataViews, ({ field, fieldNode, data, validity }: VectorFactoryArgv<FloatArray>) => {
-    const type = field.type(new FloatingPoint())!;
-    const FloatVector = floatVectors[type.precision()];
-    return new FloatVector({ field, fieldNode, validity, data: data! });
-});
+export function readBinaryVector(field: Field, state: VectorReaderContext) {
+    return new BinaryVector(readBinaryBuffers(field, state));
+}
 
-const readDateVector = bindVectorReader(createDateDataViews, ({ field, fieldNode, data, validity }: VectorFactoryArgv<Uint32Array>) => {
-    return new DateVector({ field, fieldNode, validity, data: data! });
-});
+export function readDecimalVector(field: Field, state: VectorReaderContext) {
+    const type = field.type(new Decimal())!;
+    const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint32Array);
+    return new DecimalVector({
+        scale: type.scale(),
+        precision: type.precision(),
+        field, fieldNode, validity, data
+    });
+}
 
-const readUtf8Vector = bindVectorReader(createUtf8DataViews, ({ field, fieldNode, data, offsets, validity }: VectorFactoryArgv<Uint8Array>) => {
+export function readUtf8Vector(field: Field, state: VectorReaderContext) {
+    const { fieldNode, validity, offsets, data } = readBinaryBuffers(field, state);
     return new Utf8Vector({
         field, fieldNode,
-        values: new ListVector({
-            validity,
-            offsets: offsets as Int32Array,
-            values: new Uint8Vector({ data: data! })
-        }) as any as Vector<Uint8Array | null>
+        values: new BinaryVector({
+            validity, offsets, data
+        })
     });
-});
+}
 
-const readListVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, offsets, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv<TypedArray>) => {
-    return new ListVector({
+export function readFixedSizeListVector(field: Field, state: VectorReaderContext) {
+    const type = field.type(new FixedSizeList())!;
+    const fieldNode = state.readNextNode();
+    const validity = readValidityBuffer(field, fieldNode, state);
+    return new FixedSizeListVector({
         field, fieldNode, validity,
-        offsets: offsets! as Int32Array,
-        values: readVector(field.children(0)!, messageBatch, iterator, dictionaries)!
+        size: type.listSize(),
+        values: readVector(field.children(0)!, state)
     });
-});
+}
 
-const readFixedSizeListVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv<Uint32Array>) => {
+export function readFixedSizeBinaryVector(field: Field, state: VectorReaderContext) {
+    const type = field.type(new FixedSizeBinary())!;
+    const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint8Array);
     return new FixedSizeListVector({
+        size: type.byteWidth(),
         field, fieldNode, validity,
-        listSize: field.type(new FixedSizeList())!.listSize(),
-        values: readVector(field.children(0)!, messageBatch, iterator, dictionaries)!
+        values: new Uint8Vector({ data })
     });
-});
+}
 
-const readStructVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv<ArrayLike<any>>) => {
-    let columns: Column<any>[] = [];
-    for (let i = -1, n = field.childrenLength(); ++i < n;) {
-        columns[i] = readVector<any>(field.children(i)!, messageBatch, iterator, dictionaries) as Column<any>;
+export function readFloatVector(field: Field, state: VectorReaderContext) {
+    const type = field.type(new FloatingPoint())!;
+    switch (type.precision()) {
+        case Precision.HALF:   return new Float16Vector(readNumericBuffers(field, state, Uint16Array));
+        case Precision.SINGLE: return new Float32Vector(readNumericBuffers(field, state, Float32Array));
+        case Precision.DOUBLE: return new Float64Vector(readNumericBuffers(field, state, Float64Array));
     }
-    return new StructVector({ field, fieldNode, validity, columns });
-});
+    throw new Error(`Unrecognized FloatingPoint { precision: ${type.precision()} }`);
+}
 
-// Define as computed properties for closure-compiler
-const typedVectorReaders = {
-    [Type.Int]: readIntVector,
-    [Type.Date]: readDateVector,
-    [Type.List]: readListVector,
-    [Type.Utf8]: readUtf8Vector,
-    [Type.Struct_]: readStructVector,
-    [Type.FloatingPoint]: readFloatVector,
-    [Type.FixedSizeList]: readFixedSizeListVector,
-} as { [k: number]: (...args: any[]) => Vector | null };
+export function readIntVector(field: Field, state: VectorReaderContext) {
+    const type = field.type(new Int())!;
+    if (type.isSigned()) {
+        switch (type.bitWidth()) {
+            case  8: return new  Int8Vector(readNumericBuffers(field, state, Int8Array));
+            case 16: return new Int16Vector(readNumericBuffers(field, state, Int16Array));
+            case 32: return new Int32Vector(readNumericBuffers(field, state, Int32Array));
+            case 64: return new Int64Vector(readNumericBuffers(field, state, Int32Array));
+        }
+    }
+    switch (type.bitWidth()) {
+        case  8: return new  Uint8Vector(readNumericBuffers(field, state, Uint8Array));
+        case 16: return new Uint16Vector(readNumericBuffers(field, state, Uint16Array));
+        case 32: return new Uint32Vector(readNumericBuffers(field, state, Uint32Array));
+        case 64: return new Uint64Vector(readNumericBuffers(field, state, Uint32Array));
+    }
+    throw new Error(`Unrecognized Int { isSigned: ${type.isSigned()}, bitWidth: ${type.bitWidth()} }`);
+}
 
-// and again as string-indexed keys for Uglify...
-typedVectorReaders[Type['Int']] = readIntVector;
-typedVectorReaders[Type['Date']] = readDateVector;
-typedVectorReaders[Type['List']] = readListVector;
-typedVectorReaders[Type['Utf8']] = readUtf8Vector;
-typedVectorReaders[Type['Struct_']] = readStructVector;
-typedVectorReaders[Type['FloatingPoint']] = readFloatVector;
-typedVectorReaders[Type['FixedSizeList']] = readFixedSizeListVector;
+function readListBuffers(field: Field, state: VectorReaderContext) {
+    const fieldNode = state.readNextNode();
+    const validity = readValidityBuffer(field, fieldNode, state);
+    const offsets = readDataBuffer(Int32Array, state);
+    return { field, fieldNode, validity, offsets };
+}
 
-type VectorFactory<T, V> = (argv: VectorFactoryArgv<T>) => V;
-type PrimitiveType = { bitWidth(): number; isSigned(): boolean };
-type BufferViewFactory<T extends TypedArray> = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => T;
+function readBinaryBuffers(field: Field, state: VectorReaderContext) {
+    const fieldNode = state.readNextNode();
+    const validity = readValidityBuffer(field, fieldNode, state);
+    const offsets = readDataBuffer(Int32Array, state);
+    const data = readDataBuffer(Uint8Array, state);
+    return { field, fieldNode, validity, offsets, data };
+}
 
-interface VectorFactoryArgv<T> {
-    field: Field;
-    fieldNode: FieldNode;
-    iterator: IteratorState;
-    dictionaries: Dictionaries;
-    messageBatch: MessageBatch;
-    data?: T;
-    offsets?: TypedArray;
-    validity?: Uint8Array;
-    primitiveType?: PrimitiveType;
+function readNumericBuffers<T extends TypedArray>(field: Field, state: VectorReaderContext, ArrayConstructor: TypedArrayConstructor<T>) {
+    const fieldNode = state.readNextNode();
+    const validity = readValidityBuffer(field, fieldNode, state);
+    const data = readDataBuffer(ArrayConstructor, state);
+    return { field, fieldNode, validity, data };
 }
 
-function readVectorLayout<T extends TypedArray, V>(
-    createBufferView: BufferViewFactory<T>, createVector: VectorFactory<T, V>,
-    field: Field, messageBatch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType
-) {
-    let fieldNode: FieldNode, recordBatch = messageBatch.data;
-    if (!(fieldNode = recordBatch.nodes(iterator.nodeIndex)!)) {
-        return null;
-    }
-    iterator.nodeIndex += 1;
-    let type, bitWidth, layout, buffer, bufferLength;
-    let data: T | undefined, offsets: TypedArray | undefined, validity: Uint8Array | undefined;
-    for (let i = -1, n = field.layoutLength(); ++i < n;) {
-        if (!(layout = field.layout(i)!) ||
-            !(buffer = recordBatch.buffers(iterator.bufferIndex)!)) {
-            continue;
-        }
-        iterator.bufferIndex += 1;
-        if ((type = layout.type()) === VectorType.TYPE ||
-            (bufferLength = buffer.length().low) <= 0  ||
-            (bitWidth = layout.bitWidth()) <= 0) {
-            continue;
-        } else if (type === VectorType.DATA) {
-            data = createBufferView(messageBatch, type, bitWidth, buffer.offset().low, bufferLength);
-        } else if (type === VectorType.OFFSET) {
-            offsets = createBufferView(messageBatch, type, bitWidth, buffer.offset().low, bufferLength);
-        } else if (fieldNode.nullCount().low > 0) {
-            validity = createValidityDataViews(messageBatch, type, bitWidth, buffer.offset().low, fieldNode.length().low);
-        }
-    }
-    return createVector({ data, offsets, validity, field, fieldNode, iterator, messageBatch, dictionaries, primitiveType });
+function readDataBuffer<T extends TypedArray>(ArrayConstructor: TypedArrayConstructor<T>, state: VectorReaderContext) {
+    return createTypedArray(ArrayConstructor, state.bytes, state.offset, state.readNextBuffer());
 }
 
-function createTypedArray(
-    bufferViews: TypedArrayConstructor[], offsetViews: TypedArrayConstructor[] | null,
-    batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number
-) {
-    const buffer = batch.bytes.buffer;
-    const byteLength = buffer.byteLength;
-    const byteOffset = batch.offset + offset;
-    const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || bufferViews);
-    const dataViewLength = ((byteOffset + length) <= byteLength
-        ? length
-        : byteLength - byteOffset
-    ) / DataViewType['BYTES_PER_ELEMENT'];
-    return new DataViewType(buffer, byteOffset, dataViewLength);
+function readValidityBuffer(field: Field, fieldNode: FieldNode, state: VectorReaderContext) {
+    return createValidityArray(field, fieldNode, state.bytes, state.offset, state.readNextBuffer());
 }
 
-function valueForBitWidth<T>(bitWidth: number, values: T[]) {
-    return values[bitWidth >> 4] || values[3];
+function createValidityArray(field: Field, fieldNode: FieldNode, bytes: Uint8Array, offset: number, buffer: Buffer) {
+    return field.nullable() && fieldNode.nullCount().low > 0 && createTypedArray(Uint8Array, bytes, offset, buffer) || null;
 }
 
-function createSyntheticDictionaryIndexField(field: Field, type: PrimitiveType) {
-    let layouts = [] as VectorLayout[];
-    let builder = new flatbuffers.Builder();
-    if (field.nullable()) {
-        VectorLayout.startVectorLayout(builder);
-        VectorLayout.addBitWidth(builder, 8);
-        VectorLayout.addType(builder, VectorType.VALIDITY);
-        builder.finish(VectorLayout.endVectorLayout(builder));
-        layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer()));
-        builder = new flatbuffers.Builder();
-    }
-    VectorLayout.startVectorLayout(builder);
-    VectorLayout.addBitWidth(builder, type.bitWidth());
-    VectorLayout.addType(builder, VectorType.DATA);
-    builder.finish(VectorLayout.endVectorLayout(builder));
-    layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer()));
-    return Object.create(field, {
-        layout: { value(i: number) { return layouts[i]; } },
-        layoutLength: { value() { return layouts.length; } }
-    });
+function createTypedArray<T extends TypedArray>(ArrayConstructor: TypedArrayConstructor<T>, bytes: Uint8Array, offset: number, buffer: Buffer) {
+    return new ArrayConstructor(
+        bytes.buffer,
+        bytes.byteOffset + offset + buffer.offset().low,
+        buffer.length().low / ArrayConstructor.BYTES_PER_ELEMENT
+    );
 }
diff --git a/js/src/types/arrow.ts b/js/src/types/arrow.ts
deleted file mode 100644
index e18f5da4f1fd6..0000000000000
--- a/js/src/types/arrow.ts
+++ /dev/null
@@ -1,88 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import * as Schema_ from '../format/Schema_generated';
-import * as Message_ from '../format/Message_generated';
-import Field = Schema_.org.apache.arrow.flatbuf.Field;
-import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
-
-import { BoolVector } from './vector/bool';
-import { DictionaryVector } from './dictionary';
-import { nullableMixin, fieldMixin } from './vector/traits';
-import { ListVector as ListVectorBase } from './list';
-import { Utf8Vector as Utf8VectorBase } from './utf8';
-import { Vector, Column, TypedArray } from './types';
-import { DateVector as DateVectorBase } from './vector/date';
-import { TableVector as TableVectorBase } from './table/table';
-import { StructVector as StructVectorBase } from './table/struct';
-import { FixedSizeListVector as FixedSizeListVectorBase } from './fixedsizelist';
-import {
-    LongVector as LongVectorBase,
-    Int64Vector as Int64VectorBase,
-    Uint64Vector as Uint64VectorBase,
-} from './vector/long';
-import {
-    TypedVector,
-    Int8Vector as Int8VectorBase,
-    Int16Vector as Int16VectorBase,
-    Int32Vector as Int32VectorBase,
-    Uint8Vector as Uint8VectorBase,
-    Uint16Vector as Uint16VectorBase,
-    Uint32Vector as Uint32VectorBase,
-    Float32Vector as Float32VectorBase,
-    Float64Vector as Float64VectorBase,
-} from './vector/typed';
-
-export { TypedArray, TypedVector };
-export { Column, BoolVector, DictionaryVector };
-export class ListVector extends MixinArrowTraits(ListVectorBase) {}
-export class Utf8Vector extends MixinArrowTraits(Utf8VectorBase) {}
-export class TableVector extends MixinArrowTraits(TableVectorBase) {}
-export class StructVector extends MixinArrowTraits(StructVectorBase) {}
-export class FixedSizeListVector extends MixinArrowTraits(FixedSizeListVectorBase) {}
-export class DateVector extends MixinArrowTraits(DateVectorBase) {}
-export class LongVector extends MixinArrowTraits(LongVectorBase) {}
-export class Int8Vector extends MixinArrowTraits(Int8VectorBase) {}
-export class Int16Vector extends MixinArrowTraits(Int16VectorBase) {}
-export class Int32Vector extends MixinArrowTraits(Int32VectorBase) {}
-export class Int64Vector extends MixinArrowTraits(Int64VectorBase) {}
-export class Uint8Vector extends MixinArrowTraits(Uint8VectorBase) {}
-export class Uint16Vector extends MixinArrowTraits(Uint16VectorBase) {}
-export class Uint32Vector extends MixinArrowTraits(Uint32VectorBase) {}
-export class Uint64Vector extends MixinArrowTraits(Uint64VectorBase) {}
-export class Float32Vector extends MixinArrowTraits(Float32VectorBase) {}
-export class Float64Vector extends MixinArrowTraits(Float64VectorBase) {}
-
-export function MixinArrowTraits<T extends Vector<any>, TArgv>(BaseVector: new (argv: TArgv) => T) {
-    const FieldVector = fieldMixin(BaseVector);
-    const NullableVector = nullableMixin(BaseVector);
-    const NullableFieldVector = nullableMixin(FieldVector);
-    return function(this: any, argv: TArgv & (object | { validity: Uint8Array } | { field: Field, fieldNode: FieldNode })) {
-        return new ((!isFieldArgv(argv) ? !isNullableArgv(argv) ?
-            BaseVector : NullableVector : !isNullableArgv(argv) ?
-            FieldVector : NullableFieldVector
-        ) as any)(argv);
-    } as any as { new (argv: TArgv & (object | { validity: Uint8Array } | { field: Field, fieldNode: FieldNode })): T };
-}
-
-function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } {
-    return x && x.field instanceof Field && x.fieldNode instanceof FieldNode;
-}
-
-function isNullableArgv(x: any): x is { validity: Uint8Array } {
-    return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array;
-}
diff --git a/js/src/types/fixedsizelist.ts b/js/src/types/fixedsizelist.ts
deleted file mode 100644
index 6311d891d5058..0000000000000
--- a/js/src/types/fixedsizelist.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { List, Vector } from './types';
-import { VirtualVector } from './vector/virtual';
-
-export class FixedSizeListVector<T, TArray extends List<T>> extends Vector<TArray> {
-    readonly listSize: number;
-    readonly values: Vector<T>;
-    constructor(argv: { listSize: number, values: Vector<T> }) {
-        super();
-        this.values = argv.values;
-        this.listSize = argv.listSize;
-    }
-    get(index: number) {
-        return this.values.slice<TArray>(this.listSize * index, this.listSize * (index + 1));
-    }
-    concat(...vectors: Vector<TArray>[]): Vector<TArray> {
-        return new VirtualVector(Array, this, ...vectors);
-    }
-}
diff --git a/js/src/types/list.ts b/js/src/types/list.ts
deleted file mode 100644
index ca9170b5908d5..0000000000000
--- a/js/src/types/list.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { List, Vector } from './types';
-import { VirtualVector } from './vector/virtual';
-
-export class ListVector<T, TArray extends List<T>> extends Vector<TArray> {
-    readonly offsets: Int32Array;
-    readonly values: Vector<T>;
-    constructor(argv: { offsets: Int32Array, values: Vector<T> }) {
-        super();
-        this.values = argv.values;
-        this.offsets = argv.offsets;
-    }
-    get(index: number) {
-        return this.values.slice<TArray>(this.offsets[index], this.offsets[index + 1]);
-    }
-    concat(...vectors: Vector<TArray>[]): Vector<TArray> {
-        return new VirtualVector(Array, this, ...vectors);
-    }
-}
\ No newline at end of file
diff --git a/js/src/types/table/from.ts b/js/src/types/table/from.ts
deleted file mode 100644
index ae0755961eb7d..0000000000000
--- a/js/src/types/table/from.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Column } from '../types';
-import { TableVector } from './table';
-import { readBuffers } from '../../reader/arrow';
-
-export function fromBuffers(...bytes: Array<Uint8Array | Buffer | string>) {
-    let columns: Column<any>[] = null as any;
-    for (let vectors of readBuffers(...bytes)) {
-        columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i]) as Column<any>);
-    }
-    return new TableVector({ columns });
-}
-
-TableVector.from = fromBuffers;
-
-declare module './table' {
-    namespace TableVector { export let from: typeof fromBuffers; }
-}
\ No newline at end of file
diff --git a/js/src/types/table/row.ts b/js/src/types/table/row.ts
deleted file mode 100644
index 432cfd7364ba3..0000000000000
--- a/js/src/types/table/row.ts
+++ /dev/null
@@ -1,61 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Row, Vector, Struct } from '../types';
-import { VirtualVector } from '../vector/virtual';
-
-export class RowVector<T = any> extends Vector<T> implements Row<T> {
-    readonly row: number;
-    readonly length: number;
-    readonly table: Struct<T>;
-    [Symbol.toStringTag]() { return 'Row'; }
-    constructor(table: Struct<T>, row: number) {
-        super();
-        this.row = row;
-        this.table = table;
-        this.length = table.columns.length;
-    }
-    get(index: number) {
-        const col = this.table.columns[index];
-        return col ? col.get(this.row) as T : null;
-    }
-    col(key: string) {
-        const col = this.table.col(key);
-        return col ? col.get(this.row) as T : null;
-    }
-    *[Symbol.iterator]() {
-        const { row } = this;
-        for (const col of this.table.columns) {
-            yield col ? col.get(row) : null;
-        }
-    }
-    concat(...rows: Vector<T>[]): Vector<T> {
-        return new VirtualVector(Array, this, ...rows as any[]);
-    }
-    toArray() { return [...this]; }
-    toJSON() { return this.toArray(); }
-    toString() { return `Row [${this.length})` }
-    toObject(): Record<string, any> {
-        const { row } = this, map = Object.create(null);
-        for (const col of this.table.columns) {
-            if (col && col.name) {
-                map[col.name] = col.get(row);
-            }
-        }
-        return map;
-    }
-}
diff --git a/js/src/types/table/struct.ts b/js/src/types/table/struct.ts
deleted file mode 100644
index de6a3a05692aa..0000000000000
--- a/js/src/types/table/struct.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { RowVector } from './row';
-import { toString } from './toString';
-import { VirtualVector } from '../vector/virtual';
-import { Row, Vector, Column, Struct } from '../types';
-
-export interface StructVector {
-    toString(): string;
-    toString(index: boolean): string;
-    toString(options: { index: boolean }): string;
-}
-
-export class StructVector<T = any> extends Vector<Row<T>> implements Struct<T> {
-    readonly length: number;
-    readonly columns: Column[];
-    constructor(argv: { columns: Column[] }) {
-        super();
-        this.columns = argv.columns || [];
-        if (!this.length) {
-            this.length = Math.max(...this.columns.map((col) => col.length)) | 0;
-        }
-    }
-    get(index: number): StructRow<T> {
-        return new StructRow(this, index);
-    }
-    col(name: string) {
-        return this.columns.find((col) => col.name === name) || null;
-    }
-    key(index: number) {
-        return this.columns[index] ? this.columns[index].name : null;
-    }
-    select(...columns: string[]) {
-        return new StructVector({ columns: columns.map((name) => this.col(name)!) });
-    }
-    concat(...structs: Vector<Row<T>>[]): Vector<Row<T>> {
-        return new VirtualVector(Array, this, ...structs as any[]);
-    }
-    toString(x?: any) {
-        return toString(this, x);
-    }
-}
-
-export class StructRow<T> extends RowVector<T> {
-    toString() {
-        return JSON.stringify(this);
-    }
-}
\ No newline at end of file
diff --git a/js/src/types/table/table.ts b/js/src/types/table/table.ts
deleted file mode 100644
index d9074dec2d382..0000000000000
--- a/js/src/types/table/table.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { StructVector, StructRow } from './struct';
-
-export class TableVector<T> extends StructVector<T> {
-    get(index: number): TableRow<T> {
-        return new TableRow(this, index);
-    }
-}
-
-export class TableRow<T> extends StructRow<T> {
-    toString() {
-        return this.toArray().map((x) => JSON.stringify(x)).join(', ');
-    }
-}
\ No newline at end of file
diff --git a/js/src/types/table/toString.ts b/js/src/types/table/toString.ts
deleted file mode 100644
index 85b23ca246058..0000000000000
--- a/js/src/types/table/toString.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import { Struct } from '../types';
-
-export function toString<T>(source: Struct<T>, options?: any) {
-    const index = typeof options === 'object' ? options && !!options.index
-                : typeof options === 'boolean' ? !!options
-                : false;
-    const { length } = source;
-    if (length <= 0) { return ''; }
-    const rows = new Array(length + 1);
-    const maxColumnWidths = [] as number[];
-    rows[0] = source.columns.map((_, i) => source.key(i));
-    index && rows[0].unshift('Index');
-    for (let i = -1, n = rows.length - 1; ++i < n;) {
-        rows[i + 1] = [...source.get(i)!];
-        index && rows[i + 1].unshift(i);
-    }
-    // Pass one to convert to strings and count max column widths
-    for (let i = -1, n = rows.length; ++i < n;) {
-        const row = rows[i];
-        for (let j = -1, k = row.length; ++j < k;) {
-            const val = row[j] = `${row[j]}`;
-            maxColumnWidths[j] = !maxColumnWidths[j]
-                ? val.length
-                : Math.max(maxColumnWidths[j], val.length);
-        }
-    }
-    // Pass two to pad each one to max column width
-    for (let i = -1, n = rows.length; ++i < n;) {
-        const row = rows[i];
-        for (let j = -1, k = row.length; ++j < k;) {
-            row[j] = leftPad(row[j], ' ', maxColumnWidths[j]);
-        }
-        rows[i] = row.join(', ');
-    }
-    return rows.join('\n');
-}
-
-function leftPad(str: string, fill: string, n: number) {
-    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
-}
diff --git a/js/src/types/vector/bool.ts b/js/src/types/vector/bool.ts
deleted file mode 100644
index b2eea81f87f05..0000000000000
--- a/js/src/types/vector/bool.ts
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { TypedVector } from './typed';
-
-export interface BoolVector extends TypedVector<boolean, Uint8Array> {
-    set(index: number, value: boolean): void;
-}
-
-export class BoolVector extends TypedVector<boolean, Uint8Array> {
-    static pack = pack;
-    get(index: number) {
-        return (this.data[index >> 3] & 1 << index % 8) !== 0;
-    }
-    set(index: number, value: boolean) {
-        if (index > -1 === false) {
-            return;
-        } else if (value) {
-            this.data[index >> 3] |=  (1 << (index % 8));
-        } else {
-            this.data[index >> 3] &= ~(1 << (index % 8));
-        }
-    }
-}
-
-export function pack(values: Iterable<any>) {
-    let xs = [], n, i = 0;
-    let bit = 0, byte = 0;
-    for (const value of values) {
-        value && (byte |= 1 << bit);
-        if (++bit === 8) {
-            xs[i++] = byte;
-            byte = bit = 0;
-        }
-    }
-    if (i === 0 || bit > 0) { xs[i++] = byte; }
-    if (i % 8 && (n = i + 8 - i % 8)) {
-        do { xs[i] = 0; } while (++i < n);
-    }
-    return new Uint8Array(xs);
-}
diff --git a/js/src/types/vector/date.ts b/js/src/types/vector/date.ts
deleted file mode 100644
index 82dc82e64021f..0000000000000
--- a/js/src/types/vector/date.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { TypedVector } from './typed';
-
-export class DateVector extends TypedVector<Date, Uint32Array> {
-    get(index: number): Date {
-        return new Date(4294967296   * /* 2^32 */
-            (super.get(index * 2 + 1) as any) + /* high */
-            (super.get(index * 2) as any)       /*  low */
-        );
-    }
-}
-
-(DateVector.prototype as any).stride = 2;
diff --git a/js/src/types/vector/long.ts b/js/src/types/vector/long.ts
deleted file mode 100644
index de8eb0c13710b..0000000000000
--- a/js/src/types/vector/long.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { TypedVector } from './typed';
-import { TypedArray } from '../types';
-import { flatbuffers } from 'flatbuffers';
-import Long = flatbuffers.Long;
-
-export class LongVector<T extends TypedArray> extends TypedVector<Long, T> {
-    get(index: number) {
-        return new Long(
-            super.get(index * 2) as any,    /*  low */
-            super.get(index * 2 + 1) as any /* high */
-        );
-    }
-}
-
-(LongVector.prototype as any).stride = 2;
-
-export class Int64Vector extends LongVector<Int32Array> {}
-export class Uint64Vector extends LongVector<Uint32Array> {}
\ No newline at end of file
diff --git a/js/src/types/vector/typed.ts b/js/src/types/vector/typed.ts
deleted file mode 100644
index fc093f2cb5f2a..0000000000000
--- a/js/src/types/vector/typed.ts
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from '../types';
-import { VirtualVector } from './virtual';
-import { TypedArray, TypedArrayConstructor } from '../types';
-
-export interface TypedVector<T, TArray extends TypedArray> {
-    slice(start?: number, end?: number): TArray;
-}
-
-export class TypedVector<T, TArray extends TypedArray> extends Vector<T> {
-    readonly data: TArray;
-    readonly stride: number;
-    readonly length: number;
-    constructor(argv: { data: TArray } | TArray) {
-        super();
-        const data = ArrayBuffer.isView(argv) ? argv : argv.data;
-        this.length = ((this.data = data).length / this.stride) | 0;
-    }
-    get(index: number): T | null {
-        return this.data[index] as any;
-    }
-    concat(...vectors: Vector<T>[]): Vector<T> {
-        return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors);
-    }
-    slice(start?: number, end?: number) {
-        const { data, stride } = this, from = start! | 0;
-        const to = end === undefined ? data.length : Math.max(end | 0, from);
-        return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0);
-    }
-}
-
-(TypedVector.prototype as any).stride = 1;
-
-export class Int8Vector extends TypedVector<number, Int8Array> {}
-export class Int16Vector extends TypedVector<number, Int16Array> {}
-export class Int32Vector extends TypedVector<number, Int32Array> {}
-export class Uint8Vector extends TypedVector<number, Uint8Array> {}
-export class Uint16Vector extends TypedVector<number, Uint16Array> {}
-export class Uint32Vector extends TypedVector<number, Uint32Array> {}
-export class Float32Vector extends TypedVector<number, Float32Array> {}
-export class Float64Vector extends TypedVector<number, Float64Array> {}
diff --git a/js/src/types/vector/virtual.ts b/js/src/types/vector/virtual.ts
deleted file mode 100644
index 7f56012dc7d4b..0000000000000
--- a/js/src/types/vector/virtual.ts
+++ /dev/null
@@ -1,129 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { TypedVector } from './typed';
-import { Vector, Column, TypedArray, TypedArrayConstructor } from '../types';
-
-export class VirtualVector<T> implements Column<T> {
-    readonly name: string;
-    readonly type: string;
-    readonly length: number;
-    readonly vectors: Vector<T>[];
-    readonly offsets: Uint32Array;
-    readonly ArrayType: ArrayConstructor | TypedArrayConstructor;
-    constructor(ArrayType: ArrayConstructor | TypedArrayConstructor, ...vectors: Vector<T>[]) {
-        this.vectors = vectors;
-        this.ArrayType = ArrayType;
-        this.name = (vectors[0] as any).name;
-        this.type = (vectors[0] as any).type;
-        this.length = vectors.reduce((sum, vec) => sum + vec.length, 0);
-        this.offsets = Uint32Array.from(vectors.reduce((sums, vector, index) => [...sums, vector.length + sums[index]], [0]));
-    }
-    *[Symbol.iterator]() {
-        for (const vector of this.vectors) {
-            yield* vector;
-        }
-    }
-    get nullable() {
-        return (this.vectors as Column<T>[]).some((vec) => vec.nullable);
-    }
-    get nullCount() {
-        return (this.vectors as Column<T>[]).reduce((sum, v) => sum + v.nullCount | 0, 0);
-    }
-    get metadata() {
-        return new Map<string, string>(
-            (this.vectors as Column<T>[]).reduce((entries, v) => [
-                ...entries, ...v.metadata.entries()
-            ], [] as [string, string][])
-        );
-    }
-    get(index: number) {
-        return findIndex(this.offsets, index) ? this.vectors[_vector].get(_offset) : null;
-    }
-    concat(...vectors: Vector<T>[]) {
-        return new VirtualVector(this.ArrayType, ...this.vectors, ...vectors);
-    }
-    slice(begin?: number, end?: number) {
-        const ArrayType = this.ArrayType as any;
-        // clamp begin and end values between the virtual length
-        clampRange(this.length, begin!, end);
-        const from = _from, total = _total;
-        // find the start vector index and adjusted value index offset
-        if (!findIndex(this.offsets, from)) { return new ArrayType(0); }
-        const set = ArrayType === Array ? arraySet : typedArraySet as any;
-        let index = _vector, vectors = this.vectors as TypedVector<T, TypedArray>[];
-        let vector = vectors[index], source = vector.slice(_offset, _offset + total), target = source;
-        // Perf optimization: if the first slice contains all the values we're looking for,
-        // we don't have to copy values to a target Array. If we're slicing a TypedArray,
-        // this is a significant improvement as we avoid the memcpy 🎉
-        if ((source.length / vector.stride | 0) < total) {
-            let vectorsLength = vectors.length;
-            let count = 0, length = 0, sources = [];
-            do {
-                sources.push(source);
-                length += source.length;
-                count += (source.length / vector.stride | 0);
-            } while (
-                (count  < total) &&
-                (vector = vectors[index = (++index % vectorsLength)]) &&
-                (source = vector.slice(0, Math.min(vector.length, total - count)))
-            );
-            target = new ArrayType(length);
-            for (let i = -1, j = 0, n = sources.length; ++i < n;) {
-                j = set(sources[i], target, j);
-            }
-        }
-        return target;
-    }
-}
-
-let _from = -1, _total = -1;
-function clampRange(length: number, start: number, end?: number) {
-    let total = length, from = start || 0;
-    let to = end === end && typeof end == 'number' ? end : total;
-    if (to < 0) { to = total + to; }
-    if (from < 0) { from = total - (from * -1) % total; }
-    if (to < from) { from = to; to = start; }
-    _from = from;
-    _total = !isFinite(total = (to - from)) || total < 0 ? 0 : total;
-}
-
-let _offset = -1, _vector = -1;
-function findIndex(offsets: Uint32Array, index: number) {
-    let offset = 0, left = 0, middle = 0, right = offsets.length - 1;
-    while (index < offsets[right] && index >= (offset = offsets[left])) {
-        if (left + 1 === right) {
-            _vector = left;
-            _offset = index - offset;
-            return true;
-        }
-        middle = left + ((right - left) / 2) | 0;
-        index >= offsets[middle] ? (left = middle) : (right = middle);
-    }
-    return false;
-}
-
-function arraySet<T>(source: T[], target: T[], index: number) {
-    for (let i = 0, n = source.length; i < n;) {
-        target[index++] = source[i++];
-    }
-    return index;
-}
-
-function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
-    return target.set(source, index) || index + source.length;
-}
diff --git a/js/src/vector/arrow.ts b/js/src/vector/arrow.ts
new file mode 100644
index 0000000000000..88fec7e44542b
--- /dev/null
+++ b/js/src/vector/arrow.ts
@@ -0,0 +1,245 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Schema_ from '../format/Schema';
+import * as Message_ from '../format/Message';
+import Field = Schema_.org.apache.arrow.flatbuf.Field;
+import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+
+import { Vector } from './vector';
+import { Utf8Vector as Utf8VectorBase } from './utf8';
+import { StructVector as StructVectorBase } from './struct';
+import { DictionaryVector as DictionaryVectorBase } from './dictionary';
+import {
+    ListVector as ListVectorBase,
+    BinaryVector as BinaryVectorBase,
+    FixedSizeListVector as FixedSizeListVectorBase
+} from './list';
+
+import {
+    BoolVector as BoolVectorBase,
+    Int8Vector as Int8VectorBase,
+    Int16Vector as Int16VectorBase,
+    Int32Vector as Int32VectorBase,
+    Int64Vector as Int64VectorBase,
+    Uint8Vector as Uint8VectorBase,
+    Uint16Vector as Uint16VectorBase,
+    Uint32Vector as Uint32VectorBase,
+    Uint64Vector as Uint64VectorBase,
+    Float16Vector as Float16VectorBase,
+    Float32Vector as Float32VectorBase,
+    Float64Vector as Float64VectorBase,
+    Date32Vector as Date32VectorBase,
+    Date64Vector as Date64VectorBase,
+    Time32Vector as Time32VectorBase,
+    Time64Vector as Time64VectorBase,
+    DecimalVector as DecimalVectorBase,
+    TimestampVector as TimestampVectorBase,
+} from './numeric';
+
+import { nullableMixin, fieldMixin } from './traits';
+
+function MixinArrowTraits<T extends Vector<any>, TArgv>(
+    Base: new (argv: TArgv) => T,
+    Field: new (argv: TArgv & { field: Field, fieldNode: FieldNode }) => T,
+    Nullable: new (argv: TArgv & { validity: Uint8Array }) => T,
+    NullableField: new (argv: TArgv & { validity: Uint8Array, field: Field, fieldNode: FieldNode }) => T,
+) {
+    return function(argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })) {
+        return new (!isFieldArgv(argv)
+            ? !isNullableArgv(argv) ? Base : Nullable
+            : !isNullableArgv(argv) ? Field : NullableField
+        )(argv as any);
+    } as any as { new (argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })): T };
+}
+
+function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } {
+    return x && x.field instanceof Field && x.fieldNode instanceof FieldNode;
+}
+
+function isNullableArgv(x: any): x is { validity: Uint8Array } {
+    return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array;
+}
+
+export { Vector };
+export class ListVector extends MixinArrowTraits(
+    ListVectorBase,
+    class ListVector extends fieldMixin(ListVectorBase) {} as any,
+    class ListVector extends nullableMixin(ListVectorBase) {} as any,
+    class ListVector extends nullableMixin(fieldMixin(ListVectorBase)) {} as any
+) {}
+
+export class BinaryVector extends MixinArrowTraits(
+    BinaryVectorBase,
+    class BinaryVector extends fieldMixin(BinaryVectorBase) {} as any,
+    class BinaryVector extends nullableMixin(BinaryVectorBase) {} as any,
+    class BinaryVector extends nullableMixin(fieldMixin(BinaryVectorBase)) {} as any
+) {}
+
+export class Utf8Vector extends MixinArrowTraits(
+    Utf8VectorBase,
+    class Utf8Vector extends fieldMixin(Utf8VectorBase) {} as any,
+    class Utf8Vector extends nullableMixin(Utf8VectorBase) {} as any,
+    class Utf8Vector extends nullableMixin(fieldMixin(Utf8VectorBase)) {} as any
+) {}
+
+export class BoolVector extends MixinArrowTraits(
+    BoolVectorBase,
+    class BoolVector extends fieldMixin(BoolVectorBase) {} as any,
+    class BoolVector extends nullableMixin(BoolVectorBase) {} as any,
+    class BoolVector extends nullableMixin(fieldMixin(BoolVectorBase)) {} as any
+) {}
+
+export class Int8Vector extends MixinArrowTraits(
+    Int8VectorBase,
+    class Int8Vector extends fieldMixin(Int8VectorBase) {} as any,
+    class Int8Vector extends nullableMixin(Int8VectorBase) {} as any,
+    class Int8Vector extends nullableMixin(fieldMixin(Int8VectorBase)) {} as any
+) {}
+
+export class Int16Vector extends MixinArrowTraits(
+    Int16VectorBase,
+    class Int16Vector extends fieldMixin(Int16VectorBase) {} as any,
+    class Int16Vector extends nullableMixin(Int16VectorBase) {} as any,
+    class Int16Vector extends nullableMixin(fieldMixin(Int16VectorBase)) {} as any
+) {}
+
+export class Int32Vector extends MixinArrowTraits(
+    Int32VectorBase,
+    class Int32Vector extends fieldMixin(Int32VectorBase) {} as any,
+    class Int32Vector extends nullableMixin(Int32VectorBase) {} as any,
+    class Int32Vector extends nullableMixin(fieldMixin(Int32VectorBase)) {} as any
+) {}
+
+export class Int64Vector extends MixinArrowTraits(
+    Int64VectorBase,
+    class Int64Vector extends fieldMixin(Int64VectorBase) {} as any,
+    class Int64Vector extends nullableMixin(Int64VectorBase) {} as any,
+    class Int64Vector extends nullableMixin(fieldMixin(Int64VectorBase)) {} as any
+) {}
+
+export class Uint8Vector extends MixinArrowTraits(
+    Uint8VectorBase,
+    class Uint8Vector extends fieldMixin(Uint8VectorBase) {} as any,
+    class Uint8Vector extends nullableMixin(Uint8VectorBase) {} as any,
+    class Uint8Vector extends nullableMixin(fieldMixin(Uint8VectorBase)) {} as any
+) {}
+
+export class Uint16Vector extends MixinArrowTraits(
+    Uint16VectorBase,
+    class Uint16Vector extends fieldMixin(Uint16VectorBase) {} as any,
+    class Uint16Vector extends nullableMixin(Uint16VectorBase) {} as any,
+    class Uint16Vector extends nullableMixin(fieldMixin(Uint16VectorBase)) {} as any
+) {}
+
+export class Uint32Vector extends MixinArrowTraits(
+    Uint32VectorBase,
+    class Uint32Vector extends fieldMixin(Uint32VectorBase) {} as any,
+    class Uint32Vector extends nullableMixin(Uint32VectorBase) {} as any,
+    class Uint32Vector extends nullableMixin(fieldMixin(Uint32VectorBase)) {} as any
+) {}
+
+export class Uint64Vector extends MixinArrowTraits(
+    Uint64VectorBase,
+    class Uint64Vector extends fieldMixin(Uint64VectorBase) {} as any,
+    class Uint64Vector extends nullableMixin(Uint64VectorBase) {} as any,
+    class Uint64Vector extends nullableMixin(fieldMixin(Uint64VectorBase)) {} as any
+) {}
+
+export class Date32Vector extends MixinArrowTraits(
+    Date32VectorBase,
+    class Date32Vector extends fieldMixin(Date32VectorBase) {} as any,
+    class Date32Vector extends nullableMixin(Date32VectorBase) {} as any,
+    class Date32Vector extends nullableMixin(fieldMixin(Date32VectorBase)) {} as any
+) {}
+
+export class Date64Vector extends MixinArrowTraits(
+    Date64VectorBase,
+    class Date64Vector extends fieldMixin(Date64VectorBase) {} as any,
+    class Date64Vector extends nullableMixin(Date64VectorBase) {} as any,
+    class Date64Vector extends nullableMixin(fieldMixin(Date64VectorBase)) {} as any
+) {}
+
+export class Time32Vector extends MixinArrowTraits(
+    Time32VectorBase,
+    class Time32Vector extends fieldMixin(Time32VectorBase) {} as any,
+    class Time32Vector extends nullableMixin(Time32VectorBase) {} as any,
+    class Time32Vector extends nullableMixin(fieldMixin(Time32VectorBase)) {} as any
+) {}
+
+export class Time64Vector extends MixinArrowTraits(
+    Time64VectorBase,
+    class Time64Vector extends fieldMixin(Time64VectorBase) {} as any,
+    class Time64Vector extends nullableMixin(Time64VectorBase) {} as any,
+    class Time64Vector extends nullableMixin(fieldMixin(Time64VectorBase)) {} as any
+) {}
+
+export class Float16Vector extends MixinArrowTraits(
+    Float16VectorBase,
+    class Float16Vector extends fieldMixin(Float16VectorBase) {} as any,
+    class Float16Vector extends nullableMixin(Float16VectorBase) {} as any,
+    class Float16Vector extends nullableMixin(fieldMixin(Float16VectorBase)) {} as any
+) {}
+
+export class Float32Vector extends MixinArrowTraits(
+    Float32VectorBase,
+    class Float32Vector extends fieldMixin(Float32VectorBase) {} as any,
+    class Float32Vector extends nullableMixin(Float32VectorBase) {} as any,
+    class Float32Vector extends nullableMixin(fieldMixin(Float32VectorBase)) {} as any
+) {}
+
+export class Float64Vector extends MixinArrowTraits(
+    Float64VectorBase,
+    class Float64Vector extends fieldMixin(Float64VectorBase) {} as any,
+    class Float64Vector extends nullableMixin(Float64VectorBase) {} as any,
+    class Float64Vector extends nullableMixin(fieldMixin(Float64VectorBase)) {} as any
+) {}
+
+export class StructVector extends MixinArrowTraits(
+    StructVectorBase,
+    class StructVector extends fieldMixin(StructVectorBase) {} as any,
+    class StructVector extends nullableMixin(StructVectorBase) {} as any,
+    class StructVector extends nullableMixin(fieldMixin(StructVectorBase)) {} as any
+) {}
+
+export class DecimalVector extends MixinArrowTraits(
+    DecimalVectorBase,
+    class DecimalVector extends fieldMixin(DecimalVectorBase) {} as any,
+    class DecimalVector extends nullableMixin(DecimalVectorBase) {} as any,
+    class DecimalVector extends nullableMixin(fieldMixin(DecimalVectorBase)) {} as any
+) {}
+
+export class TimestampVector extends MixinArrowTraits(
+    TimestampVectorBase,
+    class TimestampVector extends fieldMixin(TimestampVectorBase) {} as any,
+    class TimestampVector extends nullableMixin(TimestampVectorBase) {} as any,
+    class TimestampVector extends nullableMixin(fieldMixin(TimestampVectorBase)) {} as any
+) {}
+
+export class DictionaryVector extends MixinArrowTraits(
+    DictionaryVectorBase,
+    class DictionaryVector extends fieldMixin(DictionaryVectorBase) {} as any,
+    class DictionaryVector extends nullableMixin(DictionaryVectorBase) {} as any,
+    class DictionaryVector extends nullableMixin(fieldMixin(DictionaryVectorBase)) {} as any
+) {}
+
+export class FixedSizeListVector extends MixinArrowTraits(
+    FixedSizeListVectorBase,
+    class FixedSizeListVector extends fieldMixin(FixedSizeListVectorBase) {} as any,
+    class FixedSizeListVector extends nullableMixin(FixedSizeListVectorBase) {} as any,
+    class FixedSizeListVector extends nullableMixin(fieldMixin(FixedSizeListVectorBase)) {} as any
+) {}
diff --git a/js/src/types/dictionary.ts b/js/src/vector/dictionary.ts
similarity index 62%
rename from js/src/types/dictionary.ts
rename to js/src/vector/dictionary.ts
index cafa753311d3b..b7375c0904276 100644
--- a/js/src/types/dictionary.ts
+++ b/js/src/vector/dictionary.ts
@@ -15,28 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Vector, Column } from './types';
-import { VirtualVector } from './vector/virtual';
+import { Vector } from './vector';
+import { VirtualVector } from './virtual';
 
-export interface DictionaryVector<T> extends Vector<T> {
-    getValue(key: number): T;
-    getKey(index: number): number;
-}
-
-export class DictionaryVector<T> extends Vector<T> implements Column<T>, DictionaryVector<T> {
+export class DictionaryVector<T> extends Vector<T> {
+    readonly length: number;
     readonly data: Vector<T>;
-    readonly keys: Column<number>;
+    readonly keys: Vector<number>;
     constructor(argv: { data: Vector<T>, keys: Vector<number> }) {
         super();
         this.data = argv.data;
-        this.keys = argv.keys as Column<number>;
+        this.keys = argv.keys;
+        this.length = this.keys.length;
     }
-    get name () { return this.keys.name; }
-    get type () { return this.keys.type; }
-    get length () { return this.keys.length; }
-    get metadata () { return this.keys.metadata; }
-    get nullable () { return this.keys.nullable; }
-    get nullCount () { return this.keys.nullCount; }
     get(index: number) {
         return this.getValue(this.getKey(index)!);
     }
@@ -50,9 +41,8 @@ export class DictionaryVector<T> extends Vector<T> implements Column<T>, Diction
         return new VirtualVector(Array, this, ...vectors);
     }
     *[Symbol.iterator]() {
-        const { data, keys } = this;
-        for (let i = -1, n = keys.length; ++i < n;) {
-            yield data.get(keys.get(i)!);
+        for (let i = -1, n = this.length; ++i < n;) {
+            yield this.get(i);
         }
     }
 }
diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts
new file mode 100644
index 0000000000000..97913f8d8878c
--- /dev/null
+++ b/js/src/vector/list.ts
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { List } from './types';
+import { Vector } from './vector';
+import { VirtualVector } from './virtual';
+
+export class BinaryVector extends Vector<Uint8Array> {
+    readonly data: Uint8Array;
+    readonly offsets: Int32Array;
+    constructor(argv: { offsets: Int32Array, data: Uint8Array }) {
+        super();
+        this.data = argv.data;
+        this.offsets = argv.offsets;
+    }
+    get(index: number) {
+        return this.data.subarray(this.offsets[index], this.offsets[index + 1]);
+    }
+    concat(...vectors: Vector<Uint8Array>[]): Vector<Uint8Array> {
+        return new VirtualVector(Array, this, ...vectors);
+    }
+}
+
+export class ListVector<T> extends Vector<T[]> {
+    readonly offsets: Int32Array;
+    readonly values: Vector<T>;
+    constructor(argv: { offsets: Int32Array, values: Vector<T> }) {
+        super();
+        this.values = argv.values;
+        this.offsets = argv.offsets;
+    }
+    get(index: number) {
+        const { offsets, values } = this;
+        const from = offsets[index];
+        const xs = new Array(offsets[index + 1] - from);
+        for (let i = -1, n = xs.length; ++i < n;) {
+            xs[i] = values.get(i + from);
+        }
+        return xs;
+    }
+    concat(...vectors: Vector<T[]>[]): Vector<T[]> {
+        return new VirtualVector(Array, this, ...vectors);
+    }
+}
+
+export class FixedSizeListVector<T, TArray extends List<T>> extends Vector<TArray> {
+    readonly size: number;
+    readonly values: Vector<T>;
+    constructor(argv: { size: number, values: Vector<T> }) {
+        super();
+        this.size = argv.size;
+        this.values = argv.values;
+    }
+    get(index: number) {
+        return this.values.slice<TArray>(this.size * index, this.size * (index + 1));
+    }
+    concat(...vectors: Vector<TArray>[]): Vector<TArray> {
+        return new VirtualVector(Array, this, ...vectors);
+    }
+}
diff --git a/js/src/vector/numeric.ts b/js/src/vector/numeric.ts
new file mode 100644
index 0000000000000..fe4767809f465
--- /dev/null
+++ b/js/src/vector/numeric.ts
@@ -0,0 +1,168 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { VirtualVector } from './virtual';
+import { TypedArray, TypedArrayConstructor } from './types';
+
+export class NumericVector<T, TArray extends TypedArray> extends Vector<T> {
+    readonly data: TArray;
+    readonly length: number;
+    readonly stride: number;
+    constructor(argv: { data: TArray }) {
+        super();
+        const data = (ArrayBuffer.isView(argv) ? argv : argv.data) as TArray;
+        this.length = ((this.data = data).length / this.stride) | 0;
+    }
+    get(index: number) {
+        return this.data[index] as any;
+    }
+    concat(...vectors: Vector<T>[]): Vector<T> {
+        return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors);
+    }
+    slice(start?: number, end?: number) {
+        const { data, stride } = this, from = start! | 0;
+        const to = end === undefined ? data.length : Math.max(end | 0, from);
+        return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0);
+    }
+}
+
+export class FixedWidthNumericVector<T, TArray extends TypedArray> extends NumericVector<T, TArray> {
+    get(index: number) {
+        return this.data.slice(this.stride * index, this.stride * (index + 1)) as TArray;
+    }
+}
+
+export class BoolVector extends NumericVector<boolean, Uint8Array> {
+    static pack(values: Iterable<any>) {
+        let xs = [], n, i = 0;
+        let bit = 0, byte = 0;
+        for (const value of values) {
+            value && (byte |= 1 << bit);
+            if (++bit === 8) {
+                xs[i++] = byte;
+                byte = bit = 0;
+            }
+        }
+        if (i === 0 || bit > 0) { xs[i++] = byte; }
+        if (i % 8 && (n = i + 8 - i % 8)) {
+            do { xs[i] = 0; } while (++i < n);
+        }
+        return new Uint8Array(xs);
+    }
+    get(index: number) {
+        return (this.data[index >> 3] & 1 << index % 8) !== 0;
+    }
+    set(index: number, value: boolean) {
+        if (index > -1 === false) {
+            return;
+        } else if (value) {
+            this.data[index >> 3] |=  (1 << (index % 8));
+        } else {
+            this.data[index >> 3] &= ~(1 << (index % 8));
+        }
+    }
+}
+
+export class Int8Vector extends NumericVector<number, Int8Array> {}
+export class Int16Vector extends NumericVector<number, Int16Array> {}
+export class Int32Vector extends NumericVector<number, Int32Array> {}
+export class Int64Vector extends FixedWidthNumericVector<number, Int32Array> {}
+
+export class Uint8Vector extends NumericVector<number, Uint8Array> {}
+export class Uint16Vector extends NumericVector<number, Uint16Array> {}
+export class Uint32Vector extends NumericVector<number, Uint32Array> {}
+export class Uint64Vector extends FixedWidthNumericVector<number, Uint32Array> {}
+
+export class Float16Vector extends NumericVector<number, Uint16Array> {
+    get(index: number) {
+        return Math.min((super.get(index)! -  32767) / 32767, 1);
+    }
+}
+
+export class Float32Vector extends NumericVector<number, Float32Array> {}
+export class Float64Vector extends NumericVector<number, Float64Array> {}
+
+export class Date32Vector extends NumericVector<Date, Int32Array> {
+    public readonly unit: string;
+    constructor(argv: { data: Int32Array, unit: string }) {
+        super(argv);
+        this.unit = argv.unit;
+    }
+    get(index: number): Date {
+        return new Date(86400000 * (super.get(index) as any));
+    }
+}
+
+export class Date64Vector extends NumericVector<Date, Int32Array> {
+    public readonly unit: string;
+    constructor(argv: { unit: string, data: Int32Array }) {
+        super(argv);
+        this.unit = argv.unit;
+    }
+    get(index: number): Date {
+        return new Date(4294967296   * /* 2^32 */
+            (super.get(index * 2 + 1) as any) + /* high */
+            (super.get(index * 2) as any)       /*  low */
+        );
+    }
+}
+
+export class Time32Vector extends NumericVector<number, Int32Array> {
+    public readonly unit: string;
+    constructor(argv: { data: Int32Array, unit: string }) {
+        super(argv);
+        this.unit = argv.unit;
+    }
+}
+
+export class Time64Vector extends FixedWidthNumericVector<number, Uint32Array> {
+    public readonly unit: string;
+    constructor(argv: { unit: string, data: Uint32Array }) {
+        super(argv);
+        this.unit = argv.unit;
+    }
+}
+
+export class DecimalVector extends FixedWidthNumericVector<number, Uint32Array> {
+    readonly scale: number;
+    readonly precision: number;
+    constructor(argv: { precision: number, scale: number, data: Uint32Array }) {
+        super(argv);
+        this.scale = argv.scale;
+        this.precision = argv.precision;
+    }
+}
+
+export class TimestampVector extends FixedWidthNumericVector<number, Uint32Array> {
+    readonly unit: string;
+    readonly timezone: string;
+    constructor(argv: { unit: string, timezone: string, data: Uint32Array }) {
+        super(argv);
+        this.unit = argv.unit;
+        this.timezone = argv.timezone;
+    }
+}
+
+export interface NumericVectorConstructor<T, TArray extends TypedArray> {
+    readonly prototype: NumericVector<T, TArray>;
+    new (argv: { data: TArray }): NumericVector<T, TArray>;
+}
+
+(DecimalVector.prototype as any).stride = 4;
+(NumericVector.prototype as any).stride = 1;
+(FixedWidthNumericVector.prototype as any).stride = 2;
diff --git a/js/src/vector/struct.ts b/js/src/vector/struct.ts
new file mode 100644
index 0000000000000..c43f6efc48fbe
--- /dev/null
+++ b/js/src/vector/struct.ts
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { VirtualVector } from './virtual';
+
+export class StructVector<T = any> extends Vector<StructRow<T>> {
+    readonly length: number;
+    readonly columns: Vector[];
+    constructor(argv: { columns: Vector[] }) {
+        super();
+        this.columns = argv.columns || [];
+    }
+    get(index: number): StructRow<T> {
+        return new StructRow(this, index);
+    }
+    col(name: string) {
+        return this.columns.find((col) => col.name === name) || null;
+    }
+    key(index: number) {
+        return this.columns[index] ? this.columns[index].name : null;
+    }
+    select(...columns: string[]) {
+        return new StructVector({ columns: columns.map((name) => this.col(name)!) });
+    }
+    concat(...structs: Vector<StructRow<T>>[]): Vector<StructRow<T>> {
+        return new VirtualVector(Array, this, ...structs as any[]);
+    }
+    toString(options?: any) {
+        const index = typeof options === 'object' ? options && !!options.index
+                    : typeof options === 'boolean' ? !!options
+                    : false;
+        const { length } = this;
+        if (length <= 0) { return ''; }
+        const rows = new Array(length + 1);
+        const maxColumnWidths = [] as number[];
+        rows[0] = this.columns.map((_, i) => this.key(i));
+        index && rows[0].unshift('Index');
+        for (let i = -1, n = rows.length - 1; ++i < n;) {
+            rows[i + 1] = [...this.get(i)!];
+            index && rows[i + 1].unshift(i);
+        }
+        // Pass one to convert to strings and count max column widths
+        for (let i = -1, n = rows.length; ++i < n;) {
+            const row = rows[i];
+            for (let j = -1, k = row.length; ++j < k;) {
+                const val = row[j] = stringify(row[j]);
+                maxColumnWidths[j] = !maxColumnWidths[j]
+                    ? val.length
+                    : Math.max(maxColumnWidths[j], val.length);
+            }
+        }
+        // Pass two to pad each one to max column width
+        for (let i = -1, n = rows.length; ++i < n;) {
+            const row = rows[i];
+            for (let j = -1, k = row.length; ++j < k;) {
+                row[j] = leftPad(row[j], ' ', maxColumnWidths[j]);
+            }
+            rows[i] = row.join(', ');
+        }
+        return rows.join('\n');
+    }
+}
+
+export class StructRow<T = any> extends Vector<T> {
+    readonly row: number;
+    readonly length: number;
+    readonly table: StructVector<T>;
+    [Symbol.toStringTag]() { return 'Row'; }
+    constructor(table: StructVector<T>, row: number) {
+        super();
+        this.row = row;
+        this.table = table;
+        this.length = table.columns.length;
+    }
+    get(index: number) {
+        const col = this.table.columns[index];
+        return col ? col.get(this.row) as T : null;
+    }
+    col(key: string) {
+        const col = this.table.col(key);
+        return col ? col.get(this.row) as T : null;
+    }
+    *[Symbol.iterator]() {
+        const { row } = this;
+        for (const col of this.table.columns) {
+            yield col ? col.get(row) : null;
+        }
+    }
+    concat(...rows: Vector<T>[]): Vector<T> {
+        return new VirtualVector(Array, this, ...rows as any[]);
+    }
+    toArray() { return [...this]; }
+    toJSON() { return this.toArray(); }
+    toString() { return JSON.stringify(this); }
+    toObject(): Record<string, T> {
+        const { row } = this, map = Object.create(null);
+        for (const col of this.table.columns) {
+            if (col && col.name) {
+                map[col.name] = col.get(row);
+            }
+        }
+        return map;
+    }
+}
+
+function leftPad(str: string, fill: string, n: number) {
+    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
+}
+
+function stringify(x: any) {
+    return Array.isArray(x) ? JSON.stringify(x) : ArrayBuffer.isView(x) ? `[${x}]` : `${x}`;
+}
diff --git a/js/src/vector/table.ts b/js/src/vector/table.ts
new file mode 100644
index 0000000000000..b15092a72df24
--- /dev/null
+++ b/js/src/vector/table.ts
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { StructVector, StructRow } from './struct';
+import { readVectors, readVectorsAsync } from '../reader/arrow';
+
+export class Table<T> extends StructVector<T> {
+    static from(buffers?: Iterable<Uint8Array | Buffer | string>) {
+        let columns: Vector<any>[] = [];
+        if (buffers) {
+            for (let vectors of readVectors(buffers)) {
+                columns = columns.length === 0 ? vectors : vectors.map((vec, i, _vs, col = columns[i]) =>
+                    vec && col && col.concat(vec) || col || vec
+                ) as Vector<any>[];
+            }
+        }
+        return new Table({ columns });
+    }
+    static async fromAsync(buffers?: AsyncIterable<Uint8Array | Buffer | string>) {
+        let columns: Vector<any>[] = [];
+        if (buffers) {
+            for await (let vectors of readVectorsAsync(buffers)) {
+                columns = columns.length === 0 ? vectors : vectors.map((vec, i, _vs, col = columns[i]) =>
+                    vec && col && col.concat(vec) || col || vec
+                ) as Vector<any>[];
+            }
+        }
+        return new Table({ columns });
+    }
+    readonly length: number;
+    constructor(argv: { columns: Vector<any>[] }) {
+        super(argv);
+        this.length = Math.max(...this.columns.map((col) => col.length)) | 0;
+    }
+    get(index: number): TableRow<T> {
+        return new TableRow(this, index);
+    }
+}
+
+export class TableRow<T> extends StructRow<T> {
+    toString() {
+        return this.toArray().map((x) => JSON.stringify(x)).join(', ');
+    }
+}
diff --git a/js/src/types/vector/traits.ts b/js/src/vector/traits.ts
similarity index 88%
rename from js/src/types/vector/traits.ts
rename to js/src/vector/traits.ts
index 872c40b64d559..ca933f1605957 100644
--- a/js/src/types/vector/traits.ts
+++ b/js/src/vector/traits.ts
@@ -15,10 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { BoolVector } from './bool';
-import { Vector, Column } from '../types';
-import * as Schema_ from '../../format/Schema_generated';
-import * as Message_ from '../../format/Message_generated';
+import { Vector } from './vector';
+import { BoolVector } from './numeric';
+import * as Schema_ from '../format/Schema';
+import * as Message_ from '../format/Message';
 import Type = Schema_.org.apache.arrow.flatbuf.Type;
 import Field = Schema_.org.apache.arrow.flatbuf.Field;
 import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
@@ -38,7 +38,7 @@ export const nullableMixin = <T extends Vector, TArgv>(superclass: new (argv: TA
     };
 
 export const fieldMixin = <T extends Vector, TArgv>(superclass: new (argv: TArgv) => T) =>
-    class extends (superclass as Ctor<TArgv>) implements Column {
+    class extends (superclass as Ctor<TArgv>) implements Vector {
         readonly field: Field;
         readonly type: string;
         readonly length: number;
@@ -53,8 +53,8 @@ export const fieldMixin = <T extends Vector, TArgv>(superclass: new (argv: TArgv
             this.fieldNode = fieldNode;
             this.nullable = field.nullable();
             this.type = Type[field.typeType()];
+            this.length = fieldNode.length().low | 0;
             this.nullCount = fieldNode.nullCount().low;
-            this.length = (fieldNode.length().low / this.stride) | 0;
         }
         get name() { return this.field.name()!; }
         get metadata()  {
diff --git a/js/src/vector/types.ts b/js/src/vector/types.ts
new file mode 100644
index 0000000000000..363fcf2265c30
--- /dev/null
+++ b/js/src/vector/types.ts
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+export interface TypedArrayConstructor<T extends TypedArray = TypedArray> {
+    readonly prototype: T;
+    readonly BYTES_PER_ELEMENT: number;
+    new (length: number): T;
+    new (elements: Iterable<number>): T;
+    new (arrayOrArrayBuffer: ArrayLike<number> | ArrayBufferLike): T;
+    new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T;
+}
+
+export interface TypedArray extends Iterable<number> {
+    [index: number]: number;
+    readonly length: number;
+    readonly byteLength: number;
+    readonly byteOffset: number;
+    readonly buffer: ArrayBufferLike;
+    readonly BYTES_PER_ELEMENT: number;
+    [Symbol.iterator](): IterableIterator<number>;
+    slice(start?: number, end?: number): TypedArray;
+    subarray(begin: number, end?: number): TypedArray;
+    set(array: ArrayLike<number>, offset?: number): void;
+}
+
+export type List<T> = T[] | TypedArray;
+export type FloatArray = Float32Array | Float64Array;
+export type IntArray = Int8Array | Int16Array | Int32Array;
+export type UintArray = Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array;
diff --git a/js/src/types/utf8.ts b/js/src/vector/utf8.ts
similarity index 94%
rename from js/src/types/utf8.ts
rename to js/src/vector/utf8.ts
index 178704f6161f5..ba875cf333fe7 100644
--- a/js/src/types/utf8.ts
+++ b/js/src/vector/utf8.ts
@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Vector } from './types';
+import { Vector } from './vector';
+import { VirtualVector } from './virtual';
 import { TextDecoder } from 'text-encoding-utf-8';
-import { VirtualVector } from './vector/virtual';
 
 const decoder = new TextDecoder('utf-8');
 
diff --git a/js/src/types/types.ts b/js/src/vector/vector.ts
similarity index 56%
rename from js/src/types/types.ts
rename to js/src/vector/vector.ts
index f732bc0971f35..8047c89923223 100644
--- a/js/src/types/types.ts
+++ b/js/src/vector/vector.ts
@@ -15,64 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import * as Schema_ from '../format/Schema_generated';
+import * as Schema_ from '../format/Schema';
 import Type = Schema_.org.apache.arrow.flatbuf.Type;
 
-export interface TypedArrayConstructor<T extends TypedArray = TypedArray> {
-    readonly prototype: T;
-    readonly BYTES_PER_ELEMENT: number;
-    new (length: number): T;
-    new (elements: Iterable<number>): T;
-    new (arrayOrArrayBuffer: ArrayLike<number> | ArrayBufferLike): T;
-    new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T;
-}
-
-export interface TypedArray extends Iterable<number> {
-    [index: number]: number;
-    readonly length: number;
-    readonly byteLength: number;
-    readonly byteOffset: number;
-    readonly buffer: ArrayBufferLike;
-    readonly BYTES_PER_ELEMENT: number;
-    [Symbol.iterator](): IterableIterator<number>;
-    slice(start?: number, end?: number): TypedArray;
-    subarray(begin: number, end?: number): TypedArray;
-    set(array: ArrayLike<number>, offset?: number): void;
-}
-
-export type FloatArray = Float32Array | Float64Array;
-export type IntArray = Int8Array | Int16Array | Int32Array;
-export type UintArray = Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array;
-
-export type List<T> = T[] | TypedArray;
-
 export interface Vector<T = any> extends Iterable<T | null> {
-    readonly length: number;
-    get(index: number): T | null;
-    concat(...vectors: Vector<T>[]): Vector<T>;
-    slice<R = T[]>(start?: number, end?: number): R;
-}
-
-export interface Row<T = any> extends Vector<T> {
-    col(key: string): T | null;
-}
-
-export interface Column<T = any> extends Vector<T> {
     readonly name: string;
     readonly type: string;
+    readonly length: number;
     readonly nullable: boolean;
     readonly nullCount: number;
     readonly metadata: Map<string, string>;
+    get(index: number): T | null;
+    concat(...vectors: Vector<T>[]): Vector<T>;
+    slice<R = T[]>(start?: number, end?: number): R;
 }
-
-export interface Struct<T = any> extends Vector<Row<T>> {
-    readonly columns: Column[];
-    key(key: number): string | null;
-    col(key: string): Column | null;
-    select(...columns: string[]): Struct<T>;
-    concat(...structs: Vector<Row<T>>[]): Vector<Row<T>>;
-}
-
 export class Vector<T = any> implements Vector<T> {
     slice<R = T[]>(start?: number, end?: number): R {
         let { length } = this, from = start! | 0;
@@ -91,8 +47,8 @@ export class Vector<T = any> implements Vector<T> {
 }
 
 (Vector.prototype as any).name = '';
-(Vector.prototype as any).type = Type[0];
 (Vector.prototype as any).stride = 1;
 (Vector.prototype as any).nullable = !1;
 (Vector.prototype as any).nullCount = 0;
 (Vector.prototype as any).metadata = new Map();
+(Vector.prototype as any).type = Type[Type.NONE];
diff --git a/js/src/vector/virtual.ts b/js/src/vector/virtual.ts
new file mode 100644
index 0000000000000..6ec3a8eef9f4d
--- /dev/null
+++ b/js/src/vector/virtual.ts
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { NumericVector } from './numeric';
+import { TypedArray, TypedArrayConstructor } from './types';
+
+export class VirtualVector<T> implements Vector<T> {
+    readonly name: string;
+    readonly type: string;
+    readonly length: number;
+    readonly vectors: Vector<T>[];
+    readonly offsets: Uint32Array;
+    readonly ArrayType: ArrayConstructor | TypedArrayConstructor;
+    constructor(ArrayType: ArrayConstructor | TypedArrayConstructor, ...vectors: Vector<T>[]) {
+        this.vectors = vectors;
+        this.ArrayType = ArrayType;
+        this.name = (vectors[0] as any).name;
+        this.type = (vectors[0] as any).type;
+        this.length = vectors.reduce((sum, vec) => sum + vec.length, 0);
+        this.offsets = Uint32Array.from(vectors.reduce((sums, vector, index) => [...sums, vector.length + sums[index]], [0]));
+    }
+    *[Symbol.iterator]() {
+        for (const vector of this.vectors) {
+            yield* vector;
+        }
+    }
+    get nullable() {
+        return (this.vectors as Vector<T>[]).some((vec) => vec.nullable);
+    }
+    get nullCount() {
+        return (this.vectors as Vector<T>[]).reduce((sum, v) => sum + v.nullCount | 0, 0);
+    }
+    get metadata() {
+        return new Map<string, string>(
+            (this.vectors as Vector<T>[]).reduce((entries, v) => [
+                ...entries, ...v.metadata.entries()
+            ], [] as [string, string][])
+        );
+    }
+    get(index: number) {
+        // find the vector index and adjusted value offset (inlined)
+        let offsets = this.offsets, offset = 0;
+        let left = 0, middle = 0, right = offsets.length - 1;
+        while (index < offsets[right] && index >= (offset = offsets[left])) {
+            if (left + 1 === right) {
+                return this.vectors[left].get(index - offset);
+            }
+            middle = left + ((right - left) / 2) | 0;
+            index >= offsets[middle] ? (left = middle) : (right = middle);
+        }
+        return null;
+    }
+    concat(...vectors: Vector<T>[]) {
+        return new VirtualVector(this.ArrayType, ...this.vectors, ...vectors);
+    }
+    slice(begin?: number, end?: number) {
+
+        // clamp begin and end values between the virtual length (inlined)
+        // let [from, total] = clampRange(this.length, begin!, end);
+        let total = this.length, from = begin! | 0;
+        let to = end === end && typeof end == 'number' ? end : total;
+        if (to < 0) { to = total + to; }
+        if (from < 0) { from = total - (from * -1) % total; }
+        if (to < from) { from = to; to = begin! | 0; }
+        total = !isFinite(total = (to - from)) || total < 0 ? 0 : total;
+
+        // find the vector index and adjusted value offset (inlined)
+        let offsets = this.offsets, ArrayType = this.ArrayType as any;
+        let offset = 0, index = 0, middle = 0, right = offsets.length - 1;
+        while (from < offsets[right] && from >= (offset = offsets[index])) {
+            if (index + 1 === right) {
+                from -= offset;
+                let set = ArrayType === Array ? arraySet : typedArraySet as any;
+                let vectors = this.vectors as any as NumericVector<T, TypedArray>[];
+                let vector = vectors[index], source = vector.slice(from, from + total), target = source;
+                // Perf optimization: if the first slice contains all the values we're looking for,
+                // we don't have to copy values to a target Array. If we're slicing a TypedArray,
+                // this is a significant improvement as we avoid the memcpy 🎉
+                if ((source.length / vector.stride | 0) < total) {
+                    let vectorsLength = vectors.length;
+                    let count = 0, length = 0, sources = [];
+                    do {
+                        sources.push(source);
+                        length += source.length;
+                        count += (source.length / vector.stride | 0);
+                    } while (
+                        (count  < total) &&
+                        (vector = vectors[index = (++index % vectorsLength)]) &&
+                        (source = vector.slice(0, Math.min(vector.length, total - count)))
+                    );
+                    target = new ArrayType(length);
+                    for (let i = -1, j = 0, n = sources.length; ++i < n;) {
+                        j = set(sources[i], target, j);
+                    }
+                }
+                return target;
+            }
+            middle = index + ((right - index) / 2) | 0;
+            from >= offsets[middle] ? (index = middle) : (right = middle);
+        }
+        return new ArrayType(0);
+    }
+}
+
+function arraySet<T>(source: T[], target: T[], index: number) {
+    for (let i = 0, n = source.length; i < n;) {
+        target[index++] = source[i++];
+    }
+    return index;
+}
+
+function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
+    return target.set(source, index) || index + source.length;
+}
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
index 722781db658dc..66a5c21ac2dff 100644
--- a/js/test/Arrow.ts
+++ b/js/test/Arrow.ts
@@ -21,6 +21,7 @@
 const path = require('path');
 const target = process.env.TEST_TARGET!;
 const format = process.env.TEST_MODULE!;
+const useSrc = process.env.TEST_TS_SOURCE === `true`;
 
 // these are duplicated in the gulpfile :<
 const targets = [`es5`, `es2015`, `esnext`];
@@ -32,55 +33,17 @@ function throwInvalidImportError(name: string, value: string, values: string[])
 
 let modulePath = ``;
 
-if (target === `ts` || target === `apache-arrow`) modulePath = target;
+if (useSrc) modulePath = '../src';
+else if (target === `ts` || target === `apache-arrow`) modulePath = target;
 else if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets);
 else if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats);
 else modulePath = path.join(target, format);
 
-let Arrow: any = require(path.resolve(`./targets`, modulePath, `Arrow`));
+export { List } from '../src/Arrow';
+export { TypedArray } from '../src/Arrow';
+export { TypedArrayConstructor } from '../src/Arrow';
+export { NumericVectorConstructor } from '../src/Arrow';
 
-import {
-    Table as Table_,
-    Vector as Vector_,
-    readBuffers as readBuffers_,
-    BoolVector as BoolVector_,
-    TypedVector as TypedVector_,
-    ListVector as ListVector_,
-    Utf8Vector as Utf8Vector_,
-    DateVector as DateVector_,
-    Int8Vector as Int8Vector_,
-    Int16Vector as Int16Vector_,
-    Int32Vector as Int32Vector_,
-    Int64Vector as Int64Vector_,
-    Uint8Vector as Uint8Vector_,
-    Uint16Vector as Uint16Vector_,
-    Uint32Vector as Uint32Vector_,
-    Uint64Vector as Uint64Vector_,
-    Float32Vector as Float32Vector_,
-    Float64Vector as Float64Vector_,
-    StructVector as StructVector_,
-    DictionaryVector as DictionaryVector_,
-    FixedSizeListVector as FixedSizeListVector_,
-} from '../src/Arrow';
-
-export let Table = Arrow.Table as typeof Table_;
-export let Vector = Arrow.Vector as typeof Vector_;
-export let readBuffers = Arrow.readBuffers as typeof readBuffers_;
-export let BoolVector = Arrow.BoolVector as typeof BoolVector_;
-export let TypedVector = Arrow.TypedVector as typeof TypedVector_;
-export let ListVector = Arrow.ListVector as typeof ListVector_;
-export let Utf8Vector = Arrow.Utf8Vector as typeof Utf8Vector_;
-export let DateVector = Arrow.DateVector as typeof DateVector_;
-export let Int8Vector = Arrow.Int8Vector as typeof Int8Vector_;
-export let Int16Vector = Arrow.Int16Vector as typeof Int16Vector_;
-export let Int32Vector = Arrow.Int32Vector as typeof Int32Vector_;
-export let Int64Vector = Arrow.Int64Vector as typeof Int64Vector_;
-export let Uint8Vector = Arrow.Uint8Vector as typeof Uint8Vector_;
-export let Uint16Vector = Arrow.Uint16Vector as typeof Uint16Vector_;
-export let Uint32Vector = Arrow.Uint32Vector as typeof Uint32Vector_;
-export let Uint64Vector = Arrow.Uint64Vector as typeof Uint64Vector_;
-export let Float32Vector = Arrow.Float32Vector as typeof Float32Vector_;
-export let Float64Vector = Arrow.Float64Vector as typeof Float64Vector_;
-export let StructVector = Arrow.StructVector as typeof StructVector_;
-export let DictionaryVector = Arrow.DictionaryVector as typeof DictionaryVector_;
-export let FixedSizeListVector = Arrow.FixedSizeListVector as typeof FixedSizeListVector_;
+import * as Arrow_ from '../src/Arrow';
+export let Arrow: typeof Arrow_ = require(path.resolve(`./targets`, modulePath, `Arrow`));
+export default Arrow;
\ No newline at end of file
diff --git a/js/test/__snapshots__/reader-tests.ts.snap b/js/test/__snapshots__/reader-tests.ts.snap
deleted file mode 100644
index 961ce87861c3f..0000000000000
--- a/js/test/__snapshots__/reader-tests.ts.snap
+++ /dev/null
@@ -1,497 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`;
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
-Array [
-  "Hermione",
-  25,
-  Float32Array [
-    -53.235599517822266,
-    40.231998443603516,
-  ],
-]
-`;
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
-Array [
-  "Severus",
-  30,
-  Float32Array [
-    -62.22999954223633,
-    3,
-  ],
-]
-`;
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`;
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`;
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`;
-
-exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
-Array [
-  "Harry",
-  20,
-  Float32Array [
-    23,
-    -100.23652648925781,
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
-Array [
-  "Hermione",
-  25,
-  Float32Array [
-    -53.235599517822266,
-    40.231998443603516,
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
-Array [
-  "Severus",
-  30,
-  Float32Array [
-    -62.22999954223633,
-    3,
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`;
-
-exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
-Array [
-  "Harry",
-  20,
-  Float32Array [
-    23,
-    -100.23652648925781,
-  ],
-]
-`;
-
-exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`;
-
-exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
-
-exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
-
-exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
-Array [
-  "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
-  "Airbus",
-  1502880750,
-  Float32Array [
-    32.45663833618164,
-    1.8712350130081177,
-  ],
-]
-`;
-
-exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
-Array [
-  "50fb46f4-fefa-42c1-919c-0121974cdd00",
-  "Boeing",
-  1502880750,
-  Float32Array [
-    38.766666412353516,
-    -4.181231498718262,
-  ],
-]
-`;
-
-exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`;
-
-exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
-
-exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
-
-exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
-Array [
-  "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
-  "12345",
-  "Airbus",
-  1502880750,
-  Float32Array [
-    32.45663833618164,
-    1.8712350130081177,
-  ],
-]
-`;
-
-exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
-Array [
-  "50fb46f4-fefa-42c1-919c-0121974cdd00",
-  "67890",
-  "Boeing",
-  1502880750,
-  Float32Array [
-    38.766666412353516,
-    -4.181231498718262,
-  ],
-]
-`;
-
-exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"row_count"`;
-
-exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
-
-exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `1`;
-
-exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `10000`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_lat"`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"FloatingPoint"`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `35.393089294433594`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `35.393089294433594`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `35.393089294433594`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `29.533695220947266`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `29.533695220947266`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"origin_lon"`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `-97.6007308959961`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `-97.6007308959961`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `-97.6007308959961`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `-98.46977996826172`;
-
-exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `-98.46977996826172`;
-
-exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_city"`;
-
-exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Utf8"`;
-
-exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
-
-exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `"Oklahoma City"`;
-
-exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `"Oklahoma City"`;
-
-exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"Oklahoma City"`;
-
-exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"San Antonio"`;
-
-exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `"San Antonio"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`;
-
-exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`;
-
-exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
-Array [
-  null,
-  "MhRNxD4",
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `
-Array [
-  137773603,
-  "3F9HBxK",
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `
-Array [
-  410361374,
-  "aVd88fp",
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
-Array [
-  null,
-  "3loZrRf",
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `
-Array [
-  null,
-  "78SLiRw",
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `
-Array [
-  null,
-  "0ilsf82",
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `
-Array [
-  null,
-  "LjS9MbU",
-]
-`;
-
-exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
-Array [
-  null,
-  "MhRNxD4",
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `
-Array [
-  137773603,
-  "3F9HBxK",
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `
-Array [
-  410361374,
-  "aVd88fp",
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
-Array [
-  null,
-  "3loZrRf",
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `
-Array [
-  null,
-  "78SLiRw",
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `
-Array [
-  null,
-  "0ilsf82",
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `
-Array [
-  null,
-  "LjS9MbU",
-]
-`;
-
-exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `
-Array [
-  null,
-  null,
-]
-`;
diff --git a/js/test/__snapshots__/table-tests.ts.snap b/js/test/__snapshots__/table-tests.ts.snap
deleted file mode 100644
index 401b992d9d7b5..0000000000000
--- a/js/test/__snapshots__/table-tests.ts.snap
+++ /dev/null
@@ -1,1815 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`dictionary file Arrow Table creates a Table from Arrow buffers 1`] = `"example-csv"`;
-
-exports[`dictionary file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
-
-exports[`dictionary file Arrow Table creates a Table from Arrow buffers 3`] = `3`;
-
-exports[`dictionary file Arrow Table creates a Table from Arrow buffers 4`] = `
-Array [
-  "Hermione",
-  25,
-  Float32Array [
-    -53.235599517822266,
-    40.231998443603516,
-  ],
-]
-`;
-
-exports[`dictionary file Arrow Table creates a Table from Arrow buffers 5`] = `
-Array [
-  "Severus",
-  30,
-  Float32Array [
-    -62.22999954223633,
-    3,
-  ],
-]
-`;
-
-exports[`dictionary file Arrow Table creates a Table from Arrow buffers 6`] = `
-Array [
-  "Harry",
-  20,
-  Float32Array [
-    23,
-    -100.23652648925781,
-  ],
-]
-`;
-
-exports[`dictionary file Arrow Table enumerates Table rows 1`] = `
-Object {
-  "example-csv": Array [
-    "Hermione",
-    25,
-    Float32Array [
-      -53.235599517822266,
-      40.231998443603516,
-    ],
-  ],
-}
-`;
-
-exports[`dictionary file Arrow Table enumerates Table rows 2`] = `
-Object {
-  "example-csv": Array [
-    "Severus",
-    30,
-    Float32Array [
-      -62.22999954223633,
-      3,
-    ],
-  ],
-}
-`;
-
-exports[`dictionary file Arrow Table enumerates Table rows 3`] = `
-Object {
-  "example-csv": Array [
-    "Harry",
-    20,
-    Float32Array [
-      23,
-      -100.23652648925781,
-    ],
-  ],
-}
-`;
-
-exports[`dictionary file Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  Array [
-    "Hermione",
-    25,
-    Float32Array [
-      -53.235599517822266,
-      40.231998443603516,
-    ],
-  ],
-]
-`;
-
-exports[`dictionary file Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  Array [
-    "Severus",
-    30,
-    Float32Array [
-      -62.22999954223633,
-      3,
-    ],
-  ],
-]
-`;
-
-exports[`dictionary file Arrow Table enumerates Table rows compact 3`] = `
-Array [
-  Array [
-    "Harry",
-    20,
-    Float32Array [
-      23,
-      -100.23652648925781,
-    ],
-  ],
-]
-`;
-
-exports[`dictionary file Arrow Table toString() prints a pretty Table 1`] = `
-"                                                     example-csv
-[\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}]
-                   [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}]
-                   [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]"
-`;
-
-exports[`dictionary file Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,                                                      example-csv
-    0, [\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}]
-    1,                    [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}]
-    2,                    [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]"
-`;
-
-exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 1`] = `"example-csv"`;
-
-exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
-
-exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 3`] = `3`;
-
-exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 4`] = `
-Array [
-  "Hermione",
-  25,
-  Float32Array [
-    -53.235599517822266,
-    40.231998443603516,
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 5`] = `
-Array [
-  "Severus",
-  30,
-  Float32Array [
-    -62.22999954223633,
-    3,
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 6`] = `
-Array [
-  "Harry",
-  20,
-  Float32Array [
-    23,
-    -100.23652648925781,
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow Table enumerates Table rows 1`] = `
-Object {
-  "example-csv": Array [
-    "Hermione",
-    25,
-    Float32Array [
-      -53.235599517822266,
-      40.231998443603516,
-    ],
-  ],
-}
-`;
-
-exports[`dictionary stream Arrow Table enumerates Table rows 2`] = `
-Object {
-  "example-csv": Array [
-    "Severus",
-    30,
-    Float32Array [
-      -62.22999954223633,
-      3,
-    ],
-  ],
-}
-`;
-
-exports[`dictionary stream Arrow Table enumerates Table rows 3`] = `
-Object {
-  "example-csv": Array [
-    "Harry",
-    20,
-    Float32Array [
-      23,
-      -100.23652648925781,
-    ],
-  ],
-}
-`;
-
-exports[`dictionary stream Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  Array [
-    "Hermione",
-    25,
-    Float32Array [
-      -53.235599517822266,
-      40.231998443603516,
-    ],
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  Array [
-    "Severus",
-    30,
-    Float32Array [
-      -62.22999954223633,
-      3,
-    ],
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow Table enumerates Table rows compact 3`] = `
-Array [
-  Array [
-    "Harry",
-    20,
-    Float32Array [
-      23,
-      -100.23652648925781,
-    ],
-  ],
-]
-`;
-
-exports[`dictionary stream Arrow Table toString() prints a pretty Table 1`] = `
-"                                                     example-csv
-[\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}]
-                   [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}]
-                   [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]"
-`;
-
-exports[`dictionary stream Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`dictionary stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,                                                      example-csv
-    0, [\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}]
-    1,                    [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}]
-    2,                    [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]"
-`;
-
-exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`;
-
-exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
-
-exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 3`] = `2`;
-
-exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 4`] = `
-Array [
-  "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
-  "Airbus",
-  1502880750,
-  Float32Array [
-    32.45663833618164,
-    1.8712350130081177,
-  ],
-]
-`;
-
-exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 5`] = `
-Array [
-  "50fb46f4-fefa-42c1-919c-0121974cdd00",
-  "Boeing",
-  1502880750,
-  Float32Array [
-    38.766666412353516,
-    -4.181231498718262,
-  ],
-]
-`;
-
-exports[`dictionary2 file Arrow Table enumerates Table rows 1`] = `
-Object {
-  "struct": Array [
-    "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
-    "Airbus",
-    1502880750,
-    Float32Array [
-      32.45663833618164,
-      1.8712350130081177,
-    ],
-  ],
-}
-`;
-
-exports[`dictionary2 file Arrow Table enumerates Table rows 2`] = `
-Object {
-  "struct": Array [
-    "50fb46f4-fefa-42c1-919c-0121974cdd00",
-    "Boeing",
-    1502880750,
-    Float32Array [
-      38.766666412353516,
-      -4.181231498718262,
-    ],
-  ],
-}
-`;
-
-exports[`dictionary2 file Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  Array [
-    "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
-    "Airbus",
-    1502880750,
-    Float32Array [
-      32.45663833618164,
-      1.8712350130081177,
-    ],
-  ],
-]
-`;
-
-exports[`dictionary2 file Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  Array [
-    "50fb46f4-fefa-42c1-919c-0121974cdd00",
-    "Boeing",
-    1502880750,
-    Float32Array [
-      38.766666412353516,
-      -4.181231498718262,
-    ],
-  ],
-]
-`;
-
-exports[`dictionary2 file Arrow Table toString() prints a pretty Table 1`] = `
-"                                                                                                      struct
- [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}]
-[\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]"
-`;
-
-exports[`dictionary2 file Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`dictionary2 file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,                                                                                                       struct
-    0,  [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}]
-    1, [\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]"
-`;
-
-exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`;
-
-exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
-
-exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 3`] = `2`;
-
-exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 4`] = `
-Array [
-  "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
-  "12345",
-  "Airbus",
-  1502880750,
-  Float32Array [
-    32.45663833618164,
-    1.8712350130081177,
-  ],
-]
-`;
-
-exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 5`] = `
-Array [
-  "50fb46f4-fefa-42c1-919c-0121974cdd00",
-  "67890",
-  "Boeing",
-  1502880750,
-  Float32Array [
-    38.766666412353516,
-    -4.181231498718262,
-  ],
-]
-`;
-
-exports[`multi_dictionary file Arrow Table enumerates Table rows 1`] = `
-Object {
-  "struct": Array [
-    "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
-    "12345",
-    "Airbus",
-    1502880750,
-    Float32Array [
-      32.45663833618164,
-      1.8712350130081177,
-    ],
-  ],
-}
-`;
-
-exports[`multi_dictionary file Arrow Table enumerates Table rows 2`] = `
-Object {
-  "struct": Array [
-    "50fb46f4-fefa-42c1-919c-0121974cdd00",
-    "67890",
-    "Boeing",
-    1502880750,
-    Float32Array [
-      38.766666412353516,
-      -4.181231498718262,
-    ],
-  ],
-}
-`;
-
-exports[`multi_dictionary file Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  Array [
-    "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
-    "12345",
-    "Airbus",
-    1502880750,
-    Float32Array [
-      32.45663833618164,
-      1.8712350130081177,
-    ],
-  ],
-]
-`;
-
-exports[`multi_dictionary file Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  Array [
-    "50fb46f4-fefa-42c1-919c-0121974cdd00",
-    "67890",
-    "Boeing",
-    1502880750,
-    Float32Array [
-      38.766666412353516,
-      -4.181231498718262,
-    ],
-  ],
-]
-`;
-
-exports[`multi_dictionary file Arrow Table toString() prints a pretty Table 1`] = `
-"                                                                                                              struct
- [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"12345\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}]
-[\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"67890\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]"
-`;
-
-exports[`multi_dictionary file Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`multi_dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,                                                                                                               struct
-    0,  [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"12345\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}]
-    1, [\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"67890\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]"
-`;
-
-exports[`multipart count Arrow Table creates a Table from Arrow buffers 1`] = `"row_count"`;
-
-exports[`multipart count Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`;
-
-exports[`multipart count Arrow Table creates a Table from Arrow buffers 3`] = `1`;
-
-exports[`multipart count Arrow Table creates a Table from Arrow buffers 4`] = `10000`;
-
-exports[`multipart count Arrow Table enumerates Table rows 1`] = `
-Object {
-  "row_count": 10000,
-}
-`;
-
-exports[`multipart count Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  10000,
-]
-`;
-
-exports[`multipart count Arrow Table toString() prints a pretty Table 1`] = `
-"row_count
-    10000"
-`;
-
-exports[`multipart count Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`multipart count Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index, row_count
-    0,     10000"
-`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 1`] = `"origin_lat"`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 2`] = `"FloatingPoint"`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 3`] = `5`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 4`] = `35.393089294433594`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 5`] = `35.393089294433594`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 6`] = `35.393089294433594`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 7`] = `29.533695220947266`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 8`] = `29.533695220947266`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 9`] = `"origin_lon"`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 11`] = `5`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 12`] = `-97.6007308959961`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 13`] = `-97.6007308959961`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 14`] = `-97.6007308959961`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 15`] = `-98.46977996826172`;
-
-exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 16`] = `-98.46977996826172`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows 1`] = `
-Object {
-  "origin_lat": 35.393089294433594,
-  "origin_lon": -97.6007308959961,
-}
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows 2`] = `
-Object {
-  "origin_lat": 35.393089294433594,
-  "origin_lon": -97.6007308959961,
-}
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows 3`] = `
-Object {
-  "origin_lat": 35.393089294433594,
-  "origin_lon": -97.6007308959961,
-}
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows 4`] = `
-Object {
-  "origin_lat": 29.533695220947266,
-  "origin_lon": -98.46977996826172,
-}
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows 5`] = `
-Object {
-  "origin_lat": 29.533695220947266,
-  "origin_lon": -98.46977996826172,
-}
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  35.393089294433594,
-  -97.6007308959961,
-]
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  35.393089294433594,
-  -97.6007308959961,
-]
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows compact 3`] = `
-Array [
-  35.393089294433594,
-  -97.6007308959961,
-]
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows compact 4`] = `
-Array [
-  29.533695220947266,
-  -98.46977996826172,
-]
-`;
-
-exports[`multipart latlong Arrow Table enumerates Table rows compact 5`] = `
-Array [
-  29.533695220947266,
-  -98.46977996826172,
-]
-`;
-
-exports[`multipart latlong Arrow Table toString() prints a pretty Table 1`] = `
-"        origin_lat,         origin_lon
-35.393089294433594,  -97.6007308959961
-35.393089294433594,  -97.6007308959961
-35.393089294433594,  -97.6007308959961
-29.533695220947266, -98.46977996826172
-29.533695220947266, -98.46977996826172"
-`;
-
-exports[`multipart latlong Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`multipart latlong Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,         origin_lat,         origin_lon
-    0, 35.393089294433594,  -97.6007308959961
-    1, 35.393089294433594,  -97.6007308959961
-    2, 35.393089294433594,  -97.6007308959961
-    3, 29.533695220947266, -98.46977996826172
-    4, 29.533695220947266, -98.46977996826172"
-`;
-
-exports[`multipart origins Arrow Table creates a Table from Arrow buffers 1`] = `"origin_city"`;
-
-exports[`multipart origins Arrow Table creates a Table from Arrow buffers 2`] = `"Utf8"`;
-
-exports[`multipart origins Arrow Table creates a Table from Arrow buffers 3`] = `5`;
-
-exports[`multipart origins Arrow Table creates a Table from Arrow buffers 4`] = `"Oklahoma City"`;
-
-exports[`multipart origins Arrow Table creates a Table from Arrow buffers 5`] = `"Oklahoma City"`;
-
-exports[`multipart origins Arrow Table creates a Table from Arrow buffers 6`] = `"Oklahoma City"`;
-
-exports[`multipart origins Arrow Table creates a Table from Arrow buffers 7`] = `"San Antonio"`;
-
-exports[`multipart origins Arrow Table creates a Table from Arrow buffers 8`] = `"San Antonio"`;
-
-exports[`multipart origins Arrow Table enumerates Table rows 1`] = `
-Object {
-  "origin_city": "Oklahoma City",
-}
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows 2`] = `
-Object {
-  "origin_city": "Oklahoma City",
-}
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows 3`] = `
-Object {
-  "origin_city": "Oklahoma City",
-}
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows 4`] = `
-Object {
-  "origin_city": "San Antonio",
-}
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows 5`] = `
-Object {
-  "origin_city": "San Antonio",
-}
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  "Oklahoma City",
-]
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  "Oklahoma City",
-]
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows compact 3`] = `
-Array [
-  "Oklahoma City",
-]
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows compact 4`] = `
-Array [
-  "San Antonio",
-]
-`;
-
-exports[`multipart origins Arrow Table enumerates Table rows compact 5`] = `
-Array [
-  "San Antonio",
-]
-`;
-
-exports[`multipart origins Arrow Table toString() prints a pretty Table 1`] = `
-"  origin_city
-Oklahoma City
-Oklahoma City
-Oklahoma City
-  San Antonio
-  San Antonio"
-`;
-
-exports[`multipart origins Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`multipart origins Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,   origin_city
-    0, Oklahoma City
-    1, Oklahoma City
-    2, Oklahoma City
-    3,   San Antonio
-    4,   San Antonio"
-`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 1`] = `"foo"`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 3`] = `5`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 4`] = `1`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 5`] = `null`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 6`] = `3`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 7`] = `4`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 8`] = `5`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 9`] = `"bar"`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 11`] = `5`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 12`] = `1`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 13`] = `null`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 14`] = `null`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 15`] = `4`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 16`] = `5`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 17`] = `"baz"`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 18`] = `"Utf8"`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 19`] = `5`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 20`] = `"aa"`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 21`] = `null`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 22`] = `null`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 23`] = `"bbb"`;
-
-exports[`simple file Arrow Table creates a Table from Arrow buffers 24`] = `"cccc"`;
-
-exports[`simple file Arrow Table enumerates Table rows 1`] = `
-Object {
-  "bar": 1,
-  "baz": "aa",
-  "foo": 1,
-}
-`;
-
-exports[`simple file Arrow Table enumerates Table rows 2`] = `
-Object {
-  "bar": null,
-  "baz": null,
-  "foo": null,
-}
-`;
-
-exports[`simple file Arrow Table enumerates Table rows 3`] = `
-Object {
-  "bar": null,
-  "baz": null,
-  "foo": 3,
-}
-`;
-
-exports[`simple file Arrow Table enumerates Table rows 4`] = `
-Object {
-  "bar": 4,
-  "baz": "bbb",
-  "foo": 4,
-}
-`;
-
-exports[`simple file Arrow Table enumerates Table rows 5`] = `
-Object {
-  "bar": 5,
-  "baz": "cccc",
-  "foo": 5,
-}
-`;
-
-exports[`simple file Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  1,
-  1,
-  "aa",
-]
-`;
-
-exports[`simple file Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  null,
-  null,
-  null,
-]
-`;
-
-exports[`simple file Arrow Table enumerates Table rows compact 3`] = `
-Array [
-  3,
-  null,
-  null,
-]
-`;
-
-exports[`simple file Arrow Table enumerates Table rows compact 4`] = `
-Array [
-  4,
-  4,
-  "bbb",
-]
-`;
-
-exports[`simple file Arrow Table enumerates Table rows compact 5`] = `
-Array [
-  5,
-  5,
-  "cccc",
-]
-`;
-
-exports[`simple file Arrow Table toString() prints a pretty Table 1`] = `
-" foo,  bar,  baz
-   1,    1,   aa
-null, null, null
-   3, null, null
-   4,    4,  bbb
-   5,    5, cccc"
-`;
-
-exports[`simple file Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`simple file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,  foo,  bar,  baz
-    0,    1,    1,   aa
-    1, null, null, null
-    2,    3, null, null
-    3,    4,    4,  bbb
-    4,    5,    5, cccc"
-`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 1`] = `"foo"`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 3`] = `5`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 4`] = `1`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 5`] = `null`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 6`] = `3`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 7`] = `4`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 8`] = `5`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 9`] = `"bar"`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 11`] = `5`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 12`] = `1`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 13`] = `null`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 14`] = `null`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 15`] = `4`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 16`] = `5`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 17`] = `"baz"`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 18`] = `"Utf8"`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 19`] = `5`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 20`] = `"aa"`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 21`] = `null`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 22`] = `null`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 23`] = `"bbb"`;
-
-exports[`simple stream Arrow Table creates a Table from Arrow buffers 24`] = `"cccc"`;
-
-exports[`simple stream Arrow Table enumerates Table rows 1`] = `
-Object {
-  "bar": 1,
-  "baz": "aa",
-  "foo": 1,
-}
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows 2`] = `
-Object {
-  "bar": null,
-  "baz": null,
-  "foo": null,
-}
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows 3`] = `
-Object {
-  "bar": null,
-  "baz": null,
-  "foo": 3,
-}
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows 4`] = `
-Object {
-  "bar": 4,
-  "baz": "bbb",
-  "foo": 4,
-}
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows 5`] = `
-Object {
-  "bar": 5,
-  "baz": "cccc",
-  "foo": 5,
-}
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  1,
-  1,
-  "aa",
-]
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  null,
-  null,
-  null,
-]
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows compact 3`] = `
-Array [
-  3,
-  null,
-  null,
-]
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows compact 4`] = `
-Array [
-  4,
-  4,
-  "bbb",
-]
-`;
-
-exports[`simple stream Arrow Table enumerates Table rows compact 5`] = `
-Array [
-  5,
-  5,
-  "cccc",
-]
-`;
-
-exports[`simple stream Arrow Table toString() prints a pretty Table 1`] = `
-" foo,  bar,  baz
-   1,    1,   aa
-null, null, null
-   3, null, null
-   4,    4,  bbb
-   5,    5, cccc"
-`;
-
-exports[`simple stream Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`simple stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,  foo,  bar,  baz
-    0,    1,    1,   aa
-    1, null, null, null
-    2,    3, null, null
-    3,    4,    4,  bbb
-    4,    5,    5, cccc"
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 1`] = `"struct_nullable"`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 3`] = `17`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 4`] = `null`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 5`] = `
-Array [
-  null,
-  "MhRNxD4",
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 6`] = `
-Array [
-  137773603,
-  "3F9HBxK",
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 7`] = `
-Array [
-  410361374,
-  "aVd88fp",
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 8`] = `null`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 9`] = `
-Array [
-  null,
-  "3loZrRf",
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 10`] = `null`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 11`] = `null`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 12`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 13`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 14`] = `null`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 15`] = `
-Array [
-  null,
-  "78SLiRw",
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 16`] = `null`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 17`] = `null`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 18`] = `
-Array [
-  null,
-  "0ilsf82",
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 19`] = `
-Array [
-  null,
-  "LjS9MbU",
-]
-`;
-
-exports[`struct file Arrow Table creates a Table from Arrow buffers 20`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 1`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 2`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "MhRNxD4",
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 3`] = `
-Object {
-  "struct_nullable": Array [
-    137773603,
-    "3F9HBxK",
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 4`] = `
-Object {
-  "struct_nullable": Array [
-    410361374,
-    "aVd88fp",
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 5`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 6`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "3loZrRf",
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 7`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 8`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 9`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    null,
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 10`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    null,
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 11`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 12`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "78SLiRw",
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 13`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 14`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 15`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "0ilsf82",
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 16`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "LjS9MbU",
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows 17`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    null,
-  ],
-}
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  Array [
-    null,
-    "MhRNxD4",
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 3`] = `
-Array [
-  Array [
-    137773603,
-    "3F9HBxK",
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 4`] = `
-Array [
-  Array [
-    410361374,
-    "aVd88fp",
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 5`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 6`] = `
-Array [
-  Array [
-    null,
-    "3loZrRf",
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 7`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 8`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 9`] = `
-Array [
-  Array [
-    null,
-    null,
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 10`] = `
-Array [
-  Array [
-    null,
-    null,
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 11`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 12`] = `
-Array [
-  Array [
-    null,
-    "78SLiRw",
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 13`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 14`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 15`] = `
-Array [
-  Array [
-    null,
-    "0ilsf82",
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 16`] = `
-Array [
-  Array [
-    null,
-    "LjS9MbU",
-  ],
-]
-`;
-
-exports[`struct file Arrow Table enumerates Table rows compact 17`] = `
-Array [
-  Array [
-    null,
-    null,
-  ],
-]
-`;
-
-exports[`struct file Arrow Table toString() prints a pretty Table 1`] = `
-"      struct_nullable
-                 null
-     [null,\\"MhRNxD4\\"]
-[137773603,\\"3F9HBxK\\"]
-[410361374,\\"aVd88fp\\"]
-                 null
-     [null,\\"3loZrRf\\"]
-                 null
-                 null
-          [null,null]
-          [null,null]
-                 null
-     [null,\\"78SLiRw\\"]
-                 null
-                 null
-     [null,\\"0ilsf82\\"]
-     [null,\\"LjS9MbU\\"]
-          [null,null]"
-`;
-
-exports[`struct file Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`struct file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,       struct_nullable
-    0,                  null
-    1,      [null,\\"MhRNxD4\\"]
-    2, [137773603,\\"3F9HBxK\\"]
-    3, [410361374,\\"aVd88fp\\"]
-    4,                  null
-    5,      [null,\\"3loZrRf\\"]
-    6,                  null
-    7,                  null
-    8,           [null,null]
-    9,           [null,null]
-   10,                  null
-   11,      [null,\\"78SLiRw\\"]
-   12,                  null
-   13,                  null
-   14,      [null,\\"0ilsf82\\"]
-   15,      [null,\\"LjS9MbU\\"]
-   16,           [null,null]"
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 1`] = `"struct_nullable"`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 3`] = `17`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 4`] = `null`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 5`] = `
-Array [
-  null,
-  "MhRNxD4",
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 6`] = `
-Array [
-  137773603,
-  "3F9HBxK",
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 7`] = `
-Array [
-  410361374,
-  "aVd88fp",
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 8`] = `null`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 9`] = `
-Array [
-  null,
-  "3loZrRf",
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 10`] = `null`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 11`] = `null`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 12`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 13`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 14`] = `null`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 15`] = `
-Array [
-  null,
-  "78SLiRw",
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 16`] = `null`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 17`] = `null`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 18`] = `
-Array [
-  null,
-  "0ilsf82",
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 19`] = `
-Array [
-  null,
-  "LjS9MbU",
-]
-`;
-
-exports[`struct stream Arrow Table creates a Table from Arrow buffers 20`] = `
-Array [
-  null,
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 1`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 2`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "MhRNxD4",
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 3`] = `
-Object {
-  "struct_nullable": Array [
-    137773603,
-    "3F9HBxK",
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 4`] = `
-Object {
-  "struct_nullable": Array [
-    410361374,
-    "aVd88fp",
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 5`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 6`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "3loZrRf",
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 7`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 8`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 9`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    null,
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 10`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    null,
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 11`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 12`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "78SLiRw",
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 13`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 14`] = `
-Object {
-  "struct_nullable": null,
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 15`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "0ilsf82",
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 16`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    "LjS9MbU",
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows 17`] = `
-Object {
-  "struct_nullable": Array [
-    null,
-    null,
-  ],
-}
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 1`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 2`] = `
-Array [
-  Array [
-    null,
-    "MhRNxD4",
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 3`] = `
-Array [
-  Array [
-    137773603,
-    "3F9HBxK",
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 4`] = `
-Array [
-  Array [
-    410361374,
-    "aVd88fp",
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 5`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 6`] = `
-Array [
-  Array [
-    null,
-    "3loZrRf",
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 7`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 8`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 9`] = `
-Array [
-  Array [
-    null,
-    null,
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 10`] = `
-Array [
-  Array [
-    null,
-    null,
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 11`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 12`] = `
-Array [
-  Array [
-    null,
-    "78SLiRw",
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 13`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 14`] = `
-Array [
-  null,
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 15`] = `
-Array [
-  Array [
-    null,
-    "0ilsf82",
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 16`] = `
-Array [
-  Array [
-    null,
-    "LjS9MbU",
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table enumerates Table rows compact 17`] = `
-Array [
-  Array [
-    null,
-    null,
-  ],
-]
-`;
-
-exports[`struct stream Arrow Table toString() prints a pretty Table 1`] = `
-"      struct_nullable
-                 null
-     [null,\\"MhRNxD4\\"]
-[137773603,\\"3F9HBxK\\"]
-[410361374,\\"aVd88fp\\"]
-                 null
-     [null,\\"3loZrRf\\"]
-                 null
-                 null
-          [null,null]
-          [null,null]
-                 null
-     [null,\\"78SLiRw\\"]
-                 null
-                 null
-     [null,\\"0ilsf82\\"]
-     [null,\\"LjS9MbU\\"]
-          [null,null]"
-`;
-
-exports[`struct stream Arrow Table toString() prints an empty Table 1`] = `""`;
-
-exports[`struct stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = `
-"Index,       struct_nullable
-    0,                  null
-    1,      [null,\\"MhRNxD4\\"]
-    2, [137773603,\\"3F9HBxK\\"]
-    3, [410361374,\\"aVd88fp\\"]
-    4,                  null
-    5,      [null,\\"3loZrRf\\"]
-    6,                  null
-    7,                  null
-    8,           [null,null]
-    9,           [null,null]
-   10,                  null
-   11,      [null,\\"78SLiRw\\"]
-   12,                  null
-   13,                  null
-   14,      [null,\\"0ilsf82\\"]
-   15,      [null,\\"LjS9MbU\\"]
-   16,           [null,null]"
-`;
diff --git a/js/test/arrows/file/dictionary.arrow b/js/test/arrows/file/dictionary.arrow
deleted file mode 100644
index 34d41db1f200159406c459b8c076d5ce0c3d30ec..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2522
zcmeHJ&1(};5T9+*G{mHqHnv(ta_B)s!H*ykC?w(`f?#L`@#bQlX&@hEla^FG<l;rV
z_&@kZc+N?|lLzTZaumda;Mw^5y?s;GjipqtoiK0S%)I%2yPL|!#)C)KhchAzB9fJP
zNl8x9c8^77tX#Z|l_ep`07*UISzsP(9&`!Tf>YFyCl*NI8<1N7r49A1Uz3Rk-wEvR
zLio|iYm>=D?S_ycE00v|-a0$whwl6poGAiD5*{f-vL43ThfM(<#?BTZKnj_r@FRGZ
zuGRg%zeb+Aq?OUn8Q|rH>9$L?ZnM)b^@3)7y?FD+Ey%;#mt^rTwKFFovhl8Q1!_($
z<jtJ=pzH(TKK3A90m}-|Z<x}(PTw?4P-=GSX0ZOuZ}kjfUO>lJWEtL)h|5{AXHOih
zg1!KX%z6W?>@oA;tk{pQh?7R_wCw;VMkrXGRg>48IPE+nwBy|A<L8$TrndV0F}Bb!
z{+zV|>(O=$|I45qzimpJ{h--txcvLUj$sb`iJStOf#2S4nX6BGyYdM<>mX!N<D#4g
z&kD7VR)xjgNo>+M#q1dO&)hQLnOoB4t^Hvy6ry{T@m@tC>`gegsGP9~nlJaXo%T6L
zm4jK(9FSI3KM3}!CTKUi9kXe6O|aA3f(!MS#eC;+_re+Dyz_BTePX)8eVhsJQg~Ri
zKQP7SunOaH-c&-=A9<}je!f5@ARhBlD4vSTCxQ_7b2}8<C&g`reT(ODT%I>B3OXsJ
zo^z=DIUK$SRQ|LxuKaj7dh@+-=jFG-NTghDp@73?R(3W=%`Do3c>m}kswb~!;&Po?
zTz=Hd;=TFTnQ30kku!_uaa=w=WBOGeN+}~Kud`5I=O^;6vby(vr24GAVc{EsZx~`3
zxCLa~*UpghB~(KqWflh5cJi|2!iAz&F6qxsUX1AZ4rPz@ZAwZT&gbMSdh!)@&(D8-
MMaQ^*75ks=H#cC^v;Y7A

diff --git a/js/test/arrows/file/dictionary2.arrow b/js/test/arrows/file/dictionary2.arrow
deleted file mode 100644
index 1537f54db79eeeb2243e91846994737099a8ea96..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2762
zcmeHJJ!n%=6h5zwHMFU=lGb7~Jg`F%14;YL9~5cn(4k<d0TmpY=Dk+~wh75A)XkxS
zgF=T66&w^qGP?^M96Gq^;NU3YAiB6}{J#6{H8CYq*Pig@+<Wdl=X~efoA)xmx_aYQ
zT4c9hBrc)E<eUu1utYJ9DN~<}P+Nm)9+ij;V#;-bVgV7L_`B7bfF`DZ1mcPeTN6sR
z_u1HINuTn0(kIeHfM3|j7za8=5#u%B2oM>us7;==KdlyX+rSubWy7m(7FMd2t)S4T
zSKK*gb|wQmdAeSsT(8uEYQ63MLgzQ}Vu)vWr{I$VMEb2z3z&idKz^U$YlF!b9RnuF
z0D3Wi55-b?$}I&YHbJdLl#Ov1gLJvFtMlH*xTb+@R)DeLw%iJ9WvENQVkz*7l})d}
zLBaL>()RseE~syN$LsCny^mbv{T!OTr%WR@lx5?K{WP&L#C@<OA(O}%2j>x-eN^3%
zJ*=@;D*$;m0g*G-D}0kP?@Uzfj;NT78wZAf;)9x3xEa(dTN}sDhF2s3m_9G6x-Ge@
zINDN#HoRGB9MRcJo-z4+|N7?X+nu3V40ZmqDnF(cIRx<MesX|<e*%6NWn5)sbrHpr
zv4~|2*0zK(g&eN}TxYCrnqw5+JbUI5^o?_#IOm4*FYIO4D=vNc;mr_f;ONS1B;+H&
zUQuG;cLDup_u12FWi?;=H;Ky<#V%%JuK)Wmh<R8Sd&IbFKzQ;PKv++=@C-X8tjRnJ
z+AGXOAvYuR?Tl~ioq1TZ@tLpG*S8xBRgV_}@(uyo^^fC8G=|d&>u;K1$iO=y{m1LG
z;oq5`cdlD{)z@uTIpA!#nmD!|EE$8=_q6VgQ`7cNUNr6W*#A)5P2crY9{1S4*Ta6G
z?WPV++52uDC!O;icZ)f1qq#l>n4XnV{<@R(^GQGFuO}TRHJdDDoKiBI%edK0I_>8?
z{~GsXd?)GS5tnpk%IRc2oi8U->6vss>y%wLmHHQ7H{HyCT;9J!nE824@Yx4z;pOhn
z>!YKS3$NclI&9xHb}9U=@Vtp>U>RWOJzeCH&eMd=h+5Z)K4~L;LpgnCvUR8L%<$Q~
Yf?fnbcz;gcnJ0Z`9^!X5ul~dS1zMm*_5c6?

diff --git a/js/test/arrows/file/multi_dictionary.arrow b/js/test/arrows/file/multi_dictionary.arrow
deleted file mode 100644
index 113d30da7856560455dc0b63229671825d1a9fdd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3482
zcmeHK%}Z2K6hALb2R}+fEwaUcE+ib78K3h-l8Ot1Br%}^7mhRU>3}-J_ymI%Eec!|
zxNuR>!XUzBv}zNyEN~I%qCX&_MO2HH)%5$ld$02t5n4!_JaXQ-_uO;O$M2r|<~cGo
zb?rt>#I6@<6Ok4<B@NOlA%BdAobdHxHDOJlua_oYa;(ULz;@aCS*e_ZB=>^f*Wn?p
z){ZYljNKyaEciW%)nTm7oDkW>;t$N}=d<=Rjqxn_Eg0EZl*Y%4UzG~uhrylTi}Ox#
zDKl9tEPI)iav?ut55yDDGuOPck}DTVUa?%=|6S)`{LR3h$vuiaDX>VLr6f^M_%OeI
z*oKjb4aqtbH2{wW{E;kQ_U5x*mOx-e+_*+-`e3>cvZv$T#JHpxR+fjc$?eH+RV!Vc
z2ajbvXS%TDWH`v=9XGqW=nZ-0Rp(&6!Ms1}yuSqUUW0Fj3gbaMjc*KPvQnwsfg~>D
z;6?BQu*gYk9Z!?rCD?Iq2e=tLeY@mju6yOe^85k(*$NlH>^0)R_#t@O@bd`EI;6og
z4!^Yf^6BNn*K5rK7)l>>5EkQY!j?W54+8PS@ZAlJI^eqv{TlLf+Q1rXm;@8oZ7^$i
zh9_%KeNF3nAA?{m6BF03{B12`J`R@5_&J7U8q(Cllj-mys54?d3#o<fQ}1?rFvJSu
zH~R50wOF@@{j85UKw$p`)Nxk2^i5^H#GJm!S2eS?ag0gCyC2MT`uePM3?UNxk+lkN
z<D4DJxyktlY}r4BJooaCoh-6~R?87c&>omMkXn%68Q@R!iD{p&H4ptBL34*-Ww2&m
z)$?sap6VQUZ|QdyY;G8aem#^y;u!`>)uhffsAnt^b2G%fLH|Zi+zhd1vuC7Go?BfR
zEjny&&{5Q1?Sp?)Ke09RH}&&9ZT7_C1NLAdnMy~MhkAflJ^wEJdUT>Y)%m|NL4Yv(
zS?UhfH;Dv-__yxwsSXZ1-&(yN1KZ4584qDS7&i(n7qqPPQ__0&dyv-jq}Kkf)(7jL
zYF+DV?KiaE^r@c8qeJYs4$*IGy{VhiMsRH&M?J5_g>lV&F_OAo@HhvcY}B2zlWscV
zrrf!RZAS+p*@T^qBvXleG7*cpDaRe8O~SP!E`HJ?c03o0q+{t^BpQpy(n&j)&qt$G
z{|eRpxdz$HzxHh2BAfYXm4-atwML)b`*yj#y?gZe&iyyl-X)6a-NXJN_kkC{bUn<M
z04SuvtoF9Tdsu_=L7DBp`-XXWK(%_MJ^rR5>V)q2o9g(Rir?qFufgxb|LvP<1D9dM
H|DWs+-cqT+

diff --git a/js/test/arrows/file/simple.arrow b/js/test/arrows/file/simple.arrow
deleted file mode 100644
index 838db6dc8eda5376aaaf977c5f96c50397582a5f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1642
zcmeHIJxc>Y5S{x-E`~!=1UW$=g@vVpg@xdyvk-(J*x1}DwgF44l$Ms3{sezQ3QK>C
zrQP}7+>FLVB_!H8VfN$A?7W@X-AjKkI62#uh2=aD>0os#tJ0K)v~-Ln2o)D;D;5K4
zL)k~rXXvHPPphiJg2plCThPpK`t2A-4oa*);8@d=0I^%(73C1=$r12CLggrn8+oi_
zM>%{Ly9?zWLYtGTmZnBxEL!idE=T<PPw}p?K2yI088dUnqcUON|0D<ZzoB;SF@+{j
zm7s_274dtPM~qBp^Zj29hq8zI*6~^6HLG!|>=K1qD@_LLSm6wo7*Sa3Cn)RIfVw0Y
zfD-loklcoj``mlSN9H$;kH5i@h)QmDkCPidn!?9Z{xtIr`2NlOKIa)y-0pK?n{1X4
z=ORDvi*w8ya(!OMAN^hqzczRB8t+;^>-8J_tfwpUUzw%(Go&cWvb=yS$iRD_;^pu~
prFaE=t<2^C?KYHW$}@hjgWmT8pS(n{|9`)h`+nV`u6_NFc>}#ma&iCw

diff --git a/js/test/arrows/file/struct.arrow b/js/test/arrows/file/struct.arrow
deleted file mode 100644
index 3d2c018e6c27c95057f2a18395a9dc6031e6b201..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2354
zcmeHJJ5L)y5MCPt7wZBk2y#J?B~nBw5RVv4cSZyvVqWKjR}<Ng3!qal*iI-Sg_1T>
z6s4k~Oa4I`{{(&lB_(a7NY3}|?Kp^s{6L5%V|}|jbMx4po7-{9b?24_Rns&dQtAP$
zZPlULRI9S|8bc_o<w`}hECNv!;0zNz7b}$txch&=#{v40=O<W)flmN^9z*vi&~HtH
ze+<L`>bC#`0A=(+is+wGdDvKpXyMleIejtO0puY>U<_|asV;b<MmtVIFI^Z39%KR|
zeUoQA+B^Yz0Fnh83zmUz6^cW43jcA$Jv$!<dFDPMh(le*@5Edv?v~byZ{C-DKb!O4
zsu&U!#AaT*O<rxiGu0B-=Pomenxf!s__TC>e?rC_v<DCM$a7;G$n`&R%u?jS%PjJJ
zqjynWzDABiY1%i>tA32!F>+3PZ?aFWNwxU?K&|QVQPY-z;pc@f8Ls-qN37n%QY`%I
zxY&rxo9<s?F;5-9Lj%^L)FYO+Ml&1k?B2w);mNTV<9jc&i|fgxmml`GS9e{no}fK-
z)VBV>w6m*goA-#&UDS5j1ee;DZ_$xWVP-wcK6z)5gufTCRbFbJ;&0~}qu0pfG@{&q
z?#j*&Ubi*rclBPXsd}vfYg-^S&f4w+o&s!FH)vU~zCB8165A>F`SjF!_TAd@Xfl)D
zbU%K`O?jhpD~ZxZ!h2O1*z^ltawxr(8Jo$yzOWVE-8OE2b_g3cIlB=%&N4;^kpA=4
j<eA?SNop`}iY+ehfz9d?{qM=u-d|@={*HX6^nazl^uV^z

diff --git a/js/test/arrows/multi/count/records.arrow b/js/test/arrows/multi/count/records.arrow
deleted file mode 100644
index 00d883762d36959727cd26f5c3e3e4dfafeba981..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 224
zcmbQkz`!5^#2~=KAjZJPz{<b@WbuFmn3)(PfRqD>4F+65J`V#6Pz4u|?*qgFK+Fi1
ggAhzW>OT-53Bj4@;tp^bh#AP{z@!A!VJz}!0H3Q0ZvX%Q

diff --git a/js/test/arrows/multi/count/schema.arrow b/js/test/arrows/multi/count/schema.arrow
deleted file mode 100644
index dfd24e9e0018c471a053b8621cb7ea8b46a5bd9e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 184
zcmXYqF%E<<3<Ng_?hHcc3W_TzD0l!5;SUt_AR!Rn;(6DFadeJ$?QAs8E$M@5HF~L#
zr5OuZ%B3o0IZ=hY2lPcPZcuM%-_oWG1D^2Ve|IKBBkGWs!5F0fM};2y#Ta|G8z*P4
RzXiyWzHisOZ0C9~dp%Qi3m5<Z

diff --git a/js/test/arrows/multi/latlong/records.arrow b/js/test/arrows/multi/latlong/records.arrow
deleted file mode 100644
index 563d12d175d4e39b10386f98c5614cb86a493424..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 352
zcmcb^z`!5^#2~=KAjZJPz{<b@WbuFmn3)(PfK&sB4F+65J`V#6Pz4u|UjxJfK+Fo3
zgAgo0>OT-53Bj4@;to(5m^h3N(||6HEYAc~glsQJ4>}(t);5XP35J<m-#CIWNSs(O
U+4%4w7;gV?<PZoGs~4mW0F=Zhr~m)}

diff --git a/js/test/arrows/multi/latlong/schema.arrow b/js/test/arrows/multi/latlong/schema.arrow
deleted file mode 100644
index 638b2ab622f8ed55ff85783a2c1ad54321b90021..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 264
zcmZ8cI|{-;6r7jDH3Sh0ixep=EImMu;4Q>T0wDqM7?xI^BFD2^XWnihKA86z-kaHl
z0XqO2Y!R?R?LCtf&I65e6{&_$Er>DqON^1rWzV@I_L<j?W@#(lX%A1<0}L-#qmne|
z`*0hk%Q#QEZ|N7X=8JSKJwYC4<|FazapK$(DT3by9}J)DRg<l#slWYuULWc53;*jL
ADgXcg

diff --git a/js/test/arrows/multi/origins/records.arrow b/js/test/arrows/multi/origins/records.arrow
deleted file mode 100644
index 49a8c407e176e107a9007cbed26b87e3c84608ed..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 224
zcmb7+Q3`+{5JV@-2odTdqT_e~kJw%7t)WPN4UGGCX7Q4=V^h?jSzZeiEf}3nMlvo6
l>`@aJfj#35`_jFSW8A2V8h-QC8fk$0e`;lU^$MH+=mDH!3Sj^M

diff --git a/js/test/arrows/multi/origins/schema.arrow b/js/test/arrows/multi/origins/schema.arrow
deleted file mode 100644
index 0d10fb0e2d1354e5a147f49b5283632d4fdde04e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1604
zcmeIxJ!lj`6bJCRGapA1Bua|d$Pxq#Q7lBQLUI>l5-+(k7firH-tE4-9d35!oY{?D
z7p!cw6C1I!5^U^j#6|==vD3yTSSZ+OBj<lMIh25nwR8M-zINWcnN7Y4A*R5W32|7A
zh+$F6+jta;V$x5Dv3zVOZwcY+0!+h{IF7jyF^bib=<SVRLKKGY;<*KWYzfYRk2Q%E
zgIIoVAIp~#Vw4cc?~hUl@Hhk);RNjE;`jGIAwKu}{SsD}@EbvYZ(Wf0hcM#%1yK~^
zsEw6vsc(cb?TI(&|3357{+TDRJ4&&b&+YgPP)gz`4CS%>IR?4<b4+7}pBck=1U|nx
z7!;p7QtS`<f7S=b=Zbr?gYiLsQ2SZG_ndDWD5L26^BjW!PQrPZg%!98*MUJn7kY39
z?!iNN3{T-Xyn?sz9zMb+*o80f9e%<vk{O3-n1NGp7B0XnT!uPqK?@XE=)n%$hX?Qo
zp1?DB0WaY-yn_$03t!;}^kEG5bOfeh22R8O^6xiLqmCjS)$oi>LO})pAhm>Is_B%h
zG;7ATjOuK%OM^P8Bq%e5aoyO|FRRB=QN*enORC3)wY*)IY3iDpZO3(Mwq?XR<aJUk
z+wnSUvzVR<R(ot+gLJSUxov!7UZ+M&Q#F}%+fpfRFr_If%q?2dp>$#X;)-z?RB2G9
z4(G`>D72I?0^sG85Q-X@_K`Ly>C#4~93G)FdR{fNn;Ba`lXP!B!4Z`*^O;Mf-jq6W
z#$s*Qab;-}soOCnYvkfQr?Qcb%UP>M%9JyU0G%>Yk%_TfQM}FmMk+>SNi%a5W4oCP
zDzTM_y+BHlTVTD-c7aoCWO_;AK3#qvRmvi^5iY6GH67BtWRQ^2OJo&tU|orEG$h(v
z){!JlU8AI1c1m_v2)P=)U*&05$pzQAO|EJjZlleLT{UYqajGi5jpAS(%~cusbgz(h
ict8K@)(@CUEJJFWRa|<EkIr_DZv{207;paN&+Hd!R}{7Y

diff --git a/js/test/arrows/stream/dictionary.arrow b/js/test/arrows/stream/dictionary.arrow
deleted file mode 100644
index 17ca48b3a97f51ad016faa004cc53a2a5e809eec..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1776
zcmbVLL2DCH5T0$)G{mHqHnv(t_Rxcff@l#56jJdJ!9$>4yt$ZX2qYWXY)dL0a`6ZF
zAN&!Xds6V=u_x(K5D$W9*YA7#rmU}Osk6+RH}l@id^7WQHYc(qB6V4miZrBZ_gG}k
z+9jx2TN#!bm@)ug1UIoZAy?onxWzN-<d71vL5(Fy`tW@CGs^<RE@A%w#*fb4&9Y4W
zX0QQOK&gIjouBG6cmD?NY=T8f0VPwi9_BiPPlyOd&KD+71xPFSNP(3by(o<~sPn9B
zZH#jPe7$eRqgHo3h||_28T7U`@7%r%dtUpBEdQZ)Vj=@gaEmKgF%5y6m{Lggf%FJ_
zh#<tW29Els^)OCN-z2R;+%way=h1Ltkn;*Uz9y@PRz_aViamSkWC!vUSY$q!Vr7qs
zgR^2kz93H(xvRDVoERyzI;*Cxm^kewEcD~t8RN(252m~R@(H#$Vg3c{12v%U5#p~x
z#?i>McG6@J_Z|K;*)_z#PvjifOrz0@p}F~NvL~OQvkp=nH7?6#=&VrtXpgYCH;GLZ
zw}_5;f5%pXPHbh1Tl>RaC>8f=<GU(S-kW@GMLTnm6fgJm^<ysZz|lNp15BSzlqCBd
zlZ*!A*zA}+lk8502%z!mnC>E8UOr*mcQKA@FP*IK5O?yIlwVft$FA7?tv=t*cS<V8
zqpo%P_d_0n{G8Wv^NGCtA4q<jd%4;ECvGe3n~&qHI^Ua5IxCfd3#$4hI9vs*e%=Mx
z{{Hyj&A0IW>#x%Tk#_rEb89=F)8<y}gWp4jD4tW-JHB1#=G#x3o8O=Rotxq!M$XO0
faaP@*GvjIum9&vm*MCTLouA0N?e+cln3>2g$Le<8

diff --git a/js/test/arrows/stream/simple.arrow b/js/test/arrows/stream/simple.arrow
deleted file mode 100644
index 2c68c0e44b0af2ed78a47edbd038ae8fa8ab574b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1188
zcmcIiJ5Iw;5M2La6IsYbgd7PZ7aV|so>x*(asiu&HWDROj*yaLaD)_;9EOt4%&d3$
zIDjGrHd;TwnSC>FH(o>{m57vbC=<z~(04qdSaFdlIFS_0)VLHYBSAhR7f52)ec$6?
zc7^v-WNkR>&J3d$Bh?>poGB!NdI5h?f00HmfkzUnN7vrV<Di`c)KGhdq-Sh<dJQpA
zC2Emf<6MCL!xz0foUgSm*X}CSqb3{o`+xS}{g0LB9ZO`6)QB1^uh4I-j})Dd^ZVZ}
zmvW9d5Bi+-k<GePcS&NMl{SNOoOun#XZ{3)bIV9s<XB;ed4EK(SI2$ttK%!H8{?ZF
za5SP3jCY-2_-cf2M)jom4fy?ye&6#yWH|48Vkdm;5$;8QelPB^tK@vw!e8rNF21L$
fFyJ#=x2yibpY!x&{|DPP|Aw?}*L63DJsjQuh?Gj^

diff --git a/js/test/arrows/stream/struct.arrow b/js/test/arrows/stream/struct.arrow
deleted file mode 100644
index 4e97b7084f6b7c62df23acfe9e200661946314e3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1884
zcmd5+O;1`u6us|>_(~{cQ4I?<B)Di?G*n8(9U7~~qNY$AbYnn$)#jys0P4b|F1zif
zOIPmnAGqNs@DsZ1vfCz2@SJ%w)HE8vrqN5z%-l2gW9D+-6fKdch=e5~LFpAotua`(
z;zdH(6A54^r1t`tDR2cZQf)Sy&>sKBIu0-%dG3D)Y~UDR>;`PleEZ%U^bsHi(7p#q
z0^}Hj5>P*pUHCYN=-?Lw&sfa%1-yj}%;AoSJVi8G^r=%vp_8Ln_B2Murp|oyc?Jvu
z6bC*IJpI^W#lihR{5bOdyq@O;Z5_l2DqH4%0^3S;zqVdo`&9G1QrX*-7z*I$W_<^A
zeZ%Tx(qpT8*I6VbA?RVm^y_>9jy34>H`T~{;~1#*AIa*a;f-!)!=GrozxnztJg%(d
zMDMraXg;fXPW<1@Ik~2^)1UihcOHlC?CCpxe*3k+RbuLuGkjW$*_VxKHzEJ^{3aLc
z6ao4**o&qgvHWkcu$5mtntqv@nV6e8dQ*D8k;%BbDQ{<WKks&ubg+ZoRv(xOc5`p@
zJuz}0y<OMIO>diTF>4Nmnf+|e$v1;y^hdth(oOG^{&ro_cHK<bZj8IIUD-S5bK6#a
zwce$+rneffw+&2>v$qq#D1c{F0n>w%$wGQ(GXHvhcBAxRy*QpJ<hJvN2jy9Jd}$?J
Z+e*9dD#>lH;%3Hj9}5!;<>ec0a0N6*h?f8W

diff --git a/js/test/integration-tests.ts b/js/test/integration-tests.ts
new file mode 100644
index 0000000000000..4147e862b19a4
--- /dev/null
+++ b/js/test/integration-tests.ts
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Arrow from './Arrow';
+import { zip } from 'ix/iterable/zip';
+import { config, formats } from './test-config';
+
+const { Table, readVectors } = Arrow;
+
+expect.extend({
+    toEqualVector(v1: any, v2: any) {
+
+        const format = (x: any, y: any, msg= ' ') => `${
+            this.utils.printExpected(x)}${
+                msg}${
+            this.utils.printReceived(y)
+        }`;
+
+        let getFailures = new Array<string>();
+        let propsFailures = new Array<string>();
+        let iteratorFailures = new Array<string>();
+        let allFailures = [
+            { title: 'get', failures: getFailures },
+            { title: 'props', failures: propsFailures },
+            { title: 'iterator', failures: iteratorFailures }
+        ];
+
+        let props = ['name', 'type', 'length', 'nullable', 'nullCount', 'metadata'];
+        for (let i = -1, n = props.length; ++i < n;) {
+            const prop = props[i];
+            if (this.utils.stringify(v1[prop]) !== this.utils.stringify(v2[prop])) {
+                propsFailures.push(`${prop}: ${format(v1[prop], v2[prop], ' !== ')}`);
+            }
+        }
+
+        for (let i = -1, n = v1.length; ++i < n;) {
+            let x1 = v1.get(i), x2 = v2.get(i);
+            if (this.utils.stringify(x1) !== this.utils.stringify(x2)) {
+                getFailures.push(`${i}: ${format(x1, x2, ' !== ')}`);
+            }
+        }
+
+        let i = -1;
+        for (let [x1, x2] of zip(v1, v2)) {
+            ++i;
+            if (this.utils.stringify(x1) !== this.utils.stringify(x2)) {
+                iteratorFailures.push(`${i}: ${format(x1, x2, ' !== ')}`);
+            }
+        }
+
+        return {
+            pass: allFailures.every(({ failures }) => failures.length === 0),
+            message: () => [
+                `${v1.name}: (${format('cpp', 'java', ' !== ')})\n`,
+                ...allFailures.map(({ failures, title }) =>
+                    !failures.length ? `` : [`${title}:`, ...failures].join(`\n`))
+            ].join('\n')
+        };
+    }
+});
+
+describe(`Integration`, () => {
+    for (const format of formats) {
+        describe(format, () => {
+            for (const [cppArrow, javaArrow] of zip(config.cpp[format], config.java[format])) {
+                describe(`${cppArrow.name}`, () => {
+                    testReaderIntegration(cppArrow.buffers, javaArrow.buffers);
+                    testTableFromBuffersIntegration(cppArrow.buffers, javaArrow.buffers);
+                });
+            }
+        });
+    }
+});
+
+function testReaderIntegration(cppBuffers: Uint8Array[], javaBuffers: Uint8Array[]) {
+    test(`cpp and java vectors report the same values`, () => {
+        expect.hasAssertions();
+        for (const [cppVectors, javaVectors] of zip(readVectors(cppBuffers), readVectors(javaBuffers))) {
+            expect(cppVectors.length).toEqual(javaVectors.length);
+            for (let i = -1, n = cppVectors.length; ++i < n;) {
+                (expect(cppVectors[i]) as any).toEqualVector(javaVectors[i]);
+            }
+        }
+    });
+}
+
+function testTableFromBuffersIntegration(cppBuffers: Uint8Array[], javaBuffers: Uint8Array[]) {
+    test(`cpp and java tables report the same values`, () => {
+        expect.hasAssertions();
+        const cppTable = Table.from(cppBuffers);
+        const javaTable = Table.from(javaBuffers);
+        const cppVectors = cppTable.columns;
+        const javaVectors = javaTable.columns;
+        expect(cppTable.length).toEqual(javaTable.length);
+        expect(cppVectors.length).toEqual(javaVectors.length);
+        for (let i = -1, n = cppVectors.length; ++i < n;) {
+            (expect(cppVectors[i]) as any).toEqualVector(javaVectors[i]);
+        }
+    });
+}
diff --git a/js/test/reader-tests.ts b/js/test/reader-tests.ts
index a7f9f4110237f..309cec9e23b14 100644
--- a/js/test/reader-tests.ts
+++ b/js/test/reader-tests.ts
@@ -15,36 +15,55 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { readBuffers } from './Arrow';
-import arrowTestConfigurations from './test-config';
+import Arrow from './Arrow';
+const { readVectors } = Arrow;
+import { config, sources, formats } from './test-config';
 
-for (let [name, ...buffers] of arrowTestConfigurations) {
-    describe(`${name} readBuffers`, () => {
-        test(`enumerates each batch as an Array of Vectors`, () => {
-            expect.hasAssertions();
-            for (let vectors of readBuffers(...buffers)) {
-                for (let vector of vectors) {
-                    expect(vector.name).toMatchSnapshot();
-                    expect(vector.type).toMatchSnapshot();
-                    expect(vector.length).toMatchSnapshot();
-                        for (let i = -1, n = vector.length; ++i < n;) {
-                        expect(vector.get(i)).toMatchSnapshot();
+describe(`readBuffers`, () => {
+    for (const source of sources) {
+        describe(source, () => {
+            for (const format of formats) {
+                describe(format, () => {
+                    for (const { name, buffers } of config[source][format]) {
+                        describe(name, () => {
+                            testReaderIterator(buffers);
+                            testVectorIterator(buffers);
+                        });
                     }
-                }
+                });
             }
         });
-        test(`vector iterators report the same values as get`, () => {
-            expect.hasAssertions();
-            for (let vectors of readBuffers(...buffers)) {
-                for (let vector of vectors) {
-                    let i = -1, n = vector.length;
-                    for (let v of vector) {
-                        expect(++i).toBeLessThan(n);
-                        expect(v).toEqual(vector.get(i));
-                    }
-                    expect(++i).toEqual(n);
+    }
+});
+
+function testReaderIterator(buffers: Uint8Array[]) {
+    test(`reads each batch as an Array of Vectors`, () => {
+        expect.hasAssertions();
+        for (const vectors of readVectors(buffers)) {
+            for (const vector of vectors) {
+                expect(vector.name).toMatchSnapshot();
+                expect(vector.type).toMatchSnapshot();
+                expect(vector.length).toMatchSnapshot();
+                for (let i = -1, n = vector.length; ++i < n;) {
+                    expect(vector.get(i)).toMatchSnapshot();
                 }
             }
-        });
+        }
+    });
+}
+
+function testVectorIterator(buffers: Uint8Array[]) {
+    test(`vector iterators report the same values as get`, () => {
+        expect.hasAssertions();
+        for (const vectors of readVectors(buffers)) {
+            for (const vector of vectors) {
+                let i = -1, n = vector.length;
+                for (let v of vector) {
+                    expect(++i).toBeLessThan(n);
+                    expect(v).toEqual(vector.get(i));
+                }
+                expect(++i).toEqual(n);
+            }
+        }
     });
 }
diff --git a/js/test/table-tests.ts b/js/test/table-tests.ts
index d0d70059e1561..5ec04a72ff33b 100644
--- a/js/test/table-tests.ts
+++ b/js/test/table-tests.ts
@@ -15,75 +15,122 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Table, readBuffers } from './Arrow';
-import arrowTestConfigurations from './test-config';
+import Arrow from './Arrow';
+const { Table, readVectors } = Arrow;
+import { config, sources, formats } from './test-config';
 
-for (let [name, ...buffers] of arrowTestConfigurations) {
-    describe(`${name} Table`, () => {
-        test(`creates a Table from Arrow buffers`, () => {
-            expect.hasAssertions();
-            const table = Table.from(...buffers);
-            for (const vector of table.columns) {
-                expect(vector.name).toMatchSnapshot();
-                expect(vector.type).toMatchSnapshot();
-                expect(vector.length).toMatchSnapshot();
-                for (let i = -1, n = vector.length; ++i < n;) {
-                    expect(vector.get(i)).toMatchSnapshot();
-                }
-            }
-        });
-        test(`vector iterators report the same values as get`, () => {
-            expect.hasAssertions();
-            const table = Table.from(...buffers);
-            for (const vector of table.columns) {
-                let i = -1, n = vector.length;
-                for (let v of vector) {
-                    expect(++i).toBeLessThan(n);
-                    expect(v).toEqual(vector.get(i));
-                }
-                expect(++i).toEqual(n);
-            }
-        });
-        test(`batch and Table Vectors report the same values`, () => {
-            expect.hasAssertions();
-            let rowsTotal = 0;
-            let table = Table.from(...buffers);
-            for (let vectors of readBuffers(...buffers)) {
-                let rowsNow = Math.max(...vectors.map((v) => v.length));
-                for (let vi = -1, vn = vectors.length; ++vi < vn;) {
-                    let v1 = vectors[vi];
-                    let v2 = table.columns[vi];
-                    expect(v1.name).toEqual(v2.name);
-                    expect(v1.type).toEqual(v2.type);
-                    for (let i = -1, n = v1.length; ++i < n;) {
-                        expect(v1.get(i)).toEqual(v2.get(i + rowsTotal));
+describe(`Table`, () => {
+    for (const source of sources) {
+        describe(source, () => {
+            for (const format of formats) {
+                describe(format, () => {
+                    for (const { name, buffers } of config[source][format]) {
+                        describe(name, () => {
+                            testTableFromBuffers(buffers);
+                            testColumnIterators(buffers);
+                            testReaderVectorsAndTableColumns(buffers);
+                            testTableRowIterator(buffers);
+                            testTableRowIteratorCompact(buffers);
+                            testEmptyTableToString();
+                            testTableToStringPretty(buffers);
+                            testTableToStringPrettyWithIndex(buffers);
+                        });
                     }
-                }
-                rowsTotal += rowsNow;
+                });
             }
         });
-        test(`enumerates Table rows`, () => {
-            expect.hasAssertions();
-            const table = Table.from(...buffers);
-            for (const row of table) {
-                expect(row!.toObject()).toMatchSnapshot();
+    }
+});
+
+function testTableFromBuffers(buffers: Uint8Array[]) {
+    test(`creates a Table from Arrow buffers`, () => {
+        expect.hasAssertions();
+        const table = Table.from(buffers);
+        for (const vector of table.columns) {
+            expect(vector.name).toMatchSnapshot();
+            expect(vector.type).toMatchSnapshot();
+            expect(vector.length).toMatchSnapshot();
+            for (let i = -1, n = vector.length; ++i < n;) {
+                expect(vector.get(i)).toMatchSnapshot();
             }
-        });
-        test(`enumerates Table rows compact`, () => {
-            expect.hasAssertions();
-            const table = Table.from(...buffers);
-            for (const row of table) {
-                expect(row!.toArray()).toMatchSnapshot();
+        }
+    });
+}
+
+function testColumnIterators(buffers: Uint8Array[]) {
+    test(`vector iterators report the same values as get`, () => {
+        expect.hasAssertions();
+        const table = Table.from(buffers);
+        for (const vector of table.columns) {
+            let i = -1, n = vector.length;
+            for (let v of vector) {
+                expect(++i).toBeLessThan(n);
+                expect(v).toEqual(vector.get(i));
             }
-        });
-        test(`toString() prints an empty Table`, () => {
-            expect(Table.from().toString()).toMatchSnapshot();
-        });
-        test(`toString() prints a pretty Table`, () => {
-            expect(Table.from(...buffers).toString()).toMatchSnapshot();
-        });
-        test(`toString({ index: true }) prints a pretty Table with an Index column`, () => {
-            expect(Table.from(...buffers).toString({ index: true })).toMatchSnapshot();
-        });
+            expect(++i).toEqual(n);
+        }
     });
 }
+
+function testReaderVectorsAndTableColumns(buffers: Uint8Array[]) {
+    test(`batch and Table Vectors report the same values`, () => {
+        expect.hasAssertions();
+        let rowsTotal = 0;
+        let table = Table.from(buffers);
+        for (let vectors of readVectors(buffers)) {
+            let rowsNow = Math.max(...vectors.map((v) => v.length));
+            for (let vi = -1, vn = vectors.length; ++vi < vn;) {
+                let v1 = vectors[vi];
+                let v2 = table.columns[vi];
+                expect(v1.name).toEqual(v2.name);
+                expect(v1.type).toEqual(v2.type);
+                for (let i = -1, n = v1.length; ++i < n;) {
+                    expect(v1.get(i)).toEqual(v2.get(i + rowsTotal));
+                }
+            }
+            rowsTotal += rowsNow;
+        }
+    });
+}
+
+function testTableRowIterator(buffers: Uint8Array[]) {
+    test(`enumerates Table rows`, () => {
+        expect.hasAssertions();
+        const table = Table.from(buffers);
+        expect(table.length).toMatchSnapshot();
+        expect(table.columns.length).toMatchSnapshot();
+        for (const row of table) {
+            expect(row!.toObject()).toMatchSnapshot();
+        }
+    });
+}
+
+function testTableRowIteratorCompact(buffers: Uint8Array[]) {
+    test(`enumerates Table rows compact`, () => {
+        expect.hasAssertions();
+        const table = Table.from(buffers);
+        expect(table.length).toMatchSnapshot();
+        expect(table.columns.length).toMatchSnapshot();
+        for (const row of table) {
+            expect(row!.toArray()).toMatchSnapshot();
+        }
+    });
+}
+
+function testEmptyTableToString() {
+    test(`toString() prints an empty Table`, () => {
+        expect(Table.from().toString()).toMatchSnapshot();
+    });
+}
+
+function testTableToStringPretty(buffers: Uint8Array[]) {
+    test(`toString() prints a pretty Table`, () => {
+        expect(Table.from(buffers).toString()).toMatchSnapshot();
+    });
+}
+
+function testTableToStringPrettyWithIndex(buffers: Uint8Array[]) {
+    test(`toString({ index: true }) prints a pretty Table with an Index column`, () => {
+        expect(Table.from(buffers).toString({ index: true })).toMatchSnapshot();
+    });
+}
\ No newline at end of file
diff --git a/js/test/test-config.ts b/js/test/test-config.ts
index 89de1cc6c70c9..d185ecc922c47 100644
--- a/js/test/test-config.ts
+++ b/js/test/test-config.ts
@@ -17,26 +17,36 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-const arrowFormats = ['file', 'stream'];
-const arrowFileNames = ['simple', 'struct', 'dictionary', 'dictionary2', 'multi_dictionary'];
-const multipartArrows = ['count', 'latlong', 'origins'];
-export let arrowTestConfigurations = [] as (string | Buffer)[][];
+import * as glob from 'glob';
 
-arrowTestConfigurations = arrowFormats.reduce((configs, format) => {
-    return arrowFileNames.reduce((configs, name) => {
-        const arrowPath = path.resolve(__dirname, `./arrows/${format}/${name}.arrow`);
-        try {
-            const arrowFile = fs.readFileSync(arrowPath);
-            return [...configs, [`${name} ${format} Arrow`, arrowFile]];
-        } catch (e) {}
-        return configs;
-    }, configs);
-}, arrowTestConfigurations);
+export const sources = (process.env.TEST_SOURCES
+    ? JSON.parse(process.env.TEST_SOURCES + '')
+    : [`cpp`, `java`]) as ['cpp' | 'java'];
 
-arrowTestConfigurations = multipartArrows.reduce((configs, folder) => {
-    const schemaPath = path.resolve(__dirname, `./arrows/multi/${folder}/schema.arrow`);
-    const recordsPath = path.resolve(__dirname, `./arrows/multi/${folder}/records.arrow`);
-    return [...configs, [`multipart ${folder} Arrow`, fs.readFileSync(schemaPath), fs.readFileSync(recordsPath)]];
-}, arrowTestConfigurations);
+export const formats = (process.env.TEST_FORMATS
+    ? JSON.parse(process.env.TEST_FORMATS + '')
+    : [`file`, `stream`]) as ['file' | 'stream'];
 
-export default arrowTestConfigurations;
+export const config = sources.reduce((sources, source) => ({
+    ...sources,
+    [source]: formats.reduce((formats, format) => ({
+        ...formats,
+        [format]: loadArrows(source, format)
+    }), {})
+}), {}) as {
+    [k in 'cpp' | 'java']: {
+        [k in 'file' | 'stream']: Arrows
+    }
+};
+
+export type Arrows = { name: string, buffers: Uint8Array[] }[];
+
+function loadArrows(source: string, format: string) {
+    const arrows = [];
+    const filenames = glob.sync(path.resolve(__dirname, `data/${source}/${format}`, `*.arrow`));
+    for (const filename of filenames) {
+        const { name } = path.parse(filename);
+        arrows.push({ name, buffers: [fs.readFileSync(filename)] });
+    }
+    return arrows as Arrows;
+}
diff --git a/js/test/tsconfig.json b/js/test/tsconfig.json
index c1ae204212b8f..838bb1d7041e3 100644
--- a/js/test/tsconfig.json
+++ b/js/test/tsconfig.json
@@ -2,7 +2,7 @@
   "extends": "../tsconfig.json",
   "include": ["./**/*.ts"],
   "compilerOptions": {
-    "target": "ESNEXT",
+    "target": "es2015",
     "module": "commonjs",
     "allowJs": true,
     "importHelpers": false,
diff --git a/js/test/vector-tests.ts b/js/test/vector-tests.ts
index 0eca2327bff43..0aaba19ee9144 100644
--- a/js/test/vector-tests.ts
+++ b/js/test/vector-tests.ts
@@ -15,11 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { flatbuffers } from 'flatbuffers';
-import Long = flatbuffers.Long;
+import Arrow from './Arrow';
 import {
+    TypedArray,
+    TypedArrayConstructor,
+    NumericVectorConstructor,
+} from './Arrow';
+
+const {
     BoolVector,
-    TypedVector,
     Int64Vector,
     Uint64Vector,
     Int8Vector,
@@ -28,28 +32,29 @@ import {
     Uint8Vector,
     Uint16Vector,
     Uint32Vector,
+    Float16Vector,
     Float32Vector,
     Float64Vector,
-} from './Arrow';
+} = Arrow;
 
-const LongVectors = {
-    Int64Vector: [Int64Vector, Int32Array],
-    Uint64Vector: [Uint64Vector, Uint32Array]
+const FixedSizeVectors = {
+    Int64Vector: [Int64Vector, Int32Array] as [NumericVectorConstructor<number, any>, any],
+    Uint64Vector: [Uint64Vector, Uint32Array] as [NumericVectorConstructor<number, any>, any]
 };
 
-const TypedVectors = {
-    Int8Vector: [Int8Vector, Int8Array],
-    Int16Vector: [Int16Vector, Int16Array],
-    Int32Vector: [Int32Vector, Int32Array],
-    Uint8Vector: [Uint8Vector, Uint8Array],
-    Uint16Vector: [Uint16Vector, Uint16Array],
-    Uint32Vector: [Uint32Vector, Uint32Array],
-    Float32Vector: [Float32Vector, Float32Array],
-    Float64Vector: [Float64Vector, Float64Array]
+const FixedWidthVectors = {
+    Int8Vector: [Int8Vector, Int8Array] as [NumericVectorConstructor<number, any>, any],
+    Int16Vector: [Int16Vector, Int16Array] as [NumericVectorConstructor<number, any>, any],
+    Int32Vector: [Int32Vector, Int32Array] as [NumericVectorConstructor<number, any>, any],
+    Uint8Vector: [Uint8Vector, Uint8Array] as [NumericVectorConstructor<number, any>, any],
+    Uint16Vector: [Uint16Vector, Uint16Array] as [NumericVectorConstructor<number, any>, any],
+    Uint32Vector: [Uint32Vector, Uint32Array] as [NumericVectorConstructor<number, any>, any],
+    Float32Vector: [Float32Vector, Float32Array] as [NumericVectorConstructor<number, any>, any],
+    Float64Vector: [Float64Vector, Float64Array] as [NumericVectorConstructor<number, any>, any]
 };
 
-const longVectors = toMap<[typeof TypedVector, any]>(LongVectors, Object.keys(LongVectors));
-const byteVectors = toMap<[typeof TypedVector, any]>(TypedVectors, Object.keys(TypedVectors));
+const fixedSizeVectors = toMap(FixedSizeVectors, Object.keys(FixedSizeVectors));
+const fixedWidthVectors = toMap(FixedWidthVectors, Object.keys(FixedWidthVectors));
 const bytes = Array.from(
     { length: 5 },
     () => Uint8Array.from(
@@ -127,30 +132,66 @@ describe(`BoolVector`, () => {
     });
 });
 
-for (const [VectorName, [VectorType, ArrayType]] of longVectors) {
+describe('Float16Vector', () => {
+    const values = concatTyped(Uint16Array, ...bytes);
+    const vector = bytes
+        .map((b) => new Float16Vector({ data: new Uint16Array(b.buffer) }))
+        .reduce((v: any, v2) => v.concat(v2));
+    const n = values.length;
+    const clamp = (x: number) => Math.min((x -  32767) / 32767, 1);
+    test(`gets expected values`, () => {
+        let i = -1;
+        while (++i < n) {
+            expect(vector.get(i)).toEqual(clamp(values[i]));
+        }
+    });
+    test(`iterates expected values`, () => {
+        expect.hasAssertions();
+        let i = -1;
+        for (let v of vector) {
+            expect(++i).toBeLessThan(n);
+            expect(v).toEqual(clamp(values[i]));
+        }
+    });
+    test(`slices the entire array`, () => {
+        expect(vector.slice()).toEqual(values);
+    });
+    test(`slice returns a TypedArray`, () => {
+        expect(vector.slice()).toBeInstanceOf(Uint16Array);
+    });
+    test(`slices from -20 to length`, () => {
+        expect(vector.slice(-20)).toEqual(values.slice(-20));
+    });
+    test(`slices from 0 to -20`, () => {
+        expect(vector.slice(0, -20)).toEqual(values.slice(0, -20));
+    });
+    test(`slices the array from 0 to length - 20`, () => {
+        expect(vector.slice(0, n - 20)).toEqual(values.slice(0, n - 20));
+    });
+    test(`slices the array from 0 to length + 20`, () => {
+        expect(vector.slice(0, n + 20)).toEqual(
+            concatTyped(Uint16Array, values, values.slice(0, 20)));
+    });
+});
+
+for (const [VectorName, [VectorType, ArrayType]] of fixedSizeVectors) {
     describe(`${VectorName}`, () => {
         const values = concatTyped(ArrayType, ...bytes);
         const vector = bytes
-            .map((b) => new VectorType<Long, any>({
-                data: new ArrayType(b.buffer)
-            }))
+            .map((b) => new VectorType({ data: new ArrayType(b.buffer) }))
             .reduce((v: any, v2) => v.concat(v2));
         const n = values.length * 0.5;
         test(`gets expected values`, () => {
             let i = -1;
             while (++i < n) {
-                expect(vector.get(i)).toEqual(new Long(
-                    values[i * 2], values[i * 2 + 1]
-                ));
+                expect(vector.get(i)).toEqual(values.slice(2 * i, 2 * (i + 1)));
             }
         });
         test(`iterates expected values`, () => {
             let i = -1;
             for (let v of vector) {
                 expect(++i).toBeLessThan(n);
-                expect(v).toEqual(new Long(
-                    values[i * 2], values[i * 2 + 1]
-                ));
+                expect(v).toEqual(values.slice(2 * i, 2 * (i + 1)));
             }
         });
         test(`slices the entire array`, () => {
@@ -175,13 +216,11 @@ for (const [VectorName, [VectorType, ArrayType]] of longVectors) {
     });
 }
 
-for (const [VectorName, [VectorType, ArrayType]] of byteVectors) {
+for (const [VectorName, [VectorType, ArrayType]] of fixedWidthVectors) {
     describe(`${VectorName}`, () => {
         const values = concatTyped(ArrayType, ...bytes);
         const vector = bytes
-            .map((b) => new VectorType<number, any>({
-                data: new ArrayType(b.buffer)
-            }))
+            .map((b) => new VectorType({ data: new ArrayType(b.buffer) }))
             .reduce((v: any, v2) => v.concat(v2));
 
         const n = values.length;
@@ -221,14 +260,14 @@ for (const [VectorName, [VectorType, ArrayType]] of byteVectors) {
     });
 }
 
-function toMap<T>(entries: any, keys: string[]) {
+function toMap<T>(entries: Record<string, T>, keys: string[]) {
     return keys.reduce((map, key) => {
         map.set(key, entries[key] as T);
         return map;
     }, new Map<string, T>());
 }
 
-function concatTyped(ArrayType: any, ...bytes: any[]) {
+function concatTyped<T extends TypedArray>(ArrayType: TypedArrayConstructor<T>, ...bytes: any[]) {
     const BPE = ArrayType.BYTES_PER_ELEMENT;
     return bytes.reduce((v, bytes) => {
         const l = bytes.byteLength / BPE;
@@ -237,5 +276,5 @@ function concatTyped(ArrayType: any, ...bytes: any[]) {
         a.set(v);
         a.set(b, v.length);
         return a;
-    }, new ArrayType(0)) as Array<number>;
+    }, new ArrayType(0)) as T;
 }
\ No newline at end of file
diff --git a/js/tsconfig/tsconfig.es5.cls.json b/js/tsconfig/tsconfig.es5.cls.json
index 6e6f213b4d6e5..502432da04348 100644
--- a/js/tsconfig/tsconfig.es5.cls.json
+++ b/js/tsconfig/tsconfig.es5.cls.json
@@ -4,8 +4,6 @@
   "compilerOptions": {
     "target": "ES5",
     "module": "es2015",
-    "declaration": false,
-    "noEmitHelpers": true,
-    "importHelpers": false
+    "declaration": false
   }
 }

From cb5da9c1bc71fafd76f5cafc60ef237385a36b2b Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 20 Nov 2017 09:27:02 -0500
Subject: [PATCH 1264/1644] ARROW-1778: [Python] Link parquet-cpp statically,
 privately in manylinux1 wheels

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1331 from xhochy/ARROW-1778 and squashes the following commits:

65cf4c6a [Uwe L. Korn] ARROW-1778: [Python] Link parquet-cpp statically, privately in manylinux1 wheels
---
 cpp/cmake_modules/FindThrift.cmake  | 102 ++++++++++++++++++++++++++++
 python/CMakeLists.txt               |  46 +++++++++----
 python/manylinux1/Dockerfile-x86_64 |   2 +-
 python/manylinux1/build_arrow.sh    |   6 +-
 python/setup.py                     |   7 +-
 5 files changed, 145 insertions(+), 18 deletions(-)
 create mode 100644 cpp/cmake_modules/FindThrift.cmake

diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake
new file mode 100644
index 0000000000000..25f50825c3735
--- /dev/null
+++ b/cpp/cmake_modules/FindThrift.cmake
@@ -0,0 +1,102 @@
+# Copyright 2012 Cloudera Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# - Find Thrift (a cross platform RPC lib/tool)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  Thrift_HOME - When set, this path is inspected instead of standard library
+#                locations as the root of the Thrift installation.
+#                The environment variable THRIFT_HOME overrides this variable.
+#
+# This module defines
+#  THRIFT_VERSION, version string of ant if found
+#  THRIFT_INCLUDE_DIR, where to find THRIFT headers
+#  THRIFT_CONTRIB_DIR, where contrib thrift files (e.g. fb303.thrift) are installed
+#  THRIFT_STATIC_LIB, THRIFT static library
+#  THRIFT_FOUND, If false, do not try to use ant
+
+# prefer the thrift version supplied in THRIFT_HOME
+if( NOT "${THRIFT_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "${THRIFT_HOME}" _native_path )
+    list( APPEND _thrift_roots ${_native_path} )
+elseif ( Thrift_HOME )
+    list( APPEND _thrift_roots ${Thrift_HOME} )
+endif()
+
+message(STATUS "THRIFT_HOME: ${THRIFT_HOME}")
+find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h HINTS
+  ${_thrift_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "include"
+)
+
+find_path(THRIFT_CONTRIB_DIR share/fb303/if/fb303.thrift HINTS
+  ${_thrift_roots}
+  NO_DEFAULT_PATH
+)
+
+if (MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX)
+  set(THRIFT_MSVC_STATIC_LIB_SUFFIX md)
+endif()
+
+find_library(THRIFT_STATIC_LIB NAMES
+  ${CMAKE_STATIC_LIBRARY_PREFIX}thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
+  HINTS ${_thrift_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib"
+)
+
+find_program(THRIFT_COMPILER thrift HINTS
+  ${_thrift_roots}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "bin"
+)
+
+if (THRIFT_STATIC_LIB)
+  set(THRIFT_FOUND TRUE)
+  exec_program(${THRIFT_COMPILER}
+    ARGS -version OUTPUT_VARIABLE THRIFT_VERSION RETURN_VALUE THRIFT_RETURN)
+else ()
+  set(THRIFT_FOUND FALSE)
+endif ()
+
+if (THRIFT_FOUND)
+  if (NOT Thrift_FIND_QUIETLY)
+    message(STATUS "Thrift version: ${THRIFT_VERSION}")
+  endif ()
+else ()
+  if (NOT Thrift_FIND_QUIETLY)
+    set(THRIFT_ERR_MSG "Thrift compiler/libraries NOT found: ${THRIFT_RETURN}")
+    set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} (${THRIFT_INCLUDE_DIR}, ${THRIFT_STATIC_LIB}).")
+    if ( _thrift_roots )
+      set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} Looked in ${_thrift_roots}.")
+    else ()
+      set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} Looked in system search paths.")
+    endif ()
+    if ( Thrift_FIND_REQUIRED )
+      message(FATAL_ERROR "${THRIFT_ERR_MSG}")
+    else ()
+      message(STATUS "${THRIFT_ERR_MSG}")
+    endif ()
+  endif ()
+endif ()
+
+
+mark_as_advanced(
+  THRIFT_STATIC_LIB
+  THRIFT_COMPILER
+  THRIFT_INCLUDE_DIR
+)
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 8c7348298db5f..00a4ee68737c6 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -61,6 +61,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUILD_PARQUET
     "Build the PyArrow Parquet integration"
     OFF)
+  option(PYARROW_PARQUET_USE_SHARED
+    "Rely on parquet shared libraries where relevant"
+    ON)
   option(PYARROW_BUILD_PLASMA
     "Build the PyArrow Plasma integration"
     OFF)
@@ -282,24 +285,41 @@ if (PYARROW_BUILD_PARQUET)
   endif()
   include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
 
-  if (PYARROW_BUNDLE_ARROW_CPP)
-    bundle_arrow_lib(PARQUET_SHARED_LIB
-      ABI_VERSION ${PARQUET_ABI_VERSION}
-      SO_VERSION ${PARQUET_SO_VERSION})
+  if (PYARROW_PARQUET_USE_SHARED)
+    if (PYARROW_BUNDLE_ARROW_CPP)
+      bundle_arrow_lib(PARQUET_SHARED_LIB
+        ABI_VERSION ${PARQUET_ABI_VERSION}
+        SO_VERSION ${PARQUET_SO_VERSION})
+      if (MSVC)
+        bundle_arrow_implib(PARQUET_SHARED_IMP_LIB)
+      endif()
+    endif()
     if (MSVC)
-      bundle_arrow_implib(PARQUET_SHARED_IMP_LIB)
+      ADD_THIRDPARTY_LIB(parquet
+        SHARED_LIB ${PARQUET_SHARED_IMP_LIB})
+    else()
+      ADD_THIRDPARTY_LIB(parquet
+        SHARED_LIB ${PARQUET_SHARED_LIB})
     endif()
-  endif()
-  if (MSVC)
-    ADD_THIRDPARTY_LIB(parquet
-      SHARED_LIB ${PARQUET_SHARED_IMP_LIB})
+    set(LINK_LIBS
+      ${LINK_LIBS}
+      parquet_shared)
   else()
+    find_package(Thrift)
+    set(Boost_USE_STATIC_LIBS ON)
+    find_package(Boost COMPONENTS regex REQUIRED)
+    ADD_THIRDPARTY_LIB(boost_regex
+      STATIC_LIB ${Boost_REGEX_LIBRARY_RELEASE})
     ADD_THIRDPARTY_LIB(parquet
-      SHARED_LIB ${PARQUET_SHARED_LIB})
+      STATIC_LIB ${PARQUET_STATIC_LIB})
+    ADD_THIRDPARTY_LIB(thrift
+      STATIC_LIB ${THRIFT_STATIC_LIB})
+    set(LINK_LIBS
+      ${LINK_LIBS}
+      parquet_static
+      thrift_static
+      boost_regex_static)
   endif()
-  set(LINK_LIBS
-    ${LINK_LIBS}
-    parquet_shared)
   set(CYTHON_EXTENSIONS
     ${CYTHON_EXTENSIONS}
     _parquet)
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 69d8d3dbf8cef..919a32be715b0 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -29,5 +29,5 @@ RUN /check_arrow_visibility.sh
 WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BOOST_USE_SHARED=OFF -GNinja .
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BUILD_SHARED=OFF -DPARQUET_BUILD_STATIC=ON -DPARQUET_BOOST_USE_SHARED=OFF -GNinja .
 RUN ninja install
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 074bd0056a948..ced4556e92ad5 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -40,11 +40,11 @@ cd /arrow/python
 # PyArrow build configuration
 export PYARROW_BUILD_TYPE='release'
 export PYARROW_WITH_PARQUET=1
+export PYARROW_WITH_STATIC_PARQUET=1
 export PYARROW_WITH_PLASMA=1
 export PYARROW_BUNDLE_ARROW_CPP=1
-# Need as otherwise arrow_io is sometimes not linked
-export LDFLAGS="-Wl,--no-as-needed"
 export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
+export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr'
 # Ensure the target directory exists
 mkdir -p /io/dist
 
@@ -65,7 +65,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     # Clear output directory
     rm -rf dist/
     echo "=== (${PYTHON}) Building wheel ==="
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-static-parquet --bundle-arrow-cpp
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
     echo "=== (${PYTHON}) Test the existence of optional modules ==="
diff --git a/python/setup.py b/python/setup.py
index ccab8fb6581f2..2270cf79c2a8b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -82,6 +82,7 @@ def run(self):
     user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
                      ('build-type=', None, 'build type (debug or release)'),
                      ('with-parquet', None, 'build the Parquet extension'),
+                     ('with-static-parquet', None, 'link parquet statically'),
                      ('with-plasma', None, 'build the Plasma extension'),
                      ('bundle-arrow-cpp', None,
                       'bundle the Arrow C++ libraries')] +
@@ -102,6 +103,8 @@ def initialize_options(self):
 
         self.with_parquet = strtobool(
             os.environ.get('PYARROW_WITH_PARQUET', '0'))
+        self.with_static_parquet = strtobool(
+            os.environ.get('PYARROW_WITH_STATIC_PARQUET', '0'))
         self.with_plasma = strtobool(
             os.environ.get('PYARROW_WITH_PLASMA', '0'))
         self.bundle_arrow_cpp = strtobool(
@@ -144,6 +147,8 @@ def _run_cmake(self):
 
         if self.with_parquet:
             cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
+        if self.with_static_parquet:
+            cmake_options.append('-DPYARROW_PARQUET_USE_SHARED=off')
 
         if self.with_plasma:
             cmake_options.append('-DPYARROW_BUILD_PLASMA=on')
@@ -225,7 +230,7 @@ def _run_cmake(self):
             move_shared_libs(build_prefix, build_lib, "arrow_python")
             if self.with_plasma:
                 move_shared_libs(build_prefix, build_lib, "plasma")
-            if self.with_parquet:
+            if self.with_parquet and not self.with_static_parquet:
                 move_shared_libs(build_prefix, build_lib, "parquet")
 
         print('Bundling includes: ' + pjoin(build_prefix, 'include'))

From 284e6c9fbf88e77809e4b7376d9439265ad0d20d Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 20 Nov 2017 14:21:40 -0500
Subject: [PATCH 1265/1644] ARROW-1826: [JAVA] Avoid branching in copyFrom for
 fixed width scalars

cc @jacques-n , @BryanCutler , @icexelloss

Note: there was no performance regression or different behavior introduced as part of refactor patches.

This is something I had noticed in the old code during refactoring but lost track of it. copyFrom function for fixed width scalars does a double check if the cell in the source vector is NULL or not. The old code used to do the same.

There is a straightforward way to avoid double check but the new changes avoid branch all-together; since it is fine to copy garbage from source vector as long as the validity bit is copied correctly to indicate NULL or non-NULL value in the target cell.

Will add unit tests.

Author: siddharth <siddharth@dremio.com>

Closes #1325 from siddharthteotia/ARROW-1826 and squashes the following commits:

8fc5b3f1 [siddharth] added unit tests
84de0cda [siddharth] ARROW-1826: [JAVA] Avoid branching in copyFrom
---
 .../arrow/vector/NullableBigIntVector.java    |    8 +-
 .../arrow/vector/NullableBitVector.java       |    7 +-
 .../arrow/vector/NullableDateDayVector.java   |    8 +-
 .../arrow/vector/NullableDateMilliVector.java |    8 +-
 .../arrow/vector/NullableDecimalVector.java   |   10 +-
 .../arrow/vector/NullableFloat4Vector.java    |    8 +-
 .../arrow/vector/NullableFloat8Vector.java    |    8 +-
 .../arrow/vector/NullableIntVector.java       |    8 +-
 .../vector/NullableIntervalDayVector.java     |    8 +-
 .../vector/NullableIntervalYearVector.java    |    8 +-
 .../arrow/vector/NullableSmallIntVector.java  |    8 +-
 .../arrow/vector/NullableTimeMicroVector.java |    8 +-
 .../arrow/vector/NullableTimeMilliVector.java |    8 +-
 .../arrow/vector/NullableTimeNanoVector.java  |    8 +-
 .../arrow/vector/NullableTimeSecVector.java   |    8 +-
 .../arrow/vector/NullableTimeStampVector.java |    8 +-
 .../arrow/vector/NullableTinyIntVector.java   |    8 +-
 .../arrow/vector/NullableUInt1Vector.java     |    8 +-
 .../arrow/vector/NullableUInt2Vector.java     |    8 +-
 .../arrow/vector/NullableUInt4Vector.java     |    8 +-
 .../arrow/vector/NullableUInt8Vector.java     |    8 +-
 .../org/apache/arrow/vector/TestCopyFrom.java | 1037 +++++++++++++++++
 22 files changed, 1098 insertions(+), 108 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
index eca6592c54ad2..8d44c92e46664 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
@@ -137,11 +137,9 @@ public Long getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
+    valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
index bd363f5b7f1f3..086331e13d452 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
@@ -286,11 +286,8 @@ public Boolean getObject(int index) {
    * @param from      source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    BitVectorHelper.setValidityBit(valueBuffer, thisIndex, from.getBit(fromIndex));
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
index c5ddc9e87be6a..bb14d1ac0d41c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
@@ -139,11 +139,9 @@ public Integer getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
+    valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
index 7e37055dab188..97205c318317c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
@@ -144,11 +144,9 @@ public LocalDateTime getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
+    valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
index dcc551094ae28..5d2782a4c8460 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
@@ -151,13 +151,9 @@ public BigDecimal getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer,
-              thisIndex * TYPE_WIDTH, TYPE_WIDTH);
-      BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex);
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer,
+            thisIndex * TYPE_WIDTH, TYPE_WIDTH);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
index 834f16c211cdb..5d7128d2bdfa3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
@@ -138,11 +138,9 @@ public Float getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final float value = from.valueBuffer.getFloat(fromIndex * TYPE_WIDTH);
+    valueBuffer.setFloat(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
index b054d9b05c06a..7aa546150b319 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
@@ -138,11 +138,9 @@ public Double getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final double value = from.valueBuffer.getDouble(fromIndex * TYPE_WIDTH);
+    valueBuffer.setDouble(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
index e5dfbcedcf90e..7c6f29ad7c559 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
@@ -143,11 +143,9 @@ public Integer getObject(int index) {
    * @param from      source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
+    valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
index 8d2fb5661d9eb..862f6429cb8cd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
@@ -186,13 +186,9 @@ private StringBuilder getAsStringBuilderHelper(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex);
-      from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer,
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer,
               thisIndex * TYPE_WIDTH, TYPE_WIDTH);
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
index ac121d2f078ff..3fba5bfeeac00 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
@@ -170,11 +170,9 @@ private StringBuilder getAsStringBuilderHelper(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
+    valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
index 5dd1492a3e095..d5c66b115acde 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
@@ -138,11 +138,9 @@ public Short getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final short value = from.valueBuffer.getShort(fromIndex * TYPE_WIDTH);
+    valueBuffer.setShort(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
index 25da4f68d506c..ca8e1aacf6e0e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
@@ -139,11 +139,9 @@ public Long getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
+    valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
index 7d2d5d1f10fe2..a28ccfe00f372 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
@@ -140,11 +140,9 @@ public LocalDateTime getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
+    valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
index e93a63fe2a1f9..e6903c011e676 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
@@ -138,11 +138,9 @@ public Long getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
+    valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
index 8329fa7813987..85426a36948db 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
@@ -138,11 +138,9 @@ public Integer getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
+    valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
index d65cb83e15123..e1b719f66a784 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
@@ -70,11 +70,9 @@ public long get(int index) throws IllegalStateException {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
+    valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
index 106359ddd9666..cadfe608538cc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
@@ -138,11 +138,9 @@ public Byte getObject(int index) {
    * @param from source vector
    */
   public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final byte value = from.valueBuffer.getByte(fromIndex * TYPE_WIDTH);
+    valueBuffer.setByte(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
index 52f6c151bdff9..ebd9b56996cf6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
@@ -108,11 +108,9 @@ public Byte getObject(int index) {
   }
 
   public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final byte value = from.valueBuffer.getByte(fromIndex * TYPE_WIDTH);
+    valueBuffer.setByte(thisIndex * TYPE_WIDTH, value);
   }
 
   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
index 42ba6e139bf71..bedb3f35e6bac 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
@@ -109,11 +109,9 @@ public Character getObject(int index) {
   }
 
   public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final char value = from.valueBuffer.getChar(fromIndex * TYPE_WIDTH);
+    valueBuffer.setChar(thisIndex * TYPE_WIDTH, value);
   }
 
   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
index 1ec21490d8447..99b547ca4fdef 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
@@ -109,11 +109,9 @@ public Integer getObject(int index) {
   }
 
   public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
+    valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
index 408a810cfa0af..e6664a09265d7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
@@ -109,11 +109,9 @@ public Long getObject(int index) {
   }
 
   public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) {
-    if (from.isSet(fromIndex) != 0) {
-      set(thisIndex, from.get(fromIndex));
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0);
-    }
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
+    valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java
new file mode 100644
index 0000000000000..7cdfbdf452d84
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java
@@ -0,0 +1,1037 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.types.Types;
+import org.joda.time.Period;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+
+import static org.apache.arrow.vector.TestUtils.newVector;
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Tested field types:
+ *
+ * NullableInt
+ * NullableBigInt
+ * NullableFloat4
+ * NullableFloat8
+ * NullableBit
+ * NullableDecimal
+ * NullableIntervalDay
+ * NullableIntervalYear
+ * NullableSmallInt
+ * NullableTinyInt
+ * NullableVarChar
+ * NullableTimeMicro
+ * NullableTimeMilli
+ * NullableTimeStamp*
+ */
+
+public class TestCopyFrom {
+
+  private final static String EMPTY_SCHEMA_PATH = "";
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test /* NullableVarChar */
+  public void testCopyFromWithNulls() {
+    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator);
+         final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) {
+
+      vector.allocateNew();
+      int capacity = vector.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          continue;
+        }
+        byte[] b = Integer.toString(i).getBytes();
+        vector.setSafe(i, b, 0, b.length);
+      }
+
+      /* NO reAlloc() should have happened in setSafe() */
+      capacity = vector.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      vector.setValueCount(4095);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          assertNull(vector.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString());
+        }
+      }
+
+      vector2.allocateNew();
+      capacity = vector2.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      for (int i = 0; i < 4095; i++) {
+        vector2.copyFromSafe(i, i, vector);
+        if (i % 3 == 0) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+        }
+      }
+
+      /* NO reAlloc() should have happened in copyFrom */
+      capacity = vector2.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      vector2.setValueCount(4095);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+        }
+      }
+    }
+  }
+
+  @Test /* NullableVarChar */
+  public void testCopyFromWithNulls1() {
+    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator);
+         final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) {
+
+      vector.allocateNew();
+      int capacity = vector.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          continue;
+        }
+        byte[] b = Integer.toString(i).getBytes();
+        vector.setSafe(i, b, 0, b.length);
+      }
+
+      /* NO reAlloc() should have happened in setSafe() */
+      capacity = vector.getValueCapacity();
+      assertEquals(4095, capacity);
+
+      vector.setValueCount(4095);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          assertNull(vector.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString());
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024 * 10, 1024);
+
+      capacity = vector2.getValueCapacity();
+      assertEquals(1024, capacity);
+
+      for (int i = 0; i < 4095; i++) {
+        vector2.copyFromSafe(i, i, vector);
+        if (i % 3 == 0) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+        }
+      }
+
+      /* 2 reAllocs should have happened in copyFromSafe() */
+      capacity = vector2.getValueCapacity();
+      assertEquals(4096, capacity);
+
+      vector2.setValueCount(4095);
+
+      for (int i = 0; i < 4095; i++) {
+        if (i % 3 == 0) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+        }
+      }
+    }
+  }
+
+  @Test /* NullableIntVector */
+  public void testCopyFromWithNulls2() {
+    try (final NullableIntVector vector1 = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableIntVector vector2 = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, 1000 + i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, 1000 + i, vector1.get(i));
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, 1000 + i, vector2.get(i));
+        }
+      }
+    }
+  }
+
+  @Test /* NullableBigIntVector */
+  public void testCopyFromWithNulls3() {
+    try (final NullableBigIntVector vector1 = new NullableBigIntVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableBigIntVector vector2 = new NullableBigIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, 10000000000L + (long)i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  10000000000L + (long)i, vector1.get(i));
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  10000000000L + (long)i, vector2.get(i));
+        }
+      }
+    }
+  }
+
+  @Test /* NullableBitVector */
+  public void testCopyFromWithNulls4() {
+    try (final NullableBitVector vector1 = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableBitVector vector2 = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      int counter = 0;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        if ((counter&1) == 0) {
+          vector1.setSafe(i, 1);
+        } else {
+          vector1.setSafe(i, 0);
+        }
+        counter++;
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      counter = 0;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          if ((counter&1) == 0) {
+            assertTrue(vector1.getObject(i));
+          } else {
+            assertFalse(vector1.getObject(i));
+          }
+          counter++;
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      counter = 0;
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          if ((counter&1) == 0) {
+            assertTrue(vector2.getObject(i));
+          } else {
+            assertFalse(vector2.getObject(i));
+          }
+          counter++;
+        }
+      }
+    }
+  }
+
+  @Test /* NullableFloat4Vector */
+  public void testCopyFromWithNulls5() {
+    try (final NullableFloat4Vector vector1 = new NullableFloat4Vector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableFloat4Vector vector2 = new NullableFloat4Vector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, 100.25f + (float)i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  100.25f + (float)i, vector1.get(i), 0);
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  100.25f + i*1.0f, vector2.get(i), 0);
+        }
+      }
+    }
+  }
+
+  @Test /* NullableFloat8Vector */
+  public void testCopyFromWithNulls6() {
+    try (final NullableFloat8Vector vector1 = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableFloat8Vector vector2 = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, 123456.7865 + (double) i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  123456.7865 + (double) i, vector1.get(i), 0);
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  123456.7865 + (double) i, vector2.get(i), 0);
+        }
+      }
+    }
+  }
+
+  @Test /* NullableIntervalDayVector */
+  public void testCopyFromWithNulls7() {
+    try (final NullableIntervalDayVector vector1 = new NullableIntervalDayVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableIntervalDayVector vector2 = new NullableIntervalDayVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      final int days = 10;
+      final int milliseconds = 10000;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, days + i, milliseconds + i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          final Period p = vector1.getObject(i);
+          assertEquals(days + i, p.getDays());
+          assertEquals(milliseconds + i, p.getMillis());
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          final Period p = vector2.getObject(i);
+          assertEquals(days + i, p.getDays());
+          assertEquals(milliseconds + i, p.getMillis());
+        }
+      }
+    }
+  }
+
+  @Test /* NullableIntervalYearVector */
+  public void testCopyFromWithNulls8() {
+    try (final NullableIntervalYearVector vector1 = new NullableIntervalYearVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableIntervalYearVector vector2 = new NullableIntervalYearVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      final int interval = 30; /* 2 years 6 months */
+      final Period[]  periods = new Period[4096];
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, interval + i);
+        final Period p = new Period();
+        final int years = (interval + i) / org.apache.arrow.vector.util.DateUtility.yearsToMonths;
+        final int months = (interval + i) % org.apache.arrow.vector.util.DateUtility.yearsToMonths;
+        periods[i] = p.plusYears(years).plusMonths(months);;
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          final Period p = vector1.getObject(i);
+          assertEquals(interval + i, vector1.get(i));
+          assertEquals(periods[i], p);
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          final Period p = vector2.getObject(i);
+          assertEquals(periods[i], p);
+        }
+      }
+    }
+  }
+
+  @Test /* NullableSmallIntVector */
+  public void testCopyFromWithNulls9() {
+    try (final NullableSmallIntVector vector1 = new NullableSmallIntVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableSmallIntVector vector2 = new NullableSmallIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      final short val = 1000;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, val + (short)i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  val + (short)i, vector1.get(i));
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  val + (short)i, vector2.get(i));
+        }
+      }
+    }
+  }
+
+  @Test /* NullableTimeMicroVector */
+  public void testCopyFromWithNulls10() {
+    try (final NullableTimeMicroVector vector1 = new NullableTimeMicroVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableTimeMicroVector vector2 = new NullableTimeMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      final long val = 100485765432L;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, val + (long)i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  val + (long)i, vector1.get(i));
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  val + (long) i, vector2.get(i));
+        }
+      }
+    }
+  }
+
+  @Test /* NullableTimeMilliVector */
+  public void testCopyFromWithNulls11() {
+    try (final NullableTimeMilliVector vector1 = new NullableTimeMilliVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableTimeMilliVector vector2 = new NullableTimeMilliVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      final int val = 1000;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, val + i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  val + i, vector1.get(i));
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  val + i, vector2.get(i));
+        }
+      }
+    }
+  }
+
+  @Test /* NullableTinyIntVector */
+  public void testCopyFromWithNulls12() {
+    try (final NullableTinyIntVector vector1 = new NullableTinyIntVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableTinyIntVector vector2 = new NullableTinyIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      byte val = -128;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, val);
+        val++;
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      val = -128;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, val, vector1.get(i));
+          val++;
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      val = -128;
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i, val, vector2.get(i));
+          val++;
+        }
+      }
+    }
+  }
+
+  @Test /* NullableDecimalVector */
+  public void testCopyFromWithNulls13() {
+    try (final NullableDecimalVector vector1 = new NullableDecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16);
+         final NullableDecimalVector vector2 = new NullableDecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      final double baseValue = 104567897654.876543654;
+      final BigDecimal[] decimals = new BigDecimal[4096];
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        BigDecimal decimal = new BigDecimal(baseValue + (double)i);
+        vector1.setSafe(i, decimal);
+        decimals[i] = decimal;
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          final BigDecimal decimal = vector1.getObject(i);
+          assertEquals(decimals[i], decimal);
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          final BigDecimal decimal = vector2.getObject(i);
+          assertEquals(decimals[i], decimal);
+        }
+      }
+    }
+  }
+
+  @Test /* NullableTimeStampVector */
+  public void testCopyFromWithNulls14() {
+    try (final NullableTimeStampVector vector1 = new NullableTimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator);
+         final NullableTimeStampVector vector2 = new NullableTimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      vector1.allocateNew();
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(0, vector1.getValueCount());
+
+      final long val = 20145678912L;
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          continue;
+        }
+        vector1.setSafe(i, val + (long)i);
+      }
+
+      vector1.setValueCount(4096);
+
+      /* No realloc should have happened in setSafe or
+       * setValueCount
+       */
+      assertEquals(4096, vector1.getValueCapacity());
+      assertEquals(4096, vector1.getValueCount());
+
+      for (int i = 0; i < 4096; i++) {
+        if ((i&1) == 0) {
+          assertNull(vector1.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  val + (long)i, vector1.get(i));
+        }
+      }
+
+      /* set lesser initial capacity than actually needed
+       * to trigger reallocs in copyFromSafe()
+       */
+      vector2.allocateNew(1024);
+      assertEquals(1024, vector2.getValueCapacity());
+
+      for (int i = 0; i < 4096; i++) {
+        vector2.copyFromSafe(i, i, vector1);
+      }
+
+      /* 2 realloc should have happened in copyFromSafe() */
+      assertEquals(4096, vector2.getValueCapacity());
+      vector2.setValueCount(8192);
+      /* setValueCount() should have done another realloc */
+      assertEquals(8192, vector2.getValueCount());
+      assertEquals(8192, vector2.getValueCapacity());
+
+      /* check vector data after copy and realloc */
+      for (int i = 0; i < 8192; i++) {
+        if (((i&1) == 0) || (i >= 4096)) {
+          assertNull(vector2.getObject(i));
+        } else {
+          assertEquals("unexpected value at index: " + i,
+                  val + (long) i, vector2.get(i));
+        }
+      }
+    }
+  }
+}

From d887d9122a17530126a40a60944e84207d40d412 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 21 Nov 2017 14:35:22 +0100
Subject: [PATCH 1266/1644] ARROW-1830: [Python] Relax restriction that Parquet
 files in a dataset end in .parq or .parquet

It seems that some setups may not use these file extensions, and so our assumption is too rigid.

cc @dbtsai

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1340 from wesm/ARROW-1830 and squashes the following commits:

d123ae1 [Wes McKinney] Relax restriction that Parquet files in a dataset end in .parq or .parquet
---
 python/pyarrow/parquet.py            | 12 ++++--------
 python/pyarrow/tests/test_parquet.py |  2 +-
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 3023e1771e404..37da662806366 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -421,10 +421,6 @@ def read(self, columns=None, nthreads=1, partitions=None,
         return table
 
 
-def _is_parquet_file(path):
-    return path.endswith('parq') or path.endswith('parquet')
-
-
 class PartitionSet(object):
     """A data structure for cataloguing the observed Parquet partitions at a
     particular level. So if we have
@@ -556,14 +552,14 @@ def _visit_level(self, level, base_path, part_keys):
         filtered_files = []
         for path in files:
             full_path = self.pathsep.join((base_path, path))
-            if _is_parquet_file(path):
-                filtered_files.append(full_path)
-            elif path.endswith('_common_metadata'):
+            if path.endswith('_common_metadata'):
                 self.common_metadata_path = full_path
             elif path.endswith('_metadata'):
                 self.metadata_path = full_path
-            elif not self._should_silently_exclude(path):
+            elif self._should_silently_exclude(path):
                 print('Ignoring path: {0}'.format(full_path))
+            else:
+                filtered_files.append(full_path)
 
         # ARROW-1079: Filter out "private" directories starting with underscore
         filtered_directories = [self.pathsep.join((base_path, x))
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 522815fce16c8..274ff458fd6cc 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1020,7 +1020,7 @@ def _visit_level(base_dir, level, part_keys):
 
             if level == DEPTH - 1:
                 # Generate example data
-                file_path = pjoin(level_dir, 'data.parq')
+                file_path = pjoin(level_dir, guid())
 
                 filtered_df = _filter_partition(df, this_part_keys)
                 part_table = pa.Table.from_pandas(filtered_df)

From e98adc3b54aab1139239c5b475dc36a809412d99 Mon Sep 17 00:00:00 2001
From: ksdevlife <33046664+ksdevlife@users.noreply.github.com>
Date: Tue, 21 Nov 2017 09:56:48 -0500
Subject: [PATCH 1267/1644] =?UTF-8?q?ARROW-1840:=20[Website]=20The=20insta?=
 =?UTF-8?q?llation=20command=20failed=20on=20Windows10=20anaconda=20envir?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…onment

Author: ksdevlife <33046664+ksdevlife@users.noreply.github.com>

Closes #1342 from ksdevlife/1840__ and squashes the following commits:

bc0f964e [ksdevlife] [Website] The installation command failed on Windows10 anaconda environment
---
 site/install.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/install.md b/site/install.md
index 0ef2008db9061..67b26983b46c4 100644
--- a/site/install.md
+++ b/site/install.md
@@ -53,7 +53,7 @@ Install them with:
 
 ```shell
 conda install arrow-cpp=0.7.* -c conda-forge
-conda install pyarrow==0.7.* -c conda-forge
+conda install pyarrow=0.7.* -c conda-forge
 ```
 
 ### Python Wheels on PyPI (Unofficial)

From c436376b0ab880202c4e2f2735aff74b4667d98b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 21 Nov 2017 09:57:36 -0500
Subject: [PATCH 1268/1644] ARROW-1838: [C++] Conform kernel API to use Datum
 for input and output

We could also add helper methods that do automatic boxing into a `Datum` at a later time if it helps with code complexity.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1339 from wesm/ARROW-1838 and squashes the following commits:

f83a866e [Wes McKinney] Conform kernel API to use Datum for input and output
---
 cpp/src/arrow/compute/compute-test.cc          |  2 +-
 cpp/src/arrow/compute/kernel.h                 |  2 +-
 cpp/src/arrow/compute/kernels/cast.cc          | 11 +++++++----
 cpp/src/arrow/compute/kernels/hash.cc          |  5 +++--
 cpp/src/arrow/compute/kernels/util-internal.cc |  4 ++--
 5 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 58a991c60f24b..fa408ae40cd85 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -697,7 +697,7 @@ TEST_F(TestCast, PreallocatedMemory) {
   out_data->buffers.push_back(out_values);
 
   Datum out(out_data);
-  ASSERT_OK(kernel->Call(&this->ctx_, *arr->data(), &out));
+  ASSERT_OK(kernel->Call(&this->ctx_, Datum(arr), &out));
 
   // Buffer address unchanged
   ASSERT_EQ(out_values.get(), out_data->buffers[1].get());
diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h
index 0037245d610b4..7ff506ca0cf2b 100644
--- a/cpp/src/arrow/compute/kernel.h
+++ b/cpp/src/arrow/compute/kernel.h
@@ -131,7 +131,7 @@ struct ARROW_EXPORT Datum {
 /// \brief An array-valued function of a single input argument
 class ARROW_EXPORT UnaryKernel : public OpKernel {
  public:
-  virtual Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) = 0;
+  virtual Status Call(FunctionContext* ctx, const Datum& input, Datum* out) = 0;
 };
 
 }  // namespace compute
diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc
index c866054eadd8a..d595d2ea507c1 100644
--- a/cpp/src/arrow/compute/kernels/cast.cc
+++ b/cpp/src/arrow/compute/kernels/cast.cc
@@ -740,20 +740,23 @@ class CastKernel : public UnaryKernel {
         can_pre_allocate_values_(can_pre_allocate_values),
         out_type_(out_type) {}
 
-  Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) override {
+  Status Call(FunctionContext* ctx, const Datum& input, Datum* out) override {
+    DCHECK_EQ(Datum::ARRAY, input.kind());
+
+    const ArrayData& in_data = *input.array();
     ArrayData* result;
 
     if (out->kind() == Datum::NONE) {
-      out->value = std::make_shared<ArrayData>(out_type_, input.length);
+      out->value = std::make_shared<ArrayData>(out_type_, in_data.length);
     }
 
     result = out->array().get();
 
     if (!is_zero_copy_) {
       RETURN_NOT_OK(
-          AllocateIfNotPreallocated(ctx, input, can_pre_allocate_values_, result));
+          AllocateIfNotPreallocated(ctx, in_data, can_pre_allocate_values_, result));
     }
-    func_(ctx, options_, input, result);
+    func_(ctx, options_, in_data, result);
 
     RETURN_IF_ERROR(ctx);
     return Status::OK();
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index 3af41609fef28..95f039932152c 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -658,8 +658,9 @@ class HashKernelImpl : public HashKernel {
   explicit HashKernelImpl(std::unique_ptr<HashTable> hasher)
       : hasher_(std::move(hasher)) {}
 
-  Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) override {
-    RETURN_NOT_OK(Append(ctx, input));
+  Status Call(FunctionContext* ctx, const Datum& input, Datum* out) override {
+    DCHECK_EQ(Datum::ARRAY, input.kind());
+    RETURN_NOT_OK(Append(ctx, *input.array()));
     return Flush(out);
   }
 
diff --git a/cpp/src/arrow/compute/kernels/util-internal.cc b/cpp/src/arrow/compute/kernels/util-internal.cc
index df68637e088f7..28428bfcba6c6 100644
--- a/cpp/src/arrow/compute/kernels/util-internal.cc
+++ b/cpp/src/arrow/compute/kernels/util-internal.cc
@@ -34,13 +34,13 @@ Status InvokeUnaryArrayKernel(FunctionContext* ctx, UnaryKernel* kernel,
                               const Datum& value, std::vector<Datum>* outputs) {
   if (value.kind() == Datum::ARRAY) {
     Datum output;
-    RETURN_NOT_OK(kernel->Call(ctx, *value.array(), &output));
+    RETURN_NOT_OK(kernel->Call(ctx, value, &output));
     outputs->push_back(output);
   } else if (value.kind() == Datum::CHUNKED_ARRAY) {
     const ChunkedArray& array = *value.chunked_array();
     for (int i = 0; i < array.num_chunks(); i++) {
       Datum output;
-      RETURN_NOT_OK(kernel->Call(ctx, *(array.chunk(i)->data()), &output));
+      RETURN_NOT_OK(kernel->Call(ctx, Datum(array.chunk(i)), &output));
       outputs->push_back(output);
     }
   } else {

From cac09122e79b2b8faa063a087b93f63d86a6f74e Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Tue, 21 Nov 2017 09:59:24 -0500
Subject: [PATCH 1269/1644] ARROW-1841: [JS] Update text-encoding-utf-8 and
 tslib for node ESModules support

Updates the `text-encoding-utf-8` dependency to version 1.0.2, [which now supports](https://github.com/arv/text-encoding-utf-8/pull/2) ESModules in node >= v8.6.0 via the `--experimental-modules` flag.

We currently support ESModules in node in the main `apache-arrow` package, but need our dependencies to also expose ESModule forms as well. I have also issued a [PR to flatbuffers](https://github.com/google/flatbuffers/pull/4504) to add ESModules support, and are using a temporary [fork of flatbuffers](https://github.com/trxcllnt/flatbuffers-esm) in my github until that PR is merged.

This PR enables the following workflow:

```js
// file - index.mjs
// run via `node --experimental-modules index.mjs`
import util from 'util';
import * as fs from 'fs';
import { Table } from 'apache-arrow';
(async () => {
  const buffer = await util.promisify(fs.readFile)('simple.arrow');
  console.log(Table.from([buffer]).toString());
/*
 foo,  bar,  baz
   1,    1,   aa
null, null, null
   3, null, null
   4,    4,  bbb
   5,    5, cccc
*/
})();
```

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1338 from trxcllnt/update-text-encoding-utf8 and squashes the following commits:

fbecde59 [Paul Taylor] synthesize an mjs file for tslib on postinstall
4b050c95 [Paul Taylor] update text-encoding-utf-8 dependency with node ESModules support
---
 js/gulp/package-task.js | 6 ++++++
 js/package.json         | 8 ++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index 7b4b15a33e6ef..ad56d172eaec7 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -49,6 +49,12 @@ const createMainPackageJson = (target, format) => (orig) => ({
     browser: `${mainExport}.es5.min.js`,
     [`browser:es2015`]: `${mainExport}.es2015.min.js`,
     [`@std/esm`]: { esm: `mjs` },
+    // Temporary workaround until https://github.com/Microsoft/tslib/pull/44 is merged
+    scripts: {
+        postinstall: `npm i shx && npm run tslib_mjs && npm run tslib_pkg && npm r shx`,
+        tslib_mjs: `shx cp $(node -e \"console.log(require.resolve('tslib/tslib.es6.js'))\") $(node -e \"var r=require,p=r('path');console.log(p.join(p.dirname(r.resolve('tslib')),'tslib.mjs'))\")`,
+        tslib_pkg: `node -e \"var r=require,p=r('path'),f=r('fs'),k=p.join(p.dirname(r.resolve('tslib')),'package.json'),x=JSON.parse(f.readFileSync(k));x.main='tslib';f.writeFileSync(k,JSON.stringify(x))\"`
+    }
 });
   
 const createTypeScriptPackageJson = (target, format) => (orig) => ({
diff --git a/js/package.json b/js/package.json
index 24bc27f5b6325..1a110b2d5de89 100644
--- a/js/package.json
+++ b/js/package.json
@@ -25,7 +25,8 @@
     "lint": "npm-run-all -p lint:*",
     "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
     "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"",
-    "prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1"
+    "prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1",
+    "postinstall": "shx cp node_modules/tslib/tslib.es6.js node_modules/tslib/tslib.mjs"
   },
   "repository": {
     "type": "git",
@@ -54,7 +55,8 @@
   },
   "dependencies": {
     "flatbuffers": "trxcllnt/flatbuffers-esm",
-    "text-encoding": "0.6.4"
+    "text-encoding-utf-8": "^1.0.2",
+    "tslib": "^1.8.0"
   },
   "devDependencies": {
     "@std/esm": "0.13.0",
@@ -90,10 +92,8 @@
     "rxjs": "5.5.2",
     "shx": "0.2.2",
     "source-map-loader": "0.2.3",
-    "text-encoding-utf-8": "1.0.1",
     "trash": "4.1.0",
     "ts-jest": "21.2.1",
-    "tslib": "1.8.0",
     "tslint": "5.8.0",
     "typescript": "2.6.1",
     "uglifyjs-webpack-plugin": "1.0.1",

From 15ed08054b07a1c3fda6d712c947b8bacc5afc7e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 21 Nov 2017 16:22:49 -0500
Subject: [PATCH 1270/1644] ARROW-1703: [C++] Vendor exact version of jemalloc
 we depend on

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1334 from xhochy/ARROW-1703 and squashes the following commits:

7282583f [Uwe L. Korn] ARROW-1703: [C++] Vendor exact version of jemalloc we depend on
---
 LICENSE.txt                                   |  31 ++++++++++++++++++
 cpp/cmake_modules/ThirdpartyToolchain.cmake   |   6 ++--
 ...7976c60b0e6e4f4a365c751027244dada7a.tar.gz | Bin 0 -> 454719 bytes
 cpp/thirdparty/jemalloc/README.md             |  22 +++++++++++++
 4 files changed, 56 insertions(+), 3 deletions(-)
 create mode 100644 cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz
 create mode 100644 cpp/thirdparty/jemalloc/README.md

diff --git a/LICENSE.txt b/LICENSE.txt
index 84e6a4e2a2a97..30966d36f37a1 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -552,3 +552,34 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the jemalloc project
+
+https://github.com/jemalloc/jemalloc
+
+Copyright (C) 2002-2017 Jason Evans <jasone@canonware.com>.
+All rights reserved.
+Copyright (C) 2007-2012 Mozilla Foundation.  All rights reserved.
+Copyright (C) 2009-2017 Facebook, Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+1. Redistributions of source code must retain the above copyright notice(s),
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice(s),
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
+EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+--------------------------------------------------------------------------------
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 42d7eddc9c9d9..411cf7584f7f1 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -24,7 +24,7 @@ set(GFLAGS_VERSION "2.2.0")
 set(GTEST_VERSION "1.8.0")
 set(GBENCHMARK_VERSION "1.1.0")
 set(FLATBUFFERS_VERSION "1.7.1")
-set(JEMALLOC_VERSION "4.4.0")
+set(JEMALLOC_VERSION "17c897976c60b0e6e4f4a365c751027244dada7a")
 set(SNAPPY_VERSION "1.1.3")
 set(BROTLI_VERSION "v0.6.0")
 set(LZ4_VERSION "1.7.5")
@@ -471,8 +471,8 @@ if (ARROW_JEMALLOC)
     set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(JEMALLOC_VENDORED 1)
     ExternalProject_Add(jemalloc_ep
-      URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
-      CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
+      URL ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
+      CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_" && touch doc/jemalloc.html && touch doc/jemalloc.3
       ${EP_LOG_OPTIONS}
       BUILD_IN_SOURCE 1
       BUILD_COMMAND ${MAKE}
diff --git a/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz b/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..29d9266a12ded2452f3bd4f89421c76f50bdcc84
GIT binary patch
literal 454719
zcmV(%K;pk2iwFP!000001MFP;ciKp@o}c+E`ua>xY+r*7#&*WL*)s&=ggLhH1)NOo
zWRACy8t7OLRxM$ZIrG2Yuc{>w55Ka`WcE(98xNAYtE=nzRY@?CH<tr1>`Cu(sHCf+
z@1FkH!Lh%;&;P}-_+Q+teplWt@9phX5BB!UxL&UARrbFdd;wmTj>N<=!je%K#&?ao
z@BZTE{zHzvif}^TS3&%;eBiwJ;oygZeP@5Cx1;t|wO^IH`+Ls8UU{c-P^ng3>B<9H
z_}V;wJD%?NsC?;pp;LUQ_O~Cu2KmFm!QSoukI(b|U)d?|?|!%U^xIuN{^I`sc>R~;
zXf#%nFe*)a??G$W_+PzE|K)>fc~SqB{auXz!h7fAFRuTwii{3}L-G8#UZOqs_veLz
z3dTAL14;^q1tEMHNBV|76~$@ieW%+vJrt+Sv)AwTtGFmc{ruuk+`QPoOwUg{Z|jFb
z`YxX@PTH-^Ch7eb`?)DMeD<JK9zgZ8xW)a}*>Usr<?%`Fb>~nNiz<*kPZeF&O9r2_
zokvr=h0mwp>rNU49l#en&uZGJSe)0oZ(cfwAAB=0`_&K6(iNV4=;`2Zv+Ie8_{uXH
z|BPqrzHn7v2g+qx2nTe<8m62_;UJQ}IM$vrVjcV2_;9M7C^TU|7H_plq$fIY;_6V8
ztEHU}Z|fh%&R9kU6iPiAuZd?warH7I>G(3h#)8lZgT9V@QEVGW_&OL|7q9oL#b*%6
zv*N&U1pYCNRdYJYWND5suT-JHPI76SNRMJGsGc-KQFO$bOk$?6Hh;Oa&145jq{R1m
z1=w-C%x;Cri{i&!zSxI!gr|GkS$HnFp0{$ih?R-Wid$F087k-MKCt#V@gMv2tFbCM
z{f8m+d*c62d2e?q{#TLzzs3JAfArKq2SrZ?gG3J0p;)V>R$RQ+@i0u{qI6&$(kuGa
zwStTW=8#>b=t$TXQic2A$zOY`-;Nc>XZZg>$1(;a=p~3f4_V_r|KF)B`Ty?DUiq8<
z{{hD)639*bl4810q4(13fAiD7((!rzuLnUGsfVs{kN@xPmY3@PN_lVRoB#g-$M$Av
z;DqDSW}&bR3_k^&MGIQzFP6+OyD*BvKBm}FCCJbpP0ze=kiQcrrZB_6nB<1e_AvIn
z(v9)#836bQ2*S*+Oqaaz7q8INg-g4jIz1UVI3w_k|C|B~6PU(F9XMX%s_8gfh*cCw
zZ+hi2yHXlHYWO8;5OepOQKZK*R+oYFl|jX!9yUA)9y%fEdD?j}c{|!n{9fppQhXCX
zVDy7|G5V9bbUbN{S}m2CChxd;yHKYid`WfVePe=wx0>OFF_#}zeD^rme0<+}cFg^>
zB1O`?drUkTsZ<JuZ4){0)n7vQ*X;AaapmlD5MmG&gZzg)l}|G2d@`d<L+%}MIbA#*
z7%SB7u?~anUuwnwmBmlLZWjLQj+=-o%XhO)thix!wDJbFx9SFJjEud!fN~ddG?ztX
z;z${@yqm@A(~YDz=GV!t<(HaWn{qzAmSZ{zbiBO${MG5=e0gr^^h$2w>6PiS=@q-^
z+eW+7Y@HPfo28N4!(Zd}@i!cykn5!%>?N-6ICNrA65?-SR759UP(tvfz^)9=V~YH~
z89UqWbg*05PET(3tJz2R#7E`C*gF1qM7^fXmABf;Q_`r_H|L1Y-QktKb{_ptJw5~f
z#gQCq^Wg6Pz4&i`XJ>B-|5fOD(HGr$di=%l->i8vP=Sgxu+yyHpQeUG?j09JXby#^
zgXD(Kg;2rxP}J)$she7Fot`&O8g1L!e7Uxk_L?u+jp?Nvnb14|kyX4nw4tXUw2Ggi
zzgxMLln~N&!@wM-XE?Vl;_Lu&x+0L9zeXHjrJ|pBu?A*+8cEML_CIH*cM&D%|70{>
zZI~#f6Z7O$W6olC8{TJG%){*WPJ8(`9pQEz{~IK4)y5OhT-L?I{r9Uo-h++gajy4x
zvJY{-M^Jr~`+Wx8hbjFLzOzccUn}z7$Kp&^e>x6V=>1R2W8Q}Tv`JQcne@1@#FxpI
zw=DAczI&fjeCeGmtntJfnPYv$GKE4<^}|Tby(N&n9Ka`YuLS%yzv$tAr#1_g*1oUT
z>79Z6{WbA>I^g>S-t0>N`PT>*UJ@we8RC2P6@hQX^I!LPeEnZ<oxg9My?(?R_dWlq
zR1X&G|8k}B?ft($;<yMrWsC^pp^C1xQNoOrqx)L9Tl^N15jqgKDKbM5CQKqn2}C{n
z4uB2=AtO~VNsoGRA`W?cKg8&34Z!GWM+HV57K%@N6zbt<66wJ(7VGs5QGwkQ>CIvB
zlQdweF_wW5fB#4)>PJTgVQ>vbOHSzjqg1GQp5VDgM9QdWtlZKv!2=>#5vSoN?Rioh
zhe_aa@3|zz!zlb<Svr=E>V@IemS_gfZIcUaH80k{6$x|#6{J$9D?J^^XaWI8zS$D=
zri}<Ak-l+L@Ix2sbC}63Qj8d>$k%bKT!Fke)-FE8Lm4k<2^q=NG|BKJUm;JFOJbfQ
z7)n25#R**ugZp3uu}rN*_QEmU%7hSvF|@yKHnu3+#zSpj4wwQm`2_=afg3D}YD4=f
zDitcrTwt}F6f!QbJvRYoUxty)5!}#8=+#)3FcHI~Z76_Uh+dXek(QpB37a*<Gjn@@
zvb%4Z9nonWci+|84S~<+?bh4oQR7IwdM~<f8iFEiyZQP}SG;MR95veOoelBNf7Ut}
z`~116ogLxdd(rs$yxr(@3az$io}QmHF&_lmwX<%s(Sf<0)lV*t5U{tztBbBUYjwp*
z^R(H;ux@LM*IAw>TE~UcM!Wt7r?pqjlV<lluXo(+o?$I<jKyl=yw>hE>lY`rwm83N
zpSL;<K|48WcIqd!=4s;ycHTV0l7+_G##vW%-qcP`Zr$0d1{kZoI%!zy*x6CD-KckI
z3+bl<HUj~Hr;{zwId9aP^r7){0~@Hd-){*_1Ssk>{^tUNa8n%BPHV5RpY>%i6!Kz#
z=<5Jdr^IIKSadF4b-K;&MYkbdw^~OXBHU@T-!|)w&fmmIt3whJ7oElyR_NAf;Q~lN
zJh+XquP!=G7F6@B+i15h&b!Ul*#<WI4w3+mHB3EXiMGyos}M@7{hq{5GGKvkiFa=r
zxZWliVprXEjfBzZwwv{?$PL4K-B!Ds+oL#Zygq5ZZk*K{WEL%=@~+uwY(PKF4h?A9
zW#84X<^^w=)B>*T$DCn8zEGZM9v5myZ=1w+IvD!yG*jDPk=5U%;x0XXr2pD!^8fAe
z8TfA~gMm5;AO7vUJMmw+3aGdA{!jU!`tALnKjJu^W;)9H4(nD7kup6CPNXrRqh*}s
zv9J$tf&7Ma%MWV}#dv=dMzIXy0yPl&`bJp{LGS!irbN0UIElmu?BElev2Ti?^u~IQ
zqA_+xm;x1KQcA*6p%<yKjwe{;G;xN)M_Dagf*YOypc6eB1xS|V=7A<0FxK>y7VN^r
zVKg}`aG^hp<B>Vs-o}t&(jySK{KiG0uqmpgz0!_YKUTdcA%IbNu>}}cKG+~KUBV6c
zKVt+PG3sD35bEuzK=r{uKMDzwfZbjc(pPE;VZ=H}!o-N&>kFVqbvb0eMSv{0GDPJ^
zR{}P~gV1w(o{VE`{f&h1p%v*`;F8V)t$ri{DH1297pv%bofvrlnj|{l4~@T>nzjTu
zyABf%!j*)KWJ0irR0Ie_`|(3;2S`+dP0)(KCMpAlMWj;T&WT+<C0QbYoT+QkSCZB@
zwy#ig-F#1EIGTd0a1@v59GgtB5nU<*w8Bcr^0e^{Yn}8JM?mi*7)20^VFIJ17sT@7
zgJ9NwwBUY&SH2hlnk(@M7X+(QLjCy}wDNTrU0F-^rPJz&@=mE-VNwGRjOxU<#M){q
zSg~2>{))(&HYv6?p~cbZ-JT-BLOYX@rmwq9wrJ<Pm&VUX-k0gPbnh^DaV;mrlP$R>
zLX`$3QGBPO2z<LqL^|g0o3M$H`vAo<!Sv=FLpgbDi65gdkk`YIt@c&sl~zN?`bHc9
zo2;8&bhTdDDE6RDcO#RTCDpZ>&TIgznt6*SpJQICrQOobk}}?4@17D7StFSxynx;d
z6Z~G7NT?6{$tne&I7;HVb^A%+(7E*B`@G1?<+1cmx-^HB>wj%bH;AoVp)4ZrLqJgU
z^`T5^12^$UOAKRDF;ac$kkQyyo-mWZ8A6gNtB(}R&I+**Yh%eiCVuL)&WcpONRD~w
zi&xxLyoB=x@%jdoJ}{4$$|{A>%H&5yOdrq}rXH_uP^Mk+((}aM4OMWz$5Q099a>}q
zL~-gCT$2a!yF@(PFmX!BF)b(^2RbCBrCo(9Bo5_0H^NCGUMPfRNsfjnd=<N#2m^0I
zz5%sccOjl6l3S9-*dPpw%wcK~qa+$|K<!(P2A}XWT2NB}L2(G{VJ7Y7m(_8nz*eck
z8(cC!<PiRd7N?j^Tby>W*#?VY&(mYJmswQpYmf4u$+PnS<;@!;f<(3Q0)fa6EnDXi
z2@|4ARYSUT_N!14l$*sNX7hyF=ULE9dOH5JVN++7Qs<FEO7Fq(b~$tIZ`__JW^Hhj
zj0i+9`OV^HkvsH7Xfq16%S$<Z1V(Mj09Rl=FHfRmB{yx&PYi$Ka7G`QhJ!ZIr<MxD
zgh8?Lja514O>2A#wgr_I{V?XYa9!bPU)u;}(uqVJp>s*K#7cfEN2;fRQ(m84Y+oHm
zO1<hF@f@&gq|cIRfipjiFgekvo0#m|a5q5&2}#sUJ3^T6lVS*EjgV0gFUW-?Wfen<
zU#SUijJE{yAV{ZDP(b!dfP<c;k<H9f8Fx;maY2kiPdgJ#Qhiz+kVM(!Y?+%zCY-qJ
zSlXxTrUZZKi<Vh;l@l>Q0#q4H+a?=qR7}cO0Cf{Fh{9_l-0+%ubncQ|+gW7xr3-kc
zAp803H{HT1%GAr_cB4VNww6r6j^xHkN7<B&8EnOb_xNt0u2r<O?x_S^fFDo<fM-yk
zAZqz|BF!|O6e)ydLP=#|BO2!?$??LX4_+K`Y<19_i!F0Ztxo5;)S64D45ox~2P)s=
z(!n_N#^lP-Vg&GweR&f5vY~RSee;O4un<|85_0kYsctAU4f<pm_(vfoB6y{-$1xc#
zjNI}Ds_bxSw0EQ(L7^95UOOpEO#cCFl~)8`6e}KA9Pkt_Con}*&`6DiZ3%(bw<tO^
zmr2l<I=ZxoBCxn|ngq=!dF3lGr=m0l?v~18{S5o$7<iD2fh!aTZzXi1#8^i$ef2qY
z2kheNGf`Lyg%)Du)PVqseeJNxk0KaB?YzmZOBx9m0`YF8h~Q4KG3IXbETT{hLRcu)
z%VkUsJ-~MzAdoPGO#>aQ+t(vAZ@yMSJ2?qea}v6vzTzOw-zng`R0Dxm9a}Z=qkSk=
zywd@K2wBKncreov3x>hg&R)$0R|fp)EANaXl2NW=D~LG=GuBenMx(d%0~lhRq}b)x
zVMV<;ON}gD6c;~l+)*U0{j?_52?K6SWe#pup}>_{rXI-&JG=G>TU!b|G7tS%Czowf
zpLFHOdKchFkfrus7{+i-Bo*!nU>}imVW@}{3USU=X*gp{G{eRB1t=@>Txy}RM*&|9
zE!eZ=Y*r>}pYYZ!rVW)G*`RVg)R;o0ze#$AdJZ-)M~XZD3tof>9oeTSbJ^#SQ6vj)
z{>M?Zyx>+Gd`VX&&IzXUcm0o25D$^_kP!*KCS*v1)kO+%W{@X5s{W8E<;2!v86aJe
zj;_>v?UqK&jSL7=6s1VzabCCL!UhrX;W(p^(l#V!i%9TDLrn&QE!#*7I$nucDRvi0
zE&1Ys+6Orkf#CXVCuUA&;Xo!qnin2SmPL9Qi}jJ0%yptz;u`|xzD(<dv{X0q_;yL)
zbA>n;0!Rb>KB3*)YU!M|#9QeNASrh)BA`T@cRLz?r0Mr6!fUZ}vnRzz0{R5nNy-eT
zqr|!#3@jsh%bNUC5D+K7x&a5J%rHeSk4zDj)RM<Wf&po6F=nI(aC$}jK?~Cdzl>W3
zp6)qJ#Qk^;Eda~e<_kSKp5kIy1V0U9l=&Ndsbq^xYs>LMqkg9WS$hNFW^p&x(Zh(a
zRm9yr`)Ca-M<|_@d7eUd`>P_?6s!oI;typhA`43l2wBAon1&g@qJVcLvH5pc&FB?U
zpx8~nEFzIsY(pthTp=)Um}?{P(B&@DEw;`zotwocUj{&X8BIAxsRJCUTz(`^Kg*`$
zZG;8>kaO^S3n+~g%dV~>P$$X@G0~T!`DmC{2qsZn+@uNg&WP9T93i2_*ov7kz8+EG
zWROu-wUW}l;aKusc}+si-IESS!F1Jyq`lH5(hNx>IdnuiYY?zf7Pc@qJB%FNqXO5K
ziO5QyCdCZbWh|_-2W3My(}u<7#xjnn5Yqc8fpxsN8|BR>)l}A$ZRfXZK~ALpmFQ?L
zSz2A&wy1FzvaeXTeZyJL)iWFSRLsQGfY6Sw!oDjf>l-5dAt>_rz|f7$g!`!}{FsYw
zG<_Ni^NnJJGl%&GK~gK1NS9V&sh{3PZ<?FL+tXzu&8@OJHI3rSPbvzFs44~n0BVv_
z(Kf@mtU~yi$&*H|O0V1_1J<A(E#hw4Vx=EQ;IPa4n<HdvtMoC*l63&(o&uzmEBE$6
zxbCKG$M}I>2#MLwo!HAfo8H$m)Wjc!5Cd5+ES}y@K)IZ@@olgREG3}tTrJ1VG++(-
zqU$V@tOS<Ci-k1yy^gecUP3w3yVL4y+2dxN8H$y~%*GbmGE4J1DIiYgc>}Jj#aMde
zISeM#<btAxf*ao^46%qsR{&s2RU=e%q+Zgtnafr%onjAmbH#YZx@p$29U@uX20F5I
zxZr6WP?0`N33ZAzRKmnVev2kzxHJpuOEO3~x39Cz&z<wJU;Hh1%ecStwVM^y!cmq%
zk{}^Cu#gb6ef)zw=7!2PJ}Io)Im?<!pH-aJ&M(he?FPLS(U^5pK<t;HLbxJ(_%%m}
z5`eAdx5yT`Gj9W0mOoHtL*;-avC6kgrsx!RuNGTwtHZ5rtj@sqTwB4$hv{>G)Ya9p
z{IMYr8BkptGxo4GG5w&?)nyty3CUo~42s0x1rox+vc#Y*WnmjY1X;C7e4idI&ULqq
zq&k2ydKC6#PcL_VHo2owwn1fgv1$ekg&3yw@)QV2wq6xa$Z+TxpT|FKMo-2#4^d$p
zCIfy$3A9k}bMOTz5C!Or>0uV{e80-wfGoS-B7CwXGIxohl<KrxUa_<<O+SI@(w6^^
zy?5_x<4V?r&%dKj(UEbGY+V3f;=#r-25hr825-Pg=Gcx-BQ>B{Qft%_mr1-o`&&;{
zty_0X;3PBq?DKAZGcokKuT`s7-Rr!6Iu*4dqzSkSQ9Q(<iIDMv2xX?0ak10W+8`aB
z)ji=n_KanoTFzi}uin5X8d2uUUCpHkfbe6Q2jEL(ZkZ2~Mp>eFg-$GC5d?Gow8s!T
z=*!%lP)7+ry#<OV<_9EWGLKc@n}=@=t7n8OMx-?PJnUza*iDcD-*S4Q*&^Cj)msAJ
zqTv#eUBbt&U!u!oB0T$SH0XbZu#e`@jUfSW3?MXgeBbD~Os=}gcq$v8dkl#(l9Lgd
zh|!N+>V=9fxsUb*JvewP#)FK{7SVX2E{yfYeb)I*TO)aisG|wdodndS{U9h{Q!QG%
zqQhXPc!hnTL}uCa`iju@^(-4C<Jo6|2A=`*t`9dojr#*&eWBK{y@a9oHI`x`Tln(&
zuZjQ&#A~Y1D_rXmlLPN^n_Jz7Sz(`$N1x6yK7r~y^&dFV68E9k&r-PWrW3+AaMY)T
z3|M(g%V28_!7!eZ!ze$E+{ZW0fc-^0x+bn7g^&$|b^<+HB??Re8hmv|Mt8v*B)8a@
zK&HdKu&J!N07ha|Bz7kNu3DCqn{m`tw<oWanaFvnNL8V+*N@Le#A8CpZ}1!K1Y2Wh
zQ(@g0bVeG*ObueKBuc02Ee!HRnWI>?j0goeqnLO`IFwdFv_pT&_~SeojIpBwwupu(
zI~@$@!0^~#;65g&c<ode&$hp1w4oDL`_dtO$9wdSUkl}9xxkpN&My#Kdr-l16kUsu
zl#uyCJweCqAuAA512)WIsi2DBldHZMn=qLo6Hs(Ae-e<8h^fn{C(kaR07M6Y?Fq;c
zp=#)f65M1Iu+|QwCA*;$R&TEReDhfc<NeJ$)4^-39~~jhMde=$4YK0*0?C`J;Ypgh
z!1LLY?;P2yq^lQAX(1&KSvY~P5ju#DYAo*&R3$J)E;st%^8-A)tE^9IFUi12D!Xm0
zOo|iR;4NL})kO_(32H=bBp_;EJ&aY7B=va)5lE~7*M0;^skII&V+GzG<s!TimU|S+
zuID0S7Jc3`M6NF#29bFRSsPp6!01V1!pdt%`+e0P&W5rHt{Kx2ml4ZanDT#je7L>)
z>Q(3M=1*#)#A<NjS_gf@Mkzfg@{8FJ0TXC7W(ETKV8%Qi(a{ya9A|{#i6ThkS`b-+
zt_7Uf?k8kMEK>}FhKUj?O0Yvtz)BD-36FNt%an+V%$mLR77j#Wc$u4p<C(;l{)?Xk
zC*zu$>dmGa{$tV2CQSjsB1DTo{L_?ee<a2fq?l8tRc^tbl`}p}wxq)oa8ib_C@L$0
z%OS5WT--!)o?Ine!uF!}%PeEo6KX`&4`5foq8b<uWXND0ec~2t&2W>Hwk5)lv7-B)
zs#6SY1_Y|04tHf@aZfa|Y5|t?RQ043xF^rIw5%~%Xqd+dN{`H!aY7YFhXN**p%OVg
zfDYJ5fIxy^jX-IowRX2ehI^EZ;|ZG)u;4wufSDz8!3K4ON*A7WhB;4WBd(}uE8uC1
za$*qER(ORoYyzb=MRSdd0qU~On~0A8KEsGs)nsN_v6emlOm$O4?9f-k)NUe=Kkb?S
zFcxvtly;YBUt-V3`qs{uA{o>RIOt8{VLTPHKiL*}Bgde@Cj5dEnkefcxD2}^>;V<w
zQBZng-C;kU?wyS?;Y@sV!I9Udx8RWFWi#safKv{{Y!)4tnD#m5=yP(-ggrQ^THWmn
zUj?wUgG3BFE$?Fhxm9>aoyan;x0ZFslSzEdmngVEv_3`w%q&?}$uKx_UlcKZ78-5@
zf;jy=qdN;61J=`M<I3znV7p)?UWOa=v{kdIp)bv&sZt1Ahle+hfl$hkQ-^~7JWGgp
zR3z0C!dY>>_Xb58rCqOm)TJXzhlSDXNKQrM1Og?t26_-DL@1w4eD=r5Wg&X*QsohW
zwc01#Bx5eQzPlc)IaR?0n>LS9rAfi55)8`v{Kr>cJa+&>d;<!ZMwUmA)m9@hq$$8x
zh>S2rmRlW*!q#IIgO|m?5mvmFX2i2Z?+6EF-ShrM*<-q$j+UoZINO~~29}n?fkNs&
zvsQ+VSP5|@cO;g0U^(R~@w2R|mlEt+s*e7(mt~E?S%*pW4lE@aj*xAPRVp{;;xE?z
zI`8M+%@cLZxURGwMzsUFps?gv6H|^ybA?js8ZZ)l-jyZh0<5#2s=m)tEKq-9wa%PF
z8nJT%%MmYdap^@LP0!VgO}H7Ab;QwGn)V!LH%|L3U19@y$3u#R8|#qbzR3BiJ=RPn
zRck_Tz)h%K;j@Itqz<!lLM)62<_X5ka$BudBYL~Jw|B7hQ|Ev093Eg;Xe2~*272do
zhCwE=NL`q@BIHtPq$QYxH|HdQVwtd5I>gE|eIR1U-cF%WpgvZd;oV$2)4X)vGuv&M
zQ-2WE=pCMl1<#G=uGTchbcaBb8F>c)K)pFx6Yt(v?`7R?X6$d_$u&eEbN7&CUB7QX
zzG6nj*)T!iO{5|wE;vSF-TuMbFiD4Pek(lI#Wzxpo$Zr8K=Kr@GQ?@uXkR<i?)0hy
zp?9Z0g(6NKP7vn&ezDbtC)I-CCd*Y$Q3J;#%%5T*bc7l5-7;J8BZMoewT}b#<%GRg
zfMy+*WimC?W>&<!C}%CX|L%A%s<D`TIg7@|zNT-)BVtr)p>r*O&ea>BQ>|e6X*^9X
zSwt7vIkGUvhs=2}*;f$g@LsbxKao8iK^PmcWA=445<wBjY^l=|P9YDJgbCgKU}=g(
z=%a_2Xv%oZ3TzhXvn{7^8(w=cKrk3l)(Z<vUjEs&@I2}?PIO4d0I4HD_2AYunYXTZ
za2a1)HDjViUWB$q6lvx6x-|1l_r+iqcOnO%8X7kp48TCjAZ(H}Wk_Mis!|vdB3SfE
zAcT7fBlKj{6{^q&#Qx}l=0bhaZLkot>O;)xqddewJeCI=a|uDyvKIqPtRtd@K3jP(
zC^yz;h;r7Ku&S1yr2q<mH0xyS9lD{M5iNl2y5TXEh_w>;H)T;jgJHvK2dTysb~5xV
zumcQ)8P<fE-WGNrP-4nBTnTD}Pu>;bsYt>+<1wD<N{P};XNqM+Xd7M&?Zfi&EbY($
zlIr{?^Tqd*WIAQ0C1o!(*m_y}9k6S7p_x{2*ho#T2CDYP!!KSRnWemiY+wodJljWB
z>!|6r<CGc3`l>kZI9y4OX02`nVxq$oDS^=PRy0wjEAbwQ2;!2y>@VEHGNNg*p$x-n
zj0f_Hu`sPOElh+h0x%U}!&$n|Dyl5E6z+?WV=I)U1h-gjmK;}R*j6$3FnYByTa~iQ
z3ro=y^+UQAVqK8vNjwp$w<Sa3=sX?tm^$NIYtg{OfN9$SY|)cO1wB20g|ryf>hlmU
zY>0Bc01)@f{hWFS0YQl2&X@qScFp?k$nLVEB&^SzZpraOOjdW8MCd3pr1XoN8S!u(
zTszs~b7qxusJ-3uO-O)(P4EoI?7%ai1knkoC=ccYi;QXInf52hmelEF#BLQuQboMW
zeC0T3hWmm59540e{p%gbM7Q=fkB(d~0(UG4rn62zvr&R@`Jv1tJD;dAVPHCwlS=db
z9rQfnC6H{OctmfrN87l^Tty8clugaCjS?2}T3f3kZ!VK}EIDwLQE@`~q@sD{u-$fp
zmWCZTdaOFVf*3ntFS4r9T)6$pp$BZeVHZn^1<8SBnf0@CydvOgZIB*%2}Yurm#a!U
zGF3rfBoq^qP-AGqIKB+w`I<A0vXjSvVs2z-T=zCsJM6OCu<DF~FP&>F_IfsUA`I~`
zW~8hfm?+5^yJ~k?a>l37@|NFM*znL(IYp4N{@64XvDhdPOHkX0Ia<Dovl00A1lN(R
z3p~>;Xrvm(p-QJFa&?|CT|Ycdsk~cGmG7Ae2<OGZFjk%gk{+>Nm~yC}jNf>W>d<~d
zhXbLsXx(YPx^@+v**D#2Ahf&u+^sSK$j`V#dh0g=#9*7R3a`MJ{c%}xIE=+4UAUEW
z4n?cevEp*RY+{Sx4;8LVj1K6G`19!`NwmLSM<nDxqKv>$%a~LMv(Yiu0NlXVr_FlD
z@6puJxDA%C(2FU$;z5s}PiF%WS+kLbu%amKI<I(syT;IT0lL{0l3+c(_BlOIdmh|k
z5&E3;r`D`xl$?n!Rfs_4Gl?iofmqqHmoY4)WH^A#SZYU#pd=$VtRby4mRPHUX;AM)
zzU|ihEsM^Ww@|eCM7#U<{!4Q<hn#o<ny2KdBL*qNX!bgu2{LQ&Rqcx$6u5LpDN*-*
z3rx`e{M(N3xt*<_e(D^*IXw8$5WPc~j1#%<z3mLlDSC7EU*-+q6r)|WEtj{H=ONxL
z3i6^H;~f;LQq0FyvmHex*+sZ3`v;A>b^ku|J+0u#=mN}!O678P8$!6)Y0s}Ru-Omt
zn`CDEQthmcvqk8lj|F!+>{NU4S5SY~#06as`EAngj~{kAG8PL3(cj+d&VRh$+&_MA
zaUR>HaX?3rguyV4puW#|00Qc)V54T)oRz4yIXi<kclEnGkK-9<N-slLe6TV@MB`uy
z3g#v$A}KICAi)k;EM^t2zsS2gRI>@gUj9QPD9lEFs50jaqu;RN&4HSy@oa?9&vMhy
zr-`0^^J-g7u;9RkxN=i>-Pv$9kV~nY6gW7i)tY06u@Alk&ZE8*1{<Pp^$E+kGPnW4
z+6!rEGNVl-oHdnpN&t`8NA#tmMW^D&%OOu@ew+lc<ChP;6lr)&APB3^P%X9vJFMP3
z@6LJ_Jv6z?<{3R}V`q13A1A}rgeJF*+~3?&=6_}J{%q)iUAQ7bAlFR@%T@T@@Z7g(
z$5=fCjByFuNJe1-17`U`Ac|9-#+jj~%_7_%Q+T;B<K(f9N`YFf+dzDafpE(yQS+Wz
zkKG<jRATkAHU|?=AR9`;(`lI4CKC?$YRRSXHQN@0GlDx72Uw)X&EkVy696Z)ac8mh
zu&>1U+R9yC3u~8^3SlXaQTm(V6jEw*QJ|i-_6!V6zyJb@dQfaYyZVx5@et2~dCsvA
z^Z6}(#w*C;Pbf5Jr>9IeLh((1KcboSOP62~9zS2<>a3Ia3H-ycl_lHETP&0>-A%%{
zhGyiqjmeenXL*bBuJ*|lxF<q`61+ogA)~7N*%FyswP8P;4rbYT)xoX#u1~%th}BRm
z9hj&fgrE{^t2h>-7i;xoE|C!cA8&1Lz1gWvyTT%dpF8^p`#aP@e3qGq)7iIdzS|XM
zPsf&<Gqrr3H8V(Jn6_Dq{z_;zHfBnI<r^V~!A7!|31o%>9)_gRH(6Nbw(*q7u_s{r
z{k~6P!hIs=C@>Zxh?FmbGo|AA9Jp<Z3qX_PzX=&@+Oz2w3q-hzA@U(TMni5<TI9~R
z*tb37a*!GuhT3P%n>IpfsB|T4)Ns0jo^p|2!6jN%cTO#!cSfAW#F~Sdd>6Jsb>s99
z1WINct&tfMuM2LF={_WT;&;kU+%p3-HWO1rjLI`6UNH3jA`8^QEr?l8sj}Jxb}|En
zE$&XEWMRldl{|p4cTOY*z4PTQC^NA)EhBO6;pEG!jD+q}=!xc5>uEK<pYtjiS*eLZ
zZ!v?3W5uFhVi1#o2Q&F&^!#M}CF*FSbBQN`CFv*m!t((BTeKEw+j{oP^V_W4uig8q
z+IA=0y}GFlv8HBe#f=N&LK$QqY?SjNT1EtqWth^=hNF4T@(`kqXH(}832=6XaiHnb
ziS;ms;SeJ@)mqs5WC9Ga!hpjaSY+$eHJ){yooH3|Q{V}VZj~bwi3|$=51fJ7+?g>1
z@EeK*#*8q=-Eom*`X-~98bZ+&1zU$*8%4ecfgtf`&@*@@NWl*0vVTV6b!_(1MlS1*
zBy1Jq7`8$`SJdO2g=0@~+LP;b4#h-jou@^l&f;#*yoHVQz5TR8dIuF&3_eNJBctz=
zy$X(|p?N?GKx!(qh9J~>{!=l(u0uT~Z-+NJ6u3aL)w~~{2awSW4v|9h8TY;M`E{lO
zdDDgGxjS~oy%y!V;T`HRR09j(7g%cRmdT=Zxq+pK#YVgf`J(yz71o@$Rs0mQ8&+YP
zN{yta=^6YNInpCq$!jN^8qp-}^z_vFMNd+Kv7wq%1%RIQ43%XDvl<O?CF+GYMCeHI
zyO1|5u&(Dlwq2U&C9G_XIk;jRDw@oo%n=A0Vfvaa@b^~5MVuvt-X&1Ep$ddlt;X7@
z61j&*7<lGWGINARV^J4v=N;`QdREfO^Kz0Xwwf%bJ~1!$%RWTRh6T7jyHpC|ezZ-T
zKstIgxPE!Gt)Kb_NBY|V(If6V{yN+}+G?kh)z5R(cyHrA%P174iA~5%kU@d}y~FpR
zA$3qHhN61mxQixr(Nw{Q$Y=TaDzR462&u5GVL1iA5WXR9@tQ>!c(BwMW7i_T%ky-Q
zm>^P&1{o#HyR3*iM2to<xemjZ@W@J}i7J#Zj2Ld@2#`SU)Ue!_y2yz*got8(@um_=
z9cpKRM|hperc}fY6e>*XwrsSJ@mN=K>7FMs*~|j5pD^gL%H5T%GBLx9vgVYD4c=h&
zyX&!dRc62znPT+-`Khyd7E4UPM8i|21`<^-w3RzJ^b&Ba4Q00{0$>_q2EI)tyt}jb
zwyb#|Bmmz^^RF>`4!IXy94t&sXzOnNgZP@&z#2U)+Isii+a#!-!Z7m8IT)iaTa*+d
z?wqioBxT0kv&LZcpE~>i-h@c!JO^w;;GY^B&b!HLKs8L!P0#MFNO|;5WTJp|k||?c
zc$2Y}>>k0vq%Z9)Pg2zk<D!eTJ>;0GxBc!UZKiX4sST(}ifi_ZvihAIQ&wtsB5||&
z;yqyf>#cgs3hlEDvP7MFJQXx->qCJpCJ4mCC4h8z0ghD};&6kTWuDCsHLl*FtLRGv
zYexK)TbX#G*SsJ;?B^^_n_?new{n~A=O&2ShRW7It8&}D3ZZ0HBU<pyF+M5MC%v&_
z@#=v74jX(ak_Tp^yc?=!%WPtE$pH?nzN>o;0rYf0EM@m|j$_A8<mg|A^Wx}ike(nU
zY^cu6yF4Q#93sjXg_NZ`!EUFQBmI!K<7+K8=0B8JGM>d@x0&^66<x8&@~?+00mV-b
zF<S~{^eAYYW<-IVV1k)v-k@(cgI#1JJ3>S)YH`@`2=6rGaXLU+wfZvldy1G@toC88
zOyWg-Q#%`tz7yJk$dD2oo@)i3MC@uWOoLenD2n;wDSh2oS2gB*UervitQ)?2DqCw9
z@6u*dI@@?QdhUt1v&h)RK(i~FzmpmCwmt1qJlsa?LnflNyzv2D0(KbUj%sHRI3w;I
z-3*?ctbSXiWlmUi+NV2F^-d;y236lSMaZq%gSsam7va7NSy>S&Cg>c06jsoI-*~X`
zR-!P#yJhjAoGHs?#7_hhFm5ER05Cm1#-r&^`Xk}!i#(8XF<sD%@9bGtRpfOx!fRqw
zhzI+xIL%Em{dLywHO}b&-FW;((!BJOtfbLToz26Y{moijM@x^Wr@u~VXD1@AlE`Mi
zUoM-I{?dgxli24H(Xy9Q6=|GHC(Rv-G}rPP&^IAoRnl|JTp;&t<@<3N`m)UFQVar)
z@D&D?)`Tx*V+02w9_lg|Ps7I*hBC0jrZEMk4CpsKf!D6{GWB>zKJKt3&f)>b;mnUT
z8@Z>B@5_Y-6tZofI%xv3=UE3UAt~51f)&DiQAIrB!m9D3Y(uLRmMKs=PnZ}=5c6zq
zAUMpiwq>U_ggt>(xy_l164xqqa#PXo({xIBq%ajNh4|5*No8(ijzUCLwlp#^dTSxe
zBwr;~uf};0;grZDOtaeuihY3G3eJ)Ofgsp^MkC>QW&;533uhIx$u6JqXlniVVD`mE
z3My&u3(Q+z4YAooz`GTD^9)zhGnTU<sV5N`F)Qmp=B>z@hYB9(K+M)*ir>i**pfu4
zb&wf?F+3Y7wO%UUFFfr+%mqskAqUX*8po7U!z9|w$NpxPIYisd<q)a5oehXo`d1yd
z+YN8$8#;YcSK^!)D;|b~QXAI*$~<&H$jN(wQbS+FYS979g{q2Hs*y@8t6740#r%mj
zg$|4fpnm2jr{ioQg2jw0gd#V8Hm9*zjv?MUZAl8RhmdZ`oSkMqATpiAcxL#`oskk#
z&Weyx9Y6RSB9N_f#PlW!g9qZ+kD{O4v;bXn(rkZ;<K~#WoqM(Vt@lx9%sh$2yNCIu
z*L;w5Aa@?*Ok)vhK!nBYAS=EXO-RWisV+yXLO1215Cq!~yT?>GYwgv&JBT0d33I^%
zYK%TmYD%gYvsx2q-uQLch_ve9K~x!0MoAWKir5zrk4&@Xx2zCN4Aw4uI}j=fA!Z+l
z3&_I-#xAiV{56S<h`ant>RnLAWQvXP`|3QT8<*k`FD@Qz<FIw=E~`<5{E&Z)WZDIO
zzJFTc^vRBbBDWHyZkBl7&%)c!&Kl({v4vv5=^1Sns%VhsX#f4*o|4skr6ui+Z^>9h
zaUKdTJ_jaffI(PD$cqMe%)6QTWBZj(u8sMy*{bI0h?xChe@aX$aSlx*?jWeA-U5Gy
zJgC-8=mta6*l?quEf5bCn4&J60vq6;BqsiH&x;tNK4M1u;?GxjXcKQ-77l-S5OCf;
zdnoU;s1441#k~g_IbPSlG%y&kLDua_4?%pLCCZg)kYa06-9sqS6j2CA{Zn+HhB8-A
z9(Nu;wAP1)p(X&!xdmji6JpG#Dnh!5J4m<ac@>Sojf#Meor{CeO1$nk6AQkhS8tEi
zY_C#goMnOq7Ht$<#s$gPp_&5GfOA}hsYD*@$|H+7aKhW3B;s5$8?hhx%mxe0yT2*2
zc?1t47r^P!JbdFtZ?axzVm@cy0^SI_+8jpNvZzk&G26R?!EBiFBZhOZ*$>=K=aqty
z(4#i-&-};+faa0X+t&WMe`*n~7b<BUx$9nR6-g8fI}JEezsn{oz;d+p*S{jHz$J0Y
zh1cC@JWK#|r>uy*_dtM(mV9)yvlg)QZ1c$^%v7vt#GQ_?>f<Mm1?y^r0s}kZjwFYZ
zH%ldysJ=XOu=G{q=I_}o<6gJ}7vKypIo(`NCMx9Hu%8*3OHFz73!1@<kl5pgooU?|
zU5iKdY^0)l%*uA%7B@feua_KATLrzArnS@|+mK4TcM~=#ck$#)q<HEDAaVv9;3e>g
zQtR{XY{-+-iRlHQ*ZAM!k=&29dH3Tq?and@M4XRDraU01cb7GdDPfIwR*9Tqj7211
ziV2=BfM4>+Ay^+HI+Li5?mq3#g24R+hTUWBWB49{tu=P(AS#L5?`sGUD-z)S0<$mL
zMb}>#E+}J9ci7-bU<z~3J<~bS=cy_b=5tWH3?aDWI57#d0!nWg;hyz!U`e2o-y&_I
zt#w@H&^l{bw=ES}2^8Y40>WYzvW)xFQ+YsRFYflxwuT@iJIAWTW<AG+1h2Cboha3k
z3Gi}E7*(zIja$&wl#e>}&n#D8`q3WB5Ui%dp6koku(6}*3dJs@fVk6g1Uj}U2fZL-
z7c7-XiJ>AAmQIO#oAdt?ipmd8ztJ2LC6h^_qN~RbeJs)_SB@J~V=umSZjP{VL8KGz
zY{uanj&M)(@`(p9+u)Cqvw`OF&Iq{LCOJr#^<hnX>)GDr>IWm(psl^@QH*pWh_r2(
z1!~swEhcl{6|eVL5G!(j@t+n^P@ZlxR<_3d5Idjr+UQJ01K8n)@7g1ey9;bGXPZ36
z-HJy6M?3omZ+G4X&&kg2{tugbc*8`15T#wjLN6g4%mQ76EH+G4?w}uB2TP1eTognA
z$=w-yQ&o>>SPD;|QJz46)m4nHIUWE!OsF?u$)Os{IcB4hI`JTazi_xL($95n43;gq
zz+DQ2aI{a`ufqw7(ws8cSr@8!l*O}eRot7^SvFf@-_&44{tF27qKT7ihI)%DX<gu{
zG}H-`eG^|S<Uvb+1lJ5`SkrV|o;t%G9D~8cgvyKz)z;8hWq43E{e<`~JC9UG3xxEE
z(3rS@2ghnAkrQF4_JJ`E$i1d|c>7#~1l2K=j!28tK`bC~L{)OpPnE(^osWH>+72~m
z1cxJMrpH0$Ro`er0fbhpmJ}6}PxdjlO(h}U+!idC7|up)V~|;uazXoHg)<Wlh8RJE
zB(5t6@&dLVd@90xU5iSoqo=`L2p1_H4BA7}_R!QGJd?cfM0tsL6qv?vMIoGn*y7Ny
zV1H=XoDBj`o_!|!=LlH?!=b$Oj>_<-Hq;o9os*IarJ+F&6sfWT4Z%k_m}-ehQE{K>
zP|w6T(cWS&XiWLm1j4zxM~!Q~I_oR8yT7-)zth>Wo2Us#`Xh;v9#B6-mV9O>aC!cu
zCObQeau!!CZ!hSaELnOKi3g}4Je6VL`Wh!~59I~cK0;XId|VCD&*LpJLAo6Daxh4a
zNvM3BwN0I*)7db-vSS&cY+Eb=4d0!pF$}j4<6&t;W)<QBKd}77$tD+0V8n*8r=E8y
z*WjQEz?Za^Nw?3k6=C(P?;nd4)tHRQOm9+3);|Q_YvVtyFj@a#ezG%@4%SySaj;-d
zmFP1YOyinlPWCmfs4m+2poCg{3HAu5E)g#nnVQ?H(mXz3m<+2UvN&7KYS_itRdBJ|
zj}fC|0@%2~Wc`t`i7PQa(h@s+&RgMkl3z~E%ba4=kdnVm=OFZw$>K>zJb{1PWJ0H9
z-zSq$EnxFEc`d*qoKO)jw<mba<rx8|Zr_o1GVQk2R*+@F3^f8RGBV_SHW#v7U>GmW
zpO1!@@Z5R5XagS4ZBN5SekL*j!or8GCoG5+z+BPXENdQ62D~Qjc9z+=FtLf`8L$yH
za)}E!$eSKRuna+%G8Y=pC3C_>5$l-2P7>#ors1ATZV2v&m%=0n&@M7#;^a~wE_<0>
z9qZ+6cRg}l(H7Y_QHGjue&b?bpeF%>)fmr+vvszB8}_*?-X`4`{L_bcA9hf3CJ!We
z0VMK|=N*a=A&jA{DZX0|e5aW<OEk+bQw3s&CQz%v8eq@KK3b|SpD&)r-myf0Z8Gqp
z6cCWLz3?(ofY#52#A0z6uxddkW)J`!-gVc6e<+R<n02#rFVorx9c!?SQFogjamj!`
zSm=LXC7umBtSSF%G7&b!LD&waT$a^Lf^~7{4?8`hT&vl{0eba~lHG{@L}%B_vUR7(
z%L)H1H>1H_45Lbq@KD?%N1n9IlB)cQW9MVZD#kK;I~^tjXnb}gR<}n(z#01~B^V1K
z+!R<#XKg}&6lZ^fHUm-oJ=vr;^R}0^)(>tL;JD+xBP*mUym>yfFjrB+d1$myC0+Ql
zkSo^2LHlJ`83=FncL_)h;>)}#gz39**n;fdB9&~GfeLY2K~^nXrhW&0I0WnC8ZZIM
zO}I8a6Cs(cgqb<j;JxX<_!mjcSMB%E@6HI1#pm~{2K4)mYP3t&7P9IpD3Eh49K{WE
z1gAid^Z=2XZyF=6{5iO&D3`IMYmPoA$vATzL%LxyJ<K6@)HvGHqxq@2vj=}&apGr1
z!@el?!QAK3$WkRLo;+v1qqUkSi{C^)jK?crIAHq>X7<d(!9aF0^W!_tNew>eOYl`M
zIjT*Xt~V8$n}mt|(6PkoBKjAZutvV|O_-F*QzVyHGN=Ta{H8<CR@YqmuFNHFQ}s_1
z(Isc~wy7??GqjIAy%!p7Z#J??oMVY&hm&xiqNcWx@LB*9G7dzWi832PNpv#NbdvgT
z_|c$bt*vtfo3$Wo?_U{g)--H&aQ1$rID66>vfKa{2SWsYVHXGS^Y(77r1B`ZK+sZ=
zz5IL_nx$(rIW^R)MBJz;?wGk)$2o8H$|5se-=q=-YB`@K8%YfEgh!aigQ)gkZ5;`b
zqbKXDbK{lKo6|;w@i{tQ7JaHrOaOo-Hrzs2VYPB%Z(;=aIXyDd>+ZlU6_ew?LW8{<
zpE~6Xg6Gbsjq`>px$Xs)INyz78&@BDuvCH!X4AB(mLi7ynPcY%Ae&{PATV6UNK!4R
zK4T;~NHUwn%vzsr82iI@Laa%M1m@gk7mja$#F0U)YKTfX;y^gCapLyxyBQ^raylVr
z^lM|%=zTF#x7^G)14@>KetPnu+@skyB+C4x%El!(@Ex?U>HxHCYf4Vqy1A*nXqE#w
zm02=c@}k^`>VG_rZ@8Ss)LPkLFS1|%bUt_#Y_Iinm;*pSd-Fak-=o0|K~m7@s%QzV
zO1CHJSS;&(3z*QZ-l=&FojVUV<g-sga)A|@P_arZfjyoi>QH0M^Tv&6CA8@Erq$`(
z<Ddg#NGa*=3A5Pj=tdkT`XAVBNxWJmCX=D(Scx~6nIQY{vP%0#a@RA<OM~pyj)`?u
zL|CR8OHTYKVYxMmg9htRRx6KRR&4IgRS&n=SVF;5M7T@!#S*W`r&bzjAR_pcmD6Gs
zQE|rkg0M|cM&3lFpE>W=P<<6md<`bDT?nxT2g|FlJj-g4A(qqRQ9^@a7V;eJrDvXb
zh7+UMsI;mEH;KRe;6^Uv6{9GI3oAAh1w;>zCZdUCen1mr4wiAKb;NI-q;z1{c%x2d
z1CMF7{_JHbyi{f5&jKcZc&6$F+blw*$%qzo9%mAkje|ucjN@Z5#-hGn#ogotV0|X!
zJ%at3mLPhB&Eq_Q@Z`x%v=dn^BK1t5V5goCuM~<)UcuJEyT9%3zpmeymdjFKN~Ja+
z&IhHK8OJrlcYM7&t&(K=*W0)&xu*Aw1eCt@N3Va>-);Khk#n!PPNu3cH+l!peMb;g
z{ibR8avVFgx4Z9*_wLb~hx*$wEJxzf{v;rl2=M}HePo_B>mCo!!;H-)z4S)7SgAhe
z@x*?1K_g5cYe@&|SS>r4JF=XrG^(8p03KVbtNFL}CR9^+h<%rzpw@edtt*a9HNJSx
zg$P#a^)l9^X*Ds6Ig`M`WN}rMhxopg>_ei?yygvc&r~T}6_|LLlhYwMG!Z+P88frM
zFKywYz9W{&qJ_<SvuPnK1M_9uUP5mx(aTrT$cIxeD3rv2^0~9zp-Y8gdqc_)F~^}<
zf-bzmYSBAOSqU$uW{#IwK_{z!jcTjWOZ@t1b&Vr+m3QTAu5#UgjO7Ub@gTdU!%}vM
z)UQ(7@WL$m?zX7I?kV(di{q!*02}y1bfU;3ZJp6+&rzfJfWOP#X5h!dzSe%Mk6;=)
z(p(=i(?fMpYzkD!)y@T|$T(m+pxF6zt8;~Cme$aa`oXz=_<Q#gt3(k_)ohK9&M*Z%
zrx)ah-e(3B_vOR%@7+xM_;uJHHK?O6zVA;^4@+O*g=B|jPaz*vi&cHdoOUt&xlT)d
z)st!GHg(l+=B(SvE|X+Dx4xIizK@WPie$>65mn}|$T`l58Ip!D<oK*buYq}u>Ak!t
zb26BtP;jto$y>JJ83m6DAIp5Dr*henslfFpmI}T)(@(eoKk~qU0^9-k1c{q07~Kz>
zog43RLJxhDWeV)%y-4)wTf|t&!aNx@lghM~riEeHiO>V*^iu`fYL68VV-lP69QehA
z&^D<Sdc=i|ZR8FPL@^r)A`(x|yl73hG-7Wx74;Nc+#v3=QLvHDdb*Wx(?okxuATPJ
z$hdQR{+l_2Xu}Qpu#y$PArRgWJvQhO_(1M;nTRoj7#o@-hoPNuSt8~06w=(~lVxtR
zJXkjHNzk1+3sTQq%V#o~eeOz#Zvn7z-Pj<BO#x(+UDkoJ%fFZW!*}yDFPib*hf9eK
zQQ=G&0$w`s)k{xLn<v*z_*jLBE8UIBY1&Q_NF<;j@0kF10dy{g%$zGyH5sSfkbclC
z3XJP|O3C<{+z@VCcsg!t1hl>zOn=mTPOlQ~kBEzS<0Q#lWCMhZQ{r<t0(FVoh6)GS
zvdjyI?_XcVN!{}_&619|C7Hwp*sdV;sMtd=>!-mFSnOH}tOO*D^2mj-C>k5(-U5C$
ztfz2EI01OaXjEv+Vshw2iT?H(5O&2@?xvL8vn2?sD0A`>Z!aT&_tFTpS8--^OnO^S
zw2sK~j0fbcte{vF0hi53p4fa<Qe~~qGm(Mfy)!YGj?HC6#Vxu}LsI)`n<Ikez8d~G
z+v|k}GywEAvt>EMm~eRBV~w<=i5GH_`R1S)PJ&g9t0owq-HM0aRy>-&6T23qOg;<b
z9z5Y7ySf#jouzsL^QmH-4=c?FpBlz+%y>)X&U}!m@fhF6crxg7(mC<E4bYiJ2#+OX
zu8X#A{B?|hd#h_#`oKooR_DXjdA1Ir;g+X=xn0^RG2?X}%WP6?6h(7h`yxaLoeEPI
zHmk69tF%Cv*4w-pzAtGO)g1G&&TNK<ojug-`@re-tMTP^;Yqw?4=|~8cK@1HxF<Y~
zb2|x4oP!-u#O9oqYrLeu6b2^R;=1GYlkX+7{*OcTNcU3F1m@A40AfI$zn;oV2Qv@Q
z{5jjWOC|^#EDRe4&8H{7^!gLqOo3_*CQY=K2P=!QNUXR{WT^8|9oIFEfti;92@-nj
zoCuHLo(Cv`K~1@!qKFA?^c9DvkW@!0Rn|<ifvO^_(J8%VQ)_+)?Q`6tdotDdUlU|K
z@i3v<f?6L<yG7Q*Xwe>d!XWm%_LD7RYFAGlx33;Q3=N}8oYvGg-<f%d3q`l<Y?laG
zvJC)Hg5uE-&@av_Gr#wCLDq>`ou*3gh$rqva+(WC66b5@S(|Hhr>tF-Y(k!up-5hl
znvvb~Ib>59<1A)N2aOfMi2-i#k}^>$a?zIO%f!-@9^IysjIFl5t6TXUmgD0Rdsjmb
z_d$Br?`kIYAg;Z+0esI1a&S)bnK_lS7|f^U)SO^I1o=GUs?h>e`RZ2B`@LQ=s;W$k
z2ex4{guXUOxsi;Lq4&D!VyT<J%{gcO;^1LaSawI|44?Tx0|pbApwHcFU$d)pohosS
zj@MV@w#NN#OEWdiPK{UmyWK6LiSnpopU~5+>uj>jX4oaRq<|Nn<eG7EbsS5HueaJj
zi#HUumvh4L>OYSVi3_<~3BeTg-I>{y$R(3guCbWMSy=)@_VL_1=*8gtH^HOq!ogX{
zG5K$bM3)ZAwnDJ_Fn*W_dn9oTIR-I@=oUjb@o$}(aD*QIGsddPh!t9z1<G(T?oD+L
zN0ANUtqhhr7{afs=gna@vOyBP)x<6%vS?5PBgT+sV4jy{5uT5R>&Q9&OqhlO2jf+c
zH%s*WY&eUDSlo&vpKamL6jiQ*<`PtS6O|oD-SJoug?}AtUfxC{r}?#)G`?89u!~b0
z8UGGUjiHOS5R+I7D3dwj9$mAkrN(+1Ly>t#q6yD*kL~)_>{GCKc{k5p=UlwEHvTKe
zHWb_;`XW1vRcAaFkhW#}tVdSxFpp|BkCB0fT<8-uhS+=E+1%TGy^rXplT_2<<Rwu|
zD4TS&Hdps8PeE5j&+Inwt<0H-N!-d!4{bWk=b<sw3n8<=&)_0v^1>h=DTGEh8NPwv
zWij1y=IKvt6bQw+!A_RMYY``VqXtnq3N84ajo=ej&*{59^pZ~{P_t|z&7~s{W@6*c
zb}h;frL##_1I$ejiJ1UsJMF$Cc3oa$NkxENLDC>pu|lgi$8YzvvjwKwVhv%@s7XCm
z#7u^8W4vUXCzT2?qBwRQ3M6wObK}F0*gocjN+zZfZA~rEhD5~v8v1P|@Q?Yc2Uzp{
zV=VWEqs!+7%nJssWuMJ?Btn_fqIO7LzQAwJzxuwrlwe$jaLX!b$W)aRmbkQ<%v$!i
z@^KQlQ?QeW?hvpxE51FLrV}r@e_nKe`JVOSk<a<xMmD8SC2}qS!8x~?>cKOv{K`IY
ztkgLqNDarEhq-A3+M1)#g8(zhZf`UiD9jxeUYTd(<p&k8LHA=5H!z6B8RkxP`3xZX
zcM-o_Jg-9RFYe4T%Z*Z*t8Axe5?qc#>#7&#bCOG~_@ixcRbpQQ70b&z4k3*Qe{&g#
zJp)$NB20qt)ng4=p*V4<=TTlcnRcxUH%GX%{)h+fj&~3CkJ|RbyTgN5^q-wqyFUpf
z<JkzhKo2}GmWQ6t2r8ODoY*i`siWX@kX$j6jiOhao``9*btT5~6WG}Nw|7PuO!Ta&
z<{K!+C+TF$)`<-$!R0qk?8YB>mG9JEdUFh_kYD;JIvUeir78e<oBo5kh-FjWss(_J
zCSZMeJU@Fs5~hSgZax?1HsmE>f2<%nbLo@ou#QF=ksBG9o)8i0dqRW&4zTvUp@niZ
zG6qO12iMj$&_sU`{};7VNRuF(48{^4FBlrh{Hfo3G^5ZR+aUv4(18hYHYb9i3jaBA
zkGas6?K0$5nZaqfNo6j7T6R6^o=?(|B<qr-wy|a?5hH31D$mvm-jRFB@N&<LL{ira
zxgZA#k0JRh7+y@WbI1LRAUWl=mJQ*=4{rTlOkj`9F+TZDLUfY2u<$`13j=vBN-f(S
z=x3`S4O4hj76)u*t#JO%dif<g>SreAV}4Luy0sTZYPM^HCq}B^qCiMAY$l!_vi--}
z6QTQZLSWz8#VW3ZT@JLTd83;gxuGjM!6pX+K=y5u;DQq`K&C<ZUgf+Zl+lw%ICw!$
zSS;o==r*OF6L3r?Od)g-pHWme9G5hf$ihrCz>2XKPkJuw0OtF8|2>2Tp30NsTNqw9
z^s!b+ID}ksb)4UwbH&i68m*l}wy`Cl^){_pE7HWdpt|<fbQq2<*HhnzN-5TWLv(U#
zLsXjBlN!?9gj6Vt&~+<?w4k@XL{Eo`k9k}XhlkQgFIj-jMT7#bAW%)8B%V~gFVuX&
z$rRAtbLQ~0Vi?#$%mDY0Sa=i<Gwo<x?;MdSHrqO!$~&@106!i1VJ#XOazURW4!ht?
z2@8B<kdoUl7XV<%oH$4&fV4Vcn!Z9eGL~cTb_HhILfs<xvC{^vt}W&P#Ps!KHlj1{
z>BY{yMX*YmVmiKjtY#Ae7QIsI^VC4>c|J%+XLQB9h+Za6p&v|jacBjVWag!O%n=Y`
zJ+)w=hmr#gr$SbA+EDKLR7}rh&Jaz{#&{p-{<B^{lvV@I@NM^cEwKEayU{^dJP2d?
z*6G0o0e$D)E4MLkH{W&k4-R*Zb`F2o2@PM3Im(#r#ca65smbK7{z-CoWCVUsvtVp7
z%mZ>SM015CmG*ec7*)GC4*y2%FhT!zvX^R{!d2l%H;G&v7$Mo*4GLWDMeD81GMixb
zOHEnyXf5>?CcIa*h!y=nVi<SDCf?W>II>?;PKZZnNt_)g=`DJSg^YBGK}|vRt*#a$
z<56M~dy08~I6S8aopr77aaB_*%f7DW*;iwm@3F5gt+HiH3yi4po|VYyZV)KN=RH}A
zI`zl!%>9WNM5+@T&zqk#Vw-x21gM)Qgz~gwvuO%|Q#=&HUY>4sLZ*h7SB0}qW=fK~
z^#x0>7#(1cIQ#Z|iS@4HGqdv4<xC{1jT~cAX8J1wzQhiiG$kHXb%m@Q<+BG$EOqs%
zR^b@r^$B~~KyKg}(%XYBH*PJ-gy;#J99>|a7Je+RIH_lul~;~g;K*$~n;VKY%kxzC
z_EY*jl04kkD&*tdHC`;@J`sl_kyJT*3zX|PDX97p6A<bN^>=Dm37_wh0>iCwOz)^L
z0X8R*agsV^JD$Wk7rNOAhg=dsuWSeAcwG1ykvJ#Q$+byA!+g8Nwd%sb-hE|T#U!ed
z;!x%^X;wq=!gqZz%O@Emi;f7M`=>#Z7*Qx#XwZz^$BmZTCeEI4<dfti9>fv{3h1k$
z;qdc0jJljs>`X;K5{3#-4+qqkUY9WrZ`muX7(ZIoZ<;GWV%uuUcr{p?aFQw|4S2}f
z=F#3EM{oBx$UP(0sVZ+7BZjUX4ra}qL_}ZV6y^TZOwLk_QV2Tf%PS@05pBK(<<zG~
zQ?yCIB#x&rO=mFgQ^r`5<~kX3&gGcd8KJ=;cQ6SagJOO~1`zyr)>c-YtUOu!@ZFP7
zv85W<zH6^N!LzdRM4uHfW7#MkXXmMP{t{qgnF5oH#&3&}uepAJ305&!FsMZKg4k43
z#vNQEi-~L>4@UD({;6E_hwHd0$WF5X#%mq_x$O6*=NnH}D*x+$e*URHHskB+ciks{
z{qC>dJ?=hUIax^_Cl5~_#t$Ap>VEfVb!F|lwTBOTaWDQZZtw0N9dGXKeZ?3n^z-=f
zL;jb3a{oSDefa3{|5|;p`smT4hu=MVyh`8KR@c@Z|F4yQ+N!-Dx?b^whMc77bWy#V
z%JYl+Z}Rif1PhTznl?i?qL?!lvah-!EeVa-P=iJQml;$OXBC8LGNHXvJenLN=)6C$
zA>Xiv8jYC47-%{%Q5+)k+<JvEXsd07MZy@FJagi!CbO(mc2B)nCy<=YMt0KHk=5X?
zhKCp>i1(1g#Bq$cZc<?mVF+hXdC2;ioeap}vt-oD&fO%CX>rlqoHJ{WsD8)1t9LKv
zJv9W5kp@md;7u))*kdn;EUidR1}i`DY<XY~lMSZu$>!HBcVx8a^F)Oy6^&44jLDO}
ziKGxM+Z8Q?3X%xV9xTubGMl<lDcg#cckb!5GrLePg!ab91`%Kv5a{IDhmGI!Q>oxx
zUbfJ&tQo4!1Lj%N8iA~^oJ_;n$Oh_aT?usyew1giFS6jyBnqz=h0rERURnvo(qOCp
zN<6Y4?cm=Kwd|NX<<k=r1`bu6P1(xYThKb5*z;r{BAj=iWQ&bh%xD9fH8n2j#xtE`
z6Zfo1_v|@$a+HX4o)dt&+t)YQ3t0fe4fCohRYC<9GP1+9XW66;Dr+DWFCi8X&=t~G
z7<jvJLS494ZB>{y{BMK)iDMeP?{rytscdel`rwUc?6Lp6^^6tt^smJ3d)|CDnj{y`
zn`h6?`aRi%_bTUgv4xFNSy`4K;os!CFZnvG%>p=tTL07-h{fvQ4tf&1S&x)wvB5Jj
z*bh60N4p36BDq0p%^*{EOPN5czVWP^4j|&zl({|mEHmF3p0JG5aVSRG#tM0-ouSVG
zem?DMd~ItbE9?_1aU@JOEzw41Bc!Jm6SOHid#{w~cd?^#PRZE)u}tx;I!j1?Qn6O_
z#6tizuI$IjX9Vn4Y~Y&vgBkI<cwj(l)mKqP0G$mJBWD0cpXN4kNhe~7$2(xNO6Ad~
z=V{NzjMkh)9I04x51ES|h*}#qPz(!%&!zrncQf)^BE774I+#sIWtq^Mn@{-tL#x&L
zBt5uS!n%5!c;+@3rq6LIdNFZ`y*SwwZ#tHylX}L@Tej1jj<O2$B5_W|o^oz9n9Zi+
zhH|H$>wDvw{QF$d70(MQ1DC5-o#0HxR_+w(R+u8ASFg(7ooa(K@~SFMyv;)F$~K4g
zG>pB2!CJH*&vgMEQx;T}Y*yt}xlC&pf}Nr^e+-(mye7U1&<-GG1s$)f;`G<0LHz;X
zE(_w+i)H~yk_NMoE9%=_q>~1)g+xe{UaVrKn~wV;k+U5WRYaxnC}9<{-nExr&QcHP
zXr@&<7-x@bwCm0wxB5%csY*ObF68d)V7PmO_&N(nA1y=V3}KaJH%0{h(<y>=tIHwF
z`BYxv<tnwQ8mnFN8$Z$t>=t}ous7~Q?2a}1n+LIB3TqSPgfECCINhljvw^@T;hGP?
zrg)VxSkC8P3}?`bWnlH;Da!u-<gQ=A(PHieG{)s7zIX8^N(^#bM$XF7bMJ(lSkdZy
znBXq<vtee<Z>A!NP^2PGI*Ax;i~5L<llwb0HgulfMgzEB#8f6kxfoQ5+lgES5k>(b
z#iqePrH3~ppt}tsqRCApDYq;`!I#0l6S*BVG?SROZ#<KKWji;4ihGgyW^99*km6Vn
z>8XZzG$fA74YL{OD#q)+;>-6E37+b1%5;Ox?-LI#-6V+Ll5KKg+?z$A-oxL@Qbc7-
z=yhd`X$#C6xPt$fGcpI_!h3EJpHF%O-zE27KD@3WqQpuhGeM$Q0VL0ueNcSNt4X*`
zNGicI1!PMH2x`R-NEG7NWOUJ=q~c#}`pQ2uP(T-x1h8n&sc@;%+M~zt@hiUYZl6pP
z%JtPJ*vDe3Zow@o8J2K<9O+(4C0VcH6q|6#1hsDEt(mwpt!<L32ye^ll~;QPErSW~
zlQ4f^qhwoGB)*rrSe{9b;lMYDl~b2ME1s!`+A{Tffm&DQIpRbov9EX%?Xwpt^t?Y+
zBLuePyw0u1r*OQriES6;<#?|OF+KgsV@C8Jcs^<1YF&iH=F$Qeq{(O)bOQD+a(=JE
z6gHg_M^jH(o|kczHM;C;a#|%r9YblsDVU@Cvtx64dEdOCviTy|%Miz@!~-y}W5QX!
z0Ll3zw|avW8A$_wvxW-sPh4z~DL><)v#@d7G8<{Y69Bk=b!cWpm;g|a)j&*$1ZP6$
zIcWon_LptPMlY40@=Q5gbIDl26WlLPNp{9897}_I#vZnhi5H8+Qxa8$8&Ln8Ku~jP
zoyj#snGbIU27OS=T@dpY(MlxJse09`?+qOW3<iN*l%=$kb$U5+S_gG{h6BMV{eNVl
zcnu2|ka^&;tLIPLrVP+Y=S*O%_R0C*EZj=8{Hi^Sq91{<tenNcoBAypCEFy8IS(FW
z2+qvK<&5&I$wUGqfeieRc!rwNN7Zo+%xVf+<nWF>iu1iQ>1p-S-govmatmuV)_<_A
zO<S$*+QG-r{M0l`d8sF&X%i+ro!5pNROSpH$UEHJjntraKAn!&+wIHC%ND&vXQNpw
zot(98#dh6LYF&96z5NgFqBr~CMq*fv0Vbu(S(w{3fT%)`3%bi~4yKu(9KY5=Wx5^N
zcxHcX#VQobCDDn`kOp{~lM#8%0s)b#TP-t!E^J8r=0Mx{OB1Vu+`@|5SdB~}W_L!_
zPn?voDV*5QA_imo`PENmgU>bwZf|BI{&Dtr&qPtox17_nTx(|{fvYVOwn6dB(V<=4
z`8ZtKBpwlag47b?xx{7^wc|JM>VB+r;So(PS@9cuBWCdSsiqIIZel2iO6X5re!HF!
zql`!sO`~;cjGdkNhTuds46;U9ijX-uxLP?%P>g(1u*{ilfvgu}745+$rT`tsKI&o<
z(g<cDSZg*?wXwW?GAprrtmFgq1)G`ECl?|-Nt4=I{VDSdP@E%*e3&?bU+!EiY+fuZ
zFC-VB_EY!MYx#k7YyDQzQaMvDo}j3}2gsFRBTbnHknQT>4>w^`LD(0R7`=IGver(<
z)|-u{vSiqATHC+qhqoN$={)6#gRsvP$syjofhTM*Fdl3&4A?><!ZQ$oVPG4Uk-l}H
zaaM8hJmpYLw9l4G+P$|C{Ximt-_f1<-F0E2;^C`yWbT_Vy0;)m%#uv>>bc8uQHv|{
zV%i67IdQ!cM4*f9ctUv15FSGU;>_*zCCgGy5{}Q~l6LvrR$i>p?jIca#L0~9Pkr)4
zzPMDyzQKti{akKmqhFH%wQ|Wg+AbvU#;O4zX;>15RhuS7>F2sXt*c>y*7G;k`YrW`
z!=%@TtOcQFjd_w<6hrbvkDF)SXi^8q%<MrNZrl|IiBY3nM^zoY8bN+M_Kfna17g}O
zfj-;p>xo+g(QLw~w!lb<13k?nD+Weldwmn6eZ=zjoZVJN2wq+|Fd&e6@ymg3#H@#9
zCGN{mESH?`2_z+6)NX#n)XTLkw#X7I)yDF$%IExGLp4lREUY=-G)I8}CBwl0T8LDc
z4ib})4HjD-jV&~qf_tfjRhY4`62yinqX}5&#lHFSZtfk`bHV0ug0SvX9~dWL@!A_%
zh*REnI%-*^xwXYO8|GBd$rwI1C$MGo+ZSLr(M615XRsKi7@FzHpwFyjC<Y13im5kU
zh`6%MoJ#7bA(#4ucx^b+`2@kLGFj6!hZM~2H>_(^;hh)nEJTPjo{FMJ%9-EClW89z
zKs^IX0dH)LNoo{ry?d|z{F0r@&^vF^OVkrlS8-EdDJK*L3-DCp-%VBLJ4)0$QjDFG
z;zn5)MVFNu0mivm0<Rp%E=>@ZtlS#S+S#tMEG%!93MttN3PR5OI_t){XI05|ErKh%
zPelJpH+9qiI5s%)WP@|dehI>xDFde-J8@|kA)y|Hy=Ij_mJmw}c{N+o8xJHsB|8_#
zE?i{;Xv{aBIi>57>itBm!9kg5JauP}emdHt%Xv*V>pX$bCDaU<eGgEs8*OmLVMvK5
zkzc|At$cY6@hPl{jD+mAfyb<ayA?F?+aY2#QO09Rxv<;L477xQ*7WL1&jOX>4NKfB
zeokPtoO!{|0uQrracqN`*(J)K#f9c|z;5B0Rdt%-m`T@nSC{V#9)|ICURA@ZZ`#$D
z*$w+zQ=O~vmdb3{e|8t(Ym?bTQzqFHV<IIhmNVrAPr9lPqY-@*5=58i_4EZWV`6Ph
zzFf^Sg6c7ETK?lrFmj$=Lcx3OT~*mIHpYM)JZ9fl=$?2dtjxRREBbOczjP)WYeg-K
z&@;$k9tv9g*;o-MtG^r3<A)9X@MNVC{q=DpT7_@a>e>_Yef2RFc*^qf={)eYJ->Sn
z-_g*R$CD>Imz9ThLM!2{9zG0a1%o6Gb4p0Ge1<EAg>hnLHJdjnSI|bVzBxG~vmeuQ
zG2zRwmYcl(;g;adaqZFLhF@7)ZWf+0A^swl1X5j(%47yE5Oz3Is7xNIrF==qhfB#s
zooQwkgBweCDx%lVT3mEFi5l*-UV06~o2b9BUjGSPF=^)ITTe$N)^bc3jyweFUqRx?
zM$_4lZ=U{s8>H-l+ZKtKmPbD!1Cf0>5v(fL+R!kfFXRU$1mUg7_`X^)$Qbg~{ys~m
zCORU=%@H3YjSv*|Ch_UiOn8evn4anuoUo<F1;eHf(`dEc(y|#5nn{nDm2B1rM|EvO
zI%KJKW=M5K$8;1@fzyJSQ8+{1HKZqrbC?xW8f!nq<n&^Vj^TF5U%3i`N7fBq31L>f
z!o$SR@&~KNk>>gXEYCPGk;v4H&Xs)?tZ#UD#j$2_T4qAoONn7ged8>m&cP1vn>2bH
zrcCf|e40o~o6M40Or%*RP{Zx^lF2Acz+oijePeV$l+)-loBCVt2;&kR)i`u+-53~|
zP&l||5Q3$B#A!UW_tAj9Nac7!^QOSs$x)BvAgw-b!p+$o6L(nid)#<fcX@Orp07(Z
zgPCNKikd^Rv9+SP-QIzp=F$&>f)&{%A+{7T?)q7GW&+!>;l?Rbi5d|XV()Cd-4=V`
zOTw-!_dc1OaVU&-MvrI*A)&fw{qOs|jkT4>5q18alS0a@bBqzkl8|dE_sia{Ky22H
zFNhCWx9tQKB;E6o5gwi=2>I1N^+*u}wr>vmOxrQ)RjN5hvFHwa+Y8o2PFy4#hCYr5
zptP^WxLN3DCGGl|Cz%<g90g&}=RFn~j~Dwv>>)XY8i2U(5~kZ|5Q!rrU{_Z_bGl(?
zYIrw5PzGk%JnUb-N_y*6VH=ndH#{K}UK{hOI^R2|;V(R}06T?X%l6d}f#~6i+-Q1A
zCH}8bo-ZKXe5S%XLw;DyA7AsbZ~_D`_e4kmQ(<voIiexbytuI-Io(LbRY+!FEDSvV
zJjVXPl-Xd^cI4k|j(0H!CUuUvjrZW5nBD=4%YwU8dhrWj)Cj`tt7k94ZqmQtm&uYG
zmVa@k7Ru6D+g?b`)s-wW$g<}lG&z5<bkRV`h+t;40t9o(Sv>H&fpl_grX^F637a+=
zxg=9wAJrhg=)r@HiIfvuP&Scr-qG|1dC^!IyO;zBqBiYnHq?lv^5djuJn<4^Kzr<o
zW7FIz)AbS~V2+pVUv0J{Ompu*#lH9cnd-q6Uj;;LN>BJ~%$vt%PJ+hgUU;4n^iu{0
z79fgnz3eS6f5Xpit-!${Sq%0!=)N`72uOGX@~HpuUsqRF9*DcBGB;vJRBw3{fn&ZW
zF$;c#`IO1HXhb&NPn_kXu&^FeW!H&H7mL)n!;L**=R6A&XP+$nNSZu4xepIG3SA?(
ztDjopKH&(<GQ+9)jtWXWIA4XVVq(q_F}d|>Z}auhM)iBK2Y44tK5p=z$iHE1=eF!U
z<cIrOOJTHhVuS*rWI{lj11E{&OV2Q%@;Z5XLG&X1xp#_Plf&$lbbf4tf=*ddpu5QW
zL75H{ky+IIk+~SB0d$j_vLuqIDnST|3u^#pR+?tkpRf?@GQP&`+xhAEaI=%!B{eY2
z?UPh$*raCgE-_{wW*;p!lNV9(i0?wb$gCKFk>{+GW#+Qy224ujMJn%SArUn~6j&DR
zNw;hl1uG6>M@$k_d+n`yJ?HaZ*?RZxwon9M5F^8M666k)%`<u+O}C*4a=AMmFUyhp
zK16uk()6=Y51WAO#+EbX+c$(Cu0btkqE_MBmn*8tW+OQ81Vga+P~z&Ekja(3?ator
z%frpXzjfYi9>1}RULN$8MF7KBJ?9#t)+;KJ^aG&pZQrnYyl^#Dp!(sU-eld_W*FM)
zvq{WeAiE<C?-Z+AKv_MT^~1V?tB^Ky`!d2Bb&adwkcRP;#Yx7*C@|a<D)wRP5r8%c
zefa?jZ#TDp=)5}I+2KW2I2?pYKI+Fnko$`_XaI7!@A(emWn=NC3HYZyR?Ju&Fg+O$
zQsWjB8(%%WqPT3~s<C|y#@Dfk`77)M1j&yH7RMfu$soCiN4d-jev(!R?#Z%CJ)l-h
zj0hxM%EHs@o3hP)_o9Qfvy;ijD6K}%wUHzqi@yCfbmZD9IOsX)&F^y0V+TDGHdrKF
zF`V)|bLU26B5t{7NDT;eIo!HKfZY+Z^!!IQ+zCH=f6)uuIiC&(xlcoV$L$xMsG3;+
z?x`9d$P=4MNd8s~5`4=oDERD_<b4kQ4wmJAGz;^-@zU7FB;wc!eM^J#g(ld#7AxX2
zf|IOll@eAaSgNyGfuE@RivU>u2tg?@UjMzI<a=1kMUa^}QiU;Ee{><p8-;#I6%aBN
zP9TyI6*IVo%xialYw!K`&i3x1MSbrLQKjxQ!Q9ntKm%(9=zI6&@9D+I*l#!Ye@_>5
z=GJm(d*|rb@8g{iNs)fXfx9IWHswh-qV3J&P5uR_O{U>lp6B&Qo-Z&Xzh3{8jZvrU
zf+$=eg3Ec%mHpT{isE74DlU@UPU+;5JY9HeTGDUvJ1K`h_-gu#$r6su&SENkE=7)4
zCbJM`P#++5l-~Bt`hO?-^)8Px4Wc`>>9XK@FrD--I2*M3)lpdy<t;m>_vMO`e7^G5
zqZe8>%vIB{)R|x2P;=j`5^jN?lVWDW?Py)EH^VCV3d)KZsb}Mk>$L=j<HxOcuHl+F
z!H9-ES7<|AIWf9)T0u;LJVAn8Wr9Ne@o6up!)4c<iIz-ViFr2@I~A{-lOW9l2osvX
zU|ti1OLY04krs%?6Qta;zqUdbZ8hc5LzJcnR3LyBaG`$`(2jF);H{KUc_EgS;O!EF
zIYfqM8SXSw4)8KLfxXscmP{(S(6$%-L`19HUPe(Z5O?H{Yh}LZ8?kt-2W@~yjVX+K
z?R39&yGPtHTzaD>2XAOw->L`y4@RT;`uH!1%7)9<>W^+Y#trdb9<F@%ARqr_ZSCR8
z|HOa!7yR6bw%ApOlcy6Ye4#-FShWUu$}cl<rHE=qZpc;YPIDFa)%c}y$MaLHa5edF
zbQcbnW+>kzRJ;-sFNy(2Mrk5}+suSd1CI!d(5keK-oJXa`_s-*M7Q<m&CcFlw7wCw
ziT-Y9=O8oi=Zmea7d%>kUu{lPdL#^Sv{8Av3(^OzmTLO7{z5*!@My(c1y}Gw$-rC%
z`S_xuWLB<JzrIlVFIOrbUsSS5m*eGNMf&@T3ZZ-ZQGb6?iBGbQU)%dz3jJ2z-@Hp4
z04UfUE8O6{)IVNC&E2J%_F6Y05`5Fn%(n_$GIw9UKipAamvoc+_@Z+3Zf9%v)$Y!=
zUoHRfMIbigD*lHT6<0u$Bf@4(|K`W4)3<!b_~!RiuNy*;2hrcn@P1M7g^ypG&@V44
zhae9-T!+7Uf76l=<S<hLt+($E4v(n~f9VfFp%7>Aq9RqOhP00{g&qDixk`9A%}sz$
z)IvyX8hGai{xnH0Uf9??nzD<B2tHBwFLxCj`Y$R!9qk>zJ3NrvWB#Bz?~f1I!4{v*
zA1^8$UQ}dOr;Cno1pQO`&uYD~R0EY3NzDd6(FS)<&(3?3jJRVtuejG!rp5tQ<AtmB
z!ZiVQ!X&)EU!YLZi{7G3S5bO3e|sH!pI%gie(EUwB>NwHvQ=Oh-LVdfUd5omR_APY
zUM>7>3yl%KU%`B{m0+0PaSB_nYFGmF;(Vl0&Bm|a5*T{n2ciIn)47B{MTgZq7m0#c
z{xe~<qXT;-sEYR|U>uN$tBhY>RNn1wncGVr-CuNdk2eor?;PW^{z~Az?LL~{UsR3>
zHPQ&3w+GuhDDnTMRsK7!5q;)g2x<@U$L1k^vjgj|H0oC8?OPrbztD(p<@UY>y63=l
zDAZpGLH19Rzav{|_dIP7P|>v^NR|nBvRm}A{WqG@D{A#}bL-!8l3j-->B?3s$skL#
zXX2QN-n^k}(+U2qua|XJ)~o`%r&IkeFo!vRxhocx)C~<t_cy>2b-WLI{Y=DS%3#ss
zsQA{*0<W(nBZUc>Q%cNap@cnBw>odO4|Cn>dtw{6=T5VAP8}Q_ZkcyLFFyV2M(#!*
zD}M>gI96(D0Vf45D}k!M^m~6gjK^gq>?t<4014>l7Y^t3`T4x`^K?-5Ik`d-)v}WF
zI4kW49(!fqhbN_9;fGOHK!?;HmUbk<tFnU0=&byEdRq3~`BIgYPMCc!DbTc}B?aUD
zsO-DOj4mtbcRweS(vqxK_8IN~W#2FRlA^t<h+(OgdjRT<Xtizt{eB+MO)Y9K7e3cZ
z_%<x=3v~<qSc(O&`_TcRDngORgwG8H6riOx;}?T)>McTVb=x5k|B-tMf`0BV?x%xv
zq|z6#-q*Cu{5jB*b(=Z!8&;<ok7N(zZ?k^x<c49nlX|d~t3dX}XB8BvkD!}B4jRoS
zS819Dxw#jN6*ULpXxeC$hV5jt_CoZ(aL(?qb?WbOtt?(0cfrLx2uphlvGs$>_QBRs
z=cl)OU(@PpIFNRnD^OS_?Z$f%@1ov@|2HH@b!q7)daITXq?y+OL)Xn8=nZ!WPcQ9M
z$pgfx;rV%(nn!4UVP_?e6$&+0%R3knhdsCLaC$YT)!*l~IePW>xV#a5ncHl7-kZ~k
zo`H!)Z1drNtMmSNcdxU%f4uYha1$w(%%%zi&o7VWygX&7S0Nh5cGP*lzk7`K#T6kU
zOFJ$<6e;6$-3aJdz9mwrl$3Dg<%EMPFL~-+VK3>%*Yk_z;qS2d4@}<YRz%LH`K4u4
zf-C){guDq$JEa6zuFh%6OYN$dh)eTpm^2OxD$KbgQd)i)q-<KCZd@TE?k*}Vp>w?A
z1v4t6g#1#I*7L?XFyOkO(it*_-B720i#j*l1+JrR!-$?XzvkZ;9OT~%#Fke!mmG9;
zCIuqaD=Q-yz2e#E7Pm>U=bTcQE_T#(aY3yi{e7NIlLcE|O2p=hCCbtKO6An6D=#PE
zz4k>K*A<pgx~|mHyYng_W&&Yr!P6ejFL`4<kG!@TaWB9+VQuyIxUpu|<GSUzz^^#G
zWj&#`e(G!<?(A>gSleWazp-MN;o`>XlUvSK?`<j1Jg>5FO&tyPbYrWlTTVrv0)>pD
z<HMb!Qo@U4q_ye!ountpsC#ajNL}~-*X>^(l(~;GK)pib$HB{g)7HW|t-B%bwz>iF
zxKuk}iUxenS2Pp;Mw<TH&hghY{e^I^X=-RvEHCK8Csr5qg$r2d%%{74N1uM&AJcV#
zp7Z-HCHB5@06RB6#NRV*0i!~PWj)#5t_Pp<>#LWquGr9<G=ZN3SZ{Fm@kUrIfOw%B
zuk4n+6}Hv?e9Jz{7GpP~eA4;LX||~K?#;pe-_}`?2{W~vO<-sb=-C3AcAh<7-j*-0
z%*)re1LQ38{J(8#2-8>$$T`iETI*TMI?t-Eb+of}u)n=|__t`?u3DFSC)y4&N3<%`
z|GH-uSwWBMzR_dIZ3}nI9fzNZ75306zPWG*t*dO1Yk*$x8ED(6kLpn~jh0@_sTnOj
zD^BLYAD;<rx)9yBnvGwZXB5KAdcgNr0_d@2A_lN!<~xpu4ro`Q$63U9R!=HXzoQ~C
z)<{&E2H=1RaM0^=T{iBvdiAi>ye;!<m?va@4IiJ)tL1<$sG*7lGtatzcwo1s?s|?1
zCsa`JCtuRcJc74qs;VWH!a0#*z2YyvWCZh8&KtpN>PBd8TLl`GSIRNJKb94Mcn_*S
zqF3IX+nv?Mhcn<Rs9#z5)XbSq{;9mxtV%kcGdaJeon!ffS3H|Vb1R#c{{+lguJ?xD
zYNcV9a~7$U&}PnO>Z+2zWBLAe3xsabvcfN@6ragk!N{D4^lOf=_~PDXZz{o~RXCs`
zlF-lHe8WGsZB~X~)MHo#7Ui#gu(<8Ly`7)X)*wFW(xGj<#fsS3w)@S_=Jw8EXM6YX
zDDqj8f>x75A|QMmGd=T^pNsUOrUxBU#=E5Vs0&Z@r5XYVbNHA3gqG}03_Cy{>+O53
z^TIw@BpdEZ;nAo^g#*@k)NeR?vk?Bntfs>v=JUei^2g?+P*(ZN3o)Q=!cOSLb{$*3
zwZ$E%Sx^&OZ4RCb#0sGf`oMpu|MEhbHs8JtS~TAtgU|rEQG4OPm*~JjPMJdKu4s!Q
znXf9Jic=npqGkp|n`f`6(-&^5ZlTL!3X83Dyw|c3>}|vDdD-A*zdWN4{K@7%#@fd-
zda9$)SaQ(s_xVrT44OM<ie(BGuf^&Bc`e?=Z`uLd;^x|_Eez^Y15e*L2beYwu^hEF
zslV7fJa~UhROfwvNlwId7c1W>-V7;Z&<WY1z1{tPzwrjRmtn#Ad1q21Iuv19WWdJK
z&xHf^HmV*Tpfqj5jH)>pHuOZf(%Z!cTM+Ba+l-)&*^}SRUEfyiIe+JT>!UxslTLNJ
zcw4pO=3s&^8hq1k_pSF^Wn%NT+7!E8E{8Cm2Kc{jPj5M!KL*EnC&hI^iAVKH-cu1J
zKHhK@)(ftVMK{NaCz4pleyL>|y)$^)#ZnnN%dybmZowaq>Q50afJh2)4)xm&qwgL)
zik6m6T>I)%p*|K?LY*FB>Q{22k&<~i1L?Yb*>#fVGD)%cS~$(e4<FJD&gXPjHs{4H
ziVG+^zrtF?mb+fMKeu{vT^DoSefOf@(7__(l1kB|v;cbZ*TzRPkfZUN+zdto*Ufw|
z@+y^|!k8!fZfpJ=+vCq?Vq6%`mfGA5M1!(^=0i|v_0oy2m-zRV`1VHOT3r`b)Kpxz
zdI{9q3+-ZbGkUTaPFi*Ip4@(n`5SfnK8m|%1Zl;Uiw8s<;<*PU>tl)A?bhe_w0ISM
z&C<EvEbX<`Z!Jg;y|!5(%$b0kq<67oU!L*0w)6-k*TJo1`e8pp1-LG5lI91kRj$p;
zhzj$H{1GY@7JhTp4$%cUin>4%Yjv-?n_Br9lw{1CTIkGVB-D+<IYy;(@K+LSK-d`2
z=b7BsH>$`=k*sq^RbXCtG}ZHSVvd+Cr*8uj^?9SPwa}k8N^v7#-?3hD=H*+Kvrk21
zu}UpYAN@`zv*h-ZzZKLk8mi%0&#a%ugY3^AT%PTc&A~a{ejgW|c7y-hfepucbM;@a
zedR;1{7Xmv(+4Z;`&S*#pE?zTnte1HIm~wZ@ovKDx@sT&6_;4HXq-UFclPqg-}%5p
zN-lqp5YHbySelu%sVyt>lmyW)$u#6&zoNZS3q-u?d6Jc{>yEq9{_^tsN~J92-+FNu
zzj6=8{ccV`_I2(3lb$bA`8^w(?|H5)=W9FnI}Qh#qyO;s-rOxE0sggn{indI;)A&+
zBwyR5k6xm3iwo0Sq4mGfuKjZ%{7*z3|I`cfPXL=ms4o1p`TnPW)iwWppkLN^ur(@r
zVOt$yD<;FJc^WyDf%m}!are=KTlX<gWaaw|3^3-FoppQh!s1a{I=Okh7t0GHlvT-%
zQfiE{u)HuzS(V%<a}8Q<tWy|mUbSGv65|-d5SP?h+%TACS$S@X<yCH)UMrnVy&5d`
zjPjB~Q)(!}d(?4y(Yhx#WdiUT6s}OW#36WbCAxw0Dlp+#DATZpgS4b$ErZ}~mMtnC
zj$Nr2SfCz!^`ZjMb7$=G*1afrb4Hj=l48*hJBLSvl)TM%=4IhGF)ORw`J;ir^5L59
zM2AT;7m{-(k+X8txdu8Ay4=T_Cd!m=wBZM*to)_hi&|}0h`^IeH4{K0&`Je|Ro-Le
zFX*ehL9;nYgzzZLP)@jo$Pjbq$-+af7d<ajvK^Gqm)fj{AkrVzJxG$jzGgL*zr5fK
zq0s5mIURGN2M#-nBq)fvD(Sc(3QZ>*ijac=ewNhQNs}|Xnea>+xsMa2+~M#(>1l+F
z4XsK6uyYq6FvAS*%L@oc^x6Bb4qfy=KYzl{*T?@l+}YfIyYn?;ERO%RvbOT*aX$Xn
z>dM2%{}cb~U-0ASCyPzkbd%mu(~H&~)bp|FIaF6N9G}OWo2_z6gtV6;5lK2Zfe1qW
zA>p3p5FJUXsqVFx$HLA)vbJ~QVNxN&TY}wdBvg~Ql~c#K*f)d-*H%_E`;^UX$5;<Z
zJ0<-~f0|V;k^BaIj%LGTf>euMa#9UO%_02Lk*0o2di}0sjWsduIbSHUDy>vp1{g17
zXL5SVVJ12CY@$hzjMB%ZyK(WWB{yYOF%j6$lX#5Cp{JUw&m`>ixeyJJ3CCW9w2g)-
z@mVRti?*l4a*fU^$i9ofif<)zkDaNe2}d|_I7muT+LOd~&e`|6pJkVHyBifvvNTCx
zLl_ael*PzqIYpia#C@agCVdXc$w`CGQVA?v+!W%cM!m{3y@W|3Jz1z{=N8j4B+C3r
zl6h%@B|B0qI~q+8t&|_J1mPO7SpN`pYmgia3I1qDhBUJ?{x!Nx;?JYRWqzR(O-y==
zcu+yi+==E6l0gxS9Pzvdy#P`)zUbD$yT9%3zh(n|-cZ?!bbLMOpPfs*aB71?;%s{o
zlTu>bqrHhV6t?&30S=MVkIuqU4h^vm(ex7POkyc$ofwC7-Am7MU1JaX2@+ooIi(@(
zqKW=$vjZxGP*3j<_trUK#(LY4=%ZxX{+}oR|GA&9fB#)z;M<OY_wc*#9?g6IS01je
z72<!c;@^Mf!p+bB&)@$$-`MDxocjWAs);xZ8w)SPz*1j=UN7|};{0j!AzJ!AYL1d<
zCHnM~pI&2-vBSw9sT5`#)b20I6+OzUk_V)())i89QQ%@?g{qYsDpblR;^yto#p|o8
ziG6rpG^hXSllZUs`TF}0aQV71Zo2>YJ$L_CzI*WCe?9u=EIIu7|M~s@ednK(?%fay
z`X>JGy9cZJ^MCkY^}GMU|Nnv?5C3!S_gX#rwelAtccW*|b`D-u-s9ssp-K}Kxe)5{
zndBmSuIV~5RBkpo1JZ9%^E`^yqj!ixBDv*TTnbJCWNze2EjOTLQgM=8^tGm`s798O
zM+1zA){s=i(oCo3_wU{inQvx;%uCdLM@dIEb4AcW5JleQ4lp?`(28~vHe!A&q#K`H
z&uJ{Jdhs;YW~LQBEo`HFC7QYU1@?ud&1lo|6Eyp{s2THHVY49O|AMC7Ze)JkNoeY_
zMF-H8<Qt{!q-bI5@z}ICt^Kqgx9DNevlGbg)GZqKPqN&iZQ2(vSt&97h=YdIfM;OS
zrmYID+?Ps4fNG=qLc=>(E6G*T?by%WUsP7%?DW;8)hcqadv~v1`3qk<CBIfGe|h?}
zVsBsF5Nj{2^PQGZpHkCYE>{)trA?_^DOV<ew{5Y^&lPLXVN)!Z#T6@lF~xc-T)Byw
zZYsDd;OcmR*-bS8c~?b4vzuZ8ZC9+(*X<I!-txBGXNCc<NTLhdVg*01f(fH;r`5~V
zFcFmV5yriBWWg#-(Y@R$uU3*Q?%tN1{bTPxF@d*8198LqPu$*j`S<_9!^aQ)=l%Z|
z{CMxbVi$vR0+bVAoQmA3jTiFoi%Nx{w?o%^?_hJg^TX!hi;bmUrR+<a{<Y9zf`TVk
zECS89;t>gt5n%sUZ_EF0etz%$=ZMU=%mB0S{y(CtzEZgVYmXlP5B>kI__^~<)ShJ%
zKGkG&5sj0{K$52zVRo%sk5>Qs*C)+2;^G_8>oh$ZBnWKNqH3FiLB!RNuq#U@7s%3b
zhph}?Au;Kn%xrcm&Xqkf`MNmYl@P}eduNjFHX`Cob81p2!?f2wRoQz(asVUYEV4Vf
zw91l#I+xLDB&pvye+)WGrt3T{{M^e;j-%V_tYDHXI`;{(FH6?+_$0kZoOqwQ8KuaJ
z-H_aGoRAd~b#GMBn|M>A0lR~k1L4k{2J+~*9Wm3Oh4yBm*^-%1d#7snBQv3nbcJDF
zYHHJ_bC7(#!+4rZ`tcxhJI)71g`DrlEx;{oKj~|xxFkZPPt3`K8=AB$<K2gj?hdOf
zxHG7JG_9;5vy)Mez9x}m*Bc_;uWSrbpwk2i@Hpifb;DbcrKi(NPVlTrlr=YDH<Gkq
zmlJ@|NB}Amh7$mC{ATwkIy!iD{Nv`~PDH=IJ3RPdmq^O!<=?1qhjl-PyRY9IM{f@H
zINSzNjP!N?_;B~-`{RSdBkHBPc|>iiT(r6Ww`k|5cO0eg;4s>K`)+TSx}*LNH}{Ws
zca9pGp>cQrbt9rKqWyzoBwpBkyL(I(j}IC=aIs-@@G5${bGY?}{@i@IySID%Hy-iT
z?(sfGdv$P#D;mAqJUrgrdcU`M7`=Oc`0n6nC&J>kcaOIAHh16dY`16zG-|Z-!_NM3
zbo2(gsbH<t&B2fRJBL_+ztrgE4$X7(<=zek<E0Yyyt8$THM!rmXai}AdkyM`Gx5-`
zv?n{XqRqp<HMG~Go&R`G)u=Gq-h8|Hnig2Qac2<R>;2)*TTGm`<LLd%qvPG<_s2WY
z>w|-B-tnWI!yk6Hc8;D#dk06nY44AAFcokVJUDeidqx%L_m}UFc6s+;M0$Am{+&dF
zd~@(4?GDXzlNxXHHXrO`!3;Ay2Zw({$Jh%#heq_{n;rUoh@Isf+r(BJ(avoh`_*W~
zW5krX<E=H?-+8^a`+8@8YX{{A=;z1Xqn$b(+wKu6?DEhA3N-3_UO0}6W`=`$6#{XC
zk2TtT6>V<+u#3rSbpnK=T?HH7x~(_5@tmxy*=*8(-$^P<O>5a7O-({@(Nz19%}%a4
zl^$Y{O4+2<?P8zdnl8EBJsmfphniOJB`32p)%xNhg6X;=7EpE=PnDS^f1e?hI}k?#
zj3GEgu-s(0u4pKVo^{jNX!<;QMo)bAb2OdA-Q+p<+s{mnPGr6kwXhvaG^{R_C83}R
z+g3sGAhuNF*;LTQj%0<*<aN1Am?|A)N*+a39iz&>q3bHoZ3h8+vZ%^|d&I}AinlH)
z>!jJ7CRfwBeZ98RM2@xra^%9wfJGfW*qfDhd3JG&4&xC$E_|{Gx6q`@<8w)c-Ot8@
z_&VcUiq*4=>fAw^%~N_=vW>UM5i)nk!vv2=cPi>`dVo0xojmMF5j(ixG@Cf<Yx&9<
z!)c0D?)bLjH=F%4dU}!#3`b!6Pn-|(c4DW}MDox}iu0-rJZHXmy*GAWzUpkjqH#Oz
z_ItNDB{~cQlOCQ7uKE4vgyyFl0&_x;N&u0`gH6+ff}fN5aUU6juD1Zb1UiprljMxp
zwC7km!b@q^O)~-;6-FM27@7dmF)@H3ied(tAZ?b8#Md#yMA+Z}z}hO`3SQLk|Ax)b
znfwd3<K0$t5f5g%CC3Nb2kV?)xSI}48q*$VVNG@WJp}eiB#`=)e<ykeHb9yD^*WiR
z<LUbPUOGB^TB!`Lqose_!AsXsE97XSRP^_htG>(ETx+dRkzi#QW=(`5)m*xKe$ge<
zYtiL-pVq@Ua$&W~nQ&uj$%xeDh&K3M6Y~<HLnjTFgjc6~d~mP_qKep){v;i-U6Dxv
z-oWEx8+bKZSjAnMT?f5(hV<@iM72s3Rp}dI*;k{D=kAXNN=8HZNdNNRaj!R71C+{V
z_a~Qj@83t%7%4M<pyZBAJ((Pw{Bq1&19Fx+h&bEDl(vito0cZkXAG*7J2ZS}+&eK-
zY2x>WzG!!haT4b@Cvv`*US?74-S*3-*bdbr$WUfe$+wK&?5tg<_EXUJ+#gW+(`m}R
zrPxi~I{vlcRik&gPaur*p-ZP*3sa_pqv)qf{VAOx5pX-}nqC_qgbO6eKTTR?dRoDi
zXi{E&&_v+gmxR`ru^#S=?E~bAt5tjHv|4}a&zG+Dh2*-9PqK7CIJu$bf&HmrENBd7
zd;x_6?(g*v+#S7Vb-n{Hc7E6e35Ji!1)A$H;8bR?3>#!@Lw7aqoibyROwQvm-RSNF
zA@_RR$-i%HZoS!geMs~$_w<>AqO)b68Ry;6+Pm#nTsqFysIw~XmAh!n1aTOQYhsz0
zhhg-War5_wd!1K1$6KH_(MNYS!MRkM8UMF=8uth2v2b^hc$wqh3fg1#o#>KLA}p7+
zRGc&L(wm*lcb#`cF9JWb$0%!q!ZRoYRQ29YzJ@{FPkASq$u%Q&&-zWyL>g7U&k}I4
z*~Xo!boTcBy#9tW?Xvd$`p1vox9-)yr_SefUAYO|i{S6(@@;ZK(7kc>Egf|~?QM*B
zZm$myems7YpV!$Wy_}xwUVsGvZ*#DpuLi=F`XoH4RevPzD_`l7aCERbnpNEGCS#=L
zQ+4K`|5-R{iDkuo(S8vVv$H*C>!fDVE7?Dp_0Im$`=dFHVcs~){;6AUPrwI<#vkcs
zN|>Z0Z#i{^hxrfnc0Eq08%9S<M}K?!@}O{X+4b;*(BxBX6Ksr>(L0!ao{gE9CYJ2j
z+@05ln{PY&n{SI7KV-C6_8RAIw5CJ{Q1qfyL_LaF4fVwmJQghtIEp9;l+1XYoDq^_
zN~YcNZ!{u|R%`D1{}x>J-|~0$w`$bN?{d|@G{5Dq$8YWuls7Je>)%SiwJw`vvQ4XW
z4g10CdB6Yf`v*VncV6!9BfvUVH5+{%rI(|+-Q7tdrtx&+FJd;j_9A%ep3;+dM!Qs_
z?K&quqkA5Mts`6t?zi7cT8(JbKf#k+XL57|QUD&N4Hlr$TR!V&=XlajlIdlVjG_l?
z&67S8j?}~Bhx~`Zy`VkQD49&m@L7CH|LOmOUOBT;Lf_g7D_G~%>X;p!?MAfvSn0xE
za?u&h2oL~d^hX<i&<@a(Tl*$zCSDD=a{YV|ZTx4{etYy|f7Gr=zY1vMb6*%~dha1e
zW(_yz^Tr!Orx*-G|I!d&DlDQhl1t^Z=<!KFDe>E*1|Ldkx_3{sQ5|kd@1DGHM<A(U
zOYV#KQJ%MRezd`jvH5pAy4D+r80;XmdoaV#65y(-klZXB@Cc@;*{@puA3I02J9>>b
zqUFwMf7D-oYD;v-sc42!!I&U^qRq(}i(9#W96NumRI=HLSjlvhL;aODZITeJI1+Q5
zr^X&fIi(MZ{69s{ls9_LzwstL$GvfX2^eSNn8+YqB%9Ib#;cr#MY64u{$-T}WW8oM
zKv{u|mG7I&;Fm^sidmjOvV+c8D4f7Z1NT%nmT31e*qR8(bD0qI#F_1rEoEwW`-tVU
zvMBV$h^yJ0#FzE5^=JpdCh5E<3Cov3QPW*J5*T3|o1S2*Dr%0-(`>p9Ozz|U3Od}u
z>KOC{1%t4U+5qnMG&K~EL1#^1ml*tFir>7=LMuDShxRj2X#H;T{FAn_92AgBq^0QQ
zESb8FAQeYXU|3xr=5cUnyb+RYji^9iEqHo;eBLO~JGL(nGK+dCqHyN$%IOt;7+$d-
zP&S3T(Yo}khpiqA!)qaITlDg+KUs0>MdjU7e@p1}Q&BH91hMLlDpqC06q2>I?s(>$
zb{wEJn~GZ8#l&!LnLQA!kO-a=7?*3u9!9LbfHaVJmT3<PGeKN*Rx$c8x5}6!nd{Lo
z%A5>vC+r|@O5Zv{jjRl|iH=w*F`hNU>sb>`QkrJCQAV)E7?cJD>~`B!blRqI`OXsf
z(b{KM^@b{$L=^0<Kc2*3&D65N;z~#=g$^+%IfS1P+b{9;{rfDLTmup6l~fj`p~h|~
zQ8yXz)=)`NKOO?%>dma*OB^*SJDVi3(O5KJ)EYEiEJFPZHnG0Wb7B?|fep`SuBR-Z
zXoj*80nLs~u%i>?yebvnA`(VvvtbV0k1!#K&7Eur_)MxJ*Yw6S{P(#4UqGP0sXjY_
z40{^sS4Kh(cb$lrmP%)(6HmmfogB{Cf!Hqquz`4SU#d8QYIMfLAbzh$AM_G@l8y~M
z{%mAsw<lx;B=LO#zyjB3JK1=!LPyZev@<$9ghSkTCJVtV;mS;(S&K=}xw%b-a%E>J
zC-}icN9IOIpB!mZj8~rZ`V;DP3zTok(`POc<6vn*6h`GR0lWajUQc2oR^my2dcAHZ
ztl(^Zq&9?k=FuO(4bIGE<W&u;tKyj#<uy5-jk?0u6`Pncsh>X?+3AIY@pYHMDhEu>
zL||ofajb6$evE)M;-U$6r!Y<UT)Y$ubzPC-B4E2|51M^$dUlb={<i5mdh@dGq(7ee
z`UYtxl9=*l+pmiC!4l0G=IDpl#rmQ2JE(7u$_kblJu-e$>9X{Z7aDtg_!8>*9T;$T
zKWY~Y*T=H~0k8jVD1N6cmV02KQ)iP-=j{MQXbcRwvl(a6ye2psIhuT%-s#*nbTGNv
z*~QtSW68rxe{SEecME+J$v3aIBUTNSAOz$rOHOA4d2|f<m(bDFE0s5RFW>1K8DeyL
z_A<ixOj66wwty0$rzRxfwPA7=*H5me9`5_(w-u%)D{HGoif~v)UpA7>VCfI$H!2&#
zH3~){O5?I4v(1enBf~(&b>Ew$<IX5Ki|y;O%j&hre(Uv5PZRii1D|eLL6@aC(qK2I
z=`<d6dI`o0N9U4AN{qUM?iNsBZdaW&Ez9*G?g8RI%*xBKBWXTN;!!XI+kD6g+Dc|X
z3v=4h5IjcouCkKHzLH0vR-j=@AqD~^L`*2Ec~9e$l~1nu=|p0F(9=jgya6F2)lU+l
zv?BU?Pd3LkZ>_96(hic|Sy>18ytso^?O+&R^^&tmlGp{jjjup^5tTeTYtw}jk8kM_
z)Ep+~EyX-Ztc3NE>MMK!=?JHDR?MqnPF82pqG_WL!f6jYnnO0!60mGm#8FVFie59(
z_<Cb2kbc#`2w{-;<3>lsVZHu=V;i8lU9vemP}!2(&-@@Ti;kiLj+!e*sv)|-d5lpF
zG9>k^fO6xr;dIkURu`HXcKRp6F~V#gY^uNLL$PK{M*{<Io~AQgYjVYQ^<L{8`NC?{
z`Hd@Y`&pN`^LUh`Gk2%y8QXZK?ldq2VG-OqA=?4yt>+&atyT;C?8hafbKo=dbc#%r
z@grwwlPvd`fu422rp@LP0}=eN$;S|E!I|!n=Ypg^Kj;_F55H2`@sy4~sM1juXS9=V
z-)-+6nm*ncO4qr;m?VwN`g~k^bMSV@#I^EzfsQkBz3_JB_9F8(qv8xt49m$ToK$I$
zwM^$_Kl6=GLW7fAb|>WHT<a6tC&;i{cDZfLOG*t;Ad;rOPA2IkF<TYi+T%~@A7c3B
zvy0`*d^?g`j$_P7vYZ`><6YyZb&tDj-4kn)1%u{|S8h3pt+H{Y$CDYGk4DmACA331
zOQvx7y5%Ufcf@An`0L)L=E!A0v-Dc4SMrt~zhk&}TmE1^T^)>638g`-?q!+U>e_>c
z?LT@;!tQ?GCE@wzI<ljHmJQ3I=TS?R(W1iKeEgC*lX;bhNXN=K{pWK4YhT*dcyOKd
zt@ZF|USC}CAJkc&mG(RZ?2>h4gG5^~?L3*bSwW5Xa^}Zf&b!cFrCx#9gB4lhu!R%O
z<P2AkwcDCXTO;<shP}d11N~3p3qh;Nuq)=nPgPQ6nLlh2tgk~e^NNf3l7#VItOjZS
z0QNbPcr?9!%GVC6WbQ6+!b_h#3*Kn6QncR<_LnsVysgkX470OnBg)SEr_-nOiETEc
zqvP$J!^5bG(vPE$P&Ma@e6<5qDfyNkN3S-IH}@j?<KS>Tl8&ls4)F`rg|UJZar_CC
zJ#a>=t=6BKY3|Y^&ECamLFChH98HJgrqwzyu${x^BVpZ(WI{$P*>Q{8{0KXBk=m&!
zlOXF_)ReG$0zu7$$)$+hiHYKN3F1H0>LTw%r?WBrWrZk2g0sMi4=O&LiVyJ;!W*rx
z(+n{|=y+H;Mt|$$y|mP#kGzq&y5p=+)vq?9OVZK0RB?rij&@)Fy1cu8y!?eeef%~f
zfR&wiTH!Tp!DvL}T}EQr1}0Ii1`M!sU~pr}#}1pE7#*1{B<`!jl#-c*`)r6KE(wbu
z#Vj_!&4dPQIrrTO%8G47DpvgNL|-LkOk9=qWu$%B`)YpDjF^h8oQSHdM^-CVtGVy;
zV3m~^p-r(bdd!O-RghII0C`jV3Mo$U>$q$<NMcGR(J%W7IZW{@UY(M&w+t;6KFPQ7
z*;O<KBbr~b$B=53)m#<Ja$<=5;WN`%0%gkdSv6}=C}dMvGv?|_R_}PXlARGos-!pN
zn@R@~VL25&C;{A3hz&(c#d(C{qhfg|Fq&6BzkMhQDgnnp_)|7NU-3`^OTM(H<Yszu
zoMNdbrYY@NM5kq!7oG;C>oJU_=`S$98yWv7XnvjdKXcP3*UWx0_t`4;$xmu8@%r6_
zQ&OTvufFE^d)J!Jp3zB%BP_VquNVOe{UnOE<77DAr}qGZ-_@6hK8a-5;(S`q^*Hgh
z@t3l=ULq0<$Ry-x7DX9s0I7x8q%yJB9nb9VxFYrsv2=6w?Af^m>7X5SDK{h%(qU&f
zo?i1vEMg;)TP<*^nDy&qsv@--tE9fEI@)vNIdRkBOP13tSM?dQt#VDYINyqn+8-z1
zii-0sSXGx*=e0m_zKPMF^{b^#v^d|&D9W<(R#cR4;i<v6QL%*<=NsBrSob2=&=wa5
zq*wi$xqHFICCd^jp?oY`T$m#+!~C(hINt`OzydHraRDF<nNU7GTU^`<Vx;GT1&Z@c
zlpW6IK&%uOU`Ys%d0?)^`BrBa0hH%kon7P`2*KhS<r+wFz7Y{Ki!Kut7q%dsE^0x=
z`4;1B;Y!AZr4xa)Xb37UZ1Kg}q83zK*xvraF{n7-f{3m~EvPu(!h2-DDY@KUalWNf
zO$Y+YwY0@$?W{`Sv|85A6c@n9*G3eZ`o;OCP7hH!TU%Vf4=g{+=0PkM7x07V80OtT
z&AgmH2o=rEX(Yu3m|~?uf8Ol4IN!#SrsWTSEiS;8Xawd>P>KsDrka2`uZ<QLfznd0
z<v?kR%Udc<`b_~sg5tsx;^^J7QN7~wmUBl9iVH_*spN0U0Miv0kep9G7Y7ZB3lQok
z=dz`G#f68=QPpKFz2d?Rv=n#gLS1ojvQPlcy@KNMRd}Ma#ZPp}y9vU6J|)1#xi)vA
z_qV0%*Gnd-ggxKik_l?D8M7Y!_M6DqYSrD*1(Do6)qdBqdexBV+Z=V;xDLXmv4?os
zs?dV`%Id8u9Duc!BY;J`W@L9onVw4JR``!Em0~9*G+5A2D8QZ9(;X;Qj10c5QvbFQ
zim;8@1Ri3O#F{`h#efL{)8bTA&_er`m&$~nbObZyMS`UH;-|SntJOkjAcE4Bvy-Y=
zR$60FfvDoTe9r#9g2!COzNCWB+0QLD?Vm@hW@8lk*g*dD+FBuq^|0EbwAE<DSY}k~
zg++?>JMnzRCa%xAS%{wL8$9$kij#6DQ8HsWh_a>Fb0{g4V=x7L7`)LsiQ>W!aH}(2
zJv1?em0atx&8ov5H|WT30J7D2y9MYsyTrejZ4wJ9OIGTnl%}wjzY)pbiKULdk5-~}
zth=n9UZ-2sv~o;ZaIt16)-3s|a?O$w2sTSTL$X=&d0cwzkZqQHh8VMKhW*mo5O0=V
zJtwm&0+yWH?coabbC9G*?*S4%H(ASa@cx)xY3{%#UH%qtgz5CU!I^(9V8E8Q+g-d0
zi@&}^$skK~&@$1usW<0cz>{X?d^j7ur<d^<FQa)3?!XeqxYvku$~_XRJ{EcDe6aXM
zGvcUNePSevQ6M=wv!c8g`i*5t)FKdY9s24_Bho_XpV&Bl)rvwl_8lOKzOHM2v3dCV
z2Rh~8gmyEv>v(^-S6f1j4=bPQ_55}grFQ=AOGii608zVyK<`$4qD{x_mhC#fPdT49
z_7uJB4+fk=Dw(Q+*BVaey-i91Aipm6{+7?qEJ9Uwqvfgi6q|GShD+g}52SPgb5=UW
zWwn!D203+u#7KXZjr~BrjeJ$u<Sbc)cm=sfwa>|Q=7x4VuJ_s0>#}iyjX{+MrfMBq
zv}C&ZRhxZr_ek}e!2$91w_4)T$cb?#0&+fqxZRg~WRQw~NQzMmK+UIv?s#Uz*!|4T
zyea+;$Qd>pMWbPa;A+XVTelV=*2}vHU#wWZ)s#6>hnsB@KAnG{yFavXf-2szU!Pyx
zw!Tyr5mgv;*who?b4mq~8Pnkg`M&X=(Xd@>-K&3mQTwpc{OhOt_1bSA+w@_n9XSx=
z_dkY(A+GR}y~FrEnhiTz2DG-c<`=SW0)2d7J_4v~DCsjPihrMlec;>u`)+Ik@R4qJ
z9*Z`R+@WP=#)FxyHlUn)Ko)ex8&aSDoaC^HeN9ILcunMRZNB>uM~Y^@3dc4{9%W9{
zbD8qJ`6A6K<v#!)N7cH2XPIsjEd7q-p+7AFTehKjCqN0^OD(*W-`s>A*%cf{$*H17
zqSu)f7Svb>A5!y%uX}Il1h$|Yh9!{YOBN@=@6JaS)+cas$T`kt<3WE)=(n-lDA%a_
z?<>(1QzD1?d2Rd8h<MFdA2jY84zv@Fn85ADrB~vIu|FrfTD6DQ;uwiYvGfQ1x0R*d
z-K+WYHD{mw*1o^=+i!BE)xhKxdk1cgdg3dg1~IiI<GdIx8A*7wglNk2k4W?Cv<ojf
zy2{td)YzL#5OxDpO1?|kMs$pS6;$mQxvfG<u0A{##R5*kkkN-BaLbZ=%cVLIRU=#Z
z)dc#xnm6U#5NtvHeUVZVB{|`j<z>sQxRJP(opqt}00OutA8U%`bjg@~WA|S^sI<G~
zsM41k7JCj32sm-)iv8bUaKC744igTTX}!)kJ0?pWdL>MsgnQ#yE|SodZH54OUf4L;
z7RBmbBXPfovdtuVld{oW*1%%A6CS_E2aDh4ue|O7orz-UJ-+<xAb=Dd9Q|Ya(E)gq
zcz>M#;6}7%|0t<1uOIG*p5iy<oI~)!73eC9fVn46>{Bxr3$t$c6Kl*R&&p^S|ElXK
zdm>P($pv&*c=1TAV~bW63<dtj4RuG%^s|`>ZzbSDj99{oYEG(pF42Xig3D6RA&z^f
zvHWnzfx<Fr;rjznUjj#<wk~akSKAzE4rn1rJtz@z@sJU?IzQZ=k~w4A_Mq!{a2a1`
zbtA6il?uN4#^X8h!-Bn2dOA^hH(QVH5=orVTUa2pYdu1~qoh}}$nyKBO4OtIvl^`v
zb5<|3dq3jf2|9s#BLdGz{9=x-(9=DpQ}FLc$*DG17n~xz{6lytFj|Wc55KGPd$$sp
z+!$NtdreT3--V-<(3xD6bm~9k!Gq*xR0hzphyDTPW_Wht7c!6j`iGh^4KX!+)58Yz
zrgUyNyr;Hh+3H20Q|Nz(Ki@b8qFLQykjvyG9D(23A&e7m7$Lhj^E>Cc{g|B8&^q`o
z0Yy#x8@hR5e}-%td~U-eTWi$x^fYJ-A9G?31_1hNG-Z=hbwjWb2&~04OFmjb5BsnR
z9t`NgxlEROOqUWK;YFrWKdPCa&X>W3y2H4F2qL=T5s^@Xo}eAq0le*}Q+V})!E9Jl
zSapea{pfs=9+j<Lsh<I|Yxbu?Li<zhC4+=0XWe#dRP-xG{NnE{&Jwe$)dS&{`N;-J
zGL9ZS4L%aR-yeN0o>HJ#un<$p|CYOBk+bY|?L6U=Yx*V6knXxy4@pxO(~Eo$1FG{a
zZH}5Xi-ogiSk_GkgQPnR<Tlns@rA_%%-tBZLeN1v?(ivX@L|=02b?KNW$+R3S_9tz
z0G*zN4i_F{nPeALzpYe`d9nP!L!w#j-qG>)?tWcDwt-lYI1ob)XEEy8dFXS=d;tfq
z;I2d-=n<04KCwHQAI7Dkz6skBHNd!1*;TXUH_Nil@g|xvjmbBgfR+lj3@N2tr^JH%
z^^2tz<DNvLFy}t*cZI7OAVMhUbsOkOM3Wpd3EaU3x3&+AkK4!TxK?(=bxXB5nBdNu
z&wte~Uu=zT=9hLeTf!NJToL~KBB!&9bB<3Ss>0|t&M_a8pE^tQHsPkcO(tiFIdre6
zVMo$k|6&ehvlql^FvM#E&=p@0rwHkofi@$zDccrb=IV=G#v-jMZ{y2VFu}D&Cn8;0
zbl~kjEh5j!>5A99Q}J5zxu8{-wu%1oYj-oR=GW}*TDZLyS;<D*;2&&i&v0(1N_%U=
zsunLcHwbt$Y?oT>&)==E10x1;n`S0dvU6xq+_=(hGgaNL-FnZoX|QER`6h_h;E3S<
zXHP40<WXfZY}3dHee6_Z`ffs{a@jeuu0@>|C8iP#xYo`318Cyw>w5iXCT@8yD31s(
zd<a=>Xh#y67m_D&&3E|HwV_8%BCJon+nnh0GPu^B`W$eb7$SJhfRN(UZ{qF`%p)?$
zR>UEUWB6+k%K-%zPhG;nSw?%@C4yy`oTrzq{QfA*Z+A!^V!KgBxcH4_Q}VW2mTV5?
zL-XJHKxb#WRs1J?610`!D)mFCWeB(OYqS(c&!Cq$q4R{&p?3Gzd^cY_qCOa|u!`v0
zZ#{yjSb;UW_m*?U5ymrO!^*mR;>^vtQ+$lf6o1t!9>tg6$*_+Fg0>P;3<ns{ZkGJ}
zV#4L^u)yVA3aOq($`h(fVl)z6nsNXi_OT^`wwEBlGWL@QZ2?av)*;-7@88L6G@TJ{
z$mnHn*%+Ggccuy}=J9bcgd)$Tv(r<P=um_4vW<>hi;9pD62!-5`lG2_5>?F9ID5xU
z?NsqSo*--sxB(Mjta3*r)F#Kmh&cwPq*4uBHHkH|N-Pw~DRCYBL}Eg7NN4tu<YX$n
zgb7qLvA0;rD|(R`09f%%ZvsnQ>Hl<|!)`ooWoc{8z7y)Exx!jRS4m}Gye>1YCvs0e
zvzR%+VcOGbKsqy%Y8@9(`Jwk*7x|TP&Ws9scZipSi)1zWwkk(#hMq`dD+G2p<E=HQ
zK~A=Ibe;?bJ4AzwliG{dKX7bhHEpWaqZak}+y$syd;aZe)n1l6CbEYfPA{FEFx8r!
zvlB&TeI_{31w=d-ej-z*lm=Pah<F>2tu&2^gpUXPnBn9kfwV~7A0s%a&`PKehbYG`
zCX%oUEF2<o>I`vKhRFzHWi9W;SSQ}(`Xupo$O?@lw!J?G7%K&+-CF_J3--w{5RHcc
z0Y0y-R#ZdXpQ&PV{)UUuYu?ON0#$qSsWucK>OO1O0zi~KkO#ClZ@x`2;nP!bA(EL@
z;Hde1L{Z}NlNkrjt|jv}v&>JN;YR&w7p>c_vSj7DP=9sa4G}TRKlhfD&0B(>7iZ^Y
z_G5A~e_EmU@LsAB>oq<xgYbI0-Ij*DfqGm2&kSGw4@a2YjwhEAV%D7VQS{R~9cMBU
zMo=;!sF@_xL=tU?(KEtG@l6R}vTC!Pp%=INPs=&gBP1~R4BM;p6cWs<rxD&F4!_dx
zg1&?lBWsAv$HwzlF-s<n%eA9vI%W+4&f%1M8zrqe7yd~13DJ16skwMorj+HyO{!pc
z5f3=xEl;`HCJM6cM{jPQjcC@*%gf8AcBN9?lB!o_)y~*GzS%t|axgI;Py4jJ=@tLU
zyX6x;C&ZvpzrNN$_j~EYflw$La(nGj0wLIJS**n}DlhIwi$TW0;=jA7`v2Uk5>D`F
z8cT@4<si;~6}Lr(=W7Ebv8-64?5=rjOD~+#L7t?FgEHdTH(5=*2*Npm7VpQ>Z)`GT
z6&xz4nT<|nOtT9`Q(|1k8q}1CpoP>Oi^ZH4HQb~hlL6-ep~pk~;lGhspX{+Gu7At;
zeYs`u07qkH!ex(RJt9Hea^{g=F^4n73ea<=DMsLVn59nK5Q$~o?!HGme^VfbYCdUg
z=<no$=&oN?jz%QrWVpU==JTZ<`HVD(B3k+=xy;pzmSwri0^`Jc(UVg%L(sN%Ky9OW
z^SL3TYRffgRF9VHzBQOsi+epHE)Z6TTC)LC`{PmDP}NNAc&EQ5qV8LsAN|`2xC2PP
zG{6H3r&*JAnQ(C7IE)}_gz9v}v|~mURus6}E9Vi}X6gPWt-5)Huw-|qYDFyA#b<oX
z&&@ZH+nq+=*v+O|rZ8)NoV43#E@8+WA<8tHN)zGuaizs9nJp=1g!senm5OP5gf(QG
z=Y~B9M8wKW!%4zPF*(F*$*~yHhD8(9OUJiL&l|u!t^8km$aX6b@CbGQ#_*kwr)Y4W
zTde;Aa5VU1LQu4imnlrKGJ5W5VPS4IZzhDJTg-jo46|r8#NeNyc;Fay97cLYOIc_U
zXP*<dm>lBNUDdbb_M59SGq_N}(|q=l)R;Zv&?y`fnXvNd3~7O)VY^M(E{U@~2T}>e
z8}9h>0a~g)4P)FBr1`M@^T+HS7TPZGc7<xd#kQKaCvWKcTM4073+N@o*0JP`SosNy
z_Cd<9*N36Ln@#zf=o-%SnKCM;yHPd!Q?2~bs2w&s$DD8;e=IVGjeQdAR}J(0jYzzD
z?PKp=9ba1a>UIZ9ZLRI=!I_x`{b8Rex}`M;YO7T!*;%r(R)_Wdiq}A2{HOMV7|h7u
zHB;O$-vNni?dTokCS9xA(u2C!s}ujcLs+WJ*$5fPo_7PPRX3KK)YqE8EHHw(of*vU
zFoe0i>4A4ja{%8#gN07O!H1utkE2iJz~F=N>g_h+COatqjJ#bj5raIFQznm=R@EKE
zqcgm^H|X8%{6qW6Qp1Rb>$tvn0wFD5v&|Zy&m*VkFPYQ*`(_?D98nF%2HyiC@Lj#{
zjt9Gi(CxU37UQmhK%}e1lZ8Wj)fll9@yqHz<EF0t&rSVm6EL^CIL{kzt-Kp~a436N
z{wXKC>h@MoHR{eL6Hf=j_9tBlz64ec2@um6($!3{C)xkN$L$Rf^;~WQirU%5Z*<*%
zQv;|454$5H77!Bif_$kqQqT*s50<%U@d$Z$B5h-m6PC5JQ<&41(OFqTnc!-GQu+8`
z6+92(C8EMtV6^{2c(*Lj+@Jh%{<~d@e}duRLbJoKF*=OC^*l^&Wpwz}CWki-kuy0|
zv%%X~8y0L>7g8gOk<3z_BeQiv{_eem#WK9_I?JJbxtZSqWQO2q!z*DWqY@&*MDh#d
zqyA;dmpQF`@Y#gr3qIPYd-}EV$L@O0Udo@P)ixM&X3v;8XN4?F&e<Z16W1T~nsB}n
z-k#VUp64rq+GgsTr|g)CcB-phzMC!UTA+QP|JLrVv{p~=-mT{iwU>&aZ0yRtskoJw
zR}D8QXnJpTWu>)JkJ<tBgzn9M+zq6FXywE2*hp^3lKHRL#gaHZc5nQdeAPbsHNtKd
z?y0-aOBT?-6a@U9Oolf+z!}e;?d)$Kyn3^<_wH%MR~~4@Wft*QRMf>mPR=g)KhXz1
z`A^XY4LZK+eqH;d3*yuZ95cO7JYIC(r|#k3g+DZ-4mLb@*IQBaqv(va8)U$+E|(1m
z?k+QyA4!&U8|yc0Z=BxP>QH-<a8Fg+TLaQp6M_9au^nOvR%MIq?L6g_<;lcYN~5mn
zYm%J7O(;sP#t20L<+4eM7Kb?|O)gIO44IDLImEfm=&6za-fCy};1>tIF|59OCR)9@
zNVFf<>1^U_!;BL<_M6d%cJvxS!!P>DWh<v<EOt<;YsAlSihQP{vn0?>vXuKQnM%YQ
z<8X2?qPdS|R}FSGXVbKsm6Jw^+H##h&D-#YeZ@hgBckrpZeQGVb1L47CdT`qhoVV-
zZvn!^tWH*`cFC|Iku)XSMK$ccs)zg4L%SRVe{g*6tq;Q{UppE_f75j8-2sf5X}`QY
zV{CB(uyF1H-IQ)=Hby{mn724aigy<3>kpj3ZvLXE&f3atYCZVw##;7lFtE%xo%C2u
zd2$^wt!!?CM5%*t-y@FsqoY_Q^`5bjhmmZHGe5>ADNrA`E6ISSH~@IGOam`BJnsoi
z<lnu6F%cCQmDK$FkoaWmx6o>JrH(CB`N7wNXD(sbSHMQYVpx+TflY=44Qj?2K3*%y
z^M5ByrJ^@89g{>W^HaMqN~XOWr7o4C@o;NIUb1@@`?L0z$K;4sP07ompif_+p}<-G
zyiH|0Ba@`7QhCK;_vN7y<Rj_N?Hds3=0(Wx-LSY`lFj0cG03_<s#rX&zMU?~Wm1Ol
z`y3;l6C1*8O2(;hM)O2m{1h6b1nYY>o3NRl<SFPOx}Ii`wKnk)d*^VfO$|I?GEipy
zAyt`h2FhFHpX`ojh7_Ji-pSjvg41BGS;bNdL})PgP{O-g9zvt9lyU+RtR-Lr?57u3
z#^=1&j_$V`(o<kjvJ^+P7gvD`YAg{Jey%l}zcuTO5**`Hs%0$5JHjpqBh2YQlst2r
zHv<`vB^smFMpW}jU{T^!e-r=6i0wJ%1mlrylMXB+Qr159b)^NOyIL*R4DK3I*rTYr
zOkv>;1j(G1wL4?1^2Jf11>ufopsP6~+5XluL@}(iXOY8u8^DaH#6mC<e!<wvxx5N3
z?xBS?LE;7Hna2JG=J*W#Y>8*}oC=di!$*ep0dI>$3DR_xHA$dem+XRSV+Oia7;EZB
z8If)DVwPB7otg2K1ED(yDn>h|POk@v7=hLPkxbHhRFfJ_<Z`FKY%so>oLrM`b3g6X
zwYnwuG5y-|-aUG;R``Dp%rEr6Rr|Y$g?zKE&k_s=cInnv!&ikEw#$?@pZ~h_`HSVw
zULZhQ=C1R{5w7!RPs6ixN2)vzEWae>{8%EIXpQ?WpwS&xeV(Q%F-B|m?mbwmo4oRD
z)j3H9oP{g8<jjC<sW9m??R%0=r|Hmk^I(mRbSi1)QX1{ST9eHO%>IkPx1l==d#TI+
zHd?uQr9TbP%1VKmos|X^dyrL@^dJRZ586c#aAr0ba11;c*I|?Lru$sB;Z=NUi?!@O
zqpZDhwQ{fBX45ZlLt}WhB8ePRdMiY$1DFz*dk*I|;9R~n?ggG%tB*sU&K!c#fR-ln
zt&5ql?oG}bD98_FW$y1U!NLaVWk(kWZ<OgoX3~h7PwLKW?~KD)5#jxWQ%n=PFv)^$
z&iiNQbNX3*+=#d@=?1xIQtC(3Pv<);94u9-Ew5ZH11X!y*2B+s@VvCpvvd(`<mWaH
z{2?u(DfqHmbTdaJH%EXs_JS@+)&xHi8s4|Yho9S@?k_deB*;)k2{P?SyaOSSt#@Qb
z|2+>T(;3E-&j|zOx1KfNa<kctmK(WR4Qsd(c)wtxIXjnS!Ok1YA}~60EwQ0$Hzx63
zx>Q9iPT9W*_|1BW`S>kM;9|BMEkE@Kq61_PBl*Zd%-B%PWi+?BgZR~Hs9TlfR}+S{
z5ASAn#^aC<(1Ev*g)q4me*v3>y2f@)$z({Yn<5$+F*wbp?K0CIh{8<HVuVV~4KCQr
zMuYl=*0c5$|2_k5{gwCO3xBx35k1J8)nNy8vzpI8EdBDys@HXEn$2Zjj4iGFjdzE;
zY)&MEK|N?IX$}22+4>VU0H8;m(D2R9=DR-uD5Lx=6ex@OfWmwvr-hr*>%)T|kKg>)
z&dJ4RcJtag*gxLcKPHm?pFcMkaNfCblH=L3ST-(!;n>~V`R7io1jB??=b)H59{Qfk
zPyf_K{tJ>qH^8Eei4^J0hGU+r?Z#JWh>f*eP&gaZjt;kIGSB7>ZM0R2%brp7=Y<Q~
z@vmcdkY)+GE%S-t+<|RQd&uV2-Yy$A9+vuar<yVm32DQWfIw}Dc6TYZ_=(s|(&L>F
z;>wf3HKvx;i;4aF(~R@ddO@_<;1-!>AfIg}Bc|nK6B&8N9BxO&#@J*NBa(0*Osluv
zy*C<wx^@Q~`l^2gtD|JHX`%+#>J5#CAaTo>yALuR;`}6$UylU?IyuNWi7WpMH~Nc2
z4zSlx;G*QVRAuOpu`yZ7u#nq6-d1j8iJv=CF-A6>rg9n{%mj=E>KAr6(0T-P1X~vh
z&^gip)@?UY3#xn$z6Gl0pd_$Zbfm=m*l6h9gbVnQpEZBMaFso|{@%<mG<^my{(Uf+
zb(yXmQl-_M%%!C>le9s)b1aW~=e|y+>{wb`Vn6gc`|5@tV-7A!tbdn|3~sz0{i;8P
ze{xd{Netv%bDgS+{j;yr4=M!8cT+7`<b4qQo(+;@CS3aLm@e_ov<fz1+I(LEZ4#8z
z^S|<mogmqHC118`DTAC6-Nxwl83a42pu;>F?6|~Jj!(sV-OYunGFrX6PqOIlWBtGR
zy(D0jQ+wt^QoTzi-2}YXskLjESHs)yd3EhL-MPJV=e5OV1U8cbuEt`(crVoas|F4l
zW{3*e^k?2ERdK<Vlg$ctIcL{nL=`gvO<<)-GESHP_3~hfIG6em%2JdF8O!0%Rm(NC
zp*9haM(``E(cp;(URxyU)Q1ZrFt~1M0&Z4FK&QnDI1_GZYT}i|5QqZzO`@23`1>rL
zfEq|ox#!u)tO5C1i5a3^mp+4_ra3IZUnka}ZS=8Cf6uU#aoksJl|En^gbDImhAq)4
zSc&#ipr5fs-o`KRETV3>JIqwp*J1>_W#_Z?+LLqDFslYni5*7A&KrP5qP;6Ya%0>s
zs>A}7JN!m-LQ~ptLQp^~4PeTPzT}FAAi77a>G=#Ut-c>PWa*i4dNQp01w68zc{IOD
z!>X9w28s<8ZzwuT8qo~vrLCmMZ|BGjjqSq0m1WV;0f^j2)hv9FaIsgFF4fX_53(m-
z7eJ|0477FM%fVo`3_D1Tp3a+N<S#IIBtd9^GO^jjV-X|~h%9FWwKPMfNI222fr^%U
zP~H79ud1=iF32!(WAyL4=I_ol?eIAwW{52ul@wJ9xV9udqi?Tm$1gbGAxi_uoIq^@
zXkNDM*@!5As($a)ZzU+b<@}hW-Fi(F?Q>gCNLazy(bnszR^{-o*?RPbt}tgFF8Ge}
zY1H_#{Yvz$JVYfgEq-iZ4Stf$vn+r7_FH)ygLKhm?UP32kl*KwB2#JCKK#7?>3)5^
z_OXZme)u^;RA(A3%9XV4*X!R~FTB#94@(`w1*`P``NK+Fas{U{F|5cQQH*PVUu?`A
z-bA=e|E@_xY$AOmla89==wAadU^Zp?Sa=Vb6eU5kBxsh{X0$4k_kykeu^sgP2lW3(
z^#8~7|L-DaF_{m0gDa_l61?ZEVYn~jb>s<Ml1bZ}o`~37!VZ4O8q_7gQJ~h?PTb+N
z(C6$p7LPO~kQr2MMy$JePvnM-i%lF@{}$xmiCXn^P;KNk{9`$a2#lNNbT+D%oDe!J
zq7_dzZ(SoF=c%poVY=R=lUq&trv66NB8V)WcZgf~Ds1TW@F6cB)7x+41GZi*hnIq<
z!q>b5^Z;J?zg59|inD^JfO{}Jt91KhX!y22z=g>u@S@DVnho=@FY*u9Ui`{4`Vz=S
zMNB*8$B<bRw%+W`Cdo_q!gGilHnV4p-3X@S@i*z|DU5+Q?iT>yXm(;+t$q?84*K=M
zCzq_j&dsRog=2ep%`}#w4*dD~+zrniZuv&?AJhcm)NY04dFnMwM!nlKa=Z}oJ0u#m
zVIt{tVJQ$;t`Q}vZM~6%NTYF}G4MCWOrhZ~!f6mZt&g^%e`;RUhS_l2q&!i8OrV$|
zAfN*s^kE0k{g-;E2g(6>(tD-XH~<Wp(Jc(_@tioPnW8=V?Rp4y!`o07ywm)!^RxZi
zJ<GX0-W_#S6*gCsBmT$kNnoZ1@%GtlWX9m21|(q1)+={(s1I(oG*ZJF(KOiZNeJf#
z+xJaoiXfTgGc7(*Pdlx%R>SpJUAcO)^0@o(K@S0E+NWvSK8Yu-ZkPUCeL#;cvO%Iq
z>pfhleqwax9i*?%KFi}Cd!<9`aR1VUDt>pbLPV9G{KC?(4gQsr0Rq|oTGfK8jd$4{
z!r8(Pnqe9vy3Q8I58~xE2PT$K6Pzn#Z%x|0lEBvX2T_SI_oi;?#>$nqn421=W51nQ
zYE`vV3^7b8AH4jx?e}lrAp{T>^IMf=OPav~pVp(LZ_GOc<vMjalQ=InVyI7!TN4`x
zC-k|wYCp&9koW;!UGD#kx{QEXr(}i9e@<isH0++IV@GKWe-mHW^^ch)K;ilDn;&<5
z&zM2+!yMcdQzeR?nH_k({>-iWxe;uYIJ!o;4Kis}EY~}1{~X=toGo$l6w}MQdF3)2
z%KVy5R+~3f4sJBxv?I`WM}l69Nc&B4wZkpM<<FIW&SCU&T+W9|SoPs25&7}ljyUE*
zB9@zGBEb5{y}^DI9`^7%0(2~W_FQut{$43<ZUEZ8dE;4jUQ^&~NJl}iQlphn;Nmi6
z(6yDC;vBP@l*0x0UCI{dxDd8htv($Xcp-<INE|{vXb>A%$0SDD6=yKr6J1O`7;sjP
zQWXXPWV6a0NkIaIzsogP<8Oo<w|6$TU+(O@s@7q2o`{(rOZ`9wQe!U&vP?yI1w$)h
z<e+N~^~b%al>QE9key7cY6D>O$sCQCz*V>?m6q@-*$ftooSNNh<;?_~i5_Owp&6Ob
z&Ycp=B)0L}vmpy*@3IJ?S~)D8NJt0>TL}9|^Fm(KOfpyjOO3ng`CQZJ&}i8oF{h4n
z8<?+oN;v%MuNJ3EfLB3eAxR3F?^*fuEwYkoY~kXt*6&7GR@}-=^v9Fh%9TS2A5YqS
zk4NGQAnR^~H#&$A44O%5eQ2(JvPNlTx%kE~{^Q~On-ua2;XxH*^^U!usG+K7VFcCO
z){nc9S(=wej3qK^g2nJE$U<Z&`km1pzB`7i7N{V`UDYwrt5^O0`>KhY=lHIQ$Up%(
zoIcg^=_isdkO5t}fL|R}Il_z#-6@`Xah*j&B_${Izz|T3;+w|y{zfhj6`8?tmrkA1
zL*Kfh&g_r!dv@e}frcx{5mtg3H=>XCL~3UT^~eV!U}Oh*d)k{swX)kq8eLBM(;y8B
zh*~d*sa5FOZf|g&#7KrSU*Iyy0dJUSI^JQ5>A4J0t>@L@M#3rON<o=Rm=7=)1^a@o
zePO97dqZ>7GkwF>n6*#@W9rqa&wBj}dhCcW*!Wl-Bxi&KK31RK&7Pf1p8Jl7&)QV+
zd0B21bN}5<UYIq-LkZb5vEgt2N$k9Zxc@?C-rUpQ!D9cbWB~s!vGWG7?r$hT!RWCT
zfbC6;{p}g}IgEN(2Ph-=NiN`@&&ZeIL!}FNSDqL2(q%~FRhmv6(lFF;e$pmKjURk8
zM;DYgG-*?UhUNvuqg23O{#*I%+49c8tL3MaXWIu`$A5db6JfE>E6>b7)Pnw*_NPQ?
zy<-CxeH)*(<zwX;o2|12&qfs{C+%*QRi9T*(w@jSPl<wWp2oxe;Ch|z=_qSv$)taZ
zclUhN)qn<fJz8x&O4u-97*Ednqo!DJuSXBZS7^`@v|4Pn#?4RxI0ePsbdXNgqdPJG
zaM>sw&nqG_5U<!Y)<l7(N|%4Qiik6j4BfA=!aq_0;n62rJey=FAE&IO^UK%I(+fQX
z7fG~f`b+pnmvWKV{=DH%IbBTI2-@P-BftYK?ET4;6@Ek1%XA$^U|jcEo6i~NEkUQf
z1OFwyh~+DtyVdQ8K~!g&o}D3CXEv;<mhvJdrVDYZ`HJ%6P9=DSgidd<j?k1PEvZ}A
zHn8Dsx#oODq#mt&XOO_tH>m4ATN20gC|3*fkyt+Ry#*TNsDFZ_mW}{Hjc>n2#TLKu
z{9bJk<wRe+h$$%NooX4M=M8R~|40qlrmPC59`@;xKkda%TAC|!Tbv0U2GP5F8q?Cl
z3r~l{$3Uml)~kza_`E`wyaXQzYAVMEZD~mTFeb=#ATl>c<K~=p^aFUYZ`<F})BLo3
zmizv!{Vj3+h0o91-=3Mz#$9L1o&la%)o{B#+8vP*sa!z$W0)0)nak*!tr#!QiEz!v
zaTihonDP_<gJ{fAbO18qvY#Q6zV_W*ZK+qi>)+{VMgb+R0;Af;>|XujQ9g2qF}+wV
znqz8>rVD`2dES|$64qZRGZlYiVn7bQ0(qMnzG${q=L8<p7&al5JHB*BOjTp&Qb=X}
zULv+5Y>;lbD%y*el2Z(}I)Mwa$SEY6w#k^wyX|^amQQc~CAJxqnRY;ZA4#*f@%T73
zoG06cvH`6M$+SOSgNSd8brMsJ{=#fpHO|dsBhEvC3n`A7nZ^MgI9XywlzRB5-C|J$
zW~%?KsK(g%$7L*pG{;bHRqjxYO*Kz;6eq*oWQGjdn6u;tr4E+7*=Kf4k^6%SX4=Py
zhUDw&C+9+oP4M>D-(9IpoHqlf3N~-7|KO9a&y;TuKZ!0OoW`S1lF`%#Gq;nf0q#x`
zFV73_HzwW_>nOWj($(X9SN@de^|n^h8}28uEXV11k8tDw(UteP7c1;yHpG#n;r@pP
zSR#k|@JV^60y!;?6Mp+IA!RKRAc!=wFS(K6MbZDn^s$YN5{hF){uNw|{M7x_Le?|(
z>!-==3>!)nCipa1M}k!(%+=KAl&pR7xIn`nB_8+WnWB0^A7(P#n`&7R7(nKNO5I^D
z@Cz&(*VxdJ{u09;b=z&-Q`Ld*w;P#>i^!kt==z$%F|4xoNu>X*Yj9v#!Urst4?#9-
zxJ@T{6$JYAN;88+{*Y|wYcy|VLZh6_$QKX$DnUra0<2au!Hd}w&+Tbemo8rfwKsxC
zOntZ<(3diU40PoeztF?SU^ruN{swOTtLq3IG8-l~DW)2+>chu!@npotARJ{vUO4W+
zw*s+G2m2COkwZ>3XEQv`yS4s2!z(Ku-CnU`69V4Qg*H%il{9ej3C9}yZsRVVwCQM0
zSZP15!zuU*e%y5Jb7D@rjbAHnJZG%Qrs3-a#P60QB4zF+k<1&a#1VOwu_i{f>oMXD
zA=gWM%4uD?%pJU2wP#fkV=hYov85cr?7KBh+VO6UNR=6Pl6Jvm(MvuV^f{`Ioreb~
z5<7%xfL=<}0@71RnkOW4)^xHL{D8q=Y&NBvBanm~J=1IJ{Xo#c7f3doWzNYNUlq*~
zV)BXO2$d3D`F(I@JP{Uw3Xe3W&e|u3#9^+V{M`%aE75TW(Q`Lb-`l{O{n8+(-%v1l
z;DJf*YKEj;)f$>weFW1{>1oAy?r1x>^t9~VoY*O>{<#;E9-eS7ybJ0EU9_de8|wDu
z;d&&>CGiR1-e~TZa1bI?)gQ27yw4aRo`_f|Y9jkQ=Y-h!@g-d1-3-kGFz5wmrN3ip
zZWBZ<54S#cWpjqV^F{MorBpg^BA|W=KybxDBu}#AufP~NI6DT+#`Bc>^UN3ky+$da
z#``=GnuEtJA|}77Bl1Fa&#>EDNZVap(=&DM(C(iXk|>w%gI3QALgx*_>351!(l5oS
zTQFEKC%qA-tT<j?wCsb{{Z(%s`Ap99#JOKQND<*E4&G1uxtV4Cvr!N|Zkozt%{nc2
zCJj5EMsVE_6>1c4o)*XR<6wAS?yteuPe+n>2H6|Z3-OU;&j(04{O<T2)OW`3%it>Q
zZB;l02f=UE7bSQ%!B2xyH1V#y4%TklK}pyw&#!J(5Txto{V3*5hA6=forR6o7BtEa
zXPUCUD>oY3fU`QaI26}kn^%9%D0giX?^^w--)i3G_$+hiZ)kg}R-fjUSZtJi?vI`0
za<$oPdLGGvz9C<=Y87enVDSTOq4+62Jo9liBP{O~l4<_B#2z!IH$w;-+5(H|cm(D0
z^3!~&+65IX%sSI_);&*p9lhF3;lYk=63#PN$Mi=<W9mp>9$!a@JkKHDE(?(~)3dQ2
zrqk0KvnyPxQ{_+k>_yx!nmsWSwHiJz-8J{O-Q?I~Zt>S(Dt|tdKXd9NpVp!7@0a*{
z#?CN=2E0Gl;WM@h9Y=Kd&|yu#!KE3`NGUMBksqfYr+1%Zta@Hxx(J~zbFRWo(L7f3
zUWyBAuPvypI&9sfnq%=~OBb|6@|aN&#XS*qvI~$ypfDaMA`Q$5m%IpvHd3T<-!BD-
z&|!|YwN4N56TEV=$%x3c94$vzQSHe})QtR^bm}%CuAYb!luH$#Za6g{^(a&vo6nC!
z|JqZ;^D5rWK<=S4v~ch&8*{uz*dtio)$_aA`rY2MHp-sg01rBMJ_HC+#E|2(UWa18
zB$D<ZoTQ=%e~x#e?WCCu$LBHhc6?;Bk%aYBA2h}|P{g@2xnXIZ<_@+Ny0WM>cWHAr
z=AW@G$>OymcE`lAG{0xPfA*|>CN2LAES)|r6uWo8@N!Y%O@u8Q{kovoC96_Q=g1AX
zw|9-hr==(Ot6J_FQX#0R<bYq#NM+0w3;0|mM3W+K%c7thduw-EZ3J`VeX)Z7CI{v$
z9)%ir75BrP*{4=HBBr%vAv9_t5-4e?X1WcxYxX<~mDdWD*XsF$@t(veN5lCzDSU{0
zlgV#3%7_8*4wn3Y*-RK1+=0GVBY+JbK+9_80LwKh5^2lJUTGQK;u&8Sj6&S0jjrhp
zIGwV#-8(XS)RlS1Wq6I;k(D1;q4JuGQ(v@AF>KW>4neCH4x*$+?vl>0RdNXBmNa?u
zIUT$D?0l8+IREo;^lUsqoWCuEE#CN8EgBWnKUQH&XA!=}{5SMZjah}Z&jH-c{7CN}
zaxAeAHK}3hCDrq|)Cx{b-OzY%b<X!S@4IK)=f5BG_UiE9{qgSp4sr5jKOp&t(&nfb
zZhOqe2xD}&Y=`{qS+FOj%DjE}Q?{XUOEJ@JgiQGqhk<yhRCx5~iH@A$s7xe8#N?yk
z*{CTa=gZ*kM~~P?sFq{${JgNv(I&9WFkzzX=o>kKRL*|QXFH!-q>BzmtC7#H_(&}v
za7Sxp_11!VrO(KnXopqM5z{=($w8%_`c6QDfyC5iiYUp}y~YvZzKh*dv&k);JjSbI
zY_+tG$BjyqeP$9yX?bJVk>N)hk5VxQw~^8ft8bcY4ze^#cvT}8pjv1;(MlA8teeJ;
zzw=o4r$Mn5I|MJW)#or9qiDbM#ntvcjrkEqO@voDQV$cORIX%Z*;~vq4d%8w+(E>l
ziwRWjGa$fFaxQU-pYNlp9$Ix_y>)bp`MP+>dXQ{EMd#w>;+cy!@snFFJBe4H-+g4L
z=_uqNa@7f?Qe_i;$S@s+@Iy<8_u}u1Us&LwM<K^uEMF^>8}1UfJSfB_&1XfzeF?V3
zF6v=#y%<l)aPos`VY)$4IMb!TU{@xaXE;2xWYJo5E41{*VDp>^xSZgyT@gGJW%0Zi
z*LpQDHS<+SM%^OcOzHsUTiec0&d)x~nqSNW@?zGOifNgc2kaSBRgW_%L{`h`-}mTY
z5qqRAYa&D4t#}R?h(i^PfL-@##MPCR=$QuBQH%Rx!+=YXB)$?Vhsq==bYByGOw%+v
z>6;WIC3CPYtCqdH!-4yl>yek!dgfUdk~xq54SJ=VPHFyzh`|r9ji)HG`7<F9#Vt|$
zP_VE{{6II=m#A_dat1W~{R{kS>z>5@D228SF|S|>x7s|35xL5@lYk>1Q5OBKzBqFf
zTKQqDzNthPqWjHR$&HDm)if9=x?!i5Y+i29<0ME+UeeCoLBjz<TO`B_bIfe4Bn!yL
zma<!2W-%#kV6}u_KFOpO1nKb1bL3HHYiYW@6Al|DH=E{Z^DvYJ>Tq!k{`;8p<owPm
zH(Bpg{yY<%il|UNCobfH3hKw2o5)FBF($Ucru4Gj?JsCIieGu(I%aVz-p+*||3{5b
zrbcks!|gdda*V`C%IwCY$jFEMjSoa^ZtI=2?Tg9p=cPFixx1+2j1JBSIB#TReHJ$I
z=@jK(8$SCQ-Q|34%r9WPkcznlM;rNZBki(Fi|y*e(tI*FuVQpB%H?^h=h7htcaTuE
z?=-D(aE<G>^_ISc{mMc+P_Y8lYvDrUjgm`GFpqa1VT0Lfgw7L|+x>y>GOfm!97@TX
zsSnHH0s>KN=#v^$wUDl*{~2Y!{rs`j{_Qt$Ge4g6hc=5rwbiaV1rZ-O5iElgCYxq4
zhNtsYKH@|L*8I_vK(}JuWLvEkm?D!s;8Z=IA3Z8iv<9@BB|jz`WzK6$cOPNS);T~H
zY{kS<fgP_jQ<u;^)0C*Mb5%<x1Xj3T+Cw%J>oEo#Sn#*(!HZc&nY`Poa804BYAf93
z+aR%Z4NZs70PgT>iH5pUP*K}2fsWdYp;~Guiq=uW?{?de6AqU5Vy5riAR*adqHng+
z&wZBL8MNfP@zkflYGpSn5cKmMpKGb<ZNP0?1$*Q_26L`%Ve{l{zes>!i|vETc0>KT
z7W)TP`(gfC3oLfOqNYuS9$HI#+x+{1!efUV^Hd8kL_xIV_@dkv+F%h;_4`1(;O8JI
zaLrzTiJw1a_uK7xH{QNYkC=y^6OouXWgpb%L=@(F9^@B-2M-D#`lA9)DG<lud0e^k
zz=1x+7keJ&Oa44c0P)I&g~@FO{fHUD<M~0F-Igf)T@{Q2@|$vF#akL3EV48(XB_Yu
z+Xu%IGS=f4(&<9>U{194NqqUocmt@P2;X<gD1Iav=kJDZONs68EDQ*!^4qkRn4s<)
zXVe8WMA0e7oh5$gUeKp)p7hS<HAm4+%`t>cHWiM{pLKPN^6@bf_aM%<qTGWCL!%*L
z<CCm5hde&1bFB9eAk5E#c0{6=w^<k?zBE9*mH0>)CU5;su~4?UJ?5PKcr3OJe>K7^
z`))}aw$=3Co!WwRafljj_=$JM1BQAqYP0F9Xk@5o&MGvN&+Mn}q_||hvjF$z0Pwkn
zf_E_WJU7Ux8KqX;o}^>W;UI%C)A4LW0&hHx&_(bO+Yssl%dm(yn>neT3k~dK%VKui
zi4Mj@kOGIOI8x<a&hN%vBo^`ZI=vK!><7GiLCx#=YU6Be+&j(Je0Q|=Zu^xJi@F<i
zb`fv|+6zrV$#cJ(8-lrvGb?;|zC+dD!X-41X@=4B{76+R?#azBxqu7DpEt+e>8&UE
zTVYq5h0*8Edu)XaHw-hshu-NeyC|J$b8g4T3ykaJzGN3?P9*YUVw1XzQ@Z10WlF&f
z9{upTK~$Xy1XPUv6UBqe_}XJP{?KXByl!<`RPQT5>a2gOjiqX?i=^1#d1?G}x|0nX
zk;D(^D4$$RN5%rj2%%xcAdC-c)m~ph(L>GjR1nw4YV>OfiV}fb?xV#(0a;MMRz6Oe
z4|x)mPZv!b)X)aTX76-r2uXnE_8ueYJR7%xuB2TI;$F+*P`vT6`lPYC`q7K_m8%78
z8-2L!_on9?E3F6LJ!+gMaHOI?)*gL&>g}KqV-gi4-^-rF3i*(o)3I)xq*o1u4B;89
zJ#5I=@gRmk_-fiHnhaJ4@kufeb>GK9*?&G(c`~;;7i-LC3@?N?u&>FygDaez?dTQq
zuX2j;k?l*n@M)^~9q?P3^omcVzKU~A3Ku<#HxE~Mgo`{<aip^MG}k7C30{=Ej&~WJ
zd?7Lrjeax^U{+d;WW!7Xt+ZN71=-v14$%exMGWUobPz3&&NA^HEfmhWj=&u}k#<5;
zdGz}8(BYG-M)#xIqm|YQ5z^V;CsQWtXgc?zRr=NriR*gE6!PKyx)ig9$83n<@;&!$
zO)@Nha(G2zem{O3wa@Ye>+9ybZWo6d^i8i2*TIRfOp-C;y)HaKOv0_4NrOuXwAA(G
zAX?4B*T;{DPXvkZBG_G!smJ{UIeUUV%nLyz`cZDO4woX0VfQ@ge$IKfpN6L?+3!9l
zkYIjpc4N8MaxWR@Ui2kxS}pGN=+$*$ecUB*Q)%#T8>dJT*=yh*4)C)<@5;3?uy8Nq
zsZX<eQ;A2@ezRmgHAzJo57^pX<iYH~RQGz|yk#7hBZm7bGI}7_EY0r#e?Wl0goTo&
z(?&Yu@p*#Ca~lurM17bI@caH<Lhe3}9_iWR_eq%vc=cnoQSutFx*#h><=!<ZDI&-(
zex09jD#|6D@&T))cLwm1@26SWRIrPM?>@@`QGlRc9F3=t!$(Dw_%%e7RwOYS3``aV
z8}qXxAzLMc#VdldS6b^hJu-FP#$y}zv$3F`LLL||8TBN-{Ng2I6aje(PH>Q3Mt4DX
z@X8dg0c#=7cP=|M?dB*G`|6G8*!*#Xr02Jqp_hRzP`Io*g^L>^AA65+#RTublB?uP
zSlQA`IQgJL(@QTP-h;Tb=n7WpJw#h(P3A?~?-40BIBoJ)eD7f!lK@U2nVNoKi?HxB
z^%RExGh&<$Uol)zH2t`;vWmJnbF*(uXK3-O*RBv!uu#Dl@dpOp#hMULkDlfbLr&W$
zAYL0=U7E&7rl+BobP2pN$?yp9*SHjS;iY;uNs`EgE#7x2s*q<4ZS?Z=dPwV;Oc))4
z8#;|uId_-vUD%P|4;7xJH}{iMT$>C>vKc2|BwvyIMNjt^l_%kYpJ(s|=g-nK>~v&E
z#BPMbazS)=zu3omm#C_KjI7d;E`lcgIT=sg4k%9dmjUCZ1UpSEpnDVr(654MFbdpP
zR@)cAjoms0P<Fb?eH7*9mttdceh!MG+&9zI&gOHHcoFFfwFBqoJJG*WZC<dfH^I?K
z);v0@_@qz!Yz#iR=O_&B%<<zunT<rA=7?Eu43NjlQNeAP0G@DLv+0XVb3?4;hA8aU
z^FVq!jIZ=y^Qp1&%b(M(up_(%Fc98lY2(os*f7AoVA@{T>)gJ1E{`bj2*2DaokM<Z
z87Q#xHR{xX7y=Pz8HxQ>I+BAm&dvrvtcZGp*`a_E|30ccAR3#_vp{1<>z-4+&{w9T
z=g}%drq}aoXk2~CY8WRQUkf_ABtmUff@|C)%?Zi5qaR+|h|L7C)6Cn2j5{5t>BKiX
z=9#QIOzVI&g<5ZzAlqX`&&?UfCS){#^nSXRm&~oJ0S#wiOj8N;0`ZaUjIH4$VX|p~
z`chL;EP9)Ro~4>O61q+I?nNuD?}Bae>OF_01^v_w2DR<wyW9P+lr<Lai}N=1DSpd$
zWC`1=@^;qlp4Oq*zYJmE%EqnwSd+jrAM0V%D(Mw&HjQ4KohQ9&$?m^Ls(~mQU{Z*2
z^Y<B7b&=lGLf!$iNev(i(T;LdbcSqj_(zH9yBzS3<t=x6bYRR(h|%*L8rf3G!1yJE
zvRA%ZUhX6crA(Q37^THL$3RtAG5QBt<rgDs65~h_omUF%+6}VN!$c&XxKNS}ftj;e
zCeko55<?#+lk_r!Eil??tD}pv+Y3B-0f9v_)Qqktt7xPqg)9z|TpY4mQ_P}BPFamI
z7TQKs1aV|lDf}0ZqHaXbsNHk7QS=wjN5Tx-{+_zgsm(7$57d^I8{CJH!Vsod&V1C<
z=a3i`^`Ur%rXu01b~>GEq!K><!&D-+H@SS7UTKx9;b4?)v|6p_wwJT0-`l9Nz;vY=
zG2x8JksEBNW2;Ge3Te?kbFJGniRX4!TkMDoyXI*+Z>oeh?z*}1Uv{#^>0y65E;?>*
z)>zz_vwe8;wzPk3rLj`4{%I>Mthyo!)GJw8L)Y{)>VXW=eMet%#t`q+zR&9ZUVQku
z`su#g4EeLxy7ztE9-Dk?->`#At8E3E-4WdydSQ6G8j;x{TK#iLzOKQbp7C-N0=*rE
ze&OQm-|XLxk+ogH)rI%+qdkC+{(gSU9f7zPJ`HxlKa7vPYCAUzJJ`~<xrLqf;rWBp
z(B;Ts<o3CBuU>ohkxT0BrB(X%?D^7Kd*K{--`jb}GCP|?a{*FoHW?FGosOS!p7Y7r
zCxb^X=tS;BKXOyZq}X9O&<II$hmoq!4GyX1kdpask&@LQ6Am#W=kbyaa~>Q!&#VyA
zxEQ!P=?-n+>Sy14*xKGa-uzH$w=v|qc!b~1r_=FzyM1|i+3H>sk^7HUIyq~f4AQgq
z+RDmb+pBBsl_xl=aXf0qO$P+n)h2!XN16_MllZJT;9M=&ZS;!dRs(uGdI(g(&j0Il
zMz!f=2I3sIxULKULi(hzqN1)woPJa!Zo~8fk##GT_Pu)*A{BMYk&c3p==fY_PD|fW
z^qT24cq5GZf2VPonPSFHqGhYOd@pj~)KD7>)Z$2Z_td9A#$^9TI7Q>ch`-J9?t1Z9
zc%?r3?8&^F(c2XM5W2qL8}<$jYa)M~it9--sB5i*lV1>1hM@KAOxmL0-$zI%-p{gG
zFwFlpor!IPj#mc*a1;~o-4}1cDe^Ia#MK_OgG1VtICCA+{x+!HG<q9PqP3N1<;nWP
zwe?3U(d)Oz^!MsYBl_P7jrLPNPS0n3YVsFCwh0SwXJ@jvN+r-)VzKo4^agAnuUVbJ
z@99{pjp#>e`-(`5=s~pl-TH&Su0MFNV66A!9t*m#W!OcIf5*3bHfA&^;ZiUz@2J~_
z_SX3%MX*#InmXSc^@weV_LB5;lJt7I7~tWF?21;FobDKiLl$(C5g07Eb4+rO@bYNe
zR4^O5m5#3`On>8EHL<M1z|pg~H86j?fc<ei8MeCV@OdlRgvF~c5tD>=^&&yML_ND6
zf%c?-GPALxIX(TzSSaGl38Cf*`-}`h|Kfa7F<J*h>tEFdSQ_+8{4Mdn*k(z~&iifR
zi8`oHN4>sSqH;K}WNLvjqW#UyF7xbDc`JuA-e*pNhK}Xk=-S;HrBm9-hKUoAN$|sa
zN*ViuYmgi_Ga9!$h>4S%wA_r=iZh~-z1=Z0qBZws-Q;$Yax-FQkXud4cq#SL?rdnI
z)=`T#{Ce2-c}PG$>Boc2^>2?t<NP3?v|Ef>=<%D~qv+`1)$xy;hdUAd{_gPLhu!U+
z?dav-ieUGD{?FzSRa#z-Hutyb|9^{ietLJfb959P97elu-|g-0Y@=7|>~M4ccz5Th
z5$*18?Y-aL-KRJ2<@@7kpV*MS-M72PG{o^igC?+JT6)c*gICepox`m+^ylWw-M!u8
zzhOx9{c87kA0xh^AvdFUn}^4{TkrQa52JVQ58oXe?L=6}_U_Tv-sbMxo$VG)fCi0r
ze%RSRj*i}J?(G%Vb#TZx?_mG<aF-@|aCj8G+@Wc1zTDf9foQ4QyN5el$5@d2ZHu;w
zX1GTv@ZHYVE`HhhX@{1xdHA=6>-K2pKi*S4Dv!1|-)_F91=eoZ7dnTn_lG-gF>Bh4
zqxUb5j(3mWAMZr34-U4EwC|&x!yk6Hc8;D#dk06nYwwSC8WG-a43A!Dzo?jgfBF7s
zmv?S=|9I!{@cp~v-Glvl^yc74+MB~@Ym=I8^T8hMqi?L4PUqn8Z|E1B!RJ62_{|P|
zKg7oJmTh7$j%e$)j{RyhB<<v}w^H=Fzw>%;_w~;H)($Ehprapmk9O*GY`ZkS*XR`^
z|F}uRzUQ?AL}+gEs|T(PKGbOU70-72hh5BH>k}j#?JD5#_HDh<t+$YWo-kc))IfO9
zGe0Q34R=gv=yZB18J2qaTgds%)66*Kqvv==UnbJ8W^aVwE9VTPYdR(*hW9UH+(9Us
zv#V2k!mD(Jqj5?vLLcc4lRjVW<VrJyqjS3Q{o#1fPkIdunFnR8u3f1hmB%DyQ#mu=
zmifgc0YNRIa;F^I4hc?{8}j!t=l{?@4hWU?N2cVI+GRvhj^D^Loz*Z7m|Z2n4wEya
z1)7+5JmVQz$xKPxUn_s1ubWdrPKkVN5bgSxzf{u^KhCo;JGK)oRZA|WP-?rf9#J`5
zDg!dbv`ycNH4%X_U-K<}E7sKUyn64)u2#Ek>d})pw_-%}?RG+sG82*f0x(yRj@3$N
zWaU*c<mTc{3=;W_;bAs4Vl0r6N{m+i@&$YS7b22J@kReEo~Dx)jXK#p!x>N0z4S7f
zY!NL|tG75y{K08$`7$~A+@CJj#Q>I8pb2JZKX;H?1$j`CQ8KA5^QxE<B3STN^&>tt
zB-cv)DV8SWQCo<m%3r_;E$Vpt;4OD_)Sb{1cT?w_jwGXX0z}z>{Nijo?Y0Oi6?|V_
z@AhgUD0*7?%UMfA*~HW`C(WPEjTqB>`(lBV*iho`nBrlE0l9;sRH$fvFI9+(H4r{X
zCq(KK^k=OQm@yqx;r4v&fLhEM5`2Jh3!1iRV#cWP;f-6+N>TMzmoD;X`qNW<yAHnn
z4HcDj8~al~VqbGOEsM{GW7$CkO!`-V8d{bDOsjDHwvOM{w6LK~?6hG^Q<ZPEYII+V
zTQU7wZFnC}@*leS554?{B>&+QAL>=C)6CUJ1GTDM7~Q3;?(f0&Y&a(TY8K1}n4oQh
zMM_R*12Mfzh{dSqb^y7t-^vQh7XKkCtUD^KGs>^)<0y&>+Y=SGBg!uy?TE3U|E)%!
zs@{=jg(HtGe!S^`JxprP*|i3`Wwm{uQ`oxibbHIrVgIN$nhofB_WfQy^gjhBA{bpc
zDd1VTJGsf_@4US-gw6o^VWuaZQQ@mxA$*su`T^<@Iyp@PrANu=LNAnNv!W*<nQZGL
zXe#!S!8BHcin@&Yw($|8bE)>?N}_i3;4zdYbFrY`t_PN7Ky8|Tl_qb~e-LXO>jeG*
zB@BUrOa$fN&uKN?DN>QGwEo&Aw)bXCXk`<1voxTfB7$Q4LnY&uz~QmEWVJK&3FLr>
z;{tI3*etg47c5h?hH!&Lz>LIPJ%r#;JS1m7PBUg{WDcJTXi%?Sz2&5fgbo`tpX+*y
zyRQAiFKlvY6UA2h6d5#j)`+}W4+%Q@<H2?9v&>rw7VMe)=G6$6_y@q)k`Mo-P<(5+
zpcyK`M-(4XARGe;1Wg2)*$_JFzbpWr0?g>fa^HV(hhp~IZgP0qQPZa^_4IQeuW>Mc
z`C{_gdDF(RxOw9Me#gY;0=rp7KnLFjgm1Sf2ZE(DaK+yEwxXG#>k$_j+f)SME&>JK
z1ih9_{%-@lMl$U6IgQV34nn2}OF-IkIe>Y$iHsb|4x(hAzv?#p#cBo$Kx(JLaO};+
zW@@xc`<F~~9`>VWeoD}ylFgeSNx7c?GB_$AHUP;yobeyC{*?Kz{E;l14f*uxxyvD`
zIUhML1w?8>d1_#MV<pMFow1T+>JzE*GB9;4B0wHJS5CZOJUr>2&1mm9Z4akAWT_>D
z?<c8>>}>C}(?JI&M2j`o6k}xS*W~Ou<xr`J;pgmBnoV>vgVi!okpx16H12tJ6Q~hQ
zFH=AFoX#NQfK6ETG_7bx8BD}Z3F&_{P9av#S$|B1L=#9xWh8f;p@!xX;Jt`7sX9%-
zq%yMgV7$!&SlN0EQ-OW<IP+#mor!!p>A87vD_%1nh-4{>Z7U>MlspzHi!nkUHgv@=
z%Oj*<94vdgZ)K||*XC&T^fdcUJf<q+2?DKi<R!@#DM8QX+Xp*ihH?+-MYhMGhqYaN
zCrgr1Ck3l(?32qXz<i^F+&F&KoDrs3VvP-wy!7N*-X^)$&H4i%sQS7K$zc;snZ-@h
z)cMD`z1^@Q<+AA6yAagyqg1;p-~4CPe)eId`PaC4y4ig7>Hc$0j)w`_I!ItLTAp$$
ztv)LNO}a;+F*yi=(whSC>4<Zk4QObSj;hOI!4o#{pJ+fh?Iy|e8d+<ce6qiiQ-ZCw
z{r)BJRlY@ijyGc#=vfLbKJHu}p~+@4zn9XscKDnT($_-Lp8^y`aGb)UgVBhph`iGZ
zj<X>~z9Wqg0Z$H?I635iv-E~s{S@^zo?JZT&XMgE{yY&U+!atahU+fN+`KD|jWvZ|
z>Q8Tmo<(Px&nfA44fkZ_UhTur&p+L(n@i%~5>6ki=sTX8ymS4j1IybjLG0#D@y8cS
ztDa$>gA_mHWUTRfOxhW2Y#(WqfZ-L-PR>NQO|9Mv>6~yTyUeM*_?a>l=`%+JaxpDY
z>LONdxwh^NG>x;MwJX=Cg&NZmYuxR<N+%Ybe@?FR7`oqc5go+tpx^ry6>Ypg7qz9|
zbKwTKogb^xG_FVY&Alz{mSdFkjF+hiB8V@%<<<>^C26o=39g1%G`JGIuDsyTgXB+p
z!#)S5d@#(zbTZGJf7G^~C@7X_mDX!#UMw)~rI-bfgVJ$X5CQDybAPOMxNbKjogzWc
zwQmD_o?aq`9^IEyAQ@yl2AC4@a`@)2_Wqg>TX8*l9zEfJ#Bp>sjjG?BhCf~|br+7u
zw+44+Od8HVG}!cGW2is-WlX>-xk){u%tUe}C^2@i9Q{uAGYGF&n23+fa~W>E^tRvl
z`E7Y#{@Y}me^`nUDikAr3P$0{gZ$+DkpzYb0XB}nL89hP8po4qzdJ)fSI#)C&1-C)
z3gpPtm}VBY!9VjUts_PQth(anU0A_LHf%o^w$5B40mOsPZ25&8gbE=QM>%*dS8|6`
zFQ}?U+p9mwM8gMIK>j6?)W$k4vKs%wX@U8d=0@}&bh85S9KMT7zckmv5Oe)OV#1sS
z)pC*O=+c(;79{7n{hNlaLVqYb*W0xXUk<$wv=`A@S*KO`MGemDWd`NjDYF2>1Q%W0
zJG<$GZon8O8Zfy^876ST>+6TJk;MV=)OG^*P;NLn6+5l&b$1}3f=~){OUHxwn)#$`
zn#4V3k+SO%0f_aY=45%s63udYtUNrOFzb;_6g*n~;$^8YtT)qa-Q$UdNuk*Xn_B}<
z5=vz6+wUFbql1U%#?#^^QeSrv^3T0*HSMZi*w7q)vY6AKN{D2tgau_N3zOB7)Ji^j
z8-@M3ZAkteEr0+0GUHYY;PP>OB^UL*Fbg+oX$Ho8{drUNvNM)U-tyZE7w~7!lsmrd
z>=4SbBsph(C;7n%KL5cf+k{}I)2Ch`O4zynWAFZw3p!V})OqVn=7OL1#%Y#K4=afE
ztzlo`x?e)LR?t9dvKv6PK@XXbEOqGpHl9QO=M~A&C{wd)rOC8F>9W_*o`WnKi^H^0
zbINn4lrcD!Si_rf6gI7ENHIxFm5++-f8rCyw{JX$KckAYA?!GsEZ%2ysL@+Bcxy3N
zi?*)G=o^A(s-u`~HEQx%u}}3rZ7SPQH1UfwX!h4n{Y|f%JQr~~9OBW|Wc66Jy3Wj7
zwFl&yXu1OnXog!vCs_9Ar$U9ZNqRXwx8R+-F#QoB{*IUrWH~a7Fl5>_W$t<u_O_?{
zYMggMc+K0b6Rp`fxt@mC^rk*bZb8`*n*(ms0Y7~5=)1?Q6%WH(h|eL8!j^%w0k4*h
z7!!ZvP2Uw|^I`3QKau3G^{4E-2)Q>gT=6K)5|~elO^y6I2Lk8phXUB<V46%ub0Caf
zLXk1jAk|jY2QzB75Yimf2a};*=G_$M`t^HfC=<*j%=lfi{OU<6%dRd?GX$p5-9jqh
zU;l5te^o80{^46!eObNfmQ~++b^gg0sj7Ek?m<>HqfgZxB7P!`f=NI%>W?{GO*#?-
zY%ULQG6MX-yo>F=*=2=FcQwd8#9^>YKa&`0!uEWK8z+r!qxT6O6w#S>WZO3&jaM1X
zN#ZyE5PuTkISpy1XaHLq=btz$4-m2UoRAvYI21hEkegbXkD#$Y5}{BdVuoCDaF^E3
z(H%^8uY79gpn>nElhc&b1YSU)f5ibbq7NtdPZ$5`$$4zqDg0l4D*4D(LYXZ*ny)#N
zU9r@ZTi3j%W@kw@joxiVsmd97`=R&a$beS%*wFqmR;tBBZFY%6Wc$a<e&&iL8;FP)
zce5iN=UsP`J3Bs4j=&qv-Fp2lv+In(&)MW6UvUDrhx=#j;veGLLgpAW0DY~318;u5
zc*vWx-WDU9D8tsSI#eO1ehI$zW)l+f?tpDy*6*0(cy*^oEI-=QbhS50JdK>q`+O%_
z#@97)6wCT(W>FPfRh3Em+2}JnZ<&<1*;B|0da0P0LmQzgXIxS$W;0uMQ8dAgHLwEr
zxvt$+&ATu0LPB|+%<_wW;Y@0Rl4@SE0?QrZwW@ya$t_I>vM}%D{TWK7R#aW*ENkUs
zt$fN$u1ybm|MX;_nwO}NaIb+ZzvUmCNcV3zuX(pe)CvZ1vGZIF1+^u<+Z~LMSeyZS
z1zfAUw!i#EQkd|slGl+JG_RMyAJObrDGd5?o3w8P_m(~!ZsR31f*x~7ZeBnq?X)zS
zlQ?O8#mtuapDagVc2@L5(pB?$Fhxr=6&Hekb1R7Q8>YSbNZf-X?#W*8SA7uMuO@6a
z@H^lnM=`3q8jD6)9OVV{z2YF<q1ePsO1pk)Ci`FcaH7IEX50GO?cP$|Gjkk%<gGwK
z5K4eG%3@H%Q}&i6A_xOB=ET8;Q{bGBd<tmy4P>Q-N`bI6$bwMbwK>JG?~PpFTI<Hn
zMJ9#KE+i|$xRR+(D%8Wci2@*SL!|dg4ogH$oL)s*6mO9wb^NBKR8KkCBcV88TSaUh
zT$}jbo&Emnb0^wUt-Lk-v6ekZc_>Zq;@8NJ&d2ExC&^iIh40Jz2X8lzx85xO*<v3b
z?pZcpTkbM{xjZj`=5qIRk|gtshc`68p>;y-`)8y1HDHZ>)w#JI7ZU*7FrD$m4Ha%G
zr~O-8K{n)Sxk0e-OCo>G?d&V+UVTN~-dEJkzNYfD+qnr|P-XF<Q;nOThN|4$$Bi&S
zm2L|@sCv`UQ=MB>xe-99&M3KDSl&wp$#n7RsS8fYcZ+^*u1B49h>tlZWO1uOZ?4=+
zZ#?7mb+1wc)Cu1V#!Ag*bOvQ6;bmssIkW{zdeJDmMSq=i(&>)JHz9-V^oT`Bz0H9k
z#^+bg;Ix`Zm+R}Xv{-OP9WxjCFJh@ceSW*9Co>}B`lA~G!PQ&{Qm)!Xax-*=wHJ1r
zjv;+P@aNms`tWIPJqc0IRu4FIqIn~_xas7&8Qb$Q89>=8<=`m#X)zk^{39JDH$tLc
z_a^k_*S!fF_;qjjZs>$>#8-apg_x2z6|O+><THzGPLi`e$4hCeB^>s-JIki&5W*jm
zL9}$#PDc`jo~7y8;O2cr^>q9-wb8jQ?L19C-`KTZ*B7}EHTCOK&KT5uuXdPCccqae
zIVG~F28gOlz?(SgKwWVv-kFHhb3Hxh&@_=DX-)iMp&+B<t+za#h%Jy^3o(dKl94tN
zw8jQ$Y+2JCDpC<r5w@1B5WP&qT<CN#Lu^28owCtQFY(>0v+IoTi<nd7NXe?1)o(w$
zft*bJmSX!d`td>uNCe+pUq4K`>7++I^Jf*KZpYJj?)gR9>S%T{>`&j$rpeXUH#-{l
zN4Fo42U%Z#KcXF&4_LW+M}q~AoUMDvX;E+2GbX&g{-5^u1@+DPZZT<B6*Ind^*01e
z$%Lj8ekfk0i@ag=t>9-ovps20+>DS~Jg|Wk;8GyrcemcXHxhidPfuKwTs492jg!$C
z?4Myt6b7ZRVT+$eeS|86iL`0U_Rw@q#Lq$X1%qg9)N78r^uBayF8ypMF}hj>AXgao
zKx}=T*iR-{%+H;OrbN$Mzst$lN_RZy01NHt9-yb&bg(s^X&;k(2lH{{tHaH=JDr{V
z?M`P=wJ{NM1i~FUom+GY9+`INMto~w#D$mJPX!^O{#Z}m2#NtVnPJs$4_0u684OBQ
z7s%a0jJ)WQI48UKF^Sef+cP*F9UmU-zux;>l+Di2RO8+DSpx;PipjJOe`WyEFp0C-
zM4ZyKkA5p@HB>)xDx!liqih3p9&}G~cc7(xyn#6=+tUeohIT_!SOZ~1?;Yo9c(;XF
z;lRXyi&rFlN&-ikU4yR%itRF0fy1_*v@rONa3#3x^+y-!0AaQynS-XB%PU;wC~ay+
z5K_-ea{_Un`K;D>%gZ-N*X+o7DVwiODXkyM+>B%@NMXk>HXzaKjyID%()b`*hMl#%
ztfIFSSelhC5?X4V#ez%xpn&KBso02C8;+d0QD`PoO;`Qv<YsQ~JV}=op4l7z-%q-G
zJ8^E#{&<c^*C_260kUN;q;bAAt51)$b14)+oD&-z#dMt{q)s~Vqd&t6mf<jhgN$Yt
zXt+3hL055>&4y#~Nn|zbB)RCPv&^8ROSo|Y<2qkS7I^}3Ow(d_!gHn8N*6!*ot7V8
z66_f)nwvLYCVYFL@7I7XgQSB;UnAn{a@2hB=^W88EP^;Ho&kQ^BYI9oFwIA-BZ;p=
z2p+MERFI7Nc`Jh33Xe-~(SLcqmfv+>|HSkxCAXX9=eQ2;(-Qvvb$QwHa}2>psa%AF
zZpAgb95Dr;#76;B(2M8I+;==VPNSHUvh>o~2`!h`&k+u13>K55JDX&zie$mQx*0Y3
z>e^mJu+fL>nfRm)@P=PAXUO;4Lr>qxik6ZuxwlU~`b=%^f+acZ{m;E<b8gOds@`qc
zIO(c4vv{nq_z<C2y5ia<c=3DjJL4*EQ5Gs+ikBpB2I#xz)IDak;6G*DPrmS<_XCd;
z!Q|INIEM5!$UlwMX8nr!et*>~DY42}EpETJO)I)VobXS)aS2^xV>2bfm^`+-Cfg&1
zh}`CnVV6@L_P>Wc6Y%eNU@pv1p+W-+dfs1ci#poI=-xZe8r<UexOnYtj#M20T8;*2
z=q?2>b;+gpT%eZJ?0(Vlhqt5*2tbkF1PHrlBigz@1_^!c<Np>=KnHTdy~t*j`4_{7
z?8vPEM5=~tiJSiFO#op~yf#^A!j}iL8_X&-MZ7Ve92w$8++4jJ@wYZ|T=^P6FwO3l
z!NJ^V)3T#YyrVC9n)GLWlAHPIy)4)^9a^Zp6LDrmV&wSAr%l+Lzl6;YiZ1}cn?CHi
zGda}bxUg?uaio{?M|xQ@(#zY9WcKY)x9lz4{4(>X!hD{$jz1qMBKTdKck?za9m=G0
zJ7sdm+n4xdy-a9az9C_+!LH;1UUr}yYP!F3(Z3gByh*SAL69ivDu3ojvlF&Xi(ZPT
zVpE7F{rZQl%-3IzsOe?FeVMnRy+9(;oVggXk#t7M8Jk*r2Y=x({+Ao?fkB)LJ0*f0
z?(0VK%Mnd7-6m%!&NMiCRjZ%-tzk7tSf~3jhg{GVtRUgMf_0fEQpss^j!)(o_^^#V
zUG$G+l5*-g7w?DRz?{ZSi2ZYvBc07c0(@<kle6dZ?t6(x#)=%1Me0_S>*yH&Ju=($
zRa0i6e0~kzEM7$T*}xliJy)VxDeVHhco8NTLfL90(h4p<gtjV7p}xB?iRVBn{(Ac!
zb2Def)H44`CX)MTh%W!*i_gj!R|@Q{p}n&E)>JpR;4D*plU(gwO_LF!aGPe5=7Sv?
zcRkJ%Tqh0IK&jbI2l#epYTz>?<ugCmJ#xl6>}+X53$qyrk1#=Il-@OwZBU93nWw%x
z?6H9?%<{^;L&8|InX?IOa9zRs!#!tUhbY!KC8}_q3<f({Hy$Uo7gVS7YUg<C%?_?A
z^}r1z$<5#1zC752*RN^&TeXhp<;i5iuH*96j-XKpo=H2-yrthb)%K*_J~P3XSR#)b
zBf+M#<{(W!gWzYMa8V7<M*XYsd&Q>A;So$WFIZu_XO{y6m_wDBR(WS^uqa#=lS9oZ
zTg*(`KaXa^j#9kq1S++Uz58|NgU^*$b7?hooFgQLhF(W<Othl?)JC_6FXH|H$<<nB
z>=F%;_VbFWis28G6Szw*Ev1c?0j!9HxuvK*=dNLr>1^{Ct)r2a0_@Q|4a?2CG`XJ9
z8NBC+2%I94^C5+eoT4B^h9$Y*-ogLy)^ju{&Yo~MRmdiShHzo1=FoS|&mMgQEMPAi
zCeXC(69+3wM@<o#iKlkPDx(q7L1pZqk((%SPmP~z$gY%qzus=wzps7xx&7&Wo%2}L
z+P^Jv3WvR&qx$zt4NH7bA9TKiZ&$H|swQWun#**aW>Zb_wY27VSvJR@PY;Uh?b1W;
z2wix6puqISH&Q;b8-D9vyKO7Ous{$<Rpw@!>99OC>k^K_hHcfb6H&Z|a4)9`gj@l<
zazI2*w_|ag4gcgMV2Zm8Tm*yBcMQh%(;In6vMN(P9qQV;P#Bj<)5nsakj}f!*E?0`
zJAfI#6Rj!dHmv`db8slyO@lz=_2K5*&OY?iI+*}pcNDAM7HMz2-QL^X-|?`v^iYbr
zxyZc`!=6Fm+Kb?D`#nJ8e+YqV|2%Xl&NHfOiJ#)5I)m{oI?(CWOXhmDveLRwFfPIp
zwo#%3xj>{hJZJ3tDlvM3hTnV~ee=zk-)ip_*RmcbQrS|f^vri@ddpeV+#___G{$<g
z<)V-3^UE_y27OMiiSTksoFV?D#8sST(RZ!Ywbp|OLGF;;CyKTJE9}NeR1CiZDME82
zN7ytblqexQClZO2`tJ27Vl+~SyvY?V)5+&}LZk;<9Z+M;xD|1RByC6>0li)$HZ44%
zF(bgdx?de888qZx?N{l9rx;QnIr8k5FkEdm&y$#)`~M7RR{gyNWAhsPSyYI;NbR4E
zhtzt6M2DU<TsLFMw*)B#egjcdKOnmikPP;1@XyA<d%MMFCJv=BTMP#THlwg(ZEcRm
z^p+!(89{R#p%)!P3)pL=ZR$SVBPtU_K{0LXnhN*?&=fT@Zx2K(qxJZRhw)Vt-69!8
z7}3EFru$_qs}>A68F%SYd_0*yq8+2z>=AI`@HWba3kem|gVln@Yz^sTP5xUpFxl<|
zRf$rR80)1~<M#`b$vdj+fzXfT+O{t`&CNl^w0~DZ(_O?9#u6}bfnT<SuF_|e0~|`U
z{=s#_g)}vdY(G8R`emU*H=c2n&gaV-VtB?G^}w4s6C)mktt~mA8X{+PLo{sgK-|j<
z*!rXV#rKu7<}y;Xf5JsHC_1xNP=r}I-WLP~iAQ%>y=#_Wfs)h5vw8!(4ckAjTp^gI
z#GNG~4h%r+esH%nl-cSROPhHROFcB3O|QBWf%|o8AL8aeK7IIk<I}zR29U>6J92H9
z;4nuD`45f{v_=hJwX|kMD6HWtsp&*9S^H<OyI!!R4E(Pbekt&avv33|TA-)ucNf~F
zow1Ad`~834KlpLK^Ky58^YCw;)XUZ=%qq`Y)T%$md{fq=xBbkh5;RjEU##W3*>WS2
zpltZPH79#_WDm*A!0w9Mf-jYldN@t4Xg8)1a4I)v)<a=J#)QxAo`e?UO9Je?9#x}O
zFxe{B04p&1heZR$LtHE-BOtV7D?JW7BQ}XPH<B0^|4Pc=oRIg!&zsHvO|VA*{;}C<
zee<99mTUD-^|ry5QT6WKyDMu~G^KRX19#F+&it2(y0Qarf?QB^AXX+s49CN40Zs5?
zEH#Yn!U9K6%Z93YAMHG>E>$<~W;9j11I51T^+0$!&-3$!!L;sz(X>WxIC1#2tM)uT
z&ED9DF#Dc6Ut)Gj9=>nIHUUjtw?&eH-~2?f)wy@X1?h&IbgZ_YL$Nu{Y#e06h)D<o
z(1o-x39-(v%UH6TMa$A3%u|*@MVpxG{Ngo)+4Dwa(@T0_JU+`tynnY9N!V!mGLz8V
zI*TxF2EUo{r43bkhs{^m%u6^02`1{Bk;pXExgeY}B$Tjzfg95`^_5&1fw;eKy_Vwy
z^Y7fan<aT7YRQd}*Dr)dIt=?P);y!rh<=TXiA(3p{riM#MA9x})S34{2+5-K!;Ke;
zE{Rm{FlS&aTkeC*BKEyj!`yzYu3W+Gqq@|oe&Lx&E)uk7Qv2P((J`bHoYz{wdO8#)
zmWDJEE3qOqb-iJwSO{EGZ2Dgyo3!OV4et~w9{uN0zzc{@9cM1LQnUPpS9=~B%T@jg
zZU$6l_2Y|@eO&NnxyQ>5n-AdrM<!fb$baD^4gWQ7js!l@vq(aSx~__=^a*Q?{R@oK
zy?>u-uv(rY_6EoScm)0s2wQbFa7hMSf`NpG>%p;+y9tkADLwh6Gfm}c{94tqsz%*Z
zF40zQve@?x3}%$O_a0STyC<Xkx_e;8E(ooy&9JjsDe57|-KfU*)DPzpMCOMD%_MsK
zkO-w|Ce#uq9kn87==Lm@V@bo49C>UE?2n-%Y%5zbVUCAs>C`eSlDdOVh!DWzhZ<X7
z!ro{fj>VC&;)rn0-IXVU!4-$ZA+k;x5J)p7{hqP%4X5(*ntD;|^f@#=vN$8Z3iimP
zT@D@|=}Fhq2+GCu&b$Wnn9`nIFwtf?1+Zmy%h`HHjO|6w6M5f2QvJMg&k)mhq8*$b
z1Jy9THo>k<o&xr6vw2=P{k)n98c~h*!m|TcD5;@6du6el)Ig?^cIH~1HN5a_aa!w}
zZa9RXfMx$5i7_gvH(<2eY<dZ2`h!7o7IQf5Fls}q0C|~1&LvNh%~=q7Ex6Xsm^@#-
z9N+m3^##K*V0Hh7DCj^dHa4+`bDZJihfl$U_4Ify!ij(FY}y3X77}W#dbYY*&dEb{
z6>M|-O81UM$1aJ7!N)>ciuGkfS~d`E$?JWNk&{l{(%ml!d+z(;!b(*;OM#9e`{w7?
zz4kZX_+m|Y9aH78ya8=XJLKE=vk`LVN3Sgjqt=lX<@7T~xb&4m^q><%Yw0+CQV}zm
zU=qG)%_Fc|{J*j%xX^RMS#ES7EozA3q#mjYwnaaiH$CxYGRGRRw2Wz2F6ppLCs&*h
z%Q2e}=>$9Eej-VJayl|!^_SbjIR*_K^PB`X?~fjq@>*<Z&H6Y|)0scpo=o8mIDTg9
z*}=`&79g)DCm3S}raPWBjpc-%m|Afg<r)(a=Jq(MSSO|`vJxO(2)qUQ)hTW;yy`pM
z**Q6@7n{!QgxFd76aH=K-P`y62alA^wfpe>r@-8rXVju8eQW)*is6+^o$p<pN?2sC
zhejJ!38@z{RlYl_GQZSfT?CeHtso`pV;hG44pQK{?d7xP6l?UX8Lu6Bmdc+jBReVc
zh};l|$kOLS2qfrpJqN^$oeX0Uvr`Yv1lAszq%yM&*)d)6`56wG|09x6hySzLsj6gg
zRCbfhXTrOUcB9oevn2sYwl+-FKXsq-Aopfx^Iaai=I$nkR0=>U-`(rOgCCFI+`2;z
z!L+c;t%LpJo&Dq8gZ*1~>v`wrMP-HiPY)fAXlMV2U#tJNgA3O=e*12F_pqw&YAAVg
z@OI}5c%1ex2^v;CRQ65O>;>I<RKREYvlf(j4-EC*>nFA9`w`Qaz-63-)IybegZ(bn
zqor>`I}z37496X(@~`?$+AS(<1@Pq|D1_gb+cMwC;jR3e6of2u#>}qoJPP)dOgZgT
zEyN@_7T?XhJ1iJ2Srrd+tIU=@FS4cHUKJjtTcG0KmbP~GkKP}J=-KL;0(aWiF$i@$
zV!j{0)m!roF2R#706MaWFa7Fo<QLEJmgil3|ASl{!F*mOkyM(tTD}MGmoh9FHsvE?
z&tTrD;#w9@R!_br5jj{pUsq!(<b0i@>qj5!eAFS}n{KMZ3)b7Y6K#ZGR~*A#xJhC%
zKBA`UJY4sWlXM#RItKmQ_30^Uhz2-?`L4nFd*Gz?rHw=uz%tw|DYK6h3q@nZM@t6t
z2u@D}N1MbJlNw(zZ-U}%^pcB&<qd=MtR~IsQJeo-tGoD@+TVtY0h~j^Jp6~1Pt;S%
z#A>1XrNkhn5v??+^8(?CgrEd<Y^MMqB+_u^soZO8(Y<?S6?J&C=@Jx{wdDT&Jl?d1
zqKh_Yp2r+F&c{tH-4J%0`#n2C4cWJSCBXReZl<ip(KF@Oy$oUw*;A9IBOArXV8&bk
z+A4?r_z_n>VtH)I^22Nb>=H_YTw_;1BeR!?*v++1M(OPA+$k(r^&o*yXpLrZb75as
zVz0n3zT!3I*{0HuCdGY$gW^KY(1%IaJPT^qVkH8_U_U&2rhptp$CGQl$KJ8av>G~-
zhU-cjUB>-s;U4(ot^T>=bq7g2nvIR?aB|h3)>i5zs3=B;th0edikd1gH{_pu09yU|
zM&#BKHEG3x{jJ-K`8V2K)!%a;%?O`@D^f7nU3kaxXK{Zue;R()24bo?<5-ZW;hm%@
zwuv;eDLpbns@LTBC&RQilh;a(P!MxVk{#kYIj)(R7i3O8;tp5@e?A+1hS>yUywv}o
z&lp!sFj_&b?2!aYI7J`^y}d}nx10^q>2kwZiwv%17Gm&{5CV@!%@Nz`X`an4Phm4c
zJIEc;$%D1#Nq-u#Ae(=?OeaW`UF#)Ka$rgtf4@IMViNP<Q%672Y>3{|p?D-nPN|b7
zXJfl#6C;{kAkN8{ki=8Z8udxUNkP_mU&b?sT1}p8Ce-Y_H37hbsCJnw(*)vfHyP7~
z8EL#Tc-=3HdUU1Us=VBJy}Qo<Gmg8TVX^Om8OeDt_~UxPp34)#bO&+9kphI`ISoZ>
z%7G>WWZe2y_*4Dac&?3#vMxw4ylN{YRbcnrOKg=#t81JqYVDzfsT@M`dIF#vnuM(g
zFx8)8*V|7@`lEfz+>jqWm5ezXVRJeR{>Zdd4I@nL^I4ws&zf@DjpzsdGU!jIgQVEq
zIGGGO{jBr&Az+F+Ty6N>4aM{m*u=mFovT2g+;rLC6Sg(wJ%;B7j>Cxh_Y%pHGv#<m
zanEMul-+a4%F+F(+R@DoI4K^W`TSQY`w~i3yjG!F4O%A;h>`VOcAD<!Y5#0Ckr;V;
zFnl-MRq-?&+-Ac~0)Kz;01Or%_yZ`XW@<6n&t5ddhdrsAT{%*Rg|k6=5)W*^_>Lq^
zW#B_j=I9cEDK0s#O9U%cqy2e;b5NX6dqPE5aWCoihw$k=j}gr$T7B%!Dyq?6#Y!3p
zFgN%H2l}5t1X=5zPw3UVdQwOBGU{VJdgAej`Gy;`!ay)bca?go$?6|BkhPL})lS0{
zmbzaaeB*qVe^`Dh(Xs)*3e-k^V?`vvFb{vvF#9shOM7wV$qLukjk646nf2(LgZf1b
zI!IA^@80TT$2>Vn;tLBvGIp(WD#WATs|vW3tF}3k8KF~Hc$mYryj~X`PuTddLX#MM
zaCx>1-1^Xlg!MX%w@pTzNuq9pu>62w=jkBzQh@d)vnMA{=64Uu&Ow?9QA$w8l0w|2
zVf@+CQQ8i4?p{*z3B9oLTwSKH1Mj8?g-cJ(buw*4Co`Fw?Ev9dCbC!#qRDq87LyqX
z=we)%PW!A(f3U`P+!srp$64p^&asrpJZesFge~Bcs{RM;6jrNz2j~i{K8<X6F7Y(w
z>Sb^i;MXtyNCqx$#ug=4iT3)T591I0v0+HC+@r&|U@`={zvcf}mL24{hjE?XT*AsQ
zqmPO3I*~6*j1WAfr>%d&)m&LL?SIA9e6Ut_HO1bJWuISnHyM+2dR&4**fuh$rs{g`
zkuWeVns*&qV4Md(fW`^5Rlp9l6;>hjClDLl*I!pZ1q`pt+cP*7`nbX;kut4IH=iw>
z*r6FR{4Vp8JV5g~QKhw?>Q1SRbHC=;ei}u&r8E|S&xG!LKxkpW0%5h_T{PeDo{8J+
z9j$-g1k~k7NGZE<@}wDa5%L+tT@yw3mal7$I;h|HP?;mdY3W-EgByFa>>kn=(}N_=
zuDNtwVFXxL=zeV~p9TN*hkbaOXT{_FgUJ6!*kOuHP^Y0-4TQS*Lom`M+<8Lq=Ha>*
zClA&-YESs}aLye(e&{-Ik=^EmPjbVV5K3zylHzEs-K;+V5~{D`pr*RvvDs6g&xRVC
zfGmO;y4MB$bUDB)LR8PjY<fdmlQt!fJ-tfCdIKHF(?-WOOs7fM3u4D>)Gqsq*=Vuu
zJ)_rYJBbHkw}U696?SvjKRch&jk;vr8DI2XKXH}#4h>Rh$@^lb?siFJTj7WMt1f+z
zP5V_mFho1uUsiy=<5B+}y`LNRybo?WEj9M$7!Lf3ZlDg3(0QLYn;Ij~+=RZF^U{6$
zEzd<L66uW&mMBnWfIZ?O;7MvV9r>;qt&T*!n8k0u;XF-wJ1nPVg?97yE!f9xag<WR
zl6fa%J9i|qv90xA2!C6|F2!AfITq;tW%p`(f)PR1n_Zm&ZA_EEiRBAwY|M;SZIGUc
zQ+pjo=23GMcYNpIl|j&uFXV7Df)4erz6L|9+lj@?VID`3a`&!qqkUAKyUE7AhS3km
zv8%C2#wLjhG+0x>BM?|e%_vKU2@3%}dHuwvpy*5+3A?IIKABjyA~#S8jxcoc%==eS
z-sC0ALjERIerpI_C{dxA#EjW8Y9#UwgwgbRVF+SL3^+m;rd_6qNhpDVK-0<JXV9|7
z12>OQEd72YyOQ*HtR@DtEArG~RR??e%s0(Tc|Mt)H3&^%=1grccX*Oac|NJu&r@K0
ze$Vqyv3-e4NHFvAI~+eR(d&{D6(4wzW;jSLwyt5vnSWv61w-2~;7ObK;gdnv!z3Od
z66x{<Xn~k*)d}(|`CSp~?-|V1apJb8Y&IUJnOeUi68>zIao!(|5Rg?o3B0l5oTlRj
z>-7=Z+~hc51}6;QVe(Y6#ih8XvMHm|ebPaLeW&<V;0`kq(!RO&vREZ*ZR(<IJd^P-
z!&i>zLU@1)yk)huT~^;Oz_l9wgEK>TXSRd{y~$d7v$OIS-O!{vn}X{Ck?374>O}7o
zJnjbnAA4`w*hH2jiq2Q`E3$>FEvc;43&!B8GRA<Xy^UokYK%*yq?DayNvY0Ec`4xj
z_7}@pbCG1Adivfs%yc<*mN<Kyh&ZvN8lu<>QgQVMETm3@C@H2xQBKyV4Q)y_%BoOi
z>hUy+q4;ovVfd4XcR8su1cn$2C5*ftfYh%rFf?m{vP6Xdure+2p<R^l7$Q#SIO@FM
zT)E^>8}}lzMvl>2AsMB#i5Ym-u~TT1udmYI)y>xGhI0<T*IVn5(rj(6HCj(EoL{Jp
zd?R3$8|}^=mmq=#TS@u>*ABQ*k;7#Q$GeNG00NhdUT=*)Ym*cS0I?0)S6w4_Y(QqA
zt!V%)H+HeeP{x4?S&&*?kIhk5s4`Abvbu=h`nbS1!yp&}E@7EHZIDQ)6!(J5F>2-T
zjQ()z;;F7W9?;HGNm@j%@FTG`Ft{)D2Tm6|TVU7e%fIfN9_=5!-da|TvJjlcFJb_6
z$&?3(;t?9|C?icFUw*^mL<Gm4J1Rn-A*FqYR}M<gP&-6<ArPhmMdADbI22KR#i$`)
zawE^X*7?;Yr8^n_5W9tz@Djnu7a-k}zx^Mlety11_s!M?>Axumg2UrYLt;Nva}P~S
zDq^4)6fs~qi9tUHqO(8gcfRlE=^*7-hBgA(ld`S2o{eS9Syp&PCLaal2H%gAdlJ%m
zU*;Xv9lBQ^#%$QcN$^Wv(h1!Lv5lOiCcX~qhPjx%Cp|~Gh}2iDF$&Y*+G-_b4k_IR
znGU?f!Q0}QF6MB9dmNcZo~sr0<X{jdLxT-eA!U{IP(p>8^)Ur>46OyL2KO%Ed&Hoy
zb%_G9u%z;*lTx&MWj=gnD)bTyQL1`IHfg6oT4<&v)N+;6oX70tiJ3=1I15CnjnS*}
zZpGk6OevC`i!|hVt{Y;~UtkG`3Rov!oQM?WC{?5>m$YNiznM)^)$@j8nfZ{n9P*E|
zwDOFXL%A-^qC}An0rJO<t_lfzI~+06{(pZ~FcsI;Sa7pe0qs`oDs^ErNlMIWSL!h8
zfOeU{sf)a@*6SJMkxBgHMS~3X$el2*ca#fIpcREHk1v$`j(-@AoXnTmbY-~v{h-fv
z9g5OM0vJj~6Ax$P_rgFb=VsIwidmE_pX!6|AFyH4g*p65#Dx!m03bv_7<&I@WV$9}
z2vHgYOu-k0-w#RNMy?W&Sg{a5Mpt{0uKEU6nn*><P82OASI|Tf(f*ZpC#CDj+s<?)
zb(y;1vXt(HBUbx%5-UxOlP~>*dg{0DVy(QI+{vbPt@G^-vGQ%Ik$=IFw6rU~y%DKt
zlx#)mHh+5)NXOdnh4eYNMhOKo<%(YOJmGrTk0d2cG_^sA))<i+L@!!}8JvbDL{Z9#
zfv^$LkbF%zKS0?rK`Bu+*3f{O9~%6BWuz>95%B`WxRQ?+o+l-lMKErGWfTM#4r4`d
zm*hSg4XB+<|I#H*h|=H6cnYy~b*m`K4(}DdQw)7e`8J`|H|1p`>Zz~<N8T;Tt0L&B
z^Y-jjbyJG1SS~J>qED-kaUltK`U+BH9oH%|ZS`0%+eH>GaqxQ4S&1%Ca)+MYYL-7h
z!!lF0p*n<m!<pAgpuMjGuJ;G`yd=2MFhGXmM%N<Xlf1o=o={j3Ynskop-x8^(`v4o
zMv~F-h$eL9e!=~d29FG7Hwfh`r&Pn;IommKfOb1R-E!KgN~G(@%lcv5bH^gs7=<jO
zJ(zN8p?eWt)=*W<aW9dxlT_Bkr8Q>2h@z7!?uJf-uthbb2Rha4rK1fW40Bq`AXjV{
z!y?yc2V=LWLoMAciq)3tL!pO@VyGpG_7>Tt<dm<d_EyejDw)XVvyHXO=H6=Kt5Ol`
z)M~XNcO1K^6qhlI7;io4&Bq6F1E_IsH2V6k=h$H^Kg%AMB}u4N>@n#fEn+mHy|Ghw
zp`R0_$Hy9c70s3kB%o9(=k%;Trz3_r%wN!ruWB6|KO^$Y%jyIl*@e2mh%7~n%w3(7
z?Ty{J_WhXrC}vH~-0o1AoJg;W$z%$2Q#>B_rA4{!iWXaq{D?vz@`DWEy~w~-MI@P7
zdsO-B;qY-0>Vsh9NCT)50PTNZE)rBvUE=|Yj>X8<P*~)rW}*}##~+Qnz7I?dOoOC)
zgurR$c$77rJrE#cayD9eIFgsoFcUWwb025kg(jaX-3EAqs!_tV6{qX^L)34x?=(q<
z?DBo3$}8aVLg5*aPA1Y95s+qX)bo9L7l?N~B)7Ewqe0sJl~Y<!%(qS^f{vAhP@bWD
zKG1WUc*j+vmcHm9Ex>mne1ag2qnQh&Su~L&t>z_V^d&cRnDW(+dLzXC(PIHjV-KC_
z-jb6A3hzcHamekFcpvqS-vqZvuS-I<ABgAkfzqL1p?)SE_1NL@BGo|xdGa(uq=V>E
z6}xonBK3(M_)H)rEa;CD=O#q0&M`}Re}(CqV4M{hIMbo$OJe+o>?^|yNxV(w4EC}j
zv4l#=EcPdbX52#*Nge&qkq}>a9P;mWX^M7~;+SM+Q?Ab+!!8Wn+d4T7b~qTN8lRr>
zNpBfC*7f)?FxzCbR3>=+>1cuiZdPM82uH-4VmF9x{2!-Y`Vf`t`Wd$G4N)(Ixu&K{
zPlUW#c9xaXbzO#J-BeH<kjFX&!96ze2uX56;yHIoS7M{2CHZY0YD=M`$qF61iKKmx
z0g}yn#KOSVE-k6_6?99{EM{pjk>htgIV;J*N^=W?J>=b_q#DLXT3oQ<NZk5|aFD@V
z2khEQ=C5YI0|=>={K*n0`OAM+u()x@jMK;7t?)N(DHgD?3vCvYbL=zpVUEw4wSxVK
zpVW!3{xtS(O9n+X&^Gykepe5A_%hOCx0n)Ni*Yb_zFzhC!08CF1V!95=y<TesxA$s
zUB^aNC86uzD2oD3RiKEL3Q-Y?=*tz(-j=f}%eJ=U1txn_QgBwc5S!RyJy@)s%wI!&
zELyz@>=-O%b%k|iwB}ob^hFxViDcqB<fjsoWg^v!K+{oXp5}%fJ}fPKS2tSY_a3=d
z=vL5~tVttwP9kr$%=qhNK$;RT1!IgQN!m2p)FV<H7|Z`^a=@~|pQM%y`<399>_k8H
z`?<6Ju-QSomgyu;@S2vWAR$LCXuT6-&`UloPE{FG$`~GYA;3NcPR-AznUKS6Zm_r`
zt#)?HMkua$fh9T@J@9g9M!1aPtDgm~!v+y8Ifg%o#t#AN%8`ClmYSjSv$KX;^JS$=
zVZR3L#B}A|A&8Qw&#jcV42$H%N*JMs9qM9+eya{4mYifeD1&-NvWR1*q$QFk{k^5W
z?2X~9wIJv~^cg~uXB1#aDFpS>44Y%d9kH&OKZNe!AE;^6#}D(8Q9(x%OTF}msfhd4
zw3Ha+q*tvz(?85r{-IGpSD%RmrkExGQ$Vc0Wur<u`bj$eiJG&8f}luiL~{kK69r=`
zn9hVbUbg+jOuKVUn&%~+rzwsqN=;)afh?atxhSiE2cf#WmO}HjX4|?0DNjDeY@7EX
z=g4Ok9q>)%4gD@r%ru?@zZ;T!Hbv=koL6-4rlIwi#D>3iR9@wF$YA&P>2yiDfX|v(
zo`l(vKPGA7#MLeX?DCogO;!@`J*jGqylNCRIj61F*_HCzYRRieP6uVEDM3o$5FOX|
z)g0@m*{dXZ!<&hvT80!5ANzWBRPw50FgYo9SH7zUSAT);YU&KSfaxShXnOah@zH}C
zzRW#nWLcv^=8vy@@!I(kNPj`vV!-6G0N0)R!~Tt-4kG75=}dh0`H)AyNNsOF@B86T
zMd=ZWJ?-PxMjg1JFBEKrKkW}ug)5<Q#fP!!5!4un_Qx;U+q;{fz`V$XiiQK{fezTi
zYt<22AzbuBx2^!g*b5I>&mCg1=t)=aRJh70p~DOqlB&oo2?7xg$YlUb>D34^Ct49l
z_@8kLzgk)=mhZ$QAj@((4H9q7mU(NUbGX>6l9W^uTrHJ*#%wSXu7i-&Lwom9Cl24x
zS<~+%@S})@ay?~ksEiNzX-2(}K%uVN^@m6cB7Y#^yJH!Ko!<Zy5Hfvv&vFh`ni~HN
zyV7*3sH<D5zOVl#z46fS$4w%WlX31tYyhusn4o5*S!<N*P_$gnn$2fcO*B9EIs+G7
z_#is1msHyfwQN+t<}2HFB_lV=&$XPik6UvG(UOB`{RM+)%^XC2Zl?E{?ETg(B*vwP
zk*`@Qnl-NYT)Q-pD^lN1w82Y}O8%17g+(WNRu@{?UC2nz^aWqaKnd3EVGs?ua)jAJ
z0rgsB2hfx^bdoA5NiO}BFTAr#EmvZawW{iwt<GhrbV`0arI6Zet!=>H&CN#R$=Vt!
zq}nN{kop`eRj1+br+RB+qp`kP5SR;XN6`=ZZ9rCRk@K9m{4K&KKdl(X)I|PBb(cEP
zGb_^9EI|`40R(zNqh_<LjKPCisH7%}1%xa0;XPivuE8@trukv`r10gER)G*XucT_G
z@h@u{pz3W%r^%YO$SK`4%Ug01*qOrw-Q!zNt2s#Bwe*X_6$&3jW&~tfOFMoA*^b_H
zNNUY4-tS0aKzI8-3V^y(oS+bf7mTR&OI&ca5+P$Ae$U|GDa7(ih4dbZSoAwBx29e*
zTvS{b-?Q@udhaAy<OyCa3~#g0Lnp7i!pdGjfTQ_9`&d*ia)$#kBLs{4r03Z||HQ&I
z<%*0$ssdtW!Y{B{v2LT;+Wf~0G*jjWDuw;36Z8$C7LA<UOL?rEbkSwpuXESDWvuzv
zT-nL1S}oL~*Y&Jxx*(KzlUFs5NYNcD?4^-Z$xTc#B8xP{7pwt}h=oU+LKsh5tu2`I
zCr_TXRyP`iG3+E8Wpr*@_4U?TYjabL7uau~x-PW$N}OMoQ)msX=9HweUlVn-Z;4WU
zEUfU@asbOMlWM*sx3lOn=CV|-d9TRK-c>M~l+2f2@)F4JPDP=JTU4~6@xoZNpTU8E
zra+e};wAk3aGp-RC_&mUn2wrOm2#<0>E)lxKNn3xQ#~qel+k1k{)#vo24ya~xwIj#
zH)E(;dbp`2AzT9{y=a&m6ixky$kyaE?eh7ADJ(|bFnqGCAZSID_|X?^02`(PVZm05
z&K?}$DC0IqRjl5KP665)pib79e0eJ%U0`rnyMbGwesO|+b>XW~Tt$&Q0mFL*WgsKD
zWsRIJbBqrqouiWlNhFX60}wdFeCLsCSK@g_Prc?mm)TV&<)LQ2&|k1k=8gho$17Z-
zNF^Hk{1ZyBmZS<h6Oe6R^4^jR$~_p0n53=En+dvw=hwFV%7W06%HRsqs1B-{_T#9z
zFU6j=_iA!~>&6o&m&`qFc$)6}Vvfw4JC0^@9L@jHanxHGBgrQFe?<vzjOLWLS00^1
z^3@v~#?r>!Xu5EcuLTN}B2wZYc866>lcDr4B93W=7-f!#z^1w;ZlHHvR~W61t5bTo
z5>=Z}wOQuFOtzZXC$xSG#y!u!vHXuvM_lqaoYi;KvP2y?9wH4xdaq#GBh<zpaDe=g
zdx>`)+#y20lMH@1N~30|_s0SrzI2mE28<j41?l^8>H^^vi!(|7Pcqmo!=65iop4m>
zG5Fsq{BO--Y>2Pk#@OL4xuf`Z)GFKPK@!DyXAkd1kL-nJ7eRu~F_kN39fA!0sE{x5
zFnlkW1QfTg(Ws$3sl7W9!@!Qlkh+Pa)9A<Oj)I;HErciNtJ1CdjoXG5l9>?19W4mT
zvW`TTb>I@O8)wM;X|*Ru7gdt~tWyh(R?wGM&a({_MKo<wlQ^hm70w3t*+S;p`D=z_
zStDNBT-ZPKQ-E{g<>YeOUlKEzT=-@=fr3cKq{}9j)L|xMg|@g{OQ;V=5ol2LkgJYP
zIdindH|F`u=!@Yo*HAm2{jQ!A^cMaVQdtgldbo3PviGucu>az8=X8Isy_JZFh;q*i
zNxZGCjsH?lYz!aO-L%<AE_5jeUjVL<f#WOPz2tF`E8P;gg7<I!wB<Y}2D15+bIxfk
zmDc(RnFF`!xGl$BMV(tmc%PuaiYoF8234%6po&^3y%<dmU%v>$z@@tVIw*C2mevg-
z`GF`JA14Lc+P1~d3yYnzPoZ@pvxHEbB_0Ij;}<^(;<jGN(0VEf6`>X3<`%VVP^M%$
z3i>kt4UcL1j2FD-zvAsK>*+2>YgfMjWNYsV;r;>=w|@~M`PkXP2z;!8N7tA#bC*&+
zLw$bC;Z?J_PbAgp2q@N3wTMiZcp=WA-;K;Hy0mGBeI&R*OA%l)#7t7e##JJF)Ayl*
zT(iWbg<=$7{G1!jI7sgS#>FqN{7n4LmUbNlc`@8?qlX1=ga4-yy2R54l12$~T-jO_
z)$2OVhu3kiVGUvYFbE4=Cs|j7z8~b%rV(TqZ<@IDq4Jp|H0r*YdmSw$qtFO_7YyWM
zlk@G^Cv6kUKS&fP7LHxQp?+Ic*pmY07`g0=jsKxsH~GS%PlcmHEYH6A=*ENpt;7E|
z{%vkPMm;<doW6=Rf8X_9%#aw6J(+ihepX6JEQ1O?ob)Bh>gShKsK{O}dnEbQ#o)4L
z^sC1EFQf;Qh4gl6e*S8j!89QGzR!22V%4I|hdLojK--jAz55<a2c(bPF<PeMEnVAW
z4;n9-!R6q+9K~oL^iO5nO4)$tjXOQG!Xd`lpk3IxLxK=$DA~5*Fd3yloxV48?-y5G
zGE9CJSCYcFk(`EF>fRTZvXk;UqC}@zUPqj7WZ4|UXNSb`62&_oNP=Ih{9LAcCK-C3
zS5lrlrL}Lf7&1B<nK%byIBT)P6;Z4s_5gY(kXMz+#7FJqRxv>BXsNN5u%qT9zI>wM
z55F!YfRf6Ay)~7_083>uOBmOLb^w_kzU;Y3`DVsO`9YKj&Az(Lr7K0~sWkM4o*Q}P
zT18wYu!>G<6+eGYpw^5nu<+6+m)yS7#EkrnK=*%KvLx>ImHuzONxqdwCD1dL5a_Vd
zxm*47;{5mRi%NN$DY)dcOT{zgdr?fhG4G^OoKaBEJJFCoN~)1lAYn<TSZYK{*$pWZ
z9!H&5!=h5M@VatV+4P?3*3!6J3uYDav|3GAKoqmzZS-3pH9H69C=zZBP<tXy5Ujgw
zBr?3mmxe2jFeD7Y*klfV3a6f9;Zn{dcLU}AG=dF6aD4Q7L_QP*yF0sgeLw2CVP8qO
zqyO0sn&e_qTx<Aq5)PfTdgR!t8-}+W2YM^z5Wx6XT>c?iMrs58pjcOL<V<Iysn;UI
zr3^8E30^Yg`Kh42xSD&?GAcI4dK5U;2@qVSucYhGoYzzKP~rSgYYv>(Z~n}aPNxox
z?O7t9Mr~w}r&+h!zgl|r*m+G_FfO^?QM*s_-(&&jc6)t(Lg!CqamsJ(0HfGwG>l>m
z{ZH0NMSPJaT_yH?h;ELgB?%iU)Pz#}-5$p9>?`tE8nY;JSENw9?o#8f{Id+&jiu26
zWz@lEYp*YD2MueNs%VVOV3gYtyV`U8Hj{V=_|*-`z|66XVCWGn7+<m;(>O7%WFORH
zfm|5OmhvsB23TwqojZFp7#mTPRe^@s1XKow$K$J_4eeZBVxt_jEArg1cO@Lcbvx{r
zP&#Ci?;@-oHKZ+eUmzc-Xn#V7ci{RMVT`bmGC$@Bs@l^WRRjz(3i_l$RbwEDR#D}n
zKDw`CNWzsmJ=V~7k>25!q%W6Z9qQCEqUGowzG_iWM9W(d`@tBHcHJnol_K8JXu=ng
z6a(lye@Ix<C(n*#`^8?qWKUA9ajx!OP61hcX4U*^xmE0pWZO29D)EYN3=Hyw1S=ne
zaakB6kA=wZ-g0x7WGr>`TgE>Hx`cS1<ra}^j272k)C>JCHrfqHS+5_6Lt}<u7mHr{
zg+Dq&KZ@;F5sa5E13-(?wPYk+Jfq})Pn0$9S>-NH<YkByv>~4k<@@>fI{BBXf3ANv
zV~mT@3%9$JM7;wNSB{=;@4nZak`=D<J0Sd}LaCl8>U+XhYA=}31yr9#AulG@*3J=9
zz-Z-O%G7zwCtPg~nMj3^eYS;u)d3tQtLY`71(ce`%yKCzEeBwYJ{JWmN}KDI^73f<
z_ao<^(&d8?gOIMH!aQ`P2i-r=0-k1&DxN{THdkY>IIqHBbj+pHlmd*DOocQajriYW
zQay5?8LQ9<1_Pi>`7Px&#!?e-nT%o+Kl9|UV;?J~6RCs*1@>_^L08$gsPsA_VZ9N1
z2GOJbB`$3NR~lfa1MDtAk2-Nc&aEb&ilTqTFDp*5dWGhK0D<;D`2{<E@t-f>9-crZ
zf%<H_1xv(ql`j;c(qCdlwBNsTFt)qKc5Pf0`uFZ|N2DF7uU9@rFdLv_=u7o*&}=w0
z+CK6h5GsweZmZ`wYpq6eePhrtzmSgMuU4h9CV;%sVeN!^_zLv^mj&CGF^M>cmDBex
zeJrBFbt&DK#}02A;XqDQ+eCF_*bUk<0Y{P$_Z<}zfdO?;2NZVPIl?Q0eqbmqJ-$O@
zTdA*kciz0901Pw<mfoh=<4~lfT%qG8=d@LwX6$y(TUUAJ>kj{q9h``Lp<~e+?{v^?
z(qF1NXL;l`R$@9DJ~Yuia@o+6PbQ@qo&6$jG<8%nXyS(<7(7i*3CbL|o%iK=h=xjk
z1)51F*$beu1lpli{-=<R@C`g0@!%(;onad7oE;w@5YiE#l8A?sP2{7oo$-PSlP~;d
zn2DEGnn|6IU<VKwHviLKCBYRjkL1vvKmJLgCHjYhIN0s9_g)|F9i54+^IEMYpCiSV
zQkB#wJ00Ay8kihur*pRV{>*BG*p$W{v1m^Rn^qp}Kgw%vcAt7ip4e@z_4Srn!yWpU
zW2g0d%PFmuMFs{$Llh+yhS5}HLWr0)jBxYQxYL`{aQ*6uhor|MED*~UNmq+|K+_8q
z44pb94mBs&d`&es2cx>=?hvV$iNp*~*<uN-$J(!lhCM5Nxc#x5?r!SoSJzSvrm7lU
znOXlbuli+P?Z1^M>N`oh^d2Rhi;VQn###|?oTMN{cM!|-t}SMlyEKa7CN3Ixl@>*3
zjP65MZ1K{0qx#gX4tA=qF6?oNTP=TvK6J%V$cG`X%UM9go+TiO>NXBHJU28q$aO>0
zP>f3GdPDE&Hn*=(E5(3rOk3{lj7Tr*Zl;!Otr}{|O3G~OfBGWNzlo`bj~(+$Dv2wp
z%+ZU7_i`e&cQOiOaR`@1_1N*oH+~q5QMsUySend>PR)uO{~Qvb^h?L^38P?#<+5Bz
zW31+Cl8lA#e;G^iHVCiD>y7fYNB%?oc*Pl@NJsh?k*tz}J(pkNUb0&_q&bCKf{EW3
zUI~!*VFytTvie9rDPCBwkL}$-G!ElVIn3>#8Qn8Wzn5+p1(TeTOF|^ep%y=k+X2hz
z9o#0ulrO*Y3t0>$62)^s19XlRJ%KVgeTJ315=C~}AxX;Pf<<*WC3d66PE@dqc1ZN0
zcx<vfAuKKC2tjC)h2qRJ<v8}EEB#yZtcoaomT0{Gdui#aT*d!<h<>cq?rQkn%)dUg
zf82uq;aC4hdWMD{wNey(?hl8b`zWIN+?^t)QZAJ~)bOS9#}Bs^@%gE-viha`GYrQ!
z)wf?wVPCQbTqF<@-Y$SQ+PkASmMv8S7&akuuC5`!1|LpXy_+~v6zw!tw;D|fZnNHa
z(sYCzZP>52YOA$YP1$krd|8XBFtH=2&(79Qc}p2vVQE!oGsVBOW+$Kk>cfZ6_0N~m
z=Eou~qxc>TCfxYSBrT29<n5y$hUK!c6(Fum6o{#>n0FDr;Re@ql;ot_-a{6g^R2&l
zR?YL4oYYnGwuC(tdftonOW^iv4F|AwJava6UMMj+ys3KAkp5x{$uR}emjt#76@;Y*
zQKui^2bL~H_fdTi)j31_L8M>n2{KrKu;%SC5EPq@#)jCF^xbsvyS3q9!T|q4YGZZX
zZHQg$JXv>ES0S_6XskKA$G<v9yUxuRU9jVHV`MCxayTC~$z%kzCKP((RN3D`Vr!Hd
zcXWE?Upl(<A58cVL7R3W{V74qpO1ErvEtG0=cA)t_@Zt=!i>%J*RQ=H)h;cqTW|JW
z7RC4#USi}yo$V(kqdUyniUv9K3mbQO)zFjXY=(rGotsX{^_*EmxJ{3-OLCRj3UsP<
zmR)6tWAIBcM`T|HoQhxpfyALZj?kuGz<GCQvGvHY8LjC&sRc{+?J4)IXySJnP~(9!
z@M7^KV}T;n;tDWY&`lz7`{_t?cYjNYzIjsZhXWGV<k3|nC18*+6-kBlY&}UKanZNB
z=^m5<nK5;`;>cenU1}J78hclOfIlUXxMDVv(tH$SH#O2|vd|-7;uVrl*Ooj+5`LLB
zBL#(Gpd1YJlb~&&_vct(MV!tRc@9@(uP;nrh&yPQ*k^kHK%@MIP9YEa>~Y&2D)~~0
z-7|}Qt6W;@k>DXgh5e#Q6wg^dt9BM?oLomm`H*6#UbSat0fl<Z^^x--DiM34RpPSd
z(dt~*j8a0%&Sl25a{ORcQn(D=k@Py!!$6l2)os*jBloT|>2bK$4oA`<6>avbO&XaV
z`VtgRySOfFUGAfKA-k_gyw?I^xWoX4LJp1oT4R1n>;)tCG9e^3*gb+Ex+t8MOX?d6
zcgQ1Qk|vgf_#J19f<~O4$Mi5O+PL#au8tQ(JlObNyWg-L7kf_W+PjZRKcGj$=!-=~
z8+Xg3S9UIA*#$gNx;J*QXNs-LSahj!LET@GFjlN<N@xIPI?~?3c<Q%-E%_o2DN|7Z
z9~4igQZvY@R%1`-$pJM8I39l)@u+)OxRCvta7-q5x1~qXOtM0l*h;?yk8~V}?5N}d
zou6Q+V###^Mi-h*x1l?6L)v#PY2JIW-=m!hHwbjyyF;};KOWvQiG(epk{nD7H`pxy
zsJ_wc_y?y%q!80k6{ji90>nHUl9WvnqyRu(2#X>hU$wZ)Bo4#f^`tuXyLeIcuBPMb
zh%Zb322Ll+dU|dMXM`SJ;4IX%RZJ33v-X0axa^q^-ATh-wK`&1e$-1*v?vx@BwDs<
zH;O|sJuB5G(&wgHRx7`uOAv%sqC_hJA)RkP5X-|W(kw$~`LkuT&~ub02R1XAl4&**
zw}XKx=rnZ+sa$p2(W`5c^ip(UEbkM0d7W>aoxf~qa7l$P)xTzP@@EG1Cqg+rW(VnH
zsXo%#GP;@Q8lg$E2hVaEZe=$7fHRUf9ZR90VayZ0Bvhmz)eK4SuSKW$DEn}9+V=;8
ze=$DACKCv?`fUgmA$}i>+6}@mxHV8J!^zq&>Ahhrgjj*9i8)hP69=8b4G%}1NieM>
zn3+g|1cC_^0APP|2xVxQo^|%2`#<I$eX+h+2*=;j?PMYay?D-dnU`6Uf(WI%F#7yg
zOv{Wpu`s2Y7;$2fV>K{lH@nML_ntkhSFP+`EqXBKAx(5G(+fFng8MH-nb2yEChxxs
zty%uD`R_zvGMHc4xuu8_(Pt40PBT5NdlL-lN-=6gV_HAMRRT=&O`W(qcQ^_nd_cKD
z#TNL_U`XDGq;oRr--_@<-&1lc3Zc%R$So0hTf4)lb;IF(?P2@W*cJ-UWZ-QRV<hOY
zWbRCAtT_iW@-kY~&-3j}$}E)H<l=O4@`H~jR};I+;5|brDzGJ()U4F6w>g8qWTNI0
zmH(GaRrZV}XklaA-<S6Ns-b4#>^)ZOR}sNT{$@o}R?;2mT?=uqs3wADn#yNU68}zm
zLKneSNmIc$RR|z-SWki%C7Fo>q1!Cu?kG8~<D3P~$Q7<Uoq-!+%p?MXzO#ZbHXd-<
zwWA+8Dk!d$UjdT#FR$pzLHtxEABY6S?cca)7G`2W4Us~k8D&r+27PJ=csGnFq^2jp
zm{*{xk9wJi9)~bw6@ASrYe~ac3dVvsG3*G{?Cef0-}>X_73>%WkRm^=Hypf}fS3>3
zuoY0|=dt;MQ{wjl7Vbr6v_N?q(##duH7)ne@ArqErMxJg86wXckqLzBhlUAvSxE7I
z;m7e3EfNwmPH!|pBJ7J}NZL^9U8ZMw-aoL&jjBiIlRWjIsTtlGwys@Qf?euXe?pa1
z0`1cG7w4Bxv_Zd^B4S98s!8w(1!WL0OV-O-O(RQ5c|~5!$kVS&oQFED;?%y4Jbs~2
z!cmS2&3+dY1S-<eean2T5CN_fa<M8|wvomltwM>HD^&B8Diyd$B=kiUVG7A5<3Sn^
zDXf}Fhm&lf&h;b+kvEEx$!=Lmu2qWYx^Vj)p%+v>h*l}s==9_RJ&|+%9C6Bd65IuM
ziS1sKKrn6_j_5}U<d)3+NqY!&oK$wTYHK=?76|}L^#EvZ*l|P8R)_eKIZAtS%t7*=
zd%n$sw98pzOn}%<b^Mu=ERz_hotr<-1QZ^o?5aIVdQiCbyh&u-AcSLu<yz*_;qLlH
zIr|_SBLguCAM65I67Dk4uoCG*vVJOX@T9AK`Nd)F^`2LvHj?9>>9ZwcjT6<?Hpc5q
zJ#z9<hNQDV29}nmaK(E2lg~Qhfo;V&P%G+|)URgp3Rht@=@Q4L<-yg{t~(<FiL?0~
z`U&Mr-0YsZ!YMm>hiipG`(~#`ll+#t+3g#j(H=45bdl_{r7=ZRH{E)0+{RA=`WIm<
zDMtQGzH+_Nistg}Y>}RcRDwp8mdG>s!j^ETkx-^nSw-cwB*Vl-l|t?EK@tDEOaJvK
z>>19PGWBF!p4(1m((e}49kfu_xQZ0@&gosX@4*CI4!z=>zKWULA2`M&z(tGqJ%$8B
zRMnTpwb4CLSR)4=)((d&Dq18#7>{H$^jnHWFdi^}2gkjnbX{=kN9?~A-wbKQ>>xcV
zOP_DtNNL8{AZJoUO1S9!1Uu@cGe)hh)=z(L7Ry$c6WH5gXa5TH)zM*z2d+-qirl`5
zQznm?F+qBEoT^`>VF4-JpZ3rft<vrg&fba4Lw)@T{P4$<X>8JgD@^T;;beLf7ihFV
z1SUWy#J3U5DABwyd=52BA(*5ye-+n2TK{t7W}-WBzSRq-%;!@t9)bykP@5=q5UrIt
zf&_*d7<wH29sMYa%F<p#gsmMW$XRquEFE`?*%5BYwZ$$#ldsU+aDh2|+EHVC3Z{sa
zE)t7sK5#EA!LN?&h_pmu9D9$|fDIXvLKTwssg;&a^2ei&&1F1nSLo{MWNUS0-M_My
zACU$NB*W&!jbhFX3$YYgjX|j>(6}E<d5p2iXqSt92xPB5G)yZ3eT`S2Bit-HgQ0uL
z{EJp^5hsal?5e(mnLabNn_wzl+vWje^T63l<4_1}Qk*3>D@GxXI?bB3V%z{-Q4L=t
z^LNbwGlg43iv^`5j)~zB8@OyVu<S%ZLAlcwcqjCRcr5Rw({Kv3!SNma@sP+>g_&@@
z3&Koj=jF@Oy>@%A4cO)<!CuFs2`S^iZm?~JQrV{UkS56$S6%`HR;|h_7Cxyg&4Lwd
zxS@j$t835>IBaegA;BY$V+oTNI^f8{(e9zuS#I1dJ8T4Pn7ZWX0hNkBM65u}y#QWj
z$-gKo?H?|z=1iA2DIBw^CUmmYZL&_)tX8ZFrXGwmKu~qkE20O$tD*yj%A?bBV^*C`
zEV3vzvo4*Z=jh?YzNpi{@}beepiSBS?eN9%L0g*4K3@PV-H@M)(%W>tg)OsiQ{mdx
zVGr?0=%T=<Cys_R!xbI*NpC~)v59u^>NH71RNR|H36OFR-CGz$NnMaUU-b{+&mYE1
z%G;<k*H_?Vd_0F@>N%<l&t}vYt&d3-x70D7i!-+xTDaJadjI6T#rWUyqV9zwK;a}r
z;@Gfsbmcv5U7-r6;`ILbqvX6&Dx-FXO}*_0>ujqz07fS!A*nun)D2UDp{KZg1xWdo
z_>H#EC2Ky6N5=7p;;ietkwd*gV_z~3G?X?DGW!rn%=7ZkKQG%pjeweDk)tcg6?zVe
zw6h?y>)3`NwGr9&2I3;x`dD3Qj~=He0w;DaBNju3z+Cv!m69JvT6`#}IRpSC476vm
ztju9KS3uW{ro-4LH~u~LPScapJwL+Aq6wg!O8Z1l6Df2L4B#DKXXuq`)FlUKz$l@1
zt%G#Z2(ChIo~cLVlBaPnf{jVBT-Z1g?QD@j0wO8B!sS{S>+5?&-4j6E-uQh_MkVlw
zpvMrPP*o9|^mu~+FsRiCBX_x@F7RX=+-w?(wNp{bP;|-ST72GtX0RaEFO30PNzeB}
z@F}LHilih`m5Er@bjWUzpg;hCr(v;3KoFqk<rQ_qi!CbUlGsft&h)$*)_vi{7#r-C
zB4u<*;M_;UB7T~R{E<I&L-IX0T8>tf5Nt4p+8kU_Y%OU{^IvLje%PyvP}^qIa*nb=
zW2BH&gK^<e?uLFeb1y$iL`;!Vypxa|BN6!tyKY2r*7}wxfDWuEUXt3KB1|;!&nPmd
z0BD@%m9;?ho>1zHvu~L6;q7M7ja2nqoBOAL2*{>DX*wnu0%}P@4r)iMr$-BIt~gl;
zYaVIiZ2#n({}5eB5(hc~oNeHbOGO7mPjjPXMeiiWmr1SQ^*Q9X3N-o~477)1*2MMD
zKaf8fd41o-$S|3=giXeXJ5QR$p$wT>ZZsL;HvpFX09oe|lIa8SYzlzj5D8B+aflkP
z1Ek5(KNcd6E}QO?r$`Fs)}Myfp?7Y47h0@Ip^=R+|5Cvb3#DPS!tnq~_c@@JkIDva
zWT0aUOovSawz2++9MlWESA=&n!T1$xnq2?e51*{sN=PH5ZtM-Vz^<%ON^3YVpr>@4
z!h7R%3oV1vCR$2bUOUud2#G~-$l^7`Y#)z?`-PAJKV&2`yCRdfr+huJhoE&I9&=mI
z&2TnHYKv?snT{Tolr~%H(7!02#5R_YBuBAvhXksM^PW8Fc+St|X)FG$j^2fN@a-Ar
zDG#g}kCcpwFgAsZcQ4V`k+EOo)->JuN)l7Ph|sJE0)j{!PS2r<)T2vsS~8xq(lcfG
zA{Cx6_=qSJJ0hKNn}O176VH*L$4g>h&mFP0!Wj#KfTNh&F0%h$=q*c|{W2I$p=L~v
za6YR_EI3J4gdOY_JsZsKmUU$$w+#!>dDqB#a^u*7itoH74Zl%MWeJ`dzV@G`uWW_m
zq>C_}N2Xt;^WRJ5i^_*G#!Bo3uz$-x13!aT))(F~;pLRHsxT@=lndzj`R~tvx~Tj_
zbwy=|<0YePPm+(WfQEn)5Pq*!$k_Jw$E~e$eW_Wu3@8L3>1YExQ!ss_F(h$EorV60
zX|j8ATxgIVLYn|7#)ygAP@Yzt(_Ls_{8D9Vmt_5{x$K*h_(R?-Cg*U4=-ko|p?g<O
z10(kpYBpgSjQN~9rE)&{Vj^fbEZCwmL)@YtLB)GLTy!aAZXZ}$9j1M0Aic?Hi{;*J
zzyA5NmHVM4Qq+p5$DwO25)M7+v42AszVt&x9#N>9(@Cm-Z^sj~o)Y*-cTmYo6&J1g
ztEGb`7E=;%6P4+*wEr4+>}E5WEWB@;?kpkK@KSl`M?G&ibjMyWjkJeK6;PV5W5H!8
z{R(m`C|qqh`0NsiS0k2W6MZDMjR@T;&5}5_pvS?#+~NJ7$aYYWw=2m?z;x~%yaE`j
zE%xXY;&18S3(ae%!g^5$w>E>}a)3g7Nu<EHD91$60~0^E!F`N?DU33;he{6qobXS{
zjwqXD)UTvraw&>PjCT*_w#v?b?xC;w4tniqo8BlK?MB+Duv!*URqeHcHgqs%Cs4=T
zKh6U4Wf3Q@V>=cs*!HU&4jC>?q+xLqj5=Mf=OS@JZvMLdxK62(X)UnP7!cLHnB>0F
zpDaqfC_3Xb5;{zMB)!PjqXJ&GBxJfx#S&AeYAiD=Xem=i#2-@bNQ4*3qSeWwzWra(
zIw3Ly9^pE2N-h$Ku5?aL(!&U!et?`Ta=2RcRDeucS?gVRB>SVl5^7d0p`ae!Gc9J4
zJ^pA<WAz{lV9R(eNO_CLe>RXbgpWkNDMRt!kMX&oc-a2cAFsA_6eqJ|kn8-)@~)z)
z?V0jg?J9TY^l1O+^_H{iB2TYQh|o73>NxUxE{ckvR30rmDi$oC&AMK$T~aqrx%dSn
z0mQs`(Xmhh95{W*4Ncpg>S0erEG4NZg(EMa%Iwc&4j6^rJp(um+FiB)qN7W)yK=%`
zJ@P-GX<~!_LxzN7Q=&KswI+ls@5j<#fk&arwaY;<5FuwvX-dP8XU&~nwz|39Ln*u@
zy)-FVa!r8`5bM$foOsZ{*ToPSS^<X4&L`eD3Z@}>YNzl9QmjL<Oecg&QEv%uD;ezd
zvFwtNk)FJKVIba!gR}~aJ;X~<=(dsHU?-3g5|K)xFpl0Fu!CAwS{L(WKi!W<{AQ0k
zdOWJ)_p_{Wi6?$6u98%(+&L$|LB@Qew?LL6vbP@CsNS7RXF~WG5_>3gW2KQJT=ZlP
zLwFj(Zt>^rL2$`|3n#YtOuPxI%uM*wS7^GP_8cVmzt;i1`{Hu^U_#Lfl(=rp<MK7W
zm0;7@t0J<=HUx!;tj<w!?3lXI@H#7H6%y|?hDYB>b%<3YS-4gevG0)U(^LoU!-2k<
z(fhe|6XQ`<6<7G}JTzKCK1g{GmSh6}5%c24^1aV-j&a#<eLOs2vY*<kw$KMHAgXZ3
zr0PoA{pfg*Fb0w{W9MY&>`l35U;1aJD@vO0r09qIC)0w;5tjhsk~!is$8#o49;p~W
z{w(lao-apSQv%|nRtgH&I5@jo#sFesABb_2pQ<iq;BGNO!4ZJx*omLEo@1q-YUth-
z4Ha%ZJz6*;I*J^gy%GSUctlg7n4uOCBQN-Z$svq4r69}p%-?#7exrA8si2$aws3|C
zizMBPBlLUU8}{k>pJ)=Yo@WAbe^B6~BV_rieOCqyrWR_(;?P@;83_rK(DUBj(YsGY
zb4lQYLOMH|;xJ`g9i`r-mCwJIKl~RZec0OqR&3?M@YS*;U2nIfrd(2<&jRu)3&>du
zlIpM`)z6Oe#tr+d{IIN9n3_dfaDO-$`3xuF>=kQyWqC!+K*IYU4^EUu9J^I7KNHsg
z>#G&9+Lm+$UZkzzn~qDPSimD<@XmBOj#Vwi+_YD6ETW&Xv}Zlj+qeu?Vf!f;<m*x3
zEEAMv!D|y55DjSj1589R{zlO}kRK8!l3`Lb4)dwqHci4<ih<nd>=y>6MBeqZvO+m7
z?@kqf#Lemm=l-F$9H|H=^m~P%FChvN7gvKe;|h9QuPm)7pI5?gj3Q`reRmLh!q+?!
z-VyxwN>SC&7b*Dq_$(L1lr%kAd#X2__9YvZ04O#<Q9@l3-T;+=ODun)K(^kE+$ws5
zNU_vnmLP9QRK84bmI-}{ZBkPfB9gym9Ia$ril>l>{TE5--vW0@y8J1LDCRim%Fj@>
zH*4WY@UQVu!^yCQe>?x(t^T=F{qIKgX}z}6*{)vvO+2M(lQ9)@oAjc_NDIcEarhFg
zp&VhkzWm|C^0EqVl<ZJ(*-BY1J~K`nJL|9pzG6dA5oE<Q8V^ZUwX&L861q&;L;Mt9
zjjpp)IiBfUCS61kZ!}?{ueeTsCE47KN6t)oz-YpuGQ!g1BrUx;KHRGdTTAoB8V(as
z>Mo109#bR&y!~8KoEkYO<qF|6{_2TM1y?EX2Ke5?&qU$?jO5$4ROl?hz=v*F`i1)}
z`a+yoR7Q1l!<nmkSmUy41Tu7@<$7f%TwpQ{^IrQHyu61=6OZJVL<Q?P!c&g3l+yt#
z273}NAgcRdN^^+Q=Z|kZxScOiJV*#cHZhon*yswC=GTiY1=t3elk7-i*$*FbOE_{6
z<gap(g88G(?9>;zha>-zea5o`RI#NQA)pCosj2rn3u7d8IEpM!npQKI`D_+AVK$74
zC{p1Y|A#urf~0cTF%DUKsYlJOdH@835kx5BV}@ZwN(9BF(7W{RCLAcnppZvw_j(1G
zorQMzb|XC(6&@Qc(CtCaN#om~3Y;nn(CC80({Ub|FNZ6nlqh~&&?IHQSHAPzqmi0A
ziBF8t%bDamD?+4Q6~zkD|0oVE#}H3Fm$IHLshu4iqQ+T9TM5_c=@Ll=GrSi&(wM5)
zP>xQehK+P#D)r2c4*!Bk4@Q_{;e7ex$FftZ;s0uuZ$V}Hg<h~6?f+B|&Mc_10u3+W
z8Sm)fpx?-X?C(BOt&<>%d=mGfoby(jW}qPc$_)*58?`t5fr23z3)ewMP*ow5QM%`)
zp?{4|*}2n~L2wDk5KWDOURla9Bov`jMZj)7_ZoezyAjhfM7t?y7-mL7sp-*-*;)zj
z4(2fw&SL@@&!5oS_3nD!B$lFG1wzQJq=t|X<BXm02pMGP5~@zoG#Af`N;H1y_Y?$<
zTS4MkoOxu5$x4$L1lK`v7EtBA>-#M>Nw~tN)-g#|#|p@;$4-lp$0U@XboZ4+A+a8j
zHf6Ww6wprO{B!C};Vxm?u<J2SKqYw?!JFU~$;TCnE=7t<vU-IlV|ZV&>IjaHF5VTk
z1-SKCMZ;y=3GAbuQ3u^RcohCjbeHB0hR!TJtBG+CF}7Ys=ZYb}<tZ>TN3W*~y)g8I
z&d6lw_BdXMg2F606ZkYKa*0U|oCZVw0nkv_LwL()%YwtMRuvTLF)pn!UfvW?oH>JA
zPk0(gfLSq?7#&p7i~A*P@GF|XabM1};hBKj7lyC|RROsSaC^g^XOjaptg=vU<84nu
zN<2SGQBej%7kC7fN)1GaN~cPa7n`n%4etyH6o(}ats7ekrYmaxfxWx&@#%{T8mf{8
zPBL+nW-+bstBRe@>vpo2(naUIjz$VSn2Ma4p<kJuH}PqnI&R7u!7$-h11P5&c<h|H
z!q<hR;z?URq2ig=>r_DI>315+oxe&!46An9dpu9Y6Q48W!A<ISDhLbcc}i}_no_j!
zc&;bur{Xhn`n7L(F3u{_YAd18N01YBdo>c89ij%QR~Tz!Y^RyVm-=7k!fJh9IYf)j
z&?ObaH}f#4%@0Jv0cny7MP4N7wtz_Z(ROZ)x6&;>WZ?@2`onkW-t<O(99i0PZdY0<
z3hN)A-jgZRkA*81<l*<JS4&O9vxC`peC1I32H&UOlYaMGJB_`vyIe3;9FGp^oIZBm
z$N*GAn!?6YA{L}n>w+Wo7CRbG$D~8KqAiTBIb`i)3P05K#W`oejHG;4)MK($&8XtZ
zSX%h@jzVFMg}b8S<zCWr5i-IQuLF*JqgKe0ZT68CASGjDH6>sC6$k9EIb7KlFA|~*
zt8E$N)r*x*M4G#jZbU>U36B<8=d#GG7c*Y?#_$cSmu_5ea$n1Op&zPWeKK{V5u<#a
z43qK6fK}Z6NXVL$Wo$?Zc!UI(Iq}F{Q9c_tTUj=PLnHHCQQGpw!B<vYJO`m_Y^{zn
z1+y|HU0`~3IAcxu@S5d1(u);7`i+A-q8QL98(sHK(S3N(B^?qPZ*>4W7brfTZ89I@
zLnbE5RYdDf)J+g(pcuauUxi*Dy@lFrv*-`-3GOksU6bQrvsE6z`HOBlI+<KJCEPI{
zd$nJ2NT(nO>RmSkDr@KHC4uPHNNU1VXbq7Qbo?HBE<^unQF6(p#etg;SW<jw#tRzn
z=d$V%&0Jzoz}oaYuOBH4Q5|sP7UU-*ArKQJ<B-pFFp$ucx{xGzxf0w!eqRA37DWzw
zMY=E;^RvE$Bm~~#0us;WePSONCY>*u0=BmzrE=0Z;IYxd(|#g%F_H8agKsP8RjA=!
zhP?z(ds3EN<zb2UbhW~)eNdxj>qMB~H1iPo>LB_~Zt(x(M2~+*nf&8ged+Iy#h(6t
zK}-$T0+gQRD#f<W*M${km4{JhsaJQlc}kn+k4J{PfC&m~<8-0kk3V8YQ93vRKq&E}
zd6Z%R)L{?C(H2T9MMu~I-?jb3JVOPoF+W)*ee+18T4ni<?9o_xh@*97-2md4yuK9j
z$`=~g<YYuFtoWDVN4#uk-Iw&EjyB~DLaBAdcuxu7V21=Do<kvmi^}Qynz01yDu|s%
zZn&xPXaWQXqWW9+o(+x$@==x?0s;p~f(ktO07Pm?75GtEP6G9(I;g2V!pR*PsTsac
ztY11K4uGb)A@!$iGzYeAr{#vqa$0~0FyV=T5lori_#|B0{+fg<M6Zm3z9)zp#!z}x
z&jBy~ROF4D8k5na(A3BroFW&q`YviHR}o*=W9p{!*^y@;yEjlEWy~td@N<e%*oPen
zWQskE*|R?=E;NJ`XsD=;KdF!+!?hx5P+|rNte<mIX%<kJK-LISP6IphB@ri5;SWzr
z1+IHCXsH;6IxN^=jWN1`)S4>)W|%A{mgo`B?XOslq}Xc*-q18X%Gg9nXzA51W75bH
z%AApv#4cNarqq*Ov{VSK)i34%%IMR};=e`D9y=RM1tK<m9hfd_=8U5V6>$tp8VA_z
ze<p02*Kk==f~+8`REK?11ktK@$YoXe7B-5luo#TFBBY(rB5OKHgsKx7Yh}JWX!G=3
zp{^ojl)V3hxED8JHz36!|4wT0DN|)uH0U`DcEuoXGW!_O%?Omei*y`mEaJmDpp#oS
z95}UrHP<Bnl<i_iD%~mbetNStHoY;yGvdffFi8|Hc8`XUSVqHa-+T%;9!{*zm`~#i
z*{snUP|1L|B3TC8ldKI5rz^uKjQP9V2cTTWufPHXx`&yy;sYU6{ozl&J?+-f`zjKL
zo7LuqbWV%bZ^CgOvRm4LBk*{mK!~H|h9y5Ls8=0EKuvf=dXonbkCmQC?#XaWL0(9v
z8O)d(wO5449!cE7W(&qBN<#@M3gCdiBxT#j?2@K?FCST1$jwDe&l;$q>i4EYbWkhr
z2AmYAEO}m`8Yi0qR$Kci4z0IOLu0^MUS1JuO3rd;c?FBAC(pOaC$c@|TZxDqSndpq
zDNQ{1(LpB#oy{cEsk53J$vk@D>uC27@-hYjHl2zdf$Y@Bxj`vp2e(i?qWL5GIWfA3
zCy$01(QC7~hmitwoq=}?n@a2`wH|=9j)GV)IIOUPjO0kJD-oL9qaj|(!7$~^*>(Oo
z_0j0twAOa%y@YtA8!z++_qc&>`LW}%dR4+Wm(tg9Nj`}Age1{-bQzi8v_AD_QfCHz
z)}wW$ITKdU#;=NXfB|NL+b@RXO*qAN%m_seilT1GnsmhxpUn%?jv8yg0TX$xi-dxO
zjvWR6IX>NTeu&T?*AM7(5iPA9<+EivO5Af<EwAW?r1bX3AB~NdsUrm?^d_`*q!N8J
z1}@BVXuCnL`|Q~YPB?sGdw_ze=eo~EBP&j#Pl+Rw%uS!1)UheBS?;Z)5vShfnHz}}
z^Jnj*=ElEpE14)y@2m2Pla+ynsLvEIxXf5>X*(Mms@<2~nOVT0;QuT8R9I>J75ntZ
z#6~slf1}vUO9<}#twbD?fA34k=-~feLjMkz5C&X%;3c%W_$5^Rro+AZR~~L0`2Su)
z-+e26<s~%lMp=Epjq-24>HfWM6yPAY5@BBcy>Aqwga3P@{5#wzD1$Ige(p80_7GmC
z`M?9c_E(-_9O3_79p8P=<XjyK-VkdK;yM29_S3)3t?)ARZ-jeI6$^|6e!KkGKfGsE
z9Mj`9{H;eKa%0_B;mDL0iZ*v=;oRwTEOEBh%}8Aj1@)p%TGP_oT0EVg;QHKMak?vx
z`y>9RN?*+j0SY_bv2Uq!nz-|S-j-bP&slHq|H1t7|2gX}|DPj@bjuejs-7yY0H}JR
z+J*$#H0y~lN&{U_{C(=Ii+_#`Yh>0FNlixqkfr2DOMT3$*N`7gQM}nq)KfmV67xn^
zI8PjlpA*+0TcAYjhBX}zO}OAgr6ZwKD?ph7Wv#qa%U>b2=^Kp>_k7TibFY%tJ970@
zlG|Bi9cMuw3_3h{Zbz=M(cmJhor2z!y?Sbb(mO$vl3BC&L>juPr~59GTDn1%G;`-<
z&5YbjN$9kzvY;E_JFc}X*$eEZ7uZcNunoytVD6-`Kv_#I%NXrol4mb<5>nHiuEeCk
z32k;Z?IG$GMV7Jr15cD*eEAa-Y%GKNn_e`%CDJQ2cT3D~QLkPqE4?Rjw+W%4jH@gC
zdQnQ@_mvFPRnEEx(^9B!2WBuJ4**8wj0x~3ts7z6<QavKlcAqLBD~t!KiGRI?E>@N
z^_=Xq+sb;7t@-Sh>6z<4{-^%^;jyVxwfUsC`Si)tCmX$uMz`T@cx!_-cXeaE_hh}<
zXgz7It@Yi$`^2qxr~a^Cb*FdrMccqV_T<U>e~4X|{7c`>)h3+3wI}NvO-OIG8td!-
zvHn*MT>hJ)Mj$j920=Wl+`Rnc;QlZ9_buz+Bljn@%PE|~S#8W*|7L4_V{I+D{%dPZ
z`1`Mzfd8Mb|Kq<o<jO2Uzc)KE+TxI?gBF!M_(+uj5ekW%lanXB-C%Oh9tN9FpSG&4
z2GFaoLeF#B!63dxlJgZ2m#mBlXu4J?P?*Tr9Zj~Ev3#}Js5V;51*y*<&Cv%4CK6zt
zDWh?8bcd=G-D%8b2;+VoJ<6g3AG9w)JqFZ%5NZkGNclWO#mm?17tA@Cc87k?Iq-Yl
zIAXtBm=ax)21J2%n)3$oOuF2W?mbpz69V1}yD~x-cci1uJ_pO>o`CEcWlfkwIc(`n
z4ILh;0V=DzD-6lZc7ZW_B;pW4gXxg;%~6Mc|Lo22+cRh9=r`xrozv5uqqE<hk)#WC
zrOD-4EcgfXf&ya--EoYreb7_p-s$cesIl{6|6u>@H&lv#wSRWB*KRwnj!&H(=Va&f
zY=8Id!Op33^7i!PxV?w!sh%fimdqkD=a&5aqK-@7i-7}<c+9`Sati-rq?i?YJr9`P
zJ~=;{+|OS#bWy2gd9g8t&&X;afSVePBcETz@noxBzrDS!U5=-<AiS&(8AMe7i953s
z>0nX9a+kce1ata8nuoB$0K(XmvD(Gjog2Dl5(~^8=cQh^3w3+Kg%opiQ7`l-vG8Cg
zJbv#?;{Z#s?*t5?DwV|in76ywvcjv3=nD;kjsP`^b*ExQgqr~7=286lSawj_PVA|9
zU|l<@Qw_5)L2rW*ZGAp#(x2)R05f;xR*Z8~D!x8BxLGZhc_EGgm$25Ua~t}o?}(#6
zffil_@f84jMgb2+)J;~%Zy`|)rCvbcy&*~{<hlAypbXSEWveQoyJbXNKt)lWa5Snx
zZ=qtpT>5b>a&O%6*u7F42NG|+ZvE%Uw!HWJ_?f%i9R}SF$)TSOw#^%nx^Y52`(TdH
z`W7vI*h|}UE?5;wzlb(qOBM<vZ~LRyy9%7*QUf)GQ8BCPIm^*Xtx{iES^iikOwk9)
zcJV{uEq-jF7ohXwle7Kfql-e}n3dPru)y40IzpL>Yxm*f(#TuJMGmJhDKK}1<B2EA
zqi%DA((dZjij1Ts|1phAQ;6xp9tVkAEFWWeMD<m|^{RZIqVo>eE{)x;^~CjH)xW{|
za;c2=WD*CRGb<HNJu(L9PA@q=<|30_EEEJpoJ-ua=K7&IuDq?vFfa4AXRYux7owD}
zC2qV(U{~S$5l=0mMwHk<ANeq{tvkAfgZf9u1&P(4vF}#t=kjaNJIo9&>vpkl7T)tN
z5j$X6<8(7VN)fN{6LguZB2kj_Lw0RJM{nt|Q@xCxhV#tnW8^YcDQ`MHyC$$U%xCel
zf?u(tluyOEQ0r0zV8l^9dnSO$Edup!QuRb;x)$wiZ1jp+yh_!i6IV=DbIJ}&)Y@?{
z5jVXwveym<q>GYnh)y14WvBdzV_HOUA8kM)4x6Z~#O>0RRFBZe)gQA3l4yCUxkP~?
zouB^RQl+uOO$o;UW5D+!w^!%~uv`W7?kX1RZxk6TX4er$qj3<h2Uq3)+iH~?EDU@g
z_Cj~!EOWv#bfV=n*U35fZU=T==f&IogO@mqX!_!#D;=qXYM|zt^%Tf90C=U+)hVi>
zeSuHpl0a+haaLSka3cbr9m4ui%tasVbN)H?L$bm|dN-he)a)&aeOO?K4ZM9x*%<0b
z`V&tj`QTU)cEQ*U^seCct~iR02REJw+lSXapd@y7?ZO@)^!nI&gSS-32JI`(M{_ck
zc?P%>h_3t2$2Z6Av(E1Bat+PtqZr6P*4EWINQel_T48?}v+L{$5LP%2YW76-C5H2*
z4F+kbG*jbf;)innJw<lF6aigOC`4Y|>Ge8r<U7OxZZGrt7IA%ha8+!7So-;yQwqtj
z@WYbu<rO)tpXz<@rjDxO&zx`sO>?f9L59GzQ9dk5ctu1NqK&TtDo$&&$Xjm}b{_zQ
z+w#-d;mOPW)9pG`_yS4JC!Cb+kEQ9zjjkQwRk1sxYup@8wGaEAE7W_J@9AH$Y_{_e
zOVg>TqP1^~Xi25?Wy&+>3we|NWE2HD{pIl?mxd1LW|o$!B9lT}B>IJ;MWAI9s8zqZ
zv^2XSPB$ItU)>O+_H2obKq1e8q9e0O&)g|=zKsSAjL@UVR^rs49kM!9dY_?#G47rO
zBr%hNOVFQ=M)%v;w>o{|%8Rw$E-q10t>=i3Kz@YYP<+Dt7WhIj&(q41I3p{NOc%|{
zN?GGjBq{EmL4O?8!n497x@*Y+M$fr~|MfPXI=!b)`PzCUB!@kx3jZIZdekdBdSj5m
zoFj1PBf`37?_glPqPiBaz`)19h>7v9#Km_-YfwdJ+qpBKS|G<kN20|J`@zH!`GrS_
z=vP)M<zydlVkoadnRcF-JuHBCgoAWR0t-0LH203C;-1$RhS3;=V9GQn;=+@A;4c76
zAGhz=u@6$5g2R2$eu<Brcs-MBp>%oW{^|cR4tlk48egJw8LvM@s%ov;*sQkJ%6L(g
z_#h3Odd-bv6k<?w`73~KGq=fk=9a8N`;aI{Or~4w?a|KRUgvOU_s#y%9wG%ALv@rm
z^PfLEc1aPJR@6V;JJ{Q48}-Ay`m&^5zx`YLZ0}H4j}p~&eNob`{%-HI4eWrff0Ng|
zEGg^rF)A*7GNFGd;^{f?$J4vdKquq>>J<v4w{P(iL$`M=d>GRn6eK!5SWi$N=MwFo
zDFz+7hbIvLn%`6G0yG}HaxriN6gb#_vAYeFdZF;hLw`I=$(>ht1b3VB{CRcn_?2;j
z0kmX~pAP^Z;k$k9>L*NAw`8f)dAo}ZbUJ0x(zJ&igHqnmD%bbDm>hzsk}`$LMxN1A
zNC3)wP<RAA621PoWYao(8q*l70VL;T=mDwpJHlEPJRe5hTUogTe(#Yue|XT48rtIU
z7UYV+zZ?@61*oh5VnCh0sx39Uzz7igBQ4dvJD8J*+y^HXSZv=fTZt7{zs3I431Y|a
zo*=4n>&T1Y3~yDocvJ9Cr!?+H{jw7Sp@?2Hgvh7FuB_e3WC;E4Vi*ecg@I@<2(dn~
z7cv+KsT*uLm1?Cr_F{nO4F{DK<F^}3;|fKEG|KdX!JtB%piZm^mnO19!Vgi8v?{QE
z1f(1Jy8`^Rcks$=q#~jvM&3{cRt|;~$x*m8XE&F`5J@SDa9|Lzy&&WPhelPTt+zi!
z<uvl*$iA^L1PJ;rLsw&dG!j9ju|;9RmD3<3)4bE>5VJ4s<Smn}9*4e@9&Ug~?DLc(
z4m3irh!B*&syNGNJjqIMMS&@b&?+4JUOe0`@?Rusu5N21poY=4zZ>?hiiJnkHtXyF
zGpe!D#6AX}oez9Z)+0pcO9E_88T6L0KsmbWMW4$b5p85_f9e<SXsYCfBd?1alZLWQ
zKdUnG__JrKP{k^fDiK|+dP7|#x*BE{>6%3{O03&J)`woIkO$4*;OTpW(=H@PnS-=M
zO22y_N>~$JLtcC+*D6aZADSp*dh`g9nLAzt(wITjDj!N-_d}ULObCYT^U^2m)t9AD
z4D^faJ`|AyFlI8dF>Azx4Ii;mj1XAC`DpVcq@7dClR}3?nu?<XGkMS{lq{Wfg$KwD
z_^pVzz;Oof5-HO$-FdNJTW|+?v^!vcLA0&#mN8z;{_L-AZhRhXY+ipJN2{&t&#)OP
zpCdT2z0Wror0Y@c<%iTEtxTr_z3+5jEmC())jAPy93DLqH%;}QvK}NdarB5K8OJDA
zMHfcX7_pNtOq78i-P=8W^$Kvh#;@pPQ@CR(X;fUUDPSV_Cg%|{<Hr-k!~5@l7Uk3g
z;c)3nwUc??kcc%li#f3(LvD(fDk2Un7X>zc@R|Wl7;Y~|_1`^pIO<i;?fUg+QQhk<
z)$7Z6(z^a)2d#((USaWLqNB8X+cK~VcfAY81Lfy+fyKp{h@=xr?eG$vR}Wc2G3JeY
zHN_Bb9zCS=DM~_*9D>3H3U`un#tAEcUQ~!KAh#$`_u31`z)VT$nIYO^Cx)@y1;X{B
z@UQREtvU6DWC6VsNEi1MdWKz#PVcHHb4@OSMMu#5^YCc}#H4I5W7Xy4(#u?X_9dla
zVz~NwwR2Icz=EMyjwy2FHQ*VVE3M>ewCP1*_vCHo?6;G>>fz4O+gCfgXKzpUPOJae
zJ3ZPvs2-o}o$j3NAH5bt(iUIl;yEV-9%bAoZE<t@Xg1J4A!~Jxs8V(_N%-t~4h1-m
z1iSr7?1wMarB6Ww2vy7ke8Fz~0@tl6p{#7Z@L~Ng1!0zT6C++$1ZSO#*I>3jiY8~`
zV$>*w^2zy6b^<W;Sd)kIdm+j%-VOBI40vE(KMwq^7smGw=?YwGQ2fK<T^Vv43-rdj
zz7BL(vOsI7Sat!B?O^El7lj#N;kiIzzfC5+twTRdz?}d(2`4=TZsdmx^e_!4w||QP
z#PGQO-GT$gqBa0`IJt7Sj@z%A0t5?@tTqWfRn}?^#5@V^UK63{5HZ9@htH0Cd+jWL
z;wP)6)i28u(^S@Lm9K)areR2IwbSbX6DVAi?IJ(%&lQ6to<t!cAtH9V9<bo<jq49N
zTG$xt!;tGuqG`^;>BgE0!p9CH)B|ht>C<&|jiQwh80Fzb;a~(eRWY27$K0JhxvwG)
z2<5oI_)mmu3cz}FQQdPwNY1OtJ27w$%@q;cm>kckpm92!9n?m5laF=|PTuTvBnCC-
zPmY>qkV#W<3;fRRv}w7W4<#T~%krTn2!ZOYe`wYxctxn*XimXyPv2I>y?1L)snu+3
ztd)y;U8OvT+mhE77Ot%?Ui@HpM*!cb<9|07YJwVQ&S=29S+~m9Hbjk`SyeW&>zwT7
zR+{wOWUVKQ)p}yqqLx}S$78j!!L77rk7vVfqqe#GLe$Fd8r0fM)Y>@^wO%}=*6w>z
z>*WJ#J>3zN>hSC7%&|Q+Ytn42&YVAME}qn$iW;-pc=FU@!gidKac!(=$B1K*ksw9s
zI#IcIsd9pRNftcIFF3$RV2-KGA?mzO)<FdoRSi8Xgg)6$44tlvLX{hDNL8D)7JK9?
z!dFpP-muKJmUpKB15^#7!2<N>@8>7)&fZ@z0UC$ElkC3Sd-eLw{(t`C;PB}9<iAea
zXK&yA`u?~7cHC~S?+q@m{6DUTqj50#XBfrPo7=nlKR;qWPdyaE?7iPVbLds9>!I-&
zy`zYGz`Y54z}%B*NIp)-!MKWn6ZokGadC}rKYFfP0{)9%XqTOaxE0moo4g-)aK%5s
zJu6&ob&k%YknatJB0z*qUWCcP=RR7N_;*&IMMUAG(B$`=ljHXOdtey4d~5tnbT^9y
z(fJ$49kr&K)~e5r-8|`-5iSE>A>jp%q+&u4Ej@5i&q;@6__J6=+P+nwS$yuMO$rcZ
zHpaQN-q>0#TUBw@Z~gH&UahU$m3Kz|rCZs0d$j+)!(xceyDcCYG+BaNz9vQPqnouP
zDRaK#ULctB<KW`F9bGI$)VNq9h*Gc)M;Adjxq1++5!Oh<I&RljpEgVg{_2xP)$fjo
z@b*|6C0Xmw`nyB3P6RNh&Yw4<z)QEW)@Wprm()rGnF+cL_^8wU_HJdXSvzkuTNhCE
zGbV<9)B}1@r>YeUM^zVSrg4>v%qA>R5J0ekXiI>uP=m+qtprdxdtBNE(>`te+-PpS
z*{{BRUq3uK!McD3+NWrDVkb+GSCfg`tTn4wz*GaP2Y;t`qshdED<0Eu-tH;_=;3}l
z{kwg5(s{XCmC1B)-D}VL<#y8dqsD6OcH$1YHFr=O`9r6)=Y~KbzN$Gp*It=8Td9i*
z(1=<*+CSK<>9@(^$5NeN-uU&-er3X^N;i(x1Dse|?tE4L#lx^#kfNW=A|&KN36MDn
z*cBSpRv#w*PVJl|_?s|s)Lx#pf8M|aKo@>=N;mX2Md4f3?+vTtyN#S`e9G~Bw<=pt
zaIRIBANi3?m)wL7IxVH%9~BKr&6S;0(@(Uj(Ib9(<v!pky4z|`$M!x<9cer^z~H&C
zDO=f>)MrCP@_X%~A~;-QywP;L_C~Y?oXb{Xj;`F{cWI(xHi37f+n)qOuae&~)cWR@
z{ml(vYj>t$5V~*QtHvSu&KoDvHQL+U!0W@M-02(?!Y$Zi6W|jg%MtOm-yX3{0^8MZ
zt)Zvl#ELgoR7+ynHnChHCT-u0a?O(Ps3u>Dv_cj11vMR1%y*L5b~*<q8*7~oN0w?j
z6e{6T6sxi962k(KTVnLHPHoXMm+V%=SmNACYA^h4+evFFT$oxC<mwV9eEG3I0E!5@
z<Bs|pYs=60pzOChKy>e)(d>w{Wqy2JE<5z_^Qg4Ddwg=%-YypB$i-y^l8alA4Lt(x
zmq21_UG6qFpg89tav}u9+fkPJg8Suro{|lT#VrAV&|3w<YQ>R(J4KL!357Vkr&7|E
z!~c_{%>;3yCIZbWqYv-S?cqZRg^=Ph%Ip9)0vjDXddwsrrVQ_e&Kf=3(q?J%n3)75
zAZxX}L%jMuCdLro6g9>rX<qgEqvqP%6!fa{xxJPpi;>zANpZ`6x)XN{^tW>WB<d91
zl?zTo3Fd9xWS)eNYqe%=wfdh2%Nb*zj)Q3Kz^nC6)~Hu2={tZ3Oa*y1Zh|6sZ^A`I
z(l{jw)mkgqh2}q<x)=BAqaf}lL{dn?$6;w{eWkgwTK=+Z-}}H=5OI*gkgU7(_rE7p
zUsm<P$wf0(498oRl)=1Fs&@k18>&`f+H<P$Vq0Y3G`gCkr<%b*=ES>OWV-pOC-UPd
z>spdpwtsXc9V^7~kd^g0&J4?{gruLI;sq4(26N+w?3maL&Ra6RMQAxf=%Q-Tjq#|(
zMSQc9KfI!L%xxOvxLP(Qs{e!&kJiwlbA~Y(FnhqYMI4x<h{Gz910AP`y6^N`01-Ad
zLnQ5Kz2%uHj$ls}>E*SEitzNaOi{_lD(=%`Q_+YTE}k8?i%#EzQF>&>!+C1-(toot
z4cHxk`vRCqxd6cKjZrbfF+A`?PXV@Ypx{f@6#)Zva|2Ra8nTaZ*e5)n=N>Rx2v?Bc
zW-^W`Bl#@j5L12?E6ib1uv{hGV&yMstuk<jut)9obfVFsPW+bkfFW@ct;r;_Et!_p
z6y+_@)Ro(ZE5T|?CYepiw5+BmZ;@HFr_XL68?=59nLA}rd<*eL3`AyX-Y1ixR~5Ol
z;RxnC3zX!`9{`Q{^g}3W^~s+^Bv3&Z9boX*YO7iG-X5Ht?!V8hIUNFfI)@5>xAR`Y
zsF*-*+`ENgv>L6(*6soK0@kUuwyJ+ifW09`qzhTLbQ!vnt12YTfs8{T4)*Qg{z;p5
zxJbak2hj}fKt51DLdoLu=MJesDlJGwIQJ$)xTfHL1+5o}7AHaN>L<skSLiNcw@4*=
z5{UzPZ>iLQd`De0bg)*U;Md`0Zv|y^p;LkHn{)CXCX^s`Q7(oB%8sn|%&PW`s<Eo1
zp_S>Bu7W7;;0%y{x=O~W*?Un=6vh|w-%dzf@m(3OeBs-vRHXeu|6!5L4-rZj$k!Bg
z0DWr;KMalXGp)9n4gTHf+FbBA;o4V%2l8uemBI>3ol_%cUb}E;7HC%dm98hM7)tul
zmjWrNCRih>#;&zg|NeKZNxz^KWtf;E>(p}_Y{?H4X!xRn>SX+`_EErLt%eTP-Tr7b
z1;Aw?M;jLy<%R7PdN4DvfH57j585H@>kzn&trNswhdb{(HlmeiZ6wN7DPiQ^<$yuj
zd3EdLV6&P60XrRflmp-h%>z4cfRv_FZuADlg7;%U9ryG4xM%|erh~$&c~DRemOxCK
zaFgu5ufKi2^~6NIn{?Gt&EDPg?ny>1m>NJ|YjzzquNjhWQ44sQ)72TZ61T(7JKsM(
z+Uj3U<?H`r=jChYu!^F`B~(m4<SKl8R7RC>$?%+<9`Ek8+sCKd8B#hSz*tHY1s-P9
z+uUq68qNit-24(vC{cK12{z8r{AlO&_1;<M#s1OG>2IC=qqDu!S3A3VU+%CTj2p+~
z4F`7&UZ}Cczj<}<ur)(ap(nC`Sw*1fGlHc}!K+!i+simqQr5whs5*(p);BluTS-c8
z4>va}TQ7s*1h^Hk$8q&!0-(otr>oXZ=yP7yMj&9ccjPYchOz5FEy)l_)dYmP==OU5
zlEETS<lnsOI1Di23uYt%zlH2~8NH5Wt*jQH&e|$rO8U<h0!cRqq(&pFIj6)Igr@Iw
zV<auNiO&5b{X>{Dy7a3)kUtBFAuf}0h@Zv3t)2b%KX08kTNkzFR;x@r5+pgTTC3JX
zc9VZJ;Rmo%r)Tw(T}+Z95nvl9B(HX1xXfWmlN*XRN-w#a7)GT^FO#SS;_CrA_CG_s
zxh1ZVWhvE(J}wsp$Yt!)GRtWzn;SR;`+*C#v*=Sos_Sm78Kvn&SMMB!Jc5WGmf`5C
zN00p%hcKt_3=AU*eN>MgvPe=jvX*m}a%KK5Qk()>iu1s3x9Ck1x=r%VZV;`u*17kB
zJgf`qFlj}l-*ECYk`+aoO_Dw`EOTH(*LhR-2A61_L%QPrxP%g$N1yONC<Ojnr$hHs
zXVNQ`DT_u_A`<4&qscUmN=5Uf5Qe`ByKcS61*mX?zXipEa2KF@vPhMWT37l}27#ir
zm&Lx0@I}b#mQZ6R8zVjcYY&|NY~XA53tFl`8Vgs%K0aSt-*|EX(~J8C)tvTUno3o}
z2>RhvchTZeiG`eM+fXTE>aY1vB_tI78kU+4X|BO=O?Yk`1Asl)MW1w&Uz8P;b8`Ia
z-sy7g1YjMKzw;QaLqzFfNAQSKCv+GMCzILdlZ#Xj{@KG7s)eyvqqv=)adA=b0)wo+
z+A;>Z3Ii3>5Y?IdU8*<g-+IF3kQO}L1_ra&KY_!#qc8>nc8!F;pCtm`lCoeMEfZ_L
zmIa}bS>MLSLRFBlJ}R`6vU;k8lnM>j1rnRp%o%^XeKsx4L;cB%AS9O{TdEyu9;FOC
zVK0u`Su@s!{Z@rlPt4iNllS%3&yB{`i`sFmT}w$<s9MG?{}f=>&sWzj;GTrGVW?Zp
zTC(!hBz<RJP2lENi$rpV3|OrkSg940bvK@DV)ZvC)wgo=Z%*F6=L4|NGfeHg@j^bv
zWSbId(LJ@g+WgtEcEy_$KBp{FR{Ceg@Qm_C;p;?E9E&EP%8cCY!+z<-<itR|FY?<!
zuhW}MJJR{f$5iq5E@35cNE2@x5(L{!bh>^VWrJX=0ie~-+F2mv<k`IuvnOc&_10=x
zO#!Qh;DkABL2$CuK{-*g(=aM*CMz^EDl~P4)t0q5<_R*Yw5W|oMxTx0%RU20gl0D1
zsx{bT3o|#?Qkl0Zb9p(H$!GkrL%Y>!Hfk;Egj9&&A@HjW+XsmU=T8C-y*Cw$iPg^~
zU}#-u6ER4|u%yQ6(79<JzdhaEvx%Fy54Qyw9UG~@V?&0Zb+ot)ckL%CXqX+_d#5bA
zMjp>_x-qcBeT2I#RTL*hN_=!B*j|pV<WMbRe`BOgF?zC#b4pM_Q)0B{_IBIC(Cd*?
zDvB)$^cMRuc$QtEnW)g5T>&=R*8DlKdW`eJ6e9JRK?136wu{@%ZkUU$wvxrhiDJ+T
zcRGxd#S*2y=pQ5&CHd;#@<u=+RvyF*sA@4Y`;eOYFLIA-L7<|1F+E=`B25J+{j<o!
zX^_JRlD#ne3CB(pD4K<_mvlM}m0=+Uk%ts^ra}gmxFqMG7KC-7tb|wI)ns~CA!`R$
zs=Nr3Hfdcgd<6;Ddgz@=fDw@-NIljepa8%EshsO&f|PJHWZ44&5*>?Sl{Y8kw?|v>
zjf0T~=z*LRYzZ(D+LG;xqt|n4mcLb5Zu_UyJT#lN)@lx%A^`gF<qn;M*%0+4Fgzu}
z5>}@|LFn_0EtS@^k$N^{f65RniB<A)umn_QW{%p8+)}dZn8L(OQaKCR`0qpnvBbcb
zu-sNx8|#e<t9?oW@~d8VGOrc9R0@yOt?~&cNMZk($o~-`Dty*wzY-&iHXTMQPS@+X
zQ-PZI+i>A^4tF;;|3N;m$6hdvhWAq1Duk4AIAw2j_I&}TK)?`bN=VxaLUa)%s)t?#
zC=Ob_Vq;aPhggzT!V(bZk}8WA11Td-TwtFFM=fxHS?ad8UdzI&SrAe;+5;i95`AdR
z?L!v+pIJpnYv~VNN5&+or8H+z0A@BK1z3R*#pJVePDy<ME&kS&DRrS|p+^h73FUMt
zgA;n9Stx_I+Wh(a$<qrnBc+v{!KsO=CP2GdBePoN>CZn`whmAB>b`tft^mu|Fw-*N
zj3!=^pTC;maHJ6<DlbgLNK{(JID%-91hZ*?$!I!NgG8rB1&jS+i$>sxg_6&O<q2!H
zB?7e699^eg8D^GG3%Dkz_|ikW-<b3^GL-Bm)gflu^>&+;t=(XJ<K6M&1X_;MPo0WI
z^wWQ{%K7O<-fhhX)l!p(UTN+lHenn9i|G#Cu;cc6Oce1P^nhNmTEipBId(>JHg;Fn
z=Jy55=y9#C<qvFia{*|Z7PQTLXbW`B>e-Bro$mZr`K@|*vSqFK{b*9{h3-9U89(E-
z*fQDrP@_A3sa|`JejZZL&v&Z-eZdwnGgR<rwf3R#$ZQayqW2U5_(K_=&6oAdAC$VU
z=NpdI^9{%97LWzTcGk9s6+dl!w|P5T-Lrv9`%<%o(e*dA(SFyo+lX$O-(vt|+qHY#
zu8$^`)(=1)-&p;hd~Fl2RvYJy=0(DYj8DU>C(7=`$beqFdi$TiOunVJXQjS8`ZiNT
z2@|eb1#<EYAnggLGu12`fmvF_jfW!@p@jm9Iqomj!SoN{|EE=*4t@Wke*ws(Z;bUu
zYDU*;_MHCDkN>;0XipvME`P;@(PjUl{j$Cae}y@VOj208vb7WSym9}7vp4E`D6p^3
zu~eG$5QqD7^wqJ`6{9Iz6I;0>nF}ZjMlMyc2sqSnPC68sqg28!CO##~e$l?i(AGt%
zuCtMe>JzF*L%-&8snrs@z4@qw<`o4q-T1e{Ppi)RaHIeB`q%xVIg8)Bzr6KR)}x2}
zN3Va~SesLK<c}|JH&TYQRC$rAtF2U3I0o}-0*RIy>MffMuePKTio(Q#Q;(b?R4%e1
zGisC*(#f?i&+Hzwb24vSlZ-(Lt8BwUCRtPMJ=kq(myk}6wwBH?nouT+v~8zIq{e;^
z_AkEWdgG}-_OLQKxZFG3SUbzx)C;z-zhj&W61Y47fN4c}zPh%40jNtFdp}(#<dsU^
z%uGh@ZKWuZxkIiK8)S3Cwk%iG#XMxy@e?-p(2*0I-s$MCG{)bLfhddEpls;(fYGN1
z*h%PB@jXTvJp}`2=j`ui&RaO(HEqI=ARcbMj|r%8BklAP8!t64Z|7r`X$q$hR@Eqp
zCX}b&OiV0{++SHpYLbEAgyw<v&zM~^^d_0+t?BCDYC2fidVTa(=|{@gEWF^N(M%6^
z2I!UZYW)HoL#`};RQ+1|gsy78%<I-lT~y5IRlTD670?Eio`h(}VK+(V0ww3t)!n=c
z+wP8VWUJlj{(k;@{eohsFlkjh-<qP%{?<hizsW8MzCS+T@{<#hK`I^k<2jD0_!>&N
zq(Vhi*p=jMc_BK5UF>D{L;&k_JqupKt&1x3EiI8JEwd3eewTXT_1{qrP5wCQ<X3G=
z_+b*Rm!3Oy)Gn$+DHMQ66ZyS&>-ao9)0GjF*(m3gk^X^Bc6YGU9A8CP>dD+vd+#=K
zJ7tu4lGiEafCvh0Rx+DYK7g3A3c7#vrx=Z(Itl$7H}<ONs0uY}MD#XCd-fdmYf(^Z
z8crIPe#VFJh7iCx(qlEAFZ7O{uTr?kx`#cz>Zkw~_Pp+Q+Pvs=!-ACaTB2X2c1{mF
zdpj@o7j%e<t(Y!Rb#?0<@<9VoB1^khC%f&=yH_WR`Dsn|NmNd2cQ4i}^jw<O^<KR0
z)gbZOW-iiAh*a-)S@0fQbqhdp@BC;Un48tscYx%zuD{R72I0;30d(cvxtGEC`v9YR
zYCa^mr)B|hzt?4T-3&(G2hwP@6@I5ZIRXs*ogkrYpNT3lAvt$n`5ug+_y)qiB7Y})
zn494Zlfe<l;O?N)vFy@hvYnWbCXxz|pf^cZ-rN`?p^P?PWViVuqs<rTHecklsanL$
zlwsW02WtRu%hJ`sU2*9X{Qg4jO~}K6da$JQ?EPUrU>cibW6c|WpDl;}x#SyI0$dO_
zuNMLlnwS(uj}^buiI}+k&`X1|g?cmV<KWCAM30}7OpGLyS{4SI0&0PL>>LEawN!O*
z$eu%zykdU<C42t6TmAE*vbAgzw~774ZmE}@Wv4o5Ny7JJ=Sf94hLW$9nO&EJ>&~WG
z9MuVC6=yYVReVO-v}A8G>1EBYU|tuT;z=(T!h-cT)`l5-NA<#BqG!XI0c9~5xLIxz
zhlJ9;*S&~NzbxqN1#CWSRIZ-|_X;Jf-=~XL-wV(r$xu+r_nOMsN6%v41=NkZ$P%EB
zff+!A1nOI9u)EkBNA7nT7*T~Ue>>WLU!TNxTWb>G!4GGx)?im*(xgr*aA7P%65ObY
zA`(F8k-sp{4TruCe$A1IMqY%3&2+*cTSZWsanSdKc#vF*QH+86h;2ZioXnTPb28S8
zynjx;ahzpeOye<p_8p#}*#r&g?byGgz^v>qHW&<|s~{i=B^v&#aEezHI5KiD{)iX>
zRu{gWB62C^U?6!?|D<_n)oL{Gea9V<W3@>T`LTZ^T(7an3megLuK>gKa4aNzBUs_;
z6g-RE^27nT;jU5W(Qw3$*eHZ34!YKw-a?z~v&M}%vBK~??0)WJ1G*@a5R&#u8B=-O
zuD8@JxIaLl^DRYm5}tD?@*BUKE{gzQdx!67uI<Q2<U=gd0l-ik><))I-a%GRgQ!*Y
zMk%2%^y~duyShqVMB8V_Cv#6g6bF+GC@3svr=N2naM27H(fyDN<K~@cSuTq!yC0+0
zOG`QqG}2blx9LSXeGSqrm#1_v)PaXW^SmSww93o*FSetoA4L5>E|Y#q82~?|%ialv
zJ^j3{dNBLlpWez3PnKAyEiERG0wA=8w#6jivg@S5q!9&lj=ThpBwKD7{ZMk8*(b)X
zk(7*x$ynbaLnV!Heno`yq;|K0p^L_dm<X-mkG2=0=d6vIavf^C9lw{XNAdSV_6=6l
zjzlYKn;VrftNPgDL?3eW8?bmo$}YK{ELr(n+4Y8A7>tubN`FI?74+fkqh`+)t#U$2
zzX0B;z|L<;RXG*=5jO(IKaT3XZVjhOExEDNw0n|T<(<$-tdrb13&O`)D-Wcd4iHgb
zYhHVuO!D6N%<5Oks`;7L_mrKM8xIb($~N?N8N-)7%BtaSHgJ^_1{>*pkW+nGZm62v
zAIlaj3LTXOxV|Vrr?S7l5X6lKbkR8pg9iZ3KghknUe5XFel^3I8W<Y=pmpF4;LgTN
zkt4Som)@489YE&8>^=46f4)9Bk>U21C%t7weTYIpM38qs3g0K6^OQxljGi$Wy7x}x
z4m=Fx>_*XaL_vLFt(5cEEpqLN`t{OTD8;ireA_-l=XT261&AK<0jj-=5*etwj3tY7
z4HO-8-n4?=9!Mee^~)F96D@lW1xncO>lh-^O{wej!XS#OB<NBkop=GxyJzred&n-u
zR+AUrgk6dy>~MjJAi1ga%Ij}#H0YIiC^U#IoCBHiE<>LjR2T_1eHpc53|9*K+cx08
zkLpvm5A&Y3;x}}lbq^u|CKFFmba~OtJ3op;*X#*R_G4$K-xn06$`i32BMw;Zuo=@m
zk=+y#9af(REwdxD@HLtSy%iz&j1DA^0X}F}wsz0<-{Zw!Sr-QaJv{n@5O!6upp2Cp
z%?7LShL>{m31B^RXh$B5_?lRUD`34jvJ9SQLO6t0er{~NtRjr{+G@gBy6?N)YCrV^
zd$=K7%Dg*Yr@Nb~@s$%gxWMl=HVq-)GJ6{|-N`H~(VjB?Aw80xf2KXvfPpmNK3`>f
z=bfy?l}c7>-A-L&zxhVO=DMBG{>RQ*BIRYfWhcCOok)1H-e?*P!RU|MbYU$o8xw)f
zVKE4|-$&dF^C1e=jO+INcjuzw@PCD8FcoZ4KmBS*aK%{)b6K)i>#E~-JiQor&sE30
zQ1CRXOP{z`UkqXEtQOdc8=DdL%#5bi=QU+6zFzxE#v<bm`<??&b>V3Dl-p`i`-dl|
zJ8hL<EO2c#e~A&!|L8LROBecKOTB1glNjmgZ*0tdow9jFGY!seC4uuE!Vn??Zmj0%
zNo$zw(=|i(a@I!h`&Jio#QY0%hh21J0u$al9s-U0<DKj#r!SFga@&cl6vLeoU@y>%
z_Tl-9y^FRIP?=Lpj!yet=6!5KIX#5y3eJ@~juB9`l7bVBGZzRew)Y=YC0W6j>U@b!
zpU?S7(p<-ZPWlhep})-@8FV@Y&sCXeqU2=n=xqP(A!FJKdz%AJ911%hcY)UmPu%#*
zd2@K;051*~YGausb9G4Y20-Y>$XGlEq8KVXKlRa>X}@+@d*OyaI{wn@9lHLow(E_f
zX;gcC{O%`{ik(tB+4Gn<R+X(+XYZL_dWFukUwffD>^pC~TMyd3!au*@i{8i$uf4bi
zY{gGCsr<<?y_Idb&xYt-FTTa-e@HgQ)5!LL__I^({~@&i<r|YIncZLfKf8;O=Z*)z
zM7RB3P1MA0)lB+&21H`(-zjmEfO+Chhqc!s^!t|#nCU$s;NPSp7d;Skat=}u5gk^x
zcIw)ETVp`-hdehZ$k@A$DqBZ;zqYG7n1gZRg7JM7%LGv(_{=Gi3yAzGa=%C=*3&g@
z$?{uU8VsFQ%t$rrpq|5;Gq*DKdWK&V-33U`!#Ad%NGv*wT}U6o7v78DZtI1o9X{f>
za%u=yA|5M}#*f@Sv9B+@73bt+mm)&e5;(T&!T%>0=!FM7;mkH5ADLAA(;Eik84d7?
zX0-t~z^o>SIduT4b||0ZYjdE!asBJ51WP|22=@x>WWF$I(1lD5=Zu=|_tmwn_S=)a
z)2hAUqq{ZUCew)*&fD<-WjzPV`h%cs<UrYY5R@l5P@X&p%4QCf&4)ob%>xrY2+q##
zUJf9)=PeG2s2d<7uG~o!^zXA?=J`y7qQpGgs4Pfn6}$(5eCdX_zH~^RvbNIkwLX7#
zlDT517iMXZ3U%2;RaBl8wJjU=WD7~Zg9J^w%(lj*9A*ofP+>_`9ePf);Ur=qJv4;U
zqm1}T>}%yA0hF-u2M1BYN^=4!Nf(+})C{()ohP#^F_h(%Q&Y@Xiyj^xJreVsE~S(}
zxF9xVcnQOw$_;1!iem??rFiV@P<&1A&g)Hyyz2E3F9Y@`=VVV;9C{{pqAR!^`4!j&
z(hJ3=T%X=$$$bVsc{>y(0OK(^E-WL;d0|CmEKl(mDy#NU!H9x6#X&HPYL2rrzE3&^
z##>GH<_TPLvhvWy;PWLuJWiA2K2u^DMX;gw&#51hZ$%mZg*_O_`Z`iF$Nlm}s4#T)
zBI0iDoF1xhE|A1cS>xWpgU*3gp%dtuksgd;V1AE+ie()s8DcA?qHdTjG6v~Wi}UaY
zj$|AMl6I<W{nyd^EesX8icU~BpI*$vEOXEV^f$rlXdoQMEx`gwbdfi>6uyMsSN}Pl
zgUWea=!rck8i%TO(WAZN>d#w8$4B^RE*zAZ2z%@W)fo1RH>!@Klx=<jR(lV)!Wpn4
z?*Xt*XZI@#7ls9W3dZd?m?SPpqGE}f`e$_<j8P+CnwXv!?G_T!WGiQJ$lVAts*&%F
zfbq)^J-=+%lN{cxdob}HTAeqdlc76)s<@k?d{HXO!`XxJV2G!2OfN&|#%ZYsxCeuK
z2)nzQBLWL@*g!bGF(Uy&ef)AIfb{MepYbEZ9alqi#C)buVKD=P%Es_itm;#t54n1u
zqfThE2|HKAndSOgQn3@9l;ljo{%Kd6c68bPq^X5Pu<TK@k*c<LwD*2SC2#B{_R2rn
z&<eHIY6t}LkAqYaM)z1V(`@*tFoqp!B|7x}9|zTD2B`IXP<PkEOi-2lD&~}=tL>k)
z^Q-uAGWhM@K1mGV@7wn_o9p@bzj*{HkbCrK_i8wsgp@t#Q|ZC*{s9JjDG|rJ!?s0U
z&X-lqbAr1g6?1hE>6=mH7!h8v)V?f5hRr#J-pN8Td{Lr|Y2;kldcVi9KLG!{OWp%_
zUh370RhO>s=V(9S8#oqP#A!#ZvXl{e7265>^P9mk8C7>q=PM|=VX~^=gWnyp6UT)%
zl$Z1Z%|^ugkv|N^ADrW{_sZ%?f~O9~-oWSuk)VYhkU~HGz2!WAj+O^YjV%<O3e7o=
zzbS}1M0$k+(OXe3^e3@Xk_6UL?2iD+x}!;Xg>(}KE-(Uwx<juQT#o%e=^4L}#L&)t
zFr_D85omCKfGMGD0GbMh{S^w|cN_Y#S1=t!64j`60r*0ofPOBae5KlnkpQ;}Fkx45
zJlU$(Fa5X{xi{{3>|WI_$J1I6USi^{*RB6N*_Lhi@iTY3I}ExVK;!=S*<jmjy@2+=
z-wh<{G8AC+OAU4!It<}Hymz`NES*mJFrYrh)rA4$00+Tv7~JACLXOM+Kgah4ROJsx
z9MTad7S%JZ0mEHh#i&OCa}QIG@AfJ}Yjwifn7@cFqc?LPNpDy&!UJLp>1;PdT}Ta^
z$ntR?_6RSQa2;VUj@lA%-|GEQC@58vZAYQ7QQ3Z-`pig0?AwpBPH5H%7poJQb)v=U
z+?aK4=F};qT&hs*RB8fC+X`Cvk7}8zi5dmeXd!D2`ybYmS=o(nD0I8Vj40`aWVBkd
zS|Tf}LCQ*XQDx;dCL@((G$yjLYO>q4y5Xr#)l6k2C)C2SxJ{FWTGmx!6j0=WNg_K&
z0YNkuEFD54UuhL0&Mgo^ZWan?!-L%SjT`!if}<@+Lg~<!Q*!(L>J?h}EE6KE4gkSY
zLgXyt>8djNg$nu?De}ZyPEj4d?d76ff)6zm`C;q6`a`2N!vFLWg#hh9397mWpN0SU
z3;+G$ja)!(y=wDGZ}aJsr%yI|8;x$m+wj%~YwqgCdhf}4v(b9eT3hS8efNo5CuteJ
zbksF2o!tiTNsT8@*6A<&v;Q_58&5V?|D(CuTwh;bd$PXKg!I;0V;#!>l>?XmrnqR(
zWEceTta9`6lY{%e<lp1JNe=?B+jfP<9$!0f5d1!W!V-r8%Dg;5D%7B~1LY_8AzgRP
zr%zkJP&79jbcp4&gF#F$Ij>MONibHN{c#U2E#qup`D(LSZLBThVI)!q&l%Do=!XjU
zRtEEou2+nH9D04!f%Ci5*h5hpT#I$!Q%6DHAHby<7f`pd@3CrP?1iI93aHSb$ZK-Z
zJ#<c{-J#!e4*VXl?xg29!IbC<XsGUefi)}h1_I=^Ph7C1##Ut$0v>8X0*n$6dURj2
zZgCk&1|f+PHDMCvuz?7;4BZjMBUS_K3lkc$`d}d%3|WR^2abL)9da=C$Ih?&XK#+*
zo;f>5zd66|oSyC+o&ENVNHlyG25iod{Ly6S17H|a=#Jxi=sHwB+&kTU12uMD>>uo(
z{e}-@ulCQ5_S$Xd)$ysb<DBfAp6&0xJ=i&QPTroL9Jlvs7)jleGYc?s=5|1<gGz^9
z3`A<gWBv`6(+oHbRI3M9BfC|f+|OS#fZ>vJYe9tSDSU<^^KpPu59r@>K`-9k-YPP2
z$RMKnPu!UuiXA=idOkGc-SxbQq`GGJVFg=)6$TK-ri|4t#%%@n0S?Ah3mglDT9^zp
zf+{>#YO>;_F(odX$<uyxcBXNFrB>(|jNE(aR1CKRZFjL{No!a2g$6-KfEvZRQ?Vjg
zI7FJACfdafBsxg(y<r4<Y981$3hqN6f5I#PT|4q4+WLIfBCkg!p8%M-E4N~tn^N)h
z$-&KPu?%<t>Yk9=JCoglP{vhwo=rQ8)w6MyfvDm@VlXxv1N0G$QQD5ok(-!yWVR5e
zwWpKG@SZ0Vcf<e>OS&koElx^!IUNCkk2Qgr@QH)+$JFc!DPvP~C@6^@(?ha%b1yx7
zG>BrDxsW<W<OuC!if`9MXvc_QZhf3C1qM5%1|d4dy<vEG&=FoV?e#p=?G*<N@g@*u
z8LS$wmS!-SaeMq-nXCj}k6?m>h<R00*IY*F#Emm?d)F{c5gu|Fk7jAH0M<Z@YaDpA
zVlvCjpu*h4CW;E;)q%|nU3?w^Y{}DH0Zd$BM_2;A<CO2YKt_wE`BfyEp!gGcbL<_5
zlOp>*3NTNrZ0`+h71ojrBvA%SgAHRkQ-rWXiS@eLTn^7GK%M$@1-rz{qtZNNiTaS3
z(HN~3WQoxYkw;EolNwLI=UfJ`W8`AXrGN%T?nF&0Qo+GAVrn>+Ufx^^m^cPFBeuVg
z=c31*fQ=a-`lKGD@&xxB;;usX3mueLRu6ZM-oDz|J$rk)cUnC@**o1i+dq0Oh<L;y
zAy$ookw-6t(Td=WZ(;rhaHCcUt<DC^4+dG#_vw`0h5^5`v1AUeW)^Mpc^cRwfq5Kx
zD)@23dc9q#kw-CLZ>D&*w~HSNZ}DRbWeexWCuhKST;Pta9=^iSLO&=b=E43>yHMDj
zg_PLHwQ$Tr97sU#S-_+?PW5WVsaCJN;Y9I#5=X$;FkKj4B5uL*F_zaXhAL0k5r#W-
z0V@y{C5D`+e}nZU)-c+WNo)ygRw|s*c>(g|k`Lx07`<31$e>&>k9a~2c~i*{d%?_o
z1??#xFB8ONBnX)Z_Et}t>`d<Ig*O=Uy%+HzW>ev#<ANlhp3y-k`U>lP4J8GG#ueKx
z7S6(Z=flS(lt$3vE^G41kOyG}xGY_iG4z7bh#qs{WEKD~Axc36zx3FtUdH6|s~;5P
zOM4S#(ztlmUid72;)_BxcFF~@qr`SslU_6=%?w1Y{%4foB<qRHbS>Jp*yt6tc$KP2
z2dS8>CSt;&IZQASM74aG#&A9YMs(#5Vi5&HRd&kKQabJfYQbmv@nj13wy0#6uB3W2
zg5xaib=mS#a|s7hR3%BMfG}A4K19(%MFD#3yz)a>n2C$Bx+@@3F0Ys)AQ6N_u5^#p
ztk##BRuifL-ASYOBxCLeuo()x<d12|8)b7OM7^OQnIK<yASjhC=#e0}W|^Q?tlDva
z_r~~MRgd!Ov%nh`%tPxdKs01c7R#&f5-ESWKM*_a9v_|{^FpSugd!yhZ#yL*w4uc^
zUej;z3<6CwLbLcZX~1iwtHJ>R_LlFnS^;S1uGa}7X?fh#w#L7+H^S<N()sVzi%OOL
z`=Pv4|Ior044zF{pU;taipO2d?i2uz=^rqVF!1|2q3?S!hY=MCaHGNR3hB-`A`ON$
z)K#WEtbo7qD?3FrdM!LRS7ZXhydHb8nU?bteY~kEs9`MJj;eyaVKD7uA**N<14^p)
zuG}CZ$3SeEPwr3R(RFnk+_>TZz>HVb<WWl3jr?9mZ2Jc-M-!I=LC}U5v&{Z1bYs>M
ze1`_*n~rW+jBFlQhQqI5Se|2_S+!7|3#weOgA%+zGdF0hQmt1d7_$3=V&uG0^hq4I
z$7p_v{(tr1z*_>z2W~Gy{D-$z*CWl+<SRbhg9Ih!(+W8eY{zTQyo>u@5AFm!Rn^sI
zqv6~J;WgZlzk1|8n5ygMiIM}9F&((1c^=N|fv{yIFvz{gClV0?DWl6|<X97@l+Wj*
zYBU{JU~85!jGl86;w2p$Z#lgo1-BwHn%-AP^Xf{oaPpFPgwDAh;d?7xSonlZAHwGd
zeS6~`T+{S9>8-D}xWJ{)e*<JVNJXyd#$4gaYNJ6PtMa3daqGDSVM)l9r{nY1>e___
zKw&0El;wB%@7TMG=_;jr)sMJUcY;HuuVKtb6zgkyNv_)q1&rH>y2D*x-H_dDiVqP~
z3BC!+d9$@jeG-vaWM3&I>=UeD1XMPU?GpbBYZtmiWyHSh&J_?+K9%kV!wKv*ZtL=j
zUnk(lC2>p$wmYJpQt1&~q^P<hr<JtJaIAL>?Rq?ouedYrJuh1?47x-2Pmgv493J?u
zeSDXia;>|=sfRKcS+v?4U(3A6r*8Dnik3!s?c<+YuSb9UaTkcj`AtCWX<U}SZo^16
zb#sTODVk9gP(wJ4#MY+XiG;az!hYAT0|_%KVE`@Fh>E7A`oAHw;CuaMBLygd#8@d{
zbE66@6KNrkYJh^aZa`bDwQknG5#Cw?-r6_9!zk<LYZ%&F02bAQ$f_?&ioZ7u;~U$(
z(d|BVUXq_6U<XjBBS~;yil1ntNInt2T>I8Fx!L$1Xtk19xz@M9oA$2>-d6*Fb1wS>
zpRL+A^#+KxyNvEYGI}Csm-nO>Y_$Gu-A5zDO7qF490MCALV{Ib-OgUOGVMH>HHouE
zGP`Hu$~3wsB4@yDH1kxcV)$hNn3#xg66yYpP(VS5e;ku7az<~p^{N0cq9~e=4DRKn
zM`BqaS7hmc7oz;$;#+J+g9#Cv%=5`-*r}<Y!{nouXQ~GqQc49LOZYhh9tvqFlxhSc
zKgM?fl10b>o#fhqUQ*;chL8#>%IHz^$$A6t-u2A}haYoWPp`!n`+s*n8}8yo4r;79
zpJkvfz(SiB`sezF{@yr$qO;If`uWC_3j+v)=dT1HOwge4mj!HNEdg${y4t~D6#Og1
zTbG=U%Y+dnjLy5m3y(j&^I`XbKX1-kqG|kIg>NJ8$vOO2Hggj?pV_-LCv;u$*Mk)N
z14H730c-{O>aE=&v$WY*ZxYi)XPUG6&BhI+^;h}N5DWIBNps`wjth9LYe@1YJ`5Qa
zp3(+>A1RDMxZ0pUSalG~4qa^m<>=twb^be<xZ+QfjF!cZR;{7G&d0YhUG`tNl;$7)
z>Ju!DmYewJDgRMr{Rwr`|0gOxa}s)O=C2{o%uvnDa3uQ*AGl8Fms$KFY7L!$P~Y(!
z;-Ae;u_s6F-9Is*i~muSoAraJ!x56*_}81ze`EJj06-fI{?qluhu4)KL-8^6bm``X
z{-*6Jees&fK`-`5;${9!6bXH^1pcxMgzHav(E68MfVQ6eOlGY$jmrEpRb<sJvYIZk
z-mr?`&s348c9BFk{m7-|R^O@2(8>-o3T#*fHWCHYBG7j#(*%#d(gilG0%`D3x9QkF
z(?!B3&1)x_U=}myDK$y)-Re_Ef2yo;v(e&O@U<$B(kQs|<%W#K-)I8M$KT_AL^Do*
z>HN@_MAl=U?o9mCl7DH(^OtCP4Er1hL^kuBgc6@khdn!sKjS`q$m2_z-VXo8DVzL3
z{}CXCxs~%-d*3y`si(@LuBZ}D#m9Z<j@Y@#XX#1TQT}s9^6&|EFBF(Eq9f0r$l*m-
z$cwIB@Mml7!qs0ie*9jgyMVsdE<~#M)8(9@XTTCaR=uHWi3Wv#RMLbmAK{=@e|7ce
zjr`o)xRaklFIJy^Yy!L4lo>Zq^;cJa-pImY_G0%cz@ElFUp?@5U45*Jj}7s66IU<}
zC8nQtQH~&2_y#T!hyQr;Z&QA><VU1ZZdOG-KSI|j^nL58Ep@kvXYa1pq{i<^Qh48_
zG;&qTrT!$O|L0~SO_<rVlM=r%X@`SMSK+~@imvArT~8I=$S%2&EI0&|{~7-S^uyDO
zbRBkVVvj&ng?o+;5d6LB$**QhrKqo{*}|C{-=HU5I3}=G0B7=mi9%|29O#xep=EK^
zHfWP?T!^2mKZ|Bp>9!xOGitgaO5k>5o!01DiYi7?GlPzf?Kvtuig2f0GBp-oP3a=P
z!;r>ax;B(bO!3{N8)p<;bg<lz*UfCo>n8guiVw4k50k|oqns1T?vIn+vK3zKqbRwI
zg(PuaYL-csx*Vwo5FqUFHE%d1hioej#kP9vlv-ut3T}h2@5lh_MF&nyKaxu12(jy3
z`fTxu4=H4JRpA3i1_)&|7MVG5Xmv<3V`%ia`-j(yk(H;1Mj8jqq{V_zkD!@K75=X*
zoj~l5v(yqkd|YaNSl&`kTJYOy{KGPdewBE*?=<n{NQCN>qNha|r6iXGtWC9|nH%-Z
zOih)d6DbBq<&sej<rz$lnXtZL{vzJQsw_cU%#EF7f=~>O1w`^VW;r_zP>PDLq^iQf
zsW_B6q9M`OP<>HJ=4~pf&P~XP$lA!nuB*O0N@WH|B?61XiEQ;KlC==Hx7wim^WGov
z#Z&%<D@heZmWUIDpo6|Scrtz7_e0eM<drBDxh4J>Dp0^DQIa>ji+q_l;LypGF?iD8
z_K|zyk5KduI4J&mr*{f*QItx2luE{^2r}{mvs3CP23Ih?k)=|<^_&T|O8&$XN~Opt
z4t+e8A4V-?usHFb=1e1#CBKJqg@(ooA*&_7k=ddG$Y-gKq1y7ybyX|!GH8g~K&Gs2
z=+B>9@>88BzDYQFbCY|E>>7U{amHWBi>W$_YvVkFiPJ?UQB{V;>3Zr@>|tix)Tc2T
zV%^MN;y_CF&CX(u%_tyQxI{5z+ssnT#@U64+&2md^4~5bj+<uZlG(_?Sw)zoOBGv9
z7F*3Kw%$k-W7aNJ>}j%Cs_Vp*sWO7aOBDzcMZ(NN8;L?2sY1r033e}4z;23JzI36D
zM4_yvfYY-}5$l&OhP+;)AhUmFQF|^kCK5S6mCtOT${zU&vjGhyzliCRzlh1B`hxM(
z-})Lu9*@(J+0$R*^avJEm4>~{=@Xs^;tIvrYD*n;;tbUf4E&>i_!_sdAu!e^29dx1
z5P$g0PWUhp8>p%YuQDowqQ_Xj#9(zi7Fji%s!!%1<xjywQZ>Ovsx06mIUVQ*{$Q_x
z^r_Y%x}pTh3d$eIPMY6hW(9L8zlpmPKh%d_CS)@$=;|l3n&MMT0`r=x=0s8eg0(ci
zyXKD@{R6m8{R8+;3m7t=R>IAuE`0OU{O+1R1pFIb=jcxD3F1c84`0#*#GJ~nb>nkg
zeQu~PeKAw07WE_4yG=5#@;5T1RGm0e{ngZ;k^XEMpEs)thO3sJIa>LPcv|`6PF&%_
z&X+!s#pN$_BA|M&SYFQK_$)H>p0=N4fvLowIPmfVh~MNZfe62HISjfS^8*NC(d+}c
z>tyRvF<27x*qWtlEfk`oSg<jq%dNW0B~@QZLp=cE0M!avaneN}I?&zK&5dr<Pxsc0
zV<C7wC?|TmN;14BEb+uYs9}NRZ5&rc>ZB(=B6k2|fsg6fznk<jpMx3<K|#-$J>C<#
z5HcHA{vd;UL@vY~^+l(reP7n-y1i+XQGwDE>*4kTpv9R1Pj?p(FBQ-)rc>3gAvpvU
z1aG=Jk-?&nsuGhBk^ZhWHa7E~%rIZp8No1f1}}JWd&$U(K3zctLpO_6Gv;3u6c2^=
zBb(OeVl}S1eklZa?o|elk8C67$vGrnE~1WqNe^K7H>)qEOI@=L&mw;HgN%hvRKr$9
zH4_MKQwEbEnaThmTx>D`(*DU`K+4V-u&9=K5Ysh{MGdpYDcnuSA%=Q<(ddt?1AxiN
z`JCpBo|JwhZ1mhi<-6|C9rrS%N5V}kKS7IX{d@9*!4-#Y?;1GWbWP%a?YSlm!$ifb
z4MJ5PvQ^W-4OJu?wj`kZ?x>JdQ{tO8S4#TO!^5|At|axr;5PE0ZF~i=ry0l%5=r$-
z;$GC6lbf&SUG_4tzb%HO>BNGv5~;aWdl}NtUQ;N+rA~#c)XJ#TnpbHxtJdoLY8ki{
z66aRSKoyXfTaDG~{>Ym9KW5E634Jn}Vqs9K+GwIG()$7a$=W)UZm*s<9H_dakT@OZ
z;$keJ3RC!KYKy*zK=b0zy&FWCEpsW<JTyRv`nqyXRl8mWs)NMbf%RQd4ZuISw-aW{
z;!157D<yPzflwN`cj6DTT_ax<6a_?;3=GM>%DtY^5UJ)iNqH20!bb)IlP<5sz@#TG
z8%9gQXe|n(rC|(XyuVk)i?|HYk^If&KCqW1M^sRHxYRlnPzgCREwjzWH?wZwO}ntX
zgI8rm-1r%n>4tfk_KibDA#yS*teX|gU^-lDa@V3&#lKnGN3h3TcHBwlKZx#(+7(f|
zLS=Gh#i(u$(pzsdZh#5SctA_|(dWubK~#*$51d^+tK$;D1Ml5r6WviT&ZliPVNV5R
zC6r62Yp}SQ{+mLjtUHeQiO=5%{^U_FTvk?$nwzwbkK22!Kodls`r!1OMe#zv7e#@@
zC_yA)4(?h2Gh;8I*xF_mF42crUPs4rZ(fqHH{-5HmYy@yWawp(b@|!UB$g4yuT~rD
z4bo#*8q0P~>DpZ=u@|y5n&~CrYLs#C_>CY@cNCfR?%>{9tl}Nta9ndex5f2YEt1ZB
z4qoBRj01NyAqjHMhPaxT#q!C2DrfIss&O^RCDT=vtBIm&q0aSF9VP}>Zesk-Tfv6e
z&!^X|B!uhLW=?&<+FR9E&s+HoCTHd8^+uw#jR*Jl2^4-}f-n<i>{<)Prcr%U)qL!{
z+JAqzx8;N$y(`OyDyq8x*iC$CY!X!C6Ve+f?c{X*iRGVcUOW}$)hDDsUFF25tKX+v
zG{?==jVEHvR?32NwwYMD%?GaB#3dUWsb!4cK$uKcjRHZGApo*E!+3U}>ttcYR1GMl
z$vxE0lvY3ZCksV#30a}Cxp9+)N^{h-ZT)eDL0TrbwZ*l*v%#&I(7q9t0V)cH-!xFR
zc~nD%y9Qz8+(P(6(pCH^D^+hIB02HTLb6OODC;xdfPdDq5HLAUy~GQ_vUDS=wF<TP
z>LXAb#Fc)M<pN}`sXUaOLwF`Y5{6^jwkEc1+qP}nw*T0iXyQz4+qP}&>~(K@?^8W}
zsIIQ>efQu2*|C8&gX~Qe82v##BwDf1g!80T%faIv8n0u<?C|Po_L`;MXp@tcta_50
zX3|osQ=72YI;M=S4UKF%)3q{Ljm<t>c}3d`mMtyWLeg{0`vLO{-1q%P=|Is|RaTdJ
z7K;t=8_9N+5idN1g4M`x2NN4!%9=(<LEux?StVIRIg!ThrL;-9+J{9j7vp%G%rDX_
z5OGD)<aTesf2kwB=dwOO(4UDC9Ea3tv_5QxmyLZz)Gj_3?JwOyUaxjh?v;1Nd=2Sb
zATS|bC1UF^;UKEiIorqgxmmIrs;KIyci!|lMoEB}DAh7#CdQS@A~z|XD-)|`KReU{
zy`PkH+0=pJ=Qg|H%uMfcCjF~$Gop7!=S^BO`cP7y<%OM)VaB<;ku1+>pJl*!Cfmf<
zq6r7gXq`74t@P00tGi;Sg1Jp>2CPrr_{O?(u#_@k5ArDogZEd;-o3NgAl3scVQV{_
ztEXCSC0JZeNuz~0n@F-~Vju&?r#g9Dj2`XtOj+ThummXMT*8K)D*FGdaG4BsU3O|K
zC!&w^{0_3SpIbKPN|mzxGtc4EF>AeyF3>Z+tuncnd1-tj&v3i+w*D^W-i?-HwH)&0
z`r}G!11pd=mwtp|s1(<XQ9TH5X2NVKGVSeR3%QmXf7_cziCyHY8c7`D(lX2ZDJHI!
zMhRQ}ilN2PmO+a*O@+4}7Bc1?!e$vfCSIHo_Dyzzq&v|$VT`Ys2AL2spv~v1%v?|~
zC9%&BUDQ&em+0?X6kLQNRkAu9X{y{#iA#U0tEGx`#OI^KFrg}|3M?-tDc2(pMYbP$
zGWvHiu0pFF`W_zpMrx4bIHg9H&f#CHpbv0mX@jH~Iat*QV}3lavX5gvod{N?TE4?4
zOJFRuRXef@AJy48Zj{AfrGA<{`5Ohb1KTaC5@z#01EDACA&5>wbKKdzq_yt^3WtXi
z8v4Ys8a)(OAO2|APY9^-cQ6&aS6UGDZG9|vD6@E?hE<0sycQUv8ku3gcs$YmG(soK
zgH!kKKO4>ceyY#+&*ZSr&DluoQ0BGDs<rd=v+0?Mi&|p~cy-r*7W`XyCYy$)9SaFM
zDaex2;P!=-OlM)FFEYwvw9zS5e_@8v66v87R1Q_`TmOOzyxREuaKJgSEm7@K_MWEr
zN(RK@4V7guC$a0()OWUvZ*F9{Ge+~RR-5Aj_Sbv{<yVmT<9OaLX!BuRr8LMFj&Yv|
zre}jcqK14!KC;cSKqj3;eFy;<Ny4Go9|JRQ2_oj;`<{fO2oc3DtWl#W+mXKw8XaXR
zk$z11i;FrM^6D&~e*Ge?<Dc@4D|GFA1W@HwvJ7C0gSS34r!C#-+6`9Zn9W(7k8f-w
zq#jwUNa9TKMZar-;F*~Lt%a4?iyWFsk%@g$DP<Q)R^FGarZurNuVbc7C0>YYv?JQj
zEqO0T5_7q+E7|Z=o5HkM;$Lng`T|~ysS^DQn5j6lniC&UhI)%Axp;W*KDU+G%CL0>
z$|ydtTkqQ?Vx*9rnp`X`l5`|t8B;5%Xo89xESJScKoG?Y=3H}wsKIuCv&5Ntsa>R%
z|0+dTE>)#BY_2UWyS6NKE!SBzw`8xVS0&HQYn#hV!X9$8q_3%((>2vzQf8{MeUySr
zgxf#`IETF~D^Hzeg-Q+b1P~43L)a67mYPF%Vu_rtb6`Uu!xP-6t`~jGjm8YX!3Tm}
zOI76#S8EA78F^%H=;oHXV$LXbkm^Lcrez&3j(1jsI<>yD;u5C#rNptWj{o}WdF~%+
z<o1tv3(ibyO64&mxk5Bd<jomFVJHPGC0E%`b`ZDRB~Td^)fKoKt}FT>VNi0o!+6l#
zK>hipy4RD$>e91j+c?@BVSo)HDal5b7#{Tz=aX^gPQ4F;BdLxtGtb;0vr}<#ZO&Zj
z+tVKMGsB%2vLxQbo%y{tlC+8YFT9(vd$gsquThu%p6mTKk)m<uZo)TFWMb0oy)lz{
zv?13_I)h0L?T`=#7qp>w>@!;9(a3YJIp{8mO~$l=sV8)D?a*E5%Vf9%ZfiDZ4>x<a
z1iB+}dwsa>)@Jtf<RvgZOD%d)$}1{zEGsHee({y{N8?taO5a4hTil*cw+gNhEt>-e
zM7X;k9MlR~8Idlx#6W97g3f!0<FgBTW-}g&aMwf>48Jh20PJ9|8{8Ot083Fb$WrW8
zYS6hD=qnI1<=18~+qQ9xEh9NZ9_9N{Fm4=PREZvvS4unH1l0x*xSht>R8^?hc@g%k
zW}NCk=h(c%;%KC&wCJa1B*{kT<wyKx{7d-n>`Ri@qwP94kINz_Oz+`Hm+M3@$T;_^
zwH3LAQ*bSBvx1mV;8hZW$mR{=i2)?+1aqgK-Z;}+*y;Yf;Nu0!Ll<JbSUslQ03)A?
ztI*{htpxB2-n!hU?691;C%D=EH<3n&!vmA`LyVt`=c?N@7edJxLT!;c_OV*xQcd&H
z=v=jJorvoeNLG>v9pWab=!=0p!D{o-f*!LUDuD2hOm3m1LMn=jR#GfkJoA@|cF&&&
zfRcKZ-wz7s_Ff1BtWv#t<$%`Q3b^s@9Mx$t<z+;$DjTq6z0Ic3JT?1X!UCR&nz+r_
z<9rr!S(l*w>P>XTZQS+BO|;laks$JT?T9vZOOOY|o)776Vl>;MB&W>DZ2c*bhUfWN
zlM?+69E&RQ+hc01VwkZlG>t*=Vl?s=9N&G#swtbRCm(!>V&-te7w!*66NORncFud<
z)MLZD7RJY<@0P^?CBw<+9~3xUwhri(<{ABvCW-xBgpium_Lk*G^>iSJV)>fk%brT(
zd~}1N56X<Ot35DmiiOHwBZiV#>EXc)QYUx6pByKMpr}QY3O0={!%k<OhFuVL{><cd
z7iUa=Cp8_Ys>7KkMFx^CqeEyVaiknoEG3B9pg}nfIYD9}B27ftu;sy*94fYFfA2F@
zLa9;Y`_-f@^Z5~X67x&F%rqNRst4wl(>|8R26RiN$E_xG)A+v(ujnFa==tr2sWAZ;
z-LrG>m|NVxf=CBMEHHIe-55lZIJ2n7ZdrIBdJ>H{_sbdfo1Vt!ubYLL*^2qS3?VAR
z6frCxBW;Ozr@}&pfRSY(M<b%I2J_$yk6Dr7gO7K&gEp%8B1`l&MVApix{)9IxTauH
zxBCtw`d%*qa^cF8ba}p^O!Pl_^o?xp{g|?BNoQ0j`#o--p^nOB*{VJKR)1x5Su*Z4
zQcZU5WC(=wU+%o}tF8PaxTvNuk3xMj{&li6PQ4`Kp7TfAikg<{#>~y+Y1a#`j4Wb2
z#7Cb^&P1ABV_gEq2#Tp<aE6-tV88<IKVgezOb><zUUjXb%D@G$G6v^O5aB40idyZB
z0#~Gec}n=bb7WvimQCds;6=k7d~VX!Am$1M4YmrDB&1us3cO9l<VO1UMqzLf#$74E
zp(ZK`E6A`a>XwD#x9Mh96~1ek^+;xe`Hj8vK;~G=SH33-;#2)lq-On0ofQ3)nG_q%
z{%fcPy=)|>ksn9uTboR`=n1sSXg8|pia)J;=wJjY7l-r$Wsu9ICtLq|qCjCgPueI%
zgqnbSjTjQsCoZtzK7z!(pbIO3J%PXH!XbBKe@Sn%U~*(X7PW+*<0M`~<%fT2pb*&A
z=uxyE*X7+7#E+NoZ@%p<zj+gExEoX#M?VH`r=$0kE~EaQJfu@ivU5n1$*7Fnum%pK
zqc6sE6({rxA1lTyJ+Yn`Ozw9SY%e_A=^VG(>2}pD_1E%Sv3!YHWx~>ZU>@CpeO0bP
zM1lAa@?>qtvO+qIG$X*cc)#sF7>qAQi)>NULFn(KJU6k;s6?`oHZl*|;)tB7so6aH
zy*K^Kcb<Ovt+P_WCsAFeWt9)#=%<$Ff(@037xh`pjGjnq7L-&R2X)m(lhi<s>O5E=
zCLj8OUW<R#h35HABLlxut-idZa6S_((_KDMY*!UW>8v%qiUYTH7+tRxw+m`KJbk4c
zLQo}F6p+VcY(8>K^2wGgGw4_bsz9HqkoX&C%OZok-QZC>-_p+@z;9~S#%B<rH0v)}
zb-J|(^}APKDRv7nLQ)#{kdJZ{WW7|Os9m47K9sXD6V&aL(>v31Dx*#5NSsc>JR7Ew
zL<JGQ2R+3R#XAXe8>(I$Bf2m#U(&dK<@9iVB^MAF#&42Q%i<tkKLj&`+O7B=R|pfT
zMLQ{-43VQ&g2c>{s5fd0#;o3Cvw5<F4D9_YPd}kZ7D+*QNfx?+#cb=B3qw^Jh9y1e
ziF`J+3HMhb*IGD^npLGz3mqHEF_!FLwO^&)gz8ThHhA7-0ipE^str+-EUdS3|4PW?
z8i~-QXgU*(5f}td0szTG$hr9<VaZHFlzp+zix}~bES(Q5orK#(tv4`o@~+vQ1ri-S
zd%vKQOUd?`r8S8p*ZY^*UDy#2oJkuhDSS7;TU^uxefw@Wgoeh|dx7zB`0kb<#95vM
zQZHR;0N4~}F8m}y+}21kUlGj)Vc6$lyS_y9SRQFC(9sWEOt>I&L;`ro$ioD=z>jMX
zYLtKEBW}P)&I+xzpik_0Nd}n})-c+W7;S>2f-YR49Et{N0CkQc3P~sds{$U4X92oL
ztmD)c%q1u{*{EX598aKMQQBUcS<Pj#cL&eC40qG8`9p^=JN8osuD5PKN<@OoeCreA
zwmo>A$YS9)YC7=tYG)s3E*37YA=3g_*{?cuXR&bPPLDLoQN`sf*`{O~$9k|m%{?b$
z)Eu^`sx3!VBUIEFDn=Jlhr<gSsW)~OT^MNr321owJr~X`%A-sU069LId0+|xY*Q0^
zql<D|%LV-E*q;;CTR>ls9qRVVpie}&QMU_hdG@*V!b}RBNE<`y8!d*mN(%2k7$8)=
z9?S!BG3|ZneMkqM<hzDVCQvAQ%)c*{P=&c$=4Goxxkid?W2*JE<Sgu#QBGo;JerEe
zcqV_VvQF!~x2e!$EGO7qf%VH%W59j@&8Mwi`udVNxru&uOPO^`bzN3nSv0k*_XOU1
zVdLku<WzD86wPOB89kR!boHi=0`k-{stY-?9b*=*?gtl_2I>-@9g+%@9A$}B=qA~`
zjhoo3Moob}_zwm@C{5Yl+-RfpETL^R#?0busEG&s%QKl41%|lksMkcTL-<a|^9FHT
z)L7?n)UwV{lQI(xA+={6ieHt^@BrcLAx#fRKJiLY4)zWK^|1_y1`+e%?D+Pm8Uiw&
z-&M|1xV$b9OElWqU3E|bOj=xXrP>CmI2W$!R#PO%G}P<n-u|T*ZiPtuP>`4V<=F+D
zQP*wiqjLBMFpht+uph(A&1bH#ALNcZw9t$phRL!mcTW^WT`#uuqu_ig4kNxI8RS0G
ztAJ?aF`>}O`iau0u)gqyf*dLRZ?dc4LS*tyd~wBKWJ&ENIJQ4IxDUGL*~xs#O6$lL
zJ;{LP{bZoYR*`gT^}=@o)u4TWUS(&1)qnZdYm|2Aqrh?DtGzw)8y=+~!lupK_hW;n
zUJwB47oF4!9}PoK*wumwh-*;3WLWa`X=;3}I_slvQDfB1ZCiwx{preb+3d2<W@v~H
zsLqd+&V#FBO$i<hO$Fp7iHp=y*osx<IJk7~TXgFusIoh0weuEb>J(tM2io}IB{YvN
zn9BdO%rpKe^@bG?!7T3Jw0j~C*JAKzMUo&crj|xdx)4rvM%v|)aECS{n!_OT>Ls(|
zB5&QP*9Y%wY4XJ_B6#T~^iD<MnmC1Shmi#{)-c3eWcYNza^MnkPK*7uF(KAD2vj_3
zm_+#Jx>|r$oAgsbKnV&0guLRMz!&3t;PWLS!pJgM|Nj6~Cv22mClm(l{l?Mz8kr0b
z<Ymv})hhpc%W&5U-JA0>r)yVNUtiCswPPvlZByIO@JtxDX83pJ@2_d*|AAG%I>>il
zBwWm??J?ORHI&G!`L&PQ8idm`#{YK(^$tdP-j6e`Edn-p09oM_OZv;N$*bc!Roq`U
zfB5_NbPGwlj`f^cySUvIM9N1ZD@C?KK0Dx=Z!0}Q(5iPVTU;McVwzv|9~S5Kp|6)K
z<bVTV_aA5Uol>6@<T9KY#n7O9l3C$eBUfqm1@actk<d}tQNbmYoO6Hh&j;3?zhJV9
zGz2$z!BfaKT9zk9M<VRQT{lZsII3$Yq+23wAb-E=zrnbdV9;<c=&3Y8Pz~qZXRu2-
z!5vYM`mIkQZ<eS1Y2xrSQj_1FETHYxUhp0cB)LwyG*M=SR6!`N3{}<9FntGSG1?yD
zL&6DQsE%m!(b9yI#H<Hnvs6C)6y%xk6-lN=Rg@*$Df0JZ<Ykrnd<n`cI@|OzE-|9=
zEG>oX$wjdy7>e<En9+hi+GkNQ&>BS#yM2Y=T%NO#Lnx{CG$2%kI}J>OHB6`C0<l42
zv8KHy>PW>p#~Kh#kEEoO1V*zdc7w1H?dbARJT^N*?k=Kx>C>P$_eLzG^P7BqF@87@
zZVT7b2#O6fwfYoZ!9gd{C#3b=<b)K#F)Jqs+pYghkswe1v_Wvx?bZ`mj2mx0_|~X|
znZLhHEDOy>JUI{As?iAj`Z8;6ED~@t!cW~z6}1-gECa5QXIo<BU}Qar159L%-1w^q
zNX{f-KLq|p%}iCUXu6{!JPH%>l2wyu6O+&TCvd65S>#t8Dc$tJvvUa%6kJHOmL`Vg
zW5&pb1!eh2z0>~bJb9%KT27O~xFK2^9-GadDyrUy;@|2zLxp?#5v0Jd*v3@xdyK&g
zfngEfZ0CE6Fkih1tuZqBraDUuV%CJ@iJl-GN{phDpe!Ef&`L#teZjpn4x!-0TqVLY
zjt_<gU3YTuV?cPbrw|emS9m?e>f#);@(v^_9tu<~9DR#4f_l^kQ+szZ(*vgC5aXg2
zgg@S7jl@}v3?lH)Qx_`0*eS%g86sw_Gx9H7ImyI4cw=1)BI8gJiQFC6AB8x*0{Yka
z_gIYB<*-zBPNAw$v!VpX|CAR>l<u3iD5#DApJM*r-8CfABQiG2@;g&Mz1ZZ&!Nfrh
zH&x#V(z*$Ndll&ES=^Q|+VgKyf#|aY!trS|`+CQw<adLVmY>`hi{HP&eaX!*s-^JM
zTQuBVSG>leKXY;P-a;}3eEP8DFBg5n3xTV@8c8>z2P!$rA1f|Rrw5g3w&dv(fr3W4
zbd+-rjRfhDBXcjwTha(1$f$X|)6y5ib2u#S+)FPj<U&+39M>cK_xJ7msk5LfZG@D4
zm_c<tfE|G*lA?=#1ELY^(CH;RiV_I+=yj}}QTSSZ_j4EOy5j7qY>FR_;&H&0p+TGd
z<3J&&5_JNdkoa_f{|t0f2f=@8`4t~0lAaZt7AMY0T@%K2Qq5j2leSTOG8J(Muk0&7
z2O1>ai1UK2w^5NLIS$1yM}!rs1Qbs7lRB*upc5#iS?_2C%j(@H@rvu55@nT1l_638
z;tf7N{P*otr=UTkzt5V+7^3G}7u`*utRJjMOAn2N@En8_E0IF7?M4bG`k+~H(6~>|
z3epmSR7n>up;5bHfIMeoe?*fm@gO`c-D$rgROHz!mzwFHE#BC`Tnjxb+&iR}4!jn#
z7jwT9XbvJ^v=W(RSVKh7r%xC$@eG>4CSjyhgauHV(BRudD}vWWA5GO!22!Z^Af%TI
zm=vvT--RhVh4zRcH~9dq#`#4#@dnz6V<krnzt9Oiv&P~i6~12?1*OP@(c$*dm-=WQ
zLBBcb`Bhyt2eVa2`m&AfBPjtX801_)l%9{CtRoYb#mFls{#DF-$C^aA5K?>>1sa_x
zc!@rOE2l)9k$+`a0VZNk1_UpK5@mQ0j?+8-cJuVl$UQ#ZWLY{L;BTN^Sqo(UOY;~!
z`I6m2U5=W8jjsj<hLAHrm;lIdQgl^mSD&li5cH)`Ly$MFZipKcErrb5-~%<2#V}8?
zoOn1x7qgNu|5U6%q3k-bt{o@;=I>taJQzqp5qHzRo9B<3kI$QuQF|bvAZTy?`8*}y
z4ZJc?(86CRyF)JeUSn2@2qQ}OUmKBLwa`%J7WRNyNYBG|H{c9>)>B1V_J;a$62rCf
zcQN2?FJr>&?%;)RmqWJ6l4o}qRx*;fy|cTkrvANjq=NQ{H5WF*{#~L9Z83GnAf2BL
zxSv$K-S}_R+FV#PRPk<PfH5NdFYYn=9H*|voH|VGw3*%xkWvQXwR6kOh>yFQ00zru
zA}e5AoqsE4lcmu!r_;{AT^Jca34yU4Gx0`42HGF?B%_IN<0th>1Z1g>o+4aW_A08u
z88>PzDnUbI8E#GCe0_CauDiBVFG1B$$AO<+FRjA8@HUrY$Ji?|f@1MeS&02N*k@75
zteR5+H?*ZtSe#%~i-RGCEw4;^s{4+|DvHe`4p<a_iY4cJefv*$lOMvrp^%ev!Nf<w
zFY@{pArWV~l%bG8Tu$RCWPKIAf|7SPWdmhYR*@((8CLCr($5qAMb^{P2~Zi)Y6|?N
z7~3@f=8+{MyG}gDSYx0jNU*++h6t4MOn7tMUr{c*nH*Fv4T*R|WC~EDpA^D){h^(p
z8d|h%KFf3=MJj|)(5cjS@x{?(SV4STj$eB)J@oBcL}&_g8o=EGeet3QxUE!-dnRBp
zgCj>+28~kxz<kiMfi%;ED$I(>nDBg3;6*qv1#!{Zv}(CbG|{0|e)#IZNW;Ls&rg<S
z>;;g!X<mHVSPiBiByq($xG8W;>Y{&Nj!O*|bduXN*WHXWCW^(kP{ShY)o|I>8WoWM
z1rku@N|_AzE#G{)-!to0Y~DH*ZwKJ(f?lr>;}0xHKc9fapL+H@)*p@UABNYLmJ7s+
zvg2nA8<XRAj(l5I+>5a%hg!Ru@0<4@*zczikrOgFZ>o2S>5$(rV^&*Mc)P|APZ*Eo
zuhRvoDtvv;i!eN${~%vqP4n$2LLUb){~%8u2?_dFf3GjQ{C2$B1GKp{93nCk<QxJ<
z0>B(Gmr?CIk~UzI-oUftRd1D0V%C1zBmCC4xGuVUUg|<2GHK*nWwjpj|3r9d?b~D7
zAVQJ$)fNiZ%N+J9Yh)gGR2VG}$Pj)F4}bCoD)03L_RhJ)SBdh&4fnO3$K5IcAD<j;
zSGP)(x>Rdm(u@Kp{_gbj=yAQXs(b>ss^8Z`CWj%hQ!9n`pK?mi-VjsL$1v(&8FyAC
zP&7AVqv4^X^pRlTxmho*!FD>C8eNf%<RVO246Q)u^W~h*dhhOD4BmO>rf1W@`^=|T
z8!o)N_VIx+Gr+&>vUk8-r}XLksDFOZA4f|~UBH}-*d%=HY<jpitK`7y$QagjOn*E`
z%8oJhVI4P30jm%_#Gans{%BgjPa+pnlD6DWFl0747~QKAq$I0Ri}uOwZ4eCy)5-FK
z`W0v?iGm<6f6dQko_Xam$z%Jkx;<+fUnAdh7xQmdX%aANqwKxFh(|OgU<(KmZ#5~|
z*14;qII;~bbIEUQ;YY#Ujm(sK>|FI`!yFNmACNwPFAX2atL}L&GW<)t?(5y+p|feD
zN=JYFwl+<tT}0;_;@R`PRV&_*LFOx#hn+V<{c?|mLOCS2P5+$>hXF^_(m`Fob;}L`
z5JJ~WQOrF_;&CrFBx5}^l~zh_-aErkOn<BFL6|s<u}-tc_Ep+=l<4W{+Fhlrh3ek6
zr<?DKNVwP~%Lo_A`ipY*Kn&pEsi~Fd4>(OwfIfXk^>|jIC?v2ZM&>1iXN>>RAohd!
zhU3XlpT5<pJD=feonltN5cXaE?J#_tmk;bG*!AL9alj}*$6Yu>Uf1_19`HC#U;Kfa
z{0rghjxE)MJ)yUu7Z+QOY(9A<_A-n2w3Eki@~z^+B_7c*iSvpIMhXDmCbrCsf?6yI
z`1??*-m$zHa3WKUJ;GAFy3S7UHw<R~tz$Olt8<>%ct9%Zj<R({i3*S0JBfVBw@Wwl
z=R=YEXRE!!mujJYgfKgSAwyvQy?Lj{5|rOCv9S}<pTl*)N6?Ui|4bfj*QV1fHDX5-
zYt@@f_bdxyM~l_0;Zf=kTDZtu5v5UG2^oo6OrshVyif=2Y_@76j=QRVM1SY4cS$kt
zqwsYyYx7w^*iLV$A`pPk5iok}hMh#!TTMd95_Hzjsu!_vS6bczuWPg*D(%hYBCi-S
z@b>_ki<^I)<v6tIPt21Q+Hyq2sl)*Xb@AbE7SQ0NzjWHdP?Y69_0K!iXnaW1flOVi
z0wCwue-_}!DYd|n*?0yLw6t<fukq!91wSM@Asq~^$!%T_wjE(SP(UVs6{X1aa-#<l
zUjCI_#(CER$xa%QqG0@Ymce7%J$XoJCsxqRESeQHQxe27f~OK&cnza}d~}}-6{UO>
zuz0)?L%<Z7)t56h-$)Yb)9CUXo~VEJ>GuKDBIBNP-WC(i<fzQ|NzmrlcxwU;3+mgR
za@!rmj(s+{cq2Z*%{mJoeBq6@GZ5c>-=#^R23*oG1T=R8^fntLu8@#0^M?=$O5R7F
zp-85f9j(?HYA*O{28{R)Zu(9}2328-S*v%iHoQJ|uu2+qSSC;~+NPz3UXr!N<-OzK
z;(ILzg&x!hvWsfkV;#WZ14)U$7%8OOsx2m7Mm=sTAYprlT89ec>Y9N+Nv;~Kk>X-c
z0s=OR41|Rm3u=<AI+M;%&RvboUK|S@*Uo<aDobT%0ve;*9`XX29oI(*vY&K^jc2w=
z0{IH%TS%JJ>{%wFYn4Tb!kEklJH$@MK&16Rg$2{E;EWgis*U1{M1aak4A|NV=JKld
z@Q^~%=?dxG%xvLR7Hc5;O^mQ{<K9#kAQX^Ll%sF{43azWrZGzg*LAx}7;BV{UvT5W
zbMq_@j6mIcpw3^<&X9*r0Qzu@2m?FE=}iVZ&n*rv^spNPHy5R$e}jXVa#6sepeeD7
z;*h(6v?k?a+(|%BM$iy!XjUK`LE|b4uKdV68k5pJ{m-+^V9XSkMutvPa_B+4K)3+q
z?ji6pJ8TExx`_}bA^cWalu0VBXwDaNi4vMOacOq^W%e<!iq~1r^Y_aCX_>L(Z*LEy
zOpKfn9bg(`a4%Q_*&`|2>dX+>+_x#ck2bUW)Ycs1m1c%~ru(54g)th^hv%w$dT%;_
zARmNs*Ooirr?WWaC4tkc_7k3Rh2T=yf<tMDvs=nFOP@q1{vUwMF5ttN*N$DP%5YIZ
zh5{?M&PcGdG-AX2$Flz6`SkBLLH<>@=%0JXligj<)(7*5_UnbOEf#ADUGw%WuXj}_
zE`r&@a)nw1zGj&cEz0kd%cc5We8oal2&a_|zj_YWH5!q$TvOGwvjF#&Qy6mmgY#i+
zHiUnKr(EUlqfFAJFn#l=D1ou*Hn5)6g0;m<;y*L%=$)N{^q1E~orOk@*k~+*->vy;
z^LtwEI~pu01}$cd#8*6A$jK?^J^K(B!A}(?2|P8W`trZk-qB3qqv)AoZ-5$|>U5hc
z0|&TGdB7JNCfCh;n;Un_Al;4>Hq%!{D!ZJ3bW)hDEV6WoljH&V^{9o`Vl$|iMPexT
zoP#Pqu6RY&w9ZOW{6zL!ND@ga_oFc+FY_Drf7r!d;0z1P3568J^$t&Z_M=rqfBbG&
zVtim*e|LO~BFyi9rR+DrrU1!FkJremd{1G^qUGh@nr(YE{NT9HeorGiKJ)1|*9EKz
zHDAQH=16*#E8{}2gtcll2DU9^KgXpfh!jBLUxToplsMf|`khaHG@j#38F<<6!16fy
zC9=#<FJTU0XG(t1Ow0JaoGplTmSl5#Zdu*@ylmf9Pmisg{dt1TEGZV2Cir4Eu1<|w
zRn#|V3m|sm|50GTO0@_1i<r8W-XvaGP4~#t4zlv=8w3yPFO<PtX<$suVTfhB>vT)X
zOG+lcePM_hH!4$++y&4Ysdnjx1rw(Y%Wm_ej;!q1_SxCpugOu(sKzzB=n>S0BU27@
z!rVporgif6KmZAWYVEIwv;8rAa#IW0G>FasR|r%#lWf+&7F{4Z6CoFKNUO5RETLE5
zT=N;I=9Eli*|F>-BgURwHu6jX7kPyE1pS#jG*Hp!8l*_@GH_tC8CbT4(LqC%sI0=0
zu1Y72to$JgtG+C4+YSPDizdozqoxdwu(t=O9QVU*w_^Y=HEDfzT-Sk1+e44bpKkUO
zr0n0$3Ctnc9(i(#?Gf2RbncDNhSzt;#~8c^i?;6tE{&SefZ~JVuPs2xbk4yuw!nCP
z*V6aSoHZ7GRfT%sq5OyG&*=)l4+TWJazG7f2(WERZH9=Ic*GKcOPeCUYpiif>O{;)
z`e}bsK(^Wf-P0qbl;YTJ^|vc+n(`9X7jI}Um6j~*a9qe9qZ^Z#Ti_;E9Ijf`zl^Yo
zK{ZI#W)zQKno%V`xwKLIel)wvta#7NE@qIsn+5VK@QxyU-Kkiq{9qGI1d~)PEn2mZ
z(_`0Vq-)gi0fz%3;fItfC6Wp_&fM;pgXbnNn1^#CptY4oN~}&}4-rMhY>Bl~6tY)3
zur}Zp<#fM0<N&__-b21M=G~(K34CJ+Dw8W9wA}gB<1CH-Rhd?B7$tYK;3L8V6BG8@
zeU1*JizgvvFC@7z+;=lr;ghfDI#Jnq1l~vE8$Q(m4VGbJ1Q3!zJD^tw6S93?KafTp
z9U-wju6&lE!Dv(dng>kINGS@;Wy(t2?dSUsPUBkp=keF$EW)#CfCm3Kg9ha7L}gFt
z4?U*)jJv$SP2amJsKr@^S!SSUKepr3Z2t7#-9+~lNE33SPZ&T2xCv9f*zDEo6T@1$
zTTjqDVq;2}JtixW+dOGnePH5LaS8ve`v?z)S0i%x%dqfB>BRAb(C7E>(g*yaFEZTb
z5LZG|8mkQ4vZkevo}|gJ*L`jtetxQ}+&WJCwtd?)czT<YRPsK`-%oRqM4$=W@f%N4
zQIX4OHGe6MX2Gu?^|g<ZqD1qFpk=O+@yMIGJ1kP}`L%RJfyY6lQiHEYhw~OajC}$K
z37r*XpsSE+7ctty@TZQ?og{}%*lOADl3$U^IkJ9Jqh)bg{TWj&*hQ#F(I~n<LN{WH
zM*v4ED5S5bneC{UIF2h7;Zp6kP^TxkVtaF?ju8IijQUs`0W>yq$kvf53{Hmt9+p7K
zW~H};EjL2y(WqyO0ei~<l0J>^tu#}N_As8jqE@QS^jUJikiWIt%WQ`^M+t8jEa3|=
zN5T@tss9b}>qFwITV-|dgF)Uy{^e!e@)4}9C_v%lM+qm1H@NSmOP)6t3Y~X+1bjcK
zo0Jhm;5_{Tgo9UE5-1@+PB@`s&#gRRnn1CzP9X#{5*32#0B%&sdq8MTuot3y`@6Q=
z$9>kzW0lLl)dPlOmbh3&c#t==@Z*F}F|zwYlyqIMV=I=L=zwB%Ey_!&RIE1)PcuEj
zpDM(Mykq3mW<a@`mR8=og3cLw#Ku=en{L7W;d(X)T&vezpRf<!>GRpuqh8(fA5+BB
z3N1&#kBYXm9)Q3vX_4MnZkuc2lgtz06ERZ$x3Yb@%Io>q)#2ZhM@AVVPs_P=ZIg?K
ze{;v~!c9dbmzOJUad14u$<8+S2VhivxWSHOd9z~@cVM74mZd_|AI&JJ!{d4~)Kgf~
zmWgw05+FFE59kij0LT~WJM9isF2W{6&HGLw*pDSu_?l)tj62sI`v)0AwRsjok`C$0
zBvx0lWBqYO&F37aNOGZu!do}BqgQx|SgfFgUOTG27%9D<taWwX+z!~`+H<>XHfo8V
z-bG7Abpxt2ej~P3`>}`TC+W4Y<mJJW5ZUtj%abhs6-fVU{ho|OiajPfr*GP!26)@q
z{dvbkIQx({4I0-*Yzo<MXldCPJw{e$L3Fz?K*A+LNfgz~Qw$89>jG6FRo*aUuypPB
zHGjuK_(0Ln@{~7m&@`?Gp+R`OiN5>Ld7q4QGxRixah{E4e>bLsfl5!AfdROr`+V0a
zt3pZNC#?w2zK(vyC+~r(QHp&OEs3ERxe)g_J{=~>!MePAo`d!HXwiL+v)|z^{BnV%
z?l#<J!B&h|$efgPJiM3C{2p60Tx3VP<W}mX<4mvSsPsI9_}Aw1nmb}hm*w3guBc!6
z)jXGg<6@m8jRZ8qe8;739ZTG)AA}eN1hAwlJ#+0Mj4phJ%F|>kR4}RRgQjA{w98s~
zzUQ2)IND30QdSV)bP$`5#)(5yls{^K*+vLT)FVFgjyLR~k#*r8RThWE+=swb#{j@)
zW6@Jpl1<d(A@065fA?6sQ!X4aJxkG?n3#qRzO3M8Tcg?;Yx4y%y!KuzFGa%U)mvkG
zU}(-8X?8Sw*QERi-JA@5Ej1}2UNknYjUDk5$TKXBJo+mMLUYU>n<-_H)u*!N<R~d5
z3g;K*R!Q47X59x~y}Oo(0-<1J5`>wL!-?w)d?S?|nx4V_HCA6hW&_(`Qp71A0@65C
zi#Ox~(^v*cG90$1!6-SOE;P{(!xHHwR=(s9*Ap@w&it$hM$)QV3GZXzn>*WO$bfTU
zB9fOIw>@nxqC0os)o_DPhB~eA7x3<%+Wyjc*Vt_JBj)@UA*9&}?YD&t)<9l93UCJE
zQu)K<$`@0lI;wT<gN$F4Pm(X$uI<VU`FaK=B<lHh&HRFN?$1Z9en+6<c(@5>WC!GG
zY|c=uvE0Q4H}LpuP*%$Osoo-(?)}!ad!kI%F$i1RZe&eH2Bosvjp%&z^Vk}xbI`;1
zcdGQ`STuQaEX2o?I*+d~1CvEad{KI+xWYm7=x6@U5o)RwBdciE^}Xy~>0qE%j9+mU
zUY(+SE#ftFKJz>hOPZc1x%kb{fAPq;oqA5OCd@A8Jk>$Z-L|(`d;#<D`Rl9!cV(m2
zI3HF@K!F%G6(+g5WIfk+pRoNvKdC7>A{@ZFlI^*js~wyD)wR{UcuKCJk`VN_n_)&T
zL!}dU%`DJ+Sa9mO+O_K68TIp<{=P%WOhe0{!MFjWzQC*vzSwpg8oA3gX@}JKdS-jz
za!a4lZhmgQ-fb@8#tA74|0wPn59u#EMpi%N=}$p2?`{)s|JH-G#U~cS{I(5O3;hkC
zM&=w)dM+1jJ7J`Atki_a|KvxER~Tb<lc|Tba4LqR9;7N_kcB7})~_*-`6p9gsB#YU
z2Db$vAkS~yA2TjQ_cJPh_6Z6XJ`}b+6`s!i1c-{nq><I2P4r<$?><GDpnm&~aW@er
zYJqh}ShCm_#*|-m1kFiEcyIYwEfkr%-$?3`0fAeeOwrq5tE>H@BF!OJ2ww-`U9xtz
z4(l%hN@MDF2S(|Ll#)MY9n&={O;j4#FR!t^n^!<o{pfw|D^vTc;1M54DygZ7-!*uL
z@R6mi<~|2#wHT?Wmh=wk*ClV#4%*W5DEd%prgNnji&SAg5hhpuB*XN%jUaPQgsl7~
z_kIok@%HPH7(+$+7Hep(@Hh$GzX81BM*a1B)A5fTGN@UNiMVIzY3ddJ(94?b&Jegg
zhfroQ{!AoRqrzx}BIiKpw?2hG<E0LdZf!!oh$E0(i_8IMtluWMN}oPjg*OYN#S0C-
zTKLNFp<3SNS`DaGr4b_^$^I!-7@lE>w=EvMuwf4n5z%bE^LYFqmSg!W;=Tj!RxTo;
z(6{<`C(K$S2zs3YG9})huJ1bIeGC62aHv;}OS>R=b=0`@a4etqQ^O)YdxcXoqn(cm
zznyGQr#p?KzR6i-yXR>TZlTaF;;OK;-t2v`1aEcd;iU|7e{iC!n@%TJ3hILK`eS>e
zTD*xod9J-xZok-{AN+C*&KJ;098DiJH*#}77Rxr*PQcaP;Tf8h?++CAZ8KtUt^&0c
zW58CI8BNHj*_pNl8p@6F!)zmZMZCNQtWeAW`kR+?C%S$2!z2|77`&WK*+KNP@wP2C
zrG0lMP+01HwJbm0Mgg;cfXH8hKU)bMz#rsCwEo*liah@e2^r0@G|5<_)QP+?v{9%L
zt7}|q3HOjh00Zdi%d7*%_{LDdPp+R*WPoGNB}5|N00M2h?=%cA?`~SceaAR7s_qsy
z-OF>Ujv-Y2bCe;JmRpNA32Xf@Dtr-G{s@RIs+_|~5Orui+}Ha6T5x_(sOT7zqCVRG
zX{JbJv+~>H?tmBIWAuUGq9gkx0<-sEb7QW)v-DIel2kPe7P(FFN`*JgiU}cIMJ>a1
z;wJ>@3`0QN-l2s{BDLgP+w3N|9JS*U;Oy(|-ry{6(ve6nIS7s0m;trg@C~o*ziPb?
z7D=nTLNp`CG80Ipr&fx|1FrQo%8BrC<39b&E%C2UxJm(R@tJ+(o*!Ut)<0CMRQBgu
zM5^+AV3Pp!=u6(4^m3K@T;-DT>~KAjCN9ndCyNwIg{UVb!bvdWqyV^e+%Dj1y(rWj
zU%5Z?_bqAz^L-rk`bD_Tdtg=wB0gpXOr11ClSQHCHO`6~^d|D!#;)_opjK6t*iLXL
zd~Pz8xm=j&g&wr6Uz-)8DWtjj{{jiO;4;XFiuZ}DiTzK2I~fCW7)fVL3$-G?$*UZ-
zDY=X~_1x}@1#~QpvK*xq8=v*G!c705lqcBjjp0)CK_S)7sy7q3@WX`7S;B3aFOHZi
zvk_o-xwh~9^|XWbIX`&1$)U6o4&cwh_(zD0$!7823x8t52l|Ye9nMnHZpGNNavm|L
zcM!dwsy$h2w!{X)Xn^dPfwQT-70H*HI_HejO3*WFD)_l;prl~$=nP|@J~0?A&mkZm
ztAUf0&>4Ghie$BrNc3ymJ{9P^1;N)4-wPo_46&l2A->P!G-Z!8>3R)p2<eqKH^XDn
zIXw1?+$QDPBmr+ew;Khcbgy>*@{XU>oK5VzBtTPNt&VS68#CKJcdp4zFH|GJh`*M%
zH?o5eU959-oNO?>bCq^BvvnRAkKs6v-M6wML|t|#clvb@N174sW$YiZ?T!i$AX`fl
z24a`*8=i}<=zMLRfwD);Z7>k>)dH58*0u3w44y=KXznQ|^JvwA-pT#>ER%cC4Wp+L
zS5K;m<wkZ8(-S`yjhp%2sRd~hzYzG01$Rsw9|d=CJCwjOFM|XZ-(A3h>g(j48|v%$
z%eQkuP3*P6!|2<+M#?NPS8Atw)-w3leD^XSMY5fAd$3d6OM3`1y*(xKGwOpMzuq79
zhAjYD^;?h_pl>aVgMA<8@B4>-CSUb91DiUg>|I0gEly=;!vK#m3nWZzjhvprqnUba
zi;k%Xz292hnzru8$fy|hWJ)kRAkLqRkKx<!;qOY@F`*<fWz7nF7|&Jx8H#y=GLi+=
zQORQgC&JfQbhCw<E20ApO4lky)G1H#mZoKd$hDKXn1iB-%=jOf{S1rJJgYqGTQXH$
zSTpU^FycEG@(c_v+kJ=8iiKP=e>!BXvGofQREo*Hhv}TK&PSDyK5>^3s7`9Zg+>h^
z#-}xblk&2C)%bn3<Cc%*xYmZSn!dgQD}_eKk`5&Kyzf6ZKwfY?PG9b%ggI}sye>^)
zU6$9V)3wh%DlsTMfsHw*7&G-7Yn%MZ$91gMB)^R>-Xv>1oU3BVIc&`8JGSn-TGj(r
z{<zW}O7RQ>?cev|@9a6TEdp0eZ4<#4&%M<lvFKlGxNNq;mmM%-Ayf|#W&UoM1pQ|U
zXkO^ZJOjU5opmnPLe|DBAtD&^M?A1V@WIMX<t09Gkeh$V^1wb7KJyXm(ll??MdZ6n
z%xcAlcKy~bxr~jYnpf#mEVXbDDsJ)msQ3HhvYb~6)0bjw0zU1m32uULJR<CZ1cIr=
z%`H3zqoLZ8$WgCJ6rjz-VWQ5(#a7wq`kZ^7KKbjaqx*srjQVfL_P-=>p=jbXchwDA
zDeCqinp%Xv2Y1iY$QL-p1np(6i7Q-MZ~Thbf(u1mvGr7*upYey^At$dg0%>e!}L3N
zGCV)GOL<@(ar^JQ;H#7KrsGEI`#0#SnxYi@KV)-Fr26amNm6M?Kq1|!Z56Q5;Tq28
z$>We?b>@?seH+CFFA4fZQ`J<Y<+30jxWlZp$H2|qV>s!$SBI9jv<Qrr@98P<vNVR3
z@<+|o&0Z|;_W0q4W$I3?n_NCLK@lhvmEPd>tD$rOH)OQcq7h-Rn?}!g&Nh%A4ALj>
ztGyZ$@Q|p?vZLg$vIFm209J<<-L|l1fklNZLl$3W^Fk~Wp`q2LP0FtR_FV!?W+mMw
zuC}wr44LJz3l=riO$?P`g~U(ALol^t6ACI;P&y@hdU=#~%$?t3YJ^5s{I?W)&RmP?
z-VZaIZ^o|Ub=o(k+2$u*59w7N`jczpOT?-Ll5i~AnGNuTtNT7klxEiuq0o))JF(!l
zhdVhj_LDVwQ93A@v9p8{l5>2f9@9J7EYC-=a&NUf_k|lwVtWZ8){OTD=R~d&jBb;m
zb10=6>v?~9Yjw08Od1|2L}~9q2J(dbllq*aGzfmFhzX^~V7n5dcK!JPHT$09;0PIj
z|1I2oghaj;5i?XNK~}lrc|BNAVx;4IE6V<){tVtCJS`Snw)FPEAJa(L*lEk8ZtGax
zo%^)8y@dD9Kg{fMWZRf}TtXr0mE^~8bTmHGX8mn+@EKc7!J-T)XJMN1ZXWaSTF&D9
z#A%Z~G;8LX<+LL~s6T$VbM*u4V}ZxR;g6-TMU3MBw%5(#Oo^K8(vc2Gc0KA>VxE&j
z#aond?cM|WyZFNBs<MWH=-a5<J9&R)2Y2CkQnIzi_pf`hDK8B)ZtCI*eTFl?8uq|{
zj#Z_eS!dEU-;PfzeT>hV)2Gqc9|6DjNZ1=6+vW>D;PQDHg=>yv+!FE|Taz<KJeL31
zAkBQ!>Xed=@w1JSDx4K&*s$vwZ_&ZlgdhAha;3dD-hIdY6<G?i-XUc#LHqpANwDp|
z(PPDZ>Vbf<(D}esbIw0jsE$|J8wdZ3Wht~2d3h*FG2}G_Ar~NM8o^{nO~a=rUKz^K
zMO#G{RANx|0VHlOud#Aqm<Tmr7^#I)l^A$-W{pP2JTxx3sl)6b<Yd<g;5C|WM$<S*
zID75*F2{tZ(3dqa99q|sfRmZ#F|3m9`8^SNlzu)G{V-FB+i|ceA&`bA+WJGe+0!10
zM{`UlIyuD|o}0@Z%FOjKdX|1X^;P{95~4K{NAb(3o4a&_z)`;CBH2(|5sGEmgqrFh
zD;ZEpvNO~$ZAIna-|qPh=i)z%Hxau?dx1l3T*r>5(M>9#9G>>-r&^ALaN0a3FHU-_
z#Le}5XUCPr{S@L#qN1HCU(1gINK8V<p^-2CjJ?;ypgU%rV9#ypp=CA5VqiY@o%yG*
zoRRl%w@8MrD#rlvJV9U(L3^59)#WIf>;kQyFe7wYQ{VnhetNKwO&?t~;ihSsn5?;c
zux;Ix{mn3(#~NXIiI=6DLHInZye8<aG|4e}ysJTyDsi?o-Di_Ili^u>`EShpJvRni
zV{rmrb0shyHiF=D<re*663?}d+DC$^Ao@;Ctq^zkEPk!DbVdegZ((t@c$qa5*N`HR
zu-2&_&;5%`+D_l(pHfZ8RM*6#zt`sbh+KNQFTDy4=h%_O9o1_?Gj)7ts7P>_LY=FU
z&vtQayI;Nf;rGzTz+sM|_3+Ux>T9rPoHbDMbR3~9opJ<if0R;YDVNboel>rRk}t8d
zagzTrwW{G(d=-yKbw7P(sdf@=(sI3I#;8)`==qYa9BeeI<LAEN&(`<dM%IlZ^lRJg
zuYBt|SB7iZu_K-GkUpaz;R`I{CMXW+5i0~Ac^7{4+{=0LHu=Ai6VEDpbEK|!AW{iE
zSQ(dQ^%-l5%e6Tw?nXPb9}*Tm<q|xQfL4ieA4VU%N_>Es*^6PgK@G2>J?4}u8`HO*
zNdCq*Su7VJT#->_m#~S)>4<gKnKkyj$K@L>6VyKkaV4}IFrr`Spa$?Q)194RTM5!O
zm9!!q6d$oL(q*oQgx#rZsLe+teq%8ztXb?vPB<~%;2z&dj7U9y1vq%^7<(i>Hc$WH
z3O2Z3DsB|#4AiZLLi-T$TNtj`LQ(`MFXT9mr%Wce4T>l;g7Wa({Tf-k1(lkSWrjX5
zKiH8*L%;$2jZF+8GFTd!*@1sF6uHSTnVn2jHuX?QFo9S<?&?z1H2XD9j%Of2gG!O|
zUBq{U6SB7-%Y~-W$uWSE_VG64@mwv(i>R|$WFwll-n?U9_(^%`Q3=pwXq|<kZ|qfg
zDcYiXW-33WSr>*8WdlQ~vQKr|XYpFdJaO1IqiLJ;I8WQgS)~y}2g4v{FWai;5gUp>
zgr`xiZ&+QEl6BWOs_;E9fsop96Z@%~A@{g0&{bp1_2}3AwfRcyn+W*Zi-bg%nog=6
z3>UzP&!rii3(Q@@FINR5>nF)_7_tIr`YPUG^sMzo<>*kIq-xflyj!(NO>W3qyeTOM
zvpmRG@xW7%co3(Ho+}u7xnw#?R^(0m@$PhS{bLfV21HjTUBhhy-JshjJw7Kk-HbHY
z8o^2fjkWJDCqHc(?m@`Kq8VNWE-?DKnTPZob8J{Qu#LBbq~NZ9gRK%MCP`M=$fzyD
zAP;A7c;A3d?rg#r({H7$Spe&jxGVtV`u=RGE*aQm6A23b*%8pZ&K$j4?dUim#%UTj
zo*f?qWJr;TJr5@veOEHQ2FkuC&H&;cgkRC~&sXX~9byqCx{<l^UF-M91IqGfiAk0}
zBj9StPCMo6j)>}u?>@}KJ{-CU=dI=Y3VKo+H~k{(v`%lIzu%)>xx8CuAx73yD=ec-
z9w9fGw&zMUM{<E5Wv(H6a~l)aoP8Qia9kd}S}6`#9~Nu7M0HR;@e4OxLenX_Rg4M4
z)<2Mru$zx_)Xs<KIt5tutI$9f8Gy_P(M>$-SSqB|Ty5+#1%H*euD?MDjLvE9Ai$5*
z>q-k%1-WeVhkjfN{a8UgDK_%u-I5}Pf;cJ}CH%b9EA7~@7<KPaD$hdBnT)=jz`&W>
zIR$xCO=D_6|B#bD!-?HVGfkRWc35D6lYhM5ic16MIgWSy52|LokJ&!AnyhhaO{cUD
zksJMhVfKe5WklFfwQ7&_k(J}v;H8c1&N3sXFW$8YC(VFUWaRYR8sDl_D^I;b!cHDm
zi0pGSXR)%(+vV$Z4Zs`_iV@TYLtdr<1#Tf<NfmOf{teZiZ#`qgGrx2Fh_W@Wnh)CK
z^Puu{|5mz_@$eo`L;hL!yE##Bpkadu_%p^C#-r!+!>joHaep9z(0VMP>eDBR(B2Jb
zM`i+3{-XRZ2cXQIj53KBf3-9pK#i9#PEFA5w2A@<GK+W`IpA<U*iLKO9gQ1(vQDMg
zEP6nV=Q;gGh7kHZ+S<1A`FX9N(gLaxGiGIQ7bV~;fLK6!-RdFiPfJL+=l3>=@ol5h
zJDDHRLB`iMR-eN0yt^(|eEd`_SzTN_k0^S0`ODMoPjZ(9<OXU<6iTX8`>vFhQ@IVX
z=f)-;kV(fkLYRdTHBhP1T%NwU*v^FG#Fa5nAj7meJi#<#h9?^l&tXm)Eja1G5n0bT
zzHu~LUDVtMBnmlO{p@#{mnmVvi|@AQ5*EBPbjgVRbqH)-WG**&C`$>!DVZUd$6q`5
zLxTWPJlx(*5bKFo3<Ksq6=Pk*PNEM?065QOSkSM*{?T)vw<0+ZdA@4NrOg6A4VrcC
zd{Xn9xY*TD-Ot9HF_cm}N?t7(R!`knklmkJdnwXC^>A`fw%xN{x^g!fv9D*_%}I~p
z4Z0`s`<2d;phHI>7kw<Q4q!qKf0-fc?+qx<(G6TX(f$sx?^ro+?M3=c8q#2~AL}^C
zy_O}1Mr9aNB=QXbn6zfAts#;bKa<Unt)h;+uB^aE!7T(u#ekF#t5V;kN*MD0oILB}
z981&ZXv-CE%oB`_MaE@&{cBz9y{36<z`v}D=IyXw5`I!rMMWIg_>sziJ6DNq=PQeT
zH7i%3OGVLCiufM@f<S%0#oA>RwuFP1ntOZgEPrC>zUNC9%`eMYQ1M>*2B@t}sC-L(
z1K8C}uz2Tv1JJchpm2eH1I%@VY5F0v0a(#*vX^vD5nP|yMR>Ajm%*8vT>_@zFCz<g
zl}qsn>pa5UW<}pgIw_r{FD^hxg>jplQkh7rj2MYruo#Y`?~Od=IDX{bJ04u;ebp-^
z*LN}03|_WFzuN=kqFyX8loJ;g5C?u3#T>;p4)3Wv;HE&vgT==SGFVKiuL~$heu>Wx
zp|E`Ybx|yX6FyJILrOt}|C`OlCdpDgPHQ50CX>;-MQiksa$Q<YYQ(5D5$^+zlc-^+
zuVj5%8ZCC!9C@={=m7#}3Vk5sEAE`VIzIhJ%0eVRJmcx!-qG3V&e2)9kfB=GgVT16
z;Y3m22ZM|fHO)JQcb3p1w3gDUg?*v7IR+oX1{Q&+wLN|?h`ljBIz`UM>M)EyVh6_J
z_28p0ANyF7odXJ#fPIvKsy{kz<lK7X*wdj1!!BxNhp@eU=DU-&Vy36Zua26n>e*qn
ztl=|`z(HjPixS~MD0MoHfSJjMm8i<v|6*kNy<m~0umS0C9G2y-#vr?>Gt)UdJ~}=-
zKEn8H$2+I(J*RX)F@L4%c{%lRXK$ZuC>&}ear~ru89(*_S3K<juJT9pOz6bXvqIs~
zV!;0%N7h;7TB88i)nM$SKTu@gGT-U4i#{wQNp`o4m+IU}e%NGY&Y~eR{(dtnr_|E5
z5>Z}_agvr5x~R|2a-KU4t%H?QMxgMzKPI0BrYctE3N5mV-?V(jx@m;~bY0$YyHPO2
zFgY}3>64IED6v{tq)z4RHOi`L?&x%W<~*c}J!~otcL1sEyl5XEygl3Nob4Z)yBO`6
zjZoAjXx_a6G6LD2nT7?T(p)otSxl=Yp$a%lSw*Y4cKlpoh9y52$g<^^IcaNtos(nF
zZ?>8=AmYvAo4v=ImBX0S9On#aAni5>E;l+SEW#mu4+<@L+!pZOJIzL|*;>}h@F=no
z_0e<rjW>=Fz9(#w?8_&8si^2wDZ<H^@3aoF$h0wz#zF$;2nZMs{5JGs;E`J^PVow^
z@1VHitkdRBQGQ~x%Z*@n+PuJ0m$|hPix5;8NE|x1yQB5Yt$R_e9_mQGK6<NdudMLv
ziZGY97%6KDvvfL$78_QWW!M^Hk#eZgXqCz^*)VG?rus&BpuM;I_H?iFdhcxS=v~qX
z!kQWpHoX3#{StOJkT%mxzdv_YRLPu?s{9$VVU$^5CRFJM%))-Vy?6HZ_5RE38AzDs
z=p7>6kV|G~=(*$RBw?=;1uJ+~2_A}>4W9>!>pO!<6}8QPMp}?7Gy79~QV^PA#2i6W
z!2JOM9E5#@8vw3Gwnf1f9v7Stw+Kmp4Itehh=FWiQ@7OWWsfgsnX(JdSl7G?3oLSW
z)kQ7q*lv(ntlfjXoujuW25QjZ+xebYXd_5|Zv@xzT?fO1b~xZxr`NUj1A)UmK}8)A
zN!0(-6+ykp{#2`yu*mV&bH*NrB1WHE9%&KBS3c_uVoPWQt52i39mItp*Z;<}R!gt=
ztbydM<E%Qf)_-=rMYh2#2#Xs0o$bBv>>ur)b#`Bvj@ATWGIak09utVpIR-{pu7l;~
zXw74}sxBuCjjUEm7*CLXG2=Xf_WUZ3)a&O`Mydz-w2>;3K^+-+Df-CBOA<(*l109x
zY{bpde7cf_XPOSaf+{#IYUVVK6B{`Q>pVLweHR!FZ>gz@D4#h>LyVp|Ly}NfV7$$Y
zn9^CI#H^8x17mQ~AX1uU)2jj;Y2bPAofFF=aXD6aKv+s<4j|KPDzQ;=Y&>U3M6+a~
z{o$ZD&X$OZth7ATfWz3QKaB22R!Hh|TV~lOra=|Ulv4JGd8#_+DG6hLnAIys5sBG2
z<6!5%f9oJGIm=u?BZP9wZwA~hbM5u9h~|%kUjW?haC^W$Ks}U_3ebJ+-QNPjPX!^B
z7VyIvMjCY0rET~11*oE9Qd%+l{6pR%Ctnv^=**uTw7=!F%RwZ(<fZ!3Sfnp-O20>U
z8d+#ZcQhB_WX)X4bp}c>ppzo%Br$q>GZh-K4`!Hm=MHt<VB>Mt9W=PsT1Xo%F>CiY
z_&H_9fjV&VpFLAaS0!=+Grc?vql-B^GY8WxwjF-mIXx<!;|^fB=nXZR0K*Fyo7|JQ
z;kDPNX!mqt0c9hBrp!AsP)b(ZBV0Y@re)Fdlj@?tA~}8R0iTZ2^Q=Dp3v6z7n@z{X
zh_F3;giA;yjLZyMkZ~*&ZV~sj&tC3!#B+70<NUPkv>MH3vvp8LH3}`w<bp@Xot?Ae
z!~NZMr&G4Ua(jlJF3T}@xEmbzOv=AZu_cT7p;H@co!F^NU|mZ3XSwmrE`WzgD5Ks4
z=(iFbM^GCbEYkrdCK7dB1U0Wvd6H-#gj%BPz>85p*zcFz73Z$({D=qJIK-t8Rb}mm
ztzNS!XFgN(s<IarHNs+wIsez}Ebd?FY1s)TBAge=+a_Uf;{)xC8~#=3dBD4+cb=-e
zm^uB3Jo9*UWAmsySP)f1Te#plt+sF@thQhR8))95)>vg$AQyMi-{CKoFc%cov$NVN
zVTl!|$(Aq-dK^(4XNQ!7?&!a%7OXNAD#Q)pS5{M)UgC19IlW!D^wZ3}BJvjAD7;Ia
z{!4es>a1Pz(hYC@ae9+*x^vuS<WNb;{eh^w_QG*pmRMwC%-R=u+d{(M>HmN1y?JBX
z#?d(3f2B`>xNSl@)IsX-A=|Ai%Z{GPl72eUv}qo`AOT8<Nr1tjBTb&q{?4%{4ic0o
z*>24?776Um?#|B6&d$!^0Y$W1Sb1zvgkn0I>t!&RgTPCMxw)Q$xw%d-;eCGEqlEES
zI`8sj?TLpbY3lQew|kuak?UOs3EqDfr&qr#VFII(8=4n6T-1zwkq>?$%2eFo(7Rc$
zZQgvpccQ{rSQilH*p`=<TMicPzCGUEZOVIsl9A?0-g4L)Kz{gL%q3r8B%$8VU>|XG
zcKlA^fts|m`ZX}}=J4S3V0ZUTJ*=qGcfpJTQoxQEg(+(GCSgdHxiyw#^Rr^=$AiP4
zzB_xji{1(Ku=OKW(RJ>EGX-R8VzGlmqo77^dNHm*DZH8097p02TPnRnlEM38H%!K8
z@SQpr8642rTSTbvCx!64m>P{u7Afd7l!l<P@i?lC<N#2$VqhsMpy{h3o^;JJt-*yA
zu&qA!bM;H0^0BWpwxb%w)L4PvO$_bEM020}kGw$Wc9Wa1-~Bg8S*;}N{oZv$kL51m
zORvrw@&rdfx){Y%T-^QRfKT|_$GhF_gE!rizi%IPLx+?8iSwRv&RSj+j<aV6h>nWd
zq(}Iv_2XjEsZ!m1M0`Qw{HeXv3(|=Lsr104Dio$C_NhX0<&L|5>Xh3R?(ZG^tx|{a
z<20kf;-~CHIV+55E_o%NTlI7hl6U8{Y5!AacHsY>y^6zU%|EqKuKZJ*<;vSY5mbX`
z3{5Xz1IU)M-f7+mCJ<o(X8SfEf4{|iDYf2=)!1xm`YdyRL=j~pjMSYVEy>$7tekge
z`Am0(ox0Lr-uUIQie>_W`a_n2iyKW<r8;|m_wC{FPW3EUB7o_ZQg;5;JE*uGDjbwb
z_SnfBy0O-Qb<&((ZgaimJbjQem5w^mAO4D4Sm@c$hG<2%nBP?jRV%lQ$z`VQ5)Z<!
zZ^u#bqf|53x96$&K?VMe74ZhA<N6ltZ+7lChtae0>K;%qd>|amz!{C+&BP4NdXoPY
z`HxX*HU-1$v%US(y@T$~{y!g3B@-pxEF8E&Oc$oNSIgOOx<&0=l&s(zTPe5AvUY$T
zH}6HxY<WP%tA_X6sHf`wpza#p1A>ls|KHi(@$MVvh0|VbcUD)StO!Aw8%>ayp%{Jd
z5tc+jVIfzL{(0#@Q)dTnxA%_eikVDPw(@1^pT^htZ?pI^3+>khOKZD)g(lA|GJ`(P
z?EC|g5FC<t)OS<so&cD;D|H#pzA>q&fYnd;t?ALv&+LgaCk)3|F;Vzxh6y@3;xI|~
zIx&pUrPv>0DDQkQhP?g*qgF4e`}Aq5kdo2`W+}BISXSTE6Qyw1sx%^4p{puXTTR&I
zN>m~F0_9o}Z7hWMNztQe%?K!q&7$_gE}GvLN()W77w67>Rnoh>vz{_<v*!kR9~rah
zK=^7APByXm=ixyi>%rWKRCAAW@_`ePLoTA|LsDQ7g(Ai4X*ilu;9~!pA|;Trvhf&1
z5!7*au})KHFvn>O>)R~%m<kHBRxU~rRZ@z}Qd&6n&C&{R%dsnpn`-qQxzYX>Pj$m?
z(&5+E!v>$8xv<rwDs4V(P!Bpr9W{8omB)db2x%}TjF^jvX0!eBpo{rZwfWz>{k_-9
z2Kc+fLu-#JpK^381Oau1Nf(nTD%#VjLc`5EPZl&iQ|9AFd$Z*{DZ{5(Y7$1xa<k$T
zf9}0In1)qlnC^P+7=wheCA6Qn_o#N-rSfi?#)wY%vsL;It$`a~<JV?+fyAeMo6EEu
z>2%8$!P{@Pk52bMB0e~(sO|v*<LE{;QS=aIt7JLMZa6N*d}Zm`B=^ehKOh@LbYd#;
zmcy=yS?nilz^<Hw`xkEP_F<`T5?8tkVIxvB(|LV(xPM;~3F*zSBbcElVaF>Kz$41i
zF$U6vJ|;p@ad_}{?;TR^`|X1_CygJgv$v1q;1YmzLl+${DLsOq*wF_sh@GXe8DOIl
z4x0aQs1m63`^qfsB^QrotD4-5deJahPOsBkjFFj!`A9BJZuzC|0}e)jd}%;`v?y&8
zXU1+8J~V#K79SMD)=(b=$CS?7?6YWd@UmXV3-uq}W_J2xM};QBN&^}qa+Yv}J}ojx
z3asFB3~JHEtWVBwuoZsO`o@9({eO-Mu@tJ1D&a_!c}``V-^>6ifBKQb(qNTQ?J6Bv
zMaFXre}4|$0*?LamF_s-upj%Q2sSyKQBDq}k(DS9&;b6QV4SenmqEAj<P>uRE~CcF
z|Bp_)H-wKr{o8o+I}TlS2%cc<D2%e?{6@j$mM)z~|BF5S*PkBo;Qkdd(iT-Trb^f*
zUL|yaGx%v_yo0^zhu_Tt6EQ^1&P?9WszCtw&LRjbU}Ef%J(NeJIJY`nHip};q6cGB
z`Q`+Vy47;6&7ulpd_ey6fLlPZ!3V_dgY$nKyc5+}cfjn)cf0%hzt`L<tyTqkRTyjl
zLce4r@888+9XeH($j2A`y)L%xQ7b&t7+&hZ%*dO}P~YZo_>~H0ZVzrzh2T|TVe^CK
zB86b8<tV640xAj@nh8Zo_h&zyi2!KZ<_L0?6P{~h`C>T;F_0OiR9o_$Z<3Y2!cpk2
zP%iDRoPRTA`bv9zkBdaL7yGy>XU0w~%m~=`$YTn_cnFlyzI;z86R_`uG9bIUXp=*E
zIJDhrEj}RX(iw60U@P@rke0_*-SzgTe+OZb!q;w+4F<vW@3%q$UTm9C@NEt*;fYio
zM{(5mlf(fwD>WMWwxB;azwqLu8`zN{<>M$zEp#gf-#^I`X?A~BlJ9JNjID~ntOuea
z;x+a~+$}8wE=FUg7laD)OZ>)q2D2!Bi{#rD1|--bZsZ$<p2M@#qqEa{Lc<ZAGchhR
zMw;Lh1r&uocaN6yr<`X$;PX=<)w&9<F5^f$&hxUErNx>$2+3t&)P$cUg^N#^0<0@O
ztj>>1AnfcEt^}$4PFYz3f>Cfd{*VP*KQ5_xa<sd%_jYghja@VUxTNOp52wf5x|Zyp
z`M9j7BGhAkRN`?-jr}+I3RFWpuI!jqO?Rxi8v5Rs;2R&8ls-P%KK-ssFwN4&<61=V
z!^!^X(edF<u2K281O~i1K0G|lcT+sB)hKHR3L_OtgJSvDdk1g!jtk}aaY^|%+o#*P
z3Rqq|E~&70u(N;mX4kBP74+ls=_>0KKQ1Yc8*WK?eq0M}B0zN>f{a^Ttl;*NZmtqm
zFdmmw68mMo6}gR;x00(QTdA&8e3H{0Nicbp(!`2;-*2Dpb`Q4S@17iCrs6_br})V=
zk^@oFo-Hyc7&%FzSWobG+<wfqc{}PFZ_99;rO~+`=KC!S62#k*dj0-V0>TUCJ1nW!
z2XU0&mQ?lpUUptYm8eQ@%NmS4A7lpD#L#o5&>+0+%C{x8F|2J;*(SX$sfAKV$)t)}
z__m_b66~m1h$*JZDs|P{3DwX0{faib{I;sDQL&_MdBv2{r&bM=-<DOo7?%;fcGxMY
zgDK;y>)_jxN}T<>s**@mUaThbE|*tRnWKvpWvb-zisEhAO3cEMJFe;{zb%u$WHg%6
zYC4!v$qWp0RBg||u#Bqn$%XAYxkS1p_2)0h{^85#bN#5}T>DA?>9Z%#o^19vS9`1e
zroTQ|ch@#I`cF36tDPsE^>xqn+$Zje<kE21tJwzT+IaG0;}0V3M*i>W#%lWy?X~vC
z#>V=Sjm<VZ?{wDM@b_LvE<PD-9x*hD;>e_O)86L?_nCbD_|*!Y*prLJMGUCE<S#CQ
zY=(N1KeZQyHsnv8#pEIwq)rFlB`^Q<*TuziAOmNqhw7iD0XeslBuHu&77CN!7Z?BN
zyvN&FDl889>QXkzdWru_h90%_J_s*G+{DDWitx6!Ka6@kKbA*yfJSC8u+b%W-OK5#
z5{hx}c)?)clY1^3!*E!)!O%TVmKU+wfUdX)xSK3HOC$H%^T$xH?JS{P4ZL2WJLGR3
zFJo`?T{mau8St&k;P+(^tEEPW?pHtd1m5v8`}+5y{>p4^Ok4l>J-7Z>pRBF^VdH*=
zY=8dC*Z=M2{~{d?m#>rIauC*N1M#uBxn4Q{YflRNe{FTG^M(IEz~=?b;`1cE8T!eE
z4;#SsXc#8X;o-|i7il_vzOr(4b+vr8wj9OhEA3~`o~`_FvcGbQ;SeyU*(0f}eEF!o
zy!r@zy$ND4qNofPu|IhE=&wQr^jD%AB%odteL(tov{F&S&_w(d2~PJgS524}j{Ni@
z@?JiI?mu$i+!|rPz?YA*bg=XkOTSpL$NB1W=iBr8+;06Vg+JhXIuDq*{?}L6pR5<x
z|K`S%FYEsSJ}>?PQ>Yg4H?WAm(d*}%v(vXrPrvyKtRF14Wbg?TnqEFi{p%E0md;4X
z>sPQ2xzbru1AYOghR{z}VjlzT`B-zMpT#J)t&H3dW2z(;d%%{zdULpQ`j4Ys5CaEL
zFurk)&R*~D?KqE?R#pzTPxek$R^FVxao$Az*XRfF!~1>b`}O7Zm6cs1Q=-YuA~y+=
zCCqqDTun5%0wY|(dgAx>vX^>~oF5l|dbPOtLN-o}NB#)HLB!!LUZg=f^k1D}BBl3k
zn7P9jEBuIGkK@S8`YDQ-ujJx+v0^^uaso35*omVdpjc*jef8J;MbKYgtQf@w05=0h
z7;_?GF?jJz$KiNENzJI$|8bKje6gY*=t4;r)05px;Po$7<WF5-4Bd#r-Y|=MuePZP
z7?JTzHos!FFMp{X;vj5nB6@+Jy9y=zn$am0vbvgyW+0-=Xc%UrS8Fd;^bgsLgo-e6
zFkWz}8{s6N;}m+TXUKl8%I4-#KH>LYEiE~_ZvVm&k5$-tG_2&nL3nSBp(T1ghoJVo
zWyd*0UoOfA5nzU{*oZknXCBUx&26G1rixK5QkX>9(8H1m!^bc+j*9_`#&G~mEZe;=
z@4NifpZFHD0$(e4ma0<p()YR(K?(1ss@LesRP~NWa9s&`^6Tr#uan8Iy(zV)ybmY8
z#*y{Y;gr|-9UW$SQ_A&jrj!AeLK^fgSzu2o$pLsNA$hLk(vq$*&SD@E{}sD{3?1<s
z-Mui;w5xI?=!6^uP1chu)>x$oHu3vu+reBV;>!Dx#eVnqULS8C|0BnG-PnC~ifd{_
zj9F#-O>{c*B5#HD_#P8HB2$V-)Kvo4hb0W_Y7k|i=eTS_@FK%BdGtDqwcxCTB;4{I
zDRu!%yO#thV2`T3S_ZC(U6eznF{%OIgwYswE6u5RXea+TI6Q)_&mJn`Qr1xH*L+AU
zRyl!vJxsS=Agf?5g{GR&dCY?l<5S&b!}bUF3a;?#>vO2SQUEX86+`J5JBH|c>V}>h
zd(QUJp4ETVU9vqy3B!yXMQ=QR@QQu-UP!`GoJ>97uUOQqjz}KiL*W?65B$`RUokQ3
z-{QLkfq|Cv>=r%l-?_(}r+iwUKklf}%0`blzh~4PJLB%L(eH73$AK#+(N7zHd8*0n
zr?)4b&vg7sRG5u)=1DmZGGsksmVWMYxD}mM8&PLEJ3oj*%sP7yEdBbvdAbRKxRowW
zHwO<&-Lnq)<52g<M49Zd!qD~ShqEFC|2mD`&kF6yL!<qptW7;$@4sxk$%7C0KY$m7
z|M^>%Ezm0^Mco1S;gFwFhzg5T@GLpYs0*|f4ZZO#h%E;K4z)8uDRd+BuWr)~8v^v=
z!DrCWBR%j`mdpfn<F1nlAnHu!-0>*%Y_s98y#IHjBSyfNHT5uSDz|x-DO6|LMvcxl
z9#hiqZok(wM_|;IJ;Zm3z+cO;mgkC5z-)Eax#;(zvHMFl(*m#H6VW(**v+|2cRkxW
zo#Wy9HdNJkES)|5ByrFE<{f0SIZosqW)aK2dO;6ua-~NvF5TF?bwKZj7c27NmR6Z_
zBertFR!VEdx~)sM?-S%LA6>PYn)WxlCp*V`N2hy-2ez`kpdp!{-Ci^YeTr8GInr#H
z6RI;!i+d73F#(hdi%#zb%Ap%$7OJ$@z9(*6mK_J1mGC?PkAh%e9LXdLQdk=|j!-f>
z4R-)gopF>TxSPNi@>d4d;>UgHT9c#MaIRDV4cd0Y8!`>R?@{k(z>m7#n1F-+EI8mk
zW9t+3;iCDZ<@j#@qBj15(1HEpJiQPYW7|_iQQ-L@Cw5FD0S^0(nLsAbCfTRDFp&5>
zqtxQjpxB*~0X&}6-4=sQFC_B)>%LFq%ht82IotBu4MyQ#kB31YsPe`!sU(e@U;S8)
zi%?N#;%udU`2>pUY8vYivFLG<^)J+bY)An5o5bn42@KTpVsyz4=#m{1dXF(jXbGB2
zRo@&$k8?LBCgw!4cYJc`4l~&SUJHb@-i=eeCnypEW}E0mH?2D`0$_)Dc@HgJ!I}ef
zbnvc`vQuKv<spy~yjENRy{F(|c%H#xeEDeS5g|4jG?>4N9e_D2%n=rURcCsw8;8$z
zcIP<@0PD4X;a&z5eOEwId4SM(W76<Nta2BLD)rCtUi=orsWETjrlf+Q!bTgO`-y>b
z>?w{934M%C<WZIkZ;Jc_O>ScJ3iMzqi~Y@!Kdz6d)WEYN`Dp}r3E3GR`y#CcV$83}
z-S(VpjDYd=5U7GD27iI1PwN=Y9mcS*fjwP-MjD6}6u0PkzEwOj+!VG-v`$6rm32+b
zKQwhVh9W@L7$Kr%7SyOyJa!W5po@6<<I#v*wfl;JQx9`1&t&q<%BVvJQR>qYlhZYs
zNJj3BLjm>7F{%|QgMss=h&vx}N7wMZ5czM3Sz@c<B<?7H?G2_N;swcvZkWD0INRSx
z_9cF{3=*zO{5@R0zOkFbRIB}bic}_@PsV;f7zBWEq^3IS5vQ_hF|ut~f&W!3x+%*I
z*wx7O6eyMEj2vJxj2LUordS#ik*{}{swoH~$K|0!{eBiF%cg8zkjyKD<|%iG)pGN-
zh1OP)7;2Teu2m|jRm!UetF#!d_CnFHq@=c*!ufQNq#d`&kP8}NErZgoD~y@4goPO<
zY{^)nu_O{iyl#*}&i5%QF|Gzti~%A$BzAh8X1KHzzZm+LKrYB2VMFf)=~V!KfhZd#
zvdeEmLa|a!s5L4&>0szh%&Un;T7oAJBEZxYu%+kvm<dP50GGfpNK)%)3oq~)TNL97
zknKu;=<dJkQl#xiIXw&DZ}dtpgnaRaf%l<xpVzeE=*0t6jxZPP_t>i-^`R}I`fZAi
zUdoPX&QdqVINfx>U_r?O$MH{dl2e()*l3eI82)GXr*c?QkVFC@>$-j%IbP(mlmmM}
z-~k^i_NEdd_7hJvTI3Eqt9pzQfMhT)RurZzwU#-t^=5pQ=`PImCqLB{U2S2y%-FA*
z>%znsVsLsDl~1|_OcFq0(kFrLEaxyd=c)eZ?(t#GnZ}~C&fuOpQs<>eIi)a<$RJ&y
zapS<vA1M&;F%g`{#LOHa(6~ZgNc@%vTm|O`l4r*;o<~v}m{p4fR6g7G&~z1}MfLpj
z%J)N%U(*N<VkBSS`3yzQ^dd@1maa<{v6kAv9E~1<uwrNO2y8ladkH0IsK5|Zvou)q
zU+<@sCa)$LaA9-Br~UaYdE5?hQaDS&k`OKX4#)FoaO94rA#$p7mz)(}h_dk3(umo;
z#MK_aVVe%MBrf!PKeVT%|4SCc(8OinvRSjTYL*{-;D>-o?$<K=E<AemLfnA8vYw*{
z20zBB3qRQS7VoC)9_G<SV6bY43Q5RNihGB~(!2>ukVyC7@L)Egq8hVO%$tZ1l%Zl|
z3fZ8z6%O;;#4SkN=p)EnwV>t4MoMN$#VF|7w3nq38rk!CY~+S6p7cdjn=6bsBgIra
zY-2X{5M<CG?q?$e-^U<~COm_rK*mLU$@)UJpa24sBxSpMQfhYRiOPErWR7<aw(ANq
zSoWR-8OhZ!U%G2CFJ@RUhu5_P51Tt&xX2tFe=vX}B%Ntv|6CnDEP&`fWo#|T(ntf3
z{fRi&Hfks6023NVdeLBxQ&qF3YPrVXMmbuiP7n?pQ_{VQbW6_*=bRdviLz64$<>Ra
z(91gL8I-)Xa$Tvi1URA4vh=*-G^fQETkC4+TWj3wSQ@0&lJz%HlR6PU&|Oq9mTIpZ
zn>fmqHBzZx_j;-VnI;-#M{Hzjjh&&*lQMyOLnmqYgY-?(0qOufN~j^=1*jj5gYzH+
zsq+(<m@<{kTqXvV<amfTjPsZwTTbV>D0HZxhAUVE^UXoHgfe!E7+DI8VVr8o8nu4d
zV;*?^4$B@2pZ?`~wsZiKhWBj|1%-}Eoq)sz93!&x3&5#J0E<15Ohs}4=qkaXI*p{s
z`#tU1H%NB=aG-8sLucRyLsVm`vC56ViV&xeU_rsTYH2qS-r%+fn}{j_4xQ+X)+yD%
zY4`p14@Ng;xw0!$Qkv^R(WO$&+0<Ekq15S2nUZsY>ihNzZ(G$Zz?CSx>mYD)rvqa@
z$E1W|A})odjGB!AYH$(tRj!pQzNCV${A}zdi4Yu$yp3kk#Xgc^b`<<7SqzSH1jp)8
zHWKni5MtyMc_XI5DK@XJOe+YaB~%4C2iJMP=G7vDcluEbYjzxk7+gYJ63MG7<3e$=
z=37ngWSXxt)e|-AhVT<c7J;^$UPd{k#(s>>!HE%)3`dIr$TH!S@0y6MDRz}_AthI}
zCWl8^lp_ELUP;NYh`GYC9&im3<80j^l4}|GE@hD%u)yazPsj?zw`oFWr^UX@n_hG_
z#&NCyhgJ97A5oqV>WB(dh%DEMTEWt<oy4z!G)RJ9afE_DNm@lkZWOnWBCdSc;M7qP
z6J=uYQRKmSfq0l2lu*XOf;{(QlHU5prHlXoWEe!=QsQGZ#PbA;Y<-k9&L?tbGfZAw
zXw>(YRH8}GQ8|;gpvNp>ytoEdEh|gakU*r)VfDztViY<C1ICQTn0oE`pVLPxa7gkQ
zznW=#Q9si$Eh0W2VPDiWJ&qkWf7fF@Pv*CnCu(sRj(bIV$(uA2fAwH>;8~81$_#a;
zviafFDB&3bH%i28Dn`f4$(GMseCniYFYN5JdsS+*@-3j?1bF)p&Vs;=l;W3+g)5OF
z&IrtQOEHwUw&tPR<1DR}v#`z&wt66P1G5Hu%cZN_7$zDj>6PqUxh#*8c~tC=DGm!~
z5*$X+hishCqISn<F(<DfwW2}R7~VLrO~yBR7s#24sFlzn1qg;t!|;^2n(3WB3Ii4W
zl0~Wi>OE<|Xr1oq02Iz^IN!Zk;UZ4s{od=Qb&rd<R`w-E_s|Zf)LFsaG1+=WawWE5
z8Dv9g)Il3cO{}+yWRiF6kSi@=#Y~Kc&=M~HxWt=-&8L7n+ILt}Yb9k?Lu7Z!**``w
zeB?0l$Qw@Pq(b&#F^ZQ}-4PJ~&Js>csf}!<bXT&<N=GDb{F0H74y8^@jjC#k;D{e;
z#8MR?lcCTK0A=wG#sNB0WG3KAZK|6{y+go@g%&RE8BE^T!Vvn3FksAYG1sW3c-_8v
zyTnuLH2mfBWrE&;hY}&im4w=HNTHT2_kwUaSVpP9xR5a3b(UFT$X!Y_C9M`ER2;bK
zBOgoMA#e?fti-tI-tu-FV}vtbg5#+j`sV?N;iN8%G<F=me-A$RIl~19BZZyA1e5kJ
zJTE(RzWD0L9n4tq<6;J<53<${VSq4+!PcU1eE@3tas4Ot6DX(UFa>QbF2cXgOGlW9
zQq|_xq7;B6I_deNmh-iM{WZZr{@rp`q5L9)13>GlvStS6q^$C!PzL}G*8>PhFO2{b
zv2Qtn^MVHfU%&mf>HK?fL3AA+lEVhHxPYVT`~)>%pbOS0WGMi<U~xEVlbdWg4Yq`B
zel3d^hqbWs*m)B<NdzOgKq8=nn=HS)yzD$)S^WK{pU~UEnyFW{wXLRaP=&lnJGl}N
zv(X6eYq1*V_10H(ni)t>A)CiSgFUw|myDKYg-k#juP$Ha8RB?u>WZz!u#}tJFey$7
z;}*Uv)|lmL8B86^BuhX`9yaiV9AykE+9!K$;A$7y$PJfJc$CV55gapcOORXd|ELW%
z&>jc!0s~LhI?1|etTMs73DdO$fwVLmV`ht2`7AU{=&%u6%8o9Dfee36A`W5r=+*z6
z93D6<DX73w;_}2e(4m+hnJ;b`o*^bc@ptM}qh$`_bD@99+2zS_*vn$gn*3|OpBdmC
zSpR9tx*K7uMJ^JI)N_Sh8<i3<vmkV$b?XF@b8>2@f=u(c%GOS%>_e$LffGQ81jx5B
zMcNbS00i^t7f4*$<?y|2LbtdgVxMe8o%6A<8~S95EmXv8YxintiE}+H5hHoAA|6{;
zto9^Xnax^qGDdnbkPq;8S3ML1&>+?7$0>%RZH$++e4T{=#>i0k#!q3Pg(3wVh%f!&
zjkGKiP4#3`sz*cp#OTsRWtX(*OkKhrunLCM3&v#NUc>ts`%Wn5AZ+F$(=JT%x!G;H
zN00q}bRLq`i=tXyuaqf=9BR%(5CA->kygYhrkif(>7);`E}GfXF+3Vs41lEUo{z<(
z@TANr+43Fl9)bJ<Ox?qZbIdJ5TNUYldhF{!rW{b5rU_tWA$8bTESPoWgV{IVFu1IH
z@Up!sCmy{_jTYCb(q>r>HpLyM%FV{GJx3Jxd|~FV<zhnv#Tqj=F|?e#WbB)n(@O2#
z^l>YP52Zdt4jLpQp)6YBi>Aq@>st=Y$c5sN;>5tGSP#f&=%W!8GSf7QhIus<B9lWZ
zqTXaXl-Trwqz|0Q4g1;-EH%Akib`d1+@T+msmbHO%?#K{E{IIoC_{D!cA*f*gWQQp
zn|y<MhBgq*cVZl@9rtue)D2U)?y3#YOX4dk2hbFa&dzP$&(pJmy@Pip;Uo`3Jk7)5
zld(qOb7@C)$XJ2z;>beT_7+h%C0KC;8)xVXM-inXG%V~A(!R)Thy9^QEWooFYRB-}
zA9e@=XIWwwn71o4WB5xboW)tV1P7Cx`+{EOyu}*zNynu&Al#^zj6|Sn6Vg5z0)p)<
zM|O?Fs0TuZv{-N^`}%5tVmIJ}xY!V+<u}IDweO9HRw<=!(5JH^ojjRh3qdcTAQpM_
zYQ<0cD<ak<_d|ud)VCe-tgm|JhnGPdkqX-w7Cd!hVQ)iOR$r{}qn0vhqM3!QAVr%f
zAs*87MX`tONwgab{Xt6iox%A9x&LX)d|UpRj7Z57iQV_;th$)zYeCM3lb0P})oXFy
z79W->;4BvE*+HFl#e$cH`{e#62|O0mDs({t0XU%{kqG$_MY`KJDrPH;z6LoGKoTV;
z_mLeJczNKm(7akInS{RnR-~i+I;v?xEm(JeJPvd5kS{g%McVf|;>1aybh@0fCA=Lf
zb|BUv*;&I7N3Nj;!m}&6YE#+gjPJ1v?2MTyX*IAa^0E*9O-?}qT|(z-LoK6}G5~Se
z?ZGBtC}Z&1<AMT4T7o)_;=RJ1ZxtjK2%2@b&=mS8|Jud<q=kb>@$~bGy8S$jGrtwG
z8sm8z`^s-|YBZhF$fu8*o)t$<gjtR|(Q*!scv^&{S;9#q^lNhd+1eLm1K$<WQQ|ak
zbSnecQv7C%we_N)&}gZG%B1LN%W1FDxxT&G5*s$s0NsxHgW$T!4Dtpj*8~{5SAqly
z48xrq^H}Z4_D<ggk-<;w9eVHm(f;oH-GkF@4A9^l9G>o;l!bqeT`xcwe0Eb?t>|MW
z)#fkyILk8Be=hWV(u+S#3p?L2!_L20vA~f`(4*TWd6CSD0HJ6|vL4;pyDY^ec;=>C
zc#P4H+C2)}OdbI!HuiB5(iGDxEG9-Vh~0CAHF19xqwRAa)w#m}j!Lfd<x<zQLLyh;
zBK8IkfDwv3A+LwUrY+7Of?hc4eT&@*w1`AU_#V*N^iBfD8{esuih18BDq3vPA-8^H
z%3~$a@VQxEs*wxc{SG=H<%Cx*9u}uUX*QTa;b3?iWT)k2eV<OL1VvorN)F8jP?&$Q
zyD?QFyd`*omr%GEhzt^`(z=S`54>pTE}e9=v44)<qjb%Q5J`LmnOvBj39F3|eN_3>
z-H(zqS=O5z%L;s%0s_djPb!TiVKfYj5ksSk<>XoHgvZQg>tY3ok;)T1+53uf$`Eh5
zc;logRd$%bpp#q&<iZwPK6L}%iUWnU(s?LI9-^<Tp1^~aFjv5nA`6*8lHe8bd2s3H
zZj9t;Lev0a4I%?mn~^1Iab`WzzXoi!kFJ=awmNZp{=gLhRInp><-s?Sz}w*v%L5aJ
zG%lU0Pzr@2Ugm4!k@^XvAav=w=n8pc3lGKQxbA)sM-M+tjN0V`3<h-bsBd?*i8lfQ
z5)@%$P)S$RQ7DC7*%H|c6^W1z#$FvQp>pO5N{EFsIcfFS?O~!?FR#!`rOmotmV&P)
zr9L7Eg7z39HKgdAT1=cp)EXv?poY!3sx+vZ@6eN*uzvyPAYpVyEsP>zJ8Pg$C>cmv
z$w(2-JtAwj1d|Z~WmBk=d>9tFsz%BaPMBiJb&|-Hk)hg`MwZ(RS-Ze08FGcg>oY|g
z0s$nA$3Bvx;Y9#HXw{l3v$ENLuN>K6nlLI4`9eFrI0E?&C9DXIvfSZPBGidl_(2{s
z_!<(QFeMA~IotbEUHqPtqm`lHqU<tsQ{^~21Q%&eOaNV7$^cIWSt=`WYFAblY+jgV
z5e1++f)}ho29#2B<fkrO6b*Psh!rMjHh;;2{s&g1n(t)SOw1s1(cBTl&?qMJq&^%v
z<Z%Q0_MFL9sB*Ommyay(<yxztZ85b;;Q8{p!9}7tAByqY=RE|hs?zL}#Ate-^ZYq@
z&nC>iMlevy0K^DY2Xtlfp!$a9Z+7=cE6aKx6_^CIlo^U^0|f>aKjwa>lvz$A3z18=
zR3BVAK*FqYLuy&07F-m;+@*BtFj{X4RVu8o9vr}>5ett(lLF$DpAor%sF5&v@YZ5;
z2@iF&qaY;asp*7wYucl8^JLiT%3V+1^Ep)-s}pmWwChP9GN?_drV<n{sk?|tIY027
z_8YBya4|Xo<^V@hBn7#&j9^bAwoyP5M}2CMr)f(DVKqu5LhJLw9r7}RwI8M+ayOoq
z9;e(G*kS6E-CdJF#muL8H&WqmLLc5GX*p*oG4`(x5amS~%vL@?6pc@KOEGG-{+yH_
zzCd-JjfnBJ*s9L6!#uH!fC+TsA~@t8w8*GOBESq^)bljZ;YDjiZRRd7s3_^xHCbH%
z+wH9WneDhx<(6?|G&YzK;a6qrxE&E4nGYFY?Atar)O#ex#OAv4ddY-f)|LvsFvo5U
z-AB1PV0hl8(m8#Ad|agvXzGw6#K^3Ek@>o(4TX*F7%`S`JQkv~l2XY_!s=IF&mWy0
zzu9~HHs{Ass;JVdoiD#($>HvVx=s=l&wW!y7FsgYA)*AU3`hCAppFuz%^u&K-b5KJ
z7~c0_5LqmQS%cfacvKJr%<&A9O}L?beux|_xp4U=pB5y-jLYO2jXr`U*Qptmflc}q
zTp}Hm$ApelztUV6ZyPL0Dm<@J-cP<!YUR|vr7?2!skffXRYDTDa6ihkaNK7j%+eZN
z${kWT^3r-n8OZYGCBJ219F^<D<FZa{AMLfIC54J1WJjfl2CV)7n&QdBIVxU6x}jxN
zZ!=n(U9m>o?_Y$%ZJDp;LLZN;t_bx2FY?$f3r___G1`MeHgB%;Yzcj#?4A&q_NMf#
zLSngyQTgb?(~|PeGZl^D9e}0~l%A*~Lw2TdaLG6!b5Gw(wEG)|8H`Yr5G*eCm()EV
z?C&Z6LKO@nG`Be!?huG9IzWoO1m^}eq8M0$FSii9GE*Xf*!IQsDA5$HI?267%FuiQ
zM0bc2LNcTFgd_f?ZUKl1noC{|w^ehzN@B%v3(+a&UnGK3@Qx%C);)>Fw-mn>q6bp)
z({jFG%sfzhU*y@EPwSH&#dHwIJq8BMk9`VgTPfL6=p4EnEEA~iMHGEV@<x-(7zrpq
z_cXIq(R`AnZb4$Cn9{T~5Z3!{Kzq3k@F8Z9PL9HHOLH7kxpEgYpyfjaliY?KV1!-j
zIPZhk>IAm#{0)AQOfB?0YHdlG4e5*kZLTkYq-f|Tb6u2&MRAhhP=PK|M07>R*^*lh
zGI<mL$}c!hiCg_Se6*6^O;dVvJsLg_3Xle9Dgm=T9ID^|GFq|G<$Tb!lmurGzK6-n
zM{AGhi$8w(sQn05{FV5sO5<DDy|7eBt@WWC?Orb*;SDC|?5-%<QAO#Jb7I-GDzGW)
zybPas=%nR4&zj90F&}vGmJh7JPfkDv0os^Qfb571F8D@!fJ(D7FVbG*-AwBm9TgC>
zFCRU9#7COyJ`5-H@5iS<P3~SLa@v~{pyEMgtpls%X$z>o<vd+&InUr(yA2Q9om)4$
z)|uESw9<YCEp$52W@lsbw#{s=PiSVX1I?^Gg%;LvcpL4`tx?!{I-#A-b!g|wCN%Ms
zP`BDyzimT+ib;6u;PkXN*07Z)gum7GTTc<-V?tZd6U5%?6KG{^gOIrT?AG(tMkYL=
zwGDg^6v%&`@Smsn=hm2<(qem+8)<jM-}O7RW~}ljk6!Jg1W?`d)ShkLW!&pipm)T$
zJL}@_Q~tZwxl5~0r?k2*T3r{dZHU%4?}GJLjHyXHQ#AWb5LR2wcIQsSwKjbYC3!s)
zEp`;bH|{jQo6{zEP0lcIK7n+F_4b`6Ss-mPClo7qQ#AQhp&o^@JFvxR2=B<bM)oMa
zJ$ZJQ@!DJWciF@>J+*_KyNq8<+!S^#rX3q%o-L;TE-llvO=R5fnQ^~o#{Hfd_g&_C
zN{hg_-wVe5UNCO{yLp#Z*Qd1F5v_JaYn`V8&3BsDr&C&87p<;~);0v5H}5p3%gf8v
zZI->PPoziFiS&qxbXO7S5fka|U5Ipd3X$$Ik?t~)PHzfieN(!JpLA{#<`ye-F2f&_
z44sPRZK67#$k1MJ=XBpW*nYoza<sj(TbkkC@FQfOV>)uG5DKB~jEwhczPP3l?qm$D
zt~boG<9yHdnKxr!-{tA6YBo~_)S*!V8#xEDBYm${(8-H+Z>gJRQc>=Qc=_PT@CeE%
zFTT<yc!sTByj1#l$to>$#QLpOiGytjhlySm3_bgH-J#fk0=F$0ioy-?HglUDX?<2h
z8J*XSDK&~=LYVImmiakXCJ4Pi#9j-PRi5%jHExRuls4NV{TJ!2SLwnOEnTwlSeo>)
z)?4PbP?vL_^6j^r2Xru6a#q7IH&XvYzDLW#%~R`&4ExSjagcdL&^0*ur&s8xk`hBm
z`$h5U!*Fwxk<FRbSxJN5l=5aX?|4Hukqq8(PSNIp$XP^RLHk3gyw4ea`s1j7Vf2xH
zSDLyqX2gv~ga{v^AIc#16EcLrJ6D$?@_IKwU_syLXin`@8uzadAo9|WF;0;Y8j{08
znha3(Rg~48HwrZ>6Ly&aBTHAO!94Az^T8vJ*^dX2EN#6V(I@HnR~aKQA0M$2A!5F2
zpd&YvMPy{KVm8;i=|n_a-2xGN0~yPMTz=RCHwasjQrNzVj4qINiJTO$v@AqMrDG%e
z^9bYImI6ZIKhUy9qznK~huDC%VN8<L#>-b^H^SzF7(HTO1dK&?!=V9yd(mBh8_H;$
z!lZ`73dkYOVWydvFVlMJM!jJ`*==e~eR<VMw18&Tb6*ezw@>QOlF=b)DTeNBdEZD!
znv9ifZSzB~=l8PnS&&K<K0Yc%H%bv`yU;rKi@-r(DCcY756gdi^l>jfKK6$0uQy#Z
zFrEd4s?x{DBX(tpyYlhZvf&{$3py>ws*fZ)?2dZAhZ49%4SRDfX{wZdkCtN)eq@ja
zjH`O1VnR&!F#DJjm4zdBJkuOejgL?7vL7FxbByC?CMw7BA0L0*i%$c4-C@u>@Aqd#
zqNw%}aVhQXu{TSyHzrQaj5Y-e{0G5n7G4FRcN^4l%};|;myG^=e6C4~i&;=AqO5#u
zOy+{Ue7r@^{wcFyP<pg{^rfD6tr4sIBT-^{G1KM^#XkiUUX1UCKOIMbZKn)RB@B(p
zDq}Myk0ad-!eE>Y=RF6*nRo#d{}kq6m`s?12W18sI!VmL47kR}=clqCAD^LKy2JB$
z#(iGa`S^I$J^A>Uyyg?yEQl0UK0Y2*M?NANjrbKcFjx}V;(?I2jjb+4;nI~GW7sB#
zap<iG4(W*;gc!p^yP)DnRN%|Tft$KR;qxYZ%{erWvfHP|Cpp@S@t1ho3sgx5`?ZPQ
z_Mr#^NLJ%giOz&zrcac^LuWSWSzS<7l&dxkDAT-&p*Kc^N_J}y2sRpCHfC!cbmt}Y
zc4ATXqv@ESZA@Eyn^+Jzl9&{Lp=rg#i~YXu`Q*5)r!n@2u86F4yI2r2bf*v$Pum)3
zvf@Mr!r@JY^-#GRrmUoSV68?#5$Y)kp3h}OF}bP=;V8#(fRT5tpp?2x{4nt`5Q|Z2
zs}QK8ycKIj%03I^8c`Ffn)!6`x>Y%1g_wEs4(%RUlYtZ+CyQK!15nSMe|#Ur7!W{u
z!<bH>!|0q-;dPq4fw@K~075Xisbwj7ud_9SnOuca2<B34D$f*?-ekzq{xBGieNTFA
zi3C9_<+)pI;#9zl5kSlAQPb%BZ>Qay_cSY8keahxFcB$EvJ9a<_i27FP*{jN0;tA_
zBIJ@R)1E3Q(ns`Kn1RSZ!5qU$8m8DB!gfYhpmf7KM`!FYh*1mJDJvFHB}Cu@C^0AL
zah%2H^^Sw8^hevtkFa&jyF<wMdguY0+9Dz7>zKYf4T0vSnf6u`^!&biBMPGHxS!9M
zoOj^%VBz0%afXXt-1=R0)?}m@&rjW8sM9)PXK4h}dqDl|BAw)#*Ds0&OF8Qevu&!z
zZO@H~n<%Q!$;*&e!p`rBpC73hQ8K+6&x*;-Y57w$X$pyf{k4=^GH@TiayXoa2<{=m
zc2aO0WzCk2P!on>5h#R-0!A+T=u=GrK*UBZ`Xd+sv+FBh>lfH9ZjpT+^(hI(e~S68
zgUmErB5J4z4=+<(eemPZA3AUl%0xJ8A`mlwJP(8-$W9bgUGFl$3^#WaR*<t|0Ch}|
zB$bSKqRS`uRk?tSVNfVB^!^X}0qP;GQ26Lo+`s7$eG_X69B|h0#Z7|V;Bs^Cfz*b4
zIqAD`U&l~pe2Dl$JlR&LX-Bblo1|beo<48m=DPO0-Q1E{{-RKaq+M#$2{wOXG&+m{
zq0c#d{sfL8Hk&H8Rrb1?bJ=k&i|s52lN&PZOK0Gf@_Zz1D21h^ij_I1W2?bd>z-aq
z81BRIt8E<}g~Q8Yh!r|rBt|R{T=T+6mVyZ>aPUB|Cu0r3qcp%EK1Q}d8G@*!w7CW|
z&LY5}kYZo21yPm*?g$%1atW7J%(zgl0tn*VC<;7SgS<iYv-nb2JtwJ)*9=IFK8}{w
zI_A`6khsVws*kY>5dxcw44hS&u@tR@OGd?Uibd~&(a84#*NEPa(Gf6gnZPVrkJ}Bv
z+7-(+F&3#Q?NNEn>}mEAr@bmCu-2L`6$A?J?zEb#n=qDgown+%<2fHPjO;wMSF};*
z!F8d#q?wbNPt&BG`a|KsFhoy~>Zq>x5gNT#iK*~}97TF&j3fslmkcx`Vt7(>wBfs2
zHkqtPO|i<I6{)gU_E2w;IHfYeB7QDpkdTN1nX(H5F$Y+ou7`$ve7?)aC6Y>JBMXN^
zxJppu8rpH?)Y%W+VmQkV2gIt*C}2(uGK05SjE6l0*D~gqk!>4;j||2lpsH46Vhkfz
zrV8OMVz01d6RPoRsYvC_;d}|orXL4_HsnY=8nqKUDT2_Os&UImCkq4g6mU6gDJ_SF
zNI_1~JB|Kx6u@&_{V^xEzY6Ph!pPEv`$r+YuATuG<IXm%<ZxE+MPPhV63UEv1QVo1
ze@kOfr2If#WI&;0Q%;saSwD=Bs1E0_aNNKvnKENnyYz?ASR~xFZyL&O80I7>{XC0)
z?&m4GxjX3v9OuW$mdroZX1@P}XDO`9d-dLcq#o4Bi)>1Kr6={r6cGmF$=KqQ&WGR`
z;YP*c)43vaA(e^A80zY^yP*eL)|o-<`-U!ih$50|POG3MHETAv;g&a-*O22aoNkt4
zCFwyT<1l)WYu(@?suv<FZDDX2%w9M1voSFyAVuH4NGD@>tf+;>nwDO6eokvB^teC4
zXbgvVExH;jFZ9n{l`JZ^q{=#IEH8t5)x5~7z7?)<kykP)#oI&$FwVk|cO6nWTiNLB
ze00(bK;)zQURec*WOVakBEN9lupq2|+R|0-LD@e?R&u1>pmdjoT0K;;++dZ3QMiPs
z7G@`g>WAv8aHg;~yxn_yC_{G|FvVdA_M?2LLi}jr(i&uH9lxmb0H+-1Q-Ji0*k0y_
zj2y=kheIYmSaOu8fRCCQ0`82oQFF1f;}b4vLM|uMv!i9=$5MCxT#T0a^Mpv5=9yGT
z#wvdz9N03E24%omar+snmQdcK=(IXoE^XhpUnORkOLmq6p&EmKrMh?8oz=v@h<#h1
zn`zccTx}9J)923uVJ2`Yp3E}j5cAw97aYF5sQ|nWUeAeJ#J=}yGy|&;<>!8cn<JMv
z<}M}`7BA{hM*(N^0)3$J!Nd}bQ0dbB(jX-kMB%HglS44DI7CU(o4c<zK8LD)Q9`B*
zpMt@1V(B=}xULXwmDGU8Jq1nW1mGZ!bbqgxg2;u(8ftrKv-T!qA6sKXEnV9)t!Z1U
zPElwRJy0`Kh8H>hV8Cf7ogRmv#bgeHVRW@Z0aNjtc_E|R^m8)DJ&7{L>7byh7UE0A
z@}JX+#(l5$a>l}+@>>2Usvf0`I%Pf2@8saDmL$$~cs=A!^>i}5fHR~|xq|w>4$zHV
z!Q*(znYlE=tz|hgG4wDi1Vk`qR?mv@Sp7xoNKB#{P=Z`N_<Pv_ryh~#tb?4vs#+I+
z5K_-sP1h&(Z6i2wOaV4UHp{4{hETSA$IHQzb!N=3riuGhh0J26Y?ObH3C^1#&oIN%
zC^yRB&tOTc+R2Pfi;b=b1VM^9)C^LR8aEhbv9Oj2O?{`*vbAuc#zXhUxCDZu!+98?
z7TggTz%g*Ij!%Db`eyg|xchc*fA`?fh)~a|EnJb7TW^m-tBqT99AQXY=u!Y{K$O2j
z=RulEyK}J;mNsE_(jwD3*iTe$7tSc*Py+WLQHgY0tTnli{?A5QSiVj&Hg=tFF12Dy
z2@P}-Wih3p^Ze&}C<0YP8c^2CIq~AFA9vntpKkxSX!wddFP%q%<|yalqphEQf|1*B
zG_D>@T{(l7n=bw!D>UcCR)2w>R%C)|FvMorQvJKkI)ERYz9XN>zGY=b;#*RO!Ezc-
zk(!0n4f<H(^!=C;A}aI3@*cXJ8ERy+03oz%%>~|kk>)}s91vSN@A@~gd$~MSEyopc
z<s-_5YAyiIs;HAnNjs}4%7@{3mj~Y+xz|N&Fa56i=2OtPc>yZa`-VwMQ8>|d`ZWdg
zyC^G4^ov^c69IP^WOx;$$Z1Y1v!>gmDElyL((sC$VL|2`9@##par)g+lOwvt_RUSj
z`qZZcs44FYafb*}1N(ekgmbRp(P6XHNJAPR@Zp;?T#?=o^G8|vtMm%Ur*A@oG2To$
zBZ7c(GZqOmaN~*5OrA~Sv@BruhSmL9m_`{zMC(%?TWK^`oBHQxqi8Pdnp9Y^<nL7r
zed}F98ZPT>2DT|9U3DX0H&XK9r!3E)ul5!+cS#*(BgxlKNqM>@{E+^KV&I8P3{+Wk
z4Qo{-TH-oY)^wsiPzr}-MpL@>Msr@a&vj#o$iff8nygqMGmrsqd00EBqoCk|8s`NA
z1zzPrdQ)!;r9c+U7;+n^@ut@H2`3V0YiWR50<1;7S@C@RqxPmwO(&Evytu0yFKhJ+
z%J(QaEuB-SqA)x`^QM`w)XzrcNP-u&`l4*A29B^TMinFvMnnhHCZjd+M3x^QLsVc&
zzOL0j(+oG_c&J!+DUE~kb3bO&8l*qR%iuJFft>1K?i@>(VUAwhre_zy`Nh%|+DlLB
zn=qG|q*14C41%zLKmDRs|EOfX2QZOG;t7b!<1FQ5rK2&71FdFIem13kJUWj+`u1M^
z>@z=+za8;K=Q3An9eaeb!Z=r7CM*<IDJ-YpC7X)0S}maB*>m|kiq41rG5}l|ubiXA
zp91SlKw#BB^hqA17Z+Vl#I3YpFR9Ug;k)Cmz*v`O_tC5GcDIk5qvJ!=87OdX50Bq(
zpBl~*4%SuJ=DLm-!J6`Amgf2GHC@aQ9@G~@j5-f$TXV}D=q);_VR&WkvgDbHj>UDj
z$kv4=Ob<6U4xyN(6wyt_cMBu!cEw;=PaXlUjJJdu-1HHUW>-3%sSv59W<-PIwMHa>
zke!Adla&&bDGOhaV?SutN|$V6Gi~N>U5lf8F%Y;>>rp+JCw<B7cNY4}Z_J;bWOJlM
z8X*CoP~)rb#sz}wD7RY$P41K<A67^hBqI?~Tck&}a^I4{dBOD1Vy*@Gsw@>KehQmI
z$+l3L1{5>O8EOr2-eQZ*G&)BYK;01(wL@W*F78^jSudHP8b?yP6rIpp1iM@i<BX@3
zL9+FoO64JM&WAgqUp~eNGVbDq*lhG)tp6#mg+0FTlfL`J-70312ew>3p~aWg!U>!k
zL;niT|KwQaNgbl#dO=B-O3Ujd>5Al!J+v7>7ieYwst8u*IG9MMWpi)S?4;Svk5kg|
z!x3=)gi>EUQF?Z{SN+5-33mGuK%<YlI04wYiJqTc`F>bt7SDrADADK0cQFI7rwr@J
z&8hUHy2m<TvDe$sSc@w9;rXmAPPFtFcs^Qyb3RYr{8b?n7z*AQONRgm4L&e=U`_}S
zXED_t(2+>>LrRQi3fk*=sx#!-ayALTI(2?}Oz&6<UAf~fUs9}4zR!M?g(>HU%_%lm
zW~>M}RJzbx4w@vPTW|17rU9WQmF~1vIu5eQWOD^FkCPclpqq-3K|^nhZ1L79&5$yL
zrvb9YU_>@gu27i?#f&7g;$hUlh`<7;vrlCXTj&mvp`T1}Akf3SFD|F`3p4V$qUNE5
z(|s<iZe;4|a#T7ivqOFZKQq^`jggJZ_L%&IQ+fgm$@SpmBNhF?;Bc~RizJf`6Gizk
z)09fjav24MYdQ23Na0Z9_dg_BU!fBX$x$*0eupQZoM$NeTc_xRRC0VmZ$v&`kxL@|
z3_SHc@qeN30eLOTUwFVXPF$veAKs5z8E-UjGq+D7j}b!^PRG^-P0(4%L;v)neYsW$
zaQRv-$yN<jTIs587`cv?<{#<B)XvSa_vG|;pWfbcBFrz8l$*MoRA`c1npacmhlR<O
zdblg|-d+lvVv6x#YqJejSMV-H=RV5Y9i`C0OAc3**X#S&oa0+)U&stk-mlDLH#$#Y
zMW{<>A()&Z_0uDmGu*mAjuS1}5g}o)UW;d5D`Ls43b{8i-+SY|gZtsP55Z@ZmF;xj
zTV=qSUud`Ry)}GpO;lC+-cI+qHH6H(@9m5}w+d3;_jVSaTg}&Wu!$0%+mFG4P^mi`
z1Hu}gGXL4P)YQ{%fwPaxtwbI(v2+l%rgbEGOYr(P#9w)m*LRz?Bz7mos}(;hcT?yB
zt={~KIJNGK_&)BT&s9QyM<F_D(XUzI&hhnblO69JMwf||IV(rta+4(L2c#raifQC=
zrlGFrxw<>j{*DYo##1-FCfG~r0y(Wz&rg%rML-n;N-^4s@p7sfvDw1g1?keE?wpA4
zT3#U1%1c{!8H&=OxU~}A5%Iw7&m(-Bqz37PF%t>NN_S4OSeFyp(Ld_5!gqq!STN3e
z$5Oa7?Sqp$)oIknQnyD^37v+^1BzV5z!uyXig;T&<4WbVYwfQni{bV9IA+gTo??(z
zq<JehhUT`JaV^O)r{_-u^^R_ZT6M@^P8ZX6Ap{pDi|{LF0{=5o?2@lD3R$&J_{Bw3
zW2sEJbg=SpvYtg=TT2BXQZVeBz7+&wTIx^|9Op+fMg)jih{mY)pTIoKI2ix6?8899
zObo=V_utM!tmRZ|(fnty5w<m67N?O}sK>5(r}J2uFgsTR?Oeb&n>Bos2)7%d&KEv7
z#^Mh~$~9mx%#w?8nUO=6nsT!`L6ZbLnE*u>2!4?Q&9?;-oU1Civ^z6XMXsYon!xoi
zi6)lr-Dv+UcCmyMCfnpIaHrORKn7{yNS7iLz4YMXoZzfq;_Tx|8fSeBqhMI?SD(X5
z==2a3hGb;d${kG<Kn$8dr?Q?eq7Ak%c6r~lz8m{-5aDH%5iZm@cjKOWZr-&+W!4vv
z#b2V+k*OTyykK%?EFk~B8{>_KrCN7oHHzZ5Yy_YXEt@T3m<e^u(kqSLCvW2@)SZa@
zCN0W1q@ZX|2JsmiSQp^J^A#N&gEg_SPi}UoY?IC#o<G2IqmNj(9J!2>vaif?V^UH=
z4U8}pV*UbJ6$qYphWT-Ey4^rc?<`AIRhqYDs~P#-T+(&f{4LdPARnwq-G=8U{WuuY
zjXW8|)>m`j%$8jBAfaxDdL8$p@r|^%rWV***4#FP@s$X{hy;XY<D#6oJY@hXUq_TF
zUD9avGW1FKHEbJj&k`o-0=A6RqS=#(o7gj9jRRpgfzh%2@dd1Zm>Tj#)oy6YYZdoE
zUzudf$o7Uv8wxLQS6)iv^_J+66?i5WDaVS)6@-^k%UB!)wn6bVXquqoS20Zg7RD`y
zUb>+VEgIT#*IS_iQthrYb9kAsHKooCgMm*scoYSm{qS<$Q04?n%abq|E+Ii;tQ!%P
zPkQHx0F`Je0}BWE6J8+ZLe2xFqO97F&O=l}nL5geD&QbFhf~VMopr!gbdkvo_J(d)
zD<96J2ItJ|X0WrKWN3Mh!s;!#*p4ukhAoq#J7}}H1Os!&oP<VXMi6FCQ_j4&F^7gD
zmtlwvyw3H!Kptl2W|6$bK`U{sXv+3;-OhEFmL!?%sW5Uw_x#>{RE12%%(@;ynKkYu
zQ_frk-5u&DMVc2b&QpY3i#PCb8f(L0CrnZ8AG0Kd>=>iBqbI3Khg3m9@3;5&4|jg(
zp6+b#e76fkq~eFzZyMC)5$tg>pqVF^e>IJmBgLb4KaQ{mh7Y$0NMhVo@F2~gDdy6#
zMOR>B?m6*stymz_E8_~uadt6)gF-67w>Cr>9Sth+h7c-|nVfTR*V!s4m1u(jq-L=x
zS5m+QzLkfK2n7b`A{u&gjpaW3p|9p?ug*fKEgs(S#Ag%5R2S@1+l1Mt34o;#i+CD1
zb2m)=maxKUtM2X0uPr*ztZ1A{3Q_6{y7mf5Nm{u4QU|kpARdssVgI_Bg?vPn|3m_^
zSSdO@@j*ayQuh+XXjdLp$sSZ#uQ-E@0xgR9R7o)AIAt>TMEwk8`Ggcxvyt6O&r>f8
znOmMygpT}lEv>pqncPgf0<0=>%LM~n4y|KcwC<UBKdn;g<?p5ox%0$Yy^krQOdz+K
zGOGFDQ+CQq98Slw#Qx4QUP^h!8*9GG=kue@<0)r9CJP@vs!mI8!@<Q8;o~hOdA+}r
zBrris9L+Hu-5i8wNrtfx=h$13A=P|vGJL~39}$jF-cZNBL6r*1mS!-XeDbprO@x($
zLQtagxufK56Dbgy^urP(7k*x(KQxaFt#GF10MxF&BJ|J3P}p9QVqi-p(-o9OotEk8
zFGC2>HQFy(z&c@$JY(v56KOK-YnOz)PSw{MRT1+${MkwaqHU0*N=00*7$-()L$q8_
zxWg0(?HLCk9#lrr)><5lJz+#aw^x@LC*KT!&&w#E4pv5#pW|Apo6Wnx(C3ud`!*(J
z(l*~nw~YxBlf-hDNzBCCm|MG`e;1KAZpY`%-p!TYlYOHNJ;#xgB+rVIA{-im(mv|b
zgh>89M-rMnA85^Ffkm>a<Lz}DirV3w5wE6Z$PubH(rB76D9`P}X~SYFm?#9T3=E-N
zX9J~@BD`VCNVR9^yNy(P2^_wV-C{6xgE<P%Lb^5ZuNd-!rNWh}KJ<k(A|lRNDy^T{
zu}*rsk4brMjl7Rq;giq*ICI`d1*$avV^!YA3B+u&8s5aU2DzjS3I->%8_cUI@c~7o
zG-ODaBj3|$Hz;zcN<Lx=^%V!ST=I)lB}SIX8ATwPrMA-umi=V~QkqjVA0Tt{y)P9C
z?byADq7S#*^^N)ycYj^%9+phbqMY+2v+i_65jvq=on23JV+GLU%6O>KWu;8mcgLh2
z0eZqab-tdDC|-dO+4TzKXqow|-hxn(B^jx~dN-uJrVIJ-LujV6lYuu`sU7>{A*?9R
zN))(<e(t#Y^!=7FI~ZWt0}f=PA&L8(GD+ti&kDA|At?uG{qwjYIE~Z2C9^R*KK2rd
zDnrtk(|n4a8;hd>n3J(a`Y=xbf{W(n`)AT*(~UMyg5;6SmK9#d3JFD1Bk>~8gr^tc
zMeYy}A^N=-&o~Z@?v)s7VkmElXbePzVptXiDQj&}Cx-@k45~yhm1}-cuY!6JyK${~
z{y-S}IR9Q;;KfLwyXnHhWBRrwpL_a+XTQP7Z$V0L_^Yntt9Os(clDAs45~%n)bj-J
z!qDiOdOq@HdHS}s_`74VkI7-iWx#CU%axo*8$PMK@<h>#K8Lc2r#xihw#4=vJ#xwS
zg3G`YE7%`KSFje9mtH0Ug%~C|njG=)LA*&&tGxJ^dznaf91RA7ap)y$Y|bN0?m>6`
zfNPyPq@{wqlcfY6xll;u81VAq$e=0(&rH7~LWP@YHPEgAxKD}?(5N91BN66FVgU@S
zpu)J|NIn46>|pTUY^;)EVYm9OCsw}MC60m5<1#sz*nTCJ&OzKx%Gtd}wO(ZbX~Gn+
z*r(KLSbc=C!(pfZ04w)JM+KKhC@{W(dl`|@HwEHWr8O%;{6xTFLN0*;$%i*h1B%#L
zxU)p?sZ42tWNsP&8BLo3l0a_2tH2CpG>%LoLf(<#j+tK67^pb!sktN7A7fDXhf6CX
zBq%%wqDB!YyJ)$r_r2r>M2V5}!H+|KDD%DKIA;VRWg-ZNhTC%d<#SDMe}@$mT_q)K
z4W|zdGtoE+l2`N#dXz@Aa!ru|-vDEKB^exrqIj$gmm3zJGp(2TE7mpP(IqcF6oB-l
zF99-dfyMVq2z)ef3=sI_ZV=pIa2_Ik&jJR`NUWi7W~Dm-;DtF02zhB{4F)&VE`_9@
z42Y*n69E5K<Y9`0Bs31)8(1N)ln^O?=eCD!Tc!w%q$HGylNL%YyP~<B2%3`H3I5B$
z&M{9sh&+_!k4+X5%UQtgM>%O2IUAYB%eX%wB?B1_=ul(HuhTHgVcgZ-EDhotj`xa3
zj2OF%<~e$Ik@j{N-(m_4QmmRQA9Z7il*h_?_Bc;_h+*7ENi{1CjWkDx4q**GYnB2m
z(&AK*hCnPfPGHPuIBk9$3wt0%3UUUunaIJ}{=Q)WqUAK0eAqZ}OPV<18Ag(YqsXJn
z-Q4y6B+^k|X*xORlWWbULWbTVWCjm^>Qq<2AA{_mH-2p7Dnio|v_(f75}9DY+^<Ee
zm`P2Tm0?t2l<#398`Ps8vdYeUUUq8M=#Yyd2hVWM8rE!O+7wKJuK?~@U<zA>H2bWi
zp_avEl-rQx8S(%+u?>Uk7!p`zla3^JjU~-yM1Ux@K8$-O$L%H2Fhezzj~vn^X~LEA
z0Pz<u@_E2`nqeTMKR4`7r60=#D7rHHEyx5hAe~&?Bq(F(3FU5caz-)<#wn(wYyuJ)
z=fogWsl*EG!Kdkbct9DnuX=yrc(LY<*Q67(yilokZIaiP%F}HSqhrS<%-tO2CED5x
z;*$4y%1b<$yqH3sa-X%NqJBz|#X2g@B(!){Qj5MIHAR5Q3}-+GuyEpLWJP?LHa9Es
z%%?dKUbBh4+-sQ;6LVtF)ahzjdFh4;b9Cd3M66Q>mM91SxC<x`C+-nN_`rmDbb>Yb
zhn^m`sVqNcm3x9%`*Q4$C>uCSf%;(2)_-(7rWj-*&5C8gC5*P<?A<c?x1IB;JR}95
z=!H*)d5FJn&LdSwpBd*VLWl+fGAg+XjG-T%rx%|&^O8JEQ0q+GN>W3CTS>7%oDHNr
zpi<51GZrPh;>#XfVkp1V<7%};ZFkC?^D)jekWEGGR|?fVX{zsX)GOz8ipME9Q{}#T
z@}6dVm2zJVc~74mrpkS_<UOr*6>{H9j2)P6HW)yUO$}d>&P3;K#NEs<8AjE;ck#^s
z!Y2Py*`(5X&nDLSm8?>}N+>Nvj)Lu5Im-Ui|3E1tzHm%~V`>*`tvYgr>5Rk_i&2_$
z%8Ui&yC0hq8p13#tn4V}A%cnvIKJLsb|r;L-N}hUqj_Baq=iOtx%d&7;TK+bpS+Of
zwUQB93{a4O;C`B|NiN9#HKxarAx&7!u6SItu3O8FZU2SbF&GrEW?j+}1GLV(F=lpA
zoC5{-r9nUuec1r<I#sc&Z>%Ws(p3E3*@&=3`0QmWdg9IjM(>OAO*12+j(W^U5K<jx
zL`9Q1a*!az4&l{{_p{q4V+JzI;EbAsAhwgFPb3?`ZB&RIwl|$ry;oM<w3VA(75Mq+
z0`05Sf(j34n@C7tGbRa8Mngf<yddOQAY|r&DC0bwuBni{x`>ANFHi~Vt2=YTxp1MG
zU{v&DCQkN+8&2njL?QLKp&+I@+>nYU1=dx09g$;KRTmO=G3&m5GNnvqy}9DVd@-^D
zE$8zL!wwPH|5hfL<hURT-++XEsx8fCWQWF02K%v+3xVQ@n)>f%6>qp`848z)2Jt1%
zTSnQthGD7eF+zc(aNE!^YDMo}%*S6K7Zkldc1pvg7cnG^&E}1tZP-ZT*u9yMtmn4A
zd3psx*L|1FJjoMA{5DpHWsMZUj3uz-=={cO*7ERG=$hZu-O?#9`E-2oR95ABK6%C|
zlEOXNK6x#b!%Sm)nk7e-|65J(#XQZ(5F3_H!heVtUUDlZyyO${y(?1M1Rp$&RpE<w
zS06lm8&+DY!Uyj*1YPQ%R>vK3PNe*a`^P2djUT2p)2P~2?jafPH0u3-+%lV1&J6`f
zbhC^f%YTHyAEBAwd`He9kGdoGdY19)S`U<u`POly-gDAF89PdZ>!1)5{h^!8QEUu*
zK{%T@jTJtMFnyr&J0Oc!$e2r}1Ub5knelOPZwQ$yNuB0$WLS~8B|=s0xg!=@fL#sS
zm<735OK*3qR(){(B^M)6oQ@HUF0S{mAQ@3%?4Qp>hBl_C`m~H4Yn^*QmR_WojSwB>
zFwQwyddJxuw|_<xfb8ClrX+-qKvxI3d^PujsBj42X5(m7<D;<nJd<MYl>1@zILceF
zB&zla^CZ&zX}sASeV!X7s}g71D!I-lL0T8ty-^kq^A@xvDt_wJrBHWY#3e&n)W=*t
z$i)K^)-E+I+>V4UQ$(e}bR^@kumy1`o)f*%cPFT4*9w0b^hu~4L(jjS15zmrJBxc=
zR=9Oh4^a!f6?H#{`1n>vc;T%@_BazGE|*8LpOhgWuW$iT8s^XlQNooIoeXzsn(VU4
zDrPyJG%e-nv+q~CIBVL4|G<0;RSAzL%yog*a&PW1r<VISF9#T^jI?;)>p-Tvn-gJS
zik<pog{9{a^YXq^61{<bq8qrKPDtF>>lotrlzlb^4tfl0W|QbrEy^}YWzjq@ian92
ziVdPu=!tR3jm0o)%`22P#<4#Lu1VyepMj^oC;l(=eS|+o@)xGpezBrp=nd_PV$>-7
z*}#nqP@Jd$7uK+LCPuX~7-HF);WyLP%)Dswd^VyypGK7*o7MgO!Vh)cd68DFNS-(h
z%C5>0`4l*viZy3tiWvydCrAW>y$sL<W5hxxu*@Tdn^*>>*a%;m@H?b4?;hwn!ynnb
z()DLU&-H_K>*9Aip4>NoJ@zSF>u{K-Vf4?%qJ3dWsqLK<tZeMjog#boVvr6rcbS-~
zQ(W}Y)N!5SEwJ&jdS1y2YC^6hfKz~PzqsEdp!rVJC7F`$&Fxe>-%NpK6#fUB|M|}m
ziF}vtu>KUu8V{EJ`<8RG4I2<@OY>>JWyAzbW|&_2zG<AsBa_)Ba>*NKklf`oYV4NI
zauJmu<6K7Rw3qucoXo5m(^u+8PWyeN6MnlYqdF%bizsVmBI1`SGZ8AJ=U`y1^*lWV
zp)^w>BM%Xe$%ifwEOyRTz%GLG3l0Iusg;J&mCluknJk5JhjPd#m^B_#-%_y4j04J~
zW+W__#GseS;EP7ecq?Ge1R290M}0GO6#Xt*ZiwHM^*%)FPB*sV9)>cvm~7n2&?%8l
zv7wRl`ZqH8p>RID479bpn?|FcZ^TkG^75J?(J9#x&Rnj%=>ZACAkX|cCuD#n?CNf$
zoCuUK>`|skz`YDCB;z>EljZWlQ5<N7#ir^!(8hS%#g<p27kI9Hk~2c=&W)4J%*g8U
zu}=`OIp$Lj(MDF!LV&rkG{#}E_ndT;BHN*1y+wI5Cgi6p5c+g-F3FJuJygLdg0Mf#
zyh7p?0ZtezFOSLG&Yb&PyIj!7KSmXb+3_gX3*l6cIZT7tM_*L`G7`B+Tp8LL&JU@J
zsoCd7#uDHRge<FkmZa?Y!ast-EQ^t}@(%nW=(7lbY2+s&6H1hmdQp^Og3Iwz;sc+j
z#UYb!>*b*44O`JsrSY`KmXI8d9JU9HBRxr`&rzVF0}lRFr2ISu<|-6re)~2^<qtOp
zf1CbISe%tztX{UwxzRmL0;M;cf$n9MPn_=MwP^_xmtO4@kbwt7l&N4bdlzA4h1y6F
zaZ9cXx<?#J-*|IHl~BH3V_aND5fK}TL0`NVm<G*uzw8}FjMB`Li`kuM)e1pZj^+kg
zyY5;)l~LJC5yGjrZVVp`jyR4{gqAp{l!BGhFp?O|!fWc>_!yVdN0B*0!^ue-qBT%s
zW0Xb|Vt|v^UjCY-T<nk=;M_J4!GP*-A2|l}NiL#ka*&8;Z>^Lf@lTY2beP++xwg;$
zcbw}f!cGucqiUg69gkX^0ZN}>#oE-$anoCdqri;N$4MOrW1pkj;?5WJQ$8o*;uks*
zI?mP6Hs*QuRL(qaDwpwVX4)>e%I8YyRpI}wcx;qeQ3dxgnXwF8B*GO7y+I*IfO#ax
z`2Ws}K8_X+$uuu7<TvQ@(0?_`!VzAL*iTI{JiLsCFql^zM7#X)V#NXq%ktP)DgkSh
z206l<h^-+D1q*LEhSM$GgtDWJR2>Mt1*VA(&kY^E5fmmLCT@~URholIXRt}kynP~P
zYSS2-0;tA@S75l}J$6O1!^fL<5j+!<)?$U+Ss`8CgLXqonsw7rRtTN*%1U_~n=Q;g
z*_cT0NiY6zS88vXQL2pun6Bx7t<qjjUVd{}d}hj`>Hc5!6IaTl!S_dLvpdLE(A-=d
zW^ZL#StC_t&K*|c^<lhE=a&OI&o5?kJFQ|Hb8|zzmBV_CgpWlhQ}e1}ri72>PNw?I
zQl9Yf9$Y&ghC)JNDtQ<BkT5>7i<;<#X(5=1xH0Boo#c$U<8eH@ioBpVy*H(KeH;pU
zS4Tf|OSuPR6;#SI<0_|P{%EKYVgCfvKy-=3j+inoLKFm+g$wnY&%n5@`5>v|PK{je
zGDv(QWX<yOa%s>Wqlivz@_zfx_uV&#2d4+SyKjsfCw7b^)tY(zkOZRuj^s+CL)siX
z63iv+MqXn&s~)AOKm8mOE1Fo8?j)Uy*Gmqaw>YpD5c-*%;7w;oajIpcPTYg@dX1^t
zi?Z+v_qGQY?l<nREUIokKJm#k*9UDs#T%piy;T}xMguYxOkI&9j=XuIa7m&;i1YV1
zW4E$9nB^|$6Q8$|d&aWcG$TP+*<G6#3<dy?^Bx;Xk5x)!&nj*hqe$f|MarceRW(nM
zyE&+uStb%u<ft*UJ=UVVu!)w6Y^5VYWm4=5kG#Z36r^}*O&}&Kj?{(o)N5tQajc|x
zALn{hwJ7hF0#0NxbjK|4u*z(L3DoUv6^i4(+jW4JZw!qeS(Xb2ortu!iecztj;9X|
zAQ0Wd(TmQKcRF|xga@-4y*s<mhiVbiJh}4yaUzo|gQ$6b8m&;s)^|d<l+QmOlz&QR
z3p`HXJ)%bnH>Lnn?Kb~|CTVN#bkHOSpV55tv6Q<#e}pE8hxtf7IZ&UO$YUs(O#46@
zloRzZq56HQ=NWn7h#J59EC@iKJelugqipDsX37Zbs&9Nm;wj9$9ADCmXdXl}gMp*&
zYgL5ayk^7ujAz4I0oIi`;r`y2gd~AEOoIgVIZs5%;<(RSL$v9c&-Rc5^`EglaFC$;
z-yRrwbJ!j%K%eJ25b4${HwmrZ3u2fSw3WQk8Lj8PSqPNgeY3Ctea>@ZWpVyAW@jF|
zNDkEJJVOt=i(s&;`D`LKr2mXPghGMc{}w{1o5K!b!udRx03rO-;ZTgwyKf&RoX>b-
zKBdirQ96&k!-n)3PtU{d9CXC|c^+(wAmPuSrQ-MVT_@(<nNp14ar^;h;_iGZKNFub
z$1-Okry6;HxtIrI%Fo8<&7$;tsCtS^-PbgbSXVRBc)dvw5I=G8cBc^RLxwBIYkP`y
zaW4UGD;5Lh+@0%6Ab=lf894JxT2Q)97@9*vyvHdSX32#=-$`MR|KZcZgS@|qsU6O~
z?xry<*!?HFVUU5Zh%$VS`1a?HasTthIeJGVR=g+}3maeMECN;V5cKZFH0$>_N7$A5
z&XIul`R(HO;vzr^G3IlWV^IHbXF09xm>@sN^WH;g&*iMm;dUcGYoGJEuUttb=S{)V
zxiLQ0yv^m_BR_NhDQCo5EAIX#XAWCVeri7FJ;$B|f&BYZqKHH~^VyOlurHg^Lulpo
z%+2A}q&Rb5Hl_!eySZGe6ld?t<}{}Xg1uyx1ZA~nNW=Z?wyE3=*u1tZ^(?^t;tPC6
zu}TtHlx_MYf6%0%pHo-(H+4DCwV0$A9N6T$-R&di==kvM-u|xh_VD=q_9+Zk$2<|C
z7>4OZl$~GDibt5jXNvMX3B!LEaD3xjxrrmZw}uqn({{w~`4nCcumS^Mmm(Qc>;=Xg
zazpRcn1htdpN{y#?pzqz%fIx85qb?mThP5!Ism@Nlv7d`4`J|WI)1(afINTsJc`bT
z{xZN?8Lyn9+esQl!(`>rEB%9Nj@|wT%H5oa;HAXmM7(gav!~ck6zq5sCI}cVP2iW%
zQ{n1{WMb4J&o!765$IZ>;Ah{)0RfYEAQZ&GrSxU1LR?F@Y=swY*!M@!bRwrg<eAG)
zr|b?+TR#qZJ`OQfxuh2jir5RO@*n~tzrw*^+>FtqDy6VbCd`(jT}ggiUS9sma3LXp
zLC8*oiGw5^h!E|Z3gQC=k|UDor=g#qkKPLad>*?aS}o!Es~2DWxbtTFbo<9eqV~(q
z%0_3s^YkeIPai*bI#1iHo6nt%&CRtZ8|{ws<LYx~6&E*k1`Dk{ch)vK&z<(>lP6Es
z*4ImlJbMPOpFG)E2YAoc*4vxwtNBuYbylx822VFV_bGkbaNAy=-@)(BGk@J%_u-q{
zSzTRgZ{oLJuk-BbQ{4*mwhf)x+{A|xI(DxOz1v)Wwn<NQhv5Iu)PNYu;`{BRqun>%
z{k_-6+sAvmC(rrCPd}l*j*U2?IX#0g@%;;-aC3XnrJtKd475Iql5{bRKKLPJkU{Ur
zq1y*KAyU&5ja;0Gh?prSZc%U9Ta;&g(I}=|d397uT33NT^d`^zixNiHz4T)wr&lX4
zFkqj^1ECTH5VLg8&tv}_y7B5c^gfHD-p`5nO14<9V4LV?BmH>9k4@ruA(F$bb*DnB
z5A4~qyF>dcK#RuR0Xk{Da$bnPOpz~gPp!Au4~$@np4R^KeoVlT0Y+kJE>NC5n;A^y
z`k-~uUwJVOyjLJ-jG=JJVv#)1-fs&VuIKl%bJ+dPyL_%gnuM)xl-S$uo88xE@80bl
zymQ`f@9!V(;1>LPfA{@K$>#fa4yiBTwnX%P*MTh!HUkjkaXG=-8U*KA?635)VCXqN
z``kV)OaewbWM1SUu=?7Hf>Y+p;N_nxLZJA&2xS{%u)r~;Qlpepgf->|Nfx9CLRO&i
z$<3$-o2`nlz$3$Cf4nKdI;RmQ#-O~G9#9YhcYuPlW4O?e=Q}78`C`vyEx{5+jKXBI
zF*nU2K9R+lp5~4p$59MoS%%Ni^*5s-$G*>@qOW2Q$d=pz@C<(H_-VgvLp<VvBMSp@
z4U=b4uudTbVPaCu?C|23EJ`sblE{_`(xCTZg&z_y5vI@x-^(13Qd|*IF*vUZeepjS
zCjsBe3=$M+Q8{Sa!Ki+Q!nrV1_^8}<ny0MkV8T_D8yx?7jG<BqQ!rx`t${^=c!Wrg
zdX#Gn`b;x~r$iwGCD|}dWaKP(<;yHEJ5h){+8^GuXjoKUj3ze{dS#6UE96FlkBW}v
zawH&Kj3}r;qSgb@;y8_H()7TfW3N@@Rv)9oUPYSZdG%!pO%F?qQIe223rGTSum6E&
zJcaSMK<abP!$<}sP(3gjD<LJ0ri+^u3T;fMDx57WuuCNQ5kla|B)D5=sa$@<ZCJ2M
zML!aq7KIA2(=|$k*bLAbNnA-9QP^7Qn~@fW*FlsGJqjirM;Ada5E(eXi>@&Bim^PY
zqqxVOQwAp~;<My{kT5~rkm9bfyl%<rurHrOC%ok-J_p(Sd+}3B;||K{B6c++EYGYi
zRSn70Boh%~;y4KqMhWHId&}jFy};LbB0;?EjxcM=xu56gd)x004o^<^cAWRSCnwwQ
zcFTDA9-Q368!6_<#jXyDE0$VDo+lD9jgAPIOgszYvR=8I%NS$g(fOoB2?VKofO&L#
z#0kUMP}%()DQ^k(t7W#6(>J@v$6Y)L9vm9d8cg#pb4dcp>shXT+&ZB_=H#%fxjQk^
z#@=Ak@&%fW^N|zjS=9oj{F^?kDMd+Q_$-O0A_FimapvjeE6XR%1|=L68??X|ovZ1&
zBM;a#F;|JXuZ?`h8Xyw`W5l=;i3y6S5^=r!QIK-I)W1$CFf{fPmz2zi!<{5x$QsmG
zfVgQfMkAE2-6Uc>PoE~5U9qFe8Ro#$sosdC9)gt*W{3b27mcRr&!566EYT}YJN@^2
z(b)YZ^UXj2GSd&t$$5tCOCH;E#D<ww&CX}x5ZabVg`HFD7a0ym2ck#<@)TR>J}Gdz
zMDW%STH=($_OGs(55c?xk!9&<=wo!yP-S}ZB8uQ2-e91&b1}0}(@!je0~vQX6uSzb
zTu}o?I}bi!$QY=h7Y+;~c9RQ-;ybz?0Qyiyt=p7Jk(mw0m7@%m+Pzsw6E8TU6>NT4
z3V-SFCHXO)yg&|K$rX_@M@{a>CaYawu34V%9q*o=9UnN~Z||SwIi3K~QR*yyc>*);
zxv}SLAMIHctW}zuTbm?xtQcLcUs6tUP#ytbeylT?>1g)a&qmHB$pgsQ2H9|nW5rxy
z)&;kkh6OCyKwwz<l3XqM3s86%8SIAnnhsU@f>;B+gBAXM_x+0%{lun|a*TmM5?sSt
z2FwNLVL3)bb?zavDw<tqwR-G9s>-`Ua^^}>l{tGicuN_85F<bYgXdZ=n%8uiFLG*8
z^@>!rCJOfsfHqAON_DPXFQ?O`p=h>7Ll{x;eUsP8uV$cF{2<RoQoCYxaK{05g6t*5
z?MZmn*)?9EmJ(it=NT;bmydQHk!Cv@2sQU*6nHu~_X|>L8Ue=%M3owuUsmC4CS(@q
zu@*Al=j9wGp6W}9Nj)wqz>UFAS{5{izSo_=CnSAKwvgi%v+xPok;)yff=?LtFB7;0
zD~t;~u+9+Giu(^;YhM<{EQ=!cYb*yv&4OlI^301jgBa<yFA~`4`5LFOIeo1M-EUiS
z+;GKdQ*3uvB6ysT<?tdlw8umTH&|U6oq?t!Thvx4cZXN*O@h0{rO?I8>oljAQM5!%
z^a+PS%CxEW;i%uwVs)ojW8zVnHD%UAi6GwOwWw$ai=D}`-a!=Nwd(Bh-t}~WFD@%M
zQQ{LIX5{3zlK6}p4)~J5%mw`bk3*PcC^*leERii=DN(?^1j2EJDhcf^g1&F|*0}9o
zt~JTmlTc&|e@Yw#GNUVMp`l<p=Nvcq+>FO5#?5|gH)by?SOA}(NO#S$CLQ^VJWxhu
zY|P^SVi_9E_emwiA*&tN!!(*0om0R%kE5%Q(!t>cRXubicSTnfawec<h9w)8b<dRG
zReuzrgWSoQJHfCx(yBq}MjnL10xMmK3LSXfqU=DVkPZe?fH#R<=j5?STPPq|1xp4I
zbUKykQfi<=sEF)Se(3<96!LM*Yuw8z1<II;sM5R17cX*_RYx781U^l+L2+j#53)~7
zye(*WpXwRNAllHs8r#b;vC<7v^(r<gwqpw#5L_N!8bua75i<!Hcf>)h0CU^J6r=p2
z`WbHx44{vrp$AZ{Zd&lTi4w3|tjxrm+ms145z2K83yUTuLcebT<I8W?LuK1INPR6M
z+)jX-S>)V3IK9(^x+9yv8c{o`s(uA4!77=I{eCb&HJFR}4?cvzXjg&|XglyM7?#*@
zujn}1J)T_v-%jECIAzrw%p{6;c-!aDS(`df{LNM6EO1;FYc540NWu>{+U<ASdk3=*
zw|Lzk1y!paNs{p-52`s1<?Z&_{_J$T9dfkq0qpGJrWqCSK8VK{dk{zEdz3B+wdf#E
z7Y>Zdiq%XCmfr*(mkH|p!D(M%3J^CSAR|5S(M>45vjppQaUZR%>f<`KM3<>@ov~qa
z$IYX_B&D(+UIuYQCw*I`ynFEd-tpnVdtz~gD>U0^$pB*9+XzA+o_y{1GikdgMYpm`
zHo50k?6{R=Er}W!VEl@$Il4QC2XBF@=}`$q=jmD8l$DLetX0{Bx*Fq*s{pc}Vyf*x
zB{elwCh}sCme3f;f^W^l4fiA>E+^qK0<L$h;9rwTR_;b&_lNEGNBgDMr>7BJGSUqr
z9f0xj8IO2;?JZ8hY9^CD9CE`~!_ZkuoNwrM@p&3&{x{BwpY~UH0WISN@rxBvHrIQj
zVczy`<i%$aWijR#G&0;O3)S|f^FDa3x?x)q*-Bn*!X}n8_QrYXJQ|*N3G(yK`lGG-
z21$7oC%)fxhe>3k_+)q2+1@`nEJ5+))QB<)dP#P~nwb$<O21tK#`!f-nR0S|O=M)S
zcF&D4Bb(O9+8rB7{p%pT=SI%`lp>hZx&QVp)m#-qC6(hmC#Ty7Z?=!$l(L>_my%+P
zlkOw)MauMn`dC^=SC^1^3!SASXYb^2Wp8(fDnEPr?Ai17vuCSLL);R>TW9Cls=WDa
zR@z6*c7Z3_7dBPUiCy3=C#mPOS6A1T+e@9*)wb%x(c#J756kUhAMEMU1n_F{50gLg
zo_neNr2q8UlV?vh`<tu1Re#f8AFR7;n;ZQn8|~H3lg|3O=X&lFcg2g4^M{;x;5r#D
z2jP@9R^hX`xlVuKlmENA)_(Gb_F8*mV`Kfv#%3Fych)xX?-aOm`D8%LF*F%RQ97yI
zwD<YJeI}n5FoVyN^aiAe3*S$j>(LNIIe4f{%C4@imao>xpbXZ-vy~rC_E%10H%yRh
zA34HE{N<ze@+xryf*6d*KjH%D@a3bwLY3pg!_&Vix<Ml9MbQVOpGPYdwcK9PP2xTx
z{;x=Mx|g!rixs=~Q>p#SC;z$K`nRWIS{u{W|K?iB`rml+W&J<E=bqL-9d8}fNW6T6
zTU~M!M&l$%mZ_;nub_IstCvbv)+3Gf7@ZxHM{q<N!v6O1(W=2RP_;B9Gcq(y*OjDT
zf@M&WSW*QUcn}VQ&|e-!XkGk^0Jvg8tZYJPH)IRKhOBl&hPisa!H>YtW?%n7*dJz|
zzf!*qTn|s4Y*em){GMC??akHAKWyCJu=VG^eEr{k{$<VSgm8=bUt4>!{$>6j;4_E$
zmx&`Q3E>v=ztL&0ewqIV_{?+uIh4gkp^b_1UwiGzllDe_{x{b*H@?V!5Aa!eJlD_4
z;va*7=MS9!*%f;B@4Bb|INCi~T>K+E21#gPfBX2|ZukA(!Os5n$;rY(<DX}ZcGG$B
z!r6b<wO=+D7gruT@7?QQl#NQ_-v*%*g+96~E<4z?Y~fw^`0Swje*1^x-FMgmJUH5Z
zx9con6Q_Fz|G*ZSs=QseeJ1*1Jf@B;I?$IOq`t(y_bWDy>b7OaIp&|3q7hvV`Hnm$
z(;U7S4+c)?<29EX-_S+qV{}HKkT7C2jxyU^UNi@Dy!+;#hX;g{lf8fLE-bV+WoJ;^
zG?E>{+ZeK^qjav%jqH#Vae?zf;}tzSGT@cPa{z*H9H$X-HRshUXL#PF%H2^mOq<TD
zmrh8(Md6dE#h#`Kd*YyFwpjz)SfiVi*mwFc{vP^-Xj3KUsfbI7R$Gq&Vd0JFv*;~)
zks7Hj7jrZFX7BhE=I?C(wEKpyKNr>(7tyR6$<7BNasIuyKxrJiqsa5SjG33tYRh`;
z`F)tHmrlF2Xgx<4!xWx(7Jqk)$LW^p#L*ejsBFB@6q4*4f8Zi6xDj%t#oJUL9nBJp
zR^ROIZ2tpj^1X)GUPTZ(sfm9{Za_i*LjuSYj+F?qf8oQk5^x8ndpmzS=^pJKch8RA
zY@hA|%+*zmr6i3rbcaKl=we7PF@NYT)=es&0|rX#lVuVWNmO0)SyerNZ?;ugj&5<f
zBX`{G1*vW#_jr2kQIPaYYSZJ%we3n(WqU!`MXyg)Ts^3&LDZp&^RG$JO~6&<WSOe>
zDO%;|QuGVTF8<MnRX%Wl@ba7+$?E}jiIdagvz=4iCvva0k9QBY<?HVDVhIt!f`^Xp
zg`R&+S%9QD5{A1)+qoo(`T?0Kps@ojn1$rmAoRNF!UB|rZ-9aYYt`YMfw|GhO=ba>
z1vv>Ugjht>JAz7zj|<CK?Tm^ZXF&?o#t`^Bw9Q|Be7$%2e*38V?)dQR=mfW%pSBi%
z-=e*5*F|qgc4St;J?O#|DjH!v$fXW9q%pg$qRX8wOwCSSRD{CC1>nfXfHc?hVr1A2
z^rVC>u1OYXcukFVw}YW8oJP^*nG7AyePsB;G7mr!q@jC*e$~S$`oMmCP6MVQBNxuK
z&Jq-JlA9#;N6qJ0p6WQxU&P1y65_{!|GaE8bXDoUIM1E4P+|GG)p_*Q90YNat_;z0
zh_;iK1JmcHR>FJHZtf#?ANlSCg<XbED2@gLbR$vk_i@KJ-uI&{Om!FNq1kvEyBze$
zXeWfxFg3CZE9Ifo0k2xl>%D`$gEw8WM%;Tth1eR;jEaSjJb*;3`!Dn}gP&Q5cBa0^
zPexb(15dzam&%7jcu4U0rwcz7YX72e{#MzJAO#2$1e|jiN{SdwRQw?fHUj?wA4*if
zV+Cgn;A0z|^gI#I$qu&+Eo!V=c3MbwO5lhCQ4UuG=o$wdh2LNti<n_A!J}3B_APvC
z)3+broW0-Q{)t*m9oelPiSp!Ann;6a4I*?J{h{xMOw&XgiTGGwzQvCN{{h@3SCzdM
zpYktDXYfy5xMu=xSr>^oYIpI?QvEK{XI8kPJTEd;c`QEG7tINlmX@|_mR*{mtb!}b
z^UAJ0qNV-qTVZNRe5@};xMruE28U$wTD&M;ymWHsrzzc5V}9UnlS{qT^_R1D{oyc<
zvI50OTv$M1Ax-p2VCTEDgTHlg?<P?N1noBqjRu!)u3oRU&zdD=X`l;M!A@zxlT&~Q
z-%kE!7G8t0C51uqKYagw?=;_s^-|Dh2mjnXKFk$;T3VD2Tvm76tEC0s?r*<4DRj5p
zFYmA5&nMsQy**vnPz7wZZ8bX>lr}4BaJ#u35)Sg^ElvO=>z1}qHj?8rV<e68_F#K?
zS~|sWq%YPqA+pi})fbiZY3TI%7hOp;{;${zEaUv<l+Bj)SkqjyNkZt9vm(fizr>A-
z6Jv8B_UNmWXYs^OB5nC8thPYga(>P*h{<CXE6Y!WD2-mB9~fRLrdTRQ&p<<N6<~cA
zb74Fdg7t;(dgLYOM6t)N%fSev2BA6wv&P^iiV-SVH8J_Lsm1DjKm!g1MHm!sFyLOo
z0(0zh1QzJ57bHKU^{|sAexjw~c2geMzM)f1!Y6D2go<>U=2;reiu(Ro1i?aJd?Fee
zy$Hgki~@FqOM`5v#9##9LVePO)L<!NZyydv9)%PXaFzvw9AAWu@+oaDz)L{`cy_`Q
z5+DW7OeRLqD3ih}ljes6MFPA~L0JnxSqPBMULy7;iQImSGJfKFhIGvGp$S#@V*#BN
z;Ll@>eb`O=>2-IUg<JSWKS|B^`0jVvHc2e<<cHHb@>3TN)%otz_7M~0-)S|-)A{(K
z3UkJMQglXCK^0Hwx#KZ5JLL%d8V{D+qz^PHzUq)@67hDyp`XBz7BIRJ#_<dkO{%R`
zhOa^lB}<SfMi&L9M1`O0TwO#kg%lZ#qs(yR;%=PM5h79{S9?)<K@ft=L|FgP;S^1p
zmwGPNR<ZshdHUCAaor6`k+-mbfBD?bU5PUv6|};=qc0<MVbDbGON!<WWQCA&XktFJ
zIXsFWAu!~`V){k+mu^VjF+k|qH3~WuFpM;$<d=*U#ISsi&qPBv<#=#15|R<N4<j(+
zF^8klUSkYC0G96?!{H-nn8wjCkyz+O(GZ9b3*mbM1Ml52Mdl^_tC%jHMVi?UzCTW|
zq5cP{8Cju~wIaRBFqQ%W;<GNZU<@P`a=0JlZV{g}B^orL2!1OAwDFP87w!{2H_P5A
z6!Hs|D2#W2m{7Regj~H{2?>Y6y`gY<qfrzZM&^mr@R!e*opm%ZZ%W*h?;s2E>yPc9
zws6(Gxe477dh6rNp*t@v8ojq2D{m$v{##!EMEi}Y)X}URGaK<fh0yd8l6x4s>|w-a
z84$_K`YM1@Mp4Te)H-gQ{b+n6`8Yx&R!%#rybfrJn6z#XGC-F+w+8Mo5qlk)t~kks
z8`ENP^!`mdnDFJ-&Lphx(bo#*XJx76^RHsL-(Y~=U-A!2%9X(@E2r*iitvDMuXb7W
zK;?*wuTE)q0=(({b4o{C2C=Gb>SNn_=%5$#xeCl%yM>jzV0C$U*~veAfO2z~^?W(P
zXo!hGm)euc-G0`~;Hm3AP46Y;{#6EVMJzCL(rGoSC`lard!(-ib!%=y^^kjQiCh)>
zE>Z-qR)xt0Id%gXhAweC8SJ+lj}<s@NFK$;KpckW2P8eNqWh;noM@4^KWYz|AF;{-
z@q6a@)V8@WOQdY$riC^CC&CZUP}jSh>rSQ(fGXcu$ph?jy}}4<^N+3D-CRjyG>hWi
zK}><XoRi&XNU(7FYnT=BT-t6nOFAK^Q-D5=QX@jbjAO#95#37$ReFbB3ofSeSnH7Z
zgiFPg9)!rtq8rTN$PI&WMmCUq!;#2SPvI_xDG;GhCg%n7euVGN7R;YwEQidpQ5GSA
zL#*N1hhos#R`Ike@8=Bp7gGdW2O}I>?pBvWLy%a5ku-(Fr0a-|W+7sSmGj2cJe*iC
z{%mN3k+5r6rE%rpZ8O6W0fA#5%SqfNYW}f*TZAs^>56mm{_ybhyOY!1qwWFzd&1a&
z6%OyQiY7iK8dDJHRK{*JGD-5yw&Fg+=v<0Lsf*$)MxpjWsAF*sLrlqt%&vDyk#lje
z!@|FE{BZ=p858Jm%#o)D6cHw6c!ESk3|<U248n<*4LQK1XU4yPo`D!5Vrw(E!3%U9
zg#aN@z;Dz)qDbW=AA&LW0%iI=y27JS1^~s&D5fv>QFNV{-4#O``7x&|(GovdGhi+c
zz)qxM_hOuHs7FVGK57;`1}Y~`PXj&XnR|oLEl~yoA>WHen83j~_d_@cfLy4F8B^>m
zEc}cDc1%G$eLaN=1WChU_PAwre&PKhqcGHJN)dk`DiRPLJ~v>7p@DNW9ap$}+JP9B
z#gD7Y%Muog+0HJZSVm})bO7fo*be#_^8@Wi0(59KJHkS$KsPtUzM?({|HQFrVHh7%
z$k5;d!@EoAnimevWDxRQSJeNIr2e>wehE~gP+tn0i_yinIRKI(2xF3s9B$tjl*8-Q
zq8>*^lLmSP-YNGE>h01iT3*X+ctZ|t1c2X(r!Zy=Ym5<bedi*&VpAHUhb`P$;aLk>
z6W3jWb_{kQxdj6?R=&KSgq2Mhd1M16Yze_@&|=9j-QZb4Tw9M(Fe&gL!AN-ZVTdDy
zMUx1{8HPA02{JX-l0x;4mF|LxRTvqGfm4_Qv@ya!?B=pl2*A@mjP=Jt4}U7z8Q2=b
z*cV{^M`J%0*@{>f1O(9%+BAyWkHK|iMjK%`1w;rluLzRZCS74yM6~80wh;!i7McmA
zgTeYOg2d}nSfaUokwFKZ4v|iVgd>!0fFF#|4;;-UXgTmolV@;rR3`5ZSK>+rInGj_
zbYfV^kOOv%B*PqbUkU^=6qlLNiKxQbq6vt;#8#(rp*?%dC^Z`V&S$-a)L;q?A9FRz
zy(0BV6zdJ`5GWY{#w1}b;9m=$a_ln|;zl?;57Y?JjE|!|@Q&68w!#Zn>}<Me<-U%Q
zk`oG4%tv#Uvtb7s2K^hI5CGLGtT__**JIXnT=^InN^D<fL5Ke5FrMMqWO7Q9W7aUl
zb+LqTR;22zh(y*RqF^Y*H|I6VFSu`FV0IxLiCHERr+Com0{tyn22%begwvyMP@qIJ
z97E|R7%-_dawaqj%JL@1XgN(uBAF_R4T;Kx#pookEho7N`xkK(QgF4)fUl`t3|G}R
zmL$etcS(fd#8arcV0Al8KaPi>Q=IG;Ee_1CL6CH=izOmslsFU_^E8n%j}APhS&32w
ziwTl-NW{n$ft@c&RVX%fP0!KB2z|UbM2vPArh(aIQg3>W@Jq6D6^QhL!K-CFa`DO*
z1u4KDMmp!53Z%VE`sme7@=P)PnhXr$8{3h#=Wv|W=O`en)F8%?MS=G7lA-5{S>zE4
zI{2~klc@lCK!(4t)mJ8DU_v~Dg$Cw@1}hc;+mKbsO$lCNEDAD{mA5OWOc$CAoW>~f
zI7*Z3M3WS9IdT)ErY~V&NiE5sI7UAToLg~SuDRB9c5vA)aZZ@59uQgx#E+-(Xt7|$
zf|e^<UrzD$OG;Wlo^PNk(qD^}SaxtTNWeLv?M)tn5}qHDj!}0DlsgjHTzJg@r^KKL
z8264?;5ap-&zt!r#K3+TBLDIl+(o=t65oEI!?G5G>7vH%%gjXv-5ebsyd!oegkieP
zV|FFwN_B7(L8kTAi&qOwK70d`6eI{fM@sp7aluP)G=86iO7UIr-Cf45t1n>$^Od8t
zZY-J$E|<}@`J4#SwmQ<QV*!sarwkDAlCT@XR|6+|#x*Cae`9wH|FXioN)C~+Q)vJ^
zfk{6Y4k@Ql9GqXEYDb(G6W*hJp9Bv+@`+1ao&lBJgatKmO2S(PPns{1Ss~qW(EjQV
zRQ?RP>xk<LsKl(2kCTB&c!&o*PqQj!Ab5okO0ZmCsmARKP+crVElNusB666MS2VO%
zr9dywG-@GnA|<+H8o3{srd+%gIUlMqhB6#y+5H5{LR3zW!+XmGpn(EA5-TzzM7(ea
zU;%$QrzEgU_ob#t+=sbxw+zgESj2;3^FQblo4=<K8_qX`QcK)5i9=VxxK^k$LTUNW
zsv%$tbh{>0Q#leA11S+W+14_Dfh7(q1e$P8+%v*;5K!(RrHf&Xge*ad&J?Amfhy_u
z+i$+_zBxQNJ=ooS(<H%xw`rdvcnY#|Uf=Ld6+1-)c$8uHhX7BOX@>R(4*M3s%AnH}
z#Ym)Ml%X8b5kE&DdWdUmUzto)p4${s&dl?ffTnphp5esZ<r}jEIfiJ90alWTn>RP%
zrQQqRNgND?+J_=6d(QsxnSF833K?A4JR)XYe74}zIqWsk4gvcm6+?+X3YNT#1qi<J
zlM9R&2`w^UU$I+dKLX1F#A8f?f?U*;O`w})s`Q$l_s3@jEc&c0EeVt^;XRh|<~RF}
zpxztnzQCFESy>XyO0^1fy7cNdabZrF|Lw(u!MyrS+{XRO!E#XhmOX*Dzu^w`+oVxW
zg}{HOK!8Ex2@ATM`%xRhZ&g3;4?+Qxzg7MCP5jglcuuw5i$ik-PdtX5{~>?!_o;I_
zLr*>mKIz_TWf}x|JGPSlp^M&uGVce1b)&a35rQtdl0VDmtfBOit%9|S$rr>%{04Py
z!dOP?MGdLZ@<&32D15O6*L;5NiNY%uW_9+7K0NUmGY4lS8^e*%w`9%b(qj!)?2gbF
z%g|W*d<NtJi{RZpr+t#C6%7-3k`+Z-X>k!53Z0JU^Uec!X|nJs&NEt>Bs+-d+QbGI
zut?0&Zsj#Y?30n5^hu!&3O8T6L1j&xiUrm(l3Zi&w2Y~;wDCo$5eok5;zsydN)tjJ
zm&zN6*TB3FM<Gs&I=6^f!r}&*B)=vgd@{9oTUDi|mk@AbOt2VEhNR0B=UJ4ZnMMlQ
zp%Vk?{5;>BV?D8WC`ZzkNn9q<CjAvlqtLj1sY%T_gJai{%87`V9m0$$iazMDa@;7(
zC-^2&gW;b77}@(sB_%3K(cDMOV9p24T$<znNS<rSj|Z>!4o*z%2wsLSCBDcx>`BsF
zB0};x>AF_G%JY+sFHah4K0`?}O1foT>iO``AYcWrrO!IfgmMDTEFfL~iWC|w3rjx{
z$ekh#i$fdvW08WE={RTh>l!z<<Txzg;7KgTn^Mk}LJ|9gqtHZUCmYNDcyNT)(kI%v
z?kr&`6K3;go!tl5O(Mok8K>yhMQUSc6<xDaTQ)A*OiKUCe=jbe$F3hM4%+44)cRCA
z&ev?mS>}`}iad~8=I#%tyT=D7sQ881?z2URBaR=WB<Z5r$IC+1o$a0PcDskK_xHZv
zJ3Kof!*sM_hd~V!4$HeL21q|(N^ot)p~GiK)Ci>)<1P8r^u?x^^gzAleqrq1g~vuw
zjKP%r_>(H8PQC~scla$UB4-Y-(q)0XckhDzK(9y%P{5OBkz`4z&%%Zh$h2VO1z!S!
zjylq2A@`npGIFoc(%9kH32Ke-#Rsd5OJMV4LS5loCu$cQ42`OWqIM{hu(`qIYx`C8
zFUsqafKaGjAh!S6J=N~cG#;mcOhED@VD-|EMB&VS?>x4C4`6-A^@{fWU?^AgB3YB0
zhf}eAxHml?r~|WU3v+J(BQ#CcmgGFfzZ~ck7XqK8q1O{3$8d_j^lu5-mKV7$|Kc4K
zxFLEJJjTB*$8N*!7*+D7*DCFq1ohY)B=5*b1ycZuS<mn<k5N2P#%!w0;{@sx1|FkJ
zY#DC7+U;^)LDpxQ&2mTNuOuvMCDyHUlVAk<Z#=m*YQSt!!jFcY)gHAzwaE*RD?iM$
zMSHx&$EzCmug7la&9`xDP9+;>m=>$=p8M%Esx~@r*G;C>6;vR4W(QNr_e$?BGj_?S
zIJG~jFS?RV8MIwafkCK+>9cDx^O;`Xu0<D-SY8yLTw2`nU{?Dsxf|5LzNkZDGL(bA
zfCY~CAUw%ij=V5(a`#1c>rAmflOGH<&g3{w<5ocqa30I=29p)kIBCNokkW-tSc#%U
zoKZpn+`70?N$xxw>KC<zqj_rXZq?ufrb^yzm3GO?<G}1u5eNA$lvI~JD7@Md!l5w%
zg);SKpjoyf!<HT6rVWEGhgZfuA4^iFevmvfZM_LfLozE6LJ3TeS*I{U#a8v^X0%ZW
ze_UrLuCsBf23m4yA-EFBgw_)LGm7A>lxvtJ^dh#4Ta~9RP;j1nugk_!HCDcLPo}nw
z0?FhZeUNa{#2=2TIx+=`rOg`KmpNV(?rGX?)0G+=@HU;9fkT-Kl8+1p1~rKVh$>|)
zfL={u0lfaImuzv-d&a_@H#wC~Q5_ahV~S?kG^S5U)zp+!Rr^EP)DU(JK|~Y^qb?ml
z>`@n|NeXTzpRpu4y4|bhXtCmG6|%#v8m-Og42upi<?_Ld;+w2CU3k+?s;^UoTaRPU
zQI`-IlZ07W7W*-?$U3>9l7eO{Rl;b`O@h8K1tROA8mm;Q5#Sstj4WD)KFE3e*u&sS
zvgOCt0Bv#ovF?^WE(*f~;{iv{vHD%R$7Q{xKGg1UX>U#9uIjPui?9NN9?>_hTQvKG
zorvE%rtqWR4vV$<Tqz4&&^N%7((?#4ffzSYPU%TQ+EC`y3$k7i_HR`W@HMrbMeVlP
zRHLw_#RH`@@vm}|uO^!}S(DJCNG%&ng7ODZ{9$#5^OUI4p1n$E_9|<$REZMwX8+J-
z_w?J;>|X@K$s2XEDtMvLTtASWeD7d??_ih17lzim-GlAd`@5pRV!6;K2}p>3xG?Y<
z3h!gA=n!d{s!7Xm%tGonLFF3;_bHcm3hm@uVCx%otbt#1A4GK%w;hDF&?Ue$>WUJL
zN&RphiwgM@ody$=62CT-$>WVZ1Sv{aMcZ*6$I~Z=oGD5`by=;g@uJSCW2Bhz^x3SM
z$qDdy>L$!uIfd9WCjlahZ<tL2dd0hnI<)`Xfo230dWgXkJzrchIO0`RjZBB)<Jwa?
zQXdU+kDf13*4qGipwi)XU`4c58!qeO<}?kw6y^g<2FsjR*L0-Kxli?cdZ_7fW_Xr9
zSgb|L?AypMLsfao@}P%)XfCLtlBjeoDS2{oud9!O6xy~%Ls<IOt(C)<GU)B#PUyH`
z3HON`ZbB^Q8C$aS^5x)}|J-T>%(RN?{|RFfj!DLva6PYZZm6EmV*MoRO{s6MgCd0D
zNy$EK>r0GoFd7dFt*P4%>!PC^fY6TJ0mQTua495vix{V|nD)XZQPl7l{KNvnw*j7i
zT^Njh&zA=IDL{z`4X;A4AR<k__{nt>V4I=j9vj%AM@8LH5kOwYP8EyVqbdX$+I^!d
zsM$)dtDviEy4m2HMWUhCrD9g2__WxtwvkHeDo{cyT?MAranrhqnc)&<!X-~pb10My
z$uPQt(?xnwR2;b6esSI{qD;zqS(p-P%k%LHrc6Iwt)s2cZByYj4V;Cp0&I;l6TVO|
z<00B<aIjOaQGSwcI=}tq6pFAF-A=dJT>PWsl!ig%>fP-{{e1uL+y6KbFDkEP7H2%7
z=Ov}Zf1?{_LMA}s875beR$(yu{M#3771rgPsi^2!i2XE+L#OdIy;*vt0@R5*OA_2q
zQ2h7BsxeMd6qJs1>PfA1w6eF=n9HNCDWFL(6Gu^Xs$QH?4{@(ERT142hT3T0Mu;0-
z%ENZPU0lFrVLZ<n{573#o%~GSVxlJ?Y913^hWI;8G&h;+?aZng=;=b)keruJp?4e*
zdHc<q<Id~tlih|^)GXNJ$I9k>aXDwnshr*Xls26e8elZ23<UeFqP?v-Ql>t_Rb!s{
zc@gt7X}9hR@s;$TD;evMhf=oc9F46PD%kd&SWkU@T^=_j_{*b77h!_3#}s=&d`w!E
z6xDKKT5*b6red;!%BI(wK!19cZja|F+RXW57yS-vpcmTyEbuE;G;)RCTH9-F17BBf
z_&K(}^hzAurfp%9Iy#p<t8C<R+_dUwsjY6A%In!0nN@{12^-YiYLw^DF?&}h=+1HN
z8fS<tJ{6RuPH<jf4j(Lpi&@lnzRs7Wqz0JbunY#-9aX`9`O;`$B7AASRJS*<;Ve~k
zhmS&PW`w3_Mg-4#)lfnl3--jYn;wr!?ZGDH*+ik4nSqN&E*r{kC7Dz}FhaZ|fKuYL
zXh#yw><FrVVe*NgMw}ID6~AX3&T>EWNn)|=t${NM*Ww*!l9tIRV^vWOXSciiD#O--
zt=pa*eImJ91-CL1$mF=Enczmxf`InIAwM|O(n$C(b4l4u6f5=rjHGv~Tfh6Exzb!r
z%)}52x@2MHQi>&mL}hlS5*Aq699F;0)^NK^&i^p%sp7NKUj1v`|8N|d&E0Yb{s;)1
zG$PtHW*^I~@6qpjJm$VOS>2OAvDCcFr&EM_t>xs~9SKTq|4T76`ADGAN8-4Z4Q?jU
z-v*?F{2YAP9H$BM#WnkXrRRc*MK|N*-Of0o#M{WcEH~!CobCz7=mMo`JP_W#roaHX
zk;@XCZ20Ta%ym>+Ug1xuQO*rTx!dr<(j}so#$<wSQa~U}>dd7c{7nt&_OpFA3za!c
za&EB}Ym;ul&~nT|6}LW0&S&PPt>0N^Zg$(Bdt8&n^`lmRz1}9H@>V1~qpi{xR`rRn
zDl3O5t;1r?+lnA_v@ZLoY)u}F%Z|aezP~~>Wp|~RGg;m+s<1pY=D(biS!aI9^JM=k
zbL!4IuQQtU<xKiY=Jy`CqS9(q&YiqNToN6uxYC&ydu@nTtm&|HQuQaUbQM=E*OK|1
z-C~kWEGTq3mhJv3?u?dRk<Zw25oAl6O3vJuY1Wr!U!d$A`(mGIYFTpH7ckrr<RUeC
zl3#P86KGf^x^NrIQ0h=$5G=>KBRD3tBUqIy=quiA1ko<N%Ei=hzJ2MK5!n~eysM%p
ztQre;lhX876kf^|#@xITk!hUa6fOL&aZ8ykXTjJErmz!m#^a@|L^nCa(JtLIdM&${
z)7O{IV#{oQhGE^pJa^8cTU@|NCG)@7r?}&G@I}OXgL@}e06uDp`BhAOI$E~!ptEoa
zaOx=hW!kB$*1wg#GHp-=<^6or@jNYJswg9}v66CI?Us5jsQ{nY+pqD2BX-NJ$#}t5
zL0wVW)P8B34BeKtz9tX?4_H#}=At2qngz?O+qS&UH&|)!p$qc7aG!|;?XCW+GC8gM
zl7rsVHfdtZUNIP`V8%k@x=1_H!c&AsTUaPJID>!D*eqw$M3f;cnW!mi#%ih)<ycj%
zm${iY3stIgi$d#ab-kSWG*|a4Sy$5;3`NUb(=H|1!D@koB6FPD63Di<DHuBFcDWl_
z2z9;|#g|?Q>edttZ!we0D^0+d_2s2vI~2fG7yrqaD$NAM!e}DR$w6{e6OU%2FYSfX
z6b;ilQLXEEl~&=OG=9r0?1bH)VU?%k+9FY!c2f-TTY-J!jqC=XPbcGq^Xs1->y6HC
z%n&3U{k~XwhxkXwsS2?w?Y}vY<YXi9*G2Kp;`%c<m=t;a^vSJcHWP16%LLKe`m+>V
zQH(Pc2CHrvX&RG@V1Tx7)Qmd7OftMPx}uHv*b8~V<j%DyQ3XJL8~Z+|nh~jbC?T6j
z<swqW;>{<e=ip3PjA(d400wR_%wio=SP-~P-yo-!(swQLHhQ~F1N9>8u@_thiHbt(
z>U^5?0&|NER36i1$2m)6N-JK%x;_O2@6&K8VVvNrl*KEKvh#~^mSX-W%#`7DR{#8!
z7%EOFktNM=bK*uLhP=c>o-|J9*iZT{3<Ko9K6YY^^FcAFJ5H3PGDBpTjYJ9~Od;a=
zVMLk9qFBW6)^cS9Z$Rw~5+M|T_Pf7k4SkAOa@t#BfUjWgsN9wT(|E|DlPSfvS$3jS
zcX~FiFf$4dhQ`9ZT;K%&eZa<$MLBp|Bmf}$!eq<@#G~w|u1w+dMr9DBgt;0rhIPjT
zsFWav^9kKx%|_>cJ8fshdDE16=t7ZwiO~Q>;5|@Gz(L6g0)ug!AF!hFq=&cJm_5lT
z)A#}YNJIddFOwhN04CRdA_0T+-5?yX$Z1skI}=}o*=}%RIZHDqYZK`_6P>2+3d7o8
zVy0Qn$RqKx%6WO(^lt|_032svU1;h7yvR`aZY=Nvi`*Mx*z+C`H;;H_@sI2%v|#sm
zft9+^K+-#^NY;>4Ky*4jI{>NrhvVIMC(T0rM$&}Jjenjs+EmGgix&M*0`2zOc={;`
zot;DKPe3jI7CW12W3AcBKU#0L^p6em$ENw?NwMZr^ZhgPM|;)!)h^a;cdQa?)-S6|
z?TwP!R@d53tY22Q+Ruu$J67*HZR?lSuTBZRI#%yG7REYOzd9v2>sY<(SXf)L`n5*$
z{5zc@jMK1UAumv97qn*0yy5&IXs{`I@vYNoHks{cj?y6hm0yyo8M!jCjyV!#q9u5d
z1(QIoIGTt4F{1!BvU2{nT<yQBosoBqmtyBPx@X8R;@t~_b5kC9%VM1im5Vc$3#qBn
zsOdwCf{O)%6b}{0;i=j$*bLz`a&Wf4PXn0s>}SvjR|R1BisyTxhf%!)-{eb*DF?nO
z1|<3%W(;G35sg*@3EfS@nun$_mi8HfRoi8y2ub3jr1IjSt@`ptsgDdT>$veX9p7}X
z#w>y&iK_*}^(equk?dLyPwZk&fiziHJ$Fy;Do3l&o}llfFA#*9g`$HQ8|yeHI5D`#
z^W_=dv+7PDKAl&^<7)d%GUW-hNo~%m8PUC6s8U8uqH96XbWK!F(&&XvUiS;8NXjxP
zC0M~BQzBw_c;((;3QOSR0cJ}KeifNuIG+RM*%r07X_PFvQof-qh7$(oHtJ(Wb`(8f
zZyHADAOi)7sD^3KVTYhpM{p9m@Hutj)rC)KH<2h}e}HlxC4`}NMOA2BG`?~RG%E#^
zxwv!7Ix&~ELR;O%eE)$H>fF!d7IfVefb{@Hq65y?Ul&favKLT4w=Sv`%(JLPC$o|g
zt@QMjI4k3kJBzWQ-j4rBK&#Da7$1`($o@^*k(;*Mm|H5E*ySEL@zTUtFt-o*f!mx8
z0lcL%E$N7;oM|aX_S8z4i2&M7<7R4$00M@&BMFC`#KD*VQCK#9%H6*u)pi2&R-1Nx
zu^_F6?JkOu5LV2(Ejf6qELe_8ydb*Ja>{NvYs|ppSSfr#UOH*2;r7%<4d2bX1Y0oz
zt(dtUs-u>e(Hy?YkpQwhuBPTjvruA^Y7H7}Nz{<+l^K)qo{f!P=Gdz~%!&JH?sze-
zyQDseTe(Q(_!uXI3ywFV$0|3rvfiXJVGt*Hn%EsPhWR|Yv+|gAim+$oyW#XPS2fV>
zFp<hhp+_^Pp;#}c`!`8TXH-z+nd@X7=YD<6tccDtQQDrD{A}Dw*SIG{Q2M+=wTNGf
z5fDTgIVokS0Lg5KVxfdJvb?kUoDviEZm>HvF?DSDXOC25+$&5vm>0MI;{Ex99BJ-!
z8+(VVPjz)<*EWp~ilTtE#Z-CYk@#l_lOX#!U-9gSV-P%;Npz7cMx8MB>GWWwj@)Ak
zL`SM9WM0q^Bthz5QPWL0U1dPI$O9Nl=o@)cM-hfj$8_v5ram1GDWR)8YQSi-F=cF|
zYaw<226XNDsUMFJ7#(16{;&yAlen_k-{P@DAdHz))EfqrX_j!Aql>R4<1Dctwyj!f
z+J7!dfXd#}6fzN@qME#o;<wN{DDbVe6n*td20nqkLITc(mZ*|o4Ij+HVJoMksi+}K
ziHhG=_im}(tUke><mL<HTD~p2ab!}O;&C$S{p|PC<{gfdU$L3CJ=9WzAY^;x7%72E
z<Y8Rl3mT#O<_=Ot!+2StsxlaH9UWD*sCd`dan+k;j-|Iu|F_$SZJ#dRznwPrR@=l(
znpwGu3hPG*VpC<`g`#CcjYS}%VJTChg#|%ICCQGo6e<bK0;BA|jbMTcHkYVMq@;n=
z5U^T~W?0OzNK0cLyMNDdC*vejkHhDfVbjtk&Sk%uJg2zNQ*W2_&SGo|1p)rwB~w>^
zfGC~1d{Z+_qHg8RlW5FcUN7hsd1x)3c@GgXCmlIv;g$2|iiWAF6dfr=GwLME)OH_T
z6Q>N&>&mi%{8-dLvEjY@ft1}oN)%|upd<{QPc-3({a-RaNo5s$u`GV9Fx#AoDOg;=
zTrwu@s$+8}6E0jZ?f9c`G85WuRv9&$`OE9cT8#Pb7T8vY4e5ku#l7za(+N_6o3^ii
zw5{bM><J}`q<L%rp9j;QZdLl4tc_$8<Fd3wB2m!2q$T>p?4O@~)|oA}pY)$Td-Ckb
zW`A?Fx9V^D>w|T7ZF8glWTU;>dD2;5_gv3?;;sZ?f0%jximbUJotIab9G5ThZLGp)
zb90^k!YBW?v%0$RhxS@~V`F3e$;M_Io_97kH=g`q^%moq`I9B78$*+E6s42OO?#go
z+-LHE4GngtH)P%h2jf1VBN_>N*hUi$n@$vaXgnK9TT)WjT)1K1CmTG^cRh5Ual9->
zb5v5+2y0qxN$YUZEHWhG*UF|&v;`ZL7dii4;LpEu_ader3n``=KB<>PS?S_4hAyOo
zhJry0{m~0as;{0qe@>qN*?aE1g^r<-pzP^$=U<Qhoc!z2zrsf?Rk7QBySKmF?Y7_>
zt#0`CN7$SsaJLsBlD1THzb~ln|BiNvX!5$+G^3gm0zfoC%M;*)p2C9htYKo}U!{IL
zdVsSI{4Ata$f_9KydBZ#l!4b9^g^bgt0Fa!fxcU06`Dhc@%#=*DMP~nNM{E0xr~0&
zD4Ec@Rf8>mZLNuVD?;|OfW55=oxIJy@#XWM@tN!XPX-6G?*D6>?M}P6|F5rp+5aEl
z^8l$aPyca5i80HhVS4vKDLPM1kI!~at9l~dK2Tpoa?(Y;0~iLW?10@EH_ZqM@?pza
zZC2J=>y*^Ohn2O)xe8+}OGm!a;=ksK|2nBd`M!v<pH#Y5mVdWVkr4;!#mGlb4SBJc
zII+GYH{Qp7k`0yH5)To<L}T<{;)~6bXe3|h=g#)go(1&M9cHLH3L+A*(VUBKG=mTU
zvTqCBijp5q1Pu2IFEMow$zT&j^U!m{ba;bTspBl=3ZkP)guvLjC?(gNo-cagd(hX@
z3qSD<nCNN~MltgD8*$l++LVZ3Qi^tKw21^6jniO6wxaHpdxPF;+zA>k4V7Sr-=N_m
zl<M7J0fZ&)Zt<Y7NS;S8(pQ#|HvDejv&UnQ^d!gz{MvGUFVsM4(-m6u>e{Y{At|m)
zD`N<dX_c3(%A}+fwPCIbwfp=hX>##@^B?p)C178@box#C%u&sV_!Z*$V~kvrsGs?O
z3iMcXhcwKV@n7k12;wylVr6RHVi^>oge9qX5eQ!WK#SRYB}DDiN}N$nV{>AKzVTgW
z0yD>6tCMS;;>dt)%2%oq{V1y>M>eri*~ls@^o{S;Bb(Ts$Z%#dcfNhVlKibu3<pwP
zEZ_TLslNPD-e0qVX$yqFSsQEbEHZSHjx|~=$?<y1OXfRVR<5saVM_a41zTsdH}^MX
zs7g~If0UIcdYMqZKq?hw`^NVQDw)vA-P5g|9=_k(;e2az-?c$L*j!)g!H%VOnq{F#
zavJ;ZDBj}X+wRfnF)Fdwu!s8T-T?(~T@#7m^!U16cjUq9?^;9ZcCVjqcIk1mVCFCR
z#bSNthc|WVXG)xw?&wB9^g|d69>LkP{t;@=ww!AK`it}D)t2+;_SPeQ_1t;%ZTtwf
zf@aHk^m6owHawJ@4zm>Yc*ce-X}sPG?YEZIN9ug*RJb$b$6144+vCJ`t{veSkB^#-
zrH1@Ut|tN&pE@QuZqltbA&HTv_$%x8{Un((_5IQKx_@yFV_t_bugWn$N_-IGEF8#@
zBNANrD&SAU1Qf^8ef)x0OiFQs<^53<-#o&%JHx2gLy@c;cV9ISMNQd(3EnX?Y{tS=
zy97}JKW)Yj_HQ3`&S()h8V>`OC8>}#9s^%QKG(OQnHqTEtD#{j8{KKQnN6O|kFzNJ
zh-&z{JM`T|)WEPpjcJ9N<E=v2xq+YdFF1N%3&q5Fvx<A}IEDJ7jxXKQ%AWrKzLK4R
za~m9vy8boZuAl`#8b{$dj^}F>Qz|07-DZ)^aTId;(<t^C^^;cvFNJ4wTM8<Uib)4p
z6{?~3+)wbd>0bv)+H!hXiqR&}HrT<46J)o6cEud_MVx=(y|aI`x$Y!4N$QUhI{D_o
z4{>UmY-{XAxll2}qhD=;WP%}{N%;!_G%c1|+5|@WwCl`&RX$9=K@t%Czqi$S$APGo
zI2yy&TQ!b;;mJIg<}HU{F2io@4^`)<t;fR?UXgQLkP@8$nksI4AZ|Cb7}#<es(N!=
zfxFvMa(UxRM)S>D?cbFU6-~*G>@a)Z4!x5rcMPDAr`PJrtvI~;8F36Iidz)*GrkXW
zWUn3r58FQ4BSuLkYyR~(!hl$z)LqK$h~*u;j$B+=h;ktml*XFE9f=#pBKDx05t3v%
zO?Ka(>~z21J-!E0+`}s>4(7Tm#>0d$z7Hv<Ub_}e-k-o?69P`0(J4WcvTximpvr#J
z5Kto4P|lIXm7qn=MLA`3-|g&lzuW$Pw|nxB0}!bX-ydxs?{>E_-09!9kGfCkEdMVu
znbj<=`}Xj7r@G4g$*kK=ZbGi+h92*TJgoa6FVP-WtJU7zn3`x9;VHQldyoKY!w$?@
zUR_)e4I<5SQ$I<$HFPGhh$ivzFMNyRD0Uvb4-yQKr!77B5(WcV2+dOdf+l&SjQo^#
z>^>A*Yn@8PmL~dO?dfJgO?4~f6{>{uL_I-uEcpigtgE~legeuXbDT9)^7c3_<rO{>
z$}9Rhi}LDYW1?)C*Su%Aweo8C$tbVps;rt!JQI{v!#c{U;peBUx&<ex)+2C@Dar;e
zbZ2GNEm*@ncl?&hsu@=D{0DF^%BlyOh6-iXtyERBFNA8130vsQpMs~gSh3UH8$nj8
zTDUuNCY%zjh(z?HX`iAIxy!s8N(rc<G0tiLNvIPJ3wI#_yYVHX`DU&5=Qj1_8W8}c
zH6l~Gaw|5Xem<B+gskx<XhfJA4UNc5`jER2V!2-8zL`*Us}?oh-+pM5z+_#>d^91`
zP*Z}EN)9zu3$iY?AT?DW>-Ruew^4!Ig{J1A0=X3zo4X2R+K6kYK%@d>y81)tAIj7p
z_o($a68RW%YL7DfVDPQUhHHqF3QM4-o+nYUS=8tm3a6%&g`_rCri3oS?o5Y`Tf3gg
zrX^<y*o5K=%FrXPvc;S4$c+G>G<;hxJ!e!HjS)U<#y=4^`IS4-{;(d3imK2`qUf7H
zlDcRqGMY-t=$k*1>11LvkZRk?$(Sfp#7<ZklSf^~(k83~bA<Q7ZG^ez6lNpi)=jfK
zt-0t#6#si+H|4PJz;5iZSKHFgckJ_FI5m+mM}{-ae)zwQ=iG)?{*(EQGRv64ab!75
z#ICf`F2ScZZPv>{*0?+85zsqzpx=S_<gqZ>B7WXjn4JgR2`zKvLiIcW>fvj?yr=|2
z55|jbOF4JvN5-f>C4Tfd#i04}`FNjs`v3Rb#4mdPPmTZ6-fTZ9`2Vl1e)0c*h|d@Q
z|Bvhc&(4>)a=K;FB8?D{(z}n!VoJfJB<q=!$W=H*{V4X6!1KeD153)MBX>OZJ?U{s
zu;E)u>IAFBaqJJG*v=GT7l@J+OLS2c8m2W0_Ai2=moN88MF;rL-#zEb|AQ3BY1a5J
z?I)e~T1otu_80#D0H25B|0@_X^*R`3BZmnE|2X|93n}Go=YK(_e!X{!(W&2W{{Y`k
zmd(_yg|`cKl2%nPS873LX)S<gE-t1w!2Y1WNg8MURP?Mn^e_EkH?e=FTNR}#Yj$G&
zGRj)s8nO_)HwGq-65KiC!T#aFJ3`z6CkT!P4n^`F!}s$z%ElEKdUt$ycGNu(<IgwX
zd`n)@HoEL8B`7lrOsY&lMFB$b%~HE5!>zoDoP@%o0N1!lVDd3#b#w)TJEw>tf<6LF
zK?gepr3j`p-A)?&nECoG=7@HBah6i_5Gd349Sl?A#!dqN^e<ZI^9|LmqHO3n7w#p+
z>BTT3lp1=NHJZ){K~q-n&i}4CX)yAW7J}1FIYF4DzU!#~hD93UY{A9hI<K5Hyo+N@
zIcF!k-P7aUUE_Yum{pq0gyIDUF()Y?qZR1Jd2s26GDEZ=0BhPs)7WN*+COQ86jEJ8
zd?U>RKZ2RBm@ygBwrsM+q*eFUrDWA^oCf`D2s1#-g{~PZy1!V`f9`sBf7R^0=zir8
ztAMoY0jY#r3Al4>z$K4lW7W;A&9ga*3^=hHYOc4FBpcz?u}I4<CXXgIX&F!Qq)?o6
ziWD?`-rL>%-S2nzi=<mY9u;6ZRbU#En#-*VgMg~qS*zJjXG%MrDebJ+Y-epsJ8P;P
zF~f2zuAWNdLD_g5#c7f0DSn6>oMLSi4D!Xxa?WCKN4vLBv4T4C+c#_D>Ng1k@66C_
zXMWAD&Cu-H{F+^#q1pB7W-YUlvry0l8nW|Fji*V|bfkp5n0GiLA&+0Sg+Ytlg%gwC
zHHrgXaq==vQ<wkN`Rc2ZI)5(KS>&PN@lMh(hr%jdiU4Q=@6{5eJ~P@tKN<B$xv2YZ
zw25>MK{)b7WIGVQ=;b$w<PBE=JeAS#V)sU~4Yq|8N(Fk<jR@4%;_qAeCWXArE$(Ap
zV0V}k5iszypsfZ;LL_DMqBM<0+^eS5WGFHQ<yq2#C1wd441IJ1apN1>u0kbIE$Bp}
zTyjf?7dXV20ThETh?CT5h!Wp|nC!E06^G5kxOSAdOg4rODTKD~5g;)J;I3%|gY%cq
zms#dD#)guw7&YL50sXl8Q&WZ+u#Y9OKw%Yq+x}^*oWvK(q|ZvUNE~yytqKbNy-IiU
zF*R!@#bGY%hIL4D9{2Ez(vgKC6_H(4*P;seO&ZyhLDMnH#9K<$HL?7srSqB`2$1!J
zJdOconVAtcW_{FO&Qe`X6uit$Q*t*@xfvT|vkuGb3$T&9_2$lT*#s-tDf0)V3*}o+
z;NriFJXYrIJO`HVP|CVNwDd}FCGkt}%b%9b%>mBN?JdR55CT=lUq2Ew;9tn9PA#1P
zb{bCiBDiQ2i46x|tun(>3GU8ib`zFiWKRQR?8xLA0W`R}<pj<P*=@<?oZ#DUb#f!o
zso*Eh24p7Al_5AWhe$4V=agIhd8`5>Zz>|p#OAq?6$({tdWbUPnwqe*^7xOX^rXl+
z(T<hp&ex7s7g0N3S1K8rlMJHgv5+TV<MH`;Hsr{BT7_9^Iig`~aRxMEyw$IBh(z3S
z&qQ-G;*&u9?@Om{4JiMZI&!{c4uA3h%Y>?<I2ya>Zt9E0hZj=5lg2KHP#hB&9^wLJ
zjUBE^Tt{L-(z;U8R)A@E)N*Xw%TY*TdO6}8;sAJl7iwaE!N$MMqk<e;`qq*2vtZ27
z!tLj1Vdhy{Fy?7NpiZ%#g<kBtA7t83%Z$W<z3aJygJ40G_M%CKueaUujHL$eb4$yb
z+rm;Zb{#=#6vU`NW`W*J1R018!{A60C-Num7L*n`-}9z`ARMv+=ar<!U}*^vS{F3X
z2o(E?Q!--<%tfj+=9qEU>H}{WVq4-}$Qsw`Pr8?l+!&h1B7$X+>vr%H??s|p!Q@V1
zDejJ-qM|@$u>*to^Uo>l{Eo8FObmVD>;Q=84aegiO9oY!(LWHMSN+1tQSyoG#H_qU
z)xYyDFnyn!<2Im>@VV{;6|=Z-+j*3E`81%rZx~-`ELUzCg+u(}7t6u_s?R*_KPbUs
zi2>jg`@hZhlcN2{#uxj)2l+gt{RiGVonR7CRFAqtVEL#wMGYi}tl<ou2`4H+y{WQd
z6PZGs!;^0k=Q4=X%pD>$QI7+)+SiM~{yCNLo87nDXZxoMoi@qG$2$26Co*<DFZPp!
zuP#+GTlLtrZYuKGkzI8)yP$j^&7`YP0ND2b@xV}y8M(;^zJ)9HBvQ)vsf{1LeS5Nd
zYBw)DqKeH=Yn>9-`XG*RH%8eo4aP%XB#cR%S1+BUq$4}Kd$1Y2Yb0hT8g%=ZYq10c
zi#cH59UlG-lW6aq?rrby{d4zNrM<&#5fc)%bf&!rdZeHH{wdwxd`e1oOuP$JB-E!1
znDl#i-W55?#Y-0T`xkD!@E8kt6jMyRj8QPes!ShU`n^>Etf*(J@(t0|7NVu0p3%{U
zNVlw3H^l79MrUMC@!drfL3h#x^A}*jg~b6Cs>>8@aycO}8s*{_kkzUyCsQi3?QTgo
zMILzmiHB)Yv-7jqKM#OE#tqXkSR^^;M`rqD0EK^z^z)C;y=*j=I&OxE_gS}qDuIaJ
za+*#w3YM;frlGS{h!RKF!A&Jrq;q)aU!kK`Y#CRpA8K2&&7>1(j*;=JA4l?gKN^jK
zRQwX@sVj$h6{D$j{ArFMH@ox3xuA=(&O)y=sc~mg-z)9UJTX|eM~EuC(Jge(SYYEU
zKKCE0cPu1WGduLdbIO};^vN{h|D3KDEv6wC&=7EwjmtO<uRMPY1B7bxxH=QEP1(&^
zj-f49b`I0q=euS_kKtWU=cpI^p70N3*^Pkc`Xa*1M>?AWKKa<^JgPYN;kLef@u)_@
z5jhDw`f}d?6h3o5|1%v}ZA>};cRI!M|C6=#%`fNw2lzbP`Cs}&_+d6e^)1i|I+gXi
zUXWnIc+bIGke2ac90wybN5U6v^F`tVxApMVOQ+LXG~NsY*q|NKq&jPhzdP!M;c)Th
z<fMCayt|_`14X!2Sy^o1k$V5=Q#ub8j#<uQHKNcR`N@wzePX6rdR7(`&)^5RB?_s<
z5N3=CN*lVen%z=)8(vP9#qzSWDN<F))05<9qrX8&ts9a`aXcd*ghr%$vsl-)B*1Pj
zCkI2lPgk(}a$oQne&)abj|x7Z)BOL}I~#@le`9sy%l`iWpD+IZpMaIJwQu42Y1~51
zQ*UeX3XK$X4PSPa|DMlW`TvKDvhXj{`2R+GZMAU!+wOef{}1tb`17Ce1p;E}CiHL7
zuO6QE`6V6P@VfGpOC;zxgJgH*_-=Beb9RnJXyLI-F^}oaM?xh&hb>*W2{h+ex;n}E
ze(e0z)E!}UFI4&9-(Ym-VlDGmb4xhAK_B2h4r<`X33&rykKtH@0ptVEckz06?8k#B
z9=TzkZK9yAx()r?cio-Cvx8Id{3r&P5}<SG4l|#eqp)wDKlD?-(R3QZX)|$D^=291
z?{|-1AD*y38ehksUif&=4*ya71uXRXZ0B#gr`?_1{rwZC?+=I6p6~WA9K@fK;-SUq
zW&ICmSn8a@DI7*P2vYL&4dp8eQn254=n{c^^NaWM`PR^FZ)bbwyIoSZE*Tzs81)hF
z@;)Uo0=n>>MhIexaBkD^zZVxYr7S!Ka@w+=cy8)iPIA03l$(W`QL?w&nUS(o`o9JA
zYq(0b%qPhO5Z8yYX9G7F+RuX!?b_^Tu@CG&(K>hR;8`NPz{iQ7dC^krhF+xnK+pY9
zgvQMEZXAc_6x7IPpF8I?N-d|r05kEL?q~}Ov?vT0z4t*#KBs71($8WHLSz6a&~Adn
zE6JY#Oj-g+&4T8I6P`3$To|5rM?olnZrLxOAL<2-iztSANHODs1Ou`QCML$GIn}~r
z{JX`E6el7o!mlPs9vM1l!Z56c^>77nRJ$xZh6-DsoX;K$sxVlPd{KELbT=*DCqP#H
zaG@mC5qjoPdS-*Q6T-wTo5j5)&Xht$^Uv~jv{M*9BU)=%Z&hCdx&<$G5K$b2p48}I
z57c*D^%e|TUX?>OHyZ+DKFGt0{ZVvTH7vU-$4W6fhTjQ&6&>XojoAR^fT;-pjt6@+
z&0V7~MCnyVuX%UYz2J&sPVKtw9J)GYi$YAeMU%2-?KZz0p}u@R{m)#_f9dcx=f6!j
z^%c*58&AHR{~qA;@y>tgkS!?^{k7yIh_Bp{?^$K_gYp`%>-yJr9qUnfExsP&$eKyv
zNqHT!v?})rol%O&h64%hX$uR4M*HZI$2NLj<smHKA9aW!v(AObSc`qN_XbWQq{yCt
zJoaa4aWE11zv}?20KpAKdw^m`-GCm+dJ;VJQWFQND%ERXI`Fdb7+yD}s3SLFDSycg
zUCJLK47+}9YXF)BT;%Ae1~zikx$wC0n1D3ZRb*u+&CV6OViTvHL==JpcL=q-Een<D
z1?Hhk@)JeIw+o3pL*W_43*z*qJ0inY<BjL{-J32(#oN+nu7$A+N0m$a5do0rHuTAd
zodHK3bMO<4oHCBa*^n&xVO>Cn1_2-Uh$_%qo=9PF!CG<XO@-eC=<W0K*e9PWu8F5i
zGF>O6IsGB}Ucqqy?MXchDG1Ux7q!og3(BpaOV!OY?Lx?#TG<<~@(rqA`~+u)-z_c(
zD9{5nm>zxk@!<8|!3jE)i6TQcKKE6j$3q@5jyD{NA*l#6JPuLuA~Wk<81*hn>*WZ_
zAf+N=%;_yo_{dLPR$J(2GP<x;j$#YjrBFYXCsmqsqSQRkU3e_s;kX*2p!2O`MBPzi
z`;^YH$enO7y&L2P_EF3*)4-RK3q?aur%}OQLq8mwM=>}O>jLlt-UFMEEg>jOKhVqB
zJQSN#k)MO!I2Q{V19OFVL^cK>K<4^yR51?ZSlt?r1zwJ6;&}MHg)Qf!P~*c%L8$3{
zjTqEcxrz|fLPvRxOAX357;`*DY8X5K(Q^m$fRYak8lL`)-UxN8QY}Q(SM1fTyiOM6
zQXiudc+k+#Db4r`DS^Lw?!0w_VHRV&Qb%?AUyuHr{OeK6Szcal{wsXcQZ=zl%`FYh
z?L~3gFrNHwc)rNorK`;%UNq71D2b#TNk#%CMVw5jrqEEtK-HKZnRqGoE{_R?fjf<!
z*9NTsN&3?%q=1Z?P!y3!Xcn;;n4;_>Gm;AT0Z{{Jm<=cY%{q?%1SQ8j&;M*6KXd%|
zjkPD6h4cTD)%7pu{|ESdwDZ5|Op(~G6`#}z8qYKs2|OBzPU5}M8Ab^2W4J^(gQ)8p
zI+6Hc_pvhw{Gpea*MatMIrQZ=inGojjz-8*`xksn=8{gIwno5#8{V+oFU(hg2|tEL
z=$AMd$-KH4R1(f#0s^`35=aVVj!olf3ncY1bQbp->@dS7$w&b9yx6*gLr=emT_<w~
zDJac8nd%Pi#E%1gZ`T+{*b{gHN5C)&m$EQ`lM=_Vr1E>AgIjx_{EZW{asn6#9dxRK
zQ)HBdbfOb!i`m~B&W-~DqNeTr*#XA4hd~Or9yknUsRKs_ioFki%WmDA0rZb!bGrQx
zj^X=lmd2_5C4=tSy=>tLvq!OnxHG{B?_m9Y+T^gGlz=&LM#470_w?XT?D3A=8%j9e
z^Bu;M<6{_Y8F00i_+g*u5RtOPNNEuu3L<)S0fm$CFi2Bu6Obq9fV}M!JZ;H{-U-l_
zJq{op8J+Y3DFj`>5s3G*+ea9Z^u^t$MFF8P|9JdompNxJbZO*tQ{5BJ{Q$dv(;p)2
zKGGvnY>Zfk-b2q~0wTQnp)P`28U(|kK--GadqBh7j@ft&ID-{4zPL#!rmO~}JooH1
z(z4q1uK|1gAO&KD*~YBd;+IPq3S2>Vys)B6KW3;2VweCVZW<)8_LD*vLf`iYhzGkS
z%*SN_(4s-=pa-%;Ud2gqgsRs$47-d=3&Sw3OLrJ}6n_Uhi24CDpaJ`-B?QEa!f#SX
zc%u;~5ia}1KDfe}TZa1?pzJ3NTw2^VE)Hrg>irCqFZ2{+DBMl^={4Oq11R-CK-&+(
z4>bHJ=1zPA+$jqa)=2R!IeW*k5ecrP#N>s#a2H7G1GE#PYT=i-VeKc)g?^t3V7sui
z02vrQDAxm87(mx6Nw^{yz>)~uYeX+vh<gMz-$L?h61tvhrD}MLO~4zXi{k_n1ZgFc
zNaQX2GVJ=pQN9^?%1tc%UV=p86O~=B=(+0k?^?qRMX_G4sNBTZW&aaXx#VB{&+e(b
zz!%-7U%<j)LxL{@SPyZ|?OYhLOsdxRgJG`LB47(hJB=DG3qyh=?1yQ~mUWty)$3IK
zq>f*c9ebtp^4J^$KSZaW=^+EGdO+k;tJ%}Yk{3R@O}%hxE#jEwE8d=Fucw&)>{Xe&
zOs!`!W(%$QMXf0*&LsM0l?9V1#=s~7z$vY)&niPKQ&yK|tg>!NXo=k1POTc2RTL9k
zS=opmP*z>f^JEBxm<S~^J{f{5UC;{E^@LYKFo?x@jDO4EP>}Q*6K_srwZEMTR8bjx
zq0j*>1i)Tc*`4Z|!v3l<5K4u-;XKxlTKT0SWEXl_-LkOOD(~%`qZ!~{aBx~4&OlAg
zVsRl)UjJ4s1AMadN}fooBZB7Jna;v1nqekT(u#;^J0rs{Z$)s6nypOR9xB=h%Uht}
z;4^O}<!vbbQ>UG}TMsoQ<A&;qk!+^e3v=z(=}4`Otb#mbFG||6c-dk(BfL$Us`i9<
zxVPUdsZD#8IWHZd|Ebkn-K{m(5<J3&NFP+Eo%-9b-4I>hzjRoYRI8y{TXwFI5Ss|=
zH89K8ZZ{*8A1@uLov72??4p3(sMc=0bPSzAE!vntfXI;tD<NJwQbSOuxmyVwcDw35
zG|kF{It|}SLb0S1OI(>Ep-_*@SG>_EvdYqRD$4LiS1##lVlNa4%>rTmwL#ikMC}k0
zq*E?wa`7rDluN3d7$~<0b03^H3`AT_4qO)_FV^IvHKjh9jbCDND)^e|D26$US@F@_
zQZL0Y`|^4HKHR(fD914JhhD?1R5>6jX~JkZw;$gGDOL7N6shcwcwf?oj~CVIW8c1g
z{Leh^f6;5A%m94y{cmS=t=%cy|E{fezTE#lz(=`(I)ez^YVl@<T;-H!EoNbs7M1U_
zc$CHHaJB{;_0a88hB|SjGR}I#pub4h3wYgymp<EoUih{>9uNKA&9dX1_`Y+IrsL%K
z%F20=USz#x*j83tmrJZ9BW&G|QJ+nq&|+{XQpbJ?R`HQ_?4OUi8Le~II^A@E90RYP
zZg$r?MTal+ttOKzS=>o{tzc4B+nuMyy8NIV0&n`#LfDDBQ6?PB4Z!fK4q*N{EBH$1
zw#hI{JmBuEgZgg!<U7_t&7D5b-$3j>bYDv*!P>t#FWkrhFVY0WQ94AbP-wU&f_RGK
zw-5S_Ys|qB!|)vV`s$U_SnD`T@Sq}}nVcD;oz3-zny(q!*<7F2jvmjspLTn&A%D=M
zu7>t_%)}mWgXA?FdPB4{pfEDP$I)FtXK2a7Ver8p-k8zk$C%RuDI^*wq&1h?O=XOT
zsa#}>u}{9)3nPEjAKx^-hTkH*DGvN`voUtQh3+t2ngA)8i3*hJr<^HCBMCDc)j_*P
z2jE_#gJ$4f(1DQ4(7|i;7U&>=cSDD!j=?Cnwy6Dr$9{o%W-gG}+M8SW^D%V(`ss$>
z>${u1EvvwqF3|7vI<CLAv6Y9Qdv;N0mZ)3i;t*<v4+!)F0$syaKVYkW`+sMR)$74v
zqwNm7C%(6_@wDHk!)SHs{(9f<JsbFK7lzc|SSyy+9j-VM*?63BD_K+lAk<u=>jb6^
z$~|s6W^f1u7ZUj<P!<3?E6zIEFR1xzqe9DP<B;5P@8dfdeE*r>@iy99#R45F(C$5b
zvbOPTP1q9jBJYMs9!WRPK~!f7yrl{I0*>ih+2hCSM76ZSi0V~JrzKHh3tkwBoD{4N
z*zqDa<4S<!CV^XkVlcz~_Lle|>PU``{@Cq}=9W>k!xaH(IQLxfqEPVyE3P$-x9!Ip
zFs{{We{*Afv$wgf@+T3(P&O?=*i-iYhj(SuT!yU0JPuarS#lMm{R^jo^Z&K8#@OnE
zH14cD$8s<hYz8)J+&E|f^XrcFx=pX2s<&<H?dnfkjzXwY6Pu2SNsW*J`t}9Cvc(Z1
zhKmh-G7{4Wm)z*0-LzOno0p(S<W{Pjn7>q3vPPX-FH+`#WvkS9*eQFsR`zgx>LTx~
zJ?lN&Z1*Y`xeLenXHT9!D=Dx>1=gPS*4ABrqolw(LumU@X8vrVy?I}J;_*7JTBccD
zsZ*m;J6~z7Mx{=^(t3?bYk6iiVeL6(3vU7sd)B0a-@?022{+r=Y{SlpG(#QeNZ1-f
zsskN>w%MdtSZHkvsTVsSIclNG1y*^|q=MLiwJlw)BRa4O(uMnE{fX$nnuM*ftvawK
zpov23TewoN18brKg(|3=Af&-7?7;e#F1IE+@N5&t-$(g{I<PKb>zPv>SO;iNoAgS|
z6Rb+?z`E!_p~}TNZzEU`J0P|jDz`2=u-0+=YrXc9d)QFKvMOvS`K6G1v{rnyUa=Da
zcJ)@&-gu5B)n>Dv-;CPpc7e70cC=={=iO{|>#j4yz;P>F)4+Kz?5yk7+jfD~Libm#
z0?xBs_nl|<`=`456p8#U)Xfb%eU1~R8hDZ)g7d^Ku$do%vuPLD$PZz|e!p(cz#X^)
z17Ekq4QH*;eOs7u3erqYoN@B<OirM2R?Y6;mb=yzYs7&0>ze+$Zp%H)i6u{7Q{NB^
zf={*;!)lwHqf%@#Sg(3c8<FOS>N&}@no+6eEPLLue=JA=(t$ZO^vM+fQtr~V4k|Q=
z33ADHvE;g0a;>HeIDZA&JWp#{5TrKpf*?QCf}q(jy(96C53x_5^w*yBp0(Q;DtpZ%
zSv+65qe`#$2OF-pzSeoN4oABugXUzhjTItgVjI5^`%9Iiw#rF*wY+8tYkZ_H5!T?y
zuce`mEv#{+5@D@SsXa}Ef*QEU*V^?(D5!!WROiVQ5vt?${U@DgQ$#5205Hk5HO(a3
zQ$#52Ku7dhExn=$wf1!V*@oNOxT65^xH&<5(9^eSKkC#L3}~drT5m`pp-D@A-4FxT
ztBbG6;;XjQP*J=iim!Ea@lJK|wkV#L3Mz`PisE@0p}aWA3tT)WF;tX(DoRh;d~48^
zmL1yVO+67!Z5TW4lj`D|#VI4XqN4alamq-{s3^W(oH8vq6pODFr%a0x#p0dflxd-&
zSiD`FvUYXpReQ>A%Zp|gILxiv*rtmj1M##Hk%3rEa{?S=YtL`sSoZhDd%4ruKH4k2
zzfpVt?6+IRn=R9b6`9wIm(Je7Y4`p14^7G<dKSVSmu4Y4)Y*4S4rLkA*GzmhW(O;N
z`O;Z)zW&<zs&F0ldhcC#_u$Rm_5r4Sw<H-FNdx-bq;YV^8^%`5y{AjLVd68B*TMu|
zoF_0eBqd8rRtlGk^YsEFg(Wx<fl5GI{mCfME-BFd0C_-$ziCUSlmD10>OQt5#pn5%
ztNl-Q?$2)jgUMcs@gLSYYhUbt9^muEq~=q||3b0ccsWhUSthozj2)(w4YKl*BuL|#
zA(_S>`E+LjK6x7w_*iB-?#e*ZMl(hRRGG1M=oQ+?IAeDZwsKG0>&j8ZYE)erjI7*t
zyH?|s+1)sUPD&oxY=Vy8aL+CAFt$=mG^m>*OyJLG5e*_@A;zRg*H<j-JkCE;#y4!%
zqu*BH9Kb6-TPp5y@mz9R?c~Zq1j}F+t5|NMOlOwY%(VJ?5VqO(#Cj7P5Obhne=4vq
zQMn7BdG7!6Nmo8uzE}>zDf>SF-&oD(e_d;@Z?wPc{}1pfm{GzR2$sV3e)l^S@UHA#
zNuFml?OjaNGc6Z%S*`w%suh;Z+3E52&aMXcBDqOcK49P&kT+lHBS*f!2Z!A^@crQ3
z{%$Gv{-YzT74_qTM%Z+I_S5bk{}|%?2_s;8XHKy?rmj3Hg?QLK-aXiUziWY8URD_R
zYXD3NzIBsqFbJ;yD&S=Hycl5U>{r$IzmDVJ63(o;Uw_RHWm?;e&PkJ3XoIE=gSG~J
z!E}X36uJI*elI(BUKZxy&F<^7ceFEO2w*WV{7L>pvLpCls|5Ikd+B3J)H1*)C$|PG
z`ixPBN+7@8+uy${=%M@TO&4{oCGhvR|M`zDeEHjL8sRXn@&`92RvG?+!@Ev%e|YeA
z?_JmUHKD_}$(Ho>==kvMU2#GC_b}+4_xlwsbocjOzuVcFS3_BN6@=b|w$2XzzIX8E
z*3FIFaRS(OhknxcOPYDVeRKl&-rnCm0hV^hL5;Ey+AC{?BB9+CJjdBCLO(g(KE3k-
zW4XEn`YELVyEFKUGQ6CAcW0!e!!ned?%x?98Et4Ku*LRwNAQ>K@H{Rj{qMK;-yPqD
z=tUFP%xp?p_<{f4wSnL=scK>G`;)me!RNjRw)V>)%F3pi$g_L+dVlZxy~DG*(_RGA
zZf~C+zTew9F$J5{b1`Od-;!%`HM!a|quK~M4WhtGY26a9jl&a$QSPvDcye;IhuUe~
z05HxEu=TG%#i`x<=i$L_u5T;&PuGv*D5jbvFVTCq<lW_{OZlYD<|fD`7c6$NOxx`q
zz1ve#;qhx=yYG*-cTVd_IB{>90Q*ZkwZxFbB6hKpuPLl0&|2Rgrx^vt?D<YFjy_PR
z$26j#lTnD#7M+GrYcQ*9Ieyw-Znh8_hP6yp?#6x)_p>4F{hmLDq4{ASV-QjlKxj?G
zloV}Xinu7FWQ|FNISD8}r<phqpq$4T;Gga{FhnQClEVmtvF~Mlp93fkFvz77g$U^~
zNH8=eg>)PYqpKFVPf)oF_=4_y!H^SIVx9?#_Bj^m8YpKS=8Xgy11cam?TXkRkDW_5
z4lu-~it6cj8HLrPoRkRWIaI=WPEob!IM1D<*oWPpqqQzc7_{=;kfS#`<0SLFXek=I
z7<Usd#2H@c2bDhLk!Uw4HHgohH+Z4WfnT$@tf<sCJa=|k8$?&|cmQ%aJtFr?XE(+8
za23r;eZg~>DNM6zq!YX+r^);=ORIQiI`6^yj-x~@5$6bl7*Zq<N|-`Owz9a&gb^@T
zzd{k281_Wv{+L?PtcPfPJ9N(zrx&D(fZXS-QpE4+bFuF5$LezcgRw60$FI*xNr+#=
z=URU#VTccXN#Wz9;^VCT8q&u~gvv<#p}UR8I}i^~{<hGdySo2FUl;f^D~tZHfJOYm
zdzPSzuvS-B2ZOUBJXM|ndW8b}@4DOjd+!cRkgL~g19a)5V92UxRO;RCX`@N8izRN%
z;I_%w?+1f`T!$%!DTll)3{ryXo*eD&?7iLFebb=QO^!p4E3xs<vqqcl-uadDwNu*V
zW(f{ePc={pb$S=j^$Tx%yL4fUPG5xv{<(X6n1?SLXrRQ=Ec6+xG>uFHx%nyYY@;Cw
zyxAx~G$^5o4o^>;O=rpJRFPDT7W8+434q#JC9s&D7gr=MbYCP15la*HjwBn8qd2wZ
z`gHH$AKmwR2MY_0_8Cqi%$j`Gl*?4*b=9$KHC+V>r)~XzXWIv-XYZZfO$r-D171h8
zN2Eb`Lmfd#A}>x;WFi(~g04+k5<_0<c7wI2o87J@CUxI;_kMV~i8GGa5lR>hWMN?)
z-v|){0kSL-xSQ+U?u?b(;rPOxp_bYAWbDR$Y}4*OKfiod_OyHXtlOPoWM*^j$SY|K
zO%RHY-*<o5T%Q?(ZakWRina1On2+=ph`$(*XTdvH8U=9npS6ck-2O0grEzrS$K(FY
zJ&c2-e>rolWbN7N%#}{Qn;q*ZNQpC78v1LU87h^md3cXU6X(%c_#uq0Ldq@7EJyMg
z4&n@_tkfr=ZcA)qHx$bI(c*&hXlV&W*`?w65=9HnMvsh^b1G`pJJ?Lc&7kIh;-#^t
zh$p=LE;J;wQ!MwG{pTozwdB5{6I$S=^8oP6S1%pc;=&U8&h8$FlJ6R>+N_{=@XPtu
zF*|^p-~U+;^tkp958jmzO4vqW0b^kJQ;rJ1PaYCQ@t-~(0-7I;IU3a}4MqRb<It8|
z=GLU2>Jh*XIe<d0`ifzQJK<)Hp&;~0BayAsK-3SlbGp|=#<UT`@oK;A$Ux(vPwCG&
zEu#z|?^4Hf5u|@Nfo+sd9eb1*A;1k-WORfxR2oOagoihF`?%vmI|Km%EYanFXC>%c
z7IHF4CS=xUoKa<<8BD~k-uL{z3tfbzPFF;*?i#c(8z6N+T?9r+1~1r$KX$M5>2BGQ
zwN0=K3!Cb>`Eb4XR8Fw&(!}{y!!Mm+TiVra$Z1Zj)tBHb+Z~~~&ONe^P%yPH(xdHn
zyPOMylWuYv2J9)Tx-bb;(vO3&kja>zVl9HH@1TrFf}2U|gcqfY9bXu$Tmdl3QtuFr
z1w2wp1f&F&heO7;r#DD+ZV+;M3VxqkGuYbpn>WXu*V`w%Ha8P1&kW;$3kesG>|?P*
zYGSc{BIW7xO5pNSYR;){&u%^6c<954G!bShFokN&DJH4t`NQZcH-0*`yvMM=1`=9&
zFKbT6Mx*h64PCxTo)$&CM1w)%<GM|+d>`{;pw410eLj5q_5_Y1<-L{^RP43hu9r$X
ztR~Vpmrk8Y-8D6x`EHr>n(ssvHmPrKNaF*9u9RF$Lq9xEFF@9a-5a?sQJOk^v(x>4
z_xNP*@SxjuUO4S%&z`Ne_YFbl{Rx#|fyU6H;<1SosD^;VXDO%4e@RXem&y{H8NR1W
z-?q^tfHhzWI@edwH4vjrl6u3Sr^`rDOnni3`hNR(kAewpA0Kc3qk-xh&eYf9d?QT<
z(2P$xlO;wTrPgH(7JSu&y`aIRn}v(==?xHceDb3J@>4CfGwe0N$e*STmKQM7%4Hx0
zPb(u8{gX>2;~<<?W*o!zKBY_=^gsAh)!qFM7gIY#DKvx}Fs<w;iqeZD^~Y07QjXtg
zy-TMe6~y-t*wnOQqb&8Wr=l#GI?to2R}3<t-V|CujpejLy&$D~lqscP*PT`-ri?Pv
zs0w$=sZ)Z%Nv9QJqw*=pCW&h*BnnSC<;o^=W4sXwr;jj>25hX9Gt`mV@Q2X5Ym??L
z-02H<s?VLKvmS$A<jIaz#e(#@+ckMpZk5ae#9)Jy`|^4oeVGj83x~V~hn&7omT*zY
z+a7|CDt8QFoKL<E(Kzi=B4lN=udVDB9^+fK(a+Z*YknGL#k&EJhk7Lp@$scbZLwnP
zd%s3yFW73R_*~fH6kp)grFtqabLdS-D<s*n(GY~nbck<U4>)6FG;nr~&iJg9hZgq7
zS&}c!&~?T@H-l}0tZ?KZ>{{rL(hD>n;5%@xRo)7}>mnd^sk_as&~XlrR#LZ>`0-`X
z=fsZ`FEsXl$$ZMU=Nv?MBpkx=g`7MS!xTYunvP=xFZ9un08VSe8-l&1i}+3-`i*HA
zhVHKc=Z(a~kC?pD=)dyNc&zZvU5EY9>m<K+Q@00)BQ&N+bh^$UB>N1${zBDTHuO+s
zhUOw!O1d2rhIrg)T1^X=7oy_@qq!dsj!=1d^3#^+3SFglxcIGl({RbeBnvT3?wC`g
z_9F_F;Uv+>XIqi63wvl2)lx=-HM(wh@5ckX^Z2|&MX2*|sCPM)Yd;lPFdgm?5eel{
zO(Z!`WgyZqW<(XRl;*P37>3oQ0}ux&Nqviu#PB)s=oob$6m#S;Q}l2sk_Y=tHIxig
z^*J|g8NGoAZV&B87_xTwaaYj`2X#8iZ^~U2PGu!I(A-twOH}aDc?|15O@ObAyVcc=
zLVc@Hn2kzvuIci~|EyLED>}u*!(C%IdXV&mQa#wNKFZ1iu%j!WU5&2BKQW#NuyD`v
zBrC5jEA}s<DDjK6$Z$~ZWeTxWWfYvpzzfP7iH07uV>}4`D{hz|F#i+E8O*`aANp>X
zjSXPAqMl!;a3DrgS)kVYMBuTySXi^1RjhY|rd$(BD+?l6P{4gYSfs9B0+YeF-7!K>
zrxfa61Vc}j`ErW-a*DdQQ`7^gdA=M^W<8$V?sOtg3=eTSF>)S!;ZOhJ{HccC_+eRC
zJ?8WY^PR<li>sX}M4A&Xz!wy)r5p=CwV!<cSS2gbd8=5APZ?86RlbIPWVv35E*j#$
z?1?RDmlPCznN}`efcsK%ER&t4SEVIoi_;2Ll}I{S)$nwI@?oj>6)#--O)&^WrR<L(
zpeDwi$e#sD!AyWqa>|*ZPQ^i|@{Gd)F{sZVO~VQGq8kk0(2}6JPq8b-_zo5@Irt#8
zp)k`%FrMFcZ@S|5q>jxa<4kF~W4YQEOV&BX0<-dbz*Fay^W7PlW&E(SzkPCo;ks0$
zdZ94E@L3pyOD$#0N;y354uCx4210r9qK!#Gw#-7wMKDM)l-9MQ?8>^-M)%$E;n@+I
zs0kw)=P%Bx^Blk~iEzS=$>ptf&f_u07p2x-l(s~H0#*4Tx@r$O-s#k?kr{*Y7tt;G
zkh)r0amnF$n*#0B5FF$${W!t6FA1g+{pVg|iS7AiSFG`jB99}-B4V-N*Ck-cB3)KT
z`wNLU?JQ8zJ+@!B(Lv6Vgv4*LX)_TE6Y9e66B0yE-ZF^d{;ko$Q-!av`=v9b$}$?L
z?AP^&r5oysP&<V3|7$e9`U)OOBFz;ed}}WYHTM&!5xpU(G(v?QS}mGgQOke^PdRh`
z;<SN;;J48ggDSbkx2hj@p8+qb&+z@*JS7<||H>1~#lKZG%_*jhx7G}o?)Df;SjHUm
zQzLGksdmNh7(Y*hA*6yY3p!kaXXe%BG(b~3Rz`B7B_f^|j={xLKB3&8MB5V@%Zn?s
zGt*VcMg^2fOteT8_r$7fPKC@#t^$)cS=>x4DXw4x$s}PSF1|I&llh<iX;NS&)BBba
zq1eu*$b`k&pq@<J+ayuY27GY(dhg(bb6PPZ0`TIcV{smK+fm>s3b3)Sq@w^6aOVpG
zmIVB71-f*VzE9zb6(NkHs}4Qv{>t=b=f;vXUa01H>y*!z<nyK(uyVodr%iX*gh_Za
zZQjy`pF1Ha59zIX|CUwE#%k{1GjHJ1g8&zX=-gk-=l8LV7k6=lac|qWb+~{A6`fUW
z=yU|ht+nm_zi<EJWWtgwIfj^gRxXwO!^6Lw9qIM4WfPvULm0O^3O_6nNXNLJh(hcw
zqk;)kWRII=gp-H%isQB;S1`Ft$%st`Q;t7ti<>Z@>aA$%X%+i#GPPHVr!keh5L9Az
z))IcInHQ$-qc-#b#@y2Qve4^FYUIBXfGv2zYrA+&iO{Z8Lo^giYR&?p;w>LvMigDS
zt(Yh`?J1IC#Xg-|Sp^a;Rg+YTjkzmt`dJhmOICoQ$_e7*l&+SnBH6B5Hw8hmI8xP<
zsZCdDfNmkotmfpHSGN#j6phuEVpNSe2r&&)@FW372^7{Vtk^6OZpvipylgY0P+clM
z%pgD9lWZ;4=v{>CN^&gU=<T&yvbZaEy8Oz`lYIG#WdoNTE(ZfAGZPv%*|J=b9Zu{g
z*)VkmaipE*(OW8XqTbIKw1q;h2m>b_jRC>1>B&@V+>20PBiWV)N|5sa*bwGALN#0q
zH`5i&2GwM|lPK(I*C9M@VfuhZ&YcM8SwrIeP-OQxjc?ehj=h`=!zG=?SWd$U4ncFT
zuMMEE>RIoItOM{1i~F&2j3!bHkqo0OCokne6%9;WVMVS@UZ!3Y0y3_A3V)QkAJDn)
z(hXDe=woX;XMpxuG;uf%4DQ4A^W$kbJ{%Kn<n*C=m-I7BEjKT&POkFQ##N-eyuv74
zLPJP2N5d-QZiodiRFI#@Ob-k}^pDnu<GjVrUAveO;<+(Oha=Ja247xN23Gj`KJP{@
zi!R~k+a>iezRHg*>Nw9f5o)_lab@xAT1UNXudl<C_G<adXV?i*kzQ`D)5{f18zGn`
zuI3UKH=<p`X^pZ!4Fuc7tzr8Zcig=LUdWjEq=d8PD0Aq>GRRMkGIM-_3SuyDD%L~_
z6(vkUrE5TD2Nt>hov7>^Saki10B8jRU6Jdk5$ZdM*zgYtIyXInzGVym8Ac1B>404#
zm<d{TN`&g3y>u+^PxEexdg;7J`cKgRRxl?rQ7p&Kg1x&T6SYx~2Y2fUY`&63%NBWH
zJMIv}gr&YBXGz)-ET|=V5NDyXoef<XNxsPdn4zLcD|w3xi!)q9$|&-j?Js<8BgYR`
zEF$_=O!L_F;JeA^NIeH56_#}FHS}sCiG7#lFz3AXuh6s-#)o6Wm}H!)#g-m*qtisl
zMZ#hQaS<U-r7l5O1f@1{5z<W3+WGG6;BOVQRY_VGdHm#e1ih3Dic=2((dbIa8SM?e
zA!otj4D^sQz^y!1Vff8SSjcI9b>5@JBAL?mvcwo3x#KYf^+BwenGTvJX$B#IsX>wo
z4U!;>M96!z<S!M&u}1UzVu4IaRB?9bHWvjCqH7)Oz{Ti$OKi#MT)M-|FU@|Wu6`Hq
zaxsnKhbjRb-4bhp`vh{w!P)-4N&WJonTAC|t}vkgi-WBO)&(&fP3Wopa4Wa$gvd)R
ziKZO6+H<D1E%Qpf#tle0k|tfDllW>4Hji0RrNv=AD^yVVhHG5j&(Jc)7oFTN8vv;0
zYghnO-u@dTBB9ckt-@o|fEQ*^&3t0F-`pxNUaO)c1-Z(sw-;`L?6&7qfXo1+Rl=(B
zoEtyQEx#%w@0KGk(0<Q#gRWb--(hLTbCLy?$B-!@tkMABvh6=SiGcYu(kNotv)RHp
z2i$#NKs{g9R=wBzp{D5kJir2*JR+qAAnb|VP|n5I=6HS-_l$<9#pUFBM>C4A%1A+X
zFqK(AYooiyD!G+iNf`^%>L}dK3V{+0{>6APF;dQ#XyQc`vUKc{rTtii&G_1Yq;|nv
zUr}<cWP29hQ!WiHM;RAPXLr)T1`t}`!lrtCYw{`KA}yX0Vnun(<rcDenJ8#e{{EVZ
zF1<3lOJl1OXhGXk^U;4#6h(1~qNYkWv*iokH-^4@38UsCzA!Sj<W(T}Xn4ud0n4K3
z+VO*OE~7jNcXNTQ1;en9p4gK+Il$I}F}n@+H2X_sX%<XNRut4pR3vzY=Z~UPJ1YfL
z^Vf8gt8A-kw3ge<1;qirbNQeoYf8E!KgD3lZiV2LfSA}tKet?8tmRp|8&x33lgC#h
z<JB!kneRfLo!_|wlbb8U;`Fu*lDfPNO^M1`C;qizHfq@L;;UK-u>$tOE6kkHc#T<^
zKpcTr#8j~tzf34p*@I$RiC0Pm#k_1hM+LZ(#*s|!<E;Td4mh(MI$3jExydcalyX*!
zW_~&UI*r{K^UE=SDq@c{R`1fWnq8UFjr-_DQ{)#6uFIC=z0AS<4UO#^0WIoQZETi7
zUK%T5e5-IQ6acW2dGm34GY<N263-I%+<(rI;Z4ke9QWh$^72poMm20dmKa2nNOOm6
zIfQT{eGW7|{RZR~n{+iE7AQs<6hc39it#vdKlq80_+jE>{9!Xe5gfx%HswR10B63(
zBaUL>c?=f^E{aaPuI2$P-bK2Y#HwbIE>NiF?${?H(rkUj7e(^fR|a52i%i9%@m^7o
zl|-VPda(40tx3&z4EY0>ncY-rH5f635EB<Ii$~Kz!%9d5XDYK_`f>o<`^q}MiMB;i
z!=u{uyoOwtc)DtdqqMyO;ElZkrk=|t(8tUYU&$03<`pK*b+Nmou&6|;KE=@i4#tba
zC5={#FlJ)C)}TI~%vnY2SsAelh4o8^?q;*Ho2`$yEXF0vBHu@QB8`5OMfxO?weN_9
z-QUBkqmmUQmew5B(ZifE>~&_7`Lx+-$tA_0W*F5ik1`WFYjTB(UNaXc=MW}ysAaWv
zo5K_?Wp**`hLK0Jf`7$rmWbWDid#KO26S`J78?QHo5*XS5ab_<ZY-4%ECe7Vd5@(W
z8n6ef$QNsZOBF9ZR)qj^h;j!nalR3Ak_#E{27&r?&4)o;NCOf?u``b7YhuYF<}fiL
zF5w%0(IUm+{q`HrhgFh$ugY4oONqQzF1FYcRXn)brc6p*Uv4mB8~67x7BiZkVqsx3
ztnOzmUI!(+7&z!huJY5e2x=t_Du3trv9i)?tqUJ9q#6hAyx~o`yemn=xWz-znPGM`
zVL30}-*J9|lzi9yD=^L%6}#57-DAo&kz~Dw*-Cz!K*UsT6H;646(*}pUaU7k73vT+
zFg54UWTNKuDQ3w8qqwC|k5B8x9rZ2B5LP|H`7l%0nimgta5+-X>z_Od)&(ty1y~0I
zYQ(c3|336XyfF9_S0oV=(2RY(i@xchC(J}_VZv^U0~Ag^1>IP+{Et2Wwye~T#y7m5
zENWX7X_e`GD?4Bs&e&I3>b$UutdW0B^p+F672P&W0kwwo<yhmBJsB&GO*|DJQK#A5
z)3lzEO-m~~_uLJ_mQ3-;L*+~z6dNXI_$QH)L&oi}QS8$l@F8{rg_Rsy%9{IKn-2j2
zpe5)ik}1eUPB4@l@%Oic7dk12BAF#du=8UyH30dNp!nBgzt2XXY;Ht3FA1Rx?h4mz
z56!HG7`Xq&A-mD|Qd@0FO4PP{a(ST59zA(3pTG!4{-{5`X^fTLjcXHC7_3OVRtrs%
zki3>ve!R1+?%S;D;u_Y>x&AtyBGQ-#dSRzjVk4~+K_C?Si7HP?`iQOwo@LjudmSWX
z6)tRwt^9I9=z2b|-_+mIHjklyMHz2|X$KmL<BS#UI?5Pmeh&HHRdYdzB~ZpDq)=ol
zFk^#4gdU-+kx2)@dZ*RfUjj^{nA};`pv~FGQaH>`KgD7{7VmTcVQ+F%?>s&M@~=zz
zIZG?aY^ul*j;`HNbm^OdSZ?yfK1_(HZT~SL=)Me+HvfQZ)Gr5cWgI;*_}lh=_d7((
zA3o<#F)c6U*jm1rV;kswu(`SZ2lhi)&Hr6n-+1ze_F8*mV`Kfv#%3FyuXWn%PyVnv
z$39ekGC0k~&}1A%>7;Vg-scA=5xdlW(trBw$+IV${ms?hs=w*457yna&5iz(jrMBi
zNoRfCgVXX8cSX?sN`dU>J7D`$FwRl5{bqM({{&P1<SmX0CG?VgF@j?SXL0A8M1S8q
zSnE)ZB3T0C+(uW4NC%|W`Y5^VlZ%&|u81OCViCODWhNh;ROlvI8N-QalmkRBMk6<r
z&j)zvC$o``L$^P~WTe11jmII#d6yUzFhI3ix0y?!gdpKR6cSYyBNOfk>>*cikop8y
zgFQGFk7Y_Ek%vf?cXXLtZX{~JgddIlFb7_xHf=64c;r#rp%>$3#c9_Q%DZedi(t^~
zjT)vO{v|$v^TPRs1OxJnelHU@`?yh{wfgopH48FrHpXZT9&27z4L5Ca-YSO0j9Ru`
z9HxonEQkbxb-qsde$4jqYFaI#Jv`~29lYJ%JMO}-zvUoD$qKBd4+Y3Jh<E#YuXnm~
zJ2^beLrFlW@G5)f$8i{yks=4>w4a;L^0~IY$8*)rcirvRCx`oIr@P(Lz4yC?lwSRd
z75t}bK(g|T{?YCOC60$;#`9q?3erR@Rh}Z1&e}+sr5{pCOnzHvBlX<)oM9QyP|ztn
z6f<l-lF7lWVi<BV_YAsZ_LsBIMP>#u`ADSpf)iEGkDEGW+}DjpgA<4~Nzht)C5kl5
z28H>f^PRwFiT$uo3Fi%-XXIg1g?QL9BfXT)oP;~JWsKoy`}8|RvjdyMx;Yc4Z+4H5
zyKndQcMlGoj`6139b{p@+l4Q0&klA@DMMT-ah-IJcTY}__jXRLWN_%l83!Qzb-VJb
zrYN+)2-vgs%8fDYV&b?xR4;hwDoN^H2dFIdM-KLb>~JLm7{~7U$dwfv9s_#rtUr6U
z*(9?Yi7m`%_jGl25p~N!A2GBTB)y><e!yh&PTxKo^idb^&A-xbP$|JboNr7ivbbSn
zA@DC=!1<{j^uu(>>*GIK)+_sz$$tZ&@Z}(!TN{(*zfOCzQ<VSKS36(izmF{cy#?W}
zJ5m1Q62cbW0CIG^`*!aKPCiTG3SEzi1lbBR7{@=jZaG29+BDAc3-2hAN!A+zBiKIL
zOW23T;o`6fi^Roj3I>(O5%5vqE3Ekx=|$i;+QNze62U1ib)mtDKsHJOeCmdNlqJI(
zVd<YcSn@~;k9Wj#F?OT)-uK(EYaeXC-#t0n-q}^@l`%t%7y^4k!+>Gui{1DJ5D{FW
zQV9o3sg}{lJh^}&c%n?;Cvb4S8TFzepb|zK41@FrcDVk93yf`)Am~eSGQIEtiCi>6
zSQPl6=i?dQzh=MEmeb4F%U-Zsqmj@ZVbOjx96~$jQ{hsAQdD2jyb!Xf`6`U%P#}@?
z-Hmp0gn?o5AeJuCY18G<3od&5z!sFzK7!u^4^vAAAzrnQQ6;fx`gYy~{*axKK}NlR
zXO}^Yj<g8f>K^>=o*eBR{C#`xRHR;I!UT4|({2kFA2HZVHyGkZwOlqnD7}4p_<nEa
zBoBvF#J^-PmqVP1XyCk!egE~zn{N^UzwxZ8H^~}paYN4}29353Pqqo-<3=UK7Sh}}
zjxGZsqQ7zDE4&|YLl2>OQ9naX4|xT#Z!C9mt%)WTh(+|8FP~*-pZZqPwQe`L0eQsr
zye_oV9r$Vgf;3WV9n`3`9QnN|dm?HiS#Np`dxV}3w2}G@<lT;tpilmB0P}SC{%HGn
zx4V7trhD@D?W69x!&%0Wyd_&|TwMhH3s^^Pn8+9)HacZVq`;te-tQBEqNgs*F9Qog
zO!l2LbC~MBJv`p2?942PgF~J|1Ad>5xRM&TN8Hm-32{$L5!cHC;O`+E;eTxqcI;=v
zMh*exL`^t&eYUrMx_8js+5cxrQ&G~*!hsvaT|85EXJ39*%gH*xmaX>_V1jd9?Z{Pe
z5H;63a?d=+6Izi*r^)+VB_lgK!TCJdfnOzqOZ<Kolgi!?FR?oqw4laRo$T(M0d2h7
zJ>5O{z65TNz;Q4;4?NMqIxzQ6PIgbv-tE1~k991wGEwHBLN6-;9-V%7yu1AdcsY=D
z+0^!ll3?(7ns4kN0d(IV9vq$?9_--)*gihlZ8^)!%eBW)4rA7)@6={VW2CWgdr36R
zQlC^8joQ#lbRs2eKpbVnM9EB2P1apuRFLh|frp%hj_-XF)Ebb;#anet&GQHUrpx*K
zka>uEKM;_Z@j-JW)_EDK1RQP|;ixoc6sf|HYhxtxgryo0VrDE89ST8^BwL@dTcHUz
z^=rzY@Gjj)A=pi1AR|C2igqHNgs30{CkZU^%8@Pp2$J2|-`zeqJ1Rl7lH!z4J71MU
z+aJQ{Dik7n;tqU=ZqVVyct}fUL=x`yD8fr8lG)Ih^E{z*zzB|GB<l;29^q)Wd5{t*
ztBJI5KtO36oS*wN2{O7Dp;*X>8WX-}7SR1GM+9SF_YItOSYRw5z?eP^*Fc#g88+{H
zBz$)zHBrCG$y=n$>-KvMG=mETK@llZ7%aix)yYAQWszeOxpWyRsOi!-2PKJ-aAeO+
z#dO1J`hkYn-FeOFI-_vOAC2K`AqJTalkO<;{2@q(fa@McT7#bH7&WEX+dlzT@Mf2o
zy&*d9bWc!!1;-*k#1)@#8b~s*NCJ?W`<Qz%gkcY|A>~dC&@MwNp33EM1vJ+fv=9G$
zzk7^<Tx5GGhAg8-`xwN9qC&wD`vz7`Idl_B;kfr^_x0I31)DEzd20p<go!(W^BG{X
zf8qB(B(xhlLJgL1qD>Tm49!{TJW)IL@J>#*Pft|SbdF0>Ji+uu_#B+{GFVj6hIMp&
z_*RcX(-U5|!bvYYF9n3P2+kQ$E^x3c#ETnRaa<9(QU3$xrzK%eU1(H7>+bKpK0EmP
z-oYCaE{cwK1PTJKg2<96YH|}q*IghBuog+2?!Z>1>CF|~XGugB>{HATY+&f#2>!$Y
z?)U<p&;RB64KwK0iq7~bYbv`hH@tChV~17{*wuwEcRLZSL<)c?jo%RMRB=HXOp)wQ
zH05QqgE~nMWFnBV%(!5}TAmaT*dCc1CuxMMj)E=v=VWQn0C@Q`fNqH<?u6M5s`Mia
zCkNXq;;@1FlvmJ(zz909BFxI0r4#pdk3~=gk`pLH-<LQblOTX{G4}LWC^S^{?`1}{
zvhv?=@4q{i<Sc7MV_aMyZBue-*vVm~9Ui_f1@Hs^wdxPuU)B6N4c-pq&n$#FL8=3h
zI)cS0gaDZHp$jbL!uQItvcLV$f2gB}id#tLCloP^GAN7o0a)jMjd<j25NColUP{{0
zp%4{!u}n<}qKqCZ4EEr4L>n5+>J4><!gO%zHk)8{Wg5U?=+jw7L8f3{W&J!d*!rGY
z^1!FPpMff%Z&INoC{KhS<HhE{Sqn%GU2xUrNL*c|wx*D7{m6w`>HEz6VI=?o<HZPq
zr^Sg%ymOHZc;tFMV;e+5e5x5jz#?9VnU{w`zM%kq)FI>~Bj{eExiZQ^`oelXK=t4G
zMjau2&*RzVxM?fmCrZCaiDTFjc?VOH7)TmH;*;xbiUb5Q%QCSGjSDYAubuR<8tIB0
zUzL=3bb^PH{oRwDQtTsBk%J&5E`D_|Z?Gr0O3Q(cvhe7J#8fyi9Nt1jBr0J9xftXm
zfF(7?sE_8omrlsgoQomy073gfm|eqgSyTtX1H@BUL%(DslKlVdz3E~bH?lDL{Ux8G
zBWGgLp(O5<EZb*fNp^HbmVC4%nMrauO|mKVm|MD=mK4X|i`^ghk?t$pS^zZAjpiat
zvXh82u}Cb1LZMJ76o9HcI{{5lmu`)Qx{9CX73X3+@G=(lH)n@H2;x`Jmz25(-NlOh
z{DL@U=$^a3p_V`bJY+wDq<Lm3Uy`P03SjUKh!~(#RS}Atx$M`e??ujvHzKPHdPPNX
zvSLxaNdFz9C$1?eszgiRjc9Par7Sy`r8R@r3KaevP0~sPb(nB=2nzSkclhL9V&CtF
zron^F`NZk$31~us+Q18~-zmf>_BK`RhitQn(XtPY!t2$m6Id{2X3xw`EuQ<|pU@Yz
zf~se8MY*{I#T1<oizxQ;F&UoIg216AogcudlN8u-shI&>jQdC6ZrDh|f1|FP85unl
z_}J#5cQqOKT|dGHa8ZQ&0^%kFKOOnFnS&gdQgUO!E=*$?&j$vCek|^Rsk^~ABu!K5
zB%_|+&6+2qgS3xc9_)X=e{`k@%Jwa%aCGAQSj52%9bEN^OD@O2?=HLGd;=}49?8QK
zRQdxtQ@DV&4A@*Uce>qw9UUrGeB*&?)ng&s^PoHEz!UfP7oLzpo&$&=bQB)G(BpJ<
zaR&`x?eE{TaWk=V^yUq0y58))IXeF5Qx-pu!@ThLyT}LcLU!?5()#=1THJ0s){s0C
z1vI~%9+(I`)lMi3Fl5<_26n>OAlVe;>>ej=Yq03Oheal3rki08+WmgtYSIGrZ4`3s
zZ+3f15k+ZfReM%Cjl>~CFzufXkAA@i4WtU-;vJk%nNLY_O3Y?jrwE%iLa}koWEz~a
zc19AOG3xn!n4U0Oqf2ycD3)F*#g{3FftVpA4r^;`B}Vzx@!lS~+7$bQ0A{`DW++VS
z+*Aqr+`Ui_7_R3rn0(X#Q@ZO0XfiQY*-MbP=(2Q5>$(-Q-J`?P!@a#-g`GbfdOaW3
zVwMyLRv@b(w+Q3ojeG8-!V<*as7HRhl~HC9Nci{`wdBA%7j_ob6{7K!E_vCHHP6V$
zvKVO`fH_4ZB~qt^HU`y|YhD*J3@hKg)!@;AM1pVV5erM;wF_|#Fk>E_wCUS6Ug1Rl
zw40mpRiJm?H3-q(96OJ3r3hi*!8NY4pk1M-IMOq$HDB((ZtorL?r-y%0xI|8K3?^q
z$Izs+<ApQWWW!>b(Anb*z3yN<I!{$TI68c7uZ=m$>P{w%Ie=2opd&-~s%`Dfo9!Rl
z*<GSs0(Cy{J6&P11wC>Mlf2ND=3x9?q6=H*Z`6_Lhkc)yzU8R3gxUnG(bU$g+W2$-
z^<i9pyXQ{u2Cc9>DqbEb6Y2cp5jI6`?{2?6-G}7Ew>Eu5Dm?Oj1BrFMohl@Ip*Qx(
zUo5ErxVuWM!}4MZ))H_tMEl;4Z;y_7DEOV`nxH-Y61#Y`yaid-f}NwoSHR}luh}=u
z#C@83y=Rub|1~7->^2_&%)9?zEjLp4ziPGe!~L&&xc`;doFrVBf%<Im;LH0|u?OK5
zhV;!)ZoRb~n+PS~sC%8JxtmUDF^x>FYVA{}*62fb;H}a`^~q|Kyfc*vO78Tf9%b)i
zd&kyGObvLWNylp=XRXU!EQy1(#Gy^k19<{ONgU1(F5PQkobRG9oO95A$Uq01ZLxJw
zhX5u8gh<j}0hosY8UE<>(+YYPpk3cpt&m##4w3Fm&rva5SB8Gab3<P$4C;cgopf^x
z02R6GwvyQ}vMV$9#Fu-MHBHB|8Q1H#Pfw4H>(3d?|8qW<;9Bik*6DG9HSA&{m502|
zHHr(g1O1%WQW~Ztz`Q;fP!~fcRYg5^u#Z!^!kqEcN~SN&Cx+T-r|^hfHfEgy#weT}
zo}KLN+Q%)$MO-*;fRl6oIxGC9Jt6q580YMZoOIUzoE3h>EOq_rS1oU!^Rz6L_YZCj
z|9$j`p1Q|5)EfT#=#i1ZZQ;MkD%=wO%T?6Vi8emnIyyW#jk}*yY2xw3dzW5L;E$sA
zp44z+Su9|wqCO}y_6Ltr4-RC-yIkNNJ~KbJsHUtBJ9e#cj{euEm6P}XtBn?xzYhZ3
zmHzh!`P}|<>ExQ*KK<w0y%X!sN93WhaSwQa7&_ZI&B8?F-V-i+KjNM*3m1`lPq@gJ
z^Ya|_lN0D+hu^DG5RBV!q5S40dr_9J=h}X4quG{^O18V*anKV=D=OOiAZ!K>K0yT}
zx)BDtIvMvP3ih?@z4r!a4|^OsZ|Fj6$>~l5w6xJdL@@4x?u{=$yeqFu7n$O>2BCIm
zOkP;P_pRs#22C3nNlfbN8YFT<$d5~8qA<kq;cKNRP!`@{_y{-f&oA&Oh0e3-@RxQ<
zd_au1C@ws81P7f#Tf-2Pi~tfjq~$7OYT{4#1rbNjxx|B?#}U7U!E+kAbnONKDfjv<
zLwf%UgBMIkv^l_o`4OAJH5$`C4oxd$H=!3B;-EnQ4V?l-Wu(JsJf6e{#ps5I?px9(
zr@7HAVh^2_xId!y3Rz@4q#s^`G~G<=PPt6>qj28zRJzQO2E`$-OHMx$7)oD^Lx6<W
zqhLo=SeZ=23wLXU9b>Fq!QeT6`<v%pmFQoU(Va~Fp87QzPgI?!ICYPv(PSD4Z;vW<
zf0RsIVT@KVcV{r}bd-xo%r9;!)n58Ck{!ALuI^9qE&=Q_k+3iu-g(}{U=-%tz$4bc
zyFVOrNaGN3B1VU|<B_Mr%b7G8jVE{4JCYXoOmm<MjCR`{_UI|QE+g9QD-Z>gTwIhL
zlAQ_rc!GZ?-I2YB(WOwMlP<o<zahGw@A>0<!WIu9y22TPHW13Gzzw^APpaL(d2|wi
ze4&5tsXy5MGI-0!Gw=_Hs}abff`kiGP<YrE6L5$pRp>ZRo`t#idJx|ennH$!K3%TE
z!zsFui8n{kE>3q4=*0z_UuYVlnR*#{mlNh%zE2!E!{`cK0~x6lO6nWPDFGQfD<$WL
z{Zm-rVJo(M_>cC{oBdPt-D6atcOa)HyN7Ld1&*~-g&28DG40v4&<1Q;jnFghM63DS
zYV;rWbF2G5<PyskaQ6NWpqDGj{a>?DX*}%z?%{KP`#<UYg7p!{kSwA36CF^eJO~Nf
zQR?Nw{`SHCztDPUsuUSd>x#t%VCC@hGrPn2VAiK%u|(A>71B~RrROA`(=CLI^P&9d
z?HLa}mZiZ5^Ndz2BqK*Gfu3;7VMB8Ec`)iJYE&Gx?yKi@vSbQLtZQ20jXooLI@zbt
zne{UGMI+ZtluAaO#VKep{?F^q?%u2IvxC!5>566Km%(_xa`0IUL3UB}!Q#D4{=k0^
zPcdd}`KsJ3!?yF}_y?2=>T*#IbtG*kw{r~S+1Wn8$ExU>N6VL<Y8De)IrHgpP6C=I
zVRHO*_+WuLNhy&tU#~^7enJV&<(^E&VKjj)Wv@N*uA(h5(X;9cPRL#Q(FnHum?6G`
zF1huE8R!yO@k<Ge(h!WK^M3ira4Qyad>tgt!VaB{dSgeNUAIT>&?6_ZYZp#@#GN>N
z)#{^FLro)!mzLK16Z?ZF^hE|k=w86i><Ko@-bv*E1n%e^hJJuQGTNH~XF}GZcL}ek
z)~4+BfjtmFoA14SI-Kr>Kj<w8CQT@Oqbm)Ve>@9pYTtHKYKr~bO{?>B$2Q(Q4?Y#h
zXuKStPd7W0sN2c5EZvC8WLJ=r%Gx#DGh@-!7owQ-`W{d7;*u*+ZoB*I)DJvdi2%{P
zE7+gm%iw}+3DMV$i@81AvF!vtJy1IF$U0~%z2pLwSfQ9~Nw9O5D{Z|xumhkzm`jVj
zDPCnh#zUe8(l!j1;4*1eSYpl?%%Ih~3l=!`(9XUD3mjlz?7L?H(d|!Rg0n0T7B<Lg
zPBOwmL%g(Hv(5V1GkzI7mJ^LPlRzIIEO&p=&u!ZOK*J<|w8=K+-T$pssww+lv+)P_
z|NRH~d^Y<ZGYf3D#TT^s;nQ>Xil3gVw_2kqy67bz1N5dnbPE5su~B!fJvRu&D+h6h
z?eGLW1L01aJQm~CxXMPo-fY$D<yN)$1)3fd6+Yy1^0HuV^aXB%Nplz2(mqoT8!Sig
zH}*6KU6P8vR=k>B9!Axu?RBBPMhmJo3dO}k6O5%K)L*pdXhvDR0~pvPg@3YMIGzSw
ztO<?86^FRtO|{{st9KG@VqDdEO+p#qkVM6B+CdK%=>bY1Oa(8}5Pab)HQi@5N}`1C
z1<ATSA05zT*TnpA99Tv})U=I98u=JZl4t*W?^KBzGR;U_hGWo!#v{Lrw^+sIUzl*x
zC7dV8v@JW->7<928~_<3mjWycDX5$vwOBGxB-@iH>X}?EL0o+|{3V$(9u&zx!sj;i
zKYE?J^!abOT5cxae^s0HX6-@$yNA!i{ogM@uX;i=r;T1^_+bUUP)~#wqgCe#duNb~
zE>~lBG#M3TQPD&c_(NM!wj~%r%m(;wHyOUuiVH1K44>0xAfuJ-dCQbW9}3=LMe$Yr
zVkJl8Ib}66f4PSS_r20SEVX~0&#kWi7dLtTQ*G3$srA3nYCYJ0?&Bk#SvYT9pFF62
z=ed)FKU?{Ldgb)OGs-=MdWYU6I{l#c1t;F1zq;M~&y*udVv!ZBco9XD@ag*c<>lpC
zH(cuur)yqsy8eGdSYo1a0D2=nw17VZdi4V8g=>>uU$pxJdfIcRSN_0vgKH<c98b_A
z;lK|g@?+l({0_Uxb)EhgSn?7dF%VFNx{h}E)Y-d&OkX&o8ugrQ;R05n3)pu3$O}C$
zxLSjT)+Qk%{oeO3d%*O)@#SbRc6<NT8~e{I<+Td_uiRX#Zf<OR6L^#7@L%N{+%i5_
zJ-OWvT^d#=#&5sVW3})v_UDVfuE^Ll4LrkIHn|uT+%`S%%;m(rA7PgIkW(o+E&>X~
z#R%^Kyx+3s(6tY1&VbJ9Ro+KJ>v)X8m^08?4y!iK%mny-A(F_&8wE;XrxuCs3*0DH
zECVb#Zo7{$MeqxhD2nC^^lSBn$X$yo7i|E;On~@DR7AIlA4F#|fIH7s@h#Cm^gMv~
z?91A-VL1Rd><CshyMg08rxRFYb{DV&bwF{=4D_kRo1ssbq-il5ed_Z}^sxk?uULBq
z`a=)2(>A@l@p>7}mc`!+{_PGYqIt#44o;V{Ia_i%%X3U%h7|`3w&qu6<wM&A6QiQ@
zjDifO8(n}KDzbl6pX-z*`z|!gsHSLa&e%a?3cRRA!QoYtvfnWUXLmu>RJ7QVsLY+r
zWv$L!?pTm3N=`rwub6|PKJ%DAPxXz;@+1Qg@mQl>(NQ%kX3bQ^vbCa>O(=A<PTA!+
zpQ@1$Hktuj89mVu`bakxl<Z%oS;d%(HmecvV>|~khE<?|p1IM7<P2G38&x^w(S44?
zgOgl7ohOm&4>B{EBe)f7ZbeML?3F>AdfJp<f1>?BPljadN0p0uj7~~{=)#A1-w(nF
zbut(xj}<D=1YPIV{;MNVAD`xeHiVHhdgw+Nfv|@k2OY)Y%UJZni|;lk6WGhL%US>~
z>XQ$kYtZ*b?;r~Smz?Pc7Is1af8+50&}xLQ8qrm$a2q7*3&hqO=Lhmb6h!R28H>@_
zkt#_Kw?<<}I!#6QxB`HWt{`D3N=`U-E<GGW*@Q!MLBE;&=t~Ev>QBZKbja^1eAhJc
zu4}{=k}MY9lI?MTw2?*s8H)_AkYy4sRr@L_3Fga3tXGn&$zl>!6<6i6<+{E6O!$>0
z;akdQ$Ye&wkjG+{U_~)o#k`CKl>l1iM5LD{VubWCR&6n`lKu-%RjQD|COnH5WDijq
zWujcbBm^6`Pzyp#%p4tANB}liN929X8Q9L_Bn{J8p^0nH;j1~d=WyAlv^}@>d=AtA
zn7p_c5+t+Cfi`r!BpWN3Nj|0wV<Eot=GB_VZO5}oF3j4L!&$jyq%ODgWr<yCTFhR&
zdT*plS#Z-?CbaRHd!Honlp41v2h5uS%Og>_5fb+?7jP7LOyU(z2%}Emx$hR9yPw3j
ziWgPtTwcs>%FgzhYO2&%O=q5~vAWYMyRvHTOOxv4+$n`1R$mB%46jvUW?A)_+2Yp#
zmQp@uZxXZX%@p@Z9bvA5udDZ{+;cnHoSG?q^ocY_DM@rnYggOYn8Z)>%50{ftuE-j
zH{8YymfG*f%pvP#ZfwoHkFgb8beINedv8!I5SYv(m2?CFy8Z<)&^Q_wifm7G&b<*j
z-$uu$GH4qWKm>&E)9yHcWpqNZc*f~KsS*OkBXLJs_+Z$_Ox(a7Vz2@KH;nFp;SvA_
zU;(uX(rIx#kc^w##!xiELPz1Xk>rXd(y;n-o(s~w^PDXF=+Vf#ET{@N@1-~r#p&=g
zE>5B1W>%1f+PDsF87+0PP=bA(v}>sfvMVH+c*<tB7kES1ptYB6maENdHo+V;EGgD-
z9jp<@{}RU>xKbofS}05jN1+;vh|C?p!RbyW(t@2F^mCfBbBElRfhgk>-BzqWixXrS
zH31WbgP}J>*@|Q;m!v23*qmRKobfTIw->XO1qmB}%|j#bqG>R~g+~cQLubiAL5{2q
z4IA&wLxb18GzzMy#4};0oxM)StMpc8Z)0GVZe|Vl{8_fp;D0mG?C2&Rj7&zDZFD-t
z$gAut4T(pQknj+YO)DbMCF6_}hpb5h_cwoVEdUS=Wx)(#P+W5;Vu3PUp~%j~Rx5^&
z#;ORzba7pd9+Hw1MHzP|He4xm(XBI@h=hwc1{g>9qgK`#j_f~Ocy&k7D&K!4lN%7h
zbI$<|_Kl*?kT*DTCd5EQ<)gx2eC~{jYdT5OhNc<3X=FOe!kDoH&(Z!%R>e_MT~yJS
zg<~yP=#Vp&skk1fnhJ1A<(y->5QT&yzF>nTqf)GSB{!<`bkCITJeRJfM#z<PB_=)h
zXNkBOt7OhH^Cc({Kk>dpFK|!dPiqkX60fQ@t*2-UXuJk#h1N)Oh{o6=W)oOYx8QFv
zrpyw8c)s2-b2ugg7b6Wm;g|L?JH};`qV@+2EQe8x2Oj#<M<;?xa><aQQsVd#dpNuF
z$O)u?fDYwJ+`^zkj!V(U+yMu`jxJ&`KEiM?i^(I}X2kZECD<Ys8W{kEC|HUQ;#saX
zd?q7MJ`cXe7a7}jV>F!Cz_=nHPyqwYUvNw!uOAJ^<g|fq3y9@qIE^UW58h>vq$`Np
zy#2~PN=FhD_0700UuW&w7(P`c24o7#M&mtNqMH~H2txw9aR^(%pcMNOxk0Sl{^%X;
zZ3v|7+CoF5ZA}?83qp*S>Y=C7t7x4rB2b`Rbv#2yNyy=A%!q=J0)*9GL3H9SXISxP
zh<G(G1tquQO0!OxxpYZenBq<<7Bdb=m%t^l+|6Md^Xy?xX}Jogmf;2LZ?e#0c<I(N
zgElh^XmeK9m_ACAm~YHU*pla&eiC!=_7oGO$V_h01XB8BeGZSyRYqKmS-`59R*M$p
zd78R9*HNPxCtfqS9jh58RyhG}Jd&PDi&)4rSU%C`84SP7L`+WGxonwa{x6F|7nVHs
zmGq;2{he+--fQ%%KmO-d?|-M<T7`5U9PdN7T8*stU*$?I75}SJtv$qlxR1}Huhu*M
zXg$2h=Sf52D7pT~`FGcioUbeYcD9_}IPW-K_hRg<NXz5bAGqYl_|3^-K_wPf^1boM
zdzgn`{^u^{zX;B_7G3d-nfLyq)k^yRfcanfgUA2-gM8d_k47=?mSF{ke~02HhM#Y{
z#t(WU_@C+MT|2~5BDv#+ez%RWy+You$#j3zhRvUMW&G}s)z2vO218u;+hpvtq~l%}
zHjOd|MnHT8Su_qk<460#4SHq@9<HbqL|cSPot}F}GQ}Wo58dkyI;%i^FEg8sGe&kV
z18NYbx4o+g1~kmd4le@Ok!EN6Lx11~GJoK?qv=Ha>ggjSPytG(5@_7K9(m>RdS*5a
zDTQvgr0YX1(k(?PX;vEs7_aMIw<Ev%E|m}_QqV7i*LSA_QYW)BLtCb`hE_5RQT6QV
z<z!M^3OzZDpaE8DMPgjTH&_u}ortf4^LEb<qU$#L%CL08o*!o9XVr_#8}n8M367`4
zFg<ibcY>3RX9b=?Ypuw<4n~Z$Wyr_7_G*k_Iz64D!p?@NEaoYJK~{DeBBy7Sr{}%t
zN|cO!>LsZ|Io`;YgEVai(+mo6b{dcAC?@!vTsy9*A~q--6g^#gk`lN^hADAX7P{C=
zrcqo9m3ReoP(M&o(tB;0;zgi4M6SkaZm*Y}sbQVY45uBL>7U~fyZRM&Z3~K?p6H&)
ziN<Yw;_WL5tz`uC_*<lpyi55F*rBb@I7%>t@koTeMz%^O#T`^hpjL^bi4Mi|c$-XQ
zV}?v+C%t9!XT|(kHGkIB&vC!)cDvJIDnVaHwNyjMQ#$-J#2~nOTGWlMwBE*P7!E6)
zs-;DNm_aEKq;uA<Q#ZgcD<dzxS_L{CgN&t4q>sjJZD9|3IP`)GWSZCe4fZr`ywEeH
z*xxE6#u_F8{63F42@_JFLLITtV4AcV6|wr!_feQ=y^CmC#8gqgHK{N7HWZniYkaL>
zDiAVHB}U^RsB<tq+s2Qn@dKoGt(r_}*7cO%<c}`2G0}fajUNUmnG%D-6a@;VehdnK
zOpPA~D47z2!W0FHq*n}#e@+uW4S;c443g7+%j2CLAr9y(sjBoXF@Mc8F?~6aCnW!1
zHo(CJSqlJVK$^c2w~fKS<aaF@^=Vc*R6_hcjLU`nsrm{lhWtJqQ5irsi~{)G6<tsX
z{lN3IpF+pzM53K(-wT3qkk_?q#NS9D{xwoS%QAis0T(0TGQgKW5-t(b(6x1ty%da|
z4G6vyJNLsVP6$H-aww)9tq*aUQV<hKp-KY%?_DSrI3w*mQ6{bf`glx)0c1`k@*`ev
z_y<Y4L`TeSgzX^0G2$OMwCWp-R`@R8eRlyOYYJS3-}sD|zKV?weI;=41=lzHI~kK&
z29hMaFcRK**XYxZ(!^E>zlI)04dbs7Ubu;Cir=bYG!RtBFV{oABfg={kiOy<oLrod
zm_A05m_3vxCQrPP0A!K;YP`rMnHS1Duii{gEM-X!lgYhakS>dfsgi1kkWNjOHFaov
zPO=W`)?)b58F;n{EW|T3oh?R9&q$RtS8jVw5-X*vrV>*nN9l5-R4Gk=wIpk$sSQvA
zKnZ{^UC$rbbNVpN0-1$#21XC&1MuhgIwKi~)tv^Rm&~S1iOEE|IFn2?M@gmPe6yrw
zwMTjMa50%PaMNw#Bu*t=rHhG4n@Gm9m{gj*sL4uk>U%Fh<BO$}dR|s%jWX6wTd)a#
z3r88lX=Yhl6-Cm+Dl*bTD>5v@D|3>)*V1R<kkM<An;LGBo*Z(Od}S}5O~KUg%k=Cj
z7B)~h8LVK*pGA~8EFnq>4Euf9>JO&;m;Z==L6xJwpuXYX-==xgmO+CVjq|i#;Xm#E
z(7$R!(f9|Hryp3lLtgD=cAO&11Y=l@WH$YfIakekyHec{KQ@}?53GWwK4EaB@4eJ_
z+P-kYP|KX?-woNFNaC$ZB2gs;8koekh|Q3k@4ND^_UE@~2;U6*q%ZInf3d)*{oWm%
z2e>^V%W#!6miQ|<%$-I?lF*K{6xhP&ebHy}H<Dink6%SzVEm2L7c^%K{PR)Xmt7K@
zFennr$|@E^-hVHPUGZOe@{!a|(&<7)i8F+r!nUZPd)0+09p(q;VtIluG%%ry?*e2v
zfbIlH7WHRG2gr_LVCInal?oe_j(1|FBzih4xKAQe^w<(aBd>^}_bwbIen&wxo&u?4
zLKOC>6Xj$=MNbf17^}-#o?uKZWtd6<&X<fVQr{<r;e8TQHi+TAs^i1f8^7SbzdZ(N
z19MD<a?Ov##@n4nWA($_g{#zY<PK;ikGhj-m>*qqV9kL)#5bVOjt1(R@*6F`3h&eL
z&&hS(@U@&boi^u_{-0F>GKlWXJD+q`EE7gy{LNFT`d8tdI&XM1&W|Ti+wF`&u;aJb
zTF*aWhTvZke)5ahZL6QOQ^j8}g-C;!KVoCKOraS@`x%f%MxvN@A}NvwKUb6@#)Q6O
zY4t_e&9t8wFh2g>_eY`~`gz`)!bV8_gb{xi|Kdx)@oy2`ghBE^P#&4U@LOyEqo2Y`
zMc>(oKs-vnxj+2%f-#ulU~tmaFs3MK+4)H|_$?p1zRc?mKJ%sr&wT86Gp}5k=5-gF
zdCiGuK6XW#*F2c!Cm`di;cxP<S`6fGu@cCixC~6jmsQgFX?53r!Kxeof|Y&}z$O&N
z6lev4mkOIHSiwk<l@c`)#m%xvagQ%QsS0|Utfi$<)tG6t?=<ZbzlG$4MM=*HRYAGP
zO!VKucrfU=Aa5xRzwxN^Pm-kSYbE|#6<;G^KcHX8yKQ{q3=%a?7*D(r|D<WCzKIFd
zYmYns@wx&8^MCv`?x*^h=q9I!W-*+*Jd<O1YEtve68wVS1#@fP1#Sjv!x&Uz!86mu
zLZ@adS>5$tf{dtVsAgID(&UUVqze(KXW*<4FUEtOk*T!0q(W!va{S9!PV}UBEMp6@
z_LD@6_8T4js1!pO#T8RBYGz(&&r2qw!6pTV5~*bPTNTwv{0lRP>|{eU5U`y`lZx62
zfjATewy>wRF&Mj^CFw^tHs%CA3QH!i&0SB8f@`g$u)FaO>=b{)eVh1(xC7_<vPP&1
z(S`?>iTB<>rc68UyzBS}{nc92lXT2vx``XK+D`mrJq6D$Ov8$i!WwG`<cI&1u*UCV
z;)O2esVx@j?p3}J5`T^B#YwVyoPwf01i25|jXNPZNxz6Qd6nx5`Dy$jWynlH+M6kO
zoT#sqV%2A^@Z9j)OyRR5BZ*IC(n(BunJ&(XKv^4ieVo%G1Mg<^2UFMw8ldB31H8IK
zBY)${EWS%_l|S(XrGnS9px?9L4_J0kVkdyfWTamrDb$kQdn5CQK?CZK{GmZohgPx1
z#rq=hJ0XjcsX`jdWU=88KgHHf{?d-uI7v1Vn)C{lElK{?k!RDslway(TG5O%dY&bV
zP1B@{i|@q6;ycYl@ty9+Xx{--*q6lC<@i^q|C*SYiL<|1d0}>522g9F4NP308CGrK
z;=P;v28(|DRaH~;)3$c<Tm;klUevvazwwzh{*(Za$K-G@C4G~WzDY|zK}?Fx3-b9l
zfgdepzzW@W!g7TN!YTtp`Gyzq?+Y4N+*VrYH?k6NWT&9$cIvmPHXhQ31HUKM`9xBf
zN{U<2RzXWrm`aLk$~>FoNGID%Pm;wGKh0WN>Ex8m%m_0wQn2({JR>X2%+eqZnDgL2
z`ImV{#%m;Trp8rI3@wS1WKl1)q=d;7Rc#c{no)a%2h1E`60n=%-cQ=<NR=P>?>x43
z#_9O*9E9nxEdqI3az<lNa-msE7DdOVYKd@_J#P|SSQF_&jthJHL?)s&z*e#XUnVom
zM25{r*#_06)RBoKs`Xg1#*YL9)p#P^Y}-~MvEHSU#e%2O#foQ!n$5>7AiF{LSCm1F
z@BiM#{qO0>d;v1|{YSZ(y8qq4^7nB8`Y(R}d)fCN9MfLC^musa@dy9h?))E}+aiF>
ze*aOav}&zX{I6R5;r+*beC{v)7rV+jA&*!10^f53^m20^Okptt)!Lba-n2Jfg$2Gh
z9?Gz<!jINCcq&49I1aK|;kxKNcM9&4E9YCMTXg>B3`IV9%3;^k6bl63`%7m!q2rCh
zDLPTWD6`n47x?2|4BACU5l1~0bhZ$90~gH%=#32nhM<#z>i>?4?6#obu2zcVxLQWU
z^w9fteT|`Y7lkIu53zjpe`{5N&xz+b{}@f9i^rjJx_z|s)~N+OXRo(ba=srA)|_g|
znVhdVl@>WS=y~XvG8CS}LXUeLxG+6K83*MQy$*pO_q&AX(7A9!qE*ii-QIg3k$dj3
z<2p<Q^`4?<KvDdO_r9UG2ne<e7@ik4JuW$=gDmv%;8VO9;Z*60jG5#+$$^1Dcfwc<
zn{=J-b(e`%C8uXth8}n)a_c1rbW{P6d?uj@1sy(Jdd`G=bz}3>kcWIq8$xuV=10O;
zS{xJQ`0#amyIrg1&h`&aYt{Bnp;9T=8kI)1UMZTzcFbbcTD?_FL2A~iNV5(_nzd%V
zQfpSLwT*hSwOK2d8!3=_`AWTBZEiNo_43BX#wJR;))y6?_)7|r_eI586XAAL-rnK%
z%Y!{pAkR03+fmMP98{2*4yEf6a4iQ#g*ZO4D9v{tQe}Ue7G%geQ80mpe(DwD@PtaE
zBmK}nAE7f8QH;a=QG`Ff<eWlOKQxQc%mkP~T{~cth-c%Hz$4MKS>2KeRybyVW_3m;
z*m{!H4e`_3vWnD;UKqsww_s<cqVxOjPO6BICB<T1C2b%4u>H>yQA_pQb|RT%!Tg7O
z?69gpX&^h5pe(}Ls>C56|1m$SQKXaY$TP{wGaGHzwMOB`tVWs6mJ?NTLSTDH?t|Cu
zll_0~9ldJ5Jw2wCcA36E@&+~-wPm`((VidCX5Tp*#Ubv+^=j&oWRcm#jz@Ue>1{A%
zUWA?Jsp3T?rdb81n)G}kD>^SQKu|jQ84v*pBm&hiTnuNQGnuW7MRMpj7+rsG5{56~
zhZJNq;|c{S$7C^bplxAhhCv0D!`{@(l7gI+R9N<}h_$4*m;9V|p2P_8S12dRl=~?C
z&Iu4NUO0tX)mhcWU&1Mzh*8$8ucA7{7(3$$s=4G7xa*EaW6+g6C*}}P>R_`4Lp1PE
zOLHcmNP^Ng<d9|y&B|rPw&V?#e>dd~V*LCJ0s}HJfz$L&5w}^NOWXzI#m!eCB#V^r
zHzmry&I%Yy=^3yJhzLWd6s=Q97ibGiq~8{rL)i7%Q|Px(oITLO(vzn!lQ=gWdsAlK
zP;p2~OXxw0Su{qFTu>oQVaD(9SvT!e@H`#2rm)8uoO974&E=e8l;@7Pwp=i-If=qZ
zii!aIS~#1Aos$fvG;)3_udQ)Wa#$-PFgZ~Gn{94NFp+SanOaQ*N#c2+)R>rCu<ncJ
z&do+77EX)nU`erbKZ5{>8Mn*sX%L{H#30-b@pVzSyc%nYz(4Z<ND=v|lR&Bc_Z_Nx
zuKn+J?Z4yx^7h|avsJHK?7t7^KlkyuKl?Ab_QM<z%vWA64b?Z0B(^f$kt&2=GfMIG
zN9}t?xpS@D`HXVnhJ^kjqIeI^5k3BQ#DiZEQ#jG-x5r1X@{b&`eRuFDr0u=h|FONh
z_iFp>;53(8`HzRdSLC*EPU#gXM5I*_G$Sf`vi;`m!CqXlvY9Hfe|Wlg{QdTUDzsXW
z_W1AI03G2DVJ8Beg1OLr=cp^f4xNmzITYXm-(X?KK~|d=i3>%t7$Y=4jKf#7+nuRD
zh<x;jfYOj&%X(o5#YZ0A32*~{a4ihJc>QY?eXK}hIYNV$VpRGV3kAxJdILa&?eF0Q
zIlzaV(mOyD#eH}^>Pnx}fK2E`<SW^Tnn5IadD?!n{UfNy_za6fM$-i!#0M(lEfiV-
z*HSfi{_$P=&Hmy3>E4@@T&`KC+9yzyEb88H99%n{X<xHUUt;h8yi<uw9bPeZ`mk(-
zY0$f8Z{D_Fp1nei7eF@}wWa}`hG$jXERoGjLyjLKZXn~EltM8L{p~UE!K)a)=s8mQ
zkPj%Z1NcD41IodO091E4#_$QS^~A~Y7;ttD#||Cs)47L2rBC*GFQoW3(-GdrAbfaz
z#{dX?vxj16`qAUi8NYL{kwO?ea}|od_d^kEK7se)&gqYBm{tEc!Qm=b>q}rwGxsy0
zeY(4Sx-Ai}HkuOcSfoh95A68|<B*Pn(B~>SP^UqK1a>{dJ0u6Meg;Fb6X<h}%3@dO
zrUzm^oygD^d?5)aGmS82y#-Om^W=2<bgzB9w|}^^eS8StcXNe;F8!P*MN-buGfPx7
zPD~O&-ku%5-ab7#{^v{x)fo_IYG%Xzl1@re%h?A~Mqf(7g`4IvF(tZmrvOpsAj%xN
zhz~>0Iw%JNc=F`E*VXjt98>`HaVu|KVA==!FJJHMpn%v#W@1e6coVN|GWQJp&Uv@%
zw9iJDc=_t3CbQAm2a4A?8Wo|ESPY>zYJJW<>FC;iZs=YCcHH$THy{^Esqm&3awgAP
zHOEXkD9e0z8jL&t3AfCo&UNG&Y4n65S&$M%Q6!%1k_StR>JTdoWUEkjI{}|kI$K<*
zwB(>Hx7WcJ33zIKjV!tg#V~3MsG1Jn40XiqV&TVM5>W$y%7#V1Vd|aooF);%5{!P+
zxh=GWG6IJm<43Wiqf=R=uRa}t-{>1}pD2*S&UnJ7y_mlG0^qhkO#NP|=ZD?EAAT&U
zr5n~buj|seE1@xTF@=Btsga|U0P<jMC&`mDhd77e0s2-lO|&Mqb!k>9QIs{WE5fD;
zWbx9Q)=AP8b2<1E3P9WP>V3~SAB;P&)??Uc?<&%2u}TP(!7mzN&h~eiynq`n<5L(!
zSZ~y+e?+UJ!*|FTD+~vXDhm%ziNIj!jPBUGVj!rnU^_r)_%~!dE8zGU7*YGuy%tgA
z!fE$Hgax9?6rUP{c8r15gQt|W>THv8;VFK=0z5`8AxH=H7=i&~WdoKXgty{V0^{&f
zvnF8<FaXH;MI7;Z1qjufcD-OlwnL1~UFSuYi2<8qfwjX6vq2La`BF^46}+u59m43Y
z5Z)9S9MYhB0Y$?TfMA*LqV+)BJ_waMN)tp(D+=snNu}`M0?@e^g(L$>TLp4bi}>i&
zL7rKIhCiTSDUl%g==6f|1PNH?04S(l3P|d;79kD<zCg47InY=O2_*uh%U%jO;DtuF
zkSA=NNzWY2r>R4neZW3nU_b#uR@5B9QdlqO(EFl-wmB*?2y_;sartR5RUxwy4+;T`
zwanLZg(6N#dho#=;#}u_C<UE^i;nDv2=<0xQC*XEAs%Vs;Z?dR10y4>FUEZ5DU+D)
zy69wsv_-7pz&hc`oxpIRZyniJkWjD_A<cXO`hTK97HatBa2Q?5KIk%r@>tM(jCYej
zb$n1|P+i}j(o3v_lt0&{{9DE$bVjg0>x@edEuhok5Y<uK{%AvNJg-Ojh2+sF5Ua|<
z-=K(2DQB0E6SQeMk3|Kruq#y0ai0JYDnJcVR>vbue=2x*Rqml^A&p_J+4FTlB?P<U
z#v(f2#F;SO64y`zW5r#Ta=W}X<K7YjEwWNC?luV%R+y~Egl2gLy{|Y0A@f8#UQk4m
zjDsF;?n1YZce!FLcmY%Z*EMA+$H0x*=6O<lA!_mRhIGYsokfzixYK4?ZC?BlM_-~#
zU1BNI*;C^)n(kpjT#)J@kP!7lG4QmI^ep1zT!;_i`B;27r``}mGU^4W?n>y$t-M<i
zGYzX*7iq!R>7`m(0JLhPjOI`oHFa$V)MseD<h%zCQCoP7=z@;wSR)1j(us7XJBLd5
zPnBP?C-1pFdCxN^uQVBbm^mGtH_~YWl(|M$#2an2XB+vyPNlx!_&vN9+4sF*ElX`u
zoRkStXi}iQ0$IeinDsTaSzv*ko_OX;ZCBn*MQxvF73memoWns6F*ev}JX*yKF88Pl
zIsuYjd)teU#OmNi5x(s~?NFq6R})_>7v_q_>MSg2^d+&T?eMHp`|0>0moALOf=OK^
zTLBIjn*dlcD7BZ2IYeTNVWk9}2v{POVpKF}8jI0H4@SI>jL}Rn&R){#<W$4|JG4iQ
zF=mV8eAq^>uD;OM6-WWB_jG9i)8QSXt|m}OQ<AhYN8H220YPUE+CE`jJzc_SD`ip!
zlVDp~WLpg*Rt!+5Jpx{mlAht<UV)w%Q^+r^5(yz(qlw7)?1&_I+^mEusD};)fpj1V
zs*%#yofA|(up%q62|CAT)W*Jpv^?0wM*cwPHWC`Gev*4!lo^iL_le^ladiWav1ohW
zcjSqPSezyQj(*+Jf3W8QGwGa@)aYXjGmIOpDQ2V`oB9e%ayE5|0Y=9!v|&&ye8-al
z*dreIF6E#+R&_7j_d*HK-#dyBe|;AZ{KK)jAAF6ij2?rs+wGFLlHx~+0aW4wYLkEv
zvSC6E6vRxbp?=>q#PM@&vE7B=5@qfK)hqCK(o@$&d6dN6Xi!Wv%DH#4cBNON*gOZw
zgo^1k1A2Nr(u~wIRwgO>jAIJ9HhoEVc+p>5Fyb)J(z(r+HMK2=#+sS-C!q?8tC?wk
zK3kv<;=1eM?hL0nOoiyu0|~?VJWC}M3mEBEWDCIz&I?9jQLk+x%ahsR4^;imKfhSL
zq$wfJ2?XzFgCO5Q8i-f;+gMm<R!8STvbMBDIhe@Y4H&>o_OARW8w45CGz^YC+<Jpt
zfvr0|W?-7?0BB)PxHS4HFY3R6K^x-gE00PBW55At2QEe=KJnbBH{)OpA-&x`{SGIG
z^R2WsVk&BJXb;`)a-3y4^Vhhl{nL5)ix+)R4rWRp3+Vwg*w7A+eb6ipP5G=*G2`EA
zW9Ud`Dgjkoe4#?plbKcN`*Mk=XW@8(;ryKTO!7=YL1PsKFK3Xm0wFJ@l|mfndHp_!
zC&v$!IH$t|fry$SrVa<o1IkBnEE;1iY(!*n1g+8?SYa`YOCL={-l!+G8X?Nwp5GOF
z+6ynPNJXItb&=leOoyloxf5s~U+_e@{W9kxDr@|3zQ|`CVl1>f-};o6!?7Q)cW9|k
zOA$tIGLHDQ60|!U_fS9Y=@t`Ot`Hu1<_Qf9gk(twr+l8ozGiNb{+bH(M%;M<e@cd$
z^aTHkD>>{Ss8`(5z*tQ*{I;8*jY|ABh(ZuMPbN{IZu?AW!$fl_)uPdc#l4e(uK_O`
zmT)DmE4Q7fBoM=e&}cMXF^%$2Mr$DVghd~DkxA1~tCSQj|Gjq_U+_0QCKV6~{r>+r
z_KYR-n5MCA;5<>^VL)UlxsLl7?UxED7h1587jRJJ@Sq$8)n2V|$<m?$l@bFd*3h7{
z$1a2~*?`$cwxlC%YfskKTjVr#)Dw>qskH{b0mxVs4rR6)7rnPAY;Y%NjQhC_qH&b*
zMEgYy6~muZl)YP7CYn63q%Q_8G_h^04)dDlgHhaY#ARZYTx%!<W+vHUH(80P%s2BW
zS1}>2z)y6INxIWAQ2-{)rAj6Hrw@--&cH6^8Z4`;DW_jXQ)XYXYDpJk*~GHD9m^&h
z6)_+vUv$}xD7SPo2Sk%z=<!C10W-NGlVj-Qg3qL4Tt_;cOCpylQJGP~bSJSuL2^ft
zT`t3UUBN!1{z%XIY%^6Ued(b~`!SSdG6m7OZ6ffInss%Pz5r)0>!u_U4gzmm6NnuV
zE)Q8Rf*eD|Y|X{QwfIh)p?EMI$w{f+7s-swjqUN&u0v?YUvrw!m_vI&$>>>hv)dj6
z;b_t#wwD8H>5ZXgmaHR2ump6P_ZFoO)2&M6;yB;V^GUqHH{<yv_BPY&Do<m2z$i>r
zd+%|p)pz7>m-bV^6mR`VE2KCSj)aj3?>ypbd%RU94;RHXMV;=-;FdH;vo(pfc(}&l
zb|WV*m5Hk2T)*$4X6Hs1B{tg0s|BO6bIyLdJP+T~pSytqb1}Yzwy>c{Y}DL$Y!0J?
zeR*~`1|ShS5#X~y&<#fCv0+c%e%ElC`5Gand6wkVNj`@f`u-fZgEnG_W3xE~nN17<
zV|juo)MnDelvDWymx}xc^NNoX-|*>*_ZTq6`aW+#>8S!A2R(Pv?&1opb<krp0iPa9
z;BOcn{~c_;3!=IR7S9E!gyxEl0EsNso<(8j_iDMcm%0LVRB8@T<Rd*;qX%edY*t^0
zlOoxsrO8<FH{i~18e0~WG`fz`Mzk(ZuIPz~_83G0nK$S{Vwz5W=PdLzLF_{;(S>5p
zS7My>H!7BBeFbg=0nfx2+c&)rFSW2)7PTAn!xBR<#=j(+5^K-GeOX?i$zB@m3*jwv
zHAd3xtWu@4Rd*5bdjs8yOU==(K0z#{MN)|tSSgs))YonnbKPgp9WBjv#Tc}JSU4b&
z<<nb*xuSkvUY4ZP=Z?aEmN4W;Wjyy8WiD?DtMthuz9nOgM-#ZLN-A+?t+y)bZKYe*
z-|K3R@J8OHysV`eh;hwIwmNf*7t2T4>{n74%H<zK;ZlO{X1cebcjt|}z5sz=D_S`*
zzF6Y$y%NEf)DRA7Se+KcpvZCjN>{o-{)=&)HI>v9mGdTt6XzSIpX3<yY-QkWT)aO6
zWYthg`{_4mH5DXht7$H2GW~{Zp7A?J!$w+iM7F+Q&3i=}i_9DBYHmaLrv^y`^wC8k
z@qaNjRdsQJ_uXDN290SbW*gJ3>kEyHtU6;knl-))(tP<npJl%n?^T!F^QHHDvzIzL
zco(9k#|xAAk-Z971|6qaa_CG!PxRx#)~Dw;-#A~14&is5k7y#z;3V?XOZsVorE+mA
z^QJrR33VMAcQZNML+p<63IziYXLF22{E8zf5R0PfCB7*^kM4LU0=Ai#c#_8laInif
zA4l=wA?*QOQ7?W~hY-5%B%&inb+;nE093-8OCwQDobsU;IBrBBia`8PIPLfSu8&uE
z0p)Hm4nw{XnbU3r$BPW7udAgcefGtZDqimSj?{eNh&nooF~MJ0Vcu+qS{g?tYDhM=
z)Jg~yQGdShNU_vBDWaV^*lXHUz65q!4`U27&hbjRA!i{iG8RlrT6I#Zv>4^ojwI3b
z;(Woe)NH=#0+C6<<j&)ZbOLaz@BgQrfiFVsD(BDtXTJaEYPpi~|6HzBAKw4p!zbbY
zc{H^7ejYvlH@ScQ`on1W(fJ+L&c{z5JNUof-OG2*;}7)c7;1cc{O{ORw0PY4AFmDd
zKE}*`Sis-K2iMVG!sia=f71Q?eE;u_r2pqyrCx45%>R4%+~NFZSMg%{Kg{+o^K+Z?
z|JQ-d`_DZ8KjnHOwf<Kdt%vvz_wu3VV4$7TV(rIn0OacZ>wqqL72G!7Ei2~u^Rs;T
z^dYBGa$LlD^6TJ&L#A)ZtdIF5JT0D50I)F$_QkkAHp_a0p=$cqKz^l5xrDaqyFjZr
z@-7uzbnvCHl&PyP6rR%hE3uVsy~@QTE&?okD>-hP-ux7uUjs_!^kST0RHlX$Mg<EN
zW_b+tE(*mERUZl|h|-uLi=Z*)zYsNimzyfCVgFPZxJ}|+>5UuRHE^(EiMR-26qDEx
z<P@cHHY5n~MUY4lLN=}uAl>B8bk->DwcFa~x7)y|w3Unz`E?*rF}Z-544^`jX4QhI
z1Ffgf*y4WR&NJ1M=hPFfT4WAJq|@aj*ey}yV>~Idc-ieo6nakd&EBV+?2l1;R(EXk
zJV8Di)iOyCo9J??`$_VqdD}u|lPdFQID>E)W8sg}smvL-HM5g}HypqB=Jjt5<IZPW
ziCwmIO7$^|pqBOm*$r#w39vX4i^1ElR7=BRGmE%BML=A(tQ;a2O<aVMjc;iq8}~(!
zC`$EbCd=L!Bddttx)sEb;oa^9RDJITSS!8SvHXz*xg@la%`pi+MyGY&0u6CH$?}V|
z_qi3gPyF1f{vX^-|F2bRDgU44@`L_=A0K?tM~_>x^nayv&sY6(Pks&9^Y@lD2Q@K$
zPaxaQP~rojZT}ir)*hkC{~CY_Ei%_+>%5vn*4B)Ih)lAeT~Gy5A2tvvWVLQq6<X~4
zs#wshnp8lc6feUzf-KNWTm;j(TMLm=S1+^LnLQA)q8*DxM<S`W+DO6>P_6Hcb1{=*
z!Uyg5LVZr>X45N4>sj<dNVBPsO)ndZW9PDHHiU{)xa~5j+8w(x$0u}h@+u74?oxx@
z?y%Y>e=Jqk<+^%;m1XClry*mjKNCH?-euhe@Jmri(IPzvg<s)h;79l~@RTT{19<P(
zssG*`c%vx3f&CbrZa5>44)zfoyIi;&JKgcb_d*$K4V`|PzB*M)Swk7?ON5^!PDxS1
zswD-nYPgi3wGU)YZTBUET4&I*6Js9OG*{fmvLMw&p;RVi5H^7}^#T?snm|>5IPWXn
z0%d05T?G1O>CjgaW>K@UWEFlYZzY;zZ)&_5T}kzXn~k@!v85%!Mo&e@mhP_6x!mmj
z(V(gReQuTN5B0fK{a;0cH}?T^^#5A9m9qc08nw!U{(ldj&*uL>W){3O(+-mfX}|nh
z0i(3=%fHJG`E|d7aUr^Lgl|P;9D2>Sx3TL`cz?fr@cMZFaJT(GXWPfyho}38v5);Y
zMp3o?MTPCmqU{IaTnMjUZOPy*IdNC~<Cd)=RT2ery6<H@O?X~bNqp?1#~*mJ{o~Qg
z|HfNl`=?H&td$*{w}<YPfRG&g9HygFKJr>~J#vS_xm`#eI>_G!d%tJ@$zPbeBO<oe
zC%y)FF&@8bUycLavxj^S85cIL&ZVC6YCx?{1^!6Ss<Wvi-%Ee;drxih&U*DpmltPX
zf_D^SGHcHR#J$5lz7G868+ryBgW-oPVNblS>w6JsSn}Ztrf4n7_jc;_2pXejGSKI}
zQ~01|{W(6jss9C?8`ys<m1?t|y#IywKOXLX-N$GB$!gxYUE%OV1e@<pCeG0924ly^
z%hBjAmHjDVv^TFGt;%2})zz94c)ium0K-wB|3dT|`j*_1kaIH`C50brNA&2M{c&~2
z1K5{RV0z3Hp*F;uZ-5_04*Fk4_oOZZ!<egpK;WNWM62Y{B!*uA5qkWxy>s;D?e@+o
zs)%Ai!DFq9X$ak+cjxGPHc!s5*^x3nq8JY+E@)8r6e0wOVd$g(Vcaz*yC%qU7DCmh
z;)mlr67$M=7Dhe5XYJyJB?Dj4#5p2h)Zy;l%d^+c`npr#;8f_6E%XpCTBK^lwPF$$
zXQe=Cy*I&m!;WW=8)$5!VY(%h3Zj*@HPQK7=ePA(*?=y5*H$Vh|B^~d+&$be9CBbC
z;RxB-21p0P^UpreC?StbgU+apoV}HqOC)Qf7vuOGM8yBPkCx4BdXhzX3Lk+GD;0P?
z15z)WMC6EcSwx}x090>}ZgT<h=ttHI!*LKM8j84cq$DLyuJH@fs~9(h#wkjQjYN_>
zMGMd{8z}yzYU*|m!OsMbVR++VNE!}pEO?BgGOORrFpGDbsf=e#1@&4b17pRDMG3%h
zzQH)=yhSUnR`Eru1)Dh(qe6?2Od-IwQ4RI@rHz88Q2aenU|#3APu!&n^k^$>)^~PD
zuFV<-ueb0Z;IvBiV#yJRu{F^k`8${XUhmVWA9GHO)?!rFI474gMoeZgda)5)ABhyj
zXLM9M`1~{mTcV4<8$*-6LS^7jEB+n!;5No+%W}lUtD<&2H#r%o4RJJ|>R8Z;lKuNx
zdj5S0yVrM~N1oGB3yb62%K(O0O==R@tXe!+kI{!s_7(b6zN*l+xvU?C&XMo3x_Fbj
zrPrDv<YrdOVpv(pcAUg(|6MEuS)6e95}j7TSlzf|X=}#=xcv?#r<{&h-ZrSWF2|@r
ze5h(Wm%a<JLk*Juv5b@OI}#Irm}#ptWQyXGO@|p2(C=P|p}^GD7g9%d633*;A=np&
z7iSvz13$Xv-L_>u$-9eL<A9BooWL?31)0eF4&~yd8}reUsmdfSHmg%(sq6C@?i_Vq
zLeo!)sh)<)q970x^VTlilgY)utw;L|-u<j3bLI(s3!SWFS_H(W&?y*G&@~*5Y|%Q0
zPID2}rs9$_KJL!qoF`PfMdVQ+1&bjq7N8g5{%Hi8(Gxqp7LcncaxKI@33|;M7c0T6
z_pHzn3dqk15Nup13QOX-H^N{$7|Tf9w4zXbV{Asov>OM37f!|;U_it*P*+S91X4IL
zmX5llHMn*rE=Cy#sY<7eUGjN3Ml7IKLBtc#g*zkXw%sVAOD`PjLWl{~cWEE_InN~>
z5$HwCEm`f+BVH?HRWg>q>Z0BOh}t5AO{%sxoJ7|P)R)@`8w|Npd<qO9IX?x2xNw^U
zQBswNb+#4-CoU%47P!!J1A0x62FR{XWaW3>HM933UA_~jlN&?L;!$<?b{T4B0aI5G
zZ)<4$h~ATCgTvFFDE?V^((M!rB)s%maRGS*UzA_m;9P2i3*aN~*DZ)wPGF)Vw;Wz`
zNiL(x+K?egvejfegqqe?LQa!}Oi5|iK`?QMIP)wdaWGV(^BFg!h`bz3rUUE|A0(->
zlkMPqiq*pS{6uuy!N;7h<l2Bqeg_>LqE;1){&+bkcvnx!X|LZU`A(3SRtV`Wf16p`
zyK2W((JGFFd+MCxpY-{UV}@T;Afgxs0icd#!5~9P1|ZsZY0zlpC1{5j&=JqREWqSZ
zmV|=8V;J4x#9@qR+ee38fgb{e)iR7eKT2`%$o_aR7+(s1%)CB7WoYJy!k};iPhzf%
zx8(Bo7;jGl(^){+KmSXg1<Rv3q74!-U7Yq<6AYOxk1lPU7akKqkC=L)wV$gMARc$i
zd4?pV4q4GF$0eUXpIK6a`C<ml{SkTZ?qcx063;8HB)cu9tt(~~2Za(LtJC0AF;=f^
z1SyFqghTfo2mwD*2cl_c0EA;fTENKIROozK3!*I%NCP7|gZIRgpa{e1kb`vUJPEq#
zbP~&s#6?3niNbg~3C2qm_z>eD5ANbADCd!={Bg@{O&*|`6=YLFq_Su#VGjKhm13ky
zvH|)6Q>Z9<bZT^V`|UoCsFGiCzZp2+eiC8G$zI{HA2w3j;tuY5810o#gr92hvOvZs
z1WJ5f7*_^~ZB7g<3G$VVy%bt=gb{GHFkA_!>{@Lti9C5PmdLXTya4*UsN@gv#QXBX
z;{c_!9>I_!h-DF92yIdS*o&U35SK=)B@tqirtzg8da2H;p_l29z$$yX&oio!hQ0um
zUVGZKTgO_wSm6B%sCVa7dS!37go2B_fhH-q5W#435%<2N7h-L5E(qFCEC9kvQ7rH{
zbi%1HfUg6zp2y$slB5o`{M6ab8bqS`EbOcUz`(+UFvdY!C%#!|OvE-tCn;}Fw8FGQ
z$)M{A)fZtTdm_yH(2XemAym9uy5bV5a4=gVnjUKOPN9|{CTA_i$V*af7DM0p0Du}2
zcMm6Qe>@%aN*M~8P^ieE?d5n13T($iFj<@l3Wpb1v<o#n_aL%vgAX(oC^;9`6I==+
zy!tNYWuk}nSz6Gz$y%bAVlP7l5L^74$hUo}Dg_KhuD)xcvrr?_azIf=pRNIF2xNCx
z7$I)b-D9fC-R%eC;r(`3aT0Z#yU|-na*#P9qr<{NqNT7!fQzdMtL58m5p$6+Y$FRH
z^J%E0nKdGX7=_lQ8}!!E@icyMfR2G!=m-G{yUtuqax6p6NL54zhAlZhwSYUs;WVVH
z4n9T-_bG5~78FuUrG=U{SS~xoj`y2v=ys=}7fK7a<->TeBky@k>3kYj5}5>BqR6J*
z9{{5Cm<;h+r}A9OHZG}f8clUO4jLkMl0=bvPCq1q34iAJs#TavC`zxTF@+tB?b)d3
z;R+TELF9J*pgSD`nxih$IXw~<aNz~5S7HUj&PvS0fW=IKe$vPIz+=jM#et3!pY-T6
z=#T<tGyE7N^nNyrY3wEAElW6t83Q5lZ^ek2v;(~r9oPGzh{)BS4usWeRn4CmMe^Zb
zfko_kO`CO7ijv6z$NXsG{yJ4Rgfs{F-A<vHT5QE47&nX-hJi=U@X6D?)^kN<odgLS
z<4M<3R1`KLidwDMXy$4o?XSAP)9kLFD$7`O0Xj5O$sC9AJ6Fd+k7OD~@DA}pH!l|%
zl0(fJzGWXwM<)A|ZZTRNO$UQ`<TGFqy^tG3&q<iZ=wUpaiF++=G+jVect)^%0<Yr^
za5_2|vYzZ51c;=oxJ;&Y&yG6)qIkUwSR!h-BFWt@+bFfNg;q(YXempCr8^?}pi8h*
z3Tx++n~6*Jng#86qD_M_(=QF}!VQ6d=hIPY?kzC&j7u+Qxmx5@)HwM@1J;etNR_h+
zVv~$(H;IEt{er_y&sNQLW#Amj(=YPKkTuyY-{@4<XkA^g_Y(K~XWapkM?XoFQ9wl6
z{Q*5BkOIDBH?j_fm*|E4R?dBPVzHxX?S@@uhwviMygL@@TS7nQXWr@3NMQlYSA<OH
zibyeL3CD;z-76rUOQc9GeX7XVh*az=gY5}*FLA346UEgJ3dEs%V<aMtV9gy)_%anr
zhypK5kvbPvB{57Tlzirt_3KokNh%-Xk<<Dy>n4qAMl_-Yoy4|Y#BHU3&KT7UsJbJ{
zPT;xkwoF>+jxdR{vS~g0HrK69hQ%&cEHxRj$HH=BPELloVj&IpuQ(O%gy|QUUw?(E
zu+&AH_)=f?gy7NMN}-y?ak7YDAuK5fVL))uPeH-tBeFhX?`e9KV=uZkMv^2~sLWy|
zNm8q~-2{{wT%v=yhp}AlF4h8rxsb>(hFF&i=D?afu1)W>e4ugF#KU4knwZL%aRb4T
zEi|k^7Zee*n8<>6QgqzQn)nxGo;%@@#W?bh#XO&Svr<#1W<=r8`lF4n$ypy^A0J)(
zgf9Ze{erMIp<Ao^FD2&*=1sP=4$gekBT?1vYhtDKC(aN4sK>AQ^tKWD>Wc+fn%+YY
zE*KOL{%+B~6H!^de&ED!zg3E1%+aDa{~+!bnv(`cu!wZ)14X}5FAC?fKEe5^^2^#n
zQ$SU7k_bpfv+}96k!;SN-^y%u)@l6QF}X<$?Ahqj9NOf7t*&nAQ4@I(Up!By$pKvz
z?@%NQcSYkGDuyJ|#`RfRbVw-TVZK2=U0I&<G@{F_LT&(JH8tM5XFXM<3W^zXIyM5M
zIf9Z~{8>bqRrO>bT+rfXB0#M=j;vQSHUeS*U-?(gbmG)k`DBa7QRW-V&X!R%nDf$5
zHM*6kGyCyN*WAy@nvx9?h<wZ$!)@6(utc7*bFfb|wGfNPtg|_1#$y-JAAUW%4DWCa
zJM+wEoUTSc$vQV-8!OA4pbO#z$1KGcDV~^7UZ<Q!@vIk{T}_ZD##ZaWf#;40lN9<h
zIG$XkMEopv_Skh3+&lk~Tn5_*bDk|nq8dL!z5#{qcy4Lm&9$U!s9LtQ2l))NGB)Aj
z0TkXN#%sjHE84h;9w&_Lnr^XW7M#s(2|)5GpN8BmJ!iR14UW~c5*MG8vNgH|bTR@V
z_YX7*uxY^4B0M>TuYq&v2I#U&RQ-}Qxj&u0CYKKC{PVQFqt<koFML@nzWi0fyFwT`
z-W6wnI27TfJE4KZ`D{?Oq8aiePsF<cPLBuk+i&gTsKlo+o|;fK+FY${r85^FBU#Nx
zy@W)YYfzk{8H+sv5k~U;bH`{e#$z?H57+coNvv2TaPv6tU}*T(3Z3Aizcn?Mnx3}!
zs+67|^OhH_MoE%sb{hm-*`epC)tTVu*#D&&5Zxd2Q+!U>Uw~Mwx{+tPQsU`Hm+q0R
zGl)y{+uGx2ESJHU?jTS^8gV~FgA@Cu;g1CI-S`skvEdyYfaHav$8@VR!laHz*F^}F
zJUOQ(JDv_=5{xHafX<CQygKFaWecNIlVCbRPu0l}ftaU<W)J0@gGWj1dAwq1B64`}
zat5P~ju_QgC0s6&OGhN2@CW1dG{U&&u0NvK8ND&mwL^D1kW7)dFEYd{2qS-xq6~pB
zrl@rrnYqgqD~OPOTsdW&N^2fki*h2^R|f_Br&W+~iG_+3tl9G?t}BL>1)$b`V(GBy
zaC7)(`8H+<(d=`y#dbA`rCs0DB*z6fhXxeyv*=f#VM0#vG9qQTv<F~zTr;>vYM`*w
z>599(<7re<x0MFtP~KLW1wt$nUo9v!&034t7PfRoTxj2=GoR4IOwJZuk>qO4rC}B?
zDUcR^HuffSmIRI6;w4?-FBfy`-fkf=xETCw-GCZO?tL^nb{1I$Q>hzk;}OVvy!`2+
ze_8nHIog&wTfpdniZhT&OuU@=d%pES*A<iw4!FNXVUAbbguLfVtpPgF2t1l;unC$@
z;w}hYPP2vTz;Ya+PIZ6IC-$G&`^eJ^F@^b(yjYN6MWyLNYs-|(nD8g^VP%&03qv)V
z1tjPrc^6JE!V7&QBy+_vkN+|M)2*i()mgdLEZGZLPH&*Psh&xV-Li36lIwKCmo&vK
zvWLoLZJ<nO3#~c=p0YzFAti4;P&XCyC2el1qSf%WKT0UFJ>NYK++kR9e*eAXtgo}B
z<maEYWk}Wq_Qq!bLJ@VN3;A?gycv-%8yrjB8?d=q_xIm#T9<~g5EPfMyS^^de_pn3
zp`+T<zAlWj_^f9XU!d+{9lo70nx{CJ6Uk=M%rNCsa)wQF!JUe%ytpCN5KQsv9wp^K
zrfHZy5B$1Z-9=ixRerr49?*1G^i3<4eYwc<R)Jcg*IVL67wjcSC5y2_#Q-$3C6xlM
zE8W_r*7)QSFD>?Ws$?%XO>VAb&m05$c|v-h@ug}6cewJ?=ToK9LX-5*B$bw2ML*%R
zJ088|j=R6;HB!)XQMqUv3n?r47*9F`x^~=dG^JQ>9WQ>VmZ-a=OpWVi?ZGiWr=2Sr
zhrc$PhhhW?IV}!n$z#e_Ly{?vwSrD+yBiM*6*3So;b*hUQVp2HuZw4DNf*&~uSfmm
zB-WC8)I5omc?lqW7IX))!!&cmIL<Mz&AySNMYz#T6$>AIEM%%!%p9F2BG$d}wXmci
zLGSURst%JrY0FGOT1L%Lm6EEC32lzLBY+t+NU1zug7}E1@6QX!4^w;upCUdMkycrC
z7woCE?n2_qT6e*1Zo01I$LY<k73#{?61rA~=$hI@*UiCMCE+{~HJx$&Yo<81RQ&Q*
z<cBG>aNVU^HKr8X5#qJNcmTJUQea5>yPOF#=C}S<<F*Ynb2lL7j)UvYJSP>SX5CPT
z-e&G8W=z(3Hl8KV%%u9Cc*v8rqq&K7CkwhZfQwpsmOmy+syJWF-Fop^ROY7l1dCgf
zQ+1b+x!IC$zBu}Fl!+TyZDt*9%o38z++V-pJBZI@qglY-l3H^Xw_RhUKc#G=L0!%?
z@*ox#s(WLF=yqqZT~b&kUh)N1nk%e}p6z(!OI*stXD;hz9`!A-i^!_G2<y%@tIcTI
zUUxCpwbfm8MZ1-y?2om~JRAl5<&FZ>=RtNXP<PQ~?<VWrEYo}Z-N?$N-sGiTd@uY2
zZ?QYksfT!s{gQq-(BUDp(VYh2{3m&z8hI&p;~_8hE1!S*`YwG$=&qo0b1%pn9zk*(
zKhoncyVlMI%lp0Rc1q~zF^(pQQ&qIm?I?(9t#HiJ7PlQqUl=BbDM~ETg$3Q%sXpi&
zX9Tgl0Rxf$+L`Q=h$tpWbUVzXUOw_;N6}X5ty$J%$7MN)dn<b=W0Ens7kQsyXC$tl
z!A&+s`trDFrlXn+Eann(-t#LHnVXA=KPJgPZQ?F7Wp5S)P!HW~9RfKq+0VH}QZ76N
z(~eD<kdkn7SCU1Si-h-71^*7+W$94>mRPR;(5tnz&!V3<6cC?D+|KUFjTVlrTaW+-
zT8l2>rV1#iwG@jngrtbjq`{KQ`HPz`=N1YiSE2_#vo7w><}L0^k@n{Jd`V0EA7Gj<
zJ;6U=)wrGIehEIxT9>t^7g+UYjpC9jhBi)1X##Vi5-+X*=rJ>lVV$9?ExzV&ncYiy
z0LjFA=GI#`y0Zx<F)2(U&p8Y$h{{SkSqitA_Xrn7ZpL9+26iN5JI2D81KAbJtZY78
zo|~*(v`mca<odl3Z8PU;rmakS1PtBG=O1~a9uR8%iF5q2{aqVlYy6+P`uxWmx`V;E
zyIN^=H#S?Ft!B4b?v%ZzSMS%|TC>q@H7ez5t6H!3+@9NV*Zom<FztEkvgSH@@))^;
z^?>5`T%_74!>8G-(_i=`{#NUaX8HeAYL!N#QExSx6-ci(8x{C_XT8=wQ<SyPB#y<b
za`W<&o%@4)WDIwV_T={bD_dN4iU;lb13dMSFNe_u#&qBY&SV-)#u&7gqN2HBIPUr`
zxn~)AkxMbk$%j$*Vmf*!0y?A97nnBzG%&b!#+M?*l;kK8K<Rcr9HPeWW8~%k-g~os
zaB#E(lk4=KZ}(2}c@zX%1Zd)9c6RFxhV3w!5N+9O3}A&FsN(!9tE?;q`FS3N<J054
zz4puf)06hwz2o-5-uHV4DsY_ZOvaa9K+&cy$4p0#@&luAPC=jCk&Mzcbh{TA+}Q2)
z0xt|<1bWeh8Kvjob=%fnuC7X0vkR3obSIZly9hmT-ndu(a607fITs!-5^H+XT=ToV
z{ny`}w%=_3sEgQwXrKTsob3N=@90(g?dkEiHBe^Ptpq9}^_(*qWD015F~`n52N>zy
zYCn(x?p3%Z5hy8z&*9PTUVHy=|8#%*VE<qHhp%&>TqlMd)`;>|r6>nXIhP=|6+y<5
zA)X$Gsle)^>9E5?N#Wnlr{gI_&q_6?B58*dbc$Q?N-A<2WK0KQvJJ<<wd58A_L~=s
zIUMA=V!(8N^ez(oc2q2&`#|^`GcGThTG5^=Ya!8=zJw*`8ltm)yHndXuTM^o&vs5>
zW@pXl5Bc15gaZa&azc@KGMSFH@QPxhPq`=L;|H_6s|W^C#At&W@7)0;f85H$|MON^
z@By8tfGYij0xyerbMio({9y6m8i(W<oo_G}UWW8IumnhV;h=nL0WX0>4JTuuJXRc2
zINI_W28x?_oH!OCbs;AtNEhyV5BS9!ahNs^78ZCCxKO-Dz`-3q3fusF)(ksw<U0al
zbOxPz;n98o!rbThhih1hN}fz7v{+#K<Ww@iIBoR}2cU<{))|ror%s&!iN%UIKSaNV
zu2WkjVk>le$~n^z!^shey3h*{qdgx~Hdu8Cl5#or-dE%gf2#gc5)Rlzw@ByXC8~Ud
zi@Q_su22cWc#;T*Q}I)j?btzAe$nSH5~wFcI>>E=L=DaaQT`<+C*IqEeq*BZ)cNVz
zbcCb$!ug#ss>lf<6RKLm+S=MLX+RY#pt1y}CsboVjs;ScAi?w~#MEI5{qCevE5#&6
zG|1?uYZAC!j)06xiHK$zA}3!p>ngIQQYI+ECOKD<BpuMQS#J@KA+6v00p2=Mi(@!(
zyA(_p>IMTC9vuE@h-~v-j<804hlxYu=%V1@fQHP0V-b*Eiz=8yiD!{D0%SW==7<6;
zUhu-op;|U@W>a^1eu(0K>W3FHOs5|*E^DIQ&_4(LDwo6ZTS^w~OoyPpMgDtzHJSmI
z`@@2T03@UhcAA@29hMDu0Y0^z5|Jbv(+g1U34Pfz@_H(>Cv)Q_SjmWc_ZB8ONS7(f
z(a7x#JSE;>!F`VrsySM-u@=L4Q(wwR2LeO1fJ)3{QXU9d69=$OcbRf{1^6kc?fjzp
zCG^8wPD~j~$&XoUsaW6cX*J#Z@pSL_5LQ$2I4VZPyA+E*toZ?pUP%EV&(lNe@X(fc
zE1Vu(M~p7$5HwG?NYBHtt%=s4NP5l)z7kU_fDvt9xIr(8C^fH;CGyVzM~nX)U}-f7
z{*=;f*hf>)_nt;`|L|b{5O+P0H8S${4!2((?1=(-#m|DOF;k}pUf%h}xdYu)zjJCt
zAY*nl8?`7Nr$$>?IsBeQ=j0=lIU^Va*Eool>Nr+3jKtqP(wAU8S%@c6otgpO>|iO~
zoT*-lwxH8Xx0HgcO(%qR8ThFgH4`9*E;8_QQ2SkA8k%&sNDHRMas!~x@tZ-m6B|vU
zbJ#QmN#oLRe)SoYWZ^&)(<$)ov{$gt&LZB>N)_QHWT#y$=2hDE!4KR2JQ39tY4XcX
z>imazg>$rOKu9`?nvMqkJ8y7}fuq<#c~QOg5_`wh7ranAt4;+6Kwl^EraPC>7tg_Y
zyFVC@g963Mdhvpfurl@txUz~HhC1M9j?s8=D?Z`Uib3+nd?wK(CGv{-mKSG0z_?|i
z(u)`H<zHuoN-@Lau_})27=NKA(4Sw{gjxlqzR{q}s6eHO^fUX4%&gD4nIhNhQ~dIW
zHl)T|pbR6+OjOd1<`<Xe$!VH5bxl!HZ0t$D-W^XtRb1^}c-?mt)Z6Vj1&XuF-*H<D
z`!Zmffydf4JF|Z70I<~;(nf%j;VacHzQFh@MNTS96GVb=qw&!1vX(I^d_xdnYfy}N
zT7wBwpX;fYzVRq9=uaT)5=+e@f_4twW&dphOHJs4)<Rn)b~1^qQ+b9$S8}L1DGY9%
z>Is3LztP%mAubaoPNGdF?t^d#b1e2WF7CN;)7<ec+(93=JFap`Pu3?QnQqARLk4j^
zbpsd5@&1V{(cJ;Y(4jd2)EX(z*x~i4dl8IB!pK@;0J;}^wh2!sJ@iJ38AA{D<DrjR
zG}wU)D`v;*lMxDc@MP~2h*EL*ItF6G4i!U_tWk+|ZKa`^!^^39mY`8&TQ_E57hv4i
z&eu%Z`0qoaM4?!sSw9++Rh0n5{4hq_6NW(c$5Ua=cSx^NPh5aZ=xLlJ1~tYYy5s4f
zCrns)B$}X`bLm0vK^#Q5f4zVo>0V}u3+lDOg@XN1|I>JG8T~I?V<Gkvs1QUQmy*My
za05ZKq~^Go$Wiu`8wz{5|9@^@USo63QbI~pV=)yskr>}uEw*PZ6B43DWXYC4X@u@%
zcoLMhvqMQjzxe`@EM0Sa-A%KPZKG+N#%^rewvEQNZCedbY}>ZeIB9Gr_w;@4zw?<L
z?RRGPch2sP;9G`Rum{cH546p0DcT+?Vq4u510NR}>427KsIJXA>yuMnVafAX&b{GI
z!-a~hAoJXju5E6}58pR3b&2&Mfv}%HJq6HHiBMWYx4!_!VErjfTZ3b3((ihI6jj}n
zHWhGQbBD229mnV3>}iz30ct9v$%i-zTbdY7?_f+m7tiKL^6Q#IgR0u`Xbm9yO?W*$
z()awKrU*&pPCh{%{7m=y(KiK|IEHYkVj5XNH}~)Tj<Jzb1n0Ir^Hl9bl7=hKBPFM=
z=m70l7{NWKeAE{C1gLk3P;VFq+iXww(?Fs1Kf7I37Gk5C3kKEJbU65D!FZaElZw%G
zasN(T&9eM#(c?(zH1$l5n&(JRthB0B%}}p2t5nTVue7UF%~G#4tW?cYue7XON>{Hm
ztz61hw;oWd{8_X78?Zdwsg+U>SU=jdQoAghG>wbQ;Xh)Q7#b%|X`@xDyOl7dPg1GU
zQOcO0NHP0N*moCXM9OYZ3_sokwYnNBM3Yg{&>GbHoJ+2{qjH|dg(1lcV#RjFkay?^
zoCk5<&<kLPD95Ah{ME(=r`3#@`SV0sChKkywWl#&Olckgx}_k)q=MWB`!Pl-kCh(Y
zx{w|2w@Ro`^D~IfRba#C!Psl+zTX?&j?akns|jiT*2#Owi1_GYXPj;6MJYsBOB1`6
zok-W9)j;`KquDxJ)L4n=`As9aS_0Cl9N>!2!Wc7)oNa+QX)e1a265#1;PvLDIP|zE
zWG1Cx%N6g(^vLA_>!s7qlZ>z3-6w-C{u$D*(Sat$Zs=%s9xZ^_@`rxsYgX%*HjVn(
zVc1*G)3wL_lII8^APoCOyw*P_<g)E)Q4*o}&gR$l=MrKd_B|cmE9objqUAAz0sWlM
zD2KpYFX7$?Z2P$v6H*td*j~q<tV*JgYiisfpWrQ#PCYp&JBV9nm_!(mZp=`A5g4WX
zf}Q-aV;a@c=D?RP78k*8MX0@}_p?Y5(qH1o-*rcB#8*^yPzi*HUz8vGM8%hvE55XQ
z|JMHb6!K?$cv(?}AhWMW8s2wXfE6qtxeC2px`V^?h*Fm|zK<uF`MMDF%c-%RzLLY>
zh?9;Zc;4x$yN~D};0*gf4+@49+Y)3#8FGOP!76>6HGp)cQxZO&>8U=xOS&;_@}~hU
z`@ZS{%>b*uVWa6C+ruEp<2fxqy+YU-H~rV+AUQ`rn+{~SZ?th<9~(~JZx)R&aFh)_
zQncd0K`}VYTrz$cKaNM`JJc~)28|fhHAe3Q6t#M^l&Cm4z4Yvoyhss=R`(uHS@)$%
zOsa7#CeEL!cATg?gFs=a9j|R7$2>pXAbrxCu4zt%-+en4{yla_clSn+im8~WPH}$n
zd}yD&<D?X3&l`6s)(kp*bN=K~iW*ouj`@&5g%KqF6c{8X=9tDJ{*KW;?UY19!9wn=
zkzfx$u2bG<M;uu>k=tyZ@s%+B$7tsqWb_|=UCL0f%o#N%&$}f-VjxJ2S&#0!^Gf}~
z=b41GCEPfP*01d?39v=?6`59DTu_Go$ZUeyr+>$jGN~ftvOOp|wxU%L4kU#<u}8GI
zPRBYWrOBY9Ne2tYK|8Vti3FIUTMsjuiP6%x(-O&J7#)|_fuiPJ@{?D5nuB~1nGJ}G
z^TBHq;D?y3w9Qt)_kdi*<<AkHV=#rA*<zK={4As>GsIX($cCsTxfjguKQ<}7Zqcea
zcn3s$*_7EgDSRp;MorJ~fAM%CXe5X{m&GLU6*KLCC=*ZkeSnS@rJ;@z6~<-jcDwD$
z{Mb)$e}7J9R!dmP0h?O~y@@Hq7DWP?9F=U`#o7ElUYLGIVXFboLv`nnqv>Gx`tov&
zsqE^h<hdtQ8GKwL<t#?arl|r~;1Mgt!Z4KdXJ{uL+j@vZMsM|ccPOYaeSYveT$c&x
z60fE1U?u8BPS8aXy22FdL+jVKz1a?GSWSH;3_4g`{eB0kU6f>{Mc;b4Kfidx^93ZL
zmMrqSjYb3`EWm@%Q*!zBI5+h?qC@uf(C0z_a3h{tDxGRXsb|$|<&3ZI1v?Yf57H!4
z!0(kPGuLEh_xLWV&$-j4?YgIazhPZI^h^A@F(Bn_!0ZbL7a6$emsMs@UFYZnfs+j(
z!x5CI+nkNF0k$S>K*@}Y!?CxE>OKom+W(Gzn{YQCMu432D3-rEe~E`F-Hy2xvy3CT
zELer35yoi-y<sKTj#Rfo!}WV`gsgZgk-7~_`qx^D#+I^40$8vwOh}IZ(B$}Bow)N9
zLCu&~(Iw6mO8V}3IZtXaHz=462kx`G!gt-hC^Ugw`Y~nxJjND)>7%1+g4krzIDqB;
ze0Q|CgHJ<JG~b9?I&X6AUY=*eG|s&_Z)Z&<PcS24`ED-ff&1vfb`|M%0cTSf%{uA6
zF)n`Itv-az)_RbdY;}^Th>sLuISP^pJV~k+sVZ|hP+Ul7?^3;Nm&?uCs!kIv^0a<z
z7$Wy|C<Vc{RW(*a>wQKOM1yY<D^p%o(vAkJlLqoRTs*>CUCE^qCT1|>=1~kYQR+3)
zpMJCh<!st~o3&F@dj4G0k`ph)!lwft4Y$f(S8R|mr{(m*!7H3iP>cBDmZk3rehFaZ
zf*G@vCVJ}3G(>S3DA8bY#8WOgFQ>4Jo=N};)nuT;09gv>e0v%L#}|R;?mi%6r>3DC
z&kyyA{J|{<{4wjbrZ|=?X$qEBNj5cxI6azZJ|<znR%#{2fblU_Z`aG-^ZTketVhK9
zI4d5qt|vJMh|4IxM*#lKKvOW4(?xFXDohJlrdtU7%uq{iz|+~{$kXPq=@r^1&ldl?
zIq>Q!@yBys!^OZt1N&du^NouYujl*NOXYE|uU7o;3m>w~<6-W{iPZjBP+78EP`ib@
z%8%uLIO+|zrJD2!33Ix9{(j5OS{Cmt<cMUZjkiXS*oo(LaZPXIx!Fd_?|bwv(8U#I
z;ct%7Lgj*x4YsZa6ZEUVSEUj?bZS`SY;JbFhKpoui`8U6yumG7q4}Fa8B{UHUR=|D
zh?^v_gcEVwq2eBnd@G3j2<#e?aAQ+(dyY#A?vxg#4Ca$$;Ey4nGg6DdD|S-};<4vZ
z%aV|Klz8XP>U;ht>H5z+GH(bB%9OvrUCbmB({e@%`XE&Oo~d?z+@wLe(adB}HiR|4
z<SSVgH;+%7AB+rok3~rg!uU7MJt9pomZl&u&$GX{&!ZJ%nKIBZZ4a*TQB(6r%{g#g
zIhxe2Rz|T}=QQBvH6~0y7h3#^$4GzdD12ihL!vbS6czbpcP&wmHAE;J`*LNR?QXWr
z9zv8PAqoZZFy%sTjMO|SkY$7iTFI4?P&O|-baM}Im8-!~&PGM-oq22#N-EJF216yz
z6fMpaF$!&<xyAux>`=Uu;c7=eR1ZHZj-~FFpdEVem<o)O9z-oxsfys}CEid0RJYsO
zJov(Bt$R&%|Lz+EU{pmTM9Ks_wS;{3OUPj)qN7VHC9nPsG2a-*3vd*{Tj_og(`QtX
zBXb)3Tfpp_g4-B9sxs?olp?QBsmfhnA8dDyHO)`q(5&%yy~C^K5&6kgoq4>TZ1m7^
z+$KTumWZA+U6+}jqvP9@75!+YkJ(s@*4)^6{n*Qtj*W%nn#+`C>=e05`DDe>`Ecg%
z_Mh<jPPr`ozm-xoH_|UwD+*dlD(8IQSybN@r0TLOg-&B=Gg&?ivjnTK##biU9-|n{
z0pDU>G9<UI*GeJE;!mWFc>cZ}FZkygRh3!mFW9Wsu@FJ%kdqf9Ca&nvj<G;)2drC&
zmnp^TP8mPItymP$gcu(uzCSr{q4IH~(&VxnU^88gma}D2WgtqY?005DId_EF8A2ZT
zK$-_C;8(^#d0zbW^xn$_0~yCDi?0m^C^V$^6e>$O(G7wcD1_Tm*5DBxirHm#diPz+
zj|$Dleh<^}?*yxsB-D`$K=lfcz-+4HB#6`W{ynFi;bVT>@Qn3ZsKer|m<xINWrQi^
zKjV{eg#6sL^rk<+MEPoXAS?4DzTty^S!+q{_EWuSiFSn$c<^#Syg`A|Dbi3NBAQz_
z|2WY5lGjv~zw=iklX{N<Y(?sXT+`*^HQ&QATN?~KOwW=`E2L1A05YNDd`9Zus0Ft$
zzV_c_rKtseeBa)9-TZxjZ-W>2i(<(3m!6$%TU^Zh9m{v-URB2g-MPFs`4;}c@UV9O
z-rMZYgOq~$o%Rz`hi78C;dmPJQ#oJ>{{dPIP5~SE+jmy`sPI$ld0_PA>G(t^G2m=b
z$dA#iG)u=I9BHo@pUL}WEEa_2>G<bs3$>VPfqpAB*G$gR-;|({7xe}r4#ZDoY!DX(
zJrr<juqxF*aA2BLGoi3~SYMWzF*vgW5+hAA@!F<V=0($G?A4P~M`Kw<MGIGQa!=dM
zJv)5cVN*(WqI>N|G%n#^xxhb}ZQB_1yQU5+T%@tKGy6t0ubcceQnRC!T+IoNBb{oZ
zoM{47^eW)}fDg~xd|fwc+Fz0BRHRI#c-9>xuZ$l;ef1Svb!}FyXvH`(ftGQB$8Q)2
zq|}2ow#`wphn%sYE70L9G__NKP(@oUO-+fJ7AeUZtIR+xma*b0s2VO>{iMlKr4fs<
z5y<MQ1|egrU>$#WHEXIjEKp=YS138RWz5eUEL*n_Tw$xiX{rZptHx={9E1@=z!KB|
zr(<0og;23=(n2VsXR_)CQTgjaBbJrnZw0VZ1xdu|TeV-*2FSt_5=WVURJidlSs?}}
zmdxiTv>2+49oZNd{6|oPq^Dl7!s4qiae5RhnKu`ZjhHio-;&Yh>`0tivs*%EwStST
zc!&|}QUv^@^9Bw*zciD-R+KhSp#jHV`l~iwZQ|tGqlv9+4I6Dx%Vq_LnW_(oqt3|I
zEw{5#!}-@Vs(q=9ZQ>uJqic!gBZROWOf+6o%&S;KyJeDiip+Gi%_DmV&pHOqPwNM%
zrY6h7b14dQ_0IO}k1VhvHCv}5J<BEA+p^5DwI!{#8GmZifVofuVtlbB;843oYg14O
zgT^+{>@5h}jehZe#K+E95Hb_j5+y+L8NO?V?$p7#R@Y`Zj6gIhO8e$&mD&WGhYFCG
zdB*1POe18b{<mp-c=)%jnK}z+hu&$b=t}S;)U=gK<%8K8MO+2YmmQl{o)u~!r&gBf
zAMuXKYBNXIqWSd`7y+;%*>tlQttOx=K-G4wDfIH|k@0J)WJoe7H0D5o_(uc3Pcq}z
zmbhr&57oMWd`&>UM}KhKcp%ZNL8#MWpe%K44WQOV(N1CF>e8TO`O_fF{C|AgWomeV
zalw<-py@GcTl5Xq8QEQP)<1{JU^8?p!MXg^mnh<_fV8$X477R@6IVb;d?wK0i36LK
z-3N#m;6QMDP<1vfpuZCb=W+nrW1uCor17>cKp#vUU9kmV`Oug<wy9ULQevhMvv?|i
z#FaC49w=b^z>jDJYA4RMD9oQcbqhpPf+y`3X=30e<1=NyFR^fLoGlGXX=5_uFE&|a
z=RqWot$pZqHB*5wi3#LsVJ46M<7(jOP`dOm{jI1vdTwTHY+R&PH*xNXWga94Zj}PE
zh|(^COBA3`KUVaQrg2X_DGt2Iu030+RcV^G_CPxgC8=Kn9}S!h-eNc!SC<8fxWAg&
zx|XcgLJ?(f8C{CtTR8W|H0CQnJ@cFAp1PpnOkh$g{-tdAK#ynk&ZR0>up`xg%BQrN
zKnthn8_qPd5xvsn+US91W-&hE+!h-qiO0}RSGP*QK@BqF-y4C<PtG>r1GSvqcWSAE
zSTW|uAZolCIslCn14iO6Vx54RrkQ#?sexqu#5s`Ul_REqjXH5I1zF4h15!sp4r=P)
z+N5*At{GNUKN<SjSi$(|nz1#ycWzapd>UR|H(9+%&G=QRx+$Vzkvs$hHQ;Os2W8?M
zu9^rCQ=x0dUeD2?b(PYAhGMKzW9$ES#fA+C|Ae8n+S#YSfpHpUL2w|y@P2ko+L<`G
zVy2fYqx<8Krq`;*C&b(B7MWq0tFv=|$mTZwOkGPclZ_t<Rb+{3s3cI}@dQILUfc{;
zZC(p}G5e3xtH~<Q)HTM`wFI&rrr8>3=j7gb#mdgfKRZKz79&;gQx{?JS(!Pw0yb^g
zG@=(Z%mxDCJDRKp=o_+tQ3AKJe}p6&wrGGHq#9s0N)n#}1``y$#l3n7$O$~D0&bqR
z2BZxuL)Vp;*G-B+1;!`>G#keL8M<HHK+R~Bp$o|8gl9Wfo3;b0nF`UJ3I!R&Q5Ad;
z_5s)>i35NFT|iL<ZBo}(nR08`qP2i(?*yt{A&7>{^g>*U6~pMNL*o8X!=_yz%^M~|
z$MLi1(iwpAOdZ(5U)0oR!%f+0?<uOjES?*#6=|Xx&K^R~>IzkjGjwg64cOth`Dx5s
zfKdlFP#6N!G|oUQ?9d9Odo&xM_@7v40<n{LU^Iyq|7I22YBG)QowsjT!YXP5IkQO9
zzxRRwGsg-z>s@E6<DDie8fGgBWW5MLsH<U_ue=?0ld(;l<55cxX&F$}UC)VOnX?-_
zoHClPXqv4gTbT@4kVZ8a3NHbk{&@sf$tSZG6Dj-M7C5^_b89x&808I<)d9tjT{Xa@
z03vhC;z8FC{~vM5x^{GL9#~TTQGcD~uQ3y)t~H5an%Mx%{$o9{vsvZ^W`&iS!H8OU
z-#BpEfoKnnv1l0gRx!^&RIc7GvsXK3_5zxaRn{B%51mbHpa`^^s#!eiK+IAsI(Y<C
z{WE|bZo(ayK|plkV<mJ0AL2E2wQX1m{8=zpwtg@NoK1H23Vhz}$Yo8lfx@1-3e&X)
zc3@tf+AUTZMb=JMt7jfAQ)H&t+*$zPn8o-O!mI@}KnH}&))t7Rf2?aY>Q>djsqcFY
zr2zb^M(aQ@strMYWP$?0__03>l{pN^?#dWi1HN1mzBU6)*IDZDX0rw$N`P40>W8Jq
zNi=mOs90fRX9xNO)T52RmAm@Sq!q*1fK|KQ=^ddW+G#r%t0qKHASg8R&a$#xGyaJL
zY#;sR;I06Angr<41lVgGpk`D6{2_=@gn~?6i)_3T{p&frz#mbB7TGk+*k{G059&rq
zHQMVzmO8euY8nw!*DR20NVV9eLlNz<^0Cgj$cTnvnt!zBY}sfWpxU_AgY<Pb%qSQ4
zG2>%Vrn{KF;gV(CUHEhPsAm)W1WFVTTnjGgS*`^N^=Sa{h}njUN`RU;hq8+@1Z^Ha
zm*R|SV4onU*+2niyT%iZMMWi0tO5hP0$^_`gBYK-0bhV*b!K&js33$^AV#h9$G=z@
zgRh7w?9*U*sEqLhZQ49&?ZLG7F?R$|)R~3yb|iX0wVf^>oO%YCvgHEojApd{pYhy|
z6~9y0zIHN{2|TCGl0T@*Bv;VRvrAT;k!<4JtZ9>|1YnvVo-)ZIfqf(tCu$1}5jur9
zfxI^$JI4d+bXiNbfDq`^03RaO8luLV?JeaU!o02XYwigia6AFUm)U|ZlqS$7XZCT#
z|FMFn7%I2JzTsu@j034#p333tP!l#dHBtm@+-e!R|5!Wln_|;HXdANVbcpbYc5MD9
z^npvaF2tgK66%BO4WS<B(Hhm$#Ga#QlBsLbViT8vNs@Smc|1Y-htE+F#l$&9PIB+`
z&iL9t+NyInod}Q*9)it+(jB?C{~u?LT7&vwO7j?VhpJ_Nq)Jnspc?ZG5qB2;EkZqz
z2-&eYMQwrM@;@`ropn5tO2(M+mrJK&)c&nm{{-ln7g4kiGk1ib;SZA9h}n7ri+qkR
zwu%^Vf$4z!%XMyXZXOQ|-Kbs3#_1BUc)|=yLU!Ay{x3-N+Gnn>fvdC)+EcG$fNaB=
zYGCAD@wY-bxNPbt?V1E;DwZqvu%yrwu*}`GbJXGv6@eaDgR2EiMSlF}e~78bx9_Z)
zW-V;BPK@H2Y#zzLpj_BAvO$Q^HW<DsgzEYY(KoQn5Os7>P%?p`7&a=3k<!%)3{k`Z
zJ?;2d|2<KmpJe(7MTtT;n{dv|eTPsF9VyjC0bDZ~EotZ^0ILUzN4O?6>lZ2i%I$X9
z-cj8eaHjxtNzZ2hlf5vLwt;<?$rMwqDOL8nX|CYoud9AtYhdvgmIaO@2g+Ny^#m)z
znm9*Xtqu4k>{@>f6i9YXR3Wt3tYK!~yA7)L1>BFB1BY@gb+;BR8D$_&;L;M+>esCO
zr+WjS3IQy3_)}H@MF0@=G{DI7*3hTSVNKYqjxV!O6IomT=lR)_z=yE_(DNTkkX&gO
z|J<3cT-<V$1~G~T5dhQ$Xq(^@nmRPAU$oNA0joT|fGw-=hN<fjZZULW?Z&^wRpaff
zcE#3a{h#XM6$SHnIv`sX(aK-k$8)KFr1PaW0?=^Ab=snFeQ<;S={>bonb^C|A7aLT
z90t4&hyFJ?r?za?m`s+MW<{P5Mab)U{|k@jrJ1^os5WhbAbZ8iP1)xE(3$_@!u4;l
zA86MPZ31jkZ~UkHLVb{K85p=J>T}{O)wlmebN(*h4Xg+;V2&al7Av9tKbap^B{R$&
zL(E<cYS0{0h#iyDJVEDal{t3GK)($x2l((`fqP6~)IOw|Xq-oZ<;&e9+v9&Zd9Hrb
zs`-zr2nmnu{}bp#3-Zwhp^StvFr_(in2<&cSZ`|{&UF7pEX%zT3pvjAso~!wfc$2t
z&a<tZ9BwF`vikodKW8^!(ZceF5X*W1?lKqGXubbI(w>bytMg{VAfpR5NN*%%@PBT9
zs9|cz19fM@n4{mL{+DGfp1B-cn`Q9P`P#q*+z~i4CVs|63z)1xRJ8hQfEU~Rr}|^c
zs%J*o1X#?@BZs6+27ooEzGv+s{LCC*$ZSAq5oD>*up{qZDLL;zl-CK{LSgP$S`VIT
z|Nq2s>jG}>>|inw;kZwVYS)^_ihzjrEj^8-8w`zK>)p-D?^^$d3zMGxF_uw8;s$mr
zMr-Yu%nM&vAg0lptr0naUk;&7e&8a2|G;aQMs7n2RBqomn+;0V@}IyOE!3>cf1?fD
z>Lb$bt8jpO?wX}jDd6V`?I+zTuiF1d@<XphAJ|M!8Mu3udi*bKu2i{HVI>SyZ2PBS
zij)4!EnW>5jj&A&;6yLm*!(fopb^W+zOa1^L93%OVfzLA@2DyDKLhKp9<+pn?BFsG
zt^D9!ZCo>em3r+ejXk8W3CKCR&Q+z4x$*H2&?Y_l){!Z$K+Ur21V8_mGW9$5_k(M5
zzvI`4;tfg_?AI>M)q&OG?E}k#4MAemCZ3e&?c%?(=Ihr$dTC(`+@H4vuv-gC|9>2N
zl(D4-&4i7}E^}8Ifi?d8sRMscY!N7tL)^c#qWNv@UrqGwX{jT|Q5v-|u#Mk%WEuUR
zHCI1@s?C&*g4rttc>gC(0%Q@pMT1=%jWluPKS#n26{kN_2VovZV#L;5`TwnLu>J?K
zsZ)p%a}*~*|A`_4`5Qrc@;_4M```amN`OGLt0|6Zz*?ZkXsJyD(?mpI&)LUG*${$8
z(s%(VD$@l@6ZKr%k@|UC=6#MI=A`d<AZP{}o;mOB<~{F^llUj0&=x4f^xpY1ziH=l
z?(G){&12A4nAdaqjh`O;407z*{d+BbKEL=t1S%J!Q;9G{DL;V=r<(hG09Y>?JqFfO
zioaq5FVdSW@U(WPBF1y>^(CI%FlnTyZ^*@`3h;dXL+ydeQcF`)Q{n)>$U1{{Y=ijs
zG>~u2M1pd{v2jQ$z2Se_vqE~a1gsTihvcepQ4<kbuxGYA0<K~Uv8P$=5WAgU(Vw5-
zbme+?tY4*327grV7bQU-g9p)4<-wM$_VkVX1mxNXJ{vnugU`f>$gm%?GvJ(sz`|>y
zm8<G$TdVo4_2aDUx{DZ7cS-`U=-%2w2R$l7jxUY(oaBqBgKW^09=rIYe;g<bAZKt?
z{Ar?gx1~w<!|+UK@JyL;3+p<jp&}&F=hqyAUpIh(yQB}z_wP1%hwTR+kk`3OgL(yl
z=QTe(LoQq&!920jdkr%fG1S(6StStVqKa;8{L!z>?m@c{#0mFI(mj<z$cd>r0l#I(
zH!~}Of+&(5WT7(=uRsWM95>C4PzAnr5Rt_d-sjd&Q>yC%r7k!1qJMmoG;)pYt1=X4
zpA&J{v2f<7QofXX_k5hk**tcI#KolfkUg?n^!4F!q3Cfj_p<LN+<|@S%5(sQa!U#l
zvKR;9LDfo9wuA@S6i9e^$bEbR4L7mpt^4x4`ocbhftvgYp~PBuhdB4*S`xs27KxQo
zFIUD-V%vX`^c?)UgZ5!RSM$42%&Vwc8JRaf{ooBeLHg<K9kx5N8_M$TPvl7-<{V)k
zW<qn~)Gx7O5uxAGB)NHNEh7AWk@|&sYsB`?or&u7&cctm6O=>SPCZz>f`Xx+Mm5Rb
z+U7;FsdiGa<ir#h&*yjk@(yfXw>Xw~K*AmEFjA&tp-1C&KaUc_Dux%Vc*BXJ_(su-
zj}1}2P%eE$^R1G6w?U;c`ON?PVej(Ae<S|;Sp0LAf`8$Qw4JY<;^_<c;rvqj)e|=J
zVJSTK^MoQEU97JWpL3M<BK}K6TR$;mMdUi@dr59~?*P3k%@-5=;S8Bi57{mafkS0y
z*T6*@v{#|P_mtjmirBpN*y-TeKb~7!OAZ5DGekbU7&L`m0>s4UNzkPFw$<idKHPq&
z9vobJ8hVu?o^?p=?IE${&Ir(b5$F9FRMV&A1Nm7Ky?bB|QvU?zf4Ch74FbU%6a)b&
zO77@wQ1LBu6^>8P)%!R@`FRA`N0mcMvG9Qe<M~lL+N0<0G1~KC;PI2HUtpX}(dUTO
zT7xa&^P*)jAY;1e%5m$Rls?n1@Lxl>1pE*zNQ1{${l1BKU{g0mTb`<~r|TD+GrvWb
z4s5-SpeWi~MB1iuvRQbvru4&Tt@7eYt#9)rit_T4Rg!{qXqZin&b+aqMAj!g^QN3;
zfwYxhuOts76TN*#&7Wj>PPrDKiQy8Rw+53etTDAdlzam>Cm7<QNhKWWz7oGcUh#Z2
zp)aJXV%rje2iDcu#0O8ZV@ciqMAJ$Do_O#>3$Qz(8$J1;Q2#yeX&o5EK_V6F#POR*
zpt=~m(T}dr6Nk-h^95OjPO(bFf1~TN5SNaG5o4X?HVnrmXkVu&Ct69c0&05}Ulc=8
z-KFWk`F8@#_X5yg2dF4;IDw|E#~8jAi$zzKi4GZbyyZU<?bys%#3v1nOErl5F*N5y
zq*a7(6>!J}Mb3!?Gu#9CFCzqeg^_<#WDtxjB0-nj1!`s&cs8*>CW|MqJqNhS8R|hS
zi`_Ge;ff0<li=uvf|j9VYC)61ePduwD@r^r^x>_-eX6-ah9b_=Ew>~|U4x`f)bwLS
zWaykt2(Dc^CIJ&lya8Vw;LzowI-eu1hPYdWu}mYQS)7-!g2Z!Qh^N}u6%E)RPlZCl
z9l!L{p@-w);={TdTlRF+0A~>EWy^(0mCJRBJRucoTBz^WfxZ;eiXZih$L1ZXk8t>`
zCm(X|PS1xWKLnSKf;16<632TT(q%rV2e%`Z3lRENjU}l#r%<m*`;(lZEL9R*S^YIW
z-aqGTbb%5PBg_BM1nh{FCW@jGTJJ#VL?!wiHr)uK+Ly3QGy$<xAWpL2#F+=jIB@Y!
zm<Q$z;8y(uCk#%@HI^I!Di-?(NsSj%;-thX2c(%!;2sjNl`#TasCgnzUF)$wLat1J
z^$k7!v?flGHQ1qU1ZAw0FS7txu*&Vg=oJ@a3ybd+W|9oW8@sTLa)X@v>6~f?WdgUe
z*VPBg@XQc7ysroOCD@cHZBzlWZ~)Kvj)T8=7USJEsK`Or$$Cbtg({BcfP;TeKEzDI
z4JF)g5Gx^CD_s%`EGlozhK@z{5Rm{PvhQvrJH+_?=IBu4SUOm)U>?jQC3<x7VyDG;
zPIX{Pakq@UbgaxVWw>_PLJy*W&&m82+K$Vn1<!rQ#gB^}IMKadneg^q2AF+SMomt|
zrF(&c5X!Y45gStS&eCEGG{RDt8K&0IkZFJn6;FBjQagee{}t%Q&>@FVWhT*bzh;2e
z<(Pl>RA4B;6m+4*zU>D#*m9cLGGq=M$Mk9RNYJJ`8f@@sr<3$yq~UHSeTBhQ!l;d8
z3=IZj9~Dnu`l()mA1{?dqrj?bd%zup3F;3w?)?b(0sX=^Sx?kn5?V9G(+O6Njgtkw
zr*xw61Dk{PuH0})`Nbw|xY8Vv0(o}XJiENyRojJ4CB7nT6X3!6IN~07p;-U+9dO0l
z7{wBeY{S}INpLH$m0qGk=i?A%C<Eq*g>FM|s3rXN-3}75tRW+X*gCLUs$!``|0u&$
zvUfT|C<AlD$2FM)H_NXM*|!=6FJ-7dYruz(n_zd69(T-2ZVkS=!e-J8z8=Is>MTD?
zk{e4@*m?i*GfP(}X_6Z?umBgv@!VDGc}O+}5FcR4AhBu<GlzyQE>3dAhrSU_2zaU#
zyu-Oojc)Gq4WBKeUDKf-k$UltWCz&^YVJQm!&qtz7|c`H8u|dpT(uU+8xB*^UiUhQ
zYgoaxKeEnNgkdj~v3GxqVC>kNUpbwAoObhStUYgXdy@YJyFPUY2PuiG@c>^|DqEOO
zfJr0jHMQUMF_3eyFZ|q{rP-FL_;Rs!b9S*%wC`uuqCHXeYW6mw1I5UxwUP6Sr}xhp
zGow${^c_cN5K|~osuuAG76p7mh(g#8Xz^Nzs9)=EVQo51*>;i;P5O=GF9Lz$)C@6@
z;|7sMXTALr0cAIirO|()pB`(*qJ`dBMYj98&Xf%(dZg{iZ-=48@Hi8E0#7NMA5I^!
ze{`oy(d~Zlyhwnc2zFx-VH2Sup$v;o;%_{X@g_!gjjXp4Fb|QY1-rqeu<*Q#Jzx(6
zu)OucU?kW%l0;54%4HK&zI&Ru+KEHaj-YIBm_wJgNeSz&9HP|-V!s*MU(q!S9wm*D
z;SA*O^R(cagj>w6#A5didj`k9D&xGba2u5(8h0^V&OjNjqBM{r(@iYI?{xb2GsU#*
z&Hg~O@d7?t@<;C;>~p>W1#S<~-v`eunX?fU8%nTIP$Nv%l`3t*R)wuH0u-H@VPeq4
zRo)(vu`c$|dhw5Itke7MNPdV=V&~bnqRtcVtemmP%sYL#H4mwTKRx-vOl=kJj5Zb>
zXz#~!3z>QyS4alV1f5fgLe5XmW3nrVOi=G!8ScNFMzj}1Of|}C=~b$el`>|_xCun!
zHK+WX6Mrr#xgy+}rJVP@{)J#M-h3FVb~qKsrUe{4YBGU`*vAQY7sSx~0(SKACg!=P
zpx^E!)JgMC@Ua1N$8!ALn=^kMR8x|U%aZ1E5vXc6$flCrt=(G5pOya&|5=zYW<7S4
z!pIUzM3^poM1ULBm87m`CO~595Ndzok(@8HP;A@(moFKhg{KXQgRjPI&?9^hsTetT
zzR~N`tKu2Xmje>f{CEgnoJK`v5u3dHl9ZgxYY#>s{YUZ~Quu2u1|k}3*H^3Vw{I29
z{+6My!aGsz*v`)W>I<O-I%b<s)%@{~{g>Rw-jn9yw1U41U_0XvZ$~=@lUF0*9V06m
z9tVyMzWSO1&SRL|_x68*Y3z3J5<Z*5M8a--tr>)u_-bM1{ATx<wmQUC$}@tAX2-=o
zUE3*vmNg_{Jo0~=y>VvACIoiT?3l~tc{+d%Ag#|;@JleLfXr#P{7oGC;xhXPC0vq$
z5wI_I)$BMJpyc5+Gh=9bn1<**H`Hj*e>-d6u|*G~(nxGqaPdv$rzh9k$diZ7QEU_}
zx<37F#}$u4faCENn_p?}In<{Y*ZeFbfbr?$D9+O&4Btat@huyvr+k|Dox%ltSPN@;
z>EN5}Q66>AFZdNwH%go3R>mh8sAx=sqe4dVpBk*!LHNz3MII~sd9kIF>|{Uv3tU5T
z$E6YU0C!sfU-(QqRAy95-|s=l(FpY|l_J)Qj7dQQR(yReNRWO9^rCCcy*^K(+}w-b
zWRMyUp5G+|7dwF%BPvBUVem)GTvFv{UJ8dntrS~h5L{U?Uw3qK&2H?Ns!3eXNYoZI
zhA=qd6mFt_HkZtgeuk66N)q;X!xxL_uYV75pexB4IvQK-mYWwKFJGcAiTx9P7gX#Y
zTN^AS5cxnoV?(X%ND*WS0|$opYUm+kQ0(|O#GM#TV@T-*>@FR98>UN-4l49?m9>R}
zRdy+iH;G?Jhd}(YjqzrI-aq9F+55YGVbYrtGK4;3_osw-;Fd)`oA*Mz{%Pbmj}Skc
zY_o#%EBKR6=bB^b9FF>Pb+;FW1<`}WK4|1^?|>=(O!KF(GO)-9K$A_5Er2}zsAtoX
zB;naRi%MeAcoznfP=;%qX9~AH24fayBxR@;PNr2tK8XXioIlG7^vCIi2tAy;l3lV)
z6rjhG_Xs$if{px`P~cm6jDXB%T8?QF4cPwJ?H33;6yOL>$u9Nv1_dkWcgMS-al^pb
z_h-{<4I}JN6dM=Q3SoNYjDEox**1xc|3xEW7+3wir*COt{>EkFSt6@hVOM6*!mybi
zwj=^boFDu)ms&+$ib7~M>#dN|#O5&<katgx7<=*diU-PhjXj)%jhoGJT_OS|uxu5!
zV28~bE`nXl$HcPF(m+VAT@)-+Z|L&&eYtNE%CdR`FMH{Sl^k@QtevDS$&c<FY#PJB
zP1oBBK^$aHqi-r`C;ImUcv5O&`@V_9Hbd!^=1AU!_$I=cax$$@ERjBgn}JpO3lC8q
z#S%p#){)t=4h)MZ6$Kt>z03&~Tn7m)>b1#Ow=QZPh<R+@anavl-Rt$seBy%KSfZi_
zU?v)WtnRMOn#}#{8BQ+!nJ~SsL}E*9NipvsGE|<;hnGm$B+R{G!?aEkFr<^xx0O$X
zRhBGKJE4|-z5<!Wnn8|Ax3?x>$JADp<1o}ulRA8IF6@oyAkFPi1kVj-dE<Tz1LNOC
z3s-y?q=kEN)=M&FLJ!W8IoFYHMOxk*M#UyQ8ZXZWip5rYek^&~78n;{lzc7FKqJQ|
zczE)rL{nNMNBM`xt(FxR8O~iMQxEg}N1a5D_j0rhNdmJOxs3|DnDQd_8kRGTpC@=q
z)h&-DCAwN;VYhe`J|)?k!lIi%TQ+1sJSrtQo=`BK)sK5Jx5#p4&5@lu2;g(bJ3=3<
z4j9*mFayF5DSxHy_=N5L>#!kN=pHLAG^GW9TNk1r2^kVB8SO2Hh?Dzn>CD*1(C4B*
zmb&*hvR%c13-9^L09q}+H+NyoXzt|*AB-Ch`6EI4w!W<4chR%DI${*8nv4S1m?!nm
ztbC}MDgD9ISIEbS#`lIbJygMYj>}F)@4mc#b_A=35avM4D7+u(S)>5-hBp+JLD^JC
z-=YYGd~nNi?&oMF4feK5qcTe2jPyBl2}p5D(C_o7zYJ)mOc-_M;nK>KujizpZ2q1W
zBLmyTy{!r&yA3EHj4nI<M&A_@10i@aK1;h_fqFY53}(lw4qI*mQwis++b<6)2YH(I
zmPZ=~{pg9gHj(}Y9{n?^fsq}1^?P)wQ;}QI&7s8V(IDdN?jT|B^ZbMKNYIvxKQb*D
zUic8XOC)OgGkUXIn0vh_5nNieysHM^=Dkr-YdcaF1>5)EHgmN8gz|&eOv{zAOzDlw
zC$N>3<#S;H;gJ;ZJWgn&GYHvxP({nBY)lRUX@@8H7OA1bNdQYt7vM@VT5^m=rbLkR
z`Im&YedBE^iXEAo8`!%fsB3^gCGt@oevc2UMkX2=B{dFas1^h^rP08PE$vM-rrLCR
zf%c1{q;euPaq~9Yd_diZt@z+k9ae(p?jg3QYTh%iXbxo4-Mt-q-{Ayv50n(wj+K~N
z0)Ytl<kQ-#b!OmtYe=!E5GxsOAw<*I{lKk!^gBD&Ii^)A&tP0}IwHT83q(LhONk11
z|A3H((ag8qr&G)zLYT8W6pOzSG_GfdPfN1ye7O1G<Y*EWugPd2Ojwnixo!E?x(d<J
z@PN_50j~K$%)^Acp>Srq8O4^T>WXJ!Msig0KA!QTQuLf+Rs~DyYE(2u6!tMmE$e$L
zesob<!+IF%qOZP{!o_+J5)Ij3;HC}nATm<PaRZ--*ojc?oFP(1taWZ}aj{`xE9Dfp
z#$Z41L(6Q5EO`E?<qTrVsN}6My>pg|vv4J^C3K!pM*2i3QnWkDKxmmPMuxEG5!95w
zU7s)!kLlG_*T0fDpq+Tri0La&l*pBwS>)9tY#R7|d87t3+8pnvG3d7zoha2y^Zo%I
zu6AgW%w}nGukbHLAukbnagPBL(LP{;xU$-d5&GiYqC5bA^%aD9?2qnpS*y5z#HJyI
za>_U;S}n!P2qx(_msv+ILlR+uPSAe6=aM#w`2SwJn=K+uGSviSz~aeD&m%o1Cv_+{
zkAyy)%Zq2g%2sG*IypwtwHm+=alV^Gv3)-x{0&^Kz34;Ki4$};UoOwD2`?_Tw)X!{
zTqN<fRZ-vW8RE&&7~BN9wxNw!F3q9A7|nUy2sJxjw|}_4J>;HL8$?Quq}BMX%OI_5
zD;*}<NK0zX&OIW#8GZwvdRpC4Wo9=9Xq(h&TQso*w^K0R?0i94-DHjk*QAz)V6~ht
z#c&Oy+^!wU_$ow=SE&v)SJBp2dr$6cual^!;%nLg#BEZS2HK4*BD@v-oenSqLhk8!
zG=``u=u2;Hl;sjq=Y?Jaj!{{zqfbgyQ2siG!u2l~a$0Bp)z_hfq%>uf()x~pk|MQY
zj9B#h&_>M`a#GemJ_(XrrE^b>3XU}78_ERVYM{KMw(K{4uOCP%plJ3}N0%k59&PA>
zjfp0^rb&kliv^Gw+TE%`mI&uwx5y7NmJW}82}hqLM{=GX;I^?3O~)dU)PN*N@?$#}
zvl|$yzkcNl5=`@mzvC=3KlVx$$GiECF=(-_-5(w!*l~sj&^}G@;VHp_tndahF2!O8
zMS5uBXH{H2G!R1OZ~2Ge3!oln-kR+bW&qt^qyr!IS6|~zR=gKqO`3k;Uyg)%Gf$s!
z@_s@uhZFZM6bbJF4A54poqN!h+BY+(pPyj7(C@%^+L_A7t}0AmHFQq!xweU|xMgfz
zUV<iEMv|<h3A*iZOPx;A0G)jC@j+=O@MHCR2uj;Rk@D07qVdvdQcG<qQ6z8NU#r}?
zE{Z0&k@L9p$9?@UeuhmcRe*U;!HKcXnqNTy#0Vm|JcUIn*63gl?_l5-t)jCX>}R19
zN<{#q9`xh&z{nXB&H}Ma3^v$GLP+V3hmpauDyxBhQ}}t49s$4kXba-_xjVG0$IpJe
z`Mfu$%2)l>hPJP~qOYG74_aDZm8^vC{F@`kd_~0N$3NR!nAV^7V;U^OtFcZDGg_ZZ
zE-TZoq7Gl(>NmDIS2y^0yu5^JW-IGrCDk(flW^KQrI%wPZ^4*qI`vDIRbf9n_PcDC
zz;vgi(s+3}3cmSnRo!mzpA#-u)YAWO(G!43+joVY^$csT$uPK4B8E2rq3Qzrh&B4z
zuM{V(Z5Yd;ETR{G)hzFdywR#D_5*Pb%Ck<VvC~29kNf*}Zfs+zJ2|hojv>qZtWOqx
zOprC{&uf9gR?zP5(_Lw^6jXjADCQb)*UHNm3z_MwP&;;<8plCe!W|Yw(_}GMp>XOx
zD~_9XUqY#*Qns7$XnopUr-G;NL8$aJqRITuxd$LM!HNlguHkhM7;*SY%8V<Mdi~;N
znolA!c3?OEXqC+2>XD#gKy_vYui)torQwY32e}=R++s;cKq0ZO;=0R+421fq8ea&B
z{FFwxfM)T><ZEqTmnzkbYK%ksKA))s!NB3gKOeC9`Plc_^BCI%-cbbnu3qSeMj-g8
z8+B>a3_S^kgl)tKlE==0*thtGwa3gNeYa3)GmB^0F0b5_0D4ZBg2pS`63t8rK4o&K
zBx9*cfyFC&=sJKttrG$N&HFc;3_f(dvJ!Hp7Po#<5rfSPmx*Vw(B9-rZFBRZgBa6C
zMYEnpic{{1>hvL^arTZFBy;SU=*@|A(Cs$C;Nhuk>8*cMnzMzcrvo9uJ+ok%cc*~_
zy$C#g9qQ<J=o1HGk}MLk1m(e&&~xMv2@c$_f;f4yqPonXq8kg<ZVHOu$*-LK8MXc}
z1o2cuKhF#>N84b$-Ul-(h)a5hzxxJrLyJEz(u_Yv-9))}|4!Ff&Q3UfI5Fe{H_!(j
zRYaUFb?$>Du6ztB{gWFu^961{C}+U>)C<QCH+Fnm64rf%=owgm!9S-qbpRFvwuvnu
zG#L?WBgul4%Vk|ITGpNtS^fjAs=GCC@4-rl;{(F`PFhG0RQ%mJ;=5H9*+FUkb2tI?
z-^~Z{9*hgccV_oyO49<2`VSS!J;*r-e#({x8CYDjEi9NBIxe-$%Dh>^SMW5RPMv55
zL_6n`l=b8eS5Bnu-FRJ>F3IA4nu@)H#h|W6Eh^zDk{8CbepaPk84UJGJPz?+ZO0>k
zvb+*9&JIe63THJ$$#PUGB0H~xJs%t<FF{jW4cqT!GMM%VVy8M$U<Ng;V~{Fsl?89o
zoU&bON}LEp(5zcZL;PF_pL3)(*USQ6HJ~&W?$Upvgd>~8oF<17byyoOoVzz%NqKUl
zi$0xl?^d<_g@nbyRtoDC`u4pm2Df?Ysa4NY%vwwqH5(IerWqBLBMrT5otAVu{U}Yy
zOmz5L5=lvy^5nDQ(4tdL2hSc|wVx4D5K8uO5&P-93=7NjSo@0j@qNi0EfekO7_jWR
z*o7LKcRW@uMrrXLCkx@-j8iqViorYhzY$y=nUNaj98evYEH4*s)kXnC<MjgLBM@yP
z!ze%a+O5&U8||M5ImH>zNzlKB@5m^v>Ph-qV)^jb?PFb^kkZM@R>|D)>|OpEw8&7I
z$gCl>V)eF95WF~!QK&2o#t?GR@ecH`uG96eu$K(e3OcZGO=1vO@it77|FEV9y(N@b
z%coD?8=^HQLOmmNJk(8Pbe(fzVWwrWfQKce0CRm^0mWX#qmL(MpiLz+EN<oC{%H%z
zLV0f*8zIYAZ+n0aNd~TIp&@<Km?D<a%5Y`@InNsfh++=UZ5m)xm06E~bkT&zIOMZP
zUg&u>X{;R|FI3pB>j&FQjv7NeWQRRf!Pv}vEBjgP-EB@p^w)b4MNdpp{iT7M!7Wdp
zcjd*HRc)W_SY{e{fG)3zn06KtHb<Y*G)~HaMi!dpX})NNtYpx`l81D@Skr=rIgRzi
z*gjgGYqqjsIz$?II1q8&Jz%6>BiTfw-PVWOSs8+ikbZ-Y&neB`w*OEe{pUC%V@;O@
zL9;IKbiuDAc4PvF7AGh{jN!-AZrUJ<r^FV211(AFYc^FZ4F#yt;ReWp%o_fMm>}|{
zAqH#mHfiF>LzgZwo$wEeg01AnOHdVZ?Fmr=_`C548NYY{a{Yr~^QWOnr79~(_bp0i
zJO*OjZ*qCSya@HMK}mMWAH|Rq^V`RnsM2XYl;|K#MFi05Kg8fUqPq!DT&EfygZb*d
z-vJmcB8r0;gu$&j4n$B{W8$!oB^T6+CQ>DfU`*D~AEtDO@b=6W@K^2<27eIJ)uHWh
zP`lt=u_B+M<o7-diHk%~t&Qd7p^1Axzt4mxdKZfAir8l}Mj~!<jdPGxSl%R)(wnCc
z!8^6xO|k2IOTdb$>K<LSTf0SEP&H~O8LvG3QMwSFT6TqF&wf3_)ELBFXkGc2HCBk1
z(SB0HnuTMt(PuV_&IjDt3%=$6Hr2kNwLJZ3ZgABPa?&}mPllY_nyj^uQEw8RL8-86
z(8!uxxjnpKsaQhFwYWTK?n?8ZzDmt6_9g0rY5RiwhjS-AZJ;TS8J_T+x~DM5<;Gc|
zQA&DHT~LL4;mjcc?iR?JxOOq~TG%+w`CWtO6&<p1`6LgjF&0>xQ3Jz|qn;FkUq;il
zJ`%(RQejz?#Zh(|XgW|IVBa@?qsms$M+DB?u<5^@oHzR1S9JNcc0BEGN0^{SId?|D
zITKP)_?UD{Dk(zWLfYn;!Q+KY8Yydy=N^_ir>5r115AJ41FB$71`D_q6RQtQkgTXK
z-!^lnP4JDIx3`9Wa>5_}R>ho3cd=ZaXHGL%<v&EyC{?}mW$%TbHtHIFx3%CoZonXL
zDH-Zlq9A^$?p!2R@&e5{Tvb$vY{$~|wzO=W)%#$XQ*w~k(W-!!hZ9Rn`w1|!Ad~E<
z`0##kvPo=!o^6&qdMlfVJE&?xs43SHJ<9WAZ?6SNSWYc}WMz{&$%^RM5Tnn{jDsdy
zlAwW?&>be<D#qkEvVSYxba(q$W1EY@y(YUdMHtWXF#b68GLzi9vq2j<H$|96>o9h4
z5b3;fJhAVrrfu<3n;rd0eQw#lKm>7PWhbyGnxfP>S77$=(M=%{=|G*@bVYt;T`3=4
zt%y-^EZN;G*$3_A->(qEOMB+)o-l;L#NzEr+Fw|j&Pjf1X1nT;a}+cM$^H{r`#Fs6
zeE{pz5I)cD(GdQtHsq_n<(ehC&KoOC^XZQa@kB%*Rxf9P9kvt3C}&=hmKenSevLM>
zr!)N%#FDJzpOjta-Jtfrww3`5-uC4e2fy$<)E808Ut!c@)W!os=pC|)z;Ja?L_H5D
z6T`|K$avh8svSo(SmAzQACieRNkWx2rwq8QR1`N6z2L{jbAV8PCRnYSWAAdFPO@b5
zNVt~ap@rP1nv5wHmN|1}z*cXy4gc6A_hi$YEE%IJQb<Yc>q*C`zphbc|0SMZLG-at
zSYEC3IyVY_3PyJ{_{0{PEQg|IW;i&aG%X<KHf-o_pb(j|3A6s~*NLeDy>$)f$$H*v
zPKrUf^L{bL^#M5K-v~(;)L2_RaT{a?jEkiW_$4!)7h^tq4hg303=z)Pj*?*zj_QL^
z^6X%iVV#}vkvsM;3#r(PSP@1CZ?%sg4Q`$K<~n37-*#T-9Yc|)(QA!f#q`+-SV)@7
zSn(^~^(E4(SZrO-Z3z$u+)Wr>c^xGkiqDX^adk!tbTo&MNW*Vto_IClLKJbYFnoKh
zrx2LHi;}X>L^^@H;=8{`aBxkKzD&cyzB!6=Iq&C&4!8aps~sCxpnHI8A54kTPlXpn
zDz@AqfGwS!M>kwg9c_kh5|qG<Qum#z&!JD{>cy&#dF_uw9WE<on!8yf)GQ_X8&`g1
zyoR+a(O{!~WCV|vQiUNkiM!nJqu-ghlb-_NyQ>6Cez=P!XhX~h;aHs6Yq{_F?92Uq
zD<grdFfuRs$b#%OQqHGA)d(9L$p@lWMQ~(~bYU0^$!>5Pz(}M4dU6)V6zA9sP6PC6
zu4t}EJSjNIvLq+zeV18NbTQtsoN*Vx<;Yl7FB@&$g_rS>!i(3WD1wzdx&Z!jq!Eu;
zHqjxQf9vo^JCCqmUh31Tm5eg9r)$l2B)pVl!W8Vdwc>B}UlbO6Np6$*#Uw^%oyaM~
z?1%I$#@Ki_CB(SK6u3rWuX15Z%=z+MD#okYK7!WVsS()TeOX%fbby1qV@<2naGt_6
zS@12XZ$o1jJCIe=;LlUHJij=<kwP|(jq%!(8F^YUN!6n$(=VBkmb__fjN3(?Z${=X
zL%B$lj~3qTHBxYji6XHXoNS70oO=j}!&xbz;N}~bx>|}L`jxxZrHCv@eIHYT;8mV9
zbs^`&oJb9<ALO}PW)I#Qwey_oHrq|?KlmN4LDv2;H0fPzq?{0&CfhzEY=@LqX?QR%
zBX#dpwFVD38zIr*l49Z?6D-eyb+Ven^U=toh+J!W=^x%)V-aKac$7*tc}Pg?g_pCR
zxD{uFT<eL{F6Ak*9&N=SEPwjr_I%0Ta{A>4d;vTQ{Lk8_AI(%d`KK^o+x_~>3pwxY
z?u&-mT|eC~IQQe=%8bB|TWHrsuZz=mdBZ;LN$=w8qzVCwW^A*ibY#q?d%;zi2q6$+
zinL3nM$JXdlg4Zh1HCk2kVd9citI7-0pdHAMmZW)Z#i1ITK<nQq>}ZzWXUEIsvuj*
z`N)VXo*-Cxv#h2lY)i*3-tDWK=S+;Cdtf)2$c383QwjFq0dGSk_+7Gv8jcU@47p@M
zf$J3^WQdgUHZ)H4Wb_VR&H|KVA0&tF3Y%j&^YyDBJ_enT!YSMJr{|EkUGb~KwaTvj
z{i1y~@3BJ=WkdRGaBR18xnr$dTp!9VhOR5$YzRiE5214ljN)|0M+a6Vd)C}s{aHaS
zPhC|gaS&`N!c-;yhH?f64p0mZ9-olWT;Q8a4C)N_Y!L`wF`(2`)hmNB7ROXwJ?W_t
z93VwvdeU~W`zm(5(@oxoY>|lvRQ@%w{|C1~NWb^?GU2IA16;!`Y9lCZvySU@Clk%P
zDw<|N35r~Q6k^EHA$rN2b}uv^v!lmA-eCOB8#%Ctg9gZf<`6xuqT^EuD>O+NclY?U
z^qjo5wpMi7?eDiS@uc03*<e7vTruRSj~m2G*kt*m^U&!{$xdu+LD`>Ys5uoyK0!ES
z7k*FfX*FPd*B_8UIPkjT^AWO~@=7bcOp^tes!CW!lcH-0=^h7g%|FT|M<p5E^*sh6
zdoKmb0>yQU39S|F*N102Kr~fZ?C`IOAllpQ&eTUAxNVHE+XXfAE6BhA7IArH6Syj7
zmM<5RDC(_HAv-{krCE8>)>4>Nd&h^{2ko=N<Gt;j@6hH`q}7Km#mp?{gcsxK0FC?a
zJ(%s}1(?92fZ&jO)4)p}Hv)=*=XNh}j0?rtEfmlKFXjA7TEk=geeGg}WWia;Cm56n
z(`w=dk>8!7Z(E5C**Dnja0$H}llm{B1NOYm^xPDZszlN0jdj1LW`JhMFBao%-Dn)O
zsf$2pZNG~L8^G3ua@MiJ=Q$|<f(|%fTytu<LdWp%4Tk9g2Ji`_tMHfb9i$U%$56Ho
zT|C|v>rog5(Rd0;VjIbI1N36-cVQ!rVI0Yp&lzzJP=P``Vyo!KKXFjOx5jjsVNsp1
zi8AlQkz5C*-*57#KJ5sX>MFlh6b{I-iO}f}#&qwDe1W64Y=9VDh!ZdBaW2(_Kq46v
z)X`Cd)fND8f8V=w^6R)0z}p0R6POOJ8w5CB^Nx<+Y@dpSx=<@Q^#l#ycSSpNo>6#0
zG;Mb~MRCAkHdK^xPrIG*#Qk+D;+@)XG5SW8M2oftKe;tpF&FQ(r0tTBDkZ1-2RrVO
z*B^5Ie%KxirUey*b1)vA^FJLQ<|_`9SvXnNUQ8=xFYwz`o`q%rXx0nzYDrd+T9t}W
zNUgYtsFa(GFRSgY=MPL+SE3BXans7^z^>wn7ng`JzS1!noVP(JfDqI&iK#%6v_UA2
z#!@_vrb7-PmpfNk@+|dJk9W>a_Ye4dhc+B8C||Iy<9Vsnv1ch1uU?(t4O{>`r%+KG
z)y^$BQ!j!152b_B!NJER)shVBp&X0+d~P<(lNp34h_W*+mS&q7lVX9^mTxRQwdOao
zd@HH>iH#r|9lUPiw(jUv8+W_UpKA%wqINf!P>tJVk2^DJ9~>RNHmeVou5Sa9go4^R
zbLQbV83714y`tMtx1cogMTf>Bw9<4#91-=}61`QOvsGNSht2vbl)qUIEiyOVLtq5N
zvpSO-=3!Z@rm@U~XE~6Q1Ch^M2x#71t*&mtzs~Q^9S*T={Hhwi>c+1PvtHBqS)-a1
zU|ZsRm5nFGq|985?f_34Vz6-Rz^;E9aafoTwIa~DLC?M9h-+0DxW;(_z-VDfc7`WU
z-o9!|Mk%Cbajh6*V-D9<%q_MOKqyf{a<Q?5w%dMfquFj7i)Op+4tu}?$<!+-Ny+J;
zsA>+RS9`pwAbtRTTk%QN6~8Ltl%Z2t>4bsvH&7P;R$k!;Bu|}{=fO$=sui8}b?0ar
z0rA5%l?i!dH^9vOk(Q_3UVzHZGv^Fu90sy0YNM)_i6S<zWpR4;=47Y+{oe8N{WTZ6
ztZruyb^E0sopjyNG1`nm4~F?008%2|Ng$c3S54P5Yq6!7CCbX#NNnrSu!^q$5PuRY
zsz=2YB~?D1maz`L{rz71<=Oth>HcAR=ipyU6L(=k8sb))KZU}E^X!?qQWl+6r-J{)
zhO6%2H#NEDz{hpxMhy(~>3hS$kxmCbnTjj}3oHbSTgLBd;&(mqdn56?(v%k<t^BQ(
ztW{&_M5d@KEQD3s%w^h9eI{K~5?g3!SD0Nj&bm?i_aZ^*?zrb!#WaQQvTjg>jMk3Z
zQtrQaA0uQk0dSs+>bfB2mr%|C<CD!n2ingk_CyvuWamv^XzAsFpy+GD!##jI`P}CD
z|9f|E9{8hF5a2oI|CL&!l05%!Ra)hT^Z$GJ-23^z%J_c!;PvtT;ck53@Juq0#z`+!
z4A={?#7Olu@Z{<<cD@(o<v}PJ67jw~o$m#77V4rk(8mC=u?=}`E%3}Q98qu0agM!T
zr>Hnnn|uO@hi5^L0zzK%`wO35X~>yS!oZXqld>E5kq@Baz?T#ZFbG8e%uofNodo0e
zeh*J(p&^8(1-*n8V<Z5lZM1K5q{osxARdDj1xx28=)jT3k^eX-CH%S=Ar6CU=hAcE
zd8q5*`b_(R$Q||EpyzDA-N%TML?K3<1Ce4aj2$_N_$+V|xB$FM2R=mH9{HEKcD$=7
zaGlO|<Y_|^A6QRAJa=ki2MR^UCCoU@2`6sX%Q!KW%mJ(GKla*h_TID)kG6Mq_D)U;
zbZkJ>l7~de^C8e%Bm-%0q-9bYpW(EJ-mpCyb977{<W}nJXaU^L4tMun!2+~<i^$Q9
zLQ+dj?3PCE_EEDVmcT5YwHiu@;=w*3_7cAQ0~rG5tja&$L!D{=@a@6&PF#fMi43Qy
zt~xm0u;cY`GQyM=Q%RpWkQ&FBmGNR>($a)g;({4AWYs35(5it)@u5K<V$P;}8BRZN
zhxm4DM6;&@NstG;GoO9&H=aiDOO#Q`iK$j{e)EE{7>yZ&f!E*>g6?f8o2WYcNj3Pb
zpqeel9$~0qy9b|zxa)XNJQO<mes32h61O@Y^xD{te0?Xbqb>|*=w5fc3{aR%K0XRE
zYZ173mkBSf3KWaE<xx{06m}XE_!YlSm=EjdVAnwD<^(e!F=i=XB4G&x3qx2=tegtp
z>B4SF02i=%phOw`8vGW2S9NqgpDlGXH$KuHLjYlOlHldEZzjG*J&(s8J)%Amm#OKX
zSjuqcGgB}nI-gUHgak1d@7Qw*VTW|U*{te0&!kBHAiI+-Z)FJ@T<Nr*Y>Y_hsEA|z
ztzJkqQxxL}JIy#vI9bzNr&(MY&7rBmhL0Y_RM5x2V3Ie{&3@o{$O`;EPnInux}lLX
z<?w9|j#m5>ZrZSOi-PaE<H>bfuwEfOUK)D;r4ibB%K2LfXp!p=Bs4h^ss-uoZXubj
zH7w^^?7=1Wnir%0!YS)?pR=CJQmVcN_hU?QgPY^d$+qS;B}>d^Wvx(*PbqFs`3cl8
z)7d$e4NEU@c%zWXE;vtKNLo^9N^rLh(V}V^o;i=t4iEPaU$>7A6<RsXh`E^dm#H@2
znu>c%>gb|i7SyMfQez6VfVQ(1Ke6N?XL3Pa)D~R=wK@XQvS`|w(d3*j&LPYfbk8l*
z^gwq!W^4>uGEKA_T2uYi|Eyix(p8tOn2;>l)n@X>=epW--&Wi|xf%DypWC$m3KuM?
zCt$Pfzg7C4wEvbL-hbW8=l<-!cu0sj!l#UMf)uK6(UvNs&ZcEv;Evwu1*AVNRk!Gv
z4wvGhcrGUM@IeGfMsYh-RgQ0`YDw{Z;{%Yd4q||I;UA=*A$mqb$<7b^ry#dZ+Q-|6
z|7ah***_%{5;ivQB2W??=*&1EyC_*^FFe^0nyxND><8DTB`F0lmn)yeSNe_YH~WXV
zTqVvmE8|lN%C_e2oF3?zCg9VZ?Va!TPI9?wM(yq6y~AzDuVpp<_W0;kd-v?k+gz@m
z0fUOXJgvUoIsGx$m{sU>cl#7tYtAi`%e7{ep_XzRXL5MY@QIZhI_^k&6f{<>z-Q2~
zJWryFb~GkdEELfch=&0I7KJt0@3|3~@2Q;QU7?Uf|K{jasSAVeke@hangLzLksNj6
z<HH|h3SuS=2YS#m2aS;5<%Rxx&oF=z2AGOgO+FDIiuiOy(1j9oUy}|4+@Z%WijWML
zGK6wNIs?FFfIO-P2Ai7fvU7C2m&<Kt@d7am<!5KNcfLD2{KrYIQl44r@Mw3hP0X3A
zWHSEC?Gr$!Qq9Ug*xP;uv+AlZJ~LZ<iE}JfeO8U{&R*{=TxIv<WZ|;J!`nNj``_;^
zRRyPE`(*Ebma6ocC+{*<4nd_v&r$8O{oSQ13U5=~HI`fH>Z$sgXR!h@Yti|zqB*l!
zDG>{o*o~a?f&bvYA3uIc+oja1_<>^F9#RMndIX9`cl5Zyr|ZzLB(eOV=YF9(aE4IP
zJ@@!QW-fjJk;^?n$C7w^1_WLXpFxVnhbhBHnPn6uD40TaG$(IRVe*B4egWzF>PK+o
zS&ZBLb?OIv7Q;x5aot6K-XUU(?cl99#LHaKb2<F0p|;<5qDv)wVU@*|6rg#(PN9n|
zAkhupg?DWTun8@<1VWzs@Y^2rXC%&K@dTD7Jzr?S2du*h347pzK2QOOi()*dx!#L0
zF|bH>hcyczHGbj#I0ydd9Sn##bVaY5j{%EmfbD>29bfY8yY4$NWWtMStkVehr6052
zrF>4mcoIvTYXwbh5?{^dJ{(EL9D3$HvWFzpGom5i%`#B<@4Zu{)$tP}rSqX@H#)=v
zswK2L)(5jxJa(Vu*Z`@%XsPm^l0kwvJEQ5?FR1p)DNVC@ufBNCk%@w=u3Nik)QJ12
zC0jeG<)>Tugbv{2&E-3hsug!6k!9^nqRg#!h&8l%%xLhvHX+(dn=8e;AOe{tg3(e;
z04>?tc`bi2eq9z#X>ih8^+M%d(QpN5V1^EH7brFfJS;h@#5jxy*aY1HM~jq%ZP=h^
z7W}d-X0smyl6yj9t{OhEW-z!x4zbHn0F?ixr9r4Ewj7w*`yCV1Y`!GcP4Q-C8wuFf
zipf5r;2)r!IN|6qwK`-Vo=wE*Xwhn){Oa({+3DVo{Or)`4(|p`h0hBzWy)5Zhsmfb
zSseplMy!0XcQ*Aqb|tq(dJ{}xbO2DeEX4|2$wBogcJ6qI7a<gixn$G~)0d$*Q?<SI
zp?lY#GGR=|n932R(R+2HTxKU-rfCIo%DrancMbbPo#r-IUy$ky1V~wy(oLmCToKi5
z)}|W7jJY7?mX;Gq>Mg}-Uk;(~mbJ4Kop^YF+ZP<6+ZDl1a`@u>FJUb6IZQ2YEjmCz
zc{E1jezK4p9?2QXGcAo5l;$nFH+N{cx?rhi3F?X>xema1M^0To;y*olV}jp5a`WUE
zmua8RS(QMo?WAreSHERgW%f$yXlp|zgOySi?xj(&ugzJli&o1Ez+Y@({&PtwyReRz
zg~bKhLi6RhK3~uZHg-6WDaTsD+YcDo)E}L*;ly;5EL-O7glwAv8Of=nge51ahF(Dp
zfLbCOQ{WBUE8tr!fmW>0jr_3hd%aYn67Gu(u!rLajX(^VN}8rS;Q-oVPtZuDi|%yO
z&l6q3?qVjIQY1Z5{gX*wJr{9nWhF|D3urQ_NY}b#XYj`46X%T|qGb;cgP|}G1~2r}
z*T4_Q@!ji^Lr=zJI#*T(Iv-S5a`kN`3Kd?CE_rE0YmdB3St#Z%$fLVs{)p7gG!RvE
zo)B16KVL=+d`a_OSYvn4`3M?_Y4*{~cRINF_F$NX<M-~Mlz6W)xW=i1Bb`3GV#!e?
zpJ5osucR*KPtgZF@Ie&P$rOI$Q!AZa9*5+|ZVHVL$2~eiy%<j;s4*ogv$}f&C|<bl
z{c#|TwDXi&V(rD`7cbv<Q*0Ld#QxMX_X#@Ve#Y00w{ZVM`H>7QzJKw9_aR!XM#lXQ
z$ZOV;_dgnqYU|<t$31)=eYM{4N9*B5K0ogK2Rr%v>q<TkQrUU-Z13n*+V1pSJAG3T
z)1-^W?gW%a{7;%IS9Vrlf=LcXn0Y)Ax#$Is=7kjRzlCcT@o$XdHT3ipe&z!ye2Y?i
zR5r&v(JKf8ZD;}+oevN!su2JOd%E)V2hsVDDenII10ecwMYZ|6a~^mTXZ5}F_%*$h
z!}o6eU_5lzo)`{~eqMX5Rl*azRnK`GuK%w*JYSb&TtBy$biQ7PEXeS>7vp?yJn|9)
zlO(v3BUUkj@&hXx<A00uUyR_IKT7b}?E7EkYNgpq`TuU!ALjpke1JuE#j5EeGhs+h
zzcY=zcDq2%YKu;IJ&N2bJV8|+AJ`w+G$9z!YX$}yBDVn=RN?jz9|$P$OMnyJg7FDX
z5;A-CE1Zv}!tbAO@^_<VghRmBhiAemU`A`V)>7OG-p=8M=sj=X_dpFnA&Iw#)LUKQ
z9C{qA;?{)Zj(Xzg_xViR4Lp9X>xFoSXv_~5!|O2ehT`r{0<EL(_l}SEcj<Qi_QC$^
z!|W2@({G9JcH|5^JV!t9N7E~3JoF>JJ+wl$yA_b`PoW$Zb>y97%d22E^r`F1pbl}J
zH*Q2en1Lh{6tlDiqkcT5SJTlL#k;*vXD4L#CJ!AuM~5e;66gKCG=PKhIq|xF|C%0%
zUxHdH-$qZSoq^xgt`W$#LV7?dyebR-p{Rs9JKugwu9yMU(cs$o56Be1k2@Rb1}hR?
znnCj&s%~j;Q3*z9=L5gf6&QwDU4t(D_#@tyIr<@XuOZw3&Gf5pI3>Y+r)G7XpDtI`
z{Nu+yN<{2z@qCB;`~0s_SDyw(#=~vwFXeNa>%Y{`k{ZCA^}pFjum7!DwfeCB-@_+%
zz7a+}kUkXCK~KXoz*HhEfM(jjN9ms|7oY=s^5Kzw+q8FlvVU~wto&WEkBEEv_m#}j
z?Kj*1eRS-kssQKxXB_-}M!o$*dp&<Nv);?I*BAyPU9mGg@B3FXY91c%eQ&8Z3cU9-
zYQ5gywNRve?)QKyTHph&_G24_DlOTCfm_TK&;?WxzZQ8JNITlB8ggDjVGuC{^w!AV
z0Mu%Q!KGG-RBiG2QEbZugo5zsg|M7wj{GIvvpN6eBwBNx#xUY69Zd1X3k)p!ZB1g9
zh}$9&``6y_k?2zSs;*p8y!7#sGsPW71EhC_E-+&s5oLgfKCVJ%#hpguAqBV<#bNL0
zo|A{m6*du9D7>yHFWcp7^X<c<!#%Ntr%>Y}g{Z4&UcevUR*UF7Y=iq{twUgQ2{<rD
zdgmwv_ME;=Lys=J%P!JoZ`fvn*2wi&vT&#C&CD(FxP4n8%&X*U-|uS<8P)ETEbyt@
z2ZlORw8i#Nxl&P!UD0J0HEu=dx>2$Q#YGLR?Gj(_?LsbWi+Wynfcj#=9r)*?LRieG
zA~!0TFlB*iA*_H3S`||@akb~iw@1gPJlj~C+dVkY6Vs46SI4wB7<gB>9Vv31cmgk2
z+aKz+<s?hawhVvjVaf5WBse@0_H?bW(byk>LhThb^;I)#JhGvDc*IO$fFTaCi>qYC
znSD9g+c^b!oEm!$&XYC#$9pHI$NM{{@a6RE_)xDZ;~q%KEO`>0uvRw9Wh1n5yS=yj
zy4^0?@uz>cPhmYvqc#rabY4}~75afTVbi-+J0tVFNg50?=Y^U|CIp&I&}vxJhJLmM
zAiG;~;-%V<)(6?lLi2J46fUp`rFB`buryFyIZG^L%wQAg_kQSiV5=Hk7gnHxI~_zT
z#bg@`TSKhvT4sF$35$(}iV6D7<UJF71{oJd!Pb1%@MTD|Oo^eyn%S};*|J1fvPvw)
zX2_+)RAkq03reQUrHLg&*7Ca0VtJeEqMgZaVey$U8nelM3&bT3`rO+-^|{^sKOL>4
zjsfSK|5a;^RQ%6Yx!!!(|KGz$JO5+5XFI$ecE*FS9bM5;ApfL0_x{!MudDhIl-P4z
zJ7RHh+{t7h?Vh;N;1^<`hC-^3Z=L`OKbd7-={XGE!J|HWZSX|xRrt-83S5X0@L<a#
zc@0kvyW`P+q>Yx$u<nR&2`Oa=;R%<0^u|>~Z_SYg_xK|8_TlS;J>y|7UM=wAO%lRG
zOc{GLNE51L()HN(z5RCIVOpu|3uu;{Z@RJjPT_@gbrlg`Rp4MxxjqhLvYML=&<v{P
zgkBVpeZ?A>k2_=2A`Xw*yL)lAO%NIdSYWAR+tCmy^lxN>uRo+&=HtKR!RqMwzwz7>
zN{xmeo!_ItdHnRTga7;8y?p09{y@vI^L6Fp<9~}YPG=>Solxx~{9~}+M@(EvpM)AQ
z9?=Y+toKB^h{W3^9giPWVwlH>JdwHs@?a6+8*1W2c@{uNDSRBDI1fkIx6>0jRwy%C
zXh!Jg&<*?nz5O40?uagf5j$(=-@x#pH|V2ZS7Lr0)6R8zX7THA={?pD#{+TPfJf2D
zi6*1+VTX^QBYii?cL-@F>nR?~W~KXM5I#>>iZxQl-VYz~xy|)oUcgw^{$H!s)As*H
z?P2}Dhfgds|NbAZ{dci;$#diW7zXwd<%f`hV!1}UgUjm>i8ykd5&3<$OUIbwK8sOw
z@T?h4*=8!Dg@J6t6XS?qji<!hbcC0;5!f6->%iW}OJ48!yFY?q9Sqvxw2yZBwTmK)
zFmWX*xoy@S%ReDH$bg2h;%;SCq6b^Kf>A-d!Iwb~v!ff*o$2h|yRz|Q?lKS{HIm%G
z7(I~jG{_a089y?%tC|};u2{70-Y|;AT_W8zkRDBkMr6!S??6??c}#W!7n&4Kgc4|=
z{uMfCW6Lj(>ysfm4iKSD1%{N*Y_8)xAK%e<L>GE1#XF|1u(q`gBFAX!ommidL27ec
zG=8y>`Zjwcoc3Fs-z~^5vr)A-O)aDMy|~$|mhNSaU+I&aBkJ}C?s=FTmd`!81A(o5
zu$cW~<QE<!0mD8_R|~W$63E@VPQE3Z3Pn?i`%E+Xg#$8~2HtX<*GqS47S1yug%(gV
zMr28PjFx6jL5n4XGqQ6x8Wd8#Mq*}sG9L6Ms$ckP;Eg6S{P4)Tj5FZ}hq+!F0c<k@
zb*7VniEJH8Xt)f~vN4+6q`}U0+?WQ=XpO;_ZjTHzSV@HgccYNa{DpW377jhlIzXn*
zG@e9DbD`zJp~-ovtAg59M7u;!np)BqDDPC}OHE|9+Od?XWOxyndG+%yd-gu9vD%LI
zGZ(e3FV49B2l(9P{$IS<Sl0euZdKCz|8nEu{@;Ck=Gy;L=Ke#&u1H?#blU*g13+p=
z!I<2wb*^nzZq4|eG&PcwxeIlqAkExlj{q?tEfjd&J_`KFM0cwy!)j@+F2iiwC0=BD
za}Sow;ey4hc1t*PS{TEJw{PLR&G{ccw=yk&bL@Y$ay@<jxB2k?=UzTMr&;BpAX^vH
z(Yv<3s|Izj?e=<!y@AE*fj5PnCm7qH#BV+vk<=Xzhkm4af}jhAZH$<)MU&HTr1qqs
z;ea<jrOL=H#(x=j{8vTBP>!3)f061`Pwy2aCAvg_7}*FPh|I;8Din0+Hhi(veC)X0
z8Q6&*<9<K%Vr36B_4DXL5qv3>xye~1mVf7x*82?T%SLab1F-cx_wE&YYVkHu4<~~~
z2v2`p^JmALs5fPO!eEiEEH)E{UScegXuER1$hn<6=@emM>(u<|o$z%3A)hlOS}ym5
zzHQ0$UYyZ0b0mYaUs?DX*HO$5mApiz;4P>{-&DF7C`hMoDqZvy(&^h){-YR*i}Cm!
zIp+V|%hbc?jy^Zn|AyiFZkxTotj}&^-u|zgj{i}wR4f0daUTTui|c>4GyXf9n=^Re
zCgWdkHXg?R9zM4>{_3q@s*O4OzjCXbivQlMl^^Us_wkW-5ixr03l-ZVPSM_UUebGY
z5tiL~bMpO;(;Isf9bHswzeFW!9I!IwkDML8(uT#@WB<#eqXYJ3gpScXejG2MOx!S}
z3zvBH;&I5XpgJC3#4I4pqElXLpie_|+bZDf4BXNAYIlIw<w8aa!vlF3(>WYFBhTxF
zj>fQz(uErebU7}$3#tAfTE#1)=vqLx8@1bAY@^*)+8uR+s%z$RK?LeM<*6jXwxWKL
ze_5v-S82D2+E}bjnQhQgWl%V%sQGkJBZCNj;`ra{{HJHZ={9E1|4OUeOv(Rd<-z`Q
zFQ4^wXaD5L+1b2uIAE2J_fzFql6m)yk5^~M{mA*=4>A75Ni^;G<2+P->EiX&@d#n8
z@9*t6n;V;(PfvHM&W_l2c}_2Ip=#lcd+jt!PPJTa6rCSlSL*ABl}felAQ=1)YD}&J
z|NH{Oa}}{%a~1yCaJC2Dl{>-<**g~=>`(kqRyp>1e#o!S@ZK0+E)$n(!|^oedX&=f
zM{aOUzFXK)+XV_I;X4kY497j?tfNHXMkZb$Zl;OrwLQMbZQh8(7^G}rrb<IEdaC;3
zRMryMQTPynR(vx)@FG`SP<K1y_a240Wg?8mk>B-73=TddCDCgj0@B;&Xeb(D&@9L*
zA4s(pBUDWz1PyD%kqAMTds7jQDuI*2Mc{TLoWzBE1*<piQn)Kujv?Mc=6n1@H-g~?
zSs0H4jUR&6yP|cownldu>F)Em&z+dUP$RHwb)PyK<E{2_0L(%`b;d(nhq&ia7^4QQ
zzn9*<sINE+E^)M_??G{`x{H^*k*xycn-OzR80xs4ez$+(oE*J6{bBof59W*W_W0=g
z{oTD?=jA^kbI;j6JN@qH*!lOrw@)DZ@ndKEa2Njjr?dCt+vB~H69+Q)-@HB8-`hoa
z0Ool6@N|Fgq~z=$?i`%$?jOD`IWNynox`J3=V1TM{wV-IJt`4&QB$kt;EB$<^JedO
z=Q~K*ez||JfBH{q^%XmZbY4M|+s@nV<J0|}vxDtp=k3|?+oKb}6}qsyf3kD1z5ix!
zcMTALMiJWH_j`w@&dGP%2M2nmUhV-xc+Sosp;Np2$9p@ch<f~O2gm`491zo=yxrT`
zhwsq+J-~hY_@5=D03Olr{m&VcgG^_4`_1-i=t^NW1rQ#Pd*|$U?+xO2^vXFodwFuY
ze|mPh=e#~T+9m3q>>Yo<zq5C;<s2L#y<a(JCwnCaL&{Ox7^Q0e^kfUcVW-Yc_K8gU
zho^hT$7gTxjabq7?&ya-eA&CR4YhZPo=1n&XCTPY@jnqR5`)NHBHsAnyFJJ|MlwSG
zPLH>d9(;JN7lk%Y@z6c)t#i2d`e6U{-r>$3<Pu&8=!gB2y&??K{t1@Z=hlDNhNjP`
z^Efhq9Dmi=vP9$O>=R>b?|#3J9T4SV+)wrej}ZNKz7yn^EPkt>IN*hPDh;jt1X>XK
ze|1*QLZLXap2v=tNIImJtDSGZ6+W9lAN*=XDN^e(=wA}eot*BXhl=moqEV@%8F{96
z0q2ENfptIsXcD;RLl+S0dfK&$w4)TH+AHXav8XytQkel`a&d}+v`@lobe$+De1ou0
zDukunQc;cV<2TNwCofxH2JQrucf3XeO*;}BqSs^Prt#ZvSDX(YA0sf~wHHa|oa3_?
z!hmRx68{|&|0#<~9+I7Et5$pJgdQ#g&JMXk#pqWchS8_lVC?oZk7dr|%b_>yUIdSE
zHK8Y4mtJpCFOvAINo`3)uVy+#L`21Vul5hkx_k~q57B~m4twDOK%;Yhw&nSOFrQOr
zWnB8Tm`#3Q=UTu4+PnRs-y4vh3l)i@{o_WnZ3NzEziID7Vv))jegJC7K92Szi%@O1
zFS#krfrFNx_RNKSZLL){)+)~aDDuwnDI4}r%YdxCE4Lf1Uiv*?0Y6nN$&|>5d&7>`
z!!gw6Rb6h}uoB+JMJIo#ER`H@?R<?dZe56|lJo|T-2lSIs|~GgU^UpF0jCm|O(&>K
zQr3_14c2gtW1A>aJtIod4JT!3XS+VN#vcTZhU;`8m=Zyx3RRl+Qg|RAgQ`^%rPNV^
zYI166QI%3{PHj!9Vye-pYLg#hr}1O*&FLx?sx;?xl}gpBbGxdVoY_^?=IpMjM)j@|
z|JtyFX`sfUS+_<C*2_&*F`ehDur!SP;dE%_dM#Wq^i8zYCI^QbiEzT$D?OiksTIKf
z=mofcS^-2(FOcYy!eMI1VXBarW*{+DSWGjpm?|`;2{bg4)tpz5j3$`j(xmIprd)px
z;resv)}KeVRQn04r4dLFEsa5fW_=sU>ZZ_jy6%MDn8Gr6F&<Dz;Bn_aUN@5ADMPhz
zk;g{XI5K7_^S4}$X3Q)sIE~AZ6d{d=+xW$3ftDFV1X)|4Vuq|TT2M-rXhJ7S!+8yh
z12WP)28u^`=w4;hUS!5?RGkCTtZ17p-@4kDw@2ohtvRUDS83GqcG|RwXusLsIX*fm
z($D~MCmwEX$I%FPjn!(|anWuGs!p}h5-OX@tZ12=8oICo#|V3wYq8WkLF3#&FLor}
z$n6%ecCo!riN!>*?H{Qa<!=9|X9B+qAOr*1AAoRbF<FqoaJC5sW`zz0CkcSz)MB!r
z1jFC;To_Ir42~{<;nZTXpajF;RqmPI<@Q;k%V&C5AUDyKGaW_9PM~<EHx1dMX>Aa7
zuA3zxwZ32&<CAzW<-l5taIa(!(?!&9s-d-)AvlT@=`w0O)!5q0NI2=C<T7eN)!^F8
zNH~_UH2Z#*>ib!$V`u4(ou#loOJjYOLMa|^eT-|`821-g4fP(snmg35=8p8M`2+oG
z{y4vyKg_S@kMgVegZyfNF@80Fh+i!*!mkz>;8%0Ux2}z^e+x@@d~p|=LkG$!YNZM!
zX{8FB=_NGuE$ex9vvR02o0W9SZdR6<vB+0pk&g*^K5Mzx*ae`@;wH+T!AUCLHnjT0
zWrK2B5y&1q=>P8it5=MjT$OD3*nGS!(ZFPlOaf+VU=lD*s+(k&R5;mx-QGSudb7X7
zP|2~XMnpXyp$f;VvAv5PK4F&Zzm+Jl%|VsdRzzW(NOe1-R?Kf{eu`O4RbeLkVduMT
zW@^;1#4pH2YrH!M$I2goKQe7X)XTVbVwmF$r>s*tPX8qdM5C1nM08JzsuVEpiyWCB
z#86SgMw$snpLJ=a6d@{(uP?yy3#-2L7;gJ;p7vXh72IZ7Lj@GvH*91h1EzJsq{n^k
zC<^oXq~B=o6yCdoqT|57j~@9O3B^?8a%QEO&^L|TS~a0Y8oAAS-jF7VhNoJ~XX&P=
zdfR8|#;1DgXX)mr)P78_s!p#A`56-|AtZN{h}=mD$Q>mfcPz}57-r(9goQk|hX3+#
z5%f>?xt;qT%GJBIjXC$fssKI}|Eq=N?}Gq;@%ta~{Vy6p<NlXS$-3&*?}zK~-(Zbh
zo%N=};kBb!L7vR{@$o%gvg*^-u;@Z_T4w|=k;gh9@&FL6;f^ED)Hp5lyM^{kKRW5U
zqgUhL(hYjWJia{lyCBp6e%E8CtL$w65G)MFpu1mDG)85>qBLK|RZIXZeB+m(LL3y)
zDfrH}4!X!t2_jhRtd`NN!R>01#Sx*aB3D-W77(HRqm0)wue;5<=8SfB*9V1^VK9RZ
zk4!{tRB6^1r3)Wh;0Y&4myX+gN8?_ru6F#0ZrSzl+!9o`W}TA6X(pcqzIXirZoII*
z8-~*%?z{$m6v18$`|OSoeJqY~8&mxo47T)ShkH2qsrm~AW~4});o4x}Ja?{U4L_9m
zsr-w*uOj|xX1z+fUagiU5)B}-LBi!OkK7gd*QP6p{-`aJ5~b3GQe{#_s6;B@q?+%y
zaI5g)^GEsI&ia4pk9y<FuuVR6Qd+>g``^`aGxh$r(ri4e|M&2b>p!mVuormqcKa0f
z;=4yboV53TJU!lSf2YFL%qpW@aq)s+H0G#3O1R9Z`XFvT=V$r(ufqLKy6I4#*3Z28
z-%P*%sWw{={=e_zBm8>*o&Wwl?_GIa>FoRO+K<2I2Y#pL2hab`fB&8r^PSS-@0hN;
z-h7^P#hK5e1Cgi7V`qNX3%h|o!AoIJoo(lwg6a;O$uyXZ$)5+k+OHKIe>fR<@l|2E
zlI;yA7cO2Y3Y|VG&GAKIe5UJ;#3k^s>yk!^29)kJ2)t4ET1M+!%by(|JasOjX!3MD
z=I)Uft>?dIwe)E^lGoq!JLJdgsk6HO^@q6sAM*!(2U8Ee{=k&_=&TN8Z7iu%<Kt(S
ze9O;&T=(ZZ_)L80jn{JT|0<PQ?cx33J$!CB{>QMlf8(u1S2JlMK6B)MrCdqbe`@7g
z`9c2Q!zcgU_!6(d@}68dqsQI)53JPZpU?Z`^OZGcp9Pz<bA1lG0TGZvMhH&$l6Jdz
zGdO&Tw~L;r*M9LYSOfUSxqJCed<(n@e+#`{K3|O+G_R^w%asN<9^X>U=WA=ucCROJ
z`?-TpP{xE5=%e2`-srs_jLD0Q^WF`7vOVPMYtGTc8#ycApAgWijiz(RF#=A<=;sQb
zIN{>*Hh@h~w1U`S@E%y(py>KlQ{9clhb$4srHJ^-I-MRD^ZCY_BSXkbV_DxH2)cw9
z&T7|L+1YvC9XM<2f-Gy{#Y#TkTyqX_*f@wVok)vt0b>zq0v|r%h96JE3<42soWeiH
zD_R%(w-ydY`Om%_J$&x@b3^%WVV>+Z=E(n6y<vI(Uw_#D-^b_w`+xr5e8vM$=VUzK
zmzA$TDwXiD3J5Ys=)_K?x<>C6%sQ2No)h@DEc_fY*2-(udaGJqE9b>U?2M97VyzB3
z;O1H-|CZ%jVdX=$(kSoMYPHpkSGzl__3CzQb+g{8uU58en~mmfy}7%$@o}XH09F*@
z`;j*cS4y^&53Ta{<|e)}g^HW2fY?ibyir@N@9gb0cbb*v?yK6zm3&TMa3UhCNk{|;
zTZ8?PD3Z_RV!a4*;vf0k-l%5+%Eu{MCpTX1Z0_v6+*sXsS>Il*RCdd&8@rXtYPq~!
z-r1}`ZoMixDW@QIMyvwi&paKKY*$~_-gmFC5cPaxxB2pAb+fv<xmN*7R%*Meo3A!E
zR_hzh`pey&N~2QWhMs>af_Q%2jxGYALTE!$&o_P;$d)(D)s3B3yQ}rRm#x+MW@UGE
zqq(`WT5gp$c3*C`_IB%a6WP~;amO9xbNn|sZaaDqrMCk&qL;y9AY~yK$z88AJ^vk7
z@#l{E9;bscpwc&*GJTcLV)SZSQmRx8L`xzrLbb^gV6<O)9#z(U#d#(c=|oFO9hiX&
z(AA)0lB|O0GpITaBaN(!G2fb^_}X?CRj!@>d14%v2J{j&+_my2f!SzU@Bux!R=o+R
zBt|y@qz!8tHFL~w3O0e}EdaHVQD%M;5z<7xAy{qB8K|0b@dm(R^gaz#f`l^trrk6M
z_i4bgIMOPIZVozy?yUf}ll9Y>$j+Tvpch*27Lkw30?Lq&OM#g!AD0D{DIb>rw2XYb
zA=ugS@g_hQl8-k8JWoC@26jRDxIC~-`M4ygdGc|2a5LoNQoxpzk2eNAPd?rZ@PhL3
z#=z&v$9d|g)HFdYop?dydlt?9JHMxX9-^0p=gzC`gOfcAbd~U@HH_WR{v-Et3mpXA
z;QR+2z(1V-+{5Qn$DjA|v)SMV<6mo)AIASaKDRXfDj0;VjT?+Ve5*c;|2=$ed;F6I
zM{HxZ{U3%pb^otcZdED|_WyhM{D1$?|NHE}u7(471N-?(Wv#s8c%$wZAG|(anMVE9
zjg|j;k$)ykedtFCBlDcP!MPWO&sRuiTj4{|?-i6ff&suIC?W3Fgxw3^j&N-#&l}dd
z<KcSGd+!ZUTLaFbQ|R?7Fu=--yyHAWBm3)MJe@cPZpRxuU$L1RLn;fUZJEwKU(I>G
zlHTr!dQj7-@<K29Y~9S0HKdL61wneYu6{Er>o#1LM-z1QTR{@w+%}2}knrAVXWKJm
zVQ7~9+^EhN=p_-o-Jv$Nx^po6JX2-v9QjNXn?Itri`cVuB`i(RDJt#^1d1`LYimEN
z|BD-p@yQ`MI2gHupD9?%+Qo{q{>hNS5ccG^0}7+9c5ek#2ZrKy0Qu3-o!km2ioSAt
zklMX4yhUikTkGb#i`#({4Q>Svx|$r_4uf8};qCP8VtRf%?5VEo$dtQ*ijj#1x5I5V
z*!{Z#%^pE-1^BRYJ7A`fcXca}Mq%U+Z-<M})5GmhX>vRBV-oo9-N<XB%kOaFc5kVh
z0_t>h+i*wcw*+Y1za2mv=k%7l$v|(LdcWR|%=$IB9WYZja7U3px}DAiw<9%zDDZ9v
zAqsHU9iY~BJ2O5Uj>pkO7<sp&J%khV4{#fRFlzm+K#IC<_u_V_4|~0L-t91I6!vZf
z&NT9GX98aO(a8HWl~maoZvn<9s}%~+rz;T}sL}c0lUPZCg0&*LC9D`!`#;_%Y01fm
z@wWjg4nDm!FzqG#&pVr&KdVS)w*kq&(S||?q&%v85|qRfeF1B})nJFi0N5J}gRZa-
zbamwB*WvB^a+PSKzGM~Nk}p?@H{Um?L)+||)EnLaN!U6sSpzo7%heFu;pHmP#&^jo
zxW!$riW2|JRfC<`k`-tVwp<;t+gh#?@2i%qgPVzEtH8Elxhk5PyIe)qM3<{WI^B|0
zNPAnZj?mMVti&4Gl9fRBTCN6ZS<6+C`cyYVs*BrD*TzvAUu;Sqp1q#>y%&q!Z&kKv
zs>4wJ;ay*jgC3juG5guNOyUB^umw1jo5B}z8b4b%)48m8fXWAr(he?87+-9xRi3TS
zERh%H^0@1~#0~M)zawyezVdb9`yFwX`PPjtia)F8P-{Dr$(lIrT!Wc22_=0^g6_!3
zXU|t2e_b#v^Tpr4E=atJkDot(teu@bUXep#RFMN>B%XqH^J4cJv?afL;CBKyxK11&
zPzkPi7JAeS7xcaG1xA&8ww_KGLoX1d6I?HP@xD?o#|IBGRbX`i(x3}^{0Stz0mAq%
zuOmFl(9<(0|A|yHj`fx%-U(Iy3)0Rw!n;SFqadcYL%1I<#shq0cOJOIqY138{%?Hl
z#bcA1{RNQyGcoO7ocQWw_(8_rOvc`XbbomXd%yhl7oXzk3p5YXbt%#>sRtYSFV{7;
z6MJYZK)oB9jL=@SB6cEi(X6T`fUVbCyu9qo$%O~VuH0Pz7F0}A0!B8#9m%dAB<Y~R
zTbn{LNf-?Y@5&FOu<-Z>a2^*G6CaGb?qGW|*>xjV7LE6JGpKZ1igkj_e@<fE0!t+n
z>^~=+Zcp`nV5_Bd*gq$YZbvzNV5b`>seeu?-In_Jr)4Rt9T;2Z$TPQW-4t{?nqtRZ
ze68N*VmGyVKPQ3y{A^v+5riKJ05hwIQe1`gShOc&H6AhiRqQ=m*YfVgPWdI0VKs&#
zwj-Ei0=rvE*_EbyC(?b`{2mx!G4<w)BG7#biZ7N3tM9uhJ`tyDpG#hRR$AS?1?w-t
z1$a2Uiy<Y5Qn#aJs8e$OO#|a~IjQ#@IoDM=>)Ba9xWULDogd(v)ED$`?b*7KC`%gF
zxLrR)r@<gJ(ndN)1@yMA7lz{??s)j3eZ04KczV2jc)CSVF%Ay4;J>{e-yR*GZvFS(
zo9%;xqn-Bd-pjMsTjI~(3K_tP)Gcc>5{>O1lh~EF$6)YbkjG@YwA>%tnscWNkeLM%
zFq+w-L`x9wgrl~11+~K$_3~zUt6r%$x9ZhqYpY(X)t{|r6^=<wwo+7wU&7=<&!vK;
zVSY@e5t%l>F2qb)T$}Vd#HH4H%w6jV_Ih^UkKU;^Po|xd>kydwg}9IKY&}krg`}rv
zV&45xpPZ@2hThC<fq-m~uXZG6&+W}ByXx|XoLtHI6VEl3!!zx<oLlXK{g;2n33fak
z;f*wL(81>fr{rqY>xn0_i6ReE^UIr>hJx^ioRJU9-NT&vvgVXc_x?jpt%t?#3tQ}x
zjLpvTUmOj3C(-o)W<W*(S)V-QyMtqoT;j+Z2Z_|b&TO@t3)53|@mY7HNo~x@H|`Y-
zf96zuH64)?Nop1;1Skl4TC-%meHryvmXa5wEE%7);vPOMv47nq7V7Vfu-`jJZ+5p&
zUyTR7m_Kbfi7DV6dwmb%=&AdAmW=xr4~y<zAEZQ<lt^g{hkspd;bB+&AYQT$A0DL2
zmp!vTmrC>?9R8ZZVKGy}UsOgsXdYjn=J81;iU%QL3X$a=Q$(D#$a4w-#r8TvJ-~i(
zbN|mtf8A?+e-L>=czYhE=lTDxHd?KE(*Jk0R<8fC{eR!GWA)qZbo{x2|8LXL_wD-m
z7BlV@F=6cR;AA`vx*o+LmG^O>)Y-`Yb?WW+Jdl-rAC|!n^;bKsTI*G1b*Hu6SgqGu
z&DCvSz18Y=t-9N4RJW_IUVVJFo~o!Mikx-fi|%By)!pjC|F5pLdR~8P+#789@ZZJc
zKU*DG2HbFn|6PH!iE9ag!T4u%hS%)*o!P8yZZ>Kg<<*VNm%FR=PPx9iv9b4ZwOMVv
zdb!uA?!A2Va>34AY+b<rp+^_cnTz3;KZ3rDx?5La?~~A1F1=>;qtV=K_4;mYwbJbN
zSL>U#>T0K6-B@jUWv|=xHv5|!US>a*K}K7oXW>)sc6!yW+gz<z-PUTozfoP?XjEFO
zy_VN<EA@VFbEA3N_@FnanK*&eZ}w|tAWsJvx7q8hZn#|+XjH4!nzg>$sCE~^>9*ZK
zYkt;Z&5>Kv`<fujx@zVVo0YEFK1<umo7-n<T{wXIEUizZ&$!Rnx^Of1SujwZ={`&A
znlromY(M4meOBYD^xa`0{l$M~G^DTj&x?leQ~#NfkUs7|FB;N6{uh9S@U8zPU^4t7
zhK79le`Xvg9>C|tLB$gIyl99x1fLlRj$!b5k+8)-_`)z!u@k;Lyyog6p98Ih6!g1C
zjN(0fW)wuMh|i0KiZk(f(U37JJ~I}dU+&ssV|*DHnUNtrH%=6j<8vcH@jN~=66shV
zUj#y561V*N`Sk{hU-Fqz&|{x`UQ~k5zxv9UE1wsOM7);IjEo-Z<ujwgabi9*5+cUT
z=fy(Br}@li$k;ZY84Zey^LcR)F?BvK5^+49`^3W@<76fVT9rFMA{ip(GooUKp;;6W
ztaf+E`18$K%iwa0p=Xjf>DL9)IGER#mWft#$L)v!H+S8NjEQr{?Z_}X_tA`qymK#2
z4ey6~6yN8r8=;Uucif1IB6P>?h`>X4-3rGly5m-~@J4ss5=BJ1>o!CXr8{m$#a+7N
zc4VlgyKaV~JKcFJ6cFl;TQOot-Emto?9?4MBO+DZbt@vc)g3pZ;$hu&J2HgUUAIF~
zx9+$R5vc2qTZv<P-MAq`(~d8V=kfmUEnQQNyfA7<^n4*)w_S~HwHg`szxlg)|GV0(
zLHWjAU3%6(cXj{!5AwN-@z;N>btjYA_y5byN~4mz|6ge!zuX4_{^H}mzV1jCaCW}^
z)``RuBnJmE54qdZ=wci^b&ioxLub3wn+^u9_a5`!((#P5J080JD4%~Mj0es$-Z8(>
zQu^Xem^(02qiNvjDWE3w#zQ@YG161s(_<<<1+5d~wF^B-_Da2=!0h}q^hf6o6o2}Z
zp={|%+?0_t9r^h12=5%Udp??}I^>YkC^Ymz_w2P1_pN*$iaPofC=~M_KnnlIYm55}
z-SMdJJa<+`WWNZX)>gJ4AFGKJ$Sh+LS@a_({0$QD`?y~aC5l@yMA*d@R>qTPP2{bV
zoWlFD-*cW6zY$3#=bP}il5===a8Po}RN3!45}5PVbEj5xKG51R2>_9PVP!7}aM-6K
z?`i^+0*)9>haE3)#(hWN?fk9x^9WkgadMo%gE1XB75w!PJ3;Rv9q4ozQ75q{;ZMIn
zpFTLPl2d7xoLaTyG|MGtvjV@%@TF44f9iG2{{Pu~_P4fmWPc`qg-TCL0tt{1UJbCD
z^5}98(0--dJ$u`J{+w7LE{UCEI~3aGf4?&$$(A2^Km#o_*|Rj(V>FtVMk7bF34b=>
zPi1wr$f{VowvKI=sRKe-u0YKSe2Q$fyoo(m;D69~v%CqAH}QioSKvQ_Q(1!!73fed
zm)T7%3rm>oDIyAJ=Ha=Jmkc263v7w$&mWgR7ijuIsFcjIudWCFjz4Gs@q0fULUCe&
z0oqVgM?vwA%`2$G$w3jU;a(XYSC_-b3O&|h!bKjVL0rrWen)=`nov<|6Zl$=Nm@A)
zwWvi(+Jswbf<|9*UEHEBByr`q$SKkK<h=91rD6LRPiBFOwDAI_Rt#`_oLTB{&Vy##
z^zyOZr`Q-(?}eqhuq%9^KX$;y?{p=cqDJS4j1dJ1G-x<6nc1Xy@Xwak4W1VklzK>x
zF7j@#V>Y?bFq;?okAZD%+n@8Q5o3eQ#ugOp5P|ogG%r{w<gCQXQiB-N3<~ocD<2?v
zS$REd&<8jhMljEMf#-kJ1?IDAg7d-JZ`suY=1k<Nv4sUYnhcY4W<mbrOMR9j;8ap@
z@DtMjffp)(F7u^`LNNPq(uo1ThylNd0Yz_oT-au7)#Rx9xK~of2MnM$4Nb5AF(RG%
z2<m=8OXh#esuXp3#jHZ8rFs8bYEU%D&t$JO_L2Ob(QjK(ZIrg5+&8Xa$pMrVt6o?|
z(gpe=miHIdqO#by7=TPclOp?4WLGI<7~@xVkT0hyZW7NSH!cJ>E`S?A40Zu3lb(W@
z-Ny?g3(E8YyNC~PGZ9wDktzx<DsGLy5!s;O`7GjJAumxbWU(h=&@u12-ZkU6fR5q_
z`zQW^m2|oC*RyT>OZ?Gzb5YdDR?I>s&AWlzOY;vzvgL8Erd$}zj_CW1)+$*1;@VC=
zED^0liyLKH1qEKR0A?x;s3M`>|K`UQvdXPjsY=!=CBb&Mc>zw(wgw;UQhe2n7PLLr
z;fw5u`T#e-Nrfb43P=PS&+}Nzgnc<emP72Ywbj*?wHgDD8pNe(`kY*GPuu|zjmdns
z%Y!zqeR;lkzQ}ABRu&FkIW_>B0JLtwq7)QjxP1U$l)dnu5i)r>`ajt0LH+P(e{U9e
zGVAx7c!5Hz-<d58U#h`0!s6TKgH~zj{e`2H0X%ppH@Tm^(V2cSKli@>Z;$%@KYXf{
z)czlw^XdNo0Y1-ro_XFind>xp*4DK)Roo~2iRr4m17y-;4A=kaYPFKS|D(F{$m4(a
zU|1(Raq+ps{U3Y0W%fIPyqIQt_BCGSF`Kv+q&(SsLA2C-h(qi~t}sJiR%LXslFn}t
zov6gR?C32MX`#a&>I$$Of)r(URtGRyxZEDitF`JxkOVlBMsW-hJ~-{Nu7MqdVZ80n
zMs*0IZ;9w+SslA~djy6*M^#4Z$WNl!XpT-6vE=KNnYljSblis7NgB*IecRgkzP!0y
z+1NeUD^>Sjua~Nu<-O9z+U9O)d3||f@Ac;T{$90u6OK=P1e8DdHEilF!CtK1T2e)#
z0p>P6Q9t0>(O0ysrFflaFRzRf3yhk-Tv>KD7Ry^pLtEq|nSLzay<FwudG9Bb8C9_(
z)+M|&QZM{)-Qzp;Uaw;}={X@YR4uxl`QoBHab=(#-fEXF&&8xnCUo1mELWE!X;Y;V
zCD`7Z&47ESP((KszP=7ne)aMJWS?v@)hXrfkS*b1h$d51OoD3vZo3^W@34*CT`Cwe
zu=*FE`A?D8-<<SXdaS8^xLfX{Dcx>;5ByZMAKt1?34c4n-_qxe`MSd{C59pK6^zVV
ziL<+&0=C>5zrC8vX`9nU$v>JOI86!QtcSY~Os$-tg9dN056u)|G$i<y?FW8-?hbI~
z3ZV!e;kg|MC|=aBnkxDarP8GQ=meRcCb1?UQrY>(o*11b=k89$>eim?)1=X4ywa&S
z-N8S7npB#U^Y~;%Dl_hjqaV{{c@vIgO1i!6<HUCU$e+3$de0T7srCDm1o~-fN%oP1
zcO-zJU1TfK1(R|dI!%vvxf7Tj=}ad}A0#UIn^GZWXU0b4hY5Axv5-eTp=?+6SCCD)
zrM`FJ9GVUEHzY(%tbfD#XKDB2Bf~@z-j7U;`v`K~|NHP{r4pvG-+g;>`u1r5VN%55
z*|oYNDJxIhL*$MdJM$WdgYRM<x5;N-i?Ea#>U1o}*C~Cv8GzEG?eQLWEbcV1TD@Us
zEq3fi?Sgv_$X$gV(2bT~&n;Vi`)WIW4=>ch2jGX*IhQr2b!qlgGr!F{3J#Dj`eyb6
z1-@^KL0zlSR1M&3vnL;PHtRUYW*nQ0WbOT@s~qCLK1yql${QILoPMZ(INbex4FwyX
zWKzBym6d67YGM*5rW6e!k9G>B<VaI1^h5i{M`?VFAI}&4s7%VQWf-2gH}J4V@WFkL
zN3Z<jdLF-xbpSa)#=m$f4o9{NaM?^b0rHLGC+rJ6JsXrA9!(pd@omYSOjXR%9Gh3C
zJwDI?9mu<rXjr6Bl;}LOO7%N*@K$Z#ZmL%54ZMts@q#WVzw2J=S1?cyl<xjl+j~73
zTl#?3ZQS_I)X3e=qx)^`-w4<4*Y)atQ=>TzSlyG`IDFSSwWl#t_uHDwspXrxi;u{v
zjTe%e8n>ylICU1Epvn9_L!0B;*AY3%l9%1f7pzA&Ht^DPt;eW*X!51>P?%yVBf^}1
z;W8-i@|xA`m`MBZeZyL7G%A~w(k3rAOV#p<RoZNAZj`DUYt@F;EU%WUW^~bqB35$v
zaZt|4H5b1daM09m_p`-7RE_h)MF}PAcZku9@8h1WU-gr9|J!1Va*0grzOfPaKbF@j
zE6Mxcs>_c(|HIS0|1Eye+hp%~lZHbivx%&Rp6VvDXlIW7cb^}ERS)cz4X)((>cQ^%
z%KAaMw7XtkEmc?6*GhE|y;7yVQrTNyt<)<A2VmM#9YY5|*$ICK40EknYr=n5S2c^b
zV9n{&Z1{h>_odbVN6GZN`2Q<#4x(PtSjLj^BqyV1%p5kE&6Ul~)s>Cq(#GcNy;8NY
zTrF*E?7uFpRaOsP@2^(&Umv_4b29B(8~%eCwP7;tZq0UJE>5#{<y*tizeAGF)Pfj!
z?Gs6ml)Z-Dd5|Ap!;(Ym{2f07>Q7pJGO0P?F-G10yHZZ(e_E-mmdjK7{{Z$sOqi3D
z>Y-VBva|U#+n(#VVU`_7UF#;2<cM0MLm4_czD$&ZB1c#CgJae#GQ3E9ejXn#Mb;dw
z@|u_~AaN~mc-6SqoL(<}_s{Lc)NgurFGy#38~j0ixMlvg^4e-;n*Z$qK6mH;Id_b}
zGcRp_v3EUmj1l~=u9cJbKUb<NkKO-&B%kNcmioT8)Ucf;?p(6ob<lR5T(0H0UCLGx
zptH|xu^|UH9vV^{$j{{j^NoN~Aq5;Wn$6CB%5_ZVyl<ZKSL__)ZFV!4>zaXQU%eu8
zPHV-h+zSBE;4RnVh7c&MEitnnxJ}n-(Zk|WQ-+>6`{VoW+v9hKNBi%MgQNPJ(^m`Z
zlk~fGceiK%*xhA+#9Pd=o3IH~og5s#IeWh^>Ii(-yx^?#WhM4N>kQzgBSm$vr)gf~
zV1~KebM~Rl*_<VA2%Te1JwV26pPe^hDTA*a+v#5w*;hMgGY4~Lkk{#9o{n&40Ik(A
z&;3Ql{*Adle3wuVotj9@A}e(<L0YNADGFJ#Fn*~$gU`?^0-RFG5^u6nJl`)eOdk~$
z$|SE0HgL?OTqtHktki;&3|QYkZv(!Fl;t{eA{hWCAYqmH`E}rMxw64}uJ7BJf=Cw!
zObZvZxzB41b63GCjc|YnoofI_V3N)IJxnN4VRL<_@Au8l9P{P1QbpEx(1iwotPNX6
z_+kbOChY?(+qrZ*mx#8FNlDCR2+;30+aU~0QpY^_vxwujMNvR`4vLhOW;ef%DPP!v
z_*M?TRSKANZ8LXV|I0W#IXgY89~p1oeK>r3a*7qXoa-Bs_wYPNOrFC+IS(v!eJIK)
zhR522xPpAlX_Z<(s3e|ioedNZ)do{#eVjvinEQ-Pq!6Xskl@e7MP{z3m7vgwf|$_6
zi-gF`6>~5cNP#{=o<{f>P5U#@?_+9GSeiW(>c7AS%pq3*H&T2zbipwK*C1(vIeyL|
zA8kyUbL9cics`3|3&7e}G}ush3)?XP&Fj-GGP{88D%mc>bHDmz>}dE5poq}0u>>|Q
z0*OS4%Ip@?<R}}=D{0dr2(nR*DnbshZI+ThHAq_aMvavXHgjZRWF&<6xgw3b8ZZiC
zM0MGM@>zk`1?7(LGi6N<@SMp=joSyChmib!57#|zB}{yYwYk}2s6uuJEfJKxJ!O9_
zMoeVe(h86fPN-cP7hk8e$gm`kf<@WD^nox40Q`Q#;)~q&?Dv~lmRQ}WaBfkYZ{^c0
zEV5ZPEAno|c~*$k#PBN=1U@k(0G*k`!V*mQ6?@@-a^~0z_&&HLO(<#_%}!BH*tS88
zL|RJP3bPxLZNPA*1q+$~3Wm$g3}^z4V5J}4f-%_0UVv9K3X~uR82C~b&pbPJ1KZ~)
zpB77cOfO<s|CBxK9sGZ<d5gUte(tdUTV9{;{~qLXH~v$$+QsIWV+@V|tyGt*E6My{
zl}e>tneP7{;Pc$VV{bkazPwU@zkgCUc8w^mHyftXCWqTGJ7G?3LzQ5_Ft;>6(a#b9
zWO(P<B7^Ta%$+zh<MuEL*u#LJ*XOmIOsh<lP;Sr>KHzq3wEO2!-|Ktl+~~SsV*~0;
z1<?K7+50#9#@_xx{p{#N4P9y6qO?tKW?t(<F-(!GIyW;*%l~W<zFE9!UemXerTT2{
z1`rW>BL#+?bHlQ|;M(X4r~l}Uv3L0X!+(rp7%Uo8BCoo}dMTGJ0htCiC@<tT+uPr*
z|Ht@nc)ULlX4F0cm?QNyU{wWjwZJExi5Gl4d3|_t`WYIkk-KIOYE8cuE9u&P({NiY
z*m6Y`0#w($^7>AV0S7UmFMOxGn&hU5RM*5*)v03=#my_&{P=#@jfzD_s-QmLmf7bH
zw`QCU6>`!3B+O2p?8TG)+S&b3VA9?0L#lIZo~akj(2ES*;JU|sqK@RBim>})?@OCm
z_3DL%={a3Y$BBTf@VymqKM`RG0eAS48_jm#xiBn|dN}Yc#|YRw{#z8xz(@L0BzH>h
z1U2-fP|JWT6eYl*e%B7hfGl&XIzr5;S0-SPm$A@gQFADRf5b49W|hV$ito)1H-Yu?
z$--9<Vd#LevZhBDBa=D+$U71&12-0_8I-_3nWJNM$NdUEISxMFgfjq7UT$;Y8gY1X
zba=9F>`H!0dxGYGce(}@p7S6tY^qM2&}smhvN;~}%w!R;mYb2{(gVNSc7S`(y<rD<
zbTmUvFbt9CTG;*9*~y<qeQ)o5<#qjZKd*)Xa1svY^o=150xYZa)^f0@wRjdY6C9Sk
zu(dpl%E#N%1W?>8o|k2qU-u@9mV#Ow590&Fp#_W0CrB%6Cqdl1kx)(ZCABW?AfAIJ
z1ipbp-*@Qk(h)d=bPsRPpy{Zz&j1zl8UEs2&|Z5%-o$*}GC{N|YiMOq2pwR~I++1G
zB<Mg&o5-@6#_c;c3N)mmS)Rt;A)E`aNfggp;(0U?O=~<Ns$j`TLSiQIUf>ZeCH5*?
zJrx(sn%=09-KcRZ@@uqE&~_rok}9fZORE`2$R<u9I($aw=5{j-O@@+tkn9F_z})g!
z5ABx=xtWj}@NcTGnvCijZ+nCGov9a^!4TOLBBHQV=sJ-9(rbe(I(fOyU~qJQPhUo`
zi;*e_iUUX^hzCGBcC^NYq%Y)P+t31!goT{?pZyQH=NQg!p%-ZU-8Y>8HMrGrT`#A$
zAL*(H!bzUi;Y58Q_nd<t(Fz@2u@t2n=B2^Fe{=&vjGgKE-@t<7A_M<uj3z&Dhf>1F
zing}dzs_Lea9sau_o#k)s@01IFWI{~)t~-Zz(Cb<Aqn-_b{Z@+q;9+WhalK*s8Cps
zo>;K}&k8P3)Vd^mTv)X%3U5o%5={|0?p9{N&zzY72eD3PJ#?Z11o?^wKyBM+e%tke
zQq%UDeLG-Z-G0Xs$4{0^2QQv|-VOu=k#nAW_fCfh>0^YBT4CIss=l`EW*f`ERRysS
zJ#kVcdI`jexz-@m{eaoQ9N=MhdmW6x2QK>pyK&Yw9jn9D$asbX-ArGe0trN$W*-W;
z=Ru|0_ko!{Y;F8S>b0b)77-HYbEC8iiB$2w31XGRu}?_Nh%49_c&)l8weLQpJWIx7
zWy0f2Lf9UuDK(`kW24>Vb|)KatQj&OR*$2Z*lws|-j^Z@AsApKbQ`@T@QP{D>4n{6
zCX+oyNScJQYhHucUUED(Ff9%Z0^YfnJ5ln|segkDxg}jJA$$vKed^!ftZQFEiR~yA
z#nV%O+v{=17ZAR-xg*Xd#IX&{)JFOLyAL{qTO50lFG^jd{jTUpTN=Vx91BAqQZ`gB
zJUZo<_*NF{bwE{p90s%)OguL|(cgz}px#3JdM^4n6lP&9)FZ#&YS~R23ZX=f%gjsD
z?%<(^fY9Y#*SluE+vO5GZgOH8tE;E9cUYP<cS)7Kxx0RjFX$O!K2$lW|1jMo7?Emf
z8$sppA^i;75spXZJ}|lEOl+89du}ZBNzY<yp0>7RXvKh?U2Z1Yb*f4zH^`|<?v4rt
zO*aJ^sE>)QQs0px$}MdN8HdDncErd>Zwyp!9k7DDwH4Na4if;S+Onm-0u<1W#2dvO
zAxZw1R9xCYZ5539h}}OI2T`Ki4^Z?|v>?}Q5U)K@kxM@1^-topB>#Q$nu1#t0J{g>
z9uW{o#Xi#bi&;jYjr(V39GW0a76=$wgeH#b??gQP)<71-BuZt3CJ;)4%+37m84mxN
ze<`U1vH%OKXVIny$AtPW?^tnBkpahnon%`P{KP|Jfy+oKxd*E-m68cbi82`jd9Do`
zvVKRgJ}nicEUAb^S!FDrFc(epnUO<^%mJ?>-zXNkQVgJGK=zR0G_WRzVmx{e4Cr><
z;DfAI#`r4wj-3TA=v|9Pa;)_}>P*1F*DUB9FE68uS?weeVw4OuM7I(?sz#W7F%5#4
zQpSPQRY=^7B+LLs9ZVP+CnAh^96%TeJS52=$r?V7jJcP_oqIa?MuCNS&0u-_+z`h$
z!wx^hp(7ns(04LP0+tBMgd4$g3d)Q!SGBOWRGnxS-*;f1EsrO+kk3H>5mKw)>4;XM
z0AxvFrHTZKa`SjDyr{f7Of1MYeycwg7(7=<*jS>Gt4u9Ip@4%lqv@jxQIiPDBTj+_
zg}fG=l~5L90oxEUn9j01zhw);%xUfNv4-NL{}Q%{^O7$lOA&!frK})jw#+0X#u_9t
zVn%?@z;ldojxQud<0~U})FummG4>o4h(k}!Fh&q=cQB;<Z=j}r_JpJO_!5O7DyTOU
zhKF2{f10H-0Ow`m&|bj#oK&BfoQa}55m$8tYy+>5tu$LK5&{Q06IM&}>;Nh;BxpL~
zC2Ty6?z!|8ti2<{a{?X)Nm}4U_dQMl>ClN!l3z-oegH8*_6I6XAw!Nc<sDkS7~V1x
z`0aSSHi7oeb_&itqMahY=2hLA1JW>|<c(pwikjZ%W@H7}!CUr@8I6U7@vOu*?zu@l
zk2}2Biq42r&TcMxx9NZDM{nLAp6r1i_vgNGynk$*o$T!&z>s_SEb}Q$jN*+PLf>^3
ziY#L<N>-kCkupP|4g$C=ed}z%<4_CxfmQM@h;G~(i*h4LH3p%=DTx`x=M{OE<4pim
z&$d9WZ#xNP{%qiaIPOx7=JB;L6?MJ^Pslb7rmn<9p#Z~T^fhn8V9PaTgAqkAY>Cit
zTM~f+R4ptNB2TXO(f+J#P3Df!!JoQZa>KN&l#i(n#VyOw1fRTZdXL+nA9o1fQnqbI
z;F#JnspF^6trTrg1!m;i)DnIy_EJi8PxRi}lHH>{#*Eqtgea&p(HX+j(5m!i@+c>X
z-gbo92;%wu#vtNB)vbvKeNx0r&0ZsvB*buPmo!xL?n$!Y-~-5p)6~9alMOuoaG)Vk
z2*1~{gF8hblu_2B5cJCn87hetMK5JVMx1j^o)zr_H!wS*7+009k@WhbAxNLtfB%~r
zB03P#prt&zVId2p?#YnF*s%r)U_nDb4>&}V&?BlLK`@r7^%UEHZa5EpRwa#&kts_@
zoQo`uSur^P%AQ7BC3?f+RU=^zFl|Mh_lPujJ%4iE!=F$zFX+dHo+*BNgGXWL{Loy{
z^5Rb`JO+ir*%1eNiZnawbX=1?B$zg@w6xUX?LjVt=?CWev;~n+=;-|)TCC{o<p1{H
zzl9a(|Ji%j_9l*OVf6iK|B8m3O(c+wO-M3>fq4ixiT8lXi(zh=96fERZFi8=O1Fe@
zW;nn7)Vfuzs;+LeY#_|u;hBkB-F02HYSpTBy~i7W2j;TTf@O95aDWh@hMJC|4g;Rc
zzok^z3y(R1f(q8j4X$3Fs1hD@9lM9yA8h!+4q7#X7#br=)G%2~x<X72)&+aC6T^Pq
zS!mTYd2Oxv&Ad~D8H7eP!c3abvurZY$_DbR@6<}^u8d`A&$3twASR+a+yZSjnyY0q
z5yD@m_?A9sTGhPiz#z|!54Wh3?Lavo`aJlxY#mux&{0Uk%wji#Qhh8X-3|v=O&?uh
zW%sS233Y5R1MEwDBge}_J|2k+g65MOH|ocz+dNQE3tNY0Me(}YrlK+?t9`S>t1*X!
zp^Uj(5AniDyOV`W<o7|_O9O@9ze}=-x%vJp_=>r9aXZ-)!5}=f+{WtE4={ztn$URI
z>>XaVPryqDEN6pG(=#U=gcSf+iM%qW+M-yF&Y&?c*Fdu>h?bDhCZMBjqw{@DDAYPP
zQK{o`9sVs>pxsPT{aFd$SD+yoH8}+{l;dz>!QDSx1Rub9(aBf`3{F(PX#bD>g87Mb
zOhz}<ueq;nXWLwJ2G%eMg{Me(9s2wcN2jis8@1b>qF{K%qhYxX(&<dgk*P=cTNZx>
zRBq)1vIR8%?5gk$2L1nKB;jQ?ylIPuPX^$|M2%}`*BYzej9VJo<o~?$_U+Z>HSGeM
zMs$9MjOHeqdSweVKAfbWTAN^Q?&KsdtHl7lx5*xn5Dd+<z->FtArQrk5Dc3^tTIRG
z;tU;ou$c;CRAA*QHOR<~1AZZPHFKEyPGgAiHAO-G6ly6Q=LtM#>=5Z)TS?bztv=a*
z{QTLogO@LD!#+k6FVf1y8a^=D8F>6Q%ex{Izfrr8)ppFLhgrHGb8ggG`F`gy%R%(s
zMru#wzBCjYoTyOn!16>v&D`IbY@g!t90%w)z0jGZjn%|S38dfz9&o!GWMhwQw#@?c
zyjPsWmBx%<_67~JXdN=Oi^`kJ@Y}vZ=(l052XJ&LJpihv%%1<|;1I~|!J(nN2Ty+I
zoUh6`N9igM+%-65kHD!DE5;}*!*@huUvj-RciVxvtqZ%O>`&|>7ZJBbsC$HQ@N1D4
zHFD+{b|@e4HuG3@>Bk9F6_ezSL+`K+MlLK_Cr-*A{<{yZkZR;_n$MPCb9*K(NTRE`
zc@>bmR_{XJJ_W0Leo{hLbOC1$Mdhh@*qU8=CwfwwS6tQmq29JbU7Ce9{%UEVO*pIv
z;aQEMTl}S(tdDJthHF%^d4LTSg6_qa0?~<VB^pjlstC&8YSAd5YS15`hi8OWxp_KX
zg0tDb9KQ~iJu;d~z-Y<*2i4xiNpjtiuItIT$SMeHH^w9x;Hxm7oBu8fYy-!_ly^Hx
z9+d_ZWm@K@1l($XWuGQ#<%CY?r;+h*V(_t{;)L8fAf_J3tI?@roWIG(OYoVXEB$hA
z#z(tgoH_Rl=mi@A4~sJ65;;0trRcQ?fSfJ7Ughqf&BoAzPW$OHur1A3!WaNFN&WcP
z@LEb~d4R<F?MYz|5!_mHKYkrry3WMs41;M8`EXqAimy&oi%;}(9bCgh3r_}D-<;M>
zvaB7qp-`UXiL+{gEVi?wAOq(mxc8JjHKRe|jatqD??8;Mosf0Q1Q7L9{xM9QyMf-1
z@G?A0%kdQ#fHtoZFW+1(U$)Bw6o;eEOqYx7t)Z6Y`87jM3_al$TggDEwIQNgua6E=
zW;?-6n?y|K`C^t;<{&_NY0m!n;tXwd(wVgR*Mpab`$tE^7mq+$`~1cJfLXgno>&8J
z<;oJCz#4hTq#kd$wY9Qrhn8}jqAWll=`bwu?K<llsbNqKzX-Q*y_j*S^-cZZS|X%{
zqiUx8gIe2Wo=g--qy+W-`Jfo_)ofe{(Sv9$I3E!gZg?A2*wyG(wQ@DJWuv=wPoz_8
z?pP}~^)_y}WqIm1-NL$c<9&;rEB+J^8WtPF81XZogr>4WcNm(;v%D?SvSFGJ^X@ca
zMX(mN{nc5%ngsYHMg6j-7vDdB{^uieZ9RB-U@orz*nbfcjt@i*P~7T4nClnkGm9bt
zq#gOtAVfQIAE$13v@@g=o8dW7BNpy_e*Yxn#wSKIydAm{mQF4mIZFj-?kX+u9=h*G
z8dUewD!O|&I`3VSpKC}z(2$NTZPVE<02B8K4{YH{Bzi4vB?T~uthQY1pvd+V2<R9i
z?h5GlYGtxTYHreaBjLim2*H+NT#kC-&2}YU(?*(<zMSCFptO$3<?^O$@&k2EMqRd3
zU-J2<UaVZ)U?cGEiSZNC?`944y>*80dyMtwQQ_FuLkOV;S42?9MuPEa1<L__1Bn`_
zLd@Di00`Eipc2L0X>+p!60}6O`cSx8nS0kWb0OQ52oWUgFo?n888g@xc=S1Q2)L+o
z4Z5|JIx-mHW>hHyoS)bjToWStr(`F&7g4~(<&eCNVI`$ssp%;iOl@RS3_MAI7T(!m
z$F(8u0$pM#fmwkfJXnp~Un7Aa!W&mCgS4p`XsajY@U-h;@2ut)E_z)CBr<nS5#C7>
zw(0XzG*5s@)D>9(a%;D5Esogaw(eWd)QUdRl+Z=&H#@hZM?)HOD=u|kNN(!)Wvzn)
zEU+Y7p+G*U%Qd^HQ$rg8G7<RC25^Jcdo8&0t)oFmSJ&+2BuCq55;9vQcgIcbkLj{o
z`$|mxWsk&7Z3*OyeH(a@P`-@fCu_sgR3T(p+}ej{S&ff_Q|On;5YP#|ygm7p`Cz@g
zC&Ps@+!L;g#9>t^*isc~uq{@9j%a%;ZKbm9YgcnZ9p}|(rF149)>GM4!%ay%CXlbO
zr@5sdj~@?@4xa8GKHkU6mQ^IwjPWjaXOn8?*3lwEMCddH;|mI|;#(BM60k6IdGQP;
zIw=q9`UgDn0&6dcDD;FIp;RX<(m>`$9i)b_(cDA!CWDj))Bx47-_g&j0-+{k2ws#~
zx<E%Ub7Yqsu*Wt;4cIPR2WRG2nw;gQXD-qYT2_No88n9$)>J(mLDF&%$mk(btRENK
z$6tSq8vwa*7pw2vt)iT$2!~G)Uy$>z1ry-W;=#2T=jyN7<6`zED;cSiW3Weeyi)S=
z`yWUqkLG3+ymvC8n$_~Sf<R3(sqDo56;v{fLhgxB(k~)@1}03TX^a@-w89rPh}A6R
zsO+q0DNV$3z1wx8+=~t_xMTaLRNT`hc4gdJ=gPR%c4drGQ#^z&M@_LRX`~u7pH#v-
zAS3EFETyT4v!!&p=9r&FP%+A2sq+QbwdoA83oB!8JA1l^X^tE{F)047@BJk8y$wCB
zJ_V!w#jDkOMbYgG#B{W^#Kd$iGLs{+a9olC7nHns^8^p*oaMWkC5u|XbYaPUnS?Q-
zR`Y^l$)L$W-OfSeNkAHEOmh=)>b6aRx(`{BNKS&wIoPN)xj`;nJVv%uGyl!rCKL?w
z4*uv-A#g2|ApGVT(*__H$Ee~Fb!&?x^m@jFf&qdfxc}Ad42;Wv>j4ccEOv%}e#BAz
zBu&;^^NWp^<rys;JD_)mn%T4C;*$a$2Ht1`-PSswtGCt!Iw;u;y78xoaN{))t|QdO
zYa(1b)XJ>N7DI96H7tDH!t~PuS`-><W5y@4sm5z;s*dz%yym8Ar$=S?O;t?BtkMdg
zC`V!YE*(e%PMpzxjtsbmHHY82@-heQKHhmAs^v8%<JD3#il=2D>!@zhvPcxLnng=Q
z#fRlN$k^g7nk0jH+XG}V(#lDtTtSch0EU8UXm#9b&1Ah_E08))D?RIh#I^8d*k^|t
zD?|$zCp}!RqBr`>j8lf`Qd8(K5ExLc%ax@uoZk8?reuv>WsDj;wK|u8ioYqNMjW^$
z-h+{s-l#4x3B-g3YYY44W~Gz$uu@;5Ygm%6qrxc}qjmR`;AeUwt4-?ImO=?7^!Th9
zMW7_bp7buln+b}7iuod+<`o8OL@hW`lZwrgI_vS{l90>Ltc^wiiX2<YKfM+lh1&^y
z9o8^t&^mb=46YURR%}NPtMq{o$1H_aH9WeMXnI4VW-a84g?b!EJFY+8b)Q>QLTN26
zF=3U9XVD-n>wHEx8e|A^SVdJNGDie8V%xB*MN`LJ=nQkZV_|gH>Le@8^sm*@iD_GO
zxFZ2^;}(dB&${dYLSZq4YT6Ff4DtQyIF5XCJF<5rYb!Cfg$-@T*sT_fZ6{{y3bjQ*
zRksuCjE=G8FBR@pX5UZJat$=l3*W}@)9uG!MPZ3uykviUmBf#Wf=Ei1;|h!KFMZ>c
zX5eOIaRAzq!Q7V_hKHD<R5TSxPc4)64>VAPv>k%Bvn}o{sQ><!{;bT|&h$O4{2(9*
z@@<@-oIuuie`0FybAbkA2_wC|2Yd*L!CDwW&Ec4@c|Lg&#SxTRMJTYPfKUWoC?Ae>
z=_|7#Gcjsd9mAkuXNF!RE6LYP=oidGjxkNe-mQntjecGVsk689bT)n;SBlut7sq<N
zF9JSLnwbShC_*IB=#Lu)D`sWc$kdr{u!QT~Ji>Ko^yQJehcX65L^Qqrc9)NK0gl_B
zVp+ZVG9sID#bu<vE!Vn?xF9uZF}5B~v(hQP;{?yH7%vqe3k5phUM}s}72T6Jm5IBx
zL?={}07ojKe_7IY?642t@8J?$uheE~ql_yIKe|)LhSlot)egFV16Xy?E#t?Wk{{Y*
z^%J@6t#@2DT>Bc&Mt$vzosAJ9xb|7p?8o9?{WK|D?+$2*CJndifz|2^!ISV|?2X=1
z4x)ubI_Q9DBud*KpC)x=6jsor2vMRu=T8a>o3wSDng5*R3rYr`O|lVXqr(VE^qV3s
z73Yu*5=x<BAL^MVAi1My?geL(uVDD33BrJ}1#0{us@TY^AqHU}A8aT#%fLvDJh`*C
zDSCc`n>CnZK?o*jkdmi+*0CJMMTLIBnG9h>=CPgvMQDRaFTLbj?C{Z+hs$O~Bp%P2
zEMsm`n;I|o6Im&XWsTTvX?;q`c!l#3`}#=UtNbU+8Auz~B8Igp{ms%k9tKvoq3LlQ
zxP+sv;Vcz^S4A<}@VWtP(zA~(<iKY8M7fcv;kht5tF37CB`>Ow=P5;?!U5DAvN;~k
zTEOWhDP03bzFpQ{9duzj&igKMOJIgX0;9%cwzna$8YYso<~pBOjLK+XstADWdpZZh
z1#Ci==n6!ZROz#52G$YgurRb7GF>74*fw`Cu%b1iHDu(R?KaU0i4)!yV#R3#!ileo
zFfuh+W61b4xJc5RInPH+!Z~ZBg!^yr{o-l>ChfvipI0C4kAWaVdf(KL<g*MG)j)9U
zlB7=4F@?BrX+tGDsLdG)(47E8bBd@pE(>H<)ah=hf7381)FWXUx{_fa$9!r7Wz?gE
zt%Xo)Y*`9V)P<fk1_t}EG)x6D^YDU)@gAuc^|S`{S~pbMd#6~PBBZ@<qE^S4U`c)_
zXO}_S|6PP^B@#8s@g^Mk%&f{}K>^Kr74J&#n;&-3`viIDwsyg<!hd}~k*4}RHX7ld
zHet*cGBElO9h8U;TJ3G3V_rv4?GN1=@CNO5zlk=$swTkk=UT0pt6A0N#X--b=4JJ3
zM1u!*1y&3=24*3!ga$*l#Q9QqSw%W70nK_aJEdwFXQTAO{5Ks#GINTFRYw-5>4HcO
zFfKR|@L@Pbiy>%6Q`PUXSynnSDjm>`6~lMuhVoC%J+m~|QJ}85p#T`%%T#!~pa@^{
z!ffmGWRcMsF<O=-kPx%cWqZ#m*zChmF~6|3K{%Lg>C!=l-eaR^Cq`94g43<0Bs>!m
z?tO%|T=EQVdy)JMCj2ETB={YL9i#VyWM0lrA%=<p<npV#!9b|cki4b%63VU2s$Rq3
z$jP_M<f!@3-AC=T067e8hX4bdPBq+S^wv)NCj4C!cdTWaEb0`UUYjO>Jpkw8XN2+y
zBTWH(&?dzx`z#D6IXcWmra_dE+M(|tTK(#-DR5@iA)8k}{$#f8zcxP`XD4P58%Hnq
zf9S!7L2@=o-VTzBe)3Q7n}qc`diMPJ%kQDc=4Z>9sgdCG#ybOY+itZ~fok;tWz<lZ
zvuS!EF*@(XZw-qDOJ~D$lQW`&i(x4Y`N<*D-|+?Qw8HGPYF^C7`Ro)MImzE<<1Ngf
zmsHcjocAzmYRB*bqeTIZnMcbh1)M00@p81N2JmZ^U}!($1PuRF76ur}xHzA|FWT`2
z`UcqNXN6$}X3OcZ+3{wS2o2fl22dD8wH{H`!FhNH|2?w70iez;do&RO@uVS}*iE~t
zIpl?mLEmLWGe@#%?cqb+-8bL<;A3v2g?7>_sL9$QG+B+|B;yC27UOKPNUg`M%WmsW
zbjJB||Fo$cJ|(MdtgMb%9)ha|<gH!U3VK9qOtf~>v-@F|j^BV~KhQ&WO6#}4J=I3T
zcDs~SFg5}7hwVjX(NAuQ7}sODln@%4mKBhJ{HicRkB$-;sZ^LStG=%Gz@H;lFavi!
zSGAGfh^~lh^Cuf;qBcJ;`M2fzz>3!<=>zyzgGRd&pozd4?>^F%=r#4o5bW*+!aFvI
zciQE0#olP&Qh34-rN;)zLep`z5k*ADE-Vs8_0DwI_X}(|Q+ZT43d9JHvB56+s~bwS
zJcj703;Vr^(b0S7fy|j`azkCs>PQ4{ZwGRqM{~7*G#-1P;v+#?sDQ@fP`2v~+seL$
zXOi>DRta`cGrxmfn=Y`P6Gtop##B2&HuA(${S|2;;pAcztyhncyV|fjBeA;+wBPBx
zQvy#RE01X5gxus5TWil0-I}u|BSc>m+=Qnw^J;|W$;^vWFa30f#fsZr3me*rM7Ukn
zWi{N*tY!F-OO?BoQjuVk-lgJ!D#JGq_o8;?fX?nT+zke*B?{H*stGx0s8~O5SwiI9
zYGONYM{Q5u3=MRu@$Q2O<>e#|BOmQ`*|yzKa3%XtKk_0+S<6)lf#8db!u1c9_kLmy
zOeJ`*0cC=64^W`=?|&;m=~f;jX1oW__J>FNkB845{VgKPh{_8hcPX61S`sFWzTG7z
zdU8o^S6ogMqJSMrmDJ?Ai<DxEcdcQ;RoN>^lK4{hBBOXHblS>g1qIdBs7x2I^+;k3
z(pbuNK22vAlwyo-R&pUb&XRmO&BnReYvvA!<PBoCADA0iv7C&7sE)Gv0)w+d>3Lxw
z0J2+Cd792`T)V=Qm7x?fd$~JL>?Q(f@j^w?`*~?7!%0ptUg)A}E_c<^j2o{1lk9D7
zMoZCp6h@5BKx85|4vWlOuE#VJY$J!{=@>_5kYqWErjN75d6toY>0g{<>EX^hiXCI_
zsixNy1WQ}cK1-V^y!{3DmW-Dr1ox3ZHn=Vc*&Z!RoPb<a<J4c1Uw6QKL}8!mYKQ|~
zxrbLWx!8>GMNbOI9YDp*RzQrHW6XX`ZHHQxu`}C>%0%yzkuuFk2{8^9C{L8gP4g0>
zJYnpnG8-++3i35W&+ul1NQ)|o^W5P5Bm-r{qM$4W;VzIxBRvFdNPXi2rABoIsI4yC
zoFV$*!>HKX-l|+(KaL)1Uv$Iz<DQOxr9etSBxz4M5ea-kJqiW+%-)W&Y+O;61lpI}
ziyt7#=kg#~vXoPnD|?;TJy&zDF>;VbJ=_JWj}Obk$;q*X+$;G33xK(k+1Q|+!kyb0
zCl&j3KbFB$vuA#)n>IiZ$7MlVW1kdX7BEagGGGV>4f5B9PWP^Hp=n_zSj=+J%H6n;
zv`SBjDI<XpwTc|1AHeG+%<CK^g+RB@({c=Y(Pt212H;0xd5r4?5_>3|&Ny1yX<Y6<
zK>0q^eml=b3rXdX((*E@Ud-9f^JG$-qiz^Fr=-<|S-@;oEFtJxR_4VRgQ{Wg&<wSJ
zL#z1cSeRQ~@@i&kFP1Yr+6JU%tQIiD(+fa>YFWY22B#ShqDV%lbR-TmS(N$dX@->F
z5S?ji7EeIH_kjwJHLC4Z&0r;HrjXDCMwQK~l(7#s_DPn~K?+AAwQ!u#VGV$Qq$YTZ
zLW)jsQit;n3E$SZH1ucY_H?r%L~g)>LtWTqCHFF$=`UvbCl_wDTm+%hLZb){M8pNF
zu}~0@CHP+OW7?@!Z3Gebj^X%bdC5RvFA`)?4e`RvC}J^@3B`xdmDubev3R^NC~{aV
zSTUteg=B<g0Mrhj{E?1I%%lT>>wZT2%)-*@MQ&Jx<YWo>;<Q<m+ZDnG8kz~`ot#;i
zBl8T*PiK|%=VO_U5ZpzUbi?wR?RRNs=;|2bIOXoZE?6$Wp31^d)@Urx+&}HngRPa<
z*xaZyPKujR?33rc7C(Su_jmu!gJ-&7C}yk0@~oYpVrV2PauBk`DaVwQFo>K$hA{Yv
zRU23kLRgeT(1rFq!&6RUSdJ;&foC5cuSVY_TEb{#<0L>u>g(Erq~6J15)K4^yS_a(
zNq84ND7oqbHlMD@`c-l{&m?7_7ojxQ5#FGN_9q0&wGo@9edo>13*<;k;u}SEN!cj!
zY@Cep(`hpymO6|EV=uX*Ylor6$s9Z8d2EQd9F5(_)g|yU3~8sO4KnPD*Ke&@i;QP0
zUb7YLa{E<V!u!dPZDT$nfHxFxx?2w{>Un0>to>@-+2&n!Mx$RPQf&6T)1E+D>ZuZ)
zhE#wF7JR7i29r&wnSd-%Io`Wc(l>gH5|{!1#m&)5ZS2lPP+$n(9TAYj8#IW!&31a}
zXjGJAJOvAHszld6O}ga(5hYdT;IRdosa!{r)&yr+hYim&Y9pE%V?)AAD{nLG{?Y`q
z?K~yXkxRBJnqk>{J&PE(7-W4&UmB<b%uEd9!Nc@ty_!h7Wb}`i5^QUn0Dc=%hNA?{
za7Dt)aUJ&B0a!dh!H1d2=@LyGfQzsBTzM(#gTx^l_0%C7T@w)n5imBw-JFicWw1`s
za*_3{r8|>)Hv$hZm)lOrG!Op4s-bw52r7P^Fr2@trHc_>aGWN!-cf=l<ULB{^ne8M
z&1t<F^`%*z)757h>f3TPpJ4aHDt6%$ViV+uaCY&Y50Y*hO@`6P+aFVxAiQ!s2Smy?
z`P)(>Mj-*7)U)1UB8<BXbMv49T1_=>`9XUX<vTdBmq55+QgmCa{CP!&+tc(l1nPA*
z(_C|2=3qTvs4(f8!g<La7^)nGdm}*6n3*~aGI^Oz=LKhZ^(@~p?GP;zrW~S99p*K8
z)d+G(Zh3*DqF};-Y^#~jXB;^mD(AIxIzgBJ@p3j!C2QS=OO#<(xZ#)wAgPU~N`tI<
zMQO}}Xo^m=$+#CIcu9?^*g$S|!(p}JV^?2;BOAr>arYmvIZ$nP@aFItWv<rDDpTiN
z?263t2jq(_cZd3`K{A`B<2QL_Ly%tfP9BTUBW?4YpZgRJy1Vyvly7c8pp}es6g178
zbEKOOc;&bBj^sn>h}DTKqqE68P&p_<P}F_fEF3u9yh^g^aW)=f(rBFp#E|&u5<H`k
zYvhW^I#NREJN@K5bGf8-r?xmD%98d3A^sS=OwLJXiMH$GtXh=Cg+VdlLv_|Vv!H^2
z<7}Rx1#KdktZfBHv6Fc*$wwEOa4R3L)f~Tn!H%7}mlT@#tcORp>zVs1^xl?hjRHuG
z8~QZ(s&yGE$0P;kJ;_a)`J$C<QOFCRus<g?A9^sNr$5sd_L8uUV5MM`mkeiC%(h;B
z|HIZOoddUkS=ddB=lNtJic`+H0##w36(^9FMZEEH_ReD|w%2w_M%r8#cY|dQYs^Pe
zHW_Tqr4o{KV!>;0?5}5F!IEA`3^{@?%ko_dIJkJ~1zFPds|K<xj}tVX{KSXX)QQ=a
zs5xlV%h4=zZb{qB=0kqcrHxt|$DKX<+C}cFCt~yoN39i_5*;_O5F7rIh7kqb{nyRa
z*02+ly#<LT>q#cruw6~X8>8dlqMN&!nU{Ro$iG6%no6x|MUm974UZ^*wJPjE3sESA
zTh&WA3BgH<b5VxcLo<MTA9ZOq!KXRb*mE-zn;@)u4#ypLhr7PY$61n{oPd_Aus0SV
zE&6bSyqAViwN^c-Wlc{=acMe26|T!hf=rSnC%_u8ip}r_kR^IdJ}F2M!Mk)|nFDse
zb)*RBA&*x`5@>vv75JkpAf5^s0W8Q$>NrE!78)`Rm%0M$5be!srue8L^>d0;xf0@c
zUgit4&BkP+VaUO<JXT~ISPn2*7q^@0bFSym6O+b0dI|aDa&VS8&d$;|aHj)molW2%
z1ur9yuv+IAPR|%m%;+ItEGMR5a_)9tV_HVwcliLS-BLov*mB003n7e}*Ji@hA<LEJ
zx?vcOyY~y}k>X=`-=VinMp6Oq_sQON-j^1DbheHurcbxY8NN}p|0!HiLg{`tVg8k6
z`B5%f2)HNQCMiCMT$7bjSWiRHo$<Fig?P5dptRLF;IX_?#_Q^64n<dsusg11T{z4+
z4uW!o$zkRJU%Ku}w@Y)^jWGx-VHeMjF$sr<mk9cQrl>f)uysx388Dz)-P=j}Z1xbN
zLu8BUTz(OhKsh2w4=Jvj3?@k3O-^Mt)Y{=U_k7=&ydoCLT#8`k5YjDU*2BBITT$FS
zXeLRvYVx>yVsU8hP+<2aVo|F;tvsV?G}Vz1Yw>^MtjbF>C^S&M*Ice{$>~e|>H|he
zyBe@Ei1jqSYRGW!ES8JrQ5ddeLa9_K5&E07%+uMT`?w$0$9+3;I!DYaSH{Zm+x^#-
zLU4LA#)0B6nvUbx2^Jp5(L+(CUlR3z3!%woa*xqQ5EKA4W2HXYrip<o@!oKfW6JO3
zc$rVeWdASRwAqOzB+E@sOR&A16*Dj@OG^;@R}!^lS;4(LpMh~)<(s6D0v0a!wcO&N
zl~jXi6&WY8+K<ZyK00J3Shobe7LkyHGmKQW=GvYsv~RO*cxfLcO#{9s-2?V|Wc|a9
zRwgJ|Wrq|7f^6^Gr2Q0mvISLWtI`ykQd^*~jc;^}jXjequ+=)CVe@HW5NwhjXON?a
zO_!a#yGcXPlpo?UgR-h%?o>E?=R^@<+8baL?b{nxY&{x-^xK;W)`dIIrUuo$$R1|%
z0^iDVwS6VWDl|L3Tbd;u-xVu$woq@fo4mApO7xzUvUH*>PAIJ()aA*qpPSu^RGdyC
zNNNWiDP%(0L9ap|6C{Nsp%5_whe|bU74a+}P&k43S6+r{kSkbUs44{;^u$a|s3cyb
z9D62Vy!M=2%Xb5@@9m{K3{uz40MJF{k5p~8`%~5UCNp1@E5c68^sG!lrIlvAs*aOw
zS~CZf@OMK%9(d@RQ7lR&7W7sSUnAOp2D*pn0>yPS^(KqMsI=bp(_n1q3pYVG{W_l~
z7(h&GzGY?{3oq=NUbl6-r$jfVv^gYB3P5&wrkj8YMQcm_NR9s#%w0Ys(;YR)co*Di
zWOB;pLr?L8*m15QvVRv)%~f=R0y?PRIZasxnE*UB^om^kNaA?1oQ*_;xik=7EOX!^
zQ0hq@8ir7uF4NMmx>;7S?A0_hE6`_B;#JMFl1T(GWn|^A`w;?fog-8B8K@SZvjJuc
zDx;4=RZJmTA`i53o(%V(2G-h2j-XIMP82eUO4OwbGu#R%0`4jm__LUz<ZogYW6A!H
zhE1BAZ8k5#Efjm?;b#+8sG9b0G$Y^zNi^!paT^O|J|dz0fMG=<g3dC9x&M_3sHI9n
z3lii}(=mP`i$8A3Z1&lryx8U`!uA_2CcW$E(Qti`Y;SM($=TIH9>R-dC*YkXA>)(3
zf|nn+SS`vNV_jGCEPYLv7gD~Z$@SR!nO??3u+AkeQo=bUxE})ta+%G|w80>82HbKn
zGwe4;KRjNV*@~a2&fSfasdk#+Iirt~Fy=jeXXP5wN8HE1OoryLfjcHARIH`rmTZYy
zm=QgZwT=cha#W0k$sgT?o9AU(Kv_N<o#?`aQ$#YvIcVEL_=iEBY~orQ+{x#_CY&)A
zv&CYcq`u<ee82R;OI=5qD18nI;3#n%rK`k|HDnouaw-=k=BAV=pZtsSy|>pS80cfS
z8Ss0X{;5g<h2qTFs=3t`1Fum{$cC%X;Uo`E@C}9cXa2ghV0|mvZ?5DCR3f#=5V6Rm
zumR@s7*kOvg`wZe**P2rW7d^HOjDh%BCa!`{nwh|T1}-n714^Zu7U452gf(LrwUMe
zk`hRV@%4f2AhsoO=K1CqM_KDI2vb<)-N9dGN1J`EqSZPHAx7%Q3%Qi6qNvQo4hzwe
zAb~#>N>TgdhzK$&U4ZV5Tsbtp$hJ6c0BjZ~CZbRW<XI_Z7VPps*9}{8lQg<$4{u&%
zcyPHmeR9bV*+d$8MR<}4kA$t`b|LgER#@npZIR9%X@QR>9v^%Ijf*e$6%)$X@Z(jO
zygx!+GL1&zY6F$79fRu->RM%c<_0O8ZNXZCVo&C8IEoQxRMDtpkKNU$F2mpmY<p{N
zmSj)t1)A>MyKgnDHLcjFig>>1Tf{i;&OIvu5g{DgNY-e{4@U?8*mt2DWBXfUg-XD&
z7Cmf}NkDI-BAsN_s2-s(CW3G|b?0oz0juTSL4zok@L#dH2ERnTJ0ZEQPOp_hLy0^1
z$mL$0T$=l%PyECGDmOSi5~lMn%M0|IVE~Bxy~8MJ{g!IUmsyxISxH#i<rCc6E!!5k
z8a6ahv#XC+cd%?~CvM#pZEe$;MbvDiM5f(n7}$O(hKB2X?gb6YV!2)d%R!0Qz|536
zOyDwTb{4WxZlCONR99MAJXQ?9Ms>+!Cj(=ZMOM0q0#S#EOU&w)n+gPfe4wVICI0Qp
z-Mw*#0WGaJX*AdBBhWSSB?`bTCxO;DM{sPQR=Mx3C&xc<>xsci=oF`=7BeO+mQhS!
zViV9agag~CfG=uiD9N7EvuX|fs)5UGaOUGKnhGJF-cqX!UmP~e7iT-Z0>i4>E_F#f
z)~|a@t))%hWf|)z9h3+cbwc>i3URGJMk~$R)Ck77v{SFzMC+)Arb<lc)}nFsRvKvA
z{OYvL^tBf>C^XHr9?&!a?$!fUAx7=yWj4-!J9;N`WM43pa;{I<bSqb&U6H|DiAW6;
z?4zVf1OHPXPEOL?LXzf<Bx(K$lO*Qy*t=s7UpPk3ae11h;}1Ne2EQ(W2I5gScGp!T
z(;kmw6&ws0RTaC-5>BW7QaU;LdOsBWZ+1~!*&X08Xbq1Isjc=M)uzlxAO`EEO42L@
zHu3PGXuyQx<H9<vOPj{0rs*+FqcBC(Jv_y_swc2|WrS+H_9!!}W({<0^+_kK?k;34
zILXo(<^~FE%T{NoYBYn}^H56dJH<_@i&TyJp+=aD^g3AQQ3j+~T1ahsN2T0WWCUq7
zxVzd$@G-i+e>GL~a9R_FUU2F<Gm1!RY5x>pY1T`!ytjm3BI~h6Y&1#pDcU7r7APnJ
z#`PE$g$>(NGCnf(%vo`c7W-gqFLr3h8R##_Mqg>+K_ce*q0n4KMot6Jr|pt4&@^sC
za;|$AU+xmhUOgBHRt91CvuoPGvNONOS)Z=4A&!_&aZXuvW<FrBY1fx02QOa!o&8?I
z%|J1w?XN$@9eb!Uvc%8Q@-@drODj)1?73i?2{|U&8c{Xzm!2yQmyYb)))TeiARZ`h
zoONn@M{y#qIl@g<QNq2}xHjVv7DDrDhXU_$3Y|Nxp|eZOP?wNdl5>NDl-LR1(Dz0E
z&Qhn(qHH)o?wsNtI)2`{HVISD+HV)>v2l}kX5nKnT3NLQtB*@)E`3F~yktyzRyPR_
z`qFTd@Q)Bn8hjzI7d@s^LlL(XK2I>T^tr_i$QwUfRyxe9uyz6g%y~(B+CPCG_~*0!
z>;lv1R`j`A7xg!}Hw4<E1Dr$gIz>UmxKM#9K{pE5F2H{@6+z2hIe!rd=X|&-&)g(E
zHa8~;o}6>!Rj}J&c-hD!U?P#`3o&otIF^`#&02#~H=Yx_H%vQ1ilaS|V6I>d0pX5G
zdq9E5PnHw3whuk_3^sz<3tN<n@(bc%!fmpeFU!1GR-ghai%N=laI!H~T>NAIY>I}G
z?2^_2`M@%tf@WfshgkF)<@<(NW}44M&SUTF*Rylv-b3bjRR;4qt9mIH5v~Mt9a;mh
z9uE@2j_~k&)2^CsqjnE#jI9&a_V|`gG^?Q{d*n^i3hx1}oCiziID1x;Pq8y7Wq>@i
z08tFH${1rgC5Qxz+pu;C>zkcvwiE=Qwa9OHN%Tem5@T4fDl>bC!u31%`^35mYrj+2
zFJKCU%|LZ@PaOaiKmq{U$+SGlws7CTq1(zdw?{kxpfM=iDY4R@S|Jz5=GRL@E6|Iq
zVaQ2Gn3_$`_ulKM-;XVO=TQ%2qFcP;?{U;F+<-I`WtfzSgYviFAcBng3pk<e#^s>~
zc9`84dUK-*p0`v-TGFwWf~Q-%zMS;kWzcLOkf`yd1}EM(sx={+=Biiux$&P~jV4}W
zZf>W)^1K2_ezfyS3x__bvn#p<{<oZ0jjMpW%n94P)y8k7mo@$aA9st=bou*v_lcc%
z|3T;6N(S|R<(U^UEWhd5rzuX2o|~4zoC~~HgBc~aG(87k2zdf^iEdP!&D#%IsXY)S
zX46Tlq_IsWEvR8oYNZ7k${@C`Lb>|MsON{Ka@@DGh8-mCNOZ&YMxw?)GyBY3%9(4p
zaJoS^jV^;mv%JXoafqbJ9U^58%ywa`+j?xZxasnx;qSe;N2W60Ei-5j{V{D-F9yNd
zpjegiPJTzB<klOG_+P;hOdQ25e#R7lEa!S!6t78u0v^~Pf+@u*i_pGx^<XG_)^Np|
zf-0agi)DH`ERyVPUM=|Otw@~$0RpOdI?7xK+HG=R37p+%+~e%6i14`Y7KNtOWs)7Q
z$nz+Fkez0Wuc>EpefwtXzb2QDh~KOREC7?Lq1T^JKkBO20WWO~>XK9Ez_SLok-WrB
zK&g32yGxit?`If#S1WyIV8%|i@{1{wfE<OC&<4C9FmYj9=A>wMCx~VrT=B0B!rn?n
z0khg<e<u4nYOYTo-qlfC30LDw+Bak$9D2zMFP-TSVCYqb8)2Gq8vAN_a*~hGrHex&
z*qxD3A~|#=D`uJXdb6`<VJx*%7}ho|M`u`c3QAAN!Xpgnd)(cdAqmB>D9MX#>jZoZ
zm6HWXV6z!2SrxDhhH;qa{&o<X!=stJ`e$rZzAn97i#SH8>w6_hao3Z!MF~QN*xq1y
zbYr|spbJ*7Woz4{zG8Is$j3)Y3x{TH&{%+QrMXRUz!sMV5WH~^`OM$uq4%9x7A>s7
z8O6>v40Vr?%_El%T|CSZ7+VLO8VP8y582ca5u3KK-KU=SGQ{NNiuuBonVX5TX`nS$
zGsB~x>u?>-IP4ynJCtOQK*|zBXds>;CO>^mrz85!&e_NJxL8UE#<KzsF$}s0p;uXH
zfw{~;B{ksGd2Ut!p-FL`l`ub;6NEHAW}AmGT2L6cLH@8)OquG!H9{^`Pw8W)1;^;7
z3n93!9gJNmd-T-K9zXCPhu>muIHK<&(blH8v-2hqUv$-y4-G9AlR6%;&lW=L1-F`6
z$Z&W2@{LARcXYHnviHMq9}9Gt8Up22EJZV^<nl}_7jTmx!#j^!4iZWJj2@K3bTnE{
zk1rOGZ}eZ<5#HS7p=*!25@A&kSm0b{*z?#0zlnHZQX{TQf5F3{z*6A5d~CK1mSuVz
zx(hlw+>NDhE=4>+z;OjSP29J6VxYFQae-*clT%ulasH;yzQ+iy(9h9h>YcByP&Pt8
zP2T&O?5pkckTZyqqi+nY5aSz;Dcl9JZFCc{(>OeraHLqtqrsNaK7amfTi}8m4##M{
zFytP90#Vb7bJYLli;6^TT$ROf0}}iIPe8E0S!V`w-gs{AJ)mZ@3nFw}^%z*8H>T~M
ziTEMy)q@cGFcuOTdK<=`&fOup0eRiga#%O91>3-Xfy?ZuQtq<5bI21WQpkI9CI$SZ
z_5`AxSzeo^6<m~mUP_LQzVC65i-TW@3ZTz&!#Z0FXGz$OT|1K*=Sez&LkEwCDlw-z
z9yGKxe7m^Wi%dMZmE?Ue%tsV{*W{P4d2zxnu%FKNf=`%#yIP=eHFdTKz63SI7dO@z
z43ByHcw5%jzw<>gpl;YFA6}fK+g4Pz06KnAxOSER3=)3<HP6aQ5fGnI7$-5E&c?qQ
z-c1a>)typ(5b}$)aWM>HgUBxyk_R^Rme+1y1dji`{|qA!Q~crKf15*p;FsK6-S#bS
zkf)dEL0Qth$9edp!h7|bbV6)_S;5cplW}(9s{Z%>%gxW0GkC{@mwN)&>Lqe?o1ak_
z=8dKTuC>FSVM!ZG^*29*h>|BU1zoMM#wH!xL85?xL5^p8U!0s62oeYI2jB?IS?dJ~
zo0sOxN$>jopEG*%@Luwm`V;6T|Mlw<EB;rN!2jmSbvhTXlfjhv+#g_p2mkfi+kZkh
z6zR4{4{(g^LJ0Qj*OPn6_3Apyg~@=(yv7IoMkM^Yj?+MG<4*g~%hbF`O<ycG7Hj#z
zriYKMXs{=E(2NMx5Yv`G>hE=^3dW8=Ru6w>Wm%Li;M}IoDx{7hZI2tFJ*U=NW<5G(
z%jbA=3u2Z(@S4$q%LRLH0wMtSW?w{TLhkM+RNin`5t1NGfPjO|r=5RMee>CEXO#ge
zO!`em(ha#65!B)5hKxHh49hya+pGt-c@OZoMeBF#<A4RD?!ITJyi+J@?$Cy%F?<vb
z4=wfJO^=S@mDFJzp3VVrsC|#^-C7|_!PSnf73#KQwU1Qiv%iT(U|)_F*hFe;o~jN8
z-@e%2r{i=}?&t9{wF9BP(o!@oM8gM1r0dN#(&&2gjzp`K{)OVaL8kTiYU^|Q60*!7
zUM29n%EH^yZMZQ8@7>7M2MWt%XzdY9c=+tm-yT1Gbad4BM5IvW4oOFCc%e>$(T=Fd
zR~27ZYQ@FVP^Z!o$~h2pcwDq6tE@{jg5X$Dp|O+tklz(XHd-l`u%WL2f_LCY&uV_w
z(SxilDrlCru96h6pg=d!K~BK(G-B|aoS^GvtERnux7q!AN%y|k-Lm$D?~u{dBQ3*=
zh_tTd70KGQzgClki?&W&uEoA!?#dR(3RGa$6(WNsnSp_=JBFKN2RgeZ-@9sE;u!lP
zFoBZ;ng(H-i=qZcwI!i{dGz$V7YBz=3~~AM{_xrUv*D}5C;Q(X9PU4%sxD-iyXw6+
zSqb*AmaOCw18;<cbBSt{7X)Hy79WJvj}oN5FX|VNI(y)QQfOREhjdMg62T50xNLBR
z-6NK}<Mi?4;nBg<{lmxm!-K<vmp#EZvR?>-Tap;6je4#X-kQ;g1ii)aBu6z1VULxF
zmtGWu$HjEML^FfsEH~Wv6r50U2nc$-1l0*&XF1=VOX>tRN1L2+FI`ZmP>3F%L#V<+
zykN~ZAb39W%GvhmcJkv+vb~*r^+n%Q{pv0?0k?p=$)EFY#I@2KJzw43LT=eMq9P@f
z7&c>eI-Hyi&3c|@!(W#+C>D(vNvY|R?c6YZGVBBM`|+d4-|s&)IIf9-hZb<bP&}eJ
zgeZKot!a01q~=be&4j~npJJN33R$hqA*&bVGnf4us&7^uHvv{e4xFyu+ysUQ%gnph
z*&CO*YS!F1xy9jKG+?5pN_6SwJ&o9JZ9EK9PC*rI6pG+9Bxusf=YN+|v9W~nWu!=~
zw;otJ5f7ybBm2oehTF>d$8t+aZLIOGcKpH#M8xGm?c7C5kd130F}q_%<gCZeKHTYD
zJ5ZxyOBKBQN~iOCJC`~baCiY36h5m^5Kho7@d2^P*^)W_z74EltpEh-Jom6d-@5Ht
z{1}}TMI~owMWgPk`FgPAs@z#rFRXfRxJ2n{4R;w!>+r;;)!U1<O+;{-zQq8?&22~M
zsNXh&Se>V6q5e=3Wigsa;}jV-Vi6lO;CdjFq35#+m{82d6LV(O<5uIr2(%zC#iGd+
zCZ|kuO+1}}R<O~8Xubwl<jKYPm+c{fgfUKumkpZ$A3G6mrTL`sx_J2zCOiNeXlRX1
z;os{yEY0xo{((7|zvJ8cC$M)c{F7jW_5;qvLOpp<X9mo(Yi28v$Y58q+>GoVz4qG6
z>H{$#A1M4xkdvX2KLHR5hMVG#o*sO6_-z01rClM-`mw)&TQuM#(a$3+<S<h$sv$YH
zt6T1XvfB!V>q>9A!dr|DTMiq9Xi5<m-l=K7p?iRXp4)!@UdCdCEGz}8fxOupVbhDL
zQFlVqW9UO>6gz$}Hqp6@$*Q1?rOES0?0KAr_Ea(%536O6F;Mp%^W~N$gl*k(FlNUU
z+rgW1|K_HnJipy-?{tK(5@Rpq3*NHaU%VqZo3tLQl-Lcc)~In?;fgr##iX7Ru`!;9
z&4PG^yoxG>m~C*=8ew{FGzmtZ=M?T}tV6L3?aUh8N_}USFizmrD4murwiBdoccYq^
z`hVj!sq>O-#$;^49j5iRvc_d))m^1oe0IfK%DhXwy0th&Yh?+|;Zw*nGB*^z69AAM
zlat)HVFT4m=Gr*j+zn~#260Ny9v%IeT?qTuPvi)eeNXX267Wno>}sBk@)R>sCG$x-
z$}D^BOZj7rz5<N5y;vF}G@Mfbh-^m7kK=b36q!3v(5!unb`%M&KQF5MeQ^%yf(B09
zHcsXR>Lt;>``Fwx7GNWdq8Jh8QF=DX&vwkdz-%9lW{Ts25*i(&o%k%DFDEv<je%vE
zn_m4ElyL5Yz`9Mo3Hz&ATm@flB>$hothyMUcWlEViuhemtadvKr>8a-OW=_fpm4&W
zja(-*inuGPnpoo%mAkWzyA)=s;=H+{8>bqeoLzOYBSZ%TMCL!!Vo^&CZx*RqBz1ML
z)dn|Q8o3zrBztwO@v2OQpvtdb6T>?N<ha2!sg&zRL<N-w5e}PpG9W`wAH4wFIXD!>
zaZ4@4*Valc>gdiFBObhq0Z}(;2$K>9Uf(gzxAom`(gFKy8|@M*74?!=D&P+WW*&3Y
z33;4XRWX9BZPo)LhI>uHd_x5~iSHoVnr+uP&6w|ASafy`v1;tihV(slzxvo7@!IlI
zpWrv(m6=>pZr81cm<q)iwKv;Yv^${M(+L8co)i<x^OhU~WA!-%8=Q@EI1kc^mwC_T
zP^K_!)C2&dm|>2d*#finpl=qMWwVD}4zW@o*Ql9MW#f`>a+LA~na$6N*%tU{kvp!q
zCArYqfg=v=@!g0gRhP{2aqse4Di%p|BMa;Km%E(n5yY&!1-+j@G;x%5vHAcg8Y^2;
z#(Y4J;trQGGOG(^QSZq;cP-c47T<clmuzjd1VIRVwrcCTdiB^__ZfKDVU9{txSr^R
zq)3mmEg&ib*iL13l9d>FSOUould-vM-G=d)<5_Q}?Ieh^h~_rjF!)ZEDfnD+LSK+z
zmj%T~Hk<l&md!B)>4F$fj8japXyQFeG#VX+P|w7f`{>x5)hrzkA&zBSR-tivu-U$q
z``W8CIXcf5qcfiEwnt53Zf}kX90w6Dz!fdnTOrA32dQZ0JB3y*WP?*OxAa*CF&yG_
zhV+HQd^*2;;H#QJr=zII5FpkrDg$V?%z*o}pRdXC1|$L*YIDGdX2G*wYd*}P%;FW8
zBR4oOG~jl&g4i!)XIU<TY{oi<70WVLiBJ_V`Vy+evH_(FfUvj13a{JQj*47b8MvOV
zG(kc;OBW)vZs+TnI`;iIEMSi}Rt9;@xOQ}-ZDixL%+=?0I(pnnEp-?3$Fs}!hT)0n
z?Zq~S+%9%~<G5WPTkpZSQp@zL=0|HNmH@4rTCocu@7sM-Gc&P@vV`Df^kBN+3}K^_
zx(sbnQ_zk;meh2ZR!0gXa&<+!J{4jSQiak_kI{|ax#HS(;nQ?~Q6%VM1l{R~^3fm~
zsVTsO<^XHAB`5)$8MKziE{%O3Wp<F8vP_dIo#hL2**VKFGr8y?bQrW~BWec<(liA~
z>!Sd}T*|?6^Wg9=kDeYpVOcWIL|q^8cS{tp*t$2bwYuBW=Gj{*zh2i(1;q+2z7eF1
zSmC~ag;u$#ko2y_aUrTeXgMKS$d7GBd`PU+e@$<ZwIR|7kA@F%ZxL#?leY*p-{#Oa
zRsz%R%xJfBtDV=p3<J1QV9-G=t~YFi$BikuLu~u^zmtegP$6l@oK7}Ir&z%%n*zR&
zTod?(u}joW2@;7n#Hd<NXBj9ZHJWg6t5%)lmeNVKDjPMc)j`*Yyq9lAxW2q1dcG0p
z-!tzPVa?(@ElSkec(N|`XC01X&jY)ofN1@RTtZN^`tXW4C+mS#XJtNn{T>;XFp9ME
zqWe!JYo6jm1qrsYts{h?(?Ej#KSq?Y(T~|Lz=oCskAcEUrIgjBh?UltMRqXnkx;sx
z-3`6j^y#W^z~5}xVOosPNdb|gl&xfSl_f}>sIH=^Jb)&<3>kGmSKEs!%KA}kRsN$)
zRUDlGW+p~wPRru_15z10Ma{;ag>TZgp_|p!=@7S2k5R2QRzn&U)J4(K)<Up9Sf3m6
zHm!JRao%H8xQ%VxxV6X%Mw8+g&^%4CK9|tkwM3c@hnu2FbEIdR2=BBo!!!gjn+zdn
z=N?4<68O|u!?qk=K?ghmw<@aHhYuXC_?9f7E1VgiNk&9Rt%07qK%tWh0y%`OpHFfX
zo6%_+iKznJ;538%tR2rva!12=5~G*uE5|yk8jMJGJQmvEI}E~YhDiEMF9<l2FX6uJ
zCV^%t{A3(#_vKc(u@Uq4t1-!(Fxs<q_dp-}s^af&*jZV_@-0o)G5MD(!U$EzC})wo
zQZN-DkqD;(gI4S`V$rHrL3zM64yxH42=3_4#?M7?`-gdlIAfU#48td;PeqxZ=Ah7a
zNc4HdRWYjq%t{i6nF<%My)cOA3>R{dm1bGMCICGTy^xs(Um=9-<UGBoz$VKd!N97`
z4aKy5DqAfwvdYZxW*Fqn+G^F@F1fDtQ-tx6!zCJ?>DX{|B>aljUva}J3Z|lQN-E6(
z9_-fOj8bB4Q2Vz;zwXSu?HK?|B>`Qy4mR%{BG?RF-3aBWn#5cc!3!2c`^uRzX4<sL
zf!)ru*ak)1In;g#DO!hlK#2gsA;IBOQ<Vlb%P~;xyeQFd&Mg`>aBj<T2tVx<Y_P_x
ztoa{lUXyt?hOic+!6ruqBy@pq+2^47T_ovfu}mR@9EK<db_MVhGJw1>B$u3#sP?E>
zPR5CC0fHp~0&$8FN=7#aFq@;V*bKtV+Ds*u0@EZ6z?Nvh1~_Zlz?&FV?isQ;I)mAq
z13jnmKvvOG%#d$0(u?FAM^$DxhiSSn+eyXXy-YS5$fMCEqohx^nC?c?&gfBKhMDH%
zGMzJt)b10TI%$1dDvIdrS~H9EyASMEOHP)nC1r^)G?0*`A5!(0NU8g6X}#;x)rC;%
zWL@XRwAPhr<(;X+rK$6+sq?jIjeAqqixbAExP;?{@}hCv!o78*LDG_wh^>ylw?a@~
z`yP#s*QmN^ls>g;FboO}ePjR2=`^~;q!hQdhOk+<-|cwBudUCj?GFpyM&LGlaC(q>
zv(1qWTl64?T}bSG(j2q^fwbl-j48Q_atv|p3v+D(IFE|?1*#8;7XSuw6K^=kL!Mv&
z;*$h~0%&n%>L34{jTQr&8s580A*0BN<#P<UzP+g&j~iqTfw*BEA%a%KHwqGB@(L@G
zlclin8r>z|HfWWzkHKPM#*rJz^MZ3?oB7EN)Ktg~DR3?!A<<ah02_;$@^RZ-&nP#X
z$Ng{`9Jx}(vx<s;VuEkC%V;^2Fh>F3vcxg3DD;x>yrVjWVp>TX<ZyP8nPQGq_G84<
zTO0ekJ1p`2?Gc>T9;4xPE<KXX#t8hiM1gMM(NZ<}7W0nKHGtF!Xi+INfd~RP@r)$B
zd&5D1qYSZtx@p&3M71gLhQfVgW>_a5vtmeb#@8eu_(RhUw5eziK*q3~C3ug`Fe<vE
zfZZux7Rfy0EQM<g(1_`rUTy1)hg1l;`+xsim~(o0|3UKoEA$~!9vdF!@$&kmbZ`J>
zD>&j43>`SrJ4@?%VdtUV%K(C)8J3R3e9m0^P2Nm#+N*B3k8rijDoRRmcV0=BbwzC<
zaizI8PO=jv&a2XCV56@F9k{=+p}sbgvBrxLUjL_UiOFjgOQZ|lhQx4!4xSG~FU5cV
zTdIY!HzLW`qIZlw+9mT9h}T;tWe~>o=k#0@swO~z4ef&FyeX+_4LI=@v|z37juzd*
z`OLbbg$F~^FaamlJr<j^svVBW28P}N%M02PAvdivk4}8A?x$hBYq<kV(s!0k=2^Lp
zr>&-B%F%9xr!9v7(9W_unu^FJdkxphP3M9*@Lek{)d7l=8j=(G1q#`*RL;g-*|8Yk
zO7z4ZPk!g*jVS4%;(my3l%$D|I6jva0K6LY#kj<Q(|ws*!VM~rD&pP(#jZf~PJniI
zRUQ%CBV<rYx0N<icjW|et?5vQ2svma*pF^EW~8jwt2naaRES2XlZI|ZhCvrX3W9ZW
zeLBgIr_W}^@{|*LoP&l51(TQ;Wz)JK)9nX5S!Psnl(Hp`;jGfpYn(Gr(aac9Dq<c-
zmT%77h0C5-5mr1P0~!k64iwQ`9H%W0uEWS`2-(H9Ao_1ZAa6q(n4=D6fQhc;N;d|N
zy5kEf%loU>zwHqUD@_G@&PaiBcfO5SDcYZSPv6r*t?m&+rEQ9ykxfVQ3;Mt;wECJ?
zzt}w-;!%UI{l7Ln$L3Pi=M{*eqvwqsQbHH2!}oU{SC!%Qv(a!NCQ<TYRSHxj53YK`
zK@&R1ZE~2m^-!8w%c`k17saC6aaAi5xvDA1qdTCkLGn0gP17kU2JDlqJ<c&#>ssMV
zgV`spxk~{;-Bv21j0}wl3$}-#MKc`BNeX{qfq_EOY^9MNSM8^ZVw#VJu=$3+_IgZe
zZ}x9QTkF@BAl@mg2ltS8B^=D*ZaZIs^awoImq_-3nJR&DM{XLLiu;{npz@yIO{@W)
z>Uo(jvJP_{BwV8JH{E3^gk79wS6+vDQ}soNbf;IqeO?WIEV}~ysyl+c>R>&<*9Y#P
zt{QCsdEHAjtxP{Cj31)0g%?{?Q3IPc@8*m`f>*>jQNV}MSz0DHZ<<|dm@65qhXm@q
z8>Yv6-JFqM>A=09MDMr)wp-9s(BS0F&?e_rMxZbjMK^{sa~-XI3<wK2$N0l^q}O|m
z4ccKV`V8;3;T@FV3D^;uTMfOzkEB9V-|$6fd0ShM6;%4SUYGSuE!)v;kk~mSIAnjM
zWDcc9nb?u>h}JnBe6ar~09?jGV}%jn6V;tMzb*Uyn7$`dPw*#SC-vIMR@nk)?4bNF
zoisBNiG_--!{dA;WB|b)w7&Ilnqu;4xA{m5Nw4=@IYTqkd=@DwA5_^H9R^`ZdacV%
z)A5_Uij?FJsw~#6_fD@>xv3e>I9;H&JvuS}d#t8a|2l$N|K><*U$qg@9JgtfeONeD
zi~+=b)wRa0wZ;v#d=+@Tp8q{MKiCj0(}T~E*+pyVkCC~>>x^}s&04~&*#s~BS*5aN
ziDvyLotQGJd$%R_{A%Ns)yA=EI@e`WMV<B1y=L`)OGPfhI+Yq2&gx(|TM33*i9Nqs
z0~l&0{Ay;TXB5P%rKT&iw{B3waPDNs5+>z!j?1BIa84n9y14{z9LJzoPfimQ9Y>3f
zV?{O6#FE?i#Z`9Hy2pH=+~n&$$=Fqom_;@04qxttB`3@3EUTBL*J7%oC&NhWO@z=^
z6{83rK{<L^pKZOQ+X|h7_n?4Vj_}PU=2`JLl-dLW3C_}qfBn+8E|+#gTKiEd{Z!BP
zZF|gTqb@yiPwBY5qd~IU+?C-E*66BQ9(U`?yScJnqnZuD{ypd}3Otz7ueKk*KJ<}n
zNk_IZnOGLOfjG|0b8~kF6anwXPFIik3=_ebkQwF}K_<$th-zW{uKaXaEUU={aFqFM
za<N&nN2O3i$vK4*8cnivmMrHOuPCjuq1cZ;fBvk}A%N;ue_~tVz}Z(V;M7@|XU&~$
z1aDZK;zVlp`!)>o|9$o7@a3y#F4VU@KV6RN7_Uqb<DrFHNzWcOwK|%6CVNXif=~N&
zl5p62DjKww6xV~z(?XFuj*6+HP<ZDjcW1RPcYor9HPDSmOCauaPai#l<U0PkK1lw0
z@bYl~=xF%j5h(v&>>oaA)>YKkmCZ4vZAQ19VD=V>R-Ndx1OUnSrKX;%#;)9IL=mZV
zB_z@Bm2Ar$%4|9fawFo@CU6!`z`t`{Y5a|d-W|`IAaJH2aMlh1e;dS4WjJYnnIJh6
zRA4RJFB7qxi6q3~qAn8-IdLEsdljNdu&i+dn1zVX26zySk_z-nup`GwL5Z#mjV`ka
z9Ykmcmf3hY$||RILPcDDRuqsYD}h~7EwVY9oNzWR3d&1X$W(8+$S3))WJi%s3o!H<
zXJFi)PJqIW*pkgm5P~YAcgS+VA=Nh7fWKOxKLLbQE3<iK?rX=U(Z$7fl6-5JLkwJE
zs^t^7IUAO5VKzG${Us^2P@j3=FXd;^GSJl_QIbEwe3E;c$_&D+bBMKj%6Z-s<dgY;
zr%XX|5nw|~?6IbA{y8a&E?7DTy`coww6CMGy%w>*Qa-?)8+2_pbF?L@EafBG=?U{>
zP&Fp~hoggk>@%Cbsol~rl%$?GMtDQ#Il}@HWXgI0mIsyQhMmul#|ugk_=c=U8;zt~
zD9U~i7iDXyH68|<O=yzU_hSnQ)S$oejG+j}i??#lAQ6$Y>Gcz52)gq4vBOvb5kiO6
zuNXNn0vG;5L9=0ax>~$SWAT-}i0r6yKe|FY+_;X09KC9R9h&XLtu%tQLsK2AC2b#x
zE9;HMy>1!N;qxc^lwPkPn~s|7&=B0$l_ezjOR$jb{|?N!@^k`fY`F50NyGcNH*Yk-
zz}~j(yv>{I!l6cHY&!49wEhiof};e(2A&PiSQgtl<JTMR`lxxWy2!XMXRl{%)aR5}
zV1oWQos1wHe5SKugJUN^yIPLeR0AT%XBm2)X=8Mhk(UOy7u7w}8Z?*~o+sAUedp@+
z7X0SwMJdp)Dfi;lA=u5l*#GW`8x0G=bw>4(o$g4+#uN=HDsxwniH#Gu$gr>HCY|->
z@HB<Nc%<SQe1jaD19&#Zko-X^uT2}OQl>P7E&}Vu;Ub<4eV&dv-?_Y5loz(8K{C(E
zlWeq5Z^#AEzSyIzr(fLRst5;HkAHHd8HrEq<?sn}2$`ftCo3Jx9|7|)T~1Zn0<^HF
zw-n-4)iZp_qwyG~G)8+P^1}t^dhN@40yu-=q_Z(QAEL*D>46;;Pcgzrf}ZBmSc)#z
zNtQwsSMHOX7|+tli4A{S8}J^Q2Sa$<g==l#w?j^eVCNvC0q5QGe3XH!C&`qx>jvD3
zfo`kxZuKi})aQnK^li7W6}A%o$E)5hrXlSatc7%^9x_wndKMaNIQu~{0e-Rn<R8xu
zF}OOgAtDk}O<?zuh%6|MQZd0tW_mpP)y9V0ARazUcEfh!)mW;g+HY(g2NDeI-R6c5
z8r-FC40r8!$kxLgxTV+7hXMGmj#MM*asgqn0Qf4Ez`_&zPGQljq&F)Bi_9i7KPLIq
zV7J*h=mE@^u)d&&-rDR~y+?0}da-{r?B#yXD||aLDZqhofuIYiglS1v-%~}TC)+DZ
ziM`Hl*`Zx&Z0{0|lu#QTm{oYgx!KT!q<ruS`v(E>4L(gw9@OaOo-GuaPA{y#W$AK1
zxlO1O$2N^LFBlQfN?y{suvw#}iEw6uBRBYKCn=mgrxT%hP!U@?$-Pv}Gjshl*k_0+
zd9ag>BoeKexqPB8wV_ysn&zbE11m@gz>P7K7T{clGs7D_8c>ePEEV5yhvw$m3b)Wn
z2G03rHcaGkcxG5un8!=&!_;L{^VG)eq>$nS1q+~1@bb&nzHZ3=#G|eYOfa5<S`4$U
zBqBhTG!6<on1>!)!*B@){86T9{VjAC98|&o<Teo5GzYKyeQ~@g<`)Jq5JyAUC1SoK
z{M~~H#}cw_4*~{dBhJHlV5vjJpWw~beXD?&+-f+Y`YQNPd_Y?bH9>qF%#lVD5;`}Y
zTW&mFoNNpGnq8M=E%9n1_H(WA)2&ynxt0tM4!?bl$?LK3MOINJUAi^&!A<0!t-;30
zrF$<Zs199rGv5eG1bng!DQ?RxAZneP+6~vX`Zfree{jfcoZhVPW@k$tJ(l;A)8T13
zC+m}JvH-Ib{yVRJ#m?Nf;l$iVm-9Xhx8B!hpFwb4ZnlRk-TnL2Jf_ApZ>VW{O^@ap
z2Ta%yodWic#63E?P|Y7_g?u}D9O4V+m|UFA5vVzw8Yp}r%mMObkTE#{r2^VE9WPIQ
z<oo0GAITy4r$j7VFptYWXV?31Sr#*=8E6~eH+8D*qzvMKX)p;Is~<rh0rarBVYpIX
zXE*2{`QuN~uTt|z{P>gkYG__K3RDHw=N^s+<9nLF6yZ}Is6Ai177rjD${rYg{_Oj=
z`LzZRGW6(>Dsl=PPtg=M%K<4Z%&mHo&4Ly|fY=v?dNZilrE~PIQv&`*NZINDP&^Jk
zGmxOYH11jVfmcWS!=t|+J$?k4gw4q>1<bo)VJ^}J*-tQ}oI$yMbx|2YFtHz5e02S`
z`EdJnR?f1??dkOPqD*Hf{GBauM8FH(=E+?jB+rL0o;*K%`ge58pIl~8>7sR5shFZY
zdO7^|$>YC18C-3MfLGWXG~T;PQy7E*F%sH)kZf^6Bo$CAE6buJo@ea^gI#SPgS#7}
zC$T0#0|>qi4DkSigyo49JOR?W*-8uWr`KGTVk)dX%+aK%K;NW?h3{ZoZV>pY$8V@B
z1RV|Te$ocJEADS^dt+p_ML8Qy=RE{NZA&|IZyZ}+@z)#m4GLGfM*2oFO&0c9Lb0jl
zcW*P^CaXjO)8(R23`ZkjG!4Qc0k>}+dD;z!koBT((c6H86B?`e95gB`q=siNU%TZ4
zyAS!K%^j;Q-ONP-Bo1nYWH_@zC)k_j7@Av=75#*&$9YN-wvqw_17<guxw*m7olJVg
zMljI+38v-5oSeB~4A~7^r(VPE-LRn>=mYN}*p9(pnOBvbNEbuveMboq(kd}r=Ta_{
z`D`><j%AM64jcyHI)`Hg<Bl26#m*Ls-RYZqFKj~Bu$3NyC`)FpU`&@-zT3hN>Cb`C
z1uL_ya-LI?`xzL)wU>tQMcWYW^t~qON_UK56kSv_830D%GgZheCWz)YD#{X^4{hXO
z?u+D6A!YbNX&jh6(kt<R1OHj2K08|v5=mI$bo@PE{U7GFa0MHRQTXS6AA+>$(4<>3
z1vUn76YA-q-qP86y<n}5)v$FRa`JB5uf4f-ns3W2WU`Hua2bi{#+|4ft8h8AW4wRg
zrlMCDU|o~%#zDY^rbwPHlPg7);}(P3{(U?6s8h)Y`Vgq(tUA1C1%X4bnWXwy%R9Q^
z%sCz;DDXWpHC)pwQks9WtQp<SO9*1}dS|6`WMJ>g^}2XzaM9iO(}h!>Hg$C8{dDB(
zMw>eOVzpPPf2OvmG(+Ri__$cHxh;009jb_;zCMYfT0r?Bai{Cf#qT=OR&)q!fuOa~
zc2Kldt*!1ll|39>r(%46NR`o9K55S^(RmGAQ}1sHVr_O?fmqG13lOW@1^%&m7X1I`
zf5pF_GXl-l?$@I~{PF8Qe*M+xtDWPW?5phV$=&peuf808{pIe?ov-iQy*p0F>DTG)
zsvO-WbLZ_*w?1~vzpuW!i+|0(@bAv<?%n^h`^D~;Uw(P_>o33BHSh1-`Qodu{?E>b
zojUznnk!Rjx-88dr>)$o@8RVBL;i_e)P-+*`TW_z<Kg3>xsmfN;kqEnZ84a?EpLYM
zX8Y{=|B@Z^KkMJ+_CMUWy6*l}`+s-mtKFU2{@=Ot`~LqJ|7!am0)Aa;_iwJqB);yy
z+or+fNxGm&exlj%4)VnY`dXMXeowyB-Ft&R_=d%{j5gKeCeSXI%{s$!$b~F6Zel_B
zQN??+-lOr)5GNK4%Hm(eo%j7iF*=*%=BAy_5Ue|#zl!jc$VUd1;BSqgq$}>QIG=GC
zH4yajx>&<x2fP-KmklTY20aSW8@gt4vloo$W}n8?EtC<IwE}3brM?cV!I51<{i_Ao
zxDKsB2w!ZYeve!UtoJdG1}L>cmcSUBvTh3n1d*9aP&RO`XiRZdfsDrYl#6Ur@sm?F
z->z%w3(#edxW3GPZL&TW<GT&-Nu(58Mjq?bTv}xK(M@0@vC#l#Z0X>@u<kywvNh_N
zvH?Z*M6g|<$Zh5J$6{-e<zjd=w1F+(K!|E=3ydSpmb5ZvvgDYtoU%v#@g9X_2u<Ee
z0p4+ngb}wxF(jpmOzIW^3a4veWUal~VA;&EH<AKq?udnCFrr7nAzo7<)}mW;knldP
z)0uahn(Bh}(TLa2Ht%)HYY<|7pVoOp<`RoVjGxrB89-YPEXp*cNAM377^JP?pD1EI
z<-EddXYbrx8}<_IA`A372_*Ay&&%_4nU<+J`VHY28&+U8-lB{U$uyf5<;5Vdo_(EB
zGMg<F{)3U^JT0p%fru2K;9ibQpQbI;!=#WNWt_iF;CN1mI%Se$iemqovh!>N2}z)_
z`PoH<ZdTNJvaC!mWVOwKB?E`d+FrLCJreqnx3^>?jgtcZ?d$A9TL~6RNRx%uc5{Ho
zEK5sp@lyGHP&I*aj!Dk8)u<Sv>@H-fZe4IkXDR1^L0uiYqEW@Ow4`8%xTWn@jPBrm
z7|_z<Kn+uFm?;U>rT^M^kYRGvmiPmT)bW0bIonhQJfkx{88lmFnlQuY>pgl+?@>bp
zUsW!37*RludKu}c%rNR@tREp8%2rv0*c3uL#kk(86HTbY^F{5jcTGfw;sc6fh*C&f
zH7+}FgpujZ4n{XE88<Yq3m_`n0?9)j3wBy-sLF&g#Hbncev9iPfvNT5>bDGG%Yh@d
zHs-K^lW_IgY_25zaoSoZtwR5uFX?!U%10oq=y~8#T#$`|`OyWbM~`8Xqq-KgtL#fU
zDhtfFbbLW(z1ev*m0de6Zvei{`P@>Nv8{!p^h12|l4zuBPqU)8bS*ZeV<5=~sUBUE
zN$gF#5bl+L!w;WD$MuTvWD%AsSnm4|{M8Ig+O?2^+%Is}u9XLg-z<E!HG>}D8}k}o
zlOerq1h2$6#8;j<Mx*<l)E*?);W2dZhkJfBLScj@XPtMGZ)9zo+GgZ(A<nyz$b}9~
zv|dVILfh3>K9E#vEo!S+t8tyo%GT@pn6}0}g@f1jZzcb^r2G%fa^I8u|I52~zO2dr
z?!xyEJ9YZ^yZkSZ{~f%9#CM;D1W*!5fgRw>7yJ9N1w)H*{eleO!CqX<l+oNI)QSp&
ze_9sH`EX<o@P#t}$2x%x2fZ_ALeF~bJ^fDL_3Y8#%+ry0Ow{bnoKFvS7?^lf3t%Rx
z1NaXbB$2IIS)4++iu_GBfpm)HYjce_PfN%KRdNJ%NYG(ESM1JHg0my~P||z8L8J?D
zQmPDNV;TeQ1_vYeg<~?Vr^bHV`DvSAV@-PC3{c=QZ+P-B^E=<Yc>d~#BZ88rKHlBa
zKu*A8P<dw)Y(c;5AUS-x|CjxzN8&cE29?(a6@1^?^-I_wE8aS9te%G|AQFSCjwCh^
z=Wf!{_I*1-8XUl=zh_4@<K=&Vcr7mxnH~^QX?Ymoy|;&Egp+K*8rw{Mv@3tK{wVyR
z>vL0@b^8@tho?c(-nDF2%$QTN&FAg*4NMrFxFu<7J~J#j@M<hJIK_Qa*bJ{gxVTTU
z$pm6{Vs3PN`_A|y<o+#9;~+W1roc^nmv?IRc9hNWMnD@)k^nDfOJx3GR6WC^rLzmL
zO*||X>~)4d$p|b6;({AL*%6Oqv_7on#mo+v@?=knQxtYUVK%A6CUC@*(Fl6n<ovE=
zGzyVr0ATrvsXJTHm{{z9Nsv7<z_A5nWp>`4oV*&y(x5RMtqY3^N(_Ic@pE_x8wqUi
zJY2ky)C-ZBVW0L!V*z)kHJwS8bQ;i)J}w4(4&=x`B@Z7yu$ZH%5)d69dFw`;KAmXa
zrg9})K?wNL+Xgq?t7L?21YRrLB7loSF2A)Bb{K6C&FT{nw(yyZ+PSM7%kSyXm6hc4
zwf%7Vctnz`eHcRcAG%p(jY14V^a3A^8h-iff%EFV4)rypVvI~+8vFcP#;N~BreFB)
za`!*-hwf?s(tiK@^2<A4-l^UH?tJm}@Atot@UM3NW4+xcasBfQK%95g#!d9`K@<Ha
zM@JyVeDvz+%e_sps9>id$#1uVW)+|v7z4L9S@LZPc!AULaymCmTYh?)q3%RQ`{CwB
z%Thu*`ybBoQyc7gvt)CU4K_T*-_-G6(h_17lB#p7%qSR<=SVLjx4s2?^;ejUeGf=u
zPdhs7_UkRMv&5dK&EL4Gb*=0&#X|mBc0R1>?prgc-K1Pg^gJr2(|n;~){;pwM6TV_
z*@$^`<Jb^5N2%_gTBpWAJdES%*ay|>&<Fn@rpKG|B%7Ts&brSk`}loQh`*26VKtwu
zGei87dL81Q<J#JG9RTH%hzwwuhP@{G>+$6WVM&jROSjmFM5!aE^2jC|Y(^2^zV`9P
zmChPIc&l#TdP`&9dhptO_aQ9I$U@-T<!prhi}SKr6pM>F`#tg!p73Y(K!m$#nPD!#
z5jYtd3R$v3A5Wy5bY$A}4`~=psx*)(+ua~IcZ*&#K?VFd@XY}En}l|x+VK*gh9tb9
z7@Vs&0VhtknJ2Sxh<W`FY5C^?VN?G)%@*;rP(gI_T9`(<&a^hX+DXM<9QCV9#u?wg
zqd9_PIK`TcLNqIb*SZ758W32WhA%=@JPeJ%sF;k0SAtW}fQudOwdw@e^9vGFo#feM
z%(i<dT4U=J_L$D6^P;Nq;|aMBm*`Gfi7czi!0)#t_wQsmA0z)WFOYfUz<(v>2c1vS
zk)a#s_(GZ~BqBW~2?en=jMGvSX({QUnGEJzpA|NvDCDNJuG<ij*(~OTjsEOoH8}#C
zZ8M?Xt7KeY_^$K>qKi4n+-Nb&FcTEY<}NV&7{rApWHKFNZI-ABk1ehh&wu#)@cZY_
z|J<APq5XeJz?|g_W<x*e4KM#^pUpn|3`~%3&TMoYG*;R$uin^K?~>)7s=VCRu};-A
zw_Ph<I=Xppx&3CKG16SsB}|Wv?gsby+Ds4$0l{it#?7CI`o(KyEg$jti+;f)F!}|L
zL5KlC(Q?A?SD*>Oma5wg>n)sC5Oqv~A(-P~JT!Y}a>bJj&k{mB5)%0Vd2~gCW2$c^
z0wmF)n;$^q2m=fqJbHTYkNp>7!3{}Yr}LiI@r_7x)-KuWxbI@la7>GvJ{{tH^QU&=
zZNrnt4jaiTEuzXb7J)mt$}-TA*>$mPVp|WMHRI(DWB<(ut&BO)Mnzb()4VcNndNjI
zlSHqgX5@6A#9O&18*Vmu10y<Y*ZLaJRlKq}==m}}NR)XLkA#xbN339i90d~B1oIVW
z{Q%ofHru-Gxw3|@q<}g(=LiS$>fPiXHFDd9FHm(TTC{I8Zv_$qY*KdtA_-TXGpOEh
zf(k6lgdCcFNHxstxXLK4{2Bu+F@S+^7+W%t!*IgI>~qj6pOjg31|<eIX$-`PPA@3P
zt|8_adveUxFaR^fsHBV~X7g7ZGQ{@Fd_q!!fhAQ0W4Vn^GlcX_ghT*uf1;RU0>2@7
zkrTUD3y~cfDaobmQkac&8Ie0H(KW#QfzU|ZQB`c$c|%+?4SX;}t(M1~5mf?e_zPlj
zv=5<Iph4_F<zosEnM=HLCg)EI(s^zjXXZZ;mXic}RW`|x@f#%hcDBvGDPR0#Y|7rA
z6z7?od}Kt8F`msU3Tdije4%#;Ei(2ygc|0to<*zm=2){TZ*?E}C2CmCCgdux8!UtG
zQXw|Dn;XCQYl|)avP=!D3o#e7d?NPVO_tp}Ei&!?rO~u|wbgb22{G6fU7j*|PX7Jx
zHUr^@6!=6Lmd5;(?0Ol7{9-g&mEg67g=)F(%VatU1F)Sok!nPoD%P#w!=yy3PGn%k
ztN4p$NS3H*r-fEjqk1%o30zO`3GEda2|WRYl}#v2j9?&!j?KOB%`SR;pvz^5XdG~5
zAE@{}suFei=*g28cfe%THi8}7AL`DEsXP)nL^$Fxq8?;Q0c-7SdYp~Nm{|=z<|T(}
z!mK*jh&dt6rFWE~lZpj5+;qvw3~q^bHt3cy+rk;~@kfnhpPQR#F~_)Gvr`C9l_uc1
z0%jO~NMW9<?#sZ&_8@)^xDs|1IXDm`kn)zrf)wrS6{)^`Gx;Vvhif)t#Ce&{x6%@f
z*SBpvd)2{xCSr95s!K~}RxsZ&4%51Pgy5(-FfifVSV8C9&eU7N5<I?PSh5$Rdl0(U
zBYB$KDZde}+I*_;OdXkVLuzmq0nDhu0*?I_a(dr|+F~Yd&FgN|g-(cwnV`8Ues}n6
z|L|qZ6``4T=Lnk~#&)XMPE_2$9(V+({lKxcZdrn6iDN70WCm^PQcSKr@zaU<(6n4-
zZrO&=*#I48y?$pN2-xe+FGDZAa+Z__MeX~V`w+Kmu*!+3fv1M|m`q?P&XWFj`sGR(
zl5+$y8MabQ6`ipl0~-<3(`I<DhumM2yzT18=+nG1`k<r#GEc)e4*H$<w;ygpP5|&K
zqvHmn3&OmCV$i&s{m=p`AZroO9KWzGsKe)R#H94%IFlMe`S6bIC4Taj!Ftjipa+&+
zHS13Y?wKhCW6VlE!#o$em?N0R=@xgd1`vcW@Jo9on}EPo;y@4@&5n0Waox`M;Np-O
z3-H`=AJh>Sy{ed^*!CC^`H)esn=lhw*PMablr;xvo_gI6PR$u7yFQ}VRc>)HP@q5C
zP;xq2CRUd<Kn2)esZ?V@X&|+fTJLJeYOQZqU~Ak1Q&2$xvj$5VN<;liK(`}!v8V#5
z?2;Dv+A1|THvzZ~VFdBJ7)Qn4<22IPF^8Uja+XH$y=5T(Av8ixJF_9Lxg!?NOic^u
ziz5a$QJXM7S3Zg^wLul;h)6xapVlRc9Di$TP#SJMnzr#FwSxS0B!0MNFdY|3;u_6P
z=hVab^AZtm0W%+u|1VeTd{6zjITzU(#)$BJaxwc&G_H7ZTNEc$wQoZ$S{d8?$NVSD
z)8e*N!+MiWPD{)cZYp&3`BnFfngcIM;SMA^0n$*uzGjsH9f5MdD&T4X;ghWzcmavB
zMj512B@jbfDm;X`VG~mcckANxbV5p7U>~=6Z+Oo1{bZ58&MIgv9luFu3%JmuD+4x=
zq)3S<Dg~Sa%*cwE5t8nj2GBc7Rp)GgQ&?uy=aP8}00ffg1{naY*om;xY`ayGr}t`P
zXH)Dh>|-bh@<oT&{96#cyx~LxXjs?}FH24`6V`WM8XG}o`1-uer)ha{9meSxf6xh_
z^(F-LxA}t~xOs6k_};^7l)f6U1yxlAd)$NVKtqzGY19NMmHV1__;?TFo?>8f87>XM
z>m3U8R`sYmqu!f)yk2FHW$qHNvK4<5Xi4ZHh~knivQvq5@4_4ywl+6=nr0Ln8mw3e
zwi5;c3T5b8$Z4gN0%D!uG6&m?$pzA@`kM6<Gt4uHOLLaalG-Njb23O#VSQdDqcW!?
z-e~Po8)u)X1h`%nOOQNnQYLO473WP}f<qdUKau(QSLyKL+<<f>qNhc(ThAP@1dbcd
zXae=1*`h7jTEp4VDFB&1JQ`v@gj5o^iz^D~e>1`QCqPd%LTGhI!)*awbbPkUOO0J)
z%G4LmiUdux5DMW9Wfu-S0DfE)tYaV_*A~MB7x8!9Ix%AR>vF3Jg4&D&_KFx-o<KdN
zFJqbn*(j(QX#|8gl1kt>!-Temg_$m^1u4vrvuu_q2{~uX1>NdJpfaF_6}ucs@Z8M0
zNZypyHb~*#3yrB``>%<Gsqd~jwX;msrOqrXMzIXiZ%z_dE<&p(MmuyP6s-ZF^&0QR
z@EbnBDFo$Cvc>4EXQhj7g_>|bVk_H#XB}2`(~5!|E42cWrt1{*Lw%CR%>U#=MIawL
zR@^tyudRKuuvJz)oELDtFo{qYu=Nl>Z%f}0Z;=!;-MAIdjLtbs=YIeqb+8a@(5%L2
zueXijvZP@30ySQJ31jrv3Sm5P84kf31NijA+MgwlfxRCKvv2PG^Rk$yr(iG!gJvmj
zWNsZOkw~H2uXQMaQOVMES9&Tf_e3r39)u?sk<$&dNQ)g^;n@Y+Hwv(eNa_dhvfMyi
zWK@$`(jvgUwSJf6rC1?l3eH8(vg6%~^U)dgtaS4E$TmG}iWl3*cGEUE_*nbD<%~+i
z&2<LJO~bFeSz!@&je>su``?KOF!#WH!0Lm;&wH&=01ux(di;3*=%}Zr*T0ev&Lyop
z+RZj%N(M6V6D0$PZFiU$bueD58?I?94j1xsRHT!vqSSa0ddZ1rv>-Y(!Khg&*Ok?_
z2=*B^5Yf)tRy=jXHE-{tr-NR~h?YE1qql0%^`TwURY<hV#{wc+5Db!oL2C+wLFXiH
z9YY+eq)cb08T$O>;9Zk(tW0pSfylO}WUn<ZD)a!9OzdiXs)8MYC{sPCw{Mg31~u;B
zq+a;C4i{>3ZT<V@^y+O<Rk5jBp+IBK>e1$9!3Hp;=q$P4Xd;3{I{2tMnbn5Dwj=Qv
zuAyNG+H4?NToclBN$uUGbaH-lUPeH`G~T5wc|?R^{75kdE41T>>|LqvZBX}GL-<4Z
zRug&-Sj%A$J)L2ks0?%xh`Ug0Ih*9KF%W?%ZpfwC6;HGDHSeMH1bo`2q^tmHnqXX~
z4aq=Sk*t8l0wgm)&#1AD9Zg!1Ts-KoY8r?m!8+r$Hn2zj0D;UpZCV%dIo5n}2R(J5
z@pLNXXn5leie|4>IC3rOSuJLxCb?ZN{IRfhGkUTdr=!>IXaNqOepbQT*wA6Yu!cH_
zI@_H=MVpE<N^h`)^Rls)TFpllwFwo}DMgiHM^*E<+U&cifR7MOCN!%~DQF8LbbO3M
z%G!p#pOXsV5JC02shBO<?M#E>ku&YLJUg2)Wz~V&j6yb>ZimA*a?Vm4mpy`;8oczu
zOvAAUbKCL#5vks46Ai&lo20x;`0>~+ByiSx-@2zR7eMdMXRmBTa+ILH!M}m{L+#Xf
z1u@}fSKFXgRZ&#%7wU`ZN3*^-c&9Rfp3nx_ih$p8(g5X8I*!P1n%2a{MZabh=x`aP
zmhu@Y#0k7w3*dEx>Q;Rr*2<NdpKjwouO8gUi^S0ki(shHeegb|n(spt!mf~?!VbBN
z>{Kyw<4p1ZIN^ZD>Wl-q<~SnQ>@3$L-_dTku%~im=z89`JaK`JhvW$(c}s!gsogaJ
z6xbkxE+S-2Wv(G`zfndl4ij+N+B?A?8#m{V&RI`}A_=l%vqCUslj$};H%(-xq1}43
zSE8a;|ItK8Lj_5ll-es!NJ}FLP8wApw?qXJC0mz_T^589>qNxx_R1ah+N65w6^D~%
z5KwRZMiPA4NNks%jm6auF`vstd0S^P?~MfaIC$_4N=$<hV?(7S4X8$_`E{&j2yNQF
zR|j<s#vkc~jIL0ZSRAd#Q6x%Ll+sD>68cZymm4tfa$*01XnRhP*&cDvac_nP142Q!
z_E?yXNUVnX;JpxdHO{dXC~%$rLmN##Wx(tNdrglgS<iMO9Yk8X%v9Ejet4C&@DVEl
z&iV}X_$kN*-u<%4)LCP3unR4C<w4F&xytXmS8t`S68pV8dU7}<>qxAe+;lFl#YwGl
zzfJ(WtV*1BWcb&E!!Pb^ZXEsn=;i*i;lbgz&l3n8ZC3ufY;kl^Ewbr>`Q5vL0i~58
z>R+-lKe^yk1e7EU8PU8-vR~kpVVA&Ac7{m@PD_x-x`conXbetuLjcCEzF2<&-a(h!
z<MSWPw<F?ld%fLP9Ml?UD5g=3BbyEI#$7n}7;_bXpbKaw<!@YIo`Q6?oTO#GxG=P}
zqWlV9-TgC?)teyiBFYTe63)RBAf2427ZqCT8@8renD2`-J^X7rN-uGOLEK(LriM#H
z+!-tTJmE)H&i<VnRG-;FOdM<2z|V-~t#?%K#z2eCHycNof{>G>_Q)J&Z%h}klPj~H
zzyg9qp0hH%NheEmK+ug6tLs3jqt|<b+4E0cEc}oXz*)A_-dPeAAc5yem`lNaG184|
z<)Os_;Sz;T^{vYgkSRoaYo0;d1d@8`9SY%SWD$C9Dm@oG{fCz?Zhg_mtaPvn-XyK7
zQMlVVrcFm`_*+NH{-62p3i00oeLBT}Z;$`J`^BBFcEb4YU+nJe{vQARBm8TK{|;f*
zFB1d)<As2?9x$_XnpHpkM8TJ@kMjyB_4v9um#qI0JUb5?_;q;u(RB`n48C8Fmt_g2
zlEB45quCh35+@Q%Il)Xm=V#e0fk<X(x{cX9flYz{rib6$BVAFBo@toQYYg7UmZQ>s
zC(41#`kFx4WCLtG|1pf|=MYr}8nelkNtVJ4jccQiu5;!nsth2YPcC68HifG>&tMkl
zH-|B}Brpv|WG<5GVm3M}iy539*ce5p_k7=^#f=TyyNi77`UBkg;l=)Acb>v3egm$4
zqF>|OL#E_*;9IvrD+`?69#nfog9Sb<LuP55v+cZD4g_f`LF5b=MzoHGH9^G&9@If6
zc17SHbY)9+?2iYIoZQ<pvw915Z;C?67SXNqFq0+RHIqpWo<GG5TV)7kRRdtA>Dvz(
zD%vxJh5oAJ@^w$H(gqv`|1nSt@aD`UThd|1?+$LRDCPTm!jc)@5-$z-9^&rp)f~f^
z|1$i!LvuAId;xka;np|Qj$(^gPx!An42QZ*Df-siN^dG1&TiSe5h8rl=R`}OoPXXz
zHS0NSb{HhcNTLJw;`T<yr#Uw51JQiRtgXDkv?0Y{3_KH*`7xBC2$ccG>{zbT7IDFF
zwsvj8YZ#1kst=+O-{4^k5^h914!_OcW-0*J$ZS;1Xn1Vsmf2GU=6o}qlV@^Dp&_y{
zc?t_Xh{lwHJG_kenu;%0_Z4xlbc2hUv3fLSh?(RFaRCO^JO?=w{F!B_48b>K{K#)%
z-8f_?vd0KJOUAOd^K3-I^BvJsUsJ(689D3-2DZikE{9~}b8PVZwOHWb2n??`U*wP=
zU5%n4s;)mn2;N_vxhY2CLxyAOH=(o|^-iO$zIeBO6!&Wi%Fc7LTQ^WaJ;Z=U$=W^~
zQm><fHQl^nyY(r9V46r;M4`I|6sJoz$uH<KK1wk@`a+_!Bl|W73YaEm%Tv#e4HAYz
z*i4M0GRsa=Knd5nzRuf2qK~t_PkFy;o2GeB$Q|@r6jS&V4Gem~wQf=rbBw{K6RLu$
z=8ZX9F$PmfG3CM9r+_1z>JrzW(orlcyw~&bj;Sk(X|}*rs)gm=s4}s>&RnOqKb?QZ
zz>-Px1d{=Se12q8Zz+A6&dIKYMXT;bwLHdmdn(SvDQ7~H()PDk*gLx^`b3OsD{8NU
zDsF@{+(jHx7JNl9rIxKhq@=z*E6$VY5+Xf7Pg{uC20u?BrzS?|k<6IDIa5_XF|NV5
z3gl?ABg80+Y6Imz&bT5wFpY8PAjeV(yF!~b`ta4^+58BK;+Ub72twouD1LCadhokQ
z1L4}bn0I>sf<S%0GBmfC$WQCAA(&gTy~(}k0suM-*ZuPn#c&*I*=#=H!^c^bmo_}J
z+W!b;4MD-*hR}8}QWrgskSF+jKLXX{-~WzG&-N9o62(EtjrG?srX`yPJ8)R?1%XK5
zdP@ejv}cmziv=^^8e#0`2f`lstERrMvv9#q9l@Tgw|UFDuGvU15g6V|sDauDZpjF&
zU&n^h+52`2_G<U0&eK8jZdqM93lcFH2C3<C1iP57?HIwPiM5kerk11U19KESbnHFY
zE-pr!Z6cCZ20~+qd2(a21Jy*tYDQFVuAv*A1wgRNU9o^zyALhyS}GvvhJ?>r^3EbF
z>M*2HSD!4`qUS}w(Gmqo_jt`RLs-Q{eGGKrVdFvI6LxNMMC)6!gCE$t_IQbrgq}Zt
zhRPB=$vFeFc|PZA%mQJ^4?OUT<)X^RO#BGHi!!uv*9aWmOb~}nud*o`m{Vp-^kqw}
zP<TLY*l23+Htp+sahAxTd?aW}T`fo^ZNsx{0*LCVXYEBTP^X0|^M?;Y+6yumWJklI
z98K1zLA5I|o7t(su!6hJJck-ae>!qtDz?xOkccABSH7;&^tzkuUen;rw;<zy>c6IU
zj|?6<hO*9mtFh!Um8@IviK<3q9t~r)3m3hTce!NwVk=Lw*K@6Vdxk3!oSgXX-3Jg<
z{nqmG|7riOqW{4-SsfHW?fRdcFTUR0t?7U6+`aR={^uk7Yta8ZgUsHaMiStbcpp<t
z20kgyb-b{vM53vuN@xnX??g0PsIK38ui^C}E8M0?1vr(jz4uEvWq2i;e`|cf5yEs}
zdlj$%UB#$Mb0mR{&Vk*rDSeg2+fKk6*x}QqnMS%O^J&jkkL9HZ$8Ph_={N|=dR#{V
zF~x&O<y~TapvC`t48qfdTx5QyM0f{iJ$^AD#b0t>EGJ`-Wv9t`N>UJvx~ZffXqQ6)
z?Rwxtip?Bmi*JseBxfn)F@!*ul-?pqk1Nz>n#+C#qL?%}$<hV!*%>*Jn&!@n@-;Sd
z2EMcAMRszMk8;wcq1H1?M0{g<o>S^-l)#nQ1kGcxR}xB2K<;CB_+m_fJ#7n8m$OG!
z35e;o05W5o{|s_*PhGyow{(q6zAw%*Q*yxKIx;RQvuZh6R4`G<3JDox&kMW<*xZwZ
zZp#@03+0vg`uTj3PxETQ*(?|4E-V8BRY_ifTP^^v%umlue^axGL0gN-CC#l2VoX!8
zz;yIlY=P9Q2hO>W#O)+cVW(Wk3L^beY_IveEFjM&tU{GcAsrUWW=uJ;`g*@ne;mmO
zBa}3TYymrZ{u$GZ@U^ZXF#~*J<6uoz(5U$Nx8EM^zoh%3Cm4ePa3y!g*K6uuF=Mf(
z{6nH8PA7ZkUXGsB9e@-$sawiA{U6)zPjNc^V0OMEhwJv5U+W6yQJ3WZUY~Wc{GosC
z{NERMcfY<H@_)NunRmbQe;@uI+P}Qte;74d6#PW@3@z*3`G0qB4es=l(Q@><DBzen
z!#gMC`h~AzZIQ831~LzC;)zt9nWJU&+H6UK_~6b`nF_OHlo?7tKzaWR1Q@$y>T)M+
z2u=lP1R8UB$bnPK%s7h+FyO$GtT5z0H=JxRNJBGD&8NlL)SSo_h|5Q1eoQf~^VvP~
z2cDAThZHg;zStfl-)?`qoxCW<5Hq`XXJ_YbKY2_|eosxB4*$DY0@n<{c_i&u)KHRq
zx!b$bzqfnm&erbTyW8BuciAi}!4behH5ez3F0V$*szNQG8`^#eM^;rq*1T6{fX}v*
zXXy-R9Sk;kHaarPU`|08>{tEdkpZ;fWB{7;G9BSLDML#q$nK*m5JPpRV|0TzQ^A#5
zpsB2x;K3~Up)5G+g*E#5dT}N#{S{8Tvto|MFXl8fKwW~&0W7|-?;}HQN5HC7N$;;u
z9vyvd#x_WfAZ~Z^c#@XsAbI>~AkF=EI$MI?2&Nz)BdaVQ8&OxPj(Nrog&Ja(B}@vi
zBY-;@<`tw+!08N6Yfc$WW(Tka){Gn4%qoul;ST}x5iZMdHUriM@E=N<v(>Nx<K;+J
z_gk{f-X*D~TfzjCwUcdLEDBx@reSBrc}2X$+k6V_mNgETrsmCZnoNom0;7x;MM;sG
z!L+$pBHD?)IT?jH)RBMxwojko#=Fg1(tfAnpYgn*_%2tw!+-Dc-#@tGTm0|Vs=s<L
zyQY<$Z8LfQ?-vRF-MwR5r+;_w_1!<x=RbU9{yyB<{^J+->F-zf{o-F>u|Muo@jrge
zfB(RL|M*A0_+7u)A5}30nN3z#&z93;Llj`|0uUi{xX?B*U^^=evrc?Eak}ymGFJ+t
zi;maOpg{--Q*mUt7=smG+}T28{Oa!3ac<B7-4m(-oBk!eKLGiK0=>HtLcED8RJdxu
z&YoQG4tlTuTrPAcZ}{`~pNOCV42x_^`Lb9Pi;FqOGl%?s`3#_)ur$Kb3Q}<?k^F{X
zj)teD;Wro6m~B!FlWAUptJv!^!>wtvKKM|bmHF&-bNLS-R3W?9BaD7#QOwNrH6%Zn
zjU}u(rhYJZDrjps))SCt>?L6MKYTfS_ULcW5ZXta=b&x}c-YlFIDC3=xIcW%yzeGV
zi+Z#9^Xx<|gG>b^nJ%%=5FiarZ(g5`u)G=6-lj8y%7Hx&(`q<Z1@Q4Oi1y8^$A8`j
zQO47!N1R`ST##Xk50V?E^{t2I{Q9xSAZe%y%GRGmpaPY*9s=N$5gE2*-kHnAyoXWj
zY_Yy4bm`EO+)Sj31ASj5>fp7+(>J!1tCn%CHOwQc7h5h~Q92TSyz|q*EiNvg^M84L
z=B2qr_(3%S9yK8Le>_#m6G2Uqh>fB7K)T%d3H_ohB4E4VnzAgZXhmL9;7mhE5Jgcp
zZjKHp3lD=<>!I(U=4&uum#m>}$h#kC$icu2;?$RzuCcJF<%KjU0bx3CR8BgtUXBez
z*SdYhB{kR1X<*F*W?IGa^vqJCO2k^yIa-e)lfFQ$@N8^&B?-;6Z6mPLhZ)$xV2+py
zxVLJtg(+wVYAB0>RIp(0#-KLmt^x6KE~ai)g5k7c^Ky0y-cA@aYPkIBZja~)z6Vk-
zF$JPT=%Z3Ck<Oa#ZZXXJKC(sYp&Ma@-ucn+)O*Rp%I{$LK{IIn&|4Vr2t;bPZtZDe
z=yI;U^GTiyxI)D4uwV-?;G5s1`2?w+!3^10$qQU>@C101j~2*+8*a$T7*Vcz0<mvX
z<i_Pv1NMXx#IGp%0$?h_b!LGM6Tlk*r=D?qJ*bU+m^!!1%vPVn)n^R+yP0P`NwOSG
z8Y;4k0b%7hTV$5kr(whzlAAAVhbGc&_boOs#M&(zH!V+P```wpy?XPed2jb`_`&Vr
zV7JruHh&iMYr*Rr?DgiOU_lSH<J=NPJ6uzQ@QX(ZsuIF28DY#;+J=8SD++Kz5h4NW
zDzV!fMM2p&Kv7H$bB_MzQo%srz7k@zST=BAuZ+-Mv0Nm*?2WnYp`UD@>eJeyG+}~$
zSJJs<@(sE`fqcQPIgnh7DqE$1&aicAs#FV86)+#YRlsg!AvUomq)^}n;Fig|08yJ_
zQy!aB-`-SNYBIL&L{ydclr?tE*C&5%^zZWAm+%fD14fo&aPFR`b7s8$1;_~)@(!6I
z^|OZy(`Q?s1tFMib1zviiaA;)gVxk8iIO_{zQ6+fiMNM%&*1&%`K!a1n@ksHxOpp}
zjMfjCoE~OlwrCPkrTBK#G{8Jy^B}##*)*|HW}a7IJ19e{H1L4O!N)+Y;SMaASSU*S
z&Lr$$kxWzc>_Ac+>lHK44H(_I8i3&QN)Q{7T(FyhqS|vp6iZz39CNier#n)G*{z&p
z)hL~NZE(f<)`jRDQ`9I`*>YTLDXJreYuSVk0Hju$N1^>b1%rLr)m#boHc}CcE4dJg
zkWau1-fHAh5EGrFg|xM+27`DY84EOML_2AGmseIXRnqhq=LOdS_Ncl?Klqhd#g}gS
zuoCJxsS4;7BLMue%tx<n+*$Cp!}AVf;qoy@hZ=5312+RKFhu1FclyaACRZTV6y!!-
zo9i7VW9ba<E(6yU5w~fU&cwhU_2!^fn`P7}68&%$%|C-Pf;V*f3*L-DUuVd@u!hQ@
zkQ329KFbU(#sp}5T)S+uiwL-Q$M9jM*d{s6G}9D@9wFmNyV=|!N+z0(OuTn57-0MU
zjnpz>?>gKOB0Pp;94tc-zO%eU6H~BPV>7%%KfGIb48V%ZBilW5G*)uFeV^CJ%RxT`
zetf9!{Q~*^21^$$%~vP$NlHQUISw`Kx;J?$H1bUrV|V4EWpAsEOsyN8V3}Z+zA3yE
zhoIwSKA{8DX(@<&RHG7OoE%>;sC_0)^0=d%3!31&25bFi+4)MYoO}1hCCnbDFVkba
z7w}$Wj<Z2BGoz_*1m?F9&(SL3LE`RHdz<QV#V&#NehKZ-H4~riQ&;LjiS|imI6vk;
zZEJRl$P{ft^&QC)Z$Ei$$raudYea(5E13V%I!T0aJ!F|jjO39GI*gt!j=?RlDRj~b
zfvk;3NoG2-FlL?MHe>$in~B*+U^UJgX^_yqLdiswsUw~ay7yT+vHd}@zh&xsaCtaA
zv!US!$*I{VaFQEzgj8{F%n+EK*+Ux}BCN!NS4?^g(=;xY;4+*{!EObE4-{Yu1bT#W
z3?6OHG6!2(qKh##COgrev7nnrPxi2sz#&@1FzpduK6YK>eXz7nFRE!;_NCb3rFWJ+
z4tGGShV)LA2mRpV@AqLpTE<mi+v8@;jsU}!CA)5Z{|cke{q6D7M@L73qGU3|#vd)w
z>kZ&A4iBSHKcA!K;@8ELkgpog;GsXD?$q#HhI9y~OICdJ%)z)uczf+3@>1L!%ZWp$
z-d628T)83{e1LcswCsEjqN&K8Y)>u|^qSxB5B=R=^apw|%%KxS0nM@~N;!aDWC3!r
zs;R>fcc})KBNoc5ndbU}8pk9NE;$66aKV**4uVAF@@W(c4<L;r5U?~1qYkwuIH_Ic
zB>oTw3&`4D7#O=fgVIW^l9tALl+NFNT?yX@qGxh!BVB_v`xr!tgKJ%y%B)XhHdQ#?
zE07RPJT?_IX`>)f0wNMKa{&Y2tE?;wxF*g%Ux3tnac093qXkcu<C(I+ObMn8)rI9j
zQ&r&oO!bo`7;@;aGssxYt%ZmaFj-i<+iW3GbqZD6Y94*MJb7A0oz^B!jj|uwvzk-z
zps{&XDt1XCRs(D&64t~F(Ihb_w3?_jqBL-Z2y}6}WkuLil!<_O1AGJ$0wv%ttDnqU
z*1y6Behhg%*BEc)7}=ArN?j?vL&yeY^NKf%diY%KT#F}L8AWE?J(P&NLkN2BiP84o
zkvNfF2~Gt9TJ!Pl9rT|53x+e~eGFnGz@urpFgG?1Y8c!uEaA7}NdOI)8qJGyvU9m(
z&8Q|RL<P@gq|k$VqFYBY_ns}v9Bs$`nyHu!y*mw-gCOJqc@3slB2%YL3eQk8cS0SD
z2!Z}TumZwuv$Os6mt=ee*Q~63Ll;x1h%IqtoB<v}i;FoMjl)hlx2VIHMV^&t0YJWn
z?ZCQ9cnOcYW)z7XLyTLt2XiVo#JiRBzAzNaz4))>i@u>px6L0op6<xaPa!Y|yR7rz
zs1&sF0#yT2n>h5R0Lt#Gxi=d6goyK`fS4j=*=X^Eih2aX4yA$EHd2;u6>$TSAVglt
z=0oWkJOR?KKHD8hxZ>F^p^g+o9^=Q~c~tfa$3Lx7*9!z+)45;G19TbknjETSO~=_J
zTVyS)X_H-uBAb8NfAP)pqkWqp2IA)}PI@%`>;F~t{%hX9mk>d{m;BfDZ=bzfes%Zy
zAQ6hHUtZ^Gi6*UnEvBvUT0~^kw|~w|o#7DlgTo=tZXjOM_GBpxJ%Io873p7DiU;4Z
z9L{59S)SU;ax}}nl44wvpHPkuW5cAJ0vX}J19-+;QQEgJ_V<VX{p#?~gT(F9q0I#D
z43h9j>UBdREnDQnOgnLadD8+Mvg=F-+?32YCvN07ooxH&%}}Nd0zAt$SQ(sc!O?UV
zPd9c!X*M0{leq|Y;!E+YDm5y}gA)2J3fox7E^ggH3s$Cw@;<*JivVLatvHb9ZHj^E
z$?m#VA8utxl-R@^)?}qoD<sDNHlT=l>seG`lZ4PhK^r+a-c+8@k7Zrm<f0-$711Tm
zbWMTO+9X<utT9Pj7eEt)^j`3dNa}lImGPf5Ys)qGZAUE@couXaf;Ztev|&*^is6k2
zPF1Z%K&{2(fdn@UKXQxs71*c4V&MW`Nz@!ip0>93Hg(XRvvl$%bG)RBbvOatG0J&&
zZ{Ian?#V={AQkU^zn$!`EV4;Da;M+7Hm<RH5A7$%{<+-k2d6WTMw3v%MR<`|E57K8
zklqUMB09=ea<RfpHIBLetFX2sM4eP|0d@{};qKL_Ed#2=&p+PziB4gsi8Ylc*{s)d
zft=0205XGQ=j|Ww?%ug`^2Hag2HZfnmejKtRePQ92BiNE_Ke<2d<EZCYo*`c(Q61F
zYyjIuIWpb!WR#{0`)n?we<F&S+)E-rniuG*um+SimNT%205ovrH*R8dKq_yR8CEA*
zDioAZ&)bx5)&9(pw>Ph?O#zPNhWRqkr2y>XKmOs1FaG#d)&zE$%NlTZ#_a~|W?Td>
zG;z6?{2tN&Q~GyF|KG*refj_He!Y8Vx9<P@_0Ctn`~QA~e|3MXmrp+(e=HaJpNu&S
z-a(#MbkE903yOPxQh*@N91Y;IXt)z7M5*s?>ge@eBH6_TH?`-vZsA*%Zp(g!a3){f
z1tJV3vUxE&v)^og+$uhQ2&XLrknb^87(1n<Q%Eu}14rL+h7qxgF~_cq7^Co+YX*oF
zDGv+gV)9<~R5>nzxqp4S4zJ7|2WkOn?=>~v$86X4<baLZlXV!_$lRO8rQsZ!%2K{z
zxp@OPl&N~XxU>iCnm>D*n0yan({ZIgLC{Bf{t^57sq+kZrTjVSmz}_WO@ZHse%O8;
z{-2cFq);CqIzPO4{_=Tm*59Ci|CH@zcHYM9v(JWRdj4$`CoO`RX@0C!L*`C28<-~C
z6||U)=T3}cewh+;6&c)pG(RS^@-n)E!zcTHYaCx3$2M-lm~lD$;N%-Ttd+yJ*bGDQ
z{7X$&lAsNequQTLgp!*nWdaQxO>Mc+#N0?OQIE={7+W!jxE^V2gij;}@B^~1=%Atw
zB28AR$@Pa{W+!E4xZw6!@y-RMzgGjhh+=I0NN2HbB2D4i#mfN3Ly<{Qyk5>jNUN(8
zAe@5+?|E_A!NWk|yg@7VYn;s&XAn7N)9t-C=43H@v#(y^NvB^>)b!zkQokFBVMdC^
z0w<GY6{~Lu#R=}0rf(^Fp+~+F21VX;9OKwv?6-O3qf6Z!tE#?wm05aCvn-A$`5OqM
z5Gld(?b>_rD~!CJq`zJaA<2E@8*?!5EB2Km2-Q9SJR4pQC;9Q|Xw>p`IXlNRkhKDw
zY`FHG7C!djEH=2sB-Y4s0g-+qAKs*s(=tAww*(&j1=7%^$3?jaUt+oy|CLJ+=)a#)
zlBe)Brd5r!G0w}y#SmhE)cVXu>BSJjd4*qyfT8{>{Pc5VL>TTlc=rqWtC-U&{8DD)
zUyJ(dI)veE0}ShH%CqHNj?smKD*kFC$O1f+cVIMflGj>jAOrT3>}@Q8<TNrsELo)x
ze5L6tUSbShMLRhiFQ?IoK!oOoI_7k(tr<^dw48c0hQLP_xRDQa>PhbnVPBNRMc+Gn
zm?hxc$o`>GPwdK}3sZCjcO7_*^dlb-i(?<mnU^i1HT=CMuN(J@*Hdl3$XA{EcvCZI
zT%cB`8XiMpogqbEYxp=Tv(x%$<!iZ|arIh<TqII;0=4qydh|U~@r;8+H&>Msk;)OO
z#b4CzHa6PURjJw*r2--}Tlg5xUm&Jk<aL~U$P^4jV-uL2HGKe3#%oY0#K<R#eXT!q
zpe+rQ315fwB2Go{ZEeRilA~rq#w<B6tzn(bhK{pB(UP+|y^oh0cPz&VgQ}L2Xav@2
zj8t;<7`fcaF><*MV}v|NorZ`7SC5d2tsEc~>o7jp0mGj=bSwo|b}a?FcaBL6PrHl~
zSlv$L;JC}Wlngg|<7B!OgsEUbO=P1*+pr6o=(H%d<Z6tSl3n%{m+dkcF58BRoU^&h
zd{o)hDDKL3*@w2YnU$sDEfcd9S{j{k=C76_0nZXbuj!BsTZtZ)L7{p5APGxgXDtP$
z=+50zpv)j<W^0jIjpGXoOw;k3ylNdNG||?{)Qki4&Z|dkODZDVyzKRcMvYKyA?giP
zWGwYB<^<2j*(`zsO%rr0@j(Q!uvJHkNdk%1MQ@TkV`FG|nV{SUKWW+Pvqyg$KH2}_
z<@Xz3IkD&Aqi6dd@B9}iZr=?yX0FQj2Tz~$An=ouhHi*`-e;eoBnm8%<<(*`{Oq#@
ze;6+M1FF%}QtrM~9M~2?Xsw6YGu`co{ndy>!lD^^4*5UN(%E=|flNp*<ts+SQN@Kh
zaOr|)1!fF}O6cZ0%PMl}^<==k+&}~M*9IK9$agnaNP>Nd7`&8h)&D{eUuP-Wbpsce
zWn9GeZeYvfISHjB#p%lU4s%;~8Wu_7>Vtx^VINLvr$$5DCK$YCk=GbZWE{B1!-0?x
zP!B@3bubRM-D8F4O{M3$xDo*d<+bQAULowk6DO)Fw0c+=_Z9+Yka%mW%L9A~D3RhQ
zXz2oVmYhYP9>_IKR|#&wxYW~|`|<MBE|(_*)fj9z!mtnuL{%9p2^A35B&4da5{(AM
zwa`>oKxt`MmPDvgIJ;5=UcU?rQiJxih&57yWUB#e1Hr4Rh9^ik7!t$AG0ByxBhcb(
z9ySFcJ6&Q@>kNdosszapgRWSxsEUcqHYjLCxvD73AA^!2mF$4h35skZR0O2Jb~;Wj
ztTDH$yaK=jYIJ1zu#C-<8LM(EdC|p07^6`DSS7DJ;=lY39hVvOs@P9&?+L%Diz_ZP
zL?)6s>%v+TtxEsQ6Sw7l4rUE9X|DhZsZ{|dpw`_fv9&78B5Pexq}Ywvr)1jA_@qVL
zjax<uy#3X1(yCxJlo+dFhlxbIiLzMnct|BkG_L#867#UMe-wv8^ssbY$R(txO~&rS
zM2V$z<v?H+sUu>pSS>383rCD{wA%DYx)zp2`FyP`N$qR2I7?sGilP|54&l*q8UZi}
z_B-)8jZ7dZ0W7=VX$G2kKac6tlmHb3L2IHm=t4(D&K?=5PrH1LK}d~i#YlhPZT2c#
z)QXX8w1GNn&B7{z8CN&763vJLLX}sFhp>XyNg#~dL0m28SvhP0be+Ky<wnB_2G&1R
z+H07uF)Ny>BZAeB)CUQ7Jbz_8O^ZGmL0lLwN_rQK6|8FUKwQ(qtOw+lNi;xOv^r~m
zo2oRyd_{vXz_b3SBjmNVX<g_v_tXd;R$i?IAFR_1p!OzX5VDH0jxh2SYwJR(t+z%v
zSxwzqkm6d+uv*n@3|i5vzQ?i#Dnzv_mF_SothI`rNOGuZTuqUy27=65$Dpj}BJ}~D
zhQ}5wCTpyq=JD#PN=?#0D-gCxic4MUOp57JXJIk(TOkmmgr-DZn<&B$+awLBUI6o=
zrBV3okCf!+wXw43T@^iZx1ng$XSJIV=~{!PSpnOHcWRPUHS@6@cvzohZeW*%?Q~^A
zmh-e;q{f!|oQ_ffdppds#jaG0A2z$kvfWrRWVt&~geIAL1jZ!KmcCYq_1olftL6Nc
z7&$ASBiH^F-G};ey%qecW2^dHR~vVzHm+9_WmtPft1`O;kXZY@K9E9%eP<x~`uom6
z^40lk04db(Ujj&^fnOg+fx$p$5P9|kogw6z6RZKCz`o!T@Da^=go$XKXJD|<d8UE=
z!Wwg}88qlS9GNb(*A)8(-6vWytmu%e9lTm(*RZM{TR(VnTq^dW%@)E3jAjkmO(O!J
ztgI)-1)aI(&iGU3o#O@=-8kr2-3TAORu`?rRSY%O-58<ONOvk37?G?)Cu>$EozWmP
zGPx`sw7_d6B8c(Ir7(diuSSIxowS}I&ctP1Y^Yn^bUzbnMmXyn3=yjwA18u~K^Hrn
z%i=~W{8nOzvjDmjen|Dz7_zdXHXMmz2Ad@{fY@3sH3&r`7EG%y;;m*)trzlULnrUP
z?cJ>!?4EQ5?7pcySwVR^-?lqR?J6RsHw_i6Wscr9R@5C^keb9>Or)ABhy_)s2W%P=
z+f(T&^L5ExyW^X{nk;FJA5fH)+XqYeYIrN{Z==a+01IF>SMSxOU&%>Re33%rm=1*;
zUFy}OBbunS*1Rf7y96P-4Eb6~wALV1aifs>h91@TanoffYKs%25w#XQytn0ISS}`c
zeo1Snw}O0hr5Z6}p@gW-xYS9|1)(Nl@Mr%_8H%`TY7rZ_YuY1$oHsp{G#nu{;z6&y
z10M}9Y6a=FE?CSLbwoBVyVy3>bnFPCMrmamtUYP_YCEY=)vhdSY8$fqo^M*Vdc#nX
zG2yf245G(DNqNSi_Q%Vo@`xMyL*xSH$x+0guFmBckN^{g;P0)66a4GP5DmX2JvPT8
zQH^c0pYG6y-fS|OKyaf9(@lYEGaHe{D!LR14TFf~O?nrM`>bW$XW_U}f)X3{S^Kbc
zz51xnx{bOy%?Smb>g?dpu`#K0VJtalw7fP9BxIkez#1qUA--2X+MknrnlFA*#XaWH
zRNUItZCU;>{K8yd&^PgV{8GgVrP1RGakS7IQic8wk{ui0%9}dD+!mITh}a$O$U%e`
zE&g%b_EBz6{%0Pnlsa}}J^be2@CY)&dDsF<VsGF<0~6Q4CH70u*qLf<T`YdrjDvjY
ztrsjJ3=~0u(R5DGI5z0kT#ww`Ah|ILa9zDJC_UiG+IJo@>#`P)YgHJLRbfJ6^@359
zYmNeB7zJxP@|K4(T85c|5qnhR*A3fMfZ(?t1pPs5Ru*t_UO)2>&)Yk&<h`6Ss93mp
zJ0jW|7xAl&S63THss+0!UQr53q|+pWaC{CMj?*e1t%DQck^*LY5S-|=b5=${X^DVR
z-!@f9><vuA17Ob;6SAyI@he@3sI)GFZ<@EM)THv;#BJ*aybpWz8p3x+>d2=^ExM^k
z>d=Q+4Zf<!>hPyXExNBq>d*(=!XXHJ)v2k=NEAZEFBj>&u1Q_pm@VoLtxh}|4o*Fc
z8i|OWre2MZIz*3HuST6PG@QIKSi8Yf9*QRL1m)uB;SSe0j=1l!`r{S#$1CdF$+piV
zDAx`YINT#+@e8y7qoKb3+yaWG+GAb2%MmE;*G?wgdi>amR2{cf9k*5$*wgBG%hPa*
zMOu+FIywOUXhXC6&K{gwF)}jLQ1Y;v#)W0}8e1S;<*GKQ6t?oN2}<UiY5*^MY5)sq
zFXQ#-TMIlJYw|}MY&yg>%WKE*ZPJEswU2uZRqOax``A>q1>Yj28d{0Kr}nWKK4&fP
zIctMY?c-iU)jE7?9~-JV#Z&}9v0}nkHZy8#2b9B*lucZlf#VGqx|Bp&KVqKyMjkL)
z#l2t)&Yv(ic_19HvGUsD`wbGgR?~HC5A%;bgYT68wpva}N@faTPCrim6X#`xakYJi
zd0L|p_F93)iINYj`6(ue!=jw<48$x>p5-o}9*Fu{_7aF*u!Bc@3MOQ)D3O$C^4pxq
ziI*hCvpY#quhw~T3v!*`+|(EDS`#)|k95;MUIh@&QOr;`x1?;E%{X)PG@Fj*7d=_!
z5SIo?bhNwOM(c}c>Uaq){iG!}&65tKlI=}$ii~uJ=z&7@h*Zty7eo{tGd5Fd=e$I~
zO(Zilf(#?H*A{re*tj7ey|G}JHjectxi7ZP-nO^0nN@JUS~uK?OsmV#4RGxmgqS6S
zN54yisC(bI|68~Ax=BEz;Wxtz@y1^sy*POE&C~tiqZcn8{k>;tHl%E1)UOi>mF?~^
z5K%||RA_+<N)vaS2}U8IJifqc^|PZddEi0<-G#9Cz0^W{3fGF!X}k&9dgV|7P-9=G
znb5?1khpa29oxixhH>PS{XbAZ)sxPUQCo62as8@Z@NHsiy@+KS8x0X%?aG-p2N=B7
zO8O&$2#SJwoL0(eb3ngESI9rzg&Thof;+I?2-~>fHzJ!;wMAKh9VG01U2bt5R$g7p
z_$s_k-JoN+MP8)|x!$SDEZOH)Ya}4ntjL}2--paIKRo(wKLD19E)1slfWOS>Yr>bc
zo8SsjiHaV!po0wC{(g_6?yUvkbvkCLWy`A2NFL}NbE&D>y{19OBEPkEhhLXz0Te`?
zI2<Ms-m@N(IZPJj<q&`v6hZ}{`gHMD_?2tYoX{;ltFkbYm067pBc&HJWG%~7^OeG@
zai&wMY={|5s)EC3Y8jSvR2EeQ;XGLh?vg3jkMaizKMfbfBAp;L3yG4HY-eWcd)wy{
zS1=vI+WCE6DwDlr53F&pJFQIYN-jn%o8qYf&II_%3~dC7txZ>DcAA!BOu<#0m^I5F
z(h%|l)Bq%dO$@M;WtEM$ljLdo>xDb;kjG5Ui{)g@sbw)k7BHi{@~Nq3%7YO?UM+xp
zz!)%pmSH#E<k>kU13SJzp5N@HNzR<BMTYos=<`Oki~WPv^M51Cp+`=7NnNM9`<Wz6
z)Zx$6;m?NO$ju+uzT)Nica#qN{L}WtvHI&&JM*f&+w?2Pc-M`M=x&j0Z*|0NKwa+b
z9b<79+UE01)WN38OCn0=kQuf+Jl4R<M1daVTh1D2UoD*=lUTz8BZci2(Y_^`{if{K
z#)$SnvQ2lK-&?nxO{N3W=eRHg^``mno=xnQVZH^!0sp+$oHGRe0;EkdNI|C=i9_~(
zS>|uj38uO)fF6v_%Gq*aID6c3rXx(;yr2jA-ovqczWuq`6E-#HpAsx{&oA@OozDTX
zFhppFM%?f9AS2CuQJQC{w3VnArlK4~RLx+7&gr{<_<aBHC7?q?B4SJ$@*p&8+N<9S
z80d-`aAGoy0JrD=-g-E*DO5PMwIv<R8CZhUrA@0d=P}DFVWz>y(w*5mlrBe-42A?J
z7@tReT9e2mP6F;q*mO0NmgZ-s7%*ARfQKb0gA2-@aGh@4ux*)QGsr%NIG_p-uw-8?
zsrkoK$6`!upex!key&z3T+Q(gsHkEj?M$E79R9&hqids3?Bg;^Uk4P-_6ozBQxHHJ
zVu*K5)DY>QQ{-;|PEXUflGA6;$AB=|P2RltTvV1Hf0_}AYI0F7ilym@;7tJrcL_uJ
z3X0_)B(Dw*UqY}%^M6SBP46GpGC8g@V;T=HVKA$DIwH5+Iel{}W(ElnMBbDGODrfK
z(q6biYFDj0+y?AYxYdqP4xhX+TFruVV5Yv91QY<Wpl0P10{P&!wQFkX09l9PNHuwC
z$i5*1_416Ufa9U3n240PY|fi%utmz}GxI91l6kr~10&gDd3r`kS0TJ6m%6<+f+KPk
z;H7PNH#!jwsU1P`*OR5>Yu(^g^b#i9P4|xY(&W{Aj6&(jRHb=EAUyDZfEi}}*-AgP
zkvFo~bnx9Ut^c?SYKco6ZJU<5)egAJoN$h5*;$?0Sa^`*3&4R$Dk;r@v?YyFS*2o$
zqkIO0zu{uCT+`=laL@esSO;PK$<;F(rT%RdhZ4JiV!mE=$91{d`yOlX=(|S;hs~^k
z;b&kbrldEjJy{2z$<qnp39cN771Leaz@~&#f%@0rREzg*nUV;%Q2mH4ZSDRfjbT~F
zKA5?sC4D<n`--v*<A|`CK|;%JbA-_%+q7UC_ISFjNQ&XKGrX%Msi7AofW1yPMmjWa
zZR;86Msc`~!qDTQFhgF+Nyk}b$}kr2%H8m0wkEp7&|)UoKnE?2O~%x)kz@Oq0nj}o
zoZi;nO`=0Sxahs2VGGLc%|G~V1XbT6E8!3r7w0o~Iw6L2=Hm%IACZl3WzOB{?|Qm#
zP6iPh^nohoKV62otW%MISLdep0!Fke(|N!Fr#&s}`{D02F-6=Kq*g<I5Bwb)Js@g5
z2051yE=k?!^y=7Acj_wRU4y*-z?Aga!8ezsnO3@M9u}Aj-MOxKLJcRknl9cBlI!N_
z`XKr1!OO$_qod)AM~8og+_?Ry#=S09VJS*fWodPl5XQ48U)N;Lfw)B%^W-gFTSc{i
zQo}h2it2Lgu+2#E)oml9c5d304qN^ZvF~qUhR}XfiqRMQkDfd~eERoZ-|(%;zv$}w
zPex>ef|gw{*E&K*Vn?XxKOO4*u>a!Oo~<6)NAl_2#)cGo2NL{Amh=pJV@LGY7Y8r*
zn}!6NL=TC#iz1^RYFGfM;oqLW7~=T!*m!9E*v$T~jm^9ZVG5Pu#MPkft2uv=Y4qL<
zlEo_E#wK8ai}xw@nN<agzAD3~y*VT17wQg!#o*kM`h6-5Q23K;D7j|~>`8-cj6sC5
zM?6XYM(<4@cpU7czsE%xYzK#bdGyr6rU`EF>750exnuU=R*ujxgMQOr4|SWMt5@Dz
zV*beM9|h%=rekJcD$}H35G$ucsUu!OK8VjN!?;h(J+nA5FgEAfcxlxqBnVK6#*-=P
z+e{f$zMwN4maH~OY;abb8%(c2FFl&ir8qD;+1q&v_G5l_3AjwXcs_jm^t=7T-e{0a
z%z<c_QG_pz+X>R$TrQOrS@Xc0ig*n!m^%K~h8u)sirt$9Y^XQsM7`4GfgkRT`hE3P
z4oN!K9Uem2NncftYhX2N<PI9F;K;rj4GmA@ye-?KZvc5khFlpDH`q5G`w>jDW#~?Y
z(lU5YhT%|nvxYZ}HXhFWuSJ@*P~<noku@=59h3l!_@9do|2vnMjw>#m<dwN)Es}Kf
z%Q7#)@(iK`l%)Evs^xT!N7v#kT|joDxw*a@^e}Kn^C_@VgvQLcP<F5^;aJ0~o*8iT
za5{V%r!)VP@M;}0^P8bmR}x?r?0+^D__SBx<<XP+wweCEZQ2)YN>Wg_KEKJ<dHnqF
z+k@|9`~2_QJF}0zbKZYc8=netu%toq-}}!VJ$?H8@$fJEFOCkLANpF8zGt$8_Ec9}
z+^eY>ULC$V+JBOO;`6J!R0vkI$r3frRc!KgohlD(YgSpN@9Ud2!!f0s*kf$4wq{;q
z&AQmNn4pO2_JbjZ+UN1qb_28y?l+gm)OIUNfj8~SLN>A8%91LlU5LKTbuhQx%BCSS
zR;NOo*RFF}%4fH-IJ?uVC`B&LtBakjp73I_x;siNwz8OHv|Cw_bC5~74^I4SzB`P~
zCXe1ARY{C{VjrUDrAkW<JgVY;u^C6C#!O7Jy&qP&7pZ~qGX@F%8BRg*M&YRJx1TzV
zQ^#+KysDaqtb)8G;qR7eWJiwK^r)WqB830r8o`iEd-|zqp7Gi)!Cw7$ecbUjIQL%g
zqub~q4WNE%t0lY4rSkKi+v?gKrBeI_KiaAbJ#o4J{O7K9y<~vVR}pM^98xUK7N{QD
z`X`Dlx#;v$_gb7#F!G`cx&>`X)T7#qHO5m<+CrRETY--4`-xKG@7fosHs8^5TK~P?
zXe7r=)7uqz6ATG+&(wdfu>hDTvHp9F79f9O?dJzO@D$gsb434^%)50eHZ}2iPdOo*
z8ptV_Z{{yB&r-_z=8n#aqRPmCf?a4ud&vdsF#yP7sK53Nfrewcrt#1N_JS5D`VN7^
zwx{);yCBv+@Z|#9yCyB`czp|mB7?^12ci7Kh|iI-WtA8YY=E6tNwGxLojKf2v%s|V
z+efdSwwktXZUi9{xa||ON??_nmKWTl3^(+j_6!{vL=s~_rsc1h_HJ`#Ud_h05T+no
zxabzQ=}-YtZ!yYKXYe4vo#v-t7DnhJt3zXRCR>q!L3~)jexV*7Bsf8O&GU7}MqJ3Z
z?vh5tOIrq+g|OnYlT4|Byff0_NNvm!&n)Rf-7#Y#JMT&o=7=8<T!Oa%A)*`hX1spW
z^LX3ydgF2^cWGYbp7+{U@2@g?o!Xw-i1$IW@MaL9-i$#80}NFjuTRF6k#^jQyE7R_
z|6dlRS#wTr<3yyLn@X4#OhvqHBc>8fHWGMh`XqxaBw9&E&@Z^h-o&QZJc46%c=A2$
zsZEsA=Hcfr9Y+894Hs^WrBcGsO>LG1-}G#HR|O5FFf@19-;giy;ew&io1vl&!K7+O
zr1fqNYa%bV*`xBB;YZ+AbP5fUn_e%B)?jPQjwa|$5I3H7#=W`;157mMk94Y!KG`$u
zuBvCcu3Jh2lJLdmH)LO1=w1ubSgIGfr6~x0aN-9|MTVd-M^nOe*|DpW%@2D6(%IvG
zOB}M-9Uc*1buLGk9lKfa)MzPhxW;|&3Cs95xH;nzy2{?=W#v-h1mDHLe#wLH(}B8o
z0&ciQ67qiqw=a7kYyh8HEUM|MuC+w6-gNvTAk}JL`ZT5{##yx}i;Lfv>b))%L-lHl
z^})y^`ZRE$64D(6K6G|PB$^4Ub08*5#Mz-C5C*`_$$D&jka`>yVRdY80DoWL5d>%H
z5V}qWtc!$c4Z^~$6Bth1sS|W4K3SF+;%1fs1{cYh!GEz&laP2a`E)ksirkxcVPjaF
zi^Uljpg}{XDT5ou`8E|tNlvHeBP-{s!qr3q)R930&E6q~A65=<zzHOjf0$i`g%f9q
z)0|t{`@zaAqQWE_Ll$w*hy5z3Z?6k52Miw&O>l592x8*`UG?_WZ+gXPzR_*eHoq1T
z`96ja@zVNuUFM*XzDQsAKLa2RA+G|F5;EU^5+|?SYg%M)cX69&0I#Tl?jPKV%zn#X
zXOoLwKC|x)XZr7dCt5*@;a*VGCdGXPZGSCh)-EHS{o$(dDX(o-IE~UeN`e?ct4rzk
z)^&BDNU>ql4<Bgz0EG8S(R-9^iV)gf{Y0*O#VuV`za{!=nFFQj;@D$#ypH38P35(4
zNxtA9Q8~ZRuPI(MQtVPTX>}75Mpc-TFB_OGhyg-oY?>}G(wTL$t)2YA@Du~UVVEGi
zQjp#Zj(A9%>xEZRQziy%LppcC-CQ)JcUpdR4NI;oyRZrBTDO_6N4w0{P>iRjrI1G1
z3115rnIcqS;CyV(eytih9M}f>fsAFPQqBRM(K6_=X^wqgxW~A(azbde<WI;A{fZ+G
z+7<->MpJ3cvS+IQBnsBpPWf=!un58PtfhWSg-llpS3k@zwM%N7WW8g#o4@g<jY(|<
zs5BNI>TO8xOC-f!gihBr#H~{CQWDqLm>X#W@4TDRWn37$db?d#-0m~yV9fVv@uUBP
zv08!rKLe%JfIoSBrsL+UC|+NJL;V!U^jdsm$SFlaTVp&5M)pm`zI53t{V09~Sht};
zj>xVT0gYMo3tW9`PvsZ&S|V9f(41u`s8Aa^QtE-A4{B6))~epUAV_VqOl(pmU95F?
zcOQhu(sh(ae_+<ADFT??hAuFt6E3elPb6{}r9Eosqg*c<eV#mQx<C++By4qi5H|`V
zCex#^SMMiE^aoXjGCdHw@F^VeO8CDPt*fa+BcW+5f%Oh_d)PHad~Q=w&_1nuYJAKK
zNTuUzeHx1hb_D~9@%R2#v7V~HCQ(p^5aFugSm1cIO+`coVN075i^$=OtA9J_^<ES)
zYE*egk0x4UvmSjA%Ug9+Pb@QFvHGNwiewYm_A&*1c=Y>JuD|d3Mmf$-Qn^`vS}b>b
zoMgrLTXNXdr*UC?pLeiN7YQPR{P-B)JN7XW{IX1E3-I&#zpox0zI^qp$>WiW&r|SV
zgxpE1yo<!2OztttdCWayqSX~$&W`h$DT5i3<NlQ`-OkRaw)|~!&*b8!kl_hoIC6)=
zH?x!CN4V(S`3ds!o^pZKQBzAZNNn8L6n_O1@-ObJXsKMzR*cIQka2b5@TL8$v-yrm
z$EDJiKH~NxzZDB^Aw?m2o6vMNft0-im%Di7QbGm0mN>gR@Z7_}-Rf1pns)|D1it>-
zP~+y`C)n)VI)jb}u?lva33%W#8?S^ORcVHw9cvwu>Nw~OP21gjh3DD2@U+8l5Un4Z
zlsfm*;N9QX1q{AP%Hm4~@ou|&C0m&J;WxN{eb6g;g9q;?>gCg7?f${d*xStM5oC4o
zpA|!gu0D8I>(XXTXs5kAukI%FAMM}pkyqSzThP0=>-ze>7aw#@!O8qnv57g!&3E$T
zA?ar4750teNK7xSMKti%(PRqKCkrDfzNQ7&r2&8XTSNXoVlNStuCSG;fse71esk+c
z{x&(~kQinfevFJOZYbNZy+1HR<$8SR;;36^WJp2$1xhaU$Qp_f^R&6)AWFN_R$Xt6
z+JasL!8(GUzAtrC8(|ERrkErjFdACjx0?bMjU<B0jo`Nx!mMmUxt=Rwu6r5C1e)97
zo>e2>SXQ?n`LlH!?p@bbT$Xs(ihI|2EST`x8wtPZJ`PH8?T<s=S)5T(p&5>>RN)|B
zp?qD0Bh4YeB25MHv>F@r%Mqap<{KNRin-N1URS7VyEEc|+o0fz)NcQg2ImnR_1l{_
z;B2fhgj8EwT%hv?l9n0MHls<ArS<f3SZO=4sMx8SI^uTjXht#OI(5Tk>vXw8H`8?d
zCa=17B&9FY6%Mi^;JEB%x)IX19zFz>E`fd0f;$7&>p60Ebnm?a@ksZsZFm%{cZu0j
ze=`n6*WTMqlG0LNmsnEnZQQB1@q6m6%zjyBmHED+KPh*G9w#Zdm#v97ZV|EiB^Hso
z?Y4-H03dX^%S+F-+geJe<M-5Q$0e0+?aE%h<Fn;e78Yt>VqLM*OD!vPc-eVgYEiG)
z<(f+xpJ#&6(iVo(&5Kz|8H>9Q1oVl+pF0Bl7~kU0yN)(q3INh?C#Z-0j<4QtN6?2o
z+iF+nd$cT3W4cz)QfcLe!epkt61f2c;P$xg{Hacd$*(aC2quxOJrJ&SB^&@<?3(M3
z>f0;f-TJ+9ohzc;+TBVe$I-#}HZ^Fx^Ghj1wZQCWrx5+QyI#Iw0YW8nhw~Sp(3wU~
z-XD#$9d{y<VaHb>lD6ZHR5I-PN>oxh?}oFJ_l&dB`};=PHLN0<7Gr(PWs8Z<xO?{g
za94HR346n?uYkR(>yG%_H17y!g&_j2GA4teV#u=Apj5p}BexsHdq?l?Mk>v#6<Vuw
zoz@5R?GWqED>-+7ANREtM$?yd00}_x9{oW6&+q>}`oEtuTC1(yuSb9Q<JW)u`m51b
zJI6cOSJ~Z@yXhBSeL4F2%iWzjU*EZVcbtyXuhZLAIl7IeT#M~dw?1~vzpuW!i+|0(
z@bBF(zW)0E?0&KP<(FUH{rbzVcFp@craSZf!%m(4Ee$@GrpvM@7H#ELeGe!1AM#HU
zpV_on`+s}6fB16vc)0mlJ{wJz<1D!@NOBu>n&x1-EpLYMX8Y{=htN!ZmY*0>d*ZTo
z{rCP$Cf1vuWwUX9vPo2RlTZ@PF3C0<9!@4jQTCYb!XG4h{F^o1WQCb<<UNi<14RsB
z-61;#bX|p4cZl%g*IUW1rNI=ojYEc*GyD<i*)bvK34}7Es9Zwtlu_tTzpuR&)fZ@U
zi@M-3L6Q43j=0ZC3m`^WK52zReC9N&6`&O;;P_nY6e-ReXh*N}d6FhvGj(Z6r|Cs<
zmcGf7be5!J!#JgjY;uvH_>emtP!q@5C|y<=L39hc7;}LNTRG0mA^R3W<juB|<mDM8
z1ei@OsPY-;C}vf1ewNLktL1E2Ez=2(6jBE;%LbZ+d^_7tz&*(vk)Y<I23W#~>Elwf
zET+!ShHRKFGRQelK>$0`-h7gdGVG?zl8M<=X-V}f^E5h3&Wq(_oXiVk?@XH)$q6iS
zwLCe=N4a@v-f+K3`X<dM5VxCJm}b+Wyhy5InsM{Rc~)+3N`Sl?_9$e9Z-}f8({%h&
zyR!}spB^0U4<A$JJ6`f%hofRT=P1kkQaF`Vc%I<>G4eNzT<P!^gmWtv=?3sS*w~~v
zU~cC4(#;(A1(R{N^khH}rA@FCv2X9&xBU*#nOz+^)y&O_20r82>&Z8jIu7S%3O>a0
zJS+42KK<@6pLBfb`M?Ebz9?}@>3CeCtG~SK>niZo&Cu05?pm)lXRP;-8{m24yZiF4
z%Ub?g&8{$i<(B2sOx{4I_OYgu*B(%8a-WEYm1kF|ER&JS5ex6+yKI(0k`OcW@z${+
zU$2uznPthzayD`qMap9X_gQZsEkhB3m;Z7x@S$U_JXFk3!`F0~tAo3L)uD74h72J9
zPXG5S$L09{v$Q(v%Kfk8|95tG@6`DJoiD!lo&Wy`|7!gI_m7UgzZCcXThHY8f0y}p
z$^E}P&ED7k|KiS_ufDAB|1ZD#egA)ifA#(U>bw0<<@V1~*?{BPBvS_X1^gg!Wn|cH
z*hXZ>f*K^F+}wb_Hhhw~GQ3{Sd!~xZ)y8+=YaZiC#i1*d7klhyO@a)z|7w72bY2q^
z+u;tv*}-3qIDZoz4oK|HBa3pxI0z6x!1TM%!cKxdmDy=-Zowr%goZ%_@Fs`>^xXul
zUHw(3NIoQ4Sps;?AG1z%Fb<O1;PiaUvTdl#Sye}u2Cg1qgn;MZomm|`)EBix-O#QG
zph0fu65q^f0hZ(-jI)N^xZW}^>9YGEsqQT4hmUsH+uZeL!>w|wq~X9vY`Z?x`T9eZ
zJ1dANrYM^MhTfaN26f^2kfd0BPX&6*@oBGNn@YWECXPjUh1r8aK!a-zkN_bY8-9@-
zrc9<qnPF0a{B&6?D+U6Ja~5Y#&O_!(7HJRyv*iR`ZDtsy8;56|lJ>AO=K$2}V@r9N
z0T4*Ecb+e1Syc_GQG+e>#o2IC49j#TijTh@ygb}LIvT!sbol4t<L58-2P|Vtd9jl!
zmBk)7Aj`ea1_@6sZtUP#mn_Xdy8Z0Y-yT1Gbad2@Da9njHrIIIu%sVCG@bE=KtxZ>
z^v;s=vwU<0`34{zv$>s}Wz+ycfYubQ?;%9{$)6;wp37)D1NSNV&dE)admNf?N1~80
zway7ue_c!&l8+xf{(hfs@cRem?|0(=<DpX@JYk_hTbx7sQGZ192MLW^piEuX)2cEE
zH&LdU+)N%9^9w|J5Fr!}K~X$7H-IMaKq>Q?NC<5>2$)cy_!Aq-7a&cX7gYs1LMS{<
zUqj8Z%tng@QhwjYelTMEI2+{^T!w9i41&Gw{s9h3T&W0QYU=?LlhvYyX;>EzG!6l&
zs<`Yy#Ap&cNLzNYXg5fUrot{-BIO84a-#y(pq)p*??)xgNjGmXfat$oMw@IkM0(I(
zYP<jiXpidiXiCgR$s`W)oj7V^b3Xt8Vy6+-rPglnS*hZ3^U)E)BHl5M{z?<oA^w3}
zVLTvCzYNcO)NX7jT81mlyZ>MY^nrFfqBU~)h+&P&Xq?K{LpuFPtNY7jm`$d`S$58F
zHZMT<!rvBYJ~69+q)1nn$*R&VTI71aG=1DF^+XGPjs-droww;Vhj`#lI?blA!niy;
zyh6?^G*M8Fw*Irlf81%hHOpkPF6-6dpI!TMK#B{Kovhu4H4)xt`AJPV_Sq5@W(XNZ
zJs8oZF#6#KSNpIRejDb~`J`_ZZ6Q~sO7;{(rMU)Yvk`y{^giHw76;!+nThkXXpx@J
zCKra;z|1!|ZfFT|2ARg~S!6cv>~q6%978&2gTcqFKUvPmQN?LYwD0N^KO@@cVR1~8
z8T<yWk0_@sC{ESF?%Nmp`@wo*cAO8rLjBrG03%gv;lb)v-jijnuX9OdXT@+@Krn)Q
znorU);2rA-P)x>1+_Vu70%thQXM^AkgtzipNuT+oVlzX^8~H|`d)?a%;??>gbRU0j
z7HjzZ^XGp)8Xg=TygYdH^xz-+FNEcsmzg~U%;~&{GN2!hCq!xtISv4*Kv%y(=ZWkZ
za2Yeh7R(Ejk{bw=DfNTV6orSh@ZIBj_Z@&S46=h`qHY(rb^*6xo$f!7b(0nzDA>aW
z*aIWACxs!`n6r!7X%0pwyeV&fIi}R*+rPZajg{FE@5YMDX)mmM;K-dhMn@N=m%1#1
z@LQlYI*WVKtIkSuxR_!0nQc|We#oQNG4m?3BTf<vlVmHA+f0n8wGuKR984<vJ*DU}
z|4nXj_cm~IjJV)<@ciFUXz3E6_yL>!BG+gKP;##G5(r}HaU?ckv=@#Uw=``R;kOwY
zePLguq8r^i6-2=wg<Hf{E4K(^VOI3jd<@tR^d6U^AleV5l$M^BHoW-jdwWcrVl~Dj
zPPz8LQSw@`DibEsmWhc(JaNShE3a@9AwbZ0BM9xfQ7~MaGlKrGZ@kCB$(YVA5~x!p
zXW3-zcEV+uN6qe*j@|W63d{B)NaSo#+)sSNLU674cD=8+>!o-<VgejOO2(g+8`a_e
z4&pwg@nQlYA90haJb~-*jU5Sw9Y>91)ELZQU_tUeC$$XEdZd?^KkA#Mv1=UHP9`)0
z0@MAB{-{UJhR<L`qTtVT?lGo7;o<Y4rA?;sqxmEUfhX)SmH|<_BR=fn#NJ?#|G^Fz
zF&pcGWrAa!aq`<0bk}P;MOaP~X?#E{E|*`XZJ16^U4gx8`o}uWYd4_I=lXAOh7)Wb
zJ$drt&Nq*a_G?4j)HJN$vQQgpsj#`{-+p_v|FUt;|3NIl?}z+K3DG~yEg66tp&gAZ
zfIC3iH~^*r5fit%&P{R{bvVAVxASy1zWni}@1#lhlg!sz46Efa0!GOehy3C_9#!k?
zoBG&RA8%_)<38{yhSKv_z*hMng^ThyTY&KV{47Ih5n!yjE$1jLVoTb*;%zy=hL*Hb
z2?C$Xx8x)@JBMt3k-1DO$C}oJ??Zv3kZ~8qyuK|IVs_XTN;FPw#gD&rFyP9hu$7Xv
zp_0ve8_sDx`ud4c$M+z7wtI1H_MfjIbuLQrZUOfn_PtKW?S1u~)I8Z*QPkrL*1sXG
z0ioE<XtVL+_AY+hG@Ub@516_bG$I_Wi9(%k#+{{G(gzRphz)|jG<5EyFyn#?b5fn5
zy8-%F9B0`q8Kuk9vqf@zVV)+FtdxUWSscKK!1n_oZ^ef99|X!}Chy-Ai!;SyRml{b
zB958&GITgaK%@zPD0Y9h`e)cQsz0TeZsPJImj$TXH=}PrHS3zQSY*>VY6pl2<N4sG
zk>s;2^yJu9ih>oqMU-~qLNbccrl;N2pyi>^H>4y+Wy2oFI`xo{_AV^8ImFV_6a?vL
z+yEi(NKbE9Qcb|EZXgei!ZDv2IBvGm6VTwSfC)Dcg$xY){les(;|j?6%8UoLLAbal
z%)sW=0zT83jX0Q&M#Xem03%RgJ5X?zo#R(~+ShQ;g5+U%adyz8^R5PNTlIcJRfEt3
zpjB#^1_aD!9*rZ?f^gs4To33qn7?C-5x1L&({JBRZQxr0g2sL5-7q{bgmZtN*cOr{
z-|9@Ptazw6PQz!&Wxb=tl|CDku|#k-Ae$*)0hOZEGqg_&Eyl!5th0%k8c>6YS!pZV
z2*%-b*Sd<eK*FZRfsXL0$r=r8a!BU$p3Y3~ylgxxfn3`YF-n@;28m&L7_vHcSMaV3
zv?dAx=q_oABn=miKRJ)hk|ZFk$x6VZY*v`5lb_}}&_Qz@G7|tN%oH1NMX@#pW7tlT
z1NQA4rC@1gHq9hCMsu;k&ecrakX3A?nj6x9qkwe*Z6ecIvM6fVLy=VTHN;eqP~KB*
z=3Rmv=!V8ziltD0VxZl#8q!VRKu7ZnvhHAOQfh&2*}vE=5H6X7s1kdgnr%mDX3@lB
zt|4K6A5w<ad*hmBLS=@i{eG>0YI50Tlj)ey$x{Maur!)-wE!#9deD7K&#UEO-EGBI
z8WtIBv#7GEPr-<3y~EV_)a$4v=Hp0zsrf2~(J?!Ik=xT|`*>Ks4k9zXXkN?paoC2h
zAwpuHN2yvyi??o#+$&KpwZqbD0{(uo5QV`74f>P4B3PlFkjT0YI`}ixK>3l0@H)VZ
zL(z%q7#QopvH^4*oI!|&G=Q8B$({$_zLM(fzUu!TDCoWZyPW@@)avE~*y{gxXZP!`
zzO4KInGS#V|N994YW{zphU*`g7^VxdaD*Q(mNS%ak$8Anl5nAzyU+kGNyWy^pXI7M
zgz`3s(D1JZhhN;;^bKWRzI@R;&(hb{%o#rKp{REQgwE-4QJPCb&64-Y{x`3_!@|I3
znaZHqhMy=X@Eg-(6G1jLne+($<DTCzO4w?jzCAGals-0k9eTa4y1kD5UT>I%6P;hN
z-_?`#`vjAT58fp+m)Pqs%d~{3Ff%w~|M98^3PU@jG8_L|;1^cX*)L1*YKnb$J7uud
zzM#6tzLv}&_{pVo+}T6?J#caN$c1W|i7e6_cBoAU54GiNh9>PZNF<Z|C7qm>`HZwv
z*lj)=zTuaqUEAesG+$QK(GO(-%Fl}P_$D^spA#~$oXR`!7aJVV56=BP)6gaeQmF~N
z&x3TpT>f$NRaU;qN2D$V5+I&+aQ{CnKteM~vI(ksE4Igg=6FPDS*91*p6F8Z$pyB(
z=Zct}US<%~J(;Ax!go}?W@9knPj0%_e%#rXn1{o3xhVQ>pcux<yqeN%my<<4pJZ<V
zzBne#=mN3_*V=X>&FVH?4~t@vPN)kAFGc<EGN$wSBqvc0G`L7#XB7r)O41R9yefE3
zP<)C)4+&kLoRq~BU&HKf%WZ69W8)^Vjb?Fq)$E#Yc!NTtY*2@LZ-+N~eZAzi-J9tS
z)({~6w3h%lws2PIiCNYd;?(;F@L)qPR@dHKT+A~dq1kLX1wpJmNOH6NW-2n}#)IUG
zf&V?-8Ttk81O>`$v@FqyZo_=t4ZiBS8(^b$8+`bt(_)5#s$^^HVe<3rR~+ZNMAuyG
z1^b2GdaXkNARUbpqOJKY5DJko9%t+#Xa`}C?<iYX>VSXq)6+OGmEnsFrJ5AyW)M)B
z`ui)yJJTOQO@xAh3i>%?H^XXP7*c9>#c{qMZ`mp}9bTC8DjSh^r>xk?Bt0Ga$BBVv
z&v4~~@FUQD`Uc6^cD(|QM7t7qyb?fKH{8cBypKex)#tnRv!ajg^Os(2rswYSSN+X*
zZhINF<_~Up0c=C43e;7pkPp5)+<(%uo~8Xw9sG4;Zo0oy{W@Hv$3GtaaCGpG{iC1E
z%KXcE`22A6kKsjDL9)Bkaz025cQS$GV~`jNp88-@{LjseUax0Pu-*Rs`&6=@+)CgL
zm8aMA2;Q!`^;yx}xGi|}m_iZaH!Le<T&;Qa>%d^f%zfj)34ItKboJS1rc6t38vuYI
zO2hY$#QBLKpnuK^`E~RdR-`+;UEDhYc?9BDpKAA|7{sUCc~*jhFd>TJhtuK-nd9c^
zcf*$lhkqYFJ2*r?@XgQ8(-Ikn>rc{|p%~!(t_oduV7T+maHp5|H}U~+RQ0xY*K2FH
zt*txjwRNYhtuMTyU$hmy>lMA*R`g4+=$CCpzw(NH)mHRtujtopMgQOx{X<*PKYB&~
z*t&>2{vx%n_^w}AEh)0uWob@?>#t_jay~aVA;a+aD@c5Jh?e)0-8!NO_9gPrT>DNP
zHKdtD9-3=Mutl0l<e|BC1c9WPL>`)JM{r7-N#voqb_C6&nM59%Yez6onn~oLxpo8@
zrI|z?nrlb!Q<_QSp#@uFC@T#m^3YO0hQ+p}#6H;;RAF;g=I`@Rgf>6>Ecil=Hk^it
zRr<$W{o_mh<178+YyIO7`o|yj4*;(EwX1*CPTf$>@$!4fZM^>ky<|7(mVvjx2{VzF
zJ0+bq+#?d^#@}QZ(v~hc$Vq;d@KrxKe6Ia&&G9O?i)2#FPWwB3RaO_=m~+PZDw~}2
zyi!Mgi;5j5KD?wDNxFJp7(aP!E=RF?!EpD4JT^^-g%t4_kU*np-1Y7~Zol*R$eCL$
z7#Sb<mx%;eX%e9E!n2SJwirFae-B@RdDG6@ov(Iw8srCf>jCM38O+g(zZ%r}Gmc0=
zwW8LZo@8QQ>zkXCwQTz0J7k;6<!Diqea{NVjQr6L2b=B=NXioUMtTU4%$%63IE58Z
z?U)geO9`ZqH#<EgoAG6ZHsodwqs7F)6pa|N?bB_zQZ46fPR33*IYqcO3z3*lBHKC6
zP9RRvf?`x>WPc9A0DXHrOUrR`4mPl(GOf<cwe13TSAjOmI5O{oiH*Rg?c}HcdkxbE
z8dikM&E7HCN<PDCwSLpev@n@$Cx6Y7aRKoZ7G%9L0>KBxR!A4cG#{B;A~wOTrX!H&
zu!M^#LwyxtfwL3?0L;u7a|(oGHXlU9MiIg^T`aP)O0u*xzj!4?OolQP48_fmOlLHR
zDx0SziC2&Y&5Cl0ZGwQ`+-%1MfY6k7ZtH9W9UCTw?$Ef3fn#pivV-J0JYNr-6aeE<
zIp2W5bUSyUJBH7W9uNPr{{qOtyi8B0shJ1Li+b0$AHR6{|NQOotLp<4&h#DDAMHPW
zdGP#D73l{OZIDd${j4UqBYLzYlAHdP=+mAs0}S?xB_#GwhrMi?;juh>^>Y7jal$0h
zAlh*WTWf9HobOTegSo-7d4XDnq=$k`!VxgeGq}@dk6^w6R#@u=$sx&N6F*a%C4a4!
z6WgdS>miu}NPB7r;H#G}9zEVy^qO8^VC6wF>KnvL{<X2eS9UK{l9N_|>7f+CVA0%R
zW%%nENRjdHo|FhiWlHba+RMfE9PE?Q{@#Xu3DRZ=j_tj~fZ8MfCA8VQvE(%6#Yu0u
z==W{cG9Yh&w)b}%8`9J}hzK^xk{%q@!D$$PNS#LST~7qU)^9mTG!ThJ0hq;tEW%S8
zL)wY5%M6XzMrKpa%N!!a70?h4KSwDW3Swh&HcKdn=?-n1z1Ct<=0i>Xo9xu=jcHzu
z4AD+!8C<gYvZkVE({z;Z;jfSWe&iydqh~{nu^6kPQj8}ef=^%asfnl`cRlPBO$fLu
zvtLMM_4$mX{pHb<@E{wda4;Ri34H=w<`~pEpgKrUb&{Q&fD2f0Lbi&Tv$Gu7WD-H_
zHPjc#;Q?M`HpEGCyj=M0K;<fV`Sd7(T-3>%JWW903DMomT2Q1s6fB>Y`GQYwQyLqg
zG!Qfd_?&H_$b1VmW$c~?g4yMqq`hdP>O`j4#z{KLCm^G}2?yrQ<TP7Uq>YGxAeeqd
zG3JQxFu;J|mnSEHEh!swrwgB{$GMs%IKDvE29MFnWYbTLkz{W`-=huX+kAQ4gwRUW
z{glvgrlZ~p{0WiPZb)kvZM$5FwH`a=8P3MGy=N({9E3U#wP~E!B73lp8qykBiTMC-
zM0Qh??<`RdK(2J>X0NZPq*bBihHrq~U~nk^*Ld)bj^94uc$F`fsUxb;l-ZJI!x&Vb
zP$YlY9&gezhcqV48~Rk0jm6WQoElcvex7Z&xYJ7R_Z91l>k~EawF!8zG(?m%Xni;&
zn@MM6uCn~}2RsHWWBnFoBx2DH##ySV9k#vXKQ;FNm@-@SDyy=FX<9Q+)d800XqMh^
zoHv5A=%k36bxNM@csBd#&a}1+VFk5NwtqE6o?AFI_F=Pg)*+WIB7SSg-#153KChBV
zemnyFqad0FDhtCSrt_hXqy`7=r>!b8n3}&qHFJQOUC=V=!AnKU;6C;yD=(luxrMN<
z8wox!t;;eCj-@^$ID^!Uy$Q4)5|bq<B=_iZ%nHMY?e48y0ov5wV!ED3UeB}T)H+hg
zC^p=;p}0oKvl#&B{%(p1t~Xv+2**s3FUd6mDoKc1R|LFgRLi@8r=^{plo3PbaDr-j
zc<ct4<_!`YLB-_X4@eduWIe0^p)>9yuu!xDDnlMZR~ANbycG}eR3Fa8FZE{s2J6{P
z#h*O0Yu%Brx%WEcPmxZRy&GP=fi1hDeRKC#B!SyZ$KyzgyVRD9CTOUEworq33WY89
zW!oo@$sQs_H#;(Y#r#?nh!ps<8H*Bp+mQvOx=O@0#y4*OT82;dLYjpkAoNn=7N}e4
zG9p8UuKN#?`kedXQvxL(@`d+r!$&Xn4<CX1Pb>nn$`u!Da5FIKz(vwdJC9gU#Z0Qp
zpEiNt7m1cND3k*el^ntvq$KeHs}1@t+?tpjyVH!|gqUtU-)ukw8yj11d4yEr8RKZr
zLBV?KIcVr7if7!Q-AZpqMg;+7?Wmk*25c0!AvIqXb#HK(u|7()n_ZLbls4jyW1$`Y
zDvN8^U}Q#h5_O!D=dE?N^XV1KRp}IKm>`P7poy8;y`~uYR|P~fhAG+^%SRbrI<~e#
zgl2CoI~3{IjcsEC+RWYhI^RLGXr~xLP^O`XWkvAwG(UxdKMtE>GR6@*TAQ8MFu)Yz
zQQtJ5Ex6=$8#ttW3lV9ctrCr)8I0p>)N^BM1Ge<0N=4u%rJ7J{IeQPtbv4X^Dc3Ql
z6XFQ<ZN?ie;RppVd5T=nZ!=`Nc{Ra7Ph4795MO>%=sLbPYqwV`sY`&QScX}-?h#O?
zX2=-0W({;2zjLFr&aI|MqLb)hrB0<Mc~upo95XRY#tHRBCv^{wbFemB&QZRiB@yOb
zfoUiWW$9Bed_JZYMUhlM9knpQn+cyka4+_!ct04t3dgRXBv{C=_VmH4%FMz-0W>9R
zb_nI2J&k74rcb5#ZC{Gc;$>%HVHFxKGUDin_}h*;dqs++*Giy2HOuKKNc`F9lb=$!
zczqpTK>CAu41xKRM!6d3LN0<A@lPR6-h=&i3-3la#_-Sr%(xaE8OH!<95pIvHfn3#
z3wCSYUYBiHKN6;U<@yGzYfISNxeEZu(`p9&Mu>#b-0j#9^Ji;-=tJV~rk_Lu_<M(h
z;6M&I=n`VU4W4w1xbEg=cx)Kp;)F-BNeO;I96;f+$E;S2kgVj~ltVxMF=#R0fRG8o
z#7iy?6v<Y}tdR5@GcwDsmgcSD&*0bMys)O@qhdB%nu3_PH+!3n3?pX)hfw$1=zTX9
zjq!j*!{-o?41(o|Cb_zpjn2wqX44~+W_ezsP?gCbZe0M{ZOt<0hOv~m1Q-fD$Gx89
zwQXeH-MZz1Y3Kr$P6%>gt3z;R@7>U4e$0Q`c7DWJ>Eca>Erlg1bcPCS$$gMswNoEY
z_8&j``|$AP!Q($44S(2wF?{vIlSeQ2RZtGh;o@zYhtUIp14sGY`UQVNF<x|2b6k;|
zVP9KeU+TK^0<{WfsCg>0>MR@kcn`}bTRMP-xxwvi>WzJ`PM49!UvrTStPrlY!0I|7
zo02Z$M_?j8PZwtc*(9ipKfdsj#E{-Kyin1{I(yW@7(y%t=XLgYcy#b||M2lXYKxu7
zR<9`DTOTI7w1$rJ8YDkM$}1Sbk9W+vo#bV;z>GI|K1c?MCz%<>6x5Tz&4LyK_6wwx
zFnfNsWhHzhf}kIpK>~0}AVF^v>2v3&d&#$;J#sD}wJNUkE|<3biS)Ks2ev%z>7Qv&
z{|uIq=8xINuqU{|pMPRMBU9MUT(K|cq!F4k$Zb<I;Ni8?W{hg$*ms0q*U7RR71|A`
z5IT!k8l>VQ#x4W3ra3q_t|mhCF-{*h70#O_NO)wNr4<{xoM+WSYPi$Gy8l#+M37$r
z2<?U>2x){ok3$XLUQ#fvwVe?aLl3rwxebzQ*M9CN_g%@$HE0pRHtdQ*4}JRSAxw@;
z12PCY{+`AlABE}*@FFt@<(N%<Kpw|-K(rcxff7}*S2$9SqNmu~MLJt(*Dv$coW!yi
zQuXkgdxklvscb<Ymf2h{8x)$R!ZdKRrRhIIRRQdL^hJw|>5Vx*mz6(W9?l>cvn$S-
zZT&he&Nh~~E=PEPI(i=xq%jhF%Z3j3L<0`^D=#Nw9RUclL<H}xs#I!UTyQ2!XTbLq
z=~w_VNC@J|#PC{+gg$v|3rGa3s+Kb%o%ErFVyA0A!)WbMA-hbc7K_in0RnaOrWWgh
z37od_(A4tiZ5gB9I{JLEO*YB<!qIwz<xx7ps?XzZG|32~AgGb=jk*L{KoW&{4dNDY
z8EPU0@+b}<np`zj54Sc-wK(l-?M@qe6;}gV^-&b$THoZyT4J5KSa^(wBQR53&J{mn
zjYfm?H$fB5Oyn|+u=pxov7SV8)1_Wi!Y_5lMqZzaXa+G5>`5j>jIOV>vq}uOYDhIv
zpA)qO<LNLQJsPE&bHfp)wtZAF=^Dpm`+;s(Fv_im9wU$LxW^f=hz3de?21gZ;O0uX
zhI7IUg$Qjjy(SqKk%H#ixZT-NmY+~w;5O?3xHi;`Onbdnv}hTgtKqyf<Y#EW`?h8j
z!{$R2DGjqq&<m_+!D3Z7d-_f=lk+=A3J>22{<Gmpwium}0MlL#o@M1}hUS75Ej}|P
zd<LTtR!%PZ>F9z@m#HdTCEny?5KahFNtR<|WPJ`!xFs@5Xa`1<!Jeo-n@uIjBe2{8
zQ(Z7(HO&v0mx3z-jSTdCVDUD|PtO*U3#t#+*kI!Y;yiO#!(itSDZ@Nmu&E>YZ4KaP
z*7`#p0JBP&3elIjJb6x}V1+pi0%6VTYc}5}4>2X9cVssx8O<675gzT)E&K-;AiSEw
ziuU>(<J*vx3N7D&I{|y~4W2rrx&#JsQ5NGRSc8&<=m=xgS9CeV(O4$OTn<fZ+%DUu
z7jwgs<{<wr$1Ja4bHOo{gOw}M{MhGwRt0iWNBJ+4A(*?HDfT3Gpp@c?|1udQ7&K3q
zhX_EZd1(Oadu+k#C{S)l@2uFx#amIEx<{r;?!VgFXCH)OT})O#sL^a-QyN;!iuwFm
zTAFK!jeJvqa}!4G%4TnRF1AZ@bCyogq9!yU|NhZm_Q7cX)r<Y%cl$5*5C3AB5Wg{O
zKsgu1+BrBn+JE`#yMrfuv{&GIa+;5Ocvaqs*d3UL$o|@X(B1W|O}-=0*4Bfki~&Y0
zdENjIEyibOFwc%d8%KXXdb$5>cyRdbb3-Q$rhJzzjxMT2Ha(b~6ulc&PBJJo#(!Xn
zY>&?m@nZh`<Og!{D9UP&y;W<0VZM8M@Xh0)0Oa}eKYQ_G9{=!a`1tv&!<T(4W<EuG
zvWE|nJKJAbxqvNaw&tiTP{Qg;e{Rt60`8C}2B;_$Hc*C8r6B#{28C2gf^oU_0`bL`
zZjyfrIHoMn26#!yQ@E*Q0S0Jvw3r(#y0S>1)iA>9G+Ue%D5E%Kr#mOfFf*}6x3KTc
zN&#~K_FWM7cGEe`nY!r-+P7Z}XY-!#2xmo|rWSDN74pU#Zk;T|dh+cZ<EdIp2HNoG
z@$m467tbH>A00h^aWs5>`1H`BnZ?4MRD6(Z?ShhLH@TNstEWDF^>otOj23#rq?A3}
ze6$a5LVP7dfCh>8@-@6+Gq;<sC)v#W-lUtDD)mN}dV{49-BE}MmV$E#B&;_)bY|ww
zodUF>E66ZqUcgU0KB=nJ?<Ci_EzV7Ko*RZF>0u-E3S}doKl=P0Nl8Bc#y&l^PfzUA
zzJ2=k^Lw`TclPb~_UXVr{kMJkvweE1YCp4Y5AD-)`}Bi-`oH$+g{pmI-@dd@uk6!b
z?9*TE)8AC>zuUL}P;XQF_Sim+?9<pjW%lVr)jqXv&+JofpMJJaukF)B)t=h7Gy7E7
zr@4Ll#Xgm)c4gl#?9<Xdy|GW{_UWyvePQ4Js^0F{x4ZV~j(z&VKHarXU#i+)*|%TY
zr$5-IKdO>Le1@1-)va52>K_}T@p{h|xK}H{`wK%T+rW*uQMSwm-_mbkvrwHkYS#eI
zC_BcgV}ox0xkLS8LJ4U5n%#vqizAT<w3F;zzc0u4!+XhNLpi{N7l^VUNmX*)N@lKO
z3We9%#Xo1)9hXxxv)?f1O=^e}j-k1uXX(TcfN3_ys|$B&Tdp<AX$-DHGX}NgApY1K
zCr^vwb&{T;J^w620+x=zyAF;Y=M_U)58lc_Mw~TVeuO&$hM5#Z=;DfAs^%iNoU*nS
z^qt^$!?z>w<AYO|;;F-BaRR(BFrXMaYgTZf(!dSabgl`iS#utueIvyAn?lBh5n2q2
zKoKbMa*9m5j1=N2p$rnHh*=)0?M=sU1G+HLet<~y9*_mQ^wEV6k4{_Q4R3+YDBsrj
znhk3#j%{&9NZwd(gEnVk(=J_jKx6z6k4c_ZrxF#GUc9!4E>I5t+Z!(t0L+}^1Kb5(
zG*0{l`>)FUfBR;25B_7;WsqQxU?$e54qme(2fliPuiiAy!iIqI{dSxY!}J(bim2M<
zAe9`JlB}0?04~_sCm=WnHLKzL9^*D5n_LW=#!k-cVa0b#zLF3Q*8)=||9+4J*^vw<
z6^Ybkc&i=GMDp*%&uLk!fG=z1RcJ+3LA|Zjzz~V5EdH*Q;@FF-9KWfRlwgdisJyzz
zso|95NJ4=rU+w^IDqfhhsb5K(euTANy{&8H-O#cReB^PuYhWk=(~p-YKmPFO<@aDZ
zp4>`ye~NIL1KcY&-|ShJ4Z>$YVQnQYK@X8wO2pJGvD0G5d4+5?kSbh8un}9BJ_&UO
ziHGd9<csaMh|-1v8idY|$t4lYa1_GE)`M(N6rc@IU%WazJUIN$+!cm@dGz$V7YBz=
z2&`sdFd*JNXCBq&T&S9rd@nBKszgcM)2}5is%t+CU#h;aO8lBl<z~x!fk*9iRpv7M
zzpoy>cy#y@+yS5LfBWdw)0eiUi%Kda`kg;XCjyO6QySuOZoHPAP)0#_m2mF%&BWZ!
zE#dN_>$2)gSdCn<89VmGfJDbMMuqH=(`(mqYkq9rX6-)cABB)|7LpR<CC>8+n%nwX
zS%b`{Wjgi6Ydk`~OFE{tO3*;zun0m6SMzL?pX4I-hY6V-x&!JzIEnXk`{s)~`~7*}
zxkr6eykS8;XLc(YYy@$MHz+bdWjLPk(2olM-!wIVW<YoYk#%cQq90!De|zw^O<_`Q
zXN%G8>%rIS62IF_`Z@}T$T(V6sg<D1I~5_}tK}^@y^ZjXAW~MrUt<kYz3O6mTucnT
zgb96KB`_z=uQ_l5#|OW9GIcQCJF!v=^9)H2cbyE`Rwd!WIxFoh5?6bXRSO`RxE{77
z2wLz>Ppz6Tf}dH40qSt09s&jU<M=MRiG9JyqgOtgy^)1)U3O<##S1P!pC5iJVWVZa
zgHQE!26*=DW{QeId0hP#dmX0^+CxSXE)p@%B`|)Jdk3CcOPkeMjnv@do4hPA(~p!F
zw@!}=e1xt3F43H{ndFc!I6cCJdV79zXcxlKG;Ydu7RsWHZcW~GgP4IEui^j50o-^a
z$<A&dkK9+GF9BGFt9beN``)a-LH{70CGVVBe`^;u31YnI&Rlh7qv^bdd@{ZzGm?I!
zK{(xB?b~J}t-h(Z`i2;mXsh2pI(+hUfB4Pw=TCdZAfd~^qA#rzCVCU%xGMND_{yWB
zNa_8)6xt}{ptbd^^;5kv@eBRFu1u$^U)x1@*|5x}x&YZvxZw_F2CH+H3Lt`CALs%M
zcn7OJssC=npRENC*A``-9o@TN*$xgN-W_-WLt;XSQx+D1;Iy_(&ph@G<Isa&H8l8}
zZ7+QrZ_gC5?Lir8?;-HfZ^QN&0bUMY^%jGKPyn+<V(LPSNKDxy`4Qhv@_D}tdcft_
z)PQ(8(>?cE2<uqAQ($&9ey6WWfP=HW+%3vQu{2Hh%!1v>z+?f+x@9`GZIF@%vjn@l
zI*3fR{%Pe3>dPA_PTS>Q9|OHqB3^(hLXT@Y*ct@+>@&L-u&}!I-l9*K(pS)gZ6hCS
zZ?%2*H`lI#rEV0E2kQgM^ds$>_kOz@_1<ySy@@L1Sbu(I5k0s^ClRZRb5QFUTTuCg
z3;KMy*eXu8N(iExT=&1l7P)Da`7=#^1%T8J&_pFlRe0CFIy|C~#+~-9X$dtjxO?@E
z3kiW-mTUBYijx|rkuXF@ia|JEb{nUrFX7OLtFI623L}dSt#5(TPT^27JK57~cPZiq
zW~)flCO$>eN6&vCO`E_e##b@g#4zYd@^C{9)P1Lnu^XMo=xx)>4pc1;Zp(>2sXA`N
z$KOACG5mpXXxw*AwquW^NS$BM?Wc#_+x-U{3Ovl;6@Aj!Vu^bV1suL0`!~OU5t(UI
z2MV2vAL*54ky~dRZbaf1B~{NLxsHFYvk{|rPkoH3am}scJiVweZAlIW$_ut<1|bY6
zNI(*sLNZElFx#LYnKR5ma_9f;J^gOzV)zcfd-43$54Q{~&}I*6RY`9?SyHeCG8r0=
zF{E5N!S61hWmbWm7u;=7ZnIS!Px3c;v6N0HX?Y5!++cIWHt!c;lO1o6E9qugu*7*;
z(Kre7x8&3LBtN+j`HHP6^1uP*c_nMK@=Xe%b&}&^IUBRL9L-B5%ocHVm2ni+G@tpZ
zml*jqYc@Hh+?r<D<k760f$<NIzS~c32^oz59i*fXgkOYYWbEX_5G2R{76}!I3dPD`
z$ADGtD!kt*(F6z)tXaKbTL<(Xi%PJA<T7)8TvgY@YZ6~HL6XBR7HB>&O+kvLC5bh`
zoM~-nR&O+%<kNhiP3Zh?Z8^AbS3l-IIa=b#j2;VL6)1(NRK~a&eODwx-Q)`rw|xu*
zFSaZ=bN#(e*7&P;{oWusrKz1f(6wt*+H<2KxLM^DLR2o^*ukdyqm0|JW@t#SW*0qD
z+UT_Ez>*~^3yAYUE)|r)Z+<cu<1p;i!Qo5b85zxHh1f#YaBaRc#Na(*8@4x`LR!cb
z_@+u~jB}uRQyUL0GggFiM76N!=dy6IrPV9@39aanPqD^%xwsfYN+}{+?)P=OvRk_?
z6sF(Sf%v|HjPtttVb*@jA<p5^%Y$cVREWWb_8JdGWszx`Q@^eURL!H3|FTe5?80?W
ziCph{NYW^7@oA5*07`D$P#2Xp)lEBLy8cAPlh{HP#&!D*H9jyKpjR9O&V4=360;eP
zL6ty)H}@&X!sb3mJ~p=t-9FgZSV{Fn1%&|G3%^(`lt<}8MU?(1gfHnJzVh~EL>Cqt
zNSMRVt(YP!?cHNjP2JPS#_si0oAq(4=5NaXj*KnHhyZFvMh)n7uu=MI1Xtsw4y`k6
zktk2WU3ny%Up8==PB&|4p!;#%x#aB(5-LxpF&xDHVH~Hf%wd~W`AJjNcom8#dL91g
z!tVk)%yGjRlF^o^h-r%++FD~PiPq-0jA@Ub+8QKPb8CCzWpJGy6tJ&_ThK7+*Yt*q
zo<7`++l{P?ukYtXx3%XED>o{iBGvTf?YPS$k$tRouWZljw2d3`d0Ni7iD~YN;67Kv
z?j9guvXH*+y>~1Gi#poaliG@(&D)pzYz}LEHg7{4$n4$P3KTHSz@NsgB;pM&4mXFB
zTu)lPHH}>VIm6~!_9Rw-Je?2}=DPZ&kaoqkrkv&MLyC3L<|iCgy+q?Q{nf?VRcdbT
znZLa1!*z6Zm1PBq+gEq(!rDr3!$soe-)vhUGXeqpb@yW>T#h26M@M6`FC*iZPYoCv
zOvU18%a~8c%jsOkj6c>~OZ2>x2Jpw44NS~cM(V?JtUB(GRX3S=nwRxs-rl_SB!iP9
zmi0Hp5P9#_p#AcwBdonr=^B;lul6b|=A`AZ*K$>keIpEterf(hv|*ENSL2=|E|fc^
z6b}4OV<Fg4xa}uu3WKssSa~v7jOV89i`muYMkJoAX9RJUiHi{gsqvErZL<6g#g}83
zXNSWxK%Q4_v8tu$+GNWL_)ell%7YL|*#(Zfn<BsqZGJ)Y+?pyZ4$BhlL!O6#yynA}
zt=#81c=y7eSyr;cZt~5ty4X5TbBHjq$R~|C$iP%=k}Wv96}Lvwrnp72<S*zrig`Y&
zi0dx%4;|9jS~^bWK*#+SBVL`&4cROz@yP#s4Ui8!shqohE4rsU5}3lXqV_Y1dD(h4
zYlpWP5K=S<!M)Q{acrHLtGKfV;s<MT_(O?yZZJ}?>90Tl5B}f0Q0M_n4&ikladvX_
z)+}PRPtwYKmT*ITYoIR!LnQ%`aD42kFk&di#~IqW9H&*rZ(?~DoPc#jnZ;yPi3V8j
z_FPm{Z)7zaQRerskVU-4{vt`PRD=%Drij$iOLT>Cu8*Rvie)?n_s(X;^7JgJM(N23
z2E#oSNAq(>b9C%|KP?$~p7zC>*b4<l5#;M})593l)@}Vp3I=@~{7vMtwJ2}NO&zvT
z8sS-~Ffo{M20himgN75elo<u0lDzej#-3+YkCxW21v#Zv(CH-lHZDvZ+<v0-SSoTq
zcX~rMkQH>pnXaRJxg^)pH3%KWc4?zXd+~_aZ7upyT>KzDex8PkBQJs@zNA%;VPBYB
zy$%a^m@Bqn)e^W$m|OvH+Ld4wCr|36SMzvAukRn|a2-B+`R(%;<nw&3R??+g8$SM~
zpOAq?wB$dYAMX2&kP$Mzqk8VkaE2Pd;b@X)h7b??=v<vVHFwjuFDaD7=Qe?6x-hKD
zYr`>F2Wb~cHYj6FN$2SQY>16}Q&d=HIEO8<V27Rh=aD%)$NEg@a3q8ym7}44GN^+g
zBy#o)UVk@%3wCwpg`cl6pR@%UPYQRw+L#;`0>WHEjKQO4KY&yA!ce>clcHowoMbvv
z1Z-}$!{Z-bCC408gw0tU_lIE^5p5sx)ywNVp>Ovgn%U-tMA{C@P`ZzhgsU=+%02~+
zRN&~&?bzIsV{@=W7y`7mFyFll3_e*60!Y?#H6Y-Efz?XwIrNJ8oxRA$%Mpxd=0=K|
zwWRl7V?Z)7(G5?QcOA|!KWsBasC4nlu&BzJP^o~$kbeZ37YYtU!8{@d9=pWVW_@ed
zX1Z{w)^HX{fLgL|;N*?GU=G>Hn<I@Hf2&C#7|h|E?sPikyd$pW?$w?Ot8Kfwlg~vs
zVp1I;onFe^`;swtBg{anf+4JNhO!bgeuwRQ^snPJCUUi?-P;)2*V>p0)S-me-td6G
z$^dovwzXP7*Nq65S!NRm4XL8OKmezT5zst1^n!JFw;hf)-nI)u0ET=+TjsWw=&X8P
zwYvf!pS>&Gp~M}H@NR244)f@g+(G7Ad#}B&-Ab@xbZ2VJeYZcWrNjQ1(T6yG{1^Q#
zaiD~drtug=oYpIs<Hw48y=cp9z_;X7cE6fKa=1_#I#Bh5%pJ+1M~k-<Piyg(OCZ4n
z?_{KSoL5WG4a&vvUZ*$0z9D1M5u7A&fS}113v}FwhG^V|@qcW>y{C^}e76s>Mydi>
z0Z!++J-G&nUu57~l01EG0>aYG^^ofUG3l&fjX^Oj7PYXQ$$UT{b>S)_BV(W=u<Ju{
zSUB&mPKxTU^!iKOx|SQYwTI!++7ez^rhM*AO{qmN7Yc1ipzJQZrk8q6+5BSj*SwGg
zyRVQAIyi=C9fK*Ms<+6OT8|({oXZf0c&eaK7L&<wN_0oMw*fmsWO~QIo!W84(1@Ce
zAq7zfhQ@I*zA)D)3{N)!J_F*{A@77C1TeFTglwlkyQkR<Qyh+MF7ydzFvKtwL`cvj
z{PnCjpE+;<rJ+=yI?Bf>Y00yU-EZ-p0;7Rg>6m;4T<4Z5t!GD+#(6cR`jku2O~<Fg
zPl6j#t_W&t#!S-~p<A}J=RvKrvC6S^lLJLcQjPio3vU2@+DMYNp10!Wy@5O=kr<Mu
z8jmSRBM}^#I8O9}53`Skh>E%9%<w?}aTL#Ss`vtm8VQ<<fKd<?6{$5ec!-OOXw>6&
zF$}dOCv?P--+>=J(Y&*co{Kz8cA&AF_LTVk6_9f|BowpJG-5U7fJ6=iP2zjr+i#js
zwAWdgzgk97REr^ccyQK7)0IPK51V<E+N&3KokH*B9E_^TyCO!+ZFa(I*a1YM#)}=U
zCra9EIQ{^E^hcKp#M>ag#Yio2vsjY&BKX*a*a}?!tLpXdNubi}qPx6Kxvnun3(TLy
z^S&&e2CtxjXSj1(**|>#Z2uXP-Wukcp{r?M^QO%JTuO4%ywdk$Ts$p_#{psJ-!M$H
zjr%);9Zk#7Pi{S+Wu}p}kbzl%WBY&m;rWY~iNZPa-$yT>zi_W!>>s^+aq#%1d3gEi
z#UZ79wO<dPzx@8i^S>H;>)Lqv@<mS!fc%zG$=ciPZ`v3MVUF4ji_UwSVZ2?9(9O9#
zTwE5H33k)ZZ%YpQUIuKyNnR_G9f!w<O@DiX1C%b+TjKs|ThnezdlX;yOr2EiT3qt3
z{afzfGTqzCuU@`*^mxCwBlM}{c#QSHBJ^9Mxb%VDg;0qGKo4#iQ`p4$>1@?S0}Fuz
zen8veSBwvGrQm&BrFWLS@N!^>hx(JI+r_-)dZ8w$(CPzUD&}=AHZqR&Zxr6DuCbh-
z{`H{=Up>J2y!XcX5dSpq3_g`N%sFxcy9PMCsSU*3+Dl4m^Ba?1H?BP2FX_h=Y#r=o
z0(1ZczC98Y^R3Wk&mA2cC|EYD*PO)83cwrV&WZDOyg5gml@}@jUQtX5v@vGAD^uQn
zWZk}*JWF3^<_c7D;6pbh2y8;$d!qiH7w4qKxZ|{84$*tADp<B9H#F%2_-u+Rn<{xf
zSQf?4U5bTkRGB$zAjQ04zviY%OdO<RB!e|O9&2zBaCEh0qtkW&;Lx0C!7|!BhHz-f
zB?C!)ut{sX)ADubmMoVK|F$8_#cDL_a3$eJ>O{6qgB0~~^)5q#T=-;b;ZYkrZ#hj&
zx!R2+Ks!7o|MAN3Yq%YmMqunE+Q7V2JJD=(jhr`j)|vU%#E<H(&4`EGcb%2uxYBLR
zlpggU%(o>2V{XU!#H>YLd@jr`+C95y!TRoPuD`k0*xTWbzRv2(>wIyu*EQmB?5=(%
zctMx3bs!Mg>XEw;oHgO~rsdc(HyOyr@OF5qGtWQWWNPmGltzNuR;5aP`O*f<X8*Rg
zygLXRUgn{9*=vD?zWxDR%*DHQSoh3J4@x)i>QRHyZl6sx^u12#uDjKn|2E!Kep3+M
z>Fo{@0^`Vo-we#-*S|QMYD1<|>s+O!8rWSdrw~Su+VIxI6><PaTL=hT#OI>N+~{V(
zRKr?vsQ2Ch3J&;t$!&=KM+Q~TpFbn+hOIa_DqxC$fGjx`2$tDc?ynijp^0@u6Xe<F
z3zhhnGT;_(vhoC^+~XxiU<M!C+-2E>_|aL8Hq^yDD?uiX=^7wf1BHwuTS9XMPaud)
zeqw6QARzVRL}JPv=L-yLn(_jQ;fV4pRGb=sj{pa3TF?26f$>Y-p@z<eJHVV7LhTmi
zHo^WHu3#UJ?v-F>6cLbYQ-aWSMJLjYNF1&vJsDj_da_~&>j_XsD+MU7yQ>OOT9e2H
zGL$lF*VR(FVlp1;LkWJ1ZtpjesNjvbRP3l7qZ(W&DB1a#0+LmO<3BrB5tiuXyZpKZ
zq7#(lD>+CgO@UTOTwnz&s<f4_#AGO@DXA^xu+>N&MIJj;Sx;<pWjRg5o@iagn7n3v
z1o*JB8=e={@kKTf07l152ZbzQE6N7?C-B9NU?{d9=1hxP1_VNzSBI~T_MZqEm%n!5
zNV@Ae`Dz0fOO-QP1Va%Q7~oR^=b#4jC#ThfHMKg)_}vo0qc4_wf(HUmE)&q8B&J`D
z#GAiF_hHNZ;HIn6ibn6cJ};oOm_U+$(HQDbL+ucIV8>%5w1ggK95Hm{PWfwOrgSJY
zj=*lO`my}&i~ao`7N$OE)3t1ds%<y;`qxf-<3E2htU~LnV%^&4(gj=cs)jAzss+Y8
zoe<}PUmnAowwwxe3|x6dY(_CblJ56KDQeFej%)6tHhk@d-rh+XyJ2c{CJD3F5x{SA
znGxV1Sa#6N38L{COu>}AMS}PW=pa$KnW3i&7><&gt_k>I7(TZ1UO$zjxsoK^lAbz;
z`p%R0TOqJ;5NeJtL$8|Sx|`UF>Ui5(N6Od?&D-uoW<xLhI;Y`U8=Et2I4pXg+e`k`
zxZ9I^qElEx2|HL`1b-ayqu!|oSTkUk)Vy0uC6N3{27?xE_mbK@fMFFfn>1P#`=-qd
zD~LSQ9CnD@Q`=_evBKfV>Rd2l^&)Porbe&!CtZa}-cj45%3`FioOh$*2sP%l4LL*-
zbxA{v&c;%9m#?swdcXR$d}g8Dzo%4eQMr4lPJ17=P;t_=mT8nFQpyTyO-)TVuEV0Y
zr`xfItCjK8S%Me)B;%Q;7spvly2Br!Eq5E^Z;r6pa$OWYv$~!q!iGdy3*aE|6tnCD
zS^mY2!Hx%0{h;iuscGNWU60%Pj$Zqq?mXgmL9*!%&K4ik+XKv|%~qU^({hY5+!8`U
zh3+0<NR8)z*?;lk;E6e+;LX8zhaXP?f+m5Y&{;!Iw3>JIO`2CLjaX+CYK+2tB@Vfb
zF#HSt{<si)v%;wIiqL{6$(IHcyBkzudt_}u@u;tgX-3(x&3{kx5ghpAY>|x?AOo?k
z*ap<q_8Fx3gvK};xl6gl8xOHtLX`_sEFCAi_LF@#?4|w06-+++EJ^F7ppaLEedv49
zNtv^3*UJ{(l2p9U(4l1C=Ae6Hwkt)`dGYe;li`#7{U5%89ChGbidpc+lVYCDIL^Q*
z1^=$s8ECddrv=R|AloTee&-cr$ih^pnB;quC7gM@ol-#~y|cw)UfsKW`*?Z!Yd)Ez
z+tY&n7Uk*f>df4X&6n-bY5u2t{NU~%zxeX*Zr^!%K#*>PKyK4iW~CWqmYfyEYt)6|
ztU02E?HYYd($QiGP9)IO9Ek83B9dos7iDU`pDxjwL^7gdnpg<5fcTe=$E6|F0|pT$
zjy*=G7<?|uY#g|e=+m)xbBL$m5Qf-SXCkF{gHL<$nm3^;BM4R2dr@bHYimHR0YUdZ
z8KlOz=-%dMVPZDMGN$B+XKiB^OCoR74HP(eut@<DY&GcLCx|wKJEQi?g5wslZxAM7
z10^f^3n2;&a2at6aF#6D(1bN7kfn3FtQOGTvB7o_>BHy%EhZH>Q$W(M89L~ndI-K<
zgqSrRWAZQ5+CubuGn#FWZWypVI)V*-GV5;)hv{Nb=EuuLHXNb_U0Nl*&*&=#upi?l
zhG`mn90@JU-p<n*DxwRZV#9>u1WJf4%JhqRVjB-c`u_n~NH>He(ZBF$*f8q`vy9o&
zC%QV5q&=#|l`Ok=lWxXdf99JMmrk^R>1I+#M_r%GhD`)!3@r_eqV=N;f@>w1G#3sV
ziJXyMy*JXe!8rq~h;`3Rb76IcJ*t1)6}m0Ia^J?zlQQy~emI`OJ2HWx?8MJ~JdQGC
zBpVf+7~~)){Ci93Ya!xzF^*dJ6K4T-KUgyAkteSdX+lZ*O>x;2IlC3KyU-N)<f5PS
zD4p&N8+_yGcS9nU&mJB9S;v@ny-AwJn=v_w5^8ugY?upd{JZ^^!=oSeA0K>su>XX{
z(D$2kHm9x`b=9`~*-c-{bTckyiuwN|wA9G{vH#*Z49pED3ZR#dA3gqlpTXn8;H)nD
z;6eEQ5MlzsrR=7Shby*Oa$R#@=%I2qwV*LKa!-FBHsI=O)pBj*5{<;4q|Kr%VY9}y
zuq_?{HwfKKA*vA3joX&A8H%^S+=Q~Zt%okKvz7h=MqEWASnai;cd8dOsOAh_;}8O;
z|GLJ@ZbW>DYlrLpv=$hF5x?i0Ne0b}o@cG2#-@n_4}^^FLq}LEUuC%z6u2)guzR6T
z=dX*Y-=*g~u2i1fCyRPQpU==CiWea;T=>XK5m@?k@r%-~1rIozc2#qgA|X7&q1}(F
z)sd=>sjUuG1PqRctAX(Tf;h}#Rz%LoDK_9vh$_SO#;J|MI=f^kyt>5k+aBj3L_#q@
zf(fDRkB@?T5_d^(Ftkn*lcYc((ZETFRDuW21HRz^OB?S7f%k^M_XbmfnCV)ZwUR)j
ztCv}W?b3sq?NST6=!4^C1LAZ*aAjB$4(DMon)+aB=7sNZNZc16UcZg^n@qx5MKNLX
z$r@_c6sYDayjw24QD?<v_i9t{iubu+f5W|b`5hvNNPWhz@q;Y+?JEU5=@jooz)_EQ
zqOe5Rh)YL2`45V5;^;0h_{wlZv6=BWD60aY)R}^-MnbtvD3sP%C~F5p>1zMbEg}lB
zy`o<BI9~2)MR*kEoBf*W1Z)Y}oj4t8irEsO!5td-t$l{P;2Lim$bf8W?ITn>7_6D6
z#5~B!gJdV%ToFGIk34J32?E?2P9XDm1Pe&C!uSz$0$H#B77vis-8Y8Nm^T=ivhT+{
z^!%7R2T$BF4U(A#-am8)Y2w#9x)o?iTZc>L>I>#?<59Bc?oy(;5rltCVsN$lVY|D@
z8e?CB)4uFgv2403c6CUpp;29x<XtNp@MzO=X%?6Mlf7Ib_r@l#k=1@KA;#3UdmMXi
zdSqsW=)pR~tc|4W;^>ubeC`N|@t1bX@;HHG2!x4VEc2IVUhv*=4*HixGA&@Ws)AG4
zIOkE|-$8Q;Q>GS16LV^!S;Lf+(ij2#yjV`gj`PPlkQF?u427VI3SKGazvkpTGF+GN
z8lqZLtqOGd<9wbkie)vqNGiy6a*{)U>*SG5-#xid3xv4|%&e$^V;YP>oec7hqc(k6
zLM(hXT|Lf5DP-w3>{dS8nomFziy=p-eyNgCk|G~t>>F^I*_9Nbn{@e<coEvchF#2e
z$I3Ps4cAPDoBW~lB|f&*9KJ>k?^b$5eH?Rs=HTMW|B#t;<WS$$LSL(pSnpldEWai^
zi0QbTCd3kW$axs(T7bQY$pY*0hZPPinm=9N9=^dGK6W!*?*{7V`ARL*CDlw<*D^(v
zO!crdBBS$_OsghYR~WtrjbWq0@T&U4RqDdc%X&Ag>$T*0Hq<;7*6>GcP{@$g!@b7n
zMwbY06vdi4d(!4Rpbozq9+T3>i^o`o77T9SVAkSZt~!#g?iy`=8kadO--~23;=^YJ
zeIM8Me%{scJ|5x@g59pPzK24v^y?6Qk8#j}mfm|jCGh27KEZ{&ZHb&zhvCkpCdNl^
z+eMN%gw6ad@BOoEGSA9Dja_GMI$R{t`m<FY5%Du8;PYriSsUH++<dV0JuQUiH5S>3
zv*;n<z$bN`+$Mxpv*!N;fo#wqoQXlPNkS6`O9TNrgMQOe|0ei##=izZ#)ro}%#dwK
z-=`t1U`YiBPNJovh)57535|5wKvi~3pUtWvvt1?@*sQIBX>V`x#aZrsU`@VUfJHi(
z;J(eL%c;9D(;blGCvi+B%2*gmP5R^qFam2w(=+9-Es{|>hdg^pnt&t#^U-g+`qsaV
ztiGuadjX~CMw_J_=cPyt*eA<d!Llmrzgq|qqPWVT4NI}+#l46|qIcYh*M91Cu$oPy
zdYV<^F~p)#lElN}0G9m!+57jtHjZR*9NvH7RZPeeN5TT@#@%=q19sTO;KL=!ZsM;#
zBWVO<OCv=y;^M4-ul=b@U+1QgjGg2hv>PMMbXQkbS65e6S6Abmzr2vpU1#iv=v=!&
z$K%Lp94=AIaFP(Ed#>MyxiE<o&ErEaDryGkxa0A?Lja=d<nULV_mhyhgB>ewX!&Mn
zht%tatNi|%QjG-ttX}iZ6D4Y$mWhUEZK|7YK0jST9_EBnB$Bl(ND-8^Jf-^Sazt=W
zYmV7w#$sqV?4)U+h5vm1I_(Wl_3BtT8M53u#gp8mtsug!BV>2l0m-{5J{fzFNNl&I
z`(-MBMt{s4z7``l-3q6hd$H>%srct`$joQG>kNY+(DvRQ;h~Pc-|_K0mD>)7{Tq;s
zu+Dj$<qvA%u{BC^dgc0f?FJL<DFI=UV465@j5p+{40|j%Z-r;p<I^dfK1c|$+{)yJ
z0u0JMY{U|Aw02Q}(=oht$3A*}sd>H~lJ+&R8pwGN>fJuH)Ac;OlCwL!4HyYr#sL$8
zbM4;HYDpE*IXrU5v^<DT$qo^w02o`t^JUNx?u5vSNblh{i+-?NXi08E?;Vy)=AnXx
zWZLR<uY)D%%!y=lodb_3K$o<~wr$%pcWm3XZQHhO+qP}nwsB|g=1snQo6RPbKK&Cq
zU3H$)+!;W)8y-R5Nv;9o`YG7oIV;pE-(z|?@<39<Q6m2<l+6Ws!lu_CyZ7)xj207Y
zB!Qft&Ft)#d_}l5h5(ZU2WC`}xiS)e{^VM@7u52WQS<r-jfd;j7=()5MmE@x@K`D&
z`ea`B$@+*9&EZksYOpZDkc6U@2IRrxbT9Z2p@k9Hwp8PJ@s2wPI^X;?G`nGPdT(B)
zlKc<n_%t>RvoXY1w#w)4XlNiCS*h{5cyXWjoB`rgb()f_Eg&jx9Fc4)zkTjaWux;E
z_k_HCt!absW(AKEQ5N~LA@l0`Z@RB&oxl7?^#^vT^ga%jY{ra+gLmGa4yCi!SizNO
z{o07=-bOws-SK#Y{3Vj}BAg77*kFr4SWd?i$&Y!&YNtzrBu%KP%o<qX7PF)8WN3@F
z95j`AMV`en=>&{&iq0q9O+89a0ca`m6>9q7<%hD%^cijvV7L9ug+OAQo$$UdJ^Cbi
zV7oWtk0nFE?s-mqN(TiT+g?3x&W|6N;AIT~c&(dn>-46sw}fvpFA<)^T?}zdxf!`u
zP!EV%<{bB1ltl!T@OEOW^uvE7%&n-VMJ_iq_VOEUA({E!0vysf=IRe;rRHCI$Vvs6
zj@X!lN%KYmYGBEJtr*$~xEDGLJwH~~xs{+3uTV1Iycet_j*km$?p4cZnGe^~Hd9X!
zTsDUizP5y|w=+aSizsfchq|k+3n?6cvyGqNSlMxBt$ED_awWpMoShefXQ^F0dfy)3
z;|n3QzRJkY&^KAw)rOn_@5x(6MUZ9(TRGCmY1M2MxcuBI)3sj?s+~qwFd;R2Wp&Xc
z(^!LfcMAE|WbJ|>BeL1Rb`Q_@zk2uZx{UUqDfLpF6_pH$K*$aihDX@tkA*5|61(iJ
z#CW88PGX;3IB)@Q4@&dRMQ$<O!j8Y-Zf9aR5AJc>efcvc%E3szp(q^G#%)%oOeG-G
zy(V}L@6mu+N}dlnbFBLIMtWT+p3<)NM;T1Suw6Nm81Gn~c+8UBtD*ZH;w_#M3<y|&
znsWMqejblu#iRi<v?xZqvZW(xY20sI{}^mwoXP+0GMXlZpOPJi9jT1nFICaFrITEa
zLK8{$g~sq8&oyurI30C?klI}PT<@WldSQndGGy=q0ph{T|KzWVcS;lE*}ne3doj?G
z8FkR_;d-`D-D$&y>5SbytYh*ZH~{xZbm!5LnttsI|Lsre*g+Ybj!s^UIocq{4FxqK
z8$S&m`WswdLZX9V&xaQ{jSWy=6b?ol%~7GneWpWV8!jlkg5cIu&-}U-u8{@w(a<zq
zc_1Hr`ZWN1dx~%v=qPTjc4$R7HO&H82;G}D^yVWTC&I?qhpuk1@z=v0k=Yz3-{gs4
zt#H!QJUPE5P6<QfQ;>$VWal(%DrzWC!nf^Jk8ZPG45i^gp#LXz*nHElXMM7q-)8g&
zkt%Ydj^j?DZAg5=p8N#CgWIn(hME?7ub5U+lW#LY{J6$IV{+**upd@8+^)tPqr<1M
zp+-f(mHRMhd@ePoXVwncPr7LUQ<~^*Hp$JagJ%viwre*XS*2*wM7hm{)L_Z17k#Gy
zY*A`r=#h|C!vs^AWKuPWzyXrtBYA4_A?HjM_%FB5Bp!n2Tkc*@n49Bu_FE$emqh7k
zPG6cVh;#jTP`_yI-w*_ftqw5ZW8lc!a>YnjpGdkIc++!Q?@fX+-fMG&78Zn-4F2UM
z{^h7ZfLK?9gbi1BdhoF}YC7nA?#*ZTu)^komg|N%_+%^?R_&3|LH030e4S>`wSRUr
zc~*fXBsbp-m2)T|EoTj4E8+}+xA4f}qF6*OpY9@s0Mb<t8ikqe9x|e=V*Wl@L3>+#
znuWBl)Pai+{UEX~g<g6@qJC4lL;|VMrOCY}6YzLYy&m4Kg*Hb`UTgn!7#@1IB8Hq*
z5Xon<AOkhidM=A}OES8_F8Cpb?R&vLZu5`TgBH}L$93w$l(Vf=2v5uglV-g;Z4b;w
zA-uK8-<tzIN3!Z=HqI<9Y%T1ooEOWk%Nnm4-6fmqIv2K8%au>jo{j5H9k(GLv{<e=
zu$vXx2e!(44L!R>z4zf?RjZcEJ<FKQ%}q@wBby$uxvQ2Is^njw9~p15{Jpq>i3fX(
zla#Di-rbJAaj{djRjd1{RbOb;)BL48XC5;Nrg3J;cbxVgGfDSJ@~Sl{y+BrcKD`6x
zvv08lO?43Ij9=^*Pg>sJtCyG0qwim+yyI_jTyH!?el>cMT2+qSuem<0`x8Cqr$P2d
zpR2N>B+;H~Esx!sH_m~}m#E;S;ugQq0(>#@>~38{Yd+UZerIBtH{iPN{B+#*gsx0a
ziKxp%q7jEL!mbBy<zMj&1y#x30b*_$2x9}lm8b=Vd+`Qg9!G9Kj>7Pg17YZ?M*B9>
z96YAO0fN(MBO~`-v|_77+^QOb`2*#blN@>p=jXSd;|CpJTsxFMY*_h_O=X+#>uo~4
zqX#Q0d(2`3cM^P!_QG6h=Rzf2$7;ruK7e!G&V#fgi;mIqJlmI)6JhxrM><IE6a3_f
zT_&4XQQ*qiCc9)w=mO(CtSwokfcXLjp5)A9;n_));ect*Uk5+=!fKh>IDfqr98#|x
zrI@Y>TRJ_ai~@=pCPsz*9~bKyO#O7I%5u^`m8U00oQQnF{)omr6UKMpjgA_7vqTTR
zmluM8t;LkH;>i-}gYig??q{fssdXQW4Uu9G5DqGlzJq7RjS}#>=Cq;nW#iuBHp`wQ
zB;p#qn=9v~vr@@U_NUc&Q%*EcPyFEBCa+{b{d2kMpgi|GctlW2Va^4mT!TjNCPf^m
zMXHwQHs~3R0s7RPBv4b^D#rW$8wp3weL?<(RJrz2+mSkxU_Ro({;tFF?yu5)7krR^
zPB9rhXn~g?{r3AA0D1wce(1R1sy2iWpGa}=eO+KkFD)Wl+MR^dD;03w)h0$sqwj@H
zgo&xodXPKNpXbJX>-%v)5<v%t`?aObH(Np06Y6$!^R}iS#FU-YOeLv#iJ8&h0|qa^
z+&BK{+B>l^2$Mk;XX=a?VpmHvPHJ<7I2-zV7Ql3KX{#brlSgJZpCtEgkTcg;c$Y>m
zcncIe#W3hCi|bYl2G~YfphH4{Uk9#)$4kG9*)EaqAusU`)M5GJcsHt^i!A(%A4M{u
zD9*nh?%BZ~MFlVlRM&EvPI-SbEU2QJ9Np^@bR-4tm9d~4trY*Rg{z)>-b|pns?Ul*
z3p~h&vLArdl<#jPV`G_H%xpd|239GN2Di|U*TwS_9S1}iw<JEB%r0Q%iRM-CoXk-z
zZ=>M<bvEu@Nu&|EZcJ|pvrQ3=6;Z#2jnI<9FEG#7%@kyMRZU((W6yy^sic&oCX^KC
z0YMyA(kqUue_EWE2Eq1~)Z(`vwGy$08aVXqV<14|Xe~%aky%yQ0u@#S8r(jlRFN(9
zZwslFs7?NO6~i9>9pioa@Z-N2AV>^^%cuOC_i{SwIrzI|&Hd@uJ8A|;A9vr^&CSWn
z%gLkB&du4OZrA!M#d<LZZZU8L-2^G+_<*a1g1@FSJr1x!L=NGxF*rs|540H%^@3&6
zUv35r*%J%8ctzqr$lT2t)8Lw`NN6dlnj`~S0Kwa;W#lgj2GxI}i~!&XhB3<=p~~5;
zFKWNHu`kW!8d~%Pf_jucbYQ%V*k1u5FUL*dZ>~e_4y-JPwB8|6ZiBs*DAnF;zbPzg
zHi9UZ8$~X0ykqM|b?Y8wOu&t27^KQTaV8)y4CE9w6{*1De)13b5+K51VdR-bB1S$g
z4qA6~K1?y@64Pi5u?bOgjRmTh@HmmddPxqhCIh7~_%@2hd9QHuy+#wPaR#%<<2U;O
z8rqWxq7fQa9kMcI;_(Ncv`z^jjW&EPT2EbDhN!D+s!O!PSgnh9cO+{^!5bg$MmN&z
zQ#v8ntE*A)avM3OWTy8x{H$5prealz#9u)+<Zc}f*rDZd;$vhXYiKoVDQPKcDXGZk
zF;K&oKLFu&ko;t1u&U@tD0u%_9#u&>XcK=BZT#Lp5L?%4;ll^RBM7>@4ZRoW1i3FI
z_nBfYK>;hMODdDxbHVcM75w+|vp=RB$}wW89N3KG`PSW?T2G-quR)YSMP_{p?i)3i
z!Wn>~<|C{JA71P0SV=zF{3h=<CkLOGpoU{K=^f>+&xNQ^2K>dOI&e<vLVP|4Oq|l{
z2MHHOttbnPR^vNmwJI>@+sNz9e#f(^?N`X^?&RZwah;F$dVJ`GCZ-C^_N-H6!r*>X
z|1!(jq3|bTiEV@IDdA=GhHC*<QDxp~{i^e=gnS{!^-r<Fqo`71udKR<04z10S~i3U
z8u~g^eUJe|bZEeyf~VGZh(mb^O7o$eiZFgg%R}4fYv80XCb6D|P6M`I(2u+MH}d*e
z-o7{~!L@9`z$^?%UUVnH&f)sw(z5Bh3RTXB3RYZ&ShDmKT1oK{PXayFa5z|zf;}2v
zp??!T#Tf!_+5!q8VgH);hS;Iurc^hkbPf48JsF#}AnNS>Ye@$%Ui>~=M@=4>T6p9I
z4Ive^?|V^A+uZN8*>Bu7HNAKKdz_|IPbSi@!j^mUZ{Hv};_rj&-OItp_5=Q^HqY<d
z$KMf$t1<SPC4Bg~SGm3JKb;Ifyn|o&nBQv=pQg)W=<qubiY=D6uUkz^QE0@M$>-p<
z!-1!gdw!f;og6e}YMdO+l@35qxES>e812Np=n9bV%-cp7tqfN<;VS`hmg_@y?U7Nz
z=9jGi*bPRoskYozkbAqc?HFkq^OP&uD6p>h^F}oK<vE|}GQ>^qxG&lGKa8b;J4He+
zxajcp6Vm#z?e-^=eGfn^(_<I|0upv%fTJD=FrHciORnjUEXC5LOagS4noL2sd7`XY
z)D}?6+3aC<_m%NOP5AHz%TkTNer4!}!EFU|hHrqF{_4RRLN=!1u|L6L^bPds-Y+w|
zTdP$?dGPQQ8G0f*`+<}II^2$w%*{Xr7o6zR%cJ(8JY!u?LX-Ls#9<OYg0jF~SAdA0
zZ?iSn!pX?-uI~@t?Nk(AvRlCNQ1F?BLSino6Q5fa*)jvW(!9o|QPvPmhvs;79Mr4{
z55i_0W1XCHuum`pTPB;=0S2D=au`w3w3oKp)^e+33^;&r_*-e9fGZ+Mn$bUyB!t6%
zRCrCp+rT4;D~;+^0;qgu)oB!0A89f`))madzbIGGVz$fPACMRD2N8J)gP(0V7pDa)
zm8-3~3Y9M@&EVb`?WfbHC=<YiT?c;a82n(=DkXRMc(=LuP-bzg>|(JOW#ltbG8MHW
z1}KmUc`kk)A0Jl^Ha0FapnJQCR|CrMZo5(#B-p&6L%sW3#`Z&dki5^PD9UVi7RYp}
z^}G>-KWQ+1v=eb7(hg*4E9i#zjF5is2K9nS1)KZX_RG?<V@CfH*w_n8f-+0k>^XI<
z0?+Q3+xp9(j-PuYCWx8t8X1<CK^bp!a%i+h$BJqIK}7{JRvzY8N@JY|l%xY&`Pz@M
zXcHyT#Bc>#jwmokG$t6Ck%+iF=p6f6btQD{(usjelN>{Pm5$QkA#3~lAhC>kb$t@O
z#0g}e%3$QEBqy;?oLQ=|NI(T$YfH2xYEs*PI9QLQE3>4k_JQb3_m~NlAi#_}+kONo
z_)P}ZZBSeNiPNxlP)(>?87w^3ymg$alKKpcWIs&qm5`*?(W;d_Y1H*GqTacN-U-aq
zPfyPu4+NhK)Z9Cf?bHMsksPZ(jM8yb+o7<jvEyX_=BCaruU!3;k+q|HQLkbIMVjzs
zd>_zz&WiU}|55jLvsefPdD}8EEAy;m&OZ)4lTR7?&|b4UV!_z^<YeG%c_#_rX+%+U
zd%ZZl(n41`QS&%3xQykj`o66#&Dr8|cOXveEUZS&EG(=U_=%bpTS*c>J6Ag!2M9g&
zUuq2TTwQsCk24fB(%vGh*Ki>_F@5n{`(1@H*}?M&YQqp>{7@-@V5||Qtaz}vn1fs>
zU1bNG@0kG@YStlw(98<3w;z)olA=xuUGyUtofDTyf(atO;2R-d8cckEhwZt<ZjmAM
zKqP<gIdGM%L1`CCdsQm1NCx$69hMRjx}1@KO*NGG^{a%T0JoVKn#6SGg~V@>!X%i(
z?{0s;`Y%%REYcKb#mDXX$SZtYY@DoaTJkL=YgYS~fHghCp1vb*jKd}ef<!$=tD>s2
z$>p$!PGvT9s9z6UUiR$niQr#cCC~<ya5OHjXF{uNe0JcX@z}|5RA1p3>hJSw$)kQM
ziX(Z<!g1JkXJ+*k5q=Xt!<B%&gR#i=gRaczNYoz}eZlyI0XSmZI(<N=T#l8lP==&O
zS7ZJr0>KSs%YbJ-p075J14l?Sx>pbsc@#FIG`qe5XiGo**zSD6gwLG649~m<fR8@s
zQ?@6hAkJ4+=)y-{87z_`j|r~(M!n^P1F0KM_c(t#?g6l*pW|J<FQj1wyGAm1)sJT;
zFx9zLjo_`|=Vqvl>R;i+RLKs8cCU@In>#?@L|Zq9{6*`7O#q7t5`n;~#R?IzD5Wwd
z(ooGVFbe~9N6grR%&<n#tVH{f%W9a_v15OAQmrnhDcTk2kZQY0tnLb5F7<!Qf;Jik
zHN;^Fx)-BcTv%8F^`U*RlIFNkID!FELy;f}M^}l!@aZoG{Nxm)y%%I40e_Plr%tmC
zri7A8VfHNwMdlFbV?cPQUZ$I@t*wz9=W;GklHvNk3xIyMVdqKPWJ0R8X5sOqml4WR
z=%&|9S6Xv={?>TmlkG0I<sif7j_7eOHA0<fzu?CHeOtLxFkrx2?rl3xIgpSHrp(H9
z916IphkSayJ|F9e5;FL>&dJU-wo59lMPV#sfAqTaOyE&*3bFF=1SV-yk1z~wASW*y
z1k5%8%TnE|)fwSs2<lGZAK*p!3>)K`>-c4dtBo}CPtW}=$X)-fs;%L>{q+fvbNs!#
zK7KI*3;$)#^Emshf8&3`FZC(fm7e{L8T<HU<)wW3eO}Ri`klf@BwLP+)78!Omvzi1
z^4%xLqub%KEkLpc!SGLEhP3_3$DX1}fO>xU+pEB;aW^$eYCj9^QIW2Se0}!0sNp(3
zygQk6<&9pG!yfqTM7Pz)RM#0yG**`Zb>?Nai-+I(umA2w2TzZ`q*;^Y7-Oyc3O7gF
z{}K3Ejnybz$Avsd!L-cN{Y>z5tM|-zyg<y1Y2I#OpJn&(kIe5a`eO)C%N~FTSUV(f
z^MkA3IOYn)Ys~p>?)wj2SWn5r1d-jpXfHqFgB+>dz7mUriLNR0neF*7SlrtUpaZ>d
z($}aP`U-6qw|$>>$X)ljn@fWINs{#UbPxL-{_G@#;UM;W&B#YPsRlXtobf;U((sb3
zaiT6M`cpDm<a6LV4Y-4+UVN`M8U_!#0^Laa1j2jBJ9mX)E*xXZ^+cJcvw#}+k<P;*
z0jbH?Q1=1hJ&UUo159mpm8|od&-g?ZqZmPV*vEnsg(mX!X~8&H%$u-MqIp6=#@*uy
z$<n<izcwGn<IVt%SWv!6IV{1ML#BDgiZ{zE_&Ea>jDawRiAx+4oe*<4Auyh7ZEb^r
zV6FoxP7h-9Ma+&b=)u8#xYG9JI_+~hkiM{{P{MpRb5WC;WTLPrXaRC`ql(cmbiYR`
zhy|k&w+URKm|24mYd_Q6(IAj`3K|;3#O&UX6eDMPF!q@9iui!8{dyrCgeO*85+SF{
zXrY@?Tq7!R@s&Z}RQb{Xdfe4{K#qlS2J~=0Q~vw7Dvq={-PrWWL76jj`dC+tn-j9_
zvFdTt7?Q|ZKW*Buvh<?5{I!eUdZ>iMP3W4l38hS^YcOE@NlFU(_m|{$k1m;P7%#G%
z!L6Hg(Jg^`@JPTbvI8*NPBNY$-2-GpL~l4wx+T*n^Yf3Fhn&ZnVE5Aj)|n|+rQQPS
z-Ng@L9j$~j<E#011J50uqsJO12!;#KBE=v<Y!9{D_Q(}<Jy`3(g7^DQ4AG>R_J!wR
zlpDL?re9+{3!qSwKjIHW)BQ;skj})RqA2aYf^~9JO~l?zpnht@LGgqjQb@rYcA7Mc
zdCG*(og6hox~je)=yDBAKkto1-I)Tf*MaU`*AaEx2{gh_kb@7P3_HxI)5D?mG$|>S
z(1A#=QO+pP8Axj?wHZt?Y-N36;a-8?ZNUo4@-!UXUiSzDO}D?Wd$G7iLgo}#NP>1A
zO}3`I<yf>Kt8P(Pp7T0k2v&S(7`rsYWH5mYaUcF(;Jk&E+7c=#jUAcnm!igqZUi(I
zMWRu-gq+|6On5?XpMZq3JedFxI4=BM)4jt((MoDG0{&DajG<|s`()ufMBM`~a7xec
zY;8f*f=hriYiEkLOb`2I)MZBtEQz=+O1~fk9onwwJ0=}EZQokrAQh@eWp>Qbb-hI)
zO%^Ai?MusZ4<nWjGis7T>$i#f_@Adv>Kk53IFwOtOqWe67eaCqA|>G9Y5Lhj&7#s+
zDJ{8+;bz`~3rU6O21{!C<m*+^P0@%}4Z&^K5G_`$NobwiIK3+NxfpgX)%@D=Q{<tF
zt0Nga;FvV`dLuNOvB*|r5n33N0d!XpU?q*ij%jVfl%)h^vO}~}aZ?%D2dgov6*j3B
zqY{?h#%buHz+vTHFFJZ{hREx3DR3mh-XqtmNoYw!JndZVTztI!o!D`#_D`*++-1Nk
z$T_;t-8aUegqh$IZQUNpqm?RU-JV7d6;ny@mO1G3`Wti9-YkpdtwR&88&Wpv-876z
zr&|9u(T5-#U;PJGnpE8Ww5_gzq$Hakl>u$l1rQvEjD6cA=!QY)jaNyWtmgXbXU49`
zxD6G<>34|`RRuhHEM6~$$Fb-TE5Uy81neY+M+Yuf%v?sEM^G8X@$n|qlEkXhz+5oJ
z>iwh&rjmT_iBwWg45?aD32VwilJE!7Yz+!d{1#4K399o~Vhv8VUeEnWy~}lYCkX1&
zBZI!URE1tBLc_q1ZsIVPVwwYIZS7#15j)!=rJ}-47j7ebPtE}&9%o4r4nl_kxP|^@
zH2mDGjt73gSMZ{PbZKfneFWm^z5};tA(rFtcu8!l4<q<#9)?M-d3CkS`9j|FbU+;o
z5i0zm`Bxn#7VvVvF7sK1Q4KzOin&rh;MRI`A9q$)@B&`i!PooRKe?B@`qb5F(3$82
z7_<4QM1kDzQX?v-O5J7x;NeFUfKu3UmT9Vo%v@X99v6VBA&^1+DxwnI3mAE^qr2hU
z=$ua&X5Ui_#z*y2on{UtDk%Krk7TFOFb+zHe~aM<^`#zDCCN_ee|E2iolYA7q<2qv
z7RYH6J6X<VS9;DIU=P*kl*$^Je@b5c@m&Kcv~Q$G{T<(*Ee;*N@a&$wI;(1>Q^Dn%
zfwRMQ%0=Z+yN9zt#^s_mit=*$>1*Ux<M&^d4f#7q!zn6^68NH?pB$PHBqgAh0AH$s
zq|?y@87_pH8xt9Ysk=T+-#SdElOc&ur?Qts;H0?@i!nm?YH%ltN2l1C>{L^g?Zp1U
zyb$ikU`^)=_o}N^ZiJ^L1s1ON3*MYFRnphGu~_~!(><z!KGK(pYxfR?7I+gw9E0{X
zd@J#-)M8HTRKUCs+ry-mDHhyyu<x!F3j&5lybvTgX7A>*YnYz}vCtyt!8l%)2w#I-
zw$bJRX0U4Z_87F++ox2!r6|(3E>Nt}1(vRnd}eeng+=Oah}11c`ASBG#@@(YA*%VB
zdo5DVaGeLR9fWL5CKuOdT6d+wBsnc|Yf<Y1QIZYR1O!lM8NBUJL2r*IU$$32Z9_%3
zl645aD7{9Y^yKNPlJ9G!HRtVeK5g4$G{1)F#90h|U2}?2T9gt6#8x-MU!s#)>E7O>
zQ`QK}zE~TOy1cKRVD7F*i&@W_D0T;i&qOQ*LJG}V7m%bb?xqwR#&WM+#5tl~B{-Zb
z9X@SF#qOqpFgfx!6sxk*TH@s5G~18;8ruEpsb`nP@5kyr(__54{VUxYrvjFfXd(b(
z$TRf%*5BWiv10M_kDSRv(S&09jh)IIILUi7r<!<Qw51vaA)p8m<PG`Bc!Pf7H(Ba7
z@8R6?a#DYsg8u$cDgXAxyjF7R<m{Vz3)B<ksstkkusr}1I0mQ9tfft01|&>4M-Vvy
zSxZT1ayId`S>9#boUbmaA_tTV4V)Yo1ycABb@u?MDU+4yIRv?a&V-UgFp0}6#Dk)H
zDKh<Ap2dfC8dv&ZH1pkH`*)qe&pMN@b&?E<6&oBnA~Qr}MzF~AK%ohtLL+^dM%faL
zjuktjGeml)H2mKoSZYhC)D~Z%HNH%1e2K>7f}If>B0VHnY7iIqpGmk>A8(;P&O&31
z#l|E9olzPhy(Cy_;S}|sNx0M;Z=pHPVq?5Y^Z&_uw<TVq>HjG~qb>e_$}BNty7NDZ
znGOy4U*f5wHD;qRX0tYC`*-=@vY|90Y_cG3G9mmwG}YZc-&CTzg65^7b*o}Cpw>dF
zw}R%QqV=j`bD-8zqQ@Gls*SN_J~HeejW-73C`~jb>?ln(7VIcZwPxL*T{VUViA}4;
zv}V1a-DKi4gEq^WJWHgL>s_v>i?%4oWRtkgv-v5kV2J}lCajQ!`CaYAGi;=Gxt_lX
z2u1M-mGg_@76K)1rRZ5`*hnQivxt{>E?=uBS@;N*?9gk}a$)>?Xx8a-`khnqRc16E
z+8k0in?U_4T+<CC7tK(ncMMng4?n7ZXGmY;VD&V;?Q*nLspxu2+oP7vzIEm-M<p);
z-eA8ClzRsw#!!EJE=ugM?!<r78Ci>45HQhXG#X*TO|fznqRVof@t}?h;@_FcB)rtR
zR2{3JoWbIutehri?i&p@m@s~;wuwDb=Q7{X)0H^xBUswa-5O<^5$P~-Kk_jkA(efO
zTI&iZ-nlbTPEM1U8ZQmwB=VQZVND(^%kYoDvlG1aYcHSqk&csWjo}Wjn3#hBzdc*Q
zP={0jAxcGCdp?l447uJ%MhB&PUEWlJ!Lx8wS3YIaZ^BzT&|NAyz@1E3beR*wzaB97
z&<r_>L$HhU-eD1sAQq-xV_5REvlwM_Z54LCh#l=v!|Nzrmtvvm&JwzoHYFUxn)$0Y
zD%rpeq%U(3ZgbC1Z7+;I!Xg!iH}dFjU%gTn^F1YTWU(X;XV&JW(Jnu3V*eC^4}KHT
zB0e17^k_+weW>)Q^dVa3YQ*V7v_K=L=$;yt0rfMwUG4NiVKVhVV@AWRC|e$S1c7Av
zW}fTaA{z0a;D-+9=xi^v{hUZLyhM7NvfAyP;Q*?!H`vAOSkG0=UYaOHB6>hCV$?CO
zAO&Z;C+^&@7PASuNxxke=h^Oodr=UMx6z>h;_e)7`uD?f__g|U1IF7(=kP)Hi4#ST
zgn0sZ18YzeI-!%j806qmQsfQBO#1!Y7Rr`=w+g7JIrl$Wij~`)pNmXR*sIWjD2P0h
z<z(%sn3OTpp++rV6&c!g0hz~Tw26N>Q|0>PB0$uhz}86LQqsRPhqnAP{usGYyFAg-
z5+)w6ukR{}PC_<yZwE-xqD_q^i^f%<c#;(V<LXzb>-A3L_-3pmHhGllUYr<)u8&2_
zgKl%p6K58b*+a$xp3T55kO5bn=dmmu)7LVDl$6@cpWJAS;{3JfifbU+8wO)*Y1|~6
zwdh}G*iD(xMP6h|l->7CCT3X(u})lqOHXf<n0!{_Rx(Cs5#7Z^FGfJypKZxbMX8aa
zhAKzN9dLavMD;H~YX7lm*qCJ&i676V5`7Pvkn|i<xWg%vG0yCIFt!(ybTq5P#OJmn
zaq0`#6f7%vxnG3`atIAx0WDu-Ri)znEuadd@;{TDZ)IRPyLIQl^1$eON{TC!B%+GI
z^bi@*x`xW7QOK~e-$}r<0O4J*&x-oDB_EmR<Q|Kiz6#55S$||?k4P=qoFB>g!c^UK
znH|sZ637SvH%2h-2<y*NU}V@M(dTzpOkFDcqykOJjX;X@Tp?>g%7nrBV_*;JQ<&XD
zXqkZHA$&Y^l^h;gIChw?8Sddv*(*WYSb-KWxn%~&7WS{O?r~hdLM|4jB?48xtIDST
zct{q^?g)*-eyp{C>-x`O!L{(d$4<OEwg>n+qRTXP8$1>TJ}|;b2d5((*i6{kF#XpW
zU3?V$u)gk)6&V?vc6El<{0wF=w@AM?=RrMv!ppeJCqzVR;Ey;?tE1?AWy1bi`l=+{
z1+S4OJ3ChFoswx~;U<fvYfmmTcVMtoc_j1Tx3c)*22gZc84HI4yjALp%0vV)LS;#K
zQ)}Vk;wty`4a`+;M0x!V8q+K?30k1KjUs>}%Q1zNkgQUP`h<<nLPSgPYC(`06Fhr(
zc_w1=&;HKXw3_=g6qnf>ZUN<9;^xL634du!%=#b`DEE9oRLBXo;&n(X5_Q~oNeY>8
zu_H0|w#GEWDGByLAin2qyOYjhD&p%m<bYNWqSW6@84V2`n^#`Y<jV4xQM2&h`VW|<
zzZCqsghLvb2vx*WfJHtArx>j+Z2GoKy`j82k@-{?8`2%n>r8J3!I4qcfL^!Y)hHXG
z1(zKw4%Z(S7g;J9+N{`V+SIl$WIM4_=Ycrl#P*aH_ZB<>=rNezs!d$AXo}HWGzlRn
zYHa!5`gGQyB6*-=1QCS7O`}E*0<wyZ#tY-O^Fn|@2T#Pn&A9>=>|$D@5tK@eArhP0
zaZG2tW7lP1rcs!1iS6v!AUoYNPu=GxTY@QwW|*<dL<5<w>7|44ODx!eWVZ6=QiqVu
zkWDh1B~bjBak9BE4m<d=@*ObT>H|O6ci5-G)ohJG+9T2#8;SeslsmU~RB%F2YsEGl
z#6f2yL&_X8G$G{^ke&rCgLgs)Hp9TCgdg@fM*6s}E9Jwp-7JtX5K~tfW!K@uRHMr%
zSLetLOgxO}@8p)GR9Qq~F-#}XFG27Gbkb<#a)e$R#+CrLGnIIO*f3;FJHKQ(A#WT0
z8mSZ`ub_q{ifxd>GP^YQA=r7Ja<~y9r^tT8sz@33e<jONJ2sODDBx`;Gl^yNGX$_Z
z6g<d=vsLC+D0v4wKOtf@g!B9uBQeEHz;*$KR`SBB%;%PPBP0ae!$DXGu#x7h>xyCr
zCPs&mJD0OST~0}o>>8hKp736Shv^we(K#%bp%JZmWF?Uf-AF}F(|?E2m-mGpB}#eu
z%Vm|oX>KzYmKbg9gPn}V9Bm<pwq{|0ic=rvoWZl+1|<6Po-=&m-x$2(94Xh!{Oo@P
z>7Ma9+ed=3u#DoOMf1J^k|3@XCx`8CI*2pyNAIGTF9{g+Arrzvx5I^P%}cgX>v1xm
zqF6}*mWPf$A<X{KDt7Ie2A8BJ^>6mL7=$3X$Det$IqL1JRn68*vSRFiMRI%tYaJy0
zQ8~w%Tq&_T#$yL1eto9B0mYrb1rg@kh91D(q@}9G^#DHI#mTn<?A3hwb`IzsMMU^R
z6WGB>gys`+V*(r;ny5o}*1H}V5y210qi^X>^}(Jh?tFtM-x#iaz%3QdB-_>q{BSd9
zq8sg>zX<6u+U7YnAt0pUk&>R0N+(mlSWOB4Ie$XV%>>C+j2^zjtI22lfEi4s(h{mf
z89<<5`W*7ad~gPbwYmZcRkC9iwYI60Tab7JQ<p?FCOquX5PS)vF78JTifF8#K@u2*
zu8||-)FQLd2NZ+ea6~6jINl&)?fYpd0u0)KC)4XAN2JTlV{u<~kL2>LfHy>QoI<G$
z?3tPDg7~#KMZbZl6%oyieT=AMBFr>NbfOU^s7hy?B_N{sj#1qLoKNa5WBx+Sy0_Dj
zLV<<FlBLH)a(W&Nr3>qK2=ss=2VU569|12&&SAiT_D8Wkle0p^0U2&Nia);_>W^q#
zsB>S<nx9+CgTF5RcXg6UOg=In%MikNDu1u4!{eUzpvX7_DZ^hcB7Z3Tp$rwZFjI`m
z*Ok+u$gh;tmc2-qBt8HcZvD@0<Zs!<)=*jgw!s?a!33pD^^XnCV!~!V5yN4?d|1&V
z@Ny}<x-UAwbz`dwkXD|#GcTvG>wX{_r+9%z6}eq~W1C8jRM^<mMky90G^y4^;+I@e
zVA=3_t>!tacEMQxYYFmXgX5?F<<C61bYVG9Godokgb1RZ_F@<PHMVNpRSt#hIe_7v
z)dW_nf(-4rkEV=suw7HFnrp3GiP>CL^{MGts2<!~p`u;%Fsh?pq=RJh90jChNv=}i
zZjbiMtyqojYO_jnUvrHfn<^h64law;V4`o}({0R5bRLIGD0Eh*b*|TB3Wei^7C<ij
z8Z|T{6+T{Q1Wa{|j2iQ1F?1*}Mx!Vi?7&d-<DKBPg5-8d&**-iu!K2@@34|KW&9R_
zMZ<sVhL~QQt`Fs>$}HdzXwgRqqIecw3|eVdW{Mmq#q@a^zP4>G$TZ(tmu~?$-ovt4
ztid%wkm=R8gfukqASj3Vx3sr+#)Vxu!VTNNYy*wtW<X*OZc-4a6&c~=NOj<;96wEj
zmgFfnfyXe?#|dJ{$7V;+x;OtKa(T#;lV`C&r%IFtT<Ls^O7*DUv~J<Zpn!Zb&RA!I
z;07Q6)IuWVztg^JJ=u`J4z0gE!;a>C1>(8+;{)@$R`SnhOPd6u(%HChdC253Xr>-6
z?nv=D%02KniZbuT*_dMdBx)MFyp_dn&T-GK-nH9r*8Guh&tbCvtfiE-Cd|ovRE4ET
zJUcqF3b!5WU4{^WtBVvs?(`V=jQHp~PDcjmlno3`M;S~mNHdy~(GZ%8J2saZ0)%BS
zvrsr5)iC6Od2z@Gwhv~AMW)fe_kS)=A9GFem=?yHH8zZUcG%}4ER7CAY%xGW71B|)
zBx)9!<TP@&@#lI3f<g|CEKVUnJcK!Ct|^=OokzLlrLgen7AAcwPN7$wiVxxyc)2xF
zMHx<O5>E9#`va)RB~VwtR17sq;>}|qf7JVBIjv_Aw_(c^r=!6n%&VcmHX>?E*x{GB
z@u+kOoEsuTl<(@PyFkXq$X4ID{Y==Ye}X!;_H|deD96EKJ`OX^dHsj*Z-0#qf>#10
z$pRC<w0>naFV5cAFx9<O;X7A1*abTNUl7jI+1n2~M}T<|-zMYg-5vSB+m(xrtRCoY
z_~YY4l?q)4tP_}86zIWalL+bX$fnLttSnn-_H<rwsLpoG!drAAL5HtIW#==R<k9O}
zg$Z00oKVZ^XLbVu#K@gL#L#N_IZSO($&?Rq@7bz?E7x#)XjwcuMg=y&b}(pQ72HdO
z=uVHI)98!od5y!56Ehg7CY~;BO*;M;HkiHvM1*P^w^E2giAO=xm(`g{->GY$j)i;+
zh>n=I&NuJ<Vh6?y_nlO^cNH+K$^<7>In|z2W>S(Oo@SPkqWkz!IV9q^UvJK8biEGq
zmD93tFcIo>2q6b$BZCW~6?H25vYN^>WfsUVV{QB2lA%+wSvriT9f>$hOZ#OjR7}qr
z#CiT^v0i`u%hYCsN3qW29+G5V>Dn1dLVx*hXu}gAl*Gnfanpk$WPv}lIAJ3rhUX9+
zLvaRi&800)!EtdrMWN%@b{xK|{E76;ef;YG#7`VDtxLSc2U`a0vpDsz<4|V<3_$=F
zD9<o>)8@NSJ1?8lK%h?sPV-iwi~2V+r!b=I|9J=6{InT-y;crRh{)DCaL&-}oUVWB
zfSMeH`Z8IaBf4gfRt|NF<pQO}_IjMRHlv;qj47Pmij1#6OVihRx4l@&ikk3rve=UN
zy8Jfe)=7L>ZgM-vjmLty3sq}8E{9T{j!&D2l)N`lSy`NkjR-f*RI6B`a-*|PzfMa@
zgjV%2Sp7Q!dWMoSzrHvNYW~n|&luhwUS+`I{As&&)s|eysZ-8*J-4mFVr7fofE<o4
zK|OFE$wE8IiJmdeGBr2!?UbO=%5R-LLZ-uAC~uSYQ~<as-;2e0OS?lcFh-_;rf||j
z7lv_&r#mu`U495ty1o357|!&1t>f+>rj%BAVxmuUazW*X&sh*Q5H$<IneOVFdZ^5z
zB8)PGI!D|fPFLBV(GHc2%ae4L+ZK+lB~fQBL*Yb;dhx+nSb%D8F6&L&(^xD9V^0}`
ztEfTlMkzDK;Y2$_k=v4vv=J?`%mx{AkG!ee5GXgDgW9#QJ{k#Rhp!~1uyT^XT3KKr
zC!Nf;s}X(%@pnDJJoRnjef~Kj+hLy8&5o3DcQlwcZae=fwkvsYmxF6S2~!9WXFGb;
z!i}+~KR)(|qpkbT){CMXv4J!kvV<`2$n1~vHh-?)s$0uAv$#7ZYFK=HS~W9H962#B
zL16f!i?|UI1M05!+jhfJ$cbc10kRaJ=*qqgoHvvhN4^SsNjHM;bRr_eSn=J*kvv3{
z<Hl@sIdp`QUaNQp4_VZ-tChhh1J@(<d<%^w;lIydq1MNn#|3$?+&{+I)IDTQV1(O|
zRGF2*=WC8^8aw)SJa>B+r?iq5->h`+{i7k!NTX1(0K6x`I=VC0PK1@^3Vb|=vmCN1
zI1zG(mBmOs^N^Su4+$`l)ZUx)C(UcuXH{nqQp-6x#J=>ltd6EB*CVnuuAVkJ{eA@}
zC_{|2WP}Gw6gWxx+5d>!gi8x^twhX|=sKT@g|jM?n!R|D9ztL&?KXLCz|Ri>$xpzi
zfIt{K;xqt@UF0ctxE<6k1;JnR^vM`SYm(83H|T%%<n?P~gBjqVo&GfVLw`l#RR1?7
zq|1PF(x~<Ia-4f^z^Zj0Z|W`eBxOgUqPRyVgLgx_uL^X|wa3KE*lp43bU*RFq0%A?
zW3{_^H)EQ8#a<>W%7J~h`#@k~hv<Bi$&QGUFYgbRu$dBkdGROhAzhRQH`=DcgU-9o
zcmYf^ga+VuiuG&LR2hS&rp}UGg05Cm=yD*d=wl>F5C1RjXV0-Umz30qWLfvfq^c-H
zOFD^1xqSI3tN-gL<p9<<5zl_$Udiah@GD7&@pSI&ZozWmtOnW=Ga{0F_5{Owfs2w~
zX*)QE*wfylw0D_@t)?)+a|k22Y(x5-H;ub2&tf&2S80!7M9)j$fa76h>LH_(Wb?4l
zsUaaiD3B_PC*48B$0E0;?0q4wDhYux2wH|WFKsi0$s2XdToN4MVhqe{T#ol_y-bXb
zSIQ>*5n-w4CiCv`u~?pi6<nP&X{<u6X!LNu;9eM$-7aiqlsi*C`-PIE#Venp=&8CR
zYK{-pNJziK{qIkm{xT4&k%mS!7xZJ+#ss}NdE^95#xzut9t+qioSc+Pk}nk*aZZUJ
zO~kF%f(|Bw#>MeF33yPvu^{J|-={Kojf~Cm$E{X!g_|AyY%!aqb|W#f=GR@Zh3Qdh
zB+lLg+tlDa!4*GRo3i3wx}+eRQt>jYTEJkD$t|b)lJ`n@*X{%zsv3mRR&+zQ1L$^$
zhm|B27$dlR1Fu$+(Iy%|HFxuLvo9y7#d6bOjU{TCuIw;?QvXFiZb(UYKNZXeuKu*0
zq;?9-z>mnD+g#c<dTixl!_c6lvO-VCg9>n@Yw^1Y>_M}OtG%b=x7`#5=6JZC*T&Ks
zhUdcgT_uHUM>4c06+>N*d7e+K-I$Q{sM9iSiUlhvwxmguvH3}(ha-_s3F)9*4DQPD
zoaQ>oV2xV;F#R0-{X1e3Ior5YdSt;9j49(!h?&$jU^>`iB+#v^hM;=I4aZ#WLTm%4
z-I<F?nS|z{$!f2q%xfNTkz@UP?H9~*1^hcYKD`P8)X%@yA2?HB_VgEJKMR5@HRL5V
z|K54IF(xc-K2IXjP6k#L_~7fm09acXxVz27#_q<#!3Uct-q2-txpqMW$627EUx0jq
z*iLbos*$m%4%1@9`Rca5`hn7jSJ4i9uJ|jp6367F;~OkCMT$wdMFWB65`H#;U?2=;
zu)g9wUm`*9G=R{2*;w`Dexo2_!=M8tPs4I{`^yx>$d7k`%@aPr0fTL-q=dsPc-kCK
z!&J75Zc{Wlw*@fhOUBlfS8O)ghh|OEtTQ*q@`mQSc@0N^rH-Vn4j<{Pu-RtQr6F6-
zkx3@NL=iD&n8Zn`;+V==rC-)NyDdIf+nNs_B=PJGn&ao@8y?XRcm{Efq0PP{k!Jc6
z1~=a8PnFd+?%}dad5x0@MI3pFP!E>~IJupHA^;fPnf?&YSbj_Uk(aa_n*em$tXgM5
zuwV$F3c56qLmnK#+Kb)yZ7{?&m`)&Z<`IZfQUFIdo#qwPy`x-&BjiYp)vg|iEsB{9
zs8V!y1K%|kIh0Hk?@0dM0)jbp6diAh3-Zijg5auyVq|_IIH|RAG+4P;P3~3%#X%rP
z)HpeEPnUv42!lR*J*RAS4g3)dQI6&8kdl6fbxQ`obg3j``*w#3AP1N$<J<lsqcj+|
zIWOMs$Dx1KezDEjZ_;KPl-nSjH~93+uZ2xc68zy1de^KQ{=Wo#aa;U9Gw(r>siRo5
zwnM)C6sKe??OC=30D;;<+Mul1k3>JyE4lEkk)8~3mdr8p@Ise&M8pwFAgC0v!<X*}
zV~t$ODiRceTbjl|5=3|!lE)4pSS2*2t-@y<K7s;AmxHvvUw3F?#0W(;VjNk^n^UB!
z_1*eQhRlCL4@;>i!!#m>!o6?;DuTkZRv}s0O81p#RK)d@Y43)HjP1ZwW{;i6nCwag
zQFYqgNG#~Snt(T*T`S$=Lh$rs$NR3{WqY;0eEp9Uyka8?mC4Z!Nn$MbJcaYML|Ggm
zVLkVkQJCzdp(VUx5~B)Yl+$z^5h@7xtVwX($gO7nk*$YUHm6T_OGiZ`$!$4G+C+V?
zu_l)SeY4>mh2@A-Xu7Tf=cLZ;qoAcY1B~i0FC$oAuq=i@t_x2z2<m*%z)r{onxcv!
zjW-X2V%52Dm$Vx7m*1u}0}e+gH5X?&U02~EA2WMuS<`g)*gbR6NaF*or#ZJPC%z0T
z84KO8v`-2r?0Hp17SmQ!c$UVl`qKs&Yao7#(lnVkj5HOGVkT7;OJM~hT3*GLB}F~5
zNb`Xr`?UNC^!(YnQK4=Hs>zKogDT}CVLa`sqQvXDB#Jp2V3Ef77a3MS^AEoh!cung
z(w`2Tq3CURO7Q`dQpCh{M5SR&!YQfD0q~|SDQHL0-XfL_*NyHwUrxsLn<wvMH@)~X
za5?CyVJ(Xzky)ZG_JqB79Ua`B+1k8Oxz0LH!zF~JkYhn_7i5p*<i^4kaGPfI_}nPb
zcimp$@?n(cVah#_pVk?r3yb0;mqV7Vzf2@j9bAI91Bq$pD+><?6T4YL7j&8w8N1hw
zRNqG*2fxj+vcB6~yqxOl|D9O}kaFx(?ztM_J&Er3mKaym1}xcQM-|hkW-oZt@<rfv
z1WX<DFd`C#IS7ZBPAfUfXL|nbLa$=cQq2>}LI0N;usf#@MWfNsa96^4st{HTbM1vz
z>$|F@f_8oBiv26QZz9oBH36Yw9oCSc3?yGg@FyH9Rmj$IEO*blK-NcBZ+N)@&nVbb
zGu$nX1~WxvY_h<}uCw^{VCR{75cQRZdoS@du<J$Y1-EhStxvU2jr$F@2Rw?BsV;9r
zV&;fB>lKcqfyLcx@;u?I_swU-^dsX$P*X`1q#Mlx^<M@BajHS8Y=!iJN|Wf1aYd@}
z-)=UG>+iR|Y#plfXG&%}d0BEVe;?WjWazvsfeG)f$3bY{rp4z0yW6gu?v>s4IRmo5
zl&%1E0yJJ9?y~#X-=05zd<gZ?uEj!-Bjtn$gUXfx<e!^d4PcOXqo=v0(H_i&j8zQ?
z)Ma9k9yLC<*`ggJuSl?3mI#&_6hq>XW4mMbo<v-g-H>1xuqP`p>Ns5Rb6Oc5JB#b5
zYW34bnv-Ak{P{RK7T$oP#K0u*Fu4KZ%$YQY%&XKHMGuw09C#z4OR9c{1TQWPLKIiC
zC--~xWeZpuFl-Zz+b38;splODHIn5;Hx;fnL0Pjw5wRVTv9;cr#-2aZKDlmKQv&=h
zCDnZ8OPk*|xJ75Adu*BymG4?JTReN#%(Yd8pD^(Z%+WW8Yh<y1GLx$g-Ev4RnVcI^
z&w+HSG8OA{Db(}WasW8r?y<Y23T5}4=zE7)nCFoz$;Fk`_AD{onkK`!n30zjCLR_h
z46O7tu!->Ug66b_^(7t<`*u0l9ns<0DE$|wN*7nM><&Ucn4jj2+4P6Hk<sbVxjg(m
zkSR!3HSLdfqInQKo<?NZX{SWhyVfP_x!g;HU)cnAW140Yq|f-%G8T9u4^zXIzmST>
zV@&Mn(7Sj8qNcXun`-3{N|<it`ZGV=XwjeEtCrA5E0DZu>T5MugJ`KS-UumnnRGGV
z2-MLDinC?cU840m!?8u2mG$cAUnQuC3srz_Fxv3Pw*wzX3AQQ173=Wx0aAbb0(iZh
z9i1KCPqp`c0}o{pc*u^1TeV*=M0nYfJarj;1^UT(U^pWha@^FxyC!EC2g#}RMY`Ny
zEOUy}EYym-I~(<AP4Tr8iA9+abDz=-(JWq1{o7~wZQ;wAsh!^6j?Q$KMff!eC*u`g
zXC5_(WNfvEM>-wg(9g~b9U@g!`B%*%8N~~VF<GZDrRa_f@D)%|9y%>FPD93@Z5duH
z>PXAdd%f6r-~N>wHad^4G>XFMdBvujwVSi`rBz@KDayv3I=fl2r9>aOf>O|%=a{!;
zC)mcr&qGikiT*vg%3JH{?l6=V@JQw21-*f(a#|WR>}X!HRsyY7q1D+WQgY?#&LLF!
zM!uV+8!m%8T7IDBb_Fdqr9-c<CE3grOFebL8IZkdi*0MIP0KzjY(()O1$yJX6E!+;
zR-J0;hQ4OW|3+U&vi0{t?h4Zs<#2@5Cio!Iyk4Od64$PdTKg{5eUM_f$r|dF*dEI$
zq^w}(UH?~|uz0WbZ?QiHhg@~n;P~Q*@hYdllhdSb%!s~~$c$NaJ6F7`hI(dDyr&l0
zr^pdme?wLMDQ+JNsz`xAhFl!k$iz&Gtd8LtJElOjR*^iffI?<X;~CL>Xp?Eh!NQ?F
zCZ&tNNt4&dAXf!wI^LDgfIxgqIeX#=`Rs1`b+)xbA*;fTq>eYwjqRiy^HrM4)Hn!r
zpJ)69|CFG#<X-xK#^{|CdUEJIu;3_HX>c`jm1*=x5=?<S#xuh~jDn!D{AoMEw7D^p
zfvPJZ5wd!n?KL{;{!s9{m2?N*+p4StF-+D_$iNufQCeyoDk%{&z|gyLtk>`qP;tzQ
z$?11|ny-^b_NOShSA8<qVlq^AtLQ8&p=q>KNYyd-4@*)<ASzAhv8;i*q5ckd)|u91
zU$E2!wihcJTJB9TWAZL#;Lhd<Qfd)^g%_fUbS)L>qx*>`^2}qYuSHP2Wr}u8Y3HO(
zcq%{xp_wy7v<t(<kJPj4`rKQV>;<K=-1FkQ$@?F?{HJ*gx9#m9<yylMA#D$bHQ`ec
zK8X{z)^vSde|-;GMbzBvWa87xl$D}l;O*ToR#xH`RO+K2kDh+M$xN1)W}G(i)de>b
z^-J_BABbq^gNA4h9pmzzlXqBm(5hjbF8HI#5-?^jlBdHVBdq{pV@0FSl?-hsjzcNO
zUHM|i+#GemZHq?&7?1Wh&%1MB&s#s|Fa0(#_j*T`2xpW-6fh3&7B>&?0>cpzOQ51f
z`rgeVmPQnLKF-!EEXm1{7i_+6tgfOf#jsnBfV`VtY;XU(RohNm9G%_*qEThy0fKJh
z8A_DQLk-IPZn}LfiMP|K5*-36yP|@pTn!h&Ac3j})c)55UlGLgXT?l1b>7T}2xa@a
z%O;1^5E|u~7OIoY^lIMp+jEqeK?+u%9GL)`RG}jUYf*JrX&vJxt#}<aUDR{;+R*|2
zMR5h`8wz1KB~S@ciBW9UZM|S;O|^yNu0v+#4@nAis_GzJQdvw50A)B3Pj{h=q5lGy
zKxe<F_0s}rsbLoA^xkFg1HDY`X(2|)>ZLLCW)JNLcq9<NKS0;jpc>IA!(mj_sXDSV
zl(XuQh|(2TKpnn1`nh$$YoD?A4VHRJfkb>2B2g=VggAqx0_^1N&wk+eQRq<Yt;0L3
zBJ+Y9+_U}_;vcH5Zg)@@jhmO4OOW23*P*M8J$Kx{7l*H60Z(-J7F7ZHL<a7$C-{&q
z16SCJo5BOmdgVHpot=_i;Y*)x4F`k+x|5qd$ySUX4j^V1knUnd1nmJ+h{bkwWwf`6
zyiUx7Po!Hz-S_-LoseI!U(gA6Bh4K;U}TnvG)U|jNW^90h}RPvuzArvHsYoN3O^HW
z850%lpNW}vMp7`bZgOr3bC!`?I&R=q_`XE(M1`>bbaZiYcz)hG+dKIe<zi^<?Oz=I
zcvwygw^T<|jd^Yl;52;HY?tWJgB)yLWAI%rGO5dq!YXxn%0)4K%~59pGRow1;o^t0
z!@UDR*A~WdWc4Q*6QT}>P7el2j|NE%n$76Gabv9<9`PurcP}sMulFy0&gzoozPpv)
zyF4tk^GqVaPx2^yuy?U{o`r%LEhdxeWV9U*ERz_R&vWoNIsEwohr4xobdWd3S*&M<
zZOW$Qvz%rrKK1<Ye`XtP%jIacHl?M!j@d^_aGlAHwACq-VC}o}#il(Sdnp^)PI4}Z
zJ;c*uxH<^{<-&KsE{ZJtsEtrU5Q{#zq0n~2bfb6C$S!mOjNdvw-T&9Qj=8iUmDLEC
z?P=>MIa=||gt3MU+dpeybxU-2r;xL97H7#>HHVWZebN*8a|uQ>wJ6D~CTFLa(N+Pf
z8I2iaqo;Xvqeij`w~&&#nCcxLy?njDuhheR+M=iIO>XByG<G|<Q9qy!dvAP?TkwQ3
zIhObkxHndsBY_awqlX$CR%R8=NKVkY3~v3P+j=Ql5Vr_V_$e)PsJ0;jYCm44%Ue(y
zkg<Q6YO`w%c|_N2e5fW&uL8=EH9!JTGQwR(x963PUG1J**50=%%2tMu%7&^qtN0|J
z2G^SB?7T~(f2|PxrxM)WDkwNW<`IOcHLePeX<A_UboklMggD`bVT83?BQ(R3u7jER
zHwB)$ykH7y1t1loM|#DnbkC_aWq@;P##m|2$&lVaVp7aMi_;rOnazc(g)`AAUvf<w
znJKId3roNIaGb5dO`mN=K+M3=$)<}ibh1`Keoj_=Sa!DHe*tc`;1_1*WUXnuY}F{n
z${Ax~a`Lm~2mL%-{M+NF@JE_nE47Wz=GMm6#(HPH+OB%*UbEM98|!PGjkQ{}zEN*B
zyKdLraG!<a&a>Yp?$`|?&|_CS)7z-RXMMd%f8i7RTdS{cRR30M)YjJ4nj35DHF#cc
z)HfP`tA6?J|Lt!R%w-KtjzJI3D>v<ZymNn%kIaCgESdkg+yhJKg3b}Pz|P^@Hk~EO
zD9@ys@=QKy$tSDVOES&v|0o7=Mnj*|tx-A{w0bC#1D(uZA5mno7j*DyKK7XV0VfAX
zFF5+8MV4DCi;|LJrqCFgjN}!5BE~tqo<w--AE&;RJu^CVezA9P*g88rI@#YlJAv;9
z3#F3gWwLpV<4;C{m;{iwmuIi{E>6$>oeQC!1L4iy&kH$lzkpnPw(Pu;`wJk7qhWME
z@<JjiW{dNKK8}}&p^-ZgoQFzA)|g#gST=rLjfc0^9<)sq{R_mt5!An66zh&%KWGhu
zcwFzCkP1TK!!07ZXEGXjV=##)ltudCM<9)i&hn!1Q-+@#nCnQ@fqO?8Y;glHt3Sz5
zB@uo!?Daw~8ul!9tvH+#_o-7V{pYe&D~s&HvL(!wYiyz!RpGVp>*$B*M2by@f<a5%
zN-}b3mY^jQkL=hbUCKzr%pT;}E0I_@{)TP=O?j@V#^Jc)hv`zWcvtH12EkaIUJr*K
zTDQW|HA7Fbhg`;~LwRwvH6P})0z|EGN?|!OD(Cn$%BRzp|2*f-jUDaP!aQV-dj&HL
zQ)kl|69e+K8#-;z3+(MpabAT0#9J<7hHt)cCmQ=#S00Abk3ATG5Vyw^9P5r&awP)F
zrUVj!uIAv!il~z*)<7(XaN*1lB|})`pU=vk*2TqRa-hYnVjKiGW!y3t5|f>!g!Yjw
zm?eSJ!IG+<%Fi+>ur;mHX4{S~CT2^@#7oR+Iq{yRm3E$k<dO1=Wl1_E$||P~RU_@8
zJ)y+%6y$_j3FhVYuO`EZNWo}?JFdLoyxUJq==kWD1fuVxHz*C=<y?Vk+)|K<=<V1Y
z0k*hO2`uH+7oFkgzD0=xlt9m*g|m`^A|byMhu2OYC1*vp$9{zQ2qj@Gs(kM`#F33W
zQLX$Us=u?@pAnGgFALzgbk8}SIYkC6W{MgPfz8Zv9bW=wR)S*kf_mdD!yg#2BGli}
zDdO7iS6)nH9I_wkXlc?CR$hQkOF_CSCW2&&NinM=lS!=Ua-mLLephxK-FW&!O!p<H
zEyJlI0@(hEq0!_@1mNM?^dTv#O{Qs*<b)GvPg48;!YqzEtZ7qCgtTVzN=!;8)eCF=
z;Zr*>P6d@o!51c5Sov||tdWunrRVqK83`?nS>ju2oRhJ}m+d7ibJk|_p_xBQsx}8m
zPr4}{ETRE!yz!k4l!F%c@eI_69DRS_N8;#ho3$J7uH$)K&|up4cv}Q+wc);9G&VJt
zR3>;TX#B#7QO3OG+2oU5Cq_ezmLeE<8qR(tq~D5YW1Ax;(wIC;=-H+bQ_Xn|YE7In
z=28Wn^`q}GtgYW)&Fa3qYjEm%WwXDn3`qqmuB$v-PLwKha9wZgd5-IZeY^n+@dOs}
z`yu)xuo$B&cr{h!D1a~{ut$7=Ou!9=%st(q7Y5%)f@RVeER$q5$2{N7V;Q#lvLt0n
z&p>txb}1E5Ay~7VVF_Y<=s1I1D=_#p8mlK#*2v``ebIQDT>#tRq-qp>rNs}*)=AL!
zKX`qPA$>myKD3Y%<ZX@=G^tPE4`Kc8engRQ!)HLycLG3vbo*Ch%>3`u5c+|rH5_?k
zX{g8uNpWeYK;uT%{O^Aqfyg%?#pPoTJD^>}Cwh3#(K&|}+5Yj~`MC&wA<NjHJM|?*
zpInLL-b9|pS&$Zvy3L*5>k9?X4i8)Zygd1rS?80bho;Yi*wvqNh~ZCZW^l%-$f@S9
zxz1!;z8OnO(uwd*o|Tj*DyL9miYl3Hwdx#BpB`+FV?O`?7=!<cxc=#r@Bgo^H#VAa
z|9`!{R(tgSe}zxN|9^IY(qJyvztErM=JMk80OpC>YQA<Wj{ecMQ(axVu%2GGL2tRn
zBmD4M+?mI67}v(ZFhrl`Rp9sh4onTFjVtM}=X~()LuZ5%iZ&C7H(S44h3`CcF3Eic
zT1zSLd#oZ?$1+_3eKF8O&=mnG7<Q#+4ocD&_8A+myXrlE;{+ilG@um5>GvwmwTBnj
zRe0-N$W<o#4~Ivuf4HDj$*2zL#kX|)x`p{k@dU%g+0#auu5d#fxZ=yBi}Tjo!?V`$
z;g5&MWkbm(8iD12s|kHaM~RWUfVU}(<OL?Ks54Nd@6y{L$qz%8_L9-t&Icf5u0Yak
znuRWt3U8T12Q?L2i4}X63Ry1wC5C|4gD$5Q1h`g@F=Z$GrWKa!k{+Q_3TTWpjc0CT
zS2RLiJ!GPl7sNGqlnVdusK;ITyc>I=@Zc}voa!0KD5vk;ctY70h3QH7OZDAqIBDY}
zeQ+{Sf`|;`E*f~j*TkjvYMaN{>{=Q`yd0W=rZ%=Ca$`n#(cqY|7mha-7g6K52uWun
z^3w+Cr}QvcL|~J}H;M9D9q?Cb`sGuhGG{<#bLkjI-xwccQIpWg9?33G@0czrkd2K?
zW^vs;F&v+mnf8Z+(If&RcR2ho8R6AtecHWki+x)#a9IM<Wza2xD=(LgN3_J{TXF;q
zAb2Vo7#kOlgj@u31OD@+6jdI~n&E(sN3WW09mbe*w6z3D6U%@h*uDOMGQq%iV|gGn
zb!-*=M6VO{)S91`9Tre13@GhF^e?Cf&|4+y?NOpXYMvy>svwxL&`Ip66lUjzXT;F>
zdq-L({MN__B0;242`@B(;uS~rSqDYiLaT(W8Ga}eW{l$YYDvV2_tYY5oO`U!>UokS
z8|303OPXEJ>$&&|!>244!CFZy7L7-=Pm`%?5PE4mw+w&ztY<nuO0&m}h6BIT>bPNR
zR1(>EmdjXKnio8aPEHT_q|(ve@zH;dPF@R>Hyrus*u+19N0%F^lY@J94mm%B^$EY-
zJ0g|bh%Zb>EllmxadbM}{0ksgZe2fu-bQ{uww1?M7vqt*$kWFO2^)^y7Z?$P$&0h;
zHP|5|49~NREFe*E31xavq+*O9%&RI55S8D}B#*rlBcPk>b!qynRp%aLV~GO~(x_Vy
zc=K&biIEhu$&zt;hGjCK;8~`|wL9)kVMJRiPi5kYWZ5UFy%sH&ryxdRBnPczMffv6
zwohw-FF;s%K@Br@f3)rOXZXx#|Kq$XZ2nKP|EslHJz@V>*B|ZwuklIP|L1RyPUbTI
z|LC4Pe&+eiIsfT`$_MfP)vAqp%KWddJ;wijjZbp^U+jVHI+y=X!rH$$ZoNLdC<S!Y
zA6P-Cy)Dq^R~?=B5mPa-Cn_rMsMmK0xfAuVw#+RFP@aSzr8-;X(!VY$O|DwD8-G)m
z?fVH7GzJ5|{mc40(X#!#Pez$M%Ide#&m?|vAwr>eXE5^=s&{R~3I0HTxE0xz1w^Kf
zc5lfq8z@Nt?R%_DHMwFgaq>y?rfE>;eVjmlE&AZ&COv-t;AHy}@z)2KYMTP8(G^XZ
zQb?Az=oU-k0t_bH0>A92R3X(b8I??SI=9H1i)}v;m@(B#%WR!=+GRmTJ8558#+7v1
z!;$;j#4~OVcpnaB7Nfp9zLGcUTu-h{Bg7|zS8&b9y+UK1%7NK;<$@i=fAK8n&5q$T
z2oP1^W;9==_*vqFJF5t02~QunS1LZIoK}#4U`YW@LYIDO?1FW&wj=Hyl~e(0sE&)&
zBELrBe@h7S*QIUe7m?Axuq};U>WFRxveP8wH^c29^le9fXy+KWnK2iGIAz}DR{?`v
zQgy{+;cUjQV-J9bQOIv7Q`J=%uYiw%L=L`lM)03AK`UQC!(G^Tm7v$3gx8(`8+f-=
zpFd*{f5>bP266tvR1iIkSBdwPoM+_m!ib5gWO5}9ocHMK!@j{+>G@Kr<pYFe1Z)B#
zEsnB>7vh`1-6$o{-Vu-*b%G?yXxL1oiX=-#6Qv@CxS~eYuE~8QcapvCEV1$^=3I3Z
z`vrff8%uIcdO|<tEK+%%$eW@i0LG)yFZ~>s0;(&+62$cS1}87>f|=DReJ1^$eZo0u
zpOuvrJ*v9rrjUW|LT_;tmbj;Syp(9lsYqkABxwZS`5H4Ezx;}l8Dy)9Hw=MTaHPb{
z(PJgJByF^cLF~pMc0UuMoU96>cpxDrNj;4_1=~xA0?rd-KG<BEA<~Pr#==uKCE5}b
z&T6PAM4qE5F!@Cc>Zch96Q8(V=F}}bbQVPMATe};%6e6vW%{ahIr<%DY|3L?vIS$~
zs;w>zujCZ(&Qx5mBo+TS$%89-ECE^d$C!c(#xkaU8|=Set!+BDQX01wq-ZMA#H^{b
zNH5NOnBK(&tx}l3xe9mT)7%UoNK8xe3_~Z2f1IS3`}AFCjgX!fYDCQ6u=eu~A}#c|
z>AqB1lBvxX%+!hYk|ox2_r~@6ydDgIOg_<sv~pBO(2jG!{8+kMy}!R-VhaRg2xTqY
z9i+d@re?4}pm0gm1v!h?-<?veTCIxW@Ek2bRY>+}+rdIKQ#nNwfqOwuC5LgqC2kj;
z;k`s%FY-E|q;1pw65a5IXNPaSvFGq6R3OWT-pHlfriHQRV5XzcaY5Y!UKZ(9WnmSV
z89LV#myO0o;IzHy*7Jf@=P2L=oE=w0N<y7r=nVL*ac|NmI?Ao9j*C@k`wEkDu8JDZ
z%sCSVf(x;tx}6h$fSI&Jw=jv!X2#eug`NY2Vlyxgnx0eD)oK+Oa!D{j4)|H%6I=qT
zTAB)eR`}DvGcXZxBiM468`>$}PEAr=vE*k<z3=MPj}zy+2K-m6f24nf0MTWPhl%gd
zZ@VDEXd*=Xva%M=3{if!gs0Xr-z#$fSX@6gR}gCnF;#xKj40{FQp0FQsFffM*xcq}
zh?@#VD+&yLNg49L)>!<RXr4w{Wo~}1KjR&*Zcd4uIT*~2o}Boy74va?Ia@1v$GPdp
z>LUfdfFmo_c@_6b1$`cf_qndD*}PGSvVEa4Kas6HtnaEXwVf2_RdIO4Du#Qen0LnD
zfPCkScBvF`r%;zt{VCZ#57WzH`BsW~I;hEqHJ}y9hqIs+$eB&v0*)CuXn#`Y4E8<d
zok{vhiW0D}A^XMc*g4FVZ1aS=SKL0Cp)c9X_fl-(O<l>UX9}7gIFg@${d+Z6aS_(9
zGfe0B8Y~ylhh0B$3Hfs_n%AA{N$_FDWs{c4*wSe(n+4a(Ks5eGu9}I`0w*f3@O#*G
zR)GENl@_^b_I;4)D=o|eiDL1UR@9h(tz`guxs?E?uehRQa?Mq1nM7cQ1=0Z8O$XZj
zETI1f*In+tz4BV4UU>1<bkvLzC!~H;`MQg+F}-J4hb6AbrC7ZA^DoBMSjzR7g)S}3
z$=<>eZC8p*3bkBYtF)xZ11{9*+x36cs{EivtX_@9BF$@NYLG^3z2+kUD5cFNyCFUa
zb;i1`KaTEOgGoQCI9;#f-nSxuV5AjchsB8FTi^?ouQ&sA(Q@(}*N66Wg#kRF1h=w?
zdZ5;|9pm2hU^Jm_0Q+=rd^~wh4-9~GU_QMvzfznhes=Aj#(E{Yjg(7=$gdPc@w03H
zOhQEZrvs!n3LgLJTE8^386#t)h>i(ZIr1!2pU9sNtqDoEBr9oitfH81Znek_1^Ws>
z2s#NlzyHAEqU-Q2HsKzm=KvNv{p%7e1iS<0z@xeQ40WZ-$9X@!j@R{q4(Nx%1d-75
z`)y5TL4nu=5A|aD04^sJHm{gnriS$<MKn{YbDZU<?a-Ij01t)G{*Q5Chy};IX>2B+
z^8oLvA@Q|M&`6{?)E2VWcFl4Lmui<H!=}-N&-8ldVsP?oPu-xSyubMjuEezDbM@eX
zhgbu)##cX{VQ{IEF}NDY+#^dM90WngrQr4%gJuGseT+u(#y1)uwS4Nq$*W6DBCCo8
zw@^)9Or_u#E30XW<kb}mu&k}t<AU{h*_K$-54-N?v!fuR5s#rX6B?VjU=n?8ko0J7
zh)nlnn1oPu=7%drBn`57H9r+-yiy?Zi@Db}tnLR_+YXVGY3zizo;M23rW}F4@lZmL
zC>X_!L8@fxj|CP?(hpaa)+SgtrfwQ{lun%j4A639Jc+5O1t*h$)J=hu=DL^;$#z*h
ziK)h-lMg#vsSRYQ-&uCf2L=GCv9M-oY@Ft0{hT1OifvzfP74`vXEBrIb6QA^Kr@>#
z;>qS}<x}Nz{GF**wfM~K@04qDaWse)+bK9qaiK6G$uI&E4Kaa{Z<Sh=tRTN>>OI<?
z327BGWoMCA4A7QVG4r>ej1iE!DUfEARsvE?TE)!nFCwj|jV~vyV&n8hq?I*zUqo72
zgZgEpRcx&P7-<!cgLnur5ud5~t|qI}EMfwEN(m`pyQc~o69mZ?54JO_;GAYS-6WhD
zWdp;Bi3Zz+_C+KEHu2>IgFQ4~L@t=4_C>^kIh<ccD%ivP#|VY^mhD3bh4{oQ?k%2?
z&e1S(zp&!tJdMjZcoOTNt^F|is-FIBk-@Ihjn(Peb#k>VUCVfqB&*ny`eq)nA>4c!
zbT&N}&N8_H3y5^yND%^0?y5Ybn~z~yVu>HkJy*DTE*Xp#^1Cm$m;?dj&xa*}glZ8%
zKAf8j5<r3wnhaJOQ}{q^q{SE_w%;~`1)u=N^@Ze?T>|U_aGhx6A=cJ34z}s?8S7Gf
z;kLn>YxjJsk_^<QU3W}6UNB&LhUqAf-FD=}BuAV=U3E)TD6oes`IXbdV!R=ul#!bt
z1vRn3EjF<!PFpnY8#0()mt%m<VDW6o9Qy4`UgaHf8;~79mkkPmr0)1%d4V@}`(Ge%
z9WS?_4Sln2XM`ygI1a!{t}wX~Th4S!eA1}P-?Rk{%l3`2=^utS#^(ABG>BO?CiqXt
z|KPu!QTUP44V#%kuxTXd#C&LOD~IL_Ei6L&xqgAI=ohiAT4Ou>#v6zJ5NF8~Ai{%e
z>hk=PkmDP=(2AaIg+Mw>KQbt_ZObqyRVyE2S~YLI+&?{eb@aOR)7jC*Ve94T>2d3y
z=cgx_UZqpUO(@KgVsvX^K|a(}5m1-DrQ3dQm~~`gmjWn@f~GWD0$@ud=R(no_6<rP
zOnpy6h~fs9Dym|sQ#R7YNQp4_${_k=tO>j-rW1?Kgtdn4zJKEnCt(F8W!dB#L7}eK
zo?I<iMEdL%Y>9v}sVdo&nptK$OM7&Nj!r_VP&1Md1Rn%fyALZhHqrG%{xOA4c(_Xt
zC#y)`{diAzQY#{UE`4MT7=qaB5+%jSKBL^+3!E5d1C(r#XJUFw*#sL*!1@?XAsJqy
zRG>(!@3*fyoq{DN!L1*3rxd2uXG$49FgdMIe;Sxcgy&$V6q0wNOeuCJ4cn#9W-x8a
zFMc}-K%_e9ZL;_sjv`C;p1!>>3K}w330;qv1+$zjVPz7rGBFiNktc04Hx-BNf|UMs
zaUm(S@nIoxOa*5U$3_Jqm$f{`S8^`RjfUfh-Y=<DEknw2dke-&kWTE@o6C#6m&b>c
zM-1t%l9~y<rfy^jn|mmnCr_|G_Q9zC?SPhmx>-MU6#sV*qf=V?c0k|Caa?pAz0~8F
zqz`#w%oo)6wBf%zIX`-Ra(IxU<K&L<1jl&tb*I}zj1#sk#^Q5F!XAZ*0I$@wDRT#J
z7inG0EUH4vqW5)a_fZlrRK$B&+6fapEKa<6sVB(ymFd@&^m{l0CdGh=mkVd6VzvY~
zEX>)YcqW~*<+xe5{=ayTl(7ETC24IGrIxY#!T_618t?zGch-74^OPX-`w80ZUy<6Z
z%4@UwixJ$k8Adm0OWbE0xX_qo-Ogj~oK1oiikt;t7&9JvQRD|#;kG{X7gKTF_HZ0Y
z^*phv^sidv3QHCcTtbob@ppPXXOD7bZ4si!Zzc86!C4>>JX`$OvF(C1@|d_C_s%VU
zD%Q9WsGQxn8FDS!G#{x36SR1rN@g(*=EZ~YA~cNRuFH(KAHDG~<CSg3#a?Q=EbXmr
zNG2E6zD&-8caCPkxE^J^F^%XX<DgMI8iVM{0IbG?N~%Lj@MBf}m|l&KmWbMPGlc+=
z@?ud$rKXa?WKDt`x_EnvfCdlc?@UTSTb;=um)2Cw$D~_eb?1S)>P`lERByq8kPo5n
zxr$fC4a{aPWV&tU*1ZoMZ{SB9STc0n(CH6@EBtE~?hFSWym7;b;LxSRq1&Sit0qBb
z7<7FUWx$B{&JZtc=ImmFf$LmjYOxZ?1itCU4V-KD2E-i!g6b)i;r}RBZ>eh6leau$
zVlZ+AO*qw9S>E{^>%D<EJ1ZW_S0SAhc6;ehB{wDCGBcz+&w9briG3kLTB+;J=3mJe
zQo>amKb6ma(ff+V)B_$uTM9G=@&wvscDMZ=0kKT7ly1aNS3MEh*F$ZH<{YOY6@Ae>
zGhATZr2DhN8}n>UvC}Vv)SK)ROU?*pArWQU(Z{3ZIVyR63UM}1I)8Blj3YXHtT;Y^
z)~TrnAH9r|QpJbP&^n5@ei_h3e+GqON209#1GZF<DV}qN$R)g?imQ~39p)772gM|a
zCIh0}%>zbWED6ANWdH1Q_<tA-A!a)Wat_r#Y)=sC=XT>Q4q<8^;)a>%idMXq3ORjj
z`)tNJDSN6LP3iC46W+b^2QY%HbFEMlT~3SA0j(j7QAC}=v%51rU5&4jk`zr!Gta^#
zuM$uPqbWo}rS-_lJ8fsPkvPSd&AA4R!6lbU&iFBV7Xg}?1Sfg(C1=Cew_eVUbEKn!
zrsiq)6Z3iAT*FHUBFWgfNCY~S`4f!3h^2g=7n49`pQh-`MqLCaU=!0&GP21(AZ;Yn
zUzZXRYA%>6_Q}uBtvm;dzN;winJul<Jo8<pUsMymFVd0Crwc;m{MjXvBYwf@WoO#^
zGV|*T@J~kay9Z{VshP&+VInj6kYppBE@v*7Ts@zaRQ9hg$V~C{QGXeB%1q!i4@1ov
z<f^a|Y6JRa(1A%Yg7NTBHuA~gq)dFX-jgTJ{1!h#u)}rL%*b$<#e=Tq;Nc|D%%Ce-
zRfS$9FjQ$7J{Kz^6~UD7uLKU{n0o*e2+mv>NcSOWm+Q@&SiZX1QIP;s4hJFpG#r#W
zG|e2J&ws-3evR*L&OhUDh9xFgY9uhhZC!i)kvD$WKwytEY0!^JVEvqy3`}gtE{ogg
zbkwhL;z+v5WrR1LNC^F|hn~-5k}t!U7B`IQ3DvQaoN!=nthjm5;E92SeO$q;punf6
zP%(<i9xNFL7;=isZZ6T864~W!=<>`($_bnVWOZQW;ZLh@8=rY*VyTmoNyWUD{bNa#
zD9<Y%Qx7{@rw2Yy=qiw(k^aRxf|58mGAB&QOR<wg{BT}8^A2dug`;)aDSi-Xehr1i
z6DlWCR6WS~l|AdAiX3$DLmzIzpE_-Xq`53+o_x^<v+RQ}a|*^8hhd*@z7b709nbGm
zlAIVWne$Qlz0#;{d%;-!eu$g6tg~b0o{vr$o3ZPoq5~}Zh>-Yk@9YS-xLSK>XM6vq
z<0Y7)*pGa-58@IiEGn~wAQl!MPo^vw-Q?`;fl*Q1Xp}g+f_DZ<nm_?{m9a$I0AV;(
zO^hT}p7_??wD?$ARwgEfp(PqN>@1O}R2BZke^;jG3nqxCPYFC<i37_08y<EZ5OJrG
z!49C|mn;lkfGGn!@{M8=eY2DEmNsWaO+wIy8)uusIn?5UR_2#dY!ge(HY42Nje)Y*
zY2G$K)4$2IteI!L7A0uKt*l%b=hFn}haiZQmvTfvMzWAJN?Kh}V}B`YBu3vB0R}g|
z&u76j9+a<=on2UXFW51!>+(HzBu0f8V|+J}OkuqjS4$<B<g=O-z5nIA$-C{G2F2eT
z=2=W>N*)yyjAIl@vQR2rKbMwj4Sw%DH(#5gKrF`Q>ut?xs_&mn#uLo<dCSMAUt`&m
z<EGPR*-U1U;V25p_(8emjmJC2K}c^j>|ARr2rN@tNtDT@Z};YB;&>-KMw?nb6%%k*
zo+_eV$d_tHr@y8nzBV{M?Q1`yJ168L6?=2$^*{x=cEsr)KcKtt<mGy$vUBT(bj2Pt
z-@c~+8qibvJ|~u?P#1!;f9-WXI4B=ZPv1~<4O%Bb2k-Br7BEzX!f)uxz7BqK1Hi_=
zy7I;ZLg+~d*1?<kMq^Y@Fu*Jr2A(tQQT=ez4l%C+t}qhGm9f`H1&hG&eSp5W*{?SV
zIuy=$>-VwY33LuI^n3gsA%~Mr$Md=&^@gM$4Jr_;h9iw5YMe|poEpFiC<~1(RH|rh
zo6-aJh~U~Dd#(?_n~<eFhv2`l_E@RGUFUG`_1@8mJ!^=`OH0q+d;H+VHmvD`;rQOn
z1Ke}{{$z}oLKsmF68T>l`(FY5)z#IqTKGUAos5H+gEFPLR~mQ<NToxYuqlGHWU8My
z)v(j31jQ6j-eomi%~%+g7A*h$uhs9(B`Cc0BN)PxC?cK`Z-4*K?|DrW(^E_m__RSU
z{;;D9UWtNw&dRBrQg|Q=tGSv}))i%qnVnO*ElP`to>Q_fO3L}4Q}kLC6=ET^WY_Dt
zlYX?lqowv|1uKht5~KuQlFakjvCFMi%X#Tt0q04OB+&d2<VWbhTO*bXkrxBX_^Y$=
zcxgt8`GdVrJDQ(;wx68nJ1RKX6xwD=1Ixdc15LQ+l%r1cI#rqcth*0B8RPTmbH-z3
zWyLuRx~Zd~uRXv;`)`k*Kj0(BWu>;!+1%RL+F0+bSKC!@-D~!mZex9|v$0mI);H?S
zX4mby8}74k+<C^KC99ocZB*g2zTTw2@QM9xY}9J&f2%cWYin!GjkWa}Jg?UqYiob2
ze%Y?`X95g4h9<|uVU$;H+WUCt{vscFa_66iZ}yIlPxo6F`+NI89Jcmbi{FCM)}M4e
zM@i>rv=I}y{b%w?OFmh>URqpyw)~)<X8_PSzzjlkUo}eS1q^Ux?o;cv*w~?0)@cDK
zar|0#=*`~G`^S6d=dFXoS9_Po7ob9jk{~^y8@(ZbSdKl~k$W*x^&o-W`Opfy{vf0)
zwJ$LBnL|$<d`R)5Vtx?Xx+(`r0ZhhsVL_uxL1+r45Z?-1@F_HN6m-2i)O>sXIK=RC
zZ-9y~=r-4=;Brw6Uyqz?(9;9&2GD4GUf^I1Ct;H4ifnA;h7q^&1KzL=jex#>)xUT8
ze((X7z&^-p2Tyo>K-HaY`Nq5us#SqZXtts|BKy&zirmiI@er^14rQ_5D}+Ra%X~5c
z2|gQ4BJU2MNnnHMZs!XyoERI)8X+}aio$MyTTb{_nvATzkD@V}gNs5nc1mo~8}%v<
zl&P>*Rpv}TWSxo@V4QH}cgI~UEgGo^y3kLK;-d|>kA8qffQ#@hw_Q<c<wfA#MXjq2
zvMns81`lI>_zOj!$g=o5{DnchvrF?N;41V(*HiCxO;#_k?8=M&@V0decGuW>A&nHe
z*By+gS9s$yXg=(PE<259O=-Xl+wX}9+8)%WC-|4)4BYXDRf6zLSUjTF&z+N(M<?e(
zjW@8PtEF3%ldojZ5dOm3iXfWu8!+C12n1bm;{AA<=$fqg;)PRB2A?MoA#F?`Vj^I-
zEP)n5fLoURQgI@Z&}UwE0!DU^tlQxPgacUV^`F6lKKAj@3W!F0*XsH=Xy;zDumGxa
zb)%j>c>@Ld9!yN9W{*Wlv54vk30%NXVy#<yS}T|Fpg@hB&&k(McdQWhc(0$wIqE3r
zfL#)%cJ+<>0j+q@;H~aP&>60L1yv(?(5GtmDuEjMw$qVbSz!%3j_j^Iyf{lzCMk89
zT1>t}tDYAMY&o?67`#%9ILlIyu$bhOlv11+LW|`R{G=~Q0mH7Yr0A$Pw$u?p0CJ%u
zhe^mTF~$c3A@*9d$EApBYGz7o;uFJ+Bl<i0LuB{$;kEXC1S`%qZ;HgCk4%EaYY@>;
zN-MD`C<dH#Ut&aJecCa^I{Zz_?i8t$DZndfv?ZFy>8rdPV{browTEs;5*n73okeSx
zDj;O=iaf99jmLCTnBh}ovr}960GU!j#U<;GdI1<X&toiIR-+4$7`4vza2N_%N8TC@
zfeT76&W^Y(lo5?j77&8a;Ew19t|aRFuI`hR1<vshG%oT&QX4okpwVH+r`)ip7|~nS
zmw2fW8yXU8yj>9ESMUp3{ITcd{U0t*{?*z$I5?}n+&e!k37d9V2ny0w0O=a925Oja
zge8Gs+T!{i;29h)aOFj%C)5UX93C5W1#5@}TY*`5VK3Q>3u$3@Myq06jN6dpK7*VS
z%&IUI@k?3GN3}Ml&!IAJ9`XrM3=XrQu!}H%{#nNXSf#_HU~zPw2&vlnKtSW3licnK
z#{0(Pc@Y6>&jRJ_qsDl{i`mT!#U_;bM(6aEY#YVi?C6yu{z`WrtN_W$@?j!mmXI)A
zm@w7hb%O8UKe4eSI*sLwxwf!Sn6*)l-I3f<fN7EHY6#>ulX0gb>!&61WCQG0R7TuZ
z7UjN>@q)mXQFQ=g?d{&{L-swI7!TYLOpYkz*%5_cXF0nHmYh8F98pEiLIb5FvJMD@
z$|hjW<V;fz&&&f~AiH_p7%QqqRA*tG5>QdC7p*dAYcr9w6;e4aXm^1G5bdF;C=lny
zcOBR1`aNI>pt!7{1sZ87q$&p<>uUh2!|@&cds&__2kqxPa>HvwwWbr(TrHwambNGB
z0N^g}<e=1adXorMCNRNp=VL+YL|}uau$&=@cO71Y77VUd;RMzu0nhbGP++!;@ewB;
zJ0HS$d1ENV@?avB?rf!7jV+O2sgxJEP!Op8a5#by=?34UAw3#_0UZs+yJ*NeMI18x
zz0fEHDnx0Ich4jM;6<_al4=@E=m>dth=qgUt=2x%<bSH{pIvxnO1IgCTfS_Yd8BbI
z7<hxh@WzX=8K?o$PLhu-O#wrr2DII%_sqn08UnVpie1twm?cUeb;|bZ=W)xcta_TG
z)@N#h%Iu44f_XYM>4B!+m*D{TdYYiHK>nJl;GaVcWT`x>CU|*rd471nMiwbqY5E>1
zV)WMBubVCzOq62!Rs8gay;B9!)9y}E#Wc~A;b#KFxb`WRTCH{Fs-3*ROi?U3o+N#b
zEnD?l5TuBSG@ivE3^i`q8lMfLPk)5K2Cg8J>YXKF3jxiSwQ70&-=a)P)5GL4Ie3=B
z#y4yfQrS#crq=c)l(l?CO=@VeP60hlDrmCGpFvN{5g##Qlx99jsa$Jus;XxXq15(-
z0RD^XeAC5soX}H*RgOHFLqz2ZsLxjgW1ahpD}&~dE$Ln)=2p5$%paTd58BEd?<%FQ
zXQ=V`Ac_J1A#cvXtOGgg29exQTg~CIAceJl>$lK&4!m2HkPSEg<huyy-U5XX*9BQu
z^x(I?-f2-)G8iDG<-~zEi+WxE%aHFfKvbRHcsO_<0vb(G=6<nwu^txym8{4rjlK^Y
zMUQ~Wuy%X!yoAqm69%1$zGiGcv7>Zo>yT^Fo>p^=NG=_CD;IhyuC|i^511l24T}u4
zbeU5W6N$Z&IY44q=HZDXGvKDr%-k^ScZHg%T}AL17~SBLJf72Ck)X;uu^Mhh1=R?N
z2#uutims~I&Tp!j$Bg`q+%e`G8c2;Q>wr*%5ex^IbdMKUgU@7@5UTJ-G=(l;R4B+x
z!Rm6WLn?fqir991G@$Q)@cQ>9WgF1jatciOTj159&A3!&bUBM?x=!Zq77;Pd6Fd4+
z8u(M;h>g~;*9&pODi&KxiGg+<ehce~1jY*y$dRW`Nh~#{c~1v`l;sQ*K5?PHqyV^m
z08iS&Xh<<LjDV0cqP@2lXRZCiqvNBK*90D+n0SiVW6>2{@{MQ|OrL12evHB%{i0Fl
zXlt0ept1np`N5Uddg#_CB_<Aq%m5mPWxaz5CHxW1;g8WkaC)qCg+z`#w?y9;4HqqP
z?8W)PNlTn-;vy=@SvDj1C&@kqr=6U0P#knhZ%l{!ODX{3s?*}<6-R^yz20vfo$Q~L
z4M#{xygE5QKD{_^onvqmh7dVVoZ6+xcO_jIR<?__otMaty71qvn!C#!Rc?>)(_PbH
zL=hKI%iLp68D^!^vVue$7NVKJc`kLtPB?NqFyEa$6$l<uIJ~G$`<|@Q_J<gqGXw$F
zb*}v@%#G=GyJL6{Hg^g8jm6qt;Prf5J{bZgQ}v?Gby=&q2?{WdyD|WNxoqwd2p)L4
znAj#TsU#IfTbz`$G_fw|rsLuL^Z=WLf6bN()5T%VLSjm|B&Wt1iFEP=ALYmN`0sWW
zXNidM`Do!{`UYEuhObW{Om$i_J~KP4=h=0eeV0MRuM^#K=k8oIPXzz?Ebb(q&Q0^#
zjmFQg8dJ?iKijD_#iuogRcIbFh`AWD{DChht-rbKOxyO(-S5uvUjSO{l!_97<wP?U
zLSE|<I|rS<=LVBeTm?*OFG2~lbNemoGdaSh)v>V%Q0bYBvwfT<DIKlYG6O)!0JNM-
zXy{VIOz#dUA_wUIz-U$0I${9g7Q}3(UQAGT#42vte)z&-K`zu9-Ui+{-4a<;=b;@p
zz>OJPp#_DF6SC8UcQHX(T-;<DrP(`w#pGy?iDrY&b^lD0s{_-M%j1sQCDJ+nBwy;x
zMymORA2+=6;vx1%O(zF>1XP$e8#9=L-kLH<Ch{E63Dd;_Qr`%RF00U7ex?*^W6~pH
zZ8sD#aB*n1WrH4mXIUy_d`jSFBdY$X*!aq9QXGy{-iWIqx~<%{hn@!GK;b7F@0L&$
z8I%_J9Ip;?K8XsqXE&GD8n(oxgiFSJzr9#!rLrk5RZ$wSPkdG(Hp-UzspQo%nkVrQ
zPb#7*)~tjM&%peP4SJlj%CnHh)^q(??f$|oX~VLBKOo&moS|2<PNEcEw3bf_LUODE
zS~e-}wF__{?+(|No;6hnpEEz8d8G20l(3~02`F|FkFk)b|6)#&cZJ$9Ynb@sZw}3#
zXFkj@zr->Ox`AGbXP9Y8doLtcHYwsMCZC)zG>EzSz*GTi(bi;*B#+x#mQj%ok6UAe
zVB%_Gn9?1s@(B4cGlQp+OBeeELyoZA?86_&KFtD8;GYy|GxsPq8>}NPiGGT>J)%BN
z%aqw1pUi~UeDp9@gcf{{EG?J%pR2<frlZK$aMW`G4;?n{_`ZT!%m*VUaRd%CCfxGY
zGV5FNteNA71my#~59B>1@&*C>6y0;MsQ&w3r}Ur8Qmu@)FC>b<Q%AYT=#-rf#n&zF
z#>x=w#i5T};4fmhrvtak2;iwD^BR+L+Ztojfk;DIaYUMWOhNV2(Z$K(`FZPX@8n-X
z896VTf<P`!xC*g0EF8IYA5Zm_Vi=-#xh9N{Bb8kx?iOUzBlZwG7kfndo2}~w6`_>J
zc1;sCW3E}FnWT?an78KnNTK~sGmmKewT+~Fo|v8-2MLF2*GYBAs?--#e5T4hkpYD?
zP<;exDmIHb(4-n#7^nQ!4Aefqr+YukZ9a3Druv+DB`fGFOT{XN&hg$E$9d0=hW7f3
zovgSg0e#TJ@rR^*HT@^ji=>YeTec}rdlOP}tx5PmM#C_|@}N3l*v~^CkOFZQL^{`g
zzx$AQXn^Tx{L!b`A3w8x=FI;<8)y%c|Djr~)tbrt4;zig{10E@lg$5cwEwTev$^s=
z{Ly`R{LKEDbN+|jqLcq`Q|JH2`g$V&OKqdR@i_m#!Y4WZ&kz2L$zNj0OPoOLG=)2{
zL?LL1i+7;($Wp(Uy*obKJGp#&j_NQTr!^{KCgxJbsqpSu#lhl7Cr1}Yd&furIXo+W
ziHsxS7{7r9e=Ql|O0gUxRZ&6(-q=w?W||SJb5x1jyB%Yovq)azR>eE{G_`&(RNH$D
z0}O7ikG?w!UG%youKO+Mn9Nz-l431620xq~?x|rYqi;bLZT)m~(x{iQI^8kKzZVsR
z!=K-to?R?{ix;l;7{Qjrkniy@F{GVA@~Vp)S~Izz;<#5XB~P)PFp2mc#dT!*;Qp|H
z?7uTIYIcXb?}sqWxX&rz^kEteRR(oh?I^DH-<8Dim{={2`jiN$trwZO&$$If-UCj;
z0yJWRq=`Y7Qwa=GKZMy%iLRZ}a*tD}mIVW`uS)d?`j$azL|37&%44T`@J2mA&$F^>
z!fra_EogTlIX?JT8QxM1pUC?Nk9h--EgQ;th(+m9Yhos_-R9`v*iN;Mo{iNN;g;Kr
zQiz`7efLp$L45FX_u$)a%}sIWz9x*iNtbuq1(ss(o}0icJl$wGzLz`lVk@3Ec+=5R
zb+#An(+*(I-uS~wNEdkWu?BL?PC8K9Nbg#4Zas&#x@B@0MhwqFBwJBuSB7+hvW6(m
zO56@7eL5MzDObnd3T^XX5>k-|2(<}^Pu}qT=G`Gz6{Aq0-S;kyiX*mG;SsloD7W2?
zkt`^KEMpIrG#8euNDeeNvKPb=hp+nB&p@P3?Qzx^i)NyOig3`2Gl~(&r5A@1P%lK>
zU1nE4hbv^tPGmw`KC{O_Oc!&qqhGqJMW>!#IG^<7X1AUUI_65L;l4aEu@FTCQrifQ
z1LVS1gzIgqeUAs4Mk-823oPYZq^_kbN!WHxA<P)_vqDWtw8~mY^V(&sJwm&{E3+MV
zviJ6Au@&{hmK?0o4@W2G7kelBhYs*+7#(25&%sZpX9rHNdwb@&l)3Zx?T@EN2Tp%<
zgA6jh@dUzBNU5SIqXYtK)jQt%&%ayn<zIrQp$WN&?p<7f4uE3}(wSC3I^-e+zTgER
z+!M7A4rT3(RWysC<fBiEasRJDzc=Umtsf817Qb~I`J{F9^X57}a4@Iy)xf1Y*wM8`
zspiHNXX)9|$^P->!Qpm`$c)+RmdZt|m;=MLXcJ(D+B3D6(i{=`TU9soDCcx(X?6eX
z;{W&a@xPZUczt~Zsa%%hq6Yf>aR1`y^u#D#)?5HXeO5yB%8SU13nk`8XPNWSnYfhG
zn34`a6k$x@AB&KRe@Lq<eN$Ag!{c{}vKPna^yXJW=bM<~S5`@_U~=NZ-0EO9aS&de
z$v8wpsu$PDry!m1FYD{{xQ)AU6lX=jN+`xgf~i3)d;K0AhU5qMvn1qHtTrFZp)Htw
z%4dFi?U9T&4g*r6MYoS%?;SoIMZ1|1p9;itMXWz0w=s<<_$Sx_9a@z<2E^|Pv(P*O
zBo6?syb$Q@?otQSB1Od}Wp=Y#b~8wILlNJ0PCHT9L5J|rC(pQp#_2J7uMY^(PmLiV
z$i1|Vl-YfZ!I-Tlu{Fq4B58<$-+Zx?J{up+JdW}h)x&j8sp$`A|Ig?DnM8iSxC3ZP
z{@-Q;zQ_H))kpvDSNVVz3>xSGiD4)7@rFa^#v@r9qJGYn9V+Vsw=*7!6tlNOROneM
zd-AFX1Z33^cMOE`9suC;Wsxzwyt;^zLmE!Pegq2*O=oRB#ZOhzngYejqR_94)2<=F
zOME0klx&<U$?FDGOP{pSrmW*?c~N0#sn7<NDM*%$cRAQy1_Y1J<#dK!w2TNmV>u1U
zg^8}NisT_rLyLOMgLU8ptT6-X=TKLv2bUK|$A2F8<12Y($Y?x_hSB{9co}^-!7y0y
z13pN{=7GFiB(%JhxbiIAj-vra>0EiG*zGdb%`zC+C&_VHKX1Lfe06^GpTmVxsdkBx
z0Hx#CEqrkLs&#a7@n-Mm)`j!bX_P5iq%2!RSPRROdPpay2JRg!VRuHX6$XOH+dk3L
z*P-761CftX+^-q-y`VKRusSfYszI{`nlS{rfPmRPHGnASwzAtfE#A&B-cjnA+mBF*
z8Wd^lXS1<-Zew>-8k>!(U%-<L94plc!Ej}C<4i_oK5f#53iBlDJ*mFj%k#t5`M=Ni
z_b}!Wq_Qjk&sbKJRiN4>ED_;-h~`A8UUtT=P^8_!VW|QSLA{1qHVi`_#UZR0!*Lf2
z4IP&dMmT(U7K`|OKf1@YW{|koMkGY<`|XbODfyuelp{b;?A6%wUY;N9ID;Xc{)aJx
z>cJJ_8F_<HI-9QD8{sGe2>_T4;Ky)sb**nZk{v5MtayeF+IN6H>c|x!i*U+fqockX
zF{^|_STRSG%!Pbx0B91RZHH$}z{ciFi#+HJI-9WKgeikENKpkX(-Fv0zBK999S6`*
zD1-S#CMiNkK;2d7s$r@)=NAWuXJ@TfN5_XJr+^wBjUEG+mO|V|_?DyS1Tc|mU&G@8
zoCbn)R?x&_etedXrDbt#lIGj=>$yUnHK=C%#QPMEJ04DgZj$RsNg*2Xan`^C(gNwR
zZlbjk-a~{o9ggg9!e?f2MP?nv^f_^NlT(CmT8RTz@#3%~CR=3%KP>A=MPpX#m!D*x
zuaKva4{wku9G6xhnAAZR&*%cdz43TBZjDP~Hnm=#T(-_nFVFT59r9ZC#7)7&MLMZV
zZ~%<h^Wn|{(=N6rJpxq{cbrK{2G5lsh3_!FQal*$vV~W`;BNGW`>|Jgb$a$@@1phQ
z@cex5^<nGP+36dU8yAOf<motSp(+lFmyX|_U!0YxsS-IW%G{JbdLRwFDs!hHoLa5d
z$44*sTP;f>8FYYw5M9BkpeaTuP@AG(!oAtX5rjwbs-AJs8Qqt-pjmbSzW%~LS8BMI
zL3dDD?y;(#OY%`30@k!qi;iSNs2L3>@W^7tG5V$bPUHlXnv@^vrvQ$g<EWw<z~|=F
zdIng5xbm-jD;q+4N9>>l-W9skjpjxbXE~tDUopp9z!|29YUpJYLBsTYcbBgSr5`Vx
z#yZoJ!eB=f1?WXBAVDxJ9Uh(hxX1a%I9K<jP<BM<V#Yy|wm8ouTsG&oNT*xOT~D$X
zG|tGQv+v}5#}rMhi~zcBaL-N*=%KM!CzMq3YVzm{Pwn!EfAig0zeeA0JK}4gzuxGt
zJ^gj8zB+v6)=1_Vmf2bO?FzqTgw)<0>Z8ytDX7HO6xVoa!%Ht}_-{=*Vw9tQr+hs`
zz}G+_es0sqtg)>YcrXM!lsUYbiByXyoiyE_A-#!l+zxJ}fIu=*o{3m=wf>_A%>(?U
z(+%2Z)(r755G@Lm0dQx@TfJJXI7^OGU2<;3t$pxo<!-4gmqEp4$ePO|WXA8S-`it4
z`la^H9M!5CsG9m#SKk`yTT^{oqXFRW>+0zS4-1PD${9iXvdFEQJJy>`ky|^RA%0d6
z?--&+yl!7;Hv{In(wcf~p)S5{s&89nr9JidrTVt7z8$D<hi0W$<_}kuX{&D?^{uPE
zd1j@axU4QtQzB|@eaC)TTNiIf7+qZFx5-=WVvj#rLtqw?<HNFvQ!!>AXN)097*?fM
zc^KSyW8pv5;Dp#gMtwnyiYpoXc)p9|r^}aVB47r`k+Z_A{M>1mdHKX+8lvi+S@l15
z)mjw{9j?EZRo^x1f6T1!iu&S)Mzx%bxKDH&1k-DImsS({4uc_=8X^)tdrox(Hv@%E
zfj&Ku;_?<^W05%9J9&LrHbx@0N@R|S7(g*JAYjXCq0y#YxwKMJ)@vDYQl~;ld;~g=
z*vSxuk?@z{%}=P_i|dv?e_4wko1-&lw2Ip&5fhk6)TFo#om-FP<_&F9NRloKXwZ9c
z1Fl%c#fgLTG9z2FQCDLa#xfX3^ZFp-mU8lRKL*ekRywA;#B(Pc8Y46>DIiJFQDGmo
zD$ZgNowMN6B`yoH=2Ep*Z#37|H#WDHez|kLbM!T=wJN7FxpSWFT927jV$$Bx<wuFY
zjN1A(FD-`dBo;FSCMkfNh*R+ICL((;_YV$V$(tCr%~x)_1CnWpa*1U0h+BcaJaMXb
zJ=uj9FuR*m5dY*zsrQ+vzy9IqpZ_|3b8`Cjf6mS?E`R*#=YRhvr#n~I{`(L8K`<Qs
z20Fy#=JxLXBX?0Is3sK>L^<67L5k)sDwI0~s9=sLR!tpK)*{o0^9gk$!Ieo4#Pqp_
zaq@(&noXG&b9ly76dUQ_BF_gahb20lSEk?#JY}<jdK?Z8&OU%%tBRY)*eI92CsX>7
zVz!G)r!j5tp`JEEA*Ib<QA;n6sk)}2<(dj=$RksdJ>%7MnUBL=1!=(sPBL~y8&GKk
zp4TPF;TW29o7t3;z-WP_^gLRl%<kd$ag#6={dt{?)4Eezub9;``y{&ItO#{U^+YL9
z18;y<aA?4zng8<=&FH(Oic`-RD85@y7_m2DAU1<UAJ90pQuLW(r$=suigV+(C}F3|
z^FoyOW438;NAE4{y?uLla?si@byBuHFbb1mX7$Dj+W9OwpoZSU@L?#^cQIlOotFf0
z^O`d^xnw7Rs!$8t#o8#?C@~-A<pu)Nvu*W=cZkGTJYke+SQ!c<9@{eVDPthKP2Vnx
z_2cc{!NH-?oeFv*qw5M-)xD_oeiBB0?_T!tS5j(^;f*tLyIm1FsajSy!*pAsiv$aG
zmX#Xzt_oc%iGDjG^0`c9;4MNR=aBgvh#*K(6{93*{EdkpPEpyaNTU^_`g=XZ9H0B1
z<8$AmLdrGa$J3`}Sx7S!2squeNoXL&k8XeBX&T_u1b^(970&#t>vZB_r~dFhr7_Cu
z!;9A5+3VKH<(tE^qy5s2i#cj7#{X5XDfGf%dtMm+{{6Nw6K?2gb(rJs_^RZN$XTV)
z5sO2FePeKDQMBb3+qRt!I<{@wwmZ%jbZmE=bUL<e+cr8*$Hq5#Z)&D$>dl*}x^>UF
zf9~C9AKW@;ue}!f&pRV!XU>WZp1>G;h{Q}nJiyxvcY%~Yz3N9sf+}eP*e53u9Yg~^
zcLTX5WXF?Qcw$u<-Dnr{Pkj|@6+u<wC1Xy|sk7kG09qFLLILK99;<TURSPTd>ct<`
zz&D@&VwS3>k>g`adOS{k-9>O}lKRh)h5Ooa-~NeK$<-b6^LHFTyS5G8%tPHHBMAd<
z6C#2)cVwg8MaNZ3to(*78(Xr&0)bp0^>*BAa_tcE{=MgO`~!T~NnzcJNKr>|ZiI${
zpz1<IBf8;>u>Krh!F_EJc_VSkg!KjEJ))H3PFK>}XfmeUygd4m-X5(Q&ORNIlNA3-
zqpT99BTpd*q1_8klj-7A$noHw&AOcRvh`t~PbNG9s7^StPcHlqB2vhJHSP<?3;2?k
zA9~0w26jS->0(`WgRMC3nnh6MVd68r6^D4VV7mI#&e6P%iy^q@)=JF*Id-cpg{5i!
z9|Kd3xr4|-{<@Y>y+BipiuXU^!<D7vl6s5p!?(o+2zRm^!FbGN@a6Hv8_ILhOhGKH
z?3=xcxj=2#KDHEa!55II(--bfKkN{3-SuQHZQ$(5!Y;7Lcgp>v@4g4!dY0%EyxKMJ
z=q@!u^2PpPgbm_%#u&nHl!gXyjqG>~@vKQ${Ek&;2G&IL5HoyEJ^X9sqsWkMTu!@T
z1j|#P)1;R!d%&CFq??AmueC<O=hJD(dlr7;(^=Mb%R4Qq38^)&m9soD++!04KOxx0
zqn{Ihs6!9Nl<4JEck&Z19UCOP(w=^o&>;h-ntoVRBs}NZ&ev(K`S>SU&R>m9NyJdT
zCrM)hqneHchjrLxi5^pd!x5^k(RZR1N_LA_s*9_rOXJHB&)VN<6Q(U7nNpK!^B-2|
z$H}DQEEfN!BvTjm)zPA=M6wGg<M!HK>xpvxeuOg%l?-64nGF1W4fXXavFx46+j`0q
zD!K9;>%$JdIKJxP#^P<iN^3@f9eUMH7r=;yhU+wVBza#+>=Ay<l#^p1=KI#k)UMw?
z6I>UnS#fpZU?jv!@l$nF<i31LCRSN`Dg4sZ+Ee)gBFeMBKx%^av>doH2c(4$m8sBL
z4#xvJz?C?EW^W?x&aY<PwE$0La@r2iP^2Rqrm1kI4EduDF~TWm@4WerVh!1|w}V+j
zU4!C7C)SMhyqyi23E%vQ-@eIEkc?Q@Ji+;G(<qzDL85dEKvezHC{uCC{JTosukQ<s
zdu3x!>>Lxktu2fkWaPy}OCE6SL;k2-QQ`w3#=$V5=?n}G@)2nf7gGH(IW;?<6$g+k
zs4235=Wy^?Kdr0H7aj!vB46n2jf>303?7!(xARH3q|0o<*vMRiJQ;n%tr7U6Y=fnY
zzx#c6j>lI@B+*Kyt)2gyNYLyrOu)b}q}l;eW<{JC)Mgs1;Up6FkiswbYjQ9OG|0^d
zk;C1~EZmIZnZ@(mzo8ewm~!3XYf<LQLG9cfhp45SU6$=OQ<U_6KcsgiTnrPvQr#H^
zvDwH2cgRVjJ~i*v&3hwmoWXsX&o|qj;-b@^xQovBM$Z*qKi=viaE}AMA&bq9cy75o
z=QCnni=Ne|$3-z%1#Kv?XWM@pKB`kj*}8_Q;52BIsLcM!l&L3-T5<+0{QiZhg0f{3
zmv;m#amSW={}M6VTMb!*mw1az_QyJ39qRqqHL)(nOQwq>ASFWG?p_EnB2yiQ8b+=@
zA=L}CVFM4T`#F1jbjWNL%SM8yEBz0R1^V%q+rK-ZyBrWy(+Rv+tucBl-OG+`)0yu6
z`;}zZqu1vXgP*0KHJzFp-RaZ8_KjVFA56+8_zWAjocjemq2e%6yg4M(PAkttp)mn_
zVrK;_LL?PM*U#R+ZHF~hxVwQbIGQy^Uq8QY+k&6LsosDi1S5SK?EJ(62u1f>_DIX~
zsqG$QJtyRyMn%P^$0wAT898+`awh%vlr`i?I$qW9W@70yFyS?EipVHY&FofuL%OH2
z0w~P?b}?a~HoQE3czO9jUUm-hv)^EUT2lwSPkv-5eeS%f0NARH;Unm7fewWklb{Ym
zC&6)0&9zK+Z+Z*fqz7ys3J#mwbrsyL)FESZO>+=x>#u8tH2e-QYJ)x@d)iT)QXg`}
z_le&!ojTq@s9&O_?`y2Y6o~wR0N~^ZEP#!+pM*^(_3Tp!h53&t{YJ4oOJjZ6XvrZ1
z(%v7;#-ytW&H3>|UlXEI9AiXW_B+g_q<hM@K0o`-Y;r^>28RJw>T9yI+Ipc!=kwu9
zC+4ABl&x}?o612k*41fGlhY0vZ#daI<5?VgVez2ibP$KO>a~V|+jVWTJY`UZQ+Ka$
z5t7<%hKL=;^uI;27IZ5GvD|8+@!oB(^zBj=(ww;%@FIT6@xR%3cJhBu0F@W4htg9s
zgrhvxiL#L?;YY@q`QIfMAh{Nt%g-ynx1;&yZTnOzzs?TB`#)a)X1kg|xv`q1N#@04
zo+OJ8*(%Z~M!m7Z|E%VUs=ll)>XLI>qfHPzlng364{M6SKevm<ewyvMiE&l}msbr4
zgBrp_z<~L?<C(%6wfqaG<J#G`tUv!R9q=`%TND2CN17-vAE3y7vU@sMJTlty?+y<%
zw}+P+apwD1^cYozx|N78zTvm#QCBB=#46li-+3XOgMpRi_E=Y+R{t)6sbx1nTa(Li
zqt15UsRGV}z~w^H0*cn#PEjQAh9DFjb?Ji(y7#Bp;nd%uy_v?D?R8+8O!lu#zRXW|
zvWlQ|t3LyOc>>$8mgEGcLwS=+Kr>b#|3QV!@Nv+Jb@h_!El6M+bj=}{cLjQzIUO?r
z+aal!kI+d#-n^+I;VxQmJ3g-8`|zcnM}1A46p$XH4Er?}qROT+??92U%l6sebBTCz
zAy2?X7dBnHGqeR~j{(+56F{>462#X%q7ZXKDa>dr7<6f7$;&SQ@?PRuc}vtLkHMbm
z#QZ)S>r7y(%P<_t#mmBlp&CZQ>w1UEhkQrO<vqH@iS7M^ReUobc;x&iN_C_BbdxGC
zW7F91!vwslO$Ne|dI}Aqd#Yj`y^V&GrF~Un%=Sq3?HF=wmyN@y|A@HW{_@;px%T3{
z#H`z%lrp`FP1~hp&bg*YzpQ&&lct67U_SgDDYS6PGKMSvptM{zLrRUd0E-LGyKOy;
zq7DBh-<|c{Y@HqclWxhim8&wQ)qp5TSTd{S;nkaA4eXPjYo%wc>o7IJbhXID<&Tld
z#waJg`&C$25#?*hl)egeZu;BRSmU-wKC&78oQ}ZyXYdbJmDYclACW?X)_)dp?xnU^
zjPIxImwPWEV2u)?!29Mc-J@N*LaMc+Hm@OWa^`@qSZ1U3V2Q+ZyfIzKZNc)!rqAjl
zb(TY<Ic9fO&_oA*@P8Wl)N~(%$9fA(UtBHEZ>EQ&m1WMOhFrA<OthTs<0ASG1oOL8
zk=8wAQ#dUlbWrGaF;NTcfnKSq->J`KaWQ9aq+w?4e#yC%pLDurW~i<!9^C!qMk?jz
z3`tKYVK+~E0G|}!IC~`<SH*=RC#~F2vlC6#DVJSik~>DzEG|AUsJPRV(@591V&8z`
z$ivKG_i<_Z(ihejC$LCeiY2jbO%7y=Ev`0E9p4kD#6oGhWlHGFl6j|f+ZD2yr2gfx
zO+;_$0@hb_a{=K;AzhHEYojI=t>Cs)&}@OgS~;kMRxc6%f{knt0$~+ndj?&A%lMw9
zg@QqJymEMr9U~LH7y9(aiyi!@$y!93s7533s+3hU=NihAQ`f+#2r-%5hBB;68wa4=
z{+<67C)nY%6^%Z{&rZQzV9!3eCHD<XOy?`vgJVEcJceu{J#)212sd!Th&;r8J=Tu(
z&lFh=_XOwsO~<M@m4%eH#jO^6h2n&|(`QOXt~Z(vsrijBe#N@%8?C5kYzA%S->TO2
zZ%y6a6#BYb)A#M9dBp>583-`+;j*SaL+_3V96;Hn(^5GoE*x+AAQMYX$D>ZBHJ>YX
z#%N)zVVN-bXamnt4S6w2IU7YpbP5W0;pq(md{N@(0xG6LatWrO5>6f0EOx!fuqw{v
zDgwJPufmP*$BL7jw2KoJ1sh7?#f#l|+H1OUcrltPL*LJKNqBa>G-Tyfpxwplf*iip
zKq|b;GY%mGRhIVQt0;PG^;7YOyyityXOw;PWq=AL(aY1lf%Yf^i?8_$M+y*g6N@i@
z7B_#lX3{&&@#;1yNF<kLmqQ)Yup|#xBI8K(&tBB0ro$LbDFeNW(#nPod7HQoc&YfG
zqsi}q7d*eduNMjLHHV*$@zokeEuPQ!jMOcXS>`_lXQ6Ma2fi4>!5TqDOkhdqsD*hH
zWswm|o$Y5+{NTZ5TDa&lpw{-zDI`ATf67J)@?^O6e|l@jr{W*H<f~n~GEokPh$ox<
z8%DHV2QTHHcY~+6DnjSz)K0`p;ThwKz5qp0{ydFT<p$p$NMh=vo0FgGbQ_JO@dSrF
zGBf<jcYm;n!kgQwHgxrbGgK8med!$2!nns_Dx89BCh#kv0=`M5q6W7_Q8gJ%WFq^R
zn3^<q6!l;qyknqr@1|Ae8f+M;WeLCmV7-R8tpyhwZEQP}o>mw=NBRdn_I69_310`d
zjR#JzrlikfCl9dlcA@qkZr{@+>O#d?w*<?R+Kc3(G%j^rtHRQV#zr!7qEres?oaDt
z!Hf!*S48OyeRH5s?wQ=58K<(0B|#DcDOPYO(JDP{bghJAwK-dq?Eg`y?6=L3H8*mf
zw0JC&kabd0sP&<7!oZL+iQzi^nIrX*g|RUN<#u5)0Pn*iIg_%2XD^tw{gV1ZwS0{z
z&z7~Hb&__*UDa?HXrMpaliwf_?j&_Yr|!Y=6uD4My=kyE=k4w0Ql7^2)Wlr5#DJ;7
zj;j$KUD%12=9)<JZ!GTgB|5L*c*MhR-ua7s)?9UKaB0!f`@qK*EN3Qac#s`E{rmN)
zsAl!7Qs())yDJVm=R9Jpxu=3zp>}vIzGQk6;O}s(YWAUrNne0DG30wnxzCitk+BdY
zTm>IquT`y23To<?!VjH^jpg!>7{uHMnQ>M7HQ8vOtlk*uY7g>^qV93J`_aHf==?QK
ziM9$f<8)`ra6j9c27nbwQe=a^@DcM(mO1-u*tk1iGE~BaP;vP;%M><SY^eXD8izSw
z%y2t=R1}R|6e{s#`hMM<@do_$R>vK0a>T<sXX%b#D&!KyLk@mnM?U{c>p?$kzjl5h
z@s-#9HTW~xqUC|C7ynNNM@Ug_ZL9~ke?4L=KYVUg#o2jQ1u3}$mHw=!_;?WSnH|cM
zfG|5aXNo00{Uxg%c(?EniE3bwZWH$v*@H5la2w`mI-OWTY^0^tJiJ58R;MxHJ_Z@g
z@)iQnHXs$sia14I430V{>pCSyw&Ug&2Vphwq0Qr(!_9mz`$q9JW4q7!M7C|PJFIP$
z!eVygM5+vB>Zu%<m31ZCW5=>NjkkiLdinQV@bU@`Mxj$xs0Wp1a=t~$_CcbXM`i|U
zS4b<$?deIMlMs#Xc7^W9n6UG<(L?s!B3}$5Tzl&L1X96!REp-@i*jieWZjFl=8uy<
z3#bUwp47)YXV6kNa_h*y%VQR%?{kkOJ=oHzEHBPc>fh-|LN!Re%KWa~z6$JDs(%vQ
z7stwOIJc(?SJZ~Iy&k+$|GlBx!vp!4CI0u9DTUoI@zo&D%*kT;GO4-42jzCIj#8OG
zeWp)>R`eEKlGcrsp^fgQN6GG2sXc6=^PH)r%Dfh}d^UFPBU-M_@uFjB&?Q<*SCf0V
z#P+l9AJOM{MG78qzQ^Cm_BAl@A$m6|4NfXrV|wP|FgWW7LNH~3><{HYuoyFl?*(`a
zDQUiEiF6NU=jn~s-Vitg(SLbsQ5j_9awKi~<Z~6mlKhw**?t#Yu(PZTq`-oCh!s!B
zfNu%bc={n|M+7J>Zw+p@w?ZV}n(n%Qqd@_zmG3I@=i2OPcKaiY(_cw{os6N21Z!aJ
z;J-akc@DnfGEPq>oBCrX5z@k#hpzdAUQqp`!hjNi!(uJ+8_+={`|+{;TF`PvXfyN8
z1{%^5FfJdFNx|#<eLdMhLv@3Ug+wA95r&O7Dcrp(M}~#?k93c*b1Bt#5AeXV9~sRb
zzAiF0<GmT}6*l4pFw`8Yf9UIzA{K~?4Z`90%an(_e{+}FTjB7A)I~KS(_i5n3ts2!
z=QqVm64(FPh~}n0SVn5&RbO5UFx`|Dgl+#2Z*uU0Jnds%@=OCVep@c8Od1U&k7%<Z
zmw&c6541K<K<CCsf$<W{?NS}CImD8K$C}Fi^^D_!p|vF)Y*3DhSMN2P`)cz~)VaYm
zH_A92UTju`a}*lw7tExwKUe5n9a7zK$QzCJ4pt>i%^ud=#&<kxEzSU&mJrV!Qx6P@
zxI#Sh-@Ov-S8R}T6~~)p+Q5Fho^Vf?V2k7ga^3=s*UC6$-oP6kLNAm0EvwgOwAe(n
zO7riCJdvs)1*SzYB1^ZJuM{!rZrmCtvACGqi3biuVW=d2BI=t6ZYP9_C+@?JURCyX
zLcfnm?pybN=%L4Pt{v*aioIdP>u2ZX#&%&1#RZsC;oX9-H{BLqxi~v+`Kn|cyK`QR
ztHfC|C#4}PAVZV+8JIfN@!gC1Khp|A**osNnkhMRXIyiZ*@S$z`w>Ch817X&Z3I=R
z_UdBELNhzq<?3A*Vqu03NB*3ct&xw`JHOC<6P}D}wme@}_?%cB(y(8DalT)VyIeg`
z2C3%FSwtG8@m;(rwrB~F_XkZ4M8fl`rUK&)iA2X76Z@;gO?I|T0Rnp?76W@iA@L1o
z>~{;Pw8B*j6LC-964zU2eK2CU?)aY&(zO-hH@+Ta%|$cj?y35_mVK!>>-Ar(UaZ$u
zY`Cf4Wg{fqP6Lg4$RIyM=oS26CT6>s>i-7ECvO&xt$PZ+_6lA49tQk!-A}rwaAo`%
zT}B>~LJ{YkVm#~CjY)2i7L#JF-vg=UHuaJYs-KKw+f0vxQw+r}4I*l(d6w2+nSQ&8
zT65__!HZ_ceU^_)%Dk&|Sey7kj)}=9uE!>7Zg@~mef1|by<VxrhT76tD)QsqTu8*G
zmJHrA(Nn5iFQ01RTX@0k>Zr21cVlFfFRn0P;<E8GY7~M0^XFc9Qh`~pDObVOAi~Le
z#50Z>IesF$PE|C-AFE>B`HY`)&ZJcPhJ#6&*qExv&b;#(OlmV4pG>!AcrrKka|x(S
z{X(Nr+Laz?LrL~xwq~F#H^QlaPqy3slQnsVUGLG@*rv*Np?iTJpADLJ%b@1R&{uo^
z&-gr0w%hwq*_^2#%gFP}%JK^F2#CQ?(Cr-oE`(>er*7F^Fb<^k%s=>?*W=?>xBo~1
zeBf=0eZK=WNj7ABA%~VLwMb^ZILzYt`9gCU8fSXHKRI1rBb*2#BUh^FIaG4iDxdO3
zNp8&KZf$Q@BX6Gga9s38yd9W$U7~EJj+Jld_zU?wf{-+KFB`aU261}n46Ym>c=7N1
zC;<euZVI`)-y2?YqqFjU>4bWS#z-jeyzV^l8KU}ZC?hcNPJcSP3l)ojyejO7Rx7|k
z_S{arKDmfFZY%L&SjS&E2+UeO2mJ$J0b*Dnm`E0}c0|8*j_7|{IRf`6WQ((jQny+f
z$ySCq?h|z=nn=8@=Ef}4u}9`G&*FdZdd(+ILO&q$D-e`~2JjF&0%okT%Jy5$P-<dH
z@?q*2+)6!D(M)Fnm3!6Fv4B)aM&^0;z8P0Uh2KTJHEb7AsHQ9=p*=w#nAfriYGSUl
z2BJYDf70ag>LVp)V+$gDDgAL`qHk=8WoH{jXd=<?yZBDM@EI_0LU%8;GmP1TUERJE
zN99?V=2Jsn5r?DAT7x1@m|8!Sk)q*bUjiq)b+!a={Lr@cRa`~KmZQLswg<8|%Sn+l
z*DwuRkOdAk!_4kuw<~(VZhw31m4#<h8%)Vh(l+ucv2{$a@$EAbb)8`ID6^Ehul&gG
zj`y30bHISTvBiz*zRijziJQ*3ThPK0y38Tsz5cjdJB^ou35S`U01ixuS%K~^duHMc
zLC$;c&OUz5UO)a_<Pi2QBu6(Crl;eSSREr;+guTrGtX^wB?I}wg?a=A9KsgZj!&jG
z4k!X)6|ygL^a+V_wXzG7KMp2Bz7W7zF!|gxB{+yj&nHuJP$^y9Rmyi`?bD|W$5VT2
zpD5RESAXYj=7xI8=Crck=Dj_9hVvh@TYRl>q~6j;e66N8CK_5wj>C~3Z+B1pKweiT
zQ+s{%yI>AqA1BV$=ROv?2vV{4$Rx}8ba(X%6JYJQ?Bs{Ee=;N{F-=}P4k>IH_&~VG
z{%l4WgcOfwKs<ZD-1|g(2Y)3LU3r}TG(ZcSK>;13%l6A18(&Rr+nv~y!KNDbJ~GcB
z?--*5vLB~2&oIX^F)<_Otdx|%umg_6t%Qg#Npi2aM@o*xdnfM1f@b@+hlmRR&VfIb
zp}TuJ`Gv?$iND;Cn`6-3#my@~2LbaAnF4PAl>Z5)@^KI8%jA7}eSfh7OMEIyq2?0s
zx`&#ZM^+H&<13x!mcRcQs)L9R^*id{X#gKhcujn~;lW=i?L$nFFPvO}c32QU?+Ccm
zPrx+{yPphZFBs(e;A~gDPN%5ZC1IEABXR(LzhMM^FVV47a85+bn^1(yt{C@BS~?Nr
z3yQ7{Qq>rm+bPK<E3`s3Nf%Gx+Jzdi-!T?%#!g`Nb#Sj6ljGVzP}4+8HuXI<3rNJh
zuMwF#gfk+WfqB;p4sIo<7m`G8qEJm1L)@am%kN{dQgj9Egu^>;`EV`$AUOI4%IetV
z3$(#KQi-1+<idXT#)$syMpBjMo_{y>L+lqt#hk%l_-}J23Q9}6t1oHhhY;B#OTmG}
z1>2;}-2sm1AE|3dJdeB>$US`)4)dNav1=hWA9Lr*T1LcOKafpyuX0c&LA?{?FmVW=
ziX|?sb+5HxzPQ8hB5(ALc?akwV<a6<l2woTO}uVPXVcnp_SyZx-!sJ@x5#}*SjMLH
zq-aJ+39y~`?=SVV-1x35i)eLZ8TK?!&i;gkHl|;8rss5~8*CD?GSSE|U_9DQSBuHJ
z1n6&MX!=yj<t?f2j-8I(^sHp5otY{(R=yv-8=X&e4%7OgRZmvc*4$LIY!l~p(doNY
zx-4<<)LT%iA4|9!vTF<?h6b0oVAAem9aryfYGuc7juUNLLC1uV$~g|n95mGTHjtw+
zxor`Yp5Lzcf>9HuuVWE9svH)=btfYPF!seuZ0W%@7@h4)n7aJ(9B?_MxG%LY1l$(6
z8x_>CT&G|KLVBJ$|63s0i5M}Z2V%}4(}|)Ego=Gb2G=8wXviK$Z;}q8`9cq84L3U*
zm-Vv#s|_|$2I7Rs%9=O{XJGKn5h<$9N4g{%)3{9rRUY8KYwhgODX^iH*K`VN_hfT3
zD1L0&((>vkbyk+8-<Xj<V=upC3X*)s{<EMeH*gE4I23rvr&Pu%V2MZvm!SdHkajSK
zFo@?{D-0Wlm#2i!5Hx#i0=~ubLTF!n2lb2Y0E-A#HGz2&E6KJsNcZP6*wwl*AK_cx
zZ&(xUV6cc_vCt?;yI;Bsoh&LU!Z=Zw6~t;fSaaVqu!`v#gM>~rLbagyCCpr1hP**N
zVTomVMwyWk6l;k57}zt;6}aq8UkNz<tZ^pPOhS_q%H))+-rA6G8BhGqS73#_uh;-u
z2~NH%D9udiD?G-Q8{&in1LevDQlXI7BqH`%-`B<j^~Clz<@DVWikcytrpN7W0YUqw
zaald9)ZLP<ma!r1)bRSd^L^1W9hYVLX`i9BQf`fQv!*X?#cQa~#cO8FQjy51sD>e&
zmt|Xv0thOCSASyXyg$uJ^y~wAo|wEV#t#X7vn1QRvAkons9IkU3F=TvT^w57vDUhx
zbH-bYEb|*n8JX9SghY79FjR4tBJ%m_gv=6m{`>}1KbISehRPu)-3B!!H+L4z-to*k
z^)%1VHz~f8p6lbjM_`0^{~}R7-J1}=U!V@uP}ls}3cAA*jDHjQub0se(8#(YV0hiv
zjd;gdS!rzdMN*gF$(TOnVQgrHCjG4n+q!Zm>RxCXJq<Pd<WCM{Hr;*=(Mxl4)6(j!
zCldfL$BmWWvM#I>3Cw?==(;QP4`;#`=m!hUIHw$stYT`i^6=uqhyDc-<i-+6i2q{=
zj%<=OgltmWE()GR{xwnPEV;K5Bs6GWB$>n!S-Gh}At7<;!?WT=<&p3&kv#_AMi!0W
zpx-6<E1|1M-kRO~_ZUX_aIbe*YTrMo4%7_+n?(!E_QqfHTlu(Df{)dktkwTwa@JXb
zWc$a}kXafmtbe<M4`u!ydpHVV$77?ZqAq~5tv583KY$oRB=3M?)Nv>LF`C_V@{2ym
zr4Xi?{nvaPVrbr8gO395u`NDIMvVC7a}LV1`A^R;K@T`KlV7rnV<ajSD>)LIJ8G#^
zNq>;{Ojt^;9<JTN^Lg>)ZYQp9ln3~Ppq;Zi^|S%_lF4%al4doD+N|&n_;{%Tm)3wf
zyh4&I)0TLRg|t6)@x$BIvewBo<Kgd55Pj4c5HBb0PbSaih`USJg~{<Gjh#weZ_{Q?
zIW3DLQYGgnXY+z0i?go-0HMR|OD+rsPajh+J+_Ju)f)rxD;?}ci#HN2#~$khkI~Sb
z)=xf6%XNnAFHix?%pOj-OWX$&v1(=vseg?}2&BeIQJK&}@=RjepFJH4d}uDl*DWp#
z40<YtcQ{<PIra>!>=r7PR<<T2>IkNPdw8zs)Hv>*<uk!VZM<CPGC|zFN*3w*n=!Ot
zV@qpRtZ>=2E!7Iw>o#;Rk5+|2sU}n|*}XiJipxsq61g$VpJj?Su5FIjNK++Yajl$B
z`KgzQ!+%j{Ht|@{mP(HsFD_T;sfC3^MU<c5E0k01GtsKiwYj_n;HaZ`O>mv=$u;%a
z3pEfr?=$nBeZzvYxFxw1csU0WuYzcf;<hBaK{f5&b!pe2DlXy=pwZvE6VMm`uD~nk
z>nZ-TM+&r42)e3x14$dndV-H@r8$|I_%GZmFfL{mXBiimWWFE}6!$rFim~BV3ZZ;k
zi8b8d4B3-lo)VPQtKT?(T!&-BHMH*?aO_x#CCJD#YxT)$I<gGgTd(@TxtY7ss{e;l
zUc7byA%&4WK8`sqfPWK*QVEWr&m2%A+-ri4i*{=^;t8-&gMgO3F5cG$rKpwXS!fCk
z$Q}vNEd>?hExGK{v&-H)g?u^`hiE&gPRd~uhV{;7B7Uo)A?N18!h~8*!B6#3cn21$
zR`sUKR%zO3F4Lc<uIpMf)N7P!xCvYB(gm^Q$IAN4y^{p4<VQ*tEsH-dnx5gx4`qXC
z!HEp(h-_6ZHXnlS@{h@#4z&BaX-l)pHGV2-)|$#Dd6XQRoUWo!djgrQ>QAm@9N$x>
zV|!@un2RXCZ(p7Ary2Tm$^2FGqC-Mq-9_lZG0h-^U9}gPmM#h(ETn}v1c<0d#Wf&)
zi;^LMdpZH6HHK1sFP3?9s&T4b%w&j7jZGbH#$0Al<*t!7=_C)mQVpvRXQVv+ObTxX
zjSb+~nU#O+$Y+C$oYd{&AOB-hvJUzx3qL^CAff+NeYgb9rhlIV2AJuA2!>)o923tq
zL$SbFDwU!O>KLHkcI3vhb$myc;lMdZ(lF3B)?*=@{qyce)%n?0;SI?8ea7b9s!*@Y
zp;YpS^FzTbA>T>6IREFT^)rYp3|QBfXZeVOf1nc+knp@gV1HXvwb)z~VVFqnEfKYg
zMd!+<5TMH+B`(e1_`%#STgh2`1>(YZ1U9sNT7T`4X8uoG|I=)volhEnx&a8%oi)Kg
z-6~sOux3VP;i+5jMLV@kz_m>XXGRQ|b&c=k_Ofn2l5mz%S8kRwL||BZoe2-J;11HP
zP0bQp$O>0ym9ACc+X?jZz!tzBYKb7VlW56%3s4-{q++v(PK~|h7jGEY6N8hF+9H1g
zj%gOv^BW`Z=O-uZOX~WvtV5BZm{MTEJcBBIjWk^+d68o=jW8p*PIus!?_XeJwuar`
zge*Y7!k7O?JEXqa@s3ktx(w`c)B|PlECU^)9`XWE1IW%v-$j>2o}ms%-?iYuOF4M;
zwqGsp=3aq~pCnZvsdEr8@*(~^kih(1)&V01gpUt0z`ADsc^ie<&!Bo@i4W@hc{l{d
zU;DxACZ_Em-vUZL24&gBKlS*>{$G8%^=ai#w*{8JJ@g&0XjTHSO@rwbgfbq6|Kkl1
z4%zGlp%ViNbc=le!XKN`J^-ICCSqxa5^sP{)*5l()HkPwc9kvJI{^FomXB{&MquQn
z2H*AXBLN@4rx!}^0c)TX&VSrtZoz6-cc5A0AAx326kyFo*=f{0wI3qLHPUTae<2el
zMzy}OYRY)c9ORooYSW+<F8u$`xUB>RVC;M_6Tb>kfdOoj1!f776G5f}ba*d4K^w|8
zX~3?QIh&RqUGPB5stval5)8mQ93yC?3nVoFa=8MTTm+y#Z#40Sf7HNbkx<=7rHnGD
zg4ak#X4%{=?BBp-gO2&QOdY=Hi-m?k!2iy5U){L%iQ{DY*wgi3RCBZnB6xWOVLQ-1
zBz+ZhS$1PNAn3pOaa%aEDNj)LjcT5sTiH3lmX_t!a#L*3pN}2rqS~z!hvGHRd5&&X
zTid7CEokl&*a-UKNCr_o3c<f$t`Ez0!9RkaS3#<xU*>g^Q(Z@;wCX$xF%tuSe#~&`
ztUF9%fk3}Npb9_YOOVvt|9-)LRza_iLT7S#$bg&iM*utikH;R42?%2tkdqE^;QbqD
z|B*lE^Qx(9kON5gwgX+YASVe33|p75cp#$yY=&*vSc(I@aulmncAQBB0Kit5-5P>7
zkQZnNbn^@ny7~Z8zO{%f*Hl)qYc?N#0Igr;_P&;*n<&igjA?+Obhqp~$cMw^17{7S
zv<`ZB{hHJqg8yY9`kE&mZzT6c?&o<%tIa^hXQ7$<8jf2BAR?WE{hy0ZGcTLMmN^AU
z&=V&o=XxEELOToURZ%st=>*hNHT7WVae1=|D!c?)zX`2>g3j%M3GW*ZsrdMxP0-q(
zs_He;bDiW8Up95=qzQc^e^CLT*n^y^SF$1@01Ho`<R=gZ=h=Td+LkVX^Xc;ffLjfF
z5YT_?7Q9kVF%W43&KL$j$qYJgzVWk}hwG(LA{YY%8hV-y$SHw(Lb19tF?u(2E1Z6C
ztq|vY`3)n#TT=z(bW-DT?r6x>D>-B@{2@df@Zkl&%X|P%JIbP#4S;&tXZAu3gamQI
zzNUKc!_<c20N`c1$f@q4QU)1RZ!KQ{gWqq5u0RN4HMyIwB}&~NMindg1TJsDsyRP?
z(0}7voj1_hZG8wSNQ9xvtW@E?QQ&co$NByUP+X593P*j;fJ3=SfQoJi(A^*U220?i
z-=Arbk~@u-a>jRj9PYtN1lm(HW{R&$ouw-p%dXep?^d_eEYJmm&&U&24p-g`5ov7&
z5DP-)Zt<DS(ci<V^l8^T4f$M7HW}mSS2HW8__Rz<+(8`wN`--8ag{3Uo)KL@55%Im
zuf=%_X9?9T?1%`Pcw8ZkJ&41<L4?t?Q%TMji-u#e3Dg9f6qu9v=8WM&(Ny^)cgsZ&
z30ZGUy6gd)UXNrq;t9Rb$C|)nJe)2biDp%TcK>-@Y!fuCf9+H!H7fYDs)7gtl$Wky
zx*$Hxz9SQAyGUYCKM0}h1a9(gIb9MBF!FyTG!uqgJL^_~sXylQ<rj@=C84BmMbX!8
zR&z%qTnC-f=Xu<pSARxHr{wI6{Yjp2q@OkzYOCQU@jT)Aa67)>{bt=N<la#=8y-y6
zGI@11RN=h&q_;ilv?vz=;B{;5m1+R2T<*TLdjTnSp0EO$-jXg}JqPMaKhcyYeZ46R
zw=)u6E{Q~|N0h@OsSL$hBB}41{g@4-M4-?E#($xbqP!MZX7(Q8FPBOdaDh2M&k`V`
zf~R!`!}ZuUMDE6C73_Facjkv;=^Bru^>35fN=&=bv?muLk`e+(@ULBl@K$zvc%O2N
z&}-n&yc2P;FXvgZ$+_h>k9Ct|a)lQdm{H@66tl5PJ{>vksUpwRqE}+}X?NrV)yw{I
zci(OHJ)J;=87bwNUSSVk&u0=qvy+f%5Z*=JrJUU%hy)HnNyUM{J&4%lnIp3TIfqRb
z&s~UmOu~m_E@@&LgrD4Qe*Vk#qth?BqqSZ+z$!o;NK#3VQpix`*c2WU3IN!locvow
z7IxUto6R=<ZcRZs!U_oelv3&xDXaKXi$hHhf5Ji~Oe@t%teC?Yk;dcF>m17Y)uH9H
zh@zC0cp5I-LW<?y)L3W_4Dz&d)B4FaoP>GbASh>FUkCQ2eJYSTBY|4CbLpe>j3tFA
z(H~;-&jNRYzQV0>%Y2Oy5d#<yr`&lid{<yHYchSd-nXY+k_;Z%ZEHdan%=EK{<`w*
z^~^rDuNfmVB=6p|AYj;eo}k)0DE4(-cN4T@FL(s{pwE*6feJyOxi21j5bz!(^a>*R
z02wWSW^|J&a-bW0KnI4fd!(qvAm?|ECy39(fBxtbGAO`xMd)vxPqoIpdjhDwyZRli
z#u9TMqX=3xT13aCR^3v#|5Fe|`F@yfT(g!?Nc`t6**_}zeBr|FDD9v?lJluAxky-&
zUmWI>2WxZYT{2F4=R|%SPg(${y2a3qMG{gY0&F&6`6g@e&$1&m!>904n>``)Z(p$s
zyTZD~E2Xr`63_R0N1GdxTGZ2{bPp|?&~T`U4mUm((6-IuSdayI7&5!)7#)<@Y@)Lg
zH7<EdH%?eSPv)U4K;Ov{;!mFvI$FAbuCQ;Ty=ZLiBRKfOX(B*AhqWxiS#^Cdu?;%A
zy*4UXL*gu!psLuOdgx2SO<O-@0BjtRaTB@iK=c#il97pCSGL8V(D9|+b6$2gk74G=
zWtpizNs6LMd<6Y@6_*<6h^)Ctu_^lZHoUpHF(yd@Gl=A$S*$8cl4I8NMdgeqFae86
zxasO<q|og*s+YcdkV~|4cPP04BqLpa=GLfc4#%LHf{lA{1Ef_9G2xuQ49vDK28*a6
z73j-9pJ37QP~HPK9LUaub2p!aA#M!ol5Wq5bMukOHanuX_Wo>1q0Qs_I`$O%hGvQH
zQ{^j88!?$2IIYwuRgoNvcLbQ(en={QILDqZSnp68fhYcRhoIKlD&Yv*RV8SMLRIeg
z<!q(G?JYB$T(Z#fAHt{fhuOtX0Tuegv>w{cqMNYLbHWwbwSTk!{e=8fISKg1E!1Y9
zMUOBak1|9!F?SW1@LHr>)}<$6&Cr<l1HxD>`-5k*_PL|Y%`=2Lz@h22QfJ53vC3(&
zWm&W0FOgKcSBKz6@uhFcCrJ3`N<N2_{U3hH;<ZYfg9;z>2ysS@ZMqnBZ_V2!E3zFQ
z;+NX0a49xl-QxLbL7Ge1yV>eQg$~uSL5m44i{r8{DHR50AeioS*pKE2*&+*y)U=2N
z`Gsy$salXpl?X2^Y?VnKA*vcQ8MQ*r&`ypzcC`a%);YnHrAJ|%%9gHPMb(NiAkHd7
z2M-qhYT#Sh4;I&F&3}1{W(9Qng4+a32Ykmpy(aZLe|?W%+^l`5c@pQc7Ppqlq7|us
zhe1>0jlS3;?y<qm@vW2+3@Yy6GBx7T-%q);;$*ep0@1Tzc&{lRBw}C}H;KYig|CL%
z-&(Woiv&^XKnroO;z!uCDeovvN0Q(EzDqs0fQ}jbj5)9dwCLrexMVYeV4&r9!y&#W
zlmS*BNF^Qp(mQ)G+?AWQro*i-esnrx{e{D)MVLS**8TRad4m3Hu99IUp$*XA2GefA
z>(2VHGeZ4^dVRMQv-5v*NI+plF4a}!FHc=qhpNN0ThcL1$TPx0ASaatIWFoOYZnrI
z40=VTK`y)h@XWDYqy64b1hl=Y9{d*ck78$EMda8MVp0?~smJY`vA|b8Nz+6<ZBPv#
zlX|IR7OgP^)ml}#bx3)S*WTjzDsHM>$n>p=RUfp*1hfVNGCXu&&V8Z7=8VWgnH$n@
z(!am`G+PnxQ9D6;touB@MM&$No}w)RF`EptsQNuu?z<Umgc6^<AZb3{(Ny(0Brzw#
zCds+h&~MPm0X^*H+;Ztbo?N%QqZ3Jo79}vX)MidH#>}aUDmo~KC7^G1b7G@FxWdpL
z%^IzbUPrU>PhiuU{or4aq{AgE=xXKJk3n+YThXz@`QbJ6hX(P-ajnOkPs7vQeoNfF
z)Ok3<ed;#+H-$d<LdP8^HpYVtzHV85JCtz$w*oF=7_m3BP=Z0*-*O7Z(WA}(sY;Kg
zqGM@;y6vu;vuAuLIn&DT?lgA)_m7rez<EfN|3u?xAWc+bC!aP5mtNWuVVt%ddTg1B
zwp<2|fyYHR5OI$NO>O<<^yOuv5Cefz`6+}8IX5TwN@5^rr;cEb4kcjq?WUG$o-8GY
zMk(7VX$+OI*HP%!i60}kH;i;n1f!vq3-`H|jp!F_;LvRX$$e-++jErSY6w!uB0K}K
z^gl92Y1%s}<CbTl+llz$PbBEn<j@_8C2{JV+6W(SSNFMgZvGw?FTJk!W0SK3E?=6O
zc$9Cs31ZLk!;R-6sY=%JJ){bwe+b+lLlp31Xi@(pYU8;Fh5DcefvuHb5J+O-ssdTS
zFcv~o$)IgZ1N~X(ztG>peUwz2{S-u27gf(8Dmw$p@Mvye95*mk^Z9E1TG}~n#F^#f
zpWru8CtY<D1SGW=X+vJ#{vq*5au5604{OAZ)C(u6Rzx%}e@OyRe{avj({$wmpHMOf
z8jvb8>l&5`T61m^j(Kwf1NZg>AW}k~n~sgZE*=lZ*A%WV`}QU#R?K-I`uNFy&;~D3
zfepH$lH@1l{F#C~X`dDvWVBJ~9D{A$Bgx-ZJeB&5j8I{YA=Jq9`SEzo|FQHkwD$K?
zVt8U8kNFGN^WCsMdEXB{O;`%Jt3ZNvcz9YAK!8I_u_u3FfER)o$pwvteuYR*xh;e5
zC*uvyhDLoQ|NCc`66Rla=a7o-1h742iTa>XXNb)PiCV8*a`(=q3XKAfkonm94z9;e
zRqrL2$5&;ImYCXNj?e9AUDEd;yMM}!W5rX`sPM@Ny={sodpA*|$;037J4aNsKeZ7$
zs|niTk`zp;?Sn-G_=Q$XZ8-OmjY7Vy)<SJfX8tgp%k;3x{C*mFQ6*zM^-5`HdZ6aR
z@`xH%noCjd;BAHa@O(mrJ?ijtbtc(7cwY4zgR<Q6^KV_m&~?KQ!;@G#S4eKg*4yC$
zIAT6gtR_*nsq=?k#-e8h<jf(D%yc<@F`FL-C^R2T(`C-{0VqW|)}_zMd`RW&wf9MD
z)RSPffpuy5QaC84VYT7|*}~7{tL)%YKThQ;+}{TJgss1cHeEM$Tb7NFkF;U}2bx80
zUEI?j?^{yB?VG7d)F|!7@6MS7u;Tu3eT5l4Z1wbFW4%P;u3ufpcfe-IT7c)<;S4Mt
z>;6fVCuhPI2}_9I5F#hfIAC}AYS_TQPPUvAXhzu&zB#D8kU?Ob;r}8sTZBo38e*v<
z-1WIl-&<rQp~F%U!VbuI)|1c=FuEOQ(Ms8n<DTgb8nL+B3r0`HomQKlad~7GNN@NU
zYQ8gQ{V?#{@c#RXJ=C0Il-`X1G<fCEE-#I$TCSjIOuj~BW$J#u)Qk5Q(H?d{0UNkg
z%p5qwP}~-@9@iMSvu^8*2@qLoMBcrtMAnCfgdK^{umelyC1s%I%O`T+y!FN<7)Xxy
zz|W;2bwLE6#2E5o=*XMZ$Mms8#$Entyp!s%?mjv0`vCUSe$NS1djytGSqdD1?9}b-
zzE1Ux4(=}fps!+@ct4aGe~u?YWct75hU62O+yw`Bo&8YfuR@^bYD4*1UCOVV-|4Q;
z>o-6{*uD+Bh{=5NOw6puQWb7SqE27R4<|_~CN0`e&WR79-ebmW7-l!&McQcz>L>w)
zOahmC>o<PwgLb6HKs$eKjY0sR<6~fu_Izl-Y`ZHAgw{_R9$r`1zWA=PQQD-$S8BT!
z$@X>FmGc{6&4wNK>JJ|}u;L9l(Thi2kBaipm}&yU4hkjES*t%YTnxOwWbJWQS2T%v
zZAq=8WU0|zU$u3xx9jTwL*uC-My4n3>h`JpC=wb{j_y)o-9cfuy8%O?3LX8$StyMI
zuOOUH1#k4jG6+EA+tA*l>k_AlqM1YT9<W2V)V#bhF>MHfA2f9ra6Z#(aya(#`g^Ms
znGh9dW!#g!l6j>`R6kE0#NALDXZ7#0$E8_T@9*Wql%*l}oN1CbHqb~PRmdmBL&|YB
z?O!&IB9_?)wvI`MTP`2LHGxt;^L@5KNwe-&P|dHXBjg@fj>|$FWO$}4@>M9tXLXHN
zNRUY&dY6bX0}Fl;ajrRNFaSJ2e`r5?5qi-*mkfVm@Xm*=^_V|&gaNlaA{-MiZLp@}
z@{iWfrJ}iQX;t5j{cP#zRdT?4JkzmEJs6&paJhN;EOv_i1EEPsmCCET4hlRSRF!%<
z1iv!a@__WQ#&~X<g><I|?S5r$t(OfaVvCX(xl0qzb%~#_ah`s5cDBXGUvR#csD8*)
zDWVp<>GUfKl0NQ*HY|5uTQ&B&l%H*+8*+kOl)&<R8)DhN>-7iLi$|gD?fG`wLaEb2
zTb?+{;G>igVfQ9Rc;Iy19P08bYRui`>(RQbzZ*rml2x)fG)hN~%+P7uFYtEohC-oA
z?k^enf`v&etC$f#21Nomn85e#xN>gz039HV&;uE|8GX#6It%rPqy3MITL@$Qv0t|W
zq(krLRf`F4Xv{bHK|Nq%V`m64eDC*Pw3F3CGYrkonhQzC(9%NLzbSIPW&*c11FYtc
zZ=gSruO*8RciTm1!N2E|axoQCCCPyB53<lWDUUh-xkW@GF+aE;MX@Yg1s;!p35fP8
zgYV2biKj(@bqk|`g++--%w1Wn5(}oW|6H-~xs5D>mJ5$z`du@LLhW9^d@0_ZT%AC|
zfY^;BWbmt=mi!|3j+7C=rM_b$+B>t(M|ZR<l@o9>eqGuZpO*6&M^2yacQkvB_`RJP
zYvZ}lF=^536#C~weX%uvkYbKnD(XkR0>}m<(YA23*C=oTo?2u-?`ko>x2DN+nrpJ#
zRDBp6RNM=Ujb=7hwIi(j5kUcQJ#6X)i{AZrZs^0|VC8!nMn1=ghzIM%1h*WXqU)i}
zi3LZWe%}1+PRs*{=n2eMagl})&htzM@0ev;B2tH#AM6@$rIuS2;d$jV7N-|?tMvm;
z=3n4Xp6fX6(zgLUeuQu^Rb#@oTh?-;);^PLqY}3WBD22yU#Kyx#mWClkDg8Kh^Uxg
z+(7mFFkMSa@mGUk2vJ+aimzxScI*#X4>L<n0k?cB0A!79;5=_#8=fA`*%UrACWX_c
zh_TkS#|eyzwP*I8PRWxRPW-TBnaZ}KqkScN$q5f!>asR@K!tJuBeN{L*Wo%EqMrin
zznz4jKsk}7Rh`AnJ3YLY_V)G48afj`S6=^ha)^1s3WPc~>PXd%%IcLBz@*dc%+8M(
zC~%ULXEh^(3P%kQw4kCom8zxlbAkt?njkh94{TKd{EAr1!uIC6=H@l+jm^p>F1sa;
z8l5B2>DMz!e}4xv2xEKoimc}yU9nDV)Lh=FrKTz$2fL~|C{05*JGa7R{f$``+lts&
zNeBOmy1H&?SdJCDCSEQwQs;uI*!6X6T7@M^tnsBxg~6r3xl#_hua9TxHN{R@nwb^0
z9W70OgX08u3vVEuIQW~{b=7ruBj-E`fAV7tK^iP;24N~F&v0^btq6Nq2W#lY&J}gT
zD+hYn5yUnjo#<mktStK5RyuN&2TrV0pLhso>0*fto~RlX9X{zP+<Q7yhv>GpInG)*
zG}SFZE_LbFaGV?Cuh=dj{i_#A_vkGQbKkLcTr$Z96iH(7G8waLoyX#zgf7L}IuNml
z%rAFkD&Uw4R6${QlI?sz>(ezN?9=NG|IC#~iCI#(zF(c)DU*IC;HMJsl~+LBWv#!>
zM1LO~3zAMZab+(WDm68Y&NOxmYzw}uPG*5j8-7;1S<?<BbE?(|(k94?wXD+F(sl}d
zT~I%uhOX`g^YO%ZcCP3~F89IU?%1o6PC9i}F#hw^2~}a8kOwe8m8_`L3b0Q4UO+pi
zNP*Fgeea6GA$^6mnMW6#U)x$>qCU+)e!HdvbyQWAW2F;E3+EZj(Yc0jjZP>}BJHxi
zfs;A`5}?&kPcPhXrN!8hGHOX!zxRGic-qDh%$7xsKw<2LdSXC1_-`Gwy}F(grwMr<
zW;s_^Y8AInZ@BS>I#-sSo|g>7<TCv{_uMu9sO2wXJC-TH{yKDKzO=cHsH-8yp^O=V
zWO?`8ra04Z0A&QCKC%TTJ`!$I6FyIDA|qW$9rB{YcP%>>RE_8MCHZ-%#hfNdhl*xH
zBcm@CW!EZXH)$+2jQ(u2B!w;YG<^*C<f)`t<a=aEW4nrOjhc{W`@qI~P0sz3Uf8NW
zD}T4bHET!5N(cMp{8ingCE>}+kcdrddvLpcT!(51Ed0oSRW%&>Qlx#toHZ+FvGme~
z?ASZOVBiHR8<2Eb4#6jtT&iVJ*$B1yBpeAoq{_>j$?8x?+EQtdWDsJ86&wWTdUWz_
z$g4rWY^%8lSagB}jc1oDY~<k%A?SLxh4CF79l)~etE(W?mY0+44Iyq|A=ZQ+LvJ<*
z%i0r9U?$U{StOBiGMAc;Ua2dSWq6$`9jl%CH|>N;gzve|?DU%4nMUfsp-Vyml;Rj^
z(M^;&CN8=#(5r)~&D}wb5dY)~?$*ziv0)>4tDR=Uk@V5LIR~v`&1cwU6bM^I+rtbW
z_Lya!XI2QFYCb?(EtoVvC0V%*5Tn}geN*})etWkM1l`kV3-V3a&0YrGs(#b@a~63`
z3v~|b-fGHg-INcm3(|+WFT#<;ItDZ<y+1iG@enkd>>-R<)nPA4w79CPK&LCtx+^qp
ziz_tNwKan$8<iOG_B;xc1ho0czg}y83maHFT5oQEDL?Kd0t$aG-XF@h7ign3oU$#`
z+^*)rXLuN1XGijuT&XS)^~D+qjoaSRU}^F^&g`Vm*NV5jHtO0JPZ{bka1r%F?GLF1
z199y2a6CH+j7-&MsSo_s+qJIxq9+~mPGHdPr4k@$)a37hX2X};+?o@^R5G}~bDtHI
zB9V<Xh%&DU4mzvMr^Sg&Q_-Qj7hYn8+HEbI^YVg;!H<l&QH*$F=chK^JJl~#<m{x1
zpV}fCxlX&Vc}y{CWtwDqwAkLF<3I#da_4^ImDYBzR%;XyiIOz1c39Q0xs6%14rrEl
zrEehdLw%s>A&Elt5qkb0PIxg?A2^|q?`R(HvEjj8iumL43iuS)n)4R6$)FANr&MrL
zXK3&f@4|e}?<?eqZKa?Q%@gVrBO}B4AySA_`h7kk>5F9%>t{c_G!uKNGpR(w3D<c<
zorNL5r#^uOk9emuL9?sC7S+1;VHE~q=WE4bIb9?hsB@dzM)-}d=>;E;Dx{3dbrSCE
z6mzDIZxjT#qWss`#qikmH|aO()z}hvSWi5@2gC?OZX49{WUnN`$tvCq(cj#uHCNL6
zLCch{KmDSs>$tCE&+rD%S-1{bSIWF0@ok(?H^#4Hei6{HWy~+(>&<BFY46=xr-@mx
zp01alv3I(RkQ`|Yww7Vt{xWEN^3N_jyu`dH3>T|qd6G~R^!k<U#UT26eVvaZ%v+S0
zp>j`^0Rv;Lj?-W?`G~1IFD%oXG7v2|77-!AnJvl|Tz}GkzI%W=$Gl{xVM>!8)@<`x
zT762ArSRb&0uLwW)+KAdn|1@Cz<J((iD#-Mcdw>lkBi#p$g&PjkQcDjJ1iCm-p-I8
zj7HtZiLU*Hr=WHvxHqd%(TZ3DWI`HGmHrO^m_TR0v%bMUYM?KRO33y2s=lQ<E`14`
zn_FCZqggeZ07Y9iv4%|ulGWFnTuOojC6!))87QmU2(Dx8njjoiX*Aa*b$HUn8pNrD
zBhGmq)C|K1VNH%XzE!_UFg)^GL-b7kZmMr<;#;^*!>d-9l-UT=&?BCgQ}C0qt489f
zfQE`|2k-t^Nm*&PieAzVuimr@6Q0e_)GcXCY?`j~`Fqiz{ut*^h4uuHxTDJ`i7W9s
z@R&!HAQL@v(kDDf{Ks#3(H~Rezt+G?O~ijRn)S!{kFW5dL-vE;T7er3!@lQ@N=i!(
z=$t-J#dETaKudx@E}fRflVM=b{5J7mZpJJpnk!?kk46}-l^D8?2?Ophu{5Qobi(_f
zb3Gmg;-I!u;wbmAhe?3JbfcIpK5OQO;lvB^SO^P*M0(#4Cdf#eXt7j!(78c))e7Yo
zP!_0I{Ppi<o?AyeFUd+}kQKp)9lOjgk_vcA<Q>2JB+5WVOe|2rFZ54TQ5aMRSb0%r
zv)p@oL>VltY>o_w4uwHYlud{Z2<yRbWzh<qzJrG-xe_ncNWzQGa1tb|ON8)QD;nQl
zc~i=y6_7Z9Y*p_nX0`B&ipsv)tb>%mU*4eYb-NsDjzP?-m@jWNiD1FtqeFaNk)sQN
zwT2@XU{U7Cz~lU%NG#v&<E0Ek^eKCI@@HgJLUxjLE6XgN7&wSVG3BLpgL|$B^+4Mb
z=wLWK2tSNHbUp+yP*?}gZ3wH`A9J+CJzf(bk1~ijWx-NzOVS1ywY>0KzvcA@v8Rh>
zCFCQw<wg+(U$pp)6MjQt-{VqZ)y1E%MD@Gng<s=oC5>SDL3G!O;48@f4=UaoaKU3D
zaTMWOsLDvL0&N0$N=Hm9gk-ojJY}?rhs}rFhi|_no>ujUsf@UCU4xVf#F#2P8T_{L
z0`V5H+`k%rE;5L8EIxE7@NT&V_QH6DkEjQv@av{jgb$vlG(*n~d_=cMc&2oVxXSL9
zEM(;eF}`BwsG9DaLN%pjJqXFxd8!v^USC(P2Qm#W#3l`|1xY?d${nlTB*?6(6h5q#
zPh^W1Rc02m!$NQs=y41dd^2*A6znuqf-VDN_55)dC0I)`E0p?X*EzX7KDJm|%&^wj
z8w_teT|09U@9@bMN#T{1)Pp8P=}ofu^Qip*q&8HH46Y*@Cq@|8oGc=1Fe!btp$8#4
zM+_+y^QIUmS(;H`jPW!Se-#!`0>xiVdtaj{%L8uD9o*nrR{sc$eid%417SBHN)Db;
ztKiB8Yu>=Ub84Ur;cW<%_=ajAz~`#Fze>LrJbn>xM-hq%WWC&3$V<#KgKR`C!D63E
zoac}zNq#c9BX4^0a!OJk0Gabh7r;FYSW3vlYs|nzd2^L``gF&}AbYm`m68KJi2a90
zEpytKV*k}@wQ9otTdS=<+J9f+!{!}~AekQ^RWRc|Ce}iej;}rxNlQry7RRO0mIDQv
zcFpMAVRr(hIY69Fl7^t6fEFE10#MmG(MUwQfOtGdAGim-Fap)o9gW1PV?2jE7IEfi
zMbg$H9diav`~!MshFN(-@c62UpL6TU9PeF!>|vsKLQNz|5oy`u3UVwf(8B)N(Z$jJ
z-f`>va37P7u)ZuZ03;6woSwHXPhRaEoweZCzslvsg<%MiqUZW!5?KpV>tm`PtOwtK
zD93gk480L)g}9Tb!ppbh<N4%}Vh@Hp>D)?Zrl%lVDeop5{Zivhe?`JA{gse>*2_Pd
zj~_qteI7#o^ZEJgHuB|vqqeb;^#5(t9{qn`<0IV+d*|neXBVxPm#;9Z+d_S9J#oIe
zGZ>X3d6ofwRGh%OYSEX`*snONtE=+FHK<4tXi~U8;TrY^JZ#g$w)v0`W<zNJ1@Ft2
zy+qe2u<rPajpMr=tBml^WEK)jLl}r->e7vZDJzHaXSuRqWDLN=>sg5cmdt{s@4~XP
z^ey1O<k0t(l^0aPfVBkeE<qtdvn7Y003@@`Rtt|{@nQHx+5w~7w=fFd;*k<~-)@;^
zNvxZKEUKU4cNxL{xYyb{dwos^?kVvoAK=FHmx?^Hyrko_<OW8fP4Bl7^9nMC$wZtM
zRdUkqF6LUmPJOdmah66)Hfm{*1FsO|ZxrO+!jLgBs4D<xKt@|`KD1@#Q8yQJhCdDY
z#O%zbH_i`(+Kg~#3c{UiDx2^S)^0I){ip!E=N0EgL2yOTdk(N(6asfu1RSFFycoD5
zoqfRwCc(ww-DM9h%<{z8etS0szq{<gHQ=eSEdVcba1D5JY>R*^fSw%N0^l+S*8nHS
zwg|Waot0yo1Q&<boo#TtQ_$<qHn!8j%`&tFpv^F{)1l2WumzyaFs=z`ad4B_hIKLp
zyUA?BIvw0B!&(5^48uAd+APCb0NM=0nt&Du*Uy=*cza%kTGOA#ef^wyYr>7`1x4Uy
zj<gB)xdi*7FxVpKK3C|zC<r!l>`kyQ6xioQ!4@IH3k5Zu2no13lia#kQNoQ)`u;TL
z?dMGUoCcH_DAGVCd*n1A1w$eDb96w8iX!kc872q56dr}(&p{t4PLlBB4g5Amo;W!|
z<+o{k|JxLy;^at{BEWMci<2#03c)UnxGiIfz|NI1PPU*a1iLUzY-y8(od8}`+T@9x
z-==Z=Z$(8;p4=(YN-?>UBYX-qGH+L;1S--<F$t6-h6*(@FS(>FN;VR2<$b;>GH~x&
zQHCgbKW%2b&$mk~*fB{|1a_V_Yr(b!Q6bPp(YEDK5zu*ht_9i_Lxn&WC5SDBlAsf?
z^Q6!;E}6#X@AHJuG&U&!JX_{W;gce;3nLz5lmf7`CCn5~DFVANO=7GvRi1pzktZSF
zrIo=bAE$Bn#~g7I107?NBA_$bBnH}Ml0uLRB5kus5y+V=5`%0rNFm4tsbRB65^@4`
zjwH$Fj%i%|F-MH#Ge-f)+02p08$|#YLpjD81psHWMjmGr0bGm}F~*oGH}LNIJVgX=
z9ZhGAS8pz$@)SQGRL@gIOz1Xy6osCrjF`|Z7AXw8AnF#I6a}8AkeI+NRw)d;AXzMS
zNd=zp@#QhhWE#gzrXfGc<Cr-31j`fxpR1VU@k}x3MUjs)O(E#H%1IvA6oX!rGI6#^
zLQgQ|dNWtf;C<2Aa%R0bh2^1oo}4kEC*@2L=y`I+gr1Nyg@6}CJt1d`0MC;%Ch&xu
zDFnPASrT$40X)Hvc?>g|#xav=$WQV(CJsI+XNrK&l{0xfQw(}h<P&nH2=rVzlgBm1
zpckb~Le5NOo3_m~*0mILye}nGik|01OYQFdg@dI4c@-%2U5F<npz`O2LOFkTmhcG2
zk^`XV_?88*omQZn+2^kmIh9})O8`6!t9<iUW|ePjR@r@I6~QWqUObdcN*a8ZFY>5Q
zS=H&~6rmQzL;Mu43f6-HC_YP=dk7fwugZTR-fHNES%IB=V-)5c?s?;oQ%3N!S(sF_
zCXiE}`TJ9FSs0JEj1TiSS>ycI8fI$%t+9>|wPXlul>c(WoOXMbyp=4m@=O~>RVHs3
z7xYzO1%`1!UVEm8G3&N7J<+BOfFjDgM8~>Qi5{mkpRqc^`~ZIFc)?`gKraGHCeRs<
z$6h!Za`rLtSRPX1lNBLerX>zeg+m0TE@tLKO!5p9LH>PlfzZQ?g(qQa1jCC5+$;9u
z@Pj|XGoG=>*e7^guPoZn<w5TTbj*B?=Zs|Q6v14gw^5nmnK6Bc<TAX!v-R@u_0dVG
zS5^!uZa=oqiuSnjf_`Eh1Qq0`Q%&FY5^Wrw9O$xF!)WOA-L}`~N?y=ar!?)y+sPI<
zH-vFoTDOGnOB^F^S1K9o!jar%yB5r|mbxFyH-PANV<o0S#IbDoJ^PYGA)I5!nN|6c
zF|97}EFC@^;Z#D=CvN8}BF{W|P&S*^IWmX_UL(b~V3QzO#vB}OB$ga?KJL}&*_*wK
z*4wkAlZ#iSTE(f$gK2tB&ItvkDcwvaFOU~El!?2#X`Y#!F9)0%-dPrn?4*{8k7=S3
zb;0CpvlDo2ZmbOPrkG>a^Ed&dfHI#cVYT=`=fChm)SNR$Q_lZZ8_neT-{wZ`@%-0U
z_yE)Iwebip-CP3m_uiBDVB;xMT$I#_S-zInJRU1${||d72gip{r;AGhA5@j^<ayOd
zzvk<;c-q3V4kh*}(QB9!*WC&)Cd(yY-(jaL4#$+`V|RR|W<At-KlB4p*{HkB8IS*H
z`u^EJ4>JGxqU6jrrp^E6Mq>W2RU41z|G&oPna=HoqgbK2Pj_nuPTLDQ*OWGT>|sJS
z5C*I1hvpXo%-rkY48%N>`d4iSHAeW+_4<)Z`XbL;Yz$Y)F)W7qHOzi|p=P%zH5{cE
zlAc*n5YX<3#cvu+$}wOAMX!nZyIm2>WqZ<Ny8xcT_r4eWFOn;B&womhiEGBQscqnV
z*w|Rhod5VeHvb#V^|imPeTDx2U!VUEF#fDm{|q+D;mO`=C)vgn`Cnge))Mj`pFPU|
zFaOauIVZ33rYoDJsd;ko!`b29!MRhbE=s$dHXg3s8?V(xFN0JLm7R5Peoh-@=#3Uy
zA^s%qvQz`{fmjus<waEEcvaA<9^L_<Vw{CC=?7ih?vRhmv`L3&tfMkiF?0R9V=r*A
zJUnEWcwdna=ONua8sbf#!ttYu>q`LmfF{zV6ZK0=R3co(RFVOPxJm8wiK<$0o`fIe
zIqqt?BJS|4I7^4)F)Xj_%`t$edlik|edjcB>!ZQh#|6FvzfHILnGaw+(&)o)%=j#R
zaUsgaY}$V?SeTBh6aIpi)&0KI^+WWjVP`NW0j&7<_?0t>;g3%UK~WT|iUBb3m1(a<
z+p3ZBGw}o)UoNL%gQ@%#P7*wI6Gj#DntPqdgHagcjZ!7WaR2?Ufy*vX+WEUvb+(<u
zS9_Po7y4EzYXH%hj)p)dF_Np*Re%Nkg-0-OOi(a2&|}69NHu5)2<0W}4UP(3)oAis
zTHEb=tVIoRJCaHGU&Tp)kZfJj&LCJBurq0ri6}S6NMe?4^!V}gzC)*nCASU6H#pzQ
zS}+2hxC~^=3R2neuw81UE1r#z@6GFS$!|rey$weoc`Pg}Xs#<$F8oCbo#Z8{@m&3~
zq>-`TFfE@DH2+=2Q36I%O%mOZAigA6ej}m@X?`+%)lun;TwDaWivt%5k-x@D=}jIa
z77r=JAUI!G{y)I_@Ahe2S_Ef~^&jsRjIaOe&H7{hpRe&Ly#DY1aC!1Cr+U|<?y1Lb
z_I}<wK6-tES9XczrdD>IKX>X4wexWrIN^O5d4tg5EZ-eB_#W0SZ;W}LaS4pV3j3Hi
zb$AD_x}nVyc_F!T<pr1+jxH*4o&GSwbaY*R9Nhz)!Ek(!Y3lBs-zI+NL;rr&ao$eG
zSLnTJd(o{2OJwK1)AvG<yODnda5#hi5Zm_MzH@r|1~Zj=t6<?a>MLzOa()6?IJ^~k
z;*5=o#SuXEjLDp>CBWK3g_mqOuM`7XWJq+!<SKyBQZ+?r(v-t|>n$o3lH3$3fqG{u
zJb<NqOyUemdsI0XmgvS2TpO&3Tv8P(PR&>nIBsAhlbYfpe5F89w8r7l$&Y)-vS){~
zUxX>S7yRG`fOj>f&UX|Ou6~!5gg3E{DK$p*=Z<8<9d6Zlg}G0K4rTW^ugiFahUoSj
z{hqbs@(vTTb$MBvUPj_PFdPo@MxR|477u`W?i{~v;g0O-tJd3#Gt`<IHfvlSom{Lp
zTl=Nz-DcJPl!e~x&}Ujvk*&~TJ5}!ZG6u_jB}{{vcod<{#-=ByZw}w&u&Lb?j8p}O
ztGrZ+@tA6RRsuJR6S!lStYs0nX;3jnU;(1`S0-?*w>9RL460N2Eh_fxkHMy*tSa)S
zWy2>UBGNQU<p~bGc(?&5!&m_Xh_dsKE)LI}W|lyKb$S@N{g^&3g$!uySp8k9hK_HP
z<~$1ds&u4Es<husit^n;VxI65iyVU$gIn7%Y$`wq?{n4=o;;DyOSMZ>U>Yp`RfVU!
zCW0?sI3<heR%&IuYD^%J*L!hYo20;0R$34jhH-h7Q=U^|lS$Z~^nQ7H`RefO?DPy{
zZp6*=3k&dq7^!41UByujZCFj=yS(Ym*h1&uhEb^)5mDPE23#@<AuxREl)n4+<6U{%
z`7WeoZDApeKjpvJHaGo={^gN-Dvi|EgxnIuq1=0e8eH}TjET}fVa#gf)cf5EjE1K0
zH!%w1HKtbcG2T`9i<{&YSU=<i{M$)@9x-ZfUZXIs>{X0+S_!D6)=)D%%a(i)`;XE&
zd)+kK4?k1wzx8^;|GT!)c(nh%!e^@eCoCU!K#{{2zV`ep3UNtu`kr6xT|AWAm)w8j
zvE{mzn|_X41(N~P^e+7E*337{?g*a2zwmasOW(Vrn(6kf;9aDs9>=`Dw7{b*D6z9q
z*>mjaTvfboIex1#kFU7Ul>)F346X8qCl@4~j5sNNy0d7C-lgxt5(v<^FRvtos*cB!
zYg0645fA^CS?JhZ8Wrx-$m-PS&(<sZ3>J^Et1ReWlik5s-D^r7KN=5jd|+~yuQ<kQ
zt%r2YA6^E9D;ck1(QxcudF+liY1o}c!mgLf7~MY`39U{|WelTN7Hf2E))3npe|YC&
z{6ScDll)=!>a#dRyVFaigJinT+;o}8CMy9dCUj~s-r9-J`+Qij$({l&G2dz=Zv`n1
zys-z$r5ALdU+I&N@9h)>_I{ffgiKbD<(Fcl#3Lp&(?_c5w_=1?VS?t{_Zb+7w((jw
z4y|UAPlQE~rf0@_jwc%NqSOhith)buvK{Gvq!?+yO0)^<D&WI0q;7=QSA=v2hT{$>
zm|9W-c%$!ke7ti*k25>KNbyVxqttQDl0Z|qq?^tq*)xASpHP|Z6h4Vf!XlhvRLS8L
zp7jNJC0!K6xy9y&uE7mvFWrad2Xjo*cGFC8x*c;lm`a**E>KB{5Z%hz_B3UD4Cq3g
z4v!kBJz*rV5jLJWaiz-|Dd*mRJjbe3bNc=t8bK|F$!3t&1ylCY;<dv`Ti4Y)hdTm}
zvHfRm{mETAYyYn3-rcih=WHTEVx7UHANewRjxMU~Tzj1lWG}bfF0UXQTPyAb&$99i
zI+&)M>R{#b%cdED4`Bawt|!5VEZYx1`SxGEzP7%Ry#Hfu?a}`G3ZKIE9}aTsdN}+L
z!W0{hLAKD3os0tgu6%C=3w4JrjuF7%5ik#T5^ufEa4_(rcs*Y@$aWHsyBW_VoJ4sq
z>kT<>l8-|W$SI|1`%*altBc<{e1Ev-T%7IgAGUrtJ^fdyH;_k91XaapGkR_x&Lt~3
zABYt=4@NUnoHa^*6nM8Sx7!`7a8>PXvpu=giRF*pcvvAzoPRT2Do!$XiY1?c6zc1;
zwcYPV+y?svV{SrfW%s+VqE<N34n0#X?0z>X{~DN@zGL^s04&5CL3aM`Bu{e7uB4sv
zcB}|5#aD_Sm4H?(%u8J<8y$xT=?L-(*R&Z<^NDZc5+l|%hm0P@#!Glv5E`tng~x+H
z#Rfw`NvC%F#H4|ZQMVWwWAmEN2CMmwMxcF1ZyKcqY0}ZC<Pnui<q%c;81{Oh7pb3p
zFSv@Xc@m_PnX@a96RHr!xdHyp(3QKKxId5Mt$s^PC5kb&0{XGluBj(zkOw6>Y+|PD
zI^VFPO%4hg)(Ot_{x6bQF|z1=8Y{<#LQ!hA!I0I84TlaydDyY|i~;x(^jAxBo{VN%
z6r^^7lC;E6rzHJx?VoL91$`Kq&8__Z*?aTGIIiRDH~)&CqD|Yn00ol0FJ|bNq97S>
zC{iyMJC3a24uPRK5(q#u1BsHu``K^(s!sQ*S+EeKC^;b#i@;3x>C<PaQ_E9Nx!<2I
zF70z<`e_3lI)7z|Xzo6JZ5glQbk;Q3K(2f4>LbrTDOJVt)3QAHqdwcuzJ2=Er-per
zUYwHfDEa(t%+ByeqoL(Dn<7KioiRl=sHZ-B7&`658Hew(DXtBuC|`lu^Ke@XC{gy?
z!AQE1hJUlI<oSvI$v@$*d`t29>!JKahMpaazMjRWnC)Q@`K?3+!tMZ=U%p1c*TXgP
z*DV+<ya_{B^>;AxN>ehZ+grn*`%8ZhI^YM3CuaOapBewu4HN&=$+_|YfO}oGFAY#F
zru`n24@}hZM^9Hj+FSM+4tWwkM^eKoc5!1f!K;5VH7AhHQ*!%Mz<KiD!&i^*-nskd
z?cMti?r+~RU(3#+7+wdmfyGcidST1drqTQ<en#U951$U%2Qe4oVo3pRG01of?S=RL
z&_7mzoa@HHWeO||NL;C!HqiIYukU`##8+BaowU;Lr86!$H3uQS&(f9{CVmhY{Ts(k
zJX|wuk+`u>0kyiGxpaZ(M)1nhQYdA-Y>5q$)Bf;}F%u*{;Iwx37aAHi@p7;CFWO#j
zIWGx;l~FL|?gITAFSh8!$Vwl_7jD%=PvYg!LQ|$>o!^ehti%hjg?3C!IKLegc6~eH
zkA+<{Ug(dCgU`luJ2DKEwRb?{K@@n2cjk5<<lVva<WTlH7ki=CpG~VV&Xt|M+R(yI
zKlkC;`iEz){qSt%;lp@wn64SFv-<YjoVu$+6OQSNcW$Aj3y)2>Yjt#je(>7QUmc!{
zJU?Ii{QMHn&(}|na_^X!x!LdJo3;=4Mzg)q{@CWqYbtS(51(H8l<U(IhL+^j{IkwJ
z=TAG|qV9F-`Q{zLs>_DYq5<50>a*5#^(EI5nE_QQ-XvAcyR3#DEZX4y<K6=GpY`TY
z|JjPR#>>BqIdNh0q<!zfNa$y0p4Dz{{M4M1Q?hoxY`TAca>I-GtyMS2$_p5z!xK)%
zaocimW&7mnYQlsc(XF)y*TPN1Bl_cMd*N`px(M|cuI>+^?S&&~IVEveGOJMtw;B6g
z7*@gPa6dVEDv$eqY^~nZW68GV`KvgXVL#_0UcX+~ez?3~y}@|eB7XYMzKEZ0u32Aa
zte6#RYTW?X{H+cGpSyujLe<4R5GQA011&AP7o+*!^Q))65^(jEey)4ZUai_*P^w;I
zXRMdXalhDP5br)7P06LZ9!xZpjsOyf(QjU67^mcqX6Kubxp@t2m$dqP4Amk3>tPbl
zj!vcv3QTf5o+kT8Cr=M1l+&9XB_q2zjK#B)!F1+mg3?p{Yx+2%^uu3SGMU+}U!RPo
zqr>?al7S;0EE|44l~au@LVZgmo-}mF4ZY`P$RhJQ+wbbjZtJxN&SKS^rS+J(wb*~s
zC!)EBxiN9a_3H@VwX5dU8u_V9!Zh`jRdB9|>~-Kbm5;x)I66f>pEi@d-QlJ(e6(u!
zt)I$Y!wbLO>t?I3o4xMqz6B|JnlM~1`92?gE82kcA~`%dyuKvrObCtl!(#XAx_ldF
z{%TJRlr#pNI$rgVy}lF3t%fTLBni$dwCM4<fV1c4@kRMu>Wg<GfCX0^OVJ^9<20Yp
z87`rP7WY@1yp*45q0xRtyGAeNY+CH;culjH_Bbsx9S#!LZtwnHZuAV_(?W|uZ*wj7
z9`xlF&+tMmv=}rb*P?y0CNqE0AN4xvTrNU#S|-*n^Yhm5f3wT;KW6y~{;!h1-{b$@
z!Ow4hbMxus@aF8ftl?+Lm&w(azx#)r=iuR=O%4uTU6s@8>dWkE@>OyRxboqZ1oEMy
z={&@2cJDs8|LN_k-^d#Hg9Rn8Je&M_*7y6Lvwz;g{trsZ=?LJn_kRZ1cV+(<>HGcv
zE`BbG|Lr5lSI8tkME9EQ!W+B5Ot$x8f03<E&cGOZSFXL+2ue$9P6wO<<@xZZ3$o{I
zSaFVkzU@tht%M%>ObR23es#)gqS&uF(U*sy4;mbruYY!RbD>RpeQi;HT{mn4aHWUC
zg+G@E&=UccUoHG?k%4k)kS`bz&@L=8;9bn|f95zVGhR*zz?}XU?bjNfyffCpSsk^O
zzU+JX`=3s}TG!XD#O&A87lxZeR)r;tEza5E0*O<7T1+iMY2MRh{X8^TNAj}I4vv2C
zJBrzRF?z{dhhJq;E>3IQ`<vG@p8}?)I%c|e_(=;NWo{uxZv&%+7p%Y9Ab!97=_OZR
z-AabL&wR#zKH2!<@xxDcKK*p}^E>~sdC6z3v~uP%p*7ik%$1jVSiJqrr+5xZ7(7;A
zb@PJHUTH~bNCtkKbuWJowbt4tW0)-Y_SOJ$n@?v3Fk7Uj#F<UapLw5c_jH_yFgTjc
zkY_lU`}{fOP#q^nPrs4(3mcoVM#{Kw@?xy*9F9lRgCCRUql0IfGx)viXp$t!M;Om$
zV(X6f#y75*OZuYA`U80p=cQZbNa<tD@`jF5Hi%)zouWgad?-u1BBX~JwMu_cAXgzJ
z+#Vmize_<q`EKpZ@1IPOR&wD)q?E`uWT58VEv`inndvb9p+lQ*@Rt51e&b$Z@W$D9
z;hkSyi7YUuvx1$nEmrB&q9bl`m9M?>0()k21(x%s&%fwHd(yIVAN}Os@|O1BUHoyH
zF?H5ne>$2?_B;WWNQg)Oxt7jyku}<=rG8RkDf!4q2p|2Y{(d_-?3d$DED2@w^F+>v
zd_GlptAopaaJc5)#n8{;cth`9Qg`}W7NSVr^Ssj5lG}MruxM#Vy$t^5+v(=u`=9-!
zO->DY6&=&dy_TIYr3d`)>cg=}E{*2Rcmz}n&qs#?rSa9JdqrL7M@w!VUi9W{cx7$t
z#bg$}`Kv_4?`|+wAH6!O{mTB&?S~KE;sl@8?B!o?p~e19!~KUBo&ED?aC-g}uI1Iz
z9rp%gcW1KKp2mn5--Z9VG{ApCX1t#kc=3K(%Zm5YdQQBb-Z~@R>HEY^fXEM<{P&ff
zTW)2|uis3=^;eg6(&^N8dUdZ(1ZKOd3lnZ@AUySWNeg=8{x9xY)h=ScUi?gNv9y_;
zPV>Pj0zE$E9Ko)Abe&r2Zr{h{&aFJ#i<33edfLJ8RvtfSEJHg3f4n1*x25o*+a1@<
zl<A-S`(v^GNBc0H%*V?7Vgf_D=z6$3p{|Qd%VAxQ2R6l^X1r_O#QpVa=o0LSeNW#C
zn_}-1SYlE1f2vA&Nj}AYwx74?|35VXc$xTLS*P#)|L@@Eef;l#_s?6{|Gh*#H2`?d
z{?DsoHU3w<kN>@spL64XXZq9!OQ#QgogSe=W9x%CFGepdKB92mTMKn6E|a!81m1jZ
zV)*Pe^HyE9*+pw>T_DqDHTbip+SDl#4xeNbZT}&%*1so~*7EpWI>@#-K{kZe!Bg>e
zi#4bGIIeg0Z7e=RJ!Gf2eAQphy1UNx`yOgJ6%|@m<r)4=7t46F_E#UUXAj7h7xd5Y
zF7Z)s?{(gUzf_*Iwx3g@EabkQeN5c5-qxT*&doQpF8*okgST-0A0N#oFLz(ikVxw_
zTEIEye^uox^<U-k_w}FO#n1ctU;m82hD+4{T5%(%ixsq%BcH=}y|2f==HiyuqBINT
z!nWRCF|0nn=&hFn8{X$|eCGS~wZIlig1)cq^`^DGE;{_74vyO^YX*N!1OFrIb^RI!
z?U$*+^(%PucU@C!rTEFOG=@0&HCER$LdKPeK0RzQStzyz-QxQaTklJ3y^RuEzv`d2
zVE;|uF#n^d@^$$i@9TfPi=X%TAMf)&-sgY(!ucP|+T_>MqaW;Cc|M&ilB#Yc*S?F>
zJ_p({Xqk8acK6=)Bh>DwM{XtGxGNt0=c6RA($qy|M5&Zdzq+-AfBnxV8(A(YoXGw%
z{Ihvwk<1`ltpD@6JO(^E{c06CxJI|N7jw#lUh$1VZ@=LRUQDrSvoF87o_!U;xZe!=
z$9Ho1G(M`o^rLLRrmTCB?^MTHmNL`D_fg%gSJ19s|6_0Y2cVoi$UoAar+@nq_-?r?
z24S=77P>riD)JYhMbD2&H0XJA{t)V_FF%(_WD%)fC5+|u>x|?d8FKmk>>+y>1Lj`W
zo;6<prmY((ksouzq=jy-A0$()zdo9KJXL(Pc@Lm+qof11ghyRKp!PbbEW_b<TeJ%r
zsVgCJMZHC%xxp;LaWL$0pk%FV&%HB9+#_qgh#i;5VNLZdToVfb{;H)<Hyu0kWZj;F
z5jpO+{yw<Cu!O8PA5A!cRIh)EPQMqHjVtSSCx%&MRsQ}s*^q`A^G)(w=Axa?lHdRK
zrCB;USE5ut?~#LiGwR<xEaqb+bQjA;1otq*eSd7`Y&=3@Nk4rnf_+)&QpDqQNqfx7
z`i}pYNF4+y1*KHj&%R_U|3kz{aSd^~mDK$&W!hEx>#HbcKi#=^Z};QBJ>E{zm!r{>
z;q!-g?tiwO<Z0QinU((Vj1OI5evFg0(nI!->#oh-0)~9}_QF{5^SP{!*TKWDEYSom
z?uKKHE~mg8f7)Dix-wHkoi%%i{4NWC8;5bVyKkNQBzME9PvqK#C$2ezbmGoBaE8JC
z=$kdedx|hUON!d7fAHNYl97G+9sBZ_vh!TU@A7`Z>^@^yzFQiW!DFJsbJoC|->?m`
zZl>$~XkTE4%>iIO1bvh5;+uroUkAng?j!4Re4XKOeFu8@Hlz4mz{<W_J^_A?MYNjZ
zFnBYD_RjY@o-VWW>jlMbH^l1D?kq}IZT22rd0GAV){2(52F1R4dBx9rBh9lIVmHAB
zX^$B`i>}9|{d7jhPFGAWqL*kp9tytFWB`EE*xEHWK)-v~w?6g^(6@hPxcoc+@O~w}
zIztVyQ8;tG9DHp{Ms15z&bz^M=r6vtkoV>n)4k8addjPX&;KHyesV`nHn+}l)>+?V
z@E@PD%=%_G`pY-}ds$k((aP(a{ph1x!%OySuYY#@rWfa@uf7ZP_ste$-|p?-vm}S7
zTxM0q85x?n@ETq0#&=6SS$_y=b!jq)a2)r8+^>%yzRx`_Qq5{uqt356CSD~3lP=!y
zLVqsyh_l-58(4o7$$_)}dF%Re&U;=|D)bHd&-xX6b`1I~@&u18xHyf%8+#8H=+js5
zU~vpC#>?Q9v&rjFGq@XHkDcL+Cti%8!8LF}j)pTHc@9m(S!eC*Ff}aq;1XmFD+~OR
zah@f|nh)q$G0dd^kN;rh@LUKI{j=fz`k%KD|C#Nd6975~|Cgs}wwnJ_ypR99i=PYQ
z|4s+`<;$tk#ThIMkvGMi1$Eh%rgwB2TS@=%>jN(qQnlY_Y5yv-wBKI7^Z;r&{ADs-
zXG^KBeO~e!#+3L<7ehL{y6j2!zbS|H{m<op-op70fz@b#*AUO^bbxct|01nd@&9>Q
zy+8lo!Ow-y|ND>s{BZluCy$aW9V*bzpNk~HtsTtW(e&#@{0G8ncPJ1bQtsM-`Viw-
zF87w?`a++*@5unq119p~=}U&Ln;Z_wON)s<1MW1$QC7;yY#5YqK=#{mzoFNiS}Btd
z$`W<{;#dsmz{QU@<tJ1dJr#5~AP(ifryvbo1a9H=HOJAQgFQ6x)z3Hrixtw|{hRe`
zLSw_OcS}<d4A(n+04ihr(y{i<MlZ(6XcoA|<Z1TyR~OJCst&A;Dth6#R2=tpmiq^6
zwYfZdUsTkv6IWRW-)Kwx&s_#<bonrZPQhb<_vD4<<Bbn>n9<}00oOZ7S6}iq9?^RO
zlW&e>{TZ}NpUdAH!$XH>^p|;0?@x-8>c3gZ!nHZk2e=sFrM1X~)%#vu{2%!9_VmB`
z-e~Xnc-P27_Pru_j{aBWX}P5T6>>xVUjKUsKNrRS8h%LOM)K>k$=6}NliTF}5Loy>
z+kp_qYIt8!Au>G~hx8u))QCx+(!&S0&Q&Gj3Fv~52797X)_8^UfUnLuv8?Ae!b<f=
z*ZR`7n~?%m8@!!t-+!=u|M9x`)xzMoc#_`Gs})Cw2S1`)nAqj?ewKvcoXi{+YJW1b
zE)F4|uaBrrH+nig*tI6Z>7#v1twZB1S*;qkn4NmzfmlBFDA%QUit`CH7R$=f;MMSs
zQMu}$+#jOa^#I&@(~e)d-wX4(zTbXgeqFu3*kE5-TqN&1{j;tQwKWI5oF;xvE9QU7
zdt7u8o$?mHmSy!n<Tc)8RV`W7{<!<Y6*K*2-%gG<uKs=f8~y$2K%yXKZ!-E`>_on}
zx~B0pCcAnEtYNc`qHZ$Wbb3H)cJegd8_o7xp&e&pdh;>~EiagZ!>8w1Zxnw#ISd3E
zFHR2TlVfTOOkRwY8DB<b?_2d>JQdRr&3!uBbJbW+GydE-_&~z&f?IEX?R%T@m;QMR
z_TS4l%l|CWqA6DMKePAzfA8YwT>H-`e4fn4$tanPk4KR3n8#8@O03JPL_RYu2zhik
z4<aTArP+}Z`_PigpF?Vw#MBU_-Ff<B@_alxPL8KX&#dUS(FExY$49e;I?~AxR0*fO
z*nuC+wwmrw4oB08Sy-b`ZACk`zrf`)q<Rjnyg{UXuyjdc!He`(u(b5g+ex~ZXW@*{
z;1@iTw9vZbPv77&jpk>tKTkV%dpKO8hYBNRzS!QH>IJ*?X3a^5Z#!ZpqG#GsHqNcs
z#c~njH%t6mmk(n3U9LKGD1zXhw2<Vr_#pLN?>!$Kem&N-x8(ZuKMv_{eOA|*FtX*r
zk@=a9<i&Ph9DV<*9iZ7y*O#C6P6sIrkGkT2onw@IlvM9_A^mU8m{;Qe*GZXwIyqd<
zmtSmAE5g4{iwDz_bDDgg*8e`Of6XM)_GR+=WeKlK>R)|=F6GNSfhB#hNBsQU1j`Uh
z9kz69eQJPMrk{=HL8d|jI}SFd-4#*>+{mktzFa~<-;p;sRkXv8<t;-26Mi{4hU`8Z
z4A%LDL&P06SvY65`axM0wozMsI(|O-ezN>rT_Wc`_F(KE8_Vt4!~4Hq|LHnU1D*Rt
z9{N#s>BpYh^g?38WnbwvbQoR8rtPiEdHJ8Y!235H7*`K1AT>KW_+IW6V@zJ>#W6;O
z4$XyD!*y8=`CDEM!|Kl7banTyESqcZxxIGZZ=FB#qijqry!3yjm;FfEEq=CLbv=Lk
z9e1r~Z!lG_ZJz0#ShrjC-|VgKT5EFk%j_54(x)9q{SaLCO!`II9i}e1p2X34gA-K=
z7j&_lda|wWMn9u{Ka2<X5`8)MNpbVM=}W)%{!x8n09t6x$K-a*S$Md8|4sw~ZfJDI
zZ!{Im+D01w&BDLA4dCU@&A5@=Hgoo6s9$Uk4p%zmIvu{uR=<pQ=Br=k{Z>v_Hu4)B
z-2H&)@ctY-y?;R3KV$gl!fSLv1>7IwQJjsFA13qXnti-p30TvMbhQ_6Yp3)&?j;-l
zrH1eD=<xdM4q;fI*M|=-+pjYplr29f`<Wgz2#3?&D0`brB#4FU?xUY)$-LQGUsy7e
z!%OeDH(R);H64Wa<y-!x#_sIxb}(Pp6O;I>YtshdpZkq}UIFvYgz7me+?sjKy$ToJ
zS5>&n&aP9g`jh;81&fDVO25(`tMH<UV9A7cDqdHs>Kj|tc$|+tXjS^S6%A+jDqs0(
zP@0OBuZqhnP2oL%Hk~Q7zqH$npIKx2dd1w&R{C(M!*H?TxOgtz_AfJXuOrLcz9?ji
zhd(dbr?VcCFFqvySsv2w@bzzUsT2LQHDNpay8GnKm&jXQ*q2g{R()x2v`jDN`o3J_
zzs%5`qZ_U3$AWKS&BA;=|K2~zOTQJ7@{R=Pd5Y1xJrVUHd(#!Q=+a)>b&AnK?X;=r
zLA%C;xf@Eqolkz;vmGyy7$5JQ$85ZnOaXF0r82atD6W8V)9yUmxSAx#w|_5>-n;$#
z-@a6WVYnFoV06QK`hjAHwXwj>vd1sSdtau+>(8c7R_c(wKpnCdU}GzNkVk(hPrY%&
z$`HwgefPJLp4qtXz(oJ-yI@SdN*8i=tA9US)9$OEgP8ilQ)Q`KKErFWz`oPBZ>d}c
zZ`UCXoX=;Cv{Jp;jUh>4$d#H-mBnY>a<nNAq|+aTxsL&KZuNtv>w2IN>*MaTpI%H3
zhf$HgMlWB(3`qWaf#+rsO=T=MgWA82P`VjC8Pei@GBaTz?C{eYE7s@HH@{5Nfw(cb
z`ni>81dVTl{RjD?r#nT=1XR_{Yx3u{<Wsq3Nq7eYseZ6OetPnC^7LeKus;Lxm`LN}
z<Y4kGjFs~vQ7PxrZ*9p6_uSxj5sPbCqKyIgb?AMKj8diJ87~&!DS4!9bL6NWAB^_K
z!^`|I*1)Tca{@UX@z&uod6n78-rjh;AAI9sJ|~A~`N4i#I9Y~dj`f24qVxGC0}wf^
ztnbEEW$V&VvxhyE^D?*8N10owXT0)~E_N6*7uUbG<B@zlW@k_AeD15C%W<xoBy8#H
z7izk{^marfhI?;aIg<zbSR~mJ5w^ecb;jWEoSq&I-8;0_U!A5StmWW$F)8l~NuvdG
zzN_QRpwPwqAR*C6W%_al&hh-{TMUk$@DeT>mYw+m=Y*fuyJmcEof4ah2+TJxO_w>~
z!+rYB0xsHr;jzFANkB^ttiju?wy(bX)sJ-sRzK!lm&rU=uZ{Dv`9AF^{Jh2dpV{+M
zo^cNULz?ByD*tEmp8xY*{QUMeH=j-pZ_b{JV0e~%nOuGOyMI_3rAq%-S4G@heVJWN
zzDjP1(lS20lAx~~P3J*F-M#zZ{-?LEesgm8?H}~`xO(N;<o(|IXZgA0{+~_vZeI8q
z?1!eQPTl|fzO?_drpW%K`WfGQ`1xPH|KDK#AAS1y<I|Hq-Ot(bze<Z$`JdO-d;Z6F
z@nh+okDpIw=<kGMrjzf-^2hY(MZydx*^S~x!9^R-=kwz~+`Re24?o;^F`7TWG25Hm
zcs`jP&7MzQjIN*D813Eo=J@9S_u{&Id-L<hx<GC!pWcn<^A`tVeEjfyviCgsVKk$?
z>6j{YVwWlKmcvEVwCRgka`a69cy=TUQ!V0y$=>*IHvWU$#f?dF_vrY?>E!F@^JHUh
zGs$InZOMO{<nxpH?BpnUG@8thwvx|B-$Ix3`A9x|!Ed9({p8PmhpnY2hw{49*<}9X
zjpWY3L87hB078$a-;Xt8)GmBD-k;3o)5+5lt7axmoy_nxi#2~T6>FVOo|1a{BZ;vu
zW?SaL5aXABk51;+&Woe{$+L;2v2LmN;#gjKGRHtYo*sQa*;g93(R^tP_KptsCzks<
zvdmX$Ydnu*!q2s(Zt7Tu{_P#@kCPX&+{r#NB17_HMFp3y!f?ix5==T7c1Q|jZ5&Jh
zP(NGf#o_)^7v;%f$xL31r#H^(l04dtQ{N?d_x%%`9(}KTUxp68ey@Bpv4cr?|NW!A
z6G)Lq{iNQM`8v`Hbh3zLFNj7nOp9+y=T=*A@3!#T_s5e^$!?sGSk*7<VF9|eC$AMR
zWYYYjCbAwirJAQ4O=r?G&22tm?~vPkxG$fMS!vQ!S?J@$26H}JczNB~`?j-n_-04X
z=0C6k!vala$KyQ~r`$hb-A-AShrSVAtFu{nSNrDipLZT5j~;yb_^)>!ZYT2l7Y`r&
zW#^OaPm+)SCSPtRcOQK5w}(5Q{rPe7=Lh#b*?#yaxpV&$`SkwdhdUoXdHmqvBWvjD
zokw!}RsHtP{l6vK|MA7c?MIK22M?2-&%e00BQ44!9^SeCcxU_3R<d*d?!70U?A-rs
zE0I=``wt#lGxv5r-+3(8K7O#Jk6*nvdGKlS`S!!Rf0jS*e7tjS=kee4p`Y$NzRv?c
zeelqlx|4iy=i%d>yHD=jd6<0h<lz?&9&IPQ<tICj?%unz^ZE8CH>4l(>}2~d5PUuQ
zvna)jFKsP7`0M@chrEe@?d0Qa>Fu45?``u)edkYh9&X=#%*(~!?#f_FNAGP}Q;)vb
zzPlrT$@px`tKE6{x2@3dqwW9uM6Q%ClTYq^e&;iJ(~XOal1$j$Cl9wj=M7|N9zFT^
z(c_)RPabb4pFMc+iB7<y?T3Haxl0k&dk-Gz06uxN9Xj;MoyT{y3(}4Zp<E-sfBfXp
zjt=I|{m0u6A3piw@y>($o3aA_DkCMmy(9O3qND!cK5yQyrR@g~|3>qSi_Xke^4C9a
z%jXXnXdS&f49g=K#k-IFW%5uN)bM)oy_5UfpWWN}Z2SJ*Z7z90Q-9rgw7n@4x$}q%
zcC-VsKIFkqbOe|{>0g+-PZyW?md<su^J#MDlfUfn7U2$A9glXxa?)YE`{ytZMm%uy
z+O>ox|D-2vV)~PQ@N_yEKT96%eEx~-;)%#3*{7e5M7Vt}@?m^<I8Gk_AaZ6r-QcFp
zu=V~lI+1-lT{!iUjdk*6bK&ykWAW5mxXT^2ci-fwz2HIn7$G?RA^G^xC&@iK_Tj`E
z9UmV|Ox!JquB~J?9?K%QyDf|V#{A`62f7c5Fj%{HcdwjBa<TM1w_S(n;>FA;u*EXA
z7WX#>mcx&CKEt23b4M2ShaV=t=`V<%*X2+cBk`XmSx<rJ8{Ty-?(QzOxw|`hu`kWN
zl?VO#&R@27@7#O5^Ox<rZ>Plv_wW7f?A|5G;;Vjk|H)l>N#%h2tra@Edi@h?#}opX
zm+dr?S=Yn%`X|24tF#%qVuh{j?!JFFuU)^?k2IU*?e(XVxoyY&(R`H5e>@&5+sU1S
zxvb(nkv@;Ma}HcF9uD_M)BWVG+<3VsvR+S{9u*mVf3z#2Y{<b5>4)5Ra){RQSGi#K
zaq<rxP3fKJ2%fn_j`dsm6~TZa-<>BXnq&Uz)@tkeqS7$`GNua;_<?`sx2{0wx!hsK
z;}X5y-F+c{PLkQ=?RRlDzH&~7?2pABU7`nz&A2Y;LhPS1d-h_!I}1;tP1L=<-GLN`
zm@@P_Q&I8c5vplwgL?m$_rJO|+;2qXdq*eSYl3l{6#2RBu_7k>FJnU^jmZ2V;l5dt
zLPm=8R+7?*>8wlb$z=q5ejLxQ1Z8(}DEjy4VDeLlRSpk|pX*>=cwb5jpYDiW^x3@!
zAB#Hk7dflZlSETo>de;CK+tvGC%N#3l<r=!aS)l0GH!%vVJDf%x;LKA&1^FZMfBjw
z(LO4D{$Kz2Tolz;o>{10L)M-wBnm@|CMk+Rww2f~dH+k%|5C0WA9;^QkMD>Qntb}?
z{#~@FM{8b9o<1B+JNNI2>a}~fpK2u}nk&`=Nwe{rwX#O#_tJN|tvA=StP#ek9PZEN
zvOG7!8oee)#MZE44efCa7=%h8E|xBw(%8YJ8Vj53ls3N}AMQ>mcSF*cZa`^bZ~pT1
zUc@`tGX9>ez9-l8tI!R|5Y{y|EP-byhkMz^@aOtRFI=53UhPmOVb$%%#VDe0>An0>
zvTz13oX+9IK5Z0cKa7qyPJi8AvYmcxao(@0R3kY(Roy!6{8)e6Y&OpAHhZk>)#$U5
z0aMd`HhFP!pt3Hh%OC?03$jB+AAc;8#6}`94l=Zw{*xD@z30*s)6oN&bx<bG-%>%K
zhY0E<`7a7}y$a{gYu|3S1_kDfx15=B(jOnDpX;6=a~}Z}6_XpiTI~;;uSHUREvn2R
zW5NH=>~>52->{~xCGBP+cg${_)f1k!IK=c#Z~2od(Fv{vu9AO`Pv+(aac#LPgI8wp
zxII#3(0b>twLa>*>TlnV4o=4X99}}moj3DmiM+Fflv6C~QprHKlKH__awPvA9f-)+
z`1Et|aU1<bo0G%qe>{O>hjMN?c(9c(i?25A44)sYyEi@S-i&+o<|7^~Pp2+ioONsX
zh=!&0Q@;Cey-^;rAxiu8WaWyJdFY4qt`}*W8%Nq+kuP4AEndaAxNr02HlD9*qgZO=
z`B@wB7nsfBtQpDAp3Us6d((F}o7shCGd*)Qiwn(WzJ4}eTQ(kypS>52UNoLB<W5n=
z{`rL*tNXNVvG%*?x-D$a{Id$VcRQQ!P{{pkv-#@~a%ZZ$#LLDr;Oe@~r{leoDNz{>
zsfqOQGG-d2i~DFK|JO?r`DEV5rq=ll+-*-UciZ0OZrgvo+kRHPZ~tg_7~^3dO`S~`
ze|;K%-LpEd{h#~&#tI(<A58e`$FRKPe*I94PHFhjy?)h1edWU+HA&&o!{Mu8NshvA
z|CPMF5q{a&cp5&GHTdXZCZmyj{^;Ljqy1O%qg{061G#KJ%G5{qGT(AGw4C)VXRhUJ
zspYJ1IrA-N>sro-mh-;l+_ju9wOsZs=f359UCYJLa?!V3xR#5hmW#gS!na(wmd_Ei
z|0CB(_Oq>#{AXJu#m}}z$_uTL1@V13p|n=$1IV16DgrG^ay%QK>>uI6I(pI5t&2uG
zGw;Uh`pLptRA0ryLcBK3w&Xu~SdUXs=^x4+*Z*jKYB^l<;H#eItKVheyOpAe^4%8?
z9%V73YhhElRw$Hs#-GVY+KzlWwJg%9vJ_}P>GcQ8y_$p`y^8$`U3y!6kypEanf|;w
zMFohf;bb<RUXRhT3$6yQ4(J`D!{NBK56rx<v<_?^-&%5b^5Q8iC5bmY$Y!u_IIV}%
zd(DlZby<32N($^>aDiUjE$i3rR`SiSVfktoUu)@}vQGMjUU$u&-huGV%38f#2Xq}=
zzfeE=2ip_hT0g$AS@}jflLN*=+oz^(@=fSaeE)Cs4A6$YAxkRs{j7J>_u3lT`q^J@
zeQ&*tSm%>J*Z;rp*`f#kGLeDR(;`MPwg%m0dD9HzsYChFy`5?u^720}e#XxHVBwm@
z@3W=v^QG^LrSDa5dAazR>IFZp4DWfi*eHIg=)B;x8KR)0-l-$fln-~G{<ynr_Pq2#
zToKm#RR|EQe6t#JTnRWWh8Q+pqTReit;rkFK(0CSGpp;e%Bv^$Zk>AQ{QBNCzvR1C
zb)B|FS7v9u?)hO~->-|s==?4@l*7;H1Xho#;kAA^ntr?V(Y0e*ES8R@wBIpn_gQJ1
zj1+X9`d_z`(8!k|Ahr3GD7hhIJy|~CvYJ-@XpQ~xkMe<U$$pV>p7d|R-yhy47{=c>
zuZ5ucgMS~;0a`~#-=D_V`!;P}><;$<UnH{c(bIiPr@SaFGIWXd;)CT=p~&M|(D!FU
zuXls635`JDYgipEd2ySY4~DLPV13tr!o&3+u@5rMgmpUInn3>-Pd>hPexU!9ok<UZ
zd9W`#G>#Jh<-@NBXSe-HJ-#38a_OP*%x~$R1LlxCuS01`VV;b>m<Ne&-0>U@uJ=|4
z9qwpqI-IY8L8j_t-=1g*r;A&3=aWZ|AD$&{;!C{{A@brR=$P_o(@|CQFt>ky%Izqt
zCS{78JnzuZ4KG;EY3Ph5ZjwB!*|*Y$U8C8oa^aeHyDv-Z=XkTaeB(D;Y2vgulMC0q
zSzYRziL&9LcWYm?SMUv*-@kSc?rRK#8&?9HXt!7T!PzYwAOk{hZ)B#)_!$Hc3z3r>
zYmSmNPt#W)#A^RUna?#6d$??R32AHz_8a=O14RX21j5I@+Lz<%HcLl^&VpfX0FMub
z>xOf~qzVT~FQfWvKKiJ)Vr(F$VL>LJg;|kPJd8&l`jTV(%k;*LgUO4@eDlUB!?UaW
zF^hKCaGBk@GH8&vXkte{BAR+4FY)qO_<154@rM8N_o1y|yFCXsy)m2>ng~BkSQs5f
zlJEZ2+8@L_pZ)%}wHL(n)9HBhtrN<AoT$%pw$`_PT?g#R&%CaY;=Uu=TwV4hR>9`Q
zRz}Z&#l?|QX?}8eFrLx%C4Eh?)?bqyG=3=-4R(3rchc$aH|!*h7{yL4@1)lQC@n4_
zwUl0O^Tp!xL*Lx?)xWI;8NLvTu@5)bw`Nw>`8#^yTiL9?7c*2#28PxDraS)ihf%G|
zWdY%x9LwK93l_eQAKgy!1&QL~bkfKWzME!#=m*hMm$CJmgNx<#)~CaeUruZ-t;N&C
zmR`0ZwrsJSCbs_l|2}cDs{h$Ugq`-)g~gfdnx8|Qz2<qnu*ny#v01S8xP8g-_~lm^
ze4;n1{JCKX9fL=3q0PQAm5ejjPylL-reAAvUeHYq0F6L$zlqz+$Pv&EQe^79H06|L
zhU&-itY=3j(;JC)AiNasr)~Qeo=`Jc8h$n+!G=cW&qs$#sf~lo_HvkOKMg`WiqNgs
zlk#dI_ktd8>lEb~O?w#(a%0&zm0p^tSBX9wWIcDU?ex}=>9^SB)7-(qlT4}AvsVL2
zgyts>2LDpWbrOz`CLXH3I2eC_np~Cskh5@Ala0xQJos@d&iBIHX7y(8a8R_5rWPG3
z;o9Mz7(jDNFn$&s&HB9k{V8W2dhs&gM|!w(xC9QOg`wHG>pw^2RBoP*ravx?oGRY)
zg%!U}_Z-Geec;?5-QbBevi`M8vx2|YBH~k9sItDs-<^yObmbl;8@MGzxXK$M!0~#N
zLoW|W4wa#PK?alzcB1BmyxDB>a(DQU7wl`M)1;UMNthemV@mmSBl&!^mppj%AIbjF
zct)Y#=cDiS<!4bjnQbLcWwr4{&A8$n<QAKpXaFs*40~31Q1Vo6{ML*V^{^}$XzNwb
z(_E9UcV^$VV|n3Aot)n%Wq+72fAz=y9{px1ZlRqV+^3tNlX~CHq}*IO4R7}r<e5m2
zQ{?pkg8tRX>mtk7%j@|MNAJb;&myju7sH#2>;IXudYyH1X<40LN>=CpLq+wXA?VcZ
zpIubvXNl^%yo{*MPZ8CZu*x)`cFJ?|%|-KPjr5Nra~?2c6f_YB10Z*Le|cAokxNSo
zUXWJyi2kOVSNaKE5TG#}!-f594s5mh&T%2Qz{e?ey|b4W*Z#(b$6I~bVH>~A;ct7w
zrgPu-p35iq?s=z~bOE!!@6l%H;RPIei{XHDlYk*_yc{<P5qZK5eN<vKk|5W$SzZ=J
zUKLO7AySvF0d%kl=7=92@L#800?IVq+%%i;qkdRltzEDB9_!*&W_`w6;{LN}iZ?EG
z7T;ujszb`wXn%%9y?GG7%S&*!fQ^?0`0e;dcTlaiur)L>8rE>%fZV%#Fh2Y`aJqzF
zrH#JjH4DC15aNQr)?aPXEH?TYyI?E%X3KI}m-fRnUUV?}I{xuuvLF9YxgnE_(*IAk
zpXg98mg~IGhmgiA>n#KkUnMw{7B6hwb<L@l6`i}RTDWZZxbUwbgQ~A#P!PFb6R971
z8d!FR7lFvPeL9TQ@cE{5?z;sh2YvF#!QZD-EzerKV7ZI^Rl}~0E`N*nuS%o;T9Q!m
zN-98-#Kgzqx6-i5kIh?aq?_IQ(d~W_PWe6d_>^1VPM4AMH?9+!(=e?1j@Pd*%TL)5
z7PQ?UYO5C+ZjwLU7K_o`(?(^oW>hu+<M4W-C@>`|VpoSpFYCvszZC!6d9B&2f1m@_
zO9vo4Dwt2XJB-_M8|zLDK4+Bl*1l(Ja7)9`{S%DRyBimW2DtCx0R7P9Ylh~diw(`%
z%PuoC|J4u8_177i|GMPR{QO!%FbfVRQDIyYYK`hWCk}^9ucohgQizE>n~uk!OYgaz
zFZ}Zc{GU(fdgu8Rpy%>G*ZDI4XHjM4d;ZUN@N*tDmTlN;SHffu<xSMXH?Rzm3A;!o
zAFkg`H|$cq;{DeA)&0Er{@+c{lK|)K|8&j%Z;Jf={(lEQ=kEW0mJd0wMP+}#fB#4Q
zyoLRr{XF~sef@`b@$;JdKYPFb|7ZQY!Twhc_&4YO=gXD)4@H{4KmXswPfwNm`JKmq
zz6e$BFWukwKX3QvP4@qH7ux;j@Bbz%R_cFL+57YVUHqJT{#${NM+f8av3gBM$H&v7
zmlLWZ9Q>FZ&XU8?;n8eN2?mnE?;j-xM?Z|G$+g3y>9yqK_*i~_dUT?MvdN5Q6#OuG
zp)|5zpNyuX!}$<W)?%tJzT4GCxp-snF&qXDL+}mClmGR0lF3kC8~(FtA$TCJyMNue
zU*uOl(2jHQ%_Ll5u;dR&=?c`mbOtb!{&I8zd9PKH<r&M5u-d)h${X|Vcjev93Ay<3
zp?!Wh)6%>{BxlX7*(tZCp{VZ%aWFm%S8eq^(GOllgutcd|4sJ){N=?2z&ZOruZnVI
z|2J9oe*eFNpL6$rj~U;+`|$DZqc3*u-+l1p{^NmSfAQm_lvf|{ids%uR>>~@`VD@W
zC)ea3-N#G&R9RAX2EYE@sr<G#I(Rfz9KoH#{YS@>!@GpAHa>ho3;?9j)>5y4J;Y*`
z(-R}Bg=}_Xg}?v7qunR>KfSZ_a995NckUdTmA$w4?e5O~oyTW2MGuynNC#1KJbykN
zkM?(8oXp2BM|0}2Cj4_N6r*1KUI*+$ehne_)%RM{>TEoRliY6HO1^mf=ZD*OKH2^J
z$>Z(+*!^Vt(>qV@J?@hV7VZwubK~ltZP~9YPurIjGCle+bo17l9==-F$>rO^Vi|7M
z8iJcctVq`n@uIWW&xL3C+&Nq~XR*`65G_EnhK};hISYLo1kBR%UT*iIEB0u%dvf?}
zG@0sy?dcb6s$a44n{2Fma{0Y=cdrh`x`(Wc!^z<VM&jOh^!@Tie@i0~o`31Fc(VTf
z)zMi0pf@>?-|^2Iod0@6yaoHeN}DzIe^tLf|KGvSx#xe+xe?Ux53JC14@gP}f_f|M
zEWe-3cIWP**_Zws{Yc9pq*_ZneRWm?_7{wv?z>|#0EU}9*MQxlrPgjISFg(HI_&XJ
zA3Xg0&g0!L9`4+K{OLxvmE@ZytgPX0C&wGUB}g<!qGaE=aYIXZe?QvQQgfqYj%@jh
zH8bBBQOsN~<+KX}_x;}SbaFU<wt<FjdU}7=mNdYX(twDF4f`Y97H+!w_d_j8fBNgw
zSv?&u_;OU^{-=34I;$_YW_qtIjDj&rEUX#dl9`ILA13p?=ZU|}KzF0rcx6rfp@*tm
z<*%!^)?CKe$IJL@?Pa`0yo|rNOb(vW$-(>&B!TS85wiDOP9am1`lIt$x1DKY{?%F2
z_C|4Eee}_pBWb&2VN3AOt>tH_Y85NNpR#Yi(x)xW=I>?5em|2r-AabXY$a#Ul0g90
zZu#NuUmPE7U^+mk86L@qgr-HGkYD4X)iu1h4m91>)ZE1;OqIQP4T#n?YL<<Y8MT0x
za^R2l;PMJZ5a&n9^yKi61Rb6-yJ0s!=66k`(KM*hLGpCGH=-iw{Q3AtEzxM@K?C0T
z*=TQaAWs?3?Eax~8KSSh9xEfuQa22#JQ3MEe)de%Bg#VRy?w%vG;UcK$0tt@WH|49
zv17fv+50c7GCMLs)cD(jGQT{NJBtt2f?T>&old{BzBhw&;?<Qtd3nw13+Tgn58~q2
z+sxf)`n3v}envN~Uvu3L+I7FO|HzDPhTbeb>6Z0i>3yzWzn<voJh}P%ldZMC!mGp5
zH9+h+qJ`Jl^p~zV*|_GbE%)`snnsA6*EH8$qG_F`bDHLwg{E<#4U^8W>waCgQ8(On
z?9Y-l`*7)eId!cp96WmF*ri@7U|7nR>yDZWc6#@4K_L7B@1XYW#b07URxcmr$5Sz8
zUT!TMlP|`z+34$VT!dF$cWR6H4#K-{rN8q+C-g6^x4!>*J3nt?|2^M-EB@cCSdIVY
zb@d+q`7VAg;{SC{8?!5(@9*wuTI&YCB0it*Z-wlp4UtMmdz%~f*Jl5*ocJnT@AY2E
zN00A3e7yVc!IQ^3_qUVj(aD@*qqmcd^;c}tsz@dtC0E_m$3IqAOb}~gO@>w|tNyiq
z&__woDD2LFCchHwSQV*Px3vQ4<Z!XsO*2<|A<V0Ob2K>&@9bXrnyM%N2rB1aN0a%d
zN7F}g93PAyJpD#Aw++2`=l-Yok+)S7`|F*Z$GZ<c{y*DyAMd8*v(SbrL7)8f!NX6I
z@yiJ(^8Ofw=(F*BTmHT)e}=wkgECZl@n-VjaJec_*X-Eo=Rh;mj&VD0`exPcFt65|
zPV1siIX2QCHr%VT65ZMBhJUGnOH4rDLVw*>(x0TkEB8z9w_n@Oo5=ski}Ca#C%}2}
zzqF}V{C|1=KK}PEelCRnzoJtebX$JkA0Ny|Dk1p7ww|b<@<#|I>F3w~ILF0@ND5z`
z9PhKS<JW$Bhi4oepZAEpqvIcA8)Dopw0GLGtUAL$3l^RyM*3dRsn<WXZ*HFGcej?i
zp`W@gWUdRdlc)3PNaX*@RcSB|RM`(#AI`#cn-Q$AI5^6QvN$@@=)&Jmw}Kt;^yHbP
zq{^rA`@#57-zL^{O401&7>wvq{5D(s_UXd6&|Jvhq4i0?Duh43*-8!>rJL81zdk=Y
z7%Lgu2ZwS|SX#5gV2*q!{ZgKE>)-VWR>{>*KYx5ums@o3W@5uRA=mFPoZ}NRdiytW
z%)4~z6t3Sx%k7h^^`dLwMnV@q8ShP`U$5868&`d=w4G0xXC+Uep}_4%=6|@&Xx`!w
zP8AYE8s0}4`apk~rnYX#yBjx-{F8-oNPaKBUa{<d8G&kZGx@h9Oa5Ruk&AUb-0nu%
zJt{sW`N;Zb-k5&y7Y0wYhhJKFz5jXRpUa;AbNG2~UhEm1hmD-^r=0)#`?CF4*44jM
zKkuvC&;RoC|IOzA#bj@K^mKf<cP0@09R07($`$=jeo5c!fA8SuT>Wo-ArMtac1e<7
zqpUf0r^6pc(-$Yl{*%dEu84Ff8`o&X49m?$6=lNNy3Fn$TI`D%e4u?$pXHK{s+gpe
zyLtNQ_Xgc_>NKytqZh}cslDrL<J=cZmg9E2qm`H49c`|;?&;RTHBW6Y{ODRQgT^=A
zeM;<+`&BL6<*=_QZ8EUTh&eoH1tAF3qrV16h{au^E&k+i{PK9bN44Aj#?6GUPNw6l
z4&@aBFL4=sIP`89nlK{am2a)N`spcGKUHVAS9)Xf@z&wJ2kw>Olg%~J=U(Z#n((Ai
zAU*s2?Dp`*iTr)_N>GuOKGz#=XMZrKxm^^x9H!ap%YAn@!?#b@efzYJ&ctr>+WTSL
zk8Q$YxYXcP_EO4r(8q02K%wkot%~9~Rj$FKV|fY$HlrU#KN>rN-Mf)(ZvIBq)UZ;9
zMY54yVu75oK0eT=9-sCSFPGk8xjR40nqBDA<LyU}cR$|#Z0Ej3f_j+4Ec|A;wGS3$
zK<41!TNrBo&#QK0ze3z>4d1=Ix|R3=(5`4tw(ozkcCz~waq39D^@+RC8l1iRe$8CR
zvY~Iw)5xeYiTjtGL>d@o$!0Hzh5e-|9G<Z_iOW3-SJA22rF(VSv-`fyRxX&W@7<cY
zd_in)-ELewM&9~_hR2`GV6fDZBf2-wksQ0TvjoV`++}4J^ZSD&k<C*Z>@Qtx@k}Ws
zbHPG+GZEy+bV(fblP%ZIR+h|bNv6YJcperJE@#fe;cL&sEN<Gv%g)6xz6%rJ2U#vU
zT%Lr((<ec685#V^p#yDsjLwF`d#s#h9t^T1BmGxvgq8n#ad|FaQC0`+J>|=<`{$DO
zznX`$=Qk#Y7kI`w_J5gG%_{!8c_06I7eBxK&CREi!<)0`SFR|I^-rUxlRqVs!z;+D
zz2`^9QcNYd={))5Z}&dg{q)YgdmrDq`|rE=c0PW1=i%RWzqs@G&$qAs>FDV<jM1MC
zCQq;GO}F4>6XARG?AbyqcKz@E!N0HmaLRqxHv8Ylv(es_E59|k<|rZRVqELe*}*w+
z_lt+ypYHs}pZY%^-2XHIlojey9vQ>=Y;QU_o{P13B<)S+Y9>7$Q`C}p^Zvn)SmhLZ
zJmk+%LK(7aS?vkgRg=T7XE&~d_i)eJm0xfF?jL9N=aoG3Zfri#;qFhqpX{HE4)huS
z&&)Tf^x;bn=^x(9cIof44P(ud<M7Mm>dW8#!$0#X@%F1PuO`<=%iqyl{Qj%t7Ck!j
z6Lse6FoXU{TWcDZ#)cX|muYi-lg}m!T#&(>PRFxj5|4F;J#O#D6=q+N4!6@Q04VME
z><Wj!{hnXxFFHA#$E)m*a9Q{Rs4ffXcYnH->>u@m{kwR3a{X|eq{;Q8<ae<d_hD>K
z0T4^WE&I&g%YIL;-WCPvcmJ5b5=H6u)&6nU$H~<{&8B<0TliK4@Y2NnHhFY(GTpPq
zHG4ijI0$QA7Ir^aiext2u&!%)tx+;Nd1{x-nhPz30${_1S{4Ald5SE|R-Yhk$`<hp
zHk5z}?M?KaP%JiFeD((3m_I)W&(lsnABU#mo?Pm{<m<z}GQ(=q$^Li_B=&~3^1BV$
zhUmv|)ZhM-`{Pf^->(=;Zuszcdh~3#f<IQSnC}7SAFi=K<5f43<#{8_H?*;`qy4ws
z@F$szKOve$>k94IU%l+y^gB)7Y(+rbKKFsI`#4>#vZPK93{Z6YcmF0I=o#|6f4I+I
z*<nJ<!<Kxt8J_0$-|sdy;$I(SoBbuXZiTytO+4J~Hmuwo|5|tVu<eJt-42(#<6o<H
zU+e$2%?b2Bc|MvRQlY}I@=5l`5A$Jk<z&g25Du>Wqr>r)EAylIsDIf1E5kpHqah!?
zN<ND3H~bxY^y>e;q9VdQes(gMlf3Za#c2BD9}>%7S!na7%w&A(tF6S6R~DYLa*buK
zEIfPZnyX@gsO<CCh-mt5QK;!4q*ub43_~JI=9gA?-v2E9TvGo(Ih@Qd82!Cm{5Q+9
z_wnC%@bf0~|LD`tAHOmFFR#SbTjBpH()aP-ck*-773~1Cv_GCsz8}jU)1wy&6P#o>
ziW`NNd3rvdAOGRz%^!aF;f8X*&GsfYo=>Jnv*(i+qw6O(Mte8DIllS-y|^yl-u(Qr
zu8*5y;*4M3cs_q|u-TUc`(ZS*vO*LD6J_v3E|~^BbNXV|TX(UX*TH0Od^j8bA(W~~
zMD;)ZaXR_>`8?U!+e~s<UR&~?Ci(niK07%|9*rjRqb)4PBT))JAIXO=_zi>V&wYoj
zr6-3IEgUuf@kVk-bV+S>hGH|GejlsjB+0|^{)8l}rze)2Crw#`pIDQsa`TCp(sCYa
zM(oV0cTu(}RPnNQd<FY0O&}C)_Qhn5!6~|>Sc9m?&qwp6F@RucqA^Qu3T=(&aZFTG
zS?Z>aW#}KukCm#Lq6b?EEvsg_*AHiGDIrZ&hJDK{XU%?+wtO!R_m{dTPu@EiDJuA^
zF3F?aIQ3nUci%q=mW%ev_hsnd>-Wkx6Fb-!B-=mQ(~@<geo}ABd>zR*(*%aA>11>;
z8>YoKrE{w-xOZE4?fc`2wxcD+$iNNDdRTyN?a6D!3z;<ksEMpcc|&{3(R8NCrBBC$
zAvHeSmruv6H0kLJ>W0~1&SwiRuRD9+cDA`gj-JhbU<LMN=0Ziky^x*y!<1!t=o`_s
zI-9v7bYd0$dFN5`=)tFt|9a=)b|Sxj@$kW4c0SquB>DJn^5u4N_rVu`d${x2pC2cG
zesJ%T?T3$&JNG}4Pwzi|xbyLo#}6JpvWBkSc_g=A)o<_I|68*CA7Z&ZdXzkPnCyK1
z#l0PAQ6BN|&i%(b+mFbnxO?x(Cp-5)+e)OB<o<)l*37+~&vzcnwT~Zc>El=LO&)xj
ze7^ng?w{q)J0I`d+j;yqedwn<kMHxqPaizArtTzP+<EwT=kAkxcOE8RJbC!VgGbv5
zZ~4j2qr3O+?0mlc$qng;JUiL`%l7@p$)i7u&9(T_*3yH&-rs)6oA}pGKHiqz-ud|6
zHjmVI{$%Ij_T9(4T>R~>45oDS-j+4>=!@;UJMx!|&$hhUorizh3JpKn{=ZM;O8GMR
z<j&`JK9e`yxX38Ugx!7eaQk!KK!)bglaC)g-g*4w@pkgrg9o4J1U%Y)_?Ml#+mCJ~
z_Z~db0etdkJ9OxiJCE;Z7o;5-Lb*nM|M<zH9UaV_`;WIDK78`U<DCchH)RF<RYppB
zdq?j7L`VI>ecrrZOWO|~{*C4t7oC}{<gb6;md_tD&^mf|7?wveigzFT%jBUlsNwbE
zdnfm|KfAZ{+4lXr+g$R1rvAF~XnRv8a_12j>}UsMeaM5K=m;=@(!VfupDr%(EuHIR
z=hNiQCx6-DEy5kLIv(wW<)p)S_s?M-jNI=0T&qB;^|6vpJ=*!tZN-77Yrd$<WW!xX
zO}gPK<-*#{^xmZp^Q4i_7U>m+5>X3v+;-)oK6dt{GIm_~peW0fLS<e1WGl(4y6n0p
zZ94rU6BVy3(p>(?`L}800=_A$yezt|OrLB?3r$z&bz4@2e3g`Wk>zEc(p;XEMVqx%
zOLIjl8fPZ==r>*4q(z;k1uf)tQ+7>TXQ@86kjC;VE4fY{nif@6H2hH&rRcL&&4aSK
ztt;tKUeQ8QmE4whxpuGZ(yS=*j6aGdmHYCFZ|Gc_<<em~BrnjWMVfWAkTrGL)?Hdw
zyhWRrl{_i6KkB?{vo57~d0C4-UFKz{-()#o(pBk0QFTSu){TxsmepO=bcH^tkPq9g
zZgeDzyvdrjsCd7u?RZq4+DPVUU8PN{!=D#T+vQbS=-Z}cDx)c1={V$NBg3V=E7Dd*
zOGZ;$$Vz#SS_YiQCS6??c`3JXQ!eA*HH{5_S;>@jd8cntWJM`6(bPH@@?F}|M}2Ho
zHF;O&+CtrxrSvc3n?eVo%36JkuFHAV!seo^<$b%N)K`}|m(Ew6-UI}+E;H+0D|SJd
z<%O-NuB&9kZ7zyRdMiUyW}$_uka^QJ*;Qqm*L6mRWbpG;mT#_osN`LnE-UmcGMU!4
zma(a{<8>!XHmx#U0*qs0!_TCovXbmUg}jBdR_Ea@<P~L++3?HgH1hh|2Rc{E!m%;V
zb6Lz;XC0Dfq+MI-s?J&&x}r^MZCng4F@<Uy{zgVr+SWg^wv{y`U+{-KUY4Z1xUDD|
zeOXhs9x{0~S=O?u^tz%gnnt=u+cHO0*4Rk0#G6!>TC1}uySd0@z;!ldNM(iAx)loP
zv<z8O>j=r1W_8`9x^$CP_Ia0=mF<gMwosSq+AT0ASqHk!<kLdtLnTLE$r#ImEjnAR
zjqFBweeHv+u3Q#CVdGGixkwuwJTVE`CXLDmu}1T@t;$*_CY77Yx?vT{dXTQQZ7M^d
z9hZHUx20`ISsn64uJS?Npe|$$WtDBzv=X6|+DKMe(Ml^#s&A2{velciFd5araA*=n
z-cI&knrjOkn@MD2*qa#{%=(widMnF6m$%aWBfFx*=+)Uw<*qK%omI%nkTEVxT@Q&Y
zetB@Fi%nit?r74SkxVEPnWdEtD^jSaOP!dklIxH#CMUaA)`n~polTiadA&l%OV$Gm
zytS<^t6Sxo&P6R_C6cw)k(5q$Y%c9XqLQ$xbd!iQlo@DS-S>&iR?)ULtVzY>i793~
zOGI)fz16u8VJ^Z@mXwJl5s|q($;6FF7&%#Vt7nbuq>kO7Er@KE21Nwwnw00up`)@|
zgia+}vngzhGZ7Xt8agN<su66BjYA=?(B`e}VVNUY;h7#&BFSXoRCQ~MNaUq_rSmM?
zwV|h#tteUDa+b;8x_?^4ab-7^GF*9KyIa<~oNbw%IHmks%LXVz3v#f?3s<dP*NVMb
zv}GC2i!zh8Z9_A#vi3}*%QBN^A-$x9ROYFZ!%5{xBU`&-q3fC~MC2jZb;oc9i+D`!
zJZw3p+SFE5DH2mwH8&Nte3`YmuH8h0p){Blx`;$>Rcs$!)pBI!?2{nnWY@`IR<^c(
z@>cGvv^f#&X(f}R`(7R^bKZ1efpYlfmF_I0yG&r#gsmV;S=OFz$D&eknA`A^{!xdG
z%3H{yus(2jNvG@uW!I&$iYpt|S`K?TRdlN-m3$%Nf>04DS2NK?m)e;n7u8*tnPQ+i
zOV{YIR&rv9<kYn$s)Tej%k2P@g<H$^GC5f=D>7|CxXOtx`b27SM0Sa^rczEebjP<Q
zqvU~|EWNr8QchN=EZx#h9J#DCDJM@7O-DqQZgml2vLHo{=r@(Fq0(ANn~Gi61;s~3
zA8Da)+cu&kh-}h7<P=berBf@XkL(ItQ6eW~3)Nu}$rWW=nL-2jLbQxbZ({S5BGz?H
zN^j&Vxk-1HC~k6ki=@;(pntS7rTU~sRL{CCbS`AvnWIjHfSfx8>R{MeoJ@rbn$1}$
zFW8wZOGKKBkQ2Q{msi(`G;50Remu7fG72##^P!tA5j{d=Zmye27GZ-Hlj`7!Ad{65
z6q-~X&l<DYOr;B=MChR`ORAQ0CASe5%|;|nZnCG6ajTd?8_5ca;=*LLtc1Ff9dBY>
z#ALx?pj%xwMV;rm#HA0i0hu+E5AsUVxXwi)XJLW5V_QZ>OP*`GPJ!!HRDj%;fE@N|
z-4$V9$aYa>LQmdK<VsoUNXq_|XPKHVyH@0oY%LWkji}G^m7S=vsmfv)Ntu69v@>0k
zGK135Ce=emq?b&M>}NaGTiFw3Q1oTti3QL#CLg3VIq*&S5}ifX123!+Sq{3c3L8~c
zr)Y*%Z4Y9j<~rc)EHuGFCAW-M)5wZSbuMJn%004;RrX{Y95QEm7KxHq%8oNhD-(lW
z(3o%)(I;!qG!@xnqDspEnhchmlm-(369sW<J4+5q5n7^ZheKJ`fXtvhw&Un+O)QB>
ztB?^U@<b)&y=)}qc2ScHUAwY<M6%|M3M@L_igs`7T~><Ba+TZhBTADTu5~rYNjZ>g
z#5v>{Rh>-{Y6v6Mm!)R~ilMG388tc8<qRrqRf{qwYp>GHB%((aUpVNb%OYGGeG8S-
zvH(<9MU&=?Hmw+GFB4Uo4kOdggH3yu!$$U9ott>>(4KPJ_woXAU^n&_GGnr!M7h(Q
zg~cTcO4Sh&&(htru@hBh0u`yTQ(d%OQM60j7a|B#>=0d1qL<6ls=^KrPCOChIu30n
zr%Yx_6~|(RW@h@N^jVbga7-Zv%G9P-=H5&uo!U|wm+oeEs>{OV5VMgKy+CH7YV1j(
z=gHjI+sYWQ=X8vDx~@?ZL^2N*!ZOGx(FR2MvKPcSkd>^v8%0-CNYmvi20<RB?<Xs;
zE)c_J9moKQKy6z4N;EsscXZZ8#K>vV+2Rltwv+`M>?|3cjv29W5TPmm3+JXBf+7P|
zG@?n!IWB8f2L+Lyn?M$YO<EJXyERQrR;1XRYI+pdDn(wYg)h5Frz{wL(y>DNt~)E~
zq(M0cvmm2Vbh*wJk*FhW9Zn`0Vpe=<Y72(7tf9`%WD(jZ!@9d^LMF=W5E*|F!I_GF
zIe7~VKYgrBx=f#l7(FlKGGu5MY`qMnQ3Qb<bRxOy(!2+pbd4MZLCT4$BsSx)Yh}k3
zrrTFcpr}#09#YwK>RHg?mvI!6u(S;=os(hCt#>ujRRywoFvOB9Y<FWxiGXfRug=*!
zc@qwqMuZw8Z6|7n4r5lcbX?A0G+A8_A}LYhZ9Ryr=-6pC&te_~4~7hD$F{JOS5}m~
zuPjp2zr}Qy)v8Befg&M7UG<{^WiYcv#6d2-maSr^o^-B8wAmSt@X2g<Bl*PgNX`9I
zH@I2~JD9`(lZOUvkQFK0+vG?tt&3*a=@FRJs^JD@oEfY{?5agB9cPqF^TUWrRdKZI
zY^I!<A{R`T6Y0{(wl@V=RNagXsV&I=$vvhO<Jb^MQg`O@Nok-qQ=t;QsY3_Qj*A3s
z*(o|)vWCTWse+XxDuen?bpObDl3}q8t@f9BTVy-RxgjGF<Rs^Pk()LsYF^f#Kg<~i
zVi}B7P6EVeI3gPSCMI^JcV!wpS{&<QOg7d%*^Dg$#uPxbU9@vupt8R)<?RS8(eB$c
zoH%mGW6Y`<h(wZeKn+CM!j0~;E{srv&#$(*5RIAd?9|IL>41n^olO)AJa{@NBBNv@
zn8uCvFO#H`E&`4LYI3s<WXs7bc5RS`avaH2o3N1eg;r!j6Q@ToL&U0KCzWNeFPK~$
zW7-0bs>2^5YzpiMYg_aru>?DFbco7<C1lcG-bj=Llj(Ah*C?wxXRJe60X8v7jihbE
zp&*-#?H4>bGGQYA%oQhM2LWH!CP(CO)o*QTW%G%&*Yi+pX{_^b;>dcIrIOis!B(%s
z2`#b?b0bv=ljGBoRb>ji>}zSE2|5h!CmGDvM2E;`*(Rpb)@VU?ojewKT<6vyk>Vn-
ziZVCLT|}J7a+5G@9#JE8P-JBwTuqk~(^1sP)@D=eAbGjA4ks__j4Xi8d;~R4xwZ@5
zBrzjp26T7Jh@%AAMwQcy<15JO76DP3BucPjtKLy4o3^4vu+Bw;1}~PHZB*GkBBbiz
zMG?u>w051WaJHbWe$lU0V>UesOB+9?c2yk<wHgteIN1KOe7kU5i-A$6;gAtoi(^H_
zjmR}TMLN&ePW0Av+6Heij7IucQI|x`HyuV+SB1O|CN!?8+Lm}`E}kImJ2h)f@67Sa
z$V92%3d<jL&V&jMc-%y)-H5R&FWyzb#wj@bb31Y5d~NVG+1`|0+5p1W17Dusn$m%q
zfg7eOOv<5v%Eq%-=g1W6K({*jFz%Z?2>2X{ZUavD3fl?NLWYtkV`DO12BhuG^Ut{^
zvzzG=kr5F<`7+0s=skH|*;^znA%e2%g3>|!!CVe9F*PE%HDg1L+q?=kPKE!_JQK+3
zRD@J&cDF2KBuo)hNI7NjmD|ZHhZd@}>D5TkrZtUJ=7L^Vl^ytaA5t?hF~~UDnjppH
z4zZGK&$T!*d(FI(w^N6;jks)?!hGv^2jn16GxM*A$g9!$bPLN~Z(7w&(4%Ch%E=Ji
zr<EKGMIDq-SzxlY8gm?XsKt%FpB$3BqIo#QqLGK1uaT&({4RUO4pGFZ%$b=rA~Oq)
z5ED0I+?B1_dZ?xdM%{E7E(a0Ms$3SR>-aS@JF`mZPjJCY-(>ik*1W78c1>Fb1t~+C
zR%Xe_aVnPvYghyuWMHZJx<o}I(xUQ#SX6@(D%g4jCa)PhB3)9f0BufeT6vGU362iX
zOtooZM0I2tXJ(Wnxr*z|Y@wDmI<pqbQg>=&C#sm2Vm`KZZsNKr8}n1>%9W#|35Tdy
zhruf#Gn*ma&DN8(A!mPPv#G0J<|6b#9)i(h+e~DV*p$JOEk|3GnjV$Of@(RUgU6c}
zOwFH*GErf}1~nGJ)0&G7lR2&0E;acelD6(Ob6MZ=8e%r9P${vjT3vb4uSNt!VQ(Q_
z!F^VkVJk|D45V2HoT+%V&3KV@h!Z}Xda_MLqUzKpo&4Jr+Hrg>qQ6P!%|Dq`Sf8C~
zt__h!8FJlboU~#<Rpy})i6w$NC<!&er#jf(vRLJasB?3$bXcBwXX+{5ZrNq_>g?b`
z&0Gwg8at;jeG+36ucVE5UZObZ+C}DZWLQ)#M|p0{b`cdt1_TF$tw~u>GVEq=ilJZQ
zATeVDPgiXoZxNcD1F4;6Vg<>0tx-qT1A<7@=hDs<F>6{A+^ER-b!t<-N+M;!=#rxc
z``@B4sQn@?O7r;0VVWTXwGX2Ii%e>4amd!eQDNt%$dek+t*wWSfM^>I84=@1{}2Qd
z!Nq1Z^-a$AN;+<5GVZN3_&`LA%jOz%7*TI!Zp_>l>536;i=$05HhOK7E?t!^6g2Y`
zcSCE=&rIZtOkjxj$nZ#unMGekgOoorU6Xi@M8mBtQY<n`?Do`DNL*E-R8?kTqSQ&p
z&9{-r+lskuwhM{@!J1$lC>5c4&=Xwv^5Qm<a(I<EW34$kkXy`NJv=I;LstilRAj6O
zmBNN!UPjicnXecsc>L7p%DJqrTH6;Dk)uXk&(eY@QL>O*_0BV$q6%6lFOj{+p=$nD
zHY|P}I|pQ2$U6pa5)`{c6AC-dP%g~IAt)u&lD8JZmyOy4ACF9zSWxoDL61@ZsJW<1
z)Kfy0rc{ZjL5EW*j!#E+z3NefL&T^Lds7ydh+u3el|*vhH!@B7SUJ;0yD3eNN^pW8
zAazyaCl}o!=#vED5cxXdIeJ}ZeU){H1J`_ma(KvgO6^!_@N|ewDZ*ToxthRFOmmf;
zs$ZcxrK;(fJFsQTw!x*$=(U+?VkH(>g+3il|2zvGZ>;`~D7z`RjdWbjBeh+m$1-s0
zI7UXH0?7exPVN?Wiy4T#M1gPJ1a~EeW@ge07lwLt&1&wjCuFEqB#3T_<HD4$7Huxr
z7kIu29|ebq8ueN5<W#u)kltaM+J;b}uE`=-tc7+QJ-@|fePS`S45?z$9y3nQb{(${
zffxcWa-fRtV+JBC6cJ&51$iY^EJ8G@MawJIr7vqpWLsx|1!TI6W$<vyDO?gLR2j@G
z$U134BuHLPrnxeEQ%A@Yy9`CJ$W52aYZb>9Ai&8kDnyqD4WuA?o*Uet&UHDMF9_G3
zD8H@!Q8yY2wI!a)ktYJjMzRFHQrZg=*_W$2Q$od9ltZ&NA8<#w6C8ytaTL#>1u^TQ
z^>x9k28y5y#(J&zmB!BQ3^$XEgL$$u6&<y;fLWv#t&T%WWY`QpF+&P;2s8I{T{Bq_
zHwqoe%&UggjI+b+5P9ncr?Lrd8HS3#P0|7t$V1RKl?BlT{{t$bylRu#sutag{^)j;
zHBw7|Ef6QioIK3Dr*d+YEq?f}k4Du556qZlr>7P+A;c&)aA#)`AtD)ZgAXNLhkeo7
zT*x94Ez67&bu-DK(3*f})!=8dBQPW8Zcb@Ysc`95UGR9fVr(=P#}TtHqmM;6afq{P
zB#6-yX=X7Zc^m1fjEe0|W?i(wBKR?76*YyeDA`7GS@=T)yzI!_7C+IsD$mVgWg<n*
zt4tcorbzMhm^%Z1Qf*E;QHeTrtJxpy=0=?i*m@``1ei_I7DUs7`$yhLbas_6(x_~i
z%z`?&&Sda{-#!t8ry@jZc2*6Brr&0w*vL83+MvixvH^pxTHwbC+Mt|ySSc1>;e?Wd
zIGDMz59H!H9M?p_a&tY43WKO^`j~qG8p{@&oKsaAT+bPPDLHNpQ=p;d%nlEPdxj{o
z(;uXhsQPUX9VoGtd2%vBL9*h^n_e3XAjB^)<nw?WAg&}U+(K9xRyaC?O(2$*fk!Y1
zur@7rAX^)RT!?_^+K|5krmY2DW_U8uU~&!yTh+jvN}EkZIWd2kjU?t>7J?oqeBiXS
zxke_a3x4~Ir66awi5pR`Qx3f*L?>~w7Ist7;UBS+2~7)RMQf4+&s1i{b&g|L?Pl|V
zh;G6GX(3>-L3$h&zVNauEwqapMb?VB*3lm`oZe<Ni}EI2s53j`rEg^y;8q=B80p4f
zKVgZeUeI88RK0gGQ=dp@r=D25?4IBgLyE{Gn#WF*X`q98h{~#y;Yss8zKOr5)mKlN
zN_4@(piab)TlHU?aFvEz3yb6CkWp6P4JWS%ELnw>*%t-6L;y`-g@S9-^{(i&++v1`
zw2@csf+mLC4-y9O9s0XD&|C0rvZ2iXP>O~p>%fLf&XmT+*t`e1E=0Xf#7Ggmtci$O
zjDPI|ryBlEGskdyXxzi}Y69V+&D(bDT6L#}y(!DT(=%SiLC%*p3!oQ~s<P=jJN<L)
zXRHWwi;B`IYp=4A<PgAR^TgcW4FF72CdAZh@}dY45LsJkAG+z}G%Ol(#mVWfF=#XV
zaFG+@wnJ1_Yb8dtMZ|!@5Q{YoXMs=&8a&~)va4-kWHI3=vFJ*JO(+}6>@2YhQlLq;
z9%LyuRS>(fdzb*57*XY9-Ku#<s3pTzOY<H%Py>OqrNTj*n)_Q`qvJ+%HG-}ZUCNf1
zEE}TSY42q%RJ55siCjZB(CZqlf?Ncfmta%?mDY*@Xl!rF5hugiG{NnWikXa<G0{lS
zqR!3zovQFMYgi^(x|)|kpUeO#mMUCvf?%ImfUPb$dCfBc$_bZi08^%-AXG&aPF^CX
z(xcK&#|oo2v&}?LJCoudb!FE#7UwO&Rmd`}!`=h|s4G;S1TI#FHsI=HOq(vC9}!oa
zGA1+|09K|y;FSi9)dYa3*qdz=+&@*R7#1585o5R}Oi#qa*)$dbLH&@)v=~aE#-|*y
zT|c!VN6I!#44^s_{k*lDQ@uY*=C}>utCEOJ6~fB}4o*>uOf*V+*v7Ud<w01^CaYQa
za&Fq$t^V{bzz`LbgA30f!mRJcVgg__QuXTFnU!_~$VyWH0jf!Z`XpHyK#_YB8b67F
zN&&*PIA+WzhVF$nVKSXaL}kthIdLkMMrX)tAg^uHd&@dr%DjtMIQT)#x|Y)symuLb
z!=jF4mD$q(%j7aMjb&#c48w|&<Bt$hxJh)LqF?<Qs}7nb&M`U1b3;AjKqu-RB7@Q=
zM78-!aP0Kp7?iS<;d`Qrep{=nSJwl64KeS6-K}sdQ<#gCQ=lph3e+%rxjBY#GRkgF
zgPM*t*Ys~eSgHeHu6->qIL-Z?RDjrPQ#x`U2@=i_cf>A4DD^QlnX}ZKpK=q!+8TO6
zq&wr1TS&73y;qvrDQh#wawtM{(weZd8wt-5^2AJNV!7tY;T4+J!ee5LH8PrQZTdr@
zD=*7~HrS}8U`iFJ&(4sQfa)vo9p+k>0}o%2>5~eyEP@jld=@ZX-GL%uFo3OhI???%
z*lD7mw07#%gc498Z1syxpPAT2xK<3ZnF@f!I@47X1#}cm7o@$+nsmtw0%Gjg(>nYq
zifj|StSu{`HADbC<;1a*w<Nj-Kv4%-V>*omf22S1)GUB5frrW}?{OfwTUtYTalQhN
zupk)NYjPKC_^}5X!~7>eVbu242O%Tt5YH0XR{%}1Z7VVt(PVv<lLWAh`TT12_6Lxp
z9M+kla>A)sDe@&8wz%$tbC+Ezr(3|0${PS(3+Ts+)86(4a05}fES{Akya-?fUl48!
z^TTN1S@u?tauv?RUIW40A<S#e4Rros;bWr`8#Elfbd=p4G}0DdVH@-)R2$;My1P@&
z?6AOW0kFS!7y$KY!3def%A(F2TTwXNM8(R?C)Tzsr_8{oq6z|PG=DtqZz9DutOXvH
z%<L@O608lgjZ4(s&H}Tds>;!shfEeu5*hra3-g>7cp0n@o#MJIY$7%Z*d&|HT2V`e
zdd9pXURs#FDPu1SSw!BISGUf}<9Yhy?r1t4{kSfVN4;QJ;^FOqjs-wi#UO(!(+ytL
z7~H18$t&VptIfjWu$ImTmnZoXt*J4%Vnj?8Rfz7&S!sc7f*UeBUDkFX1_jQl4qnMr
z5n$?T5^VtVa27H;L<13lZxJ#urd5{~Ar=QZ5<}I55r}(>djM{by^&k=x|DMr2_3+d
zG6^D*Z3aY?;rwrVe?M6+7SbXu1vtOCtPw=AE-ct3GOn%yW&v1ZiB%Ust2(fTE0E=`
z=@7b>IqB-o)OPkCy0!V3*)}-8)EP=ThRm;tYvTNx1!K!sT~LPcEikjBQ&q6|TSeXH
z0sbb3Y=}Xj)ROvPk>ga=HUsn(idY2_9-<cbYf^Kwm4!wm3^##p52`O93FMKaH0glX
zIEX8oO<<a0B<2CR$+=LJ7Ks+A!Wn5`&5|HpX(K7$t4(aNmjdU#EWOr{jTLcT+mT|y
zirUd;0c=rWbQ!7|Ylo0^7)dZ1Y|+kINYORYelGH=t^+8SlU$aEo#ic|k~$9>96?s{
zD8kB<1)hcU2fTxT!>t7wkSa4HtOlZ(hA0-<Gm6_YAQV`E&LXf$ix)UG8&Sj!uE-GF
zC>1i_`}4rT%Z4=IxtM<Px>>+95o#$yAO<gVXG%Q5V$oFUI%seuc43|u0sfn+cnug9
z%zE=z0&d4UWJhNuwnJ++54&45Lt@lA;w5lt5op?6u3i~)VkDaOCsuYq5`aMpHbH`#
zn5Gu;NdV@{VHpyvG#{$7ytNjni>x~pH-$li4VRr0imAIYBqAsTtTqQDQVH2%hmM>x
zvNUXBWTIs?G#1^J!7t^I2skvdhl-$elRqF=*{&t(FOni_1HLv_WTe?=6^_K(lAsVa
zq#fEGCZ~tw6!Y71&Q<2j(LiyY1;l}fP<cvW=1^8^hL5H?p;lo~cX>#>A+nwt!Uiod
zXL?O*K>m>(sn?lPD?rLv){Ui$AThxb=Vl;av|8I(HjGO%<W`WFnT0(pS_t`$)}e|(
zcWV#AX(x(NzzZ}ymYgzOlND#L4IaL@8g1BIbG&J4tF-_jTBKbO^13pG;~OBNK+$L{
z-v?j2h`i3cYy{2QybS3uqQAFg88EP5I%FZ);HgtHgdr1I7IOSrwv`NAiF>y+%T1kw
z!Pkxx-Xhz2T?(q+EFi^a*R_QgWqFa&)>;x9j<D3C*T^W5L_meiRxS~ybz>+I@?UC=
zz>;^1W6bbU7U-2F6msl<x{&gwN2Scaf?#!<A%wfuS^(N2o6C%_2FYDiy$*<8EhnlO
zf1n=9kU^#vPiYufat+a?Ll{M_7n>*8XIM=5eS&obe9;^!_;+P0ijai}Ocs0OiK=rz
zWKy%XMH*C`D|t{!NM0>%)@y(`r2bh8vYwa^6M3>!>&`O6SR900ZO-s70&obq0*%7M
z?7#<jQ&_+nZKEL#Ektix!VGO2d@QmN@}hUFRSM-X`KQ^5<{%-sp$S%Lh^*rL&+V8Z
zr-U3Gb7E*5Q?oOGAeUG}77QXJCL1BELL`mkP}AGZDzD8$0-6%fN*0oT2pZ;wFY8JL
zZDk?vlTdqYzGw_YU{Hnu!y5}0*p#Y{m{9=ZcR+^g!q6Y0bjVp&_kKT4o2(2^!cCEy
z_gTbZPCU1@2oi`Ed8-hs@5tO}LSjl+Dx}%)0xf1wZ4#yu6GZ{RPt3oo5hRneq*s$^
zX{tI{ye{}wWjV-11zC;5zsUnMvc|Zs%VO{VHN-LdM9mt9kXeGnQ0mTLOa%vHW}req
z6zu0Foc=jJsoGq>;Hc8p)E0n^q~@73RHRD^%)pjcK?XqZYm!BZQe>7cW?*iiLPO$U
z6Pc++Sga1{Hx$OEYeU>aHWHp%3yEN`XvVxgRxSbkWIn}KG>M@2D*#o7tO<TilCTR?
zu>jZtQM4nF2o$*S;DDp=Z7_2P@@1uAKV>bDCD~ezeyveS^M|u^=|eznW~6%+g@G%1
z)4I<Uz$r>NFGLIoahYHfMOwC;2#hRj)U@l`K2R&NrY(YfM$)>>W}j5Ai67SShD~eP
zIyl}{>oD*Dhc0l5CU~iHTtjB`5QI#-kV(h(ZX3(F%)ttPGN?=#5*#XfjGGK4hrhj`
z93$xcmWl#^Rkn?OgXaq)Q>|?=)-}DL2>T)fE@MXtLPY+|!Zyp$T*D$_(t&|n3oWN*
z$Q9_o{EJc(V)dk!SwM=|uxvtuKrp(M=HaR!!rm`y!wV#uc2iigR6$lL0>grcWT$~1
zvnAf)SgQ@5L?A#mO7J?!4?}KyljE^9=T{<ojC892gO*;FxRb-MqPS)uOAm_X)Dkjq
ziIAafo1{}31hY^?Fn1W{;k+Q}7U!*L#j@bL#ynvNTyiUmkjx^6ZEN8JQU5wNqXBzl
zi8p#+2j6P0u6zqel;Ge^s3eTNVovoSuUWcH(}Qg@(Vf~blCru8!<(IjCMdEsq|;U!
z=`+}hhV45$m})XEEWH|>KImb~=}lB6B7e>hJHTj51Epy&wJQsY97}&%5|gtKVFo4D
zn6C*x2P0udiKsnAKq6*nFk+wuT~*{{j^owjq*{)aFN`?I5W*H=A?}h{-X|ayR39^V
zYEl)eG+0Tp6+~XtA&zWb?mz*d(I8VUW!<VGW+zUI;wjw=7&nxW+!B~Gh!C>iy+mIi
zu{rOv{8IpV=By(9g20%a{tbTQ)*_D`s$$ay#15Ke-3HtQHiHNlbMq9bMrT7_7f_=j
zz~35`N@?(*f>OzXvEHH2=OGP_JXRUnBE*LoVDrssUL<CG87Ls|BeZ?~9w$lQ@DP=>
zBdXG80*N751iM>#O!BmkUAB0jEjOK=0VRlKb=UAGRHi=!f1|lmWg1DOGS-v?s8~g?
z)z&0w>RlCl8yGHK8F08jpJX}MtJgR{O$$OGLCF!|I~}VqFi&JyuQ<d_MiJD>E&ijp
zbpnLuy(IJ!{40uq6?NHs$0Asz(_P(16g%QXwrx3W(09yzMvn+{>v3H}S5a7V`1H%N
zO+)@G7>~5G77C5#roo*`5}OE?+Lm}OHdHV+aJ^*UcP;+IF%s-<fI^~jn2S>OX01Zp
z3?7t>qE7=M*|*KJ&<Aq0vp}T)E){Dy5MDr5VDT-CtXf?uc5ZTLRY6u4ngncc1`!>K
zd}@O%BZ6ciPbZ9{#0~|xW16B2Q7o7+veq1Km2!Ke!M><K73AiR$^k|iIFA?<iCDGS
zn}h=^>^Re_5f5oV(}Cl_b89Fcat6yL#61XUrFqbbOCYc?H`pQq#UBnj+7szx5k*;D
z@HHAvA0KmB^!^9qam)ik1A{9MR+0vTM6nJL5sYxdRb}e&$%9@EmKBd!J|ylIy-r(^
zxF$!NtyWDPHP<~UKBy#?x1n(lvHe3rJK9lWOCVQFS%^W&u2<7NB)iFBRzy2X_GV3#
zM%M$;j5f1BnpBPa06`|6S5@I?t8s$_e}Ar8Y;7@p0OjP+nBoI80jn;!_=yP^o(R0E
zf=RmpFMvX5z94x)2up0dTKunuuN4&&Vv@2a+hH(^ke!A_K(IETufeOCs}Umxd|BYE
z>4-&?M&OVWfj1r{^y-!XuYoNRH0L64o)q8<AP@+v8jheqd4dqar5w&pAdW@vSO7s=
zSeO9_rC5@AIpokF1q`tb=0Jpzy@ePPz<(`?nrtLuESAkIo1qG{9jtMTg1~kFI4`Y&
zv^QaDNpV0C3G}LAl&f2yDFQaU1Xb0CPjP}|nd!77{8eS|$sznB>r3@-4swoLbJF1s
z19xwTD<$v<Q437yf-~9F`MS=tK6s{?>Gj}1C&MrekRL@K=9U?zFu$_v0?9(BEJGoU
zs#6;*sK~6>F5nFj6-54f?1t=v%outyv}1@P%y-rTbv0q3xy>RI%t<G^0%yJH54?J*
z$u}xTfQwY6`R_>aPYc6^klaWvT^>?1@d@-<dLYsXbcW;~hS`u=V3^Pu3ez7*Yb}i*
zM8yaMA;DiQZ7D!S_YWSB)FdtHP^SDnCfLyP%_2`~oW6a|C8-!Ga>ND?n7JI}A@>xm
z!vYQo>H?;R84)5}6`*HYF(zej`svY|1&=q&tH3*9U@SPT2)gQeDDYPnLCTSwj*~2e
zzCg&EseoOJ!@)A8vBluOFfgjjf1YPqK$lB%O#ms1WsAeDv{*ek7pa{80hFux!6B73
zQ*V@=yrd4xU$*HI1)^qYfPx@+o77$jr&CL!v7Uz+P~Nf)S`fi15i233O?_lpKbrun
z0`^lmqfA)ffRO!I1e+I}C2E?TymF<SPJ#JNv<a~)&7Rg19lgx9x?*|5T#zLslG<hy
zPhr)VZ#^e99n;9XWomn>gBhi&#@Qcu>2T=cbhA8{iu9(ihY6?SgR;l6&Z;^{P?moe
za)~n8wH3CDg$p3rkX>!5NkC3<L+WF(z5pQ50!jupOBZ%lqv;g^KT@h=)l>nZ8l7d(
zs?fxvVSana7^*bnGXb{(fDj-y#Gk4v!2Yp#+dw4)R>{<PrB1>gNP~q^lTBF#qKA&<
z7E;=9Wn<NuP74N7j&plioIwOb?ZnBo+okCbqOvHZ)Dy$o5aljRZ6OYn1*Z%drghc(
z#7I$=qt-0^GE<%hleD6tx5^@*BZP-E%kz-lQ7KPeXaVn8W_W($b!FF^jve+%+8X9t
zMy}NhpIA7g?u;l8`9P{|o||KVEUGdMu_QvzsIo?p05?sbxImQF3V8~?K{c~1gr%x@
zYM$Pt#K}+^fFW0hZIy<ET+uj0V@pHUMS+rTw1992*0{?oSPaWrVVOXr34kUs?M8G0
zR5Y_UGtKcYjCCzhr+XDvzZ!lpwAcvYFfzAJLfl6-x1F2B2Sm~Zov+53Sm(i5NBORU
zvr+^oLa(s%qE!%?MM$9-0s3!>Ffbo1k&w{dVvw2FU2zh1$XmnbSDF7vbFu*g^(kL?
zcAJo@R*>PC*8x4M$c;2WV#sSPO3Ug_Wc2`3K&-!`g4zqhga_2y)~*Ov#H8to1ajJt
zgG;h-)urYG0VY!B!Sy0L0r3@*W;vNyYvvZkRx8ar0yh~s;{kyJUIFoAd69)WoK3Ki
zVU%UT-juI6RO*mf4jELQg%&z|Q-P5n!v$X%PE})cb)N*<X})CJXFB0K4NN439zV6=
zPf<J}<IK(d-Qjo&1SkO6N+Tyq$kR^2mYA*AD2T`a<(YCP)lEo-tq2BIeX1HqWyq+O
zbA^y(8?-^#dOA}z8A)KTYHWZ|tHCDso1{@@GhMnemQB^inp+suh#Ts<VTigArPa)v
zBG5#VMw(#&*=bhd(ds2Fpwrr<Kt>h;+LLK2nI1pp%+tgQc*X39S3-gw9R}PjwOBJt
zt%3K%AZ0CDm2rrwc)}E{f_zY3CtH(f6Ph0LM75<<Jrky;6ASN(KC1+RG9v`bi87_7
z?ppN8x=}2lp=gy?wbX+g+7Qswc8Eem2-AZ_%;mkwy|&q`5yzSN%<)yzN|3Zo#iR%3
z9W`Vv^NkasBY+q(Xkk{U3}_5nR*g%_UWtTuIjJmpfwbbLwbcr_p6DMTI}QJG-S+q^
zG^fG>^$E!QAO&!Dw(5&BoIVCd7Wh;OrH?Gkk|Fef>ThzAuo-%bMLt^&<lqPe6xFpM
z+dw25{wxE|CYp^WYI1M{w16%wcLe`(R@z%YfLJxgny3B;oM&~wz-DC8w8pNN;jYdt
zPKB1I*m{$49JE!R83xH3en$ffG`VpTnyl_{uviGIAiFwH#}&YC)3U1rwyQB3Kiivx
z#!90`(VU5{F*zyvL#A8b^lIEAh9?J$(#Tm&EKOgX$VgcPL=uc=Wsmv-p4;X<e}=MU
z1_{$*+vRltu*eQ(OiZbQ+$A;sL77^hY_;tyY`roI_GSt92$qNC!4W8{EzvAjP;>zI
z6W{|<n}x)ILX!^+MN1s1>dJs0B39WpO>kD0iYqZ`k25Y!!|9LvJkKrb0>~p73qhTS
zQwilx=PV`7whsGV4zI4Ri#}k4N*fRq9f|jqi6t;zIp@8FL(p$zs~FY~)L+}wfmR_U
z=ol<DqyuT=X6DxF@@otq04q&bguKs;G>NLGUm(XN3;a5Hsa8lDj!Lqt0_;MLM<uB<
zK$B>#>@st{mBg#dHrSiQSxB+cA%f#cQ~^UEfz2<?*`LEOD6f%*tnRkysw|*KbDmn2
zfpUv1_{R7Iq&4DlbqL)kDOpGZ0r1+TwRw68$D<1dqE_(sgjH-qlh4)^VTsqgs=*h5
zb!P@0R6H{FS8#5iU*WPdUyzbylp*z?B%8)uM)KPD5CeCnvbI4{s!M>afatdqN3N5j
zDUhGSz?WI@JL95~czCe8p|L6rga)yI2wgkyNoA=BHtLFEG4x);60byF1}q8yLHXMt
z$Q_8v&U^$Sku;(h2oB_Iga#*s3GsU6WzVNV!oT5%$Qt2T4s;ywDd%}`TEO$w1dm;<
zByzQ#I7n{PP4gzvQeHJ7<rO!FyqBRk8m0d+H)gBc)J?E?Nw)%15JsGAt289V$$A$p
zAy9dj8agx87a^DT{^|rOX}E|D?kUm~vWOlSObFgKG+vV`%R<Hj+NH=gvrtg4(mJSl
z6|hpyDSZnNNL81Ij1C;dfyb0I7n%&XAmnKenIw~&f$4cfG@-&AWW=^|qo{?f31E-e
zIMQ*52~7cnbgbqy4=K=9p*)SY1YjKxXkwx<%J6F~$oNC=sapp1p$;(=va#hvwP8)j
z1+3dDI0BTOxeeKhIYe!ZA#W1WXGjb-v72kQNMXTFa$cb@(b<%Vz-a1%RxIaP6*2<I
z`oLor+{2)T)833vL30IW8l06?;3~+~+gyYUIWiZ-_A`1|Sxq8Hg1|x@h4#*zwh-1d
z16wo6s^(i#oli3(&4EN5v$7UMs35v>3x2osrmX`e2R&aAF(wU(`4)C~fDa{ZYiBYA
zig4tslrv7|ETI0BUCan;HN~65O@}{2k+3}sT(Q#h>R=_w3eOtz{3k8mveJ|^C9f~S
z5rN7;yw7}tq=f<Zv;$1Tk%8w}!$nn~?&IjzWLVqknimlQ3ChkAav{*CP~QSWPlZTs
z1M@ry@pAYYb{dyr-Dj4A!BT1lJD3pLMbN8jb-x)X5q1#}8b&3<>(#9tQ{-(ZZ9{-q
zYf55Z_a}7D1R7vC$FJH1-ChNok$G~mWt<;2F#r$6cxXZ&2p^cY8Q>01(tylUIDjF$
zSsF0pSPDj_7=n<m0C$aOyoR7Cl)tkxB)rnNkdSWBw;ZcBH|HDFKe%>!o*^hLvp})b
z;lv7A@K}^x(VCnD8YtqyjzG|DW!1B9DzRZv21*@3q-k!%>?mkuAa5j~qIomsu>-l<
z)lJAvhx<IW6ehG)vIUJ97juc&gdtH=WdkxCNTZ>sHstxi=$D2O=T()2&%#I0^wt)I
zNg#H`iD-GP1j3DwK)s78Qat2b7Mep7QoOO~(k38k8e;W<w3XQcVw#0<ATcuJMWVWa
z8nefOV(Tp3l+%OYTgXPjaajf~lU#`~;nXwX0|iId1z<`C);;)NYnVBWbVxc3oH;kn
zr%ch-dB_KZ^oY|$I}Vs4%gVsV1_-3F7%N23rLK6J3)!n(VCe+JK~`1J23v^i19LWs
zIypy<?igf4R}~>kuTz^OFsXw;E&`bb_EXjGTD%$X37BUBf|4%qZ((la(6#ikTJ&Fo
zLK+=P?@cGOJuQNEgC4Hg9@>Wj9CzD<J~S|}1Pmt#Bv3-;xl}eN^S3E3AM4gyD3zSZ
zn7)XWg31n@is&4b@$QK#&Ea3$JVRq3Zx}2T7^$i{w@6Tl^QR|(LYooYDWn>}ab~8r
zl2t%lkO%A{`?<4M*Su>~g%COGAh$FAPUJRriJ64TQ`l#9!<tivvL&cNOuU)7qE(Z=
zZ3-YPxhZK&`W77=b7dRY4l1DGwfRaod&mw5Z&8tEQUr7~2<^^}wvxDU+mwC&3K%HM
z;7CwAVXm}@GRdlyWl9q&M%PIVPm8jg2R(|aDvE?RFAAiXJ^ccE375RhCQ<ji30y}6
zr>nxS?qJp-JP7n9Xyuu0k{lBCCh$g(huUz;sO^F;AvMwz)y&Dv>jtI{!dqsaYa(kJ
zut_QDO=%al8F;xa*u3z?lx@hukwa5yO?tdNSd>7fkJdCKLbN4*_Ppo!WpjquVy<9_
z(vUaGZ?2xSCu$9o(%W_+$pY|c)0hrJiUxlDz#E}lEon$#*5tX6F;1dXWwj_0a!?7Y
zRqOM3>T;zXk9@S8$Mf(1_sNE|iW#cem;i2M7SqkGgs+GQL9bIQuWg}V(#<WqEzzKx
zm=~gui0~CLU-h>ULIn>~Yf5Da-9tXy289YnX0GHb3a9R$25JMIcO)??m>AIgZIl|o
z`9}1)OO=tV=x>wFE&>bUXVSHZ|5n4(G|GS~1<IU6)HdsH10krKP#Ar+l1Id+0gbFc
z&XFRC2fTk8K?n4I^0f%xpw$nz0d>H<l%txAO1zQ%ZIIDs=#UNBopKta-Edo_rX7+5
zZhH>#VI)!TMfF9vReUQ4eteo{(uiP6TP0wT*X-Y|)DRPTJ6;3g7uj%IqtKK_>pUn6
zaN6I-iLaOzj!dPbDC0;1TEc;euN8n=)eR%5ae&k?k(t&Dh)+XOYIGGHxl^{i{x;QD
zQ{v1W*lSuDp2ism`w$9NrXGZ0x|Oe*=Yw`uirisM%9BO%waNriDtSuY4AV`Hj@IcR
zMgx&U5!WQpFrbW?6?>6K51q`FzFqpS$RDD#?l!zjDVTDICK25INRrivKL^MTXRvfD
z4hg=@mds9Lf0DzXzpa9NIn}^&i}5PDQydZq%3CFzfp`ii&#-YsrfHzARVqQ^6vNX}
zuvuDH3FWQ^T{7H;v%PAyW(#DmS(Unx6s1IV0Vgg(NQR-m4HJMYLSi5&2(lyH)5tVI
zk|6)_LwCb%06j>u?*I-pc#fkiX;7v!QbCn}za4}E%Z(_oY&i&JdH?usrX}aVVxi(M
z+W0gu_R6%Cf}+OA&*N>7Pav{Mjw3-~SS0|3l?tVWQnqGix{<7u{)Uv&66V%=SjsSQ
zFt#1>4{C+Tc^ap?2AH8>SSZy1?G77<b61W@(UQ<i(ej5Ofg%9kEYWaT>+rG<Ln7-N
zRSy^jG|nQi<L<`4h3=@{LmW%PZ6IE1NRwb10nAs&r)6qG18x9{PaJI+l1gFcoLA~J
zlvjyoK%#*tR1BD#$*LcA05&7U%J6yN>?qT?-xBEjDpmk-LrQ%NLIHGki7o)*ipYD_
zxiyL0M`O#B4n|hyFmJGaL0g`|RYSNw$`TY>j71RU9B6ejY@CYxl9U*8W?Z~o>PAwL
z#BgDCil2=WKybLOls$w*T`x<39K+!b4p6-wA`^#^EVOomq5~*Dmbx`boVSAN9J(gd
zuQCceB1LiwR%NH=WSs5{LzM~&nG%l3ybiY|joK~XZsdMWvUqe9%9*Dp4r!M7HHI?)
zKRA;L#<W(Sdt8$Mi8A#)a{>Z~9d>{i6jXx4xrd`Bb#f94okn>{$wKLH#tu)TPInG@
zA4Uj(>L~C48ENLA3`+*0CL20gu<!90lIqhc19d#8N+4ybKNyQq70+%;=2S`}0`XJo
zI+;PotTrvVN#vQd!y#Oggj7??3=9S5Rh(`xh}7#r>z0Mv>Rt>(0wF3YfwE$Qvx*xB
zL5H0y9YuV>iV)w86q%0X0cGB-ygDQL4gL|J%SeYN$`Xwr=1LOM0=*I|JT9)fQ%T?9
zVh5(u_P4?LtXw>hZ6P08cjS}gXBAa+lwms=PE+!=<*p7VKKTBo@)KYnGm@SPzAA^^
zO>T?wWxxVKY&41>t<VDuAtyZ$e!yWQH6fOCVKvO*&WiI!YzL7`Xh%uo@?qXkVldFy
z7ci8x&7i8u=75cw<d~L3T&7?+7Gb$#*-?#-45+AcC0Zarb4)3$P|2}1=*r}609R5{
z0NnAZ+i&pZD=h$UMbi9+Awf=P$%GcJ3HlRvK&xq`L?H3RA!de7qAKWyS7vWQhl8B7
z8V6*S5&FWy8WeiC?Q^ZY!U3MajvA%1s*2eE6>A(Nw~cF(<YTCJQ`A7|NIc&%Fd-VO
zFGvES@Y=ArK-p_*DtW@nvlS;m&Mtg5q@j@GB%8#^5_9V}7?`AVN2x5G)&iF>+mT}_
zy7n+6;Kk(ZAU!VC^y(;rcr*C-Q-vnCeg~AQ?2|8}bfti4VkZ^fE&3;&B!Jcpodoej
zF+bJ^6=qVmaTGw_P_v#>f|ywvMUbIh8bar4K<91((13_50B0i%J_v<MQ#z=wsO;5{
z{X0zo*`f(W6;;QIC={AP*G2&?3Sj1@aU)5v2|!7$g-X!DhY3j4vqs3C6lKc$xiv|y
zIVxk$5stHM*f@9;_*}uQq~FEquGK(;kASEfbf&`&s3A8%Z$hO9ncl?3l@g<B5Xl_I
z)E4T7Awh`id4mB8+AfM$xQQB)&Pct5AFz$elQNP+u$$wKuf0hNeJyAzI2_3#>FU9J
zXc}b>EwMl=fY)_=H_9=pGpwd+5;HtJ4TY=Flpu8uqOnDFM$W<(H!(&WBv;W|WF2O?
zp5K~yChf;glG0Zx_5i;@8CC3z@SKvBvgT6aO-@xSwT5l3iYk7fE*fH}BLu)JHLOW^
zikdl+2Un{SP7(PWL=Y(<n5M=hN-@(Y%#n~VSzsV2kT@Nt8#kE32~uqR+LXV(lNps+
z8ZB>^5tb>U{f75mYayT-YK`2uxCnGZvqEGMHMeczyAhs4x6|zZ-1!s|SVEL#6&mtR
zOPdU9l4KHv)WQy~+$_#mLBq@e{o``(oChN#+eKs4upNpeR+nxEkUXrr80);w(<p+p
z#;&YaR-Fbf3iq^-UI=A@?C!1{PMlI{X#uW?pvQ!${29>c`7GbUFUA29ojMq-C1p0X
zK2Wn@h?P`s1Imbh0runJ5GFcEW+AhNBy!4Ar8|*LsE(i12|R`ue@Lo!GA=sj3t}u_
z^i&<LB4RR3K&AKq%3f(v#X7DMt$GSVUE!M~mz2?|L#9T-`vT47+5TzB_DbosTQFL2
zm1M9!!JkZ`u`F|uHD19W)Lx;eq4t+;bdLchq0kVzT-Jd=jzf~`ey>o^L7L0fA2trD
zyD+^#AP7sZQ+i1|A!%6iwW)X-kHr$<14k-2e>khN_%vWPDSYL%)|LW%5YGT&OfXPW
zR0|kGd>U!7TClU$AieZ6E-nr9Yswbr3{bQF-I(&yDjh5MU=XkP|H0j-kk0Gm#&;_#
z4P?Q8mRk<(!jNQ`4iuWz>Ra&)C<w(sGmhVffX%Qbv6wjNTdl8AW5YTtKT`%>%rLm7
zYiD;;(m_|bsHv&0us9?XJ5&@cvJ$x#jTI`yR%%$Z;8=^iagqyw4^|x*rIvKn;mD^P
z2pOiNwnC#>xf2KOAV31Vq1NUemNL}aS}F{*2d>vB@HjA)RR<GDCG9YhtO6o>tyQ)F
zvk*IpgH&x5nA$1PS`WgU(gL{_FTz340wa%dl9ag`*cqf3Y@9)uH^3NiaA4XM6uxnq
zISK#@Aj(Ur(qou_Qfmsdnh-<ogxhaK+&Kg&ipu{xA08<Pm|qp#`_uvRozz0<R3|0P
zlO&;NE#~SMtaLr6>UxV$qY{d|JH?k7>avWUCwKxlZ7^EEa#eAa;Ee`}3Dy97rE|ui
z;8(d8Y$=uK*-0)acHtBx`Ud+*?Oe8e(J1*SVGy~^`Aaff<}KKb21j2V9ad~Z3IpRb
zP+UXXx{;*vkGh`JZIh)@!y`0Bu)fu_bbyV{TEv*8YO>No5`~W587xyRDc`|>n@0N$
zHJEG*&;YH<i*^v^sL3Uu9<76{u2iScll4eJ49dw+&OGXqfB`cK8$gJHfbf20Ih=W4
z*bl6jjk7v(PW_4?6tx_qX{3A|%0^YIMkewrt`c%a$REPNicanP%H)wy*o|_fWv%o@
z!|txMBz31Jl4btH=`NIA0<E?Ja+=25Qt%!v)Ei}|gx%imMkR+VvrvL7@<)bFHip5%
zABYNAJIx$bJXJ(+fmA2>6dn7y)`G5ax{-F?MyZS{tJQ?@v0@Fy^Nqw%1`{i~Q9ecI
z_`rt>5e=#%hkuL*;BKv9wx#?H+n|jis8EtqP;#33O7WHGXoSAIMLEPaloKSLI8+%c
zYPH!r3T?IRFmI&OqQ#}S2?+X@aRO4sk>hOCsxg_y3i>kugDBUMeZV~m=ba(?gtEoK
z0|cyXoSTKLlT>}HU~%ixMZ~CXNLnp<h$xp)ymG35i6U!;44I~hyPJgBS{+ZEfN<tU
zor_>;2L(KdDk2l2rhuRaH>Pxjcy6k=yHh-jDHXPA6wS;{HwCE+1@HhKR^zsdJAf(_
z>_RP~ihs;afL1+9arUU+HjNG|>|#CAJ5BH@oiJAy5%gLxW3X0Y^r28PEG=VKC{)k6
zBCs<^?U4=IlKzuM(}-{uPITzFfXS53t*KOrqAAzvdqx#)Vkhxgww-#%V1#x)b7<|P
z--_SKbzKflRJ0pXDdk)#&@=OR8*xXhZi40HHm3eIm{*_`C5BUJsh_w_Q6U;|5nB2Q
z_R?rkHT0dz?<_76Epa4?p`+mgIfYcp#X6v`Y_wb$L?2o&IvSbkOM`kDvVHbJ)Pl$m
zNU>Of?Nb=s$;k%aF9XfHbtdS;)0F9zFfM)=(gB?P23nBfLDY#-7~61mi>{d(f}a?I
zC@oG4f(!?%s!5gC4nXqoG)RzeF3DKnI+T?X^+{Mn(ZNf#$$>@=+cYQNM(Kg6T$p=(
z64WHX(~fX3>_jd^QbPI0qJ=+|+SLBUK{hHyDuH(@fDenSW4jRoSMLS@Bd3`+jmG~J
zOApg-3?o3-Tx*U`1!$o0=0i(X3gy<Jb;lh&EalSJK?sV%H3xJciXilR#DAyx7f83L
zs_EWGm4jduN$b}nS!fvK_}*}R#*w5e8Fmu|du3y~z;y*42$@%Dg^VJK*Af7?mWqOD
z8o!X2C9TdD;TBljyTLm{sDM-zs=1)CiBNVsfM7Ueu_?eLXdDNVmY_Qek5nvJ%Ufp&
zC&Ud&b^;Lu3^m#~G($ON3JXGUQr2-uU_t@cfj78@HauGS_>b_EGZut-^C}*Tv=7x&
zRcIQJm~t^h!U0A?G45}T*v3v~7*iz)pX6_~K#Yb}QfM?nSwsrDnYrCf*+R0J;2A3L
z9t=YQ`W0;$ByguC(Q_v!K?7Ehhk(`R)EagG1-y|<RwXs};UBZ*k&tZBk}_Ar4rsLm
z0^n|}_A+-dD%Fh|j7vHQz&#4Qk~K##rpCqukP+t%{h6|+xEr)6)37E92UE`h9h6?E
zrZYJ9Nt`M0v4Vlco{x3_$S8_>!49YRVkgY0KcWQ-;KnC$Au3N+JPhn0Tt00P)fwp?
z`S_g0<gg+Y<8;F~Q)pQ=96Nay&o`x<B0CnAM$NB}(LVg?xVS0$POXJHb`m}n)IQ<`
ztg$Y7u|c<Ru;JcDEPIP8$ILIa(w0{I%A$tHBk-Rhq&j4`JE+(%5VNzbb6}Ho)F;Ud
zQUokqUoZ`&Zr)mo8j#z|i3DUfF6B&Fe(5`~0r|hUalk}rU1dVvjm5(IctEFRRL~@q
z;KYRy$dROu6iNcQPfZu&0gz6R|F1|Na565AgL59LnudCN;3=Ye4AdnjJ85?~)eC3i
zAV^9e*;LyD(wxTWrnsvdAL>TIyYJK)*!y8NC0j>}t_+<d<yF=K_$qir(%i)sQ>F6n
zAiR@ZsTTh*Z{Q-RGJ-TDCfqq!gtD=fYQUwL&eE{oa@DJ4C4+yUGKP=R7V1SZ;-H#l
zo!OTXy@E}v)vTGgxEf$+Sc)yYddllKm_|@q0ZLXI54^mOK$G5HXnjOT>B=a~DbEf6
zQO;?f5)gCNVo_;+bIGZRZW)jK1go>sq8C-IpzA1t$SQ=loH}}1oS=v&PNj@KS~iJd
z#C78o!&EC8LX`v6E*RGs$AM=&gJ6fq051M`n}!E#O?-t%tsde59E(_(*kjT~tt$|B
zH^sU#^`dan5iN;30AdEHE?~VNRueZ4w0nroR8<Yt4+lE{PG&e>TeXk6%B@K&cxt)_
zxH|{%skO4W)o_wMiJIhm3V5BdUNNMBNDQ(BTqh`T9CbibTc={vYiL$rj%PCJELv1l
zNy30Dl?1#{_g;VEWGK5LV>qK>v52Rs!gLDIHXsy9M)G(rFbGijK^e-y?+&U;4=7lT
zGmTj~$3BF{6_67MJ4)vqfX;-yP2tf@f!@yZM1C0sp9`WwMy56BT$Gmp5S(*xD(={=
z5=!s27|V*ZsN3i<?C=)=qfS{vRpHbbf^wuXgWn;yKaEE|R&tf80}P7_AFb<TLD;9$
z%rTs-T^08mRGi>TG^I?0oAY4cS4BYK=fVpW_nQ^7!T?K2XOuP<lwN4&)ulyg^tLJ8
z?gl$nP>ckY4X2ktf&!r=@gnl;i4_h@IaOgzG?03ToD<HkoWXa)8iO^mj);#$PsLXA
zH?@9U<8I?D(RgZ?sjAt<=!1qF5TK0=7HiK33E@v6%d*oV1Zj*w>%J$!k%TT~?smS{
zjIyw>zrfN`k^m9o0rUd5ai<aw^nKLuD7D^do!UxaM|l;OGQ6fJ8MwFz0mU5vzL+Rr
zX+dq`HG_snoDzA9aSelv6XqNZu*oY)yK=e3qk~$b1g~YLAz!TBA*^m0cs?+H;ID{5
zLaHZfQoUBc1@7iF1+s;S{K`VgHOf02`C4HT>LikbG4cur%*r6-XTUnn2g;sB8wa>7
z0cy-);vmjD(?P<59ab5~a~EyXN|8dTlEdKw&zx6{R3+uNrMNTfkP!$8oh+&YrvpBw
z${oTK-6A>&+X1YY`LF|kYAN#~oDX<lef%v|U<V}D2ohG(aCG3kO0_%|B{$P7o^MHo
zsYp%+I3Fv%GHjeyxkhn!)npAg*d;h%ie`2sb>jTb3J3Dk(s*bJ_(~{F5>K3xA_E#W
zsqnuP(P7oXbtBi7%5sA6%bYOp6qtcB2SH2`=M51ehdFqOj^bhtr=V2+DRucctN<&;
zXrET;V%mTU!kFxkI;C>O;J0U|k}?wgCFC*W#4$<az{YXm)>I1!Q$>!zOxF%kBXC#B
z@JM6~f!{a<19MbU4oEdB6c8|U5)W1i<OG^)oa?cZO2^IthZWzDZ$BLQsa8UR7F!vH
zTL(=9I|DDAj0QZ60E&lI0^|d%7iMCO(df)R++|uShypOg1mhV1v78<bO+XSy<N)Ct
z!h~cZ5Ud9TH0a6@*X5dJj&{BP0jOFnRnu`2XvDdWyBnM}-D#=9hN0ljZY2{cHOyA4
zTQ*KN9Z(i)DhQJJ6o&+|-l{d{D~fboMC^>Rc8P^g;v)VXrv-uQ#Bbl^D2J3ZaXWx)
zHIi6ma^c&nT>vz-0s^HLRi{GlFx`z(>H~GsOh&x^Zr-42A)^`qCdmL%PL>?q3OlfB
zvEGY~vr42YC8rAoen(I>5EiyLH6DoBxR`LM=Lu-{B30CKgxqAH_b5C@RT%X4(peqA
zc{LY`g`_pKhv_a=BBgN4)!1WIoNm%6w7_|z=+G=82gITRq@w$tY8eaYF^-Y~cMtw+
zxC*0&M+I=AMimriINeIGD-&YwGUfD8pp^rPpc*ws!A}Z@LlARIQz{)lm>Q}83?^+&
z<gH6pe`=tQit0?ONn*HOU^XqV2!?l4D3KB?!wpuXPL{OTN<>=8;Kkz@BVq&{+0qoN
zW<9j=-9Y7084p~ah&;IbhBc|RAC$eFlraL6ag~7j$qXsM#&2R|C83OYi>D9=80@<)
zB#D;}iY@pcNZZQt7)UEw!W1lkCl2tn^I}8v0mPhqix!PELfptmf!L|EitAnHbS^D>
z28$#tD8QnkX5QfK!_9&r3PVV|4O%Dkee!}-MbF~|fY?*V0xoNEE8<ellvFt*9FE?|
zqlaxu#T=}0<>Qkz;nYxh24{lWB{;*nBC2XN5onTJFf&1CWp2O0yaatZTaGYMbVn2P
zZ!D_@H&T&$I1y>UgtRJ6k**?i16oH4L80vf8*Lq27D5U2Kq^}|0M`h;N2_3=s^tyz
z$Eh>Wx@3y<fJ82fXw9U~lnxDMWfXL_LELYUrgLs9TCmKc<We5jn#}=_1uu=ueLxc{
zEohCKT7*CZW~oyU<~a<?23K`F1Ms^L)y_0V4lB_xZw2WM(lydAlu^1`=p>{O6>40q
z5&w7y1HP5Rn}|JJWzMlrQ6t$Bg`$ixPNVWf$Os4+Iuy|o2ZAx$RYEZoWG#r=IdCpj
zhIF6-b`Z9Y?}op&fG|uX1|&PB^G~LRWyeDSo3J|;sT@ztw+Z||S?5n26hGo#gdK_U
zI+raG^&I&`YH8t;7!YT;{gE<oaJL?M4+;$9h=Yf8o?$6#^n^<A=KN^V+QBYM65QZx
zYt%H&;tpVQz>fs>l3Jl@;bK&rH7sF>#c_f+gRV?ep;VF<G@z17GcJzP5#C|tVQ^4(
zVGs(OB@JOT5~f7XbS~2Zza#uUtOAx~S9EThqQFA!)np1Uc8*4@aKH_e<E_bc7=(G3
zYOxM#IF*XDa!Er<AI}bHm2tf5>WJq8@4*^^{RO%pXNW<=j>VfoGNih%%i2=<Z`Ow$
zsFhQma+tMpt2hq_Xf*q927yDsO^tZY+yvw#G2kAnk)njVhY8TynM65>3FByWPtzLA
z$`_79hh*irae%%wxRo-JGqDrmiNiP%2*J}xHhvTB0CMe>^Rt0Q55X7TjmliG#Sn9;
zu?x#6OQ2>Y<B~nnu!N#eP<j@kyM@w~WKo|KHNAjDSBo#g|LwFO_2($R0w)$(bx{if
zW`<%R>$XtGpmQ`@Hov;&p?9ldu!+G^kYtNDiy8sm_o`5i6u&bFhCHTBBvdr4tjsl~
z12boLYkpV_Suk3?`rVx{2iZ+jf&{Fpsok-tu|bUls%_`s4WwGv2vMp;)RbY3BT3pE
zu>6#vCaM&3AMgk&<fb#02d5aS)he5yXtlaKj^`N3Bnh5M5nvE&qPwl1w^mE8;V;6V
zjN6ppNQSwZX(2Vx2=U!={r^;Rw=lA!A(m+{fa=1HX2i5k!-L-e%}Ku|_#U5zb%Q<&
zMwirtD$W}odeV)lsSS(=4B~Jb7#v)5SV-U*TYuz}tR}Jnk3^~>B}T+LE%ulKK+5^b
zZ5)sZoknEHR;$uzzjbQJ;5#KB6N$Ot9;3cEP&KSxob6Fgg5E*EAe!^cIlmJMVKQ?m
z5m3M<4y~z0)?t=!u^=c+8*LmNbIecZ3tDKZqFzGoA^itDM-?i62zS^j1cvd`RF%tF
zQOF@S07tr3{)qEN`YQazg_WevqBBl`>7qA6>zcAV8mHkYUqar2jUWV_M)QHCtgacz
zf1tr}<^vSonASM#6hrDFVs)jw=14xgQ>hCQ<{b49(i4A!6+5~jO0dO9Sc<>bnmFi=
z4n`1#q*EKS_INI>Zz#t)$0Pn7=jWmjGp?*!vl8HEjLtaV5I|tSMQLVQe78=kJ(B}K
zKoB!0x+3rfHV|y3s0luhmTMJ4Lf|BdlF1ZtL9m&S4%lhLhTJZ%E6c_Q6QmkaRXRhA
zeNUFZx;`~*6L$b12PT|SS(;%G<8ON6aL_2zHwam0t`lRzi$Yy0jlB=6L^BxZ2iy_R
zI=Kk|4TN7x9WdnLM}Y@O4U$#Xz6Mr2yg_RNqZ;CI!8<<Krcgz|vqX(E3Ro>5zZC}G
zsohR)Tm+1R$RcYFV^WK%J3w6tPyr<k?=X&`7`97ljOrF}ZiO?naR*hQmRSYMZcDNG
zIzCMVu~IkT?1KHEbW{b3l)!ID!CNwwqDjlyO2mk)P2k;?gWM2W%eDSqso3@|iwXuL
z5yWt09iWUg-p`e3Vju1xK!f;5;^LwNV}U0L3xbM0F0@7cSkgdKO*gMyP@dVwI|&7j
z)*f^S;;Pfw2#i#yhk2YgBAz4;mxu;DoDK*{Bm>_&{uKfx?&#3UZW@FpcL=6j2RLW|
zf?-MF$;vD_F$@WL^BiyNZM0$cZg5Q3Fk0h}R}5K<<!f1N5=yblbxB8crpPo?g%k|(
zf;BNExNK8RcO}&-Q(UPN3Raw)>V|~^?7N7HV9@~1$U!I|Rh)ykQSdyg08tLRiVN<-
zg)KrB2t<R)?9>_1Um#{N6rr4p!W=NDa(6K*)Mayt+ATa6<O2dCl=Dm-to>3ZzSEN1
zL?(iW3zz+aa}qRoq5f(%+#nRdX@Vjl=M`R2=Ny0|sM4bnUuoejVH|OQ(4i8g#7COB
z(==DmL)s~IK*5F^78m6a07wzd&$Ps=Q)kG|fPaKs%F=Lbp_92*{OW4Olm{npFS$@9
z0S^G7&NXpv&`Ss`TaB_tHNXfIVdp~|)@fdB1wleYMWCMI5+N2+6ApBSm<CjsRSNFn
z!jFM#4OvjV6*oMnGl$gCAQb85T2Y6@=b@9>6De9gQH{EBlrGeq09#{Hxq_@ZW?Xl$
zHYpq@S3s--FM<4T=qFB6QlK0&1w3cii<;{~m0g#l2WG!w4G>Xc{lz3bpqd?`7gDMk
zE*y8_w3^jHrX$h$Hew8sXj-WlN`VIGu`?eC72(3j(=0IIMdb-3KD_q8mUYOYxeiAp
z?k&tW@IBGf%PSGH?x4&jnaW8nm|!Z!eFCHc-|L_oWcp^U7DnwLL2!TrIJc;()llOS
zCcT(4QBvH^NGPaMo;$}!Qfq=0$=V%IkUECR6U9(z9yzH$b?RgZLZ2GqFpFp@aUT!R
zbf_||)!9~;F4u_U2b5<L$>p>vqI)b+swJ3O(O!WtxiErJm1C;qD5=zp9+X}(?Ud$>
zM0=cTJ}0xpQ%>3_JhDi<VNJpXuH+ubT>u2mXI?1@BpgX#pP;*OtArv*xT+ywOFg3q
z-aT4VW|mUPqT-swv4n17<);eo98fHEPU<l_c<NFnz^RbbSXKHasdEm%SqPX>YnhWc
z&_>5TVL<h+kpH0k0uFYm6_0YO(Ljy32m?bw)nlOH@E~d1rM#DVG!%*_hb}nA-yj@=
z-lNe38TIYNW3j;Cke6?*P){A98=$W{t<esKfix2rA|d)rWT(@0Rz&CqBsRp~H2g#Q
zhC{XUZb&lV1EnSmcP_MrI*2sWEZn+snz`nhp;STRPF6(B>u9iESk5>o0R=nUt73iS
zy{SS0`A3zxvzxFBe3g*>fsb%8;Z*B{Rjmfm*pM-0B87C7$Wg6C@443}NwLR;OkAQ-
zKESwfpo`UN1+_*cT~MAJH9!<ap<dgfi)S|!!Z@%w*$h1D$T}3@h(lR%kJ8zg*+f=V
zPST0$%%rS2=XXQ@;npQtp>-1gf=zjn;fzKEc&S`!CKP^8LJ3V3<2ca0k@W}86guVF
z14XPtZlQ7+wK4{-se@yQ-#FB?E{HRYGNMgjSV=z4yDYkFD<u-il=Kd^8i)AM?D7_3
zL+B5Ru4|lWq~%zZ^bq%?&o?Fcgi{C_BHU`VLw={4tyQTbw;`|8Cs$%wP-y`#EUI|)
zhT>=J=!wiad&5mPu=`H=wUm4e&(5&lI>nmRO5jU=tAnKgxve#WwN_$r=Zu3+O2j*K
z5h-w5r(i(1CtEnz%Gs%A!8rw^q67sQW&mxw-1%Oq8V5NbAO+}ltLSP+RiJZlw*YZV
z9pC^+fWpafNP*Oj=bNS{QN9+Yb=5eXdrT*~g#DyYkG9vja446WU!SYj&}AjVYfvej
z5TSKEdc&G*HF;gLW^sU|F4%;RD>e46TERwcDWh0Xy`jS%0ru})e+ojNpeF$PvaO<=
zOiJ}B0U)L(SK-5Ua4TwX2<w2#+fl>Il!vLq^^Xsy_1m<gCK!tdoF|MCj*5qg1MZG3
zPlz`X$Y^y^>ab`653%rw)=cPQu8}qcBh>DoLTAsE&|35P%X-MARX`;XJwjMDeRXN!
z8fmQ{$%Li9XtU^wP)uckF-&|yYfi+atmHnNG=^!c9K;R3AO|J~EyR#6WdeSFVjQW~
z6T}@Bg*ko%n4hqIu<V?I(b6~S8{v7YwD{>D@Cr?5C)%XYu?21x&>4+}gwkAdOd=8$
z<awr)U#%7bTp)b_0fx_mq>7GQt3d?e6`=qh2nP~HUD8mk9#_C4sRjg_7$Ql?R8yu&
z_6{ID7d0ZlN`M*bANXF32SE2$8Wj>Q-Q1_Tz_OfE(7jS85njc?5F?ZaG#XSjkU;Ny
zO-Vq^s6-C%Ca)vUlTu6I@=<0*gp-S#Vpg|}Vli7HV-XogdA_z#a1&LvQ!nr!wSvP6
z*d?pk0qT^n4vHmtwHAPhLu<B%mxI%c%&t0jCN1V~qYjKl%YQ_;EP<a?%|h}GYWPKC
zunF)yWteg3<Z#)#GeB|9N^H{57srQ}Y6g7=R47%HW#fVtl<A}bKvgIh$^m?GrATJw
zm1vCs=A9v?#n3fdzBW>b=pKW#6+joIpb)+usRNahq90`9Q?dny#Rb<9u~~e15I{IG
zDim@!0<qkz9FU6%XO;~~#ZJPR&i7hStfEtSQK`NY4=PA_kwTz-b0u<g@c<Bnfa<^;
zq|nDk7#MN69MpjliDix&B$OddL7Pzx%xHSTbgS}_f*(|_-P);YnP&O3N^-oVI9!I2
zq)ubQA%$$hQ{h7-1+iq%aX2MBu^puq@TR4Ny_B=cSyX`SWY?E?gUQ*Ch{j4o@;H!*
z3Q;h{#k&bo^3p_4Fq{SrkHRR_-GjJ={8;ZZSL6U*4>C^_-4vl4B;jaDYdO2|>Svu7
zLGm||rc#-NI)`ef>M<lxF!ZztS%->%bQh0psgYb&#-ou*>48S6FeohS^E$}A)zE3H
z{GTMII&Ub^xsq`L!&ffw&M039Iq11^6-Gl0F~fl<dldx*(<`n?N+jb}!>@sh(&c1o
zo!?G@9~50`eU>B|2HGhkk_3Pw`yOz`4qX_P6PSt1e@jRe)iQjd)@Vt^h-#-AN(*Km
z&6^OChm=xG28aUXxtq?#s6@9X$DCxmOx**e^I(+Fb%3X%(y{v8qU(=JO*QnA7Ls&M
z9r7l%{;t+`#0ZV=mUmjEg&K{GR?cz;H2bMmE^9oFNX20XC@XnYC_}v#d~#kjJq!uF
zz;sjOF29?JQgEAkL&4H|Y$$10P+d^ii`?Z%SHq?vgsU(mf^beMtIkqWNQhk2+VSC0
z_pVS{){;;gHhG-x4sfSf2A!sYI%-EssDmg?)g^%}$(%ZqlBc3PHI+G<qDiauF({1)
z92HDaJcNy0Mlp{-mq?P1S`e{f5^G4I!jyLmJ~+&@*cj>gsE5M@5F$Y_s<rA!-7dKP
zI_Uq{8d|5)t7<9>iRH_=Lsq$ute{yysMVT{k7swdO&tndrBG$UD9+O>-?R$2SN}3`
zAeZ})cUGOpc)LoM*+lh982C7*)!>S}GjJ2Y%0;dnz!0A{2ek?&bqPc@h2YW|&`L}U
zbhT4|bVQ;iBuG<`6as6d{c}WHya2H#tqemBice<73IK1Tx^EFF0ux>b>@BQDIG<gf
zu;xjCpRG9qw5m^plxq1m7_k+Hn5T|UK}lCSb<%0xm&+!Hc&VTy7tGO^l+Nl<(+40R
zjvo@rBgF)DR6BJ<c1oJkdd`84jSG2LrtVpfRihL`4qt*62W*Ok5J^g~g0Lm^Rf&8I
zLV@Swj;)kgrtBQ@n{th%npq7X&^nK?4upq-Xffi*N^l@g^eRNS(t)d(jj5ca9<geK
zRb<{fUyvp+C=mf+Rl?YjMvv+*6(s*!9;(Wnn+4qywlB&mp#c`1iyBo#bwPb#Xi{d!
z8DbE!w<JqsiqFX1NFoTe_yp+I#@XF4tC!$AbKr3eRRtYY0cB8;3*;l@M({T1B@pJ2
zZB3<F-0RMyB|%Cv5?UN?jYsOVM8UBkNrVJ;5WS%!Noob2tfjg~>mWDOe&Ov3EqtVB
z<S>$iz|}j<@j?Z67q|us2PuJ;2%+Y=^Yr4tMwx6WNK+TyXWo!?OV(z}a0CC1N4|(<
zoMxJXLSVwtR>DmIN<mYLV03c)TyT6SB(MV<)A@W;Jmok()OC+NIS4$UlknhUQgw#9
zh@GSw7lx;*N`$du_=kvLM&goE_tll78Bz$95<5`T7P_=}29S6R)Qey-;aE?5LLdRB
z4PGvoq9a~8AxR!*GbEb?NgTPUo|jmpXto@4(Pc}?#FE^_6m$wnuud~4&P0|1a3TCL
zaozysl2@u}Y3MnbJ5BM-YZYD|sr=J!o92)Js>PvY$y}PMQUw)IeTpyvOz04emDZI9
z0}WELafp3LzyT7a$|6XyFsGNKV6s?^AU1RF{HBOKr+N-3L4E&GDC8t6P<D!F0UX#4
z@u9`=FvZ{uhVeOySBz@nE|q%FYDW(RaZZ*gSr`uD0%z&4CM%$jsWLO6^Op`N!YF8w
ztJGjTMLgf|7gDjD;zQKG_5~a$3q++dDuZcqZ+seFPU;4Lmr@Rjs9+>8t%IB=2G}W@
zCBjlV;(4j&MMLvec$OPd&MaJ3km1{8(K$fatx=s9=wahha1$U_FqMJZgQg@x+@$|q
zrr=@Do{l((I~GYq#fGkw9W5^%8yIIAu|b%}0Z2Pz1wFdNivj}~;egmlVvq<04g!tE
zIYRlQLkXy0xB~BjiZ`rDG($42aZXa^)9nD_*@Zr?h2Oo6R6GQ(QFj&QlXaQ~<f0Go
zry%GNedO0T+Z4w&#uj{kwFe$^(vsz+An&YIi4o@wJ_xLN3gJP{;nE#39xzitHbQIe
z%6ur35rr+(*hxL>MycU@;woesp)N=026{|N)P;6|9XZbv=z53@kzLrCkzxWjsnR<U
zM9c~IJ9E7yO+|~>)arGJXLr&m%vrTpjP=Tqn}QXhC^)s&G_4{AWGE8>&Qpo#RHi!)
z;*K*BnU(@F%%f&b+M$9*(c~=OBCbi9DZnQMq5KvJ7m4i}POehxnl=Q!YbRcHTx&H^
zD=jS=F@~rVR2{Iv)DmTJx=Be)tym!Cl^w+d?i_}>q4uI?u{y;N#0*6QwSx-v9_B4o
ziZG&6)TnQ}#+$TRqqbjbWnrU=O+F51CKdl+Qz8r2Jx#-(M9gq3m^u*V4ac4?08}}0
z&!RBb+!;zHfG8(tBr^DDZaKa-+)HXoI@(I|T&M#DR-d@OL)n5#gs8Mo1-5jcYU)ct
zf=rGvNgteZ?tJUeBXFEUB-wfQ7#R^*15_;l-se!ZtYQ>b(TuECl(Pc}F_Rlz5*Sdb
zfM_jp;Gr*Ijey6C?*@$@2@feaq?B2YE?Y1`wR)<El9vmN$Aywfl|X@uTw(o#2qH5}
z&oJsN0Y8qyJlD920>nsHvk*fsOYl(?3RhM?zRR>=u!xkkwGys3sq@ZIoutqzuN=tI
zTaTUTQYEp4;RkYr$cs(L1R<kY=5@eZ562?TIE8gn*1XN)*{%MlPKir#Q`dzDLdwd*
zv>>O0gfSOeWI%w2aO6`u+$ZU^6m&(qWmsw=ACXZ+i&kgAb5q^I&Z%R;iVg}Az^Y{$
zy=r(CwE$M8$r}{Qa|U$5Ay(M}sj58EA8JrnRJ#`U8SmhY6)PzMf`M2#M0<k=gFv9V
z8AK8i_;UvpK@+mVb1ioc+e-8)<htr>$}-`Fubus-)T>JK-<C?1AFX^sr70ol4)q9>
zhO@hMBnnbfh`<!7ds?UdvKAf;vYA2UpHrjj>5sdk>2&ntX*If_;g)j)N)*xGNl=OV
zlvsfVgmDjuCf7OJ2n(Ifq3S>mgG=dm4girxN%}zvJ41k!rwW&Dn=4aW#7c4WDf$?D
z8I9NhUt0|)X`&%qRaFyfbmf+ih1~#8UF*nf6Fo0TLu`l+DWBUg-vwqqi5Z0Lo4j$L
zC%E~ck_Szt*^5KsqLzN`l!XZ89LvKciFJ*cS!BbIrWHq$<a8L9C|-d+P&pX{mJ=zC
z$1T&!tI^DdeF&F_@{E&n?ywu!_o~3E{NjWkBp!9RK)X;vj3ibE)v3@;2@0V=QJW{e
zn`PlpM+xUL)|gAk(V||KO)k=vVnl<))3iNqz~82>-6*=`OwW{yp@9t#{D6}|xTrCt
zIjb{QA__rCMvdx#<bt`{GllYWz@kc~G!!8&07o{IVgrCUAW9uqf##o*)(hPkWGc?{
zf%^{5NU`xdP2q~Mt(t;OV3;~&Q{}51Kx_($3Uh=K6s5WHlxt13hP6<7cWBuvu#K!h
z_=&0LHY{bWHNcyL>D3x+aVZ1XZ!}YoFiurP4TEig5RoTAGn3Yt^Zkj9hC!vaGD&XH
zT7mP%O1a?(0ygEqGsI^pS*0XHvPvQfMx-k20Ga?)w1L{bKZP~B8Em-jt;*-|;Dc^7
z!I>o8+!tEmT`NL;rQuiW+*_GcC{m{xQS!ozASyxNn8(inr9tF$!O5We9(P*{Ou*$A
ziwS@$UiBderen63l0Q?Tfjo3V-Vv&kLF!6gNv<?GwGp}7(^^WPf!_cMLN2-sKNRZd
zSK?goI%rZ+aw$_hMW;w-$vv@}ansf8c*UBek<uTxH+b@#oLQY&$5ntYx+C}?yAsc$
z1JP5Und*G(?;1~?+ucwX;$_LG#i*2Q!|pDWtdITFY4)`{so*n%U4pEX%Gf4{-A%F`
z*nEmWLE#$X<b0~t_q2v^4TniI^TAdr=%LjUCUuYx@K+RQ1jI*}KaEx<$OmPd!8b!L
zhR=B?S<dtT$X@Z|4QD{379=<Z;Gu9VJClm71nZ;B0NMKyW}7k32v{vn!Js%G9<m&6
znO2W7ygM;!p&`4PNIxF;E-I7|rc$3rQ8LN`5;L_)fi|+aT8l_!-Wddg9yz6`w}cd;
zZA9uM&caGFp}}rB0uvJFF=Mp8HRXdGEP{N}O5qutsaY%{qgpViF&3+qmu{j6LU$sQ
zt5Lh7t{f8ydC5pz@WX(f9QK~~1SQ|8k4~A@oD+{^3c@m&Z|I#74bMoCF<WYWD205S
z_q(G8XlhynSg;rn&~hWniBHMPs`WHCRyI{tL#3aINn$NgFmORI=`oPLaw2XKayfA-
zqA^Eb1VOM8J|T^sviW?x6}X)y7i0c+u7(M9%1W}p<P|BN(>U=;Oj!Ncz~0G!kG_aX
zJy`0|tTivt)eZ$s2@x2gN$Ce%0Iq{73r{45nAU-C9%A+Hbs(L!?vbwu#)fLu6ax$5
z*>OZ6r)Cl9b|utnt&f1#O5sL93-BqKXMaOeCSnbNRR-22D$A@9@FzGxl(N7Rt10<m
z4t%dtj9c8@IHXl7XMsH0IhG5p-Gw!f>-)Q`STYc(Gy=;Cz~+UjkR`<M3N+7f2}Kg(
z3NsjWxCNU+R;q)3lTHT#i}`NaD!M}<P$lA_rFA+eQGDf%4E2oa>N4dNb&eH^GNsjl
zjSWvPb^82i3KuX5Dz)O&BRZEv)Cz<Z`V+*8UYN7Fad#q+d8p%@2F*?qXr-vq%<-`$
zOsUdA5-i5~9p?>qK5R3<M>U1jVM;;NllA~*9(u2vw^G@o)k9CLO~kp=x#WD%x~arg
zHMKb&i*>09A|ehU2dx~Nu)F|@zKjXz6UrL+Fm0n?4u!Q7@$*1)oUW<DXeqyur7@AJ
zQKuiwiMXPv_2H}zFx4eoqFQ1Kp8H|ruwzJrQH2sI=BuzOhM5<D<^bNT;oTs6(TanF
zxN36K{E!r8eG+KGdFd+5YNB1LfdSBj&N#J970gmR2w4_I5Xy*pW`Jq4nX`pdLbTFG
ztD6wZ$zw4RO~fQ<K><NHSA8)O98T1lBYz1zOf(;IHVu|F@v>Cw3b~Qggnr-+ouXLd
z#=*3g#g=IiJ1Eq>X0GKaD8&E~NbS&qnTCpBDkYs6N>pc2QFR%DaWa=$MSZw1Ejhh#
zQ)o?~#^)$iq?!X=;5^0e9x>doBQ;8PNkpC^1kOVYM=cwbxhCszcsDF6u-U+#s6O3Q
zzRCe7fTs5mm~XPk?+d39`JJf}BC+ICAIM2@uy+tl!w2gqj|sLwRDf$kvv!;_ND*Cl
zyNGj=rR38QKqj#X31~K*=Tb#au9Xx#1@!3Cl5yS<TZ8MU=IHRGaSb_q#8=3nhabde
z>A`40h!_86>&iRA<WZrVpgwhQqeR+!s<NuU=45H~Om$C#UP<{1$T@3$`VTh05@2MU
zZCT;GaUvHLnL)iR@<#FqFv+xV9WJ*Lxm9`M88ZGffI>x8jZL^xv05j9vK)*@Te}4B
z3LX?(;8-j0E=CR|5XyLC(F}<)`h-(R0t+gwFkXZ@UD@f9$;vpt(DFJDdg8>$G>X=#
zE4z;N8~nB{uIvoMolINjR>MO?{HcUsUAerY%dms~fXawDj3YjiP0K#wEL2`N`0FC<
zk4!pst>W@beU&=RMnM*nncvu^jGiYPHc(;0u|XoPOB9A=7&NLphG2QD;9%@;WufE_
zm6d{vw~-tTBEQMBG<X`(6sqSED^6=2)7;tJNv&zhU>=pMyK|a32Nc=75Nkrz;$tKr
ziBuQX#FjFT@aR;_;G&EbtdP!8FJ-!Z$iFDe#gs?)7{LtmG|k|H+`?rsP^t&Q+Eg=u
zw5rV@UUAiE=}8Ea*_2M5sZ^E3A4hE1lYVnd<;G$NaJDO)2r9_gA~>5jMvxQFH!2(`
z5CMkumYAhe3~N=HaHc~ifh>yCtuZvn#W<&MT163rQl43%)k;a?qh10yS2HJ(@jwOK
zkkl9m5a$*Eq%)V4i8-6YD_E!I>hivJQ8Q7($dairU1PLKv+)~9(#XxvtiDXIb2aK|
zwn{dHG$Z*~t?ba01ttJpK%&1|pqvWc@u76KY)6)nt7#4{T`3sI@T8iAPa|Qy){3D8
zm`QTNAWIqqFCiIKsJ$y9QCLQWQ$f}O+=(u|H8TbfG*r^48b_BxC>~7$uo^TCdFEUZ
z6mlYTS1ZuG^ofgFeVJTk0D6F>qt0c#(8|}8Dhs?@B`sJ^Y6ii2N79|E&qA>&gU@&5
zXZo|d)%0&-VJ(`r3#|Yb&~mQoHY7k1^%63JAlyJ|6Zozih8vC=)J)>tkO(?3Hl7~h
zm=LVuD0MI-GNx0?I8eG0lih_@pbk@_7m#&Y*~x*PDmZRf$Qe<4Uy@DKL_)h*%kXui
zj<t{K1)bs?3ngJnT@6aS1E~SQz=onwL?aVQM1nH4a+!G2I4!^kO&bM(Ky13y-AoIH
zaBj3pY2((UmS0gyEF6lRakcgF0Qkr`Qrbq1X%8^r^n%GBbdQ4VovsWs9jIAZfhin8
z?x4bLq-PYtNfhE3+fqx5>U3zR3w0Wk4g^2Wh7#Zt#S`K-r37iCdBnt$b7wwekOY&l
zj35``lzM;(QXi+K(%UN|ebh_f#3H>CuX%-YC&D7&5H2b0+G$@>hu_P!E)mbvY7Ip@
zKpOEVX!;)RTAW;tdWln(wTj0_V>7wqucyd%rU@kYrgHx@&>rj^5LT_$H;dXUkd;b{
z;li?6E>st+;ASAgj<*GZVplzvfEF1aok8>+<Rek;xfVWsU|?PDBA{BHo3j=|^rpxc
zf=#G*%HmGh_9E^#5K`Fo9p))}$Cs{5u!1{n6t|+R^QdAIO=9yym8!H&u7n0`Y!uyv
zM~sxJ=#I`1SNL_X%&F$)P-m#vXv0?dyD>`qFaf2y2T<6di7IMepQTdZcd7&+U7JP&
zI#W_Lg=crt&3vH_F)7PDgD5Pua2@MIsjLpb*U+#EAA**{2Qa*jJxKM+X|HlLC`M1`
z9mJp`*(c3Iz2cm}xTGDvfr7_`RDowefbHQdz;Q^vrwjw!z^=*x#~VIf79g_FXR@Kk
zwCdN#k5nt+P9N(4`hs$Ys~T0wTU2Oasg{P+l<X{?I3f@Va_yARNCQ3bwoDm52od4l
z&m7w#Rp?ta?Q*i|bFZqUO4m^<IWB~$F5|bg&?e=f8wGngaA;O&ReP<gq4Wrjct*>X
z)Ry^8s+c1%sWQvR%vH1Rx-OO)N}!FUJ|D_FC@aTCMKlE)AD4Q^0T^lA;6j49R}0u<
z7W<kCod%~hZw}9zBQz&HK3AR*-2Zrj-O*86BpC}-{gAFqV253tX?-m4hNa_)ZQ$L=
zv8P}jYnm9$hmpI|7&;W(tRts$p*d?qii8B&712vXW+~}}ZkHBDL)Xun?i^rNN<vvi
zOetc9@xT*C+G&~Q-qlX*36#15L6JH`C-Kv25jCwpkyf5IQ895~@o+hkToYr$`0uQQ
zz1B2rd}S$wU0bRL90C((<(C)?6c9zkQ2oI<_95j`-fZQ^E-ewM_t2MmLQ~<Jx+0#f
zLm|XDsv|5)g{20CH3>|sWfzmu$$IuQH>6=SN*qj8f;KX@LXt&7pu*!)4IX=E9M^+v
zClXpm$wdHN3lV9!9u6^QQ31(lGNo_h<fbmhpgbv^MwKZVo7j>g#4GW`nSY+jGr$Ul
z+iKE{@iYU-!FB7ReejbK|A9OgK(KTD=}IV-9D}MzwNvP^dvKWHM9Gz)JX(vK>9X?D
zObg+_&*}~;t#}W~FmKU1;!qgvRBctX9Vkyn8rN#>41B@V$pQ2k1s>~(Kr%=rCGv{5
zkrh!Xr#TUGm=1?`g93(h11$wb5~k;?!RX^;N2?)WDjJ!1;6aUw&C5Yud1i7t(Q5ub
z7$5IuA^VM*Cq<6FkKO3Op&A=h)DRBy%F}_7hsQeQUrHc4TB(ZcN$d-X?vhg;edd}1
z4I?(Z_f#JBG~8?@h<qtpMK)(tFdB?OxJ&Tqw-&(dZ^LI?v#g-CCukGBYE&Ym;*^5J
z3tyv4Yg3T0*lL!QRw0O9Y%OJ5YVaKta^p2~(iuDD9fPjKl^QBwQN#p=YJyU@rQdHr
zbo66_u|-S)g~F#HtW98*MCUH|A<0a${Mox;6=JQYOs!9fF_M-E?8~K-<K)rGeYlNL
z^IVh`8Wl3Afzp;>p*;wu!l90GSSUs>^606aPNgGxESY2u&4Ry#;iqObh(Qj>;Wld7
zs;5e$c?<c)kmR?<V89U6ZR4yCs${fUyOo@^1vGr41Vge+#llgaVH^_N@QO}Nv9vty
z5~xk8ZU({~c<Ey(8TWQ;h0rVF1f45^n8F_k_gk)2w<*dNz1Zr#2c=7`vBDL%<4l7h
z14?+pGR~qh&TI`^P|b&=M|X56Q36gRWil;df_I`kD5ym(M06VHaas^L!Wo>!T9T++
zte$J97tzlqbxs}5fDTTF8c#~5uHdLVL3&34D{NLg$WEUmms|lKSl=W;I0DL=ggMyn
z3Fud~U&^FcBJ^-oHxwBiMiNJfk*EME;Fxd>GJsOZ;!)f*`8tuBqEw3{)5_oMI?wU0
zJm9(33M0@{N4I9Ji0M|11Fb9NDgo3Kp8)Dawc$+dyy#WKKS>?p9H39`OIMakTZt>K
z)5ENdEGq@rbHqZa^gj?SxorxYLP1g>qC9sej%HqDO0WX)X|=!;+iJO9ttdk6U{7^c
zvZ?@mBkKSWI2yNFi#LF%pt4z(yPUud6iG{Lh>^;DhCSLJs};wgGIB<bp$6?>BT->e
z<qP7K%x%q>Qp9}OL|6&|lRPz1Z{APO{svc41GfW&0F{#^qMhahIf#mpUg|iMh>YN9
z#u%>(%@1^SQ7Lbt*em52aJ3AR68%h*?U4B*o}5>$*33U#U#+tK`!$&v3jkT`1aqn;
zF69EBD|$Ic3I?MyGHc~8V2yw@G4Fk^;j!pCt#H-CS?I##c#J9Wj2|VpoXx1cW}1pu
zA)vsnI94MyEvUDEN2O5Ks$t`FT6;tJni^w=bhar+0_0XeXNo+Hb~kmG$y_Ov_q8ZJ
zrwaiZMfDX*8kU#tG}YP$q@gkcpebEsMXBP!d1-2kp8(PKfT^&g)rIXTj~gg~F~eG|
z?OavyHpO9!2(ipRbWOK$$P$E;1s(_0U1BK)Ad*zhEl4ke48u7-VEZL@Lhgno7z|d2
zhB9h3xU~d_v#6-m0~DuI3_Zn4qxq1jdsnp+02rOSO<`4dY~dedZF!Op6%wR(!}yA&
z=hmbW>%t(w*_}rIdQq_zRf1IkUl}9heG1TamkM|!F)D&!(0$12q9!MzJau(-;6y=m
z5o&<WrgruI6a#=?ztE!7jdScLo%*8y%%jFNRkQ;%Kt}y1hza3@b<hoB$(o3v2@bgU
zV-hqc6vcSR?Sq5N^DuBua_CWcUG_%hXn}zO5yOX!G1F+Ga}E&HtH639G1{(lK6BHZ
zN%mKEc~`?kDRGGW%F?sK^AWs6d`(&2m7OokePt=2I7p=u_yw{dT8o;%Ksi+Wn=qYu
zB13rcsBlkw7ZKq+^jcI7PZu0<od+-BWo(qfjqCuTQIXdR;s8J!_{U+5a44is9rHwC
zDFa#*b@T<P0YH$4!2-JO++)<DE0oZc^2OvjL>ykH8Bet`S=Z3II+rceSZUU;n&6$I
zrVueec^nOV;pggnNU~F-8lF!Wr&@=w0vDEPwG`b$zVr!9crbgx`Jy#NTro-V?<h^&
z;&e~T+`-eye83VH%|&4V4&sIqsfi@&HDNP4THuP41SoB)9*)Q+oZwIAmPQM#wO#IV
z0!7M@YeyQlV*XmkXn{LK^Kcu04t44<z4)B)jB71=_3aL~6(Dh{Qc)kk$y=|giTbKV
zcgTxH$#opKZK`e_9OLR4aX9l<Nfn_EL46Imkm22w4GFNb`l@qJhzG+9*n!kCrkY3|
zt>xyZO+>DzrqwwW8?A}#;Zo^#Yr~-T)*{#|tpU}6HS`(wiqcolH>@@wb6k*+L<uz>
zGEH%C3I=*QOET4DBz#5D&qeL0T77o}-wAa(rw*I}>WfB)C|ZZq(W)V>W&+4_OOL~&
zOdzrRT(xqP^x@rF<;4c0L{ScnF1l?>jtEt;Nbk`S0)w1{ErCdX-D!<4&o@e78M1th
z2g8jR-D4ndD8!9(57&O_)+G2@1e<!ta1}?-6WQlTa9H#-K<{G0rImofk%a32WLor>
zsLEJtVY{;R1%(q>DH^S~0Yyb=PLUvjGztQ#S+0@f%uzxzRqDLAaGAV#)VXr4pQkPk
zz^n@;*;35|;eo6L1WQ~c=(21mf*@_}7-Te8r@`>Y#76?f>bdCaL^5DNATXq1kbafa
zO5g$e3^rFClh<*9p*1PxuI4;?m@~&>r#!=CePwktA1Y;j&a`HPauB&8X;kB&P#WOZ
z&Jt6?NRs82TWQl)k&P~$9Hbn0aZ;$Q9NHi!VZpCK`PcBC_{Ei}s-~1Rw3s<Yi%f|k
zTeTIH+0w-WGT29v^%QO~KCXCB!DIyc0zO!M$;=%c1fM8mnrYgdV_QVO&y*1hCotUj
zj<c-Ql3<{q;lajLHfVUDgs}8A*$w!vL&nvp@hRMK2BC>kne2vA;pC7okhkgFEM(w-
z4A+{-rLV6Eni{7h`?n#9-aB<Da6l-GLi82;+ZTdvl`6GGD8sAjXo2Z{3k9DNZ?Xns
zWCaH?@2k$X&ei!yNcXO^EF~d*7J7^akeY`ahhq=Gk;6QpX=;sOvW)S=x;h^~Hc~Yy
zw3MN*`%SJKk_$p9EcVjnzX94p3N|<?!~}Az>!guuRz$8CfX>kX5aYl_!A_RrJ4>Bk
zxzhrmX<ZogW|<97Ly0D4RFzsk9v`P;HL8skyH?T=f>gth0D;uXQmt||)Va@n&_o_c
z^jfM=W)bz5fH|%;6A;o9@}!-HhelYbF2tL4F>8zX8!Nrjla?&QZb1z#G!?DGVb`L)
z4<bl9U1>3E;^YfhHBF>Y?qNz6T59%i2xD==##|_+bCbDvfNY0`=zH6#2c|CKyd}6Z
z$RCDqfGR*9i=8SF5`bC~GjpdaCrV6bWlTFmyt^ZxDqEV63iUDzJ<1YAp%qxXHX`oG
z&k4tq0iCNeAdi+XUSq1X;U|Uu(UaNY0VTXl5n4hTZ8VMWwNq|N>n0THrij8ELOH9!
zP;1^$lw9~~3*~pxRGr3?v!r0uGz}Y<D86J@#Rq8-su3uV%F%9=O7UBh(_D87#Tu+d
zr9BkgLLpI$b^soBGX6R$SOD>fwL@Wbl>^rByX+kAvQ@ICQUx>YE-~Mf7%Y4u^0qa8
zo38T&U7V!~AgpzB7m-#9W(nd4ojTK@p^ZzKGT<$lPs-5O5G0Blhx3jQ0<~WVxH<nM
z=8sl?#MPa<ZAu+$6tPA*D9N*QXE!JWXe1#ZQct_%VF-*A<m!Ruas`DG-4L99R01KN
zA*S(UoTJ31G+2gi<A7NWfFBgO%5u-5n+2ajq5R^Iuv8?eyOBhMlLc2Ri?YK7X@sp+
zwlLDs8dvFtP1R@tD79x`iSZP5G}CCV8(?RLijW-)asfnqht3$8Bq10BqXg=bG-0-+
zJ8@VyDKQ|z3G9;S#nwfFGm7vjJ3V%?vZcd3v!)%Pxz^spN3Z;1N;naBcdM%opuEa0
zU!8*Bvyy9)NLCgLshkxhji_tED|cir4vVYRq?N2y_%~I&y@CNHN=q&Cf`dDE)dfit
z=1HZh=y~Ok11KJuRwz%&SVKvTQKQTl1sY|Pq&|w%6tq+iiJn^0)^XtKQUe-Zse;5o
z;@q0JoQ$!BWU4B9=yN^82*(igBUa|?Sm8uPCnP_bs?H8yf&kP|J-f8=`lOXM#SuWE
z+_G`Cp71x|)7J=MPFU5YmMOjqAPD$@4(nu?fSQuaxh7>9yw`=vOZ1~gX}`cIwJvRr
zlNnesJ|{9!eLW=Q_Gy(hm69Tn3MSDiJrHM57fB~-p)hTwLL9qSy^)cRt<ks@o)V0Q
zg>*+qWC?T=8PW`i!C#Vq{c6F?kQtz%6zBX_ms+FQHTVW%_1sKL9FomJ_^?rC;iz)~
z#8($PP{|TiVc6Y-B*`{!T1A<7#7CivUpaU>P4aN{qd{g6gk$sPnrG~M3M_xrc?bd^
z+mD7A3LGwfOf5uS?W43nSV`MATuSQL-{45aDuRSglj<B68^{V;04YzMQiQ}n8ZKz9
zgoK~3_2sXW|EMKo3R(~Bd#o3c!+Z_+AF`g-;=a-DR_arwq3cv*9^MUtD5_X#DUxC#
z*hIxEC?=pk&<5j)LnSIw9w<vqK%ww;bu!IqQYjAMap3~j>efSB&~PWoi0%;99e{%X
zJQ1sF*rsR@gr7+BCeH6%5y}afsoxFz!Lw0;29k>*wIyy^<UX$h<OpOCsiIV6-eV>q
zC0STl0LE)2|A`~XZU?5yz~FI-%s-ai56irg@htF-l8i<m8Acwhqa2G8HcCaJ-Y5mN
zWDYxn0-=SvWzJEAny6p^{UI^E)mloVaeIGBtDK6Otph`4>xKkuML|k0+?-YEa-ac?
zDIYJaErm83LnO+}qD4jEQgu~)$R;n?HX6z&jV*Uh9l~oU`O4By{*$wDO64e_kSexj
z>MEvi+`uzW0bnd?pMI5Ux&*ceX<ekpMTH(4rBKEJ_5M31Nqkftubl(U>KwqwVvVTZ
z6_i>ioKBx4uPG<|OYDd!R|K7Dv1XtH4U4IB0nmi9?}f(el)YJo+)=?m1wj@o1U~oy
z90r37V61Ks9Hmxy9EOBokj9wNR=O^Ck}KtiQpQD5(qNE{beF2^f>!}v1^u*x-m9ak
zwKB0VCb&ApiPo~|v;aq0DsyubuVgWjqt_|GRSr~_Gz8lZP)w*>%DQxV362I1KIK}e
zG%eV9XKH02rY$Ibyc$YWrB-|;32)fUgPbf&O&uXOS-GOzNK&FheVvU~VyNm!;e-F0
zI4#*tTKd?dHK}u<Ilb&f${fbB(*^qsamS75EKa<l!6*$K)eh>yku<hS69!WRuqa@&
zPT(P8P-|O|Cm9qX!;mNmJ50kQZ^)i>;uWHA;F?rAMC~d%=lTQgDAF7+R;&3`E{srX
zqC0dW09d=2*;E)Ti7A55Ca)YUg;>36j6f1{WdMq2H^50MBw3Ase4(H)0V5Q1tz|`>
z@!_@(e;e?)hL}ZJJG&d(huMOyN73~|t`Vu<xk8h)z#h`b^B-z?c#6(Jc@95Kj2dZ4
z<U&jCl#n4g?2K}mBQ_d5ghkBUP2gQXZqXB%aLNS|D@tq@iUX}RbnF=AwGLHdjz54a
zjk`_NYDzgl1l77IE%bC4{$z}l%2n?<0$cnAN;9Uc&oKZBT0n!vOmrL4{`ys-Ny5xr
zsRg;Z=r2L^XM{&uP(^j&ST_<(rA$kb0y@<SqEVdzc-3fi8hF84cfR3u)Fchal?$z4
zIqZN`t8S3QmughQB{Rz#kabg9bq&faJ7^-c#{dU$v}sYyxZfB;fHN8u%KW@hQL!}s
zRlqPnW1TY&1DTjN6&ElJquEFJj1>R{Bc;t2_M3&?NyF+OU5pz?1J=1laI|`pt7C<R
zjp}V_nSjD0SqD>65>o+gj*aF(Fq#aXDX<Y5mD0h$6di(+K}kho>z)=^vr!6_d*GxZ
z#7(&iaRtHmU20iaSGp4MLXck($ey<{(*p`l?M^F!G|u;m9-k>+PNDoWnNtjl5Dcy<
z`C^Q<aVf*X!@N+DhDwuhcgqJFt^nIq7JA~wPRg4C)FHnQ;pJ-2Fk2KiP~t$)!Y)M#
zgOAjC+&zlri8uu;W-T30?OX`YD<8>(y%0sUQ^T-`wD+}99amb0uTh&mDi~TNFUL(s
z;sLJDVVkN1LTza%pJjQ@{cT0A0sK-kuGJSGg*nm??*({x5LG_gF30er5CCaKRTK4+
zLhEW|%Jv1AwQ|JR3>{eQjwm`@XE7Z_&8$ZA!U1r!MeY|D5>k_@iLv2b7&HZ{4}n~T
zd`ST<u5x0hsOk*l8{oZo2veCwX-Y`_YZRpFg7PhADYY8#)8jhv<#;esINvb#J69|j
z1{@-v)FP;~4w`eub*lY0oG4Jydjd+W#t3#Aes{?ETuLBVE_~t~9XS76FO?yjDirZf
zLVxAp>9wW-a6zq1KAzA#qvU<5xJ%01Yw3_)pA^kg>D4K+N8H=3$yTGEB;?R$8tu0P
z2OS=Pf+~oq7J+x#D;RO*=dqf%u`H+7nQWA^AB#TswR#!FG}pR1_%KSB0}UA=^gJ8{
z<V4g?dBP{tY7#U@3VztQxTw#KLINm-v{45<E)*mIjSzJ$n$jg|!+(T-uO(QB0_(Kb
z%6OS&At^;#k;OWVSZyguFc4&F9W|f-)@e!_K_}phSu`?XsUmwiRWb%FBWD^>?;Ab`
zYG45>k0dJ;u_hlM+ylj)3tu~wG)?uRH%5gN;Y4`*Tdl^1Tbz1nE<pq8J&J?mXxWTf
z43TIZu2zvqs*kyvcu<kU;|_9^xQjbYLFE=WAPTM1l4hktvs9(!L2^{d3M_mr`cji$
zp#!7rRP7S%%TB9<D)vfxRMmCFDQM|NX!HyGqG*lISV2`Ov=m6De&BdpOR`;~#miC>
z)_l1F0wx5lOU2|@E=?8qHRm1`96CMyoU0urliE-KTwBeGA2dAnb6shDKoHxR)9@0G
zWq62DzS$xX2q}`V%rp~T>DQY)#=8?OV4^iP2+o20h%hk53OKUT*^-~{>$7ywRPpv5
z%PEUQTbzf5Mv$p5i)!x)WI%gSmK^d#vI`tBwz@Smkp(j{bEXmLx7a;6`IMj7MIY#f
zl{6ALk8i<I@o04$%Yjx>qqqZ<qqGJeO(1~;N(W%23bj~Axsus{Cyaq>28vSkz(yfr
z^{nfKRsn%(i}JyREA7Qbl%+yx%T)15P78vtgnbKZN{WdT(|GtS!#+dt(2_*X(Wr@1
znu!Eeuk7LoJO*(CqXQmk7MM$l1?Wd7)s;f-Tueg8Bc(M{l${H3vePF4mZabhGvy)c
zqN=7OL0T|UsS~mVP935d#O%;r%gjfhwHhx;{e&0`s(m=C1BXMWx#pNu2)5x2ARPsO
zfR={>iskty$ti^#&S<)Fm)C)>jC6#$(t7W$FO;dsF*D^M#P1@)isM98Z}0()R*+v1
zuM9Ide!zxR^kOTgU8cmkct(l^@@OKTg!#c?pcuz;JY}KuHnm2kG0Yt*0>&&twSYFE
zu`8obA!#+Yl|=GFOQ*WDa6rl_W?IoI*IDY|MA$!&65*6p8ZkEk=(Xyggw>xN5Mzs)
zKt)^~b0)P?fc-Xwd6pD=d3EKHj8Ndp@Zi*9Gal01g(S&}gAfGP1PCUgTN5CQLW;Ev
zF0<hx&=k_DlmW5MjEZrXZjz=+FcEbk)1a*Lmnbd_hEw9L&}TWAGei+s7VHJ;*Ex8D
z*1F`B!(CK5Fg*F)Y#n7~gUi;Nw48_7ZD2kKt~+LO99z_N)wFB$ST`i9RikdT+yW#a
zI`(Vzzbh^d_NBtb7IDktQV%{zES*8TLXVF7yV5f!tsHa%R#O1FL`PsYFC(TGGFx>t
z1L3B2t0&pVIG`jr@ERS+c;eJ)t%bXDlxEaUCnwPo$jzXxXQdg%(e<aQP6k$p71X(y
zFzGWeCqqL)5`E)5y?~xu!YhhXDoQuq4Vh}_`QW?=>O{aeBr&-{9Z6l{#Bymope*1@
zqA||k=t3d_ZPkQTjl#j^oZniamEA!Mnp$wf1V9Q<0S3|YgIIMu12j~l+$~^?p{0#J
zJk|*Ug_^?Htu7{v>yP??c=zQjO`}^AMuZH)zLIc1^(Gd`14t=_{!wdG#^5f9L;$R{
z)q&TXyQrCKeh4{@h1RO7VokhK%L^-yq{8o8XTNFTQuJ#}W1~*_FeJFfjH{XiCSz*r
zJ6S5r4M|Msm38DE!_jD-7pTCX*mDHe`0rr5AOoCYb#Zrds$&NP&RLeErz!xJRj8`N
zf-R%c3-3Wo_=ifqn8^#ZX9_~oFo(cl*LJRG6KP>7r(Q`l4@zFdrJUe`#y|xO(`1pf
zFCl8(8gf)4IUc?znm?3H8OE&E%Zyw|gg<ZrD61ebLWlU!>{~eMFxHf1JQ^z*1`$aV
zxB!$H!99&5fZS24zo7_5tQvfGDDIdukPcSfVFmX&)7mMFu!~80sO0e0kp#n`(z?nU
z*!`7OMMO$cp*<c{gmj@#(|I7*&~;LGF3>~ZZE*IuFu7_MVCvu)>^$g9NqcYt6T-#E
z<crGJs!G;|o^Em%fv(6(1|y<b<3I-@nJ}g}MK?5SK#97eFmK2{MBx(C2WV_W*@BrS
z`<XKze`4<Paj@ROjVV1H2Ta5(Yu#>*7*|Sk+ISlWnlFNHrP)XR-QeZJflM@5k&rI4
ziRY+Qr)6f!Htufq)#MOjRb=;O9_EZ;Tq?$dBrVhvcbY1s6;qky2&+{!9Qi;#3i7?6
z?#mpi9e_@%+`iCK;Is3^f0T8trXXEi2j0!_ZWQ_?ua5{kC9)kjmog0aV@Xmih$^|z
z7P-*`-^s6pUE0|=Fl$v>jJP2iq;aTrjq8w^Pics{TqqCHsv`?8FPH<+IF3ZD0jdwM
zTQRJ)BO+166}ddAG77QQhUsP(850T64Uu1Nl^}d_EuAGIugao|4ZsIyzCu*Vc5EEr
zfEr`qL*QAEMMj1A80$bGBumUlKs;O)STu$tG%SVsRpK^f`+<+IU=Y<~$@#ft<&eG#
z!8-X?5nsZx(n2I`gjD&l;x>gDuE2LgNVaUE<jS-}T0xoZ76z2u7jPgl0iaB~o@uSO
zLA+`wNJWMGkchY20a^e7udYS}iTMme(pXtO$TfgHd6pX(k6}CE4A-($k%ykrz+j#f
zbc#REA$0(Q<Kx5ihey7Sjt^23uw#`SKx5EOQ_yr%crwWX$UEoQhZG|-GA4LkTUS*C
znTy+1fws_*_}C&WXq23$wywULiwSo)LOZPz001o(I3;%jl1ruFmO09t3kk_Zh>4oR
z9YJm1Nt}!%C=^=dv-TS&gSm>rM7UisB^~g%pf<X)^{Gd+bQ=esXRc|U9JG1skl!Vw
zCRmv{wRbtled&`N3RcUrRa&xGwhXRWO1Q}=Q^Cb!)iULx#lR>3skO3Sy`}(;TAqZy
zPVS<UlUf3lTJ@C@2G<VNo+_;|F(6pz@DX}q1d_^><yN!mT%Ir>+OpO>J=S09Y4lLL
zk-nK`0+7=aJ4p;ARYpHxUTN;Up-_U9<jPZnNo%jgZK^gWd2S^A7jQbePHLGsJmyrN
z?J9?z!Sx7inqXB)jM0IYlxzghHwUUhEr*Q*mo`HM2?)q|m${2y9voDKH3(ob#!cZ{
zMo{t|M4U=rfe)%}l7qBtH6>wO4m7A$&Ztb&*?hE*Yz`O@3EtvYi<N<uqF$5k@zs%m
z>CFdTH`B-kDQ-ms9%s0HH6l<d#ck$NU9^&c))>N}K<<7#I&kxAjW;T9paZ(tVpFNo
zh4UrTNO^Q2Q7NndJ5?)7G$RmYiJXzGRoF}Mv^yqGYIQ&+0D-dTIpp9vEl9**M)nRH
z3tw+!Uc$qwQBK5mk!L=a7H>nUClR9q9xOV)sjY|a2Jj!*{w|Y{>={j3R94<HX2>Kc
z9~1*`(B@&EaRE^5QiaXI=FyP*9vQ~9F&sfKgHqhnF~hSZQp0^KKlu<yOVnkF+mQE5
z)y#8Rkd|nqwiH>LRp}VBz>))GBRT|*-qps2<df~8?&FeyjO3<xNpP2|f=*(!PhtQP
zk#Xij(O_jV!kVPo7asFenVxEooyjpQw7@dL3+`wK0g$6o%jy~rKv6p_h)F46IKypI
z=CyYatK|xXPo5&7ons%%05MNkE7fy9nnw5-a$wVi5`ngk*v^sy@&Gu6Yc3Y?fhMfd
zY(a%Xwaz;O<B7q&APE8X6jPKcT?1MSwpR51Vo7a;7x{_G8yrcE0H~smHCnEwEIq|L
zCq5)|qEfb;8>!2<)>wqb{4{{nIadTp?u1l%rqW8sKpJ4n6h<$_FJPCrG#;ECt%@gV
z3^a`|x(hQXVRe}D0Qk88Xh|I~JjxW1rT|}5dbO0Z);TN`aq3b6!7KuN(_pde%QRA7
zQFwzzt_3OprF5;<=(_TF`_bdwkGDVDxxX<V&*r;dj}LdJqr?4Ou|lu><H`rr?JDx!
zd7|~Yiu}uu?>)Ht@4Jt7{&V}Q<o|4ce&?S2y?f{J<A)oggUQ#2<9%5_o13>*@7#Tj
zd*#+-;@g-Ox%@eM_F^u7T-%$!l&*g;n$5=3`R@Mi!Fc2A@qG8i<S>*Kn@xTi7x{*~
zeSD;hl34G_)jM2~a78jdI!b0QMh6F1d8o9S+_w5;{U~fG=w7~iHaR%h4KK1G7jE@$
z<)3iog}ZAmJ)FcR$+erj)S10FnkE|)`8vIoOp<?f?aALCeY8pWv=1iFxaMDzyAPh+
zf4rO7hy7TcNJqxsZS<2l`D$zNQM-KK7i5!fxm8E)!O8siWS+d3$au~7o+or=a{KrD
zSET{E_Q5OppUjeUu5Zf*K)+mk`jlaFUAPjS@p<3)C@#0(?|*gs_cDsosQk;*7H(x>
z@Pt>ZgJbWXyOFs>_jb?g;O?1ST)5qJb<NDbvaPs%|C7~CIh4!$b!|#{i!V#9+Lyj$
ziEbt7OTcve+l`*?@$dflS^DZ1+{yjWZOC6+$uGB;*Y)FP+s$shyx+~gy4^g;+PWNe
zvzErx$JC|Uk#%{2{p{~u*w5$Q?RT`cGJV~xS(l4jvo0@n%o4<1+;P%y+;P(C!ZXgi
zba8X`7hm+K9eQ%==B&#LZ%?Y5hV3cSx_a39>Zh;2J>BZJAy`=b_U!2}EN8BK`C8Zg
z)lWZv{IV{uhUFjrWDkT}{ku#rzExkZtLJP}y<nlc3%$L1b+x~{`C^YRd;b19ckIG}
ztmknhN6c1IY$dW6vRChRQ!k!$)i1iO*Y)LR+}LZ&8vm*`_CN8ye&by&yX4A?(d2OB
z`=iM|`}oQS)A9UddYI@g->@~lYz1$*kDXq%weIR+qpU~`?aIHr|M}<t`DTpJJKDRR
zHG8D&H1%GcK268<xO`TQin`jPq$t7Pa(}cxYDPC%3^z|iKHMCoH})>_40sjmy3~K=
zXZc?s*#ANT44e;Do&5{E=~Z3)OZs!aJU=J1`DiMSoE{y`&${ls@0VZhpXBGalf%7(
zll^gWRp+%|4!ity<N4K9<zbYn&VbNKt{rng9gEl;rFP$jYMVh~9?z${hez|>!;^!9
zjpME4>XXC%7>PG*CgbV!XsXBVDBZUYk6%yL$y0Vizd5+P=#%cg7glHdywU!jjt@t>
z)A4LPKT{B#v;P68E$@GFLf-HHckpxW{+HpIPxg|N!<lOCSJ1ByQJH7ElNZMa8+%8G
zvw5=jd^8nx{KdW*6yf4THYwV=TrJxAPY8>jL|E|m(X);IDkm}a(Xi;j-dtLKvA>mU
zn4sDGP@39GKKyAbx&P$ey{&{eQ`+nCPOkJf|NZLkXMYzx^u~=Fo5{1$<ltmFmOts?
zEhjzXUlk9hye6%!kvYp_a8-B0?8f2K$>Hqk<{2#>jHX|YU#G=`>B%`wK0kTA7N74u
zKRNt%xlNlRZBc(+H@92iDzOf1p2MHZv)upsV)FFM@~efv55|YG0LrD6VBepz4h}~z
z#`7b&*A|5QBJ04K+6)UI-CDN>;*Do6h7agw{*~HT`?oIMuctrKw06gObLL__tE1M^
zmwhjP|I^7=>-xH-<9^oa^^-{3I+YuXleV}*;#~LZLzeK3^ah__wFjQ|1YOFPc>+uN
zVvk^HU*-w6#MeDzMVpC77!Ch+e=KI#_=Ed@e)8G&2c?r_cA+%1m6%wNe{}@pU%Dy(
zh`KY|Qg=q^?d+$1Y24}+rfXaj93RgPMqkescT2%;na}nQac=N0v#lMSeLFecojluM
zPd=M`z5D&>;OnV$C;9M0kp~BpZ^s9?1%}TzH<SPV@1(yPH-xP~fBDhzeD}MPk=S~3
zdFjnfU0bu)kkI{jk6+Hmhgki?25`gm;q0ROU?VPg?TQYN^u`=W|FL`L;r9JI8$;9b
zhQn8n@7}rl=k4A55AJVsr%jQX^Me@YC2Y$2&rKNX=S*S$u?p19<d2E2g5=-Azkl%G
zat~7}_k?GC6xIX3aE)9ME(n8iZ8K=RcH^~vki(FjsyFMZS{^M`oEHY`=C$MwpO5C_
z<ml-)a_KDjVKRT7jN~6V-({Aib4Sngv9lY=wVSIh5qlNq5z)o4Bu>n;L$|)1AT_N!
zFE^NGoiXn;S$P%jGFf@2mm4DcL;(Kwr5Ak*4X<(J*toCjwG&aU*TaPUO1iJBWlis+
z^jp{YVYU4tZ(&kO9<0LZ*?7M9e4}56e;H3F&wflswi+j*L!7eOR2Ie;554{AWcTa&
z#^n2%Epr~QmGr%q9v#faLH7$z+~GON?D^5j!G7|5^!+&b?qoduaU{Z57yQ+}+tR;>
zYOrXR28|@_MfrR4=IbdXCIjWH(L?0wvVwA!k{TOX*vqPrWXx*che70BSQ9aM>;HR>
z!fb7<%9Pc~Ob(BdgCh|p(`5W|GMm5d<VwHaWOA3DK1}Sh>3FP*`|@s5uMGTeaT>fW
z`+xTQ^k<ys|I5l|#r|*Pv-kG@&;Rq=-`o_9|K{wu?38E8m&w(azx#*Gsz{OD<LS|}
zS65|AUwxTfO}<KQCG+Rw!z+nwyyK(kJm_A#cOTsU^!8OQ{=<BFGTu7)TJPMQjYK6M
z{~^74<=Nza!%q6={CT7O&*8OqehlE8{SVjq^8Tkv-246i4t~z{|Me!Zo~Vx=KYVic
zF~0oy<_G%ocQM<iCwuec__x2^HA^-5#|Qm)AIv`(?d_esNN@R%UM~DKi+{c9f8~$)
zkH=$qcCy<3@az26QvW`F@X6n_ck=&x^1snm^6BS~H<SG%>y$jud|I&yJ^E*cGw3_@
z!9O<Uo7Xe3XX8KltF3ikW6GZO^_sim%j=bW3#U9Me0|z;`mZnWoU`92Is1K*3%^fj
zU?WTZ^<NVud2lx@UUc>~OW&@&c>T32m)ezadHvmab|!8$y6=}2+F%vvY8h;VxQsS?
zU00iVdUSL^T<cbBd7kJ_z+}<|H~H#@+Zun5D)MiCJ0D5205Vj*ih?!Tmrsv3KGYjF
zx9sv>ZFNn>IZ@?#dpbQj93P#`5}MconsH*#j5Skt+S5n-{^48kkuRkq{i7K3jrrc@
zQh&rcUHJ5+@B3A{vt^@Vs0iuXOZ#ohy~bo~<u!dLZoTg4*%P<c4U@d<d82g6Ih>t5
zz5FO%|K}P-dDk}_McFyTl<m=bquD7#=YAKf&-ec+U6)o?UOM^BZQsD!%_0w!ci7t-
z&t^A+IV>A!c5;BRdD<{p-{|<itvvG&Yc?!Cy~Rh4pFI<cd|~8Xck$VP{V<)($Loh|
zcE&_6U+Pzn>#DZCG{4B!*wI(X56>qD<76Y<3=a38HW<opE9udGo2%}4djq>bjrHG(
zRq~sSd!H$!?ZKzJUp#)8+`gR@c(>!HJNF;!r<=}OzHzlKuRop4!#YZij>pr%gE5yG
zKE{d}@8d!E?f76ezT$L<<*qLL#uo<S(pwMrC(l;#mhPQ_Z1<qXGvC?rA(s2$Z(sP0
zFRuPp@nnkLKCKU;5CtdmX%9bMe(rFKJrED#Tpcy?#N(@HJnN@ENO<bAep-Ika7+9b
z2jtb?{`B&!C&?v4eB#tWT^Y}K)532&ZslpO<2v#ScWeC@izvH(<kx)m)4B$JCXdvc
z*nhIQUW|_4*#3jKXx09!)A#n@JNS8R{FhC-#w9!kK=g~_>G*s7TtQ*|AF4q+bvjdZ
z`gZd1&g0MTe6joao&S*E9yw2Oc);ZF*^#vSlz)k3^fA@Y{=#Jn`ub2W?UBdf(XO}Q
z0v{N`U!^|w<d3&w_Xp?m$9l8j!zVL?hmWVp{Q2lG>HX(dEi~ia)_VBCtFs~aeeWmp
zg;CV7=bH}IIYXn+@9@BH_@KXGo?Nrv2tCBl`nY8e`(87qo4y%+r2W2?41>-E`exsl
zP4$h~R1^89f0toWGzic?{_F8_Q(MV5n~Fdm=BSShJRKv^#}*PW?72k+tbrboexZp8
zc!2-i^P;012V~7e+;Uk2x|zb&&xkZGJu0RR+&MU)vokkwaoR3BV><Qk=j@l6v-A6+
z({oyHepT;sS&8xN<bToXyLm0ynX5hZ{rKQVhA)h$C{y#v;n!rr%%6{CXuqBu9?Az6
zCBEPh@RLTj<G`*=W%7+X0zREgXLA$%bi&Q*H=^7nqi535<Z8U0!+L@#`)u(>e{Q_R
zHD$x47cLIk@Zi;98v~pg9qxy_zPBK}C?4~~jmjo|y}jq-ncfqdyJ5F2j`t+F&NA%B
z{Q7kn^F*d`V{+q0`c=64Lc8^?&XO(?`LmxC85cKI{ye`>-~8mAt4O47{!Y;&PMy5-
zSNw%0@Zjj^+gWlvn(56i*3MkEZl|0j7MbavVX7846z}S0YH1GNVA{^_t1jg@b*Fci
zJ{~Z(ad*Gt!{Rku2u>Jpt$vIsUh}bZWzDBL7VEBlw&F&36Z?Pf+vgWb06W|MR}P>R
z`#&w-$A90&&uiQNhvOf1_s0k0xtag-*?zz(lQ#YBbNNI5d&>ZVh7a`czy;CF<<cA|
zFxhzWmnL}g>1;j~3A+1yG<zPFvHSGf@sE2)FOD}CQL%kvKA()wtgrDI7wSe~-8epy
z{X3p6_oM&#jXQFdTDf<0ayXZ5uN|F@N7KFMyVLQDqwmii_YZu|#tdMcMz#a)Z<N>=
zUL7*vmp|yQxUq7{jjKdZ7a#CqGMjPs-Hw;3bA58CvX^m7=75QAT)nlZ8vhDb#{wjg
z3v|dgbZ#70GJdn!p8-LHD^B+5iM-J~d?Zhj53g<||M_HN0J|?W-6Jb^bZXb5CwmAg
zaK=8??8kl6nGQ5>%ft`WtjAP_U%cDvlsBGylnt)0OTDyfVD+7Q^z9>E%5VPKasnM4
znld!xqRQ_J%U_S=<lC*}dwvTr{BPxyzjp0=z1&~@?My!Zex^F#w`w>|k`G0ZO8zbR
z@Y@^5$sZI*s|^j`=KAgT3*U0>?Rf3NUn>jTHulESoYBSak2lw0wB1c^E%VsfLYG5Z
z(yzr49Qrj|c!%>YcSv$sI(^OpLpJ=bA<cgu|2WV=XpoSF5nf}mj7fJbeeYWOUR(Oh
z=-_00c3ZEmEU0rU;~Aur$3gdd^3)*>f!_iD{7^;i`{ioGYX{I;IdjgM4R`*pU$bY+
z;Be)+3rh^3!3esvybkwX>Vc=o_e<zGLBpA#P(}S_NW|NDy#4tjj#G$?H_y;}`uw^t
z?Ut{GkG_+AGJT8M$J<g0SLKQI+zE~2#SAOX*CahXvq$&gx`dv^^paN>d35wDUhz&O
zb4B!5M}Nf1*sW?L$6tQ)mHpRQ*Goeb?po~cdB>(T{GrVAhtt!W9whwQro+~@roIb8
zAx!D8q0hKwI_&=SH!<ets9!dyZ(~P?mewks-XZu=?^?RRe6AVs#Z&b%qkh)Ga<N$*
zj+I|zR?k*HE-^|@`&iTZ8xPV#^SUH;vK6g58;=j2>>MP3m~WbR#z!y4FL2^<DF)JT
zY{=i6J}hPTm~{D#?rRylFe)F~mqV{_^&ZXeyl<_YrTO^ICWHAKNz31l4%E&Ze|IuE
z=>4?z>~HO{-@0Y}ZFHL|5!&&W-+l#L?7QL9#RaKD@vZTk^+UQdAHQIeC8GndkU#2p
ziOutE@N!E3?Jmw0nXvxTr*ieT;Scq4+J9fIkKu(a*~h*XU<x$PY{#xBjLo{?18J*!
z`pWuOTx@NjrB@d`CeD)+lhqw?aWQcrDKS~x02db$%Nu@ju8dfB$9W>+90~Ed<Fl}X
zFCrgaXLuGG-w(QjHIep;!_Vk6v0Ke^vQEJ~MY;U#$suVXwlpv6e2-pD?>0Z(2j{O^
zqrv|3WRJdy|9^kHH~R4c5rDJ(|2b7`SN#7){oeooE`Bb`|0K5H<i&XR#gQC3M~9O=
zjIfo@j;52ZZ!O(%a=bs9kKN_r1G{#a(kMB!uV>Qg-nYwbi8f*nbYH3R61!mE`7f`m
zbdzh=$b`iUHa#beZzuF$8yWg_jeo7Vg*W)<BN1Ha97~OhfDQNV9UcF;!EdNW{o5h#
z`fmMZUB8yCXx)mt($#Ng79Nn_L>x)S;x}U@*X;+IPsjn0f3jN(lr`{9$T!L_k=|OM
z`dgr)fj;K=1QLqh+)nft>EQm@`l33Sf3_$Q{!b0aI|Do%G3)GFf7=@OU7w;M4Qd=-
zFHh`pQw?VqrF1%#Kxp8#31bsEZ33T-o_*_h6&7@kvmUv~iZndXX5q3=yrSTR<XZB0
zYzPD33Wxin>3(wOi=A0=qS2d?PPyS+_YS0!X!eJD<KzeVdvy5!%(aQVqr>@B_S$?8
z%t@lk&S=jPDmKP1_r}Nb<mrzI&JWJfgCCRq$!zrWfG5~J+6*eCL0sk&nKf2aay&iy
zdOCWsdE<(LAJp;Mv)q{LRaYyjwJjc0@P&23=pGNQk`0}v)7I1R`E)w^yzHFkIb!}p
zQAxOCtKU{vxBKS9>YpF!Z^{0N2Fdz|>cix16-e+J?`mup@1m#Hv%w#}cd2O!cV$7i
zTySRgeu<gc^D{FHv%dTB(QNW^_XS6!$k5jr1HbW?TKVvJYe_csgS4KnWw{e?b8y1b
z&KaTQ9;_LuK=ZzPsl~E*^ZK#5*aBMW#=9S?r4FnaDS56RBgf+PdV|)tW5(p`4ARhp
zce>4nr}h%?;dt~$FMj+_`PQFY;;r2qr|-Pes~W|%A;{oV3Zp~}1c`X;jJ7PQ7~|QN
zHlq}EL8Xw%IhgEC=7Z`m-VYkV2NpsdGWKJD!XB=uxtY-?ez5RSibGBp#>!Vpu{CJ9
zi{B|9Xcisjt5IKn`DSu+S-*@fGE+YnTAzG1ps0rsO%IU1$O`zt={m;wwuT3B#jSm|
z0<;_@`rtzA>Vvao<Odg$lCcBwE`&@suMYRlezM0PDlb1&{jJAFWcMbcHT|VWYkINK
zdU?6gdbv1K=^2Cd&ffUt#ol=Ta&NpJ-}sk*)%}aT>TKm*-<3c(e>`XV7u;;`Y*6E9
z8Dd@UiR0T3?5iH5=FT@en4r(S`Rw3i_I%Oo<!CS(&a;lVU}PT-=UZsDKihB*1)g5}
zqDKt%o*Mno$aMYD7e4O~EWTOzC|x|?7QV~k`F0xN?|N@V==z?_)_lQH;d_a*qK`#8
z^2zhP6{oW9xV~TDuw8gnci6h=e4EGZMtt|pf1Xp<4aS+<!Ver<pdYzY<>u;eTx^if
zT5tN~s|$>#eV5m+z~%MalZu{3fBH*Lg36Y2h0|F$s)g=(1l1Rhk}S<j$2ks)Zjn<v
zJ%vkqGrE6gUrI{|nW0wO`1ok=x&Po$oxo|x_+R=s3m-d2v+*{2Dz2@;|IeihUjMDZ
zeQdvv_7CLsF5#38@_iu79wkpt4)>lzjcu|#uxD>2$D@;(B@yk(wcqMf`@81BojF`Q
zm0lc87cPe`Gx_qx{#V0RpB&2L=990V&&m8cmep`&Aw6t7ER*Tv>#xVtnZ5$x``$Bp
z%3-m1B)k~CToLWj@qMNnI<~MzxXY)#Twi2$zlihugPwc+Fbqd*6xf5OmrfG+cp@A2
z>%GERp7cwfO+NU&=Be^LU2In`LA`u_h(LGRQq#kG>6fAPUhMS*OBZ{!1r4CT^|Zba
z5u4ozL!I6@>|cB6PmlurNG}d_Si0Is@^49+{2|%C|6u$6<Mq89Xt&m3QNCD!N-nX{
zhuJ=d3w`z)kjaI(UJzzMLRo6|5WNe2nT2@a#$s@9F{U?QhK8FKW2$VLrO%h!U%Jjd
zytUfinx;a`bg{kVrux9@InyvG1%c3e82&-7)9_-9_pIG4D@-N#nfq#GV=dge7WJbX
z^9%iON6*mYaCk3H<tC^2)3JsM0t3veHB+~4s4eP!&QOQY`0DC9ePL}bF2;Q0Lw{Ow
zDEhyBD;j_Pus=^j5ZZk|+TUOJI<~__5UZ|H+dtX9d*^Sv_aE=v{r5+^Uu-|z{r}th
z*Y-AUWL+43zDmEMN7+tDo1)n~(~7e$#a0rnv1M;bp2?adn->o)i8e*jJlL|c{Qma$
zz6(G%`bbhIJB~OLi|hspg+igKP$(2$y?nZRya&tOzHif*7S^=2)xed^Xv;bc0Dem2
zwUq?y6(nq`h`o}GbyJjax{ytnD@md@jl=xRem?J<Pk^OiPM=Nk%ym)XT{FJz_t;p6
z(tu^f2$p$kW`Z=Fo4SXNuTFy5sBI$7!f~&+kX?Qg1@`i21uJ<p9{E!;YAffCp1*i;
z{L9ht-plp@|2?90lK)c8=Tf@;ytZY$ew@S&1ufj!S)b9)x>-9%f_c9JLnqPREM2R?
zd1q;*Sz2Gq<aMuo@-9)Vz|j{QJ_@1e60KTAuTB-ouidgu(ba2&Yg^%tg~Z2xq+puX
z&;Mm2v~Tuv_4x0sn}5`2N&eSTso}=+zgF+^zuv~jy8kEfS|7@YM>5cbVCDD7-MX7Q
zU&rSv^Ixw0lq*C>b125tlORjxf2~@No&TuS%6I2KZsW6d{wKMWQjfeOfu=PY9N*Bl
zOULfPwQ97PCk|GQHyFy-o#husTh}wjcYlLD9hIQZbH?wyDI>0;aqS-PI`t%MZ7|g+
zsryiMvc5>FK5obu?(X|}pe2mHbM84E_1%KwZ_`}n7G3{lZre`u>3NN7>FDK^@$(Ls
zxWh7JEXVISRo#du7_j()i<t3vsyG9vqfl@)chrfNy%&^qH@Ds5I-6MnGrYtRe1VhA
z+=gG;@-k=76jov1(rWNsq(aa=b>cFCT)`4b*ib_Y-I>|tF6U^xJ)TI&3MO4|Fw+xb
z%hb1Fo6YAa_UX6~_gh3|Q=Q}<I6q*-LMaPkxG8aCw$@i0Drehyx%=~;b)^0$8EU%|
zHtO2xOq8N6gZq6+7fo}k4$+#op*~vaIG><^gfXQwZl~dlHUC)7+xl&&pH_uzN(Z!4
zc}@~hRek@Y{p)I29yqQw?RCFnq<O4ZSnB(*SDhWs12Z@>R4WBZ1nWP?_xd_V|6xMr
zjsLvLRU!WwE2017H^VW<T4V{%aV5675xmfDGrLcTmmsL7ogaSST{i5-S^_deQOW4>
z{r5hg+ZRtmFQ@TK5V5fdgx;&-iX|g_sQMTLKoUd(a48j$OsAeqsbnnTbLodAc=$re
zLMRKBM3QK3(bZzK2plVc2n^~eJT5`2aI4GcHuP*!q)3=RuT)$!aEnNy)rQqVM8^i&
z)5{ke_8e35A~OXqr<V3AMV(4f%hvWF1%l%%^BY}d{ReUOW|aiMOV)qoQn?)S|CI6U
zZvA)jpXK+z4o<{d+y3|dL8ZLum;ZkDVfOHSXMj>am`l)N`j#9b8?0d<Rtm()fcf0;
zuUr}woZxj=Jja!EdRL%QMxb?-+RZw1XSE^sw_W;(umw|+3bDz@&bF(x_rv=R@64O(
zXjw%K)Xs&Dt_y<UjUN^IW<f0Z0H2%<hJ6hXf@(0&m8>c(#64>I{TC`PB<j563QuR5
z-O#)Dj5X3G8Qb9?qAj+^Wu{c;LP&Q<tiNRZP0eWV1)=_X|EHt<pAUAQ?LEyYn0aIV
zCIjoScaXNz^T9|h8Tt#4`#+YoN@J`ubnf+`ga3WJclaDl_|T<kE8)^73zfZJ_mAU<
z$V<LDI^I2gb!1+107^i$zk|Hw=Y|Tf-3-z}#qj2EFfA2OCU!y51o#@$PYL*n1*>l(
zv~DsKMk(j%eHcuq-q3pon~b#i_h8f?zn`f)3}Y7&uCb#cGxcjzZg-!hpR4G9X286<
z{&(GaHGcnlx$NHQ|6BMxyuZoAPG<vNhcguaX16ngScClSjr*Q*gbsK|$#l*b`J?B*
zVA#DWtL4t-^U3Vd!-p(cJROfuhhDKaK7TlQI6FU+Vh{7`{O>^S1!k<g9Dz9Voag(;
z&T!E4Mv!#|7C^m!|Gqe$;OTfV?RmxV^z@-FHG6nIm~ZPJ#mU(uUv!)w$MZ7>Jd9;)
zr{AA?bn+4$wEDO==!lk*BgS;v3P|bJcf1q(-C#yFgTBwl(+^nO496;}Y!^&17&*I>
z$<XV5C<wffx~nEuYuXu|dLZ6(eD0J>Zi)5rw+!5l(SPs6KNK*Ye3&xy-+<FSVGM=C
zlFq#I#v7@Khes&Ha!*VpC)djh{A~h{V5kefr0$#vb|5!Fc0V|$Kyc1GPHC@j0GZL3
zq0wlkGy336yy*-T#@#s-g~|p)ObHK#=VXl8_+<W`SQ2ZCu>EmwaSrQ3X=}3242p&K
z!W`)uTX|`R=OzeHUkhN~%E*itb2&RP9mt`&f};#fVq#to2N-zOhztSgA%hIiiy4=T
zK)8UFDfA!zdeXniqB|VS&I&wB4(QzluPvVOp^#nJ@*$Vu-cXPOkO4X%oepqR+>3@L
zB<@_3YsM$<&oJbn{tN`G6DU|T>`Ar$800HNdH-Y#nfqSSjf|nS<usuga&V2)>5ku7
z9K%eHz*;jQCu1<u`qk`=`yE|RlaC&&kuVybnx0LG1``0y2QZ?z*(;|QBRwt}Jbu}8
zj$Zt9{O{evJ!k*Od3pHaU;9t@o;q8*NBB((Z2$O|7q5;TlsMcyIQ}2!#ZS)e!T&h_
z+&_3)aQ1$EdAN6U<h(c}wEgEVpY8AA@&3V+XRn^_AN=h6h^hxKj-6-w&uN@F$1kK!
z4Q+q#i14289X|O5KX-rJf3|=8KQb48+CM%Z<ey#~I=jxx-NWPkC$FCE9y%{y9lm^V
zv<D=f0`kHB!B2;1Xz%&n!Eq4{<C(MfFZ^(he%XEYjI9ye?kjZTkcc`@UcCID!~LIs
zId*<|@$Bgy9{#uo{C0nQwx?P{@18u{-G5$ip6)*1{drGny#OHTBugqZoqzwb$ER#{
z7yo~9y#L~W{dw}@;P?<f3+V3Qv0wY&`$u~PXZLXbh?M#1@WpfKDXEJ}F9Z&%9_%SV
zq<naQP=vo<9qsurou_-d&j1dibr2m;4)Vjz&84fq`G<+eg_}28oYNqtP;bBMOb1}1
z_C>SamP?IePu+4e_PAc<<IPV_=<z&A)<$bJ^p4jq<-(;M{Y@FOGf0TunUgaQiD3`g
z%Sk}<kx4+Ale@B0$^pZmAbzBjRDFq+1f@EQ;rtONWn10N*3ad&N6K{5&Z{2oue!nH
z270_DZ`ISS6+0i)!Q-?_Q99^K>haccl9<5jfA0N{T;;w|uGI~lxMm5{M{UhA!+Hvh
z)Me#OroBTUNrb1U&2=%WbU%QznGht#W<Eh5I?bJEHE(8I>;aJb4sb=ue__&6KC~&e
z)P?O$USY2473})?jN%|eXIKc}=XWBT59Z!?NwBsMjc%5pT<h!9%;>i4UmAi+46b@*
z5GuuR(gidMr7#x$<~q$j2k?D`0m}w5q4cSMe+sVtQz+w0u~^i<_^n*4ebRg70%+z1
zd0nC##iVJl1%lR8l&zstUhM-XOoEJAAnSOad@v|vwN1l9GlD9pXg4Sjhi!FN11Zzv
z@fbz0-v;-6$}p`iKtx8&TF!rRGA5ail0^6ejy}J?VNbd+LIoCE0ju?qGLv`-QmWXE
zAeAa4f}9MB{gegHa93E()6NIw*IG{~vYR$HM7SVf-kM7nbQkqJQNypp5l99pV|ul8
z_!9^n8vJBD0)mH;SsxdB)KhWMV}7l;WM~A+T_xN&=LH<ISbYDzBPVS16IY%zttVx9
z;@0s<xwQ22iaak@)#;WvmwYogp>gxydphRv$Nit_INRSn$YLPh^9QMQQ7^ldN~z+#
z+Hp)5GX_3XwOp?^%cWi?yQ&$dP}N?qUM_d*CqY&1Gh7ktZt0}f@Vv?;u<lm8o>wnl
z2J3pIS?zY}EgNg`S!P)Yq1<H*mx!r96PzRx!en2U=#qQ0T;9Ur-ZvMLdHi%d=XaBF
zJjn(#Ok%a-E8e`F892>K$*|&#(er%w*ES6FpAR<Ny3+5oT&VMm_I4idq?s-yx4Qlj
zF$>}v)Gx2}sF}(MHk5BPeXI0raEi`{fsX1&2Bn06z(9B^8+9o)bm8oKfox&823%!{
zF9)m`bXCB*^v-3y>lTd)NsrZfDe_!5db3XXtN4FkGiLDW{@+TyRE_(8@8bV&<+E!3
zcL@}2t3zjP4t!hEU2&<E_9~POK3+&p%q7=XuR`@DIKwW3im_**8fuUEJ(U4>x1<C_
zdpL3Ytn)s25t8YuqJF1?M<;5Q$6!MGaxdEg7Myg9sERaVNnSDL&^ArYrv**4?qJmB
zrPS=Q`7Uc)9U&%aAZrkE0?e<P5T9$_ZT~R!PQ|{9-|-^Bin*M>!p;L~Ud|r+_R~*?
zdwcEwes%CqKi=xOz95zusZ>*JnyUcb`SFnV&R{he<kewf*3-dw6kM8DNI*_m25Q@V
zKIJ}jY)Gj1!sDSbY%CoP#e0dvq3OL46qXM_z{P3+76$bw^%q|l!57y{<JdG^iMUY9
zNz*$*N|UTl_E(H2-J!`cWv&9)fj%K*Z=*L~0N@98zHug7FQY|g)bmvEzc(5$PS5n7
z*WBrkz1iryxpUrm!&~3ESwolpf@%D85-4V+QX4|U@u@g51(2k*=SCAc#!Uc1@f(MA
z+j|q7Fi-=)EgRyxO<V&9$u-blObV9C`R6g(=Yx??a{laG2t)h4HnUG}=tw*U$q!7p
z3$qExpm1ABvu%b5Vg}sg*E{Jo(<=omlw3WSdWkSh;<2=v>6O9_CvvFHH*7yoDw|L&
z1pJenBC(@~XO<%j(jdr3s75n2^R+qoFk9i^xfV|9q=jifU}N|hv#Xe8N1vqj@gr}<
zD=EGpyO@%(aD5SS%vD)%zUzD!#M{5*M(cPn!*mHmRY;6szVyb^>0*L6L6TH`eUxr5
zltxB3rC)8k*VUvCFx>2?P(DljMrXDBXlKC*O($tNa!x=a9+Ld(UnBZ;4-UNdS;$ie
zS;176wAm7uo=Ny9fk`^Q0Vw2mhKODS<2(B(_mP@VG1OLQhb<@V?Taw)GzKO;<4?f9
z{!p!ukJ-K)?H?g&<^1pe+IC+;u0}19p(w@ePwJ^W1R;_I<8Qkv6MEkdO{C4WP`eCb
z3Zr%9bcVFmXZtT9E=|o<q@OPPDd~Qr02yT)gfiRwLuWi-|4H9K;eoM^IAM6o;>ok!
zqoZwiiFY8teXWMVt&pKr{~;^o<l!o<k|1j}8g7D(s^d0bjr&peiW;D4`uZ#WDq%Hs
zKAzD(*Yig6;Ri7A;QU;+pL8K*F>KDxrUMQae)%Rx#ALRL1WG4(rqB7&<+bDIi?rVR
zGDqsGSs1+C-lk==<M4ikF<pvh%%G_bW*QHTRXpy`OQT4Z;W{4n+Y5zT+?35Cv9f5b
z>b`LsIwocn2#<SMtNX>aVh{{=*62i-Gz=;LlQ`^L^v<V=w~<*r)O}Ws3I}M?fm`oS
z$`X<7M)T-ceM`U3!(GXo$)db3w_>k%QY)s{v8^|Bs%&37;4UG1w3oal2z4gqNFNy$
zWRjW-yv$B!;A&C~hTBXWXQ1K6ue;Gapo560k+g1#>!XMyvfVFzZ!YlAX#eAQ#Ef!c
zUIEZf;*^XP!=uTt)AJaImbRQMsKz=cbEbtcDS$X<Pfli@6*WM6Nln-c#O!5OBID~S
zD*QmgJZ1qu#XAs4;P^W;Ov@F5|1xYz(x3>N7&gNsHyE%@!l0y9&8TIMk-qjwvg^w-
z{bV?PueNcsX|COA%8*oK^_M{IHLZSN^lnRL{L)3YyTowIz}C!`r)ZdWG^?VHF6aaW
z{ae+99L!qqg;684n!9zvT!z)ujJf1CO<-jfqEA&m;pkJj<|wgfSn<Wm7#Z=|Ph4Di
z|7Usnr=|Iy%B6Vxf3;k>+yA+RkCp!k%f;WEt&9KjC#F_+=!t&$w1vsGb>VI~e>gjm
z(0-FS6yk}ZI$b<^Bx0%X(VLF-SIy^-C>aWG7aazp5AeHGAoIxMt<96c-Jzq;^||W&
zS7t``F7UGXpK$)SS}on}|K7$Y*#A|#`FDxk{$8JJn*W(rz*7D1mSXe2R&U(R|6BNc
zUi~l3b7ug2#?O`KKYa>Ig1|1L|I6jO`F|UqJN^In{#?)e&vpV_V*OvOmaBL3{}w)<
zSO1fF?$rP1{9I-J4^P`FZ|ss7@MZJAQm@D6f2n>q|8L{7cK&BG=<VLHb3U<mbi>)T
zviBiYy2Z=7!>e@pvlkJBKfA~M%lcf){9m#Mux$QUO9}RW_0Io)8y`#mf8#utIOoMw
zXZ;yZ=0!8cj?5kg9!MsilaoPjAc^Ii{gE?~8_|1AD#Pp~<PbocOvk6w&bjkmqR=Hn
zkJI<Mi&IBQ1eM}Vf*E+R)SxpoxuPY{3=Ik=H|@-sZ0H+eq|(1gXf;#PY!fEKpqVYt
zMsF~(5KO$*!u0A<>F>VYxyt<4iP4uH0n48MwQ98*yZ@omsNCiMx{c4;`d<%v+hcai
znSW{^bfuwxav=3GztOloTlCJ_ygnfH#xeiWN(mY;b@L_}u~zu1lpJ#zjYr$w#b74c
z=Lve2ai8EjZ#t8MFQIm~N=5vSF|_mP&>Q9S?|fdPvG;QC@OdEv8@^}qw9b6)olhj=
z)BEXQE?L^8AG}O{dpsI`Q0XEwFkJUWGkKKXc1gPZgMaNlTS_`9C8QgSCJPm&=^vrK
zzIzfCVh^qpvG5fersvBLjvyJm_+f42#*29}+2Y7O_+pUDOpqVB)x)0V)+H@&?2;DA
zL6dM@%fr}x9oZL>L<B?RB9cUj5Y%2<G^{`hx%OjKX?Hs`3MAJh?rDhy$gXsIN|K1w
zQ#>jsR~L~+BF02zqKLc>%9*M5S?G2Z(XBvVkNmzen(2c4vS_X&%`@-zNs^peabF7G
zYY^~X1imW>I_6#Jw`Br`<s6R`XhcWe`*uw11%&WRnz+^<u|Cro%U3oeR=UeM;wcK9
zbC9!$QG_C=Bmy$|F}R25{tBr2V`q^ic7N1u;{9)q*qSsB?iVuy8lLC=<MhjMZ)gM_
zykD_9Y2%<1K*c+l(zjb0s<n(ROm8$r1sAcq;+NxvG-G8)elYLVW2ahPst)z-Z`}be
zaUV^+)7f?C#xSXj%&SDWXwWqz!t2xdSz=A}q=1fO7<})iV;BsR&P<)9_eOn_o=~Y8
zlv-8(sbdx5Og1T+q$oW{PmZ67>uCRHeZ6wQndh0@_fH<D{e5#o#3ZbhoelN*JxxHO
zOn%1{N#t<VKym9E(~`~V?<Dmc_>a{|lx5e}g~rBiw5v~*2}FIQ<Qu=ut^E<B>KUG|
z9{Hx21tMuOuzy)I0o6Y_`XY)_9^9KqV`czrkGsQxoXA!Oc*r10g~g|sU4&bi*(HXZ
zk007Yxt%<#nD&m4B!y5W*VGD&(RpW*4RkUcpDZuY9uB&vz20gC7o+#kyxC>tg7EAj
za~X&wWzLp#Z$4bo%3?n4^p>;&wmO|I?b3xBAHR^vUY1aHC6O?mNXE7eIX|Z7NN4!I
z^I_(n0!x+^FJ2z!<^_7t&D^>MX~A@~RzkhVoDJO+zZ2BZ`7$5-0%41O{jF~Vw?xsR
zWQq|mrM$s1DzOw~D@C3ul&(=VGGeM4EW%>x`eh2of&*AkdkXj?BN*c$mQ$lpaLfil
zW?_5&7o0wit7eooY5QPBE4{l0Y`KOz8*vNs{$M))&_0*3S*1#Fv3F*9MMIGq?Fr^$
zHuP`@UJEVFiN#d12N5C}g%ut;vSevF2BR!3WF(OKn^DjgF#1bIQ6oFm-(}pUO7To^
zW(^IRA{YN;6j3^b|E|Jb)Ebj1<k&?<od9r~+kNl6jIuU6N{w0Pd@{_c5rW=ZoUa31
zPF{=nS_S-Re{nupt*FjuyGqfa*LmZYOw>E5ute#MCLOtgP-vwV;r_+iFw(=fYDd=t
zVr2)5f{?FFYNxvM&Hd9X6uNv`wfU{(B3f&CYpqPG>#apZu3yJeBz6%syA;1<l_};-
zE{e#d(;+&Va+2_6O<#|xs|l*pt*vzHbTqk*N^CJ3qt9pNq7=Um#iCk0XWxXY<;}<(
zRM#;)K;r&eW@d0xkDdF{N!IMUI@ypjkzM<@s_jY1iUY#>Eh7*pq1*9Dptq}nd%IFn
z2-45bfM+l-NN(Wn2dpL4z3!@`VyozG`2k5Kje>cA$|}C#*Y@#~-6y~7wGUn#?3uPA
zVHFDQwz_6XW5K8g<;|NpkWD-sZAj^^Ov+U<Vnnk{lq9`_o!LS4nzsWP0?89UQK0G7
z6G*&~vzSCk<ED{fIXBwLkq=bmqYbxYNx}JG_M3S_(WkjK>EYY(!=#IxX=G0L&s()q
zy^P%_Vw{U#<*j@*nP~(_DbHK44RNNPBnXtsvtr_12L?S`99h=C!8lFU<hB&_Vo%z#
z1vkmWY+->hucgf@ZUQKo$M+V@2$tXRv+wU+cvHD>T(igiQ?NttLvLs@>#WkdtWNZK
z1=RRM*iWgkj`b@_A*~yhYo%!4{?cRe({M54-Di{ehqW0k3-=SkO>-FQZfM*S2V%9Z
zm~^97hn@&)QLR~GarHh;bjrGOdsc>JQBI+sx5qCmL)9Q<iqWJV1oBS-NNpBH-$w9!
zdkH+>UKP)`OYk&f@-In;ZY!=?iegxOpp31-3{q@w=W`EUL?<dRlR%{G#lfZO$Hb{>
zQ*r89I)5<(J+DN@w<P0RLq@|}(=a<;6N|-iEv`UK4d!hEHK*%RQw&b&wk)p()aPCA
ztn+RlTg&R26T1HyBC1i;S06JMq9)E`k!f<Tu_2dX!=)NwjTvOgXc1`=fwH==epvp}
z#r0FAvaeP5d>%6((=<JkeQ{ZyhL_|j2hn)gH;LLu-h1D<m%Kr4*$+$B;_7c`HEz6y
zNznr9?9qyY)rZF9VE&J-gT;B*n{p1RN4!o_Z38I?rk`T`suIl6n9><VyP_1dyJAr;
zVawJcX37Y!xMOP}TlykV%;G!M#GoDW=c@f+<M0W4dYSpE9eKu-!{KMid^z(2s3NN%
za)&sdc_+GjX`he#%x(qswS<d{rDCY*ttb_fFurUk#pXr`5W2RVvj3fmU0CFVFpVG?
zp&Fy1e%gKY>^KwI*`n9;X0v2ubKyWXURzbk_Sb!0{6);ixeGl>8r0a(P-3v9@Qd#q
zAEjh^B=EC>^Ef&gLO=bLoRz9IGAYv3QYs@yI5`SYGjhrRgw?S%3@eFof2XW(^1?2i
z^+{S06`kbnEmw+0I0ZXLA<1NKC<#B<4+Oj5!8Hja0k~Ws1?`4Hh@v<KB7d#$3P~Jl
zpsXZ<LinW5UmLfpPu>uYrHjUi`rHbxYY6l}#)J^I;fJYhq>A<Qd#7C5iLzWOkR(B2
zbLuiFk-}XeC}t{#<->BKt!VVbg+s>NTP7B){;Vz&WKu>puU52)?X9g!EYwW}xGiB)
zY5nRErJv6_&)Ocd`girMF?B{EFyxa_SQ0G05@{3muvS(spN)~E^{<$bO#l84j5M2B
zcY&**2RviMLr*R7f}dE^C3BZuVJU}Q*Lf9xux6K!%$1`{TJc-GWL}s4V8%=@{jHUU
zEKpb>`Xwxp?Y0UrBI4nt!mnraP<pv=>syoQ;Jh>au$2H!-z+P;xTr-9*z@|{sJGTJ
zo|Il?bz3k?;+-o?kM)Mdw$WBKV=ql@i4jhvbyQHa=i_;2*go~<a@X%2cYbE>T({d_
z6L&7vBeS-jq{8d3VU_5YpT`v7V&-4Fhx@xfKHF>W9v<%gj}BS0`r^B+)$XJYY*4=4
z`+ne)4!X|lf(6v-hTW9C_E~Rby~^Ta_7`@R&SKuYEH~-qWy!d<n>3EsU&>4RS<(Mu
zoTSd<1W4EHB#lw}tMZXXd$+Pg`fGVeXJ3|sboSTrk6sh1zU{U5bvZ}F<Vg|@QSM5l
z*mVnq%lbxVmvD{F+TwrcjOMfW8Pnavy@TB&9!)zs-aU}BV_d@rXRG|<V{&?o@9a}q
z3G!iXj%}qKGu0j0`bITmS)Kk6M9zV&c*Hbzm@C|q(AiSe`j&r?EBPc>lseXNq0yX2
zMgH$AJbv}uKckUWf#)j<&dc4O_fm^m$F$V(hl3yY4~|wTKAbLCel(O@{k+*Kb<Wg{
zo$}4E6*@f<sk{oG@g%@<UPHH5;J52$H+8-vgXyk(uK9X-E<`26M(P-caxaqDgVD+O
zx6&WQsdwrh4@m{Wyc(4<!N`gGBslUS6NteCEZ0uOX5k-JN#ZCQ-b!mL!duQ(&kkRG
z4!$CGe<{Wa(<#hZ&vp-g-fKVKKaksmQ^qd@V3h$p(`j&*4&g6PetC89&(AQ33d7W4
zG$XkFy7%UWF-oiOK>GJFr;K5w_9~<3UCg~vvIDB!^<WknS>XG1+H&#fjRhq7tzLFb
zklC*4ZY(nI>N$7T?WMXs_R>zv#OO;+I^NtJZYJKBQ8X}9RdtvPZZT7L+d4w5nLeZy
zYA}f-wj<}o<ef7BlU^+Ih^j;2+HDvUBjDxZBhBJQcm(MWu1{(R7DYXDHzWHKCVr@Q
zmMNzqNdvVVDi=!?kX$VJ6#tU4mWv^|pl|%jsqH0&q}iyrvWs3e54*{!p;9Rp;%}l!
zCliX;$ruw}TPe&k;rscOl3?fzI0<i<F(dRibY9zjWu2~#KMp;yqRZ{)iP67F1wsqf
z*jtr1BCT5Di`gvpIJEVV@HQLCSr(4gqB!AQXd#o-aL6Q0C=fMy5{f7VW4{OqxBORr
zUBK1yf6vdBCjb8bmgWDhR!jAG{_nf{zi#KVH2=5J?9X?9Z6EGEYwsN%I&QJ#ZMQ<N
zc7HswUv2tuzdsp{I{@_qpPf*nwA(L#-Z$!H|KO+ngZ<<G+585xaBw1pA}@3+=7E!Y
z`s1@U+H@W``di+w&AB`1N&2VF{&>M_jJ&r{3&WiH<-iEi-@5*{^Uei}P_KlZb-FV?
z6I)w9*2QSg@kH4PisU-Zw$sf!--qjL1d4FF5T&c@avw|Tq0SGEf{@QU|E0d>I@{g+
zL%e!q|L&;z3Z8-Uy^XlYDbszEx^wdcDt|nM<n|2}=OUck6F`BG8#y6wo#QIYIj+af
zg?|8!Rqmq`21Z^g1lY|LF0qSaM(xwi`FW@9BNbl6T<9Znu*q_!Z0-V-d5pF$w%z<c
zbOzbEcM4ng!a8quAzg4Th-aB!B1g^Cz4Re=h_>SUq*bweK)DOa%4@N|`k@)#uV^hG
z{knF#s|$7F>>}<hmgDu>4)sy`jro*|ZV7+>;32H3ckYemk4wdJEiXOSL#+%;&7D4p
z&j+(!+ZeaMG2{l+FcSzEyTx*|St^%n<!ZgvEHxU{3QXc+tx>MjtF3CeRjxK5#F~6q
zE>+6SQmf`xD{i?}t29v`SVcbtXB$sj&2qzS0!Z0yx|One>Xu85X1QEx)GKJP<`M!6
zHzE36w+!`Csx~40*;)cz0;o0{Zo{oMTjfftUULP7a<f`*mYa=IwNY-lEmyFG1}VGM
zdcEFqtF21IZSi5fRdK7pt5mDCTGd*)CG}gCTC?Qhm0M~yKnjombSafujY^$2&^JmA
zP=XJu&3Y5+snM*pDplao6cRRCZll%$0n4CIt=g&=oO021tJNAFHLA4|nlCr`uv)7C
zX03r9^JcqB74^%-W~o_2pIrP~!|Q6b;8cok1F$O1N(t=))kcjEp%g1<$}Lrz=vc$$
zVz*Lk0U;OUXw+R$hg5N@#Y(-7@4Qm047BRa5+632jVh3>mN5*aYOP87)rw`eg@UbS
zz10FMv?}F-Q!AogtzItGS_-!Y@UNp&XuQ^Jkp*h#9{XPhx3tPN3}3nA@_%Kt-zb(q
z{wB(l8kJ_P0(RlUTBS;ssMH&^W}{M;`psggjB-tM06<$!Fjv887Ha^B-qu|h1yu}W
zqu{iPrAoyGn#30kRw_&g+A4zN<vIqgQNftC>MeBD1p_qz&Lx{P`ERu)PwS;B#uWX;
zxRh(9W}PGG7SW+<2|U;Wdt<OF=mMVBo2^QvQU%wx8bG#&fpyD8-bq@*<S3QUxw_jB
z5FAApJYQ{MwgBB$iS1P|447W%FW9b5{w&E;R0erL_iBlgK@yv~)nW|~8W^rxiT!K3
zY_Ep7U#+^*yGD&AD)O`rl9s>?jS{<r=|m9qB4#X2sYbm9ih$A}8=f{%tAS2{#8tOd
zZi1Oyj0P~ngsgHbnyr#>liTDVgSl!YjC%<cO9as@HZXLUQy>vJv?*A$ikLYd7U)qf
zS1UE}6+yI$EzBeEe+z)zCTAy@AG9x{b`5;rD7keY&!?DZ^%BVlW~)-xRLK)~+#+Sl
z;8xD;mPn9txmW|y>SQR45yquO5EaZ-(51|=t6*ZlY9xqCu~{#J4$Z2IaYRpQm{(;G
z88W0qE3;Jv`7k<o3L>}A3zs7Z2+fKJH#~0EL2yh9w+X=n)&pOZ!SEGI72rlvx!@c;
zZD1N<B&rqm9pjAe1knKPAp~5^<YpB}fY2xm_HJU#K-_AJlBX%+p;at58V$?^w^l+~
za4%U4kHLMQA{eFB!e7jB$O4cP@~4hbtCyN3Oehe}skp^Tqly_{!OUqiz=bst5oO3&
zAPcdHUN<mk96mgS<SIkNfaNgv(H9CCNEtGA1ydJ5KsjOXO0`(#7(=KvAORtoT<`^+
zg0nzNwAE}-HdYCO(xXwwgsC(^R}55(<gC<-6%K!`41_Bc2ndq1(tuR2LmWdiLSfWM
z7<md}V|rqWw(zBengjvDfduGa3*APeU|<vmA67^iia*G8h+=@KieiCGueB(kD_{{~
zfyWRrHA;6#W6tavIH2lcL?FK?Nr59L0|jOofHA^#$czSLR}&MHAS%#rAQ{9OXL^%@
z6NSNWWJ7d-!{M@<cv>yi-6|z1gfQA|f&~boR;-W<@sk7Ff-E5xkg(OJ7T8s)Th3HG
zg~V$?cDJCXnvI5{MYRDL0Kp8DfngovNOD%2(0nd+ZjIE$%yhx_Rj^jA)Pm?m=OFj$
z%_iqatwf0mdVxq_L$Gume1WG;P_hc5gC#1k04Pyw5QkX2K?ISpAjVq?1V}{T2wMTN
zuhyd7fWpvUEy_raah=kc42P#sG8N2RkOLE?R1@)Aqh6xw#u&Pky%+@)u3<!gDCbm@
z;OiBF01rX^)QJJdnp21%z(h1mz~AT!&}a%4P%Th^AT9U>+7$d$lBW>RH3}`T1{t79
zsa0!1J^_z9EF7>B)}1709fkx18z>3JfLvg!cuJ||LW)6dL2I;HO7PV=&B{=fE_fD3
z31l%kf~O<}BoKxZctb;BsOuF-R45#x1;i>fv`P?=Ptf%hiY}~`Nh_dSC%3>PgKEOW
z#5@C|;%S|l8#q8*3Clt*5(ITGo{&xr=sAoN3u8iISr5I<X@+?!MqM3p9gIU&3m|}i
zE)fKDDtHpc9(Wid3Zn%bX@C~gkgzbI7V090@f5UxL4($4Z#1cVh()7Jy@g?hu>tws
z!r-DKc#3&03<{}?u><~iikSnsPc;jkgPy{CBM3~LnlxW+LP%lmkd+(Y3-kf=w_LAM
zD#H@OQ}6|#Vl2U(5aQr+ir)sLGUhm_#;FNjZ*fo?O_{%C2ph^S6rhg7(-vqCE4Ngx
z)0}{jK@hEClZpXG2qqYLp@zxcg!sk0$GCytAdD)a<C-o<k1`fQ12P7hi<7+x!w<Xw
z6;K7kff-zKXA{N&?>B@n#jFI`DzvehRakxv$QeusFjpC4CLn4>-j7x%&w&nTvWeM%
zr!Y{;v|K?SLZ*q*guaJ4*eFptG|_jAB0*4I)M>#|BS8R^IL(?=MJ1sam<d9=Q3ZvX
ztzwl1CdRbUpa^e>$Z3^|<TMfpNHi(k(UBH7ph`;{Jkg>RC*0Wr1At(S8f{S6=9sEt
ztwDBT(ozWkK9~cl0EHnL2&3GhXvZi5O+2lV^iZp?<VwV)+#+|jYDM4#?jg#Us4x@R
zULAs#N&vGL^BCftc6<v~PMHLyOl6;2V&$|TCCfGT1=0+P8EYVLXN!8IRL3HtR;8s3
z%|$y9OE1__(D~#Rbf(c1V!+acw8l82c`zZ`h4^A|NCAjVsCI}>&P*_?dQ3|cl8M3x
z8XqR9KmqeNW#V8cq0lfn1qy^BsShy;eap#7vmDHZzN-lY2;{F)MWCSf5DzeIA*7+D
zX=}1Lx{tX{Ca3v{WmidnQ%TUA#2C=LE-BRr4uUX;IRRv8_?B6ZhA@U5!$nMB0YWj<
z#4>d`?&KG-&%tEieX*u7@(tRXpa4iu?MQnQW-sL&m;sGKEmf3p3bPlp2Q#cevl$W#
zCWk<QX;DX;U^T3jXoAs3!mNTcg+M|7FpoG;bsE~VaiQ%XD=>duE@fSzF05QIaFg?a
z`o2_esUAV@VX1&QQsOF$+$AVN2Y@9|5Tt^ASAy6P;~pzw8XTY{<RSox2nT1v60C@!
zhEb+L*B~x1Q;8Ln0uZc_5vxKMSfp4vkRzHj6DezCLBX{e!~)ldSSmtFa8kkGFM@io
z9Jzdj9)uWI3k^tpm{vf+rBMibqb7~g{)FVifMC{9NKn$DNRfI9b_AAN5H_%;gt{;)
zxhSfDU@!sc)D^l^xNK@sVRGRK^QpnbIE-bCQH@qFh7#ik8VlFqIRrZl8F3cWIMzhH
zLMacWfZot}!kSz?#w2$ssv0z!XcRPMj9^4^Sqj65bDawuRUFp0NEkGS=>Y*-CAnZl
zHtB{gQ!C-0W?9W6Sd0xW(dt#I2Cz#*Ryh@}Twt=c=t?eeN!yZzS*u9?X;3u5wu2gO
zxv~bMSzQq;rpXls3|&Hjd5A?%ODr=?0`Q;AM%aho*fJD5_!n~y5>;@4DF}XPHXw0o
zG&`zlHo_K!iOnf3>;=U|djw336)a6xXdFyB$Qv;Q0I(?bBy~F%6cy1tn1aPBBqYcM
z2sN&-X!*Nh_`x!(a|MgZ&3PbDAj)eLeH4}8G4Ks>snM`;DHLgFa8XAI2u=o%at({Q
zSL0fwrN#(m9_+yyty|Cm=0H_e*|6xy$Pnb@#unu(hYl7Umo$({kVLQw08%6-Y&yC^
zAbGh+q8e$*5*NxFQU|<P;(8PoOI4sWplu+$u!bvB=D|vpc?1KFZUpF47#NT{7)-Ky
zjnbRKS*#?8BPDlX!9i@h93qGlvW6&BFdBp!p3sfKm2y)_DOhU|Ojw-IJA%oFi7b!N
zQH(JpE?R9;iYWO4dkt&_@ZdyRpyX>7ukkT3<RTFx4Kr94;&noS6r!J_N>$Pl`2yn%
zjkf6jLI=bf#UX`th9Se$<a)cz<&{i~Iy(wB5k<tsbY1*NnCEy-#-LFN(-SgZ*7@jY
zmHMnMo(&jYTs4DjL0t%}2FV9&)om#1Ht4f()w&QyXqh$-i=(4h)5GrI(ts;e8R-Ts
zQ-BAaTpmH|HKj)lS?fWuR!bC^FnnAo-XI%MH#Vqf>oj`!nCh5|DX=f4BTPUUBiLRP
zly&;Bnsg~tN`wOAiz})Y-BMJqT-x$6co)hE767J2sllb6^oZsc7YbaFK|+zq1PUf8
zMJ|`zSjmGHf(r~V2zA=(buRI%TwMti%u?ui$h2w=@(~mh#nYtwi#!JvP3qT`+@-Nq
zQVd?F;f{$RGXUn6(7GZ{Fwh+~9*aZqpeHcv0En(dal%1p6}hH{JcqdlsY&mmI9Ob&
zuLj)*;@N@NAQYHfkX`gLLG)Cq8f22fxN6YxiA56(DXwyr%Zla|mq1P4u}iU5mTA|b
z+@-NpqE>KaWyZ&lyY%J&OqmuWmy-geL4^!OA!aj|G~z#ng$1?6<xW{vX)tUAxHvN)
z%wefiDZ#6Cag)H7q5<U6#R2+MAp{!q8NrsK$pJno38S}YzhG)(p+%j*xdICk1K1EK
zFrMjL;*uCvH79G0#cM^ErdJCxy$-g3e4x{(#Pt(YRGF(GSmrPq6%?paE>cJva9NXH
z1{QA?Av+)ufgE%`4SBk9Xfsv0=!cD2Q}Rc=HZ+>3c0unFz1Fp6L;PJ}Qm9h&4EWIo
zQJyW>Oy~^e23I*Sw4l6{9}G-N#qCnRayi)G7?tTO<Um$1U%<L9d7F=GumY(YQ5=9V
z6U63*<O3g7xsZY~p+|teJ`{(_giZ&$xpd5MK`g+bGFzlL)`Xx6<Y=WOP_PoBB!d0J
zDThfyyBoF00AL{Cjv3pcLxhiE8`PjQU2y@GXif?)u!4v;{ZW|hK)KlvC@=#XvchiB
zDN}RB%T=aluEITI^b@j(4sO8(RuD!2vVtTjbNnkromLQ?fUuH4h8Dqdj9>*-WQA0w
zY$$P0L%0rB5M(4NDH?#TYC?e(MDGN~sV<H!2z9#jVeQa9LpQm!5}UOmj>d9Pj3TV!
zYP3=svb2U0!C+L$;q*O1mNdjq2y2InIQ&J6E{zP)uvm<8mV%GavaC*9T$Gk+?m)tV
zwP|&O{`8sPF=ri=948T_7mQrx^u)A-C_;lE6|^KB%Jh1|uEV&IsN8#Kpn0MQvj!#?
zEL*HJ=oGIumFpCC4c(rgBv*_ejF?_n&A{}e%?Xa?{!XbTGXQ1{TEbjG*;Z4j4uMiD
zLXbf!(NL?>Hl-#3#zii>Dq>@ao1-QuHn{5qnGU%EsY;8gBv2qR=^&va0epbTSeHg&
zz(KNba|a|NFA5vccLmAVq({<KZV{m_?3yONV$R9J(5<U!2eXD$;c^=@9h0lZ&66_h
z7w{G=b4UgVLds*|x*EB$!FdA40-Jy_r8oo$nF0C%OB2`zHCG-(tJB5Rgr);VWlkr7
z0&b-4qPGgN5H_nqy(I=57>4c=Qdp)s?3Ws?Yc3GzJS&&g41oPY4>I}>{Z*rvRU{_t
zmj+!c^y)wta}S1+0rm?wR$vF$=!~j!E(sK@vLGH%x7CDHYKRO(@p_T#Et**L0)PRz
zvVjpprw%4Mw|=+=X~;SNJp;cq=*ZyC4($$MQWz>g5QuXCS%7Xbr%XGgS>q<6%N;gI
zw5G_q2KU!WqW^(3)@%*#Bw-#GsYkG8fKubiL)_HR*0j`J&ReX>ICP??VJ~6D2bMyI
zT&hg&i?KL;`Os-FCb)OWg#&v8I|c8XvYk_-*Bo^PO0!5OE(U{U8pcoU>cJqPCMi)l
z(k|i79o_k*B0ZqgNfmCtP>K;slkRwz`-c!5^zgVcMl?vcqgdwR1I7r<3E32Ytj7SL
z*LZ`G<3^`IsX}^}#b3g;I;MA-Lkfe0g)zbFVA}>)A7p8If1pU|7OGXntuN~%m;^;G
zc)5v-l|`9bTT&c7!fLuE1~`VOPUo@%SpeOL`4633qpibvDSHvFSgLfN)A9sV*;Rq%
zL01RpLSe#nxCjhb9$Xkha?yO{qN${y(6a~GU8i+Z<|0&3r(*}U9@GOj)}VcBw1UdC
zJm?vuL$pZ;dP_+uS{@KC#1k5s+d)E?7JEeT4|4~cQKe^w`z7cRmnT@FHmCtL??NkZ
z*+6xVIt>m}OSV_ISqoWJgC2#=fZrTC=rMXm@rQE}@(xSImO$Ycarw!Ow>rm=?1kxH
zWbXk7Q>j)KO7L-=?k-LxZhm3bV^9SOxUNcH8zgZ<EW`%)X-ece+JsyfaBGpvZ>QpN
z=UY61FwS6l(KkgXG%>{INjDhq#9SrnFfnRe{ZahWc%(Gt;|k>nR?!gnFi}9!k}FU+
zkGK|Ya3g|FJlYHu@xQp@$gFV3tt8IA3QP%_f1p>1uD}X6vIGhw0~iRTqQyg7kltD#
zTr6`_k0vu6)hIzIFeR{F00TAXpQaDDCc8f3XsOcn;zE;RO!mmjSbk&Vz~bEgt;@C&
zAH&|Iw~<yc(ZhO3L7}XZ{d8r$LyDDIyiR|4ld6;ra^pjwK-O`{$P@G*)|6dW5nF<O
zORRXQrezl&#ktHb(c#yiONq`!b`+*WjrMg3Yd>zapu<94*b<oD&89dq%e1lS3c#c-
zqRj^T!_62jILSuzoX~j*@hEO9FqiNPYzFXFg{mG?k1k)D$6#k}Ptfp!sD>J;iQ)xj
zMQ-wwVhws;>1r1!O>St?!vcMWAw`$jqYBm}%6r>@NM_JNg+1c_8CHLohu~UTMz9`b
ze~|VbO_Z9r@~Fj$y6lx<5{YP_y06g;hFGOfl?zAO9pKceINK}S7vs8yvypBtK$k|b
z%A$%TCg4-L5jk|+0)rso?lpHdz}kv;VKcxAfQ;qVflJ?s>@>n=fK23`3mRxJ>VSqC
zv^7vfo8rvmLWYZVK&f%7pTFoFhe*OJ_DFmlSjch32e~Zsj*q!+L>Fa?o>Cg$1xk&Q
zp+yU=DT7Vjfa2n?<T?}6oc6Kn3NDao(EAM<O;vh)nso&QB7hQ(J~-|VK~h&XKkZ}$
zS26M@99I<%=W<nAZXIHu95CdksnW4fR_<;cXvBak^LF6eXOvKFCo|s4inS`}h^drl
zC6!ct^=K;t#tybhPw#C9TQT$XczQ0eR&KF~1x+g)YgKa7W36=MSd`Va6=t<fuM(9E
zr3+mY+*EhAxl@2~hIYl$7Hfer1=2Q5Rm}62tdXEQY9$e0cuaqNlio^(j!?5nVMb$s
zEy{HKQM6U%DMM%*74Af}xDQ{_Ah?*Op9!;qyMx^B<x_6>yPWHowh*hXJf$I8<BEs=
zdbcjFhiwLox#9<}aX*T|E>f7=;doOniMdoFkK-xrCajEGoPl&by9$I$jQ}~0iOiiB
z#th&o#kWi65Z3=Nu1czV^exlAZ!wCC@e>7!8EbJ52?Df2gPtu*fElNEox+z}2`!Nw
zeCl%ZhW=PCxhmpw!(+%f?$a{Tg^tjwD$I?^GGmL>YJ{_rPfLts<H8Zc1W{2_AR08*
zXiz~O&>k*JVaOxKyfAi!p38>9jnozgbd!6pj5-hyV*jEGC0BM{bYb=xavSW?qWhdy
zCZD<_F~~@DLPv)JK_3W2H^nGD=sMVro10K9u=u%UZzu$thN_ir5{f~Qba)Jci)*XF
zbym&9>p>u5HILPZ3tb81RQF)?(2vguBWm!bCMVZRbw&na%?~E8tHO-c5q}SviD41y
zX-%SzTHJXC0mUP`Esk0gr%$LtXJ<`=;F<sewMlOY%r-S5v>&*oTjj!<4jcud%=$2X
zVAp`tTMb>9BD~7&&8qCDC@f&<(An0IVFvj$EnFDoTti{DLSwlVMSFTY>1<|vAJk`A
zKP}S%&)r|T94oR+<YOB9SSf<gG}s!dFerdo%I%XDmd~w*VmO9g)EHubZo?)ud{JT)
z7?lXtiw&%W>#0GJ@sbjk83tQd^}`snR8VCYS(zTup1}f^qG2LbB&ae%uc~T4OkB4l
zL7fTXkThgcKQ_KjA}b_u4eQ5Q%!zR$c{Ikj;*N=Nc}q}@Oo)nfDl&Jf3jgp#6AyKC
zPIB+oNt(W_AD`<^qF6|hAgmu3H%_8tN)lqMACY(!;h%_btV;ce9Bt^J+K|w#mZhr^
z`CJ$N3yX3^za+WuB&vucZNmC-)#D^;sU(HQ`f=^&#1thR5iy<GRP9F;u<VvZbaka9
zoeHUS@lCUS#0DU#W9EYR8RM2kqG71bl+{hbs9m$%7@Wk6!IGP83>w30k}t&VsNlMq
zb*w-QW>q6_(+S&wy|Y0%H+h_H2|`JrO6=1=dHg3tq41*w8tcdr^#^}H<%^pcCM4T!
z_fM4Zm<?zr6Gqewb?LEc$-YOGK>#Ihn|^3AI0MPJXrxo9>TT0|!eA6ePhgsI*R77y
z5)szqQiwatW$}vFQJkI(a3mc`<UdOF5+4idHAas!3WPhZm5R4rqEx6A2?T3Cde0RU
zH3fhh*4XtXr!{5)9?Q(GahucSMyiDS;jx)pN{&Hk)ggQFTxJ&o8law_oUo3qwD6cQ
zQczkFJjdOCh%HwhZ*Ei>FynG<jwLL_BXKNCjK1uK;a!6<Dh)iAsprxMz@2fPYH%Cg
z1G=*G0SMsHYRYy$9eNGXiV)jS0F3&AOoX(qDGkc7c)E^V#-l+#syzfMVew1RyDVCu
z?ux*6xfPG6)QlAd7rIK1Hn<5z!3tGEaZ*u2p9G+@i4jl@20_wk1r5UxcBo$NTT(+v
zWH|)CxJ{sb@S!WVT+^Z(B8wYEP$#95Y-$O^pl+k80@fLE%9XLmv^t}aYTS3N%8st`
zxU+b*S(f-#(4V^(^g<QOvPH_MdWO1Uq2FXQWDyi<*1$wu*l<U`$vr26k)Tt?%s}4O
zB%-#+0De-60n|`Fa#{iUq$b9FnOdvBV+mCTJj!uGdS+WZe^7=_04Fent}apVSZ+f~
zQ$0cbF<e-9y@4TRoEtThgl@DLEI}U)WB+SvkDt~|mEk&6q_81W>=Oh8c$NN?CPT9&
z1_!!~+kDVtC5C}Z6iboebB(6hV+~i#aF<@u23<B4p{DHK*Nfall;{N*A!5dq)ee12
zg2uqBoS>T<jS^#rfp3%XE3zNhEYacBpkuGeAflQ?48jB{lOdp}nzA5kHjC&dH_fQq
zD$-x*AMV;nOcJ-NBp8pe94&_D(6nI97QN?m0l^|-OiN8-=rFlat;*2I3Ky6Rl%q?B
zb`Q;76+2X_a=!=MKp&8VNJ+RFH{Q4t3`HqhdJH~mxJ`yWR%8einXB{^%cd?&-U^Lz
z3DRnCc<4Ch29t^*0tFcV1xhn?nUsPtfJw`krv}5QXauylpT^!|4s$**7_bIe$h~6j
zrIpIEHQZuc2h}p@gqcg5pw*}r>3w3@Aonf#s9Awc$N-g^?C(j<7G)wg>Ugk2mhLSf
z6=}uISAA*)6rioyl+z{<b3!SX0YZ$qsJ0k@rGg9@k3q*V#4He$!z~p&qpZW6tMCkf
zA~PpKgVLB`D-9uYojVQO9l>z6lxaiS)68$dTB<S9D9wnVl8G1*1?RPWY|vBJY{jpD
z3k(ab?UtJq{S#iw7pO;dxncB_o0Re$OZY|Gqb8Z&1c%qkxOoj6Kw^G*@YrdnhLVhz
z*Ou{;ZM`USp#VG!HacRMh-DH}%ic1tR)Rr82vLIuMAv2wzDZX%yax~X_tSDKM-fWQ
zoVBbRXLi<^c&t_{7Aw&ciM4DKM-fLL7&AxgXHhOSF(&wr9CRzCTR93$)XZrVxuw`D
zGbD=3&59Tc&<wOSXzNouLnkYv07iF{AwHZCEk>7%@|Ux8+`z3fcDuqeCdw$_0SvC&
zxZ+^6V^uw--?+j<Qe1r2AdtjXpn<}LGtK!5*W|8mL(t7FI-J2KV*R*s)X!y53d3!=
zSgdF$)uITBIuBQIW@;!ko`6xoUySmqszx1&bB4^|-kyZY__%QAh62CA<4SHz3>Rfv
zaS00*Ux9QismB%B+o(1xT)CFeTg%L0Tq1X0v4DZ_tIB$uU56^=xuzCth_wOLN*M>`
zEc*-+ngeCYO#}#JD*CG01}ZUfhTeQ0GpjQUOCB-~yvcPC*RB%4AnR$K9%F19y&71k
zmy^$P$|6;ELKxA&N3xy<8Muh!dKQb%R#S`-T8vyb!J>s($CG@@>U62i8C+WC-nt6u
zq>$&~2gZumn)Kugua>EsMTKFc8;r@Pc~X;@Tb@*6kdUhhSYxO(`3p-|Qc(;!7|soz
zLTidKq^QcXOjP<XWXfCtDu)EwyUI9bXdA{RD~AAsHn5<rF^aDy(Jn#&n3c_zY*BHU
zs|Zl4HMy(8=pb&~D8m~HSx(nLLDt2L<jMdn-sA$8(F-s}D~hLOgA^K_2gVvIUbskO
z0|pEuhAXr*8p7r!8gdL-X)=~v?WprCM~y*#O@86oVqrHZ(<;L<87f$o05lkLTxC?~
zJ>WiSwNY0Dba^(IQ8Z<mo~o;ycU5kS!rCrrCk;2@X`%952M<~*cSeJ!nHzM6HOj7<
z0wr=ujeDqc+?3TCf@cuGT|6)*<H4?TC#K2-6&U(bLxQ}Eb%E7l=u^F}qWCB!#39O9
z@wz&h#DxQcxp+31(Mt`XketFOs|~x7_$JDXt^+&?cI0NSY*{kqvdSnF+FXn=CV#oy
zbpj1t+OZ`$k5uHS@JIy@+AuyrQ4Hhda(?j~www;4(eKKpZJDQR7&@gqFg)!;5k&t;
zRgJftS7D&BI$$WAQDu}HmMwDPOHMbr>L6;pT9yN=wVLu?&^^V|f}C0~Q(JlkB`0*n
zGg4O$jj9{~0gG|XQxS8N0W4^uBB!S%N>mg=lOa?PKnzTlwG_`Og3%@Tmf>Y$Lo;v(
zas-4$N9v3)B*kb@F&M2Z_|pJya5}lP(LnPmT}+B%Bs457vR+f2geG;GjE-(GjIXW&
zdF7l47?e&LAX;u}sm}mGMt^WqTWJVPD-5Aaejo$Nnwfsi5)X&e85>hqY@#MT`yo-?
zq7Un&1pT5|zS8N$XPj<aT2~s3wv!cGg@K7&-Se1fQ@N#7bPP`nVqGQimclyRf91Jq
znvV5~QZtOirvIzLxb(VNPeBhbo}Xh|qCMQC^NWYvT}nV2`)L-UrRT3~DB7@=8jDg)
z*^9QVk!&-X!oJz48o33VQ5#<cg|D)JjNKPTPkF^622a$0G*wryBNMkr*Dxg02fNo9
zdUkpgLtXT3XLEJhZ)Q+#)7aChiWrX;1*4?Ha4-o#A*)&`p<a}@38WRm9<j@SG+qpO
z_qa2k4&Dtq!{G<Vd%GCC!^_c}*O@b|)O6GtI?RbRm<$K%!uM!my*2Wu#q*~WlHPJN
z-F~4Rqmf5kT0|aM)JT3KgZ;JV!}_~@u9W}l?JC*7mgWD#x~f!<=Ko@p@m>C}TllP;
z|Lfog0&DN^XrmH5dVcit(Z+@=9p2oWf0%g8+%<y~hvGSZ+diM2wrA$coJ;%g8z>I-
zJ!i_~H->}Jn;qI1cy#KGa>^Z0U?uryUhn}zh@)^muAoq$+nEhiAQbDM%8qG^zh<YF
z(mCl2XC5(Nmc!h)=Wp|o+@ShRVj5Y=u8I=9@nU2uGq>pZTT4lJ=acydyVitPS1pD^
z)|^j2@CG!gs+COeG;g4=Y3Kc!H*)4@p7VC$Ej(v13m{AHCJTYXO9E%7OOSL!NcP5~
zIn<Cd9(m5WH=A`%y(qp4gpI8<Q%FQtD>1xZid+CY3GgzgV8nq`3hPzvL{gbxH|J;l
zHgEeHyz{0^?>L?79I>70X}}}*K@J#p>U$oq(vzZTgEeO+1*{I447PRZbK8CVK^@&0
zL3Ja<2;;eChQv;fJYa3Q&IGpVE{nd*(+gFpr1m&{+kwSG8R0BK%)C+mddVPNPwP2x
zRjo|;mA3P_M%`SDY39D}M<Q(|@qE1m5xL+)Q92hOke5a^_mTND?<LYIL5!&Utop*Y
zK$4J-%$=uL%736pJNwPB#UC-!T}9-mR(e2a?&ZEUjTf9acUw7-6}!boOe5&$^s^zG
zUN8_d$}myL3a8g&Sg-*?nxd<<Ds}B9b?vO#y-wC)tqKa^Xa2aVk^kHP#y~m0TKMyQ
zu4MnsFIF-Dm)L)9qgILKe|1ZhJNxezKC9b*$G;rz?LIx)C<l-B51zmv+$fi{1sD^c
z^9$P$!!O9s`9)k1;*FMs^NXk`H18spPUA7~e50!92a8eck(MCxz!o79W7VLyu_5^R
zQhNhGJ}D_k@z;hrq#>K6RxN${&M4GKMbWfWP_gIvcPlAcA-Jt+eJY^(_7A$>8}j2q
zd2?*K*o9WpfT%p}^t{B*D2$cn!@p2_nnjkWJ%6+Tp}6vhv+#v-9mWZxTTawxf}5|*
zB3g_iwJJbSRX#MM<-*r20LOw~n`WU5nKA4SLCfu_H|(&VVvWPJqo-~$xuSQ-pK683
zLm9qwugz8D|7gaX?#op`w)}UQ0V*c{YmGbke;c2b<^PZS&z|kSIM{f#e{fu{wx8r&
zKN>w@1+G{_rm@Hl{d?}<3k8DpKKCttFzKR(R<*BmU7#u6m!gV~W>PpGDh=T=nlT_j
zIlPA-9GZ4psX%64FMxu|_`OE$kzHN2ol-x`gKl-dyo^3&pFnK!uh6aDc=93EE8adc
zn|BjC*RgvSWsR<07Fxi~bE&(OBHc8S0Mx)+g(%%rsstcbqRaWDGxe@ank*9ZE&O$R
z29D_o-BZVSKeIhlZx@|m?0TdGIDV5$;3RaVtGiNO4_((NPd>JqIu-Rkg;wv3gnnzH
zzik8VT7ZY%Z01AlEa^;jz0Pc>^(NHlx?M@@Nt8rcopB3}7V*xu<9;neRMEoYFEv6d
zh>}jfe=}kvfFCBzPBMX+IHy@(SN?29H>uIwfCjMMq)r)7-H8aBUT1G=HuN>+)N0*Y
zVQgoM?i^No+;F<i0XB_f(IupQuS5vb+NwhwvjjqNN(tRHI^x$ZT|vmL+^tm%KOGft
z4Q7&oWJ<WDRMIh?r&_uCXof=?RvOT{`DCG<I_NxQ!N7U>G;a?7RYtBqcsJ<ZkPWxm
zm?{jy@2*4-0-RlCB-iRpCA}||o9N04gPJ|2@^9}ZM>cWw5hjgqYF%~wu8Rwee&`iK
zYB-{5dtX`ilC;gbeQaG9+n$d{<N0``-0MDy{?puQP!^LuOwbj*G^KpZof`sUd<a`V
z(iiFQjicU(FW2iA)$l#M@4Ogc{d_o>IRAp_`|rW%4dd#Do;RVDJ(xMe@#s|D@kLL!
zj~W=v8})aBN16rHbKbDPq^>GQLf$H77!%{5ifSi#2spl!tU5mbl+@_iyVDY?nACT3
zHu;t6wzl>A-kbw11euP9!|iUT_oldILrjx+s%>-0u>#I@w;v!do;zoqcOKRg-pDy0
z_Xj7&t?}uy?qrL$h2+QOgy*DJ;$!Jd!W)c4!c#NJl3F&iHtC(3ZV7KvU-)w+;hmlr
z>8|sy;&T=M|AZ`j<@-N*RZk;+|7Yp${?FU^EcO4ZQ-nu@kDk*Tc9aWb&d{@t(pqD=
zl$=3nybM|2dym;BG11ir&}gSnY%^!VAg0Ag9Tog$aN<1sx&3nY=RMxp=vMu0hNJ&%
z<o@$2=jNU7zjtz`3>V|P<P-YFF3F^^{?(rf(F6aUbh-L>##n3~C0syW0`IfI>6!Ch
zTrTIG>6@bC9QnQ&ydHX;+1x2Rv$1rO0D<ZGd1sQ#b7#^!A5T9xr_=HKnbRM?AHDBP
z`vw1-yqKS{@o1I7jIc;ls6a&mvx)xXmsbb>Jlbdmw|DBT)6FjA?5Ho|a{={Yq;_yu
zy1majLg`sSZMWLDsLS)HT^+mko`piU-YaOqZTC*eHn`(Ar-uWTqT8tg&f2%uJvbk9
zf2+zdjo`r$I?P8Nzy0$csz}}{%|C1{scZga-+mu~La|h29duOP{VkpORyCS>C%lH#
zjYOIO^)e5k8oY$?A<FYlOuyR~nX`KZ-DivTY6=CU*LbGiS}(xD=<#w|!r|qqNB`jl
zwy)}SK9tQ3QFg4A1I!LjYRFs>W^+cEx3e}TZ5xwTV4C`+yd7AmdcPSe9_LtI6<aCO
zz+$<b6c$$e4gy9PKOa=^Ic6P(N#y-B1`!N3hlF?|mGu)DB*q`APdkyx$;MJ9XBLY{
zy)cig%OYkt4GQrgr996V(_0-vQ<#{TX}`vxnrgQ-uBN?946UhpJ)>)&IY0bh6z-C-
z7PT20?Eqp<>D35KXlQ7>c6ZQ8<|W{<8u(O9^3Fpig(dT$CVlm(IH1jwSl~pX@&cJ;
zcvi}_^&kdJ9Uf7?&@xr3n%fG680o4}eX55QL%KJe%GK6R7)ai}Slh-R9N4XT+Maqd
zZ|?Uq-aUKPl8q}jB1bvfh{$E88i3?uv(2?<n>|*mO)N!i``;y&jDiuuBOx-5Yo8Q0
z0=mhpXUVaUzC?v&{TZqdnn6+}c}_(u#f#ERR#s;Op(r9KC{<DHU%Q9<yFWhLYwsQ&
z?*5PFZJtw|LJ#?BOatSQ(gMuj6Yzod_WU#$4ng7hbYNfHP7_fTvus6aF|ZZ=W4R@D
zEH{K99+`6Hl!0}?;OIX**7@w&izmMV@4bUvYZ5vC8=Rn3C61r$KKW&@eemL7Pi&}^
za|r$!+l1o?_dYJ7E#6z8sV%=!Vrz!i3e<0p1qeFHFI#?d?29Eyf>XO3g5Poj87qr2
zdFO%ino~zkKzvdrAJ#}4DEQOV!*T}$(p}8u$d6j2apB^1dS}F<An&H*Iel2~&jx&|
z3SlWj7!%I&={pa&=+zRp!)D&w1y8^VJ!E@j=6Dz0v^SV}vPgVCn4b+s+h{?c3_Bmj
zXlF7W4tgJod<538I3k%()Z7i2MgM;Apb(VG>2D7ltR?S7X>&=&%A~c71`Th6!($8)
zpKb7T)nK&nq=Y%RGr0<HhR5^S_;k>d1@{z-N`InZWZn!WlfmdT7%hK2-hrTc=S>GE
zAMjqF$QW{rRBf*D7bD|N{KCgZ7GEn6|7N5E6wsuQ%^ZnhLyuN}FzZddNoUmi5SIYI
zj{+IsPkMj&ohL)DGjbLaB~+6Pfw0|hDQ~bSyRY04`ZxVt+5bB`TiV8w^FPe<8T0>E
z8r3`h?=5`3`Nu=7U?0xTHaEX<UO`FE$PXA|=;YaqHnei2%EF-NIQ!@4UVi|i0Lo^v
zn4XH$=p1X{bG!$LlWFIadErF&7dIj5#?!eLqU|Ry4t{#PH9S>j=%Z2rU;BgU{6m}D
zI*+#9Lf`9kKD4?2f?r#A;^xLbSDXJ>jjyl)v~2!2Dy8`Ruiwr8+xRTq|A{Yr>4G_1
zoT`w4?sz<;)A)RVm7cN2ScDdtdJdC|o8@>dFK9*!3~rIA6**l<1?O8;+;XslS6-|X
zc7FwM@{UIr%2qr)A;8x_5QkO5`rj&<w*oPez>UVM1Jq^nq3{*;3g$I%x1n6YMDvGb
zEko@}^Z)ER^1ry%S}iI6%iaF(ZG4u_|Hwhv(YWumXX=kS2x~w7GuYOA@pP~KeD~-5
zCr;^t*|$3<4bR#LI`3e~N|A6j$|+!ZN2Pv|GW9CTyzxFrCK%)E<aCg-z4J*<<%d;O
zo%@}F6MoPY>4X~0oT1&QTzPi3e*jqL2d67v=(sk|zBkW+16Pt$wTvvOl_r<kTqQqU
zsK`+WGOVm5_=XgF&!es?Im?;XnTS&ai`brrg|@2LnRW;BX=nPu8BhCIcw<%0Rqkj!
z+8*>_F>wX&4CdbXOq?X@C=F>aD+&NC?@T9hIns#Iah~tw(HZ+eH=ox#hTE}{-$zUC
zgbo7ZZ^*f96TB@Www&zFpMeBkV5oU2cKZjZTyO%ekPxxHVuR2LELEkuLKSP%z%T{y
z>FV78!?%+&pT9UTWZKDDD6^0})_>*&-4&HvNdNM?pAR4XR>YZLEawrz@NdE2ig5bg
zWPTRAgJZ*#9(pJ9^YKj5vJLwM=ROT@Fi6r?x`6VJCF|KLD4gWBepebR^7LML>!;_(
zi+Xj7&IG~HF#6_3xNR!uvr&5}KO_%LoE&C;L~A})N1`{)R<r4#qEEDs(hu<f`MuP=
z);K_q&J!>Q!^!VBrq(|mD|JXDf~T3VbfLVO;TE#`MHoworV9<fu#7$Yiie#0{Hr1=
z%|uyTc*AqaY7!osckkXwE$80E!FL%TsjZ33@7q6sW2t{^8lR4_@EUfy-mq=R9in%r
zq6yp-GAmrNDHEy<BI`Q;Q0;4{-YS4p8BO7c%PhSXUrnaoJNuQnH82F$eDeb%)z9x{
z^mx8=`eSc4`fe_jxKLGZRfvQ|D-FY>QQEZ0`Y1MJ|L_0(KYz&Z-5#=pT}Wx>U1vCu
z5YZ$l5q`!<$<aJ4D@J@dlgy;62@P_N0%V3PGyt60xrks_xaijHP}3(eyBUz{AAnP(
zwoD+&&500*cIkR#O#tRGyQ@i@k}N5-fo>|Q{%$?fg&c(e8Af!v;TRT^j_NUZ@Yo5D
zXml8K_r45~c6}4hhwFd!8ASh1n{)cLo^&Zx^aZZr4$CU>Ss-C{iNKA_u9y&x&!f^#
zl4!n!yiMs&MELqCt?uNnC4W<BXN0lIuM(HX94A?gb6hMPZ1iRl<HXz;XKl%yC=8-G
zhJQFp@^j?9&wXnk+A;s~R_f~akRrcO9jub?Q7WGdrdYiDl`W5fkv1u*43#pOdgtSJ
z8V}9pg^)^7o&tGqm9<2ZvH(TytH}_|ECa`GN+X<V-0&~pSPfb=@;eFbg)H~f0h)G3
z{jpqK=dUWc8aQ&wwcyE*yFc?mDToW++k5I%%bOdslk+*z<6nDO+3lSErT}rvHZ=C8
zOTsu-p-?XKg7ZeAg8&x2;aLbSCj{FZ8_(|~<P##Ab^t7vc)c?QW6IB7XXZJjN8IOo
zc;DF{k!iS`l$D~|+KcarD)iK5sO)sIW3uJ{S~e)VQ?JAIw1V!&p--7v60a`ViXMz@
z6x(*(aFZB|4qf`|K{RB<FyRku6w<h$E9i~Sc%t^lA&6>QSo%4b4<J-Nsuj6xc+0q&
zp2nxSFCitFQ&_i{=^7+$x6W_L43K<IO13E!->5#nL7!jakB{7jVy`!U6wQ<f7P&j&
z0`yBf7LI+R;tPZT)-Z;8l-;WIXCT#7Z+NgsH>9u@$vSZEIc|lj&n=s(Z18wGI6Y<D
zyf-}^n4?)6DPb~kIj71B-E3(|TPq|nrH>mc4aFNS%oN~d<DtkF^5n+l_~t4IYVse7
zoC}lRpVXjWRg=z|*-M6Jg${F~y|F2o(PO>9(F!1CyjN{AXmVQ;2WPdbM$eZ7w)6=o
zz^*$#2qCt&eF|h?yzQ=m^BX~PMLH$oTYh9sRautNobj!%<%1c~jd5*i37s1m2DJtx
z)}a-k)mms3sMdeF09#;HU)&iw(*GvC=nR$C@LN+<+|Xnquw?3fa)iE>Yo>2Z-mFL1
zmFLuFA!@?3Fw3NyVxASo(N^&r6#P@h4to7ug2c_Re-njnL?c|<{8oWd8%zQk*l08=
z@ba6*i2h+^JD6Xz-!8ocr<W9k@H1`2exc!0SK9xZ!VI|j`Ok7v{@-%-F8||ge3tJ2
z*~?GG(irvj@Kjmo+^@+=3_yqR`@QgKuUV(y%(?|<ioesYy^b>VWg*ppq^Xv<D&)5~
z{BH?|mvZSgc0zT!tJVqcFe#k;#H`-LlccPW65dqhh0y|KG9C99J;q4uB5EIhri{T^
zcPS#>br9JLZV^gH#ky)JjLan{c`5phya?JqLAY<fAN1#E1|#QuAzp2)G_8;^Ma0Wv
zD^`Iigh~}G0sWt1PnuuGtPBPybA-$jyRpbX<xu}yC&vV+c<>}wx;W7vIjn3T<oy&7
zJlXRL4;+hDce;bQohq>uG^#G*@&Uw~2B95LoF}ewl`<4~ys<6+qSv!Xlj9TTBsx66
z%+yOqBDUX^H2>?>@G(zuv0TT*2Vh}<dUg4P50?$+V!4%xE(mm2LEr`_c$f-n(diZ@
zbhz#4M6Oq^IFZc=B+eo=5Y_~i4939(zJ-abdBK{<_3D>2kv}eCdfXg_y!vDYa2fj`
zvqQysz{8_Q`~TT{@l*Td@nL>t^}a4JJpvyWw<Qe(!S1yncyrQXT`FGdK-|2zSr@mc
zJj!KN%!|W2EU#2Y))QKWQL>B=#T8m~5r_71`}yv#Hzuew7g};^87a)hV1UiaGR&7<
zMwl&;VX;l3s4>?bjYok5(>X5UCZF0p%ubIMhrI$|x+vCzOXa&9XK1&zz;cc)g$njt
zpAdk&9Bcv^lGV=&i%6|u%=VsK)4F@`^WKKJfWUtxkK0GbdoMRyjbMlHaPOtuQfT(y
z%@i;1DA>;Tjt|s4dhpl@psA@GtZF%1d>sghq%yw%qDECaFY!jXFZtzrNAQpzz8f>>
zqY(}Es-)mVg@V09g?htzZt54>dcm{2C+er={7TjQS@!#vM02&ExK^60K1OIR2Baou
zgw`m39kfn2vAz^~8;kGSsBQWernTt3O!kJ9S(W06;%Wtja0zX&OckuJ2Ubu5>uG@H
z;(vYFzeMn7N&R))=jpn<kXdYL8JoK4#cY~omP#qI3f@bRUIpV7(2b5*`sTyc{Qu*V
zcBj`{oM-O>EII#Isg~TR|G&y-cmDsI|Ez!i-$szE@j#CC9<^WY9k$i$4J#1q>8s~2
z+xrK{dx!tpeFo(eTqSkfe)8hg0sj4E@5w)dBDK)L6XCA@;(XE`PrOlkz`N3V-ej(H
zDBeywr&tuWz3J4%Q1{L{Q|JE4088P{x#yn%I_XR7B_opwD!28p@4b7-{Yb1aUbO#x
z_~PK%|5yvuKG&xweYp!H6rLxwBX_bIadr~V%jDeORKz@)j?bA6;(P%pZD#&uep@fd
z(Zt*3B&|yjXg1f2{jkvN&IMx5Etop@<AZ=jT2MU+tJM1nqX>aw_X8~bTMD&S@6OeY
z(h@3K@PFDKqJe%lkoelR-d5&nIaE2CPsBvc#kyvN&;?mpvWe`|RuUiD=~q;KXcO}k
zVpr?nS8}#5a%Yj;xs{S)%Is@LGPcPvRVLIKdJd~Ipj*Pz2oMjAZDS#pGF=2k=YRiK
zFQEqyTnSf>kxci(&|54q%q;Ue7usT{-R0#%WAh+4*j6K#55_pyw)a>1L+}Py`G)W+
zqJXz$O})-ccP!w6!H$iUd1~>HNpiCMmdXJM8JtGug&gl_q%OOflJ9|Z*2R?qpILO;
zgOl7pAZ1Plr)|<t=DunvINzMkDXL_ISAq<2@>XkclW<O+<yXtPsu@b^V4_RHAF`8<
zyWiXh3sb`b3DPz3+R@Nun8Hpz=>~t8v0VAeO{=bs^uPS=o0qgT6~}y`AsM&&xhDHB
zGYibJ{10U}ZvWNlwLAOo7Cu(~7wDVcoUM!h^QR}-fJMxG+QO{dx^TCgKb#$&ZXRtq
zPEt}A7JfuASO~Zx^qbP`kt{ix^+g`eI_Hz2_o%eBc{2FB&%@94xrX`Ak<DBHE}8!g
zECCbdfBkO$-@<3D_%CgBv}tC^(J*`2#U&REL|S0ROo^<^7NcH)OFJG1v$DsGhnbmo
z$HUZ|yQ)dwuow?xDRu+n>E0}te{`?^$jPvxc*iY`r+Z8QuFxf_1xJ>X0TNvzA*}^H
zO>E%a%z4)tF1#=}g^`B&DTBnbbP}WWcF?#yLQ6c(QrIN)&qR3tk&ldR)9Q!$OODIK
z_tU{VJU*M|Ok&m`g=w|YKqbIU87NB#a!RxZ&}qJl!haJ+AvQQNWcC25L+7En7FkH7
zLk~Djur^+o6ohk~yhMEezL{1+b*WQu+HHbvxB2ba{=r_monKCq*=jTq3dD+#gO;#z
zgi^CLD7C3$@+0l7K|i)8IY+WcII~FDU4x4AGxB`~gC!J*({wHB37;vj#V`?<wk5P&
zLb~K0X%4igm(}xi_%H#^Y<&u*mAoW#UV*%6NUTrx5aVW3eqHtrOnZoGLh5=J#Z`~A
z>A%!I@U#0GEoBH2`1iOg83OA3m*YB*=ueD9Rz=)i>M{ji8TfvzQsBZ$P%~68b`oJX
zd_8U?S9oHQ@NkOzY!nmvo257-&bO@yYeb!&0jneono$g7kJK?un~m{(M4zFo?ZZ^Z
zTAHrXJB-T&irLvrmB#|8;H&^{X25}z8Y+)}TC=y{{3AZ4)sfI7s;-Kg-`e`f{o^LF
zi?+8KdWO<5>}~|P6~{Hq1kq7l2Qv+0Q=JT+&BriW(E^QCeLDDB7%|Klm)>U&rsACk
z^zZ(;F8@!K19++bw^WPye|RSF&i`}spBwW3Y5}HY7jvZC`F_91&z0x@DYw$D7XMSj
zyIA~B<u3p4ZG2Ys|H|U6KhQRbzj3^Cqai!-_WZr}TZ`w3u4GP=f(;6eJl22O%CV{!
zuLbA+Tl_a=hQ_G7^=}zokT|NM9U^OM|E9rfT4PK8h<NMvag~mV!;#<|)wxW3->!-8
z+ndK%c+f)8_UZflndaG7{B)`#n|v1X>nT5fIF;tNn)&TE*5j=HVAfH|Ev@DQ6vJp|
zj>q%{$3x|fN*oY7>woHi=xO{H7!R=k(}&|9tB=R*%bOq32tf1uZRGbf@_S~SW|vxa
zpMIGkN7rQkT`B&@ZN%e$>h9hC|80D3EdIxe_X+q!O30G$i<$&~<Ik1n|KQ5&Kev{2
z{-b<1|8L`SxBgpm{g)POZe2urx2nrp)m?eVG-|(H#TxwE$B$o={J&cKcl`d(YQ0>(
zlmEBy`4Zy4{RbjY-gSn0Tj0(}y!p?S=f7MweHHu9tz)qlxBsel=Ra=av#R~~b)3lE
zWH6g0kaCs{r!@XeSz}p-*j5NnwL!4SfXA?)4Dz(-J%sRUdv76dFseuSoy@KQkb+^m
z7{kzQKVw34{ZcE_gx32erlX_vrr;Pc>&&o)+l>!lf7}iPPcD%ZLZtNWO3m;<MjWWo
zzK4mqX2*3PP+)s@=AC%xU$n`D@%QhT-HZB~En0HQ=^QDE$x#gD+TG49oP;LaT6;Vl
zoEDrYT9HD$KPbGZwblXJL{3c}x$$IP3^$(6tO|EUthDnW*TAtE<EN>Y)LJ^cFbKI1
zCC&pYwHq0=I1=1&Ih}eQuWEO?-u%6X_1f<VME-qCE>79fAbyVsjNiAmOx)WsFO8oK
zRF}wMI`S=gxVLN_BVKEcMn>`K^KWH>Uxwl-<DNEG<*2)%q9UKV1~yD@T|<*LVKO@E
zlE5sYhXrT?jBBG!qPU3XIbA}Tlx`*rLqNvooi~6-F5r#46hbga`oawi&g@6XnncdO
z(EPK#7{M7DvDD5W#k^@?t9kvvEhD9RBVLlle=&X9z43$>4*gjYn}z@Xv}3yfH`5{K
z=Cl5YS90)g!AY<Qt@9{W)Ax1T%Kln{x%Qczd*{8$hupU|+6B`IUtI@{<im!cA3&R>
zAA3WuGg?d{=MZA}$=xN9XNnh>D!+&=>IV@oKedp2SmBn>Cb%{tZh%PCd1w(O<U>@?
z6KI-H%Jwyex8t)ICfrB=#nuvYI6JM&Ws|ztZptM{?<JxfNhNLbezCWhj=QgCu|bhx
zl|(?xI=x)sMrD}Y>p;ue{f?Qsw@vI!Qbi_GK&sibGk+3C8zGV6&c4j-kIR&#vjS^o
z^vb!fnEThQ<o!}Fe*EwD;r_wT?H^zLq;eB&xV1`<;OlVjXz$oc{bfLu@@dJzD)mO=
znv+z8Zdz9$IPSm2k^e)Elz8VfhkVvBhoJFWX^*v%Q<>aNVM?r)RVs5Pu8>wLWiD>{
zDVqWVIFx7Vi%WndjjNHnKcQm#RtAu`CJv&$pNnN9*K?UUhF2!-*T#l|>EAdeH(aS_
zD`#2D+qtts`jsvD1h-G<Y0zOsYjRz|a(zOx!&&gYN~SyKh7?JJo&!M2k}6J(Se~>M
z37Aa3B=Uu+w{otoz`$DKdf2&!{9VyRbr??azDY5*%s|UbnHCb$K?BJves9~m7|di&
z3I@&$W<U?L(pH;=mTAm-loM<cXq3K039s>L+4kmNDBssI^=Rd%i_JWA+saF4EimIC
zL&OhpO#<Lxk|vF9x=xOcQo;a-HMhC3PE*VO9TQFFO?;A@H(|YGo`y8Q=BDq|(T5$K
z&Rmw$`0ZKd5>(}!VFc6hY-W3Ye4An)0HOS2vfx6AmF0pUL6t+;V%`(-AIm55T2m)=
zny9m#-aPQ&DQO%Y#ZvN=BBt@=l{;3>?T-a#q)r8-wv^ZuOYf?|z|c@XYh~f&pk;e!
z&ier#bv*(*AA@1@8W`obkWuDzbkTU}RpVvwf9Xv@v2$m(=)shooh*jVg@LWBDpzqd
zd1q+qUIcGR&5~NW$hJjn(q5#aZjb@jrK2+vd`2cT_~U~p-C`(>*1$<O>^laRu@kV?
zI@%E0w5!Hf<Mt^-8eLWx5tGE&8M&icFJFC2Z!#UAEAz24U5rFz(#qlFomq#gR}DGJ
zJmG9@aukx^X$Md8Tl=YzQ(r^8PxNz5`~TM;|6Q)$?f>7x=QiTM@6P;wxt}Y~|9NkI
zkv;vF&;M#89{*FnJO6tdpX;ChceDMk^tt-{SD79d-jx;wxMcp<%Wf?Gw^qBG|F`nF
zRQ^|M%!9;ycEF!qV4bv=8)-n6q$-URP2F*_b2_o`R>(~ig0UpvZJl(o@m3SEB^rch
zSZ@R67&eI{%|IsFXhI^|d`7ZBGqwroK@&&+5@{I)z-$Totd=Zw)xl1W_zN`|=c>!5
ztX*;JGf5jlZH>{JEgiktve8TQ-rEqsRD-^MPfgnNUGAYv<30&*N$uU%*?=~Qao#+<
zFGCC7m@MY{-qX={jIR-I<$JHMoD3G2U<>0`SZMF3-B-_!GeQv6J*bXJbo@Jb?e{JE
zOVaqrZa~xEyWdCOt=xN%@b{6T;a5e3qro{~bVlBIF_XzC$)mjty3mi7t$xD#XPE+8
zm6+NQn5~DB39PH8koo1_;q$CG$y74?xAG>8e3;bYQB0kk(a^M5;6c^C!18GCcw?hn
zyXGhW8A}VZ-`qOA-CJ9+NM&vK%tn(bl*>tO87CJEeH^)~Sfg<TX%^+;nA2BibW_o@
z>qPUUc5n^uh!o9aqLnw6?b_<Qr%p`V_*GDUp*_?Ub`*uv*3T-MY#GEH?1`iFH9FV-
z0-tML|EIVCm-_!|rP%tvQN7Fma2uan@c-XA0KeeRRp$TOVfMLS^U0q7ZmAx>|Eb)#
zyZ`rAK1=<7TvRX4yWUi;U#CszjpoyVH{(?e&Tr*nv0VQn9PhJtaD2GGcjUA-H|HNF
z9xVzL6XgsCw4v0W`Huhg?NEZEbLx3sz2_4a+6>o)C57VN@Z2grdSfj80oG}scEWBa
zF+9BU$^1i`RdN<|=RW_6g{Yum*%veX>_8YvQ8#R#45k1ibtBQOLv^Hw)|oUZXxmX{
zHke^YLlKY8n|xA`eJa4sje)*4mk5oag?nc?*o+oD6^i`efwCA)OgxthgV>qm6&sp*
z@4V^EyU_*{ta}bQ%LGHE2atb=;#;{{D)+67$&R+xtwA?0?DeS4Bur7=)|{O)yx?=X
z&k!U1D7*w6B&`5UQELUDpTHO|4!B_buJawEh`AC_Jgxc=6w_oCa(+tUYE?3YuM5tK
z9a+lzJq}@*@3(&#@cuK6YKcAK(F8rH=-&BYcHWuy&TQs4aH5PK24-;Eb_J?A^qdTO
z)oEa3Fb}skSc1II(d*8Hj5hr~4*XtbcGE^8jnlSWd$o!8GAS+$r4#uolP(Y3C6xJ;
zDAtnNw8nI)unQekvH|mw=qTWl4=)W=Krc6SW@>6BHLx<ZwwF^YS+dPLgJF9xBIFeL
zW@e~q%c$76mm(?$Lvqa*1KOQsWWYoHTWi!1&5V0Nx9GbU!g#9&lc{Eeu`R1~aUUTA
zJ|dyhh&A=j$0}pT*GH^uV2WE;rsuXgq7tT?DXn^7>zI|)X2PHd{r`d-Y~dXnlz@bR
z%(`x7miGUG%zPQTU4^ehGbPU378$Rfx7T9p<{!bc`<;Swzl;BR_zx)E_we5d{yW8g
zXV*Do9Ukop6e6B<?I$ju^z0{PKIz*}DtzMEPpW)!Vn3<z$*KLM&L?N~lg1xv0}jgl
zzVp_XIMB}c(e-F78OFggrZ&Ly&dFTPK&!`2XX-hNnb#K>67kJA`wvd{gEQt<rE@Na
zO`Q2zXD)?1L;4_9AUOj~W@EK2{N2nMjRnfcqmzHEGC|<oxzignJZ<g`KNKD3m+^aA
zb6nCAhrxLCun!ADPPd#)$LFjfaLxz){?Ox6nB<>11GX?%XuTiIJdMJR(;pKHB2i?;
zhnp!n;3Qzt7=eN!-IYr9LZw1P#oM#Yx2MZn01};a7-+n|5|-XtvUDl)UowNISx^?j
z&#VzjdZV~G^2QQ|NpBMCXrc1?H$AT8|C?Tm|F2w%-~Ur?xOe$KZ{u^b{yzp{Oojt2
zg+_hvLe+4KMc+UQ)jN9mY@cTmpYHwY)MBnfHr<}7KbBWeKc-i3no}>AdjvyzR#ozS
zBrAA!5-gXu({$HOhTA9Et3zT<Jp41t#?m|syKM^2>lKlHt(_Uf-tuC^JE2a2zZM6C
zoRhabd#{-c!+0<Dp%K)fP{_O~r1$7_=xrNUpRkXLz;WIUW&<7&^m<U6Q0{)v$#~?2
zR_{rB-PwIf@XY)r`y@_@sdA@ex+5k!W;EPS)A#}NWf^n_qpS{e&EeizfO~H-owD4e
z@fWAc74UzF`U8e>U3VXnyfM@EScf9(*_E*HN;cGeuXED$4a%fBphKn*+x`-!e1^L#
zH+b+m@7(hnF;2`SUQBDo-RQ*5!rZj-V6gpq+eIe~+M&K&XD3#k$tjd}lPH}@=xd&#
z%jkF+1e2$qy{K3+4<l+iZBj<vzj22~xK)c1&bC7zMj~nA6E=C0C-l+s_+<1hbFSZR
zXKz`yK(wqcPHsR__?9dvgZ~1gRzt`i>9j4qb@fF8D}`3spN*p;YyFkx&+Td}Tjt+}
z#J>Txg;|lT2(DY@UQJtFQ7hcM+Pa)N&9vKQT;zU9Yc-YA?R&>sQ=439ani_Rc0X#{
zS+#v$TgGgkU}Vp{)V4xxY=_Xh=Y#ioC|uyd<q(Qafn>Ym+#B}QIDW}i2i+VB%b<M8
zE^AXLVylFiG(c5gvis|UOfi4105KEEOLja9%Pf$kYwGT*a4zf0mNjc3{KRIh$g2d0
zbVP$k-dR)Q^E<5XKG*uWlK+3vnRZ6=!6^0QzxphR|F6~>vH0J+`+sifvx@)!pPn}n
zr`v~7PhCnN9{Yj#MDg6>gV|ZgrT>3l?H=wP9P>Ew(f)t-Ha41WtGu}>hEq^WGI33V
z>us#76zz285n%Vr*L;y+l3n2ANW}b&ivMhE{O47!T&oB9j1L0ro^C`@eR%Yryy;c|
zvj3Z1>4EIh>l3ENFibpib}|?a?Fc!QY0jjgJNl10<cGgH!0zZjkMDh4IQ_AwHmd{w
zv<GB{PDeIbo!$`ewpdZAwCM2Ut&M#@-=H@dFHX;#sdvh}+1z0d_SGFCDhVS4SOsS?
zT+AHkE47^tol|t2W0YY?s51y$?Y-V&s;|K4gn(jrZGZx&yO^VEJ{o~1xIgWX(*7pC
zRnuAEkHU{_3!J`ZrL|NjDD<YH9)mXUr}v>ZwA{{8T7W3HqJRN?Fms0E@x<@K%)|r4
zjQ$Wa#B)VjGeM1U#DTsZ?{C<AC<hWH8D%sXxx!K{KAhLUa)3Vuu~6$C>)1eF{}HTB
z|8MX-OhAds7(3vv;fEb7MfHFzi%Gyg;7QJhyl=ppQ;HVB%(J8aBrVj`<+UI_2?G14
z|8Y#V9#5x>35PneJ=mFgqfSCGJemwUJy~ev{4}HmePfriP*>ZlE5$haDpQ4iJ3cvK
zT0v(4%dW|s#2@B5c|`_d{*4!zvdL6bDH!DN`iychts4wYN_&@ibwwa~vZ@2=wD%cU
zWz@3iZeKRUA)lTM$M1C>)=WB@t7vEVo>N51uX0g_If6S|Wme68+jcpZN*CR(rC?@B
zH`S(NYXtXGG)#f5iaNTW6BG>Jy~N$gFN_*9n*g^?n9H!5;{VI8#6oH*{?wnSU(Pb>
zn(Y58<^QZy%6Iqwex{Fg{`Z@84E<hlQy&A4*t(CEi61>oOx3aSVd72~k96|T*Nf2`
zb+A}Z9`z$Y<<bB7GlIgWyA#JNf37nBr`>Cb|7(=v`G3m|zQ5_t^YeF~|I60@W`R5H
z&PVMo%&<4@GjDKuHqUk14ACn%9ZYVIiHe|5yRdMlx<H;m(0^|HE>^HA?A)JrC++T8
zd%m-=p)-?7HZ-m>%qyBC0xG}#1O9E3@&&y1!E!O8lA1Yw9erWRjt$juERcf#&^wu{
ztaee{P02i#HUgSEDF2X3kUOi?5q!9L;UoIsfsf9{r;SjD{h#>#NoU>3mGZHu$I-a&
zwP)&&74@he2QG<&7f<)v&v$>`e*zP$)ht&!Ck-sIx!9<b(f{+#>7d6Q;BK3Ba|)mz
zDams6D$2d_K7_(0Y5Yjp-uXnQ=u=hYbVK+-S5}2g%m@3(@=q*)f%i7o+5SPWC^!MK
zup)o#^c8lYd%IMX?g~}J8WWb;{sGuKKR8|al6SWKXWyG=J8E2|hJU{ieHvW1KV3|Q
z19~-Pa;Sg?rcMw>-6!`*PCZXf7l2=Tp3@nrik)e9FrRj&ADr<ND-t>d$k3zlXnVk2
zeE=QOwQ;Wf8OrSfeCP{M&H&SiT#iFQlY5@;<k1=XK{ub*`;^<UlHW&5s?Ahmsp7Bl
za2fsrBd|9<Bk%_~49`!!QBDniK|R)gFbP#@wVR=TfiXA=(bXy71P8WNb%6r3G`|R|
z>30%4%Wt+{OI8E&S|I5H82@AZwR}ACHt~qPR90t}{mRy=g@~lrlQYU$)iyWKg(CS&
zZO!tRuqBfU`uv&(e&%(iy))H7!TFZlc@xcyZmFHg%IzfhXUvZ3rhuAy^t&0>bnf%7
zb#mR-L9xeuZ!$j%hj6QzRSFpIzGwmdu%1l`PCryag5N16Bx{zMq1IH|-0-2qluLY;
z<R$pVv|uWzC#f}(bWlP;;%0T88{PV@8b$M&a}FDa6Ge_v4ZY4gyb(bp)l?znG!Wsq
ztUdpNyMO}!2*?z^oihWNUt3)jx3P6VNWrug<n+;Zq7{(uGOY#GgPZBnThw>49eKkO
zMVPQ&BAAs_6JR_;H=Ws{OV$pEoJcxy>VwBlXcG7Z3lYY*`5hQD=pq1-v)-}=yQ)F#
zkU!#>CDS<xwV>qn;uh0cT1uFys-%S4DuDj!=CpdbuS8E+Y$$&fGZ%)X!hL$)?R=3e
zV#xy~&U}Gz`SDj5dUuVy*Y^im;2lj?T_EU0L^GjXtClKa9AY+ryrT$+UM-TOyoJ+f
zDN>^=V>MbIt;tffCRavlvLaf!`-8j^sqqw)iP}r^Wr!&g&J>wMB7Pm+7nP+m!r@jT
z$U$g|43;Beq7T<X$V4n&6JoNfA~Q?J!)DFcAVrNRCP6iER&J!21cUQ#XV@9_yqRxb
zTFqMO^%`aZC8k|CbZ@403Gu0C7n8+^0;0Z}@WX$4)*3=GvfAqCs`F!2WQSKq))J74
ztu_<<2Vn)RP>m>qjbJRDi1y(R+dmNI0ee?L<)88asYFz=h8WDiZTfkCD_SKsong=_
zp4@g5lGg}HgAHuCEiw~kXl7mxc7k$LtbS$nsF2Clk;<~x3%HAUbb3W6yq3ZS_G-Fb
zcgGe;x7aO-Fuulai6BIw`E>k&eYiT-7JHq}bGoEr@u!~h7OCBK;;S~>&w1ybloR$e
zmGax$iTlhDIMdFkKR&lZU=C>Y969A$@Z`tcpZTB^gvspfJ$0&O2&j{D$%nzewzH|*
zIsHum`eP@>Ng$Y6g+igz1?Nq4-iKvB8)T|whUlJpBk4zuXeFLMU<a(Qiy4+ky)&vZ
z`Pu8tJg4-CYX`a6kVL0nNF<=Xbukt#p%7Og6SBx*mtS)u^tHq$cc)&5{x=2PjYHRb
zf7w>_Fm0pQw&RAI#3*$Z!}+82AR01a0H?e_H%A&5bOjxvVTX;l0`PGNq6!|7eh%h?
z&XCbZ9Ck?uVy#9u;?5z1>l=mZ8{qmHe|(@DiecXTk?gv#3n)(qJND)e9Vi|qe+ZN8
zR+U)SaopMiVj2>GfX^}osOtF<?NLq0p#Dr{SoC#aB8=eP^K`WsgwrRh$5{1{iZs5m
zHxFD3(kdD<1FEkiFAd!WuiON2pJJrr+9f~_aLB*7ksNRpcQalCJlp-a8n}QH)ybVW
zANmE<$(h6g{wz2Yl@Xo=zD#uPIW^7vmS9YqCGsw$IHPSBL_)pMIQlZmmWu&!LUn$S
zHnz8y1D7;ZDWJ=bEV&^%ii!7C(a1tX32k{Jg&hCtm<?WRtl{|Yf`y$%N2T;vg=B9k
zYN%O@Dc_V(eN%?vw)*LB7pazv5g(sMfh`d5j|^&O=%CT_!Kfpd5*d596(i3lM+j88
zSptO?n%RVkn}AlCFOYIoL%BpU`BPC_gQoL=aYzQXJF^Ty+uU0PNn8JKQ1-~7GOwQt
zj!o$kgRq|h^NIQ%B}@#asCEArO(;JU6L!{R1F6jJQ6c<H+o4}h^weMfbEWtXF*}xr
zfGmmsYE<g6`(G>7a^)`m;}$-v-v2mNh6R-X^Hg(bE<dDnPRBTp@M;#G*|~Vdd{0%9
z(td`wXt{ho#f7M1M-$-63SAmMF+1DIr#e(s-me#IQKw{lQinIO@YYxcTkOvF-#eAO
zwO*X+1?*I>U?W>o!BS?bB>g};Eu=g8yXsp$>3*gFKqQbqx*hBH&TpwyQ1QM2+@MU}
zSd)6d<O01QB|FQhW~N$O@eG5896uPHOler7(BYjJoj34;Ne`^Ge!HV7(fw-;LvlN_
z@F0Av8p?n>lMzW9UMT*=!DXOC-Uq+JAWw8grQrt$E$W*=SL((8&yF%F`Hgn3w@<NF
zLOtwWt%p}&<nCY|%@3#CPzlVs3a|RFBpB#7`d9ueS*gcDK|Ze3k3+rR+#ci^?c{zU
zZR2xlCq64~aau%OOB#Q6*w5w}bSB|+Q5q`>$zK1Vxx(7jUxfo>E7fIaS<aqi+$)H2
zNvpWlx6r>;Ufv9@-3qMvBnB4$82FAS(X65aVaCD9pw}4*CpqL(u-W5#>IfD7eIx%`
zl?W6?AK&|C>!;_(7cY+ww+hZ_>>(M!ucREVKHYUcgXO|<Rb<9!4aT;0mqPu;2t;9A
z#QAupIT}AZCw2ljZyq^KrYT{cC0m)nWR<uce|@^5qzOFkVT~DfrTf13qrGFtt;am?
zt9roKP_Ge@#g_j_7dt-gufHrIbRZXL2BJm}wd}C?eGgpSX3=}oLTM$*^7@akJ`CDe
zUmdD>yNo)Y1+CTbx{3L_f>j;q^=*#AtzeWTz_NWpygT$doIv4aS|RpIuPfp&sQ=<m
z@4QC|pDX!)79$>R9L=FMvfMyR&i|Au@%-Pll6#l`<2F7^{XbFrWX1w*4n2*9u!S<}
z!WJzkL&aQ`UT{p=;OY6GtFD&_eh)n!%hmP=e&?Ns=9emzx6)6jjLe?O#|(5FLCQ*F
zLEkuy+4-3_m=XbbH&Yi*pwi&vg8-5nBy?H`VdK66GMLFzlSi1}4G^8aVKa37yP<#;
z?i0S1Th1HLib9|*d!t8DOs;qjU(g*@Rp!?AkYSV17BAwRk3m7dQ8ZC370oE1b-YBA
zis;#``M^TjF?RnLJ7L*~9Nw~j$F)r-e<%|#FYkUJ+Jc=z&LWgSU(re%e>TOE&=-J-
zt_PJfB>-xRvi8YvJf8Y515@)zxQvXTi>jUGVA;`uiMl<WIDTEGugGs106xT35$Sn@
z;SJ-Fy%-sGrl($bvP*i49(Rh`<L+=E=7SxWGa90Jx)6}9=YqqY$=4ec3WEG+6bhtN
zsz6W~HHr~J+M*%yN>9?5q*yK_E+wdB6ho3qkL!(!Dz*VxbyIRmx?qQRDNY-Pr=R*J
zLO?O9EFo_~s12`6su?&yD#A9-e+Auh9yl*|f8KK*1&>%+ddf3N88T2})If@8v9N@)
z*Yr`rrs!!z*fW`%D~K?Tp-xy8r1O`~<c79%3x-b1x};If61?om=SOeTq;CDeoK8)~
zSBue%Av3GLZ5eZ(#rsLd+3thET;`iv_sS6<yAjMAUsy*xpb91btu=cBKjRa;4PVX>
zZr1imh8tF7^P#JsR_Q1~(=<$gOo$CkfWt<(w~rO08h$qq)0tjRs^z-$d^{S@$D=_n
z(PmCjn}SSSXQnGXU#(eMO*EPm4|U^DH6;o5lfHG$$MK7@a=#1(77T?xDP!|TDaoDM
zJ6nw22#@^mgY)d?Hu)s3ebQBqCDjLX#2OGcDYJ)OPNQK@W+6~ie$ZhoW#a-)bs}ta
zM!`yvh(td1M4Tiv9+x9OTu<|fz1K5N*NOGqpC*CKx9TYj)<rGcWG0n{o-K40!4e3R
zie(yB5wW`h{VWu;qurh)iIP-tcSKgM;S~(~A8apQc(G!?2}JLm^DpwbivLdpm|j|4
z!T(pSy7gH8uSWeY|JQANR*nBs%4(C|`18(1I~e*PZEs-GoGHzhzP6_dHmjj9xJ&DM
z-DXJR>t|7&uOvhA;tGb4f0S`^V?`gtYKvQi&6Sq7Hg>^^)bvxG@!3pSoUXYx4G)W1
zpsu_=4b#yG^Dxll=sa7h2wxeVv5<-K47xlR@m6Mi&m`Z=R~f`mVL8JTV`8DFsidh_
zmLGAPjislHmI_V?x;!uQkt)L7b1C|LfR^*W|Ep+e)%-r%D^=~mgD`wmFV;xt5U*s;
zi8UImZdE?wQC|}Kb%u7rWbf@_@UAmtrd8F5A2{MKZ51YxQ#yC=n~xV<d}+GdV-%uU
z7`2REe@-L=EY;blptxoQt8~F`Oux<tIkwHYJ!_ouk9=I6?d>0YY@G<Yco!-ddGGf_
z*N^rl9aRgL<bwIQf5e8qyuMEqyW4T1*d+T)5G9E*9=Iu>AGiS><myd}+XGUJ7g(Mz
zCeCC!o_P9%Mi%k)bQZVJf0<sE7rCe)*mm>0-sq9Mv@qe%d_w#4v7P-rn*j(T_`Q9j
z&{JVEkNRS+-SFAKLjXA}F|qkzwLXl@Fj{SW+T2_sE#flTX!zA-hMhn_^(v~DlNmUo
zwEUJ=pO_)9ee!4}^Pb<ym3d+<hPxTqX^=CxEz8$8xs(wzF;>h3f7E+^Ti1qR9p>|&
zedJiC+Aq3T)O_LN^6aC$hUp@uzV~ri&2VAKWV<B<J#Mo^T3g!~V5QY;0Y&1^V6R*o
ziV>xxN}EAX5~Su`!o)Yj0(GqMTdIw%a&cvB51!qD4D@`?h!<I6mt@rpI&XTI(0N0b
zFShduLrwD|UiyY?51E!B(}PVfa`tTY4G;Co4fEaA%>xA`+5EVf7Bb+kZiGaEiWy?l
z?ZK-3Sj7kl3xTk(H5zUeF|ge1h;nS2!LiD`TViUY@tWPCE1Lt!>~(YS)1hC|<S@5?
z*w#kewYH)i5$38?yHc3>fGPKn5>4C4(LyQc@Ma~AYb-sMRr^1$axae$pYH$kQ~UYu
zulbv7iiO+9(#)0Hh_V32Hi2fPu(=rqT#~4Q#b~ye@M;7dew4sosGCHkYXcsl%Mz$*
zWblPxTu7)PeHo&6Ae<t^ez|-^P2EJ!x&CItTyrJ(n@Kt&i*1b1xZK#P+L^3#9StP2
zPa5P1NiMrPXsx^K9l%N23tw#hO?L*bwEuTL9?#Eab8m8``#)>tS}A`2XQO(z|91<Y
zRrmkYRS$mZzF_Fv^G;WtTn>&O9`C)(@m;|=D>xSg=R@B4eRJas^X};Rix<bguz-I;
zh~GaY+#p~z!NldK)6P(BGx|6z&be=;wh(usDI9cw6t9*v)t&K)g}AxY;kQz;=>FkI
zHam4S!_09XmWz(_uY{Q3lfeaAn2gmm7n7+s!>V4Q1HYRE4Gn0@$?cY`Ia0A#v>N?X
zsZM_Y$^wYoOsU#C^iDfdDPqbN9OmS4=Db2lfOzJb^O-6GT>9hpBU+-wZsc{QOcHBj
zDTlh`)(VsmAU>Msoey+Rj89IS?gz#+pD)lF{szO;KH$Gc&SE(6Z>|L4qajZ^!ZudV
zzwMlL9z2fn&EOAhl=9@|KmYHmoIVR|!TIs0m$1e2Ksy(M4xJ_Ypd+~(^TOFqK)^CZ
zbRJ&AHd-O=8)`5&rbL&3>hHXf)9FbLIdh%Ie7YE^n?XY+fzFpJKs(^|0G@Zz^Sr(=
zlhj~nr8>*3PD!YVHko2LPI6oKc;p(;wuIE`ZbMdea>K#-VD8-O=YJph(KxEN(rT>B
z%R>>ZQHVq(nz_Ybo%UJQq_QT`2?(RE2%A|!M;H|~wF=Il=oOv8eCEuC;|T-l1q$y+
z8G+^JgLk|HcAv))I{?)4ocH)YCkNPi(wTDid&-57%mb#N)JgsX%#1uM^K&xrdft19
zEY<Ber&F(^?qThWoR8jgT#&nB`(wQleY+S;y<mp(qF2M^iUjL1x_?Akh{(zb3?EGl
z4H>E00`t7|yFlvr#4N~Re(rLKhrvhfOr6HG;L&ssHyJ~f`SA2rMZuH{&p3t)KjVeG
zGQM(ugu1lGUA7Fh8ka#xVk_O>zBiP-yQ{F68qgWopf?|W0B`j$urTHHWxE%TOU0Tx
z$RXFt3167PeJ@(rEtY)ouFiA}nIaQq-+P@mp1Q_YGlz&<C=P!t2J;VEnsyj((CL#h
zbHkh}wwaTG3wq7xSk!kG7*m=3Vt1O*(pe(61>{+k{hWtR*?Exg{GL-@i#7ccruj3-
z8DAPiI9JNpqZu4glJOG8@d*cc7mOUmL#bF|5a|W~k@53=uIbz!rj21C`oa8SX-u6-
z*o0X^QvZ<fhP1}J?d?m`RHM4>Tn|kRgDaZWe42vIuSC;JM`*?jw?gvkK)xlvR}$oJ
zx-z?+&z07HFnluuKlLYj{a2|b=KrhS-T!zSpY_jwbI{wcJkLAR54N-Uevii6JL{M?
zgDwRc6OrPpye>eo&g5D=8qAZ4*QaR=9qWe?K6dnV<I<G#Urxz+<m?^1*gH6mMmOpv
zQ-eb_hpJMM!<7n|q{vW%Nm_s?I<Oed<7ZP;x0fK}6U0WxqEdj64pW6KwmL$YEyt*E
zJw(WA(~Py<YB2sc*RJBMtx!BGpIm*wxr(#-GZe(24rds$ni=71aP`v)3>b~oDP_Gt
zF^n2N?-8FyK1}l_9c&8Jc`i01QxP3{vzY?sk7znLP3jeZ>0w%7#VS(gY;md<6lzc>
zJJ!;|n~ZyBf$Z_gVwV}6n@m!D^MzJE8kpHqYfXVP5$}V#&|aNE!RlN)?ZY?GgtEL(
z;@n0E`K8CEb6e8aEyKp@U!T$CrJq(+e)-$6sWI=V=j~RLX)3_IROA3HlW>XW*w>i#
zq0<3+WY@w^b3y2&fM$%V;<h>t{WuQ&+s480eh3Tm@Tzz$mz1Y-Tdb8bC0fjn6B0jq
z=@gsZ2BLmpA3~lpWgu2kote%PfiGrIqM(O1Xp26Hz$3CZAhL<q=<A6-QL<sV)KjAt
zGP!)0iM-n}0&zH)&xcezf&u<=oHK;vnxZZUE6`?JAXq00X%MvNzZ=Z(O&&<-qM)dk
z+ljn6Kf~bm6?3KuHF;SGLJu+b<xxKM6iQU6DcV?~`sn%YvuE6$eDZAf=qN)_8>%jq
z)*FhG(zHQV6KcGcx4;3l8t_jTXTWBal~j#kOw(V1*tfe9d8Qzj()E-K5yXu!Lu6=I
zI)VMy)`qJXp{Yi`cT+lWWJV*9`Fi$V6|0q6*AgP4eYMm`I%Ia;7%0hPx38XDx&KQv
zo+zn#&Ja7$ce_RxjYpo6n?{$MbznD-d%eYUs;%U>S_;WqTPewl@S-53MKvAbwzne7
z=KQSRmWxs}oge_he!tj7s5(b~d)WE-p$!S~CO8{PVBA0zjlds0K_vl(AbH>SExs2V
zzo~+MBMTqB9uG!2DG)T3ts*%rSqgHJ;+%--$ikAf!aD1N5JmDk4ljkqZ->9Rx%gg2
zs6Xyw;OCOQM;xJeRa7}Rz7P#Mxs39>!YW7iN|B^^GF;4}Qa-Yb*Q20mXb;0G=IUQd
zw+O4-OD(;lm+4UoOSCMldri7VVfmRCTpEebyW-MF1WT^Scm?Z<D~{K#ue;{K;;59d
z;)<%D+g`*G&w$1jrsm<&USDPLCg^=}n&}HvEGsXWzDD<%bpDGM>I%!xEYvj;pI4}B
zB>uia{ckMM{k@3m7wCU|DZYwd>T49>cl#foegET5RBBOq72(Ewoofg;I;bSKrg*AO
zXX_cQ_<AxJ4&^Tw`Ch30NI^zvN#6XI7mZSt1a1#TC*$8re-uHn_RY!4znK?hU5Q9e
zS8nOgDH9DApVyo<Slqlldj%D%k<^kV6w{?1U0s1(2S15K3k{@_af!%9r<{+UKl4^3
z>slBpSsTVtJ-b;f*JQ?M8s{L3^OBvS>zfX79OTipnsS$(1rgkgF1}ea>Wj8~u6L0b
zwj7O+xt!f-(Ns;NaE^b2ur`OJiG)2uzRuVpj6u_rIvpMhWB~k>KvJ26DSMcbgWiJH
z^>~`jUlk=2bwz-nt)MpIX`L|4EBR;H{xXYqJx8pzkNv)_9as1T8n4fG4}ackKi@yN
z!EL`U*jN-U`TW+RkH%;Aei}5sXnWCMd8aCGPgQDYpI@x&#MoCA>>+9HCl&u11p3$G
zs9oC?s=97V^QiPPTyRC%wI(+G6m0s}gUuD3+dhuLEQq0LPmmkk^ZF}^RhuX&vQS=M
zT^g=)OxHfhSBSlk;NIBNa4dTIdy*{mt%@a1dz;y?TqKnW*V5+cx*&gdoY?wY`TVa+
zV3K(XIQ#r>ty-?uV)>uyHTUlP?=5_mp8uVHn0U<7<c${Rl)uL>{<(M1KK`GVd+mc4
z2YZFklfA>k7l)BYFZRO^bbz(><fq+dM-f2Bhp!?}p8a^Z`{bW{$Kj`kiBF~MUP8&p
zvnMZ}y*P+Idj5Pj{OIWTaR1=vXeXZkxOceu$(fJec%wFn)SmAI)SAtwi{6~cYoG_(
zGxG&SA|*gk^KRq5`o`#I3##?Qw?j8JCgYj)Yv_&aUxSfdv^O5^1eoqHTsw^1pZZqc
zciyx+p0R50tTWxX-(8$YwDNbqm%iJE{yZOysE3Lw36z4)mRaUtao&Z79iKR?+ksY}
z&LkO%Ckd?xg6Sy$GE_a(jWFJVw&}e3yCaFh>}YfKdH`vZ(<Pnz@}=OD+|gt$gvQGH
zQP?!LG&9tUO`dIP=TYz-xcNVTnu=;CjO`y3eCG8Cjo<bUK^u4(8b(udEGgH82>Ow_
zZ%7TOd4#t4Op!-F1P-#^7{(&-RYQ_xEJ74PnpV|6LYQy=phsRASju1!TVrGPelYKy
zIk{Meq-c*xM<N2@5g>5sFlFX4uP<5Yqvb?QJW9xc^>Oit|2dXv=(CgLYh(M596t?~
zMWBKc5JpU}E--k5FPLl{n3O9v=eH1o9d0h#0_a4#y*hYxwD;7RX(qt{V1V9F2XilX
zbo>;3ZvV9Z468u{?c2y8U~c>t@Wmei1{gTiN(BGkM_cjp7n<#5O2nstJ<o+3A{+pW
z11Vr3hUb*q)<eaoCF_Y|X8Va|T-SP}*tV30_6hKgq3yG+jnF?7Mv7&9i#RAyaCk)F
z#gkv#$4_>j{GyoC9Ck5HlN_dL9?fFX>=?~d>O=COb04D9&Uv9P2b)jvCC7JKb~^VJ
z%|aMIHW>d`5bcWk4;)w9tVswvkDO^z=j>s4;4Fu;qbL}{Bbe+=Zs%zp^Uiyd54ms2
zjaDOINA#~?TFS@RgUTYz9S@8c(``uhtDO|;3$KHy$%Ai<kBy__-D7Ix-9xH;{ZpA3
z`o}@R(O+H#-=0}tp9N)qGLL`S4Zc0Iz8=}%_kxPY=K1kq@a>iL)hcTlF8aw~?1^u%
z*d?sr#!&HpDkH`G{AKs>DEza%cl2cU<zD2WkD>bYH1ad{`)R=<xi*6I@8|oFc2)=W
zcAw&X`#*b!FM_8}_kZ3$wn{zSd$JGav<3j(jFsB^^=0Ts`)L2?gYc7RmE9*#_Ff)?
ze;u+GWsb!tf%i_SFUqO^-cysL{p1P2Ln=SnYwsWIALqmi7Ah`6EVaBFD^CfR3EobY
zJoIvw_zX>J<0g4Io()WP#R=U7kW#d>FyT%-eZSuOv%%1Fa;o$HFuhk)QS)4+QqQ+h
zl(w;&mPDMsx)(sYB6YV#P3qB&fRF(|WyP3xTJ+nRz<R5+fTa`UooT(NcSWV?@0{-*
zIqKW*-Qepe_&N=~&c74%e9dfN!1`1#!t4RTv6EPSlvw;Ax$sG1;h&NV&l3wDCl~%L
zv9J*+$#wontaF%I=g;(Z_L6ISN^d2l#-qd<BG{5E6jCZc^d(o=O0GcRm|S5yv4Tjp
z<O<1LWXrzfdalo*W&c~n|5jB{)sLG0xbA;z_}`lTw^kx5N-jo`uq<FR0mVH0)=a?A
zib;Y5MD)ZK_JIH`*wdZ{fnRCcE)>XuLLePgkV3@rP*DZ}(w5l4EVYErGR&ih!!}4m
zd)t&>4dXDWm2lljL;B!Rx<p$fOP5z)*|;6N%Amn8g$B>QIwCx~3>CgYGGx)=5;{Jt
z*nNHJ_*&5Mb(`bsEGMLk!g}T}N$cg<IdRu(Oz4G#VhTRtYL^p=KV`5$C)tXM>H5FW
z;3Se@cWrTiwdYqyge+k&OQXWqX4tSx8FcVc=@2uqzXC$+-KmGqp&k-UX65Vp97gkT
z1`kY=7tV)Y*;F__TzlbgeE7Ak9MGe+<szM~UmqFnq~d2cnvFOAa~aLX-M69-#7T~s
z_-yDOULO6!F93bz1idWnKP%%LUKaJk&yKpdg)f1*zfbo05P$xB_fm-K9lg(nINjtK
z%I5dqQ@!hYuW=oxyxL7ntWmlsm9)yiS2y@t4!%}`uhmC(&06rH9(-*CUz@?#R;Xqt
zc+m~M_JXhd;Hwv^dE&ot1H|0`aW_ERt=Ki)YVaaJ+zk+S1H|2CsAem85g_gch`Rye
zZa-Ah3tpUj;kr6xQ3PC)YA#v4n8C<ffl>6k|Mj8&EwH3Uw!IUWQpp>58M{o$M!VW#
z|H=$j)u{~PmwYnb+?JT{PK=Fc&;Qo<zj=106Z^-mU*<>u+Y|rWQ~%qZUFoNc!G8Kx
zj`q{L;Z7OuuXMcc20V4Zzt$1I8}ii@{$G5!l~sO;)k_c@__>xZ610=er3r(07rAq@
z3}w?dV{nSe6j8yVH$W0~cqe;@V~s72bWUAm)uIOeGYi-l-~YXbM0N9Xk4a?THwcyf
z%A|zVv(HWP=rr)6CP6fS_}7<T`(!{DGe%|GC(wg50!jFOvUdFp(<nGS@RiLXL+Q^w
zh5kn5{?cUDS0Q@rPDIa<O7;sm$Ch!@MWqelDK~D0>t~@Oi4Pv^gr50l5{ZoC4ME*M
zqXl#w!PTOOdi+UF&Sf2QGl#t8PrEanc0R;Uy6vRb7<XTLz4(F7{7&LwI#uUgXShf?
zPiNRQ#27_2c<HBmRSa+Ru?gOaKG)F;(Z~KGGY+1t$8ir6GFC9ylGO#%DV22i*xpZ1
z0$>jg>st&NJ1iMt(Bz|y3hbmam?jT~oqH#3K$4f+oQPX50oWvtkxSGZ4DK0EYYs-I
zTjt=WX_*j%W0v`pL!g)94}a5kX?syI`zd88Yz|&B6j`JfFtcUHij&$*CXavKj!#sP
zOLciADkqTHY%woyG+Cdm(&v>_VM46NBdf1w0*7RJoDSK8mm(`KHF2zX7@0b|gq<*V
ze!I<Sa~PLLhH!CyPv`6Wr6f|ythk~`dLc4+WuX)kkYDM%Tdm!6jhQrc<5Wv%T{WL(
zDPCft?_K*ePcv{XKhO7;a^s)3In9GnOq$B}Tud@r4x=_tx`?=pb+S}9CR65gJm+oa
zU2mw;&8TKqQ7?%#FWus$21Fk#O{ZsaTM?peS_K}NbVVxnPI%eHQ4Nhy!ezFvXX8=Z
zHq%vJfdLu9N<uOifue&x7I%}yeD>QPSiF!n5AQq2nrKr;P*5vHf}W|YOB-8%I=^oz
zDg67^Bj;D=ljPEgJRbcz9HzWk{F?e^m<DYzOnY_mD;HbIuR6b`zZ(9U{%V%i7DqVc
z-Tc=@3c&d^ttpIR8Z?Y=%B$bEejok5mHO`Yk6AAlzm`g+%mSrfmlZ&XB?U^qrWM#u
z=~Sth`l6IZ2)=rt-YM37DQvI#mMkmG8}O}N*+2}QB|LiM{N|JrpW{ade@g|&bv|X5
zcx1iTe?G~VPmw8^IU&O{kkdiI5{^_~$ugz^AD@m8O8@@t5B-;D>+dD?$JKxI&u#O?
zEfvcZ>#J9_e{6?-J+Oc6g?@wz?wNwy7WC~9`0X(K?QjFzp%%8oO>BqT_<c)M$M0L_
zr}O*o^3Oy3*TsJ${5QpabBC<k>Gt^7eY*FPBXvTYB|Ig1|6OM~7%yhd$hu`apg6~;
zfcjm0|IID@hQ~d8Eq^Lh{`iA0`uK&1K<g2Q?b9dC5Stq(VBOq6d`Zs0`QD`24a@8N
zL;cw1pFAo&c%Ti>kf=u$0E~$I!5=iLe5I|vSdVPCUqMMeJ1wm|Fp5^^z4A*-6U@UL
z%hIcmQRSJ`&&tK@gC<2n+TQ?9geb3^*`n9;W;2K6lQteX_n4?%VPbVlvV(maSV^DK
zX?|sjUxD7TJh>fmhd{4NZzW?dL2WzJuw-qM>~`&5SEqj-PdaZGUUJ5?sD&A1N8_`+
zsv0)OLmAm5|CqJ{#aJRaa;wtoQ<Ax5<*yafOYpRDA5Sc4zV+Wq3jOz~{(EkIn=c*x
zw`+czFGKUdeChg6{BLLGiTTp?pZMQ~{v-ceH+U9&Jv}!+sHeh-@4>760!_;J)G1q2
z!mso?lo)DYt&K~M;tNIhky?ZLmjedL#{Jeqm(+waj_X|G-o+eaxE0NMFQ{+o4Twl`
z@#KW@;W-oF`pH?o%^=iyt*G-l5G>@0G=cKkP%kbjtcNMW!en=}xH4zvmqiIvPk8h7
zA0c8ckp@}fKmmOG`_{b~Rc9D{ia;={`i%7BsoBS^rA{H1vD=!GciFmc_H&i|-}6E5
zjW^9q{VkvD{NJTYqg;>X|E|<(^}GDvxA0k-|2wKQ)u78CPVjFaNCzWt<KS@b<&jek
zejo2Y`R9>S+1xM{9Q9>K?o6qa>DMSI?V?L4xu||Xv<m==nW&nQrYuzz71XMxEsf1>
z4AeEcF<!J{8q1s3oNecIqM%ge99fM3m&f;@LSN(GJnbfYO_0JCoq~2UV4n=8>7eZ!
z*g(3}ZUmQ0>_x!$y~)sH8p*h|o`z{6oB&NhFm%~P7G{dTsbHtx%$sXV?VH7j3DQ-K
z%)5Rp%x58HOlQ7j+_t$gvlLP}#J=?<#tgGqqlth>XB$oiI;GVxiH>z5B1yx&A{whu
zWp!jE$?(l1v^KBx#*+`rSgixAt{Z2yjxE$IaCij+5AU<P)m@YGl1SXW2J)_oM?euJ
zSUOkjHl(wA4PC$CZl`n98h7(pxEGQi&k~*sZ?0)DU&;QP^_Lre|KC#kuUd9v_rFw1
z^}G9DZsW7m{xgE^_?N@I-KXuRyT`ju>7r5Yf;HSyaN~*ZFyM`N{m#6juNv_;9M!|#
z(CdsAleTx^^%j^vC}Ucj^<}eemXm@74J2>BpG?Q|@hqp`1x1uM%5Fffmpju_S%#{D
z&VBx+=gIueqky?;>>r5;hAthupj;3j1;-+YZ`SNiPZ#Ijh|7Ld5L%2K4beGSjCzvD
zTkk}lcj!_<x123gKagafG|o|Uc#()~A8u|uyblp@-`UmogE{>Lw=aU`xR3IF>`cdy
zNHhBPobj|zNBGzQb_>{X^Rv!eV3=Y5Zbp>7>8LXl#HJo~*Pt^TeC!D7XX0L)6<O<H
zu<<G+v36Ml($K+TsP0OTfKVP|34>3DA#|07b|P7o$}3hX2b;O!ZQQ7Lht3@islNWv
zx(w0Pmmn^?1u^Xs#Et09Y|8Cu@h=@)(@~7W%^V}aBu~7gji^*>^+waK&hk;_FF1a!
z&{QF}#b%0CxDc9;`Os|~&i&2#S-&lz63G*(A55aWT)jwI&DCrQF4o+P&l|z*6r(79
zFB=~nt+B5dPS_9S%w*A;GQy6Qu@MZU%E`a-O_tG1B}2USZQp{G-V6Uu!`&0PXArof
zb|Iw`Y8lDN=e+OC9KG^E%i+60)0<1Cyusm+YbkDQITQ>^?uPE;9*rIE<b*LXAR$1j
z)=#FMmz!8&3OTtYBr#Bd_|TMTs~by|F%z(iaaLORl`cB9lZsa<xANKdw+iJKT??`R
zhFN{q_lGAzDw+Te7VgP?k1*QKu$cDI3Y@5xe9G%8ug5UVq`dBR$d*iO)X@7$ihLJZ
zo5EZ9r580myze|2Q!0aLpfKmN`4rNY<9H4uMT=f%G98RXhWm`cvbnt4#iSk^+2V|G
z@Fk_gBDV+aho_Qwft3`qty1yGE@raZV(X=#;hiiyHq0l5$SZ4{@BF-<=W`YNe=#2n
zS6KjL+yCXVTW!Sb|8lK#XaC>AXLb9ZqU+_0f0x@&_V%CUAWdg-57l&#clu-JPezjd
zZf!OW@NWFRydwdrd^2}0^bh%Iebtion{NQ|n{On_6E)jCZ!lC3a|t_q{zbtNU@7qc
z^E}MlrtumZ9G177LrpjDeE+>OpyHKZQk8;?Pt&kcEqqGuv5#c=nvk@xq?;sP@Nu+W
zrK7q6rb*;vQx>+DLH0okvbn(nH=l~|5`5XDtT%)OxwBnUUqf`2q8Bt|t!7zj+szAN
zJI;mk{bLI$#n1-X3m=+BVtL;$&{tJ9+*_6__;8{-1ghf&=i>3bEzCE3(KPUh5T?f`
zN5{#R@^SkrKxt90D#If(warcH1qUSx&QH&eSu)#dv!GVWsZhD=1E`g<f{K!>LjAZ%
z1ggRe3eLv`Smom)YGg&>`8({>`FL^OxxmVx?Tz{q9il?d1<WA1S>Bs7igz?t7lP@L
zeZ4Tz3O1A+<M7f$5v;q{)s>K^K5PrL#6jR`th%)}S*L3%epp@fs_eMrNT;31L(?vn
z%rxBkD$I$L>2_3<zfj4Pl76JZkhkqo=Ugh114wkM`o)ffs2ClycX*id@*5jo!THAX
zpZ@dJ^W9%DbvAN2A4LA*n{Q~R2ZfFj3gztLEFGbU#AW&cK7Vo?RaE81wp$Qv_Vy3{
zwfhXS_SOEuF<NNjzfXlo;dRPZ8kvQqv=qiarKNBmNYMwYm3>fArRDGP^4n5+dudrI
zTrEeCE>Y5{k*k*T28Gp%ZKo8IA>Izkm9oH<tY%fgrRBC$%Bgm(f~AWrfERYltCrkO
zE2$gTMJ;Yh25I2A>1EvfdZXs7R;*ffq?og$T<JoJU1Z~5EoT+8iP1<$uHm=oX3(tZ
zmugC(W;SM}wv=k8ms+EnrC+7guS-f@oGM_a%b@LM47-I}Kn^#pplVl%4Ed8#GJxnR
z7U93;WQJp@MnK?Z`e3y>QmT{OX3H<8`ntQM*oh(?|8?te)M}>OPd`h!pO=*@T?oYG
z0876L$mI|}1`Jh+m#g^Y{<E~4Ql@GH60glyDd7L0K(W_p8G-AfAN^p*knXg@r^8D1
zz69DocD_~GRfSIbrj>05D9cE!2eBJ0lY>yhCfJ55?R$AG`-=^2{~+c}!THvQQgGA@
z8ZH7M$^44jU#e2rhKa|m$YbZ83R+iDlLax6oO}J?dh!HNVh)*BG|mw`HA?6pk&5*v
zZlWcHdiyU|z%UsN`Vxg?z6UXE2b*O{1gne0C?`F7X}By!CKhU%W&n?rf0KUO5x2$q
zcC^NiuYP*Acd+5s^fleQ1Xn#Hm+GKppDlw+zIQ@ykeuIF4$)%dH&t+o#iH*H`F%6a
zvl_dc+~Lhjew}vvU%Tzy!=I1ZZM~|qIT8(wdgl{WKD}8$+ID5>K>$(Ot8+~b(0{w=
z3<vWMPEL)1^KPa`Bfq5<rmULTmhagJ?l87kLEnU)S~+MW)oBu4zbe_JE?UQmoL|0o
z)R@UHBZOpNpxMfmYQ52HQSt^qgoRXz4xe_FhPgMx7e~s)ig5FdKu}1IN2pTxD=(s@
z9Y;gluFn!dI*zI@g;L9~kSSBFtWfHH@jgq{3r;1mtkqYi&(7k3UBG(3o%DV?R$x0)
z;(=A-fepeB<WqxXezDp*6FF)6mYb{0-_$7^o~A2IRILE<Js+L0!A#KNIyQU4%lYwQ
z4)L}Xp6W{{QJN<#I#<+~oO$R2c}IPb9UAqfR=S7rDV6T=Eq>tFz5YG^!b7W^nIukZ
zj`$@Ut4J*}j}nU9mm;+!NHTwPvF#L4F-Y5PN+~B?thS`sc0xJ3pe}~Vn1+`rp#W8~
zk=LcRD~Tj^0&umo7Q=<o8@0>b*T9m95iaKsQzA%v6oR6OooRQvy?&cz+xIT`w{*cL
z0kpPypo@HyT!wO3ml#U{KCod}1-?lvVHH@km2j%sXj<?4)_e2XvEDmL1s+%h9wZgm
zj<&F!QNUSR;z6jy>a8s4r(J)rr2ZhG{-p;Z;08Xx@JARzKebE#%4fSjJ~1VKO)F_&
z$=~mK-&wDeNu=M;<gb;R#>c5z)<q`#>&G>SHztkz!@epz9FI=(ij2u*2o;Q$3qc&L
z3UQDDF*HQOu~o8^1#*!EGCF9(3mdFIBw3?3{54huZ_}x3Q_WdUk$CY;O4}4%1$wM_
zHq~viJIiTr6F&ufYy|8faMl?Idmx+@N8{~UgAp&CH5PBz8j5)7%#nDz)<C>ne;nRk
zav0uTViaC$y862vS%iue&KiT)gLNQB3ulhN>v5!X)&RU-ECU^-|Lcn=_?5{2`q#v2
zQR-W$%{4tI^J#x@a<YV=@zPlooe$Q69V_j_PAvaX<1$n0rc6wtWwY@3I9LPjARBIo
zWj>C>g|j&3<6;TmD8qcbhyY)SUq1etR7<l<c>R0HqadOE`a<%HN;&z($A$eoefK-7
zRMNYck1eCnU1Z-E{kcl~-}}LQ<jrPFLjbem|J+)`ji3K+G%9!Tf4A^i8vmz{1?aJE
z<J}vT-;T#;5}uBygVUX8t?~PjH;q@bA1Bsh<bF!s=&Qu~eQz>9i`TUut9qe>O0&*;
zt4BMrcUH6Uml3?;1+2z*LZ>b^6R0VtaeAnsQ;qmV1`MJ0d*saun-sv|Lcx2bV#59M
zPZE7f#=<6U8X8G0SoM}Q5go&Inu*k1)|5TuYZBGGo_0oW=w9X@nKiS)JFgHNe3K?*
zGa@p~3AoV!FmQY0^GUqJokDyxyQ=&4A=S?I4@9Wobj_Eoe;f3}2NAgUBgk)sLiVBP
zxxoZDYGbmiz9_YwhHiW6z4Mq}=Ch%_osD;NQ^8`NjQ}$s+vWL>S${-4`B%;@%SQH_
z!K6Jn$^8S}IT@T{JiCiiL4n0Mq^xz~&3k9+41@uzyUB?*#P1x-(S(e=7ygi!`m9*e
z`}TY+Y^TfR!mOBgU#Yr5Wp&iNLl-T5Y{6-k2y}n;Xvp7o>_Wsn@rACPUNfO{?iD(B
z*}CNv!gN6xdacW=6Q-L+nC>#dBo(j;Q@#RWE?c*J1)^4BL|u%Qamts#2v_ZIN_V@T
zd)8-dqT0ebLFhq97o7fWiM|Z+6vFefi>WkmaS23kNo>Sfa`iq&jCPD`{LM~M@x+pW
zRkf2~Y3kFNQ38Vtw|a2`8}ninLd%9vDA`W9@$k;MhQ^bzZ>%GaNdsjAmP4R1xO*!?
z4G&Upr9oPif_>o8Ur*KdZ;E358=_c$Ew+v;r8NZ9GA386#Z@G)V&Ts4T1?K*I%Th2
z=tmhf%{E(IfX2*>n_%zGRoVMm45v#NAveR`y~UIZ;3$7zp1G4eIu@ES$4Xko^g1zy
zkJpTe*{^EKEobaY&zM;K2yK&rppYYlZhG#;D#y`_K}bOEn$%1(x5nL>YfjviX0Afq
z?|=XLCUC~&O`f>R$q=0`DU<c)=1YvNQ&~EVjT2;jlowO@FqQYNjlWjmm%y!gEld*i
zy<7N&JmX0flFP=JBwXW)OtNeRe<>!iFbp%1yhvc8D<ike-dNjl5#`}<m-bRRbS3$@
zdT*EWOq~DI_vE8n@)}+t{zz7@Nzct{Ao-m(rIKDorFBvnHA64A&3={r-;dsOeD(Y<
zr84(?@AAKVp3kcLzp|MoT*pz*`S&}t&j%xQKtli2sRww<uSxYV+u00mgJ@4M5m5Z2
z+(2OtXguHjdH;z%xZji@n!3x=47$9|iZ)}W4j*jZWZ*L>*g`3k^vwkA&I$?1-!7ur
z-{iEBy0*eJAh3Cr#rKbO4|mig4De%z;+29cDox2Bq9^HOGi>%->5qaVR0z(_y+nE5
zQZ)SFbaaNtzSI3M_ndnl7Z35DGnhHO@pQVFFuj;x@jtI5TaIp#M2R-0Uv0bB*QgIN
z+;DJ@#z2T9Z}bV%kJbUw2!ew@7|&e_&^qwmXJIt@ZtjpPw@(Jcp^esn=qM5pDsYhg
z`lxYeE`kx8eSF|b{tcBj4OR7JCSlH-#%hG9oWgvGha!`+NPbdDD*GV@G|^<zgQP7^
zQlLE}TkCXuN%j~i#5RJN4E&Ft?>>9RYz|MJ?H(O%yG!;Kq@iJFdg`s!P`DM+O<-_7
z^@gC`Dy^O^)@(G~1RL$0Ek<uBknTsb&#V!Y7tp#8eJU-+o(kfZRM9v3xw8JBoh@x+
ziT*E@-BL{dSIf0K{eKIeZ~pPHI~YBjoq_*PoZp<Si~sYdH|lhUp3J6CTabNQ7w(qx
zhqL3%&%Dv5!_$r9>D=Lbe(|LJ<i){HkGDEqJbEN6oWg6&F#VM*BDdrfre6Q!c;xBl
zZx@|uXEcZQe^lDqJQ;jx)8g)P_qqGreeOPYpS#c9=k9a&x%=FGzPiu<A3wE=a{%}t
E0N<^bumAu6

literal 0
HcmV?d00001

diff --git a/cpp/thirdparty/jemalloc/README.md b/cpp/thirdparty/jemalloc/README.md
new file mode 100644
index 0000000000000..272ff9c730be1
--- /dev/null
+++ b/cpp/thirdparty/jemalloc/README.md
@@ -0,0 +1,22 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+This directory contains a vendored commit from the jemalloc stable-4 branch.
+You can bump the version by downloading
+https://github.com/jemalloc/jemalloc/archive/{{ commit }}.tar.gz

From 3fb14914abfe89797f18a6cc5f62ec5cbe0ab2be Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Tue, 21 Nov 2017 18:24:49 -0500
Subject: [PATCH 1271/1644] ARROW-1268: [SITE][FOLLOWUP] Update Spark Post to
 Reflect Conf Change

The Spark conf to enable arrow has changed from "spark.sql.execution.arrow.enable" to "spark.sql.execution.arrow.enabled"

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1344 from BryanCutler/site-Spark-blog-conf-update-ARROW-1268 and squashes the following commits:

5e784351 [Bryan Cutler] update spark conf to enable arrow
---
 site/_posts/2017-07-26-spark-arrow.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/site/_posts/2017-07-26-spark-arrow.md b/site/_posts/2017-07-26-spark-arrow.md
index c4b16c0738c0c..211e5a481b482 100644
--- a/site/_posts/2017-07-26-spark-arrow.md
+++ b/site/_posts/2017-07-26-spark-arrow.md
@@ -57,7 +57,7 @@ the conversion to Arrow data can be done on the JVM and pushed back for the Spar
 executors to perform in parallel, drastically reducing the load on the driver.
 
 As of the merging of [SPARK-13534][5], the use of Arrow when calling `toPandas()`
-needs to be enabled by setting the SQLConf "spark.sql.execution.arrow.enable" to
+needs to be enabled by setting the SQLConf "spark.sql.execution.arrow.enabled" to
 "true".  Let's look at a simple usage example.
 
 ```
@@ -84,7 +84,7 @@ In [2]: %time pdf = df.toPandas()
 CPU times: user 17.4 s, sys: 792 ms, total: 18.1 s
 Wall time: 20.7 s
 
-In [3]: spark.conf.set("spark.sql.execution.arrow.enable", "true")
+In [3]: spark.conf.set("spark.sql.execution.arrow.enabled", "true")
 
 In [4]: %time pdf = df.toPandas()
 CPU times: user 40 ms, sys: 32 ms, total: 72 ms                                 
@@ -118,7 +118,7 @@ It is planned to add pyarrow as a pyspark dependency so that
 
 Currently, the controlling SQLConf is disabled by default. This can be enabled
 programmatically as in the example above or by adding the line
-"spark.sql.execution.arrow.enable=true" to `SPARK_HOME/conf/spark-defaults.conf`.
+"spark.sql.execution.arrow.enabled=true" to `SPARK_HOME/conf/spark-defaults.conf`.
 
 Also, not all Spark data types are currently supported and limited to primitive
 types. Expanded type support is in the works and expected to also be in the Spark

From fc4e2c36d2c56a8bd5d1ab17eeb406826924d3e5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 21 Nov 2017 19:01:48 -0500
Subject: [PATCH 1272/1644] ARROW-1808: [C++] Make RecordBatch, Table virtual
 interfaces for column access

While this will cause some minor API breakage in parquet-cpp and some other downstream users, this is reasonably long overdue. It will permit implementations of the RecordBatch or Table interface that do lazy IO / data loading or lazy materialization of columns.

I will write a patch to fix up parquet-cpp, and will look to see if glib is easy to fix. There's no good way to go about merging this patch since a green build is not possible, so once we're happy with the patch, I can merge this patch and then work on getting a green build in parquet-cpp so we don't have a broken build there for too long

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Kouhei Sutou <kou@clear-code.com>

Closes #1337 from wesm/ARROW-1808 and squashes the following commits:

55ce663d [Wes McKinney] clang-format
57923708 [Wes McKinney] Revert to debug build for now
06625301 [Wes McKinney] Fix test case that wasn't being run in debug builds
c9a7cb83 [Kouhei Sutou] Apply patch to fix Glib test suite
6b02e438 [Wes McKinney] Move DCHECK in SimpleRecordBatch ctor to Validate
06bced1a [Wes McKinney] Deprecate arrow::MakeTable
498bb082 [Wes McKinney] Fix glib compilation
a4e403c3 [Wes McKinney] Set build warning level via environment variable
9a61beb5 [Wes McKinney] Move boxed_columns_ to SimpleTable. Remove DecimalType backwards compat define
7691a5f1 [Wes McKinney] Make Table virtual also, refactor
bcc0cd15 [Wes McKinney] Some table refactoring to be virtual
267dd218 [Wes McKinney] Finish RecordBatch refactoring, get tests passing again. Add option to set build type in Travis CI
ef00e5b9 [Wes McKinney] Split off record batch implementation into separate files, progress refactoring
33f341df [Wes McKinney] Start making things virtual
---
 .travis.yml                                   |   2 +
 c_glib/arrow-glib/record-batch.cpp            |   5 +-
 c_glib/arrow-glib/table.cpp                   |   3 +-
 c_glib/test/test-file-writer.rb               |  16 +-
 c_glib/test/test-gio-input-stream.rb          |  18 +-
 c_glib/test/test-gio-output-stream.rb         |  18 +-
 c_glib/test/test-stream-writer.rb             |  18 +-
 ci/travis_before_script_cpp.sh                |   6 +-
 ci/travis_env_common.sh                       |   3 +
 ci/travis_script_python.sh                    |   3 +
 cpp/src/arrow/CMakeLists.txt                  |   2 +
 cpp/src/arrow/api.h                           |   1 +
 cpp/src/arrow/array.h                         |   2 +
 cpp/src/arrow/builder.cc                      |   1 -
 cpp/src/arrow/builder.h                       |   1 -
 cpp/src/arrow/column-benchmark.cc             |   1 +
 cpp/src/arrow/compute/kernel.h                |   1 +
 cpp/src/arrow/gpu/cuda_arrow_ipc.cc           |   2 +-
 cpp/src/arrow/ipc/feather-test.cc             |   5 +-
 cpp/src/arrow/ipc/feather.cc                  |   1 +
 cpp/src/arrow/ipc/ipc-json-test.cc            |   4 +-
 cpp/src/arrow/ipc/ipc-read-write-benchmark.cc |   2 +-
 cpp/src/arrow/ipc/ipc-read-write-test.cc      |  19 +-
 cpp/src/arrow/ipc/json-integration-test.cc    |   2 +-
 cpp/src/arrow/ipc/json-internal.cc            |   8 +-
 cpp/src/arrow/ipc/json.cc                     |   2 +-
 cpp/src/arrow/ipc/reader.cc                   |   4 +-
 cpp/src/arrow/ipc/reader.h                    |   3 +-
 cpp/src/arrow/ipc/test-common.h               |  44 ++-
 cpp/src/arrow/ipc/writer.cc                   |  10 +-
 cpp/src/arrow/pretty_print.cc                 |   2 +-
 cpp/src/arrow/python/python-test.cc           |   5 +-
 cpp/src/arrow/python/python_to_arrow.cc       |   8 +-
 cpp/src/arrow/record_batch.cc                 | 206 ++++++++++++
 cpp/src/arrow/record_batch.h                  | 154 +++++++++
 cpp/src/arrow/table-test.cc                   | 177 +++++-----
 cpp/src/arrow/table.cc                        | 315 ++++++------------
 cpp/src/arrow/table.h                         | 165 ++-------
 cpp/src/arrow/table_builder-test.cc           |   5 +-
 cpp/src/arrow/table_builder.cc                |   3 +-
 cpp/src/arrow/test-common.h                   |   1 -
 cpp/src/arrow/test-util.h                     |   3 +-
 cpp/src/arrow/type.h                          |  11 +-
 python/pyarrow/includes/libarrow.pxd          |  11 +-
 python/pyarrow/table.pxi                      |   8 +-
 45 files changed, 725 insertions(+), 556 deletions(-)
 create mode 100644 cpp/src/arrow/record_batch.cc
 create mode 100644 cpp/src/arrow/record_batch.h

diff --git a/.travis.yml b/.travis.yml
index 9c714a68948e5..ddadf739aab25 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -55,6 +55,7 @@ matrix:
     - export ARROW_TRAVIS_VALGRIND=1
     - export ARROW_TRAVIS_PLASMA=1
     - export ARROW_TRAVIS_CLANG_FORMAT=1
+    - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - export CC="clang-4.0"
     - export CXX="clang++-4.0"
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
@@ -74,6 +75,7 @@ matrix:
     before_script:
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_PLASMA=1
+    - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp
index f381af0a2c2c7..f23a0cf7582e2 100644
--- a/c_glib/arrow-glib/record-batch.cpp
+++ b/c_glib/arrow-glib/record-batch.cpp
@@ -150,9 +150,8 @@ garrow_record_batch_new(GArrowSchema *schema,
   }
 
   auto arrow_record_batch =
-    std::make_shared<arrow::RecordBatch>(garrow_schema_get_raw(schema),
-                                         n_rows,
-                                         arrow_columns);
+    arrow::RecordBatch::Make(garrow_schema_get_raw(schema),
+                             n_rows, arrow_columns);
   return garrow_record_batch_new_raw(&arrow_record_batch);
 }
 
diff --git a/c_glib/arrow-glib/table.cpp b/c_glib/arrow-glib/table.cpp
index 779f2ef62b8f5..e086396f8f9e0 100644
--- a/c_glib/arrow-glib/table.cpp
+++ b/c_glib/arrow-glib/table.cpp
@@ -143,8 +143,7 @@ garrow_table_new(GArrowSchema *schema,
   }
 
   auto arrow_table =
-    std::make_shared<arrow::Table>(garrow_schema_get_raw(schema),
-                                   arrow_columns);
+    arrow::Table::Make(garrow_schema_get_raw(schema), arrow_columns);
   return garrow_table_new_raw(&arrow_table);
 }
 
diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb
index 3de8e5cf34b51..67aed85f73b48 100644
--- a/c_glib/test/test-file-writer.rb
+++ b/c_glib/test/test-file-writer.rb
@@ -19,14 +19,18 @@ class TestFileWriter < Test::Unit::TestCase
   include Helper::Buildable
 
   def test_write_record_batch
+    data = [true]
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
     tempfile = Tempfile.open("arrow-ipc-file-writer")
     output = Arrow::FileOutputStream.new(tempfile.path, false)
     begin
-      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
-      schema = Arrow::Schema.new([field])
       file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
       begin
-        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        record_batch = Arrow::RecordBatch.new(schema,
+                                              data.size,
+                                              [build_boolean_array(data)])
         file_writer.write_record_batch(record_batch)
       ensure
         file_writer.close
@@ -38,8 +42,12 @@ def test_write_record_batch
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
       file_reader = Arrow::RecordBatchFileReader.new(input)
-      assert_equal(["enabled"],
+      assert_equal([field.name],
                    file_reader.schema.fields.collect(&:name))
+      assert_equal(Arrow::RecordBatch.new(schema,
+                                          data.size,
+                                          [build_boolean_array(data)]),
+                   file_reader.read_record_batch(0))
     ensure
       input.close
     end
diff --git a/c_glib/test/test-gio-input-stream.rb b/c_glib/test/test-gio-input-stream.rb
index a71a370430e6d..2adf25b3af51f 100644
--- a/c_glib/test/test-gio-input-stream.rb
+++ b/c_glib/test/test-gio-input-stream.rb
@@ -16,15 +16,21 @@
 # under the License.
 
 class TestGIOInputStream < Test::Unit::TestCase
+  include Helper::Buildable
+
   def test_reader_backend
+    data = [true]
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
     tempfile = Tempfile.open("arrow-gio-input-stream")
     output = Arrow::FileOutputStream.new(tempfile.path, false)
     begin
-      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
-      schema = Arrow::Schema.new([field])
       file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
       begin
-        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        record_batch = Arrow::RecordBatch.new(schema,
+                                              data.size,
+                                              [build_boolean_array(data)])
         file_writer.write_record_batch(record_batch)
       ensure
         file_writer.close
@@ -38,8 +44,12 @@ def test_reader_backend
     input = Arrow::GIOInputStream.new(input_stream)
     begin
       file_reader = Arrow::RecordBatchFileReader.new(input)
-      assert_equal(["enabled"],
+      assert_equal([field.name],
                    file_reader.schema.fields.collect(&:name))
+      assert_equal(Arrow::RecordBatch.new(schema,
+                                          data.size,
+                                          [build_boolean_array(data)]),
+                   file_reader.read_record_batch(0))
     ensure
       input.close
     end
diff --git a/c_glib/test/test-gio-output-stream.rb b/c_glib/test/test-gio-output-stream.rb
index adaa8c1b7b2b0..c77598ed110d5 100644
--- a/c_glib/test/test-gio-output-stream.rb
+++ b/c_glib/test/test-gio-output-stream.rb
@@ -16,17 +16,23 @@
 # under the License.
 
 class TestGIOOutputStream < Test::Unit::TestCase
+  include Helper::Buildable
+
   def test_writer_backend
+    data = [true]
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
     tempfile = Tempfile.open("arrow-gio-output-stream")
     file = Gio::File.new_for_path(tempfile.path)
     output_stream = file.append_to(:none)
     output = Arrow::GIOOutputStream.new(output_stream)
     begin
-      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
-      schema = Arrow::Schema.new([field])
       file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
       begin
-        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        record_batch = Arrow::RecordBatch.new(schema,
+                                              data.size,
+                                              [build_boolean_array(data)])
         file_writer.write_record_batch(record_batch)
       ensure
         file_writer.close
@@ -38,8 +44,12 @@ def test_writer_backend
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
       file_reader = Arrow::RecordBatchFileReader.new(input)
-      assert_equal(["enabled"],
+      assert_equal([field.name],
                    file_reader.schema.fields.collect(&:name))
+      assert_equal(Arrow::RecordBatch.new(schema,
+                                          data.size,
+                                          [build_boolean_array(data)]),
+                   file_reader.read_record_batch(0))
     ensure
       input.close
     end
diff --git a/c_glib/test/test-stream-writer.rb b/c_glib/test/test-stream-writer.rb
index c3d0e1490cef6..32754e20838b4 100644
--- a/c_glib/test/test-stream-writer.rb
+++ b/c_glib/test/test-stream-writer.rb
@@ -19,17 +19,19 @@ class TestStreamWriter < Test::Unit::TestCase
   include Helper::Buildable
 
   def test_write_record_batch
+    data = [true]
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
     tempfile = Tempfile.open("arrow-ipc-stream-writer")
     output = Arrow::FileOutputStream.new(tempfile.path, false)
     begin
-      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
-      schema = Arrow::Schema.new([field])
       stream_writer = Arrow::RecordBatchStreamWriter.new(output, schema)
       begin
         columns = [
-          build_boolean_array([true]),
+          build_boolean_array(data),
         ]
-        record_batch = Arrow::RecordBatch.new(schema, 1, columns)
+        record_batch = Arrow::RecordBatch.new(schema, data.size, columns)
         stream_writer.write_record_batch(record_batch)
       ensure
         stream_writer.close
@@ -41,10 +43,12 @@ def test_write_record_batch
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
       stream_reader = Arrow::RecordBatchStreamReader.new(input)
-      assert_equal(["enabled"],
+      assert_equal([field.name],
                    stream_reader.schema.fields.collect(&:name))
-      assert_equal(true,
-                   stream_reader.read_next.get_column(0).get_value(0))
+      assert_equal(Arrow::RecordBatch.new(schema,
+                                          data.size,
+                                          [build_boolean_array(data)]),
+                   stream_reader.read_next)
       assert_nil(stream_reader.read_next)
     ensure
       input.close
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 4998f190f9891..664f7ce5fed2d 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -91,12 +91,14 @@ fi
 if [ $TRAVIS_OS_NAME == "linux" ]; then
     cmake $CMAKE_COMMON_FLAGS \
           $CMAKE_LINUX_FLAGS \
-          -DBUILD_WARNING_LEVEL=CHECKIN \
+          -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+          -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \
           $ARROW_CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
           $CMAKE_OSX_FLAGS \
-          -DBUILD_WARNING_LEVEL=CHECKIN \
+          -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+          -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \
           $ARROW_CPP_DIR
 fi
 
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index 52c7da4e0178e..21b6e266ea678 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -38,6 +38,9 @@ export ARROW_PYTHON_PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 
 export CMAKE_EXPORT_COMPILE_COMMANDS=1
 
+export ARROW_BUILD_TYPE=${ARROW_BUILD_TYPE:=debug}
+export ARROW_BUILD_WARNING_LEVEL=${ARROW_BUILD_WARNING_LEVEL:=Production}
+
 if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
   # C++ toolchain
   export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 603201bcc166b..5f7b0a9a1afed 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -63,6 +63,7 @@ cmake -GNinja \
       -DARROW_BUILD_UTILITIES=off \
       -DARROW_PLASMA=on \
       -DARROW_PYTHON=on \
+      -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
       -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
       $ARROW_CPP_DIR
 
@@ -78,6 +79,8 @@ if [ "$PYTHON_VERSION" == "2.7" ]; then
   pip install futures
 fi
 
+export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE
+
 pip install -r requirements.txt
 python setup.py build_ext --with-parquet --with-plasma \
        install --single-version-externally-managed --record=record.text
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 496e0da9d621e..94705781fa42d 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -22,6 +22,7 @@ set(ARROW_SRCS
   compare.cc
   memory_pool.cc
   pretty_print.cc
+  record_batch.cc
   status.cc
   table.cc
   table_builder.cc
@@ -144,6 +145,7 @@ install(FILES
   compare.h
   memory_pool.h
   pretty_print.h
+  record_batch.h
   status.h
   table.h
   table_builder.h
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 5d2e859f3a4be..7cae8414a774b 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -26,6 +26,7 @@
 #include "arrow/compare.h"
 #include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/table_builder.h"
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 28756a6abdafe..dda9dd38be4a7 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -279,6 +279,8 @@ class ARROW_EXPORT Array {
   ARROW_DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
+using ArrayVector = std::vector<std::shared_ptr<Array>>;
+
 static inline std::ostream& operator<<(std::ostream& os, const Array& x) {
   os << x.ToString();
   return os;
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 3e213fcd5ca70..a42f9024545ec 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -28,7 +28,6 @@
 #include "arrow/buffer.h"
 #include "arrow/compare.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 32741b53ac438..e59e166580ae4 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -29,7 +29,6 @@
 #include "arrow/buffer.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index e50ddf6d7032d..af2c368c32975 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -19,6 +19,7 @@
 
 #include "arrow/array.h"
 #include "arrow/memory_pool.h"
+#include "arrow/table.h"
 #include "arrow/test-util.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h
index 7ff506ca0cf2b..0bfa55cfee198 100644
--- a/cpp/src/arrow/compute/kernel.h
+++ b/cpp/src/arrow/compute/kernel.h
@@ -22,6 +22,7 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/record_batch.h"
 #include "arrow/table.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/variant.h"
diff --git a/cpp/src/arrow/gpu/cuda_arrow_ipc.cc b/cpp/src/arrow/gpu/cuda_arrow_ipc.cc
index 022268e034758..a7262c8b4d4ba 100644
--- a/cpp/src/arrow/gpu/cuda_arrow_ipc.cc
+++ b/cpp/src/arrow/gpu/cuda_arrow_ipc.cc
@@ -27,8 +27,8 @@
 #include "arrow/ipc/message.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/util/visibility.h"
 
 #include "arrow/gpu/cuda_context.h"
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index 6bd16462df94d..e3de17f1f75cd 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -29,6 +29,7 @@
 #include "arrow/ipc/feather.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/pretty_print.h"
+#include "arrow/table.h"
 #include "arrow/test-util.h"
 
 namespace arrow {
@@ -376,8 +377,8 @@ TEST_F(TestTableWriter, TimeTypes) {
         schema->field(i)->type(), values->length(), buffers, values->null_count(), 0));
   }
 
-  RecordBatch batch(schema, values->length(), std::move(arrays));
-  CheckBatch(batch);
+  auto batch = RecordBatch::Make(schema, values->length(), std::move(arrays));
+  CheckBatch(*batch);
 }
 
 TEST_F(TestTableWriter, VLenPrimitiveRoundTrip) {
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index cea720bd01bc7..077dc39305b09 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -32,6 +32,7 @@
 #include "arrow/ipc/feather-internal.h"
 #include "arrow/ipc/feather_generated.h"
 #include "arrow/ipc/util.h"  // IWYU pragma: keep
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index a560f09d6fdb1..e496826f96b9b 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -31,8 +31,8 @@
 #include "arrow/ipc/json.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/memory_pool.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
@@ -269,7 +269,7 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
     std::vector<std::shared_ptr<Array>> arrays;
 
     MakeBatchArrays(schema, num_rows, &arrays);
-    auto batch = std::make_shared<RecordBatch>(schema, num_rows, arrays);
+    auto batch = RecordBatch::Make(schema, num_rows, arrays);
     batches.push_back(batch);
     ASSERT_OK(writer->WriteRecordBatch(*batch));
   }
diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
index 9ed0abde651f6..8561fb8603707 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc
@@ -63,7 +63,7 @@ std::shared_ptr<RecordBatch> MakeRecordBatch(int64_t total_size, int64_t num_fie
   }
 
   auto schema = std::make_shared<Schema>(fields);
-  return std::make_shared<RecordBatch>(schema, length, arrays);
+  return RecordBatch::Make(schema, length, arrays);
 }
 
 static void BM_WriteRecordBatch(benchmark::State& state) {  // NOLINT non-const reference
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 40cd3f0eef0b8..1fcbdac5ebc73 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -197,8 +197,8 @@ class IpcTestFixture : public io::MemoryMapFixture {
     std::vector<std::shared_ptr<Field>> fields = {f0};
     auto schema = std::make_shared<Schema>(fields);
 
-    RecordBatch batch(schema, 0, {array});
-    CheckRoundtrip(batch, buffer_size);
+    auto batch = RecordBatch::Make(schema, 0, {array});
+    CheckRoundtrip(*batch, buffer_size);
   }
 
  protected:
@@ -292,13 +292,13 @@ TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) {
   auto CheckArray = [this](const std::shared_ptr<Array>& array) {
     auto f0 = field("f0", array->type());
     auto schema = ::arrow::schema({f0});
-    RecordBatch batch(schema, array->length(), {array});
-    auto sliced_batch = batch.Slice(0, 5);
+    auto batch = RecordBatch::Make(schema, array->length(), {array});
+    auto sliced_batch = batch->Slice(0, 5);
 
     int64_t full_size;
     int64_t sliced_size;
 
-    ASSERT_OK(GetRecordBatchSize(batch, &full_size));
+    ASSERT_OK(GetRecordBatchSize(*batch, &full_size));
     ASSERT_OK(GetRecordBatchSize(*sliced_batch, &sliced_size));
     ASSERT_TRUE(sliced_size < full_size) << sliced_size << " " << full_size;
 
@@ -411,8 +411,7 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
 
     *schema = ::arrow::schema({f0});
 
-    std::vector<std::shared_ptr<Array>> arrays = {array};
-    *batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
+    *batch = RecordBatch::Make(*schema, batch_length, {array});
 
     std::stringstream ss;
     ss << "test-write-past-max-recursion-" << g_file_number++;
@@ -632,7 +631,7 @@ TEST_F(TestIpcRoundTrip, LargeRecordBatch) {
   std::vector<std::shared_ptr<Field>> fields = {f0};
   auto schema = std::make_shared<Schema>(fields);
 
-  RecordBatch batch(schema, length, {array});
+  auto batch = RecordBatch::Make(schema, length, {array});
 
   std::string path = "test-write-large-record_batch";
 
@@ -641,8 +640,8 @@ TEST_F(TestIpcRoundTrip, LargeRecordBatch) {
   ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_));
 
   std::shared_ptr<RecordBatch> result;
-  ASSERT_OK(DoLargeRoundTrip(batch, false, &result));
-  CheckReadResult(*result, batch);
+  ASSERT_OK(DoLargeRoundTrip(*batch, false, &result));
+  CheckReadResult(*result, *batch);
 
   ASSERT_EQ(length, result->num_rows());
 }
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index c7530a467b398..f487487dfda9f 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -34,8 +34,8 @@
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
 #include "arrow/pretty_print.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index bdf1ef52b40eb..bfb3d282d8772 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -28,8 +28,8 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/ipc/dictionary.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -125,8 +125,8 @@ class SchemaWriter {
 
     // Make a dummy record batch. A bit tedious as we have to make a schema
     auto schema = ::arrow::schema({arrow::field("dictionary", dictionary->type())});
-    RecordBatch batch(schema, dictionary->length(), {dictionary});
-    RETURN_NOT_OK(WriteRecordBatch(batch, writer_));
+    auto batch = RecordBatch::Make(schema, dictionary->length(), {dictionary});
+    RETURN_NOT_OK(WriteRecordBatch(*batch, writer_));
     writer_->EndObject();
     return Status::OK();
   }
@@ -1435,7 +1435,7 @@ Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr<Schema>&
     RETURN_NOT_OK(ReadArray(pool, json_columns[i], type, &columns[i]));
   }
 
-  *batch = std::make_shared<RecordBatch>(schema, num_rows, columns);
+  *batch = RecordBatch::Make(schema, num_rows, columns);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index 30a1bb81e1a1e..ea2947d5d4c36 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -24,8 +24,8 @@
 #include "arrow/buffer.h"
 #include "arrow/ipc/json-internal.h"
 #include "arrow/memory_pool.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/util/logging.h"
 
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 8e10d7d66f907..5960e81883d6b 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -37,8 +37,8 @@
 #include "arrow/ipc/message.h"
 #include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/util.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/tensor.h"
 #include "arrow/type.h"
 #include "arrow/util/bit-util.h"
@@ -307,7 +307,7 @@ static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
     arrays[i] = std::move(arr);
   }
 
-  *out = std::make_shared<RecordBatch>(schema, num_rows, std::move(arrays));
+  *out = RecordBatch::Make(schema, num_rows, std::move(arrays));
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 7581fbda5b140..627f67e2517e9 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -24,13 +24,12 @@
 #include <memory>
 
 #include "arrow/ipc/message.h"
-#include "arrow/table.h"
+#include "arrow/record_batch.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class Buffer;
-class RecordBatch;
 class Schema;
 class Status;
 class Tensor;
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 7fc139381052c..6f8a0dcc61fbc 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -30,8 +30,8 @@
 #include "arrow/builder.h"
 #include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/util/bit-util.h"
@@ -184,7 +184,7 @@ Status MakeBooleanBatchSized(const int length, std::shared_ptr<RecordBatch>* out
   std::shared_ptr<Array> a0, a1;
   RETURN_NOT_OK(MakeRandomBooleanArray(length, true, &a0));
   RETURN_NOT_OK(MakeRandomBooleanArray(length, false, &a1));
-  out->reset(new RecordBatch(schema, length, {a0, a1}));
+  *out = RecordBatch::Make(schema, length, {a0, a1});
   return Status::OK();
 }
 
@@ -203,7 +203,7 @@ Status MakeIntBatchSized(int length, std::shared_ptr<RecordBatch>* out) {
   MemoryPool* pool = default_memory_pool();
   RETURN_NOT_OK(MakeRandomInt32Array(length, false, pool, &a0));
   RETURN_NOT_OK(MakeRandomInt32Array(length, true, pool, &a1));
-  out->reset(new RecordBatch(schema, length, {a0, a1}));
+  *out = RecordBatch::Make(schema, length, {a0, a1});
   return Status::OK();
 }
 
@@ -252,7 +252,7 @@ Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out) {
     auto s = MakeRandomBinaryArray<BinaryBuilder, uint8_t>(length, true, pool, &a1);
     RETURN_NOT_OK(s);
   }
-  out->reset(new RecordBatch(schema, length, {a0, a1}));
+  *out = RecordBatch::Make(schema, length, {a0, a1});
   return Status::OK();
 }
 
@@ -261,7 +261,7 @@ Status MakeNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
   auto f0 = field("f0", null());
   auto schema = ::arrow::schema({f0});
   std::shared_ptr<Array> a0 = std::make_shared<NullArray>(length);
-  out->reset(new RecordBatch(schema, length, {a0}));
+  *out = RecordBatch::Make(schema, length, {a0});
   return Status::OK();
 }
 
@@ -284,7 +284,7 @@ Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out) {
   RETURN_NOT_OK(
       MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array));
   RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array));
-  out->reset(new RecordBatch(schema, length, {list_array, list_list_array, flat_array}));
+  *out = RecordBatch::Make(schema, length, {list_array, list_list_array, flat_array});
   return Status::OK();
 }
 
@@ -304,7 +304,7 @@ Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out) {
   RETURN_NOT_OK(
       MakeRandomListArray(list_array, 0, include_nulls, pool, &list_list_array));
   RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &flat_array));
-  out->reset(new RecordBatch(schema, 0, {list_array, list_list_array, flat_array}));
+  *out = RecordBatch::Make(schema, 0, {list_array, list_list_array, flat_array});
   return Status::OK();
 }
 
@@ -327,7 +327,7 @@ Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
   RETURN_NOT_OK(
       MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array));
   RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array));
-  out->reset(new RecordBatch(schema, length, {list_array, list_list_array, flat_array}));
+  *out = RecordBatch::Make(schema, length, {list_array, list_list_array, flat_array});
   return Status::OK();
 }
 
@@ -347,7 +347,7 @@ Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
   auto f0 = field("f0", type);
   auto schema = ::arrow::schema({f0});
   std::vector<std::shared_ptr<Array>> arrays = {array};
-  out->reset(new RecordBatch(schema, batch_length, arrays));
+  *out = RecordBatch::Make(schema, batch_length, arrays);
   return Status::OK();
 }
 
@@ -377,7 +377,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
 
   // construct batch
   std::vector<std::shared_ptr<Array>> arrays = {no_nulls, with_nulls};
-  out->reset(new RecordBatch(schema, list_batch->num_rows(), arrays));
+  *out = RecordBatch::Make(schema, list_batch->num_rows(), arrays);
   return Status::OK();
 }
 
@@ -445,7 +445,7 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
 
   // construct batch
   std::vector<std::shared_ptr<Array>> arrays = {sparse_no_nulls, sparse, dense};
-  out->reset(new RecordBatch(schema, length, arrays));
+  *out = RecordBatch::Make(schema, length, arrays);
   return Status::OK();
 }
 
@@ -526,7 +526,7 @@ Status MakeDictionary(std::shared_ptr<RecordBatch>* out) {
 
   std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2, a3, a4};
 
-  out->reset(new RecordBatch(schema, length, arrays));
+  *out = RecordBatch::Make(schema, length, arrays);
   return Status::OK();
 }
 
@@ -564,7 +564,7 @@ Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out) {
       {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type)});
 
   std::vector<std::shared_ptr<Array>> arrays = {a0, a1, a2};
-  out->reset(new RecordBatch(schema, length, arrays));
+  *out = RecordBatch::Make(schema, length, arrays);
   return Status::OK();
 }
 
@@ -584,8 +584,7 @@ Status MakeDates(std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Array> date64_array;
   ArrayFromVector<Date64Type, int64_t>(is_valid, date64_values, &date64_array);
 
-  std::vector<std::shared_ptr<Array>> arrays = {date32_array, date64_array};
-  *out = std::make_shared<RecordBatch>(schema, date32_array->length(), arrays);
+  *out = RecordBatch::Make(schema, date32_array->length(), {date32_array, date64_array});
   return Status::OK();
 }
 
@@ -604,8 +603,7 @@ Status MakeTimestamps(std::shared_ptr<RecordBatch>* out) {
   ArrayFromVector<TimestampType, int64_t>(f1->type(), is_valid, ts_values, &a1);
   ArrayFromVector<TimestampType, int64_t>(f2->type(), is_valid, ts_values, &a2);
 
-  ArrayVector arrays = {a0, a1, a2};
-  *out = std::make_shared<RecordBatch>(schema, a0->length(), arrays);
+  *out = RecordBatch::Make(schema, a0->length(), {a0, a1, a2});
   return Status::OK();
 }
 
@@ -628,8 +626,7 @@ Status MakeTimes(std::shared_ptr<RecordBatch>* out) {
   ArrayFromVector<Time32Type, int32_t>(f2->type(), is_valid, t32_values, &a2);
   ArrayFromVector<Time64Type, int64_t>(f3->type(), is_valid, t64_values, &a3);
 
-  ArrayVector arrays = {a0, a1, a2, a3};
-  *out = std::make_shared<RecordBatch>(schema, a0->length(), arrays);
+  *out = RecordBatch::Make(schema, a0->length(), {a0, a1, a2, a3});
   return Status::OK();
 }
 
@@ -665,8 +662,7 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
   RETURN_NOT_OK(b1.Finish(&a1));
   RETURN_NOT_OK(b2.Finish(&a2));
 
-  ArrayVector arrays = {a1, a2};
-  *out = std::make_shared<RecordBatch>(schema, a1->length(), arrays);
+  *out = RecordBatch::Make(schema, a1->length(), {a1, a2});
   return Status::OK();
 }
 
@@ -695,8 +691,7 @@ Status MakeDecimal(std::shared_ptr<RecordBatch>* out) {
 
   auto a2 = std::make_shared<Decimal128Array>(f1->type(), length, data);
 
-  ArrayVector arrays = {a1, a2};
-  *out = std::make_shared<RecordBatch>(schema, length, arrays);
+  *out = RecordBatch::Make(schema, length, {a1, a2});
   return Status::OK();
 }
 
@@ -716,8 +711,7 @@ Status MakeNull(std::shared_ptr<RecordBatch>* out) {
   std::shared_ptr<Array> a2;
   ArrayFromVector<Int64Type, int64_t>(f1->type(), is_valid, int_values, &a2);
 
-  ArrayVector arrays = {a1, a2};
-  *out = std::make_shared<RecordBatch>(schema, a1->length(), arrays);
+  *out = RecordBatch::Make(schema, a1->length(), {a1, a2});
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 323116f589b73..3c1db06159e65 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -32,6 +32,7 @@
 #include "arrow/ipc/metadata-internal.h"
 #include "arrow/ipc/util.h"
 #include "arrow/memory_pool.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/tensor.h"
@@ -508,12 +509,9 @@ class DictionaryWriter : public RecordBatchSerializer {
     dictionary_id_ = dictionary_id;
 
     // Make a dummy record batch. A bit tedious as we have to make a schema
-    std::vector<std::shared_ptr<Field>> fields = {
-        arrow::field("dictionary", dictionary->type())};
-    auto schema = std::make_shared<Schema>(fields);
-    RecordBatch batch(schema, dictionary->length(), {dictionary});
-
-    return RecordBatchSerializer::Write(batch, dst, metadata_length, body_length);
+    auto schema = arrow::schema({arrow::field("dictionary", dictionary->type())});
+    auto batch = RecordBatch::Make(schema, dictionary->length(), {dictionary});
+    return RecordBatchSerializer::Write(*batch, dst, metadata_length, body_length);
   }
 
  private:
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index cfbc30315fcd7..bd5f8ce10ea68 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -22,8 +22,8 @@
 
 #include "arrow/array.h"
 #include "arrow/pretty_print.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/logging.h"
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 86391a18598fd..3b7d7d884ef09 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -23,6 +23,7 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/table.h"
 #include "arrow/test-util.h"
 
 #include "arrow/python/arrow_to_pandas.h"
@@ -81,8 +82,8 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) {
   std::vector<std::shared_ptr<Field>> fields = {f1, f2, f3};
   std::vector<std::shared_ptr<Array>> cols = {arr, arr, arr};
 
-  auto schema = std::make_shared<Schema>(fields);
-  auto table = std::make_shared<Table>(schema, cols);
+  auto schema = ::arrow::schema(fields);
+  auto table = Table::Make(schema, cols);
 
   PyObject* out;
   Py_BEGIN_ALLOW_THREADS;
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index b0c6287f088a1..72cc5b6e1db67 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -32,13 +32,15 @@
 #include "arrow/builder.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/ipc/writer.h"
+#include "arrow/record_batch.h"
+#include "arrow/tensor.h"
+#include "arrow/util/logging.h"
+
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/platform.h"
 #include "arrow/python/util/datetime.h"
-#include "arrow/tensor.h"
-#include "arrow/util/logging.h"
 
 constexpr int32_t kMaxRecursionDepth = 100;
 
@@ -694,7 +696,7 @@ Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
 std::shared_ptr<RecordBatch> MakeBatch(std::shared_ptr<Array> data) {
   auto field = std::make_shared<Field>("list", data->type());
   auto schema = ::arrow::schema({field});
-  return std::shared_ptr<RecordBatch>(new RecordBatch(schema, data->length(), {data}));
+  return RecordBatch::Make(schema, data->length(), {data});
 }
 
 Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out) {
diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc
new file mode 100644
index 0000000000000..60932bdf3e4bb
--- /dev/null
+++ b/cpp/src/arrow/record_batch.cc
@@ -0,0 +1,206 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/record_batch.h"
+
+#include <algorithm>
+#include <cstdlib>
+#include <memory>
+#include <sstream>
+
+#include "arrow/array.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+/// \class SimpleRecordBatch
+/// \brief A basic, non-lazy in-memory record batch
+class SimpleRecordBatch : public RecordBatch {
+ public:
+  SimpleRecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+                    const std::vector<std::shared_ptr<Array>>& columns)
+      : RecordBatch(schema, num_rows) {
+    columns_.resize(columns.size());
+    boxed_columns_.resize(schema->num_fields());
+    for (size_t i = 0; i < columns.size(); ++i) {
+      columns_[i] = columns[i]->data();
+    }
+  }
+
+  SimpleRecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+                    std::vector<std::shared_ptr<Array>>&& columns)
+      : RecordBatch(schema, num_rows) {
+    columns_.resize(columns.size());
+    boxed_columns_.resize(schema->num_fields());
+    for (size_t i = 0; i < columns.size(); ++i) {
+      columns_[i] = columns[i]->data();
+    }
+  }
+
+  SimpleRecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+                    std::vector<std::shared_ptr<ArrayData>>&& columns)
+      : RecordBatch(schema, num_rows) {
+    columns_ = std::move(columns);
+    boxed_columns_.resize(schema->num_fields());
+  }
+
+  SimpleRecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
+                    const std::vector<std::shared_ptr<ArrayData>>& columns)
+      : RecordBatch(schema, num_rows) {
+    columns_ = columns;
+    boxed_columns_.resize(schema->num_fields());
+  }
+
+  std::shared_ptr<Array> column(int i) const override {
+    if (!boxed_columns_[i]) {
+      boxed_columns_[i] = MakeArray(columns_[i]);
+    }
+    DCHECK(boxed_columns_[i]);
+    return boxed_columns_[i];
+  }
+
+  std::shared_ptr<ArrayData> column_data(int i) const override { return columns_[i]; }
+
+  std::shared_ptr<RecordBatch> ReplaceSchemaMetadata(
+      const std::shared_ptr<const KeyValueMetadata>& metadata) const override {
+    auto new_schema = schema_->AddMetadata(metadata);
+    return RecordBatch::Make(new_schema, num_rows_, columns_);
+  }
+
+  std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length) const override {
+    std::vector<std::shared_ptr<ArrayData>> arrays;
+    arrays.reserve(num_columns());
+    for (const auto& field : columns_) {
+      int64_t col_length = std::min(field->length - offset, length);
+      int64_t col_offset = field->offset + offset;
+
+      auto new_data = std::make_shared<ArrayData>(*field);
+      new_data->length = col_length;
+      new_data->offset = col_offset;
+      new_data->null_count = kUnknownNullCount;
+      arrays.emplace_back(new_data);
+    }
+    int64_t num_rows = std::min(num_rows_ - offset, length);
+    return std::make_shared<SimpleRecordBatch>(schema_, num_rows, std::move(arrays));
+  }
+
+  Status Validate() const override {
+    if (static_cast<int>(columns_.size()) != schema_->num_fields()) {
+      return Status::Invalid("Number of columns did not match schema");
+    }
+    return RecordBatch::Validate();
+  }
+
+ private:
+  std::vector<std::shared_ptr<ArrayData>> columns_;
+
+  // Caching boxed array data
+  mutable std::vector<std::shared_ptr<Array>> boxed_columns_;
+};
+
+RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows)
+    : schema_(schema), num_rows_(num_rows) {}
+
+std::shared_ptr<RecordBatch> RecordBatch::Make(
+    const std::shared_ptr<Schema>& schema, int64_t num_rows,
+    const std::vector<std::shared_ptr<Array>>& columns) {
+  return std::make_shared<SimpleRecordBatch>(schema, num_rows, columns);
+}
+
+std::shared_ptr<RecordBatch> RecordBatch::Make(
+    const std::shared_ptr<Schema>& schema, int64_t num_rows,
+    std::vector<std::shared_ptr<Array>>&& columns) {
+  return std::make_shared<SimpleRecordBatch>(schema, num_rows, std::move(columns));
+}
+
+std::shared_ptr<RecordBatch> RecordBatch::Make(
+    const std::shared_ptr<Schema>& schema, int64_t num_rows,
+    std::vector<std::shared_ptr<ArrayData>>&& columns) {
+  return std::make_shared<SimpleRecordBatch>(schema, num_rows, std::move(columns));
+}
+
+std::shared_ptr<RecordBatch> RecordBatch::Make(
+    const std::shared_ptr<Schema>& schema, int64_t num_rows,
+    const std::vector<std::shared_ptr<ArrayData>>& columns) {
+  return std::make_shared<SimpleRecordBatch>(schema, num_rows, columns);
+}
+
+const std::string& RecordBatch::column_name(int i) const {
+  return schema_->field(i)->name();
+}
+
+bool RecordBatch::Equals(const RecordBatch& other) const {
+  if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
+    return false;
+  }
+
+  for (int i = 0; i < num_columns(); ++i) {
+    if (!column(i)->Equals(other.column(i))) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool RecordBatch::ApproxEquals(const RecordBatch& other) const {
+  if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
+    return false;
+  }
+
+  for (int i = 0; i < num_columns(); ++i) {
+    if (!column(i)->ApproxEquals(other.column(i))) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset) const {
+  return Slice(offset, this->num_rows() - offset);
+}
+
+Status RecordBatch::Validate() const {
+  for (int i = 0; i < num_columns(); ++i) {
+    auto arr_shared = this->column_data(i);
+    const ArrayData& arr = *arr_shared;
+    if (arr.length != num_rows_) {
+      std::stringstream ss;
+      ss << "Number of rows in column " << i << " did not match batch: " << arr.length
+         << " vs " << num_rows_;
+      return Status::Invalid(ss.str());
+    }
+    const auto& schema_type = *schema_->field(i)->type();
+    if (!arr.type->Equals(schema_type)) {
+      std::stringstream ss;
+      ss << "Column " << i << " type not match schema: " << arr.type->ToString() << " vs "
+         << schema_type.ToString();
+      return Status::Invalid(ss.str());
+    }
+  }
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Base record batch reader
+
+RecordBatchReader::~RecordBatchReader() {}
+
+}  // namespace arrow
diff --git a/cpp/src/arrow/record_batch.h b/cpp/src/arrow/record_batch.h
new file mode 100644
index 0000000000000..b2c4c76b3f2d3
--- /dev/null
+++ b/cpp/src/arrow/record_batch.h
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_RECORD_BATCH_H
+#define ARROW_RECORD_BATCH_H
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/array.h"
+#include "arrow/type.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class KeyValueMetadata;
+class Status;
+
+/// \class RecordBatch
+/// \brief Collection of equal-length arrays matching a particular Schema
+///
+/// A record batch is table-like data structure that is semantically a sequence
+/// of fields, each a contiguous Arrow array
+class ARROW_EXPORT RecordBatch {
+ public:
+  virtual ~RecordBatch() = default;
+
+  /// \param[in] schema The record batch schema
+  /// \param[in] num_rows length of fields in the record batch. Each array
+  /// should have the same length as num_rows
+  /// \param[in] columns the record batch fields as vector of arrays
+  static std::shared_ptr<RecordBatch> Make(
+      const std::shared_ptr<Schema>& schema, int64_t num_rows,
+      const std::vector<std::shared_ptr<Array>>& columns);
+
+  /// \brief Move-based constructor for a vector of Array instances
+  static std::shared_ptr<RecordBatch> Make(const std::shared_ptr<Schema>& schema,
+                                           int64_t num_rows,
+                                           std::vector<std::shared_ptr<Array>>&& columns);
+
+  /// \brief Construct record batch from vector of internal data structures
+  /// \since 0.5.0
+  ///
+  /// This class is only provided with an rvalue-reference for the input data,
+  /// and is intended for internal use, or advanced users.
+  ///
+  /// \param schema the record batch schema
+  /// \param num_rows the number of semantic rows in the record batch. This
+  /// should be equal to the length of each field
+  /// \param columns the data for the batch's columns
+  static std::shared_ptr<RecordBatch> Make(
+      const std::shared_ptr<Schema>& schema, int64_t num_rows,
+      std::vector<std::shared_ptr<ArrayData>>&& columns);
+
+  /// \brief Construct record batch by copying vector of array data
+  /// \since 0.5.0
+  static std::shared_ptr<RecordBatch> Make(
+      const std::shared_ptr<Schema>& schema, int64_t num_rows,
+      const std::vector<std::shared_ptr<ArrayData>>& columns);
+
+  /// \brief Determine if two record batches are exactly equal
+  /// \return true if batches are equal
+  bool Equals(const RecordBatch& other) const;
+
+  /// \brief Determine if two record batches are approximately equal
+  bool ApproxEquals(const RecordBatch& other) const;
+
+  // \return the table's schema
+  /// \return true if batches are equal
+  std::shared_ptr<Schema> schema() const { return schema_; }
+
+  /// \brief Retrieve an array from the record batch
+  /// \param[in] i field index, does not boundscheck
+  /// \return an Array object
+  virtual std::shared_ptr<Array> column(int i) const = 0;
+
+  /// \brief Retrieve an array's internaldata from the record batch
+  /// \param[in] i field index, does not boundscheck
+  /// \return an internal ArrayData object
+  virtual std::shared_ptr<ArrayData> column_data(int i) const = 0;
+
+  virtual std::shared_ptr<RecordBatch> ReplaceSchemaMetadata(
+      const std::shared_ptr<const KeyValueMetadata>& metadata) const = 0;
+
+  /// \brief Name in i-th column
+  const std::string& column_name(int i) const;
+
+  /// \return the number of columns in the table
+  int num_columns() const { return schema_->num_fields(); }
+
+  /// \return the number of rows (the corresponding length of each column)
+  int64_t num_rows() const { return num_rows_; }
+
+  /// \brief Slice each of the arrays in the record batch
+  /// \param[in] offset the starting offset to slice, through end of batch
+  /// \return new record batch
+  virtual std::shared_ptr<RecordBatch> Slice(int64_t offset) const;
+
+  /// \brief Slice each of the arrays in the record batch
+  /// \param[in] offset the starting offset to slice
+  /// \param[in] length the number of elements to slice from offset
+  /// \return new record batch
+  virtual std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length) const = 0;
+
+  /// \brief Check for schema or length inconsistencies
+  /// \return Status
+  virtual Status Validate() const;
+
+ protected:
+  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows);
+
+  std::shared_ptr<Schema> schema_;
+  int64_t num_rows_;
+
+ private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(RecordBatch);
+};
+
+/// \brief Abstract interface for reading stream of record batches
+class ARROW_EXPORT RecordBatchReader {
+ public:
+  virtual ~RecordBatchReader();
+
+  /// \return the shared schema of the record batches in the stream
+  virtual std::shared_ptr<Schema> schema() const = 0;
+
+  /// Read the next record batch in the stream. Return null for batch when
+  /// reaching end of stream
+  ///
+  /// \param[out] batch the next loaded batch, null at end of stream
+  /// \return Status
+  virtual Status ReadNext(std::shared_ptr<RecordBatch>* batch) = 0;
+};
+
+}  // namespace arrow
+
+#endif  // ARROW_RECORD_BATCH_H
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index b490310c26ae6..e77d3aa8bbc08 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -22,6 +22,7 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/test-common.h"
@@ -216,8 +217,8 @@ class TestTable : public TestBase {
 
 TEST_F(TestTable, EmptySchema) {
   auto empty_schema = ::arrow::schema({});
-  table_.reset(new Table(empty_schema, columns_));
-  ASSERT_OK(table_->ValidateColumns());
+  table_ = Table::Make(empty_schema, columns_);
+  ASSERT_OK(table_->Validate());
   ASSERT_EQ(0, table_->num_rows());
   ASSERT_EQ(0, table_->num_columns());
 }
@@ -226,20 +227,20 @@ TEST_F(TestTable, Ctors) {
   const int length = 100;
   MakeExample1(length);
 
-  table_.reset(new Table(schema_, columns_));
-  ASSERT_OK(table_->ValidateColumns());
+  table_ = Table::Make(schema_, columns_);
+  ASSERT_OK(table_->Validate());
   ASSERT_EQ(length, table_->num_rows());
   ASSERT_EQ(3, table_->num_columns());
 
-  auto array_ctor = std::make_shared<Table>(schema_, arrays_);
+  auto array_ctor = Table::Make(schema_, arrays_);
   ASSERT_TRUE(table_->Equals(*array_ctor));
 
-  table_.reset(new Table(schema_, columns_, length));
-  ASSERT_OK(table_->ValidateColumns());
+  table_ = Table::Make(schema_, columns_, length);
+  ASSERT_OK(table_->Validate());
   ASSERT_EQ(length, table_->num_rows());
 
-  ASSERT_OK(MakeTable(schema_, arrays_, &table_));
-  ASSERT_OK(table_->ValidateColumns());
+  table_ = Table::Make(schema_, arrays_);
+  ASSERT_OK(table_->Validate());
   ASSERT_EQ(length, table_->num_rows());
   ASSERT_EQ(3, table_->num_columns());
 }
@@ -248,7 +249,7 @@ TEST_F(TestTable, Metadata) {
   const int length = 100;
   MakeExample1(length);
 
-  table_.reset(new Table(schema_, columns_));
+  table_ = Table::Make(schema_, columns_);
 
   ASSERT_TRUE(table_->schema()->Equals(*schema_));
 
@@ -262,14 +263,14 @@ TEST_F(TestTable, InvalidColumns) {
   const int length = 100;
   MakeExample1(length);
 
-  table_.reset(new Table(schema_, columns_, length - 1));
-  ASSERT_RAISES(Invalid, table_->ValidateColumns());
+  table_ = Table::Make(schema_, columns_, length - 1);
+  ASSERT_RAISES(Invalid, table_->Validate());
 
   columns_.clear();
 
   // Wrong number of columns
-  table_.reset(new Table(schema_, columns_, length));
-  ASSERT_RAISES(Invalid, table_->ValidateColumns());
+  table_ = Table::Make(schema_, columns_, length);
+  ASSERT_RAISES(Invalid, table_->Validate());
 
   columns_ = {
       std::make_shared<Column>(schema_->field(0), MakeRandomArray<Int32Array>(length)),
@@ -277,15 +278,15 @@ TEST_F(TestTable, InvalidColumns) {
       std::make_shared<Column>(schema_->field(2),
                                MakeRandomArray<Int16Array>(length - 1))};
 
-  table_.reset(new Table(schema_, columns_, length));
-  ASSERT_RAISES(Invalid, table_->ValidateColumns());
+  table_ = Table::Make(schema_, columns_, length);
+  ASSERT_RAISES(Invalid, table_->Validate());
 }
 
 TEST_F(TestTable, Equals) {
   const int length = 100;
   MakeExample1(length);
 
-  table_.reset(new Table(schema_, columns_));
+  table_ = Table::Make(schema_, columns_);
 
   ASSERT_TRUE(table_->Equals(*table_));
   // Differing schema
@@ -294,7 +295,8 @@ TEST_F(TestTable, Equals) {
   auto f2 = field("f5", int16());
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
   auto other_schema = std::make_shared<Schema>(fields);
-  ASSERT_FALSE(table_->Equals(Table(other_schema, columns_)));
+  auto other = Table::Make(other_schema, columns_);
+  ASSERT_FALSE(table_->Equals(*other));
   // Differing columns
   std::vector<std::shared_ptr<Column>> other_columns = {
       std::make_shared<Column>(schema_->field(0),
@@ -303,19 +305,21 @@ TEST_F(TestTable, Equals) {
                                MakeRandomArray<UInt8Array>(length, 10)),
       std::make_shared<Column>(schema_->field(2),
                                MakeRandomArray<Int16Array>(length, 10))};
-  ASSERT_FALSE(table_->Equals(Table(schema_, other_columns)));
+
+  other = Table::Make(schema_, other_columns);
+  ASSERT_FALSE(table_->Equals(*other));
 }
 
 TEST_F(TestTable, FromRecordBatches) {
   const int64_t length = 10;
   MakeExample1(length);
 
-  auto batch1 = std::make_shared<RecordBatch>(schema_, length, arrays_);
+  auto batch1 = RecordBatch::Make(schema_, length, arrays_);
 
   std::shared_ptr<Table> result, expected;
   ASSERT_OK(Table::FromRecordBatches({batch1}, &result));
 
-  expected = std::make_shared<Table>(schema_, columns_);
+  expected = Table::Make(schema_, columns_);
   ASSERT_TRUE(result->Equals(*expected));
 
   std::vector<std::shared_ptr<Column>> other_columns;
@@ -325,18 +329,17 @@ TEST_F(TestTable, FromRecordBatches) {
   }
 
   ASSERT_OK(Table::FromRecordBatches({batch1, batch1}, &result));
-  expected = std::make_shared<Table>(schema_, other_columns);
+  expected = Table::Make(schema_, other_columns);
   ASSERT_TRUE(result->Equals(*expected));
 
   // Error states
   std::vector<std::shared_ptr<RecordBatch>> empty_batches;
   ASSERT_RAISES(Invalid, Table::FromRecordBatches(empty_batches, &result));
 
-  std::vector<std::shared_ptr<Field>> fields = {schema_->field(0), schema_->field(1)};
-  auto other_schema = std::make_shared<Schema>(fields);
+  auto other_schema = ::arrow::schema({schema_->field(0), schema_->field(1)});
 
   std::vector<std::shared_ptr<Array>> other_arrays = {arrays_[0], arrays_[1]};
-  auto batch2 = std::make_shared<RecordBatch>(other_schema, length, other_arrays);
+  auto batch2 = RecordBatch::Make(other_schema, length, other_arrays);
   ASSERT_RAISES(Invalid, Table::FromRecordBatches({batch1, batch2}, &result));
 }
 
@@ -344,11 +347,11 @@ TEST_F(TestTable, ConcatenateTables) {
   const int64_t length = 10;
 
   MakeExample1(length);
-  auto batch1 = std::make_shared<RecordBatch>(schema_, length, arrays_);
+  auto batch1 = RecordBatch::Make(schema_, length, arrays_);
 
   // generate different data
   MakeExample1(length);
-  auto batch2 = std::make_shared<RecordBatch>(schema_, length, arrays_);
+  auto batch2 = RecordBatch::Make(schema_, length, arrays_);
 
   std::shared_ptr<Table> t1, t2, t3, result, expected;
   ASSERT_OK(Table::FromRecordBatches({batch1}, &t1));
@@ -362,11 +365,10 @@ TEST_F(TestTable, ConcatenateTables) {
   std::vector<std::shared_ptr<Table>> empty_tables;
   ASSERT_RAISES(Invalid, ConcatenateTables(empty_tables, &result));
 
-  std::vector<std::shared_ptr<Field>> fields = {schema_->field(0), schema_->field(1)};
-  auto other_schema = std::make_shared<Schema>(fields);
+  auto other_schema = ::arrow::schema({schema_->field(0), schema_->field(1)});
 
   std::vector<std::shared_ptr<Array>> other_arrays = {arrays_[0], arrays_[1]};
-  auto batch3 = std::make_shared<RecordBatch>(other_schema, length, other_arrays);
+  auto batch3 = RecordBatch::Make(other_schema, length, other_arrays);
   ASSERT_OK(Table::FromRecordBatches({batch3}, &t3));
 
   ASSERT_RAISES(Invalid, ConcatenateTables({t1, t3}, &result));
@@ -376,31 +378,38 @@ TEST_F(TestTable, RemoveColumn) {
   const int64_t length = 10;
   MakeExample1(length);
 
-  Table table(schema_, columns_);
+  auto table_sp = Table::Make(schema_, columns_);
+  const Table& table = *table_sp;
 
   std::shared_ptr<Table> result;
   ASSERT_OK(table.RemoveColumn(0, &result));
 
   auto ex_schema = ::arrow::schema({schema_->field(1), schema_->field(2)});
   std::vector<std::shared_ptr<Column>> ex_columns = {table.column(1), table.column(2)};
-  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+
+  auto expected = Table::Make(ex_schema, ex_columns);
+  ASSERT_TRUE(result->Equals(*expected));
 
   ASSERT_OK(table.RemoveColumn(1, &result));
   ex_schema = ::arrow::schema({schema_->field(0), schema_->field(2)});
   ex_columns = {table.column(0), table.column(2)};
-  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+
+  expected = Table::Make(ex_schema, ex_columns);
+  ASSERT_TRUE(result->Equals(*expected));
 
   ASSERT_OK(table.RemoveColumn(2, &result));
   ex_schema = ::arrow::schema({schema_->field(0), schema_->field(1)});
   ex_columns = {table.column(0), table.column(1)};
-  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+  expected = Table::Make(ex_schema, ex_columns);
+  ASSERT_TRUE(result->Equals(*expected));
 }
 
 TEST_F(TestTable, AddColumn) {
   const int64_t length = 10;
   MakeExample1(length);
 
-  Table table(schema_, columns_);
+  auto table_sp = Table::Make(schema_, columns_);
+  const Table& table = *table_sp;
 
   std::shared_ptr<Table> result;
   // Some negative tests with invalid index
@@ -419,50 +428,32 @@ TEST_F(TestTable, AddColumn) {
   ASSERT_OK(table.AddColumn(0, columns_[0], &result));
   auto ex_schema = ::arrow::schema(
       {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)});
-  std::vector<std::shared_ptr<Column>> ex_columns = {table.column(0), table.column(0),
-                                                     table.column(1), table.column(2)};
-  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+
+  auto expected = Table::Make(
+      ex_schema, {table.column(0), table.column(0), table.column(1), table.column(2)});
+  ASSERT_TRUE(result->Equals(*expected));
 
   ASSERT_OK(table.AddColumn(1, columns_[0], &result));
   ex_schema = ::arrow::schema(
       {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)});
-  ex_columns = {table.column(0), table.column(0), table.column(1), table.column(2)};
-  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+
+  expected = Table::Make(
+      ex_schema, {table.column(0), table.column(0), table.column(1), table.column(2)});
+  ASSERT_TRUE(result->Equals(*expected));
 
   ASSERT_OK(table.AddColumn(2, columns_[0], &result));
   ex_schema = ::arrow::schema(
       {schema_->field(0), schema_->field(1), schema_->field(0), schema_->field(2)});
-  ex_columns = {table.column(0), table.column(1), table.column(0), table.column(2)};
-  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
+  expected = Table::Make(
+      ex_schema, {table.column(0), table.column(1), table.column(0), table.column(2)});
+  ASSERT_TRUE(result->Equals(*expected));
 
   ASSERT_OK(table.AddColumn(3, columns_[0], &result));
   ex_schema = ::arrow::schema(
       {schema_->field(0), schema_->field(1), schema_->field(2), schema_->field(0)});
-  ex_columns = {table.column(0), table.column(1), table.column(2), table.column(0)};
-  ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns)));
-}
-
-TEST_F(TestTable, IsChunked) {
-  ArrayVector c1, c2;
-
-  auto a1 = MakeRandomArray<Int32Array>(10);
-  auto a2 = MakeRandomArray<Int32Array>(20);
-
-  auto sch1 = arrow::schema({field("f1", int32()), field("f2", int32())});
-
-  std::vector<std::shared_ptr<Column>> columns;
-
-  std::shared_ptr<RecordBatch> batch;
-
-  columns = {column(sch1->field(0), {a1}), column(sch1->field(1), {a1})};
-  auto t1 = std::make_shared<Table>(sch1, columns);
-
-  ASSERT_FALSE(t1->IsChunked());
-
-  columns = {column(sch1->field(0), {a2}), column(sch1->field(1), {a1, a1})};
-  auto t2 = std::make_shared<Table>(sch1, columns);
-
-  ASSERT_TRUE(t2->IsChunked());
+  expected = Table::Make(
+      ex_schema, {table.column(0), table.column(1), table.column(2), table.column(0)});
+  ASSERT_TRUE(result->Equals(*expected));
 }
 
 class TestRecordBatch : public TestBase {};
@@ -475,24 +466,22 @@ TEST_F(TestRecordBatch, Equals) {
   auto f2 = field("f2", int16());
 
   vector<shared_ptr<Field>> fields = {f0, f1, f2};
-  auto schema = std::make_shared<Schema>(fields);
+  auto schema = ::arrow::schema({f0, f1, f2});
+  auto schema2 = ::arrow::schema({f0, f1});
 
   auto a0 = MakeRandomArray<Int32Array>(length);
   auto a1 = MakeRandomArray<UInt8Array>(length);
   auto a2 = MakeRandomArray<Int16Array>(length);
 
-  RecordBatch b1(schema, length, {a0, a1, a2});
-  RecordBatch b3(schema, length, {a0, a1});
-  RecordBatch b4(schema, length, {a0, a1, a1});
+  auto b1 = RecordBatch::Make(schema, length, {a0, a1, a2});
+  auto b3 = RecordBatch::Make(schema2, length, {a0, a1});
+  auto b4 = RecordBatch::Make(schema, length, {a0, a1, a1});
 
-  ASSERT_TRUE(b1.Equals(b1));
-  ASSERT_FALSE(b1.Equals(b3));
-  ASSERT_FALSE(b1.Equals(b4));
+  ASSERT_TRUE(b1->Equals(*b1));
+  ASSERT_FALSE(b1->Equals(*b3));
+  ASSERT_FALSE(b1->Equals(*b4));
 }
 
-#ifdef NDEBUG
-// In debug builds, RecordBatch ctor aborts if you construct an invalid one
-
 TEST_F(TestRecordBatch, Validate) {
   const int length = 10;
 
@@ -507,21 +496,19 @@ TEST_F(TestRecordBatch, Validate) {
   auto a2 = MakeRandomArray<Int16Array>(length);
   auto a3 = MakeRandomArray<Int16Array>(5);
 
-  RecordBatch b1(schema, length, {a0, a1, a2});
+  auto b1 = RecordBatch::Make(schema, length, {a0, a1, a2});
 
-  ASSERT_OK(b1.Validate());
+  ASSERT_OK(b1->Validate());
 
   // Length mismatch
-  RecordBatch b2(schema, length, {a0, a1, a3});
-  ASSERT_RAISES(Invalid, b2.Validate());
+  auto b2 = RecordBatch::Make(schema, length, {a0, a1, a3});
+  ASSERT_RAISES(Invalid, b2->Validate());
 
   // Type mismatch
-  RecordBatch b3(schema, length, {a0, a1, a0});
-  ASSERT_RAISES(Invalid, b3.Validate());
+  auto b3 = RecordBatch::Make(schema, length, {a0, a1, a0});
+  ASSERT_RAISES(Invalid, b3->Validate());
 }
 
-#endif
-
 TEST_F(TestRecordBatch, Slice) {
   const int length = 10;
 
@@ -529,19 +516,19 @@ TEST_F(TestRecordBatch, Slice) {
   auto f1 = field("f1", uint8());
 
   vector<shared_ptr<Field>> fields = {f0, f1};
-  auto schema = std::make_shared<Schema>(fields);
+  auto schema = ::arrow::schema(fields);
 
   auto a0 = MakeRandomArray<Int32Array>(length);
   auto a1 = MakeRandomArray<UInt8Array>(length);
 
-  RecordBatch batch(schema, length, {a0, a1});
+  auto batch = RecordBatch::Make(schema, length, {a0, a1});
 
-  auto batch_slice = batch.Slice(2);
-  auto batch_slice2 = batch.Slice(1, 5);
+  auto batch_slice = batch->Slice(2);
+  auto batch_slice2 = batch->Slice(1, 5);
 
-  ASSERT_EQ(batch_slice->num_rows(), batch.num_rows() - 2);
+  ASSERT_EQ(batch_slice->num_rows(), batch->num_rows() - 2);
 
-  for (int i = 0; i < batch.num_columns(); ++i) {
+  for (int i = 0; i < batch->num_columns(); ++i) {
     ASSERT_EQ(2, batch_slice->column(i)->offset());
     ASSERT_EQ(length - 2, batch_slice->column(i)->length());
 
@@ -567,9 +554,9 @@ TEST_F(TestTableBatchReader, ReadNext) {
   std::shared_ptr<RecordBatch> batch;
 
   columns = {column(sch1->field(0), {a1, a4, a2}), column(sch1->field(1), {a2, a2})};
-  Table t1(sch1, columns);
+  auto t1 = Table::Make(sch1, columns);
 
-  TableBatchReader i1(t1);
+  TableBatchReader i1(*t1);
 
   ASSERT_OK(i1.ReadNext(&batch));
   ASSERT_EQ(10, batch->num_rows());
@@ -584,9 +571,9 @@ TEST_F(TestTableBatchReader, ReadNext) {
   ASSERT_EQ(nullptr, batch);
 
   columns = {column(sch1->field(0), {a1}), column(sch1->field(1), {a4})};
-  Table t2(sch1, columns);
+  auto t2 = Table::Make(sch1, columns);
 
-  TableBatchReader i2(t2);
+  TableBatchReader i2(*t2);
 
   ASSERT_OK(i2.ReadNext(&batch));
   ASSERT_EQ(10, batch->num_rows());
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index fe19bf4ce0b3f..8f3f195765aa9 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -23,6 +23,7 @@
 #include <sstream>
 
 #include "arrow/array.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/logging.h"
@@ -153,171 +154,126 @@ Status Column::ValidateData() {
 }
 
 // ----------------------------------------------------------------------
-// RecordBatch methods
-
-RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows)
-    : schema_(schema), num_rows_(num_rows) {
-  boxed_columns_.resize(schema->num_fields());
-}
-
-RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-                         const std::vector<std::shared_ptr<Array>>& columns)
-    : RecordBatch(schema, num_rows) {
-  columns_.resize(columns.size());
-  for (size_t i = 0; i < columns.size(); ++i) {
-    columns_[i] = columns[i]->data();
-  }
-}
-
-RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-                         std::vector<std::shared_ptr<Array>>&& columns)
-    : RecordBatch(schema, num_rows) {
-  columns_.resize(columns.size());
-  for (size_t i = 0; i < columns.size(); ++i) {
-    columns_[i] = columns[i]->data();
-  }
-}
-
-RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-                         std::vector<std::shared_ptr<ArrayData>>&& columns)
-    : RecordBatch(schema, num_rows) {
-  columns_ = std::move(columns);
-}
-
-RecordBatch::RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-                         const std::vector<std::shared_ptr<ArrayData>>& columns)
-    : RecordBatch(schema, num_rows) {
-  columns_ = columns;
-}
-
-std::shared_ptr<Array> RecordBatch::column(int i) const {
-  if (!boxed_columns_[i]) {
-    boxed_columns_[i] = MakeArray(columns_[i]);
-  }
-  DCHECK(boxed_columns_[i]);
-  return boxed_columns_[i];
-}
-
-const std::string& RecordBatch::column_name(int i) const {
-  return schema_->field(i)->name();
-}
-
-bool RecordBatch::Equals(const RecordBatch& other) const {
-  if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
-    return false;
-  }
+// Table methods
 
-  for (int i = 0; i < num_columns(); ++i) {
-    if (!column(i)->Equals(other.column(i))) {
-      return false;
+/// \class SimpleTable
+/// \brief A basic, non-lazy in-memory table, like SimpleRecordBatch
+class SimpleTable : public Table {
+ public:
+  SimpleTable(const std::shared_ptr<Schema>& schema,
+              const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows = -1)
+      : columns_(columns) {
+    schema_ = schema;
+    if (num_rows < 0) {
+      if (columns.size() == 0) {
+        num_rows_ = 0;
+      } else {
+        num_rows_ = columns[0]->length();
+      }
+    } else {
+      num_rows_ = num_rows;
     }
   }
 
-  return true;
-}
-
-bool RecordBatch::ApproxEquals(const RecordBatch& other) const {
-  if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) {
-    return false;
-  }
+  SimpleTable(const std::shared_ptr<Schema>& schema,
+              const std::vector<std::shared_ptr<Array>>& columns, int64_t num_rows = -1) {
+    schema_ = schema;
+    if (num_rows < 0) {
+      if (columns.size() == 0) {
+        num_rows_ = 0;
+      } else {
+        num_rows_ = columns[0]->length();
+      }
+    } else {
+      num_rows_ = num_rows;
+    }
 
-  for (int i = 0; i < num_columns(); ++i) {
-    if (!column(i)->ApproxEquals(other.column(i))) {
-      return false;
+    columns_.resize(columns.size());
+    for (size_t i = 0; i < columns.size(); ++i) {
+      columns_[i] =
+          std::make_shared<Column>(schema->field(static_cast<int>(i)), columns[i]);
     }
   }
 
-  return true;
-}
-
-std::shared_ptr<RecordBatch> RecordBatch::ReplaceSchemaMetadata(
-    const std::shared_ptr<const KeyValueMetadata>& metadata) const {
-  auto new_schema = schema_->AddMetadata(metadata);
-  return std::make_shared<RecordBatch>(new_schema, num_rows_, columns_);
-}
+  std::shared_ptr<Column> column(int i) const override { return columns_[i]; }
 
-std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset) const {
-  return Slice(offset, this->num_rows() - offset);
-}
+  Status RemoveColumn(int i, std::shared_ptr<Table>* out) const override {
+    std::shared_ptr<Schema> new_schema;
+    RETURN_NOT_OK(schema_->RemoveField(i, &new_schema));
 
-std::shared_ptr<RecordBatch> RecordBatch::Slice(int64_t offset, int64_t length) const {
-  std::vector<std::shared_ptr<ArrayData>> arrays;
-  arrays.reserve(num_columns());
-  for (const auto& field : columns_) {
-    int64_t col_length = std::min(field->length - offset, length);
-    int64_t col_offset = field->offset + offset;
-
-    auto new_data = std::make_shared<ArrayData>(*field);
-    new_data->length = col_length;
-    new_data->offset = col_offset;
-    new_data->null_count = kUnknownNullCount;
-    arrays.emplace_back(new_data);
+    *out = Table::Make(new_schema, internal::DeleteVectorElement(columns_, i));
+    return Status::OK();
   }
-  int64_t num_rows = std::min(num_rows_ - offset, length);
-  return std::make_shared<RecordBatch>(schema_, num_rows, std::move(arrays));
-}
 
-Status RecordBatch::Validate() const {
-  for (int i = 0; i < num_columns(); ++i) {
-    const ArrayData& arr = *columns_[i];
-    if (arr.length != num_rows_) {
+  Status AddColumn(int i, const std::shared_ptr<Column>& col,
+                   std::shared_ptr<Table>* out) const override {
+    if (i < 0 || i > num_columns() + 1) {
+      return Status::Invalid("Invalid column index.");
+    }
+    if (col == nullptr) {
       std::stringstream ss;
-      ss << "Number of rows in column " << i << " did not match batch: " << arr.length
-         << " vs " << num_rows_;
+      ss << "Column " << i << " was null";
       return Status::Invalid(ss.str());
     }
-    const auto& schema_type = *schema_->field(i)->type();
-    if (!arr.type->Equals(schema_type)) {
+    if (col->length() != num_rows_) {
       std::stringstream ss;
-      ss << "Column " << i << " type not match schema: " << arr.type->ToString() << " vs "
-         << schema_type.ToString();
+      ss << "Added column's length must match table's length. Expected length "
+         << num_rows_ << " but got length " << col->length();
       return Status::Invalid(ss.str());
     }
+
+    std::shared_ptr<Schema> new_schema;
+    RETURN_NOT_OK(schema_->AddField(i, col->field(), &new_schema));
+
+    *out = Table::Make(new_schema, internal::AddVectorElement(columns_, i, col));
+    return Status::OK();
   }
-  return Status::OK();
-}
 
-// ----------------------------------------------------------------------
-// Table methods
+  std::shared_ptr<Table> ReplaceSchemaMetadata(
+      const std::shared_ptr<const KeyValueMetadata>& metadata) const override {
+    auto new_schema = schema_->AddMetadata(metadata);
+    return Table::Make(new_schema, columns_);
+  }
 
-Table::Table(const std::shared_ptr<Schema>& schema,
-             const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows)
-    : schema_(schema), columns_(columns) {
-  if (num_rows < 0) {
-    if (columns.size() == 0) {
-      num_rows_ = 0;
-    } else {
-      num_rows_ = columns[0]->length();
+  Status Validate() const override {
+    if (static_cast<int>(columns_.size()) != schema_->num_fields()) {
+      return Status::Invalid("Number of columns did not match schema");
     }
-  } else {
-    num_rows_ = num_rows;
-  }
-}
 
-Table::Table(const std::shared_ptr<Schema>& schema,
-             const std::vector<std::shared_ptr<Array>>& columns, int64_t num_rows)
-    : schema_(schema) {
-  if (num_rows < 0) {
-    if (columns.size() == 0) {
-      num_rows_ = 0;
-    } else {
-      num_rows_ = columns[0]->length();
+    // Make sure columns are all the same length
+    for (int i = 0; i < num_columns(); ++i) {
+      const Column* col = columns_[i].get();
+      if (col == nullptr) {
+        std::stringstream ss;
+        ss << "Column " << i << " was null";
+        return Status::Invalid(ss.str());
+      }
+      if (col->length() != num_rows_) {
+        std::stringstream ss;
+        ss << "Column " << i << " named " << col->name() << " expected length "
+           << num_rows_ << " but got length " << col->length();
+        return Status::Invalid(ss.str());
+      }
     }
-  } else {
-    num_rows_ = num_rows;
+    return Status::OK();
   }
 
-  columns_.resize(columns.size());
-  for (size_t i = 0; i < columns.size(); ++i) {
-    columns_[i] =
-        std::make_shared<Column>(schema->field(static_cast<int>(i)), columns[i]);
-  }
+ private:
+  std::vector<std::shared_ptr<Column>> columns_;
+};
+
+Table::Table() {}
+
+std::shared_ptr<Table> Table::Make(const std::shared_ptr<Schema>& schema,
+                                   const std::vector<std::shared_ptr<Column>>& columns,
+                                   int64_t num_rows) {
+  return std::make_shared<SimpleTable>(schema, columns, num_rows);
 }
 
-std::shared_ptr<Table> Table::ReplaceSchemaMetadata(
-    const std::shared_ptr<const KeyValueMetadata>& metadata) const {
-  auto new_schema = schema_->AddMetadata(metadata);
-  return std::make_shared<Table>(new_schema, columns_);
+std::shared_ptr<Table> Table::Make(const std::shared_ptr<Schema>& schema,
+                                   const std::vector<std::shared_ptr<Array>>& arrays,
+                                   int64_t num_rows) {
+  return std::make_shared<SimpleTable>(schema, arrays, num_rows);
 }
 
 Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>& batches,
@@ -351,7 +307,7 @@ Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>&
     columns[i] = std::make_shared<Column>(schema->field(i), column_arrays);
   }
 
-  *table = std::make_shared<Table>(schema, columns);
+  *table = Table::Make(schema, columns);
   return Status::OK();
 }
 
@@ -388,7 +344,7 @@ Status ConcatenateTables(const std::vector<std::shared_ptr<Table>>& tables,
     }
     columns[i] = std::make_shared<Column>(schema->field(i), column_arrays);
   }
-  *table = std::make_shared<Table>(schema, columns);
+  *table = Table::Make(schema, columns);
   return Status::OK();
 }
 
@@ -399,82 +355,19 @@ bool Table::Equals(const Table& other) const {
   if (!schema_->Equals(*other.schema())) {
     return false;
   }
-  if (static_cast<int64_t>(columns_.size()) != other.num_columns()) {
+  if (this->num_columns() != other.num_columns()) {
     return false;
   }
 
-  for (int i = 0; i < static_cast<int>(columns_.size()); i++) {
-    if (!columns_[i]->Equals(other.column(i))) {
+  for (int i = 0; i < this->num_columns(); i++) {
+    if (!this->column(i)->Equals(other.column(i))) {
       return false;
     }
   }
   return true;
 }
 
-Status Table::RemoveColumn(int i, std::shared_ptr<Table>* out) const {
-  std::shared_ptr<Schema> new_schema;
-  RETURN_NOT_OK(schema_->RemoveField(i, &new_schema));
-
-  *out = std::make_shared<Table>(new_schema, internal::DeleteVectorElement(columns_, i));
-  return Status::OK();
-}
-
-Status Table::AddColumn(int i, const std::shared_ptr<Column>& col,
-                        std::shared_ptr<Table>* out) const {
-  if (i < 0 || i > num_columns() + 1) {
-    return Status::Invalid("Invalid column index.");
-  }
-  if (col == nullptr) {
-    std::stringstream ss;
-    ss << "Column " << i << " was null";
-    return Status::Invalid(ss.str());
-  }
-  if (col->length() != num_rows_) {
-    std::stringstream ss;
-    ss << "Added column's length must match table's length. Expected length " << num_rows_
-       << " but got length " << col->length();
-    return Status::Invalid(ss.str());
-  }
-
-  std::shared_ptr<Schema> new_schema;
-  RETURN_NOT_OK(schema_->AddField(i, col->field(), &new_schema));
-
-  *out =
-      std::make_shared<Table>(new_schema, internal::AddVectorElement(columns_, i, col));
-  return Status::OK();
-}
-
-Status Table::ValidateColumns() const {
-  if (num_columns() != schema_->num_fields()) {
-    return Status::Invalid("Number of columns did not match schema");
-  }
-
-  // Make sure columns are all the same length
-  for (size_t i = 0; i < columns_.size(); ++i) {
-    const Column* col = columns_[i].get();
-    if (col == nullptr) {
-      std::stringstream ss;
-      ss << "Column " << i << " was null";
-      return Status::Invalid(ss.str());
-    }
-    if (col->length() != num_rows_) {
-      std::stringstream ss;
-      ss << "Column " << i << " named " << col->name() << " expected length " << num_rows_
-         << " but got length " << col->length();
-      return Status::Invalid(ss.str());
-    }
-  }
-  return Status::OK();
-}
-
-bool Table::IsChunked() const {
-  for (size_t i = 0; i < columns_.size(); ++i) {
-    if (columns_[i]->data()->num_chunks() > 1) {
-      return true;
-    }
-  }
-  return false;
-}
+#ifndef ARROW_NO_DEPRECATED_API
 
 Status MakeTable(const std::shared_ptr<Schema>& schema,
                  const std::vector<std::shared_ptr<Array>>& arrays,
@@ -493,15 +386,12 @@ Status MakeTable(const std::shared_ptr<Schema>& schema,
     columns.emplace_back(std::make_shared<Column>(schema->field(i), arrays[i]));
   }
 
-  *table = std::make_shared<Table>(schema, columns);
+  *table = Table::Make(schema, columns);
 
   return Status::OK();
 }
 
-// ----------------------------------------------------------------------
-// Base record batch reader
-
-RecordBatchReader::~RecordBatchReader() {}
+#endif  // ARROW_NO_DEPRECATED_API
 
 // ----------------------------------------------------------------------
 // Convert a table to a sequence of record batches
@@ -565,8 +455,7 @@ class TableBatchReader::TableBatchReaderImpl {
     }
 
     absolute_row_position_ += chunksize;
-    *out =
-        std::make_shared<RecordBatch>(table_.schema(), chunksize, std::move(batch_data));
+    *out = RecordBatch::Make(table_.schema(), chunksize, std::move(batch_data));
 
     return Status::OK();
   }
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 2cff32f74ef14..d0312d93cb93c 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -24,6 +24,7 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/record_batch.h"
 #include "arrow/type.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
@@ -33,8 +34,6 @@ namespace arrow {
 class KeyValueMetadata;
 class Status;
 
-using ArrayVector = std::vector<std::shared_ptr<Array>>;
-
 /// \class ChunkedArray
 /// \brief A data structure managing a list of primitive Arrow arrays logically
 /// as one large array
@@ -113,123 +112,28 @@ class ARROW_EXPORT Column {
   ARROW_DISALLOW_COPY_AND_ASSIGN(Column);
 };
 
-/// \class RecordBatch
-/// \brief Collection of equal-length arrays matching a particular Schema
-///
-/// A record batch is table-like data structure consisting of an internal
-/// sequence of fields, each a contiguous Arrow array
-class ARROW_EXPORT RecordBatch {
- public:
-  /// \param[in] schema The record batch schema
-  /// \param[in] num_rows length of fields in the record batch. Each array
-  /// should have the same length as num_rows
-  /// \param[in] columns the record batch fields as vector of arrays
-  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-              const std::vector<std::shared_ptr<Array>>& columns);
-
-  /// \brief Move-based constructor for a vector of Array instances
-  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-              std::vector<std::shared_ptr<Array>>&& columns);
-
-  /// \brief Construct record batch from vector of internal data structures
-  /// \since 0.5.0
-  ///
-  /// This class is only provided with an rvalue-reference for the input data,
-  /// and is intended for internal use, or advanced users.
-  ///
-  /// \param schema the record batch schema
-  /// \param num_rows the number of semantic rows in the record batch. This
-  /// should be equal to the length of each field
-  /// \param columns the data for the batch's columns
-  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-              std::vector<std::shared_ptr<ArrayData>>&& columns);
-
-  /// \brief Construct record batch by copying vector of array data
-  /// \since 0.5.0
-  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows,
-              const std::vector<std::shared_ptr<ArrayData>>& columns);
-
-  /// \brief Determine if two record batches are exactly equal
-  /// \return true if batches are equal
-  bool Equals(const RecordBatch& other) const;
-
-  /// \brief Determine if two record batches are approximately equal
-  bool ApproxEquals(const RecordBatch& other) const;
-
-  // \return the table's schema
-  /// \return true if batches are equal
-  std::shared_ptr<Schema> schema() const { return schema_; }
-
-  /// \brief Retrieve an array from the record batch
-  /// \param[in] i field index, does not boundscheck
-  /// \return an Array object
-  std::shared_ptr<Array> column(int i) const;
-
-  std::shared_ptr<ArrayData> column_data(int i) const { return columns_[i]; }
-
-  /// \brief Name in i-th column
-  const std::string& column_name(int i) const;
-
-  /// \return the number of columns in the table
-  int num_columns() const { return static_cast<int>(columns_.size()); }
-
-  /// \return the number of rows (the corresponding length of each column)
-  int64_t num_rows() const { return num_rows_; }
-
-  /// \brief Replace schema key-value metadata with new metadata (EXPERIMENTAL)
-  /// \since 0.5.0
-  ///
-  /// \param[in] metadata new KeyValueMetadata
-  /// \return new RecordBatch
-  std::shared_ptr<RecordBatch> ReplaceSchemaMetadata(
-      const std::shared_ptr<const KeyValueMetadata>& metadata) const;
-
-  /// \brief Slice each of the arrays in the record batch
-  /// \param[in] offset the starting offset to slice, through end of batch
-  /// \return new record batch
-  std::shared_ptr<RecordBatch> Slice(int64_t offset) const;
-
-  /// \brief Slice each of the arrays in the record batch
-  /// \param[in] offset the starting offset to slice
-  /// \param[in] length the number of elements to slice from offset
-  /// \return new record batch
-  std::shared_ptr<RecordBatch> Slice(int64_t offset, int64_t length) const;
-
-  /// \brief Check for schema or length inconsistencies
-  /// \return Status
-  Status Validate() const;
-
- private:
-  ARROW_DISALLOW_COPY_AND_ASSIGN(RecordBatch);
-
-  RecordBatch(const std::shared_ptr<Schema>& schema, int64_t num_rows);
-
-  std::shared_ptr<Schema> schema_;
-  int64_t num_rows_;
-  std::vector<std::shared_ptr<ArrayData>> columns_;
-
-  // Caching boxed array data
-  mutable std::vector<std::shared_ptr<Array>> boxed_columns_;
-};
-
 /// \class Table
 /// \brief Logical table as sequence of chunked arrays
 class ARROW_EXPORT Table {
  public:
+  virtual ~Table() = default;
+
   /// \brief Construct Table from schema and columns
   /// If columns is zero-length, the table's number of rows is zero
   /// \param schema The table schema (column types)
   /// \param columns The table's columns
   /// \param num_rows number of rows in table, -1 (default) to infer from columns
-  Table(const std::shared_ptr<Schema>& schema,
-        const std::vector<std::shared_ptr<Column>>& columns, int64_t num_rows = -1);
+  static std::shared_ptr<Table> Make(const std::shared_ptr<Schema>& schema,
+                                     const std::vector<std::shared_ptr<Column>>& columns,
+                                     int64_t num_rows = -1);
 
   /// \brief Construct Table from schema and arrays
   /// \param schema The table schema (column types)
   /// \param arrays The table's columns as arrays
   /// \param num_rows number of rows in table, -1 (default) to infer from columns
-  Table(const std::shared_ptr<Schema>& schema,
-        const std::vector<std::shared_ptr<Array>>& arrays, int64_t num_rows = -1);
+  static std::shared_ptr<Table> Make(const std::shared_ptr<Schema>& schema,
+                                     const std::vector<std::shared_ptr<Array>>& arrays,
+                                     int64_t num_rows = -1);
 
   // Construct table from RecordBatch, but only if all of the batch schemas are
   // equal. Returns Status::Invalid if there is some problem
@@ -242,25 +146,28 @@ class ARROW_EXPORT Table {
 
   /// \param[in] i column index, does not boundscheck
   /// \return the i-th column
-  std::shared_ptr<Column> column(int i) const { return columns_[i]; }
+  virtual std::shared_ptr<Column> column(int i) const = 0;
 
   /// \brief Remove column from the table, producing a new Table
-  Status RemoveColumn(int i, std::shared_ptr<Table>* out) const;
+  virtual Status RemoveColumn(int i, std::shared_ptr<Table>* out) const = 0;
 
   /// \brief Add column to the table, producing a new Table
-  Status AddColumn(int i, const std::shared_ptr<Column>& column,
-                   std::shared_ptr<Table>* out) const;
+  virtual Status AddColumn(int i, const std::shared_ptr<Column>& column,
+                           std::shared_ptr<Table>* out) const = 0;
 
   /// \brief Replace schema key-value metadata with new metadata (EXPERIMENTAL)
   /// \since 0.5.0
   ///
   /// \param[in] metadata new KeyValueMetadata
   /// \return new Table
-  std::shared_ptr<Table> ReplaceSchemaMetadata(
-      const std::shared_ptr<const KeyValueMetadata>& metadata) const;
+  virtual std::shared_ptr<Table> ReplaceSchemaMetadata(
+      const std::shared_ptr<const KeyValueMetadata>& metadata) const = 0;
+
+  /// \brief Perform any checks to validate the input arguments
+  virtual Status Validate() const = 0;
 
   /// \return the number of columns in the table
-  int num_columns() const { return static_cast<int>(columns_.size()); }
+  int num_columns() const { return schema_->num_fields(); }
 
   /// \return the number of rows (the corresponding length of each column)
   int64_t num_rows() const { return num_rows_; }
@@ -268,35 +175,14 @@ class ARROW_EXPORT Table {
   /// \brief Determine if semantic contents of tables are exactly equal
   bool Equals(const Table& other) const;
 
-  /// \brief Perform any checks to validate the input arguments
-  Status ValidateColumns() const;
-
-  /// \brief Return true if any column has multiple chunks
-  bool IsChunked() const;
-
- private:
-  ARROW_DISALLOW_COPY_AND_ASSIGN(Table);
+ protected:
+  Table();
 
   std::shared_ptr<Schema> schema_;
-  std::vector<std::shared_ptr<Column>> columns_;
-
   int64_t num_rows_;
-};
-
-/// \brief Abstract interface for reading stream of record batches
-class ARROW_EXPORT RecordBatchReader {
- public:
-  virtual ~RecordBatchReader();
 
-  /// \return the shared schema of the record batches in the stream
-  virtual std::shared_ptr<Schema> schema() const = 0;
-
-  /// Read the next record batch in the stream. Return null for batch when
-  /// reaching end of stream
-  ///
-  /// \param[out] batch the next loaded batch, null at end of stream
-  /// \return Status
-  virtual Status ReadNext(std::shared_ptr<RecordBatch>* batch) = 0;
+ private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(Table);
 };
 
 /// \brief Compute a sequence of record batches from a (possibly chunked) Table
@@ -322,13 +208,18 @@ ARROW_EXPORT
 Status ConcatenateTables(const std::vector<std::shared_ptr<Table>>& tables,
                          std::shared_ptr<Table>* table);
 
+#ifndef ARROW_NO_DEPRECATED_API
+
 /// \brief Construct table from multiple input tables.
 /// \return Status, fails if any schemas are different
+/// \note Deprecated since 0.8.0
 ARROW_EXPORT
 Status MakeTable(const std::shared_ptr<Schema>& schema,
                  const std::vector<std::shared_ptr<Array>>& arrays,
                  std::shared_ptr<Table>* table);
 
+#endif
+
 }  // namespace arrow
 
 #endif  // ARROW_TABLE_H
diff --git a/cpp/src/arrow/table_builder-test.cc b/cpp/src/arrow/table_builder-test.cc
index 07d9b6b2d6568..8167577e9064e 100644
--- a/cpp/src/arrow/table_builder-test.cc
+++ b/cpp/src/arrow/table_builder-test.cc
@@ -22,6 +22,7 @@
 #include "gtest/gtest.h"
 
 #include "arrow/array.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/table_builder.h"
@@ -98,7 +99,7 @@ TEST_F(TestRecordBatchBuilder, Basics) {
   ASSERT_OK(ex_b1.Finish(&a1));
   ASSERT_OK(ex_b2.Finish(&a2));
 
-  RecordBatch expected(schema, 4, {a0, a1, a2});
+  auto expected = RecordBatch::Make(schema, 4, {a0, a1, a2});
 
   // Builder attributes
   ASSERT_EQ(3, builder->num_fields());
@@ -119,7 +120,7 @@ TEST_F(TestRecordBatchBuilder, Basics) {
       ASSERT_OK(builder->Flush(&batch));
     }
 
-    ASSERT_BATCHES_EQUAL(expected, *batch);
+    ASSERT_BATCHES_EQUAL(*expected, *batch);
   }
 
   // Test setting initial capacity
diff --git a/cpp/src/arrow/table_builder.cc b/cpp/src/arrow/table_builder.cc
index a1bd95940a6db..379d886deacba 100644
--- a/cpp/src/arrow/table_builder.cc
+++ b/cpp/src/arrow/table_builder.cc
@@ -24,6 +24,7 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"
@@ -64,7 +65,7 @@ Status RecordBatchBuilder::Flush(bool reset_builders,
     }
     length = fields[i]->length();
   }
-  *batch = std::make_shared<RecordBatch>(schema_, length, std::move(fields));
+  *batch = RecordBatch::Make(schema_, length, std::move(fields));
   if (reset_builders) {
     return InitBuilders();
   } else {
diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h
index a4c4fddff7348..911adf7b6057a 100644
--- a/cpp/src/arrow/test-common.h
+++ b/cpp/src/arrow/test-common.h
@@ -30,7 +30,6 @@
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
 #include "arrow/memory_pool.h"
-#include "arrow/table.h"
 #include "arrow/test-util.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 77f489ab17797..1a34808488a83 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -35,7 +35,6 @@
 #include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -375,7 +374,7 @@ void AssertArraysEqual(const Array& expected, const Array& actual) {
 
 #define ASSERT_BATCHES_EQUAL(LEFT, RIGHT)    \
   do {                                       \
-    if (!LEFT.ApproxEquals(RIGHT)) {         \
+    if (!(LEFT).ApproxEquals(RIGHT)) {       \
       std::stringstream ss;                  \
       ss << "Left:\n";                       \
       ASSERT_OK(PrettyPrint(LEFT, 0, &ss));  \
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 70f275c0fa495..8dcc1592da01f 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -498,9 +498,9 @@ class ARROW_EXPORT StructType : public NestedType {
   std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
-class ARROW_EXPORT DecimalBaseType : public FixedSizeBinaryType {
+class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
  public:
-  explicit DecimalBaseType(int32_t byte_width, int32_t precision, int32_t scale)
+  explicit DecimalType(int32_t byte_width, int32_t precision, int32_t scale)
       : FixedSizeBinaryType(byte_width, Type::DECIMAL),
         precision_(precision),
         scale_(scale) {}
@@ -513,21 +513,18 @@ class ARROW_EXPORT DecimalBaseType : public FixedSizeBinaryType {
   int32_t scale_;
 };
 
-class ARROW_EXPORT Decimal128Type : public DecimalBaseType {
+class ARROW_EXPORT Decimal128Type : public DecimalType {
  public:
   static constexpr Type::type type_id = Type::DECIMAL;
 
   explicit Decimal128Type(int32_t precision, int32_t scale)
-      : DecimalBaseType(16, precision, scale) {}
+      : DecimalType(16, precision, scale) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   std::string name() const override { return "decimal"; }
 };
 
-// TODO(wesm): Remove this
-using DecimalType = Decimal128Type;
-
 struct UnionMode {
   enum type { SPARSE, DENSE };
 };
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index dbfd89cc378cb..73e34c7b21033 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -403,8 +403,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CChunkedArray] data()
 
     cdef cppclass CRecordBatch" arrow::RecordBatch":
-        CRecordBatch(const shared_ptr[CSchema]& schema, int64_t num_rows,
-                     const vector[shared_ptr[CArray]]& columns)
+        @staticmethod
+        shared_ptr[CRecordBatch] Make(
+            const shared_ptr[CSchema]& schema, int64_t num_rows,
+            const vector[shared_ptr[CArray]]& columns)
 
         c_bool Equals(const CRecordBatch& other)
 
@@ -427,6 +429,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CTable(const shared_ptr[CSchema]& schema,
                const vector[shared_ptr[CColumn]]& columns)
 
+        @staticmethod
+        shared_ptr[CTable] Make(
+            const shared_ptr[CSchema]& schema,
+            const vector[shared_ptr[CColumn]]& columns)
+
         @staticmethod
         CStatus FromRecordBatches(
             const vector[shared_ptr[CRecordBatch]]& batches,
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 591f3297587a3..8c5b8bbc343c0 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -724,7 +724,6 @@ cdef class RecordBatch:
             Array arr
             c_string c_name
             shared_ptr[CSchema] schema
-            shared_ptr[CRecordBatch] batch
             vector[shared_ptr[CArray]] c_arrays
             int64_t num_rows
             int64_t i
@@ -740,8 +739,8 @@ cdef class RecordBatch:
         for arr in arrays:
             c_arrays.push_back(arr.sp_array)
 
-        batch.reset(new CRecordBatch(schema, num_rows, c_arrays))
-        return pyarrow_wrap_batch(batch)
+        return pyarrow_wrap_batch(
+            CRecordBatch.Make(schema, num_rows, c_arrays))
 
 
 def table_to_blocks(PandasOptions options, Table table, int nthreads,
@@ -946,8 +945,7 @@ cdef class Table:
             else:
                 raise ValueError(type(arrays[i]))
 
-        table.reset(new CTable(c_schema, columns))
-        return pyarrow_wrap_table(table)
+        return pyarrow_wrap_table(CTable.Make(c_schema, columns))
 
     @staticmethod
     def from_batches(batches):

From 1516306d40e4b785416282fe84daa9db342e5b90 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Wed, 22 Nov 2017 13:46:56 -0500
Subject: [PATCH 1273/1644] ARROW-1047: [Java] Add Generic Reader Interface for
 Stream Format

This change decouples the reading of messages from the ReadChannel so that it is possible to build a reader that is not tied to a specific stream.  This adds a new interface `MessageReader` that will return a `Message` and the message body.  The `MessageChannelReader` implements this interface to read from a `ReadChannel` to match the current functionality.  A re-org of reading and writing packages is also done to better organize classes under an `ipc` package.

There is a slight change in behavior for the `ArrowFileReader` that should not affect any usage.  Previously,  the schema was read during initialization, and all dictionaries read just before the first record batch was read.  Now, all dictionaries are read directly after the schema, and not specifically tied to reading the first record batch.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1259 from BryanCutler/java-generic-stream-interfaces-ARROW-1047 and squashes the following commits:

43314ca1 [Bryan Cutler] ARROW-1047: [Java] Add Generic Reader Interface for Stream Format
---
 .../org/apache/arrow/tools/EchoServer.java    |   4 +-
 .../org/apache/arrow/tools/FileRoundtrip.java |   4 +-
 .../org/apache/arrow/tools/FileToStream.java  |   4 +-
 .../org/apache/arrow/tools/Integration.java   |  10 +-
 .../org/apache/arrow/tools/StreamToFile.java  |   4 +-
 .../arrow/tools/ArrowFileTestFixtures.java    |   6 +-
 .../apache/arrow/tools/EchoServerTest.java    |   4 +-
 .../main/codegen/templates/UnionVector.java   |   2 +-
 .../arrow/vector/BaseDataValueVector.java     |   2 +-
 .../vector/BaseNullableFixedWidthVector.java  |   2 +-
 .../BaseNullableVariableWidthVector.java      |   2 +-
 .../org/apache/arrow/vector/BitVector.java    |   2 +-
 .../apache/arrow/vector/BitVectorHelper.java  |   2 +-
 .../org/apache/arrow/vector/BufferBacked.java |   2 +-
 .../org/apache/arrow/vector/FieldVector.java  |   2 +-
 .../org/apache/arrow/vector/VectorLoader.java |   6 +-
 .../apache/arrow/vector/VectorUnloader.java   |   6 +-
 .../org/apache/arrow/vector/ZeroVector.java   |   2 +-
 .../vector/complex/FixedSizeListVector.java   |   2 +-
 .../arrow/vector/complex/ListVector.java      |   2 +-
 .../vector/complex/NullableMapVector.java     |   2 +-
 .../vector/{file => ipc}/ArrowFileReader.java |  59 +++++--
 .../vector/{file => ipc}/ArrowFileWriter.java |   4 +-
 .../vector/{file => ipc}/ArrowMagic.java      |   4 +-
 .../vector/{file => ipc}/ArrowReader.java     | 150 ++++++++++--------
 .../arrow/vector/ipc/ArrowStreamReader.java   | 148 +++++++++++++++++
 .../{stream => ipc}/ArrowStreamWriter.java    |  11 +-
 .../vector/{file => ipc}/ArrowWriter.java     |   9 +-
 .../InvalidArrowFileException.java            |   2 +-
 .../{file/json => ipc}/JsonFileReader.java    |  10 +-
 .../{file/json => ipc}/JsonFileWriter.java    |   6 +-
 .../vector/{file => ipc}/ReadChannel.java     |   2 +-
 .../{file => ipc}/SeekableReadChannel.java    |   2 +-
 .../vector/{file => ipc}/WriteChannel.java    |   4 +-
 .../{file => ipc/message}/ArrowBlock.java     |   3 +-
 .../{schema => ipc/message}/ArrowBuffer.java  |   2 +-
 .../message}/ArrowDictionaryBatch.java        |   2 +-
 .../message}/ArrowFieldNode.java              |   2 +-
 .../{file => ipc/message}/ArrowFooter.java    |   5 +-
 .../{schema => ipc/message}/ArrowMessage.java |   2 +-
 .../message}/ArrowRecordBatch.java            |   8 +-
 .../message}/ArrowVectorType.java             |   2 +-
 .../message}/FBSerializable.java              |   2 +-
 .../message}/FBSerializables.java             |   2 +-
 .../ipc/message/MessageChannelReader.java     | 115 ++++++++++++++
 .../vector/ipc/message/MessageReader.java     |  65 ++++++++
 .../message}/MessageSerializer.java           |  91 +++++++----
 .../{schema => ipc/message}/TypeLayout.java   |  46 +++---
 .../{schema => ipc/message}/VectorLayout.java |  23 ++-
 .../vector/stream/ArrowStreamReader.java      |  66 --------
 .../apache/arrow/vector/types/pojo/Field.java |   8 +-
 .../apache/arrow/vector/TestValueVector.java  |   6 +-
 .../arrow/vector/TestVectorUnloadLoad.java    |   4 +-
 .../vector/{file => ipc}/BaseFileTest.java    |   2 +-
 .../MessageSerializerTest.java                |  13 +-
 .../vector/{file => ipc}/TestArrowFile.java   |  61 +++----
 .../vector/{file => ipc}/TestArrowFooter.java |   4 +-
 .../{file => ipc}/TestArrowReaderWriter.java  |  21 ++-
 .../vector/{file => ipc}/TestArrowStream.java |  15 +-
 .../{file => ipc}/TestArrowStreamPipe.java    |  50 +++---
 .../{file/json => ipc}/TestJSONFile.java      |   3 +-
 61 files changed, 698 insertions(+), 408 deletions(-)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowFileReader.java (77%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowFileWriter.java (94%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowMagic.java (93%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowReader.java (65%)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java
 rename java/vector/src/main/java/org/apache/arrow/vector/{stream => ipc}/ArrowStreamWriter.java (84%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowWriter.java (95%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/InvalidArrowFileException.java (96%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file/json => ipc}/JsonFileReader.java (98%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file/json => ipc}/JsonFileWriter.java (98%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ReadChannel.java (98%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/SeekableReadChannel.java (97%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/WriteChannel.java (97%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc/message}/ArrowBlock.java (96%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowBuffer.java (97%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowDictionaryBatch.java (97%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowFieldNode.java (97%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc/message}/ArrowFooter.java (96%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowMessage.java (96%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowRecordBatch.java (94%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowVectorType.java (98%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/FBSerializable.java (95%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/FBSerializables.java (96%)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageChannelReader.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageReader.java
 rename java/vector/src/main/java/org/apache/arrow/vector/{stream => ipc/message}/MessageSerializer.java (86%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/TypeLayout.java (80%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/VectorLayout.java (89%)
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
 rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/BaseFileTest.java (99%)
 rename java/vector/src/test/java/org/apache/arrow/vector/{stream => ipc}/MessageSerializerTest.java (94%)
 rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowFile.java (94%)
 rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowFooter.java (93%)
 rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowReaderWriter.java (85%)
 rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowStream.java (88%)
 rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowStreamPipe.java (78%)
 rename java/vector/src/test/java/org/apache/arrow/vector/{file/json => ipc}/TestJSONFile.java (99%)

diff --git a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
index 3091bc4dab123..ce6b5164a8cbb 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
@@ -23,8 +23,8 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.stream.ArrowStreamReader;
-import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.apache.arrow.vector.ipc.ArrowStreamReader;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
index ab8fa6e45cecd..6e45305bf6cd4 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
@@ -22,8 +22,8 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.file.ArrowFileReader;
-import org.apache.arrow.vector.file.ArrowFileWriter;
+import org.apache.arrow.vector.ipc.ArrowFileReader;
+import org.apache.arrow.vector.ipc.ArrowFileWriter;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
index 6722b30fa7f50..3db01f40c590e 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java
@@ -21,8 +21,8 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.file.ArrowFileReader;
-import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.apache.arrow.vector.ipc.ArrowFileReader;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
 
 import java.io.File;
 import java.io.FileInputStream;
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
index d2b35e65a8172..666f1ddeabcc9 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java
@@ -22,11 +22,11 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowFileReader;
-import org.apache.arrow.vector.file.ArrowFileWriter;
-import org.apache.arrow.vector.file.json.JsonFileReader;
-import org.apache.arrow.vector.file.json.JsonFileWriter;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.ArrowFileReader;
+import org.apache.arrow.vector.ipc.ArrowFileWriter;
+import org.apache.arrow.vector.ipc.JsonFileReader;
+import org.apache.arrow.vector.ipc.JsonFileWriter;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
diff --git a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
index ef1a11f6bfac8..42d336af9b040 100644
--- a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
+++ b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
@@ -21,8 +21,8 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.file.ArrowFileWriter;
-import org.apache.arrow.vector.stream.ArrowStreamReader;
+import org.apache.arrow.vector.ipc.ArrowFileWriter;
+import org.apache.arrow.vector.ipc.ArrowStreamReader;
 
 import java.io.File;
 import java.io.FileInputStream;
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
index c56a5a3303b75..eac517d96bd48 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -28,9 +28,9 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowFileReader;
-import org.apache.arrow.vector.file.ArrowFileWriter;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.ArrowFileReader;
+import org.apache.arrow.vector.ipc.ArrowFileWriter;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Assert;
 
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 89714e4479180..d8693c596155b 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -44,8 +44,8 @@
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
-import org.apache.arrow.vector.stream.ArrowStreamReader;
-import org.apache.arrow.vector.stream.ArrowStreamWriter;
+import org.apache.arrow.vector.ipc.ArrowStreamReader;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index e44edbd47b67a..73165315ea556 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -32,7 +32,7 @@
 import org.apache.arrow.vector.BaseDataValueVector;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.util.OversizedAllocationException;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 38524ff8adf82..6d9eb1db03af5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -22,7 +22,7 @@
 import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.util.CallBack;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
index 209758e4ece9e..f82077f692f3d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
@@ -28,7 +28,7 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
index edf4987de5756..b9e5442ecf666 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
@@ -25,7 +25,7 @@
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.NullableMapVector;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index c6d404e15430c..26c817008836c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -24,7 +24,7 @@
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.BitHolder;
 import org.apache.arrow.vector.holders.NullableBitHolder;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.OversizedAllocationException;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
index 23252ca697b8d..2d4db85c583bf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
@@ -20,7 +20,7 @@
 
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 
 /**
  * Helper class for performing generic operations on a bit vector buffer.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
index a0dbf2bdcf101..332ca228a43fd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java
@@ -18,7 +18,7 @@
 
 package org.apache.arrow.vector;
 
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 
 import io.netty.buffer.ArrowBuf;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
index c2ed17eb4dd31..509eeda75000b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
@@ -20,7 +20,7 @@
 
 import java.util.List;
 
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 
 import io.netty.buffer.ArrowBuf;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 58fc80bbba17c..2cd4099c66975 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -24,9 +24,9 @@
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.schema.VectorLayout;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.VectorLayout;
 import org.apache.arrow.vector.types.pojo.Field;
 
 import com.google.common.collect.Iterators;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index fd9677312ed33..2b034894ab14d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -22,9 +22,9 @@
 import java.util.List;
 
 import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.ArrowVectorType;
 
 public class VectorUnloader {
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 3cc93a2a34f43..0ab3a7b6843c3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -28,7 +28,7 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.impl.NullReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.Field;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 6713b1c7871b5..774a10dbfb98f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -33,7 +33,7 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index afe86a692c3e1..d50d4c447fefd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -42,7 +42,7 @@
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index f95302f55f887..e223d1ce67449 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -34,7 +34,7 @@
 import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
 import org.apache.arrow.vector.complex.impl.NullableMapWriter;
 import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileReader.java
similarity index 77%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileReader.java
index d711b9c6c1e26..4cd70262261e8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileReader.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -26,32 +26,45 @@
 
 import org.apache.arrow.flatbuf.Footer;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
-import org.apache.arrow.vector.schema.ArrowMessage;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.stream.MessageSerializer;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.message.ArrowFooter;
+import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class ArrowFileReader extends ArrowReader<SeekableReadChannel> {
+public class ArrowFileReader extends ArrowReader {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(ArrowFileReader.class);
 
+  private SeekableReadChannel in;
   private ArrowFooter footer;
   private int currentDictionaryBatch = 0;
   private int currentRecordBatch = 0;
 
+  public ArrowFileReader(SeekableReadChannel in, BufferAllocator allocator) {
+    super(allocator);
+    this.in = in;
+  }
+
   public ArrowFileReader(SeekableByteChannel in, BufferAllocator allocator) {
-    super(new SeekableReadChannel(in), allocator);
+    this(new SeekableReadChannel(in), allocator);
   }
 
-  public ArrowFileReader(SeekableReadChannel in, BufferAllocator allocator) {
-    super(in, allocator);
+  @Override
+  public long bytesRead() {
+    return in.bytesRead();
+  }
+
+  @Override
+  protected void closeReadSource() throws IOException {
+    in.close();
   }
 
   @Override
-  protected Schema readSchema(SeekableReadChannel in) throws IOException {
+  protected Schema readSchema() throws IOException {
     if (footer == null) {
       if (in.size() <= (ArrowMagic.MAGIC_LENGTH * 2 + 4)) {
         throw new InvalidArrowFileException("file too small: " + in.size());
@@ -82,18 +95,30 @@ protected Schema readSchema(SeekableReadChannel in) throws IOException {
   }
 
   @Override
-  protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator allocator) throws IOException {
-    if (currentDictionaryBatch < footer.getDictionaries().size()) {
-      ArrowBlock block = footer.getDictionaries().get(currentDictionaryBatch++);
-      return readDictionaryBatch(in, block, allocator);
-    } else if (currentRecordBatch < footer.getRecordBatches().size()) {
+  public ArrowDictionaryBatch readDictionary() throws IOException {
+    if (currentDictionaryBatch >= footer.getDictionaries().size()) {
+      throw new IOException("Requested more dictionaries than defined in footer: " + currentDictionaryBatch);
+    }
+    ArrowBlock block = footer.getDictionaries().get(currentDictionaryBatch++);
+    return readDictionaryBatch(in, block, allocator);
+  }
+
+  // Returns true if a batch was read, false if no more batches
+  @Override
+  public boolean loadNextBatch() throws IOException {
+    prepareLoadNextBatch();
+
+    if (currentRecordBatch < footer.getRecordBatches().size()) {
       ArrowBlock block = footer.getRecordBatches().get(currentRecordBatch++);
-      return readRecordBatch(in, block, allocator);
+      ArrowRecordBatch batch = readRecordBatch(in, block, allocator);
+      loadRecordBatch(batch);
+      return true;
     } else {
-      return null;
+      return false;
     }
   }
 
+
   public List<ArrowBlock> getDictionaryBlocks() throws IOException {
     ensureInitialized();
     return footer.getDictionaries();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java
similarity index 94%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java
index 1d92d2bde1c6f..1b687c9f2697a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.io.IOException;
 import java.nio.channels.WritableByteChannel;
@@ -24,6 +24,8 @@
 
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.message.ArrowFooter;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java
similarity index 93%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java
index 68313e7878b71..a9310a608ae03 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java
@@ -16,7 +16,9 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
+
+import org.apache.arrow.vector.ipc.WriteChannel;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java
similarity index 65%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java
index 21fb2207eb019..6d708a03cadb0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -33,32 +33,25 @@
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
-import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
-import org.apache.arrow.vector.schema.ArrowMessage;
-import org.apache.arrow.vector.schema.ArrowMessage.ArrowMessageVisitor;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.DictionaryUtility;
 
 /**
- * Abstract class to read ArrowRecordBatches from a ReadChannel.
+ * Abstract class to read Schema and ArrowRecordBatches.
  *
- * @param <T> Type of ReadChannel to use
  */
-public abstract class ArrowReader<T extends ReadChannel> implements DictionaryProvider, AutoCloseable {
-
-  private final T in;
-  private final BufferAllocator allocator;
+public abstract class ArrowReader implements DictionaryProvider, AutoCloseable {
 
+  protected final BufferAllocator allocator;
   private VectorLoader loader;
   private VectorSchemaRoot root;
   private Map<Long, Dictionary> dictionaries;
-
   private boolean initialized = false;
 
-  protected ArrowReader(T in, BufferAllocator allocator) {
-    this.in = in;
+  protected ArrowReader(BufferAllocator allocator) {
     this.allocator = allocator;
   }
 
@@ -105,58 +98,18 @@ public Dictionary lookup(long id) {
    * @return true if a batch was read, false on EOS
    * @throws IOException
    */
-  public boolean loadNextBatch() throws IOException {
-    ensureInitialized();
-    // read in all dictionary batches, then stop after our first record batch
-    ArrowMessageVisitor<Boolean> visitor = new ArrowMessageVisitor<Boolean>() {
-      @Override
-      public Boolean visit(ArrowDictionaryBatch message) {
-        try {
-          load(message);
-        } finally {
-          message.close();
-        }
-        return true;
-      }
-
-      @Override
-      public Boolean visit(ArrowRecordBatch message) {
-        try {
-          loader.load(message);
-        } finally {
-          message.close();
-        }
-        return false;
-      }
-    };
-    root.setRowCount(0);
-    ArrowMessage message = readMessage(in, allocator);
-
-    boolean readBatch = false;
-    while (message != null) {
-      if (!message.accepts(visitor)) {
-        readBatch = true;
-        break;
-      }
-      // else read a dictionary
-      message = readMessage(in, allocator);
-    }
-
-    return readBatch;
-  }
+  public abstract boolean loadNextBatch() throws IOException;
 
   /**
    * Return the number of bytes read from the ReadChannel.
    *
    * @return number of bytes read
    */
-  public long bytesRead() {
-    return in.bytesRead();
-  }
+  public abstract long bytesRead();
 
   /**
    * Close resources, including vector schema root and dictionary vectors, and the
-   * underlying ReadChannel.
+   * underlying read source.
    *
    * @throws IOException
    */
@@ -167,12 +120,12 @@ public void close() throws IOException {
 
   /**
    * Close resources, including vector schema root and dictionary vectors. If the flag
-   * closeReadChannel is true then close the underlying ReadChannel, otherwise leave it open.
+   * closeReadChannel is true then close the underlying read source, otherwise leave it open.
    *
-   * @param closeReadChannel Flag to control if closing the underlying ReadChannel
+   * @param closeReadSource Flag to control if closing the underlying read source
    * @throws IOException
    */
-  public void close(boolean closeReadChannel) throws IOException {
+  public void close(boolean closeReadSource) throws IOException {
     if (initialized) {
       root.close();
       for (Dictionary dictionary : dictionaries.values()) {
@@ -180,15 +133,40 @@ public void close(boolean closeReadChannel) throws IOException {
       }
     }
 
-    if (closeReadChannel) {
-      in.close();
+    if (closeReadSource) {
+      closeReadSource();
     }
   }
 
-  protected abstract Schema readSchema(T in) throws IOException;
+  /**
+   * Close the underlying read source.
+   *
+   * @throws IOException
+   */
+  protected abstract void closeReadSource() throws IOException;
+
+  /**
+   * Read the Schema from the source, will be invoked at the beginning the initialization.
+   *
+   * @return the read Schema
+   * @throws IOException
+   */
+  protected abstract Schema readSchema() throws IOException;
 
-  protected abstract ArrowMessage readMessage(T in, BufferAllocator allocator) throws IOException;
+  /**
+   * Read a dictionary batch from the source, will be invoked after the schema has been read and
+   * called N times, where N is the number of dictionaries indicated by the schema Fields.
+   *
+   * @return the read ArrowDictionaryBatch
+   * @throws IOException
+   */
+  protected abstract ArrowDictionaryBatch readDictionary() throws IOException;
 
+  /**
+   * Initialize if not done previously.
+   *
+   * @throws IOException
+   */
   protected void ensureInitialized() throws IOException {
     if (!initialized) {
       initialize();
@@ -200,7 +178,7 @@ protected void ensureInitialized() throws IOException {
    * Reads the schema and initializes the vectors
    */
   private void initialize() throws IOException {
-    Schema originalSchema = readSchema(in);
+    Schema originalSchema = readSchema();
     List<Field> fields = new ArrayList<>();
     List<FieldVector> vectors = new ArrayList<>();
     Map<Long, Dictionary> dictionaries = new HashMap<>();
@@ -216,9 +194,43 @@ private void initialize() throws IOException {
     this.root = new VectorSchemaRoot(schema, vectors, 0);
     this.loader = new VectorLoader(root);
     this.dictionaries = Collections.unmodifiableMap(dictionaries);
+
+    // Read and load all dictionaries from schema
+    for (int i = 0; i < dictionaries.size(); i++) {
+      ArrowDictionaryBatch dictionaryBatch = readDictionary();
+      loadDictionary(dictionaryBatch);
+    }
   }
 
-  private void load(ArrowDictionaryBatch dictionaryBatch) {
+  /**
+   * Ensure the reader has been initialized and reset the VectorSchemaRoot row count to 0.
+   *
+   * @throws IOException
+   */
+  protected void prepareLoadNextBatch() throws IOException {
+    ensureInitialized();
+    root.setRowCount(0);
+  }
+
+  /**
+   * Load an ArrowRecordBatch to the readers VectorSchemaRoot.
+   *
+   * @param batch the record batch to load
+   */
+  protected void loadRecordBatch(ArrowRecordBatch batch) {
+    try {
+      loader.load(batch);
+    } finally {
+      batch.close();
+    }
+  }
+
+  /**
+   * Load an ArrowDictionaryBatch to the readers dictionary vectors.
+   *
+   * @param dictionaryBatch
+   */
+  protected void loadDictionary(ArrowDictionaryBatch dictionaryBatch) {
     long id = dictionaryBatch.getDictionaryId();
     Dictionary dictionary = dictionaries.get(id);
     if (dictionary == null) {
@@ -227,6 +239,10 @@ private void load(ArrowDictionaryBatch dictionaryBatch) {
     FieldVector vector = dictionary.getVector();
     VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), 0);
     VectorLoader loader = new VectorLoader(root);
-    loader.load(dictionaryBatch.getDictionary());
+    try {
+      loader.load(dictionaryBatch.getDictionary());
+    } finally {
+      dictionaryBatch.close();
+    }
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java
new file mode 100644
index 0000000000000..d1e48021885d5
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.ipc;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+
+import org.apache.arrow.flatbuf.Message;
+import org.apache.arrow.flatbuf.MessageHeader;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.ipc.ArrowReader;
+import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.MessageChannelReader;
+import org.apache.arrow.vector.ipc.message.MessageReader;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+/**
+ * This classes reads from an input stream and produces ArrowRecordBatches.
+ */
+public class ArrowStreamReader extends ArrowReader {
+
+  private MessageReader messageReader;
+
+  /**
+   * Constructs a streaming reader using the MessageReader interface. Non-blocking.
+   *
+   * @param messageReader interface to get read messages
+   * @param allocator to allocate new buffers
+   */
+  public ArrowStreamReader(MessageReader messageReader, BufferAllocator allocator) {
+    super(allocator);
+    this.messageReader = messageReader;
+  }
+
+  /**
+   * Constructs a streaming reader from a ReadableByteChannel input. Non-blocking.
+   *
+   * @param in ReadableByteChannel to read messages from
+   * @param allocator to allocate new buffers
+   */
+  public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) {
+    this(new MessageChannelReader(new ReadChannel(in)), allocator);
+  }
+
+  /**
+   * Constructs a streaming reader from an InputStream. Non-blocking.
+   *
+   * @param in InputStream to read messages from
+   * @param allocator to allocate new buffers
+   */
+  public ArrowStreamReader(InputStream in, BufferAllocator allocator) {
+    this(Channels.newChannel(in), allocator);
+  }
+
+  /**
+   * Get the number of bytes read from the stream since constructing the reader.
+   *
+   * @return number of bytes
+   */
+  @Override
+  public long bytesRead() {
+    return messageReader.bytesRead();
+  }
+
+  /**
+   * Closes the underlying read source.
+   *
+   * @throws IOException
+   */
+  @Override
+  protected void closeReadSource() throws IOException {
+    messageReader.close();
+  }
+
+  /**
+   * Load the next ArrowRecordBatch to the vector schema root if available.
+   *
+   * @return true if a batch was read, false on EOS
+   * @throws IOException
+   */
+  public boolean loadNextBatch() throws IOException {
+    prepareLoadNextBatch();
+
+    Message message = messageReader.readNextMessage();
+
+    // Reached EOS
+    if (message == null) {
+      return false;
+    }
+
+    if (message.headerType() != MessageHeader.RecordBatch) {
+      throw new IOException("Expected RecordBatch but header was " + message.headerType());
+    }
+
+    ArrowRecordBatch batch = MessageSerializer.deserializeRecordBatch(messageReader, message, allocator);
+    loadRecordBatch(batch);
+    return true;
+  }
+
+  /**
+   * Reads the schema message from the beginning of the stream.
+   *
+   * @return the deserialized arrow schema
+   */
+  @Override
+  protected Schema readSchema() throws IOException {
+    return MessageSerializer.deserializeSchema(messageReader);
+  }
+
+  /**
+   * Read a dictionary batch message, will be invoked after the schema and before normal record
+   * batches are read.
+   *
+   * @return the deserialized dictionary batch
+   * @throws IOException
+   */
+  @Override
+  protected ArrowDictionaryBatch readDictionary() throws IOException {
+    Message message = messageReader.readNextMessage();
+
+    if (message.headerType() != MessageHeader.DictionaryBatch) {
+      throw new IOException("Expected DictionaryBatch but header was " + message.headerType());
+    }
+
+    return MessageSerializer.deserializeDictionaryBatch(messageReader, message, allocator);
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java
similarity index 84%
rename from java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java
index b854cd2bb6e74..d731d05b81f16 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java
@@ -16,16 +16,13 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.stream;
+package org.apache.arrow.vector.ipc;
 
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ArrowWriter;
-import org.apache.arrow.vector.file.WriteChannel;
-import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.ArrowWriter;
+import org.apache.arrow.vector.ipc.WriteChannel;
 import org.apache.arrow.vector.types.pojo.Schema;
 
 import java.io.IOException;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
similarity index 95%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
index 7dc10b5e6299f..4b483d0105004 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.io.IOException;
 import java.nio.channels.WritableByteChannel;
@@ -30,9 +30,10 @@
 import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
-import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.stream.MessageSerializer;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.DictionaryUtility;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/InvalidArrowFileException.java
similarity index 96%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/InvalidArrowFileException.java
index 607207f41b06c..ad9d8776e33f4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/InvalidArrowFileException.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 public class InvalidArrowFileException extends RuntimeException {
   private static final long serialVersionUID = 1L;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
similarity index 98%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
index 8017b385d148a..cb11a253058e5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -16,19 +16,18 @@
  * limitations under the License.
  ******************************************************************************/
 
-package org.apache.arrow.vector.file.json;
+package org.apache.arrow.vector.ipc;
 
 import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
 import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
 import static com.fasterxml.jackson.core.JsonToken.START_ARRAY;
 import static com.fasterxml.jackson.core.JsonToken.START_OBJECT;
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.arrow.vector.schema.ArrowVectorType.*;
+import static org.apache.arrow.vector.ipc.message.ArrowVectorType.*;
 
 import java.io.File;
 import java.io.IOException;
 import java.math.BigDecimal;
-import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -42,9 +41,8 @@
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
-import org.apache.arrow.vector.file.InvalidArrowFileException;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowVectorType;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
similarity index 98%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
index 0c8507b51f598..22423b844e5db 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -16,9 +16,9 @@
  * limitations under the License.
  ******************************************************************************/
 
-package org.apache.arrow.vector.file.json;
+package org.apache.arrow.vector.ipc;
 
-import static org.apache.arrow.vector.schema.ArrowVectorType.*;
+import static org.apache.arrow.vector.ipc.message.ArrowVectorType.*;
 
 import java.io.File;
 import java.io.IOException;
@@ -33,7 +33,7 @@
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
-import org.apache.arrow.vector.schema.ArrowVectorType;
+import org.apache.arrow.vector.ipc.message.ArrowVectorType;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ReadChannel.java
similarity index 98%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ReadChannel.java
index b0eb8f3d84d9a..395fd7db5975f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ReadChannel.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/SeekableReadChannel.java
similarity index 97%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/SeekableReadChannel.java
index 46bea1314da63..62ba3b73e5377 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/SeekableReadChannel.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.io.IOException;
 import java.nio.channels.SeekableByteChannel;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java
similarity index 97%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java
index 89c9d1f9b7a44..da500aa97bee9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -25,7 +25,7 @@
 import com.google.flatbuffers.FlatBufferBuilder;
 
 import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.vector.schema.FBSerializable;
+import org.apache.arrow.vector.ipc.message.FBSerializable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java
similarity index 96%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java
index e1b4d6a8b215e..8731f77ac2c4c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java
@@ -16,10 +16,9 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc.message;
 
 import org.apache.arrow.flatbuf.Block;
-import org.apache.arrow.vector.schema.FBSerializable;
 
 import com.google.flatbuffers.FlatBufferBuilder;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java
similarity index 97%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java
index 4e0187e791b5a..6b0eeaad4d177 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
+package org.apache.arrow.vector.ipc.message;
 
 import org.apache.arrow.flatbuf.Buffer;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java
similarity index 97%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java
index 635fa3fb42307..cd23cb96b6bd7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
+package org.apache.arrow.vector.ipc.message;
 
 import com.google.flatbuffers.FlatBufferBuilder;
 import org.apache.arrow.flatbuf.DictionaryBatch;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFieldNode.java
similarity index 97%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFieldNode.java
index 3ed384ed7e280..ca0087f7089ce 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFieldNode.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
+package org.apache.arrow.vector.ipc.message;
 
 import org.apache.arrow.flatbuf.FieldNode;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java
similarity index 96%
rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java
index 1e95321fdec5b..f7794f7364bd3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java
@@ -16,16 +16,15 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc.message;
 
-import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector;
+import static org.apache.arrow.vector.ipc.message.FBSerializables.writeAllStructsToVector;
 
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.arrow.flatbuf.Block;
 import org.apache.arrow.flatbuf.Footer;
-import org.apache.arrow.vector.schema.FBSerializable;
 import org.apache.arrow.vector.types.pojo.Schema;
 
 import com.google.flatbuffers.FlatBufferBuilder;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowMessage.java
similarity index 96%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowMessage.java
index f59b4b6c1721e..92fb58e16fe6b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowMessage.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
+package org.apache.arrow.vector.ipc.message;
 
 public interface ArrowMessage extends FBSerializable, AutoCloseable {
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
similarity index 94%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
index bf0967a2797fe..6c6481e74dd93 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
@@ -16,9 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
-
-import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector;
+package org.apache.arrow.vector.ipc.message;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -113,9 +111,9 @@ public List<ArrowBuffer> getBuffersLayout() {
   @Override
   public int writeTo(FlatBufferBuilder builder) {
     RecordBatch.startNodesVector(builder, nodes.size());
-    int nodesOffset = writeAllStructsToVector(builder, nodes);
+    int nodesOffset = FBSerializables.writeAllStructsToVector(builder, nodes);
     RecordBatch.startBuffersVector(builder, buffers.size());
-    int buffersOffset = writeAllStructsToVector(builder, buffersLayout);
+    int buffersOffset = FBSerializables.writeAllStructsToVector(builder, buffersLayout);
     RecordBatch.startRecordBatch(builder);
     RecordBatch.addLength(builder, length);
     RecordBatch.addNodes(builder, nodesOffset);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java
similarity index 98%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java
index 9d2fdfaafe4aa..3342652bedd52 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
+package org.apache.arrow.vector.ipc.message;
 
 import java.util.Map;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializable.java
similarity index 95%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializable.java
index 91d60ea995b89..31f55bd522c68 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializable.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
+package org.apache.arrow.vector.ipc.message;
 
 import com.google.flatbuffers.FlatBufferBuilder;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializables.java
similarity index 96%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializables.java
index ae5aa555e745e..6717ed7ab313e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializables.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
+package org.apache.arrow.vector.ipc.message;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageChannelReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageChannelReader.java
new file mode 100644
index 0000000000000..5bc3e1fff6f96
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageChannelReader.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.ipc.message;
+
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.flatbuf.Message;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.ipc.ReadChannel;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Reads a sequence of messages using a ReadChannel.
+ */
+public class MessageChannelReader implements MessageReader {
+
+  private ReadChannel in;
+
+  /**
+   * Construct from an existing ReadChannel.
+   *
+   * @param in Channel to read messages from
+   */
+  public MessageChannelReader(ReadChannel in) {
+    this.in = in;
+  }
+
+  /**
+   * Read the next message from the ReadChannel.
+   *
+   * @return A Message or null if ReadChannel has no more messages, indicated by message length of 0
+   * @throws IOException
+   */
+  @Override
+  public Message readNextMessage() throws IOException {
+    // Read the message size. There is an i32 little endian prefix.
+    ByteBuffer buffer = ByteBuffer.allocate(4);
+    if (in.readFully(buffer) != 4) {
+      return null;
+    }
+    int messageLength = MessageSerializer.bytesToInt(buffer.array());
+    if (messageLength == 0) {
+      return null;
+    }
+
+    buffer = ByteBuffer.allocate(messageLength);
+    if (in.readFully(buffer) != messageLength) {
+      throw new IOException(
+          "Unexpected end of stream trying to read message.");
+    }
+    buffer.rewind();
+
+    return Message.getRootAsMessage(buffer);
+  }
+
+  /**
+   * Read a message body from the ReadChannel.
+   *
+   * @param message Read message that is followed by a body of data
+   * @param allocator BufferAllocator to allocate memory for body data
+   * @return ArrowBuf containing the message body data
+   * @throws IOException
+   */
+  @Override
+  public ArrowBuf readMessageBody(Message message, BufferAllocator allocator) throws IOException {
+
+    int bodyLength = (int) message.bodyLength();
+
+    // Now read the record batch body
+    ArrowBuf buffer = allocator.buffer(bodyLength);
+    if (in.readFully(buffer, bodyLength) != bodyLength) {
+      throw new IOException("Unexpected end of input trying to read batch.");
+    }
+
+    return buffer;
+  }
+
+  /**
+   * Get the number of bytes read from the ReadChannel.
+   *
+   * @return number of bytes
+   */
+  @Override
+  public long bytesRead() {
+    return in.bytesRead();
+  }
+
+  /**
+   * Close the ReadChannel.
+   *
+   * @throws IOException
+   */
+  @Override
+  public void close() throws IOException {
+    in.close();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageReader.java
new file mode 100644
index 0000000000000..b277c58295059
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageReader.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.ipc.message;
+
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.flatbuf.Message;
+import org.apache.arrow.memory.BufferAllocator;
+
+import java.io.IOException;
+
+/**
+ * Interface for reading a sequence of messages.
+ */
+public interface MessageReader {
+
+  /**
+   * Read the next message in the sequence.
+   *
+   * @return The read message or null if reached the end of the message sequence
+   * @throws IOException
+   */
+  Message readNextMessage() throws IOException;
+
+  /**
+   * When a message is followed by a body of data, read that data into an ArrowBuf. This should
+   * only be called when a Message has a body length > 0.
+   *
+   * @param message Read message that is followed by a body of data
+   * @param allocator BufferAllocator to allocate memory for body data
+   * @return An ArrowBuf containing the body of the message that was read
+   * @throws IOException
+   */
+  ArrowBuf readMessageBody(Message message, BufferAllocator allocator) throws IOException;
+
+  /**
+   * Return the current number of bytes that have been read.
+   *
+   * @return number of bytes read
+   */
+  long bytesRead();
+
+  /**
+   * Close any resource opened by the message reader, not including message body allocations.
+   *
+   * @throws IOException
+   */
+  void close() throws IOException;
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java
similarity index 86%
rename from java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java
index c397cec72f0ed..e2f8f7d9a8d7e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.stream;
+package org.apache.arrow.vector.ipc.message;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -31,14 +31,8 @@
 import org.apache.arrow.flatbuf.MetadataVersion;
 import org.apache.arrow.flatbuf.RecordBatch;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ReadChannel;
-import org.apache.arrow.vector.file.WriteChannel;
-import org.apache.arrow.vector.schema.ArrowBuffer;
-import org.apache.arrow.vector.schema.ArrowDictionaryBatch;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowMessage;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.WriteChannel;
 import org.apache.arrow.vector.types.pojo.Schema;
 
 import com.google.flatbuffers.FlatBufferBuilder;
@@ -102,12 +96,12 @@ public static long serialize(WriteChannel out, Schema schema) throws IOException
   /**
    * Deserializes a schema object. Format is from serialize().
    *
-   * @param in the channel to deserialize from
+   * @param reader the reader interface to deserialize from
    * @return the deserialized object
    * @throws IOException if something went wrong
    */
-  public static Schema deserializeSchema(ReadChannel in) throws IOException {
-    Message message = deserializeMessage(in);
+  public static Schema deserializeSchema(MessageReader reader) throws IOException {
+    Message message = reader.readNextMessage();
     if (message == null) {
       throw new IOException("Unexpected end of input. Missing schema.");
     }
@@ -119,6 +113,16 @@ public static Schema deserializeSchema(ReadChannel in) throws IOException {
         message.header(new org.apache.arrow.flatbuf.Schema()));
   }
 
+  /**
+   * Deserializes a schema object. Format is from serialize().
+   *
+   * @param in the channel to deserialize from
+   * @return the deserialized object
+   * @throws IOException if something went wrong
+   */
+  public static Schema deserializeSchema(ReadChannel in) throws IOException {
+    return deserializeSchema(new MessageChannelReader(in));
+  }
 
   /**
    * Serializes an ArrowRecordBatch. Returns the offset and length of the written batch.
@@ -184,25 +188,20 @@ public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) t
   }
 
   /**
-   * Deserializes a RecordBatch
+   * Deserializes a RecordBatch.
    *
-   * @param in      the channel to deserialize from
+   * @param reader  the reader interface to deserialize from
    * @param message the object to derialize to
    * @param alloc   to allocate buffers
    * @return the deserialized object
    * @throws IOException if something went wrong
    */
-  public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message message, BufferAllocator alloc)
+  public static ArrowRecordBatch deserializeRecordBatch(MessageReader reader, Message message, BufferAllocator alloc)
       throws IOException {
     RecordBatch recordBatchFB = (RecordBatch) message.header(new RecordBatch());
 
-    int bodyLength = (int) message.bodyLength();
-
     // Now read the record batch body
-    ArrowBuf buffer = alloc.buffer(bodyLength);
-    if (in.readFully(buffer, bodyLength) != bodyLength) {
-      throw new IOException("Unexpected end of input trying to read batch.");
-    }
+    ArrowBuf buffer = reader.readMessageBody(message, alloc);
     return deserializeRecordBatch(recordBatchFB, buffer);
   }
 
@@ -243,7 +242,14 @@ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock
     return deserializeRecordBatch(recordBatchFB, body);
   }
 
-  // Deserializes a record batch given the Flatbuffer metadata and in-memory body
+  /**
+   * Deserializes a record batch given the Flatbuffer metadata and in-memory body.
+   *
+   * @param recordBatchFB Deserialized FlatBuffer record batch
+   * @param body Read body of the record batch
+   * @return ArrowRecordBatch from metadata and in-memory body
+   * @throws IOException
+   */
   public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
                                                         ArrowBuf body) throws IOException {
     // Now read the body
@@ -314,26 +320,21 @@ public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch)
   }
 
   /**
-   * Deserializes a DictionaryBatch
+   * Deserializes a DictionaryBatch.
    *
-   * @param in      where to read from
+   * @param reader  where to read from
    * @param message the message message metadata to deserialize
    * @param alloc   the allocator for new buffers
    * @return the corresponding dictionary batch
    * @throws IOException if something went wrong
    */
-  public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
+  public static ArrowDictionaryBatch deserializeDictionaryBatch(MessageReader reader,
                                                                 Message message,
                                                                 BufferAllocator alloc) throws IOException {
     DictionaryBatch dictionaryBatchFB = (DictionaryBatch) message.header(new DictionaryBatch());
 
-    int bodyLength = (int) message.bodyLength();
-
     // Now read the record batch body
-    ArrowBuf body = alloc.buffer(bodyLength);
-    if (in.readFully(body, bodyLength) != bodyLength) {
-      throw new IOException("Unexpected end of input trying to read batch.");
-    }
+    ArrowBuf body = reader.readMessageBody(message, alloc);
     ArrowRecordBatch recordBatch = deserializeRecordBatch(dictionaryBatchFB.data(), body);
     return new ArrowDictionaryBatch(dictionaryBatchFB.id(), recordBatch);
   }
@@ -377,8 +378,16 @@ public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
     return new ArrowDictionaryBatch(dictionaryBatchFB.id(), recordBatch);
   }
 
-  public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocator alloc) throws IOException {
-    Message message = deserializeMessage(in);
+  /**
+   * Deserialize a message that is either an ArrowDictionaryBatch or ArrowRecordBatch.
+   *
+   * @param reader Interface to read messages from
+   * @param alloc Allocator for message data
+   * @return The deserialized record batch
+   * @throws IOException if the message is not an ArrowDictionaryBatch or ArrowRecordBatch
+   */
+  public static ArrowMessage deserializeMessageBatch(MessageReader reader, BufferAllocator alloc) throws IOException {
+    Message message = reader.readNextMessage();
     if (message == null) {
       return null;
     } else if (message.bodyLength() > Integer.MAX_VALUE) {
@@ -391,14 +400,26 @@ public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocat
 
     switch (message.headerType()) {
       case MessageHeader.RecordBatch:
-        return deserializeRecordBatch(in, message, alloc);
+        return deserializeRecordBatch(reader, message, alloc);
       case MessageHeader.DictionaryBatch:
-        return deserializeDictionaryBatch(in, message, alloc);
+        return deserializeDictionaryBatch(reader, message, alloc);
       default:
         throw new IOException("Unexpected message header type " + message.headerType());
     }
   }
 
+  /**
+   * Deserialize a message that is either an ArrowDictionaryBatch or ArrowRecordBatch.
+   *
+   * @param in ReadChannel to read messages from
+   * @param alloc Allocator for message data
+   * @return The deserialized record batch
+   * @throws IOException if the message is not an ArrowDictionaryBatch or ArrowRecordBatch
+   */
+  public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocator alloc) throws IOException {
+    return deserializeMessageBatch(new MessageChannelReader(in), alloc);
+  }
+
   /**
    * Serializes a message header.
    *
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java
similarity index 80%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java
index 29407bf1ab4e1..06fe9481686d7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java
@@ -16,15 +16,9 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
+package org.apache.arrow.vector.ipc.message;
 
 import static java.util.Arrays.asList;
-import static org.apache.arrow.vector.schema.VectorLayout.booleanVector;
-import static org.apache.arrow.vector.schema.VectorLayout.byteVector;
-import static org.apache.arrow.vector.schema.VectorLayout.dataVector;
-import static org.apache.arrow.vector.schema.VectorLayout.offsetVector;
-import static org.apache.arrow.vector.schema.VectorLayout.typeVector;
-import static org.apache.arrow.vector.schema.VectorLayout.validityVector;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -64,7 +58,7 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
 
       @Override
       public TypeLayout visit(Int type) {
-        return newFixedWidthTypeLayout(dataVector(type.getBitWidth()));
+        return newFixedWidthTypeLayout(VectorLayout.dataVector(type.getBitWidth()));
       }
 
       @Override
@@ -74,14 +68,14 @@ public TypeLayout visit(Union type) {
           case Dense:
             vectors = asList(
                 // TODO: validate this
-                validityVector(),
-                typeVector(),
-                offsetVector() // offset to find the vector
+                VectorLayout.validityVector(),
+                VectorLayout.typeVector(),
+                VectorLayout.offsetVector() // offset to find the vector
             );
             break;
           case Sparse:
             vectors = asList(
-                typeVector() // type of the value at the index or 0 if null
+                VectorLayout.typeVector() // type of the value at the index or 0 if null
             );
             break;
           default:
@@ -93,21 +87,21 @@ public TypeLayout visit(Union type) {
       @Override
       public TypeLayout visit(Struct type) {
         List<VectorLayout> vectors = asList(
-            validityVector()
+            VectorLayout.validityVector()
         );
         return new TypeLayout(vectors);
       }
 
       @Override
       public TypeLayout visit(Timestamp type) {
-        return newFixedWidthTypeLayout(dataVector(64));
+        return newFixedWidthTypeLayout(VectorLayout.dataVector(64));
       }
 
       @Override
       public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) {
         List<VectorLayout> vectors = asList(
-            validityVector(),
-            offsetVector()
+            VectorLayout.validityVector(),
+            VectorLayout.offsetVector()
         );
         return new TypeLayout(vectors);
       }
@@ -115,7 +109,7 @@ public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type)
       @Override
       public TypeLayout visit(FixedSizeList type) {
         List<VectorLayout> vectors = asList(
-            validityVector()
+            VectorLayout.validityVector()
         );
         return new TypeLayout(vectors);
       }
@@ -136,18 +130,18 @@ public TypeLayout visit(FloatingPoint type) {
           default:
             throw new UnsupportedOperationException("Unsupported Precision: " + type.getPrecision());
         }
-        return newFixedWidthTypeLayout(dataVector(bitWidth));
+        return newFixedWidthTypeLayout(VectorLayout.dataVector(bitWidth));
       }
 
       @Override
       public TypeLayout visit(Decimal type) {
         // TODO: check size
-        return newFixedWidthTypeLayout(dataVector(64)); // actually depends on the type fields
+        return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); // actually depends on the type fields
       }
 
       @Override
       public TypeLayout visit(Bool type) {
-        return newFixedWidthTypeLayout(booleanVector());
+        return newFixedWidthTypeLayout(VectorLayout.booleanVector());
       }
 
       @Override
@@ -161,7 +155,7 @@ public TypeLayout visit(Utf8 type) {
       }
 
       private TypeLayout newVariableWidthTypeLayout() {
-        return newPrimitiveTypeLayout(validityVector(), offsetVector(), byteVector());
+        return newPrimitiveTypeLayout(VectorLayout.validityVector(), VectorLayout.offsetVector(), VectorLayout.byteVector());
       }
 
       private TypeLayout newPrimitiveTypeLayout(VectorLayout... vectors) {
@@ -169,7 +163,7 @@ private TypeLayout newPrimitiveTypeLayout(VectorLayout... vectors) {
       }
 
       public TypeLayout newFixedWidthTypeLayout(VectorLayout dataVector) {
-        return newPrimitiveTypeLayout(validityVector(), dataVector);
+        return newPrimitiveTypeLayout(VectorLayout.validityVector(), dataVector);
       }
 
       @Override
@@ -179,21 +173,21 @@ public TypeLayout visit(Null type) {
 
       @Override
       public TypeLayout visit(Date type) {
-        return newFixedWidthTypeLayout(dataVector(64));
+        return newFixedWidthTypeLayout(VectorLayout.dataVector(64));
       }
 
       @Override
       public TypeLayout visit(Time type) {
-        return newFixedWidthTypeLayout(dataVector(type.getBitWidth()));
+        return newFixedWidthTypeLayout(VectorLayout.dataVector(type.getBitWidth()));
       }
 
       @Override
       public TypeLayout visit(Interval type) { // TODO: check size
         switch (type.getUnit()) {
           case DAY_TIME:
-            return newFixedWidthTypeLayout(dataVector(64));
+            return newFixedWidthTypeLayout(VectorLayout.dataVector(64));
           case YEAR_MONTH:
-            return newFixedWidthTypeLayout(dataVector(64));
+            return newFixedWidthTypeLayout(VectorLayout.dataVector(64));
           default:
             throw new UnsupportedOperationException("Unknown unit " + type.getUnit());
         }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java
similarity index 89%
rename from java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java
index 0871baf38edaa..e4f2f98fde37a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java
@@ -16,12 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.schema;
-
-import static org.apache.arrow.vector.schema.ArrowVectorType.DATA;
-import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET;
-import static org.apache.arrow.vector.schema.ArrowVectorType.TYPE;
-import static org.apache.arrow.vector.schema.ArrowVectorType.VALIDITY;
+package org.apache.arrow.vector.ipc.message;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -30,14 +25,14 @@
 
 public class VectorLayout implements FBSerializable {
 
-  private static final VectorLayout VALIDITY_VECTOR = new VectorLayout(VALIDITY, 1);
-  private static final VectorLayout OFFSET_VECTOR = new VectorLayout(OFFSET, 32);
-  private static final VectorLayout TYPE_VECTOR = new VectorLayout(TYPE, 32);
-  private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(DATA, 1);
-  private static final VectorLayout VALUES_64 = new VectorLayout(DATA, 64);
-  private static final VectorLayout VALUES_32 = new VectorLayout(DATA, 32);
-  private static final VectorLayout VALUES_16 = new VectorLayout(DATA, 16);
-  private static final VectorLayout VALUES_8 = new VectorLayout(DATA, 8);
+  private static final VectorLayout VALIDITY_VECTOR = new VectorLayout(ArrowVectorType.VALIDITY, 1);
+  private static final VectorLayout OFFSET_VECTOR = new VectorLayout(ArrowVectorType.OFFSET, 32);
+  private static final VectorLayout TYPE_VECTOR = new VectorLayout(ArrowVectorType.TYPE, 32);
+  private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(ArrowVectorType.DATA, 1);
+  private static final VectorLayout VALUES_64 = new VectorLayout(ArrowVectorType.DATA, 64);
+  private static final VectorLayout VALUES_32 = new VectorLayout(ArrowVectorType.DATA, 32);
+  private static final VectorLayout VALUES_16 = new VectorLayout(ArrowVectorType.DATA, 16);
+  private static final VectorLayout VALUES_8 = new VectorLayout(ArrowVectorType.DATA, 8);
 
   public static VectorLayout typeVector() {
     return TYPE_VECTOR;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
deleted file mode 100644
index 5b6300076b6c2..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.stream;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.file.ArrowReader;
-import org.apache.arrow.vector.file.ReadChannel;
-import org.apache.arrow.vector.schema.ArrowMessage;
-import org.apache.arrow.vector.types.pojo.Schema;
-
-/**
- * This classes reads from an input stream and produces ArrowRecordBatches.
- */
-public class ArrowStreamReader extends ArrowReader<ReadChannel> {
-
-  /**
-   * Constructs a streaming read, reading bytes from 'in'. Non-blocking.
-   *
-   * @param in        the stream to read from
-   * @param allocator to allocate new buffers
-   */
-  public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) {
-    super(new ReadChannel(in), allocator);
-  }
-
-  public ArrowStreamReader(InputStream in, BufferAllocator allocator) {
-    this(Channels.newChannel(in), allocator);
-  }
-
-  /**
-   * Reads the schema message from the beginning of the stream.
-   *
-   * @param in to allocate new buffers
-   * @return the deserialized arrow schema
-   */
-  @Override
-  protected Schema readSchema(ReadChannel in) throws IOException {
-    return MessageSerializer.deserializeSchema(in);
-  }
-
-  @Override
-  protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
-    return MessageSerializer.deserializeMessageBatch(in, allocator);
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index eba149bf79f65..574612833cd19 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -43,8 +43,8 @@
 import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.schema.TypeLayout;
-import org.apache.arrow.vector.schema.VectorLayout;
+import org.apache.arrow.vector.ipc.message.VectorLayout;
+import org.apache.arrow.vector.ipc.message.TypeLayout;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 
 public class Field {
@@ -117,9 +117,9 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
       }
       dictionary = new DictionaryEncoding(dictionaryFB.id(), dictionaryFB.isOrdered(), indexType);
     }
-    ImmutableList.Builder<org.apache.arrow.vector.schema.VectorLayout> layout = ImmutableList.builder();
+    ImmutableList.Builder<VectorLayout> layout = ImmutableList.builder();
     for (int i = 0; i < field.layoutLength(); ++i) {
-      layout.add(new org.apache.arrow.vector.schema.VectorLayout(field.layout(i)));
+      layout.add(new VectorLayout(field.layout(i)));
     }
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     for (int i = 0; i < field.childrenLength(); i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index c7ee202f94661..f51a874367622 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -17,7 +17,6 @@
  */
 
 package org.apache.arrow.vector;
-import org.apache.arrow.vector.holders.VarCharHolder;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 
 import static org.apache.arrow.vector.TestUtils.newNullableVarBinaryVector;
@@ -38,15 +37,14 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.schema.TypeLayout;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.TypeLayout;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.TransferPair;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 3853eecac0552..e61dbecf44101 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -39,8 +39,8 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
similarity index 99%
rename from java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index 874ba99e20f29..233b682c99bc6 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java
similarity index 94%
rename from java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java
index f968768f5e67d..239d3034ad135 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.stream;
+package org.apache.arrow.vector.ipc;
 
 import static java.util.Arrays.asList;
 import static org.junit.Assert.assertArrayEquals;
@@ -33,12 +33,11 @@
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.file.ArrowBlock;
-import org.apache.arrow.vector.file.ReadChannel;
-import org.apache.arrow.vector.file.WriteChannel;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowMessage;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowMessage;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
similarity index 94%
rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
index 8559969a20424..4387db03655d1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -40,16 +40,14 @@
 import org.apache.arrow.vector.NullableIntVector;
 import org.apache.arrow.vector.NullableTinyIntVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
-import org.apache.arrow.vector.schema.ArrowBuffer;
-import org.apache.arrow.vector.schema.ArrowMessage;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.stream.ArrowStreamReader;
-import org.apache.arrow.vector.stream.ArrowStreamWriter;
-import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.message.ArrowBuffer;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -108,52 +106,41 @@ public void testWriteRead() throws IOException {
     // read
     try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
          FileInputStream fileInputStream = new FileInputStream(file);
-         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator) {
-           @Override
-           protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator allocator) throws IOException {
-             ArrowMessage message = super.readMessage(in, allocator);
-             if (message != null) {
-               ArrowRecordBatch batch = (ArrowRecordBatch) message;
-               List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
-               for (ArrowBuffer arrowBuffer : buffersLayout) {
-                 Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
-               }
-             }
-             return message;
-           }
-         }) {
-      Schema schema = arrowReader.getVectorSchemaRoot().getSchema();
-      LOGGER.debug("reading schema: " + schema);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      VectorUnloader unloader = new VectorUnloader(root);
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
       for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
         arrowReader.loadRecordBatch(rbBlock);
         Assert.assertEquals(count, root.getRowCount());
+        ArrowRecordBatch batch = unloader.getRecordBatch();
+        List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
+        for (ArrowBuffer arrowBuffer : buffersLayout) {
+          Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+        }
         validateContent(count, root);
+        batch.close();
       }
     }
 
     // Read from stream.
     try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
          ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
-         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator) {
-           @Override
-           protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
-             ArrowMessage message = super.readMessage(in, allocator);
-             if (message != null) {
-               ArrowRecordBatch batch = (ArrowRecordBatch) message;
-               List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
-               for (ArrowBuffer arrowBuffer : buffersLayout) {
-                 Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
-               }
-             }
-             return message;
-           }
-         }) {
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
 
       VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      VectorUnloader unloader = new VectorUnloader(root);
       Schema schema = root.getSchema();
       LOGGER.debug("reading schema: " + schema);
       Assert.assertTrue(arrowReader.loadNextBatch());
+      ArrowRecordBatch batch = unloader.getRecordBatch();
+      List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
+      for (ArrowBuffer arrowBuffer : buffersLayout) {
+        Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
+      }
+      batch.close();
       Assert.assertEquals(count, root.getRowCount());
       validateContent(count, root);
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java
similarity index 93%
rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java
index 4612465323130..235e8c1646712 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import static java.util.Arrays.asList;
 import static org.junit.Assert.assertEquals;
@@ -27,6 +27,8 @@
 import java.util.List;
 
 import org.apache.arrow.flatbuf.Footer;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.message.ArrowFooter;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
similarity index 85%
rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
index 3ce01a26835d5..49e194b51b823 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import static java.nio.channels.Channels.newChannel;
 import static java.util.Arrays.asList;
@@ -37,9 +37,15 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.TestUtils;
+import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.ipc.ArrowFileReader;
+import org.apache.arrow.vector.ipc.ArrowFileWriter;
+import org.apache.arrow.vector.ipc.SeekableReadChannel;
+import org.apache.arrow.vector.ipc.message.ArrowBlock;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -87,7 +93,10 @@ public void test() throws IOException {
          ArrowFileWriter writer = new ArrowFileWriter(root, null, newChannel(out))) {
       ArrowBuf validityb = buf(validity);
       ArrowBuf valuesb = buf(values);
-      writer.writeRecordBatch(new ArrowRecordBatch(16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)));
+      ArrowRecordBatch batch = new ArrowRecordBatch(16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb));
+      VectorLoader loader = new VectorLoader(root);
+      loader.load(batch);
+      writer.writeBatch();
     }
 
     byte[] byteArray = out.toByteArray();
@@ -100,7 +109,9 @@ public void test() throws IOException {
       // TODO: dictionaries
       List<ArrowBlock> recordBatches = reader.getRecordBlocks();
       assertEquals(1, recordBatches.size());
-      ArrowRecordBatch recordBatch = (ArrowRecordBatch) reader.readMessage(channel, allocator);
+      reader.loadNextBatch();
+      VectorUnloader unloader = new VectorUnloader(reader.getVectorSchemaRoot());
+      ArrowRecordBatch recordBatch = unloader.getRecordBatch();
       List<ArrowFieldNode> nodes = recordBatch.getNodes();
       assertEquals(1, nodes.size());
       ArrowFieldNode node = nodes.get(0);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
similarity index 88%
rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
index c7e34191a3f1a..7a8586a9e708e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
@@ -16,9 +16,8 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
-import static java.util.Arrays.asList;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -27,16 +26,12 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 
-import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.NullableTinyIntVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.schema.ArrowFieldNode;
-import org.apache.arrow.vector.schema.ArrowMessage;
-import org.apache.arrow.vector.schema.ArrowRecordBatch;
-import org.apache.arrow.vector.stream.ArrowStreamReader;
-import org.apache.arrow.vector.stream.ArrowStreamWriter;
-import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.ipc.ArrowStreamReader;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.ipc.BaseFileTest;
+import org.apache.arrow.vector.ipc.MessageSerializerTest;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Assert;
 import org.junit.Test;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
similarity index 78%
rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
index f393733cc9021..65e6cea2ec2e6 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file;
+package org.apache.arrow.vector.ipc;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -30,10 +30,9 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.NullableTinyIntVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.schema.ArrowMessage;
-import org.apache.arrow.vector.stream.ArrowStreamReader;
-import org.apache.arrow.vector.stream.ArrowStreamWriter;
-import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.ipc.ArrowStreamReader;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.ipc.MessageSerializerTest;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Assert;
 import org.junit.Test;
@@ -95,37 +94,28 @@ private final class ReaderThread extends Thread {
     public ReaderThread(ReadableByteChannel sourceChannel)
         throws IOException {
       reader = new ArrowStreamReader(sourceChannel, alloc) {
-        @Override
-        protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException {
-          // Read all the batches. Each batch contains an incrementing id and then some
-          // constant data. Verify both.
-          ArrowMessage message = super.readMessage(in, allocator);
-          if (message == null) {
-            done = true;
-          } else {
-            batchesRead++;
-          }
-          return message;
-        }
 
         @Override
         public boolean loadNextBatch() throws IOException {
-          if (!super.loadNextBatch()) {
+          if (super.loadNextBatch()) {
+            batchesRead++;
+          } else {
+            done = true;
             return false;
           }
-          if (!done) {
-            VectorSchemaRoot root = getVectorSchemaRoot();
-            Assert.assertEquals(16, root.getRowCount());
-            NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
-            Assert.assertEquals((byte) (batchesRead - 1), vector.get(0));
-            for (int i = 1; i < 16; i++) {
-              if (i < 8) {
-                Assert.assertEquals((byte) (i + 1), vector.get(i));
-              } else {
-                Assert.assertTrue(vector.isNull(i));
-              }
+
+          VectorSchemaRoot root = getVectorSchemaRoot();
+          Assert.assertEquals(16, root.getRowCount());
+          NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
+          Assert.assertEquals((byte) (batchesRead - 1), vector.get(0));
+          for (int i = 1; i < 16; i++) {
+            if (i < 8) {
+              Assert.assertEquals((byte) (i + 1), vector.get(i));
+            } else {
+              Assert.assertTrue(vector.isNull(i));
             }
           }
+
           return true;
         }
       };
@@ -139,7 +129,7 @@ public void run() {
             reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectorTypes().toString(),
             reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectors().size() > 0);
         while (!done) {
-          assertTrue(reader.loadNextBatch());
+          assertTrue(reader.loadNextBatch() != done);
         }
         reader.close();
       } catch (IOException e) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
similarity index 99%
rename from java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
index 5c4c48cd26b60..c3e0b795174c0 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.file.json;
+package org.apache.arrow.vector.ipc;
 
 import java.io.File;
 import java.io.IOException;
@@ -28,7 +28,6 @@
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
-import org.apache.arrow.vector.file.BaseFileTest;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.util.Validator;
 import org.junit.Assert;

From 9b2dc77a4d95c7415edd5be087a5abafc5a7f64c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 22 Nov 2017 17:22:19 -0500
Subject: [PATCH 1274/1644] =?UTF-8?q?ARROW-1845:=20[Python]=C2=A0Expose=20?=
 =?UTF-8?q?Decimal128Type?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1348 from xhochy/ARROW-1845 and squashes the following commits:

98514320 [Wes McKinney] Fix API page in Python docs for decimal->decimal128
5fade1f5 [Uwe L. Korn] ARROW-1845: [Python] Expose Decimal128Type
---
 python/doc/source/api.rst                    |  2 +-
 python/pyarrow/__init__.py                   |  2 +-
 python/pyarrow/includes/libarrow.pxd         |  4 ++--
 python/pyarrow/lib.pxd                       |  4 ++--
 python/pyarrow/pandas_compat.py              |  2 +-
 python/pyarrow/public-api.pxi                |  2 +-
 python/pyarrow/tests/test_array.py           |  2 +-
 python/pyarrow/tests/test_convert_builtin.py | 12 ++++++------
 python/pyarrow/tests/test_convert_pandas.py  |  6 +++---
 python/pyarrow/tests/test_schema.py          |  2 +-
 python/pyarrow/tests/test_types.py           |  2 +-
 python/pyarrow/types.pxi                     | 18 +++++++++---------
 12 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 8f2f23d9fd470..fb2a286770e96 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -50,7 +50,7 @@ Type and Schema Factory Functions
    date64
    binary
    string
-   decimal
+   decimal128
    list_
    struct
    dictionary
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index c8ded2d3cb88d..c4db36e55a0d6 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -35,7 +35,7 @@
                          uint8, uint16, uint32, uint64,
                          time32, time64, timestamp, date32, date64,
                          float16, float32, float64,
-                         binary, string, decimal,
+                         binary, string, decimal128,
                          list_, struct, union, dictionary, field,
                          type_for_alias,
                          DataType, NAType,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 73e34c7b21033..f1f59384bd639 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -209,10 +209,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int byte_width()
         int bit_width()
 
-    cdef cppclass CDecimalType" arrow::DecimalType"(CFixedSizeBinaryType):
+    cdef cppclass CDecimal128Type" arrow::Decimal128Type"(CFixedSizeBinaryType):
+        CDecimal128Type(int precision, int scale)
         int precision()
         int scale()
-        CDecimalType(int precision, int scale)
 
     cdef cppclass CField" arrow::Field":
         const c_string& name()
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 6413b838f0595..5abb72ba48165 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -81,9 +81,9 @@ cdef class FixedSizeBinaryType(DataType):
         const CFixedSizeBinaryType* fixed_size_binary_type
 
 
-cdef class DecimalType(FixedSizeBinaryType):
+cdef class Decimal128Type(FixedSizeBinaryType):
     cdef:
-        const CDecimalType* decimal_type
+        const CDecimal128Type* decimal128_type
 
 
 cdef class Field:
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 0aab9a41b8025..a50ef96e79f59 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -80,7 +80,7 @@ def get_logical_type(arrow_type):
             return 'list[{}]'.format(get_logical_type(arrow_type.value_type))
         elif isinstance(arrow_type, pa.lib.TimestampType):
             return 'datetimetz' if arrow_type.tz is not None else 'datetime'
-        elif isinstance(arrow_type, pa.lib.DecimalType):
+        elif isinstance(arrow_type, pa.lib.Decimal128Type):
             return 'decimal'
         raise NotImplementedError(str(arrow_type))
 
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 90aff9e936d95..bf670c5c4e290 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -78,7 +78,7 @@ cdef public api object pyarrow_wrap_data_type(
     elif type.get().id() == _Type_FIXED_SIZE_BINARY:
         out = FixedSizeBinaryType()
     elif type.get().id() == _Type_DECIMAL:
-        out = DecimalType()
+        out = Decimal128Type()
     else:
         out = DataType()
 
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index b7b0b18333677..a4d781a3354b0 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -470,7 +470,7 @@ def test_simple_type_construction():
         (pa.binary(length=4), 'bytes'),
         (pa.string(), 'unicode'),
         (pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
-        (pa.decimal(18, 3), 'decimal'),
+        (pa.decimal128(18, 3), 'decimal'),
         (pa.timestamp('ms'), 'datetime'),
         (pa.timestamp('us', 'UTC'), 'datetimetz'),
         (pa.time32('s'), 'time'),
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index c7a0d49b40db1..4c3d9e5631ce3 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -314,7 +314,7 @@ def test_mixed_types_fails(self):
 
     def test_decimal(self):
         data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
-        type = pa.decimal(precision=7, scale=3)
+        type = pa.decimal128(precision=7, scale=3)
         arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
@@ -322,32 +322,32 @@ def test_decimal_different_precisions(self):
         data = [
             decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
         ]
-        type = pa.decimal(precision=13, scale=3)
+        type = pa.decimal128(precision=13, scale=3)
         arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_no_scale(self):
         data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
-        type = pa.decimal(precision=10)
+        type = pa.decimal128(precision=10)
         arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_negative(self):
         data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
-        type = pa.decimal(precision=10, scale=6)
+        type = pa.decimal128(precision=10, scale=6)
         arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_no_whole_part(self):
         data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
-        type = pa.decimal(precision=7, scale=7)
+        type = pa.decimal128(precision=7, scale=7)
         arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
     def test_decimal_large_integer(self):
         data = [decimal.Decimal('-394029506937548693.42983'),
                 decimal.Decimal('32358695912932.01033')]
-        type = pa.decimal(precision=23, scale=5)
+        type = pa.decimal128(precision=23, scale=5)
         arr = pa.array(data, type=type)
         assert arr.to_pylist() == data
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index b9c3a12213243..a2a1d6b6910fa 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -828,7 +828,7 @@ def test_decimal_32_from_pandas(self):
             ]
         })
         converted = pa.Table.from_pandas(expected, preserve_index=False)
-        field = pa.field('decimals', pa.decimal(7, 3))
+        field = pa.field('decimals', pa.decimal128(7, 3))
         schema = pa.schema([field])
         assert converted.schema.equals(schema)
 
@@ -851,7 +851,7 @@ def test_decimal_64_from_pandas(self):
             ]
         })
         converted = pa.Table.from_pandas(expected, preserve_index=False)
-        field = pa.field('decimals', pa.decimal(12, 6))
+        field = pa.field('decimals', pa.decimal128(12, 6))
         schema = pa.schema([field])
         assert converted.schema.equals(schema)
 
@@ -874,7 +874,7 @@ def test_decimal_128_from_pandas(self):
             ]
         })
         converted = pa.Table.from_pandas(expected, preserve_index=False)
-        field = pa.field('decimals', pa.decimal(26, 11))
+        field = pa.field('decimals', pa.decimal128(26, 11))
         schema = pa.schema([field])
         assert converted.schema.equals(schema)
 
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 116f3978333a8..dbca139e20570 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -333,7 +333,7 @@ def test_type_schema_pickling():
         pa.date64(),
         pa.timestamp('ms'),
         pa.timestamp('ns'),
-        pa.decimal(12, 2),
+        pa.decimal128(12, 2),
         pa.field('a', 'string', metadata={b'foo': b'bar'})
     ]
 
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index 9eefa33b66187..e352e35a3a3a0 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -56,7 +56,7 @@ def test_is_null():
 
 
 def test_is_decimal():
-    assert types.is_decimal(pa.decimal(19, 4))
+    assert types.is_decimal(pa.decimal128(19, 4))
     assert not types.is_decimal(pa.int32())
 
 
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index abd678bac6e58..1563b57855cd9 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -290,28 +290,28 @@ cdef class FixedSizeBinaryType(DataType):
             return self.fixed_size_binary_type.byte_width()
 
 
-cdef class DecimalType(FixedSizeBinaryType):
+cdef class Decimal128Type(FixedSizeBinaryType):
 
     cdef void init(self, const shared_ptr[CDataType]& type):
         DataType.init(self, type)
-        self.decimal_type = <const CDecimalType*> type.get()
+        self.decimal128_type = <const CDecimal128Type*> type.get()
 
     def __getstate__(self):
         return (self.precision, self.scale)
 
     def __setstate__(self, state):
-        cdef DataType reconstituted = decimal(*state)
+        cdef DataType reconstituted = decimal128(*state)
         self.init(reconstituted.sp_type)
 
     property precision:
 
         def __get__(self):
-            return self.decimal_type.precision()
+            return self.decimal128_type.precision()
 
     property scale:
 
         def __get__(self):
-            return self.decimal_type.scale()
+            return self.decimal128_type.scale()
 
 
 cdef class Field:
@@ -953,9 +953,9 @@ def float64():
     return primitive_type(_Type_DOUBLE)
 
 
-cpdef DataType decimal(int precision, int scale=0):
+cpdef DataType decimal128(int precision, int scale=0):
     """
-    Create decimal type with precision and scale
+    Create decimal type with precision and scale and 128bit width
 
     Parameters
     ----------
@@ -964,10 +964,10 @@ cpdef DataType decimal(int precision, int scale=0):
 
     Returns
     -------
-    decimal_type : DecimalType
+    decimal_type : Decimal128Type
     """
     cdef shared_ptr[CDataType] decimal_type
-    decimal_type.reset(new CDecimalType(precision, scale))
+    decimal_type.reset(new CDecimal128Type(precision, scale))
     return pyarrow_wrap_data_type(decimal_type)
 
 

From dda2d34c712a6e64bcfeafe2b1e764ba8cd017e3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 23 Nov 2017 09:41:46 -0500
Subject: [PATCH 1275/1644] ARROW-1828: [C++] Hash kernel specialization for
 BooleanType

This is a bit tedious because we want to preserve the order in which the unique values were observed.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1350 from wesm/ARROW-1828 and squashes the following commits:

576ab330 [Wes McKinney] Fix typo
498a9092 [Wes McKinney] clang-format, fix Python flakes
c6a2b8fe [Wes McKinney] Add tests without nulls
b6cd4db6 [Wes McKinney] Finish boolean hash kernel implementation, tests passing
e4d4db6f [Wes McKinney] Scaffolding
---
 cpp/src/arrow/compute/compute-test.cc         | 34 ++++++++
 cpp/src/arrow/compute/kernels/hash.cc         | 77 ++++++++++++++++++-
 cpp/src/arrow/compute/kernels/util-internal.h |  4 +
 python/pyarrow/includes/libarrow.pxd          |  3 +-
 4 files changed, 115 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index fa408ae40cd85..96edd8f01ed38 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -843,6 +843,40 @@ TEST_F(TestHashKernel, UniqueTimeTimestamp) {
                                       {});
 }
 
+TEST_F(TestHashKernel, UniqueBoolean) {
+  CheckUnique<BooleanType, bool>(&this->ctx_, boolean(), {true, true, false, true},
+                                 {true, false, true, true}, {true, false}, {});
+
+  CheckUnique<BooleanType, bool>(&this->ctx_, boolean(), {false, true, false, true},
+                                 {true, false, true, true}, {false, true}, {});
+
+  // No nulls
+  CheckUnique<BooleanType, bool>(&this->ctx_, boolean(), {true, true, false, true}, {},
+                                 {true, false}, {});
+
+  CheckUnique<BooleanType, bool>(&this->ctx_, boolean(), {false, true, false, true}, {},
+                                 {false, true}, {});
+}
+
+TEST_F(TestHashKernel, DictEncodeBoolean) {
+  CheckDictEncode<BooleanType, bool>(
+      &this->ctx_, boolean(), {true, true, false, true, false},
+      {true, false, true, true, true}, {true, false}, {}, {0, 0, 1, 0, 1});
+
+  CheckDictEncode<BooleanType, bool>(
+      &this->ctx_, boolean(), {false, true, false, true, false},
+      {true, false, true, true, true}, {false, true}, {}, {0, 0, 0, 1, 0});
+
+  // No nulls
+  CheckDictEncode<BooleanType, bool>(&this->ctx_, boolean(),
+                                     {true, true, false, true, false}, {}, {true, false},
+                                     {}, {0, 0, 1, 0, 1});
+
+  CheckDictEncode<BooleanType, bool>(&this->ctx_, boolean(),
+                                     {false, true, false, true, false}, {}, {false, true},
+                                     {}, {0, 1, 0, 1, 0});
+}
+
 TEST_F(TestHashKernel, UniqueBinary) {
   CheckUnique<BinaryType, std::string>(&this->ctx_, binary(),
                                        {"test", "", "test2", "test"},
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index 95f039932152c..e47759d4d4706 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -368,6 +368,79 @@ class HashTableKernel<Type, Action, enable_if_has_c_type<Type>> : public HashTab
   HashDictionary<Type> dict_;
 };
 
+// ----------------------------------------------------------------------
+// Hash table for boolean types
+
+template <typename Type, typename Action>
+class HashTableKernel<Type, Action, enable_if_boolean<Type>> : public HashTable {
+ public:
+  HashTableKernel(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : HashTable(type, pool) {
+    std::fill(table_, table_ + 2, kHashSlotEmpty);
+  }
+
+  Status Append(const ArrayData& arr) override {
+    auto action = static_cast<Action*>(this);
+
+    RETURN_NOT_OK(action->Reserve(arr.length));
+
+    internal::BitmapReader value_reader(arr.buffers[1]->data(), arr.offset, arr.length);
+
+#define HASH_INNER_LOOP()                                      \
+  if (slot == kHashSlotEmpty) {                                \
+    if (!Action::allow_expand) {                               \
+      throw HashException("Encountered new dictionary value"); \
+    }                                                          \
+    table_[j] = slot = static_cast<hash_slot_t>(dict_.size()); \
+    dict_.push_back(value);                                    \
+    action->ObserveNotFound(slot);                             \
+  } else {                                                     \
+    action->ObserveFound(slot);                                \
+  }
+
+    if (arr.null_count != 0) {
+      internal::BitmapReader valid_reader(arr.buffers[0]->data(), arr.offset, arr.length);
+      for (int64_t i = 0; i < arr.length; ++i) {
+        const bool is_null = valid_reader.IsNotSet();
+        const bool value = value_reader.IsSet();
+        const int j = value ? 1 : 0;
+        hash_slot_t slot = table_[j];
+        valid_reader.Next();
+        value_reader.Next();
+        if (is_null) {
+          action->ObserveNull();
+          continue;
+        }
+        HASH_INNER_LOOP();
+      }
+    } else {
+      for (int64_t i = 0; i < arr.length; ++i) {
+        const bool value = value_reader.IsSet();
+        const int j = value ? 1 : 0;
+        hash_slot_t slot = table_[j];
+        value_reader.Next();
+        HASH_INNER_LOOP();
+      }
+    }
+
+#undef HASH_INNER_LOOP
+
+    return Status::OK();
+  }
+
+  Status GetDictionary(std::shared_ptr<ArrayData>* out) override {
+    BooleanBuilder builder(pool_);
+    for (const bool value : dict_) {
+      RETURN_NOT_OK(builder.Append(value));
+    }
+    return builder.FinishInternal(out);
+  }
+
+ private:
+  hash_slot_t table_[2];
+  std::vector<bool> dict_;
+};
+
 // ----------------------------------------------------------------------
 // Hash table pass for variable-length binary types
 
@@ -698,7 +771,7 @@ Status GetUniqueKernel(FunctionContext* ctx, const std::shared_ptr<DataType>& ty
 
   switch (type->id()) {
     UNIQUE_CASE(NullType);
-    // UNIQUE_CASE(BooleanType);
+    UNIQUE_CASE(BooleanType);
     UNIQUE_CASE(UInt8Type);
     UNIQUE_CASE(Int8Type);
     UNIQUE_CASE(UInt16Type);
@@ -741,7 +814,7 @@ Status GetDictionaryEncodeKernel(FunctionContext* ctx,
 
   switch (type->id()) {
     DICTIONARY_ENCODE_CASE(NullType);
-    // DICTIONARY_ENCODE_CASE(BooleanType);
+    DICTIONARY_ENCODE_CASE(BooleanType);
     DICTIONARY_ENCODE_CASE(UInt8Type);
     DICTIONARY_ENCODE_CASE(Int8Type);
     DICTIONARY_ENCODE_CASE(UInt16Type);
diff --git a/cpp/src/arrow/compute/kernels/util-internal.h b/cpp/src/arrow/compute/kernels/util-internal.h
index 70c506286ede6..7633fed4a8fe7 100644
--- a/cpp/src/arrow/compute/kernels/util-internal.h
+++ b/cpp/src/arrow/compute/kernels/util-internal.h
@@ -59,6 +59,10 @@ template <typename T>
 using enable_if_binary =
     typename std::enable_if<std::is_base_of<BinaryType, T>::value>::type;
 
+template <typename T>
+using enable_if_boolean =
+    typename std::enable_if<std::is_same<BooleanType, T>::value>::type;
+
 template <typename T>
 using enable_if_fixed_size_binary =
     typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value>::type;
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index f1f59384bd639..32464817845f0 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -209,7 +209,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int byte_width()
         int bit_width()
 
-    cdef cppclass CDecimal128Type" arrow::Decimal128Type"(CFixedSizeBinaryType):
+    cdef cppclass CDecimal128Type \
+            " arrow::Decimal128Type"(CFixedSizeBinaryType):
         CDecimal128Type(int precision, int scale)
         int precision()
         int scale()

From 1524ed7fc6fe5aac4528bd05b0c5690fb0f625da Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 23 Nov 2017 09:48:01 -0500
Subject: [PATCH 1276/1644] ARROW-1782: [Python] Add pyarrow.compress,
 decompress APIs

This enables bytes, Buffer, or buffer-like objects to be compressed either to PyBytes or `pyarrow.Buffer`. Wanted some feedback on the API (argument names, etc.). The compression API in Arrow in general requires knowing the size of the decompressed data, but some compressors (like Snappy) are able to tell you how big the result will be based only on the input buffer

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1345 from wesm/ARROW-1782 and squashes the following commits:

f297103b [Wes McKinney] More assertions, nicer error message when decompressed size not passed
7cd0fdb7 [Wes McKinney] Tests for compress/decompress passing
38480411 [Wes McKinney] Draft implementation of pyarrow.decompress
29b82dca [Wes McKinney] Docstring for pyarrow.decompress
2eeaf9a8 [Wes McKinney] Expose ResizableBuffer as Python class, test
6968be3f [Wes McKinney] Work on compress/decompress methods
---
 cpp/src/arrow/util/compression.h     |   2 +-
 python/doc/source/api.rst            |   4 +
 python/pyarrow/__init__.py           |  17 ++-
 python/pyarrow/includes/libarrow.pxd |  43 ++++--
 python/pyarrow/io.pxi                | 209 ++++++++++++++++++++++++++-
 python/pyarrow/lib.pxd               |   7 +
 python/pyarrow/plasma.pyx            |   2 +-
 python/pyarrow/public-api.pxi        |   7 +
 python/pyarrow/tests/test_io.py      |  36 +++++
 9 files changed, 301 insertions(+), 26 deletions(-)

diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h
index ae187a7fcdf1c..de3837ec727b1 100644
--- a/cpp/src/arrow/util/compression.h
+++ b/cpp/src/arrow/util/compression.h
@@ -27,7 +27,7 @@
 namespace arrow {
 
 struct Compression {
-  enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO, BROTLI, ZSTD, LZ4 };
+  enum type { UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4, LZO };
 };
 
 class ARROW_EXPORT Codec {
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index fb2a286770e96..bb2a0420b2a08 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -195,7 +195,11 @@ Input / Output and Shared Memory
    :toctree: generated/
 
    allocate_buffer
+   compress
+   decompress
+   frombuffer
    Buffer
+   ResizableBuffer
    BufferReader
    BufferOutputStream
    NativeFile
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index c4db36e55a0d6..0456a658f1425 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -71,18 +71,21 @@
 # ARROW-1683: Remove after 0.8.0?
 from pyarrow.lib import TimestampType
 
-from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
-                         FixedSizeBufferWriter,
-                         Buffer, BufferReader, BufferOutputStream,
-                         OSFile, MemoryMappedFile, memory_map,
-                         allocate_buffer, frombuffer,
-                         memory_map, create_memory_map,
-                         have_libhdfs, have_libhdfs3, MockOutputStream)
+# Buffers, allocation
+from pyarrow.lib import (Buffer, ResizableBuffer, compress, decompress,
+                         allocate_buffer, frombuffer)
 
 from pyarrow.lib import (MemoryPool, total_allocated_bytes,
                          set_memory_pool, default_memory_pool,
                          log_memory_allocations)
 
+from pyarrow.lib import (HdfsFile, NativeFile, PythonFile,
+                         FixedSizeBufferWriter,
+                         BufferReader, BufferOutputStream,
+                         OSFile, MemoryMappedFile, memory_map,
+                         create_memory_map, have_libhdfs, have_libhdfs3,
+                         MockOutputStream)
+
 from pyarrow.lib import (ChunkedArray, Column, RecordBatch, Table,
                          concat_tables)
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 32464817845f0..5d68607efa386 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -169,26 +169,27 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CBuffer" arrow::Buffer":
         CBuffer(const uint8_t* data, int64_t size)
-        uint8_t* data()
+        const uint8_t* data()
+        uint8_t* mutable_data()
         int64_t size()
         shared_ptr[CBuffer] parent()
         c_bool is_mutable() const
+        c_bool Equals(const CBuffer& other)
 
     cdef cppclass CMutableBuffer" arrow::MutableBuffer"(CBuffer):
         CMutableBuffer(const uint8_t* data, int64_t size)
-        uint8_t* mutable_data()
+
+    cdef cppclass CResizableBuffer" arrow::ResizableBuffer"(CMutableBuffer):
+        CStatus Resize(const int64_t new_size, c_bool shrink_to_fit)
+        CStatus Reserve(const int64_t new_size)
 
     CStatus AllocateBuffer(CMemoryPool* pool, const int64_t size,
                            shared_ptr[CBuffer]* out)
 
     CStatus AllocateResizableBuffer(CMemoryPool* pool, const int64_t size,
-                                    shared_ptr[ResizableBuffer]* out)
+                                    shared_ptr[CResizableBuffer]* out)
 
-    cdef cppclass ResizableBuffer(CBuffer):
-        CStatus Resize(int64_t nbytes)
-        CStatus Reserve(int64_t nbytes)
-
-    cdef cppclass PoolBuffer(ResizableBuffer):
+    cdef cppclass PoolBuffer(CResizableBuffer):
         PoolBuffer()
         PoolBuffer(CMemoryPool*)
 
@@ -635,7 +636,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil:
 
     cdef cppclass CBufferOutputStream \
             " arrow::io::BufferOutputStream"(OutputStream):
-        CBufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
+        CBufferOutputStream(const shared_ptr[CResizableBuffer]& buffer)
 
     cdef cppclass CMockOutputStream \
             " arrow::io::MockOutputStream"(OutputStream):
@@ -661,6 +662,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
         MessageType_V1" arrow::ipc::MetadataVersion::V1"
         MessageType_V2" arrow::ipc::MetadataVersion::V2"
         MessageType_V3" arrow::ipc::MetadataVersion::V3"
+        MessageType_V4" arrow::ipc::MetadataVersion::V4"
 
     cdef cppclass CMessage" arrow::ipc::Message":
         CStatus Open(const shared_ptr[CBuffer]& metadata,
@@ -926,3 +928,26 @@ cdef extern from 'arrow/python/init.h':
 
 cdef extern from 'arrow/python/config.h' namespace 'arrow::py':
     void set_numpy_nan(object o)
+
+
+cdef extern from 'arrow/util/compression.h' namespace 'arrow' nogil:
+    enum CompressionType" arrow::Compression::type":
+        CompressionType_UNCOMPRESSED" arrow::Compression::UNCOMPRESSED"
+        CompressionType_SNAPPY" arrow::Compression::SNAPPY"
+        CompressionType_GZIP" arrow::Compression::GZIP"
+        CompressionType_BROTLI" arrow::Compression::BROTLI"
+        CompressionType_ZSTD" arrow::Compression::ZSTD"
+        CompressionType_LZ4" arrow::Compression::LZ4"
+
+    cdef cppclass CCodec" arrow::Codec":
+        @staticmethod
+        CStatus Create(CompressionType codec, unique_ptr[CCodec]* out)
+
+        CStatus Decompress(int64_t input_len, const uint8_t* input,
+                           int64_t output_len, uint8_t* output_buffer)
+
+        CStatus Compress(int64_t input_len, const uint8_t* input,
+                         int64_t output_buffer_len, uint8_t* output_buffer,
+                         int64_t* output_length)
+
+        int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input)
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 495e31b5a2176..619ba365c2df7 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -600,6 +600,23 @@ cdef class Buffer:
         # TODO(wesm): buffer slicing
         raise NotImplementedError
 
+    def equals(self, Buffer other):
+        """
+        Determine if two buffers contain exactly the same data
+
+        Parameters
+        ----------
+        other : Buffer
+
+        Returns
+        -------
+        are_equal : True if buffer contents and size are equal
+        """
+        cdef c_bool result = False
+        with nogil:
+            result = self.buffer.get().Equals(deref(other.buffer.get()))
+        return result
+
     def to_pybytes(self):
         return cp.PyBytes_FromStringAndSize(
             <const char*>self.buffer.get().data(),
@@ -644,13 +661,37 @@ cdef class Buffer:
         return self.size
 
 
+cdef class ResizableBuffer(Buffer):
+
+    cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer):
+        self.init(<shared_ptr[CBuffer]> buffer)
+
+    def resize(self, int64_t new_size, shrink_to_fit=False):
+        """
+        Resize buffer to indicated size
+
+        Parameters
+        ----------
+        new_size : int64_t
+            New size of buffer (padding may be added internally)
+        shrink_to_fit : boolean, default False
+            If new_size is less than the current size, shrink internal
+            capacity, otherwise leave at current capacity
+        """
+        cdef c_bool c_shrink_to_fit = shrink_to_fit
+        with nogil:
+            check_status((<CResizableBuffer*> self.buffer.get())
+                         .Resize(new_size, c_shrink_to_fit))
+
+
 cdef shared_ptr[PoolBuffer] _allocate_buffer(CMemoryPool* pool):
     cdef shared_ptr[PoolBuffer] result
     result.reset(new PoolBuffer(pool))
     return result
 
 
-def allocate_buffer(int64_t size, MemoryPool pool=None):
+def allocate_buffer(int64_t size, MemoryPool memory_pool=None,
+                    resizable=False):
     """
     Allocate mutable fixed-size buffer
 
@@ -658,17 +699,27 @@ def allocate_buffer(int64_t size, MemoryPool pool=None):
     ----------
     size : int
         Number of bytes to allocate (plus internal padding)
-    pool : MemoryPool, optional
+    memory_pool : MemoryPool, optional
         Uses default memory pool if not provided
+    resizable : boolean, default False
+
+    Returns
+    -------
+    buffer : Buffer or ResizableBuffer
     """
     cdef:
         shared_ptr[CBuffer] buffer
-        CMemoryPool* cpool = maybe_unbox_memory_pool(pool)
+        shared_ptr[CResizableBuffer] rz_buffer
+        CMemoryPool* cpool = maybe_unbox_memory_pool(memory_pool)
 
-    with nogil:
-        check_status(AllocateBuffer(cpool, size, &buffer))
-
-    return pyarrow_wrap_buffer(buffer)
+    if resizable:
+        with nogil:
+            check_status(AllocateResizableBuffer(cpool, size, &rz_buffer))
+        return pyarrow_wrap_resizable_buffer(rz_buffer)
+    else:
+        with nogil:
+            check_status(AllocateBuffer(cpool, size, &buffer))
+        return pyarrow_wrap_buffer(buffer)
 
 
 cdef class BufferOutputStream(NativeFile):
@@ -679,7 +730,7 @@ cdef class BufferOutputStream(NativeFile):
     def __cinit__(self, MemoryPool memory_pool=None):
         self.buffer = _allocate_buffer(maybe_unbox_memory_pool(memory_pool))
         self.wr_file.reset(new CBufferOutputStream(
-            <shared_ptr[ResizableBuffer]> self.buffer))
+            <shared_ptr[CResizableBuffer]> self.buffer))
         self.is_readable = 0
         self.is_writeable = 1
         self.is_open = True
@@ -783,3 +834,145 @@ cdef get_writer(object source, shared_ptr[OutputStream]* writer):
     else:
         raise TypeError('Unable to read from object of type: {0}'
                         .format(type(source)))
+
+
+# ---------------------------------------------------------------------
+
+cdef CompressionType _get_compression_type(object name):
+    if name is None or name == 'uncompressed':
+        return CompressionType_UNCOMPRESSED
+    elif name == 'snappy':
+        return CompressionType_SNAPPY
+    elif name == 'gzip':
+        return CompressionType_GZIP
+    elif name == 'brotli':
+        return CompressionType_BROTLI
+    elif name == 'zstd':
+        return CompressionType_ZSTD
+    elif name == 'lz4':
+        return CompressionType_LZ4
+    else:
+        raise ValueError("Unrecognized compression type: {0}"
+                         .format(str(name)))
+
+
+def compress(object buf, codec='lz4', asbytes=False, memory_pool=None):
+    """
+    Compress pyarrow.Buffer or Python object supporting the buffer (memoryview)
+    protocol
+
+    Parameters
+    ----------
+    buf : pyarrow.Buffer, bytes, or other object supporting buffer protocol
+    codec : string, default 'lz4'
+        Compression codec.
+        Supported types: {'brotli, 'gzip', 'lz4', 'snappy', 'zstd'}
+    asbytes : boolean, default False
+        Return result as Python bytes object, otherwise Buffer
+    memory_pool : MemoryPool, default None
+        Memory pool to use for buffer allocations, if any
+
+    Returns
+    -------
+    compressed : pyarrow.Buffer or bytes (if asbytes=True)
+    """
+    cdef:
+        CompressionType c_codec = _get_compression_type(codec)
+        unique_ptr[CCodec] compressor
+        cdef CBuffer* c_buf
+        cdef PyObject* pyobj
+        cdef ResizableBuffer out_buf
+
+    with nogil:
+        check_status(CCodec.Create(c_codec, &compressor))
+
+    if not isinstance(buf, Buffer):
+        buf = frombuffer(buf)
+
+    c_buf = (<Buffer> buf).buffer.get()
+
+    cdef int64_t max_output_size = (compressor.get()
+                                    .MaxCompressedLen(c_buf.size(),
+                                                      c_buf.data()))
+    cdef uint8_t* output_buffer = NULL
+
+    if asbytes:
+        pyobj = PyBytes_FromStringAndSizeNative(NULL, max_output_size)
+        output_buffer = <uint8_t*> cp.PyBytes_AS_STRING(<object> pyobj)
+    else:
+        out_buf = allocate_buffer(max_output_size, memory_pool=memory_pool,
+                                  resizable=True)
+        output_buffer = out_buf.buffer.get().mutable_data()
+
+    cdef int64_t output_length = 0
+    with nogil:
+        check_status(compressor.get()
+                     .Compress(c_buf.size(), c_buf.data(),
+                               max_output_size, output_buffer,
+                               &output_length))
+
+    if asbytes:
+        cp._PyBytes_Resize(&pyobj, <Py_ssize_t> output_length)
+        return PyObject_to_object(pyobj)
+    else:
+        out_buf.resize(output_length)
+        return out_buf
+
+
+def decompress(object buf, decompressed_size=None, codec='lz4',
+               asbytes=False, memory_pool=None):
+    """
+    Decompress data from buffer-like object
+
+    Parameters
+    ----------
+    buf : pyarrow.Buffer, bytes, or memoryview-compatible object
+    decompressed_size : int64_t, default None
+        If not specified, will be computed if the codec is able to determine
+        the uncompressed buffer size
+    codec : string, default 'lz4'
+        Compression codec.
+        Supported types: {'brotli, 'gzip', 'lz4', 'snappy', 'zstd'}
+    asbytes : boolean, default False
+        Return result as Python bytes object, otherwise Buffer
+    memory_pool : MemoryPool, default None
+        Memory pool to use for buffer allocations, if any
+
+    Returns
+    -------
+    uncompressed : pyarrow.Buffer or bytes (if asbytes=True)
+    """
+    cdef:
+        CompressionType c_codec = _get_compression_type(codec)
+        unique_ptr[CCodec] compressor
+        cdef CBuffer* c_buf
+        cdef Buffer out_buf
+
+    with nogil:
+        check_status(CCodec.Create(c_codec, &compressor))
+
+    if not isinstance(buf, Buffer):
+        buf = frombuffer(buf)
+
+    c_buf = (<Buffer> buf).buffer.get()
+
+    if decompressed_size is None:
+        raise ValueError("Must pass decompressed_size for {0} codec"
+                         .format(codec))
+
+    cdef int64_t output_size = decompressed_size
+    cdef uint8_t* output_buffer = NULL
+
+    if asbytes:
+        pybuf = cp.PyBytes_FromStringAndSize(NULL, output_size)
+        output_buffer = <uint8_t*> cp.PyBytes_AS_STRING(pybuf)
+    else:
+        out_buf = allocate_buffer(output_size, memory_pool=memory_pool)
+        output_buffer = out_buf.buffer.get().mutable_data()
+
+    with nogil:
+        check_status(compressor.get()
+                     .Decompress(c_buf.size(), c_buf.data(),
+                                 output_size, output_buffer))
+
+    return pybuf if asbytes else out_buf
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 5abb72ba48165..90f749d6db633 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -323,6 +323,11 @@ cdef class Buffer:
     cdef void init(self, const shared_ptr[CBuffer]& buffer)
 
 
+cdef class ResizableBuffer(Buffer):
+
+    cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer)
+
+
 cdef class NativeFile:
     cdef:
         shared_ptr[RandomAccessFile] rd_file
@@ -343,6 +348,8 @@ cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader)
 cdef get_writer(object source, shared_ptr[OutputStream]* writer)
 
 cdef public object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf)
+cdef public object pyarrow_wrap_resizable_buffer(
+    const shared_ptr[CResizableBuffer]& buf)
 cdef public object pyarrow_wrap_data_type(const shared_ptr[CDataType]& type)
 cdef public object pyarrow_wrap_field(const shared_ptr[CField]& field)
 cdef public object pyarrow_wrap_schema(const shared_ptr[CSchema]& type)
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index bc0e94e64906e..f2e8653d8a828 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -30,7 +30,7 @@ import collections
 import pyarrow
 
 from pyarrow.lib cimport Buffer, NativeFile, check_status
-from pyarrow.includes.libarrow cimport (CMutableBuffer, CBuffer,
+from pyarrow.includes.libarrow cimport (CBuffer, CMutableBuffer,
                                         CFixedSizeBufferWriter, CStatus)
 
 
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index bf670c5c4e290..9776f2ad76e3c 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -43,6 +43,13 @@ cdef public api object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf):
     return result
 
 
+cdef public api object pyarrow_wrap_resizable_buffer(
+    const shared_ptr[CResizableBuffer]& buf):
+    cdef ResizableBuffer result = ResizableBuffer()
+    result.init_rz(buf)
+    return result
+
+
 cdef public api bint pyarrow_is_data_type(object type_):
     return isinstance(type_, DataType)
 
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 98c465adcdb3d..e60dd35de66fe 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -182,6 +182,42 @@ def test_allocate_buffer():
     assert buf.to_pybytes()[:5] == bit
 
 
+def test_allocate_buffer_resizable():
+    buf = pa.allocate_buffer(100, resizable=True)
+    assert isinstance(buf, pa.ResizableBuffer)
+
+    buf.resize(200)
+    assert buf.size == 200
+
+
+def test_compress_decompress():
+    INPUT_SIZE = 10000
+    test_data = (np.random.randint(0, 255, size=INPUT_SIZE)
+                 .astype(np.uint8)
+                 .tostring())
+    test_buf = pa.frombuffer(test_data)
+
+    codecs = ['lz4', 'snappy', 'gzip', 'zstd', 'brotli']
+    for codec in codecs:
+        compressed_buf = pa.compress(test_buf, codec=codec)
+        compressed_bytes = pa.compress(test_data, codec=codec, asbytes=True)
+
+        assert isinstance(compressed_bytes, bytes)
+
+        decompressed_buf = pa.decompress(compressed_buf, INPUT_SIZE,
+                                         codec=codec)
+        decompressed_bytes = pa.decompress(compressed_bytes, INPUT_SIZE,
+                                           codec=codec, asbytes=True)
+
+        assert isinstance(decompressed_bytes, bytes)
+
+        assert decompressed_buf.equals(test_buf)
+        assert decompressed_bytes == test_data
+
+        with pytest.raises(ValueError):
+            pa.decompress(compressed_bytes, codec=codec)
+
+
 def test_buffer_memoryview_is_immutable():
     val = b'some data'
 

From ea0fb370e2ee0d11b2fbdb149247af3695fd394a Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Thu, 23 Nov 2017 10:24:39 -0500
Subject: [PATCH 1277/1644] ARROW-1577: [JS] add ASF release scripts

@wesm does [this](https://github.com/apache/arrow/commit/b270dbad4f4fd70e19613b93148ec2ae3596e1fd#diff-fc8acbd4f42fb5e6b0cad14928b68115R59) look good?

Author: Paul Taylor <paul.e.taylor@me.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1346 from trxcllnt/js-asf-release-scripts and squashes the following commits:

ac246cd1 [Wes McKinney] Update npm-release.sh while integration testing from release tarball is hard
469beeac [Wes McKinney] Add to READMEs about how to release and verify JS release
160f5c30 [Paul Taylor] remove tag-version-prefix from npmrc
854020d2 [Wes McKinney] Run unit tests without integration tests for now
94859222 [Wes McKinney] Add JavaScript release verification script
dd06c468 [Wes McKinney] Create tag in source release script
2c6ce86c [Paul Taylor] number of npm-release.sh script arguments is 0
e1c3a96c [Paul Taylor] npm install before release, use npm version --git-tag-version with tag-version-prefix
58700435 [Wes McKinney] Number of arguments is 2
1133fb8c [Wes McKinney] Remove upstream Arrow version tag. Bump JS version to 0.2.0
30b602f1 [Paul Taylor] add dev/release/js-source-release.sh script
be82c087 [Paul Taylor] add npm-release.sh script to publish npm modules
f72fc833 [Paul Taylor] update test_js commands in verify-release-candidate.sh
5194951d [Paul Taylor] npm pack the js sources into the release tarball
d73c47a0 [Paul Taylor] prepare js lib for release
7d9c84a0 [Paul Taylor] remove tslib patches as it doesn't work on windows
---
 ci/travis_script_integration.sh            |   2 +-
 dev/README.md                              |   8 ++
 dev/release/README                         |  13 +++
 dev/release/js-source-release.sh           | 103 +++++++++++++++++++++
 dev/release/js-verify-release-candidate.sh | 100 ++++++++++++++++++++
 dev/release/verify-release-candidate.sh    |  11 ++-
 js/.npmrc                                  |   2 +-
 js/README.md                               |  22 +++--
 js/gulp/package-task.js                    |   8 +-
 js/gulp/test-task.js                       |   3 +-
 js/gulp/util.js                            |  16 +++-
 js/gulpfile.js                             |   4 +-
 js/npm-release.sh                          |  12 ++-
 js/package.json                            |  23 +++--
 14 files changed, 292 insertions(+), 35 deletions(-)
 create mode 100755 dev/release/js-source-release.sh
 create mode 100755 dev/release/js-verify-release-candidate.sh
 mode change 100644 => 100755 js/npm-release.sh

diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 105a0cc6086df..9514a30a88b08 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -51,7 +51,7 @@ pushd $ARROW_JS_DIR
 npm run lint
 npm run build
 # create initial test data
-npm run test:createTestData
+npm run create:testdata
 # run once to write the snapshots
 npm test -- -t ts -u --integration
 # run again to test all builds against the snapshots
diff --git a/dev/README.md b/dev/README.md
index f3e963cf491bb..62ffb0a8f1416 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -113,3 +113,11 @@ Studio 2015):
 ```
 dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
 ```
+
+### Verifying the JavaScript release
+
+For JavaScript-specific releases, use a different verification script:
+
+```shell
+bash dev/release/js-verify-release-candidate.sh 0.7.0 0
+```
\ No newline at end of file
diff --git a/dev/release/README b/dev/release/README
index dd2f512a0bb1b..8e0104ed83318 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -49,3 +49,16 @@ Logon to the apache repository: https://repository.apache.org/#stagingRepositori
 Select the arrow staging repository you just just created: orgapachearrow-100x
 Click the "close" button
 Once validation has passed, click the "release" button
+
+# Releasing JavaScript
+
+* Make release branch then tag the release
+
+git checkout -b release-js-X.Y.Z
+git tag -a apache-arrow-js-X.Y.Z
+
+* Build the source release (requires NodeJS)
+
+dev/release/js-source-release.sh X.Y.Z $RC_NUM
+
+* After release vote push tag, and rebase master on release branch
\ No newline at end of file
diff --git a/dev/release/js-source-release.sh b/dev/release/js-source-release.sh
new file mode 100755
index 0000000000000..bf32acd052403
--- /dev/null
+++ b/dev/release/js-source-release.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <js-version> <rc-num>"
+  exit
+fi
+
+js_version=$1
+rc=$2
+
+tag=apache-arrow-js-${js_version}
+tagrc=${tag}-rc${rc}
+
+echo "Preparing source for tag ${tag}"
+
+tarball=${tag}.tar.gz
+
+# cd to $ARROW_HOME/js
+cd $SOURCE_DIR/../../js
+JS_SRC_DIR="$PWD"
+# npm pack the js source files
+npm install
+
+npm version --no-git-tag-version $js_version
+git add package.json
+git commit -m "[Release] Apache Arrow JavaScript $js_version"
+git tag -a ${tag}
+
+release_hash=`git rev-list $tag 2> /dev/null | head -n 1 `
+
+if [ -z "$release_hash" ]; then
+  echo "Cannot continue: unknown git tag: $tag"
+  exit
+fi
+
+echo "Using commit $release_hash"
+
+cd $SOURCE_DIR
+
+rm -rf js-tmp
+# `npm pack` writes the .tgz file to the current dir, so cd into js-tmp
+mkdir -p js-tmp
+cd js-tmp
+# run npm pack on `arrow/js`
+npm pack ${JS_SRC_DIR}
+# unzip and remove the npm pack tarball
+tar -xzf *.tgz && rm *.tgz
+# `npm pack` puts files in a dir called "package"
+cp $JS_SRC_DIR/../NOTICE.txt package
+cp $JS_SRC_DIR/../LICENSE.txt package
+# rename "package" to $tag
+mv package ${tag}
+tar czf ${tarball} ${tag}
+rm -rf ${tag}
+
+${SOURCE_DIR}/run-rat.sh ${tarball}
+
+# sign the archive
+gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
+gpg --print-md MD5 ${tarball} > ${tarball}.md5
+sha1sum $tarball > ${tarball}.sha1
+sha256sum $tarball > ${tarball}.sha256
+sha512sum $tarball > ${tarball}.sha512
+
+# check out the arrow RC folder
+svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow js-rc-tmp
+
+# add the release candidate for the tag
+mkdir -p js-rc-tmp/${tagrc}
+cp ${tarball}* js-rc-tmp/${tagrc}
+svn add js-rc-tmp/${tagrc}
+svn ci -m 'Apache Arrow JavaScript ${version} RC${rc}' js-rc-tmp/${tagrc}
+
+cd -
+
+# clean up
+rm -rf js-tmp
+
+echo "Success! The release candidate is available here:"
+echo "  https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
+echo ""
+echo "Commit SHA1: ${release_hash}"
diff --git a/dev/release/js-verify-release-candidate.sh b/dev/release/js-verify-release-candidate.sh
new file mode 100755
index 0000000000000..5a37e10f74afb
--- /dev/null
+++ b/dev/release/js-verify-release-candidate.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Requirements
+# - nodejs >= 6.0.0 (best way is to use nvm)
+
+case $# in
+  2) VERSION="$1"
+     RC_NUMBER="$2"
+     ;;
+
+  *) echo "Usage: $0 X.Y.Z RC_NUMBER"
+     exit 1
+     ;;
+esac
+
+set -ex
+
+HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
+
+ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
+
+download_dist_file() {
+  curl -f -O $ARROW_DIST_URL/$1
+}
+
+download_rc_file() {
+  download_dist_file apache-arrow-js-${VERSION}-rc${RC_NUMBER}/$1
+}
+
+import_gpg_keys() {
+  download_dist_file KEYS
+  gpg --import KEYS
+}
+
+fetch_archive() {
+  local dist_name=$1
+  download_rc_file ${dist_name}.tar.gz
+  download_rc_file ${dist_name}.tar.gz.asc
+  download_rc_file ${dist_name}.tar.gz.md5
+  download_rc_file ${dist_name}.tar.gz.sha512
+  gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
+  gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
+  if [ "$(uname)" == "Darwin" ]; then
+    shasum -a 512 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+  else
+    sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+  fi
+}
+
+setup_tempdir() {
+  cleanup() {
+    rm -fr "$TMPDIR"
+  }
+  trap cleanup EXIT
+  TMPDIR=$(mktemp -d -t "$1.XXXXX")
+}
+
+setup_tempdir "arrow-js-$VERSION"
+echo "Working in sandbox $TMPDIR"
+cd $TMPDIR
+
+VERSION=$1
+RC_NUMBER=$2
+
+TARBALL=apache-arrow-js-$1.tar.gz
+
+import_gpg_keys
+
+DIST_NAME="apache-arrow-js-${VERSION}"
+fetch_archive $DIST_NAME
+tar xvzf ${DIST_NAME}.tar.gz
+cd ${DIST_NAME}
+
+npm install
+# npx run-s clean:all lint create:testdata build
+# npm run test -- -t ts -u --integration
+# npm run test -- --integration
+npx run-s clean:all lint build
+npm run test
+
+echo 'Release candidate looks good!'
+exit 0
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 8d1bed75b8b05..7c5e7fd0abaaf 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -173,7 +173,16 @@ test_glib() {
 test_js() {
   pushd js
   npm install
-  npm run validate
+  # clean, lint, and build JS source
+  npm run clean:all
+  npm run lint
+  npm run build
+  # create initial integration test data
+  npm run create:testdata
+  # run once to write the snapshots
+  npm test -- -t ts -u --integration
+  # run again to test all builds against the snapshots
+  npm test -- --integration
   popd
 }
 
diff --git a/js/.npmrc b/js/.npmrc
index 71ffabdd55d95..b6b25d1f1816d 100644
--- a/js/.npmrc
+++ b/js/.npmrc
@@ -1,2 +1,2 @@
 save-prefix=
-package-lock=false
+package-lock=false
\ No newline at end of file
diff --git a/js/README.md b/js/README.md
index e07be6bc0aaaf..9b1753f68335e 100644
--- a/js/README.md
+++ b/js/README.md
@@ -183,22 +183,24 @@ The base `apache-arrow` package includes all the compilation targets for conveni
 The targets are also published under the `@apache-arrow` namespace:
 
 ```sh
-npm install @apache-arrow/es5-cjs # ES5 CommonJS target
-npm install @apache-arrow/es5-esm # ES5 ESModules target
-npm install @apache-arrow/es5-umd # ES5 UMD target
-npm install @apache-arrow/es2015-cjs # ES2015 CommonJS target
-npm install @apache-arrow/es2015-esm # ES2015 ESModules target
-npm install @apache-arrow/es2015-umd # ES2015 UMD target
-npm install @apache-arrow/esnext-esm # ESNext CommonJS target
-npm install @apache-arrow/esnext-esm # ESNext ESModules target
-npm install @apache-arrow/esnext-umd # ESNext UMD target
+npm install apache-arrow # <-- combined es5/CommonJS + UMD, es2015/ESModules + UMD, and TypeScript package
+npm install @apache-arrow/ts # standalone TypeScript package
+npm install @apache-arrow/es5-cjs # standalone es5/CommonJS package
+npm install @apache-arrow/es5-esm # standalone es5/ESModules package
+npm install @apache-arrow/es5-umd # standalone es5/UMD package
+npm install @apache-arrow/es2015-cjs # standalone es2015/CommonJS package
+npm install @apache-arrow/es2015-esm # standalone es2015/ESModules package
+npm install @apache-arrow/es2015-umd # standalone es2015/UMD package
+npm install @apache-arrow/esnext-esm # standalone esNext/CommonJS package
+npm install @apache-arrow/esnext-esm # standalone esNext/ESModules package
+npm install @apache-arrow/esnext-umd # standalone esNext/UMD package
 ```
 
 ### Why we package like this
 
 The JS community is a diverse group with a varied list of target environments and tool chains. Publishing multiple packages accommodates projects of all stripes.
 
-If you think we missed a compilation target and it's a blocker for adoption, please open an issue. We're here for you ❤️.
+If you think we missed a compilation target and it's a blocker for adoption, please open an issue.
 
 # License
 
diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index ad56d172eaec7..824284e6a158a 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -48,13 +48,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
     module: `${mainExport}.mjs`,
     browser: `${mainExport}.es5.min.js`,
     [`browser:es2015`]: `${mainExport}.es2015.min.js`,
-    [`@std/esm`]: { esm: `mjs` },
-    // Temporary workaround until https://github.com/Microsoft/tslib/pull/44 is merged
-    scripts: {
-        postinstall: `npm i shx && npm run tslib_mjs && npm run tslib_pkg && npm r shx`,
-        tslib_mjs: `shx cp $(node -e \"console.log(require.resolve('tslib/tslib.es6.js'))\") $(node -e \"var r=require,p=r('path');console.log(p.join(p.dirname(r.resolve('tslib')),'tslib.mjs'))\")`,
-        tslib_pkg: `node -e \"var r=require,p=r('path'),f=r('fs'),k=p.join(p.dirname(r.resolve('tslib')),'package.json'),x=JSON.parse(f.readFileSync(k));x.main='tslib';f.writeFileSync(k,JSON.stringify(x))\"`
-    }
+    [`@std/esm`]: { esm: `mjs` }
 });
   
 const createTypeScriptPackageJson = (target, format) => (orig) => ({
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
index ae6b8071d2ab7..ac9755e11ded3 100644
--- a/js/gulp/test-task.js
+++ b/js/gulp/test-task.js
@@ -73,12 +73,13 @@ const CPP_STREAM_TO_FILE = path.join(CPP_EXE_PATH, 'stream-to-file');
 const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'file-to-stream');
 
 const testFilesDir = path.join(ARROW_HOME, 'js/test/data');
+const snapshotsDir = path.join(ARROW_HOME, 'js/test/__snapshots__');
 const cppFilesDir = path.join(testFilesDir, 'cpp');
 const javaFilesDir = path.join(testFilesDir, 'java');
 const jsonFilesDir = path.join(testFilesDir, 'json');
 
 async function cleanTestData() {
-    return await del(`${testFilesDir}/**`);
+    return await del([`${testFilesDir}/**`, `${snapshotsDir}/**`]);
 }
 
 async function createTestJSON() {
diff --git a/js/gulp/util.js b/js/gulp/util.js
index e23fc398562d4..23476da2df53b 100644
--- a/js/gulp/util.js
+++ b/js/gulp/util.js
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+const fs = require('fs');
 const path = require(`path`);
 const pump = require(`pump`);
 const { Observable, ReplaySubject } = require('rxjs');
@@ -27,13 +28,26 @@ const releasesRootDir = `targets`;
 const knownTargets = [`es5`, `es2015`, `esnext`];
 const knownModules = [`cjs`, `esm`, `cls`, `umd`];
 const moduleFormatsToSkipCombosOf = { cls: true };
-const metadataFiles = [`../LICENSE.txt`, `../NOTICE.txt`, `README.md`];
 const packageJSONFields = [
   `version`, `license`, `description`,
   `author`, `homepage`, `repository`,
   `bugs`, `keywords`,  `dependencies`
 ];
 
+const metadataFiles = [`LICENSE.txt`, `NOTICE.txt`, `README.md`].map((filename) => {
+    let err = false, prefixes = [`./`, `../`];
+    let p = prefixes.find((prefix) => {
+        try {
+            fs.statSync(path.resolve(path.join(prefix, filename)));
+        } catch (e) { return false; }
+        return true;
+    });
+    if (!p) {
+        throw new Error(`Couldn't find ${filename} in ./ or ../`);
+    }
+    return path.join(p, filename);
+});
+
 // see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988
 const gCCLanguageNames = {
     es5: `ECMASCRIPT5`,
diff --git a/js/gulpfile.js b/js/gulpfile.js
index cd0ac70029aba..a60363214bb94 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -86,8 +86,8 @@ const buildConcurrent = (tasks) => () =>
             .merge(...knownTargets.map((target) =>
                 del(`${targetDir(target, `cls`)}/**`)))));
   
-gulp.task(`test:cleanTestData`, cleanTestData);
-gulp.task(`test:createTestData`, createTestData);
+gulp.task(`clean:testdata`, cleanTestData);
+gulp.task(`create:testdata`, createTestData);
 gulp.task( `test`, gulp.series(getTasks(`test`)));
 gulp.task(`debug`, gulp.series(getTasks(`debug`)));
 gulp.task(`clean`, gulp.parallel(getTasks(`clean`)));
diff --git a/js/npm-release.sh b/js/npm-release.sh
old mode 100644
new mode 100755
index 0bf70d9798538..42cd73c0cfe59
--- a/js/npm-release.sh
+++ b/js/npm-release.sh
@@ -16,5 +16,15 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -e
 
-lerna publish --yes --skip-git --force-publish=*
\ No newline at end of file
+# validate the targets pass all tests before publishing
+npm install
+# npx run-s clean:all lint create:testdata build
+# npm run test -- -t ts -u --integration
+# npm run test -- --integration
+npx run-s clean:all lint build
+npm run test
+
+# publish the JS target modules to npm
+npx lerna publish --yes --skip-git --force-publish=*
diff --git a/js/package.json b/js/package.json
index 1a110b2d5de89..9933f75a2ce57 100644
--- a/js/package.json
+++ b/js/package.json
@@ -17,16 +17,16 @@
     "debug": "gulp debug",
     "perf": "node ./perf/index.js",
     "release": "./npm-release.sh",
-    "validate": "run-s --silent lint build clean",
+    "clean:all": "run-p clean clean:testdata",
+    "clean:testdata": "gulp clean:testdata",
+    "create:testdata": "gulp create:testdata",
     "test:coverage": "gulp test -t ts --coverage",
-    "test:cleanTestData": "gulp test:cleanTestData",
-    "test:createTestData": "gulp test:createTestData",
     "doc": "shx rm -rf ./doc && esdoc",
-    "lint": "npm-run-all -p lint:*",
+    "lint": "run-p lint:*",
     "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
     "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"",
     "prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1",
-    "postinstall": "shx cp node_modules/tslib/tslib.es6.js node_modules/tslib/tslib.mjs"
+    "version": "npm install && npm run clean:all"
   },
   "repository": {
     "type": "git",
@@ -43,12 +43,15 @@
   },
   "homepage": "https://github.com/apache/arrow/blob/master/js/README.md",
   "files": [
+    "bin",
     "src",
-    "dist",
-    "types",
-    "targets",
-    "LICENSE",
-    "README.md"
+    "gulp",
+    "test",
+    "*.json",
+    "tsconfig",
+    "README.md",
+    "gulpfile.js",
+    "npm-release.sh"
   ],
   "peerDependencies": {
     "command-line-usage": "4.0.1"

From 6ec4f343ba5bff628d397a7441852b5603dc7488 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Thu, 23 Nov 2017 13:26:30 -0500
Subject: [PATCH 1278/1644] ARROW-1047: [Java] [FollowUp] Change ArrowMagic to
 be non-public class

Move ArrowMagic from vector.ipc to vector.ipc.message package.

Author: Li Jin <ice.xelloss@gmail.com>

Closes #1349 from icexelloss/ARROW-1047-followup and squashes the following commits:

0b5df959 [Li Jin] Change ArrowMagic to be non public class
---
 .../src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java
index a9310a608ae03..f71318ee678ad 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java
@@ -18,13 +18,11 @@
 
 package org.apache.arrow.vector.ipc;
 
-import org.apache.arrow.vector.ipc.WriteChannel;
-
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
-public class ArrowMagic {
+class ArrowMagic {
 
   private static final byte[] MAGIC = "ARROW1".getBytes(StandardCharsets.UTF_8);
 

From ac4bb6990fef485a3178361881943e843da0f3c7 Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Fri, 24 Nov 2017 09:12:51 -0500
Subject: [PATCH 1279/1644] ARROW-1852: [C++] Make retrieval of Plasma manager
 fd a const operation

This small patch makes it possible to retrieve the manager fd in a `const` context. For example:

```cpp
class handle {
public:
  bool connected() const {
    return client_.get_manager_fd() != -1;
  }
private:
  plasma::PlasmaClient client_;
};
```

Without this patch, it's impossible to implement the above function.

An alternative would be to simply provide such a utility function as part of `PlasmaClient`, but this changes the API. Let me know what you prefer.

Author: Matthias Vallentin <vallentin@icir.org>

Closes #1352 from mavam/topic/plasma-manager-fd and squashes the following commits:

f4d4ad37 [Matthias Vallentin] Make retrieval of manager fd a const operation
---
 cpp/src/plasma/client.cc | 2 +-
 cpp/src/plasma/client.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index dd32bdc8149a3..9bbafac38004a 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -617,7 +617,7 @@ Status PlasmaClient::Fetch(int num_object_ids, const ObjectID* object_ids) {
   return SendFetchRequest(manager_conn_, object_ids, num_object_ids);
 }
 
-int PlasmaClient::get_manager_fd() { return manager_conn_; }
+int PlasmaClient::get_manager_fd() const { return manager_conn_; }
 
 Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
   ARROW_CHECK(manager_conn_ >= 0);
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index 89df2b0b0039f..cfd11c16dbace 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -313,7 +313,7 @@ class ARROW_EXPORT PlasmaClient {
   ///
   /// \return The file descriptor for the manager connection. If there is no
   ///         connection to the manager, this is -1.
-  int get_manager_fd();
+  int get_manager_fd() const;
 
  private:
   /// This is a helper method for unmapping objects for which all references have

From 05bfb2619b19154eead36e4aa067f440fdedf49a Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 24 Nov 2017 09:32:36 -0500
Subject: [PATCH 1280/1644] ARROW-1849: [GLib] Add input checks to
 GArrowRecordBatch

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1351 from kou/glib-add-input-check-to-record-batch and squashes the following commits:

c85dde22 [Kouhei Sutou] [GLib] Follow API change in Go example
3bf1eeb4 [Kouhei Sutou] [GLib] Add index check to column readers for GArrowRecordBatch
94d9f238 [Kouhei Sutou] Always validate on creating new record batch
---
 c_glib/arrow-glib/record-batch.cpp | 54 ++++++++++++++++----
 c_glib/arrow-glib/record-batch.h   |  7 +--
 c_glib/example/go/read-batch.go    |  4 +-
 c_glib/example/go/read-stream.go   |  4 +-
 c_glib/example/go/write-batch.go   | 10 +++-
 c_glib/example/go/write-stream.go  | 10 +++-
 c_glib/test/test-record-batch.rb   | 81 ++++++++++++++++++++++--------
 7 files changed, 128 insertions(+), 42 deletions(-)

diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp
index f23a0cf7582e2..64f2020ad3efc 100644
--- a/c_glib/arrow-glib/record-batch.cpp
+++ b/c_glib/arrow-glib/record-batch.cpp
@@ -28,6 +28,23 @@
 
 #include <sstream>
 
+static inline bool
+garrow_record_batch_adjust_index(const std::shared_ptr<arrow::RecordBatch> arrow_record_batch,
+                                 gint &i)
+{
+  auto n_columns = arrow_record_batch->num_columns();
+  if (i < 0) {
+    i += n_columns;
+    if (i < 0) {
+      return false;
+    }
+  }
+  if (i >= n_columns) {
+    return false;
+  }
+  return true;
+}
+
 G_BEGIN_DECLS
 
 /**
@@ -135,13 +152,15 @@ garrow_record_batch_class_init(GArrowRecordBatchClass *klass)
  * @schema: The schema of the record batch.
  * @n_rows: The number of the rows in the record batch.
  * @columns: (element-type GArrowArray): The columns in the record batch.
+ * @error: (nullable): Return location for a #GError or %NULL.
  *
- * Returns: A newly created #GArrowRecordBatch.
+ * Returns: (nullable): A newly created #GArrowRecordBatch or %NULL on error.
  */
 GArrowRecordBatch *
 garrow_record_batch_new(GArrowSchema *schema,
                         guint32 n_rows,
-                        GList *columns)
+                        GList *columns,
+                        GError **error)
 {
   std::vector<std::shared_ptr<arrow::Array>> arrow_columns;
   for (GList *node = columns; node; node = node->next) {
@@ -152,7 +171,12 @@ garrow_record_batch_new(GArrowSchema *schema,
   auto arrow_record_batch =
     arrow::RecordBatch::Make(garrow_schema_get_raw(schema),
                              n_rows, arrow_columns);
-  return garrow_record_batch_new_raw(&arrow_record_batch);
+  auto status = arrow_record_batch->Validate();
+  if (garrow_error_check(error, status, "[record-batch][new]")) {
+    return garrow_record_batch_new_raw(&arrow_record_batch);
+  } else {
+    return NULL;
+  }
 }
 
 /**
@@ -192,15 +216,21 @@ garrow_record_batch_get_schema(GArrowRecordBatch *record_batch)
 /**
  * garrow_record_batch_get_column:
  * @record_batch: A #GArrowRecordBatch.
- * @i: The index of the target column.
+ * @i: The index of the target column. If it's negative, index is
+ *   counted backward from the end of the columns. `-1` means the last
+ *   column.
  *
- * Returns: (transfer full): The i-th column in the record batch.
+ * Returns: (transfer full) (nullable): The i-th column in the record batch
+ *   on success, %NULL on out of index.
  */
 GArrowArray *
 garrow_record_batch_get_column(GArrowRecordBatch *record_batch,
-                               guint i)
+                               gint i)
 {
   const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  if (!garrow_record_batch_adjust_index(arrow_record_batch, i)) {
+    return NULL;
+  }
   auto arrow_column = arrow_record_batch->column(i);
   return garrow_array_new_raw(&arrow_column);
 }
@@ -230,15 +260,21 @@ garrow_record_batch_get_columns(GArrowRecordBatch *record_batch)
 /**
  * garrow_record_batch_get_column_name:
  * @record_batch: A #GArrowRecordBatch.
- * @i: The index of the target column.
+ * @i: The index of the target column. If it's negative, index is
+ *   counted backward from the end of the columns. `-1` means the last
+ *   column.
  *
- * Returns: The name of the i-th column in the record batch.
+ * Returns: (nullable): The name of the i-th column in the record batch
+ *   on success, %NULL on out of index
  */
 const gchar *
 garrow_record_batch_get_column_name(GArrowRecordBatch *record_batch,
-                                    guint i)
+                                    gint i)
 {
   const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  if (!garrow_record_batch_adjust_index(arrow_record_batch, i)) {
+    return NULL;
+  }
   return arrow_record_batch->column_name(i).c_str();
 }
 
diff --git a/c_glib/arrow-glib/record-batch.h b/c_glib/arrow-glib/record-batch.h
index 021f894f3f5a3..d31edf43973d0 100644
--- a/c_glib/arrow-glib/record-batch.h
+++ b/c_glib/arrow-glib/record-batch.h
@@ -68,17 +68,18 @@ GType garrow_record_batch_get_type(void) G_GNUC_CONST;
 
 GArrowRecordBatch *garrow_record_batch_new(GArrowSchema *schema,
                                            guint32 n_rows,
-                                           GList *columns);
+                                           GList *columns,
+                                           GError **error);
 
 gboolean garrow_record_batch_equal(GArrowRecordBatch *record_batch,
                                    GArrowRecordBatch *other_record_batch);
 
 GArrowSchema *garrow_record_batch_get_schema     (GArrowRecordBatch *record_batch);
 GArrowArray  *garrow_record_batch_get_column     (GArrowRecordBatch *record_batch,
-                                                  guint i);
+                                                  gint i);
 GList        *garrow_record_batch_get_columns    (GArrowRecordBatch *record_batch);
 const gchar  *garrow_record_batch_get_column_name(GArrowRecordBatch *record_batch,
-                                                  guint i);
+                                                  gint i);
 guint         garrow_record_batch_get_n_columns  (GArrowRecordBatch *record_batch);
 gint64        garrow_record_batch_get_n_rows     (GArrowRecordBatch *record_batch);
 GArrowRecordBatch *garrow_record_batch_slice     (GArrowRecordBatch *record_batch,
diff --git a/c_glib/example/go/read-batch.go b/c_glib/example/go/read-batch.go
index ef1a7fb5a69f9..1472939cd9284 100644
--- a/c_glib/example/go/read-batch.go
+++ b/c_glib/example/go/read-batch.go
@@ -57,8 +57,8 @@ func PrintColumnValue(column *arrow.Array, i int64) {
 func PrintRecordBatch(recordBatch *arrow.RecordBatch) {
 	nColumns := recordBatch.GetNColumns()
 	for i := uint32(0); i < nColumns; i++ {
-		column := recordBatch.GetColumn(i)
-		columnName := recordBatch.GetColumnName(i)
+		column := recordBatch.GetColumn(int32(i))
+		columnName := recordBatch.GetColumnName(int32(i))
 		fmt.Printf("  %s: [", columnName)
 		nRows := recordBatch.GetNRows()
 		for j := int64(0); j < nRows; j++ {
diff --git a/c_glib/example/go/read-stream.go b/c_glib/example/go/read-stream.go
index 7bd076473f667..ed75a96c95c39 100644
--- a/c_glib/example/go/read-stream.go
+++ b/c_glib/example/go/read-stream.go
@@ -57,8 +57,8 @@ func PrintColumnValue(column *arrow.Array, i int64) {
 func PrintRecordBatch(recordBatch *arrow.RecordBatch) {
 	nColumns := recordBatch.GetNColumns()
 	for i := uint32(0); i < nColumns; i++ {
-		column := recordBatch.GetColumn(i)
-		columnName := recordBatch.GetColumnName(i)
+		column := recordBatch.GetColumn(int32(i))
+		columnName := recordBatch.GetColumnName(int32(i))
 		fmt.Printf("  %s: [", columnName)
 		nRows := recordBatch.GetNRows()
 		for j := int64(0); j < nRows; j++ {
diff --git a/c_glib/example/go/write-batch.go b/c_glib/example/go/write-batch.go
index 9dbc3c00acc50..f4d03ed922eb5 100644
--- a/c_glib/example/go/write-batch.go
+++ b/c_glib/example/go/write-batch.go
@@ -188,7 +188,10 @@ func main() {
 		BuildDoubleArray(),
 	}
 
-	recordBatch := arrow.NewRecordBatch(schema, 4, columns)
+	recordBatch, err := arrow.NewRecordBatch(schema, 4, columns)
+	if err != nil {
+		log.Fatalf("Failed to create record batch #1: %v", err)
+	}
 	_, err = writer.WriteRecordBatch(recordBatch)
 	if err != nil {
 		log.Fatalf("Failed to write record batch #1: %v", err)
@@ -198,7 +201,10 @@ func main() {
 	for i, column := range columns {
 		slicedColumns[i] = column.Slice(1, 3)
 	}
-	recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	recordBatch, err = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	if err != nil {
+		log.Fatalf("Failed to create record batch #2: %v", err)
+	}
 	_, err = writer.WriteRecordBatch(recordBatch)
 	if err != nil {
 		log.Fatalf("Failed to write record batch #2: %v", err)
diff --git a/c_glib/example/go/write-stream.go b/c_glib/example/go/write-stream.go
index 244741e8cfeb0..7225156a7be84 100644
--- a/c_glib/example/go/write-stream.go
+++ b/c_glib/example/go/write-stream.go
@@ -188,7 +188,10 @@ func main() {
 		BuildDoubleArray(),
 	}
 
-	recordBatch := arrow.NewRecordBatch(schema, 4, columns)
+	recordBatch, err := arrow.NewRecordBatch(schema, 4, columns)
+	if err != nil {
+		log.Fatalf("Failed to create record batch #1: %v", err)
+	}
 	_, err = writer.WriteRecordBatch(recordBatch)
 	if err != nil {
 		log.Fatalf("Failed to write record batch #1: %v", err)
@@ -198,7 +201,10 @@ func main() {
 	for i, column := range columns {
 		slicedColumns[i] = column.Slice(1, 3)
 	}
-	recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	recordBatch, err = arrow.NewRecordBatch(schema, 3, slicedColumns)
+	if err != nil {
+		log.Fatalf("Failed to create record batch #2: %v", err)
+	}
 	writer.WriteRecordBatch(recordBatch)
 	_, err = writer.WriteRecordBatch(recordBatch)
 	if err != nil {
diff --git a/c_glib/test/test-record-batch.rb b/c_glib/test/test-record-batch.rb
index 9fd34b7d45cff..365922f496fe2 100644
--- a/c_glib/test/test-record-batch.rb
+++ b/c_glib/test/test-record-batch.rb
@@ -18,32 +18,53 @@
 class TestTable < Test::Unit::TestCase
   include Helper::Buildable
 
-  def test_new
-    fields = [
-      Arrow::Field.new("visible", Arrow::BooleanDataType.new),
-      Arrow::Field.new("valid", Arrow::BooleanDataType.new),
-    ]
-    schema = Arrow::Schema.new(fields)
-    columns = [
-      build_boolean_array([true]),
-      build_boolean_array([false]),
-    ]
-    record_batch = Arrow::RecordBatch.new(schema, 1, columns)
-    assert_equal(1, record_batch.n_rows)
+  sub_test_case(".new") do
+    def test_valid
+      fields = [
+        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
+      ]
+      schema = Arrow::Schema.new(fields)
+      columns = [
+        build_boolean_array([true]),
+        build_boolean_array([false]),
+      ]
+      record_batch = Arrow::RecordBatch.new(schema, 1, columns)
+      assert_equal(1, record_batch.n_rows)
+    end
+
+    def test_no_columns
+      fields = [
+        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
+      ]
+      schema = Arrow::Schema.new(fields)
+      message = "[record-batch][new]: " +
+        "Invalid: Number of columns did not match schema"
+      assert_raise(Arrow::Error::Invalid.new(message)) do
+        Arrow::RecordBatch.new(schema, 0, [])
+      end
+    end
   end
 
   sub_test_case("instance methods") do
     def setup
+      @visible_field = Arrow::Field.new("visible", Arrow::BooleanDataType.new)
+      @visible_values = [true, false, true, false, true]
+      @valid_field = Arrow::Field.new("valid", Arrow::BooleanDataType.new)
+      @valid_values = [false, true, false, true, false]
+
       fields = [
-        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
-        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
+        @visible_field,
+        @valid_field,
       ]
       schema = Arrow::Schema.new(fields)
       columns = [
-        build_boolean_array([true, false, true, false, true, false]),
-        build_boolean_array([false, true, false, true, false]),
+        build_boolean_array(@visible_values),
+        build_boolean_array(@valid_values),
       ]
-      @record_batch = Arrow::RecordBatch.new(schema, 5, columns)
+      @record_batch = Arrow::RecordBatch.new(schema,
+                                             @visible_values.size,
+                                             columns)
     end
 
     def test_equal
@@ -53,7 +74,7 @@ def test_equal
       ]
       schema = Arrow::Schema.new(fields)
       columns = [
-        build_boolean_array([true, false, true, false, true, false]),
+        build_boolean_array([true, false, true, false, true]),
         build_boolean_array([false, true, false, true, false]),
       ]
       other_record_batch = Arrow::RecordBatch.new(schema, 5, columns)
@@ -66,12 +87,28 @@ def test_schema
                    @record_batch.schema.fields.collect(&:name))
     end
 
-    def test_column
-      assert_equal(5, @record_batch.get_column(1).length)
+    sub_test_case("#column") do
+      def test_positive
+        assert_equal(build_boolean_array(@valid_values),
+                     @record_batch.get_column(1))
+      end
+
+      def test_negative
+        assert_equal(build_boolean_array(@visible_values),
+                     @record_batch.get_column(-2))
+      end
+
+      def test_positive_out_of_index
+        assert_nil(@record_batch.get_column(2))
+      end
+
+      def test_negative_out_of_index
+        assert_nil(@record_batch.get_column(-3))
+      end
     end
 
     def test_columns
-      assert_equal([6, 5],
+      assert_equal([5, 5],
                    @record_batch.columns.collect(&:length))
     end
 
@@ -94,7 +131,7 @@ def test_slice
 
     def test_to_s
       assert_equal(<<-PRETTY_PRINT, @record_batch.to_s)
-visible: [true, false, true, false, true, false]
+visible: [true, false, true, false, true]
 valid: [false, true, false, true, false]
       PRETTY_PRINT
     end

From aaa04431acd998ebeef57f03456d95b534ad41f3 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sat, 25 Nov 2017 12:09:52 -0500
Subject: [PATCH 1281/1644] ARROW-1855: [GLib] Add workaround for build failure
 on macOS

Arrow-1.0.gir generation may be failed on macOS. Because Python's distutils' build option tries building universal binary implicitly. g-ir-scanner uses Python's distutils internally. We can disable it by defining ARCHFLAGS environment variable.

See also ARROW-1766.

If we put this workaround in c_glib/, users don't need to specify ARCHFLAGS environment variable explicitly.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1355 from kou/glib-add-macos-workaround and squashes the following commits:

ac4ecc6e [Kouhei Sutou] [GLib] Add workaround for macOS
---
 c_glib/arrow-glib/Makefile.am     |  5 +++++
 c_glib/arrow-gpu-glib/Makefile.am |  4 ++++
 c_glib/configure.ac               | 13 +++++++++++++
 ci/travis_before_script_c_glib.sh |  8 +-------
 4 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 5ecb1a6614268..e8073115d2a15 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -207,6 +207,11 @@ if HAVE_INTROSPECTION
 -include $(INTROSPECTION_MAKEFILE)
 INTROSPECTION_GIRS =
 INTROSPECTION_SCANNER_ARGS =
+INTROSPECTION_SCANNER_ENV =
+if OS_MACOS
+INTROSPECTION_SCANNER_ENV +=			\
+	ARCHFLAGS=
+endif
 INTROSPECTION_COMPILER_ARGS =
 
 Arrow-1.0.gir: libarrow-glib.la
diff --git a/c_glib/arrow-gpu-glib/Makefile.am b/c_glib/arrow-gpu-glib/Makefile.am
index ec9615987ee5e..11ffa5ecbe367 100644
--- a/c_glib/arrow-gpu-glib/Makefile.am
+++ b/c_glib/arrow-gpu-glib/Makefile.am
@@ -72,6 +72,10 @@ INTROSPECTION_GIRS =
 INTROSPECTION_SCANNER_ARGS =
 INTROSPECTION_SCANNER_ENV =			\
 	PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$${PKG_CONFIG_PATH}
+if OS_MACOS
+INTROSPECTION_SCANNER_ENV +=			\
+	ARCHFLAGS=
+endif
 INTROSPECTION_COMPILER_ARGS =			\
 	--includedir=$(abs_builddir)/../arrow-glib
 
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index c6fa0192c944d..f06a017a10ba1 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -34,6 +34,19 @@ AC_CONFIG_HEADERS([config.h])
 AM_INIT_AUTOMAKE([1.13 foreign])
 AM_SILENT_RULES([yes])
 
+AC_CANONICAL_HOST
+AC_MSG_CHECKING([for macOS])
+case "$host_os" in
+darwin*)
+  os_macos=yes
+  ;;
+*)
+  os_macos=no
+  ;;
+esac
+AC_MSG_RESULT([$os_macos])
+AM_CONDITIONAL(OS_MACOS, test "$os_macos" = "yes")
+
 AC_PROG_CC
 AC_PROG_CXX
 AX_CXX_COMPILE_STDCXX_11([ext], [mandatory])
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 99d05397a9f23..9d07e02a84302 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -97,13 +97,7 @@ if [ $BUILD_SYSTEM = "autotools" ]; then
   CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API"
 
   ./configure $CONFIGURE_OPTIONS
-
-  if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-    ARCHFLAGS="-arch x86_64" make -j4
-  else
-    make -j4
-  fi
-
+  make -j4
   make install
 else
   MESON_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL"

From 82e42c54b8dbd95e6b79a2c46aeb98e2a419e930 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 25 Nov 2017 12:18:36 -0500
Subject: [PATCH 1282/1644] ARROW-1777: [C++] Add ArrayData::Make static ctor
 for more convenient construction

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1353 from wesm/ARROW-1777 and squashes the following commits:

d086b1e4 [Wes McKinney] Add ArrayData::Make static ctor for more convenient construction
---
 cpp/src/arrow/array.cc                 | 53 ++++++++++++++------------
 cpp/src/arrow/array.h                  | 11 ++++++
 cpp/src/arrow/builder.cc               | 33 ++++++----------
 cpp/src/arrow/compute/compute-test.cc  |  2 +-
 cpp/src/arrow/compute/kernels/cast.cc  |  2 +-
 cpp/src/arrow/compute/kernels/hash.cc  |  7 ++--
 cpp/src/arrow/ipc/feather-test.cc      |  7 ++--
 cpp/src/arrow/ipc/feather.cc           |  2 +-
 cpp/src/arrow/python/numpy_to_arrow.cc |  7 +---
 python/pyarrow/public-api.pxi          |  2 +-
 10 files changed, 62 insertions(+), 64 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 4ceb071ac3338..4b1fabfdcd441 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -36,6 +36,20 @@
 
 namespace arrow {
 
+std::shared_ptr<ArrayData> ArrayData::Make(const std::shared_ptr<DataType>& type,
+                                           int64_t length,
+                                           std::vector<std::shared_ptr<Buffer>>&& buffers,
+                                           int64_t null_count, int64_t offset) {
+  return std::make_shared<ArrayData>(type, length, std::move(buffers), null_count,
+                                     offset);
+}
+
+std::shared_ptr<ArrayData> ArrayData::Make(const std::shared_ptr<DataType>& type,
+                                           int64_t length, int64_t null_count,
+                                           int64_t offset) {
+  return std::make_shared<ArrayData>(type, length, null_count, offset);
+}
+
 // ----------------------------------------------------------------------
 // Base array class
 
@@ -112,8 +126,7 @@ std::string Array::ToString() const {
 }
 
 NullArray::NullArray(int64_t length) {
-  BufferVector buffers = {nullptr};
-  SetData(std::make_shared<ArrayData>(null(), length, std::move(buffers), length));
+  SetData(ArrayData::Make(null(), length, {nullptr}, length));
 }
 
 // ----------------------------------------------------------------------
@@ -123,9 +136,7 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t le
                                const std::shared_ptr<Buffer>& data,
                                const std::shared_ptr<Buffer>& null_bitmap,
                                int64_t null_count, int64_t offset) {
-  BufferVector buffers = {null_bitmap, data};
-  SetData(
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+  SetData(ArrayData::Make(type, length, {null_bitmap, data}, null_count, offset));
 }
 
 const uint8_t* PrimitiveArray::raw_values() const {
@@ -165,9 +176,8 @@ ListArray::ListArray(const std::shared_ptr<DataType>& type, int64_t length,
                      const std::shared_ptr<Array>& values,
                      const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                      int64_t offset) {
-  BufferVector buffers = {null_bitmap, value_offsets};
   auto internal_data =
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
+      ArrayData::Make(type, length, {null_bitmap, value_offsets}, null_count, offset);
   internal_data->child_data.emplace_back(values->data());
   SetData(internal_data);
 }
@@ -219,9 +229,8 @@ Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPo
   }
 
   auto list_type = list(values.type());
-  auto internal_data =
-      std::make_shared<ArrayData>(list_type, num_offsets - 1, std::move(buffers),
-                                  offsets.null_count(), offsets.offset());
+  auto internal_data = ArrayData::Make(list_type, num_offsets - 1, std::move(buffers),
+                                       offsets.null_count(), offsets.offset());
   internal_data->child_data.push_back(values.data());
 
   *out = std::make_shared<ListArray>(internal_data);
@@ -276,9 +285,8 @@ BinaryArray::BinaryArray(const std::shared_ptr<DataType>& type, int64_t length,
                          const std::shared_ptr<Buffer>& data,
                          const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                          int64_t offset) {
-  BufferVector buffers = {null_bitmap, value_offsets, data};
-  SetData(
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+  SetData(ArrayData::Make(type, length, {null_bitmap, value_offsets, data}, null_count,
+                          offset));
 }
 
 StringArray::StringArray(const std::shared_ptr<ArrayData>& data) {
@@ -338,9 +346,7 @@ StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
                          const std::vector<std::shared_ptr<Array>>& children,
                          std::shared_ptr<Buffer> null_bitmap, int64_t null_count,
                          int64_t offset) {
-  BufferVector buffers = {null_bitmap};
-  SetData(
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset));
+  SetData(ArrayData::Make(type, length, {null_bitmap}, null_count, offset));
   for (const auto& child : children) {
     data_->child_data.push_back(child->data());
   }
@@ -384,9 +390,8 @@ UnionArray::UnionArray(const std::shared_ptr<DataType>& type, int64_t length,
                        const std::shared_ptr<Buffer>& value_offsets,
                        const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count,
                        int64_t offset) {
-  BufferVector buffers = {null_bitmap, type_ids, value_offsets};
-  auto internal_data =
-      std::make_shared<ArrayData>(type, length, std::move(buffers), null_count, offset);
+  auto internal_data = ArrayData::Make(
+      type, length, {null_bitmap, type_ids, value_offsets}, null_count, offset);
   for (const auto& child : children) {
     internal_data->child_data.push_back(child->data());
   }
@@ -416,9 +421,8 @@ Status UnionArray::MakeDense(const Array& type_ids, const Array& value_offsets,
                           static_cast<const UInt8Array&>(type_ids).values(),
                           static_cast<const Int32Array&>(value_offsets).values()};
   auto union_type = union_(children, UnionMode::DENSE);
-  auto internal_data =
-      std::make_shared<ArrayData>(union_type, type_ids.length(), std::move(buffers),
-                                  type_ids.null_count(), type_ids.offset());
+  auto internal_data = ArrayData::Make(union_type, type_ids.length(), std::move(buffers),
+                                       type_ids.null_count(), type_ids.offset());
   for (const auto& child : children) {
     internal_data->child_data.push_back(child->data());
   }
@@ -435,9 +439,8 @@ Status UnionArray::MakeSparse(const Array& type_ids,
   BufferVector buffers = {type_ids.null_bitmap(),
                           static_cast<const UInt8Array&>(type_ids).values(), nullptr};
   auto union_type = union_(children, UnionMode::SPARSE);
-  auto internal_data =
-      std::make_shared<ArrayData>(union_type, type_ids.length(), std::move(buffers),
-                                  type_ids.null_count(), type_ids.offset());
+  auto internal_data = ArrayData::Make(union_type, type_ids.length(), std::move(buffers),
+                                       type_ids.null_count(), type_ids.offset());
   for (const auto& child : children) {
     internal_data->child_data.push_back(child->data());
     if (child->length() != type_ids.length()) {
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index dda9dd38be4a7..ec5381d6e28a3 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -104,6 +104,17 @@ struct ARROW_EXPORT ArrayData {
     this->buffers = std::move(buffers);
   }
 
+  static std::shared_ptr<ArrayData> Make(const std::shared_ptr<DataType>& type,
+                                         int64_t length,
+                                         std::vector<std::shared_ptr<Buffer>>&& buffers,
+                                         int64_t null_count = kUnknownNullCount,
+                                         int64_t offset = 0);
+
+  static std::shared_ptr<ArrayData> Make(const std::shared_ptr<DataType>& type,
+                                         int64_t length,
+                                         int64_t null_count = kUnknownNullCount,
+                                         int64_t offset = 0);
+
   // Move constructor
   ArrayData(ArrayData&& other) noexcept
       : type(std::move(other.type)),
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index a42f9024545ec..4d7fd5ff840ea 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -219,8 +219,7 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) {
 // Null builder
 
 Status NullBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
-  BufferVector buffers = {nullptr};
-  *out = std::make_shared<ArrayData>(null(), length_, std::move(buffers), length_);
+  *out = ArrayData::Make(null(), length_, {nullptr}, length_);
   length_ = null_count_ = 0;
   return Status::OK();
 }
@@ -314,8 +313,7 @@ Status PrimitiveBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
-  BufferVector buffers = {null_bitmap_, data_};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, data_}, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -404,9 +402,7 @@ Status AdaptiveIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
       return Status::NotImplemented("Only ints of size 1,2,4,8 are supported");
   }
 
-  BufferVector buffers = {null_bitmap_, data_};
-  *out =
-      std::make_shared<ArrayData>(output_type, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(output_type, length_, {null_bitmap_, data_}, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -562,9 +558,7 @@ Status AdaptiveUIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
       return Status::NotImplemented("Only ints of size 1,2,4,8 are supported");
   }
 
-  BufferVector buffers = {null_bitmap_, data_};
-  *out =
-      std::make_shared<ArrayData>(output_type, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(output_type, length_, {null_bitmap_, data_}, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -741,8 +735,7 @@ Status BooleanBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
     // Trim buffers
     RETURN_NOT_OK(data_->Resize(bytes_required));
   }
-  BufferVector buffers = {null_bitmap_, data_};
-  *out = std::make_shared<ArrayData>(boolean(), length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(boolean(), length_, {null_bitmap_, data_}, null_count_);
 
   data_ = null_bitmap_ = nullptr;
   capacity_ = length_ = null_count_ = 0;
@@ -828,8 +821,7 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
 
-  BufferVector buffers = {null_bitmap_, data};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count_);
   return Status::OK();
 }
 
@@ -896,8 +888,7 @@ Status ListBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
     RETURN_NOT_OK(value_builder_->FinishInternal(&items));
   }
 
-  BufferVector buffers = {null_bitmap_, offsets};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, offsets}, null_count_);
   (*out)->child_data.emplace_back(std::move(items));
   Reset();
   return Status::OK();
@@ -969,8 +960,8 @@ Status BinaryBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   RETURN_NOT_OK(offsets_builder_.Finish(&offsets));
   RETURN_NOT_OK(value_data_builder_.Finish(&value_data));
 
-  BufferVector buffers = {null_bitmap_, offsets, value_data};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_, 0);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, offsets, value_data}, null_count_,
+                         0);
   Reset();
   return Status::OK();
 }
@@ -1040,8 +1031,7 @@ Status FixedSizeBinaryBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(byte_builder_.Finish(&data));
 
-  BufferVector buffers = {null_bitmap_, data};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count_);
   return Status::OK();
 }
 
@@ -1060,8 +1050,7 @@ StructBuilder::StructBuilder(const std::shared_ptr<DataType>& type, MemoryPool*
 }
 
 Status StructBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
-  BufferVector buffers = {null_bitmap_};
-  *out = std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count_);
+  *out = ArrayData::Make(type_, length_, {null_bitmap_}, null_count_);
 
   (*out)->child_data.resize(field_builders_.size());
   for (size_t i = 0; i < field_builders_.size(); ++i) {
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 96edd8f01ed38..d5158978c8de8 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -688,7 +688,7 @@ TEST_F(TestCast, PreallocatedMemory) {
   std::unique_ptr<UnaryKernel> kernel;
   ASSERT_OK(GetCastFunction(*int32(), out_type, options, &kernel));
 
-  auto out_data = std::make_shared<ArrayData>(out_type, length);
+  auto out_data = ArrayData::Make(out_type, length);
 
   shared_ptr<Buffer> out_values;
   ASSERT_OK(this->ctx_.Allocate(length * sizeof(int64_t), &out_values));
diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc
index d595d2ea507c1..d48d669922aa6 100644
--- a/cpp/src/arrow/compute/kernels/cast.cc
+++ b/cpp/src/arrow/compute/kernels/cast.cc
@@ -747,7 +747,7 @@ class CastKernel : public UnaryKernel {
     ArrayData* result;
 
     if (out->kind() == Datum::NONE) {
-      out->value = std::make_shared<ArrayData>(out_type_, in_data.length);
+      out->value = ArrayData::Make(out_type_, in_data.length);
     }
 
     result = out->array().get();
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index e47759d4d4706..66c907369e3ee 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -342,8 +342,7 @@ class HashTableKernel<Type, Action, enable_if_has_c_type<Type>> : public HashTab
     auto dict_data = dict_.buffer;
     RETURN_NOT_OK(dict_data->Resize(dict_.size * sizeof(T), false));
 
-    BufferVector buffers = {nullptr, dict_data};
-    *out = std::make_shared<ArrayData>(type_, dict_.size, std::move(buffers), 0);
+    *out = ArrayData::Make(type_, dict_.size, {nullptr, dict_data}, 0);
     return Status::OK();
   }
 
@@ -528,7 +527,7 @@ class HashTableKernel<Type, Action, enable_if_binary<Type>> : public HashTable {
     RETURN_NOT_OK(dict_offsets_.Finish(&buffers[1]));
     RETURN_NOT_OK(dict_data_.Finish(&buffers[2]));
 
-    *out = std::make_shared<ArrayData>(type_, dict_size_, std::move(buffers), 0);
+    *out = ArrayData::Make(type_, dict_size_, std::move(buffers), 0);
     return Status::OK();
   }
 
@@ -634,7 +633,7 @@ class HashTableKernel<Type, Action, enable_if_fixed_size_binary<Type>>
     BufferVector buffers = {nullptr, nullptr};
     RETURN_NOT_OK(dict_data_.Finish(&buffers[1]));
 
-    *out = std::make_shared<ArrayData>(type_, dict_size_, std::move(buffers), 0);
+    *out = ArrayData::Make(type_, dict_size_, std::move(buffers), 0);
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index e3de17f1f75cd..8ec3b0e4ae8b2 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -366,15 +366,14 @@ TEST_F(TestTableWriter, TimeTypes) {
   ArrayFromVector<Date32Type, int32_t>(is_valid, date_values_vec, &date_array);
 
   const auto& prim_values = static_cast<const PrimitiveArray&>(*values);
-  std::vector<std::shared_ptr<Buffer>> buffers = {prim_values.null_bitmap(),
-                                                  prim_values.values()};
+  BufferVector buffers = {prim_values.null_bitmap(), prim_values.values()};
 
   std::vector<std::shared_ptr<ArrayData>> arrays;
   arrays.push_back(date_array->data());
 
   for (int i = 1; i < schema->num_fields(); ++i) {
-    arrays.emplace_back(std::make_shared<ArrayData>(
-        schema->field(i)->type(), values->length(), buffers, values->null_count(), 0));
+    arrays.emplace_back(ArrayData::Make(schema->field(i)->type(), values->length(),
+                                        BufferVector(buffers), values->null_count(), 0));
   }
 
   auto batch = RecordBatch::Make(schema, values->length(), std::move(arrays));
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index 077dc39305b09..d339449c084d2 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -371,7 +371,7 @@ class TableReader::TableReaderImpl {
     buffers.push_back(SliceBuffer(buffer, offset, buffer->size() - offset));
 
     auto arr_data =
-        std::make_shared<ArrayData>(type, meta->length(), buffers, meta->null_count());
+        ArrayData::Make(type, meta->length(), std::move(buffers), meta->null_count());
     *out = MakeArray(arr_data);
     return Status::OK();
   }
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 0d2df93151913..0c0d1a9b3fdf2 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -392,9 +392,7 @@ class NumPyConverter {
       null_count = ValuesToBitmap<traits::npy_type>(arr_, null_bitmap_data_);
     }
 
-    BufferVector buffers = {null_bitmap_, data};
-    auto arr_data =
-        std::make_shared<ArrayData>(type_, length_, std::move(buffers), null_count, 0);
+    auto arr_data = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count, 0);
     return PushArray(arr_data);
   }
 
@@ -472,8 +470,7 @@ Status CastBuffer(const std::shared_ptr<DataType>& in_type,
                   const std::shared_ptr<DataType>& out_type, MemoryPool* pool,
                   std::shared_ptr<Buffer>* out) {
   // Must cast
-  std::vector<std::shared_ptr<Buffer>> buffers = {valid_bitmap, input};
-  auto tmp_data = std::make_shared<ArrayData>(in_type, length, buffers, null_count);
+  auto tmp_data = ArrayData::Make(in_type, length, {valid_bitmap, input}, null_count);
 
   std::shared_ptr<Array> tmp_array = MakeArray(tmp_data);
   std::shared_ptr<Array> casted_array;
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 9776f2ad76e3c..2fdb606a7d1c7 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -44,7 +44,7 @@ cdef public api object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf):
 
 
 cdef public api object pyarrow_wrap_resizable_buffer(
-    const shared_ptr[CResizableBuffer]& buf):
+        const shared_ptr[CResizableBuffer]& buf):
     cdef ResizableBuffer result = ResizableBuffer()
     result.init_rz(buf)
     return result

From b20beff1f801e888fa0deb5d48331634d0213cff Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sat, 25 Nov 2017 17:33:33 -0500
Subject: [PATCH 1283/1644] ARROW-1836: [C++] Remove deprecated static_visitor
 struct to avoid msvc C4996 warning

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #1359 from MaxRis/ARROW-1836 and squashes the following commits:

bbd5850b [Max Risuhin] ARROW-1836: [C++] Remove deprecated static_visitor struct to avoid msvc C4996 warning
---
 cpp/cmake_modules/SetupCxxFlags.cmake |  9 ---------
 cpp/src/arrow/util/variant.h          | 10 ----------
 2 files changed, 19 deletions(-)

diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 6b0974b3d4c19..4b1950f7a17bd 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -68,9 +68,6 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
-
-    # MSVC version of -Wno-deprecated
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
 -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
@@ -118,9 +115,6 @@ elseif ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "EVERYTHING")
     # /wdnnnn disables a warning where "nnnn" is a warning number
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}  /WX")
-
-    # MSVC version of -Wno-deprecated
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic")
     # Treat all compiler warnings as errors
@@ -140,9 +134,6 @@ else()
     # /wdnnnn disables a warning where "nnnn" is a warning number
     string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
-
-    # MSVC version of -Wno-deprecated
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
   elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
diff --git a/cpp/src/arrow/util/variant.h b/cpp/src/arrow/util/variant.h
index 923a8685f0cdb..9bfc52cb1c9a2 100644
--- a/cpp/src/arrow/util/variant.h
+++ b/cpp/src/arrow/util/variant.h
@@ -101,16 +101,6 @@ class bad_variant_access : public std::runtime_error
 
 }; // class bad_variant_access
 
-template <typename R = void>
-struct ARROW_VARIANT_DEPRECATED static_visitor
-{
-    using result_type = R;
-
-protected:
-    static_visitor() {}
-    ~static_visitor() {}
-};
-
 #if !defined(ARROW_VARIANT_MINIMIZE_SIZE)
 using type_index_t = unsigned int;
 #else

From ad82c9abaf1e564cd725591e543f828e99a3eeb2 Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Sat, 25 Nov 2017 15:43:01 -0800
Subject: [PATCH 1284/1644] ARROW-1853: [Plasma] Fix off-by-one error in retry
 processing

When a user construct a plasma client that should not perform a single retry, by passing `num_retries = 0`, nothing happens due to an off-by-one error in the retry processing. This commit fixes the issue and polishes the error output.

Author: Matthias Vallentin <vallentin@icir.org>

Closes #1358 from wesm/topic/plasma-num-retries-fix and squashes the following commits:

082ee4c [Matthias Vallentin] Fix off-by-one error in retry processing
---
 cpp/src/plasma/io.cc | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
index 2228ad94c6346..2cba8970e26ae 100644
--- a/cpp/src/plasma/io.cc
+++ b/cpp/src/plasma/io.cc
@@ -156,19 +156,14 @@ Status ConnectIpcSocketRetry(const std::string& pathname, int num_retries,
   if (timeout < 0) {
     timeout = CONNECT_TIMEOUT_MS;
   }
-
-  *fd = -1;
-  for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
-    *fd = connect_ipc_sock(pathname);
-    if (*fd >= 0) {
-      break;
-    }
-    if (num_attempts == 0) {
-      ARROW_LOG(ERROR) << "Connection to IPC socket failed for pathname " << pathname
-                       << ", retrying " << num_retries << " times";
-    }
+  *fd = connect_ipc_sock(pathname);
+  while (*fd < 0 && num_retries > 0) {
+    ARROW_LOG(ERROR) << "Connection to IPC socket failed for pathname " << pathname
+                     << ", retrying " << num_retries << " more times";
     /* Sleep for timeout milliseconds. */
     usleep(static_cast<int>(timeout * 1000));
+    *fd = connect_ipc_sock(pathname);
+    --num_retries;
   }
   /* If we could not connect to the socket, exit. */
   if (*fd == -1) {

From bf1cf3befc5a5d1e7eee92b6ab4b0935c9821ce5 Mon Sep 17 00:00:00 2001
From: Andrew Andrade <andrew@andrewandrade.ca>
Date: Sun, 26 Nov 2017 19:02:52 +0000
Subject: [PATCH 1285/1644] [Python] Add more detail to development docs
 (#1356)

* updated file path to brew install for repos dir

* added information about bundled wheel build
---
 python/doc/source/development.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 885613725a10a..1c1a308fc4a56 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -108,7 +108,7 @@ building Arrow C++:
 
 .. code-block:: shell
 
-   brew update && brew bundle --file=python/Brewfile
+   brew update && brew bundle --file=arrow/python/Brewfile
 
 On Debian/Ubuntu, you need the following minimal set of dependencies. All other
 dependencies will be automatically built by Arrow's third-party toolchain.
@@ -230,7 +230,7 @@ You should be able to run the unit tests with:
 
    ====================== 181 passed, 17 skipped in 0.98 seconds ===========
 
-You can build a wheel by running:
+To build a self-contained wheel (include Arrow C++ and Parquet C++), one can set `--bundle-arrow-cpp`:
 
 .. code-block:: shell
 

From ebb6c764687c197fbd5269b6500568d11dafaf31 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 26 Nov 2017 14:17:28 -0500
Subject: [PATCH 1286/1644] ARROW-1859: [GLib] Add GArrowDictionaryDataType

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1361 from kou/glib-dictionary-data-type and squashes the following commits:

6ccce1f8 [Kouhei Sutou] [GLib] Add GArrowDictionaryDataType
---
 c_glib/arrow-glib/Makefile.am                 |    6 +-
 c_glib/arrow-glib/array.h                     | 1225 +----------------
 c_glib/arrow-glib/array.hpp                   |    8 +-
 .../arrow-glib/{array.cpp => basic-array.cpp} |  206 +--
 c_glib/arrow-glib/basic-array.h               | 1115 +++++++++++++++
 c_glib/arrow-glib/basic-array.hpp             |   27 +
 c_glib/arrow-glib/basic-data-type.cpp         |   38 +-
 c_glib/arrow-glib/basic-data-type.h           |  104 +-
 c_glib/arrow-glib/composite-array.cpp         |  237 ++++
 c_glib/arrow-glib/composite-array.h           |  132 ++
 c_glib/arrow-glib/composite-data-type.cpp     |   86 ++
 c_glib/arrow-glib/composite-data-type.h       |   25 +
 c_glib/arrow-glib/meson.build                 |    6 +-
 c_glib/doc/reference/arrow-glib-docs.xml      |    3 +-
 c_glib/test/test-dictionary-data-type.rb      |   56 +
 15 files changed, 1765 insertions(+), 1509 deletions(-)
 rename c_glib/arrow-glib/{array.cpp => basic-array.cpp} (90%)
 create mode 100644 c_glib/arrow-glib/basic-array.h
 create mode 100644 c_glib/arrow-glib/basic-array.hpp
 create mode 100644 c_glib/arrow-glib/composite-array.cpp
 create mode 100644 c_glib/arrow-glib/composite-array.h
 create mode 100644 c_glib/test/test-dictionary-data-type.rb

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index e8073115d2a15..39222a5480162 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -45,10 +45,12 @@ libarrow_glib_la_headers =			\
 	array.h					\
 	array-builder.h				\
 	arrow-glib.h				\
+	basic-array.h				\
 	basic-data-type.h			\
 	buffer.h				\
 	chunked-array.h				\
 	column.h				\
+	composite-array.h			\
 	composite-data-type.h			\
 	data-type.h				\
 	error.h					\
@@ -86,12 +88,13 @@ libarrow_glib_la_generated_sources =		\
 	$(libarrow_glib_la_generated_headers)
 
 libarrow_glib_la_sources =			\
-	array.cpp				\
 	array-builder.cpp			\
+	basic-array.cpp				\
 	basic-data-type.cpp			\
 	buffer.cpp				\
 	chunked-array.cpp			\
 	column.cpp				\
+	composite-array.cpp			\
 	composite-data-type.cpp			\
 	error.cpp				\
 	field.cpp				\
@@ -125,6 +128,7 @@ libarrow_glib_la_cpp_headers =			\
 	array.hpp				\
 	array-builder.hpp			\
 	arrow-glib.hpp				\
+	basic-array.hpp				\
 	basic-data-type.hpp			\
 	buffer.hpp				\
 	chunked-array.hpp			\
diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h
index e988a8aae931d..9a845597d4f0f 100644
--- a/c_glib/arrow-glib/array.h
+++ b/c_glib/arrow-glib/array.h
@@ -19,1226 +19,5 @@
 
 #pragma once
 
-#include <arrow-glib/buffer.h>
-#include <arrow-glib/compute.h>
-#include <arrow-glib/data-type.h>
-
-G_BEGIN_DECLS
-
-#define GARROW_TYPE_ARRAY \
-  (garrow_array_get_type())
-#define GARROW_ARRAY(obj) \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj), GARROW_TYPE_ARRAY, GArrowArray))
-#define GARROW_ARRAY_CLASS(klass) \
-  (G_TYPE_CHECK_CLASS_CAST((klass), GARROW_TYPE_ARRAY, GArrowArrayClass))
-#define GARROW_IS_ARRAY(obj) \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_ARRAY))
-#define GARROW_IS_ARRAY_CLASS(klass) \
-  (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_ARRAY))
-#define GARROW_ARRAY_GET_CLASS(obj) \
-  (G_TYPE_INSTANCE_GET_CLASS((obj), GARROW_TYPE_ARRAY, GArrowArrayClass))
-
-typedef struct _GArrowArray         GArrowArray;
-typedef struct _GArrowArrayClass    GArrowArrayClass;
-
-/**
- * GArrowArray:
- *
- * It wraps `arrow::Array`.
- */
-struct _GArrowArray
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
-struct _GArrowArrayClass
-{
-  GObjectClass parent_class;
-};
-
-GType          garrow_array_get_type    (void) G_GNUC_CONST;
-
-gboolean       garrow_array_equal       (GArrowArray *array,
-                                         GArrowArray *other_array);
-gboolean       garrow_array_equal_approx(GArrowArray *array,
-                                         GArrowArray *other_array);
-gboolean       garrow_array_equal_range (GArrowArray *array,
-                                         gint64 start_index,
-                                         GArrowArray *other_array,
-                                         gint64 other_start_index,
-                                         gint64 end_index);
-
-gboolean       garrow_array_is_null     (GArrowArray *array,
-                                         gint64 i);
-gboolean       garrow_array_is_valid    (GArrowArray *array,
-                                         gint64 i);
-gint64         garrow_array_get_length  (GArrowArray *array);
-gint64         garrow_array_get_offset  (GArrowArray *array);
-gint64         garrow_array_get_n_nulls (GArrowArray *array);
-GArrowBuffer  *garrow_array_get_null_bitmap(GArrowArray *array);
-GArrowDataType *garrow_array_get_value_data_type(GArrowArray *array);
-GArrowType     garrow_array_get_value_type(GArrowArray *array);
-GArrowArray   *garrow_array_slice       (GArrowArray *array,
-                                         gint64 offset,
-                                         gint64 length);
-gchar         *garrow_array_to_string   (GArrowArray *array,
-                                         GError **error);
-
-GArrowArray   *garrow_array_cast        (GArrowArray *array,
-                                         GArrowDataType *target_data_type,
-                                         GArrowCastOptions *options,
-                                         GError **error);
-
-#define GARROW_TYPE_NULL_ARRAY                  \
-  (garrow_null_array_get_type())
-#define GARROW_NULL_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_NULL_ARRAY,   \
-                              GArrowNullArray))
-#define GARROW_NULL_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_NULL_ARRAY,      \
-                           GArrowNullArrayClass))
-#define GARROW_IS_NULL_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_NULL_ARRAY))
-#define GARROW_IS_NULL_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_NULL_ARRAY))
-#define GARROW_NULL_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_NULL_ARRAY,    \
-                             GArrowNullArrayClass))
-
-typedef struct _GArrowNullArray         GArrowNullArray;
-typedef struct _GArrowNullArrayClass    GArrowNullArrayClass;
-
-/**
- * GArrowNullArray:
- *
- * It wraps `arrow::NullArray`.
- */
-struct _GArrowNullArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowNullArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_null_array_get_type(void) G_GNUC_CONST;
-
-GArrowNullArray *garrow_null_array_new(gint64 length);
-
-
-#define GARROW_TYPE_PRIMITIVE_ARRAY             \
-  (garrow_primitive_array_get_type())
-#define GARROW_PRIMITIVE_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_PRIMITIVE_ARRAY,      \
-                              GArrowPrimitiveArray))
-#define GARROW_PRIMITIVE_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_PRIMITIVE_ARRAY, \
-                           GArrowPrimitiveArrayClass))
-#define GARROW_IS_PRIMITIVE_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_PRIMITIVE_ARRAY))
-#define GARROW_IS_PRIMITIVE_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_PRIMITIVE_ARRAY))
-#define GARROW_PRIMITIVE_ARRAY_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_PRIMITIVE_ARRAY,       \
-                             GArrowPrimitiveArrayClass))
-
-typedef struct _GArrowPrimitiveArray         GArrowPrimitiveArray;
-typedef struct _GArrowPrimitiveArrayClass    GArrowPrimitiveArrayClass;
-
-/**
- * GArrowPrimitiveArray:
- *
- * It wraps `arrow::PrimitiveArray`.
- */
-struct _GArrowPrimitiveArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowPrimitiveArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_primitive_array_get_type(void) G_GNUC_CONST;
-
-GArrowBuffer *garrow_primitive_array_get_buffer(GArrowPrimitiveArray *array);
-
-
-#define GARROW_TYPE_BOOLEAN_ARRAY               \
-  (garrow_boolean_array_get_type())
-#define GARROW_BOOLEAN_ARRAY(obj)                               \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BOOLEAN_ARRAY,        \
-                              GArrowBooleanArray))
-#define GARROW_BOOLEAN_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_BOOLEAN_ARRAY,   \
-                           GArrowBooleanArrayClass))
-#define GARROW_IS_BOOLEAN_ARRAY(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BOOLEAN_ARRAY))
-#define GARROW_IS_BOOLEAN_ARRAY_CLASS(klass)            \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_BOOLEAN_ARRAY))
-#define GARROW_BOOLEAN_ARRAY_GET_CLASS(obj)             \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_BOOLEAN_ARRAY, \
-                             GArrowBooleanArrayClass))
-
-typedef struct _GArrowBooleanArray         GArrowBooleanArray;
-typedef struct _GArrowBooleanArrayClass    GArrowBooleanArrayClass;
-
-/**
- * GArrowBooleanArray:
- *
- * It wraps `arrow::BooleanArray`.
- */
-struct _GArrowBooleanArray
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowBooleanArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType          garrow_boolean_array_get_type  (void) G_GNUC_CONST;
-
-GArrowBooleanArray *garrow_boolean_array_new(gint64 length,
-                                             GArrowBuffer *data,
-                                             GArrowBuffer *null_bitmap,
-                                             gint64 n_nulls);
-
-gboolean       garrow_boolean_array_get_value (GArrowBooleanArray *array,
-                                               gint64 i);
-gboolean      *garrow_boolean_array_get_values(GArrowBooleanArray *array,
-                                               gint64 *length);
-
-
-#define GARROW_TYPE_INT8_ARRAY                  \
-  (garrow_int8_array_get_type())
-#define GARROW_INT8_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT8_ARRAY,   \
-                              GArrowInt8Array))
-#define GARROW_INT8_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT8_ARRAY,      \
-                           GArrowInt8ArrayClass))
-#define GARROW_IS_INT8_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT8_ARRAY))
-#define GARROW_IS_INT8_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT8_ARRAY))
-#define GARROW_INT8_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT8_ARRAY,    \
-                             GArrowInt8ArrayClass))
-
-typedef struct _GArrowInt8Array         GArrowInt8Array;
-typedef struct _GArrowInt8ArrayClass    GArrowInt8ArrayClass;
-
-/**
- * GArrowInt8Array:
- *
- * It wraps `arrow::Int8Array`.
- */
-struct _GArrowInt8Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowInt8ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_int8_array_get_type(void) G_GNUC_CONST;
-
-GArrowInt8Array *garrow_int8_array_new(gint64 length,
-                                       GArrowBuffer *data,
-                                       GArrowBuffer *null_bitmap,
-                                       gint64 n_nulls);
-
-gint8 garrow_int8_array_get_value(GArrowInt8Array *array,
-                                  gint64 i);
-const gint8 *garrow_int8_array_get_values(GArrowInt8Array *array,
-                                          gint64 *length);
-
-
-#define GARROW_TYPE_UINT8_ARRAY                 \
-  (garrow_uint8_array_get_type())
-#define GARROW_UINT8_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT8_ARRAY,  \
-                              GArrowUInt8Array))
-#define GARROW_UINT8_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT8_ARRAY,     \
-                           GArrowUInt8ArrayClass))
-#define GARROW_IS_UINT8_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_UINT8_ARRAY))
-#define GARROW_IS_UINT8_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT8_ARRAY))
-#define GARROW_UINT8_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT8_ARRAY,   \
-                             GArrowUInt8ArrayClass))
-
-typedef struct _GArrowUInt8Array         GArrowUInt8Array;
-typedef struct _GArrowUInt8ArrayClass    GArrowUInt8ArrayClass;
-
-/**
- * GArrowUInt8Array:
- *
- * It wraps `arrow::UInt8Array`.
- */
-struct _GArrowUInt8Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowUInt8ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_uint8_array_get_type(void) G_GNUC_CONST;
-
-GArrowUInt8Array *garrow_uint8_array_new(gint64 length,
-                                         GArrowBuffer *data,
-                                         GArrowBuffer *null_bitmap,
-                                         gint64 n_nulls);
-
-guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array,
-                                    gint64 i);
-const guint8 *garrow_uint8_array_get_values(GArrowUInt8Array *array,
-                                            gint64 *length);
-
-
-#define GARROW_TYPE_INT16_ARRAY                  \
-  (garrow_int16_array_get_type())
-#define GARROW_INT16_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT16_ARRAY,  \
-                              GArrowInt16Array))
-#define GARROW_INT16_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT16_ARRAY,     \
-                           GArrowInt16ArrayClass))
-#define GARROW_IS_INT16_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT16_ARRAY))
-#define GARROW_IS_INT16_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT16_ARRAY))
-#define GARROW_INT16_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT16_ARRAY,   \
-                             GArrowInt16ArrayClass))
-
-typedef struct _GArrowInt16Array         GArrowInt16Array;
-typedef struct _GArrowInt16ArrayClass    GArrowInt16ArrayClass;
-
-/**
- * GArrowInt16Array:
- *
- * It wraps `arrow::Int16Array`.
- */
-struct _GArrowInt16Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowInt16ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_int16_array_get_type(void) G_GNUC_CONST;
-
-GArrowInt16Array *garrow_int16_array_new(gint64 length,
-                                         GArrowBuffer *data,
-                                         GArrowBuffer *null_bitmap,
-                                         gint64 n_nulls);
-
-gint16 garrow_int16_array_get_value(GArrowInt16Array *array,
-                                    gint64 i);
-const gint16 *garrow_int16_array_get_values(GArrowInt16Array *array,
-                                            gint64 *length);
-
-
-#define GARROW_TYPE_UINT16_ARRAY                 \
-  (garrow_uint16_array_get_type())
-#define GARROW_UINT16_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT16_ARRAY, \
-                              GArrowUInt16Array))
-#define GARROW_UINT16_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT16_ARRAY,    \
-                           GArrowUInt16ArrayClass))
-#define GARROW_IS_UINT16_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT16_ARRAY))
-#define GARROW_IS_UINT16_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT16_ARRAY))
-#define GARROW_UINT16_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT16_ARRAY,  \
-                             GArrowUInt16ArrayClass))
-
-typedef struct _GArrowUInt16Array         GArrowUInt16Array;
-typedef struct _GArrowUInt16ArrayClass    GArrowUInt16ArrayClass;
-
-/**
- * GArrowUInt16Array:
- *
- * It wraps `arrow::UInt16Array`.
- */
-struct _GArrowUInt16Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowUInt16ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_uint16_array_get_type(void) G_GNUC_CONST;
-
-GArrowUInt16Array *garrow_uint16_array_new(gint64 length,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array,
-                                      gint64 i);
-const guint16 *garrow_uint16_array_get_values(GArrowUInt16Array *array,
-                                              gint64 *length);
-
-
-#define GARROW_TYPE_INT32_ARRAY                 \
-  (garrow_int32_array_get_type())
-#define GARROW_INT32_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT32_ARRAY,  \
-                              GArrowInt32Array))
-#define GARROW_INT32_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT32_ARRAY,     \
-                           GArrowInt32ArrayClass))
-#define GARROW_IS_INT32_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT32_ARRAY))
-#define GARROW_IS_INT32_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT32_ARRAY))
-#define GARROW_INT32_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT32_ARRAY,   \
-                             GArrowInt32ArrayClass))
-
-typedef struct _GArrowInt32Array         GArrowInt32Array;
-typedef struct _GArrowInt32ArrayClass    GArrowInt32ArrayClass;
-
-/**
- * GArrowInt32Array:
- *
- * It wraps `arrow::Int32Array`.
- */
-struct _GArrowInt32Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowInt32ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_int32_array_get_type(void) G_GNUC_CONST;
-
-GArrowInt32Array *garrow_int32_array_new(gint64 length,
-                                         GArrowBuffer *data,
-                                         GArrowBuffer *null_bitmap,
-                                         gint64 n_nulls);
-
-gint32 garrow_int32_array_get_value(GArrowInt32Array *array,
-                                    gint64 i);
-const gint32 *garrow_int32_array_get_values(GArrowInt32Array *array,
-                                            gint64 *length);
-
-
-#define GARROW_TYPE_UINT32_ARRAY                \
-  (garrow_uint32_array_get_type())
-#define GARROW_UINT32_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT32_ARRAY, \
-                              GArrowUInt32Array))
-#define GARROW_UINT32_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT32_ARRAY,    \
-                           GArrowUInt32ArrayClass))
-#define GARROW_IS_UINT32_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT32_ARRAY))
-#define GARROW_IS_UINT32_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT32_ARRAY))
-#define GARROW_UINT32_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT32_ARRAY,  \
-                             GArrowUInt32ArrayClass))
-
-typedef struct _GArrowUInt32Array         GArrowUInt32Array;
-typedef struct _GArrowUInt32ArrayClass    GArrowUInt32ArrayClass;
-
-/**
- * GArrowUInt32Array:
- *
- * It wraps `arrow::UInt32Array`.
- */
-struct _GArrowUInt32Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowUInt32ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_uint32_array_get_type(void) G_GNUC_CONST;
-
-GArrowUInt32Array *garrow_uint32_array_new(gint64 length,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array,
-                                      gint64 i);
-const guint32 *garrow_uint32_array_get_values(GArrowUInt32Array *array,
-                                              gint64 *length);
-
-
-#define GARROW_TYPE_INT64_ARRAY                 \
-  (garrow_int64_array_get_type())
-#define GARROW_INT64_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT64_ARRAY,  \
-                              GArrowInt64Array))
-#define GARROW_INT64_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT64_ARRAY,     \
-                           GArrowInt64ArrayClass))
-#define GARROW_IS_INT64_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT64_ARRAY))
-#define GARROW_IS_INT64_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT64_ARRAY))
-#define GARROW_INT64_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT64_ARRAY,   \
-                             GArrowInt64ArrayClass))
-
-typedef struct _GArrowInt64Array         GArrowInt64Array;
-typedef struct _GArrowInt64ArrayClass    GArrowInt64ArrayClass;
-
-/**
- * GArrowInt64Array:
- *
- * It wraps `arrow::Int64Array`.
- */
-struct _GArrowInt64Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowInt64ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_int64_array_get_type(void) G_GNUC_CONST;
-
-GArrowInt64Array *garrow_int64_array_new(gint64 length,
-                                         GArrowBuffer *data,
-                                         GArrowBuffer *null_bitmap,
-                                         gint64 n_nulls);
-
-gint64 garrow_int64_array_get_value(GArrowInt64Array *array,
-                                    gint64 i);
-const gint64 *garrow_int64_array_get_values(GArrowInt64Array *array,
-                                            gint64 *length);
-
-
-#define GARROW_TYPE_UINT64_ARRAY                \
-  (garrow_uint64_array_get_type())
-#define GARROW_UINT64_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT64_ARRAY, \
-                              GArrowUInt64Array))
-#define GARROW_UINT64_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT64_ARRAY,    \
-                           GArrowUInt64ArrayClass))
-#define GARROW_IS_UINT64_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT64_ARRAY))
-#define GARROW_IS_UINT64_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT64_ARRAY))
-#define GARROW_UINT64_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT64_ARRAY,  \
-                             GArrowUInt64ArrayClass))
-
-typedef struct _GArrowUInt64Array         GArrowUInt64Array;
-typedef struct _GArrowUInt64ArrayClass    GArrowUInt64ArrayClass;
-
-/**
- * GArrowUInt64Array:
- *
- * It wraps `arrow::UInt64Array`.
- */
-struct _GArrowUInt64Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowUInt64ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_uint64_array_get_type(void) G_GNUC_CONST;
-
-GArrowUInt64Array *garrow_uint64_array_new(gint64 length,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array,
-                                      gint64 i);
-const guint64 *garrow_uint64_array_get_values(GArrowUInt64Array *array,
-                                              gint64 *length);
-
-
-#define GARROW_TYPE_FLOAT_ARRAY                 \
-  (garrow_float_array_get_type())
-#define GARROW_FLOAT_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_FLOAT_ARRAY,  \
-                              GArrowFloatArray))
-#define GARROW_FLOAT_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_FLOAT_ARRAY,     \
-                           GArrowFloatArrayClass))
-#define GARROW_IS_FLOAT_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_FLOAT_ARRAY))
-#define GARROW_IS_FLOAT_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_FLOAT_ARRAY))
-#define GARROW_FLOAT_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_FLOAT_ARRAY,   \
-                             GArrowFloatArrayClass))
-
-typedef struct _GArrowFloatArray         GArrowFloatArray;
-typedef struct _GArrowFloatArrayClass    GArrowFloatArrayClass;
-
-/**
- * GArrowFloatArray:
- *
- * It wraps `arrow::FloatArray`.
- */
-struct _GArrowFloatArray
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowFloatArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_float_array_get_type(void) G_GNUC_CONST;
-
-GArrowFloatArray *garrow_float_array_new(gint64 length,
-                                         GArrowBuffer *data,
-                                         GArrowBuffer *null_bitmap,
-                                         gint64 n_nulls);
-
-gfloat garrow_float_array_get_value(GArrowFloatArray *array,
-                                    gint64 i);
-const gfloat *garrow_float_array_get_values(GArrowFloatArray *array,
-                                            gint64 *length);
-
-
-#define GARROW_TYPE_DOUBLE_ARRAY                \
-  (garrow_double_array_get_type())
-#define GARROW_DOUBLE_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DOUBLE_ARRAY, \
-                              GArrowDoubleArray))
-#define GARROW_DOUBLE_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DOUBLE_ARRAY,    \
-                           GArrowDoubleArrayClass))
-#define GARROW_IS_DOUBLE_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DOUBLE_ARRAY))
-#define GARROW_IS_DOUBLE_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DOUBLE_ARRAY))
-#define GARROW_DOUBLE_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DOUBLE_ARRAY,  \
-                             GArrowDoubleArrayClass))
-
-typedef struct _GArrowDoubleArray         GArrowDoubleArray;
-typedef struct _GArrowDoubleArrayClass    GArrowDoubleArrayClass;
-
-/**
- * GArrowDoubleArray:
- *
- * It wraps `arrow::DoubleArray`.
- */
-struct _GArrowDoubleArray
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowDoubleArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_double_array_get_type(void) G_GNUC_CONST;
-
-GArrowDoubleArray *garrow_double_array_new(gint64 length,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-gdouble garrow_double_array_get_value(GArrowDoubleArray *array,
-                                      gint64 i);
-const gdouble *garrow_double_array_get_values(GArrowDoubleArray *array,
-                                              gint64 *length);
-
-
-#define GARROW_TYPE_BINARY_ARRAY                \
-  (garrow_binary_array_get_type())
-#define GARROW_BINARY_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_BINARY_ARRAY, \
-                              GArrowBinaryArray))
-#define GARROW_BINARY_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_BINARY_ARRAY,    \
-                           GArrowBinaryArrayClass))
-#define GARROW_IS_BINARY_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BINARY_ARRAY))
-#define GARROW_IS_BINARY_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_BINARY_ARRAY))
-#define GARROW_BINARY_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_BINARY_ARRAY,  \
-                             GArrowBinaryArrayClass))
-
-typedef struct _GArrowBinaryArray         GArrowBinaryArray;
-typedef struct _GArrowBinaryArrayClass    GArrowBinaryArrayClass;
-
-/**
- * GArrowBinaryArray:
- *
- * It wraps `arrow::BinaryArray`.
- */
-struct _GArrowBinaryArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowBinaryArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_binary_array_get_type(void) G_GNUC_CONST;
-
-GArrowBinaryArray *garrow_binary_array_new(gint64 length,
-                                           GArrowBuffer *value_offsets,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-GBytes *garrow_binary_array_get_value(GArrowBinaryArray *array,
-                                      gint64 i);
-GArrowBuffer *garrow_binary_array_get_buffer(GArrowBinaryArray *array);
-GArrowBuffer *garrow_binary_array_get_offsets_buffer(GArrowBinaryArray *array);
-
-#define GARROW_TYPE_STRING_ARRAY                \
-  (garrow_string_array_get_type())
-#define GARROW_STRING_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_STRING_ARRAY, \
-                              GArrowStringArray))
-#define GARROW_STRING_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_STRING_ARRAY,    \
-                           GArrowStringArrayClass))
-#define GARROW_IS_STRING_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STRING_ARRAY))
-#define GARROW_IS_STRING_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_STRING_ARRAY))
-#define GARROW_STRING_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_STRING_ARRAY,  \
-                             GArrowStringArrayClass))
-
-typedef struct _GArrowStringArray         GArrowStringArray;
-typedef struct _GArrowStringArrayClass    GArrowStringArrayClass;
-
-/**
- * GArrowStringArray:
- *
- * It wraps `arrow::StringArray`.
- */
-struct _GArrowStringArray
-{
-  /*< private >*/
-  GArrowBinaryArray parent_instance;
-};
-
-struct _GArrowStringArrayClass
-{
-  GArrowBinaryArrayClass parent_class;
-};
-
-GType garrow_string_array_get_type(void) G_GNUC_CONST;
-
-GArrowStringArray *garrow_string_array_new(gint64 length,
-                                           GArrowBuffer *value_offsets,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-gchar *garrow_string_array_get_string(GArrowStringArray *array,
-                                      gint64 i);
-
-
-#define GARROW_TYPE_DATE32_ARRAY                \
-  (garrow_date32_array_get_type())
-#define GARROW_DATE32_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DATE32_ARRAY, \
-                              GArrowDate32Array))
-#define GARROW_DATE32_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DATE32_ARRAY,    \
-                           GArrowDate32ArrayClass))
-#define GARROW_IS_DATE32_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DATE32_ARRAY))
-#define GARROW_IS_DATE32_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DATE32_ARRAY))
-#define GARROW_DATE32_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DATE32_ARRAY,  \
-                             GArrowDate32ArrayClass))
-
-typedef struct _GArrowDate32Array         GArrowDate32Array;
-typedef struct _GArrowDate32ArrayClass    GArrowDate32ArrayClass;
-
-/**
- * GArrowDate32Array:
- *
- * It wraps `arrow::Date32Array`.
- */
-struct _GArrowDate32Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowDate32ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_date32_array_get_type(void) G_GNUC_CONST;
-
-GArrowDate32Array *garrow_date32_array_new(gint64 length,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-gint32 garrow_date32_array_get_value(GArrowDate32Array *array,
-                                     gint64 i);
-const gint32 *garrow_date32_array_get_values(GArrowDate32Array *array,
-                                             gint64 *length);
-
-
-#define GARROW_TYPE_DATE64_ARRAY                \
-  (garrow_date64_array_get_type())
-#define GARROW_DATE64_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DATE64_ARRAY, \
-                              GArrowDate64Array))
-#define GARROW_DATE64_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DATE64_ARRAY,    \
-                           GArrowDate64ArrayClass))
-#define GARROW_IS_DATE64_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DATE64_ARRAY))
-#define GARROW_IS_DATE64_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DATE64_ARRAY))
-#define GARROW_DATE64_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DATE64_ARRAY,  \
-                             GArrowDate64ArrayClass))
-
-typedef struct _GArrowDate64Array         GArrowDate64Array;
-typedef struct _GArrowDate64ArrayClass    GArrowDate64ArrayClass;
-
-/**
- * GArrowDate64Array:
- *
- * It wraps `arrow::Date64Array`.
- */
-struct _GArrowDate64Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowDate64ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_date64_array_get_type(void) G_GNUC_CONST;
-
-GArrowDate64Array *garrow_date64_array_new(gint64 length,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-gint64 garrow_date64_array_get_value(GArrowDate64Array *array,
-                                     gint64 i);
-const gint64 *garrow_date64_array_get_values(GArrowDate64Array *array,
-                                             gint64 *length);
-
-
-#define GARROW_TYPE_TIMESTAMP_ARRAY             \
-  (garrow_timestamp_array_get_type())
-#define GARROW_TIMESTAMP_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_TIMESTAMP_ARRAY,      \
-                              GArrowTimestampArray))
-#define GARROW_TIMESTAMP_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_TIMESTAMP_ARRAY, \
-                           GArrowTimestampArrayClass))
-#define GARROW_IS_TIMESTAMP_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIMESTAMP_ARRAY))
-#define GARROW_IS_TIMESTAMP_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_TIMESTAMP_ARRAY))
-#define GARROW_TIMESTAMP_ARRAY_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_TIMESTAMP_ARRAY,       \
-                             GArrowTimestampArrayClass))
-
-typedef struct _GArrowTimestampArray         GArrowTimestampArray;
-typedef struct _GArrowTimestampArrayClass    GArrowTimestampArrayClass;
-
-/**
- * GArrowTimestampArray:
- *
- * It wraps `arrow::TimestampArray`.
- */
-struct _GArrowTimestampArray
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowTimestampArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_timestamp_array_get_type(void) G_GNUC_CONST;
-
-GArrowTimestampArray *garrow_timestamp_array_new(GArrowTimestampDataType *data_type,
-                                                 gint64 length,
-                                                 GArrowBuffer *data,
-                                                 GArrowBuffer *null_bitmap,
-                                                 gint64 n_nulls);
-
-gint64 garrow_timestamp_array_get_value(GArrowTimestampArray *array,
-                                        gint64 i);
-const gint64 *garrow_timestamp_array_get_values(GArrowTimestampArray *array,
-                                                gint64 *length);
-
-
-#define GARROW_TYPE_TIME32_ARRAY                \
-  (garrow_time32_array_get_type())
-#define GARROW_TIME32_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_TIME32_ARRAY, \
-                              GArrowTime32Array))
-#define GARROW_TIME32_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_TIME32_ARRAY,    \
-                           GArrowTime32ArrayClass))
-#define GARROW_IS_TIME32_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIME32_ARRAY))
-#define GARROW_IS_TIME32_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_TIME32_ARRAY))
-#define GARROW_TIME32_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_TIME32_ARRAY,  \
-                             GArrowTime32ArrayClass))
-
-typedef struct _GArrowTime32Array         GArrowTime32Array;
-typedef struct _GArrowTime32ArrayClass    GArrowTime32ArrayClass;
-
-/**
- * GArrowTime32Array:
- *
- * It wraps `arrow::Time32Array`.
- */
-struct _GArrowTime32Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowTime32ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_time32_array_get_type(void) G_GNUC_CONST;
-
-GArrowTime32Array *garrow_time32_array_new(GArrowTime32DataType *data_type,
-                                           gint64 length,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-gint32 garrow_time32_array_get_value(GArrowTime32Array *array,
-                                     gint64 i);
-const gint32 *garrow_time32_array_get_values(GArrowTime32Array *array,
-                                             gint64 *length);
-
-
-#define GARROW_TYPE_TIME64_ARRAY                \
-  (garrow_time64_array_get_type())
-#define GARROW_TIME64_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_TIME64_ARRAY, \
-                              GArrowTime64Array))
-#define GARROW_TIME64_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_TIME64_ARRAY,    \
-                           GArrowTime64ArrayClass))
-#define GARROW_IS_TIME64_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIME64_ARRAY))
-#define GARROW_IS_TIME64_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_TIME64_ARRAY))
-#define GARROW_TIME64_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_TIME64_ARRAY,  \
-                             GArrowTime64ArrayClass))
-
-typedef struct _GArrowTime64Array         GArrowTime64Array;
-typedef struct _GArrowTime64ArrayClass    GArrowTime64ArrayClass;
-
-/**
- * GArrowTime64Array:
- *
- * It wraps `arrow::Time64Array`.
- */
-struct _GArrowTime64Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
-struct _GArrowTime64ArrayClass
-{
-  GArrowPrimitiveArrayClass parent_class;
-};
-
-GType garrow_time64_array_get_type(void) G_GNUC_CONST;
-
-GArrowTime64Array *garrow_time64_array_new(GArrowTime64DataType *data_type,
-                                           gint64 length,
-                                           GArrowBuffer *data,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-gint64 garrow_time64_array_get_value(GArrowTime64Array *array,
-                                     gint64 i);
-const gint64 *garrow_time64_array_get_values(GArrowTime64Array *array,
-                                             gint64 *length);
-
-
-#define GARROW_TYPE_LIST_ARRAY                  \
-  (garrow_list_array_get_type())
-#define GARROW_LIST_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_LIST_ARRAY,   \
-                              GArrowListArray))
-#define GARROW_LIST_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_LIST_ARRAY,      \
-                           GArrowListArrayClass))
-#define GARROW_IS_LIST_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_LIST_ARRAY))
-#define GARROW_IS_LIST_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_LIST_ARRAY))
-#define GARROW_LIST_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_LIST_ARRAY,    \
-                             GArrowListArrayClass))
-
-typedef struct _GArrowListArray         GArrowListArray;
-typedef struct _GArrowListArrayClass    GArrowListArrayClass;
-
-/**
- * GArrowListArray:
- *
- * It wraps `arrow::ListArray`.
- */
-struct _GArrowListArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowListArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_list_array_get_type(void) G_GNUC_CONST;
-
-GArrowListArray *garrow_list_array_new(gint64 length,
-                                       GArrowBuffer *value_offsets,
-                                       GArrowArray *values,
-                                       GArrowBuffer *null_bitmap,
-                                       gint64 n_nulls);
-
-GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array);
-GArrowArray *garrow_list_array_get_value(GArrowListArray *array,
-                                         gint64 i);
-
-
-#define GARROW_TYPE_STRUCT_ARRAY                \
-  (garrow_struct_array_get_type())
-#define GARROW_STRUCT_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_STRUCT_ARRAY, \
-                              GArrowStructArray))
-#define GARROW_STRUCT_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_STRUCT_ARRAY,    \
-                           GArrowStructArrayClass))
-#define GARROW_IS_STRUCT_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_STRUCT_ARRAY))
-#define GARROW_IS_STRUCT_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_STRUCT_ARRAY))
-#define GARROW_STRUCT_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_STRUCT_ARRAY,  \
-                             GArrowStructArrayClass))
-
-typedef struct _GArrowStructArray         GArrowStructArray;
-typedef struct _GArrowStructArrayClass    GArrowStructArrayClass;
-
-/**
- * GArrowStructArray:
- *
- * It wraps `arrow::StructArray`.
- */
-struct _GArrowStructArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
-struct _GArrowStructArrayClass
-{
-  GArrowArrayClass parent_class;
-};
-
-GType garrow_struct_array_get_type(void) G_GNUC_CONST;
-
-GArrowStructArray *garrow_struct_array_new(GArrowDataType *data_type,
-                                           gint64 length,
-                                           GList *children,
-                                           GArrowBuffer *null_bitmap,
-                                           gint64 n_nulls);
-
-GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
-                                           gint i);
-GList *garrow_struct_array_get_fields(GArrowStructArray *array);
-
-G_END_DECLS
+#include <arrow-glib/basic-array.h>
+#include <arrow-glib/composite-array.h>
diff --git a/c_glib/arrow-glib/array.hpp b/c_glib/arrow-glib/array.hpp
index d2dff22c48cf9..e575c425380f2 100644
--- a/c_glib/arrow-glib/array.hpp
+++ b/c_glib/arrow-glib/array.hpp
@@ -19,9 +19,5 @@
 
 #pragma once
 
-#include <arrow/api.h>
-
-#include <arrow-glib/array.h>
-
-GArrowArray *garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array);
-std::shared_ptr<arrow::Array> garrow_array_get_raw(GArrowArray *array);
+#include <arrow-glib/basic-array.hpp>
+#include <arrow-glib/composite-array.h>
diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/basic-array.cpp
similarity index 90%
rename from c_glib/arrow-glib/array.cpp
rename to c_glib/arrow-glib/basic-array.cpp
index 4945ca3cfe903..0698a04098050 100644
--- a/c_glib/arrow-glib/array.cpp
+++ b/c_glib/arrow-glib/basic-array.cpp
@@ -24,7 +24,7 @@
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/compute.hpp>
-#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/basic-data-type.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/type.hpp>
 
@@ -86,9 +86,9 @@ garrow_primitive_array_new(GArrowDataType *data_type,
 G_BEGIN_DECLS
 
 /**
- * SECTION: array
- * @section_id: array-classes
- * @title: Array classes
+ * SECTION: basic-array
+ * @section_id: basic-array-classes
+ * @title: Basic array classes
  * @include: arrow-glib/arrow-glib.h
  *
  * #GArrowArray is a base class for all array classes such as
@@ -194,15 +194,6 @@ G_BEGIN_DECLS
  * nanoseconds since midnight in 64-bit signed integer array. It can
  * store zero or more time data. If you don't have Arrow format data,
  * you need to use #GArrowTime64ArrayBuilder to create a new array.
- *
- * #GArrowListArray is a class for list array. It can store zero or
- * more list data. If you don't have Arrow format data, you need to
- * use #GArrowListArrayBuilder to create a new array.
- *
- * #GArrowStructArray is a class for struct array. It can store zero
- * or more structs. One struct has zero or more fields. If you don't
- * have Arrow format data, you need to use #GArrowStructArrayBuilder
- * to create a new array.
  */
 
 typedef struct GArrowArrayPrivate_ {
@@ -2028,195 +2019,6 @@ garrow_time64_array_get_values(GArrowTime64Array *array,
   return reinterpret_cast<const gint64 *>(values);
 }
 
-
-G_DEFINE_TYPE(GArrowListArray,               \
-              garrow_list_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_list_array_init(GArrowListArray *object)
-{
-}
-
-static void
-garrow_list_array_class_init(GArrowListArrayClass *klass)
-{
-}
-
-/**
- * garrow_list_array_new:
- * @length: The number of elements.
- * @value_offsets: The offsets of @values in Arrow format.
- * @values: The values as #GArrowArray.
- * @null_bitmap: (nullable): The bitmap that shows null elements. The
- *   N-th element is null when the N-th bit is 0, not null otherwise.
- *   If the array has no null elements, the bitmap must be %NULL and
- *   @n_nulls is 0.
- * @n_nulls: The number of null elements. If -1 is specified, the
- *   number of nulls are computed from @null_bitmap.
- *
- * Returns: A newly created #GArrowListArray.
- *
- * Since: 0.4.0
- */
-GArrowListArray *
-garrow_list_array_new(gint64 length,
-                      GArrowBuffer *value_offsets,
-                      GArrowArray *values,
-                      GArrowBuffer *null_bitmap,
-                      gint64 n_nulls)
-{
-  const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets);
-  const auto arrow_values = garrow_array_get_raw(values);
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_data_type = arrow::list(arrow_values->type());
-  auto arrow_list_array =
-    std::make_shared<arrow::ListArray>(arrow_data_type,
-                                       length,
-                                       arrow_value_offsets,
-                                       arrow_values,
-                                       arrow_bitmap,
-                                       n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_list_array);
-  return GARROW_LIST_ARRAY(garrow_array_new_raw(&arrow_array));
-}
-
-/**
- * garrow_list_array_get_value_type:
- * @array: A #GArrowListArray.
- *
- * Returns: (transfer full): The data type of value in each list.
- */
-GArrowDataType *
-garrow_list_array_get_value_type(GArrowListArray *array)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_list_array =
-    static_cast<arrow::ListArray *>(arrow_array.get());
-  auto arrow_value_type = arrow_list_array->value_type();
-  return garrow_data_type_new_raw(&arrow_value_type);
-}
-
-/**
- * garrow_list_array_get_value:
- * @array: A #GArrowListArray.
- * @i: The index of the target value.
- *
- * Returns: (transfer full): The i-th list.
- */
-GArrowArray *
-garrow_list_array_get_value(GArrowListArray *array,
-                            gint64 i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_list_array =
-    static_cast<arrow::ListArray *>(arrow_array.get());
-  auto arrow_list =
-    arrow_list_array->values()->Slice(arrow_list_array->value_offset(i),
-                                      arrow_list_array->value_length(i));
-  return garrow_array_new_raw(&arrow_list);
-}
-
-
-G_DEFINE_TYPE(GArrowStructArray,               \
-              garrow_struct_array,             \
-              GARROW_TYPE_ARRAY)
-
-static void
-garrow_struct_array_init(GArrowStructArray *object)
-{
-}
-
-static void
-garrow_struct_array_class_init(GArrowStructArrayClass *klass)
-{
-}
-
-/**
- * garrow_struct_array_new:
- * @data_type: The data type of the struct.
- * @length: The number of elements.
- * @children: (element-type GArrowArray): The arrays for each field
- *   as #GList of #GArrowArray.
- * @null_bitmap: (nullable): The bitmap that shows null elements. The
- *   N-th element is null when the N-th bit is 0, not null otherwise.
- *   If the array has no null elements, the bitmap must be %NULL and
- *   @n_nulls is 0.
- * @n_nulls: The number of null elements. If -1 is specified, the
- *   number of nulls are computed from @null_bitmap.
- *
- * Returns: A newly created #GArrowStructArray.
- *
- * Since: 0.4.0
- */
-GArrowStructArray *
-garrow_struct_array_new(GArrowDataType *data_type,
-                        gint64 length,
-                        GList *children,
-                        GArrowBuffer *null_bitmap,
-                        gint64 n_nulls)
-{
-  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
-  std::vector<std::shared_ptr<arrow::Array>> arrow_children;
-  for (GList *node = children; node; node = node->next) {
-    GArrowArray *child = GARROW_ARRAY(node->data);
-    arrow_children.push_back(garrow_array_get_raw(child));
-  }
-  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
-  auto arrow_struct_array =
-    std::make_shared<arrow::StructArray>(arrow_data_type,
-                                         length,
-                                         arrow_children,
-                                         arrow_bitmap,
-                                         n_nulls);
-  auto arrow_array =
-    std::static_pointer_cast<arrow::Array>(arrow_struct_array);
-  return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_array));
-}
-
-/**
- * garrow_struct_array_get_field
- * @array: A #GArrowStructArray.
- * @i: The index of the field in the struct.
- *
- * Returns: (transfer full): The i-th field.
- */
-GArrowArray *
-garrow_struct_array_get_field(GArrowStructArray *array,
-                              gint i)
-{
-  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  auto arrow_struct_array =
-    static_cast<arrow::StructArray *>(arrow_array.get());
-  auto arrow_field = arrow_struct_array->field(i);
-  return garrow_array_new_raw(&arrow_field);
-}
-
-/**
- * garrow_struct_array_get_fields
- * @array: A #GArrowStructArray.
- *
- * Returns: (element-type GArrowArray) (transfer full):
- *   The fields in the struct.
- */
-GList *
-garrow_struct_array_get_fields(GArrowStructArray *array)
-{
-  const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
-  const auto arrow_struct_array =
-    static_cast<const arrow::StructArray *>(arrow_array.get());
-
-  GList *fields = NULL;
-  for (int i = 0; i < arrow_struct_array->num_fields(); ++i) {
-    auto arrow_field = arrow_struct_array->field(i);
-    GArrowArray *field = garrow_array_new_raw(&arrow_field);
-    fields = g_list_prepend(fields, field);
-  }
-
-  return g_list_reverse(fields);
-}
-
 G_END_DECLS
 
 GArrowArray *
diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h
new file mode 100644
index 0000000000000..c01a04f07d8b9
--- /dev/null
+++ b/c_glib/arrow-glib/basic-array.h
@@ -0,0 +1,1115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/buffer.h>
+#include <arrow-glib/compute.h>
+#include <arrow-glib/basic-data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_ARRAY (garrow_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowArray,
+                         garrow_array,
+                         GARROW,
+                         ARRAY,
+                         GObject)
+struct _GArrowArrayClass
+{
+  GObjectClass parent_class;
+};
+
+gboolean       garrow_array_equal       (GArrowArray *array,
+                                         GArrowArray *other_array);
+gboolean       garrow_array_equal_approx(GArrowArray *array,
+                                         GArrowArray *other_array);
+gboolean       garrow_array_equal_range (GArrowArray *array,
+                                         gint64 start_index,
+                                         GArrowArray *other_array,
+                                         gint64 other_start_index,
+                                         gint64 end_index);
+
+gboolean       garrow_array_is_null     (GArrowArray *array,
+                                         gint64 i);
+gboolean       garrow_array_is_valid    (GArrowArray *array,
+                                         gint64 i);
+gint64         garrow_array_get_length  (GArrowArray *array);
+gint64         garrow_array_get_offset  (GArrowArray *array);
+gint64         garrow_array_get_n_nulls (GArrowArray *array);
+GArrowBuffer  *garrow_array_get_null_bitmap(GArrowArray *array);
+GArrowDataType *garrow_array_get_value_data_type(GArrowArray *array);
+GArrowType     garrow_array_get_value_type(GArrowArray *array);
+GArrowArray   *garrow_array_slice       (GArrowArray *array,
+                                         gint64 offset,
+                                         gint64 length);
+gchar         *garrow_array_to_string   (GArrowArray *array,
+                                         GError **error);
+
+GArrowArray   *garrow_array_cast        (GArrowArray *array,
+                                         GArrowDataType *target_data_type,
+                                         GArrowCastOptions *options,
+                                         GError **error);
+
+#define GARROW_TYPE_NULL_ARRAY                  \
+  (garrow_null_array_get_type())
+#define GARROW_NULL_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_NULL_ARRAY,   \
+                              GArrowNullArray))
+#define GARROW_NULL_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_NULL_ARRAY,      \
+                           GArrowNullArrayClass))
+#define GARROW_IS_NULL_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_NULL_ARRAY))
+#define GARROW_IS_NULL_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_NULL_ARRAY))
+#define GARROW_NULL_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_NULL_ARRAY,    \
+                             GArrowNullArrayClass))
+
+typedef struct _GArrowNullArray         GArrowNullArray;
+typedef struct _GArrowNullArrayClass    GArrowNullArrayClass;
+
+/**
+ * GArrowNullArray:
+ *
+ * It wraps `arrow::NullArray`.
+ */
+struct _GArrowNullArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowNullArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_null_array_get_type(void) G_GNUC_CONST;
+
+GArrowNullArray *garrow_null_array_new(gint64 length);
+
+
+#define GARROW_TYPE_PRIMITIVE_ARRAY             \
+  (garrow_primitive_array_get_type())
+#define GARROW_PRIMITIVE_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_PRIMITIVE_ARRAY,      \
+                              GArrowPrimitiveArray))
+#define GARROW_PRIMITIVE_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_PRIMITIVE_ARRAY, \
+                           GArrowPrimitiveArrayClass))
+#define GARROW_IS_PRIMITIVE_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_PRIMITIVE_ARRAY))
+#define GARROW_IS_PRIMITIVE_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_PRIMITIVE_ARRAY))
+#define GARROW_PRIMITIVE_ARRAY_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_PRIMITIVE_ARRAY,       \
+                             GArrowPrimitiveArrayClass))
+
+typedef struct _GArrowPrimitiveArray         GArrowPrimitiveArray;
+typedef struct _GArrowPrimitiveArrayClass    GArrowPrimitiveArrayClass;
+
+/**
+ * GArrowPrimitiveArray:
+ *
+ * It wraps `arrow::PrimitiveArray`.
+ */
+struct _GArrowPrimitiveArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowPrimitiveArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_primitive_array_get_type(void) G_GNUC_CONST;
+
+GArrowBuffer *garrow_primitive_array_get_buffer(GArrowPrimitiveArray *array);
+
+
+#define GARROW_TYPE_BOOLEAN_ARRAY               \
+  (garrow_boolean_array_get_type())
+#define GARROW_BOOLEAN_ARRAY(obj)                               \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_BOOLEAN_ARRAY,        \
+                              GArrowBooleanArray))
+#define GARROW_BOOLEAN_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_BOOLEAN_ARRAY,   \
+                           GArrowBooleanArrayClass))
+#define GARROW_IS_BOOLEAN_ARRAY(obj)                            \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BOOLEAN_ARRAY))
+#define GARROW_IS_BOOLEAN_ARRAY_CLASS(klass)            \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_BOOLEAN_ARRAY))
+#define GARROW_BOOLEAN_ARRAY_GET_CLASS(obj)             \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_BOOLEAN_ARRAY, \
+                             GArrowBooleanArrayClass))
+
+typedef struct _GArrowBooleanArray         GArrowBooleanArray;
+typedef struct _GArrowBooleanArrayClass    GArrowBooleanArrayClass;
+
+/**
+ * GArrowBooleanArray:
+ *
+ * It wraps `arrow::BooleanArray`.
+ */
+struct _GArrowBooleanArray
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowBooleanArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType          garrow_boolean_array_get_type  (void) G_GNUC_CONST;
+
+GArrowBooleanArray *garrow_boolean_array_new(gint64 length,
+                                             GArrowBuffer *data,
+                                             GArrowBuffer *null_bitmap,
+                                             gint64 n_nulls);
+
+gboolean       garrow_boolean_array_get_value (GArrowBooleanArray *array,
+                                               gint64 i);
+gboolean      *garrow_boolean_array_get_values(GArrowBooleanArray *array,
+                                               gint64 *length);
+
+
+#define GARROW_TYPE_INT8_ARRAY                  \
+  (garrow_int8_array_get_type())
+#define GARROW_INT8_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT8_ARRAY,   \
+                              GArrowInt8Array))
+#define GARROW_INT8_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT8_ARRAY,      \
+                           GArrowInt8ArrayClass))
+#define GARROW_IS_INT8_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT8_ARRAY))
+#define GARROW_IS_INT8_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT8_ARRAY))
+#define GARROW_INT8_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT8_ARRAY,    \
+                             GArrowInt8ArrayClass))
+
+typedef struct _GArrowInt8Array         GArrowInt8Array;
+typedef struct _GArrowInt8ArrayClass    GArrowInt8ArrayClass;
+
+/**
+ * GArrowInt8Array:
+ *
+ * It wraps `arrow::Int8Array`.
+ */
+struct _GArrowInt8Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowInt8ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_int8_array_get_type(void) G_GNUC_CONST;
+
+GArrowInt8Array *garrow_int8_array_new(gint64 length,
+                                       GArrowBuffer *data,
+                                       GArrowBuffer *null_bitmap,
+                                       gint64 n_nulls);
+
+gint8 garrow_int8_array_get_value(GArrowInt8Array *array,
+                                  gint64 i);
+const gint8 *garrow_int8_array_get_values(GArrowInt8Array *array,
+                                          gint64 *length);
+
+
+#define GARROW_TYPE_UINT8_ARRAY                 \
+  (garrow_uint8_array_get_type())
+#define GARROW_UINT8_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT8_ARRAY,  \
+                              GArrowUInt8Array))
+#define GARROW_UINT8_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT8_ARRAY,     \
+                           GArrowUInt8ArrayClass))
+#define GARROW_IS_UINT8_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_UINT8_ARRAY))
+#define GARROW_IS_UINT8_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT8_ARRAY))
+#define GARROW_UINT8_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT8_ARRAY,   \
+                             GArrowUInt8ArrayClass))
+
+typedef struct _GArrowUInt8Array         GArrowUInt8Array;
+typedef struct _GArrowUInt8ArrayClass    GArrowUInt8ArrayClass;
+
+/**
+ * GArrowUInt8Array:
+ *
+ * It wraps `arrow::UInt8Array`.
+ */
+struct _GArrowUInt8Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowUInt8ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_uint8_array_get_type(void) G_GNUC_CONST;
+
+GArrowUInt8Array *garrow_uint8_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
+guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array,
+                                    gint64 i);
+const guint8 *garrow_uint8_array_get_values(GArrowUInt8Array *array,
+                                            gint64 *length);
+
+
+#define GARROW_TYPE_INT16_ARRAY                  \
+  (garrow_int16_array_get_type())
+#define GARROW_INT16_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT16_ARRAY,  \
+                              GArrowInt16Array))
+#define GARROW_INT16_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT16_ARRAY,     \
+                           GArrowInt16ArrayClass))
+#define GARROW_IS_INT16_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT16_ARRAY))
+#define GARROW_IS_INT16_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT16_ARRAY))
+#define GARROW_INT16_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT16_ARRAY,   \
+                             GArrowInt16ArrayClass))
+
+typedef struct _GArrowInt16Array         GArrowInt16Array;
+typedef struct _GArrowInt16ArrayClass    GArrowInt16ArrayClass;
+
+/**
+ * GArrowInt16Array:
+ *
+ * It wraps `arrow::Int16Array`.
+ */
+struct _GArrowInt16Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowInt16ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_int16_array_get_type(void) G_GNUC_CONST;
+
+GArrowInt16Array *garrow_int16_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
+gint16 garrow_int16_array_get_value(GArrowInt16Array *array,
+                                    gint64 i);
+const gint16 *garrow_int16_array_get_values(GArrowInt16Array *array,
+                                            gint64 *length);
+
+
+#define GARROW_TYPE_UINT16_ARRAY                 \
+  (garrow_uint16_array_get_type())
+#define GARROW_UINT16_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT16_ARRAY, \
+                              GArrowUInt16Array))
+#define GARROW_UINT16_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT16_ARRAY,    \
+                           GArrowUInt16ArrayClass))
+#define GARROW_IS_UINT16_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT16_ARRAY))
+#define GARROW_IS_UINT16_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT16_ARRAY))
+#define GARROW_UINT16_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT16_ARRAY,  \
+                             GArrowUInt16ArrayClass))
+
+typedef struct _GArrowUInt16Array         GArrowUInt16Array;
+typedef struct _GArrowUInt16ArrayClass    GArrowUInt16ArrayClass;
+
+/**
+ * GArrowUInt16Array:
+ *
+ * It wraps `arrow::UInt16Array`.
+ */
+struct _GArrowUInt16Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowUInt16ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_uint16_array_get_type(void) G_GNUC_CONST;
+
+GArrowUInt16Array *garrow_uint16_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array,
+                                      gint64 i);
+const guint16 *garrow_uint16_array_get_values(GArrowUInt16Array *array,
+                                              gint64 *length);
+
+
+#define GARROW_TYPE_INT32_ARRAY                 \
+  (garrow_int32_array_get_type())
+#define GARROW_INT32_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT32_ARRAY,  \
+                              GArrowInt32Array))
+#define GARROW_INT32_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT32_ARRAY,     \
+                           GArrowInt32ArrayClass))
+#define GARROW_IS_INT32_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT32_ARRAY))
+#define GARROW_IS_INT32_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT32_ARRAY))
+#define GARROW_INT32_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT32_ARRAY,   \
+                             GArrowInt32ArrayClass))
+
+typedef struct _GArrowInt32Array         GArrowInt32Array;
+typedef struct _GArrowInt32ArrayClass    GArrowInt32ArrayClass;
+
+/**
+ * GArrowInt32Array:
+ *
+ * It wraps `arrow::Int32Array`.
+ */
+struct _GArrowInt32Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowInt32ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_int32_array_get_type(void) G_GNUC_CONST;
+
+GArrowInt32Array *garrow_int32_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
+gint32 garrow_int32_array_get_value(GArrowInt32Array *array,
+                                    gint64 i);
+const gint32 *garrow_int32_array_get_values(GArrowInt32Array *array,
+                                            gint64 *length);
+
+
+#define GARROW_TYPE_UINT32_ARRAY                \
+  (garrow_uint32_array_get_type())
+#define GARROW_UINT32_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT32_ARRAY, \
+                              GArrowUInt32Array))
+#define GARROW_UINT32_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT32_ARRAY,    \
+                           GArrowUInt32ArrayClass))
+#define GARROW_IS_UINT32_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT32_ARRAY))
+#define GARROW_IS_UINT32_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT32_ARRAY))
+#define GARROW_UINT32_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT32_ARRAY,  \
+                             GArrowUInt32ArrayClass))
+
+typedef struct _GArrowUInt32Array         GArrowUInt32Array;
+typedef struct _GArrowUInt32ArrayClass    GArrowUInt32ArrayClass;
+
+/**
+ * GArrowUInt32Array:
+ *
+ * It wraps `arrow::UInt32Array`.
+ */
+struct _GArrowUInt32Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowUInt32ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_uint32_array_get_type(void) G_GNUC_CONST;
+
+GArrowUInt32Array *garrow_uint32_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array,
+                                      gint64 i);
+const guint32 *garrow_uint32_array_get_values(GArrowUInt32Array *array,
+                                              gint64 *length);
+
+
+#define GARROW_TYPE_INT64_ARRAY                 \
+  (garrow_int64_array_get_type())
+#define GARROW_INT64_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_INT64_ARRAY,  \
+                              GArrowInt64Array))
+#define GARROW_INT64_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_INT64_ARRAY,     \
+                           GArrowInt64ArrayClass))
+#define GARROW_IS_INT64_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_INT64_ARRAY))
+#define GARROW_IS_INT64_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_INT64_ARRAY))
+#define GARROW_INT64_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_INT64_ARRAY,   \
+                             GArrowInt64ArrayClass))
+
+typedef struct _GArrowInt64Array         GArrowInt64Array;
+typedef struct _GArrowInt64ArrayClass    GArrowInt64ArrayClass;
+
+/**
+ * GArrowInt64Array:
+ *
+ * It wraps `arrow::Int64Array`.
+ */
+struct _GArrowInt64Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowInt64ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_int64_array_get_type(void) G_GNUC_CONST;
+
+GArrowInt64Array *garrow_int64_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
+gint64 garrow_int64_array_get_value(GArrowInt64Array *array,
+                                    gint64 i);
+const gint64 *garrow_int64_array_get_values(GArrowInt64Array *array,
+                                            gint64 *length);
+
+
+#define GARROW_TYPE_UINT64_ARRAY                \
+  (garrow_uint64_array_get_type())
+#define GARROW_UINT64_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_UINT64_ARRAY, \
+                              GArrowUInt64Array))
+#define GARROW_UINT64_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_UINT64_ARRAY,    \
+                           GArrowUInt64ArrayClass))
+#define GARROW_IS_UINT64_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_UINT64_ARRAY))
+#define GARROW_IS_UINT64_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_UINT64_ARRAY))
+#define GARROW_UINT64_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_UINT64_ARRAY,  \
+                             GArrowUInt64ArrayClass))
+
+typedef struct _GArrowUInt64Array         GArrowUInt64Array;
+typedef struct _GArrowUInt64ArrayClass    GArrowUInt64ArrayClass;
+
+/**
+ * GArrowUInt64Array:
+ *
+ * It wraps `arrow::UInt64Array`.
+ */
+struct _GArrowUInt64Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowUInt64ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_uint64_array_get_type(void) G_GNUC_CONST;
+
+GArrowUInt64Array *garrow_uint64_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array,
+                                      gint64 i);
+const guint64 *garrow_uint64_array_get_values(GArrowUInt64Array *array,
+                                              gint64 *length);
+
+
+#define GARROW_TYPE_FLOAT_ARRAY                 \
+  (garrow_float_array_get_type())
+#define GARROW_FLOAT_ARRAY(obj)                         \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_FLOAT_ARRAY,  \
+                              GArrowFloatArray))
+#define GARROW_FLOAT_ARRAY_CLASS(klass)                 \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_FLOAT_ARRAY,     \
+                           GArrowFloatArrayClass))
+#define GARROW_IS_FLOAT_ARRAY(obj)                      \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_FLOAT_ARRAY))
+#define GARROW_IS_FLOAT_ARRAY_CLASS(klass)              \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_FLOAT_ARRAY))
+#define GARROW_FLOAT_ARRAY_GET_CLASS(obj)               \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_FLOAT_ARRAY,   \
+                             GArrowFloatArrayClass))
+
+typedef struct _GArrowFloatArray         GArrowFloatArray;
+typedef struct _GArrowFloatArrayClass    GArrowFloatArrayClass;
+
+/**
+ * GArrowFloatArray:
+ *
+ * It wraps `arrow::FloatArray`.
+ */
+struct _GArrowFloatArray
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowFloatArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_float_array_get_type(void) G_GNUC_CONST;
+
+GArrowFloatArray *garrow_float_array_new(gint64 length,
+                                         GArrowBuffer *data,
+                                         GArrowBuffer *null_bitmap,
+                                         gint64 n_nulls);
+
+gfloat garrow_float_array_get_value(GArrowFloatArray *array,
+                                    gint64 i);
+const gfloat *garrow_float_array_get_values(GArrowFloatArray *array,
+                                            gint64 *length);
+
+
+#define GARROW_TYPE_DOUBLE_ARRAY                \
+  (garrow_double_array_get_type())
+#define GARROW_DOUBLE_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DOUBLE_ARRAY, \
+                              GArrowDoubleArray))
+#define GARROW_DOUBLE_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DOUBLE_ARRAY,    \
+                           GArrowDoubleArrayClass))
+#define GARROW_IS_DOUBLE_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DOUBLE_ARRAY))
+#define GARROW_IS_DOUBLE_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DOUBLE_ARRAY))
+#define GARROW_DOUBLE_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DOUBLE_ARRAY,  \
+                             GArrowDoubleArrayClass))
+
+typedef struct _GArrowDoubleArray         GArrowDoubleArray;
+typedef struct _GArrowDoubleArrayClass    GArrowDoubleArrayClass;
+
+/**
+ * GArrowDoubleArray:
+ *
+ * It wraps `arrow::DoubleArray`.
+ */
+struct _GArrowDoubleArray
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowDoubleArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_double_array_get_type(void) G_GNUC_CONST;
+
+GArrowDoubleArray *garrow_double_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+gdouble garrow_double_array_get_value(GArrowDoubleArray *array,
+                                      gint64 i);
+const gdouble *garrow_double_array_get_values(GArrowDoubleArray *array,
+                                              gint64 *length);
+
+
+#define GARROW_TYPE_BINARY_ARRAY                \
+  (garrow_binary_array_get_type())
+#define GARROW_BINARY_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_BINARY_ARRAY, \
+                              GArrowBinaryArray))
+#define GARROW_BINARY_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_BINARY_ARRAY,    \
+                           GArrowBinaryArrayClass))
+#define GARROW_IS_BINARY_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_BINARY_ARRAY))
+#define GARROW_IS_BINARY_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_BINARY_ARRAY))
+#define GARROW_BINARY_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_BINARY_ARRAY,  \
+                             GArrowBinaryArrayClass))
+
+typedef struct _GArrowBinaryArray         GArrowBinaryArray;
+typedef struct _GArrowBinaryArrayClass    GArrowBinaryArrayClass;
+
+/**
+ * GArrowBinaryArray:
+ *
+ * It wraps `arrow::BinaryArray`.
+ */
+struct _GArrowBinaryArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowBinaryArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_binary_array_get_type(void) G_GNUC_CONST;
+
+GArrowBinaryArray *garrow_binary_array_new(gint64 length,
+                                           GArrowBuffer *value_offsets,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+GBytes *garrow_binary_array_get_value(GArrowBinaryArray *array,
+                                      gint64 i);
+GArrowBuffer *garrow_binary_array_get_buffer(GArrowBinaryArray *array);
+GArrowBuffer *garrow_binary_array_get_offsets_buffer(GArrowBinaryArray *array);
+
+#define GARROW_TYPE_STRING_ARRAY                \
+  (garrow_string_array_get_type())
+#define GARROW_STRING_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_STRING_ARRAY, \
+                              GArrowStringArray))
+#define GARROW_STRING_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_STRING_ARRAY,    \
+                           GArrowStringArrayClass))
+#define GARROW_IS_STRING_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRING_ARRAY))
+#define GARROW_IS_STRING_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_STRING_ARRAY))
+#define GARROW_STRING_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_STRING_ARRAY,  \
+                             GArrowStringArrayClass))
+
+typedef struct _GArrowStringArray         GArrowStringArray;
+typedef struct _GArrowStringArrayClass    GArrowStringArrayClass;
+
+/**
+ * GArrowStringArray:
+ *
+ * It wraps `arrow::StringArray`.
+ */
+struct _GArrowStringArray
+{
+  /*< private >*/
+  GArrowBinaryArray parent_instance;
+};
+
+struct _GArrowStringArrayClass
+{
+  GArrowBinaryArrayClass parent_class;
+};
+
+GType garrow_string_array_get_type(void) G_GNUC_CONST;
+
+GArrowStringArray *garrow_string_array_new(gint64 length,
+                                           GArrowBuffer *value_offsets,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+gchar *garrow_string_array_get_string(GArrowStringArray *array,
+                                      gint64 i);
+
+
+#define GARROW_TYPE_DATE32_ARRAY                \
+  (garrow_date32_array_get_type())
+#define GARROW_DATE32_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DATE32_ARRAY, \
+                              GArrowDate32Array))
+#define GARROW_DATE32_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DATE32_ARRAY,    \
+                           GArrowDate32ArrayClass))
+#define GARROW_IS_DATE32_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DATE32_ARRAY))
+#define GARROW_IS_DATE32_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DATE32_ARRAY))
+#define GARROW_DATE32_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DATE32_ARRAY,  \
+                             GArrowDate32ArrayClass))
+
+typedef struct _GArrowDate32Array         GArrowDate32Array;
+typedef struct _GArrowDate32ArrayClass    GArrowDate32ArrayClass;
+
+/**
+ * GArrowDate32Array:
+ *
+ * It wraps `arrow::Date32Array`.
+ */
+struct _GArrowDate32Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowDate32ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_date32_array_get_type(void) G_GNUC_CONST;
+
+GArrowDate32Array *garrow_date32_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+gint32 garrow_date32_array_get_value(GArrowDate32Array *array,
+                                     gint64 i);
+const gint32 *garrow_date32_array_get_values(GArrowDate32Array *array,
+                                             gint64 *length);
+
+
+#define GARROW_TYPE_DATE64_ARRAY                \
+  (garrow_date64_array_get_type())
+#define GARROW_DATE64_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_DATE64_ARRAY, \
+                              GArrowDate64Array))
+#define GARROW_DATE64_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_DATE64_ARRAY,    \
+                           GArrowDate64ArrayClass))
+#define GARROW_IS_DATE64_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_DATE64_ARRAY))
+#define GARROW_IS_DATE64_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_DATE64_ARRAY))
+#define GARROW_DATE64_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_DATE64_ARRAY,  \
+                             GArrowDate64ArrayClass))
+
+typedef struct _GArrowDate64Array         GArrowDate64Array;
+typedef struct _GArrowDate64ArrayClass    GArrowDate64ArrayClass;
+
+/**
+ * GArrowDate64Array:
+ *
+ * It wraps `arrow::Date64Array`.
+ */
+struct _GArrowDate64Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowDate64ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_date64_array_get_type(void) G_GNUC_CONST;
+
+GArrowDate64Array *garrow_date64_array_new(gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+gint64 garrow_date64_array_get_value(GArrowDate64Array *array,
+                                     gint64 i);
+const gint64 *garrow_date64_array_get_values(GArrowDate64Array *array,
+                                             gint64 *length);
+
+
+#define GARROW_TYPE_TIMESTAMP_ARRAY             \
+  (garrow_timestamp_array_get_type())
+#define GARROW_TIMESTAMP_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
+                              GARROW_TYPE_TIMESTAMP_ARRAY,      \
+                              GArrowTimestampArray))
+#define GARROW_TIMESTAMP_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_TIMESTAMP_ARRAY, \
+                           GArrowTimestampArrayClass))
+#define GARROW_IS_TIMESTAMP_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIMESTAMP_ARRAY))
+#define GARROW_IS_TIMESTAMP_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
+                           GARROW_TYPE_TIMESTAMP_ARRAY))
+#define GARROW_TIMESTAMP_ARRAY_GET_CLASS(obj)                   \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
+                             GARROW_TYPE_TIMESTAMP_ARRAY,       \
+                             GArrowTimestampArrayClass))
+
+typedef struct _GArrowTimestampArray         GArrowTimestampArray;
+typedef struct _GArrowTimestampArrayClass    GArrowTimestampArrayClass;
+
+/**
+ * GArrowTimestampArray:
+ *
+ * It wraps `arrow::TimestampArray`.
+ */
+struct _GArrowTimestampArray
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowTimestampArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_timestamp_array_get_type(void) G_GNUC_CONST;
+
+GArrowTimestampArray *garrow_timestamp_array_new(GArrowTimestampDataType *data_type,
+                                                 gint64 length,
+                                                 GArrowBuffer *data,
+                                                 GArrowBuffer *null_bitmap,
+                                                 gint64 n_nulls);
+
+gint64 garrow_timestamp_array_get_value(GArrowTimestampArray *array,
+                                        gint64 i);
+const gint64 *garrow_timestamp_array_get_values(GArrowTimestampArray *array,
+                                                gint64 *length);
+
+
+#define GARROW_TYPE_TIME32_ARRAY                \
+  (garrow_time32_array_get_type())
+#define GARROW_TIME32_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_TIME32_ARRAY, \
+                              GArrowTime32Array))
+#define GARROW_TIME32_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_TIME32_ARRAY,    \
+                           GArrowTime32ArrayClass))
+#define GARROW_IS_TIME32_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME32_ARRAY))
+#define GARROW_IS_TIME32_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_TIME32_ARRAY))
+#define GARROW_TIME32_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_TIME32_ARRAY,  \
+                             GArrowTime32ArrayClass))
+
+typedef struct _GArrowTime32Array         GArrowTime32Array;
+typedef struct _GArrowTime32ArrayClass    GArrowTime32ArrayClass;
+
+/**
+ * GArrowTime32Array:
+ *
+ * It wraps `arrow::Time32Array`.
+ */
+struct _GArrowTime32Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowTime32ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_time32_array_get_type(void) G_GNUC_CONST;
+
+GArrowTime32Array *garrow_time32_array_new(GArrowTime32DataType *data_type,
+                                           gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+gint32 garrow_time32_array_get_value(GArrowTime32Array *array,
+                                     gint64 i);
+const gint32 *garrow_time32_array_get_values(GArrowTime32Array *array,
+                                             gint64 *length);
+
+
+#define GARROW_TYPE_TIME64_ARRAY                \
+  (garrow_time64_array_get_type())
+#define GARROW_TIME64_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_TIME64_ARRAY, \
+                              GArrowTime64Array))
+#define GARROW_TIME64_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_TIME64_ARRAY,    \
+                           GArrowTime64ArrayClass))
+#define GARROW_IS_TIME64_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_TIME64_ARRAY))
+#define GARROW_IS_TIME64_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_TIME64_ARRAY))
+#define GARROW_TIME64_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_TIME64_ARRAY,  \
+                             GArrowTime64ArrayClass))
+
+typedef struct _GArrowTime64Array         GArrowTime64Array;
+typedef struct _GArrowTime64ArrayClass    GArrowTime64ArrayClass;
+
+/**
+ * GArrowTime64Array:
+ *
+ * It wraps `arrow::Time64Array`.
+ */
+struct _GArrowTime64Array
+{
+  /*< private >*/
+  GArrowPrimitiveArray parent_instance;
+};
+
+struct _GArrowTime64ArrayClass
+{
+  GArrowPrimitiveArrayClass parent_class;
+};
+
+GType garrow_time64_array_get_type(void) G_GNUC_CONST;
+
+GArrowTime64Array *garrow_time64_array_new(GArrowTime64DataType *data_type,
+                                           gint64 length,
+                                           GArrowBuffer *data,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+gint64 garrow_time64_array_get_value(GArrowTime64Array *array,
+                                     gint64 i);
+const gint64 *garrow_time64_array_get_values(GArrowTime64Array *array,
+                                             gint64 *length);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/basic-array.hpp b/c_glib/arrow-glib/basic-array.hpp
new file mode 100644
index 0000000000000..52b94a51f3dbf
--- /dev/null
+++ b/c_glib/arrow-glib/basic-array.hpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#include <arrow-glib/basic-array.h>
+
+GArrowArray *garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array);
+std::shared_ptr<arrow::Array> garrow_array_get_raw(GArrowArray *array);
diff --git a/c_glib/arrow-glib/basic-data-type.cpp b/c_glib/arrow-glib/basic-data-type.cpp
index 62f8a7e4d2fbe..dcafe06d67581 100644
--- a/c_glib/arrow-glib/basic-data-type.cpp
+++ b/c_glib/arrow-glib/basic-data-type.cpp
@@ -220,6 +220,37 @@ garrow_data_type_get_id(GArrowDataType *data_type)
 }
 
 
+G_DEFINE_ABSTRACT_TYPE(GArrowFixedWidthDataType,                 \
+                       garrow_fixed_width_data_type,             \
+                       GARROW_TYPE_DATA_TYPE)
+
+static void
+garrow_fixed_width_data_type_init(GArrowFixedWidthDataType *object)
+{
+}
+
+static void
+garrow_fixed_width_data_type_class_init(GArrowFixedWidthDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_fixed_width_data_type_get_id:
+ * @data_type: A #GArrowFixedWidthDataType.
+ *
+ * Returns: The number of bits for one data.
+ */
+gint
+garrow_fixed_width_data_type_get_bit_width(GArrowFixedWidthDataType *data_type)
+{
+  const auto arrow_data_type =
+    garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  const auto arrow_fixed_width_type =
+    std::static_pointer_cast<arrow::FixedWidthType>(arrow_data_type);
+  return arrow_fixed_width_type->bit_width();
+}
+
+
 G_DEFINE_TYPE(GArrowNullDataType,                \
               garrow_null_data_type,             \
               GARROW_TYPE_DATA_TYPE)
@@ -254,7 +285,7 @@ garrow_null_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowBooleanDataType,                \
               garrow_boolean_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_FIXED_WIDTH_DATA_TYPE)
 
 static void
 garrow_boolean_data_type_init(GArrowBooleanDataType *object)
@@ -1015,12 +1046,15 @@ garrow_data_type_new_raw(std::shared_ptr<arrow::DataType> *arrow_data_type)
   case arrow::Type::type::STRUCT:
     type = GARROW_TYPE_STRUCT_DATA_TYPE;
     break;
+  case arrow::Type::type::DICTIONARY:
+    type = GARROW_TYPE_DICTIONARY_DATA_TYPE;
+    break;
   default:
     type = GARROW_TYPE_DATA_TYPE;
     break;
   }
   data_type = GARROW_DATA_TYPE(g_object_new(type,
-                                            "data_type", arrow_data_type,
+                                            "data-type", arrow_data_type,
                                             NULL));
   return data_type;
 }
diff --git a/c_glib/arrow-glib/basic-data-type.h b/c_glib/arrow-glib/basic-data-type.h
index d3d52c6120de7..bc0192f1b0062 100644
--- a/c_glib/arrow-glib/basic-data-type.h
+++ b/c_glib/arrow-glib/basic-data-type.h
@@ -19,57 +19,45 @@
 
 #pragma once
 
+#include <arrow-glib/gobject-type.h>
 #include <arrow-glib/type.h>
 
 G_BEGIN_DECLS
 
-#define GARROW_TYPE_DATA_TYPE                   \
-  (garrow_data_type_get_type())
-#define GARROW_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DATA_TYPE,    \
-                              GArrowDataType))
-#define GARROW_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DATA_TYPE,       \
-                           GArrowDataTypeClass))
-#define GARROW_IS_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_DATA_TYPE))
-#define GARROW_IS_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DATA_TYPE))
-#define GARROW_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DATA_TYPE,     \
-                             GArrowDataTypeClass))
-
-typedef struct _GArrowDataType         GArrowDataType;
-typedef struct _GArrowDataTypeClass    GArrowDataTypeClass;
-
-/**
- * GArrowDataType:
- *
- * It wraps `arrow::DataType`.
- */
-struct _GArrowDataType
-{
-  /*< private >*/
-  GObject parent_instance;
-};
-
+#define GARROW_TYPE_DATA_TYPE (garrow_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDataType,
+                         garrow_data_type,
+                         GARROW,
+                         DATA_TYPE,
+                         GObject)
 struct _GArrowDataTypeClass
 {
   GObjectClass parent_class;
 };
 
-GType      garrow_data_type_get_type  (void) G_GNUC_CONST;
 gboolean   garrow_data_type_equal     (GArrowDataType *data_type,
                                        GArrowDataType *other_data_type);
 gchar     *garrow_data_type_to_string (GArrowDataType *data_type);
 GArrowType garrow_data_type_get_id    (GArrowDataType *data_type);
 
 
+#define GARROW_TYPE_FIXED_WIDTH_DATA_TYPE (garrow_fixed_width_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowFixedWidthDataType,
+                         garrow_fixed_width_data_type,
+                         GARROW,
+                         FIXED_WIDTH_DATA_TYPE,
+                         GArrowDataType)
+struct _GArrowFixedWidthDataTypeClass
+{
+  GArrowDataTypeClass parent_class;
+};
+
+gint garrow_fixed_width_data_type_get_bit_width(GArrowFixedWidthDataType *data_type);
+/* TODO:
+GList *garrow_fixed_width_data_type_get_buffer_layout(GArrowFixedWidthDataType *data_type);
+*/
+
+
 #define GARROW_TYPE_NULL_DATA_TYPE              \
   (garrow_null_data_type_get_type())
 #define GARROW_NULL_DATA_TYPE(obj)                              \
@@ -114,47 +102,17 @@ GType               garrow_null_data_type_get_type (void) G_GNUC_CONST;
 GArrowNullDataType *garrow_null_data_type_new      (void);
 
 
-#define GARROW_TYPE_BOOLEAN_DATA_TYPE           \
-  (garrow_boolean_data_type_get_type())
-#define GARROW_BOOLEAN_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_BOOLEAN_DATA_TYPE,    \
-                              GArrowBooleanDataType))
-#define GARROW_BOOLEAN_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_BOOLEAN_DATA_TYPE,       \
-                           GArrowBooleanDataTypeClass))
-#define GARROW_IS_BOOLEAN_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_BOOLEAN_DATA_TYPE))
-#define GARROW_IS_BOOLEAN_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_BOOLEAN_DATA_TYPE))
-#define GARROW_BOOLEAN_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_BOOLEAN_DATA_TYPE,     \
-                             GArrowBooleanDataTypeClass))
-
-typedef struct _GArrowBooleanDataType         GArrowBooleanDataType;
-typedef struct _GArrowBooleanDataTypeClass    GArrowBooleanDataTypeClass;
-
-/**
- * GArrowBooleanDataType:
- *
- * It wraps `arrow::BooleanType`.
- */
-struct _GArrowBooleanDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_BOOLEAN_DATA_TYPE (garrow_boolean_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowBooleanDataType,
+                         garrow_boolean_data_type,
+                         GARROW,
+                         BOOLEAN_DATA_TYPE,
+                         GArrowFixedWidthDataType)
 struct _GArrowBooleanDataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowFixedWidthDataTypeClass parent_class;
 };
 
-GType                  garrow_boolean_data_type_get_type (void) G_GNUC_CONST;
 GArrowBooleanDataType *garrow_boolean_data_type_new      (void);
 
 
diff --git a/c_glib/arrow-glib/composite-array.cpp b/c_glib/arrow-glib/composite-array.cpp
new file mode 100644
index 0000000000000..445103d374037
--- /dev/null
+++ b/c_glib/arrow-glib/composite-array.cpp
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#include <arrow-glib/array.hpp>
+#include <arrow-glib/buffer.hpp>
+#include <arrow-glib/compute.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/error.hpp>
+#include <arrow-glib/type.hpp>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: composite-array
+ * @section_id: composite-array-classes
+ * @title: Composite array classes
+ * @include: arrow-glib/arrow-glib.h
+ *
+ * #GArrowListArray is a class for list array. It can store zero or
+ * more list data. If you don't have Arrow format data, you need to
+ * use #GArrowListArrayBuilder to create a new array.
+ *
+ * #GArrowStructArray is a class for struct array. It can store zero
+ * or more structs. One struct has zero or more fields. If you don't
+ * have Arrow format data, you need to use #GArrowStructArrayBuilder
+ * to create a new array.
+ */
+
+G_DEFINE_TYPE(GArrowListArray,               \
+              garrow_list_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_list_array_init(GArrowListArray *object)
+{
+}
+
+static void
+garrow_list_array_class_init(GArrowListArrayClass *klass)
+{
+}
+
+/**
+ * garrow_list_array_new:
+ * @length: The number of elements.
+ * @value_offsets: The offsets of @values in Arrow format.
+ * @values: The values as #GArrowArray.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowListArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowListArray *
+garrow_list_array_new(gint64 length,
+                      GArrowBuffer *value_offsets,
+                      GArrowArray *values,
+                      GArrowBuffer *null_bitmap,
+                      gint64 n_nulls)
+{
+  const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets);
+  const auto arrow_values = garrow_array_get_raw(values);
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_data_type = arrow::list(arrow_values->type());
+  auto arrow_list_array =
+    std::make_shared<arrow::ListArray>(arrow_data_type,
+                                       length,
+                                       arrow_value_offsets,
+                                       arrow_values,
+                                       arrow_bitmap,
+                                       n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_list_array);
+  return GARROW_LIST_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
+/**
+ * garrow_list_array_get_value_type:
+ * @array: A #GArrowListArray.
+ *
+ * Returns: (transfer full): The data type of value in each list.
+ */
+GArrowDataType *
+garrow_list_array_get_value_type(GArrowListArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_list_array =
+    static_cast<arrow::ListArray *>(arrow_array.get());
+  auto arrow_value_type = arrow_list_array->value_type();
+  return garrow_data_type_new_raw(&arrow_value_type);
+}
+
+/**
+ * garrow_list_array_get_value:
+ * @array: A #GArrowListArray.
+ * @i: The index of the target value.
+ *
+ * Returns: (transfer full): The i-th list.
+ */
+GArrowArray *
+garrow_list_array_get_value(GArrowListArray *array,
+                            gint64 i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_list_array =
+    static_cast<arrow::ListArray *>(arrow_array.get());
+  auto arrow_list =
+    arrow_list_array->values()->Slice(arrow_list_array->value_offset(i),
+                                      arrow_list_array->value_length(i));
+  return garrow_array_new_raw(&arrow_list);
+}
+
+
+G_DEFINE_TYPE(GArrowStructArray,               \
+              garrow_struct_array,             \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_struct_array_init(GArrowStructArray *object)
+{
+}
+
+static void
+garrow_struct_array_class_init(GArrowStructArrayClass *klass)
+{
+}
+
+/**
+ * garrow_struct_array_new:
+ * @data_type: The data type of the struct.
+ * @length: The number of elements.
+ * @children: (element-type GArrowArray): The arrays for each field
+ *   as #GList of #GArrowArray.
+ * @null_bitmap: (nullable): The bitmap that shows null elements. The
+ *   N-th element is null when the N-th bit is 0, not null otherwise.
+ *   If the array has no null elements, the bitmap must be %NULL and
+ *   @n_nulls is 0.
+ * @n_nulls: The number of null elements. If -1 is specified, the
+ *   number of nulls are computed from @null_bitmap.
+ *
+ * Returns: A newly created #GArrowStructArray.
+ *
+ * Since: 0.4.0
+ */
+GArrowStructArray *
+garrow_struct_array_new(GArrowDataType *data_type,
+                        gint64 length,
+                        GList *children,
+                        GArrowBuffer *null_bitmap,
+                        gint64 n_nulls)
+{
+  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  std::vector<std::shared_ptr<arrow::Array>> arrow_children;
+  for (GList *node = children; node; node = node->next) {
+    GArrowArray *child = GARROW_ARRAY(node->data);
+    arrow_children.push_back(garrow_array_get_raw(child));
+  }
+  const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap);
+  auto arrow_struct_array =
+    std::make_shared<arrow::StructArray>(arrow_data_type,
+                                         length,
+                                         arrow_children,
+                                         arrow_bitmap,
+                                         n_nulls);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_struct_array);
+  return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
+/**
+ * garrow_struct_array_get_field
+ * @array: A #GArrowStructArray.
+ * @i: The index of the field in the struct.
+ *
+ * Returns: (transfer full): The i-th field.
+ */
+GArrowArray *
+garrow_struct_array_get_field(GArrowStructArray *array,
+                              gint i)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_struct_array =
+    static_cast<arrow::StructArray *>(arrow_array.get());
+  auto arrow_field = arrow_struct_array->field(i);
+  return garrow_array_new_raw(&arrow_field);
+}
+
+/**
+ * garrow_struct_array_get_fields
+ * @array: A #GArrowStructArray.
+ *
+ * Returns: (element-type GArrowArray) (transfer full):
+ *   The fields in the struct.
+ */
+GList *
+garrow_struct_array_get_fields(GArrowStructArray *array)
+{
+  const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  const auto arrow_struct_array =
+    static_cast<const arrow::StructArray *>(arrow_array.get());
+
+  GList *fields = NULL;
+  for (int i = 0; i < arrow_struct_array->num_fields(); ++i) {
+    auto arrow_field = arrow_struct_array->field(i);
+    GArrowArray *field = garrow_array_new_raw(&arrow_field);
+    fields = g_list_prepend(fields, field);
+  }
+
+  return g_list_reverse(fields);
+}
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/composite-array.h b/c_glib/arrow-glib/composite-array.h
new file mode 100644
index 0000000000000..ebf955498724b
--- /dev/null
+++ b/c_glib/arrow-glib/composite-array.h
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow-glib/basic-array.h>
+#include <arrow-glib/data-type.h>
+
+G_BEGIN_DECLS
+
+#define GARROW_TYPE_LIST_ARRAY                  \
+  (garrow_list_array_get_type())
+#define GARROW_LIST_ARRAY(obj)                          \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_LIST_ARRAY,   \
+                              GArrowListArray))
+#define GARROW_LIST_ARRAY_CLASS(klass)                  \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_LIST_ARRAY,      \
+                           GArrowListArrayClass))
+#define GARROW_IS_LIST_ARRAY(obj)                       \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
+                              GARROW_TYPE_LIST_ARRAY))
+#define GARROW_IS_LIST_ARRAY_CLASS(klass)               \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_LIST_ARRAY))
+#define GARROW_LIST_ARRAY_GET_CLASS(obj)                \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_LIST_ARRAY,    \
+                             GArrowListArrayClass))
+
+typedef struct _GArrowListArray         GArrowListArray;
+typedef struct _GArrowListArrayClass    GArrowListArrayClass;
+
+/**
+ * GArrowListArray:
+ *
+ * It wraps `arrow::ListArray`.
+ */
+struct _GArrowListArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowListArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_list_array_get_type(void) G_GNUC_CONST;
+
+GArrowListArray *garrow_list_array_new(gint64 length,
+                                       GArrowBuffer *value_offsets,
+                                       GArrowArray *values,
+                                       GArrowBuffer *null_bitmap,
+                                       gint64 n_nulls);
+
+GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array);
+GArrowArray *garrow_list_array_get_value(GArrowListArray *array,
+                                         gint64 i);
+
+
+#define GARROW_TYPE_STRUCT_ARRAY                \
+  (garrow_struct_array_get_type())
+#define GARROW_STRUCT_ARRAY(obj)                        \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
+                              GARROW_TYPE_STRUCT_ARRAY, \
+                              GArrowStructArray))
+#define GARROW_STRUCT_ARRAY_CLASS(klass)                \
+  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
+                           GARROW_TYPE_STRUCT_ARRAY,    \
+                           GArrowStructArrayClass))
+#define GARROW_IS_STRUCT_ARRAY(obj)                             \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
+                              GARROW_TYPE_STRUCT_ARRAY))
+#define GARROW_IS_STRUCT_ARRAY_CLASS(klass)             \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
+                           GARROW_TYPE_STRUCT_ARRAY))
+#define GARROW_STRUCT_ARRAY_GET_CLASS(obj)              \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
+                             GARROW_TYPE_STRUCT_ARRAY,  \
+                             GArrowStructArrayClass))
+
+typedef struct _GArrowStructArray         GArrowStructArray;
+typedef struct _GArrowStructArrayClass    GArrowStructArrayClass;
+
+/**
+ * GArrowStructArray:
+ *
+ * It wraps `arrow::StructArray`.
+ */
+struct _GArrowStructArray
+{
+  /*< private >*/
+  GArrowArray parent_instance;
+};
+
+struct _GArrowStructArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GType garrow_struct_array_get_type(void) G_GNUC_CONST;
+
+GArrowStructArray *garrow_struct_array_new(GArrowDataType *data_type,
+                                           gint64 length,
+                                           GList *children,
+                                           GArrowBuffer *null_bitmap,
+                                           gint64 n_nulls);
+
+GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
+                                           gint i);
+GList *garrow_struct_array_get_fields(GArrowStructArray *array);
+
+G_END_DECLS
diff --git a/c_glib/arrow-glib/composite-data-type.cpp b/c_glib/arrow-glib/composite-data-type.cpp
index ce3d78ca08332..7ce8a978e2036 100644
--- a/c_glib/arrow-glib/composite-data-type.cpp
+++ b/c_glib/arrow-glib/composite-data-type.cpp
@@ -21,6 +21,7 @@
 #  include <config.h>
 #endif
 
+#include <arrow-glib/basic-array.hpp>
 #include <arrow-glib/data-type.hpp>
 #include <arrow-glib/enums.h>
 #include <arrow-glib/error.hpp>
@@ -38,6 +39,8 @@ G_BEGIN_DECLS
  * #GArrowListDataType is a class for list data type.
  *
  * #GArrowStructDataType is a class for struct data type.
+ *
+ * #GArrowDictionaryDataType is a class for dictionary data type.
  */
 
 G_DEFINE_TYPE(GArrowListDataType,                \
@@ -133,4 +136,87 @@ garrow_struct_data_type_new(GList *fields)
   return data_type;
 }
 
+
+G_DEFINE_TYPE(GArrowDictionaryDataType,                \
+              garrow_dictionary_data_type,             \
+              GARROW_TYPE_FIXED_WIDTH_DATA_TYPE)
+
+static void
+garrow_dictionary_data_type_init(GArrowDictionaryDataType *object)
+{
+}
+
+static void
+garrow_dictionary_data_type_class_init(GArrowDictionaryDataTypeClass *klass)
+{
+}
+
+/**
+ * garrow_dictionary_data_type_new:
+ * @index_data_type: The data type of index.
+ * @dictionary: The dictionary.
+ * @ordered: Whether dictionary contents are ordered or not.
+ *
+ * Returns: The newly created dictionary data type.
+ */
+GArrowDictionaryDataType *
+garrow_dictionary_data_type_new(GArrowDataType *index_data_type,
+                                GArrowArray *dictionary,
+                                gboolean ordered)
+{
+  auto arrow_index_data_type = garrow_data_type_get_raw(index_data_type);
+  auto arrow_dictionary = garrow_array_get_raw(dictionary);
+  auto arrow_data_type = arrow::dictionary(arrow_index_data_type,
+                                           arrow_dictionary,
+                                           ordered);
+  return GARROW_DICTIONARY_DATA_TYPE(garrow_data_type_new_raw(&arrow_data_type));
+}
+
+/**
+ * garrow_dictionary_data_type_get_index_data_type:
+ * @data_type: The #GArrowDictionaryDataType.
+ *
+ * Returns: (transfer full): The #GArrowDataType of index.
+ */
+GArrowDataType *
+garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_type)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto arrow_dictionary_data_type =
+    std::static_pointer_cast<arrow::DictionaryType>(arrow_data_type);
+  auto arrow_index_data_type = arrow_dictionary_data_type->index_type();
+  return garrow_data_type_new_raw(&arrow_index_data_type);
+}
+
+/**
+ * garrow_dictionary_data_type_get_dictionary:
+ * @data_type: The #GArrowDictionaryDataType.
+ *
+ * Returns: (transfer full): The dictionary as #GArrowArray.
+ */
+GArrowArray *
+garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto arrow_dictionary_data_type =
+    std::static_pointer_cast<arrow::DictionaryType>(arrow_data_type);
+  auto arrow_dictionary = arrow_dictionary_data_type->dictionary();
+  return garrow_array_new_raw(&arrow_dictionary);
+}
+
+/**
+ * garrow_dictionary_data_type_is_ordered:
+ * @data_type: The #GArrowDictionaryDataType.
+ *
+ * Returns: Whether dictionary contents are ordered or not.
+ */
+gboolean
+garrow_dictionary_data_type_is_ordered(GArrowDictionaryDataType *data_type)
+{
+  auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type));
+  auto arrow_dictionary_data_type =
+    std::static_pointer_cast<arrow::DictionaryType>(arrow_data_type);
+  return arrow_dictionary_data_type->ordered();
+}
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/composite-data-type.h b/c_glib/arrow-glib/composite-data-type.h
index 9dac5bd84bed2..11b83014ff830 100644
--- a/c_glib/arrow-glib/composite-data-type.h
+++ b/c_glib/arrow-glib/composite-data-type.h
@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include <arrow-glib/basic-array.h>
 #include <arrow-glib/basic-data-type.h>
 #include <arrow-glib/field.h>
 
@@ -112,4 +113,28 @@ struct _GArrowStructDataTypeClass
 GType                 garrow_struct_data_type_get_type (void) G_GNUC_CONST;
 GArrowStructDataType *garrow_struct_data_type_new      (GList *fields);
 
+
+#define GARROW_TYPE_DICTIONARY_DATA_TYPE (garrow_dictionary_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDictionaryDataType,
+                         garrow_dictionary_data_type,
+                         GARROW,
+                         DICTIONARY_DATA_TYPE,
+                         GArrowFixedWidthDataType)
+struct _GArrowDictionaryDataTypeClass
+{
+  GArrowFixedWidthDataTypeClass parent_class;
+};
+
+GArrowDictionaryDataType *
+garrow_dictionary_data_type_new(GArrowDataType *index_data_type,
+                                GArrowArray *dictionary,
+                                gboolean ordered);
+GArrowDataType *
+garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_type);
+GArrowArray *
+garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type);
+gboolean
+garrow_dictionary_data_type_is_ordered(GArrowDictionaryDataType *data_type);
+
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/meson.build b/c_glib/arrow-glib/meson.build
index aeec4172d6284..25968e69c1047 100644
--- a/c_glib/arrow-glib/meson.build
+++ b/c_glib/arrow-glib/meson.build
@@ -18,12 +18,13 @@
 # under the License.
 
 sources = files(
-  'array.cpp',
   'array-builder.cpp',
+  'basic-array.cpp',
   'basic-data-type.cpp',
   'buffer.cpp',
   'chunked-array.cpp',
   'column.cpp',
+  'composite-array.cpp',
   'composite-data-type.cpp',
   'error.cpp',
   'field.cpp',
@@ -59,10 +60,12 @@ c_headers = files(
   'array.h',
   'array-builder.h',
   'arrow-glib.h',
+  'basic-array.h',
   'basic-data-type.h',
   'buffer.h',
   'chunked-array.h',
   'column.h',
+  'composite-array.h',
   'composite-data-type.h',
   'data-type.h',
   'error.h',
@@ -102,6 +105,7 @@ cpp_headers = files(
   'array.hpp',
   'array-builder.hpp',
   'arrow-glib.hpp',
+  'basic-array.hpp',
   'basic-data-type.hpp',
   'buffer.hpp',
   'chunked-array.hpp',
diff --git a/c_glib/doc/reference/arrow-glib-docs.xml b/c_glib/doc/reference/arrow-glib-docs.xml
index e267ea2f9d356..51e7b2a6a6cf5 100644
--- a/c_glib/doc/reference/arrow-glib-docs.xml
+++ b/c_glib/doc/reference/arrow-glib-docs.xml
@@ -40,7 +40,8 @@
     <title>Data</title>
     <chapter id="array">
       <title>Array</title>
-      <xi:include href="xml/array.xml"/>
+      <xi:include href="xml/basic-array.xml"/>
+      <xi:include href="xml/composite-array.xml"/>
     </chapter>
     <chapter id="array-builder">
       <title>Array builder</title>
diff --git a/c_glib/test/test-dictionary-data-type.rb b/c_glib/test/test-dictionary-data-type.rb
new file mode 100644
index 0000000000000..5530a0415cb28
--- /dev/null
+++ b/c_glib/test/test-dictionary-data-type.rb
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDictionaryDataType < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def setup
+    @index_data_type = Arrow::Int32DataType.new
+    @dictionary = build_string_array(["C", "C++", "Ruby"])
+    @ordered = true
+    @data_type = Arrow::DictionaryDataType.new(@index_data_type,
+                                               @dictionary,
+                                               @ordered)
+  end
+
+  def test_type
+    assert_equal(Arrow::Type::DICTIONARY, @data_type.id)
+  end
+
+  def test_to_s
+    assert_equal("dictionary<values=string, indices=int32, ordered=1>",
+                 @data_type.to_s)
+  end
+
+  def test_bit_width
+    assert_equal(32, @data_type.bit_width)
+  end
+
+  def test_index_data_type
+    assert_equal(@index_data_type, @data_type.index_data_type)
+  end
+
+  def test_dictionary
+    assert_equal(@dictionary, @data_type.dictionary)
+  end
+
+  def test_ordered?
+    assert do
+      @data_type.ordered?
+    end
+  end
+end

From 85e2d8960d5aeeb04d8b59e6e7d8a8266a7d095f Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Sun, 26 Nov 2017 14:12:22 -0800
Subject: [PATCH 1287/1644] ARROW-1758: [Python] Remove pickle=True option for
 object serialization

This closes [ARROW-1758](https://issues.apache.org/jira/browse/ARROW-1758).

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1347 from Licht-T/clean-pickle-option-for-object-serialization and squashes the following commits:

927f154 [Wes McKinney] Use cloudpickle for lambda serialization if available
ba998dd [Licht-T] CLN: Remove pickle=True option for object serialization
---
 python/pyarrow/serialization.pxi           | 14 ++------------
 python/pyarrow/serialization.py            | 13 +++++++++++--
 python/pyarrow/tests/test_serialization.py |  9 ++++++---
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index 6b7227797a836..3ee5c7d4e2770 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -47,7 +47,6 @@ cdef class SerializationContext:
     cdef:
         object type_to_type_id
         object whitelisted_types
-        object types_to_pickle
         object custom_serializers
         object custom_deserializers
 
@@ -55,11 +54,10 @@ cdef class SerializationContext:
         # Types with special serialization handlers
         self.type_to_type_id = dict()
         self.whitelisted_types = dict()
-        self.types_to_pickle = set()
         self.custom_serializers = dict()
         self.custom_deserializers = dict()
 
-    def register_type(self, type_, type_id, pickle=False,
+    def register_type(self, type_, type_id,
                       custom_serializer=None, custom_deserializer=None):
         """EXPERIMENTAL: Add type to the list of types we can serialize.
 
@@ -69,9 +67,6 @@ cdef class SerializationContext:
             The type that we can serialize.
         type_id : bytes
             A string of bytes used to identify the type.
-        pickle : bool
-            True if the serialization should be done with pickle.
-            False if it should be done efficiently with Arrow.
         custom_serializer : callable
             This argument is optional, but can be provided to
             serialize objects of the class in a particular way.
@@ -81,8 +76,6 @@ cdef class SerializationContext:
         """
         self.type_to_type_id[type_] = type_id
         self.whitelisted_types[type_id] = type_
-        if pickle:
-            self.types_to_pickle.add(type_id)
         if custom_serializer is not None:
             self.custom_serializers[type_id] = custom_serializer
             self.custom_deserializers[type_id] = custom_deserializer
@@ -102,9 +95,7 @@ cdef class SerializationContext:
 
         # use the closest match to type(obj)
         type_id = self.type_to_type_id[type_]
-        if type_id in self.types_to_pickle:
-            serialized_obj = {"data": pickle.dumps(obj), "pickle": True}
-        elif type_id in self.custom_serializers:
+        if type_id in self.custom_serializers:
             serialized_obj = {"data": self.custom_serializers[type_id](obj)}
         else:
             if is_named_tuple(type_):
@@ -125,7 +116,6 @@ cdef class SerializationContext:
             # The object was pickled, so unpickle it.
             obj = pickle.loads(serialized_obj["data"])
         else:
-            assert type_id not in self.types_to_pickle
             if type_id not in self.whitelisted_types:
                 msg = "Type ID " + str(type_id) + " not registered in " \
                       "deserialization callback"
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 2b47513fd1c85..ab25b63d5710b 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -17,12 +17,18 @@
 
 from collections import OrderedDict, defaultdict
 import sys
+import pickle
 
 import numpy as np
 
 from pyarrow import serialize_pandas, deserialize_pandas
 from pyarrow.lib import _default_serialization_context
 
+try:
+    import cloudpickle
+except ImportError:
+    cloudpickle = pickle
+
 
 def register_default_serialization_handlers(serialization_context):
 
@@ -67,9 +73,12 @@ def _deserialize_default_dict(data):
 
     serialization_context.register_type(
         type(lambda: 0), "function",
-        pickle=True)
+        custom_serializer=cloudpickle.dumps,
+        custom_deserializer=cloudpickle.loads)
 
-    serialization_context.register_type(type, "type", pickle=True)
+    serialization_context.register_type(type, "type",
+                                        custom_serializer=cloudpickle.dumps,
+                                        custom_deserializer=cloudpickle.loads)
 
     # ----------------------------------------------------------------------
     # Set up serialization for numpy with dtype object (primitive types are
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index b0c5bc49e6a58..ed4fd9ae59aee 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -23,6 +23,7 @@
 import datetime
 import string
 import sys
+import pickle
 
 import pyarrow as pa
 import numpy as np
@@ -197,7 +198,9 @@ def make_serialization_context():
     context.register_type(Baz, "Baz")
     context.register_type(Qux, "Quz")
     context.register_type(SubQux, "SubQux")
-    context.register_type(SubQuxPickle, "SubQuxPickle", pickle=True)
+    context.register_type(SubQuxPickle, "SubQuxPickle",
+                          custom_serializer=pickle.dumps,
+                          custom_deserializer=pickle.loads)
     context.register_type(Exception, "Exception")
     context.register_type(CustomError, "CustomError")
     context.register_type(Point, "Point")
@@ -338,7 +341,7 @@ def deserialize_dummy_class(serialized_obj):
         return serialized_obj
 
     pa._default_serialization_context.register_type(
-        DummyClass, "DummyClass", pickle=False,
+        DummyClass, "DummyClass",
         custom_serializer=serialize_dummy_class,
         custom_deserializer=deserialize_dummy_class)
 
@@ -357,7 +360,7 @@ def deserialize_buffer_class(serialized_obj):
         return serialized_obj
 
     pa._default_serialization_context.register_type(
-        BufferClass, "BufferClass", pickle=False,
+        BufferClass, "BufferClass",
         custom_serializer=serialize_buffer_class,
         custom_deserializer=deserialize_buffer_class)
 

From 42fc57be9c768add32b278adbee2f6f30b10005d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Nov 2017 09:42:50 +0100
Subject: [PATCH 1288/1644] ARROW-1178: [C++/Python] Add option to set
 chunksize in TableBatchReader, Table.to_batches method

This also fixes ARROW-504 by adding a chunksize option when writing tables to a RecordBatch stream in Python

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1364 from wesm/ARROW-1178 and squashes the following commits:

a31e258 [Wes McKinney] Add chunksize argument to RecordBatchWriter.write_table
dc6023a [Wes McKinney] Implement Table.to_batches, add tests
---
 cpp/src/arrow/ipc/writer.cc          |  8 +++++-
 cpp/src/arrow/ipc/writer.h           |  6 +++++
 cpp/src/arrow/table-test.cc          | 33 ++++++++++++++++++++++++
 cpp/src/arrow/table.cc               | 21 ++++++++++-----
 cpp/src/arrow/table.h                |  2 ++
 python/pyarrow/includes/libarrow.pxd |  9 ++++++-
 python/pyarrow/ipc.pxi               | 12 +++++++--
 python/pyarrow/table.pxi             | 38 ++++++++++++++++++++++++++++
 python/pyarrow/tests/test_ipc.py     | 23 +++++++++++++++++
 python/pyarrow/tests/test_table.py   | 25 ++++++++++++++++++
 10 files changed, 167 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 3c1db06159e65..fb766a9a7599c 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -651,9 +651,13 @@ Status GetTensorSize(const Tensor& tensor, int64_t* size) {
 
 RecordBatchWriter::~RecordBatchWriter() {}
 
-Status RecordBatchWriter::WriteTable(const Table& table) {
+Status RecordBatchWriter::WriteTable(const Table& table, int64_t max_chunksize) {
   TableBatchReader reader(table);
 
+  if (max_chunksize > 0) {
+    reader.set_chunksize(max_chunksize);
+  }
+
   std::shared_ptr<RecordBatch> batch;
   while (true) {
     RETURN_NOT_OK(reader.ReadNext(&batch));
@@ -666,6 +670,8 @@ Status RecordBatchWriter::WriteTable(const Table& table) {
   return Status::OK();
 }
 
+Status RecordBatchWriter::WriteTable(const Table& table) { return WriteTable(table, -1); }
+
 // ----------------------------------------------------------------------
 // Stream writer implementation
 
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index cedac45e712d6..457dcb4ec6a13 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -65,6 +65,12 @@ class ARROW_EXPORT RecordBatchWriter {
   /// \return Status
   Status WriteTable(const Table& table);
 
+  /// \brief Write Table with a particular chunksize
+  /// \param[in] table table to write
+  /// \param[in] max_chunksize maximum chunk size for table chunks
+  /// \return Status
+  Status WriteTable(const Table& table, int64_t max_chunksize);
+
   /// \brief Perform any logic necessary to finish the stream
   ///
   /// \return Status
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index e77d3aa8bbc08..8a2288710c896 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -586,4 +586,37 @@ TEST_F(TestTableBatchReader, ReadNext) {
   ASSERT_EQ(nullptr, batch);
 }
 
+TEST_F(TestTableBatchReader, Chunksize) {
+  auto a1 = MakeRandomArray<Int32Array>(10);
+  auto a2 = MakeRandomArray<Int32Array>(20);
+  auto a3 = MakeRandomArray<Int32Array>(10);
+
+  auto sch1 = arrow::schema({field("f1", int32())});
+  auto t1 = Table::Make(sch1, {column(sch1->field(0), {a1, a2, a3})});
+
+  TableBatchReader i1(*t1);
+
+  i1.set_chunksize(15);
+
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_OK(batch->Validate());
+  ASSERT_EQ(10, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_OK(batch->Validate());
+  ASSERT_EQ(15, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_OK(batch->Validate());
+  ASSERT_EQ(5, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_OK(batch->Validate());
+  ASSERT_EQ(10, batch->num_rows());
+
+  ASSERT_OK(i1.ReadNext(&batch));
+  ASSERT_EQ(nullptr, batch);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 8f3f195765aa9..129524b7e4320 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -19,6 +19,7 @@
 
 #include <algorithm>
 #include <cstdlib>
+#include <limits>
 #include <memory>
 #include <sstream>
 
@@ -403,7 +404,8 @@ class TableBatchReader::TableBatchReaderImpl {
         column_data_(table.num_columns()),
         chunk_numbers_(table.num_columns(), 0),
         chunk_offsets_(table.num_columns(), 0),
-        absolute_row_position_(0) {
+        absolute_row_position_(0),
+        max_chunksize_(std::numeric_limits<int64_t>::max()) {
     for (int i = 0; i < table.num_columns(); ++i) {
       column_data_[i] = table.column(i)->data().get();
     }
@@ -416,7 +418,7 @@ class TableBatchReader::TableBatchReaderImpl {
     }
 
     // Determine the minimum contiguous slice across all columns
-    int64_t chunksize = table_.num_rows();
+    int64_t chunksize = std::min(table_.num_rows(), max_chunksize_);
     std::vector<const Array*> chunks(table_.num_columns());
     for (int i = 0; i < table_.num_columns(); ++i) {
       auto chunk = column_data_[i]->chunk(chunk_numbers_[i]).get();
@@ -430,8 +432,7 @@ class TableBatchReader::TableBatchReaderImpl {
     }
 
     // Slice chunks and advance chunk index as appropriate
-    std::vector<std::shared_ptr<ArrayData>> batch_data;
-    batch_data.reserve(table_.num_columns());
+    std::vector<std::shared_ptr<ArrayData>> batch_data(table_.num_columns());
 
     for (int i = 0; i < table_.num_columns(); ++i) {
       // Exhausted chunk
@@ -441,7 +442,7 @@ class TableBatchReader::TableBatchReaderImpl {
       if ((chunk->length() - offset) == chunksize) {
         ++chunk_numbers_[i];
         chunk_offsets_[i] = 0;
-        if (chunk_offsets_[i] > 0) {
+        if (offset > 0) {
           // Need to slice
           slice_data = chunk->Slice(offset, chunksize)->data();
         } else {
@@ -449,9 +450,10 @@ class TableBatchReader::TableBatchReaderImpl {
           slice_data = chunk->data();
         }
       } else {
+        chunk_offsets_[i] += chunksize;
         slice_data = chunk->Slice(offset, chunksize)->data();
       }
-      batch_data.emplace_back(std::move(slice_data));
+      batch_data[i] = std::move(slice_data);
     }
 
     absolute_row_position_ += chunksize;
@@ -462,12 +464,15 @@ class TableBatchReader::TableBatchReaderImpl {
 
   std::shared_ptr<Schema> schema() const { return table_.schema(); }
 
+  void set_chunksize(int64_t chunksize) { max_chunksize_ = chunksize; }
+
  private:
   const Table& table_;
   std::vector<ChunkedArray*> column_data_;
   std::vector<int> chunk_numbers_;
   std::vector<int64_t> chunk_offsets_;
   int64_t absolute_row_position_;
+  int64_t max_chunksize_;
 };
 
 TableBatchReader::TableBatchReader(const Table& table) {
@@ -478,6 +483,10 @@ TableBatchReader::~TableBatchReader() {}
 
 std::shared_ptr<Schema> TableBatchReader::schema() const { return impl_->schema(); }
 
+void TableBatchReader::set_chunksize(int64_t chunksize) {
+  impl_->set_chunksize(chunksize);
+}
+
 Status TableBatchReader::ReadNext(std::shared_ptr<RecordBatch>* out) {
   return impl_->ReadNext(out);
 }
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index d0312d93cb93c..c813b32ad36dc 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -197,6 +197,8 @@ class ARROW_EXPORT TableBatchReader : public RecordBatchReader {
 
   Status ReadNext(std::shared_ptr<RecordBatch>* out) override;
 
+  void set_chunksize(int64_t chunksize);
+
  private:
   class TableBatchReaderImpl;
   std::unique_ptr<TableBatchReaderImpl> impl_;
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 5d68607efa386..14211787c89f2 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -456,6 +456,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         shared_ptr[CTable] ReplaceSchemaMetadata(
             const shared_ptr[CKeyValueMetadata]& metadata)
 
+    cdef cppclass RecordBatchReader:
+        CStatus ReadNext(shared_ptr[CRecordBatch]* out)
+
+    cdef cppclass TableBatchReader(RecordBatchReader):
+        TableBatchReader(const CTable& table)
+        void set_chunksize(int64_t chunksize)
+
     cdef cppclass CTensor" arrow::Tensor":
         shared_ptr[CDataType] type()
         shared_ptr[CBuffer] data()
@@ -692,7 +699,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
         CStatus Close()
         CStatus WriteRecordBatch(const CRecordBatch& batch,
                                  c_bool allow_64bit)
-        CStatus WriteTable(const CTable& table)
+        CStatus WriteTable(const CTable& table, int64_t max_chunksize)
 
     cdef cppclass CRecordBatchReader" arrow::ipc::RecordBatchReader":
         shared_ptr[CSchema] schema()
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index 27e9167750991..e5639137dd3a1 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -202,7 +202,7 @@ cdef class _RecordBatchWriter:
             check_status(self.writer.get()
                          .WriteRecordBatch(deref(batch.batch), 1))
 
-    def write_table(self, Table table):
+    def write_table(self, Table table, chunksize=None):
         """
         Write RecordBatch to stream
 
@@ -210,8 +210,16 @@ cdef class _RecordBatchWriter:
         ----------
         batch : RecordBatch
         """
+        cdef:
+            # Chunksize must be > 0 to have any impact
+            int64_t c_chunksize = -1
+
+        if chunksize is not None:
+            c_chunksize = chunksize
+
         with nogil:
-            check_status(self.writer.get().WriteTable(table.table[0]))
+            check_status(self.writer.get().WriteTable(table.table[0],
+                                                      c_chunksize))
 
     def close(self):
         """
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 8c5b8bbc343c0..b03ee26702245 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -971,6 +971,44 @@ cdef class Table:
 
         return pyarrow_wrap_table(c_table)
 
+    def to_batches(self, chunksize=None):
+        """
+        Convert Table to list of (contiguous) RecordBatch objects, with optimal
+        maximum chunk size
+
+        Parameters
+        ----------
+        chunksize : int, default None
+            Maximum size for RecordBatch chunks. Individual chunks may be
+            smaller depending on the chunk layout of individual columns
+
+        Returns
+        -------
+        batches : list of RecordBatch
+        """
+        cdef:
+            unique_ptr[TableBatchReader] reader
+            int64_t c_chunksize
+            list result = []
+            shared_ptr[CRecordBatch] batch
+
+        reader.reset(new TableBatchReader(deref(self.table)))
+
+        if chunksize is not None:
+            c_chunksize = chunksize
+            reader.get().set_chunksize(c_chunksize)
+
+        while True:
+            with nogil:
+                check_status(reader.get().ReadNext(&batch))
+
+            if batch.get() == NULL:
+                break
+
+            result.append(pyarrow_wrap_batch(batch))
+
+        return result
+
     def to_pandas(self, nthreads=None, strings_to_categorical=False,
                   memory_pool=None, zero_copy_only=False):
         """
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 5033ea95783ab..9cd5f807662d6 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -168,6 +168,29 @@ def test_stream_write_dispatch(self):
         assert_frame_equal(table.to_pandas(),
                            pd.concat([df, df], ignore_index=True))
 
+    def test_stream_write_table_batches(self):
+        # ARROW-504
+        df = pd.DataFrame({
+            'one': np.random.randn(20),
+        })
+
+        b1 = pa.RecordBatch.from_pandas(df[:10], preserve_index=False)
+        b2 = pa.RecordBatch.from_pandas(df, preserve_index=False)
+
+        table = pa.Table.from_batches([b1, b2, b1])
+
+        writer = self._get_writer(self.sink, table.schema)
+        writer.write_table(table, chunksize=15)
+        writer.close()
+
+        batches = list(pa.open_stream(pa.BufferReader(self._get_source())))
+
+        assert list(map(len, batches)) == [10, 15, 5, 10]
+        result_table = pa.Table.from_batches(batches)
+        assert_frame_equal(result_table.to_pandas(),
+                           pd.concat([df[:10], df, df[:10]],
+                                     ignore_index=True))
+
     def test_simple_roundtrip(self):
         _, batches = self.write_batches()
         file_contents = pa.BufferReader(self._get_source())
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index cd05fb8e1fce5..ab012340c0ab9 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -213,6 +213,31 @@ def test_recordbatchlist_schema_equals():
         pa.Table.from_batches([batch1, batch2])
 
 
+def test_table_to_batches():
+    df1 = pd.DataFrame({'a': list(range(10))})
+    df2 = pd.DataFrame({'a': list(range(10, 30))})
+
+    batch1 = pa.RecordBatch.from_pandas(df1, preserve_index=False)
+    batch2 = pa.RecordBatch.from_pandas(df2, preserve_index=False)
+
+    table = pa.Table.from_batches([batch1, batch2, batch1])
+
+    expected_df = pd.concat([df1, df2, df1], ignore_index=True)
+
+    batches = table.to_batches()
+    assert len(batches) == 3
+
+    assert_frame_equal(pa.Table.from_batches(batches).to_pandas(),
+                       expected_df)
+
+    batches = table.to_batches(chunksize=15)
+    assert list(map(len, batches)) == [10, 15, 5, 10]
+
+    assert_frame_equal(table.to_pandas(), expected_df)
+    assert_frame_equal(pa.Table.from_batches(batches).to_pandas(),
+                       expected_df)
+
+
 def test_table_basics():
     data = [
         pa.array(range(5)),

From 61763504e53fb5531f1d642c399b8ec7b094b337 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Nov 2017 10:36:33 -0500
Subject: [PATCH 1289/1644] [Release] Apache Arrow JavaScript 0.2.0

Change-Id: I1db065001e7fc196128e8f8c36b3406a89ccbdd5
---
 js/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/package.json b/js/package.json
index 9933f75a2ce57..8e5c42ef7d1e3 100644
--- a/js/package.json
+++ b/js/package.json
@@ -1,6 +1,6 @@
 {
   "name": "apache-arrow",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "types": "./types/Arrow.d.ts",
   "typings": "./types/Arrow.d.ts",
   "main": "./targets/es5/cjs/Arrow.js",

From 682e248a7fd78d9e6ad8ec41dd4f518951d77152 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Nov 2017 11:45:54 -0500
Subject: [PATCH 1290/1644] ARROW-1850: [C++] Use void* / const void* for
 buffers in file APIs

This makes for a more convenient / less rigid API without as need for as many usages of `reinterpret_cast<const uint8_t*>`. This does not impact downstream projects (e.g. parquet-cpp is unaffected) unless they provide implementations of these virtual interfaces.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1363 from wesm/ARROW-1850 and squashes the following commits:

af5a3488 [Wes McKinney] Update glib, arrow-gpu for API changes
5d5cf2d0 [Wes McKinney] Use void* / const void* for buffers in file APIs
---
 c_glib/arrow-glib/input-stream.cpp         |  4 ++--
 c_glib/arrow-glib/output-stream.cpp        |  4 ++--
 c_glib/arrow-glib/writeable.cpp            |  2 +-
 c_glib/arrow-glib/writeable.hpp            |  6 ++---
 cpp/src/arrow/gpu/CMakeLists.txt           |  2 +-
 cpp/src/arrow/gpu/cuda_context.cc          | 23 +++++++++----------
 cpp/src/arrow/gpu/cuda_context.h           | 10 ++++-----
 cpp/src/arrow/gpu/cuda_memory.cc           | 14 ++++++------
 cpp/src/arrow/gpu/cuda_memory.h            | 10 ++++-----
 cpp/src/arrow/io/file.cc                   | 26 +++++++++++-----------
 cpp/src/arrow/io/file.h                    | 16 ++++++-------
 cpp/src/arrow/io/hdfs.cc                   | 17 +++++++-------
 cpp/src/arrow/io/hdfs.h                    |  8 +++----
 cpp/src/arrow/io/interfaces.cc             |  9 ++++----
 cpp/src/arrow/io/interfaces.h              | 14 ++++++------
 cpp/src/arrow/io/io-file-test.cc           | 11 +++++----
 cpp/src/arrow/io/io-memory-test.cc         |  2 +-
 cpp/src/arrow/io/memory.cc                 | 19 ++++++++--------
 cpp/src/arrow/io/memory.h                  | 12 +++++-----
 cpp/src/arrow/ipc/feather.cc               |  6 ++---
 cpp/src/arrow/ipc/json-integration-test.cc |  6 ++---
 cpp/src/arrow/ipc/metadata-internal.cc     |  3 +--
 cpp/src/arrow/ipc/writer.cc                | 13 +++++------
 cpp/src/arrow/python/io.cc                 |  8 +++----
 cpp/src/arrow/python/io.h                  |  6 ++---
 cpp/src/arrow/util/io-util.h               |  4 ++--
 26 files changed, 122 insertions(+), 133 deletions(-)

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index a7a894b9d4f3e..94422241b9cfa 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -420,7 +420,7 @@ namespace garrow {
 
     arrow::Status Read(int64_t n_bytes,
                        int64_t *n_read_bytes,
-                       uint8_t *out) override {
+                       void *out) override {
       GError *error = NULL;
       *n_read_bytes = g_input_stream_read(input_stream_,
                                           out,
@@ -437,7 +437,7 @@ namespace garrow {
     }
 
     arrow::Status ReadAt(int64_t position, int64_t n_bytes,
-			 int64_t *n_read_bytes, uint8_t* out) override {
+			 int64_t *n_read_bytes, void* out) override {
 	return arrow::io::RandomAccessFile::ReadAt(
 	    position, n_bytes, n_read_bytes, out);
     }
diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index 739992fb62b0e..9939f4f086f8e 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -76,7 +76,7 @@ garrow_output_stream_file_interface_init(GArrowFileInterface *iface)
   iface->get_raw = garrow_output_stream_get_raw_file_interface;
 }
 
-static std::shared_ptr<arrow::io::Writeable>
+static std::shared_ptr<arrow::io::Writable>
 garrow_output_stream_get_raw_writeable_interface(GArrowWriteable *writeable)
 {
   auto output_stream = GARROW_OUTPUT_STREAM(writeable);
@@ -325,7 +325,7 @@ namespace garrow {
       return arrow::Status::OK();
     }
 
-    arrow::Status Write(const uint8_t *data,
+    arrow::Status Write(const void *data,
                         int64_t n_bytes) override {
       GError *error = NULL;
       gsize n_written_bytes;
diff --git a/c_glib/arrow-glib/writeable.cpp b/c_glib/arrow-glib/writeable.cpp
index eb6adfee8c985..a16e43ab17ae9 100644
--- a/c_glib/arrow-glib/writeable.cpp
+++ b/c_glib/arrow-glib/writeable.cpp
@@ -88,7 +88,7 @@ garrow_writeable_flush(GArrowWriteable *writeable,
 
 G_END_DECLS
 
-std::shared_ptr<arrow::io::Writeable>
+std::shared_ptr<arrow::io::Writable>
 garrow_writeable_get_raw(GArrowWriteable *writeable)
 {
   auto *iface = GARROW_WRITEABLE_GET_IFACE(writeable);
diff --git a/c_glib/arrow-glib/writeable.hpp b/c_glib/arrow-glib/writeable.hpp
index 2b398f8b507c1..806d36fc07957 100644
--- a/c_glib/arrow-glib/writeable.hpp
+++ b/c_glib/arrow-glib/writeable.hpp
@@ -26,13 +26,13 @@
 /**
  * GArrowWriteableInterface:
  *
- * It wraps `arrow::io::Writeable`.
+ * It wraps `arrow::io::Writable`.
  */
 struct _GArrowWriteableInterface
 {
   GTypeInterface parent_iface;
 
-  std::shared_ptr<arrow::io::Writeable> (*get_raw)(GArrowWriteable *file);
+  std::shared_ptr<arrow::io::Writable> (*get_raw)(GArrowWriteable *file);
 };
 
-std::shared_ptr<arrow::io::Writeable> garrow_writeable_get_raw(GArrowWriteable *writeable);
+std::shared_ptr<arrow::io::Writable> garrow_writeable_get_raw(GArrowWriteable *writeable);
diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt
index 3f3069b91974b..3ddf2c7974f0e 100644
--- a/cpp/src/arrow/gpu/CMakeLists.txt
+++ b/cpp/src/arrow/gpu/CMakeLists.txt
@@ -54,7 +54,7 @@ configure_file(cuda_version.h.in
   @ONLY)
 
 install(FILES
-  "${CMAKE_CURRENT_SOURCE_DIR}/cuda_version.h"
+  "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h"
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu")
 
 install(FILES
diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc
index fff8ece6c1cff..2f5ccb0a95063 100644
--- a/cpp/src/arrow/gpu/cuda_context.cc
+++ b/cpp/src/arrow/gpu/cuda_context.cc
@@ -69,28 +69,27 @@ class CudaContext::CudaContextImpl {
     return Status::OK();
   }
 
-  Status CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+  Status CopyHostToDevice(void* dst, const void* src, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
-    CU_RETURN_NOT_OK(cuMemcpyHtoD(reinterpret_cast<CUdeviceptr>(dst),
-                                  reinterpret_cast<const void*>(src),
+    CU_RETURN_NOT_OK(cuMemcpyHtoD(reinterpret_cast<CUdeviceptr>(dst), src,
                                   static_cast<size_t>(nbytes)));
     return Status::OK();
   }
 
-  Status CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+  Status CopyDeviceToHost(void* dst, const void* src, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
     CU_RETURN_NOT_OK(cuMemcpyDtoH(dst, reinterpret_cast<const CUdeviceptr>(src),
                                   static_cast<size_t>(nbytes)));
     return Status::OK();
   }
 
-  Status Free(uint8_t* device_ptr, int64_t nbytes) {
+  Status Free(void* device_ptr, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuMemFree(reinterpret_cast<CUdeviceptr>(device_ptr)));
     bytes_allocated_ -= nbytes;
     return Status::OK();
   }
 
-  Status ExportIpcBuffer(uint8_t* data, std::unique_ptr<CudaIpcMemHandle>* handle) {
+  Status ExportIpcBuffer(void* data, std::unique_ptr<CudaIpcMemHandle>* handle) {
     CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
     CUipcMemHandle cu_handle;
     CU_RETURN_NOT_OK(cuIpcGetMemHandle(&cu_handle, reinterpret_cast<CUdeviceptr>(data)));
@@ -145,7 +144,7 @@ class CudaDeviceManager::CudaDeviceManagerImpl {
     return Status::OK();
   }
 
-  Status FreeHost(uint8_t* data, int64_t nbytes) {
+  Status FreeHost(void* data, int64_t nbytes) {
     CU_RETURN_NOT_OK(cuMemFreeHost(data));
     host_bytes_allocated_ -= nbytes;
     return Status::OK();
@@ -221,7 +220,7 @@ Status CudaDeviceManager::AllocateHost(int64_t nbytes,
   return Status::OK();
 }
 
-Status CudaDeviceManager::FreeHost(uint8_t* data, int64_t nbytes) {
+Status CudaDeviceManager::FreeHost(void* data, int64_t nbytes) {
   return impl_->FreeHost(data, nbytes);
 }
 
@@ -241,22 +240,22 @@ Status CudaContext::Allocate(int64_t nbytes, std::shared_ptr<CudaBuffer>* out) {
   return Status::OK();
 }
 
-Status CudaContext::ExportIpcBuffer(uint8_t* data,
+Status CudaContext::ExportIpcBuffer(void* data,
                                     std::unique_ptr<CudaIpcMemHandle>* handle) {
   return impl_->ExportIpcBuffer(data, handle);
 }
 
-Status CudaContext::CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+Status CudaContext::CopyHostToDevice(void* dst, const void* src, int64_t nbytes) {
   return impl_->CopyHostToDevice(dst, src, nbytes);
 }
 
-Status CudaContext::CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes) {
+Status CudaContext::CopyDeviceToHost(void* dst, const void* src, int64_t nbytes) {
   return impl_->CopyDeviceToHost(dst, src, nbytes);
 }
 
 Status CudaContext::Close() { return impl_->Close(); }
 
-Status CudaContext::Free(uint8_t* device_ptr, int64_t nbytes) {
+Status CudaContext::Free(void* device_ptr, int64_t nbytes) {
   return impl_->Free(device_ptr, nbytes);
 }
 
diff --git a/cpp/src/arrow/gpu/cuda_context.h b/cpp/src/arrow/gpu/cuda_context.h
index 6471059612349..6fc2e0d08abc9 100644
--- a/cpp/src/arrow/gpu/cuda_context.h
+++ b/cpp/src/arrow/gpu/cuda_context.h
@@ -46,7 +46,7 @@ class ARROW_EXPORT CudaDeviceManager {
 
   Status AllocateHost(int64_t nbytes, std::shared_ptr<CudaHostBuffer>* buffer);
 
-  Status FreeHost(uint8_t* data, int64_t nbytes);
+  Status FreeHost(void* data, int64_t nbytes);
 
   int num_devices() const;
 
@@ -88,10 +88,10 @@ class ARROW_EXPORT CudaContext : public std::enable_shared_from_this<CudaContext
  private:
   CudaContext();
 
-  Status ExportIpcBuffer(uint8_t* data, std::unique_ptr<CudaIpcMemHandle>* handle);
-  Status CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes);
-  Status CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes);
-  Status Free(uint8_t* device_ptr, int64_t nbytes);
+  Status ExportIpcBuffer(void* data, std::unique_ptr<CudaIpcMemHandle>* handle);
+  Status CopyHostToDevice(void* dst, const void* src, int64_t nbytes);
+  Status CopyDeviceToHost(void* dst, const void* src, int64_t nbytes);
+  Status Free(void* device_ptr, int64_t nbytes);
 
   class CudaContextImpl;
   std::unique_ptr<CudaContextImpl> impl_;
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
index 949c1d7a4dd41..cbf044121a644 100644
--- a/cpp/src/arrow/gpu/cuda_memory.cc
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -101,11 +101,11 @@ CudaBuffer::CudaBuffer(const std::shared_ptr<CudaBuffer>& parent, const int64_t
       is_ipc_(false) {}
 
 Status CudaBuffer::CopyToHost(const int64_t position, const int64_t nbytes,
-                              uint8_t* out) const {
+                              void* out) const {
   return context_->CopyDeviceToHost(out, data_ + position, nbytes);
 }
 
-Status CudaBuffer::CopyFromHost(const int64_t position, const uint8_t* data,
+Status CudaBuffer::CopyFromHost(const int64_t position, const void* data,
                                 int64_t nbytes) {
   DCHECK_LE(nbytes, size_ - position) << "Copy would overflow buffer";
   return context_->CopyHostToDevice(mutable_data_ + position, data, nbytes);
@@ -134,7 +134,7 @@ CudaBufferReader::CudaBufferReader(const std::shared_ptr<CudaBuffer>& buffer)
 
 CudaBufferReader::~CudaBufferReader() {}
 
-Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
   nbytes = std::min(nbytes, size_ - position_);
   *bytes_read = nbytes;
   RETURN_NOT_OK(context_->CopyDeviceToHost(buffer, data_ + position_, nbytes));
@@ -190,7 +190,7 @@ class CudaBufferWriter::CudaBufferWriterImpl {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const void* data, int64_t nbytes) {
     if (nbytes == 0) {
       return Status::OK();
     }
@@ -214,7 +214,7 @@ class CudaBufferWriter::CudaBufferWriterImpl {
     return Status::OK();
   }
 
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) {
     std::lock_guard<std::mutex> guard(lock_);
     RETURN_NOT_OK(Seek(position));
     return Write(data, nbytes);
@@ -269,11 +269,11 @@ Status CudaBufferWriter::Seek(int64_t position) {
 
 Status CudaBufferWriter::Tell(int64_t* position) const { return impl_->Tell(position); }
 
-Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
+Status CudaBufferWriter::Write(const void* data, int64_t nbytes) {
   return impl_->Write(data, nbytes);
 }
 
-Status CudaBufferWriter::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+Status CudaBufferWriter::WriteAt(int64_t position, const void* data, int64_t nbytes) {
   return impl_->WriteAt(position, data, nbytes);
 }
 
diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h
index 9ebd2ccf0c77e..9376b4b3ffcb9 100644
--- a/cpp/src/arrow/gpu/cuda_memory.h
+++ b/cpp/src/arrow/gpu/cuda_memory.h
@@ -49,14 +49,14 @@ class ARROW_EXPORT CudaBuffer : public Buffer {
   /// \brief Copy memory from GPU device to CPU host
   /// \param[out] out a pre-allocated output buffer
   /// \return Status
-  Status CopyToHost(const int64_t position, const int64_t nbytes, uint8_t* out) const;
+  Status CopyToHost(const int64_t position, const int64_t nbytes, void* out) const;
 
   /// \brief Copy memory to device at position
   /// \param[in] position start position to copy bytes
   /// \param[in] data the host data to copy
   /// \param[in] nbytes number of bytes to copy
   /// \return Status
-  Status CopyFromHost(const int64_t position, const uint8_t* data, int64_t nbytes);
+  Status CopyFromHost(const int64_t position, const void* data, int64_t nbytes);
 
   /// \brief Expose this device buffer as IPC memory which can be used in other processes
   /// \param[out] handle the exported IPC handle
@@ -130,7 +130,7 @@ class ARROW_EXPORT CudaBufferReader : public io::BufferReader {
   /// \param[in] nbytes number of bytes to read
   /// \param[out] bytes_read actual number of bytes read
   /// \param[out] buffer pre-allocated memory to write into
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
 
   /// \brief Zero-copy read from device memory
   /// \param[in] nbytes number of bytes to read
@@ -158,9 +158,9 @@ class ARROW_EXPORT CudaBufferWriter : public io::WriteableFile {
 
   Status Seek(int64_t position) override;
 
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
 
   Status Tell(int64_t* position) const override;
 
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 1ec5e23e587d5..65a302c8beac2 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -394,11 +394,11 @@ class OSFile {
     return Status::OK();
   }
 
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
-    return FileRead(fd_, out, nbytes, bytes_read);
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* out) {
+    return FileRead(fd_, reinterpret_cast<uint8_t*>(out), nbytes, bytes_read);
   }
 
-  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) {
     std::lock_guard<std::mutex> guard(lock_);
     RETURN_NOT_OK(Seek(position));
     return Read(nbytes, bytes_read, out);
@@ -413,12 +413,12 @@ class OSFile {
 
   Status Tell(int64_t* pos) const { return FileTell(fd_, pos); }
 
-  Status Write(const uint8_t* data, int64_t length) {
+  Status Write(const void* data, int64_t length) {
     std::lock_guard<std::mutex> guard(lock_);
     if (length < 0) {
       return Status::IOError("Length must be non-negative");
     }
-    return FileWrite(fd_, data, length);
+    return FileWrite(fd_, reinterpret_cast<const uint8_t*>(data), length);
   }
 
   int fd() const { return fd_; }
@@ -504,13 +504,13 @@ Status ReadableFile::Close() { return impl_->Close(); }
 
 Status ReadableFile::Tell(int64_t* pos) const { return impl_->Tell(pos); }
 
-Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
   std::lock_guard<std::mutex> guard(impl_->lock());
   return impl_->Read(nbytes, bytes_read, out);
 }
 
 Status ReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                            uint8_t* out) {
+                            void* out) {
   return impl_->ReadAt(position, nbytes, bytes_read, out);
 }
 
@@ -570,7 +570,7 @@ Status FileOutputStream::Close() { return impl_->Close(); }
 
 Status FileOutputStream::Tell(int64_t* pos) const { return impl_->Tell(pos); }
 
-Status FileOutputStream::Write(const uint8_t* data, int64_t length) {
+Status FileOutputStream::Write(const void* data, int64_t length) {
   return impl_->Write(data, length);
 }
 
@@ -710,7 +710,7 @@ Status MemoryMappedFile::Close() {
   return Status::OK();
 }
 
-Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
   nbytes = std::max<int64_t>(
       0, std::min(nbytes, memory_map_->size() - memory_map_->position()));
   if (nbytes > 0) {
@@ -735,7 +735,7 @@ Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 }
 
 Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                                uint8_t* out) {
+                                void* out) {
   std::lock_guard<std::mutex> guard(memory_map_->lock());
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
@@ -750,7 +750,7 @@ Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes,
 
 bool MemoryMappedFile::supports_zero_copy() const { return true; }
 
-Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+Status MemoryMappedFile::WriteAt(int64_t position, const void* data, int64_t nbytes) {
   std::lock_guard<std::mutex> guard(memory_map_->lock());
 
   if (!memory_map_->opened() || !memory_map_->writable()) {
@@ -761,7 +761,7 @@ Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t
   return WriteInternal(data, nbytes);
 }
 
-Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
+Status MemoryMappedFile::Write(const void* data, int64_t nbytes) {
   std::lock_guard<std::mutex> guard(memory_map_->lock());
 
   if (!memory_map_->opened() || !memory_map_->writable()) {
@@ -773,7 +773,7 @@ Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) {
   return WriteInternal(data, nbytes);
 }
 
-Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) {
+Status MemoryMappedFile::WriteInternal(const void* data, int64_t nbytes) {
   memcpy(memory_map_->head(), data, static_cast<size_t>(nbytes));
   memory_map_->advance(nbytes);
   return Status::OK();
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 7937fea74997d..265df4d65225d 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -59,7 +59,7 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
   Status Tell(int64_t* position) const override;
 
   // Write bytes to the stream. Thread-safe
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
   int file_descriptor() const;
 
@@ -93,12 +93,12 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile {
   Status Tell(int64_t* position) const override;
 
   // Read bytes from the file. Thread-safe
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   /// \brief Thread-safe implementation of ReadAt
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* out) override;
+                void* out) override;
 
   /// \brief Thread-safe implementation of ReadAt
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
@@ -141,13 +141,13 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   Status Seek(int64_t position) override;
 
   // Required by RandomAccessFile, copies memory into out. Not thread-safe
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override;
 
   // Zero copy read. Not thread-safe
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* out) override;
+                void* out) override;
 
   /// Default implementation is thread-safe
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
@@ -155,10 +155,10 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
   bool supports_zero_copy() const override;
 
   /// Write data at the current position in the file. Thread-safe
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
   /// Write data at a particular position in the file. Thread-safe
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
 
   // @return: the size in bytes of the memory source
   Status GetSize(int64_t* size) override;
@@ -168,7 +168,7 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
  private:
   MemoryMappedFile();
 
-  Status WriteInternal(const uint8_t* data, int64_t nbytes);
+  Status WriteInternal(const void* data, int64_t nbytes);
 
   class ARROW_NO_EXPORT MemoryMap;
   std::shared_ptr<MemoryMap> memory_map_;
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 77d1f524aae85..6e3e4a7a1c7e7 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -119,7 +119,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, void* buffer) {
     tSize ret;
     if (driver_->HasPread()) {
       ret = driver_->Pread(fs_, file_, static_cast<tOffset>(position),
@@ -149,11 +149,11 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
     int64_t total_bytes = 0;
     while (total_bytes < nbytes) {
       tSize ret = driver_->Read(
-          fs_, file_, reinterpret_cast<void*>(buffer + total_bytes),
+          fs_, file_, reinterpret_cast<uint8_t*>(buffer) + total_bytes,
           static_cast<tSize>(std::min<int64_t>(buffer_size_, nbytes - total_bytes)));
       RETURN_NOT_OK(CheckReadResult(ret));
       total_bytes += ret;
@@ -212,7 +212,7 @@ HdfsReadableFile::~HdfsReadableFile() { DCHECK(impl_->Close().ok()); }
 Status HdfsReadableFile::Close() { return impl_->Close(); }
 
 Status HdfsReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                                uint8_t* buffer) {
+                                void* buffer) {
   return impl_->ReadAt(position, nbytes, bytes_read, buffer);
 }
 
@@ -223,7 +223,7 @@ Status HdfsReadableFile::ReadAt(int64_t position, int64_t nbytes,
 
 bool HdfsReadableFile::supports_zero_copy() const { return false; }
 
-Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
   return impl_->Read(nbytes, bytes_read, buffer);
 }
 
@@ -261,7 +261,7 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
+  Status Write(const void* buffer, int64_t nbytes, int64_t* bytes_written) {
     std::lock_guard<std::mutex> guard(lock_);
     tSize ret = driver_->Write(fs_, file_, reinterpret_cast<const void*>(buffer),
                                static_cast<tSize>(nbytes));
@@ -277,12 +277,11 @@ HdfsOutputStream::~HdfsOutputStream() { DCHECK(impl_->Close().ok()); }
 
 Status HdfsOutputStream::Close() { return impl_->Close(); }
 
-Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes,
-                               int64_t* bytes_read) {
+Status HdfsOutputStream::Write(const void* buffer, int64_t nbytes, int64_t* bytes_read) {
   return impl_->Write(buffer, nbytes, bytes_read);
 }
 
-Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes) {
+Status HdfsOutputStream::Write(const void* buffer, int64_t nbytes) {
   int64_t bytes_written_dummy = 0;
   return Write(buffer, nbytes, &bytes_written_dummy);
 }
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 0708b11cca550..062473b20104d 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -182,12 +182,12 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
 
   // NOTE: If you wish to read a particular range of a file in a multithreaded
   // context, you may prefer to use ReadAt to avoid locking issues
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
 
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* buffer) override;
+                void* buffer) override;
 
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
@@ -217,9 +217,9 @@ class ARROW_EXPORT HdfsOutputStream : public OutputStream {
 
   Status Close() override;
 
-  Status Write(const uint8_t* buffer, int64_t nbytes) override;
+  Status Write(const void* buffer, int64_t nbytes) override;
 
-  Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written);
+  Status Write(const void* buffer, int64_t nbytes, int64_t* bytes_written);
 
   Status Flush() override;
 
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 582cc2026d86d..04560209a62c0 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -38,7 +38,7 @@ RandomAccessFile::RandomAccessFile()
     : impl_(new RandomAccessFile::RandomAccessFileImpl()) {}
 
 Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                                uint8_t* out) {
+                                void* out) {
   std::lock_guard<std::mutex> lock(impl_->lock_);
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
@@ -51,12 +51,11 @@ Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes,
   return Read(nbytes, out);
 }
 
-Status Writeable::Write(const std::string& data) {
-  return Write(reinterpret_cast<const uint8_t*>(data.c_str()),
-               static_cast<int64_t>(data.size()));
+Status Writable::Write(const std::string& data) {
+  return Write(data.c_str(), static_cast<int64_t>(data.size()));
 }
 
-Status Writeable::Flush() { return Status::OK(); }
+Status Writable::Flush() { return Status::OK(); }
 
 }  // namespace io
 }  // namespace arrow
diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
index 82af875e7c07e..09536a44ef003 100644
--- a/cpp/src/arrow/io/interfaces.h
+++ b/cpp/src/arrow/io/interfaces.h
@@ -86,11 +86,11 @@ class ARROW_EXPORT Seekable {
   virtual Status Seek(int64_t position) = 0;
 };
 
-class ARROW_EXPORT Writeable {
+class ARROW_EXPORT Writable {
  public:
-  virtual ~Writeable() = default;
+  virtual ~Writable() = default;
 
-  virtual Status Write(const uint8_t* data, int64_t nbytes) = 0;
+  virtual Status Write(const void* data, int64_t nbytes) = 0;
 
   /// \brief Flush buffered bytes, if any
   virtual Status Flush();
@@ -102,13 +102,13 @@ class ARROW_EXPORT Readable {
  public:
   virtual ~Readable() = default;
 
-  virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0;
+  virtual Status Read(int64_t nbytes, int64_t* bytes_read, void* out) = 0;
 
   // Does not copy if not necessary
   virtual Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) = 0;
 };
 
-class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writeable {
+class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writable {
  protected:
   OutputStream() = default;
 };
@@ -138,7 +138,7 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
   /// \param[out] out The buffer to read bytes into
   /// \return Status
   virtual Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                        uint8_t* out) = 0;
+                        void* out) = 0;
 
   /// \brief Read nbytes at position, provide default implementations using Read(...), but
   /// can be overridden. Default implementation is thread-safe.
@@ -162,7 +162,7 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
 
 class ARROW_EXPORT WriteableFile : public OutputStream, public Seekable {
  public:
-  virtual Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) = 0;
+  virtual Status WriteAt(int64_t position, const void* data, int64_t nbytes) = 0;
 
  protected:
   WriteableFile() = default;
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index ee3beabd9a80e..e70431e69580a 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -135,7 +135,7 @@ TEST_F(TestFileOutputStream, Close) {
   OpenFile();
 
   const char* data = "testdata";
-  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), strlen(data)));
+  ASSERT_OK(file_->Write(data, strlen(data)));
 
   int fd = file_->file_descriptor();
   ASSERT_OK(file_->Close());
@@ -158,7 +158,7 @@ TEST_F(TestFileOutputStream, InvalidWrites) {
 
   const char* data = "";
 
-  ASSERT_RAISES(IOError, file_->Write(reinterpret_cast<const uint8_t*>(data), -1));
+  ASSERT_RAISES(IOError, file_->Write(data, -1));
 }
 
 TEST_F(TestFileOutputStream, Tell) {
@@ -170,7 +170,7 @@ TEST_F(TestFileOutputStream, Tell) {
   ASSERT_EQ(0, position);
 
   const char* data = "testdata";
-  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), 8));
+  ASSERT_OK(file_->Write(data, 8));
   ASSERT_OK(file_->Tell(&position));
   ASSERT_EQ(8, position);
 }
@@ -179,7 +179,7 @@ TEST_F(TestFileOutputStream, TruncatesNewFile) {
   ASSERT_OK(FileOutputStream::Open(path_, &file_));
 
   const char* data = "testdata";
-  ASSERT_OK(file_->Write(reinterpret_cast<const uint8_t*>(data), strlen(data)));
+  ASSERT_OK(file_->Write(data, strlen(data)));
   ASSERT_OK(file_->Close());
 
   ASSERT_OK(FileOutputStream::Open(path_, &file_));
@@ -583,8 +583,7 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) {
 
   std::shared_ptr<MemoryMappedFile> file;
   ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &file));
-  ASSERT_OK(file->Write(reinterpret_cast<const uint8_t*>(data.c_str()),
-                        static_cast<int64_t>(data.size())));
+  ASSERT_OK(file->Write(data.c_str(), static_cast<int64_t>(data.size())));
 
   std::atomic<int> correct_count(0);
   constexpr int niter = 10000;
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index 117972f1cf06e..8c2e8c3b0beba 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -93,7 +93,7 @@ TEST(TestFixedSizeBufferWriter, Basics) {
 
   std::string data = "data123456";
   auto nbytes = static_cast<int64_t>(data.size());
-  ASSERT_OK(writer.Write(reinterpret_cast<const uint8_t*>(data.c_str()), nbytes));
+  ASSERT_OK(writer.Write(data.c_str(), nbytes));
 
   ASSERT_OK(writer.Tell(&position));
   ASSERT_EQ(nbytes, position);
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index d9c84b495d21a..74ec85c0e58ca 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -79,7 +79,7 @@ Status BufferOutputStream::Tell(int64_t* position) const {
   return Status::OK();
 }
 
-Status BufferOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+Status BufferOutputStream::Write(const void* data, int64_t nbytes) {
   if (ARROW_PREDICT_FALSE(!is_open_)) {
     return Status::IOError("OutputStream is closed");
   }
@@ -116,7 +116,7 @@ Status MockOutputStream::Tell(int64_t* position) const {
   return Status::OK();
 }
 
-Status MockOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+Status MockOutputStream::Write(const void* data, int64_t nbytes) {
   extent_bytes_written_ += nbytes;
   return Status::OK();
 }
@@ -162,9 +162,10 @@ class FixedSizeBufferWriter::FixedSizeBufferWriterImpl {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const void* data, int64_t nbytes) {
     if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) {
-      internal::parallel_memcopy(mutable_data_ + position_, data, nbytes,
+      internal::parallel_memcopy(mutable_data_ + position_,
+                                 reinterpret_cast<const uint8_t*>(data), nbytes,
                                  memcopy_blocksize_, memcopy_num_threads_);
     } else {
       memcpy(mutable_data_ + position_, data, nbytes);
@@ -173,7 +174,7 @@ class FixedSizeBufferWriter::FixedSizeBufferWriterImpl {
     return Status::OK();
   }
 
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) {
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) {
     std::lock_guard<std::mutex> guard(lock_);
     RETURN_NOT_OK(Seek(position));
     return Write(data, nbytes);
@@ -210,11 +211,11 @@ Status FixedSizeBufferWriter::Tell(int64_t* position) const {
   return impl_->Tell(position);
 }
 
-Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) {
+Status FixedSizeBufferWriter::Write(const void* data, int64_t nbytes) {
   return impl_->Write(data, nbytes);
 }
 
-Status FixedSizeBufferWriter::WriteAt(int64_t position, const uint8_t* data,
+Status FixedSizeBufferWriter::WriteAt(int64_t position, const void* data,
                                       int64_t nbytes) {
   return impl_->WriteAt(position, data, nbytes);
 }
@@ -252,7 +253,7 @@ Status BufferReader::Tell(int64_t* position) const {
 
 bool BufferReader::supports_zero_copy() const { return true; }
 
-Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
+Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
   memcpy(buffer, data_ + position_, nbytes);
   *bytes_read = std::min(nbytes, size_ - position_);
   position_ += *bytes_read;
@@ -273,7 +274,7 @@ Status BufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 }
 
 Status BufferReader::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                            uint8_t* out) {
+                            void* out) {
   return RandomAccessFile::ReadAt(position, nbytes, bytes_read, out);
 }
 
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 3aec91f7237e1..8dfd582f9ee05 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -48,7 +48,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream {
   // Implement the OutputStream interface
   Status Close() override;
   Status Tell(int64_t* position) const override;
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
   /// Close the stream and return the buffer
   Status Finish(std::shared_ptr<Buffer>* result);
@@ -72,7 +72,7 @@ class ARROW_EXPORT MockOutputStream : public OutputStream {
   // Implement the OutputStream interface
   Status Close() override;
   Status Tell(int64_t* position) const override;
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
   int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
 
@@ -90,8 +90,8 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
   Status Close() override;
   Status Seek(int64_t position) override;
   Status Tell(int64_t* position) const override;
-  Status Write(const uint8_t* data, int64_t nbytes) override;
-  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
+  Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
 
   void set_memcopy_threads(int num_threads);
   void set_memcopy_blocksize(int64_t blocksize);
@@ -111,12 +111,12 @@ class ARROW_EXPORT BufferReader : public RandomAccessFile {
 
   Status Close() override;
   Status Tell(int64_t* position) const override;
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
 
   // Zero copy read
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* out) override;
+                void* out) override;
 
   /// Default implementation is thread-safe
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index d339449c084d2..d3872503edf19 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -523,10 +523,8 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
     uint32_t buffer_size = static_cast<uint32_t>(bytes_written);
 
     // Footer: metadata length, magic bytes
-    RETURN_NOT_OK(
-        stream_->Write(reinterpret_cast<const uint8_t*>(&buffer_size), sizeof(uint32_t)));
-    return stream_->Write(reinterpret_cast<const uint8_t*>(kFeatherMagicBytes),
-                          strlen(kFeatherMagicBytes));
+    RETURN_NOT_OK(stream_->Write(&buffer_size, sizeof(uint32_t)));
+    return stream_->Write(kFeatherMagicBytes, strlen(kFeatherMagicBytes));
   }
 
   Status LoadArrayMetadata(const Array& values, ArrayMetadata* meta) {
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index f487487dfda9f..f362d97015978 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -119,8 +119,7 @@ static Status ConvertArrowToJson(const std::string& arrow_path,
 
   std::string result;
   RETURN_NOT_OK(writer->Finish(&result));
-  return out_file->Write(reinterpret_cast<const uint8_t*>(result.c_str()),
-                         static_cast<int64_t>(result.size()));
+  return out_file->Write(result.c_str(), static_cast<int64_t>(result.size()));
 }
 
 static Status ValidateArrowVsJson(const std::string& arrow_path,
@@ -250,8 +249,7 @@ class TestJSONIntegration : public ::testing::Test {
     do {
       std::shared_ptr<io::FileOutputStream> out;
       RETURN_NOT_OK(io::FileOutputStream::Open(path, &out));
-      RETURN_NOT_OK(out->Write(reinterpret_cast<const uint8_t*>(data),
-                               static_cast<int64_t>(strlen(data))));
+      RETURN_NOT_OK(out->Write(data, static_cast<int64_t>(strlen(data))));
     } while (0);
     return Status::OK();
   }
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 87b4708bf6346..05202ea937217 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -945,8 +945,7 @@ Status WriteMessage(const Buffer& message, io::OutputStream* file,
 
   // Write the flatbuffer size prefix including padding
   int32_t flatbuffer_size = padded_message_length - 4;
-  RETURN_NOT_OK(
-      file->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
+  RETURN_NOT_OK(file->Write(&flatbuffer_size, sizeof(int32_t)));
 
   // Write the flatbuffer
   RETURN_NOT_OK(file->Write(message.data(), message.size()));
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index fb766a9a7599c..a36ed4159d108 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -693,7 +693,7 @@ class StreamBookKeeper {
   }
 
   // Write data and update position
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const void* data, int64_t nbytes) {
     RETURN_NOT_OK(sink_->Write(data, nbytes));
     position_ += nbytes;
     return Status::OK();
@@ -782,7 +782,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl : public StreamBookKe
 
     // Write 0 EOS message
     const int32_t kEos = 0;
-    return Write(reinterpret_cast<const uint8_t*>(&kEos), sizeof(int32_t));
+    return Write(&kEos, sizeof(int32_t));
   }
 
   Status CheckStarted() {
@@ -870,8 +870,7 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
 
   Status Start() override {
     // It is only necessary to align to 8-byte boundary at the start of the file
-    RETURN_NOT_OK(Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
-                        strlen(kArrowMagicBytes)));
+    RETURN_NOT_OK(Write(kArrowMagicBytes, strlen(kArrowMagicBytes)));
     RETURN_NOT_OK(Align());
 
     // We write the schema at the start of the file (and the end). This also
@@ -895,12 +894,10 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl
       return Status::Invalid("Invalid file footer");
     }
 
-    RETURN_NOT_OK(
-        Write(reinterpret_cast<const uint8_t*>(&footer_length), sizeof(int32_t)));
+    RETURN_NOT_OK(Write(&footer_length, sizeof(int32_t)));
 
     // Write magic bytes to end file
-    return Write(reinterpret_cast<const uint8_t*>(kArrowMagicBytes),
-                 strlen(kArrowMagicBytes));
+    return Write(kArrowMagicBytes, strlen(kArrowMagicBytes));
   }
 };
 
diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index b01358ab00b4b..cc3892928c455 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -76,7 +76,7 @@ class PythonFile {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* data, int64_t nbytes) {
+  Status Write(const void* data, int64_t nbytes) {
     PyObject* py_data =
         PyBytes_FromStringAndSize(reinterpret_cast<const char*>(data), nbytes);
     PY_RETURN_IF_ERROR(StatusCode::IOError);
@@ -130,7 +130,7 @@ Status PyReadableFile::Tell(int64_t* position) const {
   return file_->Tell(position);
 }
 
-Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) {
+Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
   PyAcquireGIL lock;
   PyObject* bytes_obj;
   ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
@@ -155,7 +155,7 @@ Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
 }
 
 Status PyReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                              uint8_t* out) {
+                              void* out) {
   std::lock_guard<std::mutex> guard(file_->lock());
   RETURN_NOT_OK(Seek(position));
   return Read(nbytes, bytes_read, out);
@@ -208,7 +208,7 @@ Status PyOutputStream::Tell(int64_t* position) const {
   return Status::OK();
 }
 
-Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) {
+Status PyOutputStream::Write(const void* data, int64_t nbytes) {
   PyAcquireGIL lock;
   position_ += nbytes;
   return file_->Write(data, nbytes);
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index bf5db5313a9db..f550de7b2848c 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -41,12 +41,12 @@ class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile {
 
   Status Close() override;
 
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override;
   Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
   // Thread-safe version
   Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
-                uint8_t* out) override;
+                void* out) override;
 
   // Thread-safe version
   Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
@@ -70,7 +70,7 @@ class ARROW_EXPORT PyOutputStream : public io::OutputStream {
 
   Status Close() override;
   Status Tell(int64_t* position) const override;
-  Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Write(const void* data, int64_t nbytes) override;
 
  private:
   std::unique_ptr<PythonFile> file_;
diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
index dbca0d8be394e..7e2a94ca82320 100644
--- a/cpp/src/arrow/util/io-util.h
+++ b/cpp/src/arrow/util/io-util.h
@@ -40,7 +40,7 @@ class StdoutStream : public OutputStream {
     return Status::OK();
   }
 
-  Status Write(const uint8_t* data, int64_t nbytes) override {
+  Status Write(const void* data, int64_t nbytes) override {
     pos_ += nbytes;
     std::cout.write(reinterpret_cast<const char*>(data), nbytes);
     return Status::OK();
@@ -63,7 +63,7 @@ class StdinStream : public InputStream {
     return Status::OK();
   }
 
-  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override {
+  Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override {
     std::cin.read(reinterpret_cast<char*>(out), nbytes);
     if (std::cin) {
       *bytes_read = nbytes;

From b19e183c152000193df227f1e931444759093af6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 27 Nov 2017 16:15:16 -0500
Subject: [PATCH 1291/1644] ARROW-1783: [Python] Provide a "component" dict
 representation of a serialized Python object with minimal allocation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For systems (like Dask) that prefer to handle their own framed buffer transport, this provides a list of memoryview-compatible objects with minimal copying / allocation from the input data structure, which can similarly be zero-copy reconstructed to the original object.

To motivate the use case, consider a dict of ndarrays:

```
data = {i: np.random.randn(1000, 1000) for i in range(50)}
```

Here, we have:

```
>>> %timeit serialized = pa.serialize(data)
52.7 µs ± 1.01 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)
```

This is about 400MB of data. Some systems may not want to double memory by assembling this into a single large buffer, like with the `to_buffer` method:

```
>>> written = serialized.to_buffer()
>>> written.size
400015456
```

We provide a `to_components` method which contains a dict with a `'data'` field containing a list of `pyarrow.Buffer` objects. This can be converted back to the original Python object using `pyarrow.deserialize_components`:

```
>>> %timeit components = serialized.to_components()
73.8 µs ± 812 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

>>> list(components.keys())
['num_buffers', 'data', 'num_tensors']

>>> len(components['data'])
101

>>> type(components['data'][0])
pyarrow.lib.Buffer
```

and

```
>>> %timeit recons = pa.deserialize_components(components)
93.6 µs ± 260 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)
```

The reason there are 101 data components (1 + 2 * 50) is that:

* 1 buffer for the serialized Union stream representing the object
* 2 buffers for each of the tensors: 1 for the metadata and 1 for the tensor body. The body is separate so that this is zero-copy from the input

Next step after this is ARROW-1784 which is to transport a pandas.DataFrame using this mechanism

cc @pitrou @jcrist @mrocklin

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1362 from wesm/ARROW-1783 and squashes the following commits:

4ec5a89f [Wes McKinney] Add missing decref on error
e8c76d42 [Wes McKinney] Acquire GIL in GetSerializedFromComponents
1d2e0e27 [Wes McKinney] Fix function documentation
fffc7bb6 [Wes McKinney] Typos, add deserialize_components to API
50d2fee5 [Wes McKinney] Finish componentwise serialization roundtrip
58174dde [Wes McKinney] More progress, stubs for reconstruction
b1e31a34 [Wes McKinney] Draft GetTensorMessage
337e1d29 [Wes McKinney] Draft SerializedPyObject::GetComponents
598ef335 [Wes McKinney] Tweak
---
 cpp/src/arrow/io/memory.cc                 |  3 +
 cpp/src/arrow/io/memory.h                  |  1 +
 cpp/src/arrow/ipc/message.cc               | 30 +++++++-
 cpp/src/arrow/ipc/message.h                | 27 ++-----
 cpp/src/arrow/ipc/reader.cc                | 13 ++--
 cpp/src/arrow/ipc/reader.h                 | 10 ++-
 cpp/src/arrow/ipc/writer.cc                | 64 ++++++++++++++---
 cpp/src/arrow/ipc/writer.h                 | 11 +++
 cpp/src/arrow/python/arrow_to_pandas.cc    |  2 +-
 cpp/src/arrow/python/arrow_to_python.cc    | 60 +++++++++++++++-
 cpp/src/arrow/python/arrow_to_python.h     | 13 ++++
 cpp/src/arrow/python/python_to_arrow.cc    | 82 +++++++++++++++++++---
 cpp/src/arrow/python/python_to_arrow.h     | 28 ++++++--
 python/doc/source/api.rst                  |  1 +
 python/pyarrow/__init__.py                 |  1 +
 python/pyarrow/includes/libarrow.pxd       | 17 +++--
 python/pyarrow/ipc.pxi                     |  4 +-
 python/pyarrow/serialization.pxi           | 62 +++++++++++++++-
 python/pyarrow/tests/test_serialization.py | 33 +++++++++
 19 files changed, 394 insertions(+), 68 deletions(-)

diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 74ec85c0e58ca..ecdf26f0a991b 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -241,6 +241,9 @@ BufferReader::BufferReader(const std::shared_ptr<Buffer>& buffer)
 BufferReader::BufferReader(const uint8_t* data, int64_t size)
     : buffer_(nullptr), data_(data), size_(size), position_(0) {}
 
+BufferReader::BufferReader(const Buffer& buffer)
+    : BufferReader(buffer.data(), buffer.size()) {}
+
 Status BufferReader::Close() {
   // no-op
   return Status::OK();
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 8dfd582f9ee05..cf370b3b6388b 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -107,6 +107,7 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
 class ARROW_EXPORT BufferReader : public RandomAccessFile {
  public:
   explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
+  explicit BufferReader(const Buffer& buffer);
   BufferReader(const uint8_t* data, int64_t size);
 
   Status Close() override;
diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc
index 21d6a69a28603..1835cefde09ee 100644
--- a/cpp/src/arrow/ipc/message.cc
+++ b/cpp/src/arrow/ipc/message.cc
@@ -236,11 +236,35 @@ Status ReadMessage(io::InputStream* file, std::unique_ptr<Message>* message) {
 // ----------------------------------------------------------------------
 // Implement InputStream message reader
 
-Status InputStreamMessageReader::ReadNextMessage(std::unique_ptr<Message>* message) {
-  return ReadMessage(stream_, message);
+/// \brief Implementation of MessageReader that reads from InputStream
+class InputStreamMessageReader : public MessageReader {
+ public:
+  explicit InputStreamMessageReader(io::InputStream* stream) : stream_(stream) {}
+
+  explicit InputStreamMessageReader(const std::shared_ptr<io::InputStream>& owned_stream)
+      : InputStreamMessageReader(owned_stream.get()) {
+    owned_stream_ = owned_stream;
+  }
+
+  ~InputStreamMessageReader() {}
+
+  Status ReadNextMessage(std::unique_ptr<Message>* message) {
+    return ReadMessage(stream_, message);
+  }
+
+ private:
+  io::InputStream* stream_;
+  std::shared_ptr<io::InputStream> owned_stream_;
+};
+
+std::unique_ptr<MessageReader> MessageReader::Open(io::InputStream* stream) {
+  return std::unique_ptr<MessageReader>(new InputStreamMessageReader(stream));
 }
 
-InputStreamMessageReader::~InputStreamMessageReader() {}
+std::unique_ptr<MessageReader> MessageReader::Open(
+    const std::shared_ptr<io::InputStream>& owned_stream) {
+  return std::unique_ptr<MessageReader>(new InputStreamMessageReader(owned_stream));
+}
 
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h
index 495474e505157..159b39a81f95d 100644
--- a/cpp/src/arrow/ipc/message.h
+++ b/cpp/src/arrow/ipc/message.h
@@ -144,6 +144,13 @@ class ARROW_EXPORT MessageReader {
  public:
   virtual ~MessageReader() = default;
 
+  /// \brief Create MessageReader that reads from InputStream
+  static std::unique_ptr<MessageReader> Open(io::InputStream* stream);
+
+  /// \brief Create MessageReader that reads from owned InputStream
+  static std::unique_ptr<MessageReader> Open(
+      const std::shared_ptr<io::InputStream>& owned_stream);
+
   /// \brief Read next Message from the interface
   ///
   /// \param[out] message an arrow::ipc::Message instance
@@ -151,26 +158,6 @@ class ARROW_EXPORT MessageReader {
   virtual Status ReadNextMessage(std::unique_ptr<Message>* message) = 0;
 };
 
-/// \brief Implementation of MessageReader that reads from InputStream
-/// \since 0.5.0
-class ARROW_EXPORT InputStreamMessageReader : public MessageReader {
- public:
-  explicit InputStreamMessageReader(io::InputStream* stream) : stream_(stream) {}
-
-  explicit InputStreamMessageReader(const std::shared_ptr<io::InputStream>& owned_stream)
-      : InputStreamMessageReader(owned_stream.get()) {
-    owned_stream_ = owned_stream;
-  }
-
-  ~InputStreamMessageReader();
-
-  Status ReadNextMessage(std::unique_ptr<Message>* message) override;
-
- private:
-  io::InputStream* stream_;
-  std::shared_ptr<io::InputStream> owned_stream_;
-};
-
 /// \brief Read encapulated RPC message from position in file
 ///
 /// Read a length-prefixed message flatbuffer starting at the indicated file
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 5960e81883d6b..ae0f8f39806b7 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -480,14 +480,12 @@ Status RecordBatchStreamReader::Open(std::unique_ptr<MessageReader> message_read
 
 Status RecordBatchStreamReader::Open(io::InputStream* stream,
                                      std::shared_ptr<RecordBatchReader>* out) {
-  std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
-  return Open(std::move(message_reader), out);
+  return Open(MessageReader::Open(stream), out);
 }
 
 Status RecordBatchStreamReader::Open(const std::shared_ptr<io::InputStream>& stream,
                                      std::shared_ptr<RecordBatchReader>* out) {
-  std::unique_ptr<MessageReader> message_reader(new InputStreamMessageReader(stream));
-  return Open(std::move(message_reader), out);
+  return Open(MessageReader::Open(stream), out);
 }
 
 std::shared_ptr<Schema> RecordBatchStreamReader::schema() const {
@@ -717,14 +715,17 @@ Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
 
   std::unique_ptr<Message> message;
   RETURN_NOT_OK(ReadContiguousPayload(file, &message));
+  return ReadTensor(*message, out);
+}
 
+Status ReadTensor(const Message& message, std::shared_ptr<Tensor>* out) {
   std::shared_ptr<DataType> type;
   std::vector<int64_t> shape;
   std::vector<int64_t> strides;
   std::vector<std::string> dim_names;
-  RETURN_NOT_OK(internal::GetTensorMetadata(*message->metadata(), &type, &shape, &strides,
+  RETURN_NOT_OK(internal::GetTensorMetadata(*message.metadata(), &type, &shape, &strides,
                                             &dim_names));
-  *out = std::make_shared<Tensor>(type, message->body(), shape, strides, dim_names);
+  *out = std::make_shared<Tensor>(type, message.body(), shape, strides, dim_names);
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 627f67e2517e9..019c9bc1f32d8 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -219,7 +219,7 @@ Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr<Schema>& sc
                        int max_recursion_depth, io::RandomAccessFile* file,
                        std::shared_ptr<RecordBatch>* out);
 
-/// EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
+/// \brief EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file
 ///
 /// \param[in] offset the file location of the start of the message
 /// \param[in] file the file where the batch is located
@@ -229,6 +229,14 @@ ARROW_EXPORT
 Status ReadTensor(int64_t offset, io::RandomAccessFile* file,
                   std::shared_ptr<Tensor>* out);
 
+/// \brief EXPERIMENTAL: Read arrow::Tensor from IPC message
+///
+/// \param[in] message a Message containing the tensor metadata and body
+/// \param[out] out the read tensor
+/// \return Status
+ARROW_EXPORT
+Status ReadTensor(const Message& message, std::shared_ptr<Tensor>* out);
+
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index a36ed4159d108..c6aa770127c88 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -560,9 +560,18 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
                           pool, kMaxNestingDepth, true);
 }
 
-static Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_size,
-                                     const Tensor& tensor, uint8_t* scratch_space,
-                                     io::OutputStream* dst) {
+namespace {
+
+Status WriteTensorHeader(const Tensor& tensor, io::OutputStream* dst,
+                         int32_t* metadata_length, int64_t* body_length) {
+  std::shared_ptr<Buffer> metadata;
+  RETURN_NOT_OK(internal::WriteTensorMessage(tensor, 0, &metadata));
+  return internal::WriteMessage(*metadata, dst, metadata_length);
+}
+
+Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_size,
+                              const Tensor& tensor, uint8_t* scratch_space,
+                              io::OutputStream* dst) {
   if (dim_index == tensor.ndim() - 1) {
     const uint8_t* data_ptr = tensor.raw_data() + offset;
     const int64_t stride = tensor.strides()[dim_index];
@@ -580,16 +589,37 @@ static Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_siz
   return Status::OK();
 }
 
-Status WriteTensorHeader(const Tensor& tensor, io::OutputStream* dst,
-                         int32_t* metadata_length, int64_t* body_length) {
-  RETURN_NOT_OK(AlignStreamPosition(dst));
-  std::shared_ptr<Buffer> metadata;
-  RETURN_NOT_OK(internal::WriteTensorMessage(tensor, 0, &metadata));
-  return internal::WriteMessage(*metadata, dst, metadata_length);
+Status GetContiguousTensor(const Tensor& tensor, MemoryPool* pool,
+                           std::unique_ptr<Tensor>* out) {
+  const auto& type = static_cast<const FixedWidthType&>(*tensor.type());
+  const int elem_size = type.bit_width() / 8;
+
+  // TODO(wesm): Do we care enough about this temporary allocation to pass in
+  // a MemoryPool to this function?
+  std::shared_ptr<Buffer> scratch_space;
+  RETURN_NOT_OK(AllocateBuffer(default_memory_pool(),
+                               tensor.shape()[tensor.ndim() - 1] * elem_size,
+                               &scratch_space));
+
+  std::shared_ptr<ResizableBuffer> contiguous_data;
+  RETURN_NOT_OK(
+      AllocateResizableBuffer(pool, tensor.size() * elem_size, &contiguous_data));
+
+  io::BufferOutputStream stream(contiguous_data);
+  RETURN_NOT_OK(WriteStridedTensorData(0, 0, elem_size, tensor,
+                                       scratch_space->mutable_data(), &stream));
+
+  out->reset(new Tensor(tensor.type(), contiguous_data, tensor.shape()));
+
+  return Status::OK();
 }
 
+}  // namespace
+
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
                    int64_t* body_length) {
+  RETURN_NOT_OK(AlignStreamPosition(dst));
+
   if (tensor.is_contiguous()) {
     RETURN_NOT_OK(WriteTensorHeader(tensor, dst, metadata_length, body_length));
     auto data = tensor.data();
@@ -619,6 +649,22 @@ Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadat
   }
 }
 
+Status GetTensorMessage(const Tensor& tensor, MemoryPool* pool,
+                        std::unique_ptr<Message>* out) {
+  const Tensor* tensor_to_write = &tensor;
+  std::unique_ptr<Tensor> temp_tensor;
+
+  if (!tensor.is_contiguous()) {
+    RETURN_NOT_OK(GetContiguousTensor(tensor, pool, &temp_tensor));
+    tensor_to_write = temp_tensor.get();
+  }
+
+  std::shared_ptr<Buffer> metadata;
+  RETURN_NOT_OK(internal::WriteTensorMessage(*tensor_to_write, 0, &metadata));
+  out->reset(new Message(metadata, tensor_to_write->data()));
+  return Status::OK();
+}
+
 Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
                        int64_t buffer_start_offset, io::OutputStream* dst,
                        int32_t* metadata_length, int64_t* body_length, MemoryPool* pool) {
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 457dcb4ec6a13..013783ee0a224 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -245,6 +245,17 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
 ARROW_EXPORT
 Status GetTensorSize(const Tensor& tensor, int64_t* size);
 
+/// \brief EXPERIMENTAL: Convert arrow::Tensor to a Message with minimal memory
+/// allocation
+///
+/// \param[in] tensor the Tensor to write
+/// \param[in] pool MemoryPool to allocate space for metadata
+/// \param[out] out the resulting Message
+/// \return Status
+ARROW_EXPORT
+Status GetTensorMessage(const Tensor& tensor, MemoryPool* pool,
+                        std::unique_ptr<Message>* out);
+
 /// \brief EXPERIMENTAL: Write arrow::Tensor as a contiguous message
 ///
 /// \param[in] tensor the Tensor to write
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 8814fc190ab3d..096bbd55c6d41 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -480,7 +480,7 @@ inline Status ConvertStruct(PandasOptions options, const ChunkedArray& data,
             Py_INCREF(Py_None);
             field_value.reset(Py_None);
           }
-          // PyDict_SetItemString does not steal the value reference
+          // PyDict_SetItemString increments reference count
           auto setitem_result =
               PyDict_SetItemString(dict_item.obj(), name.c_str(), field_value.obj());
           RETURN_IF_PYERROR();
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index 9686050b9676f..ce539a597be37 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -29,15 +29,17 @@
 
 #include "arrow/array.h"
 #include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
 #include "arrow/ipc/reader.h"
+#include "arrow/table.h"
+#include "arrow/util/logging.h"
+
 #include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
 #include "arrow/python/numpy_convert.h"
 #include "arrow/python/pyarrow.h"
 #include "arrow/python/python_to_arrow.h"
 #include "arrow/python/util/datetime.h"
-#include "arrow/table.h"
-#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace py {
@@ -286,5 +288,59 @@ Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObj
                          obj, out);
 }
 
+Status GetSerializedFromComponents(int num_tensors, int num_buffers, PyObject* data,
+                                   SerializedPyObject* out) {
+  PyAcquireGIL gil;
+  const Py_ssize_t data_length = PyList_Size(data);
+  RETURN_IF_PYERROR();
+
+  const Py_ssize_t expected_data_length = 1 + num_tensors * 2 + num_buffers;
+  if (data_length != expected_data_length) {
+    return Status::Invalid("Invalid number of buffers in data");
+  }
+
+  auto GetBuffer = [&data](Py_ssize_t index, std::shared_ptr<Buffer>* out) {
+    PyObject* py_buf = PyList_GET_ITEM(data, index);
+    return unwrap_buffer(py_buf, out);
+  };
+
+  Py_ssize_t buffer_index = 0;
+
+  // Read the union batch describing object structure
+  {
+    std::shared_ptr<Buffer> data_buffer;
+    RETURN_NOT_OK(GetBuffer(buffer_index++, &data_buffer));
+    gil.release();
+    io::BufferReader buf_reader(data_buffer);
+    std::shared_ptr<RecordBatchReader> reader;
+    RETURN_NOT_OK(ipc::RecordBatchStreamReader::Open(&buf_reader, &reader));
+    RETURN_NOT_OK(reader->ReadNext(&out->batch));
+    gil.acquire();
+  }
+
+  // Zero-copy reconstruct tensors
+  for (int i = 0; i < num_tensors; ++i) {
+    std::shared_ptr<Buffer> metadata;
+    std::shared_ptr<Buffer> body;
+    std::shared_ptr<Tensor> tensor;
+    RETURN_NOT_OK(GetBuffer(buffer_index++, &metadata));
+    RETURN_NOT_OK(GetBuffer(buffer_index++, &body));
+
+    ipc::Message message(metadata, body);
+
+    RETURN_NOT_OK(ReadTensor(message, &tensor));
+    out->tensors.emplace_back(std::move(tensor));
+  }
+
+  // Unwrap and append buffers
+  for (int i = 0; i < num_buffers; ++i) {
+    std::shared_ptr<Buffer> buffer;
+    RETURN_NOT_OK(GetBuffer(buffer_index++, &buffer));
+    out->buffers.emplace_back(std::move(buffer));
+  }
+
+  return Status::OK();
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/arrow_to_python.h b/cpp/src/arrow/python/arrow_to_python.h
index 7509f30eb4e90..9440ffb32abeb 100644
--- a/cpp/src/arrow/python/arrow_to_python.h
+++ b/cpp/src/arrow/python/arrow_to_python.h
@@ -48,6 +48,19 @@ namespace py {
 ARROW_EXPORT
 Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out);
 
+/// \brief Reconstruct SerializedPyObject from representation produced by
+/// SerializedPyObject::GetComponents.
+///
+/// \param[in] num_tensors
+/// \param[in] num_buffers
+/// \param[in] data a list containing pyarrow.Buffer instances. Must be 1 +
+/// num_tensors * 2 + num_buffers in length
+/// \param[out] out the reconstructed object
+/// \return Status
+ARROW_EXPORT
+Status GetSerializedFromComponents(int num_tensors, int num_buffers, PyObject* data,
+                                   SerializedPyObject* out);
+
 /// \brief Reconstruct Python object from Arrow-serialized representation
 /// \param[in] context Serialization context which contains custom serialization
 /// and deserialization callbacks. Can be any Python object with a
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 72cc5b6e1db67..253e9d9a7da30 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -31,7 +31,9 @@
 #include "arrow/array.h"
 #include "arrow/builder.h"
 #include "arrow/io/interfaces.h"
+#include "arrow/io/memory.h"
 #include "arrow/ipc/writer.h"
+#include "arrow/memory_pool.h"
 #include "arrow/record_batch.h"
 #include "arrow/tensor.h"
 #include "arrow/util/logging.h"
@@ -710,27 +712,89 @@ Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject
   return Status::OK();
 }
 
-Status WriteSerializedObject(const SerializedPyObject& obj, io::OutputStream* dst) {
-  int32_t num_tensors = static_cast<int32_t>(obj.tensors.size());
-  int32_t num_buffers = static_cast<int32_t>(obj.buffers.size());
-  RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&num_tensors), sizeof(int32_t)));
-  RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&num_buffers), sizeof(int32_t)));
-  RETURN_NOT_OK(ipc::WriteRecordBatchStream({obj.batch}, dst));
+Status SerializedPyObject::WriteTo(io::OutputStream* dst) {
+  int32_t num_tensors = static_cast<int32_t>(this->tensors.size());
+  int32_t num_buffers = static_cast<int32_t>(this->buffers.size());
+  RETURN_NOT_OK(
+      dst->Write(reinterpret_cast<const uint8_t*>(&num_tensors), sizeof(int32_t)));
+  RETURN_NOT_OK(
+      dst->Write(reinterpret_cast<const uint8_t*>(&num_buffers), sizeof(int32_t)));
+  RETURN_NOT_OK(ipc::WriteRecordBatchStream({this->batch}, dst));
 
   int32_t metadata_length;
   int64_t body_length;
-  for (const auto& tensor : obj.tensors) {
+  for (const auto& tensor : this->tensors) {
     RETURN_NOT_OK(ipc::WriteTensor(*tensor, dst, &metadata_length, &body_length));
   }
 
-  for (const auto& buffer : obj.buffers) {
+  for (const auto& buffer : this->buffers) {
     int64_t size = buffer->size();
-    RETURN_NOT_OK(dst->Write(reinterpret_cast<uint8_t*>(&size), sizeof(int64_t)));
+    RETURN_NOT_OK(dst->Write(reinterpret_cast<const uint8_t*>(&size), sizeof(int64_t)));
     RETURN_NOT_OK(dst->Write(buffer->data(), size));
   }
 
   return Status::OK();
 }
 
+Status SerializedPyObject::GetComponents(MemoryPool* memory_pool, PyObject** out) {
+  PyAcquireGIL py_gil;
+
+  ScopedRef result(PyDict_New());
+  PyObject* buffers = PyList_New(0);
+
+  // TODO(wesm): Not sure how pedantic we need to be about checking the return
+  // values of these functions. There are other places where we do not check
+  // PyDict_SetItem/SetItemString return value, but these failures would be
+  // quite esoteric
+  PyDict_SetItemString(result.get(), "num_tensors",
+                       PyLong_FromSize_t(this->tensors.size()));
+  PyDict_SetItemString(result.get(), "num_buffers",
+                       PyLong_FromSize_t(this->buffers.size()));
+  PyDict_SetItemString(result.get(), "data", buffers);
+  RETURN_IF_PYERROR();
+
+  Py_DECREF(buffers);
+
+  auto PushBuffer = [&buffers](const std::shared_ptr<Buffer>& buffer) {
+    PyObject* wrapped_buffer = wrap_buffer(buffer);
+    RETURN_IF_PYERROR();
+    if (PyList_Append(buffers, wrapped_buffer) < 0) {
+      Py_DECREF(wrapped_buffer);
+      RETURN_IF_PYERROR();
+    }
+    Py_DECREF(wrapped_buffer);
+    return Status::OK();
+  };
+
+  constexpr int64_t kInitialCapacity = 1024;
+
+  // Write the record batch describing the object structure
+  std::shared_ptr<io::BufferOutputStream> stream;
+  std::shared_ptr<Buffer> buffer;
+
+  py_gil.release();
+  RETURN_NOT_OK(io::BufferOutputStream::Create(kInitialCapacity, memory_pool, &stream));
+  RETURN_NOT_OK(ipc::WriteRecordBatchStream({this->batch}, stream.get()));
+  RETURN_NOT_OK(stream->Finish(&buffer));
+  py_gil.acquire();
+
+  RETURN_NOT_OK(PushBuffer(buffer));
+
+  // For each tensor, get a metadata buffer and a buffer for the body
+  for (const auto& tensor : this->tensors) {
+    std::unique_ptr<ipc::Message> message;
+    RETURN_NOT_OK(ipc::GetTensorMessage(*tensor, memory_pool, &message));
+    RETURN_NOT_OK(PushBuffer(message->metadata()));
+    RETURN_NOT_OK(PushBuffer(message->body()));
+  }
+
+  for (const auto& buf : this->buffers) {
+    RETURN_NOT_OK(PushBuffer(buf));
+  }
+
+  *out = result.release();
+  return Status::OK();
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/python_to_arrow.h b/cpp/src/arrow/python/python_to_arrow.h
index c5b6396145b7f..ce7aefa0e2409 100644
--- a/cpp/src/arrow/python/python_to_arrow.h
+++ b/cpp/src/arrow/python/python_to_arrow.h
@@ -30,6 +30,7 @@
 
 namespace arrow {
 
+class MemoryPool;
 class RecordBatch;
 class Tensor;
 
@@ -45,6 +46,26 @@ struct ARROW_EXPORT SerializedPyObject {
   std::shared_ptr<RecordBatch> batch;
   std::vector<std::shared_ptr<Tensor>> tensors;
   std::vector<std::shared_ptr<Buffer>> buffers;
+
+  /// \brief Write serialized Python object to OutputStream
+  /// \param[in,out] dst an OutputStream
+  /// \return Status
+  Status WriteTo(io::OutputStream* dst);
+
+  /// \brief Convert SerializedPyObject to a dict containing the message
+  /// components as Buffer instances with minimal memory allocation
+  ///
+  /// {
+  ///   'num_tensors': N,
+  ///   'num_buffers': K,
+  ///   'data': [Buffer]
+  /// }
+  ///
+  /// Each tensor is written as two buffers, one for the metadata and one for
+  /// the body. Therefore, the number of buffers in 'data' is 2 * N + K + 1,
+  /// with the first buffer containing the serialized record batch containing
+  /// the UnionArray that describes the whole object
+  Status GetComponents(MemoryPool* pool, PyObject** out);
 };
 
 /// \brief Serialize Python sequence as a RecordBatch plus
@@ -62,13 +83,6 @@ struct ARROW_EXPORT SerializedPyObject {
 ARROW_EXPORT
 Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out);
 
-/// \brief Write serialized Python object to OutputStream
-/// \param[in] object a serialized Python object to write out
-/// \param[out] dst an OutputStream
-/// \return Status
-ARROW_EXPORT
-Status WriteSerializedObject(const SerializedPyObject& object, io::OutputStream* dst);
-
 }  // namespace py
 }  // namespace arrow
 
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index bb2a0420b2a08..636f41d67bc6a 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -245,6 +245,7 @@ Serialization and IPC
    serialize
    serialize_to
    deserialize
+   deserialize_components
    deserialize_from
    read_serialized
    SerializedPyObject
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 0456a658f1425..bd31b21c196de 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -101,6 +101,7 @@
 
 # Serialization
 from pyarrow.lib import (deserialize_from, deserialize,
+                         deserialize_components,
                          serialize, serialize_to, read_serialized,
                          SerializedPyObject, SerializationContext,
                          SerializationCallbackError,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 14211787c89f2..91bc96dc63f89 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -689,11 +689,10 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
     c_string FormatMessageType(MessageType type)
 
     cdef cppclass CMessageReader" arrow::ipc::MessageReader":
-        CStatus ReadNextMessage(unique_ptr[CMessage]* out)
+        @staticmethod
+        unique_ptr[CMessageReader] Open(const shared_ptr[InputStream]& stream)
 
-    cdef cppclass CInputStreamMessageReader \
-            " arrow::ipc::InputStreamMessageReader":
-        CInputStreamMessageReader(const shared_ptr[InputStream]& stream)
+        CStatus ReadNextMessage(unique_ptr[CMessage]* out)
 
     cdef cppclass CRecordBatchWriter" arrow::ipc::RecordBatchWriter":
         CStatus Close()
@@ -915,12 +914,12 @@ cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:
         shared_ptr[CRecordBatch] batch
         vector[shared_ptr[CTensor]] tensors
 
+        CStatus WriteTo(OutputStream* dst)
+        CStatus GetComponents(CMemoryPool* pool, PyObject** dst)
+
     CStatus SerializeObject(object context, object sequence,
                             CSerializedPyObject* out)
 
-    CStatus WriteSerializedObject(const CSerializedPyObject& obj,
-                                  OutputStream* dst)
-
     CStatus DeserializeObject(object context,
                               const CSerializedPyObject& obj,
                               PyObject* base, PyObject** out)
@@ -928,6 +927,10 @@ cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:
     CStatus ReadSerializedObject(RandomAccessFile* src,
                                  CSerializedPyObject* out)
 
+    CStatus GetSerializedFromComponents(int num_tensors, int num_buffers,
+                                        object buffers,
+                                        CSerializedPyObject* out)
+
 
 cdef extern from 'arrow/python/init.h':
     int arrow_init_numpy() except -1
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index e5639137dd3a1..7534b0d0e87ec 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -125,9 +125,11 @@ cdef class MessageReader:
     def open_stream(source):
         cdef MessageReader result = MessageReader()
         cdef shared_ptr[InputStream] in_stream
+        cdef unique_ptr[CMessageReader] reader
         get_input_stream(source, &in_stream)
         with nogil:
-            result.reader.reset(new CInputStreamMessageReader(in_stream))
+            reader = CMessageReader.Open(in_stream)
+            result.reader.reset(reader.release())
 
         return result
 
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index 3ee5c7d4e2770..bb266b2f928ff 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -155,7 +155,7 @@ cdef class SerializedPyObject:
         def __get__(self):
             cdef CMockOutputStream mock_stream
             with nogil:
-                check_status(WriteSerializedObject(self.data, &mock_stream))
+                check_status(self.data.WriteTo(&mock_stream))
 
             return mock_stream.GetExtentBytesWritten()
 
@@ -169,7 +169,7 @@ cdef class SerializedPyObject:
 
     cdef _write_to(self, OutputStream* stream):
         with nogil:
-            check_status(WriteSerializedObject(self.data, stream))
+            check_status(self.data.WriteTo(stream))
 
     def deserialize(self, SerializationContext context=None):
         """
@@ -199,6 +199,46 @@ cdef class SerializedPyObject:
         self.write_to(sink)
         return output
 
+    @staticmethod
+    def from_components(components):
+        """
+        Reconstruct SerializedPyObject from output of
+        SerializedPyObject.to_components
+        """
+        cdef:
+            int num_tensors = components['num_tensors']
+            int num_buffers = components['num_buffers']
+            list buffers = components['data']
+            SerializedPyObject result = SerializedPyObject()
+
+        with nogil:
+            check_status(GetSerializedFromComponents(num_tensors, num_buffers,
+                                                     buffers, &result.data))
+
+        return result
+
+    def to_components(self, memory_pool=None):
+        """
+        Return the decomposed dict representation of the serialized object
+        containing a collection of Buffer objects which maximize opportunities
+        for zero-copy
+
+        Parameters
+        ----------
+        memory_pool : MemoryPool default None
+            Pool to use for necessary allocations
+
+        Returns
+
+        """
+        cdef PyObject* result
+        cdef CMemoryPool* c_pool = maybe_unbox_memory_pool(memory_pool)
+
+        with nogil:
+            check_status(self.data.GetComponents(c_pool, &result))
+
+        return PyObject_to_object(result)
+
 
 def serialize(object value, SerializationContext context=None):
     """EXPERIMENTAL: Serialize a Python sequence
@@ -291,6 +331,24 @@ def deserialize_from(source, object base, SerializationContext context=None):
     return serialized.deserialize(context)
 
 
+def deserialize_components(components, SerializationContext context=None):
+    """
+    Reconstruct Python object from output of SerializedPyObject.to_components
+
+    Parameters
+    ----------
+    components : dict
+        Output of SerializedPyObject.to_components
+    context : SerializationContext, default None
+
+    Returns
+    -------
+    object : the Python object that was originally serialized
+    """
+    serialized = SerializedPyObject.from_components(components)
+    return serialized.deserialize(context)
+
+
 def deserialize(obj, SerializationContext context=None):
     """
     EXPERIMENTAL: Deserialize Python object from Buffer or other Python object
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index ed4fd9ae59aee..d06beeac99280 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -219,6 +219,17 @@ def serialization_roundtrip(value, f):
     result = pa.deserialize_from(f, None, serialization_context)
     assert_equal(value, result)
 
+    _check_component_roundtrip(value)
+
+
+def _check_component_roundtrip(value):
+    # Test to/from components
+    serialized = pa.serialize(value)
+    components = serialized.to_components()
+    from_comp = pa.SerializedPyObject.from_components(components)
+    recons = from_comp.deserialize()
+    assert_equal(value, recons)
+
 
 @pytest.yield_fixture(scope='session')
 def large_memory_map(tmpdir_factory, size=100*1024*1024):
@@ -485,3 +496,25 @@ def test_serialize_subclasses():
     deserialized = serialized.deserialize()
     assert type(deserialized).__name__ == SerializableClass.__name__
     assert deserialized.value == 3
+
+
+def test_serialize_to_components_invalid_cases():
+    buf = pa.frombuffer(b'hello')
+
+    components = {
+        'num_tensors': 0,
+        'num_buffers': 1,
+        'data': [buf]
+    }
+
+    with pytest.raises(pa.ArrowException):
+        pa.deserialize_components(components)
+
+    components = {
+        'num_tensors': 1,
+        'num_buffers': 0,
+        'data': [buf, buf]
+    }
+
+    with pytest.raises(pa.ArrowException):
+        pa.deserialize_components(components)

From a75325a8e273f845a50ee5cdaf050ea74cb82797 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Tue, 28 Nov 2017 12:32:42 -0500
Subject: [PATCH 1292/1644] ARROW-1710: [Java] Remove Non-Nullable Vectors

This removes non-nullable vectors that are no longer part of the vector class hierarchy and renames Nullable*Vector classes to remove the Nullable prefix.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1341 from BryanCutler/java-nullable-vector-rename-ARROW-1710 and squashes the following commits:

7d930dc2 [Bryan Cutler] fixed realloc test
ff2120df [Bryan Cutler] clean up test
374dfcc5 [Bryan Cutler] properly rename BitVector file
6b7a85e7 [Bryan Cutler] remove old BitVector.java before rebase
089f7fc9 [Bryan Cutler] some minor cleanup
4e580d93 [Bryan Cutler] removed legacy BitVector
74f771f5 [Bryan Cutler] fixed remaining tests
8c5dfef6 [Bryan Cutler] fix naming in support classes
6e498e55 [Bryan Cutler] removed nullable prefix
dfed444c [Bryan Cutler] removed non-nullable vectors
---
 .../apache/arrow/tools/EchoServerTest.java    |  30 +-
 .../templates/CaseSensitiveMapWriters.java    |   2 +-
 .../codegen/templates/ComplexReaders.java     |  12 +-
 .../codegen/templates/ComplexWriters.java     |  10 +-
 .../codegen/templates/FixedValueVectors.java  | 765 ---------------
 .../main/codegen/templates/MapWriters.java    |   9 +-
 .../main/codegen/templates/UnionVector.java   |  12 +-
 .../templates/VariableLengthVectors.java      | 677 --------------
 ...hVector.java => BaseFixedWidthVector.java} |  22 +-
 ...ctor.java => BaseVariableWidthVector.java} |  16 +-
 ...bleBigIntVector.java => BigIntVector.java} |  32 +-
 .../org/apache/arrow/vector/BitVector.java    | 878 ++++++++----------
 ...eDateDayVector.java => DateDayVector.java} |  32 +-
 ...eMilliVector.java => DateMilliVector.java} |  32 +-
 ...eDecimalVector.java => DecimalVector.java} |  34 +-
 ...bleFloat4Vector.java => Float4Vector.java} |  32 +-
 ...bleFloat8Vector.java => Float8Vector.java} |  32 +-
 .../arrow/vector/GenerateSampleData.java      | 142 +--
 ...{NullableIntVector.java => IntVector.java} |  32 +-
 ...lDayVector.java => IntervalDayVector.java} |  32 +-
 ...earVector.java => IntervalYearVector.java} |  32 +-
 .../arrow/vector/NullableBitVector.java       | 513 ----------
 .../apache/arrow/vector/NullableVector.java   |  26 -
 ...mallIntVector.java => SmallIntVector.java} |  32 +-
 ...eMicroVector.java => TimeMicroVector.java} |  32 +-
 ...eMilliVector.java => TimeMilliVector.java} |  32 +-
 ...imeNanoVector.java => TimeNanoVector.java} |  32 +-
 ...eTimeSecVector.java => TimeSecVector.java} |  32 +-
 ...ector.java => TimeStampMicroTZVector.java} |  18 +-
 ...oVector.java => TimeStampMicroVector.java} |  18 +-
 ...ector.java => TimeStampMilliTZVector.java} |  18 +-
 ...iVector.java => TimeStampMilliVector.java} |  18 +-
 ...Vector.java => TimeStampNanoTZVector.java} |  18 +-
 ...noVector.java => TimeStampNanoVector.java} |  18 +-
 ...ZVector.java => TimeStampSecTZVector.java} |  18 +-
 ...SecVector.java => TimeStampSecVector.java} |  18 +-
 ...eStampVector.java => TimeStampVector.java} |  22 +-
 ...eTinyIntVector.java => TinyIntVector.java} |  32 +-
 ...lableUInt1Vector.java => UInt1Vector.java} |  26 +-
 ...lableUInt2Vector.java => UInt2Vector.java} |  26 +-
 ...lableUInt4Vector.java => UInt4Vector.java} |  26 +-
 ...lableUInt8Vector.java => UInt8Vector.java} |  26 +-
 ...BinaryVector.java => VarBinaryVector.java} |  32 +-
 ...eVarCharVector.java => VarCharVector.java} |  32 +-
 ...etter.java => VectorDefinitionSetter.java} |   2 +-
 .../complex/AbstractContainerVector.java      |   4 +-
 .../vector/complex/EmptyValuePopulator.java   |  10 +-
 .../arrow/vector/complex/MapVector.java       | 562 +++++++----
 .../vector/complex/NonNullableMapVector.java  | 352 +++++++
 .../vector/complex/NullableMapVector.java     | 512 ----------
 .../complex/impl/ComplexWriterImpl.java       |   6 +-
 .../complex/impl/NullableMapReaderImpl.java   |   8 +-
 .../impl/NullableMapWriterFactory.java        |   4 +-
 .../vector/complex/impl/PromotableWriter.java |   4 +-
 .../complex/impl/SingleMapReaderImpl.java     |   5 +-
 .../vector/complex/impl/UnionListReader.java  |   1 -
 .../arrow/vector/ipc/JsonFileReader.java      |  14 +-
 .../arrow/vector/ipc/JsonFileWriter.java      |  52 +-
 .../org/apache/arrow/vector/types/Types.java  | 186 ++--
 .../arrow/vector/util/DecimalUtility.java     |   1 -
 .../apache/arrow/vector/TestBitVector.java    | 189 ++--
 .../vector/TestBufferOwnershipTransfer.java   |   8 +-
 .../org/apache/arrow/vector/TestCopyFrom.java |  86 +-
 .../arrow/vector/TestDecimalVector.java       |   6 +-
 .../arrow/vector/TestDictionaryVector.java    |  22 +-
 .../arrow/vector/TestFixedSizeListVector.java |   8 +-
 .../apache/arrow/vector/TestListVector.java   |   8 +-
 .../apache/arrow/vector/TestMapVector.java    |   4 +-
 ...TestOversizedAllocationForValueVector.java |   4 +-
 .../arrow/vector/TestSplitAndTransfer.java    |   8 +-
 .../org/apache/arrow/vector/TestUtils.java    |   8 +-
 .../apache/arrow/vector/TestValueVector.java  | 308 +++---
 .../arrow/vector/TestVectorReAlloc.java       |  17 +-
 .../apache/arrow/vector/TestVectorReset.java  |   5 +-
 .../arrow/vector/TestVectorUnloadLoad.java    |  10 +-
 .../complex/impl/TestPromotableWriter.java    |   4 +-
 .../complex/writer/TestComplexWriter.java     |  22 +-
 .../apache/arrow/vector/ipc/BaseFileTest.java |  60 +-
 .../arrow/vector/ipc/TestArrowFile.java       |  26 +-
 .../arrow/vector/ipc/TestArrowStream.java     |   6 +-
 .../arrow/vector/ipc/TestArrowStreamPipe.java |   7 +-
 .../apache/arrow/vector/ipc/TestJSONFile.java |  10 +-
 82 files changed, 2159 insertions(+), 4320 deletions(-)
 delete mode 100644 java/vector/src/main/codegen/templates/FixedValueVectors.java
 delete mode 100644 java/vector/src/main/codegen/templates/VariableLengthVectors.java
 rename java/vector/src/main/java/org/apache/arrow/vector/{BaseNullableFixedWidthVector.java => BaseFixedWidthVector.java} (97%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{BaseNullableVariableWidthVector.java => BaseVariableWidthVector.java} (98%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableBigIntVector.java => BigIntVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableDateDayVector.java => DateDayVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableDateMilliVector.java => DateMilliVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableDecimalVector.java => DecimalVector.java} (91%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableFloat4Vector.java => Float4Vector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableFloat8Vector.java => Float8Vector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableIntVector.java => IntVector.java} (91%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableIntervalDayVector.java => IntervalDayVector.java} (91%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableIntervalYearVector.java => IntervalYearVector.java} (90%)
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableSmallIntVector.java => SmallIntVector.java} (91%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeMicroVector.java => TimeMicroVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeMilliVector.java => TimeMilliVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeNanoVector.java => TimeNanoVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeSecVector.java => TimeSecVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampMicroTZVector.java => TimeStampMicroTZVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampMicroVector.java => TimeStampMicroVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampMilliTZVector.java => TimeStampMilliTZVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampMilliVector.java => TimeStampMilliVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampNanoTZVector.java => TimeStampNanoTZVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampNanoVector.java => TimeStampNanoVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampSecTZVector.java => TimeStampSecTZVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampSecVector.java => TimeStampSecVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampVector.java => TimeStampVector.java} (88%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTinyIntVector.java => TinyIntVector.java} (91%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableUInt1Vector.java => UInt1Vector.java} (91%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableUInt2Vector.java => UInt2Vector.java} (91%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableUInt4Vector.java => UInt4Vector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableUInt8Vector.java => UInt8Vector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableVarBinaryVector.java => VarBinaryVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableVarCharVector.java => VarCharVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{NullableVectorDefinitionSetter.java => VectorDefinitionSetter.java} (94%)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index d8693c596155b..47b5541d17e4e 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -35,9 +35,9 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.NullableIntVector;
-import org.apache.arrow.vector.NullableTinyIntVector;
-import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
@@ -92,7 +92,7 @@ public static void stopEchoServer() throws IOException, InterruptedException {
 
   private void testEchoServer(int serverPort,
                               Field field,
-                              NullableTinyIntVector vector,
+                              TinyIntVector vector,
                               int batches)
       throws UnknownHostException, IOException {
     VectorSchemaRoot root = new VectorSchemaRoot(asList(field), asList((FieldVector) vector), 0);
@@ -115,7 +115,7 @@ private void testEchoServer(int serverPort,
 
       assertEquals(new Schema(asList(field)), reader.getVectorSchemaRoot().getSchema());
 
-      NullableTinyIntVector readVector = (NullableTinyIntVector) reader.getVectorSchemaRoot()
+      TinyIntVector readVector = (TinyIntVector) reader.getVectorSchemaRoot()
           .getFieldVectors().get(0);
       for (int i = 0; i < batches; i++) {
         Assert.assertTrue(reader.loadNextBatch());
@@ -140,8 +140,8 @@ public void basicTest() throws InterruptedException, IOException {
         "testField",
         new FieldType(true, new ArrowType.Int(8, true), null, null),
         Collections.<Field>emptyList());
-    NullableTinyIntVector vector =
-        new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
+    TinyIntVector vector =
+        new TinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
     Schema schema = new Schema(asList(field));
 
     // Try an empty stream, just the header.
@@ -158,13 +158,13 @@ public void basicTest() throws InterruptedException, IOException {
   public void testFlatDictionary() throws IOException {
     DictionaryEncoding writeEncoding = new DictionaryEncoding(1L, false, null);
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
-         NullableIntVector writeVector =
-             new NullableIntVector(
+         IntVector writeVector =
+             new IntVector(
                  "varchar",
                  new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
                  allocator);
-         NullableVarCharVector writeDictionaryVector =
-             new NullableVarCharVector(
+         VarCharVector writeDictionaryVector =
+             new VarCharVector(
                  "dict",
                  FieldType.nullable(VARCHAR.getType()),
                  allocator)) {
@@ -218,7 +218,7 @@ public void testFlatDictionary() throws IOException {
 
         Dictionary dictionary = reader.lookup(1L);
         Assert.assertNotNull(dictionary);
-        NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector());
+        VarCharVector dictionaryVector = ((VarCharVector) dictionary.getVector());
         Assert.assertEquals(3, dictionaryVector.getValueCount());
         Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
         Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
@@ -231,8 +231,8 @@ public void testFlatDictionary() throws IOException {
   public void testNestedDictionary() throws IOException {
     DictionaryEncoding writeEncoding = new DictionaryEncoding(2L, false, null);
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
-         NullableVarCharVector writeDictionaryVector =
-             new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
+         VarCharVector writeDictionaryVector =
+             new VarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
          ListVector writeVector = ListVector.empty("list", allocator)) {
 
       // data being written:
@@ -300,7 +300,7 @@ public void testNestedDictionary() throws IOException {
 
         Dictionary readDictionary = reader.lookup(2L);
         Assert.assertNotNull(readDictionary);
-        NullableVarCharVector dictionaryVector = ((NullableVarCharVector) readDictionary.getVector());
+        VarCharVector dictionaryVector = ((VarCharVector) readDictionary.getVector());
         Assert.assertEquals(2, dictionaryVector.getValueCount());
         Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
         Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
diff --git a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
index 5357f9b8a9d3a..7c652902eb8f5 100644
--- a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
+++ b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
@@ -23,7 +23,7 @@
 <#if mode == "Single">
 <#assign containerClass = "MapVector" />
 <#else>
-<#assign containerClass = "NullableMapVector" />
+<#assign containerClass = "MapVector" />
 </#if>
 
 <#include "/@includes/license.ftl" />
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 60347a2fcb8e2..4863ecdb63284 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -53,9 +53,9 @@
 @SuppressWarnings("unused")
 public class ${name}ReaderImpl extends AbstractFieldReader {
   
-  private final ${nullMode}${name}Vector vector;
+  private final ${name}Vector vector;
   
-  public ${name}ReaderImpl(${nullMode}${name}Vector vector){
+  public ${name}ReaderImpl(${name}Vector vector){
     super();
     this.vector = vector;
   }
@@ -69,11 +69,7 @@ public Field getField(){
   }
   
   public boolean isSet(){
-    <#if nullMode == "Nullable">
-        return !vector.isNull(idx());
-    <#else>
-    return true;
-    </#if>
+    return !vector.isNull(idx());
   }
 
   public void copyAsValue(${minor.class?cap_first}Writer writer){
@@ -88,7 +84,7 @@ public void copyAsField(String name, MapWriter writer){
 
   <#if nullMode != "Nullable">
   public void read(${minor.class?cap_first}Holder h){
-    vector.getAccessor().get(idx(), h);
+    vector.get(idx(), h);
   }
   </#if>
 
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 406bbb39c7f4a..98672d54cc38e 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -39,9 +39,9 @@
 @SuppressWarnings("unused")
 public class ${eName}WriterImpl extends AbstractFieldWriter {
 
-  final Nullable${name}Vector vector;
+  final ${name}Vector vector;
 
-  public ${eName}WriterImpl(Nullable${name}Vector vector) {
+  public ${eName}WriterImpl(${name}Vector vector) {
     this.vector = vector;
   }
 
@@ -82,7 +82,7 @@ public void write(${minor.class?cap_first}Holder h) {
     vector.getMutator().setValueCount(idx()+1);
   }
 
-  public void write(Nullable${minor.class?cap_first}Holder h) {
+  public void write(${minor.class?cap_first}Holder h) {
     mutator.addSafe(idx(), h);
     vector.getMutator().setValueCount(idx()+1);
   }
@@ -111,7 +111,7 @@ public void write(Nullable${minor.class}Holder h) {
   }
 
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-    vector.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
+    vector.setSafe(idx(), 1<#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
     vector.setValueCount(idx()+1);
   }
 
@@ -122,13 +122,11 @@ public void write(Nullable${minor.class}Holder h) {
   }
   </#if>
 
-  <#if mode == "Nullable">
   public void writeNull() {
     vector.setNull(idx());
     vector.setValueCount(idx()+1);
   }
   </#if>
-  </#if>
 }
 
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/writer/${eName}Writer.java" />
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
deleted file mode 100644
index e07416ba984d4..0000000000000
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ /dev/null
@@ -1,765 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.arrow.vector.util.DecimalUtility;
-
-import java.lang.Override;
-import java.util.concurrent.TimeUnit;
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-<#assign className = "${minor.class}Vector" />
-
-<#if type.major == "Fixed">
-<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/**
- * ${minor.class} implements a vector of fixed width values.  Elements in the vector are accessed
- * by position, starting from the logical start of the vector.  Values should be pushed onto the
- * vector sequentially, but may be randomly accessed.
- *   The width of each element is ${type.width} byte(s)
- *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-public final class ${className} extends BaseDataValueVector implements FixedWidthVector{
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
-
-  public static final int TYPE_WIDTH = ${type.width};
-
-  private final Accessor accessor = new Accessor();
-  private final Mutator mutator = new Mutator();
-
-  private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
-  private int allocationMonitor = 0;
-  <#if minor.typeParams??>
-
-    <#assign typeParams = minor.typeParams?reverse />
-    <#list typeParams as typeParam>
-  private final ${typeParam.type} ${typeParam.name};
-    </#list>
-
-  public ${className}(String name, BufferAllocator allocator<#list typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
-    super(name, allocator);
-    <#list typeParams as typeParam>
-    this.${typeParam.name} = ${typeParam.name};
-    </#list>
-  }
-  <#else>
-  public ${className}(String name, BufferAllocator allocator) {
-    super(name, allocator);
-  }
-  </#if>
-
-  @Override
-  public MinorType getMinorType() {
-    return MinorType.${minor.class?upper_case};
-  }
-
-  @Override
-  public Field getField() {
-        throw new UnsupportedOperationException("internal vector");
-  }
-
-  @Override
-  public FieldReader getReader(){
-        throw new UnsupportedOperationException("non-nullable vectors cannot be used in readers");
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
-    return valueCount * ${type.width};
-  }
-
-  @Override
-  public ArrowBuf getValidityBuffer() {
-    /* this operation is not supported for non-nullable vectors */
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public ArrowBuf getDataBuffer() {
-    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
-    return getBuffer();
-  }
-
-  @Override
-  public ArrowBuf getOffsetBuffer() {
-    /* this operation is not supported for fixed-width vectors */
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public int getValueCapacity(){
-    return (int) (data.capacity() *1.0 / ${type.width});
-  }
-
-  @Override
-  public Accessor getAccessor(){
-    return accessor;
-  }
-
-  @Override
-  public Mutator getMutator(){
-    return mutator;
-  }
-
-  int getAllocationSize() {
-    return allocationSizeInBytes;
-  }
-
-  @Override
-  public void setInitialCapacity(final int valueCount) {
-    final long size = 1L * valueCount * ${type.width};
-    if (size > MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
-    }
-    allocationSizeInBytes = (int)size;
-  }
-
-  @Override
-  public void allocateNew() {
-    if(!allocateNewSafe()){
-      throw new OutOfMemoryException("Failure while allocating buffer.");
-    }
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    long curAllocationSize = allocationSizeInBytes;
-    if (allocationMonitor > 10) {
-      curAllocationSize = Math.max(8, curAllocationSize / 2);
-      allocationMonitor = 0;
-    } else if (allocationMonitor < -2) {
-      curAllocationSize = allocationSizeInBytes * 2L;
-      allocationMonitor = 0;
-    }
-
-    try{
-      allocateBytes(curAllocationSize);
-    } catch (RuntimeException ex) {
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Allocate a new buffer that supports setting at least the provided number of values. May actually be sized bigger
-   * depending on underlying buffer rounding size. Must be called prior to using the ValueVector.
-   *
-   * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
-   *
-   * @param valueCount the number of values to allocate for
-   * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
-   */
-  @Override
-  public void allocateNew(final int valueCount) {
-    allocateBytes(valueCount * ${type.width});
-  }
-
-  @Override
-  public void reset() {
-    allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
-    allocationMonitor = 0;
-    zeroVector();
-    super.reset();
-  }
-
-  private void allocateBytes(final long size) {
-    if (size > MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
-    }
-
-    final int curSize = (int)size;
-    clear();
-    data = allocator.buffer(curSize);
-    data.readerIndex(0);
-    allocationSizeInBytes = curSize;
-  }
-
-  /**
-   * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
-   *
-   * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
-   */
-  public void reAlloc() {
-    long baseSize  = allocationSizeInBytes;
-    final int currentBufferCapacity = data.capacity();
-    if (baseSize < (long)currentBufferCapacity) {
-        baseSize = (long)currentBufferCapacity;
-    }
-    long newAllocationSize = baseSize * 2L;
-    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
-
-    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
-    }
-
-    logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", name, allocationSizeInBytes, newAllocationSize);
-    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
-    newBuf.setBytes(0, data, 0, currentBufferCapacity);
-    final int halfNewCapacity = newBuf.capacity() / 2;
-    newBuf.setZero(halfNewCapacity, halfNewCapacity);
-    newBuf.writerIndex(data.writerIndex());
-    data.release(1);
-    data = newBuf;
-    allocationSizeInBytes = (int)newAllocationSize;
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void zeroVector() {
-    data.setZero(0, data.capacity());
-  }
-
-  public TransferPair getTransferPair(BufferAllocator allocator){
-    return new TransferImpl(name, allocator);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    return new TransferImpl(ref, allocator);
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((${className}) to);
-  }
-
-  public void transferTo(${className} target){
-    target.clear();
-    target.data = data.transferOwnership(target.allocator).buffer;
-    target.data.writerIndex(data.writerIndex());
-    clear();
-  }
-
-  public void splitAndTransferTo(int startIndex, int length, ${className} target) {
-    final int startPoint = startIndex * ${type.width};
-    final int sliceLength = length * ${type.width};
-    target.clear();
-    target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
-    target.data.writerIndex(sliceLength);
-  }
-
-  private class TransferImpl implements TransferPair{
-    private ${className} to;
-
-    public TransferImpl(String name, BufferAllocator allocator){
-      to = new ${className}(name, allocator<#if minor.typeParams??><#list typeParams as typeParam>,  ${className}.this.${typeParam.name}</#list></#if>);
-    }
-
-    public TransferImpl(${className} to) {
-      this.to = to;
-    }
-
-    @Override
-    public ${className} getTo(){
-      return to;
-    }
-
-    @Override
-    public void transfer(){
-      transferTo(to);
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      splitAndTransferTo(startIndex, length, to);
-    }
-
-    @Override
-    public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, ${className}.this);
-    }
-  }
-
-  public void copyFrom(int fromIndex, int thisIndex, ${className} from){
-    <#if (type.width > 8 || minor.class == "IntervalDay")>
-    from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
-    <#else> <#-- type.width <= 8 -->
-    data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width},
-        from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width})
-    );
-    </#if> <#-- type.width -->
-  }
-
-  public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
-    while(thisIndex >= getValueCapacity()) {
-        reAlloc();
-    }
-    copyFrom(fromIndex, thisIndex, from);
-  }
-
-  public void decrementAllocationMonitor() {
-    if (allocationMonitor > 0) {
-      allocationMonitor = 0;
-    }
-    --allocationMonitor;
-  }
-
-  private void incrementAllocationMonitor() {
-    ++allocationMonitor;
-  }
-
-  public final class Accessor extends BaseDataValueVector.BaseAccessor {
-    @Override
-    public int getValueCount() {
-      return data.writerIndex() / ${type.width};
-    }
-
-    @Override
-    public boolean isNull(int index){
-      return false;
-    }
-
-    <#if (type.width > 8 || minor.class == "IntervalDay")>
-    public ${minor.javaType!type.javaType} get(int index) {
-      return data.slice(index * ${type.width}, ${type.width});
-    }
-
-      <#if (minor.class == "IntervalDay")>
-    public void get(int index, ${minor.class}Holder holder){
-      final int offsetIndex = index * ${type.width};
-      holder.days = data.getInt(offsetIndex);
-      holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-    }
-
-    public void get(int index, Nullable${minor.class}Holder holder){
-      final int offsetIndex = index * ${type.width};
-      holder.isSet = 1;
-      holder.days = data.getInt(offsetIndex);
-      holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-    }
-
-    @Override
-    public ${friendlyType} getObject(int index) {
-      final int offsetIndex = index * ${type.width};
-      final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-      final int  days   = data.getInt(offsetIndex);
-      final Period p = new Period();
-      return p.plusDays(days).plusMillis(millis);
-    }
-
-    public StringBuilder getAsStringBuilder(int index) {
-      final int offsetIndex = index * ${type.width};
-
-      int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-      final int  days   = data.getInt(offsetIndex);
-
-      final int hours  = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
-      millis     = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
-
-      final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
-      millis      = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
-
-      final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
-      millis      = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
-
-      final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
-
-      return(new StringBuilder().
-              append(days).append(dayString).
-              append(hours).append(":").
-              append(minutes).append(":").
-              append(seconds).append(".").
-              append(millis));
-    }
-
-      <#elseif minor.class == "Decimal">
-    public void get(int index, ${minor.class}Holder holder) {
-        holder.start = index * ${type.width};
-        holder.buffer = data;
-        holder.scale = scale;
-        holder.precision = precision;
-    }
-
-    public void get(int index, Nullable${minor.class}Holder holder) {
-        holder.isSet = 1;
-        holder.start = index * ${type.width};
-        holder.buffer = data;
-        holder.scale = scale;
-        holder.precision = precision;
-    }
-
-    @Override
-    public ${friendlyType} getObject(int index) {
-      return DecimalUtility.getBigDecimalFromArrowBuf(data, index, scale);
-    }
-
-      <#else>
-    public void get(int index, ${minor.class}Holder holder){
-      holder.buffer = data;
-      holder.start = index * ${type.width};
-    }
-
-    public void get(int index, Nullable${minor.class}Holder holder){
-      holder.isSet = 1;
-      holder.buffer = data;
-      holder.start = index * ${type.width};
-    }
-
-    @Override
-    public ${friendlyType} getObject(int index) {
-      return data.slice(index * ${type.width}, ${type.width})
-    }
-
-      </#if>
-    <#else> <#-- type.width <= 8 -->
-
-    public ${minor.javaType!type.javaType} get(int index) {
-      return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-    }
-      <#if type.width == 4>
-    public long getTwoAsLong(int index) {
-      return data.getLong(index * ${type.width});
-    }
-
-      </#if>
-      <#if minor.class == "DateDay" ||
-           minor.class == "TimeSec" ||
-           minor.class == "TimeMicro" ||
-           minor.class == "TimeNano">
-    @Override
-    public ${friendlyType} getObject(int index) {
-      return get(index);
-    }
-
-      <#elseif minor.class == "DateMilli" || minor.class == "TimeMilli" || minor.class == "TimeStampMilli">
-    @Override
-    public ${friendlyType} getObject(int index) {
-      org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      return ldt;
-    }
-
-      <#elseif minor.class == "TimeStampSec">
-    @Override
-    public ${friendlyType} getObject(int index) {
-      long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
-      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
-      return date;
-    }
-
-      <#elseif minor.class == "TimeStampMicro">
-    @Override
-    public ${friendlyType} getObject(int index) {
-      // value is truncated when converting microseconds to milliseconds in order to use DateTime type
-      long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
-      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
-      return date;
-    }
-
-      <#elseif minor.class == "TimeStampNano">
-    @Override
-    public ${friendlyType} getObject(int index) {
-      // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
-      long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
-      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
-      return date;
-    }
-
-      <#elseif minor.class == "IntervalYear">
-    @Override
-    public ${friendlyType} getObject(int index) {
-
-      final int value = get(index);
-
-      final int years  = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-      final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-      final Period p = new Period();
-      return p.plusYears(years).plusMonths(months);
-    }
-
-    public StringBuilder getAsStringBuilder(int index) {
-      int months  = data.getInt(index);
-
-      final int years  = (months / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-      months = (months % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-
-      final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
-      final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
-
-      return(new StringBuilder().
-             append(years).append(yearString).
-             append(months).append(monthString));
-    }
-
-      <#else>
-    @Override
-    public ${friendlyType} getObject(int index) {
-      return get(index);
-    }
-    public ${minor.javaType!type.javaType} getPrimitiveObject(int index) {
-      return get(index);
-    }
-
-      </#if>
-    public void get(int index, ${minor.class}Holder holder){
-      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-    }
-
-    public void get(int index, Nullable${minor.class}Holder holder){
-      holder.isSet = 1;
-      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-    }
-
-    </#if> <#-- type.width -->
-  }
-
-  /**
-   * ${minor.class}.Mutator implements a mutable vector of fixed width values.  Elements in the
-   * vector are accessed by position from the logical start of the vector.  Values should be pushed
-   * onto the vector sequentially, but may be randomly accessed.
-   *   The width of each element is ${type.width} byte(s)
-   *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
-   *
-   * NB: this class is automatically generated from FixedValueVectorTypes.tdd using FreeMarker.
-   */
-  public final class Mutator extends BaseDataValueVector.BaseMutator {
-
-    private Mutator(){};
-
-   /**
-    * Set the element at the given index to the given value.  Note that widths smaller than
-    * 32 bits are handled by the ArrowBuf interface.
-    *
-    * @param index   position of the bit to set
-    * @param value   value to set
-    */
-    <#if (type.width > 8) || minor.class == "IntervalDay">
-   public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-     data.setBytes(index * ${type.width}, value, 0, ${type.width});
-   }
-
-   public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     data.setBytes(index * ${type.width}, value, 0, ${type.width});
-   }
-
-      <#if (minor.class == "IntervalDay")>
-   public void set(int index, int days, int milliseconds){
-     final int offsetIndex = index * ${type.width};
-     data.setInt(offsetIndex, days);
-     data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
-   }
-
-   protected void set(int index, ${minor.class}Holder holder){
-     set(index, holder.days, holder.milliseconds);
-   }
-
-   protected void set(int index, Nullable${minor.class}Holder holder){
-     set(index, holder.days, holder.milliseconds);
-   }
-
-   public void setSafe(int index, int days, int milliseconds){
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     set(index, days, milliseconds);
-   }
-
-   public void setSafe(int index, ${minor.class}Holder holder){
-     setSafe(index, holder.days, holder.milliseconds);
-   }
-
-   public void setSafe(int index, Nullable${minor.class}Holder holder){
-     setSafe(index, holder.days, holder.milliseconds);
-   }
-
-       <#elseif minor.class == "Decimal">
-   public void set(int index, ${minor.class}Holder holder){
-     set(index, holder.start, holder.buffer);
-   }
-
-   void set(int index, Nullable${minor.class}Holder holder){
-     set(index, holder.start, holder.buffer);
-   }
-
-   public void setSafe(int index, Nullable${minor.class}Holder holder){
-     setSafe(index, holder.start, holder.buffer);
-   }
-   public void setSafe(int index, ${minor.class}Holder holder){
-     setSafe(index, holder.start, holder.buffer);
-   }
-
-   public void setSafe(int index, int start, ArrowBuf buffer){
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     set(index, start, buffer);
-   }
-
-   public void set(int index, int start, ArrowBuf buffer){
-     data.setBytes(index * ${type.width}, buffer, start, ${type.width});
-   }
-
-   public void set(int index, ${friendlyType} value){
-     DecimalUtility.checkPrecisionAndScale(value, precision, scale);
-     DecimalUtility.writeBigDecimalToArrowBuf(value, data, index);
-   }
-
-   public void setSafe(int index, ${friendlyType} value){
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     set(index, value);
-   }
-
-       <#else>
-   protected void set(int index, ${minor.class}Holder holder){
-     set(index, holder.start, holder.buffer);
-   }
-
-   public void set(int index, Nullable${minor.class}Holder holder){
-     set(index, holder.start, holder.buffer);
-   }
-
-   public void set(int index, int start, ArrowBuf buffer){
-     data.setBytes(index * ${type.width}, buffer, start, ${type.width});
-   }
-
-   public void setSafe(int index, ${minor.class}Holder holder){
-     setSafe(index, holder.start, holder.buffer);
-   }
-
-   public void setSafe(int index, Nullable${minor.class}Holder holder){
-     setSafe(index, holder.start, holder.buffer);
-   }
-
-   public void setSafe(int index, int start, ArrowBuf buffer){
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     set(index, holder);
-   }
-
-   public void set(int index, Nullable${minor.class}Holder holder){
-     data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
-   }
-       </#if>
-
-   @Override
-   public void generateTestData(int count) {
-     setValueCount(count);
-     boolean even = true;
-     final int valueCount = getAccessor().getValueCount();
-     for(int i = 0; i < valueCount; i++, even = !even) {
-       final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE;
-       for(int w = 0; w < ${type.width}; w++){
-         data.setByte(i + w, b);
-       }
-     }
-   }
-
-     <#else> <#-- type.width <= 8 -->
-   public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
-   }
-
-   public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     set(index, value);
-   }
-
-   protected void set(int index, ${minor.class}Holder holder){
-     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
-   }
-
-   public void setSafe(int index, ${minor.class}Holder holder){
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     set(index, holder);
-   }
-
-   protected void set(int index, Nullable${minor.class}Holder holder){
-     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
-   }
-
-   public void setSafe(int index, Nullable${minor.class}Holder holder){
-     while(index >= getValueCapacity()) {
-       reAlloc();
-     }
-     set(index, holder);
-   }
-
-   @Override
-   public void generateTestData(int size) {
-     setValueCount(size);
-     boolean even = true;
-     final int valueCount = getAccessor().getValueCount();
-     for(int i = 0; i < valueCount; i++, even = !even) {
-       if(even){
-         set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE);
-       }else{
-         set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE);
-       }
-     }
-   }
-
-   public void generateTestDataAlt(int size) {
-     setValueCount(size);
-     boolean even = true;
-     final int valueCount = getAccessor().getValueCount();
-     for(int i = 0; i < valueCount; i++, even = !even) {
-       if(even){
-         set(i, (${(minor.javaType!type.javaType)}) 1);
-       }else{
-         set(i, (${(minor.javaType!type.javaType)}) 0);
-       }
-     }
-   }
-
-    </#if> <#-- type.width -->
-
-   @Override
-   public void setValueCount(int valueCount) {
-     final int currentValueCapacity = getValueCapacity();
-     final int idx = (${type.width} * valueCount);
-     while(valueCount > getValueCapacity()) {
-       reAlloc();
-     }
-     if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
-       incrementAllocationMonitor();
-     } else if (allocationMonitor > 0) {
-       allocationMonitor = 0;
-     }
-     VectorTrimmer.trim(data, idx);
-     data.writerIndex(valueCount * ${type.width});
-   }
- }
-}
-
-  </#if> <#-- type.major -->
-</#list>
-</#list>
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 6afd508e68837..a5ac1b7170442 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -21,9 +21,9 @@
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" />
 <#assign index = "idx()">
 <#if mode == "Single">
-<#assign containerClass = "MapVector" />
+<#assign containerClass = "NonNullableMapVector" />
 <#else>
-<#assign containerClass = "NullableMapVector" />
+<#assign containerClass = "MapVector" />
 </#if>
 
 <#include "/@includes/license.ftl" />
@@ -51,7 +51,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
   private final Map<String, FieldWriter> fields = Maps.newHashMap();
   public ${mode}MapWriter(${containerClass} container) {
     <#if mode == "Single">
-    if (container instanceof NullableMapVector) {
+    if (container instanceof MapVector) {
       throw new IllegalArgumentException("Invalid container: " + container);
     }
     </#if>
@@ -124,7 +124,7 @@ public MapWriter map(String name) {
     FieldWriter writer = fields.get(finalName);
     if(writer == null){
       int vectorCount=container.size();
-      NullableMapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), NullableMapVector.class);
+      MapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), MapVector.class);
       writer = new PromotableWriter(vector, container, getNullableMapWriterFactory());
       if(vectorCount != container.size()) {
         writer.allocate();
@@ -214,7 +214,6 @@ public void end() {
   <#assign upperName = minor.class?upper_case />
   <#assign capName = minor.class?cap_first />
   <#assign vectName = capName />
-  <#assign vectName = "Nullable${capName}" />
 
   <#if minor.typeParams?? >
   @Override
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 73165315ea556..501933f8fbc31 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -65,7 +65,7 @@ public class UnionVector implements FieldVector {
   MapVector internalMap;
   protected ArrowBuf typeBuffer;
 
-  private NullableMapVector mapVector;
+  private MapVector mapVector;
   private ListVector listVector;
 
   private FieldReader reader;
@@ -174,10 +174,10 @@ public long getOffsetBufferAddress() {
   @Override
   public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); }
 
-  public NullableMapVector getMap() {
+  public MapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
-      mapVector = addOrGet(MinorType.MAP, NullableMapVector.class);
+      mapVector = addOrGet(MinorType.MAP, MapVector.class);
       if (internalMap.size() > vectorCount) {
         mapVector.allocateNew();
         if (callBack != null) {
@@ -195,12 +195,12 @@ public NullableMapVector getMap() {
       <#assign lowerCaseName = name?lower_case/>
       <#if !minor.typeParams?? >
 
-  private Nullable${name}Vector ${uncappedName}Vector;
+  private ${name}Vector ${uncappedName}Vector;
 
-  public Nullable${name}Vector get${name}Vector() {
+  public ${name}Vector get${name}Vector() {
     if (${uncappedName}Vector == null) {
       int vectorCount = internalMap.size();
-      ${uncappedName}Vector = addOrGet(MinorType.${name?upper_case}, Nullable${name}Vector.class);
+      ${uncappedName}Vector = addOrGet(MinorType.${name?upper_case}, ${name}Vector.class);
       if (internalMap.size() > vectorCount) {
         ${uncappedName}Vector.allocateNew();
         if (callBack != null) {
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
deleted file mode 100644
index 3934e74f11b2d..0000000000000
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ /dev/null
@@ -1,677 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.Override;
-
-import org.apache.drill.exec.exception.OutOfMemoryException;
-import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.BaseValueVector;
-import org.apache.drill.exec.vector.VariableWidthVector;
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-<#assign className = "${minor.class}Vector" />
-
-<#if type.major == "VarLen">
-<@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/**
- * ${minor.class}Vector implements a vector of variable width values.  Elements in the vector
- * are accessed by position from the logical start of the vector.  A fixed width offsetVector
- * is used to convert an element's position to it's offset from the start of the (0-based)
- * ArrowBuf.  Size is inferred by adjacent elements.
- *   The width of each element is ${type.width} byte(s)
- *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-public final class ${className} extends BaseDataValueVector implements VariableWidthVector{
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
-
-  private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
-  private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
-  private static final int MIN_BYTE_COUNT = 4096;
-
-  public final static String OFFSETS_VECTOR_NAME = "$offsets$";
-  final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(OFFSETS_VECTOR_NAME, allocator);
-
-  private final Accessor accessor;
-  private final Mutator mutator;
-
-  private final UInt${type.width}Vector.Accessor oAccessor;
-
-  private int allocationSizeInBytes = INITIAL_BYTE_COUNT;
-  private int allocationMonitor = 0;
-
-  <#if minor.typeParams??>
-     <#list minor.typeParams as typeParam>
-  private final ${typeParam.type} ${typeParam.name};
-    </#list>
-
-  public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
-    super(name, allocator);
-    this.oAccessor = offsetVector.getAccessor();
-    this.accessor = new Accessor();
-    this.mutator = new Mutator();
-    <#list minor.typeParams as typeParam>
-    this.${typeParam.name} = ${typeParam.name};
-    </#list>
-  }
-  <#else>
-  public ${className}(String name, BufferAllocator allocator) {
-    super(name, allocator);
-    this.oAccessor = offsetVector.getAccessor();
-    this.accessor = new Accessor();
-    this.mutator = new Mutator();
-  }
-  </#if>
-
-  @Override
-  public Field getField() {
-    throw new UnsupportedOperationException("internal vector");
-  }
-
-  @Override
-  public MinorType getMinorType() {
-    return MinorType.${minor.class?upper_case};
-  }
-
-  @Override
-  public FieldReader getReader(){
-    throw new UnsupportedOperationException("internal vector");
-  }
-
-  @Override
-  public int getBufferSize(){
-    if (getAccessor().getValueCount() == 0) {
-      return 0;
-    }
-    return offsetVector.getBufferSize() + data.writerIndex();
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
-
-    final int idx = offsetVector.getAccessor().get(valueCount);
-    return offsetVector.getBufferSizeFor(valueCount + 1) + idx;
-  }
-
-  @Override
-  public ArrowBuf getValidityBuffer() {
-    /* this operation is not supported for non-nullable vectors */
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public ArrowBuf getDataBuffer() {
-    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
-    return getBuffer();
-  }
-
-  @Override
-  public ArrowBuf getOffsetBuffer() {
-    /* dataBuffer associated with the underlying offsetVector */
-    return offsetVector.getDataBuffer();
-  }
-
-  @Override
-  public int getValueCapacity(){
-    return Math.max(offsetVector.getValueCapacity() - 1, 0);
-  }
-
-  @Override
-  public int getByteCapacity(){
-    return data.capacity();
-  }
-
-  @Override
-  public int getCurrentSizeInBytes() {
-    return offsetVector.getAccessor().get(getAccessor().getValueCount());
-  }
-
-  /**
-   * Return the number of bytes contained in the current var len byte vector.
-   * @return the number of bytes contained in the current var len byte vector
-   */
-  public int getVarByteLength(){
-    final int valueCount = getAccessor().getValueCount();
-    if(valueCount == 0) {
-      return 0;
-    }
-    return offsetVector.getAccessor().get(valueCount);
-  }
-
-  @Override
-  public void clear() {
-    super.clear();
-    offsetVector.clear();
-  }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    final ArrowBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(false), super.getBuffers(false), ArrowBuf.class);
-    if (clear) {
-      // does not make much sense but we have to retain buffers even when clear is set. refactor this interface.
-      for (final ArrowBuf buffer:buffers) {
-        buffer.retain(1);
-      }
-      clear();
-    }
-    return buffers;
-  }
-
-  public long getOffsetAddr(){
-    return offsetVector.getDataBuffer().memoryAddress();
-  }
-
-  public UInt${type.width}Vector getOffsetVector(){
-    return offsetVector;
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator){
-        return new TransferImpl(name, allocator);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
-    return new TransferImpl(ref, allocator);
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((${className}) to);
-  }
-
-  public void transferTo(${className} target){
-    target.clear();
-    this.offsetVector.transferTo(target.offsetVector);
-    target.data = data.transferOwnership(target.allocator).buffer;
-    target.data.writerIndex(data.writerIndex());
-    clear();
-  }
-
-  public void splitAndTransferTo(int startIndex, int length, ${className} target) {
-    UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor();
-    final int startPoint = offsetVectorAccessor.get(startIndex);
-    final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint;
-    target.clear();
-    target.offsetVector.allocateNew(length + 1);
-    offsetVectorAccessor = this.offsetVector.getAccessor();
-    final UInt4Vector.Mutator targetOffsetVectorMutator = target.offsetVector.getMutator();
-    for (int i = 0; i < length + 1; i++) {
-      targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
-    }
-    target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
-    target.getMutator().setValueCount(length);
-}
-
-  protected void copyFrom(int fromIndex, int thisIndex, ${className} from){
-    final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
-    final int start = fromOffsetVectorAccessor.get(fromIndex);
-    final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
-    final int len = end - start;
-
-    final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
-    from.data.getBytes(start, data, outputStart, len);
-    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
-  }
-
-  public boolean copyFromSafe(int fromIndex, int thisIndex, ${className} from){
-    final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
-    final int start = fromOffsetVectorAccessor.get(fromIndex);
-    final int end =   fromOffsetVectorAccessor.get(fromIndex + 1);
-    final int len = end - start;
-    final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
-
-    while(data.capacity() < outputStart + len) {
-        reAlloc();
-    }
-
-    offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len);
-    from.data.getBytes(start, data, outputStart, len);
-    return true;
-  }
-
-  private class TransferImpl implements TransferPair{
-    ${className} to;
-
-    public TransferImpl(String name, BufferAllocator allocator){
-      to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>,  ${className}.this.${typeParam.name}</#list></#if>);
-    }
-
-    public TransferImpl(${className} to){
-      this.to = to;
-    }
-
-    @Override
-    public ${className} getTo(){
-      return to;
-    }
-
-    @Override
-    public void transfer(){
-      transferTo(to);
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      splitAndTransferTo(startIndex, length, to);
-    }
-
-    @Override
-    public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, ${className}.this);
-    }
-  }
-
-  @Override
-  public void setInitialCapacity(final int valueCount) {
-    final long size = 1L * valueCount * ${type.width};
-    if (size > MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
-    }
-    allocationSizeInBytes = (int)size;
-    offsetVector.setInitialCapacity(valueCount + 1);
-  }
-
-  @Override
-  public void allocateNew() {
-    if(!allocateNewSafe()){
-      throw new OutOfMemoryException("Failure while allocating buffer.");
-    }
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    long curAllocationSize = allocationSizeInBytes;
-    if (allocationMonitor > 10) {
-      curAllocationSize = Math.max(MIN_BYTE_COUNT, curAllocationSize / 2);
-      allocationMonitor = 0;
-    } else if (allocationMonitor < -2) {
-      curAllocationSize = curAllocationSize * 2L;
-      allocationMonitor = 0;
-    }
-
-    if (curAllocationSize > MAX_ALLOCATION_SIZE) {
-      return false;
-    }
-
-    clear();
-    /* Boolean to keep track if all the memory allocations were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    try {
-      final int requestedSize = (int)curAllocationSize;
-      data = allocator.buffer(requestedSize);
-      allocationSizeInBytes = requestedSize;
-      offsetVector.allocateNew();
-    } catch (OutOfMemoryException e) {
-      clear();
-      return false;
-    }
-    data.readerIndex(0);
-    offsetVector.zeroVector();
-    return true;
-  }
-
-  @Override
-  public void allocateNew(int totalBytes, int valueCount) {
-    clear();
-    assert totalBytes >= 0;
-    try {
-      data = allocator.buffer(totalBytes);
-      offsetVector.allocateNew(valueCount + 1);
-    } catch (RuntimeException e) {
-      clear();
-      throw e;
-    }
-    data.readerIndex(0);
-    allocationSizeInBytes = totalBytes;
-    offsetVector.zeroVector();
-  }
-
-  @Override
-  public void reset() {
-    allocationSizeInBytes = INITIAL_BYTE_COUNT;
-    allocationMonitor = 0;
-    data.readerIndex(0);
-    offsetVector.zeroVector();
-    super.reset();
-  }
-
-  public void reAlloc() {
-    long baseSize = allocationSizeInBytes;
-    final int currentBufferCapacity = data.capacity();
-    if (baseSize < (long)currentBufferCapacity) {
-      baseSize = (long)currentBufferCapacity;
-    }
-    long newAllocationSize = baseSize * 2L;
-    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
-
-    if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
-      throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
-    }
-
-    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
-    newBuf.setBytes(0, data, 0, currentBufferCapacity);
-    data.release();
-    data = newBuf;
-    allocationSizeInBytes = (int)newAllocationSize;
-  }
-
-  public void decrementAllocationMonitor() {
-    if (allocationMonitor > 0) {
-      allocationMonitor = 0;
-    }
-    --allocationMonitor;
-  }
-
-  private void incrementAllocationMonitor() {
-    ++allocationMonitor;
-  }
-
-  @Override
-  public Accessor getAccessor(){
-    return accessor;
-  }
-
-  @Override
-  public Mutator getMutator() {
-    return mutator;
-  }
-
-  public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor {
-    final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
-    public long getStartEnd(int index){
-      return oAccessor.getTwoAsLong(index);
-    }
-
-    public byte[] get(int index) {
-      assert index >= 0;
-      final int startIdx = oAccessor.get(index);
-      final int length = oAccessor.get(index + 1) - startIdx;
-      assert length >= 0;
-      final byte[] dst = new byte[length];
-      data.getBytes(startIdx, dst, 0, length);
-      return dst;
-    }
-
-    @Override
-    public int getValueLength(int index) {
-      final UInt${type.width}Vector.Accessor offsetVectorAccessor = offsetVector.getAccessor();
-      return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index);
-    }
-
-    public void get(int index, ${minor.class}Holder holder){
-      holder.start = oAccessor.get(index);
-      holder.end = oAccessor.get(index + 1);
-      holder.buffer = data;
-    }
-
-    public void get(int index, Nullable${minor.class}Holder holder){
-      holder.isSet = 1;
-      holder.start = oAccessor.get(index);
-      holder.end = oAccessor.get(index + 1);
-      holder.buffer = data;
-    }
-
-    <#switch minor.class>
-      <#case "VarChar">
-    @Override
-    public ${friendlyType} getObject(int index) {
-      Text text = new Text();
-      text.set(get(index));
-      return text;
-    }
-      <#break>
-      <#case "Decimal">
-    @Override
-    public ${friendlyType} getObject(int index) {
-      return new BigDecimal(new BigInteger(get(index)), scale);
-    }
-      <#break>
-      <#default>
-    @Override
-    public ${friendlyType} getObject(int index) {
-      return get(index);
-    }
-    </#switch>
-
-    @Override
-    public int getValueCount() {
-      return Math.max(offsetVector.getAccessor().getValueCount()-1, 0);
-    }
-
-    @Override
-    public boolean isNull(int index){
-      return false;
-    }
-
-    public UInt${type.width}Vector getOffsetVector(){
-      return offsetVector;
-    }
-  }
-
-  /**
-   * Mutable${minor.class} implements a vector of variable width values.  Elements in the vector
-   * are accessed by position from the logical start of the vector.  A fixed width offsetVector
-   * is used to convert an element's position to it's offset from the start of the (0-based)
-   * ArrowBuf.  Size is inferred by adjacent elements.
-   *   The width of each element is ${type.width} byte(s)
-   *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
-   *
-   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
-   */
-  public final class Mutator extends BaseValueVector.BaseMutator implements VariableWidthVector.VariableWidthMutator {
-
-    /**
-     * Set the variable length element at the specified index to the supplied byte array.
-     *
-     * @param index   position of the bit to set
-     * @param bytes   array of bytes to write
-     */
-    protected void set(int index, byte[] bytes) {
-      assert index >= 0;
-      final int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
-      data.setBytes(currentOffset, bytes, 0, bytes.length);
-    }
-
-    public void setSafe(int index, byte[] bytes) {
-      assert index >= 0;
-
-      final int currentOffset = offsetVector.getAccessor().get(index);
-      while (data.capacity() < currentOffset + bytes.length) {
-        reAlloc();
-      }
-      offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length);
-      data.setBytes(currentOffset, bytes, 0, bytes.length);
-    }
-
-    /**
-     * Set the variable length element at the specified index to the supplied byte array.
-     *
-     * @param index   position of the bit to set
-     * @param bytes   array of bytes to write
-     * @param start   start index of bytes to write
-     * @param length  length of bytes to write
-     */
-    protected void set(int index, byte[] bytes, int start, int length) {
-      assert index >= 0;
-      final int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, bytes, start, length);
-    }
-
-    public void setSafe(int index, ByteBuffer bytes, int start, int length) {
-      assert index >= 0;
-
-      int currentOffset = offsetVector.getAccessor().get(index);
-
-      while (data.capacity() < currentOffset + length) {
-        reAlloc();
-      }
-      offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, bytes, start, length);
-    }
-
-    public void setSafe(int index, byte[] bytes, int start, int length) {
-      assert index >= 0;
-
-      final int currentOffset = offsetVector.getAccessor().get(index);
-
-      while (data.capacity() < currentOffset + length) {
-        reAlloc();
-      }
-      offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, bytes, start, length);
-    }
-
-    @Override
-    public void setValueLengthSafe(int index, int length) {
-      final int offset = offsetVector.getAccessor().get(index);
-      while(data.capacity() < offset + length ) {
-        reAlloc();
-      }
-      offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length);
-    }
-
-
-    public void setSafe(int index, int start, int end, ArrowBuf buffer){
-      final int len = end - start;
-      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-
-      while(data.capacity() < outputStart + len) {
-        reAlloc();
-      }
-
-      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
-      buffer.getBytes(start, data, outputStart, len);
-    }
-
-    public void setSafe(int index, Nullable${minor.class}Holder holder){
-      assert holder.isSet == 1;
-
-      final int start = holder.start;
-      final int end =   holder.end;
-      final int len = end - start;
-
-      int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-
-      while(data.capacity() < outputStart + len) {
-        reAlloc();
-      }
-
-      holder.buffer.getBytes(start, data, outputStart, len);
-      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
-    }
-
-    public void setSafe(int index, ${minor.class}Holder holder){
-      final int start = holder.start;
-      final int end =   holder.end;
-      final int len = end - start;
-      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-
-      while(data.capacity() < outputStart + len) {
-        reAlloc();
-      }
-
-      holder.buffer.getBytes(start, data, outputStart, len);
-      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
-    }
-
-    protected void set(int index, int start, int length, ArrowBuf buffer){
-      assert index >= 0;
-      final int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + length);
-      final ArrowBuf bb = buffer.slice(start, length);
-      data.setBytes(currentOffset, bb);
-    }
-
-    protected void set(int index, Nullable${minor.class}Holder holder){
-      final int length = holder.end - holder.start;
-      final int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, holder.buffer, holder.start, length);
-    }
-
-    protected void set(int index, ${minor.class}Holder holder){
-      final int length = holder.end - holder.start;
-      final int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, holder.buffer, holder.start, length);
-    }
-
-    @Override
-    public void setValueCount(int valueCount) {
-      if (valueCount == 0) {
-        // if no values in vector, don't try to retrieve the current value count.
-        offsetVector.getMutator().setValueCount(0);
-      } else {
-        final int currentByteCapacity = getByteCapacity();
-        final int idx = offsetVector.getAccessor().get(valueCount);
-        data.writerIndex(idx);
-        if (currentByteCapacity > idx * 2) {
-          incrementAllocationMonitor();
-        } else if (allocationMonitor > 0) {
-          allocationMonitor = 0;
-        }
-        VectorTrimmer.trim(data, idx);
-        offsetVector.getMutator().setValueCount(valueCount+1);
-      }
-    }
-
-    @Override
-    public void generateTestData(int size){
-      boolean even = true;
-      <#switch minor.class>
-      <#case "Var16Char">
-      final java.nio.charset.Charset charset = Charsets.UTF_16;
-      <#break>
-      <#case "VarChar">
-      <#default>
-      final java.nio.charset.Charset charset = Charsets.UTF_8;
-      </#switch>
-      final byte[] evenValue = new String("aaaaa").getBytes(charset);
-      final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset);
-      for(int i =0; i < size; i++, even = !even){
-        set(i, even ? evenValue : oddValue);
-        }
-      setValueCount(size);
-    }
-  }
-}
-
-</#if> <#-- type.major -->
-</#list>
-</#list>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
similarity index 97%
rename from java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index f82077f692f3d..cc056904ae8a1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -36,12 +36,12 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * BaseNullableFixedWidthVector provides an abstract interface for
+ * BaseFixedWidthVector provides an abstract interface for
  * implementing vectors of fixed width values. The vectors are nullable
  * implying that zero or more elements in the vector could be NULL.
  */
-public abstract class BaseNullableFixedWidthVector extends BaseValueVector
-        implements FixedWidthVector, FieldVector, NullableVectorDefinitionSetter {
+public abstract class BaseFixedWidthVector extends BaseValueVector
+        implements FixedWidthVector, FieldVector, VectorDefinitionSetter {
   private final byte typeWidth;
 
   protected int valueAllocationSizeInBytes;
@@ -53,7 +53,7 @@ public abstract class BaseNullableFixedWidthVector extends BaseValueVector
   protected ArrowBuf valueBuffer;
   protected int valueCount;
 
-  public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator,
+  public BaseFixedWidthVector(final String name, final BufferAllocator allocator,
                                       FieldType fieldType, final byte typeWidth) {
     super(name, allocator);
     this.typeWidth = typeWidth;
@@ -66,7 +66,7 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all
       valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth;
       validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
     } else {
-      /* specialized handling for NullableBitVector */
+      /* specialized handling for BitVector */
       valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
       validityAllocationSizeInBytes = valueAllocationSizeInBytes;
     }
@@ -311,7 +311,7 @@ public void allocateNew(int valueCount) {
     long valueBufferSize = valueCount * typeWidth;
     long validityBufferSize = getValidityBufferSizeFromCount(valueCount);
     if (typeWidth == 0) {
-      /* specialized handling for NullableBitVector */
+      /* specialized handling for BitVector */
       valueBufferSize = validityBufferSize;
     }
 
@@ -556,7 +556,7 @@ private void setReaderAndWriterIndex() {
     } else {
       validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
       if (typeWidth == 0) {
-        /* specialized handling for NullableBitVector */
+        /* specialized handling for BitVector */
         valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
       } else {
         valueBuffer.writerIndex(valueCount * typeWidth);
@@ -600,7 +600,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
    * for accounting and management purposes.
    * @param target destination vector for transfer
    */
-  public void transferTo(BaseNullableFixedWidthVector target) {
+  public void transferTo(BaseFixedWidthVector target) {
     compareTypes(target, "transferTo");
     target.clear();
     target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
@@ -617,7 +617,7 @@ public void transferTo(BaseNullableFixedWidthVector target) {
    * @param target destination vector
    */
   public void splitAndTransferTo(int startIndex, int length,
-                                 BaseNullableFixedWidthVector target) {
+                                 BaseFixedWidthVector target) {
     compareTypes(target, "splitAndTransferTo");
     target.clear();
     splitAndTransferValidityBuffer(startIndex, length, target);
@@ -629,7 +629,7 @@ public void splitAndTransferTo(int startIndex, int length,
    * Data buffer can always be split and transferred using slicing.
    */
   private void splitAndTransferValueBuffer(int startIndex, int length,
-                                           BaseNullableFixedWidthVector target) {
+                                           BaseFixedWidthVector target) {
     final int startPoint = startIndex * typeWidth;
     final int sliceLength = length * typeWidth;
     target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
@@ -640,7 +640,7 @@ private void splitAndTransferValueBuffer(int startIndex, int length,
    * the starting position of the source index.
    */
   private void splitAndTransferValidityBuffer(int startIndex, int length,
-                                              BaseNullableFixedWidthVector target) {
+                                              BaseFixedWidthVector target) {
     assert startIndex + length <= valueCount;
     int firstByteSource = BitVectorHelper.byteIndex(startIndex);
     int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
similarity index 98%
rename from java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index b9e5442ecf666..b515586bc9f36 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -24,7 +24,7 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -37,8 +37,8 @@
 import java.util.Collections;
 import java.util.List;
 
-public abstract class BaseNullableVariableWidthVector extends BaseValueVector
-        implements VariableWidthVector, FieldVector, NullableVectorDefinitionSetter {
+public abstract class BaseVariableWidthVector extends BaseValueVector
+        implements VariableWidthVector, FieldVector, VectorDefinitionSetter {
   private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
   private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT;
 
@@ -57,7 +57,7 @@ public abstract class BaseNullableVariableWidthVector extends BaseValueVector
   protected final Field field;
   private boolean cleared;
 
-  public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator,
+  public BaseVariableWidthVector(final String name, final BufferAllocator allocator,
                                          FieldType fieldType) {
     super(name, allocator);
     valueAllocationSizeInBytes = INITIAL_BYTE_COUNT;
@@ -657,7 +657,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
    * for accounting and management purposes.
    * @param target destination vector for transfer
    */
-  public void transferTo(BaseNullableVariableWidthVector target) {
+  public void transferTo(BaseVariableWidthVector target) {
     compareTypes(target, "transferTo");
     target.clear();
     target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
@@ -678,7 +678,7 @@ public void transferTo(BaseNullableVariableWidthVector target) {
    * @param target destination vector
    */
   public void splitAndTransferTo(int startIndex, int length,
-                                 BaseNullableVariableWidthVector target) {
+                                 BaseVariableWidthVector target) {
     compareTypes(target, "splitAndTransferTo");
     target.clear();
     splitAndTransferValidityBuffer(startIndex, length, target);
@@ -697,7 +697,7 @@ public void splitAndTransferTo(int startIndex, int length,
    * need to explicitly allocate the offset buffer and set the adjusted offsets
    * in the target vector.
    */
-  private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) {
+  private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseVariableWidthVector target) {
     final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH);
     final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH);
     final int dataLength = end - start;
@@ -713,7 +713,7 @@ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullab
    * Transfer the validity.
    */
   private void splitAndTransferValidityBuffer(int startIndex, int length,
-                                              BaseNullableVariableWidthVector target) {
+                                              BaseVariableWidthVector target) {
     assert startIndex + length <= valueCount;
     int firstByteSource = BitVectorHelper.byteIndex(startIndex);
     int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java
index 8d44c92e46664..ccf0c3067e043 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java
@@ -29,35 +29,35 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableBigIntVector implements a fixed width vector (8 bytes) of
+ * BigIntVector implements a fixed width vector (8 bytes) of
  * integer values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableBigIntVector extends BaseNullableFixedWidthVector {
+public class BigIntVector extends BaseFixedWidthVector {
   public static final byte TYPE_WIDTH = 8;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableBigIntVector. This doesn't allocate any memory for
+   * Instantiate a BigIntVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableBigIntVector(String name, BufferAllocator allocator) {
+  public BigIntVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableBigIntVector. This doesn't allocate any memory for
+   * Instantiate a BigIntVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public BigIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new BigIntReaderImpl(NullableBigIntVector.this);
+    reader = new BigIntReaderImpl(BigIntVector.this);
   }
 
   /**
@@ -136,21 +136,21 @@ public Long getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, BigIntVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
     valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that
+   * Same as {@link #copyFrom(int, int, BigIntVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, BigIntVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -329,22 +329,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableBigIntVector) to);
+    return new TransferImpl((BigIntVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableBigIntVector to;
+    BigIntVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableBigIntVector(ref, field.getFieldType(), allocator);
+      to = new BigIntVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableBigIntVector to) {
+    public TransferImpl(BigIntVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableBigIntVector getTo() {
+    public BigIntVector getTo() {
       return to;
     }
 
@@ -360,7 +360,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this);
+      to.copyFromSafe(fromIndex, toIndex, BigIntVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 26c817008836c..b6f7323a796b4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -18,342 +18,469 @@
 
 package org.apache.arrow.vector;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.BaseAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.impl.BitReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.BitHolder;
 import org.apache.arrow.vector.holders.NullableBitHolder;
-import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
-import io.netty.buffer.ArrowBuf;
-
 /**
- * Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start
- * of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0'
- * or '1'.
+ * BitVector implements a fixed width (1 bit) vector of
+ * boolean values which could be null. Each value in the vector corresponds
+ * to a single bit in the underlying data stream backing the vector.
  */
-public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
-
-  private final Accessor accessor = new Accessor();
-  private final Mutator mutator = new Mutator();
-
-  int valueCount;
-  private int allocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION);
-  private int allocationMonitor = 0;
+public class BitVector extends BaseFixedWidthVector {
+  private final FieldReader reader;
 
+  /**
+   * Instantiate a BitVector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name      name of the vector
+   * @param allocator allocator for memory management.
+   */
   public BitVector(String name, BufferAllocator allocator) {
-    super(name, allocator);
+    this(name, FieldType.nullable(Types.MinorType.BIT.getType()),
+            allocator);
   }
 
-  @Override
-  public void load(ArrowFieldNode fieldNode, ArrowBuf data) {
-    // When the vector is all nulls or all defined, the content of the buffer can be omitted
-    if (data.readableBytes() == 0 && fieldNode.getLength() != 0) {
-      int count = fieldNode.getLength();
-      allocateNew(count);
-      int n = getSizeFromCount(count);
-      if (fieldNode.getNullCount() == 0) {
-        // all defined
-        // create an all 1s buffer
-        // set full bytes
-        int fullBytesCount = count / 8;
-        for (int i = 0; i < fullBytesCount; ++i) {
-          this.data.setByte(i, 0xFF);
-        }
-        int remainder = count % 8;
-        // set remaining bits
-        if (remainder > 0) {
-          byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7));
-          this.data.setByte(fullBytesCount, bitMask);
-        }
-      } else if (fieldNode.getNullCount() == fieldNode.getLength()) {
-        // all null
-        // create an all 0s buffer
-        zeroVector();
-      } else {
-        throw new IllegalArgumentException("The buffer can be empty only if there's no data or it's all null or all defined");
-      }
-      this.data.writerIndex(n);
-    } else {
-      super.load(fieldNode, data);
-    }
-    this.valueCount = fieldNode.getLength();
+  /**
+   * Instantiate a BitVector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name      name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public BitVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, (byte) 0);
+    reader = new BitReaderImpl(BitVector.this);
   }
 
+  /**
+   * Get a reader that supports reading values from this vector
+   *
+   * @return Field Reader for this vector
+   */
   @Override
-  public Field getField() {
-    throw new UnsupportedOperationException("internal vector");
+  public FieldReader getReader() {
+    return reader;
   }
 
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   *
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
   @Override
-  public MinorType getMinorType() {
-    return MinorType.BIT;
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.BIT;
   }
 
+  /**
+   * Sets the desired value capacity for the vector. This function doesn't
+   * allocate any memory for the vector.
+   *
+   * @param valueCount desired number of elements in the vector
+   */
   @Override
-  public FieldReader getReader() {
-    throw new UnsupportedOperationException("internal vector");
+  public void setInitialCapacity(int valueCount) {
+    final int size = getValidityBufferSizeFromCount(valueCount);
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+    }
+    valueAllocationSizeInBytes = size;
+    validityAllocationSizeInBytes = size;
   }
 
+  /**
+   * Get the current value capacity for the vector
+   *
+   * @return number of elements that vector can hold.
+   */
   @Override
-  public int getBufferSize() {
-    return getSizeFromCount(valueCount);
+  public int getValueCapacity() {
+    return (int) (validityBuffer.capacity() * 8L);
   }
 
+  /**
+   * Get the potential buffer size for a particular number of records.
+   *
+   * @param count desired number of elements in the vector
+   * @return estimated size of underlying buffers if the vector holds
+   * a given number of elements
+   */
   @Override
-  public int getBufferSizeFor(final int valueCount) {
-    return getSizeFromCount(valueCount);
+  public int getBufferSizeFor(final int count) {
+    if (count == 0) {
+      return 0;
+    }
+    return 2 * getValidityBufferSizeFromCount(count);
   }
 
+  /**
+   * Get the size (number of bytes) of underlying buffers used by this
+   * vector
+   *
+   * @return size of underlying buffers.
+   */
   @Override
-  public ArrowBuf getValidityBuffer() {
-    /* this operation is not supported for non-nullable vectors */
-    throw new  UnsupportedOperationException();
+  public int getBufferSize() {
+    return getBufferSizeFor(valueCount);
   }
 
-  @Override
-  public ArrowBuf getDataBuffer() {
-    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
-    return getBuffer();
-  }
+  /**
+   * Slice this vector at desired index and length and transfer the
+   * corresponding data to the target vector.
+   *
+   * @param startIndex start position of the split in source vector.
+   * @param length     length of the split.
+   * @param target     destination vector
+   */
+  public void splitAndTransferTo(int startIndex, int length,
+                                 BaseFixedWidthVector target) {
+    compareTypes(target, "splitAndTransferTo");
+    target.clear();
+    target.validityBuffer = splitAndTransferBuffer(startIndex, length, target,
+            validityBuffer, target.validityBuffer);
+    target.valueBuffer = splitAndTransferBuffer(startIndex, length, target,
+            valueBuffer, target.valueBuffer);
 
-  @Override
-  public ArrowBuf getOffsetBuffer() {
-    /* this operation is not supported for fixed-width vectors */
-    throw new UnsupportedOperationException();
+    target.setValueCount(length);
   }
 
-  int getSizeFromCount(int valueCount) {
-    return (int) Math.ceil(valueCount / 8.0);
-  }
+  private ArrowBuf splitAndTransferBuffer(int startIndex, int length,
+                                          BaseFixedWidthVector target,
+                                          ArrowBuf sourceBuffer, ArrowBuf destBuffer) {
+    assert startIndex + length <= valueCount;
+    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+    int byteSizeTarget = getValidityBufferSizeFromCount(length);
+    int offset = startIndex % 8;
 
-  @Override
-  public int getValueCapacity() {
-    return (int) Math.min((long) Integer.MAX_VALUE, data.capacity() * 8L);
-  }
+    if (length > 0) {
+      if (offset == 0) {
+            /* slice */
+        if (destBuffer != null) {
+          destBuffer.release();
+        }
+        destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget);
+        destBuffer.retain(1);
+      } else {
+            /* Copy data
+             * When the first bit starts from the middle of a byte (offset != 0),
+             * copy data from src BitVector.
+             * Each byte in the target is composed by a part in i-th byte,
+             * another part in (i+1)-th byte.
+             */
+        destBuffer = allocator.buffer(byteSizeTarget);
+        destBuffer.readerIndex(0);
+        destBuffer.setZero(0, destBuffer.capacity());
+
+        for (int i = 0; i < byteSizeTarget - 1; i++) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset);
+
+          destBuffer.setByte(i, (b1 + b2));
+        }
 
-  private int getByteIndex(int index) {
-    return (int) Math.floor(index / 8.0);
+            /* Copying the last piece is done in the following manner:
+             * if the source vector has 1 or more bytes remaining, we copy
+             * the last piece as a byte formed by shifting data
+             * from the current byte and the next byte.
+             *
+             * if the source vector has no more bytes remaining
+             * (we are at the last byte), we copy the last piece as a byte
+             * by shifting data from the current byte.
+             */
+        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer,
+                  firstByteSource + byteSizeTarget, offset);
+
+          destBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+        } else {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          destBuffer.setByte(byteSizeTarget - 1, b1);
+        }
+      }
+    }
+
+    return destBuffer;
   }
 
-  @Override
-  public void setInitialCapacity(final int valueCount) {
-    allocationSizeInBytes = getSizeFromCount(valueCount);
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+  private int getBit(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = valueBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
   }
 
-  @Override
-  public void allocateNew() {
-    if (!allocateNewSafe()) {
-      throw new OutOfMemoryException();
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index position of element
+   * @return element at given index
+   */
+  public int get(int index) throws IllegalStateException {
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
     }
+    return getBit(index);
   }
 
-  @Override
-  public boolean allocateNewSafe() {
-    long curAllocationSize = allocationSizeInBytes;
-    if (allocationMonitor > 10) {
-      curAllocationSize = Math.max(8, allocationSizeInBytes / 2);
-      allocationMonitor = 0;
-    } else if (allocationMonitor < -2) {
-      curAllocationSize = allocationSizeInBytes * 2L;
-      allocationMonitor = 0;
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index position of element
+   */
+  public void get(int index, NullableBitHolder holder) {
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
     }
+    holder.isSet = 1;
+    holder.value = getBit(index);
+  }
 
-    try {
-      allocateBytes(curAllocationSize);
-    } catch (OutOfMemoryException ex) {
-      return false;
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index position of element
+   * @return element at given index
+   */
+  public Boolean getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return new Boolean(getBit(index) != 0);
     }
-    return true;
   }
 
-  @Override
-  public void reset() {
-    valueCount = 0;
-    allocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION);
-    allocationMonitor = 0;
-    zeroVector();
-    super.reset();
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   *
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from      source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, BitVector from) {
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    BitVectorHelper.setValidityBit(valueBuffer, thisIndex, from.getBit(fromIndex));
   }
 
   /**
-   * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+   * Same as {@link #copyFrom(int, int, BitVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
    *
-   * @param valueCount The number of values which can be contained within this vector.
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from      source vector
    */
-  @Override
-  public void allocateNew(int valueCount) {
-    final int size = getSizeFromCount(valueCount);
-    allocateBytes(size);
+  public void copyFromSafe(int fromIndex, int thisIndex, BitVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
   }
 
-  private void allocateBytes(final long size) {
-    if (size > MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
-    }
 
-    final int curSize = (int) size;
-    clear();
-    data = allocator.buffer(curSize);
-    zeroVector();
-    allocationSizeInBytes = curSize;
-  }
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
 
   /**
-   * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+   * Set the element at the given index to the given value.
+   *
+   * @param index position of element
+   * @param value value of element
    */
-  public void reAlloc() {
-    long baseSize  = allocationSizeInBytes;
-    final int currentBufferCapacity = data.capacity();
-    if (baseSize < (long)currentBufferCapacity) {
-      baseSize = (long)currentBufferCapacity;
+  public void set(int index, int value) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    if (value != 0) {
+      BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+    } else {
+      BitVectorHelper.setValidityBit(valueBuffer, index, 0);
     }
-    long newAllocationSize = baseSize * 2L;
-    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+  }
 
-    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+  /**
+   * Set the element at the given index to the value set in data holder.
+   * If the value in holder is not indicated as set, element in the
+   * at the given index will be null.
+   *
+   * @param index  position of element
+   * @param holder nullable data holder for value of element
+   */
+  public void set(int index, NullableBitHolder holder) throws IllegalArgumentException {
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException();
+    } else if (holder.isSet > 0) {
+      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+      if (holder.value != 0) {
+        BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+      } else {
+        BitVectorHelper.setValidityBit(valueBuffer, index, 0);
+      }
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
     }
+  }
 
-    final int curSize = (int) newAllocationSize;
-    final ArrowBuf newBuf = allocator.buffer(curSize);
-    newBuf.setZero(0, newBuf.capacity());
-    newBuf.setBytes(0, data, 0, currentBufferCapacity);
-    data.release();
-    data = newBuf;
-    allocationSizeInBytes = curSize;
+  /**
+   * Set the element at the given index to the value set in data holder.
+   *
+   * @param index  position of element
+   * @param holder data holder for value of element
+   */
+  public void set(int index, BitHolder holder) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    if (holder.value != 0) {
+      BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+    } else {
+      BitVectorHelper.setValidityBit(valueBuffer, index, 0);
+    }
   }
 
   /**
-   * {@inheritDoc}
+   * Same as {@link #set(int, int)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index position of element
+   * @param value value of element
    */
-  @Override
-  public void zeroVector() {
-    data.setZero(0, data.capacity());
+  public void setSafe(int index, int value) {
+    handleSafe(index);
+    set(index, value);
   }
 
-  public void copyFrom(int inIndex, int outIndex, BitVector from) {
-    this.mutator.set(outIndex, from.accessor.get(inIndex));
+  /**
+   * Same as {@link #set(int, NullableBitHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index  position of element
+   * @param holder nullable data holder for value of element
+   */
+  public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException {
+    handleSafe(index);
+    set(index, holder);
   }
 
-  public void copyFromSafe(int inIndex, int outIndex, BitVector from) {
-    if (outIndex >= this.getValueCapacity()) {
-      reAlloc();
-    }
-    copyFrom(inIndex, outIndex, from);
+  /**
+   * Same as {@link #set(int, BitHolder)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index  position of element
+   * @param holder data holder for value of element
+   */
+  public void setSafe(int index, BitHolder holder) {
+    handleSafe(index);
+    set(index, holder);
   }
 
-  @Override
-  public Mutator getMutator() {
-    return mutator;
+  /**
+   * Set the element at the given index to null.
+   *
+   * @param index position of element
+   */
+  public void setNull(int index) {
+    handleSafe(index);
+      /* not really needed to set the bit to 0 as long as
+       * the buffer always starts from 0.
+       */
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
   }
 
-  @Override
-  public Accessor getAccessor() {
-    return accessor;
+  /**
+   * Store the given value at a particular position in the vector. isSet indicates
+   * whether the value is NULL or not.
+   *
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void set(int index, int isSet, int value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
   }
 
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new TransferImpl(name, allocator);
+  /**
+   * Same as {@link #set(int, int, int)} except that it handles the case
+   * when index is greater than or equal to current value capacity of the
+   * vector.
+   *
+   * @param index position of the new value
+   * @param isSet 0 for NULL value, 1 otherwise
+   * @param value element value
+   */
+  public void setSafe(int index, int isSet, int value) {
+    handleSafe(index);
+    set(index, isSet, value);
   }
 
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   *
+   * @param ref       name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
     return new TransferImpl(ref, allocator);
   }
 
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   *
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
     return new TransferImpl((BitVector) to);
   }
 
-
-  public void transferTo(BitVector target) {
-    target.clear();
-    target.data = data.transferOwnership(target.allocator).buffer;
-    target.valueCount = valueCount;
-    clear();
-  }
-
-  public void splitAndTransferTo(int startIndex, int length, BitVector target) {
-    assert startIndex + length <= valueCount;
-    int firstByteSource = getByteIndex(startIndex);
-    int lastByteSource = getByteIndex(valueCount - 1);
-    int byteSizeTarget = getSizeFromCount(length);
-    int offset = startIndex % 8;
-
-    if (length > 0) {
-      if (offset == 0) {
-        target.clear();
-        // slice
-        if (target.data != null) {
-          target.data.release();
-        }
-        target.data = data.slice(firstByteSource, byteSizeTarget);
-        target.data.retain(1);
-      }
-      else {
-        // Copy data
-        // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
-        // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
-
-        target.clear();
-        target.allocateNew(byteSizeTarget * 8);
-
-        // TODO maybe do this one word at a time, rather than byte?
-
-        for (int i = 0; i < byteSizeTarget - 1; i++) {
-          byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + i, offset);
-          byte b2 = getBitsFromNextByte(this.data, firstByteSource + i + 1, offset);
-
-          target.data.setByte(i, (b1 + b2));
-        }
-
-        /* Copying the last piece is done in the following manner:
-         * if the source vector has 1 or more bytes remaining, we copy
-         * the last piece as a byte formed by shifting data
-         * from the current byte and the next byte.
-         *
-         * if the source vector has no more bytes remaining
-         * (we are at the last byte), we copy the last piece as a byte
-         * by shifting data from the current byte.
-         */
-        if((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-          byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + byteSizeTarget - 1, offset);
-          byte b2 = getBitsFromNextByte(this.data, firstByteSource + byteSizeTarget, offset);
-
-          target.data.setByte(byteSizeTarget - 1, b1 + b2);
-        }
-        else {
-          byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + byteSizeTarget - 1, offset);
-
-          target.data.setByte(byteSizeTarget - 1, b1);
-        }
-      }
-    }
-    target.getMutator().setValueCount(length);
-  }
-
-  private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) {
-    return (byte)((data.getByte(index) & 0xFF) >>> offset);
-  }
-
-  private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) {
-    return (byte)((data.getByte(index) << (8 - offset)));
-  }
-
   private class TransferImpl implements TransferPair {
     BitVector to;
 
-    public TransferImpl(String name, BufferAllocator allocator) {
-      this.to = new BitVector(name, allocator);
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new BitVector(ref, field.getFieldType(), allocator);
     }
 
     public TransferImpl(BitVector to) {
@@ -380,269 +507,4 @@ public void copyValueSafe(int fromIndex, int toIndex) {
       to.copyFromSafe(fromIndex, toIndex, BitVector.this);
     }
   }
-
-  private void decrementAllocationMonitor() {
-    if (allocationMonitor > 0) {
-      allocationMonitor = 0;
-    }
-    --allocationMonitor;
-  }
-
-  private void incrementAllocationMonitor() {
-    ++allocationMonitor;
-  }
-
-  public class Accessor extends BaseAccessor {
-
-    /**
-     * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the bit was not set.
-     *
-     * @param index position of the bit in the vector
-     * @return 1 if set, otherwise 0
-     */
-    public final int get(int index) {
-      int byteIndex = index >> 3;
-      byte b = data.getByte(byteIndex);
-      int bitIndex = index & 7;
-      return Long.bitCount(b & (1L << bitIndex));
-    }
-
-    @Override
-    public boolean isNull(int index) {
-      return false;
-    }
-
-    @Override
-    public final Boolean getObject(int index) {
-      return new Boolean(get(index) != 0);
-    }
-
-    @Override
-    public final int getValueCount() {
-      return valueCount;
-    }
-
-    public final void get(int index, BitHolder holder) {
-      holder.value = get(index);
-    }
-
-    public final void get(int index, NullableBitHolder holder) {
-      holder.isSet = 1;
-      holder.value = get(index);
-    }
-
-    /**
-     * Get the number nulls, this correspond to the number of bits set to 0 in the vector
-     *
-     * @return the number of bits set to 0
-     */
-    @Override
-    public final int getNullCount() {
-      int count = 0;
-      int sizeInBytes = getSizeFromCount(valueCount);
-
-      for (int i = 0; i < sizeInBytes; ++i) {
-        byte byteValue = data.getByte(i);
-        // Java uses two's complement binary representation, hence 11111111_b which is -1 when converted to Int
-        // will have 32bits set to 1. Masking the MSB and then adding it back solves the issue.
-        count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7);
-      }
-      int nullCount = (sizeInBytes * 8) - count;
-      // if the valueCount is not a multiple of 8, the bits on the right were counted as null bits
-      int remainder = valueCount % 8;
-      nullCount -= remainder == 0 ? 0 : 8 - remainder;
-      return nullCount;
-    }
-  }
-
-  /**
-   * MutableBit implements a vector of bit-width values. Elements in the vector are accessed by position from the
-   * logical start of the vector. Values should be pushed onto the vector sequentially, but may be randomly accessed.
-   *
-   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
-   */
-  public class Mutator extends BaseMutator {
-
-    private Mutator() {
-    }
-
-    /**
-     * Set the bit at the given index to the specified value.
-     *
-     * @param index position of the bit to set
-     * @param value value to set (either 1 or 0)
-     */
-    public final void set(int index, int value) {
-      int byteIndex = byteIndex(index);
-      int bitIndex = bitIndex(index);
-      byte currentByte = data.getByte(byteIndex);
-      byte bitMask = (byte) (1L << bitIndex);
-      if (value != 0) {
-        currentByte |= bitMask;
-      } else {
-        currentByte -= (bitMask & currentByte);
-      }
-      data.setByte(byteIndex, currentByte);
-    }
-
-    /**
-     * Set the bit at the given index to 1.
-     *
-     * @param index position of the bit to set
-     */
-    public final void setToOne(int index) {
-      int byteIndex = byteIndex(index);
-      int bitIndex = bitIndex(index);
-      byte currentByte = data.getByte(byteIndex);
-      byte bitMask = (byte) (1L << bitIndex);
-      currentByte |= bitMask;
-      data.setByte(byteIndex, currentByte);
-    }
-
-    /**
-     * set count bits to 1 in data starting at firstBitIndex
-     *
-     * @param firstBitIndex the index of the first bit to set
-     * @param count         the number of bits to set
-     */
-    public void setRangeToOne(int firstBitIndex, int count) {
-      int starByteIndex = byteIndex(firstBitIndex);
-      final int lastBitIndex = firstBitIndex + count;
-      final int endByteIndex = byteIndex(lastBitIndex);
-      final int startByteBitIndex = bitIndex(firstBitIndex);
-      final int endBytebitIndex = bitIndex(lastBitIndex);
-      if (count < 8 && starByteIndex == endByteIndex) {
-        // handles the case where we don't have a first and a last byte
-        byte bitMask = 0;
-        for (int i = startByteBitIndex; i < endBytebitIndex; ++i) {
-          bitMask |= (byte) (1L << i);
-        }
-        byte currentByte = data.getByte(starByteIndex);
-        currentByte |= bitMask;
-        data.setByte(starByteIndex, currentByte);
-      } else {
-        // fill in first byte (if it's not full)
-        if (startByteBitIndex != 0) {
-          byte currentByte = data.getByte(starByteIndex);
-          final byte bitMask = (byte) (0xFFL << startByteBitIndex);
-          currentByte |= bitMask;
-          data.setByte(starByteIndex, currentByte);
-          ++starByteIndex;
-        }
-
-        // fill in one full byte at a time
-        for (int i = starByteIndex; i < endByteIndex; i++) {
-          data.setByte(i, 0xFF);
-        }
-
-        // fill in the last byte (if it's not full)
-        if (endBytebitIndex != 0) {
-          final int byteIndex = byteIndex(lastBitIndex - endBytebitIndex);
-          byte currentByte = data.getByte(byteIndex);
-          final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7));
-          currentByte |= bitMask;
-          data.setByte(byteIndex, currentByte);
-        }
-
-      }
-    }
-
-    /**
-     * @param absoluteBitIndex the index of the bit in the buffer
-     * @return the index of the byte containing that bit
-     */
-    private int byteIndex(int absoluteBitIndex) {
-      return absoluteBitIndex >> 3;
-    }
-
-    /**
-     * @param absoluteBitIndex the index of the bit in the buffer
-     * @return the index of the bit inside the byte
-     */
-    private int bitIndex(int absoluteBitIndex) {
-      return absoluteBitIndex & 7;
-    }
-
-    public final void set(int index, BitHolder holder) {
-      set(index, holder.value);
-    }
-
-    final void set(int index, NullableBitHolder holder) {
-      set(index, holder.value);
-    }
-
-    public void setSafe(int index, int value) {
-      while (index >= getValueCapacity()) {
-        reAlloc();
-      }
-      set(index, value);
-    }
-
-    public void setSafeToOne(int index) {
-      while (index >= getValueCapacity()) {
-        reAlloc();
-      }
-      setToOne(index);
-    }
-
-    public void setSafe(int index, BitHolder holder) {
-      while (index >= getValueCapacity()) {
-        reAlloc();
-      }
-      set(index, holder.value);
-    }
-
-    public void setSafe(int index, NullableBitHolder holder) {
-      while (index >= getValueCapacity()) {
-        reAlloc();
-      }
-      set(index, holder.value);
-    }
-
-    @Override
-    public final void setValueCount(int valueCount) {
-      int currentValueCapacity = getValueCapacity();
-      BitVector.this.valueCount = valueCount;
-      int idx = getSizeFromCount(valueCount);
-      while (valueCount > getValueCapacity()) {
-        reAlloc();
-      }
-      if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
-        incrementAllocationMonitor();
-      } else if (allocationMonitor > 0) {
-        allocationMonitor = 0;
-      }
-      VectorTrimmer.trim(data, idx);
-    }
-
-    @Override
-    public final void generateTestData(int values) {
-      boolean even = true;
-      for (int i = 0; i < values; i++, even = !even) {
-        if (even) {
-          set(i, 1);
-        }
-      }
-      setValueCount(values);
-    }
-
-    public void generateTestDataAlt(int size) {
-      setValueCount(size);
-      boolean even = true;
-      final int valueCount = getAccessor().getValueCount();
-      for (int i = 0; i < valueCount; i++, even = !even) {
-        if (even) {
-          set(i, (byte) 1);
-        } else {
-          set(i, (byte) 0);
-        }
-      }
-    }
-  }
-
-  @Override
-  public void clear() {
-    this.valueCount = 0;
-    super.clear();
-  }
-}
+}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java
index bb14d1ac0d41c..ed8956c1eaa3d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java
@@ -30,35 +30,35 @@
 import org.slf4j.Logger;
 
 /**
- * NullableDateDayVector implements a fixed width (4 bytes) vector of
+ * DateDayVector implements a fixed width (4 bytes) vector of
  * date values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableDateDayVector extends BaseNullableFixedWidthVector {
+public class DateDayVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 4;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableDateDayVector. This doesn't allocate any memory for
+   * Instantiate a DateDayVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableDateDayVector(String name, BufferAllocator allocator) {
+  public DateDayVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableDateDayVector. This doesn't allocate any memory for
+   * Instantiate a DateDayVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public DateDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new DateDayReaderImpl(NullableDateDayVector.this);
+    reader = new DateDayReaderImpl(DateDayVector.this);
   }
 
   /**
@@ -138,21 +138,21 @@ public Integer getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, DateDayVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
     valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that
+   * Same as {@link #copyFrom(int, int, DateDayVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, DateDayVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -331,22 +331,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableDateDayVector) to);
+    return new TransferImpl((DateDayVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableDateDayVector to;
+    DateDayVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableDateDayVector(ref, field.getFieldType(), allocator);
+      to = new DateDayVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableDateDayVector to) {
+    public TransferImpl(DateDayVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableDateDayVector getTo() {
+    public DateDayVector getTo() {
       return to;
     }
 
@@ -362,7 +362,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this);
+      to.copyFromSafe(fromIndex, toIndex, DateDayVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java
index 97205c318317c..f21b58f85fb44 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java
@@ -32,35 +32,35 @@
 import org.slf4j.Logger;
 
 /**
- * NullableDateMilliVector implements a fixed width vector (8 bytes) of
+ * DateMilliVector implements a fixed width vector (8 bytes) of
  * date values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableDateMilliVector extends BaseNullableFixedWidthVector {
+public class DateMilliVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 8;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for
+   * Instantiate a DateMilliVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableDateMilliVector(String name, BufferAllocator allocator) {
+  public DateMilliVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for
+   * Instantiate a DateMilliVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public DateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new DateMilliReaderImpl(NullableDateMilliVector.this);
+    reader = new DateMilliReaderImpl(DateMilliVector.this);
   }
 
   /**
@@ -143,21 +143,21 @@ public LocalDateTime getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, DateMilliVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
     valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that
+   * Same as {@link #copyFrom(int, int, DateMilliVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, DateMilliVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -336,22 +336,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableDateMilliVector) to);
+    return new TransferImpl((DateMilliVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableDateMilliVector to;
+    DateMilliVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableDateMilliVector(ref, field.getFieldType(), allocator);
+      to = new DateMilliVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableDateMilliVector to) {
+    public TransferImpl(DateMilliVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableDateMilliVector getTo() {
+    public DateMilliVector getTo() {
       return to;
     }
 
@@ -367,7 +367,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this);
+      to.copyFromSafe(fromIndex, toIndex, DateMilliVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
similarity index 91%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index 5d2782a4c8460..71a74cb9f3338 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -32,11 +32,11 @@
 import java.math.BigDecimal;
 
 /**
- * NullableDecimalVector implements a fixed width vector (16 bytes) of
+ * DecimalVector implements a fixed width vector (16 bytes) of
  * decimal values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableDecimalVector extends BaseNullableFixedWidthVector {
+public class DecimalVector extends BaseFixedWidthVector {
   public static final byte TYPE_WIDTH = 16;
   private final FieldReader reader;
 
@@ -44,28 +44,28 @@ public class NullableDecimalVector extends BaseNullableFixedWidthVector {
   private final int scale;
 
   /**
-   * Instantiate a NullableDecimalVector. This doesn't allocate any memory for
+   * Instantiate a DecimalVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableDecimalVector(String name, BufferAllocator allocator,
+  public DecimalVector(String name, BufferAllocator allocator,
                                int precision, int scale) {
     this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)),
             allocator);
   }
 
   /**
-   * Instantiate a NullableDecimalVector. This doesn't allocate any memory for
+   * Instantiate a DecimalVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public DecimalVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
     org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal) fieldType.getType();
-    reader = new DecimalReaderImpl(NullableDecimalVector.this);
+    reader = new DecimalReaderImpl(DecimalVector.this);
     this.precision = arrowType.getPrecision();
     this.scale = arrowType.getScale();
   }
@@ -150,21 +150,21 @@ public BigDecimal getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, DecimalVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer,
             thisIndex * TYPE_WIDTH, TYPE_WIDTH);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that
+   * Same as {@link #copyFrom(int, int, DecimalVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, DecimalVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -385,23 +385,23 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableDecimalVector) to);
+    return new TransferImpl((DecimalVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableDecimalVector to;
+    DecimalVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision,
-              NullableDecimalVector.this.scale);
+      to = new DecimalVector(ref, allocator, DecimalVector.this.precision,
+              DecimalVector.this.scale);
     }
 
-    public TransferImpl(NullableDecimalVector to) {
+    public TransferImpl(DecimalVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableDecimalVector getTo() {
+    public DecimalVector getTo() {
       return to;
     }
 
@@ -417,7 +417,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this);
+      to.copyFromSafe(fromIndex, toIndex, DecimalVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java
index 5d7128d2bdfa3..dc78bfde2a228 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java
@@ -29,35 +29,35 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableFloat4Vector implements a fixed width vector (4 bytes) of
+ * Float4Vector implements a fixed width vector (4 bytes) of
  * float values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableFloat4Vector extends BaseNullableFixedWidthVector {
+public class Float4Vector extends BaseFixedWidthVector {
   public static final byte TYPE_WIDTH = 4;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for
+   * Instantiate a Float4Vector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableFloat4Vector(String name, BufferAllocator allocator) {
+  public Float4Vector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for
+   * Instantiate a Float4Vector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public Float4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new Float4ReaderImpl(NullableFloat4Vector.this);
+    reader = new Float4ReaderImpl(Float4Vector.this);
   }
 
   /**
@@ -137,21 +137,21 @@ public Float getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) {
+  public void copyFrom(int fromIndex, int thisIndex, Float4Vector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final float value = from.valueBuffer.getFloat(fromIndex * TYPE_WIDTH);
     valueBuffer.setFloat(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that
+   * Same as {@link #copyFrom(int, int, Float4Vector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, Float4Vector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -330,22 +330,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableFloat4Vector) to);
+    return new TransferImpl((Float4Vector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableFloat4Vector to;
+    Float4Vector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableFloat4Vector(ref, field.getFieldType(), allocator);
+      to = new Float4Vector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableFloat4Vector to) {
+    public TransferImpl(Float4Vector to) {
       this.to = to;
     }
 
     @Override
-    public NullableFloat4Vector getTo() {
+    public Float4Vector getTo() {
       return to;
     }
 
@@ -361,7 +361,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, Float4Vector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java
index 7aa546150b319..1b410b868795d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java
@@ -29,35 +29,35 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableFloat8Vector implements a fixed width vector (8 bytes) of
+ * Float8Vector implements a fixed width vector (8 bytes) of
  * double values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableFloat8Vector extends BaseNullableFixedWidthVector {
+public class Float8Vector extends BaseFixedWidthVector {
   public static final byte TYPE_WIDTH = 8;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for
+   * Instantiate a Float8Vector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableFloat8Vector(String name, BufferAllocator allocator) {
+  public Float8Vector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for
+   * Instantiate a Float8Vector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public Float8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new Float8ReaderImpl(NullableFloat8Vector.this);
+    reader = new Float8ReaderImpl(Float8Vector.this);
   }
 
   /**
@@ -137,21 +137,21 @@ public Double getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) {
+  public void copyFrom(int fromIndex, int thisIndex, Float8Vector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final double value = from.valueBuffer.getDouble(fromIndex * TYPE_WIDTH);
     valueBuffer.setDouble(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that
+   * Same as {@link #copyFrom(int, int, Float8Vector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, Float8Vector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -330,22 +330,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableFloat8Vector) to);
+    return new TransferImpl((Float8Vector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableFloat8Vector to;
+    Float8Vector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableFloat8Vector(ref, field.getFieldType(), allocator);
+      to = new Float8Vector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableFloat8Vector to) {
+    public TransferImpl(Float8Vector to) {
       this.to = to;
     }
 
     @Override
-    public NullableFloat8Vector getTo() {
+    public Float8Vector getTo() {
       return to;
     }
 
@@ -361,7 +361,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, Float8Vector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java
index 806beb5f8fedc..89e2a02f6ac2d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java
@@ -30,62 +30,62 @@
 public class GenerateSampleData {
 
   public static void generateTestData(final ValueVector vector, final int valueCount) {
-    if (vector instanceof NullableIntVector) {
-      writeIntData((NullableIntVector) vector, valueCount);
-    } else if (vector instanceof NullableDecimalVector) {
-      writeDecimalData((NullableDecimalVector) vector, valueCount);
-    } else if (vector instanceof NullableBitVector) {
-      writeBooleanData((NullableBitVector) vector, valueCount);
-    } else if (vector instanceof NullableVarCharVector) {
-      writeVarCharData((NullableVarCharVector) vector, valueCount);
-    } else if (vector instanceof NullableVarBinaryVector) {
-      writeVarBinaryData((NullableVarBinaryVector) vector, valueCount);
-    } else if (vector instanceof NullableBigIntVector) {
-      writeBigIntData((NullableBigIntVector) vector, valueCount);
-    } else if (vector instanceof NullableFloat4Vector) {
-      writeFloatData((NullableFloat4Vector) vector, valueCount);
-    } else if (vector instanceof NullableFloat8Vector) {
-      writeDoubleData((NullableFloat8Vector) vector, valueCount);
-    } else if (vector instanceof NullableDateDayVector) {
-      writeDateDayData((NullableDateDayVector) vector, valueCount);
-    } else if (vector instanceof NullableDateMilliVector) {
-      writeDateMilliData((NullableDateMilliVector) vector, valueCount);
-    } else if (vector instanceof NullableIntervalDayVector) {
-      writeIntervalDayData((NullableIntervalDayVector) vector, valueCount);
-    } else if (vector instanceof NullableIntervalYearVector) {
-      writeIntervalYearData((NullableIntervalYearVector) vector, valueCount);
-    } else if (vector instanceof NullableSmallIntVector) {
-      writeSmallIntData((NullableSmallIntVector) vector, valueCount);
-    } else if (vector instanceof NullableTinyIntVector) {
-      writeTinyIntData((NullableTinyIntVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeMicroVector) {
-      writeTimeMicroData((NullableTimeMicroVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeMilliVector) {
-      writeTimeMilliData((NullableTimeMilliVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeNanoVector) {
-      writeTimeNanoData((NullableTimeNanoVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeSecVector) {
-      writeTimeSecData((NullableTimeSecVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeStampSecVector) {
-      writeTimeStampData((NullableTimeStampSecVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeStampMicroVector) {
-      writeTimeStampData((NullableTimeStampMicroVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeStampMilliVector) {
-      writeTimeStampData((NullableTimeStampMilliVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeStampNanoVector) {
-      writeTimeStampData((NullableTimeStampNanoVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeStampSecTZVector) {
-      writeTimeStampData((NullableTimeStampSecTZVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeStampMicroTZVector) {
-      writeTimeStampData((NullableTimeStampMicroTZVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeStampMilliTZVector) {
-      writeTimeStampData((NullableTimeStampMilliTZVector) vector, valueCount);
-    } else if (vector instanceof NullableTimeStampNanoTZVector) {
-      writeTimeStampData((NullableTimeStampNanoTZVector) vector, valueCount);
+    if (vector instanceof IntVector) {
+      writeIntData((IntVector) vector, valueCount);
+    } else if (vector instanceof DecimalVector) {
+      writeDecimalData((DecimalVector) vector, valueCount);
+    } else if (vector instanceof BitVector) {
+      writeBooleanData((BitVector) vector, valueCount);
+    } else if (vector instanceof VarCharVector) {
+      writeVarCharData((VarCharVector) vector, valueCount);
+    } else if (vector instanceof VarBinaryVector) {
+      writeVarBinaryData((VarBinaryVector) vector, valueCount);
+    } else if (vector instanceof BigIntVector) {
+      writeBigIntData((BigIntVector) vector, valueCount);
+    } else if (vector instanceof Float4Vector) {
+      writeFloatData((Float4Vector) vector, valueCount);
+    } else if (vector instanceof Float8Vector) {
+      writeDoubleData((Float8Vector) vector, valueCount);
+    } else if (vector instanceof DateDayVector) {
+      writeDateDayData((DateDayVector) vector, valueCount);
+    } else if (vector instanceof DateMilliVector) {
+      writeDateMilliData((DateMilliVector) vector, valueCount);
+    } else if (vector instanceof IntervalDayVector) {
+      writeIntervalDayData((IntervalDayVector) vector, valueCount);
+    } else if (vector instanceof IntervalYearVector) {
+      writeIntervalYearData((IntervalYearVector) vector, valueCount);
+    } else if (vector instanceof SmallIntVector) {
+      writeSmallIntData((SmallIntVector) vector, valueCount);
+    } else if (vector instanceof TinyIntVector) {
+      writeTinyIntData((TinyIntVector) vector, valueCount);
+    } else if (vector instanceof TimeMicroVector) {
+      writeTimeMicroData((TimeMicroVector) vector, valueCount);
+    } else if (vector instanceof TimeMilliVector) {
+      writeTimeMilliData((TimeMilliVector) vector, valueCount);
+    } else if (vector instanceof TimeNanoVector) {
+      writeTimeNanoData((TimeNanoVector) vector, valueCount);
+    } else if (vector instanceof TimeSecVector) {
+      writeTimeSecData((TimeSecVector) vector, valueCount);
+    } else if (vector instanceof TimeStampSecVector) {
+      writeTimeStampData((TimeStampSecVector) vector, valueCount);
+    } else if (vector instanceof TimeStampMicroVector) {
+      writeTimeStampData((TimeStampMicroVector) vector, valueCount);
+    } else if (vector instanceof TimeStampMilliVector) {
+      writeTimeStampData((TimeStampMilliVector) vector, valueCount);
+    } else if (vector instanceof TimeStampNanoVector) {
+      writeTimeStampData((TimeStampNanoVector) vector, valueCount);
+    } else if (vector instanceof TimeStampSecTZVector) {
+      writeTimeStampData((TimeStampSecTZVector) vector, valueCount);
+    } else if (vector instanceof TimeStampMicroTZVector) {
+      writeTimeStampData((TimeStampMicroTZVector) vector, valueCount);
+    } else if (vector instanceof TimeStampMilliTZVector) {
+      writeTimeStampData((TimeStampMilliTZVector) vector, valueCount);
+    } else if (vector instanceof TimeStampNanoTZVector) {
+      writeTimeStampData((TimeStampNanoTZVector) vector, valueCount);
     }
   }
 
-  private static void writeTimeStampData(NullableTimeStampVector vector, int valueCount) {
+  private static void writeTimeStampData(TimeStampVector vector, int valueCount) {
     final long even = 100000;
     final long odd = 200000;
     for (int i = 0; i < valueCount; i++) {
@@ -98,7 +98,7 @@ private static void writeTimeStampData(NullableTimeStampVector vector, int value
     vector.setValueCount(valueCount);
   }
 
-  private static void writeDecimalData(NullableDecimalVector vector, int valueCount) {
+  private static void writeDecimalData(DecimalVector vector, int valueCount) {
     final BigDecimal even = new BigDecimal(0.0543278923);
     final BigDecimal odd = new BigDecimal(2.0543278923);
     for (int i = 0; i < valueCount; i++) {
@@ -111,7 +111,7 @@ private static void writeDecimalData(NullableDecimalVector vector, int valueCoun
     vector.setValueCount(valueCount);
   }
 
-  private static void writeIntData(NullableIntVector vector, int valueCount) {
+  private static void writeIntData(IntVector vector, int valueCount) {
     final int even = 1000;
     final int odd = 2000;
     for (int i = 0; i < valueCount; i++) {
@@ -124,7 +124,7 @@ private static void writeIntData(NullableIntVector vector, int valueCount) {
     vector.setValueCount(valueCount);
   }
 
-  private static void writeBooleanData(NullableBitVector vector, int valueCount) {
+  private static void writeBooleanData(BitVector vector, int valueCount) {
     final int even = 0;
     final int odd = 1;
     for (int i = 0; i < valueCount; i++) {
@@ -137,7 +137,7 @@ private static void writeBooleanData(NullableBitVector vector, int valueCount) {
     vector.setValueCount(valueCount);
   }
 
-  private static void writeIntervalYearData(NullableIntervalYearVector vector, int valueCount) {
+  private static void writeIntervalYearData(IntervalYearVector vector, int valueCount) {
     final int even = 1;
     final int odd = 2;
     for (int i = 0; i < valueCount; i++) {
@@ -150,7 +150,7 @@ private static void writeIntervalYearData(NullableIntervalYearVector vector, int
     vector.setValueCount(valueCount);
   }
 
-  private static void writeIntervalDayData(NullableIntervalDayVector vector, int valueCount) {
+  private static void writeIntervalDayData(IntervalDayVector vector, int valueCount) {
     for (int i = 0; i < valueCount; i++) {
       if (i % 2 == 0) {
         vector.setSafe(i, 1, 50);
@@ -161,7 +161,7 @@ private static void writeIntervalDayData(NullableIntervalDayVector vector, int v
     vector.setValueCount(valueCount);
   }
 
-  private static void writeTimeSecData(NullableTimeSecVector vector, int valueCount) {
+  private static void writeTimeSecData(TimeSecVector vector, int valueCount) {
     final int even = 500;
     final int odd = 900;
     for (int i = 0; i < valueCount; i++) {
@@ -174,7 +174,7 @@ private static void writeTimeSecData(NullableTimeSecVector vector, int valueCoun
     vector.setValueCount(valueCount);
   }
 
-  private static void writeTimeMilliData(NullableTimeMilliVector vector, int valueCount) {
+  private static void writeTimeMilliData(TimeMilliVector vector, int valueCount) {
     final int even = 1000;
     final int odd = 2000;
     for (int i = 0; i < valueCount; i++) {
@@ -187,7 +187,7 @@ private static void writeTimeMilliData(NullableTimeMilliVector vector, int value
     vector.setValueCount(valueCount);
   }
 
-  private static void writeTimeMicroData(NullableTimeMicroVector vector, int valueCount) {
+  private static void writeTimeMicroData(TimeMicroVector vector, int valueCount) {
     final long even = 1000000000;
     final long odd = 2000000000;
     for (int i = 0; i < valueCount; i++) {
@@ -201,7 +201,7 @@ private static void writeTimeMicroData(NullableTimeMicroVector vector, int value
 
   }
 
-  private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCount) {
+  private static void writeTimeNanoData(TimeNanoVector vector, int valueCount) {
     final long even = 1000000000;
     final long odd = 2000000000;
     for (int i = 0; i < valueCount; i++) {
@@ -214,7 +214,7 @@ private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCo
     vector.setValueCount(valueCount);
   }
 
-  private static void writeDateDayData(NullableDateDayVector vector, int valueCount) {
+  private static void writeDateDayData(DateDayVector vector, int valueCount) {
     final int even = 1000;
     final int odd = 2000;
     for (int i = 0; i < valueCount; i++) {
@@ -227,7 +227,7 @@ private static void writeDateDayData(NullableDateDayVector vector, int valueCoun
     vector.setValueCount(valueCount);
   }
 
-  private static void writeDateMilliData(NullableDateMilliVector vector, int valueCount) {
+  private static void writeDateMilliData(DateMilliVector vector, int valueCount) {
     final long even = 1000000000;
     final long odd = 2000000000;
     for (int i = 0; i < valueCount; i++) {
@@ -240,7 +240,7 @@ private static void writeDateMilliData(NullableDateMilliVector vector, int value
     vector.setValueCount(valueCount);
   }
 
-  private static void writeSmallIntData(NullableSmallIntVector vector, int valueCount) {
+  private static void writeSmallIntData(SmallIntVector vector, int valueCount) {
     final short even = 10;
     final short odd = 20;
     for (int i = 0; i < valueCount; i++) {
@@ -253,7 +253,7 @@ private static void writeSmallIntData(NullableSmallIntVector vector, int valueCo
     vector.setValueCount(valueCount);
   }
 
-  private static void writeTinyIntData(NullableTinyIntVector vector, int valueCount) {
+  private static void writeTinyIntData(TinyIntVector vector, int valueCount) {
     final byte even = 1;
     final byte odd = 2;
     for (int i = 0; i < valueCount; i++) {
@@ -266,7 +266,7 @@ private static void writeTinyIntData(NullableTinyIntVector vector, int valueCoun
     vector.setValueCount(valueCount);
   }
 
-  private static void writeBigIntData(NullableBigIntVector vector, int valueCount) {
+  private static void writeBigIntData(BigIntVector vector, int valueCount) {
     final long even = 1000000000;
     final long odd = 2000000000;
     for (int i = 0; i < valueCount; i++) {
@@ -279,7 +279,7 @@ private static void writeBigIntData(NullableBigIntVector vector, int valueCount)
     vector.setValueCount(valueCount);
   }
 
-  private static void writeFloatData(NullableFloat4Vector vector, int valueCount) {
+  private static void writeFloatData(Float4Vector vector, int valueCount) {
     final float even = 20.3f;
     final float odd = 40.2f;
     for (int i = 0; i < valueCount; i++) {
@@ -292,7 +292,7 @@ private static void writeFloatData(NullableFloat4Vector vector, int valueCount)
     vector.setValueCount(valueCount);
   }
 
-  private static void writeDoubleData(NullableFloat8Vector vector, int valueCount) {
+  private static void writeDoubleData(Float8Vector vector, int valueCount) {
     final double even = 20.2373;
     final double odd = 40.2378;
     for (int i = 0; i < valueCount; i++) {
@@ -305,7 +305,7 @@ private static void writeDoubleData(NullableFloat8Vector vector, int valueCount)
     vector.setValueCount(valueCount);
   }
 
-  private static void writeVarBinaryData(NullableVarBinaryVector vector, int valueCount) {
+  private static void writeVarBinaryData(VarBinaryVector vector, int valueCount) {
     Charset utf8Charset = Charset.forName("UTF-8");
     final byte[] even = "AAAAA1".getBytes(utf8Charset);
     final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset);
@@ -319,7 +319,7 @@ private static void writeVarBinaryData(NullableVarBinaryVector vector, int value
     vector.setValueCount(valueCount);
   }
 
-  private static void writeVarCharData(NullableVarCharVector vector, int valueCount) {
+  private static void writeVarCharData(VarCharVector vector, int valueCount) {
     Charset utf8Charset = Charset.forName("UTF-8");
     final byte[] even = "AAAAA1".getBytes(utf8Charset);
     final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java
similarity index 91%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/IntVector.java
index 7c6f29ad7c559..2364310e6d5fc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java
@@ -29,37 +29,37 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableIntVector implements a fixed width (4 bytes) vector of
+ * IntVector implements a fixed width (4 bytes) vector of
  * integer values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableIntVector extends BaseNullableFixedWidthVector {
+public class IntVector extends BaseFixedWidthVector {
   public static final byte TYPE_WIDTH = 4;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableIntVector. This doesn't allocate any memory for
+   * Instantiate a IntVector. This doesn't allocate any memory for
    * the data in vector.
    *
    * @param name      name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableIntVector(String name, BufferAllocator allocator) {
+  public IntVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableIntVector. This doesn't allocate any memory for
+   * Instantiate a IntVector. This doesn't allocate any memory for
    * the data in vector.
    *
    * @param name      name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public IntVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new IntReaderImpl(NullableIntVector.this);
+    reader = new IntReaderImpl(IntVector.this);
   }
 
   /**
@@ -142,14 +142,14 @@ public Integer getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from      source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, IntVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
     valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableIntVector)} except that
+   * Same as {@link #copyFrom(int, int, IntVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    *
@@ -157,7 +157,7 @@ public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) {
    * @param thisIndex position to copy to in this vector
    * @param from      source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, IntVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -340,22 +340,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableIntVector) to);
+    return new TransferImpl((IntVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableIntVector to;
+    IntVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableIntVector(ref, field.getFieldType(), allocator);
+      to = new IntVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableIntVector to) {
+    public TransferImpl(IntVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableIntVector getTo() {
+    public IntVector getTo() {
       return to;
     }
 
@@ -371,7 +371,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this);
+      to.copyFromSafe(fromIndex, toIndex, IntVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java
similarity index 91%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java
index 862f6429cb8cd..481a66f4e7b29 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java
@@ -30,37 +30,37 @@
 import org.joda.time.Period;
 
 /**
- * NullableIntervalDayVector implements a fixed width vector (8 bytes) of
+ * IntervalDayVector implements a fixed width vector (8 bytes) of
  * interval (days and milliseconds) values which could be null.
  * A validity buffer (bit vector) is maintained to track which elements in the
  * vector are null.
  */
-public class NullableIntervalDayVector extends BaseNullableFixedWidthVector {
+public class IntervalDayVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 8;
   private static final byte MILLISECOND_OFFSET = 4;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for
+   * Instantiate a IntervalDayVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableIntervalDayVector(String name, BufferAllocator allocator) {
+  public IntervalDayVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for
+   * Instantiate a IntervalDayVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public IntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this);
+    reader = new IntervalDayReaderImpl(IntervalDayVector.this);
   }
 
   /**
@@ -185,21 +185,21 @@ private StringBuilder getAsStringBuilderHelper(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, IntervalDayVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer,
               thisIndex * TYPE_WIDTH, TYPE_WIDTH);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that
+   * Same as {@link #copyFrom(int, int, IntervalDayVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, IntervalDayVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -388,22 +388,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableIntervalDayVector) to);
+    return new TransferImpl((IntervalDayVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableIntervalDayVector to;
+    IntervalDayVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator);
+      to = new IntervalDayVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableIntervalDayVector to) {
+    public TransferImpl(IntervalDayVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableIntervalDayVector getTo() {
+    public IntervalDayVector getTo() {
       return to;
     }
 
@@ -419,7 +419,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this);
+      to.copyFromSafe(fromIndex, toIndex, IntervalDayVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java
index 3fba5bfeeac00..2aa728f3d6337 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java
@@ -29,35 +29,35 @@
 import org.joda.time.Period;
 
 /**
- * NullableIntervalYearVector implements a fixed width (4 bytes) vector of
+ * IntervalYearVector implements a fixed width (4 bytes) vector of
  * interval (years and months) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableIntervalYearVector extends BaseNullableFixedWidthVector {
+public class IntervalYearVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 4;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for
+   * Instantiate a IntervalYearVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableIntervalYearVector(String name, BufferAllocator allocator) {
+  public IntervalYearVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for
+   * Instantiate a IntervalYearVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public IntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this);
+    reader = new IntervalYearReaderImpl(IntervalYearVector.this);
   }
 
   /**
@@ -169,21 +169,21 @@ private StringBuilder getAsStringBuilderHelper(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, IntervalYearVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
     valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that
+   * Same as {@link #copyFrom(int, int, IntervalYearVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, IntervalYearVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -348,22 +348,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableIntervalYearVector) to);
+    return new TransferImpl((IntervalYearVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableIntervalYearVector to;
+    IntervalYearVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator);
+      to = new IntervalYearVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableIntervalYearVector to) {
+    public TransferImpl(IntervalYearVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableIntervalYearVector getTo() {
+    public IntervalYearVector getTo() {
       return to;
     }
 
@@ -379,7 +379,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this);
+      to.copyFromSafe(fromIndex, toIndex, IntervalYearVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
deleted file mode 100644
index 086331e13d452..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java
+++ /dev/null
@@ -1,513 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector;
-
-import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.impl.BitReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.BitHolder;
-import org.apache.arrow.vector.holders.NullableBitHolder;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.OversizedAllocationException;
-import org.apache.arrow.vector.util.TransferPair;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * NullableBitVector implements a fixed width (1 bit) vector of
- * boolean values which could be null. Each value in the vector corresponds
- * to a single bit in the underlying data stream backing the vector.
- */
-public class NullableBitVector extends BaseNullableFixedWidthVector {
-  private final FieldReader reader;
-
-  /**
-   * Instantiate a NullableBitVector. This doesn't allocate any memory for
-   * the data in vector.
-   *
-   * @param name      name of the vector
-   * @param allocator allocator for memory management.
-   */
-  public NullableBitVector(String name, BufferAllocator allocator) {
-    this(name, FieldType.nullable(Types.MinorType.BIT.getType()),
-            allocator);
-  }
-
-  /**
-   * Instantiate a NullableBitVector. This doesn't allocate any memory for
-   * the data in vector.
-   *
-   * @param name      name of the vector
-   * @param fieldType type of Field materialized by this vector
-   * @param allocator allocator for memory management.
-   */
-  public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) {
-    super(name, allocator, fieldType, (byte) 0);
-    reader = new BitReaderImpl(NullableBitVector.this);
-  }
-
-  /**
-   * Get a reader that supports reading values from this vector
-   *
-   * @return Field Reader for this vector
-   */
-  @Override
-  public FieldReader getReader() {
-    return reader;
-  }
-
-  /**
-   * Get minor type for this vector. The vector holds values belonging
-   * to a particular type.
-   *
-   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
-   */
-  @Override
-  public Types.MinorType getMinorType() {
-    return Types.MinorType.BIT;
-  }
-
-  /**
-   * Sets the desired value capacity for the vector. This function doesn't
-   * allocate any memory for the vector.
-   *
-   * @param valueCount desired number of elements in the vector
-   */
-  @Override
-  public void setInitialCapacity(int valueCount) {
-    final int size = getValidityBufferSizeFromCount(valueCount);
-    if (size > MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
-    }
-    valueAllocationSizeInBytes = size;
-    validityAllocationSizeInBytes = size;
-  }
-
-  /**
-   * Get the current value capacity for the vector
-   *
-   * @return number of elements that vector can hold.
-   */
-  @Override
-  public int getValueCapacity() {
-    return (int) (validityBuffer.capacity() * 8L);
-  }
-
-  /**
-   * Get the potential buffer size for a particular number of records.
-   *
-   * @param count desired number of elements in the vector
-   * @return estimated size of underlying buffers if the vector holds
-   * a given number of elements
-   */
-  @Override
-  public int getBufferSizeFor(final int count) {
-    if (count == 0) {
-      return 0;
-    }
-    return 2 * getValidityBufferSizeFromCount(count);
-  }
-
-  /**
-   * Get the size (number of bytes) of underlying buffers used by this
-   * vector
-   *
-   * @return size of underlying buffers.
-   */
-  @Override
-  public int getBufferSize() {
-    return getBufferSizeFor(valueCount);
-  }
-
-  /**
-   * Slice this vector at desired index and length and transfer the
-   * corresponding data to the target vector.
-   *
-   * @param startIndex start position of the split in source vector.
-   * @param length     length of the split.
-   * @param target     destination vector
-   */
-  public void splitAndTransferTo(int startIndex, int length,
-                                 BaseNullableFixedWidthVector target) {
-    compareTypes(target, "splitAndTransferTo");
-    target.clear();
-    target.validityBuffer = splitAndTransferBuffer(startIndex, length, target,
-            validityBuffer, target.validityBuffer);
-    target.valueBuffer = splitAndTransferBuffer(startIndex, length, target,
-            valueBuffer, target.valueBuffer);
-
-    target.setValueCount(length);
-  }
-
-  private ArrowBuf splitAndTransferBuffer(int startIndex, int length,
-                                          BaseNullableFixedWidthVector target,
-                                          ArrowBuf sourceBuffer, ArrowBuf destBuffer) {
-    assert startIndex + length <= valueCount;
-    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
-    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
-    int byteSizeTarget = getValidityBufferSizeFromCount(length);
-    int offset = startIndex % 8;
-
-    if (length > 0) {
-      if (offset == 0) {
-            /* slice */
-        if (destBuffer != null) {
-          destBuffer.release();
-        }
-        destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget);
-        destBuffer.retain(1);
-      } else {
-            /* Copy data
-             * When the first bit starts from the middle of a byte (offset != 0),
-             * copy data from src BitVector.
-             * Each byte in the target is composed by a part in i-th byte,
-             * another part in (i+1)-th byte.
-             */
-        destBuffer = allocator.buffer(byteSizeTarget);
-        destBuffer.readerIndex(0);
-        destBuffer.setZero(0, destBuffer.capacity());
-
-        for (int i = 0; i < byteSizeTarget - 1; i++) {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset);
-          byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset);
-
-          destBuffer.setByte(i, (b1 + b2));
-        }
-
-            /* Copying the last piece is done in the following manner:
-             * if the source vector has 1 or more bytes remaining, we copy
-             * the last piece as a byte formed by shifting data
-             * from the current byte and the next byte.
-             *
-             * if the source vector has no more bytes remaining
-             * (we are at the last byte), we copy the last piece as a byte
-             * by shifting data from the current byte.
-             */
-        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
-                  firstByteSource + byteSizeTarget - 1, offset);
-          byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer,
-                  firstByteSource + byteSizeTarget, offset);
-
-          destBuffer.setByte(byteSizeTarget - 1, b1 + b2);
-        } else {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer,
-                  firstByteSource + byteSizeTarget - 1, offset);
-          destBuffer.setByte(byteSizeTarget - 1, b1);
-        }
-      }
-    }
-
-    return destBuffer;
-  }
-
-
-  /******************************************************************
-   *                                                                *
-   *          vector value retrieval methods                        *
-   *                                                                *
-   ******************************************************************/
-
-  private int getBit(int index) {
-    final int byteIndex = index >> 3;
-    final byte b = valueBuffer.getByte(byteIndex);
-    final int bitIndex = index & 7;
-    return Long.bitCount(b & (1L << bitIndex));
-  }
-
-  /**
-   * Get the element at the given index from the vector.
-   *
-   * @param index position of element
-   * @return element at given index
-   */
-  public int get(int index) throws IllegalStateException {
-    if (isSet(index) == 0) {
-      throw new IllegalStateException("Value at index is null");
-    }
-    return getBit(index);
-  }
-
-  /**
-   * Get the element at the given index from the vector and
-   * sets the state in holder. If element at given index
-   * is null, holder.isSet will be zero.
-   *
-   * @param index position of element
-   */
-  public void get(int index, NullableBitHolder holder) {
-    if (isSet(index) == 0) {
-      holder.isSet = 0;
-      return;
-    }
-    holder.isSet = 1;
-    holder.value = getBit(index);
-  }
-
-  /**
-   * Same as {@link #get(int)}.
-   *
-   * @param index position of element
-   * @return element at given index
-   */
-  public Boolean getObject(int index) {
-    if (isSet(index) == 0) {
-      return null;
-    } else {
-      return new Boolean(getBit(index) != 0);
-    }
-  }
-
-  /**
-   * Copy a cell value from a particular index in source vector to a particular
-   * position in this vector
-   *
-   * @param fromIndex position to copy from in source vector
-   * @param thisIndex position to copy to in this vector
-   * @param from      source vector
-   */
-  public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) {
-    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
-    BitVectorHelper.setValidityBit(valueBuffer, thisIndex, from.getBit(fromIndex));
-  }
-
-  /**
-   * Same as {@link #copyFrom(int, int, NullableBitVector)} except that
-   * it handles the case when the capacity of the vector needs to be expanded
-   * before copy.
-   *
-   * @param fromIndex position to copy from in source vector
-   * @param thisIndex position to copy to in this vector
-   * @param from      source vector
-   */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) {
-    handleSafe(thisIndex);
-    copyFrom(fromIndex, thisIndex, from);
-  }
-
-
-  /******************************************************************
-   *                                                                *
-   *          vector value setter methods                           *
-   *                                                                *
-   ******************************************************************/
-
-
-  /**
-   * Set the element at the given index to the given value.
-   *
-   * @param index position of element
-   * @param value value of element
-   */
-  public void set(int index, int value) {
-    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-    if (value != 0) {
-      BitVectorHelper.setValidityBitToOne(valueBuffer, index);
-    } else {
-      BitVectorHelper.setValidityBit(valueBuffer, index, 0);
-    }
-  }
-
-  /**
-   * Set the element at the given index to the value set in data holder.
-   * If the value in holder is not indicated as set, element in the
-   * at the given index will be null.
-   *
-   * @param index  position of element
-   * @param holder nullable data holder for value of element
-   */
-  public void set(int index, NullableBitHolder holder) throws IllegalArgumentException {
-    if (holder.isSet < 0) {
-      throw new IllegalArgumentException();
-    } else if (holder.isSet > 0) {
-      BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-      if (holder.value != 0) {
-        BitVectorHelper.setValidityBitToOne(valueBuffer, index);
-      } else {
-        BitVectorHelper.setValidityBit(valueBuffer, index, 0);
-      }
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-    }
-  }
-
-  /**
-   * Set the element at the given index to the value set in data holder.
-   *
-   * @param index  position of element
-   * @param holder data holder for value of element
-   */
-  public void set(int index, BitHolder holder) {
-    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-    if (holder.value != 0) {
-      BitVectorHelper.setValidityBitToOne(valueBuffer, index);
-    } else {
-      BitVectorHelper.setValidityBit(valueBuffer, index, 0);
-    }
-  }
-
-  /**
-   * Same as {@link #set(int, int)} except that it handles the
-   * case when index is greater than or equal to existing
-   * value capacity {@link #getValueCapacity()}.
-   *
-   * @param index position of element
-   * @param value value of element
-   */
-  public void setSafe(int index, int value) {
-    handleSafe(index);
-    set(index, value);
-  }
-
-  /**
-   * Same as {@link #set(int, NullableBitHolder)} except that it handles the
-   * case when index is greater than or equal to existing
-   * value capacity {@link #getValueCapacity()}.
-   *
-   * @param index  position of element
-   * @param holder nullable data holder for value of element
-   */
-  public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException {
-    handleSafe(index);
-    set(index, holder);
-  }
-
-  /**
-   * Same as {@link #set(int, BitHolder)} except that it handles the
-   * case when index is greater than or equal to existing
-   * value capacity {@link #getValueCapacity()}.
-   *
-   * @param index  position of element
-   * @param holder data holder for value of element
-   */
-  public void setSafe(int index, BitHolder holder) {
-    handleSafe(index);
-    set(index, holder);
-  }
-
-  /**
-   * Set the element at the given index to null.
-   *
-   * @param index position of element
-   */
-  public void setNull(int index) {
-    handleSafe(index);
-      /* not really needed to set the bit to 0 as long as
-       * the buffer always starts from 0.
-       */
-    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-  }
-
-  /**
-   * Store the given value at a particular position in the vector. isSet indicates
-   * whether the value is NULL or not.
-   *
-   * @param index position of the new value
-   * @param isSet 0 for NULL value, 1 otherwise
-   * @param value element value
-   */
-  public void set(int index, int isSet, int value) {
-    if (isSet > 0) {
-      set(index, value);
-    } else {
-      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-    }
-  }
-
-  /**
-   * Same as {@link #set(int, int, int)} except that it handles the case
-   * when index is greater than or equal to current value capacity of the
-   * vector.
-   *
-   * @param index position of the new value
-   * @param isSet 0 for NULL value, 1 otherwise
-   * @param value element value
-   */
-  public void setSafe(int index, int isSet, int value) {
-    handleSafe(index);
-    set(index, isSet, value);
-  }
-
-
-  /******************************************************************
-   *                                                                *
-   *                      vector transfer                           *
-   *                                                                *
-   ******************************************************************/
-
-
-  /**
-   * Construct a TransferPair comprising of this and and a target vector of
-   * the same type.
-   *
-   * @param ref       name of the target vector
-   * @param allocator allocator for the target vector
-   * @return {@link TransferPair}
-   */
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new TransferImpl(ref, allocator);
-  }
-
-  /**
-   * Construct a TransferPair with a desired target vector of the same type.
-   *
-   * @param to target vector
-   * @return {@link TransferPair}
-   */
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableBitVector) to);
-  }
-
-  private class TransferImpl implements TransferPair {
-    NullableBitVector to;
-
-    public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableBitVector(ref, field.getFieldType(), allocator);
-    }
-
-    public TransferImpl(NullableBitVector to) {
-      this.to = to;
-    }
-
-    @Override
-    public NullableBitVector getTo() {
-      return to;
-    }
-
-    @Override
-    public void transfer() {
-      transferTo(to);
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      splitAndTransferTo(startIndex, length, to);
-    }
-
-    @Override
-    public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this);
-    }
-  }
-}
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
deleted file mode 100644
index b2455e9e42b4b..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector;
-
-public interface NullableVector extends ValueVector {
-
-  BitVector getValidityVector();
-
-  ValueVector getValuesVector();
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java
similarity index 91%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java
index d5c66b115acde..859e62dbb8bbb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java
@@ -29,35 +29,35 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableSmallIntVector implements a fixed width (2 bytes) vector of
+ * SmallIntVector implements a fixed width (2 bytes) vector of
  * short values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableSmallIntVector extends BaseNullableFixedWidthVector {
+public class SmallIntVector extends BaseFixedWidthVector {
   public static final byte TYPE_WIDTH = 2;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for
+   * Instantiate a SmallIntVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableSmallIntVector(String name, BufferAllocator allocator) {
+  public SmallIntVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for
+   * Instantiate a SmallIntVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public SmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new SmallIntReaderImpl(NullableSmallIntVector.this);
+    reader = new SmallIntReaderImpl(SmallIntVector.this);
   }
 
   /**
@@ -137,21 +137,21 @@ public Short getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, SmallIntVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final short value = from.valueBuffer.getShort(fromIndex * TYPE_WIDTH);
     valueBuffer.setShort(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that
+   * Same as {@link #copyFrom(int, int, SmallIntVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, SmallIntVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -357,22 +357,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableSmallIntVector) to);
+    return new TransferImpl((SmallIntVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableSmallIntVector to;
+    SmallIntVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableSmallIntVector(ref, field.getFieldType(), allocator);
+      to = new SmallIntVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableSmallIntVector to) {
+    public TransferImpl(SmallIntVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableSmallIntVector getTo() {
+    public SmallIntVector getTo() {
       return to;
     }
 
@@ -388,7 +388,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this);
+      to.copyFromSafe(fromIndex, toIndex, SmallIntVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java
index ca8e1aacf6e0e..604cedffd4258 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java
@@ -30,36 +30,36 @@
 import org.slf4j.Logger;
 
 /**
- * NullableTimeMicroVector implements a fixed width vector (8 bytes) of
+ * TimeMicroVector implements a fixed width vector (8 bytes) of
  * time (microsecond resolution) values which could be null.
  * A validity buffer (bit vector) is maintained to track which elements in the
  * vector are null.
  */
-public class NullableTimeMicroVector extends BaseNullableFixedWidthVector {
+public class TimeMicroVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 8;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for
+   * Instantiate a TimeMicroVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeMicroVector(String name, BufferAllocator allocator) {
+  public TimeMicroVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for
+   * Instantiate a TimeMicroVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this);
+    reader = new TimeMicroReaderImpl(TimeMicroVector.this);
   }
 
   /**
@@ -138,21 +138,21 @@ public Long getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, TimeMicroVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
     valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that
+   * Same as {@link #copyFrom(int, int, TimeMicroVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, TimeMicroVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -331,22 +331,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeMicroVector) to);
+    return new TransferImpl((TimeMicroVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableTimeMicroVector to;
+    TimeMicroVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator);
+      to = new TimeMicroVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableTimeMicroVector to) {
+    public TransferImpl(TimeMicroVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableTimeMicroVector getTo() {
+    public TimeMicroVector getTo() {
       return to;
     }
 
@@ -362,7 +362,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this);
+      to.copyFromSafe(fromIndex, toIndex, TimeMicroVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java
index a28ccfe00f372..c3d100c5b6194 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java
@@ -31,35 +31,35 @@
 import org.slf4j.Logger;
 
 /**
- * NullableTimeMilliVector implements a fixed width (4 bytes) vector of
+ * TimeMilliVector implements a fixed width (4 bytes) vector of
  * time (millisecond resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeMilliVector extends BaseNullableFixedWidthVector {
+public class TimeMilliVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 4;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for
+   * Instantiate a TimeMilliVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeMilliVector(String name, BufferAllocator allocator) {
+  public TimeMilliVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for
+   * Instantiate a TimeMilliVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this);
+    reader = new TimeMilliReaderImpl(TimeMilliVector.this);
   }
 
   /**
@@ -139,21 +139,21 @@ public LocalDateTime getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, TimeMilliVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
     valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that
+   * Same as {@link #copyFrom(int, int, TimeMilliVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, TimeMilliVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -332,22 +332,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeMilliVector) to);
+    return new TransferImpl((TimeMilliVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableTimeMilliVector to;
+    TimeMilliVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator);
+      to = new TimeMilliVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableTimeMilliVector to) {
+    public TransferImpl(TimeMilliVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableTimeMilliVector getTo() {
+    public TimeMilliVector getTo() {
       return to;
     }
 
@@ -363,7 +363,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this);
+      to.copyFromSafe(fromIndex, toIndex, TimeMilliVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java
index e6903c011e676..97401ec8aaa0b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java
@@ -29,35 +29,35 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableTimeNanoVector implements a fixed width vector (8 bytes) of
+ * TimeNanoVector implements a fixed width vector (8 bytes) of
  * time (nanosecond resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeNanoVector extends BaseNullableFixedWidthVector {
+public class TimeNanoVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 8;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for
+   * Instantiate a TimeNanoVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeNanoVector(String name, BufferAllocator allocator) {
+  public TimeNanoVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for
+   * Instantiate a TimeNanoVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this);
+    reader = new TimeNanoReaderImpl(TimeNanoVector.this);
   }
 
   /**
@@ -137,21 +137,21 @@ public Long getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, TimeNanoVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
     valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that
+   * Same as {@link #copyFrom(int, int, TimeNanoVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, TimeNanoVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -329,22 +329,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeNanoVector) to);
+    return new TransferImpl((TimeNanoVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableTimeNanoVector to;
+    TimeNanoVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator);
+      to = new TimeNanoVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableTimeNanoVector to) {
+    public TransferImpl(TimeNanoVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableTimeNanoVector getTo() {
+    public TimeNanoVector getTo() {
       return to;
     }
 
@@ -360,7 +360,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this);
+      to.copyFromSafe(fromIndex, toIndex, TimeNanoVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java
index 85426a36948db..a7823a916f3d3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java
@@ -29,35 +29,35 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableTimeSecVector implements a fixed width (4 bytes) vector of
+ * TimeSecVector implements a fixed width (4 bytes) vector of
  * time (seconds resolution) values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableTimeSecVector extends BaseNullableFixedWidthVector {
+public class TimeSecVector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 4;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for
+   * Instantiate a TimeSecVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeSecVector(String name, BufferAllocator allocator) {
+  public TimeSecVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for
+   * Instantiate a TimeSecVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new TimeSecReaderImpl(NullableTimeSecVector.this);
+    reader = new TimeSecReaderImpl(TimeSecVector.this);
   }
 
   /**
@@ -137,21 +137,21 @@ public Integer getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, TimeSecVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
     valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that
+   * Same as {@link #copyFrom(int, int, TimeSecVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, TimeSecVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -330,22 +330,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeSecVector) to);
+    return new TransferImpl((TimeSecVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableTimeSecVector to;
+    TimeSecVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableTimeSecVector(ref, field.getFieldType(), allocator);
+      to = new TimeSecVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableTimeSecVector to) {
+    public TransferImpl(TimeSecVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableTimeSecVector getTo() {
+    public TimeSecVector getTo() {
       return to;
     }
 
@@ -361,7 +361,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this);
+      to.copyFromSafe(fromIndex, toIndex, TimeSecVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java
index c7ad141549bcb..bfe330a1e40c5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java
@@ -29,37 +29,37 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableTimeStampMicroTZVector implements a fixed width vector (8 bytes) of
+ * TimeStampMicroTZVector implements a fixed width vector (8 bytes) of
  * timestamp (microsecond resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeStampMicroTZVector extends NullableTimeStampVector {
+public class TimeStampMicroTZVector extends TimeStampVector {
   private final FieldReader reader;
   private final String timeZone;
 
   /**
-   * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampMicroTZVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) {
+  public TimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) {
     this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampMicroTZVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, fieldType, allocator);
     org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType();
     timeZone = arrowType.getTimezone();
-    reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this);
+    reader = new TimeStampMicroTZReaderImpl(TimeStampMicroTZVector.this);
   }
 
   /**
@@ -200,7 +200,7 @@ public void setSafe(int index, TimeStampMicroTZHolder holder) {
    */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref,
+    TimeStampMicroTZVector to = new TimeStampMicroTZVector(ref,
             field.getFieldType(), allocator);
     return new TransferImpl(to);
   }
@@ -212,6 +212,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeStampMicroTZVector) to);
+    return new TransferImpl((TimeStampMicroTZVector) to);
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java
index 5a34b194ce77f..85b615d8f578b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java
@@ -29,34 +29,34 @@
 import org.joda.time.LocalDateTime;
 
 /**
- * NullableTimeStampMicroVector implements a fixed width vector (8 bytes) of
+ * TimeStampMicroVector implements a fixed width vector (8 bytes) of
  * timestamp (microsecond resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeStampMicroVector extends NullableTimeStampVector {
+public class TimeStampMicroVector extends TimeStampVector {
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampMicroVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampMicroVector(String name, BufferAllocator allocator) {
+  public TimeStampMicroVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampMicroVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, fieldType, allocator);
-    reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this);
+    reader = new TimeStampMicroReaderImpl(TimeStampMicroVector.this);
   }
 
   /**
@@ -202,7 +202,7 @@ public void setSafe(int index, TimeStampMicroHolder holder) {
    */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref,
+    TimeStampMicroVector to = new TimeStampMicroVector(ref,
             field.getFieldType(), allocator);
     return new TransferImpl(to);
   }
@@ -214,6 +214,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeStampMicroVector) to);
+    return new TransferImpl((TimeStampMicroVector) to);
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java
index f9736aed7be62..9d68b564492b6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java
@@ -29,37 +29,37 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableTimeStampMilliTZVector implements a fixed width vector (8 bytes) of
+ * TimeStampMilliTZVector implements a fixed width vector (8 bytes) of
  * timestamp (millisecond resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeStampMilliTZVector extends NullableTimeStampVector {
+public class TimeStampMilliTZVector extends TimeStampVector {
   private final FieldReader reader;
   private final String timeZone;
 
   /**
-   * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampMilliTZVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) {
+  public TimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) {
     this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampMilliTZVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, fieldType, allocator);
     org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType();
     timeZone = arrowType.getTimezone();
-    reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this);
+    reader = new TimeStampMilliTZReaderImpl(TimeStampMilliTZVector.this);
   }
 
   /**
@@ -199,7 +199,7 @@ public void setSafe(int index, TimeStampMilliTZHolder holder) {
    */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref,
+    TimeStampMilliTZVector to = new TimeStampMilliTZVector(ref,
             field.getFieldType(), allocator);
     return new TransferImpl(to);
   }
@@ -211,6 +211,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeStampMilliTZVector) to);
+    return new TransferImpl((TimeStampMilliTZVector) to);
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java
index fc38a6bd507de..7e8a1d0e2a854 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java
@@ -29,34 +29,34 @@
 import org.joda.time.LocalDateTime;
 
 /**
- * NullableTimeStampMilliVector implements a fixed width vector (8 bytes) of
+ * TimeStampMilliVector implements a fixed width vector (8 bytes) of
  * timestamp (millisecond resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeStampMilliVector extends NullableTimeStampVector {
+public class TimeStampMilliVector extends TimeStampVector {
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampMilliVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampMilliVector(String name, BufferAllocator allocator) {
+  public TimeStampMilliVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampMilliVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, fieldType, allocator);
-    reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this);
+    reader = new TimeStampMilliReaderImpl(TimeStampMilliVector.this);
   }
 
   /**
@@ -200,7 +200,7 @@ public void setSafe(int index, TimeStampMilliHolder holder) {
    */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref,
+    TimeStampMilliVector to = new TimeStampMilliVector(ref,
             field.getFieldType(), allocator);
     return new TransferImpl(to);
   }
@@ -212,6 +212,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeStampMilliVector) to);
+    return new TransferImpl((TimeStampMilliVector) to);
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java
index 8eb48462c994b..e0361820137e2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java
@@ -29,37 +29,37 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableTimeStampNanoTZVector implements a fixed width vector (8 bytes) of
+ * TimeStampNanoTZVector implements a fixed width vector (8 bytes) of
  * timestamp (nanosecond resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeStampNanoTZVector extends NullableTimeStampVector {
+public class TimeStampNanoTZVector extends TimeStampVector {
   private final FieldReader reader;
   private final String timeZone;
 
   /**
-   * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampNanoTZVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) {
+  public TimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) {
     this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampNanoTZVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, fieldType, allocator);
     org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType();
     timeZone = arrowType.getTimezone();
-    reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this);
+    reader = new TimeStampNanoTZReaderImpl(TimeStampNanoTZVector.this);
   }
 
   /**
@@ -200,7 +200,7 @@ public void setSafe(int index, TimeStampNanoTZHolder holder) {
    */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref,
+    TimeStampNanoTZVector to = new TimeStampNanoTZVector(ref,
             field.getFieldType(), allocator);
     return new TransferImpl(to);
   }
@@ -212,6 +212,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeStampNanoTZVector) to);
+    return new TransferImpl((TimeStampNanoTZVector) to);
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java
index 6cc5c905783d7..fdf5d26945b90 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java
@@ -29,34 +29,34 @@
 import org.joda.time.LocalDateTime;
 
 /**
- * NullableTimeStampNanoVector implements a fixed width vector (8 bytes) of
+ * TimeStampNanoVector implements a fixed width vector (8 bytes) of
  * timestamp (nanosecond resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeStampNanoVector extends NullableTimeStampVector {
+public class TimeStampNanoVector extends TimeStampVector {
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampNanoVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampNanoVector(String name, BufferAllocator allocator) {
+  public TimeStampNanoVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampNanoVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, fieldType, allocator);
-    reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this);
+    reader = new TimeStampNanoReaderImpl(TimeStampNanoVector.this);
   }
 
   /**
@@ -201,7 +201,7 @@ public void setSafe(int index, TimeStampNanoHolder holder) {
    */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref,
+    TimeStampNanoVector to = new TimeStampNanoVector(ref,
             field.getFieldType(), allocator);
     return new TransferImpl(to);
   }
@@ -213,6 +213,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeStampNanoVector) to);
+    return new TransferImpl((TimeStampNanoVector) to);
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java
index 0593ac76dce57..201f1c317d02d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java
@@ -28,37 +28,37 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableTimeStampSecTZVector implements a fixed width vector (8 bytes) of
+ * TimeStampSecTZVector implements a fixed width vector (8 bytes) of
  * timestamp (seconds resolution) values which could be null. A validity buffer
  * (bit vector) is maintained to track which elements in the vector are null.
  */
-public class NullableTimeStampSecTZVector extends NullableTimeStampVector {
+public class TimeStampSecTZVector extends TimeStampVector {
   private final FieldReader reader;
   private final String timeZone;
 
   /**
-   * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampSecTZVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) {
+  public TimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) {
     this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampSecTZVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, fieldType, allocator);
     org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType();
     timeZone = arrowType.getTimezone();
-    reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this);
+    reader = new TimeStampSecTZReaderImpl(TimeStampSecTZVector.this);
   }
 
   /**
@@ -198,7 +198,7 @@ public void setSafe(int index, TimeStampSecTZHolder holder) {
    */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref,
+    TimeStampSecTZVector to = new TimeStampSecTZVector(ref,
             field.getFieldType(), allocator);
     return new TransferImpl(to);
   }
@@ -210,6 +210,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeStampSecTZVector) to);
+    return new TransferImpl((TimeStampSecTZVector) to);
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java
index 1afdb30d0453e..4bcd4f7bf0026 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java
@@ -29,34 +29,34 @@
 import org.joda.time.LocalDateTime;
 
 /**
- * NullableTimeStampSecVector implements a fixed width vector (8 bytes) of
+ * TimeStampSecVector implements a fixed width vector (8 bytes) of
  * timestamp (seconds resolution) values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableTimeStampSecVector extends NullableTimeStampVector {
+public class TimeStampSecVector extends TimeStampVector {
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampSecVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampSecVector(String name, BufferAllocator allocator) {
+  public TimeStampSecVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampSecVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, fieldType, allocator);
-    reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this);
+    reader = new TimeStampSecReaderImpl(TimeStampSecVector.this);
   }
 
   /**
@@ -201,7 +201,7 @@ public void setSafe(int index, TimeStampSecHolder holder) {
    */
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref,
+    TimeStampSecVector to = new TimeStampSecVector(ref,
             field.getFieldType(), allocator);
     return new TransferImpl(to);
   }
@@ -213,6 +213,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTimeStampSecVector) to);
+    return new TransferImpl((TimeStampSecVector) to);
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java
similarity index 88%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java
index e1b719f66a784..4c70b819cbf37 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java
@@ -24,21 +24,21 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableTimeStampVector is an abstract interface for fixed width vector (8 bytes)
+ * TimeStampVector is an abstract interface for fixed width vector (8 bytes)
  * of timestamp values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector {
+public abstract class TimeStampVector extends BaseFixedWidthVector {
   protected static final byte TYPE_WIDTH = 8;
 
   /**
-   * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for
+   * Instantiate a TimeStampVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
   }
 
@@ -69,21 +69,21 @@ public long get(int index) throws IllegalStateException {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, TimeStampVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
     valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that
+   * Same as {@link #copyFromSafe(int, int, TimeStampVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, TimeStampVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -188,14 +188,14 @@ public static long get(final ArrowBuf buffer, final int index) {
 
 
   public class TransferImpl implements TransferPair {
-    NullableTimeStampVector to;
+    TimeStampVector to;
 
-    public TransferImpl(NullableTimeStampVector to) {
+    public TransferImpl(TimeStampVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableTimeStampVector getTo() {
+    public TimeStampVector getTo() {
       return to;
     }
 
@@ -211,7 +211,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this);
+      to.copyFromSafe(fromIndex, toIndex, TimeStampVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java
similarity index 91%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java
index cadfe608538cc..306437f0fe9d6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java
@@ -29,35 +29,35 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableTinyIntVector implements a fixed width (1 bytes) vector of
+ * TinyIntVector implements a fixed width (1 bytes) vector of
  * byte values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableTinyIntVector extends BaseNullableFixedWidthVector {
+public class TinyIntVector extends BaseFixedWidthVector {
   public static final byte TYPE_WIDTH = 1;
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for
+   * Instantiate a TinyIntVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableTinyIntVector(String name, BufferAllocator allocator) {
+  public TinyIntVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()),
             allocator);
   }
 
   /**
-   * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for
+   * Instantiate a TinyIntVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public TinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new TinyIntReaderImpl(NullableTinyIntVector.this);
+    reader = new TinyIntReaderImpl(TinyIntVector.this);
   }
 
   /**
@@ -137,21 +137,21 @@ public Byte getObject(int index) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, TinyIntVector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final byte value = from.valueBuffer.getByte(fromIndex * TYPE_WIDTH);
     valueBuffer.setByte(thisIndex * TYPE_WIDTH, value);
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that
+   * Same as {@link #copyFrom(int, int, TinyIntVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, TinyIntVector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -357,22 +357,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableTinyIntVector) to);
+    return new TransferImpl((TinyIntVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableTinyIntVector to;
+    TinyIntVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableTinyIntVector(ref, field.getFieldType(), allocator);
+      to = new TinyIntVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableTinyIntVector to) {
+    public TransferImpl(TinyIntVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableTinyIntVector getTo() {
+    public TinyIntVector getTo() {
       return to;
     }
 
@@ -388,7 +388,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this);
+      to.copyFromSafe(fromIndex, toIndex, TinyIntVector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java
similarity index 91%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java
index ebd9b56996cf6..6901a889a6dcf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java
@@ -28,22 +28,22 @@
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
- * NullableUInt1Vector implements a fixed width (1 bytes) vector of
+ * UInt1Vector implements a fixed width (1 bytes) vector of
  * integer values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableUInt1Vector extends BaseNullableFixedWidthVector {
+public class UInt1Vector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 1;
   private final FieldReader reader;
 
-  public NullableUInt1Vector(String name, BufferAllocator allocator) {
+  public UInt1Vector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()),
             allocator);
   }
 
-  public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public UInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new UInt1ReaderImpl(NullableUInt1Vector.this);
+    reader = new UInt1ReaderImpl(UInt1Vector.this);
   }
 
   @Override
@@ -107,13 +107,13 @@ public Byte getObject(int index) {
     }
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) {
+  public void copyFrom(int fromIndex, int thisIndex, UInt1Vector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final byte value = from.valueBuffer.getByte(fromIndex * TYPE_WIDTH);
     valueBuffer.setByte(thisIndex * TYPE_WIDTH, value);
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, UInt1Vector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -279,22 +279,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableUInt1Vector) to);
+    return new TransferImpl((UInt1Vector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableUInt1Vector to;
+    UInt1Vector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableUInt1Vector(ref, field.getFieldType(), allocator);
+      to = new UInt1Vector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableUInt1Vector to) {
+    public TransferImpl(UInt1Vector to) {
       this.to = to;
     }
 
     @Override
-    public NullableUInt1Vector getTo() {
+    public UInt1Vector getTo() {
       return to;
     }
 
@@ -310,7 +310,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, UInt1Vector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java
similarity index 91%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java
index bedb3f35e6bac..968ce9151fae0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java
@@ -29,22 +29,22 @@
 import org.slf4j.Logger;
 
 /**
- * NullableUInt2Vector implements a fixed width (2 bytes) vector of
+ * UInt2Vector implements a fixed width (2 bytes) vector of
  * integer values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableUInt2Vector extends BaseNullableFixedWidthVector {
+public class UInt2Vector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 2;
   private final FieldReader reader;
 
-  public NullableUInt2Vector(String name, BufferAllocator allocator) {
+  public UInt2Vector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()),
             allocator);
   }
 
-  public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public UInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new UInt2ReaderImpl(NullableUInt2Vector.this);
+    reader = new UInt2ReaderImpl(UInt2Vector.this);
   }
 
   @Override
@@ -108,13 +108,13 @@ public Character getObject(int index) {
     }
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) {
+  public void copyFrom(int fromIndex, int thisIndex, UInt2Vector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final char value = from.valueBuffer.getChar(fromIndex * TYPE_WIDTH);
     valueBuffer.setChar(thisIndex * TYPE_WIDTH, value);
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, UInt2Vector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -280,22 +280,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableUInt2Vector) to);
+    return new TransferImpl((UInt2Vector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableUInt2Vector to;
+    UInt2Vector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableUInt2Vector(ref, field.getFieldType(), allocator);
+      to = new UInt2Vector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableUInt2Vector to) {
+    public TransferImpl(UInt2Vector to) {
       this.to = to;
     }
 
     @Override
-    public NullableUInt2Vector getTo() {
+    public UInt2Vector getTo() {
       return to;
     }
 
@@ -311,7 +311,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, UInt2Vector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java
index 99b547ca4fdef..af219cb061e48 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java
@@ -29,22 +29,22 @@
 import org.slf4j.Logger;
 
 /**
- * NullableUInt4Vector implements a fixed width (4 bytes) vector of
+ * UInt4Vector implements a fixed width (4 bytes) vector of
  * integer values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableUInt4Vector extends BaseNullableFixedWidthVector {
+public class UInt4Vector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 4;
   private final FieldReader reader;
 
-  public NullableUInt4Vector(String name, BufferAllocator allocator) {
+  public UInt4Vector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()),
             allocator);
   }
 
-  public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public UInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new UInt4ReaderImpl(NullableUInt4Vector.this);
+    reader = new UInt4ReaderImpl(UInt4Vector.this);
   }
 
   @Override
@@ -108,13 +108,13 @@ public Integer getObject(int index) {
     }
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) {
+  public void copyFrom(int fromIndex, int thisIndex, UInt4Vector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH);
     valueBuffer.setInt(thisIndex * TYPE_WIDTH, value);
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, UInt4Vector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -252,22 +252,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableUInt4Vector) to);
+    return new TransferImpl((UInt4Vector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableUInt4Vector to;
+    UInt4Vector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableUInt4Vector(ref, field.getFieldType(), allocator);
+      to = new UInt4Vector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableUInt4Vector to) {
+    public TransferImpl(UInt4Vector to) {
       this.to = to;
     }
 
     @Override
-    public NullableUInt4Vector getTo() {
+    public UInt4Vector getTo() {
       return to;
     }
 
@@ -283,7 +283,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, UInt4Vector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java
index e6664a09265d7..eae4a31729955 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java
@@ -29,22 +29,22 @@
 import org.slf4j.Logger;
 
 /**
- * NullableUInt8Vector implements a fixed width vector (8 bytes) of
+ * UInt8Vector implements a fixed width vector (8 bytes) of
  * integer values which could be null. A validity buffer (bit vector) is
  * maintained to track which elements in the vector are null.
  */
-public class NullableUInt8Vector extends BaseNullableFixedWidthVector {
+public class UInt8Vector extends BaseFixedWidthVector {
   private static final byte TYPE_WIDTH = 8;
   private final FieldReader reader;
 
-  public NullableUInt8Vector(String name, BufferAllocator allocator) {
+  public UInt8Vector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.UINT8.getType()),
             allocator);
   }
 
-  public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public UInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType, TYPE_WIDTH);
-    reader = new UInt8ReaderImpl(NullableUInt8Vector.this);
+    reader = new UInt8ReaderImpl(UInt8Vector.this);
   }
 
   @Override
@@ -108,13 +108,13 @@ public Long getObject(int index) {
     }
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) {
+  public void copyFrom(int fromIndex, int thisIndex, UInt8Vector from) {
     BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
     final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH);
     valueBuffer.setLong(thisIndex * TYPE_WIDTH, value);
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, UInt8Vector from) {
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
@@ -252,22 +252,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableUInt8Vector) to);
+    return new TransferImpl((UInt8Vector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableUInt8Vector to;
+    UInt8Vector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableUInt8Vector(ref, field.getFieldType(), allocator);
+      to = new UInt8Vector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableUInt8Vector to) {
+    public TransferImpl(UInt8Vector to) {
       this.to = to;
     }
 
     @Override
-    public NullableUInt8Vector getTo() {
+    public UInt8Vector getTo() {
       return to;
     }
 
@@ -283,7 +283,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this);
+      to.copyFromSafe(fromIndex, toIndex, UInt8Vector.this);
     }
   }
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java
index 1dfe8912c52f9..893ad7cb594a7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java
@@ -31,33 +31,33 @@
 import java.nio.ByteBuffer;
 
 /**
- * NullableVarBinaryVector implements a variable width vector of binary
+ * VarBinaryVector implements a variable width vector of binary
  * values which could be NULL. A validity buffer (bit vector) is maintained
  * to track which elements in the vector are null.
  */
-public class NullableVarBinaryVector extends BaseNullableVariableWidthVector {
+public class VarBinaryVector extends BaseVariableWidthVector {
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for
+   * Instantiate a VarBinaryVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableVarBinaryVector(String name, BufferAllocator allocator) {
+  public VarBinaryVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator);
   }
 
   /**
-   * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for
+   * Instantiate a VarBinaryVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public VarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType);
-    reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this);
+    reader = new VarBinaryReaderImpl(VarBinaryVector.this);
   }
 
   /**
@@ -156,7 +156,7 @@ public void get(int index, NullableVarBinaryHolder holder) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, VarBinaryVector from) {
     final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH);
     final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH);
     final int length = end - start;
@@ -169,14 +169,14 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from)
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that
+   * Same as {@link #copyFrom(int, int, VarBinaryVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, VarBinaryVector from) {
     final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH);
     final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH);
     final int length = end - start;
@@ -291,22 +291,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableVarBinaryVector) to);
+    return new TransferImpl((VarBinaryVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableVarBinaryVector to;
+    VarBinaryVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator);
+      to = new VarBinaryVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableVarBinaryVector to) {
+    public TransferImpl(VarBinaryVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableVarBinaryVector getTo() {
+    public VarBinaryVector getTo() {
       return to;
     }
 
@@ -322,7 +322,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this);
+      to.copyFromSafe(fromIndex, toIndex, VarBinaryVector.this);
     }
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java
index 31e409cc6263f..8a38b1d455d20 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java
@@ -32,33 +32,33 @@
 import java.nio.ByteBuffer;
 
 /**
- * NullableVarCharVector implements a variable width vector of VARCHAR
+ * VarCharVector implements a variable width vector of VARCHAR
  * values which could be NULL. A validity buffer (bit vector) is maintained
  * to track which elements in the vector are null.
  */
-public class NullableVarCharVector extends BaseNullableVariableWidthVector {
+public class VarCharVector extends BaseVariableWidthVector {
   private final FieldReader reader;
 
   /**
-   * Instantiate a NullableVarCharVector. This doesn't allocate any memory for
+   * Instantiate a VarCharVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param allocator allocator for memory management.
    */
-  public NullableVarCharVector(String name, BufferAllocator allocator) {
+  public VarCharVector(String name, BufferAllocator allocator) {
     this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator);
   }
 
   /**
-   * Instantiate a NullableVarCharVector. This doesn't allocate any memory for
+   * Instantiate a VarCharVector. This doesn't allocate any memory for
    * the data in vector.
    * @param name name of the vector
    * @param fieldType type of Field materialized by this vector
    * @param allocator allocator for memory management.
    */
-  public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) {
+  public VarCharVector(String name, FieldType fieldType, BufferAllocator allocator) {
     super(name, allocator, fieldType);
-    reader = new VarCharReaderImpl(NullableVarCharVector.this);
+    reader = new VarCharReaderImpl(VarCharVector.this);
   }
 
   /**
@@ -159,7 +159,7 @@ public void get(int index, NullableVarCharHolder holder) {
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) {
+  public void copyFrom(int fromIndex, int thisIndex, VarCharVector from) {
     final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH);
     final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH);
     final int length = end - start;
@@ -172,14 +172,14 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) {
   }
 
   /**
-   * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that
+   * Same as {@link #copyFrom(int, int, VarCharVector)} except that
    * it handles the case when the capacity of the vector needs to be expanded
    * before copy.
    * @param fromIndex position to copy from in source vector
    * @param thisIndex position to copy to in this vector
    * @param from source vector
    */
-  public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, VarCharVector from) {
     final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH);
     final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH);
     final int length = end - start;
@@ -294,22 +294,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
    */
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new TransferImpl((NullableVarCharVector) to);
+    return new TransferImpl((VarCharVector) to);
   }
 
   private class TransferImpl implements TransferPair {
-    NullableVarCharVector to;
+    VarCharVector to;
 
     public TransferImpl(String ref, BufferAllocator allocator) {
-      to = new NullableVarCharVector(ref, field.getFieldType(), allocator);
+      to = new VarCharVector(ref, field.getFieldType(), allocator);
     }
 
-    public TransferImpl(NullableVarCharVector to) {
+    public TransferImpl(VarCharVector to) {
       this.to = to;
     }
 
     @Override
-    public NullableVarCharVector getTo() {
+    public VarCharVector getTo() {
       return to;
     }
 
@@ -325,7 +325,7 @@ public void splitAndTransfer(int startIndex, int length) {
 
     @Override
     public void copyValueSafe(int fromIndex, int toIndex) {
-      to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this);
+      to.copyFromSafe(fromIndex, toIndex, VarCharVector.this);
     }
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorDefinitionSetter.java
similarity index 94%
rename from java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
rename to java/vector/src/main/java/org/apache/arrow/vector/VectorDefinitionSetter.java
index 1e0746aabaa61..2f45d3a5b6316 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorDefinitionSetter.java
@@ -18,7 +18,7 @@
 
 package org.apache.arrow.vector;
 
-public interface NullableVectorDefinitionSetter {
+public interface VectorDefinitionSetter {
 
   public void setIndexDefined(int index);
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index db0ff86df47a9..5ded1cc6f8778 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -100,8 +100,8 @@ protected boolean supportsDirectRead() {
   // return the child vector's ordinal in the composite container
   public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
 
-  public NullableMapVector addOrGetMap(String name) {
-    return addOrGet(name, FieldType.nullable(new Struct()), NullableMapVector.class);
+  public MapVector addOrGetMap(String name) {
+    return addOrGet(name, FieldType.nullable(new Struct()), MapVector.class);
   }
 
   public ListVector addOrGetList(String name) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
index a76fbbe11a1fb..cd3a2ae9e6f3c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
@@ -42,14 +42,12 @@ public void populate(int lastIndex) {
     if (lastIndex < 0) {
       throw new IndexOutOfBoundsException("index cannot be negative");
     }
-    final UInt4Vector.Accessor accessor = offsets.getAccessor();
-    final UInt4Vector.Mutator mutator = offsets.getMutator();
-    final int lastSet = Math.max(accessor.getValueCount() - 1, 0);
-    final int previousEnd = accessor.get(lastSet);//0 ? 0 : accessor.get(lastSet);
+    final int lastSet = Math.max(offsets.getValueCount() - 1, 0);
+    final int previousEnd = offsets.get(lastSet);//0 ? 0 : accessor.get(lastSet);
     for (int i = lastSet; i < lastIndex; i++) {
-      mutator.setSafe(i + 1, previousEnd);
+      offsets.setSafe(i + 1, previousEnd);
     }
-    mutator.setValueCount(lastIndex + 1);
+    offsets.setValueCount(lastIndex + 1);
   }
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 6089a67924fe0..075ae83ea4d84 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,332 +21,492 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 
-import javax.annotation.Nullable;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
+import com.google.common.collect.ObjectArrays;
 
 import io.netty.buffer.ArrowBuf;
-
+import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.NullableMapWriter;
 import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringHashMap;
+import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
-public class MapVector extends AbstractMapVector {
-  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
+public class MapVector extends NonNullableMapVector implements FieldVector {
 
   public static MapVector empty(String name, BufferAllocator allocator) {
-    FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
+    FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
     return new MapVector(name, allocator, fieldType, null);
   }
 
-  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
-  protected final FieldType fieldType;
-  public int valueCount;
+  private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
+  private final NullableMapWriter writer = new NullableMapWriter(this);
+
+  protected ArrowBuf validityBuffer;
+  private int validityAllocationSizeInBytes;
 
   // deprecated, use FieldType or static constructor instead
   @Deprecated
   public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
-    this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
+    this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack);
   }
 
-  public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
-    super(name, allocator, callBack);
-    this.fieldType = checkNotNull(fieldType);
-    this.valueCount = 0;
-  }
-
-  @Override
-  public FieldReader getReader() {
-    return reader;
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
+  public MapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
+    this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack);
   }
 
-  transient private MapTransferPair ephPair;
-
-  public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
-    if (ephPair == null || ephPair.from != from) {
-      ephPair = (MapTransferPair) from.makeTransferPair(this);
-    }
-    ephPair.copyValueSafe(fromIndex, thisIndex);
+  public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+    super(name, checkNotNull(allocator), fieldType, callBack);
+    this.validityBuffer = allocator.getEmpty();
+    this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION);
   }
 
   @Override
-  protected boolean supportsDirectRead() {
-    return true;
-  }
-
-  public Iterator<String> fieldNameIterator() {
-    return getChildFieldNames().iterator();
+  public Field getField() {
+    Field f = super.getField();
+    FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata());
+    return new Field(f.getName(), type, f.getChildren());
   }
 
   @Override
-  public void setInitialCapacity(int numRecords) {
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      v.setInitialCapacity(numRecords);
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    if (ownBuffers.size() != 1) {
+      throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size());
     }
-  }
 
-  @Override
-  public int getBufferSize() {
-    if (valueCount == 0 || size() == 0) {
-      return 0;
-    }
-    long buffer = 0;
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      buffer += v.getBufferSize();
-    }
+    ArrowBuf bitBuffer = ownBuffers.get(0);
 
-    return (int) buffer;
+    validityBuffer.release();
+    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
+    valueCount = fieldNode.getLength();
+    validityAllocationSizeInBytes = validityBuffer.capacity();
   }
 
   @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
+  public List<ArrowBuf> getFieldBuffers() {
+    List<ArrowBuf> result = new ArrayList<>(1);
+    setReaderAndWriterIndex();
+    result.add(validityBuffer);
 
-    long bufferSize = 0;
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      bufferSize += v.getBufferSizeFor(valueCount);
-    }
+    return result;
+  }
 
-    return (int) bufferSize;
+  private void setReaderAndWriterIndex() {
+    validityBuffer.readerIndex(0);
+    validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
   }
 
   @Override
-  public ArrowBuf getValidityBuffer() {
-    throw new UnsupportedOperationException();
+  @Deprecated
+  public List<BufferBacked> getFieldInnerVectors() {
+    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
   }
 
   @Override
-  public ArrowBuf getDataBuffer() {
-    throw new UnsupportedOperationException();
+  public NullableMapReaderImpl getReader() {
+    return reader;
   }
 
-  @Override
-  public ArrowBuf getOffsetBuffer() {
-    throw new UnsupportedOperationException();
+  public NullableMapWriter getWriter() {
+    return writer;
   }
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return getTransferPair(name, allocator, null);
+    return new NullableMapTransferPair(this, new MapVector(name, allocator, fieldType, null), false);
   }
 
   @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false);
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new NullableMapTransferPair(this, (MapVector) to, true);
   }
 
   @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new MapTransferPair(this, (MapVector) to);
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, null), false);
   }
 
   @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false);
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false);
   }
 
-  protected static class MapTransferPair implements TransferPair {
-    private final TransferPair[] pairs;
-    private final MapVector from;
-    private final MapVector to;
+  protected class NullableMapTransferPair extends MapTransferPair {
 
-    public MapTransferPair(MapVector from, MapVector to) {
-      this(from, to, true);
-    }
+    private MapVector target;
 
-    protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
-      this.from = from;
-      this.to = to;
-      this.pairs = new TransferPair[from.size()];
-      this.to.ephPair = null;
-
-      int i = 0;
-      FieldVector vector;
-      for (String child : from.getChildFieldNames()) {
-        int preSize = to.size();
-        vector = from.getChild(child);
-        if (vector == null) {
-          continue;
-        }
-        //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
-        // the child fields may be nested fields of the top level child. For example if the structure
-        // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
-        // But the children member of a Materialized field is a HashSet. If the fields are added in the
-        // children HashSet, and the hashCode of the Materialized field includes the hash code of the
-        // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
-        // (This is similar to what happens in ScanBatch where the children cannot be added till they are
-        // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
-        // include the hashCode of the children but is based only on MaterializedField$key.
-        final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass());
-        if (allocate && to.size() != preSize) {
-          newVector.allocateNew();
-        }
-        pairs[i++] = vector.makeTransferPair(newVector);
-      }
+    protected NullableMapTransferPair(MapVector from, MapVector to, boolean allocate) {
+      super(from, to, allocate);
+      this.target = to;
     }
 
     @Override
     public void transfer() {
-      for (final TransferPair p : pairs) {
-        p.transfer();
-      }
-      to.valueCount = from.valueCount;
-      from.clear();
-    }
-
-    @Override
-    public ValueVector getTo() {
-      return to;
+      target.clear();
+      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
+      super.transfer();
+      clear();
     }
 
     @Override
-    public void copyValueSafe(int from, int to) {
-      for (TransferPair p : pairs) {
-        p.copyValueSafe(from, to);
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      while (toIndex >= target.getValidityBufferValueCapacity()) {
+        target.reallocValidityBuffer();
       }
+      BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex));
+      super.copyValueSafe(fromIndex, toIndex);
     }
 
     @Override
     public void splitAndTransfer(int startIndex, int length) {
-      for (TransferPair p : pairs) {
-        p.splitAndTransfer(startIndex, length);
+      target.clear();
+      splitAndTransferValidityBuffer(startIndex, length, target);
+      super.splitAndTransfer(startIndex, length);
+    }
+  }
+
+  /*
+   * transfer the validity.
+   */
+  private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) {
+    assert startIndex + length <= valueCount;
+    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+    int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length);
+    int offset = startIndex % 8;
+
+    if (length > 0) {
+      if (offset == 0) {
+        // slice
+        if (target.validityBuffer != null) {
+          target.validityBuffer.release();
+        }
+        target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
+        target.validityBuffer.retain(1);
+      } else {
+        /* Copy data
+         * When the first bit starts from the middle of a byte (offset != 0),
+         * copy data from src BitVector.
+         * Each byte in the target is composed by a part in i-th byte,
+         * another part in (i+1)-th byte.
+         */
+        target.allocateValidityBuffer(byteSizeTarget);
+
+        for (int i = 0; i < byteSizeTarget - 1; i++) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset);
+
+          target.validityBuffer.setByte(i, (b1 + b2));
+        }
+
+        /* Copying the last piece is done in the following manner:
+         * if the source vector has 1 or more bytes remaining, we copy
+         * the last piece as a byte formed by shifting data
+         * from the current byte and the next byte.
+         *
+         * if the source vector has no more bytes remaining
+         * (we are at the last byte), we copy the last piece as a byte
+         * by shifting data from the current byte.
+         */
+        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
+                  firstByteSource + byteSizeTarget, offset);
+
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+        } else {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+        }
       }
-      to.setValueCount(length);
     }
   }
 
+  /**
+   * Get the value capacity of the internal validity buffer.
+   * @return number of elements that validity buffer can hold
+   */
+  private int getValidityBufferValueCapacity() {
+    return (int) (validityBuffer.capacity() * 8L);
+  }
+
+  /**
+   * Get the current value capacity for the vector
+   * @return number of elements that vector can hold.
+   */
   @Override
   public int getValueCapacity() {
-    if (size() == 0) {
-      return 0;
-    }
+    return Math.min(getValidityBufferValueCapacity(),
+            super.getValueCapacity());
+  }
 
-    final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
-      @Override
-      public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
-        return Ints.compare(
-            checkNotNull(left).getValueCapacity(),
-            checkNotNull(right).getValueCapacity()
-        );
+  /**
+   * Return the underlying buffers associated with this vector. Note that this doesn't
+   * impact the reference counts for this buffer so it only should be used for in-context
+   * access. Also note that this buffer changes regularly thus
+   * external classes shouldn't hold a reference to it (unless they change it).
+   *
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
+   *              but the returned array will be the only reference to them
+   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
+   *         vector instance.
+   */
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    setReaderAndWriterIndex();
+    final ArrowBuf[] buffers;
+    if (getBufferSize() == 0) {
+      buffers = new ArrowBuf[0];
+    } else {
+      buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false),
+              ArrowBuf.class);
+    }
+    if (clear) {
+      for (ArrowBuf buffer : buffers) {
+        buffer.retain();
       }
-    };
+      clear();
+    }
 
-    return natural.min(getChildren()).getValueCapacity();
+    return buffers;
   }
 
+  /**
+   * Close the vector and release the associated buffers.
+   */
   @Override
-  public Accessor getAccessor() {
-    throw new UnsupportedOperationException("accessor is not needed for MAP");
+  public void close() {
+    clearValidityBuffer();
+    super.close();
   }
 
+  /**
+   * Same as {@link #close()}
+   */
   @Override
-  public Mutator getMutator() {
-    throw new UnsupportedOperationException("mutator is not needed for MAP");
+  public void clear() {
+    clearValidityBuffer();
+    super.clear();
+  }
+
+  /**
+   * Release the validity buffer
+   */
+  private void clearValidityBuffer() {
+    validityBuffer.release();
+    validityBuffer = allocator.getEmpty();
   }
 
+  /**
+   * Get the size (number of bytes) of underlying buffers used by this
+   * vector
+   * @return size of underlying buffers.
+   */
   @Override
-  public Object getObject(int index) {
-    Map<String, Object> vv = new JsonStringHashMap<>();
-    for (String child : getChildFieldNames()) {
-      ValueVector v = getChild(child);
-      if (v != null && index < v.getValueCount()) {
-        Object value = v.getObject(index);
-        if (value != null) {
-          vv.put(child, value);
-        }
+  public int getBufferSize() {
+    if (valueCount == 0) {
+      return 0;
+    }
+    return super.getBufferSize() +
+            BitVectorHelper.getValidityBufferSize(valueCount);
+  }
+
+  /**
+   * Get the potential buffer size for a particular number of records.
+   * @param valueCount desired number of elements in the vector
+   * @return estimated size of underlying buffers if the vector holds
+   *         a given number of elements
+   */
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+    return super.getBufferSizeFor(valueCount)
+            + BitVectorHelper.getValidityBufferSize(valueCount);
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords);
+    super.setInitialCapacity(numRecords);
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* Boolean to keep track if all the memory allocations were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      clear();
+      allocateValidityBuffer(validityAllocationSizeInBytes);
+      success = super.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+        return false;
       }
     }
-    return vv;
+    return true;
+  }
+
+  private void allocateValidityBuffer(final long size) {
+    final int curSize = (int) size;
+    validityBuffer = allocator.buffer(curSize);
+    validityBuffer.readerIndex(0);
+    validityAllocationSizeInBytes = curSize;
+    validityBuffer.setZero(0, validityBuffer.capacity());
   }
 
   @Override
-  public boolean isNull(int index) { return false; }
+  public void reAlloc() {
+    /* reallocate the validity buffer */
+    reallocValidityBuffer();
+    super.reAlloc();
+  }
+
+  private void reallocValidityBuffer() {
+    final int currentBufferCapacity = validityBuffer.capacity();
+    long baseSize = validityAllocationSizeInBytes;
+
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
+    newBuf.setZero(0, newBuf.capacity());
+    newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+    validityBuffer.release(1);
+    validityBuffer = newBuf;
+    validityAllocationSizeInBytes = (int) newAllocationSize;
+  }
+
   @Override
-  public int getNullCount() { return 0; }
+  public long getValidityBufferAddress() {
+    return validityBuffer.memoryAddress();
+  }
 
-  public void get(int index, ComplexHolder holder) {
-    reader.setPosition(index);
-    holder.reader = reader;
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
   }
 
   @Override
-  public int getValueCount() {
-    return valueCount;
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
   }
 
-  public ValueVector getVectorById(int id) {
-  return getChildByOrdinal(id);
-}
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    return validityBuffer;
+  }
 
   @Override
-  public void setValueCount(int valueCount) {
-    for (final ValueVector v : getChildren()) {
-      v.setValueCount(valueCount);
-    }
-    MapVector.this.valueCount = valueCount;
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
   }
 
   @Override
-  public void clear() {
-    for (final ValueVector v : getChildren()) {
-      v.clear();
-    }
-    valueCount = 0;
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
   }
 
   @Override
-  public Field getField() {
-    List<Field> children = new ArrayList<>();
-    for (ValueVector child : getChildren()) {
-      children.add(child.getField());
+  public Object getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return super.getObject(index);
     }
-    return new Field(name, fieldType, children);
   }
 
   @Override
-  public MinorType getMinorType() {
-    return MinorType.MAP;
+  public void get(int index, ComplexHolder holder) {
+    holder.isSet = isSet(index);
+    super.get(index, holder);
+  }
+
+  public int getNullCount() {
+    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+  }
+
+  public boolean isNull(int index) {
+    return isSet(index) == 0;
+  }
+
+  public int isSet(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = validityBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
+  }
+
+  public void setIndexDefined(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
+    }
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+  }
+
+  public void setNull(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
+    }
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
   }
 
   @Override
-  public void close() {
-    final Collection<FieldVector> vectors = getChildren();
-    for (final FieldVector v : vectors) {
-      v.close();
+  public void setValueCount(int valueCount) {
+    assert valueCount >= 0;
+    while (valueCount > getValidityBufferValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
     }
-    vectors.clear();
+    super.setValueCount(valueCount);
+    this.valueCount = valueCount;
+  }
 
+  public void reset() {
     valueCount = 0;
-
-    super.close();
   }
 
-  public void initializeChildrenFromFields(List<Field> children) {
-    for (Field field : children) {
-      FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType());
-      vector.initializeChildrenFromFields(field.getChildren());
-    }
+  @Override
+  @Deprecated
+  public Accessor getAccessor() {
+    throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP");
   }
 
-  public List<FieldVector> getChildrenFromFields() {
-    return getChildren();
+  @Override
+  @Deprecated
+  public Mutator getMutator() {
+    throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP");
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
new file mode 100644
index 0000000000000..cc3ac4148a865
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
@@ -0,0 +1,352 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.complex;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringHashMap;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class NonNullableMapVector extends AbstractMapVector {
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NonNullableMapVector.class);
+
+  public static NonNullableMapVector empty(String name, BufferAllocator allocator) {
+    FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
+    return new NonNullableMapVector(name, allocator, fieldType, null);
+  }
+
+  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
+  protected final FieldType fieldType;
+  public int valueCount;
+
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
+  public NonNullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+    this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
+  }
+
+  public NonNullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+    super(name, allocator, callBack);
+    this.fieldType = checkNotNull(fieldType);
+    this.valueCount = 0;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  transient private MapTransferPair ephPair;
+
+  public void copyFromSafe(int fromIndex, int thisIndex, NonNullableMapVector from) {
+    if (ephPair == null || ephPair.from != from) {
+      ephPair = (MapTransferPair) from.makeTransferPair(this);
+    }
+    ephPair.copyValueSafe(fromIndex, thisIndex);
+  }
+
+  @Override
+  protected boolean supportsDirectRead() {
+    return true;
+  }
+
+  public Iterator<String> fieldNameIterator() {
+    return getChildFieldNames().iterator();
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      v.setInitialCapacity(numRecords);
+    }
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (valueCount == 0 || size() == 0) {
+      return 0;
+    }
+    long buffer = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      buffer += v.getBufferSize();
+    }
+
+    return (int) buffer;
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    long bufferSize = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      bufferSize += v.getBufferSizeFor(valueCount);
+    }
+
+    return (int) bufferSize;
+  }
+
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return getTransferPair(name, allocator, null);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return new MapTransferPair(this, new NonNullableMapVector(name, allocator, fieldType, callBack), false);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new MapTransferPair(this, (NonNullableMapVector) to);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new MapTransferPair(this, new NonNullableMapVector(ref, allocator, fieldType, callBack), false);
+  }
+
+  protected static class MapTransferPair implements TransferPair {
+    private final TransferPair[] pairs;
+    private final NonNullableMapVector from;
+    private final NonNullableMapVector to;
+
+    public MapTransferPair(NonNullableMapVector from, NonNullableMapVector to) {
+      this(from, to, true);
+    }
+
+    protected MapTransferPair(NonNullableMapVector from, NonNullableMapVector to, boolean allocate) {
+      this.from = from;
+      this.to = to;
+      this.pairs = new TransferPair[from.size()];
+      this.to.ephPair = null;
+
+      int i = 0;
+      FieldVector vector;
+      for (String child : from.getChildFieldNames()) {
+        int preSize = to.size();
+        vector = from.getChild(child);
+        if (vector == null) {
+          continue;
+        }
+        //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
+        // the child fields may be nested fields of the top level child. For example if the structure
+        // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
+        // But the children member of a Materialized field is a HashSet. If the fields are added in the
+        // children HashSet, and the hashCode of the Materialized field includes the hash code of the
+        // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
+        // (This is similar to what happens in ScanBatch where the children cannot be added till they are
+        // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
+        // include the hashCode of the children but is based only on MaterializedField$key.
+        final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass());
+        if (allocate && to.size() != preSize) {
+          newVector.allocateNew();
+        }
+        pairs[i++] = vector.makeTransferPair(newVector);
+      }
+    }
+
+    @Override
+    public void transfer() {
+      for (final TransferPair p : pairs) {
+        p.transfer();
+      }
+      to.valueCount = from.valueCount;
+      from.clear();
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      for (TransferPair p : pairs) {
+        p.copyValueSafe(from, to);
+      }
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      for (TransferPair p : pairs) {
+        p.splitAndTransfer(startIndex, length);
+      }
+      to.setValueCount(length);
+    }
+  }
+
+  @Override
+  public int getValueCapacity() {
+    if (size() == 0) {
+      return 0;
+    }
+
+    final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
+      @Override
+      public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
+        return Ints.compare(
+            checkNotNull(left).getValueCapacity(),
+            checkNotNull(right).getValueCapacity()
+        );
+      }
+    };
+
+    return natural.min(getChildren()).getValueCapacity();
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    throw new UnsupportedOperationException("accessor is not needed for MAP");
+  }
+
+  @Override
+  public Mutator getMutator() {
+    throw new UnsupportedOperationException("mutator is not needed for MAP");
+  }
+
+  @Override
+  public Object getObject(int index) {
+    Map<String, Object> vv = new JsonStringHashMap<>();
+    for (String child : getChildFieldNames()) {
+      ValueVector v = getChild(child);
+      if (v != null && index < v.getValueCount()) {
+        Object value = v.getObject(index);
+        if (value != null) {
+          vv.put(child, value);
+        }
+      }
+    }
+    return vv;
+  }
+
+  @Override
+  public boolean isNull(int index) { return false; }
+  @Override
+  public int getNullCount() { return 0; }
+
+  public void get(int index, ComplexHolder holder) {
+    reader.setPosition(index);
+    holder.reader = reader;
+  }
+
+  @Override
+  public int getValueCount() {
+    return valueCount;
+  }
+
+  public ValueVector getVectorById(int id) {
+  return getChildByOrdinal(id);
+}
+
+  @Override
+  public void setValueCount(int valueCount) {
+    for (final ValueVector v : getChildren()) {
+      v.setValueCount(valueCount);
+    }
+    NonNullableMapVector.this.valueCount = valueCount;
+  }
+
+  @Override
+  public void clear() {
+    for (final ValueVector v : getChildren()) {
+      v.clear();
+    }
+    valueCount = 0;
+  }
+
+  @Override
+  public Field getField() {
+    List<Field> children = new ArrayList<>();
+    for (ValueVector child : getChildren()) {
+      children.add(child.getField());
+    }
+    return new Field(name, fieldType, children);
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.MAP;
+  }
+
+  @Override
+  public void close() {
+    final Collection<FieldVector> vectors = getChildren();
+    for (final FieldVector v : vectors) {
+      v.close();
+    }
+    vectors.clear();
+
+    valueCount = 0;
+
+    super.close();
+  }
+
+  public void initializeChildrenFromFields(List<Field> children) {
+    for (Field field : children) {
+      FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType());
+      vector.initializeChildrenFromFields(field.getChildren());
+    }
+  }
+
+  public List<FieldVector> getChildrenFromFields() {
+    return getChildren();
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
deleted file mode 100644
index e223d1ce67449..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ /dev/null
@@ -1,512 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.memory.BaseAllocator;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
-import org.apache.arrow.vector.complex.impl.NullableMapWriter;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.OversizedAllocationException;
-import org.apache.arrow.vector.util.TransferPair;
-
-public class NullableMapVector extends MapVector implements FieldVector {
-
-  public static NullableMapVector empty(String name, BufferAllocator allocator) {
-    FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
-    return new NullableMapVector(name, allocator, fieldType, null);
-  }
-
-  private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
-  private final NullableMapWriter writer = new NullableMapWriter(this);
-
-  protected ArrowBuf validityBuffer;
-  private int validityAllocationSizeInBytes;
-
-  // deprecated, use FieldType or static constructor instead
-  @Deprecated
-  public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
-    this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack);
-  }
-
-  // deprecated, use FieldType or static constructor instead
-  @Deprecated
-  public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
-    this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack);
-  }
-
-  public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
-    super(name, checkNotNull(allocator), fieldType, callBack);
-    this.validityBuffer = allocator.getEmpty();
-    this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION);
-  }
-
-  @Override
-  public Field getField() {
-    Field f = super.getField();
-    FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata());
-    return new Field(f.getName(), type, f.getChildren());
-  }
-
-  @Override
-  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    if (ownBuffers.size() != 1) {
-      throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size());
-    }
-
-    ArrowBuf bitBuffer = ownBuffers.get(0);
-
-    validityBuffer.release();
-    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
-    valueCount = fieldNode.getLength();
-    validityAllocationSizeInBytes = validityBuffer.capacity();
-  }
-
-  @Override
-  public List<ArrowBuf> getFieldBuffers() {
-    List<ArrowBuf> result = new ArrayList<>(1);
-    setReaderAndWriterIndex();
-    result.add(validityBuffer);
-
-    return result;
-  }
-
-  private void setReaderAndWriterIndex() {
-    validityBuffer.readerIndex(0);
-    validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
-  }
-
-  @Override
-  @Deprecated
-  public List<BufferBacked> getFieldInnerVectors() {
-    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
-  }
-
-  @Override
-  public NullableMapReaderImpl getReader() {
-    return reader;
-  }
-
-  public NullableMapWriter getWriter() {
-    return writer;
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false);
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new NullableMapTransferPair(this, (NullableMapVector) to, true);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false);
-  }
-
-  protected class NullableMapTransferPair extends MapTransferPair {
-
-    private NullableMapVector target;
-
-    protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, boolean allocate) {
-      super(from, to, allocate);
-      this.target = to;
-    }
-
-    @Override
-    public void transfer() {
-      target.clear();
-      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
-      super.transfer();
-      clear();
-    }
-
-    @Override
-    public void copyValueSafe(int fromIndex, int toIndex) {
-      while (toIndex >= target.getValidityBufferValueCapacity()) {
-        target.reallocValidityBuffer();
-      }
-      BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex));
-      super.copyValueSafe(fromIndex, toIndex);
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      target.clear();
-      splitAndTransferValidityBuffer(startIndex, length, target);
-      super.splitAndTransfer(startIndex, length);
-    }
-  }
-
-  /*
-   * transfer the validity.
-   */
-  private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) {
-    assert startIndex + length <= valueCount;
-    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
-    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
-    int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length);
-    int offset = startIndex % 8;
-
-    if (length > 0) {
-      if (offset == 0) {
-        // slice
-        if (target.validityBuffer != null) {
-          target.validityBuffer.release();
-        }
-        target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
-        target.validityBuffer.retain(1);
-      } else {
-        /* Copy data
-         * When the first bit starts from the middle of a byte (offset != 0),
-         * copy data from src BitVector.
-         * Each byte in the target is composed by a part in i-th byte,
-         * another part in (i+1)-th byte.
-         */
-        target.allocateValidityBuffer(byteSizeTarget);
-
-        for (int i = 0; i < byteSizeTarget - 1; i++) {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset);
-          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset);
-
-          target.validityBuffer.setByte(i, (b1 + b2));
-        }
-
-        /* Copying the last piece is done in the following manner:
-         * if the source vector has 1 or more bytes remaining, we copy
-         * the last piece as a byte formed by shifting data
-         * from the current byte and the next byte.
-         *
-         * if the source vector has no more bytes remaining
-         * (we are at the last byte), we copy the last piece as a byte
-         * by shifting data from the current byte.
-         */
-        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
-                  firstByteSource + byteSizeTarget - 1, offset);
-          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
-                  firstByteSource + byteSizeTarget, offset);
-
-          target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
-        } else {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
-                  firstByteSource + byteSizeTarget - 1, offset);
-          target.validityBuffer.setByte(byteSizeTarget - 1, b1);
-        }
-      }
-    }
-  }
-
-  /**
-   * Get the value capacity of the internal validity buffer.
-   * @return number of elements that validity buffer can hold
-   */
-  private int getValidityBufferValueCapacity() {
-    return (int) (validityBuffer.capacity() * 8L);
-  }
-
-  /**
-   * Get the current value capacity for the vector
-   * @return number of elements that vector can hold.
-   */
-  @Override
-  public int getValueCapacity() {
-    return Math.min(getValidityBufferValueCapacity(),
-            super.getValueCapacity());
-  }
-
-  /**
-   * Return the underlying buffers associated with this vector. Note that this doesn't
-   * impact the reference counts for this buffer so it only should be used for in-context
-   * access. Also note that this buffer changes regularly thus
-   * external classes shouldn't hold a reference to it (unless they change it).
-   *
-   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
-   *              but the returned array will be the only reference to them
-   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
-   *         vector instance.
-   */
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    setReaderAndWriterIndex();
-    final ArrowBuf[] buffers;
-    if (getBufferSize() == 0) {
-      buffers = new ArrowBuf[0];
-    } else {
-      buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false),
-              ArrowBuf.class);
-    }
-    if (clear) {
-      for (ArrowBuf buffer : buffers) {
-        buffer.retain();
-      }
-      clear();
-    }
-
-    return buffers;
-  }
-
-  /**
-   * Close the vector and release the associated buffers.
-   */
-  @Override
-  public void close() {
-    clearValidityBuffer();
-    super.close();
-  }
-
-  /**
-   * Same as {@link #close()}
-   */
-  @Override
-  public void clear() {
-    clearValidityBuffer();
-    super.clear();
-  }
-
-  /**
-   * Release the validity buffer
-   */
-  private void clearValidityBuffer() {
-    validityBuffer.release();
-    validityBuffer = allocator.getEmpty();
-  }
-
-  /**
-   * Get the size (number of bytes) of underlying buffers used by this
-   * vector
-   * @return size of underlying buffers.
-   */
-  @Override
-  public int getBufferSize() {
-    if (valueCount == 0) {
-      return 0;
-    }
-    return super.getBufferSize() +
-            BitVectorHelper.getValidityBufferSize(valueCount);
-  }
-
-  /**
-   * Get the potential buffer size for a particular number of records.
-   * @param valueCount desired number of elements in the vector
-   * @return estimated size of underlying buffers if the vector holds
-   *         a given number of elements
-   */
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
-    return super.getBufferSizeFor(valueCount)
-            + BitVectorHelper.getValidityBufferSize(valueCount);
-  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords);
-    super.setInitialCapacity(numRecords);
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    /* Boolean to keep track if all the memory allocations were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      clear();
-      allocateValidityBuffer(validityAllocationSizeInBytes);
-      success = super.allocateNewSafe();
-    } finally {
-      if (!success) {
-        clear();
-        return false;
-      }
-    }
-    return true;
-  }
-
-  private void allocateValidityBuffer(final long size) {
-    final int curSize = (int) size;
-    validityBuffer = allocator.buffer(curSize);
-    validityBuffer.readerIndex(0);
-    validityAllocationSizeInBytes = curSize;
-    validityBuffer.setZero(0, validityBuffer.capacity());
-  }
-
-  @Override
-  public void reAlloc() {
-    /* reallocate the validity buffer */
-    reallocValidityBuffer();
-    super.reAlloc();
-  }
-
-  private void reallocValidityBuffer() {
-    final int currentBufferCapacity = validityBuffer.capacity();
-    long baseSize = validityAllocationSizeInBytes;
-
-    if (baseSize < (long) currentBufferCapacity) {
-      baseSize = (long) currentBufferCapacity;
-    }
-
-    long newAllocationSize = baseSize * 2L;
-    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
-
-    if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Unable to expand the buffer");
-    }
-
-    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
-    newBuf.setZero(0, newBuf.capacity());
-    newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
-    validityBuffer.release(1);
-    validityBuffer = newBuf;
-    validityAllocationSizeInBytes = (int) newAllocationSize;
-  }
-
-  @Override
-  public long getValidityBufferAddress() {
-    return validityBuffer.memoryAddress();
-  }
-
-  @Override
-  public long getDataBufferAddress() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public long getOffsetBufferAddress() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public ArrowBuf getValidityBuffer() {
-    return validityBuffer;
-  }
-
-  @Override
-  public ArrowBuf getDataBuffer() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public ArrowBuf getOffsetBuffer() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public Object getObject(int index) {
-    if (isSet(index) == 0) {
-      return null;
-    } else {
-      return super.getObject(index);
-    }
-  }
-
-  @Override
-  public void get(int index, ComplexHolder holder) {
-    holder.isSet = isSet(index);
-    super.get(index, holder);
-  }
-
-  public int getNullCount() {
-    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
-  }
-
-  public boolean isNull(int index) {
-    return isSet(index) == 0;
-  }
-
-  public int isSet(int index) {
-    final int byteIndex = index >> 3;
-    final byte b = validityBuffer.getByte(byteIndex);
-    final int bitIndex = index & 7;
-    return Long.bitCount(b & (1L << bitIndex));
-  }
-
-  public void setIndexDefined(int index) {
-    while (index >= getValidityBufferValueCapacity()) {
-      /* realloc the inner buffers if needed */
-      reallocValidityBuffer();
-    }
-    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-  }
-
-  public void setNull(int index) {
-    while (index >= getValidityBufferValueCapacity()) {
-      /* realloc the inner buffers if needed */
-      reallocValidityBuffer();
-    }
-    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
-  }
-
-  @Override
-  public void setValueCount(int valueCount) {
-    assert valueCount >= 0;
-    while (valueCount > getValidityBufferValueCapacity()) {
-      /* realloc the inner buffers if needed */
-      reallocValidityBuffer();
-    }
-    super.setValueCount(valueCount);
-    this.valueCount = valueCount;
-  }
-
-  public void reset() {
-    valueCount = 0;
-  }
-
-  @Override
-  @Deprecated
-  public Accessor getAccessor() {
-    throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP");
-  }
-
-  @Override
-  @Deprecated
-  public Mutator getMutator() {
-    throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP");
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index a6960238b9165..970b90ef510fa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -20,7 +20,7 @@
 
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.StateTool;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -131,7 +131,7 @@ public MapWriter directMap() {
     switch (mode) {
 
       case INIT:
-        mapRoot = nullableMapWriterFactory.build((NullableMapVector) container);
+        mapRoot = nullableMapWriterFactory.build((MapVector) container);
         mapRoot.setPosition(idx());
         mode = Mode.MAP;
         break;
@@ -152,7 +152,7 @@ public MapWriter rootAsMap() {
 
       case INIT:
         // TODO allow dictionaries in complex types
-        NullableMapVector map = container.addOrGetMap(name);
+        MapVector map = container.addOrGetMap(name);
         mapRoot = nullableMapWriterFactory.build(map);
         mapRoot.setPosition(idx());
         mode = Mode.MAP;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
index 06b0f4d43272f..acf155af71db2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
@@ -20,17 +20,17 @@
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.pojo.Field;
 
 public class NullableMapReaderImpl extends SingleMapReaderImpl {
 
-  private NullableMapVector nullableMapVector;
+  private MapVector nullableMapVector;
 
   public NullableMapReaderImpl(MapVector vector) {
-    super((NullableMapVector) vector);
-    this.nullableMapVector = (NullableMapVector) vector;
+    super((MapVector) vector);
+    this.nullableMapVector = (MapVector) vector;
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
index d2dcb2374d0e7..06dbf784be0a5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
@@ -18,7 +18,7 @@
 
 package org.apache.arrow.vector.complex.impl;
 
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 
 public class NullableMapWriterFactory {
   private final boolean caseSensitive;
@@ -29,7 +29,7 @@ public NullableMapWriterFactory(boolean caseSensitive) {
     this.caseSensitive = caseSensitive;
   }
 
-  public NullableMapWriter build(NullableMapVector container) {
+  public NullableMapWriter build(MapVector container) {
     return this.caseSensitive ? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 5bd439cac4269..9abd38d20ac62 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -23,7 +23,7 @@
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -94,7 +94,7 @@ private void setWriter(ValueVector v) {
     type = v.getMinorType();
     switch (type) {
       case MAP:
-        writer = nullableMapWriterFactory.build((NullableMapVector) vector);
+        writer = nullableMapWriterFactory.build((MapVector) vector);
         break;
       case LIST:
         writer = new UnionListWriter((ListVector) vector, nullableMapWriterFactory);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index c77ca4e8f23ba..0341b622e0df4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -24,6 +24,7 @@
 
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NonNullableMapVector;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -34,10 +35,10 @@
 @SuppressWarnings("unused")
 public class SingleMapReaderImpl extends AbstractFieldReader {
 
-  private final MapVector vector;
+  private final NonNullableMapVector vector;
   private final Map<String, FieldReader> fields = Maps.newHashMap();
 
-  public SingleMapReaderImpl(MapVector vector) {
+  public SingleMapReaderImpl(NonNullableMapVector vector) {
     this.vector = vector;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
index 6243a2833c055..62bc271257fb2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
@@ -20,7 +20,6 @@
 package org.apache.arrow.vector.complex.impl;
 
 import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.reader.FieldReader;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
index cb11a253058e5..04d50331b76d8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -222,7 +222,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader INT1 = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        final int size = count * NullableTinyIntVector.TYPE_WIDTH;
+        final int size = count * TinyIntVector.TYPE_WIDTH;
         ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
@@ -237,7 +237,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader INT2 = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        final int size = count * NullableSmallIntVector.TYPE_WIDTH;
+        final int size = count * SmallIntVector.TYPE_WIDTH;
         ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
@@ -252,7 +252,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader INT4 = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        final int size = count * NullableIntVector.TYPE_WIDTH;
+        final int size = count * IntVector.TYPE_WIDTH;
         ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
@@ -267,7 +267,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader INT8 = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        final int size = count * NullableBigIntVector.TYPE_WIDTH;
+        final int size = count * BigIntVector.TYPE_WIDTH;
         ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
@@ -282,7 +282,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader FLOAT4 = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        final int size = count * NullableFloat4Vector.TYPE_WIDTH;
+        final int size = count * Float4Vector.TYPE_WIDTH;
         ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
@@ -297,7 +297,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader FLOAT8 = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        final int size = count * NullableFloat8Vector.TYPE_WIDTH;
+        final int size = count * Float8Vector.TYPE_WIDTH;
         ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
@@ -312,7 +312,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     BufferReader DECIMAL = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
-        final int size = count * NullableDecimalVector.TYPE_WIDTH;
+        final int size = count * DecimalVector.TYPE_WIDTH;
         ArrowBuf buf = allocator.buffer(size);
 
         for (int i = 0; i < count; i++) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
index 22423b844e5db..067fb25b8d86c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -160,8 +160,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
       generator.writeObjectField("name", field.getName());
       int valueCount = vector.getValueCount();
       generator.writeObjectField("count", valueCount);
-      final int scale = (vector instanceof NullableDecimalVector) ?
-                            ((NullableDecimalVector) vector).getScale() : 0;
+      final int scale = (vector instanceof DecimalVector) ?
+                            ((DecimalVector) vector).getScale() : 0;
       for (int v = 0; v < vectorTypes.size(); v++) {
         ArrowVectorType vectorType = vectorTypes.get(v);
         ArrowBuf vectorBuffer = vectorBuffers.get(v);
@@ -199,86 +199,86 @@ private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer,
                                      ArrowBuf offsetBuffer, FieldVector vector,
                                      final int index, final int scale) throws IOException {
     if (bufferType.equals(TYPE)) {
-      generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH));
+      generator.writeNumber(buffer.getByte(index * TinyIntVector.TYPE_WIDTH));
     } else if (bufferType.equals(OFFSET)) {
-      generator.writeNumber(buffer.getInt(index * BaseNullableVariableWidthVector.OFFSET_WIDTH));
+      generator.writeNumber(buffer.getInt(index * BaseVariableWidthVector.OFFSET_WIDTH));
     } else if(bufferType.equals(VALIDITY)) {
       generator.writeNumber(vector.isNull(index) ? 0 : 1);
     } else if (bufferType.equals(DATA)) {
       switch (vector.getMinorType()) {
         case TINYINT:
-          generator.writeNumber(NullableTinyIntVector.get(buffer, index));
+          generator.writeNumber(TinyIntVector.get(buffer, index));
           break;
         case SMALLINT:
-          generator.writeNumber(NullableSmallIntVector.get(buffer, index));
+          generator.writeNumber(SmallIntVector.get(buffer, index));
           break;
         case INT:
-          generator.writeNumber(NullableIntVector.get(buffer, index));
+          generator.writeNumber(IntVector.get(buffer, index));
           break;
         case BIGINT:
-          generator.writeNumber(NullableBigIntVector.get(buffer, index));
+          generator.writeNumber(BigIntVector.get(buffer, index));
           break;
         case FLOAT4:
-          generator.writeNumber(NullableFloat4Vector.get(buffer, index));
+          generator.writeNumber(Float4Vector.get(buffer, index));
           break;
         case FLOAT8:
-          generator.writeNumber(NullableFloat8Vector.get(buffer, index));
+          generator.writeNumber(Float8Vector.get(buffer, index));
           break;
         case DATEDAY:
-          generator.writeNumber(NullableDateDayVector.get(buffer, index));
+          generator.writeNumber(DateDayVector.get(buffer, index));
           break;
         case DATEMILLI:
-          generator.writeNumber(NullableDateMilliVector.get(buffer, index));
+          generator.writeNumber(DateMilliVector.get(buffer, index));
           break;
         case TIMESEC:
-          generator.writeNumber(NullableTimeSecVector.get(buffer, index));
+          generator.writeNumber(TimeSecVector.get(buffer, index));
           break;
         case TIMEMILLI:
-          generator.writeNumber(NullableTimeMilliVector.get(buffer, index));
+          generator.writeNumber(TimeMilliVector.get(buffer, index));
           break;
         case TIMEMICRO:
-          generator.writeNumber(NullableTimeMicroVector.get(buffer, index));
+          generator.writeNumber(TimeMicroVector.get(buffer, index));
           break;
         case TIMENANO:
-          generator.writeNumber(NullableTimeNanoVector.get(buffer, index));
+          generator.writeNumber(TimeNanoVector.get(buffer, index));
           break;
         case TIMESTAMPSEC:
-          generator.writeNumber(NullableTimeStampSecVector.get(buffer, index));
+          generator.writeNumber(TimeStampSecVector.get(buffer, index));
           break;
         case TIMESTAMPMILLI:
-          generator.writeNumber(NullableTimeStampMilliVector.get(buffer, index));
+          generator.writeNumber(TimeStampMilliVector.get(buffer, index));
           break;
         case TIMESTAMPMICRO:
-          generator.writeNumber(NullableTimeStampMicroVector.get(buffer, index));
+          generator.writeNumber(TimeStampMicroVector.get(buffer, index));
           break;
         case TIMESTAMPNANO:
-          generator.writeNumber(NullableTimeStampNanoVector.get(buffer, index));
+          generator.writeNumber(TimeStampNanoVector.get(buffer, index));
           break;
         case TIMESTAMPSECTZ:
-          generator.writeNumber(NullableTimeStampSecTZVector.get(buffer, index));
+          generator.writeNumber(TimeStampSecTZVector.get(buffer, index));
           break;
         case TIMESTAMPMILLITZ:
-          generator.writeNumber(NullableTimeStampMilliTZVector.get(buffer, index));
+          generator.writeNumber(TimeStampMilliTZVector.get(buffer, index));
           break;
         case TIMESTAMPMICROTZ:
-          generator.writeNumber(NullableTimeStampMicroTZVector.get(buffer, index));
+          generator.writeNumber(TimeStampMicroTZVector.get(buffer, index));
           break;
         case TIMESTAMPNANOTZ:
-          generator.writeNumber(NullableTimeStampNanoTZVector.get(buffer, index));
+          generator.writeNumber(TimeStampNanoTZVector.get(buffer, index));
           break;
         case BIT:
           generator.writeNumber(BitVectorHelper.get(buffer, index));
           break;
         case VARBINARY: {
           assert offsetBuffer != null;
-          String hexString = Hex.encodeHexString(BaseNullableVariableWidthVector.get(buffer,
+          String hexString = Hex.encodeHexString(BaseVariableWidthVector.get(buffer,
                   offsetBuffer, index));
           generator.writeObject(hexString);
           break;
         }
         case VARCHAR: {
           assert offsetBuffer != null;
-          byte[] b = (BaseNullableVariableWidthVector.get(buffer, offsetBuffer, index));
+          byte[] b = (BaseVariableWidthVector.get(buffer, offsetBuffer, index));
           generator.writeString(new String(b, "UTF-8"));
           break;
         }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index c57dd6dafe9e6..7834845aad80e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -24,41 +24,41 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.NullableBigIntVector;
-import org.apache.arrow.vector.NullableBitVector;
-import org.apache.arrow.vector.NullableDateDayVector;
-import org.apache.arrow.vector.NullableDateMilliVector;
-import org.apache.arrow.vector.NullableDecimalVector;
-import org.apache.arrow.vector.NullableFloat4Vector;
-import org.apache.arrow.vector.NullableFloat8Vector;
-import org.apache.arrow.vector.NullableIntVector;
-import org.apache.arrow.vector.NullableIntervalDayVector;
-import org.apache.arrow.vector.NullableIntervalYearVector;
-import org.apache.arrow.vector.NullableSmallIntVector;
-import org.apache.arrow.vector.NullableTimeMicroVector;
-import org.apache.arrow.vector.NullableTimeMilliVector;
-import org.apache.arrow.vector.NullableTimeNanoVector;
-import org.apache.arrow.vector.NullableTimeSecVector;
-import org.apache.arrow.vector.NullableTimeStampMicroTZVector;
-import org.apache.arrow.vector.NullableTimeStampMicroVector;
-import org.apache.arrow.vector.NullableTimeStampMilliTZVector;
-import org.apache.arrow.vector.NullableTimeStampMilliVector;
-import org.apache.arrow.vector.NullableTimeStampNanoTZVector;
-import org.apache.arrow.vector.NullableTimeStampNanoVector;
-import org.apache.arrow.vector.NullableTimeStampSecTZVector;
-import org.apache.arrow.vector.NullableTimeStampSecVector;
-import org.apache.arrow.vector.NullableTinyIntVector;
-import org.apache.arrow.vector.NullableUInt1Vector;
-import org.apache.arrow.vector.NullableUInt2Vector;
-import org.apache.arrow.vector.NullableUInt4Vector;
-import org.apache.arrow.vector.NullableUInt8Vector;
-import org.apache.arrow.vector.NullableVarBinaryVector;
-import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.DateDayVector;
+import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.IntervalDayVector;
+import org.apache.arrow.vector.IntervalYearVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TimeMicroVector;
+import org.apache.arrow.vector.TimeMilliVector;
+import org.apache.arrow.vector.TimeNanoVector;
+import org.apache.arrow.vector.TimeSecVector;
+import org.apache.arrow.vector.TimeStampMicroTZVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampMilliTZVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampNanoTZVector;
+import org.apache.arrow.vector.TimeStampNanoVector;
+import org.apache.arrow.vector.TimeStampSecTZVector;
+import org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.UInt2Vector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.UInt8Vector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.BitWriterImpl;
@@ -131,304 +131,304 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     MAP(Struct.INSTANCE) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableMapVector(name, allocator, fieldType, schemaChangeCallback);
+        return new MapVector(name, allocator, fieldType, schemaChangeCallback);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new NullableMapWriter((NullableMapVector) vector);
+        return new NullableMapWriter((MapVector) vector);
       }
     },
     TINYINT(new Int(8, true)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTinyIntVector(name, fieldType, allocator);
+        return new TinyIntVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TinyIntWriterImpl((NullableTinyIntVector) vector);
+        return new TinyIntWriterImpl((TinyIntVector) vector);
       }
     },
     SMALLINT(new Int(16, true)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableSmallIntVector(name, fieldType, allocator);
+        return new SmallIntVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new SmallIntWriterImpl((NullableSmallIntVector) vector);
+        return new SmallIntWriterImpl((SmallIntVector) vector);
       }
     },
     INT(new Int(32, true)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableIntVector(name, fieldType, allocator);
+        return new IntVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new IntWriterImpl((NullableIntVector) vector);
+        return new IntWriterImpl((IntVector) vector);
       }
     },
     BIGINT(new Int(64, true)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableBigIntVector(name, fieldType, allocator);
+        return new BigIntVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new BigIntWriterImpl((NullableBigIntVector) vector);
+        return new BigIntWriterImpl((BigIntVector) vector);
       }
     },
     DATEDAY(new Date(DateUnit.DAY)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableDateDayVector(name, fieldType, allocator);
+        return new DateDayVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new DateDayWriterImpl((NullableDateDayVector) vector);
+        return new DateDayWriterImpl((DateDayVector) vector);
       }
     },
     DATEMILLI(new Date(DateUnit.MILLISECOND)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableDateMilliVector(name, fieldType, allocator);
+        return new DateMilliVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new DateMilliWriterImpl((NullableDateMilliVector) vector);
+        return new DateMilliWriterImpl((DateMilliVector) vector);
       }
     },
     TIMESEC(new Time(TimeUnit.SECOND, 32)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeSecVector(name, fieldType, allocator);
+        return new TimeSecVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeSecWriterImpl((NullableTimeSecVector) vector);
+        return new TimeSecWriterImpl((TimeSecVector) vector);
       }
     },
     TIMEMILLI(new Time(TimeUnit.MILLISECOND, 32)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeMilliVector(name, fieldType, allocator);
+        return new TimeMilliVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeMilliWriterImpl((NullableTimeMilliVector) vector);
+        return new TimeMilliWriterImpl((TimeMilliVector) vector);
       }
     },
     TIMEMICRO(new Time(TimeUnit.MICROSECOND, 64)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeMicroVector(name, fieldType, allocator);
+        return new TimeMicroVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeMicroWriterImpl((NullableTimeMicroVector) vector);
+        return new TimeMicroWriterImpl((TimeMicroVector) vector);
       }
     },
     TIMENANO(new Time(TimeUnit.NANOSECOND, 64)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeNanoVector(name, fieldType, allocator);
+        return new TimeNanoVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeNanoWriterImpl((NullableTimeNanoVector) vector);
+        return new TimeNanoWriterImpl((TimeNanoVector) vector);
       }
     },
     // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
     TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeStampSecVector(name, fieldType, allocator);
+        return new TimeStampSecVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampSecWriterImpl((NullableTimeStampSecVector) vector);
+        return new TimeStampSecWriterImpl((TimeStampSecVector) vector);
       }
     },
     // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
     TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeStampMilliVector(name, fieldType, allocator);
+        return new TimeStampMilliVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampMilliWriterImpl((NullableTimeStampMilliVector) vector);
+        return new TimeStampMilliWriterImpl((TimeStampMilliVector) vector);
       }
     },
     // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
     TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeStampMicroVector(name, fieldType, allocator);
+        return new TimeStampMicroVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampMicroWriterImpl((NullableTimeStampMicroVector) vector);
+        return new TimeStampMicroWriterImpl((TimeStampMicroVector) vector);
       }
     },
     // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
     TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeStampNanoVector(name, fieldType, allocator);
+        return new TimeStampNanoVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampNanoWriterImpl((NullableTimeStampNanoVector) vector);
+        return new TimeStampNanoWriterImpl((TimeStampNanoVector) vector);
       }
     },
     INTERVALDAY(new Interval(IntervalUnit.DAY_TIME)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableIntervalDayVector(name, fieldType, allocator);
+        return new IntervalDayVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new IntervalDayWriterImpl((NullableIntervalDayVector) vector);
+        return new IntervalDayWriterImpl((IntervalDayVector) vector);
       }
     },
     INTERVALYEAR(new Interval(IntervalUnit.YEAR_MONTH)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableIntervalYearVector(name, fieldType, allocator);
+        return new IntervalYearVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new IntervalYearWriterImpl((NullableIntervalYearVector) vector);
+        return new IntervalYearWriterImpl((IntervalYearVector) vector);
       }
     },
     //  4 byte ieee 754
     FLOAT4(new FloatingPoint(SINGLE)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableFloat4Vector(name, fieldType, allocator);
+        return new Float4Vector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new Float4WriterImpl((NullableFloat4Vector) vector);
+        return new Float4WriterImpl((Float4Vector) vector);
       }
     },
     //  8 byte ieee 754
     FLOAT8(new FloatingPoint(DOUBLE)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableFloat8Vector(name, fieldType, allocator);
+        return new Float8Vector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new Float8WriterImpl((NullableFloat8Vector) vector);
+        return new Float8WriterImpl((Float8Vector) vector);
       }
     },
     BIT(Bool.INSTANCE) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableBitVector(name, fieldType, allocator);
+        return new BitVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new BitWriterImpl((NullableBitVector) vector);
+        return new BitWriterImpl((BitVector) vector);
       }
     },
     VARCHAR(Utf8.INSTANCE) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableVarCharVector(name, fieldType, allocator);
+        return new VarCharVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new VarCharWriterImpl((NullableVarCharVector) vector);
+        return new VarCharWriterImpl((VarCharVector) vector);
       }
     },
     VARBINARY(Binary.INSTANCE) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableVarBinaryVector(name, fieldType, allocator);
+        return new VarBinaryVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new VarBinaryWriterImpl((NullableVarBinaryVector) vector);
+        return new VarBinaryWriterImpl((VarBinaryVector) vector);
       }
     },
     DECIMAL(null) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableDecimalVector(name, fieldType, allocator);
+        return new DecimalVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new DecimalWriterImpl((NullableDecimalVector) vector);
+        return new DecimalWriterImpl((DecimalVector) vector);
       }
     },
     UINT1(new Int(8, false)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableUInt1Vector(name, fieldType, allocator);
+        return new UInt1Vector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new UInt1WriterImpl((NullableUInt1Vector) vector);
+        return new UInt1WriterImpl((UInt1Vector) vector);
       }
     },
     UINT2(new Int(16, false)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableUInt2Vector(name, fieldType, allocator);
+        return new UInt2Vector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new UInt2WriterImpl((NullableUInt2Vector) vector);
+        return new UInt2WriterImpl((UInt2Vector) vector);
       }
     },
     UINT4(new Int(32, false)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableUInt4Vector(name, fieldType, allocator);
+        return new UInt4Vector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new UInt4WriterImpl((NullableUInt4Vector) vector);
+        return new UInt4WriterImpl((UInt4Vector) vector);
       }
     },
     UINT8(new Int(64, false)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableUInt8Vector(name, fieldType, allocator);
+        return new UInt8Vector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new UInt8WriterImpl((NullableUInt8Vector) vector);
+        return new UInt8WriterImpl((UInt8Vector) vector);
       }
     },
     LIST(List.INSTANCE) {
@@ -470,45 +470,45 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     TIMESTAMPSECTZ(null) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeStampSecTZVector(name, fieldType, allocator);
+        return new TimeStampSecTZVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampSecTZWriterImpl((NullableTimeStampSecTZVector) vector);
+        return new TimeStampSecTZWriterImpl((TimeStampSecTZVector) vector);
       }
     },
     TIMESTAMPMILLITZ(null) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeStampMilliTZVector(name, fieldType, allocator);
+        return new TimeStampMilliTZVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampMilliTZWriterImpl((NullableTimeStampMilliTZVector) vector);
+        return new TimeStampMilliTZWriterImpl((TimeStampMilliTZVector) vector);
       }
     },
     TIMESTAMPMICROTZ(null) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeStampMicroTZVector(name, fieldType, allocator);
+        return new TimeStampMicroTZVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampMicroTZWriterImpl((NullableTimeStampMicroTZVector) vector);
+        return new TimeStampMicroTZWriterImpl((TimeStampMicroTZVector) vector);
       }
     },
     TIMESTAMPNANOTZ(null) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableTimeStampNanoTZVector(name, fieldType, allocator);
+        return new TimeStampNanoTZVector(name, fieldType, allocator);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new TimeStampNanoTZWriterImpl((NullableTimeStampNanoTZVector) vector);
+        return new TimeStampNanoTZWriterImpl((TimeStampNanoTZVector) vector);
       }
     };
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index acf7c58a1337f..c6d734d18d89b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -19,7 +19,6 @@
 package org.apache.arrow.vector.util;
 
 import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 
 import java.math.BigDecimal;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
index ada341497878e..36365fa9d4063 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -55,17 +55,17 @@ public void testBitVectorCopyFromSafe() {
       dst.allocateNew(10);
 
       for (int i = 0; i < size; i++) {
-        src.getMutator().set(i, i % 2);
+        src.set(i, i % 2);
       }
-      src.getMutator().setValueCount(size);
+      src.setValueCount(size);
 
       for (int i = 0; i < size; i++) {
         dst.copyFromSafe(i, i, src);
       }
-      dst.getMutator().setValueCount(size);
+      dst.setValueCount(size);
 
       for (int i = 0; i < size; i++) {
-        assertEquals(src.getAccessor().getObject(i), dst.getAccessor().getObject(i));
+        assertEquals(src.getObject(i), dst.getObject(i));
       }
     }
   }
@@ -74,25 +74,23 @@ public void testBitVectorCopyFromSafe() {
   public void testSplitAndTransfer() throws Exception {
 
     try (final BitVector sourceVector = new BitVector("bitvector", allocator)) {
-      final BitVector.Mutator sourceMutator = sourceVector.getMutator();
-      final BitVector.Accessor sourceAccessor = sourceVector.getAccessor();
 
       sourceVector.allocateNew(40);
 
       /* populate the bitvector -- 010101010101010101010101..... */
       for (int i = 0; i < 40; i++) {
         if ((i & 1) == 1) {
-          sourceMutator.set(i, 1);
+          sourceVector.set(i, 1);
         } else {
-          sourceMutator.set(i, 0);
+          sourceVector.set(i, 0);
         }
       }
 
-      sourceMutator.setValueCount(40);
+      sourceVector.setValueCount(40);
 
       /* check the vector output */
       for (int i = 0; i < 40; i++) {
-        int result = sourceAccessor.get(i);
+        int result = sourceVector.get(i);
         if ((i & 1) == 1) {
           assertEquals(Integer.toString(1), Integer.toString(result));
         } else {
@@ -102,8 +100,6 @@ public void testSplitAndTransfer() throws Exception {
 
       try (final BitVector toVector = new BitVector("toVector", allocator)) {
         final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
-        final BitVector.Accessor toAccessor = toVector.getAccessor();
-        final BitVector.Mutator toMutator = toVector.getMutator();
 
         /*
          * form test cases such that we cover:
@@ -123,8 +119,8 @@ public void testSplitAndTransfer() throws Exception {
 
           /* check the toVector output after doing splitAndTransfer */
           for (int i = 0; i < length; i++) {
-            int actual = toAccessor.get(i);
-            int expected = sourceAccessor.get(start + i);
+            int actual = toVector.get(i);
+            int expected = sourceVector.get(start + i);
             assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i,
                     expected, actual);
           }
@@ -137,28 +133,24 @@ public void testSplitAndTransfer() throws Exception {
   public void testSplitAndTransfer1() throws Exception {
 
     try (final BitVector sourceVector = new BitVector("bitvector", allocator)) {
-      final BitVector.Mutator sourceMutator = sourceVector.getMutator();
-      final BitVector.Accessor sourceAccessor = sourceVector.getAccessor();
 
       sourceVector.allocateNew(8190);
 
       /* populate the bitvector */
       for (int i = 0; i < 8190; i++) {
-        sourceMutator.set(i, 1);
+        sourceVector.set(i, 1);
       }
 
-      sourceMutator.setValueCount(8190);
+      sourceVector.setValueCount(8190);
 
       /* check the vector output */
       for (int i = 0; i < 8190; i++) {
-        int result = sourceAccessor.get(i);
+        int result = sourceVector.get(i);
         assertEquals(Integer.toString(1), Integer.toString(result));
       }
 
       try (final BitVector toVector = new BitVector("toVector", allocator)) {
         final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
-        final BitVector.Accessor toAccessor = toVector.getAccessor();
-        final BitVector.Mutator toMutator = toVector.getMutator();
 
         final int[][] transferLengths = {{0, 4095}, {4095, 4095}};
 
@@ -170,8 +162,8 @@ public void testSplitAndTransfer1() throws Exception {
 
           /* check the toVector output after doing splitAndTransfer */
           for (int i = 0; i < length; i++) {
-            int actual = toAccessor.get(i);
-            int expected = sourceAccessor.get(start + i);
+            int actual = toVector.get(i);
+            int expected = sourceVector.get(start + i);
             assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i,
                     expected, actual);
           }
@@ -184,25 +176,23 @@ public void testSplitAndTransfer1() throws Exception {
   public void testSplitAndTransfer2() throws Exception {
 
     try (final BitVector sourceVector = new BitVector("bitvector", allocator)) {
-      final BitVector.Mutator sourceMutator = sourceVector.getMutator();
-      final BitVector.Accessor sourceAccessor = sourceVector.getAccessor();
 
       sourceVector.allocateNew(32);
 
       /* populate the bitvector */
       for (int i = 0; i < 32; i++) {
         if ((i & 1) == 1) {
-          sourceMutator.set(i, 1);
+          sourceVector.set(i, 1);
         } else {
-          sourceMutator.set(i, 0);
+          sourceVector.set(i, 0);
         }
       }
 
-      sourceMutator.setValueCount(32);
+      sourceVector.setValueCount(32);
 
       /* check the vector output */
       for (int i = 0; i < 32; i++) {
-        int result = sourceAccessor.get(i);
+        int result = sourceVector.get(i);
         if ((i & 1) == 1) {
           assertEquals(Integer.toString(1), Integer.toString(result));
         } else {
@@ -212,8 +202,6 @@ public void testSplitAndTransfer2() throws Exception {
 
       try (final BitVector toVector = new BitVector("toVector", allocator)) {
         final TransferPair transferPair = sourceVector.makeTransferPair(toVector);
-        final BitVector.Accessor toAccessor = toVector.getAccessor();
-        final BitVector.Mutator toMutator = toVector.getMutator();
 
         final int[][] transferLengths = {{5,22}, {5,24}, {5,25}, {5,27}, {0,31}, {5,7}, {2,3}};
 
@@ -225,8 +213,8 @@ public void testSplitAndTransfer2() throws Exception {
 
           /* check the toVector output after doing splitAndTransfer */
           for (int i = 0; i < length; i++) {
-            int actual = toAccessor.get(i);
-            int expected = sourceAccessor.get(start + i);
+            int actual = toVector.get(i);
+            int expected = sourceVector.get(start + i);
             assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i,
                     expected, actual);
           }
@@ -242,62 +230,56 @@ public void testReallocAfterVectorTransfer1() {
       int valueCapacity = vector.getValueCapacity();
       assertEquals(4096, valueCapacity);
 
-      final BitVector.Mutator mutator = vector.getMutator();
-      final BitVector.Accessor accessor = vector.getAccessor();
-
       for (int i = 0; i < valueCapacity; i++) {
         if ((i & 1) == 1) {
-          mutator.setToOne(i);
+          vector.set(i, 1);
         }
       }
 
       for (int i = 0; i < valueCapacity; i++) {
-        int val = accessor.get(i);
         if ((i & 1) == 1) {
-          assertEquals("unexpected cleared bit at index: " + i, 1, val);
+          assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i));
         }
         else {
-          assertEquals("unexpected set bit at index: " + i, 0, val);
+          assertTrue("unexpected set bit at index: " + i, vector.isNull(i));
         }
       }
 
       /* trigger first realloc */
-      mutator.setSafeToOne(valueCapacity);
+      vector.setSafe(valueCapacity, 1);
       assertEquals(valueCapacity * 2, vector.getValueCapacity());
 
       for (int i = valueCapacity; i < valueCapacity*2; i++) {
         if ((i & 1) == 1) {
-          mutator.setToOne(i);
+          vector.set(i, 1);
         }
       }
 
       for (int i = 0; i < valueCapacity*2; i++) {
-        int val = accessor.get(i);
         if (((i & 1) == 1) || (i == valueCapacity)) {
-          assertEquals("unexpected cleared bit at index: " + i, 1, val);
+          assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i));
         }
         else {
-          assertEquals("unexpected set bit at index: " + i, 0, val);
+          assertTrue("unexpected set bit at index: " + i, vector.isNull(i));
         }
       }
 
       /* trigger second realloc */
-      mutator.setSafeToOne(valueCapacity*2);
+      vector.setSafe(valueCapacity*2, 1);
       assertEquals(valueCapacity * 4, vector.getValueCapacity());
 
       for (int i = valueCapacity*2; i < valueCapacity*4; i++) {
         if ((i & 1) == 1) {
-          mutator.setToOne(i);
+          vector.set(i, 1);
         }
       }
 
       for (int i = 0; i < valueCapacity*4; i++) {
-        int val = accessor.get(i);
         if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2)) {
-          assertEquals("unexpected cleared bit at index: " + i, 1, val);
+          assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i));
         }
         else {
-          assertEquals("unexpected set bit at index: " + i, 0, val);
+          assertTrue("unexpected set bit at index: " + i, vector.isNull(i));
         }
       }
 
@@ -305,27 +287,24 @@ public void testReallocAfterVectorTransfer1() {
       TransferPair transferPair = vector.getTransferPair(allocator);
       transferPair.transfer();
       final BitVector toVector = (BitVector)transferPair.getTo();
-      final BitVector.Accessor toAccessor = toVector.getAccessor();
-      final BitVector.Mutator toMutator = toVector.getMutator();
 
       assertEquals(valueCapacity * 4, toVector.getValueCapacity());
 
       /* realloc the toVector */
-      toMutator.setSafeToOne(valueCapacity * 4);
+      toVector.setSafe(valueCapacity * 4, 1);
 
       for (int i = 0; i < toVector.getValueCapacity(); i++) {
-        int val = toAccessor.get(i);
         if (i <= valueCapacity * 4) {
           if (((i & 1) == 1) || (i == valueCapacity) ||
                   (i == valueCapacity*2) || (i == valueCapacity*4)) {
-            assertEquals("unexpected cleared bit at index: " + i, 1, val);
+            assertEquals("unexpected cleared bit at index: " + i, 1, toVector.get(i));
           }
           else {
-            assertEquals("unexpected set bit at index: " + i, 0, val);
+            assertTrue("unexpected set bit at index: " + i, toVector.isNull(i));
           }
         }
         else {
-          assertEquals("unexpected set bit at index: " + i, 0, val);
+          assertTrue("unexpected set bit at index: " + i, toVector.isNull(i));
         }
       }
 
@@ -335,7 +314,7 @@ public void testReallocAfterVectorTransfer1() {
 
   @Test
   public void testReallocAfterVectorTransfer2() {
-    try (final NullableBitVector vector = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
       vector.allocateNew(4096);
       int valueCapacity = vector.getValueCapacity();
       assertEquals(4096, valueCapacity);
@@ -396,7 +375,7 @@ public void testReallocAfterVectorTransfer2() {
       /* now transfer the vector */
       TransferPair transferPair = vector.getTransferPair(allocator);
       transferPair.transfer();
-      final NullableBitVector toVector = (NullableBitVector)transferPair.getTo();
+      final BitVector toVector = (BitVector)transferPair.getTo();
 
       assertEquals(valueCapacity * 4, toVector.getValueCapacity());
 
@@ -426,81 +405,81 @@ public void testReallocAfterVectorTransfer2() {
   public void testBitVector() {
     // Create a new value vector for 1024 integers
     try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
-      final BitVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
-      m.setValueCount(1024);
+      vector.setValueCount(1024);
 
       // Put and set a few values
-      m.set(0, 1);
-      m.set(1, 0);
-      m.set(100, 0);
-      m.set(1022, 1);
+      vector.set(0, 1);
+      vector.set(1, 0);
+      vector.set(100, 0);
+      vector.set(1022, 1);
 
-      m.setValueCount(1024);
+      vector.setValueCount(1024);
 
-      final BitVector.Accessor accessor = vector.getAccessor();
-      assertEquals(1, accessor.get(0));
-      assertEquals(0, accessor.get(1));
-      assertEquals(0, accessor.get(100));
-      assertEquals(1, accessor.get(1022));
+      assertEquals(1, vector.get(0));
+      assertEquals(0, vector.get(1));
+      assertEquals(0, vector.get(100));
+      assertEquals(1, vector.get(1022));
 
-      assertEquals(1022, accessor.getNullCount());
+      assertEquals(1020, vector.getNullCount());
 
       // test setting the same value twice
-      m.set(0, 1);
-      m.set(0, 1);
-      m.set(1, 0);
-      m.set(1, 0);
-      assertEquals(1, accessor.get(0));
-      assertEquals(0, accessor.get(1));
+      vector.set(0, 1);
+      vector.set(0, 1);
+      vector.set(1, 0);
+      vector.set(1, 0);
+      assertEquals(1, vector.get(0));
+      assertEquals(0, vector.get(1));
 
       // test toggling the values
-      m.set(0, 0);
-      m.set(1, 1);
-      assertEquals(0, accessor.get(0));
-      assertEquals(1, accessor.get(1));
+      vector.set(0, 0);
+      vector.set(1, 1);
+      assertEquals(0, vector.get(0));
+      assertEquals(1, vector.get(1));
 
       // should not change
-      assertEquals(1022, accessor.getNullCount());
+      assertEquals(1020, vector.getNullCount());
 
-      // Ensure unallocated space returns 0
-      assertEquals(0, accessor.get(3));
+      // Ensure null value
+      assertTrue(vector.isNull(3));
 
       // unset the previously set bits
-      m.set(1, 0);
-      m.set(1022, 0);
+      vector.setNull(0);
+      vector.setNull(1);
+      vector.setNull(100);
+      vector.setNull(1022);
       // this should set all the array to 0
-      assertEquals(1024, accessor.getNullCount());
+      assertEquals(1024, vector.getNullCount());
 
       // set all the array to 1
       for (int i = 0; i < 1024; ++i) {
-        assertEquals(1024 - i, accessor.getNullCount());
-        m.set(i, 1);
+        assertEquals(1024 - i, vector.getNullCount());
+        vector.set(i, 1);
       }
 
-      assertEquals(0, accessor.getNullCount());
+      assertEquals(0, vector.getNullCount());
 
       vector.allocateNew(1015);
-      m.setValueCount(1015);
+      vector.setValueCount(1015);
 
       // ensure it has been zeroed
-      assertEquals(1015, accessor.getNullCount());
+      assertEquals(1015, vector.getNullCount());
 
-      m.set(0, 1);
-      m.set(1014, 1); // ensure that the last item of the last byte is allocated
+      vector.set(0, 1);
+      vector.set(1014, 1); // ensure that the last item of the last byte is allocated
 
-      assertEquals(1013, accessor.getNullCount());
+      assertEquals(1013, vector.getNullCount());
 
       vector.zeroVector();
-      assertEquals(1015, accessor.getNullCount());
+      assertEquals(1015, vector.getNullCount());
 
       // set all the array to 1
       for (int i = 0; i < 1015; ++i) {
-        assertEquals(1015 - i, accessor.getNullCount());
-        m.set(i, 1);
+        assertEquals(1015 - i, vector.getNullCount());
+        vector.set(i, 1);
       }
 
-      assertEquals(0, accessor.getNullCount());
+      assertEquals(0, vector.getNullCount());
     }
   }
 
@@ -526,15 +505,17 @@ private void validateRange(int length, int start, int count) {
     try (BitVector bitVector = new BitVector("bits", allocator)) {
       bitVector.reset();
       bitVector.allocateNew(length);
-      bitVector.getMutator().setRangeToOne(start, count);
+      for (int i = start; i < start + count; i++) {
+        bitVector.set(i, 1);
+      }
       for (int i = 0; i < start; i++) {
-        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+        Assert.assertTrue(desc + i, bitVector.isNull(i));
       }
       for (int i = start; i < start + count; i++) {
-        Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i));
+        Assert.assertEquals(desc + i, 1, bitVector.get(i));
       }
       for (int i = start + count; i < length; i++) {
-        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+        Assert.assertTrue(desc + i, bitVector.isNull(i));
       }
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
index 4fe2861450a34..20f6754be2def 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
@@ -38,11 +38,11 @@ public void testTransferFixedWidth() {
     BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 100000, 100000);
     BufferAllocator childAllocator2 = allocator.newChildAllocator("child2", 100000, 100000);
 
-    NullableIntVector v1 = new NullableIntVector("v1", childAllocator1);
+    IntVector v1 = new IntVector("v1", childAllocator1);
     v1.allocateNew();
     v1.setValueCount(4095);
 
-    NullableIntVector v2 = new NullableIntVector("v2", childAllocator2);
+    IntVector v2 = new IntVector("v2", childAllocator2);
 
     v1.makeTransferPair(v2).transfer();
 
@@ -58,12 +58,12 @@ public void testTransferVariableidth() {
     BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 100000, 100000);
     BufferAllocator childAllocator2 = allocator.newChildAllocator("child2", 100000, 100000);
 
-    NullableVarCharVector v1 = new NullableVarCharVector("v1", childAllocator1);
+    VarCharVector v1 = new VarCharVector("v1", childAllocator1);
     v1.allocateNew();
     v1.setSafe(4094, "hello world".getBytes(), 0, 11);
     v1.setValueCount(4001);
 
-    NullableVarCharVector v2 = new NullableVarCharVector("v2", childAllocator2);
+    VarCharVector v2 = new VarCharVector("v2", childAllocator2);
 
     v1.makeTransferPair(v2).transfer();
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java
index 7cdfbdf452d84..87ffcafecd198 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java
@@ -70,8 +70,8 @@ public void terminate() throws Exception {
 
   @Test /* NullableVarChar */
   public void testCopyFromWithNulls() {
-    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator);
-         final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) {
+    try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator);
+         final VarCharVector vector2 = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) {
 
       vector.allocateNew();
       int capacity = vector.getValueCapacity();
@@ -130,8 +130,8 @@ public void testCopyFromWithNulls() {
 
   @Test /* NullableVarChar */
   public void testCopyFromWithNulls1() {
-    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator);
-         final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) {
+    try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator);
+         final VarCharVector vector2 = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) {
 
       vector.allocateNew();
       int capacity = vector.getValueCapacity();
@@ -192,10 +192,10 @@ public void testCopyFromWithNulls1() {
     }
   }
 
-  @Test /* NullableIntVector */
+  @Test /* IntVector */
   public void testCopyFromWithNulls2() {
-    try (final NullableIntVector vector1 = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableIntVector vector2 = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final IntVector vector1 = new IntVector(EMPTY_SCHEMA_PATH, allocator);
+         final IntVector vector2 = new IntVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -252,10 +252,10 @@ public void testCopyFromWithNulls2() {
     }
   }
 
-  @Test /* NullableBigIntVector */
+  @Test /* BigIntVector */
   public void testCopyFromWithNulls3() {
-    try (final NullableBigIntVector vector1 = new NullableBigIntVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableBigIntVector vector2 = new NullableBigIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final BigIntVector vector1 = new BigIntVector(EMPTY_SCHEMA_PATH, allocator);
+         final BigIntVector vector2 = new BigIntVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -314,10 +314,10 @@ public void testCopyFromWithNulls3() {
     }
   }
 
-  @Test /* NullableBitVector */
+  @Test /* BitVector */
   public void testCopyFromWithNulls4() {
-    try (final NullableBitVector vector1 = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableBitVector vector2 = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final BitVector vector1 = new BitVector(EMPTY_SCHEMA_PATH, allocator);
+         final BitVector vector2 = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -392,10 +392,10 @@ public void testCopyFromWithNulls4() {
     }
   }
 
-  @Test /* NullableFloat4Vector */
+  @Test /* Float4Vector */
   public void testCopyFromWithNulls5() {
-    try (final NullableFloat4Vector vector1 = new NullableFloat4Vector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableFloat4Vector vector2 = new NullableFloat4Vector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final Float4Vector vector1 = new Float4Vector(EMPTY_SCHEMA_PATH, allocator);
+         final Float4Vector vector2 = new Float4Vector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -454,10 +454,10 @@ public void testCopyFromWithNulls5() {
     }
   }
 
-  @Test /* NullableFloat8Vector */
+  @Test /* Float8Vector */
   public void testCopyFromWithNulls6() {
-    try (final NullableFloat8Vector vector1 = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableFloat8Vector vector2 = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final Float8Vector vector1 = new Float8Vector(EMPTY_SCHEMA_PATH, allocator);
+         final Float8Vector vector2 = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -516,10 +516,10 @@ public void testCopyFromWithNulls6() {
     }
   }
 
-  @Test /* NullableIntervalDayVector */
+  @Test /* IntervalDayVector */
   public void testCopyFromWithNulls7() {
-    try (final NullableIntervalDayVector vector1 = new NullableIntervalDayVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableIntervalDayVector vector2 = new NullableIntervalDayVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final IntervalDayVector vector1 = new IntervalDayVector(EMPTY_SCHEMA_PATH, allocator);
+         final IntervalDayVector vector2 = new IntervalDayVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -582,10 +582,10 @@ public void testCopyFromWithNulls7() {
     }
   }
 
-  @Test /* NullableIntervalYearVector */
+  @Test /* IntervalYearVector */
   public void testCopyFromWithNulls8() {
-    try (final NullableIntervalYearVector vector1 = new NullableIntervalYearVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableIntervalYearVector vector2 = new NullableIntervalYearVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final IntervalYearVector vector1 = new IntervalYearVector(EMPTY_SCHEMA_PATH, allocator);
+         final IntervalYearVector vector2 = new IntervalYearVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -651,10 +651,10 @@ public void testCopyFromWithNulls8() {
     }
   }
 
-  @Test /* NullableSmallIntVector */
+  @Test /* SmallIntVector */
   public void testCopyFromWithNulls9() {
-    try (final NullableSmallIntVector vector1 = new NullableSmallIntVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableSmallIntVector vector2 = new NullableSmallIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final SmallIntVector vector1 = new SmallIntVector(EMPTY_SCHEMA_PATH, allocator);
+         final SmallIntVector vector2 = new SmallIntVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -714,10 +714,10 @@ public void testCopyFromWithNulls9() {
     }
   }
 
-  @Test /* NullableTimeMicroVector */
+  @Test /* TimeMicroVector */
   public void testCopyFromWithNulls10() {
-    try (final NullableTimeMicroVector vector1 = new NullableTimeMicroVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableTimeMicroVector vector2 = new NullableTimeMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final TimeMicroVector vector1 = new TimeMicroVector(EMPTY_SCHEMA_PATH, allocator);
+         final TimeMicroVector vector2 = new TimeMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -777,10 +777,10 @@ public void testCopyFromWithNulls10() {
     }
   }
 
-  @Test /* NullableTimeMilliVector */
+  @Test /* TimeMilliVector */
   public void testCopyFromWithNulls11() {
-    try (final NullableTimeMilliVector vector1 = new NullableTimeMilliVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableTimeMilliVector vector2 = new NullableTimeMilliVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final TimeMilliVector vector1 = new TimeMilliVector(EMPTY_SCHEMA_PATH, allocator);
+         final TimeMilliVector vector2 = new TimeMilliVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -840,10 +840,10 @@ public void testCopyFromWithNulls11() {
     }
   }
 
-  @Test /* NullableTinyIntVector */
+  @Test /* TinyIntVector */
   public void testCopyFromWithNulls12() {
-    try (final NullableTinyIntVector vector1 = new NullableTinyIntVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableTinyIntVector vector2 = new NullableTinyIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final TinyIntVector vector1 = new TinyIntVector(EMPTY_SCHEMA_PATH, allocator);
+         final TinyIntVector vector2 = new TinyIntVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -906,10 +906,10 @@ public void testCopyFromWithNulls12() {
     }
   }
 
-  @Test /* NullableDecimalVector */
+  @Test /* DecimalVector */
   public void testCopyFromWithNulls13() {
-    try (final NullableDecimalVector vector1 = new NullableDecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16);
-         final NullableDecimalVector vector2 = new NullableDecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16)) {
+    try (final DecimalVector vector1 = new DecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16);
+         final DecimalVector vector2 = new DecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
@@ -972,10 +972,10 @@ public void testCopyFromWithNulls13() {
     }
   }
 
-  @Test /* NullableTimeStampVector */
+  @Test /* TimeStampVector */
   public void testCopyFromWithNulls14() {
-    try (final NullableTimeStampVector vector1 = new NullableTimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator);
-         final NullableTimeStampVector vector2 = new NullableTimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final TimeStampVector vector1 = new TimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator);
+         final TimeStampVector vector2 = new TimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       vector1.allocateNew();
       assertEquals(4096, vector1.getValueCapacity());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index 4d844d6d3ca0f..a48c035718990 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -60,9 +60,9 @@ public void terminate() throws Exception {
 
   @Test
   public void testValuesWriteRead() {
-    try (NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(10, scale), allocator);) {
+    try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(10, scale), allocator);) {
 
-      try (NullableDecimalVector oldConstructor = new NullableDecimalVector("decimal", allocator, 10, scale);) {
+      try (DecimalVector oldConstructor = new DecimalVector("decimal", allocator, 10, scale);) {
         assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType());
       }
 
@@ -85,7 +85,7 @@ public void testValuesWriteRead() {
 
   @Test
   public void testBigDecimalDifferentScaleAndPrecision() {
-    try (NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(4, 2), allocator);) {
+    try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(4, 2), allocator);) {
       decimalVector.allocateNew();
 
       // test BigDecimal with different scale
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
index 46a2bafa00208..ba149bcb8cb9f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java
@@ -18,7 +18,7 @@
 
 package org.apache.arrow.vector;
 
-import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
+import static org.apache.arrow.vector.TestUtils.newVarCharVector;
 import static org.junit.Assert.assertEquals;
 
 import java.nio.charset.StandardCharsets;
@@ -54,8 +54,8 @@ public void terminate() throws Exception {
   @Test
   public void testEncodeStrings() {
     // Create a new value vector
-    try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator);
-         final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) {
+    try (final VarCharVector vector = newVarCharVector("foo", allocator);
+         final VarCharVector dictionaryVector = newVarCharVector("dict", allocator);) {
       vector.allocateNew(512, 5);
 
       // set some values
@@ -77,9 +77,9 @@ public void testEncodeStrings() {
 
       try (final ValueVector encoded = (FieldVector) DictionaryEncoder.encode(vector, dictionary)) {
         // verify indices
-        assertEquals(NullableIntVector.class, encoded.getClass());
+        assertEquals(IntVector.class, encoded.getClass());
 
-        NullableIntVector index = ((NullableIntVector)encoded);
+        IntVector index = ((IntVector)encoded);
         assertEquals(5, index.getValueCount());
         assertEquals(0, index.get(0));
         assertEquals(1, index.get(1));
@@ -90,9 +90,9 @@ public void testEncodeStrings() {
         // now run through the decoder and verify we get the original back
         try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) {
           assertEquals(vector.getClass(), decoded.getClass());
-          assertEquals(vector.getValueCount(), ((NullableVarCharVector)decoded).getValueCount());
+          assertEquals(vector.getValueCount(), ((VarCharVector)decoded).getValueCount());
           for (int i = 0; i < 5; i++) {
-            assertEquals(vector.getObject(i), ((NullableVarCharVector)decoded).getObject(i));
+            assertEquals(vector.getObject(i), ((VarCharVector)decoded).getObject(i));
           }
         }
       }
@@ -102,8 +102,8 @@ public void testEncodeStrings() {
   @Test
   public void testEncodeLargeVector() {
     // Create a new value vector
-    try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator);
-         final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) {
+    try (final VarCharVector vector = newVarCharVector("foo", allocator);
+         final VarCharVector dictionaryVector = newVarCharVector("dict", allocator);) {
       vector.allocateNew();
 
       int count = 10000;
@@ -124,9 +124,9 @@ public void testEncodeLargeVector() {
 
       try (final ValueVector encoded = (FieldVector) DictionaryEncoder.encode(vector, dictionary)) {
         // verify indices
-        assertEquals(NullableIntVector.class, encoded.getClass());
+        assertEquals(IntVector.class, encoded.getClass());
 
-        NullableIntVector index = ((NullableIntVector) encoded);
+        IntVector index = ((IntVector) encoded);
         assertEquals(count, index.getValueCount());
         for (int i = 0; i < count; ++i) {
           assertEquals(i % 3, index.get(i));
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index 2af6cd53aa0c1..50438ce116dfd 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -53,7 +53,7 @@ public void terminate() throws Exception {
   @Test
   public void testIntType() {
     try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) {
-      NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
+      IntVector nested = (IntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
       vector.allocateNew();
 
       for (int i = 0; i < 10; i++) {
@@ -80,7 +80,7 @@ public void testIntType() {
   @Test
   public void testFloatTypeNullable() {
     try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) {
-      NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
+      Float4Vector nested = (Float4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
       vector.allocateNew();
 
       for (int i = 0; i < 10; i++) {
@@ -115,7 +115,7 @@ public void testFloatTypeNullable() {
   public void testNestedInList() {
     try (ListVector vector = ListVector.empty("list", allocator)) {
       FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector();
-      NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
+      IntVector innerVector = (IntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
       vector.allocateNew();
 
       for (int i = 0; i < 10; i++) {
@@ -157,7 +157,7 @@ public void testNestedInList() {
   public void testTransferPair() {
     try (FixedSizeListVector from = new FixedSizeListVector("from", allocator, 2, null, null);
          FixedSizeListVector to = new FixedSizeListVector("to", allocator, 2, null, null)) {
-      NullableFloat4Vector nested = (NullableFloat4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
+      Float4Vector nested = (Float4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
       from.allocateNew();
 
       for (int i = 0; i < 10; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index f6aa86a300876..1cff7fa886e87 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -131,8 +131,8 @@ public void testSetLastSetUsage() throws Exception {
       ArrowBuf validityBuffer = listVector.getValidityBuffer();
       ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
 
-      /* get the underlying data vector -- NullableBigIntVector */
-      NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector();
+      /* get the underlying data vector -- BigIntVector */
+      BigIntVector dataVector = (BigIntVector) listVector.getDataVector();
 
       /* check current lastSet */
       assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet()));
@@ -310,7 +310,7 @@ public void testSplitAndTransfer() throws Exception {
       final ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
 
       /* get dataVector */
-      NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector();
+      BigIntVector dataVector = (BigIntVector) listVector.getDataVector();
 
       /* check the vector output */
 
@@ -418,7 +418,7 @@ public void testSplitAndTransfer() throws Exception {
           final ArrowBuf toOffsetBuffer = toVector.getOffsetBuffer();
 
           /* get dataVector of toVector */
-          NullableBigIntVector dataVector1 = (NullableBigIntVector) toVector.getDataVector();
+          BigIntVector dataVector1 = (BigIntVector) toVector.getDataVector();
 
           for (int i = 0; i < splitLength; i++) {
             dataLength1 = offsetBuffer.getInt((start + i + 1) * ListVector.OFFSET_WIDTH) -
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
index 357df96aa2efc..e5682955bada0 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
@@ -22,7 +22,7 @@
 import java.util.Map;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.junit.After;
@@ -50,7 +50,7 @@ public void testFieldMetadata() throws Exception {
     Map<String, String> metadata = new HashMap<>();
     metadata.put("k1", "v1");
     FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata);
-    try (NullableMapVector vector = new NullableMapVector("map", allocator, type, null)) {
+    try (MapVector vector = new MapVector("map", allocator, type, null)) {
       Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata());
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
index ba2ebbf05ad0d..f14dbd6637356 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
@@ -112,10 +112,10 @@ public void testVariableVectorReallocation() {
     try {
       vector.allocateNew(expectedAllocationInBytes, 10);
       assertTrue(expectedOffsetSize <= vector.getValueCapacity());
-      assertTrue(expectedAllocationInBytes <= vector.getBuffer().capacity());
+      assertTrue(expectedAllocationInBytes <= vector.getDataBuffer().capacity());
       vector.reAlloc();
       assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity());
-      assertTrue(expectedAllocationInBytes * 2 <= vector.getBuffer().capacity());
+      assertTrue(expectedAllocationInBytes * 2 <= vector.getDataBuffer().capacity());
     } finally {
       vector.close();
     }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
index 7de3bcb33195f..80d5fe19700ac 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java
@@ -24,7 +24,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 
-import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.util.TransferPair;
 
 import org.junit.After;
@@ -45,9 +45,9 @@ public void terminate() throws Exception {
         allocator.close();
     }
 
-    @Test /* NullableVarCharVector */
+    @Test /* VarCharVector */
     public void test() throws Exception {
-        try(final NullableVarCharVector varCharVector = new NullableVarCharVector("myvector", allocator)) {
+        try(final VarCharVector varCharVector = new VarCharVector("myvector", allocator)) {
             varCharVector.allocateNew(10000, 1000);
 
             final int valueCount = 500;
@@ -61,7 +61,7 @@ public void test() throws Exception {
             varCharVector.setValueCount(valueCount);
 
             final TransferPair tp = varCharVector.getTransferPair(allocator);
-            final NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo();
+            final VarCharVector newVarCharVector = (VarCharVector) tp.getTo();
             final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}};
 
             for (final int[] startLength : startLengths) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
index a148813090900..99a1d89071c71 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java
@@ -25,13 +25,13 @@
 
 public class TestUtils {
 
-  public static NullableVarCharVector newNullableVarCharVector(String name, BufferAllocator allocator) {
-    return (NullableVarCharVector)
+  public static VarCharVector newVarCharVector(String name, BufferAllocator allocator) {
+    return (VarCharVector)
         FieldType.nullable(new ArrowType.Utf8()).createNewSingleVector(name, allocator, null);
   }
 
-  public static NullableVarBinaryVector newNullableVarBinaryVector(String name, BufferAllocator allocator) {
-    return (NullableVarBinaryVector)
+  public static VarBinaryVector newVarBinaryVector(String name, BufferAllocator allocator) {
+    return (VarBinaryVector)
         FieldType.nullable(new ArrowType.Binary()).createNewSingleVector(name, allocator, null);
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index f51a874367622..336ae1c7a0146 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -19,8 +19,8 @@
 package org.apache.arrow.vector;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 
-import static org.apache.arrow.vector.TestUtils.newNullableVarBinaryVector;
-import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
+import static org.apache.arrow.vector.TestUtils.newVarBinaryVector;
+import static org.apache.arrow.vector.TestUtils.newVarCharVector;
 import static org.apache.arrow.vector.TestUtils.newVector;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
@@ -88,9 +88,9 @@ public void terminate() throws Exception {
    *  -- Float4Vector
    *  -- Float8Vector
    *
-   *  -- NullableUInt4Vector
-   *  -- NullableIntVector
-   *  -- NullableFloat4Vector
+   *  -- UInt4Vector
+   *  -- IntVector
+   *  -- Float4Vector
    *
    * TODO:
    *
@@ -107,28 +107,26 @@ public void testFixedType1() {
 
       boolean error = false;
       int initialCapacity = 0;
-      final UInt4Vector.Mutator mutator = vector.getMutator();
-      final UInt4Vector.Accessor accessor = vector.getAccessor();
 
       vector.allocateNew(1024);
       initialCapacity = vector.getValueCapacity();
       assertEquals(1024, initialCapacity);
 
       // Put and set a few values
-      mutator.setSafe(0, 100);
-      mutator.setSafe(1, 101);
-      mutator.setSafe(100, 102);
-      mutator.setSafe(1022, 103);
-      mutator.setSafe(1023, 104);
-
-      assertEquals(100, accessor.get(0));
-      assertEquals(101, accessor.get(1));
-      assertEquals(102, accessor.get(100));
-      assertEquals(103, accessor.get(1022));
-      assertEquals(104, accessor.get(1023));
+      vector.setSafe(0, 100);
+      vector.setSafe(1, 101);
+      vector.setSafe(100, 102);
+      vector.setSafe(1022, 103);
+      vector.setSafe(1023, 104);
+
+      assertEquals(100, vector.get(0));
+      assertEquals(101, vector.get(1));
+      assertEquals(102, vector.get(100));
+      assertEquals(103, vector.get(1022));
+      assertEquals(104, vector.get(1023));
 
       try {
-        mutator.set(1024, 10000);
+        vector.set(1024, 10000);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -139,7 +137,7 @@ public void testFixedType1() {
       }
 
       try {
-        accessor.get(1024);
+        vector.get(1024);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -150,18 +148,18 @@ public void testFixedType1() {
       }
 
       /* this should trigger a realloc() */
-      mutator.setSafe(1024, 10000);
+      vector.setSafe(1024, 10000);
 
       /* underlying buffer should now be able to store double the number of values */
       assertEquals(initialCapacity * 2, vector.getValueCapacity());
 
       /* check vector data after realloc */
-      assertEquals(100, accessor.get(0));
-      assertEquals(101, accessor.get(1));
-      assertEquals(102, accessor.get(100));
-      assertEquals(103, accessor.get(1022));
-      assertEquals(104, accessor.get(1023));
-      assertEquals(10000, accessor.get(1024));
+      assertEquals(100, vector.get(0));
+      assertEquals(101, vector.get(1));
+      assertEquals(102, vector.get(100));
+      assertEquals(103, vector.get(1022));
+      assertEquals(104, vector.get(1023));
+      assertEquals(10000, vector.get(1024));
 
       /* reset the vector */
       vector.reset();
@@ -171,7 +169,8 @@ public void testFixedType1() {
 
       /* vector data should have been zeroed out */
       for(int i = 0; i < (initialCapacity * 2); i++) {
-        assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i));
+        // TODO: test vector.get(i) is 0 after unsafe get added
+        assertEquals("non-zero data not expected at index: " + i, true, vector.isNull(i));
       }
     }
   }
@@ -179,8 +178,6 @@ public void testFixedType1() {
   @Test /* IntVector */
   public void testFixedType2() {
     try (final IntVector intVector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) {
-      final IntVector.Mutator mutator = intVector.getMutator();
-      final IntVector.Accessor accessor = intVector.getAccessor();
       boolean error = false;
       int initialCapacity = 16;
 
@@ -211,12 +208,12 @@ public void testFixedType2() {
       /* populate the vector */
       int j = 1;
       for(int i = 0; i < 16; i += 2) {
-        mutator.set(i, j);
+        intVector.set(i, j);
         j++;
       }
 
       try {
-        mutator.set(16, 9);
+        intVector.set(16, 9);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -229,12 +226,12 @@ public void testFixedType2() {
       /* check vector contents */
       j = 1;
       for(int i = 0; i < 16; i += 2) {
-        assertEquals("unexpected value at index: " + i, j, accessor.get(i));
+        assertEquals("unexpected value at index: " + i, j, intVector.get(i));
         j++;
       }
 
       try {
-        accessor.get(16);
+        intVector.get(16);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -245,7 +242,7 @@ public void testFixedType2() {
       }
 
       /* this should trigger a realloc() */
-      mutator.setSafe(16, 9);
+      intVector.setSafe(16, 9);
 
       /* underlying buffer should now be able to store double the number of values */
       assertEquals(initialCapacity * 2, intVector.getValueCapacity());
@@ -253,7 +250,7 @@ public void testFixedType2() {
       /* vector data should still be intact after realloc */
       j = 1;
       for(int i = 0; i <= 16; i += 2) {
-        assertEquals("unexpected value at index: " + i, j, accessor.get(i));
+        assertEquals("unexpected value at index: " + i, j, intVector.get(i));
         j++;
       }
 
@@ -265,7 +262,7 @@ public void testFixedType2() {
 
       /* vector data should have been zeroed out */
       for(int i = 0; i < (initialCapacity * 2); i++) {
-        assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i));
+        assertEquals("non-zero data not expected at index: " + i, true, intVector.isNull(i));
       }
     }
   }
@@ -273,8 +270,6 @@ public void testFixedType2() {
   @Test /* Float4Vector */
   public void testFixedType3() {
     try (final Float4Vector floatVector = new Float4Vector(EMPTY_SCHEMA_PATH, allocator)) {
-      final Float4Vector.Mutator mutator = floatVector.getMutator();
-      final Float4Vector.Accessor accessor = floatVector.getAccessor();
       boolean error = false;
       int initialCapacity = 16;
 
@@ -304,18 +299,18 @@ public void testFixedType3() {
 
       floatVector.zeroVector();
 
-      /* populate the vector */
-      mutator.set(0, 1.5f);
-      mutator.set(2, 2.5f);
-      mutator.set(4, 3.3f);
-      mutator.set(6, 4.8f);
-      mutator.set(8, 5.6f);
-      mutator.set(10, 6.6f);
-      mutator.set(12, 7.8f);
-      mutator.set(14, 8.5f);
+      /* populate the floatVector */
+      floatVector.set(0, 1.5f);
+      floatVector.set(2, 2.5f);
+      floatVector.set(4, 3.3f);
+      floatVector.set(6, 4.8f);
+      floatVector.set(8, 5.6f);
+      floatVector.set(10, 6.6f);
+      floatVector.set(12, 7.8f);
+      floatVector.set(14, 8.5f);
 
       try {
-        mutator.set(16, 9.5f);
+        floatVector.set(16, 9.5f);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -326,17 +321,17 @@ public void testFixedType3() {
       }
 
       /* check vector contents */
-      assertEquals(1.5f, accessor.get(0), 0);
-      assertEquals(2.5f, accessor.get(2), 0);
-      assertEquals(3.3f, accessor.get(4), 0);
-      assertEquals(4.8f, accessor.get(6), 0);
-      assertEquals(5.6f, accessor.get(8), 0);
-      assertEquals(6.6f, accessor.get(10), 0);
-      assertEquals(7.8f, accessor.get(12), 0);
-      assertEquals(8.5f, accessor.get(14), 0);
+      assertEquals(1.5f, floatVector.get(0), 0);
+      assertEquals(2.5f, floatVector.get(2), 0);
+      assertEquals(3.3f, floatVector.get(4), 0);
+      assertEquals(4.8f, floatVector.get(6), 0);
+      assertEquals(5.6f, floatVector.get(8), 0);
+      assertEquals(6.6f, floatVector.get(10), 0);
+      assertEquals(7.8f, floatVector.get(12), 0);
+      assertEquals(8.5f, floatVector.get(14), 0);
 
       try {
-        accessor.get(16);
+        floatVector.get(16);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -347,21 +342,21 @@ public void testFixedType3() {
       }
 
       /* this should trigger a realloc() */
-      mutator.setSafe(16, 9.5f);
+      floatVector.setSafe(16, 9.5f);
 
       /* underlying buffer should now be able to store double the number of values */
       assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
 
       /* vector data should still be intact after realloc */
-      assertEquals(1.5f, accessor.get(0), 0);
-      assertEquals(2.5f, accessor.get(2), 0);
-      assertEquals(3.3f, accessor.get(4), 0);
-      assertEquals(4.8f, accessor.get(6), 0);
-      assertEquals(5.6f, accessor.get(8), 0);
-      assertEquals(6.6f, accessor.get(10), 0);
-      assertEquals(7.8f, accessor.get(12), 0);
-      assertEquals(8.5f, accessor.get(14), 0);
-      assertEquals(9.5f, accessor.get(16), 0);
+      assertEquals(1.5f, floatVector.get(0), 0);
+      assertEquals(2.5f, floatVector.get(2), 0);
+      assertEquals(3.3f, floatVector.get(4), 0);
+      assertEquals(4.8f, floatVector.get(6), 0);
+      assertEquals(5.6f, floatVector.get(8), 0);
+      assertEquals(6.6f, floatVector.get(10), 0);
+      assertEquals(7.8f, floatVector.get(12), 0);
+      assertEquals(8.5f, floatVector.get(14), 0);
+      assertEquals(9.5f, floatVector.get(16), 0);
 
       /* reset the vector */
       floatVector.reset();
@@ -371,7 +366,7 @@ public void testFixedType3() {
 
       /* vector data should be zeroed out */
       for(int i = 0; i < (initialCapacity * 2); i++) {
-        assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i), 0);
+        assertEquals("non-zero data not expected at index: " + i, true, floatVector.isNull(i));
       }
     }
   }
@@ -379,8 +374,6 @@ public void testFixedType3() {
   @Test /* Float8Vector */
   public void testFixedType4() {
     try (final Float8Vector floatVector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
-      final Float8Vector.Mutator mutator = floatVector.getMutator();
-      final Float8Vector.Accessor accessor = floatVector.getAccessor();
       boolean error = false;
       int initialCapacity = 16;
 
@@ -409,17 +402,17 @@ public void testFixedType4() {
       assertEquals(initialCapacity, floatVector.getValueCapacity());
 
       /* populate the vector */
-      mutator.set(0, 1.55);
-      mutator.set(2, 2.53);
-      mutator.set(4, 3.36);
-      mutator.set(6, 4.82);
-      mutator.set(8, 5.67);
-      mutator.set(10, 6.67);
-      mutator.set(12, 7.87);
-      mutator.set(14, 8.56);
+      floatVector.set(0, 1.55);
+      floatVector.set(2, 2.53);
+      floatVector.set(4, 3.36);
+      floatVector.set(6, 4.82);
+      floatVector.set(8, 5.67);
+      floatVector.set(10, 6.67);
+      floatVector.set(12, 7.87);
+      floatVector.set(14, 8.56);
 
       try {
-        mutator.set(16, 9.53);
+        floatVector.set(16, 9.53);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -429,18 +422,18 @@ public void testFixedType4() {
         error = false;
       }
 
-      /* check vector contents */
-      assertEquals(1.55, accessor.get(0), 0);
-      assertEquals(2.53, accessor.get(2), 0);
-      assertEquals(3.36, accessor.get(4), 0);
-      assertEquals(4.82, accessor.get(6), 0);
-      assertEquals(5.67, accessor.get(8), 0);
-      assertEquals(6.67, accessor.get(10), 0);
-      assertEquals(7.87, accessor.get(12), 0);
-      assertEquals(8.56, accessor.get(14), 0);
+      /* check floatVector contents */
+      assertEquals(1.55, floatVector.get(0), 0);
+      assertEquals(2.53, floatVector.get(2), 0);
+      assertEquals(3.36, floatVector.get(4), 0);
+      assertEquals(4.82, floatVector.get(6), 0);
+      assertEquals(5.67, floatVector.get(8), 0);
+      assertEquals(6.67, floatVector.get(10), 0);
+      assertEquals(7.87, floatVector.get(12), 0);
+      assertEquals(8.56, floatVector.get(14), 0);
 
       try {
-        accessor.get(16);
+        floatVector.get(16);
       }
       catch (IndexOutOfBoundsException ie) {
         error = true;
@@ -451,21 +444,21 @@ public void testFixedType4() {
       }
 
       /* this should trigger a realloc() */
-      mutator.setSafe(16, 9.53);
+      floatVector.setSafe(16, 9.53);
 
       /* underlying buffer should now be able to store double the number of values */
       assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
 
       /* vector data should still be intact after realloc */
-      assertEquals(1.55, accessor.get(0), 0);
-      assertEquals(2.53, accessor.get(2), 0);
-      assertEquals(3.36, accessor.get(4), 0);
-      assertEquals(4.82, accessor.get(6), 0);
-      assertEquals(5.67, accessor.get(8), 0);
-      assertEquals(6.67, accessor.get(10), 0);
-      assertEquals(7.87, accessor.get(12), 0);
-      assertEquals(8.56, accessor.get(14), 0);
-      assertEquals(9.53, accessor.get(16), 0);
+      assertEquals(1.55, floatVector.get(0), 0);
+      assertEquals(2.53, floatVector.get(2), 0);
+      assertEquals(3.36, floatVector.get(4), 0);
+      assertEquals(4.82, floatVector.get(6), 0);
+      assertEquals(5.67, floatVector.get(8), 0);
+      assertEquals(6.67, floatVector.get(10), 0);
+      assertEquals(7.87, floatVector.get(12), 0);
+      assertEquals(8.56, floatVector.get(14), 0);
+      assertEquals(9.53, floatVector.get(16), 0);
 
       /* reset the vector */
       floatVector.reset();
@@ -475,16 +468,16 @@ public void testFixedType4() {
 
       /* vector data should be zeroed out */
       for(int i = 0; i < (initialCapacity * 2); i++) {
-        assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i), 0);
+        assertEquals("non-zero data not expected at index: " + i, true, floatVector.isNull(i));
       }
     }
   }
 
-  @Test /* NullableUInt4Vector */
+  @Test /* UInt4Vector */
   public void testNullableFixedType1() {
 
     // Create a new value vector for 1024 integers.
-    try (final NullableUInt4Vector vector = newVector(NullableUInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) {
+    try (final UInt4Vector vector = newVector(UInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) {
       boolean error = false;
       int initialCapacity = 1024;
 
@@ -582,10 +575,10 @@ public void testNullableFixedType1() {
     }
   }
 
-  @Test /* NullableFloat4Vector */
+  @Test /* Float4Vector */
   public void testNullableFixedType2() {
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) {
+    try (final Float4Vector vector = newVector(Float4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) {
       boolean error = false;
       int initialCapacity = 16;
 
@@ -683,10 +676,10 @@ public void testNullableFixedType2() {
     }
   }
 
-  @Test /* NullableIntVector */
+  @Test /* IntVector */
   public void testNullableFixedType3() {
     // Create a new value vector for 1024 integers
-    try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) {
+    try (final IntVector vector = newVector(IntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) {
       boolean error = false;
       int initialCapacity = 1024;
 
@@ -777,9 +770,9 @@ public void testNullableFixedType3() {
     }
   }
 
-  @Test /* NullableIntVector */
+  @Test /* IntVector */
   public void testNullableFixedType4() {
-    try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) {
+    try (final IntVector vector = newVector(IntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) {
 
       /* no memory allocation has happened yet */
       assertEquals(0, vector.getValueCapacity());
@@ -870,8 +863,8 @@ public void testNullableFixedType4() {
    *
    * Covered types as of now
    *
-   *  -- NullableVarCharVector
-   *  -- NullableVarBinaryVector
+   *  -- VarCharVector
+   *  -- VarBinaryVector
    *
    * TODO:
    *
@@ -879,11 +872,11 @@ public void testNullableFixedType4() {
    *  -- VarBinaryVector
    */
 
-  @Test /* NullableVarCharVector */
+  @Test /* VarCharVector */
   public void testNullableVarType1() {
 
     // Create a new value vector for 1024 integers.
-    try (final NullableVarCharVector vector = newNullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final VarCharVector vector = newVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
       vector.allocateNew(1024 * 10, 1024);
 
       vector.set(0, STR1);
@@ -916,11 +909,11 @@ public void testNullableVarType1() {
     }
   }
 
-  @Test /* NullableVarBinaryVector */
+  @Test /* VarBinaryVector */
   public void testNullableVarType2() {
 
     // Create a new value vector for 1024 integers.
-    try (final NullableVarBinaryVector vector = newNullableVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final VarBinaryVector vector = newVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) {
       vector.allocateNew(1024 * 10, 1024);
 
       vector.set(0, STR1);
@@ -971,8 +964,6 @@ public void testNullableVarType2() {
   @Test /* Float8Vector */
   public void testReallocAfterVectorTransfer1() {
     try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
-      final Float8Vector.Mutator mutator = vector.getMutator();
-      final Float8Vector.Accessor accessor = vector.getAccessor();
       final int initialDefaultCapacity = 4096;
       boolean error = false;
 
@@ -984,7 +975,7 @@ public void testReallocAfterVectorTransfer1() {
       double baseValue = 100.375;
 
       for (int i = 0; i < initialDefaultCapacity; i++) {
-        mutator.setSafe(i, baseValue + (double)i);
+        vector.setSafe(i, baseValue + (double)i);
       }
 
       /* the above setSafe calls should not have triggered a realloc as
@@ -993,33 +984,33 @@ public void testReallocAfterVectorTransfer1() {
       assertEquals(initialDefaultCapacity, vector.getValueCapacity());
 
       for (int i = 0; i < initialDefaultCapacity; i++) {
-        double value = accessor.get(i);
+        double value = vector.get(i);
         assertEquals(baseValue + (double)i, value, 0);
       }
 
       /* this should trigger a realloc */
-      mutator.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity);
+      vector.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity);
       assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity());
 
       for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) {
-        mutator.setSafe(i, baseValue + (double)i);
+        vector.setSafe(i, baseValue + (double)i);
       }
 
       for (int i = 0; i < (initialDefaultCapacity * 2); i++) {
-        double value = accessor.get(i);
+        double value = vector.get(i);
         assertEquals(baseValue + (double)i, value, 0);
       }
 
       /* this should trigger a realloc */
-      mutator.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2));
+      vector.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2));
       assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity());
 
       for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) {
-        mutator.setSafe(i, baseValue + (double)i);
+        vector.setSafe(i, baseValue + (double)i);
       }
 
       for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
-        double value = accessor.get(i);
+        double value = vector.get(i);
         assertEquals(baseValue + (double)i, value, 0);
       }
 
@@ -1036,15 +1027,12 @@ public void testReallocAfterVectorTransfer1() {
       toVector.reAlloc();
       assertEquals(initialDefaultCapacity * 8, toVector.getValueCapacity());
 
-      final Float8Vector.Accessor toAccessor = toVector.getAccessor();
-
       for (int i = 0; i < (initialDefaultCapacity * 8); i++) {
-        double value = toAccessor.get(i);
         if (i < (initialDefaultCapacity * 4)) {
-          assertEquals(baseValue + (double)i, value, 0);
+          assertEquals(baseValue + (double)i, toVector.get(i), 0);
         }
         else {
-          assertEquals(0, value, 0);
+          assertTrue(toVector.isNull(i));
         }
       }
 
@@ -1052,9 +1040,9 @@ public void testReallocAfterVectorTransfer1() {
     }
   }
 
-  @Test /* NullableFloat8Vector */
+  @Test /* Float8Vector */
   public void testReallocAfterVectorTransfer2() {
-    try (final NullableFloat8Vector vector = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
       final int initialDefaultCapacity = 4096;
       boolean error = false;
 
@@ -1111,7 +1099,7 @@ public void testReallocAfterVectorTransfer2() {
       TransferPair transferPair = vector.getTransferPair(allocator);
       transferPair.transfer();
 
-      NullableFloat8Vector toVector = (NullableFloat8Vector)transferPair.getTo();
+      Float8Vector toVector = (Float8Vector)transferPair.getTo();
 
       /* check toVector contents before realloc */
       for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
@@ -1139,9 +1127,9 @@ public void testReallocAfterVectorTransfer2() {
     }
   }
 
-  @Test /* NullableVarCharVector */
+  @Test /* VarCharVector */
   public void testReallocAfterVectorTransfer3() {
-    try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
       /* 4096 values with 10 byte per record */
       vector.allocateNew(4096 * 10, 4096);
       int valueCapacity = vector.getValueCapacity();
@@ -1223,7 +1211,7 @@ public void testReallocAfterVectorTransfer3() {
 
       TransferPair transferPair = vector.getTransferPair(allocator);
       transferPair.transfer();
-      NullableVarCharVector toVector = (NullableVarCharVector)transferPair.getTo();
+      VarCharVector toVector = (VarCharVector)transferPair.getTo();
       valueCapacity = toVector.getValueCapacity();
 
       for (int i = 0; i < valueCapacity; i++) {
@@ -1239,9 +1227,9 @@ public void testReallocAfterVectorTransfer3() {
     }
   }
 
-  @Test /* NullableIntVector */
+  @Test /* IntVector */
   public void testReallocAfterVectorTransfer4() {
-    try (final NullableIntVector vector = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+    try (final IntVector vector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) {
 
       /* 4096 values  */
       vector.allocateNew(4096);
@@ -1316,7 +1304,7 @@ public void testReallocAfterVectorTransfer4() {
 
       TransferPair transferPair = vector.getTransferPair(allocator);
       transferPair.transfer();
-      NullableIntVector toVector = (NullableIntVector)transferPair.getTo();
+      IntVector toVector = (IntVector)transferPair.getTo();
       /* value capacity of source and target vectors should be same after
        * the transfer.
        */
@@ -1336,9 +1324,9 @@ public void testReallocAfterVectorTransfer4() {
   }
 
   @Test
-  public void testReAllocNullableFixedWidthVector() {
+  public void testReAllocFixedWidthVector() {
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) {
+    try (final Float4Vector vector = newVector(Float4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) {
       vector.allocateNew(1024);
 
       assertEquals(1024, vector.getValueCapacity());
@@ -1359,7 +1347,7 @@ public void testReAllocNullableFixedWidthVector() {
       assertEquals(104.5f, vector.get(1023), 0);
       assertEquals(105.5f, vector.get(2000), 0);
 
-      // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors
+      // Set the valueCount to be more than valueCapacity of current allocation. This is possible for ValueVectors
       // as we don't call setSafe for null values, but we do call setValueCount when all values are inserted into the
       // vector
       vector.setValueCount(vector.getValueCapacity() + 200);
@@ -1367,8 +1355,8 @@ public void testReAllocNullableFixedWidthVector() {
   }
 
   @Test
-  public void testReAllocNullableVariableWidthVector() {
-    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+  public void testReAllocVariableWidthVector() {
+    try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
       vector.allocateNew();
 
       int initialCapacity = vector.getValueCapacity();
@@ -1392,7 +1380,7 @@ public void testReAllocNullableVariableWidthVector() {
       assertArrayEquals(STR2, vector.get(initialCapacity - 1));
       assertArrayEquals(STR3, vector.get(initialCapacity + 200));
 
-      // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors
+      // Set the valueCount to be more than valueCapacity of current allocation. This is possible for ValueVectors
       // as we don't call setSafe for null values, but we do call setValueCount when the current batch is processed.
       vector.setValueCount(vector.getValueCapacity() + 200);
     }
@@ -1400,7 +1388,7 @@ public void testReAllocNullableVariableWidthVector() {
 
   @Test
   public void testFillEmptiesNotOverfill() {
-    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+    try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
       vector.allocateNew();
 
       int initialCapacity = vector.getValueCapacity();
@@ -1420,8 +1408,8 @@ public void testFillEmptiesNotOverfill() {
 
   @Test
   public void testCopyFromWithNulls() {
-    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
-         final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+    try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
+         final VarCharVector vector2 = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
 
       vector.allocateNew();
       int capacity = vector.getValueCapacity();
@@ -1480,8 +1468,8 @@ public void testCopyFromWithNulls() {
 
   @Test
   public void testCopyFromWithNulls1() {
-    try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
-         final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+    try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
+         final VarCharVector vector2 = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
 
       vector.allocateNew();
       int capacity = vector.getValueCapacity();
@@ -1544,7 +1532,7 @@ public void testCopyFromWithNulls1() {
 
   @Test
   public void testSetLastSetUsage() {
-    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
+    try (final VarCharVector vector = new VarCharVector("myvector", allocator)) {
       vector.allocateNew(1024 * 10, 1024);
 
       setBytes(0, STR1, vector);
@@ -1657,7 +1645,7 @@ public void testSetLastSetUsage() {
   @Test
   public void testVectorLoadUnload() {
 
-    try (final NullableVarCharVector vector1 = new NullableVarCharVector("myvector", allocator)) {
+    try (final VarCharVector vector1 = new VarCharVector("myvector", allocator)) {
       vector1.allocateNew(1024 * 10, 1024);
 
       vector1.set(0, STR1);
@@ -1701,7 +1689,7 @@ public void testVectorLoadUnload() {
         VectorLoader vectorLoader = new VectorLoader(schemaRoot2);
         vectorLoader.load(recordBatch);
 
-        NullableVarCharVector vector2 = (NullableVarCharVector) schemaRoot2.getVector(fieldName);
+        VarCharVector vector2 = (VarCharVector) schemaRoot2.getVector(fieldName);
         /*
          * lastSet would have internally been set by VectorLoader.load() when it invokes
          * loadFieldBuffers.
@@ -1723,7 +1711,7 @@ public void testVectorLoadUnload() {
 
   @Test
   public void testFillEmptiesUsage() {
-    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
+    try (final VarCharVector vector = new VarCharVector("myvector", allocator)) {
 
       vector.allocateNew(1024 * 10, 1024);
 
@@ -1830,10 +1818,10 @@ public void testFillEmptiesUsage() {
     }
   }
 
-  @Test /* NullableVarCharVector */
+  @Test /* VarCharVector */
   public void testGetBufferAddress1() {
 
-    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) {
+    try (final VarCharVector vector = new VarCharVector("myvector", allocator)) {
       vector.allocateNew(1024 * 10, 1024);
 
       /* populate the vector */
@@ -1866,9 +1854,9 @@ public void testGetBufferAddress1() {
     }
   }
 
-  @Test /* NullableIntVector */
+  @Test /* IntVector */
   public void testGetBufferAddress2() {
-    try (final NullableIntVector vector = new NullableIntVector("myvector", allocator)) {
+    try (final IntVector vector = new IntVector("myvector", allocator)) {
       boolean error = false;
       vector.allocateNew(16);
 
@@ -1905,7 +1893,7 @@ public void testGetBufferAddress2() {
   @Test
   public void testMultipleClose() {
     BufferAllocator vectorAllocator = allocator.newChildAllocator("vector_allocator", 0, Long.MAX_VALUE);
-    NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, vectorAllocator);
+    IntVector vector = newVector(IntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, vectorAllocator);
     vector.close();
     vectorAllocator.close();
     vector.close();
@@ -1916,7 +1904,7 @@ public void testMultipleClose() {
    * lastSet. The method is to test the lastSet property and that's why we load the vector
    * in a way that lastSet is not set automatically.
    */
-  public static void setBytes(int index, byte[] bytes, NullableVarCharVector vector) {
+  public static void setBytes(int index, byte[] bytes, VarCharVector vector) {
     final int currentOffset = vector.offsetBuffer.getInt(index * vector.OFFSET_WIDTH);
 
     BitVectorHelper.setValidityBitToOne(vector.validityBuffer, index);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index 293ffbfe19233..c0df4881f380c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -26,7 +26,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -53,14 +53,13 @@ public void terminate() throws Exception {
   @Test
   public void testFixedType() {
     try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
-      final UInt4Vector.Mutator m = vector.getMutator();
       vector.setInitialCapacity(512);
       vector.allocateNew();
 
       assertEquals(512, vector.getValueCapacity());
 
       try {
-        m.set(512, 0);
+        vector.set(512, 0);
         Assert.fail("Expected out of bounds exception");
       } catch (Exception e) {
         // ok
@@ -69,14 +68,14 @@ public void testFixedType() {
       vector.reAlloc();
       assertEquals(1024, vector.getValueCapacity());
 
-      m.set(512, 100);
-      assertEquals(100, vector.getAccessor().get(512));
+      vector.set(512, 100);
+      assertEquals(100, vector.get(512));
     }
   }
 
   @Test
   public void testNullableType() {
-    try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+    try (final VarCharVector vector = new VarCharVector("", allocator)) {
       vector.setInitialCapacity(512);
       vector.allocateNew();
 
@@ -108,7 +107,7 @@ public void testListType() {
       assertEquals(1023, vector.getValueCapacity());
 
       try {
-        vector.getOffsetVector().getAccessor().get(2014);
+        vector.getInnerValueCountAt(2014);
         Assert.fail("Expected out of bounds exception");
       } catch (Exception e) {
         // ok
@@ -122,8 +121,8 @@ public void testListType() {
 
   @Test
   public void testMapType() {
-    try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) {
-      vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+    try (final MapVector vector = MapVector.empty("", allocator)) {
+      vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), IntVector.class);
 
       vector.setInitialCapacity(512);
       vector.allocateNew();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
index d53f69489d4da..28903b1170e31 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
@@ -43,12 +43,11 @@ public void terminate() throws Exception {
   @Test
   public void testFixedTypeReset() {
     try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
-      final UInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew();
-      final int sizeBefore = vector.getAllocationSize();
+      final int sizeBefore = vector.getBufferSize();
       vector.reAlloc();
       vector.reset();
-      final int sizeAfter = vector.getAllocationSize();
+      final int sizeAfter = vector.getBufferSize();
       assertEquals(sizeBefore, sizeAfter);
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index e61dbecf44101..439a62725e49b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -236,8 +236,8 @@ public void testLoadValidityBuffer() throws IOException {
 
       vectorLoader.load(recordBatch);
 
-      NullableIntVector intDefinedVector = (NullableIntVector) newRoot.getVector("intDefined");
-      NullableIntVector intNullVector = (NullableIntVector) newRoot.getVector("intNull");
+      IntVector intDefinedVector = (IntVector) newRoot.getVector("intDefined");
+      IntVector intNullVector = (IntVector) newRoot.getVector("intNull");
       for (int i = 0; i < count; i++) {
         assertFalse("#" + i, intDefinedVector.isNull(i));
         assertEquals("#" + i, i, intDefinedVector.get(i));
@@ -283,7 +283,7 @@ public void testUnloadLoadDuplicates() throws IOException {
         FieldVector vector = field.createVector(originalVectorsAllocator);
         vector.allocateNew();
         sources.add(vector);
-        NullableIntVector intVector = (NullableIntVector)vector;
+        IntVector intVector = (IntVector)vector;
         for (int i = 0; i < count; i++) {
           intVector.set(i, i);
         }
@@ -302,8 +302,8 @@ public void testUnloadLoadDuplicates() throws IOException {
           List<FieldVector> targets = newRoot.getFieldVectors();
           Assert.assertEquals(sources.size(), targets.size());
           for (int k = 0; k < sources.size(); k++) {
-            NullableIntVector src = (NullableIntVector) sources.get(k);
-            NullableIntVector tgt = (NullableIntVector) targets.get(k);
+            IntVector src = (IntVector) sources.get(k);
+            IntVector tgt = (IntVector) targets.get(k);
             Assert.assertEquals(src.getValueCount(), tgt.getValueCount());
             for (int i = 0; i < count; i++) {
               Assert.assertEquals(src.get(i), tgt.get(i));
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index b0d6cf555e98d..38b78424dccf4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -25,7 +25,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.DirtyRootAllocator;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -55,7 +55,7 @@ public void terminate() throws Exception {
   public void testPromoteToUnion() throws Exception {
 
     try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator);
-         final NullableMapVector v = container.addOrGetMap("test");
+         final MapVector v = container.addOrGetMap("test");
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
       container.allocateNew();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 52defdc2e1b00..bd8489eb20f89 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -28,13 +28,13 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.SchemaChangeCallBack;
-import org.apache.arrow.vector.NullableFloat8Vector;
-import org.apache.arrow.vector.NullableFloat4Vector;
-import org.apache.arrow.vector.NullableBigIntVector;
-import org.apache.arrow.vector.NullableIntVector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.NonNullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
@@ -830,7 +830,7 @@ public void complexCopierWithList() {
     rootWriter.end();
     writer.setValueCount(1);
 
-    NullableMapVector mapVector = (NullableMapVector) parent.getChild("root");
+    MapVector mapVector = (MapVector) parent.getChild("root");
     TransferPair tp = mapVector.getTransferPair(allocator);
     tp.splitAndTransfer(0, 1);
     MapVector toMapVector = (MapVector) tp.getTo();
@@ -849,7 +849,7 @@ public void testSingleMapWriter1() {
     /* initialize a SingleMapWriter with empty MapVector and then lazily
      * create all vectors with expected initialCapacity.
      */
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableMapVector parent = NonNullableMapVector.empty("parent", allocator);
     SingleMapWriter singleMapWriter = new SingleMapWriter(parent);
 
     int initialCapacity = 1024;
@@ -885,10 +885,10 @@ public void testSingleMapWriter1() {
       singleMapWriter.end();
     }
 
-    NullableIntVector intVector = (NullableIntVector)parent.getChild("intField");
-    NullableBigIntVector bigIntVector = (NullableBigIntVector)parent.getChild("bigIntField");
-    NullableFloat4Vector float4Vector = (NullableFloat4Vector)parent.getChild("float4Field");
-    NullableFloat8Vector float8Vector = (NullableFloat8Vector)parent.getChild("float8Field");
+    IntVector intVector = (IntVector)parent.getChild("intField");
+    BigIntVector bigIntVector = (BigIntVector)parent.getChild("bigIntField");
+    Float4Vector float4Vector = (Float4Vector)parent.getChild("float4Field");
+    Float8Vector float8Vector = (Float8Vector)parent.getChild("float8Field");
 
     assertEquals(initialCapacity, singleMapWriter.getValueCapacity());
     assertEquals(initialCapacity, intVector.getValueCapacity());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index 233b682c99bc6..3514acaa24278 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -28,17 +28,17 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.NullableDateMilliVector;
-import org.apache.arrow.vector.NullableDecimalVector;
-import org.apache.arrow.vector.NullableIntVector;
-import org.apache.arrow.vector.NullableTimeMilliVector;
-import org.apache.arrow.vector.NullableVarBinaryVector;
-import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.TimeMilliVector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
@@ -71,7 +71,7 @@
 
 import io.netty.buffer.ArrowBuf;
 
-import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector;
+import static org.apache.arrow.vector.TestUtils.newVarCharVector;
 
 /**
  * Helps testing the file formats
@@ -172,7 +172,7 @@ private LocalDateTime makeDateTimeFromCount(int i) {
     return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i);
   }
 
-  protected void writeDateTimeData(int count, NullableMapVector parent) {
+  protected void writeDateTimeData(int count, MapVector parent) {
     Assert.assertTrue(count < 100);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
@@ -202,11 +202,11 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
     Assert.assertEquals(count, root.getRowCount());
     printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
-      long dateVal = ((NullableDateMilliVector) root.getVector("date")).get(i);
+      long dateVal = ((DateMilliVector) root.getVector("date")).get(i);
       LocalDateTime dt = makeDateTimeFromCount(i);
       LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
       Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal);
-      long timeVal = ((NullableTimeMilliVector) root.getVector("time")).get(i);
+      long timeVal = ((TimeMilliVector) root.getVector("time")).get(i);
       Assert.assertEquals(dt.getMillisOfDay(), timeVal);
       Object timestampMilliVal = root.getVector("timestamp-milli").getObject(i);
       Assert.assertEquals(dt, timestampMilliVal);
@@ -218,7 +218,7 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
   protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) {
 
     // Define dictionaries and add to provider
-    NullableVarCharVector dictionary1Vector = newNullableVarCharVector("D1", bufferAllocator);
+    VarCharVector dictionary1Vector = newVarCharVector("D1", bufferAllocator);
     dictionary1Vector.allocateNewSafe();
     dictionary1Vector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
     dictionary1Vector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
@@ -228,7 +228,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat
     Dictionary dictionary1 = new Dictionary(dictionary1Vector, new DictionaryEncoding(1L, false, null));
     provider.put(dictionary1);
 
-    NullableVarCharVector dictionary2Vector = newNullableVarCharVector("D2", bufferAllocator);
+    VarCharVector dictionary2Vector = newVarCharVector("D2", bufferAllocator);
     dictionary2Vector.allocateNewSafe();
     dictionary2Vector.set(0, "micro".getBytes(StandardCharsets.UTF_8));
     dictionary2Vector.set(1, "small".getBytes(StandardCharsets.UTF_8));
@@ -239,7 +239,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat
     provider.put(dictionary2);
 
     // Populate the vectors
-    NullableVarCharVector vector1A = newNullableVarCharVector("varcharA", bufferAllocator);
+    VarCharVector vector1A = newVarCharVector("varcharA", bufferAllocator);
     vector1A.allocateNewSafe();
     vector1A.set(0, "foo".getBytes(StandardCharsets.UTF_8));
     vector1A.set(1, "bar".getBytes(StandardCharsets.UTF_8));
@@ -252,7 +252,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat
     vector1A.close();  // Done with this vector after encoding
 
     // Write this vector using indices instead of encoding
-    NullableIntVector encodedVector1B = new NullableIntVector("varcharB", bufferAllocator);
+    IntVector encodedVector1B = new IntVector("varcharB", bufferAllocator);
     encodedVector1B.allocateNewSafe();
     encodedVector1B.set(0, 2);  // "baz"
     encodedVector1B.set(1, 1);  // "bar"
@@ -261,7 +261,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat
     encodedVector1B.set(5, 0);  // "foo"
     encodedVector1B.setValueCount(6);
 
-    NullableVarCharVector vector2 = newNullableVarCharVector("sizes", bufferAllocator);
+    VarCharVector vector2 = newVarCharVector("sizes", bufferAllocator);
     vector2.allocateNewSafe();
     vector2.set(1, "large".getBytes(StandardCharsets.UTF_8));
     vector2.set(2, "small".getBytes(StandardCharsets.UTF_8));
@@ -327,7 +327,7 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider
 
     Dictionary dictionary1 = provider.lookup(1L);
     Assert.assertNotNull(dictionary1);
-    NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary1.getVector());
+    VarCharVector dictionaryVector = ((VarCharVector) dictionary1.getVector());
     Assert.assertEquals(3, dictionaryVector.getValueCount());
     Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
     Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
@@ -335,7 +335,7 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider
 
     Dictionary dictionary2 = provider.lookup(2L);
     Assert.assertNotNull(dictionary2);
-    dictionaryVector = ((NullableVarCharVector) dictionary2.getVector());
+    dictionaryVector = ((VarCharVector) dictionary2.getVector());
     Assert.assertEquals(3, dictionaryVector.getValueCount());
     Assert.assertEquals(new Text("micro"), dictionaryVector.getObject(0));
     Assert.assertEquals(new Text("small"), dictionaryVector.getObject(1));
@@ -345,7 +345,7 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider
   protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) {
 
     // Define the dictionary and add to the provider
-    NullableVarCharVector dictionaryVector = newNullableVarCharVector("D2", bufferAllocator);
+    VarCharVector dictionaryVector = newVarCharVector("D2", bufferAllocator);
     dictionaryVector.allocateNewSafe();
     dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
     dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
@@ -395,16 +395,16 @@ protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvide
 
     Dictionary dictionary = provider.lookup(2L);
     Assert.assertNotNull(dictionary);
-    NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector());
+    VarCharVector dictionaryVector = ((VarCharVector) dictionary.getVector());
     Assert.assertEquals(2, dictionaryVector.getValueCount());
     Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
     Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
   }
 
   protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) {
-    NullableDecimalVector decimalVector1 = new NullableDecimalVector("decimal1", bufferAllocator, 10, 3);
-    NullableDecimalVector decimalVector2 = new NullableDecimalVector("decimal2", bufferAllocator, 4, 2);
-    NullableDecimalVector decimalVector3 = new NullableDecimalVector("decimal3", bufferAllocator, 16, 8);
+    DecimalVector decimalVector1 = new DecimalVector("decimal1", bufferAllocator, 10, 3);
+    DecimalVector decimalVector2 = new DecimalVector("decimal2", bufferAllocator, 4, 2);
+    DecimalVector decimalVector3 = new DecimalVector("decimal3", bufferAllocator, 16, 8);
 
     int count = 10;
     decimalVector1.allocateNew(count);
@@ -427,9 +427,9 @@ protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) {
   }
 
   protected void validateDecimalData(VectorSchemaRoot root) {
-    NullableDecimalVector decimalVector1 = (NullableDecimalVector) root.getVector("decimal1");
-    NullableDecimalVector decimalVector2 = (NullableDecimalVector) root.getVector("decimal2");
-    NullableDecimalVector decimalVector3 = (NullableDecimalVector) root.getVector("decimal3");
+    DecimalVector decimalVector1 = (DecimalVector) root.getVector("decimal1");
+    DecimalVector decimalVector2 = (DecimalVector) root.getVector("decimal2");
+    DecimalVector decimalVector3 = (DecimalVector) root.getVector("decimal3");
     int count = 10;
     Assert.assertEquals(count, root.getRowCount());
 
@@ -491,7 +491,7 @@ public void validateUnionData(int count, VectorSchemaRoot root) {
     }
   }
 
-  public void writeUnionData(int count, NullableMapVector parent) {
+  public void writeUnionData(int count, MapVector parent) {
     ArrowBuf varchar = allocator.buffer(3);
     varchar.readerIndex(0);
     varchar.setByte(0, 'a');
@@ -534,7 +534,7 @@ public void writeUnionData(int count, NullableMapVector parent) {
     varchar.release();
   }
 
-  protected void writeVarBinaryData(int count, NullableMapVector parent) {
+  protected void writeVarBinaryData(int count, MapVector parent) {
     Assert.assertTrue(count < 100);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
@@ -577,8 +577,8 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) {
     // ListVector lastSet should be the index of last value + 1
     Assert.assertEquals(listVector.getLastSet(), count);
 
-    // NullableVarBinaryVector lastSet should be the index of last value
-    NullableVarBinaryVector binaryVector = (NullableVarBinaryVector) listVector.getChildrenFromFields().get(0);
+    // VarBinaryVector lastSet should be the index of last value
+    VarBinaryVector binaryVector = (VarBinaryVector) listVector.getChildrenFromFields().get(0);
     Assert.assertEquals(binaryVector.getLastSet(), numVarBinaryValues - 1);
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
index 4387db03655d1..0cfc9ba919d85 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
@@ -36,14 +36,14 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.NullableFloat4Vector;
-import org.apache.arrow.vector.NullableIntVector;
-import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.TinyIntVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.ipc.message.ArrowBlock;
 import org.apache.arrow.vector.ipc.message.ArrowBuffer;
@@ -82,7 +82,7 @@ public void testWriteComplex() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeComplexData(count, parent);
       FieldVector root = parent.getChild("root");
       validateComplexContent(count, new VectorSchemaRoot(root));
@@ -268,7 +268,7 @@ public void testWriteReadUnion() throws IOException {
 
     // write
     try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+         MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeUnionData(count, parent);
       validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
       write(parent.getChild("root"), file, stream);
@@ -304,7 +304,7 @@ public void testWriteReadTiny() throws IOException {
 
     try (VectorSchemaRoot root = VectorSchemaRoot.create(MessageSerializerTest.testSchema(), allocator)) {
       root.getFieldVectors().get(0).allocateNew();
-      NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
+      TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0);
       for (int i = 0; i < 16; i++) {
         vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
       }
@@ -352,7 +352,7 @@ public void testWriteReadTiny() throws IOException {
 
   private void validateTinyData(VectorSchemaRoot root) {
     Assert.assertEquals(16, root.getRowCount());
-    NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
+    TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0);
     for (int i = 0; i < 16; i++) {
       if (i < 8) {
         Assert.assertEquals((byte) (i + 1), vector.get(i));
@@ -382,7 +382,7 @@ public void testWriteReadMetadata() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) {
+         MapVector vector = (MapVector) field.createVector(originalVectorAllocator)) {
       vector.allocateNewSafe();
       vector.setValueCount(0);
 
@@ -558,10 +558,10 @@ public void testWriteReadFixedSizeList() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
       FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class);
-      NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
-      NullableIntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), NullableIntVector.class);
+      Float4Vector floats = (Float4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
+      IntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), IntVector.class);
       parent.allocateNew();
 
       for (int i = 0; i < 10; i++) {
@@ -618,7 +618,7 @@ public void testWriteReadVarBin() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeVarBinaryData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateVarBinary(count, root);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
index 7a8586a9e708e..f87a0ebd54a0f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
@@ -26,7 +26,9 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 
-import org.apache.arrow.vector.NullableTinyIntVector;
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.TinyIntVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.ipc.ArrowStreamReader;
 import org.apache.arrow.vector.ipc.ArrowStreamWriter;
@@ -65,7 +67,7 @@ public void testReadWrite() throws IOException {
       int numBatches = 1;
 
       root.getFieldVectors().get(0).allocateNew();
-      NullableTinyIntVector vector = (NullableTinyIntVector)root.getFieldVectors().get(0);
+      TinyIntVector vector = (TinyIntVector)root.getFieldVectors().get(0);
       for (int i = 0; i < 16; i++) {
         vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1));
       }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
index 65e6cea2ec2e6..bd1ec94c8fd2b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
@@ -28,7 +28,7 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.TinyIntVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.ipc.ArrowStreamReader;
 import org.apache.arrow.vector.ipc.ArrowStreamWriter;
@@ -61,7 +61,7 @@ public void run() {
         writer.start();
         for (int j = 0; j < numBatches; j++) {
           root.getFieldVectors().get(0).allocateNew();
-          NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
+          TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0);
           // Send a changing batch id first
           vector.set(0, j);
           for (int i = 1; i < 16; i++) {
@@ -103,10 +103,9 @@ public boolean loadNextBatch() throws IOException {
             done = true;
             return false;
           }
-
           VectorSchemaRoot root = getVectorSchemaRoot();
           Assert.assertEquals(16, root.getRowCount());
-          NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0);
+          TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0);
           Assert.assertEquals((byte) (batchesRead - 1), vector.get(0));
           for (int i = 1; i < 16; i++) {
             if (i < 8) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
index c3e0b795174c0..625717048bf91 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
@@ -25,7 +25,7 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -73,7 +73,7 @@ public void testWriteComplexJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeComplexData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateComplexContent(root.getRowCount(), root);
@@ -95,7 +95,7 @@ public void testWriteReadUnionJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeUnionData(count, parent);
       printVectors(parent.getChildrenFromFields());
 
@@ -127,7 +127,7 @@ public void testWriteReadDateTimeJSON() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 
       writeDateTimeData(count, parent);
 
@@ -292,7 +292,7 @@ public void testWriteReadVarBinJSON() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeVarBinaryData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateVarBinary(count, root);

From ffb37dbe431271e6f2d958f8b22004383a539a7d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 28 Nov 2017 12:33:39 -0500
Subject: [PATCH 1293/1644] ARROW-1735: [C++] Test CastKernel writing into
 output array with non-zero offset

This uncovered some bugs. I inspected the other kernels that are untested and while they look fine, at some point we may want to add some more extensive unit tests about this

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1369 from wesm/ARROW-1735 and squashes the following commits:

de41d929 [Wes McKinney] Test CastKernel writing into output array with non-zero offset
---
 cpp/src/arrow/array.cc                 |  6 +--
 cpp/src/arrow/array.h                  | 11 +++--
 cpp/src/arrow/compute/compute-test.cc  | 60 ++++++++++++++++++++++++++
 cpp/src/arrow/compute/kernels/cast.cc  | 38 ++++++++--------
 cpp/src/arrow/python/arrow_to_python.h |  4 +-
 cpp/src/arrow/python/numpy_to_arrow.cc |  2 +-
 6 files changed, 92 insertions(+), 29 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 4b1fabfdcd441..0b235cc199ca1 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -103,7 +103,7 @@ static inline std::shared_ptr<ArrayData> SliceData(const ArrayData& data, int64_
   length = std::min(data.length - offset, length);
   offset += data.offset;
 
-  auto new_data = data.ShallowCopy();
+  auto new_data = data.Copy();
   new_data->length = length;
   new_data->offset = offset;
   new_data->null_count = data.null_count != 0 ? kUnknownNullCount : 0;
@@ -482,14 +482,14 @@ DictionaryArray::DictionaryArray(const std::shared_ptr<DataType>& type,
     : dict_type_(static_cast<const DictionaryType*>(type.get())) {
   DCHECK_EQ(type->id(), Type::DICTIONARY);
   DCHECK_EQ(indices->type_id(), dict_type_->index_type()->id());
-  auto data = indices->data()->ShallowCopy();
+  auto data = indices->data()->Copy();
   data->type = type;
   SetData(data);
 }
 
 void DictionaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
   this->Array::SetData(data);
-  auto indices_data = data_->ShallowCopy();
+  auto indices_data = data_->Copy();
   indices_data->type = dict_type_->index_type();
   std::shared_ptr<Array> result;
   indices_ = MakeArray(indices_data);
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index ec5381d6e28a3..ebe54adcb9eaf 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -143,9 +143,14 @@ struct ARROW_EXPORT ArrayData {
     return *this;
   }
 
-  std::shared_ptr<ArrayData> ShallowCopy() const {
-    return std::make_shared<ArrayData>(*this);
-  }
+  std::shared_ptr<ArrayData> Copy() const { return std::make_shared<ArrayData>(*this); }
+
+#ifndef ARROW_NO_DEPRECATED_API
+
+  // Deprecated since 0.8.0
+  std::shared_ptr<ArrayData> ShallowCopy() const { return Copy(); }
+
+#endif
 
   std::shared_ptr<DataType> type;
   int64_t length;
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index d5158978c8de8..c73bfa309fd95 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -709,6 +709,66 @@ TEST_F(TestCast, PreallocatedMemory) {
   ASSERT_ARRAYS_EQUAL(*expected, *result);
 }
 
+template <typename InType, typename InT, typename OutType, typename OutT>
+void CheckOffsetOutputCase(FunctionContext* ctx, const std::shared_ptr<DataType>& in_type,
+                           const vector<InT>& in_values,
+                           const std::shared_ptr<DataType>& out_type,
+                           const vector<OutT>& out_values) {
+  using OutTraits = TypeTraits<OutType>;
+
+  CastOptions options;
+
+  const int64_t length = static_cast<int64_t>(in_values.size());
+
+  shared_ptr<Array> arr, expected;
+  ArrayFromVector<InType, InT>(in_type, in_values, &arr);
+  ArrayFromVector<OutType, OutT>(out_type, out_values, &expected);
+
+  shared_ptr<Buffer> out_buffer;
+  ASSERT_OK(ctx->Allocate(OutTraits::bytes_required(length), &out_buffer));
+
+  std::unique_ptr<UnaryKernel> kernel;
+  ASSERT_OK(GetCastFunction(*in_type, out_type, options, &kernel));
+
+  const int64_t first_half = length / 2;
+
+  auto out_data = ArrayData::Make(out_type, length, {nullptr, out_buffer});
+  auto out_second_data = out_data->Copy();
+  out_second_data->offset = first_half;
+
+  Datum out_first(out_data);
+  Datum out_second(out_second_data);
+
+  // Cast each bit
+  ASSERT_OK(kernel->Call(ctx, Datum(arr->Slice(0, first_half)), &out_first));
+  ASSERT_OK(kernel->Call(ctx, Datum(arr->Slice(first_half)), &out_second));
+
+  shared_ptr<Array> result = MakeArray(out_data);
+
+  ASSERT_ARRAYS_EQUAL(*expected, *result);
+}
+
+TEST_F(TestCast, OffsetOutputBuffer) {
+  // ARROW-1735
+  vector<int32_t> v1 = {0, 10000, 2000, 1000, 0};
+  vector<int64_t> e1 = {0, 10000, 2000, 1000, 0};
+
+  auto in_type = int32();
+  auto out_type = int64();
+  CheckOffsetOutputCase<Int32Type, int32_t, Int64Type, int64_t>(&this->ctx_, in_type, v1,
+                                                                out_type, e1);
+
+  vector<bool> e2 = {false, true, true, true, false};
+
+  out_type = boolean();
+  CheckOffsetOutputCase<Int32Type, int32_t, BooleanType, bool>(&this->ctx_, in_type, v1,
+                                                               boolean(), e2);
+
+  vector<int16_t> e3 = {0, 10000, 2000, 1000, 0};
+  CheckOffsetOutputCase<Int32Type, int32_t, Int16Type, int16_t>(&this->ctx_, in_type, v1,
+                                                                int16(), e3);
+}
+
 template <typename TestType>
 class TestDictionaryCast : public TestCast {};
 
diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc
index d48d669922aa6..465be958cfabf 100644
--- a/cpp/src/arrow/compute/kernels/cast.cc
+++ b/cpp/src/arrow/compute/kernels/cast.cc
@@ -124,12 +124,7 @@ template <typename T>
 struct CastFunctor<T, NullType, typename std::enable_if<
                                     std::is_base_of<FixedWidthType, T>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
-                  const ArrayData& input, ArrayData* output) {
-    // Simply initialize data to 0
-    auto buf = output->buffers[1];
-    DCHECK_EQ(output->offset, 0);
-    memset(buf->mutable_data(), 0, buf->size());
-  }
+                  const ArrayData& input, ArrayData* output) {}
 };
 
 template <>
@@ -199,14 +194,19 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::v
                                                  !std::is_same<O, I>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {
-    using in_type = typename I::c_type;
-    DCHECK_EQ(output->offset, 0);
+    auto in_data = GetValues<typename I::c_type>(input, 1);
+    internal::BitmapWriter writer(output->buffers[1]->mutable_data(), output->offset,
+                                  input.length);
 
-    const in_type* in_data = GetValues<in_type>(input, 1);
-    uint8_t* out_data = GetMutableValues<uint8_t>(output, 1);
     for (int64_t i = 0; i < input.length; ++i) {
-      BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
+      if (*in_data++ != 0) {
+        writer.Set();
+      } else {
+        writer.Clear();
+      }
+      writer.Next();
     }
+    writer.Finish();
   }
 };
 
@@ -217,7 +217,6 @@ struct CastFunctor<O, I,
                   const ArrayData& input, ArrayData* output) {
     using in_type = typename I::c_type;
     using out_type = typename O::c_type;
-    DCHECK_EQ(output->offset, 0);
 
     auto in_offset = input.offset;
 
@@ -475,9 +474,10 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices,
 
   const index_c_type* in = GetValues<index_c_type>(*indices.data(), 1);
 
-  uint8_t* out = output->buffers[1]->mutable_data();
   int32_t byte_width =
       static_cast<const FixedSizeBinaryType&>(*output->type).byte_width();
+
+  uint8_t* out = output->buffers[1]->mutable_data() + byte_width * output->offset;
   for (int64_t i = 0; i < indices.length(); ++i) {
     if (valid_bits_reader.IsSet()) {
       const uint8_t* value = dictionary.Value(in[i]);
@@ -493,7 +493,7 @@ struct CastFunctor<
     typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {
-    DictionaryArray dict_array(input.ShallowCopy());
+    DictionaryArray dict_array(input.Copy());
 
     const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
@@ -565,7 +565,7 @@ struct CastFunctor<T, DictionaryType,
                    typename std::enable_if<std::is_base_of<BinaryType, T>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {
-    DictionaryArray dict_array(input.ShallowCopy());
+    DictionaryArray dict_array(input.Copy());
 
     const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
@@ -605,12 +605,10 @@ struct CastFunctor<T, DictionaryType,
 template <typename IndexType, typename c_type>
 void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
                                c_type* out) {
-  using index_c_type = typename IndexType::c_type;
-
   internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
                                            indices.length());
 
-  const index_c_type* in = GetValues<index_c_type>(*indices.data(), 1);
+  auto in = GetValues<typename IndexType::c_type>(*indices.data(), 1);
   for (int64_t i = 0; i < indices.length(); ++i) {
     if (valid_bits_reader.IsSet()) {
       out[i] = dictionary[in[i]];
@@ -627,7 +625,7 @@ struct CastFunctor<T, DictionaryType,
                   const ArrayData& input, ArrayData* output) {
     using c_type = typename T::c_type;
 
-    DictionaryArray dict_array(input.ShallowCopy());
+    DictionaryArray dict_array(input.Copy());
 
     const DictionaryType& type = static_cast<const DictionaryType&>(*input.type);
     const DataType& values_type = *type.dictionary()->type();
@@ -638,7 +636,7 @@ struct CastFunctor<T, DictionaryType,
 
     const c_type* dictionary = GetValues<c_type>(*type.dictionary()->data(), 1);
 
-    auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
+    auto out = GetMutableValues<c_type>(output, 1);
     const Array& indices = *dict_array.indices();
     switch (indices.type()->id()) {
       case Type::INT8:
diff --git a/cpp/src/arrow/python/arrow_to_python.h b/cpp/src/arrow/python/arrow_to_python.h
index 9440ffb32abeb..02a22f07d7e78 100644
--- a/cpp/src/arrow/python/arrow_to_python.h
+++ b/cpp/src/arrow/python/arrow_to_python.h
@@ -51,8 +51,8 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out);
 /// \brief Reconstruct SerializedPyObject from representation produced by
 /// SerializedPyObject::GetComponents.
 ///
-/// \param[in] num_tensors
-/// \param[in] num_buffers
+/// \param[in] num_tensors number of tensors in the object
+/// \param[in] num_buffers number of buffers in the object
 /// \param[in] data a list containing pyarrow.Buffer instances. Must be 1 +
 /// num_tensors * 2 + num_buffers in length
 /// \param[out] out the reconstructed object
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 0c0d1a9b3fdf2..0b1124d303df1 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -765,7 +765,7 @@ Status NumPyConverter::ConvertObjectStrings() {
   // If we saw PyBytes, convert everything to BinaryArray
   if (global_have_bytes) {
     for (size_t i = 0; i < out_arrays_.size(); ++i) {
-      auto binary_data = out_arrays_[i]->data()->ShallowCopy();
+      auto binary_data = out_arrays_[i]->data()->Copy();
       binary_data->type = ::arrow::binary();
       out_arrays_[i] = std::make_shared<BinaryArray>(binary_data);
     }

From 155bf076204dcb6bf1f68a556e9243556f5e4ebb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 28 Nov 2017 20:06:18 -0500
Subject: [PATCH 1294/1644] ARROW-1854: [Python] Use pickle to serialize numpy
 arrays of objects.

**Just posting this for discussion.** See the preceding discussion on https://issues.apache.org/jira/browse/ARROW-1854.

I think the ideal way to solve this would actually be to improve our handling of lists, which should be possible given that pickle seems to outperform us by 6x according to the benchmarks in https://issues.apache.org/jira/browse/ARROW-1854.

Note that the implementation in this PR will not handle numpy arrays of user-defined classes because it will not fall back to cloudpickle when needed.

cc @pcmoritz @wesm

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1360 from robertnishihara/numpyobject and squashes the following commits:

c37a0a08 [Wes McKinney] Fix flake
51915032 [Wes McKinney] Fix post rebase
43f2c805 [Wes McKinney] Add SerializationContext.clone method. Add pandas_serialization_context member that uses pickle for NumPy arrays with unsupported tensor types
c9440231 [Wes McKinney] Use pickle.HIGHEST_PROTOCOL, convert to Buffer then memoryview for more memory-efficient transport
cf719c3f [Robert Nishihara] Use pickle to serialize numpy arrays of objects.
---
 python/pyarrow/__init__.py                 |  1 +
 python/pyarrow/serialization.pxi           | 16 ++++++++
 python/pyarrow/serialization.py            | 46 +++++++++++++++-------
 python/pyarrow/tests/test_serialization.py |  7 ++--
 4 files changed, 53 insertions(+), 17 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index bd31b21c196de..a245fe6796023 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -125,6 +125,7 @@
 localfs = LocalFileSystem.get_instance()
 
 from pyarrow.serialization import (_default_serialization_context,
+                                   pandas_serialization_context,
                                    register_default_serialization_handlers)
 
 import pyarrow.types as types
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index bb266b2f928ff..faf164b3ebd20 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -57,6 +57,22 @@ cdef class SerializationContext:
         self.custom_serializers = dict()
         self.custom_deserializers = dict()
 
+    def clone(self):
+        """
+        Return copy of this SerializationContext
+
+        Returns
+        -------
+        clone : SerializationContext
+        """
+        result = SerializationContext()
+        result.type_to_type_id = self.type_to_type_id.copy()
+        result.whitelisted_types = self.whitelisted_types.copy()
+        result.custom_serializers = self.custom_serializers.copy()
+        result.custom_deserializers = self.custom_deserializers.copy()
+
+        return result
+
     def register_type(self, type_, type_id,
                       custom_serializer=None, custom_deserializer=None):
         """EXPERIMENTAL: Add type to the list of types we can serialize.
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index ab25b63d5710b..08e6cce751b53 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -22,7 +22,7 @@
 import numpy as np
 
 from pyarrow import serialize_pandas, deserialize_pandas
-from pyarrow.lib import _default_serialization_context
+from pyarrow.lib import _default_serialization_context, frombuffer
 
 try:
     import cloudpickle
@@ -30,6 +30,28 @@
     cloudpickle = pickle
 
 
+# ----------------------------------------------------------------------
+# Set up serialization for numpy with dtype object (primitive types are
+# handled efficiently with Arrow's Tensor facilities, see
+# python_to_arrow.cc)
+
+def _serialize_numpy_array_list(obj):
+    return obj.tolist(), obj.dtype.str
+
+
+def _deserialize_numpy_array_list(data):
+    return np.array(data[0], dtype=np.dtype(data[1]))
+
+
+def _serialize_numpy_array_pickle(obj):
+    pickled = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
+    return frombuffer(pickled)
+
+
+def _deserialize_numpy_array_pickle(data):
+    return pickle.loads(memoryview(data))
+
+
 def register_default_serialization_handlers(serialization_context):
 
     # ----------------------------------------------------------------------
@@ -80,21 +102,10 @@ def _deserialize_default_dict(data):
                                         custom_serializer=cloudpickle.dumps,
                                         custom_deserializer=cloudpickle.loads)
 
-    # ----------------------------------------------------------------------
-    # Set up serialization for numpy with dtype object (primitive types are
-    # handled efficiently with Arrow's Tensor facilities, see
-    # python_to_arrow.cc)
-
-    def _serialize_numpy_array(obj):
-        return obj.tolist(), obj.dtype.str
-
-    def _deserialize_numpy_array(data):
-        return np.array(data[0], dtype=np.dtype(data[1]))
-
     serialization_context.register_type(
         np.ndarray, 'np.array',
-        custom_serializer=_serialize_numpy_array,
-        custom_deserializer=_deserialize_numpy_array)
+        custom_serializer=_serialize_numpy_array_list,
+        custom_deserializer=_deserialize_numpy_array_list)
 
     # ----------------------------------------------------------------------
     # Set up serialization for pandas Series and DataFrame
@@ -153,3 +164,10 @@ def _deserialize_torch_tensor(data):
 
 
 register_default_serialization_handlers(_default_serialization_context)
+
+pandas_serialization_context = _default_serialization_context.clone()
+
+pandas_serialization_context.register_type(
+    np.ndarray, 'np.array',
+    custom_serializer=_serialize_numpy_array_pickle,
+    custom_deserializer=_deserialize_numpy_array_pickle)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index d06beeac99280..6d85621d411a7 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -212,11 +212,11 @@ def make_serialization_context():
 serialization_context = make_serialization_context()
 
 
-def serialization_roundtrip(value, f):
+def serialization_roundtrip(value, f, ctx=serialization_context):
     f.seek(0)
-    pa.serialize_to(value, f, serialization_context)
+    pa.serialize_to(value, f, ctx)
     f.seek(0)
-    result = pa.deserialize_from(f, None, serialization_context)
+    result = pa.deserialize_from(f, None, ctx)
     assert_equal(value, result)
 
     _check_component_roundtrip(value)
@@ -249,6 +249,7 @@ def test_primitive_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for obj in PRIMITIVE_OBJECTS:
             serialization_roundtrip(obj, mmap)
+            serialization_roundtrip(obj, mmap, pa.pandas_serialization_context)
 
 
 def test_serialize_to_buffer():

From b92c435b8f64f98593267fd72ecd61d26c23ffc0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 28 Nov 2017 20:07:11 -0500
Subject: [PATCH 1295/1644] ARROW-1684: [Python] Support selecting nested
 Parquet fields by any path prefix

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1366 from wesm/ARROW-1684 and squashes the following commits:

e63e42aa [Wes McKinney] Support selecting nested Parquet fields by any path prefix
---
 python/pyarrow/_parquet.pxd          |  1 +
 python/pyarrow/_parquet.pyx          | 29 ++++++++++++++++----
 python/pyarrow/parquet.py            | 41 ++++++++++++++++++++++++----
 python/pyarrow/tests/test_parquet.py | 22 +++++++++++++++
 4 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index 7e5e575096ddd..55b66b53d160d 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -37,6 +37,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
 
     cdef cppclass ColumnPath:
         c_string ToDotString()
+        vector[c_string] ToDotVector()
 
 
 cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index eca6b201b3ad0..147af217579e7 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -600,9 +600,11 @@ cdef class ParquetReader:
         object source
         CMemoryPool* allocator
         unique_ptr[FileReader] reader
-        column_idx_map
         FileMetaData _metadata
 
+    cdef public:
+        _column_idx_map
+
     def __cinit__(self, MemoryPool memory_pool=None):
         self.allocator = maybe_unbox_memory_pool(memory_pool)
         self._metadata = None
@@ -624,6 +626,23 @@ cdef class ParquetReader:
             check_status(OpenFile(rd_handle, self.allocator, properties,
                                   c_metadata, &self.reader))
 
+    property column_paths:
+
+        def __get__(self):
+            cdef:
+                FileMetaData container = self.metadata
+                const CFileMetaData* metadata = container._metadata
+                vector[c_string] path
+                int i = 0
+
+            paths = []
+            for i in range(0, metadata.num_columns()):
+                path = (metadata.schema().Column(i)
+                        .path().get().ToDotVector())
+                paths.append([frombytes(x) for x in path])
+
+            return paths
+
     @property
     def metadata(self):
         cdef:
@@ -729,14 +748,14 @@ cdef class ParquetReader:
             const CFileMetaData* metadata = container._metadata
             int i = 0
 
-        if self.column_idx_map is None:
-            self.column_idx_map = {}
+        if self._column_idx_map is None:
+            self._column_idx_map = {}
             for i in range(0, metadata.num_columns()):
                 col_bytes = tobytes(metadata.schema().Column(i)
                                     .path().get().ToDotString())
-                self.column_idx_map[col_bytes] = i
+                self._column_idx_map[col_bytes] = i
 
-        return self.column_idx_map[tobytes(column_name)]
+        return self._column_idx_map[tobytes(column_name)]
 
     def read_column(self, int column_index):
         cdef:
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 37da662806366..9fb890c6e6b8d 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from collections import defaultdict
 import os
 import inspect
 import json
@@ -54,6 +55,24 @@ def __init__(self, source, metadata=None, common_metadata=None):
         self.reader = ParquetReader()
         self.reader.open(source, metadata=metadata)
         self.common_metadata = common_metadata
+        self._nested_paths_by_prefix = self._build_nested_paths()
+
+    def _build_nested_paths(self):
+        paths = self.reader.column_paths
+
+        result = defaultdict(list)
+
+        def _visit_piece(i, key, rest):
+            result[key].append(i)
+
+            if len(rest) > 0:
+                nested_key = '.'.join((key, rest[0]))
+                _visit_piece(i, nested_key, rest[1:])
+
+        for i, path in enumerate(paths):
+            _visit_piece(i, path[0], path[1:])
+
+        return result
 
     @property
     def metadata(self):
@@ -75,7 +94,9 @@ def read_row_group(self, i, columns=None, nthreads=1,
         Parameters
         ----------
         columns: list
-            If not None, only these columns will be read from the row group.
+            If not None, only these columns will be read from the row group. A
+            column name may be a prefix of a nested field, e.g. 'a' will select
+            'a.b', 'a.c', and 'a.d.e'
         nthreads : int, default 1
             Number of columns to read in parallel. If > 1, requires that the
             underlying file source is threadsafe
@@ -100,7 +121,9 @@ def read(self, columns=None, nthreads=1, use_pandas_metadata=False):
         Parameters
         ----------
         columns: list
-            If not None, only these columns will be read from the file.
+            If not None, only these columns will be read from the file. A
+            column name may be a prefix of a nested field, e.g. 'a' will select
+            'a.b', 'a.c', and 'a.d.e'
         nthreads : int, default 1
             Number of columns to read in parallel. If > 1, requires that the
             underlying file source is threadsafe
@@ -143,7 +166,11 @@ def _get_column_indices(self, column_names, use_pandas_metadata=False):
         if column_names is None:
             return None
 
-        indices = list(map(self.reader.column_name_idx, column_names))
+        indices = []
+
+        for name in column_names:
+            if name in self._nested_paths_by_prefix:
+                indices.extend(self._nested_paths_by_prefix[name])
 
         if use_pandas_metadata:
             file_keyvalues = self.metadata.metadata
@@ -837,7 +864,9 @@ def read_table(source, columns=None, nthreads=1, metadata=None,
         name or directory name. For passing Python file objects or byte
         buffers, see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
     columns: list
-        If not None, only these columns will be read from the file.
+        If not None, only these columns will be read from the file. A column
+        name may be a prefix of a nested field, e.g. 'a' will select 'a.b',
+        'a.c', and 'a.d.e'
     nthreads : int, default 1
         Number of columns to read in parallel. Requires that the underlying
         file source is threadsafe
@@ -875,7 +904,9 @@ def read_pandas(source, columns=None, nthreads=1, metadata=None):
         name. For passing Python file objects or byte buffers,
         see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
     columns: list
-        If not None, only these columns will be read from the file.
+        If not None, only these columns will be read from the file. A column
+        name may be a prefix of a nested field, e.g. 'a' will select 'a.b',
+        'a.c', and 'a.d.e'
     nthreads : int, default 1
         Number of columns to read in parallel. Requires that the underlying
         file source is threadsafe
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 274ff458fd6cc..9004fc01ae4a0 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1468,6 +1468,28 @@ def test_index_column_name_duplicate(tmpdir):
     tm.assert_frame_equal(result_df, dfx)
 
 
+@parquet
+def test_parquet_nested_convenience(tmpdir):
+    # ARROW-1684
+    import pyarrow.parquet as pq
+
+    df = pd.DataFrame({
+        'a': [[1, 2, 3], None, [4, 5], []],
+        'b': [[1.], None, None, [6., 7.]],
+    })
+
+    path = str(tmpdir / 'nested_convenience.parquet')
+
+    table = pa.Table.from_pandas(df, preserve_index=False)
+    _write_table(table, path)
+
+    read = pq.read_table(path, columns=['a'])
+    tm.assert_frame_equal(read.to_pandas(), df[['a']])
+
+    read = pq.read_table(path, columns=['a', 'b'])
+    tm.assert_frame_equal(read.to_pandas(), df)
+
+
 @parquet
 def test_backwards_compatible_index_naming():
     expected_string = b"""\

From bbbbbfb1ecd29a4d435f942cec5f01226368c71c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 29 Nov 2017 10:43:20 -0500
Subject: [PATCH 1296/1644] ARROW-1844: [C++] Add initial Unique benchmarks for
 int64, variable-length strings

I also fixed a bug this surfaced in the hash table resize (unit test coverage was not adequate)

Now we have

```
$ ./release/compute-benchmark
Run on (8 X 4200.16 MHz CPU s)
2017-11-28 18:33:53
Benchmark                                                           Time           CPU Iterations
-------------------------------------------------------------------------------------------------
BM_BuildDictionary/min_time:1.000                                1352 us       1352 us       1038   2.88639GB/s
BM_BuildStringDictionary/min_time:1.000                          3994 us       3994 us        351   75.5809MB/s
BM_UniqueInt64NoNulls/16M/50/min_time:1.000/real_time           35814 us      35816 us         39   3.49023GB/s
BM_UniqueInt64NoNulls/16M/1024/min_time:1.000/real_time        119656 us     119660 us         12   1069.73MB/s
BM_UniqueInt64NoNulls/16M/10k/min_time:1.000/real_time         174924 us     174930 us          8   731.747MB/s
BM_UniqueInt64NoNulls/16M/1024k/min_time:1.000/real_time       448425 us     448440 us          3   285.443MB/s
BM_UniqueInt64WithNulls/16M/50/min_time:1.000/real_time         49511 us      49513 us         29   2.52468GB/s
BM_UniqueInt64WithNulls/16M/1024/min_time:1.000/real_time      134519 us     134523 us         10   951.541MB/s
BM_UniqueInt64WithNulls/16M/10k/min_time:1.000/real_time       191331 us     191336 us          7   668.999MB/s
BM_UniqueInt64WithNulls/16M/1024k/min_time:1.000/real_time     533597 us     533613 us          3   239.882MB/s
BM_UniqueString10bytes/16M/50/min_time:1.000/real_time         150731 us     150736 us          9    1061.5MB/s
BM_UniqueString10bytes/16M/1024/min_time:1.000/real_time       256929 us     256938 us          5   622.739MB/s
BM_UniqueString10bytes/16M/10k/min_time:1.000/real_time        414412 us     414426 us          3    386.09MB/s
BM_UniqueString10bytes/16M/1024k/min_time:1.000/real_time     1744253 us    1744308 us          1   91.7298MB/s
BM_UniqueString100bytes/16M/50/min_time:1.000/real_time        563890 us     563909 us          2   2.77093GB/s
BM_UniqueString100bytes/16M/1024/min_time:1.000/real_time      704695 us     704720 us          2   2.21727GB/s
BM_UniqueString100bytes/16M/10k/min_time:1.000/real_time       995685 us     995721 us          2   1.56927GB/s
BM_UniqueString100bytes/16M/1024k/min_time:1.000/real_time    3584108 us    3584230 us          1   446.415MB/s
```

We can also refactor the hash table implementations without worrying too much about whether we're making things slower

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1370 from wesm/ARROW-1844 and squashes the following commits:

638f1a11 [Wes McKinney] Decrease resize load factor to 0.5
2885c645 [Wes McKinney] Multiply bytes processed by state.iterations()
f7b36194 [Wes McKinney] Add initial Unique benchmarks for int64, strings
---
 cpp/src/arrow/compute/compute-benchmark.cc | 127 ++++++++++++++++++++-
 cpp/src/arrow/compute/compute-test.cc      |   4 +-
 cpp/src/arrow/compute/kernels/hash.cc      |   4 +-
 3 files changed, 129 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/compute/compute-benchmark.cc b/cpp/src/arrow/compute/compute-benchmark.cc
index 974fffcd6d99d..aa7d899c864e7 100644
--- a/cpp/src/arrow/compute/compute-benchmark.cc
+++ b/cpp/src/arrow/compute/compute-benchmark.cc
@@ -81,8 +81,131 @@ static void BM_BuildStringDictionary(
   state.SetBytesProcessed(state.iterations() * total_bytes);
 }
 
-BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+template <typename Type>
+struct HashParams {
+  using T = typename Type::c_type;
+
+  double null_percent;
+
+  void GenerateTestData(const int64_t length, const int64_t num_unique,
+                        std::shared_ptr<Array>* arr) const {
+    std::vector<int64_t> draws;
+    std::vector<T> values;
+    std::vector<bool> is_valid;
+    test::randint<int64_t>(length, 0, num_unique, &draws);
+    for (int64_t draw : draws) {
+      values.push_back(draw);
+    }
+
+    if (this->null_percent > 0) {
+      test::random_is_valid(length, this->null_percent, &is_valid);
+      ArrayFromVector<Type, T>(is_valid, values, arr);
+    } else {
+      ArrayFromVector<Type, T>(values, arr);
+    }
+  }
+
+  int64_t GetBytesProcessed(int64_t length) const { return length * sizeof(T); }
+};
+
+template <>
+struct HashParams<StringType> {
+  double null_percent;
+  int32_t byte_width;
+  void GenerateTestData(const int64_t length, const int64_t num_unique,
+                        std::shared_ptr<Array>* arr) const {
+    std::vector<int64_t> draws;
+    test::randint<int64_t>(length, 0, num_unique, &draws);
+
+    const int64_t total_bytes = this->byte_width * num_unique;
+    std::vector<uint8_t> uniques(total_bytes);
+    const uint32_t seed = 0;
+    test::random_bytes(total_bytes, seed, uniques.data());
+
+    std::vector<bool> is_valid;
+    if (this->null_percent > 0) {
+      test::random_is_valid(length, this->null_percent, &is_valid);
+    }
+
+    StringBuilder builder;
+    for (int64_t i = 0; i < length; ++i) {
+      if (this->null_percent == 0 || is_valid[i]) {
+        ABORT_NOT_OK(builder.Append(uniques.data() + this->byte_width * draws[i],
+                                    this->byte_width));
+      } else {
+        ABORT_NOT_OK(builder.AppendNull());
+      }
+    }
+    ABORT_NOT_OK(builder.Finish(arr));
+  }
+
+  int64_t GetBytesProcessed(int64_t length) const { return length * byte_width; }
+};
+
+template <typename ParamType>
+void BenchUnique(benchmark::State& state, const ParamType& params, int64_t length,
+                 int64_t num_unique) {
+  std::shared_ptr<Array> arr;
+  params.GenerateTestData(length, num_unique, &arr);
+
+  FunctionContext ctx;
+  while (state.KeepRunning()) {
+    std::shared_ptr<Array> out;
+    ABORT_NOT_OK(Unique(&ctx, Datum(arr), &out));
+  }
+  state.SetBytesProcessed(state.iterations() * params.GetBytesProcessed(length));
+}
+
+template <typename ParamType>
+void BenchDictionaryEncode(benchmark::State& state, const ParamType& params,
+                           int64_t length, int64_t num_unique) {
+  std::shared_ptr<Array> arr;
+  params.GenerateTestData(length, num_unique, &arr);
+
+  FunctionContext ctx;
+  while (state.KeepRunning()) {
+    Datum out;
+    ABORT_NOT_OK(DictionaryEncode(&ctx, Datum(arr), &out));
+  }
+  state.SetBytesProcessed(state.iterations() * params.GetBytesProcessed(length));
+}
+
+static void BM_UniqueInt64NoNulls(benchmark::State& state) {
+  BenchUnique(state, HashParams<Int64Type>{0}, state.range(0), state.range(1));
+}
+
+static void BM_UniqueInt64WithNulls(benchmark::State& state) {
+  BenchUnique(state, HashParams<Int64Type>{0.05}, state.range(0), state.range(1));
+}
+
+static void BM_UniqueString10bytes(benchmark::State& state) {
+  // Byte strings with 10 bytes each
+  BenchUnique(state, HashParams<StringType>{0.05, 10}, state.range(0), state.range(1));
+}
+
+static void BM_UniqueString100bytes(benchmark::State& state) {
+  // Byte strings with 100 bytes each
+  BenchUnique(state, HashParams<StringType>{0.05, 100}, state.range(0), state.range(1));
+}
+
+BENCHMARK(BM_BuildDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildStringDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
+
+constexpr int64_t kHashBenchmarkLength = 1 << 24;
+
+#define ADD_HASH_ARGS(WHAT)                        \
+  WHAT->Args({kHashBenchmarkLength, 50})           \
+      ->Args({kHashBenchmarkLength, 1 << 10})      \
+      ->Args({kHashBenchmarkLength, 10 * 1 << 10}) \
+      ->Args({kHashBenchmarkLength, 1 << 20})      \
+      ->MinTime(1.0)                               \
+      ->Unit(benchmark::kMicrosecond)              \
+      ->UseRealTime()
+
+ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64NoNulls));
+ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64WithNulls));
+ADD_HASH_ARGS(BENCHMARK(BM_UniqueString10bytes));
+ADD_HASH_ARGS(BENCHMARK(BM_UniqueString100bytes));
 
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index c73bfa309fd95..84af8f7c6b0a7 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -869,8 +869,8 @@ TYPED_TEST(TestHashKernelPrimitive, PrimitiveResizeTable) {
     return;
   }
 
-  const int64_t kTotalValues = 10000;
-  const int64_t kRepeats = 10;
+  const int64_t kTotalValues = 1000000;
+  const int64_t kRepeats = 5;
 
   vector<T> values;
   vector<T> uniques;
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index 66c907369e3ee..750f1d36ac7d8 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -43,7 +43,7 @@ typedef int32_t hash_slot_t;
 static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits<int32_t>::max();
 
 // The maximum load factor for the hash table before resizing.
-static constexpr double kMaxHashTableLoad = 0.7;
+static constexpr double kMaxHashTableLoad = 0.5;
 
 enum class SIMDMode : char { NOSIMD, SSE4, AVX2 };
 
@@ -260,7 +260,7 @@ struct HashDictionary<Type, enable_if_has_c_type<Type>> {
       COMPUTE_HASH;                                                              \
       while (kHashSlotEmpty != new_hash_slots[j]) {                              \
         ++j;                                                                     \
-        if (ARROW_PREDICT_FALSE(j == hash_table_size_)) {                        \
+        if (ARROW_PREDICT_FALSE(j == new_size)) {                                \
           j = 0;                                                                 \
         }                                                                        \
       }                                                                          \

From ff8efbf046a7a1e21ca05776550d29f1445eab29 Mon Sep 17 00:00:00 2001
From: Ivan Sadikov <ivan.sadikov@team.telstra.com>
Date: Wed, 29 Nov 2017 10:54:43 -0500
Subject: [PATCH 1297/1644] ARROW-1869: [JAVA] Fix LowCostIdentityHashMap name

JIRA: https://issues.apache.org/jira/browse/ARROW-1869

This PR fixes spelling error in class name for `LowCostIdentityHashMap`.
Follow-up for https://github.com/apache/arrow/pull/1150.

Author: Ivan Sadikov <ivan.sadikov@team.telstra.com>

Closes #1372 from sadikovi/fix-low-cost-identity-hash-map and squashes the following commits:

e3529f68 [Ivan Sadikov] fix low cost identity hash map name
---
 .../org/apache/arrow/memory/AllocationManager.java     |  2 +-
 ...IdentityHasMap.java => LowCostIdentityHashMap.java} | 10 +++++-----
 ...tityHasMap.java => TestLowCostIdentityHashMap.java} |  6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)
 rename java/memory/src/main/java/org/apache/arrow/memory/{LowCostIdentityHasMap.java => LowCostIdentityHashMap.java} (98%)
 rename java/memory/src/test/java/org/apache/arrow/memory/{TestLowCostIdentityHasMap.java => TestLowCostIdentityHashMap.java} (95%)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index 14687b54634ac..419be3429721b 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -76,7 +76,7 @@ public class AllocationManager {
   private final UnsafeDirectLittleEndian underlying;
   // ARROW-1627 Trying to minimize memory overhead caused by previously used IdentityHashMap
   // see JIRA for details
-  private final LowCostIdentityHasMap<BaseAllocator, BufferLedger> map = new LowCostIdentityHasMap<>();
+  private final LowCostIdentityHashMap<BaseAllocator, BufferLedger> map = new LowCostIdentityHashMap<>();
   private final ReadWriteLock lock = new ReentrantReadWriteLock();
   private final AutoCloseableLock readLock = new AutoCloseableLock(lock.readLock());
   private final AutoCloseableLock writeLock = new AutoCloseableLock(lock.writeLock());
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java b/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java
similarity index 98%
rename from java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java
rename to java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java
index 1153fb5936471..fb7033815d5f4 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java
@@ -28,7 +28,7 @@
  * that provides "getKey" method
  * @param <V>
  */
-public class LowCostIdentityHasMap<K, V extends ValueWithKeyIncluded<K>> {
+public class LowCostIdentityHashMap<K, V extends ValueWithKeyIncluded<K>> {
 
   /*
    * The internal data structure to hold values.
@@ -52,7 +52,7 @@ public class LowCostIdentityHasMap<K, V extends ValueWithKeyIncluded<K>> {
   /**
    * Creates an Map with default expected maximum size.
    */
-  public LowCostIdentityHasMap() {
+  public LowCostIdentityHashMap() {
     this(DEFAULT_MIN_SIZE);
   }
 
@@ -63,7 +63,7 @@ public LowCostIdentityHasMap() {
    *            The estimated maximum number of entries that will be put in
    *            this map.
    */
-  public LowCostIdentityHasMap(int maxSize) {
+  public LowCostIdentityHashMap(int maxSize) {
     if (maxSize >= 0) {
       this.size = 0;
       threshold = getThreshold(maxSize);
@@ -96,7 +96,7 @@ private int computeElementArraySize() {
   private Object[] newElementArray(int s) {
     return new Object[s];
   }
-  
+
   /**
    * Removes all elements from this map, leaving it empty.
    *
@@ -331,4 +331,4 @@ public V getNextValue() {
     }
     return null;
   }
-}
\ No newline at end of file
+}
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java b/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java
similarity index 95%
rename from java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java
rename to java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java
index c1196147b4925..0237b38048f52 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java
@@ -27,11 +27,11 @@
 /**
  * To test simplified implementation of IdentityHashMap
  */
-public class TestLowCostIdentityHasMap {
+public class TestLowCostIdentityHashMap {
 
   @Test
   public void testIdentityHashMap() throws Exception {
-    LowCostIdentityHasMap<String, StringWithKey> hashMap = new LowCostIdentityHasMap<>();
+    LowCostIdentityHashMap<String, StringWithKey> hashMap = new LowCostIdentityHashMap<>();
 
     StringWithKey obj1 = new StringWithKey("s1key", "s1value");
     StringWithKey obj2 = new StringWithKey("s2key", "s2value");
@@ -88,7 +88,7 @@ public void testIdentityHashMap() throws Exception {
 
   @Test
   public void testLargeMap() throws Exception {
-    LowCostIdentityHasMap<String, StringWithKey> hashMap = new LowCostIdentityHasMap<>();
+    LowCostIdentityHashMap<String, StringWithKey> hashMap = new LowCostIdentityHashMap<>();
 
     String [] keys = new String[200];
     for (int i = 0; i < 200; i++) {

From 1fd3457a77cda15ecf4068f1ced51d0683b519ab Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 29 Nov 2017 12:48:39 -0500
Subject: [PATCH 1298/1644] ARROW-1862: [GLib] Add GArrowDictionaryArray

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1365 from kou/glib-dictionary-array and squashes the following commits:

83bfa135 [Kouhei Sutou] [GLib] Add GArrowDictionaryArray
---
 c_glib/arrow-glib/basic-array.cpp         |   3 +
 c_glib/arrow-glib/composite-array.cpp     | 107 ++++++++++++++++++++++
 c_glib/arrow-glib/composite-array.h       |  21 +++++
 c_glib/arrow-glib/composite-data-type.cpp |   8 ++
 c_glib/test/test-dictionary-array.rb      |  63 +++++++++++++
 5 files changed, 202 insertions(+)
 create mode 100644 c_glib/test/test-dictionary-array.rb

diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp
index 0698a04098050..36cf4603a55ee 100644
--- a/c_glib/arrow-glib/basic-array.cpp
+++ b/c_glib/arrow-glib/basic-array.cpp
@@ -2091,6 +2091,9 @@ garrow_array_new_raw(std::shared_ptr<arrow::Array> *arrow_array)
   case arrow::Type::type::STRUCT:
     type = GARROW_TYPE_STRUCT_ARRAY;
     break;
+  case arrow::Type::type::DICTIONARY:
+    type = GARROW_TYPE_DICTIONARY_ARRAY;
+    break;
   default:
     type = GARROW_TYPE_ARRAY;
     break;
diff --git a/c_glib/arrow-glib/composite-array.cpp b/c_glib/arrow-glib/composite-array.cpp
index 445103d374037..14cc46d733ee8 100644
--- a/c_glib/arrow-glib/composite-array.cpp
+++ b/c_glib/arrow-glib/composite-array.cpp
@@ -44,6 +44,11 @@ G_BEGIN_DECLS
  * or more structs. One struct has zero or more fields. If you don't
  * have Arrow format data, you need to use #GArrowStructArrayBuilder
  * to create a new array.
+ *
+ * #GArrowDictionaryArray is a class for dictionary array. It can
+ * store data with dictionary and indices. It's space effective than
+ * normal array when the array has many same values. You can convert a
+ * normal array to dictionary array by garrow_array_dictionary_encode().
  */
 
 G_DEFINE_TYPE(GArrowListArray,               \
@@ -234,4 +239,106 @@ garrow_struct_array_get_fields(GArrowStructArray *array)
   return g_list_reverse(fields);
 }
 
+
+G_DEFINE_TYPE(GArrowDictionaryArray,            \
+              garrow_dictionary_array,          \
+              GARROW_TYPE_ARRAY)
+
+static void
+garrow_dictionary_array_init(GArrowDictionaryArray *object)
+{
+}
+
+static void
+garrow_dictionary_array_class_init(GArrowDictionaryArrayClass *klass)
+{
+}
+
+/**
+ * garrow_dictionary_array_new:
+ * @data_type: The data type of dictionary.
+ * @indices: The indices of values in dictionary.
+ *
+ * Returns: A newly created #GArrowDictionaryArray.
+ *
+ * Since: 0.8.0
+ */
+GArrowDictionaryArray *
+garrow_dictionary_array_new(GArrowDataType *data_type,
+                            GArrowArray *indices)
+{
+  const auto arrow_data_type = garrow_data_type_get_raw(data_type);
+  const auto arrow_indices = garrow_array_get_raw(indices);
+  auto arrow_dictionary_array =
+    std::make_shared<arrow::DictionaryArray>(arrow_data_type,
+                                             arrow_indices);
+  auto arrow_array =
+    std::static_pointer_cast<arrow::Array>(arrow_dictionary_array);
+  return GARROW_DICTIONARY_ARRAY(garrow_array_new_raw(&arrow_array));
+}
+
+/**
+ * garrow_dictionary_array_get_indices:
+ * @array: A #GArrowDictionaryArray.
+ *
+ * Returns: (transfer full): The indices of values in dictionary.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_dictionary_array_get_indices(GArrowDictionaryArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_dictionary_array =
+    std::static_pointer_cast<arrow::DictionaryArray>(arrow_array);
+  auto arrow_indices = arrow_dictionary_array->indices();
+  return garrow_array_new_raw(&arrow_indices);
+}
+
+/**
+ * garrow_dictionary_array_get_dictionary:
+ * @array: A #GArrowDictionaryArray.
+ *
+ * Returns: (transfer full): The dictionary of this array.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_dictionary_array_get_dictionary(GArrowDictionaryArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_dictionary_array =
+    std::static_pointer_cast<arrow::DictionaryArray>(arrow_array);
+  auto arrow_dictionary = arrow_dictionary_array->dictionary();
+  return garrow_array_new_raw(&arrow_dictionary);
+}
+
+/**
+ * garrow_dictionary_array_get_dictionary_data_type:
+ * @array: A #GArrowDictionaryArray.
+ *
+ * Returns: (transfer full): The dictionary data type of this array.
+ *
+ * Since: 0.8.0
+ */
+GArrowDictionaryDataType *
+garrow_dictionary_array_get_dictionary_data_type(GArrowDictionaryArray *array)
+{
+  auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array));
+  auto arrow_dictionary_array =
+    std::static_pointer_cast<arrow::DictionaryArray>(arrow_array);
+  auto arrow_dictionary_data_type = arrow_dictionary_array->dict_type();
+  auto const_arrow_data_type =
+    static_cast<const arrow::DataType *>(arrow_dictionary_data_type);
+  auto arrow_data_type = const_cast<arrow::DataType *>(const_arrow_data_type);
+  struct NullDeleter {
+    void operator()(arrow::DataType *data_type) {
+    }
+  };
+  std::shared_ptr<arrow::DataType>
+    shared_arrow_data_type(arrow_data_type, NullDeleter());
+  auto data_type = garrow_data_type_new_raw(&shared_arrow_data_type);
+  return GARROW_DICTIONARY_DATA_TYPE(data_type);
+}
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/composite-array.h b/c_glib/arrow-glib/composite-array.h
index ebf955498724b..c59a61681bf42 100644
--- a/c_glib/arrow-glib/composite-array.h
+++ b/c_glib/arrow-glib/composite-array.h
@@ -129,4 +129,25 @@ GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
                                            gint i);
 GList *garrow_struct_array_get_fields(GArrowStructArray *array);
 
+
+#define GARROW_TYPE_DICTIONARY_ARRAY (garrow_dictionary_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDictionaryArray,
+                         garrow_dictionary_array,
+                         GARROW,
+                         DICTIONARY_ARRAY,
+                         GArrowArray)
+struct _GArrowDictionaryArrayClass
+{
+  GArrowArrayClass parent_class;
+};
+
+GArrowDictionaryArray *
+garrow_dictionary_array_new(GArrowDataType *data_type, GArrowArray *indices);
+GArrowArray *
+garrow_dictionary_array_get_indices(GArrowDictionaryArray *array);
+GArrowArray *
+garrow_dictionary_array_get_dictionary(GArrowDictionaryArray *array);
+GArrowDictionaryDataType *
+garrow_dictionary_array_get_dictionary_data_type(GArrowDictionaryArray *array);
+
 G_END_DECLS
diff --git a/c_glib/arrow-glib/composite-data-type.cpp b/c_glib/arrow-glib/composite-data-type.cpp
index 7ce8a978e2036..5f742e50e4028 100644
--- a/c_glib/arrow-glib/composite-data-type.cpp
+++ b/c_glib/arrow-glib/composite-data-type.cpp
@@ -158,6 +158,8 @@ garrow_dictionary_data_type_class_init(GArrowDictionaryDataTypeClass *klass)
  * @ordered: Whether dictionary contents are ordered or not.
  *
  * Returns: The newly created dictionary data type.
+ *
+ * Since: 0.8.0
  */
 GArrowDictionaryDataType *
 garrow_dictionary_data_type_new(GArrowDataType *index_data_type,
@@ -177,6 +179,8 @@ garrow_dictionary_data_type_new(GArrowDataType *index_data_type,
  * @data_type: The #GArrowDictionaryDataType.
  *
  * Returns: (transfer full): The #GArrowDataType of index.
+ *
+ * Since: 0.8.0
  */
 GArrowDataType *
 garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_type)
@@ -193,6 +197,8 @@ garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_t
  * @data_type: The #GArrowDictionaryDataType.
  *
  * Returns: (transfer full): The dictionary as #GArrowArray.
+ *
+ * Since: 0.8.0
  */
 GArrowArray *
 garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type)
@@ -209,6 +215,8 @@ garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type)
  * @data_type: The #GArrowDictionaryDataType.
  *
  * Returns: Whether dictionary contents are ordered or not.
+ *
+ * Since: 0.8.0
  */
 gboolean
 garrow_dictionary_data_type_is_ordered(GArrowDictionaryDataType *data_type)
diff --git a/c_glib/test/test-dictionary-array.rb b/c_glib/test/test-dictionary-array.rb
new file mode 100644
index 0000000000000..d4f4b3443057d
--- /dev/null
+++ b/c_glib/test/test-dictionary-array.rb
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDictionaryArray < Test::Unit::TestCase
+  include Helper::Buildable
+
+  def setup
+    @index_data_type = Arrow::Int32DataType.new
+    @dictionary = build_string_array(["C", "C++", "Ruby"])
+    @ordered = false
+    @data_type = Arrow::DictionaryDataType.new(@index_data_type,
+                                               @dictionary,
+                                               @ordered)
+  end
+
+  sub_test_case(".new") do
+    def test_new
+      indices = build_int32_array([0, 2, 2, 1, 0])
+      dictionary_array = Arrow::DictionaryArray.new(@data_type, indices)
+      assert_equal(<<-STRING.chomp, dictionary_array.to_s)
+
+-- is_valid: all not null
+-- dictionary: ["C", "C++", "Ruby"]
+-- indices: [0, 2, 2, 1, 0]
+      STRING
+    end
+  end
+
+  sub_test_case("instance methods") do
+    def setup
+      super
+      @indices = build_int32_array([0, 2, 2, 1, 0])
+      @dictionary_array = Arrow::DictionaryArray.new(@data_type, @indices)
+    end
+
+    def test_indices
+      assert_equal(@indices, @dictionary_array.indices)
+    end
+
+    def test_dictionary
+      assert_equal(@dictionary, @dictionary_array.dictionary)
+    end
+
+    def test_dictionary_data_type
+      assert_equal(@data_type,
+                   @dictionary_array.dictionary_data_type)
+    end
+  end
+end

From 705d842664d6626d25843ebbb331186033126287 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Fri, 1 Dec 2017 16:14:44 -0500
Subject: [PATCH 1299/1644] ARROW-1874: [GLib] Add garrow_array_unique()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1377 from kou/glib-unique and squashes the following commits:

4385e22d [Kouhei Sutou] Add garrow_array_unique()
---
 c_glib/arrow-glib/basic-array.cpp | 33 +++++++++++++++++++++++++++++++
 c_glib/arrow-glib/basic-array.h   |  2 ++
 c_glib/test/test-unique.rb        | 31 +++++++++++++++++++++++++++++
 3 files changed, 66 insertions(+)
 create mode 100644 c_glib/test/test-unique.rb

diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp
index 36cf4603a55ee..a7afaca958354 100644
--- a/c_glib/arrow-glib/basic-array.cpp
+++ b/c_glib/arrow-glib/basic-array.cpp
@@ -563,6 +563,39 @@ garrow_array_cast(GArrowArray *array,
   return garrow_array_new_raw(&arrow_casted_array);
 }
 
+/**
+ * garrow_array_unique:
+ * @array: A #GArrowArray.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   A newly created unique elements array on success, %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_array_unique(GArrowArray *array,
+                    GError **error)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto memory_pool = arrow::default_memory_pool();
+  arrow::compute::FunctionContext context(memory_pool);
+  std::shared_ptr<arrow::Array> arrow_unique_array;
+  auto status = arrow::compute::Unique(&context,
+                                       arrow::compute::Datum(arrow_array),
+                                       &arrow_unique_array);
+  if (!status.ok()) {
+    std::stringstream message;
+    message << "[array][unique] <";
+    message << arrow_array->type()->ToString();
+    message << ">";
+    garrow_error_check(error, status, message.str().c_str());
+    return NULL;
+  }
+
+  return garrow_array_new_raw(&arrow_unique_array);
+}
+
 
 G_DEFINE_TYPE(GArrowNullArray,               \
               garrow_null_array,             \
diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h
index c01a04f07d8b9..613a59b071a5d 100644
--- a/c_glib/arrow-glib/basic-array.h
+++ b/c_glib/arrow-glib/basic-array.h
@@ -66,6 +66,8 @@ GArrowArray   *garrow_array_cast        (GArrowArray *array,
                                          GArrowDataType *target_data_type,
                                          GArrowCastOptions *options,
                                          GError **error);
+GArrowArray   *garrow_array_unique      (GArrowArray *array,
+                                         GError **error);
 
 #define GARROW_TYPE_NULL_ARRAY                  \
   (garrow_null_array_get_type())
diff --git a/c_glib/test/test-unique.rb b/c_glib/test/test-unique.rb
new file mode 100644
index 0000000000000..b94ff462b4600
--- /dev/null
+++ b/c_glib/test/test-unique.rb
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUnique < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_int32
+    assert_equal(build_int32_array([1, 3, -1, -3]),
+                 build_int32_array([1, 3, 1, -1, -3, -1]).unique)
+  end
+
+  def test_string
+    assert_equal(build_string_array(["Ruby", "Python"]),
+                 build_string_array(["Ruby", "Python", "Ruby"]).unique)
+  end
+end

From ad9105e639b115e2868483b534fb3ac7a8921de3 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Fri, 1 Dec 2017 16:17:49 -0500
Subject: [PATCH 1300/1644] ARROW-1817: [Java] Configure JsonReader to read
 floating point NaN values

The last upgrade of the Jackson JSON library changed behavior to no longer allow reading of "NaN" values  by default.  This change configures the JSON generator and parser to allow for NaN values (unquoted) alongside standard floating point numbers.  A test was added for JSON writing/reading and modified the test for Arrow file and stream .

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1375 from BryanCutler/java-JsonReader-all_non_numeric-ARROW-1817 and squashes the following commits:

4c4682a9 [Bryan Cutler] configure JsonWriter to write NaN not as strings, add test for read and write of float with NaN
1fa24f45 [Bryan Cutler] added conf for JacksonParser to allow NaN tokens
---
 .../arrow/vector/ipc/JsonFileReader.java      |  2 ++
 .../arrow/vector/ipc/JsonFileWriter.java      |  2 ++
 .../apache/arrow/vector/ipc/BaseFileTest.java | 33 +++++++++++--------
 .../apache/arrow/vector/ipc/TestJSONFile.java | 27 +++++++++++++++
 4 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
index 04d50331b76d8..f274b748e55dc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -69,6 +69,8 @@ public JsonFileReader(File inputFile, BufferAllocator allocator) throws JsonPars
     this.allocator = allocator;
     MappingJsonFactory jsonFactory = new MappingJsonFactory();
     this.parser = jsonFactory.createParser(inputFile);
+    // Allow reading NaN for floating point values
+    this.parser.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
index 067fb25b8d86c..1c9e1d3809594 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -84,6 +84,8 @@ public JsonFileWriter(File outputFile, JSONWriteConfig config) throws IOExceptio
       prettyPrinter.indentArraysWith(NopIndenter.instance);
       this.generator.setPrettyPrinter(prettyPrinter);
     }
+    // Allow writing of floating point NaN values not as strings
+    this.generator.configure(JsonGenerator.Feature.QUOTE_NON_NUMERIC_NUMBERS, false);
   }
 
   public void start(Schema schema, DictionaryProvider provider) throws IOException {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index 3514acaa24278..d26385d749496 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -47,6 +47,7 @@
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.DateMilliWriter;
+import org.apache.arrow.vector.complex.writer.Float4Writer;
 import org.apache.arrow.vector.complex.writer.IntWriter;
 import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
 import org.apache.arrow.vector.complex.writer.TimeStampMilliTZWriter;
@@ -95,10 +96,28 @@ public void tearDown() {
     DateTimeZone.setDefault(defaultTimezone);
   }
 
+  protected void writeData(int count, MapVector parent) {
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    Float4Writer float4Writer = rootWriter.float4("float");
+    for (int i = 0; i < count; i++) {
+      intWriter.setPosition(i);
+      intWriter.writeInt(i);
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+      float4Writer.setPosition(i);
+      float4Writer.writeFloat4(i == 0 ? Float.NaN : i);
+    }
+    writer.setValueCount(count);
+  }
+
   protected void validateContent(int count, VectorSchemaRoot root) {
     for (int i = 0; i < count; i++) {
       Assert.assertEquals(i, root.getVector("int").getObject(i));
       Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i));
+      Assert.assertEquals(i == 0 ? Float.NaN : i, root.getVector("float").getObject(i));
     }
   }
 
@@ -454,20 +473,6 @@ protected void validateDecimalData(VectorSchemaRoot root) {
     }
   }
 
-  protected void writeData(int count, MapVector parent) {
-    ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
-    IntWriter intWriter = rootWriter.integer("int");
-    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
-    for (int i = 0; i < count; i++) {
-      intWriter.setPosition(i);
-      intWriter.writeInt(i);
-      bigIntWriter.setPosition(i);
-      bigIntWriter.writeBigInt(i);
-    }
-    writer.setValueCount(count);
-  }
-
   public void validateUnionData(int count, VectorSchemaRoot root) {
     FieldReader unionReader = root.getVector("union").getReader();
     for (int i = 0; i < count; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
index 625717048bf91..4f9093b8c0253 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
@@ -38,6 +38,33 @@
 public class TestJSONFile extends BaseFileTest {
   private static final Logger LOGGER = LoggerFactory.getLogger(TestJSONFile.class);
 
+  @Test
+  public void testWriteRead() throws IOException {
+    File file = new File("target/mytest.json");
+    int count = COUNT;
+
+    // write
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
+      writeData(count, parent);
+      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+        JsonFileReader reader = new JsonFileReader(file, readerAllocator)
+    ) {
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateContent(count, root);
+      }
+    }
+  }
+
   @Test
   public void testWriteReadComplexJSON() throws IOException {
     File file = new File("target/mytest_complex.json");

From 3bed4ebfb344d83e3c09a33aac51f6917fc787fe Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 3 Dec 2017 16:37:22 -0500
Subject: [PATCH 1301/1644] ARROW-1865: [C++] Do not alter number of rows
 attribute when removing last column from Table

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1373 from wesm/ARROW-1865 and squashes the following commits:

bc7cd336 [Wes McKinney] Removing last column from a Table does not alter number of rows. Test in C++ and Python
---
 cpp/src/arrow/table-test.cc        | 20 ++++++++++++++++++++
 cpp/src/arrow/table.cc             |  3 ++-
 python/pyarrow/tests/test_table.py | 14 ++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 8a2288710c896..3f1c6be3a87f6 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -404,6 +404,26 @@ TEST_F(TestTable, RemoveColumn) {
   ASSERT_TRUE(result->Equals(*expected));
 }
 
+TEST_F(TestTable, RemoveColumnEmpty) {
+  // ARROW-1865
+  const int64_t length = 10;
+
+  auto f0 = field("f0", int32());
+  auto schema = ::arrow::schema({f0});
+  auto a0 = MakeRandomArray<Int32Array>(length);
+
+  auto table = Table::Make(schema, {std::make_shared<Column>(f0, a0)});
+
+  std::shared_ptr<Table> empty;
+  ASSERT_OK(table->RemoveColumn(0, &empty));
+
+  ASSERT_EQ(table->num_rows(), empty->num_rows());
+
+  std::shared_ptr<Table> added;
+  ASSERT_OK(empty->AddColumn(0, table->column(0), &added));
+  ASSERT_EQ(table->num_rows(), added->num_rows());
+}
+
 TEST_F(TestTable, AddColumn) {
   const int64_t length = 10;
   MakeExample1(length);
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 129524b7e4320..2cf6c26523965 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -202,7 +202,8 @@ class SimpleTable : public Table {
     std::shared_ptr<Schema> new_schema;
     RETURN_NOT_OK(schema_->RemoveField(i, &new_schema));
 
-    *out = Table::Make(new_schema, internal::DeleteVectorElement(columns_, i));
+    *out = Table::Make(new_schema, internal::DeleteVectorElement(columns_, i),
+                       this->num_rows());
     return Status::OK();
   }
 
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index ab012340c0ab9..e72761d32f634 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -312,6 +312,20 @@ def test_table_remove_column():
     assert t2.equals(expected)
 
 
+def test_table_remove_column_empty():
+    # ARROW-1865
+    data = [
+        pa.array(range(5)),
+    ]
+    table = pa.Table.from_arrays(data, names=['a'])
+
+    t2 = table.remove_column(0)
+    assert len(t2) == len(table)
+
+    t3 = t2.add_column(0, table[0])
+    assert t3.equals(table)
+
+
 def test_concat_tables():
     data = [
         list(range(5)),

From ff4e40707048d9b3ad4744d3928d48857d32fd70 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 3 Dec 2017 16:38:39 -0500
Subject: [PATCH 1302/1644] ARROW-1878: [GLib] Add
 garrow_array_dictionary_encode()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1381 from kou/glib-dictionary-encode and squashes the following commits:

f2da6fcb [Kouhei Sutou] Add garrow_array_dictionary_encode()
---
 c_glib/arrow-glib/basic-array.cpp     | 38 +++++++++++++++++++++++++
 c_glib/arrow-glib/basic-array.h       |  2 ++
 c_glib/test/test-dictionary-encode.rb | 41 +++++++++++++++++++++++++++
 3 files changed, 81 insertions(+)
 create mode 100644 c_glib/test/test-dictionary-encode.rb

diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp
index a7afaca958354..3c027c0e11240 100644
--- a/c_glib/arrow-glib/basic-array.cpp
+++ b/c_glib/arrow-glib/basic-array.cpp
@@ -596,6 +596,44 @@ garrow_array_unique(GArrowArray *array,
   return garrow_array_new_raw(&arrow_unique_array);
 }
 
+/**
+ * garrow_array_dictionary_encode:
+ * @array: A #GArrowArray.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full):
+ *   A newly created #GArrowDictionarArray for the @array on success,
+ *   %NULL on error.
+ *
+ * Since: 0.8.0
+ */
+GArrowArray *
+garrow_array_dictionary_encode(GArrowArray *array,
+                               GError **error)
+{
+  auto arrow_array = garrow_array_get_raw(array);
+  auto memory_pool = arrow::default_memory_pool();
+  arrow::compute::FunctionContext context(memory_pool);
+  arrow::compute::Datum dictionary_encoded_datum;
+  auto status =
+    arrow::compute::DictionaryEncode(&context,
+                                     arrow::compute::Datum(arrow_array),
+                                     &dictionary_encoded_datum);
+  if (!status.ok()) {
+    std::stringstream message;
+    message << "[array][dictionary-encode] <";
+    message << arrow_array->type()->ToString();
+    message << ">";
+    garrow_error_check(error, status, message.str().c_str());
+    return NULL;
+  }
+
+  auto arrow_dictionary_encoded_array =
+    arrow::MakeArray(dictionary_encoded_datum.array());
+
+  return garrow_array_new_raw(&arrow_dictionary_encoded_array);
+}
+
 
 G_DEFINE_TYPE(GArrowNullArray,               \
               garrow_null_array,             \
diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h
index 613a59b071a5d..420c30ecc7161 100644
--- a/c_glib/arrow-glib/basic-array.h
+++ b/c_glib/arrow-glib/basic-array.h
@@ -68,6 +68,8 @@ GArrowArray   *garrow_array_cast        (GArrowArray *array,
                                          GError **error);
 GArrowArray   *garrow_array_unique      (GArrowArray *array,
                                          GError **error);
+GArrowArray   *garrow_array_dictionary_encode(GArrowArray *array,
+                                              GError **error);
 
 #define GARROW_TYPE_NULL_ARRAY                  \
   (garrow_null_array_get_type())
diff --git a/c_glib/test/test-dictionary-encode.rb b/c_glib/test/test-dictionary-encode.rb
new file mode 100644
index 0000000000000..ea77be64ae41d
--- /dev/null
+++ b/c_glib/test/test-dictionary-encode.rb
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestDictionaryEncode < Test::Unit::TestCase
+  include Helper::Buildable
+  include Helper::Omittable
+
+  def test_int32
+    array = build_int32_array([1, 3, 1, -1, -3, -1])
+    assert_equal(<<-STRING.chomp, array.dictionary_encode.to_s)
+
+-- is_valid: all not null
+-- dictionary: [1, 3, -1, -3]
+-- indices: [0, 1, 0, 2, 3, 2]
+    STRING
+  end
+
+  def test_string
+    array = build_string_array(["Ruby", "Python", "Ruby"])
+    assert_equal(<<-STRING.chomp, array.dictionary_encode.to_s)
+
+-- is_valid: all not null
+-- dictionary: ["Ruby", "Python"]
+-- indices: [0, 1, 0]
+    STRING
+  end
+end

From 3ee75a09bb0781350f996a15828145b2171f0983 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Sun, 3 Dec 2017 16:43:00 -0500
Subject: [PATCH 1303/1644] ARROW-1719: [Java] Remove accessor and mutator
 interface

Author: Li Jin <ice.xelloss@gmail.com>

Closes #1374 from icexelloss/remove-accessor-mutator and squashes the following commits:

3423ad4f [Li Jin] Remove accessor and mutator interface
---
 .../codegen/templates/ComplexWriters.java     |   6 +-
 .../main/codegen/templates/UnionVector.java   |  12 --
 .../arrow/vector/BaseDataValueVector.java     |   2 +-
 .../arrow/vector/BaseFixedWidthVector.java    |  17 +--
 .../apache/arrow/vector/BaseValueVector.java  |  63 -----------
 .../arrow/vector/BaseVariableWidthVector.java |  12 --
 .../org/apache/arrow/vector/ValueVector.java  | 107 +++++-------------
 .../arrow/vector/VariableWidthVector.java     |  12 --
 .../apache/arrow/vector/VectorSchemaRoot.java |   2 +-
 .../org/apache/arrow/vector/ZeroVector.java   |  45 --------
 .../complex/BaseRepeatedValueVector.java      |  12 --
 .../vector/complex/FixedSizeListVector.java   |  12 --
 .../arrow/vector/complex/MapVector.java       |  12 --
 .../vector/complex/NonNullableMapVector.java  |  10 --
 .../vector/complex/RepeatedValueVector.java   |  39 -------
 .../impl/UnionFixedSizeListReader.java        |   2 +-
 .../arrow/vector/TestVectorReAlloc.java       |   2 +-
 .../apache/arrow/vector/ipc/BaseFileTest.java |   2 -
 18 files changed, 41 insertions(+), 328 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 98672d54cc38e..4a1545a173dd3 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -79,17 +79,17 @@ protected int idx() {
 
   public void write(${minor.class?cap_first}Holder h) {
     mutator.addSafe(idx(), h);
-    vector.getMutator().setValueCount(idx()+1);
+    vector.setValueCount(idx()+1);
   }
 
   public void write(${minor.class?cap_first}Holder h) {
     mutator.addSafe(idx(), h);
-    vector.getMutator().setValueCount(idx()+1);
+    vector.setValueCount(idx()+1);
   }
 
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
     mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
-    vector.getMutator().setValueCount(idx()+1);
+    vector.setValueCount(idx()+1);
   }
 
   public void setPosition(int idx) {
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 501933f8fbc31..60cd24dcc4457 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -416,18 +416,6 @@ public void copyValueSafe(int from, int to) {
     }
   }
 
-  @Override
-  @Deprecated
-  public Accessor getAccessor() {
-    throw new UnsupportedOperationException("Accessor is not supported for reading from UNION");
-  }
-
-  @Override
-  @Deprecated
-  public Mutator getMutator() {
-    throw new UnsupportedOperationException("Mutator is not supported for writing to UNION");
-  }
-
   @Override
   public FieldReader getReader() {
     if (reader == null) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index 6d9eb1db03af5..80675135e3c1c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -99,7 +99,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
 
   @Override
   public int getBufferSize() {
-    if (getAccessor().getValueCount() == 0) {
+    if (getValueCount() == 0) {
       return 0;
     }
     return data.writerIndex();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index cc056904ae8a1..77026d4e5a112 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -91,19 +91,6 @@ public BaseFixedWidthVector(final String name, final BufferAllocator allocator,
    * the top class as of now is not a good idea.
    */
 
-
-  @Override
-  @Deprecated
-  public Mutator getMutator() {
-    throw new UnsupportedOperationException("Mutator is not supported for writing to vector");
-  }
-
-  @Override
-  @Deprecated
-  public Accessor getAccessor() {
-    throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
-  }
-
   /**
    * Get the memory address of buffer that manages the validity
    * (NULL or NON-NULL nature) of elements in the vector.
@@ -711,6 +698,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length,
    *
    * @return the number of null elements.
    */
+  @Override
   public int getNullCount() {
     return BitVectorHelper.getNullCount(validityBuffer, valueCount);
   }
@@ -721,6 +709,7 @@ public int getNullCount() {
    *
    * @return valueCount for the vector
    */
+  @Override
   public int getValueCount() {
     return valueCount;
   }
@@ -730,6 +719,7 @@ public int getValueCount() {
    *
    * @param valueCount  value count to set
    */
+  @Override
   public void setValueCount(int valueCount) {
     this.valueCount = valueCount;
     final int currentValueCapacity = getValueCapacity();
@@ -789,6 +779,7 @@ public boolean isSafe(int index) {
    * @param index  position of element
    * @return true if element at given index is null, false otherwise
    */
+  @Override
   public boolean isNull(int index) {
     return (isSet(index) == 0);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index fc0ab3e5915cc..6418ea4f8a069 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -68,42 +68,6 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
     return getTransferPair(name, allocator);
   }
 
-  public abstract static class BaseAccessor implements ValueVector.Accessor {
-    protected BaseAccessor() {
-    }
-
-    @Override
-    public boolean isNull(int index) {
-      return false;
-    }
-
-    @Override
-    // override this in case your implementation is faster, see BitVector
-    public int getNullCount() {
-      int nullCount = 0;
-      for (int i = 0; i < getValueCount(); i++) {
-        if (isNull(i)) {
-          nullCount++;
-        }
-      }
-      return nullCount;
-    }
-  }
-
-  public abstract static class BaseMutator implements ValueVector.Mutator {
-    protected BaseMutator() {
-    }
-
-    @Override
-    public void generateTestData(int values) {
-    }
-
-    //TODO: consider making mutator stateless(if possible) on another issue.
-    @Override
-    public void reset() {
-    }
-  }
-
   @Override
   public Iterator<ValueVector> iterator() {
     return Collections.emptyIterator();
@@ -136,33 +100,6 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) {
     return buffer;
   }
 
-  @Override
-  public int getValueCount() {
-    return getAccessor().getValueCount();
-  }
-
-  @Override
-  public void setValueCount(int valueCount) {
-    getMutator().setValueCount(valueCount);
-  }
-
-  @Override
-
-  public Object getObject(int index) {
-    return getAccessor().getObject(index);
-  }
-
-  @Override
-
-  public int getNullCount() {
-    return getAccessor().getNullCount();
-  }
-
-  @Override
-  public boolean isNull(int index) {
-    return getAccessor().isNull(index);
-  }
-
   /* number of bytes for the validity buffer for the given valueCount */
   protected static int getValidityBufferSizeFromCount(final int valueCount) {
     return (int) Math.ceil(valueCount / 8.0);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index b515586bc9f36..98820a2367b75 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -90,18 +90,6 @@ public BaseVariableWidthVector(final String name, final BufferAllocator allocato
    * the top class as of now is not a good idea.
    */
 
-  @Override
-  @Deprecated
-  public VariableWidthMutator getMutator() {
-    throw new UnsupportedOperationException("Mutator is not supported for writing into vector");
-  }
-
-  @Override
-  @Deprecated
-  public VariableWidthAccessor getAccessor() {
-    throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
-  }
-
   /**
    * Get buffer that manages the validity (NULL or NON-NULL nature) of
    * elements in the vector. Consider it as a buffer for internal bit vector
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 0d3acf1646f53..e77c1b12703ec 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -135,18 +135,6 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
    */
   TransferPair makeTransferPair(ValueVector target);
 
-  /**
-   * @return an {@link org.apache.arrow.vector.ValueVector.Accessor accessor} that is used to read from this vector
-   * instance.
-   */
-  Accessor getAccessor();
-
-  /**
-   * @return an {@link org.apache.arrow.vector.ValueVector.Mutator mutator} that is used to write to this vector
-   * instance.
-   */
-  Mutator getMutator();
-
   /**
    * @return a {@link org.apache.arrow.vector.complex.reader.FieldReader field reader} that supports reading values
    * from this vector.
@@ -160,7 +148,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * Returns the number of bytes that is used by this vector if it holds the given number
-   * of values. The result will be the same as if Mutator.setValueCount() were called, followed
+   * of values. The result will be the same as if setValueCount() were called, followed
    * by calling getBufferSize(), but without any of the closing side-effects that setValueCount()
    * implies wrt finishing off the population of a vector. Some operations might wish to use
    * this to determine how much memory has been used by a vector so far, even though it is
@@ -182,91 +170,56 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
    */
   ArrowBuf[] getBuffers(boolean clear);
 
-  /**
-   * An abstraction that is used to read from this vector instance.
-   */
-  interface Accessor {
-    /**
-     * Get the Java Object representation of the element at the specified position. Useful for testing.
-     *
-     * @param index Index of the value to get
-     * @return the friendly java type
-     */
-    Object getObject(int index);
-
-    /**
-     * @return the number of values that is stored in this vector.
-     */
-    int getValueCount();
-
-    /**
-     * @param index the index to check for nullity
-     * @return true if the value at the given index is null, false otherwise.
-     */
-    boolean isNull(int index);
-
-    /**
-     * @return the number of null values
-     */
-    int getNullCount();
-  }
-
-  /**
-   * An abstraction that is used to write into this vector instance.
-   */
-  interface Mutator {
-    /**
-     * Sets the number of values that is stored in this vector to the given value count.
-     *
-     * @param valueCount value count to set.
-     */
-    void setValueCount(int valueCount);
-
-    /**
-     * Resets the mutator to pristine state.
-     */
-    void reset();
-
-    /**
-     * @param values the number of values to generate
-     * @deprecated this has nothing to do with value vector abstraction and should be removed.
-     */
-    @Deprecated
-    void generateTestData(int values);
-  }
-
   /**
    * Gets the underlying buffer associated with validity vector
    *
    * @return buffer
    */
-  public ArrowBuf getValidityBuffer();
+  ArrowBuf getValidityBuffer();
 
   /**
    * Gets the underlying buffer associated with data vector
    *
    * @return buffer
    */
-  public ArrowBuf getDataBuffer();
+  ArrowBuf getDataBuffer();
 
   /**
    * Gets the underlying buffer associated with offset vector
    *
    * @return buffer
    */
-  public ArrowBuf getOffsetBuffer();
+  ArrowBuf getOffsetBuffer();
 
-  /* temporarily add these methods here until we remove other vectors
-   * (non-nullable) which are under ValueVector hierarchy and still
-   * use the mutator/accessor interfaces.
+  /**
+   * Gets the number of values
+   * @return
    */
-  public int getValueCount();
+  int getValueCount();
 
-  public void setValueCount(int valueCount);
+  /**
+   * Set number of values in the vector
+   * @return
+   */
+  void setValueCount(int valueCount);
 
-  public Object getObject(int index);
+  /**
+   * Get friendly type object from the vector
+   * @param index
+   * @return
+   */
+  Object getObject(int index);
 
-  public int getNullCount();
+  /**
+   * Returns number of null elements in the vector
+   * @return
+   */
+  int getNullCount();
 
-  public boolean isNull(int index);
+  /**
+   * Check whether an element in the vector is null
+   * @param index
+   * @return
+   */
+  boolean isNull(int index);
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
index 04c00b7c8349c..593d4dceaf782 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
@@ -35,17 +35,5 @@ public interface VariableWidthVector extends ValueVector {
    */
   int getByteCapacity();
 
-  VariableWidthMutator getMutator();
-
-  VariableWidthAccessor getAccessor();
-
-  interface VariableWidthAccessor extends Accessor {
-    int getValueLength(int index);
-  }
-
   int getCurrentSizeInBytes();
-
-  interface VariableWidthMutator extends Mutator {
-    void setValueLengthSafe(int index, int length);
-  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
index 196393fb9584a..3fd33d66d4ef5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -140,7 +140,7 @@ public String contentToTSVString() {
     for (int i = 0; i < rowCount; i++) {
       row.clear();
       for (FieldVector v : fieldVectors) {
-        row.add(v.getAccessor().getObject(i));
+        row.add(v.getObject(i));
       }
       printRow(sb, row);
     }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 0ab3a7b6843c3..962a1c947071f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -60,41 +60,6 @@ public void copyValueSafe(int from, int to) {
     }
   };
 
-  private final Accessor defaultAccessor = new Accessor() {
-    @Override
-    public Object getObject(int index) {
-      return null;
-    }
-
-    @Override
-    public int getValueCount() {
-      return 0;
-    }
-
-    @Override
-    public boolean isNull(int index) {
-      return true;
-    }
-
-    @Override
-    public int getNullCount() {
-      return 0;
-    }
-  };
-
-  private final Mutator defaultMutator = new Mutator() {
-    @Override
-    public void setValueCount(int valueCount) {
-    }
-
-    @Override
-    public void reset() {
-    }
-
-    @Override
-    public void generateTestData(int values) {
-    }
-  };
 
   public ZeroVector() {
   }
@@ -186,16 +151,6 @@ public TransferPair makeTransferPair(ValueVector target) {
     return defaultPair;
   }
 
-  @Override
-  public Accessor getAccessor() {
-    return defaultAccessor;
-  }
-
-  @Override
-  public Mutator getMutator() {
-    return defaultMutator;
-  }
-
   @Override
   public FieldReader getReader() {
     return NullReader.INSTANCE;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 4648d078949a7..a9221f2f6ea56 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -277,16 +277,4 @@ public void setValueCount(int valueCount) {
             offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
     vector.setValueCount(childValueCount);
   }
-
-  @Override
-  @Deprecated
-  public RepeatedAccessor getAccessor() {
-    throw new UnsupportedOperationException("Accessor is not supported for reading from LIST.");
-  }
-
-  @Override
-  @Deprecated
-  public RepeatedMutator getMutator() {
-    throw new UnsupportedOperationException("Mutator is not supported for writing to LIST");
-  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 774a10dbfb98f..93a8127cfd3b4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -150,18 +150,6 @@ public List<BufferBacked> getFieldInnerVectors() {
     throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
   }
 
-  @Override
-  @Deprecated
-  public Accessor getAccessor() {
-    throw new UnsupportedOperationException("Accessor is not supported for reading from vector");
-  }
-
-  @Override
-  @Deprecated
-  public Mutator getMutator() {
-    throw new UnsupportedOperationException("Mutator is not supported for writing to vector");
-  }
-
   @Override
   public UnionFixedSizeListReader getReader() {
     return reader;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 075ae83ea4d84..72cc29f30ca14 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -497,16 +497,4 @@ public void setValueCount(int valueCount) {
   public void reset() {
     valueCount = 0;
   }
-
-  @Override
-  @Deprecated
-  public Accessor getAccessor() {
-    throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP");
-  }
-
-  @Override
-  @Deprecated
-  public Mutator getMutator() {
-    throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP");
-  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
index cc3ac4148a865..1cd7b70d7bfad 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
@@ -252,16 +252,6 @@ public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
     return natural.min(getChildren()).getValueCapacity();
   }
 
-  @Override
-  public Accessor getAccessor() {
-    throw new UnsupportedOperationException("accessor is not needed for MAP");
-  }
-
-  @Override
-  public Mutator getMutator() {
-    throw new UnsupportedOperationException("mutator is not needed for MAP");
-  }
-
   @Override
   public Object getObject(int index) {
     Map<String, Object> vv = new JsonStringHashMap<>();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index 91147c663f248..36401172994c5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -41,43 +41,4 @@ public interface RepeatedValueVector extends ValueVector {
    * @return the underlying data vector or null if none exists.
    */
   ValueVector getDataVector();
-
-  @Override
-  RepeatedAccessor getAccessor();
-
-  @Override
-  RepeatedMutator getMutator();
-
-  interface RepeatedAccessor extends ValueVector.Accessor {
-    /**
-     * The result includes empty, null valued cells.
-     *
-     * @return total number of cells that vector contains.
-     */
-    int getInnerValueCount();
-
-
-    /**
-     * @param index the index of the value for which we want the size
-     * @return number of cells that the value at the given index contains.
-     */
-    int getInnerValueCountAt(int index);
-
-    /**
-     * @param index value index
-     * @return true if the value at the given index is empty, false otherwise.
-     */
-    boolean isEmpty(int index);
-  }
-
-  interface RepeatedMutator extends ValueVector.Mutator {
-
-    /**
-     * Starts a new value that is a container of cells.
-     *
-     * @param index index of new value to start
-     * @return index into the child vector
-     */
-    int startNewValue(int index);
-  }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
index 4ad2f6f5fad7a..56ae379dca210 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java
@@ -80,7 +80,7 @@ public void read(int index, UnionHolder holder) {
       }
     }
     holder.reader = data.getReader();
-    holder.isSet = vector.getAccessor().isNull(idx()) ? 0 : 1;
+    holder.isSet = vector.isNull(idx()) ? 0 : 1;
   }
 
   @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index c0df4881f380c..1b13c2ed91935 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -130,7 +130,7 @@ public void testMapType() {
       assertEquals(512, vector.getValueCapacity());
 
       try {
-        vector.getAccessor().getObject(513);
+        vector.getObject(513);
         Assert.fail("Expected out of bounds exception");
       } catch (Exception e) {
         // ok
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index d26385d749496..1a7f71c72b483 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -34,11 +34,9 @@
 import org.apache.arrow.vector.TimeMilliVector;
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
-import org.apache.arrow.vector.ValueVector.Accessor;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;

From 3b438bc91ef153d5c963b676a534909d60ba3eb4 Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Sun, 3 Dec 2017 16:44:51 -0500
Subject: [PATCH 1304/1644] ARROW-1755: [C++] CMake option to link msvc crt
 statically

- CMake option `ARROW_USE_STATIC_CRT` to turn on Arrow build with statically linked MSVC C/C++ runtimes (turned OFF by default). All 3rd parties should be linked with the same version of crt.
- Debug and Release Appveyor's build jobs with `ARROW_USE_STATIC_CRT` option turned on
- Patches for lz4 and zstd libs updated to include runtime linkage selection property used by msbuild.exe

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #1333 from MaxRis/ARROW-1755 and squashes the following commits:

5c9317dc [Max Risuhin] ARROW-1755: [C++] CMake option to link msvc crt statically
---
 appveyor.yml                                  |   4 +
 ci/msvc-build.bat                             |  32 +
 cpp/CMakeLists.txt                            |   4 +
 ...lz4_msbuild_gl_runtimelibrary_params.patch | 601 ++++++++++++++++++
 ...build_wholeprogramoptimization_param.patch | 225 -------
 ...std_msbuild_gl_runtimelibrary_params.patch | 528 +++++++++++++++
 ...build_wholeprogramoptimization_param.patch | 199 ------
 cpp/cmake_modules/SetupCxxFlags.cmake         |   9 +
 cpp/cmake_modules/ThirdpartyToolchain.cmake   |  64 +-
 9 files changed, 1226 insertions(+), 440 deletions(-)
 create mode 100644 cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch
 delete mode 100644 cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch
 create mode 100644 cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch
 delete mode 100644 cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch

diff --git a/appveyor.yml b/appveyor.yml
index 55c58d0bf664d..e647b8b77080f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -40,6 +40,10 @@ environment:
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Release"
+    - JOB: "Static_Crt_Build"
+      GENERATOR: Visual Studio 14 2015 Win64
+      PYTHON: "3.5"
+      ARCH: "64"
     - JOB: "Toolchain"
       GENERATOR: Visual Studio 14 2015 Win64
       PYTHON: "3.5"
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index e8eb0945e516e..62ebcf364e77b 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -17,6 +17,38 @@
 
 @echo on
 
+if "%JOB%" == "Static_Crt_Build" (
+  mkdir cpp\build-debug
+  pushd cpp\build-debug
+
+  cmake -G "%GENERATOR%" ^
+        -DARROW_USE_STATIC_CRT=ON ^
+        -DARROW_BOOST_USE_SHARED=OFF ^
+        -DCMAKE_BUILD_TYPE=Debug ^
+        -DARROW_CXXFLAGS="/MP" ^
+        ..  || exit /B
+
+  cmake --build . --config Debug || exit /B
+  popd
+
+  mkdir cpp\build-release
+  pushd cpp\build-release
+
+  cmake -G "%GENERATOR%" ^
+        -DARROW_USE_STATIC_CRT=ON ^
+        -DARROW_BOOST_USE_SHARED=OFF ^
+        -DCMAKE_BUILD_TYPE=Release ^
+        -DARROW_CXXFLAGS="/WX /MP" ^
+        ..  || exit /B
+
+  cmake --build . --config Release || exit /B
+  ctest -VV  || exit /B
+  popd
+
+  @rem Finish Static_Crt_Build build successfully
+  exit /B 0
+)
+
 if "%JOB%" == "Build_Debug" (
   mkdir cpp\build-debug
   pushd cpp\build-debug
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f4b7b29b9d3f3..dd159ced29193 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -210,6 +210,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
       "Lz4 static lib suffix used on Windows with MSVC (default _static)")
     set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "ZStd static lib suffix used on Windows with MSVC (default _static)")
+
+    option(ARROW_USE_STATIC_CRT
+      "Build Arrow with statically linked CRT"
+      OFF)
   endif()
 endif()
 
diff --git a/cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch b/cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch
new file mode 100644
index 0000000000000..c79898d9a786c
--- /dev/null
+++ b/cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch
@@ -0,0 +1,601 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj
+index aaf81ad..096741a 100644
+--- a/visual/VS2010/datagen/datagen.vcxproj
++++ b/visual/VS2010/datagen/datagen.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -113,6 +117,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj
+index 76d12c9..34ca686 100644
+--- a/visual/VS2010/frametest/frametest.vcxproj
++++ b/visual/VS2010/frametest/frametest.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -113,6 +117,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
+index c10552a..c1ca231 100644
+--- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
++++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -117,6 +121,46 @@
+       <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj
+index e2d95c9..613ff3c 100644
+--- a/visual/VS2010/fullbench/fullbench.vcxproj
++++ b/visual/VS2010/fullbench/fullbench.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -113,6 +117,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj
+index 85d6c9b..be8b9a1 100644
+--- a/visual/VS2010/fuzzer/fuzzer.vcxproj
++++ b/visual/VS2010/fuzzer/fuzzer.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -113,6 +117,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
+index 389f13c..e30e667 100644
+--- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
++++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
+@@ -40,15 +40,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -116,6 +120,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj
+index a0b8000..38d2ce2 100644
+--- a/visual/VS2010/liblz4/liblz4.vcxproj
++++ b/visual/VS2010/liblz4/liblz4.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -115,6 +119,46 @@
+       <GenerateDebugInformation>true</GenerateDebugInformation>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
+diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj
+index 693e121..9eb005b 100644
+--- a/visual/VS2010/lz4/lz4.vcxproj
++++ b/visual/VS2010/lz4/lz4.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>Unicode</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -115,6 +119,46 @@
+       <AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
+     </Link>
+   </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+     <ClCompile>
+       <WarningLevel>Level4</WarningLevel>
diff --git a/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch b/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch
deleted file mode 100644
index ee0f8a12054bc..0000000000000
--- a/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch
+++ /dev/null
@@ -1,225 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj
-index aaf81ad..67b716f 100644
---- a/visual/VS2010/datagen/datagen.vcxproj
-+++ b/visual/VS2010/datagen/datagen.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj
-index 76d12c9..723571d 100644
---- a/visual/VS2010/frametest/frametest.vcxproj
-+++ b/visual/VS2010/frametest/frametest.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
-index c10552a..0c8f293 100644
---- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
-+++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj
-index e2d95c9..4cd88d0 100644
---- a/visual/VS2010/fullbench/fullbench.vcxproj
-+++ b/visual/VS2010/fullbench/fullbench.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj
-index 85d6c9b..3ddc77d 100644
---- a/visual/VS2010/fuzzer/fuzzer.vcxproj
-+++ b/visual/VS2010/fuzzer/fuzzer.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
-index 389f13c..038a4d2 100644
---- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
-+++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
-@@ -40,15 +40,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>DynamicLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>DynamicLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj
-index a0b8000..9aad8c2 100644
---- a/visual/VS2010/liblz4/liblz4.vcxproj
-+++ b/visual/VS2010/liblz4/liblz4.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>StaticLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>StaticLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj
-index 693e121..7e63f1e 100644
---- a/visual/VS2010/lz4/lz4.vcxproj
-+++ b/visual/VS2010/lz4/lz4.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>Unicode</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
diff --git a/cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch b/cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch
new file mode 100644
index 0000000000000..a168a814950e6
--- /dev/null
+++ b/cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch
@@ -0,0 +1,528 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+diff --git a/build/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj
+index bd8a213d..691d39fe 100644
+--- a/build/VS2010/datagen/datagen.vcxproj
++++ b/build/VS2010/datagen/datagen.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -84,6 +88,46 @@
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
+index e697318e..a5720189 100644
+--- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
++++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -84,6 +88,46 @@
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj
+index 2bff4ca4..d64fac81 100644
+--- a/build/VS2010/fullbench/fullbench.vcxproj
++++ b/build/VS2010/fullbench/fullbench.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -84,6 +88,46 @@
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
+index 12a4b931..1ffc5941 100644
+--- a/build/VS2010/fuzzer/fuzzer.vcxproj
++++ b/build/VS2010/fuzzer/fuzzer.vcxproj
+@@ -39,15 +39,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -84,6 +88,46 @@
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress;$(UniversalCRT_IncludePath);</IncludePath>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+index 364b3bea..1f34e956 100644
+--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
++++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+@@ -94,15 +94,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>DynamicLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -143,6 +147,46 @@
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
+index 6087d737..0d2a6fe0 100644
+--- a/build/VS2010/libzstd/libzstd.vcxproj
++++ b/build/VS2010/libzstd/libzstd.vcxproj
+@@ -91,15 +91,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>StaticLibrary</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -140,6 +144,46 @@
+     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
+diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
+index 438dc617..3fb5bba8 100644
+--- a/build/VS2010/zstd/zstd.vcxproj
++++ b/build/VS2010/zstd/zstd.vcxproj
+@@ -100,15 +100,19 @@
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
+   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+     <ConfigurationType>Application</ConfigurationType>
+     <UseDebugLibraries>false</UseDebugLibraries>
+-    <WholeProgramOptimization>true</WholeProgramOptimization>
+     <CharacterSet>MultiByte</CharacterSet>
+   </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
++  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
++    <WholeProgramOptimization>true</WholeProgramOptimization>
++  </PropertyGroup>
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+   <ImportGroup Label="ExtensionSettings">
+   </ImportGroup>
+@@ -149,6 +153,46 @@
+     <RunCodeAnalysis>false</RunCodeAnalysis>
+     <LibraryPath>$(LibraryPath);</LibraryPath>
+   </PropertyGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|Win32|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|Win32|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreaded'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebug'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Release|x64|MultiThreadedDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
++  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)|$(RuntimeLibrary)'=='Debug|x64|MultiThreadedDebugDLL'" Label="Configuration">
++    <ClCompile>
++      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
++    </ClCompile>
++  </ItemDefinitionGroup>
+   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+     <ClCompile>
+       <PrecompiledHeader>
diff --git a/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch b/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch
deleted file mode 100644
index 8bfb928947e06..0000000000000
--- a/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch
+++ /dev/null
@@ -1,199 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-diff --git a/build/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj
-index bd8a213..8e4dc89 100644
---- a/build/VS2010/datagen/datagen.vcxproj
-+++ b/build/VS2010/datagen/datagen.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
-index e697318..82cd4ab 100644
---- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
-+++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj
-index 2bff4ca..ced4047 100644
---- a/build/VS2010/fullbench/fullbench.vcxproj
-+++ b/build/VS2010/fullbench/fullbench.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
-index 12a4b93..227efd1 100644
---- a/build/VS2010/fuzzer/fuzzer.vcxproj
-+++ b/build/VS2010/fuzzer/fuzzer.vcxproj
-@@ -39,15 +39,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
-index 364b3be..b227320 100644
---- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
-+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
-@@ -94,15 +94,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>DynamicLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>DynamicLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
-index 6087d73..51a0572 100644
---- a/build/VS2010/libzstd/libzstd.vcxproj
-+++ b/build/VS2010/libzstd/libzstd.vcxproj
-@@ -91,15 +91,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>StaticLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>StaticLibrary</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
-diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
-index 438dc61..834ae01 100644
---- a/build/VS2010/zstd/zstd.vcxproj
-+++ b/build/VS2010/zstd/zstd.vcxproj
-@@ -100,15 +100,19 @@
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-     <ConfigurationType>Application</ConfigurationType>
-     <UseDebugLibraries>false</UseDebugLibraries>
--    <WholeProgramOptimization>true</WholeProgramOptimization>
-     <CharacterSet>MultiByte</CharacterSet>
-   </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|Win32|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-+  <PropertyGroup Condition="'$(Configuration)|$(Platform)|$(WholeProgramOptimization)'=='Release|x64|true'" Label="Configuration">
-+    <WholeProgramOptimization>true</WholeProgramOptimization>
-+  </PropertyGroup>
-   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-   <ImportGroup Label="ExtensionSettings">
-   </ImportGroup>
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 4b1950f7a17bd..4e0ace0baea0c 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -47,6 +47,15 @@ if (MSVC)
     # Set desired warning level (e.g. set /W4 for more warnings)
     set(CXX_COMMON_FLAGS "/W3")
   endif()
+
+  if (ARROW_USE_STATIC_CRT)
+    foreach (c_flag CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_DEBUG
+                    CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO
+                    CMAKE_C_FLAGS CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_DEBUG
+                    CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO)
+      string(REPLACE "/MD" "-MT" ${c_flag} "${${c_flag}}")
+    endforeach()
+  endif()
 else()
   # Common flags set below with warning level
   set(CXX_COMMON_FLAGS "")
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 411cf7584f7f1..b706aab45265d 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -130,6 +130,9 @@ endif()
 
 set(Boost_DEBUG TRUE)
 set(Boost_USE_MULTITHREADED ON)
+if (MSVC AND ARROW_USE_STATIC_CRT)
+  set(Boost_USE_STATIC_RUNTIME ON)
+endif()
 set(Boost_ADDITIONAL_VERSIONS
   "1.65.0" "1.65"
   "1.64.0" "1.64"
@@ -204,11 +207,11 @@ else()
     else()
       find_package(Boost COMPONENTS system filesystem REQUIRED)
       if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-	set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-	set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+        set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+        set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
       else()
-	set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-	set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+        set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+        set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
       endif()
       set(BOOST_SYSTEM_LIBRARY boost_system_shared)
       set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
@@ -222,11 +225,11 @@ else()
     else()
       find_package(Boost COMPONENTS system filesystem REQUIRED)
       if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
-	set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
-	set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+        set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+        set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
       else()
-	set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
-	set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+        set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+        set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
       endif()
       set(BOOST_SYSTEM_LIBRARY boost_system_static)
       set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
@@ -272,8 +275,10 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
     set(GTEST_VENDORED 1)
     set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                          -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}
-                         -Dgtest_force_shared_crt=ON
                          -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
+    if (MSVC AND NOT ARROW_USE_STATIC_CRT)
+      set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON)
+    endif()
 
     ExternalProject_Add(googletest_ep
       URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
@@ -320,6 +325,8 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
                           -DBUILD_TESTING=OFF
                           -BUILD_CONFIG_TESTS=OFF
                           -DINSTALL_HEADERS=ON
+                          -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}
+                          -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}
                           -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
 
     ExternalProject_Add(gflags_ep
@@ -434,6 +441,8 @@ if (ARROW_IPC)
       "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}"
       "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
       "-DFLATBUFFERS_BUILD_TESTS=OFF"
+      "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}"
+      "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}"
       ${EP_LOG_OPTIONS})
 
     set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
@@ -550,6 +559,8 @@ if (ARROW_WITH_ZLIB)
     set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                         -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}
                         -DCMAKE_C_FLAGS=${EP_C_FLAGS}
+                        -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}
+                        -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}
                         -DBUILD_SHARED_LIBS=OFF)
 
     ExternalProject_Add(zlib_ep
@@ -599,7 +610,9 @@ if (ARROW_WITH_SNAPPY)
     if (MSVC)
       set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                             "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
-                            "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                            "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}"
+                            "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}"
+                            "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
                             "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}")
       set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy
                         ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt
@@ -659,7 +672,9 @@ if (ARROW_WITH_BROTLI)
     set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                           "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
-                          "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                          "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
+                          "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}"
+                          "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}"
                           -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}
                           -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE}
                           -DBUILD_SHARED_LIBS=OFF)
@@ -709,9 +724,17 @@ if (ARROW_WITH_LZ4)
     set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
 
     if (MSVC)
+      if (ARROW_USE_STATIC_CRT)
+        if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+          set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug")
+        else()
+          set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded")
+        endif()
+      endif()
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
-      set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
-      set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_wholeprogramoptimization_param.patch)
+      set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140
+                                          ${LZ4_RUNTIME_LIBRARY_LINKAGE} /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
+      set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_gl_runtimelibrary_params.patch)
     else()
       set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
       set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh)
@@ -753,9 +776,18 @@ if (ARROW_WITH_ZSTD)
     set(ZSTD_INCLUDE_DIR "${ZSTD_BUILD_DIR}/lib")
 
     if (MSVC)
+      if (ARROW_USE_STATIC_CRT)
+        if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+          set(ZSTD_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug")
+        else()
+          set(ZSTD_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded")
+        endif()
+      endif()
       set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib")
-      set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
-      set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_wholeprogramoptimization_param.patch)
+      set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE}
+                             ${ZSTD_RUNTIME_LIBRARY_LINKAGE} /p:Platform=x64 /p:PlatformToolset=v140
+                             /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
+      set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_gl_runtimelibrary_params.patch)
     else()
       set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a")
       set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh)
@@ -802,7 +834,7 @@ if (ARROW_WITH_GRPC)
     set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GRPC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                           "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
-                          "-DCMAKE_C_FLAGS=${EX_C_FLAGS}"
+                          "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
                           -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX}
                           -DBUILD_SHARED_LIBS=OFF)
 

From 5dbfe301089d4f7a81fe2406ff641acd284c23ee Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 4 Dec 2017 10:49:17 +0100
Subject: [PATCH 1305/1644] ARROW-1863: [Python] PyObjectStringify could render
 bytes-like output for more types of objects

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1385 from cpcloud/ARROW-1863 and squashes the following commits:

126d917 [Phillip Cloud] Repr when trying to convert strings
ba9a8ff [Phillip Cloud] ARROW-1863: [Python] PyObjectStringify could render bytes-like output for more types of objects
---
 cpp/src/arrow/python/builtin_convert.cc      | 3 ++-
 python/pyarrow/tests/test_convert_builtin.py | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index fa0098bdf63e9..c716c47d2de23 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -667,7 +667,8 @@ class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter>
       RETURN_IF_PYERROR();
       bytes_obj = obj;
     } else if (!PyUnicode_Check(obj)) {
-      PyObjectStringify stringified(obj);
+      OwnedRef repr(PyObject_Repr(obj));
+      PyObjectStringify stringified(repr.obj());
       std::stringstream ss;
       ss << "Non bytes/unicode value encountered: " << stringified.bytes;
       return Status::Invalid(ss.str());
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 4c3d9e5631ce3..d7760da2f9b47 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -312,6 +312,13 @@ def test_mixed_types_fails(self):
         with self.assertRaises(pa.ArrowException):
             pa.array(data)
 
+    def test_mixed_types_with_specified_type_fails(self):
+        data = ['-10', '-5', {'a': 1}, '0', '5', '10']
+
+        type = pa.string()
+        with self.assertRaises(pa.ArrowInvalid):
+            pa.array(data, type=type)
+
     def test_decimal(self):
         data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
         type = pa.decimal128(precision=7, scale=3)

From 8f2dbee49dfef8b2f7bc68f8fe978f4b24fa5a79 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 4 Dec 2017 09:21:18 -0500
Subject: [PATCH 1306/1644] ARROW-1879: [Python] Dask integration tests are not
 skipped if dask is not installed

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1384 from cpcloud/ARROW-1879 and squashes the following commits:

7b9503e9 [Phillip Cloud] ARROW-1879: [Python] Dask integration tests are not skipped if dask is not installed
---
 python/testing/dask_tests/test_dask_integration.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/python/testing/dask_tests/test_dask_integration.py b/python/testing/dask_tests/test_dask_integration.py
index e678348780cd8..842c45f57d1f7 100644
--- a/python/testing/dask_tests/test_dask_integration.py
+++ b/python/testing/dask_tests/test_dask_integration.py
@@ -14,12 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+import pytest
+
 from datetime import date, timedelta
 import csv
 from random import randint
-import dask.dataframe as dd
+
 import pyarrow as pa
 
+dd = pytest.importorskip('dask.dataframe')
+
+
 def make_datafiles(tmpdir, prefix='data', num_files=20):
     rowcount = 5000
     fieldnames = ['date', 'temperature', 'dewpoint']
@@ -37,6 +43,7 @@ def make_datafiles(tmpdir, prefix='data', num_files=20):
                                  'dewpoint': dewpoint})
                 the_date += timedelta(days=1)
 
+
 def test_dask_file_read(tmpdir):
     prefix = 'data'
     make_datafiles(tmpdir, prefix)

From fe6f60c31875c2a42d71a975f1c18c0c81c20678 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 4 Dec 2017 17:46:12 -0500
Subject: [PATCH 1307/1644] ARROW-1885: [Java] Restore MapVector class names
 prior to ARROW-1710

Changes to MapVector were not part of the intended goal for ARROW-1710 and should be restored to `NullableMapVector` -> `MapVector` -> `AbstractMapVector` as existed prior to #1341.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1389 from BryanCutler/java-restore-MapVector-class-ARROW-1885 and squashes the following commits:

9ff503ab [Bryan Cutler] restore MapVector class names
3d818f0e [Bryan Cutler] renamed NonNullableMapVector to MapVector
ced1705c [Bryan Cutler] renamed MapVector to NullableMapVector
---
 .../templates/CaseSensitiveMapWriters.java    |   2 +-
 .../main/codegen/templates/MapWriters.java    |   8 +-
 .../main/codegen/templates/UnionVector.java   |  10 +-
 .../arrow/vector/BaseVariableWidthVector.java |   2 +-
 .../complex/AbstractContainerVector.java      |   4 +-
 .../arrow/vector/complex/MapVector.java       | 552 +++++++-----------
 .../vector/complex/NonNullableMapVector.java  | 342 -----------
 .../vector/complex/NullableMapVector.java     | 500 ++++++++++++++++
 .../complex/impl/ComplexWriterImpl.java       |   6 +-
 .../complex/impl/NullableMapReaderImpl.java   |   8 +-
 .../impl/NullableMapWriterFactory.java        |   4 +-
 .../vector/complex/impl/PromotableWriter.java |   4 +-
 .../complex/impl/SingleMapReaderImpl.java     |   5 +-
 .../org/apache/arrow/vector/types/Types.java  |   6 +-
 .../apache/arrow/vector/TestMapVector.java    |   4 +-
 .../arrow/vector/TestVectorReAlloc.java       |   4 +-
 .../complex/impl/TestPromotableWriter.java    |   4 +-
 .../complex/writer/TestComplexWriter.java     |   6 +-
 .../apache/arrow/vector/ipc/BaseFileTest.java |  12 +-
 .../arrow/vector/ipc/TestArrowFile.java       |  21 +-
 .../apache/arrow/vector/ipc/TestJSONFile.java |  15 +-
 21 files changed, 758 insertions(+), 761 deletions(-)
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

diff --git a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
index 7c652902eb8f5..5357f9b8a9d3a 100644
--- a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
+++ b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
@@ -23,7 +23,7 @@
 <#if mode == "Single">
 <#assign containerClass = "MapVector" />
 <#else>
-<#assign containerClass = "MapVector" />
+<#assign containerClass = "NullableMapVector" />
 </#if>
 
 <#include "/@includes/license.ftl" />
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index a5ac1b7170442..ac59e59b30888 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -21,9 +21,9 @@
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" />
 <#assign index = "idx()">
 <#if mode == "Single">
-<#assign containerClass = "NonNullableMapVector" />
-<#else>
 <#assign containerClass = "MapVector" />
+<#else>
+<#assign containerClass = "NullableMapVector" />
 </#if>
 
 <#include "/@includes/license.ftl" />
@@ -51,7 +51,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
   private final Map<String, FieldWriter> fields = Maps.newHashMap();
   public ${mode}MapWriter(${containerClass} container) {
     <#if mode == "Single">
-    if (container instanceof MapVector) {
+    if (container instanceof NullableMapVector) {
       throw new IllegalArgumentException("Invalid container: " + container);
     }
     </#if>
@@ -124,7 +124,7 @@ public MapWriter map(String name) {
     FieldWriter writer = fields.get(finalName);
     if(writer == null){
       int vectorCount=container.size();
-      MapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), MapVector.class);
+      NullableMapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), NullableMapVector.class);
       writer = new PromotableWriter(vector, container, getNullableMapWriterFactory());
       if(vectorCount != container.size()) {
         writer.allocate();
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 60cd24dcc4457..9377bd0da211d 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -48,8 +48,8 @@
 
 
 /**
- * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each
- * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods,
+ * A vector which can hold values of different types. It does so by using a NullableMapVector which contains a vector for each
+ * primitive type that is stored. NullableMapVector is used in order to take advantage of its serialization/deserialization methods,
  * as well as the addOrGet method.
  *
  * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
@@ -65,7 +65,7 @@ public class UnionVector implements FieldVector {
   MapVector internalMap;
   protected ArrowBuf typeBuffer;
 
-  private MapVector mapVector;
+  private NullableMapVector mapVector;
   private ListVector listVector;
 
   private FieldReader reader;
@@ -174,10 +174,10 @@ public long getOffsetBufferAddress() {
   @Override
   public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); }
 
-  public MapVector getMap() {
+  public NullableMapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
-      mapVector = addOrGet(MinorType.MAP, MapVector.class);
+      mapVector = addOrGet(MinorType.MAP, NullableMapVector.class);
       if (internalMap.size() > vectorCount) {
         mapVector.allocateNew();
         if (callBack != null) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index 98820a2367b75..fbadb35d7556a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -24,7 +24,7 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 5ded1cc6f8778..db0ff86df47a9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -100,8 +100,8 @@ protected boolean supportsDirectRead() {
   // return the child vector's ordinal in the composite container
   public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
 
-  public MapVector addOrGetMap(String name) {
-    return addOrGet(name, FieldType.nullable(new Struct()), MapVector.class);
+  public NullableMapVector addOrGetMap(String name) {
+    return addOrGet(name, FieldType.nullable(new Struct()), NullableMapVector.class);
   }
 
   public ListVector addOrGetList(String name) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 72cc29f30ca14..e13084588ce0b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * <p>
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,480 +21,322 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
-import com.google.common.collect.ObjectArrays;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
 
 import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.memory.BaseAllocator;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
-import org.apache.arrow.vector.complex.impl.NullableMapWriter;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
-import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.TransferPair;
 
-public class MapVector extends NonNullableMapVector implements FieldVector {
+public class MapVector extends AbstractMapVector {
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
   public static MapVector empty(String name, BufferAllocator allocator) {
-    FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
+    FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
     return new MapVector(name, allocator, fieldType, null);
   }
 
-  private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
-  private final NullableMapWriter writer = new NullableMapWriter(this);
-
-  protected ArrowBuf validityBuffer;
-  private int validityAllocationSizeInBytes;
+  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
+  protected final FieldType fieldType;
+  public int valueCount;
 
   // deprecated, use FieldType or static constructor instead
   @Deprecated
   public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
-    this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack);
+    this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
   }
 
-  // deprecated, use FieldType or static constructor instead
-  @Deprecated
-  public MapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
-    this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack);
+  public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+    super(name, allocator, callBack);
+    this.fieldType = checkNotNull(fieldType);
+    this.valueCount = 0;
   }
 
-  public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
-    super(name, checkNotNull(allocator), fieldType, callBack);
-    this.validityBuffer = allocator.getEmpty();
-    this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION);
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  transient private MapTransferPair ephPair;
+
+  public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
+    if (ephPair == null || ephPair.from != from) {
+      ephPair = (MapTransferPair) from.makeTransferPair(this);
+    }
+    ephPair.copyValueSafe(fromIndex, thisIndex);
   }
 
   @Override
-  public Field getField() {
-    Field f = super.getField();
-    FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata());
-    return new Field(f.getName(), type, f.getChildren());
+  protected boolean supportsDirectRead() {
+    return true;
+  }
+
+  public Iterator<String> fieldNameIterator() {
+    return getChildFieldNames().iterator();
   }
 
   @Override
-  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
-    if (ownBuffers.size() != 1) {
-      throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size());
+  public void setInitialCapacity(int numRecords) {
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      v.setInitialCapacity(numRecords);
     }
+  }
 
-    ArrowBuf bitBuffer = ownBuffers.get(0);
+  @Override
+  public int getBufferSize() {
+    if (valueCount == 0 || size() == 0) {
+      return 0;
+    }
+    long buffer = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      buffer += v.getBufferSize();
+    }
 
-    validityBuffer.release();
-    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
-    valueCount = fieldNode.getLength();
-    validityAllocationSizeInBytes = validityBuffer.capacity();
+    return (int) buffer;
   }
 
   @Override
-  public List<ArrowBuf> getFieldBuffers() {
-    List<ArrowBuf> result = new ArrayList<>(1);
-    setReaderAndWriterIndex();
-    result.add(validityBuffer);
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
 
-    return result;
-  }
+    long bufferSize = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      bufferSize += v.getBufferSizeFor(valueCount);
+    }
 
-  private void setReaderAndWriterIndex() {
-    validityBuffer.readerIndex(0);
-    validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
+    return (int) bufferSize;
   }
 
   @Override
-  @Deprecated
-  public List<BufferBacked> getFieldInnerVectors() {
-    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
+  public ArrowBuf getValidityBuffer() {
+    throw new UnsupportedOperationException();
   }
 
   @Override
-  public NullableMapReaderImpl getReader() {
-    return reader;
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
   }
 
-  public NullableMapWriter getWriter() {
-    return writer;
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
   }
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new MapVector(name, allocator, fieldType, null), false);
+    return getTransferPair(name, allocator, null);
   }
 
   @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new NullableMapTransferPair(this, (MapVector) to, true);
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false);
   }
 
   @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, null), false);
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new MapTransferPair(this, (MapVector) to);
   }
 
   @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false);
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false);
   }
 
-  protected class NullableMapTransferPair extends MapTransferPair {
-
-    private MapVector target;
+  protected static class MapTransferPair implements TransferPair {
+    private final TransferPair[] pairs;
+    private final MapVector from;
+    private final MapVector to;
 
-    protected NullableMapTransferPair(MapVector from, MapVector to, boolean allocate) {
-      super(from, to, allocate);
-      this.target = to;
+    public MapTransferPair(MapVector from, MapVector to) {
+      this(from, to, true);
     }
 
-    @Override
-    public void transfer() {
-      target.clear();
-      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
-      super.transfer();
-      clear();
+    protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
+      this.from = from;
+      this.to = to;
+      this.pairs = new TransferPair[from.size()];
+      this.to.ephPair = null;
+
+      int i = 0;
+      FieldVector vector;
+      for (String child : from.getChildFieldNames()) {
+        int preSize = to.size();
+        vector = from.getChild(child);
+        if (vector == null) {
+          continue;
+        }
+        //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
+        // the child fields may be nested fields of the top level child. For example if the structure
+        // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
+        // But the children member of a Materialized field is a HashSet. If the fields are added in the
+        // children HashSet, and the hashCode of the Materialized field includes the hash code of the
+        // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
+        // (This is similar to what happens in ScanBatch where the children cannot be added till they are
+        // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
+        // include the hashCode of the children but is based only on MaterializedField$key.
+        final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass());
+        if (allocate && to.size() != preSize) {
+          newVector.allocateNew();
+        }
+        pairs[i++] = vector.makeTransferPair(newVector);
+      }
     }
 
     @Override
-    public void copyValueSafe(int fromIndex, int toIndex) {
-      while (toIndex >= target.getValidityBufferValueCapacity()) {
-        target.reallocValidityBuffer();
+    public void transfer() {
+      for (final TransferPair p : pairs) {
+        p.transfer();
       }
-      BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex));
-      super.copyValueSafe(fromIndex, toIndex);
+      to.valueCount = from.valueCount;
+      from.clear();
     }
 
     @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      target.clear();
-      splitAndTransferValidityBuffer(startIndex, length, target);
-      super.splitAndTransfer(startIndex, length);
+    public ValueVector getTo() {
+      return to;
     }
-  }
 
-  /*
-   * transfer the validity.
-   */
-  private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) {
-    assert startIndex + length <= valueCount;
-    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
-    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
-    int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length);
-    int offset = startIndex % 8;
-
-    if (length > 0) {
-      if (offset == 0) {
-        // slice
-        if (target.validityBuffer != null) {
-          target.validityBuffer.release();
-        }
-        target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
-        target.validityBuffer.retain(1);
-      } else {
-        /* Copy data
-         * When the first bit starts from the middle of a byte (offset != 0),
-         * copy data from src BitVector.
-         * Each byte in the target is composed by a part in i-th byte,
-         * another part in (i+1)-th byte.
-         */
-        target.allocateValidityBuffer(byteSizeTarget);
-
-        for (int i = 0; i < byteSizeTarget - 1; i++) {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset);
-          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset);
-
-          target.validityBuffer.setByte(i, (b1 + b2));
-        }
-
-        /* Copying the last piece is done in the following manner:
-         * if the source vector has 1 or more bytes remaining, we copy
-         * the last piece as a byte formed by shifting data
-         * from the current byte and the next byte.
-         *
-         * if the source vector has no more bytes remaining
-         * (we are at the last byte), we copy the last piece as a byte
-         * by shifting data from the current byte.
-         */
-        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
-                  firstByteSource + byteSizeTarget - 1, offset);
-          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
-                  firstByteSource + byteSizeTarget, offset);
-
-          target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
-        } else {
-          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
-                  firstByteSource + byteSizeTarget - 1, offset);
-          target.validityBuffer.setByte(byteSizeTarget - 1, b1);
-        }
+    @Override
+    public void copyValueSafe(int from, int to) {
+      for (TransferPair p : pairs) {
+        p.copyValueSafe(from, to);
       }
     }
-  }
-
-  /**
-   * Get the value capacity of the internal validity buffer.
-   * @return number of elements that validity buffer can hold
-   */
-  private int getValidityBufferValueCapacity() {
-    return (int) (validityBuffer.capacity() * 8L);
-  }
-
-  /**
-   * Get the current value capacity for the vector
-   * @return number of elements that vector can hold.
-   */
-  @Override
-  public int getValueCapacity() {
-    return Math.min(getValidityBufferValueCapacity(),
-            super.getValueCapacity());
-  }
 
-  /**
-   * Return the underlying buffers associated with this vector. Note that this doesn't
-   * impact the reference counts for this buffer so it only should be used for in-context
-   * access. Also note that this buffer changes regularly thus
-   * external classes shouldn't hold a reference to it (unless they change it).
-   *
-   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
-   *              but the returned array will be the only reference to them
-   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
-   *         vector instance.
-   */
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    setReaderAndWriterIndex();
-    final ArrowBuf[] buffers;
-    if (getBufferSize() == 0) {
-      buffers = new ArrowBuf[0];
-    } else {
-      buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false),
-              ArrowBuf.class);
-    }
-    if (clear) {
-      for (ArrowBuf buffer : buffers) {
-        buffer.retain();
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      for (TransferPair p : pairs) {
+        p.splitAndTransfer(startIndex, length);
       }
-      clear();
+      to.setValueCount(length);
     }
-
-    return buffers;
   }
 
-  /**
-   * Close the vector and release the associated buffers.
-   */
   @Override
-  public void close() {
-    clearValidityBuffer();
-    super.close();
-  }
-
-  /**
-   * Same as {@link #close()}
-   */
-  @Override
-  public void clear() {
-    clearValidityBuffer();
-    super.clear();
-  }
-
-  /**
-   * Release the validity buffer
-   */
-  private void clearValidityBuffer() {
-    validityBuffer.release();
-    validityBuffer = allocator.getEmpty();
-  }
-
-  /**
-   * Get the size (number of bytes) of underlying buffers used by this
-   * vector
-   * @return size of underlying buffers.
-   */
-  @Override
-  public int getBufferSize() {
-    if (valueCount == 0) {
+  public int getValueCapacity() {
+    if (size() == 0) {
       return 0;
     }
-    return super.getBufferSize() +
-            BitVectorHelper.getValidityBufferSize(valueCount);
-  }
 
-  /**
-   * Get the potential buffer size for a particular number of records.
-   * @param valueCount desired number of elements in the vector
-   * @return estimated size of underlying buffers if the vector holds
-   *         a given number of elements
-   */
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
-    return super.getBufferSizeFor(valueCount)
-            + BitVectorHelper.getValidityBufferSize(valueCount);
-  }
+    final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
+      @Override
+      public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
+        return Ints.compare(
+            checkNotNull(left).getValueCapacity(),
+            checkNotNull(right).getValueCapacity()
+        );
+      }
+    };
 
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords);
-    super.setInitialCapacity(numRecords);
+    return natural.min(getChildren()).getValueCapacity();
   }
 
   @Override
-  public boolean allocateNewSafe() {
-    /* Boolean to keep track if all the memory allocations were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      clear();
-      allocateValidityBuffer(validityAllocationSizeInBytes);
-      success = super.allocateNewSafe();
-    } finally {
-      if (!success) {
-        clear();
-        return false;
+  public Object getObject(int index) {
+    Map<String, Object> vv = new JsonStringHashMap<>();
+    for (String child : getChildFieldNames()) {
+      ValueVector v = getChild(child);
+      if (v != null && index < v.getValueCount()) {
+        Object value = v.getObject(index);
+        if (value != null) {
+          vv.put(child, value);
+        }
       }
     }
-    return true;
-  }
-
-  private void allocateValidityBuffer(final long size) {
-    final int curSize = (int) size;
-    validityBuffer = allocator.buffer(curSize);
-    validityBuffer.readerIndex(0);
-    validityAllocationSizeInBytes = curSize;
-    validityBuffer.setZero(0, validityBuffer.capacity());
+    return vv;
   }
 
   @Override
-  public void reAlloc() {
-    /* reallocate the validity buffer */
-    reallocValidityBuffer();
-    super.reAlloc();
-  }
-
-  private void reallocValidityBuffer() {
-    final int currentBufferCapacity = validityBuffer.capacity();
-    long baseSize = validityAllocationSizeInBytes;
-
-    if (baseSize < (long) currentBufferCapacity) {
-      baseSize = (long) currentBufferCapacity;
-    }
-
-    long newAllocationSize = baseSize * 2L;
-    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
-
-    if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
-      throw new OversizedAllocationException("Unable to expand the buffer");
-    }
-
-    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
-    newBuf.setZero(0, newBuf.capacity());
-    newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
-    validityBuffer.release(1);
-    validityBuffer = newBuf;
-    validityAllocationSizeInBytes = (int) newAllocationSize;
-  }
-
+  public boolean isNull(int index) { return false; }
   @Override
-  public long getValidityBufferAddress() {
-    return validityBuffer.memoryAddress();
-  }
+  public int getNullCount() { return 0; }
 
-  @Override
-  public long getDataBufferAddress() {
-    throw new UnsupportedOperationException();
+  public void get(int index, ComplexHolder holder) {
+    reader.setPosition(index);
+    holder.reader = reader;
   }
 
   @Override
-  public long getOffsetBufferAddress() {
-    throw new UnsupportedOperationException();
+  public int getValueCount() {
+    return valueCount;
   }
 
-  @Override
-  public ArrowBuf getValidityBuffer() {
-    return validityBuffer;
-  }
+  public ValueVector getVectorById(int id) {
+  return getChildByOrdinal(id);
+}
 
   @Override
-  public ArrowBuf getDataBuffer() {
-    throw new UnsupportedOperationException();
+  public void setValueCount(int valueCount) {
+    for (final ValueVector v : getChildren()) {
+      v.setValueCount(valueCount);
+    }
+    MapVector.this.valueCount = valueCount;
   }
 
   @Override
-  public ArrowBuf getOffsetBuffer() {
-    throw new UnsupportedOperationException();
+  public void clear() {
+    for (final ValueVector v : getChildren()) {
+      v.clear();
+    }
+    valueCount = 0;
   }
 
   @Override
-  public Object getObject(int index) {
-    if (isSet(index) == 0) {
-      return null;
-    } else {
-      return super.getObject(index);
+  public Field getField() {
+    List<Field> children = new ArrayList<>();
+    for (ValueVector child : getChildren()) {
+      children.add(child.getField());
     }
+    return new Field(name, fieldType, children);
   }
 
   @Override
-  public void get(int index, ComplexHolder holder) {
-    holder.isSet = isSet(index);
-    super.get(index, holder);
+  public MinorType getMinorType() {
+    return MinorType.MAP;
   }
 
-  public int getNullCount() {
-    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
-  }
-
-  public boolean isNull(int index) {
-    return isSet(index) == 0;
-  }
-
-  public int isSet(int index) {
-    final int byteIndex = index >> 3;
-    final byte b = validityBuffer.getByte(byteIndex);
-    final int bitIndex = index & 7;
-    return Long.bitCount(b & (1L << bitIndex));
-  }
-
-  public void setIndexDefined(int index) {
-    while (index >= getValidityBufferValueCapacity()) {
-      /* realloc the inner buffers if needed */
-      reallocValidityBuffer();
+  @Override
+  public void close() {
+    final Collection<FieldVector> vectors = getChildren();
+    for (final FieldVector v : vectors) {
+      v.close();
     }
-    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
-  }
+    vectors.clear();
 
-  public void setNull(int index) {
-    while (index >= getValidityBufferValueCapacity()) {
-      /* realloc the inner buffers if needed */
-      reallocValidityBuffer();
-    }
-    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    valueCount = 0;
+
+    super.close();
   }
 
-  @Override
-  public void setValueCount(int valueCount) {
-    assert valueCount >= 0;
-    while (valueCount > getValidityBufferValueCapacity()) {
-      /* realloc the inner buffers if needed */
-      reallocValidityBuffer();
+  public void initializeChildrenFromFields(List<Field> children) {
+    for (Field field : children) {
+      FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType());
+      vector.initializeChildrenFromFields(field.getChildren());
     }
-    super.setValueCount(valueCount);
-    this.valueCount = valueCount;
   }
 
-  public void reset() {
-    valueCount = 0;
+  public List<FieldVector> getChildrenFromFields() {
+    return getChildren();
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
deleted file mode 100644
index 1cd7b70d7bfad..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
-
-import io.netty.buffer.ArrowBuf;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringHashMap;
-import org.apache.arrow.vector.util.TransferPair;
-
-public class NonNullableMapVector extends AbstractMapVector {
-  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NonNullableMapVector.class);
-
-  public static NonNullableMapVector empty(String name, BufferAllocator allocator) {
-    FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
-    return new NonNullableMapVector(name, allocator, fieldType, null);
-  }
-
-  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
-  protected final FieldType fieldType;
-  public int valueCount;
-
-  // deprecated, use FieldType or static constructor instead
-  @Deprecated
-  public NonNullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
-    this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
-  }
-
-  public NonNullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
-    super(name, allocator, callBack);
-    this.fieldType = checkNotNull(fieldType);
-    this.valueCount = 0;
-  }
-
-  @Override
-  public FieldReader getReader() {
-    return reader;
-  }
-
-  transient private MapTransferPair ephPair;
-
-  public void copyFromSafe(int fromIndex, int thisIndex, NonNullableMapVector from) {
-    if (ephPair == null || ephPair.from != from) {
-      ephPair = (MapTransferPair) from.makeTransferPair(this);
-    }
-    ephPair.copyValueSafe(fromIndex, thisIndex);
-  }
-
-  @Override
-  protected boolean supportsDirectRead() {
-    return true;
-  }
-
-  public Iterator<String> fieldNameIterator() {
-    return getChildFieldNames().iterator();
-  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      v.setInitialCapacity(numRecords);
-    }
-  }
-
-  @Override
-  public int getBufferSize() {
-    if (valueCount == 0 || size() == 0) {
-      return 0;
-    }
-    long buffer = 0;
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      buffer += v.getBufferSize();
-    }
-
-    return (int) buffer;
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
-
-    long bufferSize = 0;
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      bufferSize += v.getBufferSizeFor(valueCount);
-    }
-
-    return (int) bufferSize;
-  }
-
-  @Override
-  public ArrowBuf getValidityBuffer() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public ArrowBuf getDataBuffer() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public ArrowBuf getOffsetBuffer() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    return getTransferPair(name, allocator, null);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return new MapTransferPair(this, new NonNullableMapVector(name, allocator, fieldType, callBack), false);
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new MapTransferPair(this, (NonNullableMapVector) to);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new MapTransferPair(this, new NonNullableMapVector(ref, allocator, fieldType, callBack), false);
-  }
-
-  protected static class MapTransferPair implements TransferPair {
-    private final TransferPair[] pairs;
-    private final NonNullableMapVector from;
-    private final NonNullableMapVector to;
-
-    public MapTransferPair(NonNullableMapVector from, NonNullableMapVector to) {
-      this(from, to, true);
-    }
-
-    protected MapTransferPair(NonNullableMapVector from, NonNullableMapVector to, boolean allocate) {
-      this.from = from;
-      this.to = to;
-      this.pairs = new TransferPair[from.size()];
-      this.to.ephPair = null;
-
-      int i = 0;
-      FieldVector vector;
-      for (String child : from.getChildFieldNames()) {
-        int preSize = to.size();
-        vector = from.getChild(child);
-        if (vector == null) {
-          continue;
-        }
-        //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
-        // the child fields may be nested fields of the top level child. For example if the structure
-        // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
-        // But the children member of a Materialized field is a HashSet. If the fields are added in the
-        // children HashSet, and the hashCode of the Materialized field includes the hash code of the
-        // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
-        // (This is similar to what happens in ScanBatch where the children cannot be added till they are
-        // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
-        // include the hashCode of the children but is based only on MaterializedField$key.
-        final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass());
-        if (allocate && to.size() != preSize) {
-          newVector.allocateNew();
-        }
-        pairs[i++] = vector.makeTransferPair(newVector);
-      }
-    }
-
-    @Override
-    public void transfer() {
-      for (final TransferPair p : pairs) {
-        p.transfer();
-      }
-      to.valueCount = from.valueCount;
-      from.clear();
-    }
-
-    @Override
-    public ValueVector getTo() {
-      return to;
-    }
-
-    @Override
-    public void copyValueSafe(int from, int to) {
-      for (TransferPair p : pairs) {
-        p.copyValueSafe(from, to);
-      }
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      for (TransferPair p : pairs) {
-        p.splitAndTransfer(startIndex, length);
-      }
-      to.setValueCount(length);
-    }
-  }
-
-  @Override
-  public int getValueCapacity() {
-    if (size() == 0) {
-      return 0;
-    }
-
-    final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
-      @Override
-      public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
-        return Ints.compare(
-            checkNotNull(left).getValueCapacity(),
-            checkNotNull(right).getValueCapacity()
-        );
-      }
-    };
-
-    return natural.min(getChildren()).getValueCapacity();
-  }
-
-  @Override
-  public Object getObject(int index) {
-    Map<String, Object> vv = new JsonStringHashMap<>();
-    for (String child : getChildFieldNames()) {
-      ValueVector v = getChild(child);
-      if (v != null && index < v.getValueCount()) {
-        Object value = v.getObject(index);
-        if (value != null) {
-          vv.put(child, value);
-        }
-      }
-    }
-    return vv;
-  }
-
-  @Override
-  public boolean isNull(int index) { return false; }
-  @Override
-  public int getNullCount() { return 0; }
-
-  public void get(int index, ComplexHolder holder) {
-    reader.setPosition(index);
-    holder.reader = reader;
-  }
-
-  @Override
-  public int getValueCount() {
-    return valueCount;
-  }
-
-  public ValueVector getVectorById(int id) {
-  return getChildByOrdinal(id);
-}
-
-  @Override
-  public void setValueCount(int valueCount) {
-    for (final ValueVector v : getChildren()) {
-      v.setValueCount(valueCount);
-    }
-    NonNullableMapVector.this.valueCount = valueCount;
-  }
-
-  @Override
-  public void clear() {
-    for (final ValueVector v : getChildren()) {
-      v.clear();
-    }
-    valueCount = 0;
-  }
-
-  @Override
-  public Field getField() {
-    List<Field> children = new ArrayList<>();
-    for (ValueVector child : getChildren()) {
-      children.add(child.getField());
-    }
-    return new Field(name, fieldType, children);
-  }
-
-  @Override
-  public MinorType getMinorType() {
-    return MinorType.MAP;
-  }
-
-  @Override
-  public void close() {
-    final Collection<FieldVector> vectors = getChildren();
-    for (final FieldVector v : vectors) {
-      v.close();
-    }
-    vectors.clear();
-
-    valueCount = 0;
-
-    super.close();
-  }
-
-  public void initializeChildrenFromFields(List<Field> children) {
-    for (Field field : children) {
-      FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType());
-      vector.initializeChildrenFromFields(field.getChildren());
-    }
-  }
-
-  public List<FieldVector> getChildrenFromFields() {
-    return getChildren();
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
new file mode 100644
index 0000000000000..d887b73064643
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -0,0 +1,500 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.complex;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.NullableMapWriter;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class NullableMapVector extends MapVector implements FieldVector {
+
+  public static NullableMapVector empty(String name, BufferAllocator allocator) {
+    FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
+    return new NullableMapVector(name, allocator, fieldType, null);
+  }
+
+  private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
+  private final NullableMapWriter writer = new NullableMapWriter(this);
+
+  protected ArrowBuf validityBuffer;
+  private int validityAllocationSizeInBytes;
+
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
+  public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+    this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack);
+  }
+
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
+  public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
+    this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack);
+  }
+
+  public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+    super(name, checkNotNull(allocator), fieldType, callBack);
+    this.validityBuffer = allocator.getEmpty();
+    this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION);
+  }
+
+  @Override
+  public Field getField() {
+    Field f = super.getField();
+    FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata());
+    return new Field(f.getName(), type, f.getChildren());
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    if (ownBuffers.size() != 1) {
+      throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size());
+    }
+
+    ArrowBuf bitBuffer = ownBuffers.get(0);
+
+    validityBuffer.release();
+    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
+    valueCount = fieldNode.getLength();
+    validityAllocationSizeInBytes = validityBuffer.capacity();
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    List<ArrowBuf> result = new ArrayList<>(1);
+    setReaderAndWriterIndex();
+    result.add(validityBuffer);
+
+    return result;
+  }
+
+  private void setReaderAndWriterIndex() {
+    validityBuffer.readerIndex(0);
+    validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
+  }
+
+  @Override
+  @Deprecated
+  public List<BufferBacked> getFieldInnerVectors() {
+    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
+  }
+
+  @Override
+  public NullableMapReaderImpl getReader() {
+    return reader;
+  }
+
+  public NullableMapWriter getWriter() {
+    return writer;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new NullableMapTransferPair(this, (NullableMapVector) to, true);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
+    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false);
+  }
+
+  protected class NullableMapTransferPair extends MapTransferPair {
+
+    private NullableMapVector target;
+
+    protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, boolean allocate) {
+      super(from, to, allocate);
+      this.target = to;
+    }
+
+    @Override
+    public void transfer() {
+      target.clear();
+      target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer;
+      super.transfer();
+      clear();
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      while (toIndex >= target.getValidityBufferValueCapacity()) {
+        target.reallocValidityBuffer();
+      }
+      BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex));
+      super.copyValueSafe(fromIndex, toIndex);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      target.clear();
+      splitAndTransferValidityBuffer(startIndex, length, target);
+      super.splitAndTransfer(startIndex, length);
+    }
+  }
+
+  /*
+   * transfer the validity.
+   */
+  private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) {
+    assert startIndex + length <= valueCount;
+    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+    int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length);
+    int offset = startIndex % 8;
+
+    if (length > 0) {
+      if (offset == 0) {
+        // slice
+        if (target.validityBuffer != null) {
+          target.validityBuffer.release();
+        }
+        target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
+        target.validityBuffer.retain(1);
+      } else {
+        /* Copy data
+         * When the first bit starts from the middle of a byte (offset != 0),
+         * copy data from src BitVector.
+         * Each byte in the target is composed by a part in i-th byte,
+         * another part in (i+1)-th byte.
+         */
+        target.allocateValidityBuffer(byteSizeTarget);
+
+        for (int i = 0; i < byteSizeTarget - 1; i++) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset);
+
+          target.validityBuffer.setByte(i, (b1 + b2));
+        }
+
+        /* Copying the last piece is done in the following manner:
+         * if the source vector has 1 or more bytes remaining, we copy
+         * the last piece as a byte formed by shifting data
+         * from the current byte and the next byte.
+         *
+         * if the source vector has no more bytes remaining
+         * (we are at the last byte), we copy the last piece as a byte
+         * by shifting data from the current byte.
+         */
+        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
+                  firstByteSource + byteSizeTarget, offset);
+
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+        } else {
+          byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+                  firstByteSource + byteSizeTarget - 1, offset);
+          target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+        }
+      }
+    }
+  }
+
+  /**
+   * Get the value capacity of the internal validity buffer.
+   * @return number of elements that validity buffer can hold
+   */
+  private int getValidityBufferValueCapacity() {
+    return (int) (validityBuffer.capacity() * 8L);
+  }
+
+  /**
+   * Get the current value capacity for the vector
+   * @return number of elements that vector can hold.
+   */
+  @Override
+  public int getValueCapacity() {
+    return Math.min(getValidityBufferValueCapacity(),
+            super.getValueCapacity());
+  }
+
+  /**
+   * Return the underlying buffers associated with this vector. Note that this doesn't
+   * impact the reference counts for this buffer so it only should be used for in-context
+   * access. Also note that this buffer changes regularly thus
+   * external classes shouldn't hold a reference to it (unless they change it).
+   *
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
+   *              but the returned array will be the only reference to them
+   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this
+   *         vector instance.
+   */
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    setReaderAndWriterIndex();
+    final ArrowBuf[] buffers;
+    if (getBufferSize() == 0) {
+      buffers = new ArrowBuf[0];
+    } else {
+      buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false),
+              ArrowBuf.class);
+    }
+    if (clear) {
+      for (ArrowBuf buffer : buffers) {
+        buffer.retain();
+      }
+      clear();
+    }
+
+    return buffers;
+  }
+
+  /**
+   * Close the vector and release the associated buffers.
+   */
+  @Override
+  public void close() {
+    clearValidityBuffer();
+    super.close();
+  }
+
+  /**
+   * Same as {@link #close()}
+   */
+  @Override
+  public void clear() {
+    clearValidityBuffer();
+    super.clear();
+  }
+
+  /**
+   * Release the validity buffer
+   */
+  private void clearValidityBuffer() {
+    validityBuffer.release();
+    validityBuffer = allocator.getEmpty();
+  }
+
+  /**
+   * Get the size (number of bytes) of underlying buffers used by this
+   * vector
+   * @return size of underlying buffers.
+   */
+  @Override
+  public int getBufferSize() {
+    if (valueCount == 0) {
+      return 0;
+    }
+    return super.getBufferSize() +
+            BitVectorHelper.getValidityBufferSize(valueCount);
+  }
+
+  /**
+   * Get the potential buffer size for a particular number of records.
+   * @param valueCount desired number of elements in the vector
+   * @return estimated size of underlying buffers if the vector holds
+   *         a given number of elements
+   */
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+    return super.getBufferSizeFor(valueCount)
+            + BitVectorHelper.getValidityBufferSize(valueCount);
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords);
+    super.setInitialCapacity(numRecords);
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* Boolean to keep track if all the memory allocations were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      clear();
+      allocateValidityBuffer(validityAllocationSizeInBytes);
+      success = super.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private void allocateValidityBuffer(final long size) {
+    final int curSize = (int) size;
+    validityBuffer = allocator.buffer(curSize);
+    validityBuffer.readerIndex(0);
+    validityAllocationSizeInBytes = curSize;
+    validityBuffer.setZero(0, validityBuffer.capacity());
+  }
+
+  @Override
+  public void reAlloc() {
+    /* reallocate the validity buffer */
+    reallocValidityBuffer();
+    super.reAlloc();
+  }
+
+  private void reallocValidityBuffer() {
+    final int currentBufferCapacity = validityBuffer.capacity();
+    long baseSize = validityAllocationSizeInBytes;
+
+    if (baseSize < (long) currentBufferCapacity) {
+      baseSize = (long) currentBufferCapacity;
+    }
+
+    long newAllocationSize = baseSize * 2L;
+    newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+    if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Unable to expand the buffer");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
+    newBuf.setZero(0, newBuf.capacity());
+    newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+    validityBuffer.release(1);
+    validityBuffer = newBuf;
+    validityAllocationSizeInBytes = (int) newAllocationSize;
+  }
+
+  @Override
+  public long getValidityBufferAddress() {
+    return validityBuffer.memoryAddress();
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    return validityBuffer;
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public Object getObject(int index) {
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      return super.getObject(index);
+    }
+  }
+
+  @Override
+  public void get(int index, ComplexHolder holder) {
+    holder.isSet = isSet(index);
+    super.get(index, holder);
+  }
+
+  public int getNullCount() {
+    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+  }
+
+  public boolean isNull(int index) {
+    return isSet(index) == 0;
+  }
+
+  public int isSet(int index) {
+    final int byteIndex = index >> 3;
+    final byte b = validityBuffer.getByte(byteIndex);
+    final int bitIndex = index & 7;
+    return Long.bitCount(b & (1L << bitIndex));
+  }
+
+  public void setIndexDefined(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
+    }
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+  }
+
+  public void setNull(int index) {
+    while (index >= getValidityBufferValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
+    }
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  @Override
+  public void setValueCount(int valueCount) {
+    assert valueCount >= 0;
+    while (valueCount > getValidityBufferValueCapacity()) {
+      /* realloc the inner buffers if needed */
+      reallocValidityBuffer();
+    }
+    super.setValueCount(valueCount);
+    this.valueCount = valueCount;
+  }
+
+  public void reset() {
+    valueCount = 0;
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 970b90ef510fa..a6960238b9165 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -20,7 +20,7 @@
 
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.StateTool;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -131,7 +131,7 @@ public MapWriter directMap() {
     switch (mode) {
 
       case INIT:
-        mapRoot = nullableMapWriterFactory.build((MapVector) container);
+        mapRoot = nullableMapWriterFactory.build((NullableMapVector) container);
         mapRoot.setPosition(idx());
         mode = Mode.MAP;
         break;
@@ -152,7 +152,7 @@ public MapWriter rootAsMap() {
 
       case INIT:
         // TODO allow dictionaries in complex types
-        MapVector map = container.addOrGetMap(name);
+        NullableMapVector map = container.addOrGetMap(name);
         mapRoot = nullableMapWriterFactory.build(map);
         mapRoot.setPosition(idx());
         mode = Mode.MAP;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
index acf155af71db2..06b0f4d43272f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
@@ -20,17 +20,17 @@
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.pojo.Field;
 
 public class NullableMapReaderImpl extends SingleMapReaderImpl {
 
-  private MapVector nullableMapVector;
+  private NullableMapVector nullableMapVector;
 
   public NullableMapReaderImpl(MapVector vector) {
-    super((MapVector) vector);
-    this.nullableMapVector = (MapVector) vector;
+    super((NullableMapVector) vector);
+    this.nullableMapVector = (NullableMapVector) vector;
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
index 06dbf784be0a5..d2dcb2374d0e7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
@@ -18,7 +18,7 @@
 
 package org.apache.arrow.vector.complex.impl;
 
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 
 public class NullableMapWriterFactory {
   private final boolean caseSensitive;
@@ -29,7 +29,7 @@ public NullableMapWriterFactory(boolean caseSensitive) {
     this.caseSensitive = caseSensitive;
   }
 
-  public NullableMapWriter build(MapVector container) {
+  public NullableMapWriter build(NullableMapVector container) {
     return this.caseSensitive ? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 9abd38d20ac62..5bd439cac4269 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -23,7 +23,7 @@
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -94,7 +94,7 @@ private void setWriter(ValueVector v) {
     type = v.getMinorType();
     switch (type) {
       case MAP:
-        writer = nullableMapWriterFactory.build((MapVector) vector);
+        writer = nullableMapWriterFactory.build((NullableMapVector) vector);
         break;
       case LIST:
         writer = new UnionListWriter((ListVector) vector, nullableMapWriterFactory);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index 0341b622e0df4..c77ca4e8f23ba 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -24,7 +24,6 @@
 
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NonNullableMapVector;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -35,10 +34,10 @@
 @SuppressWarnings("unused")
 public class SingleMapReaderImpl extends AbstractFieldReader {
 
-  private final NonNullableMapVector vector;
+  private final MapVector vector;
   private final Map<String, FieldReader> fields = Maps.newHashMap();
 
-  public SingleMapReaderImpl(NonNullableMapVector vector) {
+  public SingleMapReaderImpl(MapVector vector) {
     this.vector = vector;
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 7834845aad80e..3c5fd81d572d0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -58,7 +58,7 @@
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.BitWriterImpl;
@@ -131,12 +131,12 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
     MAP(Struct.INSTANCE) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new MapVector(name, allocator, fieldType, schemaChangeCallback);
+        return new NullableMapVector(name, allocator, fieldType, schemaChangeCallback);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new NullableMapWriter((MapVector) vector);
+        return new NullableMapWriter((NullableMapVector) vector);
       }
     },
     TINYINT(new Int(8, true)) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
index e5682955bada0..357df96aa2efc 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
@@ -22,7 +22,7 @@
 import java.util.Map;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.junit.After;
@@ -50,7 +50,7 @@ public void testFieldMetadata() throws Exception {
     Map<String, String> metadata = new HashMap<>();
     metadata.put("k1", "v1");
     FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata);
-    try (MapVector vector = new MapVector("map", allocator, type, null)) {
+    try (NullableMapVector vector = new NullableMapVector("map", allocator, type, null)) {
       Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata());
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index 1b13c2ed91935..f8edf8904c53e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -26,7 +26,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -121,7 +121,7 @@ public void testListType() {
 
   @Test
   public void testMapType() {
-    try (final MapVector vector = MapVector.empty("", allocator)) {
+    try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) {
       vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), IntVector.class);
 
       vector.setInitialCapacity(512);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 38b78424dccf4..b0d6cf555e98d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -25,7 +25,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.DirtyRootAllocator;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -55,7 +55,7 @@ public void terminate() throws Exception {
   public void testPromoteToUnion() throws Exception {
 
     try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator);
-         final MapVector v = container.addOrGetMap("test");
+         final NullableMapVector v = container.addOrGetMap("test");
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
       container.allocateNew();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index bd8489eb20f89..29d39aabe6b47 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -34,7 +34,7 @@
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NonNullableMapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
@@ -830,7 +830,7 @@ public void complexCopierWithList() {
     rootWriter.end();
     writer.setValueCount(1);
 
-    MapVector mapVector = (MapVector) parent.getChild("root");
+    NullableMapVector mapVector = (NullableMapVector) parent.getChild("root");
     TransferPair tp = mapVector.getTransferPair(allocator);
     tp.splitAndTransfer(0, 1);
     MapVector toMapVector = (MapVector) tp.getTo();
@@ -849,7 +849,7 @@ public void testSingleMapWriter1() {
     /* initialize a SingleMapWriter with empty MapVector and then lazily
      * create all vectors with expected initialCapacity.
      */
-    NonNullableMapVector parent = NonNullableMapVector.empty("parent", allocator);
+    MapVector parent = MapVector.empty("parent", allocator);
     SingleMapWriter singleMapWriter = new SingleMapWriter(parent);
 
     int initialCapacity = 1024;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index 1a7f71c72b483..9eb55c9bbdaee 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -36,7 +36,7 @@
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
@@ -94,7 +94,7 @@ public void tearDown() {
     DateTimeZone.setDefault(defaultTimezone);
   }
 
-  protected void writeData(int count, MapVector parent) {
+  protected void writeData(int count, NullableMapVector parent) {
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
     IntWriter intWriter = rootWriter.integer("int");
@@ -119,7 +119,7 @@ protected void validateContent(int count, VectorSchemaRoot root) {
     }
   }
 
-  protected void writeComplexData(int count, MapVector parent) {
+  protected void writeComplexData(int count, NullableMapVector parent) {
     ArrowBuf varchar = allocator.buffer(3);
     varchar.readerIndex(0);
     varchar.setByte(0, 'a');
@@ -189,7 +189,7 @@ private LocalDateTime makeDateTimeFromCount(int i) {
     return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i);
   }
 
-  protected void writeDateTimeData(int count, MapVector parent) {
+  protected void writeDateTimeData(int count, NullableMapVector parent) {
     Assert.assertTrue(count < 100);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
@@ -494,7 +494,7 @@ public void validateUnionData(int count, VectorSchemaRoot root) {
     }
   }
 
-  public void writeUnionData(int count, MapVector parent) {
+  public void writeUnionData(int count, NullableMapVector parent) {
     ArrowBuf varchar = allocator.buffer(3);
     varchar.readerIndex(0);
     varchar.setByte(0, 'a');
@@ -537,7 +537,7 @@ public void writeUnionData(int count, MapVector parent) {
     varchar.release();
   }
 
-  protected void writeVarBinaryData(int count, MapVector parent) {
+  protected void writeVarBinaryData(int count, NullableMapVector parent) {
     Assert.assertTrue(count < 100);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
index 0cfc9ba919d85..055c34e7010f3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
@@ -42,8 +42,7 @@
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.ipc.message.ArrowBlock;
 import org.apache.arrow.vector.ipc.message.ArrowBuffer;
@@ -70,7 +69,7 @@ public void testWrite() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), file, new ByteArrayOutputStream());
     }
@@ -82,7 +81,7 @@ public void testWriteComplex() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeComplexData(count, parent);
       FieldVector root = parent.getChild("root");
       validateComplexContent(count, new VectorSchemaRoot(root));
@@ -98,7 +97,7 @@ public void testWriteRead() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
+         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), file, stream);
     }
@@ -154,7 +153,7 @@ public void testWriteReadComplex() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
+         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
       writeComplexData(count, parent);
       write(parent.getChild("root"), file, stream);
     }
@@ -195,7 +194,7 @@ public void testWriteReadMultipleRBs() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = MapVector.empty("parent", originalVectorAllocator);
+         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator);
          FileOutputStream fileOutputStream = new FileOutputStream(file)) {
       writeData(counts[0], parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
@@ -268,7 +267,7 @@ public void testWriteReadUnion() throws IOException {
 
     // write
     try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+         NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeUnionData(count, parent);
       validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
       write(parent.getChild("root"), file, stream);
@@ -382,7 +381,7 @@ public void testWriteReadMetadata() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector vector = (MapVector) field.createVector(originalVectorAllocator)) {
+         NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) {
       vector.allocateNewSafe();
       vector.setValueCount(0);
 
@@ -558,7 +557,7 @@ public void testWriteReadFixedSizeList() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
+         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
       FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class);
       Float4Vector floats = (Float4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
       IntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), IntVector.class);
@@ -618,7 +617,7 @@ public void testWriteReadVarBin() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeVarBinaryData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateVarBinary(count, root);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
index 4f9093b8c0253..6a23a8438de48 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
@@ -24,8 +24,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -45,7 +44,7 @@ public void testWriteRead() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
+         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
       writeData(count, parent);
       writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
     }
@@ -73,7 +72,7 @@ public void testWriteReadComplexJSON() throws IOException {
     // write
     try (
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
       writeComplexData(count, parent);
       writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
     }
@@ -100,7 +99,7 @@ public void testWriteComplexJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeComplexData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateComplexContent(root.getRowCount(), root);
@@ -122,7 +121,7 @@ public void testWriteReadUnionJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeUnionData(count, parent);
       printVectors(parent.getChildrenFromFields());
 
@@ -154,7 +153,7 @@ public void testWriteReadDateTimeJSON() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
 
       writeDateTimeData(count, parent);
 
@@ -319,7 +318,7 @@ public void testWriteReadVarBinJSON() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
       writeVarBinaryData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateVarBinary(count, root);

From 611a4b951e24f4f967c3d382a2027dc035fc37f0 Mon Sep 17 00:00:00 2001
From: Li Jin <ice.xelloss@gmail.com>
Date: Mon, 4 Dec 2017 17:52:48 -0500
Subject: [PATCH 1308/1644] ARROW-1785: [Format/C++/Java] Remove VectorLayout
 from serialized schemas

What's here so far removes this code from the Flatbuffers schema and the C++ implementation. This logic is a little bit entangled on the Java side, I need some help from @icexelloss or @BryanCutler or @julienledem or someone else to handle the Java refactoring. It might be easiest to preserve the ArrowVectorType/TypeLayout/VectorLayout objects for now but simply remove the Flatbuffers dependency (we do need the names of the vectors in the JSON files used for integration testing)

cc @trxcllnt since we may want to roll in the JS changes in this patch

Author: Li Jin <ice.xelloss@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Li Jin <icexelloss@Lis-MacBook-Pro.local>

Closes #1297 from wesm/ARROW-1785 and squashes the following commits:

c1e7ea9d [Li Jin] Fix comment
43ff4e3a [Li Jin] Remove Json annotation
95e8736e [Li Jin] (Refactor) Move TypeLayout and VectorLayout from ipc.message to top level. Rename VectorLayout to BufferLayout.
31cbd486 [Li Jin] Fix TypeLayout.java
890a08d7 [Li Jin] Integration test passing
2024db55 [Wes McKinney] Remove VectorLayout from Flatbuffers, C++ implementation
---
 cpp/src/arrow/ipc/json-internal.cc            |  39 -------
 cpp/src/arrow/ipc/metadata-internal.cc        |  34 +-----
 cpp/src/arrow/type.cc                         |  37 -------
 cpp/src/arrow/type.h                          |  30 ------
 format/Schema.fbs                             |  31 +-----
 integration/data/simple.json                  |  28 ++---
 integration/data/struct_example.json          |  44 +-------
 integration/integration_test.py               |  49 +--------
 .../VectorLayout.java => BufferLayout.java}   |  86 ++++++++-------
 .../vector/{ipc/message => }/TypeLayout.java  | 100 ++++++++---------
 .../org/apache/arrow/vector/VectorLoader.java |   7 +-
 .../apache/arrow/vector/VectorUnloader.java   |   4 +-
 .../arrow/vector/ipc/JsonFileReader.java      |  18 ++--
 .../arrow/vector/ipc/JsonFileWriter.java      |  21 ++--
 .../vector/ipc/message/ArrowVectorType.java   | 101 ------------------
 .../apache/arrow/vector/types/pojo/Field.java |  33 +-----
 .../apache/arrow/vector/TestListVector.java   |  10 +-
 .../apache/arrow/vector/TestValueVector.java  |   3 -
 .../vector/ipc/TestArrowReaderWriter.java     |   1 -
 .../arrow/vector/ipc/TestArrowStreamPipe.java |   3 -
 20 files changed, 145 insertions(+), 534 deletions(-)
 rename java/vector/src/main/java/org/apache/arrow/vector/{ipc/message/VectorLayout.java => BufferLayout.java} (50%)
 rename java/vector/src/main/java/org/apache/arrow/vector/{ipc/message => }/TypeLayout.java (66%)
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java

diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index bfb3d282d8772..11a6956ebfd47 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -43,22 +43,6 @@ namespace ipc {
 namespace json {
 namespace internal {
 
-static std::string GetBufferTypeName(BufferType type) {
-  switch (type) {
-    case BufferType::DATA:
-      return "DATA";
-    case BufferType::OFFSET:
-      return "OFFSET";
-    case BufferType::TYPE:
-      return "TYPE";
-    case BufferType::VALIDITY:
-      return "VALIDITY";
-    default:
-      break;
-  }
-  return "UNKNOWN";
-}
-
 static std::string GetFloatingPrecisionName(FloatingPoint::Precision precision) {
   switch (precision) {
     case FloatingPoint::HALF:
@@ -174,12 +158,9 @@ class SchemaWriter {
       RETURN_NOT_OK(WriteDictionaryMetadata(dict_type));
 
       const DataType& dictionary_type = *dict_type.dictionary()->type();
-      const DataType& index_type = *dict_type.index_type();
       RETURN_NOT_OK(WriteChildren(dictionary_type.children()));
-      WriteBufferLayout(index_type.GetBufferLayout());
     } else {
       RETURN_NOT_OK(WriteChildren(type.children()));
-      WriteBufferLayout(type.GetBufferLayout());
     }
 
     writer_->EndObject();
@@ -301,26 +282,6 @@ class SchemaWriter {
     return Status::OK();
   }
 
-  void WriteBufferLayout(const std::vector<BufferDescr>& buffer_layout) {
-    writer_->Key("typeLayout");
-    writer_->StartObject();
-    writer_->Key("vectors");
-    writer_->StartArray();
-
-    for (const BufferDescr& buffer : buffer_layout) {
-      writer_->StartObject();
-      writer_->Key("type");
-      writer_->String(GetBufferTypeName(buffer.type()));
-
-      writer_->Key("typeBitWidth");
-      writer_->Int(buffer.bit_width());
-
-      writer_->EndObject();
-    }
-    writer_->EndArray();
-    writer_->EndObject();
-  }
-
   Status WriteChildren(const std::vector<std::shared_ptr<Field>>& children) {
     writer_->Key("children");
     writer_->StartArray();
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 05202ea937217..af1d6c851582d 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -54,7 +54,6 @@ using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
 using FieldOffset = flatbuffers::Offset<flatbuf::Field>;
 using KeyValueOffset = flatbuffers::Offset<flatbuf::KeyValue>;
 using RecordBatchOffset = flatbuffers::Offset<flatbuf::RecordBatch>;
-using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
 using FBString = flatbuffers::Offset<flatbuffers::String>;
 
@@ -341,34 +340,8 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
 // TODO(wesm): Convert this to visitor pattern
 static Status TypeToFlatbuffer(FBB& fbb, const DataType& type,
                                std::vector<FieldOffset>* children,
-                               std::vector<VectorLayoutOffset>* layout,
                                flatbuf::Type* out_type, DictionaryMemo* dictionary_memo,
                                Offset* offset) {
-  std::vector<BufferDescr> buffer_layout = type.GetBufferLayout();
-  for (const BufferDescr& descr : buffer_layout) {
-    flatbuf::VectorType vector_type;
-    switch (descr.type()) {
-      case BufferType::OFFSET:
-        vector_type = flatbuf::VectorType_OFFSET;
-        break;
-      case BufferType::DATA:
-        vector_type = flatbuf::VectorType_DATA;
-        break;
-      case BufferType::VALIDITY:
-        vector_type = flatbuf::VectorType_VALIDITY;
-        break;
-      case BufferType::TYPE:
-        vector_type = flatbuf::VectorType_TYPE;
-        break;
-      default:
-        vector_type = flatbuf::VectorType_DATA;
-        break;
-    }
-    auto offset = flatbuf::CreateVectorLayout(
-        fbb, static_cast<int16_t>(descr.bit_width()), vector_type);
-    layout->push_back(offset);
-  }
-
   const DataType* value_type = &type;
 
   if (type.id() == Type::DICTIONARY) {
@@ -543,14 +516,11 @@ static Status FieldToFlatbuffer(FBB& fbb, const Field& field,
 
   flatbuf::Type type_enum;
   Offset type_offset;
-  Offset type_layout;
   std::vector<FieldOffset> children;
-  std::vector<VectorLayoutOffset> layout;
 
-  RETURN_NOT_OK(TypeToFlatbuffer(fbb, *field.type(), &children, &layout, &type_enum,
+  RETURN_NOT_OK(TypeToFlatbuffer(fbb, *field.type(), &children, &type_enum,
                                  dictionary_memo, &type_offset));
   auto fb_children = fbb.CreateVector(children);
-  auto fb_layout = fbb.CreateVector(layout);
 
   DictionaryOffset dictionary = 0;
   if (field.type()->id() == Type::DICTIONARY) {
@@ -560,7 +530,7 @@ static Status FieldToFlatbuffer(FBB& fbb, const Field& field,
 
   // TODO: produce the list of VectorTypes
   *offset = flatbuf::CreateField(fbb, fb_name, field.nullable(), type_enum, type_offset,
-                                 dictionary, fb_children, fb_layout);
+                                 dictionary, fb_children);
 
   return Status::OK();
 }
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index ae17118323450..31ad53458112c 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -474,43 +474,6 @@ std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale) {
   return std::make_shared<Decimal128Type>(precision, scale);
 }
 
-static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1);
-static const BufferDescr kOffsetBuffer(BufferType::OFFSET, 32);
-static const BufferDescr kTypeBuffer(BufferType::TYPE, 32);
-static const BufferDescr kBooleanBuffer(BufferType::DATA, 1);
-static const BufferDescr kValues64(BufferType::DATA, 64);
-static const BufferDescr kValues32(BufferType::DATA, 32);
-static const BufferDescr kValues16(BufferType::DATA, 16);
-static const BufferDescr kValues8(BufferType::DATA, 8);
-
-std::vector<BufferDescr> FixedWidthType::GetBufferLayout() const {
-  return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
-}
-
-std::vector<BufferDescr> NullType::GetBufferLayout() const { return {}; }
-
-std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
-  return {kValidityBuffer, kOffsetBuffer, kValues8};
-}
-
-std::vector<BufferDescr> FixedSizeBinaryType::GetBufferLayout() const {
-  return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
-}
-
-std::vector<BufferDescr> ListType::GetBufferLayout() const {
-  return {kValidityBuffer, kOffsetBuffer};
-}
-
-std::vector<BufferDescr> StructType::GetBufferLayout() const { return {kValidityBuffer}; }
-
-std::vector<BufferDescr> UnionType::GetBufferLayout() const {
-  if (mode_ == UnionMode::SPARSE) {
-    return {kValidityBuffer, kTypeBuffer};
-  } else {
-    return {kValidityBuffer, kTypeBuffer, kOffsetBuffer};
-  }
-}
-
 std::string Decimal128Type::ToString() const {
   std::stringstream s;
   s << "decimal(" << precision_ << ", " << scale_ << ")";
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 8dcc1592da01f..009e07db07744 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -133,20 +133,6 @@ struct Type {
   };
 };
 
-enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
-
-class BufferDescr {
- public:
-  BufferDescr(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
-
-  BufferType type() const { return type_; }
-  int bit_width() const { return bit_width_; }
-
- private:
-  BufferType type_;
-  int bit_width_;
-};
-
 class ARROW_EXPORT DataType {
  public:
   explicit DataType(Type::type id) : id_(id) {}
@@ -176,8 +162,6 @@ class ARROW_EXPORT DataType {
   /// \since 0.7.0
   virtual std::string name() const = 0;
 
-  virtual std::vector<BufferDescr> GetBufferLayout() const = 0;
-
   Type::type id() const { return id_; }
 
  protected:
@@ -201,8 +185,6 @@ class ARROW_EXPORT FixedWidthType : public DataType {
   using DataType::DataType;
 
   virtual int bit_width() const = 0;
-
-  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 class ARROW_EXPORT PrimitiveCType : public FixedWidthType {
@@ -319,8 +301,6 @@ class ARROW_EXPORT NullType : public DataType, public NoExtraMeta {
   std::string ToString() const override;
 
   std::string name() const override { return "null"; }
-
-  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 class ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta {
@@ -425,8 +405,6 @@ class ARROW_EXPORT ListType : public NestedType {
   std::string ToString() const override;
 
   std::string name() const override { return "list"; }
-
-  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 // BinaryType type is represents lists of 1-byte values.
@@ -440,8 +418,6 @@ class ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
   std::string ToString() const override;
   std::string name() const override { return "binary"; }
 
-  std::vector<BufferDescr> GetBufferLayout() const override;
-
  protected:
   // Allow subclasses to change the logical type.
   explicit BinaryType(Type::type logical_type) : DataType(logical_type) {}
@@ -461,8 +437,6 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType, public Parametri
   std::string ToString() const override;
   std::string name() const override { return "fixed_size_binary"; }
 
-  std::vector<BufferDescr> GetBufferLayout() const override;
-
   int32_t byte_width() const { return byte_width_; }
   int bit_width() const override;
 
@@ -494,8 +468,6 @@ class ARROW_EXPORT StructType : public NestedType {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   std::string name() const override { return "struct"; }
-
-  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
@@ -541,8 +513,6 @@ class ARROW_EXPORT UnionType : public NestedType {
   std::string name() const override { return "union"; }
   Status Accept(TypeVisitor* visitor) const override;
 
-  std::vector<BufferDescr> GetBufferLayout() const override;
-
   const std::vector<uint8_t>& type_codes() const { return type_codes_; }
 
   UnionMode::type mode() const { return mode_; }
diff --git a/format/Schema.fbs b/format/Schema.fbs
index 6021e92b847e7..3d739342b83c3 100644
--- a/format/Schema.fbs
+++ b/format/Schema.fbs
@@ -211,32 +211,6 @@ union Type {
   Map
 }
 
-/// ----------------------------------------------------------------------
-/// The possible types of a vector
-
-enum VectorType: short {
-  /// used in List type, Dense Union and variable length primitive types (String, Binary)
-  OFFSET,
-  /// actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
-  DATA,
-  /// Bit vector indicating if each value is null
-  VALIDITY,
-  /// Type vector used in Union type
-  TYPE
-}
-
-/// ----------------------------------------------------------------------
-/// represents the physical layout of a buffer
-/// buffers have fixed width slots of a given type
-
-table VectorLayout {
-  /// the width of a slot in the buffer (typically 1, 8, 16, 32 or 64)
-  bit_width: short;
-  /// the purpose of the vector
-  type: VectorType;
-}
-
-
 /// ----------------------------------------------------------------------
 /// user defined key value pairs to add custom metadata to arrow
 /// key namespacing is the responsibility of the user
@@ -285,10 +259,7 @@ table Field {
 
   // children apply only to Nested data types like Struct, List and Union
   children: [Field];
-  /// layout of buffers produced for this type (as derived from the Type)
-  /// does not include children
-  /// each recordbatch will return instances of those Buffers.
-  layout: [ VectorLayout ];
+
   // User-defined metadata
   custom_metadata: [ KeyValue ];
 }
diff --git a/integration/data/simple.json b/integration/data/simple.json
index bc8949912a48a..6634729193b4e 100644
--- a/integration/data/simple.json
+++ b/integration/data/simple.json
@@ -4,36 +4,20 @@
       {
         "name": "foo",
         "type": {"name": "int", "isSigned": true, "bitWidth": 32},
-        "nullable": true, "children": [],
-        "typeLayout": {
-          "vectors": [
-            {"type": "VALIDITY", "typeBitWidth": 1},
-            {"type": "DATA", "typeBitWidth": 32}
-          ]
-        }
+        "nullable": true,
+        "children": []
       },
       {
         "name": "bar",
         "type": {"name": "floatingpoint", "precision": "DOUBLE"},
-        "nullable": true, "children": [],
-        "typeLayout": {
-          "vectors": [
-            {"type": "VALIDITY", "typeBitWidth": 1},
-            {"type": "DATA", "typeBitWidth": 64}
-          ]
-        }
+        "nullable": true,
+        "children": []
       },
       {
         "name": "baz",
         "type": {"name": "utf8"},
-        "nullable": true, "children": [],
-        "typeLayout": {
-          "vectors": [
-            {"type": "VALIDITY", "typeBitWidth": 1},
-            {"type": "OFFSET", "typeBitWidth": 32},
-            {"type": "DATA", "typeBitWidth": 8}
-          ]
-        }
+        "nullable": true,
+        "children": []
       }
     ]
   },
diff --git a/integration/data/struct_example.json b/integration/data/struct_example.json
index 3ea062db7ba32..4e6cc774e3151 100644
--- a/integration/data/struct_example.json
+++ b/integration/data/struct_example.json
@@ -16,19 +16,7 @@
               "bitWidth": 32
             },
             "nullable": true,
-            "children": [],
-            "typeLayout": {
-              "vectors": [
-                {
-                  "type": "VALIDITY",
-                  "typeBitWidth": 1
-                },
-                {
-                  "type": "DATA",
-                  "typeBitWidth": 32
-                }
-              ]
-            }
+            "children": []
           },
           {
             "name": "f2",
@@ -36,33 +24,9 @@
               "name": "utf8"
             },
             "nullable": true,
-            "children": [],
-            "typeLayout": {
-              "vectors": [
-                {
-                  "type": "VALIDITY",
-                  "typeBitWidth": 1
-                },
-                {
-                  "type": "OFFSET",
-                  "typeBitWidth": 32
-                },
-                {
-                  "type": "DATA",
-                  "typeBitWidth": 8
-                }
-              ]
-            }
+            "children": []
           }
-        ],
-        "typeLayout": {
-          "vectors": [
-            {
-              "type": "VALIDITY",
-              "typeBitWidth": 1
-            }
-          ]
-        }
+        ]
       }
     ]
   },
@@ -234,4 +198,4 @@
       ]
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/integration/integration_test.py b/integration/integration_test.py
index a063fb36f49d5..5468ab19de8dc 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -111,8 +111,7 @@ def get_json(self):
             ('name', self.name),
             ('type', self._get_type()),
             ('nullable', self.nullable),
-            ('children', self._get_children()),
-            ('typeLayout', self._get_type_layout())
+            ('children', self._get_children())
         ])
 
     def _make_is_valid(self, size):
@@ -158,14 +157,6 @@ class PrimitiveType(DataType):
     def _get_children(self):
         return []
 
-    def _get_type_layout(self):
-        return OrderedDict([
-            ('vectors',
-             [OrderedDict([('type', 'VALIDITY'),
-                           ('typeBitWidth', 1)]),
-              OrderedDict([('type', 'DATA'),
-                           ('typeBitWidth', self.bit_width)])])])
-
 
 class PrimitiveColumn(Column):
 
@@ -402,14 +393,6 @@ def _get_type(self):
             ('scale', self.scale),
         ])
 
-    def _get_type_layout(self):
-        return OrderedDict([
-            ('vectors',
-             [OrderedDict([('type', 'VALIDITY'),
-                           ('typeBitWidth', 1)]),
-              OrderedDict([('type', 'DATA'),
-                           ('typeBitWidth', self.bit_width)])])])
-
     def generate_column(self, size, name=None):
         min_value, max_value = decimal_range_from_precision(self.precision)
         values = [random.randint(min_value, max_value) for _ in range(size)]
@@ -461,16 +444,6 @@ def column_class(self):
     def _get_type(self):
         return OrderedDict([('name', 'binary')])
 
-    def _get_type_layout(self):
-        return OrderedDict([
-            ('vectors',
-             [OrderedDict([('type', 'VALIDITY'),
-                           ('typeBitWidth', 1)]),
-              OrderedDict([('type', 'OFFSET'),
-                           ('typeBitWidth', 32)]),
-              OrderedDict([('type', 'DATA'),
-                           ('typeBitWidth', 8)])])])
-
     def generate_column(self, size, name=None):
         K = 7
         is_valid = self._make_is_valid(size)
@@ -572,14 +545,6 @@ def _get_type(self):
     def _get_children(self):
         return [self.value_type.get_json()]
 
-    def _get_type_layout(self):
-        return OrderedDict([
-            ('vectors',
-             [OrderedDict([('type', 'VALIDITY'),
-                           ('typeBitWidth', 1)]),
-              OrderedDict([('type', 'OFFSET'),
-                           ('typeBitWidth', 32)])])])
-
     def generate_column(self, size, name=None):
         MAX_LIST_SIZE = 4
 
@@ -633,12 +598,6 @@ def _get_type(self):
     def _get_children(self):
         return [type_.get_json() for type_ in self.field_types]
 
-    def _get_type_layout(self):
-        return OrderedDict([
-            ('vectors',
-             [OrderedDict([('type', 'VALIDITY'),
-                           ('typeBitWidth', 1)])])])
-
     def generate_column(self, size, name=None):
         is_valid = self._make_is_valid(size)
 
@@ -689,13 +648,9 @@ def get_json(self):
                 ('id', self.dictionary.id_),
                 ('indexType', self.index_type._get_type()),
                 ('isOrdered', self.dictionary.ordered)
-            ])),
-            ('typeLayout', self.index_type._get_type_layout())
+            ]))
         ])
 
-    def _get_type_layout(self):
-        return self.index_type._get_type_layout()
-
     def generate_column(self, size, name=None):
         if name is None:
             name = self.name
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
similarity index 50%
rename from java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java
rename to java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
index e4f2f98fde37a..f6529d8e55bba 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
@@ -16,33 +16,48 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.ipc.message;
+package org.apache.arrow.vector;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
-import com.google.flatbuffers.FlatBufferBuilder;
 
-public class VectorLayout implements FBSerializable {
+public class BufferLayout {
 
-  private static final VectorLayout VALIDITY_VECTOR = new VectorLayout(ArrowVectorType.VALIDITY, 1);
-  private static final VectorLayout OFFSET_VECTOR = new VectorLayout(ArrowVectorType.OFFSET, 32);
-  private static final VectorLayout TYPE_VECTOR = new VectorLayout(ArrowVectorType.TYPE, 32);
-  private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(ArrowVectorType.DATA, 1);
-  private static final VectorLayout VALUES_64 = new VectorLayout(ArrowVectorType.DATA, 64);
-  private static final VectorLayout VALUES_32 = new VectorLayout(ArrowVectorType.DATA, 32);
-  private static final VectorLayout VALUES_16 = new VectorLayout(ArrowVectorType.DATA, 16);
-  private static final VectorLayout VALUES_8 = new VectorLayout(ArrowVectorType.DATA, 8);
+  public enum BufferType {
+    DATA("DATA"),
+    OFFSET("OFFSET"),
+    VALIDITY("VALIDITY"),
+    TYPE("TYPE");
 
-  public static VectorLayout typeVector() {
-    return TYPE_VECTOR;
+    final private String name;
+
+    BufferType(String name) {
+      this.name = name;
+    }
+
+    public String getName() {
+      return name;
+    }
   }
 
-  public static VectorLayout offsetVector() {
-    return OFFSET_VECTOR;
+  private static final BufferLayout VALIDITY_BUFFER = new BufferLayout(BufferType.VALIDITY, 1);
+  private static final BufferLayout OFFSET_BUFFER = new BufferLayout(BufferType.OFFSET, 32);
+  private static final BufferLayout TYPE_BUFFER = new BufferLayout(BufferType.TYPE, 32);
+  private static final BufferLayout BIT_BUFFER = new BufferLayout(BufferType.DATA, 1);
+  private static final BufferLayout VALUES_128 = new BufferLayout(BufferType.DATA, 128);
+  private static final BufferLayout VALUES_64 = new BufferLayout(BufferType.DATA, 64);
+  private static final BufferLayout VALUES_32 = new BufferLayout(BufferType.DATA, 32);
+  private static final BufferLayout VALUES_16 = new BufferLayout(BufferType.DATA, 16);
+  private static final BufferLayout VALUES_8 = new BufferLayout(BufferType.DATA, 8);
+
+  public static BufferLayout typeBuffer() {
+    return TYPE_BUFFER;
   }
 
-  public static VectorLayout dataVector(int typeBitWidth) {
+  public static BufferLayout offsetBuffer() {
+    return OFFSET_BUFFER;
+  }
+
+  public static BufferLayout dataBuffer(int typeBitWidth) {
     switch (typeBitWidth) {
       case 8:
         return VALUES_8;
@@ -52,29 +67,30 @@ public static VectorLayout dataVector(int typeBitWidth) {
         return VALUES_32;
       case 64:
         return VALUES_64;
+      case 128:
+        return VALUES_128;
       default:
         throw new IllegalArgumentException("only 8, 16, 32, or 64 bits supported");
     }
   }
 
-  public static VectorLayout booleanVector() {
-    return BOOLEAN_VECTOR;
+  public static BufferLayout booleanVector() {
+    return BIT_BUFFER;
   }
 
-  public static VectorLayout validityVector() {
-    return VALIDITY_VECTOR;
+  public static BufferLayout validityVector() {
+    return VALIDITY_BUFFER;
   }
 
-  public static VectorLayout byteVector() {
-    return dataVector(8);
+  public static BufferLayout byteVector() {
+    return dataBuffer(8);
   }
 
   private final short typeBitWidth;
 
-  private final ArrowVectorType type;
+  private final BufferType type;
 
-  @JsonCreator
-  private VectorLayout(@JsonProperty("type") ArrowVectorType type, @JsonProperty("typeBitWidth") int typeBitWidth) {
+  private BufferLayout(BufferType type, int typeBitWidth) {
     super();
     this.type = Preconditions.checkNotNull(type);
     this.typeBitWidth = (short) typeBitWidth;
@@ -83,15 +99,11 @@ private VectorLayout(@JsonProperty("type") ArrowVectorType type, @JsonProperty("
     }
   }
 
-  public VectorLayout(org.apache.arrow.flatbuf.VectorLayout layout) {
-    this(new ArrowVectorType(layout.type()), layout.bitWidth());
-  }
-
   public int getTypeBitWidth() {
     return typeBitWidth;
   }
 
-  public ArrowVectorType getType() {
+  public BufferType getType() {
     return type;
   }
 
@@ -116,15 +128,7 @@ public boolean equals(Object obj) {
     if (getClass() != obj.getClass()) {
       return false;
     }
-    VectorLayout other = (VectorLayout) obj;
+    BufferLayout other = (BufferLayout) obj;
     return type.equals(other.type) && (typeBitWidth == other.typeBitWidth);
   }
-
-  @Override
-  public int writeTo(FlatBufferBuilder builder) {
-    ;
-    return org.apache.arrow.flatbuf.VectorLayout.createVectorLayout(builder, typeBitWidth, type.getType());
-  }
-
-
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
similarity index 66%
rename from java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java
rename to java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
index 06fe9481686d7..d6f32b4b4b1d5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.arrow.vector.ipc.message;
+package org.apache.arrow.vector;
 
 import static java.util.Arrays.asList;
 
@@ -24,6 +24,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.arrow.vector.BufferLayout.BufferType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor;
 import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
@@ -41,14 +42,11 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
 
 /**
- * The layout of vectors for a given type
- * It defines its own vectors followed by the vectors for the children
+ * The buffer layout of vectors for a given type
+ * It defines its own buffers followed by the buffers for the children
  * if it is a nested type (Struct_, List, Union)
  */
 public class TypeLayout {
@@ -58,24 +56,24 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) {
 
       @Override
       public TypeLayout visit(Int type) {
-        return newFixedWidthTypeLayout(VectorLayout.dataVector(type.getBitWidth()));
+        return newFixedWidthTypeLayout(BufferLayout.dataBuffer(type.getBitWidth()));
       }
 
       @Override
       public TypeLayout visit(Union type) {
-        List<VectorLayout> vectors;
+        List<BufferLayout> vectors;
         switch (type.getMode()) {
           case Dense:
             vectors = asList(
                 // TODO: validate this
-                VectorLayout.validityVector(),
-                VectorLayout.typeVector(),
-                VectorLayout.offsetVector() // offset to find the vector
+                BufferLayout.validityVector(),
+                BufferLayout.typeBuffer(),
+                BufferLayout.offsetBuffer() // offset to find the vector
             );
             break;
           case Sparse:
             vectors = asList(
-                VectorLayout.typeVector() // type of the value at the index or 0 if null
+                BufferLayout.typeBuffer() // type of the value at the index or 0 if null
             );
             break;
           default:
@@ -86,30 +84,30 @@ public TypeLayout visit(Union type) {
 
       @Override
       public TypeLayout visit(Struct type) {
-        List<VectorLayout> vectors = asList(
-            VectorLayout.validityVector()
+        List<BufferLayout> vectors = asList(
+            BufferLayout.validityVector()
         );
         return new TypeLayout(vectors);
       }
 
       @Override
       public TypeLayout visit(Timestamp type) {
-        return newFixedWidthTypeLayout(VectorLayout.dataVector(64));
+        return newFixedWidthTypeLayout(BufferLayout.dataBuffer(64));
       }
 
       @Override
       public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) {
-        List<VectorLayout> vectors = asList(
-            VectorLayout.validityVector(),
-            VectorLayout.offsetVector()
+        List<BufferLayout> vectors = asList(
+            BufferLayout.validityVector(),
+            BufferLayout.offsetBuffer()
         );
         return new TypeLayout(vectors);
       }
 
       @Override
       public TypeLayout visit(FixedSizeList type) {
-        List<VectorLayout> vectors = asList(
-            VectorLayout.validityVector()
+        List<BufferLayout> vectors = asList(
+            BufferLayout.validityVector()
         );
         return new TypeLayout(vectors);
       }
@@ -130,18 +128,17 @@ public TypeLayout visit(FloatingPoint type) {
           default:
             throw new UnsupportedOperationException("Unsupported Precision: " + type.getPrecision());
         }
-        return newFixedWidthTypeLayout(VectorLayout.dataVector(bitWidth));
+        return newFixedWidthTypeLayout(BufferLayout.dataBuffer(bitWidth));
       }
 
       @Override
       public TypeLayout visit(Decimal type) {
-        // TODO: check size
-        return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); // actually depends on the type fields
+        return newFixedWidthTypeLayout(BufferLayout.dataBuffer(128));
       }
 
       @Override
       public TypeLayout visit(Bool type) {
-        return newFixedWidthTypeLayout(VectorLayout.booleanVector());
+        return newFixedWidthTypeLayout(BufferLayout.booleanVector());
       }
 
       @Override
@@ -155,39 +152,46 @@ public TypeLayout visit(Utf8 type) {
       }
 
       private TypeLayout newVariableWidthTypeLayout() {
-        return newPrimitiveTypeLayout(VectorLayout.validityVector(), VectorLayout.offsetVector(), VectorLayout.byteVector());
+        return newPrimitiveTypeLayout(BufferLayout.validityVector(), BufferLayout.offsetBuffer(), BufferLayout.byteVector());
       }
 
-      private TypeLayout newPrimitiveTypeLayout(VectorLayout... vectors) {
+      private TypeLayout newPrimitiveTypeLayout(BufferLayout... vectors) {
         return new TypeLayout(asList(vectors));
       }
 
-      public TypeLayout newFixedWidthTypeLayout(VectorLayout dataVector) {
-        return newPrimitiveTypeLayout(VectorLayout.validityVector(), dataVector);
+      public TypeLayout newFixedWidthTypeLayout(BufferLayout dataVector) {
+        return newPrimitiveTypeLayout(BufferLayout.validityVector(), dataVector);
       }
 
       @Override
       public TypeLayout visit(Null type) {
-        return new TypeLayout(Collections.<VectorLayout>emptyList());
+        return new TypeLayout(Collections.<BufferLayout>emptyList());
       }
 
       @Override
       public TypeLayout visit(Date type) {
-        return newFixedWidthTypeLayout(VectorLayout.dataVector(64));
+        switch (type.getUnit()) {
+          case DAY:
+            return newFixedWidthTypeLayout(BufferLayout.dataBuffer(32));
+          case MILLISECOND:
+            return newFixedWidthTypeLayout(BufferLayout.dataBuffer(64));
+          default:
+            throw new UnsupportedOperationException("Unknown unit " + type.getUnit());
+        }
       }
 
       @Override
       public TypeLayout visit(Time type) {
-        return newFixedWidthTypeLayout(VectorLayout.dataVector(type.getBitWidth()));
+        return newFixedWidthTypeLayout(BufferLayout.dataBuffer(type.getBitWidth()));
       }
 
       @Override
-      public TypeLayout visit(Interval type) { // TODO: check size
+      public TypeLayout visit(Interval type) {
         switch (type.getUnit()) {
           case DAY_TIME:
-            return newFixedWidthTypeLayout(VectorLayout.dataVector(64));
+            return newFixedWidthTypeLayout(BufferLayout.dataBuffer(64));
           case YEAR_MONTH:
-            return newFixedWidthTypeLayout(VectorLayout.dataVector(64));
+            return newFixedWidthTypeLayout(BufferLayout.dataBuffer(32));
           default:
             throw new UnsupportedOperationException("Unknown unit " + type.getUnit());
         }
@@ -197,39 +201,37 @@ public TypeLayout visit(Interval type) { // TODO: check size
     return layout;
   }
 
-  private final List<VectorLayout> vectors;
+  private final List<BufferLayout> bufferLayouts;
 
-  @JsonCreator
-  public TypeLayout(@JsonProperty("vectors") List<VectorLayout> vectors) {
+  public TypeLayout(List<BufferLayout> bufferLayouts) {
     super();
-    this.vectors = Preconditions.checkNotNull(vectors);
+    this.bufferLayouts = Preconditions.checkNotNull(bufferLayouts);
   }
 
-  public TypeLayout(VectorLayout... vectors) {
-    this(asList(vectors));
+  public TypeLayout(BufferLayout... bufferLayouts) {
+    this(asList(bufferLayouts));
   }
 
 
-  public List<VectorLayout> getVectors() {
-    return vectors;
+  public List<BufferLayout> getBufferLayouts() {
+    return bufferLayouts;
   }
 
-  @JsonIgnore
-  public List<ArrowVectorType> getVectorTypes() {
-    List<ArrowVectorType> types = new ArrayList<>(vectors.size());
-    for (VectorLayout vector : vectors) {
+  public List<BufferType> getBufferTypes() {
+    List<BufferType> types = new ArrayList<>(bufferLayouts.size());
+    for (BufferLayout vector : bufferLayouts) {
       types.add(vector.getType());
     }
     return types;
   }
 
   public String toString() {
-    return vectors.toString();
+    return bufferLayouts.toString();
   }
 
   @Override
   public int hashCode() {
-    return vectors.hashCode();
+    return bufferLayouts.hashCode();
   }
 
   @Override
@@ -244,7 +246,7 @@ public boolean equals(Object obj) {
       return false;
     }
     TypeLayout other = (TypeLayout) obj;
-    return vectors.equals(other.vectors);
+    return bufferLayouts.equals(other.bufferLayouts);
   }
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 2cd4099c66975..c933d149f8d00 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -26,7 +26,6 @@
 
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-import org.apache.arrow.vector.ipc.message.VectorLayout;
 import org.apache.arrow.vector.types.pojo.Field;
 
 import com.google.common.collect.Iterators;
@@ -71,9 +70,9 @@ private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buf
     checkArgument(nodes.hasNext(),
         "no more field nodes for for field " + field + " and vector " + vector);
     ArrowFieldNode fieldNode = nodes.next();
-    List<VectorLayout> typeLayout = field.getTypeLayout().getVectors();
-    List<ArrowBuf> ownBuffers = new ArrayList<>(typeLayout.size());
-    for (int j = 0; j < typeLayout.size(); j++) {
+    List<BufferLayout> bufferLayouts = TypeLayout.getTypeLayout(field.getType()).getBufferLayouts();
+    List<ArrowBuf> ownBuffers = new ArrayList<>(bufferLayouts.size());
+    for (int j = 0; j < bufferLayouts.size(); j++) {
       ownBuffers.add(buffers.next());
     }
     try {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 2b034894ab14d..94981ef02f657 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -22,9 +22,9 @@
 import java.util.List;
 
 import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.vector.BufferLayout.BufferType;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-import org.apache.arrow.vector.ipc.message.ArrowVectorType;
 
 public class VectorUnloader {
 
@@ -54,7 +54,7 @@ public ArrowRecordBatch getRecordBatch() {
   private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
     nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ? vector.getNullCount() : -1));
     List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
-    List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
+    List<BufferType> expectedBuffers = TypeLayout.getTypeLayout(vector.getField().getType()).getBufferTypes();
     if (fieldBuffers.size() != expectedBuffers.size()) {
       throw new IllegalArgumentException(String.format(
           "wrong number of buffers for field %s in vector %s. found: %s",
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
index f274b748e55dc..d0a9b9e18b8f2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -23,7 +23,7 @@
 import static com.fasterxml.jackson.core.JsonToken.START_ARRAY;
 import static com.fasterxml.jackson.core.JsonToken.START_OBJECT;
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.arrow.vector.ipc.message.ArrowVectorType.*;
+import static org.apache.arrow.vector.BufferLayout.BufferType.*;
 
 import java.io.File;
 import java.io.IOException;
@@ -42,7 +42,8 @@
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
-import org.apache.arrow.vector.ipc.message.ArrowVectorType;
+import org.apache.arrow.vector.BufferLayout.BufferType;
+import org.apache.arrow.vector.TypeLayout;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -375,7 +376,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
     };
   }
 
-  private ArrowBuf readIntoBuffer(BufferAllocator allocator, ArrowVectorType bufferType,
+  private ArrowBuf readIntoBuffer(BufferAllocator allocator, BufferType bufferType,
                                   Types.MinorType type, int count) throws IOException {
     ArrowBuf buf;
 
@@ -471,7 +472,8 @@ private ArrowBuf readIntoBuffer(BufferAllocator allocator, ArrowVectorType buffe
   }
 
   private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException {
-    List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
+    TypeLayout typeLayout = TypeLayout.getTypeLayout(field.getType());
+    List<BufferType> vectorTypes = typeLayout.getBufferTypes();
     ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()];
     /*
      * The order of inner buffers is :
@@ -503,15 +505,15 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json
       vector.setInitialCapacity(valueCount);
 
       for (int v = 0; v < vectorTypes.size(); v++) {
-        ArrowVectorType vectorType = vectorTypes.get(v);
-        nextFieldIs(vectorType.getName());
+        BufferType bufferType = vectorTypes.get(v);
+        nextFieldIs(bufferType.getName());
         int innerBufferValueCount = valueCount;
-        if (vectorType.equals(OFFSET)) {
+        if (bufferType.equals(OFFSET)) {
           /* offset buffer has 1 additional value capacity */
           innerBufferValueCount = valueCount + 1;
         }
 
-        vectorBuffers[v] = readIntoBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount);
+        vectorBuffers[v] = readIntoBuffer(allocator, bufferType, vector.getMinorType(), innerBufferValueCount);
       }
 
       final int nullCount = BitVectorHelper.getNullCount(vectorBuffers[0], valueCount);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
index 1c9e1d3809594..6eb76a7a14723 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -18,7 +18,7 @@
 
 package org.apache.arrow.vector.ipc;
 
-import static org.apache.arrow.vector.ipc.message.ArrowVectorType.*;
+import static org.apache.arrow.vector.BufferLayout.BufferType.*;
 
 import java.io.File;
 import java.io.IOException;
@@ -33,7 +33,8 @@
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
-import org.apache.arrow.vector.ipc.message.ArrowVectorType;
+import org.apache.arrow.vector.BufferLayout.BufferType;
+import org.apache.arrow.vector.TypeLayout;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -152,7 +153,7 @@ private void writeBatch(VectorSchemaRoot recordBatch) throws IOException {
   }
 
   private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOException {
-    List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
+    List<BufferType> vectorTypes = TypeLayout.getTypeLayout(field.getType()).getBufferTypes();
     List<ArrowBuf> vectorBuffers = vector.getFieldBuffers();
     if (vectorTypes.size() != vectorBuffers.size()) {
       throw new IllegalArgumentException("vector types and inner vector buffers are not the same size: " + vectorTypes.size() + " != " + vectorBuffers.size());
@@ -165,16 +166,16 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
       final int scale = (vector instanceof DecimalVector) ?
                             ((DecimalVector) vector).getScale() : 0;
       for (int v = 0; v < vectorTypes.size(); v++) {
-        ArrowVectorType vectorType = vectorTypes.get(v);
+        BufferType bufferType = vectorTypes.get(v);
         ArrowBuf vectorBuffer = vectorBuffers.get(v);
-        generator.writeArrayFieldStart(vectorType.getName());
-        final int bufferValueCount = (vectorType.equals(OFFSET)) ? valueCount + 1 : valueCount;
+        generator.writeArrayFieldStart(bufferType.getName());
+        final int bufferValueCount = (bufferType.equals(OFFSET)) ? valueCount + 1 : valueCount;
         for (int i = 0; i < bufferValueCount; i++) {
-          if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR ||
+          if (bufferType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR ||
                   vector.getMinorType() == Types.MinorType.VARBINARY)) {
-            writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i, scale);
+            writeValueToGenerator(bufferType, vectorBuffer, vectorBuffers.get(v-1), vector, i, scale);
           } else {
-            writeValueToGenerator(vectorType, vectorBuffer, null, vector, i, scale);
+            writeValueToGenerator(bufferType, vectorBuffer, null, vector, i, scale);
           }
         }
         generator.writeEndArray();
@@ -197,7 +198,7 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
     generator.writeEndObject();
   }
 
-  private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer,
+  private void writeValueToGenerator(BufferType bufferType, ArrowBuf buffer,
                                      ArrowBuf offsetBuffer, FieldVector vector,
                                      final int index, final int scale) throws IOException {
     if (bufferType.equals(TYPE)) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java
deleted file mode 100644
index 3342652bedd52..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.ipc.message;
-
-import java.util.Map;
-
-import org.apache.arrow.flatbuf.VectorType;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMap.Builder;
-
-public class ArrowVectorType {
-
-  public static final ArrowVectorType DATA = new ArrowVectorType(VectorType.DATA);
-  public static final ArrowVectorType OFFSET = new ArrowVectorType(VectorType.OFFSET);
-  public static final ArrowVectorType VALIDITY = new ArrowVectorType(VectorType.VALIDITY);
-  public static final ArrowVectorType TYPE = new ArrowVectorType(VectorType.TYPE);
-
-  private static final Map<String, ArrowVectorType> typeByName;
-
-  static {
-    ArrowVectorType[] types = {DATA, OFFSET, VALIDITY, TYPE};
-    Builder<String, ArrowVectorType> builder = ImmutableMap.builder();
-    for (ArrowVectorType type : types) {
-      builder.put(type.getName(), type);
-    }
-    typeByName = builder.build();
-  }
-
-  public static ArrowVectorType fromName(String name) {
-    ArrowVectorType type = typeByName.get(name);
-    if (type == null) {
-      throw new IllegalArgumentException("Unknown type " + name);
-    }
-    return type;
-  }
-
-  private final short type;
-
-  public ArrowVectorType(short type) {
-    this.type = type;
-    // validate that the type is valid
-    getName();
-  }
-
-  @JsonCreator
-  private ArrowVectorType(String name) {
-    this.type = fromName(name).type;
-  }
-
-  public short getType() {
-    return type;
-  }
-
-  @JsonValue
-  public String getName() {
-    try {
-      return VectorType.name(type);
-    } catch (ArrayIndexOutOfBoundsException e) {
-      throw new IllegalArgumentException("Unknown type " + type);
-    }
-  }
-
-  @Override
-  public String toString() {
-    return getName();
-  }
-
-  @Override
-  public int hashCode() {
-    return type;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj instanceof ArrowVectorType) {
-      ArrowVectorType other = (ArrowVectorType) obj;
-      return type == other.type;
-    }
-    return false;
-  }
-
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 574612833cd19..b1f036a34a9a6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -43,8 +43,7 @@
 import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.ipc.message.VectorLayout;
-import org.apache.arrow.vector.ipc.message.TypeLayout;
+import org.apache.arrow.vector.TypeLayout;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 
 public class Field {
@@ -60,7 +59,6 @@ public static Field nullable(String name, ArrowType type) {
   private final String name;
   private final FieldType fieldType;
   private final List<Field> children;
-  private final TypeLayout typeLayout;
 
   @JsonCreator
   private Field(
@@ -69,16 +67,14 @@ private Field(
       @JsonProperty("type") ArrowType type,
       @JsonProperty("dictionary") DictionaryEncoding dictionary,
       @JsonProperty("children") List<Field> children,
-      @JsonProperty("typeLayout") TypeLayout typeLayout,
       @JsonProperty("metadata") Map<String, String> metadata) {
-    this(name, new FieldType(nullable, type, dictionary, metadata), children, typeLayout);
+    this(name, new FieldType(nullable, type, dictionary, metadata), children);
   }
 
   private Field(String name, FieldType fieldType, List<Field> children, TypeLayout typeLayout) {
     this.name = name;
     this.fieldType = checkNotNull(fieldType);
     this.children = children == null ? ImmutableList.<Field>of() : ImmutableList.copyOf(children);
-    this.typeLayout = checkNotNull(typeLayout);
   }
 
   // deprecated, use FieldType or static constructor instead
@@ -117,10 +113,6 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
       }
       dictionary = new DictionaryEncoding(dictionaryFB.id(), dictionaryFB.isOrdered(), indexType);
     }
-    ImmutableList.Builder<VectorLayout> layout = ImmutableList.builder();
-    for (int i = 0; i < field.layoutLength(); ++i) {
-      layout.add(new VectorLayout(field.layout(i)));
-    }
     ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
     for (int i = 0; i < field.childrenLength(); i++) {
       Field childField = convertField(field.children(i));
@@ -135,7 +127,7 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
       metadataBuilder.put(key == null ? "" : key, value == null ? "" : value);
     }
     Map<String, String> metadata = metadataBuilder.build();
-    return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()), metadata);
+    return new Field(name, nullable, type, dictionary, children, metadata);
   }
 
   /**
@@ -153,19 +145,11 @@ private static Field mutateOriginalNameIfNeeded(org.apache.arrow.flatbuf.Field f
           originalChildField.getType(),
           originalChildField.getDictionary(),
           originalChildField.getChildren(),
-          originalChildField.getTypeLayout(),
           originalChildField.getMetadata());
     }
     return originalChildField;
   }
 
-  public void validate() {
-    TypeLayout expectedLayout = TypeLayout.getTypeLayout(getType());
-    if (!expectedLayout.equals(typeLayout)) {
-      throw new IllegalArgumentException("Deserialized field does not match expected vectors. expected: " + expectedLayout + " got " + typeLayout);
-    }
-  }
-
   public int getField(FlatBufferBuilder builder) {
     int nameOffset = name == null ? -1 : builder.createString(name);
     int typeOffset = getType().getType(builder);
@@ -184,12 +168,6 @@ public int getField(FlatBufferBuilder builder) {
       childrenData[i] = children.get(i).getField(builder);
     }
     int childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, childrenData);
-    int[] buffersData = new int[typeLayout.getVectors().size()];
-    for (int i = 0; i < buffersData.length; i++) {
-      VectorLayout vectorLayout = typeLayout.getVectors().get(i);
-      buffersData[i] = vectorLayout.writeTo(builder);
-    }
-    int layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData);
     int[] metadataOffsets = new int[getMetadata().size()];
     Iterator<Entry<String, String>> metadataIterator = getMetadata().entrySet().iterator();
     for (int i = 0; i < metadataOffsets.length; i++) {
@@ -210,7 +188,6 @@ public int getField(FlatBufferBuilder builder) {
     org.apache.arrow.flatbuf.Field.addTypeType(builder, getType().getTypeID().getFlatbufID());
     org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
     org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
-    org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
     org.apache.arrow.flatbuf.Field.addCustomMetadata(builder, metadataOffset);
     if (dictionary != null) {
       org.apache.arrow.flatbuf.Field.addDictionary(builder, dictionaryOffset);
@@ -244,10 +221,6 @@ public List<Field> getChildren() {
     return children;
   }
 
-  public TypeLayout getTypeLayout() {
-    return typeLayout;
-  }
-
   @JsonInclude(Include.NON_EMPTY)
   public Map<String, String> getMetadata() {
     return fieldType.getMetadata();
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 1cff7fa886e87..1acce7e0b6654 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -170,16 +170,16 @@ public void testSetLastSetUsage() throws Exception {
       /* set lastset and arbitrary valuecount for list vector.
        *
        * NOTE: if we don't execute setLastSet() before setLastValueCount(), then
-       * the latter will corrupt the offsetVector and thus the accessor will not
-       * retrieve the correct values from underlying dataVector. Run the test
+       * the latter will corrupt the offsetBuffer and thus the accessor will not
+       * retrieve the correct values from underlying dataBuffer. Run the test
        * by commenting out next line and we should see failures from 5th assert
        * onwards. This is why doing setLastSet() is important before setValueCount()
        * once the vector has been loaded.
        *
        * Another important thing to remember is the value of lastSet itself.
        * Even though the listVector has elements till index 2 only, the lastSet should
-       * be set as 3. This is because the offsetVector has valid offsets filled till index 3.
-       * If we do setLastSet(2), the offsetVector at index 3 will contain incorrect value
+       * be set as 3. This is because the offsetBuffer has valid offsets filled till index 3.
+       * If we do setLastSet(2), the offsetBuffer at index 3 will contain incorrect value
        * after execution of setValueCount().
        *
        * correct state of the listVector
@@ -414,7 +414,7 @@ public void testSplitAndTransfer() throws Exception {
 
           transferPair.splitAndTransfer(start, splitLength);
 
-          /* get offsetVector of toVector */
+          /* get offsetBuffer of toVector */
           final ArrowBuf toOffsetBuffer = toVector.getOffsetBuffer();
 
           /* get dataVector of toVector */
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 336ae1c7a0146..601b2062ff698 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -38,7 +38,6 @@
 import org.apache.arrow.memory.RootAllocator;
 
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-import org.apache.arrow.vector.ipc.message.TypeLayout;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -711,11 +710,9 @@ public void testNullableFixedType3() {
 
       vector.setValueCount(1024);
       Field field = vector.getField();
-      TypeLayout typeLayout = field.getTypeLayout();
 
       List<ArrowBuf> buffers = vector.getFieldBuffers();
 
-      assertEquals(2, typeLayout.getVectors().size());
       assertEquals(2, buffers.size());
 
       ArrowBuf validityVectorBuf = buffers.get(0);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
index 49e194b51b823..bf42fbb83c84b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java
@@ -105,7 +105,6 @@ public void test() throws IOException {
          ArrowFileReader reader = new ArrowFileReader(channel, allocator)) {
       Schema readSchema = reader.getVectorSchemaRoot().getSchema();
       assertEquals(schema, readSchema);
-      assertTrue(readSchema.getFields().get(0).getTypeLayout().getVectorTypes().toString(), readSchema.getFields().get(0).getTypeLayout().getVectors().size() > 0);
       // TODO: dictionaries
       List<ArrowBlock> recordBatches = reader.getRecordBlocks();
       assertEquals(1, recordBatches.size());
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
index bd1ec94c8fd2b..9ec9a078f7ef0 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java
@@ -124,9 +124,6 @@ public boolean loadNextBatch() throws IOException {
     public void run() {
       try {
         assertEquals(schema, reader.getVectorSchemaRoot().getSchema());
-        assertTrue(
-            reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectorTypes().toString(),
-            reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectors().size() > 0);
         while (!done) {
           assertTrue(reader.loadNextBatch() != done);
         }

From b241eb699b6151d9b1c8809528be4c80cce26c7e Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 4 Dec 2017 22:39:44 -0500
Subject: [PATCH 1309/1644] ARROW-1839/ARROW-1871: [C++/Python] Add Decimal
 Parquet Read/Write Tests

Author: Phillip Cloud <cpcloud@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1335 from cpcloud/ARROW-1839 and squashes the following commits:

496ec796 [Wes McKinney] Make RescaleDecimal instead an instance method of Decimal128
878d9a1d [Phillip Cloud] Fix namespace
816ff42a [Phillip Cloud] Use IsPyInteger
41c447fb [Wes McKinney] Add missing license header
3b91e550 [Wes McKinney] Rename Rescale to RescaleDecimal
accc653d [Wes McKinney] Make AppendItem methods non-virtual for CRTP inlining
bbc14cf4 [Phillip Cloud] int64_t for looping
b6f8592f [Phillip Cloud] Generate a column for different scales
afe4396a [Phillip Cloud] docstring
ade030e5 [Phillip Cloud] Clarify randdecimal contract
752e905b [Phillip Cloud] Use Python C API instead of string parsing
115fbc96 [Phillip Cloud] Remove iostream include
1f6339d8 [Phillip Cloud] Clean up testing
a4b1d6bb [Phillip Cloud] Add randdecimal to util
07c356ac [Phillip Cloud] Remove inference from python objects
c7637f19 [Phillip Cloud] Refactor test
dba8a0d9 [Phillip Cloud] Add docs
4296b5c9 [Phillip Cloud] Infer the scale by looking at every Python decimal object
98e319af [Phillip Cloud] Remove debugging
2d02b44f [Phillip Cloud] More style fixes
f235ef42 [Phillip Cloud] Style fixes
97c0a6b5 [Phillip Cloud] Make sure we assign to our out variable
d4a32361 [Phillip Cloud] Move rescaling to decimal.h/cc
891137fd [Phillip Cloud] Formatting
deea4711 [Phillip Cloud] Refactor rescaling functionality
108e8914 [Phillip Cloud] Format
c28e2c17 [Phillip Cloud] Remove test
cbb2ed76 [Phillip Cloud] Rewrite infer precision and scale from python
7a2de887 [Phillip Cloud] Add rescaling
b2ad3b7f [Phillip Cloud] Rename
e0d6080a [Phillip Cloud] Formatting
fc5412a3 [Phillip Cloud] Checkpoint [ci skip]
cf3a5a26 [Phillip Cloud] Test multiple precisions and scales
853fced9 [Phillip Cloud] Make the parquet read/write from python test more robust
ff6eccf8 [Phillip Cloud] Add c++ formatting test for small number and small scale/precision
cd38abd4 [Phillip Cloud] Add test util
45eafe73 [Phillip Cloud] Add better test
d7a0aab4 [Phillip Cloud] Make sure we do not override the inferred type
eff7d2b5 [Phillip Cloud] Use array syntax
801626b4 [Phillip Cloud] Use FormatValue
e519c25f [Phillip Cloud] Add Decimal parquet tests
---
 cpp/src/arrow/python/arrow_to_pandas.cc | 20 ++----
 cpp/src/arrow/python/builtin_convert.cc | 76 +++++++++-----------
 cpp/src/arrow/python/helpers.cc         | 77 +++++++++++++++++---
 cpp/src/arrow/python/helpers.h          | 11 ++-
 cpp/src/arrow/python/numpy-internal.h   |  4 +-
 cpp/src/arrow/python/numpy_to_arrow.cc  | 35 +++++++---
 cpp/src/arrow/python/python-test.cc     | 81 +++++++++++++++------
 cpp/src/arrow/util/decimal-test.cc      |  9 +++
 cpp/src/arrow/util/decimal.cc           | 64 +++++++++++++++++
 cpp/src/arrow/util/decimal.h            |  3 +
 python/pyarrow/tests/test_parquet.py    | 44 +++++++++++-
 python/pyarrow/tests/util.py            | 93 +++++++++++++++++++++++++
 12 files changed, 411 insertions(+), 106 deletions(-)
 create mode 100644 python/pyarrow/tests/util.py

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 096bbd55c6d41..b1825cbe69d2c 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -620,13 +620,6 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
   return Status::OK();
 }
 
-static Status RawDecimalToString(const uint8_t* bytes, int scale, std::string* result) {
-  DCHECK_NE(result, nullptr);
-  Decimal128 decimal(bytes);
-  *result = decimal.ToString(scale);
-  return Status::OK();
-}
-
 static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
                               PyObject** out_values) {
   PyAcquireGIL lock;
@@ -637,19 +630,14 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
   PyObject* Decimal = Decimal_ref.obj();
 
   for (int c = 0; c < data.num_chunks(); c++) {
-    auto* arr(static_cast<arrow::Decimal128Array*>(data.chunk(c).get()));
-    auto type(std::dynamic_pointer_cast<arrow::Decimal128Type>(arr->type()));
-    const int scale = type->scale();
+    const auto& arr(static_cast<const arrow::Decimal128Array&>(*data.chunk(c).get()));
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (arr->IsNull(i)) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (arr.IsNull(i)) {
         Py_INCREF(Py_None);
         *out_values++ = Py_None;
       } else {
-        const uint8_t* raw_value = arr->GetValue(i);
-        std::string decimal_string;
-        RETURN_NOT_OK(RawDecimalToString(raw_value, scale, &decimal_string));
-        *out_values++ = internal::DecimalFromString(Decimal, decimal_string);
+        *out_values++ = internal::DecimalFromString(Decimal, arr.FormatValue(i));
         RETURN_IF_PYERROR();
       }
     }
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index c716c47d2de23..08cbae7ab2da4 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -37,14 +37,6 @@
 namespace arrow {
 namespace py {
 
-static inline bool IsPyInteger(PyObject* obj) {
-#if PYARROW_IS_PY2
-  return PyLong_Check(obj) || PyInt_Check(obj);
-#else
-  return PyLong_Check(obj);
-#endif
-}
-
 Status InvalidConversion(PyObject* obj, const std::string& expected_types,
                          std::ostream* out) {
   OwnedRef type(PyObject_Type(obj));
@@ -91,7 +83,7 @@ class ScalarVisitor {
       ++bool_count_;
     } else if (PyFloat_Check(obj)) {
       ++float_count_;
-    } else if (IsPyInteger(obj)) {
+    } else if (internal::IsPyInteger(obj)) {
       ++int_count_;
     } else if (PyDate_CheckExact(obj)) {
       ++date_count_;
@@ -390,28 +382,26 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
     }
     return Status::OK();
   }
-
-  virtual Status AppendItem(const OwnedRef& item) = 0;
 };
 
 class NullConverter : public TypedConverterVisitor<NullBuilder, NullConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     return Status::Invalid("NullConverter: passed non-None value");
   }
 };
 
 class BoolConverter : public TypedConverterVisitor<BooleanBuilder, BoolConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     return typed_builder_->Append(item.obj() == Py_True);
   }
 };
 
 class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int8_t>::max() ||
                             val < std::numeric_limits<int8_t>::min())) {
@@ -426,8 +416,8 @@ class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
 
 class Int16Converter : public TypedConverterVisitor<Int16Builder, Int16Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int16_t>::max() ||
                             val < std::numeric_limits<int16_t>::min())) {
@@ -442,8 +432,8 @@ class Int16Converter : public TypedConverterVisitor<Int16Builder, Int16Converter
 
 class Int32Converter : public TypedConverterVisitor<Int32Builder, Int32Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int32_t>::max() ||
                             val < std::numeric_limits<int32_t>::min())) {
@@ -458,8 +448,8 @@ class Int32Converter : public TypedConverterVisitor<Int32Builder, Int32Converter
 
 class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
@@ -467,8 +457,8 @@ class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter
 
 class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint8_t>::max())) {
       return Status::Invalid(
@@ -482,8 +472,8 @@ class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter
 
 class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint16_t>::max())) {
       return Status::Invalid(
@@ -497,8 +487,8 @@ class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Conver
 
 class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    uint64_t val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint32_t>::max())) {
       return Status::Invalid(
@@ -512,8 +502,8 @@ class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Conver
 
 class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
-    int64_t val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  Status AppendItem(const OwnedRef& item) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
@@ -521,13 +511,13 @@ class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Conver
 
 class Date32Converter : public TypedConverterVisitor<Date32Builder, Date32Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     int32_t t;
     if (PyDate_Check(item.obj())) {
       auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
       t = static_cast<int32_t>(PyDate_to_s(pydate));
     } else {
-      int64_t casted_val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      const auto casted_val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
       RETURN_IF_PYERROR();
       if (casted_val > std::numeric_limits<int32_t>::max()) {
         return Status::Invalid("Integer as date32 larger than INT32_MAX");
@@ -540,7 +530,7 @@ class Date32Converter : public TypedConverterVisitor<Date32Builder, Date32Conver
 
 class Date64Converter : public TypedConverterVisitor<Date64Builder, Date64Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     int64_t t;
     if (PyDate_Check(item.obj())) {
       auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
@@ -558,7 +548,7 @@ class TimestampConverter
  public:
   explicit TimestampConverter(TimeUnit::type unit) : unit_(unit) {}
 
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     int64_t t;
     if (PyDateTime_Check(item.obj())) {
       auto pydatetime = reinterpret_cast<PyDateTime_DateTime*>(item.obj());
@@ -590,7 +580,7 @@ class TimestampConverter
 
 class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     double val = PyFloat_AsDouble(item.obj());
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
@@ -599,7 +589,7 @@ class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConver
 
 class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     const char* bytes;
     Py_ssize_t length;
@@ -627,7 +617,7 @@ class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverte
 class FixedWidthBytesConverter
     : public TypedConverterVisitor<FixedSizeBinaryBuilder, FixedWidthBytesConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
     Py_ssize_t expected_length =
@@ -654,7 +644,7 @@ class FixedWidthBytesConverter
 
 class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     PyObject* bytes_obj;
     OwnedRef tmp;
     const char* bytes;
@@ -689,10 +679,10 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
  public:
   Status Init(ArrayBuilder* builder) override;
 
-  inline Status AppendItem(const OwnedRef& item) override {
+  Status AppendItem(const OwnedRef& item) {
     RETURN_NOT_OK(typed_builder_->Append());
     PyObject* item_obj = item.obj();
-    int64_t list_size = static_cast<int64_t>(PySequence_Size(item_obj));
+    const auto list_size = static_cast<int64_t>(PySequence_Size(item_obj));
     return value_converter_->AppendData(item_obj, list_size);
   }
 
@@ -703,13 +693,11 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
 class DecimalConverter
     : public TypedConverterVisitor<arrow::Decimal128Builder, DecimalConverter> {
  public:
-  inline Status AppendItem(const OwnedRef& item) {
+  Status AppendItem(const OwnedRef& item) {
     /// TODO(phillipc): Check for nan?
-    std::string string;
-    RETURN_NOT_OK(internal::PythonDecimalToString(item.obj(), &string));
-
     Decimal128 value;
-    RETURN_NOT_OK(Decimal128::FromString(string, &value));
+    const auto& type = static_cast<const DecimalType&>(*typed_builder_->type());
+    RETURN_NOT_OK(internal::DecimalFromPythonDecimal(item.obj(), type, &value));
     return typed_builder_->Append(value);
   }
 };
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 708d9916f46d3..494f929004ae9 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -15,8 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/python/helpers.h"
+#include <sstream>
+
 #include "arrow/python/common.h"
+#include "arrow/python/helpers.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 
@@ -91,20 +93,33 @@ Status PythonDecimalToString(PyObject* python_decimal, std::string* out) {
   return Status::OK();
 }
 
-Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision,
-                                     int* scale) {
-  // Call Python's str(decimal_object)
-  OwnedRef str_obj(PyObject_Str(python_decimal));
+Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precision,
+                                     int32_t* scale) {
+  DCHECK_NE(python_decimal, NULLPTR);
+  DCHECK_NE(precision, NULLPTR);
+  DCHECK_NE(scale, NULLPTR);
+
+  OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
   RETURN_IF_PYERROR();
-  PyObjectStringify str(str_obj.obj());
+  DCHECK(PyTuple_Check(as_tuple.obj()));
 
-  const char* bytes = str.bytes;
-  DCHECK_NE(bytes, nullptr);
+  OwnedRef digits(PyObject_GetAttrString(as_tuple.obj(), "digits"));
+  RETURN_IF_PYERROR();
+  DCHECK(PyTuple_Check(digits.obj()));
 
-  auto size = str.size;
+  const auto num_digits = static_cast<int32_t>(PyTuple_Size(digits.obj()));
+  RETURN_IF_PYERROR();
 
-  std::string c_string(bytes, size);
-  return Decimal128::FromString(c_string, nullptr, precision, scale);
+  OwnedRef py_exponent(PyObject_GetAttrString(as_tuple.obj(), "exponent"));
+  RETURN_IF_PYERROR();
+  DCHECK(IsPyInteger(py_exponent.obj()));
+
+  const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
+  RETURN_IF_PYERROR();
+
+  *precision = num_digits;
+  *scale = -exponent;
+  return Status::OK();
 }
 
 PyObject* DecimalFromString(PyObject* decimal_constructor,
@@ -121,6 +136,46 @@ PyObject* DecimalFromString(PyObject* decimal_constructor,
                                string_size);
 }
 
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+                                Decimal128* out) {
+  DCHECK_NE(python_decimal, NULLPTR);
+  DCHECK_NE(out, NULLPTR);
+
+  std::string string;
+  RETURN_NOT_OK(PythonDecimalToString(python_decimal, &string));
+
+  int32_t inferred_precision;
+  int32_t inferred_scale;
+
+  RETURN_NOT_OK(
+      Decimal128::FromString(string, out, &inferred_precision, &inferred_scale));
+
+  const int32_t precision = arrow_type.precision();
+  const int32_t scale = arrow_type.scale();
+
+  if (ARROW_PREDICT_FALSE(inferred_precision > precision)) {
+    std::stringstream buf;
+    buf << "Decimal type with precision " << inferred_precision
+        << " does not fit into precision inferred from first array element: "
+        << precision;
+    return Status::Invalid(buf.str());
+  }
+
+  if (scale != inferred_scale) {
+    DCHECK_NE(out, NULLPTR);
+    RETURN_NOT_OK(out->Rescale(inferred_scale, scale, out));
+  }
+  return Status::OK();
+}
+
+bool IsPyInteger(PyObject* obj) {
+#if PYARROW_IS_PY2
+  return PyLong_Check(obj) || PyInt_Check(obj);
+#else
+  return PyLong_Check(obj);
+#endif
+}
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index 719ed796e08a1..c82bdabc47614 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -29,6 +29,9 @@
 #include "arrow/util/visibility.h"
 
 namespace arrow {
+
+class Decimal128;
+
 namespace py {
 
 class OwnedRef;
@@ -44,11 +47,15 @@ Status ImportFromModule(const OwnedRef& module, const std::string& module_name,
 
 Status PythonDecimalToString(PyObject* python_decimal, std::string* out);
 
-Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision = NULLPTR,
-                                     int* scale = NULLPTR);
+Status InferDecimalPrecisionAndScale(PyObject* python_decimal,
+                                     int32_t* precision = NULLPTR,
+                                     int32_t* scale = NULLPTR);
 
 PyObject* DecimalFromString(PyObject* decimal_constructor,
                             const std::string& decimal_string);
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+                                Decimal128* out);
+bool IsPyInteger(PyObject* obj);
 
 }  // namespace internal
 }  // namespace py
diff --git a/cpp/src/arrow/python/numpy-internal.h b/cpp/src/arrow/python/numpy-internal.h
index db34d24d99da5..6c9c871a100a2 100644
--- a/cpp/src/arrow/python/numpy-internal.h
+++ b/cpp/src/arrow/python/numpy-internal.h
@@ -56,8 +56,8 @@ class Ndarray1DIndexer {
 
   bool is_strided() const { return stride_ == 1; }
 
-  T& operator[](size_type index) { return *(data_ + index * stride_); }
-  T& operator[](size_type index) const { return *(data_ + index * stride_); }
+  T& operator[](size_type index) { return data_[index * stride_]; }
+  T& operator[](size_type index) const { return data_[index * stride_]; }
 
  private:
   PyArrayObject* arr_;
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 0b1124d303df1..2316a796a21c0 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -682,24 +682,41 @@ Status NumPyConverter::ConvertDecimals() {
   Ndarray1DIndexer<PyObject*> objects(arr_);
   PyObject* object = objects[0];
 
-  int precision;
-  int scale;
+  if (type_ == NULLPTR) {
+    int32_t precision;
+    int32_t desired_scale;
 
-  RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(object, &precision, &scale));
+    int32_t tmp_precision;
+    int32_t tmp_scale;
 
-  type_ = std::make_shared<Decimal128Type>(precision, scale);
+    RETURN_NOT_OK(
+        internal::InferDecimalPrecisionAndScale(objects[0], &precision, &desired_scale));
+
+    for (int64_t i = 1; i < length_; ++i) {
+      RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
+                                                            &tmp_scale));
+      precision = std::max(precision, tmp_precision);
+
+      if (std::abs(desired_scale) < std::abs(tmp_scale)) {
+        desired_scale = tmp_scale;
+      }
+    }
+
+    type_ = ::arrow::decimal(precision, desired_scale);
+  }
 
   Decimal128Builder builder(type_, pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
+  const auto& decimal_type = static_cast<const DecimalType&>(*type_);
+  PyObject* Decimal_type_object = Decimal.obj();
+
   for (int64_t i = 0; i < length_; ++i) {
     object = objects[i];
-    if (PyObject_IsInstance(object, Decimal.obj())) {
-      std::string string;
-      RETURN_NOT_OK(internal::PythonDecimalToString(object, &string));
 
+    if (PyObject_IsInstance(object, Decimal_type_object)) {
       Decimal128 value;
-      RETURN_NOT_OK(Decimal128::FromString(string, &value));
+      RETURN_NOT_OK(internal::DecimalFromPythonDecimal(object, decimal_type, &value));
       RETURN_NOT_OK(builder.Append(value));
     } else if (PandasObjectIsNull(object)) {
       RETURN_NOT_OK(builder.AppendNull());
@@ -724,7 +741,7 @@ Status NumPyConverter::ConvertTimes() {
   Time64Builder builder(::arrow::time64(TimeUnit::MICRO), pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
-  PyObject* obj;
+  PyObject* obj = NULLPTR;
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
     if (PyTime_Check(obj)) {
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 3b7d7d884ef09..d9919ee499188 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -35,34 +35,73 @@ namespace py {
 
 TEST(PyBuffer, InvalidInputObject) { PyBuffer buffer(Py_None); }
 
-TEST(DecimalTest, TestPythonDecimalToString) {
-  PyAcquireGIL lock;
+class DecimalTest : public ::testing::Test {
+ public:
+  DecimalTest() : lock_(), decimal_module_(), decimal_constructor_() {
+    auto s = internal::ImportModule("decimal", &decimal_module_);
+    DCHECK(s.ok()) << s.message();
+    DCHECK_NE(decimal_module_.obj(), NULLPTR);
 
-  OwnedRef decimal;
-  OwnedRef Decimal;
-  ASSERT_OK(internal::ImportModule("decimal", &decimal));
-  ASSERT_NE(decimal.obj(), nullptr);
+    s = internal::ImportFromModule(decimal_module_, "Decimal", &decimal_constructor_);
+    DCHECK(s.ok()) << s.message();
 
-  ASSERT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal));
-  ASSERT_NE(Decimal.obj(), nullptr);
+    DCHECK_NE(decimal_constructor_.obj(), NULLPTR);
+  }
 
-  std::string decimal_string("-39402950693754869342983");
-  const char* format = "s#";
-  auto c_string = decimal_string.c_str();
-  ASSERT_NE(c_string, nullptr);
+  OwnedRef CreatePythonDecimal(const std::string& string_value) {
+    OwnedRef ref(internal::DecimalFromString(decimal_constructor_.obj(), string_value));
+    return ref;
+  }
 
-  auto c_string_size = decimal_string.size();
-  ASSERT_GT(c_string_size, 0);
-  OwnedRef pydecimal(PyObject_CallFunction(Decimal.obj(), const_cast<char*>(format),
-                                           c_string, c_string_size));
-  ASSERT_NE(pydecimal.obj(), nullptr);
-  ASSERT_EQ(PyErr_Occurred(), nullptr);
+ private:
+  PyAcquireGIL lock_;
+  OwnedRef decimal_module_;
+  OwnedRef decimal_constructor_;
+};
 
-  PyObject* python_object = pydecimal.obj();
-  ASSERT_NE(python_object, nullptr);
+TEST_F(DecimalTest, TestPythonDecimalToString) {
+  std::string decimal_string("-39402950693754869342983");
+
+  OwnedRef python_object = this->CreatePythonDecimal(decimal_string);
+  ASSERT_NE(python_object.obj(), nullptr);
 
   std::string string_result;
-  ASSERT_OK(internal::PythonDecimalToString(python_object, &string_result));
+  ASSERT_OK(internal::PythonDecimalToString(python_object.obj(), &string_result));
+}
+
+TEST_F(DecimalTest, TestInferPrecisionAndScale) {
+  std::string decimal_string("-394029506937548693.42983");
+  OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
+
+  int32_t precision;
+  int32_t scale;
+
+  ASSERT_OK(
+      internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));
+
+  const auto expected_precision =
+      static_cast<int32_t>(decimal_string.size() - 2);  // 1 for -, 1 for .
+  const int32_t expected_scale = 5;
+
+  ASSERT_EQ(expected_precision, precision);
+  ASSERT_EQ(expected_scale, scale);
+}
+
+TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
+  std::string decimal_string("-3.94042983E+10");
+  OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
+
+  int32_t precision;
+  int32_t scale;
+
+  ASSERT_OK(
+      internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));
+
+  const auto expected_precision = 9;
+  const int32_t expected_scale = -2;
+
+  ASSERT_EQ(expected_precision, precision);
+  ASSERT_EQ(expected_scale, scale);
 }
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 0d0c08cc4551f..e4406747d557c 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -366,4 +366,13 @@ TEST(Decimal128ParseTest, WithExponentAndNullptrScale) {
   ASSERT_EQ(expected_value, value);
 }
 
+TEST(Decimal128Test, TestSmallNumberFormat) {
+  Decimal128 value("0.2");
+  std::string expected("0.2");
+
+  const int32_t scale = 1;
+  std::string result = value.ToString(scale);
+  ASSERT_EQ(expected, result);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 447cae5c54654..e999854b10860 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -813,4 +813,68 @@ Decimal128 operator%(const Decimal128& left, const Decimal128& right) {
   return remainder;
 }
 
+static const Decimal128 ScaleMultipliers[] = {
+    Decimal128(1),
+    Decimal128(10),
+    Decimal128(100),
+    Decimal128(1000),
+    Decimal128(10000),
+    Decimal128(100000),
+    Decimal128(1000000),
+    Decimal128(10000000),
+    Decimal128(100000000),
+    Decimal128(1000000000),
+    Decimal128(10000000000),
+    Decimal128(100000000000),
+    Decimal128(1000000000000),
+    Decimal128(10000000000000),
+    Decimal128(100000000000000),
+    Decimal128(1000000000000000),
+    Decimal128(10000000000000000),
+    Decimal128(100000000000000000),
+    Decimal128(1000000000000000000),
+    Decimal128("10000000000000000000"),
+    Decimal128("100000000000000000000"),
+    Decimal128("1000000000000000000000"),
+    Decimal128("10000000000000000000000"),
+    Decimal128("100000000000000000000000"),
+    Decimal128("1000000000000000000000000"),
+    Decimal128("10000000000000000000000000"),
+    Decimal128("100000000000000000000000000"),
+    Decimal128("1000000000000000000000000000"),
+    Decimal128("10000000000000000000000000000"),
+    Decimal128("100000000000000000000000000000"),
+    Decimal128("1000000000000000000000000000000"),
+    Decimal128("10000000000000000000000000000000"),
+    Decimal128("100000000000000000000000000000000"),
+    Decimal128("1000000000000000000000000000000000"),
+    Decimal128("10000000000000000000000000000000000"),
+    Decimal128("100000000000000000000000000000000000"),
+    Decimal128("1000000000000000000000000000000000000"),
+    Decimal128("10000000000000000000000000000000000000"),
+    Decimal128("100000000000000000000000000000000000000")};
+
+Status Decimal128::Rescale(int32_t original_scale, int32_t new_scale,
+                           Decimal128* out) const {
+  DCHECK_NE(out, NULLPTR);
+  DCHECK_NE(original_scale, new_scale);
+  const int32_t delta_scale = original_scale - new_scale;
+  const int32_t abs_delta_scale = std::abs(delta_scale);
+  DCHECK_GE(abs_delta_scale, 1);
+  DCHECK_LE(abs_delta_scale, 38);
+
+  const Decimal128 scale_multiplier = ScaleMultipliers[abs_delta_scale];
+  const Decimal128 result = *this * scale_multiplier;
+
+  if (ARROW_PREDICT_FALSE(result < *this)) {
+    std::stringstream buf;
+    buf << "Rescaling decimal value from original scale " << original_scale
+        << " to new scale " << new_scale << " would cause overflow";
+    return Status::Invalid(buf.str());
+  }
+
+  *out = result;
+  return Status::OK();
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index a0423e9fce49c..1594090a0d30e 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -126,6 +126,9 @@ class ARROW_EXPORT Decimal128 {
   static Status FromString(const std::string& s, Decimal128* out,
                            int* precision = NULLPTR, int* scale = NULLPTR);
 
+  /// \brief Convert Decimal128 from one scale to another
+  Status Rescale(int32_t original_scale, int32_t new_scale, Decimal128* out) const;
+
  private:
   int64_t high_bits_;
   uint64_t low_bits_;
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 9004fc01ae4a0..d17d89e24df5f 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -16,13 +16,17 @@
 # under the License.
 
 from os.path import join as pjoin
+
 import datetime
+import decimal
 import io
-import os
 import json
+import os
+
 import pytest
 
 from pyarrow.compat import guid, u, BytesIO, unichar, frombytes
+from pyarrow.tests import util
 from pyarrow.filesystem import LocalFileSystem
 import pyarrow as pa
 from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
@@ -1564,3 +1568,41 @@ def test_backwards_compatible_index_multi_level_some_named():
     t = _read_table(path)
     result = t.to_pandas()
     tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize('precision', range(1, 39))
+def test_decimal_roundtrip(tmpdir, precision):
+    num_values = 10
+
+    columns = {}
+
+    for scale in range(0, precision + 1):
+        with util.random_seed(0):
+            random_decimal_values = [
+                util.randdecimal(precision, scale) for _ in range(num_values)
+            ]
+        column_name = 'dec_precision_{:d}_scale_{:d}'.format(precision, scale)
+        columns[column_name] = random_decimal_values
+
+    expected = pd.DataFrame(columns)
+    filename = tmpdir.join('decimals.parquet')
+    string_filename = str(filename)
+    t = pa.Table.from_pandas(expected)
+    _write_table(t, string_filename)
+    result_table = _read_table(string_filename)
+    result = result_table.to_pandas()
+    tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.xfail(
+    raises=pa.ArrowException, reason='Parquet does not support negative scale'
+)
+def test_decimal_roundtrip_negative_scale(tmpdir):
+    expected = pd.DataFrame({'decimal_num': [decimal.Decimal('1.23E4')]})
+    filename = tmpdir.join('decimals.parquet')
+    string_filename = str(filename)
+    t = pa.Table.from_pandas(expected)
+    _write_table(t, string_filename)
+    result_table = _read_table(string_filename)
+    result = result_table.to_pandas()
+    tm.assert_frame_equal(result, expected)
diff --git a/python/pyarrow/tests/util.py b/python/pyarrow/tests/util.py
new file mode 100644
index 0000000000000..a3ba9000c2f62
--- /dev/null
+++ b/python/pyarrow/tests/util.py
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Utility functions for testing
+"""
+
+import decimal
+import random
+import contextlib
+
+
+def randsign():
+    """Randomly choose either 1 or -1.
+
+    Returns
+    -------
+    sign : int
+    """
+    return random.choice((-1, 1))
+
+
+@contextlib.contextmanager
+def random_seed(seed):
+    """Set the random seed inside of a context manager.
+
+    Parameters
+    ----------
+    seed : int
+        The seed to set
+
+    Notes
+    -----
+    This function is useful when you want to set a random seed but not affect
+    the random state of other functions using the random module.
+    """
+    original_state = random.getstate()
+    random.seed(seed)
+    try:
+        yield
+    finally:
+        random.setstate(original_state)
+
+
+def randdecimal(precision, scale):
+    """Generate a random decimal value with specified precision and scale.
+
+    Parameters
+    ----------
+    precision : int
+        The maximum number of digits to generate. Must be an integer between 1
+        and 38 inclusive.
+    scale : int
+        The maximum number of digits following the decimal point.  Must be an
+        integer greater than or equal to 0.
+
+    Returns
+    -------
+    decimal_value : decimal.Decimal
+        A random decimal.Decimal object with the specifed precision and scale.
+    """
+    assert 1 <= precision <= 38, 'precision must be between 1 and 38 inclusive'
+    if scale < 0:
+        raise ValueError(
+            'randdecimal does not yet support generating decimals with '
+            'negative scale'
+        )
+    max_whole_value = 10 ** (precision - scale) - 1
+    whole = random.randint(-max_whole_value, max_whole_value)
+
+    if not scale:
+        return decimal.Decimal(whole)
+
+    max_fractional_value = 10 ** scale - 1
+    fractional = random.randint(0, max_fractional_value)
+
+    return decimal.Decimal(
+        '{}.{}'.format(whole, str(fractional).rjust(scale, '0'))
+    )

From d64947e8c650687856bd221ea4ff15c86db0ebc1 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Tue, 5 Dec 2017 13:59:26 -0500
Subject: [PATCH 1310/1644] ARROW-1881: Ignore JS tags for Python packages

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1391 from xhochy/ARROW-1881 and squashes the following commits:

1e31a74c [Korn, Uwe] ARROW-1881: Ignore JS tags for Python packages
---
 python/setup.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index 2270cf79c2a8b..b56a4fdcace6c 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -395,6 +395,16 @@ def has_ext_modules(foo):
     install_requires.append('futures')
 
 
+def parse_version(root):
+    from setuptools_scm import version_from_scm
+    import setuptools_scm.git
+    describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --exclude 'apache-arrow-js-*'"
+    version = setuptools_scm.git.parse(root, describe)
+    if not version:
+        return version_from_scm(root)
+    else:
+        return version
+
 setup(
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
@@ -413,7 +423,7 @@ def has_ext_modules(foo):
             'plasma_store = pyarrow:_plasma_store_entry_point'
         ]
     },
-    use_scm_version={"root": "..", "relative_to": __file__},
+    use_scm_version={"root": "..", "relative_to": __file__, "parse": parse_version},
     setup_requires=['setuptools_scm', 'cython >= 0.23'],
     install_requires=install_requires,
     tests_require=['pytest'],

From 33b628acf535b4ea4c21504483929eb399980405 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Tue, 5 Dec 2017 22:02:49 -0500
Subject: [PATCH 1311/1644] ARROW-1882: [C++] Reintroduce DictionaryBuilder

Readded the previous code and moved some small parts to a new common place to share logic between kernels and builder.

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1388 from xhochy/ARROW-1882 and squashes the following commits:

db9d67a8 [Wes McKinney] Consolidate some hash table constants in util/hash.h
b387b73d [Korn, Uwe] Ensure 64but integers
4743a8bb [Korn, Uwe] Fix precision loss
4c507fed [Korn, Uwe] ninja format
a1da9355 [Korn, Uwe] Reuse common hash table code
ba307794 [Korn, Uwe] ARROW-1882: [C++] Reintroduce DictionaryBuilder
---
 cpp/src/arrow/CMakeLists.txt          |   1 +
 cpp/src/arrow/array-test.cc           | 347 ++++++++++++++++++++++++++
 cpp/src/arrow/builder.cc              | 300 ++++++++++++++++++++++
 cpp/src/arrow/builder.h               | 153 ++++++++++++
 cpp/src/arrow/compute/kernels/hash.cc |  61 +----
 cpp/src/arrow/util/CMakeLists.txt     |   1 +
 cpp/src/arrow/util/hash.cc            |  38 +++
 cpp/src/arrow/util/hash.h             |  85 +++++++
 8 files changed, 927 insertions(+), 59 deletions(-)
 create mode 100644 cpp/src/arrow/util/hash.cc
 create mode 100644 cpp/src/arrow/util/hash.h

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 94705781fa42d..d645cca224480 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -38,6 +38,7 @@ set(ARROW_SRCS
   util/compression.cc
   util/cpu-info.cc
   util/decimal.cc
+  util/hash.cc
   util/key_value_metadata.cc
 )
 
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index d894df1311e9e..7ff3261ecba5e 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1619,6 +1619,353 @@ TEST_F(TestAdaptiveUIntBuilder, TestAppendVector) {
   ASSERT_TRUE(expected_->Equals(result_));
 }
 
+// ----------------------------------------------------------------------
+// Dictionary tests
+
+template <typename Type>
+class TestDictionaryBuilder : public TestBuilder {};
+
+typedef ::testing::Types<Int8Type, UInt8Type, Int16Type, UInt16Type, Int32Type,
+                         UInt32Type, Int64Type, UInt64Type, FloatType, DoubleType>
+    PrimitiveDictionaries;
+
+TYPED_TEST_CASE(TestDictionaryBuilder, PrimitiveDictionaries);
+
+TYPED_TEST(TestDictionaryBuilder, Basic) {
+  DictionaryBuilder<TypeParam> builder(default_memory_pool());
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  NumericBuilder<TypeParam> dict_builder;
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> dict_array;
+  ASSERT_OK(dict_builder.Finish(&dict_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
+  NumericBuilder<TypeParam> builder;
+  // DictionaryBuilder<TypeParam> builder;
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+
+  std::shared_ptr<Array> intermediate_result;
+  ASSERT_OK(builder.Finish(&intermediate_result));
+  DictionaryBuilder<TypeParam> dictionary_builder(default_memory_pool());
+  ASSERT_OK(dictionary_builder.AppendArray(*intermediate_result));
+  std::shared_ptr<Array> result;
+  ASSERT_OK(dictionary_builder.Finish(&result));
+
+  // Build expected data
+  NumericBuilder<TypeParam> dict_builder;
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(dict_builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> dict_array;
+  ASSERT_OK(dict_builder.Finish(&dict_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), dict_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
+  using Scalar = typename TypeParam::c_type;
+  // Skip this test for (u)int8
+  if (sizeof(Scalar) > 1) {
+    // Build the dictionary Array
+    DictionaryBuilder<TypeParam> builder(default_memory_pool());
+    // Build expected data
+    NumericBuilder<TypeParam> dict_builder;
+    Int16Builder int_builder;
+
+    // Fill with 1024 different values
+    for (int64_t i = 0; i < 1024; i++) {
+      ASSERT_OK(builder.Append(static_cast<Scalar>(i)));
+      ASSERT_OK(dict_builder.Append(static_cast<Scalar>(i)));
+      ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+    }
+    // Fill with an already existing value
+    for (int64_t i = 0; i < 1024; i++) {
+      ASSERT_OK(builder.Append(static_cast<Scalar>(1)));
+      ASSERT_OK(int_builder.Append(1));
+    }
+
+    // Finalize result
+    std::shared_ptr<Array> result;
+    ASSERT_OK(builder.Finish(&result));
+
+    // Finalize expected data
+    std::shared_ptr<Array> dict_array;
+    ASSERT_OK(dict_builder.Finish(&dict_array));
+    auto dtype = std::make_shared<DictionaryType>(int16(), dict_array);
+    std::shared_ptr<Array> int_array;
+    ASSERT_OK(int_builder.Finish(&int_array));
+
+    DictionaryArray expected(dtype, int_array);
+    ASSERT_TRUE(expected.Equals(result));
+  }
+}
+
+TEST(TestStringDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  StringDictionaryBuilder builder(default_memory_pool());
+  ASSERT_OK(builder.Append("test"));
+  ASSERT_OK(builder.Append("test2"));
+  ASSERT_OK(builder.Append("test"));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  StringBuilder str_builder;
+  ASSERT_OK(str_builder.Append("test"));
+  ASSERT_OK(str_builder.Append("test2"));
+  std::shared_ptr<Array> str_array;
+  ASSERT_OK(str_builder.Finish(&str_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), str_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestStringDictionaryBuilder, DoubleTableSize) {
+  // Build the dictionary Array
+  StringDictionaryBuilder builder(default_memory_pool());
+  // Build expected data
+  StringBuilder str_builder;
+  Int16Builder int_builder;
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    std::stringstream ss;
+    ss << "test" << i;
+    ASSERT_OK(builder.Append(ss.str()));
+    ASSERT_OK(str_builder.Append(ss.str()));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append("test1"));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> str_array;
+  ASSERT_OK(str_builder.Finish(&str_array));
+  auto dtype = std::make_shared<DictionaryType>(int16(), str_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestFixedSizeBinaryDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  std::vector<uint8_t> test{12, 12, 11, 12};
+  std::vector<uint8_t> test2{12, 12, 11, 11};
+  ASSERT_OK(builder.Append(test.data()));
+  ASSERT_OK(builder.Append(test2.data()));
+  ASSERT_OK(builder.Append(test.data()));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4));
+  ASSERT_OK(fsb_builder.Append(test.data()));
+  ASSERT_OK(fsb_builder.Append(test2.data()));
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+  auto dtype = std::make_shared<DictionaryType>(int8(), fsb_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append(0));
+  ASSERT_OK(int_builder.Append(1));
+  ASSERT_OK(int_builder.Append(0));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4));
+  Int16Builder int_builder;
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    std::vector<uint8_t> value{12, 12, static_cast<uint8_t>(i / 128),
+                               static_cast<uint8_t>(i % 128)};
+    ASSERT_OK(builder.Append(value.data()));
+    ASSERT_OK(fsb_builder.Append(value.data()));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  std::vector<uint8_t> known_value{12, 12, 0, 1};
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append(known_value.data()));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+  auto dtype = std::make_shared<DictionaryType>(int16(), fsb_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  // Build an array with different byte width
+  FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(5));
+  std::vector<uint8_t> value{100, 1, 1, 1, 1};
+  ASSERT_OK(fsb_builder.Append(value.data()));
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+
+  ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array));
+}
+
+TEST(TestDecimalDictionaryBuilder, Basic) {
+  // Build the dictionary Array
+  const auto& decimal_type = arrow::decimal(2, 0);
+  DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type, default_memory_pool());
+
+  // Test data
+  std::vector<Decimal128> test{12, 12, 11, 12};
+  for (const auto& value : test) {
+    ASSERT_OK(builder.Append(value.ToBytes().data()));
+  }
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  FixedSizeBinaryBuilder decimal_builder(decimal_type);
+  ASSERT_OK(decimal_builder.Append(Decimal128(12).ToBytes()));
+  ASSERT_OK(decimal_builder.Append(Decimal128(11).ToBytes()));
+
+  std::shared_ptr<Array> decimal_array;
+  ASSERT_OK(decimal_builder.Finish(&decimal_array));
+  auto dtype = arrow::dictionary(int8(), decimal_array);
+
+  Int8Builder int_builder;
+  ASSERT_OK(int_builder.Append({0, 0, 1, 0}));
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
+TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
+  const auto& decimal_type = arrow::decimal(21, 0);
+
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type, default_memory_pool());
+
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder(decimal_type);
+  Int16Builder int_builder;
+
+  // Fill with 1024 different values
+  for (int64_t i = 0; i < 1024; i++) {
+    const uint8_t bytes[] = {0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             12,
+                             12,
+                             static_cast<uint8_t>(i / 128),
+                             static_cast<uint8_t>(i % 128)};
+    ASSERT_OK(builder.Append(bytes));
+    ASSERT_OK(fsb_builder.Append(bytes));
+    ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
+  }
+  // Fill with an already existing value
+  const uint8_t known_value[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1};
+  for (int64_t i = 0; i < 1024; i++) {
+    ASSERT_OK(builder.Append(known_value));
+    ASSERT_OK(int_builder.Append(1));
+  }
+
+  // Finalize result
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Finalize expected data
+  std::shared_ptr<Array> fsb_array;
+  ASSERT_OK(fsb_builder.Finish(&fsb_array));
+
+  auto dtype = std::make_shared<DictionaryType>(int16(), fsb_array);
+  std::shared_ptr<Array> int_array;
+  ASSERT_OK(int_builder.Finish(&int_array));
+
+  DictionaryArray expected(dtype, int_array);
+  ASSERT_TRUE(expected.Equals(result));
+}
+
 // ----------------------------------------------------------------------
 // List tests
 
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 4d7fd5ff840ea..de132b5f6a0d1 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -34,6 +34,7 @@
 #include "arrow/util/cpu-info.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/hash-util.h"
+#include "arrow/util/hash.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
@@ -805,6 +806,305 @@ Status BooleanBuilder::Append(const std::vector<bool>& values) {
   return Status::OK();
 }
 
+// ----------------------------------------------------------------------
+// DictionaryBuilder
+
+using internal::WrappedBinary;
+
+template <typename T>
+DictionaryBuilder<T>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
+                                        MemoryPool* pool)
+    : ArrayBuilder(type, pool),
+      hash_slots_(nullptr),
+      dict_builder_(type, pool),
+      values_builder_(pool),
+      byte_width_(-1) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
+
+DictionaryBuilder<NullType>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
+                                               MemoryPool* pool)
+    : ArrayBuilder(type, pool), values_builder_(pool) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
+
+DictionaryBuilder<NullType>::~DictionaryBuilder() {}
+
+template <>
+DictionaryBuilder<FixedSizeBinaryType>::DictionaryBuilder(
+    const std::shared_ptr<DataType>& type, MemoryPool* pool)
+    : ArrayBuilder(type, pool),
+      hash_slots_(nullptr),
+      dict_builder_(type, pool),
+      values_builder_(pool),
+      byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {
+  if (!::arrow::CpuInfo::initialized()) {
+    ::arrow::CpuInfo::Init();
+  }
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::Init(int64_t elements) {
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+
+  // Fill the initial hash table
+  RETURN_NOT_OK(internal::NewHashTable(kInitialHashTableSize, pool_, &hash_table_));
+  hash_slots_ = reinterpret_cast<int32_t*>(hash_table_->mutable_data());
+  hash_table_size_ = kInitialHashTableSize;
+  mod_bitmask_ = kInitialHashTableSize - 1;
+  hash_table_load_threshold_ =
+      static_cast<int64_t>(static_cast<double>(elements) * kMaxHashTableLoad);
+
+  return values_builder_.Init(elements);
+}
+
+Status DictionaryBuilder<NullType>::Init(int64_t elements) {
+  RETURN_NOT_OK(ArrayBuilder::Init(elements));
+  return values_builder_.Init(elements);
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::Resize(int64_t capacity) {
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
+
+  if (capacity_ == 0) {
+    return Init(capacity);
+  } else {
+    return ArrayBuilder::Resize(capacity);
+  }
+}
+
+Status DictionaryBuilder<NullType>::Resize(int64_t capacity) {
+  if (capacity < kMinBuilderCapacity) {
+    capacity = kMinBuilderCapacity;
+  }
+
+  if (capacity_ == 0) {
+    return Init(capacity);
+  } else {
+    return ArrayBuilder::Resize(capacity);
+  }
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  std::shared_ptr<Array> dictionary;
+  RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
+
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
+  return Status::OK();
+}
+
+Status DictionaryBuilder<NullType>::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  std::shared_ptr<Array> dictionary = std::make_shared<NullArray>(0);
+
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
+  return Status::OK();
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::Append(const Scalar& value) {
+  RETURN_NOT_OK(Reserve(1));
+  // Based on DictEncoder<DType>::Put
+  int64_t j = HashValue(value) & mod_bitmask_;
+  hash_slot_t index = hash_slots_[j];
+
+  // Find an empty slot
+  while (kHashSlotEmpty != index && SlotDifferent(index, value)) {
+    // Linear probing
+    ++j;
+    if (j == hash_table_size_) {
+      j = 0;
+    }
+    index = hash_slots_[j];
+  }
+
+  if (index == kHashSlotEmpty) {
+    // Not in the hash table, so we insert it now
+    index = static_cast<hash_slot_t>(dict_builder_.length());
+    hash_slots_[j] = index;
+    RETURN_NOT_OK(AppendDictionary(value));
+
+    if (ARROW_PREDICT_FALSE(static_cast<int32_t>(dict_builder_.length()) >
+                            hash_table_load_threshold_)) {
+      RETURN_NOT_OK(DoubleTableSize());
+    }
+  }
+
+  RETURN_NOT_OK(values_builder_.Append(index));
+
+  return Status::OK();
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::AppendArray(const Array& array) {
+  const auto& numeric_array = static_cast<const NumericArray<T>&>(array);
+  for (int64_t i = 0; i < array.length(); i++) {
+    if (array.IsNull(i)) {
+      RETURN_NOT_OK(AppendNull());
+    } else {
+      RETURN_NOT_OK(Append(numeric_array.Value(i)));
+    }
+  }
+  return Status::OK();
+}
+
+Status DictionaryBuilder<NullType>::AppendArray(const Array& array) {
+  for (int64_t i = 0; i < array.length(); i++) {
+    RETURN_NOT_OK(AppendNull());
+  }
+  return Status::OK();
+}
+
+template <>
+Status DictionaryBuilder<FixedSizeBinaryType>::AppendArray(const Array& array) {
+  if (!type_->Equals(*array.type())) {
+    return Status::Invalid("Cannot append FixedSizeBinary array with non-matching type");
+  }
+
+  const auto& numeric_array = static_cast<const FixedSizeBinaryArray&>(array);
+  for (int64_t i = 0; i < array.length(); i++) {
+    if (array.IsNull(i)) {
+      RETURN_NOT_OK(AppendNull());
+    } else {
+      RETURN_NOT_OK(Append(numeric_array.Value(i)));
+    }
+  }
+  return Status::OK();
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::AppendNull() {
+  return values_builder_.AppendNull();
+}
+
+Status DictionaryBuilder<NullType>::AppendNull() { return values_builder_.AppendNull(); }
+
+template <typename T>
+Status DictionaryBuilder<T>::DoubleTableSize() {
+#define INNER_LOOP                                                \
+  Scalar value = GetDictionaryValue(static_cast<int64_t>(index)); \
+  int64_t j = HashValue(value) & new_mod_bitmask;
+
+  DOUBLE_TABLE_SIZE(, INNER_LOOP);
+
+  return Status::OK();
+}
+
+template <typename T>
+typename DictionaryBuilder<T>::Scalar DictionaryBuilder<T>::GetDictionaryValue(
+    int64_t index) {
+  const Scalar* data = reinterpret_cast<const Scalar*>(dict_builder_.data()->data());
+  return data[index];
+}
+
+template <>
+const uint8_t* DictionaryBuilder<FixedSizeBinaryType>::GetDictionaryValue(int64_t index) {
+  return dict_builder_.GetValue(index);
+}
+
+template <typename T>
+int64_t DictionaryBuilder<T>::HashValue(const Scalar& value) {
+  return HashUtil::Hash(&value, sizeof(Scalar), 0);
+}
+
+template <>
+int64_t DictionaryBuilder<FixedSizeBinaryType>::HashValue(const Scalar& value) {
+  return HashUtil::Hash(value, byte_width_, 0);
+}
+
+template <typename T>
+bool DictionaryBuilder<T>::SlotDifferent(hash_slot_t index, const Scalar& value) {
+  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
+  return other != value;
+}
+
+template <>
+bool DictionaryBuilder<FixedSizeBinaryType>::SlotDifferent(hash_slot_t index,
+                                                           const Scalar& value) {
+  int32_t width = static_cast<const FixedSizeBinaryType&>(*type_).byte_width();
+  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
+  return memcmp(other, value, width) != 0;
+}
+
+template <typename T>
+Status DictionaryBuilder<T>::AppendDictionary(const Scalar& value) {
+  return dict_builder_.Append(value);
+}
+
+#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                     \
+  template <>                                                                       \
+  WrappedBinary DictionaryBuilder<Type>::GetDictionaryValue(int64_t index) {        \
+    int32_t v_len;                                                                  \
+    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len); \
+    return WrappedBinary(v, v_len);                                                 \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  Status DictionaryBuilder<Type>::AppendDictionary(const WrappedBinary& value) {    \
+    return dict_builder_.Append(value.ptr_, value.length_);                         \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  Status DictionaryBuilder<Type>::AppendArray(const Array& array) {                 \
+    const BinaryArray& binary_array = static_cast<const BinaryArray&>(array);       \
+    WrappedBinary value(nullptr, 0);                                                \
+    for (int64_t i = 0; i < array.length(); i++) {                                  \
+      if (array.IsNull(i)) {                                                        \
+        RETURN_NOT_OK(AppendNull());                                                \
+      } else {                                                                      \
+        value.ptr_ = binary_array.GetValue(i, &value.length_);                      \
+        RETURN_NOT_OK(Append(value));                                               \
+      }                                                                             \
+    }                                                                               \
+    return Status::OK();                                                            \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  int64_t DictionaryBuilder<Type>::HashValue(const WrappedBinary& value) {          \
+    return HashUtil::Hash(value.ptr_, value.length_, 0);                            \
+  }                                                                                 \
+                                                                                    \
+  template <>                                                                       \
+  bool DictionaryBuilder<Type>::SlotDifferent(hash_slot_t index,                    \
+                                              const WrappedBinary& value) {         \
+    int32_t other_length;                                                           \
+    const uint8_t* other_value =                                                    \
+        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);         \
+    return !(other_length == value.length_ &&                                       \
+             0 == memcmp(other_value, value.ptr_, value.length_));                  \
+  }
+
+BINARY_DICTIONARY_SPECIALIZATIONS(StringType);
+BINARY_DICTIONARY_SPECIALIZATIONS(BinaryType);
+
+template class DictionaryBuilder<UInt8Type>;
+template class DictionaryBuilder<UInt16Type>;
+template class DictionaryBuilder<UInt32Type>;
+template class DictionaryBuilder<UInt64Type>;
+template class DictionaryBuilder<Int8Type>;
+template class DictionaryBuilder<Int16Type>;
+template class DictionaryBuilder<Int32Type>;
+template class DictionaryBuilder<Int64Type>;
+template class DictionaryBuilder<Date32Type>;
+template class DictionaryBuilder<Date64Type>;
+template class DictionaryBuilder<Time32Type>;
+template class DictionaryBuilder<Time64Type>;
+template class DictionaryBuilder<TimestampType>;
+template class DictionaryBuilder<FloatType>;
+template class DictionaryBuilder<DoubleType>;
+template class DictionaryBuilder<FixedSizeBinaryType>;
+template class DictionaryBuilder<BinaryType>;
+template class DictionaryBuilder<StringType>;
+
 // ----------------------------------------------------------------------
 // Decimal128Builder
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index e59e166580ae4..ce7b8cd197da3 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -32,6 +32,7 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/hash.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -809,6 +810,158 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder {
   std::vector<std::unique_ptr<ArrayBuilder>> field_builders_;
 };
 
+// ----------------------------------------------------------------------
+// Dictionary builder
+
+namespace internal {
+
+// TODO(ARROW-1176): Use Tensorflow's StringPiece instead of this here.
+struct WrappedBinary {
+  WrappedBinary(const uint8_t* ptr, int32_t length) : ptr_(ptr), length_(length) {}
+
+  const uint8_t* ptr_;
+  int32_t length_;
+};
+
+template <typename T>
+struct DictionaryScalar {
+  using type = typename T::c_type;
+};
+
+template <>
+struct DictionaryScalar<BinaryType> {
+  using type = WrappedBinary;
+};
+
+template <>
+struct DictionaryScalar<StringType> {
+  using type = WrappedBinary;
+};
+
+template <>
+struct DictionaryScalar<FixedSizeBinaryType> {
+  using type = uint8_t const*;
+};
+
+}  // namespace internal
+
+/// \brief Array builder for created encoded DictionaryArray from dense array
+/// data
+template <typename T>
+class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
+ public:
+  using Scalar = typename internal::DictionaryScalar<T>::type;
+
+  ~DictionaryBuilder() {}
+
+  DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
+
+  template <typename T1 = T>
+  explicit DictionaryBuilder(
+      typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool)
+      : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
+
+  /// \brief Append a scalar value
+  Status Append(const Scalar& value);
+
+  /// \brief Append a scalar null value
+  Status AppendNull();
+
+  /// \brief Append a whole dense array to the builder
+  Status AppendArray(const Array& array);
+
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
+
+ protected:
+  Status DoubleTableSize();
+  Scalar GetDictionaryValue(int64_t index);
+  int64_t HashValue(const Scalar& value);
+  bool SlotDifferent(hash_slot_t slot, const Scalar& value);
+  Status AppendDictionary(const Scalar& value);
+
+  std::shared_ptr<Buffer> hash_table_;
+  int32_t* hash_slots_;
+
+  /// Size of the table. Must be a power of 2.
+  int64_t hash_table_size_;
+
+  // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j %
+  // hash_table_size_, but uses far fewer CPU cycles
+  int64_t mod_bitmask_;
+
+  typename TypeTraits<T>::BuilderType dict_builder_;
+  AdaptiveIntBuilder values_builder_;
+  int32_t byte_width_;
+
+  /// Size at which we decide to resize
+  int64_t hash_table_load_threshold_;
+};
+
+template <>
+class ARROW_EXPORT DictionaryBuilder<NullType> : public ArrayBuilder {
+ public:
+  ~DictionaryBuilder();
+
+  DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
+  explicit DictionaryBuilder(MemoryPool* pool);
+
+  /// \brief Append a scalar null value
+  Status AppendNull();
+
+  /// \brief Append a whole dense array to the builder
+  Status AppendArray(const Array& array);
+
+  Status Init(int64_t elements) override;
+  Status Resize(int64_t capacity) override;
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
+
+ protected:
+  AdaptiveIntBuilder values_builder_;
+};
+
+class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
+ public:
+  using DictionaryBuilder::Append;
+  using DictionaryBuilder::DictionaryBuilder;
+
+  Status Append(const uint8_t* value, int32_t length) {
+    return Append(internal::WrappedBinary(value, length));
+  }
+
+  Status Append(const char* value, int32_t length) {
+    return Append(
+        internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
+  }
+
+  Status Append(const std::string& value) {
+    return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
+                                          static_cast<int32_t>(value.size())));
+  }
+};
+
+/// \brief Dictionary array builder with convenience methods for strings
+class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
+ public:
+  using DictionaryBuilder::Append;
+  using DictionaryBuilder::DictionaryBuilder;
+
+  Status Append(const uint8_t* value, int32_t length) {
+    return Append(internal::WrappedBinary(value, length));
+  }
+
+  Status Append(const char* value, int32_t length) {
+    return Append(
+        internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value), length));
+  }
+
+  Status Append(const std::string& value) {
+    return Append(internal::WrappedBinary(reinterpret_cast<const uint8_t*>(value.c_str()),
+                                          static_cast<int32_t>(value.size())));
+  }
+};
+
 // ----------------------------------------------------------------------
 // Helper functions
 
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index 750f1d36ac7d8..1face78bdebfb 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -30,21 +30,13 @@
 #include "arrow/compute/kernel.h"
 #include "arrow/compute/kernels/util-internal.h"
 #include "arrow/util/hash-util.h"
+#include "arrow/util/hash.h"
 
 namespace arrow {
 namespace compute {
 
 namespace {
 
-// Initially 1024 elements
-static constexpr int64_t kInitialHashTableSize = 1 << 10;
-
-typedef int32_t hash_slot_t;
-static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits<int32_t>::max();
-
-// The maximum load factor for the hash table before resizing.
-static constexpr double kMaxHashTableLoad = 0.5;
-
 enum class SIMDMode : char { NOSIMD, SSE4, AVX2 };
 
 #define CHECK_IMPLEMENTED(KERNEL, FUNCNAME, TYPE)                  \
@@ -54,17 +46,6 @@ enum class SIMDMode : char { NOSIMD, SSE4, AVX2 };
     return Status::NotImplemented(ss.str());                       \
   }
 
-Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr<Buffer>* out) {
-  auto hash_table = std::make_shared<PoolBuffer>(pool);
-
-  RETURN_NOT_OK(hash_table->Resize(sizeof(hash_slot_t) * size));
-  int32_t* slots = reinterpret_cast<hash_slot_t*>(hash_table->mutable_data());
-  std::fill(slots, slots + size, kHashSlotEmpty);
-
-  *out = hash_table;
-  return Status::OK();
-}
-
 // This is a slight design concession -- some hash actions have the possibility
 // of failure. Rather than introduce extra error checking into all actions, we
 // will raise an internal exception so that only the actions where errors can
@@ -129,7 +110,7 @@ class HashTable {
 
 Status HashTable::Init(int64_t elements) {
   DCHECK_EQ(elements, BitUtil::NextPower2(elements));
-  RETURN_NOT_OK(NewHashTable(elements, pool_, &hash_table_));
+  RETURN_NOT_OK(internal::NewHashTable(elements, pool_, &hash_table_));
   hash_slots_ = reinterpret_cast<hash_slot_t*>(hash_table_->mutable_data());
   hash_table_size_ = elements;
   hash_table_load_threshold_ =
@@ -238,44 +219,6 @@ struct HashDictionary<Type, enable_if_has_c_type<Type>> {
     }                                                                                    \
   }
 
-#define DOUBLE_TABLE_SIZE(SETUP_CODE, COMPUTE_HASH)                              \
-  do {                                                                           \
-    int64_t new_size = hash_table_size_ * 2;                                     \
-                                                                                 \
-    std::shared_ptr<Buffer> new_hash_table;                                      \
-    RETURN_NOT_OK(NewHashTable(new_size, pool_, &new_hash_table));               \
-    int32_t* new_hash_slots =                                                    \
-        reinterpret_cast<hash_slot_t*>(new_hash_table->mutable_data());          \
-    int64_t new_mod_bitmask = new_size - 1;                                      \
-                                                                                 \
-    SETUP_CODE;                                                                  \
-                                                                                 \
-    for (int i = 0; i < hash_table_size_; ++i) {                                 \
-      hash_slot_t index = hash_slots_[i];                                        \
-                                                                                 \
-      if (index == kHashSlotEmpty) {                                             \
-        continue;                                                                \
-      }                                                                          \
-                                                                                 \
-      COMPUTE_HASH;                                                              \
-      while (kHashSlotEmpty != new_hash_slots[j]) {                              \
-        ++j;                                                                     \
-        if (ARROW_PREDICT_FALSE(j == new_size)) {                                \
-          j = 0;                                                                 \
-        }                                                                        \
-      }                                                                          \
-                                                                                 \
-      new_hash_slots[j] = index;                                                 \
-    }                                                                            \
-                                                                                 \
-    hash_table_ = new_hash_table;                                                \
-    hash_slots_ = reinterpret_cast<hash_slot_t*>(hash_table_->mutable_data());   \
-    hash_table_size_ = new_size;                                                 \
-    hash_table_load_threshold_ =                                                 \
-        static_cast<int64_t>(static_cast<double>(new_size) * kMaxHashTableLoad); \
-    mod_bitmask_ = new_size - 1;                                                 \
-  } while (false)
-
 template <typename Type, typename Action>
 class HashTableKernel<Type, Action, enable_if_has_c_type<Type>> : public HashTable {
  public:
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 29b18a9357ad0..42613d6a5f56b 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -34,6 +34,7 @@ install(FILES
   cpu-info.h
   decimal.h
   hash-util.h
+  hash.h
   key_value_metadata.h
   logging.h
   macros.h
diff --git a/cpp/src/arrow/util/hash.cc b/cpp/src/arrow/util/hash.cc
new file mode 100644
index 0000000000000..94ba524560faf
--- /dev/null
+++ b/cpp/src/arrow/util/hash.cc
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/hash.h"
+
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+
+namespace arrow {
+namespace internal {
+
+Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr<Buffer>* out) {
+  auto hash_table = std::make_shared<PoolBuffer>(pool);
+
+  RETURN_NOT_OK(hash_table->Resize(sizeof(hash_slot_t) * size));
+  int32_t* slots = reinterpret_cast<hash_slot_t*>(hash_table->mutable_data());
+  std::fill(slots, slots + size, kHashSlotEmpty);
+
+  *out = hash_table;
+  return Status::OK();
+}
+
+}  // namespace internal
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/hash.h b/cpp/src/arrow/util/hash.h
new file mode 100644
index 0000000000000..3597342716388
--- /dev/null
+++ b/cpp/src/arrow/util/hash.h
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_HASH_H
+#define ARROW_UTIL_HASH_H
+
+#include <cstdint>
+#include <limits>
+#include <memory>
+
+namespace arrow {
+
+class Buffer;
+class MemoryPool;
+class Status;
+
+typedef int32_t hash_slot_t;
+static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits<int32_t>::max();
+
+// Initially 1024 elements
+static constexpr int kInitialHashTableSize = 1 << 10;
+
+// The maximum load factor for the hash table before resizing.
+static constexpr double kMaxHashTableLoad = 0.5;
+
+namespace internal {
+
+#define DOUBLE_TABLE_SIZE(SETUP_CODE, COMPUTE_HASH)                              \
+  do {                                                                           \
+    int64_t new_size = hash_table_size_ * 2;                                     \
+                                                                                 \
+    std::shared_ptr<Buffer> new_hash_table;                                      \
+    RETURN_NOT_OK(internal::NewHashTable(new_size, pool_, &new_hash_table));     \
+    int32_t* new_hash_slots =                                                    \
+        reinterpret_cast<hash_slot_t*>(new_hash_table->mutable_data());          \
+    int64_t new_mod_bitmask = new_size - 1;                                      \
+                                                                                 \
+    SETUP_CODE;                                                                  \
+                                                                                 \
+    for (int i = 0; i < hash_table_size_; ++i) {                                 \
+      hash_slot_t index = hash_slots_[i];                                        \
+                                                                                 \
+      if (index == kHashSlotEmpty) {                                             \
+        continue;                                                                \
+      }                                                                          \
+                                                                                 \
+      COMPUTE_HASH;                                                              \
+      while (kHashSlotEmpty != new_hash_slots[j]) {                              \
+        ++j;                                                                     \
+        if (ARROW_PREDICT_FALSE(j == new_size)) {                                \
+          j = 0;                                                                 \
+        }                                                                        \
+      }                                                                          \
+                                                                                 \
+      new_hash_slots[j] = index;                                                 \
+    }                                                                            \
+                                                                                 \
+    hash_table_ = new_hash_table;                                                \
+    hash_slots_ = reinterpret_cast<hash_slot_t*>(hash_table_->mutable_data());   \
+    hash_table_size_ = new_size;                                                 \
+    hash_table_load_threshold_ =                                                 \
+        static_cast<int64_t>(static_cast<double>(new_size) * kMaxHashTableLoad); \
+    mod_bitmask_ = new_size - 1;                                                 \
+  } while (false)
+
+Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr<Buffer>* out);
+
+}  // namespace internal
+}  // namespace arrow
+
+#endif  // ARROW_UTIL_HASH_H

From 665da71c7d73feba6f435e96c22d90b6632a8ad1 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 6 Dec 2017 17:00:45 +0100
Subject: [PATCH 1312/1644] ARROW-1890: [Python] Fix mask handling for Date32
 NumPy conversions

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1393 from xhochy/ARROW-1890 and squashes the following commits:

7ef92ea [Uwe L. Korn] ARROW-1890: [Python] Fix mask handling for Date32 NumPy conversions
---
 cpp/src/arrow/python/numpy_to_arrow.cc      | 1 +
 python/pyarrow/tests/test_convert_pandas.py | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 2316a796a21c0..bcaccc73434a4 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -120,6 +120,7 @@ int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
   for (int i = 0; i < length; ++i) {
     if (mask_values[i]) {
       ++null_count;
+      BitUtil::ClearBit(bitmap, i);
     } else {
       BitUtil::SetBit(bitmap, i);
     }
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index a2a1d6b6910fa..a43651ed92c22 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -572,6 +572,15 @@ def test_date_infer(self):
         expected['date'] = pd.to_datetime(df['date'])
         tm.assert_frame_equal(result, expected)
 
+    def test_date_mask(self):
+        arr = np.array([date(2017, 4, 3), date(2017, 4, 4)],
+                       dtype='datetime64[D]')
+        mask = [True, False]
+        result = pa.array(arr, mask=np.array(mask))
+        expected = np.array([None, date(2017, 4, 4)], dtype='datetime64[D]')
+        expected = pa.array(expected, from_pandas=True)
+        assert expected.equals(result)
+
     def test_date_objects_typed(self):
         arr = np.array([
             date(2017, 4, 3),

From aee1a0b586a611514c65ada6cb924ef77a01721e Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Wed, 6 Dec 2017 17:01:19 +0100
Subject: [PATCH 1313/1644] ARROW-1889: [Python] --exclude is not available in
 older git versions

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1392 from xhochy/ARROW-1889 and squashes the following commits:

05a10eb [Korn, Uwe] ARROW-1889: [Python] --exclude is not available in older git versions
---
 python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index b56a4fdcace6c..81115869b8f17 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -398,7 +398,7 @@ def has_ext_modules(foo):
 def parse_version(root):
     from setuptools_scm import version_from_scm
     import setuptools_scm.git
-    describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --exclude 'apache-arrow-js-*'"
+    describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --match 'apache-arrow-[0-9]*'"
     version = setuptools_scm.git.parse(root, describe)
     if not version:
         return version_from_scm(root)

From 712b9d2c98d82b6f3a37ba8a2944bfda6f186994 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 6 Dec 2017 14:10:23 -0500
Subject: [PATCH 1314/1644] ARROW-1784: [Python] Enable zero-copy
 serialization, deserialization of pandas.DataFrame via components

This patch adds a serialization path for pandas.DataFrame (and Series) that decomposes the internal BlockManager into a dictionary structure that can be serialized to the zero-copy component representation from ARROW-1783, and then reconstructed similarly.

The impact of this is that when a DataFrame has no data that requires pickling, the reconstruction is zero-copy. I will post some benchmarks to illustrate the impact of this. The performance improvements are pretty remarkable, nearly 1000x speedup on a large DataFrame.

As some follow-up work, we will need to do more efficient serialization of the different pandas Index types. We should create a new JIRA for this

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1390 from wesm/ARROW-1784 and squashes the following commits:

21adbe7d [Wes McKinney] Do not test with IntervalIndex in pandas < 0.21, since manylinux1 is pinned at 0.20.1
939c02bb [Wes McKinney] Add pandas serialization test for periods, intervals
4b4c776c [Wes McKinney] Code comment, add more serialization docs for pandas / component serialization
1ac073c3 [Wes McKinney] Complete component-based serializer for pandas.DataFrame
6b01746d [Wes McKinney] Begin refactoring
---
 python/doc/source/index.rst                 |  12 +-
 python/doc/source/ipc.rst                   |  77 ++++++
 python/manylinux1/README.md                 |   2 +-
 python/pyarrow/pandas_compat.py             | 184 +++++++++-----
 python/pyarrow/serialization.pxi            |  24 ++
 python/pyarrow/serialization.py             | 121 +++++++---
 python/pyarrow/tests/test_convert_pandas.py | 255 ++++++++++++--------
 7 files changed, 468 insertions(+), 207 deletions(-)

diff --git a/python/doc/source/index.rst b/python/doc/source/index.rst
index b933d2359f720..c35f20be86396 100644
--- a/python/doc/source/index.rst
+++ b/python/doc/source/index.rst
@@ -18,10 +18,14 @@
 Apache Arrow (Python)
 =====================
 
-Arrow is a columnar in-memory analytics layer designed to accelerate big data.
-It houses a set of canonical in-memory representations of flat and hierarchical
-data along with multiple language-bindings for structure manipulation. It also
-provides IPC and common algorithm implementations.
+Apache Arrow is a cross-language development platform for in-memory data. It
+specifies a standardized language-independent columnar memory format for flat
+and hierarchical data, organized for efficient analytic operations on modern
+hardware. It also provides computational libraries and zero-copy streaming
+messaging and interprocess communication.
+
+The Arrow Python bindings have first-class integration with NumPy, pandas, and
+built-in Python objects.
 
 This is the documentation of the Python API of Apache Arrow. For more details
 on the format and other language bindings see
diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index 17fe84e03633d..6842cb5be9f43 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -256,6 +256,83 @@ Lastly, we use this context as an additioanl argument to ``pyarrow.serialize``:
    buf = pa.serialize(val, context=context).to_buffer()
    restored_val = pa.deserialize(buf, context=context)
 
+The ``SerializationContext`` also has convenience methods ``serialize`` and
+``deserialize``, so these are equivalent statements:
+
+.. code-block:: python
+
+   buf = context.serialize(val).to_buffer()
+   restored_val = context.deserialize(buf)
+
+Component-based Serialization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For serializing Python objects containing some number of NumPy arrays, Arrow
+buffers, or other data types, it may be desirable to transport their serialized
+representation without having to produce an intermediate copy using the
+``to_buffer`` method. To motivate this, support we have a list of NumPy arrays:
+
+.. ipython:: python
+
+   import numpy as np
+   data = [np.random.randn(10, 10) for i in range(5)]
+
+The call ``pa.serialize(data)`` does not copy the memory inside each of these
+NumPy arrays. This serialized representation can be then decomposed into a
+dictionary containing a sequence of ``pyarrow.Buffer`` objects containing
+metadata for each array and references to the memory inside the arrays. To do
+this, use the ``to_components`` method:
+
+.. ipython:: python
+
+   serialized = pa.serialize(data)
+   components = serialized.to_components()
+
+The particular details of the output of ``to_components`` are not too
+important. The objects in the ``'data'`` field are ``pyarrow.Buffer`` objects,
+which are zero-copy convertible to Python ``memoryview`` objects:
+
+.. ipython:: python
+
+   memoryview(components['data'][0])
+
+A memoryview can be converted back to a ``Buffer`` with ``pyarrow.frombuffer``:
+
+.. ipython:: python
+
+   mv = memoryview(components['data'][0])
+   buf = pa.frombuffer(mv)
+
+An object can be reconstructed from its component-based representation using
+``deserialize_components``:
+
+.. ipython:: python
+
+   restored_data = pa.deserialize_components(components)
+   restored_data[0]
+
+``deserialize_components`` is also available as a method on
+``SerializationContext`` objects.
+
+Serializing pandas Objects
+--------------------------
+
+We provide a serialization context that has optimized handling of pandas
+objects like ``DataFrame`` and ``Series``. This is the
+``pyarrow.pandas_serialization_context`` member. Combined with component-based
+serialization above, this enables zero-copy transport of pandas DataFrame
+objects not containing any Python objects:
+
+.. ipython:: python
+
+   import pandas as pd
+   df = pd.DataFrame({'a': [1, 2, 3, 4, 5]})
+   context = pa.pandas_serialization_context
+   serialized_df = context.serialize(df)
+   df_components = serialized_df.to_components()
+   original_df = context.deserialize_components(df_components)
+   original_df
+
 Feather Format
 --------------
 
diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md
index a74f7a27b930a..3d462ff2f7213 100644
--- a/python/manylinux1/README.md
+++ b/python/manylinux1/README.md
@@ -37,7 +37,7 @@ git clone ../../ arrow
 # Build the native baseimage
 docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
 # Build the python packages
-docker run --rm -t -i -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
+docker run --shm-size=2g --rm -t -i -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
 # Now the new packages are located in the dist/ folder
 ls -l dist/
 ```
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index a50ef96e79f59..8459ec31bed54 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -20,6 +20,7 @@
 import json
 import re
 
+import pandas.core.internals as _int
 import numpy as np
 import pandas as pd
 
@@ -348,25 +349,85 @@ def get_datetimetz_type(values, dtype, type_):
 
     return values, type_
 
+# ----------------------------------------------------------------------
+# Converting pandas.DataFrame to a dict containing only NumPy arrays or other
+# objects friendly to pyarrow.serialize
 
-def make_datetimetz(tz):
+
+def dataframe_to_serialized_dict(frame):
+    block_manager = frame._data
+
+    blocks = []
+    axes = [ax for ax in block_manager.axes]
+
+    for block in block_manager.blocks:
+        values = block.values
+        block_data = {}
+
+        if isinstance(block, _int.DatetimeTZBlock):
+            block_data['timezone'] = values.tz.zone
+            values = values.values
+        elif isinstance(block, _int.CategoricalBlock):
+            block_data.update(dictionary=values.categories,
+                              ordered=values.ordered)
+            values = values.codes
+
+        block_data.update(
+            placement=block.mgr_locs.as_array,
+            block=values
+        )
+        blocks.append(block_data)
+
+    return {
+        'blocks': blocks,
+        'axes': axes
+    }
+
+
+def serialized_dict_to_dataframe(data):
+    reconstructed_blocks = [_reconstruct_block(block)
+                            for block in data['blocks']]
+
+    block_mgr = _int.BlockManager(reconstructed_blocks, data['axes'])
+    return pd.DataFrame(block_mgr)
+
+
+def _reconstruct_block(item):
+    # Construct the individual blocks converting dictionary types to pandas
+    # categorical types and Timestamps-with-timezones types to the proper
+    # pandas Blocks
+
+    block_arr = item['block']
+    placement = item['placement']
+    if 'dictionary' in item:
+        cat = pd.Categorical.from_codes(block_arr,
+                                        categories=item['dictionary'],
+                                        ordered=item['ordered'])
+        block = _int.make_block(cat, placement=placement,
+                                klass=_int.CategoricalBlock,
+                                fastpath=True)
+    elif 'timezone' in item:
+        dtype = _make_datetimetz(item['timezone'])
+        block = _int.make_block(block_arr, placement=placement,
+                                klass=_int.DatetimeTZBlock,
+                                dtype=dtype, fastpath=True)
+    else:
+        block = _int.make_block(block_arr, placement=placement)
+
+    return block
+
+
+def _make_datetimetz(tz):
     from pyarrow.compat import DatetimeTZDtype
     return DatetimeTZDtype('ns', tz=tz)
 
 
-def backwards_compatible_index_name(raw_name, logical_name):
-    pattern = r'^__index_level_\d+__$'
-    if raw_name == logical_name and re.match(pattern, raw_name) is not None:
-        return None
-    else:
-        return logical_name
+# ----------------------------------------------------------------------
+# Converting pyarrow.Table efficiently to pandas.DataFrame
 
 
 def table_to_blockmanager(options, table, memory_pool, nthreads=1,
                           categoricals=None):
-    import pandas.core.internals as _int
-    import pyarrow.lib as lib
-
     index_columns = []
     columns = []
     column_indexes = []
@@ -405,37 +466,13 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
 
             index_arrays.append(pd.Series(values, dtype=col_pandas.dtype))
             index_names.append(
-                backwards_compatible_index_name(raw_name, logical_name)
+                _backwards_compatible_index_name(raw_name, logical_name)
             )
             block_table = block_table.remove_column(
                 block_table.schema.get_field_index(raw_name)
             )
 
-    # Convert an arrow table to Block from the internal pandas API
-    result = lib.table_to_blocks(options, block_table, nthreads, memory_pool)
-
-    # Construct the individual blocks converting dictionary types to pandas
-    # categorical types and Timestamps-with-timezones types to the proper
-    # pandas Blocks
-    blocks = []
-    for item in result:
-        block_arr = item['block']
-        placement = item['placement']
-        if 'dictionary' in item:
-            cat = pd.Categorical(block_arr,
-                                 categories=item['dictionary'],
-                                 ordered=item['ordered'], fastpath=True)
-            block = _int.make_block(cat, placement=placement,
-                                    klass=_int.CategoricalBlock,
-                                    fastpath=True)
-        elif 'timezone' in item:
-            dtype = make_datetimetz(item['timezone'])
-            block = _int.make_block(block_arr, placement=placement,
-                                    klass=_int.DatetimeTZBlock,
-                                    dtype=dtype, fastpath=True)
-        else:
-            block = _int.make_block(block_arr, placement=placement)
-        blocks.append(block)
+    blocks = _table_to_blocks(options, block_table, nthreads, memory_pool)
 
     # Construct the row index
     if len(index_arrays) > 1:
@@ -477,31 +514,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
 
     # if we're reconstructing the index
     if has_pandas_metadata:
-
-        # Get levels and labels, and provide sane defaults if the index has a
-        # single level to avoid if/else spaghetti.
-        levels = getattr(columns, 'levels', None) or [columns]
-        labels = getattr(columns, 'labels', None) or [
-            pd.RangeIndex(len(level)) for level in levels
-        ]
-
-        # Convert each level to the dtype provided in the metadata
-        levels_dtypes = [
-            (level, col_index.get('numpy_type', level.dtype))
-            for level, col_index in zip_longest(
-                levels, column_indexes, fillvalue={}
-            )
-        ]
-        new_levels = [
-            _level if _level.dtype == _dtype else _level.astype(_dtype)
-            for _level, _dtype in levels_dtypes
-        ]
-
-        columns = pd.MultiIndex(
-            levels=new_levels,
-            labels=labels,
-            names=columns.names
-        )
+        columns = _reconstruct_columns_from_metadata(columns, column_indexes)
 
     # ARROW-1751: flatten a single level column MultiIndex for pandas 0.21.0
     columns = _flatten_single_level_multiindex(columns)
@@ -510,6 +523,55 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
     return _int.BlockManager(blocks, axes)
 
 
+def _backwards_compatible_index_name(raw_name, logical_name):
+    # Part of table_to_blockmanager
+    pattern = r'^__index_level_\d+__$'
+    if raw_name == logical_name and re.match(pattern, raw_name) is not None:
+        return None
+    else:
+        return logical_name
+
+
+def _reconstruct_columns_from_metadata(columns, column_indexes):
+    # Part of table_to_blockmanager
+
+    # Get levels and labels, and provide sane defaults if the index has a
+    # single level to avoid if/else spaghetti.
+    levels = getattr(columns, 'levels', None) or [columns]
+    labels = getattr(columns, 'labels', None) or [
+        pd.RangeIndex(len(level)) for level in levels
+    ]
+
+    # Convert each level to the dtype provided in the metadata
+    levels_dtypes = [
+        (level, col_index.get('numpy_type', level.dtype))
+        for level, col_index in zip_longest(
+            levels, column_indexes, fillvalue={}
+        )
+    ]
+    new_levels = [
+        _level if _level.dtype == _dtype else _level.astype(_dtype)
+        for _level, _dtype in levels_dtypes
+    ]
+
+    return pd.MultiIndex(
+        levels=new_levels,
+        labels=labels,
+        names=columns.names
+    )
+
+
+def _table_to_blocks(options, block_table, nthreads, memory_pool):
+    # Part of table_to_blockmanager
+
+    # Convert an arrow table to Block from the internal pandas API
+    result = pa.lib.table_to_blocks(options, block_table, nthreads,
+                                    memory_pool)
+
+    # Defined above
+    return [_reconstruct_block(item) for item in result]
+
+
 def _flatten_single_level_multiindex(index):
     if isinstance(index, pd.MultiIndex) and index.nlevels == 1:
         levels, = index.levels
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index faf164b3ebd20..cbc5e3b8408e4 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -152,6 +152,30 @@ cdef class SerializationContext:
                     obj.__dict__.update(serialized_obj)
         return obj
 
+    def serialize(self, obj):
+        """
+        Call pyarrow.serialize and pass this SerializationContext
+        """
+        return serialize(obj, context=self)
+
+    def serialize_to(self, object value, sink):
+        """
+        Call pyarrow.serialize_to and pass this SerializationContext
+        """
+        return serialize_to(value, sink, context=self)
+
+    def deserialize(self, what):
+        """
+        Call pyarrow.deserialize and pass this SerializationContext
+        """
+        return deserialize(what, context=self)
+
+    def deserialize_components(self, what):
+        """
+        Call pyarrow.deserialize_components and pass this SerializationContext
+        """
+        return deserialize_components(what, context=self)
+
 
 _default_serialization_context = SerializationContext()
 
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 08e6cce751b53..b6d2b0258bd08 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -43,15 +43,19 @@ def _deserialize_numpy_array_list(data):
     return np.array(data[0], dtype=np.dtype(data[1]))
 
 
-def _serialize_numpy_array_pickle(obj):
-    pickled = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
+def _pickle_to_buffer(x):
+    pickled = pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
     return frombuffer(pickled)
 
 
-def _deserialize_numpy_array_pickle(data):
+def _load_pickle_from_buffer(data):
     return pickle.loads(memoryview(data))
 
 
+_serialize_numpy_array_pickle = _pickle_to_buffer
+_deserialize_numpy_array_pickle = _load_pickle_from_buffer
+
+
 def register_default_serialization_handlers(serialization_context):
 
     # ----------------------------------------------------------------------
@@ -107,38 +111,6 @@ def _deserialize_default_dict(data):
         custom_serializer=_serialize_numpy_array_list,
         custom_deserializer=_deserialize_numpy_array_list)
 
-    # ----------------------------------------------------------------------
-    # Set up serialization for pandas Series and DataFrame
-
-    try:
-        import pandas as pd
-
-        def _serialize_pandas_series(obj):
-            return serialize_pandas(pd.DataFrame({obj.name: obj}))
-
-        def _deserialize_pandas_series(data):
-            deserialized = deserialize_pandas(data)
-            return deserialized[deserialized.columns[0]]
-
-        def _serialize_pandas_dataframe(obj):
-            return serialize_pandas(obj)
-
-        def _deserialize_pandas_dataframe(data):
-            return deserialize_pandas(data)
-
-        serialization_context.register_type(
-            pd.Series, 'pd.Series',
-            custom_serializer=_serialize_pandas_series,
-            custom_deserializer=_deserialize_pandas_series)
-
-        serialization_context.register_type(
-            pd.DataFrame, 'pd.DataFrame',
-            custom_serializer=_serialize_pandas_dataframe,
-            custom_deserializer=_deserialize_pandas_dataframe)
-    except ImportError:
-        # no pandas
-        pass
-
     # ----------------------------------------------------------------------
     # Set up serialization for pytorch tensors
 
@@ -165,8 +137,87 @@ def _deserialize_torch_tensor(data):
 
 register_default_serialization_handlers(_default_serialization_context)
 
+
+# ----------------------------------------------------------------------
+# pandas-specific serialization matters
+
+
 pandas_serialization_context = _default_serialization_context.clone()
 
+
+def _register_pandas_arrow_handlers(context):
+    try:
+        import pandas as pd
+    except ImportError:
+        return
+
+    def _serialize_pandas_series(obj):
+        return serialize_pandas(pd.DataFrame({obj.name: obj}))
+
+    def _deserialize_pandas_series(data):
+        deserialized = deserialize_pandas(data)
+        return deserialized[deserialized.columns[0]]
+
+    def _serialize_pandas_dataframe(obj):
+        return serialize_pandas(obj)
+
+    def _deserialize_pandas_dataframe(data):
+        return deserialize_pandas(data)
+
+    context.register_type(
+        pd.Series, 'pd.Series',
+        custom_serializer=_serialize_pandas_series,
+        custom_deserializer=_deserialize_pandas_series)
+
+    context.register_type(
+        pd.DataFrame, 'pd.DataFrame',
+        custom_serializer=_serialize_pandas_dataframe,
+        custom_deserializer=_deserialize_pandas_dataframe)
+
+
+def _register_custom_pandas_handlers(context):
+    # ARROW-1784, faster path for pandas-only visibility
+
+    try:
+        import pandas as pd
+    except ImportError:
+        return
+
+    import pyarrow.pandas_compat as pdcompat
+
+    def _serialize_pandas_dataframe(obj):
+        return pdcompat.dataframe_to_serialized_dict(obj)
+
+    def _deserialize_pandas_dataframe(data):
+        return pdcompat.serialized_dict_to_dataframe(data)
+
+    def _serialize_pandas_series(obj):
+        return _serialize_pandas_dataframe(pd.DataFrame({obj.name: obj}))
+
+    def _deserialize_pandas_series(data):
+        deserialized = _deserialize_pandas_dataframe(data)
+        return deserialized[deserialized.columns[0]]
+
+    context.register_type(
+        pd.Series, 'pd.Series',
+        custom_serializer=_serialize_pandas_series,
+        custom_deserializer=_deserialize_pandas_series)
+
+    context.register_type(
+        pd.Index, 'pd.Index',
+        custom_serializer=_pickle_to_buffer,
+        custom_deserializer=_load_pickle_from_buffer)
+
+    context.register_type(
+        pd.DataFrame, 'pd.DataFrame',
+        custom_serializer=_serialize_pandas_dataframe,
+        custom_deserializer=_deserialize_pandas_dataframe)
+
+
+_register_pandas_arrow_handlers(_default_serialization_context)
+_register_custom_pandas_handlers(pandas_serialization_context)
+
+
 pandas_serialization_context.register_type(
     np.ndarray, 'np.array',
     custom_serializer=_serialize_numpy_array_pickle,
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index a43651ed92c22..e94ee4608ec5a 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -60,70 +60,73 @@ def _alltypes_example(size=100):
     })
 
 
-class TestPandasConversion(object):
+def _check_pandas_roundtrip(df, expected=None, nthreads=1,
+                            expected_schema=None,
+                            check_dtype=True, schema=None,
+                            preserve_index=False,
+                            as_batch=False):
+    klass = pa.RecordBatch if as_batch else pa.Table
+    table = klass.from_pandas(df, schema=schema,
+                              preserve_index=preserve_index,
+                              nthreads=nthreads)
+
+    result = table.to_pandas(nthreads=nthreads)
+    if expected_schema:
+        assert table.schema.equals(expected_schema)
+    if expected is None:
+        expected = df
+    tm.assert_frame_equal(result, expected, check_dtype=check_dtype,
+                          check_index_type=('equiv' if preserve_index
+                                            else False))
+
+
+def _check_series_roundtrip(s, type_=None):
+    arr = pa.array(s, from_pandas=True, type=type_)
+
+    result = pd.Series(arr.to_pandas(), name=s.name)
+    if patypes.is_timestamp(arr.type) and arr.type.tz is not None:
+        result = (result.dt.tz_localize('utc')
+                  .dt.tz_convert(arr.type.tz))
+
+    tm.assert_series_equal(s, result)
+
+
+def _check_array_roundtrip(values, expected=None, mask=None,
+                           type=None):
+    arr = pa.array(values, from_pandas=True, mask=mask, type=type)
+    result = arr.to_pandas()
+
+    values_nulls = pd.isnull(values)
+    if mask is None:
+        assert arr.null_count == values_nulls.sum()
+    else:
+        assert arr.null_count == (mask | values_nulls).sum()
+
+    if mask is None:
+        tm.assert_series_equal(pd.Series(result), pd.Series(values),
+                               check_names=False)
+    else:
+        expected = pd.Series(np.ma.masked_array(values, mask=mask))
+        tm.assert_series_equal(pd.Series(result), expected,
+                               check_names=False)
+
+
+def _check_array_from_pandas_roundtrip(np_array):
+    arr = pa.array(np_array, from_pandas=True)
+    result = arr.to_pandas()
+    npt.assert_array_equal(result, np_array)
 
-    def setUp(self):
-        pass
-
-    def tearDown(self):
-        pass
-
-    def _check_pandas_roundtrip(self, df, expected=None, nthreads=1,
-                                expected_schema=None,
-                                check_dtype=True, schema=None,
-                                preserve_index=False,
-                                as_batch=False):
-        klass = pa.RecordBatch if as_batch else pa.Table
-        table = klass.from_pandas(df, schema=schema,
-                                  preserve_index=preserve_index,
-                                  nthreads=nthreads)
-
-        result = table.to_pandas(nthreads=nthreads)
-        if expected_schema:
-            assert table.schema.equals(expected_schema)
-        if expected is None:
-            expected = df
-        tm.assert_frame_equal(result, expected, check_dtype=check_dtype,
-                              check_index_type=('equiv' if preserve_index
-                                                else False))
-
-    def _check_series_roundtrip(self, s, type_=None):
-        arr = pa.array(s, from_pandas=True, type=type_)
-
-        result = pd.Series(arr.to_pandas(), name=s.name)
-        if patypes.is_timestamp(arr.type) and arr.type.tz is not None:
-            result = (result.dt.tz_localize('utc')
-                      .dt.tz_convert(arr.type.tz))
-
-        tm.assert_series_equal(s, result)
-
-    def _check_array_roundtrip(self, values, expected=None, mask=None,
-                               type=None):
-        arr = pa.array(values, from_pandas=True, mask=mask, type=type)
-        result = arr.to_pandas()
-
-        values_nulls = pd.isnull(values)
-        if mask is None:
-            assert arr.null_count == values_nulls.sum()
-        else:
-            assert arr.null_count == (mask | values_nulls).sum()
-
-        if mask is None:
-            tm.assert_series_equal(pd.Series(result), pd.Series(values),
-                                   check_names=False)
-        else:
-            expected = pd.Series(np.ma.masked_array(values, mask=mask))
-            tm.assert_series_equal(pd.Series(result), expected,
-                                   check_names=False)
+
+class TestPandasConversion(object):
 
     def test_all_none_objects(self):
         df = pd.DataFrame({'a': [None, None, None]})
-        self._check_pandas_roundtrip(df)
+        _check_pandas_roundtrip(df)
 
     def test_all_none_category(self):
         df = pd.DataFrame({'a': [None, None, None]})
         df['a'] = df['a'].astype('category')
-        self._check_pandas_roundtrip(df)
+        _check_pandas_roundtrip(df)
 
     def test_non_string_columns(self):
         df = pd.DataFrame({0: [1, 2, 3]})
@@ -133,14 +136,14 @@ def test_non_string_columns(self):
     def test_column_index_names_are_preserved(self):
         df = pd.DataFrame({'data': [1, 2, 3]})
         df.columns.names = ['a']
-        self._check_pandas_roundtrip(df, preserve_index=True)
+        _check_pandas_roundtrip(df, preserve_index=True)
 
     def test_multiindex_columns(self):
         columns = pd.MultiIndex.from_arrays([
             ['one', 'two'], ['X', 'Y']
         ])
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
-        self._check_pandas_roundtrip(df, preserve_index=True)
+        _check_pandas_roundtrip(df, preserve_index=True)
 
     def test_multiindex_columns_with_dtypes(self):
         columns = pd.MultiIndex.from_arrays(
@@ -151,11 +154,11 @@ def test_multiindex_columns_with_dtypes(self):
             names=['level_1', 'level_2'],
         )
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
-        self._check_pandas_roundtrip(df, preserve_index=True)
+        _check_pandas_roundtrip(df, preserve_index=True)
 
     def test_integer_index_column(self):
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')])
-        self._check_pandas_roundtrip(df, preserve_index=True)
+        _check_pandas_roundtrip(df, preserve_index=True)
 
     def test_categorical_column_index(self):
         # I *really* hope no one uses category dtypes for single level column
@@ -203,7 +206,7 @@ def test_categorical_row_index(self):
         df['a'] = df.a.astype('category')
         df = df.set_index('a')
 
-        self._check_pandas_roundtrip(df, preserve_index=True)
+        _check_pandas_roundtrip(df, preserve_index=True)
 
     def test_float_no_nulls(self):
         data = {}
@@ -218,7 +221,7 @@ def test_float_no_nulls(self):
 
         df = pd.DataFrame(data)
         schema = pa.schema(fields)
-        self._check_pandas_roundtrip(df, expected_schema=schema)
+        _check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_zero_copy_success(self):
         result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True)
@@ -312,8 +315,8 @@ def test_float_object_nulls(self):
         expected = pd.DataFrame({'floats': pd.to_numeric(arr)})
         field = pa.field('floats', pa.float64())
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(df, expected=expected,
-                                     expected_schema=schema)
+        _check_pandas_roundtrip(df, expected=expected,
+                                expected_schema=schema)
 
     def test_int_object_nulls(self):
         arr = np.array([None, 1, np.int64(3)] * 5, dtype=object)
@@ -321,8 +324,8 @@ def test_int_object_nulls(self):
         expected = pd.DataFrame({'ints': pd.to_numeric(arr)})
         field = pa.field('ints', pa.int64())
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(df, expected=expected,
-                                     expected_schema=schema)
+        _check_pandas_roundtrip(df, expected=expected,
+                                expected_schema=schema)
 
     def test_integer_no_nulls(self):
         data = OrderedDict()
@@ -347,7 +350,7 @@ def test_integer_no_nulls(self):
 
         df = pd.DataFrame(data)
         schema = pa.schema(fields)
-        self._check_pandas_roundtrip(df, expected_schema=schema)
+        _check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_integer_with_nulls(self):
         # pandas requires upcast to float dtype
@@ -395,7 +398,7 @@ def test_boolean_no_nulls(self):
         df = pd.DataFrame({'bools': np.random.randn(num_values) > 0})
         field = pa.field('bools', pa.bool_())
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(df, expected_schema=schema)
+        _check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_boolean_nulls(self):
         # pandas requires upcast to object dtype
@@ -425,7 +428,7 @@ def test_boolean_object_nulls(self):
         df = pd.DataFrame({'bools': arr})
         field = pa.field('bools', pa.bool_())
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(df, expected_schema=schema)
+        _check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_all_nulls_cast_numeric(self):
         arr = np.array([None], dtype=object)
@@ -445,7 +448,7 @@ def test_unicode(self):
         field = pa.field('strings', pa.string())
         schema = pa.schema([field])
 
-        self._check_pandas_roundtrip(df, expected_schema=schema)
+        _check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_bytes_to_binary(self):
         values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan]
@@ -456,7 +459,7 @@ def test_bytes_to_binary(self):
 
         values2 = [b'qux', b'foo', None, b'bar', b'qux', np.nan]
         expected = pd.DataFrame({'strings': values2})
-        self._check_pandas_roundtrip(df, expected)
+        _check_pandas_roundtrip(df, expected)
 
     @pytest.mark.large_memory
     def test_bytes_exceed_2gb(self):
@@ -499,7 +502,7 @@ def test_timestamps_notimezone_no_nulls(self):
         })
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(
+        _check_pandas_roundtrip(
             df,
             expected_schema=schema,
         )
@@ -514,7 +517,7 @@ def test_timestamps_notimezone_nulls(self):
         })
         field = pa.field('datetime64', pa.timestamp('ns'))
         schema = pa.schema([field])
-        self._check_pandas_roundtrip(
+        _check_pandas_roundtrip(
             df,
             expected_schema=schema,
         )
@@ -529,9 +532,9 @@ def test_timestamps_with_timezone(self):
         })
         df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
                             .to_frame())
-        self._check_pandas_roundtrip(df)
+        _check_pandas_roundtrip(df)
 
-        self._check_series_roundtrip(df['datetime64'])
+        _check_series_roundtrip(df['datetime64'])
 
         # drop-in a null and ns instead of ms
         df = pd.DataFrame({
@@ -545,7 +548,7 @@ def test_timestamps_with_timezone(self):
         df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern')
                             .to_frame())
 
-        self._check_pandas_roundtrip(df)
+        _check_pandas_roundtrip(df)
 
     def test_datetime64_to_date32(self):
         # ARROW-1718
@@ -647,13 +650,13 @@ def test_timedelta(self):
 
     def test_column_of_arrays(self):
         df, schema = dataframe_with_arrays()
-        self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
+        _check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
         table = pa.Table.from_pandas(df, schema=schema, preserve_index=False)
         assert table.schema.equals(schema)
 
         for column in df.columns:
             field = schema.field_by_name(column)
-            self._check_array_roundtrip(df[column], type=field.type)
+            _check_array_roundtrip(df[column], type=field.type)
 
     def test_column_of_arrays_to_py(self):
         # Test regression in ARROW-1199 not caught in above test
@@ -674,13 +677,13 @@ def test_column_of_arrays_to_py(self):
 
     def test_column_of_lists(self):
         df, schema = dataframe_with_lists()
-        self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
+        _check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
         table = pa.Table.from_pandas(df, schema=schema, preserve_index=False)
         assert table.schema.equals(schema)
 
         for column in df.columns:
             field = schema.field_by_name(column)
-            self._check_array_roundtrip(df[column], type=field.type)
+            _check_array_roundtrip(df[column], type=field.type)
 
     def test_column_of_lists_chunked(self):
         # ARROW-1357
@@ -732,7 +735,7 @@ def test_column_of_lists_strided(self):
         arr = df['int64'].values[::3]
         assert arr.strides[0] != 8
 
-        self._check_array_roundtrip(arr)
+        _check_array_roundtrip(arr)
 
     def test_nested_lists_all_none(self):
         data = np.array([[None, None], None], dtype=object)
@@ -751,8 +754,8 @@ def test_nested_lists_all_none(self):
 
     def test_threaded_conversion(self):
         df = _alltypes_example()
-        self._check_pandas_roundtrip(df, nthreads=2)
-        self._check_pandas_roundtrip(df, nthreads=2, as_batch=True)
+        _check_pandas_roundtrip(df, nthreads=2)
+        _check_pandas_roundtrip(df, nthreads=2, as_batch=True)
 
     def test_category(self):
         repeats = 5
@@ -770,7 +773,7 @@ def test_category(self):
                            'strings': v1 * repeats,
                            'strings2': v1 * repeats,
                            'strings3': v3 * repeats})
-        self._check_pandas_roundtrip(df)
+        _check_pandas_roundtrip(df)
 
         arrays = [
             pd.Categorical(v1 * repeats),
@@ -778,7 +781,7 @@ def test_category(self):
             pd.Categorical(v3 * repeats)
         ]
         for values in arrays:
-            self._check_array_roundtrip(values)
+            _check_array_roundtrip(values)
 
     def test_mixed_types_fails(self):
         data = pd.DataFrame({'a': ['a', 1, 2.0]})
@@ -825,9 +828,9 @@ def test_strided_data_import(self):
             df = pd.DataFrame(case, columns=columns)
             col = df['a']
 
-            self._check_pandas_roundtrip(df)
-            self._check_array_roundtrip(col)
-            self._check_array_roundtrip(col, mask=strided_mask)
+            _check_pandas_roundtrip(df)
+            _check_array_roundtrip(col)
+            _check_array_roundtrip(col, mask=strided_mask)
 
     def test_decimal_32_from_pandas(self):
         expected = pd.DataFrame({
@@ -987,11 +990,6 @@ def test_arrow_time_to_pandas(self):
 
         tm.assert_frame_equal(df, expected_df)
 
-    def _check_array_from_pandas_roundtrip(self, np_array):
-        arr = pa.array(np_array, from_pandas=True)
-        result = arr.to_pandas()
-        npt.assert_array_equal(result, np_array)
-
     def test_numpy_datetime64_columns(self):
         datetime64_ns = np.array([
                 '2007-07-13T01:23:34.123456789',
@@ -999,7 +997,7 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-13T12:34:56.432539784',
                 '2010-08-13T05:46:57.437699912'],
                 dtype='datetime64[ns]')
-        self._check_array_from_pandas_roundtrip(datetime64_ns)
+        _check_array_from_pandas_roundtrip(datetime64_ns)
 
         datetime64_us = np.array([
                 '2007-07-13T01:23:34.123456',
@@ -1007,7 +1005,7 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-13T12:34:56.432539',
                 '2010-08-13T05:46:57.437699'],
                 dtype='datetime64[us]')
-        self._check_array_from_pandas_roundtrip(datetime64_us)
+        _check_array_from_pandas_roundtrip(datetime64_us)
 
         datetime64_ms = np.array([
                 '2007-07-13T01:23:34.123',
@@ -1015,7 +1013,7 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-13T12:34:56.432',
                 '2010-08-13T05:46:57.437'],
                 dtype='datetime64[ms]')
-        self._check_array_from_pandas_roundtrip(datetime64_ms)
+        _check_array_from_pandas_roundtrip(datetime64_ms)
 
         datetime64_s = np.array([
                 '2007-07-13T01:23:34',
@@ -1023,7 +1021,7 @@ def test_numpy_datetime64_columns(self):
                 '2006-01-13T12:34:56',
                 '2010-08-13T05:46:57'],
                 dtype='datetime64[s]')
-        self._check_array_from_pandas_roundtrip(datetime64_s)
+        _check_array_from_pandas_roundtrip(datetime64_s)
 
     def test_numpy_datetime64_day_unit(self):
         datetime64_d = np.array([
@@ -1032,7 +1030,7 @@ def test_numpy_datetime64_day_unit(self):
                 '2006-01-15',
                 '2010-08-19'],
                 dtype='datetime64[D]')
-        self._check_array_from_pandas_roundtrip(datetime64_d)
+        _check_array_from_pandas_roundtrip(datetime64_d)
 
     def test_all_nones(self):
         def _check_series(s):
@@ -1079,8 +1077,8 @@ def test_partial_schema(self):
             pa.field('c', pa.int64())
         ])
 
-        self._check_pandas_roundtrip(df, schema=partial_schema,
-                                     expected_schema=expected_schema)
+        _check_pandas_roundtrip(df, schema=partial_schema,
+                                expected_schema=expected_schema)
 
     def test_structarray(self):
         ints = pa.array([None, 2, 3], type=pa.int64())
@@ -1115,7 +1113,7 @@ def test_infer_lists(self):
             pa.field('nested_strs', pa.list_(pa.list_(pa.string())))
         ])
 
-        self._check_pandas_roundtrip(df, expected_schema=expected_schema)
+        _check_pandas_roundtrip(df, expected_schema=expected_schema)
 
     def test_infer_numpy_array(self):
         data = OrderedDict([
@@ -1129,7 +1127,7 @@ def test_infer_numpy_array(self):
             pa.field('ints', pa.list_(pa.int64()))
         ])
 
-        self._check_pandas_roundtrip(df, expected_schema=expected_schema)
+        _check_pandas_roundtrip(df, expected_schema=expected_schema)
 
     def test_metadata_with_mixed_types(self):
         df = pd.DataFrame({'data': [b'some_bytes', u'some_unicode']})
@@ -1184,12 +1182,12 @@ def test_table_str_to_categorical(self):
 
     def test_table_batch_empty_dataframe(self):
         df = pd.DataFrame({})
-        self._check_pandas_roundtrip(df)
-        self._check_pandas_roundtrip(df, as_batch=True)
+        _check_pandas_roundtrip(df)
+        _check_pandas_roundtrip(df, as_batch=True)
 
         df2 = pd.DataFrame({}, index=[0, 1, 2])
-        self._check_pandas_roundtrip(df2, preserve_index=True)
-        self._check_pandas_roundtrip(df2, as_batch=True, preserve_index=True)
+        _check_pandas_roundtrip(df2, preserve_index=True)
+        _check_pandas_roundtrip(df2, as_batch=True, preserve_index=True)
 
     def test_array_from_pandas_date_with_mask(self):
         m = np.array([True, False, True])
@@ -1231,6 +1229,51 @@ def test_array_from_pandas_typed_array_with_mask(self, t, data, expected):
                                     type=pa.list_(t())).equals(result)
 
 
+def _fully_loaded_dataframe_example():
+    from distutils.version import LooseVersion
+
+    index = pd.MultiIndex.from_arrays([
+        pd.date_range('2000-01-01', periods=5).repeat(2),
+        np.tile(np.array(['foo', 'bar'], dtype=object), 5)
+    ])
+
+    c1 = pd.date_range('2000-01-01', periods=10)
+    data = {
+        0: c1,
+        1: c1.tz_localize('utc'),
+        2: c1.tz_localize('US/Eastern'),
+        3: c1[::2].tz_localize('utc').repeat(2).astype('category'),
+        4: ['foo', 'bar'] * 5,
+        5: pd.Series(['foo', 'bar'] * 5).astype('category').values,
+        6: [True, False] * 5,
+        7: np.random.randn(10),
+        8: np.random.randint(0, 100, size=10),
+        9: pd.period_range('2013', periods=10, freq='M')
+    }
+
+    if LooseVersion(pd.__version__) >= '0.21':
+        # There is an issue with pickling IntervalIndex in pandas 0.20.x
+        data[10] = pd.interval_range(start=1, freq=1, periods=10)
+
+    return pd.DataFrame(data, index=index)
+
+
+def _check_serialize_components_roundtrip(df):
+    ctx = pa.pandas_serialization_context
+
+    components = ctx.serialize(df).to_components()
+    deserialized = ctx.deserialize_components(components)
+
+    tm.assert_frame_equal(df, deserialized)
+
+
+def test_serialize_deserialize_pandas():
+    # ARROW-1784, serialize and deserialize DataFrame by decomposing
+    # BlockManager
+    df = _fully_loaded_dataframe_example()
+    _check_serialize_components_roundtrip(df)
+
+
 def _pytime_from_micros(val):
     microseconds = val % 1000000
     val //= 1000000

From aad07824a41c4721992ee8ae2ce4a6c3a154e3ec Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 6 Dec 2017 14:11:20 -0500
Subject: [PATCH 1315/1644] ARROW-1857: [Python] Add switch for boost linkage
 with static parquet in wheels

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1395 from xhochy/ARROW-1857 and squashes the following commits:

71d97bc7 [Uwe L. Korn] ARROW-1857: [Python] Add switch for boost linkage with static parquet in wheels
---
 python/CMakeLists.txt | 9 ++++++++-
 python/setup.py       | 4 ++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 00a4ee68737c6..cbbb464d05a76 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -64,6 +64,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_PARQUET_USE_SHARED
     "Rely on parquet shared libraries where relevant"
     ON)
+  option(PYARROW_BOOST_USE_SHARED
+    "Rely on boost shared libraries on linking static parquet"
+    OFF)
   option(PYARROW_BUILD_PLASMA
     "Build the PyArrow Plasma integration"
     OFF)
@@ -306,7 +309,11 @@ if (PYARROW_BUILD_PARQUET)
       parquet_shared)
   else()
     find_package(Thrift)
-    set(Boost_USE_STATIC_LIBS ON)
+    if (PYARROW_BOOST_USE_SHARED)
+      set(Boost_USE_STATIC_LIBS OFF)
+    else()
+      set(Boost_USE_STATIC_LIBS ON)
+    endif()
     find_package(Boost COMPONENTS regex REQUIRED)
     ADD_THIRDPARTY_LIB(boost_regex
       STATIC_LIB ${Boost_REGEX_LIBRARY_RELEASE})
diff --git a/python/setup.py b/python/setup.py
index 81115869b8f17..32e76ab05d836 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -105,6 +105,8 @@ def initialize_options(self):
             os.environ.get('PYARROW_WITH_PARQUET', '0'))
         self.with_static_parquet = strtobool(
             os.environ.get('PYARROW_WITH_STATIC_PARQUET', '0'))
+        self.with_static_boost = strtobool(
+            os.environ.get('PYARROW_WITH_STATIC_BOOST', '1'))
         self.with_plasma = strtobool(
             os.environ.get('PYARROW_WITH_PLASMA', '0'))
         self.bundle_arrow_cpp = strtobool(
@@ -149,6 +151,8 @@ def _run_cmake(self):
             cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
         if self.with_static_parquet:
             cmake_options.append('-DPYARROW_PARQUET_USE_SHARED=off')
+        if not self.with_static_boost:
+            cmake_options.append('-DPYARROW_BOOST_USE_SHARED=on')
 
         if self.with_plasma:
             cmake_options.append('-DPYARROW_BUILD_PLASMA=on')

From fb800135aacd91d17f34ba7e85cbf4e6e52d34e3 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 6 Dec 2017 15:59:42 -0500
Subject: [PATCH 1316/1644] ARROW-1892: [Python] Support binaries in lists

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1394 from xhochy/ARROW-1892 and squashes the following commits:

1dbf4b25 [Uwe L. Korn] ARROW-1892: [Python] Support binaries in lists
---
 cpp/src/arrow/python/arrow_to_pandas.cc |   3 +
 cpp/src/arrow/python/numpy_to_arrow.cc  | 100 ++++++++++++++++++++++++
 python/pyarrow/tests/pandas_examples.py |   8 ++
 3 files changed, 111 insertions(+)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index b1825cbe69d2c..1a1b71125f039 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -100,6 +100,7 @@ static inline bool ListTypeSupported(const DataType& type) {
     case Type::UINT64:
     case Type::FLOAT:
     case Type::DOUBLE:
+    case Type::BINARY:
     case Type::STRING:
     case Type::TIMESTAMP:
       // The above types are all supported.
@@ -695,6 +696,7 @@ class ObjectBlock : public PandasBlock {
         CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP)
         CONVERTLISTSLIKE_CASE(FloatType, FLOAT)
         CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE)
+        CONVERTLISTSLIKE_CASE(BinaryType, BINARY)
         CONVERTLISTSLIKE_CASE(StringType, STRING)
         CONVERTLISTSLIKE_CASE(ListType, LIST)
         default: {
@@ -1642,6 +1644,7 @@ class ArrowDeserializer {
       CONVERTVALUES_LISTSLIKE_CASE(TimestampType, TIMESTAMP)
       CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT)
       CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE)
+      CONVERTVALUES_LISTSLIKE_CASE(BinaryType, BINARY)
       CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING)
       CONVERTVALUES_LISTSLIKE_CASE(Decimal128Type, DECIMAL)
       CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST)
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index bcaccc73434a4..798822c1b895c 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -146,6 +146,52 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
 
 }  // namespace
 
+/// Append as many string objects from NumPy arrays to a `StringBuilder` as we
+/// can fit
+///
+/// \param[in] offset starting offset for appending
+/// \param[out] end_offset ending offset where we stopped appending. Will
+/// be length of arr if fully consumed
+/// \param[out] have_bytes true if we encountered any PyBytes object
+static Status AppendObjectBinaries(PyArrayObject* arr, PyArrayObject* mask,
+                                   int64_t offset, BinaryBuilder* builder,
+                                   int64_t* end_offset, bool* have_bytes) {
+  PyObject* obj;
+
+  Ndarray1DIndexer<PyObject*> objects(arr);
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask != nullptr) {
+    mask_values.Init(mask);
+    have_mask = true;
+  }
+
+  for (; offset < objects.size(); ++offset) {
+    OwnedRef tmp_obj;
+    obj = objects[offset];
+    if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder->AppendNull());
+      continue;
+    } else if (!PyBytes_Check(obj)) {
+      std::stringstream ss;
+      ss << "Error converting to Python objects to bytes: ";
+      RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
+      return Status::Invalid(ss.str());
+    }
+
+    const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
+    if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
+      break;
+    }
+    RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
+  }
+
+  // If we consumed the whole array, this will be the length of arr
+  *end_offset = offset;
+  return Status::OK();
+}
+
 /// Append as many string objects from NumPy arrays to a `StringBuilder` as we
 /// can fit
 ///
@@ -1193,6 +1239,59 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
   return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
 }
 
+template <>
+inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, BinaryType>(
+    const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
+  PyAcquireGIL lock;
+  // TODO: If there are bytes involed, convert to Binary representation
+  bool have_bytes = false;
+
+  Ndarray1DIndexer<uint8_t> mask_values;
+
+  bool have_mask = false;
+  if (mask_ != nullptr) {
+    mask_values.Init(mask_);
+    have_mask = true;
+  }
+
+  auto value_builder = static_cast<BinaryBuilder*>(builder->value_builder());
+
+  auto foreach_item = [&](PyObject* object, bool mask) {
+    if (mask || PandasObjectIsNull(object)) {
+      return builder->AppendNull();
+    } else if (PyArray_Check(object)) {
+      auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
+      RETURN_NOT_OK(builder->Append(true));
+
+      // TODO(uwe): Support more complex numpy array structures
+      RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+
+      int64_t offset = 0;
+      RETURN_NOT_OK(AppendObjectBinaries(numpy_array, nullptr, 0, value_builder, &offset,
+                                         &have_bytes));
+      if (offset < PyArray_SIZE(numpy_array)) {
+        return Status::Invalid("Array cell value exceeded 2GB");
+      }
+      return Status::OK();
+    } else if (PyList_Check(object)) {
+      int64_t size;
+      std::shared_ptr<DataType> inferred_type;
+      RETURN_NOT_OK(builder->Append(true));
+      RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
+      if (inferred_type->id() != Type::NA && inferred_type->id() != Type::BINARY) {
+        std::stringstream ss;
+        ss << inferred_type->ToString() << " cannot be converted to BINARY.";
+        return Status::TypeError(ss.str());
+      }
+      return AppendPySequence(object, size, inferred_type, value_builder);
+    } else {
+      return Status::TypeError("Unsupported Python type for list items");
+    }
+  };
+
+  return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
+}
+
 template <>
 inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
     const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject* list) {
@@ -1267,6 +1366,7 @@ Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type,
     LIST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
     LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
     LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
+    LIST_CASE(BINARY, NPY_OBJECT, BinaryType)
     LIST_CASE(STRING, NPY_OBJECT, StringType)
     case Type::LIST: {
       const auto& list_type = static_cast<const ListType&>(*type);
diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py
index c145e96342668..f11da3c6ce945 100644
--- a/python/pyarrow/tests/pandas_examples.py
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -110,6 +110,14 @@ def dataframe_with_lists(include_index=False):
         [0.],
         np.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.] * 2)[::2],
     ]
+    fields.append(pa.field('bytes_list', pa.list_(pa.binary())))
+    arrays['bytes_list'] = [
+        [b"1", b"f"],
+        None,
+        [b"1"],
+        [b"1", b"2", b"3"],
+        [],
+    ]
     fields.append(pa.field('str_list', pa.list_(pa.string())))
     arrays['str_list'] = [
         [u"1", u"ä"],

From 1d519d8351c0b26ccb3214116eaf3aa684e4bba0 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Wed, 6 Dec 2017 16:40:33 -0500
Subject: [PATCH 1317/1644] ARROW-1487: [C++] Implement casts from List<A> to
 List<B>, where a cast function is defined from any A to B

This closes [ARROW-1487](https://issues.apache.org/jira/browse/ARROW-1487).

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1383 from Licht-T/feature-list-to-list-cast and squashes the following commits:

53a21a5f [Wes McKinney] Fix cpplint warning
0903e5fc [Wes McKinney] Remove unneeded macro
87a00207 [Wes McKinney] Implement ListCastKernel
a8046d08 [Licht-T] Implement child cast kernel checking before cast evaluation
cca90681 [Licht-T] TST: Add test for casting from List<A> to List<B>
058aad07 [Licht-T] ENH: Implement List<A> to List<B> cast
---
 cpp/src/arrow/compute/compute-test.cc | 41 ++++++++++++++++
 cpp/src/arrow/compute/kernels/cast.cc | 67 ++++++++++++++++++++++++++-
 2 files changed, 107 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 84af8f7c6b0a7..3fc15018630da 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -801,6 +801,47 @@ TYPED_TEST(TestDictionaryCast, Basic) {
   this->CheckPass(*plain_array, *dict_array, dict_array->type(), options);
 }*/
 
+TEST_F(TestCast, ListToList) {
+  CastOptions options;
+  std::shared_ptr<Array> offsets;
+
+  vector<int32_t> offsets_values = {0, 1, 2, 5, 7, 7, 8, 10};
+  std::vector<bool> offsets_is_valid = {true, true, true, true, false, true, true, true};
+  ArrayFromVector<Int32Type, int32_t>(offsets_is_valid, offsets_values, &offsets);
+
+  shared_ptr<Array> int32_plain_array =
+      TestBase::MakeRandomArray<typename TypeTraits<Int32Type>::ArrayType>(10, 2);
+  std::shared_ptr<Array> int32_list_array;
+  ASSERT_OK(
+      ListArray::FromArrays(*offsets, *int32_plain_array, pool_, &int32_list_array));
+
+  std::shared_ptr<Array> int64_plain_array;
+  ASSERT_OK(Cast(&this->ctx_, *int32_plain_array, int64(), options, &int64_plain_array));
+  std::shared_ptr<Array> int64_list_array;
+  ASSERT_OK(
+      ListArray::FromArrays(*offsets, *int64_plain_array, pool_, &int64_list_array));
+
+  std::shared_ptr<Array> float64_plain_array;
+  ASSERT_OK(
+      Cast(&this->ctx_, *int32_plain_array, float64(), options, &float64_plain_array));
+  std::shared_ptr<Array> float64_list_array;
+  ASSERT_OK(
+      ListArray::FromArrays(*offsets, *float64_plain_array, pool_, &float64_list_array));
+
+  this->CheckPass(*int32_list_array, *int64_list_array, int64_list_array->type(),
+                  options);
+  this->CheckPass(*int32_list_array, *float64_list_array, float64_list_array->type(),
+                  options);
+  this->CheckPass(*int64_list_array, *int32_list_array, int32_list_array->type(),
+                  options);
+  this->CheckPass(*int64_list_array, *float64_list_array, float64_list_array->type(),
+                  options);
+  this->CheckPass(*float64_list_array, *int32_list_array, int32_list_array->type(),
+                  options);
+  this->CheckPass(*float64_list_array, *int64_list_array, int64_list_array->type(),
+                  options);
+}
+
 // ----------------------------------------------------------------------
 // Dictionary tests
 
diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc
index 465be958cfabf..afa05485f65b4 100644
--- a/cpp/src/arrow/compute/kernels/cast.cc
+++ b/cpp/src/arrow/compute/kernels/cast.cc
@@ -460,6 +460,49 @@ struct CastFunctor<Date32Type, Date64Type> {
   }
 };
 
+// ----------------------------------------------------------------------
+// List to List
+
+class ListCastKernel : public UnaryKernel {
+ public:
+  ListCastKernel(std::unique_ptr<UnaryKernel> child_caster,
+                 const std::shared_ptr<DataType>& out_type)
+      : child_caster_(std::move(child_caster)), out_type_(out_type) {}
+
+  Status Call(FunctionContext* ctx, const Datum& input, Datum* out) override {
+    DCHECK_EQ(Datum::ARRAY, input.kind());
+
+    const ArrayData& in_data = *input.array();
+    DCHECK_EQ(Type::LIST, in_data.type->id());
+    ArrayData* result;
+
+    if (in_data.offset != 0) {
+      return Status::NotImplemented(
+          "Casting sliced lists (non-zero offset) not yet implemented");
+    }
+
+    if (out->kind() == Datum::NONE) {
+      out->value = ArrayData::Make(out_type_, in_data.length);
+    }
+
+    result = out->array().get();
+
+    // Copy buffers from parent
+    result->buffers = in_data.buffers;
+
+    Datum casted_child;
+    RETURN_NOT_OK(child_caster_->Call(ctx, Datum(in_data.child_data[0]), &casted_child));
+    result->child_data.push_back(casted_child.array());
+
+    RETURN_IF_ERROR(ctx);
+    return Status::OK();
+  }
+
+ private:
+  std::unique_ptr<UnaryKernel> child_caster_;
+  std::shared_ptr<DataType> out_type_;
+};
+
 // ----------------------------------------------------------------------
 // Dictionary to other things
 
@@ -895,7 +938,6 @@ GET_CAST_FUNCTION(DATE64_CASES, Date64Type);
 GET_CAST_FUNCTION(TIME32_CASES, Time32Type);
 GET_CAST_FUNCTION(TIME64_CASES, Time64Type);
 GET_CAST_FUNCTION(TIMESTAMP_CASES, TimestampType);
-
 GET_CAST_FUNCTION(DICTIONARY_CASES, DictionaryType);
 
 #define CAST_FUNCTION_CASE(InType)                      \
@@ -903,6 +945,26 @@ GET_CAST_FUNCTION(DICTIONARY_CASES, DictionaryType);
     *kernel = Get##InType##CastFunc(out_type, options); \
     break
 
+namespace {
+
+Status GetListCastFunc(const DataType& in_type, const std::shared_ptr<DataType>& out_type,
+                       const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel) {
+  if (out_type->id() != Type::LIST) {
+    // Kernel will be null
+    return Status::OK();
+  }
+  const DataType& in_value_type = *static_cast<const ListType&>(in_type).value_type();
+  std::shared_ptr<DataType> out_value_type =
+      static_cast<const ListType&>(*out_type).value_type();
+  std::unique_ptr<UnaryKernel> child_caster;
+  RETURN_NOT_OK(GetCastFunction(in_value_type, out_value_type, options, &child_caster));
+  *kernel =
+      std::unique_ptr<UnaryKernel>(new ListCastKernel(std::move(child_caster), out_type));
+  return Status::OK();
+}
+
+}  // namespace
+
 Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>& out_type,
                        const CastOptions& options, std::unique_ptr<UnaryKernel>* kernel) {
   switch (in_type.id()) {
@@ -924,6 +986,9 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr<DataType>&
     CAST_FUNCTION_CASE(Time64Type);
     CAST_FUNCTION_CASE(TimestampType);
     CAST_FUNCTION_CASE(DictionaryType);
+    case Type::LIST:
+      RETURN_NOT_OK(GetListCastFunc(in_type, out_type, options, kernel));
+      break;
     default:
       break;
   }

From ad454e868bcdf8e9ebe161e0050bbea2f1f2a88a Mon Sep 17 00:00:00 2001
From: vkorukanti <venki@dremio.com>
Date: Wed, 6 Dec 2017 18:34:50 -0500
Subject: [PATCH 1318/1644] ARROW-1877: [Java] Fix incorrect equals method in
 JsonStringArrayList

Currently it uses containsAll which could return wrong results.
Ex. e1: [true, true, false], e2: [true, false, false].

Remove the equals method and fallback to super class method
which has the correct implementation.

Author: vkorukanti <venki@dremio.com>

Closes #1380 from vkorukanti/ARROW-1877 and squashes the following commits:

194629ed [vkorukanti] removed `equals` from `JsonStringHashMap`
4732ddf6 [vkorukanti] ARROW-1877: Fix incorrect equals method in JsonStringArrayList
---
 .../vector/util/JsonStringArrayList.java      | 16 ----------
 .../arrow/vector/util/JsonStringHashMap.java  | 31 -------------------
 2 files changed, 47 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
index 480bd76d445b0..b6db29a7fc64e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java
@@ -19,7 +19,6 @@
 package org.apache.arrow.vector.util;
 
 import java.util.ArrayList;
-import java.util.List;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -40,21 +39,6 @@ public JsonStringArrayList(int size) {
     super(size);
   }
 
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj) {
-      return true;
-    }
-    if (obj == null) {
-      return false;
-    }
-    if (!(obj instanceof List)) {
-      return false;
-    }
-    List<?> other = (List<?>) obj;
-    return this.size() == other.size() && this.containsAll(other);
-  }
-
   @Override
   public final String toString() {
     try {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
index 6455389d582b9..cdb44fbeeaf3b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java
@@ -19,7 +19,6 @@
 package org.apache.arrow.vector.util;
 
 import java.util.LinkedHashMap;
-import java.util.Map;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -36,36 +35,6 @@ public class JsonStringHashMap<K, V> extends LinkedHashMap<K, V> {
     mapper = new ObjectMapper();
   }
 
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj) {
-      return true;
-    }
-    if (obj == null) {
-      return false;
-    }
-    if (!(obj instanceof Map)) {
-      return false;
-    }
-    Map<?, ?> other = (Map<?, ?>) obj;
-    if (this.size() != other.size()) {
-      return false;
-    }
-    for (K key : this.keySet()) {
-      if (this.get(key) == null) {
-        if (other.get(key) == null) {
-          continue;
-        } else {
-          return false;
-        }
-      }
-      if (!this.get(key).equals(other.get(key))) {
-        return false;
-      }
-    }
-    return true;
-  }
-
   @Override
   public final String toString() {
     try {

From 7ed4ca7209f8826fbdaf08188e2695803e3043bf Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Wed, 6 Dec 2017 23:18:40 -0500
Subject: [PATCH 1319/1644] ARROW-1867: [Java] Add missing methods to BitVector
 from legacy vector class

After the ValueVector refactor, the BitVector class was missing the methods setToOne and setRangeToOne from the legacy vector class.  This PR adds them back and restores the BItVector tests to use these APIs.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1396 from BryanCutler/java-addmissing-BitVector-methods-ARROW-1867 and squashes the following commits:

16142162 [Bryan Cutler] added methods setToOne and setRangeToOne from legacy BitVector
---
 .../org/apache/arrow/vector/BitVector.java    | 66 +++++++++++++++++++
 .../apache/arrow/vector/BitVectorHelper.java  | 14 ++++
 .../apache/arrow/vector/TestBitVector.java    | 16 ++---
 3 files changed, 87 insertions(+), 9 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index b6f7323a796b4..3887da4a618f0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -444,6 +444,72 @@ public void setSafe(int index, int isSet, int value) {
     set(index, isSet, value);
   }
 
+  /**
+   * Set the element at the given index to one.
+   *
+   * @param index position of element
+   */
+  public void setToOne(int index) {
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    BitVectorHelper.setValidityBitToOne(valueBuffer, index);
+  }
+
+  /**
+   * Same as {@link #setToOne(int)} except that it handles the case when
+   * index is greater than or equal to current value capacity of the vector.
+   *
+   * @param index position of the element
+   */
+  public void setSafeToOne(int index) {
+    handleSafe(index);
+    setToOne(index);
+  }
+
+  /**
+   * Set count bits to 1 in data starting at firstBitIndex
+   *
+   * @param firstBitIndex the index of the first bit to set
+   * @param count         the number of bits to set
+   */
+  public void setRangeToOne(int firstBitIndex, int count) {
+    int startByteIndex = BitVectorHelper.byteIndex(firstBitIndex);
+    final int lastBitIndex = firstBitIndex + count;
+    final int endByteIndex = BitVectorHelper.byteIndex(lastBitIndex);
+    final int startByteBitIndex = BitVectorHelper.bitIndex(firstBitIndex);
+    final int endBytebitIndex = BitVectorHelper.bitIndex(lastBitIndex);
+    if (count < 8 && startByteIndex == endByteIndex) {
+      // handles the case where we don't have a first and a last byte
+      byte bitMask = 0;
+      for (int i = startByteBitIndex; i < endBytebitIndex; ++i) {
+        bitMask |= (byte) (1L << i);
+      }
+      BitVectorHelper.setBitMaskedByte(validityBuffer, startByteIndex, bitMask);
+      BitVectorHelper.setBitMaskedByte(valueBuffer, startByteIndex, bitMask);
+    } else {
+      // fill in first byte (if it's not full)
+      if (startByteBitIndex != 0) {
+        final byte bitMask = (byte) (0xFFL << startByteBitIndex);
+        BitVectorHelper.setBitMaskedByte(validityBuffer, startByteIndex, bitMask);
+        BitVectorHelper.setBitMaskedByte(valueBuffer, startByteIndex, bitMask);
+        ++startByteIndex;
+      }
+
+      // fill in one full byte at a time
+      for (int i = startByteIndex; i < endByteIndex; i++) {
+        validityBuffer.setByte(i, 0xFF);
+        valueBuffer.setByte(i, 0xFF);
+      }
+
+      // fill in the last byte (if it's not full)
+      if (endBytebitIndex != 0) {
+        final int byteIndex = BitVectorHelper.byteIndex(lastBitIndex - endBytebitIndex);
+        final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7));
+        BitVectorHelper.setBitMaskedByte(validityBuffer, byteIndex, bitMask);
+        BitVectorHelper.setBitMaskedByte(valueBuffer, byteIndex, bitMask);
+      }
+    }
+  }
+
 
   /******************************************************************
    *                                                                *
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
index 2d4db85c583bf..8322a1ac8fc40 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java
@@ -198,4 +198,18 @@ public static ArrowBuf loadValidityBuffer(final ArrowFieldNode fieldNode,
 
     return newBuffer;
   }
+
+  /**
+   * Set the byte of the given index in the data buffer by applying a bit mask to
+   * the current byte at that index.
+   *
+   * @param data
+   * @param byteIndex
+   * @param bitMask
+   */
+  static void setBitMaskedByte(ArrowBuf data, int byteIndex, byte bitMask) {
+    byte currentByte = data.getByte(byteIndex);
+    currentByte |= bitMask;
+    data.setByte(byteIndex, currentByte);
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
index 36365fa9d4063..a59e5cdd48fce 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java
@@ -232,7 +232,7 @@ public void testReallocAfterVectorTransfer1() {
 
       for (int i = 0; i < valueCapacity; i++) {
         if ((i & 1) == 1) {
-          vector.set(i, 1);
+          vector.setToOne(i);
         }
       }
 
@@ -246,12 +246,12 @@ public void testReallocAfterVectorTransfer1() {
       }
 
       /* trigger first realloc */
-      vector.setSafe(valueCapacity, 1);
+      vector.setSafeToOne(valueCapacity);
       assertEquals(valueCapacity * 2, vector.getValueCapacity());
 
       for (int i = valueCapacity; i < valueCapacity*2; i++) {
         if ((i & 1) == 1) {
-          vector.set(i, 1);
+          vector.setToOne(i);
         }
       }
 
@@ -265,12 +265,12 @@ public void testReallocAfterVectorTransfer1() {
       }
 
       /* trigger second realloc */
-      vector.setSafe(valueCapacity*2, 1);
+      vector.setSafeToOne(valueCapacity*2);
       assertEquals(valueCapacity * 4, vector.getValueCapacity());
 
       for (int i = valueCapacity*2; i < valueCapacity*4; i++) {
         if ((i & 1) == 1) {
-          vector.set(i, 1);
+          vector.setToOne(i);
         }
       }
 
@@ -291,7 +291,7 @@ public void testReallocAfterVectorTransfer1() {
       assertEquals(valueCapacity * 4, toVector.getValueCapacity());
 
       /* realloc the toVector */
-      toVector.setSafe(valueCapacity * 4, 1);
+      toVector.setSafeToOne(valueCapacity * 4);
 
       for (int i = 0; i < toVector.getValueCapacity(); i++) {
         if (i <= valueCapacity * 4) {
@@ -505,9 +505,7 @@ private void validateRange(int length, int start, int count) {
     try (BitVector bitVector = new BitVector("bits", allocator)) {
       bitVector.reset();
       bitVector.allocateNew(length);
-      for (int i = start; i < start + count; i++) {
-        bitVector.set(i, 1);
-      }
+      bitVector.setRangeToOne(start, count);
       for (int i = 0; i < start; i++) {
         Assert.assertTrue(desc + i, bitVector.isNull(i));
       }

From ad30138a0ec9be3dfb179d1e9425a4502d556085 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 7 Dec 2017 10:02:36 -0500
Subject: [PATCH 1320/1644] ARROW-1891: [Python] Always use NumPy NaT sentinels
 to mark nulls when converting to array

This code path could benefit from some refactoring, but this is enough to get things working properly, and we can refactor later.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1399 from wesm/ARROW-1891 and squashes the following commits:

718c70c8 [Wes McKinney] Null sentinels always possible whether or not from_pandas=True when dealing with datetime64 values
7af1bf54 [Wes McKinney] Failing test case
---
 cpp/src/arrow/python/numpy_to_arrow.cc | 11 ++++++++---
 python/pyarrow/tests/test_array.py     |  8 ++++++++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 798822c1b895c..f21b40ed3c246 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -421,7 +421,11 @@ class NumPyConverter {
     using traits = internal::arrow_traits<ArrowType::type_id>;
 
     const bool null_sentinels_possible =
-        (use_pandas_null_sentinels_ && traits::supports_nulls);
+        // NumPy has a NaT type
+        (ArrowType::type_id == Type::TIMESTAMP || ArrowType::type_id == Type::DATE32) ||
+
+        // Observing pandas's null sentinels
+        ((use_pandas_null_sentinels_ && traits::supports_nulls));
 
     if (mask_ != nullptr || null_sentinels_possible) {
       RETURN_NOT_OK(InitNullBitmap());
@@ -631,8 +635,6 @@ inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* d
 
   auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(dtype_->c_metadata);
   if (dtype_->type_num == NPY_DATETIME) {
-    const int64_t null_count = ValuesToBitmap<NPY_DATETIME>(arr_, null_bitmap_data_);
-
     // If we have inbound datetime64[D] data, this needs to be downcasted
     // separately here from int64_t to int32_t, because this data is not
     // supported in compute::Cast
@@ -642,6 +644,9 @@ inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* d
       Status s = StaticCastBuffer<int64_t, int32_t>(**data, length_, pool_, data);
       RETURN_NOT_OK(s);
     } else {
+      // TODO(wesm): This is redundant, and recomputed in VisitNative()
+      const int64_t null_count = ValuesToBitmap<NPY_DATETIME>(arr_, null_bitmap_data_);
+
       RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_), &input_type));
       if (!input_type->Equals(*type_)) {
         RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count,
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index a4d781a3354b0..92562da14150c 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -495,6 +495,14 @@ def test_array_conversions_no_sentinel_values():
     assert arr3.null_count == 0
 
 
+def test_array_from_numpy_datetimeD():
+    arr = np.array([None, datetime.date(2017, 4, 4)], dtype='datetime64[D]')
+
+    result = pa.array(arr)
+    expected = pa.array([None, datetime.date(2017, 4, 4)], type=pa.date32())
+    assert result.equals(expected)
+
+
 def test_array_from_numpy_ascii():
     arr = np.array(['abcde', 'abc', ''], dtype='|S5')
 

From aaa978b9c072d3c436838ad2a35b0f6bde1891b6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 7 Dec 2017 11:00:56 -0500
Subject: [PATCH 1321/1644] ARROW-1893: [Python] Convert memoryview to bytes
 when loading from pickle in Python 2.7

It seems somewhere in the 2.7.x series, Python 2.7 acquired the ability to load from memoryview. To be on the safe side, we'll always convert memoryview to bytes. Here's a related workaround from IPython:

https://github.com/ipython/ipython_genutils/blob/master/ipython_genutils/py3compat.py#L153

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1398 from wesm/ARROW-1893 and squashes the following commits:

cbe69134 [Wes McKinney] Not all versions of Python 2.7 can load pickles directly from memoryview
---
 python/pyarrow/compat.py             |  4 +++-
 python/pyarrow/serialization.py      | 13 +++++++++----
 python/pyarrow/tests/test_parquet.py | 20 +++++++++++---------
 3 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index 866cbdd96d063..1b19ca0e4029b 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -70,7 +70,7 @@ class Categorical(ClassPlaceholder):
 
 
 if PY2:
-    import cPickle
+    import cPickle as builtin_pickle
 
     try:
         from cdecimal import Decimal
@@ -107,6 +107,8 @@ def frombytes(o):
     def unichar(s):
         return unichr(s)
 else:
+    import pickle as builtin_pickle
+
     unicode_type = str
     def lzip(*x):
         return list(zip(*x))
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index b6d2b0258bd08..3059dfc1bfff2 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -16,18 +16,19 @@
 # under the License.
 
 from collections import OrderedDict, defaultdict
+import six
 import sys
-import pickle
 
 import numpy as np
 
 from pyarrow import serialize_pandas, deserialize_pandas
+from pyarrow.compat import builtin_pickle
 from pyarrow.lib import _default_serialization_context, frombuffer
 
 try:
     import cloudpickle
 except ImportError:
-    cloudpickle = pickle
+    cloudpickle = builtin_pickle
 
 
 # ----------------------------------------------------------------------
@@ -44,12 +45,16 @@ def _deserialize_numpy_array_list(data):
 
 
 def _pickle_to_buffer(x):
-    pickled = pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
+    pickled = builtin_pickle.dumps(x, protocol=builtin_pickle.HIGHEST_PROTOCOL)
     return frombuffer(pickled)
 
 
 def _load_pickle_from_buffer(data):
-    return pickle.loads(memoryview(data))
+    as_memoryview = memoryview(data)
+    if six.PY2:
+        return builtin_pickle.loads(as_memoryview.tobytes())
+    else:
+        return builtin_pickle.loads(as_memoryview)
 
 
 _serialize_numpy_array_pickle = _pickle_to_buffer
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index d17d89e24df5f..2543e7d17befd 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1570,19 +1570,21 @@ def test_backwards_compatible_index_multi_level_some_named():
     tm.assert_frame_equal(result, expected)
 
 
-@pytest.mark.parametrize('precision', range(1, 39))
-def test_decimal_roundtrip(tmpdir, precision):
+def test_decimal_roundtrip(tmpdir):
     num_values = 10
 
     columns = {}
 
-    for scale in range(0, precision + 1):
-        with util.random_seed(0):
-            random_decimal_values = [
-                util.randdecimal(precision, scale) for _ in range(num_values)
-            ]
-        column_name = 'dec_precision_{:d}_scale_{:d}'.format(precision, scale)
-        columns[column_name] = random_decimal_values
+    for precision in range(1, 39):
+        for scale in range(0, precision + 1):
+            with util.random_seed(0):
+                random_decimal_values = [
+                    util.randdecimal(precision, scale)
+                    for _ in range(num_values)
+                ]
+            column_name = ('dec_precision_{:d}_scale_{:d}'
+                           .format(precision, scale))
+            columns[column_name] = random_decimal_values
 
     expected = pd.DataFrame(columns)
     filename = tmpdir.join('decimals.parquet')

From 64af87bdee0626c2bb19e0e0fd097c475c57c8d1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 7 Dec 2017 15:54:24 -0500
Subject: [PATCH 1322/1644] ARROW-1884: [C++] Exclude integration test JSON
 reader/writer classes from public API

These were showing up in our Doxygen docs and may mislead users reading the public API into thinking these classes do something that they do not (they don't read general JSON)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1400 from wesm/ARROW-1884 and squashes the following commits:

9b299144 [Wes McKinney] Move integration JSON reader/writer into internal namespace, do not export symbols
---
 cpp/src/arrow/ipc/ipc-json-test.cc         | 10 ++++---
 cpp/src/arrow/ipc/json-integration-test.cc | 31 +++++++++++-----------
 cpp/src/arrow/ipc/json-internal.cc         |  7 +++--
 cpp/src/arrow/ipc/json-internal.h          |  4 +--
 cpp/src/arrow/ipc/json.cc                  | 13 +++++----
 cpp/src/arrow/ipc/json.h                   |  8 ++++--
 6 files changed, 42 insertions(+), 31 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index e496826f96b9b..12fa4bf3ed7af 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -39,6 +39,7 @@
 
 namespace arrow {
 namespace ipc {
+namespace internal {
 namespace json {
 
 void TestSchemaRoundTrip(const Schema& schema) {
@@ -46,7 +47,7 @@ void TestSchemaRoundTrip(const Schema& schema) {
   rj::Writer<rj::StringBuffer> writer(sb);
 
   writer.StartObject();
-  ASSERT_OK(internal::WriteSchema(schema, &writer));
+  ASSERT_OK(WriteSchema(schema, &writer));
   writer.EndObject();
 
   std::string json_schema = sb.GetString();
@@ -55,7 +56,7 @@ void TestSchemaRoundTrip(const Schema& schema) {
   d.Parse(json_schema);
 
   std::shared_ptr<Schema> out;
-  if (!internal::ReadSchema(d, default_memory_pool(), &out).ok()) {
+  if (!ReadSchema(d, default_memory_pool(), &out).ok()) {
     FAIL() << "Unable to read JSON schema: " << json_schema;
   }
 
@@ -70,7 +71,7 @@ void TestArrayRoundTrip(const Array& array) {
   rj::StringBuffer sb;
   rj::Writer<rj::StringBuffer> writer(sb);
 
-  ASSERT_OK(internal::WriteArray(name, array, &writer));
+  ASSERT_OK(WriteArray(name, array, &writer));
 
   std::string array_as_json = sb.GetString();
 
@@ -82,7 +83,7 @@ void TestArrayRoundTrip(const Array& array) {
   }
 
   std::shared_ptr<Array> out;
-  ASSERT_OK(internal::ReadArray(default_memory_pool(), d, array.type(), &out));
+  ASSERT_OK(ReadArray(default_memory_pool(), d, array.type(), &out));
 
   // std::cout << array_as_json << std::endl;
   CompareArraysDetailed(0, *out, array);
@@ -415,5 +416,6 @@ TEST_P(TestJsonRoundTrip, RoundTrip) {
 INSTANTIATE_TEST_CASE_P(TestJsonRoundTrip, TestJsonRoundTrip, BATCH_CASES());
 
 }  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index f362d97015978..37778fa25166d 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -50,8 +50,7 @@ DEFINE_bool(verbose, true, "Verbose output");
 namespace fs = boost::filesystem;
 
 namespace arrow {
-
-class Buffer;
+namespace ipc {
 
 bool file_exists(const char* path) {
   std::ifstream handle(path);
@@ -73,16 +72,15 @@ static Status ConvertJsonToArrow(const std::string& json_path,
   std::shared_ptr<Buffer> json_buffer;
   RETURN_NOT_OK(in_file->Read(file_size, &json_buffer));
 
-  std::unique_ptr<ipc::JsonReader> reader;
-  RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &reader));
+  std::unique_ptr<internal::json::JsonReader> reader;
+  RETURN_NOT_OK(internal::json::JsonReader::Open(json_buffer, &reader));
 
   if (FLAGS_verbose) {
     std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
   }
 
-  std::shared_ptr<ipc::RecordBatchWriter> writer;
-  RETURN_NOT_OK(
-      ipc::RecordBatchFileWriter::Open(out_file.get(), reader->schema(), &writer));
+  std::shared_ptr<RecordBatchWriter> writer;
+  RETURN_NOT_OK(RecordBatchFileWriter::Open(out_file.get(), reader->schema(), &writer));
 
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
@@ -101,15 +99,15 @@ static Status ConvertArrowToJson(const std::string& arrow_path,
   RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &in_file));
   RETURN_NOT_OK(io::FileOutputStream::Open(json_path, &out_file));
 
-  std::shared_ptr<ipc::RecordBatchFileReader> reader;
-  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file.get(), &reader));
+  std::shared_ptr<RecordBatchFileReader> reader;
+  RETURN_NOT_OK(RecordBatchFileReader::Open(in_file.get(), &reader));
 
   if (FLAGS_verbose) {
     std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
   }
 
-  std::unique_ptr<ipc::JsonWriter> writer;
-  RETURN_NOT_OK(ipc::JsonWriter::Open(reader->schema(), &writer));
+  std::unique_ptr<internal::json::JsonWriter> writer;
+  RETURN_NOT_OK(internal::json::JsonWriter::Open(reader->schema(), &writer));
 
   for (int i = 0; i < reader->num_record_batches(); ++i) {
     std::shared_ptr<RecordBatch> batch;
@@ -134,15 +132,15 @@ static Status ValidateArrowVsJson(const std::string& arrow_path,
   std::shared_ptr<Buffer> json_buffer;
   RETURN_NOT_OK(json_file->Read(file_size, &json_buffer));
 
-  std::unique_ptr<ipc::JsonReader> json_reader;
-  RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &json_reader));
+  std::unique_ptr<internal::json::JsonReader> json_reader;
+  RETURN_NOT_OK(internal::json::JsonReader::Open(json_buffer, &json_reader));
 
   // Construct Arrow reader
   std::shared_ptr<io::ReadableFile> arrow_file;
   RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &arrow_file));
 
-  std::shared_ptr<ipc::RecordBatchFileReader> arrow_reader;
-  RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(arrow_file.get(), &arrow_reader));
+  std::shared_ptr<RecordBatchFileReader> arrow_reader;
+  RETURN_NOT_OK(RecordBatchFileReader::Open(arrow_file.get(), &arrow_reader));
 
   auto json_schema = json_reader->schema();
   auto arrow_schema = arrow_reader->schema();
@@ -399,6 +397,7 @@ TEST_F(TestJSONIntegration, ErrorStates) {
   ASSERT_RAISES(Invalid, RunCommand(json_path, "", "VALIDATE"));
 }
 
+}  // namespace ipc
 }  // namespace arrow
 
 int main(int argc, char** argv) {
@@ -407,7 +406,7 @@ int main(int argc, char** argv) {
   int ret = 0;
 
   if (FLAGS_integration) {
-    arrow::Status result = arrow::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode);
+    arrow::Status result = arrow::ipc::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode);
     if (!result.ok()) {
       std::cout << "Error message: " << result.ToString() << std::endl;
       ret = 1;
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 11a6956ebfd47..4088a8f20e6a0 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -40,8 +40,11 @@
 
 namespace arrow {
 namespace ipc {
-namespace json {
 namespace internal {
+namespace json {
+
+using ::arrow::ipc::DictionaryMemo;
+using ::arrow::ipc::DictionaryTypeMap;
 
 static std::string GetFloatingPrecisionName(FloatingPoint::Precision precision) {
   switch (precision) {
@@ -1463,7 +1466,7 @@ Status ReadArray(MemoryPool* pool, const rj::Value& json_array, const Schema& sc
   return ReadArray(pool, json_array, result->type(), array);
 }
 
-}  // namespace internal
 }  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/json-internal.h b/cpp/src/arrow/ipc/json-internal.h
index 506fe682949e6..92afc14447dc7 100644
--- a/cpp/src/arrow/ipc/json-internal.h
+++ b/cpp/src/arrow/ipc/json-internal.h
@@ -92,8 +92,8 @@ using RjObject = rj::Value::ConstObject;
 
 namespace arrow {
 namespace ipc {
-namespace json {
 namespace internal {
+namespace json {
 
 Status WriteSchema(const Schema& schema, RjWriter* writer);
 Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer);
@@ -111,8 +111,8 @@ Status ReadArray(MemoryPool* pool, const rj::Value& json_obj,
 Status ReadArray(MemoryPool* pool, const rj::Value& json_obj, const Schema& schema,
                  std::shared_ptr<Array>* array);
 
-}  // namespace internal
 }  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index ea2947d5d4c36..394563c53c09d 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -33,6 +33,8 @@ using std::size_t;
 
 namespace arrow {
 namespace ipc {
+namespace internal {
+namespace json {
 
 // ----------------------------------------------------------------------
 // Writer implementation
@@ -45,7 +47,7 @@ class JsonWriter::JsonWriterImpl {
 
   Status Start() {
     writer_->StartObject();
-    RETURN_NOT_OK(json::internal::WriteSchema(*schema_, writer_.get()));
+    RETURN_NOT_OK(json::WriteSchema(*schema_, writer_.get()));
 
     // Record batches
     writer_->Key("batches");
@@ -63,7 +65,7 @@ class JsonWriter::JsonWriterImpl {
 
   Status WriteRecordBatch(const RecordBatch& batch) {
     DCHECK_EQ(batch.num_columns(), schema_->num_fields());
-    return json::internal::WriteRecordBatch(batch, writer_.get());
+    return json::WriteRecordBatch(batch, writer_.get());
   }
 
  private:
@@ -106,7 +108,7 @@ class JsonReader::JsonReaderImpl {
       return Status::IOError("JSON parsing failed");
     }
 
-    RETURN_NOT_OK(json::internal::ReadSchema(doc_, pool_, &schema_));
+    RETURN_NOT_OK(json::ReadSchema(doc_, pool_, &schema_));
 
     auto it = doc_.FindMember("batches");
     RETURN_NOT_ARRAY("batches", it, doc_);
@@ -120,8 +122,7 @@ class JsonReader::JsonReaderImpl {
     DCHECK_LT(i, static_cast<int>(record_batches_->GetArray().Size()))
         << "i out of bounds";
 
-    return json::internal::ReadRecordBatch(record_batches_->GetArray()[i], schema_, pool_,
-                                           batch);
+    return json::ReadRecordBatch(record_batches_->GetArray()[i], schema_, pool_, batch);
   }
 
   std::shared_ptr<Schema> schema() const { return schema_; }
@@ -164,5 +165,7 @@ Status JsonReader::ReadRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) c
   return impl_->ReadRecordBatch(i, batch);
 }
 
+}  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h
index 51f30f0c109f3..674c3745ed413 100644
--- a/cpp/src/arrow/ipc/json.h
+++ b/cpp/src/arrow/ipc/json.h
@@ -34,12 +34,14 @@ class RecordBatch;
 class Schema;
 
 namespace ipc {
+namespace internal {
+namespace json {
 
 /// \class JsonWriter
 /// \brief Write the JSON representation of an Arrow record batch file or stream
 ///
 /// This is used for integration testing
-class ARROW_EXPORT JsonWriter {
+class JsonWriter {
  public:
   ~JsonWriter();
 
@@ -72,7 +74,7 @@ class ARROW_EXPORT JsonWriter {
 /// \brief Read the JSON representation of an Arrow record batch file or stream
 ///
 /// This is used for integration testing
-class ARROW_EXPORT JsonReader {
+class JsonReader {
  public:
   ~JsonReader();
 
@@ -113,6 +115,8 @@ class ARROW_EXPORT JsonReader {
   std::unique_ptr<JsonReaderImpl> impl_;
 };
 
+}  // namespace json
+}  // namespace internal
 }  // namespace ipc
 }  // namespace arrow
 

From 2994815b7274f22f8a14867fcb6e6b061a2412eb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 7 Dec 2017 20:06:47 -0500
Subject: [PATCH 1323/1644] ARROW-1873: [Python] Catch more possible Python/OOM
 errors in to_pandas conversion path

I also ran into a gnarly method dispatching bug ARROW-1904 while working on this. I will address that deprecation in a separate patch

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1404 from wesm/ARROW-1873 and squashes the following commits:

10001c6a [Wes McKinney] Add various Python error checks for possible OOM. More centralized handling of array offsets
---
 cpp/src/arrow/python/arrow_to_pandas.cc | 106 +++++++++++++-----------
 1 file changed, 58 insertions(+), 48 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 1a1b71125f039..08ce37cda6351 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -240,19 +240,14 @@ class PandasBlock {
       block_arr = PyArray_SimpleNewFromDescr(1, block_dims, descr);
     }
 
-    if (block_arr == NULL) {
-      // TODO(wesm): propagating Python exception
-      return Status::OK();
-    }
+    RETURN_IF_PYERROR();
 
     PyArray_ENABLEFLAGS(reinterpret_cast<PyArrayObject*>(block_arr), NPY_ARRAY_OWNDATA);
 
     npy_intp placement_dims[1] = {num_columns_};
     PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
-    if (placement_arr == NULL) {
-      // TODO(wesm): propagating Python exception
-      return Status::OK();
-    }
+
+    RETURN_IF_PYERROR();
 
     block_arr_.reset(block_arr);
     placement_arr_.reset(placement_arr);
@@ -282,12 +277,19 @@ class PandasBlock {
   ARROW_DISALLOW_COPY_AND_ASSIGN(PandasBlock);
 };
 
+template <typename T>
+inline const T* GetPrimitiveValues(const Array& arr) {
+  const auto& prim_arr = static_cast<const PrimitiveArray&>(arr);
+  const T* raw_values = reinterpret_cast<const T*>(prim_arr.values()->data());
+  return raw_values + arr.offset();
+}
+
 template <typename T>
 inline void ConvertIntegerWithNulls(PandasOptions options, const ChunkedArray& data,
                                     double* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
     // Upcast to double, set NaN as appropriate
 
     for (int i = 0; i < arr.length(); ++i) {
@@ -300,8 +302,8 @@ template <typename T>
 inline void ConvertIntegerNoNullsSameType(PandasOptions options, const ChunkedArray& data,
                                           T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
     memcpy(out_values, in_values, sizeof(T) * arr.length());
     out_values += arr.length();
   }
@@ -311,8 +313,8 @@ template <typename InType, typename OutType>
 inline void ConvertIntegerNoNullsCast(PandasOptions options, const ChunkedArray& data,
                                       OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const InType* in_values = GetPrimitiveValues<InType>(arr);
     for (int64_t i = 0; i < arr.length(); ++i) {
       *out_values = in_values[i];
     }
@@ -323,14 +325,13 @@ static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray&
                                       PyObject** out_values) {
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto bool_arr = static_cast<BooleanArray*>(arr.get());
+    const auto& arr = static_cast<const BooleanArray&>(*data.chunk(c));
 
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (bool_arr->IsNull(i)) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (arr.IsNull(i)) {
         Py_INCREF(Py_None);
         *out_values++ = Py_None;
-      } else if (bool_arr->Value(i)) {
+      } else if (arr.Value(i)) {
         // True
         Py_INCREF(Py_True);
         *out_values++ = Py_True;
@@ -347,10 +348,9 @@ static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray&
 static void ConvertBooleanNoNulls(PandasOptions options, const ChunkedArray& data,
                                   uint8_t* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const std::shared_ptr<Array> arr = data.chunk(c);
-    auto bool_arr = static_cast<BooleanArray*>(arr.get());
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      *out_values++ = static_cast<uint8_t>(bool_arr->Value(i));
+    const auto& arr = static_cast<const BooleanArray&>(*data.chunk(c));
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      *out_values++ = static_cast<uint8_t>(arr.Value(i));
     }
   }
 }
@@ -361,17 +361,17 @@ inline Status ConvertBinaryLike(PandasOptions options, const ChunkedArray& data,
   using ArrayType = typename TypeTraits<Type>::ArrayType;
   PyAcquireGIL lock;
   for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = static_cast<ArrayType*>(data.chunk(c).get());
+    const auto& arr = static_cast<const ArrayType&>(*data.chunk(c));
 
     const uint8_t* data_ptr;
     int32_t length;
     const bool has_nulls = data.null_count() > 0;
-    for (int64_t i = 0; i < arr->length(); ++i) {
-      if (has_nulls && arr->IsNull(i)) {
+    for (int64_t i = 0; i < arr.length(); ++i) {
+      if (has_nulls && arr.IsNull(i)) {
         Py_INCREF(Py_None);
         *out_values = Py_None;
       } else {
-        data_ptr = arr->GetValue(i, &length);
+        data_ptr = arr.GetValue(i, &length);
         *out_values = WrapBytes<ArrayType>::Wrap(data_ptr, length);
         if (*out_values == nullptr) {
           PyErr_Clear();
@@ -530,13 +530,25 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
         PyObject* start = PyLong_FromLongLong(arr->value_offset(i) + chunk_offset);
         PyObject* end = PyLong_FromLongLong(arr->value_offset(i + 1) + chunk_offset);
         PyObject* slice = PySlice_New(start, end, NULL);
+        Py_XDECREF(start);
+        Py_XDECREF(end);
+
+        if (ARROW_PREDICT_FALSE(slice == nullptr)) {
+          // Fall out of loop, will return from RETURN_IF_PYERROR
+          break;
+        }
         *out_values = PyObject_GetItem(numpy_array, slice);
-        Py_DECREF(start);
-        Py_DECREF(end);
-        Py_DECREF(slice);
+
+        if (*out_values == nullptr) {
+          // Fall out of loop, will return from RETURN_IF_PYERROR
+          break;
+        }
+
+        Py_XDECREF(slice);
       }
       ++out_values;
     }
+    RETURN_IF_PYERROR();
 
     chunk_offset += arr->values()->length();
   }
@@ -548,14 +560,12 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
 template <typename T>
 inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const T*>(arr.raw_values());
-
-    const uint8_t* valid_bits = arr.null_bitmap_data();
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
 
     if (arr.null_count() > 0) {
       for (int64_t i = 0; i < arr.length(); ++i) {
-        *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i];
+        *out_values++ = arr.IsNull(i) ? na_value : in_values[i];
       }
     } else {
       memcpy(out_values, in_values, sizeof(T) * arr.length());
@@ -568,8 +578,8 @@ template <typename InType, typename OutType>
 inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_value,
                                        OutType* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const InType* in_values = GetPrimitiveValues<InType>(arr);
 
     for (int64_t i = 0; i < arr.length(); ++i) {
       *out_values++ = arr.IsNull(i) ? na_value : static_cast<OutType>(in_values[i]);
@@ -577,11 +587,11 @@ inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_valu
   }
 }
 
-template <typename InType, int64_t SHIFT>
+template <typename T, int64_t SHIFT>
 inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr = static_cast<const PrimitiveArray&>(*data.chunk(c));
-    auto in_values = reinterpret_cast<const InType*>(arr.raw_values());
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
 
     for (int64_t i = 0; i < arr.length(); ++i) {
       *out_values++ = arr.IsNull(i) ? kPandasTimestampNull
@@ -631,7 +641,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
   PyObject* Decimal = Decimal_ref.obj();
 
   for (int c = 0; c < data.num_chunks(); c++) {
-    const auto& arr(static_cast<const arrow::Decimal128Array&>(*data.chunk(c).get()));
+    const auto& arr = static_cast<const arrow::Decimal128Array&>(*data.chunk(c));
 
     for (int64_t i = 0; i < arr.length(); ++i) {
       if (arr.IsNull(i)) {
@@ -856,7 +866,7 @@ class BoolBlock : public PandasBlock {
     uint8_t* out_buffer =
         reinterpret_cast<uint8_t*>(block_data_) + rel_placement * num_rows_;
 
-    ConvertBooleanNoNulls(options_, *col->data().get(), out_buffer);
+    ConvertBooleanNoNulls(options_, *col->data(), out_buffer);
     placement_data_[rel_placement] = abs_placement;
     return Status::OK();
   }
@@ -884,7 +894,7 @@ class DatetimeBlock : public PandasBlock {
     int64_t* out_buffer =
         reinterpret_cast<int64_t*>(block_data_) + rel_placement * num_rows_;
 
-    const ChunkedArray& data = *col.get()->data();
+    const ChunkedArray& data = *col->data();
 
     if (type == Type::DATE32) {
       // Convert from days since epoch to datetime64[ns]
@@ -1089,7 +1099,7 @@ class CategoricalBlock : public PandasBlock {
   Status AllocateNDArrayFromIndices(int npy_type, const PrimitiveArray& indices) {
     npy_intp block_dims[1] = {num_rows_};
 
-    auto in_values = reinterpret_cast<const T*>(indices.raw_values());
+    const T* in_values = GetPrimitiveValues<T>(indices);
     void* data = const_cast<T*>(in_values);
 
     PyAcquireGIL lock;
@@ -1420,6 +1430,7 @@ class ArrowDeserializer {
     PyAcquireGIL lock;
 
     result_ = NewArray1DFromType(col_->type().get(), type, col_->length(), nullptr);
+    RETURN_IF_PYERROR();
     arr_ = reinterpret_cast<PyArrayObject*>(result_);
     return Status::OK();
   }
@@ -1429,8 +1440,7 @@ class ArrowDeserializer {
                                std::shared_ptr<Array> arr) {
     typedef typename internal::arrow_traits<TYPE>::T T;
 
-    const auto& prim_arr = static_cast<const PrimitiveArray&>(*arr);
-    auto in_values = reinterpret_cast<const T*>(prim_arr.raw_values());
+    const T* in_values = GetPrimitiveValues<T>(*arr);
 
     // Zero-Copy. We can pass the data pointer directly to NumPy.
     void* data = const_cast<T*>(in_values);
@@ -1528,8 +1538,8 @@ class ArrowDeserializer {
     constexpr int64_t kShift = traits::npy_shift;
 
     for (int c = 0; c < data_.num_chunks(); c++) {
-      const auto& arr = static_cast<const PrimitiveArray&>(*data_.chunk(c));
-      auto in_values = reinterpret_cast<const c_type*>(arr.raw_values());
+      const auto& arr = *data_.chunk(c);
+      const c_type* in_values = GetPrimitiveValues<c_type>(arr);
 
       for (int64_t i = 0; i < arr.length(); ++i) {
         *out_values++ = arr.IsNull(i) ? na_value : static_cast<T>(in_values[i]) / kShift;

From 543808d5b2b64df008b57efec2ab9057a9bdc723 Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <zsxwing@gmail.com>
Date: Thu, 7 Dec 2017 20:20:07 -0500
Subject: [PATCH 1324/1644] ARROW-1864: [Java] Upgrade Netty to 4.1.17

Upgrade Netty to 4.1.17 since the Netty community will deprecate 4.0.x soon. This PR includes the following changes:
- Bump Netty version.
- Implement new ByteBuf APIs added in Netty 4.1.x: a bunch of get/setXXXLE methods. They are the opposite of get/setXXX method regarding byte order. E.g., as ArrowBuf is little endian, `setInt` will put an `int` to the buffer in little endian byte order, while `setIntLE` will put `int` in big byte endian order. The method naming seems confusing anyway, and I opened a Netty issue: https://github.com/netty/netty/issues/7465. The user can call these new methods to get or set multi-byte integers in big endian byte order.
- Make ArrowByteBufAllocator overwrite AbstractByteBufAllocator.

Author: Shixiong Zhu <zsxwing@gmail.com>

Closes #1376 from zsxwing/ARROW-1864 and squashes the following commits:

96a93e18 [Shixiong Zhu] extend AbstractByteBufAllocator; add javadoc for new methods
bb973335 [Shixiong Zhu] Add comment for calculateNewCapacity
555f88ae [Shixiong Zhu] Add methods back
5e09cca6 [Shixiong Zhu] Upgrade Netty to 4.1.x
---
 .../main/java/io/netty/buffer/ArrowBuf.java   | 199 +++++++++++++++++-
 .../netty/buffer/MutableWrappedByteBuf.java   | 116 +++++++++-
 .../arrow/memory/ArrowByteBufAllocator.java   |  15 +-
 java/pom.xml                                  |   2 +-
 .../arrow/vector/util/MapWithOrdinal.java     |   4 +-
 5 files changed, 320 insertions(+), 16 deletions(-)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index e2bbe35480b66..23f5d65fbb550 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -23,6 +23,7 @@
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
 import java.nio.channels.GatheringByteChannel;
 import java.nio.channels.ScatteringByteChannel;
 import java.nio.charset.Charset;
@@ -493,6 +494,16 @@ public ArrowBuf retain() {
     return retain(1);
   }
 
+  @Override
+  public ByteBuf touch() {
+    return this;
+  }
+
+  @Override
+  public ByteBuf touch(Object hint) {
+    return this;
+  }
+
   @Override
   public long getLong(int index) {
     chk(index, 8);
@@ -505,6 +516,17 @@ public float getFloat(int index) {
     return Float.intBitsToFloat(getInt(index));
   }
 
+  /**
+   * Gets a 64-bit long integer at the specified absolute {@code index} in
+   * this buffer in Big Endian Byte Order.
+   */
+  @Override
+  public long getLongLE(int index) {
+    chk(index, 8);
+    final long v = PlatformDependent.getLong(addr(index));
+    return Long.reverseBytes(v);
+  }
+
   @Override
   public double getDouble(int index) {
     return Double.longBitsToDouble(getLong(index));
@@ -527,6 +549,17 @@ public int getInt(int index) {
     return v;
   }
 
+  /**
+   * Gets a 32-bit integer at the specified absolute {@code index} in
+   * this buffer in Big Endian Byte Order.
+   */
+  @Override
+  public int getIntLE(int index) {
+    chk(index, 4);
+    final int v = PlatformDependent.getInt(addr(index));
+    return Integer.reverseBytes(v);
+  }
+
   @Override
   public int getUnsignedShort(int index) {
     return getShort(index) & 0xFFFF;
@@ -535,10 +568,44 @@ public int getUnsignedShort(int index) {
   @Override
   public short getShort(int index) {
     chk(index, 2);
-    short v = PlatformDependent.getShort(addr(index));
+    final short v = PlatformDependent.getShort(addr(index));
     return v;
   }
 
+  /**
+   * Gets a 16-bit short integer at the specified absolute {@code index} in
+   * this buffer in Big Endian Byte Order.
+   */
+  @Override
+  public short getShortLE(int index) {
+    final short v = PlatformDependent.getShort(addr(index));
+    return Short.reverseBytes(v);
+  }
+
+  /**
+   * Gets an unsigned 24-bit medium integer at the specified absolute
+   * {@code index} in this buffer.
+   */
+  @Override
+  public int getUnsignedMedium(int index) {
+    chk(index, 3);
+    final long addr = addr(index);
+    return (PlatformDependent.getByte(addr) & 0xff) << 16 |
+        (PlatformDependent.getShort(addr + 1) & 0xffff);
+  }
+
+  /**
+   * Gets an unsigned 24-bit medium integer at the specified absolute {@code index} in
+   * this buffer in Big Endian Byte Order.
+   */
+  @Override
+  public int getUnsignedMediumLE(int index) {
+    chk(index, 3);
+    final long addr = addr(index);
+    return (PlatformDependent.getByte(addr) & 0xff) |
+        (Short.reverseBytes(PlatformDependent.getShort(addr + 1)) & 0xffff) << 8;
+  }
+
   @Override
   public ArrowBuf setShort(int index, int value) {
     chk(index, 2);
@@ -546,6 +613,44 @@ public ArrowBuf setShort(int index, int value) {
     return this;
   }
 
+  /**
+   * Sets the specified 16-bit short integer at the specified absolute {@code index}
+   * in this buffer with Big Endian byte order.
+   */
+  @Override
+  public ByteBuf setShortLE(int index, int value) {
+    chk(index, 2);
+    PlatformDependent.putShort(addr(index), Short.reverseBytes((short) value));
+    return this;
+  }
+
+  /**
+   * Sets the specified 24-bit medium integer at the specified absolute
+   * {@code index} in this buffer.
+   */
+  @Override
+  public ByteBuf setMedium(int index, int value) {
+    chk(index, 3);
+    final long addr = addr(index);
+    PlatformDependent.putByte(addr, (byte) (value >>> 16));
+    PlatformDependent.putShort(addr + 1, (short) value);
+    return this;
+  }
+
+
+  /**
+   * Sets the specified 24-bit medium integer at the specified absolute {@code index}
+   * in this buffer with Big Endian byte order.
+   */
+  @Override
+  public ByteBuf setMediumLE(int index, int value) {
+    chk(index, 3);
+    final long addr = addr(index);
+    PlatformDependent.putByte(addr, (byte) value);
+    PlatformDependent.putShort(addr + 1, Short.reverseBytes((short) (value >>> 8)));
+    return this;
+  }
+
   @Override
   public ArrowBuf setInt(int index, int value) {
     chk(index, 4);
@@ -553,6 +658,17 @@ public ArrowBuf setInt(int index, int value) {
     return this;
   }
 
+  /**
+   * Sets the specified 32-bit integer at the specified absolute {@code index}
+   * in this buffer with Big Endian byte order.
+   */
+  @Override
+  public ByteBuf setIntLE(int index, int value) {
+    chk(index, 4);
+    PlatformDependent.putInt(addr(index), Integer.reverseBytes(value));
+    return this;
+  }
+
   @Override
   public ArrowBuf setLong(int index, long value) {
     chk(index, 8);
@@ -560,6 +676,17 @@ public ArrowBuf setLong(int index, long value) {
     return this;
   }
 
+  /**
+   * Sets the specified 64-bit long integer at the specified absolute {@code index}
+   * in this buffer with Big Endian byte order.
+   */
+  @Override
+  public ByteBuf setLongLE(int index, long value) {
+    chk(index, 8);
+    PlatformDependent.putLong(addr(index), Long.reverseBytes(value));
+    return this;
+  }
+
   @Override
   public ArrowBuf setChar(int index, int value) {
     chk(index, 2);
@@ -668,16 +795,46 @@ protected short _getShort(int index) {
     return getShort(index);
   }
 
+  /** @see  {@link #getShortLE(int)} */
+  @Override
+  protected short _getShortLE(int index) {
+    return getShortLE(index);
+  }
+
   @Override
   protected int _getInt(int index) {
     return getInt(index);
   }
 
+  /** @see  {@link #getIntLE(int)} */
+  @Override
+  protected int _getIntLE(int index) {
+    return getIntLE(index);
+  }
+
+  /** @see  {@link #getUnsignedMedium(int)} */
+  @Override
+  protected int _getUnsignedMedium(int index) {
+    return getUnsignedMedium(index);
+  }
+
+  /** @see  {@link #getUnsignedMediumLE(int)} */
+  @Override
+  protected int _getUnsignedMediumLE(int index) {
+    return getUnsignedMediumLE(index);
+  }
+
   @Override
   protected long _getLong(int index) {
     return getLong(index);
   }
 
+  /** @see  {@link #getLongLE(int)} */
+  @Override
+  protected long _getLongLE(int index) {
+    return getLongLE(index);
+  }
+
   @Override
   protected void _setByte(int index, int value) {
     setByte(index, value);
@@ -688,21 +845,45 @@ protected void _setShort(int index, int value) {
     setShort(index, value);
   }
 
+  /** @see  {@link #setShortLE(int, int)} */
+  @Override
+  protected void _setShortLE(int index, int value) {
+    setShortLE(index, value);
+  }
+
   @Override
   protected void _setMedium(int index, int value) {
     setMedium(index, value);
   }
 
+  /** @see  {@link #setMediumLE(int, int)} */
+  @Override
+  protected void _setMediumLE(int index, int value) {
+    setMediumLE(index, value);
+  }
+
   @Override
   protected void _setInt(int index, int value) {
     setInt(index, value);
   }
 
+  /** @see  {@link #setIntLE(int, int)} */
+  @Override
+  protected void _setIntLE(int index, int value) {
+    setIntLE(index, value);
+  }
+
   @Override
   protected void _setLong(int index, long value) {
     setLong(index, value);
   }
 
+  /** @see  {@link #setLongLE(int, long)} */
+  @Override
+  public void _setLongLE(int index, long value) {
+    setLongLE(index, value);
+  }
+
   @Override
   public ArrowBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
     udle.getBytes(index + offset, dst, dstIndex, length);
@@ -716,16 +897,13 @@ public ArrowBuf getBytes(int index, OutputStream out, int length) throws IOExcep
   }
 
   @Override
-  protected int _getUnsignedMedium(int index) {
-    final long addr = addr(index);
-    return (PlatformDependent.getByte(addr) & 0xff) << 16 |
-        (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
-        PlatformDependent.getByte(addr + 2) & 0xff;
+  public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
+    return udle.getBytes(index + offset, out, length);
   }
 
   @Override
-  public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
-    return udle.getBytes(index + offset, out, length);
+  public int getBytes(int index, FileChannel out, long position, int length) throws IOException {
+    return udle.getBytes(index + offset, out, position, length);
   }
 
   @Override
@@ -776,6 +954,11 @@ public int setBytes(int index, ScatteringByteChannel in, int length) throws IOEx
     return udle.setBytes(index + offset, in, length);
   }
 
+  @Override
+  public int setBytes(int index, FileChannel in, long position, int length) throws IOException {
+    return udle.setBytes(index + offset, in, position, length);
+  }
+
   @Override
   public byte getByte(int index) {
     chk(index, 1);
diff --git a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
index a5683adccbc32..f0bc84cdc2db2 100644
--- a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
@@ -23,9 +23,12 @@
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
 import java.nio.channels.GatheringByteChannel;
 import java.nio.channels.ScatteringByteChannel;
 
+import io.netty.util.ByteProcessor;
+
 /**
  * This is basically a complete copy of DuplicatedByteBuf. We copy because we want to override
  * some behaviors and make
@@ -128,6 +131,16 @@ protected short _getShort(int index) {
     return buffer.getShort(index);
   }
 
+  @Override
+  public short getShortLE(int index) {
+    return buffer.getShortLE(index);
+  }
+
+  @Override
+  protected short _getShortLE(int index) {
+    return buffer.getShortLE(index);
+  }
+
   @Override
   public int getUnsignedMedium(int index) {
     return _getUnsignedMedium(index);
@@ -138,6 +151,16 @@ protected int _getUnsignedMedium(int index) {
     return buffer.getUnsignedMedium(index);
   }
 
+  @Override
+  public int getUnsignedMediumLE(int index) {
+    return buffer.getUnsignedMediumLE(index);
+  }
+
+  @Override
+  protected int _getUnsignedMediumLE(int index) {
+    return buffer.getUnsignedMediumLE(index);
+  }
+
   @Override
   public int getInt(int index) {
     return _getInt(index);
@@ -148,6 +171,16 @@ protected int _getInt(int index) {
     return buffer.getInt(index);
   }
 
+  @Override
+  public int getIntLE(int index) {
+    return buffer.getIntLE(index);
+  }
+
+  @Override
+  protected int _getIntLE(int index) {
+    return buffer.getIntLE(index);
+  }
+
   @Override
   public long getLong(int index) {
     return _getLong(index);
@@ -158,6 +191,16 @@ protected long _getLong(int index) {
     return buffer.getLong(index);
   }
 
+  @Override
+  public long getLongLE(int index) {
+    return buffer.getLongLE(index);
+  }
+
+  @Override
+  protected long _getLongLE(int index) {
+    return buffer.getLongLE(index);
+  }
+
   @Override
   public abstract ByteBuf copy(int index, int length);
 
@@ -206,6 +249,17 @@ protected void _setShort(int index, int value) {
     buffer.setShort(index, value);
   }
 
+  @Override
+  public ByteBuf setShortLE(int index, int value) {
+    buffer.setShortLE(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setShortLE(int index, int value) {
+    buffer.setShortLE(index, value);
+  }
+
   @Override
   public ByteBuf setMedium(int index, int value) {
     _setMedium(index, value);
@@ -217,6 +271,17 @@ protected void _setMedium(int index, int value) {
     buffer.setMedium(index, value);
   }
 
+  @Override
+  public ByteBuf setMediumLE(int index, int value) {
+    buffer.setMediumLE(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setMediumLE(int index, int value) {
+    buffer.setMediumLE(index, value);
+  }
+
   @Override
   public ByteBuf setInt(int index, int value) {
     _setInt(index, value);
@@ -228,6 +293,17 @@ protected void _setInt(int index, int value) {
     buffer.setInt(index, value);
   }
 
+  @Override
+  public ByteBuf setIntLE(int index, int value) {
+    buffer.setIntLE(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setIntLE(int index, int value) {
+    buffer.setIntLE(index, value);
+  }
+
   @Override
   public ByteBuf setLong(int index, long value) {
     _setLong(index, value);
@@ -239,6 +315,17 @@ protected void _setLong(int index, long value) {
     buffer.setLong(index, value);
   }
 
+  @Override
+  public ByteBuf setLongLE(int index, long value) {
+    buffer.setLongLE(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setLongLE(int index, long value) {
+    buffer.setLongLE(index, value);
+  }
+
   @Override
   public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
     buffer.setBytes(index, src, srcIndex, length);
@@ -257,6 +344,12 @@ public ByteBuf setBytes(int index, ByteBuffer src) {
     return this;
   }
 
+  @Override
+  public int setBytes(int index, FileChannel in, long position, int length)
+      throws IOException {
+    return buffer.setBytes(index, in, position, length);
+  }
+
   @Override
   public ByteBuf getBytes(int index, OutputStream out, int length)
       throws IOException {
@@ -282,6 +375,13 @@ public int setBytes(int index, ScatteringByteChannel in, int length)
     return buffer.setBytes(index, in, length);
   }
 
+
+  @Override
+  public int getBytes(int index, FileChannel out, long position, int length)
+      throws IOException {
+    return buffer.getBytes(index, out, position, length);
+  }
+
   @Override
   public int nioBufferCount() {
     return buffer.nioBufferCount();
@@ -298,12 +398,12 @@ public ByteBuffer internalNioBuffer(int index, int length) {
   }
 
   @Override
-  public int forEachByte(int index, int length, ByteBufProcessor processor) {
+  public int forEachByte(int index, int length, ByteProcessor processor) {
     return buffer.forEachByte(index, length, processor);
   }
 
   @Override
-  public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
+  public int forEachByteDesc(int index, int length, ByteProcessor processor) {
     return buffer.forEachByteDesc(index, length, processor);
   }
 
@@ -312,6 +412,18 @@ public final int refCnt() {
     return unwrap().refCnt();
   }
 
+  @Override
+  public final ByteBuf touch() {
+    unwrap().touch();
+    return this;
+  }
+
+  @Override
+  public final ByteBuf touch(Object hint) {
+    unwrap().touch(hint);
+    return this;
+  }
+
   @Override
   public final ByteBuf retain() {
     unwrap().retain();
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
index b8b5283423c82..94102992139d8 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
@@ -18,8 +18,8 @@
 
 package org.apache.arrow.memory;
 
+import io.netty.buffer.AbstractByteBufAllocator;
 import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
 import io.netty.buffer.CompositeByteBuf;
 import io.netty.buffer.ExpandableByteBuf;
 
@@ -32,7 +32,7 @@
  * otherwise non-expandable
  * ArrowBufs to be expandable.
  */
-public class ArrowByteBufAllocator implements ByteBufAllocator {
+public class ArrowByteBufAllocator extends AbstractByteBufAllocator {
 
   private static final int DEFAULT_BUFFER_SIZE = 4096;
   private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16;
@@ -142,8 +142,17 @@ public CompositeByteBuf compositeHeapBuffer(int maxNumComponents) {
     throw fail();
   }
 
+  @Override
+  protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
+    throw fail();
+  }
+
+  @Override
+  protected ByteBuf newDirectBuffer(int initialCapacity, int maxCapacity) {
+    return buffer(initialCapacity, maxCapacity);
+  }
+
   private RuntimeException fail() {
     throw new UnsupportedOperationException("Allocator doesn't support heap-based memory.");
   }
-
 }
diff --git a/java/pom.xml b/java/pom.xml
index c479d651f6b5f..162c53460cc93 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -32,7 +32,7 @@
     <dep.junit.version>4.11</dep.junit.version>
     <dep.slf4j.version>1.7.25</dep.slf4j.version>
     <dep.guava.version>18.0</dep.guava.version>
-    <dep.netty.version>4.0.49.Final</dep.netty.version>
+    <dep.netty.version>4.1.17.Final</dep.netty.version>
     <dep.jackson.version>2.7.9</dep.jackson.version>
     <dep.hadoop.version>2.7.1</dep.hadoop.version>
     <fbs.version>1.2.0-3f79e055</fbs.version>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
index 6d3b390379a56..b863fa8af86fd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -134,9 +134,9 @@ public Set<K> keySet() {
 
     @Override
     public Collection<V> values() {
-      return Lists.newArrayList(Iterables.transform(secondary.entries(), new Function<IntObjectMap.Entry<V>, V>() {
+      return Lists.newArrayList(Iterables.transform(secondary.entries(), new Function<IntObjectMap.PrimitiveEntry<V>, V>() {
         @Override
-        public V apply(IntObjectMap.Entry<V> entry) {
+        public V apply(IntObjectMap.PrimitiveEntry<V> entry) {
           return Preconditions.checkNotNull(entry).value();
         }
       }));

From fe33155099e9893470fc938b458b92d776983ef0 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 8 Dec 2017 11:41:45 +0100
Subject: [PATCH 1325/1644] ARROW-1901: [Python] Support recursive mkdir for
 DaskFilesystem

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1401 from xhochy/ARROW-1901 and squashes the following commits:

4a86b4b [Uwe L. Korn] ARROW-1901: [Python] Support recursive mkdir for DaskFilesystem
---
 python/pyarrow/filesystem.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index 926df0e30e565..ff7809575620a 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -275,8 +275,11 @@ def exists(self, path):
         return self.fs.exists(path)
 
     @implements(FileSystem.mkdir)
-    def mkdir(self, path):
-        return self.fs.mkdir(path)
+    def mkdir(self, path, create_parents=True):
+        if create_parents:
+            return self.fs.mkdirs(path)
+        else:
+            return self.fs.mkdir(path)
 
     @implements(FileSystem.open)
     def open(self, path, mode='rb'):

From fcf46d9d947eb3a31c63be3e65118cd69cdd2976 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 8 Dec 2017 11:42:19 +0100
Subject: [PATCH 1326/1644] =?UTF-8?q?ARROW-1902:=20[Python]=C2=A0Remove=20?=
 =?UTF-8?q?mkdir=20race=20condition=20from=20write=5Fto=5Fdataset?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1402 from xhochy/ARROW-1902 and squashes the following commits:

199e101 [Uwe L. Korn] ARROW-1902: [Python] Remove mkdir race condition from write_to_dataset
---
 python/pyarrow/parquet.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 9fb890c6e6b8d..d9f1bd2c36430 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -1013,7 +1013,10 @@ def write_to_dataset(table, root_path, partition_cols=None,
         fs = _ensure_filesystem(filesystem)
 
     if fs._isfilestore() and not fs.exists(root_path):
-        fs.mkdir(root_path)
+        try:
+            fs.mkdir(root_path)
+        except OSError:
+            assert fs.exists(root_path)
 
     if partition_cols is not None and len(partition_cols) > 0:
         df = table.to_pandas()

From 74b2ebed6e19f21680d7b49a43fc0fa101c7b6fb Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sat, 9 Dec 2017 23:39:49 -0500
Subject: [PATCH 1327/1644] ARROW-1908: [Python] Construction of arrow table
 from pandas DataFrame with duplicate column names crashes

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1405 from cpcloud/ARROW-1908 and squashes the following commits:

86b2b21e [Phillip Cloud] ARROW-1908: [Python] Construction of arrow table from pandas DataFrame with duplicate column names crashes
---
 cpp/src/arrow/python/builtin_convert.cc     | 32 +++++++++++++--------
 python/pyarrow/pandas_compat.py             |  6 ++++
 python/pyarrow/tests/test_convert_pandas.py |  5 ++++
 3 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 08cbae7ab2da4..cd88d557d4830 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -148,15 +148,14 @@ static constexpr int MAX_NESTING_LEVELS = 32;
 // SeqVisitor is used to infer the type.
 class SeqVisitor {
  public:
-  SeqVisitor() : max_nesting_level_(0), max_observed_level_(0) {
-    memset(nesting_histogram_, 0, MAX_NESTING_LEVELS * sizeof(int));
+  SeqVisitor() : max_nesting_level_(0), max_observed_level_(0), nesting_histogram_() {
+    std::fill(nesting_histogram_, nesting_histogram_ + MAX_NESTING_LEVELS, 0);
   }
 
   // co-recursive with VisitElem
   Status Visit(PyObject* obj, int level = 0) {
-    if (level > max_nesting_level_) {
-      max_nesting_level_ = level;
-    }
+    max_nesting_level_ = std::max(max_nesting_level_, level);
+
     // Loop through either a sequence or an iterator.
     if (PySequence_Check(obj)) {
       Py_ssize_t size = PySequence_Size(obj);
@@ -165,18 +164,26 @@ class SeqVisitor {
         if (PyArray_Check(obj)) {
           auto array = reinterpret_cast<PyArrayObject*>(obj);
           auto ptr = reinterpret_cast<const char*>(PyArray_GETPTR1(array, i));
+
           ref.reset(PyArray_GETITEM(array, ptr));
+          RETURN_IF_PYERROR();
+
           RETURN_NOT_OK(VisitElem(ref, level));
         } else {
           ref.reset(PySequence_GetItem(obj, i));
+          RETURN_IF_PYERROR();
           RETURN_NOT_OK(VisitElem(ref, level));
         }
       }
     } else if (PyObject_HasAttrString(obj, "__iter__")) {
-      OwnedRef iter = OwnedRef(PyObject_GetIter(obj));
-      PyObject* item;
+      OwnedRef iter(PyObject_GetIter(obj));
+      RETURN_IF_PYERROR();
+
+      PyObject* item = NULLPTR;
       while ((item = PyIter_Next(iter.obj()))) {
-        OwnedRef ref = OwnedRef(item);
+        RETURN_IF_PYERROR();
+
+        OwnedRef ref(item);
         RETURN_NOT_OK(VisitElem(ref, level));
       }
     } else {
@@ -242,6 +249,7 @@ class SeqVisitor {
 
   // Visits a specific element (inner part of the loop).
   Status VisitElem(const OwnedRef& item_ref, int level) {
+    DCHECK_NE(item_ref.obj(), NULLPTR);
     if (PyList_Check(item_ref.obj())) {
       RETURN_NOT_OK(Visit(item_ref.obj(), level + 1));
     } else if (PyDict_Check(item_ref.obj())) {
@@ -323,7 +331,7 @@ class SeqConverter {
 
   virtual Status AppendData(PyObject* seq, int64_t size) = 0;
 
-  virtual ~SeqConverter() {}
+  virtual ~SeqConverter() = default;
 
  protected:
   ArrayBuilder* builder_;
@@ -459,13 +467,13 @@ class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter
  public:
   Status AppendItem(const OwnedRef& item) {
     const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+    RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint8_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
     }
-    RETURN_IF_PYERROR();
     return typed_builder_->Append(static_cast<uint8_t>(val));
   }
 };
@@ -474,13 +482,13 @@ class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Conver
  public:
   Status AppendItem(const OwnedRef& item) {
     const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+    RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint16_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
     }
-    RETURN_IF_PYERROR();
     return typed_builder_->Append(static_cast<uint16_t>(val));
   }
 };
@@ -489,13 +497,13 @@ class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Conver
  public:
   Status AppendItem(const OwnedRef& item) {
     const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+    RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint32_t>::max())) {
       return Status::Invalid(
           "Cannot coerce values to array type that would "
           "lose data");
     }
-    RETURN_IF_PYERROR();
     return typed_builder_->Append(static_cast<uint32_t>(val));
   }
 };
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 8459ec31bed54..48384ad478251 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -281,6 +281,12 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1):
 
     columns_to_convert = []
     convert_types = []
+
+    if not df.columns.is_unique:
+        raise ValueError(
+            'Duplicate column names found: {}'.format(list(df.columns))
+        )
+
     for name in df.columns:
         col = df[name]
         if not isinstance(name, six.string_types):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index e94ee4608ec5a..1231844e357d1 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -227,6 +227,11 @@ def test_zero_copy_success(self):
         result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True)
         npt.assert_array_equal(result, [0, 1, 2])
 
+    def test_duplicate_column_names_does_not_crash(self):
+        df = pd.DataFrame([(1, 'a'), (2, 'b')], columns=list('aa'))
+        with pytest.raises(ValueError):
+            pa.Table.from_pandas(df)
+
     def test_dictionary_indices_boundscheck(self):
         # ARROW-1658. No validation of indices leads to segfaults in pandas
         indices = [[0, 1], [0, -1]]

From 501d60e918bd4d10c429ab34e0b8e8a87dffb732 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 10 Dec 2017 19:33:55 +0100
Subject: [PATCH 1328/1644] ARROW-1895/ARROW-1897: [Python] Add field_name to
 pandas index metadata

Author: Phillip Cloud <cpcloud@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1397 from cpcloud/ARROW-1895 and squashes the following commits:

1293b24 [Wes McKinney] Extra metadata is None in py2
3f7760f [Phillip Cloud] Fix py2 test
3bc30fd [Phillip Cloud] Operator precedence
891671b [Phillip Cloud] Unicode bytes difference
3c41905 [Phillip Cloud] Cleaner construction
cf52001 [Phillip Cloud] Use field_name to map arrow table field names to pandas names
37dca10 [Phillip Cloud] Fix string vs unicode in column_indexes and add field_name as well
f570871 [Phillip Cloud] Use categorical codes instead of object
20bf15a [Phillip Cloud] ARROW-1895: [Python] Add field_name to pandas index metadata
---
 python/pyarrow/pandas_compat.py             | 87 +++++++++++++++------
 python/pyarrow/tests/test_convert_pandas.py | 65 +++++++++++++--
 2 files changed, 124 insertions(+), 28 deletions(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 48384ad478251..668048fd645dd 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -110,7 +110,11 @@ def get_logical_type_from_numpy(pandas_collection):
     except KeyError:
         if hasattr(pandas_collection.dtype, 'tz'):
             return 'datetimetz'
-        return infer_dtype(pandas_collection)
+        result = infer_dtype(pandas_collection)
+
+        if result == 'string':
+            return 'bytes' if PY2 else 'unicode'
+        return result
 
 
 def get_extension_dtype_info(column):
@@ -122,7 +126,7 @@ def get_extension_dtype_info(column):
             'num_categories': len(cats.categories),
             'ordered': cats.ordered,
         }
-        physical_dtype = 'object'
+        physical_dtype = str(cats.codes.dtype)
     elif hasattr(dtype, 'tz'):
         metadata = {'timezone': str(dtype.tz)}
         physical_dtype = 'datetime64[ns]'
@@ -132,7 +136,7 @@ def get_extension_dtype_info(column):
     return physical_dtype, metadata
 
 
-def get_column_metadata(column, name, arrow_type):
+def get_column_metadata(column, name, arrow_type, field_name):
     """Construct the metadata for a given column
 
     Parameters
@@ -140,6 +144,10 @@ def get_column_metadata(column, name, arrow_type):
     column : pandas.Series or pandas.Index
     name : str
     arrow_type : pyarrow.DataType
+    field_name : str
+        Equivalent to `name` when `column` is a `Series`, otherwise if `column`
+        is a pandas Index then `field_name` will not be the same as `name`.
+        This is the name of the field in the arrow Table's schema.
 
     Returns
     -------
@@ -164,6 +172,7 @@ def get_column_metadata(column, name, arrow_type):
 
     return {
         'name': name,
+        'field_name': str(field_name),
         'pandas_type': logical_type,
         'numpy_type': string_dtype,
         'metadata': extra_metadata,
@@ -193,10 +202,14 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
     index_types = types[ncolumns - len(index_levels):]
 
     column_metadata = [
-        get_column_metadata(df[col_name], name=sanitized_name,
-                            arrow_type=arrow_type)
-        for col_name, sanitized_name, arrow_type in
-        zip(df.columns, column_names, df_types)
+        get_column_metadata(
+            df[col_name],
+            name=sanitized_name,
+            arrow_type=arrow_type,
+            field_name=sanitized_name
+        ) for col_name, sanitized_name, arrow_type in zip(
+            df.columns, column_names, df_types
+        )
     ]
 
     if preserve_index:
@@ -204,9 +217,13 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
             index_level_name, range(len(index_levels))
         ))
         index_column_metadata = [
-            get_column_metadata(level, name=level.name, arrow_type=arrow_type)
-            for i, (level, arrow_type) in enumerate(
-                zip(index_levels, index_types)
+            get_column_metadata(
+                level,
+                name=level.name,
+                arrow_type=arrow_type,
+                field_name=field_name,
+            ) for i, (level, arrow_type, field_name) in enumerate(
+                zip(index_levels, index_types, index_column_names)
             )
         ]
 
@@ -214,9 +231,16 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
 
         for level in getattr(df.columns, 'levels', [df.columns]):
             string_dtype, extra_metadata = get_extension_dtype_info(level)
+
+            pandas_type = get_logical_type_from_numpy(level)
+            if pandas_type == 'unicode':
+                assert not extra_metadata
+                extra_metadata = {'encoding': 'UTF-8'}
+
             column_index = {
                 'name': level.name,
-                'pandas_type': get_logical_type_from_numpy(level),
+                'field_name': level.name,
+                'pandas_type': pandas_type,
                 'numpy_type': string_dtype,
                 'metadata': extra_metadata,
             }
@@ -442,7 +466,6 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
     schema = table.schema
     row_count = table.num_rows
     metadata = schema.metadata
-    columns_metadata = None
 
     has_pandas_metadata = metadata is not None and b'pandas' in metadata
 
@@ -452,13 +475,36 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
         columns = pandas_metadata['columns']
         column_indexes = pandas_metadata.get('column_indexes', [])
         table = _add_any_metadata(table, pandas_metadata)
-        columns_metadata = pandas_metadata.get('columns', None)
 
     block_table = table
 
+    index_columns_set = frozenset(index_columns)
+
+    # 0. 'field_name' is the name of the column in the arrow Table
+    # 1. 'name' is the user-facing name of the column, that is, it came from
+    #    pandas
+    # 2. 'field_name' and 'name' differ for index columns
+    # 3. We fall back on c['name'] for backwards compatibility
+    logical_index_names = [
+        c['name'] for c in columns
+        if c.get('field_name', c['name']) in index_columns_set
+    ]
+
+    # There must be the same number of field names and physical names
+    # (fields in the arrow Table)
+    assert len(logical_index_names) == len(index_columns_set)
+
+    # It can never be the case in a released version of pyarrow that
+    # c['name'] is None *and* 'field_name' is not a key in the column metadata,
+    # because the change to allow c['name'] to be None and the change to add
+    # 'field_name' are in the same release (0.8.0)
+    assert all(
+        (c['name'] is None and 'field_name' in c) or c['name'] is not None
+        for c in columns
+    )
+
     # Build up a list of index columns and names while removing those columns
     # from the original table
-    logical_index_names = [c['name'] for c in columns[-len(index_columns):]]
     for raw_name, logical_name in zip(index_columns, logical_index_names):
         i = schema.get_field_index(raw_name)
         if i != -1:
@@ -489,15 +535,12 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
         index = pd.RangeIndex(row_count)
 
     column_strings = [x.name for x in block_table.itercolumns()]
-    if columns_metadata is not None:
-        columns_name_dict = dict(
-            (str(x['name']), x['name'])
-            for x in columns_metadata
-        )
+    if columns:
+        columns_name_dict = {
+            c.get('field_name', str(c['name'])): c['name'] for c in columns
+        }
         columns_values = [
-            columns_name_dict[y]
-            if y in columns_name_dict.keys() else y
-            for y in column_strings
+            columns_name_dict.get(name, name) for name in column_strings
         ]
     else:
         columns_values = column_strings
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 1231844e357d1..97bbb6a175b0f 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -30,7 +30,7 @@
 import pandas as pd
 import pandas.util.testing as tm
 
-from pyarrow.compat import u
+from pyarrow.compat import u, PY2
 import pyarrow as pa
 import pyarrow.types as patypes
 
@@ -160,9 +160,41 @@ def test_integer_index_column(self):
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')])
         _check_pandas_roundtrip(df, preserve_index=True)
 
+    def test_index_metadata_field_name(self):
+        # test None case, and strangely named non-index columns
+        df = pd.DataFrame(
+            [(1, 'a', 3.1), (2, 'b', 2.2), (3, 'c', 1.3)],
+            index=pd.MultiIndex.from_arrays(
+                [['c', 'b', 'a'], [3, 2, 1]],
+                names=[None, 'foo']
+            ),
+            columns=['a', None, '__index_level_0__'],
+        )
+        t = pa.Table.from_pandas(df, preserve_index=True)
+        raw_metadata = t.schema.metadata
+
+        js = json.loads(raw_metadata[b'pandas'].decode('utf8'))
+
+        col1, col2, col3, idx0, foo = js['columns']
+
+        assert col1['name'] == 'a'
+        assert col1['name'] == col1['field_name']
+
+        assert col2['name'] is None
+        assert col2['field_name'] == 'None'
+
+        assert col3['name'] == '__index_level_0__'
+        assert col3['name'] == col3['field_name']
+
+        idx0_name, foo_name = js['index_columns']
+        assert idx0_name == '__index_level_0__'
+        assert idx0['field_name'] == idx0_name
+        assert idx0['name'] is None
+
+        assert foo_name == '__index_level_1__'
+        assert foo['name'] == 'foo'
+
     def test_categorical_column_index(self):
-        # I *really* hope no one uses category dtypes for single level column
-        # indexes
         df = pd.DataFrame(
             [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],
             columns=pd.Index(list('def'), dtype='category')
@@ -174,15 +206,36 @@ def test_categorical_column_index(self):
         column_indexes, = js['column_indexes']
         assert column_indexes['name'] is None
         assert column_indexes['pandas_type'] == 'categorical'
-        assert column_indexes['numpy_type'] == 'object'
+        assert column_indexes['numpy_type'] == 'int8'
 
         md = column_indexes['metadata']
         assert md['num_categories'] == 3
         assert md['ordered'] is False
 
+    def test_string_column_index(self):
+        df = pd.DataFrame(
+            [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],
+            columns=pd.Index(list('def'), name='stringz')
+        )
+        t = pa.Table.from_pandas(df, preserve_index=True)
+        raw_metadata = t.schema.metadata
+        js = json.loads(raw_metadata[b'pandas'].decode('utf8'))
+
+        column_indexes, = js['column_indexes']
+        assert column_indexes['name'] == 'stringz'
+        assert column_indexes['name'] == column_indexes['field_name']
+        assert column_indexes['pandas_type'] == ('bytes' if PY2 else 'unicode')
+        assert column_indexes['numpy_type'] == 'object'
+
+        md = column_indexes['metadata']
+
+        if not PY2:
+            assert len(md) == 1
+            assert md['encoding'] == 'UTF-8'
+        else:
+            assert md is None or 'encoding' not in md
+
     def test_datetimetz_column_index(self):
-        # I *really* hope no one uses category dtypes for single level column
-        # indexes
         df = pd.DataFrame(
             [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],
             columns=pd.date_range(

From 84f18c579f4092ba21e1d077c4a1823432022010 Mon Sep 17 00:00:00 2001
From: Lewis John McGibbney <lewis.mcgibbney@gmail.com>
Date: Sun, 10 Dec 2017 15:25:39 -0800
Subject: [PATCH 1329/1644] ARROW-1910: [C++] Use c_glib Brewfile in README for
 installing dependencies on macOS (#1407)

---
 cpp/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/README.md b/cpp/README.md
index 2034d29fae324..92e87cb436ada 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -42,7 +42,7 @@ sudo apt-get install cmake \
 On OS X, you can use [Homebrew][1]:
 
 ```shell
-brew update && brew bundle --file=cpp/Brewfile
+brew update && brew bundle --file=c_glib/Brewfile
 ```
 
 If you are developing on Windows, see the [Windows developer guide][2].

From 97678c106054c4d9a738ddcb1c895a6ab7ca9d31 Mon Sep 17 00:00:00 2001
From: Joris Van den Bossche <jorisvandenbossche@gmail.com>
Date: Sun, 10 Dec 2017 18:40:09 -0500
Subject: [PATCH 1330/1644] ARROW-1883: [Python] Fix handling of metadata in
 to_pandas when not all columns are present

This closes [ARROW-1883](https://issues.apache.org/jira/browse/ARROW-1883).

So basically what I did in `_add_any_metadata` was replacing `col = table[i]` with:

```
idx = schema.get_field_index(raw_name)
if idx != -1:
     col = table[idx]
```

to check that the column is actually present in the schema. However, that involved some more code to get to `raw_name` (the name how the column is present in the schema), as this does not always match the name in `pandas_metadata['column'][..]['name']`. Not sure if there is a better way to get that name.
(or if it would be better to filter `pandas_metadata` earlier on, instead of checking when actually trying to process the metadata of that column)

Author: Joris Van den Bossche <jorisvandenbossche@gmail.com>

Closes #1386 from jorisvandenbossche/parquet-column-selection and squashes the following commits:

3f605ef3 [Joris Van den Bossche] add test for compatibility for arrow 0.7.1 written parquet files
ea891b25 [Joris Van den Bossche] Use field_name if available
f6bdd1d3 [Joris Van den Bossche] Merge remote-tracking branch 'upstream/master' into parquet-column-selection
5df51f18 [Joris Van den Bossche] Add additional tests
306eaba3 [Joris Van den Bossche] ARROW-1883: [Python] Fix handling of metadata in to_pandas when not all columns are present
---
 python/pyarrow/pandas_compat.py               |  39 +++++++++++++-----
 .../v0.7.1.column-metadata-handling.parquet   | Bin 0 -> 2012 bytes
 python/pyarrow/tests/test_convert_pandas.py   |  31 ++++++++++++++
 python/pyarrow/tests/test_parquet.py          |  23 +++++++++++
 4 files changed, 82 insertions(+), 11 deletions(-)
 create mode 100644 python/pyarrow/tests/data/v0.7.1.column-metadata-handling.parquet

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 668048fd645dd..b5d395fe5700b 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -641,19 +641,36 @@ def _add_any_metadata(table, pandas_metadata):
 
     schema = table.schema
 
+    index_columns = pandas_metadata['index_columns']
+    n_index_levels = len(index_columns)
+    n_columns = len(pandas_metadata['columns']) - n_index_levels
+
     # Add time zones
     for i, col_meta in enumerate(pandas_metadata['columns']):
-        if col_meta['pandas_type'] == 'datetimetz':
-            col = table[i]
-            converted = col.to_pandas()
-            tz = col_meta['metadata']['timezone']
-            tz_aware_type = pa.timestamp('ns', tz=tz)
-            with_metadata = pa.Array.from_pandas(converted.values,
-                                                 type=tz_aware_type)
-
-            field = pa.field(schema[i].name, tz_aware_type)
-            modified_columns[i] = pa.Column.from_array(field,
-                                                       with_metadata)
+
+        raw_name = col_meta.get('field_name')
+        if not raw_name:
+            # deal with metadata written with arrow < 0.8
+            raw_name = col_meta['name']
+            if i >= n_columns:
+                # index columns
+                raw_name = index_columns[i - n_columns]
+            if raw_name is None:
+                raw_name = 'None'
+
+        idx = schema.get_field_index(raw_name)
+        if idx != -1:
+            if col_meta['pandas_type'] == 'datetimetz':
+                col = table[idx]
+                converted = col.to_pandas()
+                tz = col_meta['metadata']['timezone']
+                tz_aware_type = pa.timestamp('ns', tz=tz)
+                with_metadata = pa.Array.from_pandas(converted.values,
+                                                     type=tz_aware_type)
+
+                field = pa.field(schema[idx].name, tz_aware_type)
+                modified_columns[idx] = pa.Column.from_array(field,
+                                                             with_metadata)
 
     if len(modified_columns) > 0:
         columns = []
diff --git a/python/pyarrow/tests/data/v0.7.1.column-metadata-handling.parquet b/python/pyarrow/tests/data/v0.7.1.column-metadata-handling.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..d48041f518d55ce67b24862732e29960932d6ff9
GIT binary patch
literal 2012
zcmchYO>5LZ7{_PQY|O3&UvNSef-GSTi`~ZE?u#!7OU0rG(WUj)mXOVkx{^)SO|sZ7
zy@&@<zd(EN;GrL&c&Y3c@GJ=4JbCi$sfaVlj$JkpEVK?J6P|gV|NQcwC#1Q!Tt+-9
zA^jc_5Pu#3nTH?^Z*m|SGvgTnbOFglnnW%^kwvy5i*TGH(K(Z+r2^z;@<1Cw1@YM$
z4}`1?O~grpv<IaRP+OLOB>qLDkRVMYL(JkK=FT>R{C*=X=07zm;rQH$Q?eCV+}WmM
zzY(W=#VaVoC|itj4{Gmb2<8`#a3;wV5~7T=`iFB399+J0a61LS8i&s=k$Lp>MV`!Y
zR<<IGtR~L-O_*@hVyqvb_A^CTcT<ElNv4nxjaV-tm9d_=Mo-eDw-&uwF+^yQV~j(+
zPAh13Y+$P6wDI#0>?M@mLTw}xuwSQ?Iw*wY(Y7UYVixV5KD6kZVT3$}jR)8^%7!sv
z*(gKEex#3R8JD$dKo&gf3GSLGL;CU(N1K~;Xofzu=u;RA{R3#0M8~d>Lyy)HxX1?v
zBq6J0*N*kG(T=yfwc~9-?WnDzo$TpK-vvnkSw$9lrqeb(`Aur5Mjd3icF=V^^+s+r
zim1h$8f!0gokXKrV}Sc!$8|`BTGAI5^pf7jgVJ23T+tt2EthJQ%5|zSAv{!_pxfIp
z{EZ%_WgW+_S7}Zc`)1qpO%ikh+dkYQmg$(?u+&tyin(dMN49JJtKvj1(`+Sa5-jX@
zy0~6lb-ZG3Dd@XBeC1X@@H}jL>r?bYmF8}`P6#=bOy!w*?8TV8ze$xCiIqg+ww_|k
zpV~0luE6P7ZW~WZhNqk)%LxAK3&tC3YkdpI#GU@L0Q>V+ua_J5SFWsA^z!`5{l(_W
P-3JfBdmex$`LF*5ATPr-

literal 0
HcmV?d00001

diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 97bbb6a175b0f..7609d3488c6c0 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1286,6 +1286,37 @@ def test_array_from_pandas_typed_array_with_mask(self, t, data, expected):
         assert pa.Array.from_pandas(expected,
                                     type=pa.list_(t())).equals(result)
 
+    def test_table_column_subset_metadata(self):
+        # ARROW-1883
+        df = pd.DataFrame({
+            'a': [1, 2, 3],
+            'b': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')})
+        table = pa.Table.from_pandas(df)
+
+        table_subset = table.remove_column(1)
+        result = table_subset.to_pandas()
+        tm.assert_frame_equal(result, df[['a']])
+
+        table_subset2 = table_subset.remove_column(1)
+        result = table_subset2.to_pandas()
+        tm.assert_frame_equal(result, df[['a']])
+
+        # non-default index
+        for index in [
+                pd.Index(['a', 'b', 'c'], name='index'),
+                pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')]:
+            df = pd.DataFrame({'a': [1, 2, 3],
+                               'b': [.1, .2, .3]}, index=index)
+            table = pa.Table.from_pandas(df)
+
+            table_subset = table.remove_column(1)
+            result = table_subset.to_pandas()
+            tm.assert_frame_equal(result, df[['a']])
+
+            table_subset2 = table_subset.remove_column(1)
+            result = table_subset2.to_pandas()
+            tm.assert_frame_equal(result, df[['a']].reset_index(drop=True))
+
 
 def _fully_loaded_dataframe_example():
     from distutils.version import LooseVersion
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 2543e7d17befd..79e24d8d4e6eb 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1570,6 +1570,29 @@ def test_backwards_compatible_index_multi_level_some_named():
     tm.assert_frame_equal(result, expected)
 
 
+@parquet
+def test_backwards_compatible_column_metadata_handling():
+    expected = pd.DataFrame(
+        {'a': [1, 2, 3], 'b': [.1, .2, .3],
+         'c': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')})
+    expected.index = pd.MultiIndex.from_arrays(
+        [['a', 'b', 'c'],
+         pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')],
+        names=['index', None])
+
+    path = os.path.join(
+        os.path.dirname(__file__), 'data',
+        'v0.7.1.column-metadata-handling.parquet'
+    )
+    t = _read_table(path)
+    result = t.to_pandas()
+    tm.assert_frame_equal(result, expected)
+
+    t = _read_table(path, columns=['a'])
+    result = t.to_pandas()
+    tm.assert_frame_equal(result, expected[['a']].reset_index(drop=True))
+
+
 def test_decimal_roundtrip(tmpdir):
     num_values = 10
 

From 20bbbcba5da268186ae91c5fd002938a4bd59db3 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 11 Dec 2017 10:06:25 -0500
Subject: [PATCH 1331/1644] ARROW-1906: [Python] Do not override user-supplied
 type in pyarrow.array when converting DatetimeTZ pandas data

cc @BryanCutler

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1411 from wesm/ARROW-1906 and squashes the following commits:

49392e03 [Wes McKinney] Trust supplied user type in pyarrow.array when converting datetime+tz pandas data
---
 python/pyarrow/pandas_compat.py    | 3 ++-
 python/pyarrow/tests/test_array.py | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index b5d395fe5700b..f08049f6adba0 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -369,7 +369,8 @@ def get_datetimetz_type(values, dtype, type_):
     if values.dtype.type != np.datetime64:
         return values, type_
 
-    if isinstance(dtype, DatetimeTZDtype):
+    if isinstance(dtype, DatetimeTZDtype) and type_ is None:
+        # If no user type passed, construct a tz-aware timestamp type
         tz = dtype.tz
         unit = dtype.unit
         type_ = pa.timestamp(unit, tz)
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 92562da14150c..fa38c9257854e 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -331,8 +331,12 @@ def test_cast_timestamp_unit():
     s_nyc = s.dt.tz_localize('tzlocal()').dt.tz_convert('America/New_York')
 
     us_with_tz = pa.timestamp('us', tz='America/New_York')
+
     arr = pa.Array.from_pandas(s_nyc, type=us_with_tz)
 
+    # ARROW-1906
+    assert arr.type == us_with_tz
+
     arr2 = pa.Array.from_pandas(s, type=pa.timestamp('us'))
 
     assert arr[0].as_py() == s_nyc[0]

From abb934176ff3424663bab40b62afcc0eacf4ad2b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 11 Dec 2017 10:09:16 -0500
Subject: [PATCH 1332/1644] ARROW-1905: [Python] Add more comprehensive list of
 exact type checking functions to pyarrow.types

cc @BryanCutler

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1410 from wesm/ARROW-1905 and squashes the following commits:

93bddc9f [Wes McKinney] Add new functions to API, fix docstring
dcd0829f [Wes McKinney] Add more exact type checking functions
---
 python/doc/source/api.rst          |  15 ++++
 python/pyarrow/tests/test_types.py |  23 ++++++
 python/pyarrow/types.py            | 117 +++++++++++++++++++++++++++--
 3 files changed, 149 insertions(+), 6 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 636f41d67bc6a..2d3e39c6954a1 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -71,7 +71,18 @@ Type checking functions
    is_integer
    is_signed_integer
    is_unsigned_integer
+   is_int8
+   is_int16
+   is_int32
+   is_int64
+   is_uint8
+   is_uint16
+   is_uint32
+   is_uint64
    is_floating
+   is_float16
+   is_float32
+   is_float64
    is_decimal
    is_list
    is_struct
@@ -80,7 +91,11 @@ Type checking functions
    is_temporal
    is_timestamp
    is_date
+   is_date32
+   is_date64
    is_time
+   is_time32
+   is_time64
    is_null
    is_binary
    is_unicode
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index e352e35a3a3a0..68dc499cf48b4 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import pytest
+
 import pyarrow as pa
 import pyarrow.types as types
 
@@ -161,3 +163,24 @@ def test_types_hashable():
         assert hash(type_) == hash(type_)
         in_dict[type_] = i
         assert in_dict[type_] == i
+
+
+@pytest.mark.parametrize('t,check_func', [
+    (pa.date32(), types.is_date32),
+    (pa.date64(), types.is_date64),
+    (pa.time32('s'), types.is_time32),
+    (pa.time64('ns'), types.is_time64),
+    (pa.int8(), types.is_int8),
+    (pa.int16(), types.is_int16),
+    (pa.int32(), types.is_int32),
+    (pa.int64(), types.is_int64),
+    (pa.uint8(), types.is_uint8),
+    (pa.uint16(), types.is_uint16),
+    (pa.uint32(), types.is_uint32),
+    (pa.uint64(), types.is_uint64),
+    (pa.float16(), types.is_float16),
+    (pa.float32(), types.is_float32),
+    (pa.float64(), types.is_float64)
+])
+def test_exact_primitive_types(t, check_func):
+    assert check_func(t)
diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py
index 440d7eb09daa9..24557148a6363 100644
--- a/python/pyarrow/types.py
+++ b/python/pyarrow/types.py
@@ -32,6 +32,13 @@
 _NESTED_TYPES = {lib.Type_LIST, lib.Type_STRUCT, lib.Type_UNION, lib.Type_MAP}
 
 
+def is_null(t):
+    """
+    Return True if value is an instance of a null type
+    """
+    return t.id == lib.Type_NA
+
+
 def is_boolean(t):
     """
     Return True if value is an instance of a boolean type
@@ -41,25 +48,81 @@ def is_boolean(t):
 
 def is_integer(t):
     """
-    Return True if value is an instance of an integer type
+    Return True if value is an instance of any integer type
     """
     return t.id in _INTEGER_TYPES
 
 
 def is_signed_integer(t):
     """
-    Return True if value is an instance of a signed integer type
+    Return True if value is an instance of any signed integer type
     """
     return t.id in _SIGNED_INTEGER_TYPES
 
 
 def is_unsigned_integer(t):
     """
-    Return True if value is an instance of an unsigned integer type
+    Return True if value is an instance of any unsigned integer type
     """
     return t.id in _UNSIGNED_INTEGER_TYPES
 
 
+def is_int8(t):
+    """
+    Return True if value is an instance of an int8 type
+    """
+    return t.id == lib.Type_INT8
+
+
+def is_int16(t):
+    """
+    Return True if value is an instance of an int16 type
+    """
+    return t.id == lib.Type_INT16
+
+
+def is_int32(t):
+    """
+    Return True if value is an instance of an int32 type
+    """
+    return t.id == lib.Type_INT32
+
+
+def is_int64(t):
+    """
+    Return True if value is an instance of an int64 type
+    """
+    return t.id == lib.Type_INT64
+
+
+def is_uint8(t):
+    """
+    Return True if value is an instance of an uint8 type
+    """
+    return t.id == lib.Type_UINT8
+
+
+def is_uint16(t):
+    """
+    Return True if value is an instance of an uint16 type
+    """
+    return t.id == lib.Type_UINT16
+
+
+def is_uint32(t):
+    """
+    Return True if value is an instance of an uint32 type
+    """
+    return t.id == lib.Type_UINT32
+
+
+def is_uint64(t):
+    """
+    Return True if value is an instance of an uint64 type
+    """
+    return t.id == lib.Type_UINT64
+
+
 def is_floating(t):
     """
     Return True if value is an instance of a floating point numeric type
@@ -67,6 +130,27 @@ def is_floating(t):
     return t.id in _FLOATING_TYPES
 
 
+def is_float16(t):
+    """
+    Return True if value is an instance of an float16 (half-precision) type
+    """
+    return t.id == lib.Type_HALF_FLOAT
+
+
+def is_float32(t):
+    """
+    Return True if value is an instance of an float32 (single precision) type
+    """
+    return t.id == lib.Type_FLOAT
+
+
+def is_float64(t):
+    """
+    Return True if value is an instance of an float64 (double precision) type
+    """
+    return t.id == lib.Type_DOUBLE
+
+
 def is_list(t):
     """
     Return True if value is an instance of a list type
@@ -117,11 +201,18 @@ def is_time(t):
     return t.id in _TIME_TYPES
 
 
-def is_null(t):
+def is_time32(t):
     """
-    Return True if value is an instance of a null type
+    Return True if value is an instance of a time32 type
     """
-    return t.id == lib.Type_NA
+    return t.id == lib.Type_TIME32
+
+
+def is_time64(t):
+    """
+    Return True if value is an instance of a time64 type
+    """
+    return t.id == lib.Type_TIME64
 
 
 def is_binary(t):
@@ -159,6 +250,20 @@ def is_date(t):
     return t.id in _DATE_TYPES
 
 
+def is_date32(t):
+    """
+    Return True if value is an instance of a date32 (days) type
+    """
+    return t.id == lib.Type_DATE32
+
+
+def is_date64(t):
+    """
+    Return True if value is an instance of a date64 (milliseconds) type
+    """
+    return t.id == lib.Type_DATE64
+
+
 def is_map(t):
     """
     Return True if value is an instance of a map logical type

From 6a7ce328d585fb77121dc09bc55928ca6a0ce69d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 11 Dec 2017 10:42:00 -0500
Subject: [PATCH 1333/1644] ARROW-1904: [C++] Deprecate
 PrimitiveArray::raw_values

I was mistaken about this method's handling of the offset parameter, but it doesn't work correctly for boolean data (also a subclass of `PrimitiveArray`), so I think it's better to remove this method altogether

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1409 from wesm/ARROW-1904 and squashes the following commits:

eb735555 [Wes McKinney] clang-format
fa88b4c9 [Wes McKinney] Deprecate PrimitiveArray::raw_values, add implementation for FixedSizeBinary
---
 cpp/src/arrow/array.cc   |  4 ++++
 cpp/src/arrow/array.h    |  8 ++++++++
 cpp/src/arrow/compare.cc | 12 ++++++++++--
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 0b235cc199ca1..144fbcd05c205 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -139,11 +139,15 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t le
   SetData(ArrayData::Make(type, length, {null_bitmap, data}, null_count, offset));
 }
 
+#ifndef ARROW_NO_DEPRECATED_API
+
 const uint8_t* PrimitiveArray::raw_values() const {
   return raw_values_ +
          offset() * static_cast<const FixedWidthType&>(*type()).bit_width() / CHAR_BIT;
 }
 
+#endif
+
 template <typename T>
 NumericArray<T>::NumericArray(const std::shared_ptr<ArrayData>& data)
     : PrimitiveArray(data) {
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index ebe54adcb9eaf..6721c7d2717f3 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -334,9 +334,15 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray {
   /// Does not account for any slice offset
   std::shared_ptr<Buffer> values() const { return data_->buffers[1]; }
 
+#ifndef ARROW_NO_DEPRECATED_API
+
   /// \brief Return pointer to start of raw data
+  ///
+  /// \note Deprecated since 0.8.0
   const uint8_t* raw_values() const;
 
+#endif
+
  protected:
   PrimitiveArray() {}
 
@@ -567,6 +573,8 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray {
 
   int32_t byte_width() const { return byte_width_; }
 
+  const uint8_t* raw_values() const { return raw_values_ + data_->offset * byte_width_; }
+
  protected:
   inline void SetData(const std::shared_ptr<ArrayData>& data) {
     this->PrimitiveArray::SetData(data);
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 72ef122e3481e..9f07fa7eff826 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -312,8 +312,16 @@ static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& r
   const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type());
   const int byte_width = size_meta.bit_width() / CHAR_BIT;
 
-  const uint8_t* left_data = left.values() ? left.raw_values() : nullptr;
-  const uint8_t* right_data = right.values() ? right.raw_values() : nullptr;
+  const uint8_t* left_data = nullptr;
+  const uint8_t* right_data = nullptr;
+
+  if (left.values()) {
+    left_data = left.values()->data() + left.offset() * byte_width;
+  }
+
+  if (right.values()) {
+    right_data = right.values()->data() + right.offset() * byte_width;
+  }
 
   if (left.null_count() > 0) {
     for (int64_t i = 0; i < left.length(); ++i) {

From 0095c4e7a151c79a50255e190ddbc44c7f307510 Mon Sep 17 00:00:00 2001
From: lmeyerov <lmeyerov@gmail.com>
Date: Tue, 12 Dec 2017 15:08:14 -0500
Subject: [PATCH 1334/1644] ARROW-1911: [JS] Add Graphistry to Arrow JS proof
 points

Realized Graphistry's use wasn't listed, which matters as we're the most enterprise-y user right now.

Also made the GOAI reference more clear.

Author: lmeyerov <lmeyerov@gmail.com>

Closes #1412 from lmeyerov/patch-1 and squashes the following commits:

ff5f7375 [lmeyerov] fix(more precise description)
1fe8dad2 [lmeyerov] docs(fix backlink information architecture)
3a80c4d8 [lmeyerov] docs(tweak js readme)
d78f414c [lmeyerov] docs(normalize and sort implementation order)
4968dc93 [lmeyerov] docs(graphistry and goai in main powered by)
af8c0383 [lmeyerov] docs(include JS impl in main readme.md list)
e70e534d [lmeyerov] docs(add getting involved to develop.md)
a7e8d017 [lmeyerov] docs(add and cleanup orgs)
29629260 [lmeyerov] [JS] Add Graphistry to Arrow JS projects
---
 README.md          |  5 +++--
 js/DEVELOP.md      | 24 ++++++++++++++++++++++++
 js/README.md       | 35 ++++++++++++++++++++++++++---------
 site/powered_by.md |  9 +++++++++
 4 files changed, 62 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 9dda25de91178..15a9ffea2a7a3 100644
--- a/README.md
+++ b/README.md
@@ -37,8 +37,9 @@ Arrow is a set of technologies that enable big-data systems to process and move
 Initial implementations include:
 
  - [The Arrow Format](https://github.com/apache/arrow/tree/master/format)
- - [Java implementation](https://github.com/apache/arrow/tree/master/java)
  - [C++ implementation](https://github.com/apache/arrow/tree/master/cpp)
+ - [Java implementation](https://github.com/apache/arrow/tree/master/java)
+ - [JavaScript implementation](https://github.com/apache/arrow/tree/master/js)
  - [Python interface to C++ libraries](https://github.com/apache/arrow/tree/master/python)
 
 Arrow is an [Apache Software Foundation](www.apache.org) project. Learn more at
@@ -110,4 +111,4 @@ Thank you in advance for your contributions!
 [1]: mailto:dev-subscribe@arrow.apache.org
 [2]: https://github.com/apache/arrow/tree/master/format
 [3]: https://issues.apache.org/jira/browse/ARROW
-[4]: https://github.com/apache/arrow
\ No newline at end of file
+[4]: https://github.com/apache/arrow
diff --git a/js/DEVELOP.md b/js/DEVELOP.md
index 9f586e1b39bcb..1dd999a9efbb6 100644
--- a/js/DEVELOP.md
+++ b/js/DEVELOP.md
@@ -17,6 +17,30 @@
   under the License.
 -->
 
+# Getting Involved
+Even if you do not plan to contribute to Apache Arrow itself or Arrow
+integrations in other projects, we'd be happy to have you involved:
+
+* Join the mailing list: send an email to
+  [dev-subscribe@arrow.apache.org][1]. Share your ideas and use cases for the
+  project.
+* [Follow our activity on JIRA][3]
+* [Learn the format][2]
+* Contribute code to one of the reference implementations
+
+We prefer to receive contributions in the form of GitHub pull requests. Please send pull requests against the [github.com/apache/arrow][4] repository.
+
+If you are looking for some ideas on what to contribute, check out the [JIRA
+issues][3] for the Apache Arrow project. Comment on the issue and/or contact
+[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+with your questions and ideas.
+
+If you’d like to report a bug but don’t have time to fix it, you can still post
+it on JIRA, or email the mailing list
+[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+
+
+
 # The npm scripts
 
 * `npm run clean` - cleans targets
diff --git a/js/README.md b/js/README.md
index 9b1753f68335e..e58d335cd0d42 100644
--- a/js/README.md
+++ b/js/README.md
@@ -22,7 +22,7 @@
 [![Build Status](https://travis-ci.org/apache/arrow.svg?branch=master)](https://travis-ci.org/apache/arrow)
 [![Coverage Status](https://coveralls.io/repos/github/apache/arrow/badge.svg)](https://coveralls.io/github/apache/arrow)
 
-Arrow is a set of technologies that enable big-data systems to process and transfer data quickly.
+Arrow is a set of technologies that enable big data systems to process and transfer data quickly.
 
 ## install [apache-arrow from npm](https://www.npmjs.com/package/apache-arrow)
 
@@ -32,14 +32,9 @@ Arrow is a set of technologies that enable big-data systems to process and trans
 
 # Powering Columnar In-Memory Analytics
 
-Apache Arrow is a columnar memory layout specification for encoding vectors and table-like containers of flat and nested data. The Arrow spec aligns columnar data in memory to minimize cache misses and take advantage of the latest SIMD (Single input multiple data) and GPU operations on modern processors.
+[Apache Arrow](https://github.com/apache/arrow) is a columnar memory layout specification for encoding vectors and table-like containers of flat and nested data. The Arrow spec aligns columnar data in memory to minimize cache misses and take advantage of the latest SIMD (Single input multiple data) and GPU operations on modern processors.
 
-Apache Arrow is the emerging standard for large in-memory columnar data ([Spark](https://spark.apache.org/), [Pandas](http://wesmckinney.com/blog/pandas-and-apache-arrow/), [Drill](https://drill.apache.org/), ...). By standardizing on a common binary interchange format, big data systems can reduce the costs and friction associated with cross-system communication.
-
-# Related Projects
-
-* [GoAI](http://gpuopenanalytics.com/) -- Arrow-powered GPU analytics
-* [rxjs-mapd](https://github.com/graphistry/rxjs-mapd) -- A MapD Core node-driver that returns query results as Arrow columns
+Apache Arrow is the emerging standard for large in-memory columnar data ([Spark](https://spark.apache.org/), [Pandas](http://wesmckinney.com/blog/pandas-and-apache-arrow/), [Drill](https://drill.apache.org/), [Graphistry](https://www.graphistry.com), ...). By standardizing on a common binary interchange format, big data systems can reduce the costs and friction associated with cross-system communication.
 
 # Usage
 
@@ -202,6 +197,28 @@ The JS community is a diverse group with a varied list of target environments an
 
 If you think we missed a compilation target and it's a blocker for adoption, please open an issue.
 
+# People
+
+Full list of broader Apache Arrow [committers](https://arrow.apache.org/committers/).
+
+* Brian Hulette, CCRi,  _contributor_
+* Paul Taylor, Graphistry, Inc.,  _committer_
+
+# Powered By Apache Arrow in JS 
+
+Full list of broader Apache Arrow [projects & organizations](https://github.com/apache/arrow/blob/master/site/powered_by.md).
+ 
+## Open Source Projects
+
+* [Apache Arrow](https://arrow.apache.org) -- Parent project for Powering Columnar In-Memory Analytics, including affiliated open source projects
+* [rxjs-mapd](https://github.com/graphistry/rxjs-mapd) -- A MapD Core node-driver that returns query results as Arrow columns
+
+## Companies & Organizations
+
+* [CCRi](http://www.ccri.com/) -- Commonwealth Computer Research Inc, or CCRi, is a Central Virginia based data science and software engineering company
+* [GOAI](http://gpuopenanalytics.com/) -- GPU Open Analytics Initiative standardizes on Arrow as part of creating common data frameworks that enable developers and statistical researchers to accelerate data science on GPUs
+* [Graphistry, Inc.](https://www.graphistry.com/) - An end-to-end GPU accelerated visual investigation platform used by teams for security, anti-fraud, and related investigations. Graphistry uses Arrow in its NodeJS GPU backend and client libraries, and is an early contributing member to GOAI and Arrow\[JS\] working to bring these technologies to the enterprise.
+
 # License
 
 [Apache 2.0](https://github.com/apache/arrow/blob/master/LICENSE)
@@ -209,4 +226,4 @@ If you think we missed a compilation target and it's a blocker for adoption, ple
 [1]: mailto:dev-subscribe@arrow.apache.org
 [2]: https://github.com/apache/arrow/tree/master/format
 [3]: https://issues.apache.org/jira/browse/ARROW
-[4]: https://github.com/apache/arrow
\ No newline at end of file
+[4]: https://github.com/apache/arrow
diff --git a/site/powered_by.md b/site/powered_by.md
index 38320c0ec91b0..56f6e2bcee41a 100644
--- a/site/powered_by.md
+++ b/site/powered_by.md
@@ -103,6 +103,13 @@ short description of your use case.
   Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow
   buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI,
   Python, R, and more (all backed by Apache Arrow).
+* **[GOAI][19]:** Open GPU-Accelerated Analytics Initiative for Arrow-powered 
+  analytics across GPU tools and vendors  
+* **[Graphistry][18]:** Supercharged Visual Investigation Platform used by
+  teams for security, anti-fraud, and related investigations. The Graphistry
+  team uses Arrow in its NodeJS GPU backend and client libraries, and is an
+  early contributing member to GOAI and Arrow\[JS\] focused on bringing these 
+  technologies to the enterprise.  
 * **[Quilt Data][13]:** Quilt is a data package manager, designed to make
   managing data as easy as managing code. It supports Parquet format via
   pyarrow for data access.
@@ -124,3 +131,5 @@ short description of your use case.
 [15]: https://github.com/dask/dask
 [16]: https://red-data-tools.github.io/
 [17]: https://github.com/red-data-tools/red-arrow/
+[18]: https://www.graphistry.com
+[19]: http://gpuopenanalytics.com

From 16ddb127c5e6e51d80aa24d94dffded393db78a9 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 12 Dec 2017 15:08:54 -0500
Subject: [PATCH 1335/1644] ARROW-1914: [C++] Fix build dependency for GPU
 support build

"make -j" may cause build error:

    [100%] Built target gflags_nothreads_static
    Install the project...
    -- Install configuration: "RELEASE"
    -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/cmake/gflags/gflags-config.cmake
    -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/cmake/gflags/gflags-config-version.cmake
    -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/cmake/gflags/gflags-targets.cmake
    -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/cmake/gflags/gflags-targets-release.cmake
    -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/bin/gflags_completions.sh
    -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/pkgconfig/gflags.pc
    -- Installing: /home/kou/.cmake/packages/gflags/fb801def37c922433975cbfefb3aa08d
    [ 26%] Completed 'gflags_ep'
    [ 55%] Building C object CMakeFiles/brotlienc.dir/enc/literal_cost.c.o
    [ 26%] Built target gflags_ep
    Scanning dependencies of target arrow_gpu_objlib
    [ 26%] Building CXX object src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/cuda_arrow_ipc.cc.o
    [ 59%] Building C object CMakeFiles/brotlienc.dir/enc/memory.c.o
    /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/src/arrow/gpu/cuda_arrow_ipc.cc:26:10: fatal error: arrow/ipc/Message_generated.h: No such file or directory
     #include "arrow/ipc/Message_generated.h"
              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    compilation terminated.
    src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/build.make:62: recipe for target 'src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/cuda_arrow_ipc.cc.o' failed
    make[2]: *** [src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/cuda_arrow_ipc.cc.o] Error 1
    CMakeFiles/Makefile2:2108: recipe for target 'src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/all' failed
    make[1]: *** [src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/all] Error 2
    make[1]: *** Waiting for unfinished jobs....

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1414 from kou/cpp-fix-gpu-build-dependency and squashes the following commits:

8fe28c7a [Kouhei Sutou] [C++] Fix build dependency for GPU support build
---
 cpp/src/arrow/gpu/CMakeLists.txt | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt
index 3ddf2c7974f0e..be474131e3c72 100644
--- a/cpp/src/arrow/gpu/CMakeLists.txt
+++ b/cpp/src/arrow/gpu/CMakeLists.txt
@@ -32,9 +32,6 @@ set(ARROW_GPU_SRCS
   cuda_memory.cc
 )
 
-add_custom_target(arrow_gpu_sources DEPENDS ${ARROW_GPU_SRCS})
-add_dependencies(arrow_gpu_sources metadata_fbs)
-
 set(ARROW_GPU_SHARED_LINK_LIBS
   arrow_shared
   ${CUDA_LIBRARIES}
@@ -43,6 +40,7 @@ set(ARROW_GPU_SHARED_LINK_LIBS
 
 ADD_ARROW_LIB(arrow_gpu
   SOURCES ${ARROW_GPU_SRCS}
+  DEPENDENCIES metadata_fbs
   SHARED_LINK_FLAGS ""
   SHARED_LINK_LIBS ${ARROW_GPU_SHARED_LINK_LIBS}
   STATIC_LINK_LIBS ""

From 4f1bc97033ca0c7c60e91fce463a65283e7570d2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Dec 2017 15:58:00 -0500
Subject: [PATCH 1336/1644] ARROW-1916: [Java] Include java/dev/checkstyle in
 git archive for source releases

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1416 from wesm/ARROW-1916 and squashes the following commits:

58402637 [Wes McKinney] Include java/dev/checkstyle in git archive for source releases
---
 java/.gitattributes | 1 -
 1 file changed, 1 deletion(-)

diff --git a/java/.gitattributes b/java/.gitattributes
index cb02d822677ed..596615322fb3e 100644
--- a/java/.gitattributes
+++ b/java/.gitattributes
@@ -1,3 +1,2 @@
 .gitattributes export-ignore
 .gitignore export-ignore
-/dev export-ignore

From 8d006dc44194b5e4f4c7848ebdd7f916dff2439c Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Tue, 12 Dec 2017 17:17:20 -0500
Subject: [PATCH 1337/1644] ARROW-1915: [Python] Add missing parquet decorator
 to decimal tests

In `test_parquet.py` add the `@parquet` decorator to 2 decimal tests so they can be skipped.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1415 from BryanCutler/python-parquet-decimal-tests-decorator-ARROW-1915 and squashes the following commits:

822ec433 [Bryan Cutler] add missing parquet decorator to decimal tests
---
 python/pyarrow/tests/test_parquet.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 79e24d8d4e6eb..fc8c8f0c98da9 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1593,6 +1593,7 @@ def test_backwards_compatible_column_metadata_handling():
     tm.assert_frame_equal(result, expected[['a']].reset_index(drop=True))
 
 
+@parquet
 def test_decimal_roundtrip(tmpdir):
     num_values = 10
 
@@ -1619,6 +1620,7 @@ def test_decimal_roundtrip(tmpdir):
     tm.assert_frame_equal(result, expected)
 
 
+@parquet
 @pytest.mark.xfail(
     raises=pa.ArrowException, reason='Parquet does not support negative scale'
 )

From e4770b286eb5e08697dfceb1ff59bef672a29ed7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Dec 2017 17:18:18 -0500
Subject: [PATCH 1338/1644] ARROW-1917: Fixes to enable
 verify-release-candidate.sh to work for 0.8.0

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1417 from wesm/ARROW-1917 and squashes the following commits:

e394f301 [Wes McKinney] Disable integration portion of JS. Run integration tests earlier
363c4bea [Wes McKinney] Set GI_TYPELIB_PATH when testing glib in verify-release-candidate-sh
420a5716 [Wes McKinney] [Java] Change scm git link in pom.xml
---
 dev/release/verify-release-candidate.sh | 25 +++++++++++++++----------
 java/pom.xml                            |  4 ++--
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 7c5e7fd0abaaf..f33211e26a3f6 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -165,7 +165,9 @@ test_glib() {
   make -j$NPROC
   make install
 
-  NO_MAKE=yes test/run-test.sh
+  GI_TYPELIB_PATH=$ARROW_HOME/lib/girepository-1.0 \
+                 NO_MAKE=yes \
+                 test/run-test.sh
 
   popd
 }
@@ -174,15 +176,17 @@ test_js() {
   pushd js
   npm install
   # clean, lint, and build JS source
-  npm run clean:all
-  npm run lint
-  npm run build
+  npx run-s clean:all lint build
+  npm run test
+
   # create initial integration test data
-  npm run create:testdata
+  # npm run create:testdata
+
   # run once to write the snapshots
-  npm test -- -t ts -u --integration
+  # npm test -- -t ts -u --integration
+
   # run again to test all builds against the snapshots
-  npm test -- --integration
+  # npm test -- --integration
   popd
 }
 
@@ -238,13 +242,14 @@ fetch_archive $DIST_NAME
 tar xvzf ${DIST_NAME}.tar.gz
 cd ${DIST_NAME}
 
+test_package_java
 setup_miniconda
 test_and_install_cpp
+test_integration
+test_glib
 install_parquet_cpp
 test_python
-test_glib
-test_package_java
-test_integration
+
 test_js
 
 echo 'Release candidate looks good!'
diff --git a/java/pom.xml b/java/pom.xml
index 162c53460cc93..922048c018b7a 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -41,8 +41,8 @@
   </properties>
 
   <scm>
-    <connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
-    <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
+    <connection>scm:git:https://github.com/apache/arrow.git</connection>
+    <developerConnection>scm:git:https://github.com/apache/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
     <tag>apache-arrow-0.7.1</tag>
   </scm>

From 3afb8f4e2df14fb9a512d2a82b6aeff27d013df6 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Dec 2017 17:19:42 -0500
Subject: [PATCH 1339/1644] [Release] Update CHANGELOG.md for 0.8.0

Change-Id: If8489283f73f5aa8947f742958cdb4a0b5a76084
---
 CHANGELOG.md | 290 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 290 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 485afc1497ebe..153159cb779e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,295 @@
   under the License.
 -->
 
+# Apache Arrow 0.8.0 (12 December 2017)
+
+## Bug
+
+* ARROW-1282 - Large memory reallocation by Arrow causes hang in jemalloc
+* ARROW-1341 - [C++] Deprecate arrow::MakeTable in favor of new ctor from ARROW-1334
+* ARROW-1347 - [JAVA] List null type should use consistent name for inner field
+* ARROW-1398 - [Python] No support reading columns of type decimal(19,4)
+* ARROW-1409 - [Format] Use for "page" attribute in Buffer in metadata
+* ARROW-1540 - [C++] Fix valgrind warnings in cuda-test if possible
+* ARROW-1541 - [C++] Race condition with arrow\_gpu 
+* ARROW-1543 - [C++] row\_wise\_conversion example doesn't correspond to ListBuilder constructor arguments
+* ARROW-1555 - [Python] write\_to\_dataset on s3
+* ARROW-1584 - [PYTHON] serialize\_pandas on empty dataframe
+* ARROW-1585 - serialize\_pandas round trip fails on integer columns
+* ARROW-1586 - [PYTHON] serialize\_pandas roundtrip loses columns name
+* ARROW-1609 - Plasma: Build fails with Xcode 9.0
+* ARROW-1615 - CXX flags for development more permissive than Travis CI builds
+* ARROW-1617 - [Python] Do not use symlinks in python/cmake\_modules 
+* ARROW-1620 - Python: Download Boost in manylinux1 build from bintray
+* ARROW-1624 - [C++] Follow up fixes / tweaks to compiler warnings for Plasma / LLVM 4.0, add to readme
+* ARROW-1625 - [Serialization] Support OrderedDict properly
+* ARROW-1629 - [C++] Fix problematic code paths identified by infer tool
+* ARROW-1633 - [Python] numpy "unicode" arrays not understood
+* ARROW-1640 - Resolve OpenSSL issues in Travis CI
+* ARROW-1647 - [Plasma] Potential bug when reading/writing messages.
+* ARROW-1653 - [Plasma] Use static cast to avoid compiler warning.
+* ARROW-1656 - [C++] Endianness Macro is Incorrect on Windows And Mac
+* ARROW-1657 - [C++] Multithreaded Read Test Failing on Arch Linux
+* ARROW-1658 - [Python] Out of bounds dictionary indices causes segfault after converting to pandas
+* ARROW-1663 - [Java] Follow up on ARROW-1347 and make schema backward compatible
+* ARROW-1670 - [Python] Speed up deserialization code path
+* ARROW-1672 - [Python] Failure to write Feather bytes column
+* ARROW-1673 - [Python] NumPy boolean arrays get converted to uint8 arrays on NdarrayToTensor roundtrip
+* ARROW-1676 - [C++] Correctly truncate oversized validity bitmaps when writing Feather format
+* ARROW-1678 - [Python] Incorrect serialization of numpy.float16
+* ARROW-1680 - [Python] Timestamp unit change not done in from\_pandas() conversion
+* ARROW-1686 - Documentation generation script creates "apidocs" directory under site/java
+* ARROW-1693 - [JS] Error reading dictionary-encoded integration test files
+* ARROW-1695 - [Serialization] Fix reference counting of numpy arrays created in custom serialializer
+* ARROW-1698 - [JS] File reader attempts to load the same dictionary batch more than once
+* ARROW-1704 - [GLib] Go example in test suite is broken
+* ARROW-1708 - [JS] Linter problem breaks master build
+* ARROW-1709 - [C++] Decimal.ToString is incorrect for negative scale
+* ARROW-1711 - [Python] flake8 checks still not failing builds
+* ARROW-1714 - [Python] No named pd.Series name serialized as u'None'
+* ARROW-1720 - [Python] Segmentation fault while trying to access an out-of-bound chunk
+* ARROW-1723 - Windows: \_\_declspec(dllexport) specified when building arrow static library
+* ARROW-1730 - [Python] Incorrect result from pyarrow.array when passing timestamp type
+* ARROW-1732 - [Python] RecordBatch.from\_pandas fails on DataFrame with no columns when preserve\_index=False
+* ARROW-1735 - [C++] Cast kernels cannot write into sliced output array
+* ARROW-1738 - [Python] Wrong datetime conversion when pa.array with unit
+* ARROW-1739 - [Python] Fix usages of assertRaises causing broken build
+* ARROW-1742 - C++: clang-format is not detected correct on OSX anymore
+* ARROW-1743 - [Python] Table to\_pandas fails when index contains categorical column
+* ARROW-1745 - Compilation failure on Mac OS in plasma tests
+* ARROW-1749 - [C++] Handle range of Decimal128 values that require 39 digits to be displayed
+* ARROW-1751 - [Python] Pandas 0.21.0 introduces a breaking API change for MultiIndex construction
+* ARROW-1754 - [Python] Fix buggy Parquet roundtrip when an index name is the same as a column name
+* ARROW-1756 - [Python] Observed int32 overflow in Feather write/read path
+* ARROW-1762 - [C++] unittest failure for language environment
+* ARROW-1764 - [Python] Add -c conda-forge for Windows dev installation instructions
+* ARROW-1766 - [GLib] Fix failing builds on OSX
+* ARROW-1768 - [Python] Fix suppressed exception in ParquetWriter.\_\_del\_\_
+* ARROW-1770 - [GLib] Fix GLib compiler warning
+* ARROW-1771 - [C++] ARROW-1749 Breaks Public API test in parquet-cpp
+* ARROW-1776 - [C++[ arrow::gpu::CudaContext::bytes\_allocated() isn't defined
+* ARROW-1778 - [Python] Link parquet-cpp statically, privately in manylinux1 wheels
+* ARROW-1781 - [CI] OSX Builds on Travis-CI time out often
+* ARROW-1788 - Plasma store crashes when trying to abort objects for disconnected client
+* ARROW-1791 - Integration tests generate date[DAY] values outside of reasonable range
+* ARROW-1793 - [Integration] fix a typo for README.md 
+* ARROW-1800 - [C++] Fix and simplify random\_decimals
+* ARROW-1805 - [Python] ignore non-parquet files when exploring dataset
+* ARROW-1811 - [C++/Python] Rename all Decimal based APIs to Decimal128
+* ARROW-1812 - Plasma store modifies hash table while iterating during client disconnect
+* ARROW-1829 - [Plasma] Clean up eviction policy bookkeeping
+* ARROW-1830 - [Python] Error when loading all the files in a dictionary
+* ARROW-1836 - [C++] Fix C4996 warning from arrow/util/variant.h on MSVC builds
+* ARROW-1840 - [Website] The installation command failed on Windows10 anaconda environment.
+* ARROW-1845 - [Python] Expose Decimal128Type
+* ARROW-1852 - [Plasma] Make retrieving manager file descriptor const
+* ARROW-1853 - [Plasma] Fix off-by-one error in retry processing
+* ARROW-1863 - [Python] PyObjectStringify could render bytes-like output for more types of objects
+* ARROW-1865 - [C++] Adding a column to an empty Table fails
+* ARROW-1869 - Fix typo in LowCostIdentityHashMap
+* ARROW-1871 - [Python/C++] Appending Python Decimals with different scales requires rescaling
+* ARROW-1873 - [Python] Segmentation fault when loading total 2GB of parquet files
+* ARROW-1877 - Incorrect comparison in JsonStringArrayList.equals
+* ARROW-1879 - [Python] Dask integration tests are not skipped if dask is not installed
+* ARROW-1881 - [Python] setuptools\_scm picks up JS version tags
+* ARROW-1882 - [C++] Reintroduce DictionaryBuilder
+* ARROW-1883 - [Python] BUG: Table.to\_pandas metadata checking fails if columns are not present
+* ARROW-1889 - [Python] --exclude is not available in older git versions
+* ARROW-1890 - [Python] Masking for date32 arrays not working
+* ARROW-1891 - [Python] NaT date32 values are only converted to nulls if from\_pandas is used
+* ARROW-1892 - [Python] Unknown list item type: binary
+* ARROW-1893 - [Python] test\_primitive\_serialization fails on Python 2.7.3
+* ARROW-1895 - [Python] Add field\_name to pandas index metadata
+* ARROW-1897 - [Python] Incorrect numpy\_type for pandas metadata of Categoricals
+* ARROW-1904 - [C++] Deprecate PrimitiveArray::raw\_values
+* ARROW-1906 - [Python] Creating a pyarrow.Array with timestamp of different unit is not casted
+* ARROW-1908 - [Python] Construction of arrow table from pandas DataFrame with duplicate column names crashes
+* ARROW-1910 - CPP README Brewfile link incorrect
+* ARROW-1914 - [C++] make -j may fail to build with -DARROW\_GPU=on
+* ARROW-1915 - [Python] Parquet tests should be optional
+* ARROW-1916 - [Java] Do not exclude java/dev/checkstyle from source releases
+* ARROW-1917 - [GLib] Must set GI\_TYPELIB\_PATH in verify-release-candidate.sh
+* ARROW-226 - [C++] libhdfs: feedback to help determining cause of failure in opening file path
+* ARROW-641 - [C++] Do not build/run io-hdfs-test if ARROW\_HDFS=off
+
+## Improvement
+
+* ARROW-1087 - [Python] add get\_include to expose directory containing header files
+* ARROW-1134 - [C++] Allow C++/CLI projects to build with Arrow​
+* ARROW-1178 - [Python] Create alternative to Table.from\_pandas that yields a list of RecordBatch objects with a given chunk size
+* ARROW-1226 - [C++] Improve / correct doxygen function documentation in arrow::ipc
+* ARROW-1371 - [Website] Add "Powered By" page to the website
+* ARROW-1455 - [Python] Add Dockerfile for validating Dask integration outside of usual CI
+* ARROW-1488 - [C++] Implement ArrayBuilder::Finish in terms of internal::ArrayData
+* ARROW-1498 - [GitHub] Add CONTRIBUTING.md and ISSUE\_TEMPLATE.md
+* ARROW-1503 - [Python] Add serialization callbacks for pandas objects in pyarrow.serialize
+* ARROW-1522 - [C++] Support pyarrow.Buffer as built-in type in pyarrow.serialize
+* ARROW-1523 - [C++] Add helper data struct with methods for reading a validity bitmap possibly having a non-zero offset
+* ARROW-1524 - [C++] More graceful solution for handling non-zero offsets on inputs and outputs in compute library
+* ARROW-1525 - [C++] Change functions in arrow/compare.h to not return Status
+* ARROW-1526 - [Python] Unit tests to exercise code path in PARQUET-1100
+* ARROW-1535 - [Python] Enable sdist source tarballs to build assuming that Arrow C++ libraries are available on the host system
+* ARROW-1538 - [C++] Support Ubuntu 14.04 in .deb packaging automation
+* ARROW-1539 - [C++] Remove functions deprecated as of 0.7.0 and prior releases
+* ARROW-1556 - [C++] Incorporate AssertArraysEqual function from PARQUET-1100 patch
+* ARROW-1588 - [C++/Format] Harden Decimal Format
+* ARROW-1593 - [PYTHON] serialize\_pandas should pass through the preserve\_index keyword
+* ARROW-1594 - [Python] Enable multi-threaded conversions in Table.from\_pandas
+* ARROW-1600 - [C++] Zero-copy Buffer constructor from std::string
+* ARROW-1602 - [C++] Add IsValid/IsNotNull method to arrow::Array
+* ARROW-1603 - [C++] Add BinaryArray method to get a value as a std::string
+* ARROW-1604 - [Python] Support common type aliases in cast(...) and various type= arguments
+* ARROW-1605 - [Python] pyarrow.array should be able to yield smaller integer types without an explicit cast
+* ARROW-1607 - [C++] Implement DictionaryBuilder for Decimals
+* ARROW-1613 - [Java] ArrowReader should not close the input ReadChannel
+* ARROW-1616 - [Python] Add "write" method to RecordBatchStreamWriter that dispatches to write\_table/write\_back as appropriate
+* ARROW-1626 - Add make targets to run the inter-procedural static analysis tool called "infer".
+* ARROW-1627 - [JAVA] Reduce heap usage(Phase 2) - memory footprint in AllocationManager.BufferLedger
+* ARROW-1630 - [Serialization] Support Python datetime objects
+* ARROW-1635 - Add release management guide for PMCs
+* ARROW-1641 - [C++] Do not include <mutex> in public headers
+* ARROW-1651 - [JS] Lazy row accessor in Table
+* ARROW-1652 - [JS] Separate Vector into BatchVector and CompositeVector
+* ARROW-1654 - [Python] pa.DataType cannot be pickled
+* ARROW-1662 - Move OSX Dependency management into brew bundle Brewfiles
+* ARROW-1665 - [Serialization] Support more custom datatypes in the default serialization context
+* ARROW-1666 - [GLib] Enable gtk-doc on Travis CI Mac environment
+* ARROW-1671 - [C++] Change arrow::MakeArray to not return Status
+* ARROW-1675 - [Python] Use RecordBatch.from\_pandas in FeatherWriter.write
+* ARROW-1677 - [Blog] Add blog post on Ray and Arrow Python serialization
+* ARROW-1679 - [GLib] Add garrow\_record\_batch\_reader\_read\_next()
+* ARROW-1683 - [Python] Restore "TimestampType" to pyarrow namespace
+* ARROW-1684 - [Python] Simplify user API for reading nested Parquet columns
+* ARROW-1689 - [Python] Categorical Indices Should Be Zero-Copy
+* ARROW-1691 - [Java] Conform Java Decimal type implementation to format decisions in ARROW-1588
+* ARROW-1701 - [Serialization] Support zero copy PyTorch Tensor serialization
+* ARROW-1702 - Update jemalloc in manylinux1 build
+* ARROW-1703 - [C++] Vendor exact version of jemalloc we depend on
+* ARROW-1707 - Update dev README after movement to GitBox
+* ARROW-1716 - [Format/JSON] Use string integer value for Decimals in JSON
+* ARROW-1721 - [Python] Support null mask in places where it isn't supported in numpy\_to\_arrow.cc
+* ARROW-1724 - [Packaging] Support Ubuntu 17.10
+* ARROW-1725 - [Packaging] Upload .deb for Ubuntu 17.10
+* ARROW-1726 - [GLib] Add setup description to verify C GLib build
+* ARROW-1727 - [Format] Expand Arrow streaming format to permit new dictionaries and deltas / additions to existing dictionaries
+* ARROW-1728 - [C++] Run clang-format checks in Travis CI
+* ARROW-1737 - [GLib] Use G\_DECLARE\_DERIVABLE\_TYPE
+* ARROW-1746 - [Python] Add build dependencies for Arch Linux
+* ARROW-1747 - [C++] Don't export symbols of statically linked libraries
+* ARROW-1750 - [C++] Remove the need for arrow/util/random.h
+* ARROW-1753 - [Python] Provide for matching subclasses with register\_type in serialization context
+* ARROW-1755 - [C++] Add build options for MSVC to use static runtime libraries
+* ARROW-1758 - [Python] Remove pickle=True option for object serialization
+* ARROW-1763 - [Python] DataType should be hashable
+* ARROW-1765 - [Doc] Use dependencies from conda in C++ docker build
+* ARROW-1785 - [Format/C++/Java] Remove VectorLayout metadata from Flatbuffers metadata
+* ARROW-1787 - [Python] Support reading parquet files into DataFrames in a backward compatible way
+* ARROW-1794 - [C++/Python] Rename DecimalArray to Decimal128Array
+* ARROW-1801 - [Docs] Update install instructions to use red-data-tools repos
+* ARROW-1808 - [C++] Make RecordBatch interface virtual to permit record batches that lazy-materialize columns
+* ARROW-1809 - [GLib] Use .xml instead of .sgml for GTK-Doc main file
+* ARROW-1810 - [Plasma] Remove test shell scripts
+* ARROW-1817 - Configure JsonFileReader to read NaN for floats
+* ARROW-1826 - [JAVA] Avoid branching at cell level (copyFrom)
+* ARROW-1828 - [C++] Implement hash kernel specialization for BooleanType
+* ARROW-1834 - [Doc] Build documentation in separate build folders
+* ARROW-1838 - [C++] Use compute::Datum uniformly for input argument to kernels
+* ARROW-1841 - [JS] Update text-encoding-utf-8 and tslib for node ESModules support
+* ARROW-1849 - [GLib] Add input checks to GArrowRecordBatch
+* ARROW-1850 - [C++] Use const void* in Writable::Write instead of const uint8\_t*
+* ARROW-1854 - [Python] Improve performance of serializing object dtype ndarrays
+* ARROW-1855 - [GLib] Add workaround for build failure on macOS
+* ARROW-1864 - [Java] Upgrade Netty to 4.1.x
+* ARROW-1884 - [C++] Make JsonReader/JsonWriter classes internal APIs
+* ARROW-1901 - [Python] Support recursive mkdir for DaskFilesystem
+* ARROW-1902 - [Python] Remove mkdir race condition from write\_to\_dataset 
+* ARROW-1905 - [Python] Add more functions for checking exact types in pyarrow.types
+* ARROW-1911 - Add Graphistry to Arrow JS proof points
+* ARROW-905 - [Docs] Add Dockerfile for reproducible documentation generation
+* ARROW-942 - Support integration testing on Python 2.7
+* ARROW-950 - [Site] Add Google Analytics tag
+
+## New Feature
+
+* ARROW-1032 - [JS] Support custom\_metadata
+* ARROW-1047 - [Java] Add generalized stream writer and reader interfaces that are decoupled from IO / message framing
+* ARROW-1114 - [C++] Create Record Batch Builder class as a reusable and efficient way to transpose row-by-row data to columns
+* ARROW-1250 - [Python] Define API for user type checking of array types
+* ARROW-1482 - [C++] Implement casts between date32 and date64
+* ARROW-1483 - [C++] Implement casts between time32 and time64
+* ARROW-1484 - [C++] Implement (safe and unsafe) casts between timestamps and times of different units
+* ARROW-1486 - [C++] Decide if arrow::RecordBatch needs to be copyable
+* ARROW-1487 - [C++] Implement casts from List<A> to List<B>, where a cast function is defined from any A to B
+* ARROW-1559 - [C++] Kernel implementations for "unique" (compute distinct elements of array)
+* ARROW-1573 - [C++] Implement stateful kernel function that uses DictionaryBuilder to compute dictionary indices
+* ARROW-1575 - [Python] Add pyarrow.column factory function
+* ARROW-1577 - [JS] Package release script for NPM modules
+* ARROW-1631 -  [C++] Add GRPC to ThirdpartyToolchain.cmake
+* ARROW-1637 - [C++] IPC round-trip for null type
+* ARROW-1648 - C++: Add cast from Dictionary[NullType] to NullType
+* ARROW-1649 - C++: Print number of nulls in PrettyPrint for NullArray
+* ARROW-1667 - [GLib] Support Meson
+* ARROW-1685 - [GLib] Add GArrowTableReader
+* ARROW-1690 - [GLib] Add garrow\_array\_is\_valid()
+* ARROW-1697 - [GitHub] Add ISSUE\_TEMPLATE.md
+* ARROW-1718 - [Python] Implement casts from timestamp to date32/date64 and support in Array.from\_pandas
+* ARROW-1734 - C++/Python: Add cast function on Column-level
+* ARROW-1736 - [GLib] Add GArrowCastOptions:allow-time-truncate
+* ARROW-1748 - [GLib] Add GArrowRecordBatchBuilder
+* ARROW-1752 - [Packaging] Add GPU packages for Debian and Ubuntu
+* ARROW-1767 - [C++] Support file reads and writes over 2GB on Windows
+* ARROW-1772 - [C++] Add public-api-test module in style of parquet-cpp
+* ARROW-1773 - [C++] Add casts from date/time types to compatible signed integers
+* ARROW-1775 - Ability to abort created but unsealed Plasma objects
+* ARROW-1777 - [C++] Add static ctor ArrayData::Make for nicer syntax in places
+* ARROW-1782 - [Python] Expose compressors as pyarrow.compress, pyarrow.decompress
+* ARROW-1783 - [Python] Convert SerializedPyObject to/from sequence of component buffers with minimal memory allocation / copying
+* ARROW-1784 - [Python] Read and write pandas.DataFrame in pyarrow.serialize by decomposing the BlockManager rather than coercing to Arrow format
+* ARROW-1802 - [GLib] Add Arrow GPU support
+* ARROW-1806 - [GLib] Add garrow\_record\_batch\_writer\_write\_table()
+* ARROW-1844 - [C++] Basic benchmark suite for hash kernels
+* ARROW-1857 - [Python] Add switch for boost linkage with static parquet in wheels
+* ARROW-1859 - [GLib] Add GArrowDictionaryDataType
+* ARROW-1862 - [GLib] Add GArrowDictionaryArray
+* ARROW-1874 - [GLib] Add garrow\_array\_unique()
+* ARROW-1878 - [GLib] Add garrow\_array\_dictionary\_encode()
+* ARROW-480 - [Python] Add accessors for Parquet column statistics 
+* ARROW-504 - [Python] Add adapter to write pandas.DataFrame in user-selected chunk size to streaming format
+* ARROW-507 - [C++/Python] Construct List container from offsets and values subarrays
+* ARROW-541 - [JS] Implement JavaScript-compatible implementation
+* ARROW-571 - [Python] Add APIs to build Parquet files incrementally from Arrow tables
+* ARROW-587 - Add JIRA fix version to merge tool
+* ARROW-609 - [C++] Function for casting from days since UNIX epoch to int64 date
+* ARROW-838 - [Python] Efficient construction of arrays from non-pandas 1D NumPy arrays
+* ARROW-972 - [Python] Add test cases and basic APIs for UnionArray
+
+## Sub-task
+
+* ARROW-1471 - [JAVA] Document requirements and non/requirements for ValueVector updates
+* ARROW-1472 - [JAVA] Design updated ValueVector Object Hierarchy
+* ARROW-1473 - [JAVA] Create Prototype Code Hierarchy (Implementation Phase 1)
+* ARROW-1474 - [JAVA] ValueVector hierarchy (Implementation Phase 2)
+* ARROW-1476 - [JAVA] Implement final ValueVector updates
+* ARROW-1710 - [Java] Remove non-nullable vectors in new vector class hierarchy 
+* ARROW-1717 - [Java] Remove public static helper method in vector classes for JSONReader/Writer
+* ARROW-1719 - [Java] Remove accessor/mutator 
+* ARROW-1779 - [Java] Integration test breaks without zeroing out validity vectors
+* ARROW-1819 - [Java] Remove legacy vector classes
+* ARROW-1867 - [Java] Add BitVector APIs from old vector class
+* ARROW-1885 - [Java] Restore previous MapVector class names
+
+## Task
+
+* ARROW-1369 - Support boolean types in the javascript arrow reader library
+* ARROW-1818 - Examine Java Dependencies
+* ARROW-1827 - [Java] Add checkstyle config file and header file
+
+## Test
+
+* ARROW-1549 - [JS] Integrate auto-generated Arrow test files
+* ARROW-1821 - Add integration test case to explicitly check for optional validity buffer
+* ARROW-1839 - [C++/Python] Add Decimal Parquet Read/Write Tests
+
 # Apache Arrow 0.7.1 (27 September 2017)
 
 ## Bug
@@ -1286,3 +1575,4 @@
 * ARROW-83 - Add basic test infrastructure for DecimalType
 
 
+

From 1d689e5a3e417d3a4b3de16fea7a6fe195c8f8df Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Dec 2017 17:22:29 -0500
Subject: [PATCH 1340/1644] [maven-release-plugin] prepare release
 apache-arrow-0.8.0

Change-Id: Ia6278e52e2669880cbe32d79bd2fcda48951d3fa
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index cd2b6c0de6710..b05e47b95701f 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.8.0-SNAPSHOT</version>
+  <version>0.8.0</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 7efc8e6aa470c..5917634531f97 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.8.0</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 922048c018b7a..2c7078f8963ea 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.8.0-SNAPSHOT</version>
+  <version>0.8.0</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -44,7 +44,7 @@
     <connection>scm:git:https://github.com/apache/arrow.git</connection>
     <developerConnection>scm:git:https://github.com/apache/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.7.1</tag>
+    <tag>apache-arrow-0.8.0</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index f3c86688134dc..9e9d80843d828 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.8.0-SNAPSHOT</version>
+        <version>0.8.0</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 46e06aa1e3f97..05c4efbb9b101 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.8.0</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 5afae14f40a471ef6f10af4e62ce32d3a9ca4455 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 12 Dec 2017 17:22:39 -0500
Subject: [PATCH 1341/1644] [maven-release-plugin] prepare for next development
 iteration

Change-Id: I2a7909e2f0fa87780197270982ef941d89834cca
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index b05e47b95701f..3e6582d3fd639 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.8.0</version>
+  <version>0.9.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 5917634531f97..74ec45c1d2f97 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.8.0</version>
+    <version>0.9.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 2c7078f8963ea..152deaa9c428e 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.8.0</version>
+  <version>0.9.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -44,7 +44,7 @@
     <connection>scm:git:https://github.com/apache/arrow.git</connection>
     <developerConnection>scm:git:https://github.com/apache/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.8.0</tag>
+    <tag>apache-arrow-0.7.1</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 9e9d80843d828..246f264a6def8 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.8.0</version>
+        <version>0.9.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 05c4efbb9b101..0f3e03e6902da 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.8.0</version>
+    <version>0.9.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From 9e262f46a76fd02ee5226ee9e9250a917e93fe0f Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Mon, 18 Dec 2017 00:34:56 -0500
Subject: [PATCH 1342/1644] ARROW-1924: [Python] Bring back pickle=True option
 for serialization

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1420 from pcmoritz/revert-to-pickle-arg and squashes the following commits:

bfef3aeb [Philipp Moritz] fix windows test
c1566538 [Philipp Moritz] fix remote serialization test on windows
3f58d0df [Philipp Moritz] fix windows
6a2a83dd [Philipp Moritz] add regression test
3eb93258 [Philipp Moritz] fix
518fb7d8 [Philipp Moritz] fix
b4885862 [Philipp Moritz] revert to pickle=True argument for serialization
---
 python/pyarrow/serialization.pxi           | 14 ++++++++--
 python/pyarrow/serialization.py            |  7 ++---
 python/pyarrow/tests/test_serialization.py | 30 +++++++++++++++++++---
 3 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index cbc5e3b8408e4..d95d582fe537e 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -47,6 +47,7 @@ cdef class SerializationContext:
     cdef:
         object type_to_type_id
         object whitelisted_types
+        object types_to_pickle
         object custom_serializers
         object custom_deserializers
 
@@ -54,6 +55,7 @@ cdef class SerializationContext:
         # Types with special serialization handlers
         self.type_to_type_id = dict()
         self.whitelisted_types = dict()
+        self.types_to_pickle = set()
         self.custom_serializers = dict()
         self.custom_deserializers = dict()
 
@@ -73,7 +75,7 @@ cdef class SerializationContext:
 
         return result
 
-    def register_type(self, type_, type_id,
+    def register_type(self, type_, type_id, pickle=False,
                       custom_serializer=None, custom_deserializer=None):
         """EXPERIMENTAL: Add type to the list of types we can serialize.
 
@@ -83,6 +85,9 @@ cdef class SerializationContext:
             The type that we can serialize.
         type_id : bytes
             A string of bytes used to identify the type.
+        pickle : bool
+            True if the serialization should be done with pickle.
+            False if it should be done efficiently with Arrow.
         custom_serializer : callable
             This argument is optional, but can be provided to
             serialize objects of the class in a particular way.
@@ -92,6 +97,8 @@ cdef class SerializationContext:
         """
         self.type_to_type_id[type_] = type_id
         self.whitelisted_types[type_id] = type_
+        if pickle:
+            self.types_to_pickle.add(type_id)
         if custom_serializer is not None:
             self.custom_serializers[type_id] = custom_serializer
             self.custom_deserializers[type_id] = custom_deserializer
@@ -111,7 +118,9 @@ cdef class SerializationContext:
 
         # use the closest match to type(obj)
         type_id = self.type_to_type_id[type_]
-        if type_id in self.custom_serializers:
+        if type_id in self.types_to_pickle:
+            serialized_obj = {"data": pickle.dumps(obj), "pickle": True}
+        elif type_id in self.custom_serializers:
             serialized_obj = {"data": self.custom_serializers[type_id](obj)}
         else:
             if is_named_tuple(type_):
@@ -132,6 +141,7 @@ cdef class SerializationContext:
             # The object was pickled, so unpickle it.
             obj = pickle.loads(serialized_obj["data"])
         else:
+            assert type_id not in self.types_to_pickle
             if type_id not in self.whitelisted_types:
                 msg = "Type ID " + str(type_id) + " not registered in " \
                       "deserialization callback"
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 3059dfc1bfff2..689ec15d3292d 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -104,12 +104,9 @@ def _deserialize_default_dict(data):
 
     serialization_context.register_type(
         type(lambda: 0), "function",
-        custom_serializer=cloudpickle.dumps,
-        custom_deserializer=cloudpickle.loads)
+        pickle=True)
 
-    serialization_context.register_type(type, "type",
-                                        custom_serializer=cloudpickle.dumps,
-                                        custom_deserializer=cloudpickle.loads)
+    serialization_context.register_type(type, "type", pickle=True)
 
     serialization_context.register_type(
         np.ndarray, 'np.array',
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 6d85621d411a7..f245dc299cb25 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -21,9 +21,9 @@
 
 from collections import namedtuple, OrderedDict, defaultdict
 import datetime
+import os
 import string
 import sys
-import pickle
 
 import pyarrow as pa
 import numpy as np
@@ -198,9 +198,7 @@ def make_serialization_context():
     context.register_type(Baz, "Baz")
     context.register_type(Qux, "Quz")
     context.register_type(SubQux, "SubQux")
-    context.register_type(SubQuxPickle, "SubQuxPickle",
-                          custom_serializer=pickle.dumps,
-                          custom_deserializer=pickle.loads)
+    context.register_type(SubQuxPickle, "SubQuxPickle", pickle=True)
     context.register_type(Exception, "Exception")
     context.register_type(CustomError, "CustomError")
     context.register_type(Point, "Point")
@@ -519,3 +517,27 @@ def test_serialize_to_components_invalid_cases():
 
     with pytest.raises(pa.ArrowException):
         pa.deserialize_components(components)
+
+
+@pytest.mark.skipif(os.name == 'nt', reason="deserialize_regex not pickleable")
+def test_deserialize_in_different_process():
+    from multiprocessing import Process, Queue
+    import re
+
+    regex = re.compile(r"\d+\.\d*")
+
+    serialization_context = pa.SerializationContext()
+    serialization_context.register_type(type(regex), "Regex", pickle=True)
+
+    serialized = pa.serialize(regex, serialization_context)
+    serialized_bytes = serialized.to_buffer().to_pybytes()
+
+    def deserialize_regex(serialized, q):
+        import pyarrow as pa
+        q.put(pa.deserialize(serialized))
+
+    q = Queue()
+    p = Process(target=deserialize_regex, args=(serialized_bytes, q))
+    p.start()
+    assert q.get().pattern == regex.pattern
+    p.join()

From 72e660a7c90f1846eac473c21fb718a2e23c476e Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 18 Dec 2017 16:52:22 +0100
Subject: [PATCH 1343/1644] ARROW-1933: [GLib] Fix build error with
 --with-arrow-cpp-build-dir

The option is used in building deb package.

`arrow/gpu/cuda_version.h` exists in build directory not source directory.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1429 from kou/glib-fix-build-error-with-arrow-cpp-build-dir and squashes the following commits:

879d6bc [Kouhei Sutou] [GLib] Fix build error with --with-arrow-cpp-build-dir
---
 c_glib/arrow-glib/Makefile.am     |  4 ++++
 c_glib/arrow-gpu-glib/Makefile.am | 13 ++++++++++++-
 c_glib/configure.ac               | 16 ++++++++++++++--
 c_glib/example/Makefile.am        |  4 ++++
 4 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 39222a5480162..16d0703142798 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -212,6 +212,10 @@ if HAVE_INTROSPECTION
 INTROSPECTION_GIRS =
 INTROSPECTION_SCANNER_ARGS =
 INTROSPECTION_SCANNER_ENV =
+if USE_ARROW_BUILD_DIR
+INTROSPECTION_SCANNER_ENV +=			\
+	LD_LIBRARY_PATH=$(ARROW_LIB_DIR):$${PKG_CONFIG_PATH}
+endif
 if OS_MACOS
 INTROSPECTION_SCANNER_ENV +=			\
 	ARCHFLAGS=
diff --git a/c_glib/arrow-gpu-glib/Makefile.am b/c_glib/arrow-gpu-glib/Makefile.am
index 11ffa5ecbe367..1e1c02ac4a5c0 100644
--- a/c_glib/arrow-gpu-glib/Makefile.am
+++ b/c_glib/arrow-gpu-glib/Makefile.am
@@ -70,8 +70,14 @@ if HAVE_INTROSPECTION
 -include $(INTROSPECTION_MAKEFILE)
 INTROSPECTION_GIRS =
 INTROSPECTION_SCANNER_ARGS =
-INTROSPECTION_SCANNER_ENV =			\
+INTROSPECTION_SCANNER_ENV =
+if USE_ARROW_BUILD_DIR
+INTROSPECTION_SCANNER_ENV +=			\
+	PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$(ARROW_BUILD_DIR)/src/arrow:$${PKG_CONFIG_PATH}
+else
+INTROSPECTION_SCANNER_ENV +=			\
 	PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$${PKG_CONFIG_PATH}
+endif
 if OS_MACOS
 INTROSPECTION_SCANNER_ENV +=			\
 	ARCHFLAGS=
@@ -88,6 +94,11 @@ ArrowGPU_1_0_gir_INCLUDES =			\
 	Arrow-1.0
 ArrowGPU_1_0_gir_CFLAGS =			\
 	$(AM_CPPFLAGS)
+ArrowGPU_1_0_gir_LDFLAGS =
+if USE_ARROW_BUILD_DIR
+ArrowGPU_1_0_gir_LDFLAGS +=			\
+	-L$(ARROW_LIB_DIR)
+endif
 ArrowGPU_1_0_gir_LIBS =					\
 	$(abs_builddir)/../arrow-glib/libarrow-glib.la	\
 	libarrow-gpu-glib.la
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index f06a017a10ba1..eabe7bad51227 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -89,16 +89,25 @@ AC_ARG_WITH(arrow-cpp-build-dir,
   [GARROW_ARROW_CPP_BUILD_DIR="$withval"],
   [GARROW_ARROW_CPP_BUILD_DIR=""])
 if test "x$GARROW_ARROW_CPP_BUILD_DIR" = "x"; then
+  USE_ARROW_BUILD_DIR=no
+
   PKG_CHECK_MODULES([ARROW], [arrow arrow-compute])
   PKG_CHECK_MODULES([ARROW_GPU],
                     [arrow-gpu],
                     [HAVE_ARROW_GPU=yes],
                     [HAVE_ARROW_GPU=no])
 else
-  ARROW_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src"
+  USE_ARROW_BUILD_DIR=yes
+
+  ARROW_BUILD_DIR="${GARROW_ARROW_CPP_BUILD_DIR}"
+  AC_SUBST(ARROW_BUILD_DIR)
+
+  ARROW_SOURCE_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src"
+  ARROW_BUILD_INCLUDE_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/src"
   ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}"
+  AC_SUBST(ARROW_LIB_DIR)
 
-  ARROW_CFLAGS="-I${ARROW_INCLUDE_DIR}"
+  ARROW_CFLAGS="-I${ARROW_BUILD_INCLUDE_DIR} -I${ARROW_SOURCE_INCLUDE_DIR}"
   ARROW_LIBS="-L${ARROW_LIB_DIR} -larrow"
   AC_SUBST(ARROW_CFLAGS)
   AC_SUBST(ARROW_LIBS)
@@ -115,6 +124,9 @@ else
   AC_SUBST(ARROW_GPU_LIBS)
 fi
 
+AM_CONDITIONAL([USE_ARROW_BUILD_DIR],
+               [test "$USE_ARROW_BUILD_DIR" = "yes"])
+
 AM_CONDITIONAL([HAVE_ARROW_GPU], [test "$HAVE_ARROW_GPU" = "yes"])
 if test "$HAVE_ARROW_GPU" = "yes"; then
   AC_DEFINE(HAVE_ARROW_GPU, [1], [Define to 1 if Apache Arrow supports GPU.])
diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am
index 3eaf808bc5aa6..f4aca70c657f2 100644
--- a/c_glib/example/Makefile.am
+++ b/c_glib/example/Makefile.am
@@ -33,6 +33,10 @@ AM_CFLAGS =					\
 AM_LDFLAGS =						\
 	$(GLIB_LIBS)					\
 	$(builddir)/../arrow-glib/libarrow-glib.la
+if USE_ARROW_BUILD_DIR
+AM_LDFLAGS +=					\
+	$(ARROW_LIBS)
+endif
 
 noinst_PROGRAMS =				\
 	build					\

From db72bfe2b2141ac321251b29c1b63b3940623167 Mon Sep 17 00:00:00 2001
From: Jacques Nadeau <jacques@apache.org>
Date: Sun, 17 Dec 2017 21:03:39 -0800
Subject: [PATCH 1344/1644] ARROW-1932: [Website] Update site for 0.8.0 release
 info

---
 dev/release/RELEASE_MANAGEMENT.md |   2 +-
 site/_release/0.8.0.md            | 358 ++++++++++++++++++++++++++++++
 site/_release/index.md            |   2 +
 site/index.html                   |   2 +-
 site/install.md                   |  30 +--
 5 files changed, 377 insertions(+), 17 deletions(-)
 create mode 100644 site/_release/0.8.0.md

diff --git a/dev/release/RELEASE_MANAGEMENT.md b/dev/release/RELEASE_MANAGEMENT.md
index ce7835d5f5d85..73eaf5f95b3b0 100644
--- a/dev/release/RELEASE_MANAGEMENT.md
+++ b/dev/release/RELEASE_MANAGEMENT.md
@@ -63,7 +63,7 @@ in the `_release` subdirectory. The new contents of the new entry will go into
 a new Markdown file of the form `X.Y.Z.md`. You can start by copying one of the
 other release entries.
 
-Generate a web-friendly changelog by running
+Generate a web-friendly changelog by running (python3)
 
 ```
 dev/release/changelog.py $VERSION 1
diff --git a/site/_release/0.8.0.md b/site/_release/0.8.0.md
new file mode 100644
index 0000000000000..65b4210575055
--- /dev/null
+++ b/site/_release/0.8.0.md
@@ -0,0 +1,358 @@
+---
+layout: default
+title: Apache Arrow 0.8.0 Release
+permalink: /release/0.8.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.8.0 (18 December 2017)
+
+This is a major release.
+
+## Download
+
+* [**Source Artifacts**][2]
+* [Git tag][1]
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.7.1..apache-arrow-0.8.0
+    90  Wes McKinney
+    23  Phillip Cloud
+    21  Kouhei Sutou
+    13  Licht-T
+    12  Korn, Uwe
+    12  Philipp Moritz
+    12  Uwe L. Korn
+    10  Bryan Cutler
+     5  Li Jin
+     5  Robert Nishihara
+     4  Paul Taylor
+     4  siddharth
+     3  Max Risuhin
+     3  Stephanie
+     2  Rene Sugar
+     2  Heimir Sverrisson
+     2  Brian Hulette
+     2  Yuliya Feldman
+     2  dhirschf
+     2  Matthias Vallentin
+     1  vkorukanti
+     1  Andrew Andrade
+     1  Benjamin Goldberg
+     1  Ivan Sadikov
+     1  John Jenkins
+     1  Joris Van den Bossche
+     1  Lewis John McGibbney
+     1  Lu Qi
+     1  Manuel
+     1  Nick White
+     1  Ofek Lev
+     1  Shixiong Zhu
+     1  Siddharth Teotia
+     1  Stephen G
+     1  Victor Uriarte
+     1  Wataru Shimizu
+     1  ksdevlife
+     1  lmeyerov
+     1  rvernica
+     1  Amir Malekpour
+```
+
+# Changelog
+
+## New Features and Improvements
+
+* [ARROW-1032](https://issues.apache.org/jira/browse/ARROW-1032) - [JS] Support custom_metadata
+* [ARROW-1047](https://issues.apache.org/jira/browse/ARROW-1047) - [Java] Add generalized stream writer and reader interfaces that are decoupled from IO / message framing
+* [ARROW-1087](https://issues.apache.org/jira/browse/ARROW-1087) - [Python] add get_include to expose directory containing header files
+* [ARROW-1114](https://issues.apache.org/jira/browse/ARROW-1114) - [C++] Create Record Batch Builder class as a reusable and efficient way to transpose row-by-row data to columns
+* [ARROW-1134](https://issues.apache.org/jira/browse/ARROW-1134) - [C++] Allow C++/CLI projects to build with Arrow
+* [ARROW-1178](https://issues.apache.org/jira/browse/ARROW-1178) - [Python] Create alternative to Table.from_pandas that yields a list of RecordBatch objects with a given chunk size
+* [ARROW-1226](https://issues.apache.org/jira/browse/ARROW-1226) - [C++] Improve / correct doxygen function documentation in arrow::ipc
+* [ARROW-1250](https://issues.apache.org/jira/browse/ARROW-1250) - [Python] Define API for user type checking of array types
+* [ARROW-1369](https://issues.apache.org/jira/browse/ARROW-1369) - Support boolean types in the javascript arrow reader library
+* [ARROW-1371](https://issues.apache.org/jira/browse/ARROW-1371) - [Website] Add "Powered By" page to the website
+* [ARROW-1455](https://issues.apache.org/jira/browse/ARROW-1455) - [Python] Add Dockerfile for validating Dask integration outside of usual CI
+* [ARROW-1471](https://issues.apache.org/jira/browse/ARROW-1471) - [JAVA] Document requirements and non/requirements for ValueVector updates
+* [ARROW-1472](https://issues.apache.org/jira/browse/ARROW-1472) - [JAVA] Design updated ValueVector Object Hierarchy
+* [ARROW-1473](https://issues.apache.org/jira/browse/ARROW-1473) - [JAVA] Create Prototype Code Hierarchy (Implementation Phase 1)
+* [ARROW-1474](https://issues.apache.org/jira/browse/ARROW-1474) - [JAVA] ValueVector hierarchy (Implementation Phase 2)
+* [ARROW-1476](https://issues.apache.org/jira/browse/ARROW-1476) - [JAVA] Implement final ValueVector updates
+* [ARROW-1482](https://issues.apache.org/jira/browse/ARROW-1482) - [C++] Implement casts between date32 and date64
+* [ARROW-1483](https://issues.apache.org/jira/browse/ARROW-1483) - [C++] Implement casts between time32 and time64
+* [ARROW-1484](https://issues.apache.org/jira/browse/ARROW-1484) - [C++] Implement (safe and unsafe) casts between timestamps and times of different units
+* [ARROW-1486](https://issues.apache.org/jira/browse/ARROW-1486) - [C++] Decide if arrow::RecordBatch needs to be copyable
+* [ARROW-1487](https://issues.apache.org/jira/browse/ARROW-1487) - [C++] Implement casts from List\<A> to List\<B>, where a cast function is defined from any A to B
+* [ARROW-1488](https://issues.apache.org/jira/browse/ARROW-1488) - [C++] Implement ArrayBuilder::Finish in terms of internal::ArrayData
+* [ARROW-1498](https://issues.apache.org/jira/browse/ARROW-1498) - [GitHub] Add CONTRIBUTING.md and ISSUE_TEMPLATE.md
+* [ARROW-1503](https://issues.apache.org/jira/browse/ARROW-1503) - [Python] Add serialization callbacks for pandas objects in pyarrow.serialize
+* [ARROW-1522](https://issues.apache.org/jira/browse/ARROW-1522) - [C++] Support pyarrow.Buffer as built-in type in pyarrow.serialize
+* [ARROW-1523](https://issues.apache.org/jira/browse/ARROW-1523) - [C++] Add helper data struct with methods for reading a validity bitmap possibly having a non-zero offset
+* [ARROW-1524](https://issues.apache.org/jira/browse/ARROW-1524) - [C++] More graceful solution for handling non-zero offsets on inputs and outputs in compute library
+* [ARROW-1525](https://issues.apache.org/jira/browse/ARROW-1525) - [C++] Change functions in arrow/compare.h to not return Status
+* [ARROW-1526](https://issues.apache.org/jira/browse/ARROW-1526) - [Python] Unit tests to exercise code path in PARQUET-1100
+* [ARROW-1535](https://issues.apache.org/jira/browse/ARROW-1535) - [Python] Enable sdist source tarballs to build assuming that Arrow C++ libraries are available on the host system
+* [ARROW-1538](https://issues.apache.org/jira/browse/ARROW-1538) - [C++] Support Ubuntu 14.04 in .deb packaging automation
+* [ARROW-1539](https://issues.apache.org/jira/browse/ARROW-1539) - [C++] Remove functions deprecated as of 0.7.0 and prior releases
+* [ARROW-1556](https://issues.apache.org/jira/browse/ARROW-1556) - [C++] Incorporate AssertArraysEqual function from PARQUET-1100 patch
+* [ARROW-1559](https://issues.apache.org/jira/browse/ARROW-1559) - [C++] Kernel implementations for "unique" (compute distinct elements of array)
+* [ARROW-1573](https://issues.apache.org/jira/browse/ARROW-1573) - [C++] Implement stateful kernel function that uses DictionaryBuilder to compute dictionary indices
+* [ARROW-1575](https://issues.apache.org/jira/browse/ARROW-1575) - [Python] Add pyarrow.column factory function
+* [ARROW-1577](https://issues.apache.org/jira/browse/ARROW-1577) - [JS] Package release script for NPM modules
+* [ARROW-1588](https://issues.apache.org/jira/browse/ARROW-1588) - [C++/Format] Harden Decimal Format
+* [ARROW-1593](https://issues.apache.org/jira/browse/ARROW-1593) - [PYTHON] serialize_pandas should pass through the preserve_index keyword
+* [ARROW-1594](https://issues.apache.org/jira/browse/ARROW-1594) - [Python] Enable multi-threaded conversions in Table.from_pandas
+* [ARROW-1600](https://issues.apache.org/jira/browse/ARROW-1600) - [C++] Zero-copy Buffer constructor from std::string
+* [ARROW-1602](https://issues.apache.org/jira/browse/ARROW-1602) - [C++] Add IsValid/IsNotNull method to arrow::Array
+* [ARROW-1603](https://issues.apache.org/jira/browse/ARROW-1603) - [C++] Add BinaryArray method to get a value as a std::string
+* [ARROW-1604](https://issues.apache.org/jira/browse/ARROW-1604) - [Python] Support common type aliases in cast(...) and various type= arguments
+* [ARROW-1605](https://issues.apache.org/jira/browse/ARROW-1605) - [Python] pyarrow.array should be able to yield smaller integer types without an explicit cast
+* [ARROW-1607](https://issues.apache.org/jira/browse/ARROW-1607) - [C++] Implement DictionaryBuilder for Decimals
+* [ARROW-1613](https://issues.apache.org/jira/browse/ARROW-1613) - [Java] ArrowReader should not close the input ReadChannel
+* [ARROW-1616](https://issues.apache.org/jira/browse/ARROW-1616) - [Python] Add "write" method to RecordBatchStreamWriter that dispatches to write_table/write_back as appropriate
+* [ARROW-1626](https://issues.apache.org/jira/browse/ARROW-1626) - Add make targets to run the inter-procedural static analysis tool called "infer".
+* [ARROW-1627](https://issues.apache.org/jira/browse/ARROW-1627) - [JAVA] Reduce heap usage(Phase 2) - memory footprint in AllocationManager.BufferLedger
+* [ARROW-1630](https://issues.apache.org/jira/browse/ARROW-1630) - [Serialization] Support Python datetime objects
+* [ARROW-1631](https://issues.apache.org/jira/browse/ARROW-1631) -  [C++] Add GRPC to ThirdpartyToolchain.cmake
+* [ARROW-1635](https://issues.apache.org/jira/browse/ARROW-1635) - Add release management guide for PMCs
+* [ARROW-1637](https://issues.apache.org/jira/browse/ARROW-1637) - [C++] IPC round-trip for null type
+* [ARROW-1641](https://issues.apache.org/jira/browse/ARROW-1641) - [C++] Do not include <mutex> in public headers
+* [ARROW-1648](https://issues.apache.org/jira/browse/ARROW-1648) - C++: Add cast from Dictionary[NullType] to NullType
+* [ARROW-1649](https://issues.apache.org/jira/browse/ARROW-1649) - C++: Print number of nulls in PrettyPrint for NullArray
+* [ARROW-1651](https://issues.apache.org/jira/browse/ARROW-1651) - [JS] Lazy row accessor in Table
+* [ARROW-1652](https://issues.apache.org/jira/browse/ARROW-1652) - [JS] Separate Vector into BatchVector and CompositeVector
+* [ARROW-1654](https://issues.apache.org/jira/browse/ARROW-1654) - [Python] pa.DataType cannot be pickled
+* [ARROW-1662](https://issues.apache.org/jira/browse/ARROW-1662) - Move OSX Dependency management into brew bundle Brewfiles
+* [ARROW-1665](https://issues.apache.org/jira/browse/ARROW-1665) - [Serialization] Support more custom datatypes in the default serialization context
+* [ARROW-1666](https://issues.apache.org/jira/browse/ARROW-1666) - [GLib] Enable gtk-doc on Travis CI Mac environment
+* [ARROW-1667](https://issues.apache.org/jira/browse/ARROW-1667) - [GLib] Support Meson
+* [ARROW-1671](https://issues.apache.org/jira/browse/ARROW-1671) - [C++] Change arrow::MakeArray to not return Status
+* [ARROW-1675](https://issues.apache.org/jira/browse/ARROW-1675) - [Python] Use RecordBatch.from_pandas in FeatherWriter.write
+* [ARROW-1677](https://issues.apache.org/jira/browse/ARROW-1677) - [Blog] Add blog post on Ray and Arrow Python serialization
+* [ARROW-1679](https://issues.apache.org/jira/browse/ARROW-1679) - [GLib] Add garrow_record_batch_reader_read_next()
+* [ARROW-1683](https://issues.apache.org/jira/browse/ARROW-1683) - [Python] Restore "TimestampType" to pyarrow namespace
+* [ARROW-1684](https://issues.apache.org/jira/browse/ARROW-1684) - [Python] Simplify user API for reading nested Parquet columns
+* [ARROW-1685](https://issues.apache.org/jira/browse/ARROW-1685) - [GLib] Add GArrowTableReader
+* [ARROW-1689](https://issues.apache.org/jira/browse/ARROW-1689) - [Python] Categorical Indices Should Be Zero-Copy
+* [ARROW-1690](https://issues.apache.org/jira/browse/ARROW-1690) - [GLib] Add garrow_array_is_valid()
+* [ARROW-1691](https://issues.apache.org/jira/browse/ARROW-1691) - [Java] Conform Java Decimal type implementation to format decisions in ARROW-1588
+* [ARROW-1697](https://issues.apache.org/jira/browse/ARROW-1697) - [GitHub] Add ISSUE_TEMPLATE.md
+* [ARROW-1701](https://issues.apache.org/jira/browse/ARROW-1701) - [Serialization] Support zero copy PyTorch Tensor serialization
+* [ARROW-1702](https://issues.apache.org/jira/browse/ARROW-1702) - Update jemalloc in manylinux1 build
+* [ARROW-1703](https://issues.apache.org/jira/browse/ARROW-1703) - [C++] Vendor exact version of jemalloc we depend on
+* [ARROW-1707](https://issues.apache.org/jira/browse/ARROW-1707) - Update dev README after movement to GitBox
+* [ARROW-1710](https://issues.apache.org/jira/browse/ARROW-1710) - [Java] Remove non-nullable vectors in new vector class hierarchy 
+* [ARROW-1716](https://issues.apache.org/jira/browse/ARROW-1716) - [Format/JSON] Use string integer value for Decimals in JSON
+* [ARROW-1717](https://issues.apache.org/jira/browse/ARROW-1717) - [Java] Remove public static helper method in vector classes for JSONReader/Writer
+* [ARROW-1718](https://issues.apache.org/jira/browse/ARROW-1718) - [Python] Implement casts from timestamp to date32/date64 and support in Array.from_pandas
+* [ARROW-1719](https://issues.apache.org/jira/browse/ARROW-1719) - [Java] Remove accessor/mutator 
+* [ARROW-1721](https://issues.apache.org/jira/browse/ARROW-1721) - [Python] Support null mask in places where it isn't supported in numpy_to_arrow.cc
+* [ARROW-1724](https://issues.apache.org/jira/browse/ARROW-1724) - [Packaging] Support Ubuntu 17.10
+* [ARROW-1725](https://issues.apache.org/jira/browse/ARROW-1725) - [Packaging] Upload .deb for Ubuntu 17.10
+* [ARROW-1726](https://issues.apache.org/jira/browse/ARROW-1726) - [GLib] Add setup description to verify C GLib build
+* [ARROW-1727](https://issues.apache.org/jira/browse/ARROW-1727) - [Format] Expand Arrow streaming format to permit new dictionaries and deltas / additions to existing dictionaries
+* [ARROW-1728](https://issues.apache.org/jira/browse/ARROW-1728) - [C++] Run clang-format checks in Travis CI
+* [ARROW-1734](https://issues.apache.org/jira/browse/ARROW-1734) - C++/Python: Add cast function on Column-level
+* [ARROW-1736](https://issues.apache.org/jira/browse/ARROW-1736) - [GLib] Add GArrowCastOptions:allow-time-truncate
+* [ARROW-1737](https://issues.apache.org/jira/browse/ARROW-1737) - [GLib] Use G_DECLARE_DERIVABLE_TYPE
+* [ARROW-1746](https://issues.apache.org/jira/browse/ARROW-1746) - [Python] Add build dependencies for Arch Linux
+* [ARROW-1747](https://issues.apache.org/jira/browse/ARROW-1747) - [C++] Don't export symbols of statically linked libraries
+* [ARROW-1748](https://issues.apache.org/jira/browse/ARROW-1748) - [GLib] Add GArrowRecordBatchBuilder
+* [ARROW-1750](https://issues.apache.org/jira/browse/ARROW-1750) - [C++] Remove the need for arrow/util/random.h
+* [ARROW-1752](https://issues.apache.org/jira/browse/ARROW-1752) - [Packaging] Add GPU packages for Debian and Ubuntu
+* [ARROW-1753](https://issues.apache.org/jira/browse/ARROW-1753) - [Python] Provide for matching subclasses with register_type in serialization context
+* [ARROW-1755](https://issues.apache.org/jira/browse/ARROW-1755) - [C++] Add build options for MSVC to use static runtime libraries
+* [ARROW-1758](https://issues.apache.org/jira/browse/ARROW-1758) - [Python] Remove pickle=True option for object serialization
+* [ARROW-1763](https://issues.apache.org/jira/browse/ARROW-1763) - [Python] DataType should be hashable
+* [ARROW-1765](https://issues.apache.org/jira/browse/ARROW-1765) - [Doc] Use dependencies from conda in C++ docker build
+* [ARROW-1767](https://issues.apache.org/jira/browse/ARROW-1767) - [C++] Support file reads and writes over 2GB on Windows
+* [ARROW-1772](https://issues.apache.org/jira/browse/ARROW-1772) - [C++] Add public-api-test module in style of parquet-cpp
+* [ARROW-1773](https://issues.apache.org/jira/browse/ARROW-1773) - [C++] Add casts from date/time types to compatible signed integers
+* [ARROW-1775](https://issues.apache.org/jira/browse/ARROW-1775) - Ability to abort created but unsealed Plasma objects
+* [ARROW-1777](https://issues.apache.org/jira/browse/ARROW-1777) - [C++] Add static ctor ArrayData::Make for nicer syntax in places
+* [ARROW-1779](https://issues.apache.org/jira/browse/ARROW-1779) - [Java] Integration test breaks without zeroing out validity vectors
+* [ARROW-1782](https://issues.apache.org/jira/browse/ARROW-1782) - [Python] Expose compressors as pyarrow.compress, pyarrow.decompress
+* [ARROW-1783](https://issues.apache.org/jira/browse/ARROW-1783) - [Python] Convert SerializedPyObject to/from sequence of component buffers with minimal memory allocation / copying
+* [ARROW-1784](https://issues.apache.org/jira/browse/ARROW-1784) - [Python] Read and write pandas.DataFrame in pyarrow.serialize by decomposing the BlockManager rather than coercing to Arrow format
+* [ARROW-1785](https://issues.apache.org/jira/browse/ARROW-1785) - [Format/C++/Java] Remove VectorLayout metadata from Flatbuffers metadata
+* [ARROW-1787](https://issues.apache.org/jira/browse/ARROW-1787) - [Python] Support reading parquet files into DataFrames in a backward compatible way
+* [ARROW-1794](https://issues.apache.org/jira/browse/ARROW-1794) - [C++/Python] Rename DecimalArray to Decimal128Array
+* [ARROW-1801](https://issues.apache.org/jira/browse/ARROW-1801) - [Docs] Update install instructions to use red-data-tools repos
+* [ARROW-1802](https://issues.apache.org/jira/browse/ARROW-1802) - [GLib] Add Arrow GPU support
+* [ARROW-1806](https://issues.apache.org/jira/browse/ARROW-1806) - [GLib] Add garrow_record_batch_writer_write_table()
+* [ARROW-1808](https://issues.apache.org/jira/browse/ARROW-1808) - [C++] Make RecordBatch interface virtual to permit record batches that lazy-materialize columns
+* [ARROW-1809](https://issues.apache.org/jira/browse/ARROW-1809) - [GLib] Use .xml instead of .sgml for GTK-Doc main file
+* [ARROW-1810](https://issues.apache.org/jira/browse/ARROW-1810) - [Plasma] Remove test shell scripts
+* [ARROW-1817](https://issues.apache.org/jira/browse/ARROW-1817) - Configure JsonFileReader to read NaN for floats
+* [ARROW-1818](https://issues.apache.org/jira/browse/ARROW-1818) - Examine Java Dependencies
+* [ARROW-1819](https://issues.apache.org/jira/browse/ARROW-1819) - [Java] Remove legacy vector classes
+* [ARROW-1826](https://issues.apache.org/jira/browse/ARROW-1826) - [JAVA] Avoid branching at cell level (copyFrom)
+* [ARROW-1827](https://issues.apache.org/jira/browse/ARROW-1827) - [Java] Add checkstyle config file and header file
+* [ARROW-1828](https://issues.apache.org/jira/browse/ARROW-1828) - [C++] Implement hash kernel specialization for BooleanType
+* [ARROW-1834](https://issues.apache.org/jira/browse/ARROW-1834) - [Doc] Build documentation in separate build folders
+* [ARROW-1838](https://issues.apache.org/jira/browse/ARROW-1838) - [C++] Use compute::Datum uniformly for input argument to kernels
+* [ARROW-1841](https://issues.apache.org/jira/browse/ARROW-1841) - [JS] Update text-encoding-utf-8 and tslib for node ESModules support
+* [ARROW-1844](https://issues.apache.org/jira/browse/ARROW-1844) - [C++] Basic benchmark suite for hash kernels
+* [ARROW-1849](https://issues.apache.org/jira/browse/ARROW-1849) - [GLib] Add input checks to GArrowRecordBatch
+* [ARROW-1850](https://issues.apache.org/jira/browse/ARROW-1850) - [C++] Use const void* in Writable::Write instead of const uint8_t*
+* [ARROW-1854](https://issues.apache.org/jira/browse/ARROW-1854) - [Python] Improve performance of serializing object dtype ndarrays
+* [ARROW-1855](https://issues.apache.org/jira/browse/ARROW-1855) - [GLib] Add workaround for build failure on macOS
+* [ARROW-1857](https://issues.apache.org/jira/browse/ARROW-1857) - [Python] Add switch for boost linkage with static parquet in wheels
+* [ARROW-1859](https://issues.apache.org/jira/browse/ARROW-1859) - [GLib] Add GArrowDictionaryDataType
+* [ARROW-1862](https://issues.apache.org/jira/browse/ARROW-1862) - [GLib] Add GArrowDictionaryArray
+* [ARROW-1864](https://issues.apache.org/jira/browse/ARROW-1864) - [Java] Upgrade Netty to 4.1.x
+* [ARROW-1867](https://issues.apache.org/jira/browse/ARROW-1867) - [Java] Add BitVector APIs from old vector class
+* [ARROW-1874](https://issues.apache.org/jira/browse/ARROW-1874) - [GLib] Add garrow_array_unique()
+* [ARROW-1878](https://issues.apache.org/jira/browse/ARROW-1878) - [GLib] Add garrow_array_dictionary_encode()
+* [ARROW-1884](https://issues.apache.org/jira/browse/ARROW-1884) - [C++] Make JsonReader/JsonWriter classes internal APIs
+* [ARROW-1885](https://issues.apache.org/jira/browse/ARROW-1885) - [Java] Restore previous MapVector class names
+* [ARROW-1901](https://issues.apache.org/jira/browse/ARROW-1901) - [Python] Support recursive mkdir for DaskFilesystem
+* [ARROW-1902](https://issues.apache.org/jira/browse/ARROW-1902) - [Python] Remove mkdir race condition from write_to_dataset 
+* [ARROW-1905](https://issues.apache.org/jira/browse/ARROW-1905) - [Python] Add more functions for checking exact types in pyarrow.types
+* [ARROW-1911](https://issues.apache.org/jira/browse/ARROW-1911) - Add Graphistry to Arrow JS proof points
+* [ARROW-480](https://issues.apache.org/jira/browse/ARROW-480) - [Python] Add accessors for Parquet column statistics 
+* [ARROW-504](https://issues.apache.org/jira/browse/ARROW-504) - [Python] Add adapter to write pandas.DataFrame in user-selected chunk size to streaming format
+* [ARROW-507](https://issues.apache.org/jira/browse/ARROW-507) - [C++/Python] Construct List container from offsets and values subarrays
+* [ARROW-541](https://issues.apache.org/jira/browse/ARROW-541) - [JS] Implement JavaScript-compatible implementation
+* [ARROW-571](https://issues.apache.org/jira/browse/ARROW-571) - [Python] Add APIs to build Parquet files incrementally from Arrow tables
+* [ARROW-587](https://issues.apache.org/jira/browse/ARROW-587) - Add JIRA fix version to merge tool
+* [ARROW-609](https://issues.apache.org/jira/browse/ARROW-609) - [C++] Function for casting from days since UNIX epoch to int64 date
+* [ARROW-838](https://issues.apache.org/jira/browse/ARROW-838) - [Python] Efficient construction of arrays from non-pandas 1D NumPy arrays
+* [ARROW-905](https://issues.apache.org/jira/browse/ARROW-905) - [Docs] Add Dockerfile for reproducible documentation generation
+* [ARROW-942](https://issues.apache.org/jira/browse/ARROW-942) - Support integration testing on Python 2.7
+* [ARROW-950](https://issues.apache.org/jira/browse/ARROW-950) - [Site] Add Google Analytics tag
+* [ARROW-972](https://issues.apache.org/jira/browse/ARROW-972) - [Python] Add test cases and basic APIs for UnionArray
+
+## Bug Fixes
+
+* [ARROW-1282](https://issues.apache.org/jira/browse/ARROW-1282) - Large memory reallocation by Arrow causes hang in jemalloc
+* [ARROW-1341](https://issues.apache.org/jira/browse/ARROW-1341) - [C++] Deprecate arrow::MakeTable in favor of new ctor from ARROW-1334
+* [ARROW-1347](https://issues.apache.org/jira/browse/ARROW-1347) - [JAVA] List null type should use consistent name for inner field
+* [ARROW-1398](https://issues.apache.org/jira/browse/ARROW-1398) - [Python] No support reading columns of type decimal(19,4)
+* [ARROW-1409](https://issues.apache.org/jira/browse/ARROW-1409) - [Format] Use for "page" attribute in Buffer in metadata
+* [ARROW-1540](https://issues.apache.org/jira/browse/ARROW-1540) - [C++] Fix valgrind warnings in cuda-test if possible
+* [ARROW-1541](https://issues.apache.org/jira/browse/ARROW-1541) - [C++] Race condition with arrow_gpu 
+* [ARROW-1543](https://issues.apache.org/jira/browse/ARROW-1543) - [C++] row_wise_conversion example doesn't correspond to ListBuilder constructor arguments
+* [ARROW-1549](https://issues.apache.org/jira/browse/ARROW-1549) - [JS] Integrate auto-generated Arrow test files
+* [ARROW-1555](https://issues.apache.org/jira/browse/ARROW-1555) - [Python] write_to_dataset on s3
+* [ARROW-1584](https://issues.apache.org/jira/browse/ARROW-1584) - [PYTHON] serialize_pandas on empty dataframe
+* [ARROW-1585](https://issues.apache.org/jira/browse/ARROW-1585) - serialize_pandas round trip fails on integer columns
+* [ARROW-1586](https://issues.apache.org/jira/browse/ARROW-1586) - [PYTHON] serialize_pandas roundtrip loses columns name
+* [ARROW-1609](https://issues.apache.org/jira/browse/ARROW-1609) - Plasma: Build fails with Xcode 9.0
+* [ARROW-1615](https://issues.apache.org/jira/browse/ARROW-1615) - CXX flags for development more permissive than Travis CI builds
+* [ARROW-1617](https://issues.apache.org/jira/browse/ARROW-1617) - [Python] Do not use symlinks in python/cmake_modules 
+* [ARROW-1620](https://issues.apache.org/jira/browse/ARROW-1620) - Python: Download Boost in manylinux1 build from bintray
+* [ARROW-1624](https://issues.apache.org/jira/browse/ARROW-1624) - [C++] Follow up fixes / tweaks to compiler warnings for Plasma / LLVM 4.0, add to readme
+* [ARROW-1625](https://issues.apache.org/jira/browse/ARROW-1625) - [Serialization] Support OrderedDict properly
+* [ARROW-1629](https://issues.apache.org/jira/browse/ARROW-1629) - [C++] Fix problematic code paths identified by infer tool
+* [ARROW-1633](https://issues.apache.org/jira/browse/ARROW-1633) - [Python] numpy "unicode" arrays not understood
+* [ARROW-1640](https://issues.apache.org/jira/browse/ARROW-1640) - Resolve OpenSSL issues in Travis CI
+* [ARROW-1647](https://issues.apache.org/jira/browse/ARROW-1647) - [Plasma] Potential bug when reading/writing messages.
+* [ARROW-1653](https://issues.apache.org/jira/browse/ARROW-1653) - [Plasma] Use static cast to avoid compiler warning.
+* [ARROW-1656](https://issues.apache.org/jira/browse/ARROW-1656) - [C++] Endianness Macro is Incorrect on Windows And Mac
+* [ARROW-1657](https://issues.apache.org/jira/browse/ARROW-1657) - [C++] Multithreaded Read Test Failing on Arch Linux
+* [ARROW-1658](https://issues.apache.org/jira/browse/ARROW-1658) - [Python] Out of bounds dictionary indices causes segfault after converting to pandas
+* [ARROW-1663](https://issues.apache.org/jira/browse/ARROW-1663) - [Java] Follow up on ARROW-1347 and make schema backward compatible
+* [ARROW-1670](https://issues.apache.org/jira/browse/ARROW-1670) - [Python] Speed up deserialization code path
+* [ARROW-1672](https://issues.apache.org/jira/browse/ARROW-1672) - [Python] Failure to write Feather bytes column
+* [ARROW-1673](https://issues.apache.org/jira/browse/ARROW-1673) - [Python] NumPy boolean arrays get converted to uint8 arrays on NdarrayToTensor roundtrip
+* [ARROW-1676](https://issues.apache.org/jira/browse/ARROW-1676) - [C++] Correctly truncate oversized validity bitmaps when writing Feather format
+* [ARROW-1678](https://issues.apache.org/jira/browse/ARROW-1678) - [Python] Incorrect serialization of numpy.float16
+* [ARROW-1680](https://issues.apache.org/jira/browse/ARROW-1680) - [Python] Timestamp unit change not done in from_pandas() conversion
+* [ARROW-1686](https://issues.apache.org/jira/browse/ARROW-1686) - Documentation generation script creates "apidocs" directory under site/java
+* [ARROW-1693](https://issues.apache.org/jira/browse/ARROW-1693) - [JS] Error reading dictionary-encoded integration test files
+* [ARROW-1695](https://issues.apache.org/jira/browse/ARROW-1695) - [Serialization] Fix reference counting of numpy arrays created in custom serialializer
+* [ARROW-1698](https://issues.apache.org/jira/browse/ARROW-1698) - [JS] File reader attempts to load the same dictionary batch more than once
+* [ARROW-1704](https://issues.apache.org/jira/browse/ARROW-1704) - [GLib] Go example in test suite is broken
+* [ARROW-1708](https://issues.apache.org/jira/browse/ARROW-1708) - [JS] Linter problem breaks master build
+* [ARROW-1709](https://issues.apache.org/jira/browse/ARROW-1709) - [C++] Decimal.ToString is incorrect for negative scale
+* [ARROW-1711](https://issues.apache.org/jira/browse/ARROW-1711) - [Python] flake8 checks still not failing builds
+* [ARROW-1714](https://issues.apache.org/jira/browse/ARROW-1714) - [Python] No named pd.Series name serialized as u'None'
+* [ARROW-1720](https://issues.apache.org/jira/browse/ARROW-1720) - [Python] Segmentation fault while trying to access an out-of-bound chunk
+* [ARROW-1723](https://issues.apache.org/jira/browse/ARROW-1723) - Windows: __declspec(dllexport) specified when building arrow static library
+* [ARROW-1730](https://issues.apache.org/jira/browse/ARROW-1730) - [Python] Incorrect result from pyarrow.array when passing timestamp type
+* [ARROW-1732](https://issues.apache.org/jira/browse/ARROW-1732) - [Python] RecordBatch.from_pandas fails on DataFrame with no columns when preserve_index=False
+* [ARROW-1735](https://issues.apache.org/jira/browse/ARROW-1735) - [C++] Cast kernels cannot write into sliced output array
+* [ARROW-1738](https://issues.apache.org/jira/browse/ARROW-1738) - [Python] Wrong datetime conversion when pa.array with unit
+* [ARROW-1739](https://issues.apache.org/jira/browse/ARROW-1739) - [Python] Fix usages of assertRaises causing broken build
+* [ARROW-1742](https://issues.apache.org/jira/browse/ARROW-1742) - C++: clang-format is not detected correct on OSX anymore
+* [ARROW-1743](https://issues.apache.org/jira/browse/ARROW-1743) - [Python] Table to_pandas fails when index contains categorical column
+* [ARROW-1745](https://issues.apache.org/jira/browse/ARROW-1745) - Compilation failure on Mac OS in plasma tests
+* [ARROW-1749](https://issues.apache.org/jira/browse/ARROW-1749) - [C++] Handle range of Decimal128 values that require 39 digits to be displayed
+* [ARROW-1751](https://issues.apache.org/jira/browse/ARROW-1751) - [Python] Pandas 0.21.0 introduces a breaking API change for MultiIndex construction
+* [ARROW-1754](https://issues.apache.org/jira/browse/ARROW-1754) - [Python] Fix buggy Parquet roundtrip when an index name is the same as a column name
+* [ARROW-1756](https://issues.apache.org/jira/browse/ARROW-1756) - [Python] Observed int32 overflow in Feather write/read path
+* [ARROW-1762](https://issues.apache.org/jira/browse/ARROW-1762) - [C++] unittest failure for language environment
+* [ARROW-1764](https://issues.apache.org/jira/browse/ARROW-1764) - [Python] Add -c conda-forge for Windows dev installation instructions
+* [ARROW-1766](https://issues.apache.org/jira/browse/ARROW-1766) - [GLib] Fix failing builds on OSX
+* [ARROW-1768](https://issues.apache.org/jira/browse/ARROW-1768) - [Python] Fix suppressed exception in ParquetWriter.__del__
+* [ARROW-1770](https://issues.apache.org/jira/browse/ARROW-1770) - [GLib] Fix GLib compiler warning
+* [ARROW-1771](https://issues.apache.org/jira/browse/ARROW-1771) - [C++] ARROW-1749 Breaks Public API test in parquet-cpp
+* [ARROW-1776](https://issues.apache.org/jira/browse/ARROW-1776) - [C++[ arrow::gpu::CudaContext::bytes_allocated() isn't defined
+* [ARROW-1778](https://issues.apache.org/jira/browse/ARROW-1778) - [Python] Link parquet-cpp statically, privately in manylinux1 wheels
+* [ARROW-1781](https://issues.apache.org/jira/browse/ARROW-1781) - [CI] OSX Builds on Travis-CI time out often
+* [ARROW-1788](https://issues.apache.org/jira/browse/ARROW-1788) - Plasma store crashes when trying to abort objects for disconnected client
+* [ARROW-1791](https://issues.apache.org/jira/browse/ARROW-1791) - Integration tests generate date[DAY] values outside of reasonable range
+* [ARROW-1793](https://issues.apache.org/jira/browse/ARROW-1793) - [Integration] fix a typo for README.md 
+* [ARROW-1800](https://issues.apache.org/jira/browse/ARROW-1800) - [C++] Fix and simplify random_decimals
+* [ARROW-1805](https://issues.apache.org/jira/browse/ARROW-1805) - [Python] ignore non-parquet files when exploring dataset
+* [ARROW-1811](https://issues.apache.org/jira/browse/ARROW-1811) - [C++/Python] Rename all Decimal based APIs to Decimal128
+* [ARROW-1812](https://issues.apache.org/jira/browse/ARROW-1812) - Plasma store modifies hash table while iterating during client disconnect
+* [ARROW-1821](https://issues.apache.org/jira/browse/ARROW-1821) - Add integration test case to explicitly check for optional validity buffer
+* [ARROW-1829](https://issues.apache.org/jira/browse/ARROW-1829) - [Plasma] Clean up eviction policy bookkeeping
+* [ARROW-1830](https://issues.apache.org/jira/browse/ARROW-1830) - [Python] Error when loading all the files in a dictionary
+* [ARROW-1836](https://issues.apache.org/jira/browse/ARROW-1836) - [C++] Fix C4996 warning from arrow/util/variant.h on MSVC builds
+* [ARROW-1839](https://issues.apache.org/jira/browse/ARROW-1839) - [C++/Python] Add Decimal Parquet Read/Write Tests
+* [ARROW-1840](https://issues.apache.org/jira/browse/ARROW-1840) - [Website] The installation command failed on Windows10 anaconda environment.
+* [ARROW-1845](https://issues.apache.org/jira/browse/ARROW-1845) - [Python] Expose Decimal128Type
+* [ARROW-1852](https://issues.apache.org/jira/browse/ARROW-1852) - [Plasma] Make retrieving manager file descriptor const
+* [ARROW-1853](https://issues.apache.org/jira/browse/ARROW-1853) - [Plasma] Fix off-by-one error in retry processing
+* [ARROW-1863](https://issues.apache.org/jira/browse/ARROW-1863) - [Python] PyObjectStringify could render bytes-like output for more types of objects
+* [ARROW-1865](https://issues.apache.org/jira/browse/ARROW-1865) - [C++] Adding a column to an empty Table fails
+* [ARROW-1869](https://issues.apache.org/jira/browse/ARROW-1869) - Fix typo in LowCostIdentityHashMap
+* [ARROW-1871](https://issues.apache.org/jira/browse/ARROW-1871) - [Python/C++] Appending Python Decimals with different scales requires rescaling
+* [ARROW-1873](https://issues.apache.org/jira/browse/ARROW-1873) - [Python] Segmentation fault when loading total 2GB of parquet files
+* [ARROW-1877](https://issues.apache.org/jira/browse/ARROW-1877) - Incorrect comparison in JsonStringArrayList.equals
+* [ARROW-1879](https://issues.apache.org/jira/browse/ARROW-1879) - [Python] Dask integration tests are not skipped if dask is not installed
+* [ARROW-1881](https://issues.apache.org/jira/browse/ARROW-1881) - [Python] setuptools_scm picks up JS version tags
+* [ARROW-1882](https://issues.apache.org/jira/browse/ARROW-1882) - [C++] Reintroduce DictionaryBuilder
+* [ARROW-1883](https://issues.apache.org/jira/browse/ARROW-1883) - [Python] BUG: Table.to_pandas metadata checking fails if columns are not present
+* [ARROW-1889](https://issues.apache.org/jira/browse/ARROW-1889) - [Python] --exclude is not available in older git versions
+* [ARROW-1890](https://issues.apache.org/jira/browse/ARROW-1890) - [Python] Masking for date32 arrays not working
+* [ARROW-1891](https://issues.apache.org/jira/browse/ARROW-1891) - [Python] NaT date32 values are only converted to nulls if from_pandas is used
+* [ARROW-1892](https://issues.apache.org/jira/browse/ARROW-1892) - [Python] Unknown list item type: binary
+* [ARROW-1893](https://issues.apache.org/jira/browse/ARROW-1893) - [Python] test_primitive_serialization fails on Python 2.7.3
+* [ARROW-1895](https://issues.apache.org/jira/browse/ARROW-1895) - [Python] Add field_name to pandas index metadata
+* [ARROW-1897](https://issues.apache.org/jira/browse/ARROW-1897) - [Python] Incorrect numpy_type for pandas metadata of Categoricals
+* [ARROW-1904](https://issues.apache.org/jira/browse/ARROW-1904) - [C++] Deprecate PrimitiveArray::raw_values
+* [ARROW-1906](https://issues.apache.org/jira/browse/ARROW-1906) - [Python] Creating a pyarrow.Array with timestamp of different unit is not casted
+* [ARROW-1908](https://issues.apache.org/jira/browse/ARROW-1908) - [Python] Construction of arrow table from pandas DataFrame with duplicate column names crashes
+* [ARROW-1910](https://issues.apache.org/jira/browse/ARROW-1910) - CPP README Brewfile link incorrect
+* [ARROW-1914](https://issues.apache.org/jira/browse/ARROW-1914) - [C++] make -j may fail to build with -DARROW_GPU=on
+* [ARROW-1915](https://issues.apache.org/jira/browse/ARROW-1915) - [Python] Parquet tests should be optional
+* [ARROW-1916](https://issues.apache.org/jira/browse/ARROW-1916) - [Java] Do not exclude java/dev/checkstyle from source releases
+* [ARROW-1917](https://issues.apache.org/jira/browse/ARROW-1917) - [GLib] Must set GI_TYPELIB_PATH in verify-release-candidate.sh
+* [ARROW-226](https://issues.apache.org/jira/browse/ARROW-226) - [C++] libhdfs: feedback to help determining cause of failure in opening file path
+* [ARROW-641](https://issues.apache.org/jira/browse/ARROW-641) - [C++] Do not build/run io-hdfs-test if ARROW_HDFS=off
+
+[1]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0
+[2]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/
diff --git a/site/_release/index.md b/site/_release/index.md
index 4752e980b1916..e22df92b6cb41 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -26,6 +26,7 @@ limitations under the License.
 
 Navigate to the release page for downloads and the changelog.
 
+* [0.8.0 (18 December 2017)][10]
 * [0.7.1 (1 October 2017)][9]
 * [0.7.0 (17 September 2017)][8]
 * [0.6.0 (14 August 2017)][7]
@@ -45,3 +46,4 @@ Navigate to the release page for downloads and the changelog.
 [7]: {{ site.baseurl }}/release/0.6.0.html
 [8]: {{ site.baseurl }}/release/0.7.0.html
 [9]: {{ site.baseurl }}/release/0.7.1.html
+[10]: {{ site.baseurl }}/release/0.8.0.html
diff --git a/site/index.html b/site/index.html
index 54f67c1a245d0..ffa8d54af2f4f 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,7 +7,7 @@ <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
         <p>
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
-          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.7.1 Release - October 1, 2017)</a>
+          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.8.0 Release - December 18, 2017)</a>
         </p>
       </div>
       <h4><a href="{{ site.baseurl }}/blog/"><strong>See Latest News</strong></a></h4>
diff --git a/site/install.md b/site/install.md
index 67b26983b46c4..6e67d790a9d31 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,17 +20,17 @@ limitations under the License.
 {% endcomment %}
 -->
 
-## Current Version: 0.7.1
+## Current Version: 0.8.0
 
-### Released: 1 October 2017
+### Released: 18 December 2017
 
 See the [release notes][10] for more about what's new.
 
 ### Source release
 
-* **Source Release**: [apache-arrow-0.7.1.tar.gz][6]
+* **Source Release**: [apache-arrow-0.8.0.tar.gz][6]
 * **Verification**: [sha512][3], [asc][7]
-* [Git tag 0e21f84][2]
+* [Git tag 1d689e5][2]
 
 ### Java Packages
 
@@ -52,8 +52,8 @@ Install them with:
 
 
 ```shell
-conda install arrow-cpp=0.7.* -c conda-forge
-conda install pyarrow=0.7.* -c conda-forge
+conda install arrow-cpp=0.8.* -c conda-forge
+conda install pyarrow=0.8.* -c conda-forge
 ```
 
 ### Python Wheels on PyPI (Unofficial)
@@ -61,10 +61,10 @@ conda install pyarrow=0.7.* -c conda-forge
 We have provided binary wheels on PyPI for Linux, macOS, and Windows:
 
 ```shell
-pip install pyarrow==0.7.*
+pip install pyarrow==0.8.*
 ```
 
-We recommend pinning `0.7.*` in `requirements.txt` to install the latest patch
+We recommend pinning `0.8.*` in `requirements.txt` to install the latest patch
 release.
 
 These include the Apache Arrow and Apache Parquet C++ binary libraries bundled
@@ -157,13 +157,13 @@ conda install arrow-cpp -c twosigma
 conda install pyarrow -c twosigma
 ```
 
-[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.7.1
-[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz.sha512
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.7.1%22
+[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0
+[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.8.0%22
 [5]: http://conda-forge.github.io
-[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz
-[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz
+[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
-[10]: http://arrow.apache.org/release/0.7.1.html
+[10]: http://arrow.apache.org/release/0.8.0.html

From 44c5a7c77a8bbc33916aeb718d31027a518a88b7 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Mon, 18 Dec 2017 22:46:19 -0500
Subject: [PATCH 1345/1644] ARROW-1926: [GLib] Add
 garrow_timestamp_data_type_get_unit()

This is not a 0.8.0 broker.

If 0.8.0 RC2 is dropped, I hope that 0.8.0 includes this.
If RC2 has no problem, I hope that 0.9.0 includes this.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1424 from kou/glib-add-timestamp-data-type-get-unit and squashes the following commits:

a88771b2 [Kouhei Sutou] [GLib] Add garrow_timestamp_data_type_get_unit()
---
 c_glib/arrow-glib/basic-data-type.cpp   | 18 ++++++++++++++++++
 c_glib/arrow-glib/basic-data-type.h     |  2 ++
 c_glib/test/test-timestamp-data-type.rb | 16 ++++++++++++++++
 3 files changed, 36 insertions(+)

diff --git a/c_glib/arrow-glib/basic-data-type.cpp b/c_glib/arrow-glib/basic-data-type.cpp
index dcafe06d67581..a5f7aed1b8f76 100644
--- a/c_glib/arrow-glib/basic-data-type.cpp
+++ b/c_glib/arrow-glib/basic-data-type.cpp
@@ -805,6 +805,24 @@ garrow_timestamp_data_type_new(GArrowTimeUnit unit)
   return data_type;
 }
 
+/**
+ * garrow_timestamp_data_type_get_unit:
+ * @timestamp_data_type: The #GArrowTimestampDataType.
+ *
+ * Returns: The unit of the timestamp data type.
+ *
+ * Since: 0.8.0
+ */
+GArrowTimeUnit
+garrow_timestamp_data_type_get_unit(GArrowTimestampDataType *timestamp_data_type)
+{
+  const auto arrow_data_type =
+    garrow_data_type_get_raw(GARROW_DATA_TYPE(timestamp_data_type));
+  const auto arrow_timestamp_data_type =
+    std::static_pointer_cast<arrow::TimestampType>(arrow_data_type);
+  return garrow_time_unit_from_raw(arrow_timestamp_data_type->unit());
+}
+
 
 G_DEFINE_TYPE(GArrowTimeDataType,               \
               garrow_time_data_type,            \
diff --git a/c_glib/arrow-glib/basic-data-type.h b/c_glib/arrow-glib/basic-data-type.h
index bc0192f1b0062..469590191d4e1 100644
--- a/c_glib/arrow-glib/basic-data-type.h
+++ b/c_glib/arrow-glib/basic-data-type.h
@@ -774,6 +774,8 @@ struct _GArrowTimestampDataTypeClass
 
 GType                 garrow_timestamp_data_type_get_type (void) G_GNUC_CONST;
 GArrowTimestampDataType *garrow_timestamp_data_type_new   (GArrowTimeUnit unit);
+GArrowTimeUnit
+garrow_timestamp_data_type_get_unit (GArrowTimestampDataType *timestamp_data_type);
 
 
 #define GARROW_TYPE_TIME_DATA_TYPE              \
diff --git a/c_glib/test/test-timestamp-data-type.rb b/c_glib/test/test-timestamp-data-type.rb
index 83038876926f0..bbc597c160ebc 100644
--- a/c_glib/test/test-timestamp-data-type.rb
+++ b/c_glib/test/test-timestamp-data-type.rb
@@ -29,6 +29,10 @@ def setup
     def test_to_s
       assert_equal("timestamp[s]", @data_type.to_s)
     end
+
+    def test_unit
+      assert_equal(Arrow::TimeUnit::SECOND, @data_type.unit)
+    end
   end
 
   sub_test_case("millisecond") do
@@ -39,6 +43,10 @@ def setup
     def test_to_s
       assert_equal("timestamp[ms]", @data_type.to_s)
     end
+
+    def test_unit
+      assert_equal(Arrow::TimeUnit::MILLI, @data_type.unit)
+    end
   end
 
   sub_test_case("micro") do
@@ -49,6 +57,10 @@ def setup
     def test_to_s
       assert_equal("timestamp[us]", @data_type.to_s)
     end
+
+    def test_unit
+      assert_equal(Arrow::TimeUnit::MICRO, @data_type.unit)
+    end
   end
 
   sub_test_case("nano") do
@@ -59,5 +71,9 @@ def setup
     def test_to_s
       assert_equal("timestamp[ns]", @data_type.to_s)
     end
+
+    def test_unit
+      assert_equal(Arrow::TimeUnit::NANO, @data_type.unit)
+    end
   end
 end

From 9335e28f2be9da7003638b690cefbe9dc48dc469 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 18 Dec 2017 22:53:28 -0500
Subject: [PATCH 1346/1644] ARROW-1832: [JS] Implement JSON reader for
 integration tests

Add JSON reader, as well as `js/bin/integration.js` script for running integration test validation

Author: Paul Taylor <paul.e.taylor@me.com>
Author: Brian Hulette <hulettbh@gmail.com>
Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1343 from TheNeuralBit/json-reader and squashes the following commits:

bd6e80cf [Paul Taylor] print correct error messages
f1a51bdd [Paul Taylor] update example html file for new API
bb0059b8 [Paul Taylor] fix off by one error reading buffers from metadata v3 arrows
6da297ca [Paul Taylor] update CI and JS integration scripts to invoke jest integration tests
03d82bd4 [Paul Taylor] add integration tests to gulp test task
937dbf84 [Paul Taylor] split out integration and unit tests
e0354a76 [Paul Taylor] quote enum keys to save from mangling, add JSON customMetadata map
c49f43a1 [Paul Taylor] use string indexers to protect JSON fields from closure compiler's mangler, fix es5 umd build
97f8e5e4 [Paul Taylor] really flatten buffers from json
7b6ea0a3 [Paul Taylor] fix a few json reader typos
fa352ead [Paul Taylor] update tests
f81dcb9c [Paul Taylor] move arrow2csv into src so it's distributed in the npm packages
b5f14702 [Paul Taylor] add Arrow types AST, refactor buffer + json reader to emit type AST nodes
85bf03c9 [Brian Hulette] linter fixes
793f9e56 [Brian Hulette] only use json-bignum in bin/integration
eaa5de4a [Brian Hulette] add dictionary-encoded vectors
12f99de0 [Brian Hulette] Add JSON support for Date/Time/Timestamp vectors
53490806 [Brian Hulette] move test data creation after integration_test.py
68c23494 [Paul Taylor] update npm script name in integration runner
f50356e0 [Paul Taylor] run the js build before integration.py
dcc85f92 [Brian Hulette] linter fixes
313cd587 [Brian Hulette] Add int-test to test-task
86b53b47 [Brian Hulette] Use Int128.fromString in JSON reader
148e997f [Brian Hulette] cleanup
a2befbb4 [Brian Hulette] Switch endianness of Int64/128
a1ea88f8 [Brian Hulette] Now uses Uint32 for all internal buffers
02a7838d [Brian Hulette] WIP Int64, Int128
645b844b [Brian Hulette] JS integration script uses new Table.from for JSON
c1f3f6ad [Paul Taylor] move createTypedArray and createValidityArray to VectorReaderContext
1191a277 [Paul Taylor] refactor `Table.from()` to accept a JSON object or string
02ea8a67 [Paul Taylor] refactor traits to be compatible with closure compiler's full ES6 -> ES5
01de162b [Paul Taylor] move generated format to format/fb folder, fix closure compiler es5 build
ad417411 [Brian Hulette] Fix bug with zero-length vectors
b17367cc [Brian Hulette] Add list,struct to JSON reader
e3d6d622 [Brian Hulette] linter fixes
1e64707c [Brian Hulette] Add JS integration script and integration_test.py JS runner
7e33b1c4 [Brian Hulette] Add JSON reader
---
 ci/travis_script_integration.sh               |  26 +-
 integration/integration_test.py               |  47 +-
 js/bin/integration.js                         |  86 ++++
 js/examples/read_file.html                    |   8 +-
 js/gulp/argv.js                               |   6 +-
 js/gulp/closure-task.js                       |  29 +-
 js/gulp/package-task.js                       |   7 +-
 js/gulp/test-task.js                          |  15 +-
 js/gulp/typescript-task.js                    |   4 +-
 js/gulp/util.js                               |   5 +-
 js/gulpfile.js                                |   4 +-
 js/package.json                               |  46 +-
 js/src/Arrow.ts                               |  17 +-
 js/{bin/arrow2csv.js => src/bin/arrow2csv.ts} |  78 +++-
 js/src/format/arrow.ts                        |  32 ++
 js/src/format/fb.ts                           | 234 ++++++++++
 js/src/format/{ => fb}/File.ts                |   0
 js/src/format/{ => fb}/File_generated.js      |   0
 js/src/format/{ => fb}/Message.ts             |   0
 js/src/format/{ => fb}/Message_generated.js   |   0
 js/src/format/{ => fb}/Schema.ts              |   0
 js/src/format/{ => fb}/Schema_generated.js    |  82 ++--
 js/src/format/json.ts                         | 173 ++++++++
 js/src/format/types.ts                        | 393 +++++++++++++++++
 js/src/reader/arrow.ts                        | 189 +-------
 js/src/reader/buffer.ts                       | 229 ++++++++++
 js/src/reader/format.ts                       | 112 -----
 js/src/reader/json.ts                         | 176 ++++++++
 js/src/reader/vector.ts                       | 412 +++++++++---------
 js/src/util/int.ts                            | 320 ++++++++++++++
 js/src/util/layout.ts                         | 200 +++++++++
 js/src/vector/arrow.ts                        | 267 +++---------
 js/src/vector/table.ts                        |  28 +-
 js/src/vector/traits/field.ts                 |  71 +++
 js/src/vector/{traits.ts => traits/mixins.ts} |  46 +-
 js/src/vector/traits/nullable.ts              |  70 +++
 js/src/vector/traits/nullablefield.ts         |  71 +++
 js/src/vector/traits/vectors.ts               |  75 ++++
 js/src/vector/vector.ts                       |   3 +-
 js/test/Arrow.ts                              |   3 +-
 js/test/{ => integration}/test-config.ts      |   0
 .../validate-tests.ts}                        |  74 ++--
 js/test/reader-tests.ts                       |  69 ---
 js/test/table-tests.ts                        | 136 ------
 js/test/unit/int-tests.ts                     | 227 ++++++++++
 js/test/{ => unit}/vector-tests.ts            |   5 +-
 js/tsconfig/tsconfig.es5.cls.json             |   6 +-
 47 files changed, 2978 insertions(+), 1103 deletions(-)
 create mode 100755 js/bin/integration.js
 rename js/{bin/arrow2csv.js => src/bin/arrow2csv.ts} (53%)
 mode change 100755 => 100644
 create mode 100644 js/src/format/arrow.ts
 create mode 100644 js/src/format/fb.ts
 rename js/src/format/{ => fb}/File.ts (100%)
 rename js/src/format/{ => fb}/File_generated.js (100%)
 rename js/src/format/{ => fb}/Message.ts (100%)
 rename js/src/format/{ => fb}/Message_generated.js (100%)
 rename js/src/format/{ => fb}/Schema.ts (100%)
 rename js/src/format/{ => fb}/Schema_generated.js (97%)
 create mode 100644 js/src/format/json.ts
 create mode 100644 js/src/format/types.ts
 create mode 100644 js/src/reader/buffer.ts
 delete mode 100644 js/src/reader/format.ts
 create mode 100644 js/src/reader/json.ts
 create mode 100644 js/src/util/int.ts
 create mode 100644 js/src/util/layout.ts
 create mode 100644 js/src/vector/traits/field.ts
 rename js/src/vector/{traits.ts => traits/mixins.ts} (67%)
 create mode 100644 js/src/vector/traits/nullable.ts
 create mode 100644 js/src/vector/traits/nullablefield.ts
 create mode 100644 js/src/vector/traits/vectors.ts
 rename js/test/{ => integration}/test-config.ts (100%)
 rename js/test/{integration-tests.ts => integration/validate-tests.ts} (58%)
 delete mode 100644 js/test/reader-tests.ts
 delete mode 100644 js/test/table-tests.ts
 create mode 100644 js/test/unit/int-tests.ts
 rename js/test/{ => unit}/vector-tests.ts (99%)

diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 9514a30a88b08..0c415dc4865cd 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -20,6 +20,7 @@
 set -e
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug
 
 pushd $ARROW_JAVA_DIR
 
@@ -28,8 +29,15 @@ mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)
 
 popd
 
+pushd $ARROW_JS_DIR
+
+# lint and compile JS source
+npm run lint
+npm run build
+
+popd
+
 pushd $ARROW_INTEGRATION_DIR
-export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug
 
 CONDA_ENV_NAME=arrow-integration-test
 conda create -y -q -n $CONDA_ENV_NAME python=3.5
@@ -45,21 +53,11 @@ python integration_test.py --debug
 
 popd
 
-pushd $ARROW_JS_DIR
+# pushd $ARROW_JS_DIR
 
-# lint and compile JS source
-npm run lint
-npm run build
-# create initial test data
-npm run create:testdata
-# run once to write the snapshots
-npm test -- -t ts -u --integration
-# run again to test all builds against the snapshots
-npm test -- --integration
 # run tests against source to generate coverage data
-npm run test:coverage -- --integration
+# npm run test:coverage
 # Uncomment to upload to coveralls
 # cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
 
-
-popd
\ No newline at end of file
+# popd
\ No newline at end of file
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 5468ab19de8dc..79b098a614a6c 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -868,8 +868,8 @@ def __init__(self, json_files, testers, debug=False):
         self.debug = debug
 
     def run(self):
-        for producer, consumer in itertools.product(self.testers,
-                                                    self.testers):
+        for producer, consumer in itertools.product(filter(lambda t: t.PRODUCER, self.testers),
+                                                    filter(lambda t: t.CONSUMER, self.testers)):
             self._compare_implementations(producer, consumer)
 
     def _compare_implementations(self, producer, consumer):
@@ -909,6 +909,8 @@ def _compare_implementations(self, producer, consumer):
 
 
 class Tester(object):
+    PRODUCER = False
+    CONSUMER = False
 
     def __init__(self, debug=False):
         self.debug = debug
@@ -927,6 +929,8 @@ def validate(self, json_path, arrow_path):
 
 
 class JavaTester(Tester):
+    PRODUCER = True
+    CONSUMER = True
 
     _arrow_version = load_version_from_pom()
     ARROW_TOOLS_JAR = os.environ.get(
@@ -978,6 +982,8 @@ def file_to_stream(self, file_path, stream_path):
 
 
 class CPPTester(Tester):
+    PRODUCER = True
+    CONSUMER = True
 
     EXE_PATH = os.environ.get(
         'ARROW_CPP_EXE_PATH',
@@ -1025,6 +1031,41 @@ def file_to_stream(self, file_path, stream_path):
             print(cmd)
         os.system(cmd)
 
+class JSTester(Tester):
+    PRODUCER = False
+    CONSUMER = True
+
+    INTEGRATION_EXE = os.path.join(ARROW_HOME, 'js/bin/integration.js')
+
+    name = 'JS'
+
+    def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
+        cmd = [self.INTEGRATION_EXE]
+
+        if arrow_path is not None:
+            cmd.extend(['-a', arrow_path])
+
+        if json_path is not None:
+            cmd.extend(['-j', json_path])
+
+        cmd.extend(['--mode', command])
+
+        if self.debug:
+            print(' '.join(cmd))
+
+        run_cmd(cmd)
+
+    def validate(self, json_path, arrow_path):
+        return self._run(arrow_path, json_path, 'VALIDATE')
+
+    def stream_to_file(self, stream_path, file_path):
+        # Just copy stream to file, we can read the stream directly
+        cmd = ['cp', stream_path, file_path]
+        cmd = ' '.join(cmd)
+        if self.debug:
+            print(cmd)
+        os.system(cmd)
+
 
 def get_static_json_files():
     glob_pattern = os.path.join(ARROW_HOME, 'integration', 'data', '*.json')
@@ -1032,7 +1073,7 @@ def get_static_json_files():
 
 
 def run_all_tests(debug=False):
-    testers = [CPPTester(debug=debug), JavaTester(debug=debug)]
+    testers = [CPPTester(debug=debug), JavaTester(debug=debug), JSTester(debug=debug)]
     static_json_files = get_static_json_files()
     generated_json_files = get_generated_json_files()
     json_files = static_json_files + generated_json_files
diff --git a/js/bin/integration.js b/js/bin/integration.js
new file mode 100755
index 0000000000000..fe32433d3845a
--- /dev/null
+++ b/js/bin/integration.js
@@ -0,0 +1,86 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+var path = require('path');
+var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`));
+var child_process = require(`child_process`);
+var optionList = [
+    {
+        type: String,
+        name: 'mode',
+        description: 'The integration test to run'
+    },
+    {
+        type: String,
+        name: 'arrow', alias: 'a',
+        description: 'The Arrow file to read/write'
+    },
+    {
+        type: String,
+        name: 'json', alias: 'j',
+        description: 'The JSON file to read/write'
+    }
+];
+
+var argv = require(`command-line-args`)(optionList, { partial: true });
+
+function print_usage() {
+    console.log(require('command-line-usage')([
+        {
+            header: 'integration',
+            content: 'Script for running Arrow integration tests'
+        },
+        {
+            header: 'Synopsis',
+            content: [
+                '$ integration.js -j file.json -a file.arrow --mode validate'
+            ]
+        },
+        {
+            header: 'Options',
+            optionList: [
+                ...optionList,
+                {
+                    name: 'help',
+                    description: 'Print this usage guide.'
+                }
+            ]
+        },
+    ]));
+    process.exit(1);
+}
+
+if (!argv.arrow || !argv.json || !argv.mode) {
+    return print_usage();
+}
+
+switch (argv.mode.toUpperCase()) {
+    case 'VALIDATE':
+        child_process.spawnSync(
+            gulp,
+            [`test`, `-i`].concat(process.argv.slice(2)),
+            {
+                cwd: path.resolve(__dirname, '..'),
+                stdio: ['ignore', 'inherit', 'inherit']
+            }
+        );
+        break;
+    default:
+        print_usage();
+}
diff --git a/js/examples/read_file.html b/js/examples/read_file.html
index 2a1ebaba82c50..3093622fcc1d6 100644
--- a/js/examples/read_file.html
+++ b/js/examples/read_file.html
@@ -40,7 +40,7 @@
 }
 reader.onload = function (evt) {
 
-  var arrowTable = Arrow.Table.from(new Uint8Array(evt.target.result));
+  var arrowTable = Arrow.Table.from([new Uint8Array(evt.target.result)]);
   var thead = document.getElementById("thead");
   var tbody = document.getElementById("tbody");
 
@@ -53,13 +53,13 @@
   }
 
   var header_row = document.createElement("tr");
-  for (let column of arrowTable.cols()) {
+  for (let column of arrowTable.columns) {
     addCell(header_row, "th", column.name);
   }
 
   thead.appendChild(header_row);
 
-  for (let row of arrowTable.rows(true)) {
+  for (let row of arrowTable) {
     var tr = document.createElement("tr");
     for (let cell of row) {
       addCell(tr, "td",
@@ -85,6 +85,6 @@
       <tbody id="tbody">
       </tbody>
     </table>
-    <script type="text/javascript" src="../dist/Arrow.js"></script>
+    <script type="text/javascript" src="../targets/apache-arrow/Arrow.es5.min.js"></script>
   </body>
 </html>
diff --git a/js/gulp/argv.js b/js/gulp/argv.js
index 253553c733c8a..6f80912e97e52 100644
--- a/js/gulp/argv.js
+++ b/js/gulp/argv.js
@@ -16,12 +16,14 @@
 // under the License.
 
 const argv = require(`command-line-args`)([
-    { name: `all`, alias: `a`, type: Boolean },
+    { name: `all`, type: Boolean },
     { name: 'update', alias: 'u', type: Boolean },
     { name: 'verbose', alias: 'v', type: Boolean },
     { name: `target`, type: String, defaultValue: `` },
     { name: `module`, type: String, defaultValue: `` },
     { name: `coverage`, type: Boolean, defaultValue: false },
+    { name: `json_file`, alias: `j`, type: String, defaultValue: null },
+    { name: `arrow_file`, alias: `a`, type: String, defaultValue: null },
     { name: `integration`, alias: `i`, type: Boolean, defaultValue: false },
     { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
     { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] },
@@ -36,4 +38,4 @@ argv.module && !modules.length && modules.push(argv.module);
 (argv.all || !targets.length) && targets.push(`all`);
 (argv.all || !modules.length) && modules.push(`all`);
 
-module.exports = { argv, targets, modules };
\ No newline at end of file
+module.exports = { argv, targets, modules };
diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js
index a1f0a9a698843..1bd872fd3044a 100644
--- a/js/gulp/closure-task.js
+++ b/js/gulp/closure-task.js
@@ -27,6 +27,8 @@ const gulp = require('gulp');
 const path = require('path');
 const sourcemaps = require('gulp-sourcemaps');
 const { memoizeTask } = require('./memoize-task');
+const ASTBuilders = require('ast-types').builders;
+const transformAST = require('gulp-transform-js-ast');
 const { Observable, ReplaySubject } = require('rxjs');
 const closureCompiler = require('google-closure-compiler').gulp();
 
@@ -48,6 +50,9 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
         ], { base: `./` }),
         sourcemaps.init(),
         closureCompiler(createClosureArgs(entry, externs)),
+        // Strip out closure compiler's error-throwing iterator-return methods
+        // see this issue: https://github.com/google/closure-compiler/issues/2728
+        transformAST(iteratorReturnVisitor),
         // rename the sourcemaps from *.js.map files to *.min.js.map
         sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }),
         gulp.dest(out)
@@ -62,8 +67,9 @@ const createClosureArgs = (entry, externs) => ({
     externs: `${externs}.js`,
     entry_point: `${entry}.js`,
     module_resolution: `NODE`,
-    // formatting: `PRETTY_PRINT`,
+    // formatting: `PRETTY_PRINT`, debug: true,
     compilation_level: `ADVANCED`,
+    allow_method_call_decomposing: true,
     package_json_entry_names: `module,jsnext:main,main`,
     assume_function_wrapper: true,
     js_output_file: `${mainExport}.js`,
@@ -95,3 +101,24 @@ const createClosureArgs = (entry, externs) => ({
 
 module.exports = closureTask;
 module.exports.closureTask = closureTask;
+
+const iteratorReturnVisitor = {
+    visitObjectExpression(p) {
+        const node = p.node, value = p.value;
+        if (!node.properties || !(node.properties.length === 3)) { return value; }
+        if (!propertyIsThrowingIteratorReturn(node.properties[2])) { return value; }
+        value.properties = value.properties.slice(0, 2);
+        return value;
+    }
+};
+
+function propertyIsThrowingIteratorReturn(p) {
+    if (!p || !(p.kind === 'init')) { return false; }
+    if (!p.key || !(p.key.type === 'Identifier') || !(p.key.name === 'return')) { return false; }
+    if (!p.value || !(p.value.type === 'FunctionExpression') || !p.value.params || !(p.value.params.length === 0)) { return false; }
+    if (!p.value.body || !p.value.body.body || !(p.value.body.body.length === 1) || !(p.value.body.body[0].type === 'ThrowStatement')) { return false; }
+    if (!p.value.body.body[0].argument || !(p.value.body.body[0].argument.type === 'CallExpression')) { return false; }
+    if (!p.value.body.body[0].argument.arguments || !(p.value.body.body[0].argument.arguments.length === 1)) { return false; }
+    if (!p.value.body.body[0].argument.arguments[0] || !(p.value.body.body[0].argument.arguments[0].type === 'Literal')) { return false; }
+    return p.value.body.body[0].argument.arguments[0].value === 'Not yet implemented';
+}
\ No newline at end of file
diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index 824284e6a158a..fc959643503bd 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -53,7 +53,12 @@ const createMainPackageJson = (target, format) => (orig) => ({
   
 const createTypeScriptPackageJson = (target, format) => (orig) => ({
     ...createScopedPackageJSON(target, format)(orig),
-    main: `${mainExport}.ts`, types: `${mainExport}.ts`
+    main: `${mainExport}.ts`, types: `${mainExport}.ts`,
+    dependencies: {
+        '@types/flatbuffers': '*',
+        '@types/node': '*',
+        ...orig.dependencies
+    }
 });
   
 const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
index ac9755e11ded3..f21aaf2364d03 100644
--- a/js/gulp/test-task.js
+++ b/js/gulp/test-task.js
@@ -36,16 +36,20 @@ argv.coverage && jestArgv.push(`--coverage`);
 
 const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`];
 const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`));
+const testOptions = {
+    env: { ...process.env },
+    stdio: [`ignore`, `inherit`, `inherit`],
+};
 
 const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) {
     const opts = { ...testOptions };
     const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv];
-    if (!argv.integration) {
-        args.push('test/vector-tests.ts');
-    }
+    args.push(`test/${argv.integration ? `integration/*` : `unit/*`}`);
     opts.env = { ...opts.env,
         TEST_TARGET: target,
         TEST_MODULE: format,
+        JSON_PATH: argv.json_file,
+        ARROW_PATH: argv.arrow_file,
         TEST_TS_SOURCE: !!argv.coverage,
         TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]),
         TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]),
@@ -53,10 +57,7 @@ const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function
     return !debug ?
         child_process.spawn(jest, args, opts) :
         child_process.exec(`node --inspect-brk ${jest} ${args.join(` `)}`, opts);
-}))({}, jestArgv, {
-    env: { ...process.env },
-    stdio: [`ignore`, `inherit`, `inherit`],
-});
+}))({}, jestArgv, testOptions);
 
 module.exports = testTask;
 module.exports.testTask = testTask;
diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js
index 2fd9f1350a6f1..8b755cf7f1624 100644
--- a/js/gulp/typescript-task.js
+++ b/js/gulp/typescript-task.js
@@ -52,10 +52,10 @@ function maybeCopyRawJSArrowFormatFiles(target, format) {
         return Observable.empty();
     }
     return Observable.defer(async () => {
-        const outFormatDir = path.join(targetDir(target, format), `format`);
+        const outFormatDir = path.join(targetDir(target, format), `format`, `fb`);
         await del(path.join(outFormatDir, '*.js'));
         await observableFromStreams(
-            gulp.src(path.join(`src`, `format`, `*_generated.js`)),
+            gulp.src(path.join(`src`, `format`, `fb`, `*_generated.js`)),
             gulpRename((p) => { p.basename = p.basename.replace(`_generated`, ``); }),
             gulp.dest(outFormatDir)
         ).toPromise();
diff --git a/js/gulp/util.js b/js/gulp/util.js
index 23476da2df53b..ba6ebece51bba 100644
--- a/js/gulp/util.js
+++ b/js/gulp/util.js
@@ -27,11 +27,12 @@ const npmOrgName = `@${npmPkgName}`;
 const releasesRootDir = `targets`;
 const knownTargets = [`es5`, `es2015`, `esnext`];
 const knownModules = [`cjs`, `esm`, `cls`, `umd`];
-const moduleFormatsToSkipCombosOf = { cls: true };
+const moduleFormatsToSkipCombosOf = { cls: { test: true, integration: true } };
 const packageJSONFields = [
   `version`, `license`, `description`,
   `author`, `homepage`, `repository`,
-  `bugs`, `keywords`,  `dependencies`
+  `bugs`, `keywords`,  `dependencies`,
+  `bin`
 ];
 
 const metadataFiles = [`LICENSE.txt`, `NOTICE.txt`, `README.md`].map((filename) => {
diff --git a/js/gulpfile.js b/js/gulpfile.js
index a60363214bb94..7b82962035e46 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -93,13 +93,13 @@ gulp.task(`debug`, gulp.series(getTasks(`debug`)));
 gulp.task(`clean`, gulp.parallel(getTasks(`clean`)));
 gulp.task(`build`, buildConcurrent(getTasks(`build`)));
 gulp.task(`default`,  gulp.series(`build`, `test`));
-  
+
 function getTasks(name) {
     const tasks = [];
     if (targets.indexOf(`ts`) !== -1) tasks.push(`${name}:ts`);
     if (targets.indexOf(npmPkgName) !== -1) tasks.push(`${name}:${npmPkgName}`);
     for (const [target, format] of combinations(targets, modules)) {
-        if (moduleFormatsToSkipCombosOf[format] && name === `test`) {
+        if (moduleFormatsToSkipCombosOf[format] && moduleFormatsToSkipCombosOf[format][name]) {
             continue;
         }
         tasks.push(`${name}:${taskName(target, format)}`);
diff --git a/js/package.json b/js/package.json
index 8e5c42ef7d1e3..179107b63db11 100644
--- a/js/package.json
+++ b/js/package.json
@@ -1,14 +1,10 @@
 {
-  "name": "apache-arrow",
   "version": "0.2.0",
-  "types": "./types/Arrow.d.ts",
-  "typings": "./types/Arrow.d.ts",
-  "main": "./targets/es5/cjs/Arrow.js",
-  "module": "./targets/es5/esm/Arrow.js",
-  "browser": "./targets/es5/umd/Arrow.js",
-  "jsnext:main": "./targets/es2015/esm/Arrow.js",
-  "esnext:main": "./targets/esnext/esm/Arrow.js",
+  "name": "apache-arrow",
   "description": "Apache Arrow columnar in-memory format",
+  "bin": {
+    "arrow2csv": "bin/arrow2csv"
+  },
   "scripts": {
     "lerna": "lerna",
     "test": "gulp test",
@@ -57,50 +53,54 @@
     "command-line-usage": "4.0.1"
   },
   "dependencies": {
+    "command-line-args": "4.0.7",
+    "command-line-usage": "4.0.2",
     "flatbuffers": "trxcllnt/flatbuffers-esm",
+    "json-bignum": "0.0.3",
     "text-encoding-utf-8": "^1.0.2",
-    "tslib": "^1.8.0"
+    "tslib": "1.8.1"
   },
   "devDependencies": {
-    "@std/esm": "0.13.0",
+    "@std/esm": "0.18.0",
     "@types/flatbuffers": "1.6.5",
-    "@types/glob": "5.0.33",
-    "@types/jest": "21.1.6",
-    "@types/node": "8.0.51",
+    "@types/glob": "5.0.34",
+    "@types/jest": "21.1.8",
+    "@types/node": "8.5.0",
     "@types/text-encoding": "0.0.32",
+    "ast-types": "0.10.1",
     "benchmark": "2.1.4",
-    "command-line-args": "4.0.7",
     "coveralls": "3.0.0",
     "del": "3.0.0",
     "esdoc": "1.0.3",
     "esdoc-standard-plugin": "1.0.0",
     "glob": "7.1.2",
-    "google-closure-compiler": "20171112.0.0",
+    "google-closure-compiler": "20171203.0.0",
     "gulp": "github:gulpjs/gulp#4.0",
     "gulp-json-transform": "0.4.5",
     "gulp-rename": "1.2.2",
     "gulp-sourcemaps": "2.6.1",
+    "gulp-transform-js-ast": "1.0.2",
     "gulp-typescript": "3.2.3",
-    "ix": "2.3.1",
+    "ix": "2.3.4",
     "jest": "21.2.1",
     "jest-environment-node-debug": "2.0.0",
     "json": "9.0.6",
     "lerna": "2.5.1",
-    "lint-staged": "5.0.0",
+    "lint-staged": "6.0.0",
     "merge2": "1.2.0",
     "mkdirp": "0.5.1",
     "npm-run-all": "4.1.2",
     "pump": "1.0.2",
     "rimraf": "2.6.2",
-    "rxjs": "5.5.2",
+    "rxjs": "5.5.5",
     "shx": "0.2.2",
     "source-map-loader": "0.2.3",
-    "trash": "4.1.0",
-    "ts-jest": "21.2.1",
+    "trash": "4.2.1",
+    "ts-jest": "21.2.4",
     "tslint": "5.8.0",
-    "typescript": "2.6.1",
-    "uglifyjs-webpack-plugin": "1.0.1",
-    "webpack": "3.8.1",
+    "typescript": "2.6.2",
+    "uglifyjs-webpack-plugin": "1.1.2",
+    "webpack": "3.10.0",
     "xml2js": "0.4.19"
   },
   "lint-staged": {
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index e6527b34e10dd..3a8943434eece 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -20,7 +20,8 @@ import { Vector } from './vector/vector';
 import { Utf8Vector } from './vector/utf8';
 import { DictionaryVector } from './vector/dictionary';
 import { StructVector, StructRow } from './vector/struct';
-import { readVectors, readVectorsAsync } from './reader/arrow';
+import { read, readAsync } from './reader/arrow';
+import { Uint64, Int64, Int128 } from './util/int';
 import { ListVector, BinaryVector, FixedSizeListVector } from './vector/list';
 
 import {
@@ -44,8 +45,16 @@ import {
     TimestampVector,
 } from './vector/numeric';
 
+// closure compiler always erases static method names:
+// https://github.com/google/closure-compiler/issues/1776
+// set them via string indexers to save them from the mangler
+Table['from'] = Table.from;
+Table['fromAsync'] = Table.fromAsync;
+BoolVector['pack'] = BoolVector.pack;
+
+export { read, readAsync };
 export { Table, Vector, StructRow };
-export { readVectors, readVectorsAsync };
+export { Uint64, Int64, Int128 };
 export { NumericVectorConstructor } from './vector/numeric';
 export { List, TypedArray, TypedArrayConstructor } from './vector/types';
 export {
@@ -80,8 +89,8 @@ try {
     const Arrow = eval('exports');
     if (typeof Arrow === 'object') {
         // string indexers tell closure compiler not to rename these properties
-        Arrow['readVectors'] = readVectors;
-        Arrow['readVectorsAsync'] = readVectorsAsync;
+        Arrow['read'] = read;
+        Arrow['readAsync'] = readAsync;
         Arrow['Table'] = Table;
         Arrow['Vector'] = Vector;
         Arrow['StructRow'] = StructRow;
diff --git a/js/bin/arrow2csv.js b/js/src/bin/arrow2csv.ts
old mode 100755
new mode 100644
similarity index 53%
rename from js/bin/arrow2csv.js
rename to js/src/bin/arrow2csv.ts
index f316b84b69e58..01ef0b848ce75
--- a/js/bin/arrow2csv.js
+++ b/js/src/bin/arrow2csv.ts
@@ -1,4 +1,4 @@
-#! /usr/bin/env node
+// #! /usr/bin/env node
 
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
@@ -17,13 +17,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-var fs = require('fs');
-var Table = require('../dist/Arrow.js').Table;
-var optionList = [
+/* tslint:disable */
+
+import * as Arrow from '../Arrow';
+
+(function() {
+
+const fs = require('fs');
+const { parse } = require('json-bignum');
+const optionList = [
     {
         type: String,
-        name: 'schema',
-        alias: 's', multiple: true,
+        name: 'schema', alias: 's',
+        optional: true, multiple: true,
         typeLabel: '[underline]{columns}',
         description: 'A space-delimited list of column names'
     },
@@ -34,12 +40,10 @@ var optionList = [
     }
 ];
 
-var argv = require(`command-line-args`)(optionList, { partial: true });
-var files = [argv.file, ...(argv._unknown || [])].filter(Boolean);
-
-// console.log(JSON.stringify(argv));
+const argv = require(`command-line-args`)(optionList, { partial: true });
+const files = [argv.file, ...(argv._unknown || [])].filter(Boolean);
 
-if (!argv.schema || !files.length) {
+if (!files.length) {
     console.log(require('command-line-usage')([
         {
             header: 'arrow2csv',
@@ -81,9 +85,51 @@ if (!argv.schema || !files.length) {
 }
 
 files.forEach((source) => {
-    var allColumns = Table.from(fs.readFileSync(source));
-    var selectedColumns = new Table(argv.schema.map((columnName) => {
-        return allColumns.getColumn(columnName);
-    }));
-    console.log(selectedColumns.toString());
+    let table: any, input = fs.readFileSync(source);
+    try {
+        table = Arrow.Table.from([input]);
+    } catch (e) {
+        table = Arrow.Table.from(parse(input + ''));
+    }
+    if (argv.schema && argv.schema.length) {
+        table = table.select(...argv.schema);
+    }
+    printTable(table);
 });
+
+function printTable(table: Arrow.Table<any>) {
+    let header = [...table.columns.map((_, i) => table.key(i))].map(stringify);
+    let maxColumnWidths = header.map(x => x.length);
+    // Pass one to convert to strings and count max column widths
+    for (let i = -1, n = table.length - 1; ++i < n;) {
+        let val,
+            row = [i, ...table.get(i)];
+        for (let j = -1, k = row.length; ++j < k; ) {
+            val = stringify(row[j]);
+            maxColumnWidths[j] = Math.max(maxColumnWidths[j], val.length);
+        }
+    }
+    console.log(header.map((x, j) => leftPad(x, ' ', maxColumnWidths[j])).join(' | '));
+    // Pass two to pad each one to max column width
+    for (let i = -1, n = table.length; ++i < n; ) {
+        console.log(
+            [...table.get(i)]
+                .map(stringify)
+                .map((x, j) => leftPad(x, ' ', maxColumnWidths[j]))
+                .join(' | ')
+        );
+    }
+}
+
+function leftPad(str: string, fill: string, n: number) {
+    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
+}
+
+function stringify(x: any) {
+    return typeof x === 'string' ? `"${x}"`
+              : Array.isArray(x) ? JSON.stringify(x)
+              : ArrayBuffer.isView(x) ? `[${x}]`
+                                      : `${x}`;
+}
+
+})();
\ No newline at end of file
diff --git a/js/src/format/arrow.ts b/js/src/format/arrow.ts
new file mode 100644
index 0000000000000..14adf9040a47f
--- /dev/null
+++ b/js/src/format/arrow.ts
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { footerFromByteBuffer, messageFromByteBuffer } from './fb';
+import { schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON } from './json';
+import {
+    IntBitWidth, TimeBitWidth,
+    VisitorNode, Visitor, Footer, Block, Message, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode,
+    Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
+} from './types';
+
+export {
+    IntBitWidth, TimeBitWidth,
+    footerFromByteBuffer, messageFromByteBuffer,
+    schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON,
+    VisitorNode, Visitor, Footer, Block, Message, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode,
+    Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_ as Map,
+};
diff --git a/js/src/format/fb.ts b/js/src/format/fb.ts
new file mode 100644
index 0000000000000..fdf7f7b0ed99a
--- /dev/null
+++ b/js/src/format/fb.ts
@@ -0,0 +1,234 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as File_ from './fb/File';
+import * as Schema_ from './fb/Schema';
+import * as Message_ from './fb/Message';
+import { flatbuffers } from 'flatbuffers';
+import ByteBuffer = flatbuffers.ByteBuffer;
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
+import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
+import _Footer = File_.org.apache.arrow.flatbuf.Footer;
+import _Block = File_.org.apache.arrow.flatbuf.Block;
+import _Message = Message_.org.apache.arrow.flatbuf.Message;
+import _Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+import _Field = Schema_.org.apache.arrow.flatbuf.Field;
+import _RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
+import _DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch;
+import _FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+import _Buffer = Schema_.org.apache.arrow.flatbuf.Buffer;
+import _DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
+import _Null = Schema_.org.apache.arrow.flatbuf.Null;
+import _Int = Schema_.org.apache.arrow.flatbuf.Int;
+import _FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
+import _Binary = Schema_.org.apache.arrow.flatbuf.Binary;
+import _Bool = Schema_.org.apache.arrow.flatbuf.Bool;
+import _Utf8 = Schema_.org.apache.arrow.flatbuf.Utf8;
+import _Decimal = Schema_.org.apache.arrow.flatbuf.Decimal;
+import _Date = Schema_.org.apache.arrow.flatbuf.Date;
+import _Time = Schema_.org.apache.arrow.flatbuf.Time;
+import _Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp;
+import _Interval = Schema_.org.apache.arrow.flatbuf.Interval;
+import _List = Schema_.org.apache.arrow.flatbuf.List;
+import _Struct = Schema_.org.apache.arrow.flatbuf.Struct_;
+import _Union = Schema_.org.apache.arrow.flatbuf.Union;
+import _FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary;
+import _FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
+import _Map = Schema_.org.apache.arrow.flatbuf.Map;
+
+import {
+    IntBitWidth, TimeBitWidth,
+    Footer, Block, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode,
+    Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
+} from './types';
+
+export function footerFromByteBuffer(bb: ByteBuffer) {
+    const f = _Footer.getRootAsFooter(bb), s = f.schema()!;
+    return new Footer(
+        dictionaryBatchesFromFooter(f), recordBatchesFromFooter(f),
+        new Schema(f.version(), fieldsFromSchema(s), customMetadata(s), s.endianness())
+    );
+}
+
+export function messageFromByteBuffer(bb: ByteBuffer) {
+    const m = _Message.getRootAsMessage(bb)!, type = m.headerType(), version = m.version();
+    switch (type) {
+        case MessageHeader.Schema: return schemaFromMessage(version, m.header(new _Schema())!);
+        case MessageHeader.RecordBatch: return recordBatchFromMessage(version, m.header(new _RecordBatch())!);
+        case MessageHeader.DictionaryBatch: return dictionaryBatchFromMessage(version, m.header(new _DictionaryBatch())!);
+    }
+    return null;
+    // throw new Error(`Unrecognized Message type '${type}'`);
+}
+
+function schemaFromMessage(version: MetadataVersion, s: _Schema) {
+    return new Schema(version, fieldsFromSchema(s), customMetadata(s), s.endianness());
+}
+
+function recordBatchFromMessage(version: MetadataVersion, b: _RecordBatch) {
+    return new RecordBatch(version, b.length(), fieldNodesFromRecordBatch(b), buffersFromRecordBatch(b, version));
+}
+
+function dictionaryBatchFromMessage(version: MetadataVersion, d: _DictionaryBatch) {
+    return new DictionaryBatch(version, recordBatchFromMessage(version, d.data()!), d.id(), d.isDelta());
+}
+
+function dictionaryBatchesFromFooter(f: _Footer) {
+    const blocks = [] as Block[];
+    for (let b: _Block, i = -1, n = f && f.dictionariesLength(); ++i < n;) {
+        if (b = f.dictionaries(i)!) {
+            blocks.push(new Block(b.metaDataLength(), b.bodyLength(), b.offset()));
+        }
+    }
+    return blocks;
+}
+
+function recordBatchesFromFooter(f: _Footer) {
+    const blocks = [] as Block[];
+    for (let b: _Block, i = -1, n = f && f.recordBatchesLength(); ++i < n;) {
+        if (b = f.recordBatches(i)!) {
+            blocks.push(new Block(b.metaDataLength(), b.bodyLength(), b.offset()));
+        }
+    }
+    return blocks;
+}
+
+function fieldsFromSchema(s: _Schema) {
+    const fields = [] as Field[];
+    for (let i = -1, n = s && s.fieldsLength(); ++i < n;) {
+        fields.push(field(s.fields(i)!));
+    }
+    return fields;
+}
+
+function fieldsFromField(f: _Field) {
+    const fields = [] as Field[];
+    for (let i = -1, n = f && f.childrenLength(); ++i < n;) {
+        fields.push(field(f.children(i)!));
+    }
+    return fields;
+}
+
+function fieldNodesFromRecordBatch(b: _RecordBatch) {
+    const fieldNodes = [] as FieldNode[];
+    for (let i = -1, n = b.nodesLength(); ++i < n;) {
+        fieldNodes.push(fieldNodeFromRecordBatch(b.nodes(i)!));
+    }
+    return fieldNodes;
+}
+
+function buffersFromRecordBatch(b: _RecordBatch, version: MetadataVersion) {
+    const buffers = [] as Buffer[];
+    for (let i = -1, n = b.buffersLength(); ++i < n;) {
+        let buffer = b.buffers(i)!;
+        // If this Arrow buffer was written before version 4,
+        // advance the buffer's bb_pos 8 bytes to skip past
+        // the now-removed page id field.
+        if (version < MetadataVersion.V4) {
+            buffer.bb_pos += (8 * (i + 1));
+        }
+        buffers.push(bufferFromRecordBatch(buffer));
+    }
+    return buffers;
+}
+
+function field(f: _Field) {
+    return new Field(
+        f.name()!,
+        typeFromField(f),
+        f.typeType(),
+        f.nullable(),
+        fieldsFromField(f),
+        customMetadata(f),
+        dictionaryEncodingFromField(f)
+    );
+}
+
+function dictionaryEncodingFromField(f: _Field) {
+    let t: _Int | null;
+    let e: _DictionaryEncoding | null;
+    if (e = f.dictionary()) {
+        if (t = e.indexType()) {
+            return new DictionaryEncoding(new Int(t.isSigned(), t.bitWidth() as IntBitWidth), e.id(), e.isOrdered());
+        }
+        return new DictionaryEncoding(null, e.id(), e.isOrdered());
+    }
+    return undefined;
+}
+
+function customMetadata(parent?: _Schema | _Field | null) {
+    const data = new Map<string, string>();
+    if (parent) {
+        for (let entry, key, i = -1, n = parent.customMetadataLength() | 0; ++i < n;) {
+            if ((entry = parent.customMetadata(i)) && (key = entry.key()) != null) {
+                data.set(key, entry.value()!);
+            }
+        }
+    }
+    return data;
+}
+
+function fieldNodeFromRecordBatch(f: _FieldNode) {
+    return new FieldNode(f.length(), f.nullCount());
+}
+
+function bufferFromRecordBatch(b: _Buffer) {
+    return new Buffer(b.offset(), b.length());
+}
+
+function typeFromField(f: _Field) {
+    switch (f.typeType()) {
+        case Type.NONE: return nullFromField(f.type(new _Null())!);
+        case Type.Null: return nullFromField(f.type(new _Null())!);
+        case Type.Int: return intFromField(f.type(new _Int())!);
+        case Type.FloatingPoint: return floatingPointFromField(f.type(new _FloatingPoint())!);
+        case Type.Binary: return binaryFromField(f.type(new _Binary())!);
+        case Type.Utf8: return utf8FromField(f.type(new _Utf8())!);
+        case Type.Bool: return boolFromField(f.type(new _Bool())!);
+        case Type.Decimal: return decimalFromField(f.type(new _Decimal())!);
+        case Type.Date: return dateFromField(f.type(new _Date())!);
+        case Type.Time: return timeFromField(f.type(new _Time())!);
+        case Type.Timestamp: return timestampFromField(f.type(new _Timestamp())!);
+        case Type.Interval: return intervalFromField(f.type(new _Interval())!);
+        case Type.List: return listFromField(f.type(new _List())!);
+        case Type.Struct_: return structFromField(f.type(new _Struct())!);
+        case Type.Union: return unionFromField(f.type(new _Union())!);
+        case Type.FixedSizeBinary: return fixedSizeBinaryFromField(f.type(new _FixedSizeBinary())!);
+        case Type.FixedSizeList: return fixedSizeListFromField(f.type(new _FixedSizeList())!);
+        case Type.Map: return mapFromField(f.type(new _Map())!);
+    }
+    throw new Error(`Unrecognized type ${f.typeType()}`);
+}
+
+function nullFromField(_type: _Null) { return new Null(); }
+function intFromField(_type: _Int) { return new Int(_type.isSigned(), _type.bitWidth() as IntBitWidth); }
+function floatingPointFromField(_type: _FloatingPoint) { return new FloatingPoint(_type.precision()); }
+function binaryFromField(_type: _Binary) { return new Binary(); }
+function utf8FromField(_type: _Utf8) { return new Utf8(); }
+function boolFromField(_type: _Bool) { return new Bool(); }
+function decimalFromField(_type: _Decimal) { return new Decimal(_type.scale(), _type.precision()); }
+function dateFromField(_type: _Date) { return new Date(_type.unit()); }
+function timeFromField(_type: _Time) { return new Time(_type.unit(), _type.bitWidth() as TimeBitWidth); }
+function timestampFromField(_type: _Timestamp) { return new Timestamp(_type.unit(), _type.timezone()); }
+function intervalFromField(_type: _Interval) { return new Interval(_type.unit()); }
+function listFromField(_type: _List) { return new List(); }
+function structFromField(_type: _Struct) { return new Struct(); }
+function unionFromField(_type: _Union) { return new Union(_type.mode(), (_type.typeIdsArray() || []) as Type[]); }
+function fixedSizeBinaryFromField(_type: _FixedSizeBinary) { return new FixedSizeBinary(_type.byteWidth()); }
+function fixedSizeListFromField(_type: _FixedSizeList) { return new FixedSizeList(_type.listSize()); }
+function mapFromField(_type: _Map) { return new Map_(_type.keysSorted()); }
diff --git a/js/src/format/File.ts b/js/src/format/fb/File.ts
similarity index 100%
rename from js/src/format/File.ts
rename to js/src/format/fb/File.ts
diff --git a/js/src/format/File_generated.js b/js/src/format/fb/File_generated.js
similarity index 100%
rename from js/src/format/File_generated.js
rename to js/src/format/fb/File_generated.js
diff --git a/js/src/format/Message.ts b/js/src/format/fb/Message.ts
similarity index 100%
rename from js/src/format/Message.ts
rename to js/src/format/fb/Message.ts
diff --git a/js/src/format/Message_generated.js b/js/src/format/fb/Message_generated.js
similarity index 100%
rename from js/src/format/Message_generated.js
rename to js/src/format/fb/Message_generated.js
diff --git a/js/src/format/Schema.ts b/js/src/format/fb/Schema.ts
similarity index 100%
rename from js/src/format/Schema.ts
rename to js/src/format/fb/Schema.ts
diff --git a/js/src/format/Schema_generated.js b/js/src/format/fb/Schema_generated.js
similarity index 97%
rename from js/src/format/Schema_generated.js
rename to js/src/format/fb/Schema_generated.js
index f89cf2fa8fa0a..ebed8a90645c8 100644
--- a/js/src/format/Schema_generated.js
+++ b/js/src/format/fb/Schema_generated.js
@@ -31,65 +31,65 @@ org.apache.arrow.flatbuf.MetadataVersion = {
   /**
    * 0.1.0
    */
-  V1: 0, 0: 'V1',
+  'V1': 0, 0: 'V1',
 
   /**
    * 0.2.0
    */
-  V2: 1, 1: 'V2',
+  'V2': 1, 1: 'V2',
 
   /**
    * 0.3.0 -> 0.7.1
    */
-  V3: 2, 2: 'V3',
+  'V3': 2, 2: 'V3',
 
   /**
    * >= 0.8.0
    */
-  V4: 3, 3: 'V4'
+  'V4': 3, 3: 'V4'
 };
 
 /**
  * @enum
  */
 org.apache.arrow.flatbuf.UnionMode = {
-  Sparse: 0, 0: 'Sparse',
-  Dense: 1, 1: 'Dense',
+  'Sparse': 0, 0: 'Sparse',
+  'Dense': 1, 1: 'Dense',
 };
 
 /**
  * @enum
  */
 org.apache.arrow.flatbuf.Precision = {
-  HALF: 0, 0: 'HALF',
-  SINGLE: 1, 1: 'SINGLE',
-  DOUBLE: 2, 2: 'DOUBLE',
+  'HALF': 0, 0: 'HALF',
+  'SINGLE': 1, 1: 'SINGLE',
+  'DOUBLE': 2, 2: 'DOUBLE',
 };
 
 /**
  * @enum
  */
 org.apache.arrow.flatbuf.DateUnit = {
-  DAY: 0, 0: 'DAY',
-  MILLISECOND: 1, 1: 'MILLISECOND',
+  'DAY': 0, 0: 'DAY',
+  'MILLISECOND': 1, 1: 'MILLISECOND',
 };
 
 /**
  * @enum
  */
 org.apache.arrow.flatbuf.TimeUnit = {
-  SECOND: 0, 0: 'SECOND',
-  MILLISECOND: 1, 1: 'MILLISECOND',
-  MICROSECOND: 2, 2: 'MICROSECOND',
-  NANOSECOND: 3, 3: 'NANOSECOND',
+  'SECOND': 0, 0: 'SECOND',
+  'MILLISECOND': 1, 1: 'MILLISECOND',
+  'MICROSECOND': 2, 2: 'MICROSECOND',
+  'NANOSECOND': 3, 3: 'NANOSECOND',
 };
 
 /**
  * @enum
  */
 org.apache.arrow.flatbuf.IntervalUnit = {
-  YEAR_MONTH: 0, 0: 'YEAR_MONTH',
-  DAY_TIME: 1, 1: 'DAY_TIME',
+  'YEAR_MONTH': 0, 0: 'YEAR_MONTH',
+  'DAY_TIME': 1, 1: 'DAY_TIME',
 };
 
 /**
@@ -100,24 +100,24 @@ org.apache.arrow.flatbuf.IntervalUnit = {
  * @enum
  */
 org.apache.arrow.flatbuf.Type = {
-  NONE: 0, 0: 'NONE',
-  Null: 1, 1: 'Null',
-  Int: 2, 2: 'Int',
-  FloatingPoint: 3, 3: 'FloatingPoint',
-  Binary: 4, 4: 'Binary',
-  Utf8: 5, 5: 'Utf8',
-  Bool: 6, 6: 'Bool',
-  Decimal: 7, 7: 'Decimal',
-  Date: 8, 8: 'Date',
-  Time: 9, 9: 'Time',
-  Timestamp: 10, 10: 'Timestamp',
-  Interval: 11, 11: 'Interval',
-  List: 12, 12: 'List',
-  Struct_: 13, 13: 'Struct_',
-  Union: 14, 14: 'Union',
-  FixedSizeBinary: 15, 15: 'FixedSizeBinary',
-  FixedSizeList: 16, 16: 'FixedSizeList',
-  Map: 17, 17: 'Map'
+  'NONE': 0, 0: 'NONE',
+  'Null': 1, 1: 'Null',
+  'Int': 2, 2: 'Int',
+  'FloatingPoint': 3, 3: 'FloatingPoint',
+  'Binary': 4, 4: 'Binary',
+  'Utf8': 5, 5: 'Utf8',
+  'Bool': 6, 6: 'Bool',
+  'Decimal': 7, 7: 'Decimal',
+  'Date': 8, 8: 'Date',
+  'Time': 9, 9: 'Time',
+  'Timestamp': 10, 10: 'Timestamp',
+  'Interval': 11, 11: 'Interval',
+  'List': 12, 12: 'List',
+  'Struct_': 13, 13: 'Struct_',
+  'Union': 14, 14: 'Union',
+  'FixedSizeBinary': 15, 15: 'FixedSizeBinary',
+  'FixedSizeList': 16, 16: 'FixedSizeList',
+  'Map': 17, 17: 'Map'
 };
 
 /**
@@ -130,22 +130,22 @@ org.apache.arrow.flatbuf.VectorType = {
   /**
    * used in List type, Dense Union and variable length primitive types (String, Binary)
    */
-  OFFSET: 0, 0: 'OFFSET',
+  'OFFSET': 0, 0: 'OFFSET',
 
   /**
    * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
    */
-  DATA: 1, 1: 'DATA',
+  'DATA': 1, 1: 'DATA',
 
   /**
    * Bit vector indicating if each value is null
    */
-  VALIDITY: 2, 2: 'VALIDITY',
+  'VALIDITY': 2, 2: 'VALIDITY',
 
   /**
    * Type vector used in Union type
    */
-  TYPE: 3, 3: 'TYPE'
+  'TYPE': 3, 3: 'TYPE'
 };
 
 /**
@@ -155,8 +155,8 @@ org.apache.arrow.flatbuf.VectorType = {
  * @enum
  */
 org.apache.arrow.flatbuf.Endianness = {
-  Little: 0, 0: 'Little',
-  Big: 1, 1: 'Big',
+  'Little': 0, 0: 'Little',
+  'Big': 1, 1: 'Big',
 };
 
 /**
diff --git a/js/src/format/json.ts b/js/src/format/json.ts
new file mode 100644
index 0000000000000..3da3db6d5fea3
--- /dev/null
+++ b/js/src/format/json.ts
@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Schema_ from './fb/Schema';
+import { flatbuffers } from 'flatbuffers';
+import Long = flatbuffers.Long;
+import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
+import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
+import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
+import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
+import {
+    IntBitWidth, TimeBitWidth,
+    Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode,
+    Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
+} from './types';
+
+export function schemaFromJSON(s: any): Schema {
+    // todo: metadataFromJSON
+    return new Schema(
+        MetadataVersion.V4,
+        fieldsFromJSON(s['fields']),
+        customMetadata(s['customMetadata'])
+    );
+}
+
+export function recordBatchFromJSON(b: any): RecordBatch {
+    return new RecordBatch(
+        MetadataVersion.V4,
+        new Long(b['count'], 0),
+        fieldNodesFromJSON(b['columns']),
+        buffersFromJSON(b['columns'])
+    );
+}
+
+export function dictionaryBatchFromJSON(b: any): DictionaryBatch {
+    return new DictionaryBatch(
+        MetadataVersion.V4,
+        recordBatchFromJSON(b['data']),
+        new Long(b['id'], 0), b['isDelta']
+    );
+}
+
+function fieldsFromJSON(fs: any[]): Field[] {
+    return (fs || []).map(fieldFromJSON);
+}
+
+function fieldNodesFromJSON(xs: any[]): FieldNode[] {
+    return (xs || []).reduce<FieldNode[]>((fieldNodes, column: any) => [
+        ...fieldNodes,
+        new FieldNode(
+            new Long(column['count'], 0),
+            new Long(nullCountFromJSON(column['VALIDITY']), 0)
+        ),
+        ...fieldNodesFromJSON(column['children'])
+    ], [] as FieldNode[]);
+}
+
+function buffersFromJSON(xs: any[], buffers: Buffer[] = []): Buffer[] {
+    for (let i = -1, n = (xs || []).length; ++i < n;) {
+        const column = xs[i];
+        column['VALIDITY'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['VALIDITY'].length, 0)));
+        column['OFFSET'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['OFFSET'].length, 0)));
+        column['DATA'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['DATA'].length, 0)));
+        buffers = buffersFromJSON(column['children'], buffers);
+    }
+    return buffers;
+}
+
+function nullCountFromJSON(validity: number[]) {
+    return (validity || []).reduce((sum, val) => sum + +(val === 0), 0);
+}
+
+function fieldFromJSON(f: any) {
+    return new Field(
+        f['name'],
+        typeFromJSON(f['type']),
+        namesToTypeMap[f['type']['name']],
+        f.nullable,
+        fieldsFromJSON(f['children']),
+        customMetadata(f['customMetadata']),
+        dictionaryEncodingFromJSON(f['dictionary'])
+    );
+}
+
+function dictionaryEncodingFromJSON(d: any) {
+    return !d ? null : new DictionaryEncoding(
+        d.indexType ? intFromJSON(d.indexType) : null,
+        new Long(d.id, 0), d.isOrdered
+    );
+}
+
+function customMetadata(metadata?: any) {
+    return new Map<string, string>(Object.entries(metadata || {}));
+}
+
+const namesToTypeMap: { [n: string]: Type }  = {
+    'NONE': Type.NONE,
+    'null': Type.Null,
+    'int': Type.Int,
+    'floatingpoint': Type.FloatingPoint,
+    'binary': Type.Binary,
+    'bool': Type.Bool,
+    'utf8': Type.Utf8,
+    'decimal': Type.Decimal,
+    'date': Type.Date,
+    'time': Type.Time,
+    'timestamp': Type.Timestamp,
+    'interval': Type.Interval,
+    'list': Type.List,
+    'struct': Type.Struct_,
+    'union': Type.Union,
+    'fixedsizebinary': Type.FixedSizeBinary,
+    'fixedsizelist': Type.FixedSizeList,
+    'map': Type.Map,
+};
+
+function typeFromJSON(t: any) {
+    switch (namesToTypeMap[t['name']]) {
+        case Type.NONE: return nullFromJSON(t);
+        case Type.Null: return nullFromJSON(t);
+        case Type.Int: return intFromJSON(t);
+        case Type.FloatingPoint: return floatingPointFromJSON(t);
+        case Type.Binary: return binaryFromJSON(t);
+        case Type.Utf8: return utf8FromJSON(t);
+        case Type.Bool: return boolFromJSON(t);
+        case Type.Decimal: return decimalFromJSON(t);
+        case Type.Date: return dateFromJSON(t);
+        case Type.Time: return timeFromJSON(t);
+        case Type.Timestamp: return timestampFromJSON(t);
+        case Type.Interval: return intervalFromJSON(t);
+        case Type.List: return listFromJSON(t);
+        case Type.Struct_: return structFromJSON(t);
+        case Type.Union: return unionFromJSON(t);
+        case Type.FixedSizeBinary: return fixedSizeBinaryFromJSON(t);
+        case Type.FixedSizeList: return fixedSizeListFromJSON(t);
+        case Type.Map: return mapFromJSON(t);
+    }
+    throw new Error(`Unrecognized type ${t['name']}`);
+}
+
+function nullFromJSON(_type: any) { return new Null(); }
+function intFromJSON(_type: any) { return new Int(_type['isSigned'], _type['bitWidth'] as IntBitWidth); }
+function floatingPointFromJSON(_type: any) { return new FloatingPoint(Precision[_type['precision']] as any); }
+function binaryFromJSON(_type: any) { return new Binary(); }
+function utf8FromJSON(_type: any) { return new Utf8(); }
+function boolFromJSON(_type: any) { return new Bool(); }
+function decimalFromJSON(_type: any) { return new Decimal(_type['scale'], _type['precision']); }
+function dateFromJSON(_type: any) { return new Date(DateUnit[_type['unit']] as any); }
+function timeFromJSON(_type: any) { return new Time(TimeUnit[_type['unit']] as any, _type['bitWidth'] as TimeBitWidth); }
+function timestampFromJSON(_type: any) { return new Timestamp(TimeUnit[_type['unit']] as any, _type['timezone']); }
+function intervalFromJSON(_type: any) { return new Interval(IntervalUnit[_type['unit']] as any); }
+function listFromJSON(_type: any) { return new List(); }
+function structFromJSON(_type: any) { return new Struct(); }
+function unionFromJSON(_type: any) { return new Union(_type['mode'], (_type['typeIdsArray'] || []) as Type[]); }
+function fixedSizeBinaryFromJSON(_type: any) { return new FixedSizeBinary(_type['byteWidth']); }
+function fixedSizeListFromJSON(_type: any) { return new FixedSizeList(_type['listSize']); }
+function mapFromJSON(_type: any) { return new Map_(_type['keysSorted']); }
diff --git a/js/src/format/types.ts b/js/src/format/types.ts
new file mode 100644
index 0000000000000..09df8ccbbdf7c
--- /dev/null
+++ b/js/src/format/types.ts
@@ -0,0 +1,393 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/* tslint:disable:class-name */
+
+import { align } from '../util/layout';
+import * as Schema_ from './fb/Schema';
+import * as Message_ from './fb/Message';
+import { flatbuffers } from 'flatbuffers';
+import Long = flatbuffers.Long;
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
+import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
+import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
+import UnionMode = Schema_.org.apache.arrow.flatbuf.UnionMode;
+import Endianness = Schema_.org.apache.arrow.flatbuf.Endianness;
+import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
+import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
+import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
+
+export type IntBitWidth = 8 | 16 | 32 | 64;
+export type TimeBitWidth = IntBitWidth | 128;
+
+export interface VisitorNode {
+    accept(visitor: Visitor): any;
+}
+
+export abstract class Visitor<T = any> {
+    visit(node: VisitorNode): T {
+        return node.accept(this);
+    }
+    visitMany(nodes: VisitorNode[]): T[] {
+        return nodes.map((node) => this.visit(node));
+    }
+    abstract visitFooter(node: Footer): any;
+    abstract visitBlock(node: Block): any;
+    abstract visitMessage(node: Message): any;
+    abstract visitSchema(node: Schema): any;
+    abstract visitField(node: Field): any;
+    abstract visitBuffer(node: Buffer): any;
+    abstract visitFieldNode(node: FieldNode): any;
+    abstract visitRecordBatch(node: RecordBatch): any;
+    abstract visitDictionaryBatch(node: DictionaryBatch): any;
+    abstract visitDictionaryEncoding(node: DictionaryEncoding): any;
+    abstract visitNullFieldType(node: Null): any;
+    abstract visitIntFieldType(node: Int): any;
+    abstract visitFloatingPointFieldType(node: FloatingPoint): any;
+    abstract visitBinaryFieldType(node: Binary): any;
+    abstract visitBoolFieldType(node: Bool): any;
+    abstract visitUtf8FieldType(node: Utf8): any;
+    abstract visitDecimalFieldType(node: Decimal): any;
+    abstract visitDateFieldType(node: Date): any;
+    abstract visitTimeFieldType(node: Time): any;
+    abstract visitTimestampFieldType(node: Timestamp): any;
+    abstract visitIntervalFieldType(node: Interval): any;
+    abstract visitListFieldType(node: List): any;
+    abstract visitStructFieldType(node: Struct): any;
+    abstract visitUnionFieldType(node: Union): any;
+    abstract visitFixedSizeBinaryFieldType(node: FixedSizeBinary): any;
+    abstract visitFixedSizeListFieldType(node: FixedSizeList): any;
+    abstract visitMapFieldType(node: Map_): any;
+}
+
+export class Footer implements VisitorNode {
+    constructor(public dictionaryBatches: Block[], public recordBatches: Block[], public schema: Schema) {}
+    accept(visitor: Visitor): any {
+        return visitor.visitFooter(this);
+    }
+}
+
+export class Block implements VisitorNode {
+    constructor(public metaDataLength: number, public bodyLength: Long, public offset: Long) {}
+    accept(visitor: Visitor): any {
+        return visitor.visitBlock(this);
+    }
+}
+
+export class Message implements VisitorNode {
+    constructor(public version: MetadataVersion, public bodyLength: Long, public headerType: MessageHeader) {}
+    isSchema(): this is Schema { return this.headerType === MessageHeader.Schema; }
+    isRecordBatch(): this is RecordBatch { return this.headerType === MessageHeader.RecordBatch; }
+    isDictionaryBatch(): this is DictionaryBatch { return this.headerType === MessageHeader.DictionaryBatch; }
+    accept(visitor: Visitor): any {
+        visitor.visitMessage(this);
+    }
+}
+
+export class Schema extends Message {
+    public dictionaries: Map<string, Field>;
+    constructor(version: MetadataVersion, public fields: Field[], public customMetadata?: Map<string, string>, public endianness = Endianness.Little) {
+        super(version, Long.ZERO, MessageHeader.Schema);
+        const dictionaries = [] as Field[];
+        for (let f: Field, i = -1, n = fields.length; ++i < n;) {
+            if ((f = fields[i])) {
+                f.dictionary && dictionaries.push(f);
+                dictionaries.push(...f.dictionaries);
+            }
+        }
+        this.dictionaries = new Map<string, Field>(dictionaries.map<[string, Field]>((f) => [
+            f.dictionary!.dictionaryId.toFloat64().toString(), f
+        ]));
+    }
+    accept(visitor: Visitor): any {
+        return visitor.visitSchema(this);
+    }
+}
+
+export class RecordBatch extends Message {
+    constructor(version: MetadataVersion, public length: Long, public fieldNodes: FieldNode[], public buffers: Buffer[]) {
+        super(version, new Long(buffers.reduce((s, b) => align(s + b.length.low + (b.offset.low - s), 8), 0), 0), MessageHeader.RecordBatch);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitRecordBatch(this);
+    }
+}
+
+export class DictionaryBatch extends Message {
+    constructor(version: MetadataVersion, public dictionary: RecordBatch, public dictionaryId: Long, public isDelta: boolean) {
+        super(version, dictionary.bodyLength, MessageHeader.DictionaryBatch);
+    }
+    get fieldNodes(): FieldNode[] { return this.dictionary.fieldNodes; }
+    get buffers(): Buffer[] { return this.dictionary.buffers; }
+    accept(visitor: Visitor) {
+        return visitor.visitDictionaryBatch(this);
+    }
+    static atomicDictionaryId = 0;
+}
+
+export class Field implements VisitorNode {
+    public dictionaries: Field[];
+    constructor(public name: string,
+                public type: FieldType,
+                public typeType: Type,
+                public nullable = false,
+                public children: Field[] = [],
+                public metadata?: Map<string, string> | null,
+                public dictionary?: DictionaryEncoding | null) {
+        const dictionaries = [] as Field[];
+        for (let f: Field, i = -1, n = children.length; ++i < n;) {
+            if ((f = children[i])) {
+                f.dictionary && dictionaries.push(f);
+                dictionaries.push(...f.dictionaries);
+            }
+        }
+        this.dictionaries = dictionaries;
+    }
+    accept(visitor: Visitor): any {
+        return visitor.visitField(this);
+    }
+    indexField() {
+        return !this.dictionary ? this : new Field(
+            this.name,
+            this.dictionary.indexType, this.dictionary.indexType.type,
+            this.nullable, this.children, this.metadata, this.dictionary
+        );
+    }
+    toString() { return `Field name[${this.name}], nullable[${this.nullable}], type[${this.type.toString()}]`; }
+}
+
+export class Buffer implements VisitorNode {
+    constructor(public offset: Long, public length: Long) {}
+    accept(visitor: Visitor) {
+        return visitor.visitBuffer(this);
+    }
+}
+
+export class FieldNode implements VisitorNode {
+    constructor(public length: Long, public nullCount: Long) {}
+    accept(visitor: Visitor) {
+        return visitor.visitFieldNode(this);
+    }
+}
+
+export class DictionaryEncoding implements VisitorNode {
+    public isOrdered: boolean;
+    public dictionaryId: Long;
+    public indexType: Int;
+    constructor(indexType?: Int | null, dictionaryId?: Long | null, isOrdered?: boolean | null) {
+        this.isOrdered = isOrdered || false;
+        /* a dictionary index defaults to signed 32 bit int if unspecified */
+        this.indexType = indexType || new Int(true, 32);
+        this.dictionaryId = dictionaryId || new Long(DictionaryBatch.atomicDictionaryId++, 0);
+    }
+    accept(visitor: Visitor): any {
+        return visitor.visitDictionaryEncoding(this);
+    }
+}
+
+export abstract class FieldType implements VisitorNode {
+    constructor(public type: Type) {}
+    abstract accept(visitor: Visitor): any;
+    isNull(): this is Null { return this.type === Type.Null; }
+    isInt(): this is Int { return this.type === Type.Int; }
+    isFloatingPoint(): this is FloatingPoint { return this.type === Type.FloatingPoint; }
+    isBinary(): this is Binary { return this.type === Type.Binary; }
+    isUtf8(): this is Utf8 { return this.type === Type.Utf8; }
+    isBool(): this is Bool { return this.type === Type.Bool; }
+    isDecimal(): this is Decimal { return this.type === Type.Decimal; }
+    isDate(): this is Date { return this.type === Type.Date; }
+    isTime(): this is Time { return this.type === Type.Time; }
+    isTimestamp(): this is Timestamp { return this.type === Type.Timestamp; }
+    isInterval(): this is Interval { return this.type === Type.Interval; }
+    isList(): this is List { return this.type === Type.List; }
+    isStruct(): this is Struct { return this.type === Type.Struct_; }
+    isUnion(): this is Union { return this.type === Type.Union; }
+    isFixedSizeBinary(): this is FixedSizeBinary { return this.type === Type.FixedSizeBinary; }
+    isFixedSizeList(): this is FixedSizeList { return this.type === Type.FixedSizeList; }
+    isMap(): this is Map_ { return this.type === Type.Map; }
+}
+
+export class Null extends FieldType {
+    toString() { return `Null`; }
+    constructor() {
+        super(Type.Null);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitNullFieldType(this);
+    }
+}
+
+export class Int extends FieldType {
+    toString() { return `Int isSigned[${this.isSigned}], bitWidth[${this.bitWidth}]`; }
+    constructor(public isSigned: boolean, public bitWidth: IntBitWidth) {
+        super(Type.Int);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitIntFieldType(this);
+    }
+}
+
+export class FloatingPoint extends FieldType {
+    toString() { return `FloatingPoint precision`; }
+    constructor(public precision: Precision) {
+        super(Type.FloatingPoint);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitFloatingPointFieldType(this);
+    }
+}
+
+export class Binary extends FieldType {
+    toString() { return `Binary`; }
+    constructor() {
+        super(Type.Binary);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitBinaryFieldType(this);
+    }
+}
+
+export class Utf8 extends FieldType {
+    toString() { return `Utf8`; }
+    constructor() {
+        super(Type.Utf8);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitUtf8FieldType(this);
+    }
+}
+
+export class Bool extends FieldType {
+    toString() { return `Bool`; }
+    constructor() {
+        super(Type.Bool);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitBoolFieldType(this);
+    }
+}
+
+export class Decimal extends FieldType {
+    toString() { return `Decimal scale[${this.scale}], precision[${this.precision}]`; }
+    constructor(public scale: number, public precision: number) {
+        super(Type.Decimal);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitDecimalFieldType(this);
+    }
+}
+
+export class Date extends FieldType {
+    toString() { return `Date unit[${this.unit}]`; }
+    constructor(public unit: DateUnit) {
+        super(Type.Date);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitDateFieldType(this);
+    }
+}
+
+export class Time extends FieldType {
+    toString() { return `Time unit[${this.unit}], bitWidth[${this.bitWidth}]`; }
+    constructor(public unit: TimeUnit, public bitWidth: TimeBitWidth) {
+        super(Type.Time);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitTimeFieldType(this);
+    }
+}
+
+export class Timestamp extends FieldType {
+    toString() { return `Timestamp unit[${this.unit}], timezone[${this.timezone}]`; }
+    constructor(public unit: TimeUnit, public timezone?: string | null) {
+        super(Type.Timestamp);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitTimestampFieldType(this);
+    }
+}
+
+export class Interval extends FieldType {
+    toString() { return `Interval unit[${this.unit}]`; }
+    constructor(public unit: IntervalUnit) {
+        super(Type.Interval);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitIntervalFieldType(this);
+    }
+}
+
+export class List extends FieldType {
+    toString() { return `List`; }
+    constructor() {
+        super(Type.List);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitListFieldType(this);
+    }
+}
+
+export class Struct extends FieldType {
+    toString() { return `Struct`; }
+    constructor() {
+        super(Type.Struct_);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitStructFieldType(this);
+    }
+}
+
+export class Union extends FieldType {
+    toString() { return `Union mode[${this.mode}], typeIds[${this.typeIds}]`; }
+    constructor(public mode: UnionMode, public typeIds: Type[]) {
+        super(Type.Union);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitUnionFieldType(this);
+    }
+}
+
+export class FixedSizeBinary extends FieldType {
+    toString() { return `FixedSizeBinary byteWidth[${this.byteWidth}]`; }
+    constructor(public byteWidth: number) {
+        super(Type.FixedSizeBinary);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitFixedSizeBinaryFieldType(this);
+    }
+}
+
+export class FixedSizeList extends FieldType {
+    toString() { return `FixedSizeList listSize[${this.listSize}]`; }
+    constructor(public listSize: number) {
+        super(Type.FixedSizeList);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitFixedSizeListFieldType(this);
+    }
+}
+
+export class Map_ extends FieldType {
+    toString() { return `Map keysSorted[${this.keysSorted}]`; }
+    constructor(public keysSorted: boolean) {
+        super(Type.Map);
+    }
+    accept(visitor: Visitor) {
+        return visitor.visitMapFieldType(this);
+    }
+}
diff --git a/js/src/reader/arrow.ts b/js/src/reader/arrow.ts
index 8d3aafc722160..cf8a3d6a281a2 100644
--- a/js/src/reader/arrow.ts
+++ b/js/src/reader/arrow.ts
@@ -15,181 +15,34 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { readJSON } from './json';
+import { readBuffers, readBuffersAsync } from './buffer';
+import { readVectors, readVectorsAsync } from './vector';
 import { Vector } from '../vector/vector';
-import { flatbuffers } from 'flatbuffers';
-import { readVector, readValueVector } from './vector';
-import {
-    readFileFooter, readFileMessages,
-    readStreamSchema, readStreamMessages
-} from './format';
 
-import * as File_ from '../format/File';
-import * as Schema_ from '../format/Schema';
-import * as Message_ from '../format/Message';
+export { readJSON };
+export { readBuffers, readBuffersAsync };
+export { readVectors, readVectorsAsync };
 
-import ByteBuffer = flatbuffers.ByteBuffer;
-import Footer = File_.org.apache.arrow.flatbuf.Footer;
-import Field = Schema_.org.apache.arrow.flatbuf.Field;
-import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
-import Message = Message_.org.apache.arrow.flatbuf.Message;
-import ArrowBuffer = Schema_.org.apache.arrow.flatbuf.Buffer;
-import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
-import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
-import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
-import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
-import DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch;
-import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
-
-export type ArrowReaderContext = {
-    schema?: Schema;
-    footer?: Footer | null;
-    dictionaries: Map<string, Vector>;
-    dictionaryEncodedFields: Map<string, Field>;
-    readMessages: (bb: ByteBuffer, footer: Footer) => Iterable<Message>;
-};
-
-export interface VectorReaderContext {
-    offset: number;
-    bytes: Uint8Array;
-    batch: RecordBatch;
-    dictionaries: Map<string, Vector>;
-    readNextNode(): FieldNode;
-    readNextBuffer(): ArrowBuffer;
-}
-
-export function* readVectors(buffers: Iterable<Uint8Array | Buffer | string>, context?: ArrowReaderContext) {
-    const context_ = context || {} as ArrowReaderContext;
-    for (const buffer of buffers) {
-        yield* readBuffer(toByteBuffer(buffer), context_);
+export function* read(sources: Iterable<Uint8Array | Buffer | string> | object | string) {
+    let input: any = sources;
+    let batches: Iterable<Vector[]>;
+    if (typeof input === 'string') {
+        try { input = JSON.parse(input); }
+        catch (e) { input = sources; }
     }
-}
-
-export async function* readVectorsAsync(buffers: AsyncIterable<Uint8Array | Buffer | string>, context?: ArrowReaderContext) {
-    const context_ = context || {} as ArrowReaderContext;
-    for await (const buffer of buffers) {
-        yield* readBuffer(toByteBuffer(buffer), context_);
-    }
-}
-
-function* readBuffer(bb: ByteBuffer, readerContext: ArrowReaderContext) {
-
-    let { schema, footer, readMessages, dictionaryEncodedFields, dictionaries } = readerContext;
-
-    if (!schema) {
-        ({ schema, footer, readMessages, dictionaryEncodedFields } = readSchema(bb));
-        readerContext.schema = schema;
-        readerContext.readMessages = readMessages;
-        readerContext.dictionaryEncodedFields = dictionaryEncodedFields;
-        readerContext.dictionaries = dictionaries = new Map<string, Vector>();
-    }
-
-    const fieldsLength = schema.fieldsLength();
-    const context = new BufferReaderContext(bb.bytes(), dictionaries);
-
-    for (const message of readMessages(bb, footer!)) {
-
-        let id: string;
-        let field: Field;
-        let vector: Vector;
-        let vectors: Array<Vector>;
-
-        context.message = message;
-
-        if (message.headerType() === MessageHeader.DictionaryBatch) {
-            let batch: DictionaryBatch;
-            if (batch = message.header(new DictionaryBatch())!) {
-                context.batch = batch.data()!;
-                id = batch.id().toFloat64().toString();
-                field = dictionaryEncodedFields.get(id)!;
-                vector = readValueVector(field, context);
-                if (batch.isDelta() && dictionaries.has(id)) {
-                    vector = dictionaries.get(id)!.concat(vector);
-                }
-                dictionaries.set(id, vector);
-            }
-            continue;
-        }
-
-        vectors = new Array<Vector>(fieldsLength);
-        context.batch = message.header(new RecordBatch())!;
-
-        for (let i = -1; ++i < fieldsLength;) {
-            if ((field = schema.fields(i)!) || (vectors[i] = null as any)) {
-                vectors[i] = readVector(field, context);
-            }
-        }
-
-        yield vectors;
-    }
-}
-
-function readSchema(bb: ByteBuffer) {
-    let schema: Schema, readMessages, footer = readFileFooter(bb);
-    if (footer) {
-        schema = footer.schema()!;
-        readMessages = readFileMessages;
-    } else if (schema = readStreamSchema(bb)!) {
-        readMessages = readStreamMessages;
+    if (!input || typeof input !== 'object') {
+        batches = (typeof input === 'string') ? readVectors(readBuffers([input])) : [];
     } else {
-        throw new Error('Invalid Arrow buffer');
-    }
-    return { schema, footer, readMessages, dictionaryEncodedFields: readDictionaryEncodedFields(schema, new Map<string, Field>()) };
-}
-
-function readDictionaryEncodedFields(parent: Schema | Field, fields: Map<string, Field>) {
-    let field: Field, encoding: DictionaryEncoding, id: string;
-    let getField = parent instanceof Field ? parent.children : parent.fields;
-    let getFieldCount = parent instanceof Field ? parent.childrenLength : parent.fieldsLength;
-    for (let i = -1, n = getFieldCount.call(parent); ++i < n;) {
-        if (field = getField.call(parent, i)!) {
-            if ((encoding = field.dictionary()!) &&
-                (id = encoding.id().toFloat64().toString())) {
-                !fields.has(id) && fields.set(id, field);
-            }
-            readDictionaryEncodedFields(field, fields);
-        }
+        batches = (typeof input[Symbol.iterator] === 'function')
+            ? readVectors(readBuffers(input))
+            : readVectors(readJSON(input));
     }
-    return fields;
+    yield* batches;
 }
 
-function toByteBuffer(bytes?: Uint8Array | Buffer | string) {
-    let arr: Uint8Array = bytes as any || new Uint8Array(0);
-    if (typeof bytes === 'string') {
-        arr = new Uint8Array(bytes.length);
-        for (let i = -1, n = bytes.length; ++i < n;) {
-            arr[i] = bytes.charCodeAt(i);
-        }
-        return new ByteBuffer(arr);
+export async function* readAsync(sources: AsyncIterable<Uint8Array | Buffer | string>) {
+    for await (let vectors of readVectorsAsync(readBuffersAsync(sources))) {
+        yield vectors;
     }
-    return new ByteBuffer(arr);
 }
-
-class BufferReaderContext implements VectorReaderContext {
-    public offset: number;
-    public batch: RecordBatch;
-    private nodeIndex: number;
-    private bufferIndex: number;
-    private metadataVersion: MetadataVersion;
-    constructor(public bytes: Uint8Array,
-                public dictionaries: Map<string, Vector>) {
-    }
-    set message(m: Message) {
-        this.nodeIndex = 0;
-        this.bufferIndex = 0;
-        this.offset = m.bb.position();
-        this.metadataVersion = m.version();
-    }
-    public readNextNode() {
-        return this.batch.nodes(this.nodeIndex++)!;
-    }
-    public readNextBuffer() {
-        const buffer = this.batch.buffers(this.bufferIndex++)!;
-        // If this Arrow buffer was written before version 4,
-        // advance the buffer's bb_pos 8 bytes to skip past
-        // the now-removed page id field.
-        if (this.metadataVersion < MetadataVersion[`V4`]) {
-            buffer.bb_pos += (8 * this.bufferIndex);
-        }
-        return buffer;
-    }
-}
\ No newline at end of file
diff --git a/js/src/reader/buffer.ts b/js/src/reader/buffer.ts
new file mode 100644
index 0000000000000..c7b90507e396f
--- /dev/null
+++ b/js/src/reader/buffer.ts
@@ -0,0 +1,229 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import { VectorLayoutReader } from './vector';
+import { TypedArray, TypedArrayConstructor } from '../vector/types';
+import { footerFromByteBuffer, messageFromByteBuffer } from '../format/fb';
+import { Footer, Schema, RecordBatch, DictionaryBatch, Field, Buffer, FieldNode } from '../format/arrow';
+import ByteBuffer = flatbuffers.ByteBuffer;
+
+export function* readBuffers<T extends Uint8Array | NodeBuffer | string>(sources: Iterable<T>) {
+    let schema: Schema | null = null;
+    let readMessages: ((bb: ByteBuffer) => IterableIterator<RecordBatch | DictionaryBatch>) | null = null;
+    for (const source of sources) {
+        const bb = toByteBuffer(source);
+        if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) {
+            for (const message of readMessages(bb)) {
+                yield {
+                    schema, message, reader: new BufferVectorLayoutReader(
+                        bb,
+                        (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes),
+                        (function* (buffers) { yield* buffers; })(message.buffers)
+                    ) as VectorLayoutReader
+                };
+            }
+        }
+    }
+}
+
+export async function* readBuffersAsync<T extends Uint8Array | NodeBuffer | string>(sources: AsyncIterable<T>) {
+    let schema: Schema | null = null;
+    let readMessages: ((bb: ByteBuffer) => IterableIterator<RecordBatch | DictionaryBatch>) | null = null;
+    for await (const source of sources) {
+        const bb = toByteBuffer(source);
+        if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) {
+            for (const message of readMessages(bb)) {
+                yield {
+                    schema, message, reader: new BufferVectorLayoutReader(
+                        bb,
+                        (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes),
+                        (function* (buffers) { yield* buffers; })(message.buffers)
+                    ) as VectorLayoutReader
+                };
+            }
+        }
+    }
+}
+
+function toByteBuffer(bytes?: Uint8Array | NodeBuffer | string) {
+    let arr: Uint8Array = bytes as any || new Uint8Array(0);
+    if (typeof bytes === 'string') {
+        arr = new Uint8Array(bytes.length);
+        for (let i = -1, n = bytes.length; ++i < n;) {
+            arr[i] = bytes.charCodeAt(i);
+        }
+        return new ByteBuffer(arr);
+    }
+    return new ByteBuffer(arr);
+}
+
+function readSchema(bb: ByteBuffer) {
+    let schema: Schema, readMessages, footer: Footer | null;
+    if (footer = readFileSchema(bb)) {
+        schema = footer.schema!;
+        readMessages = readFileMessages(footer);
+    } else if (schema = readStreamSchema(bb)!) {
+        readMessages = readStreamMessages;
+    } else {
+        throw new Error('Invalid Arrow buffer');
+    }
+    return { schema, readMessages };
+}
+
+const PADDING = 4;
+const MAGIC_STR = 'ARROW1';
+const MAGIC = new Uint8Array(MAGIC_STR.length);
+for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
+    MAGIC[i] = MAGIC_STR.charCodeAt(i);
+}
+
+function checkForMagicArrowString(buffer: Uint8Array, index = 0) {
+    for (let i = -1, n = MAGIC.length; ++i < n;) {
+        if (MAGIC[i] !== buffer[index + i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+const magicLength = MAGIC.length;
+const magicAndPadding = magicLength + PADDING;
+const magicX2AndPadding = magicLength * 2 + PADDING;
+
+function readStreamSchema(bb: ByteBuffer) {
+    if (!checkForMagicArrowString(bb.bytes(), 0)) {
+        for (const message of readMessages(bb)) {
+            if (message.isSchema()) {
+                return message as Schema;
+            }
+        }
+    }
+    return null;
+}
+
+function* readStreamMessages(bb: ByteBuffer) {
+    for (const message of readMessages(bb)) {
+        if (message.isRecordBatch()) {
+            yield message;
+        } else if (message.isDictionaryBatch()) {
+            yield message;
+        } else {
+            continue;
+        }
+        // position the buffer after the body to read the next message
+        bb.setPosition(bb.position() + message.bodyLength.low);
+    }
+}
+
+function readFileSchema(bb: ByteBuffer) {
+    let fileLength = bb.capacity(), footerLength: number, footerOffset: number;
+    if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
+        (!checkForMagicArrowString(bb.bytes(), 0) /*                        Missing magic start    */) ||
+        (!checkForMagicArrowString(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
+        (/*                                                    Invalid footer length  */
+        (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
+        (footerLength + magicX2AndPadding > fileLength))) {
+        return null;
+    }
+    bb.setPosition(footerOffset - footerLength);
+    return footerFromByteBuffer(bb);
+}
+
+function readFileMessages(footer: Footer) {
+    return function* (bb: ByteBuffer) {
+        for (let i = -1, batches = footer.dictionaryBatches, n = batches.length; ++i < n;) {
+            bb.setPosition(batches[i].offset.low);
+            yield readMessage(bb, bb.readInt32(bb.position())) as DictionaryBatch;
+        }
+        for (let i = -1, batches = footer.recordBatches, n = batches.length; ++i < n;) {
+            bb.setPosition(batches[i].offset.low);
+            yield readMessage(bb, bb.readInt32(bb.position())) as RecordBatch;
+        }
+    };
+}
+
+function* readMessages(bb: ByteBuffer) {
+    let length: number, message: Schema | RecordBatch | DictionaryBatch;
+    while (bb.position() < bb.capacity() &&
+          (length = bb.readInt32(bb.position())) > 0) {
+        if (message = readMessage(bb, length)!) {
+            yield message;
+        }
+    }
+}
+
+function readMessage(bb: ByteBuffer, length: number) {
+    bb.setPosition(bb.position() + PADDING);
+    const message = messageFromByteBuffer(bb);
+    bb.setPosition(bb.position() + length);
+    return message;
+}
+
+class BufferVectorLayoutReader implements VectorLayoutReader {
+    private offset: number;
+    private bytes: Uint8Array;
+    constructor(bb: ByteBuffer, private fieldNodes: Iterator<FieldNode>, private buffers: Iterator<Buffer>) {
+        this.bytes = bb.bytes();
+        this.offset = bb.position();
+    }
+    readContainerLayout(field: Field) {
+        const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value;
+        return {
+            field, fieldNode,
+            validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value)
+        };
+    }
+    readFixedWidthLayout<T extends TypedArray>(field: Field, dataType: TypedArrayConstructor<T>) {
+        const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value;
+        return {
+            field, fieldNode,
+            validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value),
+            data: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, dataType)
+        };
+    }
+    readBinaryLayout(field: Field) {
+        const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value;
+        return {
+            field, fieldNode,
+            validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value),
+            offsets: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Int32Array),
+            data: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Uint8Array)
+        };
+    }
+    readVariableWidthLayout(field: Field) {
+        const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value;
+        return {
+            field, fieldNode,
+            validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value),
+            offsets: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Int32Array)
+        };
+    }
+}
+
+function createValidityArray(bytes: Uint8Array, field: Field, fieldNode: FieldNode, offset: number, buffer: Buffer) {
+    return field.nullable && fieldNode.nullCount.low > 0 &&
+        createTypedArray(bytes, field, fieldNode, offset, buffer, Uint8Array) || null;
+}
+
+function createTypedArray<T extends TypedArray>(bytes: Uint8Array, _field: Field, _fieldNode: FieldNode, offset: number, buffer: Buffer, ArrayConstructor: TypedArrayConstructor<T>): T {
+    return new ArrayConstructor(
+        bytes.buffer,
+        bytes.byteOffset + offset + buffer.offset.low,
+        buffer.length.low / ArrayConstructor.BYTES_PER_ELEMENT
+    );
+}
diff --git a/js/src/reader/format.ts b/js/src/reader/format.ts
deleted file mode 100644
index fd8f1b40d918a..0000000000000
--- a/js/src/reader/format.ts
+++ /dev/null
@@ -1,112 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { flatbuffers } from 'flatbuffers';
-import * as File_ from '../format/File';
-import * as Schema_ from '../format/Schema';
-import * as Message_ from '../format/Message';
-import ByteBuffer = flatbuffers.ByteBuffer;
-import Footer = File_.org.apache.arrow.flatbuf.Footer;
-import Schema = Schema_.org.apache.arrow.flatbuf.Schema;
-import Message = Message_.org.apache.arrow.flatbuf.Message;
-import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
-
-const PADDING = 4;
-const MAGIC_STR = 'ARROW1';
-const MAGIC = new Uint8Array(MAGIC_STR.length);
-for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
-    MAGIC[i] = MAGIC_STR.charCodeAt(i);
-}
-
-export function _checkMagic(buffer: Uint8Array, index = 0) {
-    for (let i = -1, n = MAGIC.length; ++i < n;) {
-        if (MAGIC[i] !== buffer[index + i]) {
-            return false;
-        }
-    }
-    return true;
-}
-
-const magicLength = MAGIC.length;
-const magicAndPadding = magicLength + PADDING;
-const magicX2AndPadding = magicLength * 2 + PADDING;
-
-export function readStreamSchema(bb: ByteBuffer) {
-    if (!_checkMagic(bb.bytes(), 0)) {
-        for (const message of readMessages(bb)) {
-            if (message.headerType() === MessageHeader.Schema) {
-                return message.header(new Schema());
-            }
-        }
-    }
-    return null;
-}
-
-export function readFileFooter(bb: ByteBuffer) {
-    let fileLength = bb.capacity();
-    let footerLength: number, footerOffset: number;
-    if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
-        (!_checkMagic(bb.bytes(), 0) /*                        Missing magic start    */) ||
-        (!_checkMagic(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
-        (/*                                                    Invalid footer length  */
-        (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
-        (footerLength + magicX2AndPadding > fileLength))) {
-        return null;
-    }
-    bb.setPosition(footerOffset - footerLength);
-    return Footer.getRootAsFooter(bb);
-}
-
-export function* readFileMessages(bb: ByteBuffer, footer: Footer) {
-    for (let i = -1, n = footer.dictionariesLength(); ++i < n;) {
-        bb.setPosition(footer.dictionaries(i)!.offset().low);
-        yield readMessage(bb, bb.readInt32(bb.position()));
-    }
-    for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) {
-        bb.setPosition(footer.recordBatches(i)!.offset().low);
-        yield readMessage(bb, bb.readInt32(bb.position()));
-    }
-}
-
-export function readMessage(bb: ByteBuffer, length: number) {
-    bb.setPosition(bb.position() + PADDING);
-    const message = Message.getRootAsMessage(bb);
-    bb.setPosition(bb.position() + length);
-    return message;
-}
-
-export function* readMessages(bb: ByteBuffer) {
-    let length;
-    while (bb.position() < bb.capacity() &&
-          (length = bb.readInt32(bb.position())) > 0) {
-        yield readMessage(bb, length);
-    }
-}
-
-export function* readStreamMessages(bb: ByteBuffer) {
-    for (const message of readMessages(bb)) {
-        switch (message.headerType()) {
-            case MessageHeader.RecordBatch:
-            case MessageHeader.DictionaryBatch:
-                yield message;
-                break;
-            default: continue;
-        }
-        // position the buffer after the body to read the next message
-        bb.setPosition(bb.position() + message.bodyLength().low);
-    }
-}
diff --git a/js/src/reader/json.ts b/js/src/reader/json.ts
new file mode 100644
index 0000000000000..49431496354e8
--- /dev/null
+++ b/js/src/reader/json.ts
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Schema_ from '../format/fb/Schema';
+import { Int64, Int128 } from '../util/int';
+import { VectorLayoutReader } from './vector';
+import { TextEncoder } from 'text-encoding-utf-8';
+import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
+import { TypedArray, TypedArrayConstructor } from '../vector/types';
+import { schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON } from '../format/json';
+import { Schema, RecordBatch, DictionaryBatch, Field, Buffer, FieldNode } from '../format/arrow';
+export { Schema, RecordBatch, DictionaryBatch };
+
+export function* readJSON(json: any) {
+    const schema = schemaFromJSON(json['schema']);
+    for (const batch of (json['dictionaries'] || [])) {
+        const message = dictionaryBatchFromJSON(batch);
+        yield {
+            schema, message, reader: new JSONVectorLayoutReader(
+                flattenDataSources(batch['data']['columns']),
+                (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes),
+                (function* (buffers) { yield* buffers; })(message.buffers)
+            ) as VectorLayoutReader
+        };
+    }
+    for (const batch of (json['batches'] || [])) {
+        const message = recordBatchFromJSON(batch);
+        yield {
+            schema, message, reader: new JSONVectorLayoutReader(
+                flattenDataSources(batch['columns']),
+                (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes),
+                (function* (buffers) { yield* buffers; })(message.buffers)
+            ) as VectorLayoutReader
+        };
+    }
+}
+
+function flattenDataSources(xs: any[]): any[][] {
+    return (xs || []).reduce<any[][]>((buffers, column: any) => [
+        ...buffers,
+        ...(column['VALIDITY'] && [column['VALIDITY']] || []),
+        ...(column['OFFSET'] && [column['OFFSET']] || []),
+        ...(column['DATA'] && [column['DATA']] || []),
+        ...flattenDataSources(column['children'])
+    ], [] as any[][]);
+}
+
+class JSONVectorLayoutReader implements VectorLayoutReader {
+    constructor(private sources: any[][], private fieldNodes: Iterator<FieldNode>, private buffers: Iterator<Buffer>) {}
+    readContainerLayout(field: Field) {
+        const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value;
+        return {
+            field, fieldNode,
+            validity: createValidityArray(sources, field, fieldNode, buffers.next().value)
+        };
+    }
+    readFixedWidthLayout<T extends TypedArray>(field: Field, dataType: TypedArrayConstructor<T>) {
+        const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value;
+        return {
+            field, fieldNode,
+            validity: createValidityArray(sources, field, fieldNode, buffers.next().value),
+            data: createDataArray(sources, field, fieldNode, buffers.next().value, dataType)
+        };
+    }
+    readBinaryLayout(field: Field) {
+        const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value;
+        return {
+            field, fieldNode,
+            validity: createValidityArray(sources, field, fieldNode, buffers.next().value),
+            offsets: new Int32Array(sources[buffers.next().value.offset.low]),
+            data: createDataArray(sources, field, fieldNode, buffers.next().value, Uint8Array)
+        };
+    }
+    readVariableWidthLayout(field: Field) {
+        const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value;
+        return {
+            field, fieldNode,
+            validity: createValidityArray(sources, field, fieldNode, buffers.next().value),
+            offsets: new Int32Array(sources[buffers.next().value.offset.low]),
+        };
+    }
+}
+
+function createValidityArray(sources: any[][], field: Field, fieldNode: FieldNode, buffer: Buffer) {
+    return field.nullable && fieldNode.nullCount.low > 0 &&
+        booleanFromJSON(sources[buffer.offset.low]) || null;
+}
+
+const encoder = new TextEncoder('utf-8');
+
+function createDataArray<T extends TypedArray>(sources: any[][], field: Field, _fieldNode: FieldNode, buffer: Buffer, ArrayConstructor: TypedArrayConstructor<T>): T {
+    let type = field.type, data: ArrayLike<number> | ArrayBufferLike;
+    if (type.isTimestamp() === true) {
+        data = int64sFromJSON(sources[buffer.offset.low] as string[]);
+    } else if ((type.isInt() || type.isTime()) && type.bitWidth === 64) {
+        data = int64sFromJSON(sources[buffer.offset.low] as string[]);
+    } else if (type.isDate() && type.unit === DateUnit.MILLISECOND) {
+        data = int64sFromJSON(sources[buffer.offset.low] as string[]);
+    } else if (type.isDecimal() === true) {
+        data = decimalFromJSON(sources[buffer.offset.low] as string[]);
+    } else if (type.isBinary() === true) {
+        data = binaryFromJSON(sources[buffer.offset.low] as string[]);
+    } else if (type.isBool() === true) {
+        data = booleanFromJSON(sources[buffer.offset.low] as number[]).buffer;
+    } else if (type.isUtf8() === true) {
+        data = encoder.encode((sources[buffer.offset.low] as string[]).join(''));
+    } else {
+        data = (sources[buffer.offset.low]).map((x) => +x);
+    }
+    return new ArrayConstructor(data);
+}
+
+function int64sFromJSON(values: string[]) {
+    const data = new Uint32Array(values.length * 2);
+    for (let i = -1, n = values.length; ++i < n;) {
+        // Force all values (even numbers) to be parsed as strings since
+        // pulling out high and low bits seems to lose precision sometimes
+        // For example:
+        //     > -4613034156400212000 >>> 0
+        //     721782784
+        // The correct lower 32-bits are 721782752
+        Int64.fromString(values[i].toString(), new Uint32Array(data.buffer, data.byteOffset + 2 * i * 4, 2));
+    }
+    return data.buffer;
+}
+
+function decimalFromJSON(values: string[]) {
+    const data = new Uint32Array(values.length * 4);
+    for (let i = -1, n = values.length; ++i < n;) {
+        Int128.fromString(values[i], new Uint32Array(data.buffer, data.byteOffset + 4 * 4 * i, 4));
+    }
+    return data.buffer;
+}
+
+function binaryFromJSON(values: string[]) {
+    // "DATA": ["49BC7D5B6C47D2","3F5FB6D9322026"]
+    // There are definitely more efficient ways to do this... but it gets the
+    // job done.
+    const joined = values.join('');
+    const data = new Uint8Array(joined.length / 2);
+    for (let i = 0; i < joined.length; i += 2) {
+        data[i >> 1] = parseInt(joined.substr(i, 2), 16);
+    }
+    return data.buffer;
+}
+
+function booleanFromJSON(arr: number[]) {
+    let xs = [], n, i = 0;
+    let bit = 0, byte = 0;
+    for (const value of arr) {
+        value && (byte |= 1 << bit);
+        if (++bit === 8) {
+            xs[i++] = byte;
+            byte = bit = 0;
+        }
+    }
+    if (i === 0 || bit > 0) { xs[i++] = byte; }
+    if (i % 8 && (n = i + 8 - i % 8)) {
+        do { xs[i] = 0; } while (++i < n);
+    }
+    return new Uint8Array(xs);
+}
diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts
index 0f95b769e141f..3bd6d2bb67650 100644
--- a/js/src/reader/vector.ts
+++ b/js/src/reader/vector.ts
@@ -15,10 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { VectorReaderContext } from './arrow';
-import * as Schema_ from '../format/Schema';
-import * as Message_ from '../format/Message';
+import * as Schema_ from '../format/fb/Schema';
 import { TypedArray, TypedArrayConstructor } from '../vector/types';
+import { Schema, RecordBatch, DictionaryBatch, Field, FieldNode } from '../format/arrow';
+import { Int, Date, Time, Timestamp, Decimal, FixedSizeList, FixedSizeBinary, FloatingPoint } from '../format/arrow';
 import {
     Vector, BoolVector, BinaryVector, DictionaryVector,
     Int8Vector, Int16Vector, Int32Vector, Int64Vector,
@@ -28,244 +28,228 @@ import {
     Date32Vector, Date64Vector, Time32Vector, Time64Vector, TimestampVector,
 } from '../vector/arrow';
 
-import Int = Schema_.org.apache.arrow.flatbuf.Int;
-import Date = Schema_.org.apache.arrow.flatbuf.Date;
-import Time = Schema_.org.apache.arrow.flatbuf.Time;
 import Type = Schema_.org.apache.arrow.flatbuf.Type;
-import Field = Schema_.org.apache.arrow.flatbuf.Field;
-import Buffer = Schema_.org.apache.arrow.flatbuf.Buffer;
-import Decimal = Schema_.org.apache.arrow.flatbuf.Decimal;
 import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
 import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
-// import Interval = Schema_.org.apache.arrow.flatbuf.Interval;
-import Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp;
-// import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
 import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
-import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
-import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
-import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
-import FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary;
+// import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
 
-export function readVector(field: Field, state: VectorReaderContext) {
-    return readDictionaryVector(field, state) || readValueVector(field, state);
+export interface ContainerLayout {
+    fieldNode: FieldNode;
+    validity: Uint8Array | null | void;
 }
 
-/* a dictionary index defaults to signed 32 bit int if unspecified */
-const defaultDictionaryIndexType = { bitWidth: () => 32, isSigned: () => true } as Int;
-const intVectors = [
-    [/* unsigned */ [Uint8Vector,  Uint8Array ],  /* signed */ [Int8Vector , Int8Array ]],
-    [/* unsigned */ [Uint16Vector, Uint16Array],  /* signed */ [Int16Vector, Int16Array]],
-    [/* unsigned */ [Uint32Vector, Uint32Array],  /* signed */ [Int32Vector, Int32Array]],,
-    [/* unsigned */ [Uint64Vector, Uint32Array],  /* signed */ [Int64Vector, Int32Array]]
-] as [any, TypedArrayConstructor][][];
-
-export function readDictionaryVector(field: Field, state: VectorReaderContext) {
-    const encoding = field.dictionary()!;
-    if (encoding) {
-        const type = encoding.indexType() || defaultDictionaryIndexType;
-        const data = state.dictionaries.get(encoding.id().toFloat64().toString())!;
-        const [IntVector, IntArray] = intVectors[type.bitWidth() >>> 4]![+type.isSigned()];
-        const { fieldNode, validity, data: keys } = readNumericBuffers(field, state, IntArray);
-        return new DictionaryVector({
-            validity, data, field, fieldNode,
-            keys: new IntVector({ field, fieldNode, data: keys })
-        });
-    }
-    return null;
+export interface VariableWidthLayout {
+    fieldNode: FieldNode;
+    offsets: Int32Array;
+    validity: Uint8Array | null | void;
 }
 
-export function readValueVector(field: Field, state: VectorReaderContext): Vector {
-    switch (field.typeType()) {
-        case Type.NONE: return readNullVector();
-        case Type.Null: return readNullVector();
-        // case Type.Map: return readMapVector(field, state);
-        case Type.Int: return readIntVector(field, state);
-        case Type.Bool: return readBoolVector(field, state);
-        case Type.Date: return readDateVector(field, state);
-        case Type.List: return readListVector(field, state);
-        case Type.Utf8: return readUtf8Vector(field, state);
-        case Type.Time: return readTimeVector(field, state);
-        // case Type.Union: return readUnionVector(field, state);
-        case Type.Binary: return readBinaryVector(field, state);
-        case Type.Decimal: return readDecimalVector(field, state);
-        case Type.Struct_: return readStructVector(field, state);
-        case Type.FloatingPoint: return readFloatVector(field, state);
-        case Type.Timestamp: return readTimestampVector(field, state);
-        case Type.FixedSizeList: return readFixedSizeListVector(field, state);
-        case Type.FixedSizeBinary: return readFixedSizeBinaryVector(field, state);
-    }
-    throw new Error(`Unrecognized Vector { name: ${Type[field.typeType()]}, type: ${field.typeType()} }`);
-}
-
-export function readNullVector() {
-    return new Vector();
+export interface BinaryLayout extends FixedWidthLayout<Uint8Array> {
+    offsets: Int32Array;
 }
 
-export function readBoolVector(field: Field, state: VectorReaderContext) {
-    return new BoolVector(readNumericBuffers(field, state, Uint8Array));
+export interface FixedWidthLayout<T extends TypedArray> {
+    fieldNode: FieldNode;
+    data: T;
+    validity: Uint8Array | null | void;
 }
 
-export function readDateVector(field: Field, state: VectorReaderContext) {
-    const type = field.type(new Date())!;
-    switch (type.unit()) {
-        case DateUnit.DAY: return new Date32Vector({ ...readNumericBuffers(field, state, Int32Array), unit: DateUnit[type.unit()] });
-        case DateUnit.MILLISECOND: return new Date64Vector({ ...readNumericBuffers(field, state, Int32Array), unit: DateUnit[type.unit()] });
+export function* readVectors(messages: Iterable<{ schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorLayoutReader }>) {
+    const dictionaries = new Map<string, Vector>();
+    for (const { schema, message, reader } of messages) {
+        yield* readMessageVectors(schema, message, new VectorReader(dictionaries, reader));
     }
-    throw new Error(`Unrecognized Date { unit: ${type.unit()} }`);
 }
 
-export function readTimeVector(field: Field, state: VectorReaderContext) {
-    const type = field.type(new Time())!;
-    switch (type.bitWidth()) {
-        case 32: return new Time32Vector({ ...readNumericBuffers(field, state, Int32Array), unit: TimeUnit[type.unit()] });
-        case 64: return new Time64Vector({ ...readNumericBuffers(field, state, Uint32Array), unit: TimeUnit[type.unit()] });
+export async function* readVectorsAsync(messages: AsyncIterable<{ schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorLayoutReader }>) {
+    const dictionaries = new Map<string, Vector>();
+    for await (const { schema, message, reader } of messages) {
+        yield* readMessageVectors(schema, message, new VectorReader(dictionaries, reader));
     }
-    throw new Error(`Unrecognized Time { unit: ${type.unit()}, bitWidth: ${type.bitWidth()} }`);
 }
 
-export function readTimestampVector(field: Field, state: VectorReaderContext) {
-    const type = field.type(new Timestamp())!;
-    const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint32Array);
-    return new TimestampVector({
-        field, fieldNode, validity, data,
-        timezone: type.timezone()!,
-        unit: TimeUnit[type.unit()],
-    });
-}
-
-export function readListVector(field: Field, state: VectorReaderContext) {
-    const { fieldNode, validity, offsets } = readListBuffers(field, state);
-    return new ListVector({
-        field, fieldNode, validity, offsets,
-        values: readVector(field.children(0)!, state)
-    });
-}
-
-export function readStructVector(field: Field, state: VectorReaderContext) {
-    const n = field.childrenLength();
-    const columns = new Array<Vector>(n);
-    const fieldNode = state.readNextNode();
-    const validity = readValidityBuffer(field, fieldNode, state);
-    for (let i = -1, child: Field; ++i < n;) {
-        if (child = field.children(i)!) {
-            columns[i] = readVector(child, state);
+function* readMessageVectors(schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorReader) {
+    if (message.isRecordBatch() === true) {
+        yield schema.fields.map((field) => reader.readVector(field));
+    } else if (message.isDictionaryBatch()) {
+        let id = message.dictionaryId.toFloat64().toString();
+        let vector = reader.readValueVector(schema.dictionaries.get(id)!);
+        if (message.isDelta) {
+            vector = reader.dictionaries.get(id)!.concat(vector);
         }
+        reader.dictionaries.set(id, vector);
     }
-    return new StructVector({ field, fieldNode, validity, columns });
-}
-
-export function readBinaryVector(field: Field, state: VectorReaderContext) {
-    return new BinaryVector(readBinaryBuffers(field, state));
-}
-
-export function readDecimalVector(field: Field, state: VectorReaderContext) {
-    const type = field.type(new Decimal())!;
-    const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint32Array);
-    return new DecimalVector({
-        scale: type.scale(),
-        precision: type.precision(),
-        field, fieldNode, validity, data
-    });
-}
-
-export function readUtf8Vector(field: Field, state: VectorReaderContext) {
-    const { fieldNode, validity, offsets, data } = readBinaryBuffers(field, state);
-    return new Utf8Vector({
-        field, fieldNode,
-        values: new BinaryVector({
-            validity, offsets, data
-        })
-    });
-}
-
-export function readFixedSizeListVector(field: Field, state: VectorReaderContext) {
-    const type = field.type(new FixedSizeList())!;
-    const fieldNode = state.readNextNode();
-    const validity = readValidityBuffer(field, fieldNode, state);
-    return new FixedSizeListVector({
-        field, fieldNode, validity,
-        size: type.listSize(),
-        values: readVector(field.children(0)!, state)
-    });
 }
 
-export function readFixedSizeBinaryVector(field: Field, state: VectorReaderContext) {
-    const type = field.type(new FixedSizeBinary())!;
-    const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint8Array);
-    return new FixedSizeListVector({
-        size: type.byteWidth(),
-        field, fieldNode, validity,
-        values: new Uint8Vector({ data })
-    });
+export interface VectorLayoutReader {
+    readBinaryLayout(field: Field): BinaryLayout;
+    readContainerLayout(field: Field): ContainerLayout;
+    readVariableWidthLayout(field: Field): VariableWidthLayout;
+    readFixedWidthLayout<T extends TypedArray>(field: Field, TypedArrayConstructor: TypedArrayConstructor<T>): FixedWidthLayout<T>;
 }
 
-export function readFloatVector(field: Field, state: VectorReaderContext) {
-    const type = field.type(new FloatingPoint())!;
-    switch (type.precision()) {
-        case Precision.HALF:   return new Float16Vector(readNumericBuffers(field, state, Uint16Array));
-        case Precision.SINGLE: return new Float32Vector(readNumericBuffers(field, state, Float32Array));
-        case Precision.DOUBLE: return new Float64Vector(readNumericBuffers(field, state, Float64Array));
+export class VectorReader implements VectorLayoutReader {
+    constructor(public dictionaries: Map<string, Vector>, protected layout: VectorLayoutReader) {}
+    readVector(field: Field): Vector {
+        return this.readDictionaryVector(field) || this.readValueVector(field);
     }
-    throw new Error(`Unrecognized FloatingPoint { precision: ${type.precision()} }`);
-}
-
-export function readIntVector(field: Field, state: VectorReaderContext) {
-    const type = field.type(new Int())!;
-    if (type.isSigned()) {
-        switch (type.bitWidth()) {
-            case  8: return new  Int8Vector(readNumericBuffers(field, state, Int8Array));
-            case 16: return new Int16Vector(readNumericBuffers(field, state, Int16Array));
-            case 32: return new Int32Vector(readNumericBuffers(field, state, Int32Array));
-            case 64: return new Int64Vector(readNumericBuffers(field, state, Int32Array));
+    readDictionaryVector(field: Field) {
+        const encoding = field.dictionary;
+        if (encoding) {
+            const keys = this.readIntVector(field.indexField());
+            const data = this.dictionaries.get(encoding.dictionaryId.toFloat64().toString())!;
+            return new DictionaryVector({
+                field, data, keys,
+                validity: (keys as any).validity,
+                fieldNode: (keys as any).fieldNode,
+            });
         }
+        return null;
     }
-    switch (type.bitWidth()) {
-        case  8: return new  Uint8Vector(readNumericBuffers(field, state, Uint8Array));
-        case 16: return new Uint16Vector(readNumericBuffers(field, state, Uint16Array));
-        case 32: return new Uint32Vector(readNumericBuffers(field, state, Uint32Array));
-        case 64: return new Uint64Vector(readNumericBuffers(field, state, Uint32Array));
+    readValueVector(field: Field) {
+        switch (field.typeType) {
+            case Type.NONE: return this.readNullVector();
+            case Type.Null: return this.readNullVector();
+            // case Type.Map: return this.readMapVector(field);
+            case Type.Int: return this.readIntVector(field);
+            case Type.Bool: return this.readBoolVector(field);
+            case Type.Date: return this.readDateVector(field);
+            case Type.List: return this.readListVector(field);
+            case Type.Utf8: return this.readUtf8Vector(field);
+            case Type.Time: return this.readTimeVector(field);
+            // case Type.Union: return this.readUnionVector(field);
+            case Type.Binary: return this.readBinaryVector(field);
+            case Type.Decimal: return this.readDecimalVector(field);
+            case Type.Struct_: return this.readStructVector(field);
+            case Type.FloatingPoint: return this.readFloatVector(field);
+            case Type.Timestamp: return this.readTimestampVector(field);
+            case Type.FixedSizeList: return this.readFixedSizeListVector(field);
+            case Type.FixedSizeBinary: return this.readFixedSizeBinaryVector(field);
+        }
+        throw new Error(`Unrecognized ${field.toString()}`);
+    }
+    readNullVector() {
+        return new Vector();
+    }
+    readBoolVector(field: Field) {
+        return new BoolVector(this.readFixedWidthLayout(field, Uint8Array));
+    }
+    readDateVector(field: Field) {
+        const type = field.type as Date;
+        switch (type.unit) {
+            case DateUnit.DAY: return new Date32Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: DateUnit[type.unit] });
+            case DateUnit.MILLISECOND: return new Date64Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: DateUnit[type.unit] });
+        }
+        throw new Error(`Unrecognized ${type.toString()}`);
+    }
+    readTimeVector(field: Field) {
+        const type = field.type as Time;
+        switch (type.bitWidth) {
+            case 32: return new Time32Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: TimeUnit[type.unit] });
+            case 64: return new Time64Vector({ ...this.readFixedWidthLayout(field, Uint32Array), unit: TimeUnit[type.unit] });
+        }
+        throw new Error(`Unrecognized ${type.toString()}`);
+    }
+    readTimestampVector(field: Field) {
+        const type = field.type as Timestamp;
+        const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint32Array);
+        return new TimestampVector({
+            field, fieldNode, validity, data,
+            timezone: type.timezone!,
+            unit: TimeUnit[type.unit],
+        });
+    }
+    readListVector(field: Field) {
+        const { fieldNode, validity, offsets } = this.readVariableWidthLayout(field);
+        return new ListVector({
+            field, fieldNode, validity, offsets,
+            values: this.readVector(field.children[0])
+        });
+    }
+    readStructVector(field: Field) {
+        const { fieldNode, validity } = this.readContainerLayout(field);
+        return new StructVector({
+            field, fieldNode, validity,
+            columns: field.children.map((field) => this.readVector(field))
+        });
+    }
+    readBinaryVector(field: Field) {
+        return new BinaryVector(this.readBinaryLayout(field));
+    }
+    readDecimalVector(field: Field) {
+        const type = field.type as Decimal;
+        const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint32Array);
+        return new DecimalVector({
+            scale: type.scale,
+            precision: type.precision,
+            field, fieldNode, validity, data
+        });
+    }
+    readUtf8Vector(field: Field) {
+        const { fieldNode, validity, offsets, data } = this.readBinaryLayout(field);
+        return new Utf8Vector({
+            field, fieldNode,
+            values: new BinaryVector({
+                validity, offsets, data
+            })
+        });
+    }
+    readFixedSizeListVector(field: Field) {
+        const type = field.type as FixedSizeList;
+        const { fieldNode, validity } = this.readContainerLayout(field);
+        return new FixedSizeListVector({
+            field, fieldNode, validity,
+            size: type.listSize,
+            values: this.readVector(field.children[0])
+        });
+    }
+    readFixedSizeBinaryVector(field: Field) {
+        const type = field.type as FixedSizeBinary;
+        const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint8Array);
+        return new FixedSizeListVector({
+            size: type.byteWidth,
+            field, fieldNode, validity,
+            values: new Uint8Vector({ data })
+        });
+    }
+    readFloatVector(field: Field) {
+        const type = field.type as FloatingPoint;
+        switch (type.precision) {
+            case Precision.HALF:   return new Float16Vector(this.readFixedWidthLayout(field, Uint16Array));
+            case Precision.SINGLE: return new Float32Vector(this.readFixedWidthLayout(field, Float32Array));
+            case Precision.DOUBLE: return new Float64Vector(this.readFixedWidthLayout(field, Float64Array));
+        }
+        throw new Error(`Unrecognized FloatingPoint { precision: ${type.precision} }`);
+    }
+    readIntVector(field: Field) {
+        const type = field.type as Int;
+        if (type.isSigned) {
+            switch (type.bitWidth) {
+                case  8: return new  Int8Vector(this.readFixedWidthLayout(field, Int8Array));
+                case 16: return new Int16Vector(this.readFixedWidthLayout(field, Int16Array));
+                case 32: return new Int32Vector(this.readFixedWidthLayout(field, Int32Array));
+                case 64: return new Int64Vector(this.readFixedWidthLayout(field, Int32Array));
+            }
+        }
+        switch (type.bitWidth) {
+            case  8: return new  Uint8Vector(this.readFixedWidthLayout(field, Uint8Array));
+            case 16: return new Uint16Vector(this.readFixedWidthLayout(field, Uint16Array));
+            case 32: return new Uint32Vector(this.readFixedWidthLayout(field, Uint32Array));
+            case 64: return new Uint64Vector(this.readFixedWidthLayout(field, Uint32Array));
+        }
+        throw new Error(`Unrecognized Int { isSigned: ${type.isSigned}, bitWidth: ${type.bitWidth} }`);
+    }
+    readContainerLayout(field: Field) {
+        return this.layout.readContainerLayout(field);
+    }
+    readBinaryLayout(field: Field) {
+        return this.layout.readBinaryLayout(field);
+    }
+    readVariableWidthLayout(field: Field) {
+        return this.layout.readVariableWidthLayout(field);
+    }
+    readFixedWidthLayout<T extends TypedArray>(field: Field, TypedArrayConstructor: TypedArrayConstructor<T>) {
+        return this.layout.readFixedWidthLayout(field, TypedArrayConstructor);
     }
-    throw new Error(`Unrecognized Int { isSigned: ${type.isSigned()}, bitWidth: ${type.bitWidth()} }`);
-}
-
-function readListBuffers(field: Field, state: VectorReaderContext) {
-    const fieldNode = state.readNextNode();
-    const validity = readValidityBuffer(field, fieldNode, state);
-    const offsets = readDataBuffer(Int32Array, state);
-    return { field, fieldNode, validity, offsets };
-}
-
-function readBinaryBuffers(field: Field, state: VectorReaderContext) {
-    const fieldNode = state.readNextNode();
-    const validity = readValidityBuffer(field, fieldNode, state);
-    const offsets = readDataBuffer(Int32Array, state);
-    const data = readDataBuffer(Uint8Array, state);
-    return { field, fieldNode, validity, offsets, data };
-}
-
-function readNumericBuffers<T extends TypedArray>(field: Field, state: VectorReaderContext, ArrayConstructor: TypedArrayConstructor<T>) {
-    const fieldNode = state.readNextNode();
-    const validity = readValidityBuffer(field, fieldNode, state);
-    const data = readDataBuffer(ArrayConstructor, state);
-    return { field, fieldNode, validity, data };
-}
-
-function readDataBuffer<T extends TypedArray>(ArrayConstructor: TypedArrayConstructor<T>, state: VectorReaderContext) {
-    return createTypedArray(ArrayConstructor, state.bytes, state.offset, state.readNextBuffer());
-}
-
-function readValidityBuffer(field: Field, fieldNode: FieldNode, state: VectorReaderContext) {
-    return createValidityArray(field, fieldNode, state.bytes, state.offset, state.readNextBuffer());
-}
-
-function createValidityArray(field: Field, fieldNode: FieldNode, bytes: Uint8Array, offset: number, buffer: Buffer) {
-    return field.nullable() && fieldNode.nullCount().low > 0 && createTypedArray(Uint8Array, bytes, offset, buffer) || null;
-}
-
-function createTypedArray<T extends TypedArray>(ArrayConstructor: TypedArrayConstructor<T>, bytes: Uint8Array, offset: number, buffer: Buffer) {
-    return new ArrayConstructor(
-        bytes.buffer,
-        bytes.byteOffset + offset + buffer.offset().low,
-        buffer.length().low / ArrayConstructor.BYTES_PER_ELEMENT
-    );
 }
diff --git a/js/src/util/int.ts b/js/src/util/int.ts
new file mode 100644
index 0000000000000..9088e7b995573
--- /dev/null
+++ b/js/src/util/int.ts
@@ -0,0 +1,320 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const carryBit16 = 1 << 16;
+
+function intAsHex(value: number): string {
+    if (value < 0) {
+        value = 0xFFFFFFFF + value + 1;
+    }
+    return `0x${value.toString(16)}`;
+}
+
+const kInt32DecimalDigits = 8;
+const kPowersOfTen = [1,
+                      10,
+                      100,
+                      1000,
+                      10000,
+                      100000,
+                      1000000,
+                      10000000,
+                      100000000];
+
+export class BaseInt64 {
+    constructor (protected buffer: Uint32Array) {}
+
+    high(): number { return this.buffer[1]; }
+    low (): number { return this.buffer[0]; }
+
+    protected _times(other: BaseInt64) {
+        // Break the left and right numbers into 16 bit chunks
+        // so that we can multiply them without overflow.
+        const L = new Uint32Array([
+            this.buffer[1] >>> 16,
+            this.buffer[1] & 0xFFFF,
+            this.buffer[0] >>> 16,
+            this.buffer[0] & 0xFFFF
+        ]);
+
+        const R = new Uint32Array([
+            other.buffer[1] >>> 16,
+            other.buffer[1] & 0xFFFF,
+            other.buffer[0] >>> 16,
+            other.buffer[0] & 0xFFFF
+        ]);
+
+        let product = L[3] * R[3];
+        this.buffer[0] = product & 0xFFFF;
+
+        let sum = product >>> 16;
+
+        product = L[2] * R[3];
+        sum += product;
+
+        product = (L[3] * R[2]) >>> 0;
+        sum += product;
+
+        this.buffer[0] += sum << 16;
+
+        this.buffer[1] = (sum >>> 0 < product ? carryBit16 : 0);
+
+        this.buffer[1] += sum >>> 16;
+        this.buffer[1] += L[1] * R[3] + L[2] * R[2] + L[3] * R[1];
+        this.buffer[1] += (L[0] * R[3] + L[1] * R[2] + L[2] * R[1] + L[3] * R[0]) << 16;
+
+        return this;
+      }
+
+    protected _plus(other: BaseInt64) {
+        const sum = (this.buffer[0] + other.buffer[0]) >>> 0;
+        this.buffer[1] += other.buffer[1];
+        if (sum < (this.buffer[0] >>> 0)) {
+          ++this.buffer[1];
+        }
+        this.buffer[0] = sum;
+    }
+
+    lessThan(other: BaseInt64): boolean {
+        return this.buffer[1] < other.buffer[1] ||
+            (this.buffer[1] === other.buffer[1] && this.buffer[0] < other.buffer[0]);
+    }
+
+    equals(other: BaseInt64): boolean {
+        return this.buffer[1] === other.buffer[1] && this.buffer[0] == other.buffer[0];
+    }
+
+    greaterThan(other: BaseInt64): boolean {
+        return other.lessThan(this);
+    }
+
+    hex(): string {
+        return `${intAsHex(this.buffer[1])} ${intAsHex(this.buffer[0])}`;
+    }
+}
+
+export class Uint64 extends BaseInt64 {
+    times(other: Uint64): Uint64 {
+        this._times(other);
+        return this;
+    }
+
+    plus(other: Uint64): Uint64 {
+        this._plus(other);
+        return this;
+    }
+
+    static multiply(left: Uint64, right: Uint64): Uint64 {
+        let rtrn = new Uint64(new Uint32Array(left.buffer));
+        return rtrn.times(right);
+    }
+
+    static add(left: Uint64, right: Uint64): Uint64 {
+        let rtrn = new Uint64(new Uint32Array(left.buffer));
+        return rtrn.plus(right);
+    }
+}
+
+export class Int64 extends BaseInt64 {
+    negate(): Int64 {
+        this.buffer[0] = ~this.buffer[0] + 1;
+        this.buffer[1] = ~this.buffer[1];
+
+        if (this.buffer[0] == 0) { ++this.buffer[1]; }
+        return this;
+    }
+
+    times(other: Int64): Int64 {
+        this._times(other);
+        return this;
+    }
+
+    plus(other: Int64): Int64 {
+        this._plus(other);
+        return this;
+    }
+
+    lessThan(other: Int64): boolean {
+        // force high bytes to be signed
+        const this_high = this.buffer[1] << 0;
+        const other_high = other.buffer[1] << 0;
+        return this_high < other_high ||
+            (this_high === other_high && this.buffer[0] < other.buffer[0]);
+    }
+
+    static fromString(str: string, out_buffer = new Uint32Array(2)): Int64 {
+        // TODO: Assert that out_buffer is 0 and length = 2
+        const negate = str.startsWith('-');
+        const length = str.length;
+
+        let out = new Int64(out_buffer);
+        for (let posn = negate ? 1 : 0; posn < length;) {
+            const group = kInt32DecimalDigits < length - posn ?
+                          kInt32DecimalDigits : length - posn;
+            const chunk = new Int64(new Uint32Array([parseInt(str.substr(posn, group), 10), 0]));
+            const multiple = new Int64(new Uint32Array([kPowersOfTen[group], 0]));
+
+            out.times(multiple);
+            out.plus(chunk);
+
+            posn += group;
+        }
+
+        return negate ? out.negate() : out;
+    }
+
+    static multiply(left: Int64, right: Int64): Int64 {
+        let rtrn = new Int64(new Uint32Array(left.buffer));
+        return rtrn.times(right);
+    }
+
+    static add(left: Int64, right: Int64): Int64 {
+        let rtrn = new Int64(new Uint32Array(left.buffer));
+        return rtrn.plus(right);
+    }
+}
+
+export class Int128 {
+    constructor (private buffer: Uint32Array) {
+        // buffer[3] MSB (high)
+        // buffer[2]
+        // buffer[1]
+        // buffer[0] LSB (low)
+    }
+
+    high(): Int64 {
+        return new Int64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset + 8, 2));
+    }
+
+    low(): Int64 {
+        return new Int64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset, 2));
+    }
+
+    negate(): Int128 {
+        this.buffer[0] = ~this.buffer[0] + 1;
+        this.buffer[1] = ~this.buffer[1];
+        this.buffer[2] = ~this.buffer[2];
+        this.buffer[3] = ~this.buffer[3];
+
+        if (this.buffer[0] == 0) { ++this.buffer[1]; }
+        if (this.buffer[1] == 0) { ++this.buffer[2]; }
+        if (this.buffer[2] == 0) { ++this.buffer[3]; }
+        return this;
+    }
+
+    times(other: Int128): Int128 {
+        // Break the left and right numbers into 32 bit chunks
+        // so that we can multiply them without overflow.
+        const L0 = new Uint64(new Uint32Array([this.buffer[3],  0]));
+        const L1 = new Uint64(new Uint32Array([this.buffer[2],  0]));
+        const L2 = new Uint64(new Uint32Array([this.buffer[1],  0]));
+        const L3 = new Uint64(new Uint32Array([this.buffer[0],  0]));
+
+        const R0 = new Uint64(new Uint32Array([other.buffer[3], 0]));
+        const R1 = new Uint64(new Uint32Array([other.buffer[2], 0]));
+        const R2 = new Uint64(new Uint32Array([other.buffer[1], 0]));
+        const R3 = new Uint64(new Uint32Array([other.buffer[0], 0]));
+
+        let product = Uint64.multiply(L3, R3);
+        this.buffer[0] = product.low();
+
+        let sum = new Uint64(new Uint32Array([product.high(), 0]));
+
+        product = Uint64.multiply(L2, R3);
+        sum.plus(product);
+
+        product = Uint64.multiply(L3, R2);
+        sum.plus(product);
+
+        this.buffer[1] = sum.low();
+
+        this.buffer[3] = (sum.lessThan(product) ? 1 : 0);
+
+        this.buffer[2] = sum.high();
+        let high = new Uint64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset + 8, 2));
+
+        high.plus(Uint64.multiply(L1, R3))
+            .plus(Uint64.multiply(L2, R2))
+            .plus(Uint64.multiply(L3, R1));
+        this.buffer[3] += Uint64.multiply(L0, R3)
+                        .plus(Uint64.multiply(L1, R2))
+                        .plus(Uint64.multiply(L2, R1))
+                        .plus(Uint64.multiply(L3, R0)).low();
+
+        return this;
+    }
+
+    plus(other: Int128): Int128 {
+        let sums = new Uint32Array(4);
+        sums[3] = (this.buffer[3] + other.buffer[3]) >>> 0;
+        sums[2] = (this.buffer[2] + other.buffer[2]) >>> 0;
+        sums[1] = (this.buffer[1] + other.buffer[1]) >>> 0;
+        sums[0] = (this.buffer[0] + other.buffer[0]) >>> 0;
+
+        if (sums[0] < (this.buffer[0] >>> 0)) {
+            ++sums[1];
+        }
+        if (sums[1] < (this.buffer[1] >>> 0)) {
+            ++sums[2];
+        }
+        if (sums[2] < (this.buffer[2] >>> 0)) {
+            ++sums[3];
+        }
+
+        this.buffer[3] = sums[3];
+        this.buffer[2] = sums[2];
+        this.buffer[1] = sums[1];
+        this.buffer[0] = sums[0];
+
+        return this;
+    }
+
+    hex(): string {
+        return `${intAsHex(this.buffer[3])} ${intAsHex(this.buffer[2])} ${intAsHex(this.buffer[1])} ${intAsHex(this.buffer[0])}`;
+    }
+
+    static multiply(left: Int128, right: Int128): Int128 {
+        let rtrn = new Int128(new Uint32Array(left.buffer));
+        return rtrn.times(right);
+    }
+
+    static add(left: Int128, right: Int128): Int128 {
+        let rtrn = new Int128(new Uint32Array(left.buffer));
+        return rtrn.plus(right);
+    }
+
+    static fromString(str: string, out_buffer = new Uint32Array(4)): Int128 {
+        // TODO: Assert that out_buffer is 0 and length = 4
+        const negate = str.startsWith('-');
+        const length = str.length;
+
+        let out = new Int128(out_buffer);
+        for (let posn = negate ? 1 : 0; posn < length;) {
+            const group = kInt32DecimalDigits < length - posn ?
+                          kInt32DecimalDigits : length - posn;
+            const chunk = new Int128(new Uint32Array([parseInt(str.substr(posn, group), 10), 0, 0, 0]));
+            const multiple = new Int128(new Uint32Array([kPowersOfTen[group], 0, 0, 0]));
+
+            out.times(multiple);
+            out.plus(chunk);
+
+            posn += group;
+        }
+
+        return negate ? out.negate() : out;
+    }
+}
diff --git a/js/src/util/layout.ts b/js/src/util/layout.ts
new file mode 100644
index 0000000000000..c064ee9d7d0b0
--- /dev/null
+++ b/js/src/util/layout.ts
@@ -0,0 +1,200 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { TextEncoder } from 'text-encoding-utf-8';
+import { TypedArrayConstructor, TypedArray } from '../vector/types';
+
+export function align(value: number, alignment: number) {
+    return value + padding(value, alignment);
+}
+
+export function padding(value: number, alignment: number) {
+    return (value % alignment === 0 ? 0 : alignment - value % alignment);
+}
+
+export type NullableLayout = { nullCount: number, validity: Uint8Array };
+export type BufferLayout<TArray = ArrayLike<number>> = { data: TArray };
+export type DictionaryLayout<TArray = ArrayLike<number>> = { data: TArray, keys: number[] };
+export type VariableWidthLayout<TArray = ArrayLike<number>> = { data: TArray, offsets: number[] };
+export type VariableWidthDictionaryLayout<TArray = ArrayLike<number>> = { data: TArray, keys: number[], offsets: number[] };
+
+export type values<T, TNull> = ArrayLike<T | TNull | null | undefined>;
+export type BufferValueWriter<T> = (src: ArrayLike<T>, dst: number[], index: number) => boolean | void;
+export type BufferWriter<T, TNull> = (values: values<T, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout;
+export type BufferLayoutWriter<T, TNull> = (write: BufferValueWriter<T>, values: values<T, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout;
+
+const writeNumeric64Value = writeFixedWidthValue.bind(null, 64);
+const writeNumeric128Value = writeFixedWidthValue.bind(null, 128);
+const utf8Encoder = new TextEncoder() as { encode: (s: string) => Uint8Array };
+
+const stride1Encode = writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 1));
+const stride1FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, stride1Encode));
+const stride2FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 2)));
+const stride4FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 4)));
+
+export const writeBools = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeBooleanValue), Uint8Array)                                                as <TNull>(values: values<boolean | number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint8Array>;
+export const writeInt8s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int8Array)                                                 as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Int8Array>;
+export const writeInt16s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int16Array)                                               as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Int16Array>;
+export const writeInt32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int32Array)                                               as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Int32Array>;
+export const writeInt64s = writeTypedLayout.bind(null, stride2FixedWidth.bind(null, writeNumeric64Value), Int32Array)                                             as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Int32Array>;
+export const writeUint8s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint8Array)                                               as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint8Array>;
+export const writeUint16s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint16Array)                                             as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint16Array>;
+export const writeUint32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint32Array)                                             as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint32Array>;
+export const writeUint64s = writeTypedLayout.bind(null, stride2FixedWidth.bind(null, writeNumeric64Value), Uint32Array)                                           as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint32Array>;
+export const writeDecimals = writeTypedLayout.bind(null, stride4FixedWidth.bind(null, writeNumeric128Value), Uint32Array)                                         as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint32Array>;
+export const writeFloat32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Float32Array)                                           as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Float32Array>;
+export const writeFloat64s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Float64Array)                                           as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Float64Array>;
+export const writeVariableWidth = writeVariableWidthLayout.bind(null, stride1Encode)                                                                              as <T, TNull>(writeValue: BufferValueWriter<T>, values: values<T, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthLayout<Uint8Array>;
+export const writeBinary = writeTypedLayout.bind(null, writeVariableWidth.bind(null, writeBinaryValue))                                                           as <TNull>(values: values<Iterable<number>, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthLayout<Uint8Array>;
+export const writeUtf8s = writeTypedLayout.bind(null, writeVariableWidth.bind(null, writeUtf8Value), Uint8Array)                                                  as <TNull>(values: values<string, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthLayout<Uint8Array>;
+export const writeDictionaryEncoded = writeDictionaryLayout.bind(null, stride1Encode)                                                                             as <T, TNull>(writeValue: BufferValueWriter<T>, values: values<T, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint8Array>;
+export const writeDictionaryEncodedBools = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeBooleanValue), Uint8Array)        as <TNull>(values: values<boolean | number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint8Array>;
+export const writeDictionaryEncodedInt8s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int8Array)         as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Int8Array>;
+export const writeDictionaryEncodedInt16s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int16Array)       as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Int16Array>;
+export const writeDictionaryEncodedInt32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int32Array)       as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Int32Array>;
+export const writeDictionaryEncodedInt64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride2FixedWidth, writeNumeric64Value), Int32Array)     as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Int32Array>;
+export const writeDictionaryEncodedUint8s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint8Array)       as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint8Array>;
+export const writeDictionaryEncodedUint16s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint16Array)     as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint16Array>;
+export const writeDictionaryEncodedUint32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint32Array)     as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint32Array>;
+export const writeDictionaryEncodedUint64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride2FixedWidth, writeNumeric64Value), Uint32Array)   as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint32Array>;
+export const writeDictionaryEncodedDecimals = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride4FixedWidth, writeNumeric128Value), Uint32Array) as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint32Array>;
+export const writeDictionaryEncodedFloat32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Float32Array)   as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Float32Array>;
+export const writeDictionaryEncodedFloat64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Float64Array)   as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Float64Array>;
+export const writeDictionaryEncodedVariableWidth = writeDictionaryLayout.bind(null, writeVariableWidth)                                                           as <T, TNull>(writeValue: BufferValueWriter<T>, values: values<T, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthDictionaryLayout<Uint8Array>;
+export const writeDictionaryEncodedBinary = writeTypedLayout.bind(null, writeDictionaryEncodedVariableWidth.bind(null, writeBinaryValue))                         as <TNull>(values: values<Iterable<number>, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthDictionaryLayout<Uint8Array>;
+export const writeDictionaryEncodedUtf8s = writeTypedLayout.bind(null, writeDictionaryEncodedVariableWidth.bind(null, writeUtf8Value), Uint8Array)                as <TNull>(values: values<string, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthDictionaryLayout<Uint8Array>;
+
+function writeFixedWidthLayoutWithStride<T, TNull>(
+    stride: number,
+    writeValue: BufferValueWriter<T>,
+    values: values<T, TNull>
+) {
+    let index = -stride;
+    const data = [] as number[];
+    const length = values.length;
+    while ((index += stride) < length) {
+        writeValue(values as ArrayLike<T>, data, index);
+    }
+    return { data: data as ArrayLike<number> };
+}
+
+function writeFixedWidthLayout<T, TNull>(
+    writeLayout: BufferLayoutWriter<T, TNull>,
+    writeValue: BufferValueWriter<T>,
+    values: values<T, TNull>,
+    nulls?: ArrayLike<TNull>
+) {
+    return writeLayout(writeValue, values, nulls);
+}
+
+function writeValidityLayout<T, TNull>(
+    writeLayout: BufferLayoutWriter<T, TNull>,
+    writeValue: BufferValueWriter<T>,
+    values: values<T, TNull>,
+    nulls?: ArrayLike<TNull>
+) {
+    let nullCount = 0;
+    let nullsLength = nulls && nulls.length || 0;
+    let validity = new Uint8Array(align(values.length >>> 3, 8)).fill(255);
+    return {
+        ...writeLayout(writeValueOrValidity, values),
+        nullCount, validity: (nullCount > 0 && validity) || new Uint8Array(0)
+    } as BufferLayout & NullableLayout;
+    function writeValueOrValidity(src: ArrayLike<T>, dst: number[], index: number) {
+        writeValue(src, dst, index);
+        let i = -1, x = src[index] as T | TNull;
+        let isNull = x === null || x === undefined;
+        while (!isNull && ++i < nullsLength) {
+            isNull = x === nulls![i];
+        }
+        if (isNull) {
+            nullCount++;
+            validity[index >> 3] &= ~(1 << (index % 8));
+        }
+    }
+}
+
+function writeVariableWidthLayout<T, TNull>(
+    writeLayout: BufferLayoutWriter<T, TNull>,
+    writeValue: BufferValueWriter<T>,
+    values: values<T, TNull>,
+    nulls?: ArrayLike<TNull>
+) {
+    let offsets = [0], offsetsIndex = 0;
+    return { ...writeLayout(writeValueAndOffset, values, nulls), offsets } as VariableWidthLayout;
+    function writeValueAndOffset(src: ArrayLike<T>, dst: number[], index: number) {
+        if (!writeValue(src, dst, index)) {
+            offsets[++offsetsIndex] = dst.length;
+        }
+    }
+}
+
+function writeDictionaryLayout<T, TNull>(
+    writeLayout: BufferLayoutWriter<T, TNull>,
+    writeValue: BufferValueWriter<T>,
+    values: values<T, TNull>,
+    nulls?: ArrayLike<TNull>
+) {
+    let keys = [] as number[], keysIndex = 0, keysMap = Object.create(null);
+    return { ...writeLayout(writeKeysOrValues, values, nulls), keys };
+    function writeKeysOrValues(src: ArrayLike<T>, dst: number[], index: number) {
+        const x: any = src[index];
+        if (x in keysMap) {
+            return (keys[index] = keysMap[x]) || true;
+        } else if (!writeValue(src, dst, index)) {
+            keys[index] = keysMap[x] = keysIndex++;
+        }
+    }
+}
+
+function writeTypedLayout<T, TNull, TArray extends TypedArray>(
+    writeBuffers: BufferWriter<T, TNull>,
+    ArrayBufferView: TypedArrayConstructor<TArray>,
+    values: values<T, TNull>,
+    nulls?: ArrayLike<TNull>
+) {
+    const result = writeBuffers(values, nulls);
+    result.data = new ArrayBufferView(result.data);
+    return result as BufferLayout<TArray>;
+}
+
+function writeBooleanValue(src: ArrayLike<boolean>, dst: number[], index: number) {
+    if (src[index]) {
+        let i = index >>> 3;
+        let b = dst[i] || 0;
+        dst[i] = b | 1 << (index % 8);
+    }
+}
+
+function writeNumericValue(src: ArrayLike<number>, dst: number[], index: number) {
+    dst[index] = +src[index];
+}
+
+function writeFixedWidthValue(bitWidth: number, src: ArrayLike<number>, dst: number[], index: number) {
+    const bytesLen = bitWidth / 32;
+    for (let i = -1; ++i < bytesLen;) {
+        dst[index + i] = src[index + i];
+    }
+}
+
+function writeUtf8Value(src: ArrayLike<string>, dst: number[], index: number) {
+    dst.push(...utf8Encoder.encode(src[index]));
+}
+
+function writeBinaryValue(src: ArrayLike<Iterable<number>>, dst: number[], index: number) {
+    dst.push(...src[index]);
+}
diff --git a/js/src/vector/arrow.ts b/js/src/vector/arrow.ts
index 88fec7e44542b..64a43bdab1c6b 100644
--- a/js/src/vector/arrow.ts
+++ b/js/src/vector/arrow.ts
@@ -15,49 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import * as Schema_ from '../format/Schema';
-import * as Message_ from '../format/Message';
-import Field = Schema_.org.apache.arrow.flatbuf.Field;
-import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
-
 import { Vector } from './vector';
-import { Utf8Vector as Utf8VectorBase } from './utf8';
-import { StructVector as StructVectorBase } from './struct';
-import { DictionaryVector as DictionaryVectorBase } from './dictionary';
-import {
-    ListVector as ListVectorBase,
-    BinaryVector as BinaryVectorBase,
-    FixedSizeListVector as FixedSizeListVectorBase
-} from './list';
-
-import {
-    BoolVector as BoolVectorBase,
-    Int8Vector as Int8VectorBase,
-    Int16Vector as Int16VectorBase,
-    Int32Vector as Int32VectorBase,
-    Int64Vector as Int64VectorBase,
-    Uint8Vector as Uint8VectorBase,
-    Uint16Vector as Uint16VectorBase,
-    Uint32Vector as Uint32VectorBase,
-    Uint64Vector as Uint64VectorBase,
-    Float16Vector as Float16VectorBase,
-    Float32Vector as Float32VectorBase,
-    Float64Vector as Float64VectorBase,
-    Date32Vector as Date32VectorBase,
-    Date64Vector as Date64VectorBase,
-    Time32Vector as Time32VectorBase,
-    Time64Vector as Time64VectorBase,
-    DecimalVector as DecimalVectorBase,
-    TimestampVector as TimestampVectorBase,
-} from './numeric';
-
-import { nullableMixin, fieldMixin } from './traits';
+import * as vectors from './traits/vectors';
+import * as fieldVectors from './traits/field';
+import * as nullableVectors from './traits/nullable';
+import * as nullableFieldVectors from './traits/nullablefield';
+import { Field, FieldNode } from '../format/arrow';
+import { isFieldArgv, isNullableArgv } from './traits/mixins';
 
 function MixinArrowTraits<T extends Vector<any>, TArgv>(
     Base: new (argv: TArgv) => T,
     Field: new (argv: TArgv & { field: Field, fieldNode: FieldNode }) => T,
     Nullable: new (argv: TArgv & { validity: Uint8Array }) => T,
-    NullableField: new (argv: TArgv & { validity: Uint8Array, field: Field, fieldNode: FieldNode }) => T,
+    NullableField: new (argv: TArgv & { validity: Uint8Array, field: Field, fieldNode: FieldNode }) => T
 ) {
     return function(argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })) {
         return new (!isFieldArgv(argv)
@@ -67,179 +37,52 @@ function MixinArrowTraits<T extends Vector<any>, TArgv>(
     } as any as { new (argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })): T };
 }
 
-function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } {
-    return x && x.field instanceof Field && x.fieldNode instanceof FieldNode;
-}
-
-function isNullableArgv(x: any): x is { validity: Uint8Array } {
-    return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array;
-}
-
 export { Vector };
-export class ListVector extends MixinArrowTraits(
-    ListVectorBase,
-    class ListVector extends fieldMixin(ListVectorBase) {} as any,
-    class ListVector extends nullableMixin(ListVectorBase) {} as any,
-    class ListVector extends nullableMixin(fieldMixin(ListVectorBase)) {} as any
-) {}
-
-export class BinaryVector extends MixinArrowTraits(
-    BinaryVectorBase,
-    class BinaryVector extends fieldMixin(BinaryVectorBase) {} as any,
-    class BinaryVector extends nullableMixin(BinaryVectorBase) {} as any,
-    class BinaryVector extends nullableMixin(fieldMixin(BinaryVectorBase)) {} as any
-) {}
-
-export class Utf8Vector extends MixinArrowTraits(
-    Utf8VectorBase,
-    class Utf8Vector extends fieldMixin(Utf8VectorBase) {} as any,
-    class Utf8Vector extends nullableMixin(Utf8VectorBase) {} as any,
-    class Utf8Vector extends nullableMixin(fieldMixin(Utf8VectorBase)) {} as any
-) {}
-
-export class BoolVector extends MixinArrowTraits(
-    BoolVectorBase,
-    class BoolVector extends fieldMixin(BoolVectorBase) {} as any,
-    class BoolVector extends nullableMixin(BoolVectorBase) {} as any,
-    class BoolVector extends nullableMixin(fieldMixin(BoolVectorBase)) {} as any
-) {}
-
-export class Int8Vector extends MixinArrowTraits(
-    Int8VectorBase,
-    class Int8Vector extends fieldMixin(Int8VectorBase) {} as any,
-    class Int8Vector extends nullableMixin(Int8VectorBase) {} as any,
-    class Int8Vector extends nullableMixin(fieldMixin(Int8VectorBase)) {} as any
-) {}
-
-export class Int16Vector extends MixinArrowTraits(
-    Int16VectorBase,
-    class Int16Vector extends fieldMixin(Int16VectorBase) {} as any,
-    class Int16Vector extends nullableMixin(Int16VectorBase) {} as any,
-    class Int16Vector extends nullableMixin(fieldMixin(Int16VectorBase)) {} as any
-) {}
-
-export class Int32Vector extends MixinArrowTraits(
-    Int32VectorBase,
-    class Int32Vector extends fieldMixin(Int32VectorBase) {} as any,
-    class Int32Vector extends nullableMixin(Int32VectorBase) {} as any,
-    class Int32Vector extends nullableMixin(fieldMixin(Int32VectorBase)) {} as any
-) {}
-
-export class Int64Vector extends MixinArrowTraits(
-    Int64VectorBase,
-    class Int64Vector extends fieldMixin(Int64VectorBase) {} as any,
-    class Int64Vector extends nullableMixin(Int64VectorBase) {} as any,
-    class Int64Vector extends nullableMixin(fieldMixin(Int64VectorBase)) {} as any
-) {}
-
-export class Uint8Vector extends MixinArrowTraits(
-    Uint8VectorBase,
-    class Uint8Vector extends fieldMixin(Uint8VectorBase) {} as any,
-    class Uint8Vector extends nullableMixin(Uint8VectorBase) {} as any,
-    class Uint8Vector extends nullableMixin(fieldMixin(Uint8VectorBase)) {} as any
-) {}
-
-export class Uint16Vector extends MixinArrowTraits(
-    Uint16VectorBase,
-    class Uint16Vector extends fieldMixin(Uint16VectorBase) {} as any,
-    class Uint16Vector extends nullableMixin(Uint16VectorBase) {} as any,
-    class Uint16Vector extends nullableMixin(fieldMixin(Uint16VectorBase)) {} as any
-) {}
-
-export class Uint32Vector extends MixinArrowTraits(
-    Uint32VectorBase,
-    class Uint32Vector extends fieldMixin(Uint32VectorBase) {} as any,
-    class Uint32Vector extends nullableMixin(Uint32VectorBase) {} as any,
-    class Uint32Vector extends nullableMixin(fieldMixin(Uint32VectorBase)) {} as any
-) {}
-
-export class Uint64Vector extends MixinArrowTraits(
-    Uint64VectorBase,
-    class Uint64Vector extends fieldMixin(Uint64VectorBase) {} as any,
-    class Uint64Vector extends nullableMixin(Uint64VectorBase) {} as any,
-    class Uint64Vector extends nullableMixin(fieldMixin(Uint64VectorBase)) {} as any
-) {}
-
-export class Date32Vector extends MixinArrowTraits(
-    Date32VectorBase,
-    class Date32Vector extends fieldMixin(Date32VectorBase) {} as any,
-    class Date32Vector extends nullableMixin(Date32VectorBase) {} as any,
-    class Date32Vector extends nullableMixin(fieldMixin(Date32VectorBase)) {} as any
-) {}
-
-export class Date64Vector extends MixinArrowTraits(
-    Date64VectorBase,
-    class Date64Vector extends fieldMixin(Date64VectorBase) {} as any,
-    class Date64Vector extends nullableMixin(Date64VectorBase) {} as any,
-    class Date64Vector extends nullableMixin(fieldMixin(Date64VectorBase)) {} as any
-) {}
-
-export class Time32Vector extends MixinArrowTraits(
-    Time32VectorBase,
-    class Time32Vector extends fieldMixin(Time32VectorBase) {} as any,
-    class Time32Vector extends nullableMixin(Time32VectorBase) {} as any,
-    class Time32Vector extends nullableMixin(fieldMixin(Time32VectorBase)) {} as any
-) {}
-
-export class Time64Vector extends MixinArrowTraits(
-    Time64VectorBase,
-    class Time64Vector extends fieldMixin(Time64VectorBase) {} as any,
-    class Time64Vector extends nullableMixin(Time64VectorBase) {} as any,
-    class Time64Vector extends nullableMixin(fieldMixin(Time64VectorBase)) {} as any
-) {}
-
-export class Float16Vector extends MixinArrowTraits(
-    Float16VectorBase,
-    class Float16Vector extends fieldMixin(Float16VectorBase) {} as any,
-    class Float16Vector extends nullableMixin(Float16VectorBase) {} as any,
-    class Float16Vector extends nullableMixin(fieldMixin(Float16VectorBase)) {} as any
-) {}
-
-export class Float32Vector extends MixinArrowTraits(
-    Float32VectorBase,
-    class Float32Vector extends fieldMixin(Float32VectorBase) {} as any,
-    class Float32Vector extends nullableMixin(Float32VectorBase) {} as any,
-    class Float32Vector extends nullableMixin(fieldMixin(Float32VectorBase)) {} as any
-) {}
-
-export class Float64Vector extends MixinArrowTraits(
-    Float64VectorBase,
-    class Float64Vector extends fieldMixin(Float64VectorBase) {} as any,
-    class Float64Vector extends nullableMixin(Float64VectorBase) {} as any,
-    class Float64Vector extends nullableMixin(fieldMixin(Float64VectorBase)) {} as any
-) {}
-
-export class StructVector extends MixinArrowTraits(
-    StructVectorBase,
-    class StructVector extends fieldMixin(StructVectorBase) {} as any,
-    class StructVector extends nullableMixin(StructVectorBase) {} as any,
-    class StructVector extends nullableMixin(fieldMixin(StructVectorBase)) {} as any
-) {}
-
-export class DecimalVector extends MixinArrowTraits(
-    DecimalVectorBase,
-    class DecimalVector extends fieldMixin(DecimalVectorBase) {} as any,
-    class DecimalVector extends nullableMixin(DecimalVectorBase) {} as any,
-    class DecimalVector extends nullableMixin(fieldMixin(DecimalVectorBase)) {} as any
-) {}
-
-export class TimestampVector extends MixinArrowTraits(
-    TimestampVectorBase,
-    class TimestampVector extends fieldMixin(TimestampVectorBase) {} as any,
-    class TimestampVector extends nullableMixin(TimestampVectorBase) {} as any,
-    class TimestampVector extends nullableMixin(fieldMixin(TimestampVectorBase)) {} as any
-) {}
-
-export class DictionaryVector extends MixinArrowTraits(
-    DictionaryVectorBase,
-    class DictionaryVector extends fieldMixin(DictionaryVectorBase) {} as any,
-    class DictionaryVector extends nullableMixin(DictionaryVectorBase) {} as any,
-    class DictionaryVector extends nullableMixin(fieldMixin(DictionaryVectorBase)) {} as any
-) {}
-
-export class FixedSizeListVector extends MixinArrowTraits(
-    FixedSizeListVectorBase,
-    class FixedSizeListVector extends fieldMixin(FixedSizeListVectorBase) {} as any,
-    class FixedSizeListVector extends nullableMixin(FixedSizeListVectorBase) {} as any,
-    class FixedSizeListVector extends nullableMixin(fieldMixin(FixedSizeListVectorBase)) {} as any
-) {}
+export const MixinListVector = MixinArrowTraits(vectors.ListVector as any, fieldVectors.ListVector as any, nullableVectors.ListVector as any, nullableFieldVectors.ListVector as any);
+export class ListVector extends MixinListVector {}
+export const MixinBinaryVector = MixinArrowTraits(vectors.BinaryVector as any, fieldVectors.BinaryVector as any, nullableVectors.BinaryVector as any, nullableFieldVectors.BinaryVector as any);
+export class BinaryVector extends MixinBinaryVector {}
+export const MixinUtf8Vector = MixinArrowTraits(vectors.Utf8Vector as any, fieldVectors.Utf8Vector as any, nullableVectors.Utf8Vector as any, nullableFieldVectors.Utf8Vector as any);
+export class Utf8Vector extends MixinUtf8Vector {}
+export const MixinBoolVector = MixinArrowTraits(vectors.BoolVector as any, fieldVectors.BoolVector as any, nullableVectors.BoolVector as any, nullableFieldVectors.BoolVector as any);
+export class BoolVector extends MixinBoolVector {}
+export const MixinInt8Vector = MixinArrowTraits(vectors.Int8Vector as any, fieldVectors.Int8Vector as any, nullableVectors.Int8Vector as any, nullableFieldVectors.Int8Vector as any);
+export class Int8Vector extends MixinInt8Vector {}
+export const MixinInt16Vector = MixinArrowTraits(vectors.Int16Vector as any, fieldVectors.Int16Vector as any, nullableVectors.Int16Vector as any, nullableFieldVectors.Int16Vector as any);
+export class Int16Vector extends MixinInt16Vector {}
+export const MixinInt32Vector = MixinArrowTraits(vectors.Int32Vector as any, fieldVectors.Int32Vector as any, nullableVectors.Int32Vector as any, nullableFieldVectors.Int32Vector as any);
+export class Int32Vector extends MixinInt32Vector {}
+export const MixinInt64Vector = MixinArrowTraits(vectors.Int64Vector as any, fieldVectors.Int64Vector as any, nullableVectors.Int64Vector as any, nullableFieldVectors.Int64Vector as any);
+export class Int64Vector extends MixinInt64Vector {}
+export const MixinUint8Vector = MixinArrowTraits(vectors.Uint8Vector as any, fieldVectors.Uint8Vector as any, nullableVectors.Uint8Vector as any, nullableFieldVectors.Uint8Vector as any);
+export class Uint8Vector extends MixinUint8Vector {}
+export const MixinUint16Vector = MixinArrowTraits(vectors.Uint16Vector as any, fieldVectors.Uint16Vector as any, nullableVectors.Uint16Vector as any, nullableFieldVectors.Uint16Vector as any);
+export class Uint16Vector extends MixinUint16Vector {}
+export const MixinUint32Vector = MixinArrowTraits(vectors.Uint32Vector as any, fieldVectors.Uint32Vector as any, nullableVectors.Uint32Vector as any, nullableFieldVectors.Uint32Vector as any);
+export class Uint32Vector extends MixinUint32Vector {}
+export const MixinUint64Vector = MixinArrowTraits(vectors.Uint64Vector as any, fieldVectors.Uint64Vector as any, nullableVectors.Uint64Vector as any, nullableFieldVectors.Uint64Vector as any);
+export class Uint64Vector extends MixinUint64Vector {}
+export const MixinDate32Vector = MixinArrowTraits(vectors.Date32Vector as any, fieldVectors.Date32Vector as any, nullableVectors.Date32Vector as any, nullableFieldVectors.Date32Vector as any);
+export class Date32Vector extends MixinDate32Vector {}
+export const MixinDate64Vector = MixinArrowTraits(vectors.Date64Vector as any, fieldVectors.Date64Vector as any, nullableVectors.Date64Vector as any, nullableFieldVectors.Date64Vector as any);
+export class Date64Vector extends MixinDate64Vector {}
+export const MixinTime32Vector = MixinArrowTraits(vectors.Time32Vector as any, fieldVectors.Time32Vector as any, nullableVectors.Time32Vector as any, nullableFieldVectors.Time32Vector as any);
+export class Time32Vector extends MixinTime32Vector {}
+export const MixinTime64Vector = MixinArrowTraits(vectors.Time64Vector as any, fieldVectors.Time64Vector as any, nullableVectors.Time64Vector as any, nullableFieldVectors.Time64Vector as any);
+export class Time64Vector extends MixinTime64Vector {}
+export const MixinFloat16Vector = MixinArrowTraits(vectors.Float16Vector as any, fieldVectors.Float16Vector as any, nullableVectors.Float16Vector as any, nullableFieldVectors.Float16Vector as any);
+export class Float16Vector extends MixinFloat16Vector {}
+export const MixinFloat32Vector = MixinArrowTraits(vectors.Float32Vector as any, fieldVectors.Float32Vector as any, nullableVectors.Float32Vector as any, nullableFieldVectors.Float32Vector as any);
+export class Float32Vector extends MixinFloat32Vector {}
+export const MixinFloat64Vector = MixinArrowTraits(vectors.Float64Vector as any, fieldVectors.Float64Vector as any, nullableVectors.Float64Vector as any, nullableFieldVectors.Float64Vector as any);
+export class Float64Vector extends MixinFloat64Vector {}
+export const MixinStructVector = MixinArrowTraits(vectors.StructVector as any, fieldVectors.StructVector as any, nullableVectors.StructVector as any, nullableFieldVectors.StructVector as any);
+export class StructVector extends MixinStructVector {}
+export const MixinDecimalVector = MixinArrowTraits(vectors.DecimalVector as any, fieldVectors.DecimalVector as any, nullableVectors.DecimalVector as any, nullableFieldVectors.DecimalVector as any);
+export class DecimalVector extends MixinDecimalVector {}
+export const MixinTimestampVector = MixinArrowTraits(vectors.TimestampVector as any, fieldVectors.TimestampVector as any, nullableVectors.TimestampVector as any, nullableFieldVectors.TimestampVector as any);
+export class TimestampVector extends MixinTimestampVector {}
+export const MixinDictionaryVector = MixinArrowTraits(vectors.DictionaryVector as any, fieldVectors.DictionaryVector as any, nullableVectors.DictionaryVector as any, nullableFieldVectors.DictionaryVector as any);
+export class DictionaryVector extends MixinDictionaryVector {}
+export const MixinFixedSizeListVector = MixinArrowTraits(vectors.FixedSizeListVector as any, fieldVectors.FixedSizeListVector as any, nullableVectors.FixedSizeListVector as any, nullableFieldVectors.FixedSizeListVector as any);
+export class FixedSizeListVector extends MixinFixedSizeListVector {}
diff --git a/js/src/vector/table.ts b/js/src/vector/table.ts
index b15092a72df24..ca2b66a22da80 100644
--- a/js/src/vector/table.ts
+++ b/js/src/vector/table.ts
@@ -17,27 +17,29 @@
 
 import { Vector } from './vector';
 import { StructVector, StructRow } from './struct';
-import { readVectors, readVectorsAsync } from '../reader/arrow';
+import { read, readAsync } from '../reader/arrow';
+
+function concatVectors(tableVectors: Vector<any>[], batchVectors: Vector<any>[]) {
+    return tableVectors.length === 0 ? batchVectors : batchVectors.map((vec, i, _vs, col = tableVectors[i]) =>
+        vec && col && col.concat(vec) || col || vec
+    ) as Vector<any>[];
+}
 
 export class Table<T> extends StructVector<T> {
-    static from(buffers?: Iterable<Uint8Array | Buffer | string>) {
+    static from(sources?: Iterable<Uint8Array | Buffer | string> | object | string) {
         let columns: Vector<any>[] = [];
-        if (buffers) {
-            for (let vectors of readVectors(buffers)) {
-                columns = columns.length === 0 ? vectors : vectors.map((vec, i, _vs, col = columns[i]) =>
-                    vec && col && col.concat(vec) || col || vec
-                ) as Vector<any>[];
+        if (sources) {
+            for (let vectors of read(sources)) {
+                columns = concatVectors(columns, vectors);
             }
         }
         return new Table({ columns });
     }
-    static async fromAsync(buffers?: AsyncIterable<Uint8Array | Buffer | string>) {
+    static async fromAsync(sources?: AsyncIterable<Uint8Array | Buffer | string>) {
         let columns: Vector<any>[] = [];
-        if (buffers) {
-            for await (let vectors of readVectorsAsync(buffers)) {
-                columns = columns.length === 0 ? vectors : vectors.map((vec, i, _vs, col = columns[i]) =>
-                    vec && col && col.concat(vec) || col || vec
-                ) as Vector<any>[];
+        if (sources) {
+            for await (let vectors of readAsync(sources)) {
+                columns = columns = concatVectors(columns, vectors);
             }
         }
         return new Table({ columns });
diff --git a/js/src/vector/traits/field.ts b/js/src/vector/traits/field.ts
new file mode 100644
index 0000000000000..9f68f507c6f10
--- /dev/null
+++ b/js/src/vector/traits/field.ts
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from '../vector';
+import * as vectors from './vectors';
+import { fieldMixin } from './mixins';
+import { Field, FieldNode } from '../../format/arrow';
+export { Field, FieldNode, Vector };
+
+export const FieldListVector = fieldMixin(vectors.ListVector);
+export class ListVector extends FieldListVector {}
+export const FieldBinaryVector = fieldMixin(vectors.BinaryVector);
+export class BinaryVector extends FieldBinaryVector {}
+export const FieldUtf8Vector = fieldMixin(vectors.Utf8Vector);
+export class Utf8Vector extends FieldUtf8Vector {}
+export const FieldBoolVector = fieldMixin(vectors.BoolVector);
+export class BoolVector extends FieldBoolVector {}
+export const FieldInt8Vector = fieldMixin(vectors.Int8Vector);
+export class Int8Vector extends FieldInt8Vector {}
+export const FieldInt16Vector = fieldMixin(vectors.Int16Vector);
+export class Int16Vector extends FieldInt16Vector {}
+export const FieldInt32Vector = fieldMixin(vectors.Int32Vector);
+export class Int32Vector extends FieldInt32Vector {}
+export const FieldInt64Vector = fieldMixin(vectors.Int64Vector);
+export class Int64Vector extends FieldInt64Vector {}
+export const FieldUint8Vector = fieldMixin(vectors.Uint8Vector);
+export class Uint8Vector extends FieldUint8Vector {}
+export const FieldUint16Vector = fieldMixin(vectors.Uint16Vector);
+export class Uint16Vector extends FieldUint16Vector {}
+export const FieldUint32Vector = fieldMixin(vectors.Uint32Vector);
+export class Uint32Vector extends FieldUint32Vector {}
+export const FieldUint64Vector = fieldMixin(vectors.Uint64Vector);
+export class Uint64Vector extends FieldUint64Vector {}
+export const FieldDate32Vector = fieldMixin(vectors.Date32Vector);
+export class Date32Vector extends FieldDate32Vector {}
+export const FieldDate64Vector = fieldMixin(vectors.Date64Vector);
+export class Date64Vector extends FieldDate64Vector {}
+export const FieldTime32Vector = fieldMixin(vectors.Time32Vector);
+export class Time32Vector extends FieldTime32Vector {}
+export const FieldTime64Vector = fieldMixin(vectors.Time64Vector);
+export class Time64Vector extends FieldTime64Vector {}
+export const FieldFloat16Vector = fieldMixin(vectors.Float16Vector);
+export class Float16Vector extends FieldFloat16Vector {}
+export const FieldFloat32Vector = fieldMixin(vectors.Float32Vector);
+export class Float32Vector extends FieldFloat32Vector {}
+export const FieldFloat64Vector = fieldMixin(vectors.Float64Vector);
+export class Float64Vector extends FieldFloat64Vector {}
+export const FieldStructVector = fieldMixin(vectors.StructVector);
+export class StructVector extends FieldStructVector {}
+export const FieldDecimalVector = fieldMixin(vectors.DecimalVector);
+export class DecimalVector extends FieldDecimalVector {}
+export const FieldTimestampVector = fieldMixin(vectors.TimestampVector);
+export class TimestampVector extends FieldTimestampVector {}
+export const FieldDictionaryVector = fieldMixin(vectors.DictionaryVector);
+export class DictionaryVector extends FieldDictionaryVector {}
+export const FieldFixedSizeListVector = fieldMixin(vectors.FixedSizeListVector);
+export class FixedSizeListVector extends FieldFixedSizeListVector {}
\ No newline at end of file
diff --git a/js/src/vector/traits.ts b/js/src/vector/traits/mixins.ts
similarity index 67%
rename from js/src/vector/traits.ts
rename to js/src/vector/traits/mixins.ts
index ca933f1605957..011ac31352db6 100644
--- a/js/src/vector/traits.ts
+++ b/js/src/vector/traits/mixins.ts
@@ -15,13 +15,27 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Vector } from './vector';
-import { BoolVector } from './numeric';
-import * as Schema_ from '../format/Schema';
-import * as Message_ from '../format/Message';
+import { Vector } from '../vector';
+import { BoolVector } from '../numeric';
+import * as Schema_ from '../../format/fb/Schema';
+import { Field, FieldNode } from '../../format/arrow';
 import Type = Schema_.org.apache.arrow.flatbuf.Type;
-import Field = Schema_.org.apache.arrow.flatbuf.Field;
-import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+
+function isField(x: any): x is Field {
+    return x instanceof Field;
+}
+
+function isFieldNode(x: any): x is FieldNode {
+    return x instanceof FieldNode;
+}
+
+export function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } {
+    return x && isField(x.field) && isFieldNode(x.fieldNode);
+}
+
+export function isNullableArgv(x: any): x is { validity: Uint8Array } {
+    return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array;
+}
 
 type Ctor<TArgv> = new (argv: TArgv) => Vector;
 
@@ -51,19 +65,11 @@ export const fieldMixin = <T extends Vector, TArgv>(superclass: new (argv: TArgv
             const { field, fieldNode } = argv;
             this.field = field;
             this.fieldNode = fieldNode;
-            this.nullable = field.nullable();
-            this.type = Type[field.typeType()];
-            this.length = fieldNode.length().low | 0;
-            this.nullCount = fieldNode.nullCount().low;
-        }
-        get name() { return this.field.name()!; }
-        get metadata()  {
-            const { field } = this, data = new Map<string, string>();
-            for (let entry, key, i = -1, n = field && field.customMetadataLength() | 0; ++i < n;) {
-                if ((entry = field.customMetadata(i)) && (key = entry.key()) != null) {
-                    data.set(key, entry.value()!);
-                }
-            }
-            return data;
+            this.nullable = field.nullable;
+            this.type = Type[field.typeType];
+            this.length = fieldNode.length.low | 0;
+            this.nullCount = fieldNode.nullCount.low;
         }
+        get name() { return this.field.name!; }
+        get metadata()  { return this.field.metadata!; }
     };
diff --git a/js/src/vector/traits/nullable.ts b/js/src/vector/traits/nullable.ts
new file mode 100644
index 0000000000000..1393e5fd1bf68
--- /dev/null
+++ b/js/src/vector/traits/nullable.ts
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from '../vector';
+import * as vectors from './vectors';
+import { nullableMixin } from './mixins';
+
+export { Vector };
+export const NullableListVector = nullableMixin(vectors.ListVector);
+export class ListVector extends NullableListVector {}
+export const NullableBinaryVector = nullableMixin(vectors.BinaryVector);
+export class BinaryVector extends NullableBinaryVector {}
+export const NullableUtf8Vector = nullableMixin(vectors.Utf8Vector);
+export class Utf8Vector extends NullableUtf8Vector {}
+export const NullableBoolVector = nullableMixin(vectors.BoolVector);
+export class BoolVector extends NullableBoolVector {}
+export const NullableInt8Vector = nullableMixin(vectors.Int8Vector);
+export class Int8Vector extends NullableInt8Vector {}
+export const NullableInt16Vector = nullableMixin(vectors.Int16Vector);
+export class Int16Vector extends NullableInt16Vector {}
+export const NullableInt32Vector = nullableMixin(vectors.Int32Vector);
+export class Int32Vector extends NullableInt32Vector {}
+export const NullableInt64Vector = nullableMixin(vectors.Int64Vector);
+export class Int64Vector extends NullableInt64Vector {}
+export const NullableUint8Vector = nullableMixin(vectors.Uint8Vector);
+export class Uint8Vector extends NullableUint8Vector {}
+export const NullableUint16Vector = nullableMixin(vectors.Uint16Vector);
+export class Uint16Vector extends NullableUint16Vector {}
+export const NullableUint32Vector = nullableMixin(vectors.Uint32Vector);
+export class Uint32Vector extends NullableUint32Vector {}
+export const NullableUint64Vector = nullableMixin(vectors.Uint64Vector);
+export class Uint64Vector extends NullableUint64Vector {}
+export const NullableDate32Vector = nullableMixin(vectors.Date32Vector);
+export class Date32Vector extends NullableDate32Vector {}
+export const NullableDate64Vector = nullableMixin(vectors.Date64Vector);
+export class Date64Vector extends NullableDate64Vector {}
+export const NullableTime32Vector = nullableMixin(vectors.Time32Vector);
+export class Time32Vector extends NullableTime32Vector {}
+export const NullableTime64Vector = nullableMixin(vectors.Time64Vector);
+export class Time64Vector extends NullableTime64Vector {}
+export const NullableFloat16Vector = nullableMixin(vectors.Float16Vector);
+export class Float16Vector extends NullableFloat16Vector {}
+export const NullableFloat32Vector = nullableMixin(vectors.Float32Vector);
+export class Float32Vector extends NullableFloat32Vector {}
+export const NullableFloat64Vector = nullableMixin(vectors.Float64Vector);
+export class Float64Vector extends NullableFloat64Vector {}
+export const NullableStructVector = nullableMixin(vectors.StructVector);
+export class StructVector extends NullableStructVector {}
+export const NullableDecimalVector = nullableMixin(vectors.DecimalVector);
+export class DecimalVector extends NullableDecimalVector {}
+export const NullableTimestampVector = nullableMixin(vectors.TimestampVector);
+export class TimestampVector extends NullableTimestampVector {}
+export const NullableDictionaryVector = nullableMixin(vectors.DictionaryVector);
+export class DictionaryVector extends NullableDictionaryVector {}
+export const NullableFixedSizeListVector = nullableMixin(vectors.FixedSizeListVector);
+export class FixedSizeListVector extends NullableFixedSizeListVector {}
\ No newline at end of file
diff --git a/js/src/vector/traits/nullablefield.ts b/js/src/vector/traits/nullablefield.ts
new file mode 100644
index 0000000000000..8cbee62e43dc3
--- /dev/null
+++ b/js/src/vector/traits/nullablefield.ts
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from '../vector';
+import * as vectors from './vectors';
+import { nullableMixin, fieldMixin } from './mixins';
+import { Field, FieldNode } from '../../format/arrow';
+export { Vector, Field, FieldNode };
+
+export const NullableFieldListVector = nullableMixin(fieldMixin(vectors.ListVector));
+export class ListVector extends NullableFieldListVector {}
+export const NullableFieldBinaryVector = nullableMixin(fieldMixin(vectors.BinaryVector));
+export class BinaryVector extends NullableFieldBinaryVector {}
+export const NullableFieldUtf8Vector = nullableMixin(fieldMixin(vectors.Utf8Vector));
+export class Utf8Vector extends NullableFieldUtf8Vector {}
+export const NullableFieldBoolVector = nullableMixin(fieldMixin(vectors.BoolVector));
+export class BoolVector extends NullableFieldBoolVector {}
+export const NullableFieldInt8Vector = nullableMixin(fieldMixin(vectors.Int8Vector));
+export class Int8Vector extends NullableFieldInt8Vector {}
+export const NullableFieldInt16Vector = nullableMixin(fieldMixin(vectors.Int16Vector));
+export class Int16Vector extends NullableFieldInt16Vector {}
+export const NullableFieldInt32Vector = nullableMixin(fieldMixin(vectors.Int32Vector));
+export class Int32Vector extends NullableFieldInt32Vector {}
+export const NullableFieldInt64Vector = nullableMixin(fieldMixin(vectors.Int64Vector));
+export class Int64Vector extends NullableFieldInt64Vector {}
+export const NullableFieldUint8Vector = nullableMixin(fieldMixin(vectors.Uint8Vector));
+export class Uint8Vector extends NullableFieldUint8Vector {}
+export const NullableFieldUint16Vector = nullableMixin(fieldMixin(vectors.Uint16Vector));
+export class Uint16Vector extends NullableFieldUint16Vector {}
+export const NullableFieldUint32Vector = nullableMixin(fieldMixin(vectors.Uint32Vector));
+export class Uint32Vector extends NullableFieldUint32Vector {}
+export const NullableFieldUint64Vector = nullableMixin(fieldMixin(vectors.Uint64Vector));
+export class Uint64Vector extends NullableFieldUint64Vector {}
+export const NullableFieldDate32Vector = nullableMixin(fieldMixin(vectors.Date32Vector));
+export class Date32Vector extends NullableFieldDate32Vector {}
+export const NullableFieldDate64Vector = nullableMixin(fieldMixin(vectors.Date64Vector));
+export class Date64Vector extends NullableFieldDate64Vector {}
+export const NullableFieldTime32Vector = nullableMixin(fieldMixin(vectors.Time32Vector));
+export class Time32Vector extends NullableFieldTime32Vector {}
+export const NullableFieldTime64Vector = nullableMixin(fieldMixin(vectors.Time64Vector));
+export class Time64Vector extends NullableFieldTime64Vector {}
+export const NullableFieldFloat16Vector = nullableMixin(fieldMixin(vectors.Float16Vector));
+export class Float16Vector extends NullableFieldFloat16Vector {}
+export const NullableFieldFloat32Vector = nullableMixin(fieldMixin(vectors.Float32Vector));
+export class Float32Vector extends NullableFieldFloat32Vector {}
+export const NullableFieldFloat64Vector = nullableMixin(fieldMixin(vectors.Float64Vector));
+export class Float64Vector extends NullableFieldFloat64Vector {}
+export const NullableFieldStructVector = nullableMixin(fieldMixin(vectors.StructVector));
+export class StructVector extends NullableFieldStructVector {}
+export const NullableFieldDecimalVector = nullableMixin(fieldMixin(vectors.DecimalVector));
+export class DecimalVector extends NullableFieldDecimalVector {}
+export const NullableFieldTimestampVector = nullableMixin(fieldMixin(vectors.TimestampVector));
+export class TimestampVector extends NullableFieldTimestampVector {}
+export const NullableFieldDictionaryVector = nullableMixin(fieldMixin(vectors.DictionaryVector));
+export class DictionaryVector extends NullableFieldDictionaryVector {}
+export const NullableFieldFixedSizeListVector = nullableMixin(fieldMixin(vectors.FixedSizeListVector));
+export class FixedSizeListVector extends NullableFieldFixedSizeListVector {}
\ No newline at end of file
diff --git a/js/src/vector/traits/vectors.ts b/js/src/vector/traits/vectors.ts
new file mode 100644
index 0000000000000..f9e05fd4eff1b
--- /dev/null
+++ b/js/src/vector/traits/vectors.ts
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from '../vector';
+import { Utf8Vector } from '../utf8';
+import { StructVector } from '../struct';
+import { DictionaryVector } from '../dictionary';
+import {
+    ListVector,
+    BinaryVector,
+    FixedSizeListVector
+} from '../list';
+
+import {
+    BoolVector,
+    Int8Vector,
+    Int16Vector,
+    Int32Vector,
+    Int64Vector,
+    Uint8Vector,
+    Uint16Vector,
+    Uint32Vector,
+    Uint64Vector,
+    Float16Vector,
+    Float32Vector,
+    Float64Vector,
+    Date32Vector,
+    Date64Vector,
+    Time32Vector,
+    Time64Vector,
+    DecimalVector,
+    TimestampVector,
+} from '../numeric';
+
+export {
+    Vector,
+    BoolVector,
+    ListVector,
+    Utf8Vector,
+    Int8Vector,
+    Int16Vector,
+    Int32Vector,
+    Int64Vector,
+    Uint8Vector,
+    Uint16Vector,
+    Uint32Vector,
+    Uint64Vector,
+    Date32Vector,
+    Date64Vector,
+    Time32Vector,
+    Time64Vector,
+    BinaryVector,
+    StructVector,
+    Float16Vector,
+    Float32Vector,
+    Float64Vector,
+    DecimalVector,
+    TimestampVector,
+    DictionaryVector,
+    FixedSizeListVector,
+};
diff --git a/js/src/vector/vector.ts b/js/src/vector/vector.ts
index 8047c89923223..9565478ad969b 100644
--- a/js/src/vector/vector.ts
+++ b/js/src/vector/vector.ts
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import * as Schema_ from '../format/Schema';
+import * as Schema_ from '../format/fb/Schema';
 import Type = Schema_.org.apache.arrow.flatbuf.Type;
 
 export interface Vector<T = any> extends Iterable<T | null> {
@@ -29,6 +29,7 @@ export interface Vector<T = any> extends Iterable<T | null> {
     concat(...vectors: Vector<T>[]): Vector<T>;
     slice<R = T[]>(start?: number, end?: number): R;
 }
+
 export class Vector<T = any> implements Vector<T> {
     slice<R = T[]>(start?: number, end?: number): R {
         let { length } = this, from = start! | 0;
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
index 66a5c21ac2dff..87641e52bf3f8 100644
--- a/js/test/Arrow.ts
+++ b/js/test/Arrow.ts
@@ -39,6 +39,7 @@ else if (!~targets.indexOf(target)) throwInvalidImportError('target', target, ta
 else if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats);
 else modulePath = path.join(target, format);
 
+export { Int64, Uint64, Int128 } from '../src/Arrow';
 export { List } from '../src/Arrow';
 export { TypedArray } from '../src/Arrow';
 export { TypedArrayConstructor } from '../src/Arrow';
@@ -46,4 +47,4 @@ export { NumericVectorConstructor } from '../src/Arrow';
 
 import * as Arrow_ from '../src/Arrow';
 export let Arrow: typeof Arrow_ = require(path.resolve(`./targets`, modulePath, `Arrow`));
-export default Arrow;
\ No newline at end of file
+export default Arrow;
diff --git a/js/test/test-config.ts b/js/test/integration/test-config.ts
similarity index 100%
rename from js/test/test-config.ts
rename to js/test/integration/test-config.ts
diff --git a/js/test/integration-tests.ts b/js/test/integration/validate-tests.ts
similarity index 58%
rename from js/test/integration-tests.ts
rename to js/test/integration/validate-tests.ts
index 4147e862b19a4..c8778ba2b33c2 100644
--- a/js/test/integration-tests.ts
+++ b/js/test/integration/validate-tests.ts
@@ -15,11 +15,31 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import Arrow from './Arrow';
+import * as fs from 'fs';
+import * as path from 'path';
+
+if (!process.env.JSON_PATH || !process.env.ARROW_PATH) {
+    throw new Error('Integration tests need paths to both json and arrow files');
+}
+
+const jsonPath = path.resolve(process.env.JSON_PATH + '');
+const arrowPath = path.resolve(process.env.ARROW_PATH + '');
+
+if (!fs.existsSync(jsonPath) || !fs.existsSync(arrowPath)) {
+    throw new Error('Integration tests need both json and arrow files to exist');
+}
+
+/* tslint:disable */
+const { parse } = require('json-bignum');
+
+const jsonData = parse(fs.readFileSync(jsonPath, 'utf8'));
+const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)];
+
+import Arrow from '../Arrow';
 import { zip } from 'ix/iterable/zip';
-import { config, formats } from './test-config';
+import { toArray } from 'ix/iterable/toArray';
 
-const { Table, readVectors } = Arrow;
+const { Table, read } = Arrow;
 
 expect.extend({
     toEqualVector(v1: any, v2: any) {
@@ -65,7 +85,7 @@ expect.extend({
         return {
             pass: allFailures.every(({ failures }) => failures.length === 0),
             message: () => [
-                `${v1.name}: (${format('cpp', 'java', ' !== ')})\n`,
+                `${v1.name}: (${format('json', 'arrow', ' !== ')})\n`,
                 ...allFailures.map(({ failures, title }) =>
                     !failures.length ? `` : [`${title}:`, ...failures].join(`\n`))
             ].join('\n')
@@ -74,41 +94,35 @@ expect.extend({
 });
 
 describe(`Integration`, () => {
-    for (const format of formats) {
-        describe(format, () => {
-            for (const [cppArrow, javaArrow] of zip(config.cpp[format], config.java[format])) {
-                describe(`${cppArrow.name}`, () => {
-                    testReaderIntegration(cppArrow.buffers, javaArrow.buffers);
-                    testTableFromBuffersIntegration(cppArrow.buffers, javaArrow.buffers);
-                });
-            }
-        });
-    }
+    testReaderIntegration();
+    testTableFromBuffersIntegration();
 });
 
-function testReaderIntegration(cppBuffers: Uint8Array[], javaBuffers: Uint8Array[]) {
-    test(`cpp and java vectors report the same values`, () => {
+function testReaderIntegration() {
+    test(`json and arrow buffers report the same values`, () => {
         expect.hasAssertions();
-        for (const [cppVectors, javaVectors] of zip(readVectors(cppBuffers), readVectors(javaBuffers))) {
-            expect(cppVectors.length).toEqual(javaVectors.length);
-            for (let i = -1, n = cppVectors.length; ++i < n;) {
-                (expect(cppVectors[i]) as any).toEqualVector(javaVectors[i]);
+        const jsonVectors = toArray(read(jsonData));
+        const binaryVectors = toArray(read(arrowBuffers));
+        for (const [jVectors, bVectors] of zip(jsonVectors, binaryVectors)) {
+            expect(jVectors.length).toEqual(bVectors.length);
+            for (let i = -1, n = jVectors.length; ++i < n;) {
+                (expect(jVectors[i]) as any).toEqualVector(bVectors[i]);
             }
         }
     });
 }
 
-function testTableFromBuffersIntegration(cppBuffers: Uint8Array[], javaBuffers: Uint8Array[]) {
-    test(`cpp and java tables report the same values`, () => {
+function testTableFromBuffersIntegration() {
+    test(`json and arrow buffers report the same values`, () => {
         expect.hasAssertions();
-        const cppTable = Table.from(cppBuffers);
-        const javaTable = Table.from(javaBuffers);
-        const cppVectors = cppTable.columns;
-        const javaVectors = javaTable.columns;
-        expect(cppTable.length).toEqual(javaTable.length);
-        expect(cppVectors.length).toEqual(javaVectors.length);
-        for (let i = -1, n = cppVectors.length; ++i < n;) {
-            (expect(cppVectors[i]) as any).toEqualVector(javaVectors[i]);
+        const jsonTable = Table.from(jsonData);
+        const binaryTable = Table.from(arrowBuffers);
+        const jsonVectors = jsonTable.columns;
+        const binaryVectors = binaryTable.columns;
+        expect(jsonTable.length).toEqual(binaryTable.length);
+        expect(jsonVectors.length).toEqual(binaryVectors.length);
+        for (let i = -1, n = jsonVectors.length; ++i < n;) {
+            (expect(jsonVectors[i]) as any).toEqualVector(binaryVectors[i]);
         }
     });
 }
diff --git a/js/test/reader-tests.ts b/js/test/reader-tests.ts
deleted file mode 100644
index 309cec9e23b14..0000000000000
--- a/js/test/reader-tests.ts
+++ /dev/null
@@ -1,69 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Arrow from './Arrow';
-const { readVectors } = Arrow;
-import { config, sources, formats } from './test-config';
-
-describe(`readBuffers`, () => {
-    for (const source of sources) {
-        describe(source, () => {
-            for (const format of formats) {
-                describe(format, () => {
-                    for (const { name, buffers } of config[source][format]) {
-                        describe(name, () => {
-                            testReaderIterator(buffers);
-                            testVectorIterator(buffers);
-                        });
-                    }
-                });
-            }
-        });
-    }
-});
-
-function testReaderIterator(buffers: Uint8Array[]) {
-    test(`reads each batch as an Array of Vectors`, () => {
-        expect.hasAssertions();
-        for (const vectors of readVectors(buffers)) {
-            for (const vector of vectors) {
-                expect(vector.name).toMatchSnapshot();
-                expect(vector.type).toMatchSnapshot();
-                expect(vector.length).toMatchSnapshot();
-                for (let i = -1, n = vector.length; ++i < n;) {
-                    expect(vector.get(i)).toMatchSnapshot();
-                }
-            }
-        }
-    });
-}
-
-function testVectorIterator(buffers: Uint8Array[]) {
-    test(`vector iterators report the same values as get`, () => {
-        expect.hasAssertions();
-        for (const vectors of readVectors(buffers)) {
-            for (const vector of vectors) {
-                let i = -1, n = vector.length;
-                for (let v of vector) {
-                    expect(++i).toBeLessThan(n);
-                    expect(v).toEqual(vector.get(i));
-                }
-                expect(++i).toEqual(n);
-            }
-        }
-    });
-}
diff --git a/js/test/table-tests.ts b/js/test/table-tests.ts
deleted file mode 100644
index 5ec04a72ff33b..0000000000000
--- a/js/test/table-tests.ts
+++ /dev/null
@@ -1,136 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Arrow from './Arrow';
-const { Table, readVectors } = Arrow;
-import { config, sources, formats } from './test-config';
-
-describe(`Table`, () => {
-    for (const source of sources) {
-        describe(source, () => {
-            for (const format of formats) {
-                describe(format, () => {
-                    for (const { name, buffers } of config[source][format]) {
-                        describe(name, () => {
-                            testTableFromBuffers(buffers);
-                            testColumnIterators(buffers);
-                            testReaderVectorsAndTableColumns(buffers);
-                            testTableRowIterator(buffers);
-                            testTableRowIteratorCompact(buffers);
-                            testEmptyTableToString();
-                            testTableToStringPretty(buffers);
-                            testTableToStringPrettyWithIndex(buffers);
-                        });
-                    }
-                });
-            }
-        });
-    }
-});
-
-function testTableFromBuffers(buffers: Uint8Array[]) {
-    test(`creates a Table from Arrow buffers`, () => {
-        expect.hasAssertions();
-        const table = Table.from(buffers);
-        for (const vector of table.columns) {
-            expect(vector.name).toMatchSnapshot();
-            expect(vector.type).toMatchSnapshot();
-            expect(vector.length).toMatchSnapshot();
-            for (let i = -1, n = vector.length; ++i < n;) {
-                expect(vector.get(i)).toMatchSnapshot();
-            }
-        }
-    });
-}
-
-function testColumnIterators(buffers: Uint8Array[]) {
-    test(`vector iterators report the same values as get`, () => {
-        expect.hasAssertions();
-        const table = Table.from(buffers);
-        for (const vector of table.columns) {
-            let i = -1, n = vector.length;
-            for (let v of vector) {
-                expect(++i).toBeLessThan(n);
-                expect(v).toEqual(vector.get(i));
-            }
-            expect(++i).toEqual(n);
-        }
-    });
-}
-
-function testReaderVectorsAndTableColumns(buffers: Uint8Array[]) {
-    test(`batch and Table Vectors report the same values`, () => {
-        expect.hasAssertions();
-        let rowsTotal = 0;
-        let table = Table.from(buffers);
-        for (let vectors of readVectors(buffers)) {
-            let rowsNow = Math.max(...vectors.map((v) => v.length));
-            for (let vi = -1, vn = vectors.length; ++vi < vn;) {
-                let v1 = vectors[vi];
-                let v2 = table.columns[vi];
-                expect(v1.name).toEqual(v2.name);
-                expect(v1.type).toEqual(v2.type);
-                for (let i = -1, n = v1.length; ++i < n;) {
-                    expect(v1.get(i)).toEqual(v2.get(i + rowsTotal));
-                }
-            }
-            rowsTotal += rowsNow;
-        }
-    });
-}
-
-function testTableRowIterator(buffers: Uint8Array[]) {
-    test(`enumerates Table rows`, () => {
-        expect.hasAssertions();
-        const table = Table.from(buffers);
-        expect(table.length).toMatchSnapshot();
-        expect(table.columns.length).toMatchSnapshot();
-        for (const row of table) {
-            expect(row!.toObject()).toMatchSnapshot();
-        }
-    });
-}
-
-function testTableRowIteratorCompact(buffers: Uint8Array[]) {
-    test(`enumerates Table rows compact`, () => {
-        expect.hasAssertions();
-        const table = Table.from(buffers);
-        expect(table.length).toMatchSnapshot();
-        expect(table.columns.length).toMatchSnapshot();
-        for (const row of table) {
-            expect(row!.toArray()).toMatchSnapshot();
-        }
-    });
-}
-
-function testEmptyTableToString() {
-    test(`toString() prints an empty Table`, () => {
-        expect(Table.from().toString()).toMatchSnapshot();
-    });
-}
-
-function testTableToStringPretty(buffers: Uint8Array[]) {
-    test(`toString() prints a pretty Table`, () => {
-        expect(Table.from(buffers).toString()).toMatchSnapshot();
-    });
-}
-
-function testTableToStringPrettyWithIndex(buffers: Uint8Array[]) {
-    test(`toString({ index: true }) prints a pretty Table with an Index column`, () => {
-        expect(Table.from(buffers).toString({ index: true })).toMatchSnapshot();
-    });
-}
\ No newline at end of file
diff --git a/js/test/unit/int-tests.ts b/js/test/unit/int-tests.ts
new file mode 100644
index 0000000000000..74c96e8efbb6d
--- /dev/null
+++ b/js/test/unit/int-tests.ts
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import {
+    Int64,
+    Uint64,
+    Int128
+} from '../Arrow';
+
+describe(`Uint64`, () => {
+    test(`gets expected high/low bytes`, () => {
+        let i = new Uint64(new Uint32Array([5, 0]));
+        expect(i.high()).toEqual(0);
+        expect(i.low()).toEqual(5);
+    });
+    test(`adds 32-bit numbers`, () => {
+        let a = new Uint64(new Uint32Array([5, 0]));
+        let b = new Uint64(new Uint32Array([9, 0]));
+        let expected = new Uint64(new Uint32Array([14, 0]));
+        expect(a.plus(b)).toEqual(expected);
+    });
+    test(`addition overflows 32-bit numbers`, () => {
+        let a = new Uint64(new Uint32Array([0xffffffff, 0]));
+        let b = new Uint64(new Uint32Array([9, 0]));
+        let expected = new Uint64(new Uint32Array([8, 1]));
+        expect(a.plus(b)).toEqual(expected);
+    });
+    test(`multiplies 32-bit numbers`, () => {
+        let a = new Uint64(new Uint32Array([5, 0]));
+        let b = new Uint64(new Uint32Array([9, 0]));
+        let expected = new Uint64(new Uint32Array([45, 0]));
+        expect(a.times(b)).toEqual(expected);
+    });
+    test(`multiplication overflows 32-bit numbers`, () => {
+        let a = new Uint64(new Uint32Array([0x80000000, 0]));
+        let b = new Uint64(new Uint32Array([3, 0]));
+        let expected = new Uint64(new Uint32Array([0x80000000, 1]));
+        expect(a.times(b)).toEqual(expected);
+    });
+    test(`multiplication is associative`, () => {
+        let a = new Uint64(new Uint32Array([0x80000000, 0]));
+        let b = new Uint64(new Uint32Array([3, 0]));
+        expect(Uint64.multiply(a, b)).toEqual(Uint64.multiply(b,a));
+    });
+    test(`lessThan works on 32-bit numbers`, () => {
+        let a = new Uint64(new Uint32Array([0x0000abcd, 0]));
+        let b = new Uint64(new Uint32Array([0x0000abcf, 0]));
+        expect(a.lessThan(b)).toBeTruthy();
+    });
+    test(`lessThan works on 64-bit numbers`, () => {
+        let a = new Uint64(new Uint32Array([123, 32]));
+        let b = new Uint64(new Uint32Array([568, 32]));
+        expect(a.lessThan(b)).toBeTruthy();
+    });
+});
+
+describe(`Int64`, () => {
+    test(`gets expected high/low bytes`, () => {
+        let i = new Int64(new Uint32Array([5, 0]));
+        expect(i.high()).toEqual(0);
+        expect(i.low()).toEqual(5);
+    });
+    test(`adds 32-bit numbers`, () => {
+        let a = new Int64(new Uint32Array([5, 0]));
+        let b = new Int64(new Uint32Array([9, 0]));
+        let expected = new Int64(new Uint32Array([14, 0]));
+        expect(a.plus(b)).toEqual(expected);
+    });
+    test(`adds negative 32-bit numbers`, () => {
+        let a = new Int64(new Uint32Array([56789 ,  0]));
+        let b = new Int64(new Uint32Array([-66789, -1]));
+        let expected = new Int64(new Uint32Array([-10000, -1]));
+        expect(a.plus(b)).toEqual(expected);
+    });
+    test(`addition overflows 32-bit numbers`, () => {
+        let a = new Int64(new Uint32Array([0xffffffff, 0]));
+        let b = new Int64(new Uint32Array([9, 0]));
+        let expected = new Int64(new Uint32Array([8, 1]));
+        expect(a.plus(b)).toEqual(expected);
+    });
+    test(`multiplies 32-bit numbers`, () => {
+        let a = new Int64(new Uint32Array([5, 0]));
+        let b = new Int64(new Uint32Array([9, 0]));
+        let expected = new Int64(new Uint32Array([45, 0]));
+        expect(a.times(b)).toEqual(expected);
+    });
+    test(`multiplication overflows 32-bit numbers`, () => {
+        let a = new Int64(new Uint32Array([0x80000000, 0]));
+        let b = new Int64(new Uint32Array([3, 0]));
+        let expected = new Int64(new Uint32Array([0x80000000, 1]));
+        expect(a.times(b)).toEqual(expected);
+    });
+    test(`multiplication works on negative numbers`, () => {
+        let a = new Int64(new Uint32Array([-5, -1]));
+        let b = new Int64(new Uint32Array([-100, -1]));
+        expect(a.times(b)).toEqual(new Int64(new Uint32Array([    500,  0])));
+        expect(a.times(b)).toEqual(new Int64(new Uint32Array([ -50000, -1])));
+        expect(a.times(b)).toEqual(new Int64(new Uint32Array([5000000,  0])));
+    });
+    test(`multiplication is associative`, () => {
+        let a = new Int64(new Uint32Array([0x80000000, 0]));
+        let b = new Int64(new Uint32Array([3, 0]));
+        expect(Int64.multiply(a, b)).toEqual(Int64.multiply(b,a));
+    });
+    test(`lessThan works on 32-bit numbers`, () => {
+        let a = new Int64(new Uint32Array([0x0000abcd, 0]));
+        let b = new Int64(new Uint32Array([0x0000abcf, 0]));
+        expect(a.lessThan(b)).toBeTruthy();
+    });
+    test(`lessThan works on 64-bit numbers`, () => {
+        let a = new Int64(new Uint32Array([123, 32]));
+        let b = new Int64(new Uint32Array([568, 32]));
+        expect(a.lessThan(b)).toBeTruthy();
+    });
+    test(`lessThan works on negative numbers`, () => {
+        let a = new Int64(new Uint32Array([0,   -158]));
+        let b = new Int64(new Uint32Array([-3,    -1]));
+        expect(a.lessThan(b)).toBeTruthy();
+    });
+    test(`lessThan works on mixed numbers`, () => {
+        let a = new Int64(new Uint32Array([-3, -1]));
+        let b = new Int64(new Uint32Array([ 0,  3]));
+        expect(a.lessThan(b)).toBeTruthy();
+    });
+    test(`negate works on 32-bit number`, () => {
+        expect (new Int64(new Uint32Array([123456, 0])).negate()).toEqual(new Int64(new Uint32Array([-123456, -1])));
+    });
+    test(`double negation is noop`, () => {
+        let test     = new Int64(new Uint32Array([6789, 12345]));
+        let expected = new Int64(new Uint32Array([6789, 12345]));
+        expect(test.negate().negate()).toEqual(expected);
+    });
+    test(`negate works on 64-bit number`, () => {
+        expect (new Int64(new Uint32Array([0xb74abf15, 0x62c])).negate()).toEqual(new Int64(new Uint32Array([0x48b540eb, 0xfffff9d3])));
+    });
+    test(`fromString parses string`, () => {
+        expect(Int64.fromString('6789123456789')).toEqual(new Int64(new Uint32Array([0xb74abf15, 0x62c])));
+    });
+    test(`fromString parses negative string`, () => {
+        expect(Int64.fromString('-6789123456789')).toEqual(new Int64(new Uint32Array([0x48b540eb, 0xfffff9d3])));
+    });
+});
+
+describe(`Int128`, () => {
+    test(`gets expected bytes`, () => {
+        let i = new Int128(new Uint32Array([4, 3, 2, 1]));
+        expect(i.high().high()).toEqual(1);
+        expect(i.high().low() ).toEqual(2);
+        expect(i.low().high() ).toEqual(3);
+        expect(i.low().low()  ).toEqual(4);
+    });
+    test(`adds 32-bit numbers`, () => {
+        let a = new Int128(new Uint32Array([5, 0, 0, 0]));
+        let b = new Int128(new Uint32Array([9, 0, 0, 0]));
+        let expected = new Int128(new Uint32Array([14, 0, 0, 0]));
+        expect(a.plus(b)).toEqual(expected);
+    });
+    test(`adds negative 32-bit numbers`, () => {
+        let a = new Int128(new Uint32Array([56789 ,  0, 0, 0]));
+        let b = new Int128(new Uint32Array([-66789, -1, -1, -1]));
+        let expected = new Int128(new Uint32Array([-10000, -1, -1, -1]));
+        expect(a.plus(b)).toEqual(expected);
+    });
+    test(`addition overflows 32-bit numbers`, () => {
+        let a = new Int128(new Uint32Array([0xffffffff, 0, 0, 0]));
+        let b = new Int128(new Uint32Array([9, 0, 0, 0]));
+        let expected = new Int128(new Uint32Array([8, 1, 0, 0]));
+        expect(a.plus(b)).toEqual(expected);
+    });
+    test(`multiplies 32-bit numbers`, () => {
+        let a = new Int128(new Uint32Array([5, 0, 0, 0]));
+        let b = new Int128(new Uint32Array([9, 0, 0, 0]));
+        let expected = new Int128(new Uint32Array([45, 0, 0, 0]));
+        expect(a.times(b)).toEqual(expected);
+    });
+    test(`multiplication overflows 32-bit numbers`, () => {
+        let a = new Int128(new Uint32Array([0x80000000, 0, 0, 0]));
+        let b = new Int128(new Uint32Array([3, 0, 0, 0]));
+        let expected = new Int128(new Uint32Array([0x80000000, 1, 0, 0]));
+        expect(a.times(b)).toEqual(expected);
+    });
+    test(`multiplication works on negative numbers`, () => {
+        let a = new Int128(new Uint32Array([-5, -1, -1, -1]));
+        let b = new Int128(new Uint32Array([-100, -1, -1, -1]));
+        expect(a.times(b)).toEqual(new Int128(new Uint32Array([    500,   0,  0,  0])));
+        expect(a.times(b)).toEqual(new Int128(new Uint32Array([ -50000,  -1, -1, -1])));
+        expect(a.times(b)).toEqual(new Int128(new Uint32Array([5000000,   0,  0,  0])));
+    });
+    test(`multiplication is associative`, () => {
+        let a = new Int128(new Uint32Array([4, 3, 2, 1]));
+        let b = new Int128(new Uint32Array([3, 0, 0, 0]));
+        expect(Int128.multiply(a, b)).toEqual(Int128.multiply(b,a));
+    });
+    test(`multiplication can produce 128-bit number`, () => {
+        let a = new Int128(new Uint32Array([0, 0xf0000000, 0, 0]));
+        let b = new Int128(new Uint32Array([0, 0x10000000, 0, 0]));
+        expect(a.times(b)).toEqual(new Int128(new Uint32Array([0x00000000, 0x00000000, 0x00000000, 0xf000000])));
+    });
+    test(`fromString parses string`, () => {
+        expect(Int128.fromString('1002111867823618826746863804903129070'))
+            .toEqual(new Int64(new Uint32Array([0x00c0ffee,
+                                               0x00c0ffee,
+                                               0x00c0ffee,
+                                               0x00c0ffee])));
+    });
+    test(`fromString parses negative string`, () => {
+        expect(Int128.fromString('-12345678901234567890123456789012345678'))
+            .toEqual(new Int64(new Uint32Array([0x21c70cb2,
+                                                0x3bb66faf,
+                                                0x0ffdccec,
+                                                0xf6b64f09])));
+    });
+});
diff --git a/js/test/vector-tests.ts b/js/test/unit/vector-tests.ts
similarity index 99%
rename from js/test/vector-tests.ts
rename to js/test/unit/vector-tests.ts
index 0aaba19ee9144..75706229ab172 100644
--- a/js/test/vector-tests.ts
+++ b/js/test/unit/vector-tests.ts
@@ -15,12 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import Arrow from './Arrow';
-import {
+import Arrow, {
     TypedArray,
     TypedArrayConstructor,
     NumericVectorConstructor,
-} from './Arrow';
+} from '../Arrow';
 
 const {
     BoolVector,
diff --git a/js/tsconfig/tsconfig.es5.cls.json b/js/tsconfig/tsconfig.es5.cls.json
index 502432da04348..4df18aa595d92 100644
--- a/js/tsconfig/tsconfig.es5.cls.json
+++ b/js/tsconfig/tsconfig.es5.cls.json
@@ -2,8 +2,10 @@
 {
   "extends": "./tsconfig.base.json",
   "compilerOptions": {
-    "target": "ES5",
+    "target": "es2015",
     "module": "es2015",
-    "declaration": false
+    "declaration": false,
+    "noEmitHelpers": true,
+    "importHelpers": true
   }
 }

From d023b40dc81ef43b9b033e88add1859c437bb700 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 18 Dec 2017 23:36:19 -0500
Subject: [PATCH 1347/1644] ARROW-1922: Blog post on JAVA vector changes

cc @wesm , @jacques-n , @BryanCutler , @icexelloss

A small post on recent improvements in JAVA vectors. Suggestions are welcome :)

Author: siddharth <siddharth@dremio.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1419 from siddharthteotia/ARROW-1922 and squashes the following commits:

ebdd986a [Wes McKinney] Minor tweaks to post, add Dremio link
eaedd879 [siddharth] review comments
57050195 [siddharth] correct typo
c2af13c2 [siddharth] ARROW-1922: Blog post on JAVA vector changes
---
 site/_data/contributors.yml                   |   3 +
 .../2017-12-19-java-vector-improvements.md    | 104 ++++++++++++++++++
 2 files changed, 107 insertions(+)
 create mode 100644 site/_posts/2017-12-19-java-vector-improvements.md

diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml
index 2a8d95b77b239..19ca53ad48599 100644
--- a/site/_data/contributors.yml
+++ b/site/_data/contributors.yml
@@ -32,4 +32,7 @@
 - name: Michael König
   apacheId: MathMagique
   githubId: MathMagique
+- name: Siddharth Teotia
+  apacheId: siddharthteotia
+  githubId: siddharthteotia
 # End contributors.yml
diff --git a/site/_posts/2017-12-19-java-vector-improvements.md b/site/_posts/2017-12-19-java-vector-improvements.md
new file mode 100644
index 0000000000000..2854861fe090b
--- /dev/null
+++ b/site/_posts/2017-12-19-java-vector-improvements.md
@@ -0,0 +1,104 @@
+---
+layout: post
+title: "Improvements to Java Vector API in Apache Arrow 0.8.0"
+excerpt: "This post describes the recent improvements in Java Vector code"
+date: 2017-12-19 00:00:00
+author: Siddharth Teotia
+categories: [application]
+---
+
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+This post gives insight into the major improvements in the Java implementation
+of vectors. We undertook this work over the last 10 weeks since the last Arrow
+release.
+
+## Design Goals
+
+1. Improved maintainability and extensibility
+2. Improved heap memory usage
+3. No performance overhead on hot code paths
+
+## Background
+
+### Improved maintainability and extensibility
+
+We use templates in several places for compile time Java code generation for
+different vector classes, readers, writers etc. Templates are helpful as the
+developers don't have to write a lot of duplicate code.
+
+However, we realized that over a period of time some specific Java
+templates became extremely complex with giant if-else blocks, poor code indentation
+and documentation. All this impacted the ability to easily extend these templates
+for adding new functionality or improving the existing infrastructure.
+
+So we evaluated the usage of templates for compile time code generation and
+decided not to use complex templates in some places by writing small amount of
+duplicate code which is elegant, well documented and extensible.
+
+### Improved heap usage
+
+We did extensive memory analysis downstream in [Dremio][1] where Arrow is used
+heavily for in-memory query execution on columnar data. The general conclusion
+was that Arrow's Java vector classes have non-negligible heap overhead and
+volume of objects was too high. There were places in code where we were
+creating objects unnecessarily and using structures that could be substituted
+with better alternatives.
+
+### No performance overhead on hot code paths
+
+Java vectors used delegation and abstraction heavily throughout the object
+hierarchy. The performance critical get/set methods of vectors went through a
+chain of function calls back and forth between different objects before doing
+meaningful work. We also evaluated the usage of branches in vector APIs and
+reimplemented some of them by avoiding branches completely.
+
+We took inspiration from how the Java memory code in `ArrowBuf` works. For all
+the performance critical methods, `ArrowBuf` bypasses all the netty object
+hierarchy, grabs the target virtual address and directly interacts with the
+memory.
+
+There were cases where branches could be avoided all together.
+
+In case of nullable vectors, we were doing multiple checks to confirm if
+the value at a given position in the vector is null or not.
+
+## Our implementation approach
+
+- For scalars, the inheritance tree was simplified by writing different
+abstract base classes for fixed and variable width scalars.
+- The base classes contained all the common functionality across different
+types.
+- The individual subclasses implemented type specific APIs for fixed and
+variable width scalar vectors.
+- For the performance critical methods, all the work is done either in
+the vector class or corresponding ArrowBuf. There is no delegation to any
+internal object.
+- The mutator and accessor based access to vector APIs is removed. These
+objects led to unnecessary heap overhead and complicated the use of APIs.
+- Both scalar and complex vectors directly interact with underlying buffers
+that manage the offsets, data and validity. Earlier we were creating different
+inner vectors for each vector and delegating all the functionality to inner
+vectors. This introduced a lot of bugs in memory management, excessive heap
+overhead and performance penalty due to chain of delegations.
+- We reduced the number of vector classes by removing non-nullable vectors.
+In the new implementation, all vectors in Java are nullable in nature.
+
+[1]: https://www.dremio.com/
\ No newline at end of file

From bcfd83d5046d76dbe3714fd3c6e8e1e2d794490e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 19 Dec 2017 09:39:06 -0500
Subject: [PATCH 1348/1644] ARROW-1935: [Website] Remove link to nightly
 builds. Fix signature / checksum links, add verification instructions

Website fixes per ASF policies and feedback in ARROW-1935, ARROW-1936

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1434 from wesm/ARROW-1935 and squashes the following commits:

41f128ec [Wes McKinney] Remove link to nightly builds. Fix signature / checksum links, add KEYS file, verification instructions
---
 site/install.md | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/site/install.md b/site/install.md
index 6e67d790a9d31..ec30e0469cdc1 100644
--- a/site/install.md
+++ b/site/install.md
@@ -29,8 +29,9 @@ See the [release notes][10] for more about what's new.
 ### Source release
 
 * **Source Release**: [apache-arrow-0.8.0.tar.gz][6]
-* **Verification**: [sha512][3], [asc][7]
+* **Verification**: [sha512][3], [asc][7] ([verification instructions][12])
 * [Git tag 1d689e5][2]
+* [PGP keys for release signatures][11]
 
 ### Java Packages
 
@@ -144,26 +145,15 @@ These repositories are managed at
 [red-data-tools/arrow-packages][9]. If you have any feedback, please
 send it to the project instead of Apache Arrow project.
 
-### Nightly Development Builds
-
-To assist with development and debugging, some nightly builds are
-available. These builds are not releases and not necessarily produced on ASF
-infrastructure. They are to be used strictly for development.
-
-* **conda packages** for C++ and Python (Linux only)
-
-```
-conda install arrow-cpp -c twosigma
-conda install pyarrow -c twosigma
-```
-
 [1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/
 [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0
-[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512
+[3]: https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512
 [4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.8.0%22
 [5]: http://conda-forge.github.io
 [6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz
-[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc
+[7]: https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
 [10]: http://arrow.apache.org/release/0.8.0.html
+[11]: http://www.apache.org/dist/arrow/KEYS
+[12]: https://www.apache.org/dyn/closer.cgi#verify
\ No newline at end of file

From ac789bafee598326ca9aeaaaa84468017b9f2598 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 19 Dec 2017 09:40:11 -0500
Subject: [PATCH 1349/1644] ARROW-232: [Python] Add unit test for writing
 Parquet file from chunked table

This requires PARQUET-1092 https://github.com/apache/parquet-cpp/pull/426

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1425 from wesm/ARROW-232 and squashes the following commits:

da8d9998 [Wes McKinney] Add unit test to validate PARQUET-1092
---
 python/pyarrow/tests/test_parquet.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index fc8c8f0c98da9..c2bb31c9bcf51 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -118,6 +118,24 @@ def test_pandas_parquet_2_0_rountrip(tmpdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@parquet
+def test_chunked_table_write(tmpdir):
+    # ARROW-232
+    df = alltypes_sample(size=10)
+
+    # The nanosecond->ms conversion is a nuisance, so we just avoid it here
+    del df['datetime']
+
+    batch = pa.RecordBatch.from_pandas(df)
+    table = pa.Table.from_batches([batch] * 3)
+    _check_roundtrip(table, version='2.0')
+
+    df, _ = dataframe_with_lists()
+    batch = pa.RecordBatch.from_pandas(df)
+    table = pa.Table.from_batches([batch] * 3)
+    _check_roundtrip(table, version='2.0')
+
+
 @parquet
 def test_pandas_parquet_datetime_tz():
     import pyarrow.parquet as pq

From 9895181610c0a5ef1ba836300ea2036e1a3e5621 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 19 Dec 2017 10:29:30 -0500
Subject: [PATCH 1350/1644] ARROW-1934: [Website] 0.8.0 release highlights blog
 post

I took a hack at this after poring through the changelog, others please let me know if you'd like to add or change anything. I need to incorporate Sidd's blog post and add a link to that here. I can publish all of this sometime tomorrow morning New York time and post to social media etc.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1432 from wesm/ARROW-1934 and squashes the following commits:

4aa6bc6a [Wes McKinney] Tweaks for publication
da9e65d4 [Wes McKinney] Start drafting 0.8.0 release blog post
---
 site/_posts/2017-12-19-0.8.0-release.md | 192 ++++++++++++++++++++++++
 1 file changed, 192 insertions(+)
 create mode 100644 site/_posts/2017-12-19-0.8.0-release.md

diff --git a/site/_posts/2017-12-19-0.8.0-release.md b/site/_posts/2017-12-19-0.8.0-release.md
new file mode 100644
index 0000000000000..c65d30832b7b8
--- /dev/null
+++ b/site/_posts/2017-12-19-0.8.0-release.md
@@ -0,0 +1,192 @@
+---
+layout: post
+title: "Apache Arrow 0.8.0 Release"
+date: "2017-12-19 00:01:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.8.0 release. It is the
+product of 10 weeks of development andincludes [**286 resolved JIRAs**][1] with
+many new features and bug fixes to the various language implementations. This
+is the largest release since 0.3.0 earlier this year.
+
+As part of work towards a stabilizing the Arrow format and making a 1.0.0
+release sometime in 2018, we made a series of backwards-incompatible changes to
+the serialized Arrow metadata that requires Arrow readers and writers (0.7.1
+and earlier) to upgrade in order to be compatible with 0.8.0 and higher. We
+expect future backwards-incompatible changes to be rare going forward.
+
+See the [Install Page][2] to learn how to get the libraries for your
+platform. The [complete changelog][3] is also available.
+
+We discuss some highlights from the release and other project news in this
+post.
+
+## Projects "Powered By" Apache Arrow
+
+A growing ecosystem of projects are using Arrow to solve in-memory analytics
+and data interchange problems. We have added a new [Powered By][18] page to the
+Arrow website where we can acknowledge open source projects and companies which
+are using Arrow. If you would like to add your project to the list as an Arrow
+user, please let us know.
+
+## New Arrow committers
+
+Since the last release, we have added 5 new Apache committers:
+
+* [Phillip Cloud][5], who has mainly contributed to C++ and Python
+* [Bryan Cutler][13], who has mainly contributed to Java and Spark integration
+* [Li Jin][14], who has mainly contributed to Java and Spark integration
+* [Paul Taylor][4], who has mainly contributed to JavaScript
+* [Siddharth Teotia][15], who has mainly contributed to Java
+
+Welcome to the Arrow team, and thank you for your contributions!
+
+## Improved Java vector API, performance improvements
+
+Siddharth Teotia led efforts to revamp the Java vector API to make things
+simpler and faster. As part of this, we removed the dichotomy between nullable
+and non-nullable vectors.
+
+See [Sidd's blog post][10] for more about these changes.
+
+## Decimal support in C++, Python, consistency with Java
+
+[Phillip Cloud][5] led efforts this release to harden details about exact
+decimal values in the Arrow specification and ensure a consistent
+implementation across Java, C++, and Python.
+
+Arrow now supports decimals represented internally as a 128-bit little-endian
+integer, with a set precision and scale (as defined in many SQL-based
+systems). As part of this work, we needed to change Java's internal
+representation from big- to little-endian.
+
+We are now integration testing decimals between Java, C++, and Python, which
+will facilitate Arrow adoption in Apache Spark and other systems that use both
+Java and Python.
+
+Decimal data can now be read and written by the [Apache Parquet C++
+library][6], including via pyarrow.
+
+In the future, we may implement support for smaller-precision decimals
+represented by 32- or 64-bit integers.
+
+## C++ improvements: expanded kernels library and more
+
+In C++, we have continued developing the new `arrow::compute` submodule
+consisting of native computation fuctions for Arrow data. New contributor
+[Licht Takeuchi][7] helped expand the supported types for type casting in
+`compute::Cast`. We have also implemented new kernels `Unique` and
+`DictionaryEncode` for computing the distinct elements of an array and
+dictionary encoding (conversion to categorical), respectively.
+
+We expect the C++ computation "kernel" library to be a major expansion area for
+the project over the next year and beyond. Here, we can also implement SIMD-
+and GPU-accelerated versions of basic in-memory analytics functionality.
+
+As minor breaking API change in C++, we have made the `RecordBatch` and `Table`
+APIs "virtual" or abstract interfaces, to enable different implementations of a
+record batch or table which conform to the standard interface. This will help
+enable features like lazy IO or column loading.
+
+There was significant work improving the C++ library generally and supporting
+work happening in Python and C. See the change log for full details.
+
+## GLib C improvements: Meson build, GPU support
+
+Developing of the GLib-based C bindings has generally tracked work happening in
+the C++ library. These bindings are being used to develop [data science tools
+for Ruby users][8] and elsewhere.
+
+The C bindings now support the [Meson build system][9] in addition to
+autotools, which enables them to be built on Windows.
+
+The Arrow GPU extension library is now also supported in the C bindings.
+
+## JavaScript: first independent release on NPM
+
+[Brian Hulette][11] and [Paul Taylor][4] have been continuing to drive efforts
+on the TypeScript-based JavaScript implementation.
+
+Since the last release, we made a first JavaScript-only Apache release, version
+0.2.0, which is [now available on NPM][12]. We decided to make separate
+JavaScript releases to enable the JS library to release more frequently than
+the rest of the project.
+
+## Python improvements
+
+In addition to some of the new features mentioned above, we have made a variety
+of usability and performance improvements for integrations with pandas, NumPy,
+Dask, and other Python projects which may make use of pyarrow, the Arrow Python
+library.
+
+Some of these improvements include:
+
+* [Component-based serialization][16] for more flexible and memory-efficient
+  transport of large or complex Python objects
+* Substantially improved serialization performance for pandas objects when
+  using `pyarrow.serialize` and `pyarrow.deserialize`. This includes a special
+  `pyarrow.pandas_serialization_context` which further accelerates certain
+  internal details of pandas serialization * Support zero-copy reads for
+* `pandas.DataFrame` using `pyarrow.deserialize` for objects without Python
+  objects
+* Multithreaded conversions from `pandas.DataFrame` to `pyarrow.Table` (we
+  already supported multithreaded conversions from Arrow back to pandas)
+* More efficient conversion from 1-dimensional NumPy arrays to Arrow format
+* New generic buffer compression and decompression APIs `pyarrow.compress` and
+  `pyarrow.decompress`
+* Enhanced Parquet cross-compatibility with [fastparquet][17] and improved Dask
+  support
+* Python support for accessing Parquet row group column statistics
+
+## Upcoming Roadmap
+
+The 0.8.0 release includes some API and format changes, but upcoming releases
+will focus on ompleting and stabilizing critical functionality to move the
+project closer to a 1.0.0 release.
+
+With the ecosystem of projects using Arrow expanding rapidly, we will be
+working to improve and expand the libraries in support of downstream use cases.
+
+We continue to look for more JavaScript, Julia, R, Rust, and other programming
+language developers to join the project and expand the available
+implementations and bindings to more languages.
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.8.0
+[2]: https://arrow.apache.org/install
+[3]: https://arrow.apache.org/release/0.7.0.html
+[3]: https://github.com/kou
+[4]: https://github.com/trxcllnt
+[5]: https://github.com/cpcloud
+[6]: https://github.com/apache/parquet-cpp
+[7]: https://github.com/licht-t
+[8]: https://github.com/red-data-tools
+[9]: https://mesonbuild.com
+[10]: https://arrow.apache.org/blog/2017/12/19/java-vector-improvements/
+[11]: https://github.com/TheNeuralBit
+[12]: http://npmjs.org/package/apache-arrow
+[13]: https://github.com/BryanCutler
+[14]: https://github.com/icexelloss
+[15]: https://github.com/siddharthteotia
+[16]: http://arrow.apache.org/docs/python/ipc.html
+[17]: https://github.com/dask/fastparquet
+[18]: http://arrow.apache.org/powered_by/

From ba009564d32b5b681128cff345ae9177ef39f82a Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Wed, 20 Dec 2017 11:52:17 -0800
Subject: [PATCH 1351/1644] ARROW-1939: Correct links in release blog post

cc @wesm

Author: siddharth <siddharth@dremio.com>

Closes #1436 from siddharthteotia/ARROW-1939 and squashes the following commits:

31f1be14 [siddharth] ARROW-1939: Correct links in release blog post
---
 site/_posts/2017-12-19-0.8.0-release.md | 66 ++++++++++++-------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/site/_posts/2017-12-19-0.8.0-release.md b/site/_posts/2017-12-19-0.8.0-release.md
index c65d30832b7b8..8c1f464ed7200 100644
--- a/site/_posts/2017-12-19-0.8.0-release.md
+++ b/site/_posts/2017-12-19-0.8.0-release.md
@@ -44,7 +44,7 @@ post.
 ## Projects "Powered By" Apache Arrow
 
 A growing ecosystem of projects are using Arrow to solve in-memory analytics
-and data interchange problems. We have added a new [Powered By][18] page to the
+and data interchange problems. We have added a new [Powered By][19] page to the
 Arrow website where we can acknowledge open source projects and companies which
 are using Arrow. If you would like to add your project to the list as an Arrow
 user, please let us know.
@@ -53,11 +53,11 @@ user, please let us know.
 
 Since the last release, we have added 5 new Apache committers:
 
-* [Phillip Cloud][5], who has mainly contributed to C++ and Python
-* [Bryan Cutler][13], who has mainly contributed to Java and Spark integration
-* [Li Jin][14], who has mainly contributed to Java and Spark integration
-* [Paul Taylor][4], who has mainly contributed to JavaScript
-* [Siddharth Teotia][15], who has mainly contributed to Java
+* [Phillip Cloud][6], who has mainly contributed to C++ and Python
+* [Bryan Cutler][14], who has mainly contributed to Java and Spark integration
+* [Li Jin][15], who has mainly contributed to Java and Spark integration
+* [Paul Taylor][5], who has mainly contributed to JavaScript
+* [Siddharth Teotia][16], who has mainly contributed to Java
 
 Welcome to the Arrow team, and thank you for your contributions!
 
@@ -67,11 +67,11 @@ Siddharth Teotia led efforts to revamp the Java vector API to make things
 simpler and faster. As part of this, we removed the dichotomy between nullable
 and non-nullable vectors.
 
-See [Sidd's blog post][10] for more about these changes.
+See [Sidd's blog post][11] for more about these changes.
 
 ## Decimal support in C++, Python, consistency with Java
 
-[Phillip Cloud][5] led efforts this release to harden details about exact
+[Phillip Cloud][6] led efforts this release to harden details about exact
 decimal values in the Arrow specification and ensure a consistent
 implementation across Java, C++, and Python.
 
@@ -85,7 +85,7 @@ will facilitate Arrow adoption in Apache Spark and other systems that use both
 Java and Python.
 
 Decimal data can now be read and written by the [Apache Parquet C++
-library][6], including via pyarrow.
+library][7], including via pyarrow.
 
 In the future, we may implement support for smaller-precision decimals
 represented by 32- or 64-bit integers.
@@ -94,7 +94,7 @@ represented by 32- or 64-bit integers.
 
 In C++, we have continued developing the new `arrow::compute` submodule
 consisting of native computation fuctions for Arrow data. New contributor
-[Licht Takeuchi][7] helped expand the supported types for type casting in
+[Licht Takeuchi][8] helped expand the supported types for type casting in
 `compute::Cast`. We have also implemented new kernels `Unique` and
 `DictionaryEncode` for computing the distinct elements of an array and
 dictionary encoding (conversion to categorical), respectively.
@@ -115,20 +115,20 @@ work happening in Python and C. See the change log for full details.
 
 Developing of the GLib-based C bindings has generally tracked work happening in
 the C++ library. These bindings are being used to develop [data science tools
-for Ruby users][8] and elsewhere.
+for Ruby users][9] and elsewhere.
 
-The C bindings now support the [Meson build system][9] in addition to
+The C bindings now support the [Meson build system][10] in addition to
 autotools, which enables them to be built on Windows.
 
 The Arrow GPU extension library is now also supported in the C bindings.
 
 ## JavaScript: first independent release on NPM
 
-[Brian Hulette][11] and [Paul Taylor][4] have been continuing to drive efforts
+[Brian Hulette][12] and [Paul Taylor][5] have been continuing to drive efforts
 on the TypeScript-based JavaScript implementation.
 
 Since the last release, we made a first JavaScript-only Apache release, version
-0.2.0, which is [now available on NPM][12]. We decided to make separate
+0.2.0, which is [now available on NPM][13]. We decided to make separate
 JavaScript releases to enable the JS library to release more frequently than
 the rest of the project.
 
@@ -141,7 +141,7 @@ library.
 
 Some of these improvements include:
 
-* [Component-based serialization][16] for more flexible and memory-efficient
+* [Component-based serialization][17] for more flexible and memory-efficient
   transport of large or complex Python objects
 * Substantially improved serialization performance for pandas objects when
   using `pyarrow.serialize` and `pyarrow.deserialize`. This includes a special
@@ -154,7 +154,7 @@ Some of these improvements include:
 * More efficient conversion from 1-dimensional NumPy arrays to Arrow format
 * New generic buffer compression and decompression APIs `pyarrow.compress` and
   `pyarrow.decompress`
-* Enhanced Parquet cross-compatibility with [fastparquet][17] and improved Dask
+* Enhanced Parquet cross-compatibility with [fastparquet][18] and improved Dask
   support
 * Python support for accessing Parquet row group column statistics
 
@@ -173,20 +173,20 @@ implementations and bindings to more languages.
 
 [1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.8.0
 [2]: https://arrow.apache.org/install
-[3]: https://arrow.apache.org/release/0.7.0.html
-[3]: https://github.com/kou
-[4]: https://github.com/trxcllnt
-[5]: https://github.com/cpcloud
-[6]: https://github.com/apache/parquet-cpp
-[7]: https://github.com/licht-t
-[8]: https://github.com/red-data-tools
-[9]: https://mesonbuild.com
-[10]: https://arrow.apache.org/blog/2017/12/19/java-vector-improvements/
-[11]: https://github.com/TheNeuralBit
-[12]: http://npmjs.org/package/apache-arrow
-[13]: https://github.com/BryanCutler
-[14]: https://github.com/icexelloss
-[15]: https://github.com/siddharthteotia
-[16]: http://arrow.apache.org/docs/python/ipc.html
-[17]: https://github.com/dask/fastparquet
-[18]: http://arrow.apache.org/powered_by/
+[3]: https://arrow.apache.org/release/0.8.0.html
+[4]: https://github.com/kou
+[5]: https://github.com/trxcllnt
+[6]: https://github.com/cpcloud
+[7]: https://github.com/apache/parquet-cpp
+[8]: https://github.com/licht-t
+[9]: https://github.com/red-data-tools
+[10]: https://mesonbuild.com
+[11]: https://arrow.apache.org/blog/2017/12/19/java-vector-improvements/
+[12]: https://github.com/TheNeuralBit
+[13]: http://npmjs.org/package/apache-arrow
+[14]: https://github.com/BryanCutler
+[15]: https://github.com/icexelloss
+[16]: https://github.com/siddharthteotia
+[17]: http://arrow.apache.org/docs/python/ipc.html
+[18]: https://github.com/dask/fastparquet
+[19]: http://arrow.apache.org/powered_by/

From ce95db3e3a678ac391125bd899c7ec4f1bbf4b97 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 21 Dec 2017 17:11:52 -0500
Subject: [PATCH 1352/1644] ARROW-1931: [C++] Suppress C4996 deprecation
 warning in MSVC builds for now

It's reasonably harmless to suppress these warnings for the time being. When we upgrade to a new release of googletest, we can remove this again

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1433 from wesm/ARROW-1931 and squashes the following commits:

09c3722f [Wes McKinney] Add tr1 define to CMAKE_CXX_FLAGS
6fe636ec [Wes McKinney] Rearrange appveyor build jobs for faster feedback
6ace398e [Wes McKinney] Use CXX_COMMON_FLAGS, all in one place
b786a405 [Wes McKinney] Silence std::tr1 tuple warning everywhere
549e0833 [Wes McKinney] Silence std::tr1 namespace warning only when building gtest
13a77e6b [Wes McKinney] Silence tr1 deprecation warning in MSVC 2017
ccf13188 [Wes McKinney] Add /bigobj flag, suppress C4996 deprecation warning for now
---
 appveyor.yml                          | 18 +++++++++---------
 cpp/cmake_modules/SetupCxxFlags.cmake | 11 +++++++++++
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index e647b8b77080f..ea7922bf658ef 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -20,16 +20,13 @@ os: Visual Studio 2015
 
 environment:
   matrix:
-    - JOB: "Cmake_Script_Tests"
-      GENERATOR: NMake Makefiles
-      PYTHON: "3.5"
-      ARCH: "64"
-      CONFIGURATION: "Release"
     - JOB: "Build"
-      GENERATOR: NMake Makefiles
+      GENERATOR: Visual Studio 15 2017 Win64
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Release"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      BOOST_ROOT: C:\Libraries\boost_1_64_0
     - JOB: "Build_Debug"
       GENERATOR: Visual Studio 14 2015 Win64
       PYTHON: "3.5"
@@ -49,13 +46,16 @@ environment:
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Release"
+    - JOB: "Cmake_Script_Tests"
+      GENERATOR: NMake Makefiles
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Release"
     - JOB: "Build"
-      GENERATOR: Visual Studio 15 2017 Win64
+      GENERATOR: NMake Makefiles
       PYTHON: "3.5"
       ARCH: "64"
       CONFIGURATION: "Release"
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-      BOOST_ROOT: C:\Libraries\boost_1_64_0
 
   MSVC_DEFAULT_OPTIONS: ON
   BOOST_ROOT: C:\Libraries\boost_1_63_0
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 4e0ace0baea0c..97aed6b274976 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -34,6 +34,14 @@ if (MSVC)
   # headers will see dllimport
   add_definitions(-DARROW_EXPORTING)
 
+  # ARROW-1931 See https://github.com/google/googletest/issues/1318
+  #
+  # This is added to CMAKE_CXX_FLAGS instead of CXX_COMMON_FLAGS since only the
+  # former is passed into the external projects
+  if (MSVC_VERSION VERSION_GREATER 1900)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_SILENCE_TR1_NAMESPACE_DEPRECATION_WARNING")
+  endif()
+
   if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     # clang-cl
     set(CXX_COMMON_FLAGS "-EHsc")
@@ -56,6 +64,9 @@ if (MSVC)
       string(REPLACE "/MD" "-MT" ${c_flag} "${${c_flag}}")
     endforeach()
   endif()
+
+  # Support large object code
+  set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /bigobj")
 else()
   # Common flags set below with warning level
   set(CXX_COMMON_FLAGS "")

From 65f5add61829bd413aa8175ad2644cdd362d0c50 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Thu, 21 Dec 2017 18:05:15 -0500
Subject: [PATCH 1353/1644] ARROW-1944: [C++] Fix ARROW_STATIC_LIB in FindArrow

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1440 from pcmoritz/findarrow-libs-fix and squashes the following commits:

bb161b4d [Philipp Moritz] fix static lib path in FindArrow
---
 cpp/cmake_modules/FindArrow.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/FindArrow.cmake b/cpp/cmake_modules/FindArrow.cmake
index 12f76b6c2b3e8..bce4404a47809 100644
--- a/cpp/cmake_modules/FindArrow.cmake
+++ b/cpp/cmake_modules/FindArrow.cmake
@@ -97,8 +97,8 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
     set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib)
     set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib)
   else()
-    set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_LIB_NAME}.a)
-    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_PYTHON_LIB_NAME}.a)
+    set(ARROW_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}.a)
+    set(ARROW_PYTHON_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}.a)
 
     set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
     set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})

From 8986521255f48a2aa775921eac0175b4e7afaa16 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Fri, 22 Dec 2017 11:24:43 -0800
Subject: [PATCH 1354/1644] ARROW-1943: [JAVA] handle setInitialCapacity for
 deeply nested lists

The current implementation of setInitialCapacity() uses a factor of 5 for every level we go into list:

So if the schema is LIST (LIST (LIST (LIST (LIST (LIST (LIST (BIGINT)))))) and we start with an initial capacity of 128, we end up throwing OversizedAllocationException from the BigIntVector because at every level we increased the capacity by 5 and by the time we reached inner scalar that actually stores the data, we were well over max size limit per vector (1MB).

We saw this problem downstream when we failed to read deeply nested JSON data.

The potential fix is to use the factor of 5 only when we are down to the leaf vector. As the depth increases and we are still working with complex/list, we don't use the factor of 5.

cc @jacques-n , @BryanCutler , @icexelloss

Author: siddharth <siddharth@dremio.com>

Closes #1439 from siddharthteotia/ARROW-1943 and squashes the following commits:

d0adbade [siddharth] unit tests
e2f21a8b [siddharth] fix imports
d103436b [siddharth] ARROW-1943: handle setInitialCapacity for deeply nested lists
---
 .../complex/BaseRepeatedValueVector.java      |   8 +-
 .../apache/arrow/vector/TestListVector.java   | 128 ++++++++++++++++++
 2 files changed, 135 insertions(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index a9221f2f6ea56..9a23fd8c356cb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -29,6 +29,8 @@
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.BaseFixedWidthVector;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
@@ -134,7 +136,11 @@ public FieldVector getDataVector() {
   @Override
   public void setInitialCapacity(int numRecords) {
     offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH;
-    vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+    if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) {
+      vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+    } else {
+     vector.setInitialCapacity(numRecords);
+    }
   }
 
   @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 1acce7e0b6654..e2023f4461879 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -559,6 +559,134 @@ public void testNestedListVector() throws Exception {
     }
   }
 
+  @Test
+  public void testNestedListVector1() throws Exception {
+    try (ListVector listVector = ListVector.empty("sourceVector", allocator)) {
+
+      MinorType listType = MinorType.LIST;
+      MinorType scalarType = MinorType.BIGINT;
+
+      listVector.addOrGetVector(FieldType.nullable(listType.getType()));
+
+      ListVector innerList1 = (ListVector)listVector.getDataVector();
+      innerList1.addOrGetVector(FieldType.nullable(listType.getType()));
+
+      ListVector innerList2 = (ListVector)innerList1.getDataVector();
+      innerList2.addOrGetVector(FieldType.nullable(listType.getType()));
+
+      ListVector innerList3 = (ListVector)innerList2.getDataVector();
+      innerList3.addOrGetVector(FieldType.nullable(listType.getType()));
+
+      ListVector innerList4 = (ListVector)innerList3.getDataVector();
+      innerList4.addOrGetVector(FieldType.nullable(listType.getType()));
+
+      ListVector innerList5 = (ListVector)innerList4.getDataVector();
+      innerList5.addOrGetVector(FieldType.nullable(listType.getType()));
+
+      ListVector innerList6 = (ListVector)innerList5.getDataVector();
+      innerList6.addOrGetVector(FieldType.nullable(scalarType.getType()));
+
+      listVector.setInitialCapacity(128);
+    }
+  }
+
+  @Test
+  public void testNestedListVector2() throws Exception {
+    try (ListVector listVector = ListVector.empty("sourceVector", allocator)) {
+      listVector.setInitialCapacity(1);
+      UnionListWriter listWriter = listVector.getWriter();
+      /* allocate memory */
+      listWriter.allocate();
+
+      /* write one or more inner lists at index 0 */
+      listWriter.setPosition(0);
+      listWriter.startList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(50);
+      listWriter.list().bigInt().writeBigInt(100);
+      listWriter.list().bigInt().writeBigInt(200);
+      listWriter.list().endList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(75);
+      listWriter.list().bigInt().writeBigInt(125);
+      listWriter.list().endList();
+
+      listWriter.endList();
+
+      /* write one or more inner lists at index 1 */
+      listWriter.setPosition(1);
+      listWriter.startList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(15);
+      listWriter.list().bigInt().writeBigInt(20);
+      listWriter.list().endList();
+
+      listWriter.list().startList();
+      listWriter.list().bigInt().writeBigInt(25);
+      listWriter.list().bigInt().writeBigInt(30);
+      listWriter.list().bigInt().writeBigInt(35);
+      listWriter.list().endList();
+
+      listWriter.endList();
+
+      assertEquals(2, listVector.getLastSet());
+
+      listVector.setValueCount(2);
+
+      assertEquals(2, listVector.getValueCount());
+
+      /* get listVector value at index 0 -- the value itself is a listvector */
+      Object result = listVector.getObject(0);
+      ArrayList<ArrayList<Long>> resultSet = (ArrayList<ArrayList<Long>>) result;
+      ArrayList<Long> list;
+
+      assertEquals(2, resultSet.size());              /* 2 inner lists at index 0 */
+      assertEquals(3, resultSet.get(0).size());       /* size of first inner list */
+      assertEquals(2, resultSet.get(1).size());      /* size of second inner list */
+
+      list = resultSet.get(0);
+      assertEquals(new Long(50), list.get(0));
+      assertEquals(new Long(100), list.get(1));
+      assertEquals(new Long(200), list.get(2));
+
+      list = resultSet.get(1);
+      assertEquals(new Long(75), list.get(0));
+      assertEquals(new Long(125), list.get(1));
+
+       /* get listVector value at index 1 -- the value itself is a listvector */
+      result = listVector.getObject(1);
+      resultSet = (ArrayList<ArrayList<Long>>) result;
+
+      assertEquals(2, resultSet.size());              /* 3 inner lists at index 1 */
+      assertEquals(2, resultSet.get(0).size());       /* size of first inner list */
+      assertEquals(3, resultSet.get(1).size());      /* size of second inner list */
+
+      list = resultSet.get(0);
+      assertEquals(new Long(15), list.get(0));
+      assertEquals(new Long(20), list.get(1));
+
+      list = resultSet.get(1);
+      assertEquals(new Long(25), list.get(0));
+      assertEquals(new Long(30), list.get(1));
+      assertEquals(new Long(35), list.get(2));
+
+      /* check underlying bitVector */
+      assertFalse(listVector.isNull(0));
+      assertFalse(listVector.isNull(1));
+
+      /* check underlying offsets */
+      final ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
+
+      /* listVector has 2 lists at index 0 and 3 lists at index 1 */
+      assertEquals(0, offsetBuffer.getInt(0 * ListVector.OFFSET_WIDTH));
+      assertEquals(2, offsetBuffer.getInt(1 * ListVector.OFFSET_WIDTH));
+      assertEquals(4, offsetBuffer.getInt(2 * ListVector.OFFSET_WIDTH));
+    }
+  }
+
   @Test
   public void testGetBufferAddress() throws Exception {
     try (ListVector listVector = ListVector.empty("vector", allocator)) {

From 16c79cc94e2440321bcad1ebbef53ea1266b94e8 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Thu, 28 Dec 2017 13:34:32 -0800
Subject: [PATCH 1355/1644] ARROW-1951: [Python] Add memcopy threads argument
 to PlasmaClient put.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1451 from robertnishihara/numthreads and squashes the following commits:

5e2c7ee [Robert Nishihara] Fix tests.
55cb8ac [Robert Nishihara] Revert old change
0903726 [Robert Nishihara] Move memcopy_threads from serialization context to put.
9281de1 [Robert Nishihara] Expose memcopy threads to serialization context.
---
 python/pyarrow/plasma.pyx           | 7 +++++--
 python/pyarrow/tests/test_plasma.py | 6 ++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index f2e8653d8a828..9b3e4093a2f2b 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -370,7 +370,7 @@ cdef class PlasmaClient:
                                          object_buffers[i].metadata_size))
         return result
 
-    def put(self, object value, ObjectID object_id=None,
+    def put(self, object value, ObjectID object_id=None, int memcopy_threads=6,
             serialization_context=None):
         """
         Store a Python value into the object store.
@@ -382,6 +382,9 @@ cdef class PlasmaClient:
         object_id : ObjectID, default None
             If this is provided, the specified object ID will be used to refer
             to the object.
+        memcopy_threads : int, default 6
+            The number of threads to use to write the serialized object into
+            the object store for large objects.
         serialization_context : pyarrow.SerializationContext, default None
             Custom serialization and deserialization context.
 
@@ -394,7 +397,7 @@ cdef class PlasmaClient:
         serialized = pyarrow.serialize(value, serialization_context)
         buffer = self.create(target_id, serialized.total_bytes)
         stream = pyarrow.FixedSizeBufferWriter(buffer)
-        stream.set_memcopy_threads(4)
+        stream.set_memcopy_threads(memcopy_threads)
         serialized.write_to(stream)
         self.seal(target_id)
         return target_id
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index b28bd60c48ad4..decdc73d911b4 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -310,12 +310,14 @@ def __init__(self, val):
         serialization_context = pa.SerializationContext()
         serialization_context.register_type(CustomType, 20*b"\x00")
 
-        object_id = self.plasma_client.put(val, None, serialization_context)
+        object_id = self.plasma_client.put(
+            val, None, serialization_context=serialization_context)
 
         with pytest.raises(pa.ArrowSerializationError):
             result = self.plasma_client.get(object_id)
 
-        result = self.plasma_client.get(object_id, -1, serialization_context)
+        result = self.plasma_client.get(
+            object_id, -1, serialization_context=serialization_context)
         assert result.val == val.val
 
     def test_store_arrow_objects(self):

From 2ca69d1dc2ff8d794639ba059455a21462eb0675 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 28 Dec 2017 15:27:50 -0800
Subject: [PATCH 1356/1644] ARROW-1948: [Java] Load ListVector validity buffer
 with BitVectorHelper to handle all non-null

Need to properly set the ListVector validity buffer for the case when the field has all non-nulls.  This is done already in `BitVectorHelper.loadValidityBuffer`, so just need to build the buffer with a call to that function.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1447 from BryanCutler/java-ListVector-non-null-validity-buffer-ARROW-1948 and squashes the following commits:

0d82345 [Bryan Cutler] used BitVectorHelper to properly set the validity buffer
---
 .../main/java/org/apache/arrow/vector/complex/ListVector.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index d50d4c447fefd..0c1daf4905222 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -124,7 +124,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers
     ArrowBuf offBuffer = ownBuffers.get(1);
 
     validityBuffer.release();
-    validityBuffer = bitBuffer.retain(allocator);
+    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
     offsetBuffer.release();
     offsetBuffer = offBuffer.retain(allocator);
 

From 0401e6d7f8f1aa8f62b1f783fd65e1bd1ced7c50 Mon Sep 17 00:00:00 2001
From: Viktor Gal <viktor.gal@maeth.com>
Date: Fri, 29 Dec 2017 17:49:55 +0100
Subject: [PATCH 1357/1644] ARROW-1945:  [C++] Fix doxygen documentation of
 array.h

this is just a small fix in doxygen documentation of the c++ api

Author: Viktor Gal <viktor.gal@maeth.com>

Closes #1442 from vigsterkr/doxygen_fix and squashes the following commits:

3557aef [Viktor Gal] fix doxygen example in array.h
---
 cpp/src/arrow/array.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 6721c7d2717f3..0ae1ddd8ea221 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -65,9 +65,9 @@ struct Decimal;
 /// could cast from int64 to float64 like so:
 ///
 /// Int64Array arr = GetMyData();
-/// auto new_data = arr->data()->ShallowCopy();
+/// auto new_data = arr.data()->ShallowCopy();
 /// new_data->type = arrow::float64();
-/// Float64Array double_arr(new_data);
+/// DoubleArray double_arr(new_data);
 ///
 /// This object is also useful in an analytics setting where memory may be
 /// reused. For example, if we had a group of operations all returning doubles,

From d86fdc71b659958878fb100e811c66f7a1cd1b08 Mon Sep 17 00:00:00 2001
From: Cheng Lian <liancheng@users.noreply.github.com>
Date: Fri, 29 Dec 2017 08:54:05 -0800
Subject: [PATCH 1358/1644] Fix a comment typo (#1448)

---
 .../java/org/apache/arrow/vector/util/DictionaryUtility.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
index cf0596c8c1fb4..6b46dbae385bb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java
@@ -73,7 +73,7 @@ public static Field toMessageFormat(Field field, DictionaryProvider provider, Se
   }
 
   /**
-   * Convert field and child fields that have a dictionary encoding to message format, so fields
+   * Convert field and child fields that have a dictionary encoding to memory format, so fields
    * have the index type
    */
   public static Field toMemoryFormat(Field field, BufferAllocator allocator, Map<Long, Dictionary> dictionaries) {

From 4daffadb92fc527e982d8011b4ebc483503c97ff Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Fri, 29 Dec 2017 14:47:05 -0500
Subject: [PATCH 1359/1644] ARROW-1953: [JS] Fix JS build

this is necessary until gulp publishes 4.0.0 builds to npm

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1453 from trxcllnt/fix-js-gulp-build and squashes the following commits:

0dddbcc [Paul Taylor] Merge branch 'master' into fix-js-gulp-build
dc329fa [Paul Taylor] Merge branch 'master' into fix-js-gulp-build
d9c1b0c [Paul Taylor] set gulp dependency to specific commit
---
 js/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/package.json b/js/package.json
index 179107b63db11..3903d1eedc442 100644
--- a/js/package.json
+++ b/js/package.json
@@ -75,7 +75,7 @@
     "esdoc-standard-plugin": "1.0.0",
     "glob": "7.1.2",
     "google-closure-compiler": "20171203.0.0",
-    "gulp": "github:gulpjs/gulp#4.0",
+    "gulp": "github:gulpjs/gulp#6d71a658c61edb3090221579d8f97dbe086ba2ed",
     "gulp-json-transform": "0.4.5",
     "gulp-rename": "1.2.2",
     "gulp-sourcemaps": "2.6.1",

From 0f38a9503bcde872b705976bd314ccb6c0d0f8d5 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 29 Dec 2017 14:19:38 -0800
Subject: [PATCH 1360/1644] ARROW-1947: [Plasma] Change Client Create and Get
 to use Buffers

- Create now takes in a pointer to a shared pointer of Buffer and returns a MutableBuffer.
- Object Buffers data and metadata are pointers to shared pointers of Buffer.

Author: Philipp Moritz <pcmoritz@gmail.com>
Author: William Paul <wapaul1@berkeley.edu>

Closes #1444 from Wapaul1/plasma_buffer_api and squashes the following commits:

7fe1cee [Philipp Moritz] fix size of MutableBuffer returned by plasma::Create
aeed751 [Philipp Moritz] more linting
b3274e0 [Philipp Moritz] fix
463dbeb [Philipp Moritz] fix plasma python extension
a055fa8 [Philipp Moritz] fix linting
fc62dda [William Paul] Added metadata buffer
4d8cbb8 [William Paul] Create and Get use Buffers now
---
 cpp/src/plasma/client.cc            | 44 ++++++++++++++++++-----------
 cpp/src/plasma/client.h             | 18 +++++++-----
 cpp/src/plasma/test/client_tests.cc | 36 ++++++++++++++---------
 python/pyarrow/plasma.pyx           | 14 ++++-----
 4 files changed, 67 insertions(+), 45 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 9bbafac38004a..0dd1c44d71c74 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -40,6 +40,7 @@
 #include <thread>
 #include <vector>
 
+#include "arrow/buffer.h"
 #include "plasma/common.h"
 #include "plasma/fling.h"
 #include "plasma/io.h"
@@ -53,6 +54,8 @@
 
 namespace plasma {
 
+using arrow::MutableBuffer;
+
 // Number of threads used for memcopy and hash computations.
 constexpr int64_t kThreadPoolSize = 8;
 constexpr int64_t kBytesInMB = 1 << 20;
@@ -145,7 +148,8 @@ void PlasmaClient::increment_object_count(const ObjectID& object_id, PlasmaObjec
 }
 
 Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
-                            uint8_t* metadata, int64_t metadata_size, uint8_t** data) {
+                            uint8_t* metadata, int64_t metadata_size,
+                            std::shared_ptr<Buffer>* data) {
   ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
                    << data_size << " and metadata size " << metadata_size;
   RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
@@ -162,14 +166,16 @@ Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
   ARROW_CHECK(object.metadata_size == metadata_size);
   // The metadata should come right after the data.
   ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
-  *data = lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) +
-          object.data_offset;
+  *data = std::make_shared<MutableBuffer>(
+      lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) +
+          object.data_offset,
+      data_size);
   // If plasma_create is being called from a transfer, then we will not copy the
   // metadata here. The metadata will be written along with the data streamed
   // from the transfer.
   if (metadata != NULL) {
     // Copy the metadata to the buffer.
-    memcpy(*data + object.data_size, metadata, metadata_size);
+    memcpy((*data)->mutable_data() + object.data_size, metadata, metadata_size);
   }
   // Increment the count of the number of instances of this object that this
   // client is using. A call to PlasmaClient::Release is required to decrement
@@ -203,10 +209,12 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
       ARROW_CHECK(object_entry->second->is_sealed)
           << "Plasma client called get on an unsealed object that it created";
       PlasmaObject* object = &object_entry->second->object;
-      object_buffers[i].data = lookup_mmapped_file(object->handle.store_fd);
-      object_buffers[i].data = object_buffers[i].data + object->data_offset;
+      uint8_t* data = lookup_mmapped_file(object->handle.store_fd);
+      object_buffers[i].data =
+          std::make_shared<Buffer>(data + object->data_offset, object->data_size);
+      object_buffers[i].metadata = std::make_shared<Buffer>(
+          data + object->data_offset + object->data_size, object->metadata_size);
       object_buffers[i].data_size = object->data_size;
-      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
       object_buffers[i].metadata_size = object->metadata_size;
       // Increment the count of the number of instances of this object that this
       // client is using. A call to PlasmaClient::Release is required to
@@ -254,13 +262,15 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
       // The object was retrieved. The user will be responsible for releasing
       // this object.
       int fd = recv_fd(store_conn_);
-      ARROW_CHECK(fd >= 0);
-      object_buffers[i].data =
+      uint8_t* data =
           lookup_or_mmap(fd, object->handle.store_fd, object->handle.mmap_size);
+      ARROW_CHECK(fd >= 0);
       // Finish filling out the return values.
-      object_buffers[i].data = object_buffers[i].data + object->data_offset;
+      object_buffers[i].data =
+          std::make_shared<Buffer>(data + object->data_offset, object->data_size);
+      object_buffers[i].metadata = std::make_shared<Buffer>(
+          data + object->data_offset + object->data_size, object->metadata_size);
       object_buffers[i].data_size = object->data_size;
-      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
       object_buffers[i].metadata_size = object->metadata_size;
       // Increment the count of the number of instances of this object that this
       // client is using. A call to PlasmaClient::Release is required to
@@ -438,14 +448,16 @@ static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
   XXH64_state_t hash_state;
   XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
   if (obj_buffer.data_size >= kBytesInMB) {
-    compute_object_hash_parallel(&hash_state,
-                                 reinterpret_cast<unsigned char*>(obj_buffer.data),
-                                 obj_buffer.data_size);
+    compute_object_hash_parallel(
+        &hash_state, reinterpret_cast<const unsigned char*>(obj_buffer.data->data()),
+        obj_buffer.data_size);
   } else {
-    XXH64_update(&hash_state, reinterpret_cast<unsigned char*>(obj_buffer.data),
+    XXH64_update(&hash_state,
+                 reinterpret_cast<const unsigned char*>(obj_buffer.data->data()),
                  obj_buffer.data_size);
   }
-  XXH64_update(&hash_state, reinterpret_cast<unsigned char*>(obj_buffer.metadata),
+  XXH64_update(&hash_state,
+               reinterpret_cast<const unsigned char*>(obj_buffer.metadata->data()),
                obj_buffer.metadata_size);
   return XXH64_digest(&hash_state);
 }
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index cfd11c16dbace..78793f1a73a64 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -26,11 +26,13 @@
 #include <string>
 #include <unordered_map>
 
+#include "arrow/buffer.h"
 #include "arrow/status.h"
 #include "arrow/util/visibility.h"
 #include "plasma/common.h"
 
 using arrow::Status;
+using arrow::Buffer;
 
 namespace plasma {
 
@@ -41,14 +43,16 @@ constexpr int64_t kL3CacheSizeBytes = 100000000;
 
 /// Object buffer data structure.
 struct ObjectBuffer {
+  /// The data buffer.
+  std::shared_ptr<Buffer> data;
   /// The size in bytes of the data object.
   int64_t data_size;
-  /// The address of the data object.
-  uint8_t* data;
+  /// The metadata buffer.
+  std::shared_ptr<Buffer> metadata;
   /// The metadata size in bytes.
   int64_t metadata_size;
-  /// The address of the metadata.
-  uint8_t* metadata;
+  /// The device number.
+  int device_num;
 };
 
 /// Configuration options for the plasma client.
@@ -107,11 +111,11 @@ class ARROW_EXPORT PlasmaClient {
   ///        should be NULL.
   /// \param metadata_size The size in bytes of the metadata. If there is no
   ///        metadata, this should be 0.
-  /// \param data The address of the newly created object will be written here.
+  /// \param data A buffer containing the address of the newly created object
+  ///        will be written here.
   /// \return The return status.
   Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
-                int64_t metadata_size, uint8_t** data);
-
+                int64_t metadata_size, std::shared_ptr<Buffer>* data);
   /// Get some objects from the Plasma Store. This function will block until the
   /// objects have all been created and sealed in the Plasma Store or the
   /// timeout
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index f44ed2510c582..5cd3063bb4309 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -71,7 +71,7 @@ TEST_F(TestPlasmaStore, ContainsTest) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t* data;
+  std::shared_ptr<Buffer> data;
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client_.Seal(object_id));
   // Avoid race condition of Plasma Manager waiting for notification.
@@ -94,16 +94,20 @@ TEST_F(TestPlasmaStore, GetTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
+  std::shared_ptr<Buffer> data_buffer;
   uint8_t* data;
-  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  ARROW_CHECK_OK(
+      client_.Create(object_id, data_size, metadata, metadata_size, &data_buffer));
+  data = data_buffer->mutable_data();
   for (int64_t i = 0; i < data_size; i++) {
     data[i] = static_cast<uint8_t>(i % 4);
   }
   ARROW_CHECK_OK(client_.Seal(object_id));
 
   ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  const uint8_t* object_data = object_buffer.data->data();
   for (int64_t i = 0; i < data_size; i++) {
-    ASSERT_EQ(data[i], object_buffer.data[i]);
+    ASSERT_EQ(data[i], object_data[i]);
   }
 }
 
@@ -116,18 +120,18 @@ TEST_F(TestPlasmaStore, MultipleGetTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t* data;
+  std::shared_ptr<Buffer> data;
   ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data));
-  data[0] = 1;
+  data->mutable_data()[0] = 1;
   ARROW_CHECK_OK(client_.Seal(object_id1));
 
   ARROW_CHECK_OK(client_.Create(object_id2, data_size, metadata, metadata_size, &data));
-  data[0] = 2;
+  data->mutable_data()[0] = 2;
   ARROW_CHECK_OK(client_.Seal(object_id2));
 
   ARROW_CHECK_OK(client_.Get(object_ids, 2, -1, object_buffer));
-  ASSERT_EQ(object_buffer[0].data[0], 1);
-  ASSERT_EQ(object_buffer[1].data[0], 2);
+  ASSERT_EQ(object_buffer[0].data->data()[0], 1);
+  ASSERT_EQ(object_buffer[1].data->data()[0], 2);
 }
 
 TEST_F(TestPlasmaStore, AbortTest) {
@@ -143,11 +147,13 @@ TEST_F(TestPlasmaStore, AbortTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t* data;
+  std::shared_ptr<Buffer> data;
+  uint8_t* data_ptr;
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  data_ptr = data->mutable_data();
   // Write some data.
   for (int64_t i = 0; i < data_size / 2; i++) {
-    data[i] = static_cast<uint8_t>(i % 4);
+    data_ptr[i] = static_cast<uint8_t>(i % 4);
   }
   // Attempt to abort. Test that this fails before the first release.
   Status status = client_.Abort(object_id);
@@ -162,15 +168,17 @@ TEST_F(TestPlasmaStore, AbortTest) {
 
   // Create the object successfully this time.
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  data_ptr = data->mutable_data();
   for (int64_t i = 0; i < data_size; i++) {
-    data[i] = static_cast<uint8_t>(i % 4);
+    data_ptr[i] = static_cast<uint8_t>(i % 4);
   }
   ARROW_CHECK_OK(client_.Seal(object_id));
 
   // Test that we can get the object.
   ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  const uint8_t* buffer_ptr = object_buffer.data->data();
   for (int64_t i = 0; i < data_size; i++) {
-    ASSERT_EQ(data[i], object_buffer.data[i]);
+    ASSERT_EQ(data_ptr[i], buffer_ptr[i]);
   }
 }
 
@@ -187,7 +195,7 @@ TEST_F(TestPlasmaStore, MultipleClientTest) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t* data;
+  std::shared_ptr<Buffer> data;
   ARROW_CHECK_OK(client2_.Create(object_id, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client2_.Seal(object_id));
   // Test that the first client can get the object.
@@ -227,7 +235,7 @@ TEST_F(TestPlasmaStore, ManyObjectTest) {
     int64_t data_size = 100;
     uint8_t metadata[] = {5};
     int64_t metadata_size = sizeof(metadata);
-    uint8_t* data;
+    std::shared_ptr<Buffer> data;
     ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
 
     if (i % 3 == 0) {
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index 9b3e4093a2f2b..abeec32783398 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -81,7 +81,7 @@ cdef extern from "plasma/client.h" nogil:
 
         CStatus Create(const CUniqueID& object_id, int64_t data_size,
                        const uint8_t* metadata, int64_t metadata_size,
-                       uint8_t** data)
+                       const shared_ptr[CBuffer]* data)
 
         CStatus Get(const CUniqueID* object_ids, int64_t num_objects,
                     int64_t timeout_ms, CObjectBuffer* object_buffers)
@@ -118,9 +118,9 @@ cdef extern from "plasma/client.h" nogil:
 
     cdef struct CObjectBuffer" plasma::ObjectBuffer":
         int64_t data_size
-        uint8_t* data
+        shared_ptr[CBuffer] data
         int64_t metadata_size
-        uint8_t* metadata
+        shared_ptr[CBuffer] metadata
 
 
 def make_object_id(object_id):
@@ -245,10 +245,8 @@ cdef class PlasmaClient:
             check_status(self.client.get().Get(ids.data(), ids.size(),
                          timeout_ms, result[0].data()))
 
-    cdef _make_plasma_buffer(self, ObjectID object_id, uint8_t* data,
+    cdef _make_plasma_buffer(self, ObjectID object_id, shared_ptr[CBuffer] buffer,
                              int64_t size):
-        cdef shared_ptr[CBuffer] buffer
-        buffer.reset(new CBuffer(data, size))
         result = PlasmaBuffer(object_id, self)
         result.init(buffer)
         return result
@@ -296,12 +294,12 @@ cdef class PlasmaClient:
                 not be created because the plasma store is unable to evict
                 enough objects to create room for it.
         """
-        cdef uint8_t* data
+        cdef shared_ptr[CBuffer] data
         with nogil:
             check_status(self.client.get().Create(object_id.data, data_size,
                                                   <uint8_t*>(metadata.data()),
                                                   metadata.size(), &data))
-        return self._make_mutable_plasma_buffer(object_id, data, data_size)
+        return self._make_mutable_plasma_buffer(object_id, data.get().mutable_data(), data_size)
 
     def get_buffers(self, object_ids, timeout_ms=-1):
         """

From 37c3265b8a7a9e3ccb3063faeafa537d5b334830 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Sat, 30 Dec 2017 20:05:08 -0800
Subject: [PATCH 1361/1644] ARROW-1946: [JAVA] Add APIs to decimal vector for
 writing big endian data
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We recently moved Dremio to LE Decimal format (similar to Arrow). As part of that we introduced some APIs in decimal vector which take a big endian data and swap the bytes while writing into the ArrowBuf of decimal vector.

The advantage of these APIs is that caller would not have to allocate an additional memory and write( and read) source big endian twice for swapping into new memory and using that to write into the vector.

We can directly swap bytes while writing into the vector – just read once and swap while writing.

cc @jacques-n , @BryanCutler , @icexelloss

Author: siddharth <siddharth@dremio.com>

Closes #1443 from siddharthteotia/ARROW-1946 and squashes the following commits:

7805b62b [siddharth] unit tests
c89efbf8 [siddharth] ARROW-1946: Add APIs to decimal vector for writing big endian data
---
 .../templates/AbstractFieldWriter.java        |  4 +
 .../AbstractPromotableFieldWriter.java        |  8 ++
 .../codegen/templates/ComplexWriters.java     |  7 ++
 .../apache/arrow/vector/DecimalVector.java    | 47 +++++++++++
 .../arrow/vector/TestDecimalVector.java       | 79 +++++++++++++++++++
 5 files changed, 145 insertions(+)

diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 853f67fd0dd56..fce6876025a91 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -67,6 +67,10 @@ public void write(${name}Holder holder) {
   public void write${minor.class}(${friendlyType} value) {
     fail("${name}");
   }
+
+  public void writeBigEndianBytesToDecimal(byte[] value) {
+    fail("${name}");
+  }
   </#if>
 
   </#list></#list>
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 228c2c531f98f..7f4a13d4f06e8 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.vector.types.Types;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 
 <@pp.dropOutputFile />
@@ -82,6 +84,12 @@ public void write(${name}Holder holder) {
     getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
   }
 
+  <#if minor.class == "Decimal">
+  public void writeBigEndianBytesToDecimal(byte[] value) {
+    getWriter(Types.MinorType.DECIMAL).writeBigEndianBytesToDecimal(value);
+  }
+  </#if>
+
   </#list></#list>
   public void writeNull() {
   }
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 4a1545a173dd3..24994d72ac1d8 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -120,6 +120,11 @@ public void write(Nullable${minor.class}Holder h) {
     vector.setSafe(idx(), value);
     vector.setValueCount(idx()+1);
   }
+
+  public void writeBigEndianBytesToDecimal(byte[] value) {
+    vector.setBigEndianSafe(idx(), value);
+    vector.setValueCount(idx()+1);
+  }
   </#if>
 
   public void writeNull() {
@@ -146,6 +151,8 @@ public interface ${eName}Writer extends BaseWriter {
 <#if minor.class == "Decimal">
 
   public void write${minor.class}(${friendlyType} value);
+
+  public void writeBigEndianBytesToDecimal(byte[] value);
 </#if>
 }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index 71a74cb9f3338..a043575081fb3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -195,6 +195,43 @@ public void set(int index, ArrowBuf buffer) {
     valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH);
   }
 
+  /**
+   * Set the decimal element at given index to the provided array of bytes.
+   * Decimal is now implemented as Little Endian. This API allows the user
+   * to pass a decimal value in the form of byte array in BE byte order.
+   *
+   * Consumers of Arrow code can use this API instead of first swapping
+   * the source bytes (doing a write and read) and then finally writing to
+   * ArrowBuf of decimal vector.
+   *
+   * This method takes care of adding the necessary padding if the length
+   * of byte array is less then 16 (length of decimal type).
+   *
+   * @param index position of element
+   * @param value array of bytes containing decimal in big endian byte order.
+   */
+  public void setBigEndian(int index, byte[] value) {
+    assert value.length <= TYPE_WIDTH;
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    final int length = value.length;
+    int startIndex = index * TYPE_WIDTH;
+    if (length == TYPE_WIDTH) {
+      for (int i = TYPE_WIDTH - 1; i >= 3; i-=4) {
+        valueBuffer.setByte(startIndex, value[i]);
+        valueBuffer.setByte(startIndex + 1, value[i-1]);
+        valueBuffer.setByte(startIndex + 2, value[i-2]);
+        valueBuffer.setByte(startIndex + 3, value[i-3]);
+        startIndex += 4;
+      }
+    } else {
+      for (int i = length - 1; i >= 0; i--) {
+        valueBuffer.setByte(startIndex, value[i]);
+        startIndex++;
+      }
+      valueBuffer.setZero(startIndex, TYPE_WIDTH - length);
+    }
+  }
+
   /**
    * Set the element at the given index to the given value.
    *
@@ -262,6 +299,16 @@ public void setSafe(int index, ArrowBuf buffer) {
     set(index, buffer);
   }
 
+  /**
+   * Same as {@link #setBigEndian(int, byte[])} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   */
+  public void setBigEndianSafe(int index, byte[] value) {
+    handleSafe(index);
+    setBigEndian(index, value);
+  }
+
   /**
    * Same as {@link #set(int, int, ArrowBuf)} except that it handles the
    * case when index is greater than or equal to existing
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index a48c035718990..8c86452fcc3bf 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -111,4 +111,83 @@ public void testBigDecimalDifferentScaleAndPrecision() {
       }
     }
   }
+
+  @Test
+  public void testWriteBigEndian() {
+    try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(38, 9), allocator);) {
+      decimalVector.allocateNew();
+      BigDecimal decimal1 = new BigDecimal("123456789.000000000");
+      BigDecimal decimal2 = new BigDecimal("11.123456789");
+      BigDecimal decimal3 = new BigDecimal("1.000000000");
+      BigDecimal decimal4 = new BigDecimal("0.111111111");
+      BigDecimal decimal5 = new BigDecimal("987654321.123456789");
+      BigDecimal decimal6 = new BigDecimal("222222222222.222222222");
+      BigDecimal decimal7 = new BigDecimal("7777777777777.666666667");
+      BigDecimal decimal8 = new BigDecimal("1212121212.343434343");
+
+      byte[] decimalValue1 = decimal1.unscaledValue().toByteArray();
+      byte[] decimalValue2 = decimal2.unscaledValue().toByteArray();
+      byte[] decimalValue3 = decimal3.unscaledValue().toByteArray();
+      byte[] decimalValue4 = decimal4.unscaledValue().toByteArray();
+      byte[] decimalValue5 = decimal5.unscaledValue().toByteArray();
+      byte[] decimalValue6 = decimal6.unscaledValue().toByteArray();
+      byte[] decimalValue7 = decimal7.unscaledValue().toByteArray();
+      byte[] decimalValue8 = decimal8.unscaledValue().toByteArray();
+
+      decimalVector.setBigEndian(0, decimalValue1);
+      decimalVector.setBigEndian(1, decimalValue2);
+      decimalVector.setBigEndian(2, decimalValue3);
+      decimalVector.setBigEndian(3, decimalValue4);
+      decimalVector.setBigEndian(4, decimalValue5);
+      decimalVector.setBigEndian(5, decimalValue6);
+      decimalVector.setBigEndian(6, decimalValue7);
+      decimalVector.setBigEndian(7, decimalValue8);
+
+      decimalVector.setValueCount(8);
+      assertEquals(8, decimalVector.getValueCount());
+      assertEquals(decimal1, decimalVector.getObject(0));
+      assertEquals(decimal2, decimalVector.getObject(1));
+      assertEquals(decimal3, decimalVector.getObject(2));
+      assertEquals(decimal4, decimalVector.getObject(3));
+      assertEquals(decimal5, decimalVector.getObject(4));
+      assertEquals(decimal6, decimalVector.getObject(5));
+      assertEquals(decimal7, decimalVector.getObject(6));
+      assertEquals(decimal8, decimalVector.getObject(7));
+    }
+  }
+
+  @Test
+  public void testBigDecimalReadWrite() {
+    try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(38, 9), allocator);) {
+      decimalVector.allocateNew();
+      BigDecimal decimal1 = new BigDecimal("123456789.000000000");
+      BigDecimal decimal2 = new BigDecimal("11.123456789");
+      BigDecimal decimal3 = new BigDecimal("1.000000000");
+      BigDecimal decimal4 = new BigDecimal("-0.111111111");
+      BigDecimal decimal5 = new BigDecimal("-987654321.123456789");
+      BigDecimal decimal6 = new BigDecimal("-222222222222.222222222");
+      BigDecimal decimal7 = new BigDecimal("7777777777777.666666667");
+      BigDecimal decimal8 = new BigDecimal("1212121212.343434343");
+
+      decimalVector.set(0, decimal1);
+      decimalVector.set(1, decimal2);
+      decimalVector.set(2, decimal3);
+      decimalVector.set(3, decimal4);
+      decimalVector.set(4, decimal5);
+      decimalVector.set(5, decimal6);
+      decimalVector.set(6, decimal7);
+      decimalVector.set(7, decimal8);
+
+      decimalVector.setValueCount(8);
+      assertEquals(8, decimalVector.getValueCount());
+      assertEquals(decimal1, decimalVector.getObject(0));
+      assertEquals(decimal2, decimalVector.getObject(1));
+      assertEquals(decimal3, decimalVector.getObject(2));
+      assertEquals(decimal4, decimalVector.getObject(3));
+      assertEquals(decimal5, decimalVector.getObject(4));
+      assertEquals(decimal6, decimalVector.getObject(5));
+      assertEquals(decimal7, decimalVector.getObject(6));
+      assertEquals(decimal8, decimalVector.getObject(7));
+    }
+  }
 }

From 4a3e6e361685cc3350e679ef3461103ffc4df0a2 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Tue, 2 Jan 2018 11:30:45 -0500
Subject: [PATCH 1362/1644] ARROW-1941: [Python] Fix empty list roundtrip in
 to_pandas

This closes [ARROW-1941](https://issues.apache.org/jira/browse/ARROW-1941).

Author: Licht-T <licht-t@outlook.jp>

Closes #1449 from Licht-T/fix-empty-list-roundtrip and squashes the following commits:

165dc6f [Licht-T] TST: Add test for the empty list roundtrip
0ddfd87 [Licht-T] BUG: Fix empty list roundtrip
---
 cpp/src/arrow/python/arrow_to_pandas.cc     |  2 ++
 python/pyarrow/tests/test_convert_pandas.py | 12 ++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 08ce37cda6351..e21bbda055953 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -90,6 +90,7 @@ struct WrapBytes<FixedSizeBinaryArray> {
 
 static inline bool ListTypeSupported(const DataType& type) {
   switch (type.id()) {
+    case Type::NA:
     case Type::UINT8:
     case Type::INT8:
     case Type::UINT16:
@@ -695,6 +696,7 @@ class ObjectBlock : public PandasBlock {
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
       switch (list_type->value_type()->id()) {
+        CONVERTLISTSLIKE_CASE(FloatType, NA)
         CONVERTLISTSLIKE_CASE(UInt8Type, UINT8)
         CONVERTLISTSLIKE_CASE(Int8Type, INT8)
         CONVERTLISTSLIKE_CASE(UInt16Type, UINT16)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 7609d3488c6c0..76b55cf90431c 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1317,6 +1317,18 @@ def test_table_column_subset_metadata(self):
             result = table_subset2.to_pandas()
             tm.assert_frame_equal(result, df[['a']].reset_index(drop=True))
 
+    def test_empty_list_roundtrip(self):
+        empty_list_array = np.empty((3,), dtype=object)
+        empty_list_array.fill([])
+
+        df = pd.DataFrame({'a': np.array(['1', '2', '3']),
+                           'b': empty_list_array})
+        tbl = pa.Table.from_pandas(df)
+
+        result = tbl.to_pandas()
+
+        tm.assert_frame_equal(result, df)
+
 
 def _fully_loaded_dataframe_example():
     from distutils.version import LooseVersion

From 3a69efce59f807c60b8d218abf533e4f1b45ed62 Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Thu, 4 Jan 2018 10:11:13 -0500
Subject: [PATCH 1363/1644] ARROW-1920 [C++/Python] Add experimental reader for
 Apache ORC files

This adds support for reading ORC files in the C++ library, as well as python bindings for this functionality.

Author: Jim Crist <jiminy.crist@gmail.com>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1418 from jcrist/orc-adapter and squashes the following commits:

7e0400eb [Jim Crist] lint
d6d32b50 [Uwe L. Korn] Hide symbols introduced by orc static lib
a2966406 [Jim Crist] Tweak error message
f45ac3dd [Jim Crist] Read reads as a table
57bc63dd [Jim Crist] Use `vector<int>` instead of `list<uint64_t>`
1d539276 [Jim Crist] date32 instead of date64
4b7a3a5d [Jim Crist] Add brief docs
e7835445 [Jim Crist] More fixups
33f5b10f [Jim Crist] Turn off ARROW_ORC on windows
86a23550 [Jim Crist] Cleanups
2cfdd924 [Jim Crist] Fix build when dependencies aren't already installed
876c3a36 [Jim Crist] Use fPIC on protobuf as well
f4a29f87 [Jim Crist] Ensure -fPIC on orc build
7cf1659e [Jim Crist] Build python orc support on travis
2adf938a [Jim Crist] Add ORC support
5c791046 [Jim Crist] Add cmake support for liborc
---
 ci/travis_script_python.sh                    |   2 +-
 cpp/CMakeLists.txt                            |  22 +
 cpp/cmake_modules/FindProtobuf.cmake          |  89 +++
 cpp/cmake_modules/ThirdpartyToolchain.cmake   |  73 ++
 cpp/src/arrow/CMakeLists.txt                  |   5 +
 cpp/src/arrow/adapters/orc/CMakeLists.txt     |  25 +
 cpp/src/arrow/adapters/orc/adapter.cc         | 697 ++++++++++++++++++
 cpp/src/arrow/adapters/orc/adapter.h          | 105 +++
 cpp/src/arrow/symbols.map                     |   4 +
 python/CMakeLists.txt                         |  11 +
 .../scripts/check_arrow_visibility.sh         |   5 +-
 python/pyarrow/_orc.pxd                       |  50 ++
 python/pyarrow/_orc.pyx                       | 111 +++
 python/pyarrow/orc.py                         | 149 ++++
 python/setup.py                               |   9 +
 15 files changed, 1355 insertions(+), 2 deletions(-)
 create mode 100644 cpp/cmake_modules/FindProtobuf.cmake
 create mode 100644 cpp/src/arrow/adapters/orc/CMakeLists.txt
 create mode 100644 cpp/src/arrow/adapters/orc/adapter.cc
 create mode 100644 cpp/src/arrow/adapters/orc/adapter.h
 create mode 100644 python/pyarrow/_orc.pxd
 create mode 100644 python/pyarrow/_orc.pyx
 create mode 100644 python/pyarrow/orc.py

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 5f7b0a9a1afed..444386fdf955e 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -82,7 +82,7 @@ fi
 export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE
 
 pip install -r requirements.txt
-python setup.py build_ext --with-parquet --with-plasma \
+python setup.py build_ext --with-parquet --with-plasma --with-orc\
        install --single-version-externally-managed --record=record.text
 popd
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index dd159ced29193..ede13af5d413b 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -119,6 +119,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow GPU extensions (requires CUDA installation)"
     OFF)
 
+  option(ARROW_ORC
+    "Build the Arrow ORC adapter"
+    ON)
+
   option(ARROW_JEMALLOC
     "Build the Arrow jemalloc-based allocator"
     OFF)
@@ -226,6 +230,17 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
   set(ARROW_WITH_ZSTD ON)
 endif()
 
+if (MSVC)
+  # ORC doesn't build on windows
+  set(ARROW_ORC OFF)
+endif()
+
+if(ARROW_ORC)
+  set(ARROW_WITH_LZ4 ON)
+  set(ARROW_WITH_SNAPPY ON)
+  set(ARROW_WITH_ZLIB ON)
+endif()
+
 if(NOT ARROW_BUILD_TESTS)
   set(NO_TESTS 1)
 endif()
@@ -526,6 +541,13 @@ if (ARROW_WITH_GRPC)
     ${ARROW_STATIC_LINK_LIBS})
 endif()
 
+if (ARROW_ORC)
+  SET(ARROW_STATIC_LINK_LIBS
+    orc
+    protobuf
+    ${ARROW_STATIC_LINK_LIBS})
+endif()
+
 if (ARROW_STATIC_LINK_LIBS)
   add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
 endif()
diff --git a/cpp/cmake_modules/FindProtobuf.cmake b/cpp/cmake_modules/FindProtobuf.cmake
new file mode 100644
index 0000000000000..a42f4493af494
--- /dev/null
+++ b/cpp/cmake_modules/FindProtobuf.cmake
@@ -0,0 +1,89 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# PROTOBUF_HOME environmental variable is used to check for Protobuf headers and static library
+
+# PROTOBUF_INCLUDE_DIR: directory containing headers
+# PROTOBUF_LIBS: directory containing Protobuf libraries
+# PROTOBUF_STATIC_LIB: location of protobuf.a
+# PROTOC_STATIC_LIB: location of protoc.a
+# PROTOBUF_EXECUTABLE: location of protoc
+# PROTOBUF_FOUND is set if Protobuf is found
+
+
+if( NOT "${PROTOBUF_HOME}" STREQUAL "")
+    file (TO_CMAKE_PATH "${PROTOBUF_HOME}" _protobuf_path)
+endif()
+
+message (STATUS "PROTOBUF_HOME: ${PROTOBUF_HOME}")
+
+find_path (PROTOBUF_INCLUDE_DIR google/protobuf/io/zero_copy_stream.h HINTS
+  ${_protobuf_path}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "include")
+
+find_path (PROTOBUF_INCLUDE_DIR google/protobuf/io/coded_stream.h HINTS
+  ${_protobuf_path}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "include")
+
+find_library (PROTOBUF_LIBRARY NAMES protobuf PATHS
+  ${_protobuf_path}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib")
+
+find_library (PROTOC_LIBRARY NAMES protoc PATHS
+  ${_protobuf_path}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib")
+
+find_program(PROTOBUF_EXECUTABLE protoc HINTS
+  ${_protobuf_path}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "bin")
+
+if (PROTOBUF_INCLUDE_DIR AND PROTOBUF_LIBRARY AND PROTOC_LIBRARY AND PROTOBUF_EXECUTABLE)
+  set (PROTOBUF_FOUND TRUE)
+  get_filename_component (PROTOBUF_LIBS ${PROTOBUF_LIBRARY} PATH)
+  set (PROTOBUF_LIB_NAME protobuf)
+  set (PROTOC_LIB_NAME protoc)
+  set (PROTOBUF_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOBUF_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set (PROTOC_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
+else ()
+  set (PROTOBUF_FOUND FALSE)
+endif ()
+
+if (PROTOBUF_FOUND)
+  message (STATUS "Found the Protobuf headers: ${PROTOBUF_INCLUDE_DIR}")
+  message (STATUS "Found the Protobuf library: ${PROTOBUF_STATIC_LIB}")
+  message (STATUS "Found the Protoc library: ${PROTOC_STATIC_LIB}")
+  message (STATUS "Found the Protoc executable: ${PROTOBUF_EXECUTABLE}")
+else()
+  if (_protobuf_path)
+    set (PROTOBUF_ERR_MSG "Could not find Protobuf. Looked in ${_protobuf_path}.")
+  else ()
+    set (PROTOBUF_ERR_MSG "Could not find Protobuf in system search paths.")
+  endif()
+
+  if (Protobuf_FIND_REQUIRED)
+    message (FATAL_ERROR "${PROTOBUF_ERR_MSG}")
+  else ()
+    message (STATUS "${PROTOBUF_ERR_MSG}")
+  endif ()
+endif()
+
+mark_as_advanced (
+  PROTOBUF_INCLUDE_DIR
+  PROTOBUF_LIBS
+  PROTOBUF_STATIC_LIB
+  PROTOC_STATIC_LIB
+)
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index b706aab45265d..4f64434170655 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -29,7 +29,9 @@ set(SNAPPY_VERSION "1.1.3")
 set(BROTLI_VERSION "v0.6.0")
 set(LZ4_VERSION "1.7.5")
 set(ZSTD_VERSION "1.2.0")
+set(PROTOBUF_VERSION "2.6.0")
 set(GRPC_VERSION "94582910ad7f82ad447ecc72e6548cb669e4f7a9") # v1.6.5
+set(ORC_VERSION "cf00b67795717ab3eb04e950780ed6d104109017")
 
 string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
 
@@ -721,6 +723,7 @@ if (ARROW_WITH_LZ4)
 
   if("${LZ4_HOME}" STREQUAL "")
     set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep")
+    set(LZ4_HOME "${LZ4_BUILD_DIR}")
     set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib")
 
     if (MSVC)
@@ -865,3 +868,73 @@ if (ARROW_WITH_GRPC)
   endif()
 
 endif()
+
+if (ARROW_ORC)
+  # protobuf
+  if ("${PROTOBUF_HOME}" STREQUAL "")
+    set (PROTOBUF_PREFIX "${THIRDPARTY_DIR}/protobuf_ep-install")
+    set (PROTOBUF_HOME "${PROTOBUF_PREFIX}")
+    set (PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include")
+    set (PROTOBUF_STATIC_LIB "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set (PROTOBUF_SRC_URL "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz")
+
+    ExternalProject_Add(protobuf_ep
+      CONFIGURE_COMMAND "./configure" "--disable-shared" "--prefix=${PROTOBUF_PREFIX}" "CXXFLAGS=${EP_CXX_FLAGS}"
+      BUILD_IN_SOURCE 1
+      URL ${PROTOBUF_SRC_URL}
+      LOG_DOWNLOAD 1
+      LOG_CONFIGURE 1
+      LOG_BUILD 1
+      LOG_INSTALL 1
+      BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}")
+
+    set (PROTOBUF_VENDORED 1)
+  else ()
+    find_package (Protobuf REQUIRED)
+    set (PROTOBUF_VENDORED 0)
+  endif ()
+
+  include_directories (SYSTEM ${PROTOBUF_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(protobuf
+    STATIC_LIB ${PROTOBUF_STATIC_LIB})
+
+  if (PROTOBUF_VENDORED)
+    add_dependencies (protobuf protobuf_ep)
+  endif ()
+
+  # orc
+  set(ORC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/orc_ep-install")
+  set(ORC_HOME "${ORC_PREFIX}")
+  set(ORC_INCLUDE_DIR "${ORC_PREFIX}/include")
+  set(ORC_STATIC_LIB "${ORC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}orc${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+  # Since LZ4 isn't installed, the header file is in ${LZ4_HOME}/lib instead of
+  # ${LZ4_HOME}/include, which forces us to specify the include directory
+  # manually as well.
+  set (ORC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                      -DCMAKE_INSTALL_PREFIX=${ORC_PREFIX}
+                      -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}
+                      -DBUILD_LIBHDFSPP=OFF
+                      -DBUILD_JAVA=OFF
+                      -DBUILD_TOOLS=OFF
+                      -DBUILD_CPP_TESTS=OFF
+                      -DINSTALL_VENDORED_LIBS=OFF
+                      -DPROTOBUF_HOME=${PROTOBUF_HOME}
+                      -DLZ4_HOME=${LZ4_HOME}
+                      -DLZ4_INCLUDE_DIR=${LZ4_INCLUDE_DIR}
+                      -DSNAPPY_HOME=${SNAPPY_HOME}
+                      -DZLIB_HOME=${ZLIB_HOME})
+
+  ExternalProject_Add(orc_ep
+    GIT_REPOSITORY "https://github.com/apache/orc"
+    GIT_TAG ${ORC_VERSION}
+    BUILD_BYPRODUCTS ${ORC_STATIC_LIB}
+    CMAKE_ARGS ${ORC_CMAKE_ARGS})
+
+  include_directories(SYSTEM ${ORC_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(orc
+    STATIC_LIB ${ORC_STATIC_LIB})
+
+  add_dependencies(orc_ep protobuf lz4_static snappy zlib)
+  add_dependencies(orc orc_ep)
+endif()
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index d645cca224480..ad86256e0be34 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -90,6 +90,11 @@ if (ARROW_WITH_ZSTD)
   SET(ARROW_SRCS util/compression_zstd.cc ${ARROW_SRCS})
 endif()
 
+if (ARROW_ORC)
+  add_subdirectory(adapters/orc)
+  SET(ARROW_SRCS adapters/orc/adapter.cc ${ARROW_SRCS})
+endif()
+
 if (NOT ARROW_BOOST_HEADER_ONLY)
   set(ARROW_SRCS ${ARROW_SRCS}
     io/hdfs.cc
diff --git a/cpp/src/arrow/adapters/orc/CMakeLists.txt b/cpp/src/arrow/adapters/orc/CMakeLists.txt
new file mode 100644
index 0000000000000..eb7194cd42113
--- /dev/null
+++ b/cpp/src/arrow/adapters/orc/CMakeLists.txt
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######################################
+# arrow_orc
+#######################################
+
+# Headers: top level
+install(FILES
+        adapter.h
+        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/adapters/orc")
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
new file mode 100644
index 0000000000000..473c90f925124
--- /dev/null
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -0,0 +1,697 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/adapters/orc/adapter.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <list>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/io/interfaces.h"
+#include "arrow/memory_pool.h"
+#include "arrow/record_batch.h"
+#include "arrow/status.h"
+#include "arrow/table.h"
+#include "arrow/table_builder.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+#include "orc/OrcFile.hh"
+
+// alias to not interfere with nested orc namespace
+namespace liborc = orc;
+
+namespace arrow {
+namespace adapters {
+namespace orc {
+
+#define ORC_THROW_NOT_OK(s)                   \
+  do {                                        \
+    Status _s = (s);                          \
+    if (!_s.ok()) {                           \
+      std::stringstream ss;                   \
+      ss << "Arrow error: " << _s.ToString(); \
+      throw liborc::ParseError(ss.str());     \
+    }                                         \
+  } while (0)
+
+class ArrowInputFile : public liborc::InputStream {
+ public:
+  explicit ArrowInputFile(const std::shared_ptr<io::ReadableFileInterface>& file)
+      : file_(file) {}
+
+  uint64_t getLength() const override {
+    int64_t size;
+    ORC_THROW_NOT_OK(file_->GetSize(&size));
+    return static_cast<uint64_t>(size);
+  }
+
+  uint64_t getNaturalReadSize() const override { return 128 * 1024; }
+
+  void read(void* buf, uint64_t length, uint64_t offset) override {
+    int64_t bytes_read;
+
+    ORC_THROW_NOT_OK(file_->ReadAt(offset, length, &bytes_read, buf));
+
+    if (static_cast<uint64_t>(bytes_read) != length) {
+      throw liborc::ParseError("Short read from arrow input file");
+    }
+  }
+
+  const std::string& getName() const override {
+    static const std::string filename("ArrowInputFile");
+    return filename;
+  }
+
+ private:
+  std::shared_ptr<io::ReadableFileInterface> file_;
+};
+
+struct StripeInformation {
+  uint64_t offset;
+  uint64_t length;
+  uint64_t num_rows;
+};
+
+Status GetArrowType(const liborc::Type* type, std::shared_ptr<DataType>* out) {
+  // When subselecting fields on read, liborc will set some nodes to nullptr,
+  // so we need to check for nullptr before progressing
+  if (type == nullptr) {
+    *out = null();
+    return Status::OK();
+  }
+  liborc::TypeKind kind = type->getKind();
+  switch (kind) {
+    case liborc::BOOLEAN:
+      *out = boolean();
+      break;
+    case liborc::BYTE:
+      *out = int8();
+      break;
+    case liborc::SHORT:
+      *out = int16();
+      break;
+    case liborc::INT:
+      *out = int32();
+      break;
+    case liborc::LONG:
+      *out = int64();
+      break;
+    case liborc::FLOAT:
+      *out = float32();
+      break;
+    case liborc::DOUBLE:
+      *out = float64();
+      break;
+    case liborc::VARCHAR:
+    case liborc::STRING:
+      *out = utf8();
+      break;
+    case liborc::BINARY:
+      *out = binary();
+      break;
+    case liborc::CHAR:
+      *out = fixed_size_binary(type->getMaximumLength());
+      break;
+    case liborc::TIMESTAMP:
+      *out = timestamp(TimeUnit::NANO);
+      break;
+    case liborc::DATE:
+      *out = date32();
+      break;
+    case liborc::DECIMAL: {
+      if (type->getPrecision() == 0) {
+        // In HIVE 0.11/0.12 precision is set as 0, but means max precision
+        *out = decimal(38, 6);
+      } else {
+        *out = decimal(type->getPrecision(), type->getScale());
+      }
+      break;
+    }
+    case liborc::LIST: {
+      if (type->getSubtypeCount() != 1) {
+        return Status::Invalid("Invalid Orc List type");
+      }
+      std::shared_ptr<DataType> elemtype;
+      RETURN_NOT_OK(GetArrowType(type->getSubtype(0), &elemtype));
+      *out = list(elemtype);
+      break;
+    }
+    case liborc::MAP: {
+      if (type->getSubtypeCount() != 2) {
+        return Status::Invalid("Invalid Orc Map type");
+      }
+      std::shared_ptr<DataType> keytype;
+      std::shared_ptr<DataType> valtype;
+      RETURN_NOT_OK(GetArrowType(type->getSubtype(0), &keytype));
+      RETURN_NOT_OK(GetArrowType(type->getSubtype(1), &valtype));
+      *out = list(struct_({field("key", keytype), field("value", valtype)}));
+      break;
+    }
+    case liborc::STRUCT: {
+      int size = type->getSubtypeCount();
+      std::vector<std::shared_ptr<Field>> fields;
+      for (int child = 0; child < size; ++child) {
+        std::shared_ptr<DataType> elemtype;
+        RETURN_NOT_OK(GetArrowType(type->getSubtype(child), &elemtype));
+        std::string name = type->getFieldName(child);
+        fields.push_back(field(name, elemtype));
+      }
+      *out = struct_(fields);
+      break;
+    }
+    case liborc::UNION: {
+      int size = type->getSubtypeCount();
+      std::vector<std::shared_ptr<Field>> fields;
+      std::vector<uint8_t> type_codes;
+      for (int child = 0; child < size; ++child) {
+        std::shared_ptr<DataType> elemtype;
+        RETURN_NOT_OK(GetArrowType(type->getSubtype(child), &elemtype));
+        fields.push_back(field("_union_" + std::to_string(child), elemtype));
+        type_codes.push_back(static_cast<uint8_t>(child));
+      }
+      *out = union_(fields, type_codes);
+      break;
+    }
+    default: {
+      std::stringstream ss;
+      ss << "Unknown Orc type kind: " << kind;
+      return Status::Invalid(ss.str());
+    }
+  }
+  return Status::OK();
+}
+
+// The number of rows to read in a ColumnVectorBatch
+constexpr int64_t kReadRowsBatch = 1000;
+
+// The numer of nanoseconds in a second
+constexpr int64_t kOneSecondNanos = 1000000000LL;
+
+class ORCFileReader::Impl {
+ public:
+  Impl() {}
+  ~Impl() {}
+
+  Status Open(const std::shared_ptr<io::ReadableFileInterface>& file, MemoryPool* pool) {
+    std::unique_ptr<ArrowInputFile> io_wrapper(new ArrowInputFile(file));
+    liborc::ReaderOptions options;
+    std::unique_ptr<liborc::Reader> liborc_reader;
+    try {
+      liborc_reader = createReader(std::move(io_wrapper), options);
+    } catch (const liborc::ParseError& e) {
+      return Status::IOError(e.what());
+    }
+    pool_ = pool;
+    reader_ = std::move(liborc_reader);
+
+    return Init();
+  }
+
+  Status Init() {
+    int64_t nstripes = reader_->getNumberOfStripes();
+    stripes_.resize(nstripes);
+    std::unique_ptr<liborc::StripeInformation> stripe;
+    for (int i = 0; i < nstripes; ++i) {
+      stripe = reader_->getStripe(i);
+      stripes_[i] = StripeInformation(
+          {stripe->getOffset(), stripe->getLength(), stripe->getNumberOfRows()});
+    }
+    return Status::OK();
+  }
+
+  int64_t NumberOfStripes() { return stripes_.size(); }
+
+  int64_t NumberOfRows() { return reader_->getNumberOfRows(); }
+
+  Status ReadSchema(std::shared_ptr<Schema>* out) {
+    const liborc::Type& type = reader_->getType();
+    return GetArrowSchema(type, out);
+  }
+
+  Status GetArrowSchema(const liborc::Type& type, std::shared_ptr<Schema>* out) {
+    if (type.getKind() != liborc::STRUCT) {
+      return Status::NotImplemented(
+          "Only ORC files with a top-level struct "
+          "can be handled");
+    }
+    int size = type.getSubtypeCount();
+    std::vector<std::shared_ptr<Field>> fields;
+    for (int child = 0; child < size; ++child) {
+      std::shared_ptr<DataType> elemtype;
+      RETURN_NOT_OK(GetArrowType(type.getSubtype(child), &elemtype));
+      std::string name = type.getFieldName(child);
+      fields.push_back(field(name, elemtype));
+    }
+    std::list<std::string> keys = reader_->getMetadataKeys();
+    std::shared_ptr<KeyValueMetadata> metadata;
+    if (!keys.empty()) {
+      metadata = std::make_shared<KeyValueMetadata>();
+      for (auto it = keys.begin(); it != keys.end(); ++it) {
+        metadata->Append(*it, reader_->getMetadataValue(*it));
+      }
+    }
+
+    *out = std::make_shared<Schema>(fields, metadata);
+    return Status::OK();
+  }
+
+  Status Read(std::shared_ptr<Table>* out) {
+    liborc::RowReaderOptions opts;
+    return ReadTable(opts, out);
+  }
+
+  Status Read(const std::vector<int>& include_indices, std::shared_ptr<Table>* out) {
+    liborc::RowReaderOptions opts;
+    RETURN_NOT_OK(SelectIndices(&opts, include_indices));
+    return ReadTable(opts, out);
+  }
+
+  Status ReadStripe(int64_t stripe, std::shared_ptr<RecordBatch>* out) {
+    liborc::RowReaderOptions opts;
+    RETURN_NOT_OK(SelectStripe(&opts, stripe));
+    return ReadBatch(opts, stripes_[stripe].num_rows, out);
+  }
+
+  Status ReadStripe(int64_t stripe, const std::vector<int>& include_indices,
+                    std::shared_ptr<RecordBatch>* out) {
+    liborc::RowReaderOptions opts;
+    RETURN_NOT_OK(SelectIndices(&opts, include_indices));
+    RETURN_NOT_OK(SelectStripe(&opts, stripe));
+    return ReadBatch(opts, stripes_[stripe].num_rows, out);
+  }
+
+  Status SelectStripe(liborc::RowReaderOptions* opts, int64_t stripe) {
+    if (stripe < 0 || stripe >= NumberOfStripes()) {
+      std::stringstream ss;
+      ss << "Out of bounds stripe: " << stripe;
+      return Status::Invalid(ss.str());
+    }
+    opts->range(stripes_[stripe].offset, stripes_[stripe].length);
+    return Status::OK();
+  }
+
+  Status SelectIndices(liborc::RowReaderOptions* opts,
+                       const std::vector<int>& include_indices) {
+    std::list<uint64_t> include_indices_list;
+    for (auto it = include_indices.begin(); it != include_indices.end(); ++it) {
+      if (*it < 0) {
+        return Status::Invalid("Negative field index");
+      }
+      include_indices_list.push_back(*it);
+    }
+    opts->includeTypes(include_indices_list);
+    return Status::OK();
+  }
+
+  Status ReadTable(const liborc::RowReaderOptions& row_opts,
+                   std::shared_ptr<Table>* out) {
+    liborc::RowReaderOptions opts(row_opts);
+    std::vector<std::shared_ptr<RecordBatch>> batches(stripes_.size());
+    for (size_t stripe = 0; stripe < stripes_.size(); stripe++) {
+      opts.range(stripes_[stripe].offset, stripes_[stripe].length);
+      RETURN_NOT_OK(ReadBatch(opts, stripes_[stripe].num_rows, &batches[stripe]));
+    }
+    return Table::FromRecordBatches(batches, out);
+  }
+
+  Status ReadBatch(const liborc::RowReaderOptions& opts, int64_t nrows,
+                   std::shared_ptr<RecordBatch>* out) {
+    std::unique_ptr<liborc::RowReader> rowreader;
+    std::unique_ptr<liborc::ColumnVectorBatch> batch;
+    try {
+      rowreader = reader_->createRowReader(opts);
+      batch = rowreader->createRowBatch(std::min(nrows, kReadRowsBatch));
+    } catch (const liborc::ParseError& e) {
+      return Status::Invalid(e.what());
+    }
+    const liborc::Type& type = rowreader->getSelectedType();
+    std::shared_ptr<Schema> schema;
+    RETURN_NOT_OK(GetArrowSchema(type, &schema));
+
+    std::unique_ptr<RecordBatchBuilder> builder;
+    RETURN_NOT_OK(RecordBatchBuilder::Make(schema, pool_, nrows, &builder));
+
+    // The top-level type must be a struct to read into an arrow table
+    const auto& struct_batch = static_cast<liborc::StructVectorBatch&>(*batch);
+
+    while (rowreader->next(*batch)) {
+      for (int i = 0; i < builder->num_fields(); i++) {
+        RETURN_NOT_OK(AppendBatch(type.getSubtype(i), struct_batch.fields[i], 0,
+                                  batch->numElements, builder->GetField(i)));
+      }
+    }
+    RETURN_NOT_OK(builder->Flush(out));
+    return Status::OK();
+  }
+
+  Status AppendBatch(const liborc::Type* type, liborc::ColumnVectorBatch* batch,
+                     int64_t offset, int64_t length, ArrayBuilder* builder) {
+    if (type == nullptr) {
+      return Status::OK();
+    }
+    liborc::TypeKind kind = type->getKind();
+    switch (kind) {
+      case liborc::STRUCT:
+        return AppendStructBatch(type, batch, offset, length, builder);
+      case liborc::LIST:
+        return AppendListBatch(type, batch, offset, length, builder);
+      case liborc::MAP:
+        return AppendMapBatch(type, batch, offset, length, builder);
+      case liborc::LONG:
+        return AppendNumericBatch<Int64Builder, liborc::LongVectorBatch, int64_t>(
+            batch, offset, length, builder);
+      case liborc::INT:
+        return AppendNumericBatchCast<Int32Builder, int32_t, liborc::LongVectorBatch,
+                                      int64_t>(batch, offset, length, builder);
+      case liborc::SHORT:
+        return AppendNumericBatchCast<Int16Builder, int16_t, liborc::LongVectorBatch,
+                                      int64_t>(batch, offset, length, builder);
+      case liborc::BYTE:
+        return AppendNumericBatchCast<Int8Builder, int8_t, liborc::LongVectorBatch,
+                                      int64_t>(batch, offset, length, builder);
+      case liborc::DOUBLE:
+        return AppendNumericBatch<DoubleBuilder, liborc::DoubleVectorBatch, double>(
+            batch, offset, length, builder);
+      case liborc::FLOAT:
+        return AppendNumericBatchCast<FloatBuilder, float, liborc::DoubleVectorBatch,
+                                      double>(batch, offset, length, builder);
+      case liborc::BOOLEAN:
+        return AppendBoolBatch(batch, offset, length, builder);
+      case liborc::VARCHAR:
+      case liborc::STRING:
+        return AppendBinaryBatch<StringBuilder>(batch, offset, length, builder);
+      case liborc::BINARY:
+        return AppendBinaryBatch<BinaryBuilder>(batch, offset, length, builder);
+      case liborc::CHAR:
+        return AppendFixedBinaryBatch(batch, offset, length, builder);
+      case liborc::DATE:
+        return AppendNumericBatchCast<Date32Builder, int32_t, liborc::LongVectorBatch,
+                                      int64_t>(batch, offset, length, builder);
+      case liborc::TIMESTAMP:
+        return AppendTimestampBatch(batch, offset, length, builder);
+      case liborc::DECIMAL:
+        return AppendDecimalBatch(type, batch, offset, length, builder);
+      default:
+        std::stringstream ss;
+        ss << "Not implemented type kind: " << kind;
+        return Status::NotImplemented(ss.str());
+    }
+  }
+
+  Status AppendStructBatch(const liborc::Type* type, liborc::ColumnVectorBatch* cbatch,
+                           int64_t offset, int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<StructBuilder*>(abuilder);
+    auto batch = static_cast<liborc::StructVectorBatch*>(cbatch);
+
+    const uint8_t* valid_bytes = nullptr;
+    if (batch->hasNulls) {
+      valid_bytes = reinterpret_cast<const uint8_t*>(batch->notNull.data()) + offset;
+    }
+    RETURN_NOT_OK(builder->Append(length, valid_bytes));
+
+    for (int i = 0; i < builder->num_fields(); i++) {
+      RETURN_NOT_OK(AppendBatch(type->getSubtype(i), batch->fields[i], offset, length,
+                                builder->field_builder(i)));
+    }
+    return Status::OK();
+  }
+
+  Status AppendListBatch(const liborc::Type* type, liborc::ColumnVectorBatch* cbatch,
+                         int64_t offset, int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<ListBuilder*>(abuilder);
+    auto batch = static_cast<liborc::ListVectorBatch*>(cbatch);
+    liborc::ColumnVectorBatch* elements = batch->elements.get();
+    const liborc::Type* elemtype = type->getSubtype(0);
+
+    const bool has_nulls = batch->hasNulls;
+    for (int i = offset; i < length + offset; i++) {
+      if (!has_nulls || batch->notNull[i]) {
+        int64_t start = batch->offsets[i];
+        int64_t end = batch->offsets[i + 1];
+        RETURN_NOT_OK(builder->Append());
+        RETURN_NOT_OK(AppendBatch(elemtype, elements, start, end - start,
+                                  builder->value_builder()));
+      } else {
+        RETURN_NOT_OK(builder->AppendNull());
+      }
+    }
+    return Status::OK();
+  }
+
+  Status AppendMapBatch(const liborc::Type* type, liborc::ColumnVectorBatch* cbatch,
+                        int64_t offset, int64_t length, ArrayBuilder* abuilder) {
+    auto list_builder = static_cast<ListBuilder*>(abuilder);
+    auto struct_builder = static_cast<StructBuilder*>(list_builder->value_builder());
+    auto batch = static_cast<liborc::MapVectorBatch*>(cbatch);
+    liborc::ColumnVectorBatch* keys = batch->keys.get();
+    liborc::ColumnVectorBatch* vals = batch->elements.get();
+    const liborc::Type* keytype = type->getSubtype(0);
+    const liborc::Type* valtype = type->getSubtype(1);
+
+    const bool has_nulls = batch->hasNulls;
+    for (int i = offset; i < length + offset; i++) {
+      RETURN_NOT_OK(list_builder->Append());
+      int64_t start = batch->offsets[i];
+      int64_t list_length = batch->offsets[i + 1] - start;
+      if (list_length && (!has_nulls || batch->notNull[i])) {
+        RETURN_NOT_OK(struct_builder->Append(list_length, nullptr));
+        RETURN_NOT_OK(AppendBatch(keytype, keys, start, list_length,
+                                  struct_builder->field_builder(0)));
+        RETURN_NOT_OK(AppendBatch(valtype, vals, start, list_length,
+                                  struct_builder->field_builder(1)));
+      }
+    }
+    return Status::OK();
+  }
+
+  template <class builder_type, class batch_type, class elem_type>
+  Status AppendNumericBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset,
+                            int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<builder_type*>(abuilder);
+    auto batch = static_cast<batch_type*>(cbatch);
+
+    if (length == 0) {
+      return Status::OK();
+    }
+    const uint8_t* valid_bytes = nullptr;
+    if (batch->hasNulls) {
+      valid_bytes = reinterpret_cast<const uint8_t*>(batch->notNull.data()) + offset;
+    }
+    const elem_type* source = batch->data.data() + offset;
+    RETURN_NOT_OK(builder->Append(source, length, valid_bytes));
+    return Status::OK();
+  }
+
+  template <class builder_type, class target_type, class batch_type, class source_type>
+  Status AppendNumericBatchCast(liborc::ColumnVectorBatch* cbatch, int64_t offset,
+                                int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<builder_type*>(abuilder);
+    auto batch = static_cast<batch_type*>(cbatch);
+
+    if (length == 0) {
+      return Status::OK();
+    }
+    int start = builder->length();
+
+    const uint8_t* valid_bytes = nullptr;
+    if (batch->hasNulls) {
+      valid_bytes = reinterpret_cast<const uint8_t*>(batch->notNull.data()) + offset;
+    }
+    RETURN_NOT_OK(builder->AppendNulls(valid_bytes, length));
+
+    const source_type* source = batch->data.data() + offset;
+    target_type* target = reinterpret_cast<target_type*>(builder->data()->mutable_data());
+
+    std::copy(source, source + length, target + start);
+
+    return Status::OK();
+  }
+
+  Status AppendBoolBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset,
+                         int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<BooleanBuilder*>(abuilder);
+    auto batch = static_cast<liborc::LongVectorBatch*>(cbatch);
+
+    if (length == 0) {
+      return Status::OK();
+    }
+    int start = builder->length();
+
+    const uint8_t* valid_bytes = nullptr;
+    if (batch->hasNulls) {
+      valid_bytes = reinterpret_cast<const uint8_t*>(batch->notNull.data()) + offset;
+    }
+    RETURN_NOT_OK(builder->AppendNulls(valid_bytes, length));
+
+    const int64_t* source = batch->data.data() + offset;
+    uint8_t* target = reinterpret_cast<uint8_t*>(builder->data()->mutable_data());
+
+    for (int i = 0; i < length; i++) {
+      if (source[i]) {
+        BitUtil::SetBit(target, start + i);
+      } else {
+        BitUtil::ClearBit(target, start + i);
+      }
+    }
+    return Status::OK();
+  }
+
+  Status AppendTimestampBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset,
+                              int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<TimestampBuilder*>(abuilder);
+    auto batch = static_cast<liborc::TimestampVectorBatch*>(cbatch);
+
+    if (length == 0) {
+      return Status::OK();
+    }
+    int start = builder->length();
+
+    const uint8_t* valid_bytes = nullptr;
+    if (batch->hasNulls) {
+      valid_bytes = reinterpret_cast<const uint8_t*>(batch->notNull.data()) + offset;
+    }
+    RETURN_NOT_OK(builder->AppendNulls(valid_bytes, length));
+
+    const int64_t* seconds = batch->data.data() + offset;
+    const int64_t* nanos = batch->nanoseconds.data() + offset;
+    int64_t* target = reinterpret_cast<int64_t*>(builder->data()->mutable_data());
+
+    for (int i = 0; i < length; i++) {
+      // TODO: boundscheck this, as ORC supports higher resolution timestamps
+      // than arrow for nanosecond resolution
+      target[start + i] = seconds[i] * kOneSecondNanos + nanos[i];
+    }
+    return Status::OK();
+  }
+
+  template <class builder_type>
+  Status AppendBinaryBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset,
+                           int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<builder_type*>(abuilder);
+    auto batch = static_cast<liborc::StringVectorBatch*>(cbatch);
+
+    const bool has_nulls = batch->hasNulls;
+    for (int i = offset; i < length + offset; i++) {
+      if (!has_nulls || batch->notNull[i]) {
+        RETURN_NOT_OK(builder->Append(batch->data[i], batch->length[i]));
+      } else {
+        RETURN_NOT_OK(builder->AppendNull());
+      }
+    }
+    return Status::OK();
+  }
+
+  Status AppendFixedBinaryBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset,
+                                int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<FixedSizeBinaryBuilder*>(abuilder);
+    auto batch = static_cast<liborc::StringVectorBatch*>(cbatch);
+
+    const bool has_nulls = batch->hasNulls;
+    for (int i = offset; i < length + offset; i++) {
+      if (!has_nulls || batch->notNull[i]) {
+        RETURN_NOT_OK(builder->Append(batch->data[i]));
+      } else {
+        RETURN_NOT_OK(builder->AppendNull());
+      }
+    }
+    return Status::OK();
+  }
+
+  Status AppendDecimalBatch(const liborc::Type* type, liborc::ColumnVectorBatch* cbatch,
+                            int64_t offset, int64_t length, ArrayBuilder* abuilder) {
+    auto builder = static_cast<Decimal128Builder*>(abuilder);
+
+    const bool has_nulls = cbatch->hasNulls;
+    if (type->getPrecision() == 0 || type->getPrecision() > 18) {
+      auto batch = static_cast<liborc::Decimal128VectorBatch*>(cbatch);
+      for (int i = offset; i < length + offset; i++) {
+        if (!has_nulls || batch->notNull[i]) {
+          RETURN_NOT_OK(builder->Append(
+              Decimal128(batch->values[i].getHighBits(), batch->values[i].getLowBits())));
+        } else {
+          RETURN_NOT_OK(builder->AppendNull());
+        }
+      }
+    } else {
+      auto batch = static_cast<liborc::Decimal64VectorBatch*>(cbatch);
+      for (int i = offset; i < length + offset; i++) {
+        if (!has_nulls || batch->notNull[i]) {
+          RETURN_NOT_OK(builder->Append(Decimal128(batch->values[i])));
+        } else {
+          RETURN_NOT_OK(builder->AppendNull());
+        }
+      }
+    }
+    return Status::OK();
+  }
+
+ private:
+  MemoryPool* pool_;
+  std::unique_ptr<liborc::Reader> reader_;
+  std::vector<StripeInformation> stripes_;
+};
+
+ORCFileReader::ORCFileReader() { impl_.reset(new ORCFileReader::Impl()); }
+
+ORCFileReader::~ORCFileReader() {}
+
+Status ORCFileReader::Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+                           MemoryPool* pool, std::unique_ptr<ORCFileReader>* reader) {
+  auto result = std::unique_ptr<ORCFileReader>(new ORCFileReader());
+  RETURN_NOT_OK(result->impl_->Open(file, pool));
+  *reader = std::move(result);
+  return Status::OK();
+}
+
+Status ORCFileReader::ReadSchema(std::shared_ptr<Schema>* out) {
+  return impl_->ReadSchema(out);
+}
+
+Status ORCFileReader::Read(std::shared_ptr<Table>* out) { return impl_->Read(out); }
+
+Status ORCFileReader::Read(const std::vector<int>& include_indices,
+                           std::shared_ptr<Table>* out) {
+  return impl_->Read(include_indices, out);
+}
+
+Status ORCFileReader::ReadStripe(int64_t stripe, std::shared_ptr<RecordBatch>* out) {
+  return impl_->ReadStripe(stripe, out);
+}
+
+Status ORCFileReader::ReadStripe(int64_t stripe, const std::vector<int>& include_indices,
+                                 std::shared_ptr<RecordBatch>* out) {
+  return impl_->ReadStripe(stripe, include_indices, out);
+}
+
+int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); }
+
+int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); }
+
+}  // namespace orc
+}  // namespace adapters
+}  // namespace arrow
diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h
new file mode 100644
index 0000000000000..6438658fd45bb
--- /dev/null
+++ b/cpp/src/arrow/adapters/orc/adapter.h
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_ORC_CONVERTER_H
+#define ARROW_ORC_CONVERTER_H
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "arrow/io/interfaces.h"
+#include "arrow/memory_pool.h"
+#include "arrow/record_batch.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+namespace adapters {
+
+namespace orc {
+
+/// \class ORCFileReader
+/// \brief Read an Arrow Table or RecordBatch from an ORC file.
+class ARROW_EXPORT ORCFileReader {
+ public:
+  ~ORCFileReader();
+
+  /// \brief Create a new ORC reader
+  ///
+  /// \param[in] file the data source
+  /// \param[in] pool a MemoryPool to use for buffer allocations
+  /// \param[out] reader the returned reader object
+  /// \return Status
+  static Status Open(const std::shared_ptr<io::ReadableFileInterface>& file,
+                     MemoryPool* pool, std::unique_ptr<ORCFileReader>* reader);
+
+  /// \brief Return the schema read from the ORC file
+  ///
+  /// \param[out] out the returned Schema object
+  Status ReadSchema(std::shared_ptr<Schema>* out);
+
+  /// \brief Read the file as a Table
+  ///
+  /// The table will be composed of one record batch per stripe.
+  ///
+  /// \param[out] out the returned RecordBatch
+  Status Read(std::shared_ptr<Table>* out);
+
+  /// \brief Read the file as a Table
+  ///
+  /// The table will be composed of one record batch per stripe.
+  ///
+  /// \param[in] include_indices the selected field indices to read
+  /// \param[out] out the returned RecordBatch
+  Status Read(const std::vector<int>& include_indices, std::shared_ptr<Table>* out);
+
+  /// \brief Read a single stripe as a RecordBatch
+  ///
+  /// \param[in] stripe the stripe index
+  /// \param[out] out the returned RecordBatch
+  Status ReadStripe(int64_t stripe, std::shared_ptr<RecordBatch>* out);
+
+  /// \brief Read a single stripe as a RecordBatch
+  ///
+  /// \param[in] stripe the stripe index
+  /// \param[in] include_indices the selected field indices to read
+  /// \param[out] out the returned RecordBatch
+  Status ReadStripe(int64_t stripe, const std::vector<int>& include_indices,
+                    std::shared_ptr<RecordBatch>* out);
+
+  /// \brief The number of stripes in the file
+  int64_t NumberOfStripes();
+
+  /// \brief The number of rows in the file
+  int64_t NumberOfRows();
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+  ORCFileReader();
+};
+
+}  // namespace orc
+
+}  // namespace adapters
+
+}  // namespace arrow
+
+#endif  // ARROW_ORC_CONVERTER_H
diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map
index f216d865001d2..c5d23793ccbae 100644
--- a/cpp/src/arrow/symbols.map
+++ b/cpp/src/arrow/symbols.map
@@ -55,6 +55,8 @@
     ERR_getErrorString;
     # jemalloc
     je_arrow_*;
+    # ORC destructors
+    _ZThn8_N3orc*;
 
     extern "C++" {
       # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
@@ -65,6 +67,8 @@
 
       # Statically linked C++ dependencies
       boost::*;
+      google::*;
+      orc::*;
       snappy::*;
     };
 };
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index cbbb464d05a76..e9de08ba19730 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -70,6 +70,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUILD_PLASMA
     "Build the PyArrow Plasma integration"
     OFF)
+  option(PYARROW_BUILD_ORC
+    "Build the PyArrow ORC integration"
+    OFF)
   option(PYARROW_BUNDLE_ARROW_CPP
     "Bundle the Arrow C++ libraries"
     OFF)
@@ -357,6 +360,14 @@ if (PYARROW_BUILD_PLASMA)
   file(COPY ${PLASMA_EXECUTABLE} DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY})
 endif()
 
+
+if (PYARROW_BUILD_ORC)
+  ## ORC
+  set(CYTHON_EXTENSIONS
+      ${CYTHON_EXTENSIONS}
+      _orc)
+endif()
+
 ############################################################
 # Setup and build Cython modules
 ############################################################
diff --git a/python/manylinux1/scripts/check_arrow_visibility.sh b/python/manylinux1/scripts/check_arrow_visibility.sh
index 27a30f7479bb6..bed357edf664a 100755
--- a/python/manylinux1/scripts/check_arrow_visibility.sh
+++ b/python/manylinux1/scripts/check_arrow_visibility.sh
@@ -17,10 +17,13 @@
 # under the License.
 
 nm -D -C /arrow-dist/lib64/libarrow.so > nm_arrow.log
+grep ' T ' nm_arrow.log | grep -v arrow > visible_symbols.log
 
-if [[ `grep ' T ' nm_arrow.log | grep -v arrow | wc -l` -eq 2 ]]
+if [[ `cat visible_symbols.log | wc -l` -eq 2 ]]
 then
     exit 0
 fi
 
+cat visible_symbols.log
+
 exit 1
diff --git a/python/pyarrow/_orc.pxd b/python/pyarrow/_orc.pxd
new file mode 100644
index 0000000000000..411691510423c
--- /dev/null
+++ b/python/pyarrow/_orc.pxd
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from libc.string cimport const_char
+from libcpp.vector cimport vector as std_vector
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
+                                        CTable, CMemoryPool,
+                                        CKeyValueMetadata,
+                                        CRecordBatch,
+                                        CTable,
+                                        RandomAccessFile, OutputStream,
+                                        TimeUnit)
+
+
+cdef extern from "arrow/adapters/orc/adapter.h" namespace "arrow::adapters::orc" nogil:
+    cdef cppclass ORCFileReader:
+
+        @staticmethod
+        CStatus Open(const shared_ptr[RandomAccessFile]& file,
+                     CMemoryPool* pool,
+                     unique_ptr[ORCFileReader]* reader)
+
+        CStatus ReadSchema(shared_ptr[CSchema]* out)
+
+        CStatus ReadStripe(int64_t stripe, shared_ptr[CRecordBatch]* out)
+        CStatus ReadStripe(int64_t stripe, std_vector[int], shared_ptr[CRecordBatch]* out)
+
+        CStatus Read(shared_ptr[CTable]* out)
+        CStatus Read(std_vector[int], shared_ptr[CTable]* out)
+
+        int64_t NumberOfStripes()
+
+        int64_t NumberOfRows()
diff --git a/python/pyarrow/_orc.pyx b/python/pyarrow/_orc.pyx
new file mode 100644
index 0000000000000..7ff4bac6dc95f
--- /dev/null
+++ b/python/pyarrow/_orc.pyx
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from cython.operator cimport dereference as deref
+from libcpp.vector cimport vector as std_vector
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport *
+from pyarrow.lib cimport (check_status,
+                          MemoryPool, maybe_unbox_memory_pool,
+                          Schema, pyarrow_wrap_schema,
+                          RecordBatch,
+                          pyarrow_wrap_table,
+                          get_reader)
+import six
+
+
+cdef class ORCReader:
+    cdef:
+        object source
+        CMemoryPool* allocator
+        unique_ptr[ORCFileReader] reader
+
+    def __cinit__(self, MemoryPool memory_pool=None):
+        self.allocator = maybe_unbox_memory_pool(memory_pool)
+
+    def open(self, object source):
+        cdef:
+            shared_ptr[RandomAccessFile] rd_handle
+
+        self.source = source
+
+        get_reader(source, &rd_handle)
+        with nogil:
+            check_status(ORCFileReader.Open(rd_handle, self.allocator,
+                                             &self.reader))
+
+    def schema(self):
+        """
+        The arrow schema for this file.
+
+        Returns
+        -------
+        schema : pyarrow.Schema
+        """
+        cdef:
+            shared_ptr[CSchema] sp_arrow_schema
+
+        with nogil:
+            check_status(deref(self.reader).ReadSchema(&sp_arrow_schema))
+
+        return pyarrow_wrap_schema(sp_arrow_schema)
+
+    def nrows(self):
+        return deref(self.reader).NumberOfRows();
+
+    def nstripes(self):
+        return deref(self.reader).NumberOfStripes();
+
+    def read_stripe(self, n, include_indices=None):
+        cdef:
+            shared_ptr[CRecordBatch] sp_record_batch
+            RecordBatch batch
+            int64_t stripe
+            std_vector[int] indices
+
+        stripe = n
+
+        if include_indices is None:
+            with nogil:
+                check_status(deref(self.reader).ReadStripe(stripe, &sp_record_batch))
+        else:
+            indices = include_indices
+            with nogil:
+                check_status(deref(self.reader).ReadStripe(stripe, indices, &sp_record_batch))
+
+        batch = RecordBatch()
+        batch.init(sp_record_batch)
+        return batch
+
+    def read(self, include_indices=None):
+        cdef:
+            shared_ptr[CTable] sp_table
+            std_vector[int] indices
+
+        if include_indices is None:
+            with nogil:
+                check_status(deref(self.reader).Read(&sp_table))
+        else:
+            indices = include_indices
+            with nogil:
+                check_status(deref(self.reader).Read(indices, &sp_table))
+
+        return pyarrow_wrap_table(sp_table)
diff --git a/python/pyarrow/orc.py b/python/pyarrow/orc.py
new file mode 100644
index 0000000000000..22451d521825e
--- /dev/null
+++ b/python/pyarrow/orc.py
@@ -0,0 +1,149 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from itertools import count
+from numbers import Integral
+
+from pyarrow import _orc
+from pyarrow import types
+from pyarrow.lib import Schema
+
+
+def _is_map(typ):
+    return (types.is_list(typ) and
+            types.is_struct(typ.value_type) and
+            typ.value_type.num_children == 2 and
+            typ.value_type[0].name == 'key' and
+            typ.value_type[1].name == 'value')
+
+
+def _traverse(typ, counter):
+    if isinstance(typ, Schema) or types.is_struct(typ):
+        for field in typ:
+            path = (field.name,)
+            yield path, next(counter)
+            for sub, c in _traverse(field.type, counter):
+                yield path + sub, c
+    elif _is_map(typ):
+        for sub_c in _traverse(typ.value_type, counter):
+            yield sub_c
+    elif types.is_list(typ):
+        # Skip one index for list type, since this can never be selected
+        # directly
+        next(counter)
+        for sub_c in _traverse(typ.value_type, counter):
+            yield sub_c
+    elif types.is_union(typ):
+        # Union types not supported, just skip the indexes
+        for dtype in typ:
+            next(counter)
+            for sub_c in _traverse(dtype, counter):
+                pass
+
+
+def _schema_to_indices(schema):
+    return {'.'.join(i): c for i, c in _traverse(schema, count(1))}
+
+
+class ORCFile(object):
+    """
+    Reader interface for a single ORC file
+
+    Parameters
+    ----------
+    source : str or pyarrow.io.NativeFile
+        Readable source. For passing Python file objects or byte buffers,
+        see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
+    """
+    def __init__(self, source):
+        self.reader = _orc.ORCReader()
+        self.reader.open(source)
+        self._column_index_lookup = _schema_to_indices(self.schema)
+
+    @property
+    def schema(self):
+        """The file schema, as an arrow schema"""
+        return self.reader.schema()
+
+    @property
+    def nrows(self):
+        """The number of rows in the file"""
+        return self.reader.nrows()
+
+    @property
+    def nstripes(self):
+        """The number of stripes in the file"""
+        return self.reader.nstripes()
+
+    def _select_indices(self, columns=None):
+        if columns is None:
+            return None
+
+        schema = self.schema
+        indices = []
+        for col in columns:
+            if isinstance(col, Integral):
+                col = int(col)
+                if 0 <= col < len(schema):
+                    col = schema[col].name
+                else:
+                    raise ValueError("Column indices must be in 0 <= ind < %d,"
+                                     " got %d" % (len(schema), col))
+            if col in self._column_index_lookup:
+                indices.append(self._column_index_lookup[col])
+            else:
+                raise ValueError("Unknown column name %r" % col)
+
+        return indices
+
+    def read_stripe(self, n, columns=None):
+        """Read a single stripe from the file.
+
+        Parameters
+        ----------
+        n : int
+            The stripe index
+        columns : list
+            If not None, only these columns will be read from the stripe. A
+            column name may be a prefix of a nested field, e.g. 'a' will select
+            'a.b', 'a.c', and 'a.d.e'
+
+        Returns
+        -------
+        pyarrow.lib.RecordBatch
+            Content of the stripe as a RecordBatch.
+        """
+        include_indices = self._select_indices(columns)
+        return self.reader.read_stripe(n, include_indices=include_indices)
+
+    def read(self, columns=None):
+        """Read the whole file.
+
+        Parameters
+        ----------
+        columns : list
+            If not None, only these columns will be read from the file. A
+            column name may be a prefix of a nested field, e.g. 'a' will select
+            'a.b', 'a.c', and 'a.d.e'
+
+        Returns
+        -------
+        pyarrow.lib.Table
+            Content of the file as a Table.
+        """
+        include_indices = self._select_indices(columns)
+        return self.reader.read(include_indices=include_indices)
diff --git a/python/setup.py b/python/setup.py
index 32e76ab05d836..3d3831dc048c6 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -84,6 +84,7 @@ def run(self):
                      ('with-parquet', None, 'build the Parquet extension'),
                      ('with-static-parquet', None, 'link parquet statically'),
                      ('with-plasma', None, 'build the Plasma extension'),
+                     ('with-orc', None, 'build the ORC extension'),
                      ('bundle-arrow-cpp', None,
                       'bundle the Arrow C++ libraries')] +
                     _build_ext.user_options)
@@ -109,12 +110,15 @@ def initialize_options(self):
             os.environ.get('PYARROW_WITH_STATIC_BOOST', '1'))
         self.with_plasma = strtobool(
             os.environ.get('PYARROW_WITH_PLASMA', '0'))
+        self.with_orc = strtobool(
+            os.environ.get('PYARROW_WITH_ORC', '0'))
         self.bundle_arrow_cpp = strtobool(
             os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
 
     CYTHON_MODULE_NAMES = [
         'lib',
         '_parquet',
+        '_orc',
         'plasma']
 
     def _run_cmake(self):
@@ -157,6 +161,9 @@ def _run_cmake(self):
         if self.with_plasma:
             cmake_options.append('-DPYARROW_BUILD_PLASMA=on')
 
+        if self.with_orc:
+            cmake_options.append('-DPYARROW_BUILD_ORC=on')
+
         if len(self.cmake_cxxflags) > 0:
             cmake_options.append('-DPYARROW_CXXFLAGS="{0}"'
                                  .format(self.cmake_cxxflags))
@@ -284,6 +291,8 @@ def _failure_permitted(self, name):
             return True
         if name == 'plasma' and not self.with_plasma:
             return True
+        if name == '_orc' and not self.with_orc:
+            return True
         return False
 
     def _get_inplace_dir(self):

From cb9c52e4b1d81446ddcfaf4485a8284006dc68d0 Mon Sep 17 00:00:00 2001
From: Adam Seibert <seibs@users.noreply.github.com>
Date: Thu, 4 Jan 2018 20:38:32 -0500
Subject: [PATCH 1364/1644] ARROW-1909: [C++] Enables building with benchmarks
 on windows

These changes were necessary to compile on Windows with "-DARROW_BUILD_BENCHMARKS=ON".  I added Shwlapi based on https://github.com/google/benchmark/issues/202.

Author: Adam Seibert <seibs@users.noreply.github.com>

Closes #1406 from seibs/ARROW-1909 and squashes the following commits:

98602cd0 [Adam Seibert] ARROW-1909: [C++] Enables building with benchmarks on windows
---
 cpp/src/arrow/compute/compute-benchmark.cc | 2 +-
 cpp/src/arrow/util/CMakeLists.txt          | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/compute/compute-benchmark.cc b/cpp/src/arrow/compute/compute-benchmark.cc
index aa7d899c864e7..44df441394ad4 100644
--- a/cpp/src/arrow/compute/compute-benchmark.cc
+++ b/cpp/src/arrow/compute/compute-benchmark.cc
@@ -191,7 +191,7 @@ static void BM_UniqueString100bytes(benchmark::State& state) {
 BENCHMARK(BM_BuildDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildStringDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
 
-constexpr int64_t kHashBenchmarkLength = 1 << 24;
+constexpr int kHashBenchmarkLength = 1 << 24;
 
 #define ADD_HASH_ARGS(WHAT)                        \
   WHAT->Args({kHashBenchmarkLength, 50})           \
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 42613d6a5f56b..a36dffb52e294 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -57,6 +57,11 @@ if (ARROW_BUILD_BENCHMARKS)
     target_link_libraries(arrow_benchmark_main
       benchmark
     )
+  elseif(MSVC)
+    target_link_libraries(arrow_benchmark_main
+      benchmark
+      Shlwapi.lib
+  )
   else()
 	  target_link_libraries(arrow_benchmark_main
       benchmark

From 0115e93c9d891a1973f023645435866eb77cb420 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Thu, 4 Jan 2018 22:33:04 -0500
Subject: [PATCH 1365/1644] ARROW-1919: [Plasma] Test that object ids are 20
 bytes

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1421 from pcmoritz/plasma-object-ids and squashes the following commits:

fc779087 [Philipp Moritz] fixes
9f613c0d [Philipp Moritz] fix windows test
f1d7ca05 [Philipp Moritz] fix linting
6be7f4a9 [Philipp Moritz] Test that object ids are 20 bytes
---
 python/pyarrow/plasma.pyx           |  3 +++
 python/pyarrow/tests/test_plasma.py | 10 +++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index abeec32783398..29e233b6e4e67 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -136,6 +136,9 @@ cdef class ObjectID:
         CUniqueID data
 
     def __cinit__(self, object_id):
+        if not isinstance(object_id, bytes) or len(object_id) != 20:
+            raise ValueError("Object ID must by 20 bytes,"
+                             " is " + str(object_id))
         self.data = CUniqueID.from_binary(object_id)
 
     def __richcmp__(ObjectID self, ObjectID object_id, operation):
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index decdc73d911b4..9ea6476670441 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -362,7 +362,7 @@ def test_store_pandas_dataframe(self):
         # Read the DataFrame.
         [data] = self.plasma_client.get_buffers([object_id])
         reader = pa.RecordBatchStreamReader(pa.BufferReader(data))
-        result = reader.get_next_batch().to_pandas()
+        result = reader.read_next_batch().to_pandas()
 
         pd.util.testing.assert_frame_equal(df, result)
 
@@ -747,3 +747,11 @@ def test_use_one_memory_mapped_file(self):
         with pytest.raises(pa.lib.PlasmaStoreFull):
             create_object(self.plasma_client, DEFAULT_PLASMA_STORE_MEMORY + 1,
                           0)
+
+
+@pytest.mark.plasma
+def test_object_id_size():
+    import pyarrow.plasma as plasma
+    with pytest.raises(ValueError):
+        plasma.ObjectID("hello")
+    plasma.ObjectID(20 * b"0")

From 4dbce607d50031a405af39d36e08cd03c5ffc764 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Fri, 5 Jan 2018 13:08:59 -0800
Subject: [PATCH 1366/1644] ARROW-1962: [Java] Adding reset to ValueVector
 interface

Adding `reset()` to the ValueVector interface and implementing where it is not done already.  Removing unused abstract class BaseDataValueVector that is not used anymore by the UnionVector.

Expanded reset tests to check that valueCount is 0, and buffers have same capacity and zeroed out.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1455 from BryanCutler/java-reset-ValueVector-ARROW-1962 and squashes the following commits:

da994e1 [Bryan Cutler] typo
a52e7db [Bryan Cutler] expanded reset documentations
1526a83 [Bryan Cutler] improved vector reset testing
a251d10 [Bryan Cutler] reset should zero data buffer and set value count to 0
bf2a16a [Bryan Cutler] add reset to NullableMapVector to zero validity buffer
7fbde5b [Bryan Cutler] need to zero out vector buffers when reset
b59addf [Bryan Cutler] adding reset to ValueVector interface, removing BaseDataValueVector
---
 .../main/codegen/templates/UnionVector.java   |   8 +-
 .../arrow/vector/BaseDataValueVector.java     | 129 ------------------
 .../arrow/vector/BaseFixedWidthVector.java    |   2 +
 .../arrow/vector/BaseVariableWidthVector.java |   2 +
 .../org/apache/arrow/vector/ValueVector.java  |  11 +-
 .../org/apache/arrow/vector/ZeroVector.java   |   4 +
 .../complex/BaseRepeatedValueVector.java      |   7 +
 .../vector/complex/FixedSizeListVector.java   |   7 +
 .../arrow/vector/complex/ListVector.java      |   7 +
 .../arrow/vector/complex/MapVector.java       |   8 ++
 .../vector/complex/NullableMapVector.java     |  13 +-
 .../apache/arrow/vector/TestVectorReset.java  | 109 ++++++++++++++-
 12 files changed, 165 insertions(+), 142 deletions(-)
 delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 9377bd0da211d..aa8178a92f80a 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -29,7 +29,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
-import org.apache.arrow.vector.BaseDataValueVector;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
@@ -318,6 +317,13 @@ public void clear() {
     internalMap.clear();
   }
 
+  @Override
+  public void reset() {
+    valueCount = 0;
+    typeBuffer.setZero(0, typeBuffer.capacity());
+    internalMap.reset();
+  }
+
   @Override
   public Field getField() {
     List<org.apache.arrow.vector.types.pojo.Field> childFields = new ArrayList<>();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
deleted file mode 100644
index 80675135e3c1c..0000000000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
-
-import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.TransferPair;
-
-
-public abstract class BaseDataValueVector extends BaseValueVector implements BufferBacked {
-
-  public static void load(ArrowFieldNode fieldNode, List<BufferBacked> vectors, List<ArrowBuf> buffers) {
-    int expectedSize = vectors.size();
-    if (buffers.size() != expectedSize) {
-      throw new IllegalArgumentException("Illegal buffer count, expected " + expectedSize + ", got: " + buffers.size());
-    }
-    for (int i = 0; i < expectedSize; i++) {
-      vectors.get(i).load(fieldNode, buffers.get(i));
-    }
-  }
-
-  public static void truncateBufferBasedOnSize(List<ArrowBuf> buffers, int bufferIndex, int byteSize) {
-    if (bufferIndex >= buffers.size()) {
-      throw new IllegalArgumentException("no buffer at index " + bufferIndex + ": " + buffers);
-    }
-    ArrowBuf buffer = buffers.get(bufferIndex);
-    if (buffer.writerIndex() < byteSize) {
-      throw new IllegalArgumentException("can not truncate buffer to a larger size " + byteSize + ": " + buffer.writerIndex());
-    }
-    buffer.writerIndex(byteSize);
-  }
-
-  public static List<ArrowBuf> unload(List<BufferBacked> vectors) {
-    List<ArrowBuf> result = new ArrayList<>(vectors.size());
-    for (BufferBacked vector : vectors) {
-      result.add(vector.unLoad());
-    }
-    return result;
-  }
-
-  protected ArrowBuf data;
-
-  public BaseDataValueVector(String name, BufferAllocator allocator) {
-    super(name, allocator);
-    data = allocator.getEmpty();
-  }
-
-  @Override
-  public void clear() {
-    data.release();
-    data = allocator.getEmpty();
-    super.clear();
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return getTransferPair(ref, allocator);
-  }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    ArrowBuf[] out;
-    if (getBufferSize() == 0) {
-      out = new ArrowBuf[0];
-    } else {
-      out = new ArrowBuf[]{data};
-      data.readerIndex(0);
-      if (clear) {
-        data.retain(1);
-      }
-    }
-    if (clear) {
-      clear();
-    }
-    return out;
-  }
-
-  @Override
-  public int getBufferSize() {
-    if (getValueCount() == 0) {
-      return 0;
-    }
-    return data.writerIndex();
-  }
-
-  public ArrowBuf getBuffer() {
-    return data;
-  }
-
-  @Override
-  public void load(ArrowFieldNode fieldNode, ArrowBuf data) {
-    this.data.release();
-    this.data = data.retain(allocator);
-  }
-
-  @Override
-  public ArrowBuf unLoad() {
-    return this.data.readerIndex(0);
-  }
-
-  /**
-   * This method has a similar effect of allocateNew() without actually clearing and reallocating
-   * the value vector. The purpose is to move the value vector to a "mutate" state
-   */
-  public void reset() {
-  }
-}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index 77026d4e5a112..702db9f528152 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -208,7 +208,9 @@ private void initValueBuffer() {
    * Reset the vector to initial state. Same as {@link #zeroVector()}.
    * Note that this method doesn't release any memory.
    */
+  @Override
   public void reset() {
+    valueCount = 0;
     zeroVector();
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index fbadb35d7556a..fff329a9b9d66 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -193,6 +193,7 @@ private int getOffsetBufferValueCapacity() {
   public void zeroVector() {
     initValidityBuffer();
     initOffsetBuffer();
+    valueBuffer.setZero(0, valueBuffer.capacity());
   }
 
   /* zero out the validity buffer */
@@ -212,6 +213,7 @@ private void initOffsetBuffer() {
   public void reset() {
     zeroVector();
     lastSet = -1;
+    valueCount = 0;
   }
 
   /**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index e77c1b12703ec..24cf59a0da81d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -100,10 +100,19 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
   void close();
 
   /**
-   * Release the underlying ArrowBuf and reset the ValueVector to empty.
+   * Release any owned ArrowBuf and reset the ValueVector to the initial state. If the
+   * vector has any child vectors, they will also be cleared.
    */
   void clear();
 
+  /**
+   * Reset the ValueVector to the initial state without releasing any owned ArrowBuf.
+   * Buffer capacities will remain unchanged and any previous data will be zeroed out.
+   * This includes buffers for data, validity, offset, etc. If the vector has any
+   * child vectors, they will also be reset.
+   */
+  void reset();
+
   /**
    * Get information about how this field is materialized.
    *
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 962a1c947071f..2d3c543d03ffa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -72,6 +72,10 @@ public void close() {
   public void clear() {
   }
 
+  @Override
+  public void reset() {
+  }
+
   @Override
   public Field getField() {
     return new Field(DATA_VECTOR_NAME, FieldType.nullable(new Null()), null);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 9a23fd8c356cb..d0a664ac01da2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -186,6 +186,13 @@ public void clear() {
     super.clear();
   }
 
+  @Override
+  public void reset() {
+    offsetBuffer.setZero(0, offsetBuffer.capacity());
+    vector.reset();
+    valueCount = 0;
+  }
+
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
     final ArrowBuf[] buffers;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 93a8127cfd3b4..9314a2566b124 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -271,6 +271,13 @@ public void clear() {
     super.clear();
   }
 
+  @Override
+  public void reset() {
+    validityBuffer.setZero(0, validityBuffer.capacity());
+    vector.reset();
+    valueCount = 0;
+  }
+
   @Override
   public ArrowBuf[] getBuffers(boolean clear) {
     setReaderAndWriterIndex();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 0c1daf4905222..8aeeb7e5a2886 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -513,6 +513,13 @@ public void clear() {
     lastSet = 0;
   }
 
+  @Override
+  public void reset() {
+    super.reset();
+    validityBuffer.setZero(0, validityBuffer.capacity());
+    lastSet = 0;
+  }
+
   /**
    * Return the underlying buffers associated with this vector. Note that this doesn't
    * impact the reference counts for this buffer so it only should be used for in-context
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index e13084588ce0b..6eab6ef3dcdcc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -302,6 +302,14 @@ public void clear() {
     valueCount = 0;
   }
 
+  @Override
+  public void reset() {
+    for (final ValueVector v : getChildren()) {
+      v.reset();
+    }
+    valueCount = 0;
+  }
+
   @Override
   public Field getField() {
     List<Field> children = new ArrayList<>();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index d887b73064643..fb84d23b5074e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -305,6 +305,15 @@ public void clear() {
     super.clear();
   }
 
+  /**
+   * Reset this vector to empty, does not release buffers
+   */
+  @Override
+  public void reset() {
+    super.reset();
+    validityBuffer.setZero(0, validityBuffer.capacity());
+  }
+
   /**
    * Release the validity buffer
    */
@@ -493,8 +502,4 @@ public void setValueCount(int valueCount) {
     super.setValueCount(valueCount);
     this.valueCount = valueCount;
   }
-
-  public void reset() {
-    valueCount = 0;
-  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
index 28903b1170e31..84ea9657f8c9f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
@@ -19,13 +19,23 @@
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.*;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.FieldType;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.nio.charset.StandardCharsets;
+
 public class TestVectorReset {
 
   private BufferAllocator allocator;
@@ -40,15 +50,100 @@ public void terminate() throws Exception {
     allocator.close();
   }
 
+  private void resetVectorAndVerify(ValueVector vector, ArrowBuf[] bufs) {
+    int[] sizeBefore = new int[bufs.length];
+    for (int i = 0; i < bufs.length; i++) {
+      sizeBefore[i] = bufs[i].capacity();
+    }
+    vector.reset();
+    for (int i = 0; i < bufs.length; i++) {
+      assertEquals(sizeBefore[i], bufs[i].capacity());
+      verifyBufferZeroed(bufs[i]);
+    }
+    assertEquals(0, vector.getValueCount());
+  }
+
+  private void verifyBufferZeroed(ArrowBuf buf) {
+    for (int i = 0; i < buf.capacity(); i++) {
+      assertTrue((byte) 0 == buf.getByte(i));
+    }
+  }
+
   @Test
   public void testFixedTypeReset() {
-    try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
-      vector.allocateNew();
-      final int sizeBefore = vector.getBufferSize();
-      vector.reAlloc();
-      vector.reset();
-      final int sizeAfter = vector.getBufferSize();
-      assertEquals(sizeBefore, sizeAfter);
+    try (final UInt4Vector vector = new UInt4Vector("UInt4", allocator)) {
+      vector.allocateNewSafe();
+      vector.setNull(0);
+      vector.setValueCount(1);
+      resetVectorAndVerify(vector, vector.getBuffers(false));
+    }
+  }
+
+  @Test
+  public void testVariableTypeReset() {
+    try (final VarCharVector vector = new VarCharVector("VarChar", allocator)) {
+      vector.allocateNewSafe();
+      vector.set(0, "a".getBytes(StandardCharsets.UTF_8));
+      vector.setLastSet(0);
+      vector.setValueCount(1);
+      resetVectorAndVerify(vector, vector.getBuffers(false));
+      assertEquals(-1, vector.getLastSet());
+    }
+  }
+
+  @Test
+  public void testListTypeReset() {
+    try (final ListVector variableList = new ListVector("VarList", allocator, FieldType.nullable(MinorType.INT.getType()), null);
+         final FixedSizeListVector fixedList = new FixedSizeListVector("FixedList", allocator, FieldType.nullable(new FixedSizeList(2)), null)
+    ) {
+      // ListVector
+      variableList.allocateNewSafe();
+      variableList.startNewValue(0);
+      variableList.endValue(0, 0);
+      variableList.setValueCount(1);
+      resetVectorAndVerify(variableList, variableList.getBuffers(false));
+      assertEquals(0, variableList.getLastSet());
+
+      // FixedSizeListVector
+      fixedList.allocateNewSafe();
+      fixedList.setNull(0);
+      fixedList.setValueCount(1);
+      resetVectorAndVerify(fixedList, fixedList.getBuffers(false));
+    }
+  }
+
+  @Test
+  public void testMapTypeReset() {
+    try (final MapVector mapVector = new MapVector("Map", allocator, FieldType.nullable(MinorType.INT.getType()), null);
+         final NullableMapVector nullableMapVector = new NullableMapVector("NullableMap", allocator, FieldType.nullable(MinorType.INT.getType()), null)
+    ) {
+      // MapVector
+      mapVector.allocateNewSafe();
+      IntVector mapChild = mapVector.addOrGet("child", FieldType.nullable(new Int(32, true)), IntVector.class);
+      mapChild.setNull(0);
+      mapVector.setValueCount(1);
+      resetVectorAndVerify(mapVector, mapVector.getBuffers(false));
+
+      // NullableMapVector
+      nullableMapVector.allocateNewSafe();
+      nullableMapVector.setNull(0);
+      nullableMapVector.setValueCount(1);
+      resetVectorAndVerify(nullableMapVector, nullableMapVector.getBuffers(false));
+    }
+  }
+
+  @Test
+  public void testUnionTypeReset() {
+    try (final UnionVector vector = new UnionVector("Union", allocator, null);
+         final IntVector dataVector = new IntVector("Int", allocator)
+    ) {
+      vector.getBufferSize();
+      vector.allocateNewSafe();
+      dataVector.allocateNewSafe();
+      vector.addVector(dataVector);
+      dataVector.setNull(0);
+      vector.setValueCount(1);
+      resetVectorAndVerify(vector, vector.getBuffers(false));
     }
   }
 }

From bda85bb1d58b71b822b358653665407de0cd4cad Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 7 Jan 2018 12:19:29 -0500
Subject: [PATCH 1367/1644] ARROW-1970: [GLib] Add
 garrow_chunked_array_get_value_data_type() and
 garrow_chunked_array_get_value_type()

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1458 from kou/glib-add-chunked-array-get-value-type and squashes the following commits:

4d99a079 [Kouhei Sutou] [GLib] Add garrow_chunked_array_get_value_data_type() and garrow_chunked_array_get_value_type()
---
 c_glib/arrow-glib/chunked-array.cpp | 35 +++++++++++++++++++++++++++++
 c_glib/arrow-glib/chunked-array.h   |  5 +++++
 c_glib/test/test-chunked-array.rb   | 18 +++++++++++++++
 3 files changed, 58 insertions(+)

diff --git a/c_glib/arrow-glib/chunked-array.cpp b/c_glib/arrow-glib/chunked-array.cpp
index 62d666fbcaaba..69195c57b1768 100644
--- a/c_glib/arrow-glib/chunked-array.cpp
+++ b/c_glib/arrow-glib/chunked-array.cpp
@@ -23,6 +23,8 @@
 
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/chunked-array.hpp>
+#include <arrow-glib/data-type.hpp>
+#include <arrow-glib/type.hpp>
 
 G_BEGIN_DECLS
 
@@ -163,6 +165,39 @@ garrow_chunked_array_equal(GArrowChunkedArray *chunked_array,
   return arrow_chunked_array->Equals(arrow_other_chunked_array);
 }
 
+/**
+ * garrow_chunked_array_get_value_data_type:
+ * @chunked_array: A #GArrowChunkedArray.
+ *
+ * Returns: (transfer full): The #GArrowDataType of the value of
+ *   the chunked array.
+ *
+ * Since: 0.9.0
+ */
+GArrowDataType *
+garrow_chunked_array_get_value_data_type(GArrowChunkedArray *chunked_array)
+{
+  auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+  auto arrow_type = arrow_chunked_array->type();
+  return garrow_data_type_new_raw(&arrow_type);
+}
+
+/**
+ * garrow_chunked_array_get_value_type:
+ * @chunked_array: A #GArrowChunkedArray.
+ *
+ * Returns: The #GArrowType of the value of the chunked array.
+ *
+ * Since: 0.9.0
+ */
+GArrowType
+garrow_chunked_array_get_value_type(GArrowChunkedArray *chunked_array)
+{
+  auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+  auto arrow_type = arrow_chunked_array->type();
+  return garrow_type_from_raw(arrow_type->id());
+}
+
 /**
  * garrow_chunked_array_get_length:
  * @chunked_array: A #GArrowChunkedArray.
diff --git a/c_glib/arrow-glib/chunked-array.h b/c_glib/arrow-glib/chunked-array.h
index c5f986a631835..0c3c81a744ceb 100644
--- a/c_glib/arrow-glib/chunked-array.h
+++ b/c_glib/arrow-glib/chunked-array.h
@@ -70,6 +70,11 @@ GArrowChunkedArray *garrow_chunked_array_new(GList *chunks);
 gboolean garrow_chunked_array_equal(GArrowChunkedArray *chunked_array,
                                     GArrowChunkedArray *other_chunked_array);
 
+GArrowDataType *
+garrow_chunked_array_get_value_data_type(GArrowChunkedArray *chunked_array);
+GArrowType
+garrow_chunked_array_get_value_type(GArrowChunkedArray *chunked_array);
+
 guint64 garrow_chunked_array_get_length (GArrowChunkedArray *chunked_array);
 guint64 garrow_chunked_array_get_n_nulls(GArrowChunkedArray *chunked_array);
 guint   garrow_chunked_array_get_n_chunks (GArrowChunkedArray *chunked_array);
diff --git a/c_glib/test/test-chunked-array.rb b/c_glib/test/test-chunked-array.rb
index cde7a8b0c61f1..9287058e1abc6 100644
--- a/c_glib/test/test-chunked-array.rb
+++ b/c_glib/test/test-chunked-array.rb
@@ -31,6 +31,24 @@ def test_equal
                  Arrow::ChunkedArray.new(chunks2))
   end
 
+  def test_value_data_type
+    chunks = [
+      build_boolean_array([true, false]),
+      build_boolean_array([true]),
+    ]
+    assert_equal(Arrow::BooleanDataType.new,
+                 Arrow::ChunkedArray.new(chunks).value_data_type)
+  end
+
+  def test_value_type
+    chunks = [
+      build_boolean_array([true, false]),
+      build_boolean_array([true]),
+    ]
+    assert_equal(Arrow::Type::BOOL,
+                 Arrow::ChunkedArray.new(chunks).value_type)
+  end
+
   def test_length
     chunks = [
       build_boolean_array([true, false]),

From 234ff713dd9ecbcc2824d336b4e4f9d059a490dd Mon Sep 17 00:00:00 2001
From: Albert Shieh <adshieh@gmail.com>
Date: Sun, 7 Jan 2018 12:28:19 -0500
Subject: [PATCH 1368/1644] ARROW-1958: [Python] Error in pandas conversion for
 datetimetz row index

Fix conversion of datetimetz row index for non-UTC time zones in to_pandas.

Author: Albert Shieh <adshieh@gmail.com>

Closes #1454 from adshieh/master and squashes the following commits:

6f413027 [Albert Shieh] Fix pandas conversion for datetimetz row index.
---
 python/pyarrow/pandas_compat.py             |  9 ++++++++-
 python/pyarrow/tests/test_convert_pandas.py | 10 ++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index f08049f6adba0..f3089d2a012a6 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -459,6 +459,8 @@ def _make_datetimetz(tz):
 
 def table_to_blockmanager(options, table, memory_pool, nthreads=1,
                           categoricals=None):
+    from pyarrow.compat import DatetimeTZDtype
+
     index_columns = []
     columns = []
     column_indexes = []
@@ -517,7 +519,12 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
                 # non-writeable arrays when calling MultiIndex.from_arrays
                 values = values.copy()
 
-            index_arrays.append(pd.Series(values, dtype=col_pandas.dtype))
+            if isinstance(col_pandas.dtype, DatetimeTZDtype):
+                index_array = (pd.Series(values).dt.tz_localize('utc')
+                               .dt.tz_convert(col_pandas.dtype.tz))
+            else:
+                index_array = pd.Series(values, dtype=col_pandas.dtype)
+            index_arrays.append(index_array)
             index_names.append(
                 _backwards_compatible_index_name(raw_name, logical_name)
             )
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 76b55cf90431c..83b1da135eea4 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -254,6 +254,16 @@ def test_datetimetz_column_index(self):
         md = column_indexes['metadata']
         assert md['timezone'] == 'America/New_York'
 
+    def test_datetimetz_row_index(self):
+        df = pd.DataFrame({
+            'a': pd.date_range(
+                start='2017-01-01', periods=3, tz='America/New_York'
+            )
+        })
+        df = df.set_index('a')
+
+        _check_pandas_roundtrip(df, preserve_index=True)
+
     def test_categorical_row_index(self):
         df = pd.DataFrame({'a': [1, 2, 3], 'b': [1, 2, 3]})
         df['a'] = df.a.astype('category')

From 95d894da250ee7d5dfccb09e2d349e5748cc5fb8 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Sun, 7 Jan 2018 13:48:08 -0800
Subject: [PATCH 1369/1644] ARROW-1972: [Python] Import pyarrow in
 DeserializeObject.

Author: Robert Nishihara <robertnishihara@gmail.com>
Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1463 from robertnishihara/segfaultfix and squashes the following commits:

ec8a6c5 [Robert Nishihara] Add comment.
6222340 [Philipp Moritz] fix tests, linting, add license
3e969db [Robert Nishihara] Simplify tests.
8aa3fca [Philipp Moritz] add regression test
bfa0851 [Robert Nishihara] Import pyarrow in DeserializeObject.
---
 cpp/src/arrow/python/arrow_to_python.cc    |  1 +
 python/pyarrow/tests/deserialize_buffer.py | 26 ++++++++++++++++++++++
 python/pyarrow/tests/test_serialization.py | 14 ++++++++++++
 3 files changed, 41 insertions(+)
 create mode 100644 python/pyarrow/tests/deserialize_buffer.py

diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index ce539a597be37..c060ab8bfd6db 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -284,6 +284,7 @@ Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObj
                          PyObject** out) {
   PyAcquireGIL lock;
   PyDateTime_IMPORT;
+  import_pyarrow();
   return DeserializeList(context, *obj.batch->column(0), 0, obj.batch->num_rows(), base,
                          obj, out);
 }
diff --git a/python/pyarrow/tests/deserialize_buffer.py b/python/pyarrow/tests/deserialize_buffer.py
new file mode 100644
index 0000000000000..982dc6695d590
--- /dev/null
+++ b/python/pyarrow/tests/deserialize_buffer.py
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This file is called from a test in test_serialization.py.
+
+import sys
+
+import pyarrow as pa
+
+with open(sys.argv[1], 'rb') as f:
+    data = f.read()
+    pa.deserialize(data)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index f245dc299cb25..6116556386b1a 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -541,3 +541,17 @@ def deserialize_regex(serialized, q):
     p.start()
     assert q.get().pattern == regex.pattern
     p.join()
+
+
+def test_deserialize_buffer_in_different_process():
+    import tempfile
+    import subprocess
+
+    f = tempfile.NamedTemporaryFile(delete=False)
+    b = pa.serialize(pa.frombuffer(b'hello')).to_buffer()
+    f.write(b.to_pybytes())
+    f.close()
+
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    python_file = os.path.join(dir_path, 'deserialize_buffer.py')
+    subprocess.check_call(['python', python_file, f.name])

From ee134f0e4b285e17f33a56e7750836dd82a13735 Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Mon, 8 Jan 2018 15:17:12 -0500
Subject: [PATCH 1370/1644] ARROW-1969: [C++] Don't build ORC extension by
 default

- Turns off building optional ORC extension by default
- Fixes travis builds to turn on ORC extension for a few branches
- Adds trivial import test to python build
- Adds documentation on how to build optional ORC extension

Author: Jim Crist <jiminy.crist@gmail.com>

Closes #1457 from jcrist/orc-off-by-default and squashes the following commits:

fc9898d8 [Jim Crist] Document how to build ORC integration
950ae38e [Jim Crist] ORC integration is off by default
---
 .travis.yml                       | 2 ++
 ci/travis_before_script_cpp.sh    | 4 ++++
 ci/travis_script_python.sh        | 2 ++
 cpp/CMakeLists.txt                | 2 +-
 cpp/README.md                     | 7 +++++++
 python/doc/source/development.rst | 6 ++++++
 python/manylinux1/build_arrow.sh  | 2 +-
 7 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ddadf739aab25..58d6786aab16a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -54,6 +54,7 @@ matrix:
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_VALGRIND=1
     - export ARROW_TRAVIS_PLASMA=1
+    - export ARROW_TRAVIS_ORC=1
     - export ARROW_TRAVIS_CLANG_FORMAT=1
     - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - export CC="clang-4.0"
@@ -75,6 +76,7 @@ matrix:
     before_script:
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_PLASMA=1
+    - export ARROW_TRAVIS_ORC=1
     - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 664f7ce5fed2d..fd2c1644638c4 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -84,6 +84,10 @@ if [ $ARROW_TRAVIS_PLASMA == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON"
 fi
 
+if [ $ARROW_TRAVIS_ORC == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_ORC=ON"
+fi
+
 if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON"
 fi
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 444386fdf955e..9e74906d03739 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -63,6 +63,7 @@ cmake -GNinja \
       -DARROW_BUILD_UTILITIES=off \
       -DARROW_PLASMA=on \
       -DARROW_PYTHON=on \
+      -DARROW_ORC=on \
       -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
       -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
       $ARROW_CPP_DIR
@@ -88,6 +89,7 @@ popd
 
 python -c "import pyarrow.parquet"
 python -c "import pyarrow.plasma"
+python -c "import pyarrow.orc"
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
   export PLASMA_VALGRIND=1
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index ede13af5d413b..0558cba495a83 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -121,7 +121,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
 
   option(ARROW_ORC
     "Build the Arrow ORC adapter"
-    ON)
+    OFF)
 
   option(ARROW_JEMALLOC
     "Build the Arrow jemalloc-based allocator"
diff --git a/cpp/README.md b/cpp/README.md
index 92e87cb436ada..39a1ccac64818 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -145,6 +145,13 @@ The CUDA toolchain used to build the library can be customized by using the
 This library is still in Alpha stages, and subject to API changes without
 deprecation warnings.
 
+### Building Apache ORC integration (optional)
+
+The optional arrow reader for the Apache ORC format (found in the
+`arrow::adapters::orc` namespace) can be built by passing `-DARROW_ORC=on`.
+This is currently not supported on windows. Note that this functionality is
+still in Alpha stages, and subject to API changes without deprecation warnings.
+
 ### API documentation
 
 To generate the (html) API documentation, run the following command in the apidoc
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 1c1a308fc4a56..01844fa18d133 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -175,6 +175,9 @@ Now build and install the Arrow C++ libraries:
 If you don't want to build and install the Plasma in-memory object store,
 you can omit the ``-DARROW_PLASMA=on`` flag.
 
+To add support for the experimental Apache ORC integration, include
+``-DARROW_ORC=on`` in these flags.
+
 Now, optionally build and install the Apache Parquet libraries in your
 toolchain:
 
@@ -205,6 +208,9 @@ Now, build pyarrow:
 If you did not build parquet-cpp, you can omit ``--with-parquet`` and if
 you did not build with plasma, you can omit ``--with-plasma``.
 
+If you built with the experimental Apache ORC integration, include
+``--with-orc`` in these flags.
+
 You should be able to run the unit tests with:
 
 .. code-block:: shell
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index ced4556e92ad5..4d816bec9b6a4 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -58,7 +58,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
     mkdir -p "${ARROW_BUILD_DIR}"
     pushd "${ARROW_BUILD_DIR}"
-    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON ..
+    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON -DARROW_ORC=ON ..
     make -j5 install
     popd
 

From 1b0732cde515f9ec9b42f98061f9204796c9e154 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 10 Jan 2018 23:34:47 +0900
Subject: [PATCH 1371/1644] ARROW-1965: [GLib] Add
 garrow_array_builder_get_value_data_type()

garrow_array_builder_get_value_type() is also added for convenient.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1456 from kou/glib-add-builder-get-value-type and squashes the following commits:

5a4b1229 [Kouhei Sutou] [GLib] Add garrow_array_builder_get_value_data_type()
---
 c_glib/arrow-glib/array-builder.cpp |  34 ++++++
 c_glib/arrow-glib/array-builder.h   |   4 +
 c_glib/test/test-array-builder.rb   | 156 ++++++++++++++++++++++++++++
 3 files changed, 194 insertions(+)

diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp
index a5df681421c25..7625bcd51ee0f 100644
--- a/c_glib/arrow-glib/array-builder.cpp
+++ b/c_glib/arrow-glib/array-builder.cpp
@@ -24,6 +24,7 @@
 #include <arrow-glib/array-builder.hpp>
 #include <arrow-glib/data-type.hpp>
 #include <arrow-glib/error.hpp>
+#include <arrow-glib/type.hpp>
 
 template <typename BUILDER, typename VALUE>
 gboolean
@@ -327,6 +328,39 @@ garrow_array_builder_release_ownership(GArrowArrayBuilder *builder)
   priv->have_ownership = FALSE;
 }
 
+/**
+ * garrow_array_builder_get_value_data_type:
+ * @builder: A #GArrowArrayBuilder.
+ *
+ * Returns: (transfer full): The #GArrowDataType of the value of
+ *   the array builder.
+ *
+ * Since: 0.9.0
+ */
+GArrowDataType *
+garrow_array_builder_get_value_data_type(GArrowArrayBuilder *builder)
+{
+  auto arrow_builder = garrow_array_builder_get_raw(builder);
+  auto arrow_type = arrow_builder->type();
+  return garrow_data_type_new_raw(&arrow_type);
+}
+
+/**
+ * garrow_array_builder_get_value_type:
+ * @builder: A #GArrowArrayBuilder.
+ *
+ * Returns: The #GArrowType of the value of the array builder.
+ *
+ * Since: 0.9.0
+ */
+GArrowType
+garrow_array_builder_get_value_type(GArrowArrayBuilder *builder)
+{
+  auto arrow_builder = garrow_array_builder_get_raw(builder);
+  auto arrow_type = arrow_builder->type();
+  return garrow_type_from_raw(arrow_type->id());
+}
+
 /**
  * garrow_array_builder_finish:
  * @builder: A #GArrowArrayBuilder.
diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h
index 19dadb30999bd..ea95f31e8fae1 100644
--- a/c_glib/arrow-glib/array-builder.h
+++ b/c_glib/arrow-glib/array-builder.h
@@ -37,6 +37,10 @@ struct _GArrowArrayBuilderClass
 
 void garrow_array_builder_release_ownership(GArrowArrayBuilder *builder);
 
+GArrowDataType *
+garrow_array_builder_get_value_data_type(GArrowArrayBuilder *builder);
+GArrowType garrow_array_builder_get_value_type(GArrowArrayBuilder *builder);
+
 GArrowArray        *garrow_array_builder_finish   (GArrowArrayBuilder *builder,
                                                    GError **error);
 
diff --git a/c_glib/test/test-array-builder.rb b/c_glib/test/test-array-builder.rb
index 92976a424ccad..a773131e15b5d 100644
--- a/c_glib/test/test-array-builder.rb
+++ b/c_glib/test/test-array-builder.rb
@@ -76,6 +76,18 @@ def test_negative
   end
 end
 
+module ArrayBuilderValueTypeTests
+  def test_value_data_type
+    assert_equal(value_data_type,
+                 build_array(sample_values).value_data_type)
+  end
+
+  def test_value_type
+    assert_equal(value_data_type.id,
+                 build_array(sample_values).value_type)
+  end
+end
+
 class TestArrayBuilder < Test::Unit::TestCase
   include Helper::Buildable
   include Helper::Omittable
@@ -93,6 +105,10 @@ def create_builder
       Arrow::BooleanArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::BooleanDataType.new
+    end
+
     def builder_class_name
       "boolean-array-builder"
     end
@@ -101,6 +117,10 @@ def sample_values
       [true, false, true]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -115,6 +135,10 @@ def create_builder
       Arrow::IntArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::Int8DataType.new
+    end
+
     def builder_class_name
       "int-array-builder"
     end
@@ -123,6 +147,10 @@ def sample_values
       [1, -2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -137,6 +165,10 @@ def create_builder
       Arrow::UIntArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::UInt8DataType.new
+    end
+
     def builder_class_name
       "uint-array-builder"
     end
@@ -145,6 +177,10 @@ def sample_values
       [1, 2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -159,6 +195,10 @@ def create_builder
       Arrow::Int8ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::Int8DataType.new
+    end
+
     def builder_class_name
       "int8-array-builder"
     end
@@ -167,6 +207,10 @@ def sample_values
       [1, -2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -181,6 +225,10 @@ def create_builder
       Arrow::UInt8ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::UInt8DataType.new
+    end
+
     def builder_class_name
       "uint8-array-builder"
     end
@@ -189,6 +237,10 @@ def sample_values
       [1, 2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -203,6 +255,10 @@ def create_builder
       Arrow::Int16ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::Int16DataType.new
+    end
+
     def builder_class_name
       "int16-array-builder"
     end
@@ -211,6 +267,10 @@ def sample_values
       [1, -2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -225,6 +285,10 @@ def create_builder
       Arrow::UInt16ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::UInt16DataType.new
+    end
+
     def builder_class_name
       "uint16-array-builder"
     end
@@ -233,6 +297,10 @@ def sample_values
       [1, 2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -247,6 +315,10 @@ def create_builder
       Arrow::Int32ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::Int32DataType.new
+    end
+
     def builder_class_name
       "int32-array-builder"
     end
@@ -255,6 +327,10 @@ def sample_values
       [1, -2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -269,6 +345,10 @@ def create_builder
       Arrow::UInt32ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::UInt32DataType.new
+    end
+
     def builder_class_name
       "uint32-array-builder"
     end
@@ -277,6 +357,10 @@ def sample_values
       [1, 2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -291,6 +375,10 @@ def create_builder
       Arrow::Int64ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::Int64DataType.new
+    end
+
     def builder_class_name
       "int64-array-builder"
     end
@@ -299,6 +387,10 @@ def sample_values
       [1, -2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -313,6 +405,10 @@ def create_builder
       Arrow::UInt64ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::UInt64DataType.new
+    end
+
     def builder_class_name
       "uint64-array-builder"
     end
@@ -321,6 +417,10 @@ def sample_values
       [1, 2, 3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -335,6 +435,10 @@ def create_builder
       Arrow::FloatArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::FloatDataType.new
+    end
+
     def builder_class_name
       "float-array-builder"
     end
@@ -343,6 +447,10 @@ def sample_values
       [1.1, -2.2, 3.3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -357,6 +465,10 @@ def create_builder
       Arrow::DoubleArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::DoubleDataType.new
+    end
+
     def builder_class_name
       "double-array-builder"
     end
@@ -365,6 +477,10 @@ def sample_values
       [1.1, -2.2, 3.3]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -379,6 +495,10 @@ def create_builder
       Arrow::Date32ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::Date32DataType.new
+    end
+
     def builder_class_name
       "date32-array-builder"
     end
@@ -391,6 +511,10 @@ def sample_values
       ]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -405,6 +529,10 @@ def create_builder
       Arrow::Date64ArrayBuilder.new
     end
 
+    def value_data_type
+      Arrow::Date64DataType.new
+    end
+
     def builder_class_name
       "date64-array-builder"
     end
@@ -417,6 +545,10 @@ def sample_values
       ]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -432,6 +564,10 @@ def create_builder
       Arrow::TimestampArrayBuilder.new(data_type)
     end
 
+    def value_data_type
+      Arrow::TimestampDataType.new(:milli)
+    end
+
     def builder_class_name
       "timestamp-array-builder"
     end
@@ -444,6 +580,10 @@ def sample_values
       ]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -459,6 +599,10 @@ def create_builder
       Arrow::Time32ArrayBuilder.new(data_type)
     end
 
+    def value_data_type
+      Arrow::Time32DataType.new(:second)
+    end
+
     def builder_class_name
       "time32-array-builder"
     end
@@ -471,6 +615,10 @@ def sample_values
       ]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end
@@ -486,6 +634,10 @@ def create_builder
       Arrow::Time64ArrayBuilder.new(data_type)
     end
 
+    def value_data_type
+      Arrow::Time64DataType.new(:micro)
+    end
+
     def builder_class_name
       "time64-array-builder"
     end
@@ -498,6 +650,10 @@ def sample_values
       ]
     end
 
+    sub_test_case("value type") do
+      include ArrayBuilderValueTypeTests
+    end
+
     sub_test_case("#append_values") do
       include ArrayBuilderAppendValuesTests
     end

From d0a93ba3fac91fe12b83d2ef7fccf97435c4f4ec Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Wed, 10 Jan 2018 17:04:15 -0500
Subject: [PATCH 1372/1644] ARROW-1980: [Python] Fix race condition in
 write_to_dataset

One race condition was already fixed, but another one exists when
writing by partition.

Author: Jim Crist <jiminy.crist@gmail.com>

Closes #1468 from jcrist/fix-race-condition-parquet-partition and squashes the following commits:

180c4a24 [Jim Crist] Fix race condition in write_to_dataset
---
 python/pyarrow/parquet.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index d9f1bd2c36430..151e0df8a22d0 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -966,6 +966,14 @@ def write_table(table, where, row_group_size=None, version='1.0',
 """.format(_parquet_writer_arg_docs)
 
 
+def _mkdir_if_not_exists(fs, path):
+    if fs._isfilestore() and not fs.exists(path):
+        try:
+            fs.mkdir(path)
+        except OSError:
+            assert fs.exists(path)
+
+
 def write_to_dataset(table, root_path, partition_cols=None,
                      filesystem=None, preserve_index=True, **kwargs):
     """
@@ -1012,11 +1020,7 @@ def write_to_dataset(table, root_path, partition_cols=None,
     else:
         fs = _ensure_filesystem(filesystem)
 
-    if fs._isfilestore() and not fs.exists(root_path):
-        try:
-            fs.mkdir(root_path)
-        except OSError:
-            assert fs.exists(root_path)
+    _mkdir_if_not_exists(fs, root_path)
 
     if partition_cols is not None and len(partition_cols) > 0:
         df = table.to_pandas()
@@ -1034,8 +1038,7 @@ def write_to_dataset(table, root_path, partition_cols=None,
             subtable = Table.from_pandas(subgroup,
                                          preserve_index=preserve_index)
             prefix = "/".join([root_path, subdir])
-            if fs._isfilestore() and not fs.exists(prefix):
-                fs.mkdir(prefix)
+            _mkdir_if_not_exists(fs, prefix)
             outfile = compat.guid() + ".parquet"
             full_path = "/".join([prefix, outfile])
             with fs.open(full_path, 'wb') as f:

From 3e14e12427bf9b81b8c43e54c2def509c1ca75f0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 10 Jan 2018 17:05:47 -0500
Subject: [PATCH 1373/1644] ARROW-1978: [Website] Consolidate Powered By
 project list, add more visibly to front page

There can be a fine line between what is an open source project and what is a company and what is a product or organization, so to keep things simple I have consolidated things into a single list without passing any judgments. I have also added a link to the Powered By page that is visible without digging into any menus on the front page

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1465 from wesm/website-powered-by-simplify and squashes the following commits:

7c43e0bd [Wes McKinney] Suggest adding to Powered By by pull request instead of e-mailing
9a61eed0 [Wes McKinney] Consolidate Powered By project list, add more visibly to front page
---
 site/index.html    |  4 ++++
 site/powered_by.md | 45 ++++++++++++++++++---------------------------
 2 files changed, 22 insertions(+), 27 deletions(-)

diff --git a/site/index.html b/site/index.html
index ffa8d54af2f4f..87995cbabed48 100644
--- a/site/index.html
+++ b/site/index.html
@@ -38,6 +38,10 @@ <h2>Standard</h2>
           projects, including Calcite, Cassandra, Drill, Hadoop, HBase, Ibis,
           Impala, Kudu, Pandas, Parquet, Phoenix, Spark, and Storm making it
           the de-facto standard for columnar in-memory analytics.</p>
+
+          <p>Learn more about projects that are <a href="{{ site.baseurl
+          }}/powered_by/">Powered By Apache Arrow</a></p>
+
         </div>
      </div> <!-- close "row" div -->
 
diff --git a/site/powered_by.md b/site/powered_by.md
index 56f6e2bcee41a..4033202679817 100644
--- a/site/powered_by.md
+++ b/site/powered_by.md
@@ -43,11 +43,9 @@ names, etc.) like "arrow-foo". These are permitted. Nominative use of trademarks
 in descriptions is also always allowed, as in "BigCoProduct is a widget for
 Apache Arrow".
 
-### Open Source Projects
-
-To add yourself to the list, please email dev@arrow.apache.org with your
+To add yourself to the list, please open a pull request adding your
 organization name, URL, a list of which Arrow components you are using, and a
-short description of your use case.
+short description of your use case. See the following for some examples.
 
 * **[Apache Parquet][3]:** A columnar storage format available to any project
   in the Hadoop ecosystem, regardless of the choice of data processing
@@ -61,10 +59,23 @@ short description of your use case.
 * **[Dask][15]:** Python library for parallel and distributed execution of
   dynamic task graphs. Dask supports using pyarrow for accessing Parquet
   files
+* **[Dremio][9]:** A self-service data platform. Dremio makes it easy for
+  users to discover, curate, accelerate, and share data from any source.
+  It includes a distributed SQL execution engine based on Apache Arrow.
+  Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow
+  buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI,
+  Python, R, and more (all backed by Apache Arrow).
 * **[GeoMesa][8]:** A suite of tools that enables large-scale geospatial query
   and analytics on distributed computing systems. GeoMesa supports query
   results in the Arrow IPC format, which can then be used for in-browser
   visualizations and/or further analytics.
+* **[GOAI][19]:** Open GPU-Accelerated Analytics Initiative for Arrow-powered
+  analytics across GPU tools and vendors
+* **[Graphistry][18]:** Supercharged Visual Investigation Platform used by
+  teams for security, anti-fraud, and related investigations. The Graphistry
+  team uses Arrow in its NodeJS GPU backend and client libraries, and is an
+  early contributing member to GOAI and Arrow\[JS\] focused on bringing these
+  technologies to the enterprise.
 * **[libgdf][14]:** A C library of CUDA-based analytics functions and GPU IPC
   support for structured data. Uses the Arrow IPC format and targets the Arrow
   memory layout in its analytic functions. This work is part of the [GPU Open
@@ -75,6 +86,9 @@ short description of your use case.
 * **[pandas][12]:** data analysis toolkit for Python programmers. pandas
   supports reading and writing Parquet files using pyarrow. Several pandas
   core developers are also contributors to Apache Arrow.
+* **[Quilt Data][13]:** Quilt is a data package manager, designed to make
+  managing data as easy as managing code. It supports Parquet format via
+  pyarrow for data access.
 * **[Ray][5]:** A flexible, high-performance distributed execution framework
   with a focus on machine learning and AI applications. Uses Arrow to
   efficiently store Python data structures containing large arrays of numerical
@@ -91,29 +105,6 @@ short description of your use case.
   Arrow Tables and RecordBatches in addition to the Python Database API
   Specification 2.0.
 
-### Companies and Organizations
-
-To add yourself to the list, please email dev@arrow.apache.org with your
-organization name, URL, a list of which Arrow components you are using, and a
-short description of your use case.
-
-* **[Dremio][9]:** A self-service data platform. Dremio makes it easy for
-  users to discover, curate, accelerate, and share data from any source.
-  It includes a distributed SQL execution engine based on Apache Arrow.
-  Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow
-  buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI,
-  Python, R, and more (all backed by Apache Arrow).
-* **[GOAI][19]:** Open GPU-Accelerated Analytics Initiative for Arrow-powered 
-  analytics across GPU tools and vendors  
-* **[Graphistry][18]:** Supercharged Visual Investigation Platform used by
-  teams for security, anti-fraud, and related investigations. The Graphistry
-  team uses Arrow in its NodeJS GPU backend and client libraries, and is an
-  early contributing member to GOAI and Arrow\[JS\] focused on bringing these 
-  technologies to the enterprise.  
-* **[Quilt Data][13]:** Quilt is a data package manager, designed to make
-  managing data as easy as managing code. It supports Parquet format via
-  pyarrow for data access.
-
 [1]: https://www.apache.org/foundation/marks/
 [2]: https://www.apache.org/foundation/marks/faq/
 [3]: https://parquet.apache.org/

From 289e205c16bdeb5a63b629ce5df05e4164b78e72 Mon Sep 17 00:00:00 2001
From: Chris Bartak <chris.bartak@advisorsexcel.com>
Date: Wed, 10 Jan 2018 17:07:14 -0500
Subject: [PATCH 1374/1644] ARROW-1977: [C++] Update windows dev docs

Author: Chris Bartak <chris.bartak@advisorsexcel.com>

Closes #1466 from chris-b1/windows-cpp-devdoc and squashes the following commits:

0990ddc0 [Chris Bartak] remove python from cpp conda env
4dab8b05 [Chris Bartak] updates windows c++ env docs
---
 cpp/apidoc/Windows.md | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index e7e83f1946b56..aa3d31f1f7b90 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -55,20 +55,16 @@ previous step:
 activate arrow-dev
 ```
 
-We are using [cmake][4] tool to support Windows builds.
+We are using the [cmake][4] tool to support Windows builds.
 To allow cmake to pick up 3rd party dependencies, you should set
 `ARROW_BUILD_TOOLCHAIN` environment variable to contain `Library` folder
 path of new created on previous step `arrow-dev` conda environment.
-For instance, if `Miniconda` was installed to default destination, `Library`
-folder path for `arrow-dev` conda environment will be as following:
 
+To set `ARROW_BUILD_TOOLCHAIN` environment variable visible only for current terminal
+session you can run following. `%CONDA_PREFIX` is set by conda to the current environment
+root by the `activate` script.
 ```shell
-C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library
-```
-
-To set `ARROW_BUILD_TOOLCHAIN` environment variable visible only for current terminal session you can run following:
-```shell
-set ARROW_BUILD_TOOLCHAIN=C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library
+set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 ```
 
 To validate value of `ARROW_BUILD_TOOLCHAIN` environment variable you can run following terminal command:

From 9eae5089b5099548f67bbeca9927ca81df9d5de8 Mon Sep 17 00:00:00 2001
From: Adam Seibert <seibs@users.noreply.github.com>
Date: Wed, 10 Jan 2018 17:10:39 -0500
Subject: [PATCH 1375/1644] ARROW-764: [C++] Improves performance of CopyBitmap
 and adds benchmarks

I took a swing at improving the CopyBitmap performance (benchmarks below).  I'm a C/C++ novice, so I thought I'd get some feedback before I went too much further.

**Starting Point**
```
Run on (4 X 2208 MHz CPU s)
12/13/17 21:15:18
Benchmark                                        Time           CPU Iterations
------------------------------------------------------------------------------
BM_CopyBitmap/97.6563k/0/min_time:1.000       4779 us       4758 us        289   20.0445MB/s
BM_CopyBitmap/976.563k/0/min_time:1.000      47740 us      47476 us         26   20.0875MB/s
BM_CopyBitmap/97.6563k/4/min_time:1.000       4858 us       4866 us        289   19.5991MB/s
BM_CopyBitmap/976.563k/4/min_time:1.000      48117 us      47953 us         29   19.8879MB/s
```

**Using stanford bithacks for SetBitTo**
```
Run on (4 X 2208 MHz CPU s)
12/13/17 21:22:05
Benchmark                                        Time           CPU Iterations
------------------------------------------------------------------------------
BM_CopyBitmap/97.6563k/0/min_time:1.000       1647 us       1649 us        815   57.8415MB/s
BM_CopyBitmap/976.563k/0/min_time:1.000      16368 us      16397 us         81   58.1629MB/s
BM_CopyBitmap/97.6563k/4/min_time:1.000       1599 us       1610 us        815   59.2186MB/s
BM_CopyBitmap/976.563k/4/min_time:1.000      16026 us      16011 us         81   59.5644MB/s
```

**memcpy + shifting**
*This solution provides varying performance depending on whether or not the bit offset is a multiple of 8*
```
Run on (4 X 2208 MHz CPU s)
12/13/17 21:23:44
Benchmark                                        Time           CPU Iterations
------------------------------------------------------------------------------
BM_CopyBitmap/97.6563k/0/min_time:1.000          5 us          5 us     280000   18.9651GB/s
BM_CopyBitmap/976.563k/0/min_time:1.000         62 us         61 us      22400   15.1721GB/s
BM_CopyBitmap/97.6563k/4/min_time:1.000        171 us        170 us       6892   560.872MB/s
BM_CopyBitmap/976.563k/4/min_time:1.000       1639 us       1639 us        896   581.782MB/s
```

Author: Adam Seibert <seibs@users.noreply.github.com>

Closes #1422 from seibs/ARROW-764 and squashes the following commits:

c813c8e2 [Adam Seibert] ARROW-764: [C++] Improves performance of CopyBitmap and adds benchmarks
---
 cpp/src/arrow/util/CMakeLists.txt        |  6 ++-
 cpp/src/arrow/util/bit-util-benchmark.cc | 58 ++++++++++++++++++++++++
 cpp/src/arrow/util/bit-util-test.cc      | 17 +++----
 cpp/src/arrow/util/bit-util.cc           | 32 ++++++++++++-
 cpp/src/arrow/util/bit-util.h            |  9 ++--
 5 files changed, 104 insertions(+), 18 deletions(-)
 create mode 100644 cpp/src/arrow/util/bit-util-benchmark.cc

diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index a36dffb52e294..8b61a3acfe709 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -63,10 +63,10 @@ if (ARROW_BUILD_BENCHMARKS)
       Shlwapi.lib
   )
   else()
-	  target_link_libraries(arrow_benchmark_main
+    target_link_libraries(arrow_benchmark_main
       benchmark
       pthread
-	  )
+    )
   endif()
 
   # TODO(wesm): Some benchmarks include gtest.h
@@ -80,4 +80,6 @@ ADD_ARROW_TEST(key-value-metadata-test)
 ADD_ARROW_TEST(rle-encoding-test)
 ADD_ARROW_TEST(stl-util-test)
 
+ADD_ARROW_BENCHMARK(bit-util-benchmark)
+
 add_subdirectory(variant)
diff --git a/cpp/src/arrow/util/bit-util-benchmark.cc b/cpp/src/arrow/util/bit-util-benchmark.cc
new file mode 100644
index 0000000000000..8969dd80b157e
--- /dev/null
+++ b/cpp/src/arrow/util/bit-util-benchmark.cc
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include <vector>
+
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/memory_pool.h"
+#include "arrow/test-util.h"
+#include "arrow/util/bit-util.h"
+
+namespace arrow {
+namespace BitUtil {
+
+static void BM_CopyBitmap(benchmark::State& state) {  // NOLINT non-const reference
+  const int kBufferSize = state.range(0);
+
+  std::shared_ptr<Buffer> buffer;
+  ASSERT_OK(AllocateBuffer(default_memory_pool(), kBufferSize, &buffer));
+  memset(buffer->mutable_data(), 0, kBufferSize);
+  test::random_bytes(kBufferSize, 0, buffer->mutable_data());
+
+  const int num_bits = kBufferSize * 8;
+  const uint8_t* src = buffer->data();
+
+  std::shared_ptr<Buffer> copy;
+  while (state.KeepRunning()) {
+    ABORT_NOT_OK(CopyBitmap(default_memory_pool(), src, state.range(1), num_bits, &copy));
+  }
+  state.SetBytesProcessed(state.iterations() * kBufferSize * sizeof(int8_t));
+}
+
+BENCHMARK(BM_CopyBitmap)
+    ->Args({100000, 0})
+    ->Args({1000000, 0})
+    ->Args({100000, 4})
+    ->Args({1000000, 4})
+    ->MinTime(1.0)
+    ->Unit(benchmark::kMicrosecond);
+
+}  // namespace BitUtil
+}  // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc
index 92bdcb5fc0846..4c64dea374d01 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -165,19 +165,20 @@ TEST(BitUtilTests, TestCopyBitmap) {
   memset(buffer->mutable_data(), 0, kBufferSize);
   test::random_bytes(kBufferSize, 0, buffer->mutable_data());
 
-  const int num_bits = kBufferSize * 8;
-
   const uint8_t* src = buffer->data();
 
+  std::vector<int64_t> lengths = {kBufferSize * 8 - 4, kBufferSize * 8};
   std::vector<int64_t> offsets = {0, 12, 16, 32, 37, 63, 64, 128};
-  for (int64_t offset : offsets) {
-    const int64_t copy_length = num_bits - offset;
+  for (int64_t num_bits : lengths) {
+    for (int64_t offset : offsets) {
+      const int64_t copy_length = num_bits - offset;
 
-    std::shared_ptr<Buffer> copy;
-    ASSERT_OK(CopyBitmap(default_memory_pool(), src, offset, copy_length, &copy));
+      std::shared_ptr<Buffer> copy;
+      ASSERT_OK(CopyBitmap(default_memory_pool(), src, offset, copy_length, &copy));
 
-    for (int64_t i = 0; i < copy_length; ++i) {
-      ASSERT_EQ(BitUtil::GetBit(src, i + offset), BitUtil::GetBit(copy->data(), i));
+      for (int64_t i = 0; i < copy_length; ++i) {
+        ASSERT_EQ(BitUtil::GetBit(src, i + offset), BitUtil::GetBit(copy->data(), i));
+      }
     }
   }
 }
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 4dd91e99ad9da..c77f0d008b502 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -109,9 +109,37 @@ Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t
   std::shared_ptr<Buffer> buffer;
   RETURN_NOT_OK(GetEmptyBitmap(pool, length, &buffer));
   uint8_t* dest = buffer->mutable_data();
-  for (int64_t i = 0; i < length; ++i) {
-    BitUtil::SetBitTo(dest, i, BitUtil::GetBit(data, i + offset));
+
+  int64_t byte_offset = offset / 8;
+  int64_t bit_offset = offset % 8;
+  int64_t num_bytes = BitUtil::BytesForBits(length);
+  int64_t bits_to_zero = num_bytes * 8 - length;
+
+  if (bit_offset > 0) {
+    uint32_t carry_mask = BitUtil::kBitmask[bit_offset] - 1U;
+    uint32_t carry_shift = 8U - static_cast<uint32_t>(bit_offset);
+
+    uint32_t carry = 0U;
+    if (BitUtil::BytesForBits(length + bit_offset) > num_bytes) {
+      carry = (data[byte_offset + num_bytes] & carry_mask) << carry_shift;
+    }
+
+    int64_t i = num_bytes - 1;
+    while (i + 1 > 0) {
+      uint8_t cur_byte = data[byte_offset + i];
+      dest[i] = static_cast<uint8_t>((cur_byte >> bit_offset) | carry);
+      carry = (cur_byte & carry_mask) << carry_shift;
+      --i;
+    }
+  } else {
+    std::memcpy(dest, data + byte_offset, static_cast<size_t>(num_bytes));
+  }
+
+  for (int64_t i = length; i < length + bits_to_zero; ++i) {
+    // Both branches may copy extra bits - unsetting to match specification.
+    BitUtil::SetBitTo(dest, i, false);
   }
+
   *out = buffer;
   return Status::OK();
 }
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index cab3c9ee703c9..86c17d16801b9 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -139,13 +139,10 @@ static inline void SetArrayBit(uint8_t* bits, int i, bool is_set) {
 }
 
 static inline void SetBitTo(uint8_t* bits, int64_t i, bool bit_is_set) {
-  // TODO: speed up. See https://graphics.stanford.edu/~seander/bithacks.html
+  // https://graphics.stanford.edu/~seander/bithacks.html
   // "Conditionally set or clear bits without branching"
-  if (bit_is_set) {
-    SetBit(bits, i);
-  } else {
-    ClearBit(bits, i);
-  }
+  bits[i / 8] ^= static_cast<uint8_t>(-static_cast<uint8_t>(bit_is_set) ^ bits[i / 8]) &
+                 kBitmask[i % 8];
 }
 
 // Returns the minimum number of bits needed to represent the value of 'x'

From f82b7e4f57bf6d5aa283823f483fabdda59d56ad Mon Sep 17 00:00:00 2001
From: Jin Hai <haijin.chn@gmail.com>
Date: Wed, 10 Jan 2018 15:32:11 -0800
Subject: [PATCH 1376/1644] ARROW-1927: [Plasma] Add delete function

Hi, I just add the delete function for Plasma and tested. JIRA ticked:
https://issues.apache.org/jira/browse/ARROW-1927

Author: Jin Hai <haijin.chn@gmail.com>
Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1427 from JinHai-CN/plasma-delete and squashes the following commits:

c6df5be [Philipp Moritz] rebase
424c1b7 [Philipp Moritz] fix linting
1d76437 [Philipp Moritz] fix tests
0ca115a [Jin Hai] Fixed two bugs according to the comments
ce27077 [Jin Hai] Update the unit test cases
8b6804e [Jin Hai] ARROW-1927: [Plasma] Try to fix unit-test fault
be88990 [Jin Hai] ARROW-1927: [Plasma] Add 3 test cases for delete function
baf82b9 [Jin Hai] ARROW-1927: [Plasma] Update according to the CI error
53e24eb [Jin Hai] ARROW-1927: [Plasma] Update according to the comments and CI error
c9984a4 [Jin Hai] ARROW-1927: [Plasma] Add delete function
---
 cpp/src/plasma/client.cc            | 17 +++++++++--
 cpp/src/plasma/client.h             |  3 +-
 cpp/src/plasma/eviction_policy.cc   | 10 +++++++
 cpp/src/plasma/eviction_policy.h    |  5 ++++
 cpp/src/plasma/format/plasma.fbs    |  6 +++-
 cpp/src/plasma/store.cc             | 46 ++++++++++++++++++++++++++---
 cpp/src/plasma/store.h              |  9 ++++++
 cpp/src/plasma/test/client_tests.cc | 25 ++++++++++++++++
 8 files changed, 112 insertions(+), 9 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 0dd1c44d71c74..d74c0f412d97f 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -513,9 +513,20 @@ Status PlasmaClient::Abort(const ObjectID& object_id) {
 }
 
 Status PlasmaClient::Delete(const ObjectID& object_id) {
-  // TODO(rkn): In the future, we can use this method to give hints to the
-  // eviction policy about when an object will no longer be needed.
-  return Status::NotImplemented("PlasmaClient::Delete is not implemented.");
+  RETURN_NOT_OK(FlushReleaseHistory());
+  // If the object is in used, client can't send the remove message.
+  if (objects_in_use_.count(object_id) > 0) {
+    return Status::UnknownError("PlasmaClient::Object is in use.");
+  } else {
+    // If we don't already have a reference to the object, we can try to remove the object
+    RETURN_NOT_OK(SendDeleteRequest(store_conn_, object_id));
+    std::vector<uint8_t> buffer;
+    RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaDeleteReply, &buffer));
+    ObjectID object_id2;
+    DCHECK_GT(buffer.size(), 0);
+    RETURN_NOT_OK(ReadDeleteReply(buffer.data(), buffer.size(), &object_id2));
+    return Status::OK();
+  }
 }
 
 Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) {
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index 78793f1a73a64..35182f8403201 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -174,7 +174,8 @@ class ARROW_EXPORT PlasmaClient {
   Status Seal(const ObjectID& object_id);
 
   /// Delete an object from the object store. This currently assumes that the
-  /// object is present and has been sealed.
+  /// object is present, has been sealed and not used by another client. Otherwise,
+  /// it is a no operation.
   ///
   /// @todo We may want to allow the deletion of objects that are not present or
   ///       haven't been sealed.
diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
index a7758fd2c0e74..66a3b2ea298ac 100644
--- a/cpp/src/plasma/eviction_policy.cc
+++ b/cpp/src/plasma/eviction_policy.cc
@@ -102,4 +102,14 @@ void EvictionPolicy::end_object_access(const ObjectID& object_id,
   cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
 }
 
+void EvictionPolicy::remove_object(const ObjectID& object_id) {
+  /* If the object is in the LRU cache, remove it. */
+  cache_.remove(object_id);
+
+  auto entry = store_info_->objects[object_id].get();
+  int64_t size = entry->info.data_size + entry->info.metadata_size;
+  ARROW_CHECK(memory_used_ >= size);
+  memory_used_ -= size;
+}
+
 }  // namespace plasma
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
index cebf35b1c1b08..b076309552952 100644
--- a/cpp/src/plasma/eviction_policy.h
+++ b/cpp/src/plasma/eviction_policy.h
@@ -120,6 +120,11 @@ class EvictionPolicy {
   int64_t choose_objects_to_evict(int64_t num_bytes_required,
                                   std::vector<ObjectID>* objects_to_evict);
 
+  /// This method will be called when an object is going to be removed
+  ///
+  /// @param object_id The ID of the object that is now being used.
+  void remove_object(const ObjectID& object_id);
+
  private:
   /// The amount of memory (in bytes) currently being used.
   int64_t memory_used_;
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
index b6d03b8a3c10d..ea6dc8bb98da5 100644
--- a/cpp/src/plasma/format/plasma.fbs
+++ b/cpp/src/plasma/format/plasma.fbs
@@ -76,7 +76,11 @@ enum PlasmaError:int {
   // Trying to access an object that doesn't exist.
   ObjectNonexistent,
   // Trying to create an object but there isn't enough space in the store.
-  OutOfMemory
+  OutOfMemory,
+  // Trying to delete an object but it's not sealed.
+  ObjectNotSealed,
+  // Trying to delete an object but it's in use.
+  ObjectInUse
 }
 
 // Plasma store messages
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index c6a19a5471883..dde7f9cdfa8eb 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -411,6 +411,39 @@ int PlasmaStore::abort_object(const ObjectID& object_id, Client* client) {
   }
 }
 
+int PlasmaStore::delete_object(ObjectID& object_id) {
+  auto entry = get_object_table_entry(&store_info_, object_id);
+  // TODO(rkn): This should probably not fail, but should instead throw an
+  // error. Maybe we should also support deleting objects that have been
+  // created but not sealed.
+  if (entry == NULL) {
+    // To delete an object it must be in the object table.
+    return PlasmaError_ObjectNonexistent;
+  }
+
+  if (entry->state != PLASMA_SEALED) {
+    // To delete an object it must have been sealed.
+    return PlasmaError_ObjectNotSealed;
+  }
+
+  if (entry->clients.size() != 0) {
+    // To delete an object, there must be no clients currently using it.
+    return PlasmaError_ObjectInUse;
+  }
+
+  eviction_policy_.remove_object(object_id);
+
+  dlfree(entry->pointer);
+  store_info_.objects.erase(object_id);
+  // Inform all subscribers that the object has been deleted.
+  ObjectInfoT notification;
+  notification.object_id = object_id.binary();
+  notification.is_deletion = true;
+  push_notification(&notification);
+
+  return PlasmaError_OK;
+}
+
 void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
   for (const auto& object_id : object_ids) {
     ARROW_LOG(DEBUG) << "deleting object " << object_id.hex();
@@ -626,18 +659,23 @@ Status PlasmaStore::process_message(Client* client) {
       RETURN_NOT_OK(ReadGetRequest(input, input_size, object_ids_to_get, &timeout_ms));
       process_get_request(client, object_ids_to_get, timeout_ms);
     } break;
-    case MessageType_PlasmaReleaseRequest:
+    case MessageType_PlasmaReleaseRequest: {
       RETURN_NOT_OK(ReadReleaseRequest(input, input_size, &object_id));
       release_object(object_id, client);
-      break;
-    case MessageType_PlasmaContainsRequest:
+    } break;
+    case MessageType_PlasmaDeleteRequest: {
+      RETURN_NOT_OK(ReadDeleteRequest(input, input_size, &object_id));
+      int error_code = delete_object(object_id);
+      HANDLE_SIGPIPE(SendDeleteReply(client->fd, object_id, error_code), client->fd);
+    } break;
+    case MessageType_PlasmaContainsRequest: {
       RETURN_NOT_OK(ReadContainsRequest(input, input_size, &object_id));
       if (contains_object(object_id) == OBJECT_FOUND) {
         HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd);
       } else {
         HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd);
       }
-      break;
+    } break;
     case MessageType_PlasmaSealRequest: {
       unsigned char digest[kDigestSize];
       RETURN_NOT_OK(ReadSealRequest(input, input_size, &object_id, &digest[0]));
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index a72c6259a9cea..7eada5a126991 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -83,6 +83,15 @@ class PlasmaStore {
   /// @return 1 if the abort succeeds, else 0.
   int abort_object(const ObjectID& object_id, Client* client);
 
+  /// Delete an specific object by object_id that have been created in the hash table.
+  ///
+  /// @param object_id Object ID of the object to be deleted.
+  /// @return One of the following error codes:
+  ///  - PlasmaError_OK, if the object was delete successfully.
+  ///  - PlasmaError_ObjectNonexistent, if ths object isn't existed.
+  ///  - PlasmaError_ObjectInUse, if the object is in use.
+  int delete_object(ObjectID& object_id);
+
   /// Delete objects that have been created in the hash table. This should only
   /// be called on objects that are returned by the eviction policy to evict.
   ///
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index 5cd3063bb4309..f19c2bfbdb380 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -58,6 +58,31 @@ class TestPlasmaStore : public ::testing::Test {
   PlasmaClient client2_;
 };
 
+TEST_F(TestPlasmaStore, DeleteTest) {
+  ObjectID object_id = ObjectID::from_random();
+
+  // Test for deleting non-existance object.
+  Status result = client_.Delete(object_id);
+  ASSERT_EQ(result.IsPlasmaObjectNonexistent(), true);
+
+  // Test for the object being in local Plasma store.
+  // First create object.
+  int64_t data_size = 100;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  std::shared_ptr<Buffer> data;
+  ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+  ARROW_CHECK_OK(client_.Seal(object_id));
+
+  // Object is in use, can't be delete.
+  result = client_.Delete(object_id);
+  ASSERT_EQ(result.IsUnknownError(), true);
+
+  // Avoid race condition of Plasma Manager waiting for notification.
+  ARROW_CHECK_OK(client_.Release(object_id));
+  ARROW_CHECK_OK(client_.Delete(object_id));
+}
+
 TEST_F(TestPlasmaStore, ContainsTest) {
   ObjectID object_id = ObjectID::from_random();
 

From b49e8f3765350f9b22d631fcd49181c2fbefbc75 Mon Sep 17 00:00:00 2001
From: devin-petersohn <devin.petersohn@gmail.com>
Date: Wed, 10 Jan 2018 18:47:45 -0500
Subject: [PATCH 1377/1644] ARROW-1971: [Python] Add pandas serialization to
 the default

Moving pandas register into default register.

Author: devin-petersohn <devin.petersohn@gmail.com>

Closes #1462 from devin-petersohn/jira/1971_pandas_serialization and squashes the following commits:

b3dfd5b6 [devin-petersohn] Removing slower codepath
2ed31371 [devin-petersohn] Moving pandas register into default register
---
 python/pyarrow/serialization.py | 124 ++++++++++++--------------------
 1 file changed, 44 insertions(+), 80 deletions(-)

diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 689ec15d3292d..61f2e83f3193d 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -21,7 +21,6 @@
 
 import numpy as np
 
-from pyarrow import serialize_pandas, deserialize_pandas
 from pyarrow.compat import builtin_pickle
 from pyarrow.lib import _default_serialization_context, frombuffer
 
@@ -61,6 +60,48 @@ def _load_pickle_from_buffer(data):
 _deserialize_numpy_array_pickle = _load_pickle_from_buffer
 
 
+# ----------------------------------------------------------------------
+# pandas-specific serialization matters
+
+def _register_custom_pandas_handlers(context):
+    # ARROW-1784, faster path for pandas-only visibility
+
+    try:
+        import pandas as pd
+    except ImportError:
+        return
+
+    import pyarrow.pandas_compat as pdcompat
+
+    def _serialize_pandas_dataframe(obj):
+        return pdcompat.dataframe_to_serialized_dict(obj)
+
+    def _deserialize_pandas_dataframe(data):
+        return pdcompat.serialized_dict_to_dataframe(data)
+
+    def _serialize_pandas_series(obj):
+        return _serialize_pandas_dataframe(pd.DataFrame({obj.name: obj}))
+
+    def _deserialize_pandas_series(data):
+        deserialized = _deserialize_pandas_dataframe(data)
+        return deserialized[deserialized.columns[0]]
+
+    context.register_type(
+        pd.Series, 'pd.Series',
+        custom_serializer=_serialize_pandas_series,
+        custom_deserializer=_deserialize_pandas_series)
+
+    context.register_type(
+        pd.Index, 'pd.Index',
+        custom_serializer=_pickle_to_buffer,
+        custom_deserializer=_load_pickle_from_buffer)
+
+    context.register_type(
+        pd.DataFrame, 'pd.DataFrame',
+        custom_serializer=_serialize_pandas_dataframe,
+        custom_deserializer=_deserialize_pandas_dataframe)
+
+
 def register_default_serialization_handlers(serialization_context):
 
     # ----------------------------------------------------------------------
@@ -136,90 +177,13 @@ def _deserialize_torch_tensor(data):
         # no torch
         pass
 
-
-register_default_serialization_handlers(_default_serialization_context)
+    _register_custom_pandas_handlers(serialization_context)
 
 
-# ----------------------------------------------------------------------
-# pandas-specific serialization matters
-
+register_default_serialization_handlers(_default_serialization_context)
 
 pandas_serialization_context = _default_serialization_context.clone()
 
-
-def _register_pandas_arrow_handlers(context):
-    try:
-        import pandas as pd
-    except ImportError:
-        return
-
-    def _serialize_pandas_series(obj):
-        return serialize_pandas(pd.DataFrame({obj.name: obj}))
-
-    def _deserialize_pandas_series(data):
-        deserialized = deserialize_pandas(data)
-        return deserialized[deserialized.columns[0]]
-
-    def _serialize_pandas_dataframe(obj):
-        return serialize_pandas(obj)
-
-    def _deserialize_pandas_dataframe(data):
-        return deserialize_pandas(data)
-
-    context.register_type(
-        pd.Series, 'pd.Series',
-        custom_serializer=_serialize_pandas_series,
-        custom_deserializer=_deserialize_pandas_series)
-
-    context.register_type(
-        pd.DataFrame, 'pd.DataFrame',
-        custom_serializer=_serialize_pandas_dataframe,
-        custom_deserializer=_deserialize_pandas_dataframe)
-
-
-def _register_custom_pandas_handlers(context):
-    # ARROW-1784, faster path for pandas-only visibility
-
-    try:
-        import pandas as pd
-    except ImportError:
-        return
-
-    import pyarrow.pandas_compat as pdcompat
-
-    def _serialize_pandas_dataframe(obj):
-        return pdcompat.dataframe_to_serialized_dict(obj)
-
-    def _deserialize_pandas_dataframe(data):
-        return pdcompat.serialized_dict_to_dataframe(data)
-
-    def _serialize_pandas_series(obj):
-        return _serialize_pandas_dataframe(pd.DataFrame({obj.name: obj}))
-
-    def _deserialize_pandas_series(data):
-        deserialized = _deserialize_pandas_dataframe(data)
-        return deserialized[deserialized.columns[0]]
-
-    context.register_type(
-        pd.Series, 'pd.Series',
-        custom_serializer=_serialize_pandas_series,
-        custom_deserializer=_deserialize_pandas_series)
-
-    context.register_type(
-        pd.Index, 'pd.Index',
-        custom_serializer=_pickle_to_buffer,
-        custom_deserializer=_load_pickle_from_buffer)
-
-    context.register_type(
-        pd.DataFrame, 'pd.DataFrame',
-        custom_serializer=_serialize_pandas_dataframe,
-        custom_deserializer=_deserialize_pandas_dataframe)
-
-
-_register_pandas_arrow_handlers(_default_serialization_context)
-_register_custom_pandas_handlers(pandas_serialization_context)
-
-
 pandas_serialization_context.register_type(
     np.ndarray, 'np.array',
     custom_serializer=_serialize_numpy_array_pickle,

From 99e58da5d58b8587657413b4bcca14a773d6de36 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Thu, 11 Jan 2018 18:35:05 -0500
Subject: [PATCH 1378/1644] ARROW-1979: [JS] Fix JS builds hanging in es2015

Also fixes [ARROW-1903](https://issues.apache.org/jira/browse/ARROW-1903)

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1471 from trxcllnt/fix-js-es2015-builds and squashes the following commits:

62db3381 [Paul Taylor] update dependencies and add es6+ umd targets to jest transform ignore patterns to fix ci
6ff18e94 [Paul Taylor] ship es2015 commonJS in main package to avoid confusion
74e828af [Paul Taylor] fix typings issues (ARROW-1903)
---
 js/README.md                          |  2 +-
 js/gulp/arrow-task.js                 |  8 +++---
 js/gulp/package-task.js               |  6 ++---
 js/gulp/test-task.js                  |  2 +-
 js/package.json                       | 38 +++++++++++++--------------
 js/src/text-encoding-utf-8.d.ts       |  4 ---
 js/src/vector/numeric.ts              |  7 ++---
 js/src/vector/virtual.ts              |  2 +-
 js/test/Arrow.ts                      |  2 +-
 js/test/integration/validate-tests.ts |  2 +-
 10 files changed, 35 insertions(+), 38 deletions(-)
 delete mode 100644 js/src/text-encoding-utf-8.d.ts

diff --git a/js/README.md b/js/README.md
index e58d335cd0d42..b427923e37ea1 100644
--- a/js/README.md
+++ b/js/README.md
@@ -178,7 +178,7 @@ The base `apache-arrow` package includes all the compilation targets for conveni
 The targets are also published under the `@apache-arrow` namespace:
 
 ```sh
-npm install apache-arrow # <-- combined es5/CommonJS + UMD, es2015/ESModules + UMD, and TypeScript package
+npm install apache-arrow # <-- combined es5/UMD, es2015/CommonJS/ESModules/UMD, and TypeScript package
 npm install @apache-arrow/ts # standalone TypeScript package
 npm install @apache-arrow/es5-cjs # standalone es5/CommonJS package
 npm install @apache-arrow/es5-esm # standalone es5/ESModules package
diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js
index cc33ee14497b6..d1e8046e67ab9 100644
--- a/js/gulp/arrow-task.js
+++ b/js/gulp/arrow-task.js
@@ -28,8 +28,8 @@ const { Observable, ReplaySubject } = require('rxjs');
 
 const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, format) {
     const out = targetDir(target);
-    const srcGlob = `src/**/*.ts`;
-    const es5Glob = `${targetDir(`es5`, `cjs`)}/**/*.js`;
+    const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`;
+    const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`;
     const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`;
     const es5UmdGlob = `${targetDir(`es5`, `umd`)}/**/*.js`;
     const es5UmdMaps = `${targetDir(`es5`, `umd`)}/**/*.map`;
@@ -38,8 +38,8 @@ const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, forma
     const ch_ext = (ext) => gulpRename((p) => { p.extname = ext; });
     const append = (ap) => gulpRename((p) => { p.basename += ap; });
     return Observable.forkJoin(
-      observableFromStreams(gulp.src(srcGlob), gulp.dest(out)), // copy src ts files
-      observableFromStreams(gulp.src(es5Glob), gulp.dest(out)), // copy es5 cjs files
+      observableFromStreams(gulp.src(dtsGlob), gulp.dest(out)), // copy d.ts files
+      observableFromStreams(gulp.src(cjsGlob), gulp.dest(out)), // copy es2015 cjs files
       observableFromStreams(gulp.src(esmGlob), ch_ext(`.mjs`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs`
       observableFromStreams(gulp.src(es5UmdGlob), append(`.es5.min`), gulp.dest(out)), // copy es5 umd files and add `.min`
       observableFromStreams(gulp.src(es5UmdMaps),                     gulp.dest(out)), // copy es5 umd sourcemap files, but don't rename
diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index fc959643503bd..2976d0ad45d09 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -46,8 +46,8 @@ const createMainPackageJson = (target, format) => (orig) => ({
     name: npmPkgName,
     main: mainExport,
     module: `${mainExport}.mjs`,
-    browser: `${mainExport}.es5.min.js`,
-    [`browser:es2015`]: `${mainExport}.es2015.min.js`,
+    dist: `${mainExport}.es5.min.js`,
+    [`dist:es2015`]: `${mainExport}.es2015.min.js`,
     [`@std/esm`]: { esm: `mjs` }
 });
   
@@ -67,7 +67,7 @@ const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
         (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }),
         { name: `${npmOrgName}/${packageName(target, format)}`,
           version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`,
-          browser: undefined, [`browser:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined }
+          dist: undefined, [`dist:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined }
       )
     )
 );
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
index f21aaf2364d03..ab280b092635c 100644
--- a/js/gulp/test-task.js
+++ b/js/gulp/test-task.js
@@ -34,7 +34,7 @@ argv.update && jestArgv.push(`-u`);
 argv.verbose && jestArgv.push(`--verbose`);
 argv.coverage && jestArgv.push(`--coverage`);
 
-const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`];
+const debugArgv = [`--runInBand`, `--env`, `node-debug`];
 const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`));
 const testOptions = {
     env: { ...process.env },
diff --git a/js/package.json b/js/package.json
index 3903d1eedc442..d68e7a6279e61 100644
--- a/js/package.json
+++ b/js/package.json
@@ -49,10 +49,8 @@
     "gulpfile.js",
     "npm-release.sh"
   ],
-  "peerDependencies": {
-    "command-line-usage": "4.0.1"
-  },
   "dependencies": {
+    "@types/text-encoding-utf-8": "1.0.1",
     "command-line-args": "4.0.7",
     "command-line-usage": "4.0.2",
     "flatbuffers": "trxcllnt/flatbuffers-esm",
@@ -61,45 +59,44 @@
     "tslib": "1.8.1"
   },
   "devDependencies": {
-    "@std/esm": "0.18.0",
+    "@std/esm": "0.19.1",
     "@types/flatbuffers": "1.6.5",
     "@types/glob": "5.0.34",
-    "@types/jest": "21.1.8",
-    "@types/node": "8.5.0",
-    "@types/text-encoding": "0.0.32",
+    "@types/jest": "22.0.1",
+    "@types/node": "9.3.0",
     "ast-types": "0.10.1",
     "benchmark": "2.1.4",
     "coveralls": "3.0.0",
     "del": "3.0.0",
-    "esdoc": "1.0.3",
+    "esdoc": "1.0.4",
     "esdoc-standard-plugin": "1.0.0",
     "glob": "7.1.2",
-    "google-closure-compiler": "20171203.0.0",
+    "google-closure-compiler": "20180101.0.0",
     "gulp": "github:gulpjs/gulp#6d71a658c61edb3090221579d8f97dbe086ba2ed",
     "gulp-json-transform": "0.4.5",
     "gulp-rename": "1.2.2",
-    "gulp-sourcemaps": "2.6.1",
+    "gulp-sourcemaps": "2.6.3",
     "gulp-transform-js-ast": "1.0.2",
     "gulp-typescript": "3.2.3",
     "ix": "2.3.4",
-    "jest": "21.2.1",
+    "jest": "22.0.5",
     "jest-environment-node-debug": "2.0.0",
     "json": "9.0.6",
-    "lerna": "2.5.1",
+    "lerna": "2.6.0",
     "lint-staged": "6.0.0",
-    "merge2": "1.2.0",
+    "merge2": "1.2.1",
     "mkdirp": "0.5.1",
     "npm-run-all": "4.1.2",
     "pump": "1.0.2",
     "rimraf": "2.6.2",
-    "rxjs": "5.5.5",
+    "rxjs": "5.5.6",
     "shx": "0.2.2",
     "source-map-loader": "0.2.3",
     "trash": "4.2.1",
-    "ts-jest": "21.2.4",
-    "tslint": "5.8.0",
+    "ts-jest": "22.0.1",
+    "tslint": "5.9.1",
     "typescript": "2.6.2",
-    "uglifyjs-webpack-plugin": "1.1.2",
+    "uglifyjs-webpack-plugin": "1.1.6",
     "webpack": "3.10.0",
     "xml2js": "0.4.19"
   },
@@ -134,9 +131,12 @@
       "/node_modules/"
     ],
     "transform": {
-      ".(ts|tsx)": "<rootDir>/node_modules/ts-jest/preprocessor.js",
-      ".(js|jsx)": "<rootDir>/node_modules/babel-jest/build/index.js"
+      ".(ts|tsx)": "./node_modules/ts-jest/preprocessor.js",
+      ".(js|jsx)": "./node_modules/babel-jest/build/index.js"
     },
+    "transformIgnorePatterns": [
+      "/node_modules/", "/(es2015|esnext)\/umd/"
+    ],
     "testRegex": "(.*(-|\\.)(test|spec)s?)\\.(ts|tsx|js)$"
   }
 }
diff --git a/js/src/text-encoding-utf-8.d.ts b/js/src/text-encoding-utf-8.d.ts
deleted file mode 100644
index 68ba4dfd9a346..0000000000000
--- a/js/src/text-encoding-utf-8.d.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-declare module 'text-encoding-utf-8' {
-    import * as TextEncoding from 'text-encoding';
-    export = TextEncoding;
-}
diff --git a/js/src/vector/numeric.ts b/js/src/vector/numeric.ts
index fe4767809f465..830d6082bcc4a 100644
--- a/js/src/vector/numeric.ts
+++ b/js/src/vector/numeric.ts
@@ -34,10 +34,10 @@ export class NumericVector<T, TArray extends TypedArray> extends Vector<T> {
     concat(...vectors: Vector<T>[]): Vector<T> {
         return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors);
     }
-    slice(start?: number, end?: number) {
+    slice<R = TArray>(start?: number, end?: number): R {
         const { data, stride } = this, from = start! | 0;
         const to = end === undefined ? data.length : Math.max(end | 0, from);
-        return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0);
+        return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0) as any as R;
     }
 }
 
@@ -49,7 +49,8 @@ export class FixedWidthNumericVector<T, TArray extends TypedArray> extends Numer
 
 export class BoolVector extends NumericVector<boolean, Uint8Array> {
     static pack(values: Iterable<any>) {
-        let xs = [], n, i = 0;
+        let n = 0, i = 0;
+        let xs: number[] = [];
         let bit = 0, byte = 0;
         for (const value of values) {
             value && (byte |= 1 << bit);
diff --git a/js/src/vector/virtual.ts b/js/src/vector/virtual.ts
index 6ec3a8eef9f4d..42db78706db51 100644
--- a/js/src/vector/virtual.ts
+++ b/js/src/vector/virtual.ts
@@ -93,7 +93,7 @@ export class VirtualVector<T> implements Vector<T> {
                 // this is a significant improvement as we avoid the memcpy 🎉
                 if ((source.length / vector.stride | 0) < total) {
                     let vectorsLength = vectors.length;
-                    let count = 0, length = 0, sources = [];
+                    let count = 0, length = 0, sources = [] as any[];
                     do {
                         sources.push(source);
                         length += source.length;
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
index 87641e52bf3f8..f2c4e930f92e4 100644
--- a/js/test/Arrow.ts
+++ b/js/test/Arrow.ts
@@ -16,7 +16,7 @@
 // under the License.
 
 /* tslint:disable */
-// Dynamically load an Ix target build based on command line arguments
+// Dynamically load an Arrow target build based on command line arguments
 
 const path = require('path');
 const target = process.env.TEST_TARGET!;
diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts
index c8778ba2b33c2..c612d62ad0c04 100644
--- a/js/test/integration/validate-tests.ts
+++ b/js/test/integration/validate-tests.ts
@@ -37,7 +37,7 @@ const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)];
 
 import Arrow from '../Arrow';
 import { zip } from 'ix/iterable/zip';
-import { toArray } from 'ix/iterable/toArray';
+import { toArray } from 'ix/iterable/toarray';
 
 const { Table, read } = Arrow;
 

From 9be080fc7747a701c57c86cbe48b33bd48dcba07 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 17 Jan 2018 12:21:08 -0500
Subject: [PATCH 1379/1644] ARROW-1991: [Website] Fix Docker documentation
 build

@kou something is failing here, I'm not sure what's different here vs. Travis CI (not sure if this is what's failing the doc build):

```
  DOC   Building HTML
../arrow-glib-docs.xml:25: warning: failed to load external entity "../xml/gtkdocentities.ent"
  %gtkdocentities;
                  ^
Entity: line 1:
 %gtkdocentities;
                 ^
../arrow-glib-docs.xml:29: parser error : Entity 'package_name' not defined
    <title>&package_name; Reference Manual</title>
                         ^
../arrow-glib-docs.xml:31: parser error : Entity 'package_string' not defined
      for &package_string;.
                          ^
warning: failed to load external entity "../xml/basic-array.xml"
../arrow-glib-docs.xml:43: element include: XInclude error : could not load ../xml/basic-array.xml, and no fallback was found
warning: failed to load external entity "../xml/composite-array.xml"
../arrow-glib-docs.xml:44: element include: XInclude error : could not load ../xml/composite-array.xml, and no fallback was found
../xml/array-builder.xml:25: warning: failed to load external entity "../xml/xml/gtkdocentities.ent"
  %gtkdocentities;
```

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Kouhei Sutou <kou@clear-code.com>

Closes #1472 from wesm/fix-gen-apidocs and squashes the following commits:

5b907acd [Wes McKinney] Add explicit instructions for uploading API docs to website
5734a651 [Wes McKinney] Use JDK7 for Java on Ubuntu 16.04
0fcb1e11 [Wes McKinney] Use gcc 4.9 rather than default gcc because of gcc5 ABI issues
dbf8be8e [Kouhei Sutou] Disable auto-reconfigure
b1b5050b [Kouhei Sutou] Fix GLib doc build
8b2d7e43 [Wes McKinney] Fixes for glib doc build
9da9e143 [Wes McKinney] Add BOOST_ROOT
---
 c_glib/arrow-glib/Makefile.am       |  7 ++-
 dev/docker-compose.yml              |  4 +-
 dev/gen_apidocs/Dockerfile          | 20 +++++---
 dev/gen_apidocs/create_documents.sh | 73 +++++++++++++++++++----------
 dev/release/RELEASE_MANAGEMENT.md   |  9 ++++
 5 files changed, 78 insertions(+), 35 deletions(-)

diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am
index 16d0703142798..9148f8a583a14 100644
--- a/c_glib/arrow-glib/Makefile.am
+++ b/c_glib/arrow-glib/Makefile.am
@@ -16,6 +16,7 @@
 # under the License.
 
 CLEANFILES =
+DISTCLEANFILES =
 
 EXTRA_DIST =					\
 	meson.build
@@ -169,6 +170,10 @@ BUILT_SOURCES =					\
 	stamp-enums.c				\
 	stamp-enums.h
 
+DISTCLEANFILES +=				\
+	stamp-enums.c				\
+	stamp-enums.h
+
 EXTRA_DIST +=					\
 	enums.c.template			\
 	enums.h.template
@@ -214,7 +219,7 @@ INTROSPECTION_SCANNER_ARGS =
 INTROSPECTION_SCANNER_ENV =
 if USE_ARROW_BUILD_DIR
 INTROSPECTION_SCANNER_ENV +=			\
-	LD_LIBRARY_PATH=$(ARROW_LIB_DIR):$${PKG_CONFIG_PATH}
+	LD_LIBRARY_PATH=$(ARROW_LIB_DIR):$${LD_LIBRARY_PATH}
 endif
 if OS_MACOS
 INTROSPECTION_SCANNER_ENV +=			\
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
index 4b9014894003b..a73fd1bfbbaf0 100644
--- a/dev/docker-compose.yml
+++ b/dev/docker-compose.yml
@@ -17,7 +17,7 @@
 version: '3'
 services:
   gen_apidocs:
-    build: 
+    build:
       context: gen_apidocs
     volumes:
      - ../..:/apache-arrow
@@ -29,7 +29,7 @@ services:
     volumes:
      - ../..:/apache-arrow
   dask_integration:
-    build: 
+    build:
       context: dask_integration
     volumes:
      - ../..:/apache-arrow
diff --git a/dev/gen_apidocs/Dockerfile b/dev/gen_apidocs/Dockerfile
index ca4718e637842..da740ee0773d2 100644
--- a/dev/gen_apidocs/Dockerfile
+++ b/dev/gen_apidocs/Dockerfile
@@ -14,19 +14,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM ubuntu:14.04
-# Prerequsites for apt-add-repository
-RUN apt-get update && apt-get install -y \
-    software-properties-common python-software-properties
+FROM ubuntu:16.04
+
 # Basic OS dependencies
-RUN apt-add-repository -y ppa:ubuntu-toolchain-r/test && \
-    apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y \
         wget \
         rsync \
         git \
         gcc-4.9 \
         g++-4.9 \
-        build-essential
+        build-essential \
+        software-properties-common
+
+# Java build fails with default JDK8
+RUN add-apt-repository ppa:openjdk-r/ppa &&\
+    apt-get update &&\
+    apt-get install -y openjdk-7-jdk &&\
+    update-java-alternatives -s java-1.7.0-openjdk-amd64
+
 # This will install conda in /home/ubuntu/miniconda
 RUN wget -O /tmp/miniconda.sh \
     https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
@@ -73,6 +78,7 @@ RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \
         doxygen \
         maven \
         -c conda-forge
+
 ADD . /apache-arrow
 WORKDIR /apache-arrow
 CMD arrow/dev/gen_apidocs/create_documents.sh
diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh
index 566d9cee79c85..54031262b3a5d 100755
--- a/dev/gen_apidocs/create_documents.sh
+++ b/dev/gen_apidocs/create_documents.sh
@@ -27,6 +27,7 @@ export ARROW_HOME=$(pwd)/dist
 export PARQUET_HOME=$(pwd)/dist
 CONDA_BASE=/home/ubuntu/miniconda
 export LD_LIBRARY_PATH=$(pwd)/dist/lib:${CONDA_BASE}/lib:${LD_LIBRARY_PATH}
+export PKG_CONFIG_PATH=$(pwd)/dist/lib/pkgconfig:${PKG_CONFIG_PATH}
 export PATH=${CONDA_BASE}/bin:${PATH}
 
 # Prepare the asf-site before copying api docs
@@ -38,16 +39,38 @@ git clone --branch=asf-site \
     https://git-wip-us.apache.org/repos/asf/arrow-site.git asf-site
 popd
 
+# Make Java documentation
+export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+wget http://mirrors.gigenet.com/apache/maven/maven-3/3.5.2/binaries/apache-maven-3.5.2-bin.tar.gz
+tar xvf apache-maven-3.5.2-bin.tar.gz
+export PATH=$(pwd)/apache-maven-3.5.2/bin:$PATH
+
+pushd arrow/java
+rm -rf target/site/apidocs/*
+mvn -Drat.skip=true install
+mvn -Drat.skip=true site
+mkdir -p ../site/asf-site/docs/java/
+rsync -r target/site/apidocs/ ../site/asf-site/docs/java/
+popd
+
 # Make Python documentation (Depends on C++ )
 # Build Arrow C++
 source activate pyarrow-dev
 
 export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
+export BOOST_ROOT=$CONDA_PREFIX
 export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX
+export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:${LD_LIBRARY_PATH}
+export PKG_CONFIG_PATH=$CONDA_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH}
+
+export CC=gcc-4.9
+export CXX=g++-4.9
 
-rm -rf arrow/cpp/build_docs
-mkdir arrow/cpp/build_docs
-pushd arrow/cpp/build_docs
+CPP_BUILD_DIR=$(pwd)/arrow/cpp/build_docs
+
+rm -rf $CPP_BUILD_DIR
+mkdir $CPP_BUILD_DIR
+pushd $CPP_BUILD_DIR
 cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
       -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
       -DARROW_PYTHON=on \
@@ -58,6 +81,28 @@ make -j4
 make install
 popd
 
+# Build c_glib documentation
+pushd arrow/c_glib
+if [ -f Makefile ]; then
+    # Ensure updating to prevent auto re-configure
+    touch configure **/Makefile
+    make distclean
+    # Work around for 'make distclean' removes doc/reference/xml/
+    git checkout doc/reference/xml
+fi
+./autogen.sh
+rm -rf build_docs
+mkdir build_docs
+pushd build_docs
+../configure \
+    --prefix=${AROW_HOME} \
+    --enable-gtk-doc
+make -j4 GTK_DOC_V_XREF=": "
+mkdir -p ../../site/asf-site/docs/c_glib
+rsync -r doc/reference/html/ ../../site/asf-site/docs/c_glib
+popd
+popd
+
 # Build Parquet C++
 rm -rf parquet-cpp/build_docs
 mkdir parquet-cpp/build_docs
@@ -83,19 +128,6 @@ mkdir -p ../site/asf-site/docs/python
 rsync -r doc/_build/html/ ../site/asf-site/docs/python
 popd
 
-# Build c_glib documentation
-pushd arrow/c_glib
-rm -rf doc/reference/html/*
-./autogen.sh
-./configure \
-    --with-arrow-cpp-build-dir=$(pwd)/../cpp/build \
-    --with-arrow-cpp-build-type=$ARROW_BUILD_TYPE \
-    --enable-gtk-doc
-LD_LIBRARY_PATH=$(pwd)/../cpp/build/$ARROW_BUILD_TYPE make GTK_DOC_V_XREF=": "
-mkdir -p ../site/asf-site/docs/c_glib
-rsync -r doc/reference/html/ ../site/asf-site/docs/c_glib
-popd
-
 # Make C++ documentation
 pushd arrow/cpp/apidoc
 rm -rf html/*
@@ -103,12 +135,3 @@ doxygen Doxyfile
 mkdir -p ../../site/asf-site/docs/cpp
 rsync -r html/ ../../site/asf-site/docs/cpp
 popd
-
-# Make Java documentation
-pushd arrow/java
-rm -rf target/site/apidocs/*
-mvn -Drat.skip=true install
-mvn -Drat.skip=true site
-mkdir -p ../site/asf-site/docs/java/
-rsync -r target/site/apidocs/ ../site/asf-site/docs/java/
-popd
diff --git a/dev/release/RELEASE_MANAGEMENT.md b/dev/release/RELEASE_MANAGEMENT.md
index 73eaf5f95b3b0..0f8c2202fef51 100644
--- a/dev/release/RELEASE_MANAGEMENT.md
+++ b/dev/release/RELEASE_MANAGEMENT.md
@@ -112,6 +112,15 @@ software must be built in order to create the documentation, so this
 step may take some time to run, especially the first time around as the
 Docker container will also have to be built.
 
+To upload the updated documentation to the website, navigate to `site/asf-site`
+and commit all changes:
+
+```
+pushd site/asf-site
+git add .
+git commit -m "Updated API documentation for version X.Y.Z"
+```
+
 After successfully creating the API documentation the website can be
 run locally to browse the API documentation from the top level
 `Documentation` menu. To run the website issue the command:

From 9e2fc04ddf80d2a04ec789a7ca82c2929d31f967 Mon Sep 17 00:00:00 2001
From: kmiku7 <kakoimiku@gmail.com>
Date: Wed, 17 Jan 2018 12:26:31 -0500
Subject: [PATCH 1380/1644] ARROW-2002: [Python] check write_queue is not full
 and writer_thread is alive before enqueue new record when download file.

use pyarrow download file will raise queue.Full exceptions sometimes.
jira: https://issues.apache.org/jira/browse/ARROW-2002

Author: kmiku7 <kakoimiku@gmail.com>

Closes #1485 from kmiku7/master and squashes the following commits:

8d5f905d [kmiku7] fix queue.FULL exception when writer thread write data slowly.
722182b8 [kmiku7] Merge pull request #1 from apache/master
---
 python/pyarrow/io.pxi | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 619ba365c2df7..5449872ff101f 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -312,6 +312,12 @@ cdef class NativeFile:
                 pybuf = cp.PyBytes_FromStringAndSize(<const char*>buf,
                                                      bytes_read)
 
+                if writer_thread.is_alive():
+                    while write_queue.full():
+                        time.sleep(0.01)
+                else:
+                    break
+
                 write_queue.put_nowait(pybuf)
         finally:
             free(buf)

From 1ffce26e3fc659521f42984d8e87bbb93be0ed2c Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Wed, 17 Jan 2018 14:50:09 -0500
Subject: [PATCH 1381/1644] ARROW-1856: [Python] Auto-detect Parquet ABI
 version when using PARQUET_HOME

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1477 from xhochy/ARROW-1856 and squashes the following commits:

a34ade44 [Korn, Uwe] ARROW-1856: [Python] Auto-detect Parquet ABI version when using PARQUET_HOME
---
 cpp/cmake_modules/FindParquet.cmake | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
index 0339ec56ae20a..8bbe05f127f87 100644
--- a/cpp/cmake_modules/FindParquet.cmake
+++ b/cpp/cmake_modules/FindParquet.cmake
@@ -60,8 +60,22 @@ if(PARQUET_HOME)
         PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
         PATH_SUFFIXES "lib")
     get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
-    set(PARQUET_ABI_VERSION "1.0.0")
-    set(PARQUET_SO_VERSION "1")
+
+    # Try to autodiscover the Parquet ABI version
+    get_filename_component(PARQUET_LIB_REALPATH ${PARQUET_LIBRARIES} REALPATH)
+    get_filename_component(PARQUET_EXT_REALPATH ${PARQUET_LIB_REALPATH} EXT)
+    string(REGEX MATCH ".([0-9]+.[0-9]+.[0-9]+)" HAS_ABI_VERSION ${PARQUET_EXT_REALPATH})
+    if (HAS_ABI_VERSION)
+      if (APPLE)
+        string(REGEX REPLACE ".([0-9]+.[0-9]+.[0-9]+).dylib" "\\1" PARQUET_ABI_VERSION ${PARQUET_EXT_REALPATH})
+      else()
+        string(REGEX REPLACE ".so.([0-9]+.[0-9]+.[0-9]+)" "\\1" PARQUET_ABI_VERSION ${PARQUET_EXT_REALPATH})
+      endif()
+      string(REGEX REPLACE "([0-9]+).[0-9]+.[0-9]+" "\\1" PARQUET_SO_VERSION ${PARQUET_ABI_VERSION})
+    else()
+      set(PARQUET_ABI_VERSION "1.0.0")
+      set(PARQUET_SO_VERSION "1")
+    endif()
 else()
     pkg_check_modules(PARQUET parquet)
     if (PARQUET_FOUND)

From 58a24c5ba243f89302f90ef62550cee252968c9d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 18 Jan 2018 10:56:15 -0500
Subject: [PATCH 1382/1644] ARROW-2004: [C++] Add shrink_to_fit parameter to
 BufferBuilder::Resize, add Reserve method

I also relaxed the requirement to pass `const uint8_t*` so that one can pass `const void*` when writing to a `BufferBuilder`. This will not affect any downstream users

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1486 from wesm/ARROW-2004 and squashes the following commits:

2d6660a8 [Wes McKinney] Add shrink_to_fit parameter to BufferBuilder::Resize, add Reserve method, relax pointer type in Append
---
 cpp/src/arrow/buffer-test.cc | 25 ++++++++++++++++++++++
 cpp/src/arrow/buffer.h       | 41 ++++++++++++++++++++++++++----------
 2 files changed, 55 insertions(+), 11 deletions(-)

diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index 5fd2706f0466b..398cc06363a6f 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -194,4 +194,29 @@ TEST(TestBuffer, SliceMutableBuffer) {
   ASSERT_TRUE(slice->Equals(expected));
 }
 
+TEST(TestBufferBuilder, ResizeReserve) {
+  const std::string data = "some data";
+  auto data_ptr = data.c_str();
+
+  BufferBuilder builder;
+
+  ASSERT_OK(builder.Append(data_ptr, 9));
+  ASSERT_EQ(9, builder.length());
+
+  ASSERT_OK(builder.Resize(128));
+  ASSERT_EQ(128, builder.capacity());
+
+  // Do not shrink to fit
+  ASSERT_OK(builder.Resize(64, false));
+  ASSERT_EQ(128, builder.capacity());
+
+  // Shrink to fit
+  ASSERT_OK(builder.Resize(64));
+  ASSERT_EQ(64, builder.capacity());
+
+  // Reserve elements
+  ASSERT_OK(builder.Reserve(60));
+  ASSERT_EQ(128, builder.capacity());
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 450a4c78b5bbb..b50b1a1aa041d 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -25,6 +25,7 @@
 #include <string>
 #include <type_traits>
 
+#include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
@@ -32,13 +33,12 @@
 
 namespace arrow {
 
-class MemoryPool;
-
 // ----------------------------------------------------------------------
 // Buffer classes
 
-/// Immutable API for a chunk of bytes which may or may not be owned by the
-/// class instance.
+/// \class Buffer
+/// \brief Object containing a pointer to a piece of contiguous memory with a
+/// particular size. Base class does not own its memory
 ///
 /// Buffers have two related notions of length: size and capacity. Size is
 /// the number of bytes that might have valid data. Capacity is the number
@@ -133,7 +133,8 @@ ARROW_EXPORT
 std::shared_ptr<Buffer> SliceMutableBuffer(const std::shared_ptr<Buffer>& buffer,
                                            const int64_t offset, const int64_t length);
 
-/// A Buffer whose contents can be mutated. May or may not own its data.
+/// \class MutableBuffer
+/// \brief A Buffer whose contents can be mutated. May or may not own its data.
 class ARROW_EXPORT MutableBuffer : public Buffer {
  public:
   MutableBuffer(uint8_t* data, const int64_t size) : Buffer(data, size) {
@@ -148,6 +149,8 @@ class ARROW_EXPORT MutableBuffer : public Buffer {
   MutableBuffer() : Buffer(NULLPTR, 0) {}
 };
 
+/// \class ResizableBuffer
+/// \brief A mutable buffer that can be resized
 class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
  public:
   /// Change buffer reported size to indicated size, allocating memory if
@@ -190,13 +193,22 @@ class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
   MemoryPool* pool_;
 };
 
+/// \class BufferBuilder
+/// \brief A class for incrementally building a contiguous chunk of in-memory data
 class ARROW_EXPORT BufferBuilder {
  public:
-  explicit BufferBuilder(MemoryPool* pool)
+  explicit BufferBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
       : pool_(pool), data_(NULLPTR), capacity_(0), size_(0) {}
 
-  /// Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
-  Status Resize(const int64_t elements) {
+  /// \brief Resizes the buffer to the nearest multiple of 64 bytes
+  ///
+  /// \param elements the new capacity of the of the builder. Will be rounded
+  /// up to a multiple of 64 bytes for padding
+  /// \param shrink_to_fit if new capacity smaller than existing size,
+  /// reallocate internal buffer. Set to false to avoid reallocations when
+  /// shrinking the builder
+  /// \return Status
+  Status Resize(const int64_t elements, bool shrink_to_fit = true) {
     // Resize(0) is a no-op
     if (elements == 0) {
       return Status::OK();
@@ -205,7 +217,7 @@ class ARROW_EXPORT BufferBuilder {
       buffer_ = std::make_shared<PoolBuffer>(pool_);
     }
     int64_t old_capacity = capacity_;
-    RETURN_NOT_OK(buffer_->Resize(elements));
+    RETURN_NOT_OK(buffer_->Resize(elements, shrink_to_fit));
     capacity_ = buffer_->capacity();
     data_ = buffer_->mutable_data();
     if (capacity_ > old_capacity) {
@@ -214,7 +226,14 @@ class ARROW_EXPORT BufferBuilder {
     return Status::OK();
   }
 
-  Status Append(const uint8_t* data, int64_t length) {
+  /// \brief Ensure that builder can accommodate the additional number of bytes
+  /// without the need to perform allocations
+  ///
+  /// \param size number of additional bytes to make space for
+  /// \return Status
+  Status Reserve(const int64_t size) { return Resize(size_ + size, false); }
+
+  Status Append(const void* data, int64_t length) {
     if (capacity_ < length + size_) {
       int64_t new_capacity = BitUtil::NextPower2(length + size_);
       RETURN_NOT_OK(Resize(new_capacity));
@@ -248,7 +267,7 @@ class ARROW_EXPORT BufferBuilder {
   }
 
   // Unsafe methods don't check existing size
-  void UnsafeAppend(const uint8_t* data, int64_t length) {
+  void UnsafeAppend(const void* data, int64_t length) {
     memcpy(data_ + size_, data, static_cast<size_t>(length));
     size_ += length;
   }

From bc9f9e532ea2a16810d5ce14e1dfc3272628cb95 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 18 Jan 2018 11:01:23 -0500
Subject: [PATCH 1383/1644] ARROW-1966: [C++] Accommodate JAVA_HOME on Linux
 that includes the jre/ directory, or is the full path to directory with
 libjvm

Some users ran into a rough edge where they had a non-standard JRE directory (possibly related to some recent changes by Oracle in their JDK installer)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1487 from wesm/ARROW-1966 and squashes the following commits:

7e14923d [Wes McKinney] Add note to API documentation about JAVA_HOME
f77b31e6 [Wes McKinney] Accommodate a JAVA_HOME containing the jre/ directory, or an absolute path to directory containing libjvm
---
 cpp/apidoc/HDFS.md                | 4 ++++
 cpp/src/arrow/io/hdfs-internal.cc | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/cpp/apidoc/HDFS.md b/cpp/apidoc/HDFS.md
index d54ad270c05f4..d3671fb7691ba 100644
--- a/cpp/apidoc/HDFS.md
+++ b/cpp/apidoc/HDFS.md
@@ -50,6 +50,10 @@ export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
 * `ARROW_LIBHDFS_DIR` (optional): explicit location of `libhdfs.so` if it is
 installed somewhere other than `$HADOOP_HOME/lib/native`.
 
+To accommodate distribution-specific nuances, the `JAVA_HOME` variable may be
+set to the root path for the Java SDK, the JRE path itself, or to the directory
+containing the `libjvm` library.
+
 ### Mac Specifics
 
 The installed location of Java on OS X can vary, however the following snippet
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index 9cd1c5052fe8d..545b2d17d2e78 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -147,7 +147,7 @@ static std::vector<fs::path> get_potential_libjvm_paths() {
   file_name = "jvm.dll";
 #elif __APPLE__
   search_prefixes = {""};
-  search_suffixes = {"", "/jre/lib/server"};
+  search_suffixes = {"", "/jre/lib/server", "/lib/server"};
   file_name = "libjvm.dylib";
 
 // SFrame uses /usr/libexec/java_home to find JAVA_HOME; for now we are
@@ -175,7 +175,7 @@ static std::vector<fs::path> get_potential_libjvm_paths() {
       "/usr/lib/jvm/default",                     // alt centos
       "/usr/java/latest",                         // alt centos
   };
-  search_suffixes = {"/jre/lib/amd64/server"};
+  search_suffixes = {"", "/jre/lib/amd64/server", "/lib/amd64/server"};
   file_name = "libjvm.so";
 #endif
   // From direct environment variable

From a11da7f914e117675d9e662fa7b5f78dd02c61ed Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 19 Jan 2018 12:46:40 -0500
Subject: [PATCH 1384/1644] ARROW-2005: [Python] Fix incorrect flake8 config
 path to Cython lint config

This was silently passing even though the config file was not found. This first build should fail, then I will fix the flakes

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1488 from wesm/ARROW-2005 and squashes the following commits:

4305e1d1 [Wes McKinney] Fix Cython flakes
894d4ab3 [Wes McKinney] Fix incorrect flake8 config path to Cython lint config
---
 ci/travis_lint.sh         |  6 +++---
 python/pyarrow/_orc.pxd   |  8 +++++---
 python/pyarrow/_orc.pyx   | 12 +++++++-----
 python/pyarrow/plasma.pyx |  8 +++++---
 4 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
index e234b7b015b8d..6a2a0be18cf9f 100755
--- a/ci/travis_lint.sh
+++ b/ci/travis_lint.sh
@@ -35,10 +35,10 @@ popd
 # Fail fast on style checks
 sudo pip install flake8
 
-PYARROW_DIR=$TRAVIS_BUILD_DIR/python/pyarrow
+PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 
-flake8 --count $PYARROW_DIR
+flake8 --count $PYTHON_DIR/pyarrow
 
 # Check Cython files with some checks turned off
 flake8 --count --config=$PYTHON_DIR/.flake8.cython \
-       $PYARROW_DIR
+       $PYTHON_DIR/pyarrow
diff --git a/python/pyarrow/_orc.pxd b/python/pyarrow/_orc.pxd
index 411691510423c..c07a19442b577 100644
--- a/python/pyarrow/_orc.pxd
+++ b/python/pyarrow/_orc.pxd
@@ -29,9 +29,10 @@ from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus,
                                         TimeUnit)
 
 
-cdef extern from "arrow/adapters/orc/adapter.h" namespace "arrow::adapters::orc" nogil:
-    cdef cppclass ORCFileReader:
+cdef extern from "arrow/adapters/orc/adapter.h" \
+        namespace "arrow::adapters::orc" nogil:
 
+    cdef cppclass ORCFileReader:
         @staticmethod
         CStatus Open(const shared_ptr[RandomAccessFile]& file,
                      CMemoryPool* pool,
@@ -40,7 +41,8 @@ cdef extern from "arrow/adapters/orc/adapter.h" namespace "arrow::adapters::orc"
         CStatus ReadSchema(shared_ptr[CSchema]* out)
 
         CStatus ReadStripe(int64_t stripe, shared_ptr[CRecordBatch]* out)
-        CStatus ReadStripe(int64_t stripe, std_vector[int], shared_ptr[CRecordBatch]* out)
+        CStatus ReadStripe(int64_t stripe, std_vector[int],
+                           shared_ptr[CRecordBatch]* out)
 
         CStatus Read(shared_ptr[CTable]* out)
         CStatus Read(std_vector[int], shared_ptr[CTable]* out)
diff --git a/python/pyarrow/_orc.pyx b/python/pyarrow/_orc.pyx
index 7ff4bac6dc95f..cf04f48a32319 100644
--- a/python/pyarrow/_orc.pyx
+++ b/python/pyarrow/_orc.pyx
@@ -50,7 +50,7 @@ cdef class ORCReader:
         get_reader(source, &rd_handle)
         with nogil:
             check_status(ORCFileReader.Open(rd_handle, self.allocator,
-                                             &self.reader))
+                                            &self.reader))
 
     def schema(self):
         """
@@ -69,10 +69,10 @@ cdef class ORCReader:
         return pyarrow_wrap_schema(sp_arrow_schema)
 
     def nrows(self):
-        return deref(self.reader).NumberOfRows();
+        return deref(self.reader).NumberOfRows()
 
     def nstripes(self):
-        return deref(self.reader).NumberOfStripes();
+        return deref(self.reader).NumberOfStripes()
 
     def read_stripe(self, n, include_indices=None):
         cdef:
@@ -85,11 +85,13 @@ cdef class ORCReader:
 
         if include_indices is None:
             with nogil:
-                check_status(deref(self.reader).ReadStripe(stripe, &sp_record_batch))
+                (check_status(deref(self.reader)
+                              .ReadStripe(stripe, &sp_record_batch)))
         else:
             indices = include_indices
             with nogil:
-                check_status(deref(self.reader).ReadStripe(stripe, indices, &sp_record_batch))
+                (check_status(deref(self.reader)
+                              .ReadStripe(stripe, indices, &sp_record_batch)))
 
         batch = RecordBatch()
         batch.init(sp_record_batch)
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index 29e233b6e4e67..32f6d189da08c 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -248,8 +248,8 @@ cdef class PlasmaClient:
             check_status(self.client.get().Get(ids.data(), ids.size(),
                          timeout_ms, result[0].data()))
 
-    cdef _make_plasma_buffer(self, ObjectID object_id, shared_ptr[CBuffer] buffer,
-                             int64_t size):
+    cdef _make_plasma_buffer(self, ObjectID object_id,
+                             shared_ptr[CBuffer] buffer, int64_t size):
         result = PlasmaBuffer(object_id, self)
         result.init(buffer)
         return result
@@ -302,7 +302,9 @@ cdef class PlasmaClient:
             check_status(self.client.get().Create(object_id.data, data_size,
                                                   <uint8_t*>(metadata.data()),
                                                   metadata.size(), &data))
-        return self._make_mutable_plasma_buffer(object_id, data.get().mutable_data(), data_size)
+        return self._make_mutable_plasma_buffer(object_id,
+                                                data.get().mutable_data(),
+                                                data_size)
 
     def get_buffers(self, object_ids, timeout_ms=-1):
         """

From 305b54ce0ca092fb243e3c69e5fb186cdd90266f Mon Sep 17 00:00:00 2001
From: Justin Dunham <riboflavin@justindunham.net>
Date: Fri, 19 Jan 2018 12:49:13 -0500
Subject: [PATCH 1385/1644] ARROW-1872: [Website] Minor edits and addition of
 YAML for versions

Hi, in this version I've:

* Added a YAML file for tracking versions and outputting related links etc.
* Done some minor cleanup on HTML and images
* Added a small paragraph about what Arrow is to the front page

Author: Justin Dunham <riboflavin@justindunham.net>

Closes #1483 from riboflavin/master and squashes the following commits:

9ffb5263 [Justin Dunham] fix responsive layout
fece5578 [Justin Dunham] update front page description
6174a5fb [Justin Dunham] small image cleanups
173ead2c [Justin Dunham] slight improvements to mobile and html layout
fe9007d5 [Justin Dunham] add version vars, small amount of copy, and clean up formatting slightly
---
 site/_data/versions.yml |  29 ++++++++++
 site/img/copy.png       | Bin 23204 -> 64271 bytes
 site/img/copy2.png      | Bin 37973 -> 0 bytes
 site/img/shared.png     | Bin 37973 -> 20925 bytes
 site/img/shared2.png    | Bin 23204 -> 0 bytes
 site/index.html         | 118 +++++++++++++++++++++-------------------
 site/install.md         |  20 +++----
 7 files changed, 100 insertions(+), 67 deletions(-)
 create mode 100644 site/_data/versions.yml
 delete mode 100644 site/img/copy2.png
 delete mode 100644 site/img/shared2.png

diff --git a/site/_data/versions.yml b/site/_data/versions.yml
new file mode 100644
index 0000000000000..0d04183868dcf
--- /dev/null
+++ b/site/_data/versions.yml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Database of contributors to Apache Arrow (WIP)
+# Blogs and other pages use this data
+#
+current:
+  number: '0.8.0'
+  date: '18 December 2017'
+  git-tag: '1d689e5'
+  github-tag-link: 'https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0'
+  release-notes: 'http://arrow.apache.org/release/0.8.0.html'
+  mirrors: 'https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/'
+  mirrors-tar: 'https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz'
+  java-artifacts: 'http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.8.0%22'
+  asc: 'https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc'
+  sha512: 'https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512'
\ No newline at end of file
diff --git a/site/img/copy.png b/site/img/copy.png
index a1e04999eb3fd3bd7c350a0850659740d4a8cd03..55ff71ece1e597f51a5038757b6604e7dc82aeba 100644
GIT binary patch
literal 64271
zcmeFZg<D)f&p3*^yA*ea;tqx4u*KbryW8Sc+=^?9ySpu3ba8j5xI0|>e&0{;bN_?8
z^XzlZ$z+n5%;Zdx$!^3qC23S7A|wb12vk`aNfihPC=mz<ND&0M4@k^15y!^`#zI_C
z90H;?4*A6x_T!n{R7OP+0>YaX0wN#;0^;cd6mSRu;l>UD@z)3fLLdzS0^c#KO<Cxp
z0?|=M+XVsw8T;QC5+Wmu;Dh>{m712TmZE}yse>J>@plIkGgePK#}8}>2q8~_k6Sx4
zS7UNdJ6n4f0Z(Dd|6mAw-2Vf!QIh`$#nncbQcLk0xrBqW896sA7b`oZ2ogCtxsdaB
za{(1essC>N@gz)X>FVkzz{cj`;lb*`$?D*2!N$SQ&(FsGneFpumJbXT7cYBPV^0=)
z7pngv@;~TEnz@)dTRFN~IoOl`qiby9;N~h!N%@b_|NQ({KV7ZN{}(5Fm;WBt#{k*>
z)v$4}va|h9+8<4Y{y_yK9PAvO&0JhQ=!<X*{Ri{^fc-B%|Ah~5wle$Z>A(JnaQrv^
z{|@`V@xNI*xH^1D&DqLS*51|3`GeVixBmR!t^dC<{+pH%+rNSTZ-V$QDgOuhA!ZRI
zA-4ZHWg<vX0GK)m2r&p*NpUq#$dhdNJbm@_riIq~qJ$hVZ*oM$UmoIO-k3E3ILIlG
zB*!KB`8&}hLmOu$`HM_H$iK$Zc2aewVnE5#Qy@}PbROhfi-Nm6^m)Qvd=$@~67SDC
zH8(ci=5@~B<^>Ky;;F63|Nrv;OW^;IKwcdjQgX1W<+K=h8kBC4QZ{j-t@t;&@{;g}
zqxrAHUqdpTzFAAAM6*7`+njv$#Y@uXzMqaJlHd~mkK$bqmpD|+EBZRVyAJ5AQR_S$
zVD|*^{Ic*O<iUP*bLF&&`gPaFZ)_KL`Lub;Z!SmhO&_;R@X0{ct8xLf{PmZ0=B4r8
z*K_9CCJ~R31g^<PqLQN*Mio?9=+hZcBDb;1_@Sczs+O+SQdgqN*`0MBj-2{0wKV*R
zibqGE(Z4k>%;h-XrJKHGm9+j$J<1`HqQxd$HO@9J&&wyB)1A_$<alQqRM#pvtji<4
z*P>}bG3de?C#~*ZzG&<5mH>Fmdf%^VJ<Qe%4!|zoLf0>6U--(}3~y&uehX*kt<K{v
zc{m~*r>2B8vues^b4|Jt$@CW=yJlNk=fjmD|I%s?)O<wET-N$BGOtWTk5fipR#TNf
za$MppS2+E&4fmov(9kxmEMRBD-$-l?HTtWtX&>;I2q&Nm9r57W6sGz}k`Ea^r9vbx
zD3yLnUl=bOOnOi5DB0s}sZp2UF0_g^vb<A~AAH7yTR&h)j+x7{uWY}?yUp_w<e%tT
z4F`lP{J>AML6)gYIOC3OTlUFvyMZYz;JKkjJ#1D=<cj|&lSrrqnok<^Of-r_GDL^Q
z|Ar7Q96y^?5SUXPcq6xQovUnZJS{dYX8Ny+W4z#^Ag~}%iYs17B!-?AmJXE~)fVof
z%7W8VrZhqp`hOLY{6I;dfK*)R&w@aMg20B^xhPWiwz+~p7A5E-Rejx6Dn;H|QdBja
z{-ZK0k$Kfp)YI1DSn$fV?&9VBY8w{u%s1d*CDUnRDGQaM-{Y-~YH|5L-p;7h+p1Lw
zu4^{jo6T7FkV@2%Nl}aaxh8%^9Cg?Qzhd=>1%73q(SNosfsR~;iuxw#`<O4hH*j#g
zz;rc3#DZ0?8)lgOy^A^Xfc$;fw~P6Ldx3>fh(?a+aNyq<@kL);kE`cEkuemj^*J>U
z_3*4P!NJdAlesz5VhRSCl@e7I)(nTxO@HID1nc8%k<P}Dcvk9Yn2u|51#9!UQEfxT
zD9auZ=Y+jLx;FB9ycPhgGyKQFt73jqxh@fchz<QN7+ce?KZ@EAqRYB@m}UQocjg~!
ztJi`>gWr(d7ar4j80!K}7-3&rAJU_1G9qjcMJF6*>QuC#8Hh&u=d?F|Js`Pd^R!~j
zvlFjBt00t@06zcxemxzqo>p0#8y80(atZzAFnk;Xhvx!l7=|*gr;X%qh}$b3-J#1Z
zUeq`Qx$Dccr$9i^dy6b*Oj2Q?_l~hHT8Lqvu89O@K{s~o#t>mYowGUR`G56pBkP_J
zzn8FGGgSSJl>nxn_a3z)gRG{2fHg-Q30Y%7(PV;!`z4~kAOHE~D%8n|Kd6;SSAYp1
z;V4xV?O_7UkUd>2RX0ZL=DqD_8mP&Vt;;~l`a@Jc<j#c)aHWgjrYkK7oLd|te(;*O
zH<DgK>xpWfKQKf(Ml>AZ6#i~U@u{N*ht<0aXZ~vJ*#_bVSUJ<lYLr}95bL6^7_V0(
z{!-AP0(DeFY=T0Z?p%i?$Do!$C^YIDr7PBgi-sK%*s)TR;r|*{L15)#TJ7!TmzI5T
z+H~#?r)wVJl;PveQ)J_Q(3gkD&Vd>Pl>(tqx<hRXI@<E=^t{0u9L!E}boQmwFY9KW
zcBG|%Zg?xb?LS%=@WWH`{i3!V6;W<6;suurqC4yHF(F`zm~Za+V^cr3(4}O;crZkh
zTD%6an~)!4;!fEC2S3zUWNA=k44$qXrMHFk>T7XW(|cUzcsz}Q$r)y(@Oly3HA0nx
zBe$61QhK?Nv`1Zl=@xvkuA|0((iP+1sH-g47()GQue9Zb;`nX`_`+0_$kGzKq5obj
zV4VC$-+Ix~E~G82`KLdh|LIRryz7z=cTI%sQ0Yz`f1x9>!@|xH$&QVjF_XIxS&hD1
z%SnWZ7K*YLX|K!GAx2X$^<$9l`?N}0W8!f5n77pg0e&w_)baX@DEm*YF|GU0fx_tN
zpJS5ds5)NBAWOYB5K_62w$4(I*B@vEBiD-{XZ@jWZ2`ipzCssr3_`u15zEPzz8`&m
zI;*8npz$NI^A9uoncmPYvpzv5>UOH&Z&(gF9O2`^V?<TWjJ$;$pQ&uSOceV|A>_}A
zPyx<*t*JVnNqE@>!@P>WC~_~7?6RpALfo~jM_;~+0J<l4!4Vm=m&lU}12Ywn=8jUk
z$@VvqU(}pUX9Q`b;D09n`k%sikrd)|N<t^X(#7>)L}sUz%(Hvx4~s-X<A*pd;@sIx
z>dtfY0G-3XEC#L><a=J9#ig2UpoThbnGRfFv@7u@-oN^2hWctvEdJ4bxRXe~NgKP<
zQAbGql0|C5Zvwe{<ZJqc&)%CNI=SJ3M>HhD0Rww?v;~;ezxsZ8Hym1Xoz~_JccsyN
zEZ3o2AGuW&N`XTCX15Yrv(aD1p5}r$b3qeY^WC0fEI8z`^!G%P5=+N*##$(ijXd|_
zB>jnmLoV2r8~k`L)SmIz<e|B;yjPb(G&ffnr5J@N#MGTRB)m3VHVe>;XV7E7s-=bU
z98-=IewD@6(nR2hXQop1dUed&F|MXQ4Kgo<3>%~V_xAQewH=V$`@F!TpeHT<P08r`
z5ZJ*GrQNFohG~L!*kxm73-!ri3GHH!=DcIA!Ov$L(^W^Q_j+Bo_k{eShQK?<8dm-&
zGXX<e?{Pl(mOZEL85RkdBbie+36BmEKDkW`^{|`_P-*J_dbu>R@a->Ir#eMP$8Vgo
zw7$d@qaPDWp^iegdiTy8CtuRf-nCkbaNZuiaiA<kURgGW6kpSQ(M>O)Ap#|&3Iif0
z$)#3aDF(synkQZe7gazmdGt7H4<bo#nfEVgqi<C!xu!0HFrGMDD;KMDF%?K`$p$nc
zETz1KvLCyhrQXLz%T<+$GlR^%nK-TM5%6s@*{O~wT-uynULRjngLuDIjXNab7^*{X
zVN-o;kHlFGNxB~1@^keL6$L40`s~Yka<SI$7Dy9U{3E<E)=<;ppS=Vfq7oE>&pQL@
zGEc<0%wgxE8c2X`#59E>k2AII)?{x@)%O9;C}TMJPXO^ETJaFH+>rb5y;>`U6!ii!
zP=+>Hhtk=7%dXiE!o3rPIClmXS@3j#n(|67M=FbYD~@^Ev;PeoVKb4(_Bk12XseJk
zpGVqQaj#xeh1m`UZNfi;J};On{s#b_;WKW%nI&l062U&HD;}P=xC<l_<1eI5Y`;C1
z?L~jFmCl}PaBbGXq8^VaIzT0HRk<b<8FEB}Fv%33x|X(x$DY8Fh=J)4z7pswJW9?5
z*g0|v9-N9#E!Q3_bE}6l;fT8)^v97rGTlM`MRv>lYaOco*Idl*GEIGWZZ+z&LJ8XW
zax>$n#xfl9F$=4+0Tp@`rB!k}uTBY|voOH3p_+olUWP0PGup^74*a|{SNN^djp8|E
zA$p+D5!R<6Hek{GJH3Zs#KZ2-1f?Xn<x<F@t^cgiTcUV&y;!<^8lIm5vMgaQLMXCJ
zOS~@A3g*2}lLjZ{TTlF58y4K8B#0`N8#1X$Z~g7)7A(6%oupn)k#|1&Jj!{ktA9Ru
zlzVrkv?OW4c_BZGgmJobi3zjS2evh(XpJOlJ-&YF_RmtBW?Kj&(WYI(W}DvCUy{*R
zMd|iJI>%ZsUZ=6HvWPIqYO9tkq57QoJu6@<Tc|pMS$!g)fYoSi=D<p4jQ#|%^whVY
z(L%U+HiN|w`u4m;_bsMKP=Uy@6mI{Ex4p6IwS=ct%1-4{)bJm`EMLSa?=OrySWU55
zzjt;rO>^3MY_lZIsvukK;0T3FV$T@Orvy*9+zTQ&j`YFIH@PY&`cS~PA~Fx2-1Y1o
zEs>R!y&B!cikIi-$gME^h<ra$?jl|zq%*qv8{oZ{2&^l<PXxJOpjk7?pX(sWGS$L^
zglMJCwL6@N>)v$yGEWVkt5=bYurX53&PafDa&PY9$(bev{c<?ZKI!K-i$9dpx&QVM
zv<0!N2iF!k)xgYhIBMJx9*k@f{d}=q3@rSCLv>(J-tPB}DWD1RIXaWlNuzN(G9dcW
zZ%E0|ye6aUNg;$kiqO!+{hI2prkKe(<l~ZQ`<Dk?VjcWit2QxDT(azNi%>|Mh9JVp
zbK50oF!~F9g>kMjuGJ0B5hg(o9SAK`4Oc`#*oNG)?~=SD<lbtKQiM=#-?$EugsIi%
zcrl3gd3rV)T#OF3{tP~=<5j=x6k!&{GEiK!>_WGO=5Zx_Znifsb@^m$JJoCcXUGL?
zld?IcTOz!*Ioe-GBg0X-u$<kL3D(UOBKhaiFj@wjrTo(?knFx$H@t-(2eYYHWGe%9
zU;YD2J1-d7_&i~tMtBsL_;PH_obd0HMWk_@GH{2p!VH3QH5xz4SS0t!*|f|?TlL5c
zT=VEjMQz6I75Vc$GdyZ58<^!Qh%oxCgO_)6sg+P=hm=xe442hJ;%2jJfIR~6BgQ27
zHVGp`Fvm<30Xu8dVqSf`NVD_HYrx}~cZx%rU)5U_mTCS>tvvT;@!ymxmeAsdh=}2n
zsf@7HVu-EV16l)fmyX=+_9U9B=Ow`==ylS>E32Ni8|0-8P8q!5L&z&8q<v#UfKk$l
zGmE$m62*L1%9WMK@zwoa4)q)E9g^&iMCuPSpQ}1>Ry|`uS#D~ai0|(Ss?QwC(e>tF
z@)&70eU~sg6mYv^s@Wp7eYO7t0N--6+s~OQEJo!UYJ`=<hMI>GR7X>~xXQ=Sqh2Rz
zJC102i4)V;zPlc4aruOS1NWusMQss@HB5P5O9)^eX)wmroxAK@I##B4HqFTu^wK0f
z$DF6qsr=fyKgpny$5E)_G#^8D$l0mNXleROI;Z~f@{9T{D~*S{M?GK3`~^xFWNreu
z*P=rIkwobQN^UVWoIZhrRUS-c*EjEzjkw_Lu7{dLPRI&}1qU9^X<YC-S5xJYPh?FN
z0oKkwB@=IWUc*1F&lKVok0iqBlt2zm{^!MG2~r#Yi;>fc%TS9ahg6Ra-)-u>BJBpl
zn6{+cFbDp8pen-3(#D9d<iD4u#ZCM!D!b|A=u70UV<O^p7B~s|)q6L4l$ZfCC=eDu
zk;~y3;u+NES%qqNKK5toH$K)o#-*+Fh+p=KLk5rHJ>ZqABsJ2UYS;^r*b@gjA@4#4
ziYdbwhK|b9(c2q1FQPeftbe0zEt)`&nZ=U%cQ}dOF3LA;z3#Pt1Exy*b!=oxiuU4U
zW`Urj@uT<SHUVde|CX^BD&J8{^AHUjUAi3Q(o&eL1=qKZJ4Iid1=8_mVFQBwa=5rS
z$$n3oB91lBUfR!ye+!)`U7ghwOk4Lac|kbUp|ac3+0s}RM19QQ&say|m}~SR@QA~n
zImi7^2Uad9_x#U5YBpsdw_$6jqnPU@DUSAyk#T%}Ugl1nJ2ZacWJ>^8f2iXq?VktQ
z^5K44>G3iYrCN!ZY*kwdZgJ5(Mlcm|Gv_=HXL;K5=g7;A(5y(Sk>wP!$*Y#;012_E
z5jB1SvpFhUV$D!bv*dVIROXc4mi8SAra2>zm<9$cKJ6%EHcwq01ND!D{%weLA52Os
zQgN2$xiRR{itM{pGJsld9gTatGtC8ADFwr+OB@Glsy0Inoi=h-m0=ljhNKItZVRi%
zWi(bs*K+}*VA=mGx)J&>Y=9fbzxMr~|MuS{fas7yULSax88s*C9R`Qwm)B;2i2B#b
z&bl#7Zqdue)#iyl`Fh}ibv|ar)hFelF_Avs_q5DMWL-;s)Jo^Q(sx~ZoS)cA{_R0)
zIf*xAYS2_Zd0}vSB}Xxv`iPL|PmbjMIHz#PLteo5MBzSiA0ws*>y$6K*7OpMWZUa{
zu+aC!sINTjDU04bKC=J{(mJ4d^bFDgFyWgo4*p}y#7x3Cy<J)5dxiGF+{|)f>$6Cx
zwc83Fap^)fS!tq?cM__1)ofzhO(Aoq_RCf9gWlg69|qBix(Ylw4H$oBww)9>PXNnU
zF2H<6N7X~bK@eQ-2}AQ_D;Rb3aFT3DD$tdqW)im`c8{I=r}xqKYeTZzD`|BjOIsj%
z%}Z-F8Qx!I@RwBbjqGNAf1UOpr3)}!o#|e4=<(bXuw7I&mK*&EJutmOf0$Kl3<d}b
zwU|?Qsthz#2hPc7QHbgiBI@K*DvygUSxXt)7v^8A8huN23nS5oE|Y8=Lo-lE@$kx?
z(UE-N(br*Wt-fE~@#91`+GyS~%-f^*!lACH!47*WhBI6V|NQ(rWtrKI;WF-=`9hGA
zH$|>6<LUbMT@VG=bjk^u!E<=s!?<3?cf?g~eB_La1%B_0yzXbKd6Ds(&YbxZmO9~O
zXFKzBv$dk~v!>_MBx2FOS&DS(hH7bpi>jHv+OmaC>MoTyq}+3QK-Hs+(G@))v!FjS
z7`u9hBe#hPtgF&x^%Lo-;;dP7166*(&}*mr#;1W*8T~=G^QiE4wA>F8eN(fPN42zU
zsl|5U^_+~|b?o(3I@))z-O^UdM2+5FuM>FFR(O4fH0QfuuhVR<uu&N%%=vxg)Pb#v
z32jx4ULzFsn3u)(YMz%vT#2l9bW8J|MBpWTday>lWUc?aUT|Zsz5of_B1|-<rA5#3
z`<Cmv(Z$zi#Lm!QrS7OXbKZdnnImfc)jFXS!*$|^g!dnq0w>i{fR>>?%3OIWb6S`C
z<#l+a=l=S#vEV0HyYai{j{asb&4K9$NUR5h5o*9MfL6WnV<bcMSj&;;4ZBa~T5N0M
ztb>?ask`d0vNE`!5q#wdpQ}IHgSQg!4(%&@j+C!#F_HBU6CfzC&w*ljiV~@iSG7EE
zr9P+a>8Q)btjEQV;wN-s!SE;D7i#&&MsE4st^-x7qV6MUwY5ETtG?-JazB8XUq0_D
zDyD;o!|Q~4u(eW&KH0<B32gz*js^A-la!lOIxds7g;7<mS_bj5wzq_Po8|3bV<A5;
zt~IMQ!pU~h8J)-~y8|a#IiZ%X>(u!pV7ij%urCg_(om!C9xI$w0%-q~T4_)q{61e9
zd6YJSo}X3^Qe>fzHM~F)g>wm5pXAsC8?pXyhM*&ZxCRYuGE8zi(~=f(67SjT&xC1>
zrvCx~EMFWtYzLG(W8uOfx&--Wc9>%if{JABxV8bXsR7rZQ*yRvN8?rzi(GAa$u}2)
zMQk7;3XCB%dH}Mx4Xg@Y7xD|qngLs5<J|KtvD$6r__9SusL%kTpQ*)6n_{z*U}yG2
zNHXz*+0k~`Hr|yb@5T!Kr2%nd2R_Xa3Tg}Q;BXU7Z?y+3BAKX5x?<J&{sYs6@McKa
zD(r*Rs3RO7_9_G`EY`wAGH0f4)$!_ILsjqNm0k~$@J`S#Bg)P8rapxWe0F#)T9GL1
zo+%@W2O4^R@x|a8a4?;iyezsbr-@)hK3MMv+5ieZa2_kS@{bFhCov!VjW@%b{c4Tz
zSGUC$#&M1{I~4V|COHP;=C(?PhP-DQ7eISv&sq@TS_O0tbhpIPtA_GCR4N^hDaz7b
z0A{F&MCt-c=B{5B-kSV|lc1j3oX5lN-<}=Xm^AOz`?4oAU9CgG6VWGI&N*4Z5$q&1
z!B-}A-I%t}h9R|E;>*1Z<0SNS@73b0vE$t_&R4T3yt$>{u9Zs;SSyo9Lv_#BIoHmA
zL+65<L><*`;}M56Z^<6#%KZ97)aD!>aBlAULBr3-#*Y*wbizNAN2^(F!T?@LZC{6m
z5)WdXo;yyN3aY7uZ>^r=*6on2L}VW8z1Eq>DOqNHx+^_CDfmcfpntj#@+xt$yUn}2
zu1VL|)sa#N7se@vv%NM_-@ptvzn|*m>3<<{9<%Q{cquKbx=Mmr6KX1YthDzV>eXsL
zRXU#RTt4k(V=~m6Yu?}8hP$7oJyJq_7#$(jn9p&Vfwz8}%|2a(I$lrrc_Paj+V9d>
z-2lt9!vRSiXy0LLNFFfh*$z-cvf=d1y)asElJ1QbH+)D-3zbQzorFd=sJer2o8aMt
z)&c}v36Bl(JZwq+$hXGQVtQdM;&-6DA)Ud=b#>XbygB6Y9@!Wn(z8Y7Ne$3cu?3L@
zJRhreM(j9%CT<s4gk_#=&zFpfQ_PnWXGq*8o@T7!?q5fx{9*ZIo2~Q8(4X&k?c0L(
zj+#9*C9K=Zx78e`#n@&*O{?e5n^+n)yJXZNupOAIP<~K)a*ya2{^Ux%S>}?5#<7XH
zR6VYas4ZwUX{%!1I-C?qmD2J#K*?UyMWY&CKA31&R+83eSDF^c-9i<@U>Qd)J~SEg
zDtLcL;($ajXZ&$ofb8b8*mQpX-K;+Ms|Ui=4rr$JI}_i1t85wGUT>x#OX4OWI}Qr;
zs954Zf5-bHW|pKtVU=gY#U5yF0O?JvL*)_wum+NS!K=Xl;=c$!e0sob!K7CjB<@+-
z)VMHkWL!k&+d8JufzOIpEN?J(t{R=<d|UQg$#Ll3S@Luw_oO0#m>t62BqAsLBR?Bx
zT>D9OLS)m<2+Q+R7W{rdNB<30*?sV951>alkA?EG)_&LtNA{}Th}RL@GA2zEi}|v>
zgPTxw`6HgL{}IQqJp%Bp83LvP`iFQOQY&?+x-P-6_PsF<Z&Cd3m=C-cR+|N0gcpjU
zSV0#2?|6Jj9k7Q9lfhfm4+!xXk`Z3pZUAAa-;_r``uoPWbpzJ|LoA5cl}Og0s-ULi
z`>bdmolVkU20yQ((FSXR$a)&d<!FREga0%K)%T!s7&FD44H3car|!*Y><5yF+eE}Q
zIWqA<=|TSP;!{lInr!F(h)10Jy&avkD?uw2c?8bcm0Ss20Wl8MCtY@6I&fYvWGqEs
z^p+o)920pBoa$UYyF&@Vb|3<Y#-Pp|ys-Gh2W6*#msvBURJHoGc4x*%$gA!778Zu&
zkXJK5ID1$&#W869QW*?6cQi4l!qBjvg=<6Jpy{Jf8Z%al8j#6n+(QoUekBicS!N_u
zvKCgyn(mO?Hf`5o%KYOf_`v;V!@*`$_9zLiiSJ{o>j#9VT_uFopjE9>9o8#vA3O!o
zA?Xf<KZF}GChJb}Ko}3xiQSUB0{{)Lcn;*RXI}?;0pQ>R*aD<`ZpG?0smbX6G%0Q3
zOc^ISNZ6(tq*n&Qe8b{}DgncO_ye5Z@Z~z*+sfOzYd=*}D`Dm22YM%S9jQMOfAg2K
zl@KvR5&ZEG7bFPc7aJ}vN48or%w5rPw9$5!y+Yo6sJlqfn@_ZaBS8FJTtoAylJT(u
z4F0Wv%Q(I@BlO7tY<JPkf{O6M#%(`Ty4q6^3_CjC@SyAiI7{~~9AtTcd_no+GRypc
ziT@4p1-d2NjBK;OK_@kY6v{@-4+@+ru2wA9RkaA`cCen>8Sq{DJUj&SJeTi8=ZPd)
z8{N_Wu@Xj2QfSg0A1CXm*)#(}#Uf7W!Nh%YOT{4UO<pl5q8&cFMiiRxerd`K{Na)U
z`*xra(?lcj{U;K{8#vm3u{W`g4(Gv0Gh+@+RRQ~>oJjl{egpSN48$30;4MLB(WyOU
z?hRiZ`iDTo0sYd$nWgJlG@D7A$D0<L9C`h+{26U_kHRfPd<+5!{xFMO<pGjnUpAd(
zyJEG_dPry=Kbz_&oX9wL7$RnDzpfd^`4fF(^X~hZCQ5qz<#@0)Q127M>N)k4C*mrE
zJq(uUfb*Dc&PL1P#3}zf3j4b@$XAymc7`V+hY9ic1gZ5_MAqmANn6uKGSQcX5083#
zhG@Kzvl5PC`6^{*qRcQ?-=>K}mWXTeI3n)uTa<Cdz!z?n>PM;5$+#rBe8b><fy4gv
zfhnHYRT4E@#v%(VBa4ik%;ynzF8MeBcTI&n)6e{)by&*B$Il1eRQ6C#ht~vPMNzUB
z!j{}Uf{zy(YzMIr1hp6dN|>mTe1m4JdbjWImyCCt^)xn8s-?@fyYg^-XH3seEuRiv
zSzwh-nroxAl%iENUwdN308?EcA5SzH&njr|W}H!u?pyW5#dSnUKT=y5Q8;Ar=$!_|
z={P+s)7(tb9Os0aG04AP&oylNxArtHXf}m;-2EkKj$}|PhBC4=;Utky|McoArwo=V
zFQEuZCJLb5cyI6g0aX)(pTZdxXbtLw4dTL7ZE5Atxz?R@qShn136DD(h&X#V&y_w@
z&4BFHXoj`cV;O14z|hRJD=XW@u~)wz`MSoTHekr$a-mg0h5^`?I5S8rdITQY!e~1!
zOldGTwu@iwp%c;QjD~}a_x6c=<h}T0>ji`cTRczX({}r1ec!#cx<bByC-g4ZFS_#i
z{4x!H0(N691_2+IM{uB<)_SKHlpE)d-(8H!*gHE@T=@Y77VHmjo_L<%!A5DRMD1-C
zL#oh+)ms@w_(~d)rc>=2q!YARvqaWO5Jtu1qbbdNMS^Q;MbTTZwxm9!x<A^+mbW)+
z_s_}WLQU*kMHFOR&J$9>QVg`^CXA*p#5U~l#VJ2~zt+ob!LEaE-}<M<O@$VmN2EIl
z$abzSV{2lkITWJ>%~H!Qt};~8uEiK^%g6>5^ZU8nG>^{BD<C66k~DmN#5JGqK4=(u
zH3vFJ&=#fc`uALKs<ETKpo~GwahWPANY-<`v)rbEraIYGkK=}%8&;)3rXXK#ELv}c
zuauIG!IG*?#4?hHZs~OUVzF4H!?AX%(%>NJK)X1Tu0@R*sf67dNL|Qt<)Bu00%&T%
zCKsLyYy-9c`r3{sQv6_IVKl0J7m)Kiyb$VdG=96(X!7Q^6i*E-w+ee|5j)IVHyS?7
z4{g9hZK8~m5+EoS^1c7E=q^J6d?_+<x5FgTKgpZ&yvhc1P5lwpG-+EDM`Z$F&!kHY
z4}k~PuEW>Ld&kUxZCjTnTF95RGhCVm8k2Z0#pcLe&9n|NhKh*LEziw>o*~grMxoX;
zLq%P6B;8xK5RN|Nt?`vh7AN%7UV^Zr7%dzB3U`vBnBDS0YneNu{fgRFy)b<<IFSI)
z8A8^1jN+A<4P$+9V}5sm_08rNdL2AbgWrxB$vZ1wG(EHG<Ysaw<^z)EH^lUGtZaO}
z#n&OTHXUpVFi>SVr^hB>!2xl>=YhFFK4RImPAY5$L+llwN6)`A;jDBcr=hq*ga`GA
zT~Wy<7>z+{c|c=E!q6lpZS|ib-j#|jcOCTH-b#;ZK`=(x)qfJN9&bytQ%3jx><;72
zJ@)>o1y%^sJcxe3@zV;u4r5{&IH+_}*9h*LbR|RZZcJoOfh_1iMYbUwKLDndW!t3%
z4~U<InCMyu_kubRp^M+kwV%+al|}Nk&jrnqG~znMMTTq>x#dXVJJ4^Hx6*imm{mKS
zd4eU&{0pYBZ*(<VpL24kg3J!NQB-=1HT}?wl-;HzA9`D5AJ2jX#|j=B8fB*n@UYz&
zqdl4}M3Y=ddNp8=T~dMY8<GQ|GH6lG7-sn>iW_A++nFDZtIaTbs@1Rk$Gu+S_uX&_
z9;8^hE;6_R3O3rV6iR)mB#4-xBj#S`aOL@LBl=&x%1a!Ck6c9=iE_eNm!>CA>1BC*
zP@{LzJV(--3pxQeW+U2&MG(*SoqLXVI#bl#bbI#fWIzHS>I-TX`VBRG5vT9ZMXA2p
zeP*Q~s-MH4r=cLHP1Ma(&|L=tHJP_=#mwg>_-CkV0#SHAOi@(knVZP|e*Rt|<qChU
zv~O_{pF(d1(6xqy94TrhT0vL&{XQ2thjx{C-X=|}#rQ|hV&DY3NP+DJeG~1S!22rR
z%EbP{DiX%8P2^+GUFaneZjV?p(-A~9oRvKSn$^`lH5z-LYEb8Ikf?eVpS(;M4&GcB
zHH2QB#*m4Qv8ooX`?GjEc;Z}!9k=19`cQNgmL@oIn2!0##5;0m`%+ZGT*41nZ7ONH
zMT4!{UnJ^#zfGgj0=O4OOTn#wTGs1O_KoJ4zQW79{lY1$Wk=ik1L<avMnf<R#{b!?
zg<_7sev9@=A3++x5+W+9f#R_C3!{ll2ccZXWz=QBaf*`y<@@2^IK2BIM88A#P<H!o
z6jUtt6KtWPV0yE`W|F?g;=4*D(T-vQr8EPCb_Z32d(8Lnp&!LLC_=C?Zci8@NgWpI
zEtwC*2PvO5)|tpAQ(R`AotU)b?i4{JPMY3QcW^7v5fY|B>K5C(PeKnoe25*`Y43C(
z_w)&l&qW7Z*wrLxxT6>-xrD80qskl`?nWxFQ`em!Np=hwj4&u~O(`{MrYJ}W|1^{$
zs2RkW8~B?x6obqKq3Qsd+aY+qb(oZ(15iDvR)`Jdc<EKkzatz7l%HUc0DvJk{d=-O
z&y8cTShAN*o4;t0ZumW`6;ufce05mLhn<-CAYY2+T2;d|Y{a%HanN_*FNrxFN{}o>
zRZxaFaus$9S6FZlgmjRk?F<pZFQC@r=!BALT+;FJOsAm9g$?;@7+?0LOcsL?tpNMK
z{l>V_*_cD+bIas+koqV37OTVwz0<OyR@uCdk_s=z<osj1awO-P2U6V+(oBnSdX=;`
z$k$KSo?USF*1%=bH#kdYEZW~x&vAq~Ia4`Z#MqS<D;Zx-4$B<T58_PKu-QLp*H?Hm
z!?nK`Fs8Kr(CxR`t5Zpfg!D72E@RK_<j4AXO87f2*MKBPM-K(e9K^N7A&}51h^&d<
z=LSycH41xN8`2C#=b$;X%IKMV{~i12oe+}1rO-z%>LI4?X(mB2rmbRo<0i5%KQ#UY
z8h&bI2{hbJwX2M@1b>2(K`BzpN@pz_MViwcK)7ib*ni-$>5M_=5-wfAk@WPdE>NtW
zD}}}>p@7ei{lq-8vU#uMe(2d15HYHt(~8~b#llt;BTeVBL(Mw+H!kORFa837&TjPQ
z147u*Ld0p!TJP1eocs2xPh2i!cpzD@ZUB~>s>j@0cL%Fk|7DqZqzc*f@r;~?F4Ot~
z?565R8q>UWHS{!kQ2)vBr%ib6Gm8nWeyIH()w140@xn1vmwV(2b`-R&aY41v!U|pe
z3mCQmt=k;zvd>Vi3sHrR3oeB<x8MA@J-l(rdKtpHB{a-U3D4JC<9*48K0m3(T~-lf
zz4Pt2UCj*&f|>K)*e;<A`<_xxOM3QPW*ffjLug$g_$VIf+xIL^4?yN?M%wD}?t~J6
zkx=a3heI7dNqMZQiQv_j5cox5jtyFJrNk7pAH(uQjMGLNznGfJZFQrUBb|eMu05jd
zh>L#8ld^_IQ3wM#ZFBiLaNG&ianX9AU6(#-dAvFv(G5=cmJrxm&@be3m8^rf8uU31
z<4!ekHB{cJ&ni&?CIM}{B(;gQ^F)(7E<A_y+niB#C(YhK9mWLDv^R<i?!1G8e2`g*
zAvE&~H?~jboH;H16eh~}679A=f25Td<!AgcIEz59O@8tt27H$FCS+OkK35GtYJSoq
zyYa|JJ3~qND=y6veU|lWIzCaP7Zg2s0eNbXBvAIa8u{hrqXImbE2&AP;i(n(1uq(c
z#4HLY`Qz-}kuApg9oVIh<1m&4qc^diz2i}VWY}%5`j8?Fhn&Sx1|!ZG!epKTbv{QJ
zlRJ!jpZsA5o!@co{xU9fgkamb7t$n7nKZ5Pj|6<GpbzzzaZWQmBN-ljq@U-x&6bMV
zpG^*Ry2)DE--I#;41{((JnJ|Mv)!-6GKP?Z<Mv{AEYfWU8#H})a#;d~{RH$@ZJ=eY
z69dLYfNeI0(SPfGqqY{0Ylx<8`_otlu(?u<eHaHDehP73F1LK=PHm|r0Domm?w#tB
z@Wf{w`=x-%bh(Ujkj)qSXL&=ET<PA#R+Z$6L6ZZ>$T|Z4q=pOE+uQA>XeZU8w{q)4
z{%<<snQc+C8fp^JFvEE}$#-<MGv*8C)S-pxx?#|R(Sve>Hv()R@v<?o!WAPX(2RJ=
zr_!Ou1d&%iGReHH@n)$UYUfoQ_R{gCu4o9#I2|q3#^|W!Y8B!zYjrVaDNOWha{!Kh
z<PVT7h^{GoND5GYGfOLF6CTE(vBsMs!LmVaLVV4Yp#{K?G|07%PjUGc3LybPgVWP1
z^k;Xe7t=M?HPM*8a{<1CTY(gM`8Qw*w$xq~?{TM_KcO938>n91_h&DkIm{Nt{f6hG
zcMeP62cp~qqiy6vYq*`D>l&LZ=kYY|o194ZP668Q{IwWmGMd!-Wb7E{0BKYex5nGm
z?p615+|;_Vnk;CJD#DIjgN_EAyFU6i!(WV_TJ+Tyi{5EFxC!7dv9*!)N1#e=%KT1z
z3BuYi%_+T>A@iRZ$q2I`T>KKPjJfA`%$puAZ%T~UvQ*QMY$9CJ=BS1B$5P(t3<hh8
zO>Wp7jw2ipI2;7*Q-~JBHAInXBGHnI85Ak&SUkHD61gksGcvZPB6vphv&Nv$AuWRL
zgKR)DMvq$Gc^JUEo=gu^7s699{aPo-1z{M&ieADT#$AeG1ILS9X%jZa+Sx2k6!Y(h
z6C03k=#tY(ada(x-%)53;XE4hBvTpE&+7KZ@u;iQ`k(p8vfw5v(<oux+u)?GKfhoB
zzVyZBBu{-a;uT&H)1m&1I<K{n(c`-WBMrWS1!8+iqf#T{9e>&)1_ADk-_hj}kkk5B
zt7i=7LnQO7+RH|Fiif(|Y$!Ss@L$X(kCx<R*<h>jV+weGK&w~dbTVL*vtg670Ipfp
z`nS;rK(t0_j@dPll~+-h%a#aj2xbca6Epj_<y<iK8CM;~nRge#9nGt<t;ULr0tN7S
zaJdG<l;{ipjP!QT@F~s=!kXqTY3^8yi;f}a`PqY%x%UcrOTsolxA7OxYFl0v>oK5i
z$$C?<L2r6<#a%km><r2shV3VAMJdCjmWTSmQIAeN4HL8IwWQ)6ADk_6-suv$``LgK
zYYSEu><v4Ika=bze}~j3(F(fX`~U^7%#mtCRZo1q#ZD9Aw-^`MxO*V?x2sDx3X*#d
zp|6UNqW0nm9S0T$jo3QzO|{fm>&GB~xXziif;<Thsi0&RUmtKO7S1B;;8IbNaz_ul
zS&E?g`v#lzZ_Pt}zUW>*&yf9kCofD5ClrI6{9^Uc?s}`I+>@m#Dy~SH22AK<bF>4Z
zV~j3OvM=kVOVBr<`T@tsnZgUoBd}52B!Ni7OlH9X78x)0dAP@3K_^lV_bmL9P|?7t
zKoR4~*#4gX7-JYk!IaS|gFhBANb2g*O?Esm3HevZ%D&#bwKUejnNPRR7A#LiOo%Wb
z5J#~;UJpx^t5n8)WL(}s#gIUtgCXmY+<uaxF#4t9*@8o;*L8*^C-?UT^H1)TeWo%B
z*<<btsX9GaYBalK=cL*!ZO}z2XqcHT%rt?p=Sb6kH6!1u87z*)1wRJs5o`!ax3~s^
z547thIn<BbZl~z{ZOm*Jd)N;YzAVZ4t?!?BjU8BUa85249k)Q@l_h3;(*H+<azn3`
zScaWU!JgHa^YZNUkuF~BwKlzhp^b#dQKZCtG7fI{p}J?l3-p_YKWuN{F)!nhJkl`2
znS9rrpV&`>y<5TbHvwX4>&cr*S`?Y1rguT^%~k$2&9stv?!?cnG1(B^rtG=%b0^r|
z>q=|60=Cdwtyux@^Cn>2>fM;8U&9(oZKs~yRM3cb@dL+=e;u0zNI~jB7AO}L7jHuz
zbpdO@Ur*r29}Mu|@U;|T)wU|g-NM#<z1$u%)c-I}%Xx&_(%1&c$4Ok|jCqSv>SL?X
zm@aUHdwK`m0Qh~ZCy>*VuTcHT$A4q@mU)1vQsuRA{TSU)TaQbab6XGDWxci{Z)f-L
zOJN_U`s%YfwdjjDQ)=&)j8*au`S5ByF+5|iDB9IYgQPaI4zs30`^b)1{row0?#ZAB
zIJRLzxIxHHM9dYZSQjBGez4=uu@_8cc)mf-!?}oC8P_w&bGS3;M4VU1F{t^{ex7-K
zJ3oKZ=eg`rT?a_-FRajV(besLAG#G3m;ACjlrqZ(EOxG^`8Qos@|wPOREq}s!!L^^
zqRoq$^9fbXMISSf(<bz(WkA26HLDjyk&<3Ny^1mo)5$7ldEnJjI%4Hwt!E^|bswC*
zNZa&@=Ip5&&bn|9K@TPOw%=DF6<?&BNH3k|Xd?MUmu5EWnbnYBQ8mSf*%<6L>+Nqn
zZUqa$1DHZvAAaW=RVTQt(pE4vHCNrS+&fg$h<v_kRgm7;q4Iy8V1ZpQcovJo@2wyk
z`WALEM#DmsV^e-09#-W8b&l@iOD%FA@9}xrW%xuyNDA4ZZ2a!QvW3-Sd~mYpZYTdl
zYnnOwa)WgP$AEI;>gwuM(7N7DN7V0d`Q-C7@Ns|n1Y6&)R89K|I6uLE;AgGL>|(Tz
zWJ%UQAMBg37>1ZQBSpg_NIJfd)j+N2V&<v|>;P+ZFSJ7-Mn3y=#9IAEBW~XSwmgOF
z@OnCCmC$f7PnyfC&+z-<vyYh>ezWinYqEW`AsssI(F=ez&#lu@_a9pj|3;hM>axFU
zO^5|~?T_$+lm%Y}GiZez>9y^JS3~%6$?y42$_8_0@kH5zO3%@CR_5}i^n0;%(|L0G
ziCR!(hGA2767N!b4Z;t35b~5$7TyN5;r%k;J78<m$3Ey9W5?RSoe51YsQmW$g+XXQ
zlj-@$WI$NdfA4wwZDLAT^sO!+CqsI%14y`W%K0Pp4EN-{Bhw(-lurKKZ~A;)8)YU;
z1p>D-D0wV|KE^jJZuq(Z{d>a$)^)@|d1-Oj&ifWi#t4jsue_{iRI4pQIiVih$_g>e
z+?~=CLA^uuuDIwAit(#8{L%EWncUUl3FsY^O_dqix0ao^Y9(U=gzx+cW&+a&B#kd^
z=g*!NB4j)JnInCYBeMZH{dt4mQVRBoMm9o)h@1_~;|Ecv`#iDc_nN;R_W~m?E(_S?
zbHK|C2Q>E9OIh$6;#;PHIS^|*N?DgO&DP`+2{@%l?G4}T$}}T}+R9U2Z&P%=SL_(N
zX(}ycU*}0aVp3X73qv1<5nBLIhn2JEGZ6wRK;YT4+2J~8w4EkbX@m;IBj(IWYK!U#
z7+rsvYd_G0L`2j`@`Xp6ZQuHHbllG|;WO5Z%w&hL<C_u}#1_mO6c>;LYTfi^C+im~
ziaW@<(~7uPI3MfE`lj!Px~j3UO<?H1WEC#^QwVbv1qjE284(BeB(h|>tR<Cdw?&^R
z{p63Nqa!Mt)F27%9@2UP!p%}B;w|5la|wVsG8lYXJGv#x(E_RNCmk6pypC1S4SH=6
zNo}@vm|a6|*3g*=#A%%wO?UpG2OfVB9S?OfTFMW+73@B`Yz?Oq$uiA4$J0F@MfqEh
zE7#)IDVZi3vjt|<K1*S}{kt~pP=EDml}D7B+hF4FOxs5Cx|=ds<gpLSld#5HP4GYy
zbz#0(|FQ2`{WfCq^bqqEqgG%|PFFggCSFu3yF92fE2W$FK;i9VAf*wK>xylQXMK>j
zI1;qAfuBzYMYsz%20322QEJ$!h|%fAO=XhHVljO<WrA9Qfl@L*)#4ndSw*pamX_!#
zhIte=H2FO%hlC713*+gb4MN;n@GXzaj3dN)ne4P0WBv{O#qcbdEPYAuNy_(F+P(%_
z0cCG`Po`Ps=rrHsoR3;8{_L+Z7NHv<20?EkT(E6^cO-3rW#8qACp{in=rfB;D1pxH
z^M^X$i`vx4>wjn~pLi}GvPaT_(P;QhR8%*%Q}4@kr6%{+_YDyUp9R~{NKH}#i7SM}
zRKN3Vj8{1QiF|l5&Z}o*DRLV?#E-zIiR1P8zT?~X(x~&$W3pbQTeE@3VWBF%?`~Fe
zYA37TolFU=qCcXGnJ6yym&>U(qG$Gks0$Eq@gn<s&QbXA#|A8m;@M8#Dr`iOv>|J^
zSU#SM$OXR@u!dzMGM*fT;!&wAqNA+Fi_1B}T$ac=6eS?&l6$W%v%L)RIMLSJ@7qgv
z^o%^N=0`aq*(2g>as`}Lz`}oBxoz1UgACBJgh+O2BszA!uOq}^qW#S|uKi)`l~{mu
zn$Hqwm9M7a6+KF(NXM{sM!ebhXJ~*cj01=JWi=>CyI&*A=Uou`{2)dQJ$9?;vMcL`
zniCVS+gsBzHanXgFF8O}jB27A2fuSH(b1%T`L?5+-S~Bkjd|fSb&EeVFC^ju;U@J3
zzN4roW`Ell_O;9MDb{(9CerXkCWyDEl3Z?9v*WB{hZ-pds><FBb~&brqcQn$B*`bY
zInc>SnjPA8u2WV0978rtu2F(9Clw6oo^r(=5s|%?YR^mVe~Tu1`n()VGD^*t^e~;}
zA6ZM4!84(FJIo1%1K!~%id)cqnnJ8xkZ+~f!%g(^s6pZ-<4tBA95C%d0TUy$0T!^U
z9{IY*8`%zPG55Bq@dx}Evh+scmQ$&Xw1la@CI-l0NiF0)D_i$nNslJM49IrB97aVJ
zkONSJJfb61s_w3AX42Q)1sLRRC_8wB{7+lAniM-_cB$PH+YnM=nxPZZkwG(J+xUoi
zw_a>bhW#!|nYb%t=f;Q!e|?;Nt0surO<LFM53urFK)9RPoApMJEm*6FIj|gYr`JO7
z3N0rp5vM7ha^82ahVV^|g5Tj8v!S}CMn)QWZ-EL-*XnXr<Buq!C?cv1;z%~#a3<0*
zG|7g<>v{wmwuvm_vLTFxF=o7Owl-ZwBde=BAK7gCh<|^dM5TP+?o`NP5AiVYyML_V
z3*;>#?IiN5C6m(fd-Ou#j5A2?GK&XpcKGVI95oztC~8{s$`Vnphu?4ZQv&PnO7IzT
zhFS<5GIv7GIQ-8FuB7ukc1`jYiVLya5&*f~;fn@4cB9tpSZQpzv7>1C=NwI77sScx
zz+ELU7#>u*N#w{Be^H7&U{RJ<%{1U$soLvH8}AAv6hdf|puTG|=Ab{vA8(Y_%|fT;
zO|5~w6d9`Ti4pU)l-8}ok#jSgs$E*k_wfWio}El|E-@5kqfP*@PMNJo$Etd6d8Snt
zTa>Z3LNMvd_?${(fn=@_<MNM<PJ3Qg3s|v5l5r#Zw08`hNZrs0H<F$GrX9r{toa64
z#vJ;lL--Ms!A+=^r;^KB>58F4Jl<qqo*k<f$Ut73m7A}dlH`zW$T9&pL0^v;*}hQ=
z|J~OG?cjs3^V+`?;X0tRI!HjzHX4Wwt9CT=;3Sz{@=rj4gYyGVBYd-G#tlqFqlQe}
z6mw;IdCN18?OXtw!da5QHo2M8(jk(CCjr5)pP&s{+F@|L%<ejeunvh|IgNP!V8;*2
zU<?~bM0T*IYBDj#4fAd{I`Oc}y$OZ6-EnN}zj$@6FS72$L38%>>ID&y#kH2WXLpI&
z3nO5QYhvUsYMo1XcK6DvqAv|>Vmac(LmVL{zWf*pXi`JQLHx#CoPTSwG|x#2B(to~
zg`eNZ{y5NA<ZY_OoH{lo6nVZIVWZ#7&ts9S=ydb={$?3Ynrvx`>fY!#enkNd7h4i#
zcjLfyB!yk@gGCZrIOqWfDO4v6tl0YKfvoJn*{sYwAMz#netlA$X|JU0OjW(&R#PX<
zC_`44!GD0=npsU<)u~hfr#`i_!j@m-^r?{*wS#Q2N;)uUJ_*$9k*f8(f4IERAxd%0
zLe;6ZOKym7Q9aVePOFe-x9{p8d4%Upf75*vZhr!AI63%baiXdu;i}URGdJ*Hx!3J>
zF=EFjpg;o4m#l?s48rU4nvNi)Pd??aGMy^u9ja5&5SA6&t+)kjClFt=D{NG0mDF!R
znufw$-04gR`$8a7z`StK%>UyK&)COzeb25LfN*ZNTDOG$X~w^QD)}JIL^iXOQSCZ;
z@TdCKj@s03M9ZX$*C1f>=k71FFAt=nY`F4nxhmC+4X#2l+Pu}KMY{Vh_CHC*fgeA!
zoa};rv@gscoyubM`P7JFI_F!@yD@m&FEh`kO==B?hC25uQ$F?6<8si2msEm3b5THV
zxhB>bK7?^(zI4_Cfh=zIdHf*p60k|Wsr%dwmC8k09ge-AN={T=>3mVs=Pr3nUo<d1
zcgsD;eouXh>IbXAm0T0Ny$#dm2A{kzZ?erfiT2ZzMf;8Dft9b7)VrS)f|Z}J+QXLW
z2bOHS*~eMBp39JBaC`hkxDnn1trkj~u6rS*AXKW@3L_SouuoEtogE8xssrvpYS2ti
zvzX1TVu}*J9pkb(uxoE8`Vu=pa%iXD+ji6mCJj7l>PEJo_T`sfYy=HfY3fRCON+hR
zud;VghDfl?45{L`CQ&lwUEZq4+iP3B!VL4!8-Y9Im)^Gf)&V(OVupDK`bXzAI~51m
zH4T3a4q~wVMvYJnq@5~@Vc#kGZ{9Ssj@Y3=9Uc602L*NIc(YwF3%Cnw$AD$y%7A5s
zOHN+^Y&%+h8OBA+ry!?}YR4PLu+!Q9dAn9icg5m#`yiI6WxT+~3iTXby^z47D@{~w
zA>`2`i`Vx-DLj^x%lLr!G@)=g=Jiqgad9!ML9w~<?2UM&Xf+uuGHWrueSk*D=e;}_
zxKs3A-{vqrjXK(Uv0bFdmVx$YA)Gad4tWld_$w47D-VdjIOR;tTeRwYze6GWzEL5W
zDGXJTT6M*GsID?}X856+{_7vPBGp?j;M>LAe4hXjw1&)Laq8I|X~FFV^-<&b;si33
zF0?a9tf~%+h*A3lYwP6bOPwj!Udl^t2-?7p1SHi-noYVqy2FtJf}SIr4t}RGVzCEL
z_S!f4B|=nck-k95g}u3`o<5^Bo^}%V&7wJD&>%OE#ICdn$-%=;U2$CQhm_Upm6>3L
z_SH#_&Aw=((`1@eGipx9eb_f-o8kRmZBCq`RdNB`|Kq;^cvKDSCoULMX`fq<|0)hF
zyGhJ<)@{c7^$o2XsLnW;tpUF;ckJ#{MK0EuOvOeYH~PZI00r1P7RKq^uC+uK$`7jN
zZM3p%{w|x!e2NtISHHT)I!Nd9=vS*w!Id!Z0avFeFN75xbH%7E<n%6QSdVnqvsU=L
zSsTf=0if?yZ}EF>$GNs-wnde5&v-xO({LwMew{L;NDjUV-h{ir_^}<3o%;iY3@#S7
ztM%B*6YcOd0pgLi`;#O`caouW6K~2Ic3hXRA^aN(F!Ri|_WcdxA-ZdFDY>!sg>O(2
z3n%KRZ#c6@7ld;mlm^{Ebt;SX6W>mDD&@*d+pj|;(B*bh@+{=6EB_2{rJ3Y`S_|o>
zqCp)l;gZV{f-k=^RPB!knQ_xG(zo>uChGGVa<?RSk+_A3ORP*Mu{usXzB8FS;Z&G$
z==EUJw7Yu921(kaLsTQO8kWhVSH($#+VKtO<y%4{8q0`P8A`+wFoi6gH3<G^!84a5
zm-;(F2PPM<GO1F=JDmHKA*5j2HfJ=nKi>ZXEJ4%0(1t(O|4}d!JOm_ScL2rmrRtRi
z#EFDE*&GaTV1~KC!9zmln^qL_TTc0;72~&rqb_xI_I@A%>?-*-75SF?@vf?{9rZM3
ztG566dN`4}oTFz$@-q`?Q|myKXp>$?yF7#Qj{y6|x%J_V)kk!f%G;>zbnqo0scWm)
z$BqM^0f)D>kP7)uChXJTNKlD<RLIh*AfJO!;(25Bj3SLh$FW7SBf-%=gyWN>Mjbu^
z9Alc&r*%2@DA7*T;laQ$Z)YHxwBMc&=7ILg5DNG{u>J1^%RozH8i{TIyMmQ1$&k{t
zT?5e=+g?J4<izpS-l@Ao+R=rDWSTFO3VnxMU8w4FTXEfz?_`U7m73sU8!}|V0V7r|
zn)w}EwcA$p+3K&gs}`J-EySIAVRg9hR7pkaq`nO?Q_ZmL4lk}-enxTQnxQOuOMtEg
zch$$o0mQacdRn^^ehSzPNaE&#$H1N7MG(a|ck{Qty|ILzcT|&~0DFKl!F<qF@_8zn
z0zL=Uc2%YAtS62;mF&m~!-?rI;3W78SOlH}+S4;YQWV9qz5w0<z5}iRJ;}DVqCKAi
z_KWv{_K5eYhvEV-xjhAy<&_M~29hdCW?S`V`xK86mep8!!-$jQYzutuKJTGG@*vT1
ztau(g4s4gl=Jw9TdOg;HIvFr-Vs)R=p{4cf_h$Uuj>vU|z5^~DyW7E6&AaQOT)%1K
zT*MeXe9BQnm%sSH_b7kj;8DB0f5p7Ju-z7vN8YM_!+*nAKEJrjDc=hB|9Vqw_7;2e
z>9}CnO_EjLO?9Vg>onk7$JO9K;Dp%{WB4%XP5}-CKL>Mw;`3}t{w#45f8I>oX}|&K
zLEvQK+m#{#%GZ%UH{Sg~<`-kZOThL$0R9Q$o4akbEC<E%G6>r2#J9$u0Cly#3tN9W
zb?pm&0G<Pzo5V4wk_|Ws+AkzRJAtLZiEkB{37ixU2fKi;g8bWm4cC^L2bWzg`%&Y^
z!Ij`|9Z>sp%FrV_1$5*D`3=g`yCO+@l`wEoL40QMxy7T&8p}6;xN%^7J;rd9^f*@N
zadDg(2wnyfoF~B(pt%=_eBC5(AY1u%=-D|4Xd~7$7hDML19`EmN;a5J)6}^H_kV%+
zcsh6mw6sjy<vkw;o&oJ?2icP}r-5?AX)b+=?;H*Mlw$4pA8Mxfd`Hr|@L#~e<6LkF
zP$HSAyK4F0XbbsHcLLc@2GJdS5$7+|7im4F6)~?-`TV}gy&X9C`1ZCC%mTLqNm~@-
z%qI!j+{AmI3eE?T#aVnx(fjEVIq<*(-@a_wvLh!>ocQN!uf6vEE~&J0D#_bWtf7{#
zy2$Jaoz@(4%rW~fUc7kk_3PJ9-MDe%&R}=KNA>UD|AIk-2Hh}x`0!^IE?hYK@y8#3
zAr_E3TD~@`SZ1`-c+4U?_>Xmd6dVIi2hDjaNYboKA;o#%R8Z+Yn=PP`1ol;5OeE2Z
zfO?6~BpwNBl-wAn=)n;ZfbD?fRZ`$MB_u(8fh1=xcp5nAd#ohZoo|b2yv|E}u5tWu
zyz<5C4Z!<S?gkfwJei93RbHOyTO;1r_rVU}{XqT2c5~XSe1H`125$p-($;Lol%Q=I
zh!4JPxl^u=(16ciYj7`+B>1KiJ20meC3Y$&*#88Ff(L*ehs7X1*S2=(SUyE%KLO!{
zemt;K-T)*(tAOI5lW%Whbst2!V}XOiYT&@97wLMi1SsZ>b+@(cSwUU==eN}NJU9#J
zmFP-QpAMW?fxEi0;%=-<PvY0Xa-a=O1A~Ef<=eAjKad#AsmB72l|*}s6KAx!@?)iZ
zEeYF>M4tiL_y222h8ComNSdDmC)zt&kiI-g?4O3`&+Wr{EA8LDh(#T6`Z{qLYBX?Z
zO6bAyI+B{coC^toWY50wESOu5g&ZqMzCAI%v38hN<i{RgK#l>v6b=KgfGdGx%m$#u
zIvOjtE>+9__95xF!Cv5WFb`NqYs$Z$^j`tCGZ7z@(Ag4=ptI{q<u|W^&ywVDa3aWe
z=Eid7JD2%B1`PW(xDhD%G<6}Z`<u_d4OveI2LLCPe}KOO#dJz}XRj3QfaKfQo4`o0
z63hkHf`>p9CnL)%FEhsXl%tcto#2Du9>B+COCR9qWYF1dFTL{0E2HPmoxAUd5hI?P
zJbCit)ERZo>6Rr$es8I&s;YF~efOP+mushk4m#+kTWz(K6HRf|s#TL-eDTHI|MaIn
z-Sz$Ne}5Qhoru-{Ca|&o=$p6%XnsA+>*Uy!XovZJVA{r>YkByoq<t6o0+`!GKB8JI
z+SUI9H-YmT$yIIxvHi`BE0!%$a18QZ)M12i+InB0PJHPSp4R}=zX+ZM&jIgOG$!S3
zG@WHMC*~9RdpU~hL%<|pd-N(_0_+b_Vtp;CBj#!D=zaeWa7_GHFdJA;YwFm6^w)u%
z!5WaH3p5aop$n=?RWzXipM!6#4}*7t2SL7Lb6HW$Crs<0^f|B(xDA{IBo*-)x27|a
ztwHj27&smL3)ltB0eXL&_!S48#@bz}9XpX;FVJ2<aT2-<Tm@DE#WG{t<1vfq0e`$F
z2fRbU=Rx)+3-NK_)eDiN>ou?=%VST~S37~@)Ot9<*vEWk+L6#!y<QDo0xy91KyRd8
z8qpfDETIem{{hB=4+HJEy<(eVKVQzdUZ2*=Kr;C9AuP+KD34x`0N(&-1N)!%V!J&G
z>sL40x%%=b7J$YkZS$3j<+MifWyHKADc~JoFJRjp>+S}Q9TMCqvEA{yBadIAu!Dhp
z{|V5Ry1q?5y+&UIlJq29pn<$WbwPEhie@zM3F6-l-U3?V^2`&C6UciSaDezZxE&~M
z>6CRj>;6Vti35)w8{cNP1rLKifw@3wsxISVd@O4#!rlxVL`H%2z_Km@GeDH`HpKAm
zJnE+YCb$}Guptni`#3lt$MzcBX+nL>^@XU*POGP<6kLD5Hs^Ks%)Xy}`}XboC~=MT
zABo7g93)3hoN<EZv_Ybs`dpn%o$QwaC)qq)&Jp=K_vc!|BRQD`I!g4id=%^sJ_RIL
z{Xl<UKavo5Kav>{&z)SAs56i8{tTA{TDBzcKfuSp^T7L-+<5Q#`{VL8Vp;j}jEr@?
zCSfZsEKkXovp3g{kJF4m<^h+8ZUynB%V%kxd_FEaRQ{dPd`bNfm<h`7ouFo=h7?bO
z6TzdPGuun?Numa#Va4lo>^=?nEF8EV0%ri<UYZ(ch<=U4XKCk=j5u&PNZbU@0`ZN^
zbnWS2`Fb2oBT*a3`L*CrzyT<qj?0QtDb1dwJQC~-EPpw0V3Sz5WTC`-p10>-%->eA
z4f}$hfp>#vK_*Ph4{)OHlXSfX_JRaHTwiUplX#31W2IINCQw2&8f*<D4U!0t{efjf
zM3NM*JsPHX)J~^^oxlZP9gxg>jBU_v%g-$*l9>F_bH(w%$#Q#OJFf%wk=O_F<;C*j
zQSxSA5-Td4FC$D+r#uKg0UU$!67SKlSZDcpK7Cux)j^b4M_&R?0EYsfiS2(3Tmzl~
zQEZ=iV!jyO(Z@3=Q2l%s+yQKJTk1TRe2yQEBgxx?61qzAM%Gnzrh1yzzz2zbKR5=&
ziN5I^uZ^|y#{uvp&V4JfQ~ng(1(ddS)VkbtfAjT>Am}u3G*}4!2;!g_ZLFz*!8EZ=
zF`Z=|1$=XMBC<Z+$rpjyATKd*KCCOwuc8uhHt<b3lSF)?Cc5RACq9)LpshrS=h|Gf
zyBJq~96*31Xc+JXMWR2b>}Z%}?hPdHw}S;hauVzE7|%IvkXXNQ_TSj1c$D-=Uc?yi
zZtw+QzmQZJz7|MqR)Q6|^VUqN*8hQeyoIA>E3tibR~`E|>Z81l$yDQg?g&idGNfbJ
z&A|Ip3>R_CGpsY^WD5Tx_$s&><i|lSTP5d``aW<L@cAa`A`NITNqUY3d>+2t`Ns80
za2sgNphfabEGG^kz8#(c_5;^}p92R-C7-S?q;-GIFXEu%fPWg84kSL-<(s6^(z;Bi
z#P)msMsN^t0CaM3(7Ob9KY6K?Hy_`T=f2JV6*w?InJ+jIpS(7@{Jc5h(^!L|?HHdw
z_gY)b+oNGk$r(f*2JFL<5GO~;kEjNUb>!=b=f3PnVnkzgP9{!LH376l%Ik`G%FB<}
z61&I1yMP|(&YF~MSFDrM204;5Jv^g<<Hu9LvCw7ZDC*R_mGZ|l9ewm=^#6hHgNuP}
zh+`k8tr7|J46r|#2fFe;sIGTQ>}<VT3q9}d-%FGJ6|~js6YnVBF;5}z9l(M27r=oe
zN@D|1jBoB3?a0n6aXS$>a9NkGeo0D{meyrDksnybbA2$7U^yuE1=j=XXz;jjuC%nC
zy40=z`)9H^Apak5a%+%8d>T~I9VcxiKFzgNdn%VoVLOoMGhi0@5ooU+nm-~=j#CJ`
z5%?kyJu;kT64Nwx9mAVD{uc@M0v`a9u#S?bv+{Q8)%7L9mmJ3}$$+Hm7O((BG0wc@
z`D1)%9c|Z7z^}m{f$hkTO<cBE)WJ#Mqu|Y;tM7#>lOzrFEDhKxoX}^21AyeAJz@uq
zvOOnX2iJkGf}tQv9FXF5TaNj<CQ-nDgDZgKz`ENiwk_X2bs_<>{67GR)z`peP%h<p
z%j4T}?VGmGXAh8XS5}t92PEkp4eXgcgzo$EajL!$Yy-+|Bidqf$L3l;i3Bc@wAAC%
zTzTd3w$HI)HrNNWcMPJ4*XhDA#quQ-dQg4~{u3Mu90Q|7eZ}jpKJG$=w}H>qSDYW`
z+S-=O$oPKH5oNWt4ZSv>29=~|YT&ap>p-CQr@e1nap3R`#g2A7cqbSHe1o|bL@{l9
z-aw*FS%&9_gOkBTa5eZPh|=1+@@2;|cI8BZC<z${UIu4_yFgwl<;};p=KOmk?hhne
z%gZw|->1#Z<x1(g>(27{wB!yZlj!r{MeyyGWN2d=36CV<8{qCXmeG=IUm?*>;Dca&
zOEQ$DX{^t=Tzw)q4!HdH0(cbIXP*X99IG5h+A|)-vf6r_L`FU24*~Vj5p9uFJ_96F
zj#o*#SOd{8ySUm^T{9Z+xkySL18)OQ07*-G#epQ2vmNJ>pQFM3;KyJQh+-Ss)5-Js
zM^M%~fP`xa_%4_Olzh6nke2-|&o`cIZv<}ulY#Mn1^Hz*E;A|58^c?2)Fpoh@NG?k
z6ZOeyois;$X3UroX3m@$Hf-1s#*7&g_S<j2z!&kw?0y_RJLl6$4Po)5skoS?(vhEe
z2>g^oc<iyq!qll#!yDf4hS0BHze-87u&1AXI!nK9-MTPt+_<pE9(#mg!-h4aZ>k;T
zabB0-OFk^7xA02o%ER+{8yfWLNp}*TkI;;xz<&VAPkZ_g*&CHH#QXqU3M$!mqZwNf
zq47BeJq+FjocL=&OC%a@`Tg%mX8Vm3{!;KHu-tn={sp2Z5FVP8Z6enjWm|m_T?g7L
z>PFJ{NpNI)3rqRirh$%dVYaR2l&d)n=;d)TcOt(Dv~<uRX`|vmWXE%0Sq6Mtbb#I;
zya#Lq4n|Sp02Z%Xdo;gT3mo7iJG+9@fNk(CFG^eMigm1@AlvReEd+)i44jOH1K%Xe
z2d=i%`Bln!30zR2Jc9Un_A}4?D2K7xMefh?<;%m>S6>}|`qQ5VzGXP9Sg|5pcG+cN
z$&w{u`|Y<60|ySwQakyV6M$GuQ*kj(`Ek>xP2rJ89tnT>%U`lO9(?e@AfefDN9Uo}
z%Iejt!=;yA8h-Pe--KtLc_yq~yEgpupZ^SeJ9OA-r=1!IT2t*PkE?Vq`LLKi9~N5`
z<GjA%h8r?nNf!AA=a{7Y`0H<c;~PHzu9k^ZsAqF7xT2FQ?@#&rfK5Qs&{^UbGz{pO
zxf)nsV-ovedAxIsFK=H5Zw9Nutw6$dDR4~Co0QLAPMC=9av4dDdhPlGNzW?KmhpmY
zuao~F<TBuxn4}vtP}2={rYc*e0Xx|b!TrFuj%Ptf3<#DXEMITMjX=V10{9R(0$86!
zA=YQwc+P2yCFY6wy%z^MJx%Wd`-5+SmqBb(%tKI%B&N3x#R2Lja0}QAI6>?QBr4W_
zE|>w7nAcg<^LTAsB^gcN$v~30vQmPk!fw9#=5X3+r-du7xFYPb%Pyh1x;i62Yu2m@
z{Lp4oS=$vS;kxa%+rqSI)4~~NoDrUS>Z$O;3opE`ObODHPd*ub@Pi)&$V1qB@4d4&
z@^cVbJ0mfBUB-3>M2}A;;*&4>s;jQbJXQPb<0I{HqNisR_`fb_nFM4paA~(Q#Xj|8
z@G#gFJP2w*UShv1?>m!7WIsO`ECaseSdPdO)5h|XtlMNTHGjOmc9$+~zwIo)$ND6#
zI#~AM;M1TZ%4=2SDRG?!67lJ|mSCrN3ET|yko+$w_gE0rX0iM@=*;K*SHOYI!Rjn<
zIru%W&U{+WTPrb-^}Paq4J2FMmv#IGTmkaw;yJ-Bl9)c;lcer}dQ9WE2he-=L!e%*
z_fB9sp)^N;90acivp~$lY4gIbiib-sxg=b4(M92~!ww5IHFaHY)F|i5I?=mw`|Y;}
zJu;FIJuF8ad1M$eWJu<9dG5LAvgI+oMSJeKXP7WyLgtm(dFP!oB4sJFX3YvuJn=+B
zKEqtn+Gd+=f&^y3fC1UEmCIe_!tY;&^Ups&oN&Sk;n-u3%}9@F@@=$T5;%S@D~lgL
zetdZATi+V^b@Xuk_16bU+=2xQvbtO%JLQy9GEb9BWiIJSa-M(w`5=k;#3w!x?!EWk
zaQEGJXHAh<z4yKE%}C&%|NQ6Red%q|J2iRo<cyT@+hk$(?AhVdpZ;_hI&^6HEp*ql
z-o3rSOyCPcXG<hL+I44&eM*9VE|3U(608CCF<~Ew{f^V@%M}mf_+GFLm<=ugkATO)
z1`x&k=FR7iaou@z{r6wMk>LG6kGSJPtNJ_XD~aVC3bd0v`Ky#Rg>|9^d=N=`t_B<&
z{s(*zbjBzj2c9?x-9tU!0)GUDgWrJzL5w%O=WUgk-{Y;|8{h`;G4Nebx7JfP7`C+@
z%N9y3!^!Ak@FO6Bl(>8WoDX!r>&{l<KyJT?d9w)qi~;WezX7`ITO|?_$-=I??wXMv
z2|ykerdQ_5E3XU(9(Z6zTyDALmT<=%cZ8)&mxhNQemLBD=bd5q-FFX%9CApw_S$R1
zym|A2OH<cfcU?vfTso6bEL^y-VM)qz`3Z+Gc<|ux>tFvm%co~Wl3_Z@P85mPg%@6!
zwNY<WzI~qaa}e1cp+EfL58>#ekIrz-HP?i5&pkI=n!D+yo5I;=pB>)xrZ<H*zxmDK
zOJDj@Fy1!)_P4*ys<H0B{`Iff{vnsozW2TFWrXN0Z+T0YGiOdX=bUr0bRPM!(QH4G
z<WHTr4{EpFb_;KN+uJe%7u($fj>pml8Laly6|#{^7J+eHP?`O}mxV*XuE4%w|B6IF
z20tE1ihOzbSMU`0Ik2z&5%@y20a#Y-SFw*0)MLc9*~d=<Ujy@i_Z#n(({{@{DM2Ec
zq#HG$(I)A+8gOuU20Rar0Dl7xARR4n5VGUzk@+joC4CY&6X+TF1;{T+aoIpHJa4I3
zm-Tp@4=h*j&%42Sz(MwQV0?QAo&0@KT;}P~+atld6&wIQ0(_f22V4szHF2P`e#;R5
z4gLXq9x<NNmWiGcy&>g$Q{wWD<Ur!^_P4)1BQ}<*hsQlYdUh1agv&hUk+i)0^2=GA
z`-Y5@tVodl{ont+!J}lo4?OTd<}EQ?@*|0nfc@=ne+x$)byUNqWy2*?jkVDY(wAR;
zdA4Nc=4j=={_DTOd*1V&aN>z4W@Kc_lqs1vieE<$C!KUsM%E-iBS((Rh>+f+FMs*V
z*>aoY?Zz8#%m|boqxtjaXZQQz4}UoGAo1(#;qb!`Zy-5(cqUGqn2{gvwTH+=+ToI|
z9u4idD`XRuECyXce7qmMK7Ry~n~#F^U;x+$*l(u-UoI{L>R|@RzZ}@#mDnHau6o|z
zqiE8nfP~F4$ac57ual=SalSnUe5tVnDZFbm;Dbog^EL2Gn)pF*CFl&|6E)z#6iLZU
z&fTx@05}!=8k`H9{MP~{PYmipTJ|>%fY#;P>vcf4xb-+FTm^m$9AG<Q(9G9Ik=Z_=
zJ2^fCZUzT{cY+TANuSGnD?l9dcH;avAQ6oB&S}fVr6s);ZWxxx<jIBP#N`}4JEKRB
zZYb1GQMoLnCq|JxM55%LAmb!1dPLm#Yx+07@r{{RC>G+flbf0icNuA~z4i*f``z!d
zeMXW9_x(iCGo<Imr6fsEtjqY?+S<(HV_bfZjrZcG)Lce0zsqup_Y{}cMvNE{KKaQ{
zh6^sZAWWY=JzE-+Aiea`OWC=kO*!<?LmTR~?#YuUXJr~^do0uNe7lV6W;vevhXK8b
zT_v?tB=LFGSUSFD`<;k=;M<(*x$%BSf)l_R@F=jq%mQ;jUd(Ubi~TPj*VE45K!bd_
zIS=@PsdQw!)y3}M72xt&l5W(16IYU+uYpTx;&d<tboQ%m?Cf^b)!<t2b#OH}1N;>1
z0VK^1il%R?9huj96p7DIfP}}lB0WDtfobyTxNNiJ%d%b>g@d_uy2R&zcQm*dd<1L*
z`hu^5KY+QwGFmNT$BxasBH#JWce1@O5(Pa#dSmpWII&BN^nkdJMGuf4JwN#1gB$kP
z=#4Rt<m2LtFV2X;w%cx-?b&hfkb7+0OB44G*+@x(WxK)IkI(P0!w%W9(Jy}Si!9tU
z#w!x5FMjch*;17Ik!(Y(@4WNQ%dU?*?zn7EPK-01#6<!madWvXis$j#{XCZa?QefO
zeD<@S&DwRu5l3XpdwP4^4>WxE@P>Uwdb{*6={Zuo50}~&En1YdBZkMiy6HHXdiMp|
zkQ0Ab$~r0&eLApG`-FYjqeh|fJs%GC2EK^cNA3Uik?(`Ez%^hFP)skBHuaUdtX=xo
zvDFu@yTC6ysb=*m!@mw}E_&%CRdj!`$>Y=g&!8nO&@19w)v4fV;G0%wi=8tLfWD#p
z5jen~2;7fz3oz_u5C=$3TivPigMb71C13$K1N;em4Lk-+=ip&o?Uh((9H2K)pmodu
z{{*`5j{x55qrl1S3n1#<c34I%htsCTiQ7+fNlM&6eCw^ZW}A-n0KM;h@5{V3k_z|B
zxVOcn9QWUR>|-Abk|w<=E=~DqEWZQDkH5RzWVw1z-1nnr#ZAu=8N(F`#p$P?9)9$r
zA7vgE%h5CBJ{w7wA6-BB<dd^@_36_m^X7c>o8Js)opn}t=%I(Qa6L|XhHQsq#qjTb
z_q$meZU5K5{`Kr$^g`tci5scowH~kIk3T;19360g4mN%~j?K=W`OIgswz&t%dJJ>J
zwC&ClIMcg{+C5B?BA3{_-*}(KcZ0l@I=2EBbwk~Hee8Q<!Sn5`Ut`gxEi>A*`|XAR
zUqamXV*G{R6F`!4KCr(=v5Z#riI}mQj<LPtssA(JIq)qI$Fisgf;vpzLP?HiNxE4B
zKKvv-Ujsg;X~3nfL%>4Nl@bRD>(mQ#FnA|$aQrj487v1%t4Tp!w(M`dp2>u~9~=sP
z4J3XJw3hGS)X|c!+p@O@yMSK-y*@tz7XWpyCrqh$uMiKH2Yo|+o#-i9xpHL&Cw7U5
z6SyS7<s>~bal^5m5WO&ZJ|r%aCr{2k5vJ!x^5U|R1j&z}OO{;9a_OsF^pv<C$mKMN
zf+WMu&3c_AN$$JJ69~)qQ)nKA?GV;ydOcF+HO?^G@4c8uaf!=xem{{%uN9ZJy!Lx|
zqIkbiAGXQ1>G?6OM9T7G+e{bB)WalEs~}zV1%^|pTQ7mcST9Id%Q!0h9ry;grwx^B
z@6qPtB(FXDu(W+ioeb3%Ej=OL=PYm}_z!S0sC(eAk|lyR3yJqFvHMS8KmRaT0%9De
z9V#wMJ^{7`@dc_wD@cVU4Q#0zxSbn21@!BN8+1U(%Ooy73M6B?hxY^$geVT8ZIOIE
z!zkciz_s8^ur*LjlTVigcF>=Exz>FhI1xyEZU)}~4wQpHlxj{I@-ewr{H-p$A4IiY
zAQ>gTTc?DVsA6XJ_mo&dDzOYyF}{}tYKNUEK4<Y?;2faGpgUv~m7WVuXl3R0OYJo;
zgSnUjJ^=LUTm`-k^!98I${D78LeEcsu(?VN4gVCl0ob3FhCW<(*<pXx_etRT4l6Mg
zlr+$bG@u9MQsA0<cS(M5km$Std<QsKejaQGqL{X=1Spo_0ChMxA2`rY22t{DiD4aa
zv|NeJ4De>)wMz;|f^UJFz&F6oU@H*C0j>bis7QR;N<i=gmDr?QVh>HlBJhGF((|fj
zI-B_TtX&G#7EcG=CB6il1^%@K4bXn=hxxwfxuoqN@IOGC{w??*P>1m)gVP4F9K}3(
zfF^+$#_47wwpow0FEU5fx4JX$qlP;4&PlYA^Z*TX=ELp*x3|SLL89>#&<*XRTms!C
z+D@#080QX#rvjJ4ToSq!EC7mxpa^1m!&)nnjMIUG^7nynYKjB6Wp|Xs`}#i$l+=6{
ztjZPcgy0*Z&tNZb3-}v&7OVv>=RCk+Z_kr-xdu9OsY~0?mcI!`fd2+dL3fBRKc55I
zrhD!xiG9O1yLTsweRVwWh2;pK=jd*52bc}!gY}>>iT&d*B$x?)2r|2GEJF_r_r860
z-ox2l(7t}u;ka$RbGo3uR7EoykclSQd^E6urrrw<26Mq0&|M;liG$!W&h_{_4(!~%
zO&tsV1Rew{fKmn8N>U_xE;qTi;LE@P`$Dh@w5MFMzec<-Cj~t{E+gbg50}}#hFdv4
z33dZ-2R{KfgDb!ch>vZGQe=;cWp<|MWy+pKXV+C<kNPfAS6iQDXX>Ieb#2*)f$f0g
zzPm^dRcCf@zk7AVl%aKfQd>~s^YMAvW^n{K5IhTR29k8i&SKCKDb*+X2FFXmHQ;^_
z!#VAdlJyT-wmbMFxD}iYI$NxF0`R3Zk)F=hb_>f-#HWRd>OskTOZh0c2y`d$Q6+J(
zi-XE@gq;f<Oy3Ir6FdYYC{F<2JpVs?=K&{0k@n#xgCtQ93`E6@m=ANtoa1>0%<&AT
zXZ$=dhcjnAbIv(uQ88!Ck`xdT5tJOi=joloW|-ZbU1n!ycjx{6p6OIwRbAa(_3yW;
zdkia~Qf4-58%@aD;1@Uw6hAkE;>60mI&rewTr&=z=r4t9;0Gwr#<1ZP_yY9&><kyd
zLkuSF!k}f##^W^38n@H{VAHEnOQjQz&9W*sYf?iQvTS{F-wC`9ZSV_e-b)Ifl~bAU
zIx~;15BFtz=DUK|=LXOb2EzF;7`%_BLUrsrcCrroCmaXkpjgdvu}nIH`5g(~<M}Y}
z5g(*J$VXzOG_os!lJLo{oQf%w0}E}GZS<{dLM6v+!7Q1F8RWwLf1hsNg5AN2_8E8;
zbom<IirQ&%UluDh#opuaCmatQ;0pK^jI+Ap$I8)t6;!(J6>v5pm6-zQ1^5)!hHZE&
zYZh-DHSjGh-pcwCu`r(Xt?`yLt^qH}Y~K26i`Z(#e{xMA-5>%<GIvJa8<Ka+z7b=J
zR^tPV(|G&p``o3FlbAgJHP!9nb*h+e2RA`Q-9c=war4@o0E!zM*Kn;`20fu0YyfM(
z2=Lzf5PpVSiIhu->vh-^yeE^MP8N&CJa?_(a99`=KMKAeh?uQ0^W3Zh18OXVQp8m*
zfs*j4a*-8$2XEMq!JBC~*ccU~*#LX9e+%ou&afdo0`J3*pjj~}II1FmtYBXNEB=$<
zY`7SPLb4HHF>5pZCE*x202C`#F`u5Fh7b9Fs^|H>%^?Vqj_upGf1_*HuCMN~#~z=i
zM2{7)6|Tpe3!A}51UKcQh8ywm!l(EjtB;5@m-lr8j-vshYW$!<gF5kn#a;PgbW^_R
z^W2Fio_J#}4j_kdc-&qCHDQYq=?-8QXQ3OhhN%M_%Tytz=;;JY!^+SH76ZLI7s6oh
z!C)HX(!9sL?|fMJ0KBKGavu{sFV>V)_5;V=od$aB@*#RiCN7VSm>%Rp$x0<qDuL>k
zfL@eG!OFNcn3ar`vQJzJpX;G7B=fIesgAA0PPfB-&<nJx#$`id9`6Sm3M<M5G2_w`
zwH@37x5B=#0@Q?UPNbJZ@0y4_73Ydf&xMCU5!({#gLy8s<Ka%IsdH!=6`A?)I+b9n
zGGn*R7H~d11vkJxuskS=@?nnkQBW+DkHo5wMkq6{*XKdunr~Bpj)NxoPPmj_sRZg;
z3FtK#23BsNVrI7JO+m2={2x3C=YdZ{%^_H_Vh`&on|3FF4Zm@&lM0-z_zC5AF#Ej;
zR#Z8${{8zm=BvtG@cQh{2LK=71ATY#vA=`5b?fGNi*5JacW;y$Fznl4@gU`@PKkee
z;=dX%%o<hQH$A+Wavgb(&qLq7eOr>}J^5$!hxwS`^?W#R74sazrSD(p<+xCD|Km0L
z5FC*Es4I=p^N@ZmcVV4k_bkx+sTe&OHh{%pL4`X}RLK-zAH!OZp4Tcxl*7Q`zP3Ji
z0*=mM+}TEDS!@Y!XS+iwT&V;~B~Wn*_;jcHwnfFkigK&MZC&^;SSi_E6$A^j@l$Q6
zxLnT0Iuhs%cp0`z?U1dA>P~pOFnn^(mCemRf9lzUFD-Xt{LQ|6Y54)Tg~88zF#gA`
zwb)WZW9~j-eKu3#+L<`7Mpy-pS2nw43=;&88xd10@<ryeiTfGyei&a>c05UaGS7!k
z>X*Q^pcgM6X1ZRx$?MmLgSBCAuzTrI(91aj^g^|TLYf|@n_-bcW+Ih#xQ~`*`A)FB
znx(eobD{t}3w*#US*Zj{C6I#>*q=CVgM3xSImq&SQRrRS8g$P;2!}(m@%D)+RAS7Q
zYaxsS7mtMxVB1vaWaE)qHFx4R)6i9K<>Y*>E#%F3ZA5#3PoIy#RiH>~1;HxHZwQk<
z{W~kJfu89bs<<4|Zp?l=z};{=C`!${mOP%aHFf)UI}Yyw@X5WVEnJVubZorMgAWIG
znd*5u9uz@(TE2kf^}}M`&1=GL>vv%{sEPZCXscxA=Tf*GIzb`KhtlW3I<91;5-62G
z4oYBg;`kD}<shy?N70!K*m&OwR_^}*EBs(K2$eOsVS7u${u6Y)?*v+C(94#}ena>p
zSsVaXlr?D~J=5^%y*Hc%55w88F$6PgGJIHH-n1VZ^hnqRQXNaCWx1aYdgA^KbG=f`
zC&RH=r_Ez+2X{h$@SN3znNCAUmTha6H-v-W5>V`120KF!aJ%L;`!=lLLJ`<r<l!Y)
zu0VMSWw;ROp8$HrRw__lOxC(+B=~?=vLY%0pX^FjJ4@havh9=AQt%_phT3LURCC}b
z(Bty~^o8xA54;Ih>{cF{4TRw(-NL#_Y|!OC3w$!x9tZbLmXYDD6pu>r8`GM!kUx)k
zDtrb*U=7$64o%I!!SqAWDmvb>5M{r*)Qxvaa%GnDp<LWwuZDTC;kXXWi+K$ejv*}9
z<>?$=*;j!32dt0shpfqR$i;D6-r_YGmn&VCEAt726&WX`aOe)6*MW33-p?Pl4&zT+
ze`ne2T5G_#tET*;{dl%7GIQq4vbBZ#^;qtQZQx)y7bZYH%{*w^!Y1G~?8Bk!^L_1_
z=AzIH#z4u6s02#Fr-)8Q6|>@v^&aRs_!NA#w6<BOj64=A^dDdZya85Td%#-oKD+|s
zKnsm^+0y*K3V>U{#_BjQohKl<UmKQvn!XoI&pd?uFk535(lEV0;dS^5)&m7!FL(=H
z2g^fC=DTdoyuUV(##fcbIUKJI?=qm^SrZgw_rOrlLb$S}JpI!6d&~tfKRdAY+H3b7
zIdbHdEn2jA?ECM(A4EoSmY>vT4ZIHK?6S))i+}gscWX?ZJb78Z#k4ekWZRzcR6`hF
z_2?2yEb-+s%PjNjb=O@tk!|sy^M@XKs1bw9<7;T!ZMU8C`RAWcVEou(UwrXJuJc#b
zvJcnw+3a{1Yy{6jz7(0)sn@b&2OS4x+ES3~mvlWJ%fmNNvLY)1FN%`Y-V*3X);ECD
z;1{UvmaL>&b1*$&J5UUK3m?G?Fd4LD__8(g{wfIeh5f*;f~5FS^mK=Qpa3a^CG&4Z
zxfE;xn}9<0HFz0*11%Z8Y)#qkiA)qP|AZN3t1A2r@i?`7cLdAoL3kZZyI|&<VOVHL
z3zd%xNm!xnIIOA&%53v%_f$9N0ZYIv7zkg1-rQ`?Bf;X=*cGb}xKJc^F!`{X`FQv~
zk9q1&T4#bDma{<bZ=p<2&ylb+99^hPl+w*y0*y1TF0HL4C19iYFKAIq@>^LpLt|%y
ztQh$ihJuZR9-zHo75D(22cPgX!-sw26{(-;Onv%n1XjX^v)gMA*cAQ&g%Uj^JvYzh
z%VYTw%<ns}Jt#=lgf~FJ7aGI#<5jdMoWgcyGg{csvRW2)13iXcK*7B{<YBHMes*BN
z<H~ei=Z33+nb|{z4C(&qr=M=zvuDp2d-dw25H%mUvIe6@jp~G_W{Yv-#;rnk)hIp@
z@OJm^-M`^2za}KIGOP|>vz{+4V;Rsb*6`I=UtNZ``4(Mw-F3azy#^N4Y!e<qcvja_
zW1jDTEny${FVv)v4%hS|q;oj9tXM6i=pp&J9v|zlk`-AAc#)K>c9(!pRQtob5QISO
zKBBVRhifxb`t4b5BfSlD0-uZr!fOyLRDxkSOl`Ohgk9l!I09~jhoQWE*pzom=A|X6
z_X3}smxAx$br=9zGHidAmxM#0JPYA%P!&~TLunbj0ikgU;aF8TL--6Qb)TB^ZL{fC
zYK^F9N|)0l#?R>Ou}oxvO>Y))H)qUHYsL^6=M27z{X0U#?pcM8=O<XW_QUc#d7xXY
z5o3E6(mO_8{~%B<gpcWWC(dv^gtV(V_1e7-&V!dgk5)~XWujO+3^s%F!3QJPr<WhA
z@@C$@9=8u%@8vDkQbJjkz&tlnR(q5J6<Z11Lx$djyP(idIW<u>;TjE<SVt0B8+@W%
z2z~$s(Cbi<HD#lM=S)yEtq+B+9LYjPOXkZbS&w%+XagU@GcXu5!&=$51zqbEnVz%V
zU?un*?t`JACDSZhQ@~#Yf*uToKwUPkY0JIXtPR2^7oJeZ*KGhNLAYLP;#l<#9s+kj
zu{_MQBQHLzbc4EBFLPM6*K+ZMP<xNQZshzG4r5hlRW}Z2zN4^lNH!3^u>3UmB>60K
zhJE2Q*Z?w_jf)Mbo}w?o<vw7%#c1Jp&F46H9uzlDJ=Z70o}fo0gsrG@RUqySHb7nB
z4zN35C}^f*JjG-lk9jHx2rtW`#?m%^y+1XbPBo9gV_#77n))B;jqzIb8qB2`f5ZUr
z)$y3^%f~ebFzEY9%aqme$;)M8Rf)@wfbtdfPULF}`&D#bRwdd}*c_HXeDUNkzEV{A
zFM*Xw@@hB`hUP!{yr-C~*mTQx1O?Bgppdr=-hj6uX@|04<9{p|ep~Q~)Irn5Z0X0_
zlg#b{p7-xy0O;cX11^GOcoyTgT<q5B0-wS|FbuTxuw`qD;Fkt$0@={RRkHk-K(0JK
z=3H;(1#lTW2f65u57({TyJyg4IFoL_GnoG!iou$YiN0Vr*A`GvAx8QoODloW;O4Xf
zsfCN*lV=+>pZ@EEg-XQAZx)P#N5JmBjbIb#nwo!&={vAF^n>%@GqB9Mfp6{{48K9B
zOqnezOF!Ok$elvh21OVDVc-~`dqE4~J2CH=raoYUa1(q0lR+!i<1Ncw>9;PFfY;dj
z@Hlv#4u?Tdq3N~3!gZV}Y}mFt8>b?AyswKz^qAinurPcHA+OBV9?Rk!)SiRMkW(+r
zlGXkaup!?cY~*X@jaJQ5;PorvDN-Td0rdWC1iG57Y<GhH!Ceq+5oWFhJq!N?E30C+
zaKDCgthehxI0zI(SAybZEl?EcG0`LR6pR7QIKuw%iemgJd3*|bR&wd_@p@|k*MZ{X
zT<}4oT(i+-qhs(^h3Odr&K^5<?Bcw6w=-|TeY4tXt364Wwonw;HA1Z6C!c(>DTCx&
z_UY3{(HHL942M#5tL&0|4&iBiQfI9;osc?v5^`{0x{z;&CtxbnH}g7{Ab^@O-wy2B
z(yOB|)Fbj5C=P>lXXbhs2xo_twP5P8hDv@-=6#y>c<t_L2AWTskHTOGqAs}~tBco4
z!l$6;CYQn|97s=Qu7L;OZpb8PSY2kEe;-Qrg>X-&w#1V;z8UA6cYIZQDvk7XT|~(M
zg`8KonLn~LhqnTK_$r#bWl=6ZJRU>dZ_OfXtqtNqmaO)dz*ihX7ibRkQTQkV#0o~N
zRP}JI05<*(hF%x;hraL{=m{ALr-7c55l~xh_He9W`X_n!US0wg1(*H2fp5Ic1g$nz
zzGXQs=a;V>M?d}aQ}o9lfB1J2QRmK`qxS9F&l8%)Cn~QktZ3Ujk)$n*9z8l5Hf&hL
zK-#EBj~)?UbxsTM*P?kFI&^5nYwHo;>WTB%rcE0MG|$afa{jM9%jPQDmLAsSio#d4
zEmK@Iu+D4GhmV`#;MBZb(-oP8+q%2}2#QEBJrq(j=5^<FnX>R85|(Sj)Cb2pZFA2~
zoj%ICe{c@a?F9;-`fLgTX$WURCpZ+w!Yi;TYyf+M0_Y8R3HpQKN5G3v%w{7Jj?;Yh
zhRt9YJO*EY*Fqn-0G@@{z<so^f7wzk`yD=fc=Xm=Z$%$|_+i|*_*V`cJ9doz{`bGf
z;v&@;&J`D#4zD~xlqqajXSz>3@kGQ&2jjTu!iYB5V1roHgfLG1&n(5!OE0|?eel5t
z5nVA6Pyf-1E3O!AwbfP;A1j<2yrO(mv~6w#iJ6MRC%0L4>#x6lV$ZzHdhL1L`4D0^
ztX;D|!f+_hrm-Phv*}^t_z-j)T09mg#Gf9n7?z7UZ{?Lw@DP``z1FWU37=Yj3gZL#
zhTm<Vz6+mlHuq;{6F31zr&c<B5MBTU(1x%Z818v++wm|827wi|X2n$uW`4{|$dBHg
zok0<2MRhkk1ykW;@EF{FAgloo!!QW@hxk}kJ8bIIsnKn>-4;!nG%4!aw{O&`Q>SR!
zv}w`c!Gj~lRg`-W*?K%G3a8NG4?&~-_uoI_zqO*@e)}!Xhu)l|{rvOK(Pfui77L;S
z4?Hm9tIDxwYW(=|@&3uMxtureT2ka>ivPqDPmJDu_uWjJYI>a+oCO!dO|Tc(oi!0M
zSwmKadoX=_ygqfT&X*T1eZN917jxdqE05Zk=kksv<NA{D$yhA4Zlzsa-Jw~n%X#iI
zyg5-G0vCY;eZx)c)NUUKR{G@mGHe8!fR(W0V;mrP790&fgBJD=f}w0}{(jTb_nzGv
zwoGMVJkufI@hEyUr*DJT#dhE^o(!*ogDn+G9*=Ps>bOX{YWDv7??)G3d~xhaS#G)I
zVvmmv{_3l*9(Q}_?fLfGZ=(?-M#P>MMaLRztPyufef8B>u_#c0*bQYj)!+X1x46rU
ze?5<uU3S@6tSGJw%lBboVHI+y*T(L&u3fvvUK>3>ZQFXe{AGHdKKkgRXyU|)(ZB!w
z@2Fe1ZgHB5GCe`Ob{z{e{<0!Y!@S#VX4=MWSK1Fh{1EXK=E!{V=cDm98+eb4KTeI~
zGo3+$21R@)CR$;I6{7dvdoNDY?iD>s{IO~D<(Cfe;$lajDGpa%b=Byz&pwO8zVgZ|
z5r2XjAOByu&&!<Go8H$e;Q!z|&`VjeQVHa>1XR@}tNkTlWnwKn@8f6fKd!1C!7A)<
zD%=lGq;|Gp7hM8Yyk7G@dHxQ*4RQw99i-V!v;+JXIzkWzK^(Z8PfMon{EBc2DAW{i
z=2O9A{w71nUzmD~_kvH^BR~((>97K5g*vXX)T{XW?6c2CyY04HwDi(T&mBXdC}`TW
zX)JvB7xn1stFMlJ`Q?|$aX{Bzdu{aMi!a7v!|9b*UK#0;Q8?Uq<Bbt-1;qm4(n~Lm
z_c1Qxxaz8_qA_E}%#%9>iu(*0Fd$CL?ls5nD4K$~&l_*N5pBKo)=~HF-D8pBHU*D)
zRG1j%?z``fMTj1xmtTH4dhWUB;&}92op8blk=<$&CQOL<SN1qQh0C$W9vh1v<MY_g
zJ@?%BSnRsG;f5RHwDg3TN00IG#~+V9Pl_Uip5jZvrI*a(SBMqByxzPv2f%Z%53CL)
zE0sWAOQ7WO$!l@fXkymDZ6JINwnh^xrpRfq=I}JUEDcA0o|Zp3>#Vb`KkTr>;xFT7
z$fuQW!-pPvsKFk4?4bbqgD3B2_t|Hk0d!}q!ISo{G&D!hNX^D$@B&Z%ulMWMZw&DT
zFG+dv#z~o%3~5$%m5qb(gt!)CUeU5PIQHQoco7sktw2xTbO_<Y)Ob9`Z{Skc1a^WG
z;3d%W6^<*U$*STsMS~5u7P@wvSFmW0KKf`ZFdR#xw@1&*nP;9EZM4xwaYJvo_10T2
zYS*q^EG`~?_~D3i8;=p<PeY?FUAjbimGq7z$Kp6<M=y}$dR}<Jo>KnvW2Kd1;p7+|
zJx-cJM$w@+MN7}4UZbzS{yHA?eBOEI#onI97F#Sj{q)mgG3P!CCiAaIQn0-C+H27^
z+iVkUw%KOUQ%^k=Ex-Ko(HCEQ5eubNR#_$9=0NM_&6~$U%x*S4S&AD6fO9WIreS;=
zZ@h6_4x4Ve>D)3i%{*JUHtptG2K3IJ4?jVbuR+T=oPYMui&Ztvs)o<YLDcg^O2Vh^
zTmZjvC<{{|H=!c15v)(vnFh<Q9K9O<ml7#fh@p~AFN=BB6HafCU0tuG_?GGAd@H0a
zPv3psdFLHHL3&n(!R!j+W%B}ASquJ`fKkwF2@YP?qxUH&R(^zFid`Sy=Q*a-M%RVL
zWGeGMW!nexcTfQ7`MD1~-mrfN&#K}wJw4Aq|9mVcI0y4(&hDTsw%8)_?I|02yMgT9
zQ3Tix#`j+0ZF*ny*eH}vKKbN$ywH<RJ{gM#1&#xy7g=PHIDQ)~2P)hBblPdB#a&T)
zbsY1fhh^uT)uUx**VM`@uN(`GOpTo$BE2pSbPjK!b?eqG7GZXAnP0o9%)cHQ!|^?s
zSj_FY=bq6q#~c&wvdb>f0}nh99dyt^aaWu9IQr<L;|wUc?8b6`Jy*SZ_nt?<>3MQY
zlE+^JW*w!+Qg7sba5Z=jW@QRY-<tE>DsmNn3fgd8SpLDnI+v5{T~2OuIasFi=Q_P{
zLxv2A#jWDkv@@Au^*kD0L2S7zYCZ2E&17D}x)4|THkUJn3G33shj^SOw<XtyeUt0z
z%X~@r)SWY}@M+7_XA}DWeag2>r_LJ<R|lSoPvt<T9e()Xr!$~*8uQldf7N`Ilmd-i
z0~&({G&+`>BIrK61dA{};vZCOdP|n4AzGQI>K~|JU%%syJ6xA3ep}-iR4AFYv8h5!
z3XUd(Tn%=BZZH_`gwG)KQ5arxTHXLrw}-rh_!Ui;!4_bveG)tmPl4jd@F6U-#bY-5
zlzl9I^j_Hgq(D#{^z7L)T5`!HBRw~INNi~AI&!@tNJGfR<6RU%3IoT?C<fMAYpn=z
z8y#@K0nsU^oDz@Y2}iA$#_lXdjouvvk)D*j_ue}?>7<k5<<Ka(ZJl-2iMyEe%;+If
zWVpYLwA+34e&E1?@faQ(O&f6j*C|d@VWgPSY|zCqN19`c<ffZ$iW}3j&ptc$@NB;M
z=CLSCdYwG>M;>`3K34ZlHhe$+_+uQe>lJeaH0!5fa1Dfqmb;h_UDC|XZ<dGSm+T%?
zRCrG8`qTTh?Y7&-W$9k!HR|Q9Pfsa5T<9vi^Ugcxjs^3aDo8V#Z}cfby;d9x=DF3Q
z=ylVfLx*^m^fZ$3q;E@KHf(ZTay}VPdN>~r%!}j793xg=tt5Qv&Y5>mLR%VfpQH|l
zCBRCg&Ki|bL&iibMML*@p0w|#67<p6eugR?w@=m@@inj^#AYF;nlgT3Wg3+C8E>%|
z-5p!7y)j*1Z!tdNGa6I7h|;Y&D=vpo=^H9o4kAOrVd7n37g!m50(QKT-BJoV&G4Gx
zJyw?|KxGyLjp5Auv!EwyZwTq#!~FLU!a6M)(<9;-mJ2VuFn*eL0J06C9+@-FI3sR^
zF2DTpxKY&mL3d_67AAOr9C&P48!<gQM;>`(ES?kq3I_)lD}sVI#sgP0*>&Wr?DySw
zUo5U1e`8mb0!q<k{NlA^Jp5T^bjvNb#0|5aB^y7x-S*mRuehtq_<cJ~?~dJ8`|i7M
z?45DUQ4lFMWMSTXC}tF92OoTJ?D^5VVFRZ4a*U7NSqde^-IiN!88?cGHZ3%Y?&DzZ
zXP$W`_Kw&ESU?N+=m7GtH0%m`Y+nJdyRytPlQcbJ=bUp+wBd#uMn@fWR6I!C^C2F8
zFvl=iex4K0ljd^3YryrEjq|Dder{R0P7CWTYp)+g*EQE%6MOFLhD<l(_d3!$xWf)R
z#GW|?y61o7$dPm7cfXKF^AXZ9UGwC*OXfeUcfW8xllk`8L--){!*=5e`#UDgvhv>X
zx)13ZZ+)^*iPooMuB*p-9tUPac@a0j*G!kIs|T8PT;Up?$#FNPQdr=|ns|sN?*3AU
z=wUI9vf8sO@$%9z+$!v=pa}V@C~R_DJNDTY)(0O*-i61&LCnF7Gr3PEmj4aI;A+U$
zlJR(qTY=sm=M_XU1+-+?vNd`5<s2yH?8ckNSRacG8+f}Ath{~dcU+E7u)ghN17oA&
zn_GHy6hVq5*Ke}PCUH2uFYaR)<9zVJ2V<{~Z)52>N}3)UUttdom%>HwOK5NvWXU(Y
z46mT^%`Y1q$KUw0@0(~Y>vd6hnXZCKuasUN-$rvBlj6z7$_CUlJqFF=)iY(cPPcB|
z;&nE7dXP+0Gd{bx5bCiISaZ!a<MrXku)gW#@jKqB9IJ^hH<y!CyxME*Z0H8oV?*%b
z+{U?4l+QIqu7c4rIsW+L<MSRiT9%sUT~FUP-+U7{>WU7#q!fmFX!P`XK74D=veE<R
zc~k&+J!qDtgTgI$!)OjL*Mp+~4ui?<wp2hQ&GVsG=-zwpjlFJq3WJ3_Qa{9Fz3eh`
zooOq$^#pm{==C!XiY?7+)-Fx+Vppm{$^4r~$9|a)k2we&g}UjwkH`A!ufN6z?4?)$
z06+jqL_t*Xd5zgk>UFvC#v8|D$t;%)b_jXAIm7xov#R9rsWYcCeu@vB_xi`JOzMkS
z>nVCX-UUAHR6L3REqRZaj`5e91{G4+F9;Nuoe#@l->~c;fj*$<(FZ;V6g=NSF!xEu
z5!Ne~+%_(($Z~32As;^4Ds(=Ff52{_x9N5m4u<s@!V@~PHEQN7<ZZr$JU0phE!mKS
zfHn*^bXw{>#`89adiERxq!3DOa}cqf8@o#M(4?D<r=F6KZrImG&pd?Xu%F@AU3cA#
z`LogL-o1NF`o0dR7S|h20VW}wB5l)6Hx0?eQ-z<N8^dTJFN&W`mdsbC?FCwA=C1_D
z`|J+CA#PUJBbGFUp>Ngc^-}C6EgZLk@AlhokHw+q$G7GbO51I>T`V^Ij?qOIT@;s>
zo-akDBG!TKz6EG`Dk!eM{`&YFhBWk|C>HH%-D#(t;&EZduZU2%1c9UY&@1WNhhbfa
zBjnNUP2Vt7&=}^Pd+v$Hj{WnW|BPOJ_0@RnnQ!rVo%)vFfB*a6SSZ_lrYBMFpXHzk
zv`h7_yY7kwn8)HVDTMSQ`WB>JTYAS7IHoUlzbc4AenS50(==JB*QevIi^pnLNQ=~g
z>7^*E3G3pK&VNiclr?d@$#_G9Twyvq5n6E@)^RyLh6lkA_7BU7)aAi$rta`F_>}bq
z1alv)yfhU`ijrRqR7DGMnuhZu;3C)#^!}Uy55NFOcC&^3Se-L74`SIEe}LKYt-Ojm
z(|Q{|8?{UozE9S6OX#@?-2|CZ%@l8@bz$F%wi%|p?dj{&mqYx?X*OX(_^N~r`_%B%
zYj`XueEP$xczotlpn6>tsh&qIUF<13KK}UQc)&4lX~y0>MX26BMS<sb#~pW!MT?EU
zLdNb(zhwRU@4v@_M{#@q{r5-iufWjsOudC$)9)L%FEL`oh|wh=-QAti(jbg3Y3VUW
zhopdXBZ5kIgD^s*L5UCDAl(hm{QiE&@jQP3J9uyR`>yMCp4Svx+eP|(+8y#(#RDlT
z^uOCH`wZ!X_M`1={CE((`sv;HQDsM=m1E#Sk_|o*y)3IF=AK2#dDL||Dn0F&M_ckQ
z!$>L%lo;G$O-F`#gZsm;Nt|LCBWodyGUGM-RqX@$nw^DkNRObPQXYhD1GHS>uOM~L
zhF#F+)$5vjF&w4b=M+?zExvr=Kb^6|W@Gj{(MXgC%<qm+;U$!m5#II9+M`>cop~`U
zgQhLF3$YfKPD=ZLOBXnrl%VW0Dj3gLf^3pz(HrVPP=Vza3i)zTQ}MltX?W6K;V#y%
z(^5E!#PlS>7Wk=)%iG2l=sasc?vd>jXiP>~I}s9jNuoJ@o{al;vdOl^WxG73*{n22
zbGZ(Sm=*)hQ1QK##*TVsbjiFrGCcE;;XOeBfh>m3#hj9DbE0{)T*F-FXM);=i6hna
zrrtuxM*Z;b8XyfI32p4qeu|2Cq-Pqc`v4AI`A~;5^3CbC^VOg})2Q4Hlz>EIh))no
z)9Fhc;@<4=Gdk=o&N5>LiDEW{39+9x#b_%HdpiQ#k8Idin*>Xr7pmyc?|2$B`#Lap
z+R%fBVEx#C>I>93GN6XEjkg{&xQ;d{E;{edPk;aFnC^!2Pz2*-hbDPEWBS^#=^mV{
z;64GIds#g2C9&i#BF@PoUgZ30Ts^)0vPdxRiscm95M8%0U-<H#Di^TbOh_&qVbi3x
z4M*9)Ha1L8a-w<htn%fs4Oq#5qC44ip<0jU+~R_W5@ZKjOg7a7H%))v<jUBy&*@g{
z4bb#=rNR8X7VOWnO?(d2A4FzvCY{$s$p#-q{TI8}JSq~GY~a1JR)qw*uh^jNKKrCN
zD1$n!kYuaol(ow!(E(-mWZJ^R<`sCp8#^HWo=q8?;9K@r&by`+@~Trp&YykYAXdPw
z`_`70CTFXK{k+2AHJ>$Yp5FozMWXQwit4s}Q^IB%uQ$2nxp(+}qN8`v)A7kCc3Hh$
zaI|1>$Me5Y4&OBK!vAhaB<O>qgm|E1W)>Ee$mv<ap-U`!-BFY(RgG&A>$U>DJloy?
zVFaBBz>)?0oy&EXJI?v6Lc{mX&q1Z;y*p0$s`3p+*ewB3gsidAe1jB<i{Q5NEB1xj
z+`}Pf2Mntt)J~blRdgLJDeT}7`~QZ5*3&59xGY0u{sjkzpk3T@#5>C%K?;|LinQT7
z!OWql7TayX&Fp+0Z4z^Js~o4zbNJ7*%)ihY4?I^yD74!QkB7hMo5Zx8RaJl+bxV2K
z7V^9uhxuW@(bKYe!;1R2nBe44v^|=d?n}mi(b#1TE99W+jAMDA=tf7D)sWiRMida&
z%isi!0Mk5~$I{XQqL-8sx88VD6fC`ldVs0nJipvH+snWzK*s(!wEsrC&21BBJqsI_
zG_@WjL(Sc3qc=t*4f(hL$?~e&={5~uHTb}f2C$WHG+W(;8=H6h^n<97Ha&OJ5WlXZ
z2^=1-mZTdb$kUecCxJ*b1CfbNrjs1}(>p;U+K%ZsL1!eLEh%ee>z=%B1o7_EaoM+y
zd5;AHZxrJX+UWh?*4GSfP?$rO%zJu=vZ_z<QHZAFvW!k>Q;m^ruFnb23S$=NghP(k
z49Rg#`OW6D>nvu~bq`owOyyK`c3J%|!XB~H!{txV$kj~VGTS}D?bXs}i5!Q8sQX-{
zi6GgE+>x`Z%$dEn-BEjQcnkJwPq`#Ig@LG%A8h<x`pkVjs;fICjyWEX+aWr5iEgqd
zGT}Oq4~`6@>({3y&#>_W({?c<LrV8RCP3<vqHhkHB3Q3zfz4aHwc2SxiKIhld-9?;
z#I=1zQ9Ci9zFn=TIA&UpczJos#%?wTLfxsJh&L6tY5XcO$v%H^yaCrnjP(My{A+z6
zk2el9w`-|%Rt1#Y%NPFVwz;h$Q_WCBZ54=bBWgCLc}J*6l@?#xF3=*=8PH*JW`)ln
z<jN8A>n^iWv=2C#(^PkI`!hJ*sY3QMUqp@J_Cn8DA*RjCcyx&i^X6AKS9v?XT%&4F
zL55kFKU00NSxXPJ%CYO=SnGo~%^^~9a(<u=d|)d9_ivV=AhR+L-Bo|hDxi8OUb(R$
zvTcyMSHihlK&F;~3n|{~z^XJSvuQn1<b0&hly6F$e*cZ|`aMo!K<S{y07C5R5ug8(
zPtgJR^KDJ~tK!{BgunCy?$hBIo(3o~*Z5{RwfJXu{c8~b^1A(5T2W3f-roFYz>vz!
z-;R?Z?YN92Jp(laixWo|a1{8%UFn94P$@>6qRswL#>mXy3!vUT#P2%1Z7&$F&GZCH
zsWL98gAp@GMn+up{$`=<mRtKRx%>A|<l%H?Ivbr;=NjVb<MXexOxmuF7MQqzR-|=+
ze0*82-JC^6=Q3d@>G6+LUW^9qo}(%}IroqGAjIFt0GuwQNK~OPGW)2SSg#45T=%-6
za3fH6(l)oG?3SpgD++DRLuXM6t2;CucO~1de@tza@4sx;sey3$o$%kEhB^jrPcpSb
zO3&TCc{mC0l!;ZJJVlo+MXz}&22CyKuZ=bOFxvfzsY>c3|8XaU7QC`03g*VKZo&qh
zWebnj&Z^7oV(o+#tEfX%?3_tYX4?j!TH312(*m1sb$b|7*))n~Bkz!nkiO#V1tw=y
zxP1!I)*>-9js<m_?TfrTm^M_sNx<pmSljQo;xqF0vimRH$sFqW*YGkp*<0pbaEl`i
z-X;YjV~Q|}5vw`=x1Exa-4@XVQ(O)24_sDbHJnZ1?jr>Ia=NHzFy%h|TQf_KHi89B
zl=tW&aZbhTUTjQiMD^UCphj@h+wVY?FAgNribbY<3UR1k0MV3s0(mh~w3lD==ma)7
z$$5X``txv%l_E-Y6@FO6VTM!Xjvv>{Un9;61qW}x>AB{Ep=Y5(9kJ%A3v8s+3>q2d
z*PL2jT#)}En~oDDp#7$VS<zxb*Ae9wrCm6nG46Yo<S<RFd!st^R|Wf1g!Yoycp4N2
zD_>1f=l;-rUijja?WYLNfnJlf4%RMV9hF8>6xzPoab0m=RoSL++}IJv#+yRND9=@k
zItg{Y4b0vr<W92h@mH7B+5&KII|svWCCHD{`$0YAWgpmOfJ&n}*(p=yen(SgbLjzN
zJU+l9XK+)5J1om#!z<59>D+&5&_+ljX4o4n3uhgfb)?ZQ^cdN^><PC=3ziYpY}U}O
zD<D(_^HYW96H-Nt${qiDK@N<K*1?L6m=JsBIKSneCXr(E>m~C*c6<OS5bpvYanRyL
zH?kR(Lb7s(oVUn*Xqmv(ojzE4d7wK<WOt?e*#)AqlDfZsoSQ*>yZvj|<+@rhlmYa^
za$br2c<Ph04<GUj=|(!o`3y)bIl~es+_YG|JVcYvyB_n5XW~d9V#O|C5F-@(8HHDt
z9}&O(Lb6-0s8ALDnc?<}sT4mvxL0`v#m86WHoNR`9sjqU3oAKT9Y5ZyCjZ&OW;ShQ
zcH%+*p~ecaE=WC5cB_evIVZ+CStZmhSnmJE@>PF0+VMm4!K_7a(9x{lk__x~?+G*s
zmS>P$Y*`yOR(<;J=`h6oy?;e)-pO70O52J0_ceFGL%72ECKVBUNnd~tSkx7CKQQ?8
zTTi=xrQ74AIKYS?U=faOaGdts4sKgs$I_KS5^O&&ar3FitQ{lRofYL<Y7fp;*|WR(
z{t!3d?)K{}cA&+w?L3DFuXY{nG}L)x>2l%R3Ot3F>)?dX=ByupcY(w`$|LeL=#2En
z{P{Iii%#hY)t+P9uY>H`l%9vg+7gYB?*$f{zVmm<U!z|1!ruEcuB6vfxK%64jhmTL
z4tR2t8<$?7SDo3M1Ef`rHIn0kSPRxt_UA?w0Xbza1$chn1`guX*!p}%hlSM<{+4Id
zYwD}6)@9cXlU*B#Zx$n=@`(&D?t!~Wu|cE3d~n`jcRkjS3tvYgGtd6TXVNngZ7ETx
z3mAY8#-%O%m?)2Mx6Q<4kC@9@&%x@!E#p1gu;}U`INQ47b*J<VzugJho=h6cLvpOO
zI{X^l@JDO+WO<Frjcd`7b_=1#IBfwu8L?_sft9B>U#t>_s{{f0gSE&CRsZST8gFFA
z!f3PgxCnReX@4>^oFMwpxZ#1LqJ_9U!sCc|EN98AYw10cBwGG#mj?!Ggkd;IPuXU{
zOMU$zQDbvg^#S6u9fg#2sMs8dR>=8}Q?<b!lERF^AQDHp3**5S?mmRe*of$Y=V87=
zYxh8Fkm9lSm~}OP&@*^!4Mp~Jx`fnrmJ>SWi`g!BxK4?mRYAdl@DW}49r%VFsQ1&g
zKew6l*Ow+U_59~|CPN!~mDU_>d8Vn1gPOL`_pt)b&6>Oaz227g2_NWH)&8j|vIGup
zTL}&6a@*@BT-e&2RXg1FgvAAICHhU?Z(P5E5zA{QIkA3v7<80+mA$0@?~eIE>g><!
z{!c4IygUQRu<+iFxwO(4>|3?5eWtT<?8j&;sW;F<ur6|s8u&n`49u4fy6anrALL1A
zM|N0?3b^pPOmx>>B_9KQpz&V~J@w2?givF`-8)VPJiG8^9!^^2$8Fv`+_d^%bB*ts
zmb;-H!+YeFs{Q>5C__J7McA!j8IyHIA6b&k9TTj_yp8DNuTL|5UqFyltVG|;x;v%k
z@eM{|4Xol>JqK$PSFgU%sWAt+hX&wy3gid~Xq|s+9ugU@imH=VxBr`Ef-m%DK<G_8
za0hLT_NFil9d47eJj(l1hN1z5*1URX-XA{r<!_9K)GGvuE$z6s&EN0%l4g(cQicl=
zEgKiiBqgh7nU1e-Hfw8~dD*D?=z1_-erDr+=xe3Bm~t*UnK+QveRYG8wjai#ux8!D
zi!tD?DGE`pr(*LKnr|%8nfP2Rt4sawG97;>iS9=Hd_;gzDQ%hgdRqPSN8mCeg3Vh4
z0Sdgm$_OT>n8}e*O0)vgMNL<nm|bNGcB&<IdUh6%JRt84dN8Q^P=AW_KhROFEY~;>
zUmW^Qf8LPxMf<xqb+G=veLp;4F+KVgdkfsO<^Qg@h_7DWNU|eOeth}w1O+sd!4b^L
z&xxh<x^5kaw{C$cAM%N*Ir6Zoo@zL%zwAut$G)>9E@pczSkyxH4C?&zteqG2u4iZh
zKnpmx>7d86C;KUH7DznY<^hg3DI|A39!rn3I~LL%v6t>_)k5n(Oa0gZH|7*L5<@Jb
zK+dxwv@f=E+>hj&ug@N&5S5ZKCf5{xRaHF2-@B5Zqv8!Z;%2u&45Z_luKbbX_xWfQ
ztiS4+fz_{I?~3)$>2~;EIc;RrGVn=2-#%=#y>;&tA_c(Z^ou@6Be-7yA}lT&?uR^O
zv<`L8RziiOMAql%y5s-)zg!8^`AMFUPu?Dxi*dRNACoXt9ghp^5d^>5$q#gU&>jS)
zuF~i>h`uldC7$~Rv?f7V&cCXj9@ZN{0+^oq+Se83wv5zBJbd-`NOWJeN;*yy`Y*dx
z>S>8FkqW~*hW!P*zyFP%pnm?`{Hu9^lV#(Qi+=-%I(ag%;VC-y^N8>In^pZfcAEBA
zS(SRN(;`gASCV*8n=1?i{Kat(FFEEA#v}%lF~8d^pthd++`0FR7l47#`}WP+fgfE!
zo1V77#oX{F7xNwgnZR=kvv^6i&6ioaeq7*zJ@;|O;aon~qz^zaHc!7<i)V%lWgQke
zvZ@wh8<)00@J*o5hR76Ux(L#482!|#ypS*LbKpyJ&^p&8k{06{3sag2|3o6awQOtl
z^Yn*#B)Yb`XZW$KP5ak}GRQOe_;}rYfrziCSd3r&sBqjY$NP&?R?&;q8-QLX^I8r$
zkfQVT(@ui9-0SIkFQnFDAUuYI6sk!bc%Bse!1yHDQ9R%#q^X_Aph&;TvO^>53MY9B
z-!><FzvwyGA55_8|7wsatLmQB8_d70ha9-?3Akx<%$<RS1lxt-__5EeCPjAsfTaE0
z<JsyJSg$hAZ;9g$;_d)@GLw9ZU_5Rws^or4LVse~Xd^4UU3?A%#)&}7qxu9@@gQ?w
zkBdtz&4vjjSb`($ic1n(CNyE3j(`)3w_gPc<BnE7^4(r~<`u={4tKYd8})Ag0a&e7
z4-y_;fOcdeQ*<t_b0pk=>Tr)dUWw{GCsm=2s3FIQTc?c#k*@HnOMw=P`)Su8LU;2-
zt8V23ODI5K0Iz8a5N~l3RsA~XRI7(yw=;t4d_F(??;%9uiv7CxJIy@L7QX>%Gk*}>
zQPhXU2RZ>RJ>n6^invZXD4)wJ32~JkIkE;io>1IpG`>`NQ-kwUqzJ5#m@I}YquE$(
z2)DOfggE*NnFl{!o|03nd{Psc*5{?@zK=R$pUvv({Xpe<<W^ek(EkXg2MQu0j+ofn
zMU%LrLx+4_Z#5bEV^G!T`<zE>cYZz!0rJkFsq)ZRz7^(r&I>IQ(o!d}8{v%eo{<a)
zPp3A1RnrLn-1ol2X-Ccf*cIUfxDnAW*<{o00{}Xq?6#6O<7=)+zlFXdhPf4uF^P+)
zYdly|CfwSA8XYI$Lz-0+Nio2o3v3-NFVaH2j85gG?cd}{=v_5G9w5a!pcVE(r=XGs
zGgumi)eLr$W34%UA(d<-JLUY-x{=dNX2z>5Y_@v2yF}HY_T@~hz7*NDToBndKn&%J
z;=T$ttk=p7Ls&<6oelwuBF}U;H-0lI4Dl>bAe*7?kfryR#JfKkgnL!B{rxWHZ@C@e
zbB<nNOW&&fzAI&E+ilFgK*Nf^GKb3P(t+AfZ@Nhc+YEcZjHr(AWbygS)UWiwdN+v{
zEJ(E9YVz*~*;2A_?X`WmKvmP2@s@IDdYOSo_le7HnTC+RnE|Z?E*jznEMej;;|$cP
z<{$1!PHQh&+^=P4sdxCya+Wb6#K1LXF|E(ylL0}2<9WJydFuDSVc4JA+TL}=JBtH2
zq}>|`dCQCYF5XY%56bll`b+;>*oI;%-!mJYl$0DJ>q5b7MJ!CIkqx;~L8)sg`?=at
zBDwo1R*ST~uaITVR*Uyy6z(w`73v_eE4($c8F|sN*V;PsGj$e{s;*h}@>Pi<j1&e+
z=jtWGiE5gf90ACSVrOgHcb(e(P&HCbE}j8z$_EVJ$VaTio!6%PU7L5`NA^$OJj4SE
z2BLfD3(5Y<-2lE@V-n9PA9;|Qr_t>D8!Nrz9y8JePa7(>@#Uh($Z;RzRR>mAH_P%2
zqZGdApz7$tvCGGi(#auXNBiQ<!5un4;?C__31B)AIdj}dOB<FE+_uV=?5g3`KnM@^
zMR@XAnZGy9U&XePhgnh1WQgFc^$JSm?hCl8TNusPo!V0wVddnC7r1q_bxZAI^xRyy
zikU%TTPiM8O8UR;>SFmt3h9dO4xx-g&A^nxSeivRF6c-&b{UEz4FE(gnYF%CC6Otr
zt$#ZvNe3toaawpxPjJ;LMEr2W8y4vsa_JNvmRl33$5OZQJS!pFM)4=>Absz;b_Y$4
z;limVRQkA7G+jin-FePMZ4uL_?qmo-P2d3O0^e*h)zIJ8_o!XvEU>DLY4E7DedIl>
z;jUf!#bT^=PQWl~ojnrXbDj=brd-IlI10%IOGDkLhb{Z7<~&<DD)51{{bNQaqBox^
zRaB0(rxu^k=0q>U5-fRcP?w{t<+L@q6+dnrizWN5`iY`|rO86}?D!Gw$FUyxbD5p!
zR#Np$pV2D@9l%zvALs&fUr6RgKd^;BLhL<|HTTQnmm}Ag#U~V~%c)Epz5hGbNU|A>
zOo>NyqcSsHoo`>;k7ofw^&k;jF5^zLztkn-E(3-!Q75>W0Y7<FW8aW^Sxb**4+!p}
z?JE#SK%Zz)A``<{Iwe%cWeMmF$AEMJm{3_3V=CWJ-xF*YZYzOYDaUq?TgbHB_;{oT
z8>{0~!35q4ed8lZX><7YT}oP7$ZV6{J2VSqs7zAk{>PaI^#f4Isnnq@mZra}^v1Ju
zeb8BIGj6^klh&V(x-I0|=KnGVLUq=WI|-LGuc+`cazaF4(j#h0aUA5!aMP68iMMAM
zf3#uy_P~9_m8ec)@YIGdNPzGq*X{m?v5vz(@gjq8lulfsB}?I*%9?gnhqvTd(wAGm
zz({EvCtjSa-wiEogcUFFU`#E0HfGTE^GCRppl#lk!{<FAMmjkMW9pf(CAsR9eG<9Y
zbigTk7Unb!S}ZM9<a8n9F2xO6n2_BzYR#H=c7Cv7T0GGSHCf!rsp&qpN_dzdUp`2{
zyxOiz8yR+`UlsCV^nZ*#9GPj!0=3%)hR^!{zfV}lc0Wgu0J@)A4MfnPjUw74H8-Lz
z`^hZKlDj{N#`JUxTp^P@&u9N#qfa&WesjXX3q*T*>?F2uIzsI42L*VQQ>>u1`HV|l
z!>UUKWFPQun<iNy^>iU8=T3g2oT@gRtoO~)M>XumAGmQ0zx#zjQYJq>DVitG(chTW
zj|h%yTN0f^^v>v6L&ST;Py8GKRnO>Mk+p2{W}JraqAtY^z$>VXa;G-pBB7q|e4{Tz
zf<tcE=O36>1L|pcNY^$R{qQ@0JhBE=IAJ^=nfk|VzGdRu{k@~o)f*n($`(GpAD0FF
z)f37yW_$Xc7D6UB^v)f7QF%rNlPYO5^?gyIqZiRKRaB1;0VS{fxwW}!f2_TpF%~U`
z6iIA2OyBpFLfmrA+B_EbbT_(&#M^VY{(<)kDp>zmwk5)z>HEYUT3$~}v|LTe9QOWq
z0U=N<(J;rnW23c6^$1bn><~;S8YD+}fflcs?8=zy5&`xh3*FgQK9OXP!bjcEj<Tz@
z{YG?R+DgJ5C8D<v<ip<CF(z6;O^mYA!NLjfY{SXSAX_u7$;VyDLh_6GOGH{>3GX=5
zbo>6WqiAdWdlnbjnd)en)aZ|Acm7_p4Y$$U^|yle)-XIMo%BJo5S<Xr&8F)s^ssUE
z$q;cH)6M;>+q1bB@jDenTQ%J6<*t^kxoaKxv3uM<qr01Ytl;}t4S{hr@6YHg9;zdi
zRb;~YCX<JX#{|PLO6bR>FBjX(=pXKd4%&Qz*aM+fVX~*#0%Z0udhY6?mtoBOJmGTm
z*6<9}>Ym3UiWWG*E;c0^u;`mkewJa27KyKPUIB_xIO@26!nEWoCGt2X^|TPXX`&8h
zpi}<vA1gXhiYU1vPBSAdIso8qYXXoMvgu3e+y{P!w0Hiv7hKdENw3gH{X+Cr3wzFZ
zZnXnLI&^12-o~!Yx6=$mJIv;7SZ0%a)$95?3%n=n*!XBG2V<fXd+C&0O}Fx)fC63+
zs+7#$o`_Z6Z9~x<@mRRS8upFM96_5rm1_1WYs3@Io+mTnPhZELN&4Q-pYsU{Fgwc+
zv=e@RO=_%tSwCL00Q*H@{QrQwz$k>NPF<{eA#u(M+A%^td-g)0Fj}GL4eqwj7qKNm
zNVH8)$2+UQdEKMnw*+6pmZQ^BNZw#aD1VQs)BmML%Ct&63^_r)0q1Mi*I{XE#8xA$
zD05LQLhUy)%ULcyLa6y<DtiVzgLGbIUR9pZKcM!Egx`)BNXvK3YmA;t)nK!Rz)__b
zipqZ~|6=Je_;K!l@F694LoJc%<N!*N9ve-juD_Qay+E>i1uZU+{*GWEG7ft3-^tB&
zIC7T%IGwy2FcnBL;&0!>;32+<l0K~D!6pfE<N5b$`Bb$ubpWl_#C1+IA|>>u<u{q6
zG^@)$%FwYhBf90Rd;!LO5r-q&ixq*(7{`F)gcOq3|E~U$b7R=;5x?AsMaieAhS-(U
zd`@)ajI$b03kty&%M%$Pcun>#W-$r2&;jSqEesWChOc~&&C%!(qF9DlJvI=bVP!9<
z3@dOJpwt5_oHM~5sU}%pD)d9x)J|gA!-h6M30)8d^Ie(f5I{B*wq~3fa7nIQhRnI4
zO^|}W)nVdd0M0T96hF#;1>^#O3+F#J^r`bOa1e|iIT=aSm9dSVfOOSG@|m-8baQx#
z=;OhBS+_XXQ|cpuJZw`ig4$(W+(_he{ni-1hZ(hqm+@sVAOn$7oS}ze_EOk!TYGbr
z<Vc^+Hnl(got)kTZYnNWb;W=-lv<r&kkmt160f-@Y62srboKR$NUPNB`4&wzUMGV}
z&p9>>%e~j!gB%Ic(7U6)7U=qw*t5}u+9(-2*~{lo{o1QDY+&EVp@kX}eZR`-v6OWp
z<sDvf*xM`NAy`syZrgGvzLJIZOu@M!pBZ*(aPlC?p@{K4x%7Hlz=O6y?a+dF%OQ73
zjxQ75U6#V7`9HwdMSV2@^$Lw6FS|*HewStud=wdI$%U1^JP~<0ucgUYLVUH?6OXt4
zrSNBcj|Cd6P4mC7(!BWc;@gjd7qGVR;t49>h+sseWj<ZhxTRE0nC5u7XDhc}Zbkn1
z(-@_aLiKGEcqP^YiHNvpb-FF|40r%F!jdgpX;9xH*e%INYjuRH$`mkJ=Pj5f(&ks5
zO-B~YTI|PJlmHd9^VZED9nPH|FTuv$=yG|BV}Emv^{j7S2#>sW`J(z>7xhb3RW8_O
znc;Mxq4+H&B%Do0&)(Ea*ZA8ZwnUF?3sBz9xn*9528SeJ&yjl}uaR<2XbNpP^0TF=
zMzp2y*V4C34GNMZnhH<2#AoEaJVPV8mb7*NW9qI5mNlP~2izB2@jbj+*mgOz)mFm-
z1xJUdlm?Ix22;Akan48^-{o_=)}Q-@FCXx^Ur1r(gl~ycM}<IM4qw2q%)zq)&jOAE
zG!^W26j$F4r%S12=@S<OMIv0Sg2^%a-G<)l#<P};M|kz@tB-56h-8g^$STeMOY`zK
zK<q;c_%)Jm*2BK(JJx(RE2n9KQ0(InsGd08UIL`xX}-_bG|9PVy8eUL!Ay<f7eZUP
z@uTeC7Gj+1KJU}o@UhwUDSjn7g0_Z_8aE$JN}T6MKn*|;6Tx%{Ax#9dH&1P7L=D?R
zsxwfFgmdf&#`cs)g3+P=g0q2KH|Sk>W?0of!Y$Fh`0avPP&&A$D?7Y0XsfC?13Nmc
z{88|~GC6hOVaiRTmK2n~3M5Lt{%StCypp|@&2gUnL<P$$$5KE2q*f@M79#Q)Zy!{s
z>WJPdJdGvNnU+PHKe(oBOd1x+YbCbe#1Fr&UHDZtK)eQ@(5s_s2r2n;oUxc5cS{g_
z=V{WqAA0ufid@J0FKsZY7RJ&RFz@2gm_%gV5bCS_2&%9o>jR@Vhu9h9aDDp2PK~T7
zs2F(gMm54_Uqqe+oe;!FG+SC2)1{*fvn(yjeC(>^<_rv~M1%I7*_D1XNAU||Xl%+^
ze!JhI(zCCNGwU=z74~TAo)qSWL#pX?Y?c<1rZ)6W1$&oY3Ku5d<ZMZbMCT_p72Z-`
zMUVc7T2`~7bR6=qb7p_WyN|e`X`jTL!Qh}w>Sy7eR?EJ3vg)q;0(izdjcQ3fZFaB&
z8B?}}FL84}W1y{y*W5Fu3fbDlIQz2yC`VR0v~-r?-wH{zM#9a57};?R9;mnq+&q{h
zF=xZlC0V$swCuxXJIDv3d~tLx7O{aZ0lyu|^M6#;cyMQql6UVr*9Uaoe&pp#!d&xN
zA`=dkk^`B!ibOVm_{)0wJ-GeNogiLA`>YU?e!6Nax|i2sIOQ$wZ2owkLRcDY`z=uO
zSvXpf)^y0RmNi=WIVL7_+<(GkYC>BnmqRQKzM<oC&_gzMarZ;CPlkKYTPT(zN?Hq=
zkpHd;r(HMw*o9<kBL}NDG%Uaj2xbqrdeMOkEYKibuVk%03Km(!iOu+%xg=LS$k^HO
zo!Fx2>WwiwGHhIJH_AVv4^K<0gkZLJV8}V|Vhxn<*78y1Ta)WmH<nQN@h@rYu^`Ac
zU;_Mq$S$&7)5>t~CeDb=fWDVwA^*9piBAZ8c9^Vt;E_Z#or5&#Us~+X`m8}u<f?@s
z&bIAG#v5&3tUh<D9dGc%t5+2=v0H0A6gad<!=)?(kv#An1)eEZ#web%G2a+L7uh*5
z|BtL3AN5!W&r7GHvdG6@>EEBJ&a7edT4loI!zDTgoG9Dswb$5-fNDA2hFa0tlv~`>
zv87C_0iZXNF}38?Ut=TRvyP;%V`EBn+UjSXv~vX1$-h$VOdu4@v(fy1%{9x&Mfk#Q
zja=to?nebECv5AIBOXv~Hw-7ojT|Caw-WBMSAeAKFBw+Yi*W;KEi`kS0%Xk4ml=~3
z%n78Zm0>jgM`dAQ4%E58$~E{26l)fv-avTxFvt-wg9f}zA*uO?J%>aV*i9{a;Dq+>
zSZ3n4*61Wt%tCAEII*P@WPm+&oojeyCa+bAchP|PX+6rvHlmoya#Qk@IRC9iNQySr
zVF&PWQN?WaDA!SWzvu)-1>qc_11yjh_%<#Ihtis?8L2_Ff{!E_2m!!P>pWUEs_d7_
z^>bgoh8fSYmU2DkS&|x8&F3T6A`hg)hbR~rNb=n8YlFmK$MVFyN|$**+^n?pmuco)
zi8NK><ebvITbAe!#anQEQuSq>rbliWlRs%7?iFzMdmCI_2_JlSCyFNByB|w2?KuZ^
zr^Wa~X)HD8NJ?eR`Nykg(`L6cSAB|WX5zk<4LGxyIMf+42`mS$z7Q=^3rt7d=|FxV
zEIUfs=UAQeLE=x8(x@D}fZ7R1<76(>;Aq#L8&EW1A;nk&NBK!5ZpvD`mqJb00k-fA
z4+4!Bkdp37ITSRtlfW3D;RXsg8hePXq2|Qj9*tTjBmXGRBgc&6$(+Qc*Wxl0r0`P7
zP=$B-ql|q$|FH3zF#6OmZ#fo_oR#``a?=WDj)qwltuJOlUQFPvT^tny`Y4|i^rD4g
z(u>e~^#F+`>?+x;YkLLT7dyxoS2q3kZV$=gJS51}o0ApQ5Xn24ZVV&@yU$e2N9;|P
zCCRfs*6bF?;loe+$uRGC+K-6iJF*Sm%p-lX^P5PR#3Zxh@oLF$MTo)6#KLHA&}b+$
zAh82J9`*bqNr!Co3}<$qafIXJ_>0b-9%8VC7pKvmj{FL=PEu_^r2Y_49~DP;56n7z
z$!;>91PK$d3^vwTOYLyF>^s0TU&Z$)2+ML)O8dIdpCe`rvKL0PsBR2>CZP?_`1j%%
zfP+n9&Vm*029o2;#D<|6V2Ri&;lc^eM<d=B7e}9r_BI*Z6$>Vh><(pHXqe><Mq;mB
z{4w3ie<cF&odcloyHM?MOhh!nj6jZ_M9Li4XjbmdCxeZC*oVuQs>7ZoJlsc0CU&F1
z(mxguwE8kTHwRQj)>GzTG-ksS$D3QeV3LbIs1R@s%K4jH`Ge48z-ARjjyeSW)L$`L
z2Lp{o?9iL|)f(5qkgJo{P;4`L2Ol?jfhC#Zg+Iaxm}Xcb<ANsh;|-En)fr0jQ)n}f
zXNkC_I?o9)C+F&M3*TQ;SN;EI_XD^>TNJ)iw12wg5_{p?9xhJkZB3CbYX@HV%hWGB
zY<*d52Vqru-X_X4h2MJzA1`tfLjJQ&-$}2Lc%U0$gr4YAkG7d*&Gr}Ybe#0QL%IP(
zgOPCE+j3+9X2ZmRG0kmTy)-nTrMHL?lKMwEfwFP(=5)iTpOJ`gd;$q9g8n7*G1p#G
zYG!0Fk*sA~H5CeDjvJIG$vpYfoVDx%IVx6tR$eVod%|XIDQ4dPM{5{<_+!6DdVZGr
zfZkFH#>NVl4*45f@LX3-Td-4T3t$hmZvBAceWO%xkiB^VnWJ{Os8}q2H$>F{k~qdR
zU!&!98{Pi_tdbPKZ;s}L>IwromDNKT6s+uFHuT0VUA^#sMv-g0TeYKO@klM66DN%%
zErFUTIz0EVFqSM3sSVDpCq0G3{7m+vdDd>F;CSS9dFF4Fh-?36OvAp${u!3f!la2#
z{y4q2@w!W9e_HcDl-lB$qGA<ncIrU9qntq1p$V7ug*WBeq*e2z)EB|B$E#Z>f{y>r
zP+Q&7pMubW^rm663TZnvv=NSGpu}2XTI-gqs#oV^3O<TSo{qxYi6Aj7mzb#D+7mQK
z`pUX5-EecBBRa~jv?K0l!o99mxRSWqvRK4Zmr+%}G-<AIoS3gI;#&*>0BygMHj;vw
zN=1w?ZL8Nfr;$EGH@$5zt<kcbA{!y1!y#$fa&w->{tJQ|@B0@Wfc&J3{T^lgH+eV$
zNB)Zhu287>-szy#v{96uk_oA2#A>5wc7i|l&OQVFiR*+y@+iEqR+9afSBqgOf$FIr
zZZ4Mjnkm*op1n;f(o01RP;sbNVAD8K|1t2{w%-VKSy3e<A54|_l09?umy1TPG%!u7
zNN5oc6h_M4bU`(9x&CCe&^F?ZxAenxr8nV(58bPi^os56_nX<Ez98gJ^<R!pY~IVd
z-n$i(YuXMF<4Y%cP!-Nv>H?0~aVs^px-3o&!|2T7PGDx{AwB@VnRCwqgQui)jk?TX
zcwaDJY^vGFPArjS>*rU_F<`R;g^IzMp^H^LVa~G5LN8?v&2N=Sgw>$EiT|i*ClL1H
zO^bJAea}6=$xfYd6-841!t9#a4IsYRHh}Y=U&_YHp+J9nYfr5#u<&#x>n=jaK%qf!
zr!*J->iW0ZVm;QUZ4q8yFMal64`P-Lyo6AZqPfVDITH{mhPzvPGHtkp9L~LH!RTlX
zt#rxO6OEPTgx&<rVyST^MnSl|N|6v+SR_!gFIAJaw<|IeE?=x3mS5mKX$XP;SKqU7
z*V^$ysi{lgwv|ZI=&{EOa<CK0na=qJHD6#Qq=Q@w00<0{iKY$y+Wi$R2lor~4pi=D
z`^5SE+p+Ef2M0?tNP%TfN#fIvdZgaanHlPu;<niQ#ioB{<FkoUO2$P8iRJBL9OMI~
zaFAHG^jgQ);OB`Jqx|Hj$KejiQL<&3Y%wnt>gUwYrCi7H%n1Q}%K_JzR7xB{X}1F!
zhYJnnNswul_&L68sUWed(_Gx>)xY@n*(^H<{(1_o0Ig{RWwdr`v5`vBZ+)HJcmQVZ
zfBvO@#}Q{gnS*?J*w2RoSair?$#!EiOp7wB*sKysCP&60mNcs8%sS+H-!k04@kv7o
zQ)X3fB|bH6)}h;P{CTIZ6_F1*uFTP!X3V^Mp}UYETpH|F)yUKBLLhZ9QSlX_mREd=
z7?0f^IU*Hu_Mx~uz%ltVd-VJeJ*=1-3}^b=GWC73Xd$WmF!Jrz!?jGRbHx<*9=l4;
zA%1HsL7iqLm^+6^G*OnSXKWsQj9{B{#wXlpRqN*rSX{U!v23#ZjD~#Bh@gVMClxMb
z63Sdjh2tD*UB)rDBtQX+^k;>k15u|z_PIps!0DP!Va*JNdLY^g0mVnVfqOzND}S6<
z<A69NJJ+A1LWN|vFTd7sAnUcPMZG3Uy@O_Zdw&+{Np_t$0}HK7Z7I}Od1u^EmS^eg
zagzhKDl2HwGwhblOyBA;>4446r9dsGHBp*5mEwU>ppeVilfcn@ao?Ne1k;A62G2NO
z5y1-B$8C$qKQ)uNLyX;~jsqp9_d>@Lmx%2+-zT530}~^qdIw=drKwf_pF%mr{03(C
zLXMH^jC0rAlns@T%UMzp=z+tZETxxGD*so6mO6in@B`lN`#7q8Yz-1|U-N)z4sT_J
zDHx@}>Z{jo)s^SI6b)$YgmsN1SfVshNKq6@Z$LgzsG$e1J}Yz%Qn${Da=g=IsQ?4c
z2|;q`1b@P_xB@3eokbjH?n37ESfUJYr0obK3mS8m3avixrNW<z#y?H2EdDOK7Cs)y
zO6?B`y2_LGf$f;+v0S>&xP927G}Kk$@wbm-iHb^~RGN0sq!PTCK~!<+SH;TToXB>+
zMU+;GBvCTaUEmnJiq!NgN>u->N9np)Xtj}0o<fXi+qB!D@!R^U;;U7Cy0ehuqIadY
zfi47gw9ux09P|2jiEG(aJ#)gb{M<`vutd=r&YP$$UQOjrb{rs0r}*KTpUwdcJ6x{z
zT<Ddic8Vue!EXb=LG2?tO_A7Kqfr)@+>2osT{{$<hNX>`7vkYdZ`%$Lt+q|JN~WlL
zJFJ}hdlXpXV5hF~w95H<`|8DpfO5q1_g~*hG+QMa4}J^0n<;%v!OLkj#dEiBT}z?L
z7I7vVb&}$EH}Q3^=#(aP(KvdCO86a-P_F2sCVnj(hmJedQ=m*EQ_R8e_07bA$U5`M
z*z4;meJIDtW=|p>)o4DSotB`DTH2NSL<!qCw-@DDzI5chZ+6gKEQkZ4L!S*Vm1yJG
zzYno4!<>^%ilpnQm0Ky%d3dEcNuy|xc9tDqLOQl!{^g|^`IMCTUi9<_lJo@a<$g!a
zA^l)u3VakL^E0SbEp$6ryYu<!esWx9VY+v)INg*T3FI|?Co>_rLc#$B3{Pia^zQ{e
z;|b5b{5$t1A$$HNCvG>#z3T%8h?Y8Q<!Gs0WtgYC<C!o$AbQ<22hP(@4YA)HhP8;q
zam@Z1!P7QOb(kC&$-Z+CviWM;EQUuZ<ThzPHZrtPuA3oLqU=f_l&Nb(cfn8kpGySj
zb0B=Ug-@F!4zXclLnri-XC~`%@mK5*i_Vt@bz)x6Uv*YANv=Wn1IK6Y#eZ>jzV|*X
zF8pbNdC<kr^|p*}Rc{@1LNiyHDws4_Gr;C!_2X6a8#}6A+VW5vYj$F34kCMo_z>%Q
zX4{`4@R>-2C9i*Cx$Y@346b(|iNq-;kvt4ZC9=Fq`22+g2_6PczEPneL4tp7r)Lky
ztBq`Wg($`5WvKJ-?*`?LBbj`YNWD7wX|VB|3~ux7xLh7dm47Y-B)H9^^w4d@>5Igt
z^Bt3<u40fM(Jq>J>-!4<1S|-QOA=rHBO^VPcnT^;GjN`8%H+<LFY`4`OU3D1J6U&j
z6ul_lzZ9AsAh^w)7r1rRX@s^{c*lLF8C%Gz&-_JWR*+%#yB>91J!|Gpky7|wws)r(
z9O1^%DXwL1xXJ1Z!Jv9iB~`Yk%?+^-jxbEuQa3a-%+VtN7W{GvK-EBzV(Yh!&9{*S
z_0XRPZ7N<6XnUWJil`t_I(tE63?!!8rZCZk;FM>R@VSBZbg&rX8{z$Cs&=sj{FX_4
zH3Y3#*!4d}qz&6N*B)xESEiK!L6+`!6GWC{v_#h;jBQW+NGFj*Go_g8fBGuMYX|>A
zn#0woYc^iP&U3XQGxvr(o7gAfk|?h^w0A!V%?C$v@+?}=|9vonCjY)gt7^nFCX#Nv
z`u#^^U}q$YMw8<H^M*HR*yxYY79dRlD&~(9utL_~1XWYVw6KY9)sQ^ftdemKqRVVM
zMi|VN4pT<HHnVs~c8)j^c^r2rc9^TNnhZuJK6jC0HFwaaC)h4oBTF-rmmZ27<Qs0h
zQq;~BwaN`c%sGm_Qc}+k=T9`soR-KDk>rR3<m03iW53e)E%IHBk;mmzN=ZDF8U02o
z^hMW^UVPC+1y4Ju#9@OTr|mD@$Nkx=N+ik1BfcKPxPF5mRWhxi3x42(o(-{lmU(^<
zxWbrsZMyqgu7cFC$KPCEi^9)so9Cja+{jOl&pPNIbsC=T_EnIWq?w@2K>TibbyqBs
z<&AbkqFM<zpt^s$g7arYF;bYIEOAg1{wz(mXE(($(A_Y&Anvn{w|}}}<NoQM7P?!M
zmDqEyS6x8?!N3opSu<}T>cXtnD}#Cnm;}hP=<QoT<}^;zXVTOP2h@K6%aMM%n{xEs
zvutV4iX}32glr2wH(+UXws+dnrfm20?#+Q{WAF3^BhxW{q)s9m!-AY$?;yXy%Xu<s
zq-h5%b>Gu-!gGTq`79=}lDJ=M)qLu=7|)>GE9s<!RQbOx@v}z`uq5d95sf*r)5QO{
z^|*K>mCjZ-tj1^MH;$|YQVY2)iXJ_A9*k9()E6TUu!<Rwzf7Z+Pt3HsS!`xe_%xK}
zNb;-p>y(!OvYX6YDsitUjTF>-1nD><ti}(()f>d782rShSDfd!EJkYl5B%iZ26O!D
zioiJnHUt(Rc_^7H$DpU%omy6YU8My)kwt>j!Ed!Y!`v;%Q!_iQUx}mM%^tSi95Tp5
zGl}F=Y5r0EwODsZvxg0z<87bRtS9%*eeDbG=o~8?ojwSNu}I(m*!At2SrSOrTV!dx
zAEkjB{l}}ljzj`wmP-0J@fhZ!;s585P@^sAAoV+C*?zINjpGMT&wT>9l%$ffSL+mQ
zRrIJc>kjU4)t?rgyb)vt3z&cmwIU1nT{(Y<@5gqos@{ln0EBI~whA;2L?S<!o5o^E
z@n{LPe-vy_)e_tKWK1A>#q2L}QFYLM-1TNSoy(@Sc7%Uek)yIfchUTd(O37Cu1VyK
zM)l9;4c*D}D@LC4#Y~}lUv_-+*m(b$3wDvCJ!$YN7ujK>c4x>_+Bc9%N^2Y8sj^qh
zz$x*xgJf<Jls>Ghk57f5pfN-y(0)a#2$)IP@gofVrLN{|YNL0M9RrfIkZK(N+=;#q
z#5SJZ$QA?x^4ff5Ez`s$BCppr_$UUET|r1)+xANBIY?vTIO|k7?>NKNMluMVut7a|
z-)gR=*+woeV%OjL<xJ<V&Png|KSr05k9cG4(*Jt(TOqG!`<64NurBp+cKc$hyXn)m
z8dJ>+CLJPa936k3_eS{)IZcO_08+)I1h20Rx!Vmz<*9bYw{i&#F)Qk|Np`woJ%P1P
zAXbdE4WSwdY8-q=IEg+(Y)fvoxi(b);H+*9-?~>3ESJix8T((OGa1r6VC^;iN6)4?
zvG_GloVj84{eMCw#=0)Mns=+=pmn#;$R3c6Zp|51G?Irm`93oyELRo(JRFn%8exFm
zOC!m3T>jbrnp~;&{!CJpFrDw#-JZDePS1c}BipXnE+%aAHMlK_L(0h1af-*>*p*V2
z97}7D+V+*9X<%La%$d&{Hw;RNp6mp9cLO>0i*81{Y#-}tF0t$BTySGSnv-Yhz>ILN
zzIedC4@WFx`>ztl8~XyJWf)wwU^O!AK@J?)my5oOn4g=<KtRjnz+&z#mP5BSRf1e;
zvW84LSLA616S{>^J=FTr|6WErD-;p4mqYc?g<yrBKXeN<e1DmjHdo5Z-f&X8HV@DW
z5m;05FgGb5)!^0y4Na=h(F98W0})#Yo9Tm*W|9p)0U)qHc}Yt6$akOB5&7-K89UBN
zZHZD|zu6RZD-FCp`sYj9aARoi0;~9zF=47nxQk@ulLF$_<v~cVrZjmp4nx1q;h%CN
z&e#L)K&04y9hRBDJ_Hk&K|1{B_BT3iCmFvO>w(2|7A}Fb8wRqi6%Ipgyq#LeCQ9*5
zOTG&&Ic~_mO589;cH^|+`>Mhc!-OQ#nqKRLd~h!bpdK&@6|!0&nV<tR%fZl#)*h=@
zNnH6^=V*RDaZ6(Q_4rsylJ(3ZtCgs9;rDmCeUp}RD3;ChTZJ*p6f-aSQ+yJUfmS{c
zGEg>a`ZZi$ddW@noB}HVq=)!~tQw!E$#$OH_?MNuJsVS7F+9$o62wi@?Bt21WPcGk
zp0cXv9nd66g&}v+kw#X<JQElgv0%@FY^sxsGcNm{Dy~%s5R;$IQQg5$Tku)7md~%I
ze!~{b^mu=rG}zsNTv$$JA?+OpM{jUh<6#^co+rpXnsQk_^c^R3<O^Cn4aese)#J9h
z1=ZU3NEwF`6ha;6g<N_Dm$d!`^KCOyi#4W)*XL-I;@1v^1(JZ7|MUt^%M-}yj?5eh
zW}g<?w3kw29lmhsPa7nNh`WzGe)2yU=5BXc4BFxo8yh-=La7UEbD?2o&0Fbj2g1LQ
zSkZHz+fzT^&2|?frwA!-ft}QiOx*AQeC=>T5BGOsxyKi5IW{;3rPI8}3X_5-l$zaI
zu5ZCtM2cPc$dR2-FQvp$ntn;{)WvqEhk7nT{e(UZL|1ZvD8RGw#<#cAs^>To&)(T{
zg7r%yrkfqaKgM2-gB_gAeUYcDtu~GB99p-DmZ2)4QxpU<th4Bwg@<Yu+3d)?S8-NL
zq(kk=Y&t<PT$@VBon$m$_%!6$UFzcM&YVaHdeKgF&+H)u%c5s$2QOZsF5=$`x_{8Y
zJpR#vx5;$EyE(F(&PPL47vEgyuD~V@3s)~HoU2hER)+Mf<*2Vi+oKPbb^TdY{{`}l
zU+zA7cDlQ^7Pw0zcac8-vpxmFra;O(AC|0vIM)V}S?OpA34<_4Up5Nm@#KXOhuz*O
zXPf+s-raf6zpL-A{T`B+^k{R|G!Taagl*jyQ@j1|a~N1k95O$qdj^bb4+{3eu$&*#
zBRzMS_^V$FRDT*ws#w5<tGbmBz+xoD@RqaAitArKaw=QWZ%Y~Xe=Jt@1f!=6L(2ww
ze}bX;xqj?d`)pAi`MGK>QCSJaTlif*&0k-iN84|m@o=nX3Ovtk3cCO1&E`)<xuXaZ
zyrOje{W5#RceK}evSm$D1exWZ*6_`j3pB{6K%wK!6>#3xlUekQxm1_aFn~CW#yS1`
zoi*9;VG+|z@swmPvvHJCNh(LgdWtt2635M6Uu9hWwplK(^MPHQm&3UrqVh&y9$1n4
zLi%xgy2Gw_tC8_Mai^8~Jv%g}S{{c%p6}z{@DNn!!LhI;XPJ0#`jrJ2$m)XN@ZT0F
zAh6dL5K)p7#^fZY$|@N~uI484EazQPZ3KSJi(oqcuBvveoR!bOid9roJ*{CyVxWt>
z!{&*kW5-H8EcNJ&aGD!;W;1@joDNq5_G`4wZ+#&EuK%<H4Ww3*>rw{<yJVXBzG&)I
zp)X+(8sho3l_gN7>GY@N!<(%^W_rgiDw%v7C%YFfL_=OH%IQn}U2c2*oAneKw|<UH
z+ISwr%*k+D5G0j|%{dO<FvmzsbWLN5B6z1H@uv~+?5gzb3z<huxnLy}`?u5D#$|o4
zyk_wivM*updRKmUqI|E3*3KmiIvK>3l&k;j#zlFqjc1pkf7YwE{Ef7H!Hlfe_CGgK
zt_U2f-Y&bh!|?h9me%p+X>!3KKF$csVGkar@bvAtcJzT{DUPdd(;orn?Z7x<`hs2-
zv!-puNiU!o9!M548U;E1=qBgUA|2Gea1k*#`o&Vs4RlI_G(em-O*DR@4SrA89QoJ_
zvs+VW`8cj|N-=OruMsdz&s}C3!?DSBP(W+MUvW;Kd){O0uy#jkbe0=E+M3O*ZTWur
zM&VCpu1NgEUB*3~FBIP4uFs2-aY+hzQ^I@JAENmqAab%oMj6-itNFN3=4eu*(-f;|
zrQY;z;NJri3fig+)32TlkOlk6X35OJZz4*YF4MJ2zZ0*%@?EyOP8Cc$C>1>MN!^`|
zXT1USmd)M{$!n5+Rmoi3ey_AJ-AvI{^)`=08X_vARY1Jdq<-u;%Vn6rter&*diCJN
zULBs8Mc(<?f3jON+a33`Nw-Gzep0A--6d*&GH!U^o3bj+1d>c%&e%RuT3028laN&~
z2J?_R<N13Ey>6t^<Vjh6wUi6dZ{sZ+&88WoQu(PZfM(OPVXM|Aqd<FcpkYT>Z>bu7
zeYl7}dw<|~5W(gND>X=0uwgghVn^u|Gh8IWF0hz8B9De;W*E18>dnk|Hu2MDKi#~7
zZL-*e=V^%Qg}eEsFXwC6-)Jm<ww=@Pv^OQY`sSEp^=l!&tE+D7q9tW5^?ZJtgQUtu
z(R+@qd)lKi>)rchyXr<ut7YC|2Njqz=MwJz<10<#^V2qaSMj5vGI2w@poF)?f1gGJ
zxhV2FpYPsumqQYzsU(@*1mt_oVkbl2RuR$E^s7zUv&axUCf@A*X;irA{=64(!74hH
zCHSISJDU{m_V#)=bhayT`3|mO_Tt;<=?UJ#tA$A0w^o#-pAJ7QY{;uO8jQsqW+7x;
z23;&%_NnjQ6jUG+mD<G+gwlUag&WOQ7bgoVN}mVglSo}>#s7AWtiS1YHB5r~SqT1j
z8!J*w>q^2_U+f<Ig*!7!yEXvvHL$((>xpu$>2uJ)f#b-htVuQ23!XuNbax{2A=ezm
z;o{!L`FOM&$%BOyH^IdWiw9|@zpWniq9BV0aVAruj51W#IV1nP4&00$#xDW%T8eAI
z66umR1wJhzMlLIo**!{sHF+;~h9>QzU8p;6|9su;HR_h*8;xyicA>^;5)GTuc4ZGe
zwVh!x;Mnzc?u+eT!^zC;s>MiGFn{;E%gzgR5z8+;eG$^c?AN`OcsgWGB$n_Eg*LrK
zl7JfE%=Qy)+|={VWUJ>S9rb>0v_!0O=$w*8ceG%Wk7*}iOp=9*lS=E?B`Mc$IGai=
z%XMXKfS%KG?x~&YTP4jL!9;;{Ndj%<^7PTqO|GT`41KPeSPl!`&*c`%3d->bN?y@Q
z3yM^iN0fPf9%Z`?|EImT?2035*L4XHB)Gdb?oK0(TWB1D26uONcMWbq(u4$ecXxMp
zceiZbcb$FKk2t5k%uzK)`K(8-`P4m$^rGNUdz8TjWSQ`a85r_CSiLjsiQJGf%dd<J
z<e(~M*-ht_65&NVBiqbgA)+rNeF<n2!8DWOe6X-MwY>#n1))b6U5ZNfr3y%7tRkVO
zA(5&Dw8HQ7UiUYNQ%Mm?vCXgeB<(wdQ<;ozkv#fMs<*Ei!p_{>anE;FWX#~aV@-*n
zH7XjZHM5D`ARBB~t98KfTC?RXCbZcKwT|=}x`=*!9wEv<W9`$=ghMzr8&S~`(e=CP
z%fiLdXz!2Gwc-anQ)xCDlc%$F^0#j$EVI2L^{#lPoz!^87Ayugz^F4U52H1E$$pkv
z8_c)<>zs|4_~trGVs2Wr6&G8Qve`0muI+}%DZMqkxc+w_Ga<Vpp<eNmb+DK7-TXRC
zT@cH}P7Tw_JL~|{Y;9kI3<E|>LtLoNX54J^?)~haYz3?DZ^_q{nk=7d{_IoWtiLk+
z;V!PL%8;9Ies*d)F#SkRXM3MGTgczk0zql>5z)JRWH;<O_P*UCD!6_MT$d+_Q@&)V
z@eRyz_@4h-rl#ZvY;nqZd!EgyXeCq+^B<#lAiTDdglj93NSu1EgN^i4J00LSeqP%_
zPHz~Ai5QEMU)qu*<<&~Bo9}n_ss$LyW}8qyUyv`RWp3lnR3F^Nc6g4$46zE71#MX6
zR`7l}zJLA0MnBgaRm{+beg1`&t3T2C(|F|j<G=fM7{T&m?FX`0vsv$pB<hiGj;xD|
z0aabIP>5klH{>Eo-iQx06ufv&ssRZ^2%fA@*VintViupEA)mz%3~Y2e@$o8|ml#rQ
zhQ3|6N=?h$yt_&T57gV({EgkMI~B?VnqK(6w@obl)3QFRYw^D7?ibf&XGE+irlw3K
z%%`pr)pUJ#VH%ec_3?n>myyCfU5xklzjH3q0&8;966otEhKK7fi^)$BoAifom5u#+
zd>>g!AR~w%^JgYI7?YJ|LmXZYmQR<P+sBD6ro1SwFqVuF_H%wP^*Br^)4cKgm9K|A
z?|ZO-du?W5@T0@WCVi9=rp#mauQR&<C#qwqs=6F6#C{{3b|Hm4s6*p>dVFJqK{hAV
z=Of2_x)Q6d5g!3Q2`6eZ{uIRU0Itm6Tk*Kc1JP24n^OW(j}1RA+9f4soM~r_!TzS0
zSF0Dx9M^d-QlJs#k*d5dEG#||twSm`xth%1HhaEa;LzX_R`X=ZX~bnIT1)mPpQEvf
zpJ9eEp4eYEil@$1jidU|Lz`Z1&|CWblF`RGl70j+JUd1E4vE|N#u<b2oqUKQWsR}w
zxMzpCu0r|Uzp{Egb+68?iSvG8c(&|sWMYQaU>@lzrDu<bM-X!^COP`|Nc?X%+E-eG
z)h;tDB#ULRr*>Cn(ThO;PZ4TGDPyl4X0enu8fhZ{?_l(+v^E%l--~bl3>c!pgS|78
zi7Js~<B1bNnjX8mAS>@97+761Q;9}Ny&7blN=wemh+#{@Q@XMda#CGr`5whP2-dO2
z8_&uKQy;ljQkw~r=Dm<Mi6ApGquuiV&H+Ccd$!mer{eSb^V8OQ<6Drf&rX4sW}6{v
zJszo;!%s51-HXv81cx?7*lArJYTy?NQ9u1kOV2x*7)0&}H1W9ntkZu}E2)$~9_)z@
z{eZU=l!!RQfeAYhktz4t;<u>~G&j;-d4Rs@T-jpdUrcW1@2#>Hy{gkYOQTpv0^U0b
zkaFPadbb)d3U5c&s@xR#Z}^sLCOwf9v^zwReNZ_Xd7WMN_A%>XTdth6?!+U%zgbM4
z1~69W2f%_!KPe|WDR1y+Ovk1Qj=y0{E_d%m<Aehyck-xA%ZIE@?mkn+8gk>KH7Tpf
z*`RVdk(w0DC#-hQZ-gxJ#TRu0&f1+5iWqq(RFbuzk(;Chsgs~uC1iPh#);u>IkNLE
zIM6@u!#HJWyzmR=Q4ys8oC+szT3?L{9d;OZ-6klKvK)`<N<_3ZH<o!kwJlo3(qDC*
zcm%g(nCt38T4w*8N5q)26T`+nr#!yNWo_*;b?2=&rZaSY!hW;mB?g$}E|*YQ^LhG_
zJU}v-(&!laCJp>oo%DEEM&;NL21jG?0SWXS?d|B?b!VCvhkerCpw598U}D(45`$-g
z_}q3B!H|cGZOPRSw$p;WJCEDEmdQTpx{yZ!VQyG`Eq)Y9oY=?xnk>W&M9#yJ<V^Iy
z)!7~r9scIx#@yR$`Y0vZYs_xPrGk>v%~1SGSE>|=ky&eTtTC-L=A5|g;rOdpS{#ok
z-u}R4o>u8GWyYM^UsrMCD>cM#=W_^`b?51#a&W)E5LYnAjWw4<bIjkHrup?}9)ID!
z+ff6<^S5AO=?k<W#k0db5Cc$nT(>ZtW#7xtsV6P&D?RPF+G{sv2G8n;fUZ0(Xm-&X
zD7=&2KK>}VtyaMyg@CZo<nfNpLPdGO=cGfD`uozjhX&yaXm7@_Z$3b<{L{kf+4?A2
zRbwNF&?<=U2Ss3e-oHh)m<Q{lciMIcap`D`>fOmo8U--Z5HE=TP#93t!&G@4LM!}t
zOQm~G2DxZN%7dImKU>~f4DXM%4h82(HdhXJohxU#IwebnUUacLu^8nVh^7Vn%;-{m
z)hBJnz|H13JL$sd*F8S1EvTv;%pJ-df%=mqWxs$COZ@7lg1T^KAd3E~AlC2{`S~<r
zr1dws_h3lgigw^vw@xm%7(2J=-v*2-zneTA*GMBdLQA^DGQqWlQMj*~iBnQZk8d8W
zP~-x@AXR+%!6-E8<<Hqequvt7BqpY%6mS<9D2cN@uAq~iYTxv@!QI7emuPn?@?L2l
z=3UaFxwQm<->|Av?W=bYtt>BQQ{oX1M3_@(1hCz-QD=%R{PZ*JnkFV9H3}n{Y^VDd
z+Vrd|S{=)S&1z$nVQKl!heR;%fB5;7?g&$==of1RwQ^#)GbQY_T91gC*9+6zL!OD9
zJ#8A~ORU-g=*tWJjW7nQOX?p{I=f5%QgCV<sJI_<DF<Fn$Lz4NH3&c=qAvb23bB1?
z6?y@F$!PD*?dcGD`1Os2$cr2tBx@J^XW!pbvQb=+PMuQR$H}Ox^HfZxaYf15JGrWA
z2tihT+E!Z*cZAG`=$r9ZWW+g1pQ%&^lu0HGaatSCL`X_BT6oXh?Z=eNFQB8y_>4HN
z*t0c{*Ru)b=8F0B=5?8C6)E#_g!PwC07>_8h0eJVbMge{j+}R(7(f3J5+Gzi2rT1%
z_%l(Rc-*80K;|ZuA*QoD8ENSULfCHSaU1=X`x1%_(mNI84I&u<Q>uz&>G19dz`nF>
zUKHf04?oY1=xnYmAR&zztAx8LnA;Rv)rf+%0a691Y+Ra5#aZW7yDtn_c%KlrX?mrT
zvK~kG+^241;I0x`nA()}xa|!wJ935Dg(d*o>le@5aCuk`m)vb1m}u;_?1_YmZDZi~
z(1tNom;Mz$V1N2Il15g;<ZP;#aNZaCQ`koW+adEAMUO>XDIdQRxwy172C7&neFeF!
zmaF(L%|@QjkCm?Dg^|aa?D^$maf*MJV;$({uB%%hn7kE(eVEU?r5o*J@IbfK1_rdW
zX;_+>wP-YM=hEXXZ+4OrU>K|e_-Si~Z0A}FXK+NJw}~r<d0bvB{hwK@=KL4`zpV2<
zDlr4=tYVn@vwP(Yj+C+JdAQ>i3cGx6G=oJnAg2ep&P&j;6k4@-+26h4@oW(3Xg*D%
z>WHLjx?D%5Tq;-^Fy~iOQ5Y?r&HC;t35PoqsVNhNu<_&3w{vp#)*;y@rFCV|@c1yq
zXYLeR@MhGmN8NQO2%ib^M`ax2$oK$9@%1yol*et;_x_i+U(wqMgd|4j96-pSLCuzu
z@6jrk&%qUQ_TcD&MG=AA#8yFAZI=@?Wd`sDt6ow|Un15uq>npiZLxq13;k2Z(jE=y
z{FrqYj{*e0N+fky;X-_r;MB3v2iPxb$<Q;!bi8nXo1WOAFj$=h2vG}eDub~6I;mMC
zobP)#5ettmdhUi_wG`m6$arY6OAa{ZFI3)z9zvv#$CX7pxy|@c$R?OFiv{1zBxWev
zGtSw?7|7$@3G4~^s?7YkQ@SvtLpi-FvWQ|2Cq+&xPS|{WGG9v@AHt|T(piUuy_45|
z{(^VcOB_jHz`PlYv}hj;f7ZS{Jb8J9wBu=_N5={4#-G+&oNhRvk8xT50!S6c8gEGZ
zgA6<f!dRlaiM7SEeU2bjLw8*GIBKo5GaunX5$EU1!jQ>*0`Sd`P}P4%e0z$yU#b=R
zgZwZF6Y0j&y>8<KYZK8WX?n@|0Z`v>Q}KHKq%Z;b5=g}P|2p!#AiYQv6l0XHDThVl
zDfG>BXrg)xXh@(U^n%AnosuEnyyom#_#&c9w0-tFh!0N>!xUxcjJks-nRDM>H5ROT
z7;i6{HcLx9;_8>)>)kL(k-E1FD#Mukjk_p%N-WTPBgaj3!_EK2$^jN!jK<7yENE2C
z)zxTdps&65mVAJ(ZxpIRGcr-PZdWKxDMw-u?~K)=aD8}zIv-V_7A((oryArW9hI$Z
z1Ym0^sP*X1C1E5iZ@5h~>nM4QK71<&fzLSH&F7d$ka;Do;(2643<oxAJWerD6B54j
z_NMK`WoYNye@~CD7zOWM`4|?*`G2)Setz73J>NG{lRf3t#&5PzPtU`N-q267^-;`D
z5|NJ!weEJbxvAM)!8_=xuEra$SGbktxJ86n|HfO}HR`&h_xKZCBPT0*W3?8a`(U1s
zgoll1IGy_SK9$4ee$n7+Rw0(#!B}m*h#EAV7ql3_ZFBjSa(7r@#h%bJ`V(^?K%+qC
z9Fk$RQ|~JiiQLUGzWnE)wA?l(>FKx`+R79^;Oq9Y(qGkYqFCu3t!VdZhNtQ-AiqSO
zC8hqS5109+`LOriQJH!DsTSE?m!T#pkS24E)Z1C$T(1s&gm16y*IJnW>F-YG3%@XW
zM8p0!^6}aD*7jfsvq0*^S5XTCj$-EfotkSY;@@`cv$=`0YHwF(6AlBCPntaLUx-~C
z)@xg)Ct;gTOt62GVc`U7e$y>YQ>XIy<U}-RykNWD9JKMT^Oi7q_!2{NjA=T<&YZ!V
z!IV%QBX$BAV%}ba9ytKugy}eKWF3-n&qIYS$g%(B9U3FBb}Q?UOysV2{by}*(DBHi
z4I}%T-Lzd6$(|sHkp(uRiUN)TLv_BZpGEL;U7^&p6c{;6wmcI}``!+t!7?tPq87)V
z^7dKEO)mi_7V?edc*L<s3@@&aQw1drQS19=O5JA<V`&^;UFJrAdG>)4kgnT2cD6f@
zG5_>fc}(0mw<|N?n^0I@d`oWfQ-uNOoJ0F>%?KT)EQ%tOmb=iKi3ws<A!~XnU1p`l
z^EUX813xE{=pXaSVmE%q0VQ#@o}2EWA}X1HKNic^Cg&UlGP#Dc2*1L#V<XZr!8g-2
zHI5UmNL$W5Sc3-G7LgqZqjV#V`7Y%U2Gt;(dS_$e8+%Aad<0XVp&>A%wq<zsVSODm
zd($YDdQr{^6oiC1aa*NoOn9J({Sl1JPMl&Bc5%Hog!Z^D7VZ;wXA>Igr6)wM2^sD5
z42!%UH;i^6Ix4u<fZMlDe`lT10W(mAiwM>{6C_()^cF)BEo%XJ1RX<MIhGc*7zVfx
zZKE#KShge#L7lTY?;gR5B7--^b^I+%LO%x6=Ds2l^Yx1Jr%ycNH){}YYU?^BN6~}L
zw1!lq#eM*W(kJ_`@*7tIWzO-iG7=bQedU~~-V7mHa{b913gfcdg!*N-@O4+&GI5N`
zwZsv^imdwca`=5(2nm|>CXs*)m4tEfro=A4Ftd^j*U|$`%9kRn`1-@+)54g_-!%YA
z(*Y5(@9v`mZ1(hewO-SkI)t`|cBf{1-?0jsl)s5>C~jl)$1Q$U-_dic<nrqE{({Cn
zzwqavJtwF9VA4e^C`h2WDZAW-4Jh^62Z*{Ez?8~P8dhN4BQ*#|+H=S7$E7>*Y#Z<r
z!Mq65v_m3TWgXCLIxGAbxj%h%92K!69nv=yqa>dYkfofCiO$IUFgGX~0O28lkRv$A
zXK;u4vq19jAf?jC>HDAyS~B|R2HTKaTTvpmXZA04ZZ?wmTUe)!VDPqcQBgQ~OxmE)
z;svLX6V1@j6d_rODi-3{0kM8H+yM5~=kF^vWG-C}bJ})B1kpxt15i!*ebeOU1oks?
zT@GgZ^{|=9ZnbyhVWwA%sO%eM0JVPO(B(XrTvpD=4{`S!EXw*EV`aRzV2SQV)}(>K
zbQeg(DmeLD46X1G3~lj<&E-xM6KdwRd*j@iVyQ!%^eI*UGF^trR9Kl?gsvFhNR*sA
z0f`*HB7HII`%NLG8_m_Gqiwu7ydwT9i|25h1F4PAf#MTmj#Kj!TN6iaF6murTyDSD
z*Iem~9h#KL&;T~QLV_m^ZLRfvVS}VVTfXgcr>)W5jr4uaw>V4YkEcXM?r~w_6?e1(
z)?jbz-nE1GBf&s4Vp#&~#3yIn<`ZR5W=PcVPVaHbr6fy=^#3@b#pm3Wbm!OaUo(!f
zaE4rYC?>*jV+!w9=$MR^9o337HViE$4m+F1B->tBgF6mcCGJPuFxokZ&IyME_4p$q
zPHzPJ7x{*S1vfLYH5gEyO)0~XC1X~LhxOgzNf=n^#iiI1{W#v_Z*QJL47;fW>2BGd
zZiwCA%Q?U2B&AmGL=pRgK?6eC?jAts?Fx!DDZQ}*_!BP+hn39<)%P9JQ$WWB=N|i;
zJ>(9|P#7Grni}-t4Hg53_+Yc+u*DWNMotuqn=k<QgJmPm-P2FO3_B44E>T#q1Y#m_
ze)D;U9usl|4E7Vw9X6xMHxsvxy{6_@66zgjM4zgP>y=XI6~%ERH~A0{t`g1j67p)&
zdrdfq2h5J_uEBDq&lt(b-DsR;=F{ae=u*ltRS{1eooZb!c&ga{ahhoaU^0}%l9Uy{
z{#(!tgyQ~$Mp6XhQbZ&&9v+|SPJK2zpUc054w_aZAU9OM^)_eh;v>7W$c;m&O6=08
zh2AjuYVH+b$(ig;*6}BX1Nv_CA3e=gS+i{=Q<r}urDp`LqRNqS#!&FS7>%QqUxFoI
z*qDbQs&iLTMEXG4LWdx5+(mtxu9JVM8e3Y8-8phFtJT7XaBqM<Dr+$!>_9j}f|I<+
zT;P9>{|P#%;qj%lw>IR6gT|8>S))k)mv88;$3E>|<jH0m7>$ja-ShlF5$EF`_y|8y
zgQTenI&cE3L`3#oGRt^i2l~2ANCTEGTwc9(d;1Bwf)l6xMzPU4&>gE&EU&+M^%s!D
zGl|Q<A5hbF4AeP-%v#pJbWE0Zi5CeeC<fu{tq<_#$&PA<-(>Jkq^+Er0F5Yp&mw$_
zHIj0FA!K79c)(vJIt*@~{TgCqW6Ll(^Gul%-vl*Xn6NjpENV||D}C$v1df>45<T1f
zVt)DYhmOOd|7g#OA)+6?5J}&uL~-WCPzppkhZs5>bxFoU&%l}|zv87uvwrkpS3z)G
zIWE@#M0CLBFcL0kbh+NzACnUr-S~S0rxH%~CHm0!q<d>o(G%h*#7JV?YLYD^NOrhb
zPZ-SH(?TKrJIDtttrJAG<<iLgkLpL2ne#A0I*i!Vj0$}_B|~9Yr*CzLE>chZxiLlq
z$63F~!a=dJF2}PChp#p?DJU|x9#w<XxbnKbwDg>r(Z^f$`|57byibDd6(hv^aw~k{
zx`GdsmwQ`_m>6rOBPW11Y~}A2u{J~|EUx%6<tKa|e0xC-eOeaS-o>sK7fIGQ>_KU8
zT~|=iMqws+vWe3hE}D_+db=dd4FvR84L4F69ADlYoow)wyYl_Hj5*1RxT(7LUoe4$
znP5iLY#dqJ(@j)|(Z0&r%Z+_?7Lke6_b^JpLcJN%%Z&`EO|#FL*_cVeN@919<tBg4
zdW*{y>HkT#*5JpYdB6qC8U{mX73#|i{fKF?Og5eeR)85Csrlw@y9C{?nLa+xHnBLW
z#BKM~qE{Iu#i8OIGrN`UMN2x?m}06oer%H$Y*KdNWsK|dnx3*UD&q%$pAT_fEqP*l
zOC7<bexm<s9<H*i6}D=Xg=V(}F10$^c6ecYbp@48bukPOBr!dcTW@2tJc7Tjx{PtN
zzLe^{8SC$k5v+yZH+BgIeqqs-=N$D_Yg1<2qE5Pt6&_=U|7=TkXeu)u$<oi?punl2
zLbJh(!Uvy?x$GLS@!dKt`}8iu2ab5HESq58<rz-m(hwvy*5~kac(JDcJljy5XQMu_
z-B_yg@1vqZbp|2aT~J*2gaE7bQ+`4A;gR?ZB>h%#DVsiRSB^7w$mITP3ynQ_m`AFT
zh%L)!v^eL&qAog~dvg2qSX=G-^h~^un6S)9F}p&J@i4sAy8loL=;%6r*d~;h!FJF=
zqUl>!+t;{SvHtVQv0Tr$Oz`$anXeph8FL;jt@r9H{Z4$LuRGYkOI0Nwi_<^Q!qdaJ
z(Kz;fs^Mv#3ghrUly-WVbO-sFG5yVSz81L%#`t%~!RK@zCMlBE!{EQj?AjfXcZ6vN
zhgC9-e<^T97xYe!zKsov@<+|k*R&u%8D*_voma043_P(8eNY8mSljHsPK_RETJv$;
zbKwm~4Oij@d7AzMimic<gL5O_h5YC&hNLOj;6RkpDIVQ<Al=N0ypgW^yiAqd5YM=W
z59y(dVJKry6Tj@#ZZiikH9TQURya)MOmE*sD;U8n?-rvca4zeZ=8TEbR*0DWc|>`+
zG`}9DI0l1X<6KqQEKbRdDt*jAC&nNybMA=JU=?>~Ub;2(5glsbX3NEJ_J#%s%`vF>
zhow8(v8`UN2jyZp%>BN^l~|*$t&)0hupcu#`J=q{kKw}Dn!X?ssa4j&yW{hDXj@Nc
zFWb-oUsp+AoFH^qo5ya=95K$P57Vavi~XPG5(N9>(L+DWBKE(l|J`cv16^sG^J}M7
zeN+JDPXs#Xk7m;sf$r+6oo9-W{y(e#-P%b<@w3<bN5GuT{~odP<zwF+U)ut1LTXr(
z^CEM?Uq=00Iu$u0oDez>X+zq~Ebju}Vij-O3sq6Mcy-t?sZJe84cd2swLb_kP|zHQ
z>A;mGvCNAo{6lT5g)S0Qdt_dxq=nNs55C9(qMY3n?+wH0j?q(nm>9{G0S5uQAWGkO
z!>p|l-)eHSLggH{J!(Zu=kLy2wG^{hIG<oX4E!<%jS#e-8P=&U4BgRYLRSZRbx$H4
z`y}3$pV_T~$TiiU<!wC`(*Dzt+cb67Rq?YTdxeS?5zFZPc0aD$TdHX*soE5zFqCT|
zq?KR6FMpazB2X{)-g{mY?!<~>Af&X_!M=)&>w6`s#RH7c$v7XUOs+G%Yet6k8G^sB
z<(zc*2bCJGgF#>rxMU<Ht%0__b8nRtK1x1<VSi|M7*>GU??FSh>UsOlx<*TY=6JqI
zQ>aOWwzE?8)S{Rs==EE>k&Bx6{T&y3p|)p!=Hc^GqF2++RcT?BJSgn<i$UV^0T()Y
ztPbChD@~+4V`N;<<gpTrM0V)yY@$s?%x!q-;!@v+n2OMfWEAnAw)VyG95XW>P735J
z!%vI4+S2Ri&fU&&<NDVs${(Eq5BjT5EiKpzHtH006s>CW*mNm94JZ6@ki0z8R!lVu
z1&_?4H!JIY3#y|<vR2%v9{p&?x}}&NJZkdoJdM}Kv+#oYDWQYIVu;+_wB0GDv~@vR
zK4ztq;>9$NHF@`RT%$hv(Xr)3Gjk_Bk(g?4*wHT0-iX8TVq;@DTqjjt(!+$0)WegJ
zu*Dn2Hi($@VAypKit#QXaM8DX_WN*BYtu>kC(UsUAUEu=t#6~dB^>U=*MaemDbH`)
zC{4mOr$klO`qeQ>n*{MVAvD`_@_imgE23bOT8Aa(dl<h)FG7p3ly-5k5h+J|&e59b
zDi0g<IxoC2*}gp_js*V+qQNhB?9V@LktRS4;c~*sK58FdhJnS$XM!9XZI=U!Fc{$A
z21X!ly<#e&ku>(V75JlL<8Z4CvuoXzq(y*$df7u&Vapp4kP_UV%fkY;-w_b(pbgPN
z7n=#aL?a0CB6*{;+;1{sDIB7tSPT-$91T(9`>o7iafHDGD8x5vWCdqO(Y?RcH2g(_
z;c`b{ye=-G3|;<&Ygi<d<12H}X-1X!C;1g<5qg#G;h!ce9`OUcSiuSjwO;0_6g^%p
z8cEy)3|l&DFjV(#k@(UWqRiA#8M@HrLkS=g)~|I&FLzqVF*Qv>b~;;N1#_q<rp$<U
zE^!9zHY)9!&=%IM!Kqfvbhb<xc2|6WKS?<!WxctGKVhily~samSYea$dnZR3eC2R|
zR8%Dcs`O23HX5IaWYJ%As%i;VSXe*^j)`4^3egy@c~k4*GRMF4X{v-`ebO6tl}G%@
zd*z;Eqdu$$adZG?o-z9rb28K?&yCn$S|`$f_o?z=rdx>IcFNA+t4ivz=D3h@@Ngm+
z&HD>ugCmd$DZQ7y9Yw)Vn3m6k<dz=!>u|L2IR==~b`&JHVR|XJMhW@f>{yQvx%nb;
zVAMVj6#rEbL+|P;&DkyfU4!SgPhr0q%2mRQP-A3eRJmo));4ALcu<7EzTN4HD?5mr
zn)5ZU(3(?>LN3qVHq{u8EoA^fl%Ce|*icV{J^m7kGm4RaYQQPfVM={vlHW8L?0TK;
z+X|7}fx+l?ez<DI#AF1ztMi|nTT3y+ue-}kLlUSngupa#2+xkk2$d+NzO$hFL_e0v
zBk}F20nqK`9uV#bsI(Rm{;&Bnku^VCYzQD?AUaNYGzC=DUXUS2F_eP=HCriUs)L0H
zrb3*ZqwkXP|I7Z=ISyK&C7l-3tK|pI5+gPpIUMv{7@qzGo(AKT#z(v5J?|Y7M0J`-
zi$qSC*l%&(KARY%1F7K{_szFrQ9>J^6m=|BIVfT>Gx3uG<mYTP^`|B7q6xJ6b?UJi
zM&<fLooCpoK_UBWoa6L=2X-!S3+C3ty{ZfUX@hV^%pv4lgJ`a0JgrT?kjZd91a>xB
z{aqAjzKzO?n?wi`uAF(8ulSWRmP#?o_G{D>I!?X1>=sE;B!GZ>+_oN1J#CuaKMCAw
zSpA;vnilnEqU<pLQhUlqd_nBY4VKqz%ZJS)wi3r*VatO#;yzI5TY8ecEKEI`N#chM
zg-!l>M`V_478mgFoehNo;7xQtf3<jYNp<DvtBT0fch~5n42P?rD8hAPcV0`Ccjs-Q
z3Vc&BNmnO8gzfdA)+UsGfbwhPAi`rE>&ss@6QnFUP!iZ)3cFG5aXeo|p5>|CIk>>3
z5GOu`DSLb_pdXZr?9_O##6T1^OR{G?*UO0f^+X@GR9#yh`^J(`)4w>$L`U6=_IErl
zu3ZoQ+|!sgNjAGkKRYii!aCJtYkQ+YJc+OIBQuKXWdCrVKsd}bHUe~XEHyhb;vgXT
z>!y860`a+Z`Y#i@1njO^R!EVkZG3WOymji{&&15kq$nC#IQY%$U$1M11I}c)I}jj!
z<5qXZqlx?H4=r0kUZ_uj?_vK&V>y!?b|h;m|JK5(!xL>GUxv?ZkbT0{KdoQ7nglMf
zy}abNHiw$@21#<x#5kSsg9d1<Qug<X3`G41=g$NqIFcx}g&f{nnNU4Ii}%-VKN{}0
zSr^87TM1N^)>Z>yaAJQla*ByV8`jJeQFmXlxql_TK9a*kCkz5vA!J(2Tj$>!YS}UE
z??HY-N=W3#at39h`(hc*^GmCc$Wmvik_e$wwkX&{I4C7J*?FM-A0PTOd4#c2xW^v-
zS#7)clNOFJ{OIx#xj~c&oDga%uAhN4bI{02atG4P@@`VZxcdO8Giwa-vYoOK$ky@R
z(_&l4C;2<(S@aAxcWpo<ngG+k@T=C>64^t;mlbbN@XW>@?T;k%z&>XVL+8@a<X-8L
ziOXg6iQMduw~&#2<|U<{;y`c}8M#Bg<G?pLxf*I2Xx5XlfQ8TgIpG4I5DQ5<Vy$KP
z>UN$GTitb9nFd|5BW7Sb<H+NtP{o8=nu4B&IP?Y7M?}P^q#)ki9Rl@3xzPE$6Ceq=
z*;ePqoBPik@&4X9;VXi~2b$h~lG0bv4$Voi^)OcCd~*&!K?}!+St~74viWlyCJf(>
zi&?Jdrm;OlqHbblhW&Q@1-xg#CqRS$rl!oCn;crsLi9cnows?mZDu0%y!E;Y`}lbp
z#6fo*WM8MPv@jd!@1qi?cvykWF&V!%M4{PHg5I&-+uHJSbwXsmd5mOkOg*$}ou^KJ
z<p{$U)4{TBEVceabC>ONGWI`TV~q7>?f6#9{IHWWV&hxjG)4o#a9_(_!w3_@o3Z0p
z{BJC|VdE?Ue($q1Hh+(LpR_OrBjQCka&9C;CM%=SFo`r+&d-eW)IUOpz#Xcey$6qr
zhq;I`0i!0zn_$|H2tyvRo#racVay__qd$h=9JAN}5n*9fJh@Bf#o=X?E?bD3u;7!`
z@u?`-14`_Wxb<%->fP8`<5SX7dP;K0_M2<iM1nAUM$k1}CX2RXU$w4D(Lh1O%le8R
z!rko*0vpirU&~&jp%s366W8bOFfk`$DGK12&lp_h5y#;yM05hP5wu7ri4tfhe_%*v
zacjv1`Jt=2zM0U3BqJ$%OF-Y<o7c(Ub*nE0jw=woTE3$nXo{yRtIE;lE~<q=XHH=r
z2>?8WTZ%Cb*z)YhqQ&F;zWAdK*Ms0OWsy;&Z5NiFfyl5XJo;$9J`787eJ@sQdD>~w
zc=D2-wnU`=enZa`zQ(Nf5y>G;1|+|INVYK_eEBofPXGPK8-)-JLWxW2dylnFRaq&4
zC16S(xude8d5JCbtCUkNoj>vn-iX^CmDY7ji|}9|!$;ihfcz1i#RW?6f?Z(wawql8
zp!!l!9Hw_~l7D4*T&Cn?58Lg<&}O=Nh%myJFeo6x`|1MX6a`5nFJ6{)M`hy0w0vo;
zw3}l;Nx)%dbiZ$pTkXEvB2_!9U|`|&f1>-m1OO^_)DTtR&XQk@DJCYTMO9WBBozYM
zp9Z#${Li}we02N`=D<ut@?uo*6)NLQH|#B7xm^1<4o*g@h=xL6sChPh1#z-Xk;BYo
z;{ImW=^_f%%VaSh3<d~2YLYaXgP0RB;Nv}^7KOeL%<Kwe44mCyIUH@5AY9fM*t;^;
zVfck{mOF^0B#L&DgHD}8krOZ1^6dNDge!MaNA|Y_G{(g_ecSaYN>CsdaksQbdEN5c
zOFSGce`Edlw;NCG(Aa%5mh9b$t$rPil2LHw69A+a$9~F?POnoU20z_azM?)C{?{T$
zVoXYJL|#p`VPSuY)K`^Z9?OM6Y22w^YCOrtu1#{l=6NWnEWtY?Hf^sVC*_?0v%%$$
z;DWpM<RHc)=J1tAg_NZ$?RoE<*+B~CDg-WOj^E}L^;x<W{F`tUK`j{YRytKKZZy-B
zuZa_#5ooVv$P2=asPRHgpI#2rTgin<GNu9>W5yx+I;r8u&);lxJMhT-bcg9Dz5_@(
zh@!i@<Pc&+9UDc5lL)Rjeh;~^o2yOY4dm5SVHS!h%`Jw14{v7YjI62u-IO>?^u)%<
zi}9t<KiQ&s8Hhd}gVv`pJ|t>oL-$eB6N}>FhFcag3Wp`ERS118ZK{cYzgWW@bjWL#
zu^+Er#yW7!-GVNrUv;71f{s@lnqRXXXNO75>D!Q05Ui}T-m{s^(~}6K5loQKAk0;r
zZh4tBJn5UJpZX0l5EsoMshZWMkq$?;@-PW&vBuT<(!)K=b(<eRqiPltqY3O^qY87C
z^{S?1v&2w}ohY(h=6vbio$BZ)A@cY0hJW9AuDN1eV3q{=T4-wnQL>kF4fx?$3veNu
zB0?Z~dkF~(@`Cg~zY`I6p#blqQ!z;==5>T2MrLMA?OtxUHm+lb)-^;5lU<{KqiL#J
zWAJsyeD5JVe3|@~sfUs<@(MXFOmPvYznYj3>g@#PKGN)KI~#mixquY=$86=ksOKrx
z0ZRM3;EGG5l$uF)?N?Xml3$)#%aE*ZF4GzS0<<+Ac=_0zhzKNUF5kG`Y+8lMr1&|$
z0lYS`Opmu}?C?>_=r2fOP$U7`C(9UPfqY>LxTL4?{7Lj!@YWw*j~xFQQ_fOaKM9R?
zg(^^SKJPyTU6VK?W}CwHdNE4R=4T~mB8)Hceo{>DFE2M!(^IiUKk!Y8rJSRhqc6`(
z?4&=9{YqmQ7CRX`JC%0p?Uklv73v?<UvADQ@)4HTM=IXi&c7Iy*6}5Jr4(ah6<9wM
zIEDVNsUZGHS%~5B*95CD65u<~BRBk<pt?*j@f@tKS}h~9QH(`d$3*Y4%hdY_6j~lg
zIaqFulPDu&yiewkCdc@Mz5gs<ddi|e*bTDNb9xWVW&JoPam^equME}JKb5{4@Vd>r
z8Y)<@jStI8fDf@V`+O#cERXDMSEJA7NDvH0bD^L2v*q*Rm$J3$;$ifGb&}sOGf6Y~
zvTL=u8(i}XgKym6WFwLRzjHc#OTW8}lZse=r~OvJ+=+9rXEb*$$mPWFc0a9sx~?Ht
zlY!>rK?(y~7<-^umQCHeQ9%D?OQtz(MC+944;SQa!)?H6c1HJ`=r+LhIOSR^b@&q9
zVOn=i5=9<qrK+X{9`(LO3DIY$uEp<ZTn{z@J!hoy3uBtu);*q^!PUG<Z<3Fm_Z1W7
zE-E&ujUf?<1W(Rx4wt*%u!r!~$^E1n5B!227giZST7WUL%Jsnyc67)fD?>@K_}NwG
zVTy~WXDOn|=Ed&xm!*xhAvNZ2p``bp1s~i*{KFrbTj;u+07OJ`-!g~+epd?g^B30L
zU&mu2f8F<WX)?~;Zh2NUBANndwUpmIYFZI3|MF7NG7pnWQO~{#HdPR~n@If(%*--V
zU|c^u;Hu@twNkuiN1e=bn@dy!Ry9;3X7PMfAuudXw*Odg-RukE@5!DWCl6O_Ye%_(
z^mmGtiBK0lv}-MgNV*sZ0YCzQAdmYx=HHeVff*rS$O~ZF<^BPs?~)eC08suKSGm0<
zk28DgV$k)2PQjh5GgHX*uwV%gI=*nn{9VSx<8VQEG1!3LDE#4GC2LpPtMH@lGi@HH
zu&Eqpir9uor=@C71Szv|R7?R7%sm}a%pnMic^F)0sDWc8W5HNr3jV1IWLB198=*<c
z+h|jbc*8-F@|Q95_pxK-j<>1tJW9MBa@^dhmcr&<a**UPho43p8(IXj`uZW{#6Y+4
zabo7oGT52@)Ag++dBD?{nUE_G5U7gtZN7MluQc+iHC$fxtwf1Gv+N9>HKM5`-UIbm
zYY2H?Q~rGD-?d$lbvD&%c(aUKB5W+&7`NOnrkn0EB`FdD;8u?Y^Pv%8+DIY5XPBUB
zSKFmD5`FN4IPe3{sYHIe3_{)HP!pad`+FGYCB?J$pwlPhb(oMSVab5R2K2<Q&-M;u
zhzsrLoW%{CacFhyn66C`%IQt1P!Cn;=h2%g_{=UM^=uACxFjhLQT}iw`ZZ#}n{b>|
zYPKkCPgL_z$?zwNC{et=j3&+Nd|~*Izr8aZ(jhR`3=>EI0!Ax0<_z`})9B;`{HotE
z2Cw*5ZiJ2TPl*|hlJfe-G*eJ@-)4>qceT}K2G2%>(^)?;EIl09%fDO)51dDo>X5Mp
z4AG)-=vc)_vazJ@$>7@~o^sACmW%mF^!lhzfM%F{qZbb<ZTXva2SqLAo~@@~NFu5@
z_LI?UZiPl<QMa2Edcd9{sUqylS)VV{Vf)WZ4E2X((Re7sq8SyNZn1!2OXS@_E=l_^
zDQoGeU2F(aDE;p}r;5T<-Y*5y#qDjTr*aFJIDxFvBDqE%wn>AEjd**foh@>gX=rV0
zFE+zxFG6&q4XUIPjljpmfWn;}_xT4FHBZ7-{KlS-BU{Rqd}HU-O9*)AsapRU$B6nV
zO0MPy<I`{{^On*(F-afATpL}9W89BEpFUYFIAuTAJx&=5la|Cn;vN{2MqXT+y^TV=
z-`JGYf4!sW<;R`ShLegPKMidp<Cgru%%j@G1TzP*uZ*@qmC{1QVtNG`P$#E(`t3;>
zDR6dm3z?rJmj@MQAlZzvxJ7q@QsWJ5WLX33?SfLv3v=bu4cTE$bH2zm1kTFaocjsZ
zfyEI4nav6qY(hRpl;pwDSxm$iUqS{Od*9LkU2n^y@+@MG9(e*}anb{8v@q$TY&!zL
zz00O^AJY!7_&xScwx|)+%3ednU%&7h2U}O)VfOHoi>3j|bK>sGFm(5I@zZH}+Q<L{
zI&DshpOn(Z3|TMmE1LJRGpKW^(^BWo)W!L6i9Ukyj(5x&O)B*fS%OV|N5%hvF0lc0
zGV1@!c|XR#s6J?|vlLg2UeW&ry+08U{sREX{!4xT2lEmHf4~|}NXeZTPM~Z|XQCZT
z@}t<@^nb8e<3~b8z!K!uPU#RvL4MQIdY&Y&r%77@Bg<IyVE}^y#*e4{Ht#-^M`E&w
zu>%7A5JL(%auL$`T)loy^qa1=fx+YCWO7i&F=RmT@ewOeDYb!t?nR}0GY!~q&h4l>
zN?O8?%c0SZ&I!D$<I5p=yT7;1#u}CJXV{4hNFEJeEwB<iVKZ4$n2}caP_&WDm`H$%
zUF-ML(Ltv|uZ#O0shvdWVD+Emo4fV)&{3q*W`}UD{-?EXFpT;hC+ed&?$N1xI-HM~
zx8sH{2)r)Jp@}yTxc=|*$XQd~O#gzj#}nEE&d2@M?FDXSrZt{8@6+N5HN*cf@EB6)
zb3U&^jqag?Q+Ze5llDpCM~u$Kf^Z09tZT3!rbrNIt;a)UAKyNf$Kg=u64YSfy_!Z#
zNQZMF@{?^LWNx9^IM-V`rxy4BBK7~5xBoxb{r_+Nw_pBmkNp4QQ^NWj_zx}0zlOE?
Tw~{Q>$094KAW<%65cGclM8kc9

literal 23204
zcmeFZWmuG5*9HvSV1Nq93@9N8(gGq~LrB-qLxXf9-6f(j2&f214&5*`(jg%rAYIa(
zGebAug>gUc`yR*l@B98e9)~b<x!1n-UVE>#&b`jH4OLN=xs6YbkA;PGTTWI|4GRk=
z9t#V*_{KHh$<4vGr@$|47d06PtfF4ZwX0_@WOZDyun0)5{$gXrC*A`Z3|nhxyJ{;b
z3Yt3Fvp+X;G%;uQw0{A##=;Wz6a+rno4Y=z@wB&da1rzrq5aiD5cqucn1hz)R})uT
z5n62}6`E&`&gL{c?A+{6Xd(DCG&I7_W)^~KlG1;~fo~$TR<5ou1UWc7JUrMvxY!+?
zEjc&^1OzyqJmq-$lnrRX=Hlhx`rMPv!G-R3Cx82qG<Pv|wtnGi?dU*r)$el?M>kgy
zTH31@{rl(lI9;tR{_`dWmp{`2CdhI1FC3igPdNVV8xR$~dMfzL(f)<Axr+<XKZHm4
zSIhtP>_6}M-Co7Y(bW;qg0r=$oP(>mGa&5x{AzTNr+=jX-&_3OzM#(5=D@6fNpt>@
z{-4kOv=`>Mn)v@v#BVMCdJ3o+f-lVR?~+0Aix-|wVPT13$w`W9cw%o&T~E{)9`C?d
z64ZO(Vq23+f>c2&rvIp#B0se5v=?!fNGcVSjNbD>jrnlA#ltHhfXkNji+elP?p`2j
zzdt3I^yo^8o5Aezmw9X_`fMNG5we8VT0Fuf62rp&?~niL;J^Fezh>}XSMdM4C%lm^
zrhJb;DvszILs0z#J(A9>-P^m>cSI|-a{L*0#&7gx)7v4!(ce_=;&zdJW4kmrl=_cv
zO7X_g;LcxZul<j)vJj=D>0lFa&;?@Q+@QJop#cqqTkQYzB>VH7CQu9y5M#xfH~)8b
zLF+eF`U{dNP!j)ko+Ac`;fd9~``ZtViU>AK{IJV{!S8;rItSkX#IWAR692nX@)*UW
z%Df%w2>!L66dXVdjD>mnpZD+M=2&U<g?l{w&md_4F&r90X{@05n~Tezx26O-CV296
zAaz1qxM#aRN}M`fFnt|aoRhp~)r-?-udr~|9=%5>hh!Gq00tKHwW_N`-Qydp|8ANl
zaYA+L=6-0%!FC@p>pM^*=#cUF#O0vHU}rN+6_3|c4u=qQ_%Z5vL6d;c%7p*aXq6I3
znzkm{LX_j}7%K@auZ8<k(Uy_-QhZ0_YqBviF8K5UB61SKQEVNia4AqpDJ+8(w0Acp
zEzWQ>Y=BmUE+}k<qqr^4%-!yzso&(vSoOKHeOVT9?Mi+N`BHa6*)!weiMQvc=V^WL
zDn)NrH-wV&W>!$OxcP1b2g?QJIohYSAz4RTKll3`f!@18n_!^p`q5;rK8c{4yR?<Y
znd5{mbO#4e;#%)MeIbp)q{s+Py%JsmasDNio1WbscHK*SW#6YigGw$EmdC(8i+F4!
z92eUD-C0#8(t1zxU0=8ZdMyV5-NwX*OVANf2lWrqR;r)s+p6~2#n*`YR6Z4QnAG1G
z5oTOfroK>U-9D}PF_M>>SFek#d+Id)2G6~q2I5oIQ^GTGF@GfC<9~M2)s{7Za`bbt
z&({*Dl#QMk9TK5V7$UNGV;5%{T(g{KK5W<fwCqg-TRCwfcxevlbN`xKPs!_wdfSoB
z51b(fTLt?aZ+jwFa>?#x_Vu4Of+mc7RwlYh@L%@z?)xlF=ta>r^9ZNO6!8?8y6^4c
zRo3f@^$n~nl=BVMGzjjSXXD<Wd9Mv@t<ZgX4_pbhA5b<`1(<EYlv-KMiTw+U8DY&k
z_*)}=_=PeXl)Fw|E-_2E;7;M&eH$6<hpd|8zWU(C>;{f<zuKj1rlO2J95-}p$a76R
zCmXzY{Yu8&5TFUiFxutlcQjtN)vOua#!(ZTqYuuqRJ>$1iwm8%W@;BFGQ9A3)tAPQ
zJ`q%3q#|mp2<+Br%j_o?wWs#KDug|qyG$jym(LmR<KD}2`OFkv6cBbI;Qm&(Tckck
zR9XwHjeZjv&^y8Xy(G~M@#aU;4Lk{~pfC6;{qI;A;~o+@1c~s}dc^5|ARpzSD7G%L
zA0OD9_z)saFeLAmgvis%vWm|N6$-AAEP&Rg!7W@*lMrMRv0P!gWGI;JO2d@1olu2?
zcWfSvh+?#}giEIAHUs>~XZxh|$HkXwmV^!RM>5dJvDLMW$OUjOpawi}SDXUfNNNep
zzsFxsdngH2Xr6&4_->%8(LQq9S)>1PjYZF%fY9+i&Tb#+O>Ep!b-+cq8+!B#D&QTc
zdbkCxV})r++s&W7of5A3k(D$2gx4!<kd)22z}&BF!plbK=WNxh<zNsaN)S=24~A8k
zd9P1AmMI$#v!ElhXFk0wj1I`olIp<tlIPkE4+@EbCz{y3c=z55Q*hFopw9KJ7W_H9
z!!O^!jH6ylneU3Kq^+@@i%ZaR<u%jcj58NE!(`oQUyR%^3v0qdfpMdP<b{lenYZ?+
zT<k5~vIDY5<cw;v^6C>RhT68keLEjvvc1&rEpOX`l^(}cnfSG}&K!p6KE2#=4%!-F
z?r?|w6y%?I9l>_1-8v?g1DUDS1mi@TmswaJ{R7ON=<U@?$)n$j#Sk3fCQS)0kKII)
z;u?(h_UHK*xoNSQn@1sgsyi$!46XNL)S(r8_l%Am3*-0=3JopuyCLxxLtG~Xt4_z{
zWmP;MS{D|#l-Lpl@~mQhzCpCmJ<<Nk=x!Hl%V$-naToBMM66;`Tw1{^H_S@JL@J(m
z4+#Dg-x+V7t`X2HYhg{8*5o@()mxhE^mE=CcM6H$d+uIm@t*9chJRx{-#^ic!h-Gg
zl(5al$En@gywZ*6Aysv{Wu(UsDpLkq)fZ~(c)nYTh{Pr4ng|P{@VTv-haH9&s$rM5
zosa*|%bRRT=xqAAeU{O}yQFPMo`!8FHyy&wv<XA{!Np^Xs(0+gQI(@7iF(VqFRczl
z&wU*l_j}?At#|tH{&C(4q1$;G_pL8S;v|<Px?~Mhe`)j&oP}$lyy@_E`bd?3`*z3f
zBK}ZkUrEAsutE2j&_79h1;wo=@ttg7nP*r*PY4QZO>ejmGgIKiu@#S++RBfVs`=I^
z@u+)-u*5e|Dk*)0(z4jnzV#^N>vtlxN$M$e2pAV*LqO!L+-`TPM!Veew<}aRSH3lA
zna&auSLrwgp@>`VJ~k`DVvCRSu@~UN&7+@<ZuHD3Wr<6!@(AxUs-jfZ>CnyUV!v8*
zF`9ZjmKnyF=@q%P{{+2{kAcPMoTa?;H|TvM1sv13bEW|FZ-}Mo1Y&N~t=_r6p*qNp
z81Tnzu=riczcKg!vu=p;CJX70e@1!pJJ82!Y&#RS7cO=!$+d@F_KR0)Q6|3&c6#@P
zKscY0k%c|Nr`Jr|8oNQAMJXChb*#*c_)OZf<L3#+#+~PQ5nlRJt0ds8jG+UUU^Dxy
z3Z@@DyYOBF=Z4u^EO5`Q>i3D)aI7B{%bSbnI8~+jlSd0phOLiwA;+2F(c=-#$8>>W
zOIN|ZH?$tmT>}m^Q!OEQI$T+v5W7)}4)k>}{mZAj*tjh;@2Ou)s9eJ(`X(Rb;x;nF
z`kJNPEK}?xv7FG2->^jU1Qhp?Hc;#yu2ui7ubL^hZkS~^p#--An77r@wM=yg>_*;0
zR(6%|w;ef|@8jHfr5b3o{k)Fl)6JkwVo&Yt^@Vfg?qqf4S}qX-dc#z-t<)9J$$KO=
z>J|NE>^?Bn$3&|Bkm^N}wlH`czM3cQL@pDAx2&-R3BFqPoR7fv(4051#3hmqY>MpK
zWpA^;`7PnGEyL{SqH^1-c=NBd#9}mhcsW)U8__Fu#5ZD_hFO#yLy&{A_-_J{E;UIl
z$lJeqiUIUyQ-Wat{6`s<t!?}L^4H@$KTiF6zDI%l?PURMC022zv9NHyhk;@^xYMv?
zI#4Vljbmv+d=q%$JlIXO!lRV$$|cY>nWfyjl?>>m3$Iu8W;-*Ww5&SraUCoayQo%)
zxqC9iG8_vB14@zNN{)Znm+wRcxZzH=#fUBE0?Yix2T|#n6~Gv>4t9<`TK<6tuq<a*
zGEq_mh2tG;Cz>YkPJ0UJwOl!#x={uP`pF8@h}$BM`V57i&KL>__H9pd^m$iA=p8Is
zxY{B0g^URx7E834`PVpv)?aPfaw!^``fif(tIKHTc|Mwxna8rGd(qEceenIcoI@tl
zvlRf4*_|pmcX>wlJO`z02)~2TN0{&6FJ;y)O(=*;Z=M?w6gc=t+}`|XDzo>kqQnJ%
z!Shk61dRhue+7>WQ%5YQpzuE5gfo{762_sv$Lhghcx6g?S5OjPd+o~x#K2{11y45v
z-EEcm)MTdDWNOK^E=EUd86!C31FB+)_+IJ)9bTI*`o~96Q4^b=9oW{%!18>r&7$wM
zYSS%KB4;Mg)A=UU$}79IKLG%%aVZqpnA7C)#GUnzOe!LK0LtXutKA2yQCqGdJ#3Rr
zLnegJV&6_3x}_Fso)7PSvFH;PI-SBfwMFSu{CL<QtsFWQj*w=p+8|tC`eIfk?1Ng7
z@KG%+c!b+UjNYecgl*<~<|m$e<}Tx0X9}|i$&3oKI)8HwT0LyPO&8|B;xT;KVmV0b
zUgp0-I4=D)lee$LH~cfX0FOykfT2}BKD>d^=EOif-qveIDRZl@a*^PPC;RcToXC7W
z37o~}5D0fG2i>>ZD_bKRItLk3)T1bCb-`_Y*)p4(9{8JaP9fr%&M&IcL^js@K5(5b
zk29WRVydd}Hw`tP5dNg?xTEE{BRs*^dz0n8e^gEFu=mM!9|eq7Z<w#v$v8P2L6Q0~
zZU`zq@!&fuXuB$VZ=Z61?IFFaSil9IhlQYOB!oBSl~^@SKPo(6^OGesM|^%glE6M;
zM8G}DztO|N!o7%`YWqCz$-OMM#gUE1tI3o-Ph~lAn0OK3uN)3oVXhN&LN)s_uMlP!
zN62JD|MXcl<z$s!k&+Q*357Ixj2|&m%RYrP!cW)i-{V$d3m2-29P^(z-0H-uR4Yu&
zZb%iC8_l#<pyka`Xg#r#KHgmzg)s#rBO47H$G(6EXGZFh@fu$NfP9b4ft%w2agLbr
z7sE?Qr?~l-_y<c%>o@gttLn3la=aZ^48~bz>vc^n$o#JRqHS9vRBf00IY47Smn_ux
zM9Qif7%OcPY$GELF3*W0QSmt*=8Xwyr@NoYv30rC7m|n4d;3LCgfy{*caCv{F6(i+
z`>8)qhp4hHuOsy@eQ1~_q2#0E$_Dc#otNG}E0rR)GMDIWR+hBWC!_*$r4Ns_*^)Ce
zw<ecyxu(eS{MKXoGhvbnKGc&$+KjWdw#g~zA_FyFyh@n+Gtf#qpPqB~wV<(?qp}#%
z^To!MbnBD5gNKrP*=Ze~zN4SM_j7kvoT0LAAFiw{;{QmC(?@-x?$HCG)HjxvYFF@V
zKOB30A5i0}5F;(fC&^Q`&PIK<6>^L>omoSU-alDVErsjo-r1;9G9)U2^7ya3qjI)q
zT;kO2le?iIZX9uNUN!F@()V`xpGU9wA6=>sj(!)0a*EzL{(y(0Nt)B?p`_H9kf*Ob
zt-=OG&S4V;1fScFmde2)zXzv1KQoH_BP6em6Qt(vd^ghN#6@#ouCBdjbFt$qZ1Z|q
zGaFk((L4;GuJ(D2qMB-L4|#t#iPrex@Ug~gi`N!}>lF~M#K5=@h8jAUeeIQ~%jEK^
zn{>KNQs+zJEv4=NN8}WS86KW<hdjejFPJ8ti6AupMC*XCHqe9dXM<z%R4Fw1IBjbx
zA)(y>(Dv7vc*+uU+Yv;wDtYl1R}00En&<?t0J(Y`Ff=J}kN3o~oZBGHu}SrGjn44i
z>Su{lX$4lVs(R`-SHn0PTp<ZAQYkNNIUTET4N|H&z8jM8kw`XPOt{f@et+syc~sxP
zKmBQO&1BQp<xd|wp$;HWu3ic&#ifB^Vjokr*17?*ggT+K%4V+-MeuilZd&)@4!pJv
z`vzXlG*YE9%R7B#OL>Xjho=jse$vl}FO{nu%u=VG%62nv#plyU<13Vxx5vC$x$%h>
z9b7q;kFMZ}WLa|K)ij({-Z~L&%jjwh5h2d0{xzQCDUDq&x;Le`dDD+IwUq5RV8+Of
zkpf%7#~xL$8deVl4;?nghDt)mpIu*`^!f^CiJ2a_H#u;>)=}N98mS^RI4Y^I!hJTT
z>$^R8h_kM4wE(hYnpU0}l(p_89O*}@@2M3hs>*Xn?GrwDjrx@L3G!ivC-=f(UMTYn
zio+JabZRz~PoecxL8uylWDSm+b}a+b-MqLTX=Yt?>uX0DOYAbWQ)x5}n?^r=G2oOG
zr!U)a*t~BU<mo?(o&v3ek*sXyO&{K0!Fg|BgIDKsKB*H@>8ja9p|4KR+&?HcD*|@U
zm@jo@ah2`zj_$$~nMlfZJ>x8u3`E60uKtmk$iNn#nPN7`fbjC4F8;W{Q?_;{Fj@}g
z*P}n2REMM<vD^32R^y@4zfu*VV~p?O$bn~T8FGK73f^OjPpzQ@UBkv@VFv8cFyjlf
z`ST>af2TZ*kdGhM93KbvDY=ewqg4(#MkUg80u4X1&w?2d@$Gj-z=M|twcpim6?(!B
zu97SBi~v?Sd)x_Nh>xtZ|Aaxcd^0?CI2FVvv{Qn44GPAb7GlR<0=b$;T7jfJttTIa
zu0A%6j_hr_7?FI`sA)2o@GYCq9x$?C1+c?=e!$E^%jmBFO+-_)prTP5q*Fj&mug8R
zx-wz@bvX-9h!j@PyDN;!8T*k$oW>R}&DM6BF34LazZetBH<C38Vq|vB$GbWoy!Qoq
z>*QN`n@e`2Eyb^yTH6@)RMPl!?}B<;=6z=fhCe(%x>FAho&Rv(>y<=GxV%%$e3{Cp
z*Al-nRG(PTGWLG*C9;^j7F2?g!e^oZ;p5n14xb^eo<6z<ZHJ+v5P>`$;iq(wpYXYd
z`Y+Qw^-9DrIVSI40(l?+2ssCh8O;)Cj;wOLFmieSDH#Vs;P;6(0hoLr>RuZszM41w
zkXp%cFuw?rDpMNJraaiHu&|$<+41I%&iBXH)mb!i_`LC4GOlEVp)07WxJQ|>fdTH<
znYOjQYd&d}xp75|(2(Hy#uUxS_{`U!R~o`UJ;2e+|J=x0$0{<Z#+X#h@qL<pQdu~>
zuj);gfA~BsWJYUWLP`ej_ql-pw@!0VkI#}70z0D8DlNfDA5_j?H2Mj-!#6oVAvhOc
zJ}H+yckGb&^imyDXc9l~EyeLiQ~}R7<aH2S1_VjorD`j|r(Ti6M+8E*Pjon$e5Dk$
zL+`+P7eJ7PNDw3_i)A1bP6(P_-{d1d3Z_lyAlcZH`<1Yxz#kfZO$SCYk+LRhQKrli
z7~Xg&Z`irlWx-p(G;3?otFi}s-$JJ%BPF1SNo#>sdn!8>r&az|aiCZTPJ{0|sPO$%
zH1-+feE=C=AxPqe_|EpOt%OR1B-xxY)$Ft+^{X>h`q_GFAC?CiN6wNcW@VKl>VN)_
zN_lLv{~Y<zTzbHbz=qA3Q09!Gn@66yh<l$pN1rn$%_xR=^n6rf{Gs>ez^Ci9;z-^p
z!i40v3V^f-!BLYx^{l8Srf}Xj1d2sNZ!`;{a?YS7(Tm%4Gc*Ouu*!Cltrk2pEVze6
zKPQJV@xtE!K5NOs17QxnO^qJ5y)fS#emU`tD58&_&dA@XB_?Jp(C<FExc+X2^Ytg;
zAT7b6`}miN$4NO3u+0LR-|5NsYS@U!uJ7k{LZGzXa&ufEe$K?qB{(9{^)P&3>D31;
z1c?##n}s$Rt@zw32OUh)Dvvd8X>R7DNzJc}1ITfvl2~Qjr9$4zLf`y{koiFEND@cQ
z1h+`D63W1^@oDiSCc;}>J|dCo`o4SyjG<m_Em{F-=51cq@K^a0H_A2T`m}s=(V@7I
zRAV1J<_4UG@BJL`AWRQ0ef9B<u0Pp*-J$tD)YR-W)i&Ggw8-|<A{+RR<_8RoU6O*J
zOh<BaB)5Fi#L4iOAvmW@6o~Q<y<frcIcFEwZ!X}35M`H#>{P4GA;MQ1g<d~uy%}rw
zOM@-UMD{vyh4X%S{OWTg#IigjXuJbtkq#-d-{F*C&GlRhoQA#k9O8=##62oMGtl(#
zt78mcnP$;}9+7vngp1H6CXOtxnS5X3<gVjDH0C^{BJ&z~mQZL>NFC3h9*>+R8Cq%(
zs&YiT4&j|3)0|On9<zG*XZ_HjHv-h^inKF4B}x!4OS~m=Bm^~2{nOBykpuL{JO@s3
z5J;4USJ1@7zHyri{w3dTT8krrj*R%9WOmR~TwSZvHTVa7$JE+q{ZR2d?IzaG5X!j>
zxX10D2V{AVrsI`>XGkQ{$Oq9fiaZx{N=wWAqkAkIP0*O)sZqv}_#6qjugTXKe6?|$
z$<<5&_=q{`Zahpd|B>PIbx{1>kTRNjjRFeLAZUi=BG(G}Xa59>CF1%@1VimU7okJ%
zK;}}DUY3_MxlJs7C2UEaR>iI#kOX{qI`6r{+*f>mb^XHC)PE&$@O)neLtW7~38%?<
z^)UhPc6_xDfylz+d=pd}^|Ry4bj>`#AEe8%+(kd9gY+Jlwh6Zxz_IcFY`hy8Voi(?
z$~;f;5^YS~JW&Yr)3*a|ZL9_d>nS3Mqt7jI@FWTCN;9&~OD2AKOdf!UnA@nq8ciW$
zzm-Sx7^i`|*9eSa^$m?Z1=q6r;z(6yS-Jw=4ZVocjN88Ty!xYNhA@@Y`+3z682gOY
znN+{ZznXtfa5UDl3z}i_Rm_HFjA?za&n;Bux3jf9fW`Sf#Mn`;7pM202EcXTeIQHb
zf#(A;aJj#$Hh^0>AdQg4<hL+t9pQ%A#V`5B_l|a)g`}ES#dL8hJAI+mQ{zioU4qNO
zL*d<^hVWm(0N8i;8D>dXeP6cehrk-@?k<x&c`D$A=Tbi3N|d<RsQG&I*_Fde4Znl-
zXj_4g!}UNyOn){k&3inYV?i0j3rt1}DsLyg#Zr7!-OL34>qNW(-Ue5$-*~d3Vz{eX
z2a9hf1GUHTTa%O)nB2e16@$<H!|PH96Vt+)6v=#`dh{w>QGRs8>~$9U)$6sfl`Qm8
zD1SL29={l-yhT(9$rPdS$|<W6{wo^+m|Bnqe)&N2Wfr=~0<}}8*$?}0Gq*u`9@b{a
z%pYg!`kfQ5m{WJE^2z|@!RmXX{YD!z+~VJ*rcq(WMy=Ota02^OfF0%v%W#>1g&+!K
zEy5CDO%}|)?QVkEg_=jUphlJkb^V-gL(j1UdH!ycM+!}h&PT8AAfFuL5+YXTNFW28
zBKZ<Vajh?Bkfu|~tqX0tDhVuQyWnYZA<en3gx<`5brvZ06VKP=4HWGu35ZQ=Qf8w=
zY1TgSs*3yB_Y{6qF-Dmr7s)arvRF`Za6<4d_>je2{nc|XCUEfI6&t6So$8ZeGSqsk
zCY;W%qN+fA;97&fYDz$&<K_GYgWo7+H5O&0U^KZ)A2QQ5q~}MdmhrVhgY$P}0{}=T
z4&)BTtW5^)1@{5fvfS1Apnfo{JpEKi7c*+JqJp8|7-9Fn+%9I|8&kl%HwkL7Y@<fX
zW{<$YvHy<Y(GPJ(pCFS06ZpKY6jW;LV<b;vIy&wEqdfH~74~I{*3Q}oylT;=*cJ;f
z1N-MI|J5DMdo3Kp<~rp$urfkro<yNV<G4nA$xELY#jSte8;|{QZWz9EXiM!*fSP{l
znEo&8$fGA3+0L^`y5$Ju{L9JEGjg?84)SpdN969Sd9M(UNd}fB7E=_yLn%A?Mn753
zVaXeRk4FLi`SaLqW+9G|(hX`<mjZlzRqlp{$)bI%k00w(g-U9jF4SnJZ<Wk(?OkWS
z$ke9VjTbda(*}{S?g^`Q>e)o9eer^|4Z<105VOFmnGeYV;4}QllM8^0Dow1;r_6OY
z<&pFMw2{C^L9F1+OpA%tsapFHDHjo5`yVOd`QH2M6<Q(1A^r|o_{LEcwxhhdWfn@o
z(X6nMVN<G1o(V^#bm?&XcwT$moby8WtqFk&o8hiu+Y4<4(b0he9FgUsFv=xsZ|ijJ
z3Y%!=nfV_WyZ!ZHV*9Ywk{8E2OI{OM+4<)IA^0=fNHTW)FHbx$^KI4LF>KwiAvLMt
z?qZ1!{UYrw{H^)K#Ke#Zh!H9-zixT8-_rl|Rse!ol$D`wzUKK3e770DY76;7y^1P}
zN#!b%1bqR>%69~GaX^OcC0<<!1{XkvC<|&>qyM=;-rg`Qv!tHIjZLdSyI&)^xS}V4
z&uBPHDWT;KmEdBf-FT^1zE<IS&iv=e3^@{3R&qWEySy*VlP?&M#d$l{VWey<T+4lk
zM(Glp^`Tr_%kQ6`yR7sUW*!cyOL>}4)wq%6Jr&jiPev=mJho25x3NrG^T2mx_Scv*
zRk_;5fDN^rCh%-$%fG$?wAiSI?Tpp!)R;EIJpKa>1p~5*^@^ds{K*R2sv4kUFQCW#
zQ9%1=%1MGcla=;i&kFq?incA1L8GAwB0la!d9NluF{{%?F+vSfwF_6O%Pe~GvgIqQ
zmX${ra5W`6zWwa5QdU#c@MLBFKY7WV)Ibk=$9(o#`{9W(D4X3Rn?a4MdIf)x8hxp8
zf_-J~WV%eG;8d+g6<gKM0YUf8)u}Na;YW@f3Qc+njEbL)qZpJ*LrEB=8&0=t%|*}9
z&zc<@&zGZrMCZLy*2u@K)JPjK5gO4OCz@#L9G6i24=M!<C-g4b0hOeoO8KMs`PX`$
z<QEhO?y~(6J>-OScGCLc_EfDF(y(4JS|MHPscLM>$#HL@z+n5k`=+KH5fvN_M<&NG
zBfknQw$yz$HcO^a)%D&0qezI;H&#T*FHflgr0UB)zhnfI%!rK$EURi4d5|(vJf%_c
zQxe<>_Sr0vImiCz+9IlMw^tnIkr*Mx%S?pkOt-<5tENkjza^&A@;$?J;JGY+?;Pc!
z&WEogkQ1Clz?TXUijU=+_vhZ<cy)E2`&@L*T%1M(@9gj!meO~DZ_gH(g>ogT*MB@C
zv>!qmiB7st+AAqsWUR!guFDe~Y5(XGz5F@twUWextUs9OvL4J@${%)Ftn=D6RAE3e
zS!fAzxl8`916eN4MQwUxyT*g;k$gHHImVT@bP+j?eUQ^dpQVA=(}(6&flx7WUc1lr
z$IA)bhW^J(zURbG#ZASje|Y{}d9={Y@c}r}5CFMmi)N{15b@jeK#qs5@OgbU3XQ{~
zg-FL2L#i@M4kc}*PQti4=JA0RC6l$N2HR(E?ii$w^iulyLo_pY$&*%m;@pc834bHq
z8yW^FNj2hcPSWONsQvcfjt$4s-LQ{DIW>W#-y(ko0GY5t_F@fF!R*y^&zCK+m9&3q
z0K{T$=J=1JS=@Zs8y7z+Xypo(uk#R02Z`Osq4q_Ko4uiO*G_wQufbv^<?(4%+Zye!
z(t-EGI0^Y51-9NlyQ6clfozOX)imb9;UV)}j<ck#Nr<X?F<lhkbZf0^-AE`=_-cq^
zyf<72?FnK<55`PGc#zT>kT-zJ9@GAOJ2F~umY`ObZJ~4ByaC-p(9IS)lKSo&cUtQ8
zrR~^44x9T0Z(m!0zI{VUw?4%iN9(<I`NiA-jD?oq#y16y`cHd;eNG)zWaDO0N-yD)
zu}1!Jdke6Yz<zj^#e__IB(;$Rsjdt+KL<;^`CX~<m*N?YGYzZ>JhtvIt&8r34B~HM
z*w>6LXQh@t5#@vi)^ww^Hh#tj6GFZ)jB6a%>pJ!ih1if2*wmb7Ou50lhk1OL&vh)5
zG}pZvf$?TZ^@NWfXYPO;_x80!Pgc_fz0b;cx3$l3pujrY1Jp#;R%ZI_jYraAY%W5!
zO5Za$ULDT{6YBc=H=G~$1yrgl3$!JBuQTfv8;gfVQF2?UsNAn#N%r}U3Z5=CR~Que
z671u8+UH*@$yRMv6Ht@%;w99ASDiJ^z@>+0q|6ekPCrpO=}?<;H{ZQtcHcBYpj{bq
zyg2K%7Z7Xcox8s_*ipW0>KawQ)5Y5CH_BsM<Egh9mEcg*`I?a0;mUS5Eq2fQgiyjl
zTjhg_T!jjC%jUH}W2LHva_MmLxEY?#-M3kKj`hRJ!Wc=P`B27}eu5s`c7(#{z+>|0
z>km9r)*dX9PLcXWiZ&ch`&4ce)n2=EQG+ll$Xv-EF&$jTt^{0C0)x-^AXL;|&%Pph
z3NXVGoj9Rfo{^&3?V7ujB>TZNz4o(BL2jQe-9rVh9sk~FfnspnA$tL(!!q{T04{7K
z2n3SUcnHsVd4I7;UR5$0@vVn%Mt|Na@4ZwxjW3bMw3p>#>YE4nhSYox`ajs}yV+|c
zHTrK=p1!ky=Dm<EPw|xmx4lJqTpX{+Xw1FKS>pCI0OuQTE)HZW1S9Gn+fP=8TNQT5
zauWuKdkDgFoXIvxP8+R8s~wu}eQH$CRwkx)yr%@ycaqAG3L_<+5@0O7yf}@$m%jE4
zuf~0g-0NW665wgWg^w4bn!Qdvm7&0H^iqjvyx2#60K7488h%=&>8<wSX<);CKsB7Q
z?*v>P1`tmPK|Hrn{WxpQeSjZzGi!Nuvsq2TMlO8SaEEwGaKPi{&eo6AXJ#E#9`AD0
zsHy=;+QDpSWR3)RVl4}*+5e)^%3t(+PaRV!DcX<5LG~a%r3K(nlKx>v@6xBpkL~7V
z6ezoz)AC-98U={tFtr!;t$`&gu%w6bwCnO=+_kQ2cXdk5<S@JecMGkOo!alyS3(o{
zInz!HWrhTGO=@+?g7Fi!yQwsaeW@!Xubp}>4RnOwx&QRh-JU+yi2aSxNa2g41^OKf
zwFzrH5G})V(hT)OlLXyt&=4fZ8(wJTO|LTgB3+meh)!>7W`ob`)7FmL7?7XymdZS%
zU@w5LU`MD;$!(UlHlfu}g!*QQu#u?c%!34f!1)`Wrbj|_>%7pj&BV9UnJxtSp#W+i
zR#n))8}0E%(yzUf49L=5CG2(z1n%NkKTL_K9~>I4`&kQ*iq??BZ|BIX=;cmjxPcl-
z680*c5vrBstULW#+oYgi<My~@y!N#w=hF(;n6Ygy&)8aOg~FsPpR(%)3k^|`B%0=Z
zt{v$q6hCYG&Wm_$)^640E0~ox6WUbYs%deo!$_YVuOO5Pv$NnON#5(Z6r0xKRTYkP
z2NSlzr_HA#b0QER9-nSi&A<;~^rc7DqL?D|NTD8?;~-W1O9zj}(J#7>w@y)wGpoL*
zhbU^-vRWB#9>;0nRo=*p%}B`Wy~u#G^eN|Vwq^^>p?@yR`XEB?Rj))mOf^4^u40&}
z7Q0_jy?qC8q1HIZazb~n1L)S3_{eX()ZA|v&LeL6uwIBib%WeHWD^*}opzFoIl=&@
z=Zjz{0+gXuad(pS^qK3srPt9FtMy~`CJS<=YQgwvGc5g#+g9!M)(@Scg2O2V23(O2
zgML4iHiWf2t&kddtP>F;2ig=oHX15&`*Q3Q4Af+6ovZ_f;^9jDJ8cg6-(5tFklK!(
z4Hspx57$)<4I#-x$Y7YKU!65-y57EO%AFO(=H#Ws76;*VS$cP$t3fr)ty8?rNdV^x
zEwbRsU*smj@4isPCAyIwh%_HlijG*bq1Wd$@A~vA8na!Cx}&=YU`31dYwX0+r3dX4
zj-N^2AxAJi`|IF3=!Q#-XJeJhNh=*<YP(bKiQmZS{-}|oVf8+#Sb*?qO7Ld&N-~$*
z*7fNAyK$c|!LU_diEx;TUrX<_56WOY*^t-GXmYhbJ-CzK5cBSJY@uLhcimEq-o)#b
zjSlTP&k`w-uI@?JXv59}EgiSiNzn^T<c@n}x2;1#%!E0)%Sj+h9DM>}!5R0dIFS6j
z!opARb7BR*;4Q%=f4&^(SjZ*dn>S@pP#_><ztzrf_@WuFxf>#_&i0yOi<~Mw_x>bB
zebz{qaaunk!uKvl&&uY&^~tvCR5Sg-eyF5P$-6|CUV*krBcH7b;{=u$#MFH{x^SxW
z`=>Q5@x+?jCVbI=W8yfHe1OQwAe}P)nTzO%vrN9PN&wg+wZa-ns7Vw{5rvZc02?J;
zeEoyXWs)$B@$eiRYQ-lzCD}I7w-~67{n|yMV;s>fHJ8@cyq{zZcI34y--1suKIrp%
zmow6{r>CMk@+Ls#-X*pQz^(tPN0hA-6Jl%J{lcQp%1!4fwV9C=ufhMo|7gxwYyUH+
zY1j95)>QI0_b~EmZY=TZ7k3!;tvb_Fb}~X4n$4)(qGI{8`c@C{bvNJnAkD0W7p1|q
zV|+21{qLq2PlVl{zIWV(9{>wS5gOBDIYI5U_<`7s`nn@?p@g=o)Zh%i>*|jb=fX&-
zC+X9X`C4l<&`TrhW75VPM*nGlvh@6-8WwjK{p3@(J)Q0N)upP$y)0*=H}L6-)=P@z
z8^7~suMAUiW6rC^3KLy;C+*7G@`?-g<GR*9`i^H5XUzm$ij+QIs>(9oC)I46RMs5a
z?GvSbT()Qqm)U`D<Rcr_qgR*W>l-Dw)@DW->-RYEQZE_Bu!+S|Sa5DqEgD6?Eat$u
zkr5quxxJdr-nXFfK|tQ1Kp;<YX!KQ_Z=s9zaMJgWS~&OhwT<|vJ6ujOvep@lH2lYk
zpJ%!8$67OqHIT+4QsOX&wh{xoT`pQ~yGwq*$4&Fz0LO5RGR0wrf@j(}t!5Oqru;O_
zZ%ek($jkKu+Oz!K2bp=i@(hjE+7-Xs19^{(UQCmDj4q}Q>ke3#=r+XK`mR0k<Mp-A
zRw~^<;97=Z;iv+fY5HAZpw^)dH=Z@Up8H~3j4i(`fmCQlvMNA7wae14XSw(A{#4Db
za$STJ1OV;(f&oW!Z;UO@uR!}MWE*eY0-0$Gk&eom)?huHWEmlY?;oNS#RUW&0ctP=
zK>Iopn_i4&5YHFW*<dqAo&x_itxMDp%aT&TQ({?RK{-13ZdG?$X=AQ?_k9cf>|x~S
z7Kc(*Tlp1ssb)|p9Hxp~w@uu3jduOS5(AlbE;mVRVblj;Hdc@cz>u_ogvGIf9^sew
zm(;aX7h32a4t}ZM(!kzIS;=<`YpbZ?@GvNJ(#iHY+-|OUPbK8et*H^EUAHOti9s%I
z2G43BL+)brP{}BZv>RtCH79xGL_b%4NwSCZQgQT~9&-naze+(DJAE$@Z1d=VV~(Ir
z-F0l-yANj-e!8l;9{zBauP<-y<;|OQ9l0yTw!dprC{V3iHnm&9tt_G`yDR9n4s!kK
z1FL&;0F|sB5QG}Non~A<GC68MH*mOQ;4gI{{7<Z-lULSxkcvh9W=U(_i79x49aaaY
zo=JO;LO6alJ}Jaz0$_HA2B1v&V*!4k`dBR`>|l5D?t{!6<h^uwzpv(FJ?j>EAI*2V
zHjk}HdfYjFyvoqDpD1VE1aL*s16yked8&#}%Qmm;qQQBs4z82ZRCL^(9g;Q}Z*@Mm
zxYYYowxIl>LW2}He*5j8)yLjYk+_0SgE%6+BV(O-d9u7e3i<d(JK5CgGQJpL(tl-f
zTMdBm{FLh?0(i7XKh%yXbMK|0wR$bV&%lre#RWqe97l?DsQ5#)aD2{2RRv7m%~qx{
z*ON<AIU#3YSQh-{n(jhLGD}_s4`fJhe<Wu>Ck1YudeDd2_LeF#>{gXxlZ`hpN<r-I
z1ve%yRD})g-kqELl)ZLuIl9%#O797QQDlfsZqM)nd7JLc_T-r=idh4X6J~3?oJY@R
zq^j_Ypzao{uxmjkxpT30VI)C?U&AyPEP7MGI}z!!JJKNlBuK|M9>6x7zAiSNdT4&x
znuDW2P$u8CRJ>oOS|T!FdpSLSL;_S5MFm!AJ;*@gzn|GXxg7!voYpMRe8h5$l0cLX
znHe0pa3~bI^(#hRs{btb+~A;xj%V?!@O4(_*1+Ct&Zy4AEm93}4}SoYc{VxP+Dhtu
zLDp6cU*eqfL(t<pI=ySh7lCT*?2bX_mQR^{o%?_^5t#Ka&+zM<v7_OPoE6SOlUDPA
z4cDwwZYX%G;9=kvb%W|%=y*>{-QY9wwfYX$*UWY1yARt8@g0bnasQ^$G9tx{!24Pu
z9vE+rqZv>k=()mlo3K*<6KuTQ?el8*gPY%Kvvf{~UX_83SS(oTcszXmFhoFHyOAlz
z7rpM>_NWcFEfZb@FJ?epqek-c2jK}zc|~fx@?wgDKB$qvM74;p11V*;zb~Z$C?3ZO
zAq7la3uErjv@!7ycRE=$kdMBk7&f12IH|piBtdR^Xh~PG*3o-^n!hD=PcKdLS7GEY
zk|Qn9gX>f>wh9axL8-RUghX~I#x52jrwgYZ-o)F^H9ae`N_u75wn22<Wo6W+5)xUs
zAF9{E)WTh<)%Ni3@t5I-XWK#~K4NyhEYnDnGBYPzwqq|Df6EjU6<stOl&m<?Jnu`1
zNJT{}BJI302U!t@H~0H+e{qXfF+HbJETDc@Ii%{suYg~(U;jgIr)7OWx~v630qk3V
z>Me~7mz@fQdZDS>;T5L3gsIxezq!*uu}mP-ZU|$pGvQXoIEKtXo|33>CB2HIS8C%Q
z@N@l%0K0+X-V1I#@=eT#((HsvLQ<^={-brKhd-1>kLTx&ShH9iH0-j!mAf`7uU3MR
zKM{(2VF>Af4yF28KQf}Vem$=cLZ~n%`7b;07>pKLOiV#z;JaW#rUR8lrl*{@f5>^!
zPmr-gHFtDQAh#aA14`t+wvq+RD9nX5pGuZIhcAFHbm;!7?KQ#B$nkN_hdyRfRznTS
zteo_3iy8`yhr#ndfMh>2%U4;cEUne_){8hneay!e#`hxm{IYWh@xM!N_BIe^<*xoS
z8y(Y42x4T3XDX#X<{uoDa4z*1foLXzKJGk9zo#^sgnHwhnEeGdhv)?KzXAHAJhQAO
zUctf*kUH2Dbg5AeGt*noBRSY!k9a7ENoMIr!CiIrxD0(0SkiaF7jb)iSpm}LW%7R+
z4uBoOStMg>X8~BB{4w1RJ!=L@bx>D)y&c<KvoZYW#hoL|L8`N|b%f64=Ib>=9Ppdp
z$!alo0`9>zAHGc)d{j)GOiBJr2e~rQ&St&d9l&7XN-Hgd_0NBcd2f^j#6Y!<EEE3b
zx#dXF75DfOt(;OSk2A;sIw_JMF?W^1<h`FpB?A0QIOm*e6R~6o3IlPlMyMD+`fHw{
zcbRP0K1+uR8pp3}kSK4<2D^C5o7XYi?uvuco>-pphY$v6=Xh$DRJB~P{bBFKXzt+D
zM|UW}@54F3maG#jvgQPPg9_WjUp5UScIC5>h4(qEJWk%s6SO^U3jq6p{&1E=IcZ{+
zTLB+hM*a7ShFQUSOtN4C&(V!Td}`hI>}w#fYsO2bis)LCsfx(}P!w2{g<bV%9p|q!
z)3cP2N<<MFzk@Qw<J7yGGS>!&%;u0U?;z$AY76`w3TDi#d$a}%Oww4K1eJjuJhRfG
z|I)1^1Vgb+{rE<TKN%sysbkO&NE%IpCcT$_%YNCvc+1t?yJi~`@tCH=XuNBz4`4^0
zy?o$jpxb{3Un<?yNOY(QctMWVDo}tHI)v#Jc~;fFztLxi!C(c;QPi655mEHyW#DdR
zp&<-?UH>inEvaANTsE;OqU+?!1BPpzDT$LIh~_^<s6iT5Jp`++l~r0KLT+)6x89M%
zkea^EXq`oat^hqym0Xj16wAy|6sEBMEd%ditH80}q+iT&uM;q^aW7jwTU!Qce=hUd
zxkt5ZBrZ(Ozi$!IaN;!F>b_#%fM1uO>3BK$#HOjPlcH90#9ks|fEX!vqFS~QSnpQT
zU^nL427|kRw*F31g7Jq6{Yh9$#|?HB;k<hpXv}AT0{3Lis<QCTx5#>QX=+}SCX<z@
z^0x0QV8H5_6qAN|k+z#~SM6ZUwnx8A*eowF67Q!GK({gO*X5y|UP5{!K2c&fUYlW#
zQ0$%i=>19EmD^~&D;7daFOw2+$K~_9)D;i^ze*F-a8EXvJXq@5)=1i{@ewb?<Qtlf
z1sC-?MeWogR<G8&pfC=!9e!s(Lr+=5Hw7KjdkP9rQNfVm{I1Ptz=1rUWohGpJHZ23
z1({`YNyZf49CZ1w41pic$!zD+9VxDj+!Vcgoe7#!Z+MnHuh+)iwhuS_%e@VgqC>GE
z;!TYBN3Vh?z?00fb3~r%ZC_|8rR1!bi!`?(!oi?B*wlPJS7jlDlV4Q&RsVRw6bWnu
z){B=djJTd2aZn<3C@_>j(1Rd4jZY}>@f@>Si~PK)e?7gQaZ7|)`!F$@eea6>t26g|
zH4l<<`CA@gf=#^|@V@2x1&3B$4*Djx8z=*P6DYml0*$*;ANjd90Hv}GeJ16S6?=<V
zZG-p-$vt=rV-6>$%J0e+3cNZV6TY^3fIlMyL(rm8G2i;v%*=K|wGYh}<Dvu9pljx+
zQu+|RlvKD!!8}z6hugv17QJ+xb;aKu0HE2q%?+Ll=fp=;)aAYeievmVzsKPth!gE|
z!<ENst#tIb{EF&~${MdKV0tJVR5w$86Q^eH#jYoX!I(N&vY3iA?Fwmt(woSRJ<&T6
z%1D!rit=oxDuBBGz?c~;#s@z96W;M>q<^_l*=Io(EW1~{;xx~;!D21T(MwW^_^e$G
z2UNuO<we8{6nrtxkckYe`aY8fV7`8MBeO2+Z|d?P8NdFLj~q~UR4`8?#fHaRkLTN?
zhT`4+IGs%#J8+6792t2>VBW=raY{I4dZ7MTqb~EP?By#l-5WogqcXaOwqBjm$_awr
z<cb;Zd}TZ>m4q@Gsyxm$_ApeY4jR=Q`hZV;Ajaa1?mFZ&HhUM?r*bIz?#-&uQsjLX
zEo*<;6XjHP&0d~8jN&8QQYB;K%A^;hiAWBfVI3z~(nMQxC<cG)D7Zxw^_At6;eeo`
z2|s#!%}snkDu)9c5{ADhjX=KzD7=i^KxEvS>A131f!v^rtYk^OnsZ)D$AeL&R_t#i
zSJ>c+<CBu8i@?HpPWD9?&sTkJYB}R04uo@^E2Phm;mQ)_$bbs$wMEJ+h|Dy*ar>l@
zp$cxA92YYz>MF4SdIp}3R=!gBM+UJli-8;gBA{+fr^#{wlEN=jLeuMIspekh_LPiy
z8}`KdVN}uMm6u}V0N<t<941K&oH#Xt8g27A=620;k)@nkvyk)IMilhj)Hjza=hctn
zn#8F4`3%S!5#vxruQh^&=xKLveIK-BLS!6?Vc;eazdZ9JeW0}M6^RShuPS`1F5m(R
z^yW7q^|CWV#^q<XFJf~Uzj*;g^G5)u@fsDcLW@hpWYjd;wORBf^=!(s0u;nrUWhO;
zW=57=+5OvEpw|0K)W?Tc)Mv_vodzYKByaecg?-O@HHIBfTh>*nehY>dcwK0dreGfO
z_OX+EMDqRX=Jq|`zeFVB0jWKp1i$=?A_%y{@aTt()2FoS<UFLt_C4ezAF+YMk2Emg
zE~XNI9Ldma8vVZKM$@PM9qRV{>Fg2UlCa4>sEg510e2b{-{fm3-=E~QHKv~NRm41C
zg&`|)8#zj#?Mh{}MwjiRj}0`>bjJ{`4wJ1hNPW|cetah^tb6)b{}(_R89SkmDzniJ
z-bL24leEoeQ30#(MADFjUUl=5A+UM58=qK;yGORR=3EZT8mD(+!jBv#q)*Rw(GBi9
z{*x10j3UOi1hk}6qBRTg6(J|}4c{B$xx2T$w<u1%M$%Rlge`&qMck_XMp(a_RKLH}
zH&XE2V6tK*%YC+YYLjr%wfI?)XJ-sjH-XgBYy3U-LHmm6xC=E@Hf%@jTgFYhJUWAd
z)!1@6i;f^|Juaalm#*HO^=wFmV?eU$?e(G3t=fu`uO3q^uMK}pA2co=m)qdV-8X(s
z`MGEpxL`84%wFOc5N}HTF3tod%-QAOx$Ux1DyL3S_bj$t;A_tWw{k+M+K&wUqxiTg
z76$tLoN=coBO~7<HWELUB?_C0qwBeM-mJNw<{M4!_xW2C#t&6t>z&R-WB?a%q`00T
z_fBUnNE3AuEcSytAXAfLGlbiwB7&9WFI(Q$^Ljg@BmI@cQ{8N)#vTa!6rZovjBX0M
z1>M@LkE=i3=Ww5UM)&g2dn3mq%++pq%s)}E@4<tE`XryR2=}H;j<NWcI6kO}^vx`?
zhT>6Cy+-!~9>z_J<7xr-i(qOzuf?_F+*3!()59&BEnV(%xk0kA0Hd=O2$ks9%FQ1?
zd-;uhCNsRA2|ZVVPI`3o@*c#_Yz1)mf$L%~%`2M31y6x%Q5F>vCeRCf^UL6t(Gkj+
zv1cEAcnQAi&F9<3^%!XEt@b7G<x%*e_9pX^Dy%f1Y$;`1^D?3(%r2y--IbHP2@?Wu
zwyg{%cq`&|V<*Ds7$?ph!z)^3Tf0sTZ%%)|lz0f-{{Y24ya*Yq^~k0fL<J<-?r$tk
zW<EQabfIGy%Zv=aRr91s&;gc^8c;sHWXLKyDp+A{Qk7I$b<S(~!Qgx^yO|%_XFNkC
zIH|<A^}y8<;c<Cg3+diqkbCI8#0`t?NPJ0YJ5&0BCr2;8Rj%Alzi(&P8D8bpRt7z+
zbl=@YOfqS**ZW*HG@?x37#x>+*%n;x+$%8kJ3Du6(@1DEGspONeXPK_?Y3OSdXc7`
zRDmBViW#fm@sk$4Ji6<c93!)u(tgeH-q@20D}9fzO3i(Mwo8lFb@G!CmQ#~_Wo1KS
zm?N|k?@7n~wo`=Pw>w%X*CM&L?^)d57%mOF9GY=}SQ84(ufq!522b2a<APsGhg!;<
zSyAcoRwija+?UVF-f4g_<Z)hwd6xLOFL!v@+uRNGCgGa41raPS>jkUu+pQXMF%i@O
zqHNzzQ@V{<AnnL9vhj!Kwb%cttWUeIG_89~?N-~7HAuK>FRB=4NhVV2Ki+bE`hhK9
z@)w^?hb732V)WRv>vj%?s(u^YMlb#9_g#J*B#eKkwYP>@Q}%sx8Vuf=9^UD!Sy{5^
z3LB?+k*T~RBBfg9$(!i3-plE3uhjTP_-0TzZm)prb-ns5<MgfD>m$=v364iAb3%Ee
z`9GXpoC`w8sT&gcyOWPsJMNwu?u;?g`}?nd#kjYX)Z&JdO`mFJYbARHcMgDU?dlrR
zg@rYjeKIB6&)z=;CeEB5i3;-gS!a)qP45#tWPekJ@|&49gyn`k=+xu&aTw|H&GJr~
z@Y3SQUR(-T$u~-`y%sr5yA}}Q-B!wwE7L*RJP93l6cTcJft)YU<+C?7f+lO8aY2dR
z60a7nG$i8KTC}G;I${Rk>^=UevO`q#jzyBAr+-60`CN)-l(%^B*9{n@1>?+#@)HD&
zd%$J#m`1(nMbbj9)A>Z6@kP;`=HOQU?g0V0()8K9rO&f)kfGO-54{%0&pw3_?iK?r
zd00eXqz;BXbl@ZG9*3GoYx1OfIR`^8z=@lvXM0%J#*D?pCggKIspj<&?}?$%t+ud#
zkGecRWb5I6JkDCB)%ZmAh8@jp6o^{oOkH)zx1{;b*=M34U^{yB6T?p!Nt~IU_V5PH
z@UZVr^L5~I^%2gP@6N-QFZ^n8O`l|+HH@{N^YU3WzneHxlk1p1viNaYiHb1Te0m#u
z*Xc|~e?JdXe^(R#k`CO1HcBUtMRWM<mpA+Imie)J_mXmOkgqQn#_QSDV6+p)f(zBi
zIw=dStB-3MpQIzbVA9;jMlsXy2}1+QlFnNBH0ze3W77W5<f!SH;%J^4!pIw~HxhGK
z!oZPjb_>?dxZ8uzBVdU+#`a1N-_JA7hCjGCpPQjb=Cz(~au6_w3T+)c=k=Y!9Wmgh
zn|h<s#^4CvM>`ynu?uL=81%eDMQf+N5Iv0fN9{EM_mE@~rRLd4IHD3;Wa4^ZLbQNA
zBEv<cahq3Vfw_KTAuy>Xo^U!s30QIa>sA&u9y3082aeYrsBeu$Y;*ZdtNC0+g_18$
zK?C9($Q6~>M~bYUh+8AK;0$bqhI+<a<wo3*g+dkY8j?CT?^+=T>ouI@-ILs{dPnp)
zp=$c+R#fq7s_)A|XonX@qJuCIOlaoerRj^}y8d+C>eeVI?d^ty5uW7J!w#eCj_STe
z)6xDr4U?o$-l@`O1ce0k46D)k-w)-Tq=n^jfqO{IOVHkC#L%n_qF-dE@tNIP*Kl)d
zf>3$Zc=I_QvKY&w;lQn!vcE8UJJb9Feow4Yje-Yd&En^aHq?3Fj}Pu0_y5`IBp@sf
zStk$J3C6giC}wm~4jYZx>;b+RH8SqSdeBLAQ^$4O!j$CEF{5SBipxyG%!Gh0ZzpRd
z<W^Iys;my<i`w2XQ<IS-FHdEDcl#dG7WK#SA02QmA>QXAq;(#4bqp~lPrVzK%d$JU
zNGC{8mnm^e|JYB@N0Vn&C@L)ZIS5&W+=Vuj$~8XN+H|~b@96K5O3$h_IfHs&_>&73
zhl-}5yNFc_+CNWVW|Ll~uA97hnCV*=g@i2!+%D3u#}qWfivl^Is`r?0IdHfaTNIoY
z>~1Lsw8pH(eCQv_5U0{Bx6|C~cuJkXQ$TXog2wPsWKHPWbOuywWWS?W)*R`JqK3rv
z`RVZIPWcDk2`Ed;HgU5F`?xQIWW+Yo+cm?pwSA3VAOdcNc$Ro}TNKvm>uI+izG+-O
z{j^^bn_h>C_KYqo@I6vyR>-mv`g2``M<-#RwvtsoI@iqm!O~QKU&VzDy?LhCKS}#Z
za8t0xQ}s(?0Hh5W1s5=j+XYKmGj5p*cITU&0Cnev%xus~dT1bLd}mqjXSyDP@}JVS
z{r^upSN;zL`o~v^a>P0+VM$+#+30Yk9Bt7mXYOm*$`q5vk$W7IP1`uy%@yNJhlnA^
zgfZ$RIYtaJXpnm_!_Z(1Gh?4A_O*Y)_lNyuUa$G#^UP;H&+~kq_vd;CGcFxDBzXh{
zN$uJ|OH8CH4SsT01{Tkvn(YPchiN;iZo;O#v4iFpZ~Q?1dH+a~NyXcch}9d-<Yu=p
zseJ-uvKTQfUjblsaCCek#*XUlkW(3Lj6=%hh{|;EM823NdAOxid>pGcO5KejiQ06F
zaaEth?!I32cGlob<Kef5<)YGOqXJnk*LWUdaQGvBO+jN1DHy7%$tQQv7pby?W<Z4A
zpRj(7ZZx^$_faBs1%h6IKWg4|o&-^X93QvGd)QbBqqL>oSbDEyPD}R`bqu<G0*<M&
zFO~@w@#V952pvR#BE>;bI#!{^!QE~myH(j`htrG+VF&B&b;Um?Ks|IFsAsVWZ<SO2
z^Y)GvhtYbOXiOX0Y0Vl}NgM*)Q>89?LDng!z`S-Ce?<O;8XuH<3-StcjIONJWj<Y&
z0>)pigQo67rHYojo3?8S>}m}`S=6Pu*Lf1lhc6{}nS~kI{<d-J_lxn5lx+yWW>XuB
z8e1(!Km#4n&EhDXjN3{Bf*>eW3luz%O8;U<`4@cOVkbd;1!C!prjo5H6$pF!04aux
zrT$1fSmImDI{(*Xx8|R}R}oF}v%u(f$(|O<)^`vEweJUF1PSa`a|jT&=W`+GjD^W>
zuM=y+24Y0kn8Q{z5^R_+Fvlx31;qL9Rag*%9Q6cZY$H6{<n!)NFVk*V%6~G)d`)*C
z4P`5wDe}~#4b%m=)R!Bc7c{8ym2s!xhqA>^AGX>lAgC+T`o({zF&MC3-#tm48c^Tx
z`-USubbYRd*VgL7-<{m>?~tiqVt4s1eeIhKI!U|a3b5I210RWv{_{$MsizZq%Q%_#
zqc^k+NFpLCt1oC!NPsJVzUB8j2m43ZbNlP*OMj_COWGx(>TuQa)<eT18`FAjHG!56
zBkcrm?A5?O?4A6@)&gjAbMOsx(VHRL@xqAHeLH3aj=_c1`i&53#Rn_z59ezw+94uC
zn{Z2@Jxgk)&fG3?ZHP-x3FxJNE$G8uCl^WL!5DUr88F9ZcMTD=iMaq`1x~@E=0n>=
zRb6>TdB=&l94~xdS##^-M^`!acA(igdKu59WAb1WwgmF#U%y%gO>%)&vp}m;T#=w?
zIrVa8`uwf8<ar_;s%bCoH8pp4Jc!ZTki_MV8j%aHU4|S@9Do-d`4$+!MMNymtHky1
zRV=U7DR!c$Szr*Gk+Tqa?nGjsC$ob{#HD<29tk2VpW!C|wu;+{!|@D1Q|797t+tjk
z<_kgz5KRiqFKfU%W&uG7_VE)<FB8*fYu1{3KxK{&#>FjAn0p!UoskF_BbuAwu}1nM
z@Sze$xC^dHdhWx`f5axfG7TYnG{tq<YWJ5UQ<~?<ojB&g=JC3=)!E8SF0ClX?@K}+
zPPU|9h$1mmi^;)%sd=ZPXdIlI#AUO_xm<H^ruvdvG&pHCd=IiRRp7rnRkJ+pLu>f@
z9Zq6T=pTcwjaCE?gQty#dROmytPlK3A-Ks$6)=2HO>2p3ATQ3>JQE)*yq~kyh?}Qo
zCcI+o!#Eo@wAh7X9a6CmtLfyj%;A#u>3J;DkA+_<ReX6Zd}?CDllqw|4Re*Yghp6R
z{1B7VpkMUVBDdl#&)|)dKV1PU00}qBv6pL@uWFdbolc<7y)gb3-lCJ0@R^%^&iMeG
z)%1tt(DN=XSJx&u4yfZAq&e>-F#)3D6Gcw-j&!xCcwd)|^rW9dhcDzsh0eUHz|~Vg
zj7h5IDHlCnrykdJP|F+OhJzHx7`U5qJotq@KMwqwAC(Y4SCc3x9itl&5GPJ_Dy@#w
zjCmPG+k)N{P2V3^(wjT|FhVKuYF*9;ELson<`K;s=`2@Mw%MuL@dws%OGqA5yesPV
zMS9J>6-u#(P4}|M?cVs{`Joc1`U0<U{iXH1bv;5h+Y@waacqxyZH8a9r#}zuDtVvG
zvGEhYPI~e(1jscmJLL_K+3aAfl>s+)WVI<ZJf!=u(*8y+3fCf(f<o&p3JM6Gw2D3F
zE*ENky5N@Nnq_50Lo@b#32=~;$6`lZXNNO_eW@-J>!z0vH7vFX!izMSEUfWoUZKc6
zT$mZyB~z9<<jV{V-e;&}ITCCH=e-{HevJ1HnPVg*Kr*|zmm-k>bzi0^RNYDsfj(R4
z_@E(*WDyxPv|8(89gg${6Y3`1q+Fc!0Gmu|h^ibn)-PU<UgZ-W$%1G(3ahBvbrq%b
zyhFS*ngvzVo({0BmEx(!bmy&8{sGFB8ZXzivec|Ry&^pptLE#kWhm#eS*Q>y+*5c$
zn;2R81X;+=U^%-Dj}sK^g0GxS_RN`p(IRu#{qfag{g0K}<WMwEKWwVEE$#WDH|{J<
z%NaV$DnAS+isFHH=I35a)At~Cd}@*u^oBbPiXKKSgE7@T2P@i+tZ2iXFUsCkUCn5S
z`0QobF+hDnMfdkD-zFW?^@DQKpSd=XS~|ns{U@zL{2Ld}nf8VrrB%02x*=mv=B_yt
zjTb<tR?izpnoh*mdsIlucmSte;eHfZbZGsn6WJ^;2W~a+5!T^WZ0ru(rKRanp?2V5
zeYEu{&oJXG$5Qq3)1pzcrLb<xcN6jbbx)9&@R>>=#dS<@#=ts(c;V2G1J0v8Ek-q_
zM5jSWhN3T3p)M|*Hl2*YRZ9!4ROywS?dUYT)iP=+P9uxd4HjPR79-ISh~Y%FJVksc
z!|L6`Y9><~R{4OP=w&_C;Al%YCnU>ZH2)?2@knfT%zLYIa2G~d>HKOhi<}6{6K^Cu
zx#Ft3q#y0WeqD-0UH*wXV8%T@ykHQN&qfR6f=mx2EV|<-Hy5T^a9mff@!nZGmBWo>
z;aQ2tC<=m~EvekOEc`+4AWjaWkz!5f@yV|i`QBwy=gIFNwKQ6qChWd`m%aZDPGn^B
z;XBu8SPat}jmI#RSOg_J!`v_r7j`d+!;q8@CQ6=hmPRd*#uvKLFBo?UTu(GM(84z6
zlCSAR>VM&#y2E{nc?~VGAUxZ57FKAzx7JxUqbX!?$f%Zm;8>9M+uJ>gNyw{LOx1sC
z!Ok4O6Dnf09!?SExvw0Tg{$bPxNKu=00HaU_KFxUMhaGtvY4*32@OFj=Fyr?#78kp
zh9l;nx}3T~KP_$mlU08|W8lR}_APgn8UQIC?`lV!+#S>gsAc3-NFcL2{+b@#6j6OE
zjoi6hed6-vk)A4KUzQXN{$YX2@T{yM;W0r?_9JlFsruib-ry}m7oRFT4Q*isT1Kx;
z0*;x){u_lqGs6(r5_1?&*up2TV5S1!hd+MB`K-N4o#AT5N%s4NyhJL%Isco;J}tCk
zNzyA3c2=fsCrWbn+FH%m9*!!67&+J;8N+wKh+5&K@Ct)8i<n4!vXY%luA`mus+xee
z_ZURNJ{9bZY51Z6pguVtTgLQv7runrhBVi+!JSF0f0_+)w%;+1<W`a*bqBwA#0Mj-
z{CQx$+c~t2R+JUMGujlOcy!mc)Sm(Thjrmm;4j<w$wZ%@X`L{z5&1`Ig5OU*q&oHc
zHdsRW15_#25}YryEw!!JcU`{qasT)6pK<;>7XLRF!sMnPPl<gz+Y)KZy2_`HHaE4p
KQ1*-0-M;}*C#gyR

diff --git a/site/img/copy2.png b/site/img/copy2.png
deleted file mode 100644
index 7869daddefe9f5476f678395e3aae91442ffddb6..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 37973
zcmeFZWmuG7*Ec-DfRfT364Kp6D-0nG5>kV7*HBV3AfPamG)SW$BGLvmh;)Z^41$C-
zNJ%~8|GMt`x$gHk-f!=R_ruMXnRCyMbFIDCZ>_z87y7#DB!mov004kQQ$y7d005-`
z06_FDd|XRp{5d7=1L$X{ejiXZ#I%Wfxb3ZB?gs!6;eH^Z1Ojqi(cwCLcQH0YnCWOk
z?YumMZ0)_CI0yxKc;k8l0P;al+*c0=ge_Z;hr6dAG)RH{pB_-$_rJ};>}>yZLAWWf
zo9XDYJ@E2%V3QP*5E5ZmBxGY_llQfEgc_>C{yiM`q`>ZsKzKuig#!Zvg#yKeynLO6
zMP+4Wg+;`K#l!@0Jp}!NJrTA+f}VaH|C;2#=23O<v-5TFM!0x+vi+Uc_KBB2LV=zA
z??V6m_pkRvxH$gbl|23aO%{$o;lIBKiwcPd|M%Ruq4IxQp%1(~ynP+~{BZLtO3MG!
z^MADc-*x`gU*Fja;f3RZuZx|gC&IxOHyC02_w5wL{yqBty~qD^E<Ilt2OQP^j28X(
z=>Kf{cYk@|zl8q>BmTwZe_C;DRwR@c{_ibQB&=Q$TL1u*0h+2R#zDaCJc1yqM~u;4
z6ge8|PrV4s;b7G}Dk>1_5L4Bf_!jsSs~im#QGh{pNn{*}ni@m}s6t+<nVM(}d1im@
z7o$Dx-ncig?djXP9$aVSnJb=~=hYgrhZYs%WrLI=sggm4%7hxg|9$vh9{le)_+MH0
z{~I3ka6)Ru-qyIjqw0}>BnEjjrMq|tRqb3==xx^jX&DdX2y1oOJXz3Sb(o;wy<?8r
z?rCZ;{hxS7^+;LQ-aw~b?LV8@;y}LaCAUf!{s;Q0Q72QH8&i6u{O3V<UW|Oz!vk{p
z_@BH{Hh`?E(tKC7_#Y5j%)`bl7N<Nb_@BtCZw!=8QIi~{`A>{R@euaBK(i+YJp51S
z{r~b+m#KS(#NLFY3K)HSawair<V%G;EwX+6wYz^${R-*Pz7=icrbDQK(l#tKz9(to
zkq$B}EC-d<_lae)q#0G)bx8FKEZGlq`yCF@5(%U_QE&T(6;XLU@Y;Q4$C91wmf>+&
z%|2yO1p-rl+Qqt$zw4~Zx}Wc`9Li7!&vSs?*k*6?+sS=gZ|s#?&G_IusMjB`wtQ;W
zvuas~u-iDs1UQ@HRv$3LZH{A}aDxfqvq#typU;jQOg%B_b=ntC&DZ`U&b;xV?w6P2
z5_V_VNd_f?<^s&tCVk}E_|ZIKUqaz|0!JE2?sF-jzsfe2N|sD59Df})gJMT{Eede^
zN~QOAkA{ybO88KOmo<}OTa64y7YzNOQcGf|%=sR=_tOr)Z+5xQ7=R$7ce`H$I3`i#
z-7frSdhsc-*ok2=aMRD}l(kbQ^+XM^KO3rSD;1V6UL&b`G;s4euDTeNyjDg)!Yy61
zNsnuFju+78aaFxxu5^CZbME8Kk24lJuf7*#1c$rk&!7NC3aWV<JoCpoRhv<}Jy&i{
z**nwM*Ol_(zLP@TMT%oqGm3Nd6)6r<7~88`R%@sjK!$9MLxyavxfFYGSLL7XtjvgB
z@fF$~fvAg_7>-<rY(j@=E7yVQ&EU06>ZV_}1oZmfn!-V?>MPd+;#VibQYF_t4Q+B{
z%qbp<c}_Fy`xBEhZ|<Ah4bf%0kt(kq@`Wl1F-Dc2MZEQDdi>WE_=r1^zfEk?g&>A+
zl+{&}rEX$tJaQ9wV{d+6@A!Ey4~y$*_lduEbEHp<vN+bu*x_7*%UE(CqL#qwzA~@A
zNJsRGE95%O*$J*+=eenu^L^c9924)pY#7ZVXd4!fmokeXedh6-)RuW9A>6sc{b;OS
zKFuRL)xsgyLe44ovzy<bZ+(eryuf(K2LVLxS`HT@+QovuKW$F(m7mkyKe+*f1DLFy
zj$C`jMOA^W^AFaF{jZ+$Nbe}jJG=Kwp6C#?IW_RvXIEbCJ(y*F&cKPI=^c(H#3Nnm
zdl17i%yJgwf~;66y^ntUZFr;0=U7p)%VgZ<HWtfW7^=oHS;T3Is7o~rp!@@&DMKe^
z-0Rr|nR{r$RwR$Z@S3@!k}B8Cpyv7q!x@3`n^@+PQlhoswRL&F?R;;}Bk&8QYlm#s
zo7qQXdA<3Ae(p{_=Z^gn<*MD2S!!E>0X5=%Tn{54lkPJ!T{%ro?Jwqohbn%A1RQjK
zj#(1S#$gMW5e_G~j+(d1<BLG*MMSc+Q>P>rg5v95{qle8^3BoCHie>w^`^%Hb?U)(
zeXDGQbiav^pQ#~TyW5gCUwPH=UQSlJ(m00gy=rLkdwA9+X%Jq*C%0uQOFg=vc@Y|(
zzd0(=jg}GZM%%b&3)8M<XTJ;F9bfwWs8BaL&tqjU*3QVg?k>TY!Gi+BLVLbDh}*o`
z>>K5;m3{Q#n9R>7-*WW3K@p>ti$ZfuXrWyj@<*#H7C&};1QI-*py?^-p)Q>~U+w5I
zIc}HAh1&j2c&idQN<PrIYIBUI9|6&0vg$);r8p(Uqm8RhUYh`iPS#6GOyC-k+JBK5
zi^I+_gZsFp>G(0dLRJip*+)tP4Na-R<&u3)x%Zw6b-9L~*njNm8oBH6E!E_nv;4~Y
zpnF)G=O$M1o`~+D`EK*s^Ygs`JH?JtV&4NGeoC)^OiJ{)vtSsN48sG4+3$l=QTqb=
ze|c@lrjKJb$`yw}R7Q}12lmM^P8xGv=JjZ;s&CtyW;;X(g@TESw9jW4_oJZ!dF?`x
zH^PpO*AHhz&Rco5e4Zq|7BxQn-9i+Y%}S`ALAvpbD`WW@KPa_VUVh|p##6|dyM`{J
zSa$e@dy;5ftVEAkiQ`K9agcl3KeXN{$Eg%$$$9}u+*=UqV9lKu5^Fm~zQi0TBhh>h
znJ+0J64BhWC4(1pS?x+|Iq{^JBW{Z|%(?iQPo>S0kEzU|gLWzX2=82TplhOn-KmF6
zF6FWa{w!Bd#KW?@#6E5LS5HTtSsC<W5RXqxQMIpY2TIg;<ug%Um7i_ht&7&x|IG1y
zzXvH*hVmz~Drv0WtMD*n^P`se(XbP(P)^0FMLkfy0OEd`WUtvJbC-L*lrSLB6sE9s
zz|`yg!E&CN{GslrbK%~AZ+^r!$GmwaZGku?{9L{NM~2iwD`TgQ4YS_CBjuEyA0eLp
z_J)PeL5S`7YHw1`e#gM$rSqg`w@s&%yzL$SrF@fT2)`|6rq6W0o%|o#@Dq+p8nm6p
z2LGX>d3bRWEx_yaxSRJsbSy1SZ*LZ7@oD{sisw@MD^5SPw`6dT{a?2nY%xylTTvOQ
zY5rfeTUd%ad9|glT$s!LyB{0G(pXuyH2z5i&wps`G%6hMXQYY0Ur7VG6k<+v+y-OL
zQ^yOrI!kT`&wb2JPhE;Xhf4{a_3ktE{9_n+@+6yl0Q#*U4Gm(fZ88YIKbU$ul^H$B
z)%$)crR_FmuC`<rqTvHNqp`$;MoD67YR$}u^YO?4tTv&~XXBC{eJ!%YuL80n`&Bqs
z9@cmPO;Px8##{9dYQ!iIctoH;RWn7pix}j|xl4Zcx`c>CghD=9>&o+`8tMakzA6`g
z9nAvqcWzp%enlGg_F?s=glP!K#_x238kyvYUxLcr2Oj%EXP^l{;%M*;D+00tWF|1*
zCY*qc5wsV;Sx3OCer${!Bn`WSC*XEc`!M$qf>+=t{%k1|68J<YJMo(8k98F5++NUW
zslQ@HU#c8{-|$EWXG>rrAg)Dy=|096!efa5+9){;8JxNtz+@2vfhwUxb?}{l=Jt@s
zaA}UPI^|$*fDG*=K0AHG6}`TjtL_f?J(CFxrQSY`VHOpThUG*9H^aMqw~)s{G-Yg}
z9JzlluVNaw4W>ubP8jnhgCKNjmMl&!!|Yelg;bv?<ROY6L^xAjwFn&(sVBajG6x%f
zC7>yr{(Bx^10Y4P{n8o_w_53&iAKi4$qD)CjjXX3p`1bL#9NgdBEH$$oQLXZT-;*#
zqv@)IrKz|>Grs9wE@;ShMO9d`Z~AKRiK;8hMqWWu;_Cf<-dhOH8rN_LLF8V$u{SgJ
ztmUP9(=vARnI3^K@&LKM>{(5tD(0$A@|H{Tb>oCg+CpRkn(5vT<y!dwtDE-d(2`UO
z24g>|ZwIpEE_pTSKK4e2IQk6V;y}W<;=`BM%76Hk*SQ`~@J|?woZAMzv!jLE3ek}S
z1WK(%cy_OY3z;_{(^rovy1OO2e3XbYKv1kS(gf>*u?3Bya}4BMH3Z-CHjp@Q!&j)W
zGS!mCk#vna(Gn<K&bk47gwRTZWKT6i{R`=mF`WD*Lve@vjc&+y0f?9iD#m&B2Ofi#
z)4?r$UZoGw!|{M7rEZk|DBtcDDGLPsSdqXr>W8QN8d-bVCD`G2vhJ#x@<K7aWU<Kv
zdbUR1m40aUNq0QbUw|x=r!OGi6s8^3ZRR_0k9kRTTh9K;X$uYyQN#u~BQWL_kE9V{
zsRjzaTslQ#6}Eao@rr816RMht{LoHT$a2epqdFzA2Or}Ta08}R_NmR0p@T8Z9N*9t
zQkt{Q-=c{$L+9u-x~je}r*DY2Z>Uc7Kakf>NE7K(gcesyPrvkG?=Gp<ZHKs8;g<aU
z_fqgDpRqxkurvx3Qbf+Ql=b|;N6G0~Pmp+{C`DH<yUv5{^*6`Zd(}$LptZ<;Wttvi
z{^%I-KhlH)K&GT1QD<;@H_e@R1F)-#PC3b6iUnSdZe9}0gBRn|L}J3+kI;8EajORV
zwez^%D_FVn(m9oir}Zy>wXU0}--En-grq%MLk?-NpNNY2<9&Z%@Az@c3L7$V_glEr
z_!EKOK!x@f$do1`74u|Xn0n|PGs2B+S%Yq(>VcJ{??D6d0j|n|<+mG(V54YC&qZKr
z`-4%6?&^Y+p3S>sH6HSds_wVNJOCG*FU^B-XtZyGlb36^Du&m}Z+J%TkXV7Qb*kG3
z)YT7HMb2Yw<?`6y$dahm+EgEDU~Vxnmq%l1^dY4sJBctK;tkb(!5*-cveUlUH1QKj
z7LFwUyVx3|zob8DC`(S3hP={ViP?$2e_u8wzPg2UTCiQK^}b@_>vSB@CIfLa4<J)Z
z?`ff^ct=3TwR4j6jL`$#>O5G=?UM2RTc0HFL!G9i&1CQq1n3|^0NT~pSF_^10E0Iu
zh*AKe9u=aZfDyV3YFHILm<~eAo3Mx{T`}HCvP$}fzBO*JM6TB*;_}c=jw5CK{`PQ(
z6s&4yQFZ&F{jUrS*iYPL4hKf7s((Iwc1;6i7X!LmqI2ZQ^)9;EeQ1>l_+R<BTZMR2
z1OZjG;ujigtipgCPhM4b91pjfvyJl5)pmIx$~UH4@J%A$VWbOX;dlYWJ&+oPxgk4j
z&3;ypyL?wGa_eALw#Qp?10vO#E35tIsydJKpw0GVxa%at_C#%NK!Y={$E+1R+BX)C
zlWchtoNnm`T|5fqx?IiZJ$ToccU=E2*Wu*tvR7Wkpef|5PX6;U@~xDth5T5x-g?l*
zrn?Db3H67K=7Q*jCqVl~tZ=+uKH@XKVoS9l&u<>g9}^&qUUt%3*jpxuv=VYacWfK%
zE|-Mrm%qotm?TPWmw9T3Ea<1;Wz%MXUdk_epAZD-$E&(CiygdSpFQPDsZHk|q~2KM
z?4S6Mww7RgM97QNiOEdaAAOxW1jK)QV>-Hac|D-+w3=1a;;WsTbIqh)AT_i0spU8I
zq54kR#E%%t-k1pd=?2C@x)~^>861v(fN=A5Jen({Ed(Di1~Cl~JOFl6nBc!)@EZLB
zf<8wC?cpN=PVi$R%K#iHIDUSWQUI@^?jk)L=YomlX;_Sk1MY~mk|q+MiNClAgcv>p
zrTVa&Z<iH1Jv(gZSA%$Nz4BO8oLRr6nsu@?FWb$e&&@wz?V+m<Sp}A^n7^peLD1}N
zzx22#P@{dL7nlXK`I!yb67+L*%bTR*GOpbx+^-lT9I4Tq{NYb1qj;#jfA5nNs?U_2
zjKh07Kmq8DUgScOx4ghc6pBATNit-18Nzob;HFa`-q>dbKta&AHLx=?+AaG9N{k^!
zYP9#Y_zX?;pnFJdSJm$%3<TM|)hMdfV|UA`%}zbuTh4BpW$%<Lh8?F-G=RcX@Q-ML
z2h=hZ=8fjnq($h0qr)VOSX*Va*l*{FZ3!df%KnCkC^V~5M-3Q)jDh>0qr(yq=%5w?
zx)gcVZ*yvBfs>Guh)!<idNE|;d<yn9#wP^vLuN8Lkxc0$m_@Dxb;(4ah>vijN1+`N
zG@Hr|_-I3@M28qhq+L5RHl&*5lZU?nmKH0etUl)PE(ydRX--bKb{aE#&-JoZ#BdJp
zGTZrXA2-g-I>pKFZ)>|@*XdgD@EB$|IFFD{qv5l&<Z-nZ{3Da!+$YPwcV53cUvB#@
z+l9+>52MfgGwXVN66Tk1_u$@+wGY?z14l+j1Egnf1Xiv){016qR&&BHoN{$ZY6r;T
z1&{VsyL}cO3D5bcXEGiAjFdmR-nRO=r&aWt*bq6CVPPN;PTz$r!4R>TSj0VZ?`}a_
zb+3b<{x4=DdU`IOt<VL6(SUqnx|(K)Oxp{$n&w-^1oMG1(*P+D3t1kMA+Iz2jZS0T
zrp#;kT@@thDh>8^&3pOAvi5$KM-z~riAjc1{y%Ca@oQuji{cm)66A%^TK%EMn9rWX
ze{9#i!o@9V?+G>xl$;GJh1d|=&WiV^eSJShwgC+B_06BjXgY$XY&YDSxC{Ft6|MEb
z<C-6P3IAhL=c@e}MRwC8t}fLtD$JB=*J|xNqix|iV=6<F9~JQ%mR>gy<_%IG=n?-A
z6>wZT@RPtHSJL8e!0AZ9h;#{wyLJzh9UTr(rXFf<<(RL1iK-i{h^QUtG0(7qwP=|)
z`h9XQafo66^_l%y$Kw+DKYyOCW@$#4`?94f*0~rcbJzb6r?vRHtCvBaMA`f6V-yfM
z+OUk3^k(b<zo5J3ckiy@SG-;F$_SH-Bv*%qAcQsn5V_}x<5|&R50NXzhvVjN##bBa
zp7X`gd7io_z<wN!vWG0pdGo~mk&5}T#!kXf`vRI$l6)G##!bO~<}&2Jl0z!HHp3hI
z3EymOmUL!~{Id1Kl4JH3xxTX%N55tK);BFn?wH}};5?0!$CRPy(JGzx&4zr2zcvRf
z(A3omoz)Y~x|U;fIwTb<*(Zi`DCD+bH%M}l-5jeHA9WI)-8~~8V)o!=e?~A>r!8E1
z<!)Uy6X(44+RmFl+v}mU`C8AL4B0#4A2wB?jaaXF|7)s0U#v(nEw3@!(cpeLBl8-4
z>>Z}gJRz3bc|D9TdG`2M9VWc#&VZHxN0c+ownTtR%ug(%2<SHu0RSiHVhd1=W1*j!
zX4MJK)RnvYA8sXnjqTD$t*D06KA6?l?6O^%L#8YsZR+$j8wQCW(~Oc~KZo1^QB+QM
z34^;rK0A`&e-h~Oe<N2#dGUQHhVt{)pXIKGF4+vT4?4O}!`{Gv*O)#rGt{HA*l>oS
z1s4v#K^>d59D#2d)9>}6*a6RlV<k`4%}f8T35R)hOwiO+qe+G=D9MZ`HPwMH!?xNX
zGc2|$UOzy2M<4`t_UC;U@wKS1F@;CJJmgnV?{h)!%P$@J>hhF~^b2;GZpDrG^huBp
zASgF#Vc4u8Vd?U!y#X#^Mr2KZjyv!Z#vH_DMg^)=gJ0;-JIFc<Cao}fL-xmV3cas=
zy`5DgF2J{Q)E`pWD#fyd5+)`k!<yuc+4gB8I@j{tF7pDnKj@r^rTI(-gn#jQ&udv+
z6VlIal^Z$kwqOInLVM!kL1;Y1r1F&oE5B_g51&gro_BFQww|qR1poFdSSs~j#x)H&
zoZ0ehE5`8IPBjK^35b)gI8-Uq1{0(maAY{pe*kpf4c|L7*pF{_w&&ired(YSf63%_
z#x6rm4E&C<EhO@0Pe3pTgWf6oNL+`Hqe?sclgJ2yIDcWQBhB1dDDuc3e!b<Qh-s|@
z4%Z2QsWsLfr*1d!f`9h)-0Oca&@<jxH%cP%#BH-HujDeXqa2+m*k3F1hWsvT*US6U
zs4TcR>k#YCw+!<)MER1zieEaVj|}nN87R(4yk*Gc|4e@LQ0cNm>`}k@T88e-TA!ZZ
z==YjBrmI0S2&qc{=Kg07N&~q1zHKDLuadn4O@In0ybDKc2k(mq5mGsok;j_C(VxaN
zg-CSq<d9P^?w5&H0#d9maynm%+-F$iX$DjvkIBbeOmp~>2O93pxEbYW8E)rLk8GN6
zC%!jhz0*`su$^s(&zf#`hB=^l-tpTt-(k4$nu^l#dZu-2UvEjSKfk?2PDt8mFV~@B
zC&OVe|MU2Z`Gib1x=Y=;vqDoC*GhwF*?P81Q1hLpG4mvW5^8w;J^@)GGLe^g!mN^*
zV$`6SxEaq8+15^3NPWieiZp;Ap<PSl4%4l;rUO<4T{)pHKJaG8?5Hc~{RUy{^O_(S
zdv14@lx6MUbo(D&!c674@rOX^50ai%xnmEDeoQf_v5WtH<vSu;622YRT4dKa!)!+<
zedClZ>L>UhR%m2eO<{F|$%uMrr6NMWZ!hpOWH|NGIWby;6INaD((bzR^>eEq9is6~
z>f>j7!Yr3}hlv#_lBl2(j7ije{8?Nq(=>u*Eg(eS<49hX`ZQh6Jo&^GKiIEB>E}cE
zTZqEcX?$->W)8Fx$_c>St4}&E=I$i1%Y$)V)%GSayA@Z`5)<VLcnbOrIeQXYL<6I>
zV`iu(cac?jOH&wCP~MV&rj5*`6jVOHG{OwJCFD-syP@I<mh5Ep<qKv|Q~1GQJ(thg
z9iKB%hZh<s)sz8#Z|iV*cp~KBN!K*)V*gA;<UV~?BjK0!k64@n2zRIr`(9SxI^LUy
z@$4Bo@Xt%-f1}~4C=@%ie1WB`CMX=ArV_xf0~b=B5#6HeN@|!!lMgUrj<E*&bQ+~h
z_7CM&0&@eOBiMz(StjebBru&<r3f!3N#8Tju$q{`&O&ko1U@ChZIXWLVXT{${QZ#N
z_nOBInp0^#OL#D#eWx-CP9QdMH`<RaTgrRVs=S%L-a}^g53L!)?_MH<<vW5>J#jpk
znc)K38HO-$NxZ*G7>0*_Il3T`y<ITjN`vLw$j-WOYoZ=B(GL@W5wtTu1&5PuC=Y=H
zBTZHAEwPWH>?JZ1XU;Kud%j*1z0O|pxFdeftkLztULw_wLk~C%GRNn^t~I=&v&t32
z4s<T5%BCE+Wr)g7|Gestldq#EdoHLVMbrJ5Oyt*h=8?C{P!1aeC-b^5c?SKA8OgL-
zx1LkBl@HhV#4gk+2{Z^Qg1ECuXQ&sBFXU(8QB9%0#C1OKm7Dv?i2172tW}`|T`x6R
z!f_tqY?KNzrPl%z%QlYwGme@iV7h%7z-CdRYt1k7ij5^V{@dV>-aGepa;L)ulPT$8
zg~nb&MN>J({rNP<N3kj<+SFL$9mcLdVJ(`Uv$lT`Ib2qall#8Xf@Uy%=#012(dS?l
zvh!BH+x>EAJtlMm4(-1rwq8PeZ2oE(r?*wDW+>0U>2+{wxtsBpfVl;8m0mqmr44mN
z>Ha81dZ4e@BAuluZx1DA^mJaO8km@l5Xs-dM{WV;@_A2~82WPAp~+_-P8R5*nevq1
z6Qz`efX*Z9LSslelsPcDO6EDd`u9Ibx&+V|jx))oydAY_`11inYTR2B$eA<xf*2vN
z2_G9R26ycU9%g>lC5SH|{aMpY?migH@#r(tCH<!as{quZVqS|fQNz&hpNBIf1t{+j
zEHv}8Ai+rQ72^o+Y8iNnANdw*e&<sJ^W@tC#uG+DTh&3Y3q0y@{0&Si$r>u8U7F1*
zVf~7Jgpi5Ea9O$dvERLo;b<8T;FxrRxL*0{T+IQAHwzsdAFA>14f5ai!S|d+!jWaW
ziG*)%Gd=pZWfB>g4WZuc1*o|`xKdLPGf|>_%;R+_Ys*da*liNdc_|iC&JZ%W0uZaZ
zR@!Ro=bm9_kz?E3D+%agM|&=oX<yz9yrDhTtIb`~aYKl$MwMXN*r*%1`QG1d)hj$y
zN-5I>o!@t$k4e}A$)*%dj7sq)v;Q?!G=d&kxKkIFd?ny;n(aslRL>1T5+O6Yg!YsG
z-7`7|wZ*p_#4Di0&G@=Nvd9Y{bD9BiT_nAWkr1}X7nzTghP<sE*B8zEh-J(+n?0hX
z@G-;l%6h4QA8xv#!AjT){_EfxQ$}q|I7j8MLFSE-c1v`fDV=42y!6}175SD%V}BO{
zR@%JIKxIa6IjmW)8XSP#k2a=b7J}9yI76cB^Mt8V(nepD!d}oZ4@YALo$YPXxKnYS
zZyl;+%ItgG_XoXofKV_$=FZ4sgJFDx3itAzrPEE32z3R)&<g%Z*Xigb`Vk+u+tbt+
zgww2N4os<kZo{cA>@nTcug4x29r9sgIT=J*gBiI~lti{}RqE4{M)OpekP!rU`Y6aS
zn;sb9Tr3B2Wn5R!Nl^f1K=+q};ttWEK;XyXyF3-s=JZVD9G}q0bK8donjA`eT7`)&
zJ${PLgpS}LEyrgWsSleG*3<r1kiu~_Zg$$z*+0d=4`BWmF#@o~F3nZ4fxg_1Jzy*w
z745rduq;@fU=6v6lok`?O(y^AZg@VP_Bp{)_FTd4=jR9oOQpcbnc6?Y-d_bUi72?9
zHw}Dhl@n8AOdF}C-1rc+YxW9gvNOB7ImVurU!_~>hjpvjzk?m@X7iRZBell2Qlatg
z-_QspI%96RRr*x>UJg9SLx!NaE1lE*?xB<}dxW<M)FP9>YoPEL-<iMdp**SLZk#k9
zf}K9E%x{2O<rN|O>>5{FP6APQToz33AD8rI1%TD5j-fuW2=(zUiEy&Nod%=c%dnIz
z+COE=gLGnt#XWMDr-1IO7NE@5cO54{r$cE=w293xyMD%!DmBH)b(2SklPYi<vJ-2K
zVNAELye&3N5XfotoFF))P+Pj03g&%&<=cN2(TrE`|4x~#-`3(3*zChhN&{LfPrx%p
zT>-|wxn919lA^)=%KX@vnt^oqk__RBJ-<?f_z&ltms&X4yG>{r1@mPxmslZVW$vX$
z>gMJ)5)g(E@5{zC7j`q)+S?xziqKy|b_oWRHS6PKHWCUek_c=F{jUgoLo3I3DuUX>
z@I;`!x7gcOYUYjanxh)pDcT}3soXUSAW#L<FC(uW=y$2o_T3hUBt0F$khs)ZGVaI#
z<pCQT9-s)MDNY@?D6@CnjJ^YUNuA}Yb-%iK;5nVb^DJ=(<wMoAWVQIb`%eg?`luEk
zE~24nP$)r)0B+E%09oS@&2YtAKR#b1zCOd(_$>xU{KnTO#gS}VCXQh>yxy<7kjuZ+
z$R7<*e0-xqm<Yf9%RMXd=|a`ZM3EbUd_a6VlgO=+;2$K$GAo=cu!!OSs~CPv$4*np
zb{jA++WtKYA$(#$r=+ft#n`)UM5ce1WHXrb1Hm33|Dpj-{E3nYQh~Iu6cr=Gm#6hY
zC1P~sh@%gr#TR|g86S3=;Q}y7Dh(~5HJ~g~gSH<-SYa1kr`JYcEpz9+{_X{wcY`1Z
zmGpgsssN-2cEbrOK2_iaAwiHYA?YE(L~p#ePaW7>8Qe~+%zZ$QxuFa{WtPJyY7Pnv
zlq#KBcW@?S=8#6Wk<2NZOA`|D*l}hElbD_Qrbwp~L>>V+3NHsQXthC?c`}R=U{884
zEf_dlodxe0H5VX4XbJqnsW0M{_QPMEvK(fPdrzL*g!{2+O2st{!hBQUYvp_zw>L85
z4eEbBv<qN%bAl2`cNdVO(Y;s-jL_>Jj{ZK%h^+onEM`$T?;zC6aO7L*JNGyWne7#h
zYVu2vA_Q$FDppV>MMB~?-u`-gbpB_Q!$o=7+$0|41pgx#I#$<iBEsB^Uj<@D&fSP3
zjD_*qBA41bk#o(}Vfq_<7-yGeSec(=b9S2b->A<WU0}eRAn*XjKv%Qe8NI;X1QxF3
zCd?yH#CxGiSevrvyf0)%Ha)3Z_-WcoCb`TAAwpKTXkla=$XDTDd}+QfopnnJ46`GM
zTv1_8CY>Zr$RZ1>ThBO%+4cU2<6dkV>kHU7zPrLVne_e+h?MmwlvuABZmfzoO<~6)
z!XO8<*kJ@p+OK)0&~N+osHQbOMGJ~GvAJ3F<v+0|OCva^mey))^w3H?GI0@wA+r01
zy@%N#6DFosW;X$6R}r8+U%IJ0N`4C~bL5?tRwKZ~pBnL2%<Z|pJwAf-m8aXtmT0RZ
zZ-7ui1bdL8jN&nkvQx3x{@mG(4Cbd6eY@Q7Pcid(YHZ7`pN1<fDqgry1j+b19C$`h
zP5Tq#%_T#8c78LkyZRC6#m7Q0h@3KEq~4zv)~qg$>-k7}_SPs;=aJOkz=|fS(Rte!
z!@8r2wj(!<VnpVWCu17-2@rHxt02~7Z3Q%(7>w3EEN)Wjk&q)Laa66u2$*Y8@yGz>
z3N(9fjdEZuFwR_FX0WJO%sFx>)^v!=t9oN%Kc-iwi7a|y*AaE@klxKb*S{y`Z)bnO
zrM$C|>;;+Fvr|@27<!_o94nZ86Jf8{6ZD{eDZilI(wn-_ZY_57;DUM|pUYq+0aLCj
z&Tn&I%tn2q(P^Z~{QV*sE{{#}XP*RQ!vIA4u>qp{UuR23xQ~d|)S~TJC>s=#57@ow
zv1(hqA{3VdV+%pH4i>SpB(Dj63T%xlL2BW)uKisnEz0~OVyr@`I`9BRghmD)hVhI1
zoex$C#M>;29E)g!v7?(P>6}cMCheyX>B@<r*wHzxgXbLO$n816*<1$;NkFGu!DWHA
zDuKP<_g{q)uHLt^@M8e5f~(rv@{NZg`0`n*uWi_bIX>>)Gl;3B5r=%e3Zky-m{>k5
z1PD_UpzgulB99L)g6tHx!x{47vZJ*%`F_9%{h1bjuo)4Br1f?Z_)3Kl!#e8(6n?Vl
zOfp(-qim(`U4^qr){ND>**PT36$7A&TpVuAxBcLpDT$oS{n5x#=-0g<k@=xLjFd9L
zw!l7z#-y)4W*0g7)ehiAeBL(b?-ZgYa3n|9r)fXosiw&P0GDH63*;dQ2Pa@)$-1!E
zOFnOjAm;tGV=`>n!N<+8YWjU?;Zep;XpbBv)t^Dkm&#}SIFIqO2%x3n<*EMSDct+z
z&qkH->y07&=8*O1A)<H4`+IoJ7%QShyk3S94pYE2Jc1-2e?=KO)8b5=bd`gF^}ZlB
z?r8gom)D~~A`qQ-49L@h<|6psp?=*hM|&WC^+j4%lU@=v0-!8%0A{k6wABOtyCp_+
z#Qi0d=VWkm?d~rTgEoPU;*lBzqG_o;VKuY2I9GK>IcgmDMVRxJ#xdUu`3hIg2sP59
z<f#0m30g1#$>j%c!o5a>M?q;Hg7|PG1m#Z|GgIfa4XYSo;`PP4;%CE<DJe(5{^P#c
zB5Wp*LmKh;JiGGXzLxqSAtwIMIQCKX?b5O7Dm7&Hr4#(Ysw#fP@=`5<u04AKYrM7>
ziVT?db>UEHlj;DfLigP_BX8Ja8m2CK2m0a$*u9}W2u5(WRD<<5oJZ(J$aSMfUD}8(
ziQgd0Fx<$YfQsz^<4_72V$O@E4!bYprE?^rIh;5vu26;-cQb7eV!0Cmpany15y)mS
z=+lpp4OlG<(Ho0N2&9FC?&q@4HpI|DzA^B@BVBI8wKh=EQ1tNJ;}XRH<~p}&{*pTL
zp?6;mc0VG?mD}En^Mw-ZM}*vh>#0J`Y$Tp2mM`D8dt8KmjTu~0rOkQG{#OYk5^l#w
zJ>eAY?u>f(tkxPt1kBSCPM}2{VAZQX_EUdX2@XW1BU31$nc+D13lXNhi0^nQ%rL>2
zZQ}EO@LXN_9%<5+<@3YbATxp{H>V)8aQG-9<^mW{bOV&3$?147*7>yX?q!0lbR%Qe
z9t=uM?Y&}V9P>A}T!=AsUYTL3ujB5!B=UYm-ZbAV7Vy2s!j<HVl?Oa8Vf}2{V~A+P
z-W%#Yz6tZWn_y)@d4IPZ_8IZkfNlPK-zCIjqJt;L)M{|ZR{Pu6yF+p61>^d=J}l{j
zPZ%Rr;DTqjm~Nk0r06!1G1M_2d}|C48T3AX16XX3Py;O8gji=uPDSCoDUic8_Nj5K
zkpM?5E7BaU<Wv%0&<r@nUy4-2GU$^BIV_c6Q#+{h8ECZ{V&qsAJtA%$%P&8$+AB${
zUQ)>XwXQ?q)i*D3og`M<oezdOp?f5Oo6eUlkBNhfeq|h((SJ8KZM2B1V8Sh2%!gsM
zhRrl@S1Hi^6b2_n)nXpXL88U^g2C9DU|WKsk&l3_<>)-{@j`uv0o}0Xbi4{@;O<`A
zO(RRlw&**rrPfJmQ%l>HtCKksYisLLfvW9X5!*=rb_lImruEae$K9+Esset$Cf-iw
z1unYe`A?cpdVE}P)X9|+eDYK&z;mf}VYbFj{aeMne6LSNr!9K~7&hH3l?d70^WnN&
zx?~>Nd^lM?n2ywYeg?6;H64{))hNf%rLjFIB)$JM?etZ@Lxl{El2T=3$h1{|Q=ssB
zK0;RD8t6l}*5UnrjYS!K+8~Dn{e3tl{=%(L(7U5m?e4lNAJ40@qj5bY^9Hx6MqSyT
zR9mkzt!u4A57gfItq)CJoi4Eo#q-b63O=^~{@!Bh&tbpt`xFlK#|aFQgj>lZ3=%yR
zW;MnB{emCOyqXVUj<Hc>kB@(SuM5Aq3N)R4>3mmuDYQ<~aUkpY3`ZVi(kY4Zd51z(
z2XDT3^q!xI{G~jRLy}(c<z&wVu^B;-QReN1PfUd3`0`})QrBzDI{`&}djFUmI;e$Y
z<bu**2^YdvscAS9v?wYbiA@)c3uH8dO#83Begs>V2C__<B@d7Vo30js*%O<He{PD*
z24R2%W?+0`o_aZH>8aUThlfQTo*&RG3bWAVXOx6-uf;!m%r!>(JE0Pv%+xx(*G#=9
zo4Y+6aeZDk@|4d+SLy0E!k?YK#fJ0*7;dIyd{|3HeITbu_Dy^u`ey1fVvU47L|W!N
z!A?!QQ-%qpdU>OZg}dGzeyR^eHTSXqV|qSDh3JEtYA!_*xtIvAK`jbZ)e3#<(ZAF%
zo}9u7O4xN!z}qXrA-v{|PP=BAd|JT=JltEj15?juTn<<B?QtB=_we6aT$pt#2&-&4
zneBgibv(6GoFQQT#H(#RKcw<xXU-ylQMwV9z~InwJZWLFx7b2dQBpEn^nk!^Z6F&m
z_o*7*4tf`U^=ERn!Hvj+%=+{0mn{LPTMF7^Hsa>${n}uzMJ#LhCrP(S0l9PBf$&fl
z)`M<xzQgguWU9*QJ?;lH+)`DNp=bM%>@$xRUq7S`3H$b({G=w}*F-OLr+&Qhcx!5I
zF63m}C2YT)U~(~J&x08;qC%6@<odm0vh`xuEt9bi=-B1F-dS=*TQZ-dz$a&1(^7pf
zt}iD)4U0WKqU(GALgHf-7go+2+9MJBVm-xbw|NsWH=X<Y%UpDY{6FWrl5iXKY@6m#
z3`#do17oA}4`JMAUrv9d+>WlS=FeN9zpCcYzbzb4StrZ`DK>ZFGz0h+bJZ&p6)FuY
z-8An2@I8dgeg?H=2wFVeYrUvy*_pD5Xkg}sS`pC-)oZ2mIfPy8d^}MMH~C<_@c5%=
z-lIB49k;$T-jkCgrR!<v?~w;YeF&Nj<FQS*sj9rZ*tzxL0<Wv{^#Wari#qirmdOq>
zB}-=uEaBYOs-2(xnHK0mQWp7X923%820Zt{Wbz{c#=!yzycwOF<yh8Pa<ej~#D06l
z#%!w#goMk37~-0ZZ)4NC(DCh<z*oD8%yMt+IBSVpi5@C9IWOI&J>5&H4dPghJ-B&t
z_f}wNUiBUchKDgk_%g1mn9p>_JFO8U;ny%>6&)7YNg4RwRg;kF5!>Q(*eC@qIs2I@
zoFpI&GIXop+p)q@(Kjh%;<y)<2^;MJE22eO05}9bf<D|^D0H4H=gM0p0w%e!z&Rsz
zRb-kN<#vIemnl?dt=}4Q)%mP+Rcm4=veWrZ8PJ*a<7^Brq;lILN1J7GQMx#yrJrMG
zF3w(jtjl`+Fq1)hNEFArPc2uy+yxR-mVP=#Q0(*Uk|f1hHdmKK*)=-w`T*QFsohLH
zYIp0tiztoQ6Go-;Rmvt}nTUIGa&wl5QMH&ISn0TsQ`^;v`1guO;<&0P{Gm$Qh04@p
zTlsR-xEt1(L7PuZApUp(F|PB*rY+oY{*w=WqJv6E5EBWbl!W}QL}R$~9Pxf?l860N
zFmu7LIn>j$ymdUu?rA&BP5LI5TI99_2;F!P$Ua^PVHgn2Ekm8NNVE#{t$Gd0koB3Q
zbZVf?*y%ery#0B2^|1~U!QA^#Mgl)|%)~JxcIgsA$$tYgfx_YL>c#UfkPC_gU})fS
zi_kWrT}CseTG91kP<tqFf125j_KDSgPTk!_J=H09r}#vRt;zTC_UfkfJ6Vuj<=-j8
zuZZXEg>NqQ+8oE<=)_zIw_W;1@syme<$4XwfA$+Z=w`LyQMflor+PqK{7j1O$T?Bw
z$E`a5tx3@OcrOOAQ6&|)6c(7$D5B%dxT!(mNham#G<P06<B*wM^dOc2q0Wd1%;kGj
zL086{2p+gZC^GX~L3ctP1b4JEvLph3kDC<Lx+15TpuyI2BF5g78$@)FpEo~)Ff&~?
zk=l)X&hjPCq`@qyw3GDFzGnmY!i1*Tf3pWsX4F{f6Jh-c@Qa3Lt7A3c)-sckb1J3f
z%mLi)Gd<^2S>VU_EVZ7{lPNcn0h$_vD$!r670}g81W&{QdShzeeQ2GH1iR?u%g=;Y
z4&!jiX?TQ1$~{ZL;){8mBgCqasrz|(zCv&G!pzUU^g5gbJJR-7`6SA%+IcyLy6fSz
z(zfT5kOvsn#JttBKHRQPThIzyyGwe^PR#lZE4=^y&Xl0EUO=XdW^SZRkCn*4AY`fH
z(K~_0kj3U;nr?q~sF_IP%N_XB0Gm2FM)J<cLz#CMKjMWOr)C^lx>-V|S6IvR3Ov0o
zerDQi`{eMOR8~*PviJ_Yl06Y?s#ChT^b46e?>NW|Wuj;r?J(Eo?ru?S&i<jjXgn=u
z{CGGv?p!GXbOyO3*oBZN`-ni7ZBXj+idiZ=H$q8U`Un5;FpKIF*;|$b-q(RIg2?Gf
z98n$3@4E6awG2OLN#6&X*(#4R&2?^UX<=?{p0YU-t*?#%i|f?X(W&k@>%vZt$LE!7
zqU`FYuZ<pav$}V=-;FSFQy9SJ?v_%@`26@5=_=EDXW=I#@kx8)Wi+Gbr*@Wz={Guz
zTR6?#*p<ZUbr<BUm+bQ^;Bdr6eg!RF*_X~grz?=B3Vrsf{z1rEw$t`Oy5QsbYBQEO
z?f6wyU!^bAJKgj?Gop%X&%3|&iX*C)B5uM$OnJBu(@m{wy6t>pn#6U-c)QQh=XhKN
z$<q1{+f{mL)Aes}pPtt4&hp%lV(eA}Z?r|IUlGT}E1<yiz-AWc1A=BdA99`UyEu!+
zk$;4E&?(Ra`ukY)ehpnSr~)jErq!Ahy%Th0$69gaQFFjr%`#6?5m~+_BblNo?X}1h
zAk7hp!iDs7*##$7kQ=ME#J2DZ_I~u!Vxp#=6Ai{6E_S89#XEk*rK4j9)67m%#<S(z
zT`hQT(f1E|6!E&h1j<0T{5!|w;LLP{Cy+$u7tu4w(O&J3M?QcEWFHHwhG3eWV0@G#
z%fr_{aDK-9hrq;TxzP`O#S=!r3sx|~{Q@Lhd5`d?Xx-Mx7V8!E#7~|_xI(8r9xe!%
zkSQK#>TB0@v?!#cw<Ki^94}M>Ux+HUUuexjhvHeg?vgU?j!Y)?eVsU<XgnpV6J8v=
zEcilE?S$%(5+a~h;km$P+(bImGNZfdV(s);Nq;a+9ORQ7m*WzOa10wD^!DxV7A-sB
zUJfr`<5qU^vLUMwh!&f<`lfzZe^Pj<uZF9iS^s2?8HzCrC#-~Q@0L6;_%<J~NEg;I
zu$$aj8!vBp>e)lw8Dy72xjcxUe>&B|3tclLxXZ=e#A$a~5XK^6cydpMO0|wuqZ-e<
z%je!>D~UhCBmHGvWa<jE7si<t|5W#UD+NSgZtx|aXzkb(yBl}1YA^mAUp``Y4e|^>
z4@VCcmte~6c-$frnDjbDQ%zA~g_yXSf{e*n6K_zn#WYy$2tRUNDP0PXu9T(>?z=l2
z3<qIFQY?TRR|nHPoUPD&Vwaa0ZMmsHrzaN<{rcoy^8cheqv{Q@%=K%L^{%5;UHb<d
z!@z*^gUEEqyu?tUqeN+2ZZv-ScgR+APX^g6OLJfOmO7jq{wym|g4KiLAqq6HXcxlt
zlkDJAqFLNttg0|BBCIr+f3{jp3B@qc_!3vQy*oosnN2bzdrH@gCayf<4B~&xPJ1eS
z-@M;SK<{69&z445?KIVOuDl&(iNwHb^wa9C0iBf<oi$|0f`CMuKtP_K`?RJ{TG*|z
za01836(g+DTP>bD-RD}Iym=s~?E!3;;w}H<e9B}bdAo)QgPa~@1;DX(3TBC7gvF*w
z$7Me;EiN(waMXm;N0clby%YP+@z>sbu8z{uXukc7DV0`Fg{91a>ZL*IBP>1Ekv7mj
z?UWMxERzYFVHF>#s8?NW-ShEC<+<-xs3}{!AB|h1twvB_<qHMf?t}Pdm)j28ia$k&
zPw+xXbioLo1&M3caZ!&g$0C;W@3(&*o|e_9|BU57(cJ!|IIruG`L~*@^f7991ZiZr
z@SJYzLnPu=@>ilcHMY?z341~NpEa=9y$976MG7z8PpC)*d~eVYXv&LE6fe1R1f1*I
z4g9d<V(c9~m0lS7ON_bNqMDtDH{cE_f*w15Ypnfo33(fV#IqZ{X4E-dB_yid8uPFT
zFFIny0V0%U#ptl#%>47d`2kIPk5hM$IB(0}By)@~<c#C`i%t4xV?D;SBf&zBvBB3k
zDUfH<20Tg?II>!0&9bN04QSjl#Kcd_!S%4|3bB#rVf$knmmlzHL(DaEGOob0{^1M_
z33{Y(&T7(6U;!YdiVe1FkSgwlE5gzVnOgEjTPJd?{3HbP(e2!$`STTR+CkHk?|(D8
zsGdIpT5C*#ugx5D4!+g_LIF8nMuK2B_$c$j@W`Jvb4jDjA0y6!4c6TsL*c@J#C61+
zOmA_)T=ZI|DM^96-Z;kkd`@eudsAlmEpn*_A<`IS&@GR-xhX0X)QDz|(V~s?eOoTC
zmG_|f=0S4Y(IoY;F5quat5o?hKZdn}WBg0>GNUO?(}rcuMFJ^Xckn$ors1A)T)I9{
zMdEMtc72k#jePIS>j))y;JFoM@cF2BS|E-09H}}g0u`%5|B+}IdAPwO2x5xLa(x$#
zzn>!SK0J(N)yGAOV8y_XufLG193Ms4vXnE`FMN2EU#5Opp?&;MkuSv~r0%%*3rF45
z&}JE5UmPZp!==r|CJC}X0x}riwV<AbUu_(Hy`d?r3xRrfz>1P7=XrSAh@oi2_NFpR
zO51zR@DbS`cVdm{OPnVPJm3=txCHh>p8*f?5Z;CenU!3FfPnX9Q9eBb4Ap^zj7Q|Z
z0*-D(qhpIJaCpo!M7W(4Wk>7oQ+?%qXDM>CkYA%~NX?OGoAJTjmXsI+qPC`0XpwF}
zs_#6_u$`C`O?C*Uk+Sk)hM8j%3<z_6STV}`B(2$p5U>0N)C+KFeR#7hrcaJHYhM{e
zFH2U*KYO^Ku^(Qqs<{2(8FUS#ILnjX<y^zqfiMgvuCTxvLllL8k^n{wOXr4}to2J;
z(2#`Ay&P|rL-8;4E=)Gm+cx;nC}X-|8zoGD7{kN$yX3h@?O4v2qC8|A?dTttO!Y0C
zRjqMRt;5~m;rmt#?KoA0t1SfIaw>SDKI|<Qwm8&t%rr8${xs@c3CD%~KCEDBbK9>R
zI9&Em^-J;z;r<iD`7gR!D_K?eOl3XsyiG_-Q5a`tq_+X;V~v+LJEZAVEO}ZikXZ&+
z7<73+SMtbXrT0!+#-GC!1GMK3ef<Z{DwdciYvR5O@BL8XK|v(t;2R)}`8@2oBS$j!
zmGg_x`6Y<IAh4a>s4d4^knhK7CaDMC4&3<Kh>x-G%RlkA10HXF3MB@;7-B68dXF$b
zCFC?xRY)IQRTJ_Sc&nbMS`6$RsrSW<NaESbTe=qtQi(7wam8JMi8{i0pzn|Jx=;6B
zgNN_3TQ6pbYWYh9kyV0+qoS1MPv|Ic^)L9WW3A7i`TIrrSiX%<2XX@k*>4+(Mv<n5
z3%9-t|4Rl@LmFoDd!bt^bmtudJ@yfIyh;;k=Yo+2tPFXQ&^PZty<{xKhX2?90*DEq
znw205#g~DJ=G4f|%QiZ@<Xtg=L@6%uPXeWWjnCtebOn`yIa)u07bo^3GCJ`2kgD6B
z_UxPXk4EXjY3I(Tc@pOw@P5;=ZlR68!?*3d{#AN~i<nkltl3nD!0M$w!*=7hw_BY`
zchh9zztC@PW{u9|fTlgNhD-<(N{*Ts<Vj`1#b0@RO^jO(h?!nwoXd!cRfeiP*?IHl
zx>fa&Ge11O;F1;A7y-Pze(HhV2q$IP|57Z|#~NRc7VF1B56+cWf(fdxSMREqI*(yN
zvA>DDjqI<PRnX{hUgNHD)QCc$=|~~&r?1%<@ffCVi01Fueil0+0dHR`;+4McEqj;%
z5(=l^m|oa;61*Ju#5l7pOm>-~6fWmAH0{TZbB-z#+M_|IMX$l)ZyI(b(hQ!Dw=?Zi
z%h0KcLtmk1uZSsjU}%$A(~G%6tPgUb&@!NshGqi}tK&$4-wt9p6Sd`h9D>W5Odxvw
zxFbLLEKov)hLqNYDPl6q5Ai??>K;mE^u<h4!v4V*2^g*{6}R5E3P6)y(5VeYg2D4}
zxG2_b2*fN<Nkd1l;MYtetSF4MWux2yN#t0~lSTT%y+kOVi6_#BkO<%*ipLjMhhD>g
znQe4n{e06lO-cO|G4D_P9=eU65_zz(_CyBPFW)K$^vb(4`{(drG&w@vXj)u)!i4x%
zD@WF_5BDKKlz6{t7$?=E2ryGvU(D|taJHjxNjrjRlgf`qlmm_6doYtdtwG>N;!HwU
zgL7yqQ3cVZ8-q+gTbk4KF_NA|V0uEK^i4JN%>VLsifI91D~%x78?7%j?@nlxyML*T
z+-Ziypp@+->_lZ`z3pNIPgO&LUl9R05)&mcpzg41!dY2H@t@+7J~H2bwV#&o_St_7
zcu3p7HsA!ixXk)Q=~eeZ4PQ4hRTK4z8oQ8!E4wfR!d}b3k`to_=ycgXlkdldaclrK
zB=Bd$&6eH!cHvL4iNufBh<AwqPI)$DGV!_Olj`kk)nNOy(Y*e0eA?75$98#{w1@g@
zYDl|`l5kr8T?HAiDAnfZ=cFP>E+49%0Gi*t5W`XoYkA-Ys_%J;-#wK}x}tQDc3kJv
z<gnYn0NuD`mbnrjM{2e+B4g&7A5#Zs!>%Ol5*P^2YH*~2toint2%YLL1BCevW(?WJ
zsWrMsp1yIbIO#6K)#iMCD%tVtmNz^6bSe*fPlVF@A_`6)MC&a!6oRDGYmCya&2e@&
zQg+HpnTkVp`?3~}4~Uj@cD>Qbb^PlO!+ExxawJ@y{B>^DErnkMKMj3n`avp5@lUz=
z7sOlH&C*o&ua7=?E<nZIr$xl=x`^v8cAIjAtiEI!=_z~<qFg1KiYuP`=(*t9m;P|Z
zIrC_=m>y1i_SSjppIUb{_G4y^23+Y<U>;DhHFmTc7c<ddV2$e7wN4zK#oy3Yg!}?R
zv$im)Y^jwT3?Zb29GdDuJaF0*6mo^@oTmh=TvLs6y-hO(OxrqMob0Sk+a*a-^PAKw
zISuCg+&mb)U1|Bb_VZef+rUELl1EpP6SArN>hfs#4erA4`1RTCwUGy8k~n8UTkv5o
z_h1y+VK`UAWNX-ku@oG_x2`j^g-o5d|9iE>Wf?dU?k|s~j#cWsbtdih4+G#T33|`&
z?8Rc1aCJ;YfHt7@_SnJ(=Qz-slx}&G^#07}=}#2iq4wwcx5s=e%6Wy_ZmydiHM*O-
zjg@Lf2izPDD@9MXnAx`6T%8tJ{T|2`e;-XmOY`=gu<$Sxiv(8!!T40OnYp=c9y7JC
zN2J<Jr}1Ul4c;9uH(^+i&!t@r7~kUM^I$tZn^ONZA3uY>k0>UB1vb}IS^K~7;N@w?
z#7%;%S7pNA@oSiFMaKvNg(LZpD^QN(^}{o3Fp-Lk6J^s__kAm_4^6w!t>5E%WDWg`
zF!lElH`gY)QXT>bKMPY!pXg(puYUI8>G3azmcu@N6Wn%I;*&5;Uu^vT#}^+j+-j!3
zHu2U-Q1uScWL|U>nd0;x+?Dv%_v8Yl>$8$aMfV~9hpx8_ild9Vg<)`af<u4+f#B{g
z2?-M1f?IGHTn7m@c(C9SB)A86cXt>dSa1k#U-P{8y><V5RsCaXs^(0e)4Naaz4ltG
zS>n}%y{~LbyswYv#O@DGkLu>X%pFv;+b7qV4dNe!(M4C<w?Dcq<Jrv=#a#jjf$qV4
zMZ?vKi?R&>SRuJ$QwU6u@mR(B+9n(NH5w2vUk}j5ELdmy-C27-1HhQ=QT?+0Tt)lS
zS4!6zRZTay`q?@gt%Er|2UaE)kIqmBxb_M~_|T}}S{R@8Dm9W+CZ^qYphwm7wvtMI
zg!6@g*A2O#)nI{#|3!zxn5WY1FY)32S;fEdEM<XGfEtXkaQiB0@eTE;7Y%%dvRvR)
zlOpGg`3q!8+*BidRNb)Zwrn>?YME}#`S<F2-Cs}C`>OVL)LglNaHJ9wWj_Eh>yG~5
zt?7f3n)+TXT53iTvmT0kbYJt_)d5h;4HJmnTFc^pP%r?h=&t)-XS7moVU$t(-Q>_<
zy~^}|^29Lc3#mscKU}El{t5)d<#F@LQriAX)kmRnsBIE((`A253MqFKUTxgXw8;@(
z`C`#p_jR_K80bH&IuEnX3GSwwR_RNjcy9KD*CR9nkc{?LUsMgB-ICdEx3*G=&+Yk@
zbH4awn*`g@u6r^bSnN}FnC#cAE6+lSID#CkHkNG8xHiRdbZMrJ@<p=O^~u4Xz;|Cj
zjBsuneKt^;HS`|=)&U6XYX}sEDIZuuhANQTgL;jqJ5UOQ`z}WJta-Nbpef0d+50rB
z>1I4{Tbv%~C^{<9#GY<Urz;G_gTm4vE*3B&Qh^AheY~g|#)istx)_nEY`qmn@}^%$
z;B*m$!l`tiy@2HT+i^3T@y82HV~vZ(yY6!l1>v^-;;|7TwGUu(gjK#5yW{*UTIIEj
zAWpj6xT~8nW%0)uHWXE9mx0$5ae>0Y_N$l_ir%Dn8W$@r<D>!|$k;{u3srg63za#m
zR7vh<y|0!#ocPoVK2}{&xQuYRk%mE+4y><2*H?33pJEd%?9^QHC58xLIDkGdUl(48
z0L)CB-}G8+zMcQ?J&9xd;Q0HwQ8d0o`z<0*00OiC9(@9^LRU$1Wdx;<F!~>YG!x9m
z<}5etj;6up*zPca!GunEm?sCWEsC`rICJ9BM%gwekHi*S4Qfbc));bY37mGy7}LrS
zay>KvQ1@0+5sykq_v#}LWT7v67WIsi6OycI1yE8^uUbK%J@c{E>vH7z>AGD?mTt+K
z_?@;9hud_fzM2)IX5e!Fqb;3@V(0s#EysHIy@^hB0Iv4|rVKz-nVfrL_0u(?{Hx#u
zQAq#WfaQrYyW|5~#p!YH$?~xP+BGU0ay8xyO1v*Y#yD5*uqB%cUJ?`B`oUrkFNuY|
zKPM9TOKp&$!MJg%Pb^2tpf?8a-@Zh@HKyh1I;&Q^1}Dd)MZe8UV3jbGkay;S(FIip
zB+KFKx^@iEQMNhGVG1v_W`xa3F$l#&o9dPF;VRC9|JH8Z$hk>h?@#v!5?C>AXm`G#
zon2dgr7s|y6yfCBm{T~=^l5S3isD~7I2@DX8UOMG?5|}&Ff16fuT<JC^I>5B)M$1)
zq9w3c@P!VTOY-wE&R6;g_!FlKHMew#+hSN)>LoBL*Ef%)W}32{4Nx}^FYe1(QVFDM
z0=Lm&xKx;sZkQP{VMx)>RXIz9ewVJ)+UD-XH_FXh(ZYwE8Ha6!PMdhmYl2_!@pQb?
zE`TQxFEPc(d+m&WV{IME6=Rku&Ym^HIgIj1$2hjiuyY_H&9dl3rUt$Ce<s@+2)iHK
zt^&9%WKiF5X9CjXdaz`wE+0){&D-UnflZBSK9jlr3Y*shnBt)^>hF!p@lQqn{Op<m
zX!~e`Jnk=s2#RhJu%-QBK1IiV6ru9Vr^Y<GP=e66dmT>xrke0<#gR!<U4^%1fS4VE
z%q)CM8?3_EQ{xFn$d3;vtpEI$ALA)NkSm04A?<5BJ*)$vU@hzfMr}NVqU95Wv)Ovv
z(o-r}LUYt`EY-I-#m2toC`V_=Tvv%2Tb{ID*Y}${Q<wr9N?Da@Kk)}el&EOLX_t7c
z47KP&8Lh^Q=v!=&QwfaKElmJvey0?eBkMvdXpK5KAhpH2uRfi#ouVYdINg;OQ`#H(
zDilnsgOcfB{Q^EW{ry$)n`{bS#9+`&m@t*poSkHv{Fkvvd-0Fk=)IbdJr~IufsJr}
zs?MaLLKMV?Hy#-TQ#Yc~W`pT6+b@<eH%zA#*xUG2O8>hcZ68)+-HC4qd19(4#%3&d
zICV^U+<nCKm5O_v-AG(9%ssy4%Yvem{_JM?+lv=4suPAs8=T6dc0B02QB1*(Hs-qP
zV)iq1wl>cDYBKuCY&<8Ye(vS3L%rHjS|I3x+;10CPtx)H$3?Ff2(tDbwv2(kqC>0s
z<?(8>#8;tT(5rJiB}iYG)8pM`8PGgNwr8uRINFCF=~%o;=a_BrbU8Tf#%Vb9QER4H
z`^e$-UOr2i81~LfE=I8U>&@u~lIs}*B2GoaygZb|czon!rZ~YjuPgO|M~P`G0nww|
zWkM2Vv40dzA4tY34D6XuQf@zkk2T7Gy2iSo1GjdUxjtMRlk1sC0d}uK?YV{=*>UEy
z(@wD;+Ly|x@9Z{dh$2va_0KW<Vd3~n$0dD0W!vv^R*RTAj!rR^uJdG-5yr{EKI5t2
zwj&%$?E`fD+$JB))L(6}l>Okf(M^GxIrRUYkBxE9ReN6;v;KM~$LV)x76~z))nq1>
z1WK&I@^CVHLl$y?({IfZ&9Xytxe8A@Y57Ec!T-0fmx;mlyTb(N>`l4}{ooN(4N!U)
zb(B9nV(+o*q(-7#s*q1DuK#{6j(An#JylS(WgBSO5uh|bb8y|EuE`uM_c|i_M8C=7
zNPjzemgEhYq@gmMi5)8>@3TT=|C6o`W`}*ewp!g+h=01eoHrzw@sr{sj7zU{$VN+*
zBwy7GSQl0bYeo%D$4xmLaTi$|@Oz{uADdS>C;ILa^2V|a$ZWP&Y}<9h9O%(`1Wb1J
z@?!YHbjij%LR~BbFftuzHW7bGL|*+1P!_xGVM7?p5_S5U8_C9%UNt_b&C}LIAkBlx
z*nVm!Uz+AxEe1v5O;<-OocedPJYV^4u0LGEhZde{SDk}9%lu40ASkP!b(jK)BqI4e
zL>*u;am`v9?fyw~pM<^%P$H%57QwX=t?bg72UFpI`~zhGI;WEfG>s1-<OpR!EXkJ{
z#@7zTk$0cH!9H;8lYbcaTn^VSXOCznU57ZyJ8#jJr?_rp6(|}+iDSbb#`VMDbfKWp
zWtZ$@AG12pD_e+Bm-;?|>!&3UnRW8oJzbemx<wfWmVk)bG?_t*F4jmXDeL`go@`+8
z4V&`o1f^Yj4W+3U#uDDbw~o|#j~y#X9o+#6B*O5KbQg_uy`dB1NyDF;K0W*Hwn3Bh
zV?U@s_8^ze#DGjpM%)$@0k=fsm^B(z6nvdebTCc&_-(-}9OApcXNXg_`03P8&gnE<
zzdM7_^2+d4vF!<AQB>}2QE@^iWzMh8U>a%5h!YM=^=NG3^{i-}rJY$VL*5j9vps!I
zDS2tF8p;gmS2eBCI_Jo%16N$s1R;xa^%r5G)#M*LxKbX>%3IDq2%}pu4n0ewxZM5Q
zM;d<?xj(EqG8gg#%OOOmeiAD58DnAySZuyIRFy*_b^25Deo;)0*qUV8N}L%zq5JUC
z(VxWn0SJbXVfJ=gd8O27^Rp^75)^CU@8GYVBOB0hnqHFCSZtf$B1>!l)HtXPL!-Ew
z6#+-b#NAd%gjE>G+j>)<{*p$9Z|E{?;hIDGLRR7IbF~wi{Uegd>D`JkOtuo+2Yw8x
z`QCp{osKBrs<?7o<*kFJErvToeCq9&a(%)3IAxz%dds4VQL$(zpV!nCwZ`4ZmTEc5
zVnPvHIJZKdqBAJP{aQj$vqdHQQe98X!v6h(E9i6)J332DR~<hWP>Lsh&}I-5g~N$!
zxIV*as$>-#W16^0p`*fJ96zo43={-yt->=mhGMi4TbzR3tE7yi+Qi9}kC&-JNH;-D
z37}$4*c%=rkjHy*%wj<z5(lCiy^ev?+W=EcMv&NtU73!(H8mp<g_*suYK&M_kqUOu
zA$2o#I@%b(QIaWp!!@Nt52gwYnun(}S^l@``u+5pd$0LGep@R${-Wcx5ef@*d!-wX
zYvUUe%t_%}9Dlz+JuY?{B*DsEx;qE#wwQ$@`Y%?e$|U5m<5o%-q)BpV_K!0&2j{Go
zGyB)?SGUBB(WFrpP|fPc5dTcK?GK%e>l%$own#h-pACb-Dgg!shY{a@eu2OGDT=(u
zM2S%Vx8$0QW0cIto(^V^5AYb*qx++S+2Tf{8uBQYMt*kt#)CivYfrsT*ZouePXsS)
zqhzs+KGTj02~92{*l3T$1pETFSt(aAAXXBefMBw*FF?%r`Kc;QIZTBGtaC19`bZd-
z`lN{ZEAb-{22BEEM8D-G_E;i;@^Hko%Kqj{KtR-{`#Psi`NDVVEn-mjbDwpNrXWlz
zJL8+CMn~}*tLDc{Ba!BDs_SkAP(QNMiW)O((tPMfv>$ct8QuiN#2Fb<*W$*)Bw(Tf
z9hekMLoneKDn^jF*2Zs6DIgx&{5^IVp!8Z?nzUQ8A3=^n7+#4Zx8{N9)MUjT<NL6c
zgVrfI6IvfRuF}}1C4-JNj~kZ<atrRlsY1IRWZ%SK#{sP$!I{sc@{B(mPiWMnkL7n)
z2vd;kbdx2znYU&&dJ?FL1(z597Rbw%Pp&cmiq|6(n#jYPUkeho-o{U-Z?7$dHvzc-
zjk{PT?e{bJ3~riR`^~;6a&i&w(M-X8%y^u{{t7#f+4N57l01!TUVI;P-5>?Pvd6gL
z>U}oW0%98i1^EKG%>5^0dS|@TWzF2K@_?~po=R~?DF5_o$FLFU9>yA+Yp18tyVEYn
zyCCOv_=cePO^HpFBUDbx>W?+v6R_UwScLsv$q9b+ZHa2OvA`sR{gA)RY5*qvQ3J-v
z(?Ur;E_`Ryo#8(8h1zXQaCgDM)+p`f3I|&S$J}RtZM_`#p)bY}p-Vb(W44~ekXrLi
zI6F>8P7OfG1LKlfuOI|-eF;0ktn2Q#E6gsR4H#QSFnXJ9I>_cd)^s^6kYk%~nD|pw
z6eo`r>Sh`5DruS=6uWLy@#L;yZ-&IRkC9&wC>~%LAPiL=I`!9pXTuk2;{92DppYzl
zj9_iEhEy!;R-3&aslf3*tm-eV853eTXD7Ylvzq9T>fy2#YLovBl=oMMa&5GjF>_2F
zM|u@sS`SHMePTG)<(w`?`SpA#7c#w%t(|uAO%YJAYb|BYc^y48(;x3R##(&-#K|Ah
zi?Fp_ZU9wx?I{T^M$n1c*<B@qBN^$Wsrwnn1S#T@W9dJ%71#w=b6o{I^x+YuAgH5=
zUemW~OJ!Ip7J6~4C@os_$M3sm3CgyYhM76x2<o574Q9tD`OEzZ(8gG3Jz(;p;!J2n
z82QSjnVe@vS*3P9RMwmK6}93<i*!;n1+}PTAsZ0f083${3l6|?fI8o52_SA*wvR6?
zcFafQJYKOhr`vUA0d;wrYaglQLIp-Q_xzz&uoHPyOalprQ9y4Pi-o>!N|e>c>#i$d
ztJ?)kvZmoO-MX)fuBX0r(AaYg(|o-K`>QQ+Vm5uN-PVKh=0kaZ4jfk53Y-%3PvuB8
zaQ*EDI3&HKIM@h&WTA>2pYPBd)0gQg_2FC4NJ4BeAFA()!d4GZ?lp{m%yQ!%y?5mi
zqI&PKbzzo|c_sei>84yF-zC@giOI<D_ha#gIt559I!~vj8)PSBB{FIfxUfgF`aUj*
zhNH5=0Ms@Dc;+7J_ygbfq4D}?vl*WCHb{3&)=;Oyqm22v1RKyiIr`wQx!8zDcP|n@
z42B*$$7A(#Yn18ERzf*V`v~c`4vqw{(W7%kuM%QzjP8M}EZ95-tlGRE29hC0pAS$l
z^LQ|%VNoMOcl#F2t0*05?M62}Yvm=SSM;<)bCH8Iv#A$eji5l)=MZ!{6t_~EO*_er
zNMY}6lwyk!?m$Z47cqZ7zB#4aLu;w0P?zXW<c8@KzH22Cz#Gl0q4wecdo>u8C4e$n
zRE93d)FFgnr4rnS1ul4Vo4&pzl)PO!(9XUFu8GN9S#MvG^>>|UyZ|w-fw4f~B*&FP
ztniS_e27Hp-%E}A_=DZj8^#EDatQYE1%IW!=BBjY{}hT%`4Qkyj=nPVF+S}u3_7+I
z7CzWQq;4yj<Fwan@u+&dir`7Fyg^ucPm-#S<RRJiss;56JcZ59w@FU(Yr(weLO40e
zeVH$4bAgUmgl>-d$0z~O?EQzH=I%K#R-;%gA~T6baDIcigs@F^lyoOr!0x-!4S%+&
zPpSxJ>2M@byE4DPHCR?EbRA3pGSKt0qr5~{fAPCE`tJe>bw;kGgYMkc9~h=ccA-%?
z|9pXB4$5sR1bln*m0fHpR1?6;T!37A`2f5Scr+<b&71yKso!O}K4V^fjA_?qa1WOX
z0JyGLx>I5*s7!@_v;N>D^SoYlCl^}_t7*4Rzl~}TBoRZ2o|KdO5M*dctZh%$Dptz1
z)LR7egYk4=yrOKK3{aI8dKFG$87_`U98}nu6@aJgwFm~$3dSzjEz>{Pp^U+GOA5m3
z&k(=B2{6n!H=klM$GAHQb7i-V=S8sxnbGwrsvzgBJlInqPmOvI^rOCbBmoGi6FSFv
zw=j6Q3+q?-)m^1R#m<sL`(vpzwqTDR)<RW3m5s8l)8Fuaca`SDNjhOyI*trSTHp=e
zI^wm<ubsCwFcl$LLLcW+dc)n2W4jUng1+Kvm=vf<yS@Dd=yajOD{>fq+w%|<Dyt|;
z;kmW`s<bdU-51i|{B0<K4Un*zp7ENA@F@*Q5>9QVD72xoE==uR8XGvqw=a4YU_2%D
z<5QM|!`L19t&np`r&o|I;IKn~Qy*f*^9sc{faDks81%Fyo(i-r2<z24>f@Vkn6O)5
z{$ihG1yGLehzMIvKelwP#wM=A^S2d-H{@E4RaEVt=BCyC=iD0mgbtM=STr!xt3)9D
zDtf~y1hL*TLehoZ8|=p-ku#R-yNvR7reT!Th$(r6&*Fx^<2hsCp#CCjfyWQsMx=K5
z?d*9N7u`9zzBTU0<gXnp<l(ff4J5{^>&XGk9h?C2gKs(Gh~$A!0cxp)XZo%>jm<{b
zCOnia?VO5XP83R;*&(T)(TCE-*<2#2hqL4%G-PI0-G*Y$s7vqgg9HLp55I|w8i1A>
z(Ap4!f84T`$gS8x_Y2W1t-y&?H!N}s{d?>-CAh_zacL%|qglF*MD5LW$q?7@=6ozV
z)}P_Z07by!oT<ZpUDAlnNG9yn8cxTN5yb$~mqlr&l<4(-nAa(p!pyhVbd-i}Wy8?-
z*T=SQ#?zA-$DM;-<mS%MKZV)c3vi$9TqXkMitU5{j(-HSsunw*ENo>K$@D0=Y?prn
z1fuZ_HeN>-BJcr-aqFXTp-}UNsiH#8%CZ9E`A?DvQ@vk-5x3SCr~3j_<W()m!D7X?
zr22O^h}#tCdkV+!R4BHJB66X>@r(qkGtmFQx??Vq!`7gFS_nAw@q&FKF*KA8VVNkt
zkojl#BHG;1uC)ZZf(fO!xN?v~j^#QX^qU+Bd3CPI;Rq5U%<FQSvB0^5ZuA-PW~0I8
z5QR7ex7$qYH53Z+!AP<^e~Qy}fX>q;Up5i$gTwK`gYMKz3XZN21(FA!m%S}nI){K(
zfy4+}dpGFD3|qm?Pow%Yk$vCM*kQwXO*T_S>Vr50Q76k7&@8qM_zS={66o}~9oz#w
z)YBjg{sQIYRt-2t*lu6~Y!uX-x37S7<#)R2iEs3Zy7^KJ$yd@Gcko7lNbc$j+(CzB
zfRZB;Eb;8yJrM`eOmq&u5Iqb0@>`bD4vx|*)|R*1HV;a^D0~0z%^#Fn(e@OQu1cnG
zFHfHxSq9Lml+*s}ds>%t@YI${8y&Q`CaQZ0AUR?aLl8jYA;ZJzJl#c9>`DC)JR{K?
ze0{`)5WtNL<bezeNr~QThMB=Mz`K0kTS-nmx=8a9xjTq9@V}oPiCpjIry|eih;G4W
zycdt+*T{o+$p5_GGb;b+4f-FOOFBYWh#LVD7=>zVaMI0zsKfI&X$xu#N9L9{JmqX1
zOwUoU2yLbI7gH%U3C3PFHcEeh)H-J=qYYSBE9lr5Jy*J8khFRNy(MIHhQpxr4Oy2{
z{in|}G;;g58Rq%R4vcexW^2KhZkj-+4Va^XJCc$VLc2nZT=gd~_GeFssaeu$6V%Nq
zwtX2JHv-cdytlwhQN|)@wu4K)c$^t9Mo8Ho67XthsRL*0j069M(&&*rfKCG2v0Eq)
z*G<b1SA79Xdv_wBT5`FrNf8pKEn+_Z^+WQF?+x0iKWkz=IrB4>NMf$sN|0Ch|7??Z
z=q=wgxyTZ+`<kC20ny7(Z@;R@&)k!0BeEg;3C0xt>L7Xj*RYcK8MN%f<47U@g|Gto
zvcqxWResmsyr}~~)=30=-yyH~hWyeR!nJ_qVdtkK4|5T;f-~D7GP*vh+Hk*b+I5?`
zYOLV>R#8gpc7`16P=B`UmD9!kSgwBgpI!uCHJgy@5$i8Q{NlFA!^RUX072-p2`6pt
zjhxb6G5$u)@A8zXt9R{OH#|=^eKhj+gg42L!E2`u*b*?Q>e%a`)UQlx3b81<<KsYh
zhX8$`z;*^yf|RKqBhd0P(2NSP4#a8U)2N!=eJ(tdp|JZTHm~<s8-`fxIfk4iCP-*T
zFmP2Yrtu%`N<kZ5&X(a#`f@9jRb}Q~ht4MW+e58Vk?p(e3y;^Q&3@`SJtV337`4pR
zg{R1Fv(67QHUS2(bx4VXDDD`aX{esT_Qu~ZcVR~S70@269G)FssD~ntI7UNi6AQ+#
zB9;k#!*gUt;eJm@E$m=c7})mKUcTt>urZy(4i$FYbhq6DtL=gDEwk>+a|{*%I#l}8
zE1cM8uO%1FcHW4Wp-SBxMI3+u$8NE27L(0+9c+c$Z|gL&M;?%%*@mnYAVl+HveAs+
zSx8Ts=a@_)G^M|PON6v%laPy*@~>8>j0)C37fS~}!4aij1G~4qG%JO7dbuVNRvL)v
zx(dRcSFY$=s;u!p7v^)o)>E!*Gd{CG-$#n>ddy0eMfSXl8zLA<(}iS{V2o9~9m>YZ
z%ER{RnJ2Rc3da;5u_Fk?Sk_}IPjkk9W~Sh@q+68Le~Yw$Dh9U;hwY16PhJJtTG%t2
zrUJDG-RZ}Ls0TbhM@b!@S<+E1%+jom<I!D1mz)P=iBFe2q#q;Z6N`NQBSLYZbAWEt
zxRB`MlHISts5jjsdnwd&7>PbOI{|Gp1{v}^qkPd(x@c|i6m*wpdnmtxWRB3yeE23u
z-j2zO-UUnsV85ZXc3b5~23Nicf>|B9-Y)BHs%fgjM`wK6xmU04S!nn}^VZ*=4JxYm
zLG8RRWb!Dd!$!Uu?XOV`Wd1T6{eSidPNX$?zV_zW#m~{5x~L1H>c}3}Eh-sk76JP(
za4go;EQt;#v}LqQC_`ROYNe7<b%58N@cSkgiJb_%<KdrmBViACFk<I!a-5>j7G*$b
zB}lJIxBQf0eGWOBVGY$RUPHpvlkP4TCn_^a2>UdLN(11|0u(eoAS^x3wqtU$<WRo7
z`nM%oUi)sSozSs*#AAXWzobey4v)9q1;GO$9djs16rOS6w|Q1xcHRo}JFKA`x8cq@
zm{>;|l3PqQ%PD=OaQ4|l1xVge)`F!&;USvPx^v)^tmCXA)dJX;F+32u)6^?$Indii
zOS&}5UnqDqq)0bge=Hj;G-mk8D({;(Plpz^!30KH6Y>FZ#o*f3$)e>>LW#oAxF}PJ
zqJcwwTL+944k3LkfD48-WCXr)ca&Ap+e`U}0#4zFD0gmW^8*d^4D&v92<m|v4I*8y
zIB4w;;I8xSTTN&{yY;q+P#B#mTVCe>VZpuapgs#R^3T$p6>D$nKh;^v6S|gYyaq*|
zc~KrBObrUu?;8OY1Suo3FegV6*ln2EM|YciM9$+92>8I{W7_c`iL9yC9)T+0_z|Jj
zQ3AM{flhG%DICU!>e=qjX436wjp2hhM+(@p3<J6U@EXVv!Ebzp&rjTwK*rMWf3)Z1
zNJ<il40kg>Y*caR?4Te|W9Tm|_=8TB3B>$J&R-sc9%S!O+9BQ&EnDecG~+llI;L93
z4THq-47XYG!eYpi6sNEqs$eYTKszH05`=O@zyhGA)v?3>#%F?6g}S+FdjFY+6m1Vx
z9H9WTmfvWXrvekSfA#D9zvtl~69;8#>+w`cgpe3qLcpu7Ox@fS<A69`x$CbGaODT9
z9?}3jbfJ3ilRe_7WUibgn64<v3jX$XZ(<rU1a5{$ImIZ7?SY)$U%Z0FVn^HMC5YpQ
zAVni>OFHqdjXeC>lAPMue3tg7KKNJi3sMv&ifNBSwBdjjYWhF+yw*7#PFB4<F?v%7
zFXFm@DYp4pq-{=6Z=_hit0r<WpBskrNCr1@%uzG$45B?L9lqroorU26f2Vl!e4%?Z
ztB%hq>mlEGWJxhPHEzx8I(I(>7?h>Kk25;JU7>&jBGM>B5H>mQ%(b4|TWbOmetlWL
z!Fbo)Gli<q5XcnuIpTXTx`{z=mp!5C8EqJ=Nu(DmqHGKL7uk3Jloe0|X=tbLr&bM~
zcLh1R4~1G2n#YHCjmOe2N-jieT-Gk0OjIW6nS`t9rTwCB7E3$%@wmiB4g)^O6^Rhh
zfoN$!I<y=Jjm_r8i0i3>3+Zv?8WXo|2_Zi8IL`X;RYyKc?MNnIwoJ`eiPu331!WVV
z6F*fXnaJGe&?z%}1N~+FqK7|f+Jt>`u|lM?isKqPUIQ5<7I%PK+DSHdJ>h$JvLwM>
z`#u&Z2hj;P0&<L98=w}qSyOexmS4v6+|=1=GL`UjDsz;00Pi?F05wcP8*zzlK1+p%
z7QUs5J=jBy8C|w(@4Iu$lCu@p3{r&LvZ}4ba3tXojSvybYd7CEL?;u>j1?<#hc2jd
zu#N537baaL``<_CeALZo^Ag}gT01a(jAZL0p+0ecdC*Oz1S2GC&^?DG$1DneHL|`P
zxwnfU8-FgDpp-z#nY-ttM1jORhCw!$Ch_%u%k!FpAeD47GJjI=mcNGy!J#OQW34hP
zM0<y6&z6a*&nW|{;)`w<>i6wVqAtgSBqN#GSks5>oHHT~s)-tviQ)9ovKBfiD@O7U
zy*f}=p&uoE61SOrS7KZmOtY|NR>?Rq@Wg4gXIErO_FWj+_00z&pQ4@hgH=2$TxuhJ
ziH)Zt`D`?1DMT5XLV`VbIB+1Xb<0`xh!bu<lEv)boPd~YyNMDNbEdgIv{w`;{Af!^
zzVNH4iGiDT>7_2dDN)(nB3vUaXCr-~-%qmnO(#(Oz9mbFN|#33{o#PL;XV@!WX9vz
zvP-;@lFgQ><)0{O&RyXU!?pjmc?l!hp1#oYc|&1-f+>#D_)vi9<46;aiJt!Pt&9Y4
zLg$E6s(*xvR1qWF;LXo=I>_xc+h|f7xO<M-+~%hsk=vj6kuK}LefdTbu8TBjypQ@*
zGTh8Pm-8fakw$Kigj0zxFOM58H!rC(dT>*Urdt;EVTXl2T(*|mQE@XNP}=)O;*Rut
z;z<JY!1&2>>37H>7alG=9%@EM%>WnAKie-Z-}~1bvKLh}b1}fSC*P7Jpi3N|<wDtr
z-O!mgZutDF976mcR3U$*!fjnA1kTa>{bU$~`UcHf7}_NtN*}s6HzkXwOrL+_wz)fc
z42y)VRtYYCb~-<l&F0pj#h_`EU$d!>HPn%PnXDNoG0v2`$Npz23Ej5_sdth5*N&8J
zLj(8tGP`#4GZYaaPSDTdRzf{zQF+<qG(z78&C(_ori^(!6jN@yikNA2yt2P<KP|oU
zXA?sVY-#F^v!^7iI2)1V<F**Qc^nM&#7-_EvM4i%b#PAcO#FlhEU^?r2YY$AiY6ny
zx-99|Yq`U&N04oJi=hPA0$GYE+lq{lUmv{{;mo41>OH&EVUyC%zyOYwKO+-iNm!4K
z^dRnTEv4$SS5ZN`TTg$;=d!2r@ct^}1;53uR=yYlK>3doNZAD!Pwx2?XmSaolp{!#
z1!gLY_Zxwc8*4LJS)k7cEU|xc87RfdD1N?z`G`FlY9ER&F<H&6avFr&ujTBFz-YFD
zpb=CZDH=*(OfNuaiWnq~8{bKm7uPa~3^f|eh{S==h{+tqSs)RPwqMfxh^u`6BiG!g
zvXvc5ycQv17#gRn^ItU4gP>wn2e<!Xoije_W28;L7BbUNKF0o9!u44cr^*kz)B#1d
zdrFmwl3poT2d(P<0;`By28B;}DN&$!Tw0{+N#WTn%yTm?eeB6=ugi!zCI+BpBIu_D
z3XHY)*27P2|LcD6%fzTEq*q4en2Jrpw#vXgXDUVw3qTG27_ge@D>ddS(rxXLgWy5l
z!*-?ujdbEZ!B9z5;X+Qm`k3#HN`}L^G=N6ulUq2QMgTQv8I}^6<WyK^?GdR`@<V6V
z2dxs?_B!uwO4D>)ZcOP@BP!%|_oGd0>`1#6iDEoFk^8lvKC*)t(Sp5d#8TDt>_M~M
zEiNOEa^%K%3k}A_+n?Bt-y%hZr2Ai=8<t6gaI?C9_`&)Jn^W*9Xp}+DBYP!5f$^K#
zgrC=vc`9xNW1F6CIs>==yK|rlN8cV*bGQ~#f&jKPdfwRJgu-62(Lo^wk*)5}ND7EP
z2lY=U)Gn(q9PqMF{`;oY$(F)Xx{;F+DUI9Rw-_=sQrF)lX6cX4zUSLDl4X{m{$V!|
zJ-m2dF~xM)!pK!^6S`gmCVMMK#V84;Sd<;J(oo_PQmV;mnoyPuU<&1-82(X&ZAP|q
zg~fx1z66oc)iJ?SkaZz}w2%++VVl+BO;aZ$klGCi7WxRr3GV71eB{P23T`;csmJYC
zp4uSAxC|o65kehQ!q2AjB_o6qybtx+87xJs>qYrCPV?o&k%9C^Bo$CPFHYo0+a0eo
zYE~La3SY0d907`M{&h*7ELkL9*xqE`)e9pF0BK7#%XF_g5OB1g9@;{8p01W1t^h{c
z($r7IGC(5h21p5Y0CG-XIXf8|4z<`J$529OG6VfV<V<9?>g(;#CV04zFg#S(9h)9z
zO-Wi@Q_z0ViHQ|Q5BWIeo=8{?wv9Bj+G%B60BC*Cm~nCC5s*STc^+&y(6(ZCL}vz3
z=PoCz6lR|RdY$gkQr!{2VyJ%+>;i0qBcRjW07#M9K5KZe?v^#IRjcO7Edqo+D?mnB
z6H2uyztH?5Odk_I5fKLO#To#ND$OF5*SoHrfk<oIJys6wevh_?+G6b?W)Eqtlug|E
z`XAnbXmM3R5i)Xwh3JYjAWR#_7Q#FzlrIx$(&^8iESA@7@PwR<AaiqfJt%<)|3E@M
z!L3}ZwP~s?4fEDQ<J?4!g2o7^#S2nz;Qx3@n%+kx>b1}rH0eu~D(2f%MI}dof=yus
z<?-SciU7}%RM}4DhYJsiUay!Up%Xf`$MpA|Zx2)U3rnm%cxNDu20aC2hAh}rV#foI
z^q%kl+Bhdr#n(JJ7}fWY#ZU@5S@~}@cv-<8B*9E5k}foYu>rV&li{VLW?o)z!3zX~
z$MU9ZfkIN=bc>aw+!hO)y^$}3NYaSW4B-YqojbnRU^PkmpM2SKC)Mq4zi>GWk1qWY
zV8*&#iC@mBxdHS(VFTcG7BTfNq_ZaQ{w7;Y^xqfy9l%O+KiQHCLdH5wa}a1DXrE5W
zJf0@?2KY0jz=)(23Oi1MvG>;}{6gP>u0|a|1zdU|t^(9nQ~ZOK#v0De#a|jMECx-r
zqUQs2V^2U+V(c3Z^|9@RxKJe&JoUn*+tt^rwKCTF;`#Fkpb9Ro7ppNA3b!g{2+jR=
zGTrig2Dl}2fPho@`QhSGaFstdpjgEmh_2ACLYgaz7SJl96G_NPn-9fzVEJT{02cAU
zxw`ckV~72IOwKPA-@Vsill=&>?|eZ1uWjBishr3X<u78{1>pDE%Sn4l`Gmo`Wo4pF
zX%-WYoCmmG`TZHd>xY=uo&zFg$EK|P{K@^zs&E2Ep)E%L+wE86fJF60bnFJWJ&@!<
zYb;&F5*8cQ{cMVhi<Lw(w*d+2iX1?K^1XNVfH|dcnMX{UA9;;y%`F$<A@r<8jjaGA
z<qST6Yc>2LX$2O%Ho*f>{?<Ehe|T|`;9k1`?mf3Yb>|kw1_M5iQ={fT@rvnu*T<{N
zg6#Dn(SHb(MK<V7DPJrYT*Ls&f&|gKy}W3(QG5TSahHF_P^mT?tl$q4U|PYH3r{QF
z9;RDi+zR&o6X!Wmmd95;wPF~3oT)(Sn1c96<{O&QB7PDa<u~P=QMBc*Kei<I=JOIn
zAfEQO(a-oapWfjZLWcG})<PDZ!t^$}LhN>x$^-taKHVL#FIO4|afgSzB?&R@0rrtL
z7p{rhKlBu2hJtvZctg#{O~YS2LnQfJ`n1-7j?Gy8fw)o;DVJH;*6Adk@6ASN6oFlN
z;|`hP`@y{fmxsJ|K0m<dAo^lx&$`N_H=Ro#Pibzx$<?ON04g(OT%b0Hx;<B>X9pM@
z^u3og^u0`6b?nB94jQ-<5KF*5yE)s6iY^}ZpaQtm|MtPnQ@0+z1owtP#8Nh|u?u|Q
zo2WLW>Ab1fzKI1bEIYpaht`8pi+Tsjb@$<wVdTMjx$%PE(`@m7N(o^~1J)**rlXE}
zSaSF=owYxQA%b8L&CZiD7hzE0t2%VG8oE+dl8UWqk!LEuJ2JiZ$5YSQ8E|2!x0z?_
zM{Btl6D|X|$4W8|gYj=jn;LZH0KWFg$D+4}Z7tlKAxZ|z^>(Y+S(d(lABbyH3JNC4
zX-qwb!46-w@5=6|2b-<1;mPkU?81_?)qkr^t~I%`k-ECNKkBTf`;OphFvnBpJuh}H
zHElWWn_jDa+H)2UkAW=&5}5EznU~fg_h87lLZ~3Wq``9gMxVjJzmYkhk#OOi$^+9&
zYyU390HTeZMk_@QIESa9k02PFSmOxWq7fve``X7D3@%t`9WJ7vd^Z+QkkEg;PsaOI
zV2r3a-#T5PYmcH|VM^fk8+VB{g`ze<3-9-Qr&FD8;5w_})=Q)%utg+F$yrdQS@wfl
z_1iK9&SH(F@@{Jbo3R}4e!4UM7|<9`?F9g~A{x4%hhpwg4*7J$oM|hs21u+axGkUr
zc;=L~fVl-ZUD|SkgFD5C=!QRn_grl6YrU`Svy=|PeL_y}n89>CMIJ?c;p!JTs<?F1
zT2hu$P<^-HwZJ(E3awlEf5JU`+n9TFd+WFvT5{~uuYo{^Lmdd8>3>N|wLvX|T=@IL
zpPd5ucXZnDd#wDWXD$SL%58<<RT}{7mcjj@?WV}XwQ}|1cy8n1Nl77piiUx3{&d+<
zG#-E#W%lho8(2zk<C`hMen(?do<rj{v9Kd7VAX@pw-9>Og~2XhN<v!pqNPr+O45IJ
z>Wv_ZcG*k)5xrDvJxvi>XEFLiGgk)96ns9TrtHcvC33Z3DrXt9Jt1iR%6lhVDs#KB
zUI#lF!vyiXv_{|`u8p~|^Vn95xIM66xG;>Lx~_?=`#k_$bQ|D->^wCExim(odM3B}
z5wo>%Z1LNblLy(P;bv4tv4~I|?JrtX(4K&^&vxhpa@dz=dMnhPu!7;vcL8>(sDDU6
zjm{&fa%VDmw*(rhwr=(Bbj4MPW`~zBerg3AU#lt-9Qu=QF4TJHu&RqtH6h&zO+`t3
zkJ~zYSjrUjPVen&eX|L*9PjNAtdAXgWywAr?lQXLw!K{~3J~GPIUv2RBbP&Wd;dbJ
ziyABVoHrbP42QeU;vKW}8&4Hun~Au)1>Fmw9PX+_Oy^KSh9Wwh*7%2|cxl{1z1uW<
zy}i1IeZnSvgv$LsT@$u@n0TpN6F&b|d!U|EeJN@j3A964-L&qxs3O7n16l@aR~Hsa
zDDm7Xw8s)&B@@eHPr6a;7hoFh!ofp{>a;ZORKdu_tlN@tEipq++HH}hzgT?`v!#*U
z<Dru?FZOzZPKa5;4NcWfFvBKqO=2guPy$;<<Po`2i#Srn;j$rGnZViK#?iA>_>7PF
zap4kDxPaj&Zn9()E+n6(lBn$6e&NqgPKPMs{dt~6T<C=egXO)-Ir$u8pLagmI>?3E
z1AFJjn8}r<8P~~0I(@?4ZfDWk%bMhG1;&O%MdxT$!!0%ae9vty{xpsf44`m1z~?j9
z059h1`&-QkhIWrS@A0RgP`lXz1HSF?*{iwv_Qm>Dl$Cyb-@Fg0wDs=Kua_%VcT);w
z#F6SwZpeIk9|Ej^Abp^SnR^{Qoen38fE5zox8OUx?bt$UDSof(J*DS$q=*iBnA*W~
z97=Q2capUVZwQ)PnBRmtUv(m692O+E*Sz<CnD+NQt!8kuDQwa_NV+~ejWgZ(aPoCH
zr(6E4wXNTRRmZ_0`gSy_O!<32RY{0f6a(-^l*xSZoa#{MV|UxVYbufAxFzes_lC&v
zV*A|;olZ8-O^ju4*bPYmss>S=3Dv>cN?Yn#a#VT?(^uCbSP@WYRsW9B6_A4_;1J&n
zP?fEw!%3nzbM%nbbKxmifKVoHehxZ<l>NCzcit^qAx)p_2L1hcRGn0~cp5sav+I`^
z&)gA*SxV3t<xER(`#0Tc`R{16?U-SI@^;GU{W_mRuLZfswC7xos>c2LjP`m>H-cs(
z7}j~YkyxtFN#9|Ja7Gkv`^|K%=_GeiM5y+#YX|=wyIc)zA}g?U#&t~9Ew5bvEWVgu
zFg&PVE^)7|+OhoJP~m^eVrtSlTn{`|88&`Wiid7hj(&J0-1bAJo9t(q<@`6zW|Q~A
z&i_)PQrl*b{jMS@g`8)Y6PayK*4I1Bz3icvat$)gcZo-#i^FB(xaCTaEH?@D1Op->
z<+rvNT6R@9e(9Dt`b!IJf1d@<sjHs7uReHhT%?k=5oYc?mH+%RW{%VPoS-I0WxKxC
znCWW&d2}dAq2Y~9{i5P?n)NhsY27pnVM8<9SEE8=<p!#w1MU7k4cp`MPgg^)&b}b^
zSvNglM8Vw`3lv1Ilc!3<s@!NJv$4dby0ax?KLWdFG-CbX((X)~XaAX2+uJWpSuX3=
zx+SHFOy8;FhOw^pD&JmR%dvHa8(Im~v|Rq3Tlkgq_Ul3Yv0EVRi68%bv<V)f|1}X~
z&66T3pr9hy<O-e(tG^T^CTC%yqt&5d?Y(n%u%KgPt$w0ba&02YY-;;i)=N#_+<8&(
zr~bpw$ORks+PXw?_l4=_j2&o+*;91XuLX>TTIZ$|?{?wN^TL9KgaO5VZhnexeR*e{
z#`us@qqdfh$Mx|c7kVEu^^dQ?%;a-SrtHx_jK#a1%#!bgMcFGAvmT~MWX)@95&r%V
zalon_sy|*j87qm(r|aM}NcT(kgb`#6gqnc)z%-3+)OoR-t~8wA&i<f?%Q2>fd|l8i
z_a8V`9n+3UcFLW$vXY6#@<7?Y5o~50`1gSr9z=wz0F(B&U{Mt?+%XutuYQBBQ>PZG
z`}G4=iE4SuYNSLT@NUW4eT@00>2?42bPr+=s?9d1F-e)?6{21m+}>wN7f0uX@_NO&
zNQa}%%fj%X;|#2|=x;EV1s(bJC8i!$wTS6dUMph0tmu}|jh$@k*PpiQ5a}dj#*<R!
zLV?L{2hKLf^OjkqHa3Jd$6doJ<4}XC(r?^1sY?07B<4O3ThP`h_b=+}JI}4Xci0(+
zA1qFM#pjV%SB4(NCgjWoidKKj^9bdvKQ^trA1jAmVKFZg?Pl658?TJah&PfUADitx
zr8t``FXlYQ$H8U6d1#%1X>Xq}nTRpGU5JNpCJ6T=6?W3#PWo1J7AM#z7*#}|Sy7gO
zr71Xc8L!K)-$uTMd|4%1?39*&QZ?=LDSlq)r^NX(x3GUS6t(*cANAcslSixXom$_m
z)T7v+to`@*W}6`I%_d?cqrg^8YtIXT&e+v^>_yN@9+NJ|bz;KLuPm9sUD%i47wgJj
zuG8naxpipkhOqK{<#GQM+`Ypbh&L#dsLVOGr=&P)dh24|Qfq2i=J>|-3+ekm({veq
zrP@akZ_*DNTz^^ey9*2(r~}JyclEj4`Sq{`qP|l8uC<fzfOw8oXo{ed<vSjN2um}p
zS;nZz#S`>T0XOP`I1xXq$n(g)Pd1;ffeOqCs~G`+ciLO%C<+9>ru8s7Hm3h(x;YS3
zNH}<cNYLiD`MLRWwyLk{KC|y$#`E9TvrK6geHy@)p8?ZT*R<Qq-5Er?KBjN6P6O9z
ztld=MF{OBT1kIE)Y2fXzk2aNA_yil>!@QDxC3(<9WVf^Fx>vnsy%%<s#M9j-qAeD@
zRLN?iczh@h_OV)tV)qRt^(6HT>%QTNH5Uf0r4JAb3`1Xs4rKAs#jZ>H{8i<*wDQ>(
znc}?1{|eVTC0P=Frzv%qiEVaG*NJYWo((9lwGsJ@DgG?4KOK)uw3W(!Y9Lr_e6L|S
zvbCP(wv@%fFB~VYHQcB_;TqKD>NP6FSN=7|Sev+i)bMFWtKdE&^7Cf<Q256On?DU9
z^Yz7m>%V3_#$9x6hBuCoHEEPcnU7QZEq3;@3EiA+spsb4L}JP+2HO_F>%6S(i6fZx
zVVmP(u7+MzJ7PcE^~j{E7FnwW3&`7%FBRo|T~C`8TSfONQt~S=@+Qjo5G9zYnKdR>
z5F47P_k!SG8cUAJ@gaq4jv3teH?QH5c|4wBzCmtZ&jPQ`?aGd#=lyG)7#V`<J4WoV
zUCw#rxZk>k_^(y492@k0ye3kX3UAF2u1YJML!yn1e$P&EaB?TRzEd|Ws5&Td>yb|I
zrHQA^_=rrM&rMz*?8dt#VWphb3Jx*|CS5uFT<GF8-iwvqIbP0VQ-4^mszz+u0ag=H
zO>$?rQ)kzkNj;n{`0|Evv9nv=mUXIeNFeX?do8zC21ZDNGMU8#_J%;cl`L%|V&`*q
zPAr9j#6X~ESc~~<xd#(-3LH`>!7cI>R-mRtH9bv7xpg_t;H;#;`-|0kA(+_bj&H^x
zIAn}wLe~ttTC}5${wp&ZAForZyzl3tT~`VCl`B+02UOI~Ueke%6~8g2<n+;)@JY2E
zbSiuhk(pg~E@qOOmX9A%GTy%nYX4zpkXH3-u)USjqP~WHyriL_a|*DTRYYkGH|*MX
z=k*QTy1F?n=DR6(<I5eFt*2a_rDJ$k+0dS9dRgKUwY$X!%PZciwhHF$(lsqr{6Wxa
zrsI<A0De>D-j-q7S_#I@&~zKEv5`(<MV;x@Dg7256Bcv$b`tVHya_EMr_%#RzdOPm
zyF(8%{ZJ^yoo%H@pe}(jndREu$&A}?KNKI_(f&L;v-<h!D{BY+C$=}cy3-!}OGj5{
zH@7K})v@}+m6#kg?@Ij@^{uUPI5>Rx_cD^|m^Q(QB3AnF`PN6(Zpz8XeTJ=zbtl=E
zJ~rvi@~kQM5zX*RVdRj6$9N|jsdFv;)_&o1BvP#{FMXegW&hKxngb9oRFQ{5)NSVW
zeMx!B;^^Yy<uS!q?^Es&HkQvEqi%<2BBjOJUOz6(JI>frS&4?R&LZj3`7GFm=kI?~
zulyRXWfzEBd?t^`k?z-8TE%UCM@RW)B!;U!C`poE^7%o+xbWRO&?~z616*pj`cMt2
z)6|>VfzxRrYMrSn|JPt5o<;c|6b-_7$8X~T$HL6kL>ldyWA0I6IHM}IY(v}cKjbg%
zmr$&#`NRpf8OA9*`^4{9Y0iByb9bS^0P<ndC%^G~noK_P#psS(fxK}#A5Ex)7}_8N
z%CSJu>l>o=r6EU!C(l1o1$al{y)%u|09B}8z#oZtIX<3oqu_?=Es_uZ*z^}S%@rOj
zYWF7c5~j4K{X%cQF146eeCiL}I9y8#55NIsQb%9J?l4mCcP)+%5Lz0)f)GM5Lya6g
zz352qiB8biJhh9-Ue8zfIXi!<JfMl5eL;5UU&G7NCY<KV)(n=XI!~5Ln=|slyKy)7
z@y_&JVOxC)9xLJ)GJbWHHBpAgtk&zn*l`?M@;W*)9S({67HLq5YpgbNn@*nX&>!V5
zg56o1Tr<$9z)uIzHZ(uTj0m0r2MA1ionfU$pBD=isn&DBrbjtPgfX={{+3|hYgcJw
zn%QQ@_vHR-D&ws{_*Wwi=mv7q0@4<_Y>g;XGXhYi68(H=Itk>0IfAQCayqA(RwXk6
zHM1jB;SQx|+m+TD$}<)Ihj-NWP1P3Z9^an!PLLNnkWq@fWQ^l=pA}_k@KEz}@yZp<
z$ZaiMMd&FbLSMTTub~W@C<lz`TNbGymWUCU+Xc6{YT)!d83PsW3ygzL0F)v<uarN%
zf{+jxfy1|m-K8mT8c}r7LTu6(IMFs8EWbL-{wA!+bYP5tAtqnv;OnrDz0DDP+SUD@
z8g<VefNUm~eGSZJP#&l{&JDQhv=KLD1Gb3w9V;=;fhw%qu}p-^StmIr+i!z7*9VV|
z1L+M`=o3K>e(PhV7y6JN#wPpDsie#cvhe3+GPg+T7qT?p5Cz<v?!)h@H)WEfNx?Mh
z0hywznO`=QVX8b3Jd3fVv0t7pv?sDT2ib8ttopYvmm>&3?|Ob?e+@yUM}U}E{oxB+
zT`J3Vq>H1b1^jVaL4)|AaLM+6v0Kq6CvoOU%y$q=-%ufa?{za&ASe{cwo*fk&wz74
z1{z-wCcFV9aYN#_GR<0JRsTL*(rV~WY|}4|C-2wb{ref|)g^OpEAtqv0YQ}s1A+h#
zRBQq?Rs!j8aZtE^C%sL4>2!oiFah)VCZ%Zo($xCjg91plE+v5~^#2~Hsla*1#=Q;t
z??=GzUXFAHU*!M$8{mT!0}w1`W<xYDjne-;7(ZwIKjYv+V$lY|F-cRo{u|-{j0V6L
zmjC<TF9QP2HD_9BxdP1p9pq&+;FL)J@A&{9{{K&j1`RcTd(>1Gqu`CC;v+w&d%6DH
zPixl_?O&D8!9qX&Au^|b9%!8Nbl&}N=Q4=~oQVfx?51zSRRn-er%3>rw3}X`*IeU!
z-V(2`c*6<84Q1=~{R~e}4OnEn-EPOCBqt2t^!?YquUvJp?k>|w`&GB|A7cu(bIC+^
zSo*Tp)5%9KYK%t>s#pPOmaRfXG@sj?v-e?MF==$jd4X{h7vJ8F=9PbYS*_?jPDs4Q
z%i_le11n75sPfffj8>)E(Qj8yY4<E49OLM+!eTio!@JC)adWQK)y|uWe|OpN{<Wp#
z<f2@L1dwbmq2Pe4-};x=!*_SP)8wtGXaK7pi~AC%J?1KOGBkAXatA|%>{c0fzL~F0
z_)RCh$T^K6KX+yozC(Ndv6Ub$Jc^XMGu|InU|mgk695=>X;RSxu;hywz&RcIg}C~?
zmxrZK59?lV58WP~i$*lxKLyFJ?9NuoSsyIc<;<3APeoIRT9_8F*Fmx#O+uq7crhix
z(XKXMrqQ>R>tstBmRJl&s(~`#PPDJ<F;NfAl<e8X!MoYzG{z0c7)o`HoZU`O@yPGC
zM98_WaNL)fli5sS45ml^YOtCwyPddG7+p)+R;FK{oWQ`vS8b)?btQ(2ekrXhcbKVv
z8vMq&YBO9xKO>19DF(I=+!tCMqFuz~%Rgr|wS{aeeyUnKQ34-TV*qKGe7Q)!keJZW
z6{X1GVf#(`-<QgNjMt#@Yviaw{a{8Wd1q3>>x4BRD(HEDQXkFNbLMxtLN;+<{$v5G
zHu>p4jQpS4&&Q*-fAVGLp1X_Zvc0TVO7)hO%9<%HkB3~88;*D9>(dNx*OJPXjXx;~
z&z{rajILKnY&+YUURL)W+6ly0#AMn%-&#Z)K$`ZnNA#)|E#B!`%5vaY(f6&EEmTfB
zNhFo8TJXCH3>OQ(YSkkdE=H8Us1sgV8j2D=+<#W8(MWUC$-c`NxnmzjM1Q4F$9KuR
z7?jLkH7b0yXjc*CN`)+f!3N18x>dXII_|P$sz_wf8F0=@;hC;dC4#!znbR?{chzOW
z1Tr2dymUFXJqqUQfz4p^WvjiJS7046MjT31kk{Jew8a0Et+wtBt8^tj4G~_9z!E-;
zx8N;BqD_UOYQ9yQNe!q0AGujNcmEb-S+#zo#859UiDeYa+<|6o-S5heRSu7fj{cZq
zJ`jile~B#EtwG5oJ35Z{ze})o18fz>ZX1VS2U0TSRL?h-;|6}?EYeb~XlZbiH}La(
z4P3qGOQ%l$WEI{5Zo9IVc2Dbir6&`E$A>Q!T&nigS`HSl5!%X2Ae2<EzL`aPBMeDQ
zsT)x+2vG|(L=KG;mLpyKnUeIQAN`d;8^y$~8nKR0Q&m6CL$WBgB_!@c@~!aBhpiI*
z`LpglE_c%Ant6xStuGN_c(cF)H&H5R2{}{%mD6eca=3z*FndLK9k&_C|J}M7D>Vs(
z>58zqV3c%hB3K1ugBip-MC2m=@&8-PtoWCp+&mpv;-0L2d*xG(SHi#hv*#LTmxi(S
z7~XmKU8j=yeRZgDcI@^=XJhv8#U3hf_;^0G`+Cl1GyebAfnAALFD}=|=j8XyeB0Cg
zrCcXq=ekdS_bywsdX~3PYx2Yu_H(L=v*!7$>1_y-5U<ofcI_YU@%k@YmO3tea4Gmo
zTW)ppxy3D@%nPb&-W1A~irwruutrrn=hgh|H}~{--!8WglT^2^kWxP(;_ZL<Y~A$O
z+spb&52&;QdnhIArBALqdHIxhdHT&>(_a>0y;VQDHm}XTEY&A`l4V~XYw)_=O!Cq)
zw<{jryxPO}Fma8$$KypW4@RiG;$J4W)Y9|8RsHnH%$hRLh|Su&o22<wU(apc{av=`
z*#3Dwna5lo$Cf>~Ab5Mn+|&M|#czw^jN@NBJWV;V#vJHkPe+HlH$4kyUO6$xZNdNc
z$+^exF`NC`n0IY;v~qIDroyvE*A~~x{(g4S`)#AQw6({j&reHt?TbCV(d4*JrkiT)
zvB)jCufvP(EnD{Fpu|a~V>36K)E{S>`dUQb!T$fgu{m>mK3;g5ZXLMp*Hy2ZugkX`
zJrjRU%Kl|bg|?c|*J|zX-+zQ(nCEQzYg6C5x3b*&@tLYyYk<ywyZZe4)T7)_i|zcs
z$6hjUcHDC<_Cvv?b&jdlOFim#bMW(*C(c>2oHe*Fy(+gGSZvq<i;d3u-YbV>JQ@Oz
zb^of*+hb`w>-)P}*$ph->5qeK-&E~+|Mnx_RKXkfHWknJtzEhM@1LUb*Ik~b%^MAN
zo?e-<$gacgT;*rB*``0N&TR7Cc)Wf7oCA+uM;JV1+odxn<D<r$N#4;{wrJ|d^)9WO
zb({OgQdz%GoGHM>5g=>z&z~tqsmuG1j`Nq_wmN%{bbQz7S^bymw43d<y_V~e%hQuW
z<o`UH@Vp{|Z{m-S+eN><dYhYbr}+OSvx<V3Kd<fJ$!yYmK415e958FSs6DWknQ%fk
z{I(0QRt<Uo(a&XCj^FwD*ZC51J=XnwaaKa{R`r8`wy(;zmks{|$4q-;`)7g+J5Zay
z+V28q`m_8cuhMr<t1HNMs=e>}YV*VH$|c{QA8ygLn^W<dXXmS=Ex%uET>S1K_qxCH
z)*qbmc&44$?apsEJC0he<`e8-VeIYfVQ~d6z6w-3@G<3I&g*r5|MAV<|L0#HtJd}W
zrE=_t3l(PC27RqIah;tg^daWKy>P)f3*}EVX{rZETIm66LJ5{DbLUE2cU+5TJv1%>
zwQE=EO_$P!H&mE99f7UGq!{+Cz{ORt*2ttTU?y6!D<*3VqT!>$28!Q8*(+R#_K}Ac
zuzfQ5Rnf|gz;p~nK#PG@@UhYWZMbnR3M+wgH9o6OFFgiqYe1Vo4oRS~c2ox<H<?D+
nqaiRF0;3^7b_g`C5d2ZU-*tVHTT{mu1|aZs^>bP0l+XkKpBSAK

diff --git a/site/img/shared.png b/site/img/shared.png
index 7869daddefe9f5476f678395e3aae91442ffddb6..b079ad0c6b4e5209b343cc7b4829da9c97d329cd 100644
GIT binary patch
literal 20925
zcmeFZWl)@3(>4mj;5N8>@C0`lTmlJ_;O-XOJutXKf)g|Z3GNysI3&Ojg1fuBe?xZm
z^Ss}wI)6{qTXm?LVp#ODUM<(^?wbe|B{@tqQZzU?I86DM(r@A50HSbk@S-S4u$Gt^
zQcl<pprwSO1RPvd9QvIxBJ6i+)0b}*;o!U(;NSv+;oxpyO#xeQaBduMa63kDa6+kY
za72z7&2L0te?WD7sqF#>hmQC34G))|K@1yx+*(b`RZH=;kg0<mo3WXLi8-66og<7I
z4o<{V2=>v=+|`)c)6Ul3MaUCO`-egZ_W7xqotF9!iK`8mR!dQZTFSxMoSK)7hmC_)
z6pfmiTEyARLg=lu?BDLN-@vq1uC9(k?Cc&M9&8@mY!1$r?3{vvg6tez>|9(gU=%N0
zyzE_#Jzv<n(ET&WzvD=oyO=s#JGxps*i%1^Yi#1+<_e~zeVXV$U;nJr)!O2}GugZR
z-4<+v>`#AT=Vaqx|IgU4t|CvZLQ)QPj?U&TF0k=Mc}4zE{$FkXo#!8VWoK)1*h-()
zBg*-g{(rXpKj~Gh99$h>TywTImA7{_cZN;&w>Q_{-v93uf5#GGf7<x}V8lOM{?iI$
zvnZMf`+ts16fH^_SPcg!4ks@yq2>v{pNW{ErheO(oi?9Ow#|kp{e?iqfIz;*uV<wq
zP+zO!{<BHDyGhhMML2{y9Fd8dhIW4b$HAR{d&^I*Szf2Zxy9AZ&ABG06(2UAY{5;z
z&GZ3jap3>|{4d}@2Ne+TP*m`4TL$1~sDSNNH8-HO;bd9RwBrzRfO$D*Zr{Q`CnZbo
z#0tsLMK5rU2tWz{=Mk?(?LUs-dy~%ypr!uvEa?E8ef4H}SW^EN(r8uVp{&$phc%#R
zbrla7sE$_%06oQc%92lCFHptMU93N3o!a{0!Q(JtL4L1fd~uPc)<(^kVl&$Gcr{SI
z5K2L#TR*V3DwuY$eiv`vdvt&F^=jH_UV_<u5u2pmRTYDKusoB`L}F<WA&9SWC~Gl3
zEf+L1X8X__@5do*Hl~4JVK=fx%(El5%C%RRuRVG6;OVIl`eGsV@lxxzoOPslHD3H`
z!rqJ08-l2g7(>e`Spl$O3391iUO>ziN$m!&<i#3$T2He9<MpSGSbdt7huu=25HSB<
z&+ZJh(d9Y1(C|o}7=a(vkRCy`VT!qhu;2qj@l2HX9t!LnDRlGqXaF^n!M5}6HEAVM
z7uSX~EvuC|6mbhVM1^T~v+KE$MMbDI9a-P|voaTxKlyR-o3)Z6+pgs@E5$9e_!BOi
zb{S+jDl1E}<i3{aU<9sRU46+_ANq)s`c1$LnFmd0Khh$K({?mBc3S%uxy-FBfcuX3
z;4mJ1Z&R?}iG3}5<u(#F`(n{zDvh{!OHa&bd+PC^^tF%AL{^x-X@-Y<TuxmHUL~eu
zwP>fK&X?T0j#wMA%7?PREiW$Jrk=f~uW5V_*`++58RlDyEJo^sPE0*}A=YVpZr52N
zv}0rLE2wt?w_CgI-1`~B5<dR!p)QAW^ZZuXL+FBbo^Nk+)R`UJcV<Z@5W&(ZD-VrJ
zcxR#quYIT00XQ!;V1oUz&Ro5M3ui0BaOaZ;t%hlhHsT#?Q==RjrAFEXG%jtT&Ax+p
zvdZ~Yhm-AF(84H&lv%^~YcG=6ej9(YnVbm;Mt7@&8H_^492WAVFy#EHk1-Y=iv6|z
ze8JJ!;J&TV4y+E)?03bWi#><5m%?U@wdgbF$cT0a=@`19&KUN1m3fz_Mlb2R2<<p#
z+YLHc_E41v6{H_MC?MP6$AYCY(5)*r>D$S2{K{46r<Xp+!^g20F68D5mgmL5=askb
z9rb0@sBvE6KJDcTje1NqjsQlDW&M(-dmWlZD;aXxzJY~@(ofg-xek3Qlh;=Sl=VF*
z(g%xNX8L3nf}+d!af$=A*(v7dZPBt0(3}bQ-QWc(EC|&2tA?l%k@DO+u<azh&wsd<
z?lSbWrlrf`<YT;|K{X?lka&<zp#au!^$?BC+nC|hX|dg!(8Z}w)GlI>@!9G@tp;bf
zSY?e*<K2FId#f0$t@JQ83ulKH6g^%qo6}P;f(uRII(j`*2{v<?2;F-aNUI06vV7<@
zT)X{^HD!?Wb1*-rfqt@`vb({8EUa0>A%rd@i&Cc~9sVcbkM#Y7y27Q$Ntv6WcV_Ed
zuEi!y!52M-X%iLZkuid7J-zH0)0(H;cD===rZ??As60g<)%k>e8s8#fRcB;zVERRh
z`=YMhj8QE9O!|b<7}8N^XfE)5=sF2mG**8KY=vxcwd--V;8^S|FGu%_<C%<H5yvXR
zN%{h_2K@_G75AZfFt&ohCBL`#u%}qp4G;Ph4O9w2)$ZrnYsTTiN>{h%RX~AP@jsqD
z?{mz54V0!nhH1{THcT}@3Z7AiQ+5G7*^O~1_wVkNiLdyzgj+q#<xX}vQlCDJMwpD?
zmg$vWW$ad2{J=d!(}U2|SKhZsd9v=h*fdXXqD1G1x^bStAue65iO4O^Nf}Xx3b*iM
z%z(8@<QV|8OIR>Tm>b1n35h~glg7rt-2ElkJI%j$uMN@G?5ShteD3DkqDD5sYvnN9
zFAXZ0o=(k8`tAj7xo6zKJ6M&M+8SZL-5^&ICLSirETAF`iSGMkB(vzH^-MY06QI`;
z_I;!FNJtb6LYOF#?^t-mi!wHe0H(owgD}kEILQeY;j=Lujb?zneSa<paP~~Cp0KWP
z8}x!y+mnw(Bdt#-(!m^pUBkP9=a89#4r5Lq(4ySg+}Qppd0<+_K1HpEl7f@P%{O>t
zYut>Cmt?WKk7L5S%R&g~3-hO4Sqr)Sr?nJ@fOS;_cmK!I`e?nT>*A_3!HLTVhp&mB
zP{PIE1Kl4dejdhOM2MT9p7}9@H&w-|da6iMBWf^Xu?wkEdWiA$Rg74}I*aMD&$TuS
zb0*EzWJf?jSsuw%<n&*=8o9^x&Ko-IE@kDl7-pMbuJU6{#SWeyVCOhOR5d^K3ver1
zx(m(OJC!f2dg37KK~Kd%cggoJI8u<yx$-;De`TooU#A*i^TZlO)gtP@K9w3!#X#N1
zI1!xG_ODa@|Cr{PFNyaH@L{^o_u4bx&e@fArtfXg2vqh6G!;^S!^&qtXiR>S)}(Y;
zKo;}pGI5OB+|BDZamqY8n%3dst|-xxT(?qk(u{)nxN?A?51K?<7Xlk!OJUGRK|(xM
znK@@07bhGCLh;gr_zbAVEbGNIy@h9_TQw9a^~O6-_YD0jzKl4q8pf8<OlrUZ?Ga|=
z>5Fb-_B4K!%CcUm)o*X~lHLOE6vZ7ZFFkHcE)Je?0D5i^ND&lWEOFF}=Yaw|UVOjp
zFFpKVxTP^8z}r`EL7XLv9ko=l>=6NsxxHBA;{abjWVNMZRp%4+^&{*sq;a8C<1a~P
z2;nb5o|H1i-}(^`m_I6fjOqxvYQr?qeL6r9NpUHCN^2&9F%O_*5oeG&qxhlZ59%}@
z6dvg}#Is}Q{~-g5{vk^NK1i2Qg;8FZ@6YR7*AJLNZ9Rxo^I@j0UlGO#d)gZ|*!CQe
zBgaTqggCi8KubceGO6ohU^bz{6lN1VOTVk>E#g3DL=WDb3^HL&?g?U>%=SMmMOHjE
zu8;I}&pAD$;?3?cw5+YPwS@(Y+jDKvFsIIyBny<z(E{Sn={<ifBoD+@T2e?fLgGP6
z&pP^bRRc%9AcXgBcVwqzf;OHU_<J?Ty;NhNR8PT|tk$P1z^w>xQ6%Y8A*z*Kb~hi@
zqlPGgK4zJ`?VH3Flr?%s&!_<2@??+pi#7Tso4(B@n7MM%i){6AF7|VM7YXJR=n<M2
zLT4uQ8QvqV&AT|}eb6#eyx_@}LiCW_Wb~2I*J_V?Q!oTYXM6rBGLag<PmW*9uM~b-
zu>siehM}j4e2JWZycr7ljs_`xS4aB;yxPkLjOj^qu2RW&9XE9tQ}X*ke17H@&*<!;
z0(4P=Cof3i0IkT@cG*|8uj$NO(0_A_PBRnXfFN49M`f$rIqJ}jiY{7av9QdHct!nk
zw|*0^8s_~XP-=);Rxo9tQmjIx!WSk$+!FAe(i_H|Mwr^&qFzlLPuKq_l~6B2d(T&c
zAe_S6S@j{TU<wn4!=BI#Su==xiO5fcajf^Qtxv)v11qkfoS^dc9_6J+f7S!<nEUr^
z6Vg1aS5>TSKPz2?kjp$00u=R#t9=+lGRwee)#Uu=@>LZZmNP9|As-{eX@HIIF$_(F
zP;N@&PhE@lfU%Oew-SZPVCMYzJz(cZj1Pmw{g6Y*W@lWKl)Ko&2KF@eGNupma{?SJ
zJfe`j56o%xbacCdZze1i-|M0K@U45aWnWzL^{;hJYY+Er<-N6)9?9Y6BNY@%2XGA)
zBky4eu&E{o??tk6?ZuF!#U^l9x=o}JEb$r4(e*GaF|@7KVECY6jf~y`_q4)$c}UgK
z1+XxQ>ihw%uMCGrc7Ol=YUb|Hj{m7n=viT(^5czyp=MiGR7K5_H&>&N87FN(Hr?>x
z^yW%Jl{yy=Q=<4)lhZXk^99;_>cwNff_+VXqs!$(pQv;Tx+A))uZCQim5~XT^QRe1
zyEP0Rxsl#{>!OPei+;1KHI8}HO;i?B^0!F)hC=|pA~$HVEygrh%xnr{ID6VI&nZDE
zbX7_>Ix7`{EX0;$_}<qb)K0Eq^g6r4e9<n7Kcv+EN4#54bXkMtzLVodMTp0v>Vg5K
zfkw$^!AGX|`pM_$)EUX5GC3%KRYzZdE4r&4&gPEeHp*P+kG8-z*7^13!3yBiu9%(7
z1ozS>L>ghtR&Zfkf^kVh`ipLt4M!11p>gb)A7hsyVxNyoQBT<pYM6c77DZu;?+P`D
z_;kc6@QkrN(|FQ1$rxT>Y0vp}R8F<uh~M@cnLiC%5lv_%ZOp>?!#QArXbkm7KCtv%
z({p7}-{H$GoWYUi-`E1&0~4#4w+IQR9?>T)O1IKR{P5-b-1sXh(a)QK&D2da(C{sW
zlHYgY)F<o@CqvqcS019aulPu^@Rl94vqwEA)?SZtSQj2ysvex)pUgLaMVrV{!EDNT
z9-9+VZ!_vZ#3KpW<I>oQz4laye_;+E*3{wALCT87@b<}CN@tee=|qSvZm&Q44q+`q
ze6N}959u5rP-A6oH+K|dNxqE)2xcPozi}MOD#@vP^HNzpTs#6baa~(=!csK?eYXnT
zclV0tyQFdy=N!THmm&lBMJXB>j4ySE|A42Dvvx;BPB`OahcJO`1CQAgKfN)J1Lwwq
z$eFAwa%oAoaCW-97e8JKs(w7f5^|Cx4@`&<M@12dav2G1tj9KeZ)VQu$ox?YyH)Mt
zOJX2OVyw7!7O%M<Us8GZLKBOLN}q5Dn~nVR{bhd+zK@Z3q!UgTU)|A;4e+_k#<=CF
zWIer~GFBlqTnu+POJVM!Pdl86GXTYT*x8rGgbK6mqp6-qWUCl`9Z{kAj6<rzh%w&=
z9UcT1v^3u*zWG7r9#^kW^^_4x+~N?}uf_myq5}<CD%NHxI+gflaG#jRVI#B)phtD7
zX&TFFns=|+E|63!Z-hoGz5L>IwX+Oi*1*FW8GJ$0a(^4Wcm7^99ZxbYZkmj-U`j*a
z6*-%amk`~>s{`c)Ud}Pft+?$|WMs||QEoY9H8Zc;7l3O}YJ{J%gx>6CqHnB?<}s>B
zs47+P2H_)aCNsdC@o6bM*^1^BX}52gye_2^ccGX0XY=>;Z<HE=1IPOM<nEER1>-|m
z!b0Qaq@UKeDj6B78JSMzNQr4@k@~`HxICwOGt-ifZomJy!_{-23mJa(n!R&#6O}DW
zvqp>%u^oK}#rmdMPInY@B=KWT)OroW=m<lDsH*SuYQk;7&;7GC(cwcxL9hrRN!iFD
z7tTvK_+gUeBa9&DS&3O{?PFBO@(nsD=%teKantrsCcN;C8rhM8JF>zxQ?Bd61XmwH
zay~b&0=tC|M_6CKn3kaRM0aSu^rCfr_0##~La>9}cw2BUqXU6DH$N`4hT);?Gi5d)
z)x}~`-PEP`7T>YhJu2W6iXNo%$c?Z<yuIbC5gXep&BT4RaIiTSPB<$(*k&bicHlKw
z^w_eTT!uOs*jPV;2_1_!sgGpH3{({m5di#32g{uPBB5RKf}v=U#o*!qlr@?TWI?HY
zUV35dbP-{;t}I??s)$^z6b#feqgoZQGJjG}IuV3ND!>eC3t9+Ema6^+OgEa0QMvpU
zRx1ai(nc-nW7O8wI6o1FDRkE|Te&>lkL-@sDL=6@&4;i$O&ARcEb1cPtD&ALO&tZd
z$hLq$N=@yBLLLV$%k?=SAH{+>x-U<N@#`neq*Fo2(Tu2aAc8>!>#QQ2!%j_D?kV+i
zFbfQH9O1$4SMT5Lf8M<TD+y9{&i*iymo$35!-?7~D8GXj^KK3zEgHliO~;wPj0QMY
zL2J{ykkd?*4FSRWVnFEL{F=<cgoc3@Z@**##E%+}b33dLC`1h6tnjlTKQS)KNn*Ma
zWhGIU&PUym&ID_*ym5B0{~+I-ZXoSrPHErxvKW)zsq}^-AY=U;VQsU{&+}0!NGZr{
z;aPmB$u4p_|Ey9zA^ZraDC%QqOZ@e#w3=X^`$_Jh6nzyT5j+~PihUP?osr46{oiB)
zF6|w*P_1~s(0GZ*!w_6|7@XzzBe05#d(`Y4<0%rv@6AtjP*~fY#gK|Io7&Vb2ZBpS
zzuCOKyc|T3VV5(-Z~Q0V7cMS|QaKscD#n__4ASg4Uqv7ozeqfm=oDaUpJpsN^1sut
z#-3>49Q5VoA<)5?$8-xedjjf9q<{>PDQs_e6A>s8G!3*GY<57`VRU}2;(tl-!h&81
zhj}D`n2Oeg3PQ^Cs|m^v@>y>@+GwX1>&wf%;lcsIvxABPB(pI5;Ov48!eRr@I^cms
zB$<>#V((L{`gnbkC0n1MWun|0rGSMHfvB?=TUtfr&tT^Wf>*{}ay3T!6&pqz-`~+4
zM13EL03;#~YmsfrA+KWVH0@vds%824EJ`MzwJS_wL^9+30|5{IXO|}_fC&q2M}RaE
z!vofffU6GkoU-RmK;pO_@l7-PFa(u$b5Ph@vj{pUiS}uxm2H$^4r?w_JPeVF1AQd|
zAdwhadO*dHatJY$DM}2o-0w`A7~SglwP@0HOYJ40Kj20DdJOb=cvmr9ZY#~x%@5@&
z8IGzR!)T48TW3pmOXOwyFg%S?FW(hBDPiH=N*X4`yYem8C@P3h{|;^VCB7z_KTrtt
z=V)O1Mcc8oX_~`NgCx_A?8JZ-ly6=|4}~8%rRmA7Fss@qLa{`ynv@{a1tfjAvxYK4
zy{#~KP)y=J{aLVd4l%HJC8LI>O)mjHeni5L21$0%L~2Rt+3h>qk4i6#vuP%6@IKav
zapkK@ZH6jgWl{)%UEi}vJRJ{=QCy&+IGc)q_o2kZ18@Uu@i12_P<HQ7Wayfp`0|~e
z5~Sp=xR2)MO3=1vne_tXED+VD<MVH4V$c+sHJA7adpLXoMusiwz-K8VxR1u5;*>C=
zlDs?DcmhoT%A0Dx_+dNHVa!pxFeZ&GYWYW;r-@f^Q65k9Am|JdL{b=~iAgbVL>b@m
zY~`Pw?J&j*)3cJnEAeh7x+sAS(dLnz6q^6pE%kSxKn5qso5ZB_dm$7Q4xdu}5pM~)
z!cycnz^-^j;2JtT+hstdJ%H+n^F0?tL||r1K|~^g@z$1t`sq?3-4O#M*1C4sGPOcD
zL*pTSTZ&Ab*fHGKgh#F9b{Sa}Lo(TQMG3RjiCtg&BpoE~_)pooCtoIp{ZR;N041#o
znRzQH9i<9+^-U4uGuCuC3Q8=hEX}j8<$|k1iNURY)t^x-{hHnr8UhE;YuX^gA@9Wh
z^7<Jyl-#WmJ{^@)JOPq!N2C26c+r)cq%t3<>@@U&kO`G)X-H3%)`Rd1U8Ea!Epa-U
zJ|=JA-<joK2f(Le{MG|-IwrtpVA5zC06dI|!qDgK4RZo~!+)IMT1DzVLIg0buw9XV
zQ~njMt7td`K9a@(IHe21=%5GeVx)m;6i2&kMCh`%q5k!w>YL(ikg2bkHc;T_4}r47
zTwds^io;7-9OX~&8w(!EnG(Ad!7g|qG&YF%YTJVh1x}CuYaqG=2ZK3@YXJHW!b93o
zc`)J)NC&$Wi2`+i2KXl!0y8?WqawH=-?9l;^_G(g6URU~FKLBDE*^9aiiL22H#(hr
zN?tPe7m1zGN?UHaY+)3k{iFFX;lRZ++8a@aI)F?AiHV!V*7!@}&|2*5vJIH<h$c=~
zp=KVxZzZe*&*)s4*FvaW@m2p>@Dc%FUquBelr4amX)xf8*br?q+`0MBa5C?<Un;RA
z(1>`$QeGBsRvw~sgn58d_CGQu4m42;fPBUv*2f$r$x-bH_+`c?B$ANpqOUTr`zAC_
zUnN$HD3_9}5hEQ?7C;%${8#+pabBW8)Yb;!ereK_d*9-8Ys(Vkn(^?L?vJ4v393en
z)h3J<e|~Eznelm~F_S3gJ;;(~@JR-r#E=)rlJ~4oDqF%e&>#>=979}@p@l4oB%uQ>
zMrMHyH`V`n^#*kQ4RjZDn2DW}K*K-$MXKtrDA2|yh*S8s0#~4*Feb`CZ}9=6B$Wt6
zl|`W12G5!xrUH`XN2xGm31x9|E(&bSuy>OG2q4VGJ%`5{n!^|hB~JBb{f%@|jVZaj
zDRI2_AxEl`d?lKJeYTD1j>!uCOxTqov9!y*DE%K6b&LRsQwQPHMLS3;2Rqb}D!oS+
zq;nmjOux&~RmiQR8uHY<i|#ImOzGC4tT-CW{YQY>s5`{fkxJX(I0MdKZ9NZ%lo?bZ
zF)>YESQCW-TwYqCuF;F~sw|x5IONtrRaL0^F@M145dXW}ml*I^M47-=Y`cJkfY{Jq
z)D5#COw^{TCf6K{VcH-=<`s8SU1z)8OX{|p;8v49aRaMY|H=Y2;F!KcAlVq|0VUm%
z-!fuKjz#>)URSmGj>}c)t18n?Cz%u4qS(3yT~Hg;1u7I?Gdvvn?YD5q%D-nsT?I_1
zzO~8ShDvTJ6(OwgnU)SJgmX`Swk!KUO(9<G_{mKIauAj*Att3M?pqi3uQW;LL<6^u
z<`75WO#+@Hn9r|a%mtl%!;qQnrI8TAx35ZO>Q@AXEj$FuO8Q7WZ8Z+Z(T2adkE6Je
z7UTlBYw+ga0h&BVEtF~pv|>7?(rXY(cak@MZPOh;8bBl?Y3Up<X}SlUVenwDJXF3l
zUcUwtV<f+lmtS^Qu&~$|R$;15$ZkoBz5lR+(f9!M&&kB;YivcG0p;3F#U@oT?&634
z)Rm}^s*ogKNr1rv&%^+7syM!k$bB1t>vJS2Uux`tHi!X4G_YRkT3c~%&GB{Xd~wJK
zX%eeuxoVY3PqU-SyHy`WfoJ0$`!njiU~ucn#r`boQjN%Sip_k@G9BJL*%;C^fyKcj
zR-%`$UZJlLi;W$DF>kq#Nao7me({hG=1~Z{6aN|_JgLbP_6V(H4<7khrrS_pM8)TE
z;-3b(JoVhTIo}N;<G07KpRT4SPzYM>=CYN_(#?J0SjmS3fSgaY6Hb=O>aZRUq^V~}
z^rcz-pp%wkX&C(=Oh`Lx080C~uf>G$WCcAlp0>xI6M5(Z*n^OuWg<Cg*~;cCBx!V_
zeg!fX;ABM4%+}kLz!w_9mR{%nyF`<{Mm~Yzd+WsvYnf7}(E4&eutc-M9`UPA-8&>X
zz(DiZmiTv)YD_Xd{GvLW$pR^1!WTNVHrv1Q>%Gr+c=S|pVy7?A2v|zPT<QU_6Bmk~
z_|(I?egk;#DNK}SqClppPl_xr-O=hl`VpnNm=<9EtYf*rLWK+s%ucb~?<%e=`|8Yl
zFXT{|jL){QMkb1o8OELRsbck#-xfpZ+V?l-H6aEcxCm*k*}}NZST#x=frb5qY<k4$
z{0<9zR*dyQKT_B$PzvK(^<NzclEsela9Bo<=viTP8$5dgEPb%R<FQuI*G75e0I4^(
zru`QFXT4|A*wTeVpZNJ@<9=cZ@RHo_HyDP%4mQPMtRSY&ZgwJ_&u-AT@JraoBD89L
z>x;c<K92*vPu!MDTbGOei(D=nKcoF0uEUf@-34S^pe1~Wc4v5l{o0<~ITR5r6(<C+
zEMs~AdG=?LZ48`n0(dM{3b?*}k0cg#EIE<E_ZWZ$k3~NZa>ow%Bq5K3jEO2!xu6=;
zovH|RyhPD?^B>96wbo;{H)q=yW6EN7_v<N!?zx3UxHHboZ|~u|91K?zAW+v(u0Mzj
z{VCuqL05#*M$-|n(uKqCg71xf(=lod*Jm-C8%C+~)#GsCD>9{L4mZy_8L7P1n_lu<
zx&HT8%gAJ6@9*jl+aE<u*9VinTAgk?28+r(A0;Z{^9jRe$_(DGVDSoqiIhWag2J{U
z;O=sq4AR@D65eNxglyS&+(%_tXE~Bp|M+lAR^aJVW5ME9h02j^HJHeJ(L-eT#s6$n
zZkCAMAS2QsH^OG3)C)Pn7w!eCyNS>nSUmG5EF_vpl8Xz1kOQwAiiu2|hdV276L4I~
z1-8E=uwW8%P$vyz6?*)*Qw)F~H|>k9Qx>~-{c(HPo{d)M$!8LgB_T4YOWn170W&NR
zV%+#uJgG*78BL&oxvu~ZV62-p!|!4`Nm+0`aY^)cZ<Q6L*meNLC^^;ddOd~D;%BPe
z7AD%#LmH6L3jghOu^IEW?H`;7%UoeYLW%2%yJXIkz~{g1+V3yOl1Dy7h!3C|eeZ{E
zjpj+Ewl<J(N}xO4Vuds&LjJXexJqyjN=dVvudB`rKH1MemZLt3ZM%k`;EOMPa%e+!
z+nun(3w_;Wq4F5KxX<!84_|@0hgoQO?9@lm;PyQb5mBVMRu?#^v=qa?fMQ%{cL4_u
z^mjs2JUY*&P&asTsJK!&yaq0n=lLlHgU@eG%dVL3u2$n{)cAwX{UZI}1den+XDIwR
zsqdkmOaEO+fQAFpxTq3(dgl%LK^e@Em<}(i_d;h34)65Ppegcc5$vc(cJsNH3qI@z
zBTF^ILae`YBoO*OTwQNwuV(pMb72UJrVbFM^V(=Y<c%ok3Jsin(WLP^F}|zBQG)9y
zOSMm(3?F`|=F3n=0`UciWKtDanox~AcTvv_zEJr*H(a0)>+)s~rNNeZ{$bgT;Nog^
z!TZ9=VivZ6boNS*1+Tu!{%q?D-8Ro^59ijx!?xRU5mDv7;e%=9S{eYYiV5s~t`Pm2
z=qd@$n+>ZwUBt;Sd~@0|oA;aioDxhV$V7b`VFt{m-hRHBd<fd^@82ireKNopb9mf2
zJBfK1@4Ud-bLiU&D3~H!cr=#^bFPjj@y4<JUG!!mR|+{u)iP7a%{hbROjw{#S$LlZ
zW{6U2C>|Dv_!n*UTikg|1@RWlseF$hrwbfE6$TfGnw%E>E~guvR`=#T7Gvh@L>`Mv
z%B#`Y8ap@=nEN=B_v>fvxxck_9pP%M7EipUv&9CV#b^2)G+hj`RtRh^A(mbI<Q@@V
zCkNkb<zXH|tB=}_c<=6ZN-Fl2g0S5Mjs$jz20m_o_88(29>Ee^14$h<w?5qLPL%5l
z`&=!D(Is#>OFAUYyG`l~Ui9sthc=fQw!inCwJp?I&vk_*Tb^JS^(VMc62eu#*NAa)
z-AgZJWo1#Yw?O2SjBX-dq&$hk4%#0d$eX@Ppcq4><>J5>T3bz<SuU|7shnLTvO-%q
zL6VNX8TT2S(m@FLoq#AB0XCW}k0IX4;3c>aXw4dVU(iBC>vwd4%){sC3s;n#I^4OA
zq#^-HMN*D@87{PsM3g(7tYjG{t1-tky(PhRYyBT`XX<RiDyyBg!s%lBaaOx>!Qae9
z&o(|eghE#dywArqGrhSf!`*kw8idvbHZxo#RST6czZ)BqN8=%!Z{@|En19ZFylsC(
zU!J!d=BxM~2pPT<K4_#1CW;eM?He7#?`(ledX58?oqpN<`XXQCb9q1`;FVjL{a&cn
zY~Uk^{8{0FBSYrtRlB3N|NXh<MVpp`kwE;yy4N&+KHp!sO05B0!v7W%wlS<FHb*=5
z2-7W!K6+<AYN6*_G^n|g^tWn4qAzsoG3;h)nSH4EJ7Y&=i1ZUp#TI#DFGE70KCqA8
zhUCbA!z5?QD-mQc@35e&JFz8?S%18W09c3%<^<4M?@oRQ{#s6X*=KpaGfpEAJFPVS
zStcrDck2t3mV!!%-l|K)I?Hn<f(9c^aC7X~*!zR594c69x@F68^f;I9rBqz<>KF6u
z0L%$yF7|>SLN5Ma?qT{m{6J*r%jt930YQ)=zWDh-xf^3QTOR^VXh2=xCoJ*N+l~yN
zLa{3Pfn8QABpU+fEy~sbg=%2!cmFtox~~{sxw`{VgjYa6zj>Wr9TbxHEIAT|m7Z{)
z4X#ZM<&ujOVS*(DFSs$WK5(+1A(1S+*uXG9e%4F!{hFin<vxb4Q)5&KRvu%&z&Ewj
zT+2R^=4O~#v3t0?4qc!R%Cp$a_J5!CLyK=F(_>-#+eq{wPhQ*oh3+k^y?P`}HKBC~
z6H9-*Oqbt>N(1U1I&f9~yiMdRPVAvyb<v>JTd>PyS;Wf;hFmg?%lCpSzY~9<ixc@p
z>A3`N$xkj9_DG<qhvTL5vwna$R#!;WP>MmU?>Qa)bATvy{7#TiaU?D11)${U^C;z2
zk{DSP4|!X}b~B4BNc8c}g^_aI@bN}7RtSn?63^{T;k?Iwkm4AvWWPys$HpmYWgTwq
zOitLQ1A+9Xj#_AGjdXlxP|C3KoJ?E>n}xbrME*=y!U~PugDJM_f`L*Yb9+iHS9z#x
zUo4jDmyjPsO%(#OrR)@~bLZ<G$uvLnDr<@3y4`aSdOh=pGlg}tMg7oPp0V^n$GWDn
z_Ftv2-D85i1%t{ze+<IKf%nJhzfKr1bknqx$>9MuCM(#808?^^%x618&|f9UR+W3L
zL@;JhqJJBj<*-eV+Vi?vOHjr@XCh=Rn`3m_n;M-CIfpotf9CEKe>Y5-R}~gkGkDN;
zYoAxsLy7yI(&Z=T#e}W{N$3jsCX*a3qHk`{cQ|9A*LyTVcSl{P-~{t7#{|VJk!e)6
z0y=!c9~z{lX)qXR)3p0TgwKA?%9t`FUy(6PiOQ#pW6eCp@V%FDq1-w#NgksOx=C^t
zWGtb^hmA#$3FohWWuk<SW)Z9?og$3V)ZoZMkYc0hom;c?{E&}F=dxcnDHKMs8bp~S
zQW$FL%R$F7!g;nK02#w$c#U$LM3$Pl);(l&#n9J;5iJ|1v;{M%KNsSN-U;0<sRR>a
zCuYm^UP)7it>OjN;54zyMPp-gaHK&dw~n1xlC|QZ*xLkxDH%3}dRI6&gnli(Tt}_$
zWu=l#rP#>m6}J{Jq!CC|>fkX=f!eGB-<*QAYwv!!VeLIoS$GO^N<_OwVz|bNFp|zj
zb9Od^ea(AB=3?)oJYMZQ{}(!dA)%@E_gv%wMQvZsU>fOR@dAd4%Q`lhGHJW3g0G-#
zF{?q#EQ~KnROd3o1Ob+rOQsk$>-`Db1zQ}_!&a+ORL4AC<T3r|=+3^hBd88?Nc=8T
z6zw>4vca9^tFP-5ltl%Dmnk<t-jOGMCZ{q8Z^#iIrnE=5>?J*wsmM%VcuX;Do<^?v
z?(LGaJ?qd8cJ@%KpN`FtDM(4|eV{6Heu{;`;HfZ#G<C!8g*QZl&tburjxwi30I#35
z*ZRc1PQ64U#97$X`~|B|MkA@S{QiE!qI<!34J_v*^p@)V6-{PySt>;XCii$-Z&~Ui
zvC;J5n|xRh1rET8IgD?Nb3_CG@M$N4CbH(p2#hF23or7Ug$7Q5V0TZ;I>^PZe>6#b
zLozZN*(BcG8nY{s&wWp;OGsRo!!;$chtSe=W7-O+)gHr>lESC2|8H$#1cNwZ5r3%l
zfy}pIa*VM34uy8ICU?xm1_A>fc2Y310^J3$GFUvk%UTcN>$6@&;-)E@z+4X#v`5)9
z40{>hUs`iy89x@ONBO^=5W%<OsKmuB>gb&-K7;7=e`5!P#P#y*h-%$cz#9-%VGgU%
z|4dd`k7h27&geg3KIFn(@|NJX9Co!RkD(HKPqrYH3^A_aslDa_Li{5QIz~S9C_}S6
z-Y0S@>$AZS3)d53>3wAxCC-EDc>IZS3w|pGd39Nfx{8kcvong?LIeqzfOC~8ri%r?
z(DcWgOu(U7*De=E#b{;r`x|T4z;+K~nH`1jQ=Pz%4{&l}$3h9YBp`%-bJ$&qjhecz
z4>2Z(He4JC@t!I`s1>u|IhUT3V-%29tZc=rFq;awR8a3A6<)tyc~kystkzmJXcgb7
zi7!t+fSqHVWyoc7c%5<awhj?2M4~0pOglk3gT-ixWnp?B&S%0t>}thOOP6l0q63?3
zlpdCdXckFIN9hiPBO<j!s>|i@YojYrI}Ag%M}IB9h`#7yGW01kRWUj8E0kcFI)5o7
zXSk9iY?!HSS~ocXj>y6+hm$C5O)@cK0t?wv*g&WmVOD}X&m|#tX_fveBL)#zXzV`y
z{0kS+ie&PxOF@wXY+8e-J))O=;fjPs)^t`thNg~aYN+X<$mco&LT;ddSrqJ@R?sUo
zikbl~kWp(M%r~TY-g<(vtx+mp#ASV<w#d<z2vkkl<i+mq;_Xc(eZFButihs^%9(_<
z-@&1VGy%``jf0n9QVp1}zf_Br%?k3V!HQKh|5j}3pk|M7M#)2ZgZFf!eue`}vdI5V
zEsk_R*$YX}!oY+~p`S7EAeo5BOK2b<6AKRI@<dvz^{FW83(0#D01YF1CNnj|@Kgw}
zWXwxBVfSNI!EU0eNjeH86(sWf*BMkxN2K`ORdD3UeH=b75EfOBLi!M35w3cd@Z0zT
z8tl!UTvt7&Jr|3nWo|sMwuEniR7+#_?Nx*g*K#GW3P~9R*8i$P8Vb-3wZlJ|iKgj{
z*~-+8ySH%~dMDt=dfY_lW_tKj!Ah{!<Z@HVcZ2UiQ3RIu!^>UF)udM9HAMbX`=azz
za6+cibBPImu|bY-4EgE7=La~3+|k#fM<yLR0D~$^4hb8ylWXUTBSw&OZ*%EeE(QCR
zg1>NT2|pKjh9QgA$t)ihloUIXX>9E89enZqf&_|UWz+pWs|M91?zPrzB?YS~7)i=Z
z6wajbgcaHcb^lMOj20vVV15~eJ(og#q2-HqOruBQ90iUFgdRuKw}t}cWQ0e#zGxCm
zW?{yJOr4&R|0XiIOZxYgC7pu-^j?|?qsAI=yxPGn!dl~7jfa{PDPW-WE;o*=+!BA$
zpLix{=81U5U5jDjph@)Kc~xEm%A6L;pn6t(uH3)ToJ*Xh-VwUmVxS9<TPp|8i98!I
z+WG)B1bNM&a4zZn8ycpKkAm~zWea&9Oze6K_Z#yh@EHD4A{t#G%o5ktAo?^y;{=&D
zSC0DP_dT3Gu&JsLtLJY_I$Ycr#ZfC9(vmF|P2dH~pK)gOvGy)7V@dHwNI`_D<J48=
z1=a}4p1=J7rMAN8o_db-IUA{onGe0^!&5d3_K4e}97%XS>kiN#mc0Q9Ar_-}jkZbV
zlC0vgx8RYql2-iuoPllGF}j_5D#aK+49cWk!23^7kp<AI@~MydBGwl>6Z=>u-+*tJ
z<rXv25}SacyFK@Pd`gTBqtjJXmk}!sR+N+H>wkBgMGwWIZ>LxZ@<wt6cTXxl3mTWE
zCyq)C7x)6-?;)xNuYhC&QQ%ODgdfF-wg@;A{U@aagV1`)IW!9WJs0molEp72Gjq&p
zMvI_bsHON}Xma!%k6PwKUym|y+wpHmRya%${&O<rG#v!dVUp1v+UQ^e3N*wVv)Y9s
z{7fOXwG^h+=K@${%}#dNRvgPF;eIr1vHcyFRnTeBA@84z6pTyVnc{+L{cbi3fqasO
zl8oZ(xn}h|YW57S-_ThE$<0$y=u3h~WPkeXNN<6|;g208`^3K?KUveK2gD8V?YMws
zg-+F+V!E=aQJh3^Ds9uHs3q@nq6&@BuSdvnzS}E5Y`D@x03#k}y>{QkR!{urc)FuH
z-uG{@S>f<tHj};sUJvQyc9I*s+xZ;nA+D_<VW2b1sB=Y1muo}4knoM)EAGS~>ECmW
zKy{R8dkVRVK{KIk@!9*Y)UqJ3i(~yzi9M@Y#*Z$a1ai-cldSdDT?S-=f90VuMqCj;
zyo+H`f?U=USBUr*N8oh(cL*4rIezxJ_QXX5lIB9mH%qU>YLD2<6OQo9`hWE%qt3_~
zIhGQW4PzzA9LGp+fU77h3W6a%y&*MFU9Zfea!R?;dWTdC?@c9yic9w=3mYDgje^gk
z-dLPXotRUOk>ghj++^JOJsE#oByE0h;_Ky*jp)?q3_cTa73lUy|3~g1&*-5f=grtU
zX!jWU@Z(`-h<IBOQfumEOR9P<&!#6YA_%RVo5^obb+GQSchLT%l*NG`1H{pPFC&H9
zj5so_@mW#5)Zl@~O1gu0VuG^rkt1fJFte3-zL-ENyeCRWzDJ!0I$dP_v$YN>;Jdt1
zB1vDk6U2D(0^^a8QZrzA*LHT|-pAl<#Xm~g`D&#VKpsFVhEW#Ke`HdG^*8_GNCQz|
z8v6<&nlz=}WQ7_r5qwO_fb$s?s2Pf4SKkkCzIC(9{@CEUz$)P4Kz+v5?U|#TOQT}`
zS8(IS0e1*sIB(Pys;!ZnF`9b_QZU!dW@8pQulM+Y^moJacgy-}_76ErZ;_~^ZX}jJ
zhrW3O(~>)x9O2?yP&eE40F*wyc@3UM!5!__GOee<!HcmO(KpQ`v>zkLB`xC8L*wa|
ztW=+F^q&=TcyYfWrq}55uc}I<27t_0efk8Vb~~5~KZUPNkrmbXnMkz>ejEZ@=h)d+
zzb$@}?IkKehPw~*8C|z^1GePe{)e8}4!oB)$5ZrFmx+c_v3V<5CGA{$(L2s*;q*{;
zXp)N5->s=&0Ms3aNkFErCnVS<?`e8Z`%PX2w2JKQ%NH~Zjd>Yh)`gi>(RcKKu!FKf
zCL|3dq-fYn;X_RR<P@mi1BqWb1J78xsn@Y-deb<RqzI`1wn2-z39AYEdM+ffbB(p|
zh0Q-!q;f>61>wDkOjOQvp;an^|4iRT4@Kwo$5>%ML4_iM5SYx5867dJ@18$Q_tMM3
zGFH@O`*=iiUR+-x;7!Rz@jX;;(i@^@a`>R9aPxq20DOFdl)n^r&Tg(nJS-uePG=*O
z{Q_4rICXOlUeJ-@?R@Iy7YsByC<<~#Xn|x@ZcA?LKPK#BBpd<-$x(lY9Fq6~x?7;x
z0re4cq*Ix2B<|STxrpC_87wZy@V)vmfrR4Z7OhzC{*m8~1Rya~0N*5luBG8`@4-2#
zrDyo-*i7SKnMiGUV(#9o-uIk8Bv4aA;ljRQ1!e_IoeEHrVvYZk_FQ@gz|a#xGNE3T
z-O>+-k6oupyG4Nt>Z4t6gDntL49oUc+k80nKJ(a;Ber63GTk7A!(XOv!J#}U9jw;0
zqD~a53d{fKhC`wUc<4#T<rPegt`p?7YdTraNwHkQWj5>q>MGy3bJ?lc?2ZB53qmjK
z;GcR!uJDX1xAdX3R;1n95-~Ke!rU`)&J>9Zr&a!vFf^B!GB&rKh3nH6DH5vGY!){#
z5BSHuiN8kWk=Su{ERaIa70&geNozpi$O;?Y`MSX3uJ%fp{HtQ{lqvHmA!LH-;IGT=
z_WNTZ!qFh<FV{t2Yn9R_{{=H*2nZmNcbd6#<f2D4#}Apn<Ar3e6gVF?s=z699+GBY
zG5WuSCraG}hZ1Bt{JEw%JTEuw0D9aJT_P`9^<#<>v%2bF=9YyCFAHV$J(4)P#BT1B
zPNt#YD^M@DFX>|Y38`73x^dV8H^5@iV^BlL0)(&Ug{e8d4S<R+C;}A5Y_3hdg^|TT
zEH4$n-Ek#lhmyrC$DmjyVt^x}ihE$X78bwyz=eZ?w?(u7wfqHI`UQBAv-+_>*1V$s
z<ugvam3z_py#Z!KBG6PQ*S9v^8_xYeBqtY~Kzp`~L6~n<VF2jWgjIpMa?ZH&6ZMEr
zX}m<cr|QrK)Op9BN_?4M_l8K?FRQdm`T&odt!zp1jgRZkI>ZmDXoj(#czzWg5Zdhf
zd#bp&0>!ZvuPyy^X2m=G>Oq6pF-vh%YAnAeHu;K+gGEKH%+yCcfC97YB;o84`kO5?
zMD3ldjysr8tNtX3)SimSVSLyvkPQZJd+75|r8|UyY*KFo&2QBY@db_P{YV8*50{!y
zSs`46uz)x#Jf>R@K?4^aBo)splDs?&E8I(I-~+rRTqd#~f*)MjpDG&p*RYA@=NgOC
z*-=Fzb;RKA*MH2Ul>hQy;=;ld)oT9XmIQ+|DNhNRw>Do(X;lQ^%g4u%nhTG{TzRMf
zbX66MzuB`*XXpX6@zPH(QM@rNW@%$kA*k~whE;3jw_w)RD{nV$v)~+&!`7W;!$qJ4
zJfU|KzHqinKRg8&XZFzK)cHk9$#25)KB!EM)2o<8D^YMLt|3a0nJC$`rxIu`IyIPC
zw1-vrJ9*p|xzeiO!<SEYxmlG5P7u0P#}dhl+uY(uykrayXh3~t7pHump0s~^d-=`R
z>vmYb-fEoT6NhOP`1shN-u@J8q*gELX54P~!d18GeTn5@X<%<m2Yr7;6n<=kZtOrd
z&g6w4nGn%uRc1VhFDiMh1D((NZX@AM$cXLTVbp1c%XtLYc53OqY~E{RsysSAWCS}q
zwT%_ef3jhU-6dsqmFnK&G{pgh>X#)3>a(&{Jd2AtqV`!!v5|&!b(X2`6r{7G$o2L9
z`;*<VWY<}C^Hr?A=_O(gB)?`w(@s)a&+lzDQ&dBugdx5C$FfD1b1q5;7<Ki^&J7No
zHXeO83AQ_TNg7SQ>2B8gBT6EkRAer8biykI4dC-Uk!9*v+5umvc67dZCn(<A<oj{k
z<;M7~Be!gpF;7%WV3JLhnKOUrd92<zzxrsG_EbH`>vMH+-sBZ)C8zzw>uTKJ=ZLte
z4X&A|dg%f#5#W>M3ir!TVmg^SNu`%<cQ5Focp+a7j|VW1cE3IfwCs$RuKBv#sgAG-
z*O$~eG^d!Gh3@4IoMwFD0Mm`7kC$1D7;r)#{3hkx37VhZy4l+sKHN0uU(R`M>ReD(
zk=0mdzPs4`%zJS+C1%@=I=^4^X|d`o8sm0nSaq+><9&6U|6EP-Tv4vyb6(;~o2>g`
zc{;?|O{&p>*sak)(ur~tmz2_)$JW~f#q_%0Gkjmoi73W<NPicbNZoo=O3dp{owaLD
z2OZw8=#@NPr+07``RQ(%8LEcct;^ARwS7ypxs8so(@>Zn0Z!PiyhF1n;WzVhZqXP=
z8+IwQYzft9c_(uQDyt+j%rHXNv|lsctuZuhIk`HRsD6*BTkl@eD(n;+^I+R-9xACZ
z<<iqMwfT6Ox$5G%X`z(xSY=eF&XWC(AA8%}Atu|7zny1v+xgplX^<VWJuTLU{CuP`
z?be-(BgC*-F;@O5iV9`tmEhEl9Qzl0ZKqgHq1ol6_c7|vQ{FW!>j=G)5a5PH2k_36
z6swoVC$AFDmgva5Qk*^&l@^lJpZ(o%yO?Dn)-k>8I5(DpS&3)wvpA*O^T~_)<mMrc
z9!jg*V(k-qcGi;uEY6vT6Sou>_LHX;O^Vf<4ddQ8i(?m^1E;*gySrM)9-q@_(FvJr
z$vDKc(X7)T_F_s&H?bofuYGmAdb{q#ZF`das%E5prO`eaLbXH$7MdtldtF2J2zx<U
z>`+B`I~KdtK1Si<s^>nFYQQ1(aV&jLeF!T!NV+f##v^mA_j<SU9~X|de;1eA&3>?7
zUru$kebX0le2st1wfI;yWqoUE@5g>isa0tbL@`+sgBKv%$7s7Zbj=<*K)c_WCp7yr
z#AYv)=;HaOlb@So`wK4E5h7*K{SC+B*u&O_QKy@|&Gt-}Y2n}PuUv$ilyzGuRy*)4
zpF3UD*tm8c>`UT}ONfCQF_Z4l26y<*q+;+i`}*truCREXRKjunc;431TQ=)Hi8b*@
zO`LGKXqB)mBG(II6fL_u;s>u`XP?y;F_#RVu~&G`zW)2O+I%NJgEFYb$SIhzlTGg6
z+r4y~hdQ|<KND;=eWWv{dN1$YS9rT{PBD^R(>x{P*`sPWyV;T%?dVgpStxZX)SG6s
z{v%(jM_lQ-;axSgZ}I5>C7JiOtLvc2{-+6dHY(A^giPT|Q}?%SHfr^Tn=eKl$}w@m
zsi7pZ8RbR1KD0UAo~kyt-NcoX37anYmD>7EyM3o!VrIK)6<c?351wnNe0#a<8J@rU
zPp>nqqDK)#U984gl7hNw(3U~3_IgT?gvD0G?ycMW0;z^v3(oW+zN9?54>S5cY<haL
z_lZo0q-Kvon&9Wd!uA(`zfZ5h=+Y*`3{tz!OgA-D;0WjZol~ZzS--tLJLVy{{jm5b
zD)r2N*x*dR5n}#y6=$RqfxW?Ee*f)<TVPvnu=v`HC^Ai0>2YwXS-$kWtP#PAch-$-
zI(N-(chJSKyDv<)e?=^7H@?doy0WYr>%z0>`g&}+aQl^8h_lG&RSO~wSu_H8`vO~W
zB#9$GP%L5Ln7MHWC4Ou6JCeJdJ%YXG4?g=Lv?Qa75z&~_*m2o{8U+1x)~bTxg%<G9
z5vMyiG$xDRCyg}s71yt+I+FV7s0keu(`xeAAM@x0qH@`^Ac-XtTZdqS$f?4iE08DY
zbOvT7y0smZc)Uy;oPY4?_4iGrGKsRmi;knNy+FC%nZ|iMXROX>_E2$YQlA(mtb|sV
zu_{mZaUQTHE$VvvkYRjV9Se#GRtL^B1-1nfzJU=O9JcJ!{{m>8<#+b>k3<NT=<7SP
zHN6MLb(Ih1VOg1D$VSx~ZkAQ%ahyhV*|`?62{uRJ1rUl_4x|rVTm%cE=<K47v{vES
z&NmxlGA=bc6qZu&WwvnHG7qB{RyowPcBbr(83IXjoB5@>?$zxke<!z!Y)a(iG?z0c
zp{>XR$m>^bz!~a4Be?R|skUBQ`r~j1S!dv{ihOeSU^}^X8t%BS-W|-XFb=vH$r2f>
zm$xf@m2wcRgAWty%y?yf-}%*3mcSkX(|0S6p;H_C!kOYLa#aPLRjn6nC+?nmd4`$W
zo7kC)L}N?bF=J~~eUVW9%n8=pna9{Pt2l$-BL=MpOuBKorJC(&=E{Ct<3bG!ryi#h
zDU6?mYd(QpzZ(;`yf{3=<t*FwulT97xvg9{la$J!EEm<~zITy$9os=y#L3&|!Rw(M
zLI?BPL*Smb^50~aZd5Nz1qVliv{RY7%vBpV$n6$>onH)2a&x+rWS31hQWz8HR`Lwu
z?}n0e@C=h5Ot<CgJHIAc-5L2*zhC<!i#P@B{3c^FG9h01vB7eVUII=BhD4WF*Y(&T
z3nv$uUU?@`ot@j^X}Pa01y4*l>7r|zh4ili?Y^@zbZWFZd(Bpw%U#((vUy7qJ?xEQ
z7dPsiQ*}l@wepz%%+!!r+ysj~U%r%*Fh9M1$WwW|&BcN$W!JoPJ!kz4r#EX!{H4wh
zcb@cm?J(3*L9~ZuobBT7OFq!f4}Juja#awK!m=O)tf`=0gsZnUVHsKB<J-BfnOx^2
zfUllN(6?<&8qz9e%lOSZjU81@wu~xhIC$;s*#jZ$Ew%^;gvCVa6J*!-NfxhFjFqq}
znD?4K8m7OW+T111w&Fk9!PK>b74ED-Q(gl<OB+4=bxnH1r?AbOR*e^~H(R*(l^GFs
z`)I^fiBraR3ZvW0Oa1ckgdd_*^so(Wd+1#noUyT^(&URmqX)>~!H7nqT|-fx1;Rbr
zInSKc#ebg08wG@Hk~G!FnXIB&=I7BavA_Vd-Z(i+G>REsBS=?|<}_pvJmW|PDSSiy
zfVn;+`gHw>13^_%<VY2I&Ill(Mf+tc4u<11K1rAr`=^+Z5Ad~?m}rONGW3M-iy&B*
zz#(0S;2Z=Ad!sSDmcF0Mbz4{T!RGcV39|D7kVOlNQ>zZ;cisea;1sPsK$)n1^J*6h
z;1VoHy54`xmn;rLdBzQcssnS)#h)fo)FdHPVOKhlpEbXo>Fj@O_y!wE?d8I|VvDC2
z_h5imPPmwGKw_!AsMiBDx&Of{Dq!N`Si|RQttdl|b*iYnZ-XW{Eji0Wc=z+*W_|vq
zn*;v1$+7T3y${`cUQHzf`W#BCiu#Pf7k`Ud|64rq6iWQ_cqYLLdj7Z4=-UF;$KIC8
zuAlUXE|xJOA;1e4SlZa=G^FcB6*<d9h$jfCXVxpB?tp*2kt{ZY6+wu<yGZ7frj9`Q
zQUJ8QT2gYCj?;jFSGQNFNOsrFFD~|LL);H%Rc%)%GP6qTblyP^gb!72M_d-CJCn%3
znh$0yN1yn~?MO&yZZGwz){8W53+ncpSdpiR+VAWdzKL-5Reki`o+)UX_c@@s82>-*
zT>Cqe>l!Yp6%`e!2}w%rA%|qbV9KOfsYX(aLx_^3!HSSZv^UBzIgDzQN~VQz8ss$7
za#%(*GY(}=hMZ5|#5fK!Sl`&~!@sbvYya~7@Lb<_z0do+-}Surdq4N{`meGx1Y?J)
zE{}lH8f9sJ?2gDFcDU?(o<n<>sI4zdz(WX96d#rwtA6D6B>HlmHgI_sq1lAhDSS_U
zrVZdLL)!bbcm%bzLe5BqNAj|noUN>W1|@7hxE}nE&bB?yvqH-NyMA}iFWp8R%Tlw@
zviH{iC{1}o9mnfmre;#p)7Qo3y4Cs?FVC~&YlffuA616lj;jW>L!h)_P}CS#&~agN
z)CL}`tVaxk2ACXCvds~|jY?a)$}d>d0PMmp>X=(b3Zyhyn@O(em9%#l9owd9%qSTM
zpS*KLCxgt1XK_Ic?3udi>O5+w?S`qQZ+|y4_<mVXkD_r8p+ZXB1gadyRmglj8v#`5
zM=<qVX6}CzWORl*5i^4C@P`FSof)|8s^>>II-ATIjyh(s8<ze9=0#RB$>)*08;36z
zojF7sTMS>^R5L3KWZ;z1bMs_{Qu(1vg1q+Lm=I26AzgqVs`>mK`+!#==pF(POMi61
z{H9jWFB#5dO}R|>be!dhxR>ap_*rXk{_bFUnEvx^mK#WK8>*C@C1<EOMcWM73x+i*
z07*y|4CA$(@m+IMk*4g7LOx)!pE!H#Hd5h^|A?iiitb)fcOU5d1KPjW`5G<G247-E
zQtc`uewUa}*A>kV9<~=;AN(`6Ph*P%FOAb$VD_k_*Gld4qg|?5vBS0RCjdf}dRFa*
z*=CRPL<r$stiDP2ic?kHk5JsRqt4}@<63&FFsnL7QNHOdPQ7S4FrIp!J-&_SCu(pv
zQSiH7Uv3CEm$*6)>UAS;b@>$KsdTGXntyQF(uh^}^v5Ao?<^+{91&V3l=bH*g;6U&
z9gsZOM@001nf0HNN%VEBGe9=WUDwMKxr6|vYnZrfqYjK6Na3<ijT(9~&&4kp6ZVgW
zonyh*=MLnqq~fsBX`Uc7H#=&Z3inGQm@f7}j-?pGythrA7OUWx2;Jq^DA2gB$l?pR
z2gM7T0>V;s=l7A<qne^g9mMG$&@wku^jG{PrR>hI2?&K0L2rS1cW?Eo!8hrd=aK2I
zl}*^tJ~7bxdhtQEQa6EuR0a7vJFRSw*bi?mTVeN4DSE?8P&sF_`fV4z73IS3nsi^J
zy2f3}Pcxod&XOaO<YOwSRPkl8{=NWUhrG)*TK$M}8dfCI+e(a1L|$!WVL!jR$9$D<
zoe%SdTlCFLizWT|jf;YN_amnB9jS+J-+Uq`Sy_O>LmTr)j}knH3w#!&Lig4I0*;+)
zNB8@sGKV$s0NkdYNq3;n&&T9<D`i#%l-pM$&0wd#e2K9-*AmOvw)*47zrn0^z>6t&
z_OPzgO<WO<zbV*%Y&3Bqxq@c7^zF&a-vEr=SWKrrKvs(>F<GV_S4otdOD%FQ0yBe?
z!|p&!`~e3)O~N-tZMqIjUXOf|g!x#`W{sW(T`1l0bif`KQO&&}8}YoRUYmjO#P93;
zMit{y48I@m*puT*wh1%OhH)S`IQyjR3x&!)5)h0t@hh<%aOU9sn)X5yDA|P;i|gPO
zH&toEKDmwRI_PQ+M7g*QlclBTb0f&}-Zjdey}LbildQE?`v>3Mo!=fhy6j%C??;v$
zQq1l!e%be{eux6LC*w!sU4dz5!WKKQHTQlD$(8wD#WyCz^mP@l`a{rft_9%rVKUJ>
z+b%l32@PtCm&Hbmh4YGoV{rkk!h7JED1n0DFBJMq&(*eAc-<L78~=BQbopD1Vod0G
zp#PUb&q-UhiFkO5K_?UqU9#UkW6w7kh;d+-yzL0yd8Gg0B%0v0)rT^ivw_g_4+d$R
ztF?Py5(G)OQ!!Ha-2GU4GvZTFc~OAO35tlaYyff(5rK$-1amG(tH60gbuPmfEJ$?Q
z;x$;tC@a4SukRNZCOW#uFBkcs>iQ^~0}&|>grEaVR>fL-$Dm00%NdlpO(ew>#g`-r
zqw3vK+wqExK?7O|ogJpSfs6L;aO3b@yxTA=`N^uPC&JvIG0ECv<{btw05%JACfZt!
z=D-YtM==`#!e#Mfrp2koweVZNxlufk82a>8KwA&2Z>7_KMES^%-9?1>TulQP;q7#r
zuvWHGO@>HjqVxxhFy4_C@%~z908~>3v_9?ep=qK6mVz;1@MF~Y2BOzP7Y`65vzqw9
zd2`WZ97k#3H-bGOXgcH9aF83E*19ONrAZ3gEO$Isit0kXPHH(8a0QynQ7GUyl!r|l
zt?cZr*0P{2S*DB#mu9a%Z@TuRdoAd6Db|D>y!rxyY;Soz^kP9`;J8eB+(oh5KI1c4
z6qy}n_H~hu?5kzxXF^)O?Bi~oOqS@P%~D2aUOe!E4fF8=^V$n14J~`Hh;-^6**zL|
znIwK0rMSU?Jr}L61La#8C7NLN5@Ycwoz=&0eaLCD2X@nSkH|3qotC&%wB4=I%FUuI
zkN}lL{PJN%THTNqIJCf=zQ&0D#5dp@Y8sz<*XMoum*FvJ&@isZgZ>0X5r;w4HW?B*
zuK&|uDTQ&UL{i?G92Wen7i`fKiq*51;(wy1WGQE9=as)!Q<07u0*?CMZv4}k|3Cl7
eBcOM9Y85j>xPU2dmfpX(?yNQXbfJ~UqrU(Kkb;T;

literal 37973
zcmeFZWmuG7*Ec-DfRfT364Kp6D-0nG5>kV7*HBV3AfPamG)SW$BGLvmh;)Z^41$C-
zNJ%~8|GMt`x$gHk-f!=R_ruMXnRCyMbFIDCZ>_z87y7#DB!mov004kQQ$y7d005-`
z06_FDd|XRp{5d7=1L$X{ejiXZ#I%Wfxb3ZB?gs!6;eH^Z1Ojqi(cwCLcQH0YnCWOk
z?YumMZ0)_CI0yxKc;k8l0P;al+*c0=ge_Z;hr6dAG)RH{pB_-$_rJ};>}>yZLAWWf
zo9XDYJ@E2%V3QP*5E5ZmBxGY_llQfEgc_>C{yiM`q`>ZsKzKuig#!Zvg#yKeynLO6
zMP+4Wg+;`K#l!@0Jp}!NJrTA+f}VaH|C;2#=23O<v-5TFM!0x+vi+Uc_KBB2LV=zA
z??V6m_pkRvxH$gbl|23aO%{$o;lIBKiwcPd|M%Ruq4IxQp%1(~ynP+~{BZLtO3MG!
z^MADc-*x`gU*Fja;f3RZuZx|gC&IxOHyC02_w5wL{yqBty~qD^E<Ilt2OQP^j28X(
z=>Kf{cYk@|zl8q>BmTwZe_C;DRwR@c{_ibQB&=Q$TL1u*0h+2R#zDaCJc1yqM~u;4
z6ge8|PrV4s;b7G}Dk>1_5L4Bf_!jsSs~im#QGh{pNn{*}ni@m}s6t+<nVM(}d1im@
z7o$Dx-ncig?djXP9$aVSnJb=~=hYgrhZYs%WrLI=sggm4%7hxg|9$vh9{le)_+MH0
z{~I3ka6)Ru-qyIjqw0}>BnEjjrMq|tRqb3==xx^jX&DdX2y1oOJXz3Sb(o;wy<?8r
z?rCZ;{hxS7^+;LQ-aw~b?LV8@;y}LaCAUf!{s;Q0Q72QH8&i6u{O3V<UW|Oz!vk{p
z_@BH{Hh`?E(tKC7_#Y5j%)`bl7N<Nb_@BtCZw!=8QIi~{`A>{R@euaBK(i+YJp51S
z{r~b+m#KS(#NLFY3K)HSawair<V%G;EwX+6wYz^${R-*Pz7=icrbDQK(l#tKz9(to
zkq$B}EC-d<_lae)q#0G)bx8FKEZGlq`yCF@5(%U_QE&T(6;XLU@Y;Q4$C91wmf>+&
z%|2yO1p-rl+Qqt$zw4~Zx}Wc`9Li7!&vSs?*k*6?+sS=gZ|s#?&G_IusMjB`wtQ;W
zvuas~u-iDs1UQ@HRv$3LZH{A}aDxfqvq#typU;jQOg%B_b=ntC&DZ`U&b;xV?w6P2
z5_V_VNd_f?<^s&tCVk}E_|ZIKUqaz|0!JE2?sF-jzsfe2N|sD59Df})gJMT{Eede^
zN~QOAkA{ybO88KOmo<}OTa64y7YzNOQcGf|%=sR=_tOr)Z+5xQ7=R$7ce`H$I3`i#
z-7frSdhsc-*ok2=aMRD}l(kbQ^+XM^KO3rSD;1V6UL&b`G;s4euDTeNyjDg)!Yy61
zNsnuFju+78aaFxxu5^CZbME8Kk24lJuf7*#1c$rk&!7NC3aWV<JoCpoRhv<}Jy&i{
z**nwM*Ol_(zLP@TMT%oqGm3Nd6)6r<7~88`R%@sjK!$9MLxyavxfFYGSLL7XtjvgB
z@fF$~fvAg_7>-<rY(j@=E7yVQ&EU06>ZV_}1oZmfn!-V?>MPd+;#VibQYF_t4Q+B{
z%qbp<c}_Fy`xBEhZ|<Ah4bf%0kt(kq@`Wl1F-Dc2MZEQDdi>WE_=r1^zfEk?g&>A+
zl+{&}rEX$tJaQ9wV{d+6@A!Ey4~y$*_lduEbEHp<vN+bu*x_7*%UE(CqL#qwzA~@A
zNJsRGE95%O*$J*+=eenu^L^c9924)pY#7ZVXd4!fmokeXedh6-)RuW9A>6sc{b;OS
zKFuRL)xsgyLe44ovzy<bZ+(eryuf(K2LVLxS`HT@+QovuKW$F(m7mkyKe+*f1DLFy
zj$C`jMOA^W^AFaF{jZ+$Nbe}jJG=Kwp6C#?IW_RvXIEbCJ(y*F&cKPI=^c(H#3Nnm
zdl17i%yJgwf~;66y^ntUZFr;0=U7p)%VgZ<HWtfW7^=oHS;T3Is7o~rp!@@&DMKe^
z-0Rr|nR{r$RwR$Z@S3@!k}B8Cpyv7q!x@3`n^@+PQlhoswRL&F?R;;}Bk&8QYlm#s
zo7qQXdA<3Ae(p{_=Z^gn<*MD2S!!E>0X5=%Tn{54lkPJ!T{%ro?Jwqohbn%A1RQjK
zj#(1S#$gMW5e_G~j+(d1<BLG*MMSc+Q>P>rg5v95{qle8^3BoCHie>w^`^%Hb?U)(
zeXDGQbiav^pQ#~TyW5gCUwPH=UQSlJ(m00gy=rLkdwA9+X%Jq*C%0uQOFg=vc@Y|(
zzd0(=jg}GZM%%b&3)8M<XTJ;F9bfwWs8BaL&tqjU*3QVg?k>TY!Gi+BLVLbDh}*o`
z>>K5;m3{Q#n9R>7-*WW3K@p>ti$ZfuXrWyj@<*#H7C&};1QI-*py?^-p)Q>~U+w5I
zIc}HAh1&j2c&idQN<PrIYIBUI9|6&0vg$);r8p(Uqm8RhUYh`iPS#6GOyC-k+JBK5
zi^I+_gZsFp>G(0dLRJip*+)tP4Na-R<&u3)x%Zw6b-9L~*njNm8oBH6E!E_nv;4~Y
zpnF)G=O$M1o`~+D`EK*s^Ygs`JH?JtV&4NGeoC)^OiJ{)vtSsN48sG4+3$l=QTqb=
ze|c@lrjKJb$`yw}R7Q}12lmM^P8xGv=JjZ;s&CtyW;;X(g@TESw9jW4_oJZ!dF?`x
zH^PpO*AHhz&Rco5e4Zq|7BxQn-9i+Y%}S`ALAvpbD`WW@KPa_VUVh|p##6|dyM`{J
zSa$e@dy;5ftVEAkiQ`K9agcl3KeXN{$Eg%$$$9}u+*=UqV9lKu5^Fm~zQi0TBhh>h
znJ+0J64BhWC4(1pS?x+|Iq{^JBW{Z|%(?iQPo>S0kEzU|gLWzX2=82TplhOn-KmF6
zF6FWa{w!Bd#KW?@#6E5LS5HTtSsC<W5RXqxQMIpY2TIg;<ug%Um7i_ht&7&x|IG1y
zzXvH*hVmz~Drv0WtMD*n^P`se(XbP(P)^0FMLkfy0OEd`WUtvJbC-L*lrSLB6sE9s
zz|`yg!E&CN{GslrbK%~AZ+^r!$GmwaZGku?{9L{NM~2iwD`TgQ4YS_CBjuEyA0eLp
z_J)PeL5S`7YHw1`e#gM$rSqg`w@s&%yzL$SrF@fT2)`|6rq6W0o%|o#@Dq+p8nm6p
z2LGX>d3bRWEx_yaxSRJsbSy1SZ*LZ7@oD{sisw@MD^5SPw`6dT{a?2nY%xylTTvOQ
zY5rfeTUd%ad9|glT$s!LyB{0G(pXuyH2z5i&wps`G%6hMXQYY0Ur7VG6k<+v+y-OL
zQ^yOrI!kT`&wb2JPhE;Xhf4{a_3ktE{9_n+@+6yl0Q#*U4Gm(fZ88YIKbU$ul^H$B
z)%$)crR_FmuC`<rqTvHNqp`$;MoD67YR$}u^YO?4tTv&~XXBC{eJ!%YuL80n`&Bqs
z9@cmPO;Px8##{9dYQ!iIctoH;RWn7pix}j|xl4Zcx`c>CghD=9>&o+`8tMakzA6`g
z9nAvqcWzp%enlGg_F?s=glP!K#_x238kyvYUxLcr2Oj%EXP^l{;%M*;D+00tWF|1*
zCY*qc5wsV;Sx3OCer${!Bn`WSC*XEc`!M$qf>+=t{%k1|68J<YJMo(8k98F5++NUW
zslQ@HU#c8{-|$EWXG>rrAg)Dy=|096!efa5+9){;8JxNtz+@2vfhwUxb?}{l=Jt@s
zaA}UPI^|$*fDG*=K0AHG6}`TjtL_f?J(CFxrQSY`VHOpThUG*9H^aMqw~)s{G-Yg}
z9JzlluVNaw4W>ubP8jnhgCKNjmMl&!!|Yelg;bv?<ROY6L^xAjwFn&(sVBajG6x%f
zC7>yr{(Bx^10Y4P{n8o_w_53&iAKi4$qD)CjjXX3p`1bL#9NgdBEH$$oQLXZT-;*#
zqv@)IrKz|>Grs9wE@;ShMO9d`Z~AKRiK;8hMqWWu;_Cf<-dhOH8rN_LLF8V$u{SgJ
ztmUP9(=vARnI3^K@&LKM>{(5tD(0$A@|H{Tb>oCg+CpRkn(5vT<y!dwtDE-d(2`UO
z24g>|ZwIpEE_pTSKK4e2IQk6V;y}W<;=`BM%76Hk*SQ`~@J|?woZAMzv!jLE3ek}S
z1WK(%cy_OY3z;_{(^rovy1OO2e3XbYKv1kS(gf>*u?3Bya}4BMH3Z-CHjp@Q!&j)W
zGS!mCk#vna(Gn<K&bk47gwRTZWKT6i{R`=mF`WD*Lve@vjc&+y0f?9iD#m&B2Ofi#
z)4?r$UZoGw!|{M7rEZk|DBtcDDGLPsSdqXr>W8QN8d-bVCD`G2vhJ#x@<K7aWU<Kv
zdbUR1m40aUNq0QbUw|x=r!OGi6s8^3ZRR_0k9kRTTh9K;X$uYyQN#u~BQWL_kE9V{
zsRjzaTslQ#6}Eao@rr816RMht{LoHT$a2epqdFzA2Or}Ta08}R_NmR0p@T8Z9N*9t
zQkt{Q-=c{$L+9u-x~je}r*DY2Z>Uc7Kakf>NE7K(gcesyPrvkG?=Gp<ZHKs8;g<aU
z_fqgDpRqxkurvx3Qbf+Ql=b|;N6G0~Pmp+{C`DH<yUv5{^*6`Zd(}$LptZ<;Wttvi
z{^%I-KhlH)K&GT1QD<;@H_e@R1F)-#PC3b6iUnSdZe9}0gBRn|L}J3+kI;8EajORV
zwez^%D_FVn(m9oir}Zy>wXU0}--En-grq%MLk?-NpNNY2<9&Z%@Az@c3L7$V_glEr
z_!EKOK!x@f$do1`74u|Xn0n|PGs2B+S%Yq(>VcJ{??D6d0j|n|<+mG(V54YC&qZKr
z`-4%6?&^Y+p3S>sH6HSds_wVNJOCG*FU^B-XtZyGlb36^Du&m}Z+J%TkXV7Qb*kG3
z)YT7HMb2Yw<?`6y$dahm+EgEDU~Vxnmq%l1^dY4sJBctK;tkb(!5*-cveUlUH1QKj
z7LFwUyVx3|zob8DC`(S3hP={ViP?$2e_u8wzPg2UTCiQK^}b@_>vSB@CIfLa4<J)Z
z?`ff^ct=3TwR4j6jL`$#>O5G=?UM2RTc0HFL!G9i&1CQq1n3|^0NT~pSF_^10E0Iu
zh*AKe9u=aZfDyV3YFHILm<~eAo3Mx{T`}HCvP$}fzBO*JM6TB*;_}c=jw5CK{`PQ(
z6s&4yQFZ&F{jUrS*iYPL4hKf7s((Iwc1;6i7X!LmqI2ZQ^)9;EeQ1>l_+R<BTZMR2
z1OZjG;ujigtipgCPhM4b91pjfvyJl5)pmIx$~UH4@J%A$VWbOX;dlYWJ&+oPxgk4j
z&3;ypyL?wGa_eALw#Qp?10vO#E35tIsydJKpw0GVxa%at_C#%NK!Y={$E+1R+BX)C
zlWchtoNnm`T|5fqx?IiZJ$ToccU=E2*Wu*tvR7Wkpef|5PX6;U@~xDth5T5x-g?l*
zrn?Db3H67K=7Q*jCqVl~tZ=+uKH@XKVoS9l&u<>g9}^&qUUt%3*jpxuv=VYacWfK%
zE|-Mrm%qotm?TPWmw9T3Ea<1;Wz%MXUdk_epAZD-$E&(CiygdSpFQPDsZHk|q~2KM
z?4S6Mww7RgM97QNiOEdaAAOxW1jK)QV>-Hac|D-+w3=1a;;WsTbIqh)AT_i0spU8I
zq54kR#E%%t-k1pd=?2C@x)~^>861v(fN=A5Jen({Ed(Di1~Cl~JOFl6nBc!)@EZLB
zf<8wC?cpN=PVi$R%K#iHIDUSWQUI@^?jk)L=YomlX;_Sk1MY~mk|q+MiNClAgcv>p
zrTVa&Z<iH1Jv(gZSA%$Nz4BO8oLRr6nsu@?FWb$e&&@wz?V+m<Sp}A^n7^peLD1}N
zzx22#P@{dL7nlXK`I!yb67+L*%bTR*GOpbx+^-lT9I4Tq{NYb1qj;#jfA5nNs?U_2
zjKh07Kmq8DUgScOx4ghc6pBATNit-18Nzob;HFa`-q>dbKta&AHLx=?+AaG9N{k^!
zYP9#Y_zX?;pnFJdSJm$%3<TM|)hMdfV|UA`%}zbuTh4BpW$%<Lh8?F-G=RcX@Q-ML
z2h=hZ=8fjnq($h0qr)VOSX*Va*l*{FZ3!df%KnCkC^V~5M-3Q)jDh>0qr(yq=%5w?
zx)gcVZ*yvBfs>Guh)!<idNE|;d<yn9#wP^vLuN8Lkxc0$m_@Dxb;(4ah>vijN1+`N
zG@Hr|_-I3@M28qhq+L5RHl&*5lZU?nmKH0etUl)PE(ydRX--bKb{aE#&-JoZ#BdJp
zGTZrXA2-g-I>pKFZ)>|@*XdgD@EB$|IFFD{qv5l&<Z-nZ{3Da!+$YPwcV53cUvB#@
z+l9+>52MfgGwXVN66Tk1_u$@+wGY?z14l+j1Egnf1Xiv){016qR&&BHoN{$ZY6r;T
z1&{VsyL}cO3D5bcXEGiAjFdmR-nRO=r&aWt*bq6CVPPN;PTz$r!4R>TSj0VZ?`}a_
zb+3b<{x4=DdU`IOt<VL6(SUqnx|(K)Oxp{$n&w-^1oMG1(*P+D3t1kMA+Iz2jZS0T
zrp#;kT@@thDh>8^&3pOAvi5$KM-z~riAjc1{y%Ca@oQuji{cm)66A%^TK%EMn9rWX
ze{9#i!o@9V?+G>xl$;GJh1d|=&WiV^eSJShwgC+B_06BjXgY$XY&YDSxC{Ft6|MEb
z<C-6P3IAhL=c@e}MRwC8t}fLtD$JB=*J|xNqix|iV=6<F9~JQ%mR>gy<_%IG=n?-A
z6>wZT@RPtHSJL8e!0AZ9h;#{wyLJzh9UTr(rXFf<<(RL1iK-i{h^QUtG0(7qwP=|)
z`h9XQafo66^_l%y$Kw+DKYyOCW@$#4`?94f*0~rcbJzb6r?vRHtCvBaMA`f6V-yfM
z+OUk3^k(b<zo5J3ckiy@SG-;F$_SH-Bv*%qAcQsn5V_}x<5|&R50NXzhvVjN##bBa
zp7X`gd7io_z<wN!vWG0pdGo~mk&5}T#!kXf`vRI$l6)G##!bO~<}&2Jl0z!HHp3hI
z3EymOmUL!~{Id1Kl4JH3xxTX%N55tK);BFn?wH}};5?0!$CRPy(JGzx&4zr2zcvRf
z(A3omoz)Y~x|U;fIwTb<*(Zi`DCD+bH%M}l-5jeHA9WI)-8~~8V)o!=e?~A>r!8E1
z<!)Uy6X(44+RmFl+v}mU`C8AL4B0#4A2wB?jaaXF|7)s0U#v(nEw3@!(cpeLBl8-4
z>>Z}gJRz3bc|D9TdG`2M9VWc#&VZHxN0c+ownTtR%ug(%2<SHu0RSiHVhd1=W1*j!
zX4MJK)RnvYA8sXnjqTD$t*D06KA6?l?6O^%L#8YsZR+$j8wQCW(~Oc~KZo1^QB+QM
z34^;rK0A`&e-h~Oe<N2#dGUQHhVt{)pXIKGF4+vT4?4O}!`{Gv*O)#rGt{HA*l>oS
z1s4v#K^>d59D#2d)9>}6*a6RlV<k`4%}f8T35R)hOwiO+qe+G=D9MZ`HPwMH!?xNX
zGc2|$UOzy2M<4`t_UC;U@wKS1F@;CJJmgnV?{h)!%P$@J>hhF~^b2;GZpDrG^huBp
zASgF#Vc4u8Vd?U!y#X#^Mr2KZjyv!Z#vH_DMg^)=gJ0;-JIFc<Cao}fL-xmV3cas=
zy`5DgF2J{Q)E`pWD#fyd5+)`k!<yuc+4gB8I@j{tF7pDnKj@r^rTI(-gn#jQ&udv+
z6VlIal^Z$kwqOInLVM!kL1;Y1r1F&oE5B_g51&gro_BFQww|qR1poFdSSs~j#x)H&
zoZ0ehE5`8IPBjK^35b)gI8-Uq1{0(maAY{pe*kpf4c|L7*pF{_w&&ired(YSf63%_
z#x6rm4E&C<EhO@0Pe3pTgWf6oNL+`Hqe?sclgJ2yIDcWQBhB1dDDuc3e!b<Qh-s|@
z4%Z2QsWsLfr*1d!f`9h)-0Oca&@<jxH%cP%#BH-HujDeXqa2+m*k3F1hWsvT*US6U
zs4TcR>k#YCw+!<)MER1zieEaVj|}nN87R(4yk*Gc|4e@LQ0cNm>`}k@T88e-TA!ZZ
z==YjBrmI0S2&qc{=Kg07N&~q1zHKDLuadn4O@In0ybDKc2k(mq5mGsok;j_C(VxaN
zg-CSq<d9P^?w5&H0#d9maynm%+-F$iX$DjvkIBbeOmp~>2O93pxEbYW8E)rLk8GN6
zC%!jhz0*`su$^s(&zf#`hB=^l-tpTt-(k4$nu^l#dZu-2UvEjSKfk?2PDt8mFV~@B
zC&OVe|MU2Z`Gib1x=Y=;vqDoC*GhwF*?P81Q1hLpG4mvW5^8w;J^@)GGLe^g!mN^*
zV$`6SxEaq8+15^3NPWieiZp;Ap<PSl4%4l;rUO<4T{)pHKJaG8?5Hc~{RUy{^O_(S
zdv14@lx6MUbo(D&!c674@rOX^50ai%xnmEDeoQf_v5WtH<vSu;622YRT4dKa!)!+<
zedClZ>L>UhR%m2eO<{F|$%uMrr6NMWZ!hpOWH|NGIWby;6INaD((bzR^>eEq9is6~
z>f>j7!Yr3}hlv#_lBl2(j7ije{8?Nq(=>u*Eg(eS<49hX`ZQh6Jo&^GKiIEB>E}cE
zTZqEcX?$->W)8Fx$_c>St4}&E=I$i1%Y$)V)%GSayA@Z`5)<VLcnbOrIeQXYL<6I>
zV`iu(cac?jOH&wCP~MV&rj5*`6jVOHG{OwJCFD-syP@I<mh5Ep<qKv|Q~1GQJ(thg
z9iKB%hZh<s)sz8#Z|iV*cp~KBN!K*)V*gA;<UV~?BjK0!k64@n2zRIr`(9SxI^LUy
z@$4Bo@Xt%-f1}~4C=@%ie1WB`CMX=ArV_xf0~b=B5#6HeN@|!!lMgUrj<E*&bQ+~h
z_7CM&0&@eOBiMz(StjebBru&<r3f!3N#8Tju$q{`&O&ko1U@ChZIXWLVXT{${QZ#N
z_nOBInp0^#OL#D#eWx-CP9QdMH`<RaTgrRVs=S%L-a}^g53L!)?_MH<<vW5>J#jpk
znc)K38HO-$NxZ*G7>0*_Il3T`y<ITjN`vLw$j-WOYoZ=B(GL@W5wtTu1&5PuC=Y=H
zBTZHAEwPWH>?JZ1XU;Kud%j*1z0O|pxFdeftkLztULw_wLk~C%GRNn^t~I=&v&t32
z4s<T5%BCE+Wr)g7|Gestldq#EdoHLVMbrJ5Oyt*h=8?C{P!1aeC-b^5c?SKA8OgL-
zx1LkBl@HhV#4gk+2{Z^Qg1ECuXQ&sBFXU(8QB9%0#C1OKm7Dv?i2172tW}`|T`x6R
z!f_tqY?KNzrPl%z%QlYwGme@iV7h%7z-CdRYt1k7ij5^V{@dV>-aGepa;L)ulPT$8
zg~nb&MN>J({rNP<N3kj<+SFL$9mcLdVJ(`Uv$lT`Ib2qall#8Xf@Uy%=#012(dS?l
zvh!BH+x>EAJtlMm4(-1rwq8PeZ2oE(r?*wDW+>0U>2+{wxtsBpfVl;8m0mqmr44mN
z>Ha81dZ4e@BAuluZx1DA^mJaO8km@l5Xs-dM{WV;@_A2~82WPAp~+_-P8R5*nevq1
z6Qz`efX*Z9LSslelsPcDO6EDd`u9Ibx&+V|jx))oydAY_`11inYTR2B$eA<xf*2vN
z2_G9R26ycU9%g>lC5SH|{aMpY?migH@#r(tCH<!as{quZVqS|fQNz&hpNBIf1t{+j
zEHv}8Ai+rQ72^o+Y8iNnANdw*e&<sJ^W@tC#uG+DTh&3Y3q0y@{0&Si$r>u8U7F1*
zVf~7Jgpi5Ea9O$dvERLo;b<8T;FxrRxL*0{T+IQAHwzsdAFA>14f5ai!S|d+!jWaW
ziG*)%Gd=pZWfB>g4WZuc1*o|`xKdLPGf|>_%;R+_Ys*da*liNdc_|iC&JZ%W0uZaZ
zR@!Ro=bm9_kz?E3D+%agM|&=oX<yz9yrDhTtIb`~aYKl$MwMXN*r*%1`QG1d)hj$y
zN-5I>o!@t$k4e}A$)*%dj7sq)v;Q?!G=d&kxKkIFd?ny;n(aslRL>1T5+O6Yg!YsG
z-7`7|wZ*p_#4Di0&G@=Nvd9Y{bD9BiT_nAWkr1}X7nzTghP<sE*B8zEh-J(+n?0hX
z@G-;l%6h4QA8xv#!AjT){_EfxQ$}q|I7j8MLFSE-c1v`fDV=42y!6}175SD%V}BO{
zR@%JIKxIa6IjmW)8XSP#k2a=b7J}9yI76cB^Mt8V(nepD!d}oZ4@YALo$YPXxKnYS
zZyl;+%ItgG_XoXofKV_$=FZ4sgJFDx3itAzrPEE32z3R)&<g%Z*Xigb`Vk+u+tbt+
zgww2N4os<kZo{cA>@nTcug4x29r9sgIT=J*gBiI~lti{}RqE4{M)OpekP!rU`Y6aS
zn;sb9Tr3B2Wn5R!Nl^f1K=+q};ttWEK;XyXyF3-s=JZVD9G}q0bK8donjA`eT7`)&
zJ${PLgpS}LEyrgWsSleG*3<r1kiu~_Zg$$z*+0d=4`BWmF#@o~F3nZ4fxg_1Jzy*w
z745rduq;@fU=6v6lok`?O(y^AZg@VP_Bp{)_FTd4=jR9oOQpcbnc6?Y-d_bUi72?9
zHw}Dhl@n8AOdF}C-1rc+YxW9gvNOB7ImVurU!_~>hjpvjzk?m@X7iRZBell2Qlatg
z-_QspI%96RRr*x>UJg9SLx!NaE1lE*?xB<}dxW<M)FP9>YoPEL-<iMdp**SLZk#k9
zf}K9E%x{2O<rN|O>>5{FP6APQToz33AD8rI1%TD5j-fuW2=(zUiEy&Nod%=c%dnIz
z+COE=gLGnt#XWMDr-1IO7NE@5cO54{r$cE=w293xyMD%!DmBH)b(2SklPYi<vJ-2K
zVNAELye&3N5XfotoFF))P+Pj03g&%&<=cN2(TrE`|4x~#-`3(3*zChhN&{LfPrx%p
zT>-|wxn919lA^)=%KX@vnt^oqk__RBJ-<?f_z&ltms&X4yG>{r1@mPxmslZVW$vX$
z>gMJ)5)g(E@5{zC7j`q)+S?xziqKy|b_oWRHS6PKHWCUek_c=F{jUgoLo3I3DuUX>
z@I;`!x7gcOYUYjanxh)pDcT}3soXUSAW#L<FC(uW=y$2o_T3hUBt0F$khs)ZGVaI#
z<pCQT9-s)MDNY@?D6@CnjJ^YUNuA}Yb-%iK;5nVb^DJ=(<wMoAWVQIb`%eg?`luEk
zE~24nP$)r)0B+E%09oS@&2YtAKR#b1zCOd(_$>xU{KnTO#gS}VCXQh>yxy<7kjuZ+
z$R7<*e0-xqm<Yf9%RMXd=|a`ZM3EbUd_a6VlgO=+;2$K$GAo=cu!!OSs~CPv$4*np
zb{jA++WtKYA$(#$r=+ft#n`)UM5ce1WHXrb1Hm33|Dpj-{E3nYQh~Iu6cr=Gm#6hY
zC1P~sh@%gr#TR|g86S3=;Q}y7Dh(~5HJ~g~gSH<-SYa1kr`JYcEpz9+{_X{wcY`1Z
zmGpgsssN-2cEbrOK2_iaAwiHYA?YE(L~p#ePaW7>8Qe~+%zZ$QxuFa{WtPJyY7Pnv
zlq#KBcW@?S=8#6Wk<2NZOA`|D*l}hElbD_Qrbwp~L>>V+3NHsQXthC?c`}R=U{884
zEf_dlodxe0H5VX4XbJqnsW0M{_QPMEvK(fPdrzL*g!{2+O2st{!hBQUYvp_zw>L85
z4eEbBv<qN%bAl2`cNdVO(Y;s-jL_>Jj{ZK%h^+onEM`$T?;zC6aO7L*JNGyWne7#h
zYVu2vA_Q$FDppV>MMB~?-u`-gbpB_Q!$o=7+$0|41pgx#I#$<iBEsB^Uj<@D&fSP3
zjD_*qBA41bk#o(}Vfq_<7-yGeSec(=b9S2b->A<WU0}eRAn*XjKv%Qe8NI;X1QxF3
zCd?yH#CxGiSevrvyf0)%Ha)3Z_-WcoCb`TAAwpKTXkla=$XDTDd}+QfopnnJ46`GM
zTv1_8CY>Zr$RZ1>ThBO%+4cU2<6dkV>kHU7zPrLVne_e+h?MmwlvuABZmfzoO<~6)
z!XO8<*kJ@p+OK)0&~N+osHQbOMGJ~GvAJ3F<v+0|OCva^mey))^w3H?GI0@wA+r01
zy@%N#6DFosW;X$6R}r8+U%IJ0N`4C~bL5?tRwKZ~pBnL2%<Z|pJwAf-m8aXtmT0RZ
zZ-7ui1bdL8jN&nkvQx3x{@mG(4Cbd6eY@Q7Pcid(YHZ7`pN1<fDqgry1j+b19C$`h
zP5Tq#%_T#8c78LkyZRC6#m7Q0h@3KEq~4zv)~qg$>-k7}_SPs;=aJOkz=|fS(Rte!
z!@8r2wj(!<VnpVWCu17-2@rHxt02~7Z3Q%(7>w3EEN)Wjk&q)Laa66u2$*Y8@yGz>
z3N(9fjdEZuFwR_FX0WJO%sFx>)^v!=t9oN%Kc-iwi7a|y*AaE@klxKb*S{y`Z)bnO
zrM$C|>;;+Fvr|@27<!_o94nZ86Jf8{6ZD{eDZilI(wn-_ZY_57;DUM|pUYq+0aLCj
z&Tn&I%tn2q(P^Z~{QV*sE{{#}XP*RQ!vIA4u>qp{UuR23xQ~d|)S~TJC>s=#57@ow
zv1(hqA{3VdV+%pH4i>SpB(Dj63T%xlL2BW)uKisnEz0~OVyr@`I`9BRghmD)hVhI1
zoex$C#M>;29E)g!v7?(P>6}cMCheyX>B@<r*wHzxgXbLO$n816*<1$;NkFGu!DWHA
zDuKP<_g{q)uHLt^@M8e5f~(rv@{NZg`0`n*uWi_bIX>>)Gl;3B5r=%e3Zky-m{>k5
z1PD_UpzgulB99L)g6tHx!x{47vZJ*%`F_9%{h1bjuo)4Br1f?Z_)3Kl!#e8(6n?Vl
zOfp(-qim(`U4^qr){ND>**PT36$7A&TpVuAxBcLpDT$oS{n5x#=-0g<k@=xLjFd9L
zw!l7z#-y)4W*0g7)ehiAeBL(b?-ZgYa3n|9r)fXosiw&P0GDH63*;dQ2Pa@)$-1!E
zOFnOjAm;tGV=`>n!N<+8YWjU?;Zep;XpbBv)t^Dkm&#}SIFIqO2%x3n<*EMSDct+z
z&qkH->y07&=8*O1A)<H4`+IoJ7%QShyk3S94pYE2Jc1-2e?=KO)8b5=bd`gF^}ZlB
z?r8gom)D~~A`qQ-49L@h<|6psp?=*hM|&WC^+j4%lU@=v0-!8%0A{k6wABOtyCp_+
z#Qi0d=VWkm?d~rTgEoPU;*lBzqG_o;VKuY2I9GK>IcgmDMVRxJ#xdUu`3hIg2sP59
z<f#0m30g1#$>j%c!o5a>M?q;Hg7|PG1m#Z|GgIfa4XYSo;`PP4;%CE<DJe(5{^P#c
zB5Wp*LmKh;JiGGXzLxqSAtwIMIQCKX?b5O7Dm7&Hr4#(Ysw#fP@=`5<u04AKYrM7>
ziVT?db>UEHlj;DfLigP_BX8Ja8m2CK2m0a$*u9}W2u5(WRD<<5oJZ(J$aSMfUD}8(
ziQgd0Fx<$YfQsz^<4_72V$O@E4!bYprE?^rIh;5vu26;-cQb7eV!0Cmpany15y)mS
z=+lpp4OlG<(Ho0N2&9FC?&q@4HpI|DzA^B@BVBI8wKh=EQ1tNJ;}XRH<~p}&{*pTL
zp?6;mc0VG?mD}En^Mw-ZM}*vh>#0J`Y$Tp2mM`D8dt8KmjTu~0rOkQG{#OYk5^l#w
zJ>eAY?u>f(tkxPt1kBSCPM}2{VAZQX_EUdX2@XW1BU31$nc+D13lXNhi0^nQ%rL>2
zZQ}EO@LXN_9%<5+<@3YbATxp{H>V)8aQG-9<^mW{bOV&3$?147*7>yX?q!0lbR%Qe
z9t=uM?Y&}V9P>A}T!=AsUYTL3ujB5!B=UYm-ZbAV7Vy2s!j<HVl?Oa8Vf}2{V~A+P
z-W%#Yz6tZWn_y)@d4IPZ_8IZkfNlPK-zCIjqJt;L)M{|ZR{Pu6yF+p61>^d=J}l{j
zPZ%Rr;DTqjm~Nk0r06!1G1M_2d}|C48T3AX16XX3Py;O8gji=uPDSCoDUic8_Nj5K
zkpM?5E7BaU<Wv%0&<r@nUy4-2GU$^BIV_c6Q#+{h8ECZ{V&qsAJtA%$%P&8$+AB${
zUQ)>XwXQ?q)i*D3og`M<oezdOp?f5Oo6eUlkBNhfeq|h((SJ8KZM2B1V8Sh2%!gsM
zhRrl@S1Hi^6b2_n)nXpXL88U^g2C9DU|WKsk&l3_<>)-{@j`uv0o}0Xbi4{@;O<`A
zO(RRlw&**rrPfJmQ%l>HtCKksYisLLfvW9X5!*=rb_lImruEae$K9+Esset$Cf-iw
z1unYe`A?cpdVE}P)X9|+eDYK&z;mf}VYbFj{aeMne6LSNr!9K~7&hH3l?d70^WnN&
zx?~>Nd^lM?n2ywYeg?6;H64{))hNf%rLjFIB)$JM?etZ@Lxl{El2T=3$h1{|Q=ssB
zK0;RD8t6l}*5UnrjYS!K+8~Dn{e3tl{=%(L(7U5m?e4lNAJ40@qj5bY^9Hx6MqSyT
zR9mkzt!u4A57gfItq)CJoi4Eo#q-b63O=^~{@!Bh&tbpt`xFlK#|aFQgj>lZ3=%yR
zW;MnB{emCOyqXVUj<Hc>kB@(SuM5Aq3N)R4>3mmuDYQ<~aUkpY3`ZVi(kY4Zd51z(
z2XDT3^q!xI{G~jRLy}(c<z&wVu^B;-QReN1PfUd3`0`})QrBzDI{`&}djFUmI;e$Y
z<bu**2^YdvscAS9v?wYbiA@)c3uH8dO#83Begs>V2C__<B@d7Vo30js*%O<He{PD*
z24R2%W?+0`o_aZH>8aUThlfQTo*&RG3bWAVXOx6-uf;!m%r!>(JE0Pv%+xx(*G#=9
zo4Y+6aeZDk@|4d+SLy0E!k?YK#fJ0*7;dIyd{|3HeITbu_Dy^u`ey1fVvU47L|W!N
z!A?!QQ-%qpdU>OZg}dGzeyR^eHTSXqV|qSDh3JEtYA!_*xtIvAK`jbZ)e3#<(ZAF%
zo}9u7O4xN!z}qXrA-v{|PP=BAd|JT=JltEj15?juTn<<B?QtB=_we6aT$pt#2&-&4
zneBgibv(6GoFQQT#H(#RKcw<xXU-ylQMwV9z~InwJZWLFx7b2dQBpEn^nk!^Z6F&m
z_o*7*4tf`U^=ERn!Hvj+%=+{0mn{LPTMF7^Hsa>${n}uzMJ#LhCrP(S0l9PBf$&fl
z)`M<xzQgguWU9*QJ?;lH+)`DNp=bM%>@$xRUq7S`3H$b({G=w}*F-OLr+&Qhcx!5I
zF63m}C2YT)U~(~J&x08;qC%6@<odm0vh`xuEt9bi=-B1F-dS=*TQZ-dz$a&1(^7pf
zt}iD)4U0WKqU(GALgHf-7go+2+9MJBVm-xbw|NsWH=X<Y%UpDY{6FWrl5iXKY@6m#
z3`#do17oA}4`JMAUrv9d+>WlS=FeN9zpCcYzbzb4StrZ`DK>ZFGz0h+bJZ&p6)FuY
z-8An2@I8dgeg?H=2wFVeYrUvy*_pD5Xkg}sS`pC-)oZ2mIfPy8d^}MMH~C<_@c5%=
z-lIB49k;$T-jkCgrR!<v?~w;YeF&Nj<FQS*sj9rZ*tzxL0<Wv{^#Wari#qirmdOq>
zB}-=uEaBYOs-2(xnHK0mQWp7X923%820Zt{Wbz{c#=!yzycwOF<yh8Pa<ej~#D06l
z#%!w#goMk37~-0ZZ)4NC(DCh<z*oD8%yMt+IBSVpi5@C9IWOI&J>5&H4dPghJ-B&t
z_f}wNUiBUchKDgk_%g1mn9p>_JFO8U;ny%>6&)7YNg4RwRg;kF5!>Q(*eC@qIs2I@
zoFpI&GIXop+p)q@(Kjh%;<y)<2^;MJE22eO05}9bf<D|^D0H4H=gM0p0w%e!z&Rsz
zRb-kN<#vIemnl?dt=}4Q)%mP+Rcm4=veWrZ8PJ*a<7^Brq;lILN1J7GQMx#yrJrMG
zF3w(jtjl`+Fq1)hNEFArPc2uy+yxR-mVP=#Q0(*Uk|f1hHdmKK*)=-w`T*QFsohLH
zYIp0tiztoQ6Go-;Rmvt}nTUIGa&wl5QMH&ISn0TsQ`^;v`1guO;<&0P{Gm$Qh04@p
zTlsR-xEt1(L7PuZApUp(F|PB*rY+oY{*w=WqJv6E5EBWbl!W}QL}R$~9Pxf?l860N
zFmu7LIn>j$ymdUu?rA&BP5LI5TI99_2;F!P$Ua^PVHgn2Ekm8NNVE#{t$Gd0koB3Q
zbZVf?*y%ery#0B2^|1~U!QA^#Mgl)|%)~JxcIgsA$$tYgfx_YL>c#UfkPC_gU})fS
zi_kWrT}CseTG91kP<tqFf125j_KDSgPTk!_J=H09r}#vRt;zTC_UfkfJ6Vuj<=-j8
zuZZXEg>NqQ+8oE<=)_zIw_W;1@syme<$4XwfA$+Z=w`LyQMflor+PqK{7j1O$T?Bw
z$E`a5tx3@OcrOOAQ6&|)6c(7$D5B%dxT!(mNham#G<P06<B*wM^dOc2q0Wd1%;kGj
zL086{2p+gZC^GX~L3ctP1b4JEvLph3kDC<Lx+15TpuyI2BF5g78$@)FpEo~)Ff&~?
zk=l)X&hjPCq`@qyw3GDFzGnmY!i1*Tf3pWsX4F{f6Jh-c@Qa3Lt7A3c)-sckb1J3f
z%mLi)Gd<^2S>VU_EVZ7{lPNcn0h$_vD$!r670}g81W&{QdShzeeQ2GH1iR?u%g=;Y
z4&!jiX?TQ1$~{ZL;){8mBgCqasrz|(zCv&G!pzUU^g5gbJJR-7`6SA%+IcyLy6fSz
z(zfT5kOvsn#JttBKHRQPThIzyyGwe^PR#lZE4=^y&Xl0EUO=XdW^SZRkCn*4AY`fH
z(K~_0kj3U;nr?q~sF_IP%N_XB0Gm2FM)J<cLz#CMKjMWOr)C^lx>-V|S6IvR3Ov0o
zerDQi`{eMOR8~*PviJ_Yl06Y?s#ChT^b46e?>NW|Wuj;r?J(Eo?ru?S&i<jjXgn=u
z{CGGv?p!GXbOyO3*oBZN`-ni7ZBXj+idiZ=H$q8U`Un5;FpKIF*;|$b-q(RIg2?Gf
z98n$3@4E6awG2OLN#6&X*(#4R&2?^UX<=?{p0YU-t*?#%i|f?X(W&k@>%vZt$LE!7
zqU`FYuZ<pav$}V=-;FSFQy9SJ?v_%@`26@5=_=EDXW=I#@kx8)Wi+Gbr*@Wz={Guz
zTR6?#*p<ZUbr<BUm+bQ^;Bdr6eg!RF*_X~grz?=B3Vrsf{z1rEw$t`Oy5QsbYBQEO
z?f6wyU!^bAJKgj?Gop%X&%3|&iX*C)B5uM$OnJBu(@m{wy6t>pn#6U-c)QQh=XhKN
z$<q1{+f{mL)Aes}pPtt4&hp%lV(eA}Z?r|IUlGT}E1<yiz-AWc1A=BdA99`UyEu!+
zk$;4E&?(Ra`ukY)ehpnSr~)jErq!Ahy%Th0$69gaQFFjr%`#6?5m~+_BblNo?X}1h
zAk7hp!iDs7*##$7kQ=ME#J2DZ_I~u!Vxp#=6Ai{6E_S89#XEk*rK4j9)67m%#<S(z
zT`hQT(f1E|6!E&h1j<0T{5!|w;LLP{Cy+$u7tu4w(O&J3M?QcEWFHHwhG3eWV0@G#
z%fr_{aDK-9hrq;TxzP`O#S=!r3sx|~{Q@Lhd5`d?Xx-Mx7V8!E#7~|_xI(8r9xe!%
zkSQK#>TB0@v?!#cw<Ki^94}M>Ux+HUUuexjhvHeg?vgU?j!Y)?eVsU<XgnpV6J8v=
zEcilE?S$%(5+a~h;km$P+(bImGNZfdV(s);Nq;a+9ORQ7m*WzOa10wD^!DxV7A-sB
zUJfr`<5qU^vLUMwh!&f<`lfzZe^Pj<uZF9iS^s2?8HzCrC#-~Q@0L6;_%<J~NEg;I
zu$$aj8!vBp>e)lw8Dy72xjcxUe>&B|3tclLxXZ=e#A$a~5XK^6cydpMO0|wuqZ-e<
z%je!>D~UhCBmHGvWa<jE7si<t|5W#UD+NSgZtx|aXzkb(yBl}1YA^mAUp``Y4e|^>
z4@VCcmte~6c-$frnDjbDQ%zA~g_yXSf{e*n6K_zn#WYy$2tRUNDP0PXu9T(>?z=l2
z3<qIFQY?TRR|nHPoUPD&Vwaa0ZMmsHrzaN<{rcoy^8cheqv{Q@%=K%L^{%5;UHb<d
z!@z*^gUEEqyu?tUqeN+2ZZv-ScgR+APX^g6OLJfOmO7jq{wym|g4KiLAqq6HXcxlt
zlkDJAqFLNttg0|BBCIr+f3{jp3B@qc_!3vQy*oosnN2bzdrH@gCayf<4B~&xPJ1eS
z-@M;SK<{69&z445?KIVOuDl&(iNwHb^wa9C0iBf<oi$|0f`CMuKtP_K`?RJ{TG*|z
za01836(g+DTP>bD-RD}Iym=s~?E!3;;w}H<e9B}bdAo)QgPa~@1;DX(3TBC7gvF*w
z$7Me;EiN(waMXm;N0clby%YP+@z>sbu8z{uXukc7DV0`Fg{91a>ZL*IBP>1Ekv7mj
z?UWMxERzYFVHF>#s8?NW-ShEC<+<-xs3}{!AB|h1twvB_<qHMf?t}Pdm)j28ia$k&
zPw+xXbioLo1&M3caZ!&g$0C;W@3(&*o|e_9|BU57(cJ!|IIruG`L~*@^f7991ZiZr
z@SJYzLnPu=@>ilcHMY?z341~NpEa=9y$976MG7z8PpC)*d~eVYXv&LE6fe1R1f1*I
z4g9d<V(c9~m0lS7ON_bNqMDtDH{cE_f*w15Ypnfo33(fV#IqZ{X4E-dB_yid8uPFT
zFFIny0V0%U#ptl#%>47d`2kIPk5hM$IB(0}By)@~<c#C`i%t4xV?D;SBf&zBvBB3k
zDUfH<20Tg?II>!0&9bN04QSjl#Kcd_!S%4|3bB#rVf$knmmlzHL(DaEGOob0{^1M_
z33{Y(&T7(6U;!YdiVe1FkSgwlE5gzVnOgEjTPJd?{3HbP(e2!$`STTR+CkHk?|(D8
zsGdIpT5C*#ugx5D4!+g_LIF8nMuK2B_$c$j@W`Jvb4jDjA0y6!4c6TsL*c@J#C61+
zOmA_)T=ZI|DM^96-Z;kkd`@eudsAlmEpn*_A<`IS&@GR-xhX0X)QDz|(V~s?eOoTC
zmG_|f=0S4Y(IoY;F5quat5o?hKZdn}WBg0>GNUO?(}rcuMFJ^Xckn$ors1A)T)I9{
zMdEMtc72k#jePIS>j))y;JFoM@cF2BS|E-09H}}g0u`%5|B+}IdAPwO2x5xLa(x$#
zzn>!SK0J(N)yGAOV8y_XufLG193Ms4vXnE`FMN2EU#5Opp?&;MkuSv~r0%%*3rF45
z&}JE5UmPZp!==r|CJC}X0x}riwV<AbUu_(Hy`d?r3xRrfz>1P7=XrSAh@oi2_NFpR
zO51zR@DbS`cVdm{OPnVPJm3=txCHh>p8*f?5Z;CenU!3FfPnX9Q9eBb4Ap^zj7Q|Z
z0*-D(qhpIJaCpo!M7W(4Wk>7oQ+?%qXDM>CkYA%~NX?OGoAJTjmXsI+qPC`0XpwF}
zs_#6_u$`C`O?C*Uk+Sk)hM8j%3<z_6STV}`B(2$p5U>0N)C+KFeR#7hrcaJHYhM{e
zFH2U*KYO^Ku^(Qqs<{2(8FUS#ILnjX<y^zqfiMgvuCTxvLllL8k^n{wOXr4}to2J;
z(2#`Ay&P|rL-8;4E=)Gm+cx;nC}X-|8zoGD7{kN$yX3h@?O4v2qC8|A?dTttO!Y0C
zRjqMRt;5~m;rmt#?KoA0t1SfIaw>SDKI|<Qwm8&t%rr8${xs@c3CD%~KCEDBbK9>R
zI9&Em^-J;z;r<iD`7gR!D_K?eOl3XsyiG_-Q5a`tq_+X;V~v+LJEZAVEO}ZikXZ&+
z7<73+SMtbXrT0!+#-GC!1GMK3ef<Z{DwdciYvR5O@BL8XK|v(t;2R)}`8@2oBS$j!
zmGg_x`6Y<IAh4a>s4d4^knhK7CaDMC4&3<Kh>x-G%RlkA10HXF3MB@;7-B68dXF$b
zCFC?xRY)IQRTJ_Sc&nbMS`6$RsrSW<NaESbTe=qtQi(7wam8JMi8{i0pzn|Jx=;6B
zgNN_3TQ6pbYWYh9kyV0+qoS1MPv|Ic^)L9WW3A7i`TIrrSiX%<2XX@k*>4+(Mv<n5
z3%9-t|4Rl@LmFoDd!bt^bmtudJ@yfIyh;;k=Yo+2tPFXQ&^PZty<{xKhX2?90*DEq
znw205#g~DJ=G4f|%QiZ@<Xtg=L@6%uPXeWWjnCtebOn`yIa)u07bo^3GCJ`2kgD6B
z_UxPXk4EXjY3I(Tc@pOw@P5;=ZlR68!?*3d{#AN~i<nkltl3nD!0M$w!*=7hw_BY`
zchh9zztC@PW{u9|fTlgNhD-<(N{*Ts<Vj`1#b0@RO^jO(h?!nwoXd!cRfeiP*?IHl
zx>fa&Ge11O;F1;A7y-Pze(HhV2q$IP|57Z|#~NRc7VF1B56+cWf(fdxSMREqI*(yN
zvA>DDjqI<PRnX{hUgNHD)QCc$=|~~&r?1%<@ffCVi01Fueil0+0dHR`;+4McEqj;%
z5(=l^m|oa;61*Ju#5l7pOm>-~6fWmAH0{TZbB-z#+M_|IMX$l)ZyI(b(hQ!Dw=?Zi
z%h0KcLtmk1uZSsjU}%$A(~G%6tPgUb&@!NshGqi}tK&$4-wt9p6Sd`h9D>W5Odxvw
zxFbLLEKov)hLqNYDPl6q5Ai??>K;mE^u<h4!v4V*2^g*{6}R5E3P6)y(5VeYg2D4}
zxG2_b2*fN<Nkd1l;MYtetSF4MWux2yN#t0~lSTT%y+kOVi6_#BkO<%*ipLjMhhD>g
znQe4n{e06lO-cO|G4D_P9=eU65_zz(_CyBPFW)K$^vb(4`{(drG&w@vXj)u)!i4x%
zD@WF_5BDKKlz6{t7$?=E2ryGvU(D|taJHjxNjrjRlgf`qlmm_6doYtdtwG>N;!HwU
zgL7yqQ3cVZ8-q+gTbk4KF_NA|V0uEK^i4JN%>VLsifI91D~%x78?7%j?@nlxyML*T
z+-Ziypp@+->_lZ`z3pNIPgO&LUl9R05)&mcpzg41!dY2H@t@+7J~H2bwV#&o_St_7
zcu3p7HsA!ixXk)Q=~eeZ4PQ4hRTK4z8oQ8!E4wfR!d}b3k`to_=ycgXlkdldaclrK
zB=Bd$&6eH!cHvL4iNufBh<AwqPI)$DGV!_Olj`kk)nNOy(Y*e0eA?75$98#{w1@g@
zYDl|`l5kr8T?HAiDAnfZ=cFP>E+49%0Gi*t5W`XoYkA-Ys_%J;-#wK}x}tQDc3kJv
z<gnYn0NuD`mbnrjM{2e+B4g&7A5#Zs!>%Ol5*P^2YH*~2toint2%YLL1BCevW(?WJ
zsWrMsp1yIbIO#6K)#iMCD%tVtmNz^6bSe*fPlVF@A_`6)MC&a!6oRDGYmCya&2e@&
zQg+HpnTkVp`?3~}4~Uj@cD>Qbb^PlO!+ExxawJ@y{B>^DErnkMKMj3n`avp5@lUz=
z7sOlH&C*o&ua7=?E<nZIr$xl=x`^v8cAIjAtiEI!=_z~<qFg1KiYuP`=(*t9m;P|Z
zIrC_=m>y1i_SSjppIUb{_G4y^23+Y<U>;DhHFmTc7c<ddV2$e7wN4zK#oy3Yg!}?R
zv$im)Y^jwT3?Zb29GdDuJaF0*6mo^@oTmh=TvLs6y-hO(OxrqMob0Sk+a*a-^PAKw
zISuCg+&mb)U1|Bb_VZef+rUELl1EpP6SArN>hfs#4erA4`1RTCwUGy8k~n8UTkv5o
z_h1y+VK`UAWNX-ku@oG_x2`j^g-o5d|9iE>Wf?dU?k|s~j#cWsbtdih4+G#T33|`&
z?8Rc1aCJ;YfHt7@_SnJ(=Qz-slx}&G^#07}=}#2iq4wwcx5s=e%6Wy_ZmydiHM*O-
zjg@Lf2izPDD@9MXnAx`6T%8tJ{T|2`e;-XmOY`=gu<$Sxiv(8!!T40OnYp=c9y7JC
zN2J<Jr}1Ul4c;9uH(^+i&!t@r7~kUM^I$tZn^ONZA3uY>k0>UB1vb}IS^K~7;N@w?
z#7%;%S7pNA@oSiFMaKvNg(LZpD^QN(^}{o3Fp-Lk6J^s__kAm_4^6w!t>5E%WDWg`
zF!lElH`gY)QXT>bKMPY!pXg(puYUI8>G3azmcu@N6Wn%I;*&5;Uu^vT#}^+j+-j!3
zHu2U-Q1uScWL|U>nd0;x+?Dv%_v8Yl>$8$aMfV~9hpx8_ild9Vg<)`af<u4+f#B{g
z2?-M1f?IGHTn7m@c(C9SB)A86cXt>dSa1k#U-P{8y><V5RsCaXs^(0e)4Naaz4ltG
zS>n}%y{~LbyswYv#O@DGkLu>X%pFv;+b7qV4dNe!(M4C<w?Dcq<Jrv=#a#jjf$qV4
zMZ?vKi?R&>SRuJ$QwU6u@mR(B+9n(NH5w2vUk}j5ELdmy-C27-1HhQ=QT?+0Tt)lS
zS4!6zRZTay`q?@gt%Er|2UaE)kIqmBxb_M~_|T}}S{R@8Dm9W+CZ^qYphwm7wvtMI
zg!6@g*A2O#)nI{#|3!zxn5WY1FY)32S;fEdEM<XGfEtXkaQiB0@eTE;7Y%%dvRvR)
zlOpGg`3q!8+*BidRNb)Zwrn>?YME}#`S<F2-Cs}C`>OVL)LglNaHJ9wWj_Eh>yG~5
zt?7f3n)+TXT53iTvmT0kbYJt_)d5h;4HJmnTFc^pP%r?h=&t)-XS7moVU$t(-Q>_<
zy~^}|^29Lc3#mscKU}El{t5)d<#F@LQriAX)kmRnsBIE((`A253MqFKUTxgXw8;@(
z`C`#p_jR_K80bH&IuEnX3GSwwR_RNjcy9KD*CR9nkc{?LUsMgB-ICdEx3*G=&+Yk@
zbH4awn*`g@u6r^bSnN}FnC#cAE6+lSID#CkHkNG8xHiRdbZMrJ@<p=O^~u4Xz;|Cj
zjBsuneKt^;HS`|=)&U6XYX}sEDIZuuhANQTgL;jqJ5UOQ`z}WJta-Nbpef0d+50rB
z>1I4{Tbv%~C^{<9#GY<Urz;G_gTm4vE*3B&Qh^AheY~g|#)istx)_nEY`qmn@}^%$
z;B*m$!l`tiy@2HT+i^3T@y82HV~vZ(yY6!l1>v^-;;|7TwGUu(gjK#5yW{*UTIIEj
zAWpj6xT~8nW%0)uHWXE9mx0$5ae>0Y_N$l_ir%Dn8W$@r<D>!|$k;{u3srg63za#m
zR7vh<y|0!#ocPoVK2}{&xQuYRk%mE+4y><2*H?33pJEd%?9^QHC58xLIDkGdUl(48
z0L)CB-}G8+zMcQ?J&9xd;Q0HwQ8d0o`z<0*00OiC9(@9^LRU$1Wdx;<F!~>YG!x9m
z<}5etj;6up*zPca!GunEm?sCWEsC`rICJ9BM%gwekHi*S4Qfbc));bY37mGy7}LrS
zay>KvQ1@0+5sykq_v#}LWT7v67WIsi6OycI1yE8^uUbK%J@c{E>vH7z>AGD?mTt+K
z_?@;9hud_fzM2)IX5e!Fqb;3@V(0s#EysHIy@^hB0Iv4|rVKz-nVfrL_0u(?{Hx#u
zQAq#WfaQrYyW|5~#p!YH$?~xP+BGU0ay8xyO1v*Y#yD5*uqB%cUJ?`B`oUrkFNuY|
zKPM9TOKp&$!MJg%Pb^2tpf?8a-@Zh@HKyh1I;&Q^1}Dd)MZe8UV3jbGkay;S(FIip
zB+KFKx^@iEQMNhGVG1v_W`xa3F$l#&o9dPF;VRC9|JH8Z$hk>h?@#v!5?C>AXm`G#
zon2dgr7s|y6yfCBm{T~=^l5S3isD~7I2@DX8UOMG?5|}&Ff16fuT<JC^I>5B)M$1)
zq9w3c@P!VTOY-wE&R6;g_!FlKHMew#+hSN)>LoBL*Ef%)W}32{4Nx}^FYe1(QVFDM
z0=Lm&xKx;sZkQP{VMx)>RXIz9ewVJ)+UD-XH_FXh(ZYwE8Ha6!PMdhmYl2_!@pQb?
zE`TQxFEPc(d+m&WV{IME6=Rku&Ym^HIgIj1$2hjiuyY_H&9dl3rUt$Ce<s@+2)iHK
zt^&9%WKiF5X9CjXdaz`wE+0){&D-UnflZBSK9jlr3Y*shnBt)^>hF!p@lQqn{Op<m
zX!~e`Jnk=s2#RhJu%-QBK1IiV6ru9Vr^Y<GP=e66dmT>xrke0<#gR!<U4^%1fS4VE
z%q)CM8?3_EQ{xFn$d3;vtpEI$ALA)NkSm04A?<5BJ*)$vU@hzfMr}NVqU95Wv)Ovv
z(o-r}LUYt`EY-I-#m2toC`V_=Tvv%2Tb{ID*Y}${Q<wr9N?Da@Kk)}el&EOLX_t7c
z47KP&8Lh^Q=v!=&QwfaKElmJvey0?eBkMvdXpK5KAhpH2uRfi#ouVYdINg;OQ`#H(
zDilnsgOcfB{Q^EW{ry$)n`{bS#9+`&m@t*poSkHv{Fkvvd-0Fk=)IbdJr~IufsJr}
zs?MaLLKMV?Hy#-TQ#Yc~W`pT6+b@<eH%zA#*xUG2O8>hcZ68)+-HC4qd19(4#%3&d
zICV^U+<nCKm5O_v-AG(9%ssy4%Yvem{_JM?+lv=4suPAs8=T6dc0B02QB1*(Hs-qP
zV)iq1wl>cDYBKuCY&<8Ye(vS3L%rHjS|I3x+;10CPtx)H$3?Ff2(tDbwv2(kqC>0s
z<?(8>#8;tT(5rJiB}iYG)8pM`8PGgNwr8uRINFCF=~%o;=a_BrbU8Tf#%Vb9QER4H
z`^e$-UOr2i81~LfE=I8U>&@u~lIs}*B2GoaygZb|czon!rZ~YjuPgO|M~P`G0nww|
zWkM2Vv40dzA4tY34D6XuQf@zkk2T7Gy2iSo1GjdUxjtMRlk1sC0d}uK?YV{=*>UEy
z(@wD;+Ly|x@9Z{dh$2va_0KW<Vd3~n$0dD0W!vv^R*RTAj!rR^uJdG-5yr{EKI5t2
zwj&%$?E`fD+$JB))L(6}l>Okf(M^GxIrRUYkBxE9ReN6;v;KM~$LV)x76~z))nq1>
z1WK&I@^CVHLl$y?({IfZ&9Xytxe8A@Y57Ec!T-0fmx;mlyTb(N>`l4}{ooN(4N!U)
zb(B9nV(+o*q(-7#s*q1DuK#{6j(An#JylS(WgBSO5uh|bb8y|EuE`uM_c|i_M8C=7
zNPjzemgEhYq@gmMi5)8>@3TT=|C6o`W`}*ewp!g+h=01eoHrzw@sr{sj7zU{$VN+*
zBwy7GSQl0bYeo%D$4xmLaTi$|@Oz{uADdS>C;ILa^2V|a$ZWP&Y}<9h9O%(`1Wb1J
z@?!YHbjij%LR~BbFftuzHW7bGL|*+1P!_xGVM7?p5_S5U8_C9%UNt_b&C}LIAkBlx
z*nVm!Uz+AxEe1v5O;<-OocedPJYV^4u0LGEhZde{SDk}9%lu40ASkP!b(jK)BqI4e
zL>*u;am`v9?fyw~pM<^%P$H%57QwX=t?bg72UFpI`~zhGI;WEfG>s1-<OpR!EXkJ{
z#@7zTk$0cH!9H;8lYbcaTn^VSXOCznU57ZyJ8#jJr?_rp6(|}+iDSbb#`VMDbfKWp
zWtZ$@AG12pD_e+Bm-;?|>!&3UnRW8oJzbemx<wfWmVk)bG?_t*F4jmXDeL`go@`+8
z4V&`o1f^Yj4W+3U#uDDbw~o|#j~y#X9o+#6B*O5KbQg_uy`dB1NyDF;K0W*Hwn3Bh
zV?U@s_8^ze#DGjpM%)$@0k=fsm^B(z6nvdebTCc&_-(-}9OApcXNXg_`03P8&gnE<
zzdM7_^2+d4vF!<AQB>}2QE@^iWzMh8U>a%5h!YM=^=NG3^{i-}rJY$VL*5j9vps!I
zDS2tF8p;gmS2eBCI_Jo%16N$s1R;xa^%r5G)#M*LxKbX>%3IDq2%}pu4n0ewxZM5Q
zM;d<?xj(EqG8gg#%OOOmeiAD58DnAySZuyIRFy*_b^25Deo;)0*qUV8N}L%zq5JUC
z(VxWn0SJbXVfJ=gd8O27^Rp^75)^CU@8GYVBOB0hnqHFCSZtf$B1>!l)HtXPL!-Ew
z6#+-b#NAd%gjE>G+j>)<{*p$9Z|E{?;hIDGLRR7IbF~wi{Uegd>D`JkOtuo+2Yw8x
z`QCp{osKBrs<?7o<*kFJErvToeCq9&a(%)3IAxz%dds4VQL$(zpV!nCwZ`4ZmTEc5
zVnPvHIJZKdqBAJP{aQj$vqdHQQe98X!v6h(E9i6)J332DR~<hWP>Lsh&}I-5g~N$!
zxIV*as$>-#W16^0p`*fJ96zo43={-yt->=mhGMi4TbzR3tE7yi+Qi9}kC&-JNH;-D
z37}$4*c%=rkjHy*%wj<z5(lCiy^ev?+W=EcMv&NtU73!(H8mp<g_*suYK&M_kqUOu
zA$2o#I@%b(QIaWp!!@Nt52gwYnun(}S^l@``u+5pd$0LGep@R${-Wcx5ef@*d!-wX
zYvUUe%t_%}9Dlz+JuY?{B*DsEx;qE#wwQ$@`Y%?e$|U5m<5o%-q)BpV_K!0&2j{Go
zGyB)?SGUBB(WFrpP|fPc5dTcK?GK%e>l%$own#h-pACb-Dgg!shY{a@eu2OGDT=(u
zM2S%Vx8$0QW0cIto(^V^5AYb*qx++S+2Tf{8uBQYMt*kt#)CivYfrsT*ZouePXsS)
zqhzs+KGTj02~92{*l3T$1pETFSt(aAAXXBefMBw*FF?%r`Kc;QIZTBGtaC19`bZd-
z`lN{ZEAb-{22BEEM8D-G_E;i;@^Hko%Kqj{KtR-{`#Psi`NDVVEn-mjbDwpNrXWlz
zJL8+CMn~}*tLDc{Ba!BDs_SkAP(QNMiW)O((tPMfv>$ct8QuiN#2Fb<*W$*)Bw(Tf
z9hekMLoneKDn^jF*2Zs6DIgx&{5^IVp!8Z?nzUQ8A3=^n7+#4Zx8{N9)MUjT<NL6c
zgVrfI6IvfRuF}}1C4-JNj~kZ<atrRlsY1IRWZ%SK#{sP$!I{sc@{B(mPiWMnkL7n)
z2vd;kbdx2znYU&&dJ?FL1(z597Rbw%Pp&cmiq|6(n#jYPUkeho-o{U-Z?7$dHvzc-
zjk{PT?e{bJ3~riR`^~;6a&i&w(M-X8%y^u{{t7#f+4N57l01!TUVI;P-5>?Pvd6gL
z>U}oW0%98i1^EKG%>5^0dS|@TWzF2K@_?~po=R~?DF5_o$FLFU9>yA+Yp18tyVEYn
zyCCOv_=cePO^HpFBUDbx>W?+v6R_UwScLsv$q9b+ZHa2OvA`sR{gA)RY5*qvQ3J-v
z(?Ur;E_`Ryo#8(8h1zXQaCgDM)+p`f3I|&S$J}RtZM_`#p)bY}p-Vb(W44~ekXrLi
zI6F>8P7OfG1LKlfuOI|-eF;0ktn2Q#E6gsR4H#QSFnXJ9I>_cd)^s^6kYk%~nD|pw
z6eo`r>Sh`5DruS=6uWLy@#L;yZ-&IRkC9&wC>~%LAPiL=I`!9pXTuk2;{92DppYzl
zj9_iEhEy!;R-3&aslf3*tm-eV853eTXD7Ylvzq9T>fy2#YLovBl=oMMa&5GjF>_2F
zM|u@sS`SHMePTG)<(w`?`SpA#7c#w%t(|uAO%YJAYb|BYc^y48(;x3R##(&-#K|Ah
zi?Fp_ZU9wx?I{T^M$n1c*<B@qBN^$Wsrwnn1S#T@W9dJ%71#w=b6o{I^x+YuAgH5=
zUemW~OJ!Ip7J6~4C@os_$M3sm3CgyYhM76x2<o574Q9tD`OEzZ(8gG3Jz(;p;!J2n
z82QSjnVe@vS*3P9RMwmK6}93<i*!;n1+}PTAsZ0f083${3l6|?fI8o52_SA*wvR6?
zcFafQJYKOhr`vUA0d;wrYaglQLIp-Q_xzz&uoHPyOalprQ9y4Pi-o>!N|e>c>#i$d
ztJ?)kvZmoO-MX)fuBX0r(AaYg(|o-K`>QQ+Vm5uN-PVKh=0kaZ4jfk53Y-%3PvuB8
zaQ*EDI3&HKIM@h&WTA>2pYPBd)0gQg_2FC4NJ4BeAFA()!d4GZ?lp{m%yQ!%y?5mi
zqI&PKbzzo|c_sei>84yF-zC@giOI<D_ha#gIt559I!~vj8)PSBB{FIfxUfgF`aUj*
zhNH5=0Ms@Dc;+7J_ygbfq4D}?vl*WCHb{3&)=;Oyqm22v1RKyiIr`wQx!8zDcP|n@
z42B*$$7A(#Yn18ERzf*V`v~c`4vqw{(W7%kuM%QzjP8M}EZ95-tlGRE29hC0pAS$l
z^LQ|%VNoMOcl#F2t0*05?M62}Yvm=SSM;<)bCH8Iv#A$eji5l)=MZ!{6t_~EO*_er
zNMY}6lwyk!?m$Z47cqZ7zB#4aLu;w0P?zXW<c8@KzH22Cz#Gl0q4wecdo>u8C4e$n
zRE93d)FFgnr4rnS1ul4Vo4&pzl)PO!(9XUFu8GN9S#MvG^>>|UyZ|w-fw4f~B*&FP
ztniS_e27Hp-%E}A_=DZj8^#EDatQYE1%IW!=BBjY{}hT%`4Qkyj=nPVF+S}u3_7+I
z7CzWQq;4yj<Fwan@u+&dir`7Fyg^ucPm-#S<RRJiss;56JcZ59w@FU(Yr(weLO40e
zeVH$4bAgUmgl>-d$0z~O?EQzH=I%K#R-;%gA~T6baDIcigs@F^lyoOr!0x-!4S%+&
zPpSxJ>2M@byE4DPHCR?EbRA3pGSKt0qr5~{fAPCE`tJe>bw;kGgYMkc9~h=ccA-%?
z|9pXB4$5sR1bln*m0fHpR1?6;T!37A`2f5Scr+<b&71yKso!O}K4V^fjA_?qa1WOX
z0JyGLx>I5*s7!@_v;N>D^SoYlCl^}_t7*4Rzl~}TBoRZ2o|KdO5M*dctZh%$Dptz1
z)LR7egYk4=yrOKK3{aI8dKFG$87_`U98}nu6@aJgwFm~$3dSzjEz>{Pp^U+GOA5m3
z&k(=B2{6n!H=klM$GAHQb7i-V=S8sxnbGwrsvzgBJlInqPmOvI^rOCbBmoGi6FSFv
zw=j6Q3+q?-)m^1R#m<sL`(vpzwqTDR)<RW3m5s8l)8Fuaca`SDNjhOyI*trSTHp=e
zI^wm<ubsCwFcl$LLLcW+dc)n2W4jUng1+Kvm=vf<yS@Dd=yajOD{>fq+w%|<Dyt|;
z;kmW`s<bdU-51i|{B0<K4Un*zp7ENA@F@*Q5>9QVD72xoE==uR8XGvqw=a4YU_2%D
z<5QM|!`L19t&np`r&o|I;IKn~Qy*f*^9sc{faDks81%Fyo(i-r2<z24>f@Vkn6O)5
z{$ihG1yGLehzMIvKelwP#wM=A^S2d-H{@E4RaEVt=BCyC=iD0mgbtM=STr!xt3)9D
zDtf~y1hL*TLehoZ8|=p-ku#R-yNvR7reT!Th$(r6&*Fx^<2hsCp#CCjfyWQsMx=K5
z?d*9N7u`9zzBTU0<gXnp<l(ff4J5{^>&XGk9h?C2gKs(Gh~$A!0cxp)XZo%>jm<{b
zCOnia?VO5XP83R;*&(T)(TCE-*<2#2hqL4%G-PI0-G*Y$s7vqgg9HLp55I|w8i1A>
z(Ap4!f84T`$gS8x_Y2W1t-y&?H!N}s{d?>-CAh_zacL%|qglF*MD5LW$q?7@=6ozV
z)}P_Z07by!oT<ZpUDAlnNG9yn8cxTN5yb$~mqlr&l<4(-nAa(p!pyhVbd-i}Wy8?-
z*T=SQ#?zA-$DM;-<mS%MKZV)c3vi$9TqXkMitU5{j(-HSsunw*ENo>K$@D0=Y?prn
z1fuZ_HeN>-BJcr-aqFXTp-}UNsiH#8%CZ9E`A?DvQ@vk-5x3SCr~3j_<W()m!D7X?
zr22O^h}#tCdkV+!R4BHJB66X>@r(qkGtmFQx??Vq!`7gFS_nAw@q&FKF*KA8VVNkt
zkojl#BHG;1uC)ZZf(fO!xN?v~j^#QX^qU+Bd3CPI;Rq5U%<FQSvB0^5ZuA-PW~0I8
z5QR7ex7$qYH53Z+!AP<^e~Qy}fX>q;Up5i$gTwK`gYMKz3XZN21(FA!m%S}nI){K(
zfy4+}dpGFD3|qm?Pow%Yk$vCM*kQwXO*T_S>Vr50Q76k7&@8qM_zS={66o}~9oz#w
z)YBjg{sQIYRt-2t*lu6~Y!uX-x37S7<#)R2iEs3Zy7^KJ$yd@Gcko7lNbc$j+(CzB
zfRZB;Eb;8yJrM`eOmq&u5Iqb0@>`bD4vx|*)|R*1HV;a^D0~0z%^#Fn(e@OQu1cnG
zFHfHxSq9Lml+*s}ds>%t@YI${8y&Q`CaQZ0AUR?aLl8jYA;ZJzJl#c9>`DC)JR{K?
ze0{`)5WtNL<bezeNr~QThMB=Mz`K0kTS-nmx=8a9xjTq9@V}oPiCpjIry|eih;G4W
zycdt+*T{o+$p5_GGb;b+4f-FOOFBYWh#LVD7=>zVaMI0zsKfI&X$xu#N9L9{JmqX1
zOwUoU2yLbI7gH%U3C3PFHcEeh)H-J=qYYSBE9lr5Jy*J8khFRNy(MIHhQpxr4Oy2{
z{in|}G;;g58Rq%R4vcexW^2KhZkj-+4Va^XJCc$VLc2nZT=gd~_GeFssaeu$6V%Nq
zwtX2JHv-cdytlwhQN|)@wu4K)c$^t9Mo8Ho67XthsRL*0j069M(&&*rfKCG2v0Eq)
z*G<b1SA79Xdv_wBT5`FrNf8pKEn+_Z^+WQF?+x0iKWkz=IrB4>NMf$sN|0Ch|7??Z
z=q=wgxyTZ+`<kC20ny7(Z@;R@&)k!0BeEg;3C0xt>L7Xj*RYcK8MN%f<47U@g|Gto
zvcqxWResmsyr}~~)=30=-yyH~hWyeR!nJ_qVdtkK4|5T;f-~D7GP*vh+Hk*b+I5?`
zYOLV>R#8gpc7`16P=B`UmD9!kSgwBgpI!uCHJgy@5$i8Q{NlFA!^RUX072-p2`6pt
zjhxb6G5$u)@A8zXt9R{OH#|=^eKhj+gg42L!E2`u*b*?Q>e%a`)UQlx3b81<<KsYh
zhX8$`z;*^yf|RKqBhd0P(2NSP4#a8U)2N!=eJ(tdp|JZTHm~<s8-`fxIfk4iCP-*T
zFmP2Yrtu%`N<kZ5&X(a#`f@9jRb}Q~ht4MW+e58Vk?p(e3y;^Q&3@`SJtV337`4pR
zg{R1Fv(67QHUS2(bx4VXDDD`aX{esT_Qu~ZcVR~S70@269G)FssD~ntI7UNi6AQ+#
zB9;k#!*gUt;eJm@E$m=c7})mKUcTt>urZy(4i$FYbhq6DtL=gDEwk>+a|{*%I#l}8
zE1cM8uO%1FcHW4Wp-SBxMI3+u$8NE27L(0+9c+c$Z|gL&M;?%%*@mnYAVl+HveAs+
zSx8Ts=a@_)G^M|PON6v%laPy*@~>8>j0)C37fS~}!4aij1G~4qG%JO7dbuVNRvL)v
zx(dRcSFY$=s;u!p7v^)o)>E!*Gd{CG-$#n>ddy0eMfSXl8zLA<(}iS{V2o9~9m>YZ
z%ER{RnJ2Rc3da;5u_Fk?Sk_}IPjkk9W~Sh@q+68Le~Yw$Dh9U;hwY16PhJJtTG%t2
zrUJDG-RZ}Ls0TbhM@b!@S<+E1%+jom<I!D1mz)P=iBFe2q#q;Z6N`NQBSLYZbAWEt
zxRB`MlHISts5jjsdnwd&7>PbOI{|Gp1{v}^qkPd(x@c|i6m*wpdnmtxWRB3yeE23u
z-j2zO-UUnsV85ZXc3b5~23Nicf>|B9-Y)BHs%fgjM`wK6xmU04S!nn}^VZ*=4JxYm
zLG8RRWb!Dd!$!Uu?XOV`Wd1T6{eSidPNX$?zV_zW#m~{5x~L1H>c}3}Eh-sk76JP(
za4go;EQt;#v}LqQC_`ROYNe7<b%58N@cSkgiJb_%<KdrmBViACFk<I!a-5>j7G*$b
zB}lJIxBQf0eGWOBVGY$RUPHpvlkP4TCn_^a2>UdLN(11|0u(eoAS^x3wqtU$<WRo7
z`nM%oUi)sSozSs*#AAXWzobey4v)9q1;GO$9djs16rOS6w|Q1xcHRo}JFKA`x8cq@
zm{>;|l3PqQ%PD=OaQ4|l1xVge)`F!&;USvPx^v)^tmCXA)dJX;F+32u)6^?$Indii
zOS&}5UnqDqq)0bge=Hj;G-mk8D({;(Plpz^!30KH6Y>FZ#o*f3$)e>>LW#oAxF}PJ
zqJcwwTL+944k3LkfD48-WCXr)ca&Ap+e`U}0#4zFD0gmW^8*d^4D&v92<m|v4I*8y
zIB4w;;I8xSTTN&{yY;q+P#B#mTVCe>VZpuapgs#R^3T$p6>D$nKh;^v6S|gYyaq*|
zc~KrBObrUu?;8OY1Suo3FegV6*ln2EM|YciM9$+92>8I{W7_c`iL9yC9)T+0_z|Jj
zQ3AM{flhG%DICU!>e=qjX436wjp2hhM+(@p3<J6U@EXVv!Ebzp&rjTwK*rMWf3)Z1
zNJ<il40kg>Y*caR?4Te|W9Tm|_=8TB3B>$J&R-sc9%S!O+9BQ&EnDecG~+llI;L93
z4THq-47XYG!eYpi6sNEqs$eYTKszH05`=O@zyhGA)v?3>#%F?6g}S+FdjFY+6m1Vx
z9H9WTmfvWXrvekSfA#D9zvtl~69;8#>+w`cgpe3qLcpu7Ox@fS<A69`x$CbGaODT9
z9?}3jbfJ3ilRe_7WUibgn64<v3jX$XZ(<rU1a5{$ImIZ7?SY)$U%Z0FVn^HMC5YpQ
zAVni>OFHqdjXeC>lAPMue3tg7KKNJi3sMv&ifNBSwBdjjYWhF+yw*7#PFB4<F?v%7
zFXFm@DYp4pq-{=6Z=_hit0r<WpBskrNCr1@%uzG$45B?L9lqroorU26f2Vl!e4%?Z
ztB%hq>mlEGWJxhPHEzx8I(I(>7?h>Kk25;JU7>&jBGM>B5H>mQ%(b4|TWbOmetlWL
z!Fbo)Gli<q5XcnuIpTXTx`{z=mp!5C8EqJ=Nu(DmqHGKL7uk3Jloe0|X=tbLr&bM~
zcLh1R4~1G2n#YHCjmOe2N-jieT-Gk0OjIW6nS`t9rTwCB7E3$%@wmiB4g)^O6^Rhh
zfoN$!I<y=Jjm_r8i0i3>3+Zv?8WXo|2_Zi8IL`X;RYyKc?MNnIwoJ`eiPu331!WVV
z6F*fXnaJGe&?z%}1N~+FqK7|f+Jt>`u|lM?isKqPUIQ5<7I%PK+DSHdJ>h$JvLwM>
z`#u&Z2hj;P0&<L98=w}qSyOexmS4v6+|=1=GL`UjDsz;00Pi?F05wcP8*zzlK1+p%
z7QUs5J=jBy8C|w(@4Iu$lCu@p3{r&LvZ}4ba3tXojSvybYd7CEL?;u>j1?<#hc2jd
zu#N537baaL``<_CeALZo^Ag}gT01a(jAZL0p+0ecdC*Oz1S2GC&^?DG$1DneHL|`P
zxwnfU8-FgDpp-z#nY-ttM1jORhCw!$Ch_%u%k!FpAeD47GJjI=mcNGy!J#OQW34hP
zM0<y6&z6a*&nW|{;)`w<>i6wVqAtgSBqN#GSks5>oHHT~s)-tviQ)9ovKBfiD@O7U
zy*f}=p&uoE61SOrS7KZmOtY|NR>?Rq@Wg4gXIErO_FWj+_00z&pQ4@hgH=2$TxuhJ
ziH)Zt`D`?1DMT5XLV`VbIB+1Xb<0`xh!bu<lEv)boPd~YyNMDNbEdgIv{w`;{Af!^
zzVNH4iGiDT>7_2dDN)(nB3vUaXCr-~-%qmnO(#(Oz9mbFN|#33{o#PL;XV@!WX9vz
zvP-;@lFgQ><)0{O&RyXU!?pjmc?l!hp1#oYc|&1-f+>#D_)vi9<46;aiJt!Pt&9Y4
zLg$E6s(*xvR1qWF;LXo=I>_xc+h|f7xO<M-+~%hsk=vj6kuK}LefdTbu8TBjypQ@*
zGTh8Pm-8fakw$Kigj0zxFOM58H!rC(dT>*Urdt;EVTXl2T(*|mQE@XNP}=)O;*Rut
z;z<JY!1&2>>37H>7alG=9%@EM%>WnAKie-Z-}~1bvKLh}b1}fSC*P7Jpi3N|<wDtr
z-O!mgZutDF976mcR3U$*!fjnA1kTa>{bU$~`UcHf7}_NtN*}s6HzkXwOrL+_wz)fc
z42y)VRtYYCb~-<l&F0pj#h_`EU$d!>HPn%PnXDNoG0v2`$Npz23Ej5_sdth5*N&8J
zLj(8tGP`#4GZYaaPSDTdRzf{zQF+<qG(z78&C(_ori^(!6jN@yikNA2yt2P<KP|oU
zXA?sVY-#F^v!^7iI2)1V<F**Qc^nM&#7-_EvM4i%b#PAcO#FlhEU^?r2YY$AiY6ny
zx-99|Yq`U&N04oJi=hPA0$GYE+lq{lUmv{{;mo41>OH&EVUyC%zyOYwKO+-iNm!4K
z^dRnTEv4$SS5ZN`TTg$;=d!2r@ct^}1;53uR=yYlK>3doNZAD!Pwx2?XmSaolp{!#
z1!gLY_Zxwc8*4LJS)k7cEU|xc87RfdD1N?z`G`FlY9ER&F<H&6avFr&ujTBFz-YFD
zpb=CZDH=*(OfNuaiWnq~8{bKm7uPa~3^f|eh{S==h{+tqSs)RPwqMfxh^u`6BiG!g
zvXvc5ycQv17#gRn^ItU4gP>wn2e<!Xoije_W28;L7BbUNKF0o9!u44cr^*kz)B#1d
zdrFmwl3poT2d(P<0;`By28B;}DN&$!Tw0{+N#WTn%yTm?eeB6=ugi!zCI+BpBIu_D
z3XHY)*27P2|LcD6%fzTEq*q4en2Jrpw#vXgXDUVw3qTG27_ge@D>ddS(rxXLgWy5l
z!*-?ujdbEZ!B9z5;X+Qm`k3#HN`}L^G=N6ulUq2QMgTQv8I}^6<WyK^?GdR`@<V6V
z2dxs?_B!uwO4D>)ZcOP@BP!%|_oGd0>`1#6iDEoFk^8lvKC*)t(Sp5d#8TDt>_M~M
zEiNOEa^%K%3k}A_+n?Bt-y%hZr2Ai=8<t6gaI?C9_`&)Jn^W*9Xp}+DBYP!5f$^K#
zgrC=vc`9xNW1F6CIs>==yK|rlN8cV*bGQ~#f&jKPdfwRJgu-62(Lo^wk*)5}ND7EP
z2lY=U)Gn(q9PqMF{`;oY$(F)Xx{;F+DUI9Rw-_=sQrF)lX6cX4zUSLDl4X{m{$V!|
zJ-m2dF~xM)!pK!^6S`gmCVMMK#V84;Sd<;J(oo_PQmV;mnoyPuU<&1-82(X&ZAP|q
zg~fx1z66oc)iJ?SkaZz}w2%++VVl+BO;aZ$klGCi7WxRr3GV71eB{P23T`;csmJYC
zp4uSAxC|o65kehQ!q2AjB_o6qybtx+87xJs>qYrCPV?o&k%9C^Bo$CPFHYo0+a0eo
zYE~La3SY0d907`M{&h*7ELkL9*xqE`)e9pF0BK7#%XF_g5OB1g9@;{8p01W1t^h{c
z($r7IGC(5h21p5Y0CG-XIXf8|4z<`J$529OG6VfV<V<9?>g(;#CV04zFg#S(9h)9z
zO-Wi@Q_z0ViHQ|Q5BWIeo=8{?wv9Bj+G%B60BC*Cm~nCC5s*STc^+&y(6(ZCL}vz3
z=PoCz6lR|RdY$gkQr!{2VyJ%+>;i0qBcRjW07#M9K5KZe?v^#IRjcO7Edqo+D?mnB
z6H2uyztH?5Odk_I5fKLO#To#ND$OF5*SoHrfk<oIJys6wevh_?+G6b?W)Eqtlug|E
z`XAnbXmM3R5i)Xwh3JYjAWR#_7Q#FzlrIx$(&^8iESA@7@PwR<AaiqfJt%<)|3E@M
z!L3}ZwP~s?4fEDQ<J?4!g2o7^#S2nz;Qx3@n%+kx>b1}rH0eu~D(2f%MI}dof=yus
z<?-SciU7}%RM}4DhYJsiUay!Up%Xf`$MpA|Zx2)U3rnm%cxNDu20aC2hAh}rV#foI
z^q%kl+Bhdr#n(JJ7}fWY#ZU@5S@~}@cv-<8B*9E5k}foYu>rV&li{VLW?o)z!3zX~
z$MU9ZfkIN=bc>aw+!hO)y^$}3NYaSW4B-YqojbnRU^PkmpM2SKC)Mq4zi>GWk1qWY
zV8*&#iC@mBxdHS(VFTcG7BTfNq_ZaQ{w7;Y^xqfy9l%O+KiQHCLdH5wa}a1DXrE5W
zJf0@?2KY0jz=)(23Oi1MvG>;}{6gP>u0|a|1zdU|t^(9nQ~ZOK#v0De#a|jMECx-r
zqUQs2V^2U+V(c3Z^|9@RxKJe&JoUn*+tt^rwKCTF;`#Fkpb9Ro7ppNA3b!g{2+jR=
zGTrig2Dl}2fPho@`QhSGaFstdpjgEmh_2ACLYgaz7SJl96G_NPn-9fzVEJT{02cAU
zxw`ckV~72IOwKPA-@Vsill=&>?|eZ1uWjBishr3X<u78{1>pDE%Sn4l`Gmo`Wo4pF
zX%-WYoCmmG`TZHd>xY=uo&zFg$EK|P{K@^zs&E2Ep)E%L+wE86fJF60bnFJWJ&@!<
zYb;&F5*8cQ{cMVhi<Lw(w*d+2iX1?K^1XNVfH|dcnMX{UA9;;y%`F$<A@r<8jjaGA
z<qST6Yc>2LX$2O%Ho*f>{?<Ehe|T|`;9k1`?mf3Yb>|kw1_M5iQ={fT@rvnu*T<{N
zg6#Dn(SHb(MK<V7DPJrYT*Ls&f&|gKy}W3(QG5TSahHF_P^mT?tl$q4U|PYH3r{QF
z9;RDi+zR&o6X!Wmmd95;wPF~3oT)(Sn1c96<{O&QB7PDa<u~P=QMBc*Kei<I=JOIn
zAfEQO(a-oapWfjZLWcG})<PDZ!t^$}LhN>x$^-taKHVL#FIO4|afgSzB?&R@0rrtL
z7p{rhKlBu2hJtvZctg#{O~YS2LnQfJ`n1-7j?Gy8fw)o;DVJH;*6Adk@6ASN6oFlN
z;|`hP`@y{fmxsJ|K0m<dAo^lx&$`N_H=Ro#Pibzx$<?ON04g(OT%b0Hx;<B>X9pM@
z^u3og^u0`6b?nB94jQ-<5KF*5yE)s6iY^}ZpaQtm|MtPnQ@0+z1owtP#8Nh|u?u|Q
zo2WLW>Ab1fzKI1bEIYpaht`8pi+Tsjb@$<wVdTMjx$%PE(`@m7N(o^~1J)**rlXE}
zSaSF=owYxQA%b8L&CZiD7hzE0t2%VG8oE+dl8UWqk!LEuJ2JiZ$5YSQ8E|2!x0z?_
zM{Btl6D|X|$4W8|gYj=jn;LZH0KWFg$D+4}Z7tlKAxZ|z^>(Y+S(d(lABbyH3JNC4
zX-qwb!46-w@5=6|2b-<1;mPkU?81_?)qkr^t~I%`k-ECNKkBTf`;OphFvnBpJuh}H
zHElWWn_jDa+H)2UkAW=&5}5EznU~fg_h87lLZ~3Wq``9gMxVjJzmYkhk#OOi$^+9&
zYyU390HTeZMk_@QIESa9k02PFSmOxWq7fve``X7D3@%t`9WJ7vd^Z+QkkEg;PsaOI
zV2r3a-#T5PYmcH|VM^fk8+VB{g`ze<3-9-Qr&FD8;5w_})=Q)%utg+F$yrdQS@wfl
z_1iK9&SH(F@@{Jbo3R}4e!4UM7|<9`?F9g~A{x4%hhpwg4*7J$oM|hs21u+axGkUr
zc;=L~fVl-ZUD|SkgFD5C=!QRn_grl6YrU`Svy=|PeL_y}n89>CMIJ?c;p!JTs<?F1
zT2hu$P<^-HwZJ(E3awlEf5JU`+n9TFd+WFvT5{~uuYo{^Lmdd8>3>N|wLvX|T=@IL
zpPd5ucXZnDd#wDWXD$SL%58<<RT}{7mcjj@?WV}XwQ}|1cy8n1Nl77piiUx3{&d+<
zG#-E#W%lho8(2zk<C`hMen(?do<rj{v9Kd7VAX@pw-9>Og~2XhN<v!pqNPr+O45IJ
z>Wv_ZcG*k)5xrDvJxvi>XEFLiGgk)96ns9TrtHcvC33Z3DrXt9Jt1iR%6lhVDs#KB
zUI#lF!vyiXv_{|`u8p~|^Vn95xIM66xG;>Lx~_?=`#k_$bQ|D->^wCExim(odM3B}
z5wo>%Z1LNblLy(P;bv4tv4~I|?JrtX(4K&^&vxhpa@dz=dMnhPu!7;vcL8>(sDDU6
zjm{&fa%VDmw*(rhwr=(Bbj4MPW`~zBerg3AU#lt-9Qu=QF4TJHu&RqtH6h&zO+`t3
zkJ~zYSjrUjPVen&eX|L*9PjNAtdAXgWywAr?lQXLw!K{~3J~GPIUv2RBbP&Wd;dbJ
ziyABVoHrbP42QeU;vKW}8&4Hun~Au)1>Fmw9PX+_Oy^KSh9Wwh*7%2|cxl{1z1uW<
zy}i1IeZnSvgv$LsT@$u@n0TpN6F&b|d!U|EeJN@j3A964-L&qxs3O7n16l@aR~Hsa
zDDm7Xw8s)&B@@eHPr6a;7hoFh!ofp{>a;ZORKdu_tlN@tEipq++HH}hzgT?`v!#*U
z<Dru?FZOzZPKa5;4NcWfFvBKqO=2guPy$;<<Po`2i#Srn;j$rGnZViK#?iA>_>7PF
zap4kDxPaj&Zn9()E+n6(lBn$6e&NqgPKPMs{dt~6T<C=egXO)-Ir$u8pLagmI>?3E
z1AFJjn8}r<8P~~0I(@?4ZfDWk%bMhG1;&O%MdxT$!!0%ae9vty{xpsf44`m1z~?j9
z059h1`&-QkhIWrS@A0RgP`lXz1HSF?*{iwv_Qm>Dl$Cyb-@Fg0wDs=Kua_%VcT);w
z#F6SwZpeIk9|Ej^Abp^SnR^{Qoen38fE5zox8OUx?bt$UDSof(J*DS$q=*iBnA*W~
z97=Q2capUVZwQ)PnBRmtUv(m692O+E*Sz<CnD+NQt!8kuDQwa_NV+~ejWgZ(aPoCH
zr(6E4wXNTRRmZ_0`gSy_O!<32RY{0f6a(-^l*xSZoa#{MV|UxVYbufAxFzes_lC&v
zV*A|;olZ8-O^ju4*bPYmss>S=3Dv>cN?Yn#a#VT?(^uCbSP@WYRsW9B6_A4_;1J&n
zP?fEw!%3nzbM%nbbKxmifKVoHehxZ<l>NCzcit^qAx)p_2L1hcRGn0~cp5sav+I`^
z&)gA*SxV3t<xER(`#0Tc`R{16?U-SI@^;GU{W_mRuLZfswC7xos>c2LjP`m>H-cs(
z7}j~YkyxtFN#9|Ja7Gkv`^|K%=_GeiM5y+#YX|=wyIc)zA}g?U#&t~9Ew5bvEWVgu
zFg&PVE^)7|+OhoJP~m^eVrtSlTn{`|88&`Wiid7hj(&J0-1bAJo9t(q<@`6zW|Q~A
z&i_)PQrl*b{jMS@g`8)Y6PayK*4I1Bz3icvat$)gcZo-#i^FB(xaCTaEH?@D1Op->
z<+rvNT6R@9e(9Dt`b!IJf1d@<sjHs7uReHhT%?k=5oYc?mH+%RW{%VPoS-I0WxKxC
znCWW&d2}dAq2Y~9{i5P?n)NhsY27pnVM8<9SEE8=<p!#w1MU7k4cp`MPgg^)&b}b^
zSvNglM8Vw`3lv1Ilc!3<s@!NJv$4dby0ax?KLWdFG-CbX((X)~XaAX2+uJWpSuX3=
zx+SHFOy8;FhOw^pD&JmR%dvHa8(Im~v|Rq3Tlkgq_Ul3Yv0EVRi68%bv<V)f|1}X~
z&66T3pr9hy<O-e(tG^T^CTC%yqt&5d?Y(n%u%KgPt$w0ba&02YY-;;i)=N#_+<8&(
zr~bpw$ORks+PXw?_l4=_j2&o+*;91XuLX>TTIZ$|?{?wN^TL9KgaO5VZhnexeR*e{
z#`us@qqdfh$Mx|c7kVEu^^dQ?%;a-SrtHx_jK#a1%#!bgMcFGAvmT~MWX)@95&r%V
zalon_sy|*j87qm(r|aM}NcT(kgb`#6gqnc)z%-3+)OoR-t~8wA&i<f?%Q2>fd|l8i
z_a8V`9n+3UcFLW$vXY6#@<7?Y5o~50`1gSr9z=wz0F(B&U{Mt?+%XutuYQBBQ>PZG
z`}G4=iE4SuYNSLT@NUW4eT@00>2?42bPr+=s?9d1F-e)?6{21m+}>wN7f0uX@_NO&
zNQa}%%fj%X;|#2|=x;EV1s(bJC8i!$wTS6dUMph0tmu}|jh$@k*PpiQ5a}dj#*<R!
zLV?L{2hKLf^OjkqHa3Jd$6doJ<4}XC(r?^1sY?07B<4O3ThP`h_b=+}JI}4Xci0(+
zA1qFM#pjV%SB4(NCgjWoidKKj^9bdvKQ^trA1jAmVKFZg?Pl658?TJah&PfUADitx
zr8t``FXlYQ$H8U6d1#%1X>Xq}nTRpGU5JNpCJ6T=6?W3#PWo1J7AM#z7*#}|Sy7gO
zr71Xc8L!K)-$uTMd|4%1?39*&QZ?=LDSlq)r^NX(x3GUS6t(*cANAcslSixXom$_m
z)T7v+to`@*W}6`I%_d?cqrg^8YtIXT&e+v^>_yN@9+NJ|bz;KLuPm9sUD%i47wgJj
zuG8naxpipkhOqK{<#GQM+`Ypbh&L#dsLVOGr=&P)dh24|Qfq2i=J>|-3+ekm({veq
zrP@akZ_*DNTz^^ey9*2(r~}JyclEj4`Sq{`qP|l8uC<fzfOw8oXo{ed<vSjN2um}p
zS;nZz#S`>T0XOP`I1xXq$n(g)Pd1;ffeOqCs~G`+ciLO%C<+9>ru8s7Hm3h(x;YS3
zNH}<cNYLiD`MLRWwyLk{KC|y$#`E9TvrK6geHy@)p8?ZT*R<Qq-5Er?KBjN6P6O9z
ztld=MF{OBT1kIE)Y2fXzk2aNA_yil>!@QDxC3(<9WVf^Fx>vnsy%%<s#M9j-qAeD@
zRLN?iczh@h_OV)tV)qRt^(6HT>%QTNH5Uf0r4JAb3`1Xs4rKAs#jZ>H{8i<*wDQ>(
znc}?1{|eVTC0P=Frzv%qiEVaG*NJYWo((9lwGsJ@DgG?4KOK)uw3W(!Y9Lr_e6L|S
zvbCP(wv@%fFB~VYHQcB_;TqKD>NP6FSN=7|Sev+i)bMFWtKdE&^7Cf<Q256On?DU9
z^Yz7m>%V3_#$9x6hBuCoHEEPcnU7QZEq3;@3EiA+spsb4L}JP+2HO_F>%6S(i6fZx
zVVmP(u7+MzJ7PcE^~j{E7FnwW3&`7%FBRo|T~C`8TSfONQt~S=@+Qjo5G9zYnKdR>
z5F47P_k!SG8cUAJ@gaq4jv3teH?QH5c|4wBzCmtZ&jPQ`?aGd#=lyG)7#V`<J4WoV
zUCw#rxZk>k_^(y492@k0ye3kX3UAF2u1YJML!yn1e$P&EaB?TRzEd|Ws5&Td>yb|I
zrHQA^_=rrM&rMz*?8dt#VWphb3Jx*|CS5uFT<GF8-iwvqIbP0VQ-4^mszz+u0ag=H
zO>$?rQ)kzkNj;n{`0|Evv9nv=mUXIeNFeX?do8zC21ZDNGMU8#_J%;cl`L%|V&`*q
zPAr9j#6X~ESc~~<xd#(-3LH`>!7cI>R-mRtH9bv7xpg_t;H;#;`-|0kA(+_bj&H^x
zIAn}wLe~ttTC}5${wp&ZAForZyzl3tT~`VCl`B+02UOI~Ueke%6~8g2<n+;)@JY2E
zbSiuhk(pg~E@qOOmX9A%GTy%nYX4zpkXH3-u)USjqP~WHyriL_a|*DTRYYkGH|*MX
z=k*QTy1F?n=DR6(<I5eFt*2a_rDJ$k+0dS9dRgKUwY$X!%PZciwhHF$(lsqr{6Wxa
zrsI<A0De>D-j-q7S_#I@&~zKEv5`(<MV;x@Dg7256Bcv$b`tVHya_EMr_%#RzdOPm
zyF(8%{ZJ^yoo%H@pe}(jndREu$&A}?KNKI_(f&L;v-<h!D{BY+C$=}cy3-!}OGj5{
zH@7K})v@}+m6#kg?@Ij@^{uUPI5>Rx_cD^|m^Q(QB3AnF`PN6(Zpz8XeTJ=zbtl=E
zJ~rvi@~kQM5zX*RVdRj6$9N|jsdFv;)_&o1BvP#{FMXegW&hKxngb9oRFQ{5)NSVW
zeMx!B;^^Yy<uS!q?^Es&HkQvEqi%<2BBjOJUOz6(JI>frS&4?R&LZj3`7GFm=kI?~
zulyRXWfzEBd?t^`k?z-8TE%UCM@RW)B!;U!C`poE^7%o+xbWRO&?~z616*pj`cMt2
z)6|>VfzxRrYMrSn|JPt5o<;c|6b-_7$8X~T$HL6kL>ldyWA0I6IHM}IY(v}cKjbg%
zmr$&#`NRpf8OA9*`^4{9Y0iByb9bS^0P<ndC%^G~noK_P#psS(fxK}#A5Ex)7}_8N
z%CSJu>l>o=r6EU!C(l1o1$al{y)%u|09B}8z#oZtIX<3oqu_?=Es_uZ*z^}S%@rOj
zYWF7c5~j4K{X%cQF146eeCiL}I9y8#55NIsQb%9J?l4mCcP)+%5Lz0)f)GM5Lya6g
zz352qiB8biJhh9-Ue8zfIXi!<JfMl5eL;5UU&G7NCY<KV)(n=XI!~5Ln=|slyKy)7
z@y_&JVOxC)9xLJ)GJbWHHBpAgtk&zn*l`?M@;W*)9S({67HLq5YpgbNn@*nX&>!V5
zg56o1Tr<$9z)uIzHZ(uTj0m0r2MA1ionfU$pBD=isn&DBrbjtPgfX={{+3|hYgcJw
zn%QQ@_vHR-D&ws{_*Wwi=mv7q0@4<_Y>g;XGXhYi68(H=Itk>0IfAQCayqA(RwXk6
zHM1jB;SQx|+m+TD$}<)Ihj-NWP1P3Z9^an!PLLNnkWq@fWQ^l=pA}_k@KEz}@yZp<
z$ZaiMMd&FbLSMTTub~W@C<lz`TNbGymWUCU+Xc6{YT)!d83PsW3ygzL0F)v<uarN%
zf{+jxfy1|m-K8mT8c}r7LTu6(IMFs8EWbL-{wA!+bYP5tAtqnv;OnrDz0DDP+SUD@
z8g<VefNUm~eGSZJP#&l{&JDQhv=KLD1Gb3w9V;=;fhw%qu}p-^StmIr+i!z7*9VV|
z1L+M`=o3K>e(PhV7y6JN#wPpDsie#cvhe3+GPg+T7qT?p5Cz<v?!)h@H)WEfNx?Mh
z0hywznO`=QVX8b3Jd3fVv0t7pv?sDT2ib8ttopYvmm>&3?|Ob?e+@yUM}U}E{oxB+
zT`J3Vq>H1b1^jVaL4)|AaLM+6v0Kq6CvoOU%y$q=-%ufa?{za&ASe{cwo*fk&wz74
z1{z-wCcFV9aYN#_GR<0JRsTL*(rV~WY|}4|C-2wb{ref|)g^OpEAtqv0YQ}s1A+h#
zRBQq?Rs!j8aZtE^C%sL4>2!oiFah)VCZ%Zo($xCjg91plE+v5~^#2~Hsla*1#=Q;t
z??=GzUXFAHU*!M$8{mT!0}w1`W<xYDjne-;7(ZwIKjYv+V$lY|F-cRo{u|-{j0V6L
zmjC<TF9QP2HD_9BxdP1p9pq&+;FL)J@A&{9{{K&j1`RcTd(>1Gqu`CC;v+w&d%6DH
zPixl_?O&D8!9qX&Au^|b9%!8Nbl&}N=Q4=~oQVfx?51zSRRn-er%3>rw3}X`*IeU!
z-V(2`c*6<84Q1=~{R~e}4OnEn-EPOCBqt2t^!?YquUvJp?k>|w`&GB|A7cu(bIC+^
zSo*Tp)5%9KYK%t>s#pPOmaRfXG@sj?v-e?MF==$jd4X{h7vJ8F=9PbYS*_?jPDs4Q
z%i_le11n75sPfffj8>)E(Qj8yY4<E49OLM+!eTio!@JC)adWQK)y|uWe|OpN{<Wp#
z<f2@L1dwbmq2Pe4-};x=!*_SP)8wtGXaK7pi~AC%J?1KOGBkAXatA|%>{c0fzL~F0
z_)RCh$T^K6KX+yozC(Ndv6Ub$Jc^XMGu|InU|mgk695=>X;RSxu;hywz&RcIg}C~?
zmxrZK59?lV58WP~i$*lxKLyFJ?9NuoSsyIc<;<3APeoIRT9_8F*Fmx#O+uq7crhix
z(XKXMrqQ>R>tstBmRJl&s(~`#PPDJ<F;NfAl<e8X!MoYzG{z0c7)o`HoZU`O@yPGC
zM98_WaNL)fli5sS45ml^YOtCwyPddG7+p)+R;FK{oWQ`vS8b)?btQ(2ekrXhcbKVv
z8vMq&YBO9xKO>19DF(I=+!tCMqFuz~%Rgr|wS{aeeyUnKQ34-TV*qKGe7Q)!keJZW
z6{X1GVf#(`-<QgNjMt#@Yviaw{a{8Wd1q3>>x4BRD(HEDQXkFNbLMxtLN;+<{$v5G
zHu>p4jQpS4&&Q*-fAVGLp1X_Zvc0TVO7)hO%9<%HkB3~88;*D9>(dNx*OJPXjXx;~
z&z{rajILKnY&+YUURL)W+6ly0#AMn%-&#Z)K$`ZnNA#)|E#B!`%5vaY(f6&EEmTfB
zNhFo8TJXCH3>OQ(YSkkdE=H8Us1sgV8j2D=+<#W8(MWUC$-c`NxnmzjM1Q4F$9KuR
z7?jLkH7b0yXjc*CN`)+f!3N18x>dXII_|P$sz_wf8F0=@;hC;dC4#!znbR?{chzOW
z1Tr2dymUFXJqqUQfz4p^WvjiJS7046MjT31kk{Jew8a0Et+wtBt8^tj4G~_9z!E-;
zx8N;BqD_UOYQ9yQNe!q0AGujNcmEb-S+#zo#859UiDeYa+<|6o-S5heRSu7fj{cZq
zJ`jile~B#EtwG5oJ35Z{ze})o18fz>ZX1VS2U0TSRL?h-;|6}?EYeb~XlZbiH}La(
z4P3qGOQ%l$WEI{5Zo9IVc2Dbir6&`E$A>Q!T&nigS`HSl5!%X2Ae2<EzL`aPBMeDQ
zsT)x+2vG|(L=KG;mLpyKnUeIQAN`d;8^y$~8nKR0Q&m6CL$WBgB_!@c@~!aBhpiI*
z`LpglE_c%Ant6xStuGN_c(cF)H&H5R2{}{%mD6eca=3z*FndLK9k&_C|J}M7D>Vs(
z>58zqV3c%hB3K1ugBip-MC2m=@&8-PtoWCp+&mpv;-0L2d*xG(SHi#hv*#LTmxi(S
z7~XmKU8j=yeRZgDcI@^=XJhv8#U3hf_;^0G`+Cl1GyebAfnAALFD}=|=j8XyeB0Cg
zrCcXq=ekdS_bywsdX~3PYx2Yu_H(L=v*!7$>1_y-5U<ofcI_YU@%k@YmO3tea4Gmo
zTW)ppxy3D@%nPb&-W1A~irwruutrrn=hgh|H}~{--!8WglT^2^kWxP(;_ZL<Y~A$O
z+spb&52&;QdnhIArBALqdHIxhdHT&>(_a>0y;VQDHm}XTEY&A`l4V~XYw)_=O!Cq)
zw<{jryxPO}Fma8$$KypW4@RiG;$J4W)Y9|8RsHnH%$hRLh|Su&o22<wU(apc{av=`
z*#3Dwna5lo$Cf>~Ab5Mn+|&M|#czw^jN@NBJWV;V#vJHkPe+HlH$4kyUO6$xZNdNc
z$+^exF`NC`n0IY;v~qIDroyvE*A~~x{(g4S`)#AQw6({j&reHt?TbCV(d4*JrkiT)
zvB)jCufvP(EnD{Fpu|a~V>36K)E{S>`dUQb!T$fgu{m>mK3;g5ZXLMp*Hy2ZugkX`
zJrjRU%Kl|bg|?c|*J|zX-+zQ(nCEQzYg6C5x3b*&@tLYyYk<ywyZZe4)T7)_i|zcs
z$6hjUcHDC<_Cvv?b&jdlOFim#bMW(*C(c>2oHe*Fy(+gGSZvq<i;d3u-YbV>JQ@Oz
zb^of*+hb`w>-)P}*$ph->5qeK-&E~+|Mnx_RKXkfHWknJtzEhM@1LUb*Ik~b%^MAN
zo?e-<$gacgT;*rB*``0N&TR7Cc)Wf7oCA+uM;JV1+odxn<D<r$N#4;{wrJ|d^)9WO
zb({OgQdz%GoGHM>5g=>z&z~tqsmuG1j`Nq_wmN%{bbQz7S^bymw43d<y_V~e%hQuW
z<o`UH@Vp{|Z{m-S+eN><dYhYbr}+OSvx<V3Kd<fJ$!yYmK415e958FSs6DWknQ%fk
z{I(0QRt<Uo(a&XCj^FwD*ZC51J=XnwaaKa{R`r8`wy(;zmks{|$4q-;`)7g+J5Zay
z+V28q`m_8cuhMr<t1HNMs=e>}YV*VH$|c{QA8ygLn^W<dXXmS=Ex%uET>S1K_qxCH
z)*qbmc&44$?apsEJC0he<`e8-VeIYfVQ~d6z6w-3@G<3I&g*r5|MAV<|L0#HtJd}W
zrE=_t3l(PC27RqIah;tg^daWKy>P)f3*}EVX{rZETIm66LJ5{DbLUE2cU+5TJv1%>
zwQE=EO_$P!H&mE99f7UGq!{+Cz{ORt*2ttTU?y6!D<*3VqT!>$28!Q8*(+R#_K}Ac
zuzfQ5Rnf|gz;p~nK#PG@@UhYWZMbnR3M+wgH9o6OFFgiqYe1Vo4oRS~c2ox<H<?D+
nqaiRF0;3^7b_g`C5d2ZU-*tVHTT{mu1|aZs^>bP0l+XkKpBSAK

diff --git a/site/img/shared2.png b/site/img/shared2.png
deleted file mode 100644
index a1e04999eb3fd3bd7c350a0850659740d4a8cd03..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 23204
zcmeFZWmuG5*9HvSV1Nq93@9N8(gGq~LrB-qLxXf9-6f(j2&f214&5*`(jg%rAYIa(
zGebAug>gUc`yR*l@B98e9)~b<x!1n-UVE>#&b`jH4OLN=xs6YbkA;PGTTWI|4GRk=
z9t#V*_{KHh$<4vGr@$|47d06PtfF4ZwX0_@WOZDyun0)5{$gXrC*A`Z3|nhxyJ{;b
z3Yt3Fvp+X;G%;uQw0{A##=;Wz6a+rno4Y=z@wB&da1rzrq5aiD5cqucn1hz)R})uT
z5n62}6`E&`&gL{c?A+{6Xd(DCG&I7_W)^~KlG1;~fo~$TR<5ou1UWc7JUrMvxY!+?
zEjc&^1OzyqJmq-$lnrRX=Hlhx`rMPv!G-R3Cx82qG<Pv|wtnGi?dU*r)$el?M>kgy
zTH31@{rl(lI9;tR{_`dWmp{`2CdhI1FC3igPdNVV8xR$~dMfzL(f)<Axr+<XKZHm4
zSIhtP>_6}M-Co7Y(bW;qg0r=$oP(>mGa&5x{AzTNr+=jX-&_3OzM#(5=D@6fNpt>@
z{-4kOv=`>Mn)v@v#BVMCdJ3o+f-lVR?~+0Aix-|wVPT13$w`W9cw%o&T~E{)9`C?d
z64ZO(Vq23+f>c2&rvIp#B0se5v=?!fNGcVSjNbD>jrnlA#ltHhfXkNji+elP?p`2j
zzdt3I^yo^8o5Aezmw9X_`fMNG5we8VT0Fuf62rp&?~niL;J^Fezh>}XSMdM4C%lm^
zrhJb;DvszILs0z#J(A9>-P^m>cSI|-a{L*0#&7gx)7v4!(ce_=;&zdJW4kmrl=_cv
zO7X_g;LcxZul<j)vJj=D>0lFa&;?@Q+@QJop#cqqTkQYzB>VH7CQu9y5M#xfH~)8b
zLF+eF`U{dNP!j)ko+Ac`;fd9~``ZtViU>AK{IJV{!S8;rItSkX#IWAR692nX@)*UW
z%Df%w2>!L66dXVdjD>mnpZD+M=2&U<g?l{w&md_4F&r90X{@05n~Tezx26O-CV296
zAaz1qxM#aRN}M`fFnt|aoRhp~)r-?-udr~|9=%5>hh!Gq00tKHwW_N`-Qydp|8ANl
zaYA+L=6-0%!FC@p>pM^*=#cUF#O0vHU}rN+6_3|c4u=qQ_%Z5vL6d;c%7p*aXq6I3
znzkm{LX_j}7%K@auZ8<k(Uy_-QhZ0_YqBviF8K5UB61SKQEVNia4AqpDJ+8(w0Acp
zEzWQ>Y=BmUE+}k<qqr^4%-!yzso&(vSoOKHeOVT9?Mi+N`BHa6*)!weiMQvc=V^WL
zDn)NrH-wV&W>!$OxcP1b2g?QJIohYSAz4RTKll3`f!@18n_!^p`q5;rK8c{4yR?<Y
znd5{mbO#4e;#%)MeIbp)q{s+Py%JsmasDNio1WbscHK*SW#6YigGw$EmdC(8i+F4!
z92eUD-C0#8(t1zxU0=8ZdMyV5-NwX*OVANf2lWrqR;r)s+p6~2#n*`YR6Z4QnAG1G
z5oTOfroK>U-9D}PF_M>>SFek#d+Id)2G6~q2I5oIQ^GTGF@GfC<9~M2)s{7Za`bbt
z&({*Dl#QMk9TK5V7$UNGV;5%{T(g{KK5W<fwCqg-TRCwfcxevlbN`xKPs!_wdfSoB
z51b(fTLt?aZ+jwFa>?#x_Vu4Of+mc7RwlYh@L%@z?)xlF=ta>r^9ZNO6!8?8y6^4c
zRo3f@^$n~nl=BVMGzjjSXXD<Wd9Mv@t<ZgX4_pbhA5b<`1(<EYlv-KMiTw+U8DY&k
z_*)}=_=PeXl)Fw|E-_2E;7;M&eH$6<hpd|8zWU(C>;{f<zuKj1rlO2J95-}p$a76R
zCmXzY{Yu8&5TFUiFxutlcQjtN)vOua#!(ZTqYuuqRJ>$1iwm8%W@;BFGQ9A3)tAPQ
zJ`q%3q#|mp2<+Br%j_o?wWs#KDug|qyG$jym(LmR<KD}2`OFkv6cBbI;Qm&(Tckck
zR9XwHjeZjv&^y8Xy(G~M@#aU;4Lk{~pfC6;{qI;A;~o+@1c~s}dc^5|ARpzSD7G%L
zA0OD9_z)saFeLAmgvis%vWm|N6$-AAEP&Rg!7W@*lMrMRv0P!gWGI;JO2d@1olu2?
zcWfSvh+?#}giEIAHUs>~XZxh|$HkXwmV^!RM>5dJvDLMW$OUjOpawi}SDXUfNNNep
zzsFxsdngH2Xr6&4_->%8(LQq9S)>1PjYZF%fY9+i&Tb#+O>Ep!b-+cq8+!B#D&QTc
zdbkCxV})r++s&W7of5A3k(D$2gx4!<kd)22z}&BF!plbK=WNxh<zNsaN)S=24~A8k
zd9P1AmMI$#v!ElhXFk0wj1I`olIp<tlIPkE4+@EbCz{y3c=z55Q*hFopw9KJ7W_H9
z!!O^!jH6ylneU3Kq^+@@i%ZaR<u%jcj58NE!(`oQUyR%^3v0qdfpMdP<b{lenYZ?+
zT<k5~vIDY5<cw;v^6C>RhT68keLEjvvc1&rEpOX`l^(}cnfSG}&K!p6KE2#=4%!-F
z?r?|w6y%?I9l>_1-8v?g1DUDS1mi@TmswaJ{R7ON=<U@?$)n$j#Sk3fCQS)0kKII)
z;u?(h_UHK*xoNSQn@1sgsyi$!46XNL)S(r8_l%Am3*-0=3JopuyCLxxLtG~Xt4_z{
zWmP;MS{D|#l-Lpl@~mQhzCpCmJ<<Nk=x!Hl%V$-naToBMM66;`Tw1{^H_S@JL@J(m
z4+#Dg-x+V7t`X2HYhg{8*5o@()mxhE^mE=CcM6H$d+uIm@t*9chJRx{-#^ic!h-Gg
zl(5al$En@gywZ*6Aysv{Wu(UsDpLkq)fZ~(c)nYTh{Pr4ng|P{@VTv-haH9&s$rM5
zosa*|%bRRT=xqAAeU{O}yQFPMo`!8FHyy&wv<XA{!Np^Xs(0+gQI(@7iF(VqFRczl
z&wU*l_j}?At#|tH{&C(4q1$;G_pL8S;v|<Px?~Mhe`)j&oP}$lyy@_E`bd?3`*z3f
zBK}ZkUrEAsutE2j&_79h1;wo=@ttg7nP*r*PY4QZO>ejmGgIKiu@#S++RBfVs`=I^
z@u+)-u*5e|Dk*)0(z4jnzV#^N>vtlxN$M$e2pAV*LqO!L+-`TPM!Veew<}aRSH3lA
zna&auSLrwgp@>`VJ~k`DVvCRSu@~UN&7+@<ZuHD3Wr<6!@(AxUs-jfZ>CnyUV!v8*
zF`9ZjmKnyF=@q%P{{+2{kAcPMoTa?;H|TvM1sv13bEW|FZ-}Mo1Y&N~t=_r6p*qNp
z81Tnzu=riczcKg!vu=p;CJX70e@1!pJJ82!Y&#RS7cO=!$+d@F_KR0)Q6|3&c6#@P
zKscY0k%c|Nr`Jr|8oNQAMJXChb*#*c_)OZf<L3#+#+~PQ5nlRJt0ds8jG+UUU^Dxy
z3Z@@DyYOBF=Z4u^EO5`Q>i3D)aI7B{%bSbnI8~+jlSd0phOLiwA;+2F(c=-#$8>>W
zOIN|ZH?$tmT>}m^Q!OEQI$T+v5W7)}4)k>}{mZAj*tjh;@2Ou)s9eJ(`X(Rb;x;nF
z`kJNPEK}?xv7FG2->^jU1Qhp?Hc;#yu2ui7ubL^hZkS~^p#--An77r@wM=yg>_*;0
zR(6%|w;ef|@8jHfr5b3o{k)Fl)6JkwVo&Yt^@Vfg?qqf4S}qX-dc#z-t<)9J$$KO=
z>J|NE>^?Bn$3&|Bkm^N}wlH`czM3cQL@pDAx2&-R3BFqPoR7fv(4051#3hmqY>MpK
zWpA^;`7PnGEyL{SqH^1-c=NBd#9}mhcsW)U8__Fu#5ZD_hFO#yLy&{A_-_J{E;UIl
z$lJeqiUIUyQ-Wat{6`s<t!?}L^4H@$KTiF6zDI%l?PURMC022zv9NHyhk;@^xYMv?
zI#4Vljbmv+d=q%$JlIXO!lRV$$|cY>nWfyjl?>>m3$Iu8W;-*Ww5&SraUCoayQo%)
zxqC9iG8_vB14@zNN{)Znm+wRcxZzH=#fUBE0?Yix2T|#n6~Gv>4t9<`TK<6tuq<a*
zGEq_mh2tG;Cz>YkPJ0UJwOl!#x={uP`pF8@h}$BM`V57i&KL>__H9pd^m$iA=p8Is
zxY{B0g^URx7E834`PVpv)?aPfaw!^``fif(tIKHTc|Mwxna8rGd(qEceenIcoI@tl
zvlRf4*_|pmcX>wlJO`z02)~2TN0{&6FJ;y)O(=*;Z=M?w6gc=t+}`|XDzo>kqQnJ%
z!Shk61dRhue+7>WQ%5YQpzuE5gfo{762_sv$Lhghcx6g?S5OjPd+o~x#K2{11y45v
z-EEcm)MTdDWNOK^E=EUd86!C31FB+)_+IJ)9bTI*`o~96Q4^b=9oW{%!18>r&7$wM
zYSS%KB4;Mg)A=UU$}79IKLG%%aVZqpnA7C)#GUnzOe!LK0LtXutKA2yQCqGdJ#3Rr
zLnegJV&6_3x}_Fso)7PSvFH;PI-SBfwMFSu{CL<QtsFWQj*w=p+8|tC`eIfk?1Ng7
z@KG%+c!b+UjNYecgl*<~<|m$e<}Tx0X9}|i$&3oKI)8HwT0LyPO&8|B;xT;KVmV0b
zUgp0-I4=D)lee$LH~cfX0FOykfT2}BKD>d^=EOif-qveIDRZl@a*^PPC;RcToXC7W
z37o~}5D0fG2i>>ZD_bKRItLk3)T1bCb-`_Y*)p4(9{8JaP9fr%&M&IcL^js@K5(5b
zk29WRVydd}Hw`tP5dNg?xTEE{BRs*^dz0n8e^gEFu=mM!9|eq7Z<w#v$v8P2L6Q0~
zZU`zq@!&fuXuB$VZ=Z61?IFFaSil9IhlQYOB!oBSl~^@SKPo(6^OGesM|^%glE6M;
zM8G}DztO|N!o7%`YWqCz$-OMM#gUE1tI3o-Ph~lAn0OK3uN)3oVXhN&LN)s_uMlP!
zN62JD|MXcl<z$s!k&+Q*357Ixj2|&m%RYrP!cW)i-{V$d3m2-29P^(z-0H-uR4Yu&
zZb%iC8_l#<pyka`Xg#r#KHgmzg)s#rBO47H$G(6EXGZFh@fu$NfP9b4ft%w2agLbr
z7sE?Qr?~l-_y<c%>o@gttLn3la=aZ^48~bz>vc^n$o#JRqHS9vRBf00IY47Smn_ux
zM9Qif7%OcPY$GELF3*W0QSmt*=8Xwyr@NoYv30rC7m|n4d;3LCgfy{*caCv{F6(i+
z`>8)qhp4hHuOsy@eQ1~_q2#0E$_Dc#otNG}E0rR)GMDIWR+hBWC!_*$r4Ns_*^)Ce
zw<ecyxu(eS{MKXoGhvbnKGc&$+KjWdw#g~zA_FyFyh@n+Gtf#qpPqB~wV<(?qp}#%
z^To!MbnBD5gNKrP*=Ze~zN4SM_j7kvoT0LAAFiw{;{QmC(?@-x?$HCG)HjxvYFF@V
zKOB30A5i0}5F;(fC&^Q`&PIK<6>^L>omoSU-alDVErsjo-r1;9G9)U2^7ya3qjI)q
zT;kO2le?iIZX9uNUN!F@()V`xpGU9wA6=>sj(!)0a*EzL{(y(0Nt)B?p`_H9kf*Ob
zt-=OG&S4V;1fScFmde2)zXzv1KQoH_BP6em6Qt(vd^ghN#6@#ouCBdjbFt$qZ1Z|q
zGaFk((L4;GuJ(D2qMB-L4|#t#iPrex@Ug~gi`N!}>lF~M#K5=@h8jAUeeIQ~%jEK^
zn{>KNQs+zJEv4=NN8}WS86KW<hdjejFPJ8ti6AupMC*XCHqe9dXM<z%R4Fw1IBjbx
zA)(y>(Dv7vc*+uU+Yv;wDtYl1R}00En&<?t0J(Y`Ff=J}kN3o~oZBGHu}SrGjn44i
z>Su{lX$4lVs(R`-SHn0PTp<ZAQYkNNIUTET4N|H&z8jM8kw`XPOt{f@et+syc~sxP
zKmBQO&1BQp<xd|wp$;HWu3ic&#ifB^Vjokr*17?*ggT+K%4V+-MeuilZd&)@4!pJv
z`vzXlG*YE9%R7B#OL>Xjho=jse$vl}FO{nu%u=VG%62nv#plyU<13Vxx5vC$x$%h>
z9b7q;kFMZ}WLa|K)ij({-Z~L&%jjwh5h2d0{xzQCDUDq&x;Le`dDD+IwUq5RV8+Of
zkpf%7#~xL$8deVl4;?nghDt)mpIu*`^!f^CiJ2a_H#u;>)=}N98mS^RI4Y^I!hJTT
z>$^R8h_kM4wE(hYnpU0}l(p_89O*}@@2M3hs>*Xn?GrwDjrx@L3G!ivC-=f(UMTYn
zio+JabZRz~PoecxL8uylWDSm+b}a+b-MqLTX=Yt?>uX0DOYAbWQ)x5}n?^r=G2oOG
zr!U)a*t~BU<mo?(o&v3ek*sXyO&{K0!Fg|BgIDKsKB*H@>8ja9p|4KR+&?HcD*|@U
zm@jo@ah2`zj_$$~nMlfZJ>x8u3`E60uKtmk$iNn#nPN7`fbjC4F8;W{Q?_;{Fj@}g
z*P}n2REMM<vD^32R^y@4zfu*VV~p?O$bn~T8FGK73f^OjPpzQ@UBkv@VFv8cFyjlf
z`ST>af2TZ*kdGhM93KbvDY=ewqg4(#MkUg80u4X1&w?2d@$Gj-z=M|twcpim6?(!B
zu97SBi~v?Sd)x_Nh>xtZ|Aaxcd^0?CI2FVvv{Qn44GPAb7GlR<0=b$;T7jfJttTIa
zu0A%6j_hr_7?FI`sA)2o@GYCq9x$?C1+c?=e!$E^%jmBFO+-_)prTP5q*Fj&mug8R
zx-wz@bvX-9h!j@PyDN;!8T*k$oW>R}&DM6BF34LazZetBH<C38Vq|vB$GbWoy!Qoq
z>*QN`n@e`2Eyb^yTH6@)RMPl!?}B<;=6z=fhCe(%x>FAho&Rv(>y<=GxV%%$e3{Cp
z*Al-nRG(PTGWLG*C9;^j7F2?g!e^oZ;p5n14xb^eo<6z<ZHJ+v5P>`$;iq(wpYXYd
z`Y+Qw^-9DrIVSI40(l?+2ssCh8O;)Cj;wOLFmieSDH#Vs;P;6(0hoLr>RuZszM41w
zkXp%cFuw?rDpMNJraaiHu&|$<+41I%&iBXH)mb!i_`LC4GOlEVp)07WxJQ|>fdTH<
znYOjQYd&d}xp75|(2(Hy#uUxS_{`U!R~o`UJ;2e+|J=x0$0{<Z#+X#h@qL<pQdu~>
zuj);gfA~BsWJYUWLP`ej_ql-pw@!0VkI#}70z0D8DlNfDA5_j?H2Mj-!#6oVAvhOc
zJ}H+yckGb&^imyDXc9l~EyeLiQ~}R7<aH2S1_VjorD`j|r(Ti6M+8E*Pjon$e5Dk$
zL+`+P7eJ7PNDw3_i)A1bP6(P_-{d1d3Z_lyAlcZH`<1Yxz#kfZO$SCYk+LRhQKrli
z7~Xg&Z`irlWx-p(G;3?otFi}s-$JJ%BPF1SNo#>sdn!8>r&az|aiCZTPJ{0|sPO$%
zH1-+feE=C=AxPqe_|EpOt%OR1B-xxY)$Ft+^{X>h`q_GFAC?CiN6wNcW@VKl>VN)_
zN_lLv{~Y<zTzbHbz=qA3Q09!Gn@66yh<l$pN1rn$%_xR=^n6rf{Gs>ez^Ci9;z-^p
z!i40v3V^f-!BLYx^{l8Srf}Xj1d2sNZ!`;{a?YS7(Tm%4Gc*Ouu*!Cltrk2pEVze6
zKPQJV@xtE!K5NOs17QxnO^qJ5y)fS#emU`tD58&_&dA@XB_?Jp(C<FExc+X2^Ytg;
zAT7b6`}miN$4NO3u+0LR-|5NsYS@U!uJ7k{LZGzXa&ufEe$K?qB{(9{^)P&3>D31;
z1c?##n}s$Rt@zw32OUh)Dvvd8X>R7DNzJc}1ITfvl2~Qjr9$4zLf`y{koiFEND@cQ
z1h+`D63W1^@oDiSCc;}>J|dCo`o4SyjG<m_Em{F-=51cq@K^a0H_A2T`m}s=(V@7I
zRAV1J<_4UG@BJL`AWRQ0ef9B<u0Pp*-J$tD)YR-W)i&Ggw8-|<A{+RR<_8RoU6O*J
zOh<BaB)5Fi#L4iOAvmW@6o~Q<y<frcIcFEwZ!X}35M`H#>{P4GA;MQ1g<d~uy%}rw
zOM@-UMD{vyh4X%S{OWTg#IigjXuJbtkq#-d-{F*C&GlRhoQA#k9O8=##62oMGtl(#
zt78mcnP$;}9+7vngp1H6CXOtxnS5X3<gVjDH0C^{BJ&z~mQZL>NFC3h9*>+R8Cq%(
zs&YiT4&j|3)0|On9<zG*XZ_HjHv-h^inKF4B}x!4OS~m=Bm^~2{nOBykpuL{JO@s3
z5J;4USJ1@7zHyri{w3dTT8krrj*R%9WOmR~TwSZvHTVa7$JE+q{ZR2d?IzaG5X!j>
zxX10D2V{AVrsI`>XGkQ{$Oq9fiaZx{N=wWAqkAkIP0*O)sZqv}_#6qjugTXKe6?|$
z$<<5&_=q{`Zahpd|B>PIbx{1>kTRNjjRFeLAZUi=BG(G}Xa59>CF1%@1VimU7okJ%
zK;}}DUY3_MxlJs7C2UEaR>iI#kOX{qI`6r{+*f>mb^XHC)PE&$@O)neLtW7~38%?<
z^)UhPc6_xDfylz+d=pd}^|Ry4bj>`#AEe8%+(kd9gY+Jlwh6Zxz_IcFY`hy8Voi(?
z$~;f;5^YS~JW&Yr)3*a|ZL9_d>nS3Mqt7jI@FWTCN;9&~OD2AKOdf!UnA@nq8ciW$
zzm-Sx7^i`|*9eSa^$m?Z1=q6r;z(6yS-Jw=4ZVocjN88Ty!xYNhA@@Y`+3z682gOY
znN+{ZznXtfa5UDl3z}i_Rm_HFjA?za&n;Bux3jf9fW`Sf#Mn`;7pM202EcXTeIQHb
zf#(A;aJj#$Hh^0>AdQg4<hL+t9pQ%A#V`5B_l|a)g`}ES#dL8hJAI+mQ{zioU4qNO
zL*d<^hVWm(0N8i;8D>dXeP6cehrk-@?k<x&c`D$A=Tbi3N|d<RsQG&I*_Fde4Znl-
zXj_4g!}UNyOn){k&3inYV?i0j3rt1}DsLyg#Zr7!-OL34>qNW(-Ue5$-*~d3Vz{eX
z2a9hf1GUHTTa%O)nB2e16@$<H!|PH96Vt+)6v=#`dh{w>QGRs8>~$9U)$6sfl`Qm8
zD1SL29={l-yhT(9$rPdS$|<W6{wo^+m|Bnqe)&N2Wfr=~0<}}8*$?}0Gq*u`9@b{a
z%pYg!`kfQ5m{WJE^2z|@!RmXX{YD!z+~VJ*rcq(WMy=Ota02^OfF0%v%W#>1g&+!K
zEy5CDO%}|)?QVkEg_=jUphlJkb^V-gL(j1UdH!ycM+!}h&PT8AAfFuL5+YXTNFW28
zBKZ<Vajh?Bkfu|~tqX0tDhVuQyWnYZA<en3gx<`5brvZ06VKP=4HWGu35ZQ=Qf8w=
zY1TgSs*3yB_Y{6qF-Dmr7s)arvRF`Za6<4d_>je2{nc|XCUEfI6&t6So$8ZeGSqsk
zCY;W%qN+fA;97&fYDz$&<K_GYgWo7+H5O&0U^KZ)A2QQ5q~}MdmhrVhgY$P}0{}=T
z4&)BTtW5^)1@{5fvfS1Apnfo{JpEKi7c*+JqJp8|7-9Fn+%9I|8&kl%HwkL7Y@<fX
zW{<$YvHy<Y(GPJ(pCFS06ZpKY6jW;LV<b;vIy&wEqdfH~74~I{*3Q}oylT;=*cJ;f
z1N-MI|J5DMdo3Kp<~rp$urfkro<yNV<G4nA$xELY#jSte8;|{QZWz9EXiM!*fSP{l
znEo&8$fGA3+0L^`y5$Ju{L9JEGjg?84)SpdN969Sd9M(UNd}fB7E=_yLn%A?Mn753
zVaXeRk4FLi`SaLqW+9G|(hX`<mjZlzRqlp{$)bI%k00w(g-U9jF4SnJZ<Wk(?OkWS
z$ke9VjTbda(*}{S?g^`Q>e)o9eer^|4Z<105VOFmnGeYV;4}QllM8^0Dow1;r_6OY
z<&pFMw2{C^L9F1+OpA%tsapFHDHjo5`yVOd`QH2M6<Q(1A^r|o_{LEcwxhhdWfn@o
z(X6nMVN<G1o(V^#bm?&XcwT$moby8WtqFk&o8hiu+Y4<4(b0he9FgUsFv=xsZ|ijJ
z3Y%!=nfV_WyZ!ZHV*9Ywk{8E2OI{OM+4<)IA^0=fNHTW)FHbx$^KI4LF>KwiAvLMt
z?qZ1!{UYrw{H^)K#Ke#Zh!H9-zixT8-_rl|Rse!ol$D`wzUKK3e770DY76;7y^1P}
zN#!b%1bqR>%69~GaX^OcC0<<!1{XkvC<|&>qyM=;-rg`Qv!tHIjZLdSyI&)^xS}V4
z&uBPHDWT;KmEdBf-FT^1zE<IS&iv=e3^@{3R&qWEySy*VlP?&M#d$l{VWey<T+4lk
zM(Glp^`Tr_%kQ6`yR7sUW*!cyOL>}4)wq%6Jr&jiPev=mJho25x3NrG^T2mx_Scv*
zRk_;5fDN^rCh%-$%fG$?wAiSI?Tpp!)R;EIJpKa>1p~5*^@^ds{K*R2sv4kUFQCW#
zQ9%1=%1MGcla=;i&kFq?incA1L8GAwB0la!d9NluF{{%?F+vSfwF_6O%Pe~GvgIqQ
zmX${ra5W`6zWwa5QdU#c@MLBFKY7WV)Ibk=$9(o#`{9W(D4X3Rn?a4MdIf)x8hxp8
zf_-J~WV%eG;8d+g6<gKM0YUf8)u}Na;YW@f3Qc+njEbL)qZpJ*LrEB=8&0=t%|*}9
z&zc<@&zGZrMCZLy*2u@K)JPjK5gO4OCz@#L9G6i24=M!<C-g4b0hOeoO8KMs`PX`$
z<QEhO?y~(6J>-OScGCLc_EfDF(y(4JS|MHPscLM>$#HL@z+n5k`=+KH5fvN_M<&NG
zBfknQw$yz$HcO^a)%D&0qezI;H&#T*FHflgr0UB)zhnfI%!rK$EURi4d5|(vJf%_c
zQxe<>_Sr0vImiCz+9IlMw^tnIkr*Mx%S?pkOt-<5tENkjza^&A@;$?J;JGY+?;Pc!
z&WEogkQ1Clz?TXUijU=+_vhZ<cy)E2`&@L*T%1M(@9gj!meO~DZ_gH(g>ogT*MB@C
zv>!qmiB7st+AAqsWUR!guFDe~Y5(XGz5F@twUWextUs9OvL4J@${%)Ftn=D6RAE3e
zS!fAzxl8`916eN4MQwUxyT*g;k$gHHImVT@bP+j?eUQ^dpQVA=(}(6&flx7WUc1lr
z$IA)bhW^J(zURbG#ZASje|Y{}d9={Y@c}r}5CFMmi)N{15b@jeK#qs5@OgbU3XQ{~
zg-FL2L#i@M4kc}*PQti4=JA0RC6l$N2HR(E?ii$w^iulyLo_pY$&*%m;@pc834bHq
z8yW^FNj2hcPSWONsQvcfjt$4s-LQ{DIW>W#-y(ko0GY5t_F@fF!R*y^&zCK+m9&3q
z0K{T$=J=1JS=@Zs8y7z+Xypo(uk#R02Z`Osq4q_Ko4uiO*G_wQufbv^<?(4%+Zye!
z(t-EGI0^Y51-9NlyQ6clfozOX)imb9;UV)}j<ck#Nr<X?F<lhkbZf0^-AE`=_-cq^
zyf<72?FnK<55`PGc#zT>kT-zJ9@GAOJ2F~umY`ObZJ~4ByaC-p(9IS)lKSo&cUtQ8
zrR~^44x9T0Z(m!0zI{VUw?4%iN9(<I`NiA-jD?oq#y16y`cHd;eNG)zWaDO0N-yD)
zu}1!Jdke6Yz<zj^#e__IB(;$Rsjdt+KL<;^`CX~<m*N?YGYzZ>JhtvIt&8r34B~HM
z*w>6LXQh@t5#@vi)^ww^Hh#tj6GFZ)jB6a%>pJ!ih1if2*wmb7Ou50lhk1OL&vh)5
zG}pZvf$?TZ^@NWfXYPO;_x80!Pgc_fz0b;cx3$l3pujrY1Jp#;R%ZI_jYraAY%W5!
zO5Za$ULDT{6YBc=H=G~$1yrgl3$!JBuQTfv8;gfVQF2?UsNAn#N%r}U3Z5=CR~Que
z671u8+UH*@$yRMv6Ht@%;w99ASDiJ^z@>+0q|6ekPCrpO=}?<;H{ZQtcHcBYpj{bq
zyg2K%7Z7Xcox8s_*ipW0>KawQ)5Y5CH_BsM<Egh9mEcg*`I?a0;mUS5Eq2fQgiyjl
zTjhg_T!jjC%jUH}W2LHva_MmLxEY?#-M3kKj`hRJ!Wc=P`B27}eu5s`c7(#{z+>|0
z>km9r)*dX9PLcXWiZ&ch`&4ce)n2=EQG+ll$Xv-EF&$jTt^{0C0)x-^AXL;|&%Pph
z3NXVGoj9Rfo{^&3?V7ujB>TZNz4o(BL2jQe-9rVh9sk~FfnspnA$tL(!!q{T04{7K
z2n3SUcnHsVd4I7;UR5$0@vVn%Mt|Na@4ZwxjW3bMw3p>#>YE4nhSYox`ajs}yV+|c
zHTrK=p1!ky=Dm<EPw|xmx4lJqTpX{+Xw1FKS>pCI0OuQTE)HZW1S9Gn+fP=8TNQT5
zauWuKdkDgFoXIvxP8+R8s~wu}eQH$CRwkx)yr%@ycaqAG3L_<+5@0O7yf}@$m%jE4
zuf~0g-0NW665wgWg^w4bn!Qdvm7&0H^iqjvyx2#60K7488h%=&>8<wSX<);CKsB7Q
z?*v>P1`tmPK|Hrn{WxpQeSjZzGi!Nuvsq2TMlO8SaEEwGaKPi{&eo6AXJ#E#9`AD0
zsHy=;+QDpSWR3)RVl4}*+5e)^%3t(+PaRV!DcX<5LG~a%r3K(nlKx>v@6xBpkL~7V
z6ezoz)AC-98U={tFtr!;t$`&gu%w6bwCnO=+_kQ2cXdk5<S@JecMGkOo!alyS3(o{
zInz!HWrhTGO=@+?g7Fi!yQwsaeW@!Xubp}>4RnOwx&QRh-JU+yi2aSxNa2g41^OKf
zwFzrH5G})V(hT)OlLXyt&=4fZ8(wJTO|LTgB3+meh)!>7W`ob`)7FmL7?7XymdZS%
zU@w5LU`MD;$!(UlHlfu}g!*QQu#u?c%!34f!1)`Wrbj|_>%7pj&BV9UnJxtSp#W+i
zR#n))8}0E%(yzUf49L=5CG2(z1n%NkKTL_K9~>I4`&kQ*iq??BZ|BIX=;cmjxPcl-
z680*c5vrBstULW#+oYgi<My~@y!N#w=hF(;n6Ygy&)8aOg~FsPpR(%)3k^|`B%0=Z
zt{v$q6hCYG&Wm_$)^640E0~ox6WUbYs%deo!$_YVuOO5Pv$NnON#5(Z6r0xKRTYkP
z2NSlzr_HA#b0QER9-nSi&A<;~^rc7DqL?D|NTD8?;~-W1O9zj}(J#7>w@y)wGpoL*
zhbU^-vRWB#9>;0nRo=*p%}B`Wy~u#G^eN|Vwq^^>p?@yR`XEB?Rj))mOf^4^u40&}
z7Q0_jy?qC8q1HIZazb~n1L)S3_{eX()ZA|v&LeL6uwIBib%WeHWD^*}opzFoIl=&@
z=Zjz{0+gXuad(pS^qK3srPt9FtMy~`CJS<=YQgwvGc5g#+g9!M)(@Scg2O2V23(O2
zgML4iHiWf2t&kddtP>F;2ig=oHX15&`*Q3Q4Af+6ovZ_f;^9jDJ8cg6-(5tFklK!(
z4Hspx57$)<4I#-x$Y7YKU!65-y57EO%AFO(=H#Ws76;*VS$cP$t3fr)ty8?rNdV^x
zEwbRsU*smj@4isPCAyIwh%_HlijG*bq1Wd$@A~vA8na!Cx}&=YU`31dYwX0+r3dX4
zj-N^2AxAJi`|IF3=!Q#-XJeJhNh=*<YP(bKiQmZS{-}|oVf8+#Sb*?qO7Ld&N-~$*
z*7fNAyK$c|!LU_diEx;TUrX<_56WOY*^t-GXmYhbJ-CzK5cBSJY@uLhcimEq-o)#b
zjSlTP&k`w-uI@?JXv59}EgiSiNzn^T<c@n}x2;1#%!E0)%Sj+h9DM>}!5R0dIFS6j
z!opARb7BR*;4Q%=f4&^(SjZ*dn>S@pP#_><ztzrf_@WuFxf>#_&i0yOi<~Mw_x>bB
zebz{qaaunk!uKvl&&uY&^~tvCR5Sg-eyF5P$-6|CUV*krBcH7b;{=u$#MFH{x^SxW
z`=>Q5@x+?jCVbI=W8yfHe1OQwAe}P)nTzO%vrN9PN&wg+wZa-ns7Vw{5rvZc02?J;
zeEoyXWs)$B@$eiRYQ-lzCD}I7w-~67{n|yMV;s>fHJ8@cyq{zZcI34y--1suKIrp%
zmow6{r>CMk@+Ls#-X*pQz^(tPN0hA-6Jl%J{lcQp%1!4fwV9C=ufhMo|7gxwYyUH+
zY1j95)>QI0_b~EmZY=TZ7k3!;tvb_Fb}~X4n$4)(qGI{8`c@C{bvNJnAkD0W7p1|q
zV|+21{qLq2PlVl{zIWV(9{>wS5gOBDIYI5U_<`7s`nn@?p@g=o)Zh%i>*|jb=fX&-
zC+X9X`C4l<&`TrhW75VPM*nGlvh@6-8WwjK{p3@(J)Q0N)upP$y)0*=H}L6-)=P@z
z8^7~suMAUiW6rC^3KLy;C+*7G@`?-g<GR*9`i^H5XUzm$ij+QIs>(9oC)I46RMs5a
z?GvSbT()Qqm)U`D<Rcr_qgR*W>l-Dw)@DW->-RYEQZE_Bu!+S|Sa5DqEgD6?Eat$u
zkr5quxxJdr-nXFfK|tQ1Kp;<YX!KQ_Z=s9zaMJgWS~&OhwT<|vJ6ujOvep@lH2lYk
zpJ%!8$67OqHIT+4QsOX&wh{xoT`pQ~yGwq*$4&Fz0LO5RGR0wrf@j(}t!5Oqru;O_
zZ%ek($jkKu+Oz!K2bp=i@(hjE+7-Xs19^{(UQCmDj4q}Q>ke3#=r+XK`mR0k<Mp-A
zRw~^<;97=Z;iv+fY5HAZpw^)dH=Z@Up8H~3j4i(`fmCQlvMNA7wae14XSw(A{#4Db
za$STJ1OV;(f&oW!Z;UO@uR!}MWE*eY0-0$Gk&eom)?huHWEmlY?;oNS#RUW&0ctP=
zK>Iopn_i4&5YHFW*<dqAo&x_itxMDp%aT&TQ({?RK{-13ZdG?$X=AQ?_k9cf>|x~S
z7Kc(*Tlp1ssb)|p9Hxp~w@uu3jduOS5(AlbE;mVRVblj;Hdc@cz>u_ogvGIf9^sew
zm(;aX7h32a4t}ZM(!kzIS;=<`YpbZ?@GvNJ(#iHY+-|OUPbK8et*H^EUAHOti9s%I
z2G43BL+)brP{}BZv>RtCH79xGL_b%4NwSCZQgQT~9&-naze+(DJAE$@Z1d=VV~(Ir
z-F0l-yANj-e!8l;9{zBauP<-y<;|OQ9l0yTw!dprC{V3iHnm&9tt_G`yDR9n4s!kK
z1FL&;0F|sB5QG}Non~A<GC68MH*mOQ;4gI{{7<Z-lULSxkcvh9W=U(_i79x49aaaY
zo=JO;LO6alJ}Jaz0$_HA2B1v&V*!4k`dBR`>|l5D?t{!6<h^uwzpv(FJ?j>EAI*2V
zHjk}HdfYjFyvoqDpD1VE1aL*s16yked8&#}%Qmm;qQQBs4z82ZRCL^(9g;Q}Z*@Mm
zxYYYowxIl>LW2}He*5j8)yLjYk+_0SgE%6+BV(O-d9u7e3i<d(JK5CgGQJpL(tl-f
zTMdBm{FLh?0(i7XKh%yXbMK|0wR$bV&%lre#RWqe97l?DsQ5#)aD2{2RRv7m%~qx{
z*ON<AIU#3YSQh-{n(jhLGD}_s4`fJhe<Wu>Ck1YudeDd2_LeF#>{gXxlZ`hpN<r-I
z1ve%yRD})g-kqELl)ZLuIl9%#O797QQDlfsZqM)nd7JLc_T-r=idh4X6J~3?oJY@R
zq^j_Ypzao{uxmjkxpT30VI)C?U&AyPEP7MGI}z!!JJKNlBuK|M9>6x7zAiSNdT4&x
znuDW2P$u8CRJ>oOS|T!FdpSLSL;_S5MFm!AJ;*@gzn|GXxg7!voYpMRe8h5$l0cLX
znHe0pa3~bI^(#hRs{btb+~A;xj%V?!@O4(_*1+Ct&Zy4AEm93}4}SoYc{VxP+Dhtu
zLDp6cU*eqfL(t<pI=ySh7lCT*?2bX_mQR^{o%?_^5t#Ka&+zM<v7_OPoE6SOlUDPA
z4cDwwZYX%G;9=kvb%W|%=y*>{-QY9wwfYX$*UWY1yARt8@g0bnasQ^$G9tx{!24Pu
z9vE+rqZv>k=()mlo3K*<6KuTQ?el8*gPY%Kvvf{~UX_83SS(oTcszXmFhoFHyOAlz
z7rpM>_NWcFEfZb@FJ?epqek-c2jK}zc|~fx@?wgDKB$qvM74;p11V*;zb~Z$C?3ZO
zAq7la3uErjv@!7ycRE=$kdMBk7&f12IH|piBtdR^Xh~PG*3o-^n!hD=PcKdLS7GEY
zk|Qn9gX>f>wh9axL8-RUghX~I#x52jrwgYZ-o)F^H9ae`N_u75wn22<Wo6W+5)xUs
zAF9{E)WTh<)%Ni3@t5I-XWK#~K4NyhEYnDnGBYPzwqq|Df6EjU6<stOl&m<?Jnu`1
zNJT{}BJI302U!t@H~0H+e{qXfF+HbJETDc@Ii%{suYg~(U;jgIr)7OWx~v630qk3V
z>Me~7mz@fQdZDS>;T5L3gsIxezq!*uu}mP-ZU|$pGvQXoIEKtXo|33>CB2HIS8C%Q
z@N@l%0K0+X-V1I#@=eT#((HsvLQ<^={-brKhd-1>kLTx&ShH9iH0-j!mAf`7uU3MR
zKM{(2VF>Af4yF28KQf}Vem$=cLZ~n%`7b;07>pKLOiV#z;JaW#rUR8lrl*{@f5>^!
zPmr-gHFtDQAh#aA14`t+wvq+RD9nX5pGuZIhcAFHbm;!7?KQ#B$nkN_hdyRfRznTS
zteo_3iy8`yhr#ndfMh>2%U4;cEUne_){8hneay!e#`hxm{IYWh@xM!N_BIe^<*xoS
z8y(Y42x4T3XDX#X<{uoDa4z*1foLXzKJGk9zo#^sgnHwhnEeGdhv)?KzXAHAJhQAO
zUctf*kUH2Dbg5AeGt*noBRSY!k9a7ENoMIr!CiIrxD0(0SkiaF7jb)iSpm}LW%7R+
z4uBoOStMg>X8~BB{4w1RJ!=L@bx>D)y&c<KvoZYW#hoL|L8`N|b%f64=Ib>=9Ppdp
z$!alo0`9>zAHGc)d{j)GOiBJr2e~rQ&St&d9l&7XN-Hgd_0NBcd2f^j#6Y!<EEE3b
zx#dXF75DfOt(;OSk2A;sIw_JMF?W^1<h`FpB?A0QIOm*e6R~6o3IlPlMyMD+`fHw{
zcbRP0K1+uR8pp3}kSK4<2D^C5o7XYi?uvuco>-pphY$v6=Xh$DRJB~P{bBFKXzt+D
zM|UW}@54F3maG#jvgQPPg9_WjUp5UScIC5>h4(qEJWk%s6SO^U3jq6p{&1E=IcZ{+
zTLB+hM*a7ShFQUSOtN4C&(V!Td}`hI>}w#fYsO2bis)LCsfx(}P!w2{g<bV%9p|q!
z)3cP2N<<MFzk@Qw<J7yGGS>!&%;u0U?;z$AY76`w3TDi#d$a}%Oww4K1eJjuJhRfG
z|I)1^1Vgb+{rE<TKN%sysbkO&NE%IpCcT$_%YNCvc+1t?yJi~`@tCH=XuNBz4`4^0
zy?o$jpxb{3Un<?yNOY(QctMWVDo}tHI)v#Jc~;fFztLxi!C(c;QPi655mEHyW#DdR
zp&<-?UH>inEvaANTsE;OqU+?!1BPpzDT$LIh~_^<s6iT5Jp`++l~r0KLT+)6x89M%
zkea^EXq`oat^hqym0Xj16wAy|6sEBMEd%ditH80}q+iT&uM;q^aW7jwTU!Qce=hUd
zxkt5ZBrZ(Ozi$!IaN;!F>b_#%fM1uO>3BK$#HOjPlcH90#9ks|fEX!vqFS~QSnpQT
zU^nL427|kRw*F31g7Jq6{Yh9$#|?HB;k<hpXv}AT0{3Lis<QCTx5#>QX=+}SCX<z@
z^0x0QV8H5_6qAN|k+z#~SM6ZUwnx8A*eowF67Q!GK({gO*X5y|UP5{!K2c&fUYlW#
zQ0$%i=>19EmD^~&D;7daFOw2+$K~_9)D;i^ze*F-a8EXvJXq@5)=1i{@ewb?<Qtlf
z1sC-?MeWogR<G8&pfC=!9e!s(Lr+=5Hw7KjdkP9rQNfVm{I1Ptz=1rUWohGpJHZ23
z1({`YNyZf49CZ1w41pic$!zD+9VxDj+!Vcgoe7#!Z+MnHuh+)iwhuS_%e@VgqC>GE
z;!TYBN3Vh?z?00fb3~r%ZC_|8rR1!bi!`?(!oi?B*wlPJS7jlDlV4Q&RsVRw6bWnu
z){B=djJTd2aZn<3C@_>j(1Rd4jZY}>@f@>Si~PK)e?7gQaZ7|)`!F$@eea6>t26g|
zH4l<<`CA@gf=#^|@V@2x1&3B$4*Djx8z=*P6DYml0*$*;ANjd90Hv}GeJ16S6?=<V
zZG-p-$vt=rV-6>$%J0e+3cNZV6TY^3fIlMyL(rm8G2i;v%*=K|wGYh}<Dvu9pljx+
zQu+|RlvKD!!8}z6hugv17QJ+xb;aKu0HE2q%?+Ll=fp=;)aAYeievmVzsKPth!gE|
z!<ENst#tIb{EF&~${MdKV0tJVR5w$86Q^eH#jYoX!I(N&vY3iA?Fwmt(woSRJ<&T6
z%1D!rit=oxDuBBGz?c~;#s@z96W;M>q<^_l*=Io(EW1~{;xx~;!D21T(MwW^_^e$G
z2UNuO<we8{6nrtxkckYe`aY8fV7`8MBeO2+Z|d?P8NdFLj~q~UR4`8?#fHaRkLTN?
zhT`4+IGs%#J8+6792t2>VBW=raY{I4dZ7MTqb~EP?By#l-5WogqcXaOwqBjm$_awr
z<cb;Zd}TZ>m4q@Gsyxm$_ApeY4jR=Q`hZV;Ajaa1?mFZ&HhUM?r*bIz?#-&uQsjLX
zEo*<;6XjHP&0d~8jN&8QQYB;K%A^;hiAWBfVI3z~(nMQxC<cG)D7Zxw^_At6;eeo`
z2|s#!%}snkDu)9c5{ADhjX=KzD7=i^KxEvS>A131f!v^rtYk^OnsZ)D$AeL&R_t#i
zSJ>c+<CBu8i@?HpPWD9?&sTkJYB}R04uo@^E2Phm;mQ)_$bbs$wMEJ+h|Dy*ar>l@
zp$cxA92YYz>MF4SdIp}3R=!gBM+UJli-8;gBA{+fr^#{wlEN=jLeuMIspekh_LPiy
z8}`KdVN}uMm6u}V0N<t<941K&oH#Xt8g27A=620;k)@nkvyk)IMilhj)Hjza=hctn
zn#8F4`3%S!5#vxruQh^&=xKLveIK-BLS!6?Vc;eazdZ9JeW0}M6^RShuPS`1F5m(R
z^yW7q^|CWV#^q<XFJf~Uzj*;g^G5)u@fsDcLW@hpWYjd;wORBf^=!(s0u;nrUWhO;
zW=57=+5OvEpw|0K)W?Tc)Mv_vodzYKByaecg?-O@HHIBfTh>*nehY>dcwK0dreGfO
z_OX+EMDqRX=Jq|`zeFVB0jWKp1i$=?A_%y{@aTt()2FoS<UFLt_C4ezAF+YMk2Emg
zE~XNI9Ldma8vVZKM$@PM9qRV{>Fg2UlCa4>sEg510e2b{-{fm3-=E~QHKv~NRm41C
zg&`|)8#zj#?Mh{}MwjiRj}0`>bjJ{`4wJ1hNPW|cetah^tb6)b{}(_R89SkmDzniJ
z-bL24leEoeQ30#(MADFjUUl=5A+UM58=qK;yGORR=3EZT8mD(+!jBv#q)*Rw(GBi9
z{*x10j3UOi1hk}6qBRTg6(J|}4c{B$xx2T$w<u1%M$%Rlge`&qMck_XMp(a_RKLH}
zH&XE2V6tK*%YC+YYLjr%wfI?)XJ-sjH-XgBYy3U-LHmm6xC=E@Hf%@jTgFYhJUWAd
z)!1@6i;f^|Juaalm#*HO^=wFmV?eU$?e(G3t=fu`uO3q^uMK}pA2co=m)qdV-8X(s
z`MGEpxL`84%wFOc5N}HTF3tod%-QAOx$Ux1DyL3S_bj$t;A_tWw{k+M+K&wUqxiTg
z76$tLoN=coBO~7<HWELUB?_C0qwBeM-mJNw<{M4!_xW2C#t&6t>z&R-WB?a%q`00T
z_fBUnNE3AuEcSytAXAfLGlbiwB7&9WFI(Q$^Ljg@BmI@cQ{8N)#vTa!6rZovjBX0M
z1>M@LkE=i3=Ww5UM)&g2dn3mq%++pq%s)}E@4<tE`XryR2=}H;j<NWcI6kO}^vx`?
zhT>6Cy+-!~9>z_J<7xr-i(qOzuf?_F+*3!()59&BEnV(%xk0kA0Hd=O2$ks9%FQ1?
zd-;uhCNsRA2|ZVVPI`3o@*c#_Yz1)mf$L%~%`2M31y6x%Q5F>vCeRCf^UL6t(Gkj+
zv1cEAcnQAi&F9<3^%!XEt@b7G<x%*e_9pX^Dy%f1Y$;`1^D?3(%r2y--IbHP2@?Wu
zwyg{%cq`&|V<*Ds7$?ph!z)^3Tf0sTZ%%)|lz0f-{{Y24ya*Yq^~k0fL<J<-?r$tk
zW<EQabfIGy%Zv=aRr91s&;gc^8c;sHWXLKyDp+A{Qk7I$b<S(~!Qgx^yO|%_XFNkC
zIH|<A^}y8<;c<Cg3+diqkbCI8#0`t?NPJ0YJ5&0BCr2;8Rj%Alzi(&P8D8bpRt7z+
zbl=@YOfqS**ZW*HG@?x37#x>+*%n;x+$%8kJ3Du6(@1DEGspONeXPK_?Y3OSdXc7`
zRDmBViW#fm@sk$4Ji6<c93!)u(tgeH-q@20D}9fzO3i(Mwo8lFb@G!CmQ#~_Wo1KS
zm?N|k?@7n~wo`=Pw>w%X*CM&L?^)d57%mOF9GY=}SQ84(ufq!522b2a<APsGhg!;<
zSyAcoRwija+?UVF-f4g_<Z)hwd6xLOFL!v@+uRNGCgGa41raPS>jkUu+pQXMF%i@O
zqHNzzQ@V{<AnnL9vhj!Kwb%cttWUeIG_89~?N-~7HAuK>FRB=4NhVV2Ki+bE`hhK9
z@)w^?hb732V)WRv>vj%?s(u^YMlb#9_g#J*B#eKkwYP>@Q}%sx8Vuf=9^UD!Sy{5^
z3LB?+k*T~RBBfg9$(!i3-plE3uhjTP_-0TzZm)prb-ns5<MgfD>m$=v364iAb3%Ee
z`9GXpoC`w8sT&gcyOWPsJMNwu?u;?g`}?nd#kjYX)Z&JdO`mFJYbARHcMgDU?dlrR
zg@rYjeKIB6&)z=;CeEB5i3;-gS!a)qP45#tWPekJ@|&49gyn`k=+xu&aTw|H&GJr~
z@Y3SQUR(-T$u~-`y%sr5yA}}Q-B!wwE7L*RJP93l6cTcJft)YU<+C?7f+lO8aY2dR
z60a7nG$i8KTC}G;I${Rk>^=UevO`q#jzyBAr+-60`CN)-l(%^B*9{n@1>?+#@)HD&
zd%$J#m`1(nMbbj9)A>Z6@kP;`=HOQU?g0V0()8K9rO&f)kfGO-54{%0&pw3_?iK?r
zd00eXqz;BXbl@ZG9*3GoYx1OfIR`^8z=@lvXM0%J#*D?pCggKIspj<&?}?$%t+ud#
zkGecRWb5I6JkDCB)%ZmAh8@jp6o^{oOkH)zx1{;b*=M34U^{yB6T?p!Nt~IU_V5PH
z@UZVr^L5~I^%2gP@6N-QFZ^n8O`l|+HH@{N^YU3WzneHxlk1p1viNaYiHb1Te0m#u
z*Xc|~e?JdXe^(R#k`CO1HcBUtMRWM<mpA+Imie)J_mXmOkgqQn#_QSDV6+p)f(zBi
zIw=dStB-3MpQIzbVA9;jMlsXy2}1+QlFnNBH0ze3W77W5<f!SH;%J^4!pIw~HxhGK
z!oZPjb_>?dxZ8uzBVdU+#`a1N-_JA7hCjGCpPQjb=Cz(~au6_w3T+)c=k=Y!9Wmgh
zn|h<s#^4CvM>`ynu?uL=81%eDMQf+N5Iv0fN9{EM_mE@~rRLd4IHD3;Wa4^ZLbQNA
zBEv<cahq3Vfw_KTAuy>Xo^U!s30QIa>sA&u9y3082aeYrsBeu$Y;*ZdtNC0+g_18$
zK?C9($Q6~>M~bYUh+8AK;0$bqhI+<a<wo3*g+dkY8j?CT?^+=T>ouI@-ILs{dPnp)
zp=$c+R#fq7s_)A|XonX@qJuCIOlaoerRj^}y8d+C>eeVI?d^ty5uW7J!w#eCj_STe
z)6xDr4U?o$-l@`O1ce0k46D)k-w)-Tq=n^jfqO{IOVHkC#L%n_qF-dE@tNIP*Kl)d
zf>3$Zc=I_QvKY&w;lQn!vcE8UJJb9Feow4Yje-Yd&En^aHq?3Fj}Pu0_y5`IBp@sf
zStk$J3C6giC}wm~4jYZx>;b+RH8SqSdeBLAQ^$4O!j$CEF{5SBipxyG%!Gh0ZzpRd
z<W^Iys;my<i`w2XQ<IS-FHdEDcl#dG7WK#SA02QmA>QXAq;(#4bqp~lPrVzK%d$JU
zNGC{8mnm^e|JYB@N0Vn&C@L)ZIS5&W+=Vuj$~8XN+H|~b@96K5O3$h_IfHs&_>&73
zhl-}5yNFc_+CNWVW|Ll~uA97hnCV*=g@i2!+%D3u#}qWfivl^Is`r?0IdHfaTNIoY
z>~1Lsw8pH(eCQv_5U0{Bx6|C~cuJkXQ$TXog2wPsWKHPWbOuywWWS?W)*R`JqK3rv
z`RVZIPWcDk2`Ed;HgU5F`?xQIWW+Yo+cm?pwSA3VAOdcNc$Ro}TNKvm>uI+izG+-O
z{j^^bn_h>C_KYqo@I6vyR>-mv`g2``M<-#RwvtsoI@iqm!O~QKU&VzDy?LhCKS}#Z
za8t0xQ}s(?0Hh5W1s5=j+XYKmGj5p*cITU&0Cnev%xus~dT1bLd}mqjXSyDP@}JVS
z{r^upSN;zL`o~v^a>P0+VM$+#+30Yk9Bt7mXYOm*$`q5vk$W7IP1`uy%@yNJhlnA^
zgfZ$RIYtaJXpnm_!_Z(1Gh?4A_O*Y)_lNyuUa$G#^UP;H&+~kq_vd;CGcFxDBzXh{
zN$uJ|OH8CH4SsT01{Tkvn(YPchiN;iZo;O#v4iFpZ~Q?1dH+a~NyXcch}9d-<Yu=p
zseJ-uvKTQfUjblsaCCek#*XUlkW(3Lj6=%hh{|;EM823NdAOxid>pGcO5KejiQ06F
zaaEth?!I32cGlob<Kef5<)YGOqXJnk*LWUdaQGvBO+jN1DHy7%$tQQv7pby?W<Z4A
zpRj(7ZZx^$_faBs1%h6IKWg4|o&-^X93QvGd)QbBqqL>oSbDEyPD}R`bqu<G0*<M&
zFO~@w@#V952pvR#BE>;bI#!{^!QE~myH(j`htrG+VF&B&b;Um?Ks|IFsAsVWZ<SO2
z^Y)GvhtYbOXiOX0Y0Vl}NgM*)Q>89?LDng!z`S-Ce?<O;8XuH<3-StcjIONJWj<Y&
z0>)pigQo67rHYojo3?8S>}m}`S=6Pu*Lf1lhc6{}nS~kI{<d-J_lxn5lx+yWW>XuB
z8e1(!Km#4n&EhDXjN3{Bf*>eW3luz%O8;U<`4@cOVkbd;1!C!prjo5H6$pF!04aux
zrT$1fSmImDI{(*Xx8|R}R}oF}v%u(f$(|O<)^`vEweJUF1PSa`a|jT&=W`+GjD^W>
zuM=y+24Y0kn8Q{z5^R_+Fvlx31;qL9Rag*%9Q6cZY$H6{<n!)NFVk*V%6~G)d`)*C
z4P`5wDe}~#4b%m=)R!Bc7c{8ym2s!xhqA>^AGX>lAgC+T`o({zF&MC3-#tm48c^Tx
z`-USubbYRd*VgL7-<{m>?~tiqVt4s1eeIhKI!U|a3b5I210RWv{_{$MsizZq%Q%_#
zqc^k+NFpLCt1oC!NPsJVzUB8j2m43ZbNlP*OMj_COWGx(>TuQa)<eT18`FAjHG!56
zBkcrm?A5?O?4A6@)&gjAbMOsx(VHRL@xqAHeLH3aj=_c1`i&53#Rn_z59ezw+94uC
zn{Z2@Jxgk)&fG3?ZHP-x3FxJNE$G8uCl^WL!5DUr88F9ZcMTD=iMaq`1x~@E=0n>=
zRb6>TdB=&l94~xdS##^-M^`!acA(igdKu59WAb1WwgmF#U%y%gO>%)&vp}m;T#=w?
zIrVa8`uwf8<ar_;s%bCoH8pp4Jc!ZTki_MV8j%aHU4|S@9Do-d`4$+!MMNymtHky1
zRV=U7DR!c$Szr*Gk+Tqa?nGjsC$ob{#HD<29tk2VpW!C|wu;+{!|@D1Q|797t+tjk
z<_kgz5KRiqFKfU%W&uG7_VE)<FB8*fYu1{3KxK{&#>FjAn0p!UoskF_BbuAwu}1nM
z@Sze$xC^dHdhWx`f5axfG7TYnG{tq<YWJ5UQ<~?<ojB&g=JC3=)!E8SF0ClX?@K}+
zPPU|9h$1mmi^;)%sd=ZPXdIlI#AUO_xm<H^ruvdvG&pHCd=IiRRp7rnRkJ+pLu>f@
z9Zq6T=pTcwjaCE?gQty#dROmytPlK3A-Ks$6)=2HO>2p3ATQ3>JQE)*yq~kyh?}Qo
zCcI+o!#Eo@wAh7X9a6CmtLfyj%;A#u>3J;DkA+_<ReX6Zd}?CDllqw|4Re*Yghp6R
z{1B7VpkMUVBDdl#&)|)dKV1PU00}qBv6pL@uWFdbolc<7y)gb3-lCJ0@R^%^&iMeG
z)%1tt(DN=XSJx&u4yfZAq&e>-F#)3D6Gcw-j&!xCcwd)|^rW9dhcDzsh0eUHz|~Vg
zj7h5IDHlCnrykdJP|F+OhJzHx7`U5qJotq@KMwqwAC(Y4SCc3x9itl&5GPJ_Dy@#w
zjCmPG+k)N{P2V3^(wjT|FhVKuYF*9;ELson<`K;s=`2@Mw%MuL@dws%OGqA5yesPV
zMS9J>6-u#(P4}|M?cVs{`Joc1`U0<U{iXH1bv;5h+Y@waacqxyZH8a9r#}zuDtVvG
zvGEhYPI~e(1jscmJLL_K+3aAfl>s+)WVI<ZJf!=u(*8y+3fCf(f<o&p3JM6Gw2D3F
zE*ENky5N@Nnq_50Lo@b#32=~;$6`lZXNNO_eW@-J>!z0vH7vFX!izMSEUfWoUZKc6
zT$mZyB~z9<<jV{V-e;&}ITCCH=e-{HevJ1HnPVg*Kr*|zmm-k>bzi0^RNYDsfj(R4
z_@E(*WDyxPv|8(89gg${6Y3`1q+Fc!0Gmu|h^ibn)-PU<UgZ-W$%1G(3ahBvbrq%b
zyhFS*ngvzVo({0BmEx(!bmy&8{sGFB8ZXzivec|Ry&^pptLE#kWhm#eS*Q>y+*5c$
zn;2R81X;+=U^%-Dj}sK^g0GxS_RN`p(IRu#{qfag{g0K}<WMwEKWwVEE$#WDH|{J<
z%NaV$DnAS+isFHH=I35a)At~Cd}@*u^oBbPiXKKSgE7@T2P@i+tZ2iXFUsCkUCn5S
z`0QobF+hDnMfdkD-zFW?^@DQKpSd=XS~|ns{U@zL{2Ld}nf8VrrB%02x*=mv=B_yt
zjTb<tR?izpnoh*mdsIlucmSte;eHfZbZGsn6WJ^;2W~a+5!T^WZ0ru(rKRanp?2V5
zeYEu{&oJXG$5Qq3)1pzcrLb<xcN6jbbx)9&@R>>=#dS<@#=ts(c;V2G1J0v8Ek-q_
zM5jSWhN3T3p)M|*Hl2*YRZ9!4ROywS?dUYT)iP=+P9uxd4HjPR79-ISh~Y%FJVksc
z!|L6`Y9><~R{4OP=w&_C;Al%YCnU>ZH2)?2@knfT%zLYIa2G~d>HKOhi<}6{6K^Cu
zx#Ft3q#y0WeqD-0UH*wXV8%T@ykHQN&qfR6f=mx2EV|<-Hy5T^a9mff@!nZGmBWo>
z;aQ2tC<=m~EvekOEc`+4AWjaWkz!5f@yV|i`QBwy=gIFNwKQ6qChWd`m%aZDPGn^B
z;XBu8SPat}jmI#RSOg_J!`v_r7j`d+!;q8@CQ6=hmPRd*#uvKLFBo?UTu(GM(84z6
zlCSAR>VM&#y2E{nc?~VGAUxZ57FKAzx7JxUqbX!?$f%Zm;8>9M+uJ>gNyw{LOx1sC
z!Ok4O6Dnf09!?SExvw0Tg{$bPxNKu=00HaU_KFxUMhaGtvY4*32@OFj=Fyr?#78kp
zh9l;nx}3T~KP_$mlU08|W8lR}_APgn8UQIC?`lV!+#S>gsAc3-NFcL2{+b@#6j6OE
zjoi6hed6-vk)A4KUzQXN{$YX2@T{yM;W0r?_9JlFsruib-ry}m7oRFT4Q*isT1Kx;
z0*;x){u_lqGs6(r5_1?&*up2TV5S1!hd+MB`K-N4o#AT5N%s4NyhJL%Isco;J}tCk
zNzyA3c2=fsCrWbn+FH%m9*!!67&+J;8N+wKh+5&K@Ct)8i<n4!vXY%luA`mus+xee
z_ZURNJ{9bZY51Z6pguVtTgLQv7runrhBVi+!JSF0f0_+)w%;+1<W`a*bqBwA#0Mj-
z{CQx$+c~t2R+JUMGujlOcy!mc)Sm(Thjrmm;4j<w$wZ%@X`L{z5&1`Ig5OU*q&oHc
zHdsRW15_#25}YryEw!!JcU`{qasT)6pK<;>7XLRF!sMnPPl<gz+Y)KZy2_`HHaE4p
KQ1*-0-M;}*C#gyR

diff --git a/site/index.html b/site/index.html
index 87995cbabed48..ec80075c59200 100644
--- a/site/index.html
+++ b/site/index.html
@@ -1,74 +1,78 @@
 ---
 layout: default
 ---
-    <div class="container">
-      <div class="jumbotron">
+<div class="container">
+    <div class="jumbotron">
         <h1>Apache Arrow</h1>
-        <p class="lead">Powering Columnar In-Memory Analytics</p>
+        <p class="lead">A cross-language development platform for in-memory data</p>
         <p>
-          <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
-          <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.8.0 Release - December 18, 2017)</a>
+            <a class="btn btn-lg btn-success" style="white-space: normal;" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
+            <a class="btn btn-lg btn-primary" style="white-space: normal;" href="{{ site.baseurl }}/install/" role="button">Install ({{site.data.versions['current'].number}} Release - {{site.data.versions['current'].date}})</a>
         </p>
-      </div>
-      <h4><a href="{{ site.baseurl }}/blog/"><strong>See Latest News</strong></a></h4>
-      <div class="row">
+    </div>
+    <div class="row">
+        <div class="col-xs-12">
+            <h4>
+                <a href="{{ site.baseurl }}/blog/"><strong>See Latest News</strong></a>
+            </h4>
+        </div>
+    </div>
+    <div class="row">
+        <div class="col-xs-12">
+            <p>Apache Arrow is a cross-language development platform for in-memory data. It specifies a standardized language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic operations on modern hardware. It also provides computational libraries and zero-copy streaming messaging and interprocess communication. Languages currently supported include C, C++, Java, JavaScript, Python, and Ruby.</p>
+        </div>
+    </div>
+    <div class="row">
         <div class="col-lg-4">
-          <h2>Fast</h2>
-          <p>Apache Arrow&#8482; enables execution engines to take advantage of
- the latest SIMD (Single input multiple data) operations included in modern
- processors, for native vectorized optimization of analytical data
- processing. Columnar layout is optimized for data locality for better
- performance on modern hardware like CPUs and GPUs.</p>
-
-          <p>The Arrow memory format supports <strong>zero-copy reads</strong>
-          for lightning-fast data access without serialization overhead.</p>
-
+            <h2>Fast</h2>
+            <p>Apache Arrow&#8482; enables execution engines to take advantage of the latest SIMD (Single input multiple data) operations included in modern processors, for native vectorized optimization of analytical data processing. Columnar layout is optimized for data locality for better performance on modern hardware like CPUs and GPUs.</p>
+            <p>The Arrow memory format supports <strong>zero-copy reads</strong> for lightning-fast data access without serialization overhead.</p>
         </div>
         <div class="col-lg-4">
-          <h2>Flexible</h2>
-          <p>Arrow acts as a new high-performance interface between various
-          systems. It is also focused on supporting a wide variety of
-          industry-standard programming languages. Java, C, C++, Python, Ruby,
-          and JavaScript implementations are in progress and more languages are
-          welcome.</p>
+            <h2>Flexible</h2>
+            <p>Arrow acts as a new high-performance interface between various systems. It is also focused on supporting a wide variety of industry-standard programming languages. Java, C, C++, Python, Ruby, and JavaScript implementations are in progress and more languages are welcome.
+            </p>
         </div>
         <div class="col-lg-4">
-          <h2>Standard</h2>
-          <p>Apache Arrow is backed by key developers of 13 major open source
-          projects, including Calcite, Cassandra, Drill, Hadoop, HBase, Ibis,
-          Impala, Kudu, Pandas, Parquet, Phoenix, Spark, and Storm making it
-          the de-facto standard for columnar in-memory analytics.</p>
-
-          <p>Learn more about projects that are <a href="{{ site.baseurl
-          }}/powered_by/">Powered By Apache Arrow</a></p>
+            <h2>Standard</h2>
+            <p>Apache Arrow is backed by key developers of 13 major open source projects, including Calcite, Cassandra, Drill, Hadoop, HBase, Ibis, Impala, Kudu, Pandas, Parquet, Phoenix, Spark, and Storm making it the de-facto standard for columnar in-memory analytics.</p>
+            <p>Learn more about projects that are <a href="{{ site.baseurl }}/powered_by/">Powered By Apache Arrow</a></p>
+        </div>
+    </div>
+    <!-- close "row" div -->
 
+    <div class="row">
+        <div class="col-xs-12">
+            <h2>Performance Advantage of Columnar In-Memory</h2>
+        </div>
+        <div class="col-lg-offset-2 col-lg-8 col-xs-12">
+            <img src="img/simd.png" alt="SIMD" class="img-responsive" />
         </div>
-     </div> <!-- close "row" div -->
+    </div>
 
-<h2>Performance Advantage of Columnar In-Memory</h2>
-<div align="center">
-  <img src="img/simd.png" alt="SIMD" style="width:60%" />
+    <div class="row">
+        <div class="col-xs-12">
+            <h2>Advantages of a Common Data Layer</h2>
+        </div>
+        <div class="col-lg-6 col-lg-offset-0 col-sm-8 col-sm-offset-2 col-xs-10 col-xs-offset-1">
+            <img src="img/copy.png" alt="common data layer" class="img-responsive" />
+            <ul>
+                <li>Each system has its own internal memory format</li>
+                <li>70-80% computation wasted on serialization and deserialization</li>
+                <li>Similar functionality implemented in multiple projects</li>
+            </ul>
+        </div>
+        <div class="col-lg-6 col-lg-offset-0 col-sm-8 col-sm-offset-2 col-xs-10 col-xs-offset-1">
+            <img src="img/shared.png" alt="common data layer" class="img-responsive" />
+            <ul>
+                <li>All systems utilize the same memory format</li>
+                <li>No overhead for cross-system communication</li>
+                <li>Projects can share functionality (eg, Parquet-to-Arrow reader)</li>
+            </ul>
+        </div>
+    </div>
 </div>
-<h2>Advantages of a Common Data Layer</h2>
+<!-- /container -->  
 
-<div class="row">
-<div class="col-lg-4" style="width:50%">
-<img src="img/copy2.png" alt="common data layer" style="width:100%" />
-<ul>
-    <li>Each system has its own internal memory format</li>
-    <li>70-80% computation wasted on serialization and deserialization</li>
-    <li>Similar functionality implemented in multiple projects</li>
-</ul>
-</div>
-<div class="col-lg-4" style="width:50%">
-<img src="img/shared2.png" alt="common data layer" style="width:100%" />
-<ul>
-    <li>All systems utilize the same memory format</li>
-    <li>No overhead for cross-system communication</li>
-    <li>Projects can share functionality (eg, Parquet-to-Arrow reader)</li>
-</ul>
-</div>
-</div>
-    </div> <!-- /container -->
-  </body>
+</body>
 </html>
diff --git a/site/install.md b/site/install.md
index ec30e0469cdc1..f795299676eb5 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,9 +20,9 @@ limitations under the License.
 {% endcomment %}
 -->
 
-## Current Version: 0.8.0
+## Current Version: {{site.data.versions['current'].number}}
 
-### Released: 18 December 2017
+### Released: {{site.data.versions['current'].date}}
 
 See the [release notes][10] for more about what's new.
 
@@ -30,7 +30,7 @@ See the [release notes][10] for more about what's new.
 
 * **Source Release**: [apache-arrow-0.8.0.tar.gz][6]
 * **Verification**: [sha512][3], [asc][7] ([verification instructions][12])
-* [Git tag 1d689e5][2]
+* [Git tag {{site.data.versions['current'].git-tag}}][2]
 * [PGP keys for release signatures][11]
 
 ### Java Packages
@@ -145,15 +145,15 @@ These repositories are managed at
 [red-data-tools/arrow-packages][9]. If you have any feedback, please
 send it to the project instead of Apache Arrow project.
 
-[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0
-[3]: https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.8.0%22
+[1]: {{site.data.versions['current'].mirrors}}
+[2]: {{site.data.versions['current'].github-tag-link}}
+[3]: {{site.data.versions['current'].sha512}}
+[4]: {{site.data.versions['current'].java-artifacts}}
 [5]: http://conda-forge.github.io
-[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz
-[7]: https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc
+[6]: {{site.data.versions['current'].mirrors-tar}}
+[7]: {{site.data.versions['current'].asc}}
 [8]: https://github.com/red-data-tools/parquet-glib
 [9]: https://github.com/red-data-tools/arrow-packages
-[10]: http://arrow.apache.org/release/0.8.0.html
+[10]: {{site.data.versions['current'].release-notes}}
 [11]: http://www.apache.org/dist/arrow/KEYS
 [12]: https://www.apache.org/dyn/closer.cgi#verify
\ No newline at end of file

From 9e4a6e4baa3dc18380a8173b07bf33f8764bf7ac Mon Sep 17 00:00:00 2001
From: Adam Seibert <seibs@users.noreply.github.com>
Date: Fri, 19 Jan 2018 15:41:43 -0500
Subject: [PATCH 1386/1644] ARROW-1930: [C++] Adds Slice operation to
 ChunkedArray and Column

Replicates `Slice` api from Array to ChunkedArray and Column.

Author: Adam Seibert <seibs@users.noreply.github.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1491 from seibs/ARROW-1930 and squashes the following commits:

1f03793b [Wes McKinney] Tweak doxygen comments
d920d80c [Adam Seibert] ARROW-1930: [C++] Adds Slice operation to ChunkedArray and Column
---
 cpp/src/arrow/table-test.cc | 31 +++++++++++++++++++++++++++++++
 cpp/src/arrow/table.cc      | 24 ++++++++++++++++++++++++
 cpp/src/arrow/table.h       | 36 +++++++++++++++++++++++++++++++++++-
 3 files changed, 90 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 3f1c6be3a87f6..99e4dd5db5146 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -108,6 +108,21 @@ TEST_F(TestChunkedArray, EqualsDifferingLengths) {
   ASSERT_TRUE(one_->Equals(*another_.get()));
 }
 
+TEST_F(TestChunkedArray, SliceEquals) {
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(100));
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(50));
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(50));
+  Construct();
+
+  std::shared_ptr<ChunkedArray> slice = one_->Slice(125, 50);
+  ASSERT_EQ(slice->length(), 50);
+  ASSERT_TRUE(slice->Equals(one_->Slice(125, 50)));
+
+  std::shared_ptr<ChunkedArray> slice2 = one_->Slice(75)->Slice(25)->Slice(25, 50);
+  ASSERT_EQ(slice2->length(), 50);
+  ASSERT_TRUE(slice2->Equals(slice));
+}
+
 class TestColumn : public TestChunkedArray {
  protected:
   void Construct() override {
@@ -158,6 +173,22 @@ TEST_F(TestColumn, ChunksInhomogeneous) {
   ASSERT_RAISES(Invalid, column_->ValidateData());
 }
 
+TEST_F(TestColumn, SliceEquals) {
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(100));
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(50));
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(50));
+  one_field_ = field("column", int32());
+  Construct();
+
+  std::shared_ptr<Column> slice = one_col_->Slice(125, 50);
+  ASSERT_EQ(slice->length(), 50);
+  ASSERT_TRUE(slice->Equals(one_col_->Slice(125, 50)));
+
+  std::shared_ptr<Column> slice2 = one_col_->Slice(75)->Slice(25)->Slice(25, 50);
+  ASSERT_EQ(slice2->length(), 50);
+  ASSERT_TRUE(slice2->Equals(slice));
+}
+
 TEST_F(TestColumn, Equals) {
   std::vector<bool> null_bitmap(100, true);
   std::vector<int32_t> data(100, 1);
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 2cf6c26523965..14877ccb537c2 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -102,6 +102,30 @@ bool ChunkedArray::Equals(const std::shared_ptr<ChunkedArray>& other) const {
   return Equals(*other.get());
 }
 
+std::shared_ptr<ChunkedArray> ChunkedArray::Slice(int64_t offset, int64_t length) const {
+  DCHECK_LE(offset, length_);
+
+  int curr_chunk = 0;
+  while (offset >= chunk(curr_chunk)->length()) {
+    offset -= chunk(curr_chunk)->length();
+    curr_chunk++;
+  }
+
+  ArrayVector new_chunks;
+  while (length > 0 && curr_chunk < num_chunks()) {
+    new_chunks.push_back(chunk(curr_chunk)->Slice(offset, length));
+    length -= chunk(curr_chunk)->length() - offset;
+    offset = 0;
+    curr_chunk++;
+  }
+
+  return std::make_shared<ChunkedArray>(new_chunks);
+}
+
+std::shared_ptr<ChunkedArray> ChunkedArray::Slice(int64_t offset) const {
+  return Slice(offset, length_);
+}
+
 Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks)
     : field_(field) {
   data_ = std::make_shared<ChunkedArray>(chunks);
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index c813b32ad36dc..570a650e7fa4a 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -44,6 +44,7 @@ class ARROW_EXPORT ChunkedArray {
   /// \return the total length of the chunked array; computed on construction
   int64_t length() const { return length_; }
 
+  /// \return the total number of nulls among all chunks
   int64_t null_count() const { return null_count_; }
 
   int num_chunks() const { return static_cast<int>(chunks_.size()); }
@@ -53,6 +54,20 @@ class ARROW_EXPORT ChunkedArray {
 
   const ArrayVector& chunks() const { return chunks_; }
 
+  /// \brief Construct a zero-copy slice of the chunked array with the
+  /// indicated offset and length
+  ///
+  /// \param[in] offset the position of the first element in the constructed
+  /// slice
+  /// \param[in] length the length of the slice. If there are not enough
+  /// elements in the chunked array, the length will be adjusted accordingly
+  ///
+  /// \return a new object wrapped in std::shared_ptr<ChunkedArray>
+  std::shared_ptr<ChunkedArray> Slice(int64_t offset, int64_t length) const;
+
+  /// \brief Slice from offset until end of the chunked array
+  std::shared_ptr<ChunkedArray> Slice(int64_t offset) const;
+
   std::shared_ptr<DataType> type() const;
 
   bool Equals(const ChunkedArray& other) const;
@@ -67,8 +82,9 @@ class ARROW_EXPORT ChunkedArray {
   ARROW_DISALLOW_COPY_AND_ASSIGN(ChunkedArray);
 };
 
+/// \class Column
 /// \brief An immutable column data structure consisting of a field (type
-/// metadata) and a logical chunked data array
+/// metadata) and a chunked data array
 class ARROW_EXPORT Column {
  public:
   Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
@@ -97,6 +113,24 @@ class ARROW_EXPORT Column {
   /// \return the column's data as a chunked logical array
   std::shared_ptr<ChunkedArray> data() const { return data_; }
 
+  /// \brief Construct a zero-copy slice of the column with the indicated
+  /// offset and length
+  ///
+  /// \param[in] offset the position of the first element in the constructed
+  /// slice
+  /// \param[in] length the length of the slice. If there are not enough
+  /// elements in the column, the length will be adjusted accordingly
+  ///
+  /// \return a new object wrapped in std::shared_ptr<Column>
+  std::shared_ptr<Column> Slice(int64_t offset, int64_t length) const {
+    return std::make_shared<Column>(field_, data_->Slice(offset, length));
+  }
+
+  /// \brief Slice from offset until end of the column
+  std::shared_ptr<Column> Slice(int64_t offset) const {
+    return std::make_shared<Column>(field_, data_->Slice(offset));
+  }
+
   bool Equals(const Column& other) const;
   bool Equals(const std::shared_ptr<Column>& other) const;
 

From e4460847f3387c6c1a8bb77edd2aedc69e7250d3 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Fri, 19 Jan 2018 14:49:17 -0800
Subject: [PATCH 1387/1644] ARROW-2011: [Python] Allow setting the pickler in
 the serialization context.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1493 from robertnishihara/cloudpickle and squashes the following commits:

57fb46f [Robert Nishihara] Fix test (it didn't work without cloudpickle).
a884bb4 [Robert Nishihara] Add test.
14e1536 [Robert Nishihara] Allow setting the pickler in the serialization context.
---
 python/pyarrow/serialization.pxi           | 26 +++++++++++++--
 python/pyarrow/tests/test_serialization.py | 39 ++++++++++++++++++++++
 2 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index d95d582fe537e..e7a39905f1f65 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -50,6 +50,8 @@ cdef class SerializationContext:
         object types_to_pickle
         object custom_serializers
         object custom_deserializers
+        object pickle_serializer
+        object pickle_deserializer
 
     def __init__(self):
         # Types with special serialization handlers
@@ -58,6 +60,23 @@ cdef class SerializationContext:
         self.types_to_pickle = set()
         self.custom_serializers = dict()
         self.custom_deserializers = dict()
+        self.pickle_serializer = pickle.dumps
+        self.pickle_deserializer = pickle.loads
+
+    def set_pickle(self, serializer, deserializer):
+        """
+        Set the serializer and deserializer to use for objects that are to be
+        pickled.
+
+        Parameters
+        ----------
+        serializer : callable
+            The serializer to use (e.g., pickle.dumps or cloudpickle.dumps).
+        deserializer : callable
+            The deserializer to use (e.g., pickle.dumps or cloudpickle.dumps).
+        """
+        self.pickle_serializer = serializer
+        self.pickle_deserializer = deserializer
 
     def clone(self):
         """
@@ -72,6 +91,8 @@ cdef class SerializationContext:
         result.whitelisted_types = self.whitelisted_types.copy()
         result.custom_serializers = self.custom_serializers.copy()
         result.custom_deserializers = self.custom_deserializers.copy()
+        result.pickle_serializer = self.pickle_serializer
+        result.pickle_deserializer = self.pickle_deserializer
 
         return result
 
@@ -119,7 +140,8 @@ cdef class SerializationContext:
         # use the closest match to type(obj)
         type_id = self.type_to_type_id[type_]
         if type_id in self.types_to_pickle:
-            serialized_obj = {"data": pickle.dumps(obj), "pickle": True}
+            serialized_obj = {"data": self.pickle_serializer(obj),
+                              "pickle": True}
         elif type_id in self.custom_serializers:
             serialized_obj = {"data": self.custom_serializers[type_id](obj)}
         else:
@@ -139,7 +161,7 @@ cdef class SerializationContext:
 
         if "pickle" in serialized_obj:
             # The object was pickled, so unpickle it.
-            obj = pickle.loads(serialized_obj["data"])
+            obj = self.pickle_deserializer(serialized_obj["data"])
         else:
             assert type_id not in self.types_to_pickle
             if type_id not in self.whitelisted_types:
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 6116556386b1a..e4681e3a59751 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -555,3 +555,42 @@ def test_deserialize_buffer_in_different_process():
     dir_path = os.path.dirname(os.path.realpath(__file__))
     python_file = os.path.join(dir_path, 'deserialize_buffer.py')
     subprocess.check_call(['python', python_file, f.name])
+
+
+def test_set_pickle():
+    # Use a custom type to trigger pickling.
+    class Foo(object):
+        pass
+
+    context = pa.SerializationContext()
+    context.register_type(Foo, 'Foo', pickle=True)
+
+    test_object = Foo()
+
+    # Define a custom serializer and deserializer to use in place of pickle.
+
+    def dumps1(obj):
+        return b'custom'
+
+    def loads1(serialized_obj):
+        return serialized_obj + b' serialization 1'
+
+    # Test that setting a custom pickler changes the behavior.
+    context.set_pickle(dumps1, loads1)
+    serialized = pa.serialize(test_object, context=context).to_buffer()
+    deserialized = pa.deserialize(serialized.to_pybytes(), context=context)
+    assert deserialized == b'custom serialization 1'
+
+    # Define another custom serializer and deserializer.
+
+    def dumps2(obj):
+        return b'custom'
+
+    def loads2(serialized_obj):
+        return serialized_obj + b' serialization 2'
+
+    # Test that setting another custom pickler changes the behavior again.
+    context.set_pickle(dumps2, loads2)
+    serialized = pa.serialize(test_object, context=context).to_buffer()
+    deserialized = pa.deserialize(serialized.to_pybytes(), context=context)
+    assert deserialized == b'custom serialization 2'

From d135974a0d3dd9a9fbbb10da4c5dbc65f9324234 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Sat, 20 Jan 2018 13:41:23 -0800
Subject: [PATCH 1388/1644] ARROW-2000: [Plasma] Deduplicate file descriptors
 when replying to GetRequest.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1479 from robertnishihara/deduplicatefiledescriptors and squashes the following commits:

9be9643 [Robert Nishihara] Fix bug.
8a827cf [Robert Nishihara] Remove mmap_size from PlasmaObject.
ab30d7d [Robert Nishihara] Fix tests.
2916e87 [Robert Nishihara] Remove mmap_size from PlasmaObjectSpec, and file_descriptor -> fd.
7f5c618 [Robert Nishihara] Deduplicate file descriptors when store replies to Get.
ab12d63 [Robert Nishihara] Make Create return a MutableBuffer.
---
 cpp/src/plasma/client.cc                   | 45 +++++++--------
 cpp/src/plasma/client.h                    |  5 +-
 cpp/src/plasma/format/plasma.fbs           | 20 +++++--
 cpp/src/plasma/malloc.cc                   | 10 ++++
 cpp/src/plasma/malloc.h                    |  6 ++
 cpp/src/plasma/plasma.h                    | 12 +---
 cpp/src/plasma/protocol.cc                 | 49 ++++++++++------
 cpp/src/plasma/protocol.h                  | 11 ++--
 cpp/src/plasma/store.cc                    | 67 +++++++++++++---------
 cpp/src/plasma/test/client_tests.cc        | 14 ++---
 cpp/src/plasma/test/serialization_tests.cc | 26 +++++++--
 python/pyarrow/plasma.pyx                  |  4 +-
 12 files changed, 165 insertions(+), 104 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index d74c0f412d97f..a683da0022b18 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -54,8 +54,6 @@
 
 namespace plasma {
 
-using arrow::MutableBuffer;
-
 // Number of threads used for memcopy and hash computations.
 constexpr int64_t kThreadPoolSize = 8;
 constexpr int64_t kBytesInMB = 1 << 20;
@@ -130,7 +128,7 @@ void PlasmaClient::increment_object_count(const ObjectID& object_id, PlasmaObjec
     // Increment the count of the number of objects in the memory-mapped file
     // that are being used. The corresponding decrement should happen in
     // PlasmaClient::Release.
-    auto entry = mmap_table_.find(object->handle.store_fd);
+    auto entry = mmap_table_.find(object->store_fd);
     ARROW_CHECK(entry != mmap_table_.end());
     ARROW_CHECK(entry->second.count >= 0);
     // Update the in_use_object_bytes_.
@@ -149,7 +147,7 @@ void PlasmaClient::increment_object_count(const ObjectID& object_id, PlasmaObjec
 
 Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
                             uint8_t* metadata, int64_t metadata_size,
-                            std::shared_ptr<Buffer>* data) {
+                            std::shared_ptr<MutableBuffer>* data) {
   ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
                    << data_size << " and metadata size " << metadata_size;
   RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
@@ -157,7 +155,10 @@ Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
   RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaCreateReply, &buffer));
   ObjectID id;
   PlasmaObject object;
-  RETURN_NOT_OK(ReadCreateReply(buffer.data(), buffer.size(), &id, &object));
+  int store_fd;
+  int64_t mmap_size;
+  RETURN_NOT_OK(
+      ReadCreateReply(buffer.data(), buffer.size(), &id, &object, &store_fd, &mmap_size));
   // If the CreateReply included an error, then the store will not send a file
   // descriptor.
   int fd = recv_fd(store_conn_);
@@ -167,9 +168,7 @@ Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
   // The metadata should come right after the data.
   ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
   *data = std::make_shared<MutableBuffer>(
-      lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) +
-          object.data_offset,
-      data_size);
+      lookup_or_mmap(fd, store_fd, mmap_size) + object.data_offset, data_size);
   // If plasma_create is being called from a transfer, then we will not copy the
   // metadata here. The metadata will be written along with the data streamed
   // from the transfer.
@@ -209,7 +208,7 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
       ARROW_CHECK(object_entry->second->is_sealed)
           << "Plasma client called get on an unsealed object that it created";
       PlasmaObject* object = &object_entry->second->object;
-      uint8_t* data = lookup_mmapped_file(object->handle.store_fd);
+      uint8_t* data = lookup_mmapped_file(object->store_fd);
       object_buffers[i].data =
           std::make_shared<Buffer>(data + object->data_offset, object->data_size);
       object_buffers[i].metadata = std::make_shared<Buffer>(
@@ -236,8 +235,19 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
   std::vector<ObjectID> received_object_ids(num_objects);
   std::vector<PlasmaObject> object_data(num_objects);
   PlasmaObject* object;
+  std::vector<int> store_fds;
+  std::vector<int64_t> mmap_sizes;
   RETURN_NOT_OK(ReadGetReply(buffer.data(), buffer.size(), received_object_ids.data(),
-                             object_data.data(), num_objects));
+                             object_data.data(), num_objects, store_fds, mmap_sizes));
+
+  // We mmap all of the file descriptors here so that we can avoid look them up
+  // in the subsequent loop based on just the store file descriptor and without
+  // having to know the relevant file descriptor received from recv_fd.
+  for (size_t i = 0; i < store_fds.size(); i++) {
+    int fd = recv_fd(store_conn_);
+    ARROW_CHECK(fd >= 0);
+    lookup_or_mmap(fd, store_fds[i], mmap_sizes[i]);
+  }
 
   for (int i = 0; i < num_objects; ++i) {
     DCHECK(received_object_ids[i] == object_ids[i]);
@@ -246,12 +256,6 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
       // If the object was already in use by the client, then the store should
       // have returned it.
       DCHECK_NE(object->data_size, -1);
-      // We won't use this file descriptor, but the store sent us one, so we
-      // need to receive it and then close it right away so we don't leak file
-      // descriptors.
-      int fd = recv_fd(store_conn_);
-      close(fd);
-      ARROW_CHECK(fd >= 0);
       // We've already filled out the information for this object, so we can
       // just continue.
       continue;
@@ -259,12 +263,7 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
     // If we are here, the object was not currently in use, so we need to
     // process the reply from the object store.
     if (object->data_size != -1) {
-      // The object was retrieved. The user will be responsible for releasing
-      // this object.
-      int fd = recv_fd(store_conn_);
-      uint8_t* data =
-          lookup_or_mmap(fd, object->handle.store_fd, object->handle.mmap_size);
-      ARROW_CHECK(fd >= 0);
+      uint8_t* data = lookup_mmapped_file(object->store_fd);
       // Finish filling out the return values.
       object_buffers[i].data =
           std::make_shared<Buffer>(data + object->data_offset, object->data_size);
@@ -296,7 +295,7 @@ Status PlasmaClient::UnmapObject(const ObjectID& object_id) {
   // Decrement the count of the number of objects in this memory-mapped file
   // that the client is using. The corresponding increment should have
   // happened in plasma_get.
-  int fd = object_entry->second->object.handle.store_fd;
+  int fd = object_entry->second->object.store_fd;
   auto entry = mmap_table_.find(fd);
   ARROW_CHECK(entry != mmap_table_.end());
   ARROW_CHECK(entry->second.count >= 1);
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index 35182f8403201..a1e10a9c29969 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -31,8 +31,9 @@
 #include "arrow/util/visibility.h"
 #include "plasma/common.h"
 
-using arrow::Status;
 using arrow::Buffer;
+using arrow::MutableBuffer;
+using arrow::Status;
 
 namespace plasma {
 
@@ -115,7 +116,7 @@ class ARROW_EXPORT PlasmaClient {
   ///        will be written here.
   /// \return The return status.
   Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
-                int64_t metadata_size, std::shared_ptr<Buffer>* data);
+                int64_t metadata_size, std::shared_ptr<MutableBuffer>* data);
   /// Get some objects from the Plasma Store. This function will block until the
   /// objects have all been created and sealed in the Plasma Store or the
   /// timeout
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
index ea6dc8bb98da5..33803f7799ba0 100644
--- a/cpp/src/plasma/format/plasma.fbs
+++ b/cpp/src/plasma/format/plasma.fbs
@@ -89,8 +89,6 @@ struct PlasmaObjectSpec {
   // Index of the memory segment (= memory mapped file) that
   // this object is allocated in.
   segment_index: int;
-  // Size in bytes of this segment (needed to call mmap).
-  mmap_size: ulong;
   // The offset in bytes in the memory mapped file of the data.
   data_offset: ulong;
   // The size in bytes of the data.
@@ -117,6 +115,12 @@ table PlasmaCreateReply {
   plasma_object: PlasmaObjectSpec;
   // Error that occurred for this call.
   error: PlasmaError;
+  // The file descriptor in the store that corresponds to the file descriptor
+  // being sent to the client right after this message.
+  store_fd: int;
+  // The size in bytes of the segment for the store file descriptor (needed to
+  // call mmap).
+  mmap_size: long;
 }
 
 table PlasmaAbortRequest {
@@ -156,9 +160,17 @@ table PlasmaGetReply {
   // objects if not all requested objects are stored and sealed
   // in the local Plasma store.
   object_ids: [string];
-  // Plasma object information, in the same order as their IDs.
+  // Plasma object information, in the same order as their IDs. The number of
+  // elements in both object_ids and plasma_objects arrays must agree.
   plasma_objects: [PlasmaObjectSpec];
-  // The number of elements in both object_ids and plasma_objects arrays must agree.
+  // A list of the file descriptors in the store that correspond to the file
+  // descriptors being sent to the client. The length of this list is the number
+  // of file descriptors that the store will send to the client after this
+  // message.
+  store_fds: [int];
+  // Size in bytes of the segment for each store file descriptor (needed to call
+  // mmap). This list must have the same length as store_fds.
+  mmap_sizes: [long];
 }
 
 table PlasmaReleaseRequest {
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
index 52d362013f1ae..3c5d107b2bbe3 100644
--- a/cpp/src/plasma/malloc.cc
+++ b/cpp/src/plasma/malloc.cc
@@ -197,4 +197,14 @@ void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_size, ptrdiff_t* offse
   *offset = 0;
 }
 
+int64_t get_mmap_size(int fd) {
+  for (const auto& entry : mmap_records) {
+    if (entry.second.fd == fd) {
+      return entry.second.size;
+    }
+  }
+  ARROW_LOG(FATAL) << "failed to find entry in mmap_records for fd " << fd;
+  return -1;  // This code is never reached.
+}
+
 void set_malloc_granularity(int value) { change_mparam(M_GRANULARITY, value); }
diff --git a/cpp/src/plasma/malloc.h b/cpp/src/plasma/malloc.h
index 0df720db59817..cb8c600b14b3b 100644
--- a/cpp/src/plasma/malloc.h
+++ b/cpp/src/plasma/malloc.h
@@ -23,6 +23,12 @@
 
 void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_length, ptrdiff_t* offset);
 
+/// Get the mmap size corresponding to a specific file descriptor.
+///
+/// @param fd The file descriptor to look up.
+/// @return The size of the corresponding memory-mapped file.
+int64_t get_mmap_size(int fd);
+
 void set_malloc_granularity(int value);
 
 #endif  // MALLOC_H
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
index 603ff8a4fac6c..2d07c919a18f4 100644
--- a/cpp/src/plasma/plasma.h
+++ b/cpp/src/plasma/plasma.h
@@ -64,20 +64,12 @@ struct Client;
 /// Mapping from object IDs to type and status of the request.
 typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectRequestMap;
 
-/// Handle to access memory mapped file and map it into client address space.
-struct object_handle {
+// TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
+struct PlasmaObject {
   /// The file descriptor of the memory mapped file in the store. It is used as
   /// a unique identifier of the file in the client to look up the corresponding
   /// file descriptor on the client's side.
   int store_fd;
-  /// The size in bytes of the memory mapped file.
-  int64_t mmap_size;
-};
-
-// TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
-struct PlasmaObject {
-  /// Handle for memory mapped file the object is stored in.
-  object_handle handle;
   /// The offset in bytes in the memory mapped file of the data.
   ptrdiff_t data_offset;
   /// The offset in bytes in the memory mapped file of the metadata.
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index c0ebb88fe5019..6c0bc0cab28bb 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -73,30 +73,32 @@ Status ReadCreateRequest(uint8_t* data, size_t size, ObjectID* object_id,
   return Status::OK();
 }
 
-Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object,
-                       int error_code) {
+Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error_code,
+                       int64_t mmap_size) {
   flatbuffers::FlatBufferBuilder fbb;
-  PlasmaObjectSpec plasma_object(object->handle.store_fd, object->handle.mmap_size,
-                                 object->data_offset, object->data_size,
+  PlasmaObjectSpec plasma_object(object->store_fd, object->data_offset, object->data_size,
                                  object->metadata_offset, object->metadata_size);
-  auto message =
-      CreatePlasmaCreateReply(fbb, fbb.CreateString(object_id.binary()), &plasma_object,
-                              static_cast<PlasmaError>(error_code));
+  auto message = CreatePlasmaCreateReply(
+      fbb, fbb.CreateString(object_id.binary()), &plasma_object,
+      static_cast<PlasmaError>(error_code), object->store_fd, mmap_size);
   return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
 }
 
 Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id,
-                       PlasmaObject* object) {
+                       PlasmaObject* object, int* store_fd, int64_t* mmap_size) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
   DCHECK(verify_flatbuffer(message, data, size));
   *object_id = ObjectID::from_binary(message->object_id()->str());
-  object->handle.store_fd = message->plasma_object()->segment_index();
-  object->handle.mmap_size = message->plasma_object()->mmap_size();
+  object->store_fd = message->plasma_object()->segment_index();
   object->data_offset = message->plasma_object()->data_offset();
   object->data_size = message->plasma_object()->data_size();
   object->metadata_offset = message->plasma_object()->metadata_offset();
   object->metadata_size = message->plasma_object()->metadata_size();
+
+  *store_fd = message->store_fd();
+  *mmap_size = message->mmap_size();
+
   return plasma_error_status(message->error());
 }
 
@@ -389,24 +391,29 @@ Status ReadGetRequest(uint8_t* data, size_t size, std::vector<ObjectID>& object_
 Status SendGetReply(
     int sock, ObjectID object_ids[],
     std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
-    int64_t num_objects) {
+    int64_t num_objects, const std::vector<int>& store_fds,
+    const std::vector<int64_t>& mmap_sizes) {
   flatbuffers::FlatBufferBuilder fbb;
   std::vector<PlasmaObjectSpec> objects;
 
-  for (int i = 0; i < num_objects; ++i) {
+  ARROW_CHECK(store_fds.size() == mmap_sizes.size());
+
+  for (int64_t i = 0; i < num_objects; ++i) {
     const PlasmaObject& object = plasma_objects[object_ids[i]];
-    objects.push_back(PlasmaObjectSpec(object.handle.store_fd, object.handle.mmap_size,
-                                       object.data_offset, object.data_size,
-                                       object.metadata_offset, object.metadata_size));
+    objects.push_back(PlasmaObjectSpec(object.store_fd, object.data_offset,
+                                       object.data_size, object.metadata_offset,
+                                       object.metadata_size));
   }
   auto message =
       CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
-                           fbb.CreateVectorOfStructs(objects.data(), num_objects));
+                           fbb.CreateVectorOfStructs(objects.data(), num_objects),
+                           fbb.CreateVector(store_fds), fbb.CreateVector(mmap_sizes));
   return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
 }
 
 Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
-                    PlasmaObject plasma_objects[], int64_t num_objects) {
+                    PlasmaObject plasma_objects[], int64_t num_objects,
+                    std::vector<int>& store_fds, std::vector<int64_t>& mmap_sizes) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
   DCHECK(verify_flatbuffer(message, data, size));
@@ -415,13 +422,17 @@ Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
   }
   for (uoffset_t i = 0; i < num_objects; ++i) {
     const PlasmaObjectSpec* object = message->plasma_objects()->Get(i);
-    plasma_objects[i].handle.store_fd = object->segment_index();
-    plasma_objects[i].handle.mmap_size = object->mmap_size();
+    plasma_objects[i].store_fd = object->segment_index();
     plasma_objects[i].data_offset = object->data_offset();
     plasma_objects[i].data_size = object->data_size();
     plasma_objects[i].metadata_offset = object->metadata_offset();
     plasma_objects[i].metadata_size = object->metadata_size();
   }
+  ARROW_CHECK(message->store_fds()->size() == message->mmap_sizes()->size());
+  for (uoffset_t i = 0; i < message->store_fds()->size(); i++) {
+    store_fds.push_back(message->store_fds()->Get(i));
+    mmap_sizes.push_back(message->mmap_sizes()->Get(i));
+  }
   return Status::OK();
 }
 
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
index e8c334f9181fc..44263a6418439 100644
--- a/cpp/src/plasma/protocol.h
+++ b/cpp/src/plasma/protocol.h
@@ -46,10 +46,11 @@ Status SendCreateRequest(int sock, ObjectID object_id, int64_t data_size,
 Status ReadCreateRequest(uint8_t* data, size_t size, ObjectID* object_id,
                          int64_t* data_size, int64_t* metadata_size);
 
-Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error);
+Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error,
+                       int64_t mmap_size);
 
 Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id,
-                       PlasmaObject* object);
+                       PlasmaObject* object, int* store_fd, int64_t* mmap_size);
 
 Status SendAbortRequest(int sock, ObjectID object_id);
 
@@ -81,10 +82,12 @@ Status ReadGetRequest(uint8_t* data, size_t size, std::vector<ObjectID>& object_
 Status SendGetReply(
     int sock, ObjectID object_ids[],
     std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
-    int64_t num_objects);
+    int64_t num_objects, const std::vector<int>& store_fds,
+    const std::vector<int64_t>& mmap_sizes);
 
 Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
-                    PlasmaObject plasma_objects[], int64_t num_objects);
+                    PlasmaObject plasma_objects[], int64_t num_objects,
+                    std::vector<int>& store_fds, std::vector<int64_t>& mmap_sizes);
 
 /* Plasma Release message functions. */
 
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index dde7f9cdfa8eb..80dd525e3e3b4 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -192,8 +192,7 @@ int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
   entry->state = PLASMA_CREATED;
 
   store_info_.objects[object_id] = std::move(entry);
-  result->handle.store_fd = fd;
-  result->handle.mmap_size = map_size;
+  result->store_fd = fd;
   result->data_offset = offset;
   result->metadata_offset = offset + data_size;
   result->data_size = data_size;
@@ -211,8 +210,7 @@ void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
   DCHECK(object != NULL);
   DCHECK(entry != NULL);
   DCHECK(entry->state == PLASMA_SEALED);
-  object->handle.store_fd = entry->fd;
-  object->handle.mmap_size = entry->map_size;
+  object->store_fd = entry->fd;
   object->data_offset = entry->offset;
   object->metadata_offset = entry->offset + entry->info.data_size;
   object->data_size = entry->info.data_size;
@@ -220,34 +218,44 @@ void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
 }
 
 void PlasmaStore::return_from_get(GetRequest* get_req) {
+  // Figure out how many file descriptors we need to send.
+  std::unordered_set<int> fds_to_send;
+  std::vector<int> store_fds;
+  std::vector<int64_t> mmap_sizes;
+  for (const auto& object_id : get_req->object_ids) {
+    PlasmaObject& object = get_req->objects[object_id];
+    int fd = object.store_fd;
+    if (object.data_size != -1 && fds_to_send.count(fd) == 0) {
+      fds_to_send.insert(fd);
+      store_fds.push_back(fd);
+      mmap_sizes.push_back(get_mmap_size(fd));
+    }
+  }
+
   // Send the get reply to the client.
   Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], get_req->objects,
-                          get_req->object_ids.size());
+                          get_req->object_ids.size(), store_fds, mmap_sizes);
   warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd);
   // If we successfully sent the get reply message to the client, then also send
   // the file descriptors.
   if (s.ok()) {
     // Send all of the file descriptors for the present objects.
-    for (const auto& object_id : get_req->object_ids) {
-      PlasmaObject& object = get_req->objects[object_id];
-      // We use the data size to indicate whether the object is present or not.
-      if (object.data_size != -1) {
-        int error_code = send_fd(get_req->client->fd, object.handle.store_fd);
-        // If we failed to send the file descriptor, loop until we have sent it
-        // successfully. TODO(rkn): This is problematic for two reasons. First
-        // of all, sending the file descriptor should just succeed without any
-        // errors, but sometimes I see a "Message too long" error number.
-        // Second, looping like this allows a client to potentially block the
-        // plasma store event loop which should never happen.
-        while (error_code < 0) {
-          if (errno == EMSGSIZE) {
-            ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying.";
-            error_code = send_fd(get_req->client->fd, object.handle.store_fd);
-            continue;
-          }
-          warn_if_sigpipe(error_code, get_req->client->fd);
-          break;
+    for (int store_fd : store_fds) {
+      int error_code = send_fd(get_req->client->fd, store_fd);
+      // If we failed to send the file descriptor, loop until we have sent it
+      // successfully. TODO(rkn): This is problematic for two reasons. First
+      // of all, sending the file descriptor should just succeed without any
+      // errors, but sometimes I see a "Message too long" error number.
+      // Second, looping like this allows a client to potentially block the
+      // plasma store event loop which should never happen.
+      while (error_code < 0) {
+        if (errno == EMSGSIZE) {
+          ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying.";
+          error_code = send_fd(get_req->client->fd, store_fd);
+          continue;
         }
+        warn_if_sigpipe(error_code, get_req->client->fd);
+        break;
       }
     }
   }
@@ -640,10 +648,15 @@ Status PlasmaStore::process_message(Client* client) {
           ReadCreateRequest(input, input_size, &object_id, &data_size, &metadata_size));
       int error_code =
           create_object(object_id, data_size, metadata_size, client, &object);
-      HANDLE_SIGPIPE(SendCreateReply(client->fd, object_id, &object, error_code),
-                     client->fd);
+      int64_t mmap_size = 0;
+      if (error_code == PlasmaError_OK) {
+        mmap_size = get_mmap_size(object.store_fd);
+      }
+      HANDLE_SIGPIPE(
+          SendCreateReply(client->fd, object_id, &object, error_code, mmap_size),
+          client->fd);
       if (error_code == PlasmaError_OK) {
-        warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
+        warn_if_sigpipe(send_fd(client->fd, object.store_fd), client->fd);
       }
     } break;
     case MessageType_PlasmaAbortRequest: {
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index f19c2bfbdb380..63b56934f3599 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -70,7 +70,7 @@ TEST_F(TestPlasmaStore, DeleteTest) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<Buffer> data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client_.Seal(object_id));
 
@@ -96,7 +96,7 @@ TEST_F(TestPlasmaStore, ContainsTest) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<Buffer> data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client_.Seal(object_id));
   // Avoid race condition of Plasma Manager waiting for notification.
@@ -119,7 +119,7 @@ TEST_F(TestPlasmaStore, GetTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<Buffer> data_buffer;
+  std::shared_ptr<MutableBuffer> data_buffer;
   uint8_t* data;
   ARROW_CHECK_OK(
       client_.Create(object_id, data_size, metadata, metadata_size, &data_buffer));
@@ -145,7 +145,7 @@ TEST_F(TestPlasmaStore, MultipleGetTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<Buffer> data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data));
   data->mutable_data()[0] = 1;
   ARROW_CHECK_OK(client_.Seal(object_id1));
@@ -172,7 +172,7 @@ TEST_F(TestPlasmaStore, AbortTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<Buffer> data;
+  std::shared_ptr<MutableBuffer> data;
   uint8_t* data_ptr;
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
   data_ptr = data->mutable_data();
@@ -220,7 +220,7 @@ TEST_F(TestPlasmaStore, MultipleClientTest) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<Buffer> data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(client2_.Create(object_id, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client2_.Seal(object_id));
   // Test that the first client can get the object.
@@ -260,7 +260,7 @@ TEST_F(TestPlasmaStore, ManyObjectTest) {
     int64_t data_size = 100;
     uint8_t metadata[] = {5};
     int64_t metadata_size = sizeof(metadata);
-    std::shared_ptr<Buffer> data;
+    std::shared_ptr<MutableBuffer> data;
     ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
 
     if (i % 3 == 0) {
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
index b593b6ae94890..656b2cc6b9bca 100644
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -63,8 +63,7 @@ PlasmaObject random_plasma_object(void) {
   int random = rand_r(&seed);
   PlasmaObject object;
   memset(&object, 0, sizeof(object));
-  object.handle.store_fd = random + 7;
-  object.handle.mmap_size = random + 42;
+  object.store_fd = random + 7;
   object.data_offset = random + 1;
   object.metadata_offset = random + 2;
   object.data_size = random + 3;
@@ -94,13 +93,19 @@ TEST(PlasmaSerialization, CreateReply) {
   int fd = create_temp_file();
   ObjectID object_id1 = ObjectID::from_random();
   PlasmaObject object1 = random_plasma_object();
-  ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0));
+  int64_t mmap_size1 = 1000000;
+  ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0, mmap_size1));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateReply);
   ObjectID object_id2;
   PlasmaObject object2;
   memset(&object2, 0, sizeof(object2));
-  ARROW_CHECK_OK(ReadCreateReply(data.data(), data.size(), &object_id2, &object2));
+  int store_fd;
+  int64_t mmap_size2;
+  ARROW_CHECK_OK(ReadCreateReply(data.data(), data.size(), &object_id2, &object2,
+                                 &store_fd, &mmap_size2));
   ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(object1.store_fd, store_fd);
+  ASSERT_EQ(mmap_size1, mmap_size2);
   ASSERT_EQ(memcmp(&object1, &object2, sizeof(object1)), 0);
   close(fd);
 }
@@ -158,13 +163,20 @@ TEST(PlasmaSerialization, GetReply) {
   std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> plasma_objects;
   plasma_objects[object_ids[0]] = random_plasma_object();
   plasma_objects[object_ids[1]] = random_plasma_object();
-  ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2));
+  std::vector<int> store_fds = {1, 2, 3};
+  std::vector<int64_t> mmap_sizes = {100, 200, 300};
+  ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2, store_fds, mmap_sizes));
+
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetReply);
   ObjectID object_ids_return[2];
   PlasmaObject plasma_objects_return[2];
+  std::vector<int> store_fds_return;
+  std::vector<int64_t> mmap_sizes_return;
   memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
   ARROW_CHECK_OK(ReadGetReply(data.data(), data.size(), object_ids_return,
-                              &plasma_objects_return[0], 2));
+                              &plasma_objects_return[0], 2, store_fds_return,
+                              mmap_sizes_return));
+
   ASSERT_EQ(object_ids[0], object_ids_return[0]);
   ASSERT_EQ(object_ids[1], object_ids_return[1]);
   ASSERT_EQ(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
@@ -173,6 +185,8 @@ TEST(PlasmaSerialization, GetReply) {
   ASSERT_EQ(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
                    sizeof(PlasmaObject)),
             0);
+  ASSERT_TRUE(store_fds == store_fds_return);
+  ASSERT_TRUE(mmap_sizes == mmap_sizes_return);
   close(fd);
 }
 
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index 32f6d189da08c..801d094194b71 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -81,7 +81,7 @@ cdef extern from "plasma/client.h" nogil:
 
         CStatus Create(const CUniqueID& object_id, int64_t data_size,
                        const uint8_t* metadata, int64_t metadata_size,
-                       const shared_ptr[CBuffer]* data)
+                       const shared_ptr[CMutableBuffer]* data)
 
         CStatus Get(const CUniqueID* object_ids, int64_t num_objects,
                     int64_t timeout_ms, CObjectBuffer* object_buffers)
@@ -297,7 +297,7 @@ cdef class PlasmaClient:
                 not be created because the plasma store is unable to evict
                 enough objects to create room for it.
         """
-        cdef shared_ptr[CBuffer] data
+        cdef shared_ptr[CMutableBuffer] data
         with nogil:
             check_status(self.client.get().Create(object_id.data, data_size,
                                                   <uint8_t*>(metadata.data()),

From 1bbaf7e669a580531be30cd2f8ade8b560466774 Mon Sep 17 00:00:00 2001
From: Simbarashe Nyatsanga <simnyatsanga@gmail.com>
Date: Sun, 21 Jan 2018 01:12:07 +0200
Subject: [PATCH 1389/1644] [Python] Fix small typos in bytes, String/UTF-8 and
 FixedSizeBinary type check exceptions. (#1495)

---
 cpp/src/arrow/python/numpy_to_arrow.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index f21b40ed3c246..c5c02e355ded6 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -175,7 +175,7 @@ static Status AppendObjectBinaries(PyArrayObject* arr, PyArrayObject* mask,
       continue;
     } else if (!PyBytes_Check(obj)) {
       std::stringstream ss;
-      ss << "Error converting to Python objects to bytes: ";
+      ss << "Error converting from Python objects to bytes: ";
       RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
       return Status::Invalid(ss.str());
     }
@@ -230,7 +230,7 @@ static Status AppendObjectStrings(PyArrayObject* arr, PyArrayObject* mask, int64
       *have_bytes = true;
     } else {
       std::stringstream ss;
-      ss << "Error converting to Python objects to String/UTF8: ";
+      ss << "Error converting from Python objects to String/UTF8: ";
       RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
       return Status::Invalid(ss.str());
     }
@@ -278,7 +278,7 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
       tmp_obj.reset(obj);
     } else if (!PyBytes_Check(obj)) {
       std::stringstream ss;
-      ss << "Error converting to Python objects to FixedSizeBinary: ";
+      ss << "Error converting from Python objects to FixedSizeBinary: ";
       RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
       return Status::Invalid(ss.str());
     }

From ed272430e310102c750cf997cc2ad5dace2d3323 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 21 Jan 2018 22:10:34 +0900
Subject: [PATCH 1390/1644] ARROW-2012: [GLib] Support "make distclean"

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1494 from kou/glib-support-distclean and squashes the following commits:

d660e0f8 [Kouhei Sutou] [GLib] Support "make distclean"
---
 c_glib/configure.ac                           |  2 +-
 c_glib/doc/reference/Makefile.am              |  4 +--
 c_glib/doc/reference/arrow-glib-docs.xml      |  4 +--
 .../gtkdocentities.ent.in => entities.xml.in} | 12 +++----
 c_glib/doc/reference/meson.build              | 13 +++++++-
 c_glib/doc/reference/xml/Makefile.am          | 20 ------------
 c_glib/doc/reference/xml/meson.build          | 31 -------------------
 dev/gen_apidocs/create_documents.sh           |  2 --
 8 files changed, 22 insertions(+), 66 deletions(-)
 rename c_glib/doc/reference/{xml/gtkdocentities.ent.in => entities.xml.in} (76%)
 delete mode 100644 c_glib/doc/reference/xml/Makefile.am
 delete mode 100644 c_glib/doc/reference/xml/meson.build

diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index eabe7bad51227..f4f2c99bbc39e 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -143,7 +143,7 @@ AC_CONFIG_FILES([
   arrow-gpu-glib/arrow-gpu-glib.pc
   doc/Makefile
   doc/reference/Makefile
-  doc/reference/xml/Makefile
+  doc/reference/entities.xml
   example/Makefile
   example/lua/Makefile
   tool/Makefile
diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am
index 4c005c237b300..454c2b0692da6 100644
--- a/c_glib/doc/reference/Makefile.am
+++ b/c_glib/doc/reference/Makefile.am
@@ -15,9 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-SUBDIRS =                                       \
-	xml
-
 DOC_MODULE = arrow-glib
 
 DOC_MAIN_SGML_FILE = $(DOC_MODULE)-docs.xml
@@ -72,4 +69,5 @@ CLEANFILES +=					\
 	$(DOC_MODULE).types
 
 EXTRA_DIST +=					\
+	entities.xml.in				\
 	meson.build
diff --git a/c_glib/doc/reference/arrow-glib-docs.xml b/c_glib/doc/reference/arrow-glib-docs.xml
index 51e7b2a6a6cf5..23d1e9a0f271a 100644
--- a/c_glib/doc/reference/arrow-glib-docs.xml
+++ b/c_glib/doc/reference/arrow-glib-docs.xml
@@ -21,10 +21,10 @@
                "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd"
 [
   <!ENTITY % local.common.attrib "xmlns:xi  CDATA  #FIXED 'http://www.w3.org/2003/XInclude'">
-  <!ENTITY % gtkdocentities SYSTEM "xml/gtkdocentities.ent">
+  <!ENTITY % gtkdocentities SYSTEM "entities.xml">
   %gtkdocentities;
 ]>
-<book id="index">
+<book id="index" xmlns:xi="http://www.w3.org/2003/XInclude">
   <bookinfo>
     <title>&package_name; Reference Manual</title>
     <releaseinfo>
diff --git a/c_glib/doc/reference/xml/gtkdocentities.ent.in b/c_glib/doc/reference/entities.xml.in
similarity index 76%
rename from c_glib/doc/reference/xml/gtkdocentities.ent.in
rename to c_glib/doc/reference/entities.xml.in
index dc0cf1a0d8d4a..aa5addb4e8431 100644
--- a/c_glib/doc/reference/xml/gtkdocentities.ent.in
+++ b/c_glib/doc/reference/entities.xml.in
@@ -16,9 +16,9 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<!ENTITY package "@package@">
-<!ENTITY package_bugreport "@package_bugreport@">
-<!ENTITY package_name "@package_name@">
-<!ENTITY package_string "@package_string@">
-<!ENTITY package_url "@package_url@">
-<!ENTITY package_version "@package_version@">
+<!ENTITY package "@PACKAGE@">
+<!ENTITY package_bugreport "@PACKAGE_BUGREPORT@">
+<!ENTITY package_name "@PACKAGE_NAME@">
+<!ENTITY package_string "@PACKAGE_STRING@">
+<!ENTITY package_url "@PACKAGE_URL@">
+<!ENTITY package_version "@PACKAGE_VERSION@">
diff --git a/c_glib/doc/reference/meson.build b/c_glib/doc/reference/meson.build
index 3374fbde5b9ed..431aa0a5c82a1 100644
--- a/c_glib/doc/reference/meson.build
+++ b/c_glib/doc/reference/meson.build
@@ -17,7 +17,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
-subdir('xml')
+entities_conf = configuration_data()
+entities_conf.set('PACKAGE', meson.project_name())
+entities_conf.set('PACKAGE_BUGREPORT',
+                  'https://issues.apache.org/jira/browse/ARROW')
+entities_conf.set('PACKAGE_NAME', meson.project_name())
+entities_conf.set('PACKAGE_STRING',
+                  ' '.join([meson.project_name(), version]))
+entities_conf.set('PACKAGE_URL', 'https://arrow.apache.org/')
+entities_conf.set('PACKAGE_VERSION', version)
+configure_file(input: 'entities.xml.in',
+               output: 'entities.xml',
+               configuration: entities_conf)
 
 private_headers = [
 ]
diff --git a/c_glib/doc/reference/xml/Makefile.am b/c_glib/doc/reference/xml/Makefile.am
deleted file mode 100644
index 833cfddc69078..0000000000000
--- a/c_glib/doc/reference/xml/Makefile.am
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-EXTRA_DIST =					\
-	gtkdocentities.ent.in			\
-	meson.build
diff --git a/c_glib/doc/reference/xml/meson.build b/c_glib/doc/reference/xml/meson.build
deleted file mode 100644
index 5b65042764fee..0000000000000
--- a/c_glib/doc/reference/xml/meson.build
+++ /dev/null
@@ -1,31 +0,0 @@
-# -*- indent-tabs-mode: nil -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-entities_conf = configuration_data()
-entities_conf.set('package', meson.project_name())
-entities_conf.set('package_bugreport',
-                  'https://issues.apache.org/jira/browse/ARROW')
-entities_conf.set('package_name', meson.project_name())
-entities_conf.set('package_string',
-                  ' '.join([meson.project_name(), version]))
-entities_conf.set('package_url', 'https://arrow.apache.org/')
-entities_conf.set('package_version', version)
-configure_file(input: 'gtkdocentities.ent.in',
-               output: 'gtkdocentities.ent',
-               configuration: entities_conf)
diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh
index 54031262b3a5d..3100d3b984b3a 100755
--- a/dev/gen_apidocs/create_documents.sh
+++ b/dev/gen_apidocs/create_documents.sh
@@ -87,8 +87,6 @@ if [ -f Makefile ]; then
     # Ensure updating to prevent auto re-configure
     touch configure **/Makefile
     make distclean
-    # Work around for 'make distclean' removes doc/reference/xml/
-    git checkout doc/reference/xml
 fi
 ./autogen.sh
 rm -rf build_docs

From 422efd9635ea6f249adec7e1fda4834f6ac46cc4 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 22 Jan 2018 14:13:19 -0500
Subject: [PATCH 1391/1644] ARROW-1580: [Python] Instructions for setting up
 nightly builds on Linux

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1489 from cpcloud/ARROW-1580 and squashes the following commits:

ff815678 [Phillip Cloud] Move to sphinx
700d2c5e [Phillip Cloud] Remove link to nightlies
9fbe9ac9 [Phillip Cloud] Add build artifact location
cb9f2a5a [Phillip Cloud] [Python] Instructions for setting up nightly builds on Linux
---
 python/doc/source/development.rst | 73 +++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 01844fa18d133..af93d8d1a52c4 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -331,3 +331,76 @@ Getting ``python-test.exe`` to run is a bit tricky because your
    set PYTHONPATH=%CONDA_ENV%\Lib;%CONDA_ENV%\Lib\site-packages;%CONDA_ENV%\python35.zip;%CONDA_ENV%\DLLs;%CONDA_ENV%
 
 Now ``python-test.exe`` or simply ``ctest`` (to run all tests) should work.
+
+Nightly Builds of `arrow-cpp`, `parquet-cpp`, and `pyarrow` for Linux
+---------------------------------------------------------------------
+
+Nightly builds of Linux conda packages for ``arrow-cpp``, ``parquet-cpp``, and
+``pyarrow`` can be automated using an open source tool called `scourge
+<https://github.com/cpcloud/scourge>`_.
+
+``scourge`` is new, so please report any feature requests or bugs to the
+`scourge issue tracker <https://github.com/cpcloud/scourge/issues>`_.
+
+To get scourge you need to clone the source and install it in development mode.
+
+To setup your own nightly builds:
+
+#. Clone and install scourge
+#. Create a script that calls scourge
+#. Run that script as a cronjob once per day
+
+First, clone and install scourge (you also need to `install docker
+<https://docs.docker.com/engine/installation>`):
+
+
+.. code:: sh
+
+   git clone https://github.com/cpcloud/scourge
+   cd scourge
+   python setup.py develop
+   which scourge
+
+Second, create a shell script that calls scourge:
+
+.. code:: sh
+
+   function build() {
+     # make sure we got a working directory
+     workingdir="${1}"
+     [ -z "${workingdir}" ] && echo "Must provide a working directory" && exit 1
+     scourge="/path/to/scourge"
+
+     # get the hash of master for building parquet
+     PARQUET_ARROW_VERSION="$("${scourge}" sha apache/arrow master)"
+
+     # setup the build for each package
+     "${scourge}" init arrow-cpp@master parquet-cpp@master pyarrow@master
+
+     # build the packages with some constraints (the -c arguments)
+     # -e sets environment variables on a per package basis
+     "${scourge}" build \
+       -e parquet-cpp:PARQUET_ARROW_VERSION="${PARQUET_ARROW_VERSION}" \
+       -c "python >=2.7,<3|>=3.5" \
+       -c "numpy >= 1.11" \
+       -c "r-base >=3.3.2"
+   }
+
+   workingdir="$(date +'%Y%m%d_%H_%M_%S')"
+   mkdir -p "${workingdir}"
+   build "${workingdir}" > "${workingdir}"/scourge.log 2>&1
+
+Third, run that script as a cronjob once per day:
+
+.. code:: sh
+
+   crontab -e
+
+then in the scratch file that's opened:
+
+.. code:: sh
+
+   @daily /path/to/the/above/script.sh
+
+The build artifacts (conda packages) will be located in
+``${workingdir}/artifacts/linux-64``.

From 72dea17fefde50676489470189c5e0492fd01510 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Tue, 23 Jan 2018 08:53:04 -0500
Subject: [PATCH 1392/1644] ARROW-1997: [C++/Python] Ignore zero-copy-option in
 to_pandas when `strings_to_categorical` is True

This closes [ARROW-1997](https://issues.apache.org/jira/browse/ARROW-1997).

The problem is
```python
>>> import pandas as pd
>>> import pyarrow as pa
>>>
>>> df = pd.DataFrame({
... 'Foo': ['A', 'A', 'B', 'B']
... })
>>> table = pa.Table.from_pandas(df)
>>> df = table.to_pandas(strings_to_categorical=True)
<class 'pandas.core.categorical.Categorical'>
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "table.pxi", line 1043, in pyarrow.lib.Table.to_pandas
  File "pyarrow/pandas_compat.py", line 535, in table_to_blockmanager
    blocks = _table_to_blocks(options, block_table, nthreads, memory_pool)
  File "pyarrow/pandas_compat.py", line 629, in _table_to_blocks
    return [_reconstruct_block(item) for item in result]
  File "pyarrow/pandas_compat.py", line 436, in _reconstruct_block
    ordered=item['ordered'])
  File "/home/rito/miniconda3/envs/pyarrow-dev-27/lib/python2.7/site-packages/pandas/core/categorical.py", line 624, in from_codes
    raise ValueError("codes need to be between -1 and "
ValueError: codes need to be between -1 and len(categories)-1
```

When `strings_to_categorical=True`, the categorical index is newly created in `to_pandas` procedure.
But, this passes data to Python by zero-copy, so the array is deallocated.
https://github.com/Licht-T/arrow/blob/be58af6dd0333652abbe2333ee5968df3f2e371f/cpp/src/arrow/python/arrow_to_pandas.cc#L1040

Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1480 from Licht-T/fix-to_pandas-with-strings_to_categorical and squashes the following commits:

61eac9c1 [Wes McKinney] Adjust error message
c1bc3539 [Licht-T] TST: Add test for to_pandas no-NA strings with strings_to_categorical
cce3f50c [Licht-T] BUG: Avoid zero-copy-option in to_pandas when strings_to_categorical is True
---
 cpp/src/arrow/python/arrow_to_pandas.cc     | 16 ++++++++++++----
 python/pyarrow/tests/test_convert_pandas.py | 21 ++++++++++++++++++++-
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index e21bbda055953..5c8c970e1e058 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -963,7 +963,7 @@ class DatetimeTZBlock : public DatetimeBlock {
 class CategoricalBlock : public PandasBlock {
  public:
   explicit CategoricalBlock(PandasOptions options, MemoryPool* pool, int64_t num_rows)
-      : PandasBlock(options, num_rows, 1), pool_(pool) {}
+      : PandasBlock(options, num_rows, 1), pool_(pool), needs_copy_(false) {}
 
   Status Allocate() override {
     return Status::NotImplemented(
@@ -996,14 +996,20 @@ class CategoricalBlock : public PandasBlock {
       return Status::OK();
     };
 
-    if (data.num_chunks() == 1 && indices_first.null_count() == 0) {
+    if (!needs_copy_ && data.num_chunks() == 1 && indices_first.null_count() == 0) {
       RETURN_NOT_OK(CheckIndices(indices_first, dict_arr_first.dictionary()->length()));
       RETURN_NOT_OK(AllocateNDArrayFromIndices<T>(npy_type, indices_first));
     } else {
       if (options_.zero_copy_only) {
         std::stringstream ss;
-        ss << "Needed to copy " << data.num_chunks() << " chunks with "
-           << indices_first.null_count() << " indices nulls, but zero_copy_only was True";
+        if (needs_copy_) {
+          ss << "Need to allocate categorical memory, "
+             << "but only zero-copy conversions allowed.";
+        } else {
+          ss << "Needed to copy " << data.num_chunks() << " chunks with "
+             << indices_first.null_count()
+             << " indices nulls, but zero_copy_only was True";
+        }
         return Status::Invalid(ss.str());
       }
       RETURN_NOT_OK(AllocateNDArray(npy_type, 1));
@@ -1034,6 +1040,7 @@ class CategoricalBlock : public PandasBlock {
     std::shared_ptr<Column> converted_col;
     if (options_.strings_to_categorical &&
         (col->type()->id() == Type::STRING || col->type()->id() == Type::BINARY)) {
+      needs_copy_ = true;
       compute::FunctionContext ctx(pool_);
 
       Datum out;
@@ -1135,6 +1142,7 @@ class CategoricalBlock : public PandasBlock {
   MemoryPool* pool_;
   OwnedRef dictionary_;
   bool ordered_;
+  bool needs_copy_;
 };
 
 Status MakeBlock(PandasOptions options, PandasBlock::type type, int64_t num_rows,
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 83b1da135eea4..5acb9c3dbe9a1 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1237,7 +1237,22 @@ def test_decimal_metadata(self):
         assert data_column['numpy_type'] == 'object'
         assert data_column['metadata'] == {'precision': 26, 'scale': 11}
 
-    def test_table_str_to_categorical(self):
+    def test_table_str_to_categorical_without_na(self):
+        values = ['a', 'a', 'b', 'b', 'c']
+        df = pd.DataFrame({'strings': values})
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
+        table = pa.Table.from_pandas(df, schema=schema)
+
+        result = table.to_pandas(strings_to_categorical=True)
+        expected = pd.DataFrame({'strings': pd.Categorical(values)})
+        tm.assert_frame_equal(result, expected, check_dtype=True)
+
+        with pytest.raises(pa.ArrowInvalid):
+            table.to_pandas(strings_to_categorical=True,
+                            zero_copy_only=True)
+
+    def test_table_str_to_categorical_with_na(self):
         values = [None, 'a', 'b', np.nan]
         df = pd.DataFrame({'strings': values})
         field = pa.field('strings', pa.string())
@@ -1248,6 +1263,10 @@ def test_table_str_to_categorical(self):
         expected = pd.DataFrame({'strings': pd.Categorical(values)})
         tm.assert_frame_equal(result, expected, check_dtype=True)
 
+        with pytest.raises(pa.ArrowInvalid):
+            table.to_pandas(strings_to_categorical=True,
+                            zero_copy_only=True)
+
     def test_table_batch_empty_dataframe(self):
         df = pd.DataFrame({})
         _check_pandas_roundtrip(df)

From 0930b1d0ed9b649ba3e538a13960c8407ac6bc12 Mon Sep 17 00:00:00 2001
From: yosuke shiro <ys2014hs@gmail.com>
Date: Tue, 23 Jan 2018 09:02:02 -0500
Subject: [PATCH 1393/1644] ARROW-2018: [C++] fix Build instruction on macOS
 and Homebrew

Author: yosuke shiro <ys2014hs@gmail.com>

Closes #1496 from shiro615/build-instruction-on-macos-and-homebrew-is-incomplete and squashes the following commits:

6b32f687 [yosuke shiro] [C++] fix Build instruction on macOS and Homebrew
---
 cpp/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cpp/README.md b/cpp/README.md
index 39a1ccac64818..d2262a68512ce 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -42,6 +42,8 @@ sudo apt-get install cmake \
 On OS X, you can use [Homebrew][1]:
 
 ```shell
+git clone https://github.com/apache/arrow.git
+cd arrow
 brew update && brew bundle --file=c_glib/Brewfile
 ```
 

From 0a490222c4078a7cb0ff085cd5c9884fdda57998 Mon Sep 17 00:00:00 2001
From: Panchen Xue <pan.panchen.xue@gmail.com>
Date: Tue, 23 Jan 2018 22:03:50 -0500
Subject: [PATCH 1394/1644] ARROW-1712: [C++] Add method to BinaryBuilder to
 reserve space for value data

Modified BinaryBuilder::Resize(int64_t) so that when building BinaryArrays with a known size, space is also reserved for value_data_builder_ to prevent internal reallocation.

Author: Panchen Xue <pan.panchen.xue@gmail.com>

Closes #1481 from xuepanchen/master and squashes the following commits:

707b67bf [Panchen Xue] ARROW-1712: [C++] Fix lint errors
360e6018 [Panchen Xue] Merge branch 'master' of https://github.com/xuepanchen/arrow
d4bbd151 [Panchen Xue] ARROW-1712: [C++] Modify test case for BinaryBuilder::ReserveData() and change arguments for offsets_builder_.Resize()
77f8f3c1 [Panchen Xue] Merge pull request #5 from apache/master
bc5db7d3 [Panchen Xue] ARROW-1712: [C++] Remove unneeded data member in BinaryBuilder and modify test case
5a5b70e2 [Panchen Xue] Merge pull request #4 from apache/master
8e4c8925 [Panchen Xue] Merge pull request #3 from xuepanchen/xuepanchen-arrow-1712
d3c8202b [Panchen Xue] ARROW-1945: [C++] Fix a small typo
0b078955 [Panchen Xue] ARROW-1945: [C++] Add data_capacity_ to track capacity of value data
18f90fb8 [Panchen Xue] ARROW-1945: [C++] Add data_capacity_ to track capacity of value data
bbc65270 [Panchen Xue] ARROW-1945: [C++] Update test case for BinaryBuild data value space reservation
15e045cc [Panchen Xue] Add test case for array-test.cc
5a5593e5 [Panchen Xue] Update again ReserveData(int64_t) method for BinaryBuilder
9b5e8059 [Panchen Xue] Update ReserveData(int64_t) method signature for BinaryBuilder
8dd5eaa9 [Panchen Xue] Update builder.cc
b002e0bd [Panchen Xue] Remove override keyword from ReserveData(int64_t) method for BinaryBuilder
de318f47 [Panchen Xue] Implement ReserveData(int64_t) method for BinaryBuilder
e0434e61 [Panchen Xue] Add ReserveData(int64_t) and value_data_capacity() for methods for BinaryBuilder
5ebfb320 [Panchen Xue] Add capacity() method for TypedBufferBuilder
5b73c1c5 [Panchen Xue] Update again BinaryBuilder::Resize(int64_t capacity) in builder.cc
d021c54b [Panchen Xue] Merge pull request #2 from xuepanchen/xuepanchen-arrow-1712
232024e3 [Panchen Xue] Update BinaryBuilder::Resize(int64_t capacity) in builder.cc
c2f8dc4e [Panchen Xue] Merge pull request #1 from apache/master
---
 cpp/src/arrow/array-test.cc | 39 +++++++++++++++++++++++++++++++++++++
 cpp/src/arrow/buffer.h      |  1 +
 cpp/src/arrow/builder.cc    | 18 +++++++++++++----
 cpp/src/arrow/builder.h     |  5 +++++
 4 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 7ff3261ecba5e..c53da8591e94e 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1155,6 +1155,45 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) {
   }
 }
 
+TEST_F(TestBinaryBuilder, TestCapacityReserve) {
+  vector<string> strings = {"aaaaa", "bbbbbbbbbb", "ccccccccccccccc", "dddddddddd"};
+  int N = static_cast<int>(strings.size());
+  int reps = 15;
+  int64_t length = 0;
+  int64_t capacity = 1000;
+  int64_t expected_capacity = BitUtil::RoundUpToMultipleOf64(capacity);
+
+  ASSERT_OK(builder_->ReserveData(capacity));
+
+  ASSERT_EQ(length, builder_->value_data_length());
+  ASSERT_EQ(expected_capacity, builder_->value_data_capacity());
+
+  for (int j = 0; j < reps; ++j) {
+    for (int i = 0; i < N; ++i) {
+      ASSERT_OK(builder_->Append(strings[i]));
+      length += static_cast<int>(strings[i].size());
+
+      ASSERT_EQ(length, builder_->value_data_length());
+      ASSERT_EQ(expected_capacity, builder_->value_data_capacity());
+    }
+  }
+
+  int extra_capacity = 500;
+  expected_capacity = BitUtil::RoundUpToMultipleOf64(length + extra_capacity);
+
+  ASSERT_OK(builder_->ReserveData(extra_capacity));
+
+  ASSERT_EQ(length, builder_->value_data_length());
+  ASSERT_EQ(expected_capacity, builder_->value_data_capacity());
+
+  Done();
+
+  ASSERT_EQ(reps * N, result_->length());
+  ASSERT_EQ(0, result_->null_count());
+  ASSERT_EQ(reps * 40, result_->value_data()->size());
+  ASSERT_EQ(expected_capacity, result_->value_data()->capacity());
+}
+
 TEST_F(TestBinaryBuilder, TestZeroLength) {
   // All buffers are null
   Done();
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index b50b1a1aa041d..44c352a93f273 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -333,6 +333,7 @@ class ARROW_EXPORT TypedBufferBuilder : public BufferBuilder {
 
   const T* data() const { return reinterpret_cast<const T*>(data_); }
   int64_t length() const { return size_ / sizeof(T); }
+  int64_t capacity() const { return capacity_ / sizeof(T); }
 };
 
 /// \brief Allocate a fixed size mutable buffer from a memory pool
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index de132b5f6a0d1..db901526fc2ee 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -1165,13 +1165,13 @@ Status ListBuilder::Init(int64_t elements) {
   DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
   // one more then requested for offsets
-  return offsets_builder_.Resize((elements + 1) * sizeof(int64_t));
+  return offsets_builder_.Resize((elements + 1) * sizeof(int32_t));
 }
 
 Status ListBuilder::Resize(int64_t capacity) {
   DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
   // one more then requested for offsets
-  RETURN_NOT_OK(offsets_builder_.Resize((capacity + 1) * sizeof(int64_t)));
+  RETURN_NOT_OK(offsets_builder_.Resize((capacity + 1) * sizeof(int32_t)));
   return ArrayBuilder::Resize(capacity);
 }
 
@@ -1216,16 +1216,26 @@ Status BinaryBuilder::Init(int64_t elements) {
   DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
   // one more then requested for offsets
-  return offsets_builder_.Resize((elements + 1) * sizeof(int64_t));
+  return offsets_builder_.Resize((elements + 1) * sizeof(int32_t));
 }
 
 Status BinaryBuilder::Resize(int64_t capacity) {
   DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
   // one more then requested for offsets
-  RETURN_NOT_OK(offsets_builder_.Resize((capacity + 1) * sizeof(int64_t)));
+  RETURN_NOT_OK(offsets_builder_.Resize((capacity + 1) * sizeof(int32_t)));
   return ArrayBuilder::Resize(capacity);
 }
 
+Status BinaryBuilder::ReserveData(int64_t elements) {
+  if (value_data_length() + elements > value_data_capacity()) {
+    if (value_data_length() + elements > std::numeric_limits<int32_t>::max()) {
+      return Status::Invalid("Cannot reserve capacity larger than 2^31 - 1 for binary");
+    }
+    RETURN_NOT_OK(value_data_builder_.Reserve(elements));
+  }
+  return Status::OK();
+}
+
 Status BinaryBuilder::AppendNextOffset() {
   const int64_t num_bytes = value_data_builder_.length();
   if (ARROW_PREDICT_FALSE(num_bytes > kMaximumCapacity)) {
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index ce7b8cd197da3..d1611f60cd924 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -682,10 +682,15 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
 
   Status Init(int64_t elements) override;
   Status Resize(int64_t capacity) override;
+  /// \brief Ensures there is enough allocated capacity to append the indicated
+  /// number of bytes to the value data buffer without additional allocations
+  Status ReserveData(int64_t elements);
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
   /// \return size of values buffer so far
   int64_t value_data_length() const { return value_data_builder_.length(); }
+  /// \return capacity of values buffer
+  int64_t value_data_capacity() const { return value_data_builder_.capacity(); }
 
   /// Temporary access to a value.
   ///

From 2126ebf8a755e3ee884058be4aae83585a55107e Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Wed, 24 Jan 2018 20:33:06 -0500
Subject: [PATCH 1395/1644] ARROW-2025: [C++] Creating multiple equivalent
 `HadoopFileSystem`s works fine

Previously creating two instances of `HadoopFileSystem` using the same init parameters would result in both pointing to the same `hdfsFS` object. If one `HadoopFileSystem` disconnected then the underlying `hdfsFS` would be closed for both instances.

To fix this, we force a new instance of `hdfsFS` on connect, removing this cacheing behavior.

Author: Jim Crist <jiminy.crist@gmail.com>

Closes #1499 from jcrist/no-cache-hdfs and squashes the following commits:

f8ff1351 [Jim Crist] Add test
bf6627e8 [Jim Crist] Force libhdfs/libhdfs3 to return new FS on connect
---
 cpp/src/arrow/io/hdfs-internal.cc |  5 +++++
 cpp/src/arrow/io/hdfs-internal.h  |  4 ++++
 cpp/src/arrow/io/hdfs.cc          |  1 +
 cpp/src/arrow/io/io-hdfs-test.cc  | 15 +++++++++++++++
 4 files changed, 25 insertions(+)

diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index 545b2d17d2e78..efceb8ae6b403 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -310,6 +310,10 @@ void LibHdfsShim::BuilderSetKerbTicketCachePath(hdfsBuilder* bld,
   this->hdfsBuilderSetKerbTicketCachePath(bld, kerbTicketCachePath);
 }
 
+void LibHdfsShim::BuilderSetForceNewInstance(hdfsBuilder* bld) {
+  this->hdfsBuilderSetForceNewInstance(bld);
+}
+
 hdfsFS LibHdfsShim::BuilderConnect(hdfsBuilder* bld) {
   return this->hdfsBuilderConnect(bld);
 }
@@ -490,6 +494,7 @@ Status LibHdfsShim::GetRequiredSymbols() {
   GET_SYMBOL_REQUIRED(this, hdfsBuilderSetNameNodePort);
   GET_SYMBOL_REQUIRED(this, hdfsBuilderSetUserName);
   GET_SYMBOL_REQUIRED(this, hdfsBuilderSetKerbTicketCachePath);
+  GET_SYMBOL_REQUIRED(this, hdfsBuilderSetForceNewInstance);
   GET_SYMBOL_REQUIRED(this, hdfsBuilderConnect);
   GET_SYMBOL_REQUIRED(this, hdfsCreateDirectory);
   GET_SYMBOL_REQUIRED(this, hdfsDelete);
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
index df925cf62823a..f0fce23c229ab 100644
--- a/cpp/src/arrow/io/hdfs-internal.h
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -51,6 +51,7 @@ struct LibHdfsShim {
   void (*hdfsBuilderSetUserName)(hdfsBuilder* bld, const char* userName);
   void (*hdfsBuilderSetKerbTicketCachePath)(hdfsBuilder* bld,
                                             const char* kerbTicketCachePath);
+  void (*hdfsBuilderSetForceNewInstance)(hdfsBuilder* bld);
   hdfsFS (*hdfsBuilderConnect)(hdfsBuilder* bld);
 
   int (*hdfsDisconnect)(hdfsFS fs);
@@ -95,6 +96,7 @@ struct LibHdfsShim {
     this->hdfsBuilderSetNameNodePort = nullptr;
     this->hdfsBuilderSetUserName = nullptr;
     this->hdfsBuilderSetKerbTicketCachePath = nullptr;
+    this->hdfsBuilderSetForceNewInstance = nullptr;
     this->hdfsBuilderConnect = nullptr;
     this->hdfsDisconnect = nullptr;
     this->hdfsOpenFile = nullptr;
@@ -138,6 +140,8 @@ struct LibHdfsShim {
 
   void BuilderSetKerbTicketCachePath(hdfsBuilder* bld, const char* kerbTicketCachePath);
 
+  void BuilderSetForceNewInstance(hdfsBuilder* bld);
+
   hdfsFS BuilderConnect(hdfsBuilder* bld);
 
   int Disconnect(hdfsFS fs);
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 6e3e4a7a1c7e7..6c569ae1e2786 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -335,6 +335,7 @@ class HadoopFileSystem::HadoopFileSystemImpl {
     if (!config->kerb_ticket.empty()) {
       driver_->BuilderSetKerbTicketCachePath(builder, config->kerb_ticket.c_str());
     }
+    driver_->BuilderSetForceNewInstance(builder);
     fs_ = driver_->BuilderConnect(builder);
 
     if (fs_ == nullptr) {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 5305b4774624d..f2ded6ff4b945 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -178,6 +178,21 @@ TYPED_TEST(TestHadoopFileSystem, ConnectsAgain) {
   ASSERT_OK(client->Disconnect());
 }
 
+TYPED_TEST(TestHadoopFileSystem, MultipleClients) {
+  SKIP_IF_NO_DRIVER();
+
+  std::shared_ptr<HadoopFileSystem> client1;
+  std::shared_ptr<HadoopFileSystem> client2;
+  ASSERT_OK(HadoopFileSystem::Connect(&this->conf_, &client1));
+  ASSERT_OK(HadoopFileSystem::Connect(&this->conf_, &client2));
+  ASSERT_OK(client1->Disconnect());
+
+  // client2 continues to function after equivalent client1 has shutdown
+  std::vector<HdfsPathInfo> listing;
+  EXPECT_OK(client2->ListDirectory(this->scratch_dir_, &listing));
+  ASSERT_OK(client2->Disconnect());
+}
+
 TYPED_TEST(TestHadoopFileSystem, MakeDirectory) {
   SKIP_IF_NO_DRIVER();
 

From 6bb1d1b35f21ce34327cf92893bda417c2a9a4f1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 25 Jan 2018 10:38:32 -0500
Subject: [PATCH 1396/1644] ARROW-2003: [Python] Remove use of fastpath
 parameter to pandas.core.internals.make_block

Apparently this argument is not used at all in pandas, and the pandas developers wish to simply remove the argument rather than go through a deprecation cycle

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1507 from wesm/ARROW-2003 and squashes the following commits:

a8382262 [Wes McKinney] Remove use of fastpath parameter to pandas.core.internals.make_block
---
 python/pyarrow/pandas_compat.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index f3089d2a012a6..4a30fb3b44a4e 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -435,13 +435,12 @@ def _reconstruct_block(item):
                                         categories=item['dictionary'],
                                         ordered=item['ordered'])
         block = _int.make_block(cat, placement=placement,
-                                klass=_int.CategoricalBlock,
-                                fastpath=True)
+                                klass=_int.CategoricalBlock)
     elif 'timezone' in item:
         dtype = _make_datetimetz(item['timezone'])
         block = _int.make_block(block_arr, placement=placement,
                                 klass=_int.DatetimeTZBlock,
-                                dtype=dtype, fastpath=True)
+                                dtype=dtype)
     else:
         block = _int.make_block(block_arr, placement=placement)
 

From db83fb400d932782ebb32a93582f8ab9cbd1130b Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <pitrou@free.fr>
Date: Thu, 25 Jan 2018 17:07:17 +0100
Subject: [PATCH 1397/1644] [C++] Update README for linting (#1515)

* [C++] Update README for linting

There are hidden gotchas when trying to move the linting Makefile targets,
mention them.

* Mention the LLVM releases page
---
 cpp/README.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/cpp/README.md b/cpp/README.md
index d2262a68512ce..b063248b30af4 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -252,6 +252,24 @@ Logging IWYU to /tmp/arrow-cpp-iwyu.gT7XXV
 ...
 ```
 
+### Linting
+
+We require that you follow a certain coding style in the C++ code base.
+You can check your code abides by that coding style by running:
+
+    make lint
+
+You can also fix any formatting errors automatically:
+
+    make format
+
+These commands require `clang-format-4.0` (and not any other version).
+You may find the required packages at http://releases.llvm.org/download.html
+or use the Debian/Ubuntu APT repositories on https://apt.llvm.org/.
+
+Also, if under a Python 3 environment, you need to install a compatible
+version of `cpplint` using `pip install cpplint`.
+
 ## Continuous Integration
 
 Pull requests are run through travis-ci for continuous integration.  You can avoid

From 68b119b7c290f240c47cf54a2932bfd4794a10f8 Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Thu, 25 Jan 2018 11:37:42 -0500
Subject: [PATCH 1398/1644] ARROW-2029: [Python] NativeFile.tell errors after
 close

Previously checking if the file was closed was subclass specific, and wasn't caught in the hdfs backed file, leading to program crashes.

This adds a check in `NativeFile.tell` for the file being open, and a test on a few subclasses of `NativeFile` to assure the error is raised.

Note that since most python file-like objects raise a `ValueError` for operations after close, I changed the type of the existing error for these cases. This could be changed back, but an error should at least be thrown.

Author: Jim Crist <jiminy.crist@gmail.com>

Closes #1502 from jcrist/hdfs-tell-on-closed and squashes the following commits:

8a9dc947 [Jim Crist] NativeFile.tell errors after close
---
 python/pyarrow/io.pxi           | 13 +++++++------
 python/pyarrow/tests/test_io.py | 26 +++++++++++++++++++++++++-
 2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 5449872ff101f..bb363bacc2e24 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -91,20 +91,20 @@ cdef class NativeFile:
         self._assert_writeable()
         file[0] = <shared_ptr[OutputStream]> self.wr_file
 
+    def _assert_open(self):
+        if not self.is_open:
+            raise ValueError("I/O operation on closed file")
+
     def _assert_readable(self):
+        self._assert_open()
         if not self.is_readable:
             raise IOError("only valid on readonly files")
 
-        if not self.is_open:
-            raise IOError("file not open")
-
     def _assert_writeable(self):
+        self._assert_open()
         if not self.is_writeable:
             raise IOError("only valid on writeable files")
 
-        if not self.is_open:
-            raise IOError("file not open")
-
     def size(self):
         """
         Return file size
@@ -120,6 +120,7 @@ cdef class NativeFile:
         Return current stream position
         """
         cdef int64_t position
+        self._assert_open()
         with nogil:
             if self.is_readable:
                 check_status(self.rd_file.get().Tell(&position))
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index e60dd35de66fe..3f7aa2e1c83bd 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -257,7 +257,7 @@ def test_inmemory_write_after_closed():
     f.write(b'ok')
     f.get_result()
 
-    with pytest.raises(IOError):
+    with pytest.raises(ValueError):
         f.write(b'not ok')
 
 
@@ -503,3 +503,27 @@ def test_native_file_modes(tmpdir):
 
     with pa.memory_map(path, 'r+b') as f:
         assert f.mode == 'rb+'
+
+
+def test_native_file_raises_ValueError_after_close(tmpdir):
+    path = os.path.join(str(tmpdir), guid())
+    with open(path, 'wb') as f:
+        f.write(b'foooo')
+
+    with pa.OSFile(path, mode='rb') as os_file:
+        pass
+
+    with pa.memory_map(path, mode='rb') as mmap_file:
+        pass
+
+    files = [os_file,
+             mmap_file]
+
+    methods = [('tell', ()),
+               ('seek', (0,)),
+               ('size', ())]
+
+    for f in files:
+        for method, args in methods:
+            with pytest.raises(ValueError):
+                getattr(f, method)(*args)

From 1a9d024781e8435e6ae010c55c32c9a9d7fa1e16 Mon Sep 17 00:00:00 2001
From: Sidd <siddharth@dremio.com>
Date: Thu, 25 Jan 2018 10:33:27 -0800
Subject: [PATCH 1399/1644] ARROW-2019: [JAVA] Control the memory allocated for
 inner vector in LIST (#1497)

* ARROW-2019: [JAVA] Control the memory allocated for inner vector in LIST

* address review comments
---
 .../arrow/vector/BaseVariableWidthVector.java | 36 ++++++++++
 .../complex/BaseRepeatedValueVector.java      | 32 +++++++++
 .../arrow/vector/complex/ListVector.java      | 57 ++++++++++++++--
 .../apache/arrow/vector/TestListVector.java   | 68 +++++++++++++++++++
 .../apache/arrow/vector/TestValueVector.java  | 36 ++++++++++
 .../arrow/vector/TestVectorReAlloc.java       |  4 +-
 6 files changed, 224 insertions(+), 9 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index fff329a9b9d66..d1190ceb7b672 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -169,6 +169,42 @@ public void setInitialCapacity(int valueCount) {
     offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH;
   }
 
+  /**
+   * Sets the desired value capacity for the vector. This function doesn't
+   * allocate any memory for the vector.
+   * @param valueCount desired number of elements in the vector
+   * @param density average number of bytes per variable width element
+   */
+  public void setInitialCapacity(int valueCount, double density) {
+    final long size = (long) (valueCount * density);
+    if (size < 1) {
+      throw new IllegalArgumentException("With the provided density and value count, potential capacity of the data buffer is 0");
+    }
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+    }
+    valueAllocationSizeInBytes = (int) size;
+    validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
+    /* to track the end offset of last data element in vector, we need
+     * an additional slot in offset buffer.
+     */
+    offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH;
+  }
+
+  /**
+   * Get the density of this ListVector
+   * @return density
+   */
+  public double getDensity() {
+    if (valueCount == 0) {
+      return 0.0D;
+    }
+    final int startOffset = offsetBuffer.getInt(0);
+    final int endOffset = offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
+    final double totalListSize = endOffset - startOffset;
+    return totalListSize/valueCount;
+  }
+
   /**
    * Get the current value capacity for the vector
    * @return number of elements that vector can hold.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index d0a664ac01da2..50ee3a7573efe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -143,6 +143,38 @@ public void setInitialCapacity(int numRecords) {
     }
   }
 
+  /**
+   * Specialized version of setInitialCapacity() for ListVector. This is
+   * used by some callers when they want to explicitly control and be
+   * conservative about memory allocated for inner data vector. This is
+   * very useful when we are working with memory constraints for a query
+   * and have a fixed amount of memory reserved for the record batch. In
+   * such cases, we are likely to face OOM or related problems when
+   * we reserve memory for a record batch with value count x and
+   * do setInitialCapacity(x) such that each vector allocates only
+   * what is necessary and not the default amount but the multiplier
+   * forces the memory requirement to go beyond what was needed.
+   *
+   * @param numRecords value count
+   * @param density density of ListVector. Density is the average size of
+   *                list per position in the List vector. For example, a
+   *                density value of 10 implies each position in the list
+   *                vector has a list of 10 values.
+   *                A density value of 0.1 implies out of 10 positions in
+   *                the list vector, 1 position has a list of size 1 and
+   *                remaining positions are null (no lists) or empty lists.
+   *                This helps in tightly controlling the memory we provision
+   *                for inner data vector.
+   */
+  public void setInitialCapacity(int numRecords, double density) {
+    offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH;
+    final int innerValueCapacity = (int)(numRecords * density);
+    if (innerValueCapacity < 1) {
+      throw new IllegalArgumentException("With the provided density and value count, potential value capacity for the data vector is 0");
+    }
+    vector.setInitialCapacity(innerValueCapacity);
+  }
+
   @Override
   public int getValueCapacity() {
     final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 8aeeb7e5a2886..b472dae069431 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -31,12 +31,7 @@
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.BufferBacked;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.BitVectorHelper;
+import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
@@ -102,6 +97,54 @@ public void initializeChildrenFromFields(List<Field> children) {
     addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren());
   }
 
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
+    super.setInitialCapacity(numRecords);
+  }
+
+  /**
+   * Specialized version of setInitialCapacity() for ListVector. This is
+   * used by some callers when they want to explicitly control and be
+   * conservative about memory allocated for inner data vector. This is
+   * very useful when we are working with memory constraints for a query
+   * and have a fixed amount of memory reserved for the record batch. In
+   * such cases, we are likely to face OOM or related problems when
+   * we reserve memory for a record batch with value count x and
+   * do setInitialCapacity(x) such that each vector allocates only
+   * what is necessary and not the default amount but the multiplier
+   * forces the memory requirement to go beyond what was needed.
+   *
+   * @param numRecords value count
+   * @param density density of ListVector. Density is the average size of
+   *                list per position in the List vector. For example, a
+   *                density value of 10 implies each position in the list
+   *                vector has a list of 10 values.
+   *                A density value of 0.1 implies out of 10 positions in
+   *                the list vector, 1 position has a list of size 1 and
+   *                remaining positions are null (no lists) or empty lists.
+   *                This helps in tightly controlling the memory we provision
+   *                for inner data vector.
+   */
+  public void setInitialCapacity(int numRecords, double density) {
+    validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
+    super.setInitialCapacity(numRecords, density);
+  }
+
+  /**
+   * Get the density of this ListVector
+   * @return density
+   */
+  public double getDensity() {
+    if (valueCount == 0) {
+      return 0.0D;
+    }
+    final int startOffset = offsetBuffer.getInt(0);
+    final int endOffset = offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
+    final double totalListSize = endOffset - startOffset;
+    return totalListSize/valueCount;
+  }
+
   @Override
   public List<FieldVector> getChildrenFromFields() {
     return singletonList(getDataVector());
@@ -623,7 +666,7 @@ public int getNullCount() {
    */
   @Override
   public int getValueCapacity() {
-    return Math.min(getValidityBufferValueCapacity(), super.getValueCapacity());
+    return getValidityAndOffsetValueCapacity();
   }
 
   private int getValidityAndOffsetValueCapacity() {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index e2023f4461879..d49a677f67922 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -112,6 +112,9 @@ public void testCopyFrom() throws Exception {
       result = outVector.getObject(2);
       resultSet = (ArrayList<Long>) result;
       assertEquals(0, resultSet.size());
+
+      /* 3+0+0/3 */
+      assertEquals(1.0D, inVector.getDensity(), 0);
     }
   }
 
@@ -209,6 +212,9 @@ public void testSetLastSetUsage() throws Exception {
       listVector.setLastSet(3);
       listVector.setValueCount(10);
 
+      /* (3+2+3)/10 */
+      assertEquals(0.8D, listVector.getDensity(), 0);
+
       index = 0;
       offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH);
       assertEquals(Integer.toString(0), Integer.toString(offset));
@@ -709,6 +715,8 @@ public void testGetBufferAddress() throws Exception {
       listWriter.bigInt().writeBigInt(300);
       listWriter.endList();
 
+      listVector.setValueCount(2);
+
       /* check listVector contents */
       Object result = listVector.getObject(0);
       ArrayList<Long> resultSet = (ArrayList<Long>) result;
@@ -739,6 +747,9 @@ public void testGetBufferAddress() throws Exception {
       assertEquals(2, buffers.size());
       assertEquals(bitAddress, buffers.get(0).memoryAddress());
       assertEquals(offsetAddress, buffers.get(1).memoryAddress());
+
+      /* (3+2)/2 */
+      assertEquals(2.5, listVector.getDensity(), 0);
     }
   }
 
@@ -753,4 +764,61 @@ public void testConsistentChildName() throws Exception {
       assertTrue(emptyVectorStr.contains(ListVector.DATA_VECTOR_NAME));
     }
   }
+
+  @Test
+  public void testSetInitialCapacity() {
+    try (final ListVector vector = ListVector.empty("", allocator)) {
+      vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+      /**
+       * use the default multiplier of 5,
+       * 512 * 5 => 2560 * 4 => 10240 bytes => 16KB => 4096 value capacity.
+       */
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+      assertEquals(512, vector.getValueCapacity());
+      assertEquals(4096, vector.getDataVector().getValueCapacity());
+
+      /* use density as 4 */
+      vector.setInitialCapacity(512, 4);
+      vector.allocateNew();
+      assertEquals(512, vector.getValueCapacity());
+      assertEquals(512*4, vector.getDataVector().getValueCapacity());
+
+      /**
+       * inner value capacity we pass to data vector is 512 * 0.1 => 51
+       * For an int vector this is 204 bytes of memory for data buffer
+       * and 7 bytes for validity buffer.
+       * and with power of 2 allocation, we allocate 256 bytes and 8 bytes
+       * for the data buffer and validity buffer of the inner vector. Thus
+       * value capacity of inner vector is 64
+       */
+      vector.setInitialCapacity(512, 0.1);
+      vector.allocateNew();
+      assertEquals(512, vector.getValueCapacity());
+      assertEquals(64, vector.getDataVector().getValueCapacity());
+
+      /**
+       * inner value capacity we pass to data vector is 512 * 0.01 => 5
+       * For an int vector this is 20 bytes of memory for data buffer
+       * and 1 byte for validity buffer.
+       * and with power of 2 allocation, we allocate 32 bytes and 1 bytes
+       * for the data buffer and validity buffer of the inner vector. Thus
+       * value capacity of inner vector is 8
+       */
+      vector.setInitialCapacity(512, 0.01);
+      vector.allocateNew();
+      assertEquals(512, vector.getValueCapacity());
+      assertEquals(8, vector.getDataVector().getValueCapacity());
+
+      boolean error = false;
+      try {
+        vector.setInitialCapacity(5, 0.1);
+      } catch (IllegalArgumentException e) {
+        error = true;
+      } finally {
+        assertTrue(error);
+      }
+    }
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 601b2062ff698..992bb6264a1cf 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -1908,4 +1908,40 @@ public static void setBytes(int index, byte[] bytes, VarCharVector vector) {
     vector.offsetBuffer.setInt((index + 1) * vector.OFFSET_WIDTH, currentOffset + bytes.length);
     vector.valueBuffer.setBytes(currentOffset, bytes, 0, bytes.length);
   }
+
+  @Test /* VarCharVector */
+  public void testSetInitialCapacity() {
+    try (final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
+
+      /* use the default 8 data bytes on average per element */
+      vector.setInitialCapacity(4096);
+      vector.allocateNew();
+      assertEquals(4096, vector.getValueCapacity());
+      assertEquals(4096 * 8, vector.getDataBuffer().capacity());
+
+      vector.setInitialCapacity(4096, 1);
+      vector.allocateNew();
+      assertEquals(4096, vector.getValueCapacity());
+      assertEquals(4096, vector.getDataBuffer().capacity());
+
+      vector.setInitialCapacity(4096, 0.1);
+      vector.allocateNew();
+      assertEquals(4096, vector.getValueCapacity());
+      assertEquals(512, vector.getDataBuffer().capacity());
+
+      vector.setInitialCapacity(4096, 0.01);
+      vector.allocateNew();
+      assertEquals(4096, vector.getValueCapacity());
+      assertEquals(64, vector.getDataBuffer().capacity());
+
+      boolean error = false;
+      try {
+        vector.setInitialCapacity(5, 0.1);
+      } catch (IllegalArgumentException e) {
+        error = true;
+      } finally {
+        assertTrue(error);
+      }
+    }
+  }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index f8edf8904c53e..ca039c52f9715 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -104,7 +104,7 @@ public void testListType() {
       vector.setInitialCapacity(512);
       vector.allocateNew();
 
-      assertEquals(1023, vector.getValueCapacity());
+      assertEquals(512, vector.getValueCapacity());
 
       try {
         vector.getInnerValueCountAt(2014);
@@ -114,7 +114,7 @@ public void testListType() {
       }
 
       vector.reAlloc();
-      assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+      assertEquals(1024, vector.getValueCapacity());
       assertEquals(0, vector.getOffsetBuffer().getInt(2014 * ListVector.OFFSET_WIDTH));
     }
   }

From 8edd62e1bda0bf38f0fce872167be99826d28da5 Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Thu, 25 Jan 2018 17:01:02 -0500
Subject: [PATCH 1400/1644] ARROW-2031: [Python] HadoopFileSystem is pickleable

Adds support for pickling `HadoopFileSystem`

A few additional small fixes:
- Adds a check that `driver` is one of {'libhdfs3', 'libhdfs'}
- A few small tweaks to the hdfs tests to make them easier to run locally.

Author: Jim Crist <jiminy.crist@gmail.com>

Closes #1505 from jcrist/pickle-hdfs-filesystem and squashes the following commits:

b19ed3e0 [Jim Crist] Compat with older cython versions
1f747264 [Jim Crist] HadoopFileSystem is pickleable
---
 python/pyarrow/hdfs.py            |  4 ++++
 python/pyarrow/io-hdfs.pxi        | 20 ++++++++++++++++----
 python/pyarrow/tests/test_hdfs.py | 23 ++++++++++++++++++++---
 3 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/python/pyarrow/hdfs.py b/python/pyarrow/hdfs.py
index 3c9d04188a6ca..3f2014b65c097 100644
--- a/python/pyarrow/hdfs.py
+++ b/python/pyarrow/hdfs.py
@@ -36,6 +36,10 @@ def __init__(self, host="default", port=0, user=None, kerb_ticket=None,
 
         self._connect(host, port, user, kerb_ticket, driver)
 
+    def __reduce__(self):
+        return (HadoopFileSystem, (self.host, self.port, self.user,
+                                   self.kerb_ticket, self.driver))
+
     @implements(FileSystem.isdir)
     def isdir(self, path):
         return super(HadoopFileSystem, self).isdir(path)
diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi
index e653813235862..3abf045f93336 100644
--- a/python/pyarrow/io-hdfs.pxi
+++ b/python/pyarrow/io-hdfs.pxi
@@ -59,29 +59,41 @@ cdef class HadoopFileSystem:
 
     cdef readonly:
         bint is_open
-
-    def __cinit__(self):
-        pass
+        str host
+        str user
+        str kerb_ticket
+        str driver
+        int port
 
     def _connect(self, host, port, user, kerb_ticket, driver):
         cdef HdfsConnectionConfig conf
 
         if host is not None:
             conf.host = tobytes(host)
+        self.host = host
+
         conf.port = port
+        self.port = port
+
         if user is not None:
             conf.user = tobytes(user)
+        self.user = user
+
         if kerb_ticket is not None:
             conf.kerb_ticket = tobytes(kerb_ticket)
+        self.kerb_ticket = kerb_ticket
 
         if driver == 'libhdfs':
             with nogil:
                 check_status(HaveLibHdfs())
             conf.driver = HdfsDriver_LIBHDFS
-        else:
+        elif driver == 'libhdfs3':
             with nogil:
                 check_status(HaveLibHdfs3())
             conf.driver = HdfsDriver_LIBHDFS3
+        else:
+            raise ValueError("unknown driver: %r" % driver)
+        self.driver = driver
 
         with nogil:
             check_status(CHadoopFileSystem.Connect(&conf, &self.client))
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 51b6ba25bd657..b62458cd73689 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -18,6 +18,7 @@
 from io import BytesIO
 from os.path import join as pjoin
 import os
+import pickle
 import random
 import unittest
 
@@ -36,7 +37,7 @@
 
 def hdfs_test_client(driver='libhdfs'):
     host = os.environ.get('ARROW_HDFS_TEST_HOST', 'localhost')
-    user = os.environ['ARROW_HDFS_TEST_USER']
+    user = os.environ.get('ARROW_HDFS_TEST_USER', None)
     try:
         port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 20500))
     except ValueError:
@@ -72,6 +73,22 @@ def tearDownClass(cls):
         cls.hdfs.delete(cls.tmp_path, recursive=True)
         cls.hdfs.close()
 
+    def test_unknown_driver(self):
+        with pytest.raises(ValueError):
+            hdfs_test_client(driver="not_a_driver_name")
+
+    def test_pickle(self):
+        s = pickle.dumps(self.hdfs)
+        h2 = pickle.loads(s)
+        assert h2.is_open
+        assert h2.host == self.hdfs.host
+        assert h2.port == self.hdfs.port
+        assert h2.user == self.hdfs.user
+        assert h2.kerb_ticket == self.hdfs.kerb_ticket
+        assert h2.driver == self.hdfs.driver
+        # smoketest unpickled client works
+        h2.ls(self.tmp_path)
+
     def test_cat(self):
         path = pjoin(self.tmp_path, 'cat-test')
 
@@ -299,7 +316,7 @@ class TestLibHdfs(HdfsTestCases, unittest.TestCase):
     @classmethod
     def check_driver(cls):
         if not pa.have_libhdfs():
-            pytest.fail('No libhdfs available on system')
+            pytest.skip('No libhdfs available on system')
 
     def test_orphaned_file(self):
         hdfs = hdfs_test_client()
@@ -318,4 +335,4 @@ class TestLibHdfs3(HdfsTestCases, unittest.TestCase):
     @classmethod
     def check_driver(cls):
         if not pa.have_libhdfs3():
-            pytest.fail('No libhdfs3 available on system')
+            pytest.skip('No libhdfs3 available on system')

From 51046a0ac80913df99605ca4d78d8561fe3101d5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 25 Jan 2018 23:17:28 +0100
Subject: [PATCH 1401/1644] ARROW-1961: [Python] Preserve pre-existing schema
 metadata in Parquet files when passing flavor='spark'

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1511 from wesm/ARROW-1961 and squashes the following commits:

e13b6b4 [Wes McKinney] Preserve pre-existing schema metadata when sanitizing fields when passing flavor='spark'
---
 python/pyarrow/parquet.py            |  4 +++-
 python/pyarrow/tests/test_parquet.py | 22 ++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 151e0df8a22d0..3a0924a27ceb2 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -215,7 +215,9 @@ def _sanitize_schema(schema, flavor):
                 sanitized_fields.append(sanitized_field)
             else:
                 sanitized_fields.append(field)
-        return pa.schema(sanitized_fields), schema_changed
+
+        new_schema = pa.schema(sanitized_fields, metadata=schema.metadata)
+        return new_schema, schema_changed
     else:
         return schema, False
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index c2bb31c9bcf51..7c2edb378df61 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -748,6 +748,28 @@ def test_sanitized_spark_field_names():
     assert result.schema[0].name == expected_name
 
 
+def _roundtrip_pandas_dataframe(df, write_kwargs):
+    table = pa.Table.from_pandas(df)
+
+    buf = io.BytesIO()
+    _write_table(table, buf, **write_kwargs)
+
+    buf.seek(0)
+    table1 = _read_table(buf)
+    return table1.to_pandas()
+
+
+@parquet
+def test_spark_flavor_preserves_pandas_metadata():
+    df = _test_dataframe(size=100)
+    df.index = np.arange(0, 10 * len(df), 10)
+    df.index.name = 'foo'
+
+    result = _roundtrip_pandas_dataframe(df, {'version': '2.0',
+                                              'flavor': 'spark'})
+    tm.assert_frame_equal(result, df)
+
+
 @parquet
 def test_fixed_size_binary():
     t0 = pa.binary(10)

From bfce44beb918807b17b5c94a6c4efdb3d7ff6e5f Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 25 Jan 2018 23:39:29 +0100
Subject: [PATCH 1402/1644] ARROW-2017: [Python] Use unsigned PyLong API for
 uint64 values over int64 range

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1504 from wesm/ARROW-2017 and squashes the following commits:

56e67dc [Wes McKinney] Use unsigned PyLong API for uint64 values over int64 range
---
 cpp/src/arrow/python/builtin_convert.cc | 2 +-
 python/pyarrow/tests/test_array.py      | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index cd88d557d4830..0879b3f98d770 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -511,7 +511,7 @@ class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Conver
 class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Converter> {
  public:
   Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+    const auto val = static_cast<int64_t>(PyLong_AsUnsignedLongLong(item.obj()));
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index fa38c9257854e..2d991119f85b1 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -485,6 +485,12 @@ def test_logical_type(type, expected):
     assert get_logical_type(type) == expected
 
 
+def test_array_uint64_from_py_over_range():
+    arr = pa.array([2 ** 63], type=pa.uint64())
+    expected = pa.array(np.array([2 ** 63], dtype='u8'))
+    assert arr.equals(expected)
+
+
 def test_array_conversions_no_sentinel_values():
     arr = np.array([1, 2, 3, 4], dtype='int8')
     refcount = sys.getrefcount(arr)

From f680dac68ef5bc911499ae0b62e14c46046816a1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 26 Jan 2018 10:28:08 -0500
Subject: [PATCH 1403/1644] ARROW-2007: [Python] Implement float32 conversions,
 use NumPy dtype when possible for inner arrays

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1509 from wesm/ARROW-2007 and squashes the following commits:

cd12626d [Wes McKinney] Pin thrift-cpp in Appveyor
326c82e1 [Wes McKinney] Pin Thrift 0.10.0 in toolchain
e334f4e2 [Wes McKinney] Add explicit type check
db046597 [Wes McKinney] Implement float32 conversions, use NumPy dtype when possible for inner arrays rather than dispatching to the generic sequence routine
---
 ci/msvc-build.bat                       |  2 +-
 ci/travis_before_script_cpp.sh          |  2 +-
 cpp/src/arrow/python/builtin_convert.cc | 11 +++++++++++
 cpp/src/arrow/python/numpy_to_arrow.cc  | 13 ++++++++++++-
 python/pyarrow/tests/test_array.py      | 17 +++++++++++++++++
 5 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 62ebcf364e77b..94eb16a5e506b 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -81,7 +81,7 @@ conda info -a
 
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython ^
-      thrift-cpp
+      thrift-cpp=0.10.0
 
 if "%JOB%" == "Toolchain" (
 
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index fd2c1644638c4..2f164c4168d0d 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -47,7 +47,7 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
         zlib \
         cmake \
         curl \
-        thrift-cpp \
+        thrift-cpp=0.10.0 \
         ninja
 
   # HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 0879b3f98d770..71f2fde5b3920 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -586,6 +586,15 @@ class TimestampConverter
   TimeUnit::type unit_;
 };
 
+class Float32Converter : public TypedConverterVisitor<FloatBuilder, Float32Converter> {
+ public:
+  Status AppendItem(const OwnedRef& item) {
+    float val = static_cast<float>(PyFloat_AsDouble(item.obj()));
+    RETURN_IF_PYERROR();
+    return typed_builder_->Append(val);
+  }
+};
+
 class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConverter> {
  public:
   Status AppendItem(const OwnedRef& item) {
@@ -740,6 +749,8 @@ std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type
     case Type::TIMESTAMP:
       return std::make_shared<TimestampConverter>(
           static_cast<const TimestampType&>(*type).unit());
+    case Type::FLOAT:
+      return std::make_shared<Float32Converter>();
     case Type::DOUBLE:
       return std::make_shared<DoubleConverter>();
     case Type::BINARY:
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index c5c02e355ded6..b5a75aeedd5eb 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -1008,10 +1008,21 @@ Status NumPyConverter::ConvertObjectsInfer() {
       return ConvertTimes();
     } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
       return ConvertDecimals();
-    } else if (PyList_Check(obj) || PyArray_Check(obj)) {
+    } else if (PyList_Check(obj)) {
       std::shared_ptr<DataType> inferred_type;
       RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
       return ConvertLists(inferred_type);
+    } else if (PyArray_Check(obj)) {
+      std::shared_ptr<DataType> inferred_type;
+      PyArray_Descr* dtype = PyArray_DESCR(reinterpret_cast<PyArrayObject*>(obj));
+
+      if (dtype->type_num == NPY_OBJECT) {
+        RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
+      } else {
+        RETURN_NOT_OK(
+            NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype), &inferred_type));
+      }
+      return ConvertLists(inferred_type);
     } else {
       const std::string supported_types =
           "string, bool, float, int, date, time, decimal, list, array";
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 2d991119f85b1..1d5d30071902a 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -513,6 +513,23 @@ def test_array_from_numpy_datetimeD():
     assert result.equals(expected)
 
 
+def test_array_from_py_float32():
+    data = [[1.2, 3.4], [9.0, 42.0]]
+
+    t = pa.float32()
+
+    arr1 = pa.array(data[0], type=t)
+    arr2 = pa.array(data, type=pa.list_(t))
+
+    expected1 = np.array(data[0], dtype=np.float32)
+    expected2 = pd.Series([np.array(data[0], dtype=np.float32),
+                           np.array(data[1], dtype=np.float32)])
+
+    assert arr1.type == t
+    assert arr1.equals(pa.array(expected1))
+    assert arr2.equals(pa.array(expected2))
+
+
 def test_array_from_numpy_ascii():
     arr = np.array(['abcde', 'abc', ''], dtype='|S5')
 

From a95465b8ce7a32feeaae3e13d0a64102ffa590d9 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 26 Jan 2018 10:30:08 -0500
Subject: [PATCH 1404/1644] ARROW-2035: [C++] Update vendored cpplint.py to a
 Py3-compatible one

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Antoine Pitrou <antoine@python.org>

Closes #1516 from pitrou/ARROW-2035-update-cpplint and squashes the following commits:

8f2f892b [Wes McKinney] Fix IWYU errors surfaced by newer cpplint
f5175dda [Antoine Pitrou] ARROW-2035: [C++] Update vendored cpplint.py to a Py3-compatible one
---
 cpp/README.md                                 |    3 -
 cpp/build-support/cpplint.py                  | 1638 +++++++++--------
 cpp/src/arrow/adapters/orc/adapter.cc         |    1 +
 cpp/src/arrow/array.cc                        |    1 +
 cpp/src/arrow/array.h                         |    1 +
 cpp/src/arrow/builder.cc                      |    1 +
 cpp/src/arrow/compute/context.h               |    2 +
 cpp/src/arrow/compute/kernels/hash.cc         |    1 +
 .../arrow/compute/kernels/util-internal.cc    |    1 +
 cpp/src/arrow/compute/kernels/util-internal.h |    1 +
 cpp/src/arrow/ipc/feather.cc                  |    1 +
 cpp/src/arrow/ipc/reader.cc                   |    1 +
 cpp/src/arrow/pretty_print.cc                 |    1 +
 cpp/src/arrow/python/arrow_to_python.cc       |    1 +
 cpp/src/arrow/python/io.cc                    |    1 +
 cpp/src/arrow/python/io.h                     |    2 +
 cpp/src/arrow/python/numpy_to_arrow.cc        |    1 +
 cpp/src/arrow/record_batch.cc                 |    1 +
 cpp/src/arrow/table.cc                        |    1 +
 cpp/src/arrow/table_builder.cc                |    1 +
 cpp/src/arrow/type.cc                         |    2 +
 cpp/src/arrow/type_traits.h                   |    1 +
 cpp/src/arrow/util/io-util.h                  |    1 +
 cpp/src/plasma/events.cc                      |    2 +
 cpp/src/plasma/plasma.h                       |    1 +
 cpp/src/plasma/protocol.h                     |    2 +
 cpp/src/plasma/store.cc                       |    2 +
 cpp/src/plasma/store.h                        |    2 +
 28 files changed, 928 insertions(+), 746 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index b063248b30af4..ef2e1fd1b1259 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -267,9 +267,6 @@ These commands require `clang-format-4.0` (and not any other version).
 You may find the required packages at http://releases.llvm.org/download.html
 or use the Debian/Ubuntu APT repositories on https://apt.llvm.org/.
 
-Also, if under a Python 3 environment, you need to install a compatible
-version of `cpplint` using `pip install cpplint`.
-
 ## Continuous Integration
 
 Pull requests are run through travis-ci for continuous integration.  You can avoid
diff --git a/cpp/build-support/cpplint.py b/cpp/build-support/cpplint.py
index ccc25d4c56b1a..95c0c32595d81 100755
--- a/cpp/build-support/cpplint.py
+++ b/cpp/build-support/cpplint.py
@@ -44,6 +44,8 @@
 import codecs
 import copy
 import getopt
+import glob
+import itertools
 import math  # for log
 import os
 import re
@@ -51,16 +53,47 @@
 import string
 import sys
 import unicodedata
+import xml.etree.ElementTree
+
+# if empty, use defaults
+_header_extensions = set([])
+
+# if empty, use defaults
+_valid_extensions = set([])
+
+
+# Files with any of these extensions are considered to be
+# header files (and will undergo different style checks).
+# This set can be extended by using the --headers
+# option (also supported in CPPLINT.cfg)
+def GetHeaderExtensions():
+  if not _header_extensions:
+    return set(['h', 'hpp', 'hxx', 'h++', 'cuh'])
+  return _header_extensions
+
+# The allowed extensions for file names
+# This is set by --extensions flag
+def GetAllExtensions():
+  if not _valid_extensions:
+    return GetHeaderExtensions().union(set(['c', 'cc', 'cpp', 'cxx', 'c++', 'cu']))
+  return _valid_extensions
+
+def GetNonHeaderExtensions():
+  return GetAllExtensions().difference(GetHeaderExtensions())
 
 
 _USAGE = """
-Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...]
-                   [--counting=total|toplevel|detailed] [--root=subdir]
-                   [--linelength=digits]
+Syntax: cpplint.py [--verbose=#] [--output=emacs|eclipse|vs7|junit]
+                   [--filter=-x,+y,...]
+                   [--counting=total|toplevel|detailed] [--repository=path]
+                   [--root=subdir] [--linelength=digits] [--recursive]
+                   [--exclude=path]
+                   [--headers=ext1,ext2]
+                   [--extensions=hpp,cpp,...]
         <file> [file] ...
 
   The style guidelines this tries to follow are those in
-    http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
+    https://google.github.io/styleguide/cppguide.html
 
   Every problem is given a confidence score from 1-5, with 5 meaning we are
   certain of the problem, and 1 meaning it could be a legitimate construct.
@@ -71,17 +104,26 @@
   suppresses errors of all categories on that line.
 
   The files passed in will be linted; at least one file must be provided.
-  Default linted extensions are .cc, .cpp, .cu, .cuh and .h.  Change the
-  extensions with the --extensions flag.
+  Default linted extensions are %s.
+  Other file types will be ignored.
+  Change the extensions with the --extensions flag.
 
   Flags:
 
-    output=vs7
-      By default, the output is formatted to ease emacs parsing.  Visual Studio
-      compatible output (vs7) may also be used.  Other formats are unsupported.
+    output=emacs|eclipse|vs7|junit
+      By default, the output is formatted to ease emacs parsing.  Output
+      compatible with eclipse (eclipse), Visual Studio (vs7), and JUnit
+      XML parsers such as those used in Jenkins and Bamboo may also be
+      used.  Other formats are unsupported.
 
     verbose=#
       Specify a number 0-5 to restrict errors to certain verbosity levels.
+      Errors with lower verbosity levels have lower confidence and are more
+      likely to be false positives.
+
+    quiet
+      Supress output other than linting errors, such as information about
+      which files have been processed and excluded.
 
     filter=-x,+y,...
       Specify a comma-separated list of category-filters to apply: only
@@ -105,17 +147,40 @@
       also be printed. If 'detailed' is provided, then a count
       is provided for each category like 'build/class'.
 
+    repository=path
+      The top level directory of the repository, used to derive the header
+      guard CPP variable. By default, this is determined by searching for a
+      path that contains .git, .hg, or .svn. When this flag is specified, the
+      given path is used instead. This option allows the header guard CPP
+      variable to remain consistent even if members of a team have different
+      repository root directories (such as when checking out a subdirectory
+      with SVN). In addition, users of non-mainstream version control systems
+      can use this flag to ensure readable header guard CPP variables.
+
+      Examples:
+        Assuming that Alice checks out ProjectName and Bob checks out
+        ProjectName/trunk and trunk contains src/chrome/ui/browser.h, then
+        with no --repository flag, the header guard CPP variable will be:
+
+        Alice => TRUNK_SRC_CHROME_BROWSER_UI_BROWSER_H_
+        Bob   => SRC_CHROME_BROWSER_UI_BROWSER_H_
+
+        If Alice uses the --repository=trunk flag and Bob omits the flag or
+        uses --repository=. then the header guard CPP variable will be:
+
+        Alice => SRC_CHROME_BROWSER_UI_BROWSER_H_
+        Bob   => SRC_CHROME_BROWSER_UI_BROWSER_H_
+
     root=subdir
-      The root directory used for deriving header guard CPP variable.
-      By default, the header guard CPP variable is calculated as the relative
-      path to the directory that contains .git, .hg, or .svn.  When this flag
-      is specified, the relative path is calculated from the specified
-      directory. If the specified directory does not exist, this flag is
-      ignored.
+      The root directory used for deriving header guard CPP variables. This
+      directory is relative to the top level directory of the repository which
+      by default is determined by searching for a directory that contains .git,
+      .hg, or .svn but can also be controlled with the --repository flag. If
+      the specified directory does not exist, this flag is ignored.
 
       Examples:
-        Assuming that src/.git exists, the header guard CPP variables for
-        src/chrome/browser/ui/browser.h are:
+        Assuming that src is the top level directory of the repository, the
+        header guard CPP variables for src/chrome/browser/ui/browser.h are:
 
         No flag => CHROME_BROWSER_UI_BROWSER_H_
         --root=chrome => BROWSER_UI_BROWSER_H_
@@ -128,11 +193,36 @@
       Examples:
         --linelength=120
 
+    recursive
+      Search for files to lint recursively. Each directory given in the list
+      of files to be linted is replaced by all files that descend from that
+      directory. Files with extensions not in the valid extensions list are
+      excluded.
+
+    exclude=path
+      Exclude the given path from the list of files to be linted. Relative
+      paths are evaluated relative to the current directory and shell globbing
+      is performed. This flag can be provided multiple times to exclude
+      multiple files.
+
+      Examples:
+        --exclude=one.cc
+        --exclude=src/*.cc
+        --exclude=src/*.cc --exclude=test/*.cc
+
     extensions=extension,extension,...
       The allowed file extensions that cpplint will check
 
       Examples:
-        --extensions=hpp,cpp
+        --extensions=%s
+
+    headers=extension,extension,...
+      The allowed header extensions that cpplint will consider to be header files
+      (by default, only files with extensions %s
+      will be assumed to be headers)
+
+      Examples:
+        --headers=%s
 
     cpplint.py supports per-directory configurations specified in CPPLINT.cfg
     files. CPPLINT.cfg file can contain a number of key=value pairs.
@@ -142,6 +232,7 @@
       filter=+filter1,-filter2,...
       exclude_files=regex
       linelength=80
+      root=subdir
 
     "set noparent" option prevents cpplint from traversing directory tree
     upwards looking for more .cfg files in parent directories. This option
@@ -153,22 +244,28 @@
 
     "exclude_files" allows to specify a regular expression to be matched against
     a file name. If the expression matches, the file is skipped and not run
-    through liner.
+    through the linter.
+
+    "linelength" specifies the allowed line length for the project.
 
-    "linelength" allows to specify the allowed line length for the project.
+    The "root" option is similar in function to the --root flag (see example
+    above).
 
     CPPLINT.cfg has an effect on files in the same directory and all
-    sub-directories, unless overridden by a nested configuration file.
+    subdirectories, unless overridden by a nested configuration file.
 
       Example file:
         filter=-build/include_order,+build/include_alpha
-        exclude_files=.*\.cc
+        exclude_files=.*\\.cc
 
     The above example disables build/include_order warning and enables
     build/include_alpha as well as excludes all .cc from being
     processed by linter, in the current directory (where the .cfg
-    file is located) and all sub-directories.
-"""
+    file is located) and all subdirectories.
+""" % (list(GetAllExtensions()),
+       ','.join(list(GetAllExtensions())),
+       GetHeaderExtensions(),
+       ','.join(GetHeaderExtensions()))
 
 # We categorize each error message we print.  Here are the categories.
 # We want an explicit list so we can list them all in cpplint --filter=.
@@ -177,15 +274,19 @@
 _ERROR_CATEGORIES = [
     'build/class',
     'build/c++11',
+    'build/c++14',
+    'build/c++tr1',
     'build/deprecated',
     'build/endif_comment',
     'build/explicit_make_pair',
     'build/forward_decl',
     'build/header_guard',
     'build/include',
+    'build/include_subdir',
     'build/include_alpha',
     'build/include_order',
     'build/include_what_you_use',
+    'build/namespaces_literals',
     'build/namespaces',
     'build/printf_format',
     'build/storage_class',
@@ -196,7 +297,6 @@
     'readability/check',
     'readability/constructors',
     'readability/fn_size',
-    'readability/function',
     'readability/inheritance',
     'readability/multiline_comment',
     'readability/multiline_string',
@@ -227,6 +327,7 @@
     'whitespace/comma',
     'whitespace/comments',
     'whitespace/empty_conditional_body',
+    'whitespace/empty_if_body',
     'whitespace/empty_loop_body',
     'whitespace/end_of_line',
     'whitespace/ending_newline',
@@ -245,6 +346,7 @@
 # compatibility they may still appear in NOLINT comments.
 _LEGACY_ERROR_CATEGORIES = [
     'readability/streams',
+    'readability/function',
     ]
 
 # The default state of the category filter. This is overridden by the --filter=
@@ -253,6 +355,16 @@
 # All entries here should start with a '-' or '+', as in the --filter= flag.
 _DEFAULT_FILTERS = ['-build/include_alpha']
 
+# The default list of categories suppressed for C (not C++) files.
+_DEFAULT_C_SUPPRESSED_CATEGORIES = [
+    'readability/casting',
+    ]
+
+# The default list of categories suppressed for Linux Kernel files.
+_DEFAULT_KERNEL_SUPPRESSED_CATEGORIES = [
+    'whitespace/tab',
+    ]
+
 # We used to check for high-bit characters, but after much discussion we
 # decided those were OK, as long as they were in UTF-8 and didn't represent
 # hard-coded international strings, which belong in a separate i18n file.
@@ -346,6 +458,7 @@
     'random',
     'ratio',
     'regex',
+    'scoped_allocator',
     'set',
     'sstream',
     'stack',
@@ -393,6 +506,19 @@
     'cwctype',
     ])
 
+# Type names
+_TYPES = re.compile(
+    r'^(?:'
+    # [dcl.type.simple]
+    r'(char(16_t|32_t)?)|wchar_t|'
+    r'bool|short|int|long|signed|unsigned|float|double|'
+    # [support.types]
+    r'(ptrdiff_t|size_t|max_align_t|nullptr_t)|'
+    # [cstdint.syn]
+    r'(u?int(_fast|_least)?(8|16|32|64)_t)|'
+    r'(u?int(max|ptr)_t)|'
+    r')$')
+
 
 # These headers are excluded from [build/include] and [build/include_order]
 # checks:
@@ -402,20 +528,23 @@
 _THIRD_PARTY_HEADERS_PATTERN = re.compile(
     r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$')
 
+# Pattern for matching FileInfo.BaseName() against test file name
+_test_suffixes = ['_test', '_regtest', '_unittest']
+_TEST_FILE_SUFFIX = '(' + '|'.join(_test_suffixes) + r')$'
+
+# Pattern that matches only complete whitespace, possibly across multiple lines.
+_EMPTY_CONDITIONAL_BODY_PATTERN = re.compile(r'^\s*$', re.DOTALL)
 
 # Assertion macros.  These are defined in base/logging.h and
-# testing/base/gunit.h.  Note that the _M versions need to come first
-# for substring matching to work.
+# testing/base/public/gunit.h.
 _CHECK_MACROS = [
     'DCHECK', 'CHECK',
-    'EXPECT_TRUE_M', 'EXPECT_TRUE',
-    'ASSERT_TRUE_M', 'ASSERT_TRUE',
-    'EXPECT_FALSE_M', 'EXPECT_FALSE',
-    'ASSERT_FALSE_M', 'ASSERT_FALSE',
+    'EXPECT_TRUE', 'ASSERT_TRUE',
+    'EXPECT_FALSE', 'ASSERT_FALSE',
     ]
 
 # Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE
-_CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS])
+_CHECK_REPLACEMENT = dict([(macro_var, {}) for macro_var in _CHECK_MACROS])
 
 for op, replacement in [('==', 'EQ'), ('!=', 'NE'),
                         ('>=', 'GE'), ('>', 'GT'),
@@ -424,16 +553,12 @@
   _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement
   _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement
   _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement
-  _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement
-  _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement
 
 for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'),
                             ('>=', 'LT'), ('>', 'LE'),
                             ('<=', 'GT'), ('<', 'GE')]:
   _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement
   _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement
-  _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement
-  _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement
 
 # Alternative tokens and their replacements.  For full list, see section 2.5
 # Alternative tokens [lex.digraph] in the C++ standard.
@@ -482,6 +607,12 @@
                         r'(?:\s+(volatile|__volatile__))?'
                         r'\s*[{(]')
 
+# Match strings that indicate we're working on a C (not C++) file.
+_SEARCH_C_FILE = re.compile(r'\b(?:LINT_C_FILE|'
+                            r'vim?:\s*.*(\s*|:)filetype=c(\s*|:|$))')
+
+# Match string that indicates we're working on a Linux Kernel file.
+_SEARCH_KERNEL_FILE = re.compile(r'\b(?:LINT_KERNEL_FILE)')
 
 _regexp_compile_cache = {}
 
@@ -493,16 +624,64 @@
 # This is set by --root flag.
 _root = None
 
+# The top level repository directory. If set, _root is calculated relative to
+# this directory instead of the directory containing version control artifacts.
+# This is set by the --repository flag.
+_repository = None
+
+# Files to exclude from linting. This is set by the --exclude flag.
+_excludes = None
+
+# Whether to supress PrintInfo messages
+_quiet = False
+
 # The allowed line length of files.
 # This is set by --linelength flag.
 _line_length = 80
 
-# The allowed extensions for file names
-# This is set by --extensions flag.
-_valid_extensions = set(['cc', 'h', 'cpp', 'cu', 'cuh'])
+try:
+  xrange(1, 0)
+except NameError:
+  #  -- pylint: disable=redefined-builtin
+  xrange = range
+
+try:
+  unicode
+except NameError:
+  #  -- pylint: disable=redefined-builtin
+  basestring = unicode = str
+
+try:
+  long(2)
+except NameError:
+  #  -- pylint: disable=redefined-builtin
+  long = int
+
+if sys.version_info < (3,):
+  #  -- pylint: disable=no-member
+  # BINARY_TYPE = str
+  itervalues = dict.itervalues
+  iteritems = dict.iteritems
+else:
+  # BINARY_TYPE = bytes
+  itervalues = dict.values
+  iteritems = dict.items
+
+def unicode_escape_decode(x):
+  if sys.version_info < (3,):
+    return codecs.unicode_escape_decode(x)[0]
+  else:
+    return x
+
+# {str, bool}: a map from error categories to booleans which indicate if the
+# category should be suppressed for every line.
+_global_error_suppressions = {}
+
+
+
 
 def ParseNolintSuppressions(filename, raw_line, linenum, error):
-  """Updates the global list of error-suppressions.
+  """Updates the global list of line error-suppressions.
 
   Parses any NOLINT comments on the current line, updating the global
   error_suppressions store.  Reports an error if the NOLINT comment
@@ -533,24 +712,45 @@ def ParseNolintSuppressions(filename, raw_line, linenum, error):
                 'Unknown NOLINT error category: %s' % category)
 
 
+def ProcessGlobalSuppresions(lines):
+  """Updates the list of global error suppressions.
+
+  Parses any lint directives in the file that have global effect.
+
+  Args:
+    lines: An array of strings, each representing a line of the file, with the
+           last element being empty if the file is terminated with a newline.
+  """
+  for line in lines:
+    if _SEARCH_C_FILE.search(line):
+      for category in _DEFAULT_C_SUPPRESSED_CATEGORIES:
+        _global_error_suppressions[category] = True
+    if _SEARCH_KERNEL_FILE.search(line):
+      for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES:
+        _global_error_suppressions[category] = True
+
+
 def ResetNolintSuppressions():
   """Resets the set of NOLINT suppressions to empty."""
   _error_suppressions.clear()
+  _global_error_suppressions.clear()
 
 
 def IsErrorSuppressedByNolint(category, linenum):
   """Returns true if the specified error category is suppressed on this line.
 
   Consults the global error_suppressions map populated by
-  ParseNolintSuppressions/ResetNolintSuppressions.
+  ParseNolintSuppressions/ProcessGlobalSuppresions/ResetNolintSuppressions.
 
   Args:
     category: str, the category of the error.
     linenum: int, the current line number.
   Returns:
-    bool, True iff the error should be suppressed due to a NOLINT comment.
+    bool, True iff the error should be suppressed due to a NOLINT comment or
+    global suppression.
   """
-  return (linenum in _error_suppressions.get(category, set()) or
+  return (_global_error_suppressions.get(category, False) or
+          linenum in _error_suppressions.get(category, set()) or
           linenum in _error_suppressions.get(None, set()))
 
 
@@ -589,6 +789,11 @@ def Search(pattern, s):
   return _regexp_compile_cache[pattern].search(s)
 
 
+def _IsSourceExtension(s):
+  """File extension (excluding dot) matches a source file extension."""
+  return s in GetNonHeaderExtensions()
+
+
 class _IncludeState(object):
   """Tracks line numbers for includes, and the order in which includes appear.
 
@@ -626,6 +831,8 @@ class _IncludeState(object):
 
   def __init__(self):
     self.include_list = [[]]
+    self._section = None
+    self._last_header = None
     self.ResetSection('')
 
   def FindHeader(self, header):
@@ -769,9 +976,16 @@ def __init__(self):
 
     # output format:
     # "emacs" - format that emacs can parse (default)
+    # "eclipse" - format that eclipse can parse
     # "vs7" - format that Microsoft Visual Studio 7 can parse
+    # "junit" - format that Jenkins, Bamboo, etc can parse
     self.output_format = 'emacs'
 
+    # For JUnit output, save errors and failures until the end so that they
+    # can be written into the XML
+    self._junit_errors = []
+    self._junit_failures = []
+
   def SetOutputFormat(self, output_format):
     """Sets the output format for errors."""
     self.output_format = output_format
@@ -840,10 +1054,69 @@ def IncrementErrorCount(self, category):
 
   def PrintErrorCounts(self):
     """Print a summary of errors by category, and the total."""
-    for category, count in self.errors_by_category.iteritems():
-      sys.stderr.write('Category \'%s\' errors found: %d\n' %
+    for category, count in sorted(iteritems(self.errors_by_category)):
+      self.PrintInfo('Category \'%s\' errors found: %d\n' %
                        (category, count))
-    sys.stderr.write('Total errors found: %d\n' % self.error_count)
+    if self.error_count > 0:
+      self.PrintInfo('Total errors found: %d\n' % self.error_count)
+
+  def PrintInfo(self, message):
+    if not _quiet and self.output_format != 'junit':
+      sys.stderr.write(message)
+
+  def PrintError(self, message):
+    if self.output_format == 'junit':
+      self._junit_errors.append(message)
+    else:
+      sys.stderr.write(message)
+
+  def AddJUnitFailure(self, filename, linenum, message, category, confidence):
+    self._junit_failures.append((filename, linenum, message, category,
+        confidence))
+
+  def FormatJUnitXML(self):
+    num_errors = len(self._junit_errors)
+    num_failures = len(self._junit_failures)
+
+    testsuite = xml.etree.ElementTree.Element('testsuite')
+    testsuite.attrib['name'] = 'cpplint'
+    testsuite.attrib['errors'] = str(num_errors)
+    testsuite.attrib['failures'] = str(num_failures)
+
+    if num_errors == 0 and num_failures == 0:
+      testsuite.attrib['tests'] = str(1)
+      xml.etree.ElementTree.SubElement(testsuite, 'testcase', name='passed')
+
+    else:
+      testsuite.attrib['tests'] = str(num_errors + num_failures)
+      if num_errors > 0:
+        testcase = xml.etree.ElementTree.SubElement(testsuite, 'testcase')
+        testcase.attrib['name'] = 'errors'
+        error = xml.etree.ElementTree.SubElement(testcase, 'error')
+        error.text = '\n'.join(self._junit_errors)
+      if num_failures > 0:
+        # Group failures by file
+        failed_file_order = []
+        failures_by_file = {}
+        for failure in self._junit_failures:
+          failed_file = failure[0]
+          if failed_file not in failed_file_order:
+            failed_file_order.append(failed_file)
+            failures_by_file[failed_file] = []
+          failures_by_file[failed_file].append(failure)
+        # Create a testcase for each file
+        for failed_file in failed_file_order:
+          failures = failures_by_file[failed_file]
+          testcase = xml.etree.ElementTree.SubElement(testsuite, 'testcase')
+          testcase.attrib['name'] = failed_file
+          failure = xml.etree.ElementTree.SubElement(testcase, 'failure')
+          template = '{0}: {1} [{2}] [{3}]'
+          texts = [template.format(f[1], f[2], f[3], f[4]) for f in failures]
+          failure.text = '\n'.join(texts)
+
+    xml_decl = '<?xml version="1.0" encoding="UTF-8" ?>\n'
+    return xml_decl + xml.etree.ElementTree.tostring(testsuite, 'utf-8').decode('utf-8')
+
 
 _cpplint_state = _CppLintState()
 
@@ -944,6 +1217,9 @@ def Check(self, error, filename, linenum):
       filename: The name of the current file.
       linenum: The number of the line to check.
     """
+    if not self.in_a_function:
+      return
+
     if Match(r'T(EST|est)', self.current_function):
       base_trigger = self._TEST_TRIGGER
     else:
@@ -986,7 +1262,7 @@ def FullName(self):
     return os.path.abspath(self._filename).replace('\\', '/')
 
   def RepositoryName(self):
-    """FullName after removing the local path to the repository.
+    r"""FullName after removing the local path to the repository.
 
     If we have a real absolute path name here we can try to do something smart:
     detecting the root of the checkout and truncating /path/to/checkout from
@@ -1000,6 +1276,20 @@ def RepositoryName(self):
     if os.path.exists(fullname):
       project_dir = os.path.dirname(fullname)
 
+      # If the user specified a repository path, it exists, and the file is
+      # contained in it, use the specified repository path
+      if _repository:
+        repo = FileInfo(_repository).FullName()
+        root_dir = project_dir
+        while os.path.exists(root_dir):
+          # allow case insensitive compare on Windows
+          if os.path.normcase(root_dir) == os.path.normcase(repo):
+            return os.path.relpath(fullname, root_dir).replace('\\', '/')
+          one_up_dir = os.path.dirname(root_dir)
+          if one_up_dir == root_dir:
+            break
+          root_dir = one_up_dir
+
       if os.path.exists(os.path.join(project_dir, ".svn")):
         # If there's a .svn file in the current directory, we recursively look
         # up the directory tree for the top of the SVN checkout
@@ -1014,12 +1304,13 @@ def RepositoryName(self):
 
       # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by
       # searching up from the current path.
-      root_dir = os.path.dirname(fullname)
-      while (root_dir != os.path.dirname(root_dir) and
-             not os.path.exists(os.path.join(root_dir, ".git")) and
-             not os.path.exists(os.path.join(root_dir, ".hg")) and
-             not os.path.exists(os.path.join(root_dir, ".svn"))):
-        root_dir = os.path.dirname(root_dir)
+      root_dir = current_dir = os.path.dirname(fullname)
+      while current_dir != os.path.dirname(current_dir):
+        if (os.path.exists(os.path.join(current_dir, ".git")) or
+            os.path.exists(os.path.join(current_dir, ".hg")) or
+            os.path.exists(os.path.join(current_dir, ".svn"))):
+          root_dir = current_dir
+        current_dir = os.path.dirname(current_dir)
 
       if (os.path.exists(os.path.join(root_dir, ".git")) or
           os.path.exists(os.path.join(root_dir, ".hg")) or
@@ -1049,7 +1340,7 @@ def BaseName(self):
     return self.Split()[1]
 
   def Extension(self):
-    """File extension - text following the final period."""
+    """File extension - text following the final period, includes that period."""
     return self.Split()[2]
 
   def NoExtension(self):
@@ -1058,7 +1349,7 @@ def NoExtension(self):
 
   def IsSource(self):
     """File has a source file extension."""
-    return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx')
+    return _IsSourceExtension(self.Extension()[1:])
 
 
 def _ShouldPrintError(category, confidence, linenum):
@@ -1114,15 +1405,18 @@ def Error(filename, linenum, category, confidence, message):
   if _ShouldPrintError(category, confidence, linenum):
     _cpplint_state.IncrementErrorCount(category)
     if _cpplint_state.output_format == 'vs7':
-      sys.stderr.write('%s(%s):  %s  [%s] [%d]\n' % (
+      _cpplint_state.PrintError('%s(%s): warning: %s  [%s] [%d]\n' % (
           filename, linenum, message, category, confidence))
     elif _cpplint_state.output_format == 'eclipse':
       sys.stderr.write('%s:%s: warning: %s  [%s] [%d]\n' % (
           filename, linenum, message, category, confidence))
+    elif _cpplint_state.output_format == 'junit':
+        _cpplint_state.AddJUnitFailure(filename, linenum, message, category,
+            confidence)
     else:
-      sys.stderr.write('%s:%s:  %s  [%s] [%d]\n' % (
-          filename, linenum, message, category, confidence))
-
+      final_message = '%s:%s:  %s  [%s] [%d]\n' % (
+          filename, linenum, message, category, confidence)
+      sys.stderr.write(final_message)
 
 # Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard.
 _RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile(
@@ -1204,8 +1498,18 @@ def CleanseRawStrings(raw_lines):
     while delimiter is None:
       # Look for beginning of a raw string.
       # See 2.14.15 [lex.string] for syntax.
-      matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line)
-      if matched:
+      #
+      # Once we have matched a raw string, we check the prefix of the
+      # line to make sure that the line is not part of a single line
+      # comment.  It's done this way because we remove raw strings
+      # before removing comments as opposed to removing comments
+      # before removing raw strings.  This is because there are some
+      # cpplint checks that requires the comments to be preserved, but
+      # we don't want to check comments that are inside raw strings.
+      matched = Match(r'^(.*?)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line)
+      if (matched and
+          not Match(r'^([^\'"]|\'(\\.|[^\'])*\'|"(\\.|[^"])*")*//',
+                    matched.group(1))):
         delimiter = ')' + matched.group(2) + '"'
 
         end = matched.group(3).find(delimiter)
@@ -1624,7 +1928,7 @@ def CheckForCopyright(filename, lines, error):
 
   # We'll say it should occur by line 10. Don't forget there's a
   # dummy line at the front.
-  for line in xrange(1, min(len(lines), 11)):
+  for line in range(1, min(len(lines), 11)):
     if re.search(r'Copyright', lines[line], re.I): break
   else:                       # means no copyright line was found
     error(filename, 0, 'legal/copyright', 5,
@@ -1666,11 +1970,16 @@ def GetHeaderGuardCPPVariable(filename):
   filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename)
   # Replace 'c++' with 'cpp'.
   filename = filename.replace('C++', 'cpp').replace('c++', 'cpp')
-  
+
   fileinfo = FileInfo(filename)
   file_path_from_root = fileinfo.RepositoryName()
   if _root:
-    file_path_from_root = re.sub('^' + _root + os.sep, '', file_path_from_root)
+    suffix = os.sep
+    # On Windows using directory separator will leave us with
+    # "bogus escape error" unless we properly escape regex.
+    if suffix == '\\':
+      suffix += '\\'
+    file_path_from_root = re.sub('^' + _root + suffix, '', file_path_from_root)
   return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_'
 
 
@@ -1697,6 +2006,11 @@ def CheckForHeaderGuard(filename, clean_lines, error):
     if Search(r'//\s*NOLINT\(build/header_guard\)', i):
       return
 
+  # Allow pragma once instead of header guards
+  for i in raw_lines:
+    if Search(r'^\s*#pragma\s+once', i):
+      return
+
   cppvar = GetHeaderGuardCPPVariable(filename)
 
   ifndef = ''
@@ -1773,28 +2087,30 @@ def CheckForHeaderGuard(filename, clean_lines, error):
 
 
 def CheckHeaderFileIncluded(filename, include_state, error):
-  """Logs an error if a .cc file does not include its header."""
+  """Logs an error if a source file does not include its header."""
 
   # Do not check test files
-  if filename.endswith('_test.cc') or filename.endswith('_unittest.cc'):
-    return
-
   fileinfo = FileInfo(filename)
-  headerfile = filename[0:len(filename) - 2] + 'h'
-  if not os.path.exists(headerfile):
+  if Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()):
     return
-  headername = FileInfo(headerfile).RepositoryName()
-  first_include = 0
-  for section_list in include_state.include_list:
-    for f in section_list:
-      if headername in f[0] or f[0] in headername:
-        return
-      if not first_include:
-        first_include = f[1]
 
-  error(filename, first_include, 'build/include', 5,
-        '%s should include its header file %s' % (fileinfo.RepositoryName(),
-                                                  headername))
+  for ext in GetHeaderExtensions():
+      basefilename = filename[0:len(filename) - len(fileinfo.Extension())]
+      headerfile = basefilename + '.' + ext
+      if not os.path.exists(headerfile):
+        continue
+      headername = FileInfo(headerfile).RepositoryName()
+      first_include = None
+      for section_list in include_state.include_list:
+        for f in section_list:
+          if headername in f[0] or f[0] in headername:
+            return
+          if not first_include:
+            first_include = f[1]
+
+      error(filename, first_include, 'build/include', 5,
+            '%s should include its header file %s' % (fileinfo.RepositoryName(),
+                                                      headername))
 
 
 def CheckForBadCharacters(filename, lines, error):
@@ -1815,7 +2131,7 @@ def CheckForBadCharacters(filename, lines, error):
     error: The function to call with any errors found.
   """
   for linenum, line in enumerate(lines):
-    if u'\ufffd' in line:
+    if unicode_escape_decode('\ufffd') in line:
       error(filename, linenum, 'readability/utf8', 5,
             'Line contains invalid UTF-8 (or Unicode replacement character).')
     if '\0' in line:
@@ -1997,7 +2313,8 @@ def IsForwardClassDeclaration(clean_lines, linenum):
 class _BlockInfo(object):
   """Stores information about a generic block of code."""
 
-  def __init__(self, seen_open_brace):
+  def __init__(self, linenum, seen_open_brace):
+    self.starting_linenum = linenum
     self.seen_open_brace = seen_open_brace
     self.open_parentheses = 0
     self.inline_asm = _NO_ASM
@@ -2046,17 +2363,16 @@ def IsBlockInfo(self):
 class _ExternCInfo(_BlockInfo):
   """Stores information about an 'extern "C"' block."""
 
-  def __init__(self):
-    _BlockInfo.__init__(self, True)
+  def __init__(self, linenum):
+    _BlockInfo.__init__(self, linenum, True)
 
 
 class _ClassInfo(_BlockInfo):
   """Stores information about a class."""
 
   def __init__(self, name, class_or_struct, clean_lines, linenum):
-    _BlockInfo.__init__(self, False)
+    _BlockInfo.__init__(self, linenum, False)
     self.name = name
-    self.starting_linenum = linenum
     self.is_derived = False
     self.check_namespace_indentation = True
     if class_or_struct == 'struct':
@@ -2124,9 +2440,8 @@ class _NamespaceInfo(_BlockInfo):
   """Stores information about a namespace."""
 
   def __init__(self, name, linenum):
-    _BlockInfo.__init__(self, False)
+    _BlockInfo.__init__(self, linenum, False)
     self.name = name or ''
-    self.starting_linenum = linenum
     self.check_namespace_indentation = True
 
   def CheckEnd(self, filename, clean_lines, linenum, error):
@@ -2145,7 +2460,7 @@ def CheckEnd(self, filename, clean_lines, linenum, error):
     # deciding what these nontrivial things are, so this check is
     # triggered by namespace size only, which works most of the time.
     if (linenum - self.starting_linenum < 10
-        and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)):
+        and not Match(r'^\s*};*\s*(//|/\*).*\bnamespace\b', line)):
       return
 
     # Look for matching comment at end of namespace.
@@ -2162,18 +2477,18 @@ def CheckEnd(self, filename, clean_lines, linenum, error):
     # expected namespace.
     if self.name:
       # Named namespace
-      if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) +
-                    r'[\*/\.\\\s]*$'),
+      if not Match((r'^\s*};*\s*(//|/\*).*\bnamespace\s+' +
+                    re.escape(self.name) + r'[\*/\.\\\s]*$'),
                    line):
         error(filename, linenum, 'readability/namespace', 5,
               'Namespace should be terminated with "// namespace %s"' %
               self.name)
     else:
       # Anonymous namespace
-      if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line):
+      if not Match(r'^\s*};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line):
         # If "// namespace anonymous" or "// anonymous namespace (more text)",
         # mention "// anonymous namespace" as an acceptable form
-        if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', line):
+        if Match(r'^\s*}.*\b(namespace anonymous|anonymous namespace)\b', line):
           error(filename, linenum, 'readability/namespace', 5,
                 'Anonymous namespace should be terminated with "// namespace"'
                 ' or "// anonymous namespace"')
@@ -2445,7 +2760,7 @@ def Update(self, filename, clean_lines, linenum, error):
     #   class LOCKABLE API Object {
     #   };
     class_decl_match = Match(
-        r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?'
+        r'^(\s*(?:template\s*<[\w\s<>,:=]*>\s*)?'
         r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))'
         r'(.*)$', line)
     if (class_decl_match and
@@ -2512,9 +2827,9 @@ def Update(self, filename, clean_lines, linenum, error):
         if not self.SeenOpenBrace():
           self.stack[-1].seen_open_brace = True
         elif Match(r'^extern\s*"[^"]*"\s*\{', line):
-          self.stack.append(_ExternCInfo())
+          self.stack.append(_ExternCInfo(linenum))
         else:
-          self.stack.append(_BlockInfo(True))
+          self.stack.append(_BlockInfo(linenum, True))
           if _MATCH_ASM.match(line):
             self.stack[-1].inline_asm = _BLOCK_ASM
 
@@ -2626,7 +2941,8 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum,
             r'\s+(register|static|extern|typedef)\b',
             line):
     error(filename, linenum, 'build/storage_class', 5,
-          'Storage class (static, extern, typedef, etc) should be first.')
+          'Storage-class specifier (static, extern, typedef, etc) should be '
+          'at the beginning of the declaration.')
 
   if Match(r'\s*#\s*endif\s*[^/\s]+', line):
     error(filename, linenum, 'build/endif_comment', 5,
@@ -2665,9 +2981,7 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum,
   base_classname = classinfo.name.split('::')[-1]
 
   # Look for single-argument constructors that aren't marked explicit.
-  # Technically a valid construct, but against style. Also look for
-  # non-single-argument constructors which are also technically valid, but
-  # strongly suggest something is wrong.
+  # Technically a valid construct, but against style.
   explicit_constructor_match = Match(
       r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*'
       r'\(((?:[^()]|\([^()]*\))*)\)'
@@ -2694,6 +3008,7 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum,
       constructor_args[i] = constructor_arg
       i += 1
 
+    variadic_args = [arg for arg in constructor_args if '&&...' in arg]
     defaulted_args = [arg for arg in constructor_args if '=' in arg]
     noarg_constructor = (not constructor_args or  # empty arg list
                          # 'void' arg specifier
@@ -2704,7 +3019,10 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum,
                           # all but at most one arg defaulted
                           (len(constructor_args) >= 1 and
                            not noarg_constructor and
-                           len(defaulted_args) >= len(constructor_args) - 1))
+                           len(defaulted_args) >= len(constructor_args) - 1) or
+                          # variadic arguments with zero or one argument
+                          (len(constructor_args) <= 2 and
+                           len(variadic_args) >= 1))
     initializer_list_constructor = bool(
         onearg_constructor and
         Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0]))
@@ -2717,7 +3035,7 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum,
         onearg_constructor and
         not initializer_list_constructor and
         not copy_constructor):
-      if defaulted_args:
+      if defaulted_args or variadic_args:
         error(filename, linenum, 'runtime/explicit', 5,
               'Constructors callable with one argument '
               'should be marked explicit.')
@@ -2728,10 +3046,6 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum,
       if noarg_constructor:
         error(filename, linenum, 'runtime/explicit', 5,
               'Zero-parameter constructors should not be marked explicit.')
-      else:
-        error(filename, linenum, 'runtime/explicit', 0,
-              'Constructors that require multiple arguments '
-              'should not be marked explicit.')
 
 
 def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error):
@@ -2786,6 +3100,7 @@ def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error):
       error(filename, linenum, 'whitespace/parens', 2,
             'Extra space after (')
     if (Search(r'\w\s+\(', fncall) and
+        not Search(r'_{0,2}asm_{0,2}\s+_{0,2}volatile_{0,2}\s+\(', fncall) and
         not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and
         not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and
         not Search(r'\bcase\s+\(', fncall)):
@@ -2844,7 +3159,7 @@ def CheckForFunctionLengths(filename, clean_lines, linenum,
   """Reports for long function bodies.
 
   For an overview why this is done, see:
-  http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions
+  https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions
 
   Uses a simplistic algorithm assuming other style guidelines
   (especially spacing) are followed.
@@ -2879,7 +3194,7 @@ def CheckForFunctionLengths(filename, clean_lines, linenum,
 
   if starting_func:
     body_found = False
-    for start_linenum in xrange(linenum, clean_lines.NumLines()):
+    for start_linenum in range(linenum, clean_lines.NumLines()):
       start_line = lines[start_linenum]
       joined_line += ' ' + start_line.lstrip()
       if Search(r'(;|})', start_line):  # Declarations and trivial functions
@@ -2923,9 +3238,7 @@ def CheckComment(line, filename, linenum, next_line_start, error):
   commentpos = line.find('//')
   if commentpos != -1:
     # Check if the // may be in quotes.  If so, ignore it
-    # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison
-    if (line.count('"', 0, commentpos) -
-        line.count('\\"', 0, commentpos)) % 2 == 0:   # not in quotes
+    if re.sub(r'\\.', '', line[0:commentpos]).count('"') % 2 == 0:
       # Allow one space for new scopes, two spaces otherwise:
       if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and
           ((commentpos >= 1 and
@@ -3174,8 +3487,8 @@ def CheckOperatorSpacing(filename, clean_lines, linenum, error):
   # macro context and don't do any checks.  This avoids false
   # positives.
   #
-  # Note that && is not included here.  Those are checked separately
-  # in CheckRValueReference
+  # Note that && is not included here.  This is because there are too
+  # many false positives due to RValue references.
   match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line)
   if match:
     error(filename, linenum, 'whitespace/operators', 3,
@@ -3209,7 +3522,7 @@ def CheckOperatorSpacing(filename, clean_lines, linenum, error):
   #
   # We also allow operators following an opening parenthesis, since
   # those tend to be macros that deal with operators.
-  match = Search(r'(operator|[^\s(<])(?:L|UL|ULL|l|ul|ull)?<<([^\s,=<])', line)
+  match = Search(r'(operator|[^\s(<])(?:L|UL|LL|ULL|l|ul|ll|ull)?<<([^\s,=<])', line)
   if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and
       not (match.group(1) == 'operator' and match.group(2) == ';')):
     error(filename, linenum, 'whitespace/operators', 3,
@@ -3313,22 +3626,90 @@ def CheckCommaSpacing(filename, clean_lines, linenum, error):
           'Missing space after ;')
 
 
-def CheckBracesSpacing(filename, clean_lines, linenum, error):
+def _IsType(clean_lines, nesting_state, expr):
+  """Check if expression looks like a type name, returns true if so.
+
+  Args:
+    clean_lines: A CleansedLines instance containing the file.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
+    expr: The expression to check.
+  Returns:
+    True, if token looks like a type.
+  """
+  # Keep only the last token in the expression
+  last_word = Match(r'^.*(\b\S+)$', expr)
+  if last_word:
+    token = last_word.group(1)
+  else:
+    token = expr
+
+  # Match native types and stdint types
+  if _TYPES.match(token):
+    return True
+
+  # Try a bit harder to match templated types.  Walk up the nesting
+  # stack until we find something that resembles a typename
+  # declaration for what we are looking for.
+  typename_pattern = (r'\b(?:typename|class|struct)\s+' + re.escape(token) +
+                      r'\b')
+  block_index = len(nesting_state.stack) - 1
+  while block_index >= 0:
+    if isinstance(nesting_state.stack[block_index], _NamespaceInfo):
+      return False
+
+    # Found where the opening brace is.  We want to scan from this
+    # line up to the beginning of the function, minus a few lines.
+    #   template <typename Type1,  // stop scanning here
+    #             ...>
+    #   class C
+    #     : public ... {  // start scanning here
+    last_line = nesting_state.stack[block_index].starting_linenum
+
+    next_block_start = 0
+    if block_index > 0:
+      next_block_start = nesting_state.stack[block_index - 1].starting_linenum
+    first_line = last_line
+    while first_line >= next_block_start:
+      if clean_lines.elided[first_line].find('template') >= 0:
+        break
+      first_line -= 1
+    if first_line < next_block_start:
+      # Didn't find any "template" keyword before reaching the next block,
+      # there are probably no template things to check for this block
+      block_index -= 1
+      continue
+
+    # Look for typename in the specified range
+    for i in xrange(first_line, last_line + 1, 1):
+      if Search(typename_pattern, clean_lines.elided[i]):
+        return True
+    block_index -= 1
+
+  return False
+
+
+def CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error):
   """Checks for horizontal spacing near commas.
 
   Args:
     filename: The name of the current file.
     clean_lines: A CleansedLines instance containing the file.
     linenum: The number of the line to check.
+    nesting_state: A NestingState instance which maintains information about
+                   the current stack of nested blocks being parsed.
     error: The function to call with any errors found.
   """
   line = clean_lines.elided[linenum]
 
   # Except after an opening paren, or after another opening brace (in case of
   # an initializer list, for instance), you should have spaces before your
-  # braces. And since you should never have braces at the beginning of a line,
-  # this is an easy test.
+  # braces when they are delimiting blocks, classes, namespaces etc.
+  # And since you should never have braces at the beginning of a line,
+  # this is an easy test.  Except that braces used for initialization don't
+  # follow the same rule; we often don't want spaces before those.
   match = Match(r'^(.*[^ ({>]){', line)
+
   if match:
     # Try a bit harder to check for brace initialization.  This
     # happens in one of the following forms:
@@ -3358,6 +3739,7 @@ def CheckBracesSpacing(filename, clean_lines, linenum, error):
     # There is a false negative with this approach if people inserted
     # spurious semicolons, e.g. "if (cond){};", but we will catch the
     # spurious semicolon with a separate check.
+    leading_text = match.group(1)
     (endline, endlinenum, endpos) = CloseExpression(
         clean_lines, linenum, len(match.group(1)))
     trailing_text = ''
@@ -3366,7 +3748,11 @@ def CheckBracesSpacing(filename, clean_lines, linenum, error):
     for offset in xrange(endlinenum + 1,
                          min(endlinenum + 3, clean_lines.NumLines() - 1)):
       trailing_text += clean_lines.elided[offset]
-    if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text):
+    # We also suppress warnings for `uint64_t{expression}` etc., as the style
+    # guide recommends brace initialization for integral types to avoid
+    # overflow/truncation.
+    if (not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text)
+        and not _IsType(clean_lines, nesting_state, leading_text)):
       error(filename, linenum, 'whitespace/braces', 5,
             'Missing space before {')
 
@@ -3409,406 +3795,6 @@ def IsDecltype(clean_lines, linenum, column):
     return True
   return False
 
-
-def IsTemplateParameterList(clean_lines, linenum, column):
-  """Check if the token ending on (linenum, column) is the end of template<>.
-
-  Args:
-    clean_lines: A CleansedLines instance containing the file.
-    linenum: the number of the line to check.
-    column: end column of the token to check.
-  Returns:
-    True if this token is end of a template parameter list, False otherwise.
-  """
-  (_, startline, startpos) = ReverseCloseExpression(
-      clean_lines, linenum, column)
-  if (startpos > -1 and
-      Search(r'\btemplate\s*$', clean_lines.elided[startline][0:startpos])):
-    return True
-  return False
-
-
-def IsRValueType(typenames, clean_lines, nesting_state, linenum, column):
-  """Check if the token ending on (linenum, column) is a type.
-
-  Assumes that text to the right of the column is "&&" or a function
-  name.
-
-  Args:
-    typenames: set of type names from template-argument-list.
-    clean_lines: A CleansedLines instance containing the file.
-    nesting_state: A NestingState instance which maintains information about
-                   the current stack of nested blocks being parsed.
-    linenum: the number of the line to check.
-    column: end column of the token to check.
-  Returns:
-    True if this token is a type, False if we are not sure.
-  """
-  prefix = clean_lines.elided[linenum][0:column]
-
-  # Get one word to the left.  If we failed to do so, this is most
-  # likely not a type, since it's unlikely that the type name and "&&"
-  # would be split across multiple lines.
-  match = Match(r'^(.*)(\b\w+|[>*)&])\s*$', prefix)
-  if not match:
-    return False
-
-  # Check text following the token.  If it's "&&>" or "&&," or "&&...", it's
-  # most likely a rvalue reference used inside a template.
-  suffix = clean_lines.elided[linenum][column:]
-  if Match(r'&&\s*(?:[>,]|\.\.\.)', suffix):
-    return True
-
-  # Check for known types and end of templates:
-  #   int&& variable
-  #   vector<int>&& variable
-  #
-  # Because this function is called recursively, we also need to
-  # recognize pointer and reference types:
-  #   int* Function()
-  #   int& Function()
-  if (match.group(2) in typenames or
-      match.group(2) in ['char', 'char16_t', 'char32_t', 'wchar_t', 'bool',
-                         'short', 'int', 'long', 'signed', 'unsigned',
-                         'float', 'double', 'void', 'auto', '>', '*', '&']):
-    return True
-
-  # If we see a close parenthesis, look for decltype on the other side.
-  # decltype would unambiguously identify a type, anything else is
-  # probably a parenthesized expression and not a type.
-  if match.group(2) == ')':
-    return IsDecltype(
-        clean_lines, linenum, len(match.group(1)) + len(match.group(2)) - 1)
-
-  # Check for casts and cv-qualifiers.
-  #   match.group(1)  remainder
-  #   --------------  ---------
-  #   const_cast<     type&&
-  #   const           type&&
-  #   type            const&&
-  if Search(r'\b(?:const_cast\s*<|static_cast\s*<|dynamic_cast\s*<|'
-            r'reinterpret_cast\s*<|\w+\s)\s*$',
-            match.group(1)):
-    return True
-
-  # Look for a preceding symbol that might help differentiate the context.
-  # These are the cases that would be ambiguous:
-  #   match.group(1)  remainder
-  #   --------------  ---------
-  #   Call         (   expression &&
-  #   Declaration  (   type&&
-  #   sizeof       (   type&&
-  #   if           (   expression &&
-  #   while        (   expression &&
-  #   for          (   type&&
-  #   for(         ;   expression &&
-  #   statement    ;   type&&
-  #   block        {   type&&
-  #   constructor  {   expression &&
-  start = linenum
-  line = match.group(1)
-  match_symbol = None
-  while start >= 0:
-    # We want to skip over identifiers and commas to get to a symbol.
-    # Commas are skipped so that we can find the opening parenthesis
-    # for function parameter lists.
-    match_symbol = Match(r'^(.*)([^\w\s,])[\w\s,]*$', line)
-    if match_symbol:
-      break
-    start -= 1
-    line = clean_lines.elided[start]
-
-  if not match_symbol:
-    # Probably the first statement in the file is an rvalue reference
-    return True
-
-  if match_symbol.group(2) == '}':
-    # Found closing brace, probably an indicate of this:
-    #   block{} type&&
-    return True
-
-  if match_symbol.group(2) == ';':
-    # Found semicolon, probably one of these:
-    #   for(; expression &&
-    #   statement; type&&
-
-    # Look for the previous 'for(' in the previous lines.
-    before_text = match_symbol.group(1)
-    for i in xrange(start - 1, max(start - 6, 0), -1):
-      before_text = clean_lines.elided[i] + before_text
-    if Search(r'for\s*\([^{};]*$', before_text):
-      # This is the condition inside a for-loop
-      return False
-
-    # Did not find a for-init-statement before this semicolon, so this
-    # is probably a new statement and not a condition.
-    return True
-
-  if match_symbol.group(2) == '{':
-    # Found opening brace, probably one of these:
-    #   block{ type&& = ... ; }
-    #   constructor{ expression && expression }
-
-    # Look for a closing brace or a semicolon.  If we see a semicolon
-    # first, this is probably a rvalue reference.
-    line = clean_lines.elided[start][0:len(match_symbol.group(1)) + 1]
-    end = start
-    depth = 1
-    while True:
-      for ch in line:
-        if ch == ';':
-          return True
-        elif ch == '{':
-          depth += 1
-        elif ch == '}':
-          depth -= 1
-          if depth == 0:
-            return False
-      end += 1
-      if end >= clean_lines.NumLines():
-        break
-      line = clean_lines.elided[end]
-    # Incomplete program?
-    return False
-
-  if match_symbol.group(2) == '(':
-    # Opening parenthesis.  Need to check what's to the left of the
-    # parenthesis.  Look back one extra line for additional context.
-    before_text = match_symbol.group(1)
-    if linenum > 1:
-      before_text = clean_lines.elided[linenum - 1] + before_text
-    before_text = match_symbol.group(1)
-
-    # Patterns that are likely to be types:
-    #   [](type&&
-    #   for (type&&
-    #   sizeof(type&&
-    #   operator=(type&&
-    #
-    if Search(r'(?:\]|\bfor|\bsizeof|\boperator\s*\S+\s*)\s*$', before_text):
-      return True
-
-    # Patterns that are likely to be expressions:
-    #   if (expression &&
-    #   while (expression &&
-    #   : initializer(expression &&
-    #   , initializer(expression &&
-    #   ( FunctionCall(expression &&
-    #   + FunctionCall(expression &&
-    #   + (expression &&
-    #
-    # The last '+' represents operators such as '+' and '-'.
-    if Search(r'(?:\bif|\bwhile|[-+=%^(<!?:,&*]\s*)$', before_text):
-      return False
-
-    # Something else.  Check that tokens to the left look like
-    #   return_type function_name
-    match_func = Match(r'^(.*\S.*)\s+\w(?:\w|::)*(?:<[^<>]*>)?\s*$',
-                       match_symbol.group(1))
-    if match_func:
-      # Check for constructors, which don't have return types.
-      if Search(r'\b(?:explicit|inline)$', match_func.group(1)):
-        return True
-      implicit_constructor = Match(r'\s*(\w+)\((?:const\s+)?(\w+)', prefix)
-      if (implicit_constructor and
-          implicit_constructor.group(1) == implicit_constructor.group(2)):
-        return True
-      return IsRValueType(typenames, clean_lines, nesting_state, linenum,
-                          len(match_func.group(1)))
-
-    # Nothing before the function name.  If this is inside a block scope,
-    # this is probably a function call.
-    return not (nesting_state.previous_stack_top and
-                nesting_state.previous_stack_top.IsBlockInfo())
-
-  if match_symbol.group(2) == '>':
-    # Possibly a closing bracket, check that what's on the other side
-    # looks like the start of a template.
-    return IsTemplateParameterList(
-        clean_lines, start, len(match_symbol.group(1)))
-
-  # Some other symbol, usually something like "a=b&&c".  This is most
-  # likely not a type.
-  return False
-
-
-def IsDeletedOrDefault(clean_lines, linenum):
-  """Check if current constructor or operator is deleted or default.
-
-  Args:
-    clean_lines: A CleansedLines instance containing the file.
-    linenum: The number of the line to check.
-  Returns:
-    True if this is a deleted or default constructor.
-  """
-  open_paren = clean_lines.elided[linenum].find('(')
-  if open_paren < 0:
-    return False
-  (close_line, _, close_paren) = CloseExpression(
-      clean_lines, linenum, open_paren)
-  if close_paren < 0:
-    return False
-  return Match(r'\s*=\s*(?:delete|default)\b', close_line[close_paren:])
-
-
-def IsRValueAllowed(clean_lines, linenum, typenames):
-  """Check if RValue reference is allowed on a particular line.
-
-  Args:
-    clean_lines: A CleansedLines instance containing the file.
-    linenum: The number of the line to check.
-    typenames: set of type names from template-argument-list.
-  Returns:
-    True if line is within the region where RValue references are allowed.
-  """
-  # Allow region marked by PUSH/POP macros
-  for i in xrange(linenum, 0, -1):
-    line = clean_lines.elided[i]
-    if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line):
-      if not line.endswith('PUSH'):
-        return False
-      for j in xrange(linenum, clean_lines.NumLines(), 1):
-        line = clean_lines.elided[j]
-        if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line):
-          return line.endswith('POP')
-
-  # Allow operator=
-  line = clean_lines.elided[linenum]
-  if Search(r'\boperator\s*=\s*\(', line):
-    return IsDeletedOrDefault(clean_lines, linenum)
-
-  # Allow constructors
-  match = Match(r'\s*(?:[\w<>]+::)*([\w<>]+)\s*::\s*([\w<>]+)\s*\(', line)
-  if match and match.group(1) == match.group(2):
-    return IsDeletedOrDefault(clean_lines, linenum)
-  if Search(r'\b(?:explicit|inline)\s+[\w<>]+\s*\(', line):
-    return IsDeletedOrDefault(clean_lines, linenum)
-
-  if Match(r'\s*[\w<>]+\s*\(', line):
-    previous_line = 'ReturnType'
-    if linenum > 0:
-      previous_line = clean_lines.elided[linenum - 1]
-    if Match(r'^\s*$', previous_line) or Search(r'[{}:;]\s*$', previous_line):
-      return IsDeletedOrDefault(clean_lines, linenum)
-
-  # Reject types not mentioned in template-argument-list
-  while line:
-    match = Match(r'^.*?(\w+)\s*&&(.*)$', line)
-    if not match:
-      break
-    if match.group(1) not in typenames:
-      return False
-    line = match.group(2)
-
-  # All RValue types that were in template-argument-list should have
-  # been removed by now.  Those were allowed, assuming that they will
-  # be forwarded.
-  #
-  # If there are no remaining RValue types left (i.e. types that were
-  # not found in template-argument-list), flag those as not allowed.
-  return line.find('&&') < 0
-
-
-def GetTemplateArgs(clean_lines, linenum):
-  """Find list of template arguments associated with this function declaration.
-
-  Args:
-    clean_lines: A CleansedLines instance containing the file.
-    linenum: Line number containing the start of the function declaration,
-             usually one line after the end of the template-argument-list.
-  Returns:
-    Set of type names, or empty set if this does not appear to have
-    any template parameters.
-  """
-  # Find start of function
-  func_line = linenum
-  while func_line > 0:
-    line = clean_lines.elided[func_line]
-    if Match(r'^\s*$', line):
-      return set()
-    if line.find('(') >= 0:
-      break
-    func_line -= 1
-  if func_line == 0:
-    return set()
-
-  # Collapse template-argument-list into a single string
-  argument_list = ''
-  match = Match(r'^(\s*template\s*)<', clean_lines.elided[func_line])
-  if match:
-    # template-argument-list on the same line as function name
-    start_col = len(match.group(1))
-    _, end_line, end_col = CloseExpression(clean_lines, func_line, start_col)
-    if end_col > -1 and end_line == func_line:
-      start_col += 1  # Skip the opening bracket
-      argument_list = clean_lines.elided[func_line][start_col:end_col]
-
-  elif func_line > 1:
-    # template-argument-list one line before function name
-    match = Match(r'^(.*)>\s*$', clean_lines.elided[func_line - 1])
-    if match:
-      end_col = len(match.group(1))
-      _, start_line, start_col = ReverseCloseExpression(
-          clean_lines, func_line - 1, end_col)
-      if start_col > -1:
-        start_col += 1  # Skip the opening bracket
-        while start_line < func_line - 1:
-          argument_list += clean_lines.elided[start_line][start_col:]
-          start_col = 0
-          start_line += 1
-        argument_list += clean_lines.elided[func_line - 1][start_col:end_col]
-
-  if not argument_list:
-    return set()
-
-  # Extract type names
-  typenames = set()
-  while True:
-    match = Match(r'^[,\s]*(?:typename|class)(?:\.\.\.)?\s+(\w+)(.*)$',
-                  argument_list)
-    if not match:
-      break
-    typenames.add(match.group(1))
-    argument_list = match.group(2)
-  return typenames
-
-
-def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error):
-  """Check for rvalue references.
-
-  Args:
-    filename: The name of the current file.
-    clean_lines: A CleansedLines instance containing the file.
-    linenum: The number of the line to check.
-    nesting_state: A NestingState instance which maintains information about
-                   the current stack of nested blocks being parsed.
-    error: The function to call with any errors found.
-  """
-  # Find lines missing spaces around &&.
-  # TODO(unknown): currently we don't check for rvalue references
-  # with spaces surrounding the && to avoid false positives with
-  # boolean expressions.
-  line = clean_lines.elided[linenum]
-  match = Match(r'^(.*\S)&&', line)
-  if not match:
-    match = Match(r'(.*)&&\S', line)
-  if (not match) or '(&&)' in line or Search(r'\boperator\s*$', match.group(1)):
-    return
-
-  # Either poorly formed && or an rvalue reference, check the context
-  # to get a more accurate error message.  Mostly we want to determine
-  # if what's to the left of "&&" is a type or not.
-  typenames = GetTemplateArgs(clean_lines, linenum)
-  and_pos = len(match.group(1))
-  if IsRValueType(typenames, clean_lines, nesting_state, linenum, and_pos):
-    if not IsRValueAllowed(clean_lines, linenum, typenames):
-      error(filename, linenum, 'build/c++11', 3,
-            'RValue references are an unapproved C++ feature.')
-  else:
-    error(filename, linenum, 'whitespace/operators', 3,
-          'Missing spaces around &&')
-
-
 def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error):
   """Checks for additional blank line issues related to sections.
 
@@ -3906,10 +3892,13 @@ def CheckBraces(filename, clean_lines, linenum, error):
     # used for brace initializers inside function calls.  We don't detect this
     # perfectly: we just don't complain if the last non-whitespace character on
     # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the
-    # previous line starts a preprocessor block.
+    # previous line starts a preprocessor block. We also allow a brace on the
+    # following line if it is part of an array initialization and would not fit
+    # within the 80 character limit of the preceding line.
     prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0]
     if (not Search(r'[,;:}{(]\s*$', prevline) and
-        not Match(r'\s*#', prevline)):
+        not Match(r'\s*#', prevline) and
+        not (GetLineWidth(prevline) > _line_length - 2 and '[]' in prevline)):
       error(filename, linenum, 'whitespace/braces', 4,
             '{ should almost always be at the end of the previous line')
 
@@ -4085,13 +4074,14 @@ def CheckTrailingSemicolon(filename, clean_lines, linenum, error):
     # In addition to macros, we also don't want to warn on
     #  - Compound literals
     #  - Lambdas
-    #  - alignas specifier with anonymous structs:
+    #  - alignas specifier with anonymous structs
+    #  - decltype
     closing_brace_pos = match.group(1).rfind(')')
     opening_parenthesis = ReverseCloseExpression(
         clean_lines, linenum, closing_brace_pos)
     if opening_parenthesis[2] > -1:
       line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]]
-      macro = Search(r'\b([A-Z_]+)\s*$', line_prefix)
+      macro = Search(r'\b([A-Z_][A-Z0-9_]*)\s*$', line_prefix)
       func = Match(r'^(.*\])\s*$', line_prefix)
       if ((macro and
            macro.group(1) not in (
@@ -4100,6 +4090,7 @@ def CheckTrailingSemicolon(filename, clean_lines, linenum, error):
                'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or
           (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or
           Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or
+          Search(r'\bdecltype$', line_prefix) or
           Search(r'\s+=\s*$', line_prefix)):
         match = None
     if (match and
@@ -4136,6 +4127,14 @@ def CheckTrailingSemicolon(filename, clean_lines, linenum, error):
       # outputting warnings for the matching closing brace, if there are
       # nested blocks with trailing semicolons, we will get the error
       # messages in reversed order.
+
+      # We need to check the line forward for NOLINT
+      raw_lines = clean_lines.raw_lines
+      ParseNolintSuppressions(filename, raw_lines[endlinenum-1], endlinenum-1,
+                              error)
+      ParseNolintSuppressions(filename, raw_lines[endlinenum], endlinenum,
+                              error)
+
       error(filename, endlinenum, 'readability/braces', 4,
             "You don't need a ; after a }")
 
@@ -4159,7 +4158,7 @@ def CheckEmptyBlockBody(filename, clean_lines, linenum, error):
   line = clean_lines.elided[linenum]
   matched = Match(r'\s*(for|while|if)\s*\(', line)
   if matched:
-    # Find the end of the conditional expression
+    # Find the end of the conditional expression.
     (end_line, end_linenum, end_pos) = CloseExpression(
         clean_lines, linenum, line.find('('))
 
@@ -4174,6 +4173,75 @@ def CheckEmptyBlockBody(filename, clean_lines, linenum, error):
         error(filename, end_linenum, 'whitespace/empty_loop_body', 5,
               'Empty loop bodies should use {} or continue')
 
+    # Check for if statements that have completely empty bodies (no comments)
+    # and no else clauses.
+    if end_pos >= 0 and matched.group(1) == 'if':
+      # Find the position of the opening { for the if statement.
+      # Return without logging an error if it has no brackets.
+      opening_linenum = end_linenum
+      opening_line_fragment = end_line[end_pos:]
+      # Loop until EOF or find anything that's not whitespace or opening {.
+      while not Search(r'^\s*\{', opening_line_fragment):
+        if Search(r'^(?!\s*$)', opening_line_fragment):
+          # Conditional has no brackets.
+          return
+        opening_linenum += 1
+        if opening_linenum == len(clean_lines.elided):
+          # Couldn't find conditional's opening { or any code before EOF.
+          return
+        opening_line_fragment = clean_lines.elided[opening_linenum]
+      # Set opening_line (opening_line_fragment may not be entire opening line).
+      opening_line = clean_lines.elided[opening_linenum]
+
+      # Find the position of the closing }.
+      opening_pos = opening_line_fragment.find('{')
+      if opening_linenum == end_linenum:
+        # We need to make opening_pos relative to the start of the entire line.
+        opening_pos += end_pos
+      (closing_line, closing_linenum, closing_pos) = CloseExpression(
+          clean_lines, opening_linenum, opening_pos)
+      if closing_pos < 0:
+        return
+
+      # Now construct the body of the conditional. This consists of the portion
+      # of the opening line after the {, all lines until the closing line,
+      # and the portion of the closing line before the }.
+      if (clean_lines.raw_lines[opening_linenum] !=
+          CleanseComments(clean_lines.raw_lines[opening_linenum])):
+        # Opening line ends with a comment, so conditional isn't empty.
+        return
+      if closing_linenum > opening_linenum:
+        # Opening line after the {. Ignore comments here since we checked above.
+        bodylist = list(opening_line[opening_pos+1:])
+        # All lines until closing line, excluding closing line, with comments.
+        bodylist.extend(clean_lines.raw_lines[opening_linenum+1:closing_linenum])
+        # Closing line before the }. Won't (and can't) have comments.
+        bodylist.append(clean_lines.elided[closing_linenum][:closing_pos-1])
+        body = '\n'.join(bodylist)
+      else:
+        # If statement has brackets and fits on a single line.
+        body = opening_line[opening_pos+1:closing_pos-1]
+
+      # Check if the body is empty
+      if not _EMPTY_CONDITIONAL_BODY_PATTERN.search(body):
+        return
+      # The body is empty. Now make sure there's not an else clause.
+      current_linenum = closing_linenum
+      current_line_fragment = closing_line[closing_pos:]
+      # Loop until EOF or find anything that's not whitespace or else clause.
+      while Search(r'^\s*$|^(?=\s*else)', current_line_fragment):
+        if Search(r'^(?=\s*else)', current_line_fragment):
+          # Found an else clause, so don't log an error.
+          return
+        current_linenum += 1
+        if current_linenum == len(clean_lines.elided):
+          break
+        current_line_fragment = clean_lines.elided[current_linenum]
+
+      # The body is empty and there's no else clause until EOF or other code.
+      error(filename, end_linenum, 'whitespace/empty_if_body', 4,
+            ('If statement had no body and no else clause'))
+
 
 def FindCheckMacro(line):
   """Find a replaceable CHECK-like macro.
@@ -4393,6 +4461,7 @@ def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state,
   # raw strings,
   raw_lines = clean_lines.lines_without_raw_strings
   line = raw_lines[linenum]
+  prev = raw_lines[linenum - 1] if linenum > 0 else ''
 
   if line.find('\t') != -1:
     error(filename, linenum, 'whitespace/tab', 1,
@@ -4416,22 +4485,27 @@ def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state,
   cleansed_line = clean_lines.elided[linenum]
   while initial_spaces < len(line) and line[initial_spaces] == ' ':
     initial_spaces += 1
-  if line and line[-1].isspace():
-    error(filename, linenum, 'whitespace/end_of_line', 4,
-          'Line ends in whitespace.  Consider deleting these extra spaces.')
   # There are certain situations we allow one space, notably for
   # section labels, and also lines containing multi-line raw strings.
-  elif ((initial_spaces == 1 or initial_spaces == 3) and
-        not Match(scope_or_label_pattern, cleansed_line) and
-        not (clean_lines.raw_lines[linenum] != line and
-             Match(r'^\s*""', line))):
+  # We also don't check for lines that look like continuation lines
+  # (of lines ending in double quotes, commas, equals, or angle brackets)
+  # because the rules for how to indent those are non-trivial.
+  if (not Search(r'[",=><] *$', prev) and
+      (initial_spaces == 1 or initial_spaces == 3) and
+      not Match(scope_or_label_pattern, cleansed_line) and
+      not (clean_lines.raw_lines[linenum] != line and
+           Match(r'^\s*""', line))):
     error(filename, linenum, 'whitespace/indent', 3,
           'Weird number of spaces at line-start.  '
           'Are you using a 2-space indent?')
 
+  if line and line[-1].isspace():
+    error(filename, linenum, 'whitespace/end_of_line', 4,
+          'Line ends in whitespace.  Consider deleting these extra spaces.')
+
   # Check if the line is a header guard.
   is_header_guard = False
-  if file_extension == 'h':
+  if file_extension in GetHeaderExtensions():
     cppvar = GetHeaderGuardCPPVariable(filename)
     if (line.startswith('#ifndef %s' % cppvar) or
         line.startswith('#define %s' % cppvar) or
@@ -4445,20 +4519,23 @@ def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state,
   #
   # The "$Id:...$" comment may also get very long without it being the
   # developers fault.
+  #
+  # Doxygen documentation copying can get pretty long when using an overloaded
+  # function declaration
   if (not line.startswith('#include') and not is_header_guard and
       not Match(r'^\s*//.*http(s?)://\S*$', line) and
-      not Match(r'^// \$Id:.*#[0-9]+ \$$', line)):
+      not Match(r'^\s*//\s*[^\s]*$', line) and
+      not Match(r'^// \$Id:.*#[0-9]+ \$$', line) and
+      not Match(r'^\s*/// [@\\](copydoc|copydetails|copybrief) .*$', line)):
     line_width = GetLineWidth(line)
-    extended_length = int((_line_length * 1.25))
-    if line_width > extended_length:
-      error(filename, linenum, 'whitespace/line_length', 4,
-            'Lines should very rarely be longer than %i characters' %
-            extended_length)
-    elif line_width > _line_length:
+    if line_width > _line_length:
       error(filename, linenum, 'whitespace/line_length', 2,
             'Lines should be <= %i characters long' % _line_length)
 
   if (cleansed_line.count(';') > 1 and
+      # allow simple single line lambdas
+      not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}\n\r]*\}',
+                line) and
       # for loops are allowed two ;'s (and may run over two lines).
       cleansed_line.find('for') == -1 and
       (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or
@@ -4479,9 +4556,8 @@ def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state,
   CheckOperatorSpacing(filename, clean_lines, linenum, error)
   CheckParenthesisSpacing(filename, clean_lines, linenum, error)
   CheckCommaSpacing(filename, clean_lines, linenum, error)
-  CheckBracesSpacing(filename, clean_lines, linenum, error)
+  CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error)
   CheckSpacingForFunctionCall(filename, clean_lines, linenum, error)
-  CheckRValueReference(filename, clean_lines, linenum, nesting_state, error)
   CheckCheck(filename, clean_lines, linenum, error)
   CheckAltTokens(filename, clean_lines, linenum, error)
   classinfo = nesting_state.InnermostClass()
@@ -4517,31 +4593,17 @@ def _DropCommonSuffixes(filename):
   Returns:
     The filename with the common suffix removed.
   """
-  for suffix in ('test.cc', 'regtest.cc', 'unittest.cc',
-                 'inl.h', 'impl.h', 'internal.h'):
+  for suffix in itertools.chain(
+      ('%s.%s' % (test_suffix.lstrip('_'), ext)
+       for test_suffix, ext in itertools.product(_test_suffixes, GetNonHeaderExtensions())),
+      ('%s.%s' % (suffix, ext)
+       for suffix, ext in itertools.product(['inl', 'imp', 'internal'], GetHeaderExtensions()))):
     if (filename.endswith(suffix) and len(filename) > len(suffix) and
         filename[-len(suffix) - 1] in ('-', '_')):
       return filename[:-len(suffix) - 1]
   return os.path.splitext(filename)[0]
 
 
-def _IsTestFilename(filename):
-  """Determines if the given filename has a suffix that identifies it as a test.
-
-  Args:
-    filename: The input filename.
-
-  Returns:
-    True if 'filename' looks like a test, False otherwise.
-  """
-  if (filename.endswith('_test.cc') or
-      filename.endswith('_unittest.cc') or
-      filename.endswith('_regtest.cc')):
-    return True
-  else:
-    return False
-
-
 def _ClassifyInclude(fileinfo, include, is_system):
   """Figures out what kind of header 'include' is.
 
@@ -4570,6 +4632,10 @@ def _ClassifyInclude(fileinfo, include, is_system):
   # those already checked for above.
   is_cpp_h = include in _CPP_HEADERS
 
+  # Headers with C++ extensions shouldn't be considered C system headers
+  if is_system and os.path.splitext(include)[1] in ['.hpp', '.hxx', '.h++']:
+      is_system = False
+
   if is_system:
     if is_cpp_h:
       return _CPP_SYS_HEADER
@@ -4582,9 +4648,11 @@ def _ClassifyInclude(fileinfo, include, is_system):
   target_dir, target_base = (
       os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName())))
   include_dir, include_base = os.path.split(_DropCommonSuffixes(include))
+  target_dir_pub = os.path.normpath(target_dir + '/../public')
+  target_dir_pub = target_dir_pub.replace('\\', '/')
   if target_base == include_base and (
       include_dir == target_dir or
-      include_dir == os.path.normpath(target_dir + '/../public')):
+      include_dir == target_dir_pub):
     return _LIKELY_MY_HEADER
 
   # If the target and include share some initial basename
@@ -4628,7 +4696,7 @@ def CheckIncludeLine(filename, clean_lines, linenum, include_state, error):
   # naming convention but not the include convention.
   match = Match(r'#include\s*"([^/]+\.h)"', line)
   if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)):
-    error(filename, linenum, 'build/include', 4,
+    error(filename, linenum, 'build/include_subdir', 4,
           'Include the directory when naming .h files')
 
   # we shouldn't include a file more than once. actually, there are a
@@ -4643,11 +4711,16 @@ def CheckIncludeLine(filename, clean_lines, linenum, include_state, error):
       error(filename, linenum, 'build/include', 4,
             '"%s" already included at %s:%s' %
             (include, filename, duplicate_line))
-    elif (include.endswith('.cc') and
+      return
+
+    for extension in GetNonHeaderExtensions():
+      if (include.endswith('.' + extension) and
           os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)):
-      error(filename, linenum, 'build/include', 4,
-            'Do not include .cc files from other packages')
-    elif not _THIRD_PARTY_HEADERS_PATTERN.match(include):
+        error(filename, linenum, 'build/include', 4,
+              'Do not include .' + extension + ' files from other packages')
+        return
+
+    if not _THIRD_PARTY_HEADERS_PATTERN.match(include):
       include_state.include_list[-1].append((include, linenum))
 
       # We want to ensure that headers appear in the right order:
@@ -4701,7 +4774,7 @@ def _GetTextInside(text, start_pattern):
 
   # Give opening punctuations to get the matching close-punctuations.
   matching_punctuation = {'(': ')', '{': '}', '[': ']'}
-  closing_punctuation = set(matching_punctuation.itervalues())
+  closing_punctuation = set(itervalues(matching_punctuation))
 
   # Find the position to start extracting text.
   match = re.search(start_pattern, text, re.M)
@@ -4756,6 +4829,9 @@ def _GetTextInside(text, start_pattern):
 _RE_PATTERN_CONST_REF_PARAM = (
     r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT +
     r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')')
+# Stream types.
+_RE_PATTERN_REF_STREAM_PARAM = (
+    r'(?:.*stream\s*&\s*' + _RE_PATTERN_IDENT + r')')
 
 
 def CheckLanguage(filename, clean_lines, linenum, file_extension,
@@ -4792,15 +4868,13 @@ def CheckLanguage(filename, clean_lines, linenum, file_extension,
   if match:
     include_state.ResetSection(match.group(1))
 
-  # Make Windows paths like Unix.
-  fullname = os.path.abspath(filename).replace('\\', '/')
-  
+
   # Perform other checks now that we are sure that this is not an include line
   CheckCasts(filename, clean_lines, linenum, error)
   CheckGlobalStatic(filename, clean_lines, linenum, error)
   CheckPrintf(filename, clean_lines, linenum, error)
 
-  if file_extension == 'h':
+  if file_extension in GetHeaderExtensions():
     # TODO(unknown): check that 1-arg constructors are explicit.
     #                How to tell it's a constructor?
     #                (handled in CheckForNonStandardConstructs for now)
@@ -4861,9 +4935,14 @@ def CheckLanguage(filename, clean_lines, linenum, file_extension,
           % (match.group(1), match.group(2)))
 
   if Search(r'\busing namespace\b', line):
-    error(filename, linenum, 'build/namespaces', 5,
-          'Do not use namespace using-directives.  '
-          'Use using-declarations instead.')
+    if Search(r'\bliterals\b', line):
+      error(filename, linenum, 'build/namespaces_literals', 5,
+            'Do not use namespace using-directives.  '
+            'Use using-declarations instead.')
+    else:
+      error(filename, linenum, 'build/namespaces', 5,
+            'Do not use namespace using-directives.  '
+            'Use using-declarations instead.')
 
   # Detect variable-length arrays.
   match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line)
@@ -4907,12 +4986,12 @@ def CheckLanguage(filename, clean_lines, linenum, file_extension,
   # Check for use of unnamed namespaces in header files.  Registration
   # macros are typically OK, so we allow use of "namespace {" on lines
   # that end with backslashes.
-  if (file_extension == 'h'
+  if (file_extension in GetHeaderExtensions()
       and Search(r'\bnamespace\s*{', line)
       and line[-1] != '\\'):
     error(filename, linenum, 'build/namespaces', 4,
           'Do not use unnamed namespaces in header files.  See '
-          'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces'
+          'https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces'
           ' for more information.')
 
 
@@ -4933,9 +5012,13 @@ def CheckGlobalStatic(filename, clean_lines, linenum, error):
 
   # Check for people declaring static/global STL strings at the top level.
   # This is dangerous because the C++ language does not guarantee that
-  # globals with constructors are initialized before the first access.
+  # globals with constructors are initialized before the first access, and
+  # also because globals can be destroyed when some threads are still running.
+  # TODO(unknown): Generalize this to also find static unique_ptr instances.
+  # TODO(unknown): File bugs for clang-tidy to find these.
   match = Match(
-      r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)',
+      r'((?:|static +)(?:|const +))(?::*std::)?string( +const)? +'
+      r'([a-zA-Z0-9_:]+)\b(.*)',
       line)
 
   # Remove false positives:
@@ -4955,15 +5038,20 @@ def CheckGlobalStatic(filename, clean_lines, linenum, error):
   #   matching identifiers.
   #    string Class::operator*()
   if (match and
-      not Search(r'\bstring\b(\s+const)?\s*\*\s*(const\s+)?\w', line) and
+      not Search(r'\bstring\b(\s+const)?\s*[\*\&]\s*(const\s+)?\w', line) and
       not Search(r'\boperator\W', line) and
-      not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(3))):
-    error(filename, linenum, 'runtime/string', 4,
-          'For a static/global string constant, use a C style string instead: '
-          '"%schar %s[]".' %
-          (match.group(1), match.group(2)))
+      not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(4))):
+    if Search(r'\bconst\b', line):
+      error(filename, linenum, 'runtime/string', 4,
+            'For a static/global string constant, use a C style string '
+            'instead: "%schar%s %s[]".' %
+            (match.group(1), match.group(2) or '', match.group(3)))
+    else:
+      error(filename, linenum, 'runtime/string', 4,
+            'Static/global string variables are not permitted.')
 
-  if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line):
+  if (Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line) or
+      Search(r'\b([A-Za-z0-9_]*_)\(CHECK_NOTNULL\(\1\)\)', line)):
     error(filename, linenum, 'runtime/init', 4,
           'You seem to be initializing a member variable with itself.')
 
@@ -5208,7 +5296,8 @@ def CheckForNonConstReference(filename, clean_lines, linenum,
 
   decls = ReplaceAll(r'{[^}]*}', ' ', line)  # exclude function body
   for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls):
-    if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter):
+    if (not Match(_RE_PATTERN_CONST_REF_PARAM, parameter) and
+        not Match(_RE_PATTERN_REF_STREAM_PARAM, parameter)):
       error(filename, linenum, 'runtime/references', 2,
             'Is this a non-const reference? '
             'If so, make const or use a pointer: ' +
@@ -5231,7 +5320,7 @@ def CheckCasts(filename, clean_lines, linenum, error):
   # Parameterless conversion functions, such as bool(), are allowed as they are
   # probably a member operator declaration or default constructor.
   match = Search(
-      r'(\bnew\s+|\S<\s*(?:const\s+)?)?\b'
+      r'(\bnew\s+(?:const\s+)?|\S<\s*(?:const\s+)?)?\b'
       r'(int|float|double|bool|char|int32|uint32|int64|uint64)'
       r'(\([^)].*)', line)
   expecting_function = ExpectingFunctionArgs(clean_lines, linenum)
@@ -5372,63 +5461,12 @@ def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error):
   if context.endswith(' operator++') or context.endswith(' operator--'):
     return False
 
-  # A single unnamed argument for a function tends to look like old
-  # style cast.  If we see those, don't issue warnings for deprecated
-  # casts, instead issue warnings for unnamed arguments where
-  # appropriate.
-  #
-  # These are things that we want warnings for, since the style guide
-  # explicitly require all parameters to be named:
-  #   Function(int);
-  #   Function(int) {
-  #   ConstMember(int) const;
-  #   ConstMember(int) const {
-  #   ExceptionMember(int) throw (...);
-  #   ExceptionMember(int) throw (...) {
-  #   PureVirtual(int) = 0;
-  #   [](int) -> bool {
-  #
-  # These are functions of some sort, where the compiler would be fine
-  # if they had named parameters, but people often omit those
-  # identifiers to reduce clutter:
-  #   (FunctionPointer)(int);
-  #   (FunctionPointer)(int) = value;
-  #   Function((function_pointer_arg)(int))
-  #   Function((function_pointer_arg)(int), int param)
-  #   <TemplateArgument(int)>;
-  #   <(FunctionPointerTemplateArgument)(int)>;
+  # A single unnamed argument for a function tends to look like old style cast.
+  # If we see those, don't issue warnings for deprecated casts.
   remainder = line[match.end(0):]
   if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)',
            remainder):
-    # Looks like an unnamed parameter.
-
-    # Don't warn on any kind of template arguments.
-    if Match(r'^\s*>', remainder):
-      return False
-
-    # Don't warn on assignments to function pointers, but keep warnings for
-    # unnamed parameters to pure virtual functions.  Note that this pattern
-    # will also pass on assignments of "0" to function pointers, but the
-    # preferred values for those would be "nullptr" or "NULL".
-    matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder)
-    if matched_zero and matched_zero.group(1) != '0':
-      return False
-
-    # Don't warn on function pointer declarations.  For this we need
-    # to check what came before the "(type)" string.
-    if Match(r'.*\)\s*$', line[0:match.start(0)]):
-      return False
-
-    # Don't warn if the parameter is named with block comments, e.g.:
-    #  Function(int /*unused_param*/);
-    raw_line = clean_lines.raw_lines[linenum]
-    if '/*' in raw_line:
-      return False
-
-    # Passed all filters, issue warning here.
-    error(filename, linenum, 'readability/function', 3,
-          'All parameters should be named in a function')
-    return True
+    return False
 
   # At this point, all that should be left is actual casts.
   error(filename, linenum, 'readability/casting', 4,
@@ -5482,12 +5520,15 @@ def ExpectingFunctionArgs(clean_lines, linenum):
     ('<limits>', ('numeric_limits',)),
     ('<list>', ('list',)),
     ('<map>', ('map', 'multimap',)),
-    ('<memory>', ('allocator',)),
+    ('<memory>', ('allocator', 'make_shared', 'make_unique', 'shared_ptr',
+                  'unique_ptr', 'weak_ptr')),
     ('<queue>', ('queue', 'priority_queue',)),
     ('<set>', ('set', 'multiset',)),
     ('<stack>', ('stack',)),
     ('<string>', ('char_traits', 'basic_string',)),
     ('<tuple>', ('tuple',)),
+    ('<unordered_map>', ('unordered_map', 'unordered_multimap')),
+    ('<unordered_set>', ('unordered_set', 'unordered_multiset')),
     ('<utility>', ('pair',)),
     ('<vector>', ('vector',)),
 
@@ -5498,18 +5539,26 @@ def ExpectingFunctionArgs(clean_lines, linenum):
     ('<slist>', ('slist',)),
     )
 
-_RE_PATTERN_STRING = re.compile(r'\bstring\b')
+_HEADERS_MAYBE_TEMPLATES = (
+    ('<algorithm>', ('copy', 'max', 'min', 'min_element', 'sort',
+                     'transform',
+                    )),
+    ('<utility>', ('forward', 'make_pair', 'move', 'swap')),
+    )
 
-_re_pattern_algorithm_header = []
-for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap',
-                  'transform'):
-  # Match max<type>(..., ...), max(..., ...), but not foo->max, foo.max or
-  # type::max().
-  _re_pattern_algorithm_header.append(
-      (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'),
-       _template,
-       '<algorithm>'))
+_RE_PATTERN_STRING = re.compile(r'\bstring\b')
 
+_re_pattern_headers_maybe_templates = []
+for _header, _templates in _HEADERS_MAYBE_TEMPLATES:
+  for _template in _templates:
+    # Match max<type>(..., ...), max(..., ...), but not foo->max, foo.max or
+    # type::max().
+    _re_pattern_headers_maybe_templates.append(
+        (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'),
+            _template,
+            _header))
+
+# Other scripts may reach in and modify this pattern.
 _re_pattern_templates = []
 for _header, _templates in _HEADERS_CONTAINING_TEMPLATES:
   for _template in _templates:
@@ -5540,7 +5589,7 @@ def FilesBelongToSameModule(filename_cc, filename_h):
   some false positives. This should be sufficiently rare in practice.
 
   Args:
-    filename_cc: is the path for the .cc file
+    filename_cc: is the path for the source (e.g. .cc) file
     filename_h: is the path for the header path
 
   Returns:
@@ -5548,20 +5597,23 @@ def FilesBelongToSameModule(filename_cc, filename_h):
     bool: True if filename_cc and filename_h belong to the same module.
     string: the additional prefix needed to open the header file.
   """
+  fileinfo_cc = FileInfo(filename_cc)
+  if not fileinfo_cc.Extension().lstrip('.') in GetNonHeaderExtensions():
+    return (False, '')
 
-  if not filename_cc.endswith('.cc'):
+  fileinfo_h = FileInfo(filename_h)
+  if not fileinfo_h.Extension().lstrip('.') in GetHeaderExtensions():
     return (False, '')
-  filename_cc = filename_cc[:-len('.cc')]
-  if filename_cc.endswith('_unittest'):
-    filename_cc = filename_cc[:-len('_unittest')]
-  elif filename_cc.endswith('_test'):
-    filename_cc = filename_cc[:-len('_test')]
+
+  filename_cc = filename_cc[:-(len(fileinfo_cc.Extension()))]
+  matched_test_suffix = Search(_TEST_FILE_SUFFIX, fileinfo_cc.BaseName())
+  if matched_test_suffix:
+    filename_cc = filename_cc[:-len(matched_test_suffix.group(1))]
+
   filename_cc = filename_cc.replace('/public/', '/')
   filename_cc = filename_cc.replace('/internal/', '/')
 
-  if not filename_h.endswith('.h'):
-    return (False, '')
-  filename_h = filename_h[:-len('.h')]
+  filename_h = filename_h[:-(len(fileinfo_h.Extension()))]
   if filename_h.endswith('-inl'):
     filename_h = filename_h[:-len('-inl')]
   filename_h = filename_h.replace('/public/', '/')
@@ -5622,7 +5674,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
   required = {}  # A map of header name to linenumber and the template entity.
                  # Example of required: { '<functional>': (1219, 'less<>') }
 
-  for linenum in xrange(clean_lines.NumLines()):
+  for linenum in range(clean_lines.NumLines()):
     line = clean_lines.elided[linenum]
     if not line or line[0] == '#':
       continue
@@ -5636,7 +5688,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
       if prefix.endswith('std::') or not prefix.endswith('::'):
         required['<string>'] = (linenum, 'string')
 
-    for pattern, template, header in _re_pattern_algorithm_header:
+    for pattern, template, header in _re_pattern_headers_maybe_templates:
       if pattern.search(line):
         required[header] = (linenum, template)
 
@@ -5645,8 +5697,13 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
       continue
 
     for pattern, template, header in _re_pattern_templates:
-      if pattern.search(line):
-        required[header] = (linenum, template)
+      matched = pattern.search(line)
+      if matched:
+        # Don't warn about IWYU in non-STL namespaces:
+        # (We check only the first match per line; good enough.)
+        prefix = line[:matched.start()]
+        if prefix.endswith('std::') or not prefix.endswith('::'):
+          required[header] = (linenum, template)
 
   # The policy is that if you #include something in foo.h you don't need to
   # include it again in foo.cc. Here, we will look at possible includes.
@@ -5671,7 +5728,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
 
   # include_dict is modified during iteration, so we iterate over a copy of
   # the keys.
-  header_keys = include_dict.keys()
+  header_keys = list(include_dict.keys())
   for header in header_keys:
     (same_module, common_path) = FilesBelongToSameModule(abs_filename, header)
     fullpath = common_path + header
@@ -5683,11 +5740,13 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
   # didn't include it in the .h file.
   # TODO(unknown): Do a better job of finding .h files so we are confident that
   # not having the .h file means there isn't one.
-  if filename.endswith('.cc') and not header_found:
-    return
+  if not header_found:
+    for extension in GetNonHeaderExtensions():
+      if filename.endswith('.' + extension):
+        return
 
   # All the lines have been processed, report the errors found.
-  for required_header_unstripped in required:
+  for required_header_unstripped in sorted(required, key=required.__getitem__):
     template = required[required_header_unstripped][1]
     if required_header_unstripped.strip('<>"') not in include_dict:
       error(filename, required[required_header_unstripped][0],
@@ -5719,31 +5778,6 @@ def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error):
           ' OR use pair directly OR if appropriate, construct a pair directly')
 
 
-def CheckDefaultLambdaCaptures(filename, clean_lines, linenum, error):
-  """Check that default lambda captures are not used.
-
-  Args:
-    filename: The name of the current file.
-    clean_lines: A CleansedLines instance containing the file.
-    linenum: The number of the line to check.
-    error: The function to call with any errors found.
-  """
-  line = clean_lines.elided[linenum]
-
-  # A lambda introducer specifies a default capture if it starts with "[="
-  # or if it starts with "[&" _not_ followed by an identifier.
-  match = Match(r'^(.*)\[\s*(?:=|&[^\w])', line)
-  if match:
-    # Found a potential error, check what comes after the lambda-introducer.
-    # If it's not open parenthesis (for lambda-declarator) or open brace
-    # (for compound-statement), it's not a lambda.
-    line, _, pos = CloseExpression(clean_lines, linenum, len(match.group(1)))
-    if pos >= 0 and Match(r'^\s*[{(]', line[pos:]):
-      error(filename, linenum, 'build/c++11',
-            4,  # 4 = high confidence
-            'Default lambda captures are an unapproved C++ feature.')
-
-
 def CheckRedundantVirtual(filename, clean_lines, linenum, error):
   """Check if line contains a redundant "virtual" function-specifier.
 
@@ -5851,11 +5885,9 @@ def IsBlockInNameSpace(nesting_state, is_forward_declaration):
     Whether or not the new block is directly in a namespace.
   """
   if is_forward_declaration:
-    if len(nesting_state.stack) >= 1 and (
-        isinstance(nesting_state.stack[-1], _NamespaceInfo)):
-      return True
-    else:
-      return False
+    return len(nesting_state.stack) >= 1 and (
+      isinstance(nesting_state.stack[-1], _NamespaceInfo))
+
 
   return (len(nesting_state.stack) > 1 and
           nesting_state.stack[-1].check_namespace_indentation and
@@ -5905,7 +5937,7 @@ def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum,
 
 def ProcessLine(filename, file_extension, clean_lines, line,
                 include_state, function_state, nesting_state, error,
-                extra_check_functions=[]):
+                extra_check_functions=None):
   """Processes a single line in the file.
 
   Args:
@@ -5942,11 +5974,11 @@ def ProcessLine(filename, file_extension, clean_lines, line,
   CheckPosixThreading(filename, clean_lines, line, error)
   CheckInvalidIncrement(filename, clean_lines, line, error)
   CheckMakePairUsesDeduction(filename, clean_lines, line, error)
-  CheckDefaultLambdaCaptures(filename, clean_lines, line, error)
   CheckRedundantVirtual(filename, clean_lines, line, error)
   CheckRedundantOverrideOrFinal(filename, clean_lines, line, error)
-  for check_fn in extra_check_functions:
-    check_fn(filename, clean_lines, line, error)
+  if extra_check_functions:
+    for check_fn in extra_check_functions:
+      check_fn(filename, clean_lines, line, error)
 
 def FlagCxx11Features(filename, clean_lines, linenum, error):
   """Flag those c++11 features that we only allow in certain places.
@@ -5959,8 +5991,14 @@ def FlagCxx11Features(filename, clean_lines, linenum, error):
   """
   line = clean_lines.elided[linenum]
 
-  # Flag unapproved C++11 headers.
   include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line)
+
+  # Flag unapproved C++ TR1 headers.
+  if include and include.group(1).startswith('tr1/'):
+    error(filename, linenum, 'build/c++tr1', 5,
+          ('C++ TR1 headers such as <%s> are unapproved.') % include.group(1))
+
+  # Flag unapproved C++11 headers.
   if include and include.group(1) in ('cfenv',
                                       'condition_variable',
                                       'fenv.h',
@@ -5994,8 +6032,27 @@ def FlagCxx11Features(filename, clean_lines, linenum, error):
              'they may let you use it.') % top_name)
 
 
+def FlagCxx14Features(filename, clean_lines, linenum, error):
+  """Flag those C++14 features that we restrict.
+
+  Args:
+    filename: The name of the current file.
+    clean_lines: A CleansedLines instance containing the file.
+    linenum: The number of the line to check.
+    error: The function to call with any errors found.
+  """
+  line = clean_lines.elided[linenum]
+
+  include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line)
+
+  # Flag unapproved C++14 headers.
+  if include and include.group(1) in ('scoped_allocator', 'shared_mutex'):
+    error(filename, linenum, 'build/c++14', 5,
+          ('<%s> is an unapproved C++14 header.') % include.group(1))
+
+
 def ProcessFileData(filename, file_extension, lines, error,
-                    extra_check_functions=[]):
+                    extra_check_functions=None):
   """Performs lint checks and reports any errors to the given error function.
 
   Args:
@@ -6019,14 +6076,14 @@ def ProcessFileData(filename, file_extension, lines, error,
   ResetNolintSuppressions()
 
   CheckForCopyright(filename, lines, error)
-
+  ProcessGlobalSuppresions(lines)
   RemoveMultiLineComments(filename, lines, error)
   clean_lines = CleansedLines(lines)
 
-  if file_extension == 'h':
+  if file_extension in GetHeaderExtensions():
     CheckForHeaderGuard(filename, clean_lines, error)
 
-  for line in xrange(clean_lines.NumLines()):
+  for line in range(clean_lines.NumLines()):
     ProcessLine(filename, file_extension, clean_lines, line,
                 include_state, function_state, nesting_state, error,
                 extra_check_functions)
@@ -6034,9 +6091,9 @@ def ProcessFileData(filename, file_extension, lines, error,
   nesting_state.CheckCompletedBlocks(filename, error)
 
   CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error)
-  
+
   # Check that the .cc file has included its header if it exists.
-  if file_extension == 'cc':
+  if _IsSourceExtension(file_extension):
     CheckHeaderFileIncluded(filename, include_state, error)
 
   # We check here rather than inside ProcessLine so that we see raw
@@ -6092,36 +6149,56 @@ def ProcessConfigOverrides(filename):
             if base_name:
               pattern = re.compile(val)
               if pattern.match(base_name):
-                sys.stderr.write('Ignoring "%s": file excluded by "%s". '
-                                 'File path component "%s" matches '
-                                 'pattern "%s"\n' %
-                                 (filename, cfg_file, base_name, val))
+                _cpplint_state.PrintInfo('Ignoring "%s": file excluded by '
+                    '"%s". File path component "%s" matches pattern "%s"\n' %
+                    (filename, cfg_file, base_name, val))
                 return False
           elif name == 'linelength':
             global _line_length
             try:
                 _line_length = int(val)
             except ValueError:
-                sys.stderr.write('Line length must be numeric.')
+                _cpplint_state.PrintError('Line length must be numeric.')
+          elif name == 'extensions':
+              global _valid_extensions
+              try:
+                  extensions = [ext.strip() for ext in val.split(',')]
+                  _valid_extensions = set(extensions)
+              except ValueError:
+                  sys.stderr.write('Extensions should be a comma-separated list of values;'
+                                   'for example: extensions=hpp,cpp\n'
+                                   'This could not be parsed: "%s"' % (val,))
+          elif name == 'headers':
+              global _header_extensions
+              try:
+                  extensions = [ext.strip() for ext in val.split(',')]
+                  _header_extensions = set(extensions)
+              except ValueError:
+                  sys.stderr.write('Extensions should be a comma-separated list of values;'
+                                   'for example: extensions=hpp,cpp\n'
+                                   'This could not be parsed: "%s"' % (val,))
+          elif name == 'root':
+            global _root
+            _root = val
           else:
-            sys.stderr.write(
+            _cpplint_state.PrintError(
                 'Invalid configuration option (%s) in file %s\n' %
                 (name, cfg_file))
 
     except IOError:
-      sys.stderr.write(
+      _cpplint_state.PrintError(
           "Skipping config file '%s': Can't open for reading\n" % cfg_file)
       keep_looking = False
 
   # Apply all the accumulated filters in reverse order (top-level directory
   # config options having the least priority).
-  for filter in reversed(cfg_filters):
-     _AddFilters(filter)
+  for cfg_filter in reversed(cfg_filters):
+     _AddFilters(cfg_filter)
 
   return True
 
 
-def ProcessFile(filename, vlevel, extra_check_functions=[]):
+def ProcessFile(filename, vlevel, extra_check_functions=None):
   """Does google-lint on a single file.
 
   Args:
@@ -6170,7 +6247,7 @@ def ProcessFile(filename, vlevel, extra_check_functions=[]):
         lf_lines.append(linenum + 1)
 
   except IOError:
-    sys.stderr.write(
+    _cpplint_state.PrintError(
         "Skipping input '%s': Can't open for reading\n" % filename)
     _RestoreFilters()
     return
@@ -6180,9 +6257,9 @@ def ProcessFile(filename, vlevel, extra_check_functions=[]):
 
   # When reading from stdin, the extension is unknown, so no cpplint tests
   # should rely on the extension.
-  if filename != '-' and file_extension not in _valid_extensions:
-    sys.stderr.write('Ignoring %s; not a valid file name '
-                     '(%s)\n' % (filename, ', '.join(_valid_extensions)))
+  if filename != '-' and file_extension not in GetAllExtensions():
+    _cpplint_state.PrintError('Ignoring %s; not a valid file name '
+                     '(%s)\n' % (filename, ', '.join(GetAllExtensions())))
   else:
     ProcessFileData(filename, file_extension, lines, Error,
                     extra_check_functions)
@@ -6205,7 +6282,7 @@ def ProcessFile(filename, vlevel, extra_check_functions=[]):
         Error(filename, linenum, 'whitespace/newline', 1,
               'Unexpected \\r (^M) found; better to use only \\n')
 
-  sys.stderr.write('Done processing %s\n' % filename)
+  _cpplint_state.PrintInfo('Done processing %s\n' % filename)
   _RestoreFilters()
 
 
@@ -6216,10 +6293,11 @@ def PrintUsage(message):
     message: The optional error message.
   """
   sys.stderr.write(_USAGE)
+
   if message:
     sys.exit('\nFATAL ERROR: ' + message)
   else:
-    sys.exit(1)
+    sys.exit(0)
 
 
 def PrintCategories():
@@ -6247,8 +6325,13 @@ def ParseArguments(args):
                                                  'counting=',
                                                  'filter=',
                                                  'root=',
+                                                 'repository=',
                                                  'linelength=',
-                                                 'extensions='])
+                                                 'extensions=',
+                                                 'exclude=',
+                                                 'headers=',
+                                                 'quiet',
+                                                 'recursive'])
   except getopt.GetoptError:
     PrintUsage('Invalid arguments.')
 
@@ -6256,13 +6339,15 @@ def ParseArguments(args):
   output_format = _OutputFormat()
   filters = ''
   counting_style = ''
+  recursive = False
 
   for (opt, val) in opts:
     if opt == '--help':
       PrintUsage(None)
     elif opt == '--output':
-      if val not in ('emacs', 'vs7', 'eclipse'):
-        PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.')
+      if val not in ('emacs', 'vs7', 'eclipse', 'junit'):
+        PrintUsage('The only allowed output formats are emacs, vs7, eclipse '
+                   'and junit.')
       output_format = val
     elif opt == '--verbose':
       verbosity = int(val)
@@ -6277,22 +6362,47 @@ def ParseArguments(args):
     elif opt == '--root':
       global _root
       _root = val
+    elif opt == '--repository':
+      global _repository
+      _repository = val
     elif opt == '--linelength':
       global _line_length
       try:
-          _line_length = int(val)
+        _line_length = int(val)
       except ValueError:
-          PrintUsage('Line length must be digits.')
+        PrintUsage('Line length must be digits.')
+    elif opt == '--exclude':
+      global _excludes
+      if not _excludes:
+        _excludes = set()
+      _excludes.update(glob.glob(val))
     elif opt == '--extensions':
       global _valid_extensions
       try:
-          _valid_extensions = set(val.split(','))
+        _valid_extensions = set(val.split(','))
       except ValueError:
           PrintUsage('Extensions must be comma seperated list.')
+    elif opt == '--headers':
+      global _header_extensions
+      try:
+          _header_extensions = set(val.split(','))
+      except ValueError:
+        PrintUsage('Extensions must be comma seperated list.')
+    elif opt == '--recursive':
+      recursive = True
+    elif opt == '--quiet':
+      global _quiet
+      _quiet = True
 
   if not filenames:
     PrintUsage('No files were specified.')
 
+  if recursive:
+    filenames = _ExpandDirectories(filenames)
+
+  if _excludes:
+    filenames = _FilterExcludedFiles(filenames)
+
   _SetOutputFormat(output_format)
   _SetVerboseLevel(verbosity)
   _SetFilters(filters)
@@ -6300,21 +6410,63 @@ def ParseArguments(args):
 
   return filenames
 
+def _ExpandDirectories(filenames):
+  """Searches a list of filenames and replaces directories in the list with
+  all files descending from those directories. Files with extensions not in
+  the valid extensions list are excluded.
+
+  Args:
+    filenames: A list of files or directories
+
+  Returns:
+    A list of all files that are members of filenames or descended from a
+    directory in filenames
+  """
+  expanded = set()
+  for filename in filenames:
+      if not os.path.isdir(filename):
+        expanded.add(filename)
+        continue
+
+      for root, _, files in os.walk(filename):
+        for loopfile in files:
+          fullname = os.path.join(root, loopfile)
+          if fullname.startswith('.' + os.path.sep):
+            fullname = fullname[len('.' + os.path.sep):]
+          expanded.add(fullname)
+
+  filtered = []
+  for filename in expanded:
+      if os.path.splitext(filename)[1][1:] in GetAllExtensions():
+          filtered.append(filename)
+
+  return filtered
+
+def _FilterExcludedFiles(filenames):
+  """Filters out files listed in the --exclude command line switch. File paths
+  in the switch are evaluated relative to the current working directory
+  """
+  exclude_paths = [os.path.abspath(f) for f in _excludes]
+  return [f for f in filenames if os.path.abspath(f) not in exclude_paths]
 
 def main():
   filenames = ParseArguments(sys.argv[1:])
+  backup_err = sys.stderr
+  try:
+    # Change stderr to write with replacement characters so we don't die
+    # if we try to print something containing non-ASCII characters.
+    sys.stderr = codecs.StreamReader(sys.stderr, 'replace')
 
-  # Change stderr to write with replacement characters so we don't die
-  # if we try to print something containing non-ASCII characters.
-  sys.stderr = codecs.StreamReaderWriter(sys.stderr,
-                                         codecs.getreader('utf8'),
-                                         codecs.getwriter('utf8'),
-                                         'replace')
+    _cpplint_state.ResetErrorCounts()
+    for filename in filenames:
+      ProcessFile(filename, _cpplint_state.verbose_level)
+    _cpplint_state.PrintErrorCounts()
 
-  _cpplint_state.ResetErrorCounts()
-  for filename in filenames:
-    ProcessFile(filename, _cpplint_state.verbose_level)
-  _cpplint_state.PrintErrorCounts()
+    if _cpplint_state.output_format == 'junit':
+      sys.stderr.write(_cpplint_state.FormatJUnitXML())
+
+  finally:
+    sys.stderr = backup_err
 
   sys.exit(_cpplint_state.error_count > 0)
 
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index 473c90f925124..dd8cc7d9e60a8 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -23,6 +23,7 @@
 #include <memory>
 #include <sstream>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "arrow/buffer.h"
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 144fbcd05c205..3d72761ed18e5 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <cstring>
 #include <sstream>
+#include <utility>
 
 #include "arrow/buffer.h"
 #include "arrow/compare.h"
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 0ae1ddd8ea221..f0a786131b2b5 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -22,6 +22,7 @@
 #include <cstdint>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "arrow/buffer.h"
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index db901526fc2ee..a740299dfe194 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -22,6 +22,7 @@
 #include <cstring>
 #include <limits>
 #include <sstream>
+#include <utility>
 #include <vector>
 
 #include "arrow/array.h"
diff --git a/cpp/src/arrow/compute/context.h b/cpp/src/arrow/compute/context.h
index 051c91bf049fa..09838195a52ee 100644
--- a/cpp/src/arrow/compute/context.h
+++ b/cpp/src/arrow/compute/context.h
@@ -18,6 +18,8 @@
 #ifndef ARROW_COMPUTE_CONTEXT_H
 #define ARROW_COMPUTE_CONTEXT_H
 
+#include <memory>
+
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 #include "arrow/type_fwd.h"
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index 1face78bdebfb..8fac7965d0ed9 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -23,6 +23,7 @@
 #include <mutex>
 #include <sstream>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "arrow/builder.h"
diff --git a/cpp/src/arrow/compute/kernels/util-internal.cc b/cpp/src/arrow/compute/kernels/util-internal.cc
index 28428bfcba6c6..0734365859b5a 100644
--- a/cpp/src/arrow/compute/kernels/util-internal.cc
+++ b/cpp/src/arrow/compute/kernels/util-internal.cc
@@ -17,6 +17,7 @@
 
 #include "arrow/compute/kernels/util-internal.h"
 
+#include <memory>
 #include <vector>
 
 #include "arrow/array.h"
diff --git a/cpp/src/arrow/compute/kernels/util-internal.h b/cpp/src/arrow/compute/kernels/util-internal.h
index 7633fed4a8fe7..2f611320a7687 100644
--- a/cpp/src/arrow/compute/kernels/util-internal.h
+++ b/cpp/src/arrow/compute/kernels/util-internal.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_COMPUTE_KERNELS_UTIL_INTERNAL_H
 #define ARROW_COMPUTE_KERNELS_UTIL_INTERNAL_H
 
+#include <memory>
 #include <vector>
 
 #include "arrow/compute/kernel.h"
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index d3872503edf19..f440c19efe414 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -22,6 +22,7 @@
 #include <memory>
 #include <sstream>  // IWYU pragma: keep
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "flatbuffers/flatbuffers.h"
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index ae0f8f39806b7..cc3b6e55783e3 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -22,6 +22,7 @@
 #include <sstream>
 #include <string>
 #include <type_traits>
+#include <utility>
 #include <vector>
 
 #include <flatbuffers/flatbuffers.h>  // IWYU pragma: export
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index bd5f8ce10ea68..994f528ea4bad 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <memory>
 #include <ostream>
 #include <sstream>
 #include <string>
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index c060ab8bfd6db..c67e5410eb6ee 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -22,6 +22,7 @@
 #include <cstdint>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include <numpy/arrayobject.h>
diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index cc3892928c455..9d32ead524b93 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -19,6 +19,7 @@
 
 #include <cstdint>
 #include <cstdlib>
+#include <memory>
 #include <mutex>
 #include <string>
 
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index f550de7b2848c..0632d28faf789 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -18,6 +18,8 @@
 #ifndef PYARROW_IO_H
 #define PYARROW_IO_H
 
+#include <memory>
+
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
 #include "arrow/util/visibility.h"
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index b5a75aeedd5eb..a1161fe32e100 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -29,6 +29,7 @@
 #include <memory>
 #include <sstream>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "arrow/array.h"
diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc
index 60932bdf3e4bb..d418cc4a2e66c 100644
--- a/cpp/src/arrow/record_batch.cc
+++ b/cpp/src/arrow/record_batch.cc
@@ -21,6 +21,7 @@
 #include <cstdlib>
 #include <memory>
 #include <sstream>
+#include <utility>
 
 #include "arrow/array.h"
 #include "arrow/status.h"
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 14877ccb537c2..8cfd67faef1ee 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -22,6 +22,7 @@
 #include <limits>
 #include <memory>
 #include <sstream>
+#include <utility>
 
 #include "arrow/array.h"
 #include "arrow/record_batch.h"
diff --git a/cpp/src/arrow/table_builder.cc b/cpp/src/arrow/table_builder.cc
index 379d886deacba..8e9babcc3997a 100644
--- a/cpp/src/arrow/table_builder.cc
+++ b/cpp/src/arrow/table_builder.cc
@@ -21,6 +21,7 @@
 #include <cstdlib>
 #include <memory>
 #include <sstream>
+#include <utility>
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 31ad53458112c..0a2889f040026 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -20,6 +20,8 @@
 #include <climits>
 #include <sstream>
 #include <string>
+#include <utility>
+#include <vector>
 
 #include "arrow/array.h"
 #include "arrow/compare.h"
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 4bfce9b5f0c53..ede52e9b84bb6 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_TYPE_TRAITS_H
 #define ARROW_TYPE_TRAITS_H
 
+#include <memory>
 #include <type_traits>
 
 #include "arrow/type_fwd.h"
diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
index 7e2a94ca82320..d1af6c666a156 100644
--- a/cpp/src/arrow/util/io-util.h
+++ b/cpp/src/arrow/util/io-util.h
@@ -19,6 +19,7 @@
 #define ARROW_UTIL_IO_UTIL_H
 
 #include <iostream>
+#include <memory>
 
 #include "arrow/buffer.h"
 #include "arrow/io/interfaces.h"
diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
index 4e4ecfaaaca31..ce29e6c321d5d 100644
--- a/cpp/src/plasma/events.cc
+++ b/cpp/src/plasma/events.cc
@@ -17,6 +17,8 @@
 
 #include "plasma/events.h"
 
+#include <utility>
+
 #include <errno.h>
 
 namespace plasma {
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
index 2d07c919a18f4..bb9cdae601146 100644
--- a/cpp/src/plasma/plasma.h
+++ b/cpp/src/plasma/plasma.h
@@ -27,6 +27,7 @@
 #include <string.h>
 #include <unistd.h>  // pid_t
 
+#include <memory>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
index 44263a6418439..101a3faa7675e 100644
--- a/cpp/src/plasma/protocol.h
+++ b/cpp/src/plasma/protocol.h
@@ -18,6 +18,8 @@
 #ifndef PLASMA_PROTOCOL_H
 #define PLASMA_PROTOCOL_H
 
+#include <memory>
+#include <unordered_map>
 #include <vector>
 
 #include "arrow/status.h"
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 80dd525e3e3b4..316a27f63f680 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -44,9 +44,11 @@
 #include <unistd.h>
 
 #include <deque>
+#include <memory>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 
 #include "plasma/common.h"
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index 7eada5a126991..7e716d284f389 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -19,7 +19,9 @@
 #define PLASMA_STORE_H
 
 #include <deque>
+#include <memory>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #include "plasma/common.h"

From 074eafc686db58178de4439fa63ecf728ec9d4ab Mon Sep 17 00:00:00 2001
From: yosuke shiro <ys2014hs@gmail.com>
Date: Fri, 26 Jan 2018 10:32:08 -0500
Subject: [PATCH 1405/1644] ARROW-2043: [C++] change description from OS X to
 macOS

Author: yosuke shiro <ys2014hs@gmail.com>

Closes #1521 from shiro615/change-description-from-osx-to-macos and squashes the following commits:

ab03f72f [yosuke shiro] [C++] change description from OS X to macOS
---
 cpp/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/README.md b/cpp/README.md
index ef2e1fd1b1259..52169974de41e 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -39,7 +39,7 @@ sudo apt-get install cmake \
      libboost-system-dev
 ```
 
-On OS X, you can use [Homebrew][1]:
+On macOS, you can use [Homebrew][1]:
 
 ```shell
 git clone https://github.com/apache/arrow.git

From a71bc838337856b72c2d1b22f8cc9741a103438c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 26 Jan 2018 18:13:03 -0500
Subject: [PATCH 1406/1644] ARROW-2010: [C++] Do not suppress shorten-64-to-32
 warnings from clang, fix warnings in ORC adapter

These are warnings we want to see to prevent silent truncation of values without an explicit cast.

See also bug ARROW-2032 that I encountered while working on this

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1506 from wesm/ARROW-2010 and squashes the following commits:

b999a815 [Wes McKinney] Add explicit cast to socklen_t
f06e1fd2 [Wes McKinney] Do not suppress shorten-64-to-32 warnings from clang, fix warnings in ORC adapter
---
 cpp/cmake_modules/SetupCxxFlags.cmake |  2 +-
 cpp/src/arrow/adapters/orc/adapter.cc | 47 ++++++++++++++-------------
 cpp/src/plasma/fling.cc               |  2 +-
 3 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 97aed6b274976..d901bde47c631 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -100,7 +100,7 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
 -Wno-cast-align -Wno-vla-extension -Wno-shift-sign-overflow \
 -Wno-used-but-marked-unused -Wno-missing-variable-declarations \
 -Wno-gnu-zero-variadic-macro-arguments -Wconversion -Wno-sign-conversion \
--Wno-disabled-macro-expansion -Wno-shorten-64-to-32")
+-Wno-disabled-macro-expansion")
 
     # Version numbers where warnings are introduced
     if ("${COMPILER_VERSION}" VERSION_GREATER "3.3")
diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc
index dd8cc7d9e60a8..f253808e34ffc 100644
--- a/cpp/src/arrow/adapters/orc/adapter.cc
+++ b/cpp/src/arrow/adapters/orc/adapter.cc
@@ -106,6 +106,8 @@ Status GetArrowType(const liborc::Type* type, std::shared_ptr<DataType>* out) {
     return Status::OK();
   }
   liborc::TypeKind kind = type->getKind();
+  const int subtype_count = static_cast<int>(type->getSubtypeCount());
+
   switch (kind) {
     case liborc::BOOLEAN:
       *out = boolean();
@@ -136,7 +138,7 @@ Status GetArrowType(const liborc::Type* type, std::shared_ptr<DataType>* out) {
       *out = binary();
       break;
     case liborc::CHAR:
-      *out = fixed_size_binary(type->getMaximumLength());
+      *out = fixed_size_binary(static_cast<int>(type->getMaximumLength()));
       break;
     case liborc::TIMESTAMP:
       *out = timestamp(TimeUnit::NANO);
@@ -145,16 +147,18 @@ Status GetArrowType(const liborc::Type* type, std::shared_ptr<DataType>* out) {
       *out = date32();
       break;
     case liborc::DECIMAL: {
-      if (type->getPrecision() == 0) {
+      const int precision = static_cast<int>(type->getPrecision());
+      const int scale = static_cast<int>(type->getScale());
+      if (precision == 0) {
         // In HIVE 0.11/0.12 precision is set as 0, but means max precision
         *out = decimal(38, 6);
       } else {
-        *out = decimal(type->getPrecision(), type->getScale());
+        *out = decimal(precision, scale);
       }
       break;
     }
     case liborc::LIST: {
-      if (type->getSubtypeCount() != 1) {
+      if (subtype_count != 1) {
         return Status::Invalid("Invalid Orc List type");
       }
       std::shared_ptr<DataType> elemtype;
@@ -163,7 +167,7 @@ Status GetArrowType(const liborc::Type* type, std::shared_ptr<DataType>* out) {
       break;
     }
     case liborc::MAP: {
-      if (type->getSubtypeCount() != 2) {
+      if (subtype_count != 2) {
         return Status::Invalid("Invalid Orc Map type");
       }
       std::shared_ptr<DataType> keytype;
@@ -174,9 +178,8 @@ Status GetArrowType(const liborc::Type* type, std::shared_ptr<DataType>* out) {
       break;
     }
     case liborc::STRUCT: {
-      int size = type->getSubtypeCount();
       std::vector<std::shared_ptr<Field>> fields;
-      for (int child = 0; child < size; ++child) {
+      for (int child = 0; child < subtype_count; ++child) {
         std::shared_ptr<DataType> elemtype;
         RETURN_NOT_OK(GetArrowType(type->getSubtype(child), &elemtype));
         std::string name = type->getFieldName(child);
@@ -186,10 +189,9 @@ Status GetArrowType(const liborc::Type* type, std::shared_ptr<DataType>* out) {
       break;
     }
     case liborc::UNION: {
-      int size = type->getSubtypeCount();
       std::vector<std::shared_ptr<Field>> fields;
       std::vector<uint8_t> type_codes;
-      for (int child = 0; child < size; ++child) {
+      for (int child = 0; child < subtype_count; ++child) {
         std::shared_ptr<DataType> elemtype;
         RETURN_NOT_OK(GetArrowType(type->getSubtype(child), &elemtype));
         fields.push_back(field("_union_" + std::to_string(child), elemtype));
@@ -260,7 +262,7 @@ class ORCFileReader::Impl {
           "Only ORC files with a top-level struct "
           "can be handled");
     }
-    int size = type.getSubtypeCount();
+    int size = static_cast<int>(type.getSubtypeCount());
     std::vector<std::shared_ptr<Field>> fields;
     for (int child = 0; child < size; ++child) {
       std::shared_ptr<DataType> elemtype;
@@ -450,7 +452,7 @@ class ORCFileReader::Impl {
     const liborc::Type* elemtype = type->getSubtype(0);
 
     const bool has_nulls = batch->hasNulls;
-    for (int i = offset; i < length + offset; i++) {
+    for (int64_t i = offset; i < length + offset; i++) {
       if (!has_nulls || batch->notNull[i]) {
         int64_t start = batch->offsets[i];
         int64_t end = batch->offsets[i + 1];
@@ -475,7 +477,7 @@ class ORCFileReader::Impl {
     const liborc::Type* valtype = type->getSubtype(1);
 
     const bool has_nulls = batch->hasNulls;
-    for (int i = offset; i < length + offset; i++) {
+    for (int64_t i = offset; i < length + offset; i++) {
       RETURN_NOT_OK(list_builder->Append());
       int64_t start = batch->offsets[i];
       int64_t list_length = batch->offsets[i + 1] - start;
@@ -517,7 +519,7 @@ class ORCFileReader::Impl {
     if (length == 0) {
       return Status::OK();
     }
-    int start = builder->length();
+    int64_t start = builder->length();
 
     const uint8_t* valid_bytes = nullptr;
     if (batch->hasNulls) {
@@ -541,7 +543,7 @@ class ORCFileReader::Impl {
     if (length == 0) {
       return Status::OK();
     }
-    int start = builder->length();
+    int64_t start = builder->length();
 
     const uint8_t* valid_bytes = nullptr;
     if (batch->hasNulls) {
@@ -552,7 +554,7 @@ class ORCFileReader::Impl {
     const int64_t* source = batch->data.data() + offset;
     uint8_t* target = reinterpret_cast<uint8_t*>(builder->data()->mutable_data());
 
-    for (int i = 0; i < length; i++) {
+    for (int64_t i = 0; i < length; i++) {
       if (source[i]) {
         BitUtil::SetBit(target, start + i);
       } else {
@@ -570,7 +572,7 @@ class ORCFileReader::Impl {
     if (length == 0) {
       return Status::OK();
     }
-    int start = builder->length();
+    int64_t start = builder->length();
 
     const uint8_t* valid_bytes = nullptr;
     if (batch->hasNulls) {
@@ -582,7 +584,7 @@ class ORCFileReader::Impl {
     const int64_t* nanos = batch->nanoseconds.data() + offset;
     int64_t* target = reinterpret_cast<int64_t*>(builder->data()->mutable_data());
 
-    for (int i = 0; i < length; i++) {
+    for (int64_t i = 0; i < length; i++) {
       // TODO: boundscheck this, as ORC supports higher resolution timestamps
       // than arrow for nanosecond resolution
       target[start + i] = seconds[i] * kOneSecondNanos + nanos[i];
@@ -597,9 +599,10 @@ class ORCFileReader::Impl {
     auto batch = static_cast<liborc::StringVectorBatch*>(cbatch);
 
     const bool has_nulls = batch->hasNulls;
-    for (int i = offset; i < length + offset; i++) {
+    for (int64_t i = offset; i < length + offset; i++) {
       if (!has_nulls || batch->notNull[i]) {
-        RETURN_NOT_OK(builder->Append(batch->data[i], batch->length[i]));
+        RETURN_NOT_OK(
+            builder->Append(batch->data[i], static_cast<int32_t>(batch->length[i])));
       } else {
         RETURN_NOT_OK(builder->AppendNull());
       }
@@ -613,7 +616,7 @@ class ORCFileReader::Impl {
     auto batch = static_cast<liborc::StringVectorBatch*>(cbatch);
 
     const bool has_nulls = batch->hasNulls;
-    for (int i = offset; i < length + offset; i++) {
+    for (int64_t i = offset; i < length + offset; i++) {
       if (!has_nulls || batch->notNull[i]) {
         RETURN_NOT_OK(builder->Append(batch->data[i]));
       } else {
@@ -630,7 +633,7 @@ class ORCFileReader::Impl {
     const bool has_nulls = cbatch->hasNulls;
     if (type->getPrecision() == 0 || type->getPrecision() > 18) {
       auto batch = static_cast<liborc::Decimal128VectorBatch*>(cbatch);
-      for (int i = offset; i < length + offset; i++) {
+      for (int64_t i = offset; i < length + offset; i++) {
         if (!has_nulls || batch->notNull[i]) {
           RETURN_NOT_OK(builder->Append(
               Decimal128(batch->values[i].getHighBits(), batch->values[i].getLowBits())));
@@ -640,7 +643,7 @@ class ORCFileReader::Impl {
       }
     } else {
       auto batch = static_cast<liborc::Decimal64VectorBatch*>(cbatch);
-      for (int i = offset; i < length + offset; i++) {
+      for (int64_t i = offset; i < length + offset; i++) {
         if (!has_nulls || batch->notNull[i]) {
           RETURN_NOT_OK(builder->Append(Decimal128(batch->values[i])));
         } else {
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
index b84648b25a9e7..819ec1623055b 100644
--- a/cpp/src/plasma/fling.cc
+++ b/cpp/src/plasma/fling.cc
@@ -23,7 +23,7 @@ void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len)
   msg->msg_iov = iov;
   msg->msg_iovlen = 1;
   msg->msg_control = buf;
-  msg->msg_controllen = buf_len;
+  msg->msg_controllen = static_cast<socklen_t>(buf_len);
   msg->msg_name = NULL;
   msg->msg_namelen = 0;
 }

From 6299a9cfb314dc7a03fdafca41419a6be4300225 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 26 Jan 2018 18:59:05 -0500
Subject: [PATCH 1407/1644] ARROW-2032: [C++] ORC ep installs on each call to
 ninja build

ExternalProject_add using git is sadly always triggering something.
Using HTTP archives instead don't need to check if they are at the
correct revision.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1519 from xhochy/ARROW-2032 and squashes the following commits:

f5ac8228 [Uwe L. Korn] ARROW-2032: [C++] ORC ep installs on each call to ninja build
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 4f64434170655..69812b97cc770 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -926,8 +926,7 @@ if (ARROW_ORC)
                       -DZLIB_HOME=${ZLIB_HOME})
 
   ExternalProject_Add(orc_ep
-    GIT_REPOSITORY "https://github.com/apache/orc"
-    GIT_TAG ${ORC_VERSION}
+    URL "https://github.com/apache/orc/archive/${ORC_VERSION}.tar.gz"
     BUILD_BYPRODUCTS ${ORC_STATIC_LIB}
     CMAKE_ARGS ${ORC_CMAKE_ARGS})
 

From edde5c19ad5c441429ae80cce32db80bd52ed364 Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Sun, 28 Jan 2018 16:47:41 +0100
Subject: [PATCH 1408/1644] ARROW-1999: [Python] Type checking in
 `from_numpy_dtype`

- Adds type checking to the C++ `NumPyDtypeToArrow` and `GetTensorType` to
  ensure `dtype` is actually a dtype object.
- Add conversion of non-dtype objects in `pa.from_numpy_dtype`.
- Adds tests to check a wider variety of inputs to
  `pa.from_numpy_dtype`, and ensure proper errors.

Author: Jim Crist <jiminy.crist@gmail.com>

Closes #1523 from jcrist/from_numpy_dtype and squashes the following commits:

e9de101 [Jim Crist] Type checking in `from_numpy_dtype`
---
 cpp/src/arrow/python/numpy_convert.cc |  6 ++++++
 python/pyarrow/tests/test_schema.py   | 27 ++++++++++++++++++++++++++-
 python/pyarrow/types.pxi              |  1 +
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 9ed2d73d42b57..124745edecf81 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -84,6 +84,9 @@ NumPyBuffer::~NumPyBuffer() { Py_XDECREF(arr_); }
     break;
 
 Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out) {
+  if (!PyArray_DescrCheck(dtype)) {
+    return Status::TypeError("Did not pass numpy.dtype object");
+  }
   PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
   int type_num = cast_npy_type_compat(descr->type_num);
 
@@ -145,6 +148,9 @@ Status GetNumPyType(const DataType& type, int* type_num) {
 }
 
 Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
+  if (!PyArray_DescrCheck(dtype)) {
+    return Status::TypeError("Did not pass numpy.dtype object");
+  }
   PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
 
   int type_num = cast_npy_type_compat(descr->type_num);
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index dbca139e20570..90efe3f7e950a 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -154,8 +154,21 @@ def test_time_types():
         pa.time64('s')
 
 
-def test_type_from_numpy_dtype_timestamps():
+def test_from_numpy_dtype():
     cases = [
+        (np.dtype('bool'), pa.bool_()),
+        (np.dtype('int8'), pa.int8()),
+        (np.dtype('int16'), pa.int16()),
+        (np.dtype('int32'), pa.int32()),
+        (np.dtype('int64'), pa.int64()),
+        (np.dtype('uint8'), pa.uint8()),
+        (np.dtype('uint16'), pa.uint16()),
+        (np.dtype('uint32'), pa.uint32()),
+        (np.dtype('float16'), pa.float16()),
+        (np.dtype('float32'), pa.float32()),
+        (np.dtype('float64'), pa.float64()),
+        (np.dtype('U'), pa.string()),
+        (np.dtype('S'), pa.binary()),
         (np.dtype('datetime64[s]'), pa.timestamp('s')),
         (np.dtype('datetime64[ms]'), pa.timestamp('ms')),
         (np.dtype('datetime64[us]'), pa.timestamp('us')),
@@ -166,6 +179,18 @@ def test_type_from_numpy_dtype_timestamps():
         result = pa.from_numpy_dtype(dt)
         assert result == pt
 
+    # Things convertible to numpy dtypes work
+    assert pa.from_numpy_dtype('U') == pa.string()
+    assert pa.from_numpy_dtype(np.unicode) == pa.string()
+    assert pa.from_numpy_dtype('int32') == pa.int32()
+    assert pa.from_numpy_dtype(bool) == pa.bool_()
+
+    with pytest.raises(NotImplementedError):
+        pa.from_numpy_dtype(np.dtype('O'))
+
+    with pytest.raises(TypeError):
+        pa.from_numpy_dtype('not_convertible_to_dtype')
+
 
 def test_field():
     t = pa.string()
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 1563b57855cd9..a3cbeefb028c7 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -1207,6 +1207,7 @@ def from_numpy_dtype(object dtype):
     Convert NumPy dtype to pyarrow.DataType
     """
     cdef shared_ptr[CDataType] c_type
+    dtype = np.dtype(dtype)
     with nogil:
         check_status(NumPyDtypeToArrow(dtype, &c_type))
 

From 450bf474f5add5f0ab09008a3057d1b57811ad6b Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sun, 28 Jan 2018 16:59:41 +0100
Subject: [PATCH 1409/1644] ARROW-1835: [C++] Create Arrow schema from
 std::tuple types

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1478 from xhochy/ARROW-1835 and squashes the following commits:

9728740 [Korn, Uwe] Remove documentation tag that conflicts with gcc
a690248 [Korn, Uwe] ARROW-1835: [C++] Create Arrow schema from std::tuple types
---
 cpp/src/arrow/CMakeLists.txt |   2 +
 cpp/src/arrow/stl-test.cc    |  78 ++++++++++++++++++
 cpp/src/arrow/stl.h          | 153 +++++++++++++++++++++++++++++++++++
 cpp/src/arrow/type.h         |  13 +++
 4 files changed, 246 insertions(+)
 create mode 100644 cpp/src/arrow/stl-test.cc
 create mode 100644 cpp/src/arrow/stl.h

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index ad86256e0be34..74674bebb43be 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -153,6 +153,7 @@ install(FILES
   pretty_print.h
   record_batch.h
   status.h
+  stl.h
   table.h
   table_builder.h
   tensor.h
@@ -183,6 +184,7 @@ ADD_ARROW_TEST(memory_pool-test)
 ADD_ARROW_TEST(pretty_print-test)
 ADD_ARROW_TEST(public-api-test)
 ADD_ARROW_TEST(status-test)
+ADD_ARROW_TEST(stl-test)
 ADD_ARROW_TEST(type-test)
 ADD_ARROW_TEST(table-test)
 ADD_ARROW_TEST(table_builder-test)
diff --git a/cpp/src/arrow/stl-test.cc b/cpp/src/arrow/stl-test.cc
new file mode 100644
index 0000000000000..c85baa3a11e3f
--- /dev/null
+++ b/cpp/src/arrow/stl-test.cc
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/stl.h"
+
+namespace arrow {
+namespace stl {
+
+TEST(TestSchemaFromTuple, PrimitiveTypesVector) {
+  Schema expected_schema(
+      {field("column1", int8(), false), field("column2", int16(), false),
+       field("column3", int32(), false), field("column4", int64(), false),
+       field("column5", uint8(), false), field("column6", uint16(), false),
+       field("column7", uint32(), false), field("column8", uint64(), false),
+       field("column9", boolean(), false), field("column10", utf8(), false)});
+
+  std::shared_ptr<Schema> schema =
+      SchemaFromTuple<std::tuple<int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t,
+                                 uint32_t, uint64_t, bool, std::string>>::
+          MakeSchema(std::vector<std::string>({"column1", "column2", "column3", "column4",
+                                               "column5", "column6", "column7", "column8",
+                                               "column9", "column10"}));
+  ASSERT_TRUE(expected_schema.Equals(*schema));
+}
+
+TEST(TestSchemaFromTuple, PrimitiveTypesTuple) {
+  Schema expected_schema(
+      {field("column1", int8(), false), field("column2", int16(), false),
+       field("column3", int32(), false), field("column4", int64(), false),
+       field("column5", uint8(), false), field("column6", uint16(), false),
+       field("column7", uint32(), false), field("column8", uint64(), false),
+       field("column9", boolean(), false), field("column10", utf8(), false)});
+
+  std::shared_ptr<Schema> schema = SchemaFromTuple<
+      std::tuple<int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t, uint32_t, uint64_t,
+                 bool, std::string>>::MakeSchema(std::make_tuple("column1", "column2",
+                                                                 "column3", "column4",
+                                                                 "column5", "column6",
+                                                                 "column7", "column8",
+                                                                 "column9", "column10"));
+  ASSERT_TRUE(expected_schema.Equals(*schema));
+}
+
+TEST(TestSchemaFromTuple, SimpleList) {
+  Schema expected_schema({field("column1", list(utf8()), false)});
+  std::shared_ptr<Schema> schema =
+      SchemaFromTuple<std::tuple<std::vector<std::string>>>::MakeSchema({"column1"});
+
+  ASSERT_TRUE(expected_schema.Equals(*schema));
+}
+
+TEST(TestSchemaFromTuple, NestedList) {
+  Schema expected_schema({field("column1", list(list(boolean())), false)});
+  std::shared_ptr<Schema> schema =
+      SchemaFromTuple<std::tuple<std::vector<std::vector<bool>>>>::MakeSchema(
+          {"column1"});
+
+  ASSERT_TRUE(expected_schema.Equals(*schema));
+}
+
+}  // namespace stl
+}  // namespace arrow
diff --git a/cpp/src/arrow/stl.h b/cpp/src/arrow/stl.h
new file mode 100644
index 0000000000000..3250b5a320464
--- /dev/null
+++ b/cpp/src/arrow/stl.h
@@ -0,0 +1,153 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_STL_H
+#define ARROW_STL_H
+
+#include <string>
+#include <tuple>
+#include <vector>
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+class Schema;
+
+namespace stl {
+
+/// Traits meta class to map standard C/C++ types to equivalent Arrow types.
+template <typename T>
+struct ConversionTraits {};
+
+#define ARROW_STL_CONVERSION(c_type, ArrowType_) \
+  template <>                                    \
+  struct ConversionTraits<c_type> {              \
+    using ArrowType = ArrowType_;                \
+    constexpr static bool nullable = false;      \
+  };
+
+ARROW_STL_CONVERSION(bool, BooleanType)
+ARROW_STL_CONVERSION(int8_t, Int8Type)
+ARROW_STL_CONVERSION(int16_t, Int16Type)
+ARROW_STL_CONVERSION(int32_t, Int32Type)
+ARROW_STL_CONVERSION(int64_t, Int64Type)
+ARROW_STL_CONVERSION(uint8_t, UInt8Type)
+ARROW_STL_CONVERSION(uint16_t, UInt16Type)
+ARROW_STL_CONVERSION(uint32_t, UInt32Type)
+ARROW_STL_CONVERSION(uint64_t, UInt64Type)
+ARROW_STL_CONVERSION(float, FloatType)
+ARROW_STL_CONVERSION(double, DoubleType)
+ARROW_STL_CONVERSION(std::string, StringType)
+
+template <typename value_c_type>
+struct ConversionTraits<std::vector<value_c_type>> {
+  using ArrowType = meta::ListType<typename ConversionTraits<value_c_type>::ArrowType>;
+  constexpr static bool nullable = false;
+};
+
+/// Build an arrow::Schema based upon the types defined in a std::tuple-like structure.
+///
+/// While the type information is available at compile-time, we still need to add the
+/// column names at runtime, thus these methods are not constexpr.
+template <typename Tuple, std::size_t N = std::tuple_size<Tuple>::value>
+struct SchemaFromTuple {
+  using Element = typename std::tuple_element<N - 1, Tuple>::type;
+  using ArrowType = typename ConversionTraits<Element>::ArrowType;
+
+  // Implementations that take a vector-like object for the column names.
+
+  /// Recursively build a vector of arrow::Field from the defined types.
+  ///
+  /// In most cases MakeSchema is the better entrypoint for the Schema creation.
+  static std::vector<std::shared_ptr<Field>> MakeSchemaRecursion(
+      const std::vector<std::string>& names) {
+    std::vector<std::shared_ptr<Field>> ret =
+        SchemaFromTuple<Tuple, N - 1>::MakeSchemaRecursion(names);
+    ret.push_back(field(names[N - 1], std::make_shared<ArrowType>(),
+                        ConversionTraits<Element>::nullable));
+    return ret;
+  }
+
+  /// Build a Schema from the types of the tuple-like structure passed in as template
+  /// parameter assign the column names at runtime.
+  ///
+  /// An example usage of this API can look like the following:
+  ///
+  /// \code{.cpp}
+  /// using TupleType = std::tuple<int, std::vector<std::string>>;
+  /// std::shared_ptr<Schema> schema =
+  ///   SchemaFromTuple<TupleType>::MakeSchema({"int_column", "list_of_strings_column"});
+  /// \endcode
+  static std::shared_ptr<Schema> MakeSchema(const std::vector<std::string>& names) {
+    return std::make_shared<Schema>(MakeSchemaRecursion(names));
+  }
+
+  // Implementations that take a tuple-like object for the column names.
+
+  /// Recursively build a vector of arrow::Field from the defined types.
+  ///
+  /// In most cases MakeSchema is the better entrypoint for the Schema creation.
+  template <typename NamesTuple>
+  static std::vector<std::shared_ptr<Field>> MakeSchemaRecursionT(
+      const NamesTuple& names) {
+    std::vector<std::shared_ptr<Field>> ret =
+        SchemaFromTuple<Tuple, N - 1>::MakeSchemaRecursionT(names);
+    ret.push_back(field(std::get<N - 1>(names), std::make_shared<ArrowType>(),
+                        ConversionTraits<Element>::nullable));
+    return ret;
+  }
+
+  /// Build a Schema from the types of the tuple-like structure passed in as template
+  /// parameter assign the column names at runtime.
+  ///
+  /// An example usage of this API can look like the following:
+  ///
+  /// \code{.cpp}
+  /// using TupleType = std::tuple<int, std::vector<std::string>>;
+  /// std::shared_ptr<Schema> schema =
+  ///   SchemaFromTuple<TupleType>::MakeSchema({"int_column", "list_of_strings_column"});
+  /// \endcode
+  template <typename NamesTuple>
+  static std::shared_ptr<Schema> MakeSchema(const NamesTuple& names) {
+    return std::make_shared<Schema>(MakeSchemaRecursionT<NamesTuple>(names));
+  }
+};
+
+template <typename Tuple>
+struct SchemaFromTuple<Tuple, 0> {
+  static std::vector<std::shared_ptr<Field>> MakeSchemaRecursion(
+      const std::vector<std::string>& names) {
+    std::vector<std::shared_ptr<Field>> ret;
+    ret.reserve(names.size());
+    return ret;
+  }
+
+  template <typename NamesTuple>
+  static std::vector<std::shared_ptr<Field>> MakeSchemaRecursionT(
+      const NamesTuple& names) {
+    std::vector<std::shared_ptr<Field>> ret;
+    ret.reserve(std::tuple_size<NamesTuple>::value);
+    return ret;
+  }
+};
+/// @endcond
+
+}  // namespace stl
+}  // namespace arrow
+
+#endif  // ARROW_STL_H
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 009e07db07744..cfee6fd0e2363 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -407,6 +407,19 @@ class ARROW_EXPORT ListType : public NestedType {
   std::string name() const override { return "list"; }
 };
 
+namespace meta {
+
+/// Additional ListType class that can be instantiated with only compile-time arguments.
+template <typename T>
+class ARROW_EXPORT ListType : public ::arrow::ListType {
+ public:
+  using ValueType = T;
+
+  ListType() : ::arrow::ListType(std::make_shared<T>()) {}
+};
+
+}  // namespace meta
+
 // BinaryType type is represents lists of 1-byte values.
 class ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
  public:

From 05439532e70c105f8f282e2963dc31e0340ec503 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sun, 28 Jan 2018 17:30:59 +0100
Subject: [PATCH 1410/1644] ARROW-1646: [Python] Handle NumPy scalar types

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1475 from xhochy/ARROW-1646 and squashes the following commits:

7d85879 [Uwe L. Korn] flake8
eb4c08d [Korn, Uwe] ARROW-1646: [Python] pyarrow.array cannot handle NumPy scalar types
---
 cpp/src/arrow/python/builtin_convert.cc      |  34 +
 cpp/src/arrow/python/numpy_convert.cc        |   3 +
 cpp/src/arrow/python/numpy_convert.h         |   2 +
 cpp/src/arrow/python/numpy_interop.h         |   1 +
 python/pyarrow/tests/test_convert_builtin.py | 805 +++++++++++--------
 5 files changed, 492 insertions(+), 353 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 71f2fde5b3920..f7a370cdc4c06 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -32,6 +32,7 @@
 #include "arrow/util/logging.h"
 
 #include "arrow/python/helpers.h"
+#include "arrow/python/numpy_convert.h"
 #include "arrow/python/util/datetime.h"
 
 namespace arrow {
@@ -93,6 +94,21 @@ class ScalarVisitor {
       ++binary_count_;
     } else if (PyUnicode_Check(obj)) {
       ++unicode_count_;
+    } else if (PyArray_CheckAnyScalarExact(obj)) {
+      std::shared_ptr<DataType> type;
+      RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &type));
+      if (is_integer(type->id())) {
+        ++int_count_;
+      } else if (is_floating(type->id())) {
+        ++float_count_;
+      } else if (type->id() == Type::TIMESTAMP) {
+        ++timestamp_count_;
+      } else {
+        std::ostringstream ss;
+        ss << "Found a NumPy scalar with Arrow dtype that we cannot handle: ";
+        ss << type->ToString();
+        return Status::Invalid(ss.str());
+      }
     } else {
       // TODO(wesm): accumulate error information somewhere
       static std::string supported_types =
@@ -575,6 +591,24 @@ class TimestampConverter
           t = PyDateTime_to_ns(pydatetime);
           break;
       }
+    } else if (PyArray_CheckAnyScalarExact(item.obj())) {
+      // numpy.datetime64
+      std::shared_ptr<DataType> type;
+      RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(item.obj()), &type));
+      if (type->id() != Type::TIMESTAMP) {
+        std::ostringstream ss;
+        ss << "Expected np.datetime64 but got: ";
+        ss << type->ToString();
+        return Status::Invalid(ss.str());
+      }
+      const TimestampType& ttype = static_cast<const TimestampType&>(*type);
+      if (unit_ != ttype.unit()) {
+        return Status::NotImplemented(
+            "Cannot convert NumPy datetime64 objects with differing unit");
+      }
+
+      PyDatetimeScalarObject* obj = reinterpret_cast<PyDatetimeScalarObject*>(item.obj());
+      t = obj->obval;
     } else {
       t = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
       RETURN_IF_PYERROR();
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 124745edecf81..c2d055fceed5a 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -152,7 +152,10 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
     return Status::TypeError("Did not pass numpy.dtype object");
   }
   PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
+  return NumPyDtypeToArrow(descr, out);
+}
 
+Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr<DataType>* out) {
   int type_num = cast_npy_type_compat(descr->type_num);
 
   switch (type_num) {
diff --git a/cpp/src/arrow/python/numpy_convert.h b/cpp/src/arrow/python/numpy_convert.h
index 93c4848926cfc..220e38f2e1e02 100644
--- a/cpp/src/arrow/python/numpy_convert.h
+++ b/cpp/src/arrow/python/numpy_convert.h
@@ -56,6 +56,8 @@ bool is_contiguous(PyObject* array);
 
 ARROW_EXPORT
 Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out);
+ARROW_EXPORT
+Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr<DataType>* out);
 
 Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out);
 Status GetNumPyType(const DataType& type, int* type_num);
diff --git a/cpp/src/arrow/python/numpy_interop.h b/cpp/src/arrow/python/numpy_interop.h
index b93200cc8972d..8c569e232c121 100644
--- a/cpp/src/arrow/python/numpy_interop.h
+++ b/cpp/src/arrow/python/numpy_interop.h
@@ -40,6 +40,7 @@
 #endif
 
 #include <numpy/arrayobject.h>
+#include <numpy/arrayscalars.h>
 #include <numpy/ufuncobject.h>
 
 namespace arrow {
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index d7760da2f9b47..fa603b1a92fa2 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -23,6 +23,8 @@
 
 import datetime
 import decimal
+import numpy as np
+import six
 
 
 class StrangeIterable:
@@ -33,356 +35,453 @@ def __iter__(self):
         return self.lst.__iter__()
 
 
-class TestConvertIterable(unittest.TestCase):
-
-    def test_iterable_types(self):
-        arr1 = pa.array(StrangeIterable([0, 1, 2, 3]))
-        arr2 = pa.array((0, 1, 2, 3))
-
-        assert arr1.equals(arr2)
-
-    def test_empty_iterable(self):
-        arr = pa.array(StrangeIterable([]))
-        assert len(arr) == 0
-        assert arr.null_count == 0
-        assert arr.type == pa.null()
-        assert arr.to_pylist() == []
-
-
-class TestLimitedConvertIterator(unittest.TestCase):
-    def test_iterator_types(self):
-        arr1 = pa.array(iter(range(3)), type=pa.int64(), size=3)
-        arr2 = pa.array((0, 1, 2))
-        assert arr1.equals(arr2)
-
-    def test_iterator_size_overflow(self):
-        arr1 = pa.array(iter(range(3)), type=pa.int64(), size=2)
-        arr2 = pa.array((0, 1))
-        assert arr1.equals(arr2)
-
-    def test_iterator_size_underflow(self):
-        arr1 = pa.array(iter(range(3)), type=pa.int64(), size=10)
-        arr2 = pa.array((0, 1, 2))
-        assert arr1.equals(arr2)
-
-
-class TestConvertSequence(unittest.TestCase):
-
-    def test_sequence_types(self):
-        arr1 = pa.array([1, 2, 3])
-        arr2 = pa.array((1, 2, 3))
-
-        assert arr1.equals(arr2)
-
-    def test_boolean(self):
-        expected = [True, None, False, None]
-        arr = pa.array(expected)
-        assert len(arr) == 4
-        assert arr.null_count == 2
-        assert arr.type == pa.bool_()
-        assert arr.to_pylist() == expected
-
-    def test_empty_list(self):
-        arr = pa.array([])
-        assert len(arr) == 0
-        assert arr.null_count == 0
-        assert arr.type == pa.null()
-        assert arr.to_pylist() == []
-
-    def test_all_none(self):
-        arr = pa.array([None, None])
-        assert len(arr) == 2
-        assert arr.null_count == 2
-        assert arr.type == pa.null()
-        assert arr.to_pylist() == [None, None]
-
-    def test_integer(self):
-        expected = [1, None, 3, None]
-        arr = pa.array(expected)
-        assert len(arr) == 4
-        assert arr.null_count == 2
-        assert arr.type == pa.int64()
-        assert arr.to_pylist() == expected
-
-    def test_garbage_collection(self):
-        import gc
-
-        # Force the cyclic garbage collector to run
-        gc.collect()
-
-        bytes_before = pa.total_allocated_bytes()
-        pa.array([1, None, 3, None])
-        gc.collect()
-        assert pa.total_allocated_bytes() == bytes_before
-
-    def test_double(self):
-        data = [1.5, 1, None, 2.5, None, None]
-        arr = pa.array(data)
-        assert len(arr) == 6
-        assert arr.null_count == 3
-        assert arr.type == pa.float64()
-        assert arr.to_pylist() == data
-
-    def test_unicode(self):
-        data = [u'foo', u'bar', None, u'mañana']
-        arr = pa.array(data)
-        assert len(arr) == 4
-        assert arr.null_count == 1
-        assert arr.type == pa.string()
-        assert arr.to_pylist() == data
-
-    def test_bytes(self):
-        u1 = b'ma\xc3\xb1ana'
-        data = [b'foo',
-                u1.decode('utf-8'),  # unicode gets encoded,
-                None]
-        arr = pa.array(data)
-        assert len(arr) == 3
-        assert arr.null_count == 1
-        assert arr.type == pa.binary()
-        assert arr.to_pylist() == [b'foo', u1, None]
-
-    def test_utf8_to_unicode(self):
-        # ARROW-1225
-        data = [b'foo', None, b'bar']
-        arr = pa.array(data, type=pa.string())
-        assert arr[0].as_py() == u'foo'
-
-        # test a non-utf8 unicode string
-        val = (u'mañana').encode('utf-16-le')
-        with pytest.raises(pa.ArrowException):
-            pa.array([val], type=pa.string())
-
-    def test_fixed_size_bytes(self):
-        data = [b'foof', None, b'barb', b'2346']
-        arr = pa.array(data, type=pa.binary(4))
-        assert len(arr) == 4
-        assert arr.null_count == 1
-        assert arr.type == pa.binary(4)
-        assert arr.to_pylist() == data
-
-    def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
-        data = [b'foo', None, b'barb', b'2346']
-        with self.assertRaises(pa.ArrowInvalid):
-            pa.array(data, type=pa.binary(4))
-
-    def test_date(self):
-        data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1),
-                datetime.date(2040, 2, 26)]
-        arr = pa.array(data)
-        assert len(arr) == 4
-        assert arr.type == pa.date64()
-        assert arr.null_count == 1
-        assert arr[0].as_py() == datetime.date(2000, 1, 1)
-        assert arr[1].as_py() is None
-        assert arr[2].as_py() == datetime.date(1970, 1, 1)
-        assert arr[3].as_py() == datetime.date(2040, 2, 26)
-
-    def test_date32(self):
-        data = [datetime.date(2000, 1, 1), None]
-        arr = pa.array(data, type=pa.date32())
-
-        data2 = [10957, None]
-        arr2 = pa.array(data2, type=pa.date32())
-
-        for x in [arr, arr2]:
-            assert len(x) == 2
-            assert x.type == pa.date32()
-            assert x.null_count == 1
-            assert x[0].as_py() == datetime.date(2000, 1, 1)
-            assert x[1] is pa.NA
-
-        # Overflow
-        data3 = [2**32, None]
-        with pytest.raises(pa.ArrowException):
-            pa.array(data3, type=pa.date32())
-
-    def test_timestamp(self):
-        data = [
-            datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
-            None,
-            datetime.datetime(2006, 1, 13, 12, 34, 56, 432539),
-            datetime.datetime(2010, 8, 13, 5, 46, 57, 437699)
-        ]
-        arr = pa.array(data)
-        assert len(arr) == 4
-        assert arr.type == pa.timestamp('us')
-        assert arr.null_count == 1
-        assert arr[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                   23, 34, 123456)
-        assert arr[1].as_py() is None
-        assert arr[2].as_py() == datetime.datetime(2006, 1, 13, 12,
-                                                   34, 56, 432539)
-        assert arr[3].as_py() == datetime.datetime(2010, 8, 13, 5,
-                                                   46, 57, 437699)
-
-    def test_timestamp_with_unit(self):
-        data = [
-            datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
-        ]
-
-        s = pa.timestamp('s')
-        ms = pa.timestamp('ms')
-        us = pa.timestamp('us')
-        ns = pa.timestamp('ns')
-
-        arr_s = pa.array(data, type=s)
-        assert len(arr_s) == 1
-        assert arr_s.type == s
-        assert arr_s[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                     23, 34, 0)
-
-        arr_ms = pa.array(data, type=ms)
-        assert len(arr_ms) == 1
-        assert arr_ms.type == ms
-        assert arr_ms[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                      23, 34, 123000)
-
-        arr_us = pa.array(data, type=us)
-        assert len(arr_us) == 1
-        assert arr_us.type == us
-        assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                      23, 34, 123456)
-
-        arr_ns = pa.array(data, type=ns)
-        assert len(arr_ns) == 1
-        assert arr_ns.type == ns
-        assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
-                                                      23, 34, 123456)
-
-    def test_timestamp_from_int_with_unit(self):
-        data = [1]
-
-        s = pa.timestamp('s')
-        ms = pa.timestamp('ms')
-        us = pa.timestamp('us')
-        ns = pa.timestamp('ns')
-
-        arr_s = pa.array(data, type=s)
-        assert len(arr_s) == 1
-        assert arr_s.type == s
-        assert str(arr_s[0]) == "Timestamp('1970-01-01 00:00:01')"
-
-        arr_ms = pa.array(data, type=ms)
-        assert len(arr_ms) == 1
-        assert arr_ms.type == ms
-        assert str(arr_ms[0]) == "Timestamp('1970-01-01 00:00:00.001000')"
-
-        arr_us = pa.array(data, type=us)
-        assert len(arr_us) == 1
-        assert arr_us.type == us
-        assert str(arr_us[0]) == "Timestamp('1970-01-01 00:00:00.000001')"
-
-        arr_ns = pa.array(data, type=ns)
-        assert len(arr_ns) == 1
-        assert arr_ns.type == ns
-        assert str(arr_ns[0]) == "Timestamp('1970-01-01 00:00:00.000000001')"
-
-        with pytest.raises(pa.ArrowException):
-            class CustomClass():
-                pass
-            pa.array([1, CustomClass()], type=ns)
-            pa.array([1, CustomClass()], type=pa.date32())
-            pa.array([1, CustomClass()], type=pa.date64())
-
-    def test_mixed_nesting_levels(self):
-        pa.array([1, 2, None])
-        pa.array([[1], [2], None])
-        pa.array([[1], [2], [None]])
-
-        with self.assertRaises(pa.ArrowInvalid):
-            pa.array([1, 2, [1]])
-
-        with self.assertRaises(pa.ArrowInvalid):
-            pa.array([1, 2, []])
-
-        with self.assertRaises(pa.ArrowInvalid):
-            pa.array([[1], [2], [None, [1]]])
-
-    def test_list_of_int(self):
-        data = [[1, 2, 3], [], None, [1, 2]]
-        arr = pa.array(data)
-        assert len(arr) == 4
-        assert arr.null_count == 1
-        assert arr.type == pa.list_(pa.int64())
-        assert arr.to_pylist() == data
-
-    def test_mixed_types_fails(self):
-        data = ['a', 1, 2.0]
-        with self.assertRaises(pa.ArrowException):
-            pa.array(data)
-
-    def test_mixed_types_with_specified_type_fails(self):
-        data = ['-10', '-5', {'a': 1}, '0', '5', '10']
-
-        type = pa.string()
-        with self.assertRaises(pa.ArrowInvalid):
-            pa.array(data, type=type)
-
-    def test_decimal(self):
-        data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
-        type = pa.decimal128(precision=7, scale=3)
-        arr = pa.array(data, type=type)
-        assert arr.to_pylist() == data
-
-    def test_decimal_different_precisions(self):
-        data = [
-            decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
-        ]
-        type = pa.decimal128(precision=13, scale=3)
-        arr = pa.array(data, type=type)
-        assert arr.to_pylist() == data
-
-    def test_decimal_no_scale(self):
-        data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
-        type = pa.decimal128(precision=10)
-        arr = pa.array(data, type=type)
-        assert arr.to_pylist() == data
-
-    def test_decimal_negative(self):
-        data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
-        type = pa.decimal128(precision=10, scale=6)
-        arr = pa.array(data, type=type)
-        assert arr.to_pylist() == data
-
-    def test_decimal_no_whole_part(self):
-        data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
-        type = pa.decimal128(precision=7, scale=7)
-        arr = pa.array(data, type=type)
-        assert arr.to_pylist() == data
-
-    def test_decimal_large_integer(self):
-        data = [decimal.Decimal('-394029506937548693.42983'),
-                decimal.Decimal('32358695912932.01033')]
-        type = pa.decimal128(precision=23, scale=5)
-        arr = pa.array(data, type=type)
-        assert arr.to_pylist() == data
-
-    def test_range_types(self):
-        arr1 = pa.array(range(3))
-        arr2 = pa.array((0, 1, 2))
-        assert arr1.equals(arr2)
-
-    def test_empty_range(self):
-        arr = pa.array(range(0))
-        assert len(arr) == 0
-        assert arr.null_count == 0
-        assert arr.type == pa.null()
-        assert arr.to_pylist() == []
-
-    def test_structarray(self):
-        ints = pa.array([None, 2, 3], type=pa.int64())
-        strs = pa.array([u'a', None, u'c'], type=pa.string())
-        bools = pa.array([True, False, None], type=pa.bool_())
-        arr = pa.StructArray.from_arrays(
-            ['ints', 'strs', 'bools'],
-            [ints, strs, bools])
-
-        expected = [
-            {'ints': None, 'strs': u'a', 'bools': True},
-            {'ints': 2, 'strs': None, 'bools': False},
-            {'ints': 3, 'strs': u'c', 'bools': None},
-        ]
-
-        pylist = arr.to_pylist()
-        assert pylist == expected, (pylist, expected)
+def test_iterable_types():
+    arr1 = pa.array(StrangeIterable([0, 1, 2, 3]))
+    arr2 = pa.array((0, 1, 2, 3))
+
+    assert arr1.equals(arr2)
+
+
+def test_empty_iterable():
+    arr = pa.array(StrangeIterable([]))
+    assert len(arr) == 0
+    assert arr.null_count == 0
+    assert arr.type == pa.null()
+    assert arr.to_pylist() == []
+
+
+def test_limited_iterator_types():
+    arr1 = pa.array(iter(range(3)), type=pa.int64(), size=3)
+    arr2 = pa.array((0, 1, 2))
+    assert arr1.equals(arr2)
+
+
+def test_limited_iterator_size_overflow():
+    arr1 = pa.array(iter(range(3)), type=pa.int64(), size=2)
+    arr2 = pa.array((0, 1))
+    assert arr1.equals(arr2)
+
+
+def test_limited_iterator_size_underflow():
+    arr1 = pa.array(iter(range(3)), type=pa.int64(), size=10)
+    arr2 = pa.array((0, 1, 2))
+    assert arr1.equals(arr2)
+
+
+def _as_list(xs):
+    return xs
+
+
+def _as_tuple(xs):
+    return tuple(xs)
+
+
+def _as_dict_values(xs):
+    dct = {k: v for k, v in enumerate(xs)}
+    return six.viewvalues(dct)
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+def test_sequence_types(seq):
+    arr1 = pa.array(seq([1, 2, 3]))
+    arr2 = pa.array([1, 2, 3])
+
+    assert arr1.equals(arr2)
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+def test_sequence_boolean(seq):
+    expected = [True, None, False, None]
+    arr = pa.array(seq(expected))
+    assert len(arr) == 4
+    assert arr.null_count == 2
+    assert arr.type == pa.bool_()
+    assert arr.to_pylist() == expected
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+def test_sequence_numpy_boolean(seq):
+    expected = [np.bool(True), None, np.bool(False), None]
+    arr = pa.array(seq(expected))
+    assert len(arr) == 4
+    assert arr.null_count == 2
+    assert arr.type == pa.bool_()
+    assert arr.to_pylist() == expected
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+def test_empty_list(seq):
+    arr = pa.array(seq([]))
+    assert len(arr) == 0
+    assert arr.null_count == 0
+    assert arr.type == pa.null()
+    assert arr.to_pylist() == []
+
+
+def test_sequence_all_none():
+    arr = pa.array([None, None])
+    assert len(arr) == 2
+    assert arr.null_count == 2
+    assert arr.type == pa.null()
+    assert arr.to_pylist() == [None, None]
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+def test_sequence_integer(seq):
+    expected = [1, None, 3, None]
+    arr = pa.array(seq(expected))
+    assert len(arr) == 4
+    assert arr.null_count == 2
+    assert arr.type == pa.int64()
+    assert arr.to_pylist() == expected
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+@pytest.mark.parametrize("np_scalar", [np.int16, np.int32, np.int64, np.uint16,
+                                       np.uint32, np.uint64])
+def test_sequence_numpy_integer(seq, np_scalar):
+    expected = [np_scalar(1), None, np_scalar(3), None]
+    arr = pa.array(seq(expected))
+    assert len(arr) == 4
+    assert arr.null_count == 2
+    assert arr.type == pa.int64()
+    assert arr.to_pylist() == expected
+
+
+def test_garbage_collection():
+    import gc
+
+    # Force the cyclic garbage collector to run
+    gc.collect()
+
+    bytes_before = pa.total_allocated_bytes()
+    pa.array([1, None, 3, None])
+    gc.collect()
+    assert pa.total_allocated_bytes() == bytes_before
+
+
+def test_sequence_double():
+    data = [1.5, 1, None, 2.5, None, None]
+    arr = pa.array(data)
+    assert len(arr) == 6
+    assert arr.null_count == 3
+    assert arr.type == pa.float64()
+    assert arr.to_pylist() == data
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+@pytest.mark.parametrize("np_scalar", [np.float16, np.float32, np.float64])
+def test_sequence_numpy_double(seq, np_scalar):
+    data = [np_scalar(1.5), np_scalar(1), None, np_scalar(2.5), None, None]
+    arr = pa.array(seq(data))
+    assert len(arr) == 6
+    assert arr.null_count == 3
+    assert arr.type == pa.float64()
+    assert arr.to_pylist() == data
+
+
+def test_sequence_unicode():
+    data = [u'foo', u'bar', None, u'mañana']
+    arr = pa.array(data)
+    assert len(arr) == 4
+    assert arr.null_count == 1
+    assert arr.type == pa.string()
+    assert arr.to_pylist() == data
+
+
+def test_sequence_bytes():
+    u1 = b'ma\xc3\xb1ana'
+    data = [b'foo',
+            u1.decode('utf-8'),  # unicode gets encoded,
+            None]
+    arr = pa.array(data)
+    assert len(arr) == 3
+    assert arr.null_count == 1
+    assert arr.type == pa.binary()
+    assert arr.to_pylist() == [b'foo', u1, None]
+
+
+def test_sequence_utf8_to_unicode():
+    # ARROW-1225
+    data = [b'foo', None, b'bar']
+    arr = pa.array(data, type=pa.string())
+    assert arr[0].as_py() == u'foo'
+
+    # test a non-utf8 unicode string
+    val = (u'mañana').encode('utf-16-le')
+    with pytest.raises(pa.ArrowException):
+        pa.array([val], type=pa.string())
+
+
+def test_sequence_fixed_size_bytes():
+    data = [b'foof', None, b'barb', b'2346']
+    arr = pa.array(data, type=pa.binary(4))
+    assert len(arr) == 4
+    assert arr.null_count == 1
+    assert arr.type == pa.binary(4)
+    assert arr.to_pylist() == data
+
+
+def test_fixed_size_bytes_does_not_accept_varying_lengths():
+    data = [b'foo', None, b'barb', b'2346']
+    with pytest.raises(pa.ArrowInvalid):
+        pa.array(data, type=pa.binary(4))
+
+
+def test_sequence_date():
+    data = [datetime.date(2000, 1, 1), None, datetime.date(1970, 1, 1),
+            datetime.date(2040, 2, 26)]
+    arr = pa.array(data)
+    assert len(arr) == 4
+    assert arr.type == pa.date64()
+    assert arr.null_count == 1
+    assert arr[0].as_py() == datetime.date(2000, 1, 1)
+    assert arr[1].as_py() is None
+    assert arr[2].as_py() == datetime.date(1970, 1, 1)
+    assert arr[3].as_py() == datetime.date(2040, 2, 26)
+
+
+def test_sequence_date32():
+    data = [datetime.date(2000, 1, 1), None]
+    arr = pa.array(data, type=pa.date32())
+
+    data2 = [10957, None]
+    arr2 = pa.array(data2, type=pa.date32())
+
+    for x in [arr, arr2]:
+        assert len(x) == 2
+        assert x.type == pa.date32()
+        assert x.null_count == 1
+        assert x[0].as_py() == datetime.date(2000, 1, 1)
+        assert x[1] is pa.NA
+
+    # Overflow
+    data3 = [2**32, None]
+    with pytest.raises(pa.ArrowException):
+        pa.array(data3, type=pa.date32())
+
+
+def test_sequence_timestamp():
+    data = [
+        datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
+        None,
+        datetime.datetime(2006, 1, 13, 12, 34, 56, 432539),
+        datetime.datetime(2010, 8, 13, 5, 46, 57, 437699)
+    ]
+    arr = pa.array(data)
+    assert len(arr) == 4
+    assert arr.type == pa.timestamp('us')
+    assert arr.null_count == 1
+    assert arr[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                               23, 34, 123456)
+    assert arr[1].as_py() is None
+    assert arr[2].as_py() == datetime.datetime(2006, 1, 13, 12,
+                                               34, 56, 432539)
+    assert arr[3].as_py() == datetime.datetime(2010, 8, 13, 5,
+                                               46, 57, 437699)
+
+
+def test_sequence_numpy_timestamp():
+    data = [
+        np.datetime64(datetime.datetime(2007, 7, 13, 1, 23, 34, 123456)),
+        None,
+        np.datetime64(datetime.datetime(2006, 1, 13, 12, 34, 56, 432539)),
+        np.datetime64(datetime.datetime(2010, 8, 13, 5, 46, 57, 437699))
+    ]
+    arr = pa.array(data)
+    assert len(arr) == 4
+    assert arr.type == pa.timestamp('us')
+    assert arr.null_count == 1
+    assert arr[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                               23, 34, 123456)
+    assert arr[1].as_py() is None
+    assert arr[2].as_py() == datetime.datetime(2006, 1, 13, 12,
+                                               34, 56, 432539)
+    assert arr[3].as_py() == datetime.datetime(2010, 8, 13, 5,
+                                               46, 57, 437699)
+
+
+def test_sequence_timestamp_with_unit():
+    data = [
+        datetime.datetime(2007, 7, 13, 1, 23, 34, 123456),
+    ]
+
+    s = pa.timestamp('s')
+    ms = pa.timestamp('ms')
+    us = pa.timestamp('us')
+    ns = pa.timestamp('ns')
+
+    arr_s = pa.array(data, type=s)
+    assert len(arr_s) == 1
+    assert arr_s.type == s
+    assert arr_s[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                 23, 34, 0)
+
+    arr_ms = pa.array(data, type=ms)
+    assert len(arr_ms) == 1
+    assert arr_ms.type == ms
+    assert arr_ms[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                  23, 34, 123000)
+
+    arr_us = pa.array(data, type=us)
+    assert len(arr_us) == 1
+    assert arr_us.type == us
+    assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                  23, 34, 123456)
+
+    arr_ns = pa.array(data, type=ns)
+    assert len(arr_ns) == 1
+    assert arr_ns.type == ns
+    assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+                                                  23, 34, 123456)
+
+
+def test_sequence_timestamp_from_int_with_unit():
+    data = [1]
+
+    s = pa.timestamp('s')
+    ms = pa.timestamp('ms')
+    us = pa.timestamp('us')
+    ns = pa.timestamp('ns')
+
+    arr_s = pa.array(data, type=s)
+    assert len(arr_s) == 1
+    assert arr_s.type == s
+    assert str(arr_s[0]) == "Timestamp('1970-01-01 00:00:01')"
+
+    arr_ms = pa.array(data, type=ms)
+    assert len(arr_ms) == 1
+    assert arr_ms.type == ms
+    assert str(arr_ms[0]) == "Timestamp('1970-01-01 00:00:00.001000')"
+
+    arr_us = pa.array(data, type=us)
+    assert len(arr_us) == 1
+    assert arr_us.type == us
+    assert str(arr_us[0]) == "Timestamp('1970-01-01 00:00:00.000001')"
+
+    arr_ns = pa.array(data, type=ns)
+    assert len(arr_ns) == 1
+    assert arr_ns.type == ns
+    assert str(arr_ns[0]) == "Timestamp('1970-01-01 00:00:00.000000001')"
+
+    with pytest.raises(pa.ArrowException):
+        class CustomClass():
+            pass
+        pa.array([1, CustomClass()], type=ns)
+        pa.array([1, CustomClass()], type=pa.date32())
+        pa.array([1, CustomClass()], type=pa.date64())
+
+
+def test_sequence_mixed_nesting_levels():
+    pa.array([1, 2, None])
+    pa.array([[1], [2], None])
+    pa.array([[1], [2], [None]])
+
+    with pytest.raises(pa.ArrowInvalid):
+        pa.array([1, 2, [1]])
+
+    with pytest.raises(pa.ArrowInvalid):
+        pa.array([1, 2, []])
+
+    with pytest.raises(pa.ArrowInvalid):
+        pa.array([[1], [2], [None, [1]]])
+
+
+def test_sequence_list_of_int():
+    data = [[1, 2, 3], [], None, [1, 2]]
+    arr = pa.array(data)
+    assert len(arr) == 4
+    assert arr.null_count == 1
+    assert arr.type == pa.list_(pa.int64())
+    assert arr.to_pylist() == data
+
+
+def test_sequence_mixed_types_fails():
+    data = ['a', 1, 2.0]
+    with pytest.raises(pa.ArrowException):
+        pa.array(data)
+
+
+def test_sequence_mixed_types_with_specified_type_fails():
+    data = ['-10', '-5', {'a': 1}, '0', '5', '10']
+
+    type = pa.string()
+    with pytest.raises(pa.ArrowInvalid):
+        pa.array(data, type=type)
+
+
+def test_sequence_decimal():
+    data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
+    type = pa.decimal128(precision=7, scale=3)
+    arr = pa.array(data, type=type)
+    assert arr.to_pylist() == data
+
+
+def test_sequence_decimal_different_precisions():
+    data = [
+        decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
+    ]
+    type = pa.decimal128(precision=13, scale=3)
+    arr = pa.array(data, type=type)
+    assert arr.to_pylist() == data
+
+
+def test_sequence_decimal_no_scale():
+    data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
+    type = pa.decimal128(precision=10)
+    arr = pa.array(data, type=type)
+    assert arr.to_pylist() == data
+
+
+def test_sequence_decimal_negative():
+    data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
+    type = pa.decimal128(precision=10, scale=6)
+    arr = pa.array(data, type=type)
+    assert arr.to_pylist() == data
+
+
+def test_sequence_decimal_no_whole_part():
+    data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
+    type = pa.decimal128(precision=7, scale=7)
+    arr = pa.array(data, type=type)
+    assert arr.to_pylist() == data
+
+
+def test_sequence_decimal_large_integer():
+    data = [decimal.Decimal('-394029506937548693.42983'),
+            decimal.Decimal('32358695912932.01033')]
+    type = pa.decimal128(precision=23, scale=5)
+    arr = pa.array(data, type=type)
+    assert arr.to_pylist() == data
+
+
+def test_range_types():
+    arr1 = pa.array(range(3))
+    arr2 = pa.array((0, 1, 2))
+    assert arr1.equals(arr2)
+
+
+def test_empty_range():
+    arr = pa.array(range(0))
+    assert len(arr) == 0
+    assert arr.null_count == 0
+    assert arr.type == pa.null()
+    assert arr.to_pylist() == []
+
+
+def test_structarray():
+    ints = pa.array([None, 2, 3], type=pa.int64())
+    strs = pa.array([u'a', None, u'c'], type=pa.string())
+    bools = pa.array([True, False, None], type=pa.bool_())
+    arr = pa.StructArray.from_arrays(
+        ['ints', 'strs', 'bools'],
+        [ints, strs, bools])
+
+    expected = [
+        {'ints': None, 'strs': u'a', 'bools': True},
+        {'ints': 2, 'strs': None, 'bools': False},
+        {'ints': 3, 'strs': u'c', 'bools': None},
+    ]
+
+    pylist = arr.to_pylist()
+    assert pylist == expected, (pylist, expected)

From e50b1b3263007f20d9d0e29ace7c15685e334296 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sun, 28 Jan 2018 17:31:52 +0100
Subject: [PATCH 1411/1644] ARROW-2028: [Python] extra_cmake_args needs to be
 passed through shlex.split

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1501 from xhochy/ARROW-2028 and squashes the following commits:

212f365 [Korn, Uwe] ARROW-2028: [Python] extra_cmake_args needs to be passed through shlex.split
---
 python/setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index 3d3831dc048c6..076d7e489b5f3 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -21,6 +21,7 @@
 import os
 import os.path as osp
 import re
+import shlex
 import shutil
 import sys
 
@@ -180,8 +181,9 @@ def _run_cmake(self):
         cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                              .format(self.build_type.lower()))
 
+        extra_cmake_args = shlex.split(self.extra_cmake_args)
         if sys.platform != 'win32':
-            cmake_command = (['cmake', self.extra_cmake_args] +
+            cmake_command = (['cmake'] + extra_cmake_args +
                              cmake_options + [source])
 
             print("-- Runnning cmake for pyarrow")
@@ -197,13 +199,11 @@ def _run_cmake(self):
             self.spawn(args)
             print("-- Finished cmake --build for pyarrow")
         else:
-            import shlex
             cmake_generator = 'Visual Studio 14 2015 Win64'
             if not is_64_bit:
                 raise RuntimeError('Not supported on 32-bit Windows')
 
             # Generate the build files
-            extra_cmake_args = shlex.split(self.extra_cmake_args)
             cmake_command = (['cmake'] + extra_cmake_args +
                              cmake_options +
                              [source, '-G', cmake_generator])

From d3226349fc61a0ffbb2139f259053ae787e500c8 Mon Sep 17 00:00:00 2001
From: Licht-T <licht-t@outlook.jp>
Date: Sun, 28 Jan 2018 17:33:20 +0100
Subject: [PATCH 1412/1644] ARROW-1992: [C++/Python] Fix segfault when string
 to categorical empty string array

This closes [ARROW-1992](https://issues.apache.org/jira/browse/ARROW-1992).

Author: Licht-T <licht-t@outlook.jp>

Closes #1508 from Licht-T/fix-segfault-when-string_to_categorical-empty-string-array and squashes the following commits:

afea4be [Licht-T] BUG: Fix segfault when to_pandas the empty string array with string_to_categorical=True
f90e7b8 [Licht-T] TST: Add test for to_pandas the empty string array with string_to_categorical=True
---
 cpp/src/arrow/compute/kernels/hash.cc       |  8 +++++++-
 python/pyarrow/tests/test_convert_pandas.py | 15 +++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index 8fac7965d0ed9..acbf403987b40 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -407,12 +407,18 @@ class HashTableKernel<Type, Action, enable_if_binary<Type>> : public HashTable {
   }
 
   Status Append(const ArrayData& arr) override {
+    constexpr uint8_t empty_value = 0;
     if (!initialized_) {
       RETURN_NOT_OK(Init());
     }
 
     const int32_t* offsets = GetValues<int32_t>(arr, 1);
-    const uint8_t* data = GetValues<uint8_t>(arr, 2);
+    const uint8_t* data;
+    if (arr.buffers[2].get() == nullptr) {
+      data = &empty_value;
+    } else {
+      data = GetValues<uint8_t>(arr, 2);
+    }
 
     auto action = static_cast<Action*>(this);
     RETURN_NOT_OK(action->Reserve(arr.length));
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 5acb9c3dbe9a1..fa265e55cfd76 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1237,6 +1237,21 @@ def test_decimal_metadata(self):
         assert data_column['numpy_type'] == 'object'
         assert data_column['metadata'] == {'precision': 26, 'scale': 11}
 
+    def test_table_empty_str(self):
+        values = ['', '', '', '', '']
+        df = pd.DataFrame({'strings': values})
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
+        table = pa.Table.from_pandas(df, schema=schema)
+
+        result1 = table.to_pandas(strings_to_categorical=False)
+        expected1 = pd.DataFrame({'strings': values})
+        tm.assert_frame_equal(result1, expected1, check_dtype=True)
+
+        result2 = table.to_pandas(strings_to_categorical=True)
+        expected2 = pd.DataFrame({'strings': pd.Categorical(values)})
+        tm.assert_frame_equal(result2, expected2, check_dtype=True)
+
     def test_table_str_to_categorical_without_na(self):
         values = ['a', 'a', 'b', 'b', 'c']
         df = pd.DataFrame({'strings': values})

From 0621765defc022e0fae68f4cac52835699b98500 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 30 Jan 2018 10:31:20 +0100
Subject: [PATCH 1413/1644] ARROW-2048: [Python/C++] Upate Thrift pin to 0.11

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1528 from xhochy/ARROW-2048 and squashes the following commits:

aa0212a [Uwe L. Korn] ARROW-2048: [Python/C++] Upate Thrift pin to 0.11
---
 ci/msvc-build.bat                         |  2 +-
 ci/travis_before_script_cpp.sh            |  2 +-
 cpp/src/arrow/stl.h                       |  1 +
 python/manylinux1/Dockerfile-x86_64       |  2 +-
 python/manylinux1/Dockerfile-x86_64_base  |  3 +++
 python/manylinux1/scripts/build_bison.sh  | 26 +++++++++++++++++++++++
 python/manylinux1/scripts/build_thrift.sh |  2 +-
 7 files changed, 34 insertions(+), 4 deletions(-)
 create mode 100755 python/manylinux1/scripts/build_bison.sh

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 94eb16a5e506b..9651772ca3fe9 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -81,7 +81,7 @@ conda info -a
 
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython ^
-      thrift-cpp=0.10.0
+      thrift-cpp=0.11.0
 
 if "%JOB%" == "Toolchain" (
 
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 2f164c4168d0d..7c1d726d4d37e 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -47,7 +47,7 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
         zlib \
         cmake \
         curl \
-        thrift-cpp=0.10.0 \
+        thrift-cpp=0.11.0 \
         ninja
 
   # HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
diff --git a/cpp/src/arrow/stl.h b/cpp/src/arrow/stl.h
index 3250b5a320464..1e31ca769ae0b 100644
--- a/cpp/src/arrow/stl.h
+++ b/cpp/src/arrow/stl.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_STL_H
 #define ARROW_STL_H
 
+#include <memory>
 #include <string>
 #include <tuple>
 #include <vector>
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 919a32be715b0..9c00e7ea256c9 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2048
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index 0160aa4eea509..ec7893080f65b 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -42,6 +42,9 @@ ADD scripts/build_flatbuffers.sh /
 RUN /build_flatbuffers.sh
 ENV FLATBUFFERS_HOME /usr
 
+ADD scripts/build_bison.sh /
+RUN /build_bison.sh
+
 ADD scripts/build_thrift.sh /
 RUN /build_thrift.sh
 ENV THRIFT_HOME /usr
diff --git a/python/manylinux1/scripts/build_bison.sh b/python/manylinux1/scripts/build_bison.sh
new file mode 100755
index 0000000000000..29cc0be6adf6c
--- /dev/null
+++ b/python/manylinux1/scripts/build_bison.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -ex
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+wget http://ftp.gnu.org/gnu/bison/bison-3.0.4.tar.gz
+tar xf bison-3.0.4.tar.gz
+pushd bison-3.0.4
+./configure --prefix=/usr
+make -j4
+make install
+popd
+rm -rf bison-3.0.4 bison-3.0.4.tar.gz
diff --git a/python/manylinux1/scripts/build_thrift.sh b/python/manylinux1/scripts/build_thrift.sh
index 28aa75b7413de..aaec4ad6bad41 100755
--- a/python/manylinux1/scripts/build_thrift.sh
+++ b/python/manylinux1/scripts/build_thrift.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-export THRIFT_VERSION=0.10.0
+export THRIFT_VERSION=0.11.0
 wget http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz
 tar xf thrift-${THRIFT_VERSION}.tar.gz
 pushd thrift-${THRIFT_VERSION}

From 40dd9cc25a46aa56a5d852fbc8ebdbc55b5fe8d6 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <pitrou@free.fr>
Date: Tue, 30 Jan 2018 10:16:51 -0500
Subject: [PATCH 1414/1644] ARROW-2033: [Python] Fix pa.array() with iterator
 input

Iterator (not iterable) input was broken with pa.array() unless both type and size were explicitly passed.

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Antoine Pitrou <pitrou@free.fr>

Closes #1513 from pitrou/ARROW-2033-pa-array-iterator and squashes the following commits:

0013889a [Wes McKinney] Code review comments
dc95be29 [Antoine Pitrou] Fix pyarrow.array with iterator input

Change-Id: I930c3309e3fde12e65ede066b47985f67f7f4037
---
 cpp/src/arrow/python/builtin_convert.cc      | 174 ++++++++++---------
 cpp/src/arrow/python/builtin_convert.h       |  18 +-
 python/pyarrow/array.pxi                     |  15 +-
 python/pyarrow/includes/libarrow.pxd         |  13 +-
 python/pyarrow/tests/test_convert_builtin.py |  19 ++
 5 files changed, 141 insertions(+), 98 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index f7a370cdc4c06..b41c55d9c3773 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -172,38 +172,26 @@ class SeqVisitor {
   Status Visit(PyObject* obj, int level = 0) {
     max_nesting_level_ = std::max(max_nesting_level_, level);
 
-    // Loop through either a sequence or an iterator.
-    if (PySequence_Check(obj)) {
-      Py_ssize_t size = PySequence_Size(obj);
-      for (int64_t i = 0; i < size; ++i) {
-        OwnedRef ref;
-        if (PyArray_Check(obj)) {
-          auto array = reinterpret_cast<PyArrayObject*>(obj);
-          auto ptr = reinterpret_cast<const char*>(PyArray_GETPTR1(array, i));
-
-          ref.reset(PyArray_GETITEM(array, ptr));
-          RETURN_IF_PYERROR();
+    // Loop through a sequence
+    if (!PySequence_Check(obj))
+      return Status::TypeError("Object is not a sequence or iterable");
 
-          RETURN_NOT_OK(VisitElem(ref, level));
-        } else {
-          ref.reset(PySequence_GetItem(obj, i));
-          RETURN_IF_PYERROR();
-          RETURN_NOT_OK(VisitElem(ref, level));
-        }
-      }
-    } else if (PyObject_HasAttrString(obj, "__iter__")) {
-      OwnedRef iter(PyObject_GetIter(obj));
-      RETURN_IF_PYERROR();
+    Py_ssize_t size = PySequence_Size(obj);
+    for (int64_t i = 0; i < size; ++i) {
+      OwnedRef ref;
+      if (PyArray_Check(obj)) {
+        auto array = reinterpret_cast<PyArrayObject*>(obj);
+        auto ptr = reinterpret_cast<const char*>(PyArray_GETPTR1(array, i));
 
-      PyObject* item = NULLPTR;
-      while ((item = PyIter_Next(iter.obj()))) {
+        ref.reset(PyArray_GETITEM(array, ptr));
         RETURN_IF_PYERROR();
 
-        OwnedRef ref(item);
+        RETURN_NOT_OK(VisitElem(ref, level));
+      } else {
+        ref.reset(PySequence_GetItem(obj, i));
+        RETURN_IF_PYERROR();
         RETURN_NOT_OK(VisitElem(ref, level));
       }
-    } else {
-      return Status::TypeError("Object is not a sequence or iterable");
     }
     return Status::OK();
   }
@@ -285,25 +273,45 @@ class SeqVisitor {
   }
 };
 
-Status InferArrowSize(PyObject* obj, int64_t* size) {
+// Convert *obj* to a sequence if necessary
+// Fill *size* to its length.  If >= 0 on entry, *size* is an upper size
+// bound that may lead to truncation.
+Status ConvertToSequenceAndInferSize(PyObject* obj, PyObject** seq, int64_t* size) {
   if (PySequence_Check(obj)) {
-    *size = static_cast<int64_t>(PySequence_Size(obj));
-  } else if (PyObject_HasAttrString(obj, "__iter__")) {
+    // obj is already a sequence
+    int64_t real_size = static_cast<int64_t>(PySequence_Size(obj));
+    if (*size < 0) {
+      *size = real_size;
+    } else {
+      *size = std::min(real_size, *size);
+    }
+    Py_INCREF(obj);
+    *seq = obj;
+  } else if (*size < 0) {
+    // unknown size, exhaust iterator
+    *seq = PySequence_List(obj);
+    RETURN_IF_PYERROR();
+    *size = static_cast<int64_t>(PyList_GET_SIZE(*seq));
+  } else {
+    // size is known but iterator could be infinite
+    Py_ssize_t i, n = *size;
     PyObject* iter = PyObject_GetIter(obj);
+    RETURN_IF_PYERROR();
     OwnedRef iter_ref(iter);
-    *size = 0;
-    PyObject* item;
-    while ((item = PyIter_Next(iter))) {
-      OwnedRef item_ref(item);
-      *size += 1;
+    PyObject* lst = PyList_New(n);
+    RETURN_IF_PYERROR();
+    for (i = 0; i < n; i++) {
+      PyObject* item = PyIter_Next(iter);
+      if (!item) break;
+      PyList_SET_ITEM(lst, i, item);
     }
-  } else {
-    return Status::TypeError("Object is not a sequence or iterable");
-  }
-  if (PyErr_Occurred()) {
-    // Not a sequence
-    PyErr_Clear();
-    return Status::TypeError("Object is not a sequence or iterable");
+    // Shrink list if len(iterator) < size
+    if (i < n && PyList_SetSlice(lst, i, n, NULL)) {
+      Py_DECREF(lst);
+      return Status::UnknownError("failed to resize list");
+    }
+    *seq = lst;
+    *size = std::min<int64_t>(i, *size);
   }
   return Status::OK();
 }
@@ -325,7 +333,10 @@ Status InferArrowType(PyObject* obj, std::shared_ptr<DataType>* out_type) {
 
 Status InferArrowTypeAndSize(PyObject* obj, int64_t* size,
                              std::shared_ptr<DataType>* out_type) {
-  RETURN_NOT_OK(InferArrowSize(obj, size));
+  if (!PySequence_Check(obj)) {
+    return Status::TypeError("Object is not a sequence");
+  }
+  *size = static_cast<int64_t>(PySequence_Size(obj));
 
   // For 0-length sequences, refuse to guess
   if (*size == 0) {
@@ -382,27 +393,8 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
           RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
         }
       }
-    } else if (PyObject_HasAttrString(obj, "__iter__")) {
-      PyObject* iter = PyObject_GetIter(obj);
-      OwnedRef iter_ref(iter);
-      PyObject* item;
-      int64_t i = 0;
-      // To allow people with long generators to only convert a subset, stop
-      // consuming at size.
-      while ((item = PyIter_Next(iter)) && i < size) {
-        OwnedRef ref(item);
-        if (ref.obj() == Py_None) {
-          RETURN_NOT_OK(this->typed_builder_->AppendNull());
-        } else {
-          RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
-        }
-        ++i;
-      }
-      if (size != i) {
-        RETURN_NOT_OK(this->typed_builder_->Resize(i));
-      }
     } else {
-      return Status::TypeError("Object is not a sequence or iterable");
+      return Status::TypeError("Object is not a sequence");
     }
     return Status::OK();
   }
@@ -830,38 +822,56 @@ Status AppendPySequence(PyObject* obj, int64_t size,
   return converter->AppendData(obj, size);
 }
 
-Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out) {
+static Status ConvertPySequenceReal(PyObject* obj, int64_t size,
+                                    const std::shared_ptr<DataType>* type,
+                                    MemoryPool* pool, std::shared_ptr<Array>* out) {
   PyAcquireGIL lock;
-  std::shared_ptr<DataType> type;
-  int64_t size;
-  RETURN_NOT_OK(InferArrowTypeAndSize(obj, &size, &type));
-  return ConvertPySequence(obj, pool, out, type, size);
-}
 
-Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
-                         const std::shared_ptr<DataType>& type, int64_t size) {
-  PyAcquireGIL lock;
+  PyObject* seq;
+  ScopedRef tmp_seq_nanny;
+
+  std::shared_ptr<DataType> real_type;
+
+  RETURN_NOT_OK(ConvertToSequenceAndInferSize(obj, &seq, &size));
+  tmp_seq_nanny.reset(seq);
+  if (type == nullptr) {
+    RETURN_NOT_OK(InferArrowType(seq, &real_type));
+  } else {
+    real_type = *type;
+  }
+  DCHECK_GE(size, 0);
+
   // Handle NA / NullType case
-  if (type->id() == Type::NA) {
+  if (real_type->id() == Type::NA) {
     out->reset(new NullArray(size));
     return Status::OK();
   }
 
   // Give the sequence converter an array builder
   std::unique_ptr<ArrayBuilder> builder;
-  RETURN_NOT_OK(MakeBuilder(pool, type, &builder));
-  RETURN_NOT_OK(AppendPySequence(obj, size, type, builder.get()));
+  RETURN_NOT_OK(MakeBuilder(pool, real_type, &builder));
+  RETURN_NOT_OK(AppendPySequence(seq, size, real_type, builder.get()));
   return builder->Finish(out);
 }
 
-Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
-                         const std::shared_ptr<DataType>& type) {
-  int64_t size;
-  {
-    PyAcquireGIL lock;
-    RETURN_NOT_OK(InferArrowSize(obj, &size));
-  }
-  return ConvertPySequence(obj, pool, out, type, size);
+Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out) {
+  return ConvertPySequenceReal(obj, -1, nullptr, pool, out);
+}
+
+Status ConvertPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
+                         MemoryPool* pool, std::shared_ptr<Array>* out) {
+  return ConvertPySequenceReal(obj, -1, &type, pool, out);
+}
+
+Status ConvertPySequence(PyObject* obj, int64_t size, MemoryPool* pool,
+                         std::shared_ptr<Array>* out) {
+  return ConvertPySequenceReal(obj, size, nullptr, pool, out);
+}
+
+Status ConvertPySequence(PyObject* obj, int64_t size,
+                         const std::shared_ptr<DataType>& type, MemoryPool* pool,
+                         std::shared_ptr<Array>* out) {
+  return ConvertPySequenceReal(obj, size, &type, pool, out);
 }
 
 Status CheckPythonBytesAreFixedLength(PyObject* obj, Py_ssize_t expected_length) {
diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h
index cde7a1bd4cfdc..4bd3f08edf162 100644
--- a/cpp/src/arrow/python/builtin_convert.h
+++ b/cpp/src/arrow/python/builtin_convert.h
@@ -39,11 +39,11 @@ class Status;
 
 namespace py {
 
+// These three functions take a sequence input, not arbitrary iterables
 ARROW_EXPORT arrow::Status InferArrowType(PyObject* obj,
                                           std::shared_ptr<arrow::DataType>* out_type);
 ARROW_EXPORT arrow::Status InferArrowTypeAndSize(
     PyObject* obj, int64_t* size, std::shared_ptr<arrow::DataType>* out_type);
-ARROW_EXPORT arrow::Status InferArrowSize(PyObject* obj, int64_t* size);
 
 ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj, int64_t size,
                                             const std::shared_ptr<arrow::DataType>& type,
@@ -53,15 +53,21 @@ ARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj, int64_t size,
 ARROW_EXPORT
 Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out);
 
-// Size inference
+// Type inference only
 ARROW_EXPORT
-Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
-                         const std::shared_ptr<DataType>& type);
+Status ConvertPySequence(PyObject* obj, int64_t size, MemoryPool* pool,
+                         std::shared_ptr<Array>* out);
+
+// Size inference only
+ARROW_EXPORT
+Status ConvertPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
+                         MemoryPool* pool, std::shared_ptr<Array>* out);
 
 // No inference
 ARROW_EXPORT
-Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>* out,
-                         const std::shared_ptr<DataType>& type, int64_t size);
+Status ConvertPySequence(PyObject* obj, int64_t size,
+                         const std::shared_ptr<DataType>& type, MemoryPool* pool,
+                         std::shared_ptr<Array>* out);
 
 ARROW_EXPORT
 Status InvalidConversion(PyObject* obj, const std::string& expected_type_name,
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index cca9425881b00..caeefd2ff4f6a 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -21,14 +21,21 @@ cdef _sequence_to_array(object sequence, object size, DataType type,
     cdef shared_ptr[CArray] out
     cdef int64_t c_size
     if type is None:
-        with nogil:
-            check_status(ConvertPySequence(sequence, pool, &out))
+        if size is None:
+            with nogil:
+                check_status(ConvertPySequence(sequence, pool, &out))
+        else:
+            c_size = size
+            with nogil:
+                check_status(
+                    ConvertPySequence(sequence, c_size, pool, &out)
+                )
     else:
         if size is None:
             with nogil:
                 check_status(
                     ConvertPySequence(
-                        sequence, pool, &out, type.sp_type
+                        sequence, type.sp_type, pool, &out,
                     )
                 )
         else:
@@ -36,7 +43,7 @@ cdef _sequence_to_array(object sequence, object size, DataType type,
             with nogil:
                 check_status(
                     ConvertPySequence(
-                        sequence, pool, &out, type.sp_type, c_size
+                        sequence, c_size, type.sp_type, pool, &out,
                     )
                 )
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 91bc96dc63f89..2e83f0701ce2e 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -852,13 +852,14 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
     CStatus ConvertPySequence(object obj, CMemoryPool* pool,
                               shared_ptr[CArray]* out)
-    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
-                              shared_ptr[CArray]* out,
-                              const shared_ptr[CDataType]& type)
-    CStatus ConvertPySequence(object obj, CMemoryPool* pool,
-                              shared_ptr[CArray]* out,
+    CStatus ConvertPySequence(object obj, const shared_ptr[CDataType]& type,
+                              CMemoryPool* pool, shared_ptr[CArray]* out)
+    CStatus ConvertPySequence(object obj, int64_t size, CMemoryPool* pool,
+                              shared_ptr[CArray]* out)
+    CStatus ConvertPySequence(object obj, int64_t size,
                               const shared_ptr[CDataType]& type,
-                              int64_t size)
+                              CMemoryPool* pool,
+                              shared_ptr[CArray]* out)
 
     CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
 
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index fa603b1a92fa2..2b317dfbc3fee 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -23,6 +23,7 @@
 
 import datetime
 import decimal
+import itertools
 import numpy as np
 import six
 
@@ -68,6 +69,24 @@ def test_limited_iterator_size_underflow():
     assert arr1.equals(arr2)
 
 
+def test_iterator_without_size():
+    expected = pa.array((0, 1, 2))
+    arr1 = pa.array(iter(range(3)))
+    assert arr1.equals(expected)
+    # Same with explicit type
+    arr1 = pa.array(iter(range(3)), type=pa.int64())
+    assert arr1.equals(expected)
+
+
+def test_infinite_iterator():
+    expected = pa.array((0, 1, 2))
+    arr1 = pa.array(itertools.count(0), size=3)
+    assert arr1.equals(expected)
+    # Same with explicit type
+    arr1 = pa.array(itertools.count(0), type=pa.int64(), size=3)
+    assert arr1.equals(expected)
+
+
 def _as_list(xs):
     return xs
 

From 673125fd416cbd2e5c2cb9cb6a4c925adecdaf2c Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 30 Jan 2018 11:29:37 -0500
Subject: [PATCH 1415/1644] ARROW-2054: [C++] Fix compilation warnings

One of them is a pointer aliasing offense (thus a real bug), the other ones could merely be ignored.

Author: Antoine Pitrou <antoine@python.org>

Closes #1533 from pitrou/ARROW-2054-cpp-warnings and squashes the following commits:

2c0d17eb [Antoine Pitrou] ARROW-2054: [C++] Fix compilation warnings
---
 cpp/src/arrow/ipc/json-internal.cc      |  4 ++--
 cpp/src/arrow/python/builtin_convert.cc |  2 ++
 cpp/src/arrow/python/io.cc              | 16 ++++++++++------
 cpp/src/plasma/fling.cc                 |  2 +-
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index 4088a8f20e6a0..204bbc40b0072 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -866,7 +866,7 @@ static Status GetField(const rj::Value& obj, const DictionaryMemo* dictionary_me
   if (dictionary_memo != nullptr && it_dictionary != json_field.MemberEnd()) {
     // Field is dictionary encoded. We must have already
     RETURN_NOT_OBJECT("dictionary", it_dictionary, json_field);
-    int64_t dictionary_id;
+    int64_t dictionary_id = -1;
     bool is_ordered;
     std::shared_ptr<DataType> index_type;
     RETURN_NOT_OK(ParseDictionary(it_dictionary->value.GetObject(), &dictionary_id,
@@ -1346,7 +1346,7 @@ static Status ReadDictionaries(const rj::Value& doc, const DictionaryTypeMap& id
 
   for (const rj::Value& val : dictionary_array) {
     DCHECK(val.IsObject());
-    int64_t dictionary_id;
+    int64_t dictionary_id = -1;
     std::shared_ptr<Array> dictionary;
     RETURN_NOT_OK(
         ReadDictionary(val.GetObject(), id_to_field, pool, &dictionary_id, &dictionary));
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index b41c55d9c3773..63d388925c711 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -582,6 +582,8 @@ class TimestampConverter
         case TimeUnit::NANO:
           t = PyDateTime_to_ns(pydatetime);
           break;
+        default:
+          return Status::UnknownError("Invalid time unit");
       }
     } else if (PyArray_CheckAnyScalarExact(item.obj())) {
       // numpy.datetime64
diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index 9d32ead524b93..2cff046085e69 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -26,6 +26,7 @@
 #include "arrow/io/memory.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 #include "arrow/python/common.h"
 
@@ -133,12 +134,14 @@ Status PyReadableFile::Tell(int64_t* position) const {
 
 Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
   PyAcquireGIL lock;
-  PyObject* bytes_obj;
+
+  PyObject* bytes_obj = NULL;
   ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
+  DCHECK(bytes_obj != NULL);
 
   *bytes_read = PyBytes_GET_SIZE(bytes_obj);
   std::memcpy(out, PyBytes_AS_STRING(bytes_obj), *bytes_read);
-  Py_DECREF(bytes_obj);
+  Py_XDECREF(bytes_obj);
 
   return Status::OK();
 }
@@ -146,11 +149,12 @@ Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
 Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   PyAcquireGIL lock;
 
-  PyObject* bytes_obj;
+  PyObject* bytes_obj = NULL;
   ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
+  DCHECK(bytes_obj != NULL);
 
   *out = std::make_shared<PyBuffer>(bytes_obj);
-  Py_DECREF(bytes_obj);
+  Py_XDECREF(bytes_obj);
 
   return Status::OK();
 }
@@ -172,13 +176,13 @@ Status PyReadableFile::ReadAt(int64_t position, int64_t nbytes,
 Status PyReadableFile::GetSize(int64_t* size) {
   PyAcquireGIL lock;
 
-  int64_t current_position;
+  int64_t current_position = -1;
 
   ARROW_RETURN_NOT_OK(file_->Tell(&current_position));
 
   ARROW_RETURN_NOT_OK(file_->Seek(0, 2));
 
-  int64_t file_size;
+  int64_t file_size = -1;
   ARROW_RETURN_NOT_OK(file_->Tell(&file_size));
 
   // Restore previous file position
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
index 819ec1623055b..26afd87066c2b 100644
--- a/cpp/src/plasma/fling.cc
+++ b/cpp/src/plasma/fling.cc
@@ -43,7 +43,7 @@ int send_fd(int conn, int fd) {
   header->cmsg_level = SOL_SOCKET;
   header->cmsg_type = SCM_RIGHTS;
   header->cmsg_len = CMSG_LEN(sizeof(int));
-  *reinterpret_cast<int*>(CMSG_DATA(header)) = fd;
+  memcpy(CMSG_DATA(header), reinterpret_cast<void*>(&fd), sizeof(int));
 
   // Send file descriptor.
   ssize_t r = sendmsg(conn, &msg, 0);

From 8d78376979965a203207899084820631394de017 Mon Sep 17 00:00:00 2001
From: moriyoshi <mozo@mozo.jp>
Date: Tue, 30 Jan 2018 11:42:57 -0500
Subject: [PATCH 1416/1644] ARROW-2047: [Python] Use sys.executable instead of
 one in the search path.

* It currently relies on the behavior of `subprocess.check_call()` that searches for the executable in the search path. Because of it, the test run needs a valid search path setup.

Author: moriyoshi <mozo@mozo.jp>

Closes #1525 from moriyoshi/moriyoshi/use-sys-executable and squashes the following commits:

f4c92227 [moriyoshi] Use sys.executable instead of one in path
---
 python/pyarrow/tests/test_serialization.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index e4681e3a59751..9cad81fc98f1e 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -554,7 +554,7 @@ def test_deserialize_buffer_in_different_process():
 
     dir_path = os.path.dirname(os.path.realpath(__file__))
     python_file = os.path.join(dir_path, 'deserialize_buffer.py')
-    subprocess.check_call(['python', python_file, f.name])
+    subprocess.check_call([sys.executable, python_file, f.name])
 
 
 def test_set_pickle():

From 5c704bce42e3fa71ea4586368962d41173b3e17b Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 30 Jan 2018 15:05:14 -0500
Subject: [PATCH 1417/1644] ARROW-1705: [Python] allow building array from
 dicts

Accept passing a list of dicts to pa.array() if a struct type is given.

Based on PR #1513.

Author: Antoine Pitrou <antoine@python.org>

Closes #1530 from pitrou/ARROW-1705-struct-array-from-dicts and squashes the following commits:

2b9133af [Antoine Pitrou] ARROW-1705: [Python] allow building array from dicts
---
 cpp/src/arrow/python/builtin_convert.cc      | 289 +++++++++++++------
 python/pyarrow/tests/test_convert_builtin.py |  25 ++
 2 files changed, 220 insertions(+), 94 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 63d388925c711..1b3c101758eec 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -23,6 +23,8 @@
 #include <limits>
 #include <sstream>
 #include <string>
+#include <utility>
+#include <vector>
 
 #include "arrow/python/builtin_convert.h"
 
@@ -356,7 +358,11 @@ class SeqConverter {
     return Status::OK();
   }
 
-  virtual Status AppendData(PyObject* seq, int64_t size) = 0;
+  // Append a single (non-sequence) Python datum to the underlying builder
+  virtual Status AppendSingle(PyObject* obj) = 0;
+
+  // Append the contents of a Python sequence to the underlying builder
+  virtual Status AppendMultiple(PyObject* seq, int64_t size) = 0;
 
   virtual ~SeqConverter() = default;
 
@@ -377,47 +383,57 @@ class TypedConverter : public SeqConverter {
   BuilderType* typed_builder_;
 };
 
+// We use the CRTP trick here to devirtualize the AppendItem() and AppendNull()
+// method calls.
 template <typename BuilderType, class Derived>
 class TypedConverterVisitor : public TypedConverter<BuilderType> {
  public:
-  Status AppendData(PyObject* obj, int64_t size) override {
+  Status AppendSingle(PyObject* obj) override {
+    if (obj == Py_None) {
+      return static_cast<Derived*>(this)->AppendNull();
+    } else {
+      return static_cast<Derived*>(this)->AppendItem(obj);
+    }
+  }
+
+  Status AppendMultiple(PyObject* obj, int64_t size) override {
     /// Ensure we've allocated enough space
     RETURN_NOT_OK(this->typed_builder_->Reserve(size));
     // Iterate over the items adding each one
     if (PySequence_Check(obj)) {
       for (int64_t i = 0; i < size; ++i) {
         OwnedRef ref(PySequence_GetItem(obj, i));
-        if (ref.obj() == Py_None) {
-          RETURN_NOT_OK(this->typed_builder_->AppendNull());
-        } else {
-          RETURN_NOT_OK(static_cast<Derived*>(this)->AppendItem(ref));
-        }
+        RETURN_NOT_OK(static_cast<Derived*>(this)->AppendSingle(ref.obj()));
       }
     } else {
       return Status::TypeError("Object is not a sequence");
     }
     return Status::OK();
   }
+
+  // Append a missing item (default implementation)
+  Status AppendNull() { return this->typed_builder_->AppendNull(); }
 };
 
 class NullConverter : public TypedConverterVisitor<NullBuilder, NullConverter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     return Status::Invalid("NullConverter: passed non-None value");
   }
 };
 
 class BoolConverter : public TypedConverterVisitor<BooleanBuilder, BoolConverter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    return typed_builder_->Append(item.obj() == Py_True);
-  }
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) { return typed_builder_->Append(obj == Py_True); }
 };
 
 class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(obj));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int8_t>::max() ||
                             val < std::numeric_limits<int8_t>::min())) {
@@ -432,8 +448,9 @@ class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
 
 class Int16Converter : public TypedConverterVisitor<Int16Builder, Int16Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(obj));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int16_t>::max() ||
                             val < std::numeric_limits<int16_t>::min())) {
@@ -448,8 +465,9 @@ class Int16Converter : public TypedConverterVisitor<Int16Builder, Int16Converter
 
 class Int32Converter : public TypedConverterVisitor<Int32Builder, Int32Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(obj));
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<int32_t>::max() ||
                             val < std::numeric_limits<int32_t>::min())) {
@@ -464,8 +482,9 @@ class Int32Converter : public TypedConverterVisitor<Int32Builder, Int32Converter
 
 class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    const auto val = static_cast<int64_t>(PyLong_AsLongLong(obj));
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
@@ -473,8 +492,9 @@ class Int64Converter : public TypedConverterVisitor<Int64Builder, Int64Converter
 
 class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(obj));
     RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint8_t>::max())) {
@@ -488,8 +508,9 @@ class UInt8Converter : public TypedConverterVisitor<UInt8Builder, UInt8Converter
 
 class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(obj));
     RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint16_t>::max())) {
@@ -503,8 +524,9 @@ class UInt16Converter : public TypedConverterVisitor<UInt16Builder, UInt16Conver
 
 class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    const auto val = static_cast<uint64_t>(PyLong_AsLongLong(obj));
     RETURN_IF_PYERROR();
 
     if (ARROW_PREDICT_FALSE(val > std::numeric_limits<uint32_t>::max())) {
@@ -518,8 +540,9 @@ class UInt32Converter : public TypedConverterVisitor<UInt32Builder, UInt32Conver
 
 class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    const auto val = static_cast<int64_t>(PyLong_AsUnsignedLongLong(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    const auto val = static_cast<int64_t>(PyLong_AsUnsignedLongLong(obj));
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
@@ -527,13 +550,14 @@ class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Conver
 
 class Date32Converter : public TypedConverterVisitor<Date32Builder, Date32Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     int32_t t;
-    if (PyDate_Check(item.obj())) {
-      auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
+    if (PyDate_Check(obj)) {
+      auto pydate = reinterpret_cast<PyDateTime_Date*>(obj);
       t = static_cast<int32_t>(PyDate_to_s(pydate));
     } else {
-      const auto casted_val = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      const auto casted_val = static_cast<int64_t>(PyLong_AsLongLong(obj));
       RETURN_IF_PYERROR();
       if (casted_val > std::numeric_limits<int32_t>::max()) {
         return Status::Invalid("Integer as date32 larger than INT32_MAX");
@@ -546,13 +570,14 @@ class Date32Converter : public TypedConverterVisitor<Date32Builder, Date32Conver
 
 class Date64Converter : public TypedConverterVisitor<Date64Builder, Date64Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     int64_t t;
-    if (PyDate_Check(item.obj())) {
-      auto pydate = reinterpret_cast<PyDateTime_Date*>(item.obj());
+    if (PyDate_Check(obj)) {
+      auto pydate = reinterpret_cast<PyDateTime_Date*>(obj);
       t = PyDate_to_ms(pydate);
     } else {
-      t = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      t = static_cast<int64_t>(PyLong_AsLongLong(obj));
       RETURN_IF_PYERROR();
     }
     return typed_builder_->Append(t);
@@ -564,10 +589,11 @@ class TimestampConverter
  public:
   explicit TimestampConverter(TimeUnit::type unit) : unit_(unit) {}
 
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     int64_t t;
-    if (PyDateTime_Check(item.obj())) {
-      auto pydatetime = reinterpret_cast<PyDateTime_DateTime*>(item.obj());
+    if (PyDateTime_Check(obj)) {
+      auto pydatetime = reinterpret_cast<PyDateTime_DateTime*>(obj);
 
       switch (unit_) {
         case TimeUnit::SECOND:
@@ -585,10 +611,10 @@ class TimestampConverter
         default:
           return Status::UnknownError("Invalid time unit");
       }
-    } else if (PyArray_CheckAnyScalarExact(item.obj())) {
+    } else if (PyArray_CheckAnyScalarExact(obj)) {
       // numpy.datetime64
       std::shared_ptr<DataType> type;
-      RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(item.obj()), &type));
+      RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &type));
       if (type->id() != Type::TIMESTAMP) {
         std::ostringstream ss;
         ss << "Expected np.datetime64 but got: ";
@@ -601,10 +627,9 @@ class TimestampConverter
             "Cannot convert NumPy datetime64 objects with differing unit");
       }
 
-      PyDatetimeScalarObject* obj = reinterpret_cast<PyDatetimeScalarObject*>(item.obj());
-      t = obj->obval;
+      t = reinterpret_cast<PyDatetimeScalarObject*>(obj)->obval;
     } else {
-      t = static_cast<int64_t>(PyLong_AsLongLong(item.obj()));
+      t = static_cast<int64_t>(PyLong_AsLongLong(obj));
       RETURN_IF_PYERROR();
     }
     return typed_builder_->Append(t);
@@ -616,8 +641,9 @@ class TimestampConverter
 
 class Float32Converter : public TypedConverterVisitor<FloatBuilder, Float32Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    float val = static_cast<float>(PyFloat_AsDouble(item.obj()));
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    float val = static_cast<float>(PyFloat_AsDouble(obj));
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
@@ -625,8 +651,9 @@ class Float32Converter : public TypedConverterVisitor<FloatBuilder, Float32Conve
 
 class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConverter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
-    double val = PyFloat_AsDouble(item.obj());
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    double val = PyFloat_AsDouble(obj);
     RETURN_IF_PYERROR();
     return typed_builder_->Append(val);
   }
@@ -634,22 +661,23 @@ class DoubleConverter : public TypedConverterVisitor<DoubleBuilder, DoubleConver
 
 class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     PyObject* bytes_obj;
     const char* bytes;
     Py_ssize_t length;
     OwnedRef tmp;
 
-    if (PyUnicode_Check(item.obj())) {
-      tmp.reset(PyUnicode_AsUTF8String(item.obj()));
+    if (PyUnicode_Check(obj)) {
+      tmp.reset(PyUnicode_AsUTF8String(obj));
       RETURN_IF_PYERROR();
       bytes_obj = tmp.obj();
-    } else if (PyBytes_Check(item.obj())) {
-      bytes_obj = item.obj();
+    } else if (PyBytes_Check(obj)) {
+      bytes_obj = obj;
     } else {
       std::stringstream ss;
       ss << "Error converting to Binary type: ";
-      RETURN_NOT_OK(InvalidConversion(item.obj(), "bytes", &ss));
+      RETURN_NOT_OK(InvalidConversion(obj, "bytes", &ss));
       return Status::Invalid(ss.str());
     }
     // No error checking
@@ -662,22 +690,23 @@ class BytesConverter : public TypedConverterVisitor<BinaryBuilder, BytesConverte
 class FixedWidthBytesConverter
     : public TypedConverterVisitor<FixedSizeBinaryBuilder, FixedWidthBytesConverter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     PyObject* bytes_obj;
     OwnedRef tmp;
     Py_ssize_t expected_length =
         std::dynamic_pointer_cast<FixedSizeBinaryType>(typed_builder_->type())
             ->byte_width();
-    if (PyUnicode_Check(item.obj())) {
-      tmp.reset(PyUnicode_AsUTF8String(item.obj()));
+    if (PyUnicode_Check(obj)) {
+      tmp.reset(PyUnicode_AsUTF8String(obj));
       RETURN_IF_PYERROR();
       bytes_obj = tmp.obj();
-    } else if (PyBytes_Check(item.obj())) {
-      bytes_obj = item.obj();
+    } else if (PyBytes_Check(obj)) {
+      bytes_obj = obj;
     } else {
       std::stringstream ss;
       ss << "Error converting to FixedSizeBinary type: ";
-      RETURN_NOT_OK(InvalidConversion(item.obj(), "bytes", &ss));
+      RETURN_NOT_OK(InvalidConversion(obj, "bytes", &ss));
       return Status::Invalid(ss.str());
     }
     // No error checking
@@ -689,13 +718,13 @@ class FixedWidthBytesConverter
 
 class UTF8Converter : public TypedConverterVisitor<StringBuilder, UTF8Converter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     PyObject* bytes_obj;
     OwnedRef tmp;
     const char* bytes;
     Py_ssize_t length;
 
-    PyObject* obj = item.obj();
     if (PyBytes_Check(obj)) {
       tmp.reset(
           PyUnicode_FromStringAndSize(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)));
@@ -724,75 +753,114 @@ class ListConverter : public TypedConverterVisitor<ListBuilder, ListConverter> {
  public:
   Status Init(ArrayBuilder* builder) override;
 
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     RETURN_NOT_OK(typed_builder_->Append());
-    PyObject* item_obj = item.obj();
-    const auto list_size = static_cast<int64_t>(PySequence_Size(item_obj));
-    return value_converter_->AppendData(item_obj, list_size);
+    const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
+    return value_converter_->AppendMultiple(obj, list_size);
   }
 
  protected:
-  std::shared_ptr<SeqConverter> value_converter_;
+  std::unique_ptr<SeqConverter> value_converter_;
+};
+
+class StructConverter : public TypedConverterVisitor<StructBuilder, StructConverter> {
+ public:
+  Status Init(ArrayBuilder* builder) override;
+
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
+    RETURN_NOT_OK(typed_builder_->Append());
+    if (!PyDict_Check(obj)) {
+      return Status::TypeError("dict value expected for struct type");
+    }
+    // NOTE we're ignoring any extraneous dict items
+    for (int i = 0; i < num_fields_; i++) {
+      PyObject* nameobj = PyList_GET_ITEM(field_name_list_.obj(), i);
+      PyObject* valueobj = PyDict_GetItem(obj, nameobj);  // borrowed
+      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(value_converters_[i]->AppendSingle(valueobj ? valueobj : Py_None));
+    }
+
+    return Status::OK();
+  }
+
+  // Append a missing item
+  Status AppendNull() {
+    RETURN_NOT_OK(typed_builder_->AppendNull());
+    // Need to also insert a missing item on all child builders
+    // (compare with ListConverter)
+    for (int i = 0; i < num_fields_; i++) {
+      RETURN_NOT_OK(value_converters_[i]->AppendSingle(Py_None));
+    }
+    return Status::OK();
+  }
+
+ protected:
+  std::vector<std::unique_ptr<SeqConverter>> value_converters_;
+  OwnedRef field_name_list_;
+  int num_fields_;
 };
 
 class DecimalConverter
     : public TypedConverterVisitor<arrow::Decimal128Builder, DecimalConverter> {
  public:
-  Status AppendItem(const OwnedRef& item) {
+  // Append a non-missing item
+  Status AppendItem(PyObject* obj) {
     /// TODO(phillipc): Check for nan?
     Decimal128 value;
     const auto& type = static_cast<const DecimalType&>(*typed_builder_->type());
-    RETURN_NOT_OK(internal::DecimalFromPythonDecimal(item.obj(), type, &value));
+    RETURN_NOT_OK(internal::DecimalFromPythonDecimal(obj, type, &value));
     return typed_builder_->Append(value);
   }
 };
 
 // Dynamic constructor for sequence converters
-std::shared_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type) {
+std::unique_ptr<SeqConverter> GetConverter(const std::shared_ptr<DataType>& type) {
   switch (type->id()) {
     case Type::NA:
-      return std::make_shared<NullConverter>();
+      return std::unique_ptr<SeqConverter>(new NullConverter);
     case Type::BOOL:
-      return std::make_shared<BoolConverter>();
+      return std::unique_ptr<SeqConverter>(new BoolConverter);
     case Type::INT8:
-      return std::make_shared<Int8Converter>();
+      return std::unique_ptr<SeqConverter>(new Int8Converter);
     case Type::INT16:
-      return std::make_shared<Int16Converter>();
+      return std::unique_ptr<SeqConverter>(new Int16Converter);
     case Type::INT32:
-      return std::make_shared<Int32Converter>();
+      return std::unique_ptr<SeqConverter>(new Int32Converter);
     case Type::INT64:
-      return std::make_shared<Int64Converter>();
+      return std::unique_ptr<SeqConverter>(new Int64Converter);
     case Type::UINT8:
-      return std::make_shared<UInt8Converter>();
+      return std::unique_ptr<SeqConverter>(new UInt8Converter);
     case Type::UINT16:
-      return std::make_shared<UInt16Converter>();
+      return std::unique_ptr<SeqConverter>(new UInt16Converter);
     case Type::UINT32:
-      return std::make_shared<UInt32Converter>();
+      return std::unique_ptr<SeqConverter>(new UInt32Converter);
     case Type::UINT64:
-      return std::make_shared<UInt64Converter>();
+      return std::unique_ptr<SeqConverter>(new UInt64Converter);
     case Type::DATE32:
-      return std::make_shared<Date32Converter>();
+      return std::unique_ptr<SeqConverter>(new Date32Converter);
     case Type::DATE64:
-      return std::make_shared<Date64Converter>();
+      return std::unique_ptr<SeqConverter>(new Date64Converter);
     case Type::TIMESTAMP:
-      return std::make_shared<TimestampConverter>(
-          static_cast<const TimestampType&>(*type).unit());
+      return std::unique_ptr<SeqConverter>(
+          new TimestampConverter(static_cast<const TimestampType&>(*type).unit()));
     case Type::FLOAT:
-      return std::make_shared<Float32Converter>();
+      return std::unique_ptr<SeqConverter>(new Float32Converter);
     case Type::DOUBLE:
-      return std::make_shared<DoubleConverter>();
+      return std::unique_ptr<SeqConverter>(new DoubleConverter);
     case Type::BINARY:
-      return std::make_shared<BytesConverter>();
+      return std::unique_ptr<SeqConverter>(new BytesConverter);
     case Type::FIXED_SIZE_BINARY:
-      return std::make_shared<FixedWidthBytesConverter>();
+      return std::unique_ptr<SeqConverter>(new FixedWidthBytesConverter);
     case Type::STRING:
-      return std::make_shared<UTF8Converter>();
+      return std::unique_ptr<SeqConverter>(new UTF8Converter);
     case Type::LIST:
-      return std::make_shared<ListConverter>();
-    case Type::DECIMAL: {
-      return std::make_shared<DecimalConverter>();
-    }
+      return std::unique_ptr<SeqConverter>(new ListConverter);
     case Type::STRUCT:
+      return std::unique_ptr<SeqConverter>(new StructConverter);
+    case Type::DECIMAL:
+      return std::unique_ptr<SeqConverter>(new DecimalConverter);
     default:
       return nullptr;
   }
@@ -811,17 +879,50 @@ Status ListConverter::Init(ArrayBuilder* builder) {
   return value_converter_->Init(typed_builder_->value_builder());
 }
 
+Status StructConverter::Init(ArrayBuilder* builder) {
+  builder_ = builder;
+  typed_builder_ = static_cast<StructBuilder*>(builder);
+  StructType* struct_type = static_cast<StructType*>(builder->type().get());
+
+  num_fields_ = typed_builder_->num_fields();
+  DCHECK_EQ(num_fields_, struct_type->num_children());
+
+  field_name_list_.reset(PyList_New(num_fields_));
+  RETURN_IF_PYERROR();
+
+  // Initialize the child converters and field names
+  for (int i = 0; i < num_fields_; i++) {
+    const std::string& field_name(struct_type->child(i)->name());
+    std::shared_ptr<DataType> field_type(struct_type->child(i)->type());
+
+    auto value_converter = GetConverter(field_type);
+    if (value_converter == nullptr) {
+      return Status::NotImplemented("value type not implemented");
+    }
+    RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
+    value_converters_.push_back(std::move(value_converter));
+
+    // Store the field name as a PyObject, for dict matching
+    PyObject* nameobj =
+        PyUnicode_FromStringAndSize(field_name.c_str(), field_name.size());
+    RETURN_IF_PYERROR();
+    PyList_SET_ITEM(field_name_list_.obj(), i, nameobj);
+  }
+
+  return Status::OK();
+}
+
 Status AppendPySequence(PyObject* obj, int64_t size,
                         const std::shared_ptr<DataType>& type, ArrayBuilder* builder) {
   PyDateTime_IMPORT;
-  std::shared_ptr<SeqConverter> converter = GetConverter(type);
+  auto converter = GetConverter(type);
   if (converter == nullptr) {
     std::stringstream ss;
     ss << "No type converter implemented for " << type->ToString();
     return Status::NotImplemented(ss.str());
   }
   RETURN_NOT_OK(converter->Init(builder));
-  return converter->AppendData(obj, size);
+  return converter->AppendMultiple(obj, size);
 }
 
 static Status ConvertPySequenceReal(PyObject* obj, int64_t size,
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 2b317dfbc3fee..bbdf6e71e0f1d 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -504,3 +504,28 @@ def test_structarray():
 
     pylist = arr.to_pylist()
     assert pylist == expected, (pylist, expected)
+
+
+def test_struct_from_dicts():
+    ty = pa.struct([pa.field('a', pa.int32()),
+                    pa.field('b', pa.string()),
+                    pa.field('c', pa.bool_())])
+    arr = pa.array([], type=ty)
+    assert arr.to_pylist() == []
+
+    data = [{'a': 5, 'b': 'foo', 'c': True},
+            {'a': 6, 'b': 'bar', 'c': False}]
+    arr = pa.array(data, type=ty)
+    assert arr.to_pylist() == data
+
+    # With omitted values
+    data = [{'a': 5, 'c': True},
+            None,
+            {},
+            {'a': None, 'b': 'bar'}]
+    arr = pa.array(data, type=ty)
+    expected = [{'a': 5, 'b': None, 'c': True},
+                None,
+                {'a': None, 'b': None, 'c': None},
+                {'a': None, 'b': 'bar', 'c': None}]
+    assert arr.to_pylist() == expected

From 3e63084bee72c097932cc07ca9deae15a2e97fbe Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Tue, 30 Jan 2018 17:20:09 -0500
Subject: [PATCH 1418/1644] ARROW-2036: [Python] Support standard IOBase
 methods on NativeFile

Adds support for most common file methods, adding enough to use
`io.TextIOWrapper`.

Added attribtes/methods:

- `closed` attribute
- `readable`, `writable`, `seekable` methods
- `read1` alias for `read` to support `TextIOWrapper` on python 2
- No-op `flush` method

Also refactored the cython internals a bit, adding default settings for
`is_readable` and `is_writable`, which makes subclasses not need to set
them in all places.

Also renamed `is_writeable` to `is_writable` for common spelling with
the standard python method `writable`.

Author: Jim Crist <jiminy.crist@gmail.com>

Closes #1517 from jcrist/full-python-file-interface and squashes the following commits:

f3bc9546 [Jim Crist] Support standard IOBase methods on NativeFile
---
 python/pyarrow/io-hdfs.pxi      |   7 +-
 python/pyarrow/io.pxi           | 122 ++++++++++++++++++--------------
 python/pyarrow/ipc.pxi          |   2 +-
 python/pyarrow/ipc.py           |   4 +-
 python/pyarrow/lib.pxd          |   4 +-
 python/pyarrow/tests/test_io.py |  66 +++++++++++++++--
 6 files changed, 136 insertions(+), 69 deletions(-)

diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi
index 3abf045f93336..83b14b687830d 100644
--- a/python/pyarrow/io-hdfs.pxi
+++ b/python/pyarrow/io-hdfs.pxi
@@ -413,9 +413,7 @@ cdef class HadoopFileSystem:
                                    &wr_handle))
 
             out.wr_file = <shared_ptr[OutputStream]> wr_handle
-
-            out.is_readable = False
-            out.is_writeable = 1
+            out.is_writable = True
         else:
             with nogil:
                 check_status(self.client.get()
@@ -423,7 +421,6 @@ cdef class HadoopFileSystem:
 
             out.rd_file = <shared_ptr[RandomAccessFile]> rd_handle
             out.is_readable = True
-            out.is_writeable = 0
 
         if c_buffer_size == 0:
             c_buffer_size = 2 ** 16
@@ -431,7 +428,7 @@ cdef class HadoopFileSystem:
         out.mode = mode
         out.buffer_size = c_buffer_size
         out.parent = _HdfsFileNanny(self, out)
-        out.is_open = True
+        out.closed = False
         out.own_file = True
 
         return out
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index bb363bacc2e24..bd508cf57ee8d 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -39,13 +39,14 @@ cdef extern from "Python.h":
 
 
 cdef class NativeFile:
-
     def __cinit__(self):
-        self.is_open = False
+        self.closed = True
         self.own_file = False
+        self.is_readable = False
+        self.is_writable = False
 
     def __dealloc__(self):
-        if self.is_open and self.own_file:
+        if self.own_file and not self.closed:
             self.close()
 
     def __enter__(self):
@@ -65,34 +66,52 @@ cdef class NativeFile:
 
         def __get__(self):
             # Emulate built-in file modes
-            if self.is_readable and self.is_writeable:
+            if self.is_readable and self.is_writable:
                 return 'rb+'
             elif self.is_readable:
                 return 'rb'
-            elif self.is_writeable:
+            elif self.is_writable:
                 return 'wb'
             else:
                 raise ValueError('File object is malformed, has no mode')
 
+    def readable(self):
+        self._assert_open()
+        return self.is_readable
+
+    def writable(self):
+        self._assert_open()
+        return self.is_writable
+
+    def seekable(self):
+        self._assert_open()
+        return self.is_readable
+
     def close(self):
-        if self.is_open:
+        if not self.closed:
             with nogil:
                 if self.is_readable:
                     check_status(self.rd_file.get().Close())
                 else:
                     check_status(self.wr_file.get().Close())
-        self.is_open = False
+        self.closed = True
+
+    def flush(self):
+        """Flush the buffer stream, if applicable.
+
+        No-op to match the IOBase interface."""
+        self._assert_open()
 
     cdef read_handle(self, shared_ptr[RandomAccessFile]* file):
         self._assert_readable()
         file[0] = <shared_ptr[RandomAccessFile]> self.rd_file
 
     cdef write_handle(self, shared_ptr[OutputStream]* file):
-        self._assert_writeable()
+        self._assert_writable()
         file[0] = <shared_ptr[OutputStream]> self.wr_file
 
     def _assert_open(self):
-        if not self.is_open:
+        if self.closed:
             raise ValueError("I/O operation on closed file")
 
     def _assert_readable(self):
@@ -100,10 +119,10 @@ cdef class NativeFile:
         if not self.is_readable:
             raise IOError("only valid on readonly files")
 
-    def _assert_writeable(self):
+    def _assert_writable(self):
         self._assert_open()
-        if not self.is_writeable:
-            raise IOError("only valid on writeable files")
+        if not self.is_writable:
+            raise IOError("only valid on writable files")
 
     def size(self):
         """
@@ -175,7 +194,7 @@ cdef class NativeFile:
         Write byte from any object implementing buffer protocol (bytes,
         bytearray, ndarray, pyarrow.Buffer)
         """
-        self._assert_writeable()
+        self._assert_writable()
 
         if isinstance(data, six.string_types):
             data = tobytes(data)
@@ -224,6 +243,12 @@ cdef class NativeFile:
 
         return PyObject_to_object(obj)
 
+    def read1(self, nbytes=None):
+        """Read and return up to n bytes.
+
+        Alias for read, needed to match the IOBase interface."""
+        return self.read(nbytes=None)
+
     def read_buffer(self, nbytes=None):
         cdef:
             int64_t c_nbytes
@@ -333,7 +358,7 @@ cdef class NativeFile:
         Pipe file-like object to file
         """
         write_queue = Queue(50)
-        self._assert_writeable()
+        self._assert_writable()
 
         buffer_size = buffer_size or DEFAULT_BUFFER_SIZE
 
@@ -390,16 +415,14 @@ cdef class PythonFile(NativeFile):
 
         if mode.startswith('w'):
             self.wr_file.reset(new PyOutputStream(handle))
-            self.is_readable = 0
-            self.is_writeable = 1
+            self.is_writable = True
         elif mode.startswith('r'):
             self.rd_file.reset(new PyReadableFile(handle))
-            self.is_readable = 1
-            self.is_writeable = 0
+            self.is_readable = True
         else:
             raise ValueError('Invalid file mode: {0}'.format(mode))
 
-        self.is_open = True
+        self.closed = False
 
 
 cdef class MemoryMappedFile(NativeFile):
@@ -409,11 +432,6 @@ cdef class MemoryMappedFile(NativeFile):
     cdef:
         object path
 
-    def __cinit__(self):
-        self.is_open = False
-        self.is_readable = 0
-        self.is_writeable = 0
-
     @staticmethod
     def create(path, size):
         cdef:
@@ -426,11 +444,11 @@ cdef class MemoryMappedFile(NativeFile):
 
         cdef MemoryMappedFile result = MemoryMappedFile()
         result.path = path
-        result.is_readable = 1
-        result.is_writeable = 1
+        result.is_readable = True
+        result.is_writable = True
         result.wr_file = <shared_ptr[OutputStream]> handle
         result.rd_file = <shared_ptr[RandomAccessFile]> handle
-        result.is_open = True
+        result.closed = False
 
         return result
 
@@ -444,14 +462,14 @@ cdef class MemoryMappedFile(NativeFile):
 
         if mode in ('r', 'rb'):
             c_mode = FileMode_READ
-            self.is_readable = 1
+            self.is_readable = True
         elif mode in ('w', 'wb'):
             c_mode = FileMode_WRITE
-            self.is_writeable = 1
+            self.is_writable = True
         elif mode in ('r+', 'r+b', 'rb+'):
             c_mode = FileMode_READWRITE
-            self.is_readable = 1
-            self.is_writeable = 1
+            self.is_readable = True
+            self.is_writable = True
         else:
             raise ValueError('Invalid file mode: {0}'.format(mode))
 
@@ -460,7 +478,7 @@ cdef class MemoryMappedFile(NativeFile):
 
         self.wr_file = <shared_ptr[OutputStream]> handle
         self.rd_file = <shared_ptr[RandomAccessFile]> handle
-        self.is_open = True
+        self.closed = False
 
 
 def memory_map(path, mode='r'):
@@ -484,7 +502,7 @@ def memory_map(path, mode='r'):
 def create_memory_map(path, size):
     """
     Create memory map at indicated path of the given size, return open
-    writeable file object
+    writable file object
 
     Parameters
     ----------
@@ -513,16 +531,14 @@ cdef class OSFile(NativeFile):
             shared_ptr[Readable] handle
             c_string c_path = encode_file_path(path)
 
-        self.is_readable = self.is_writeable = 0
-
         if mode in ('r', 'rb'):
             self._open_readable(c_path, maybe_unbox_memory_pool(memory_pool))
         elif mode in ('w', 'wb'):
-            self._open_writeable(c_path)
+            self._open_writable(c_path)
         else:
             raise ValueError('Invalid file mode: {0}'.format(mode))
 
-        self.is_open = True
+        self.closed = False
 
     cdef _open_readable(self, c_string path, CMemoryPool* pool):
         cdef shared_ptr[ReadableFile] handle
@@ -530,15 +546,15 @@ cdef class OSFile(NativeFile):
         with nogil:
             check_status(ReadableFile.Open(path, pool, &handle))
 
-        self.is_readable = 1
+        self.is_readable = True
         self.rd_file = <shared_ptr[RandomAccessFile]> handle
 
-    cdef _open_writeable(self, c_string path):
+    cdef _open_writable(self, c_string path):
         cdef shared_ptr[FileOutputStream] handle
 
         with nogil:
             check_status(FileOutputStream.Open(path, &handle))
-        self.is_writeable = 1
+        self.is_writable = True
         self.wr_file = <shared_ptr[OutputStream]> handle
 
 
@@ -546,9 +562,8 @@ cdef class FixedSizeBufferWriter(NativeFile):
 
     def __cinit__(self, Buffer buffer):
         self.wr_file.reset(new CFixedSizeBufferWriter(buffer.buffer))
-        self.is_readable = 0
-        self.is_writeable = 1
-        self.is_open = True
+        self.is_writable = True
+        self.closed = False
 
     def set_memcopy_threads(self, int num_threads):
         cdef CFixedSizeBufferWriter* writer = \
@@ -738,14 +753,13 @@ cdef class BufferOutputStream(NativeFile):
         self.buffer = _allocate_buffer(maybe_unbox_memory_pool(memory_pool))
         self.wr_file.reset(new CBufferOutputStream(
             <shared_ptr[CResizableBuffer]> self.buffer))
-        self.is_readable = 0
-        self.is_writeable = 1
-        self.is_open = True
+        self.is_writable = True
+        self.closed = False
 
     def get_result(self):
         with nogil:
             check_status(self.wr_file.get().Close())
-        self.is_open = False
+        self.closed = True
         return pyarrow_wrap_buffer(<shared_ptr[CBuffer]> self.buffer)
 
 
@@ -753,9 +767,8 @@ cdef class MockOutputStream(NativeFile):
 
     def __cinit__(self):
         self.wr_file.reset(new CMockOutputStream())
-        self.is_readable = 0
-        self.is_writeable = 1
-        self.is_open = True
+        self.is_writable = True
+        self.closed = False
 
     def size(self):
         return (<CMockOutputStream*>self.wr_file.get()).GetExtentBytesWritten()
@@ -780,9 +793,8 @@ cdef class BufferReader(NativeFile):
             self.buffer = frombuffer(obj)
 
         self.rd_file.reset(new CBufferReader(self.buffer.buffer))
-        self.is_readable = 1
-        self.is_writeable = 0
-        self.is_open = True
+        self.is_readable = True
+        self.closed = False
 
 
 def frombuffer(object obj):
@@ -834,8 +846,8 @@ cdef get_writer(object source, shared_ptr[OutputStream]* writer):
     if isinstance(source, NativeFile):
         nf = source
 
-        if not nf.is_writeable:
-            raise IOError('Native file is not writeable')
+        if not nf.is_writable:
+            raise IOError('Native file is not writable')
 
         nf.write_handle(writer)
     else:
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index 7534b0d0e87ec..a30a228ae878f 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -429,7 +429,7 @@ def write_tensor(Tensor tensor, NativeFile dest):
         int32_t metadata_length
         int64_t body_length
 
-    dest._assert_writeable()
+    dest._assert_writable()
 
     with nogil:
         check_status(
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index f264f089c4071..4081fc50e6df6 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -65,7 +65,7 @@ class RecordBatchStreamWriter(lib._RecordBatchWriter):
     Parameters
     ----------
     sink : str, pyarrow.NativeFile, or file-like Python object
-        Either a file path, or a writeable file object
+        Either a file path, or a writable file object
     schema : pyarrow.Schema
         The Arrow schema for data to be written to the file
     """
@@ -96,7 +96,7 @@ class RecordBatchFileWriter(lib._RecordBatchFileWriter):
     Parameters
     ----------
     sink : str, pyarrow.NativeFile, or file-like Python object
-        Either a file path, or a writeable file object
+        Either a file path, or a writable file object
     schema : pyarrow.Schema
         The Arrow schema for data to be written to the file
     """
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 90f749d6db633..161562c040c30 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -333,8 +333,8 @@ cdef class NativeFile:
         shared_ptr[RandomAccessFile] rd_file
         shared_ptr[OutputStream] wr_file
         bint is_readable
-        bint is_writeable
-        bint is_open
+        bint is_writable
+        readonly bint closed
         bint own_file
 
     # By implementing these "virtual" functions (all functions in Cython
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 3f7aa2e1c83bd..da26b101db260 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from io import BytesIO
+from io import BytesIO, TextIOWrapper
 import gc
 import os
 import pytest
@@ -482,27 +482,48 @@ def test_native_file_modes(tmpdir):
 
     with pa.OSFile(path, mode='r') as f:
         assert f.mode == 'rb'
+        assert f.readable()
+        assert not f.writable()
+        assert f.seekable()
 
     with pa.OSFile(path, mode='rb') as f:
         assert f.mode == 'rb'
+        assert f.readable()
+        assert not f.writable()
+        assert f.seekable()
 
     with pa.OSFile(path, mode='w') as f:
         assert f.mode == 'wb'
+        assert not f.readable()
+        assert f.writable()
+        assert not f.seekable()
 
     with pa.OSFile(path, mode='wb') as f:
         assert f.mode == 'wb'
+        assert not f.readable()
+        assert f.writable()
+        assert not f.seekable()
 
     with open(path, 'wb') as f:
         f.write(b'foooo')
 
     with pa.memory_map(path, 'r') as f:
         assert f.mode == 'rb'
+        assert f.readable()
+        assert not f.writable()
+        assert f.seekable()
 
     with pa.memory_map(path, 'r+') as f:
         assert f.mode == 'rb+'
+        assert f.readable()
+        assert f.writable()
+        assert f.seekable()
 
     with pa.memory_map(path, 'r+b') as f:
         assert f.mode == 'rb+'
+        assert f.readable()
+        assert f.writable()
+        assert f.seekable()
 
 
 def test_native_file_raises_ValueError_after_close(tmpdir):
@@ -511,19 +532,56 @@ def test_native_file_raises_ValueError_after_close(tmpdir):
         f.write(b'foooo')
 
     with pa.OSFile(path, mode='rb') as os_file:
-        pass
+        assert not os_file.closed
+    assert os_file.closed
 
     with pa.memory_map(path, mode='rb') as mmap_file:
-        pass
+        assert not mmap_file.closed
+    assert mmap_file.closed
 
     files = [os_file,
              mmap_file]
 
     methods = [('tell', ()),
                ('seek', (0,)),
-               ('size', ())]
+               ('size', ()),
+               ('flush', ()),
+               ('readable', ()),
+               ('writable', ()),
+               ('seekable', ())]
 
     for f in files:
         for method, args in methods:
             with pytest.raises(ValueError):
                 getattr(f, method)(*args)
+
+
+def test_native_file_TextIOWrapper(tmpdir):
+    data = (u'foooo\n'
+            u'barrr\n'
+            u'bazzz\n')
+
+    path = os.path.join(str(tmpdir), guid())
+    with open(path, 'wb') as f:
+        f.write(data.encode('utf-8'))
+
+    with TextIOWrapper(pa.OSFile(path, mode='rb')) as fil:
+        assert fil.readable()
+        res = fil.read()
+        assert res == data
+    assert fil.closed
+
+    with TextIOWrapper(pa.OSFile(path, mode='rb')) as fil:
+        # Iteration works
+        lines = list(fil)
+        assert ''.join(lines) == data
+
+    # Writing
+    path2 = os.path.join(str(tmpdir), guid())
+    with TextIOWrapper(pa.OSFile(path2, mode='wb')) as fil:
+        assert fil.writable()
+        fil.write(data)
+
+    with TextIOWrapper(pa.OSFile(path2, mode='rb')) as fil:
+        res = fil.read()
+        assert res == data

From cd0676f86097a7461d550b08ff17bdb0d45c4929 Mon Sep 17 00:00:00 2001
From: yosuke shiro <ys2014hs@gmail.com>
Date: Wed, 31 Jan 2018 11:29:48 +0900
Subject: [PATCH 1419/1644] ARROW-2064: [GLib] Add common build problems link
 to the install section

I think I should add "Add common build problems" link to the install section. I could not find the the link when installing.

Author: yosuke shiro <ys2014hs@gmail.com>

Closes #1537 from shiro615/add-common-build-problems-link-to-the-install-section and squashes the following commits:

12dfc9bc [yosuke shiro] [GLib] Add common build problems link to the install section
---
 c_glib/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/c_glib/README.md b/c_glib/README.md
index 1f67d7ea52ccb..d801fc83b5324 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -53,6 +53,8 @@ recommended that you use packages.
 Note that the packages are "unofficial". "Official" packages will be
 released in the future.
 
+If you find problems when installing please see [common build problems](https://github.com/apache/arrow/blob/master/c_glib/README.md#common-build-problems).
+
 ### Package
 
 See [install document](../site/install.md) for details.

From e112995fdfa4917ec5b683eead5b07a7921d1600 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 30 Jan 2018 22:35:06 -0500
Subject: [PATCH 1420/1644] ARROW-2062: [Python] Do not use memory maps in
 test_serialization.py to try to improve Travis CI flakiness

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1536 from wesm/ARROW-2062 and squashes the following commits:

22300cd8 [Wes McKinney] Do not use memory maps in serialization tests
ff5141fc [Wes McKinney] Add large_buffer fixture
---
 python/pyarrow/tests/test_serialization.py | 103 +++++++++++----------
 1 file changed, 52 insertions(+), 51 deletions(-)

diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 9cad81fc98f1e..284c7fb4ca78d 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -210,11 +210,12 @@ def make_serialization_context():
 serialization_context = make_serialization_context()
 
 
-def serialization_roundtrip(value, f, ctx=serialization_context):
-    f.seek(0)
-    pa.serialize_to(value, f, ctx)
-    f.seek(0)
-    result = pa.deserialize_from(f, None, ctx)
+def serialization_roundtrip(value, scratch_buffer, ctx=serialization_context):
+    writer = pa.FixedSizeBufferWriter(scratch_buffer)
+    pa.serialize_to(value, writer, ctx)
+
+    reader = pa.BufferReader(scratch_buffer)
+    result = pa.deserialize_from(reader, None, ctx)
     assert_equal(value, result)
 
     _check_component_roundtrip(value)
@@ -230,6 +231,10 @@ def _check_component_roundtrip(value):
 
 
 @pytest.yield_fixture(scope='session')
+def large_buffer(size=100*1024*1024):
+    return pa.allocate_buffer(size)
+
+
 def large_memory_map(tmpdir_factory, size=100*1024*1024):
     path = (tmpdir_factory.mktemp('data')
             .join('pyarrow-serialization-tmp-file').strpath)
@@ -243,11 +248,11 @@ def large_memory_map(tmpdir_factory, size=100*1024*1024):
     return path
 
 
-def test_primitive_serialization(large_memory_map):
-    with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        for obj in PRIMITIVE_OBJECTS:
-            serialization_roundtrip(obj, mmap)
-            serialization_roundtrip(obj, mmap, pa.pandas_serialization_context)
+def test_primitive_serialization(large_buffer):
+    for obj in PRIMITIVE_OBJECTS:
+        serialization_roundtrip(obj, large_buffer)
+        serialization_roundtrip(obj, large_buffer,
+                                pa.pandas_serialization_context)
 
 
 def test_serialize_to_buffer():
@@ -258,34 +263,31 @@ def test_serialize_to_buffer():
             assert_equal(value, result)
 
 
-def test_complex_serialization(large_memory_map):
-    with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        for obj in COMPLEX_OBJECTS:
-            serialization_roundtrip(obj, mmap)
+def test_complex_serialization(large_buffer):
+    for obj in COMPLEX_OBJECTS:
+        serialization_roundtrip(obj, large_buffer)
 
 
-def test_custom_serialization(large_memory_map):
-    with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        for obj in CUSTOM_OBJECTS:
-            serialization_roundtrip(obj, mmap)
+def test_custom_serialization(large_buffer):
+    for obj in CUSTOM_OBJECTS:
+        serialization_roundtrip(obj, large_buffer)
 
 
-def test_default_dict_serialization(large_memory_map):
+def test_default_dict_serialization(large_buffer):
     pytest.importorskip("cloudpickle")
-    with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        obj = defaultdict(lambda: 0, [("hello", 1), ("world", 2)])
-        serialization_roundtrip(obj, mmap)
+
+    obj = defaultdict(lambda: 0, [("hello", 1), ("world", 2)])
+    serialization_roundtrip(obj, large_buffer)
 
 
-def test_numpy_serialization(large_memory_map):
-    with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        for t in ["bool", "int8", "uint8", "int16", "uint16", "int32",
-                  "uint32", "float16", "float32", "float64"]:
-            obj = np.random.randint(0, 10, size=(100, 100)).astype(t)
-            serialization_roundtrip(obj, mmap)
+def test_numpy_serialization(large_buffer):
+    for t in ["bool", "int8", "uint8", "int16", "uint16", "int32",
+              "uint32", "float16", "float32", "float64"]:
+        obj = np.random.randint(0, 10, size=(100, 100)).astype(t)
+        serialization_roundtrip(obj, large_buffer)
 
 
-def test_datetime_serialization(large_memory_map):
+def test_datetime_serialization(large_buffer):
     data = [
         #  Principia Mathematica published
         datetime.datetime(year=1687, month=7, day=5),
@@ -309,32 +311,31 @@ def test_datetime_serialization(large_memory_map):
         datetime.datetime(year=1970, month=1, day=3, hour=4,
                           minute=0, second=0)
     ]
-    with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        for d in data:
-            serialization_roundtrip(d, mmap)
+    for d in data:
+        serialization_roundtrip(d, large_buffer)
 
 
-def test_torch_serialization(large_memory_map):
+def test_torch_serialization(large_buffer):
     pytest.importorskip("torch")
     import torch
-    with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        # These are the only types that are supported for the
-        # PyTorch to NumPy conversion
-        for t in ["float32", "float64",
-                  "uint8", "int16", "int32", "int64"]:
-            obj = torch.from_numpy(np.random.randn(1000).astype(t))
-            serialization_roundtrip(obj, mmap)
-
-
-def test_numpy_immutable(large_memory_map):
-    with pa.memory_map(large_memory_map, mode="r+") as mmap:
-        obj = np.zeros([10])
-        mmap.seek(0)
-        pa.serialize_to(obj, mmap, serialization_context)
-        mmap.seek(0)
-        result = pa.deserialize_from(mmap, None, serialization_context)
-        with pytest.raises(ValueError):
-            result[0] = 1.0
+    # These are the only types that are supported for the
+    # PyTorch to NumPy conversion
+    for t in ["float32", "float64",
+              "uint8", "int16", "int32", "int64"]:
+        obj = torch.from_numpy(np.random.randn(1000).astype(t))
+        serialization_roundtrip(obj, large_buffer)
+
+
+def test_numpy_immutable(large_buffer):
+    obj = np.zeros([10])
+
+    writer = pa.FixedSizeBufferWriter(large_buffer)
+    pa.serialize_to(obj, writer, serialization_context)
+
+    reader = pa.BufferReader(large_buffer)
+    result = pa.deserialize_from(reader, None, serialization_context)
+    with pytest.raises(ValueError):
+        result[0] = 1.0
 
 
 # see https://issues.apache.org/jira/browse/ARROW-1695

From 0d6817a05e1316c60dbdaa055c18fabf30240a5e Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Tue, 30 Jan 2018 22:36:10 -0500
Subject: [PATCH 1421/1644] ARROW-2042: [Plasma] Revert API change of
 plasma::Create to output a MutableBuffer

This reverts a part of the changes from https://github.com/apache/arrow/pull/1479. This is needed for https://github.com/apache/arrow/pull/1445 so we can return a CudaBuffer from plasma::Create.

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1520 from pcmoritz/revert-mutable-buffer and squashes the following commits:

0d36c734 [Philipp Moritz] fix plasma python bindings
259127d5 [Philipp Moritz] revert plasma::Create API back to Buffer
---
 cpp/src/plasma/client.cc            |  4 +++-
 cpp/src/plasma/client.h             |  3 +--
 cpp/src/plasma/test/client_tests.cc | 14 +++++++-------
 python/pyarrow/plasma.pyx           |  4 ++--
 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index a683da0022b18..6e9b6968a8673 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -54,6 +54,8 @@
 
 namespace plasma {
 
+using arrow::MutableBuffer;
+
 // Number of threads used for memcopy and hash computations.
 constexpr int64_t kThreadPoolSize = 8;
 constexpr int64_t kBytesInMB = 1 << 20;
@@ -147,7 +149,7 @@ void PlasmaClient::increment_object_count(const ObjectID& object_id, PlasmaObjec
 
 Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
                             uint8_t* metadata, int64_t metadata_size,
-                            std::shared_ptr<MutableBuffer>* data) {
+                            std::shared_ptr<Buffer>* data) {
   ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
                    << data_size << " and metadata size " << metadata_size;
   RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index a1e10a9c29969..d6372f44a7f28 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -32,7 +32,6 @@
 #include "plasma/common.h"
 
 using arrow::Buffer;
-using arrow::MutableBuffer;
 using arrow::Status;
 
 namespace plasma {
@@ -116,7 +115,7 @@ class ARROW_EXPORT PlasmaClient {
   ///        will be written here.
   /// \return The return status.
   Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
-                int64_t metadata_size, std::shared_ptr<MutableBuffer>* data);
+                int64_t metadata_size, std::shared_ptr<Buffer>* data);
   /// Get some objects from the Plasma Store. This function will block until the
   /// objects have all been created and sealed in the Plasma Store or the
   /// timeout
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index 63b56934f3599..f19c2bfbdb380 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -70,7 +70,7 @@ TEST_F(TestPlasmaStore, DeleteTest) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<MutableBuffer> data;
+  std::shared_ptr<Buffer> data;
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client_.Seal(object_id));
 
@@ -96,7 +96,7 @@ TEST_F(TestPlasmaStore, ContainsTest) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<MutableBuffer> data;
+  std::shared_ptr<Buffer> data;
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client_.Seal(object_id));
   // Avoid race condition of Plasma Manager waiting for notification.
@@ -119,7 +119,7 @@ TEST_F(TestPlasmaStore, GetTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<MutableBuffer> data_buffer;
+  std::shared_ptr<Buffer> data_buffer;
   uint8_t* data;
   ARROW_CHECK_OK(
       client_.Create(object_id, data_size, metadata, metadata_size, &data_buffer));
@@ -145,7 +145,7 @@ TEST_F(TestPlasmaStore, MultipleGetTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<MutableBuffer> data;
+  std::shared_ptr<Buffer> data;
   ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data));
   data->mutable_data()[0] = 1;
   ARROW_CHECK_OK(client_.Seal(object_id1));
@@ -172,7 +172,7 @@ TEST_F(TestPlasmaStore, AbortTest) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<MutableBuffer> data;
+  std::shared_ptr<Buffer> data;
   uint8_t* data_ptr;
   ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
   data_ptr = data->mutable_data();
@@ -220,7 +220,7 @@ TEST_F(TestPlasmaStore, MultipleClientTest) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  std::shared_ptr<MutableBuffer> data;
+  std::shared_ptr<Buffer> data;
   ARROW_CHECK_OK(client2_.Create(object_id, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client2_.Seal(object_id));
   // Test that the first client can get the object.
@@ -260,7 +260,7 @@ TEST_F(TestPlasmaStore, ManyObjectTest) {
     int64_t data_size = 100;
     uint8_t metadata[] = {5};
     int64_t metadata_size = sizeof(metadata);
-    std::shared_ptr<MutableBuffer> data;
+    std::shared_ptr<Buffer> data;
     ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
 
     if (i % 3 == 0) {
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index 801d094194b71..32f6d189da08c 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -81,7 +81,7 @@ cdef extern from "plasma/client.h" nogil:
 
         CStatus Create(const CUniqueID& object_id, int64_t data_size,
                        const uint8_t* metadata, int64_t metadata_size,
-                       const shared_ptr[CMutableBuffer]* data)
+                       const shared_ptr[CBuffer]* data)
 
         CStatus Get(const CUniqueID* object_ids, int64_t num_objects,
                     int64_t timeout_ms, CObjectBuffer* object_buffers)
@@ -297,7 +297,7 @@ cdef class PlasmaClient:
                 not be created because the plasma store is unable to evict
                 enough objects to create room for it.
         """
-        cdef shared_ptr[CMutableBuffer] data
+        cdef shared_ptr[CBuffer] data
         with nogil:
             check_status(self.client.get().Create(object_id.data, data_size,
                                                   <uint8_t*>(metadata.data()),

From 0e04f6d2bd6984b5afd33ae0dd1b9eae96a681a9 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 31 Jan 2018 16:34:14 -0500
Subject: [PATCH 1422/1644] ARROW-2070: [Python] Fix chdir logic in setup.py

In some conditions, setup.py may change the current directory and omit restoring the previous one, which can fail some operations.

Author: Antoine Pitrou <antoine@python.org>

Closes #1540 from pitrou/ARROW-2070-chdir-in-setuppy and squashes the following commits:

f043b681 [Antoine Pitrou] ARROW-2070: [Python] Fix chdir logic in setup.py
---
 python/setup.py | 308 +++++++++++++++++++++++++-----------------------
 1 file changed, 159 insertions(+), 149 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index 076d7e489b5f3..cfc771fe870ab 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import contextlib
 import glob
 import os
 import os.path as osp
@@ -47,6 +48,16 @@
 setup_dir = os.path.abspath(os.path.dirname(__file__))
 
 
+@contextlib.contextmanager
+def changed_dir(dirname):
+    oldcwd = os.getcwd()
+    os.chdir(dirname)
+    try:
+        yield
+    finally:
+        os.chdir(oldcwd)
+
+
 class clean(_clean):
 
     def run(self):
@@ -59,6 +70,7 @@ def run(self):
 
 
 class build_ext(_build_ext):
+    _found_names = ()
 
     def build_extensions(self):
         numpy_incl = pkg_resources.resource_filename('numpy', 'core/include')
@@ -129,162 +141,160 @@ def _run_cmake(self):
         # The staging directory for the module being built
         build_temp = pjoin(os.getcwd(), self.build_temp)
         build_lib = os.path.join(os.getcwd(), self.build_lib)
-
-        # Change to the build directory
         saved_cwd = os.getcwd()
+
         if not os.path.isdir(self.build_temp):
             self.mkpath(self.build_temp)
-        os.chdir(self.build_temp)
-
-        # Detect if we built elsewhere
-        if os.path.isfile('CMakeCache.txt'):
-            cachefile = open('CMakeCache.txt', 'r')
-            cachedir = re.search('CMAKE_CACHEFILE_DIR:INTERNAL=(.*)',
-                                 cachefile.read()).group(1)
-            cachefile.close()
-            if (cachedir != build_temp):
-                return
-
-        static_lib_option = ''
-
-        cmake_options = [
-            '-DPYTHON_EXECUTABLE=%s' % sys.executable,
-            static_lib_option,
-        ]
 
-        if self.with_parquet:
-            cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
-        if self.with_static_parquet:
-            cmake_options.append('-DPYARROW_PARQUET_USE_SHARED=off')
-        if not self.with_static_boost:
-            cmake_options.append('-DPYARROW_BOOST_USE_SHARED=on')
-
-        if self.with_plasma:
-            cmake_options.append('-DPYARROW_BUILD_PLASMA=on')
-
-        if self.with_orc:
-            cmake_options.append('-DPYARROW_BUILD_ORC=on')
-
-        if len(self.cmake_cxxflags) > 0:
-            cmake_options.append('-DPYARROW_CXXFLAGS="{0}"'
-                                 .format(self.cmake_cxxflags))
-
-        if self.bundle_arrow_cpp:
-            cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
-            # ARROW-1090: work around CMake rough edges
-            if 'ARROW_HOME' in os.environ and sys.platform != 'win32':
-                pkg_config = pjoin(os.environ['ARROW_HOME'], 'lib',
-                                   'pkgconfig')
-                os.environ['PKG_CONFIG_PATH'] = pkg_config
-                del os.environ['ARROW_HOME']
-
-        cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
-                             .format(self.build_type.lower()))
-
-        extra_cmake_args = shlex.split(self.extra_cmake_args)
-        if sys.platform != 'win32':
-            cmake_command = (['cmake'] + extra_cmake_args +
-                             cmake_options + [source])
-
-            print("-- Runnning cmake for pyarrow")
-            self.spawn(cmake_command)
-            print("-- Finished cmake for pyarrow")
-            args = ['make']
-            if os.environ.get('PYARROW_BUILD_VERBOSE', '0') == '1':
-                args.append('VERBOSE=1')
-
-            if 'PYARROW_PARALLEL' in os.environ:
-                args.append('-j{0}'.format(os.environ['PYARROW_PARALLEL']))
-            print("-- Running cmake --build for pyarrow")
-            self.spawn(args)
-            print("-- Finished cmake --build for pyarrow")
-        else:
-            cmake_generator = 'Visual Studio 14 2015 Win64'
-            if not is_64_bit:
-                raise RuntimeError('Not supported on 32-bit Windows')
-
-            # Generate the build files
-            cmake_command = (['cmake'] + extra_cmake_args +
-                             cmake_options +
-                             [source, '-G', cmake_generator])
-            if "-G" in self.extra_cmake_args:
-                cmake_command = cmake_command[:-2]
-
-            print("-- Runnning cmake for pyarrow")
-            self.spawn(cmake_command)
-            print("-- Finished cmake for pyarrow")
-            # Do the build
-            print("-- Running cmake --build for pyarrow")
-            self.spawn(['cmake', '--build', '.', '--config', self.build_type])
-            print("-- Finished cmake --build for pyarrow")
-
-        if self.inplace:
-            # a bit hacky
-            build_lib = saved_cwd
-
-        # Move the libraries to the place expected by the Python
-        # build
-
-        try:
-            os.makedirs(pjoin(build_lib, 'pyarrow'))
-        except OSError:
-            pass
+        # Change to the build directory
+        with changed_dir(self.build_temp):
+            # Detect if we built elsewhere
+            if os.path.isfile('CMakeCache.txt'):
+                cachefile = open('CMakeCache.txt', 'r')
+                cachedir = re.search('CMAKE_CACHEFILE_DIR:INTERNAL=(.*)',
+                                     cachefile.read()).group(1)
+                cachefile.close()
+                if (cachedir != build_temp):
+                    return
+
+            static_lib_option = ''
+
+            cmake_options = [
+                '-DPYTHON_EXECUTABLE=%s' % sys.executable,
+                static_lib_option,
+            ]
+
+            if self.with_parquet:
+                cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
+            if self.with_static_parquet:
+                cmake_options.append('-DPYARROW_PARQUET_USE_SHARED=off')
+            if not self.with_static_boost:
+                cmake_options.append('-DPYARROW_BOOST_USE_SHARED=on')
 
-        if sys.platform == 'win32':
-            build_prefix = ''
-        else:
-            build_prefix = self.build_type
+            if self.with_plasma:
+                cmake_options.append('-DPYARROW_BUILD_PLASMA=on')
+
+            if self.with_orc:
+                cmake_options.append('-DPYARROW_BUILD_ORC=on')
+
+            if len(self.cmake_cxxflags) > 0:
+                cmake_options.append('-DPYARROW_CXXFLAGS="{0}"'
+                                     .format(self.cmake_cxxflags))
+
+            if self.bundle_arrow_cpp:
+                cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
+                # ARROW-1090: work around CMake rough edges
+                if 'ARROW_HOME' in os.environ and sys.platform != 'win32':
+                    pkg_config = pjoin(os.environ['ARROW_HOME'], 'lib',
+                                       'pkgconfig')
+                    os.environ['PKG_CONFIG_PATH'] = pkg_config
+                    del os.environ['ARROW_HOME']
+
+            cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
+                                 .format(self.build_type.lower()))
+
+            extra_cmake_args = shlex.split(self.extra_cmake_args)
+            if sys.platform != 'win32':
+                cmake_command = (['cmake'] + extra_cmake_args +
+                                 cmake_options + [source])
+
+                print("-- Runnning cmake for pyarrow")
+                self.spawn(cmake_command)
+                print("-- Finished cmake for pyarrow")
+                args = ['make']
+                if os.environ.get('PYARROW_BUILD_VERBOSE', '0') == '1':
+                    args.append('VERBOSE=1')
+
+                if 'PYARROW_PARALLEL' in os.environ:
+                    args.append('-j{0}'.format(os.environ['PYARROW_PARALLEL']))
+                print("-- Running cmake --build for pyarrow")
+                self.spawn(args)
+                print("-- Finished cmake --build for pyarrow")
+            else:
+                cmake_generator = 'Visual Studio 14 2015 Win64'
+                if not is_64_bit:
+                    raise RuntimeError('Not supported on 32-bit Windows')
+
+                # Generate the build files
+                cmake_command = (['cmake'] + extra_cmake_args +
+                                 cmake_options +
+                                 [source, '-G', cmake_generator])
+                if "-G" in self.extra_cmake_args:
+                    cmake_command = cmake_command[:-2]
+
+                print("-- Runnning cmake for pyarrow")
+                self.spawn(cmake_command)
+                print("-- Finished cmake for pyarrow")
+                # Do the build
+                print("-- Running cmake --build for pyarrow")
+                self.spawn(['cmake', '--build', '.', '--config', self.build_type])
+                print("-- Finished cmake --build for pyarrow")
+
+            if self.inplace:
+                # a bit hacky
+                build_lib = saved_cwd
+
+            # Move the libraries to the place expected by the Python
+            # build
+
+            try:
+                os.makedirs(pjoin(build_lib, 'pyarrow'))
+            except OSError:
+                pass
 
-        if self.bundle_arrow_cpp:
-            print(pjoin(build_lib, 'pyarrow'))
-            move_shared_libs(build_prefix, build_lib, "arrow")
-            move_shared_libs(build_prefix, build_lib, "arrow_python")
+            if sys.platform == 'win32':
+                build_prefix = ''
+            else:
+                build_prefix = self.build_type
+
+            if self.bundle_arrow_cpp:
+                print(pjoin(build_lib, 'pyarrow'))
+                move_shared_libs(build_prefix, build_lib, "arrow")
+                move_shared_libs(build_prefix, build_lib, "arrow_python")
+                if self.with_plasma:
+                    move_shared_libs(build_prefix, build_lib, "plasma")
+                if self.with_parquet and not self.with_static_parquet:
+                    move_shared_libs(build_prefix, build_lib, "parquet")
+
+            print('Bundling includes: ' + pjoin(build_prefix, 'include'))
+            if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
+                shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
+            shutil.move(pjoin(build_prefix, 'include'),
+                        pjoin(build_lib, 'pyarrow'))
+
+            # Move the built C-extension to the place expected by the Python build
+            self._found_names = []
+            for name in self.CYTHON_MODULE_NAMES:
+                built_path = self.get_ext_built(name)
+                if not os.path.exists(built_path):
+                    print(built_path)
+                    if self._failure_permitted(name):
+                        print('Cython module {0} failure permitted'.format(name))
+                        continue
+                    raise RuntimeError('pyarrow C-extension failed to build:',
+                                       os.path.abspath(built_path))
+
+                ext_path = pjoin(build_lib, self._get_cmake_ext_path(name))
+                if os.path.exists(ext_path):
+                    os.remove(ext_path)
+                self.mkpath(os.path.dirname(ext_path))
+                print('Moving built C-extension', built_path,
+                      'to build path', ext_path)
+                shutil.move(self.get_ext_built(name), ext_path)
+                self._found_names.append(name)
+
+                if os.path.exists(self.get_ext_built_api_header(name)):
+                    shutil.move(self.get_ext_built_api_header(name),
+                                pjoin(os.path.dirname(ext_path), name + '_api.h'))
+
+            # Move the plasma store
             if self.with_plasma:
-                move_shared_libs(build_prefix, build_lib, "plasma")
-            if self.with_parquet and not self.with_static_parquet:
-                move_shared_libs(build_prefix, build_lib, "parquet")
-
-        print('Bundling includes: ' + pjoin(build_prefix, 'include'))
-        if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
-            shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
-        shutil.move(pjoin(build_prefix, 'include'),
-                    pjoin(build_lib, 'pyarrow'))
-
-        # Move the built C-extension to the place expected by the Python build
-        self._found_names = []
-        for name in self.CYTHON_MODULE_NAMES:
-            built_path = self.get_ext_built(name)
-            if not os.path.exists(built_path):
-                print(built_path)
-                if self._failure_permitted(name):
-                    print('Cython module {0} failure permitted'.format(name))
-                    continue
-                raise RuntimeError('pyarrow C-extension failed to build:',
-                                   os.path.abspath(built_path))
-
-            ext_path = pjoin(build_lib, self._get_cmake_ext_path(name))
-            if os.path.exists(ext_path):
-                os.remove(ext_path)
-            self.mkpath(os.path.dirname(ext_path))
-            print('Moving built C-extension', built_path,
-                  'to build path', ext_path)
-            shutil.move(self.get_ext_built(name), ext_path)
-            self._found_names.append(name)
-
-            if os.path.exists(self.get_ext_built_api_header(name)):
-                shutil.move(self.get_ext_built_api_header(name),
-                            pjoin(os.path.dirname(ext_path), name + '_api.h'))
-
-        # Move the plasma store
-        if self.with_plasma:
-            build_py = self.get_finalized_command('build_py')
-            source = os.path.join(self.build_type, "plasma_store")
-            target = os.path.join(build_lib,
-                                  build_py.get_package_dir('pyarrow'),
-                                  "plasma_store")
-            shutil.move(source, target)
-
-        os.chdir(saved_cwd)
+                build_py = self.get_finalized_command('build_py')
+                source = os.path.join(self.build_type, "plasma_store")
+                target = os.path.join(build_lib,
+                                      build_py.get_package_dir('pyarrow'),
+                                      "plasma_store")
+                shutil.move(source, target)
 
     def _failure_permitted(self, name):
         if name == '_parquet' and not self.with_parquet:

From 1ed4019a2249be68de81854c79acf5f353c18d51 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 1 Feb 2018 08:19:42 -0500
Subject: [PATCH 1423/1644] ARROW-2072: [Python] Fix crash in
 decimal128.byte_width

Author: Antoine Pitrou <antoine@python.org>

Closes #1542 from pitrou/ARROW-2072-decimal-byte-width-crash and squashes the following commits:

bc1e15b1 [Antoine Pitrou] ARROW-2072: [Python] Fix crash in decimal128.byte_width
---
 python/pyarrow/tests/test_types.py | 10 ++++++++++
 python/pyarrow/types.pxi           |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index 68dc499cf48b4..ad683e9a2ea00 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -184,3 +184,13 @@ def test_types_hashable():
 ])
 def test_exact_primitive_types(t, check_func):
     assert check_func(t)
+
+
+def test_fixed_size_binary_byte_width():
+    ty = pa.binary(5)
+    assert ty.byte_width == 5
+
+
+def test_decimal_byte_width():
+    ty = pa.decimal128(19, 4)
+    assert ty.byte_width == 16
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index a3cbeefb028c7..849a0e016a60d 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -293,7 +293,7 @@ cdef class FixedSizeBinaryType(DataType):
 cdef class Decimal128Type(FixedSizeBinaryType):
 
     cdef void init(self, const shared_ptr[CDataType]& type):
-        DataType.init(self, type)
+        FixedSizeBinaryType.init(self, type)
         self.decimal128_type = <const CDecimal128Type*> type.get()
 
     def __getstate__(self):

From 2d649f9a75250b5a908369ed0246218712830fc4 Mon Sep 17 00:00:00 2001
From: Panchen Xue <pan.panchen.xue@gmail.com>
Date: Thu, 1 Feb 2018 12:14:48 -0500
Subject: [PATCH 1424/1644] ARROW-1623: [C++] Add convenience method to
 construct Buffer from a string that owns its memory

Add static member function Buffer::FromString to create a new buffer that owns its memory from given std::string. The memory is allocated from a given memory pool or the default one if not specified.

Author: Panchen Xue <pan.panchen.xue@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1518 from xuepanchen/ARROW-1623 and squashes the following commits:

e6f7355f [Wes McKinney] clang-format
ce18950a [Panchen Xue] Add Buffer::FromString method that takes default memory pool and modify test cast
f2c5e3ea [Panchen Xue] ARROW-1623: [C++] Add test case for Buffer::FromString method
2385637c [Panchen Xue] ARROW-1623: [C++] Add Buffer::FromString to construct a buffer that owns its memory from a std::string
---
 cpp/src/arrow/buffer-test.cc | 17 +++++++++++++++++
 cpp/src/arrow/buffer.cc      | 12 ++++++++++++
 cpp/src/arrow/buffer.h       | 14 ++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
index 398cc06363a6f..a24384a383395 100644
--- a/cpp/src/arrow/buffer-test.cc
+++ b/cpp/src/arrow/buffer-test.cc
@@ -52,6 +52,23 @@ TEST(TestBuffer, FromStdString) {
   ASSERT_EQ(static_cast<int64_t>(val.size()), buf.size());
 }
 
+TEST(TestBuffer, FromStdStringWithMemory) {
+  std::string expected = "hello, world";
+  std::shared_ptr<Buffer> buf;
+
+  {
+    std::string temp = "hello, world";
+    ASSERT_OK(Buffer::FromString(temp, &buf));
+    ASSERT_EQ(0, memcmp(buf->data(), temp.c_str(), temp.size()));
+    ASSERT_EQ(static_cast<int64_t>(temp.size()), buf->size());
+  }
+
+  // Now temp goes out of scope and we check if created buffer
+  // is still valid to make sure it actually owns its space
+  ASSERT_EQ(0, memcmp(buf->data(), expected.c_str(), expected.size()));
+  ASSERT_EQ(static_cast<int64_t>(expected.size()), buf->size());
+}
+
 TEST(TestBuffer, Resize) {
   PoolBuffer buf;
 
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 1b8e4375445bb..29e2c242a3f4a 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -58,6 +58,18 @@ bool Buffer::Equals(const Buffer& other) const {
                              !memcmp(data_, other.data_, static_cast<size_t>(size_))));
 }
 
+Status Buffer::FromString(const std::string& data, MemoryPool* pool,
+                          std::shared_ptr<Buffer>* out) {
+  auto size = static_cast<int64_t>(data.size());
+  RETURN_NOT_OK(AllocateBuffer(pool, size, out));
+  std::copy(data.c_str(), data.c_str() + size, (*out)->mutable_data());
+  return Status::OK();
+}
+
+Status Buffer::FromString(const std::string& data, std::shared_ptr<Buffer>* out) {
+  return FromString(data, default_memory_pool(), out);
+}
+
 PoolBuffer::PoolBuffer(MemoryPool* pool) : ResizableBuffer(nullptr, 0) {
   if (pool == nullptr) {
     pool = default_memory_pool();
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 44c352a93f273..d12eeb4df9eed 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -97,6 +97,20 @@ class ARROW_EXPORT Buffer {
   Status Copy(const int64_t start, const int64_t nbytes,
               std::shared_ptr<Buffer>* out) const;
 
+  /// \brief Construct a new buffer that owns its memory from a std::string
+  ///
+  /// \param[in] data a std::string object
+  /// \param[in] pool a memory pool
+  /// \param[out] out the created buffer
+  ///
+  /// \return Status message
+  static Status FromString(const std::string& data, MemoryPool* pool,
+                           std::shared_ptr<Buffer>* out);
+
+  /// \brief Construct a new buffer that owns its memory from a std::string
+  /// using the default memory pool
+  static Status FromString(const std::string& data, std::shared_ptr<Buffer>* out);
+
   int64_t capacity() const { return capacity_; }
   const uint8_t* data() const { return data_; }
   uint8_t* mutable_data() { return mutable_data_; }

From ff28c7647c1910f1a0d1d97b8ba68b2b554e5ce1 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Thu, 1 Feb 2018 12:17:46 -0500
Subject: [PATCH 1425/1644] ARROW-2024: [Python] Remove torch serialization
 from default serialization context.

See discussion in #1223.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1538 from robertnishihara/dontimportpytorch and squashes the following commits:

09a8cfa1 [Robert Nishihara] Fix bug.
8992e0fb [Robert Nishihara] Fix.
83ffb70f [Robert Nishihara] Remove torch serialization from default serialization context.
---
 python/pyarrow/__init__.py                  |  5 +-
 python/pyarrow/serialization.py             | 66 ++++++++++++---------
 python/pyarrow/tests/test_convert_pandas.py |  2 +-
 python/pyarrow/tests/test_serialization.py  | 56 +++++++++--------
 4 files changed, 73 insertions(+), 56 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index a245fe6796023..8b3cba92414f8 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -124,9 +124,10 @@
 
 localfs = LocalFileSystem.get_instance()
 
-from pyarrow.serialization import (_default_serialization_context,
+from pyarrow.serialization import (default_serialization_context,
                                    pandas_serialization_context,
-                                   register_default_serialization_handlers)
+                                   register_default_serialization_handlers,
+                                   register_torch_serialization_handlers)
 
 import pyarrow.types as types
 
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 61f2e83f3193d..c8b72b74896c9 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -22,7 +22,8 @@
 import numpy as np
 
 from pyarrow.compat import builtin_pickle
-from pyarrow.lib import _default_serialization_context, frombuffer
+from pyarrow.lib import (SerializationContext, _default_serialization_context,
+                         frombuffer)
 
 try:
     import cloudpickle
@@ -102,6 +103,31 @@ def _deserialize_pandas_series(data):
         custom_deserializer=_deserialize_pandas_dataframe)
 
 
+def register_torch_serialization_handlers(serialization_context):
+    # ----------------------------------------------------------------------
+    # Set up serialization for pytorch tensors
+
+    try:
+        import torch
+
+        def _serialize_torch_tensor(obj):
+            return obj.numpy()
+
+        def _deserialize_torch_tensor(data):
+            return torch.from_numpy(data)
+
+        for t in [torch.FloatTensor, torch.DoubleTensor, torch.HalfTensor,
+                  torch.ByteTensor, torch.CharTensor, torch.ShortTensor,
+                  torch.IntTensor, torch.LongTensor]:
+            serialization_context.register_type(
+                t, "torch." + t.__name__,
+                custom_serializer=_serialize_torch_tensor,
+                custom_deserializer=_deserialize_torch_tensor)
+    except ImportError:
+        # no torch
+        pass
+
+
 def register_default_serialization_handlers(serialization_context):
 
     # ----------------------------------------------------------------------
@@ -154,37 +180,21 @@ def _deserialize_default_dict(data):
         custom_serializer=_serialize_numpy_array_list,
         custom_deserializer=_deserialize_numpy_array_list)
 
-    # ----------------------------------------------------------------------
-    # Set up serialization for pytorch tensors
-
-    try:
-        import torch
-
-        def _serialize_torch_tensor(obj):
-            return obj.numpy()
+    _register_custom_pandas_handlers(serialization_context)
 
-        def _deserialize_torch_tensor(data):
-            return torch.from_numpy(data)
 
-        for t in [torch.FloatTensor, torch.DoubleTensor, torch.HalfTensor,
-                  torch.ByteTensor, torch.CharTensor, torch.ShortTensor,
-                  torch.IntTensor, torch.LongTensor]:
-            serialization_context.register_type(
-                t, "torch." + t.__name__,
-                custom_serializer=_serialize_torch_tensor,
-                custom_deserializer=_deserialize_torch_tensor)
-    except ImportError:
-        # no torch
-        pass
-
-    _register_custom_pandas_handlers(serialization_context)
+def default_serialization_context():
+    context = SerializationContext()
+    register_default_serialization_handlers(context)
+    return context
 
 
 register_default_serialization_handlers(_default_serialization_context)
 
-pandas_serialization_context = _default_serialization_context.clone()
 
-pandas_serialization_context.register_type(
-    np.ndarray, 'np.array',
-    custom_serializer=_serialize_numpy_array_pickle,
-    custom_deserializer=_deserialize_numpy_array_pickle)
+def pandas_serialization_context():
+    context = default_serialization_context()
+    context.register_type(np.ndarray, 'np.array',
+                          custom_serializer=_serialize_numpy_array_pickle,
+                          custom_deserializer=_deserialize_numpy_array_pickle)
+    return context
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index fa265e55cfd76..ca2f1e3611963 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1404,7 +1404,7 @@ def _fully_loaded_dataframe_example():
 
 
 def _check_serialize_components_roundtrip(df):
-    ctx = pa.pandas_serialization_context
+    ctx = pa.pandas_serialization_context()
 
     components = ctx.serialize(df).to_components()
     deserialized = ctx.deserialize_components(components)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 284c7fb4ca78d..7a420106f9fb6 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -190,8 +190,7 @@ class CustomError(Exception):
 
 
 def make_serialization_context():
-
-    context = pa._default_serialization_context
+    context = pa.default_serialization_context()
 
     context.register_type(Foo, "Foo")
     context.register_type(Bar, "Bar")
@@ -207,26 +206,27 @@ def make_serialization_context():
     return context
 
 
-serialization_context = make_serialization_context()
+global_serialization_context = make_serialization_context()
 
 
-def serialization_roundtrip(value, scratch_buffer, ctx=serialization_context):
+def serialization_roundtrip(value, scratch_buffer,
+                            context=global_serialization_context):
     writer = pa.FixedSizeBufferWriter(scratch_buffer)
-    pa.serialize_to(value, writer, ctx)
+    pa.serialize_to(value, writer, context=context)
 
     reader = pa.BufferReader(scratch_buffer)
-    result = pa.deserialize_from(reader, None, ctx)
+    result = pa.deserialize_from(reader, None, context=context)
     assert_equal(value, result)
 
-    _check_component_roundtrip(value)
+    _check_component_roundtrip(value, context=context)
 
 
-def _check_component_roundtrip(value):
+def _check_component_roundtrip(value, context=global_serialization_context):
     # Test to/from components
-    serialized = pa.serialize(value)
+    serialized = pa.serialize(value, context=context)
     components = serialized.to_components()
     from_comp = pa.SerializedPyObject.from_components(components)
-    recons = from_comp.deserialize()
+    recons = from_comp.deserialize(context=context)
     assert_equal(value, recons)
 
 
@@ -252,7 +252,7 @@ def test_primitive_serialization(large_buffer):
     for obj in PRIMITIVE_OBJECTS:
         serialization_roundtrip(obj, large_buffer)
         serialization_roundtrip(obj, large_buffer,
-                                pa.pandas_serialization_context)
+                                pa.pandas_serialization_context())
 
 
 def test_serialize_to_buffer():
@@ -318,22 +318,26 @@ def test_datetime_serialization(large_buffer):
 def test_torch_serialization(large_buffer):
     pytest.importorskip("torch")
     import torch
+
+    serialization_context = pa.default_serialization_context()
+    pa.register_torch_serialization_handlers(serialization_context)
     # These are the only types that are supported for the
     # PyTorch to NumPy conversion
     for t in ["float32", "float64",
               "uint8", "int16", "int32", "int64"]:
         obj = torch.from_numpy(np.random.randn(1000).astype(t))
-        serialization_roundtrip(obj, large_buffer)
+        serialization_roundtrip(obj, large_buffer,
+                                context=serialization_context)
 
 
 def test_numpy_immutable(large_buffer):
     obj = np.zeros([10])
 
     writer = pa.FixedSizeBufferWriter(large_buffer)
-    pa.serialize_to(obj, writer, serialization_context)
+    pa.serialize_to(obj, writer, global_serialization_context)
 
     reader = pa.BufferReader(large_buffer)
-    result = pa.deserialize_from(reader, None, serialization_context)
+    result = pa.deserialize_from(reader, None, global_serialization_context)
     with pytest.raises(ValueError):
         result[0] = 1.0
 
@@ -351,12 +355,12 @@ def serialize_dummy_class(obj):
     def deserialize_dummy_class(serialized_obj):
         return serialized_obj
 
-    pa._default_serialization_context.register_type(
-        DummyClass, "DummyClass",
-        custom_serializer=serialize_dummy_class,
-        custom_deserializer=deserialize_dummy_class)
+    context = pa.default_serialization_context()
+    context.register_type(DummyClass, "DummyClass",
+                          custom_serializer=serialize_dummy_class,
+                          custom_deserializer=deserialize_dummy_class)
 
-    pa.serialize(DummyClass())
+    pa.serialize(DummyClass(), context=context)
 
 
 def test_buffer_serialization():
@@ -370,13 +374,14 @@ def serialize_buffer_class(obj):
     def deserialize_buffer_class(serialized_obj):
         return serialized_obj
 
-    pa._default_serialization_context.register_type(
+    context = pa.default_serialization_context()
+    context.register_type(
         BufferClass, "BufferClass",
         custom_serializer=serialize_buffer_class,
         custom_deserializer=deserialize_buffer_class)
 
-    b = pa.serialize(BufferClass()).to_buffer()
-    assert pa.deserialize(b).to_pybytes() == b"hello"
+    b = pa.serialize(BufferClass(), context=context).to_buffer()
+    assert pa.deserialize(b, context=context).to_pybytes() == b"hello"
 
 
 @pytest.mark.skip(reason="extensive memory requirements")
@@ -485,15 +490,16 @@ def test_serialize_subclasses():
     # with register_type will result in faster and more memory
     # efficient serialization.
 
-    serialization_context.register_type(
+    context = pa.default_serialization_context()
+    context.register_type(
         Serializable, "Serializable",
         custom_serializer=serialize_serializable,
         custom_deserializer=deserialize_serializable)
 
     a = SerializableClass()
-    serialized = pa.serialize(a)
+    serialized = pa.serialize(a, context=context)
 
-    deserialized = serialized.deserialize()
+    deserialized = serialized.deserialize(context=context)
     assert type(deserialized).__name__ == SerializableClass.__name__
     assert deserialized.value == 3
 

From 0ada87531dca52d51d4f60d3148a9ba733d96a48 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 1 Feb 2018 19:11:28 +0100
Subject: [PATCH 1426/1644] ARROW-1861: [Python] Rework benchmark suite [skip
 ci]

This PR focusses on:
* ASV setup fixes
* splitting the benchmark file
* improving the array from/to pylist conversion benchmarks

Author: Antoine Pitrou <antoine@python.org>

Closes #1543 from pitrou/ARROW-1861-rework-benchmarks and squashes the following commits:

b608579 [Antoine Pitrou] ARROW-1861: [Python] Rework benchmark suite [skip ci]
---
 python/README-benchmarks.md                   |  54 ++++
 python/asv.conf.json                          |  98 +++++-
 python/benchmarks/array_ops.py                |  35 +++
 python/benchmarks/common.py                   |  84 +++++
 python/benchmarks/convert_builtins.py         | 295 ++++++++++++++++++
 .../{array.py => convert_pandas.py}           |  42 +--
 6 files changed, 561 insertions(+), 47 deletions(-)
 create mode 100644 python/README-benchmarks.md
 create mode 100644 python/benchmarks/array_ops.py
 create mode 100644 python/benchmarks/common.py
 create mode 100644 python/benchmarks/convert_builtins.py
 rename python/benchmarks/{array.py => convert_pandas.py} (59%)

diff --git a/python/README-benchmarks.md b/python/README-benchmarks.md
new file mode 100644
index 0000000000000..6389665b075d9
--- /dev/null
+++ b/python/README-benchmarks.md
@@ -0,0 +1,54 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Benchmarks
+
+The `pyarrow` package comes with a suite of benchmarks meant to
+run with [ASV](https://asv.readthedocs.io).  You'll need to install
+the `asv` package first (`pip install asv`).
+
+## Running with your local tree
+
+When developing, the simplest and fastest way to run the benchmark suite
+against your local changes is to use the `asv dev` command.  This will
+use your current Python interpreter and environment.
+
+## Running with arbitrary revisions
+
+ASV allows to store results and generate graphs of the benchmarks over
+the project's evolution.  Doing this requires a bit more massaging
+currently.
+
+First you have to install our ASV fork:
+
+```shell
+pip install git+https://github.com/pitrou/asv.git@issue-547-specify-project-subdir
+```
+
+<!--- TODO remove the above once https://github.com/airspeed-velocity/asv/pull/611 is merged -->
+
+Then you need to set up a few environment variables:
+
+```shell
+export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.1
+export PYARROW_BUNDLE_ARROW_CPP=1
+```
+
+Now you should be ready to run `asv run` or whatever other command
+suits your needs.
diff --git a/python/asv.conf.json b/python/asv.conf.json
index 2a1dd42aba136..150153c8020f9 100644
--- a/python/asv.conf.json
+++ b/python/asv.conf.json
@@ -28,12 +28,17 @@
 
     // The URL or local path of the source code repository for the
     // project being benchmarked
-    "repo": "https://github.com/apache/arrow/",
+    "repo": "..",
+
+    // The Python project's subdirectory in your repo.  If missing or
+    // the empty string, the project is assumed to be located at the root
+    // of the repository.
+    "repo_subdir": "python",
 
     // List of branches to benchmark. If not provided, defaults to "master"
-    // (for git) or "tip" (for mercurial).
+    // (for git) or "default" (for mercurial).
     // "branches": ["master"], // for git
-    // "branches": ["tip"],    // for mercurial
+    // "branches": ["default"],    // for mercurial
 
     // The DVCS being used.  If not set, it will be automatically
     // determined from "repo" by looking at the protocol in the URL
@@ -46,22 +51,72 @@
     // If missing or the empty string, the tool will be automatically
     // determined by looking for tools on the PATH environment
     // variable.
-    "environment_type": "virtualenv",
+    "environment_type": "conda",
+    "conda_channels": ["conda-forge", "defaults"],
 
     // the base URL to show a commit for the project.
     "show_commit_url": "https://github.com/apache/arrow/commit/",
 
     // The Pythons you'd like to test against.  If not provided, defaults
     // to the current version of Python used to run `asv`.
-    // "pythons": ["2.7", "3.3"],
+    "pythons": ["3.6"],
 
     // The matrix of dependencies to test.  Each key is the name of a
     // package (in PyPI) and the values are version numbers.  An empty
-    // list indicates to just test against the default (latest)
-    // version.
+    // list or empty string indicates to just test against the default
+    // (latest) version. null indicates that the package is to not be
+    // installed. If the package to be tested is only available from
+    // PyPi, and the 'environment_type' is conda, then you can preface
+    // the package name by 'pip+', and the package will be installed via
+    // pip (with all the conda available packages installed first,
+    // followed by the pip installed packages).
+    //
     // "matrix": {
-    //     "numpy": ["1.6", "1.7"]
+    //     "numpy": ["1.6", "1.7"],
+    //     "six": ["", null],        // test with and without six installed
+    //     "pip+emcee": [""],   // emcee is only available for install with pip.
     // },
+    "matrix": {
+        "boost-cpp": [],
+        "cmake": [],
+        "cython": [],
+        "numpy": ["1.14"],
+        "pandas": ["0.22"],
+        "pip+setuptools_scm": [],
+    },
+
+    // Combinations of libraries/python versions can be excluded/included
+    // from the set to test. Each entry is a dictionary containing additional
+    // key-value pairs to include/exclude.
+    //
+    // An exclude entry excludes entries where all values match. The
+    // values are regexps that should match the whole string.
+    //
+    // An include entry adds an environment. Only the packages listed
+    // are installed. The 'python' key is required. The exclude rules
+    // do not apply to includes.
+    //
+    // In addition to package names, the following keys are available:
+    //
+    // - python
+    //     Python version, as in the *pythons* variable above.
+    // - environment_type
+    //     Environment type, as above.
+    // - sys_platform
+    //     Platform, as in sys.platform. Possible values for the common
+    //     cases: 'linux2', 'win32', 'cygwin', 'darwin'.
+    //
+    // "exclude": [
+    //     {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
+    //     {"environment_type": "conda", "six": null}, // don't run without six on conda
+    // ],
+    //
+    // "include": [
+    //     // additional env for python2.7
+    //     {"python": "2.7", "numpy": "1.8"},
+    //     // additional env if run on windows+conda
+    //     {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""},
+    // ],
 
     // The directory (relative to the current directory) that benchmarks are
     // stored in.  If not provided, defaults to "benchmarks"
@@ -71,7 +126,6 @@
     // environments in.  If not provided, defaults to "env"
     "env_dir": ".asv/env",
 
-
     // The directory (relative to the current directory) that raw benchmark
     // results are stored in.  If not provided, defaults to "results".
     "results_dir": ".asv/results",
@@ -86,5 +140,29 @@
     // `asv` will cache wheels of the recent builds in each
     // environment, making them faster to install next time.  This is
     // number of builds to keep, per environment.
-    // "wheel_cache_size": 0
+    // "wheel_cache_size": 0,
+
+    // The commits after which the regression search in `asv publish`
+    // should start looking for regressions. Dictionary whose keys are
+    // regexps matching to benchmark names, and values corresponding to
+    // the commit (exclusive) after which to start looking for
+    // regressions.  The default is to start from the first commit
+    // with results. If the commit is `null`, regression detection is
+    // skipped for the matching benchmark.
+    //
+    // "regressions_first_commits": {
+    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
+    //    "another_benchmark": null,   // Skip regression detection altogether
+    // }
+
+    // The thresholds for relative change in results, after which `asv
+    // publish` starts reporting regressions. Dictionary of the same
+    // form as in ``regressions_first_commits``, with values
+    // indicating the thresholds.  If multiple entries match, the
+    // maximum is taken. If no entry matches, the default is 5%.
+    //
+    // "regressions_thresholds": {
+    //    "some_benchmark": 0.01,     // Threshold of 1%
+    //    "another_benchmark": 0.5,   // Threshold of 50%
+    // }
 }
diff --git a/python/benchmarks/array_ops.py b/python/benchmarks/array_ops.py
new file mode 100644
index 0000000000000..70ee7f1e1fcfc
--- /dev/null
+++ b/python/benchmarks/array_ops.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+import pyarrow as pa
+
+
+class ScalarAccess(object):
+    n = 10 ** 5
+
+    def setUp(self):
+        self._array = pa.array(list(range(self.n)), type=pa.int64())
+        self._array_items = list(self._array)
+
+    def time_getitem(self):
+        for i in range(self.n):
+            self._array[i]
+
+    def time_as_py(self):
+        for item in self._array_items:
+            item.as_py()
diff --git a/python/benchmarks/common.py b/python/benchmarks/common.py
new file mode 100644
index 0000000000000..7dd42fde5abe1
--- /dev/null
+++ b/python/benchmarks/common.py
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import codecs
+import os
+import sys
+import unicodedata
+
+import numpy as np
+
+
+def _multiplicate_sequence(base, target_size):
+    q, r = divmod(target_size, len(base))
+    return [base] * q + [base[:r]]
+
+
+def get_random_bytes(n):
+    rnd = np.random.RandomState(42)
+    # Computing a huge random bytestring can be costly, so we get at most
+    # 100KB and duplicate the result as needed
+    base_size = 100003
+    q, r = divmod(n, base_size)
+    if q == 0:
+        result = rnd.bytes(r)
+    else:
+        base = rnd.bytes(base_size)
+        result = b''.join(_multiplicate_sequence(base, n))
+    assert len(result) == n
+    return result
+
+
+def get_random_ascii(n):
+    arr = np.frombuffer(get_random_bytes(n), dtype=np.int8) & 0x7f
+    result, _ = codecs.ascii_decode(arr)
+    assert isinstance(result, str)
+    assert len(result) == n
+    return result
+
+
+def _random_unicode_letters(n):
+    """
+    Generate a string of random unicode letters (slow).
+    """
+    def _get_more_candidates():
+        return rnd.randint(0, sys.maxunicode, size=n).tolist()
+
+    rnd = np.random.RandomState(42)
+    out = []
+    candidates = []
+
+    while len(out) < n:
+        if not candidates:
+            candidates = _get_more_candidates()
+        ch = chr(candidates.pop())
+        # XXX Do we actually care that the code points are valid?
+        if unicodedata.category(ch)[0] == 'L':
+            out.append(ch)
+    return out
+
+
+_1024_random_unicode_letters = _random_unicode_letters(1024)
+
+
+def get_random_unicode(n):
+    indices = np.frombuffer(get_random_bytes(n * 2), dtype=np.int16) & 1023
+    unicode_arr = np.array(_1024_random_unicode_letters)[indices]
+
+    result = ''.join(unicode_arr.tolist())
+    assert len(result) == n, (len(result), len(unicode_arr))
+    return result
diff --git a/python/benchmarks/convert_builtins.py b/python/benchmarks/convert_builtins.py
new file mode 100644
index 0000000000000..92b2b850f2a0a
--- /dev/null
+++ b/python/benchmarks/convert_builtins.py
@@ -0,0 +1,295 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from functools import partial
+import itertools
+
+import numpy as np
+import pyarrow as pa
+
+from . import common
+
+
+DEFAULT_NONE_PROB = 0.3
+
+
+# TODO:
+# - test dates and times
+# - test decimals
+
+class BuiltinsGenerator(object):
+
+    def __init__(self, seed=42):
+        self.rnd = np.random.RandomState(seed)
+
+    def sprinkle_nones(self, lst, prob):
+        """
+        Sprinkle None entries in list *lst* with likelihood *prob*.
+        """
+        for i, p in enumerate(self.rnd.random_sample(size=len(lst))):
+            if p < prob:
+                lst[i] = None
+
+    def generate_int_list(self, n, none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of Python ints with *none_prob* probability of
+        an entry being None.
+        """
+        data = list(range(n))
+        self.sprinkle_nones(data, none_prob)
+        return data
+
+    def generate_float_list(self, n, none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of Python floats with *none_prob* probability of
+        an entry being None.
+        """
+        # Make sure we get Python floats, not np.float64
+        data = list(map(float, self.rnd.uniform(0.0, 1.0, n)))
+        assert len(data) == n
+        self.sprinkle_nones(data, none_prob)
+        return data
+
+    def generate_bool_list(self, n, none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of Python bools with *none_prob* probability of
+        an entry being None.
+        """
+        # Make sure we get Python bools, not np.bool_
+        data = [bool(x >= 0.5) for x in self.rnd.uniform(0.0, 1.0, n)]
+        assert len(data) == n
+        self.sprinkle_nones(data, none_prob)
+        return data
+
+    def _generate_varying_sequences(self, random_factory, n, min_size, max_size, none_prob):
+        """
+        Generate a list of *n* sequences of varying size between *min_size*
+        and *max_size*, with *none_prob* probability of an entry being None.
+        The base material for each sequence is obtained by calling
+        `random_factory(<some size>)`
+        """
+        base_size = 10000
+        base = random_factory(base_size + max_size)
+        data = []
+        for i in range(n):
+            off = self.rnd.randint(base_size)
+            if min_size == max_size:
+                size = min_size
+            else:
+                size = self.rnd.randint(min_size, max_size + 1)
+            data.append(base[off:off + size])
+        self.sprinkle_nones(data, none_prob)
+        assert len(data) == n
+        return data
+
+    def generate_fixed_binary_list(self, n, size, none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of bytestrings with a fixed *size*.
+        """
+        return self._generate_varying_sequences(common.get_random_bytes, n,
+                                                size, size, none_prob)
+
+
+    def generate_varying_binary_list(self, n, min_size, max_size,
+                                     none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of bytestrings with a random size between
+        *min_size* and *max_size*.
+        """
+        return self._generate_varying_sequences(common.get_random_bytes, n,
+                                                min_size, max_size, none_prob)
+
+
+    def generate_ascii_string_list(self, n, min_size, max_size,
+                                   none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of ASCII strings with a random size between
+        *min_size* and *max_size*.
+        """
+        return self._generate_varying_sequences(common.get_random_ascii, n,
+                                                min_size, max_size, none_prob)
+
+
+    def generate_unicode_string_list(self, n, min_size, max_size,
+                                     none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of unicode strings with a random size between
+        *min_size* and *max_size*.
+        """
+        return self._generate_varying_sequences(common.get_random_unicode, n,
+                                                min_size, max_size, none_prob)
+
+
+    def generate_int_list_list(self, n, min_size, max_size,
+                               none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of lists of Python ints with a random size between
+        *min_size* and *max_size*.
+        """
+        return self._generate_varying_sequences(
+            partial(self.generate_int_list, none_prob=none_prob),
+            n, min_size, max_size, none_prob)
+
+
+    def generate_dict_list(self, n, none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of dicts with a random size between *min_size* and
+        *max_size*.
+        Each dict has the form `{'u': int value, 'v': float value, 'w': bool value}`
+        """
+        ints = self.generate_int_list(n, none_prob=none_prob)
+        floats = self.generate_float_list(n, none_prob=none_prob)
+        bools = self.generate_bool_list(n, none_prob=none_prob)
+        dicts = []
+        # Keep half the Nones, omit the other half
+        keep_nones = itertools.cycle([True, False])
+        for u, v, w in zip(ints, floats, bools):
+            d = {}
+            if u is not None or next(keep_nones):
+                d['u'] = u
+            if v is not None or next(keep_nones):
+                d['v'] = v
+            if w is not None or next(keep_nones):
+                d['w'] = w
+            dicts.append(d)
+        self.sprinkle_nones(dicts, none_prob)
+        assert len(dicts) == n
+        return dicts
+
+    def get_type_and_builtins(self, n, type_name):
+        """
+        Return a `(arrow type, list)` tuple where the arrow type
+        corresponds to the given logical *type_name*, and the list
+        is a list of *n* random-generated Python objects compatible
+        with the arrow type.
+        """
+        size = None
+
+        if type_name in ('bool', 'ascii', 'unicode', 'int64 list', 'struct'):
+            kind = type_name
+        elif type_name.startswith(('int', 'uint')):
+            kind = 'int'
+        elif type_name.startswith('float'):
+            kind = 'float'
+        elif type_name == 'binary':
+            kind = 'varying binary'
+        elif type_name.startswith('binary'):
+            kind = 'fixed binary'
+            size = int(type_name[6:])
+            assert size > 0
+        else:
+            raise ValueError("unrecognized type %r" % (type_name,))
+
+        if kind in ('int', 'float'):
+            ty = getattr(pa, type_name)()
+        elif kind == 'bool':
+            ty = pa.bool_()
+        elif kind == 'fixed binary':
+            ty = pa.binary(size)
+        elif kind == 'varying binary':
+            ty = pa.binary()
+        elif kind in ('ascii', 'unicode'):
+            ty = pa.string()
+        elif kind == 'int64 list':
+            ty = pa.list_(pa.int64())
+        elif kind == 'struct':
+            ty = pa.struct([pa.field('u', pa.int64()),
+                            pa.field('v', pa.float64()),
+                            pa.field('w', pa.bool_())])
+
+        factories = {
+            'int': self.generate_int_list,
+            'float': self.generate_float_list,
+            'bool': self.generate_bool_list,
+            'fixed binary': partial(self.generate_fixed_binary_list,
+                                    size=size),
+            'varying binary': partial(self.generate_varying_binary_list,
+                                      min_size=3, max_size=40),
+            'ascii': partial(self.generate_ascii_string_list,
+                             min_size=3, max_size=40),
+            'unicode': partial(self.generate_unicode_string_list,
+                               min_size=3, max_size=40),
+            'int64 list': partial(self.generate_int_list_list,
+                                  min_size=0, max_size=20),
+            'struct': self.generate_dict_list,
+        }
+        data = factories[kind](n)
+        return ty, data
+
+
+class ConvertPyListToArray(object):
+    """
+    Benchmark pa.array(list of values, type=...)
+    """
+    size = 10 ** 5
+    types = ('int32', 'uint32', 'int64', 'uint64',
+             'float32', 'float64', 'bool',
+             'binary', 'binary10', 'ascii', 'unicode',
+             'int64 list', 'struct')
+
+    param_names = ['type']
+    params = [types]
+
+    def setup(self, type_name):
+        gen = BuiltinsGenerator()
+        self.ty, self.data = gen.get_type_and_builtins(self.size, type_name)
+
+    def time_convert(self, *args):
+        pa.array(self.data, type=self.ty)
+
+
+class InferPyListToArray(object):
+    """
+    Benchmark pa.array(list of values) with type inference
+    """
+    size = 10 ** 5
+    types = ('int64', 'float64', 'bool', 'binary', 'ascii', 'unicode',
+             'int64 list')
+    # TODO add 'struct' when supported
+
+    param_names = ['type']
+    params = [types]
+
+    def setup(self, type_name):
+        gen = BuiltinsGenerator()
+        self.ty, self.data = gen.get_type_and_builtins(self.size, type_name)
+
+    def time_infer(self, *args):
+        arr = pa.array(self.data)
+        assert arr.type == self.ty
+
+
+class ConvertArrayToPyList(object):
+    """
+    Benchmark pa.array.to_pylist()
+    """
+    size = 10 ** 5
+    types = ('int32', 'uint32', 'int64', 'uint64',
+             'float32', 'float64', 'bool',
+             'binary', 'binary10', 'ascii', 'unicode',
+             'int64 list', 'struct')
+
+    param_names = ['type']
+    params = [types]
+
+    def setup(self, type_name):
+        gen = BuiltinsGenerator()
+        self.ty, self.data = gen.get_type_and_builtins(self.size, type_name)
+        self.arr = pa.array(self.data, type=self.ty)
+
+    def time_convert(self, *args):
+        self.arr.to_pylist()
diff --git a/python/benchmarks/array.py b/python/benchmarks/convert_pandas.py
similarity index 59%
rename from python/benchmarks/array.py
rename to python/benchmarks/convert_pandas.py
index e22c0f7fc9e70..c4a7a59cb77dc 100644
--- a/python/benchmarks/array.py
+++ b/python/benchmarks/convert_pandas.py
@@ -17,21 +17,7 @@
 
 import numpy as np
 import pandas as pd
-import pyarrow as A
-
-
-class PyListConversions(object):
-    param_names = ('size',)
-    params = (1, 10 ** 5, 10 ** 6, 10 ** 7)
-
-    def setup(self, n):
-        self.data = list(range(n))
-
-    def time_from_pylist(self, n):
-        A.from_pylist(self.data)
-
-    def peakmem_from_pylist(self, n):
-        A.from_pylist(self.data)
+import pyarrow as pa
 
 
 class PandasConversionsBase(object):
@@ -46,37 +32,19 @@ def setup(self, n, dtype):
 
 class PandasConversionsToArrow(PandasConversionsBase):
     param_names = ('size', 'dtype')
-    params = ((1, 10 ** 5, 10 ** 6, 10 ** 7), ('int64', 'float64', 'float64_nans', 'str'))
+    params = ((10, 10 ** 6), ('int64', 'float64', 'float64_nans', 'str'))
 
     def time_from_series(self, n, dtype):
-        A.Table.from_pandas(self.data)
-
-    def peakmem_from_series(self, n, dtype):
-        A.Table.from_pandas(self.data)
+        pa.Table.from_pandas(self.data)
 
 
 class PandasConversionsFromArrow(PandasConversionsBase):
     param_names = ('size', 'dtype')
-    params = ((1, 10 ** 5, 10 ** 6, 10 ** 7), ('int64', 'float64', 'float64_nans', 'str'))
+    params = ((10, 10 ** 6), ('int64', 'float64', 'float64_nans', 'str'))
 
     def setup(self, n, dtype):
         super(PandasConversionsFromArrow, self).setup(n, dtype)
-        self.arrow_data = A.Table.from_pandas(self.data)
+        self.arrow_data = pa.Table.from_pandas(self.data)
 
     def time_to_series(self, n, dtype):
         self.arrow_data.to_pandas()
-
-    def peakmem_to_series(self, n, dtype):
-        self.arrow_data.to_pandas()
-
-
-class ScalarAccess(object):
-    param_names = ('size',)
-    params = (1, 10 ** 5, 10 ** 6, 10 ** 7)
-
-    def setUp(self, n):
-        self._array = A.from_pylist(list(range(n)))
-
-    def time_as_py(self, n):
-        for i in range(n):
-            self._array[i].as_py()

From c1d77a130fc571c5d7e8016a5405f9833cb6ac78 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 1 Feb 2018 19:19:14 +0100
Subject: [PATCH 1427/1644] ARROW-2076: [Python] Display slowest test durations

Author: Antoine Pitrou <antoine@python.org>

Closes #1541 from pitrou/slowest-test-durations and squashes the following commits:

cf5e9c8 [Antoine Pitrou] [Python] Display slowest test durations
---
 ci/msvc-build.bat          | 2 +-
 ci/travis_script_python.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 9651772ca3fe9..58dfc2a146572 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -145,6 +145,6 @@ pushd python
 
 set PYARROW_CXXFLAGS=/WX
 python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel  || exit /B
-py.test pyarrow -v -s --parquet || exit /B
+py.test pyarrow -r sxX --durations=15 -v -s --parquet || exit /B
 
 popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 9e74906d03739..7c896df9c840f 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -96,7 +96,7 @@ if [ $TRAVIS_OS_NAME == "linux" ]; then
 fi
 
 PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
-python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
+python -m pytest -vv -r sxX --durations=15 -s $PYARROW_PATH --parquet
 
 if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
   # Build documentation once

From f84af8f79c0793f304743298cf40daf3a0bbf042 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Thu, 1 Feb 2018 13:59:05 -0800
Subject: [PATCH 1428/1644] ARROW-2065: [Python] Fix bug in
 SerializationContext.clone().

One issue is that I don't think the test fails on the current master.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1539 from robertnishihara/fixclonebug and squashes the following commits:

61534b8b [Robert Nishihara] Fix test.
35a559a8 [Robert Nishihara] Fix bug in which SerializationContext.clone() doesn't copy a field.
---
 python/pyarrow/serialization.pxi           |  1 +
 python/pyarrow/tests/test_serialization.py | 27 ++++++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index e7a39905f1f65..44bf8773e6cb0 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -89,6 +89,7 @@ cdef class SerializationContext:
         result = SerializationContext()
         result.type_to_type_id = self.type_to_type_id.copy()
         result.whitelisted_types = self.whitelisted_types.copy()
+        result.types_to_pickle = self.types_to_pickle.copy()
         result.custom_serializers = self.custom_serializers.copy()
         result.custom_deserializers = self.custom_deserializers.copy()
         result.pickle_serializer = self.pickle_serializer
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 7a420106f9fb6..23059e12d8be5 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -248,6 +248,33 @@ def large_memory_map(tmpdir_factory, size=100*1024*1024):
     return path
 
 
+def test_clone():
+    context = pa.SerializationContext()
+
+    class Foo(object):
+        pass
+
+    def custom_serializer(obj):
+        return 0
+
+    def custom_deserializer(serialized_obj):
+        return (serialized_obj, 'a')
+
+    context.register_type(Foo, 'Foo', custom_serializer=custom_serializer,
+                          custom_deserializer=custom_deserializer)
+
+    new_context = context.clone()
+
+    f = Foo()
+    serialized = pa.serialize(f, context=context)
+    deserialized = serialized.deserialize(context=context)
+    assert deserialized == (0, 'a')
+
+    serialized = pa.serialize(f, context=new_context)
+    deserialized = serialized.deserialize(context=new_context)
+    assert deserialized == (0, 'a')
+
+
 def test_primitive_serialization(large_buffer):
     for obj in PRIMITIVE_OBJECTS:
         serialization_roundtrip(obj, large_buffer)

From e3277471a3392a6f276455fc2543fdee461e9de6 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Thu, 1 Feb 2018 17:23:33 -0500
Subject: [PATCH 1429/1644] ARROW-1990: [JS] C++ Refactor, Add DataFrame

This PR moves the `Table` class out of the Vector hierarchy and adds optimized dataframe operations to it. Currently implements an optimized `scan()` method, `filter(predicate)`, `count()`, and `countBy(column_name)` (only works on dictionary-encoded columns).

Some usage examples, based on the file generated by `js/test/data/tables/generate.py`:
``` js
> let table = Table.from(...);
> table.count()
1000000
> table.filter(col('lat').gteq(0)).count()
499718
> table.countBy('origin').toJSON()
{ Charlottesville: 166839,
  'New York': 166251,
  'San Francisco': 166642,
  Seattle: 166659,
  'Terre Haute': 166756,
  'Washington, DC': 166853 }
> table.filter(col('lng').gteq(0)).countBy('origin').toJSON()
{ Charlottesville: 83109,
  'New York': 83221,
  'San Francisco': 83515,
  Seattle: 83362,
  'Terre Haute': 83314,
  'Washington, DC': 83479 }
```
There are performance tests for the dataframe operations, to run them you must first generate the test data by running `npm run create:perfdata`.

The PR also includes @trxcllnt's refactor of the JS implementation to make it more closely resemble the C++ implementation. This refactor resolves multiple JIRAs: ARROW-1903, ARROW-1898, ARROW-1502, ARROW-1952 (partially), and ARROW-1985

Author: Paul Taylor <paul.e.taylor@me.com>
Author: Brian Hulette <brian.hulette@ccri.com>
Author: Brian Hulette <hulettbh@gmail.com>

Closes #1482 from TheNeuralBit/table-scan-perf and squashes the following commits:

52f1e0e4 [Brian Hulette] <, > are not commutative, misc cleanup
04b18380 [Brian Hulette] even more table tests
16b9ccb1 [Brian Hulette] Merge pull request #4 from trxcllnt/js-cpp-refactor
fe300dfa [Paul Taylor] fix closure es5/umd toString() iterator
3d5240a7 [Paul Taylor] fix more externs
10c48adc [Paul Taylor] Merge branch 'table-scan-perf' of github.com:ccri/arrow into js-cpp-refactor
dbe7f81c [Brian Hulette] Add more Table unit tests
19109625 [Brian Hulette] Add optional bind callback to scan
5bdf17f5 [Brian Hulette] Fix perf
8cf2473f [Brian Hulette] Merge remote-tracking branch 'origin/master' into table-scan-perf
4a41b180 [Paul Taylor] add src/predicate to the list of exports we should save from uglify
5a91fab8 [Paul Taylor] add more view, predicate externs
f6adfb35 [Brian Hulette] Create predicate namespace
f7bb0ed9 [Paul Taylor] Merge branch 'table-scan-perf' of github.com:ccri/arrow into js-cpp-refactor
e148ee46 [Paul Taylor] Merge branch 'extern-woes' into js-cpp-refactor
25cdc4aa [Paul Taylor] add src/predicate to the list of exports we should save from uglify
dc7c7285 [Paul Taylor] add more view, predicate externs
25e6af72 [Brian Hulette] Create predicate namespace
579ab1fe [Brian Hulette] Merge pull request #2 from trxcllnt/js-cpp-refactor
f3cde1aa [Paul Taylor] fix lint
9769773e [Paul Taylor] fix vector perf tests
016ba78a [Brian Hulette] Merge pull request #1 from trxcllnt/js-cpp-refactor
272d2936 [Paul Taylor] Merge pull request #4 from ccri/empty-table
7bc7363e [Brian Hulette] Fix exception for empty Table
8ddce0a1 [Paul Taylor] check bounds in getChildAt(i) to avoid NPEs
f1dead0a [Paul Taylor] compute chunked nested childData list correctly
18807c6c [Paul Taylor] rename ChunkData's fields so it's more clear they're not semantically similar to other similarly named fields
7e43b78f [Paul Taylor] add test:integration npm script
a5f200fb [Paul Taylor] Merge pull request #3 from ccri/table-from-struct
c8cd286b [Brian Hulette] Add Table.fromStruct
a00415eb [Brian Hulette] Fix perf
54d4f5b1 [Paul Taylor] lazily allocate table and recordbatch columns, support NestedView's getChildAt(i) method in ChunkedView
40b36383 [Paul Taylor] run integration tests with local data for coverage stats
fe31ee01 [Paul Taylor] slice the flat data values before returning an iterator of them
e537789d [Paul Taylor] make it easier to run all integration tests from local data
c0fd2f96 [Paul Taylor] use the dictionary of the last chunked vector list for chunked dictionary vectors
e33c0688 [Paul Taylor] Merge pull request #2 from ccri/fixed-size-list
5bb63af4 [Brian Hulette] Don't read OFFSET vector for FixedSizeList
614b688c [Paul Taylor] add asEpochMs to date and timestamp vectors
87334a51 [Paul Taylor] Merge branch 'table-scan-perf' of github.com:ccri/arrow into js-cpp-refactor
b7f5bfba [Paul Taylor] rename numRows to length, add table.getColumn()
e81082f9 [Paul Taylor] export vector views, allow cloning data as another type
700a47cb [Paul Taylor] export visitors
e859e13c [Paul Taylor] fix package.json bin entry
0620cfdc [Brian Hulette] use Math.fround
0126dc4d [Brian Hulette] Don't recompute total length
e761eeeb [Brian Hulette] Rename asJSON to toJSON
6c91ed4a [Paul Taylor] Merge branch 'master' of github.com:apache/arrow into js-cpp-refactor-merge_with-table-scan-perf
d2b18d55 [Paul Taylor] Merge remote-tracking branch 'ccri/table-scan-perf' into js-cpp-refactor-merge_with-table-scan-perf
f3f3b867 [Paul Taylor] rename table.ts to recordbatch.ts in preparation for merging latest
e3f629d0 [Paul Taylor] fix rest of the mangling issues
fa7c17a3 [Paul Taylor] passing all tests except es5 umd mangler ones
e20decd5 [Brian Hulette] Add license headers
edcbdbed [Brian Hulette] cleanup
20717d59 [Brian Hulette] Fixed countBy(string)
72448870 [Brian Hulette] Add table unit tests...
67191477 [Brian Hulette] Add DataFrame.countBy operation
2f4a3491 [Brian Hulette] Minor tweaks
2e118aba [Brian Hulette] linter
a788db31 [Brian Hulette] Cleanup
a9fff890 [Brian Hulette] Move Table out of the Vector hierarchy
1d60aa14 [Brian Hulette] Moved DataFrame ops to Table. DataFrame is now an interface
e8979ba5 [Brian Hulette] Refactor DataFrame to extend Vector<StructRow>
6a41d687 [Brian Hulette] clean up table benchmarks
2744c63c [Brian Hulette] Remove Chunked/Simple DataFrame distinction
aa999f87 [Brian Hulette] Add DictionaryVector optimization for equals predicate
4d9e8c06 [Brian Hulette] Add concept of predicates for filtering dataframes
796f45dd [Brian Hulette] add DataFrame filter and count ops
30f03309 [Brian Hulette] Add basic DataFrame impl ...
a1edac20 [Brian Hulette] Add perf tests for table scans
d18d9153 [Paul Taylor] fix struct and map rows
61dc6990 [Paul Taylor] WIP -- refactor types to closer match arrow-cpp
62db3381 [Paul Taylor] update dependencies and add es6+ umd targets to jest transform ignore patterns to fix ci
6ff18e94 [Paul Taylor] ship es2015 commonJS in main package to avoid confusion
74e828af [Paul Taylor] fix typings issues (ARROW-1903)
---
 js/bin/integration.js                         |  56 +-
 js/gulp/argv.js                               |  31 +-
 js/gulp/closure-task.js                       |   8 +-
 js/gulp/package-task.js                       |  23 +-
 js/gulp/test-task.js                          |  10 +-
 js/gulp/typescript-task.js                    |   8 +-
 js/gulp/uglify-task.js                        |  23 +-
 js/gulp/util.js                               |   7 +-
 js/package.json                               |  32 +-
 js/perf/index.js                              | 162 ++++-
 .../vector/utf8.ts => perf/table_config.js}   |  46 +-
 js/src/Arrow.externs.js                       | 680 ++++++++++++++++++
 js/src/Arrow.externs.ts                       |  84 ---
 js/src/Arrow.ts                               | 345 ++++++---
 js/src/bin/arrow2csv.ts                       |  54 +-
 js/src/data.ts                                | 327 +++++++++
 js/src/{format => }/fb/File.ts                |   2 +
 js/src/{format => }/fb/File_generated.js      |   0
 js/src/{format => }/fb/Message.ts             |   4 +
 js/src/{format => }/fb/Message_generated.js   |   0
 js/src/{format => }/fb/Schema.ts              |  23 +
 js/src/{format => }/fb/Schema_generated.js    |   0
 js/src/format/arrow.ts                        |  32 -
 js/src/format/fb.ts                           | 234 ------
 js/src/format/json.ts                         | 173 -----
 js/src/format/types.ts                        | 393 ----------
 js/src/ipc/metadata.ts                        |  89 +++
 js/src/{ => ipc}/reader/arrow.ts              |  26 +-
 js/src/ipc/reader/binary.ts                   | 449 ++++++++++++
 js/src/ipc/reader/json.ts                     | 323 +++++++++
 js/src/ipc/reader/vector.ts                   | 131 ++++
 js/src/predicate.ts                           | 225 ++++++
 js/src/reader/buffer.ts                       | 229 ------
 js/src/reader/json.ts                         | 176 -----
 js/src/reader/vector.ts                       | 255 -------
 js/src/recordbatch.ts                         |  70 ++
 js/src/table.ts                               | 344 +++++++++
 js/src/type.ts                                | 578 +++++++++++++++
 js/src/util/bit.ts                            | 127 ++++
 js/src/util/compat.ts                         |  49 ++
 js/src/util/layout.ts                         |  11 +-
 js/src/vector.ts                              | 441 ++++++++++++
 js/src/vector/arrow.ts                        |  88 ---
 js/src/vector/chunked.ts                      | 117 +++
 js/src/vector/dictionary.ts                   |  46 +-
 js/src/vector/flat.ts                         | 306 ++++++++
 js/src/vector/list.ts                         | 143 ++--
 js/src/vector/nested.ts                       | 217 ++++++
 js/src/vector/numeric.ts                      | 169 -----
 js/src/vector/struct.ts                       | 127 ----
 js/src/vector/table.ts                        |  61 --
 js/src/vector/traits/field.ts                 |  71 --
 js/src/vector/traits/mixins.ts                |  75 --
 js/src/vector/traits/nullable.ts              |  70 --
 js/src/vector/traits/nullablefield.ts         |  71 --
 js/src/vector/traits/vectors.ts               |  75 --
 js/src/vector/types.ts                        |  43 --
 js/src/vector/vector.ts                       |  55 --
 js/src/vector/view.ts                         |   8 +
 js/src/vector/virtual.ts                      | 129 ----
 js/src/visitor.ts                             | 154 ++++
 js/test/Arrow.ts                              |  16 +-
 js/test/data/tables/generate.py               |  53 ++
 js/test/integration/validate-tests.ts         |  98 ++-
 js/test/unit/int-tests.ts                     |   7 +-
 js/test/unit/table-tests.ts                   | 475 ++++++++++++
 js/test/unit/vector-tests.ts                  | 136 ++--
 67 files changed, 6025 insertions(+), 3065 deletions(-)
 rename js/{src/vector/utf8.ts => perf/table_config.js} (51%)
 create mode 100644 js/src/Arrow.externs.js
 delete mode 100644 js/src/Arrow.externs.ts
 create mode 100644 js/src/data.ts
 rename js/src/{format => }/fb/File.ts (99%)
 rename js/src/{format => }/fb/File_generated.js (100%)
 rename js/src/{format => }/fb/Message.ts (99%)
 rename js/src/{format => }/fb/Message_generated.js (100%)
 rename js/src/{format => }/fb/Schema.ts (99%)
 rename js/src/{format => }/fb/Schema_generated.js (100%)
 delete mode 100644 js/src/format/arrow.ts
 delete mode 100644 js/src/format/fb.ts
 delete mode 100644 js/src/format/json.ts
 delete mode 100644 js/src/format/types.ts
 create mode 100644 js/src/ipc/metadata.ts
 rename js/src/{ => ipc}/reader/arrow.ts (62%)
 create mode 100644 js/src/ipc/reader/binary.ts
 create mode 100644 js/src/ipc/reader/json.ts
 create mode 100644 js/src/ipc/reader/vector.ts
 create mode 100644 js/src/predicate.ts
 delete mode 100644 js/src/reader/buffer.ts
 delete mode 100644 js/src/reader/json.ts
 delete mode 100644 js/src/reader/vector.ts
 create mode 100644 js/src/recordbatch.ts
 create mode 100644 js/src/table.ts
 create mode 100644 js/src/type.ts
 create mode 100644 js/src/util/bit.ts
 create mode 100644 js/src/util/compat.ts
 create mode 100644 js/src/vector.ts
 delete mode 100644 js/src/vector/arrow.ts
 create mode 100644 js/src/vector/chunked.ts
 create mode 100644 js/src/vector/flat.ts
 create mode 100644 js/src/vector/nested.ts
 delete mode 100644 js/src/vector/numeric.ts
 delete mode 100644 js/src/vector/struct.ts
 delete mode 100644 js/src/vector/table.ts
 delete mode 100644 js/src/vector/traits/field.ts
 delete mode 100644 js/src/vector/traits/mixins.ts
 delete mode 100644 js/src/vector/traits/nullable.ts
 delete mode 100644 js/src/vector/traits/nullablefield.ts
 delete mode 100644 js/src/vector/traits/vectors.ts
 delete mode 100644 js/src/vector/types.ts
 delete mode 100644 js/src/vector/vector.ts
 create mode 100644 js/src/vector/view.ts
 delete mode 100644 js/src/vector/virtual.ts
 create mode 100644 js/src/visitor.ts
 create mode 100644 js/test/data/tables/generate.py
 create mode 100644 js/test/unit/table-tests.ts

diff --git a/js/bin/integration.js b/js/bin/integration.js
index fe32433d3845a..2aeb14d0e3425 100755
--- a/js/bin/integration.js
+++ b/js/bin/integration.js
@@ -17,6 +17,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+var fs = require('fs');
+var glob = require('glob');
 var path = require('path');
 var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`));
 var child_process = require(`child_process`);
@@ -29,12 +31,14 @@ var optionList = [
     {
         type: String,
         name: 'arrow', alias: 'a',
-        description: 'The Arrow file to read/write'
+        multiple: true, defaultValue: [],
+        description: 'The Arrow file[s] to read/write'
     },
     {
         type: String,
         name: 'json', alias: 'j',
-        description: 'The JSON file to read/write'
+        multiple: true, defaultValue: [],
+        description: 'The JSON file[s] to read/write'
     }
 ];
 
@@ -66,20 +70,60 @@ function print_usage() {
     process.exit(1);
 }
 
-if (!argv.arrow || !argv.json || !argv.mode) {
+let jsonPaths = argv.json;
+let arrowPaths = argv.arrow;
+
+if (!argv.mode) {
+    return print_usage();
+}
+
+let mode = argv.mode.toUpperCase();
+if (mode === 'VALIDATE' && !jsonPaths.length) {
+    jsonPaths = glob.sync(path.resolve(__dirname, `../test/data/json/`, `*.json`));
+    if (!arrowPaths.length) {
+        [jsonPaths, arrowPaths] = jsonPaths.reduce(([jsonPaths, arrowPaths], jsonPath) => {
+            const { name } = path.parse(jsonPath);
+            for (const source of ['cpp', 'java']) {
+                for (const format of ['file', 'stream']) {
+                    const arrowPath = path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`);
+                    if (fs.existsSync(arrowPath)) {
+                        jsonPaths.push(jsonPath);
+                        arrowPaths.push(arrowPath);
+                        console.log('-j', jsonPath, '-a', arrowPath, '\\');
+                    }
+                }
+            }
+            return [jsonPaths, arrowPaths];
+        }, [[], []]);
+    }
+} else if (!jsonPaths.length) {
     return print_usage();
 }
 
-switch (argv.mode.toUpperCase()) {
+switch (mode) {
     case 'VALIDATE':
+        const args = [`test`, `-i`].concat(argv._unknown || []);
+        jsonPaths.forEach((p, i) => {
+            args.push('-j', p, '-a', arrowPaths[i]);
+        });
         child_process.spawnSync(
-            gulp,
-            [`test`, `-i`].concat(process.argv.slice(2)),
+            gulp, args,
             {
                 cwd: path.resolve(__dirname, '..'),
                 stdio: ['ignore', 'inherit', 'inherit']
             }
         );
+        // for (let i = -1, n = jsonPaths.length; ++i < n;) {
+        //     const jsonPath = jsonPaths[i];
+        //     const arrowPath = arrowPaths[i];
+        //     child_process.spawnSync(
+        //         gulp, args.concat(['-j', jsonPath, '-a', arrowPath]),
+        //         {
+        //             cwd: path.resolve(__dirname, '..'),
+        //             stdio: ['ignore', 'inherit', 'inherit']
+        //         }
+        //     );
+        // }
         break;
     default:
         print_usage();
diff --git a/js/gulp/argv.js b/js/gulp/argv.js
index 6f80912e97e52..8a83820c1fe59 100644
--- a/js/gulp/argv.js
+++ b/js/gulp/argv.js
@@ -15,6 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+const fs = require('fs');
+const glob = require('glob');
+const path = require('path');
+
 const argv = require(`command-line-args`)([
     { name: `all`, type: Boolean },
     { name: 'update', alias: 'u', type: Boolean },
@@ -22,13 +26,11 @@ const argv = require(`command-line-args`)([
     { name: `target`, type: String, defaultValue: `` },
     { name: `module`, type: String, defaultValue: `` },
     { name: `coverage`, type: Boolean, defaultValue: false },
-    { name: `json_file`, alias: `j`, type: String, defaultValue: null },
-    { name: `arrow_file`, alias: `a`, type: String, defaultValue: null },
     { name: `integration`, alias: `i`, type: Boolean, defaultValue: false },
     { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
     { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] },
-    { name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] },
-    { name: `formats`, alias: `f`, type: String, multiple: true, defaultValue: [`file`, `stream`] },
+    { name: `json_files`, alias: `j`, type: String, multiple: true, defaultValue: [] },
+    { name: `arrow_files`, alias: `a`, type: String, multiple: true, defaultValue: [] },
 ], { partial: true });
 
 const { targets, modules } = argv;
@@ -38,4 +40,25 @@ argv.module && !modules.length && modules.push(argv.module);
 (argv.all || !targets.length) && targets.push(`all`);
 (argv.all || !modules.length) && modules.push(`all`);
 
+if (argv.coverage && (!argv.json_files || !argv.json_files.length)) {
+
+    let [jsonPaths, arrowPaths] = glob
+        .sync(path.resolve(__dirname, `../test/data/json/`, `*.json`))
+        .reduce((paths, jsonPath) => {
+            const { name } = path.parse(jsonPath);
+            const [jsonPaths, arrowPaths] = paths;
+            ['cpp', 'java'].forEach((source) => ['file', 'stream'].forEach((format) => {
+                const arrowPath = path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`);
+                if (fs.existsSync(arrowPath)) {
+                    jsonPaths.push(jsonPath);
+                    arrowPaths.push(arrowPath);
+                }
+            }));
+            return paths;
+        }, [[], []]);
+
+    argv.json_files = jsonPaths;
+    argv.arrow_files = arrowPaths;
+}
+
 module.exports = { argv, targets, modules };
diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js
index 1bd872fd3044a..0b2ef1b846b81 100644
--- a/js/gulp/closure-task.js
+++ b/js/gulp/closure-task.js
@@ -36,7 +36,7 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
     const src = targetDir(target, `cls`);
     const out = targetDir(target, format);
     const entry = path.join(src, mainExport);
-    const externs = path.join(src, `${mainExport}.externs`);
+    const externs = path.join(`src/Arrow.externs.js`);
     return observableFromStreams(
         gulp.src([
 /*   external libs first --> */ `node_modules/tslib/package.json`,
@@ -46,7 +46,6 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
                                 `node_modules/text-encoding-utf-8/package.json`,
                                 `node_modules/text-encoding-utf-8/src/encoding.js`,
 /*    then sources globs --> */ `${src}/**/*.js`,
-/* and exclusions last -->  */ `!${src}/Arrow.externs.js`,
         ], { base: `./` }),
         sourcemaps.init(),
         closureCompiler(createClosureArgs(entry, externs)),
@@ -60,14 +59,15 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
 }))({});
 
 const createClosureArgs = (entry, externs) => ({
+    externs,
     third_party: true,
     warning_level: `QUIET`,
     dependency_mode: `STRICT`,
     rewrite_polyfills: false,
-    externs: `${externs}.js`,
     entry_point: `${entry}.js`,
     module_resolution: `NODE`,
-    // formatting: `PRETTY_PRINT`, debug: true,
+    // formatting: `PRETTY_PRINT`,
+    // debug: true,
     compilation_level: `ADVANCED`,
     allow_method_call_decomposing: true,
     package_json_entry_names: `module,jsnext:main,main`,
diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index 2976d0ad45d09..c42b3fc323321 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -45,10 +45,11 @@ const createMainPackageJson = (target, format) => (orig) => ({
     ...createTypeScriptPackageJson(target, format)(orig),
     name: npmPkgName,
     main: mainExport,
+    types: `${mainExport}.d.ts`,
     module: `${mainExport}.mjs`,
     dist: `${mainExport}.es5.min.js`,
     [`dist:es2015`]: `${mainExport}.es2015.min.js`,
-    [`@std/esm`]: { esm: `mjs` }
+    [`@std/esm`]: { esm: `mjs`, warnings: false, sourceMap: true }
 });
   
 const createTypeScriptPackageJson = (target, format) => (orig) => ({
@@ -63,18 +64,20 @@ const createTypeScriptPackageJson = (target, format) => (orig) => ({
   
 const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
     conditionallyAddStandardESMEntry(target, format)(
-      packageJSONFields.reduce(
-        (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }),
-        { name: `${npmOrgName}/${packageName(target, format)}`,
-          version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`,
-          dist: undefined, [`dist:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined }
-      )
+        packageJSONFields.reduce(
+            (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }),
+            {
+                name: `${npmOrgName}/${packageName(target, format)}`,
+                version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`,
+                dist: undefined, [`dist:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined
+            }
+        )
     )
 );
   
 const conditionallyAddStandardESMEntry = (target, format) => (packageJSON) => (
-    format !== `esm`
-      ? packageJSON
-      : { ...packageJSON, [`@std/esm`]: { esm: `js` } }
+    format !== `esm` && format !== `cls`
+        ?      packageJSON
+        : { ...packageJSON, [`@std/esm`]: { esm: `js`, warnings: false, sourceMap: true } }
 );
   
\ No newline at end of file
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
index ab280b092635c..7f655548eb8ef 100644
--- a/js/gulp/test-task.js
+++ b/js/gulp/test-task.js
@@ -44,15 +44,15 @@ const testOptions = {
 const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) {
     const opts = { ...testOptions };
     const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv];
-    args.push(`test/${argv.integration ? `integration/*` : `unit/*`}`);
+    if (!argv.coverage) {
+        args.push(`test/${argv.integration ? `integration/*` : `unit/*`}`);
+    }
     opts.env = { ...opts.env,
         TEST_TARGET: target,
         TEST_MODULE: format,
-        JSON_PATH: argv.json_file,
-        ARROW_PATH: argv.arrow_file,
         TEST_TS_SOURCE: !!argv.coverage,
-        TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]),
-        TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]),
+        JSON_PATHS: JSON.stringify(Array.isArray(argv.json_files) ? argv.json_files : [argv.json_files]),
+        ARROW_PATHS: JSON.stringify(Array.isArray(argv.arrow_files) ? argv.arrow_files : [argv.arrow_files]),
     };
     return !debug ?
         child_process.spawn(jest, args, opts) :
diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js
index 8b755cf7f1624..c42357adb2f75 100644
--- a/js/gulp/typescript-task.js
+++ b/js/gulp/typescript-task.js
@@ -34,7 +34,7 @@ const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target
     const tsProject = ts.createProject(path.join(`tsconfig`, tsconfigFile), { typescript: require(`typescript`) });
     const { stream: { js, dts } } = observableFromStreams(
       tsProject.src(), sourcemaps.init(),
-      tsProject(ts.reporter.fullReporter(true))
+      tsProject(ts.reporter.defaultReporter())
     );
     const writeDTypes = observableFromStreams(dts, gulp.dest(out));
     const writeJS = observableFromStreams(js, sourcemaps.write(), gulp.dest(out));
@@ -52,12 +52,12 @@ function maybeCopyRawJSArrowFormatFiles(target, format) {
         return Observable.empty();
     }
     return Observable.defer(async () => {
-        const outFormatDir = path.join(targetDir(target, format), `format`, `fb`);
+        const outFormatDir = path.join(targetDir(target, format), `fb`);
         await del(path.join(outFormatDir, '*.js'));
         await observableFromStreams(
-            gulp.src(path.join(`src`, `format`, `fb`, `*_generated.js`)),
+            gulp.src(path.join(`src`, `fb`, `*_generated.js`)),
             gulpRename((p) => { p.basename = p.basename.replace(`_generated`, ``); }),
             gulp.dest(outFormatDir)
         ).toPromise();
     });
-}
\ No newline at end of file
+}
diff --git a/js/gulp/uglify-task.js b/js/gulp/uglify-task.js
index 5c605cb7882bd..9ba3e41a16f41 100644
--- a/js/gulp/uglify-task.js
+++ b/js/gulp/uglify-task.js
@@ -29,7 +29,7 @@ const webpack = require(`webpack`);
 const { memoizeTask } = require('./memoize-task');
 const { Observable, ReplaySubject } = require('rxjs');
 const UglifyJSPlugin = require(`uglifyjs-webpack-plugin`);
-const esmRequire = require(`@std/esm`)(module, { cjs: true, esm: `js` });
+const esmRequire = require(`@std/esm`)(module, { cjs: true, esm: `js`, warnings: false });
 
 const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJS(target, format) {
 
@@ -84,11 +84,20 @@ module.exports = uglifyTask;
 module.exports.uglifyTask = uglifyTask;
 
 const reservePublicNames = ((ESKeywords) => function reservePublicNames(target, format) {
-    const publicModulePath = `../${targetDir(target, format)}/${mainExport}.js`;
-    return [
-        ...ESKeywords,
-        ...reserveExportedNames(esmRequire(publicModulePath))
+    const src = targetDir(target, format);
+    const publicModulePaths = [
+        `../${src}/data.js`,
+        `../${src}/type.js`,
+        `../${src}/table.js`,
+        `../${src}/vector.js`,
+        `../${src}/util/int.js`,
+        `../${src}/predicate.js`,
+        `../${src}/recordbatch.js`,
+        `../${src}/${mainExport}.js`,
     ];
+    return publicModulePaths.reduce((keywords, publicModulePath) => [
+        ...keywords, ...reserveExportedNames(esmRequire(publicModulePath, { warnings: false }))
+    ], [...ESKeywords]);
 })(ESKeywords);
 
 // Reflect on the Arrow modules to come up with a list of keys to save from Uglify's
@@ -104,8 +113,8 @@ const reserveExportedNames = (entryModule) => (
         .map((name) => [name, entryModule[name]])
         .reduce((reserved, [name, value]) => {
             const fn = function() {};
-            const ownKeys = value && Object.getOwnPropertyNames(value) || [];
-            const protoKeys = typeof value === `function` && Object.getOwnPropertyNames(value.prototype) || [];
+            const ownKeys = value && typeof value === 'object' && Object.getOwnPropertyNames(value) || [];
+            const protoKeys = typeof value === `function` && Object.getOwnPropertyNames(value.prototype || {}) || [];
             const publicNames = [...ownKeys, ...protoKeys].filter((x) => x !== `default` && x !== `undefined` && !(x in fn));
             return [...reserved, name, ...publicNames];
         }, []
diff --git a/js/gulp/util.js b/js/gulp/util.js
index ba6ebece51bba..f35a447e70830 100644
--- a/js/gulp/util.js
+++ b/js/gulp/util.js
@@ -87,7 +87,7 @@ const ESKeywords = [
     // EventTarget
     `addListener`, `removeListener`, `addEventListener`, `removeEventListener`,
     // Arrow properties
-    `low`, `high`, `data`, `index`, `field`, `validity`, `columns`, `fieldNode`, `subarray`,
+    `low`, `high`, `data`, `index`, `field`, `columns`, 'numCols', 'numRows', `values`, `valueOffsets`, `nullBitmap`, `subarray`
 ];
 
 function taskName(target, format) {
@@ -108,14 +108,13 @@ function targetDir(target, format) {
 
 function logAndDie(e) {
     if (e) {
-        console.error(e);
         process.exit(1);
     }
 }
 
 function observableFromStreams(...streams) {
-    const pumped = streams.length <= 1 ? streams[0]
-        : pump(...streams, logAndDie);
+    if (streams.length <= 0) { return Observable.empty(); }
+    const pumped = streams.length <= 1 ? streams[0] : pump(...streams, logAndDie);
     const fromEvent = Observable.fromEvent.bind(null, pumped);
     const streamObs = fromEvent(`data`)
                .merge(fromEvent(`error`).flatMap((e) => Observable.throw(e)))
diff --git a/js/package.json b/js/package.json
index d68e7a6279e61..1c8b23604ab85 100644
--- a/js/package.json
+++ b/js/package.json
@@ -3,7 +3,7 @@
   "name": "apache-arrow",
   "description": "Apache Arrow columnar in-memory format",
   "bin": {
-    "arrow2csv": "bin/arrow2csv"
+    "arrow2csv": "bin/arrow2csv.js"
   },
   "scripts": {
     "lerna": "lerna",
@@ -12,6 +12,8 @@
     "clean": "gulp clean",
     "debug": "gulp debug",
     "perf": "node ./perf/index.js",
+    "test:integration": "node ./bin/integration.js --mode validate",
+    "create:perfdata": "python ./test/data/tables/generate.py ./test/data/tables/tracks.arrow",
     "release": "./npm-release.sh",
     "clean:all": "run-p clean clean:testdata",
     "clean:testdata": "gulp clean:testdata",
@@ -51,18 +53,18 @@
   ],
   "dependencies": {
     "@types/text-encoding-utf-8": "1.0.1",
-    "command-line-args": "4.0.7",
-    "command-line-usage": "4.0.2",
+    "command-line-args": "5.0.1",
+    "command-line-usage": "4.1.0",
     "flatbuffers": "trxcllnt/flatbuffers-esm",
     "json-bignum": "0.0.3",
     "text-encoding-utf-8": "^1.0.2",
-    "tslib": "1.8.1"
+    "tslib": "1.9.0"
   },
   "devDependencies": {
-    "@std/esm": "0.19.1",
+    "@std/esm": "0.19.7",
     "@types/flatbuffers": "1.6.5",
-    "@types/glob": "5.0.34",
-    "@types/jest": "22.0.1",
+    "@types/glob": "5.0.35",
+    "@types/jest": "22.1.0",
     "@types/node": "9.3.0",
     "ast-types": "0.10.1",
     "benchmark": "2.1.4",
@@ -77,13 +79,13 @@
     "gulp-rename": "1.2.2",
     "gulp-sourcemaps": "2.6.3",
     "gulp-transform-js-ast": "1.0.2",
-    "gulp-typescript": "3.2.3",
+    "gulp-typescript": "3.2.4",
     "ix": "2.3.4",
-    "jest": "22.0.5",
+    "jest": "22.1.4",
     "jest-environment-node-debug": "2.0.0",
     "json": "9.0.6",
-    "lerna": "2.6.0",
-    "lint-staged": "6.0.0",
+    "lerna": "2.7.1",
+    "lint-staged": "6.0.1",
     "merge2": "1.2.1",
     "mkdirp": "0.5.1",
     "npm-run-all": "4.1.2",
@@ -100,6 +102,9 @@
     "webpack": "3.10.0",
     "xml2js": "0.4.19"
   },
+  "@std/esm": {
+    "warnings": false
+  },
   "lint-staged": {
     "*.@(ts)": [
       "tslint --fix",
@@ -126,7 +131,7 @@
       "lcov"
     ],
     "coveragePathIgnorePatterns": [
-      "format\\/(File|Message|Schema|Tensor)_generated\\.(js|ts)$",
+      "fb\\/(File|Message|Schema|Tensor)_generated\\.(js|ts)$",
       "test\\/.*\\.(ts|tsx|js)$",
       "/node_modules/"
     ],
@@ -135,7 +140,8 @@
       ".(js|jsx)": "./node_modules/babel-jest/build/index.js"
     },
     "transformIgnorePatterns": [
-      "/node_modules/", "/(es2015|esnext)\/umd/"
+      "/node_modules/",
+      "/(es2015|esnext)/umd/"
     ],
     "testRegex": "(.*(-|\\.)(test|spec)s?)\\.(ts|tsx|js)$"
   }
diff --git a/js/perf/index.js b/js/perf/index.js
index 9eac40e64ac71..42cb6abe29cb7 100644
--- a/js/perf/index.js
+++ b/js/perf/index.js
@@ -16,29 +16,41 @@
 // under the License.
 
 // Use the ES5 UMD target as perf baseline
-// const { Table, readVectors } = require('../targets/es5/umd');
-// const { Table, readVectors } = require('../targets/es5/cjs');
-const { Table, readVectors } = require('../targets/es2015/umd');
-// const { Table, readVectors } = require('../targets/es2015/cjs');
+// const { predicate, Table, read: readBatches } = require('../targets/es5/umd');
+// const { predicate, Table, read: readBatches } = require('../targets/es5/cjs');
+// const { predicate, Table, read: readBatches } = require('../targets/es2015/umd');
+const { predicate, Table, read: readBatches } = require('../targets/es2015/cjs');
+const { col } = predicate;
 
-const config = require('./config');
 const Benchmark = require('benchmark');
 
 const suites = [];
 
-for (let { name, buffers} of config) {
-    const parseSuite = new Benchmark.Suite(`Parse ${name}`, { async: true });
-    const sliceSuite = new Benchmark.Suite(`Slice ${name} vectors`, { async: true });
-    const iterateSuite = new Benchmark.Suite(`Iterate ${name} vectors`, { async: true });
-    const getByIndexSuite = new Benchmark.Suite(`Get ${name} values by index`, { async: true });
-    parseSuite.add(createFromTableTest(name, buffers));
-    parseSuite.add(createReadVectorsTest(name, buffers));
-    for (const vector of Table.from(buffers).columns) {
-        sliceSuite.add(createSliceTest(vector));
-        iterateSuite.add(createIterateTest(vector));
-        getByIndexSuite.add(createGetByIndexTest(vector));
-    }
-    suites.push(getByIndexSuite, iterateSuite, sliceSuite, parseSuite);
+for (let { name, buffers } of require('./table_config')) {
+    const parseSuiteName = `Parse "${name}"`;
+    const sliceSuiteName = `Slice "${name}" vectors`;
+    const iterateSuiteName = `Iterate "${name}" vectors`;
+    const getByIndexSuiteName = `Get "${name}" values by index`;
+    const sliceToArraySuiteName = `Slice toArray "${name}" vectors`;
+    suites.push(createTestSuite(parseSuiteName, createFromTableTest(name, buffers)));
+    suites.push(createTestSuite(parseSuiteName, createReadBatchesTest(name, buffers)));
+    const table = Table.from(buffers), schema = table.schema;
+    suites.push(...schema.fields.map((f, i) => createTestSuite(getByIndexSuiteName, createGetByIndexTest(table.getColumnAt(i), f.name))));
+    suites.push(...schema.fields.map((f, i) => createTestSuite(iterateSuiteName, createIterateTest(table.getColumnAt(i), f.name))));
+    suites.push(...schema.fields.map((f, i) => createTestSuite(sliceToArraySuiteName, createSliceToArrayTest(table.getColumnAt(i), f.name))));
+    suites.push(...schema.fields.map((f, i) => createTestSuite(sliceSuiteName, createSliceTest(table.getColumnAt(i), f.name))));
+}
+
+for (let {name, buffers, countBys, counts} of require('./table_config')) {
+    const table = Table.from(buffers);
+
+    const dfCountBySuiteName = `DataFrame Count By "${name}"`;
+    const dfFilterCountSuiteName = `DataFrame Filter-Scan Count "${name}"`;
+    const dfDirectCountSuiteName = `DataFrame Direct Count "${name}"`;
+
+    suites.push(...countBys.map((countBy) => createTestSuite(dfCountBySuiteName, createDataFrameCountByTest(table, countBy))));
+    suites.push(...counts.map(({ col, test, value }) => createTestSuite(dfFilterCountSuiteName, createDataFrameFilterCountTest(table, col, test, value))));
+    suites.push(...counts.map(({ col, test, value }) => createTestSuite(dfDirectCountSuiteName, createDataFrameDirectCountTest(table, col, test, value))));
 }
 
 console.log('Running apache-arrow performance tests...\n');
@@ -52,7 +64,7 @@ function run() {
             var str = x.toString();
             var meanMsPerOp = Math.round(x.stats.mean * 100000)/100;
             var sliceOf60FPS = Math.round((meanMsPerOp / (1000/60)) * 100000)/1000;
-            return `${str} (avg: ${meanMsPerOp}ms, or ${sliceOf60FPS}% of a frame @ 60FPS) ${x.suffix || ''}`;
+            return `${str}\n   avg: ${meanMsPerOp}ms\n   ${sliceOf60FPS}% of a frame @ 60FPS ${x.suffix || ''}`;
         }).join('\n') + '\n');
         if (suites.length > 0) {
             setTimeout(run, 1000);
@@ -61,47 +73,60 @@ function run() {
     .run({ async: true });
 }
 
+function createTestSuite(name, test) {
+    return new Benchmark.Suite(name, { async: true }).add(test);
+}
+
 function createFromTableTest(name, buffers) {
     let table;
     return {
         async: true,
-        name: `Table.from`,
+        name: `Table.from\n`,
         fn() { table = Table.from(buffers); }
     };
 }
 
-function createReadVectorsTest(name, buffers) {
-    let vectors;
+function createReadBatchesTest(name, buffers) {
+    let recordBatch;
     return {
         async: true,
-        name: `readVectors`,
-        fn() { for (vectors of readVectors(buffers)) {} }
+        name: `readBatches\n`,
+        fn() { for (recordBatch of readBatches(buffers)) {} }
     };
 }
 
-function createSliceTest(vector) {
+function createSliceTest(vector, name) {
     let xs;
     return {
         async: true,
-        name: `name: '${vector.name}', length: ${vector.length}, type: ${vector.type}`,
+        name: `name: '${name}', length: ${vector.length}, type: ${vector.type}\n`,
         fn() { xs = vector.slice(); }
     };
 }
 
-function createIterateTest(vector) {
+function createSliceToArrayTest(vector, name) {
+    let xs;
+    return {
+        async: true,
+        name: `name: '${name}', length: ${vector.length}, type: ${vector.type}\n`,
+        fn() { xs = vector.slice().toArray(); }
+    };
+}
+
+function createIterateTest(vector, name) {
     let value;
     return {
         async: true,
-        name: `name: '${vector.name}', length: ${vector.length}, type: ${vector.type}`,
+        name: `name: '${name}', length: ${vector.length}, type: ${vector.type}\n`,
         fn() { for (value of vector) {} }
     };
 }
 
-function createGetByIndexTest(vector) {
+function createGetByIndexTest(vector, name) {
     let value;
     return {
         async: true,
-        name: `name: '${vector.name}', length: ${vector.length}, type: ${vector.type}`,
+        name: `name: '${name}', length: ${vector.length}, type: ${vector.type}\n`,
         fn() {
             for (let i = -1, n = vector.length; ++i < n;) {
                 value = vector.get(i);
@@ -109,3 +134,80 @@ function createGetByIndexTest(vector) {
         }
     };
 }
+
+function createDataFrameDirectCountTest(table, column, test, value) {
+    let sum, colidx = table.schema.fields.findIndex((c)=>c.name === column);
+
+    if (test == 'gteq') {
+        op = function () {
+            sum = 0;
+            let batches = table.batches;
+            let numBatches = batches.length;
+            for (let batchIndex = -1; ++batchIndex < numBatches;) {
+                // load batches
+                const batch = batches[batchIndex];
+                const vector = batch.getChildAt(colidx);
+                // yield all indices
+                for (let index = -1; ++index < batch.length;) {
+                    sum += (vector.get(index) >= value);
+                }
+            }
+        }
+    } else if (test == 'eq') {
+        op = function() {
+            sum = 0;
+            let batches = table.batches;
+            let numBatches = batches.length;
+            for (let batchIndex = -1; ++batchIndex < numBatches;) {
+                // load batches
+                const batch = batches[batchIndex];
+                const vector = batch.getChildAt(colidx);
+                // yield all indices
+                for (let index = -1; ++index < batch.length;) {
+                    sum += (vector.get(index) === value);
+                }
+            }
+        }
+    } else {
+        throw new Error(`Unrecognized test "${test}"`);
+    }
+
+    return {
+        async: true,
+        name: `name: '${column}', length: ${table.length}, type: ${table.getColumnAt(colidx).type}, test: ${test}, value: ${value}\n`,
+        fn: op
+    };
+}
+
+function createDataFrameCountByTest(table, column) {
+    let colidx = table.schema.fields.findIndex((c)=> c.name === column);
+
+    return {
+        async: true,
+        name: `name: '${column}', length: ${table.length}, type: ${table.getColumnAt(colidx).type}\n`,
+        fn() {
+            table.countBy(column);
+        }
+    };
+}
+
+function createDataFrameFilterCountTest(table, column, test, value) {
+    let colidx = table.schema.fields.findIndex((c)=> c.name === column);
+    let df;
+
+    if (test == 'gteq') {
+        df = table.filter(col(column).gteq(value));
+    } else if (test == 'eq') {
+        df = table.filter(col(column).eq(value));
+    } else {
+        throw new Error(`Unrecognized test "${test}"`);
+    }
+
+    return {
+        async: true,
+        name: `name: '${column}', length: ${table.length}, type: ${table.getColumnAt(colidx).type}, test: ${test}, value: ${value}\n`,
+        fn() {
+            df.count();
+        }
+    };
+}
diff --git a/js/src/vector/utf8.ts b/js/perf/table_config.js
similarity index 51%
rename from js/src/vector/utf8.ts
rename to js/perf/table_config.js
index ba875cf333fe7..e3c332c870f38 100644
--- a/js/src/vector/utf8.ts
+++ b/js/perf/table_config.js
@@ -15,26 +15,34 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Vector } from './vector';
-import { VirtualVector } from './virtual';
-import { TextDecoder } from 'text-encoding-utf-8';
+const fs = require('fs');
+const path = require('path');
+const glob = require('glob');
 
-const decoder = new TextDecoder('utf-8');
+const config = [];
+const filenames = glob.sync(path.resolve(__dirname, `../test/data/tables/`, `*.arrow`));
 
-export class Utf8Vector extends Vector<string> {
-    readonly values: Vector<Uint8Array | null>;
-    constructor(argv: { values: Vector<Uint8Array | null> }) {
-        super();
-        this.values = argv.values;
-    }
-    get(index: number) {
-        const chars = this.getCodePoints(index);
-        return chars ? decoder.decode(chars) : null;
-    }
-    getCodePoints(index: number) {
-        return this.values.get(index);
-    }
-    concat(...vectors: Vector<string>[]): Vector<string> {
-        return new VirtualVector(Array, this, ...vectors);
+countBys = {
+    "tracks": ['origin', 'destination']
+}
+counts = {
+    "tracks": [
+        {col: 'lat',    test: 'gteq', value: 0        },
+        {col: 'lng',    test: 'gteq', value: 0        },
+        {col: 'origin', test:   'eq', value: 'Seattle'},
+    ]
+}
+
+for (const filename of filenames) {
+    const { name } = path.parse(filename);
+    if (name in counts) {
+        config.push({
+            name,
+            buffers: [fs.readFileSync(filename)],
+            countBys: countBys[name],
+            counts: counts[name],
+        });
     }
 }
+
+module.exports = config;
diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
new file mode 100644
index 0000000000000..438ac8b736cac
--- /dev/null
+++ b/js/src/Arrow.externs.js
@@ -0,0 +1,680 @@
+// @ts-nocheck
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/* tslint:disable */
+
+/**
+ * @fileoverview Closure Compiler externs for Arrow
+ * @externs
+ * @suppress {duplicate,checkTypes}
+ */
+/** @type {symbol} */
+Symbol.iterator;
+/** @type {symbol} */
+Symbol.asyncIterator;
+
+var Table = function() {};
+/** @type {?} */
+Table.from = function() {};
+/** @type {?} */
+Table.fromAsync = function() {};
+/** @type {?} */
+Table.empty = function() {};
+/** @type {?} */
+Table.prototype.schema;
+/** @type {?} */
+Table.prototype.length;
+/** @type {?} */
+Table.prototype.numCols;
+/** @type {?} */
+Table.prototype.get;
+/** @type {?} */
+Table.prototype.getColumn;
+/** @type {?} */
+Table.prototype.getColumnAt;
+/** @type {?} */
+Table.prototype.getColumnIndex;
+/** @type {?} */
+Table.prototype.toArray;
+/** @type {?} */
+Table.prototype.select;
+/** @type {?} */
+Table.prototype.rowsToString;
+/** @type {?} */
+Table.prototype.batchesUnion;
+/** @type {?} */
+Table.prototype.batches;
+/** @type {?} */
+Table.prototype.countBy;
+/** @type {?} */
+Table.prototype.scan;
+
+var CountByResult = function() {};
+/** @type {?} */
+CountByResult.prototype.asJSON;
+
+var col = function () {};
+var lit = function () {};
+
+var Value = function() {};
+/** @type {?} */
+Value.prototype.gteq;
+/** @type {?} */
+Value.prototype.lteq;
+/** @type {?} */
+Value.prototype.eq;
+
+var Col = function() {};
+/** @type {?} */
+Col.prototype.bind;
+var Or = function() {};
+var And = function() {};
+var GTeq = function () {};
+/** @type {?} */
+GTeq.prototype.and;
+/** @type {?} */
+GTeq.prototype.or;
+var LTeq = function () {};
+/** @type {?} */
+LTeq.prototype.and;
+/** @type {?} */
+LTeq.prototype.or;
+var Equals = function () {};
+/** @type {?} */
+Equals.prototype.and;
+/** @type {?} */
+Equals.prototype.or;
+var Predicate = function() {};
+/** @type {?} */
+Predicate.prototype.bind;
+/** @type {?} */
+Predicate.prototype.and;
+/** @type {?} */
+Predicate.prototype.or;
+/** @type {?} */
+Predicate.prototype.ands;
+var Literal = function() {};
+
+var TableToStringIterator = function() {};
+/** @type {?} */
+TableToStringIterator.prototype.pipe;
+
+var RecordBatch = function() {};
+/** @type {?} */
+RecordBatch.from = function() {};
+/** @type {?} */
+RecordBatch.prototype.numCols;
+/** @type {?} */
+RecordBatch.prototype.length;
+/** @type {?} */
+RecordBatch.prototype.schema;
+/** @type {?} */
+RecordBatch.prototype.columns;
+/** @type {?} */
+RecordBatch.prototype.select;
+
+var Vector = function() {};
+/** @type {?} */
+Vector.create = function() {};
+/** @type {?} */
+Vector.prototype.data;
+/** @type {?} */
+Vector.prototype.type;
+/** @type {?} */
+Vector.prototype.length;
+/** @type {?} */
+Vector.prototype.nullCount;
+/** @type {?} */
+Vector.prototype.nullBitmap;
+/** @type {?} */
+Vector.prototype.isValid;
+/** @type {?} */
+Vector.prototype.get;
+/** @type {?} */
+Vector.prototype.set;
+/** @type {?} */
+Vector.prototype.toArray;
+/** @type {?} */
+Vector.prototype.concat;
+/** @type {?} */
+Vector.prototype.slice;
+/** @type {?} */
+Vector.prototype.acceptTypeVisitor;
+
+var BaseInt64 = function() {};
+/** @type {?} */
+BaseInt64.prototype.lessThan;
+/** @type {?} */
+BaseInt64.prototype.equals;
+/** @type {?} */
+BaseInt64.prototype.greaterThan;
+/** @type {?} */
+BaseInt64.prototype.hex;
+
+var Uint64 = function() {};
+/** @type {?} */
+Uint64.add = function() {};
+/** @type {?} */
+Uint64.multiply = function() {};
+/** @type {?} */
+Uint64.prototype.times;
+/** @type {?} */
+Uint64.prototype.plus
+
+var Int64 = function() {};
+/** @type {?} */
+Int64.add = function() {};
+/** @type {?} */
+Int64.multiply = function() {};
+/** @type {?} */
+Int64.fromString = function() {};
+/** @type {?} */
+Int64.prototype.negate
+/** @type {?} */
+Int64.prototype.times
+/** @type {?} */
+Int64.prototype.plus
+/** @type {?} */
+Int64.prototype.lessThan
+
+var Int128 = function() {};
+/** @type {?} */
+Int128.add = function() {};
+/** @type {?} */
+Int128.multiply = function() {};
+/** @type {?} */
+Int128.fromString = function() {};
+/** @type {?} */
+Int128.prototype.negate
+/** @type {?} */
+Int128.prototype.times
+/** @type {?} */
+Int128.prototype.plus
+/** @type {?} */
+Int128.prototype.hex
+
+var Type = function() {};
+/** @type {?} */
+Type.NONE = function() {};
+/** @type {?} */
+Type.Null = function() {};
+/** @type {?} */
+Type.Int = function() {};
+/** @type {?} */
+Type.Float = function() {};
+/** @type {?} */
+Type.Binary = function() {};
+/** @type {?} */
+Type.Utf8 = function() {};
+/** @type {?} */
+Type.Bool = function() {};
+/** @type {?} */
+Type.Decimal = function() {};
+/** @type {?} */
+Type.Date = function() {};
+/** @type {?} */
+Type.Time = function() {};
+/** @type {?} */
+Type.Timestamp = function() {};
+/** @type {?} */
+Type.Interval = function() {};
+/** @type {?} */
+Type.List = function() {};
+/** @type {?} */
+Type.Struct = function() {};
+/** @type {?} */
+Type.Union = function() {};
+/** @type {?} */
+Type.FixedSizeBinary = function() {};
+/** @type {?} */
+Type.FixedSizeList = function() {};
+/** @type {?} */
+Type.Map = function() {};
+/** @type {?} */
+Type.Dictionary = function() {};
+/** @type {?} */
+Type.DenseUnion = function() {};
+/** @type {?} */
+Type.SparseUnion = function() {};
+
+var DataType = function() {};
+/** @type {?} */
+DataType.isNull = function() {};
+/** @type {?} */
+DataType.isInt = function() {};
+/** @type {?} */
+DataType.isFloat = function() {};
+/** @type {?} */
+DataType.isBinary = function() {};
+/** @type {?} */
+DataType.isUtf8 = function() {};
+/** @type {?} */
+DataType.isBool = function() {};
+/** @type {?} */
+DataType.isDecimal = function() {};
+/** @type {?} */
+DataType.isDate = function() {};
+/** @type {?} */
+DataType.isTime = function() {};
+/** @type {?} */
+DataType.isTimestamp = function() {};
+/** @type {?} */
+DataType.isInterval = function() {};
+/** @type {?} */
+DataType.isList = function() {};
+/** @type {?} */
+DataType.isStruct = function() {};
+/** @type {?} */
+DataType.isUnion = function() {};
+/** @type {?} */
+DataType.isDenseUnion = function() {};
+/** @type {?} */
+DataType.isSparseUnion = function() {};
+/** @type {?} */
+DataType.isFixedSizeBinary = function() {};
+/** @type {?} */
+DataType.isFixedSizeList = function() {};
+/** @type {?} */
+DataType.isMap = function() {};
+/** @type {?} */
+DataType.isDictionary = function() {};
+/** @type {?} */
+DataType.prototype.ArrayType;
+
+var Schema = function() {};
+/** @type {?} */
+Schema.from = function() {};
+/** @type {?} */
+Schema.prototype.fields;
+/** @type {?} */
+Schema.prototype.version;
+/** @type {?} */
+Schema.prototype.metadata;
+/** @type {?} */
+Schema.prototype.dictionaries;
+/** @type {?} */
+Schema.prototype.select;
+var Field = function() {};
+/** @type {?} */
+Field.prototype.name;
+/** @type {?} */
+Field.prototype.type;
+/** @type {?} */
+Field.prototype.nullable;
+/** @type {?} */
+Field.prototype.metadata;
+var Null = function() {};
+var Int8 = function() {};
+var Int16 = function() {};
+var Int32 = function() {};
+var Int64 = function() {};
+var Uint8 = function() {};
+var Uint16 = function() {};
+var Uint32 = function() {};
+var Uint64 = function() {};
+var Float16 = function() {};
+var Float32 = function() {};
+var Float64 = function() {};
+var Binary = function() {};
+var Utf8 = function() {};
+var Bool = function() {};
+var Decimal = function() {};
+var Date_ = function() {};
+var Time = function() {};
+var Timestamp = function() {};
+var Interval = function() {};
+var List = function() {};
+var Struct = function() {};
+var Union = function() {};
+var DenseUnion = function() {};
+var SparseUnion = function() {};
+var FixedSizeBinary = function() {};
+var FixedSizeList = function() {};
+var Map_ = function() {};
+var Dictionary = function() {};
+
+var BaseData = function() {};
+/** @type {?} */
+BaseData.prototype.type;
+/** @type {?} */
+BaseData.prototype.clone;
+/** @type {?} */
+BaseData.prototype.slice;
+/** @type {?} */
+BaseData.prototype.length;
+/** @type {?} */
+BaseData.prototype.offset;
+/** @type {?} */
+BaseData.prototype.typeId;
+/** @type {?} */
+BaseData.prototype.childData;
+/** @type {?} */
+BaseData.prototype.nullBitmap;
+/** @type {?} */
+BaseData.prototype.nullCount;
+
+var BoolData = function() {};
+var NestedData = function() {};
+var SparseUnionData = function() {};
+var ChunkedData = function() {};
+
+var FlatData = function() {};
+/** @type {?} */
+FlatData.prototype.values;
+
+var FlatListData = function() {};
+/** @type {?} */
+FlatListData.prototype.values;
+/** @type {?} */
+FlatListData.prototype.valueOffsets;
+
+var DictionaryData = function() {};
+/** @type {?} */
+DictionaryData.prototype.indicies;
+/** @type {?} */
+DictionaryData.prototype.dictionary;
+
+var ListData = function() {};
+/** @type {?} */
+ListData.prototype.values;
+/** @type {?} */
+ListData.prototype.valueOffsets;
+
+var UnionData = function() {};
+/** @type {?} */
+UnionData.prototype.typeIds;
+
+var DenseUnionData = function() {};
+/** @type {?} */
+DenseUnionData.prototype.valueOffsets;
+
+var ChunkedData = function() {};
+/** @type {?} */
+ChunkedData.computeOffsets = function() {};
+
+var FlatVector = function() {};
+/** @type {?} */
+FlatVector.prototype.values;
+/** @type {?} */
+FlatVector.prototype.lows;
+/** @type {?} */
+FlatVector.prototype.highs;
+/** @type {?} */
+FlatVector.prototype.asInt32;
+
+var ListVectorBase = function() {};
+/** @type {?} */
+ListVectorBase.prototype.values;
+/** @type {?} */
+ListVectorBase.prototype.valueOffsets;
+/** @type {?} */
+ListVectorBase.prototype.getValueOffset;
+/** @type {?} */
+ListVectorBase.prototype.getValueLength;
+
+var NestedVector = function() {};
+/** @type {?} */
+NestedVector.prototype.childData;
+/** @type {?} */
+NestedVector.prototype.getChildAt;
+
+var NullVector = function() {};
+var BoolVector = function() {};
+/** @type {?} */
+BoolVector.from = function() {};
+/** @type {?} */
+BoolVector.prototype.values;
+var IntVector = function() {};
+/** @type {?} */
+IntVector.from = function() {};
+
+var FloatVector = function() {};
+/** @type {?} */
+FloatVector.from = function() {};
+
+var DateVector = function() {};
+var DecimalVector = function() {};
+var TimeVector = function() {};
+var TimestampVector = function() {};
+var IntervalVector = function() {};
+var BinaryVector = function() {};
+var FixedSizeBinaryVector = function() {};
+var Utf8Vector = function() {};
+var ListVector = function() {};
+var FixedSizeListVector = function() {};
+var MapVector = function() {};
+var StructVector = function() {};
+var UnionVector = function() {};
+
+var DictionaryVector = function() {};
+/** @type {?} */
+DictionaryVector.prototype.getKey;
+/** @type {?} */
+DictionaryVector.prototype.getValue;
+
+var FlatView = function() {};
+/** @type {?} */
+FlatView.prototype.get;
+/** @type {?} */
+FlatView.prototype.clone;
+/** @type {?} */
+FlatView.prototype.isValid;
+/** @type {?} */
+FlatView.prototype.toArray;
+/** @type {?} */
+FlatView.prototype.set;
+
+var PrimitiveView = function() {};
+/** @type {?} */
+PrimitiveView.prototype.size;
+/** @type {?} */
+PrimitiveView.prototype.clone;
+
+var NullView = function() {};
+/** @type {?} */
+NullView.prototype.get;
+/** @type {?} */
+NullView.prototype.clone;
+/** @type {?} */
+NullView.prototype.isValid;
+/** @type {?} */
+NullView.prototype.toArray;
+/** @type {?} */
+NullView.prototype.set;
+
+var BoolView = function() {};
+/** @type {?} */
+BoolView.prototype.get;
+/** @type {?} */
+BoolView.prototype.clone;
+/** @type {?} */
+BoolView.prototype.isValid;
+/** @type {?} */
+BoolView.prototype.toArray;
+/** @type {?} */
+BoolView.prototype.set;
+
+var ValidityView = function() {};
+/** @type {?} */
+ValidityView.prototype.get;
+/** @type {?} */
+ValidityView.prototype.clone;
+/** @type {?} */
+ValidityView.prototype.isValid;
+/** @type {?} */
+ValidityView.prototype.toArray;
+/** @type {?} */
+ValidityView.prototype.set;
+
+var DictionaryView = function() {};
+/** @type {?} */
+DictionaryView.prototype.get;
+/** @type {?} */
+DictionaryView.prototype.clone;
+/** @type {?} */
+DictionaryView.prototype.isValid;
+/** @type {?} */
+DictionaryView.prototype.toArray;
+/** @type {?} */
+DictionaryView.prototype.set;
+
+var ListViewBase = function() {};
+/** @type {?} */
+ListViewBase.prototype.get;
+/** @type {?} */
+ListViewBase.prototype.clone;
+/** @type {?} */
+ListViewBase.prototype.isValid;
+/** @type {?} */
+ListViewBase.prototype.toArray;
+/** @type {?} */
+ListViewBase.prototype.set;
+
+var NestedView = function() {};
+/** @type {?} */
+NestedView.prototype.get;
+/** @type {?} */
+NestedView.prototype.clone;
+/** @type {?} */
+NestedView.prototype.isValid;
+/** @type {?} */
+NestedView.prototype.toArray;
+/** @type {?} */
+NestedView.prototype.set;
+
+var ChunkedView = function() {};
+/** @type {?} */
+ChunkedView.prototype.get;
+/** @type {?} */
+ChunkedView.prototype.clone;
+/** @type {?} */
+ChunkedView.prototype.isValid;
+/** @type {?} */
+ChunkedView.prototype.toArray;
+/** @type {?} */
+ChunkedView.prototype.set;
+
+var ListView = function() {};
+var FixedSizeListView = function() {};
+var BinaryView = function() {};
+var Utf8View = function() {};
+var UnionView = function() {};
+var DenseUnionView = function() {};
+var StructView = function() {};
+var MapView = function() {};
+var NullView = function() {};
+var FixedSizeView = function() {};
+var Float16View = function() {};
+var DateDayView = function() {};
+var DateMillisecondView = function() {};
+var TimestampDayView = function() {};
+var TimestampSecondView = function() {};
+var TimestampMillisecondView = function() {};
+var TimestampMicrosecondView = function() {};
+var TimestampNanosecondView = function() {};
+var IntervalYearMonthView = function() {};
+var IntervalYearView = function() {};
+var IntervalMonthView = function() {};
+
+var TypeVisitor = function() {};
+/** @type {?} */
+TypeVisitor.visitTypeInline = function() {};
+/** @type {?} */
+TypeVisitor.prototype.visit;
+/** @type {?} */
+TypeVisitor.prototype.visitMany;
+/** @type {?} */
+TypeVisitor.prototype.visitNull;
+/** @type {?} */
+TypeVisitor.prototype.visitBool;
+/** @type {?} */
+TypeVisitor.prototype.visitInt;
+/** @type {?} */
+TypeVisitor.prototype.visitFloat;
+/** @type {?} */
+TypeVisitor.prototype.visitUtf8;
+/** @type {?} */
+TypeVisitor.prototype.visitBinary;
+/** @type {?} */
+TypeVisitor.prototype.visitFixedSizeBinary;
+/** @type {?} */
+TypeVisitor.prototype.visitDate;
+/** @type {?} */
+TypeVisitor.prototype.visitTimestamp;
+/** @type {?} */
+TypeVisitor.prototype.visitTime;
+/** @type {?} */
+TypeVisitor.prototype.visitDecimal;
+/** @type {?} */
+TypeVisitor.prototype.visitList;
+/** @type {?} */
+TypeVisitor.prototype.visitStruct;
+/** @type {?} */
+TypeVisitor.prototype.visitUnion;
+/** @type {?} */
+TypeVisitor.prototype.visitDictionary;
+/** @type {?} */
+TypeVisitor.prototype.visitInterval;
+/** @type {?} */
+TypeVisitor.prototype.visitFixedSizeList;
+/** @type {?} */
+TypeVisitor.prototype.visitMap;
+
+var VectorVisitor = function() {};
+/** @type {?} */
+VectorVisitor.visitTypeInline = function() {};
+/** @type {?} */
+VectorVisitor.prototype.visit;
+/** @type {?} */
+VectorVisitor.prototype.visitMany;
+/** @type {?} */
+VectorVisitor.prototype.visitNullVector;
+/** @type {?} */
+VectorVisitor.prototype.visitBoolVector;
+/** @type {?} */
+VectorVisitor.prototype.visitIntVector;
+/** @type {?} */
+VectorVisitor.prototype.visitFloatVector;
+/** @type {?} */
+VectorVisitor.prototype.visitUtf8Vector;
+/** @type {?} */
+VectorVisitor.prototype.visitBinaryVector;
+/** @type {?} */
+VectorVisitor.prototype.visitFixedSizeBinaryVector;
+/** @type {?} */
+VectorVisitor.prototype.visitDateVector;
+/** @type {?} */
+VectorVisitor.prototype.visitTimestampVector;
+/** @type {?} */
+VectorVisitor.prototype.visitTimeVector;
+/** @type {?} */
+VectorVisitor.prototype.visitDecimalVector;
+/** @type {?} */
+VectorVisitor.prototype.visitListVector;
+/** @type {?} */
+VectorVisitor.prototype.visitStructVector;
+/** @type {?} */
+VectorVisitor.prototype.visitUnionVector;
+/** @type {?} */
+VectorVisitor.prototype.visitDictionaryVector;
+/** @type {?} */
+VectorVisitor.prototype.visitIntervalVector;
+/** @type {?} */
+VectorVisitor.prototype.visitFixedSizeListVector;
+/** @type {?} */
+VectorVisitor.prototype.visitMapVector;
\ No newline at end of file
diff --git a/js/src/Arrow.externs.ts b/js/src/Arrow.externs.ts
deleted file mode 100644
index c23930271183d..0000000000000
--- a/js/src/Arrow.externs.ts
+++ /dev/null
@@ -1,84 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-/**
- * @fileoverview Closure Compiler externs for Arrow
- * @externs
- * @suppress {duplicate,checkTypes}
- */
-/** @type {symbol} */
-Symbol.iterator;
-/** @type {symbol} */
-Symbol.asyncIterator;
-
-let RowVector = function() {};
-/** @type {?} */
-RowVector.prototype.toJSON;
-/** @type {?} */
-RowVector.prototype.toArray;
-/** @type {?} */
-RowVector.prototype.toObject;
-/** @type {?} */
-RowVector.prototype.toString;
-
-let Table = function() {};
-/** @type {?} */
-(<any> Table).from;
-/** @type {?} */
-Table.prototype.columns;
-/** @type {?} */
-Table.prototype.length;
-/** @type {?} */
-Table.prototype.col;
-/** @type {?} */
-Table.prototype.key;
-/** @type {?} */
-Table.prototype.select;
-/** @type {?} */
-Table.prototype.toString;
-
-let Vector = function() {};
-/** @type {?} */
-Vector.prototype.length;
-/** @type {?} */
-Vector.prototype.name;
-/** @type {?} */
-Vector.prototype.type;
-/** @type {?} */
-Vector.prototype.get;
-/** @type {?} */
-Vector.prototype.concat;
-/** @type {?} */
-Vector.prototype.slice;
-/** @type {?} */
-Vector.prototype.metadata;
-/** @type {?} */
-Vector.prototype.nullable;
-/** @type {?} */
-Vector.prototype.nullCount;
-
-let BoolVector = function() {};
-/** @type {?} */
-(<any> BoolVector).pack;
-/** @type {?} */
-BoolVector.prototype.set;
-
-let DictionaryVector = function() {};
-/** @type {?} */
-DictionaryVector.prototype.getKey;
-/** @type {?} */
-DictionaryVector.prototype.getValue;
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index 3a8943434eece..e58aa69865bb3 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -15,109 +15,258 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Table } from './vector/table';
-import { Vector } from './vector/vector';
-import { Utf8Vector } from './vector/utf8';
-import { DictionaryVector } from './vector/dictionary';
-import { StructVector, StructRow } from './vector/struct';
-import { read, readAsync } from './reader/arrow';
-import { Uint64, Int64, Int128 } from './util/int';
-import { ListVector, BinaryVector, FixedSizeListVector } from './vector/list';
-
-import {
-    BoolVector,
-    Int8Vector,
-    Int16Vector,
-    Int32Vector,
-    Int64Vector,
-    Uint8Vector,
-    Uint16Vector,
-    Uint32Vector,
-    Uint64Vector,
-    Float16Vector,
-    Float32Vector,
-    Float64Vector,
-    Date32Vector,
-    Date64Vector,
-    Time32Vector,
-    Time64Vector,
-    DecimalVector,
-    TimestampVector,
-} from './vector/numeric';
-
-// closure compiler always erases static method names:
-// https://github.com/google/closure-compiler/issues/1776
-// set them via string indexers to save them from the mangler
-Table['from'] = Table.from;
-Table['fromAsync'] = Table.fromAsync;
-BoolVector['pack'] = BoolVector.pack;
+import * as type_ from './type';
+import * as data_ from './data';
+import * as vector_ from './vector';
+import * as util_ from './util/int';
+import * as visitor_ from './visitor';
+import * as view_ from './vector/view';
+import * as predicate_ from './predicate';
+import { Vector } from './vector';
+import { RecordBatch } from './recordbatch';
+import { Schema, Field, Type } from './type';
+import { Table, DataFrame, NextFunc, BindFunc, CountByResult } from './table';
+import { read, readAsync } from './ipc/reader/arrow';
+
+export import View = vector_.View;
+export import VectorLike = vector_.VectorLike;
+export import TypedArray = type_.TypedArray;
+export import IntBitWidth = type_.IntBitWidth;
+export import TimeBitWidth = type_.TimeBitWidth;
+export import TypedArrayConstructor = type_.TypedArrayConstructor;
 
 export { read, readAsync };
-export { Table, Vector, StructRow };
-export { Uint64, Int64, Int128 };
-export { NumericVectorConstructor } from './vector/numeric';
-export { List, TypedArray, TypedArrayConstructor } from './vector/types';
-export {
-    BoolVector,
-    ListVector,
-    Utf8Vector,
-    Int8Vector,
-    Int16Vector,
-    Int32Vector,
-    Int64Vector,
-    Uint8Vector,
-    Uint16Vector,
-    Uint32Vector,
-    Uint64Vector,
-    Date32Vector,
-    Date64Vector,
-    Time32Vector,
-    Time64Vector,
-    BinaryVector,
-    StructVector,
-    Float16Vector,
-    Float32Vector,
-    Float64Vector,
-    DecimalVector,
-    TimestampVector,
-    DictionaryVector,
-    FixedSizeListVector,
-};
-
-/* These exports are needed for the closure umd targets */
+export { Table, DataFrame, NextFunc, BindFunc, CountByResult };
+export { Field, Schema, RecordBatch, Vector, Type };
+
+export namespace util {
+    export import Uint64 = util_.Uint64;
+    export import Int64 = util_.Int64;
+    export import Int128 = util_.Int128;
+}
+
+export namespace data {
+    export import BaseData = data_.BaseData;
+    export import FlatData = data_.FlatData;
+    export import BoolData = data_.BoolData;
+    export import FlatListData = data_.FlatListData;
+    export import DictionaryData = data_.DictionaryData;
+    export import NestedData = data_.NestedData;
+    export import ListData = data_.ListData;
+    export import UnionData = data_.UnionData;
+    export import SparseUnionData = data_.SparseUnionData;
+    export import DenseUnionData = data_.DenseUnionData;
+    export import ChunkedData = data_.ChunkedData;
+}
+
+export namespace type {
+    export import Schema = type_.Schema;
+    export import Field = type_.Field;
+    export import Null = type_.Null;
+    export import Int = type_.Int;
+    export import Int8 = type_.Int8;
+    export import Int16 = type_.Int16;
+    export import Int32 = type_.Int32;
+    export import Int64 = type_.Int64;
+    export import Uint8 = type_.Uint8;
+    export import Uint16 = type_.Uint16;
+    export import Uint32 = type_.Uint32;
+    export import Uint64 = type_.Uint64;
+    export import Float = type_.Float;
+    export import Float16 = type_.Float16;
+    export import Float32 = type_.Float32;
+    export import Float64 = type_.Float64;
+    export import Binary = type_.Binary;
+    export import Utf8 = type_.Utf8;
+    export import Bool = type_.Bool;
+    export import Decimal = type_.Decimal;
+    export import Date_ = type_.Date_;
+    export import Time = type_.Time;
+    export import Timestamp = type_.Timestamp;
+    export import Interval = type_.Interval;
+    export import List = type_.List;
+    export import Struct = type_.Struct;
+    export import Union = type_.Union;
+    export import DenseUnion = type_.DenseUnion;
+    export import SparseUnion = type_.SparseUnion;
+    export import FixedSizeBinary = type_.FixedSizeBinary;
+    export import FixedSizeList = type_.FixedSizeList;
+    export import Map_ = type_.Map_;
+    export import Dictionary = type_.Dictionary;
+}
+
+export namespace vector {
+    export import Vector = vector_.Vector;
+    export import NullVector = vector_.NullVector;
+    export import BoolVector = vector_.BoolVector;
+    export import IntVector = vector_.IntVector;
+    export import FloatVector = vector_.FloatVector;
+    export import DateVector = vector_.DateVector;
+    export import DecimalVector = vector_.DecimalVector;
+    export import TimeVector = vector_.TimeVector;
+    export import TimestampVector = vector_.TimestampVector;
+    export import IntervalVector = vector_.IntervalVector;
+    export import BinaryVector = vector_.BinaryVector;
+    export import FixedSizeBinaryVector = vector_.FixedSizeBinaryVector;
+    export import Utf8Vector = vector_.Utf8Vector;
+    export import ListVector = vector_.ListVector;
+    export import FixedSizeListVector = vector_.FixedSizeListVector;
+    export import MapVector = vector_.MapVector;
+    export import StructVector = vector_.StructVector;
+    export import UnionVector = vector_.UnionVector;
+    export import DictionaryVector = vector_.DictionaryVector;
+}
+
+export namespace visitor {
+    export import TypeVisitor = visitor_.TypeVisitor;
+    export import VectorVisitor = visitor_.VectorVisitor;
+}
+
+export namespace view {
+    export import ChunkedView = view_.ChunkedView;
+    export import DictionaryView = view_.DictionaryView;
+    export import ListView = view_.ListView;
+    export import FixedSizeListView = view_.FixedSizeListView;
+    export import BinaryView = view_.BinaryView;
+    export import Utf8View = view_.Utf8View;
+    export import UnionView = view_.UnionView;
+    export import DenseUnionView = view_.DenseUnionView;
+    export import NestedView = view_.NestedView;
+    export import StructView = view_.StructView;
+    export import MapView = view_.MapView;
+    export import FlatView = view_.FlatView;
+    export import NullView = view_.NullView;
+    export import BoolView = view_.BoolView;
+    export import ValidityView = view_.ValidityView;
+    export import PrimitiveView = view_.PrimitiveView;
+    export import FixedSizeView = view_.FixedSizeView;
+    export import Float16View = view_.Float16View;
+    export import DateDayView = view_.DateDayView;
+    export import DateMillisecondView = view_.DateMillisecondView;
+    export import TimestampDayView = view_.TimestampDayView;
+    export import TimestampSecondView = view_.TimestampSecondView;
+    export import TimestampMillisecondView = view_.TimestampMillisecondView;
+    export import TimestampMicrosecondView = view_.TimestampMicrosecondView;
+    export import TimestampNanosecondView = view_.TimestampNanosecondView;
+    export import IntervalYearMonthView = view_.IntervalYearMonthView;
+    export import IntervalYearView = view_.IntervalYearView;
+    export import IntervalMonthView = view_.IntervalMonthView;
+}
+
+export namespace predicate {
+    export import col = predicate_.col;
+    export import lit = predicate_.lit;
+
+    export import Or = predicate_.Or;
+    export import Col = predicate_.Col;
+    export import And = predicate_.And;
+    export import GTeq = predicate_.GTeq;
+    export import LTeq = predicate_.LTeq;
+    export import Value = predicate_.Value;
+    export import Equals = predicate_.Equals;
+    export import Literal = predicate_.Literal;
+    export import Predicate = predicate_.Predicate;
+
+    export import PredicateFunc = predicate_.PredicateFunc;
+}
+
+/* These exports are needed for the closure and uglify umd targets */
 try {
-    const Arrow = eval('exports');
-    if (typeof Arrow === 'object') {
-        // string indexers tell closure compiler not to rename these properties
+    let Arrow: any = eval('exports');
+    if (Arrow && typeof Arrow === 'object') {
+        // string indexers tell closure and uglify not to rename these properties
+        Arrow['data'] = data;
+        Arrow['type'] = type;
+        Arrow['util'] = util;
+        Arrow['view'] = view;
+        Arrow['vector'] = vector;
+        Arrow['visitor'] = visitor;
+        Arrow['predicate'] = predicate;
+
         Arrow['read'] = read;
         Arrow['readAsync'] = readAsync;
-        Arrow['Table'] = Table;
+
+        Arrow['Type'] = Type;
+        Arrow['Field'] = Field;
+        Arrow['Schema'] = Schema;
         Arrow['Vector'] = Vector;
-        Arrow['StructRow'] = StructRow;
-        Arrow['BoolVector'] = BoolVector;
-        Arrow['ListVector'] = ListVector;
-        Arrow['Utf8Vector'] = Utf8Vector;
-        Arrow['Int8Vector'] = Int8Vector;
-        Arrow['Int16Vector'] = Int16Vector;
-        Arrow['Int32Vector'] = Int32Vector;
-        Arrow['Int64Vector'] = Int64Vector;
-        Arrow['Uint8Vector'] = Uint8Vector;
-        Arrow['Uint16Vector'] = Uint16Vector;
-        Arrow['Uint32Vector'] = Uint32Vector;
-        Arrow['Uint64Vector'] = Uint64Vector;
-        Arrow['Date32Vector'] = Date32Vector;
-        Arrow['Date64Vector'] = Date64Vector;
-        Arrow['Time32Vector'] = Time32Vector;
-        Arrow['Time64Vector'] = Time64Vector;
-        Arrow['BinaryVector'] = BinaryVector;
-        Arrow['StructVector'] = StructVector;
-        Arrow['Float16Vector'] = Float16Vector;
-        Arrow['Float32Vector'] = Float32Vector;
-        Arrow['Float64Vector'] = Float64Vector;
-        Arrow['DecimalVector'] = DecimalVector;
-        Arrow['TimestampVector'] = TimestampVector;
-        Arrow['DictionaryVector'] = DictionaryVector;
-        Arrow['FixedSizeListVector'] = FixedSizeListVector;
+        Arrow['RecordBatch'] = RecordBatch;
+
+        Arrow['Table'] = Table;
+        Arrow['CountByResult'] = CountByResult;
     }
 } catch (e) { /* not the UMD bundle */ }
-/* end closure exports */
+/* end umd exports */
+
+// closure compiler erases static properties/methods:
+// https://github.com/google/closure-compiler/issues/1776
+// set them via string indexers to save them from the mangler
+Schema['from'] = Schema.from;
+Table['from'] = Table.from;
+Table['fromAsync'] = Table.fromAsync;
+Table['empty'] = Table.empty;
+Vector['create'] = Vector.create;
+RecordBatch['from'] = RecordBatch.from;
+
+util_.Uint64['add'] = util_.Uint64.add;
+util_.Uint64['multiply'] = util_.Uint64.multiply;
+
+util_.Int64['add'] = util_.Int64.add;
+util_.Int64['multiply'] = util_.Int64.multiply;
+util_.Int64['fromString'] = util_.Int64.fromString;
+
+util_.Int128['add'] = util_.Int128.add;
+util_.Int128['multiply'] = util_.Int128.multiply;
+util_.Int128['fromString'] = util_.Int128.fromString;
+
+data_.ChunkedData['computeOffsets'] = data_.ChunkedData.computeOffsets;
+
+(type_.Type as any)['NONE'] = type_.Type.NONE;
+(type_.Type as any)['Null'] = type_.Type.Null;
+(type_.Type as any)['Int'] = type_.Type.Int;
+(type_.Type as any)['Float'] = type_.Type.Float;
+(type_.Type as any)['Binary'] = type_.Type.Binary;
+(type_.Type as any)['Utf8'] = type_.Type.Utf8;
+(type_.Type as any)['Bool'] = type_.Type.Bool;
+(type_.Type as any)['Decimal'] = type_.Type.Decimal;
+(type_.Type as any)['Date'] = type_.Type.Date;
+(type_.Type as any)['Time'] = type_.Type.Time;
+(type_.Type as any)['Timestamp'] = type_.Type.Timestamp;
+(type_.Type as any)['Interval'] = type_.Type.Interval;
+(type_.Type as any)['List'] = type_.Type.List;
+(type_.Type as any)['Struct'] = type_.Type.Struct;
+(type_.Type as any)['Union'] = type_.Type.Union;
+(type_.Type as any)['FixedSizeBinary'] = type_.Type.FixedSizeBinary;
+(type_.Type as any)['FixedSizeList'] = type_.Type.FixedSizeList;
+(type_.Type as any)['Map'] = type_.Type.Map;
+(type_.Type as any)['Dictionary'] = type_.Type.Dictionary;
+(type_.Type as any)['DenseUnion'] = type_.Type.DenseUnion;
+(type_.Type as any)['SparseUnion'] = type_.Type.SparseUnion;
+
+type_.DataType['isNull'] = type_.DataType.isNull;
+type_.DataType['isInt'] = type_.DataType.isInt;
+type_.DataType['isFloat'] = type_.DataType.isFloat;
+type_.DataType['isBinary'] = type_.DataType.isBinary;
+type_.DataType['isUtf8'] = type_.DataType.isUtf8;
+type_.DataType['isBool'] = type_.DataType.isBool;
+type_.DataType['isDecimal'] = type_.DataType.isDecimal;
+type_.DataType['isDate'] = type_.DataType.isDate;
+type_.DataType['isTime'] = type_.DataType.isTime;
+type_.DataType['isTimestamp'] = type_.DataType.isTimestamp;
+type_.DataType['isInterval'] = type_.DataType.isInterval;
+type_.DataType['isList'] = type_.DataType.isList;
+type_.DataType['isStruct'] = type_.DataType.isStruct;
+type_.DataType['isUnion'] = type_.DataType.isUnion;
+type_.DataType['isDenseUnion'] = type_.DataType.isDenseUnion;
+type_.DataType['isSparseUnion'] = type_.DataType.isSparseUnion;
+type_.DataType['isFixedSizeBinary'] = type_.DataType.isFixedSizeBinary;
+type_.DataType['isFixedSizeList'] = type_.DataType.isFixedSizeList;
+type_.DataType['isMap'] = type_.DataType.isMap;
+type_.DataType['isDictionary'] = type_.DataType.isDictionary;
+
+vector_.BoolVector['from'] = vector_.BoolVector.from;
+vector_.IntVector['from'] = vector_.IntVector.from;
+vector_.FloatVector['from'] = vector_.FloatVector.from;
+
+visitor_.TypeVisitor['visitTypeInline'] = visitor_.TypeVisitor.visitTypeInline;
+visitor_.VectorVisitor['visitTypeInline'] = visitor_.VectorVisitor.visitTypeInline;
\ No newline at end of file
diff --git a/js/src/bin/arrow2csv.ts b/js/src/bin/arrow2csv.ts
index 01ef0b848ce75..ee9561323785b 100644
--- a/js/src/bin/arrow2csv.ts
+++ b/js/src/bin/arrow2csv.ts
@@ -1,4 +1,4 @@
-// #! /usr/bin/env node
+#! /usr/bin/env node
 
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
@@ -19,11 +19,9 @@
 
 /* tslint:disable */
 
+import * as fs from 'fs';
 import * as Arrow from '../Arrow';
 
-(function() {
-
-const fs = require('fs');
 const { parse } = require('json-bignum');
 const optionList = [
     {
@@ -36,12 +34,13 @@ const optionList = [
     {
         type: String,
         name: 'file', alias: 'f',
+        optional: false, multiple: true,
         description: 'The Arrow file to read'
     }
 ];
 
 const argv = require(`command-line-args`)(optionList, { partial: true });
-const files = [argv.file, ...(argv._unknown || [])].filter(Boolean);
+const files = [...argv.file, ...(argv._unknown || [])].filter(Boolean);
 
 if (!files.length) {
     console.log(require('command-line-usage')([
@@ -85,51 +84,16 @@ if (!files.length) {
 }
 
 files.forEach((source) => {
-    let table: any, input = fs.readFileSync(source);
+    debugger;
+    let table: Arrow.Table, input = fs.readFileSync(source);
     try {
-        table = Arrow.Table.from([input]);
+        table = Arrow.Table.from(input);
     } catch (e) {
+        debugger;
         table = Arrow.Table.from(parse(input + ''));
     }
     if (argv.schema && argv.schema.length) {
         table = table.select(...argv.schema);
     }
-    printTable(table);
+    table.rowsToString().pipe(process.stdout);
 });
-
-function printTable(table: Arrow.Table<any>) {
-    let header = [...table.columns.map((_, i) => table.key(i))].map(stringify);
-    let maxColumnWidths = header.map(x => x.length);
-    // Pass one to convert to strings and count max column widths
-    for (let i = -1, n = table.length - 1; ++i < n;) {
-        let val,
-            row = [i, ...table.get(i)];
-        for (let j = -1, k = row.length; ++j < k; ) {
-            val = stringify(row[j]);
-            maxColumnWidths[j] = Math.max(maxColumnWidths[j], val.length);
-        }
-    }
-    console.log(header.map((x, j) => leftPad(x, ' ', maxColumnWidths[j])).join(' | '));
-    // Pass two to pad each one to max column width
-    for (let i = -1, n = table.length; ++i < n; ) {
-        console.log(
-            [...table.get(i)]
-                .map(stringify)
-                .map((x, j) => leftPad(x, ' ', maxColumnWidths[j]))
-                .join(' | ')
-        );
-    }
-}
-
-function leftPad(str: string, fill: string, n: number) {
-    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
-}
-
-function stringify(x: any) {
-    return typeof x === 'string' ? `"${x}"`
-              : Array.isArray(x) ? JSON.stringify(x)
-              : ArrayBuffer.isView(x) ? `[${x}]`
-                                      : `${x}`;
-}
-
-})();
\ No newline at end of file
diff --git a/js/src/data.ts b/js/src/data.ts
new file mode 100644
index 0000000000000..81d19a3cf63cd
--- /dev/null
+++ b/js/src/data.ts
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { popcnt_bit_range } from './util/bit';
+import { VectorLike, Vector } from './vector';
+import { VectorType, TypedArray, TypedArrayConstructor, Dictionary } from './type';
+import { Int, Bool, FlatListType, List, FixedSizeList, Struct, Map_ } from './type';
+import { DataType, FlatType, ListType, NestedType, SingleNestedType, DenseUnion, SparseUnion } from './type';
+
+export function toTypedArray<T extends TypedArray>(ArrayType: TypedArrayConstructor<T>, values?: T | ArrayLike<number> | Iterable<number> | null): T {
+    if (!ArrayType && ArrayBuffer.isView(values)) { return values; }
+    return values instanceof ArrayType ? values
+         : !values || !ArrayBuffer.isView(values) ? ArrayType.from(values || [])
+         : new ArrayType(values.buffer, values.byteOffset, values.byteLength / ArrayType.BYTES_PER_ELEMENT);
+}
+
+export type Data<T extends DataType> = DataTypes<T>[T['TType']] & BaseData<T>;
+export interface DataTypes<T extends DataType> {
+/*                [Type.NONE]*/  0: BaseData<T>;
+/*                [Type.Null]*/  1: FlatData<T>;
+/*                 [Type.Int]*/  2: FlatData<T>;
+/*               [Type.Float]*/  3: FlatData<T>;
+/*              [Type.Binary]*/  4: FlatListData<T>;
+/*                [Type.Utf8]*/  5: FlatListData<T>;
+/*                [Type.Bool]*/  6: BoolData;
+/*             [Type.Decimal]*/  7: FlatData<T>;
+/*                [Type.Date]*/  8: FlatData<T>;
+/*                [Type.Time]*/  9: FlatData<T>;
+/*           [Type.Timestamp]*/ 10: FlatData<T>;
+/*            [Type.Interval]*/ 11: FlatData<T>;
+/*                [Type.List]*/ 12: ListData<List<T>>;
+/*              [Type.Struct]*/ 13: NestedData<Struct>;
+/*               [Type.Union]*/ 14: UnionData;
+/*     [Type.FixedSizeBinary]*/ 15: FlatData<T>;
+/*       [Type.FixedSizeList]*/ 16: SingleNestedData<FixedSizeList<T>>;
+/*                 [Type.Map]*/ 17: NestedData<Map_>;
+/*  [Type.DenseUnion]*/ DenseUnion: DenseUnionData;
+/*[Type.SparseUnion]*/ SparseUnion: SparseUnionData;
+/*[  Type.Dictionary]*/ Dictionary: DictionaryData<any>;
+}
+// When slicing, we do not know the null count of the sliced range without
+// doing some computation. To avoid doing this eagerly, we set the null count
+// to -1 (any negative number will do). When Array::null_count is called the
+// first time, the null count will be computed. See ARROW-33
+export type kUnknownNullCount = -1;
+export const kUnknownNullCount = -1;
+
+export class BaseData<T extends DataType = DataType> implements VectorLike {
+    public type: T;
+    public length: number;
+    public offset: number;
+    // @ts-ignore
+    public childData: Data<any>[];
+    protected _nullCount: number | kUnknownNullCount;
+    protected /*  [VectorType.OFFSET]:*/ 0?: Int32Array;
+    protected /*    [VectorType.DATA]:*/ 1?: T['TArray'];
+    protected /*[VectorType.VALIDITY]:*/ 2?: Uint8Array;
+    protected /*    [VectorType.TYPE]:*/ 3?: Int8Array;
+    constructor(type: T, length: number, offset?: number, nullCount?: number) {
+        this.type = type;
+        this.length = Math.floor(Math.max(length || 0, 0));
+        this.offset = Math.floor(Math.max(offset || 0, 0));
+        this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
+    }
+    public get typeId() { return this.type.TType; }
+    public get nullBitmap() { return this[VectorType.VALIDITY]; }
+    public get nullCount() {
+        let nullCount = this._nullCount;
+        let nullBitmap: Uint8Array | undefined;
+        if (nullCount === -1 && (nullBitmap = this[VectorType.VALIDITY])) {
+            this._nullCount = nullCount = this.length - popcnt_bit_range(nullBitmap, this.offset, this.offset + this.length);
+        }
+        return nullCount;
+    }
+    public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new BaseData(type, length, offset, nullCount);
+    }
+    public slice(offset: number, length: number) {
+        return length <= 0 ? this : this.sliceInternal(this.clone(
+            this.type, length, this.offset + offset, +(this._nullCount === 0) - 1
+        ) as any, offset, length);
+    }
+    protected sliceInternal(clone: this, offset: number, length: number) {
+        let arr: any;
+        // If typeIds exist, slice the typeIds buffer
+        (arr = this[VectorType.TYPE]) && (clone[VectorType.TYPE] = this.sliceData(arr, offset, length));
+        // If offsets exist, only slice the offsets buffer
+        (arr = this[VectorType.OFFSET]) && (clone[VectorType.OFFSET] = this.sliceOffsets(arr, offset, length)) ||
+            // Otherwise if no offsets, slice the data buffer
+            (arr = this[VectorType.DATA]) && (clone[VectorType.DATA] = this.sliceData(arr, offset, length));
+        return clone;
+    }
+    protected sliceData(data: T['TArray'] & TypedArray, offset: number, length: number) {
+        return data.subarray(offset, offset + length);
+    }
+    protected sliceOffsets(valueOffsets: Int32Array, offset: number, length: number) {
+        return valueOffsets.subarray(offset, offset + length + 1);
+    }
+}
+
+export class FlatData<T extends FlatType> extends BaseData<T> {
+    public /*    [VectorType.DATA]:*/ 1: T['TArray'];
+    public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
+    public get values() { return this[VectorType.DATA]; }
+    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, data: Iterable<number>, offset?: number, nullCount?: number) {
+        super(type, length, offset, nullCount);
+        this[VectorType.DATA] = toTypedArray(this.ArrayType, data);
+        this[VectorType.VALIDITY] = toTypedArray(Uint8Array, nullBitmap);
+    }
+    public get ArrayType(): T['ArrayType'] { return this.type.ArrayType; }
+    public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new (this.constructor as any)(type, length, this[VectorType.VALIDITY], this[VectorType.DATA], offset, nullCount) as FlatData<R>;
+    }
+}
+
+export class BoolData extends FlatData<Bool> {
+    protected sliceData(data: Uint8Array) { return data; }
+}
+
+export class FlatListData<T extends FlatListType> extends FlatData<T> {
+    public /*  [VectorType.OFFSET]:*/ 0: Int32Array;
+    public /*    [VectorType.DATA]:*/ 1: T['TArray'];
+    public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
+    public get values() { return this[VectorType.DATA]; }
+    public get valueOffsets() { return this[VectorType.OFFSET]; }
+    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable<number>, data: T['TArray'], offset?: number, nullCount?: number) {
+        super(type, length, nullBitmap, data, offset, nullCount);
+        this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
+    }
+    public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new FlatListData(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this[VectorType.DATA], offset, nullCount);
+    }
+}
+
+export class DictionaryData<T extends DataType> extends BaseData<Dictionary<T>> {
+    protected _dictionary: Vector<T>;
+    protected _indicies: Data<Int<any>>;
+    public get indicies() { return this._indicies; }
+    public get dictionary() { return this._dictionary; }
+    constructor(type: Dictionary<T>, dictionary: Vector<T>, indicies: Data<Int<any>>) {
+        super(type, indicies.length, (indicies as any)._nullCount);
+        this._indicies = indicies;
+        this._dictionary = dictionary;
+        this.length = this._indicies.length;
+    }
+    public get nullCount() { return this._indicies.nullCount; }
+    public clone<R extends Dictionary<T>>(type: R, length = this.length, offset = this.offset) {
+        const data = this._dictionary.data.clone(type.dictionary as any);
+        return new DictionaryData<R>(
+            this.type as any,
+            this._dictionary.clone(data) as any,
+            this._indicies.slice(offset - this.offset, length)
+        ) as any;
+    }
+    protected sliceInternal(clone: this, _offset: number, _length: number) {
+        clone.length = clone._indicies.length;
+        clone._nullCount = (clone._indicies as any)._nullCount;
+        return clone;
+    }
+}
+
+export class NestedData<T extends NestedType = NestedType> extends BaseData<T> {
+    public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
+    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, childData: Data<any>[], offset?: number, nullCount?: number) {
+        super(type, length, offset, nullCount);
+        this.childData = childData;
+        this[VectorType.VALIDITY] = toTypedArray(Uint8Array, nullBitmap);
+    }
+    public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new NestedData<R>(type, length, this[VectorType.VALIDITY], this.childData, offset, nullCount);
+    }
+    protected sliceInternal(clone: this, offset: number, length: number) {
+        if (!this[VectorType.OFFSET]) {
+            clone.childData = this.childData.map((child) => child.slice(offset, length));
+        }
+        return super.sliceInternal(clone, offset, length);
+    }
+}
+
+export class SingleNestedData<T extends SingleNestedType> extends NestedData<T> {
+    protected _valuesData: Data<T>;
+    public get values() { return this._valuesData; }
+    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueChildData: Data<T>, offset?: number, nullCount?: number) {
+        super(type, length, nullBitmap, [valueChildData], offset, nullCount);
+        this._valuesData = valueChildData;
+    }
+}
+
+export class ListData<T extends ListType> extends SingleNestedData<T> {
+    public /*  [VectorType.OFFSET]:*/ 0: Int32Array;
+    public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
+    public get valueOffsets() { return this[VectorType.OFFSET]; }
+    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable<number>, valueChildData: Data<T>, offset?: number, nullCount?: number) {
+        super(type, length, nullBitmap, valueChildData, offset, nullCount);
+        this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
+    }
+    public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new ListData<R>(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this._valuesData as any, offset, nullCount);
+    }
+}
+
+export class UnionData<T extends (DenseUnion | SparseUnion) = any> extends NestedData<T> {
+    public /*    [VectorType.TYPE]:*/ 3: T['TArray'];
+    public get typeIds() { return this[VectorType.TYPE]; }
+    constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, typeIds: Iterable<number>, childData: Data<any>[], offset?: number, nullCount?: number) {
+        super(type, length, nullBitmap, childData, offset, nullCount);
+        this[VectorType.TYPE] = toTypedArray(Int8Array, typeIds);
+    }
+    public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new UnionData<R>(type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this.childData, offset, nullCount);
+    }
+}
+
+export class SparseUnionData extends UnionData<SparseUnion> {
+    constructor(type: SparseUnion, length: number, nullBitmap: Uint8Array | null | undefined, typeIds: Iterable<number>, childData: Data<any>[], offset?: number, nullCount?: number) {
+        super(type, length, nullBitmap, typeIds, childData, offset, nullCount);
+    }
+    public clone<R extends SparseUnion>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new SparseUnionData(
+            type,
+            length,
+            this[VectorType.VALIDITY],
+            this[VectorType.TYPE],
+            this.childData,
+            offset, nullCount
+        ) as any as UnionData<R>;
+    }
+}
+
+export class DenseUnionData extends UnionData<DenseUnion> {
+    public /*  [VectorType.OFFSET]:*/ 0: Int32Array;
+    public get valueOffsets() { return this[VectorType.OFFSET]; }
+    constructor(type: DenseUnion, length: number, nullBitmap: Uint8Array | null | undefined, typeIds: Iterable<number>, valueOffsets: Iterable<number>, childData: Data<any>[], offset?: number, nullCount?: number) {
+        super(type, length, nullBitmap, typeIds, childData, offset, nullCount);
+        this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
+    }
+    public clone<R extends DenseUnion>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new DenseUnionData(
+            type,
+            length,
+            this[VectorType.VALIDITY],
+            this[VectorType.TYPE],
+            this[VectorType.OFFSET],
+            this.childData,
+            offset, nullCount
+        ) as any as UnionData<R>;
+    }
+}
+
+export class ChunkedData<T extends DataType> extends BaseData<T> {
+    // @ts-ignore
+    protected _chunkData: Data<T>[];
+    protected _chunkVectors: Vector<T>[];
+    protected _chunkOffsets: Uint32Array;
+    public get chunkVectors() { return this._chunkVectors; }
+    public get chunkOffsets() { return this._chunkOffsets; }
+    public get chunkData() {
+        return this._chunkData || (
+               this._chunkData = this._chunkVectors.map(({ data }) => data));
+    }
+    constructor(type: T, length: number, chunkVectors: Vector<T>[], offset?: number, nullCount?: number, chunkOffsets?: Uint32Array) {
+        super(type, length, offset, nullCount);
+        this._chunkVectors = chunkVectors;
+        this._chunkOffsets = chunkOffsets || ChunkedData.computeOffsets(chunkVectors);
+    }
+    public get nullCount() {
+        let nullCount = this._nullCount;
+        if (nullCount === -1) {
+            this._nullCount = nullCount = this._chunkVectors.reduce((x, c) => x + c.nullCount, 0);
+        }
+        return nullCount;
+    }
+    public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
+        return new ChunkedData<R>(
+            type, length,
+            this._chunkVectors.map((vec) => vec.clone(vec.data.clone(type))) as any,
+            offset, nullCount, this._chunkOffsets
+        );
+    }
+    protected sliceInternal(clone: this, offset: number, length: number) {
+        const chunks = this._chunkVectors;
+        const offsets = this._chunkOffsets;
+        const chunkSlices: Vector<T>[] = [];
+        for (let childIndex = -1, numChildren = chunks.length; ++childIndex < numChildren;) {
+            const child = chunks[childIndex];
+            const childLength = child.length;
+            const childOffset = offsets[childIndex];
+            // If the child is to the right of the slice boundary, exclude
+            if (childOffset >= offset + length) { continue; }
+            // If the child is to the left of of the slice boundary, exclude
+            if (offset >= childOffset + childLength) { continue; }
+            // If the child is between both left and right boundaries, include w/o slicing
+            if (childOffset >= offset && (childOffset + childLength) <= offset + length) {
+                chunkSlices.push(child);
+                continue;
+            }
+            // If the child overlaps one of the slice boundaries, include that slice
+            const begin = Math.max(0, offset - childOffset);
+            const end = begin + Math.min(childLength - begin, (offset + length) - childOffset);
+            chunkSlices.push(child.slice(begin, end));
+        }
+        clone._chunkVectors = chunkSlices;
+        clone._chunkOffsets = ChunkedData.computeOffsets(chunkSlices);
+        return clone;
+    }
+    static computeOffsets<T extends DataType>(childVectors: Vector<T>[]) {
+        const childOffsets = new Uint32Array(childVectors.length + 1);
+        for (let index = 0, length = childOffsets.length, childOffset = childOffsets[0] = 0; ++index < length;) {
+            childOffsets[index] = (childOffset += childVectors[index - 1].length);
+        }
+        return childOffsets;
+    }
+}
diff --git a/js/src/format/fb/File.ts b/js/src/fb/File.ts
similarity index 99%
rename from js/src/format/fb/File.ts
rename to js/src/fb/File.ts
index 56f50ed20e936..f4ba865ff040b 100644
--- a/js/src/format/fb/File.ts
+++ b/js/src/fb/File.ts
@@ -14,6 +14,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -175,6 +176,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
diff --git a/js/src/format/fb/File_generated.js b/js/src/fb/File_generated.js
similarity index 100%
rename from js/src/format/fb/File_generated.js
rename to js/src/fb/File_generated.js
diff --git a/js/src/format/fb/Message.ts b/js/src/fb/Message.ts
similarity index 99%
rename from js/src/format/fb/Message.ts
rename to js/src/fb/Message.ts
index 4610fbef2e1c8..537c65d1f8c93 100644
--- a/js/src/format/fb/Message.ts
+++ b/js/src/fb/Message.ts
@@ -45,6 +45,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -110,6 +111,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -265,6 +267,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -369,6 +372,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
diff --git a/js/src/format/fb/Message_generated.js b/js/src/fb/Message_generated.js
similarity index 100%
rename from js/src/format/fb/Message_generated.js
rename to js/src/fb/Message_generated.js
diff --git a/js/src/format/fb/Schema.ts b/js/src/fb/Schema.ts
similarity index 99%
rename from js/src/format/fb/Schema.ts
rename to js/src/fb/Schema.ts
index d9b45ed20089c..4a4aeb65599be 100644
--- a/js/src/format/fb/Schema.ts
+++ b/js/src/fb/Schema.ts
@@ -165,6 +165,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -221,6 +222,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -273,6 +275,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -325,6 +328,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -420,6 +424,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -495,6 +500,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -617,6 +623,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -701,6 +708,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -771,6 +779,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -823,6 +832,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -875,6 +885,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -945,6 +956,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -997,6 +1009,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -1092,6 +1105,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -1164,6 +1178,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -1255,6 +1270,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -1363,6 +1379,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -1435,6 +1452,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -1527,6 +1545,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -1620,6 +1639,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -1741,6 +1761,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -2026,6 +2047,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
@@ -2089,6 +2111,7 @@ export namespace org.apache.arrow.flatbuf {
     /**
      * @type {flatbuffers.ByteBuffer}
      */
+    // @ts-ignore
     bb: flatbuffers.ByteBuffer;
 
     /**
diff --git a/js/src/format/fb/Schema_generated.js b/js/src/fb/Schema_generated.js
similarity index 100%
rename from js/src/format/fb/Schema_generated.js
rename to js/src/fb/Schema_generated.js
diff --git a/js/src/format/arrow.ts b/js/src/format/arrow.ts
deleted file mode 100644
index 14adf9040a47f..0000000000000
--- a/js/src/format/arrow.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { footerFromByteBuffer, messageFromByteBuffer } from './fb';
-import { schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON } from './json';
-import {
-    IntBitWidth, TimeBitWidth,
-    VisitorNode, Visitor, Footer, Block, Message, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode,
-    Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
-} from './types';
-
-export {
-    IntBitWidth, TimeBitWidth,
-    footerFromByteBuffer, messageFromByteBuffer,
-    schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON,
-    VisitorNode, Visitor, Footer, Block, Message, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode,
-    Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_ as Map,
-};
diff --git a/js/src/format/fb.ts b/js/src/format/fb.ts
deleted file mode 100644
index fdf7f7b0ed99a..0000000000000
--- a/js/src/format/fb.ts
+++ /dev/null
@@ -1,234 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import * as File_ from './fb/File';
-import * as Schema_ from './fb/Schema';
-import * as Message_ from './fb/Message';
-import { flatbuffers } from 'flatbuffers';
-import ByteBuffer = flatbuffers.ByteBuffer;
-import Type = Schema_.org.apache.arrow.flatbuf.Type;
-import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
-import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
-import _Footer = File_.org.apache.arrow.flatbuf.Footer;
-import _Block = File_.org.apache.arrow.flatbuf.Block;
-import _Message = Message_.org.apache.arrow.flatbuf.Message;
-import _Schema = Schema_.org.apache.arrow.flatbuf.Schema;
-import _Field = Schema_.org.apache.arrow.flatbuf.Field;
-import _RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
-import _DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch;
-import _FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
-import _Buffer = Schema_.org.apache.arrow.flatbuf.Buffer;
-import _DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
-import _Null = Schema_.org.apache.arrow.flatbuf.Null;
-import _Int = Schema_.org.apache.arrow.flatbuf.Int;
-import _FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
-import _Binary = Schema_.org.apache.arrow.flatbuf.Binary;
-import _Bool = Schema_.org.apache.arrow.flatbuf.Bool;
-import _Utf8 = Schema_.org.apache.arrow.flatbuf.Utf8;
-import _Decimal = Schema_.org.apache.arrow.flatbuf.Decimal;
-import _Date = Schema_.org.apache.arrow.flatbuf.Date;
-import _Time = Schema_.org.apache.arrow.flatbuf.Time;
-import _Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp;
-import _Interval = Schema_.org.apache.arrow.flatbuf.Interval;
-import _List = Schema_.org.apache.arrow.flatbuf.List;
-import _Struct = Schema_.org.apache.arrow.flatbuf.Struct_;
-import _Union = Schema_.org.apache.arrow.flatbuf.Union;
-import _FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary;
-import _FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
-import _Map = Schema_.org.apache.arrow.flatbuf.Map;
-
-import {
-    IntBitWidth, TimeBitWidth,
-    Footer, Block, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode,
-    Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
-} from './types';
-
-export function footerFromByteBuffer(bb: ByteBuffer) {
-    const f = _Footer.getRootAsFooter(bb), s = f.schema()!;
-    return new Footer(
-        dictionaryBatchesFromFooter(f), recordBatchesFromFooter(f),
-        new Schema(f.version(), fieldsFromSchema(s), customMetadata(s), s.endianness())
-    );
-}
-
-export function messageFromByteBuffer(bb: ByteBuffer) {
-    const m = _Message.getRootAsMessage(bb)!, type = m.headerType(), version = m.version();
-    switch (type) {
-        case MessageHeader.Schema: return schemaFromMessage(version, m.header(new _Schema())!);
-        case MessageHeader.RecordBatch: return recordBatchFromMessage(version, m.header(new _RecordBatch())!);
-        case MessageHeader.DictionaryBatch: return dictionaryBatchFromMessage(version, m.header(new _DictionaryBatch())!);
-    }
-    return null;
-    // throw new Error(`Unrecognized Message type '${type}'`);
-}
-
-function schemaFromMessage(version: MetadataVersion, s: _Schema) {
-    return new Schema(version, fieldsFromSchema(s), customMetadata(s), s.endianness());
-}
-
-function recordBatchFromMessage(version: MetadataVersion, b: _RecordBatch) {
-    return new RecordBatch(version, b.length(), fieldNodesFromRecordBatch(b), buffersFromRecordBatch(b, version));
-}
-
-function dictionaryBatchFromMessage(version: MetadataVersion, d: _DictionaryBatch) {
-    return new DictionaryBatch(version, recordBatchFromMessage(version, d.data()!), d.id(), d.isDelta());
-}
-
-function dictionaryBatchesFromFooter(f: _Footer) {
-    const blocks = [] as Block[];
-    for (let b: _Block, i = -1, n = f && f.dictionariesLength(); ++i < n;) {
-        if (b = f.dictionaries(i)!) {
-            blocks.push(new Block(b.metaDataLength(), b.bodyLength(), b.offset()));
-        }
-    }
-    return blocks;
-}
-
-function recordBatchesFromFooter(f: _Footer) {
-    const blocks = [] as Block[];
-    for (let b: _Block, i = -1, n = f && f.recordBatchesLength(); ++i < n;) {
-        if (b = f.recordBatches(i)!) {
-            blocks.push(new Block(b.metaDataLength(), b.bodyLength(), b.offset()));
-        }
-    }
-    return blocks;
-}
-
-function fieldsFromSchema(s: _Schema) {
-    const fields = [] as Field[];
-    for (let i = -1, n = s && s.fieldsLength(); ++i < n;) {
-        fields.push(field(s.fields(i)!));
-    }
-    return fields;
-}
-
-function fieldsFromField(f: _Field) {
-    const fields = [] as Field[];
-    for (let i = -1, n = f && f.childrenLength(); ++i < n;) {
-        fields.push(field(f.children(i)!));
-    }
-    return fields;
-}
-
-function fieldNodesFromRecordBatch(b: _RecordBatch) {
-    const fieldNodes = [] as FieldNode[];
-    for (let i = -1, n = b.nodesLength(); ++i < n;) {
-        fieldNodes.push(fieldNodeFromRecordBatch(b.nodes(i)!));
-    }
-    return fieldNodes;
-}
-
-function buffersFromRecordBatch(b: _RecordBatch, version: MetadataVersion) {
-    const buffers = [] as Buffer[];
-    for (let i = -1, n = b.buffersLength(); ++i < n;) {
-        let buffer = b.buffers(i)!;
-        // If this Arrow buffer was written before version 4,
-        // advance the buffer's bb_pos 8 bytes to skip past
-        // the now-removed page id field.
-        if (version < MetadataVersion.V4) {
-            buffer.bb_pos += (8 * (i + 1));
-        }
-        buffers.push(bufferFromRecordBatch(buffer));
-    }
-    return buffers;
-}
-
-function field(f: _Field) {
-    return new Field(
-        f.name()!,
-        typeFromField(f),
-        f.typeType(),
-        f.nullable(),
-        fieldsFromField(f),
-        customMetadata(f),
-        dictionaryEncodingFromField(f)
-    );
-}
-
-function dictionaryEncodingFromField(f: _Field) {
-    let t: _Int | null;
-    let e: _DictionaryEncoding | null;
-    if (e = f.dictionary()) {
-        if (t = e.indexType()) {
-            return new DictionaryEncoding(new Int(t.isSigned(), t.bitWidth() as IntBitWidth), e.id(), e.isOrdered());
-        }
-        return new DictionaryEncoding(null, e.id(), e.isOrdered());
-    }
-    return undefined;
-}
-
-function customMetadata(parent?: _Schema | _Field | null) {
-    const data = new Map<string, string>();
-    if (parent) {
-        for (let entry, key, i = -1, n = parent.customMetadataLength() | 0; ++i < n;) {
-            if ((entry = parent.customMetadata(i)) && (key = entry.key()) != null) {
-                data.set(key, entry.value()!);
-            }
-        }
-    }
-    return data;
-}
-
-function fieldNodeFromRecordBatch(f: _FieldNode) {
-    return new FieldNode(f.length(), f.nullCount());
-}
-
-function bufferFromRecordBatch(b: _Buffer) {
-    return new Buffer(b.offset(), b.length());
-}
-
-function typeFromField(f: _Field) {
-    switch (f.typeType()) {
-        case Type.NONE: return nullFromField(f.type(new _Null())!);
-        case Type.Null: return nullFromField(f.type(new _Null())!);
-        case Type.Int: return intFromField(f.type(new _Int())!);
-        case Type.FloatingPoint: return floatingPointFromField(f.type(new _FloatingPoint())!);
-        case Type.Binary: return binaryFromField(f.type(new _Binary())!);
-        case Type.Utf8: return utf8FromField(f.type(new _Utf8())!);
-        case Type.Bool: return boolFromField(f.type(new _Bool())!);
-        case Type.Decimal: return decimalFromField(f.type(new _Decimal())!);
-        case Type.Date: return dateFromField(f.type(new _Date())!);
-        case Type.Time: return timeFromField(f.type(new _Time())!);
-        case Type.Timestamp: return timestampFromField(f.type(new _Timestamp())!);
-        case Type.Interval: return intervalFromField(f.type(new _Interval())!);
-        case Type.List: return listFromField(f.type(new _List())!);
-        case Type.Struct_: return structFromField(f.type(new _Struct())!);
-        case Type.Union: return unionFromField(f.type(new _Union())!);
-        case Type.FixedSizeBinary: return fixedSizeBinaryFromField(f.type(new _FixedSizeBinary())!);
-        case Type.FixedSizeList: return fixedSizeListFromField(f.type(new _FixedSizeList())!);
-        case Type.Map: return mapFromField(f.type(new _Map())!);
-    }
-    throw new Error(`Unrecognized type ${f.typeType()}`);
-}
-
-function nullFromField(_type: _Null) { return new Null(); }
-function intFromField(_type: _Int) { return new Int(_type.isSigned(), _type.bitWidth() as IntBitWidth); }
-function floatingPointFromField(_type: _FloatingPoint) { return new FloatingPoint(_type.precision()); }
-function binaryFromField(_type: _Binary) { return new Binary(); }
-function utf8FromField(_type: _Utf8) { return new Utf8(); }
-function boolFromField(_type: _Bool) { return new Bool(); }
-function decimalFromField(_type: _Decimal) { return new Decimal(_type.scale(), _type.precision()); }
-function dateFromField(_type: _Date) { return new Date(_type.unit()); }
-function timeFromField(_type: _Time) { return new Time(_type.unit(), _type.bitWidth() as TimeBitWidth); }
-function timestampFromField(_type: _Timestamp) { return new Timestamp(_type.unit(), _type.timezone()); }
-function intervalFromField(_type: _Interval) { return new Interval(_type.unit()); }
-function listFromField(_type: _List) { return new List(); }
-function structFromField(_type: _Struct) { return new Struct(); }
-function unionFromField(_type: _Union) { return new Union(_type.mode(), (_type.typeIdsArray() || []) as Type[]); }
-function fixedSizeBinaryFromField(_type: _FixedSizeBinary) { return new FixedSizeBinary(_type.byteWidth()); }
-function fixedSizeListFromField(_type: _FixedSizeList) { return new FixedSizeList(_type.listSize()); }
-function mapFromField(_type: _Map) { return new Map_(_type.keysSorted()); }
diff --git a/js/src/format/json.ts b/js/src/format/json.ts
deleted file mode 100644
index 3da3db6d5fea3..0000000000000
--- a/js/src/format/json.ts
+++ /dev/null
@@ -1,173 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import * as Schema_ from './fb/Schema';
-import { flatbuffers } from 'flatbuffers';
-import Long = flatbuffers.Long;
-import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
-import Type = Schema_.org.apache.arrow.flatbuf.Type;
-import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
-import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
-import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
-import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
-import {
-    IntBitWidth, TimeBitWidth,
-    Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode,
-    Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
-} from './types';
-
-export function schemaFromJSON(s: any): Schema {
-    // todo: metadataFromJSON
-    return new Schema(
-        MetadataVersion.V4,
-        fieldsFromJSON(s['fields']),
-        customMetadata(s['customMetadata'])
-    );
-}
-
-export function recordBatchFromJSON(b: any): RecordBatch {
-    return new RecordBatch(
-        MetadataVersion.V4,
-        new Long(b['count'], 0),
-        fieldNodesFromJSON(b['columns']),
-        buffersFromJSON(b['columns'])
-    );
-}
-
-export function dictionaryBatchFromJSON(b: any): DictionaryBatch {
-    return new DictionaryBatch(
-        MetadataVersion.V4,
-        recordBatchFromJSON(b['data']),
-        new Long(b['id'], 0), b['isDelta']
-    );
-}
-
-function fieldsFromJSON(fs: any[]): Field[] {
-    return (fs || []).map(fieldFromJSON);
-}
-
-function fieldNodesFromJSON(xs: any[]): FieldNode[] {
-    return (xs || []).reduce<FieldNode[]>((fieldNodes, column: any) => [
-        ...fieldNodes,
-        new FieldNode(
-            new Long(column['count'], 0),
-            new Long(nullCountFromJSON(column['VALIDITY']), 0)
-        ),
-        ...fieldNodesFromJSON(column['children'])
-    ], [] as FieldNode[]);
-}
-
-function buffersFromJSON(xs: any[], buffers: Buffer[] = []): Buffer[] {
-    for (let i = -1, n = (xs || []).length; ++i < n;) {
-        const column = xs[i];
-        column['VALIDITY'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['VALIDITY'].length, 0)));
-        column['OFFSET'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['OFFSET'].length, 0)));
-        column['DATA'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['DATA'].length, 0)));
-        buffers = buffersFromJSON(column['children'], buffers);
-    }
-    return buffers;
-}
-
-function nullCountFromJSON(validity: number[]) {
-    return (validity || []).reduce((sum, val) => sum + +(val === 0), 0);
-}
-
-function fieldFromJSON(f: any) {
-    return new Field(
-        f['name'],
-        typeFromJSON(f['type']),
-        namesToTypeMap[f['type']['name']],
-        f.nullable,
-        fieldsFromJSON(f['children']),
-        customMetadata(f['customMetadata']),
-        dictionaryEncodingFromJSON(f['dictionary'])
-    );
-}
-
-function dictionaryEncodingFromJSON(d: any) {
-    return !d ? null : new DictionaryEncoding(
-        d.indexType ? intFromJSON(d.indexType) : null,
-        new Long(d.id, 0), d.isOrdered
-    );
-}
-
-function customMetadata(metadata?: any) {
-    return new Map<string, string>(Object.entries(metadata || {}));
-}
-
-const namesToTypeMap: { [n: string]: Type }  = {
-    'NONE': Type.NONE,
-    'null': Type.Null,
-    'int': Type.Int,
-    'floatingpoint': Type.FloatingPoint,
-    'binary': Type.Binary,
-    'bool': Type.Bool,
-    'utf8': Type.Utf8,
-    'decimal': Type.Decimal,
-    'date': Type.Date,
-    'time': Type.Time,
-    'timestamp': Type.Timestamp,
-    'interval': Type.Interval,
-    'list': Type.List,
-    'struct': Type.Struct_,
-    'union': Type.Union,
-    'fixedsizebinary': Type.FixedSizeBinary,
-    'fixedsizelist': Type.FixedSizeList,
-    'map': Type.Map,
-};
-
-function typeFromJSON(t: any) {
-    switch (namesToTypeMap[t['name']]) {
-        case Type.NONE: return nullFromJSON(t);
-        case Type.Null: return nullFromJSON(t);
-        case Type.Int: return intFromJSON(t);
-        case Type.FloatingPoint: return floatingPointFromJSON(t);
-        case Type.Binary: return binaryFromJSON(t);
-        case Type.Utf8: return utf8FromJSON(t);
-        case Type.Bool: return boolFromJSON(t);
-        case Type.Decimal: return decimalFromJSON(t);
-        case Type.Date: return dateFromJSON(t);
-        case Type.Time: return timeFromJSON(t);
-        case Type.Timestamp: return timestampFromJSON(t);
-        case Type.Interval: return intervalFromJSON(t);
-        case Type.List: return listFromJSON(t);
-        case Type.Struct_: return structFromJSON(t);
-        case Type.Union: return unionFromJSON(t);
-        case Type.FixedSizeBinary: return fixedSizeBinaryFromJSON(t);
-        case Type.FixedSizeList: return fixedSizeListFromJSON(t);
-        case Type.Map: return mapFromJSON(t);
-    }
-    throw new Error(`Unrecognized type ${t['name']}`);
-}
-
-function nullFromJSON(_type: any) { return new Null(); }
-function intFromJSON(_type: any) { return new Int(_type['isSigned'], _type['bitWidth'] as IntBitWidth); }
-function floatingPointFromJSON(_type: any) { return new FloatingPoint(Precision[_type['precision']] as any); }
-function binaryFromJSON(_type: any) { return new Binary(); }
-function utf8FromJSON(_type: any) { return new Utf8(); }
-function boolFromJSON(_type: any) { return new Bool(); }
-function decimalFromJSON(_type: any) { return new Decimal(_type['scale'], _type['precision']); }
-function dateFromJSON(_type: any) { return new Date(DateUnit[_type['unit']] as any); }
-function timeFromJSON(_type: any) { return new Time(TimeUnit[_type['unit']] as any, _type['bitWidth'] as TimeBitWidth); }
-function timestampFromJSON(_type: any) { return new Timestamp(TimeUnit[_type['unit']] as any, _type['timezone']); }
-function intervalFromJSON(_type: any) { return new Interval(IntervalUnit[_type['unit']] as any); }
-function listFromJSON(_type: any) { return new List(); }
-function structFromJSON(_type: any) { return new Struct(); }
-function unionFromJSON(_type: any) { return new Union(_type['mode'], (_type['typeIdsArray'] || []) as Type[]); }
-function fixedSizeBinaryFromJSON(_type: any) { return new FixedSizeBinary(_type['byteWidth']); }
-function fixedSizeListFromJSON(_type: any) { return new FixedSizeList(_type['listSize']); }
-function mapFromJSON(_type: any) { return new Map_(_type['keysSorted']); }
diff --git a/js/src/format/types.ts b/js/src/format/types.ts
deleted file mode 100644
index 09df8ccbbdf7c..0000000000000
--- a/js/src/format/types.ts
+++ /dev/null
@@ -1,393 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-/* tslint:disable:class-name */
-
-import { align } from '../util/layout';
-import * as Schema_ from './fb/Schema';
-import * as Message_ from './fb/Message';
-import { flatbuffers } from 'flatbuffers';
-import Long = flatbuffers.Long;
-import Type = Schema_.org.apache.arrow.flatbuf.Type;
-import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
-import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
-import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
-import UnionMode = Schema_.org.apache.arrow.flatbuf.UnionMode;
-import Endianness = Schema_.org.apache.arrow.flatbuf.Endianness;
-import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
-import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
-import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
-
-export type IntBitWidth = 8 | 16 | 32 | 64;
-export type TimeBitWidth = IntBitWidth | 128;
-
-export interface VisitorNode {
-    accept(visitor: Visitor): any;
-}
-
-export abstract class Visitor<T = any> {
-    visit(node: VisitorNode): T {
-        return node.accept(this);
-    }
-    visitMany(nodes: VisitorNode[]): T[] {
-        return nodes.map((node) => this.visit(node));
-    }
-    abstract visitFooter(node: Footer): any;
-    abstract visitBlock(node: Block): any;
-    abstract visitMessage(node: Message): any;
-    abstract visitSchema(node: Schema): any;
-    abstract visitField(node: Field): any;
-    abstract visitBuffer(node: Buffer): any;
-    abstract visitFieldNode(node: FieldNode): any;
-    abstract visitRecordBatch(node: RecordBatch): any;
-    abstract visitDictionaryBatch(node: DictionaryBatch): any;
-    abstract visitDictionaryEncoding(node: DictionaryEncoding): any;
-    abstract visitNullFieldType(node: Null): any;
-    abstract visitIntFieldType(node: Int): any;
-    abstract visitFloatingPointFieldType(node: FloatingPoint): any;
-    abstract visitBinaryFieldType(node: Binary): any;
-    abstract visitBoolFieldType(node: Bool): any;
-    abstract visitUtf8FieldType(node: Utf8): any;
-    abstract visitDecimalFieldType(node: Decimal): any;
-    abstract visitDateFieldType(node: Date): any;
-    abstract visitTimeFieldType(node: Time): any;
-    abstract visitTimestampFieldType(node: Timestamp): any;
-    abstract visitIntervalFieldType(node: Interval): any;
-    abstract visitListFieldType(node: List): any;
-    abstract visitStructFieldType(node: Struct): any;
-    abstract visitUnionFieldType(node: Union): any;
-    abstract visitFixedSizeBinaryFieldType(node: FixedSizeBinary): any;
-    abstract visitFixedSizeListFieldType(node: FixedSizeList): any;
-    abstract visitMapFieldType(node: Map_): any;
-}
-
-export class Footer implements VisitorNode {
-    constructor(public dictionaryBatches: Block[], public recordBatches: Block[], public schema: Schema) {}
-    accept(visitor: Visitor): any {
-        return visitor.visitFooter(this);
-    }
-}
-
-export class Block implements VisitorNode {
-    constructor(public metaDataLength: number, public bodyLength: Long, public offset: Long) {}
-    accept(visitor: Visitor): any {
-        return visitor.visitBlock(this);
-    }
-}
-
-export class Message implements VisitorNode {
-    constructor(public version: MetadataVersion, public bodyLength: Long, public headerType: MessageHeader) {}
-    isSchema(): this is Schema { return this.headerType === MessageHeader.Schema; }
-    isRecordBatch(): this is RecordBatch { return this.headerType === MessageHeader.RecordBatch; }
-    isDictionaryBatch(): this is DictionaryBatch { return this.headerType === MessageHeader.DictionaryBatch; }
-    accept(visitor: Visitor): any {
-        visitor.visitMessage(this);
-    }
-}
-
-export class Schema extends Message {
-    public dictionaries: Map<string, Field>;
-    constructor(version: MetadataVersion, public fields: Field[], public customMetadata?: Map<string, string>, public endianness = Endianness.Little) {
-        super(version, Long.ZERO, MessageHeader.Schema);
-        const dictionaries = [] as Field[];
-        for (let f: Field, i = -1, n = fields.length; ++i < n;) {
-            if ((f = fields[i])) {
-                f.dictionary && dictionaries.push(f);
-                dictionaries.push(...f.dictionaries);
-            }
-        }
-        this.dictionaries = new Map<string, Field>(dictionaries.map<[string, Field]>((f) => [
-            f.dictionary!.dictionaryId.toFloat64().toString(), f
-        ]));
-    }
-    accept(visitor: Visitor): any {
-        return visitor.visitSchema(this);
-    }
-}
-
-export class RecordBatch extends Message {
-    constructor(version: MetadataVersion, public length: Long, public fieldNodes: FieldNode[], public buffers: Buffer[]) {
-        super(version, new Long(buffers.reduce((s, b) => align(s + b.length.low + (b.offset.low - s), 8), 0), 0), MessageHeader.RecordBatch);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitRecordBatch(this);
-    }
-}
-
-export class DictionaryBatch extends Message {
-    constructor(version: MetadataVersion, public dictionary: RecordBatch, public dictionaryId: Long, public isDelta: boolean) {
-        super(version, dictionary.bodyLength, MessageHeader.DictionaryBatch);
-    }
-    get fieldNodes(): FieldNode[] { return this.dictionary.fieldNodes; }
-    get buffers(): Buffer[] { return this.dictionary.buffers; }
-    accept(visitor: Visitor) {
-        return visitor.visitDictionaryBatch(this);
-    }
-    static atomicDictionaryId = 0;
-}
-
-export class Field implements VisitorNode {
-    public dictionaries: Field[];
-    constructor(public name: string,
-                public type: FieldType,
-                public typeType: Type,
-                public nullable = false,
-                public children: Field[] = [],
-                public metadata?: Map<string, string> | null,
-                public dictionary?: DictionaryEncoding | null) {
-        const dictionaries = [] as Field[];
-        for (let f: Field, i = -1, n = children.length; ++i < n;) {
-            if ((f = children[i])) {
-                f.dictionary && dictionaries.push(f);
-                dictionaries.push(...f.dictionaries);
-            }
-        }
-        this.dictionaries = dictionaries;
-    }
-    accept(visitor: Visitor): any {
-        return visitor.visitField(this);
-    }
-    indexField() {
-        return !this.dictionary ? this : new Field(
-            this.name,
-            this.dictionary.indexType, this.dictionary.indexType.type,
-            this.nullable, this.children, this.metadata, this.dictionary
-        );
-    }
-    toString() { return `Field name[${this.name}], nullable[${this.nullable}], type[${this.type.toString()}]`; }
-}
-
-export class Buffer implements VisitorNode {
-    constructor(public offset: Long, public length: Long) {}
-    accept(visitor: Visitor) {
-        return visitor.visitBuffer(this);
-    }
-}
-
-export class FieldNode implements VisitorNode {
-    constructor(public length: Long, public nullCount: Long) {}
-    accept(visitor: Visitor) {
-        return visitor.visitFieldNode(this);
-    }
-}
-
-export class DictionaryEncoding implements VisitorNode {
-    public isOrdered: boolean;
-    public dictionaryId: Long;
-    public indexType: Int;
-    constructor(indexType?: Int | null, dictionaryId?: Long | null, isOrdered?: boolean | null) {
-        this.isOrdered = isOrdered || false;
-        /* a dictionary index defaults to signed 32 bit int if unspecified */
-        this.indexType = indexType || new Int(true, 32);
-        this.dictionaryId = dictionaryId || new Long(DictionaryBatch.atomicDictionaryId++, 0);
-    }
-    accept(visitor: Visitor): any {
-        return visitor.visitDictionaryEncoding(this);
-    }
-}
-
-export abstract class FieldType implements VisitorNode {
-    constructor(public type: Type) {}
-    abstract accept(visitor: Visitor): any;
-    isNull(): this is Null { return this.type === Type.Null; }
-    isInt(): this is Int { return this.type === Type.Int; }
-    isFloatingPoint(): this is FloatingPoint { return this.type === Type.FloatingPoint; }
-    isBinary(): this is Binary { return this.type === Type.Binary; }
-    isUtf8(): this is Utf8 { return this.type === Type.Utf8; }
-    isBool(): this is Bool { return this.type === Type.Bool; }
-    isDecimal(): this is Decimal { return this.type === Type.Decimal; }
-    isDate(): this is Date { return this.type === Type.Date; }
-    isTime(): this is Time { return this.type === Type.Time; }
-    isTimestamp(): this is Timestamp { return this.type === Type.Timestamp; }
-    isInterval(): this is Interval { return this.type === Type.Interval; }
-    isList(): this is List { return this.type === Type.List; }
-    isStruct(): this is Struct { return this.type === Type.Struct_; }
-    isUnion(): this is Union { return this.type === Type.Union; }
-    isFixedSizeBinary(): this is FixedSizeBinary { return this.type === Type.FixedSizeBinary; }
-    isFixedSizeList(): this is FixedSizeList { return this.type === Type.FixedSizeList; }
-    isMap(): this is Map_ { return this.type === Type.Map; }
-}
-
-export class Null extends FieldType {
-    toString() { return `Null`; }
-    constructor() {
-        super(Type.Null);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitNullFieldType(this);
-    }
-}
-
-export class Int extends FieldType {
-    toString() { return `Int isSigned[${this.isSigned}], bitWidth[${this.bitWidth}]`; }
-    constructor(public isSigned: boolean, public bitWidth: IntBitWidth) {
-        super(Type.Int);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitIntFieldType(this);
-    }
-}
-
-export class FloatingPoint extends FieldType {
-    toString() { return `FloatingPoint precision`; }
-    constructor(public precision: Precision) {
-        super(Type.FloatingPoint);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitFloatingPointFieldType(this);
-    }
-}
-
-export class Binary extends FieldType {
-    toString() { return `Binary`; }
-    constructor() {
-        super(Type.Binary);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitBinaryFieldType(this);
-    }
-}
-
-export class Utf8 extends FieldType {
-    toString() { return `Utf8`; }
-    constructor() {
-        super(Type.Utf8);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitUtf8FieldType(this);
-    }
-}
-
-export class Bool extends FieldType {
-    toString() { return `Bool`; }
-    constructor() {
-        super(Type.Bool);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitBoolFieldType(this);
-    }
-}
-
-export class Decimal extends FieldType {
-    toString() { return `Decimal scale[${this.scale}], precision[${this.precision}]`; }
-    constructor(public scale: number, public precision: number) {
-        super(Type.Decimal);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitDecimalFieldType(this);
-    }
-}
-
-export class Date extends FieldType {
-    toString() { return `Date unit[${this.unit}]`; }
-    constructor(public unit: DateUnit) {
-        super(Type.Date);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitDateFieldType(this);
-    }
-}
-
-export class Time extends FieldType {
-    toString() { return `Time unit[${this.unit}], bitWidth[${this.bitWidth}]`; }
-    constructor(public unit: TimeUnit, public bitWidth: TimeBitWidth) {
-        super(Type.Time);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitTimeFieldType(this);
-    }
-}
-
-export class Timestamp extends FieldType {
-    toString() { return `Timestamp unit[${this.unit}], timezone[${this.timezone}]`; }
-    constructor(public unit: TimeUnit, public timezone?: string | null) {
-        super(Type.Timestamp);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitTimestampFieldType(this);
-    }
-}
-
-export class Interval extends FieldType {
-    toString() { return `Interval unit[${this.unit}]`; }
-    constructor(public unit: IntervalUnit) {
-        super(Type.Interval);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitIntervalFieldType(this);
-    }
-}
-
-export class List extends FieldType {
-    toString() { return `List`; }
-    constructor() {
-        super(Type.List);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitListFieldType(this);
-    }
-}
-
-export class Struct extends FieldType {
-    toString() { return `Struct`; }
-    constructor() {
-        super(Type.Struct_);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitStructFieldType(this);
-    }
-}
-
-export class Union extends FieldType {
-    toString() { return `Union mode[${this.mode}], typeIds[${this.typeIds}]`; }
-    constructor(public mode: UnionMode, public typeIds: Type[]) {
-        super(Type.Union);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitUnionFieldType(this);
-    }
-}
-
-export class FixedSizeBinary extends FieldType {
-    toString() { return `FixedSizeBinary byteWidth[${this.byteWidth}]`; }
-    constructor(public byteWidth: number) {
-        super(Type.FixedSizeBinary);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitFixedSizeBinaryFieldType(this);
-    }
-}
-
-export class FixedSizeList extends FieldType {
-    toString() { return `FixedSizeList listSize[${this.listSize}]`; }
-    constructor(public listSize: number) {
-        super(Type.FixedSizeList);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitFixedSizeListFieldType(this);
-    }
-}
-
-export class Map_ extends FieldType {
-    toString() { return `Map keysSorted[${this.keysSorted}]`; }
-    constructor(public keysSorted: boolean) {
-        super(Type.Map);
-    }
-    accept(visitor: Visitor) {
-        return visitor.visitMapFieldType(this);
-    }
-}
diff --git a/js/src/ipc/metadata.ts b/js/src/ipc/metadata.ts
new file mode 100644
index 0000000000000..88b7e52983b8e
--- /dev/null
+++ b/js/src/ipc/metadata.ts
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/* tslint:disable:class-name */
+
+import { align } from '../util/bit';
+import { Schema, Long, MessageHeader, MetadataVersion } from '../type';
+
+export class Footer {
+    constructor(public dictionaryBatches: FileBlock[], public recordBatches: FileBlock[], public schema: Schema) {}
+}
+
+export class FileBlock {
+    constructor(public metaDataLength: number, public bodyLength: Long, public offset: Long) {}
+}
+
+export class Message {
+    public bodyLength: number;
+    public version: MetadataVersion;
+    public headerType: MessageHeader;
+    constructor(version: MetadataVersion, bodyLength: Long | number, headerType: MessageHeader) {
+        this.version = version;
+        this.headerType = headerType;
+        this.bodyLength = typeof bodyLength === 'number' ? bodyLength : bodyLength.low;
+    }
+    static isSchema(m: Message): m is Schema { return m.headerType === MessageHeader.Schema; }
+    static isRecordBatch(m: Message): m is RecordBatchMetadata { return m.headerType === MessageHeader.RecordBatch; }
+    static isDictionaryBatch(m: Message): m is DictionaryBatch { return m.headerType === MessageHeader.DictionaryBatch; }
+}
+
+export class RecordBatchMetadata extends Message {
+    public length: number;
+    public nodes: FieldMetadata[];
+    public buffers: BufferMetadata[];
+    constructor(version: MetadataVersion, length: Long | number, nodes: FieldMetadata[], buffers: BufferMetadata[]) {
+        super(version, buffers.reduce((s, b) => align(s + b.length + (b.offset - s), 8), 0), MessageHeader.RecordBatch);
+        this.nodes = nodes;
+        this.buffers = buffers;
+        this.length = typeof length === 'number' ? length : length.low;
+    }
+}
+
+export class DictionaryBatch extends Message {
+    public id: number;
+    public isDelta: boolean;
+    public data: RecordBatchMetadata;
+    constructor(version: MetadataVersion, data: RecordBatchMetadata, id: Long | number, isDelta: boolean = false) {
+        super(version, data.bodyLength, MessageHeader.DictionaryBatch);
+        this.isDelta = isDelta;
+        this.data = data;
+        this.id = typeof id === 'number' ? id : id.low;
+    }
+    private static atomicDictionaryId = 0;
+    public static getId() { return DictionaryBatch.atomicDictionaryId++; }
+    public get nodes(): FieldMetadata[] { return this.data.nodes; }
+    public get buffers(): BufferMetadata[] { return this.data.buffers; }
+}
+
+export class BufferMetadata {
+    public offset: number;
+    public length: number;
+    constructor(offset: Long | number, length: Long | number) {
+        this.offset = typeof offset === 'number' ? offset : offset.low;
+        this.length = typeof length === 'number' ? length : length.low;
+    }
+}
+
+export class FieldMetadata {
+    public length: number;
+    public nullCount: number;
+    constructor(length: Long | number, nullCount: Long | number) {
+        this.length = typeof length === 'number' ? length : length.low;
+        this.nullCount = typeof nullCount === 'number' ? nullCount : nullCount.low;
+    }
+}
diff --git a/js/src/reader/arrow.ts b/js/src/ipc/reader/arrow.ts
similarity index 62%
rename from js/src/reader/arrow.ts
rename to js/src/ipc/reader/arrow.ts
index cf8a3d6a281a2..af535900cbf46 100644
--- a/js/src/reader/arrow.ts
+++ b/js/src/ipc/reader/arrow.ts
@@ -16,33 +16,33 @@
 // under the License.
 
 import { readJSON } from './json';
-import { readBuffers, readBuffersAsync } from './buffer';
-import { readVectors, readVectorsAsync } from './vector';
-import { Vector } from '../vector/vector';
+import { RecordBatch } from '../../recordbatch';
+import { readBuffers, readBuffersAsync } from './binary';
+import { readRecordBatches, readRecordBatchesAsync, TypeDataLoader } from './vector';
+import { Schema } from '../../type';
+import { Message } from '../metadata';
 
-export { readJSON };
+export { readJSON, RecordBatch };
 export { readBuffers, readBuffersAsync };
-export { readVectors, readVectorsAsync };
+export { readRecordBatches, readRecordBatchesAsync };
 
 export function* read(sources: Iterable<Uint8Array | Buffer | string> | object | string) {
     let input: any = sources;
-    let batches: Iterable<Vector[]>;
+    let messages: Iterable<{ schema: Schema, message: Message, loader: TypeDataLoader }>;
     if (typeof input === 'string') {
         try { input = JSON.parse(input); }
         catch (e) { input = sources; }
     }
     if (!input || typeof input !== 'object') {
-        batches = (typeof input === 'string') ? readVectors(readBuffers([input])) : [];
+        messages = (typeof input === 'string') ? readBuffers([input]) : [];
     } else {
-        batches = (typeof input[Symbol.iterator] === 'function')
-            ? readVectors(readBuffers(input))
-            : readVectors(readJSON(input));
+        messages = (typeof input[Symbol.iterator] === 'function') ? readBuffers(input) : readJSON(input);
     }
-    yield* batches;
+    yield* readRecordBatches(messages);
 }
 
 export async function* readAsync(sources: AsyncIterable<Uint8Array | Buffer | string>) {
-    for await (let vectors of readVectorsAsync(readBuffersAsync(sources))) {
-        yield vectors;
+    for await (let recordBatch of readRecordBatchesAsync(readBuffersAsync(sources))) {
+        yield recordBatch;
     }
 }
diff --git a/js/src/ipc/reader/binary.ts b/js/src/ipc/reader/binary.ts
new file mode 100644
index 0000000000000..6e3c7fc5cf080
--- /dev/null
+++ b/js/src/ipc/reader/binary.ts
@@ -0,0 +1,449 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from '../../vector';
+import { flatbuffers } from 'flatbuffers';
+import { TypeDataLoader } from './vector';
+import { Message, Footer, FileBlock, RecordBatchMetadata, DictionaryBatch, BufferMetadata, FieldMetadata, } from '../metadata';
+import {
+    Schema, Field,
+    DataType, Dictionary,
+    Null, TimeBitWidth,
+    Binary, Bool, Utf8, Decimal,
+    Date_, Time, Timestamp, Interval,
+    List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
+} from '../../type';
+
+import {
+    Int8,  Uint8,
+    Int16, Uint16,
+    Int32, Uint32,
+    Int64, Uint64,
+    Float16, Float64, Float32,
+} from '../../type';
+
+import ByteBuffer = flatbuffers.ByteBuffer;
+
+type MessageReader = (bb: ByteBuffer) => IterableIterator<RecordBatchMetadata | DictionaryBatch>;
+
+export function* readBuffers<T extends Uint8Array | Buffer | string>(sources: Iterable<T> | Uint8Array | Buffer | string) {
+    let schema: Schema | null = null;
+    let dictionaries = new Map<number, Vector>();
+    let readMessages: MessageReader | null = null;
+    if (ArrayBuffer.isView(sources) || typeof sources === 'string') {
+        sources = [sources as T];
+    }
+    for (const source of sources) {
+        const bb = toByteBuffer(source);
+        if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) {
+            for (const message of readMessages(bb)) {
+                yield {
+                    schema, message,
+                    loader: new BinaryDataLoader(
+                        bb,
+                        arrayIterator(message.nodes),
+                        arrayIterator(message.buffers),
+                        dictionaries
+                    )
+                };
+            }
+        }
+    }
+}
+
+export async function* readBuffersAsync<T extends Uint8Array | Buffer | string>(sources: AsyncIterable<T>) {
+    let schema: Schema | null = null;
+    let dictionaries = new Map<number, Vector>();
+    let readMessages: MessageReader | null = null;
+    for await (const source of sources) {
+        const bb = toByteBuffer(source);
+        if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) {
+            for (const message of readMessages(bb)) {
+                yield {
+                    schema, message,
+                    loader: new BinaryDataLoader(
+                        bb,
+                        arrayIterator(message.nodes),
+                        arrayIterator(message.buffers),
+                        dictionaries
+                    )
+                };
+            }
+        }
+    }
+}
+
+export class BinaryDataLoader extends TypeDataLoader {
+    private bytes: Uint8Array;
+    private messageOffset: number;
+    constructor(bb: ByteBuffer, nodes: Iterator<FieldMetadata>, buffers: Iterator<BufferMetadata>, dictionaries: Map<number, Vector>) {
+        super(nodes, buffers, dictionaries);
+        this.bytes = bb.bytes();
+        this.messageOffset = bb.position();
+    }
+    protected readOffsets<T extends DataType>(type: T, buffer?: BufferMetadata) { return this.readData(type, buffer); }
+    protected readTypeIds<T extends DataType>(type: T, buffer?: BufferMetadata) { return this.readData(type, buffer); }
+    protected readData<T extends DataType>(_type: T, { length, offset }: BufferMetadata = this.getBufferMetadata()) {
+        return new Uint8Array(this.bytes.buffer, this.bytes.byteOffset + this.messageOffset + offset, length);
+    }
+}
+
+function* arrayIterator(arr: Array<any>) { yield* arr; }
+
+function toByteBuffer(bytes?: Uint8Array | Buffer | string) {
+    let arr: Uint8Array = bytes as any || new Uint8Array(0);
+    if (typeof bytes === 'string') {
+        arr = new Uint8Array(bytes.length);
+        for (let i = -1, n = bytes.length; ++i < n;) {
+            arr[i] = bytes.charCodeAt(i);
+        }
+        return new ByteBuffer(arr);
+    }
+    return new ByteBuffer(arr);
+}
+
+function readSchema(bb: ByteBuffer) {
+    let schema: Schema, readMessages, footer: Footer | null;
+    if (footer = readFileSchema(bb)) {
+        schema = footer.schema;
+        readMessages = readFileMessages(footer);
+    } else if (schema = readStreamSchema(bb)!) {
+        readMessages = readStreamMessages;
+    } else {
+        throw new Error('Invalid Arrow buffer');
+    }
+    return { schema, readMessages };
+}
+
+const PADDING = 4;
+const MAGIC_STR = 'ARROW1';
+const MAGIC = new Uint8Array(MAGIC_STR.length);
+for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
+    MAGIC[i] = MAGIC_STR.charCodeAt(i);
+}
+
+function checkForMagicArrowString(buffer: Uint8Array, index = 0) {
+    for (let i = -1, n = MAGIC.length; ++i < n;) {
+        if (MAGIC[i] !== buffer[index + i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+const magicLength = MAGIC.length;
+const magicAndPadding = magicLength + PADDING;
+const magicX2AndPadding = magicLength * 2 + PADDING;
+
+function readStreamSchema(bb: ByteBuffer) {
+    if (!checkForMagicArrowString(bb.bytes(), 0)) {
+        for (const message of readMessages(bb)) {
+            if (Message.isSchema(message)) {
+                return message as Schema;
+            }
+        }
+    }
+    return null;
+}
+
+function* readStreamMessages(bb: ByteBuffer) {
+    for (const message of readMessages(bb)) {
+        if (Message.isRecordBatch(message)) {
+            yield message;
+        } else if (Message.isDictionaryBatch(message)) {
+            yield message;
+        } else {
+            continue;
+        }
+        // position the buffer after the body to read the next message
+        bb.setPosition(bb.position() + message.bodyLength);
+    }
+}
+
+function readFileSchema(bb: ByteBuffer) {
+    let fileLength = bb.capacity(), footerLength: number, footerOffset: number;
+    if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
+        (!checkForMagicArrowString(bb.bytes(), 0) /*                        Missing magic start    */) ||
+        (!checkForMagicArrowString(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
+        (/*                                                    Invalid footer length  */
+        (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
+        (footerLength + magicX2AndPadding > fileLength))) {
+        return null;
+    }
+    bb.setPosition(footerOffset - footerLength);
+    return footerFromByteBuffer(bb);
+}
+
+function readFileMessages(footer: Footer) {
+    return function* (bb: ByteBuffer) {
+        for (let i = -1, batches = footer.dictionaryBatches, n = batches.length; ++i < n;) {
+            bb.setPosition(batches[i].offset.low);
+            yield readMessage(bb, bb.readInt32(bb.position())) as DictionaryBatch;
+        }
+        for (let i = -1, batches = footer.recordBatches, n = batches.length; ++i < n;) {
+            bb.setPosition(batches[i].offset.low);
+            yield readMessage(bb, bb.readInt32(bb.position())) as RecordBatchMetadata;
+        }
+    };
+}
+
+function* readMessages(bb: ByteBuffer) {
+    let length: number, message: Schema | RecordBatchMetadata | DictionaryBatch;
+    while (bb.position() < bb.capacity() &&
+          (length = bb.readInt32(bb.position())) > 0) {
+        if (message = readMessage(bb, length)!) {
+            yield message;
+        }
+    }
+}
+
+function readMessage(bb: ByteBuffer, length: number) {
+    bb.setPosition(bb.position() + PADDING);
+    const message = messageFromByteBuffer(bb);
+    bb.setPosition(bb.position() + length);
+    return message;
+}
+
+import * as File_ from '../../fb/File';
+import * as Schema_ from '../../fb/Schema';
+import * as Message_ from '../../fb/Message';
+
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
+import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
+import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
+import _Footer = File_.org.apache.arrow.flatbuf.Footer;
+import _Block = File_.org.apache.arrow.flatbuf.Block;
+import _Message = Message_.org.apache.arrow.flatbuf.Message;
+import _Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+import _Field = Schema_.org.apache.arrow.flatbuf.Field;
+import _RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
+import _DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch;
+import _FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+import _Buffer = Schema_.org.apache.arrow.flatbuf.Buffer;
+import _DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
+import _Null = Schema_.org.apache.arrow.flatbuf.Null;
+import _Int = Schema_.org.apache.arrow.flatbuf.Int;
+import _FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
+import _Binary = Schema_.org.apache.arrow.flatbuf.Binary;
+import _Bool = Schema_.org.apache.arrow.flatbuf.Bool;
+import _Utf8 = Schema_.org.apache.arrow.flatbuf.Utf8;
+import _Decimal = Schema_.org.apache.arrow.flatbuf.Decimal;
+import _Date = Schema_.org.apache.arrow.flatbuf.Date;
+import _Time = Schema_.org.apache.arrow.flatbuf.Time;
+import _Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp;
+import _Interval = Schema_.org.apache.arrow.flatbuf.Interval;
+import _List = Schema_.org.apache.arrow.flatbuf.List;
+import _Struct = Schema_.org.apache.arrow.flatbuf.Struct_;
+import _Union = Schema_.org.apache.arrow.flatbuf.Union;
+import _FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary;
+import _FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
+import _Map = Schema_.org.apache.arrow.flatbuf.Map;
+
+function footerFromByteBuffer(bb: ByteBuffer) {
+    const dictionaryFields = new Map<number, Field<Dictionary>>();
+    const f = _Footer.getRootAsFooter(bb), s = f.schema()!;
+    return new Footer(
+        dictionaryBatchesFromFooter(f), recordBatchesFromFooter(f),
+        new Schema(fieldsFromSchema(s, dictionaryFields), customMetadata(s), f.version(), dictionaryFields)
+    );
+}
+
+function messageFromByteBuffer(bb: ByteBuffer) {
+    const m = _Message.getRootAsMessage(bb)!, type = m.headerType(), version = m.version();
+    switch (type) {
+        case MessageHeader.Schema: return schemaFromMessage(version, m.header(new _Schema())!, new Map());
+        case MessageHeader.RecordBatch: return recordBatchFromMessage(version, m.header(new _RecordBatch())!);
+        case MessageHeader.DictionaryBatch: return dictionaryBatchFromMessage(version, m.header(new _DictionaryBatch())!);
+    }
+    return null;
+    // throw new Error(`Unrecognized Message type '${type}'`);
+}
+
+function schemaFromMessage(version: MetadataVersion, s: _Schema, dictionaryFields: Map<number, Field<Dictionary>>) {
+    return new Schema(fieldsFromSchema(s, dictionaryFields), customMetadata(s), version, dictionaryFields);
+}
+
+function recordBatchFromMessage(version: MetadataVersion, b: _RecordBatch) {
+    return new RecordBatchMetadata(version, b.length(), fieldNodesFromRecordBatch(b), buffersFromRecordBatch(b, version));
+}
+
+function dictionaryBatchFromMessage(version: MetadataVersion, d: _DictionaryBatch) {
+    return new DictionaryBatch(version, recordBatchFromMessage(version, d.data()!), d.id(), d.isDelta());
+}
+
+function dictionaryBatchesFromFooter(f: _Footer) {
+    const blocks = [] as FileBlock[];
+    for (let b: _Block, i = -1, n = f && f.dictionariesLength(); ++i < n;) {
+        if (b = f.dictionaries(i)!) {
+            blocks.push(new FileBlock(b.metaDataLength(), b.bodyLength(), b.offset()));
+        }
+    }
+    return blocks;
+}
+
+function recordBatchesFromFooter(f: _Footer) {
+    const blocks = [] as FileBlock[];
+    for (let b: _Block, i = -1, n = f && f.recordBatchesLength(); ++i < n;) {
+        if (b = f.recordBatches(i)!) {
+            blocks.push(new FileBlock(b.metaDataLength(), b.bodyLength(), b.offset()));
+        }
+    }
+    return blocks;
+}
+
+function fieldsFromSchema(s: _Schema, dictionaryFields: Map<number, Field<Dictionary>> | null) {
+    const fields = [] as Field[];
+    for (let i = -1, c: Field | null, n = s && s.fieldsLength(); ++i < n;) {
+        if (c = field(s.fields(i)!, dictionaryFields)) {
+            fields.push(c);
+        }
+    }
+    return fields;
+}
+
+function fieldsFromField(f: _Field, dictionaryFields: Map<number, Field<Dictionary>> | null) {
+    const fields = [] as Field[];
+    for (let i = -1, c: Field | null, n = f && f.childrenLength(); ++i < n;) {
+        if (c = field(f.children(i)!, dictionaryFields)) {
+            fields.push(c);
+        }
+    }
+    return fields;
+}
+
+function fieldNodesFromRecordBatch(b: _RecordBatch) {
+    const fieldNodes = [] as FieldMetadata[];
+    for (let i = -1, n = b.nodesLength(); ++i < n;) {
+        fieldNodes.push(fieldNodeFromRecordBatch(b.nodes(i)!));
+    }
+    return fieldNodes;
+}
+
+function buffersFromRecordBatch(b: _RecordBatch, version: MetadataVersion) {
+    const buffers = [] as BufferMetadata[];
+    for (let i = -1, n = b.buffersLength(); ++i < n;) {
+        let buffer = b.buffers(i)!;
+        // If this Arrow buffer was written before version 4,
+        // advance the buffer's bb_pos 8 bytes to skip past
+        // the now-removed page id field.
+        if (version < MetadataVersion.V4) {
+            buffer.bb_pos += (8 * (i + 1));
+        }
+        buffers.push(bufferFromRecordBatch(buffer));
+    }
+    return buffers;
+}
+
+function field(f: _Field, dictionaryFields: Map<number, Field<Dictionary>> | null) {
+    let name = f.name()!;
+    let field: Field | void;
+    let nullable = f.nullable();
+    let metadata = customMetadata(f);
+    let dataType: DataType<any> | null;
+    let keysMeta: _Int | null, id: number;
+    let dictMeta: _DictionaryEncoding | null;
+    if (!dictionaryFields || !(dictMeta = f.dictionary())) {
+        if (dataType = typeFromField(f, fieldsFromField(f, dictionaryFields))) {
+            field = new Field(name, dataType, nullable, metadata);
+        }
+    } else if (dataType = dictionaryFields.has(id = dictMeta.id().low)
+                        ? dictionaryFields.get(id)!.type.dictionary
+                        : typeFromField(f, fieldsFromField(f, null))) {
+        dataType = new Dictionary(dataType,
+            // a dictionary index defaults to signed 32 bit int if unspecified
+            (keysMeta = dictMeta.indexType()) ? intFromField(keysMeta)! : new Int32(),
+            id, dictMeta.isOrdered()
+        );
+        field = new Field(name, dataType, nullable, metadata);
+        dictionaryFields.has(id) || dictionaryFields.set(id, field as Field<Dictionary>);
+    }
+    return field || null;
+}
+
+function customMetadata(parent?: _Schema | _Field | null) {
+    const data = new Map<string, string>();
+    if (parent) {
+        for (let entry, key, i = -1, n = parent.customMetadataLength() | 0; ++i < n;) {
+            if ((entry = parent.customMetadata(i)) && (key = entry.key()) != null) {
+                data.set(key, entry.value()!);
+            }
+        }
+    }
+    return data;
+}
+
+function fieldNodeFromRecordBatch(f: _FieldNode) {
+    return new FieldMetadata(f.length(), f.nullCount());
+}
+
+function bufferFromRecordBatch(b: _Buffer) {
+    return new BufferMetadata(b.offset(), b.length());
+}
+
+function typeFromField(f: _Field, children?: Field[]): DataType<any> | null {
+    switch (f.typeType()) {
+        case Type.NONE: return null;
+        case Type.Null: return nullFromField(f.type(new _Null())!);
+        case Type.Int: return intFromField(f.type(new _Int())!);
+        case Type.FloatingPoint: return floatFromField(f.type(new _FloatingPoint())!);
+        case Type.Binary: return binaryFromField(f.type(new _Binary())!);
+        case Type.Utf8: return utf8FromField(f.type(new _Utf8())!);
+        case Type.Bool: return boolFromField(f.type(new _Bool())!);
+        case Type.Decimal: return decimalFromField(f.type(new _Decimal())!);
+        case Type.Date: return dateFromField(f.type(new _Date())!);
+        case Type.Time: return timeFromField(f.type(new _Time())!);
+        case Type.Timestamp: return timestampFromField(f.type(new _Timestamp())!);
+        case Type.Interval: return intervalFromField(f.type(new _Interval())!);
+        case Type.List: return listFromField(f.type(new _List())!, children || []);
+        case Type.Struct_: return structFromField(f.type(new _Struct())!, children || []);
+        case Type.Union: return unionFromField(f.type(new _Union())!, children || []);
+        case Type.FixedSizeBinary: return fixedSizeBinaryFromField(f.type(new _FixedSizeBinary())!);
+        case Type.FixedSizeList: return fixedSizeListFromField(f.type(new _FixedSizeList())!, children || []);
+        case Type.Map: return mapFromField(f.type(new _Map())!, children || []);
+    }
+    throw new Error(`Unrecognized type ${f.typeType()}`);
+}
+
+function nullFromField           (_type: _Null)                             { return new Null();                                                                }
+function intFromField            (_type: _Int)                              { switch (_type.bitWidth()) {
+                                                                                  case  8: return _type.isSigned() ? new  Int8() : new  Uint8();
+                                                                                  case 16: return _type.isSigned() ? new Int16() : new Uint16();
+                                                                                  case 32: return _type.isSigned() ? new Int32() : new Uint32();
+                                                                                  case 64: return _type.isSigned() ? new Int64() : new Uint64();
+                                                                              }
+                                                                              return null;                                                                      }
+function floatFromField          (_type: _FloatingPoint)                    { switch (_type.precision()) {
+                                                                                  case Precision.HALF: return new Float16();
+                                                                                  case Precision.SINGLE: return new Float32();
+                                                                                  case Precision.DOUBLE: return new Float64();
+                                                                              }
+                                                                              return null;                                                                      }
+function binaryFromField         (_type: _Binary)                           { return new Binary();                                                              }
+function utf8FromField           (_type: _Utf8)                             { return new Utf8();                                                                }
+function boolFromField           (_type: _Bool)                             { return new Bool();                                                                }
+function decimalFromField        (_type: _Decimal)                          { return new Decimal(_type.scale(), _type.precision());                             }
+function dateFromField           (_type: _Date)                             { return new Date_(_type.unit());                                                   }
+function timeFromField           (_type: _Time)                             { return new Time(_type.unit(), _type.bitWidth() as TimeBitWidth);                  }
+function timestampFromField      (_type: _Timestamp)                        { return new Timestamp(_type.unit(), _type.timezone());                             }
+function intervalFromField       (_type: _Interval)                         { return new Interval(_type.unit());                                                }
+function listFromField           (_type: _List, children: Field[])          { return new List(children);                                                        }
+function structFromField         (_type: _Struct, children: Field[])        { return new Struct(children);                                                      }
+function unionFromField          (_type: _Union, children: Field[])         { return new Union(_type.mode(), (_type.typeIdsArray() || []) as Type[], children); }
+function fixedSizeBinaryFromField(_type: _FixedSizeBinary)                  { return new FixedSizeBinary(_type.byteWidth());                                    }
+function fixedSizeListFromField  (_type: _FixedSizeList, children: Field[]) { return new FixedSizeList(_type.listSize(), children);                             }
+function mapFromField            (_type: _Map, children: Field[])           { return new Map_(_type.keysSorted(), children);                                    }
diff --git a/js/src/ipc/reader/json.ts b/js/src/ipc/reader/json.ts
new file mode 100644
index 0000000000000..10819986f6d33
--- /dev/null
+++ b/js/src/ipc/reader/json.ts
@@ -0,0 +1,323 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from '../../vector';
+import { flatbuffers } from 'flatbuffers';
+import { TypeDataLoader } from './vector';
+import { packBools } from '../../util/bit';
+import * as IntUtil from '../../util/int';
+import { TextEncoder } from 'text-encoding-utf-8';
+import { RecordBatchMetadata, DictionaryBatch, BufferMetadata, FieldMetadata } from '../metadata';
+import {
+    Schema, Field,
+    DataType, Dictionary,
+    Null, TimeBitWidth,
+    Binary, Bool, Utf8, Decimal,
+    Date_, Time, Timestamp, Interval,
+    List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
+} from '../../type';
+
+import {
+    Int8,  Uint8,
+    Int16, Uint16,
+    Int32, Uint32,
+    Int64, Uint64,
+    Float16, Float64, Float32,
+} from '../../type';
+
+import Long = flatbuffers.Long;
+
+export function* readJSON(json: any) {
+    const schema = schemaFromJSON(json['schema']);
+    const dictionaries = new Map<number, Vector>();
+    for (const batch of (json['dictionaries'] || [])) {
+        const message = dictionaryBatchFromJSON(batch);
+        yield {
+            schema, message,
+            loader: new JSONDataLoader(
+                flattenDataSources(batch['data']['columns']),
+                arrayIterator(message.nodes),
+                arrayIterator(message.buffers),
+                dictionaries
+            )
+        };
+    }
+    for (const batch of (json['batches'] || [])) {
+        const message = recordBatchFromJSON(batch);
+        yield {
+            schema, message,
+            loader: new JSONDataLoader(
+                flattenDataSources(batch['columns']),
+                arrayIterator(message.nodes),
+                arrayIterator(message.buffers),
+                dictionaries
+            )
+        };
+    }
+}
+
+function* arrayIterator(arr: Array<any>) { yield* arr; }
+function flattenDataSources(xs: any[]): any[][] {
+    return (xs || []).reduce<any[][]>((buffers, column: any) => [
+        ...buffers,
+        ...(column['VALIDITY'] && [column['VALIDITY']] || []),
+        ...(column['OFFSET'] && [column['OFFSET']] || []),
+        ...(column['DATA'] && [column['DATA']] || []),
+        ...flattenDataSources(column['children'])
+    ], [] as any[][]);
+}
+
+const utf8Encoder = new TextEncoder('utf-8');
+
+export class JSONDataLoader extends TypeDataLoader {
+    constructor(private sources: any[][], nodes: Iterator<FieldMetadata>, buffers: Iterator<BufferMetadata>, dictionaries: Map<number, Vector>) {
+        super(nodes, buffers, dictionaries);
+    }
+    protected readNullBitmap<T extends DataType>(_type: T, nullCount: number, { offset } = this.getBufferMetadata()) {
+        return nullCount <= 0 ? new Uint8Array(0) : packBools(this.sources[offset]);
+    }
+    protected readOffsets<T extends DataType>(_type: T, { offset }: BufferMetadata = this.getBufferMetadata()) {
+        return new Int32Array(this.sources[offset]);
+    }
+    protected readTypeIds<T extends DataType>(_type: T, { offset }: BufferMetadata = this.getBufferMetadata()) {
+        return new Int8Array(this.sources[offset]);
+    }
+    protected readData<T extends DataType>(type: T, { offset }: BufferMetadata = this.getBufferMetadata()) {
+        const { sources } = this;
+        if (DataType.isTimestamp(type) === true) {
+            return new Uint8Array(int64DataFromJSON(sources[offset] as string[]));
+        } else if ((DataType.isInt(type) || DataType.isTime(type)) && type.bitWidth === 64) {
+            return new Uint8Array(int64DataFromJSON(sources[offset] as string[]));
+        } else if (DataType.isDate(type) && type.unit === DateUnit.MILLISECOND) {
+            return new Uint8Array(int64DataFromJSON(sources[offset] as string[]));
+        } else if (DataType.isDecimal(type) === true) {
+            return new Uint8Array(decimalDataFromJSON(sources[offset] as string[]));
+        } else if (DataType.isBinary(type) === true) {
+            return new Uint8Array(binaryDataFromJSON(sources[offset] as string[]));
+        } else if (DataType.isBool(type) === true) {
+            return new Uint8Array(packBools(sources[offset] as number[]).buffer);
+        } else if (DataType.isUtf8(type) === true) {
+            return utf8Encoder.encode((sources[offset] as string[]).join(''));
+        } else {
+            return toTypedArray(type.ArrayType, sources[offset].map((x) => +x)) as any;
+        }
+    }
+}
+
+function int64DataFromJSON(values: string[]) {
+    const data = new Uint32Array(values.length * 2);
+    for (let i = -1, n = values.length; ++i < n;) {
+        // Force all values (even numbers) to be parsed as strings since
+        // pulling out high and low bits seems to lose precision sometimes
+        // For example:
+        //     > -4613034156400212000 >>> 0
+        //     721782784
+        // The correct lower 32-bits are 721782752
+        IntUtil.Int64.fromString(values[i].toString(), new Uint32Array(data.buffer, data.byteOffset + 2 * i * 4, 2));
+    }
+    return data.buffer;
+}
+
+function decimalDataFromJSON(values: string[]) {
+    const data = new Uint32Array(values.length * 4);
+    for (let i = -1, n = values.length; ++i < n;) {
+        IntUtil.Int128.fromString(values[i], new Uint32Array(data.buffer, data.byteOffset + 4 * 4 * i, 4));
+    }
+    return data.buffer;
+}
+
+function binaryDataFromJSON(values: string[]) {
+    // "DATA": ["49BC7D5B6C47D2","3F5FB6D9322026"]
+    // There are definitely more efficient ways to do this... but it gets the
+    // job done.
+    const joined = values.join('');
+    const data = new Uint8Array(joined.length / 2);
+    for (let i = 0; i < joined.length; i += 2) {
+        data[i >> 1] = parseInt(joined.substr(i, 2), 16);
+    }
+    return data.buffer;
+}
+
+import * as Schema_ from '../../fb/Schema';
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
+import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
+import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
+import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
+import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
+import { toTypedArray } from '../../data';
+
+function schemaFromJSON(s: any): Schema {
+    const dictionaryFields = new Map<number, Field<Dictionary>>();
+    return new Schema(
+        fieldsFromJSON(s['fields'], dictionaryFields),
+        customMetadata(s['customMetadata']),
+        MetadataVersion.V4, dictionaryFields
+    );
+}
+
+function recordBatchFromJSON(b: any): RecordBatchMetadata {
+    return new RecordBatchMetadata(
+        MetadataVersion.V4,
+        b['count'],
+        fieldNodesFromJSON(b['columns']),
+        buffersFromJSON(b['columns'])
+    );
+}
+
+function dictionaryBatchFromJSON(b: any): DictionaryBatch {
+    return new DictionaryBatch(
+        MetadataVersion.V4,
+        recordBatchFromJSON(b['data']),
+        b['id'], b['isDelta']
+    );
+}
+
+function fieldsFromJSON(fs: any[], dictionaryFields: Map<number, Field<Dictionary>> | null): Field[] {
+    return (fs || [])
+        .map((f) => fieldFromJSON(f, dictionaryFields))
+        .filter((f) => f != null) as Field[];
+}
+
+function fieldNodesFromJSON(xs: any[]): FieldMetadata[] {
+    return (xs || []).reduce<FieldMetadata[]>((fieldNodes, column: any) => [
+        ...fieldNodes,
+        new FieldMetadata(
+            new Long(column['count'], 0),
+            new Long(nullCountFromJSON(column['VALIDITY']), 0)
+        ),
+        ...fieldNodesFromJSON(column['children'])
+    ], [] as FieldMetadata[]);
+}
+
+function buffersFromJSON(xs: any[], buffers: BufferMetadata[] = []): BufferMetadata[] {
+    for (let i = -1, n = (xs || []).length; ++i < n;) {
+        const column = xs[i];
+        column['VALIDITY'] && buffers.push(new BufferMetadata(new Long(buffers.length, 0), new Long(column['VALIDITY'].length, 0)));
+        column['OFFSET'] && buffers.push(new BufferMetadata(new Long(buffers.length, 0), new Long(column['OFFSET'].length, 0)));
+        column['DATA'] && buffers.push(new BufferMetadata(new Long(buffers.length, 0), new Long(column['DATA'].length, 0)));
+        buffers = buffersFromJSON(column['children'], buffers);
+    }
+    return buffers;
+}
+
+function nullCountFromJSON(validity: number[]) {
+    return (validity || []).reduce((sum, val) => sum + +(val === 0), 0);
+}
+
+function fieldFromJSON(f: any, dictionaryFields: Map<number, Field<Dictionary>> | null) {
+    let name = f['name'];
+    let field: Field | void;
+    let nullable = f['nullable'];
+    let dataType: DataType<any> | null;
+    let id: number, keysMeta: any, dictMeta: any;
+    let metadata = customMetadata(f['customMetadata']);
+    if (!dictionaryFields || !(dictMeta = f['dictionary'])) {
+        if (dataType = typeFromJSON(f['type'], fieldsFromJSON(f['children'], dictionaryFields))) {
+            field = new Field(name, dataType, nullable, metadata);
+        }
+    } else if (dataType = dictionaryFields.has(id = dictMeta['id'])
+                        ? dictionaryFields.get(id)!.type.dictionary
+                        : typeFromJSON(f['type'], fieldsFromJSON(f['children'], null))) {
+        dataType = new Dictionary(dataType,
+            // a dictionary index defaults to signed 32 bit int if unspecified
+            (keysMeta = dictMeta['indexType']) ? intFromJSON(keysMeta)! : new Int32(),
+            id, dictMeta['isOrdered']
+        );
+        field = new Field(name, dataType, nullable, metadata);
+        dictionaryFields.has(id) || dictionaryFields.set(id, field as Field<Dictionary>);
+    }
+    return field || null;
+}
+
+function customMetadata(metadata?: any) {
+    return new Map<string, string>(Object.entries(metadata || {}));
+}
+
+const namesToTypeMap: { [n: string]: Type }  = {
+    'NONE': Type.NONE,
+    'null': Type.Null,
+    'int': Type.Int,
+    'floatingpoint': Type.FloatingPoint,
+    'binary': Type.Binary,
+    'bool': Type.Bool,
+    'utf8': Type.Utf8,
+    'decimal': Type.Decimal,
+    'date': Type.Date,
+    'time': Type.Time,
+    'timestamp': Type.Timestamp,
+    'interval': Type.Interval,
+    'list': Type.List,
+    'struct': Type.Struct_,
+    'union': Type.Union,
+    'fixedsizebinary': Type.FixedSizeBinary,
+    'fixedsizelist': Type.FixedSizeList,
+    'map': Type.Map,
+};
+
+function typeFromJSON(t: any, children?: Field[]) {
+    switch (namesToTypeMap[t['name']]) {
+        case Type.NONE: return null;
+        case Type.Null: return nullFromJSON(t);
+        case Type.Int: return intFromJSON(t);
+        case Type.FloatingPoint: return floatingPointFromJSON(t);
+        case Type.Binary: return binaryFromJSON(t);
+        case Type.Utf8: return utf8FromJSON(t);
+        case Type.Bool: return boolFromJSON(t);
+        case Type.Decimal: return decimalFromJSON(t);
+        case Type.Date: return dateFromJSON(t);
+        case Type.Time: return timeFromJSON(t);
+        case Type.Timestamp: return timestampFromJSON(t);
+        case Type.Interval: return intervalFromJSON(t);
+        case Type.List: return listFromJSON(t, children || []);
+        case Type.Struct_: return structFromJSON(t, children || []);
+        case Type.Union: return unionFromJSON(t, children || []);
+        case Type.FixedSizeBinary: return fixedSizeBinaryFromJSON(t);
+        case Type.FixedSizeList: return fixedSizeListFromJSON(t, children || []);
+        case Type.Map: return mapFromJSON(t, children || []);
+    }
+    throw new Error(`Unrecognized type ${t['name']}`);
+}
+
+function nullFromJSON           (_type: any)                    { return new Null();                                                                  }
+function intFromJSON            (_type: any)                    { switch (_type['bitWidth']) {
+                                                                      case  8: return _type['isSigned'] ? new  Int8() : new  Uint8();
+                                                                      case 16: return _type['isSigned'] ? new Int16() : new Uint16();
+                                                                      case 32: return _type['isSigned'] ? new Int32() : new Uint32();
+                                                                      case 64: return _type['isSigned'] ? new Int64() : new Uint64();
+                                                                  }
+                                                                  return null;                                                                        }
+function floatingPointFromJSON  (_type: any)                    { switch (Precision[_type['precision']] as any) {
+                                                                      case Precision.HALF: return new Float16();
+                                                                      case Precision.SINGLE: return new Float32();
+                                                                      case Precision.DOUBLE: return new Float64();
+                                                                  }
+                                                                  return null;                                                                        }
+function binaryFromJSON         (_type: any)                    { return new Binary();                                                                }
+function utf8FromJSON           (_type: any)                    { return new Utf8();                                                                  }
+function boolFromJSON           (_type: any)                    { return new Bool();                                                                  }
+function decimalFromJSON        (_type: any)                    { return new Decimal(_type['scale'], _type['precision']);                             }
+function dateFromJSON           (_type: any)                    { return new Date_(DateUnit[_type['unit']] as any);                                   }
+function timeFromJSON           (_type: any)                    { return new Time(TimeUnit[_type['unit']] as any, _type['bitWidth'] as TimeBitWidth); }
+function timestampFromJSON      (_type: any)                    { return new Timestamp(TimeUnit[_type['unit']] as any, _type['timezone']);            }
+function intervalFromJSON       (_type: any)                    { return new Interval(IntervalUnit[_type['unit']] as any);                            }
+function listFromJSON           (_type: any, children: Field[]) { return new List(children);                                                          }
+function structFromJSON         (_type: any, children: Field[]) { return new Struct(children);                                                        }
+function unionFromJSON          (_type: any, children: Field[]) { return new Union(_type['mode'], (_type['typeIdsArray'] || []) as Type[], children); }
+function fixedSizeBinaryFromJSON(_type: any)                    { return new FixedSizeBinary(_type['byteWidth']);                                     }
+function fixedSizeListFromJSON  (_type: any, children: Field[]) { return new FixedSizeList(_type['listSize'], children);                              }
+function mapFromJSON            (_type: any, children: Field[]) { return new Map_(_type['keysSorted'], children);                                     }
diff --git a/js/src/ipc/reader/vector.ts b/js/src/ipc/reader/vector.ts
new file mode 100644
index 0000000000000..809069c6d9864
--- /dev/null
+++ b/js/src/ipc/reader/vector.ts
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from '../../vector';
+import { RecordBatch } from '../../recordbatch';
+import { TypeVisitor } from '../../visitor';
+import { FlatType, NestedType, ListType } from '../../type';
+import { Message, FieldMetadata, BufferMetadata } from '../metadata';
+import { FlatData, ListData, NestedData, SingleNestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data';
+import {
+    Schema, Field,
+    Dictionary,
+    Null, Int, Float,
+    Binary, Bool, Utf8, Decimal,
+    Date_, Time, Timestamp, Interval,
+    List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
+    UnionMode, SparseUnion, DenseUnion, FlatListType, DataType,
+} from '../../type';
+
+export function* readRecordBatches(messages: Iterable<{ schema: Schema, message: Message, loader: TypeDataLoader }>) {
+    for (const { schema, message, loader } of messages) {
+        yield* readRecordBatch(schema, message, loader);
+    }
+}
+
+export async function* readRecordBatchesAsync(messages: AsyncIterable<{ schema: Schema, message: Message, loader: TypeDataLoader }>) {
+    for await (const { schema, message, loader } of messages) {
+        yield* readRecordBatch(schema, message, loader);
+    }
+}
+
+export function* readRecordBatch(schema: Schema, message: Message, loader: TypeDataLoader) {
+    if (Message.isRecordBatch(message)) {
+        yield new RecordBatch(schema, message.length, loader.visitFields(schema.fields));
+    } else if (Message.isDictionaryBatch(message)) {
+        const dictionaryId = message.id;
+        const dictionaries = loader.dictionaries;
+        const dictionaryField = schema.dictionaries.get(dictionaryId)!;
+        const dictionaryDataType = (dictionaryField.type as Dictionary).dictionary;
+        let dictionaryVector = Vector.create(loader.visit(dictionaryDataType));
+        if (message.isDelta && dictionaries.has(dictionaryId)) {
+            dictionaryVector = dictionaries.get(dictionaryId)!.concat(dictionaryVector);
+        }
+        dictionaries.set(dictionaryId, dictionaryVector);
+    }
+}
+
+export abstract class TypeDataLoader extends TypeVisitor {
+
+    public dictionaries: Map<number, Vector>;
+    protected nodes: Iterator<FieldMetadata>;
+    protected buffers: Iterator<BufferMetadata>;
+
+    constructor(nodes: Iterator<FieldMetadata>, buffers: Iterator<BufferMetadata>, dictionaries: Map<number, Vector>) {
+        super();
+        this.nodes = nodes;
+        this.buffers = buffers;
+        this.dictionaries = dictionaries;
+    }
+
+    public visitFields(fields: Field[]) { return fields.map((field) => this.visit(field.type)); }
+
+    public visitNull           (type: Null)            { return this.visitNullType(type);   }
+    public visitInt            (type: Int)             { return this.visitFlatType(type);   }
+    public visitFloat          (type: Float)           { return this.visitFlatType(type);   }
+    public visitBinary         (type: Binary)          { return this.visitFlatList(type);   }
+    public visitUtf8           (type: Utf8)            { return this.visitFlatList(type);   }
+    public visitBool           (type: Bool)            { return this.visitBoolType(type);   }
+    public visitDecimal        (type: Decimal)         { return this.visitFlatType(type);   }
+    public visitDate           (type: Date_)           { return this.visitFlatType(type);   }
+    public visitTime           (type: Time)            { return this.visitFlatType(type);   }
+    public visitTimestamp      (type: Timestamp)       { return this.visitFlatType(type);   }
+    public visitInterval       (type: Interval)        { return this.visitFlatType(type);   }
+    public visitList           (type: List)            { return this.visitListType(type);   }
+    public visitStruct         (type: Struct)          { return this.visitNestedType(type); }
+    public visitUnion          (type: Union)           { return this.visitUnionType(type);  }
+    public visitFixedSizeBinary(type: FixedSizeBinary) { return this.visitFlatType(type);   }
+    public visitFixedSizeList  (type: FixedSizeList)   { return this.visitFixedSizeListType(type); }
+    public visitMap            (type: Map_)            { return this.visitNestedType(type); }
+    public visitDictionary     (type: Dictionary)      {
+        return new DictionaryData(type, this.dictionaries.get(type.id)!, this.visit(type.indicies));
+    }
+    protected getFieldMetadata() { return this.nodes.next().value; }
+    protected getBufferMetadata() { return this.buffers.next().value; }
+    protected readNullBitmap<T extends DataType>(type: T, nullCount: number, buffer = this.getBufferMetadata()) {
+        return nullCount > 0 && this.readData(type, buffer) || new Uint8Array(0);
+    }
+    protected abstract readData<T extends DataType>(type: T, buffer?: BufferMetadata): any;
+    protected abstract readOffsets<T extends DataType>(type: T, buffer?: BufferMetadata): any;
+    protected abstract readTypeIds<T extends DataType>(type: T, buffer?: BufferMetadata): any;
+    protected visitNullType(type: Null, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
+        return new FlatData<any>(type, length, this.readNullBitmap(type, nullCount), new Uint8Array(0), 0, nullCount);
+    }
+    protected visitFlatType<T extends FlatType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
+        return new FlatData<T>(type, length, this.readNullBitmap(type, nullCount), this.readData(type), 0, nullCount);
+    }
+    protected visitBoolType(type: Bool, { length, nullCount }: FieldMetadata = this.getFieldMetadata(), data?: Uint8Array) {
+        return new BoolData(type, length, this.readNullBitmap(type, nullCount), data || this.readData(type), 0, nullCount);
+    }
+    protected visitFlatList<T extends FlatListType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
+        return new FlatListData<T>(type, length, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.readData(type), 0, nullCount);
+    }
+    protected visitListType<T extends ListType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
+        return new ListData<T>(type, length, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children![0].type), 0, nullCount);
+    }
+    protected visitFixedSizeListType<T extends FixedSizeList>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
+        return new SingleNestedData<T>(type, length, this.readNullBitmap(type, nullCount), this.visit(type.children![0].type), 0, nullCount);
+    }
+    protected visitNestedType<T extends NestedType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
+        return new NestedData<T>(type, length, this.readNullBitmap(type, nullCount), this.visitFields(type.children), 0, nullCount);
+    }
+    protected visitUnionType(type: DenseUnion | SparseUnion, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
+        return type.mode === UnionMode.Sparse ?
+            new SparseUnionData(type as SparseUnion, length, this.readNullBitmap(type, nullCount), this.readTypeIds(type), this.visitFields(type.children), 0, nullCount) :
+            new DenseUnionData(type as DenseUnion, length, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.readTypeIds(type), this.visitFields(type.children), 0, nullCount);
+    }
+}
diff --git a/js/src/predicate.ts b/js/src/predicate.ts
new file mode 100644
index 0000000000000..9d55274bd880b
--- /dev/null
+++ b/js/src/predicate.ts
@@ -0,0 +1,225 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { RecordBatch } from './recordbatch';
+import { Vector, DictionaryVector } from './vector';
+
+export type ValueFunc<T> = (idx: number, cols: RecordBatch) => T | null;
+export type PredicateFunc = (idx: number, cols: RecordBatch) => boolean;
+
+export abstract class Value<T> {
+    eq(other: Value<T> | T): Predicate {
+        if (!(other instanceof Value)) { other = new Literal(other); }
+        return new Equals(this, other);
+    }
+    lteq(other: Value<T> | T): Predicate {
+        if (!(other instanceof Value)) { other = new Literal(other); }
+        return new LTeq(this, other);
+    }
+    gteq(other: Value<T> | T): Predicate {
+        if (!(other instanceof Value)) { other = new Literal(other); }
+        return new GTeq(this, other);
+    }
+}
+
+export class Literal<T= any> extends Value<T> {
+    constructor(public v: T) { super(); }
+}
+
+export class Col<T= any> extends Value<T> {
+    // @ts-ignore
+    public vector: Vector;
+    // @ts-ignore
+    public colidx: number;
+
+    constructor(public name: string) { super(); }
+    bind(batch: RecordBatch) {
+        if (!this.colidx) {
+            // Assume column index doesn't change between calls to bind
+            //this.colidx = cols.findIndex(v => v.name.indexOf(this.name) != -1);
+            this.colidx = -1;
+            const fields = batch.schema.fields;
+            for (let idx = -1; ++idx < fields.length;) {
+                if (fields[idx].name === this.name) {
+                    this.colidx = idx;
+                    break;
+                }
+            }
+            if (this.colidx < 0) { throw new Error(`Failed to bind Col "${this.name}"`); }
+        }
+        this.vector = batch.getChildAt(this.colidx)!;
+        return this.vector.get.bind(this.vector);
+    }
+}
+
+export abstract class Predicate {
+    abstract bind(batch: RecordBatch): PredicateFunc;
+    and(expr: Predicate): Predicate { return new And(this, expr); }
+    or(expr: Predicate): Predicate { return new Or(this, expr); }
+    ands(): Predicate[] { return [this]; }
+}
+
+export abstract class ComparisonPredicate<T= any> extends Predicate {
+    constructor(public readonly left: Value<T>, public readonly right: Value<T>) {
+        super();
+    }
+
+    bind(batch: RecordBatch) {
+        if (this.left instanceof Literal) {
+            if (this.right instanceof Literal) {
+                return this._bindLitLit(batch, this.left, this.right);
+            } else { // right is a Col
+
+                return this._bindLitCol(batch, this.left, this.right as Col);
+            }
+        } else { // left is a Col
+            if (this.right instanceof Literal) {
+                return this._bindColLit(batch, this.left as Col, this.right);
+            } else { // right is a Col
+                return this._bindColCol(batch, this.left as Col, this.right as Col);
+            }
+        }
+    }
+
+    protected abstract _bindLitLit(batch: RecordBatch, left: Literal, right: Literal): PredicateFunc;
+    protected abstract _bindColCol(batch: RecordBatch, left: Col, right: Col): PredicateFunc;
+    protected abstract _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc;
+    protected abstract _bindLitCol(batch: RecordBatch, lit: Literal, col: Col): PredicateFunc;
+}
+
+export abstract class CombinationPredicate extends Predicate {
+    constructor(public readonly left: Predicate, public readonly right: Predicate) {
+        super();
+    }
+}
+
+export class And extends CombinationPredicate {
+    bind(batch: RecordBatch) {
+        const left = this.left.bind(batch);
+        const right = this.right.bind(batch);
+        return (idx: number, batch: RecordBatch) => left(idx, batch) && right(idx, batch);
+    }
+    ands(): Predicate[] { return this.left.ands().concat(this.right.ands()); }
+}
+
+export class Or extends CombinationPredicate {
+    bind(batch: RecordBatch) {
+        const left = this.left.bind(batch);
+        const right = this.right.bind(batch);
+        return (idx: number, batch: RecordBatch) => left(idx, batch) || right(idx, batch);
+    }
+}
+
+export class Equals extends ComparisonPredicate {
+    protected _bindLitLit(_batch: RecordBatch, left: Literal, right: Literal): PredicateFunc {
+        const rtrn: boolean = left.v == right.v;
+        return () => rtrn;
+    }
+
+    protected _bindColCol(batch: RecordBatch, left: Col, right: Col): PredicateFunc {
+        const left_func = left.bind(batch);
+        const right_func = right.bind(batch);
+        return (idx: number, batch: RecordBatch) => left_func(idx, batch) == right_func(idx, batch);
+    }
+
+    protected _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc {
+        const col_func = col.bind(batch);
+        if (col.vector instanceof DictionaryVector) {
+            // Assume that there is only one key with the value `lit.v`
+            // TODO: add lazily-computed reverse dictionary lookups, associated
+            // with col.vector.data so that we only have to do this once per
+            // dictionary
+            let key = -1;
+            let dict = col.vector;
+            let data = dict.dictionary!;
+            for (let len = data.length; ++key < len;) {
+                if (data.get(key) === lit.v) {
+                    break;
+                }
+            }
+
+            if (key == data.length) {
+                // the value doesn't exist in the dictionary - always return
+                // false
+                // TODO: special-case of PredicateFunc that encapsulates this
+                // "always false" behavior. That way filtering operations don't
+                // have to bother checking
+                return () => false;
+            } else {
+                return (idx: number) => {
+                    return dict.getKey(idx) === key;
+                };
+            }
+        } else {
+            return (idx: number, cols: RecordBatch) => col_func(idx, cols) == lit.v;
+        }
+    }
+
+    protected _bindLitCol(batch: RecordBatch, lit: Literal, col: Col) {
+        // Equals is comutative
+        return this._bindColLit(batch, col, lit);
+    }
+}
+
+export class LTeq extends ComparisonPredicate {
+    protected _bindLitLit(_batch: RecordBatch, left: Literal, right: Literal): PredicateFunc {
+        const rtrn: boolean = left.v <= right.v;
+        return () => rtrn;
+    }
+
+    protected _bindColCol(batch: RecordBatch, left: Col, right: Col): PredicateFunc {
+        const left_func = left.bind(batch);
+        const right_func = right.bind(batch);
+        return (idx: number, cols: RecordBatch) => left_func(idx, cols) <= right_func(idx, cols);
+    }
+
+    protected _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc {
+        const col_func = col.bind(batch);
+        return (idx: number, cols: RecordBatch) => col_func(idx, cols) <= lit.v;
+    }
+
+    protected _bindLitCol(batch: RecordBatch, lit: Literal, col: Col) {
+        const col_func = col.bind(batch);
+        return (idx: number, cols: RecordBatch) => lit.v <= col_func(idx, cols);
+    }
+}
+
+export class GTeq extends ComparisonPredicate {
+    protected _bindLitLit(_batch: RecordBatch, left: Literal, right: Literal): PredicateFunc {
+        const rtrn: boolean = left.v >= right.v;
+        return () => rtrn;
+    }
+
+    protected _bindColCol(batch: RecordBatch, left: Col, right: Col): PredicateFunc {
+        const left_func = left.bind(batch);
+        const right_func = right.bind(batch);
+        return (idx: number, cols: RecordBatch) => left_func(idx, cols) >= right_func(idx, cols);
+    }
+
+    protected _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc {
+        const col_func = col.bind(batch);
+        return (idx: number, cols: RecordBatch) => col_func(idx, cols) >= lit.v;
+    }
+
+    protected _bindLitCol(batch: RecordBatch, lit: Literal, col: Col) {
+        const col_func = col.bind(batch);
+        return (idx: number, cols: RecordBatch) => lit.v >= col_func(idx, cols);
+    }
+}
+
+export function lit(v: any): Value<any> { return new Literal(v); }
+export function col(n: string): Col<any> { return new Col(n); }
diff --git a/js/src/reader/buffer.ts b/js/src/reader/buffer.ts
deleted file mode 100644
index c7b90507e396f..0000000000000
--- a/js/src/reader/buffer.ts
+++ /dev/null
@@ -1,229 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { flatbuffers } from 'flatbuffers';
-import { VectorLayoutReader } from './vector';
-import { TypedArray, TypedArrayConstructor } from '../vector/types';
-import { footerFromByteBuffer, messageFromByteBuffer } from '../format/fb';
-import { Footer, Schema, RecordBatch, DictionaryBatch, Field, Buffer, FieldNode } from '../format/arrow';
-import ByteBuffer = flatbuffers.ByteBuffer;
-
-export function* readBuffers<T extends Uint8Array | NodeBuffer | string>(sources: Iterable<T>) {
-    let schema: Schema | null = null;
-    let readMessages: ((bb: ByteBuffer) => IterableIterator<RecordBatch | DictionaryBatch>) | null = null;
-    for (const source of sources) {
-        const bb = toByteBuffer(source);
-        if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) {
-            for (const message of readMessages(bb)) {
-                yield {
-                    schema, message, reader: new BufferVectorLayoutReader(
-                        bb,
-                        (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes),
-                        (function* (buffers) { yield* buffers; })(message.buffers)
-                    ) as VectorLayoutReader
-                };
-            }
-        }
-    }
-}
-
-export async function* readBuffersAsync<T extends Uint8Array | NodeBuffer | string>(sources: AsyncIterable<T>) {
-    let schema: Schema | null = null;
-    let readMessages: ((bb: ByteBuffer) => IterableIterator<RecordBatch | DictionaryBatch>) | null = null;
-    for await (const source of sources) {
-        const bb = toByteBuffer(source);
-        if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) {
-            for (const message of readMessages(bb)) {
-                yield {
-                    schema, message, reader: new BufferVectorLayoutReader(
-                        bb,
-                        (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes),
-                        (function* (buffers) { yield* buffers; })(message.buffers)
-                    ) as VectorLayoutReader
-                };
-            }
-        }
-    }
-}
-
-function toByteBuffer(bytes?: Uint8Array | NodeBuffer | string) {
-    let arr: Uint8Array = bytes as any || new Uint8Array(0);
-    if (typeof bytes === 'string') {
-        arr = new Uint8Array(bytes.length);
-        for (let i = -1, n = bytes.length; ++i < n;) {
-            arr[i] = bytes.charCodeAt(i);
-        }
-        return new ByteBuffer(arr);
-    }
-    return new ByteBuffer(arr);
-}
-
-function readSchema(bb: ByteBuffer) {
-    let schema: Schema, readMessages, footer: Footer | null;
-    if (footer = readFileSchema(bb)) {
-        schema = footer.schema!;
-        readMessages = readFileMessages(footer);
-    } else if (schema = readStreamSchema(bb)!) {
-        readMessages = readStreamMessages;
-    } else {
-        throw new Error('Invalid Arrow buffer');
-    }
-    return { schema, readMessages };
-}
-
-const PADDING = 4;
-const MAGIC_STR = 'ARROW1';
-const MAGIC = new Uint8Array(MAGIC_STR.length);
-for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
-    MAGIC[i] = MAGIC_STR.charCodeAt(i);
-}
-
-function checkForMagicArrowString(buffer: Uint8Array, index = 0) {
-    for (let i = -1, n = MAGIC.length; ++i < n;) {
-        if (MAGIC[i] !== buffer[index + i]) {
-            return false;
-        }
-    }
-    return true;
-}
-
-const magicLength = MAGIC.length;
-const magicAndPadding = magicLength + PADDING;
-const magicX2AndPadding = magicLength * 2 + PADDING;
-
-function readStreamSchema(bb: ByteBuffer) {
-    if (!checkForMagicArrowString(bb.bytes(), 0)) {
-        for (const message of readMessages(bb)) {
-            if (message.isSchema()) {
-                return message as Schema;
-            }
-        }
-    }
-    return null;
-}
-
-function* readStreamMessages(bb: ByteBuffer) {
-    for (const message of readMessages(bb)) {
-        if (message.isRecordBatch()) {
-            yield message;
-        } else if (message.isDictionaryBatch()) {
-            yield message;
-        } else {
-            continue;
-        }
-        // position the buffer after the body to read the next message
-        bb.setPosition(bb.position() + message.bodyLength.low);
-    }
-}
-
-function readFileSchema(bb: ByteBuffer) {
-    let fileLength = bb.capacity(), footerLength: number, footerOffset: number;
-    if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
-        (!checkForMagicArrowString(bb.bytes(), 0) /*                        Missing magic start    */) ||
-        (!checkForMagicArrowString(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
-        (/*                                                    Invalid footer length  */
-        (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
-        (footerLength + magicX2AndPadding > fileLength))) {
-        return null;
-    }
-    bb.setPosition(footerOffset - footerLength);
-    return footerFromByteBuffer(bb);
-}
-
-function readFileMessages(footer: Footer) {
-    return function* (bb: ByteBuffer) {
-        for (let i = -1, batches = footer.dictionaryBatches, n = batches.length; ++i < n;) {
-            bb.setPosition(batches[i].offset.low);
-            yield readMessage(bb, bb.readInt32(bb.position())) as DictionaryBatch;
-        }
-        for (let i = -1, batches = footer.recordBatches, n = batches.length; ++i < n;) {
-            bb.setPosition(batches[i].offset.low);
-            yield readMessage(bb, bb.readInt32(bb.position())) as RecordBatch;
-        }
-    };
-}
-
-function* readMessages(bb: ByteBuffer) {
-    let length: number, message: Schema | RecordBatch | DictionaryBatch;
-    while (bb.position() < bb.capacity() &&
-          (length = bb.readInt32(bb.position())) > 0) {
-        if (message = readMessage(bb, length)!) {
-            yield message;
-        }
-    }
-}
-
-function readMessage(bb: ByteBuffer, length: number) {
-    bb.setPosition(bb.position() + PADDING);
-    const message = messageFromByteBuffer(bb);
-    bb.setPosition(bb.position() + length);
-    return message;
-}
-
-class BufferVectorLayoutReader implements VectorLayoutReader {
-    private offset: number;
-    private bytes: Uint8Array;
-    constructor(bb: ByteBuffer, private fieldNodes: Iterator<FieldNode>, private buffers: Iterator<Buffer>) {
-        this.bytes = bb.bytes();
-        this.offset = bb.position();
-    }
-    readContainerLayout(field: Field) {
-        const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value;
-        return {
-            field, fieldNode,
-            validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value)
-        };
-    }
-    readFixedWidthLayout<T extends TypedArray>(field: Field, dataType: TypedArrayConstructor<T>) {
-        const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value;
-        return {
-            field, fieldNode,
-            validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value),
-            data: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, dataType)
-        };
-    }
-    readBinaryLayout(field: Field) {
-        const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value;
-        return {
-            field, fieldNode,
-            validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value),
-            offsets: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Int32Array),
-            data: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Uint8Array)
-        };
-    }
-    readVariableWidthLayout(field: Field) {
-        const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value;
-        return {
-            field, fieldNode,
-            validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value),
-            offsets: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Int32Array)
-        };
-    }
-}
-
-function createValidityArray(bytes: Uint8Array, field: Field, fieldNode: FieldNode, offset: number, buffer: Buffer) {
-    return field.nullable && fieldNode.nullCount.low > 0 &&
-        createTypedArray(bytes, field, fieldNode, offset, buffer, Uint8Array) || null;
-}
-
-function createTypedArray<T extends TypedArray>(bytes: Uint8Array, _field: Field, _fieldNode: FieldNode, offset: number, buffer: Buffer, ArrayConstructor: TypedArrayConstructor<T>): T {
-    return new ArrayConstructor(
-        bytes.buffer,
-        bytes.byteOffset + offset + buffer.offset.low,
-        buffer.length.low / ArrayConstructor.BYTES_PER_ELEMENT
-    );
-}
diff --git a/js/src/reader/json.ts b/js/src/reader/json.ts
deleted file mode 100644
index 49431496354e8..0000000000000
--- a/js/src/reader/json.ts
+++ /dev/null
@@ -1,176 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import * as Schema_ from '../format/fb/Schema';
-import { Int64, Int128 } from '../util/int';
-import { VectorLayoutReader } from './vector';
-import { TextEncoder } from 'text-encoding-utf-8';
-import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
-import { TypedArray, TypedArrayConstructor } from '../vector/types';
-import { schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON } from '../format/json';
-import { Schema, RecordBatch, DictionaryBatch, Field, Buffer, FieldNode } from '../format/arrow';
-export { Schema, RecordBatch, DictionaryBatch };
-
-export function* readJSON(json: any) {
-    const schema = schemaFromJSON(json['schema']);
-    for (const batch of (json['dictionaries'] || [])) {
-        const message = dictionaryBatchFromJSON(batch);
-        yield {
-            schema, message, reader: new JSONVectorLayoutReader(
-                flattenDataSources(batch['data']['columns']),
-                (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes),
-                (function* (buffers) { yield* buffers; })(message.buffers)
-            ) as VectorLayoutReader
-        };
-    }
-    for (const batch of (json['batches'] || [])) {
-        const message = recordBatchFromJSON(batch);
-        yield {
-            schema, message, reader: new JSONVectorLayoutReader(
-                flattenDataSources(batch['columns']),
-                (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes),
-                (function* (buffers) { yield* buffers; })(message.buffers)
-            ) as VectorLayoutReader
-        };
-    }
-}
-
-function flattenDataSources(xs: any[]): any[][] {
-    return (xs || []).reduce<any[][]>((buffers, column: any) => [
-        ...buffers,
-        ...(column['VALIDITY'] && [column['VALIDITY']] || []),
-        ...(column['OFFSET'] && [column['OFFSET']] || []),
-        ...(column['DATA'] && [column['DATA']] || []),
-        ...flattenDataSources(column['children'])
-    ], [] as any[][]);
-}
-
-class JSONVectorLayoutReader implements VectorLayoutReader {
-    constructor(private sources: any[][], private fieldNodes: Iterator<FieldNode>, private buffers: Iterator<Buffer>) {}
-    readContainerLayout(field: Field) {
-        const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value;
-        return {
-            field, fieldNode,
-            validity: createValidityArray(sources, field, fieldNode, buffers.next().value)
-        };
-    }
-    readFixedWidthLayout<T extends TypedArray>(field: Field, dataType: TypedArrayConstructor<T>) {
-        const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value;
-        return {
-            field, fieldNode,
-            validity: createValidityArray(sources, field, fieldNode, buffers.next().value),
-            data: createDataArray(sources, field, fieldNode, buffers.next().value, dataType)
-        };
-    }
-    readBinaryLayout(field: Field) {
-        const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value;
-        return {
-            field, fieldNode,
-            validity: createValidityArray(sources, field, fieldNode, buffers.next().value),
-            offsets: new Int32Array(sources[buffers.next().value.offset.low]),
-            data: createDataArray(sources, field, fieldNode, buffers.next().value, Uint8Array)
-        };
-    }
-    readVariableWidthLayout(field: Field) {
-        const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value;
-        return {
-            field, fieldNode,
-            validity: createValidityArray(sources, field, fieldNode, buffers.next().value),
-            offsets: new Int32Array(sources[buffers.next().value.offset.low]),
-        };
-    }
-}
-
-function createValidityArray(sources: any[][], field: Field, fieldNode: FieldNode, buffer: Buffer) {
-    return field.nullable && fieldNode.nullCount.low > 0 &&
-        booleanFromJSON(sources[buffer.offset.low]) || null;
-}
-
-const encoder = new TextEncoder('utf-8');
-
-function createDataArray<T extends TypedArray>(sources: any[][], field: Field, _fieldNode: FieldNode, buffer: Buffer, ArrayConstructor: TypedArrayConstructor<T>): T {
-    let type = field.type, data: ArrayLike<number> | ArrayBufferLike;
-    if (type.isTimestamp() === true) {
-        data = int64sFromJSON(sources[buffer.offset.low] as string[]);
-    } else if ((type.isInt() || type.isTime()) && type.bitWidth === 64) {
-        data = int64sFromJSON(sources[buffer.offset.low] as string[]);
-    } else if (type.isDate() && type.unit === DateUnit.MILLISECOND) {
-        data = int64sFromJSON(sources[buffer.offset.low] as string[]);
-    } else if (type.isDecimal() === true) {
-        data = decimalFromJSON(sources[buffer.offset.low] as string[]);
-    } else if (type.isBinary() === true) {
-        data = binaryFromJSON(sources[buffer.offset.low] as string[]);
-    } else if (type.isBool() === true) {
-        data = booleanFromJSON(sources[buffer.offset.low] as number[]).buffer;
-    } else if (type.isUtf8() === true) {
-        data = encoder.encode((sources[buffer.offset.low] as string[]).join(''));
-    } else {
-        data = (sources[buffer.offset.low]).map((x) => +x);
-    }
-    return new ArrayConstructor(data);
-}
-
-function int64sFromJSON(values: string[]) {
-    const data = new Uint32Array(values.length * 2);
-    for (let i = -1, n = values.length; ++i < n;) {
-        // Force all values (even numbers) to be parsed as strings since
-        // pulling out high and low bits seems to lose precision sometimes
-        // For example:
-        //     > -4613034156400212000 >>> 0
-        //     721782784
-        // The correct lower 32-bits are 721782752
-        Int64.fromString(values[i].toString(), new Uint32Array(data.buffer, data.byteOffset + 2 * i * 4, 2));
-    }
-    return data.buffer;
-}
-
-function decimalFromJSON(values: string[]) {
-    const data = new Uint32Array(values.length * 4);
-    for (let i = -1, n = values.length; ++i < n;) {
-        Int128.fromString(values[i], new Uint32Array(data.buffer, data.byteOffset + 4 * 4 * i, 4));
-    }
-    return data.buffer;
-}
-
-function binaryFromJSON(values: string[]) {
-    // "DATA": ["49BC7D5B6C47D2","3F5FB6D9322026"]
-    // There are definitely more efficient ways to do this... but it gets the
-    // job done.
-    const joined = values.join('');
-    const data = new Uint8Array(joined.length / 2);
-    for (let i = 0; i < joined.length; i += 2) {
-        data[i >> 1] = parseInt(joined.substr(i, 2), 16);
-    }
-    return data.buffer;
-}
-
-function booleanFromJSON(arr: number[]) {
-    let xs = [], n, i = 0;
-    let bit = 0, byte = 0;
-    for (const value of arr) {
-        value && (byte |= 1 << bit);
-        if (++bit === 8) {
-            xs[i++] = byte;
-            byte = bit = 0;
-        }
-    }
-    if (i === 0 || bit > 0) { xs[i++] = byte; }
-    if (i % 8 && (n = i + 8 - i % 8)) {
-        do { xs[i] = 0; } while (++i < n);
-    }
-    return new Uint8Array(xs);
-}
diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts
deleted file mode 100644
index 3bd6d2bb67650..0000000000000
--- a/js/src/reader/vector.ts
+++ /dev/null
@@ -1,255 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import * as Schema_ from '../format/fb/Schema';
-import { TypedArray, TypedArrayConstructor } from '../vector/types';
-import { Schema, RecordBatch, DictionaryBatch, Field, FieldNode } from '../format/arrow';
-import { Int, Date, Time, Timestamp, Decimal, FixedSizeList, FixedSizeBinary, FloatingPoint } from '../format/arrow';
-import {
-    Vector, BoolVector, BinaryVector, DictionaryVector,
-    Int8Vector, Int16Vector, Int32Vector, Int64Vector,
-    Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector,
-    Utf8Vector, ListVector, FixedSizeListVector, StructVector,
-    Float16Vector, Float32Vector, Float64Vector, DecimalVector,
-    Date32Vector, Date64Vector, Time32Vector, Time64Vector, TimestampVector,
-} from '../vector/arrow';
-
-import Type = Schema_.org.apache.arrow.flatbuf.Type;
-import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
-import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
-import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
-// import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
-
-export interface ContainerLayout {
-    fieldNode: FieldNode;
-    validity: Uint8Array | null | void;
-}
-
-export interface VariableWidthLayout {
-    fieldNode: FieldNode;
-    offsets: Int32Array;
-    validity: Uint8Array | null | void;
-}
-
-export interface BinaryLayout extends FixedWidthLayout<Uint8Array> {
-    offsets: Int32Array;
-}
-
-export interface FixedWidthLayout<T extends TypedArray> {
-    fieldNode: FieldNode;
-    data: T;
-    validity: Uint8Array | null | void;
-}
-
-export function* readVectors(messages: Iterable<{ schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorLayoutReader }>) {
-    const dictionaries = new Map<string, Vector>();
-    for (const { schema, message, reader } of messages) {
-        yield* readMessageVectors(schema, message, new VectorReader(dictionaries, reader));
-    }
-}
-
-export async function* readVectorsAsync(messages: AsyncIterable<{ schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorLayoutReader }>) {
-    const dictionaries = new Map<string, Vector>();
-    for await (const { schema, message, reader } of messages) {
-        yield* readMessageVectors(schema, message, new VectorReader(dictionaries, reader));
-    }
-}
-
-function* readMessageVectors(schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorReader) {
-    if (message.isRecordBatch() === true) {
-        yield schema.fields.map((field) => reader.readVector(field));
-    } else if (message.isDictionaryBatch()) {
-        let id = message.dictionaryId.toFloat64().toString();
-        let vector = reader.readValueVector(schema.dictionaries.get(id)!);
-        if (message.isDelta) {
-            vector = reader.dictionaries.get(id)!.concat(vector);
-        }
-        reader.dictionaries.set(id, vector);
-    }
-}
-
-export interface VectorLayoutReader {
-    readBinaryLayout(field: Field): BinaryLayout;
-    readContainerLayout(field: Field): ContainerLayout;
-    readVariableWidthLayout(field: Field): VariableWidthLayout;
-    readFixedWidthLayout<T extends TypedArray>(field: Field, TypedArrayConstructor: TypedArrayConstructor<T>): FixedWidthLayout<T>;
-}
-
-export class VectorReader implements VectorLayoutReader {
-    constructor(public dictionaries: Map<string, Vector>, protected layout: VectorLayoutReader) {}
-    readVector(field: Field): Vector {
-        return this.readDictionaryVector(field) || this.readValueVector(field);
-    }
-    readDictionaryVector(field: Field) {
-        const encoding = field.dictionary;
-        if (encoding) {
-            const keys = this.readIntVector(field.indexField());
-            const data = this.dictionaries.get(encoding.dictionaryId.toFloat64().toString())!;
-            return new DictionaryVector({
-                field, data, keys,
-                validity: (keys as any).validity,
-                fieldNode: (keys as any).fieldNode,
-            });
-        }
-        return null;
-    }
-    readValueVector(field: Field) {
-        switch (field.typeType) {
-            case Type.NONE: return this.readNullVector();
-            case Type.Null: return this.readNullVector();
-            // case Type.Map: return this.readMapVector(field);
-            case Type.Int: return this.readIntVector(field);
-            case Type.Bool: return this.readBoolVector(field);
-            case Type.Date: return this.readDateVector(field);
-            case Type.List: return this.readListVector(field);
-            case Type.Utf8: return this.readUtf8Vector(field);
-            case Type.Time: return this.readTimeVector(field);
-            // case Type.Union: return this.readUnionVector(field);
-            case Type.Binary: return this.readBinaryVector(field);
-            case Type.Decimal: return this.readDecimalVector(field);
-            case Type.Struct_: return this.readStructVector(field);
-            case Type.FloatingPoint: return this.readFloatVector(field);
-            case Type.Timestamp: return this.readTimestampVector(field);
-            case Type.FixedSizeList: return this.readFixedSizeListVector(field);
-            case Type.FixedSizeBinary: return this.readFixedSizeBinaryVector(field);
-        }
-        throw new Error(`Unrecognized ${field.toString()}`);
-    }
-    readNullVector() {
-        return new Vector();
-    }
-    readBoolVector(field: Field) {
-        return new BoolVector(this.readFixedWidthLayout(field, Uint8Array));
-    }
-    readDateVector(field: Field) {
-        const type = field.type as Date;
-        switch (type.unit) {
-            case DateUnit.DAY: return new Date32Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: DateUnit[type.unit] });
-            case DateUnit.MILLISECOND: return new Date64Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: DateUnit[type.unit] });
-        }
-        throw new Error(`Unrecognized ${type.toString()}`);
-    }
-    readTimeVector(field: Field) {
-        const type = field.type as Time;
-        switch (type.bitWidth) {
-            case 32: return new Time32Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: TimeUnit[type.unit] });
-            case 64: return new Time64Vector({ ...this.readFixedWidthLayout(field, Uint32Array), unit: TimeUnit[type.unit] });
-        }
-        throw new Error(`Unrecognized ${type.toString()}`);
-    }
-    readTimestampVector(field: Field) {
-        const type = field.type as Timestamp;
-        const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint32Array);
-        return new TimestampVector({
-            field, fieldNode, validity, data,
-            timezone: type.timezone!,
-            unit: TimeUnit[type.unit],
-        });
-    }
-    readListVector(field: Field) {
-        const { fieldNode, validity, offsets } = this.readVariableWidthLayout(field);
-        return new ListVector({
-            field, fieldNode, validity, offsets,
-            values: this.readVector(field.children[0])
-        });
-    }
-    readStructVector(field: Field) {
-        const { fieldNode, validity } = this.readContainerLayout(field);
-        return new StructVector({
-            field, fieldNode, validity,
-            columns: field.children.map((field) => this.readVector(field))
-        });
-    }
-    readBinaryVector(field: Field) {
-        return new BinaryVector(this.readBinaryLayout(field));
-    }
-    readDecimalVector(field: Field) {
-        const type = field.type as Decimal;
-        const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint32Array);
-        return new DecimalVector({
-            scale: type.scale,
-            precision: type.precision,
-            field, fieldNode, validity, data
-        });
-    }
-    readUtf8Vector(field: Field) {
-        const { fieldNode, validity, offsets, data } = this.readBinaryLayout(field);
-        return new Utf8Vector({
-            field, fieldNode,
-            values: new BinaryVector({
-                validity, offsets, data
-            })
-        });
-    }
-    readFixedSizeListVector(field: Field) {
-        const type = field.type as FixedSizeList;
-        const { fieldNode, validity } = this.readContainerLayout(field);
-        return new FixedSizeListVector({
-            field, fieldNode, validity,
-            size: type.listSize,
-            values: this.readVector(field.children[0])
-        });
-    }
-    readFixedSizeBinaryVector(field: Field) {
-        const type = field.type as FixedSizeBinary;
-        const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint8Array);
-        return new FixedSizeListVector({
-            size: type.byteWidth,
-            field, fieldNode, validity,
-            values: new Uint8Vector({ data })
-        });
-    }
-    readFloatVector(field: Field) {
-        const type = field.type as FloatingPoint;
-        switch (type.precision) {
-            case Precision.HALF:   return new Float16Vector(this.readFixedWidthLayout(field, Uint16Array));
-            case Precision.SINGLE: return new Float32Vector(this.readFixedWidthLayout(field, Float32Array));
-            case Precision.DOUBLE: return new Float64Vector(this.readFixedWidthLayout(field, Float64Array));
-        }
-        throw new Error(`Unrecognized FloatingPoint { precision: ${type.precision} }`);
-    }
-    readIntVector(field: Field) {
-        const type = field.type as Int;
-        if (type.isSigned) {
-            switch (type.bitWidth) {
-                case  8: return new  Int8Vector(this.readFixedWidthLayout(field, Int8Array));
-                case 16: return new Int16Vector(this.readFixedWidthLayout(field, Int16Array));
-                case 32: return new Int32Vector(this.readFixedWidthLayout(field, Int32Array));
-                case 64: return new Int64Vector(this.readFixedWidthLayout(field, Int32Array));
-            }
-        }
-        switch (type.bitWidth) {
-            case  8: return new  Uint8Vector(this.readFixedWidthLayout(field, Uint8Array));
-            case 16: return new Uint16Vector(this.readFixedWidthLayout(field, Uint16Array));
-            case 32: return new Uint32Vector(this.readFixedWidthLayout(field, Uint32Array));
-            case 64: return new Uint64Vector(this.readFixedWidthLayout(field, Uint32Array));
-        }
-        throw new Error(`Unrecognized Int { isSigned: ${type.isSigned}, bitWidth: ${type.bitWidth} }`);
-    }
-    readContainerLayout(field: Field) {
-        return this.layout.readContainerLayout(field);
-    }
-    readBinaryLayout(field: Field) {
-        return this.layout.readBinaryLayout(field);
-    }
-    readVariableWidthLayout(field: Field) {
-        return this.layout.readVariableWidthLayout(field);
-    }
-    readFixedWidthLayout<T extends TypedArray>(field: Field, TypedArrayConstructor: TypedArrayConstructor<T>) {
-        return this.layout.readFixedWidthLayout(field, TypedArrayConstructor);
-    }
-}
diff --git a/js/src/recordbatch.ts b/js/src/recordbatch.ts
new file mode 100644
index 0000000000000..07d94a9d49629
--- /dev/null
+++ b/js/src/recordbatch.ts
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Schema, Struct, DataType } from './type';
+import { flatbuffers } from 'flatbuffers';
+import { View, Vector, StructVector } from './vector';
+import { Data, NestedData } from './data';
+
+import Long = flatbuffers.Long;
+
+export class RecordBatch extends StructVector {
+    public static from(vectors: Vector[]) {
+        return new RecordBatch(Schema.from(vectors),
+            Math.max(...vectors.map((v) => v.length)),
+            vectors
+        );
+    }
+    public readonly schema: Schema;
+    public readonly length: number;
+    public readonly numCols: number;
+    constructor(schema: Schema, data: Data<Struct>, view: View<Struct>);
+    constructor(schema: Schema, numRows: Long | number, cols: Data<any> | Vector[]);
+    constructor(...args: any[]) {
+        if (typeof args[1] !== 'number') {
+            const data = args[1] as Data<Struct>;
+            super(data, args[2]);
+            this.schema = args[0];
+            this.length = data.length;
+        } else {
+            const [schema, numRows, cols] = args;
+            const childData: Data<any>[] = new Array(cols.length);
+            for (let index = -1, length = cols.length; ++index < length;) {
+                const col: Data<any> | Vector = cols[index];
+                childData[index] = col instanceof Vector ? col.data : col;
+            }
+            super(new NestedData(new Struct(schema.fields), numRows, null, childData));
+            this.schema = schema;
+            this.length = numRows;
+        }
+        this.numCols = this.schema.fields.length;
+    }
+    public clone<R extends Struct>(data: Data<R>, view: View<R> = this.view.clone(data)): this {
+        return new RecordBatch(this.schema, data as any, view) as any;
+    }
+    public getChildAt<R extends DataType = DataType>(index: number): Vector<R> | null {
+        return index < 0 || index >= this.numCols ? null : super.getChildAt<R>(index);
+    }
+    public select(...columnNames: string[]) {
+        const fields = this.schema.fields;
+        const namesToKeep = columnNames.reduce((xs, x) => (xs[x] = true) && xs, Object.create(null));
+        return new RecordBatch(
+            this.schema.select(...columnNames), this.length,
+            this.childData.filter((_, i) => namesToKeep[fields[i].name])
+        );
+    }
+}
diff --git a/js/src/table.ts b/js/src/table.ts
new file mode 100644
index 0000000000000..3e50d16e3724d
--- /dev/null
+++ b/js/src/table.ts
@@ -0,0 +1,344 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { RecordBatch } from './recordbatch';
+import { Col, Predicate } from './predicate';
+import { Schema, Field, Struct } from './type';
+import { read, readAsync } from './ipc/reader/arrow';
+import { isPromise, isAsyncIterable } from './util/compat';
+import { Vector, DictionaryVector, IntVector, StructVector } from './vector';
+import { ChunkedView } from './vector/chunked';
+
+export type NextFunc = (idx: number, batch: RecordBatch) => void;
+export type BindFunc = (batch: RecordBatch) => void;
+
+export interface DataFrame {
+    filter(predicate: Predicate): DataFrame;
+    scan(next: NextFunc, bind?: BindFunc): void;
+    count(): number;
+    countBy(col: (Col|string)): CountByResult;
+}
+
+export class Table implements DataFrame {
+    static empty() { return new Table(new Schema([]), []); }
+    static from(sources?: Iterable<Uint8Array | Buffer | string> | object | string) {
+        if (sources) {
+            let schema: Schema | undefined;
+            let recordBatches: RecordBatch[] = [];
+            for (let recordBatch of read(sources)) {
+                schema = schema || recordBatch.schema;
+                recordBatches.push(recordBatch);
+            }
+            return new Table(schema || new Schema([]), recordBatches);
+        }
+        return Table.empty();
+    }
+    static async fromAsync(sources?: AsyncIterable<Uint8Array | Buffer | string>) {
+        if (isAsyncIterable(sources)) {
+            let schema: Schema | undefined;
+            let recordBatches: RecordBatch[] = [];
+            for await (let recordBatch of readAsync(sources)) {
+                schema = schema || recordBatch.schema;
+                recordBatches.push(recordBatch);
+            }
+            return new Table(schema || new Schema([]), recordBatches);
+        } else if (isPromise(sources)) {
+            return Table.from(await sources);
+        } else if (sources) {
+            return Table.from(sources);
+        }
+        return Table.empty();
+    }
+    static fromStruct(struct: StructVector) {
+        const schema = new Schema(struct.type.children);
+        const chunks = struct.view instanceof ChunkedView ?
+                            (struct.view.chunkVectors as StructVector[]) :
+                            [struct];
+        return new Table(chunks.map((chunk) => new RecordBatch(schema, chunk.length, chunk.view.childData)));
+    }
+
+    public readonly schema: Schema;
+    public readonly length: number;
+    public readonly numCols: number;
+    // List of inner RecordBatches
+    public readonly batches: RecordBatch[];
+    // List of inner Vectors, possibly spanning batches
+    protected readonly _columns: Vector<any>[] = [];
+    // Union of all inner RecordBatches into one RecordBatch, possibly chunked.
+    // If the Table has just one inner RecordBatch, this points to that.
+    // If the Table has multiple inner RecordBatches, then this is a Chunked view
+    // over the list of RecordBatches. This allows us to delegate the responsibility
+    // of indexing, iterating, slicing, and visiting to the Nested/Chunked Data/Views.
+    public readonly batchesUnion: RecordBatch;
+
+    constructor(batches: RecordBatch[]);
+    constructor(...batches: RecordBatch[]);
+    constructor(schema: Schema, batches: RecordBatch[]);
+    constructor(schema: Schema, ...batches: RecordBatch[]);
+    constructor(...args: any[]) {
+        let schema: Schema;
+        let batches: RecordBatch[];
+        if (args[0] instanceof Schema) {
+            schema = args[0];
+            batches = Array.isArray(args[1][0]) ? args[1][0] : args[1];
+        } else if (args[0] instanceof RecordBatch) {
+            schema = (batches = args)[0].schema;
+        } else {
+            schema = (batches = args[0])[0].schema;
+        }
+        this.schema = schema;
+        this.batches = batches;
+        this.batchesUnion = batches.length == 0 ?
+            new RecordBatch(schema, 0, []) :
+            batches.reduce((union, batch) => union.concat(batch));
+        this.length = this.batchesUnion.length;
+        this.numCols = this.batchesUnion.numCols;
+    }
+    public get(index: number): Struct['TValue'] {
+        return this.batchesUnion.get(index)!;
+    }
+    public getColumn(name: string) {
+        return this.getColumnAt(this.getColumnIndex(name));
+    }
+    public getColumnAt(index: number) {
+        return index < 0 || index >= this.numCols
+            ? null
+            : this._columns[index] || (
+              this._columns[index] = this.batchesUnion.getChildAt(index)!);
+    }
+    public getColumnIndex(name: string) {
+        return this.schema.fields.findIndex((f) => f.name === name);
+    }
+    public [Symbol.iterator](): IterableIterator<Struct['TValue']> {
+        return this.batchesUnion[Symbol.iterator]() as any;
+    }
+    public filter(predicate: Predicate): DataFrame {
+        return new FilteredDataFrame(this.batches, predicate);
+    }
+    public scan(next: NextFunc, bind?: BindFunc) {
+        const batches = this.batches, numBatches = batches.length;
+        for (let batchIndex = -1; ++batchIndex < numBatches;) {
+            // load batches
+            const batch = batches[batchIndex];
+            if (bind) { bind(batch); }
+            // yield all indices
+            for (let index = -1, numRows = batch.length; ++index < numRows;) {
+                next(index, batch);
+            }
+        }
+    }
+    public count(): number { return this.length; }
+    public countBy(name: Col | string): CountByResult {
+        const batches = this.batches, numBatches = batches.length;
+        const count_by = typeof name === 'string' ? new Col(name) : name;
+        // Assume that all dictionary batches are deltas, which means that the
+        // last record batch has the most complete dictionary
+        count_by.bind(batches[numBatches - 1]);
+        const vector = count_by.vector as DictionaryVector;
+        if (!(vector instanceof DictionaryVector)) {
+            throw new Error('countBy currently only supports dictionary-encoded columns');
+        }
+        // TODO: Adjust array byte width based on overall length
+        // (e.g. if this.length <= 255 use Uint8Array, etc...)
+        const counts: Uint32Array = new Uint32Array(vector.dictionary.length);
+        for (let batchIndex = -1; ++batchIndex < numBatches;) {
+            // load batches
+            const batch = batches[batchIndex];
+            // rebind the countBy Col
+            count_by.bind(batch);
+            const keys = (count_by.vector as DictionaryVector).indicies;
+            // yield all indices
+            for (let index = -1, numRows = batch.length; ++index < numRows;) {
+                let key = keys.get(index);
+                if (key !== null) { counts[key]++; }
+            }
+        }
+        return new CountByResult(vector.dictionary, IntVector.from(counts));
+    }
+    public select(...columnNames: string[]) {
+        return new Table(this.batches.map((batch) => batch.select(...columnNames)));
+    }
+    public toString(separator?: string) {
+        let str = '';
+        for (const row of this.rowsToString(separator)) {
+            str += row + '\n';
+        }
+        return str;
+    }
+    public rowsToString(separator = ' | '): TableToStringIterator {
+        return new TableToStringIterator(tableRowsToString(this, separator));
+    }
+}
+
+class FilteredDataFrame implements DataFrame {
+    private predicate: Predicate;
+    private batches: RecordBatch[];
+    constructor (batches: RecordBatch[], predicate: Predicate) {
+        this.batches = batches;
+        this.predicate = predicate;
+    }
+    public scan(next: NextFunc, bind?: BindFunc) {
+        // inlined version of this:
+        // this.parent.scan((idx, columns) => {
+        //     if (this.predicate(idx, columns)) next(idx, columns);
+        // });
+        const batches = this.batches;
+        const numBatches = batches.length;
+        for (let batchIndex = -1; ++batchIndex < numBatches;) {
+            // load batches
+            const batch = batches[batchIndex];
+            // TODO: bind batches lazily
+            // If predicate doesn't match anything in the batch we don't need
+            // to bind the callback
+            if (bind) { bind(batch); }
+            const predicate = this.predicate.bind(batch);
+            // yield all indices
+            for (let index = -1, numRows = batch.length; ++index < numRows;) {
+                if (predicate(index, batch)) { next(index, batch); }
+            }
+        }
+    }
+    public count(): number {
+        // inlined version of this:
+        // let sum = 0;
+        // this.parent.scan((idx, columns) => {
+        //     if (this.predicate(idx, columns)) ++sum;
+        // });
+        // return sum;
+        let sum = 0;
+        const batches = this.batches;
+        const numBatches = batches.length;
+        for (let batchIndex = -1; ++batchIndex < numBatches;) {
+            // load batches
+            const batch = batches[batchIndex];
+            const predicate = this.predicate.bind(batch);
+            // yield all indices
+            for (let index = -1, numRows = batch.length; ++index < numRows;) {
+                if (predicate(index, batch)) { ++sum; }
+            }
+        }
+        return sum;
+    }
+    public filter(predicate: Predicate): DataFrame {
+        return new FilteredDataFrame(
+            this.batches,
+            this.predicate.and(predicate)
+        );
+    }
+    public countBy(name: Col | string): CountByResult {
+        const batches = this.batches, numBatches = batches.length;
+        const count_by = typeof name === 'string' ? new Col(name) : name;
+        // Assume that all dictionary batches are deltas, which means that the
+        // last record batch has the most complete dictionary
+        count_by.bind(batches[numBatches - 1]);
+        const vector = count_by.vector as DictionaryVector;
+        if (!(vector instanceof DictionaryVector)) {
+            throw new Error('countBy currently only supports dictionary-encoded columns');
+        }
+        // TODO: Adjust array byte width based on overall length
+        // (e.g. if this.length <= 255 use Uint8Array, etc...)
+        const counts: Uint32Array = new Uint32Array(vector.dictionary.length);
+        for (let batchIndex = -1; ++batchIndex < numBatches;) {
+            // load batches
+            const batch = batches[batchIndex];
+            const predicate = this.predicate.bind(batch);
+            // rebind the countBy Col
+            count_by.bind(batch);
+            const keys = (count_by.vector as DictionaryVector).indicies;
+            // yield all indices
+            for (let index = -1, numRows = batch.length; ++index < numRows;) {
+                let key = keys.get(index);
+                if (key !== null && predicate(index, batch)) { counts[key]++; }
+            }
+        }
+        return new CountByResult(vector.dictionary, IntVector.from(counts));
+    }
+}
+
+export class CountByResult extends Table implements DataFrame {
+    constructor(values: Vector, counts: IntVector<any>) {
+        super(
+            new RecordBatch(new Schema([
+                new Field('values', values.type),
+                new Field('counts', counts.type)
+            ]),
+            counts.length, [values, counts]
+        ));
+    }
+    public toJSON(): Object {
+        const values = this.getColumnAt(0)!;
+        const counts = this.getColumnAt(1)!;
+        const result = {} as { [k: string]: number | null };
+        for (let i = -1; ++i < this.length;) {
+            result[values.get(i)] = counts.get(i);
+        }
+        return result;
+    }
+}
+
+export class TableToStringIterator implements IterableIterator<string> {
+    constructor(private iterator: IterableIterator<string>) {}
+    [Symbol.iterator]() { return this.iterator; }
+    next(value?: any) { return this.iterator.next(value); }
+    throw(error?: any) { return this.iterator.throw && this.iterator.throw(error) || { done: true, value: '' }; }
+    return(value?: any) { return this.iterator.return && this.iterator.return(value) || { done: true, value: '' }; }
+    pipe(stream: NodeJS.WritableStream) {
+        let res: IteratorResult<string>;
+        let write = () => {
+            if (stream['writable']) {
+                do {
+                    if ((res = this.next()).done) { break; }
+                } while (stream['write'](res.value + '\n', 'utf8'));
+            }
+            if (!res || !res.done) {
+                stream['once']('drain', write);
+            } else if (!(stream as any)['isTTY']) {
+                stream['end']('\n');
+            }
+        };
+        write();
+    }
+}
+
+function* tableRowsToString(table: Table, separator = ' | ') {
+    const fields = table.schema.fields;
+    const header = ['row_id', ...fields.map((f) => `${f}`)].map(stringify);
+    const maxColumnWidths = header.map(x => x.length);
+    // Pass one to convert to strings and count max column widths
+    for (let i = -1, n = table.length - 1; ++i < n;) {
+        let val, row = [i, ...table.get(i)];
+        for (let j = -1, k = row.length; ++j < k; ) {
+            val = stringify(row[j]);
+            maxColumnWidths[j] = Math.max(maxColumnWidths[j], val.length);
+        }
+    }
+    yield header.map((x, j) => leftPad(x, ' ', maxColumnWidths[j])).join(separator);
+    for (let i = -1; ++i < table.length;) {
+        yield [i, ...table.get(i)]
+            .map((x) => stringify(x))
+            .map((x, j) => leftPad(x, ' ', maxColumnWidths[j]))
+            .join(separator);
+    }
+}
+
+function leftPad(str: string, fill: string, n: number) {
+    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
+}
+
+function stringify(x: any) {
+    return typeof x === 'string' ? `"${x}"` : ArrayBuffer.isView(x) ? `[${x}]` : JSON.stringify(x);
+}
diff --git a/js/src/type.ts b/js/src/type.ts
new file mode 100644
index 0000000000000..6f382bd5b2b05
--- /dev/null
+++ b/js/src/type.ts
@@ -0,0 +1,578 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Schema_ from './fb/Schema';
+import * as Message_ from './fb/Message';
+import { Vector, View } from './vector';
+import { flatbuffers } from 'flatbuffers';
+import { DictionaryBatch } from './ipc/metadata';
+import { TypeVisitor, VisitorNode } from './visitor';
+
+export import Long = flatbuffers.Long;
+export import ArrowType = Schema_.org.apache.arrow.flatbuf.Type;
+export import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
+export import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
+export import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
+export import UnionMode = Schema_.org.apache.arrow.flatbuf.UnionMode;
+export import VectorType = Schema_.org.apache.arrow.flatbuf.VectorType;
+export import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
+export import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
+export import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
+
+export class Schema {
+    public static from(vectors: Vector[]) {
+        return new Schema(vectors.map((v, i) => new Field('' + i, v.type)));
+    }
+    // @ts-ignore
+    protected _bodyLength: number;
+    // @ts-ignore
+    protected _headerType: MessageHeader;
+    public readonly fields: Field[];
+    public readonly version: MetadataVersion;
+    public readonly metadata?: Map<string, string>;
+    public readonly dictionaries: Map<number, Field<Dictionary>>;
+    constructor(fields: Field[],
+                metadata?: Map<string, string>,
+                version: MetadataVersion = MetadataVersion.V4,
+                dictionaries: Map<number, Field<Dictionary>> = new Map()) {
+        this.fields = fields;
+        this.version = version;
+        this.metadata = metadata;
+        this.dictionaries = dictionaries;
+    }
+    public get bodyLength() { return this._bodyLength; }
+    public get headerType() { return this._headerType; }
+    public select(...fieldNames: string[]): Schema {
+        const namesToKeep = fieldNames.reduce((xs, x) => (xs[x] = true) && xs, Object.create(null));
+        const newDictFields = new Map(), newFields = this.fields.filter((f) => namesToKeep[f.name]);
+        this.dictionaries.forEach((f, dictId) => (namesToKeep[f.name]) && newDictFields.set(dictId, f));
+        return new Schema(newFields, this.metadata, this.version, newDictFields);
+    }
+    public static [Symbol.toStringTag] = ((prototype: Schema) => {
+        prototype._bodyLength = 0;
+        prototype._headerType = MessageHeader.Schema;
+        return 'Schema';
+    })(Schema.prototype);
+}
+
+export class Field<T extends DataType = DataType> {
+    public readonly type: T;
+    public readonly name: string;
+    public readonly nullable: boolean;
+    public readonly metadata?: Map<string, string> | null;
+    constructor(name: string, type: T, nullable = false, metadata?: Map<string, string> | null) {
+        this.name = name;
+        this.type = type;
+        this.nullable = nullable;
+        this.metadata = metadata;
+    }
+    public toString() { return `${this.name}: ${this.type}`; }
+    public get typeId(): T['TType'] { return this.type.TType; }
+    public get [Symbol.toStringTag](): string { return 'Field'; }
+    public get indicies(): T | Int<any> {
+        return DataType.isDictionary(this.type) ? this.type.indicies : this.type;
+    }
+}
+
+export type TimeBitWidth = 32 | 64;
+export type IntBitWidth = 8 | 16 | 32 | 64;
+
+export type NumericType = Int | Float | Date_ | Time | Interval | Timestamp;
+export type FixedSizeType = Int64 |  Uint64 | Decimal | FixedSizeBinary;
+export type PrimitiveType = NumericType | FixedSizeType;
+
+export type FlatListType = Utf8 | Binary; // <-- these types have `offset`, `data`, and `validity` buffers
+export type FlatType = Bool | PrimitiveType | FlatListType; // <-- these types have `data` and `validity` buffers
+export type ListType = List<any>; // <-- these types have `offset` and `validity` buffers
+export type NestedType = Map_ | Struct | List<any> | FixedSizeList<any> | Union<any>; // <-- these types have `validity` buffer and nested childData
+export type SingleNestedType = List<any> | FixedSizeList<any>; // <-- these are nested types that can only have a single child
+
+/**
+ * *
+ * Main data type enumeration:
+ * *
+ * Data types in this library are all *logical*. They can be expressed as
+ * either a primitive physical type (bytes or bits of some fixed size), a
+ * nested type consisting of other data types, or another data type (e.g. a
+ * timestamp encoded as an int64)
+ */
+ export enum Type {
+    NONE            =  0,  // The default placeholder type
+    Null            =  1,  // A NULL type having no physical storage
+    Int             =  2,  // Signed or unsigned 8, 16, 32, or 64-bit little-endian integer
+    Float           =  3,  // 2, 4, or 8-byte floating point value
+    Binary          =  4,  // Variable-length bytes (no guarantee of UTF8-ness)
+    Utf8            =  5,  // UTF8 variable-length string as List<Char>
+    Bool            =  6,  // Boolean as 1 bit, LSB bit-packed ordering
+    Decimal         =  7,  // Precision-and-scale-based decimal type. Storage type depends on the parameters.
+    Date            =  8,  // int32_t days or int64_t milliseconds since the UNIX epoch
+    Time            =  9,  // Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight
+    Timestamp       = 10,  // Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond)
+    Interval        = 11,  // YEAR_MONTH or DAY_TIME interval in SQL style
+    List            = 12,  // A list of some logical data type
+    Struct          = 13,  // Struct of logical types
+    Union           = 14,  // Union of logical types
+    FixedSizeBinary = 15,  // Fixed-size binary. Each value occupies the same number of bytes
+    FixedSizeList   = 16,  // Fixed-size list. Each value occupies the same number of bytes
+    Map             = 17,  // Map of named logical types
+    Dictionary      = 'Dictionary',  // Dictionary aka Category type
+    DenseUnion      = 'DenseUnion',  // Dense Union of logical types
+    SparseUnion     = 'SparseUnion',  // Sparse Union of logical types
+}
+
+export interface DataType<TType extends Type = any> {
+    readonly TType: TType;
+    readonly TArray: any;
+    readonly TValue: any;
+    readonly ArrayType: any;
+}
+
+export abstract class DataType<TType extends Type = any> implements Partial<VisitorNode> {
+
+    // @ts-ignore
+    public [Symbol.toStringTag]: string;
+
+    static            isNull (x: DataType): x is Null            { return x.TType === Type.Null;            }
+    static             isInt (x: DataType): x is Int             { return x.TType === Type.Int;             }
+    static           isFloat (x: DataType): x is Float           { return x.TType === Type.Float;           }
+    static          isBinary (x: DataType): x is Binary          { return x.TType === Type.Binary;          }
+    static            isUtf8 (x: DataType): x is Utf8            { return x.TType === Type.Utf8;            }
+    static            isBool (x: DataType): x is Bool            { return x.TType === Type.Bool;            }
+    static         isDecimal (x: DataType): x is Decimal         { return x.TType === Type.Decimal;         }
+    static            isDate (x: DataType): x is Date_           { return x.TType === Type.Date;            }
+    static            isTime (x: DataType): x is Time            { return x.TType === Type.Time;            }
+    static       isTimestamp (x: DataType): x is Timestamp       { return x.TType === Type.Timestamp;       }
+    static        isInterval (x: DataType): x is Interval        { return x.TType === Type.Interval;        }
+    static            isList (x: DataType): x is List            { return x.TType === Type.List;            }
+    static          isStruct (x: DataType): x is Struct          { return x.TType === Type.Struct;          }
+    static           isUnion (x: DataType): x is Union           { return x.TType === Type.Union;           }
+    static      isDenseUnion (x: DataType): x is DenseUnion      { return x.TType === Type.DenseUnion;      }
+    static     isSparseUnion (x: DataType): x is SparseUnion     { return x.TType === Type.SparseUnion;     }
+    static isFixedSizeBinary (x: DataType): x is FixedSizeBinary { return x.TType === Type.FixedSizeBinary; }
+    static   isFixedSizeList (x: DataType): x is FixedSizeList   { return x.TType === Type.FixedSizeList;   }
+    static             isMap (x: DataType): x is Map_            { return x.TType === Type.Map;             }
+    static      isDictionary (x: DataType): x is Dictionary      { return x.TType === Type.Dictionary;      }
+
+    constructor(public readonly TType: TType,
+                public readonly children?: Field[]) {}
+
+    acceptTypeVisitor(visitor: TypeVisitor): any {
+        switch (this.TType) {
+            case Type.Null:            return DataType.isNull(this)            && visitor.visitNull(this)            || null;
+            case Type.Int:             return DataType.isInt(this)             && visitor.visitInt(this)             || null;
+            case Type.Float:           return DataType.isFloat(this)           && visitor.visitFloat(this)           || null;
+            case Type.Binary:          return DataType.isBinary(this)          && visitor.visitBinary(this)          || null;
+            case Type.Utf8:            return DataType.isUtf8(this)            && visitor.visitUtf8(this)            || null;
+            case Type.Bool:            return DataType.isBool(this)            && visitor.visitBool(this)            || null;
+            case Type.Decimal:         return DataType.isDecimal(this)         && visitor.visitDecimal(this)         || null;
+            case Type.Date:            return DataType.isDate(this)            && visitor.visitDate(this)            || null;
+            case Type.Time:            return DataType.isTime(this)            && visitor.visitTime(this)            || null;
+            case Type.Timestamp:       return DataType.isTimestamp(this)       && visitor.visitTimestamp(this)       || null;
+            case Type.Interval:        return DataType.isInterval(this)        && visitor.visitInterval(this)        || null;
+            case Type.List:            return DataType.isList(this)            && visitor.visitList(this)            || null;
+            case Type.Struct:          return DataType.isStruct(this)          && visitor.visitStruct(this)          || null;
+            case Type.Union:           return DataType.isUnion(this)           && visitor.visitUnion(this)           || null;
+            case Type.FixedSizeBinary: return DataType.isFixedSizeBinary(this) && visitor.visitFixedSizeBinary(this) || null;
+            case Type.FixedSizeList:   return DataType.isFixedSizeList(this)   && visitor.visitFixedSizeList(this)   || null;
+            case Type.Map:             return DataType.isMap(this)             && visitor.visitMap(this)             || null;
+            case Type.Dictionary:      return DataType.isDictionary(this)      && visitor.visitDictionary(this)      || null;
+            default: return null;
+        }
+    }
+    protected static [Symbol.toStringTag] = ((proto: DataType) => {
+        (<any> proto).ArrayType = Array;
+        return proto[Symbol.toStringTag] = 'DataType';
+    })(DataType.prototype);
+}
+
+export interface Null extends DataType<Type.Null> { TArray: void; TValue: null; }
+export class Null extends DataType<Type.Null> {
+    constructor() {
+        super(Type.Null);
+    }
+    public toString() { return `Null`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitNull(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Null) => {
+        return proto[Symbol.toStringTag] = 'Null';
+    })(Null.prototype);
+}
+
+export interface Int<TValueType = any, TArrayType extends IntArray = IntArray> extends DataType<Type.Int> { TArray: TArrayType; TValue: TValueType; }
+export class Int<TValueType = any, TArrayType extends IntArray = IntArray> extends DataType<Type.Int> {
+    constructor(public readonly isSigned: boolean,
+                public readonly bitWidth: IntBitWidth) {
+        super(Type.Int);
+    }
+    public get ArrayType(): TypedArrayConstructor<TArrayType> {
+        switch (this.bitWidth) {
+            case  8: return (this.isSigned ? Int8Array : Uint8Array) as any;
+            case 16: return (this.isSigned ? Int16Array : Uint16Array) as any;
+            case 32: return (this.isSigned ? Int32Array : Uint32Array) as any;
+            case 64: return (this.isSigned ? Int32Array : Uint32Array) as any;
+        }
+        throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
+    }
+    public toString() { return `${this.isSigned ? `I` : `Ui`}nt${this.bitWidth}`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitInt(this); }
+    protected static [Symbol.toStringTag] = ((proto: Int) => {
+        return proto[Symbol.toStringTag] = 'Int';
+    })(Int.prototype);
+}
+
+export class Int8 extends Int<number, Int8Array> { constructor() { super(true, 8); } }
+export class Int16 extends Int<number, Int16Array> { constructor() { super(true, 16); } }
+export class Int32 extends Int<number, Int32Array> { constructor() { super(true, 32); } }
+export class Int64 extends Int<Int32Array, Int32Array> { constructor() { super(true, 64); } }
+export class Uint8 extends Int<number, Uint8Array> { constructor() { super(false, 8); } }
+export class Uint16 extends Int<number, Uint16Array> { constructor() { super(false, 16); } }
+export class Uint32 extends Int<number, Uint32Array> { constructor() { super(false, 32); } }
+export class Uint64 extends Int<Uint32Array, Uint32Array> { constructor() { super(false, 64); } }
+
+export interface Float<TArrayType extends FloatArray = FloatArray> extends DataType<Type.Float> { TArray: TArrayType; TValue: number; }
+export class Float<TArrayType extends FloatArray = FloatArray> extends DataType<Type.Float> {
+    constructor(public readonly precision: Precision) {
+        super(Type.Float);
+    }
+    // @ts-ignore
+    public get ArrayType(): TypedArrayConstructor<TArrayType> {
+        switch (this.precision) {
+            case Precision.HALF: return Uint16Array as any;
+            case Precision.SINGLE: return Float32Array as any;
+            case Precision.DOUBLE: return Float64Array as any;
+        }
+        throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
+    }
+    public toString() { return `Float${(this.precision << 5) || 16}`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitFloat(this); }
+    protected static [Symbol.toStringTag] = ((proto: Float) => {
+        return proto[Symbol.toStringTag] = 'Float';
+    })(Float.prototype);
+}
+
+export class Float16 extends Float<Uint16Array> { constructor() { super(Precision.HALF); } }
+export class Float32 extends Float<Float32Array> { constructor() { super(Precision.SINGLE); } }
+export class Float64 extends Float<Float64Array> { constructor() { super(Precision.DOUBLE); } }
+
+export interface Binary extends DataType<Type.Binary> { TArray: Uint8Array; TValue: Uint8Array; }
+export class Binary extends DataType<Type.Binary> {
+    constructor() {
+        super(Type.Binary);
+    }
+    public toString() { return `Binary`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitBinary(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Binary) => {
+        (<any> proto).ArrayType = Uint8Array;
+        return proto[Symbol.toStringTag] = 'Binary';
+    })(Binary.prototype);
+}
+
+export interface Utf8 extends DataType<Type.Utf8> { TArray: Uint8Array; TValue: string; }
+export class Utf8 extends DataType<Type.Utf8> {
+    constructor() {
+        super(Type.Utf8);
+    }
+    public toString() { return `Utf8`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitUtf8(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Utf8) => {
+        (<any> proto).ArrayType = Uint8Array;
+        return proto[Symbol.toStringTag] = 'Utf8';
+    })(Utf8.prototype);
+}
+
+export interface Bool extends DataType<Type.Bool> { TArray: Uint8Array; TValue: boolean; }
+export class Bool extends DataType<Type.Bool> {
+    constructor() {
+        super(Type.Bool);
+    }
+    public toString() { return `Bool`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitBool(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Bool) => {
+        (<any> proto).ArrayType = Uint8Array;
+        return proto[Symbol.toStringTag] = 'Bool';
+    })(Bool.prototype);
+}
+
+export interface Decimal extends DataType<Type.Decimal> { TArray: Uint32Array; TValue: Uint32Array; }
+export class Decimal extends DataType<Type.Decimal> {
+    constructor(public readonly scale: number,
+                public readonly precision: number) {
+        super(Type.Decimal);
+    }
+    public toString() { return `Decimal[${this.precision}e${this.scale > 0 ? `+` : ``}${this.scale}]`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitDecimal(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Decimal) => {
+        (<any> proto).ArrayType = Uint32Array;
+        return proto[Symbol.toStringTag] = 'Decimal';
+    })(Decimal.prototype);
+}
+
+/* tslint:disable:class-name */
+export interface Date_ extends DataType<Type.Date> { TArray: Int32Array; TValue: Date; }
+export class Date_ extends DataType<Type.Date> {
+    constructor(public readonly unit: DateUnit) {
+        super(Type.Date);
+    }
+    public toString() { return `Date${(this.unit + 1) * 32}<${DateUnit[this.unit]}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitDate(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Date_) => {
+        (<any> proto).ArrayType = Int32Array;
+        return proto[Symbol.toStringTag] = 'Date';
+    })(Date_.prototype);
+}
+
+export interface Time extends DataType<Type.Time> { TArray: Uint32Array; TValue: number; }
+export class Time extends DataType<Type.Time> {
+    constructor(public readonly unit: TimeUnit,
+                public readonly bitWidth: TimeBitWidth) {
+        super(Type.Time);
+    }
+    public toString() { return `Time${this.bitWidth}<${TimeUnit[this.unit]}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitTime(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Time) => {
+        (<any> proto).ArrayType = Uint32Array;
+        return proto[Symbol.toStringTag] = 'Time';
+    })(Time.prototype);
+}
+
+export interface Timestamp extends DataType<Type.Timestamp> { TArray: Int32Array; TValue: number; }
+export class Timestamp extends DataType<Type.Timestamp> {
+    constructor(public unit: TimeUnit, public timezone?: string | null) {
+        super(Type.Timestamp);
+    }
+    public toString() { return `Timestamp<${TimeUnit[this.unit]}${this.timezone ? `, ${this.timezone}` : ``}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitTimestamp(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Timestamp) => {
+        (<any> proto).ArrayType = Int32Array;
+        return proto[Symbol.toStringTag] = 'Timestamp';
+    })(Timestamp.prototype);
+}
+
+export interface Interval extends DataType<Type.Interval> { TArray: Int32Array; TValue: Int32Array; }
+export class Interval extends DataType<Type.Interval> {
+    constructor(public unit: IntervalUnit) {
+        super(Type.Interval);
+    }
+    public toString() { return `Interval<${IntervalUnit[this.unit]}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitInterval(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Interval) => {
+        (<any> proto).ArrayType = Int32Array;
+        return proto[Symbol.toStringTag] = 'Interval';
+    })(Interval.prototype);
+}
+
+export interface List<T extends DataType = any> extends DataType<Type.List>  { TArray: any; TValue: Vector<T>; }
+export class List<T extends DataType = any> extends DataType<Type.List> {
+    constructor(public children: Field[]) {
+        super(Type.List, children);
+    }
+    public toString() { return `List<${this.valueType}>`; }
+    public get ArrayType() { return this.valueType.ArrayType; }
+    public get valueType() { return this.children[0].type as T; }
+    public get valueField() { return this.children[0] as Field<T>; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitList(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: List) => {
+        return proto[Symbol.toStringTag] = 'List';
+    })(List.prototype);
+}
+
+export interface Struct extends DataType<Type.Struct> { TArray: any; TValue: View<any>; }
+export class Struct extends DataType<Type.Struct> {
+    constructor(public children: Field[]) {
+        super(Type.Struct, children);
+    }
+    public toString() { return `Struct<${this.children.map((f) => f.type).join(`, `)}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitStruct(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Struct) => {
+        return proto[Symbol.toStringTag] = 'Struct';
+    })(Struct.prototype);
+}
+
+export interface Union<TType extends Type = any> extends DataType<TType> { TArray: Int8Array; TValue: any; }
+export class Union<TType extends Type = any> extends DataType<TType> {
+    constructor(public readonly mode: UnionMode,
+                public readonly typeIds: ArrowType[],
+                public readonly children: Field[]) {
+        super(<TType> (mode === UnionMode.Sparse ? Type.SparseUnion : Type.DenseUnion), children);
+    }
+    public toString() { return `${this[Symbol.toStringTag]}<${this.typeIds.map((x) => Type[x]).join(` | `)}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitUnion(this); }
+    protected static [Symbol.toStringTag] = ((proto: Union) => {
+        (<any> proto).ArrayType = Int8Array;
+        return proto[Symbol.toStringTag] = 'Union';
+    })(Union.prototype);
+}
+
+export class DenseUnion extends Union<Type.DenseUnion> {
+    constructor(typeIds: ArrowType[], children: Field[]) {
+        super(UnionMode.Dense, typeIds, children);
+    }
+    protected static [Symbol.toStringTag] = ((proto: DenseUnion) => {
+        return proto[Symbol.toStringTag] = 'DenseUnion';
+    })(DenseUnion.prototype);
+}
+
+export class SparseUnion extends Union<Type.SparseUnion> {
+    constructor(typeIds: ArrowType[], children: Field[]) {
+        super(UnionMode.Sparse, typeIds, children);
+    }
+    protected static [Symbol.toStringTag] = ((proto: SparseUnion) => {
+        return proto[Symbol.toStringTag] = 'SparseUnion';
+    })(SparseUnion.prototype);
+}
+
+export interface FixedSizeBinary extends DataType<Type.FixedSizeBinary> { TArray: Uint8Array; TValue: Uint8Array; }
+export class FixedSizeBinary extends DataType<Type.FixedSizeBinary> {
+    constructor(public readonly byteWidth: number) {
+        super(Type.FixedSizeBinary);
+    }
+    public toString() { return `FixedSizeBinary[${this.byteWidth}]`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitFixedSizeBinary(this); }
+    protected static [Symbol.toStringTag] = ((proto: FixedSizeBinary) => {
+        (<any> proto).ArrayType = Uint8Array;
+        return proto[Symbol.toStringTag] = 'FixedSizeBinary';
+    })(FixedSizeBinary.prototype);
+}
+
+export interface FixedSizeList<T extends DataType = any> extends DataType<Type.FixedSizeList> { TArray: any; TValue: Vector<T>; }
+export class FixedSizeList<T extends DataType = any> extends DataType<Type.FixedSizeList> {
+    constructor(public readonly listSize: number,
+                public readonly children: Field[]) {
+        super(Type.FixedSizeList, children);
+    }
+    public get ArrayType() { return this.valueType.ArrayType; }
+    public get valueType() { return this.children[0].type as T; }
+    public get valueField() { return this.children[0] as Field<T>; }
+    public toString() { return `FixedSizeList[${this.listSize}]<${this.valueType}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitFixedSizeList(this); }
+    protected static [Symbol.toStringTag] = ((proto: FixedSizeList) => {
+        return proto[Symbol.toStringTag] = 'FixedSizeList';
+    })(FixedSizeList.prototype);
+}
+
+/* tslint:disable:class-name */
+export interface Map_ extends DataType<Type.Map> { TArray: Uint8Array; TValue: View<any>; }
+export class Map_ extends DataType<Type.Map> {
+    constructor(public readonly keysSorted: boolean,
+                public readonly children: Field[]) {
+        super(Type.Map, children);
+    }
+    public toString() { return `Map<${this.children.join(`, `)}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitMap(this); }
+    protected static [Symbol.toStringTag] = ((proto: Map_) => {
+        return proto[Symbol.toStringTag] = 'Map_';
+    })(Map_.prototype);
+}
+
+export interface Dictionary<T extends DataType = any> extends DataType<Type.Dictionary> { TArray: T['TArray']; TValue: T['TValue']; }
+export class Dictionary<T extends DataType> extends DataType<Type.Dictionary> {
+    public readonly id: number;
+    public readonly dictionary: T;
+    public readonly indicies: Int<any>;
+    public readonly isOrdered: boolean;
+    constructor(dictionary: T, indicies: Int<any>, id?: Long | number | null, isOrdered?: boolean | null) {
+        super(Type.Dictionary);
+        this.indicies = indicies;
+        this.dictionary = dictionary;
+        this.isOrdered = isOrdered || false;
+        this.id = id == null ? DictionaryBatch.getId() : typeof id === 'number' ? id : id.low;
+    }
+    public get ArrayType() { return this.dictionary.ArrayType; }
+    public toString() { return `Dictionary<${this.dictionary}, ${this.indicies}>`; }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return visitor.visitDictionary(this);
+    }
+    protected static [Symbol.toStringTag] = ((proto: Dictionary) => {
+        return proto[Symbol.toStringTag] = 'Dictionary';
+    })(Dictionary.prototype);
+}
+export interface IterableArrayLike<T = any> extends ArrayLike<T>, Iterable<T> {}
+
+export interface TypedArrayConstructor<T extends TypedArray = TypedArray> {
+    readonly prototype: T;
+    readonly BYTES_PER_ELEMENT: number;
+    new (length: number): T;
+    new (elements: Iterable<number>): T;
+    new (arrayOrArrayBuffer: ArrayLike<number> | ArrayBufferLike): T;
+    new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T;
+    of(...items: number[]): T;
+    from(arrayLike: ArrayLike<number> | Iterable<number>, mapfn?: (v: number, k: number) => number, thisArg?: any): T;
+}
+
+export type FloatArray = Uint16Array | Float32Array | Float64Array;
+export type IntArray = Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array;
+
+export interface TypedArray extends Iterable<number> {
+    [index: number]: number;
+    readonly length: number;
+    readonly byteLength: number;
+    readonly byteOffset: number;
+    readonly buffer: ArrayBufferLike;
+    readonly BYTES_PER_ELEMENT: number;
+    [Symbol.toStringTag]: any;
+    [Symbol.iterator](): IterableIterator<number>;
+    entries(): IterableIterator<[number, number]>;
+    keys(): IterableIterator<number>;
+    values(): IterableIterator<number>;
+    copyWithin(target: number, start: number, end?: number): this;
+    every(callbackfn: (value: number, index: number, array: TypedArray) => boolean, thisArg?: any): boolean;
+    fill(value: number, start?: number, end?: number): this;
+    filter(callbackfn: (value: number, index: number, array: TypedArray) => any, thisArg?: any): TypedArray;
+    find(predicate: (value: number, index: number, obj: TypedArray) => boolean, thisArg?: any): number | undefined;
+    findIndex(predicate: (value: number, index: number, obj: TypedArray) => boolean, thisArg?: any): number;
+    forEach(callbackfn: (value: number, index: number, array: TypedArray) => void, thisArg?: any): void;
+    includes(searchElement: number, fromIndex?: number): boolean;
+    indexOf(searchElement: number, fromIndex?: number): number;
+    join(separator?: string): string;
+    lastIndexOf(searchElement: number, fromIndex?: number): number;
+    map(callbackfn: (value: number, index: number, array: TypedArray) => number, thisArg?: any): TypedArray;
+    reduce(callbackfn: (previousValue: number, currentValue: number, currentIndex: number, array: TypedArray) => number): number;
+    reduce(callbackfn: (previousValue: number, currentValue: number, currentIndex: number, array: TypedArray) => number, initialValue: number): number;
+    reduce<U>(callbackfn: (previousValue: U, currentValue: number, currentIndex: number, array: TypedArray) => U, initialValue: U): U;
+    reduceRight(callbackfn: (previousValue: number, currentValue: number, currentIndex: number, array: TypedArray) => number): number;
+    reduceRight(callbackfn: (previousValue: number, currentValue: number, currentIndex: number, array: TypedArray) => number, initialValue: number): number;
+    reduceRight<U>(callbackfn: (previousValue: U, currentValue: number, currentIndex: number, array: TypedArray) => U, initialValue: U): U;
+    reverse(): TypedArray;
+    set(array: ArrayLike<number>, offset?: number): void;
+    slice(start?: number, end?: number): TypedArray;
+    some(callbackfn: (value: number, index: number, array: TypedArray) => boolean, thisArg?: any): boolean;
+    sort(compareFn?: (a: number, b: number) => number): this;
+    subarray(begin: number, end?: number): TypedArray;
+    toLocaleString(): string;
+    toString(): string;
+}
diff --git a/js/src/util/bit.ts b/js/src/util/bit.ts
new file mode 100644
index 0000000000000..2308bf6a2e03c
--- /dev/null
+++ b/js/src/util/bit.ts
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { TypedArray } from '../type';
+
+export function align(value: number, alignment: number) {
+    return value + padding(value, alignment);
+}
+
+export function padding(value: number, alignment: number) {
+    return (value % alignment === 0 ? 0 : alignment - value % alignment);
+}
+
+export function getBool(_data: any, _index: number, byte: number, bit: number) {
+    return (byte & 1 << bit) !== 0;
+}
+
+export function getBit(_data: any, _index: number, byte: number, bit: number): 0 | 1 {
+    return (byte & 1 << bit) >> bit as (0 | 1);
+}
+
+export function setBool(bytes: Uint8Array, index: number, value: any) {
+    return value ?
+        !!(bytes[index >> 3] |=  (1 << (index % 8))) || true :
+        !(bytes[index >> 3] &= ~(1 << (index % 8))) && false ;
+}
+
+export function packBools(values: Iterable<any>) {
+    let n = 0, i = 0;
+    let xs: number[] = [];
+    let bit = 0, byte = 0;
+    for (const value of values) {
+        value && (byte |= 1 << bit);
+        if (++bit === 8) {
+            xs[i++] = byte;
+            byte = bit = 0;
+        }
+    }
+    if (i === 0 || bit > 0) { xs[i++] = byte; }
+    if (i % 8 && (n = i + 8 - i % 8)) {
+        do { xs[i] = 0; } while (++i < n);
+    }
+    return new Uint8Array(xs);
+}
+
+export function* iterateBits<T>(bytes: Uint8Array, begin: number, length: number, context: any,
+                                get: (context: any, index: number, byte: number, bit: number) => T) {
+    let bit = begin % 8;
+    let byteIndex = begin >> 3;
+    let index = 0, remaining = length;
+    for (; remaining > 0; bit = 0) {
+        let byte = bytes[byteIndex++];
+        do {
+            yield get(context, index++, byte, bit);
+        } while (--remaining > 0 && ++bit < 8);
+    }
+}
+
+/**
+ * Compute the population count (the number of bits set to 1) for a range of bits in a Uint8Array.
+ * @param vector The Uint8Array of bits for which to compute the population count.
+ * @param lhs The range's left-hand side (or start) bit
+ * @param rhs The range's right-hand side (or end) bit
+ */
+export function popcnt_bit_range(data: Uint8Array, lhs: number, rhs: number): number {
+    if (rhs - lhs <= 0) { return 0; }
+    // If the bit range is less than one byte, sum the 1 bits in the bit range
+    if (rhs - lhs < 8) {
+        let sum = 0;
+        for (const bit of iterateBits(data, lhs, rhs - lhs, data, getBit)) {
+            sum += bit;
+        }
+        return sum;
+    }
+    // Get the next lowest multiple of 8 from the right hand side
+    const rhsInside = rhs >> 3 << 3;
+    // Get the next highest multiple of 8 from the left hand side
+    const lhsInside = lhs + (lhs % 8 === 0 ? 0 : 8 - lhs % 8);
+    return (
+        // Get the popcnt of bits between the left hand side, and the next highest multiple of 8
+        popcnt_bit_range(data, lhs, lhsInside) +
+        // Get the popcnt of bits between the right hand side, and the next lowest multiple of 8
+        popcnt_bit_range(data, rhsInside, rhs) +
+        // Get the popcnt of all bits between the left and right hand sides' multiples of 8
+        popcnt_array(data, lhsInside >> 3, (rhsInside - lhsInside) >> 3)
+    );
+}
+
+export function popcnt_array(arr: TypedArray, byteOffset?: number, byteLength?: number) {
+    let cnt = 0, pos = byteOffset! | 0;
+    const view = new DataView(arr.buffer, arr.byteOffset, arr.byteLength);
+    const len =  byteLength === void 0 ? arr.byteLength : pos + byteLength;
+    while (len - pos >= 4) {
+        cnt += popcnt_uint32(view.getUint32(pos));
+        pos += 4;
+    }
+    while (len - pos >= 2) {
+        cnt += popcnt_uint32(view.getUint16(pos));
+        pos += 2;
+    }
+    while (len - pos >= 1) {
+        cnt += popcnt_uint32(view.getUint8(pos));
+        pos += 1;
+    }
+    return cnt;
+}
+
+export function popcnt_uint32(uint32: number): number {
+    let i = uint32 | 0;
+    i = i - ((i >>> 1) & 0x55555555);
+    i = (i & 0x33333333) + ((i >>> 2) & 0x33333333);
+    return (((i + (i >>> 4)) & 0x0F0F0F0F) * 0x01010101) >>> 24;
+}
diff --git a/js/src/util/compat.ts b/js/src/util/compat.ts
new file mode 100644
index 0000000000000..7a4232ee8c32e
--- /dev/null
+++ b/js/src/util/compat.ts
@@ -0,0 +1,49 @@
+export interface Subscription {
+    unsubscribe: () => void;
+}
+
+export interface Observer<T> {
+    closed?: boolean;
+    next: (value: T) => void;
+    error: (err: any) => void;
+    complete: () => void;
+}
+
+export interface Observable<T> {
+    subscribe: (observer: Observer<T>) => Subscription;
+}
+
+/**
+ * @ignore
+ */
+export function isPromise(x: any): x is PromiseLike<any> {
+    return x != null && Object(x) === x && typeof x['then'] === 'function';
+}
+
+/**
+ * @ignore
+ */
+export function isObservable(x: any): x is Observable<any> {
+    return x != null && Object(x) === x && typeof x['subscribe'] === 'function';
+}
+
+/**
+ * @ignore
+ */
+export function isArrayLike(x: any): x is ArrayLike<any> {
+    return x != null && Object(x) === x && typeof x['length'] === 'number';
+}
+
+/**
+ * @ignore
+ */
+export function isIterable(x: any): x is Iterable<any> {
+    return x != null && Object(x) === x && typeof x[Symbol.iterator] !== 'undefined';
+}
+
+/**
+ * @ignore
+ */
+export function isAsyncIterable(x: any): x is AsyncIterable<any> {
+    return x != null && Object(x) === x && typeof x[Symbol.asyncIterator] !== 'undefined';
+}
diff --git a/js/src/util/layout.ts b/js/src/util/layout.ts
index c064ee9d7d0b0..29698fb3d2b93 100644
--- a/js/src/util/layout.ts
+++ b/js/src/util/layout.ts
@@ -15,16 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { align } from './bit';
 import { TextEncoder } from 'text-encoding-utf-8';
-import { TypedArrayConstructor, TypedArray } from '../vector/types';
-
-export function align(value: number, alignment: number) {
-    return value + padding(value, alignment);
-}
-
-export function padding(value: number, alignment: number) {
-    return (value % alignment === 0 ? 0 : alignment - value % alignment);
-}
+import { TypedArrayConstructor, TypedArray } from '../type';
 
 export type NullableLayout = { nullCount: number, validity: Uint8Array };
 export type BufferLayout<TArray = ArrayLike<number>> = { data: TArray };
diff --git a/js/src/vector.ts b/js/src/vector.ts
new file mode 100644
index 0000000000000..d9ca97b5fd120
--- /dev/null
+++ b/js/src/vector.ts
@@ -0,0 +1,441 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Data, ChunkedData, FlatData, BoolData, FlatListData, NestedData, DictionaryData } from './data';
+import { VisitorNode, TypeVisitor, VectorVisitor } from './visitor';
+import { DataType, ListType, FlatType, NestedType, FlatListType, TimeUnit } from './type';
+import { IterableArrayLike, Precision, DateUnit, IntervalUnit, UnionMode } from './type';
+
+export interface VectorLike { length: number; nullCount: number; }
+
+export interface View<T extends DataType> {
+    clone(data: Data<T>): this;
+    isValid(index: number): boolean;
+    get(index: number): T['TValue'] | null;
+    set(index: number, value: T['TValue']): void;
+    toArray(): IterableArrayLike<T['TValue'] | null>;
+    [Symbol.iterator](): IterableIterator<T['TValue'] | null>;
+}
+
+export class Vector<T extends DataType = any> implements VectorLike, View<T>, VisitorNode {
+    public static create<T extends DataType>(data: Data<T>): Vector<T> {
+        return createVector(data);
+    }
+    public static concat<T extends DataType>(source?: Vector<T> | null, ...others: Vector<T>[]): Vector<T> {
+        return others.reduce((a, b) => a ? a.concat(b) : b, source!);
+    }
+    public type: T;
+    public length: number;
+    public readonly data: Data<T>;
+    public readonly view: View<T>;
+    constructor(data: Data<T>, view: View<T>) {
+        this.data = data;
+        this.type = data.type;
+        this.length = data.length;
+        let nulls: Uint8Array;
+        if ((<any> data instanceof ChunkedData) && !(view instanceof ChunkedView)) {
+            this.view = new ChunkedView(data);
+        } else if (!(view instanceof ValidityView) && (nulls = data.nullBitmap!) && nulls.length > 0 && data.nullCount > 0) {
+            this.view = new ValidityView(data, view);
+        } else {
+            this.view = view;
+        }
+    }
+
+    public get nullCount() { return this.data.nullCount; }
+    public get nullBitmap() { return this.data.nullBitmap; }
+    public get [Symbol.toStringTag]() {
+        return `Vector<${this.type[Symbol.toStringTag]}>`;
+    }
+    public toJSON(): any { return this.toArray(); }
+    public clone<R extends T>(data: Data<R>, view: View<R> = this.view.clone(data) as any): this {
+        return new (this.constructor as any)(data, view);
+    }
+    public isValid(index: number): boolean {
+        return this.view.isValid(index);
+    }
+    public get(index: number): T['TValue'] | null {
+        return this.view.get(index);
+    }
+    public set(index: number, value: T['TValue']): void {
+        return this.view.set(index, value);
+    }
+    public toArray(): IterableArrayLike<T['TValue'] | null> {
+        return this.view.toArray();
+    }
+    public [Symbol.iterator](): IterableIterator<T['TValue'] | null> {
+        return this.view[Symbol.iterator]();
+    }
+    public concat(...others: Vector<T>[]): this {
+        if ((others = others.filter(Boolean)).length === 0) {
+            return this;
+        }
+        const { view } = this;
+        const vecs = !(view instanceof ChunkedView)
+            ? [this, ...others]
+            : [...view.chunkVectors, ...others];
+        const offsets = ChunkedData.computeOffsets(vecs);
+        const chunksLength = offsets[offsets.length - 1];
+        const chunkedData = new ChunkedData(this.type, chunksLength, vecs, 0, -1, offsets);
+        return this.clone(chunkedData, new ChunkedView(chunkedData)) as this;
+    }
+    public slice(begin?: number, end?: number): this {
+        let { length } = this;
+        let size = (this.view as any).size || 1;
+        let total = length, from = (begin || 0) * size;
+        let to = (typeof end === 'number' ? end : total) * size;
+        if (to < 0) { to = total - (to * -1) % total; }
+        if (from < 0) { from = total - (from * -1) % total; }
+        if (to < from) { [from, to] = [to, from]; }
+        total = !isFinite(total = (to - from)) || total < 0 ? 0 : total;
+        const slicedData = this.data.slice(from, Math.min(total, length));
+        return this.clone(slicedData, this.view.clone(slicedData)) as this;
+    }
+
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return TypeVisitor.visitTypeInline(visitor, this.type);
+    }
+    public acceptVectorVisitor(visitor: VectorVisitor): any {
+        return VectorVisitor.visitTypeInline(visitor, this.type, this);
+    }
+}
+
+export abstract class FlatVector<T extends FlatType> extends Vector<T> {
+    public get values() { return this.data.values; }
+    public lows(): IntVector<Int32> { return this.asInt32(0, 2); }
+    public highs(): IntVector<Int32> { return this.asInt32(1, 2); }
+    public asInt32(offset: number = 0, stride: number = 2): IntVector<Int32> {
+        let data = (this.data as FlatData<any>).clone(new Int32());
+        if (offset > 0) {
+            data = data.slice(offset, this.length - offset);
+        }
+        const int32s = new IntVector(data, new PrimitiveView(data, stride));
+        int32s.length = this.length / stride | 0;
+        return int32s;
+    }
+}
+
+export abstract class ListVectorBase<T extends (ListType | FlatListType)> extends Vector<T> {
+    public get values() { return this.data.values; }
+    public get valueOffsets() { return this.data.valueOffsets; }
+    public getValueOffset(index: number) {
+        return this.valueOffsets[index];
+    }
+    public getValueLength(index: number) {
+        return this.valueOffsets[index + 1] - this.valueOffsets[index];
+    }
+}
+
+export abstract class NestedVector<T extends NestedType> extends Vector<T>  {
+    // @ts-ignore
+    public readonly view: NestedView<T>;
+    // @ts-ignore
+    protected _childData: Data<any>[];
+    public getChildAt<R extends DataType = DataType>(index: number): Vector<R> | null {
+        return this.view.getChildAt<R>(index);
+    }
+    public get childData(): Data<any>[] {
+        let data: Data<T> | Data<any>[];
+        if ((data = this._childData)) {
+            // Return the cached childData reference first
+            return data as Data<any>[];
+        } else if (!(<any> (data = this.data) instanceof ChunkedData)) {
+            // If data isn't chunked, cache and return NestedData's childData
+            return this._childData = (data as NestedData<T>).childData;
+        }
+        // Otherwise if the data is chunked, concatenate the childVectors from each chunk
+        // to construct a single chunked Vector for each column. Then return the ChunkedData
+        // instance from each unified chunked column as the childData of a chunked NestedVector
+        const chunks = ((data as ChunkedData<T>).chunkVectors as NestedVector<T>[]);
+        return this._childData = chunks
+            .reduce<(Vector<T> | null)[][]>((cols, chunk) => chunk.childData
+            .reduce<(Vector<T> | null)[][]>((cols, _, i) => (
+                (cols[i] || (cols[i] = [])).push(chunk.getChildAt(i))
+            ) && cols || cols, cols), [] as Vector<T>[][])
+        .map((vecs) => Vector.concat<T>(...vecs).data);
+    }
+}
+
+import { List, Binary, Utf8, Bool, } from './type';
+import { Null, Int, Float, Decimal, Date_, Time, Timestamp, Interval } from './type';
+import { Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Float16, Float32, Float64 } from './type';
+import { Struct, Union, SparseUnion, DenseUnion, FixedSizeBinary, FixedSizeList, Map_, Dictionary } from './type';
+
+import { ChunkedView } from './vector/chunked';
+import { DictionaryView } from './vector/dictionary';
+import { ListView, FixedSizeListView, BinaryView, Utf8View } from './vector/list';
+import { UnionView, DenseUnionView, NestedView, StructView, MapView } from './vector/nested';
+import { FlatView, NullView, BoolView, ValidityView, PrimitiveView, FixedSizeView, Float16View } from './vector/flat';
+import { DateDayView, DateMillisecondView, IntervalYearMonthView } from './vector/flat';
+import { TimestampDayView, TimestampSecondView, TimestampMillisecondView, TimestampMicrosecondView, TimestampNanosecondView } from './vector/flat';
+import { packBools } from './util/bit';
+
+export class NullVector extends Vector<Null> {
+    constructor(data: Data<Null>, view: View<Null> = new NullView(data)) {
+        super(data, view);
+    }
+}
+
+export class BoolVector extends Vector<Bool> {
+    public static from(data: IterableArrayLike<boolean>) {
+        return new BoolVector(new BoolData(new Bool(), data.length, null, packBools(data)));
+    }
+    public get values() { return this.data.values; }
+    constructor(data: Data<Bool>, view: View<Bool> = new BoolView(data)) {
+        super(data, view);
+    }
+}
+
+export class IntVector<T extends Int = Int<any>> extends FlatVector<T> {
+    public static from(data: Int8Array): IntVector<Int8>;
+    public static from(data: Int16Array): IntVector<Int16>;
+    public static from(data: Int32Array): IntVector<Int32>;
+    public static from(data: Uint8Array): IntVector<Uint8>;
+    public static from(data: Uint16Array): IntVector<Uint16>;
+    public static from(data: Uint32Array): IntVector<Uint32>;
+    public static from(data: Int32Array, is64: true): IntVector<Int64>;
+    public static from(data: Uint32Array, is64: true): IntVector<Uint64>;
+    public static from(data: any, is64?: boolean) {
+        if (is64 === true) {
+            return data instanceof Int32Array
+                ? new IntVector(new FlatData(new Int64(), data.length, null, data))
+                : new IntVector(new FlatData(new Uint64(), data.length, null, data));
+        }
+        switch (data.constructor) {
+            case Int8Array: return new IntVector(new FlatData(new Int8(), data.length, null, data));
+            case Int16Array: return new IntVector(new FlatData(new Int16(), data.length, null, data));
+            case Int32Array: return new IntVector(new FlatData(new Int32(), data.length, null, data));
+            case Uint8Array: return new IntVector(new FlatData(new Uint8(), data.length, null, data));
+            case Uint16Array: return new IntVector(new FlatData(new Uint16(), data.length, null, data));
+            case Uint32Array: return new IntVector(new FlatData(new Uint32(), data.length, null, data));
+        }
+        throw new TypeError('Unrecognized Int data');
+    }
+    static defaultView<T extends Int>(data: Data<T>) {
+        return data.type.bitWidth <= 32 ? new FlatView(data) : new FixedSizeView(data, (data.type.bitWidth / 32) | 0);
+    }
+    constructor(data: Data<T>, view: View<T> = IntVector.defaultView(data)) {
+        super(data, view);
+    }
+}
+
+export class FloatVector<T extends Float = Float<any>> extends FlatVector<T> {
+    public static from(data: Uint16Array): FloatVector<Float16>;
+    public static from(data: Float32Array): FloatVector<Float32>;
+    public static from(data: Float64Array): FloatVector<Float64>;
+    public static from(data: any) {
+        switch (data.constructor) {
+            case Uint16Array: return new FloatVector(new FlatData(new Float16(), data.length, null, data));
+            case Float32Array: return new FloatVector(new FlatData(new Float32(), data.length, null, data));
+            case Float64Array: return new FloatVector(new FlatData(new Float64(), data.length, null, data));
+        }
+        throw new TypeError('Unrecognized Float data');
+    }
+    static defaultView<T extends Float>(data: Data<T>): FlatView<any> {
+        return data.type.precision !== Precision.HALF ? new FlatView(data) : new Float16View(data as Data<Float16>);
+    }
+    constructor(data: Data<T>, view: View<T> = FloatVector.defaultView(data)) {
+        super(data, view);
+    }
+}
+
+export class DateVector extends FlatVector<Date_> {
+    static defaultView<T extends Date_>(data: Data<T>) {
+        return data.type.unit === DateUnit.DAY ? new DateDayView(data) : new DateMillisecondView(data, 2);
+    }
+    constructor(data: Data<Date_>, view: View<Date_> = DateVector.defaultView(data)) {
+        super(data, view);
+    }
+    public lows(): IntVector<Int32> {
+        return this.type.unit === DateUnit.DAY ? this.asInt32(0, 1) : this.asInt32(0, 2);
+    }
+    public highs(): IntVector<Int32> {
+        return this.type.unit === DateUnit.DAY ? this.asInt32(0, 1) : this.asInt32(1, 2);
+    }
+    public asEpochMilliseconds(): IntVector<Int32> {
+        let data = (this.data as FlatData<any>).clone(new Int32());
+        switch (this.type.unit) {
+            case DateUnit.DAY: return new IntVector(data, new TimestampDayView(data as any, 1) as any);
+            case DateUnit.MILLISECOND: return new IntVector(data, new TimestampMillisecondView(data as any, 2) as any);
+        }
+        throw new TypeError(`Unrecognized date unit "${DateUnit[this.type.unit]}"`);
+    }
+}
+
+export class DecimalVector extends FlatVector<Decimal> {
+    constructor(data: Data<Decimal>, view: View<Decimal> = new FixedSizeView(data, 4)) {
+        super(data, view);
+    }
+}
+
+export class TimeVector extends FlatVector<Time> {
+    static defaultView<T extends Time>(data: Data<T>) {
+        return data.type.bitWidth <= 32 ? new FlatView(data) : new FixedSizeView(data, (data.type.bitWidth / 32) | 0);
+    }
+    constructor(data: Data<Time>, view: View<Time> = TimeVector.defaultView(data)) {
+        super(data, view);
+    }
+    public lows(): IntVector<Int32> {
+        return this.type.bitWidth <= 32 ? this.asInt32(0, 1) : this.asInt32(0, 2);
+    }
+    public highs(): IntVector<Int32> {
+        return this.type.bitWidth <= 32 ? this.asInt32(0, 1) : this.asInt32(1, 2);
+    }
+}
+
+export class TimestampVector extends FlatVector<Timestamp> {
+    constructor(data: Data<Timestamp>, view: View<Timestamp> = new FixedSizeView(data, 2)) {
+        super(data, view);
+    }
+    public asEpochMilliseconds(): IntVector<Int32> {
+        let data = (this.data as FlatData<any>).clone(new Int32());
+        switch (this.type.unit) {
+            case TimeUnit.SECOND: return new IntVector(data, new TimestampSecondView(data as any, 1) as any);
+            case TimeUnit.MILLISECOND: return new IntVector(data, new TimestampMillisecondView(data as any, 2) as any);
+            case TimeUnit.MICROSECOND: return new IntVector(data, new TimestampMicrosecondView(data as any, 2) as any);
+            case TimeUnit.NANOSECOND: return new IntVector(data, new TimestampNanosecondView(data as any, 2) as any);
+        }
+        throw new TypeError(`Unrecognized time unit "${TimeUnit[this.type.unit]}"`);
+    }
+}
+
+export class IntervalVector extends FlatVector<Interval> {
+    static defaultView<T extends Interval>(data: Data<T>) {
+        return data.type.unit === IntervalUnit.YEAR_MONTH ? new IntervalYearMonthView(data) : new FixedSizeView(data, 2);
+    }
+    constructor(data: Data<Interval>, view: View<Interval> = IntervalVector.defaultView(data)) {
+        super(data, view);
+    }
+    public lows(): IntVector<Int32> {
+        return this.type.unit === IntervalUnit.YEAR_MONTH ? this.asInt32(0, 1) : this.asInt32(0, 2);
+    }
+    public highs(): IntVector<Int32> {
+        return this.type.unit === IntervalUnit.YEAR_MONTH ? this.asInt32(0, 1) : this.asInt32(1, 2);
+    }
+}
+
+export class BinaryVector extends ListVectorBase<Binary> {
+    constructor(data: Data<Binary>, view: View<Binary> = new BinaryView(data)) {
+        super(data, view);
+    }
+    public asUtf8() {
+        return new Utf8Vector((this.data as FlatListData<any>).clone(new Utf8()));
+    }
+}
+
+export class FixedSizeBinaryVector extends FlatVector<FixedSizeBinary> {
+    constructor(data: Data<FixedSizeBinary>, view: View<FixedSizeBinary> = new FixedSizeView(data, data.type.byteWidth)) {
+        super(data, view);
+    }
+}
+
+export class Utf8Vector extends ListVectorBase<Utf8> {
+    constructor(data: Data<Utf8>, view: View<Utf8> = new Utf8View(data)) {
+        super(data, view);
+    }
+    public asBinary() {
+        return new BinaryVector((this.data as FlatListData<any>).clone(new Binary()));
+    }
+}
+
+export class ListVector<T extends DataType = DataType> extends ListVectorBase<List<T>> {
+    constructor(data: Data<List<T>>, view: View<List<T>> = new ListView(data)) {
+        super(data, view);
+    }
+}
+
+export class FixedSizeListVector extends Vector<FixedSizeList> {
+    constructor(data: Data<FixedSizeList>, view: View<FixedSizeList> = new FixedSizeListView(data)) {
+        super(data, view);
+    }
+}
+
+export class MapVector extends NestedVector<Map_> {
+    constructor(data: Data<Map_>, view: View<Map_> = new MapView(data)) {
+        super(data, view);
+    }
+    public asStruct() {
+        return new StructVector((this.data as NestedData<any>).clone(new Struct(this.type.children)));
+    }
+}
+
+export class StructVector extends NestedVector<Struct> {
+    constructor(data: Data<Struct>, view: View<Struct> = new StructView(data)) {
+        super(data, view);
+    }
+    public asMap(keysSorted: boolean = false) {
+        return new MapVector((this.data as NestedData<any>).clone(new Map_(keysSorted, this.type.children)));
+    }
+}
+
+export class UnionVector<T extends (SparseUnion | DenseUnion) = any> extends NestedVector<T> {
+    constructor(data: Data<T>, view: View<T> = <any> (data.type.mode === UnionMode.Sparse ? new UnionView<SparseUnion>(data as Data<SparseUnion>) : new DenseUnionView(data as Data<DenseUnion>))) {
+        super(data, view);
+    }
+}
+
+export class DictionaryVector<T extends DataType = DataType> extends Vector<Dictionary<T>> {
+    // @ts-ignore
+    public readonly indicies: Vector<Int>;
+    // @ts-ignore
+    public readonly dictionary: Vector<T>;
+    constructor(data: Data<Dictionary<T>>, view: View<Dictionary<T>> = new DictionaryView<T>(data.dictionary, new IntVector(data.indicies))) {
+        super(data as Data<any>, view);
+        if (data instanceof DictionaryData && view instanceof DictionaryView) {
+            this.indicies = view.indicies;
+            this.dictionary = data.dictionary;
+        } else if (data instanceof ChunkedData && view instanceof ChunkedView) {
+            const chunks = view.chunkVectors as DictionaryVector<T>[];
+            // Assume the last chunk's dictionary data is the most up-to-date,
+            // including data from DictionaryBatches that were marked as deltas
+            this.dictionary = chunks[chunks.length - 1].dictionary;
+            this.indicies = chunks.reduce<Vector<Int> | null>(
+                (idxs: Vector<Int> | null, dict: DictionaryVector<T>) =>
+                    !idxs ? dict.indicies! : idxs.concat(dict.indicies!),
+                null
+            )!;
+        } else {
+            throw new TypeError(`Unrecognized DictionaryVector view`);
+        }
+    }
+    public getKey(index: number) { return this.indicies.get(index); }
+    public getValue(key: number) { return this.dictionary.get(key); }
+}
+
+export const createVector = ((VectorLoader: new <T extends DataType>(data: Data<T>) => TypeVisitor) => (
+    <T extends DataType>(data: Data<T>) => TypeVisitor.visitTypeInline(new VectorLoader(data), data.type) as Vector<T>
+))(class VectorLoader<T extends DataType> extends TypeVisitor {
+    constructor(private data: Data<T>) { super(); }
+    visitNull           (_type: Null)            { return new NullVector(this.data);            }
+    visitInt            (_type: Int)             { return new IntVector(this.data);             }
+    visitFloat          (_type: Float)           { return new FloatVector(this.data);           }
+    visitBinary         (_type: Binary)          { return new BinaryVector(this.data);          }
+    visitUtf8           (_type: Utf8)            { return new Utf8Vector(this.data);            }
+    visitBool           (_type: Bool)            { return new BoolVector(this.data);            }
+    visitDecimal        (_type: Decimal)         { return new DecimalVector(this.data);         }
+    visitDate           (_type: Date_)           { return new DateVector(this.data);            }
+    visitTime           (_type: Time)            { return new TimeVector(this.data);            }
+    visitTimestamp      (_type: Timestamp)       { return new TimestampVector(this.data);       }
+    visitInterval       (_type: Interval)        { return new IntervalVector(this.data);        }
+    visitList           (_type: List)            { return new ListVector(this.data);            }
+    visitStruct         (_type: Struct)          { return new StructVector(this.data);          }
+    visitUnion          (_type: Union)           { return new UnionVector(this.data);           }
+    visitFixedSizeBinary(_type: FixedSizeBinary) { return new FixedSizeBinaryVector(this.data); }
+    visitFixedSizeList  (_type: FixedSizeList)   { return new FixedSizeListVector(this.data);   }
+    visitMap            (_type: Map_)            { return new MapVector(this.data);             }
+    visitDictionary     (_type: Dictionary)      { return new DictionaryVector(this.data);      }
+});
diff --git a/js/src/vector/arrow.ts b/js/src/vector/arrow.ts
deleted file mode 100644
index 64a43bdab1c6b..0000000000000
--- a/js/src/vector/arrow.ts
+++ /dev/null
@@ -1,88 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from './vector';
-import * as vectors from './traits/vectors';
-import * as fieldVectors from './traits/field';
-import * as nullableVectors from './traits/nullable';
-import * as nullableFieldVectors from './traits/nullablefield';
-import { Field, FieldNode } from '../format/arrow';
-import { isFieldArgv, isNullableArgv } from './traits/mixins';
-
-function MixinArrowTraits<T extends Vector<any>, TArgv>(
-    Base: new (argv: TArgv) => T,
-    Field: new (argv: TArgv & { field: Field, fieldNode: FieldNode }) => T,
-    Nullable: new (argv: TArgv & { validity: Uint8Array }) => T,
-    NullableField: new (argv: TArgv & { validity: Uint8Array, field: Field, fieldNode: FieldNode }) => T
-) {
-    return function(argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })) {
-        return new (!isFieldArgv(argv)
-            ? !isNullableArgv(argv) ? Base : Nullable
-            : !isNullableArgv(argv) ? Field : NullableField
-        )(argv as any);
-    } as any as { new (argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })): T };
-}
-
-export { Vector };
-export const MixinListVector = MixinArrowTraits(vectors.ListVector as any, fieldVectors.ListVector as any, nullableVectors.ListVector as any, nullableFieldVectors.ListVector as any);
-export class ListVector extends MixinListVector {}
-export const MixinBinaryVector = MixinArrowTraits(vectors.BinaryVector as any, fieldVectors.BinaryVector as any, nullableVectors.BinaryVector as any, nullableFieldVectors.BinaryVector as any);
-export class BinaryVector extends MixinBinaryVector {}
-export const MixinUtf8Vector = MixinArrowTraits(vectors.Utf8Vector as any, fieldVectors.Utf8Vector as any, nullableVectors.Utf8Vector as any, nullableFieldVectors.Utf8Vector as any);
-export class Utf8Vector extends MixinUtf8Vector {}
-export const MixinBoolVector = MixinArrowTraits(vectors.BoolVector as any, fieldVectors.BoolVector as any, nullableVectors.BoolVector as any, nullableFieldVectors.BoolVector as any);
-export class BoolVector extends MixinBoolVector {}
-export const MixinInt8Vector = MixinArrowTraits(vectors.Int8Vector as any, fieldVectors.Int8Vector as any, nullableVectors.Int8Vector as any, nullableFieldVectors.Int8Vector as any);
-export class Int8Vector extends MixinInt8Vector {}
-export const MixinInt16Vector = MixinArrowTraits(vectors.Int16Vector as any, fieldVectors.Int16Vector as any, nullableVectors.Int16Vector as any, nullableFieldVectors.Int16Vector as any);
-export class Int16Vector extends MixinInt16Vector {}
-export const MixinInt32Vector = MixinArrowTraits(vectors.Int32Vector as any, fieldVectors.Int32Vector as any, nullableVectors.Int32Vector as any, nullableFieldVectors.Int32Vector as any);
-export class Int32Vector extends MixinInt32Vector {}
-export const MixinInt64Vector = MixinArrowTraits(vectors.Int64Vector as any, fieldVectors.Int64Vector as any, nullableVectors.Int64Vector as any, nullableFieldVectors.Int64Vector as any);
-export class Int64Vector extends MixinInt64Vector {}
-export const MixinUint8Vector = MixinArrowTraits(vectors.Uint8Vector as any, fieldVectors.Uint8Vector as any, nullableVectors.Uint8Vector as any, nullableFieldVectors.Uint8Vector as any);
-export class Uint8Vector extends MixinUint8Vector {}
-export const MixinUint16Vector = MixinArrowTraits(vectors.Uint16Vector as any, fieldVectors.Uint16Vector as any, nullableVectors.Uint16Vector as any, nullableFieldVectors.Uint16Vector as any);
-export class Uint16Vector extends MixinUint16Vector {}
-export const MixinUint32Vector = MixinArrowTraits(vectors.Uint32Vector as any, fieldVectors.Uint32Vector as any, nullableVectors.Uint32Vector as any, nullableFieldVectors.Uint32Vector as any);
-export class Uint32Vector extends MixinUint32Vector {}
-export const MixinUint64Vector = MixinArrowTraits(vectors.Uint64Vector as any, fieldVectors.Uint64Vector as any, nullableVectors.Uint64Vector as any, nullableFieldVectors.Uint64Vector as any);
-export class Uint64Vector extends MixinUint64Vector {}
-export const MixinDate32Vector = MixinArrowTraits(vectors.Date32Vector as any, fieldVectors.Date32Vector as any, nullableVectors.Date32Vector as any, nullableFieldVectors.Date32Vector as any);
-export class Date32Vector extends MixinDate32Vector {}
-export const MixinDate64Vector = MixinArrowTraits(vectors.Date64Vector as any, fieldVectors.Date64Vector as any, nullableVectors.Date64Vector as any, nullableFieldVectors.Date64Vector as any);
-export class Date64Vector extends MixinDate64Vector {}
-export const MixinTime32Vector = MixinArrowTraits(vectors.Time32Vector as any, fieldVectors.Time32Vector as any, nullableVectors.Time32Vector as any, nullableFieldVectors.Time32Vector as any);
-export class Time32Vector extends MixinTime32Vector {}
-export const MixinTime64Vector = MixinArrowTraits(vectors.Time64Vector as any, fieldVectors.Time64Vector as any, nullableVectors.Time64Vector as any, nullableFieldVectors.Time64Vector as any);
-export class Time64Vector extends MixinTime64Vector {}
-export const MixinFloat16Vector = MixinArrowTraits(vectors.Float16Vector as any, fieldVectors.Float16Vector as any, nullableVectors.Float16Vector as any, nullableFieldVectors.Float16Vector as any);
-export class Float16Vector extends MixinFloat16Vector {}
-export const MixinFloat32Vector = MixinArrowTraits(vectors.Float32Vector as any, fieldVectors.Float32Vector as any, nullableVectors.Float32Vector as any, nullableFieldVectors.Float32Vector as any);
-export class Float32Vector extends MixinFloat32Vector {}
-export const MixinFloat64Vector = MixinArrowTraits(vectors.Float64Vector as any, fieldVectors.Float64Vector as any, nullableVectors.Float64Vector as any, nullableFieldVectors.Float64Vector as any);
-export class Float64Vector extends MixinFloat64Vector {}
-export const MixinStructVector = MixinArrowTraits(vectors.StructVector as any, fieldVectors.StructVector as any, nullableVectors.StructVector as any, nullableFieldVectors.StructVector as any);
-export class StructVector extends MixinStructVector {}
-export const MixinDecimalVector = MixinArrowTraits(vectors.DecimalVector as any, fieldVectors.DecimalVector as any, nullableVectors.DecimalVector as any, nullableFieldVectors.DecimalVector as any);
-export class DecimalVector extends MixinDecimalVector {}
-export const MixinTimestampVector = MixinArrowTraits(vectors.TimestampVector as any, fieldVectors.TimestampVector as any, nullableVectors.TimestampVector as any, nullableFieldVectors.TimestampVector as any);
-export class TimestampVector extends MixinTimestampVector {}
-export const MixinDictionaryVector = MixinArrowTraits(vectors.DictionaryVector as any, fieldVectors.DictionaryVector as any, nullableVectors.DictionaryVector as any, nullableFieldVectors.DictionaryVector as any);
-export class DictionaryVector extends MixinDictionaryVector {}
-export const MixinFixedSizeListVector = MixinArrowTraits(vectors.FixedSizeListVector as any, fieldVectors.FixedSizeListVector as any, nullableVectors.FixedSizeListVector as any, nullableFieldVectors.FixedSizeListVector as any);
-export class FixedSizeListVector extends MixinFixedSizeListVector {}
diff --git a/js/src/vector/chunked.ts b/js/src/vector/chunked.ts
new file mode 100644
index 0000000000000..c0087fdd75833
--- /dev/null
+++ b/js/src/vector/chunked.ts
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { ChunkedData } from '../data';
+import { View, Vector, NestedVector } from '../vector';
+import { DataType, TypedArray, IterableArrayLike } from '../type';
+
+export class ChunkedView<T extends DataType> implements View<T> {
+    public chunkVectors: Vector<T>[];
+    public chunkOffsets: Uint32Array;
+    protected _children: Vector<any>[];
+    constructor(data: ChunkedData<T>) {
+        this.chunkVectors = data.chunkVectors;
+        this.chunkOffsets = data.chunkOffsets;
+    }
+    public clone(data: ChunkedData<T>): this {
+        return new ChunkedView(data) as this;
+    }
+    public *[Symbol.iterator](): IterableIterator<T['TValue'] | null> {
+        for (const vector of this.chunkVectors) {
+            yield* vector;
+        }
+    }
+    public getChildAt<R extends DataType = DataType>(index: number) {
+        return index < 0 ? null
+            : (this._children || (this._children = []))[index] ||
+              (this._children[index] = Vector.concat<R>(
+                  ...(<any> this.chunkVectors as NestedVector<any>[])
+                         .map((chunk) => chunk.getChildAt<R>(index))));
+    }
+    public isValid(index: number): boolean {
+        // binary search to find the child vector and value index offset (inlined for speed)
+        let offsets = this.chunkOffsets, pos = 0;
+        let lhs = 0, mid = 0, rhs = offsets.length - 1;
+        while (index < offsets[rhs] && index >= (pos = offsets[lhs])) {
+            if (lhs + 1 === rhs) {
+                return this.chunkVectors[lhs].isValid(index - pos);
+            }
+            mid = lhs + ((rhs - lhs) / 2) | 0;
+            index >= offsets[mid] ? (lhs = mid) : (rhs = mid);
+        }
+        return false;
+    }
+    public get(index: number): T['TValue'] | null {
+        // binary search to find the child vector and value index offset (inlined for speed)
+        let offsets = this.chunkOffsets, pos = 0;
+        let lhs = 0, mid = 0, rhs = offsets.length - 1;
+        while (index < offsets[rhs] && index >= (pos = offsets[lhs])) {
+            if (lhs + 1 === rhs) {
+                return this.chunkVectors[lhs].get(index - pos);
+            }
+            mid = lhs + ((rhs - lhs) / 2) | 0;
+            index >= offsets[mid] ? (lhs = mid) : (rhs = mid);
+        }
+        return null;
+    }
+    public set(index: number, value: T['TValue'] | null): void {
+        // binary search to find the child vector and value index offset (inlined for speed)
+        let offsets = this.chunkOffsets, pos = 0;
+        let lhs = 0, mid = 0, rhs = offsets.length - 1;
+        while (index < offsets[rhs] && index >= (pos = offsets[lhs])) {
+            if (lhs + 1 === rhs) {
+                return this.chunkVectors[lhs].set(index - pos, value);
+            }
+            mid = lhs + ((rhs - lhs) / 2) | 0;
+            index >= offsets[mid] ? (lhs = mid) : (rhs = mid);
+        }
+    }
+    public toArray(): IterableArrayLike<T['TValue'] | null> {
+        const chunks = this.chunkVectors;
+        const numChunks = chunks.length;
+        if (numChunks === 1) {
+            return chunks[0].toArray();
+        }
+        let sources = new Array<any>(numChunks);
+        let sourcesLen = 0, ArrayType: any = Array;
+        for (let index = -1; ++index < numChunks;) {
+            let source = chunks[index].toArray();
+            sourcesLen += (sources[index] = source).length;
+            if (ArrayType !== source.constructor) {
+                ArrayType = source.constructor;
+            }
+        }
+        let target = new ArrayType(sourcesLen);
+        let setValues = ArrayType === Array ? arraySet : typedArraySet as any;
+        for (let index = -1, offset = 0; ++index < numChunks;) {
+            offset = setValues(sources[index], target, offset);
+        }
+        return target;
+    }
+}
+
+function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
+    return target.set(source, index) || index + source.length;
+}
+
+function arraySet(source: any[], target: any[], index: number) {
+    let dstIdx = index - 1, srcIdx = -1, srcLen = source.length;
+    while (++srcIdx < srcLen) {
+        target[++dstIdx] = source[srcIdx];
+    }
+    return dstIdx;
+}
diff --git a/js/src/vector/dictionary.ts b/js/src/vector/dictionary.ts
index b7375c0904276..385729814efb9 100644
--- a/js/src/vector/dictionary.ts
+++ b/js/src/vector/dictionary.ts
@@ -15,34 +15,36 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Vector } from './vector';
-import { VirtualVector } from './virtual';
+import { Data } from '../data';
+import { View, Vector } from '../vector';
+import { IterableArrayLike, DataType, Dictionary, Int } from '../type';
 
-export class DictionaryVector<T> extends Vector<T> {
-    readonly length: number;
-    readonly data: Vector<T>;
-    readonly keys: Vector<number>;
-    constructor(argv: { data: Vector<T>, keys: Vector<number> }) {
-        super();
-        this.data = argv.data;
-        this.keys = argv.keys;
-        this.length = this.keys.length;
+export class DictionaryView<T extends DataType> implements View<T> {
+    public indicies: Vector<Int>;
+    public dictionary: Vector<T>;
+    constructor(dictionary: Vector<T>, indicies: Vector<Int>) {
+        this.indicies = indicies;
+        this.dictionary = dictionary;
     }
-    get(index: number) {
-        return this.getValue(this.getKey(index)!);
+    public clone(data: Data<Dictionary<T>>): this {
+        return new DictionaryView(data.dictionary, this.indicies.clone(data.indicies)) as this;
     }
-    getKey(index: number) {
-        return this.keys.get(index);
+    public isValid(index: number): boolean {
+        return this.indicies.isValid(index);
     }
-    getValue(key: number) {
-        return this.data.get(key);
+    public get(index: number): T['TValue'] {
+        return this.dictionary.get(this.indicies.get(index));
     }
-    concat(...vectors: Vector<T>[]): Vector<T> {
-        return new VirtualVector(Array, this, ...vectors);
+    public set(index: number, value: T['TValue']): void {
+        this.dictionary.set(this.indicies.get(index), value);
     }
-    *[Symbol.iterator]() {
-        for (let i = -1, n = this.length; ++i < n;) {
-            yield this.get(i);
+    public toArray(): IterableArrayLike<T['TValue']> {
+        return [...this];
+    }
+    public *[Symbol.iterator](): IterableIterator<T['TValue']> {
+        const values = this.dictionary, indicies = this.indicies;
+        for (let index = -1, n = indicies.length; ++index < n;) {
+            yield values.get(indicies.get(index));
         }
     }
 }
diff --git a/js/src/vector/flat.ts b/js/src/vector/flat.ts
new file mode 100644
index 0000000000000..a32bd9d397ff0
--- /dev/null
+++ b/js/src/vector/flat.ts
@@ -0,0 +1,306 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Data } from '../data';
+import { View } from '../vector';
+import { getBool, setBool, iterateBits } from '../util/bit';
+import { Bool, Float16, Date_, Interval, Null, Int32, Timestamp } from '../type';
+import { DataType, FlatType, PrimitiveType, IterableArrayLike } from '../type';
+
+export class FlatView<T extends FlatType> implements View<T> {
+    public length: number;
+    public values: T['TArray'];
+    constructor(data: Data<T>) {
+        this.length = data.length;
+        this.values = data.values;
+    }
+    public clone(data: Data<T>): this {
+        return new (<any> this.constructor)(data) as this;
+    }
+    public isValid(): boolean {
+        return true;
+    }
+    public get(index: number): T['TValue'] {
+        return this.values[index];
+    }
+    public set(index: number, value: T['TValue']): void {
+        return this.values[index] = value;
+    }
+    public toArray(): IterableArrayLike<T['TValue']> {
+        return this.values.subarray(0, this.length);
+    }
+    public [Symbol.iterator](): IterableIterator<T['TValue']> {
+        return this.values.subarray(0, this.length)[Symbol.iterator]() as IterableIterator<T['TValue']>;
+    }
+}
+
+export class NullView implements View<Null> {
+    public length: number;
+    constructor(data: Data<Null>) {
+        this.length = data.length;
+    }
+    public clone(data: Data<Null>): this {
+        return new (<any> this.constructor)(data) as this;
+    }
+    public isValid(): boolean {
+        return true;
+    }
+    public set(): void {}
+    public get() { return null; }
+    public toArray(): IterableArrayLike<null> {
+        return [...this];
+    }
+    public *[Symbol.iterator](): IterableIterator<null> {
+        for (let index = -1, length = this.length; ++index < length;) {
+            yield null;
+        }
+    }
+}
+
+export class BoolView extends FlatView<Bool> {
+    protected offset: number;
+    constructor(data: Data<Bool>) {
+        super(data);
+        this.offset = data.offset;
+    }
+    public toArray() { return [...this]; }
+    public get(index: number): boolean {
+        const boolBitIndex = this.offset + index;
+        return getBool(null, index, this.values[boolBitIndex >> 3], boolBitIndex % 8);
+    }
+    public set(index: number, value: boolean): void {
+        setBool(this.values, this.offset + index, value);
+    }
+    public [Symbol.iterator](): IterableIterator<boolean> {
+        return iterateBits<boolean>(this.values, this.offset, this.length, this.values, getBool);
+    }
+}
+
+export class ValidityView<T extends DataType> implements View<T> {
+    protected view: View<T>;
+    protected length: number;
+    protected offset: number;
+    protected nullBitmap: Uint8Array;
+    constructor(data: Data<T>, view: View<T>) {
+        this.view = view;
+        this.length = data.length;
+        this.offset = data.offset;
+        this.nullBitmap = data.nullBitmap!;
+    }
+    public clone(data: Data<T>): this {
+        return new ValidityView(data, this.view.clone(data)) as this;
+    }
+    public toArray(): IterableArrayLike<T['TValue'] | null> {
+        return [...this];
+    }
+    public isValid(index: number): boolean {
+        const nullBitIndex = this.offset + index;
+        return getBool(null, index, this.nullBitmap[nullBitIndex >> 3], nullBitIndex % 8);
+    }
+    public get(index: number): T['TValue'] | null {
+        const nullBitIndex = this.offset + index;
+        return this.getNullable(this.view, index, this.nullBitmap[nullBitIndex >> 3], nullBitIndex % 8);
+    }
+    public set(index: number, value: T['TValue'] | null): void {
+        if (setBool(this.nullBitmap, this.offset + index, value != null)) {
+            this.view.set(index, value);
+        }
+    }
+    public [Symbol.iterator](): IterableIterator<T['TValue'] | null> {
+        return iterateBits<T['TValue'] | null>(this.nullBitmap, this.offset, this.length, this.view, this.getNullable);
+    }
+    protected getNullable(view: View<T>, index: number, byte: number, bit: number) {
+        return getBool(view, index, byte, bit) ? view.get(index) : null;
+    }
+}
+
+export class PrimitiveView<T extends PrimitiveType> extends FlatView<T> {
+    public size: number;
+    public ArrayType: T['ArrayType'];
+    constructor(data: Data<T>, size?: number) {
+        super(data);
+        this.size = size || 1;
+        this.ArrayType = data.type.ArrayType;
+    }
+    public clone(data: Data<T>): this {
+        return new (<any> this.constructor)(data, this.size) as this;
+    }
+    protected getValue(values: T['TArray'], index: number, size: number): T['TValue'] {
+        return values[index * size];
+    }
+    protected setValue(values: T['TArray'], index: number, size: number, value: T['TValue']): void {
+        values[index * size] = value;
+    }
+    public get(index: number): T['TValue'] {
+        return this.getValue(this.values, index, this.size);
+    }
+    public set(index: number, value: T['TValue']): void {
+        return this.setValue(this.values, index, this.size, value);
+    }
+    public toArray(): IterableArrayLike<T['TValue']> {
+        return this.size > 1 ?
+            new this.ArrayType(this) :
+            this.values.subarray(0, this.length);
+    }
+    public *[Symbol.iterator](): IterableIterator<T['TValue']> {
+        const get = this.getValue;
+        const { size, values, length } = this;
+        for (let index = -1; ++index < length;) {
+            yield get(values, index, size);
+        }
+    }
+}
+
+export class FixedSizeView<T extends PrimitiveType> extends PrimitiveView<T> {
+    public toArray(): IterableArrayLike<T['TValue']> {
+        return this.values;
+    }
+    protected getValue(values: T['TArray'], index: number, size: number): T['TValue'] {
+        return values.subarray(index * size, index * size + size);
+    }
+    protected setValue(values: T['TArray'], index: number, size: number, value: T['TValue']): void {
+        values.set((value as T['TArray']).subarray(0, size), index * size);
+    }
+}
+
+export class Float16View extends PrimitiveView<Float16> {
+    public toArray() { return new Float32Array(this); }
+    protected getValue(values: Uint16Array, index: number, size: number): number {
+        return (values[index * size] - 32767) / 32767;
+    }
+    protected setValue(values: Uint16Array, index: number, size: number, value: number): void {
+        values[index * size] = (value * 32767) + 32767;
+    }
+}
+
+export class DateDayView extends PrimitiveView<Date_> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): Date {
+        return epochDaysToDate(values, index * size);
+    }
+    protected setValue(values: Int32Array, index: number, size: number, value: Date): void {
+        values[index * size] = value.valueOf() / 86400000;
+    }
+}
+
+export class DateMillisecondView extends FixedSizeView<Date_> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): Date {
+        return epochMillisecondsLongToDate(values, index * size);
+    }
+    protected setValue(values: Int32Array, index: number, size: number, value: Date): void {
+        const epochMs = value.valueOf();
+        values[index * size] = (epochMs % 4294967296) | 0;
+        values[index * size + size] = (epochMs / 4294967296) | 0;
+    }
+}
+
+export class TimestampDayView extends PrimitiveView<Timestamp> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): number {
+        return epochDaysToMs(values, index * size);
+    }
+    protected setValue(values: Int32Array, index: number, size: number, epochMs: number): void {
+        values[index * size] = (epochMs / 86400000) | 0;
+    }
+}
+
+export class TimestampSecondView extends PrimitiveView<Timestamp> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): number {
+        return epochSecondsToMs(values, index * size);
+    }
+    protected setValue(values: Int32Array, index: number, size: number, epochMs: number): void {
+        values[index * size] = (epochMs / 1000) | 0;
+    }
+}
+
+export class TimestampMillisecondView extends PrimitiveView<Timestamp> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): number {
+        return epochMillisecondsLongToMs(values, index * size);
+    }
+    protected setValue(values: Int32Array, index: number, size: number, epochMs: number): void {
+        values[index * size] = (epochMs % 4294967296) | 0;
+        values[index * size + size] = (epochMs / 4294967296) | 0;
+    }
+}
+
+export class TimestampMicrosecondView extends PrimitiveView<Timestamp> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): number {
+        return epochMicrosecondsLongToMs(values, index * size);
+    }
+    protected setValue(values: Int32Array, index: number, size: number, epochMs: number): void {
+        values[index * size] = ((epochMs / 1000) % 4294967296) | 0;
+        values[index * size + size] = ((epochMs / 1000) / 4294967296) | 0;
+    }
+}
+
+export class TimestampNanosecondView extends PrimitiveView<Timestamp> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): number {
+        return epochNanosecondsLongToMs(values, index * size);
+    }
+    protected setValue(values: Int32Array, index: number, size: number, epochMs: number): void {
+        values[index * size] = ((epochMs / 1000000) % 4294967296) | 0;
+        values[index * size + size] = ((epochMs / 1000000) / 4294967296) | 0;
+    }
+}
+
+export class IntervalYearMonthView extends PrimitiveView<Interval> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): Int32Array {
+        const interval = values[index * size];
+        return new Int32Array([interval / 12, /* years */ interval % 12  /* months */]);
+    }
+    protected setValue(values: Int32Array, index: number, size: number, value: Int32Array): void {
+        values[index * size] = (value[0] * 12) + (value[1] % 12);
+    }
+}
+
+export class IntervalYearView extends PrimitiveView<Int32> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): number {
+        return values[index * size] / 12;
+    }
+    protected setValue(values: Int32Array, index: number, size: number, value: number): void {
+        values[index * size] = (value * 12) + (values[index * size] % 12);
+    }
+}
+
+export class IntervalMonthView extends PrimitiveView<Int32> {
+    public toArray() { return [...this]; }
+    protected getValue(values: Int32Array, index: number, size: number): number {
+        return values[index * size] % 12;
+    }
+    protected setValue(values: Int32Array, index: number, size: number, value: number): void {
+        values[index * size] = (values[index * size] * 12) + (value % 12);
+    }
+}
+
+export function epochSecondsToMs(data: Int32Array, index: number) { return 1000 * data[index]; }
+export function epochDaysToMs(data: Int32Array, index: number) { return 86400000 * data[index]; }
+export function epochMillisecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1]) + data[index]; }
+export function epochMicrosecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1] / 1000) + (data[index] / 1000); }
+export function epochNanosecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1] / 1000000) + (data[index] / 1000000); }
+
+export function epochMillisecondsToDate(epochMs: number) { return new Date(epochMs); }
+export function epochDaysToDate(data: Int32Array, index: number) { return epochMillisecondsToDate(epochDaysToMs(data, index)); }
+export function epochSecondsToDate(data: Int32Array, index: number) { return epochMillisecondsToDate(epochSecondsToMs(data, index)); }
+export function epochNanosecondsLongToDate(data: Int32Array, index: number) { return epochMillisecondsToDate(epochNanosecondsLongToMs(data, index)); }
+export function epochMillisecondsLongToDate(data: Int32Array, index: number) { return epochMillisecondsToDate(epochMillisecondsLongToMs(data, index)); }
diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts
index 97913f8d8878c..3d365ceac29b2 100644
--- a/js/src/vector/list.ts
+++ b/js/src/vector/list.ts
@@ -15,60 +15,115 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { List } from './types';
-import { Vector } from './vector';
-import { VirtualVector } from './virtual';
+import { Data } from '../data';
+import { View, Vector, createVector } from '../vector';
+import { TextEncoder, TextDecoder } from 'text-encoding-utf-8';
+import { List, Binary, Utf8, FixedSizeList, FlatListType } from '../type';
+import { ListType, DataType, IterableArrayLike } from '../type';
 
-export class BinaryVector extends Vector<Uint8Array> {
-    readonly data: Uint8Array;
-    readonly offsets: Int32Array;
-    constructor(argv: { offsets: Int32Array, data: Uint8Array }) {
-        super();
-        this.data = argv.data;
-        this.offsets = argv.offsets;
+export const encodeUtf8 = ((encoder) =>
+    encoder.encode.bind(encoder) as (input?: string) => Uint8Array
+)(new TextEncoder('utf-8'));
+
+export const decodeUtf8 = ((decoder) =>
+    decoder.decode.bind(decoder) as (input?: ArrayBufferLike | ArrayBufferView) => string
+)(new TextDecoder('utf-8'));
+
+export abstract class ListViewBase<T extends (ListType | FlatListType | FixedSizeList)> implements View<T> {
+    public length: number;
+    public values: T['TArray'];
+    public valueOffsets?: Int32Array;
+    constructor(data: Data<T>) {
+        this.length = data.length;
+        this.values = data.values;
+    }
+    public clone(data: Data<T>): this {
+        return new (<any> this.constructor)(data) as this;
+    }
+    public isValid(): boolean {
+        return true;
+    }
+    public toArray(): IterableArrayLike<T['TValue']> {
+        return [...this];
+    }
+    public get(index: number): T['TValue'] {
+        return this.getList(this.values, index, this.valueOffsets);
+    }
+    public set(index: number, value: T['TValue']): void {
+        return this.setList(this.values, index, value, this.valueOffsets);
+    }
+    public *[Symbol.iterator](): IterableIterator<T['TValue']> {
+        const get = this.getList, length = this.length;
+        const values = this.values, valueOffsets = this.valueOffsets;
+        for (let index = -1; ++index < length;) {
+            yield get(values, index, valueOffsets);
+        }
+    }
+    protected abstract getList(values: T['TArray'], index: number, valueOffsets?: Int32Array): T['TValue'];
+    protected abstract setList(values: T['TArray'], index: number, value: T['TValue'], valueOffsets?: Int32Array): void;
+}
+
+export abstract class VariableListViewBase<T extends (ListType | FlatListType)> extends ListViewBase<T> {
+    constructor(data: Data<T>) {
+        super(data);
+        this.length = data.length;
+        this.valueOffsets = data.valueOffsets;
+    }
+}
+
+export class ListView<T extends DataType> extends VariableListViewBase<List<T>> {
+    constructor(data: Data<List<T>>) {
+        super(data);
+        this.values = createVector(data.values);
     }
-    get(index: number) {
-        return this.data.subarray(this.offsets[index], this.offsets[index + 1]);
+    protected getList(values: Vector<T>, index: number, valueOffsets: Int32Array) {
+        return values.slice(valueOffsets[index], valueOffsets[index + 1]) as Vector<T>;
     }
-    concat(...vectors: Vector<Uint8Array>[]): Vector<Uint8Array> {
-        return new VirtualVector(Array, this, ...vectors);
+    protected setList(values: Vector<T>, index: number, value: Vector<T>, valueOffsets: Int32Array): void {
+        let idx = -1;
+        let offset = valueOffsets[index];
+        let end = Math.min(value.length, valueOffsets[index + 1] - offset);
+        while (offset < end) {
+            values.set(offset++, value.get(++idx));
+        }
     }
 }
 
-export class ListVector<T> extends Vector<T[]> {
-    readonly offsets: Int32Array;
-    readonly values: Vector<T>;
-    constructor(argv: { offsets: Int32Array, values: Vector<T> }) {
-        super();
-        this.values = argv.values;
-        this.offsets = argv.offsets;
-    }
-    get(index: number) {
-        const { offsets, values } = this;
-        const from = offsets[index];
-        const xs = new Array(offsets[index + 1] - from);
-        for (let i = -1, n = xs.length; ++i < n;) {
-            xs[i] = values.get(i + from);
+export class FixedSizeListView<T extends DataType> extends ListViewBase<FixedSizeList<T>> {
+    public size: number;
+    constructor(data: Data<FixedSizeList<T>>) {
+        super(data);
+        this.size = data.type.listSize;
+        this.values = createVector(data.values);
+    }
+    protected getList(values: Vector<T>, index: number) {
+        const size = this.size;
+        return values.slice(index *= size, index + size) as Vector<T>;
+    }
+    protected setList(values: Vector<T>, index: number, value: Vector<T>): void {
+        let size = this.size;
+        for (let idx = -1, offset = index * size; ++idx < size;) {
+            values.set(offset + idx, value.get(++idx));
         }
-        return xs;
     }
-    concat(...vectors: Vector<T[]>[]): Vector<T[]> {
-        return new VirtualVector(Array, this, ...vectors);
+}
+
+export class BinaryView extends VariableListViewBase<Binary> {
+    protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) {
+        return values.subarray(valueOffsets[index], valueOffsets[index + 1]);
+    }
+    protected setList(values: Uint8Array, index: number, value: Uint8Array, valueOffsets: Int32Array): void {
+        const offset = valueOffsets[index];
+        values.set(value.subarray(0, valueOffsets[index + 1] - offset), offset);
     }
 }
 
-export class FixedSizeListVector<T, TArray extends List<T>> extends Vector<TArray> {
-    readonly size: number;
-    readonly values: Vector<T>;
-    constructor(argv: { size: number, values: Vector<T> }) {
-        super();
-        this.size = argv.size;
-        this.values = argv.values;
-    }
-    get(index: number) {
-        return this.values.slice<TArray>(this.size * index, this.size * (index + 1));
-    }
-    concat(...vectors: Vector<TArray>[]): Vector<TArray> {
-        return new VirtualVector(Array, this, ...vectors);
+export class Utf8View extends VariableListViewBase<Utf8> {
+    protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) {
+        return decodeUtf8(values.subarray(valueOffsets[index], valueOffsets[index + 1]));
+    }
+    protected setList(values: Uint8Array, index: number, value: string, valueOffsets: Int32Array): void {
+        const offset = valueOffsets[index];
+        values.set(encodeUtf8(value).subarray(0, valueOffsets[index + 1] - offset), offset);
     }
 }
diff --git a/js/src/vector/nested.ts b/js/src/vector/nested.ts
new file mode 100644
index 0000000000000..d0fb24ca99682
--- /dev/null
+++ b/js/src/vector/nested.ts
@@ -0,0 +1,217 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Data } from '../data';
+import { View, Vector } from '../vector';
+import { IterableArrayLike } from '../type';
+import { DataType, NestedType, DenseUnion, SparseUnion, Struct, Map_ } from '../type';
+
+export abstract class NestedView<T extends NestedType> implements View<T> {
+    public length: number;
+    public numChildren: number;
+    public childData: Data<any>[];
+    protected _children: Vector<any>[];
+    constructor(data: Data<T>, children?: Vector<any>[]) {
+        this.length = data.length;
+        this.childData = data.childData;
+        this.numChildren = data.childData.length;
+        this._children = children || new Array(this.numChildren);
+    }
+    public clone(data: Data<T>): this {
+        return new (<any> this.constructor)(data, this._children) as this;
+    }
+    public isValid(): boolean {
+        return true;
+    }
+    public toArray(): IterableArrayLike<T['TValue']> {
+        return [...this];
+    }
+    public toJSON(): any { return this.toArray(); }
+    public toString() {
+        return [...this].map((x) => stringify(x)).join(', ');
+    }
+    public get(index: number): T['TValue'] {
+        return this.getNested(this, index);
+    }
+    public set(index: number, value: T['TValue']): void {
+        return this.setNested(this, index, value);
+    }
+    protected abstract getNested(self: NestedView<T>, index: number): T['TValue'];
+    protected abstract setNested(self: NestedView<T>, index: number, value: T['TValue']): void;
+    public getChildAt<R extends DataType = DataType>(index: number): Vector<R> | null {
+        return index < 0 || index >= this.numChildren
+            ? null
+            : (this._children[index] as Vector<R>) ||
+              (this._children[index] = Vector.create<R>(this.childData[index]));
+    }
+    public *[Symbol.iterator](): IterableIterator<T['TValue']> {
+        const get = this.getNested;
+        const length = this.length;
+        for (let index = -1; ++index < length;) {
+            yield get(this, index);
+        }
+    }
+}
+
+export class UnionView<T extends (DenseUnion | SparseUnion) = SparseUnion> extends NestedView<T> {
+    // @ts-ignore
+    public typeIds: Int8Array;
+    // @ts-ignore
+    public valueOffsets?: Int32Array;
+    constructor(data: Data<T>, children?: Vector<any>[]) {
+        super(data, children);
+        this.length = data.length;
+        this.typeIds = data.typeIds;
+    }
+    protected getNested(self: UnionView<T>, index: number): T['TValue'] {
+        return self.getChildValue(self, index, self.typeIds, self.valueOffsets);
+    }
+    protected setNested(self: UnionView<T>, index: number, value: T['TValue']): void {
+        return self.setChildValue(self, index, value, self.typeIds, self.valueOffsets);
+    }
+    protected getChildValue(self: NestedView<T>, index: number, typeIds: Int8Array, _valueOffsets?: any): any | null {
+        const child = self.getChildAt(typeIds[index]);
+        return child ? child.get(index) : null;
+    }
+    protected setChildValue(self: NestedView<T>, index: number, value: T['TValue'], typeIds: Int8Array, _valueOffsets?: any): any | null {
+        const child = self.getChildAt(typeIds[index]);
+        return child ? child.set(index, value) : null;
+    }
+    public *[Symbol.iterator](): IterableIterator<T['TValue']> {
+        const length = this.length;
+        const get = this.getChildValue;
+        const { typeIds, valueOffsets } = this;
+        for (let index = -1; ++index < length;) {
+            yield get(this, index, typeIds, valueOffsets);
+        }
+    }
+}
+
+export class DenseUnionView extends UnionView<DenseUnion> {
+    public valueOffsets: Int32Array;
+    constructor(data: Data<DenseUnion>, children?: Vector<any>[]) {
+        super(data, children);
+        this.valueOffsets = data.valueOffsets;
+    }
+    protected getNested(self: DenseUnionView, index: number): any | null {
+        return self.getChildValue(self, index, self.typeIds, self.valueOffsets);
+    }
+    protected getChildValue(self: NestedView<DenseUnion>, index: number, typeIds: Int8Array, valueOffsets: any): any | null {
+        const child = self.getChildAt(typeIds[index]);
+        return child ? child.get(valueOffsets[index]) : null;
+    }
+    protected setChildValue(self: NestedView<DenseUnion>, index: number, value: any, typeIds: Int8Array, valueOffsets?: any): any | null {
+        const child = self.getChildAt(typeIds[index]);
+        return child ? child.set(valueOffsets[index], value) : null;
+    }
+}
+
+export class StructView extends NestedView<Struct> {
+    protected getNested(self: StructView, index: number) {
+        return new RowView(self as any, self._children, index);
+    }
+    protected setNested(self: StructView, index: number, value: any): void {
+        let idx = -1, len = self.numChildren, child: Vector | null;
+        if (!(value instanceof NestedView || value instanceof Vector)) {
+            while (++idx < len) {
+                if (child = self.getChildAt(idx)) {
+                    child.set(index, value[idx]);
+                }
+            }
+        } else {
+            while (++idx < len) {
+                if (child = self.getChildAt(idx)) {
+                    child.set(index, value.get(idx));
+                }
+            }
+        }
+    }
+}
+
+export class MapView extends NestedView<Map_> {
+    public typeIds: { [k: string]: number };
+    constructor(data: Data<Map_>, children?: Vector<any>[]) {
+        super(data, children);
+        this.typeIds = data.type.children.reduce((xs, x, i) =>
+            (xs[x.name] = i) && xs || xs, Object.create(null));
+    }
+    protected getNested(self: MapView, index: number) {
+        return new MapRowView(self as any, self._children, index);
+    }
+    protected setNested(self: MapView, index: number, value: { [k: string]: any }): void {
+        let typeIds = self.typeIds as any, child: Vector | null;
+        if (!(value instanceof NestedView || value instanceof Vector)) {
+            for (const key in typeIds) {
+                if (child = self.getChildAt(typeIds[key])) {
+                    child.set(index, value[key]);
+                }
+            }
+        } else {
+            for (const key in typeIds) {
+                if (child = self.getChildAt(typeIds[key])) {
+                    child.set(index, value.get(key as any));
+                }
+            }
+        }
+    }
+}
+
+export class RowView extends UnionView<SparseUnion> {
+    protected rowIndex: number;
+    constructor(data: Data<SparseUnion> & NestedView<any>, children?: Vector<any>[], rowIndex?: number) {
+        super(data, children);
+        this.rowIndex = rowIndex || 0;
+        this.length = data.numChildren;
+    }
+    public clone(data: Data<SparseUnion> & NestedView<any>): this {
+        return new (<any> this.constructor)(data, this._children, this.rowIndex) as this;
+    }
+    protected getChildValue(self: RowView, index: number, _typeIds: any, _valueOffsets?: any): any | null {
+        const child = self.getChildAt(index);
+        return child ? child.get(self.rowIndex) : null;
+    }
+    protected setChildValue(self: RowView, index: number, value: any, _typeIds: any, _valueOffsets?: any): any | null {
+        const child = self.getChildAt(index);
+        return child ? child.set(self.rowIndex, value) : null;
+    }
+}
+
+export class MapRowView extends RowView {
+    // @ts-ignore
+    public typeIds: any;
+    public toJSON() {
+        const get = this.getChildValue;
+        const result = {} as { [k: string]: any };
+        const typeIds = this.typeIds as { [k: string]: number };
+        for (const name in typeIds) {
+            result[name] = get(this, name, typeIds, null);
+        }
+        return result;
+    }
+    protected getChildValue(self: MapRowView, key: any, typeIds: any, _valueOffsets: any): any | null {
+        const child = self.getChildAt(typeIds[key]);
+        return child ? child.get(self.rowIndex) : null;
+    }
+    protected setChildValue(self: MapRowView, key: any, value: any, typeIds: any, _valueOffsets?: any): any | null {
+        const child = self.getChildAt(typeIds[key]);
+        return child ? child.set(self.rowIndex, value) : null;
+    }
+}
+
+function stringify(x: any) {
+    return typeof x === 'string' ? `"${x}"` : Array.isArray(x) ? JSON.stringify(x) : ArrayBuffer.isView(x) ? `[${x}]` : `${x}`;
+}
diff --git a/js/src/vector/numeric.ts b/js/src/vector/numeric.ts
deleted file mode 100644
index 830d6082bcc4a..0000000000000
--- a/js/src/vector/numeric.ts
+++ /dev/null
@@ -1,169 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from './vector';
-import { VirtualVector } from './virtual';
-import { TypedArray, TypedArrayConstructor } from './types';
-
-export class NumericVector<T, TArray extends TypedArray> extends Vector<T> {
-    readonly data: TArray;
-    readonly length: number;
-    readonly stride: number;
-    constructor(argv: { data: TArray }) {
-        super();
-        const data = (ArrayBuffer.isView(argv) ? argv : argv.data) as TArray;
-        this.length = ((this.data = data).length / this.stride) | 0;
-    }
-    get(index: number) {
-        return this.data[index] as any;
-    }
-    concat(...vectors: Vector<T>[]): Vector<T> {
-        return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors);
-    }
-    slice<R = TArray>(start?: number, end?: number): R {
-        const { data, stride } = this, from = start! | 0;
-        const to = end === undefined ? data.length : Math.max(end | 0, from);
-        return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0) as any as R;
-    }
-}
-
-export class FixedWidthNumericVector<T, TArray extends TypedArray> extends NumericVector<T, TArray> {
-    get(index: number) {
-        return this.data.slice(this.stride * index, this.stride * (index + 1)) as TArray;
-    }
-}
-
-export class BoolVector extends NumericVector<boolean, Uint8Array> {
-    static pack(values: Iterable<any>) {
-        let n = 0, i = 0;
-        let xs: number[] = [];
-        let bit = 0, byte = 0;
-        for (const value of values) {
-            value && (byte |= 1 << bit);
-            if (++bit === 8) {
-                xs[i++] = byte;
-                byte = bit = 0;
-            }
-        }
-        if (i === 0 || bit > 0) { xs[i++] = byte; }
-        if (i % 8 && (n = i + 8 - i % 8)) {
-            do { xs[i] = 0; } while (++i < n);
-        }
-        return new Uint8Array(xs);
-    }
-    get(index: number) {
-        return (this.data[index >> 3] & 1 << index % 8) !== 0;
-    }
-    set(index: number, value: boolean) {
-        if (index > -1 === false) {
-            return;
-        } else if (value) {
-            this.data[index >> 3] |=  (1 << (index % 8));
-        } else {
-            this.data[index >> 3] &= ~(1 << (index % 8));
-        }
-    }
-}
-
-export class Int8Vector extends NumericVector<number, Int8Array> {}
-export class Int16Vector extends NumericVector<number, Int16Array> {}
-export class Int32Vector extends NumericVector<number, Int32Array> {}
-export class Int64Vector extends FixedWidthNumericVector<number, Int32Array> {}
-
-export class Uint8Vector extends NumericVector<number, Uint8Array> {}
-export class Uint16Vector extends NumericVector<number, Uint16Array> {}
-export class Uint32Vector extends NumericVector<number, Uint32Array> {}
-export class Uint64Vector extends FixedWidthNumericVector<number, Uint32Array> {}
-
-export class Float16Vector extends NumericVector<number, Uint16Array> {
-    get(index: number) {
-        return Math.min((super.get(index)! -  32767) / 32767, 1);
-    }
-}
-
-export class Float32Vector extends NumericVector<number, Float32Array> {}
-export class Float64Vector extends NumericVector<number, Float64Array> {}
-
-export class Date32Vector extends NumericVector<Date, Int32Array> {
-    public readonly unit: string;
-    constructor(argv: { data: Int32Array, unit: string }) {
-        super(argv);
-        this.unit = argv.unit;
-    }
-    get(index: number): Date {
-        return new Date(86400000 * (super.get(index) as any));
-    }
-}
-
-export class Date64Vector extends NumericVector<Date, Int32Array> {
-    public readonly unit: string;
-    constructor(argv: { unit: string, data: Int32Array }) {
-        super(argv);
-        this.unit = argv.unit;
-    }
-    get(index: number): Date {
-        return new Date(4294967296   * /* 2^32 */
-            (super.get(index * 2 + 1) as any) + /* high */
-            (super.get(index * 2) as any)       /*  low */
-        );
-    }
-}
-
-export class Time32Vector extends NumericVector<number, Int32Array> {
-    public readonly unit: string;
-    constructor(argv: { data: Int32Array, unit: string }) {
-        super(argv);
-        this.unit = argv.unit;
-    }
-}
-
-export class Time64Vector extends FixedWidthNumericVector<number, Uint32Array> {
-    public readonly unit: string;
-    constructor(argv: { unit: string, data: Uint32Array }) {
-        super(argv);
-        this.unit = argv.unit;
-    }
-}
-
-export class DecimalVector extends FixedWidthNumericVector<number, Uint32Array> {
-    readonly scale: number;
-    readonly precision: number;
-    constructor(argv: { precision: number, scale: number, data: Uint32Array }) {
-        super(argv);
-        this.scale = argv.scale;
-        this.precision = argv.precision;
-    }
-}
-
-export class TimestampVector extends FixedWidthNumericVector<number, Uint32Array> {
-    readonly unit: string;
-    readonly timezone: string;
-    constructor(argv: { unit: string, timezone: string, data: Uint32Array }) {
-        super(argv);
-        this.unit = argv.unit;
-        this.timezone = argv.timezone;
-    }
-}
-
-export interface NumericVectorConstructor<T, TArray extends TypedArray> {
-    readonly prototype: NumericVector<T, TArray>;
-    new (argv: { data: TArray }): NumericVector<T, TArray>;
-}
-
-(DecimalVector.prototype as any).stride = 4;
-(NumericVector.prototype as any).stride = 1;
-(FixedWidthNumericVector.prototype as any).stride = 2;
diff --git a/js/src/vector/struct.ts b/js/src/vector/struct.ts
deleted file mode 100644
index c43f6efc48fbe..0000000000000
--- a/js/src/vector/struct.ts
+++ /dev/null
@@ -1,127 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from './vector';
-import { VirtualVector } from './virtual';
-
-export class StructVector<T = any> extends Vector<StructRow<T>> {
-    readonly length: number;
-    readonly columns: Vector[];
-    constructor(argv: { columns: Vector[] }) {
-        super();
-        this.columns = argv.columns || [];
-    }
-    get(index: number): StructRow<T> {
-        return new StructRow(this, index);
-    }
-    col(name: string) {
-        return this.columns.find((col) => col.name === name) || null;
-    }
-    key(index: number) {
-        return this.columns[index] ? this.columns[index].name : null;
-    }
-    select(...columns: string[]) {
-        return new StructVector({ columns: columns.map((name) => this.col(name)!) });
-    }
-    concat(...structs: Vector<StructRow<T>>[]): Vector<StructRow<T>> {
-        return new VirtualVector(Array, this, ...structs as any[]);
-    }
-    toString(options?: any) {
-        const index = typeof options === 'object' ? options && !!options.index
-                    : typeof options === 'boolean' ? !!options
-                    : false;
-        const { length } = this;
-        if (length <= 0) { return ''; }
-        const rows = new Array(length + 1);
-        const maxColumnWidths = [] as number[];
-        rows[0] = this.columns.map((_, i) => this.key(i));
-        index && rows[0].unshift('Index');
-        for (let i = -1, n = rows.length - 1; ++i < n;) {
-            rows[i + 1] = [...this.get(i)!];
-            index && rows[i + 1].unshift(i);
-        }
-        // Pass one to convert to strings and count max column widths
-        for (let i = -1, n = rows.length; ++i < n;) {
-            const row = rows[i];
-            for (let j = -1, k = row.length; ++j < k;) {
-                const val = row[j] = stringify(row[j]);
-                maxColumnWidths[j] = !maxColumnWidths[j]
-                    ? val.length
-                    : Math.max(maxColumnWidths[j], val.length);
-            }
-        }
-        // Pass two to pad each one to max column width
-        for (let i = -1, n = rows.length; ++i < n;) {
-            const row = rows[i];
-            for (let j = -1, k = row.length; ++j < k;) {
-                row[j] = leftPad(row[j], ' ', maxColumnWidths[j]);
-            }
-            rows[i] = row.join(', ');
-        }
-        return rows.join('\n');
-    }
-}
-
-export class StructRow<T = any> extends Vector<T> {
-    readonly row: number;
-    readonly length: number;
-    readonly table: StructVector<T>;
-    [Symbol.toStringTag]() { return 'Row'; }
-    constructor(table: StructVector<T>, row: number) {
-        super();
-        this.row = row;
-        this.table = table;
-        this.length = table.columns.length;
-    }
-    get(index: number) {
-        const col = this.table.columns[index];
-        return col ? col.get(this.row) as T : null;
-    }
-    col(key: string) {
-        const col = this.table.col(key);
-        return col ? col.get(this.row) as T : null;
-    }
-    *[Symbol.iterator]() {
-        const { row } = this;
-        for (const col of this.table.columns) {
-            yield col ? col.get(row) : null;
-        }
-    }
-    concat(...rows: Vector<T>[]): Vector<T> {
-        return new VirtualVector(Array, this, ...rows as any[]);
-    }
-    toArray() { return [...this]; }
-    toJSON() { return this.toArray(); }
-    toString() { return JSON.stringify(this); }
-    toObject(): Record<string, T> {
-        const { row } = this, map = Object.create(null);
-        for (const col of this.table.columns) {
-            if (col && col.name) {
-                map[col.name] = col.get(row);
-            }
-        }
-        return map;
-    }
-}
-
-function leftPad(str: string, fill: string, n: number) {
-    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
-}
-
-function stringify(x: any) {
-    return Array.isArray(x) ? JSON.stringify(x) : ArrayBuffer.isView(x) ? `[${x}]` : `${x}`;
-}
diff --git a/js/src/vector/table.ts b/js/src/vector/table.ts
deleted file mode 100644
index ca2b66a22da80..0000000000000
--- a/js/src/vector/table.ts
+++ /dev/null
@@ -1,61 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from './vector';
-import { StructVector, StructRow } from './struct';
-import { read, readAsync } from '../reader/arrow';
-
-function concatVectors(tableVectors: Vector<any>[], batchVectors: Vector<any>[]) {
-    return tableVectors.length === 0 ? batchVectors : batchVectors.map((vec, i, _vs, col = tableVectors[i]) =>
-        vec && col && col.concat(vec) || col || vec
-    ) as Vector<any>[];
-}
-
-export class Table<T> extends StructVector<T> {
-    static from(sources?: Iterable<Uint8Array | Buffer | string> | object | string) {
-        let columns: Vector<any>[] = [];
-        if (sources) {
-            for (let vectors of read(sources)) {
-                columns = concatVectors(columns, vectors);
-            }
-        }
-        return new Table({ columns });
-    }
-    static async fromAsync(sources?: AsyncIterable<Uint8Array | Buffer | string>) {
-        let columns: Vector<any>[] = [];
-        if (sources) {
-            for await (let vectors of readAsync(sources)) {
-                columns = columns = concatVectors(columns, vectors);
-            }
-        }
-        return new Table({ columns });
-    }
-    readonly length: number;
-    constructor(argv: { columns: Vector<any>[] }) {
-        super(argv);
-        this.length = Math.max(...this.columns.map((col) => col.length)) | 0;
-    }
-    get(index: number): TableRow<T> {
-        return new TableRow(this, index);
-    }
-}
-
-export class TableRow<T> extends StructRow<T> {
-    toString() {
-        return this.toArray().map((x) => JSON.stringify(x)).join(', ');
-    }
-}
diff --git a/js/src/vector/traits/field.ts b/js/src/vector/traits/field.ts
deleted file mode 100644
index 9f68f507c6f10..0000000000000
--- a/js/src/vector/traits/field.ts
+++ /dev/null
@@ -1,71 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from '../vector';
-import * as vectors from './vectors';
-import { fieldMixin } from './mixins';
-import { Field, FieldNode } from '../../format/arrow';
-export { Field, FieldNode, Vector };
-
-export const FieldListVector = fieldMixin(vectors.ListVector);
-export class ListVector extends FieldListVector {}
-export const FieldBinaryVector = fieldMixin(vectors.BinaryVector);
-export class BinaryVector extends FieldBinaryVector {}
-export const FieldUtf8Vector = fieldMixin(vectors.Utf8Vector);
-export class Utf8Vector extends FieldUtf8Vector {}
-export const FieldBoolVector = fieldMixin(vectors.BoolVector);
-export class BoolVector extends FieldBoolVector {}
-export const FieldInt8Vector = fieldMixin(vectors.Int8Vector);
-export class Int8Vector extends FieldInt8Vector {}
-export const FieldInt16Vector = fieldMixin(vectors.Int16Vector);
-export class Int16Vector extends FieldInt16Vector {}
-export const FieldInt32Vector = fieldMixin(vectors.Int32Vector);
-export class Int32Vector extends FieldInt32Vector {}
-export const FieldInt64Vector = fieldMixin(vectors.Int64Vector);
-export class Int64Vector extends FieldInt64Vector {}
-export const FieldUint8Vector = fieldMixin(vectors.Uint8Vector);
-export class Uint8Vector extends FieldUint8Vector {}
-export const FieldUint16Vector = fieldMixin(vectors.Uint16Vector);
-export class Uint16Vector extends FieldUint16Vector {}
-export const FieldUint32Vector = fieldMixin(vectors.Uint32Vector);
-export class Uint32Vector extends FieldUint32Vector {}
-export const FieldUint64Vector = fieldMixin(vectors.Uint64Vector);
-export class Uint64Vector extends FieldUint64Vector {}
-export const FieldDate32Vector = fieldMixin(vectors.Date32Vector);
-export class Date32Vector extends FieldDate32Vector {}
-export const FieldDate64Vector = fieldMixin(vectors.Date64Vector);
-export class Date64Vector extends FieldDate64Vector {}
-export const FieldTime32Vector = fieldMixin(vectors.Time32Vector);
-export class Time32Vector extends FieldTime32Vector {}
-export const FieldTime64Vector = fieldMixin(vectors.Time64Vector);
-export class Time64Vector extends FieldTime64Vector {}
-export const FieldFloat16Vector = fieldMixin(vectors.Float16Vector);
-export class Float16Vector extends FieldFloat16Vector {}
-export const FieldFloat32Vector = fieldMixin(vectors.Float32Vector);
-export class Float32Vector extends FieldFloat32Vector {}
-export const FieldFloat64Vector = fieldMixin(vectors.Float64Vector);
-export class Float64Vector extends FieldFloat64Vector {}
-export const FieldStructVector = fieldMixin(vectors.StructVector);
-export class StructVector extends FieldStructVector {}
-export const FieldDecimalVector = fieldMixin(vectors.DecimalVector);
-export class DecimalVector extends FieldDecimalVector {}
-export const FieldTimestampVector = fieldMixin(vectors.TimestampVector);
-export class TimestampVector extends FieldTimestampVector {}
-export const FieldDictionaryVector = fieldMixin(vectors.DictionaryVector);
-export class DictionaryVector extends FieldDictionaryVector {}
-export const FieldFixedSizeListVector = fieldMixin(vectors.FixedSizeListVector);
-export class FixedSizeListVector extends FieldFixedSizeListVector {}
\ No newline at end of file
diff --git a/js/src/vector/traits/mixins.ts b/js/src/vector/traits/mixins.ts
deleted file mode 100644
index 011ac31352db6..0000000000000
--- a/js/src/vector/traits/mixins.ts
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from '../vector';
-import { BoolVector } from '../numeric';
-import * as Schema_ from '../../format/fb/Schema';
-import { Field, FieldNode } from '../../format/arrow';
-import Type = Schema_.org.apache.arrow.flatbuf.Type;
-
-function isField(x: any): x is Field {
-    return x instanceof Field;
-}
-
-function isFieldNode(x: any): x is FieldNode {
-    return x instanceof FieldNode;
-}
-
-export function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } {
-    return x && isField(x.field) && isFieldNode(x.fieldNode);
-}
-
-export function isNullableArgv(x: any): x is { validity: Uint8Array } {
-    return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array;
-}
-
-type Ctor<TArgv> = new (argv: TArgv) => Vector;
-
-export const nullableMixin = <T extends Vector, TArgv>(superclass: new (argv: TArgv) => T) =>
-    class extends (superclass as Ctor<TArgv>) {
-        readonly validity: Vector<boolean>;
-        constructor(argv: TArgv & { validity: Uint8Array }) {
-            super(argv);
-            this.validity = new BoolVector({ data: argv.validity });
-        }
-        get(index: number) {
-            return this.validity.get(index) ? super.get(index) : null;
-        }
-    };
-
-export const fieldMixin = <T extends Vector, TArgv>(superclass: new (argv: TArgv) => T) =>
-    class extends (superclass as Ctor<TArgv>) implements Vector {
-        readonly field: Field;
-        readonly type: string;
-        readonly length: number;
-        readonly stride: number;
-        readonly nullable: boolean;
-        readonly nullCount: number;
-        readonly fieldNode: FieldNode;
-        constructor(argv: TArgv & { field: Field, fieldNode: FieldNode }) {
-            super(argv);
-            const { field, fieldNode } = argv;
-            this.field = field;
-            this.fieldNode = fieldNode;
-            this.nullable = field.nullable;
-            this.type = Type[field.typeType];
-            this.length = fieldNode.length.low | 0;
-            this.nullCount = fieldNode.nullCount.low;
-        }
-        get name() { return this.field.name!; }
-        get metadata()  { return this.field.metadata!; }
-    };
diff --git a/js/src/vector/traits/nullable.ts b/js/src/vector/traits/nullable.ts
deleted file mode 100644
index 1393e5fd1bf68..0000000000000
--- a/js/src/vector/traits/nullable.ts
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from '../vector';
-import * as vectors from './vectors';
-import { nullableMixin } from './mixins';
-
-export { Vector };
-export const NullableListVector = nullableMixin(vectors.ListVector);
-export class ListVector extends NullableListVector {}
-export const NullableBinaryVector = nullableMixin(vectors.BinaryVector);
-export class BinaryVector extends NullableBinaryVector {}
-export const NullableUtf8Vector = nullableMixin(vectors.Utf8Vector);
-export class Utf8Vector extends NullableUtf8Vector {}
-export const NullableBoolVector = nullableMixin(vectors.BoolVector);
-export class BoolVector extends NullableBoolVector {}
-export const NullableInt8Vector = nullableMixin(vectors.Int8Vector);
-export class Int8Vector extends NullableInt8Vector {}
-export const NullableInt16Vector = nullableMixin(vectors.Int16Vector);
-export class Int16Vector extends NullableInt16Vector {}
-export const NullableInt32Vector = nullableMixin(vectors.Int32Vector);
-export class Int32Vector extends NullableInt32Vector {}
-export const NullableInt64Vector = nullableMixin(vectors.Int64Vector);
-export class Int64Vector extends NullableInt64Vector {}
-export const NullableUint8Vector = nullableMixin(vectors.Uint8Vector);
-export class Uint8Vector extends NullableUint8Vector {}
-export const NullableUint16Vector = nullableMixin(vectors.Uint16Vector);
-export class Uint16Vector extends NullableUint16Vector {}
-export const NullableUint32Vector = nullableMixin(vectors.Uint32Vector);
-export class Uint32Vector extends NullableUint32Vector {}
-export const NullableUint64Vector = nullableMixin(vectors.Uint64Vector);
-export class Uint64Vector extends NullableUint64Vector {}
-export const NullableDate32Vector = nullableMixin(vectors.Date32Vector);
-export class Date32Vector extends NullableDate32Vector {}
-export const NullableDate64Vector = nullableMixin(vectors.Date64Vector);
-export class Date64Vector extends NullableDate64Vector {}
-export const NullableTime32Vector = nullableMixin(vectors.Time32Vector);
-export class Time32Vector extends NullableTime32Vector {}
-export const NullableTime64Vector = nullableMixin(vectors.Time64Vector);
-export class Time64Vector extends NullableTime64Vector {}
-export const NullableFloat16Vector = nullableMixin(vectors.Float16Vector);
-export class Float16Vector extends NullableFloat16Vector {}
-export const NullableFloat32Vector = nullableMixin(vectors.Float32Vector);
-export class Float32Vector extends NullableFloat32Vector {}
-export const NullableFloat64Vector = nullableMixin(vectors.Float64Vector);
-export class Float64Vector extends NullableFloat64Vector {}
-export const NullableStructVector = nullableMixin(vectors.StructVector);
-export class StructVector extends NullableStructVector {}
-export const NullableDecimalVector = nullableMixin(vectors.DecimalVector);
-export class DecimalVector extends NullableDecimalVector {}
-export const NullableTimestampVector = nullableMixin(vectors.TimestampVector);
-export class TimestampVector extends NullableTimestampVector {}
-export const NullableDictionaryVector = nullableMixin(vectors.DictionaryVector);
-export class DictionaryVector extends NullableDictionaryVector {}
-export const NullableFixedSizeListVector = nullableMixin(vectors.FixedSizeListVector);
-export class FixedSizeListVector extends NullableFixedSizeListVector {}
\ No newline at end of file
diff --git a/js/src/vector/traits/nullablefield.ts b/js/src/vector/traits/nullablefield.ts
deleted file mode 100644
index 8cbee62e43dc3..0000000000000
--- a/js/src/vector/traits/nullablefield.ts
+++ /dev/null
@@ -1,71 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from '../vector';
-import * as vectors from './vectors';
-import { nullableMixin, fieldMixin } from './mixins';
-import { Field, FieldNode } from '../../format/arrow';
-export { Vector, Field, FieldNode };
-
-export const NullableFieldListVector = nullableMixin(fieldMixin(vectors.ListVector));
-export class ListVector extends NullableFieldListVector {}
-export const NullableFieldBinaryVector = nullableMixin(fieldMixin(vectors.BinaryVector));
-export class BinaryVector extends NullableFieldBinaryVector {}
-export const NullableFieldUtf8Vector = nullableMixin(fieldMixin(vectors.Utf8Vector));
-export class Utf8Vector extends NullableFieldUtf8Vector {}
-export const NullableFieldBoolVector = nullableMixin(fieldMixin(vectors.BoolVector));
-export class BoolVector extends NullableFieldBoolVector {}
-export const NullableFieldInt8Vector = nullableMixin(fieldMixin(vectors.Int8Vector));
-export class Int8Vector extends NullableFieldInt8Vector {}
-export const NullableFieldInt16Vector = nullableMixin(fieldMixin(vectors.Int16Vector));
-export class Int16Vector extends NullableFieldInt16Vector {}
-export const NullableFieldInt32Vector = nullableMixin(fieldMixin(vectors.Int32Vector));
-export class Int32Vector extends NullableFieldInt32Vector {}
-export const NullableFieldInt64Vector = nullableMixin(fieldMixin(vectors.Int64Vector));
-export class Int64Vector extends NullableFieldInt64Vector {}
-export const NullableFieldUint8Vector = nullableMixin(fieldMixin(vectors.Uint8Vector));
-export class Uint8Vector extends NullableFieldUint8Vector {}
-export const NullableFieldUint16Vector = nullableMixin(fieldMixin(vectors.Uint16Vector));
-export class Uint16Vector extends NullableFieldUint16Vector {}
-export const NullableFieldUint32Vector = nullableMixin(fieldMixin(vectors.Uint32Vector));
-export class Uint32Vector extends NullableFieldUint32Vector {}
-export const NullableFieldUint64Vector = nullableMixin(fieldMixin(vectors.Uint64Vector));
-export class Uint64Vector extends NullableFieldUint64Vector {}
-export const NullableFieldDate32Vector = nullableMixin(fieldMixin(vectors.Date32Vector));
-export class Date32Vector extends NullableFieldDate32Vector {}
-export const NullableFieldDate64Vector = nullableMixin(fieldMixin(vectors.Date64Vector));
-export class Date64Vector extends NullableFieldDate64Vector {}
-export const NullableFieldTime32Vector = nullableMixin(fieldMixin(vectors.Time32Vector));
-export class Time32Vector extends NullableFieldTime32Vector {}
-export const NullableFieldTime64Vector = nullableMixin(fieldMixin(vectors.Time64Vector));
-export class Time64Vector extends NullableFieldTime64Vector {}
-export const NullableFieldFloat16Vector = nullableMixin(fieldMixin(vectors.Float16Vector));
-export class Float16Vector extends NullableFieldFloat16Vector {}
-export const NullableFieldFloat32Vector = nullableMixin(fieldMixin(vectors.Float32Vector));
-export class Float32Vector extends NullableFieldFloat32Vector {}
-export const NullableFieldFloat64Vector = nullableMixin(fieldMixin(vectors.Float64Vector));
-export class Float64Vector extends NullableFieldFloat64Vector {}
-export const NullableFieldStructVector = nullableMixin(fieldMixin(vectors.StructVector));
-export class StructVector extends NullableFieldStructVector {}
-export const NullableFieldDecimalVector = nullableMixin(fieldMixin(vectors.DecimalVector));
-export class DecimalVector extends NullableFieldDecimalVector {}
-export const NullableFieldTimestampVector = nullableMixin(fieldMixin(vectors.TimestampVector));
-export class TimestampVector extends NullableFieldTimestampVector {}
-export const NullableFieldDictionaryVector = nullableMixin(fieldMixin(vectors.DictionaryVector));
-export class DictionaryVector extends NullableFieldDictionaryVector {}
-export const NullableFieldFixedSizeListVector = nullableMixin(fieldMixin(vectors.FixedSizeListVector));
-export class FixedSizeListVector extends NullableFieldFixedSizeListVector {}
\ No newline at end of file
diff --git a/js/src/vector/traits/vectors.ts b/js/src/vector/traits/vectors.ts
deleted file mode 100644
index f9e05fd4eff1b..0000000000000
--- a/js/src/vector/traits/vectors.ts
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from '../vector';
-import { Utf8Vector } from '../utf8';
-import { StructVector } from '../struct';
-import { DictionaryVector } from '../dictionary';
-import {
-    ListVector,
-    BinaryVector,
-    FixedSizeListVector
-} from '../list';
-
-import {
-    BoolVector,
-    Int8Vector,
-    Int16Vector,
-    Int32Vector,
-    Int64Vector,
-    Uint8Vector,
-    Uint16Vector,
-    Uint32Vector,
-    Uint64Vector,
-    Float16Vector,
-    Float32Vector,
-    Float64Vector,
-    Date32Vector,
-    Date64Vector,
-    Time32Vector,
-    Time64Vector,
-    DecimalVector,
-    TimestampVector,
-} from '../numeric';
-
-export {
-    Vector,
-    BoolVector,
-    ListVector,
-    Utf8Vector,
-    Int8Vector,
-    Int16Vector,
-    Int32Vector,
-    Int64Vector,
-    Uint8Vector,
-    Uint16Vector,
-    Uint32Vector,
-    Uint64Vector,
-    Date32Vector,
-    Date64Vector,
-    Time32Vector,
-    Time64Vector,
-    BinaryVector,
-    StructVector,
-    Float16Vector,
-    Float32Vector,
-    Float64Vector,
-    DecimalVector,
-    TimestampVector,
-    DictionaryVector,
-    FixedSizeListVector,
-};
diff --git a/js/src/vector/types.ts b/js/src/vector/types.ts
deleted file mode 100644
index 363fcf2265c30..0000000000000
--- a/js/src/vector/types.ts
+++ /dev/null
@@ -1,43 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-export interface TypedArrayConstructor<T extends TypedArray = TypedArray> {
-    readonly prototype: T;
-    readonly BYTES_PER_ELEMENT: number;
-    new (length: number): T;
-    new (elements: Iterable<number>): T;
-    new (arrayOrArrayBuffer: ArrayLike<number> | ArrayBufferLike): T;
-    new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T;
-}
-
-export interface TypedArray extends Iterable<number> {
-    [index: number]: number;
-    readonly length: number;
-    readonly byteLength: number;
-    readonly byteOffset: number;
-    readonly buffer: ArrayBufferLike;
-    readonly BYTES_PER_ELEMENT: number;
-    [Symbol.iterator](): IterableIterator<number>;
-    slice(start?: number, end?: number): TypedArray;
-    subarray(begin: number, end?: number): TypedArray;
-    set(array: ArrayLike<number>, offset?: number): void;
-}
-
-export type List<T> = T[] | TypedArray;
-export type FloatArray = Float32Array | Float64Array;
-export type IntArray = Int8Array | Int16Array | Int32Array;
-export type UintArray = Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array;
diff --git a/js/src/vector/vector.ts b/js/src/vector/vector.ts
deleted file mode 100644
index 9565478ad969b..0000000000000
--- a/js/src/vector/vector.ts
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import * as Schema_ from '../format/fb/Schema';
-import Type = Schema_.org.apache.arrow.flatbuf.Type;
-
-export interface Vector<T = any> extends Iterable<T | null> {
-    readonly name: string;
-    readonly type: string;
-    readonly length: number;
-    readonly nullable: boolean;
-    readonly nullCount: number;
-    readonly metadata: Map<string, string>;
-    get(index: number): T | null;
-    concat(...vectors: Vector<T>[]): Vector<T>;
-    slice<R = T[]>(start?: number, end?: number): R;
-}
-
-export class Vector<T = any> implements Vector<T> {
-    slice<R = T[]>(start?: number, end?: number): R {
-        let { length } = this, from = start! | 0;
-        let to = end === undefined ? length : Math.max(end | 0, from);
-        let result = new Array<T | null>(to - Math.min(from, to));
-        for (let i = -1, n = result.length; ++i < n;) {
-            result[i] = this.get(i + from);
-        }
-        return result as any;
-    }
-    *[Symbol.iterator]() {
-        for (let i = -1, n = this.length; ++i < n;) {
-            yield this.get(i);
-        }
-    }
-}
-
-(Vector.prototype as any).name = '';
-(Vector.prototype as any).stride = 1;
-(Vector.prototype as any).nullable = !1;
-(Vector.prototype as any).nullCount = 0;
-(Vector.prototype as any).metadata = new Map();
-(Vector.prototype as any).type = Type[Type.NONE];
diff --git a/js/src/vector/view.ts b/js/src/vector/view.ts
new file mode 100644
index 0000000000000..c314a3180a5bf
--- /dev/null
+++ b/js/src/vector/view.ts
@@ -0,0 +1,8 @@
+export { ChunkedView } from './chunked';
+export { DictionaryView } from './dictionary';
+export { ListView, FixedSizeListView, BinaryView, Utf8View } from './list';
+export { UnionView, DenseUnionView, NestedView, StructView, MapView } from './nested';
+export { FlatView, NullView, BoolView, ValidityView, PrimitiveView, FixedSizeView, Float16View } from './flat';
+export { DateDayView, DateMillisecondView } from './flat';
+export { IntervalYearMonthView, IntervalYearView, IntervalMonthView } from './flat';
+export { TimestampDayView, TimestampSecondView, TimestampMillisecondView, TimestampMicrosecondView, TimestampNanosecondView } from './flat';
diff --git a/js/src/vector/virtual.ts b/js/src/vector/virtual.ts
deleted file mode 100644
index 42db78706db51..0000000000000
--- a/js/src/vector/virtual.ts
+++ /dev/null
@@ -1,129 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { Vector } from './vector';
-import { NumericVector } from './numeric';
-import { TypedArray, TypedArrayConstructor } from './types';
-
-export class VirtualVector<T> implements Vector<T> {
-    readonly name: string;
-    readonly type: string;
-    readonly length: number;
-    readonly vectors: Vector<T>[];
-    readonly offsets: Uint32Array;
-    readonly ArrayType: ArrayConstructor | TypedArrayConstructor;
-    constructor(ArrayType: ArrayConstructor | TypedArrayConstructor, ...vectors: Vector<T>[]) {
-        this.vectors = vectors;
-        this.ArrayType = ArrayType;
-        this.name = (vectors[0] as any).name;
-        this.type = (vectors[0] as any).type;
-        this.length = vectors.reduce((sum, vec) => sum + vec.length, 0);
-        this.offsets = Uint32Array.from(vectors.reduce((sums, vector, index) => [...sums, vector.length + sums[index]], [0]));
-    }
-    *[Symbol.iterator]() {
-        for (const vector of this.vectors) {
-            yield* vector;
-        }
-    }
-    get nullable() {
-        return (this.vectors as Vector<T>[]).some((vec) => vec.nullable);
-    }
-    get nullCount() {
-        return (this.vectors as Vector<T>[]).reduce((sum, v) => sum + v.nullCount | 0, 0);
-    }
-    get metadata() {
-        return new Map<string, string>(
-            (this.vectors as Vector<T>[]).reduce((entries, v) => [
-                ...entries, ...v.metadata.entries()
-            ], [] as [string, string][])
-        );
-    }
-    get(index: number) {
-        // find the vector index and adjusted value offset (inlined)
-        let offsets = this.offsets, offset = 0;
-        let left = 0, middle = 0, right = offsets.length - 1;
-        while (index < offsets[right] && index >= (offset = offsets[left])) {
-            if (left + 1 === right) {
-                return this.vectors[left].get(index - offset);
-            }
-            middle = left + ((right - left) / 2) | 0;
-            index >= offsets[middle] ? (left = middle) : (right = middle);
-        }
-        return null;
-    }
-    concat(...vectors: Vector<T>[]) {
-        return new VirtualVector(this.ArrayType, ...this.vectors, ...vectors);
-    }
-    slice(begin?: number, end?: number) {
-
-        // clamp begin and end values between the virtual length (inlined)
-        // let [from, total] = clampRange(this.length, begin!, end);
-        let total = this.length, from = begin! | 0;
-        let to = end === end && typeof end == 'number' ? end : total;
-        if (to < 0) { to = total + to; }
-        if (from < 0) { from = total - (from * -1) % total; }
-        if (to < from) { from = to; to = begin! | 0; }
-        total = !isFinite(total = (to - from)) || total < 0 ? 0 : total;
-
-        // find the vector index and adjusted value offset (inlined)
-        let offsets = this.offsets, ArrayType = this.ArrayType as any;
-        let offset = 0, index = 0, middle = 0, right = offsets.length - 1;
-        while (from < offsets[right] && from >= (offset = offsets[index])) {
-            if (index + 1 === right) {
-                from -= offset;
-                let set = ArrayType === Array ? arraySet : typedArraySet as any;
-                let vectors = this.vectors as any as NumericVector<T, TypedArray>[];
-                let vector = vectors[index], source = vector.slice(from, from + total), target = source;
-                // Perf optimization: if the first slice contains all the values we're looking for,
-                // we don't have to copy values to a target Array. If we're slicing a TypedArray,
-                // this is a significant improvement as we avoid the memcpy 🎉
-                if ((source.length / vector.stride | 0) < total) {
-                    let vectorsLength = vectors.length;
-                    let count = 0, length = 0, sources = [] as any[];
-                    do {
-                        sources.push(source);
-                        length += source.length;
-                        count += (source.length / vector.stride | 0);
-                    } while (
-                        (count  < total) &&
-                        (vector = vectors[index = (++index % vectorsLength)]) &&
-                        (source = vector.slice(0, Math.min(vector.length, total - count)))
-                    );
-                    target = new ArrayType(length);
-                    for (let i = -1, j = 0, n = sources.length; ++i < n;) {
-                        j = set(sources[i], target, j);
-                    }
-                }
-                return target;
-            }
-            middle = index + ((right - index) / 2) | 0;
-            from >= offsets[middle] ? (index = middle) : (right = middle);
-        }
-        return new ArrayType(0);
-    }
-}
-
-function arraySet<T>(source: T[], target: T[], index: number) {
-    for (let i = 0, n = source.length; i < n;) {
-        target[index++] = source[i++];
-    }
-    return index;
-}
-
-function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
-    return target.set(source, index) || index + source.length;
-}
diff --git a/js/src/visitor.ts b/js/src/visitor.ts
new file mode 100644
index 0000000000000..3f46f6681c7e7
--- /dev/null
+++ b/js/src/visitor.ts
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { Type, DataType, Dictionary } from './type';
+import { Utf8, Binary, Decimal, FixedSizeBinary } from './type';
+import { List, FixedSizeList, Union, Map_, Struct } from './type';
+import { Bool, Null, Int, Float, Date_, Time, Interval, Timestamp } from './type';
+
+export interface VisitorNode {
+    acceptTypeVisitor(visitor: TypeVisitor): any;
+    acceptVectorVisitor(visitor: VectorVisitor): any;
+    // acceptMessageVisitor(visitor: MessageVisitor): any;
+}
+
+export abstract class TypeVisitor {
+    visit(node: Partial<VisitorNode>): any {
+        return node.acceptTypeVisitor!(this);
+    }
+    visitMany(nodes: Partial<VisitorNode>[]): any[] {
+        return nodes.map((node) => this.visit(node));
+    }
+    abstract visitNull(node: Null): any;
+    abstract visitBool(node: Bool): any;
+    abstract visitInt(node: Int): any;
+    abstract visitFloat(node: Float): any;
+    abstract visitUtf8(node: Utf8): any;
+    abstract visitBinary(node: Binary): any;
+    abstract visitFixedSizeBinary(node: FixedSizeBinary): any;
+    abstract visitDate(node: Date_): any;
+    abstract visitTimestamp(node: Timestamp): any;
+    abstract visitTime(node: Time): any;
+    abstract visitDecimal(node: Decimal): any;
+    abstract visitList(node: List): any;
+    abstract visitStruct(node: Struct): any;
+    abstract visitUnion(node: Union<any>): any;
+    abstract visitDictionary(node: Dictionary): any;
+    abstract visitInterval(node: Interval): any;
+    abstract visitFixedSizeList(node: FixedSizeList): any;
+    abstract visitMap(node: Map_): any;
+
+    static visitTypeInline<T extends DataType>(visitor: TypeVisitor, type: T): any {
+        switch (type.TType) {
+            case Type.Null:            return visitor.visitNull(type            as any as Null);
+            case Type.Int:             return visitor.visitInt(type             as any as Int);
+            case Type.Float:           return visitor.visitFloat(type           as any as Float);
+            case Type.Binary:          return visitor.visitBinary(type          as any as Binary);
+            case Type.Utf8:            return visitor.visitUtf8(type            as any as Utf8);
+            case Type.Bool:            return visitor.visitBool(type            as any as Bool);
+            case Type.Decimal:         return visitor.visitDecimal(type         as any as Decimal);
+            case Type.Date:            return visitor.visitDate(type            as any as Date_);
+            case Type.Time:            return visitor.visitTime(type            as any as Time);
+            case Type.Timestamp:       return visitor.visitTimestamp(type       as any as Timestamp);
+            case Type.Interval:        return visitor.visitInterval(type        as any as Interval);
+            case Type.List:            return visitor.visitList(type            as any as List<T>);
+            case Type.Struct:          return visitor.visitStruct(type          as any as Struct);
+            case Type.Union:           return visitor.visitUnion(type           as any as Union);
+            case Type.FixedSizeBinary: return visitor.visitFixedSizeBinary(type as any as FixedSizeBinary);
+            case Type.FixedSizeList:   return visitor.visitFixedSizeList(type   as any as FixedSizeList);
+            case Type.Map:             return visitor.visitMap(type             as any as Map_);
+            case Type.Dictionary:      return visitor.visitDictionary(type      as any as Dictionary);
+            default: return null;
+        }
+    }
+}
+
+export abstract class VectorVisitor {
+    visit(node: Partial<VisitorNode>): any {
+        return node.acceptVectorVisitor!(this);
+    }
+    visitMany(nodes: Partial<VisitorNode>[]): any[] {
+        return nodes.map((node) => this.visit(node));
+    }
+    abstract visitNullVector(node: Vector<Null>): any;
+    abstract visitBoolVector(node: Vector<Bool>): any;
+    abstract visitIntVector(node: Vector<Int>): any;
+    abstract visitFloatVector(node: Vector<Float>): any;
+    abstract visitUtf8Vector(node: Vector<Utf8>): any;
+    abstract visitBinaryVector(node: Vector<Binary>): any;
+    abstract visitFixedSizeBinaryVector(node: Vector<FixedSizeBinary>): any;
+    abstract visitDateVector(node: Vector<Date_>): any;
+    abstract visitTimestampVector(node: Vector<Timestamp>): any;
+    abstract visitTimeVector(node: Vector<Time>): any;
+    abstract visitDecimalVector(node: Vector<Decimal>): any;
+    abstract visitListVector(node: Vector<List>): any;
+    abstract visitStructVector(node: Vector<Struct>): any;
+    abstract visitUnionVector(node: Vector<Union<any>>): any;
+    abstract visitDictionaryVector(node: Vector<Dictionary>): any;
+    abstract visitIntervalVector(node: Vector<Interval>): any;
+    abstract visitFixedSizeListVector(node: Vector<FixedSizeList>): any;
+    abstract visitMapVector(node: Vector<Map_>): any;
+
+    static visitTypeInline<T extends DataType>(visitor: VectorVisitor, type: T, vector: Vector<T>): any {
+        switch (type.TType) {
+            case Type.Null:            return visitor.visitNullVector(vector            as any as Vector<Null>);
+            case Type.Int:             return visitor.visitIntVector(vector             as any as Vector<Int>);
+            case Type.Float:           return visitor.visitFloatVector(vector           as any as Vector<Float>);
+            case Type.Binary:          return visitor.visitBinaryVector(vector          as any as Vector<Binary>);
+            case Type.Utf8:            return visitor.visitUtf8Vector(vector            as any as Vector<Utf8>);
+            case Type.Bool:            return visitor.visitBoolVector(vector            as any as Vector<Bool>);
+            case Type.Decimal:         return visitor.visitDecimalVector(vector         as any as Vector<Decimal>);
+            case Type.Date:            return visitor.visitDateVector(vector            as any as Vector<Date_>);
+            case Type.Time:            return visitor.visitTimeVector(vector            as any as Vector<Time>);
+            case Type.Timestamp:       return visitor.visitTimestampVector(vector       as any as Vector<Timestamp>);
+            case Type.Interval:        return visitor.visitIntervalVector(vector        as any as Vector<Interval>);
+            case Type.List:            return visitor.visitListVector(vector            as any as Vector<List<T>>);
+            case Type.Struct:          return visitor.visitStructVector(vector          as any as Vector<Struct>);
+            case Type.Union:           return visitor.visitUnionVector(vector           as any as Vector<Union>);
+            case Type.FixedSizeBinary: return visitor.visitFixedSizeBinaryVector(vector as any as Vector<FixedSizeBinary>);
+            case Type.FixedSizeList:   return visitor.visitFixedSizeListVector(vector   as any as Vector<FixedSizeList>);
+            case Type.Map:             return visitor.visitMapVector(vector             as any as Vector<Map_>);
+            case Type.Dictionary:      return visitor.visitDictionaryVector(vector      as any as Vector<Dictionary>);
+            default: return null;
+        }
+    }
+}
+
+// import { Footer, Block } from './ipc/message';
+// import { Field, FieldNode, Buffer } from './ipc/message';
+// import { Message, Schema, RecordBatch, DictionaryBatch } from './ipc/message';
+
+// export abstract class MessageVisitor {
+//     visit(node: VisitorNode): any {
+//         return node.acceptMessageVisitor(this);
+//     }
+//     visitMany(nodes: VisitorNode[]): any[] {
+//         return nodes.map((node) => this.visit(node));
+//     }
+//     abstract visitFooter(node: Footer): any;
+//     abstract visitBlock(node: Block): any;
+//     abstract visitMessage(node: Message): any;
+//     abstract visitSchema(node: Schema): any;
+//     abstract visitField<T extends DataType>(node: Field<T>): any;
+//     abstract visitBuffer(node: Buffer): any;
+//     abstract visitFieldNode(node: FieldNode): any;
+//     abstract visitDataType<T extends Type>(node: DataType<T>): any;
+//     abstract visitDictionary(node: Dictionary): any;
+//     abstract visitRecordBatch(node: RecordBatch): any;
+//     abstract visitDictionaryBatch(node: DictionaryBatch): any;
+// }
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
index f2c4e930f92e4..5921c4cecb9ee 100644
--- a/js/test/Arrow.ts
+++ b/js/test/Arrow.ts
@@ -39,12 +39,16 @@ else if (!~targets.indexOf(target)) throwInvalidImportError('target', target, ta
 else if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats);
 else modulePath = path.join(target, format);
 
-export { Int64, Uint64, Int128 } from '../src/Arrow';
-export { List } from '../src/Arrow';
-export { TypedArray } from '../src/Arrow';
-export { TypedArrayConstructor } from '../src/Arrow';
-export { NumericVectorConstructor } from '../src/Arrow';
+import { read, readAsync } from '../src/Arrow';
+export { read, readAsync };
+import { View,  VectorLike } from '../src/Arrow';
+export { View,  VectorLike };
+import { Table, Field, Schema, RecordBatch, Type } from '../src/Arrow';
+export { Table, Field, Schema, RecordBatch, Type };
+
+import { TypedArray, TypedArrayConstructor, IntBitWidth, TimeBitWidth } from '../src/Arrow';
+export { TypedArray, TypedArrayConstructor, IntBitWidth, TimeBitWidth };
 
 import * as Arrow_ from '../src/Arrow';
-export let Arrow: typeof Arrow_ = require(path.resolve(`./targets`, modulePath, `Arrow`));
+export let Arrow = require(path.resolve(`./targets`, modulePath, `Arrow`)) as typeof Arrow_;
 export default Arrow;
diff --git a/js/test/data/tables/generate.py b/js/test/data/tables/generate.py
new file mode 100644
index 0000000000000..da19c6a0728c0
--- /dev/null
+++ b/js/test/data/tables/generate.py
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pyarrow as pa
+import random
+import numpy as np
+import pandas as pd
+
+
+cities = [u'Charlottesville', u'New York', u'San Francisco', u'Seattle', u'Terre Haute', u'Washington, DC']
+
+def generate_batch(batch_len):
+    return pa.RecordBatch.from_arrays([
+        pa.Array.from_pandas(pd.Series(np.random.uniform(-90,90,batch_len), dtype="float32")),
+        pa.Array.from_pandas(pd.Series(np.random.uniform(-180,180,batch_len), dtype="float32")),
+        pa.Array.from_pandas(pd.Categorical((random.choice(cities) for i in range(batch_len)), cities)),
+        pa.Array.from_pandas(pd.Categorical((random.choice(cities) for i in range(batch_len)), cities))
+    ], ['lat', 'lng', 'origin', 'destination'])
+
+def write_record_batches(fd, batch_len, num_batches):
+    writer = pa.ipc.RecordBatchStreamWriter(fd, generate_batch(1).schema)
+    for batch in range(num_batches):
+        writer.write_batch(generate_batch(batch_len))
+
+    writer.close()
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filename', help='number of batches')
+    parser.add_argument('-n', '--num-batches', help='number of batches', type=int, default=10)
+    parser.add_argument('-b', '--batch-size', help='size of each batch', type=int, default=100000)
+
+    args = parser.parse_args()
+
+    print "Writing {} {}-element batches to '{}'".format(args.num_batches, args.batch_size, args.filename)
+    with open(args.filename, 'w') as fd:
+        write_record_batches(fd, args.batch_size, args.num_batches)
diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts
index c612d62ad0c04..f81634279c943 100644
--- a/js/test/integration/validate-tests.ts
+++ b/js/test/integration/validate-tests.ts
@@ -18,28 +18,41 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-if (!process.env.JSON_PATH || !process.env.ARROW_PATH) {
-    throw new Error('Integration tests need paths to both json and arrow files');
-}
+import Arrow from '../Arrow';
+import { zip } from 'ix/iterable/zip';
+import { toArray } from 'ix/iterable/toarray';
 
-const jsonPath = path.resolve(process.env.JSON_PATH + '');
-const arrowPath = path.resolve(process.env.ARROW_PATH + '');
+/* tslint:disable */
+const { parse: bignumJSONParse } = require('json-bignum');
 
-if (!fs.existsSync(jsonPath) || !fs.existsSync(arrowPath)) {
-    throw new Error('Integration tests need both json and arrow files to exist');
-}
+const { Table, read } = Arrow;
 
-/* tslint:disable */
-const { parse } = require('json-bignum');
+if (!process.env.JSON_PATHS || !process.env.ARROW_PATHS) {
+    throw new Error('Integration tests need paths to both json and arrow files');
+}
 
-const jsonData = parse(fs.readFileSync(jsonPath, 'utf8'));
-const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)];
+function resolvePathArgs(paths: string) {
+    let pathsArray = JSON.parse(paths) as string | string[];
+    return (Array.isArray(pathsArray) ? pathsArray : [pathsArray])
+        .map((p) => path.resolve(p))
+        .map((p) => {
+            if (fs.existsSync(p)) {
+                return p;
+            }
+            console.warn(`Could not find file "${p}"`);
+            return undefined;
+        });
+}
 
-import Arrow from '../Arrow';
-import { zip } from 'ix/iterable/zip';
-import { toArray } from 'ix/iterable/toarray';
+const getOrReadFileBuffer = ((cache: any) => function getFileBuffer(path: string, ...args: any[]) {
+    return cache[path] || (cache[path] = fs.readFileSync(path, ...args));
+})({});
 
-const { Table, read } = Arrow;
+const jsonAndArrowPaths = toArray(zip(
+    resolvePathArgs(process.env.JSON_PATHS!),
+    resolvePathArgs(process.env.ARROW_PATHS!)
+))
+.filter(([p1, p2]) => p1 !== undefined && p2 !== undefined) as [string, string][];
 
 expect.extend({
     toEqualVector(v1: any, v2: any) {
@@ -59,10 +72,14 @@ expect.extend({
             { title: 'iterator', failures: iteratorFailures }
         ];
 
-        let props = ['name', 'type', 'length', 'nullable', 'nullCount', 'metadata'];
+        let props = [
+            // 'name', 'nullable', 'metadata',
+            'type', 'length', 'nullCount'
+        ];
+
         for (let i = -1, n = props.length; ++i < n;) {
             const prop = props[i];
-            if (this.utils.stringify(v1[prop]) !== this.utils.stringify(v2[prop])) {
+            if (`${v1[prop]}` !== `${v2[prop]}`) {
                 propsFailures.push(`${prop}: ${format(v1[prop], v2[prop], ' !== ')}`);
             }
         }
@@ -94,35 +111,44 @@ expect.extend({
 });
 
 describe(`Integration`, () => {
-    testReaderIntegration();
-    testTableFromBuffersIntegration();
+    for (const [jsonFilePath, arrowFilePath] of jsonAndArrowPaths) {
+        let { name, dir } = path.parse(arrowFilePath);
+        dir = dir.split(path.sep).slice(-2).join(path.sep);
+        const json = bignumJSONParse(getOrReadFileBuffer(jsonFilePath, 'utf8'));
+        const arrowBuffer = getOrReadFileBuffer(arrowFilePath) as Uint8Array;
+        describe(path.join(dir, name), () => {
+            testReaderIntegration(json, arrowBuffer);
+            testTableFromBuffersIntegration(json, arrowBuffer);
+        });
+    }
 });
 
-function testReaderIntegration() {
-    test(`json and arrow buffers report the same values`, () => {
+function testReaderIntegration(jsonData: any, arrowBuffer: Uint8Array) {
+    test(`json and arrow record batches report the same values`, () => {
         expect.hasAssertions();
-        const jsonVectors = toArray(read(jsonData));
-        const binaryVectors = toArray(read(arrowBuffers));
-        for (const [jVectors, bVectors] of zip(jsonVectors, binaryVectors)) {
-            expect(jVectors.length).toEqual(bVectors.length);
-            for (let i = -1, n = jVectors.length; ++i < n;) {
-                (expect(jVectors[i]) as any).toEqualVector(bVectors[i]);
+        const jsonRecordBatches = toArray(read(jsonData));
+        const binaryRecordBatches = toArray(read(arrowBuffer));
+        for (const [jsonRecordBatch, binaryRecordBatch] of zip(jsonRecordBatches, binaryRecordBatches)) {
+            expect(jsonRecordBatch.length).toEqual(binaryRecordBatch.length);
+            expect(jsonRecordBatch.numCols).toEqual(binaryRecordBatch.numCols);
+            for (let i = -1, n = jsonRecordBatch.numCols; ++i < n;) {
+                (jsonRecordBatch.getChildAt(i) as any).name = jsonRecordBatch.schema.fields[i].name;
+                (expect(jsonRecordBatch.getChildAt(i)) as any).toEqualVector(binaryRecordBatch.getChildAt(i));
             }
         }
     });
 }
 
-function testTableFromBuffersIntegration() {
-    test(`json and arrow buffers report the same values`, () => {
+function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
+    test(`json and arrow tables report the same values`, () => {
         expect.hasAssertions();
         const jsonTable = Table.from(jsonData);
-        const binaryTable = Table.from(arrowBuffers);
-        const jsonVectors = jsonTable.columns;
-        const binaryVectors = binaryTable.columns;
+        const binaryTable = Table.from(arrowBuffer);
         expect(jsonTable.length).toEqual(binaryTable.length);
-        expect(jsonVectors.length).toEqual(binaryVectors.length);
-        for (let i = -1, n = jsonVectors.length; ++i < n;) {
-            (expect(jsonVectors[i]) as any).toEqualVector(binaryVectors[i]);
+        expect(jsonTable.numCols).toEqual(binaryTable.numCols);
+        for (let i = -1, n = jsonTable.numCols; ++i < n;) {
+            (jsonTable.getColumnAt(i) as any).name = jsonTable.schema.fields[i].name;
+            (expect(jsonTable.getColumnAt(i)) as any).toEqualVector(binaryTable.getColumnAt(i));
         }
     });
 }
diff --git a/js/test/unit/int-tests.ts b/js/test/unit/int-tests.ts
index 74c96e8efbb6d..fbd92e7c7323d 100644
--- a/js/test/unit/int-tests.ts
+++ b/js/test/unit/int-tests.ts
@@ -15,11 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import {
-    Int64,
-    Uint64,
-    Int128
-} from '../Arrow';
+import Arrow from '../Arrow';
+const { Int64, Uint64, Int128 } = Arrow.util;
 
 describe(`Uint64`, () => {
     test(`gets expected high/low bytes`, () => {
diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts
new file mode 100644
index 0000000000000..36d2ae9038147
--- /dev/null
+++ b/js/test/unit/table-tests.ts
@@ -0,0 +1,475 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Arrow from '../Arrow';
+
+const { predicate, Table } = Arrow;
+
+const { col, lit } = predicate;
+
+const F32 = 0, I32 = 1, DICT = 2;
+const test_data = [
+    {name: `single record batch`,
+     table: Table.from({
+          'schema': {
+            'fields': [
+              {
+                'name': 'f32',
+                'type': {
+                  'name': 'floatingpoint',
+                  'precision': 'SINGLE'
+                },
+                'nullable': false,
+                'children': [],
+              },
+              {
+                'name': 'i32',
+                'type': {
+                  'name': 'int',
+                  'isSigned': true,
+                  'bitWidth': 32
+                },
+                'nullable': false,
+                'children': [],
+              },
+              {
+                'name': 'dictionary',
+                'type': {
+                  'name': 'utf8'
+                },
+                'nullable': false,
+                'children': [],
+                'dictionary': {
+                  'id': 0,
+                  'indexType': {
+                    'name': 'int',
+                    'isSigned': true,
+                    'bitWidth': 8
+                  },
+                  'isOrdered': false
+                }
+              }
+            ]
+          },
+          'dictionaries': [{
+            'id': 0,
+            'data': {
+              'count': 3,
+              'columns': [
+                {
+                  'name': 'DICT0',
+                  'count': 3,
+                  'VALIDITY': [],
+                  'OFFSET': [
+                    0,
+                    1,
+                    2,
+                    3
+                  ],
+                  'DATA': [
+                    'a',
+                    'b',
+                    'c',
+                  ]
+                }
+              ]
+            }
+          }],
+          'batches': [{
+            'count': 7,
+            'columns': [
+              {
+                'name': 'f32',
+                'count': 7,
+                'VALIDITY': [],
+                'DATA': [-0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3]
+              },
+              {
+                'name': 'i32',
+                'count': 7,
+                'VALIDITY': [],
+                'DATA': [-1, 1, -1, 1, -1, 1, -1]
+              },
+              {
+                'name': 'dictionary',
+                'count': 7,
+                'VALIDITY': [],
+                'DATA': [0, 1, 2, 0, 1, 2, 0]
+              }
+            ]
+          }]
+        }),
+        // Use Math.fround to coerce to float32
+     values: [
+         [Math.fround(-0.3), -1, 'a'],
+         [Math.fround(-0.2),  1, 'b'],
+         [Math.fround(-0.1), -1, 'c'],
+         [Math.fround( 0  ),  1, 'a'],
+         [Math.fround( 0.1), -1, 'b'],
+         [Math.fround( 0.2),  1, 'c'],
+         [Math.fround( 0.3), -1, 'a']
+     ]},
+     {name: `multiple record batches`,
+      table: Table.from({
+          'schema': {
+            'fields': [
+              {
+                'name': 'f32',
+                'type': {
+                  'name': 'floatingpoint',
+                  'precision': 'SINGLE'
+                },
+                'nullable': false,
+                'children': [],
+              },
+              {
+                'name': 'i32',
+                'type': {
+                  'name': 'int',
+                  'isSigned': true,
+                  'bitWidth': 32
+                },
+                'nullable': false,
+                'children': [],
+              },
+              {
+                'name': 'dictionary',
+                'type': {
+                  'name': 'utf8'
+                },
+                'nullable': false,
+                'children': [],
+                'dictionary': {
+                  'id': 0,
+                  'indexType': {
+                    'name': 'int',
+                    'isSigned': true,
+                    'bitWidth': 8
+                  },
+                  'isOrdered': false
+                }
+              }
+            ]
+          },
+          'dictionaries': [{
+            'id': 0,
+            'data': {
+              'count': 3,
+              'columns': [
+                {
+                  'name': 'DICT0',
+                  'count': 3,
+                  'VALIDITY': [],
+                  'OFFSET': [
+                    0,
+                    1,
+                    2,
+                    3
+                  ],
+                  'DATA': [
+                    'a',
+                    'b',
+                    'c',
+                  ]
+                }
+              ]
+            }
+          }],
+          'batches': [{
+            'count': 3,
+            'columns': [
+              {
+                'name': 'f32',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [-0.3, -0.2, -0.1]
+              },
+              {
+                'name': 'i32',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [-1, 1, -1]
+              },
+              {
+                'name': 'dictionary',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [0, 1, 2]
+              }
+            ]
+          }, {
+            'count': 3,
+            'columns': [
+              {
+                'name': 'f32',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [0, 0.1, 0.2]
+              },
+              {
+                'name': 'i32',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [1, -1, 1]
+              },
+              {
+                'name': 'dictionary',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [0, 1, 2]
+              }
+            ]
+          }, {
+            'count': 3,
+            'columns': [
+              {
+                'name': 'f32',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [0.3, 0.2, 0.1]
+              },
+              {
+                'name': 'i32',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [-1, 1, -1]
+              },
+              {
+                'name': 'dictionary',
+                'count': 3,
+                'VALIDITY': [],
+                'DATA': [0, 1, 2]
+              }
+            ]
+          }]
+      }),
+      values: [
+            [Math.fround(-0.3), -1, 'a'],
+            [Math.fround(-0.2),  1, 'b'],
+            [Math.fround(-0.1), -1, 'c'],
+            [Math.fround( 0  ),  1, 'a'],
+            [Math.fround( 0.1), -1, 'b'],
+            [Math.fround( 0.2),  1, 'c'],
+            [Math.fround( 0.3), -1, 'a'],
+            [Math.fround( 0.2),  1, 'b'],
+            [Math.fround( 0.1), -1, 'c'],
+      ]}
+];
+
+describe(`Table`, () => {
+    test(`can create an empty table`, () => {
+        expect(Table.empty().length).toEqual(0);
+    });
+    test(`Table.from([]) creates an empty table`, () => {
+        expect(Table.from([]).length).toEqual(0);
+    });
+    test(`Table.from() creates an empty table`, () => {
+        expect(Table.from().length).toEqual(0);
+    });
+    for (let datum of test_data) {
+        describe(datum.name, () => {
+            const table = datum.table;
+            const values = datum.values;
+
+            test(`has the correct length`, () => {
+                expect(table.length).toEqual(values.length);
+            });
+            test(`gets expected values`, () => {
+                for (let i = -1; ++i < values.length;) {
+                    expect(table.get(i).toArray()).toEqual(values[i]);
+                }
+            });
+            test(`iterates expected values`, () => {
+                let i = 0;
+                for (let row of table) {
+                    expect(row.toArray()).toEqual(values[i++]);
+                }
+            });
+            describe(`scan()`, () => {
+                test(`yields all values`, () => {
+                    let expected_idx = 0;
+                    table.scan((idx, batch) => {
+                        const columns = batch.schema.fields.map((_, i) => batch.getChildAt(i)!);
+                        expect(columns.map((c) => c.get(idx))).toEqual(values[expected_idx++]);
+                    });
+                });
+                test(`calls bind function with every batch`, () => {
+                    let bind = jest.fn();
+                    table.scan(() => {}, bind);
+                    for (let batch of table.batches) {
+                        expect(bind).toHaveBeenCalledWith(batch);
+                    }
+                })
+            })
+            test(`count() returns the correct length`, () => {
+                expect(table.count()).toEqual(values.length);
+            });
+            test(`getColumnIndex`, () => {
+                expect(table.getColumnIndex('i32')).toEqual(I32);
+                expect(table.getColumnIndex('f32')).toEqual(F32);
+                expect(table.getColumnIndex('dictionary')).toEqual(DICT);
+            });
+            const filter_tests = [
+                {
+                    name:     `filter on f32 >= 0`,
+                    filtered: table.filter(col('f32').gteq(0)),
+                    expected: values.filter((row) => row[F32] >= 0)
+                }, {
+                    name:     `filter on 0 <= f32`,
+                    filtered: table.filter(lit(0).lteq(col('f32'))),
+                    expected: values.filter((row)=>0 <= row[F32])
+                }, {
+                    name:     `filter on i32 <= 0`,
+                    filtered: table.filter(col('i32').lteq(0)),
+                    expected: values.filter((row) => row[I32] <= 0)
+                }, {
+                    name:     `filter on 0 >= i32`,
+                    filtered: table.filter(lit(0).gteq(col('i32'))),
+                    expected: values.filter((row)=>0 >= row[I32])
+                }, {
+                    name:     `filter on f32 <= -.25 || f3 >= .25`,
+                    filtered: table.filter(col('f32').lteq(-.25).or(col('f32').gteq(.25))),
+                    expected: values.filter((row)=>row[F32] <= -.25 || row[F32] >= .25)
+                }, {
+                    name:     `filter method combines predicates (f32 >= 0 && i32 <= 0)`,
+                    filtered: table.filter(col('i32').lteq(0)).filter(col('f32').gteq(0)),
+                    expected: values.filter((row) => row[I32] <= 0 && row[F32] >= 0)
+                }, {
+                    name:     `filter on dictionary == 'a'`,
+                    filtered: table.filter(col('dictionary').eq('a')),
+                    expected: values.filter((row) => row[DICT] === 'a')
+                }, {
+                    name:     `filter on 'a' == dictionary (commutativity)`,
+                    filtered: table.filter(lit('a').eq(col('dictionary'))),
+                    expected: values.filter((row)=>row[DICT] === 'a')
+                }, {
+                    name:     `filter on f32 >= i32`,
+                    filtered: table.filter(col('f32').gteq(col('i32'))),
+                    expected: values.filter((row)=>row[F32] >= row[I32])
+                }, {
+                    name:     `filter on f32 <= i32`,
+                    filtered: table.filter(col('f32').lteq(col('i32'))),
+                    expected: values.filter((row)=>row[F32] <= row[I32])
+                }
+            ];
+            for (let this_test of filter_tests) {
+                const { name, filtered, expected } = this_test;
+                describe(name, () => {
+                    test(`count() returns the correct length`, () => {
+                        expect(filtered.count()).toEqual(expected.length);
+                    });
+                    describe(`scan()`, () => {
+                        test(`iterates over expected values`, () => {
+                            let expected_idx = 0;
+                            filtered.scan((idx, batch) => {
+                                const columns = batch.schema.fields.map((_, i) => batch.getChildAt(i)!);
+                                expect(columns.map((c) => c.get(idx))).toEqual(expected[expected_idx++]);
+                            });
+                        });
+                        test(`calls bind function on every batch`, () => {
+                            // Techincally, we only need to call bind on
+                            // batches with data that match the predicate, so
+                            // this test may fail in the future if we change
+                            // that - and that's ok!
+                            let bind = jest.fn();
+                            filtered.scan(() => {}, bind);
+                            for (let batch of table.batches) {
+                                expect(bind).toHaveBeenCalledWith(batch);
+                            }
+                        });
+                    });
+                });
+            }
+            test(`countBy on dictionary returns the correct counts`, () => {
+                // Make sure countBy works both with and without the Col wrapper
+                // class
+                let expected: {[key: string]: number} = {'a': 0, 'b': 0, 'c': 0};
+                for (let row of values) {
+                    expected[row[DICT]] += 1;
+                }
+
+                expect(table.countBy(col('dictionary')).toJSON()).toEqual(expected);
+                expect(table.countBy('dictionary').toJSON()).toEqual(expected);
+            });
+            test(`countBy on dictionary with filter returns the correct counts`, () => {
+                let expected: {[key: string]: number} = {'a': 0, 'b': 0, 'c': 0};
+                for (let row of values) {
+                    if (row[I32] === 1) { expected[row[DICT]] += 1; }
+                }
+
+                expect(table.filter(col('i32').eq(1)).countBy('dictionary').toJSON()).toEqual(expected);
+            });
+            test(`countBy on non dictionary column throws error`, () => {
+                expect(() => { table.countBy('i32'); }).toThrow();
+                expect(() => { table.filter(col('dict').eq('a')).countBy('i32'); }).toThrow();
+            });
+            test(`countBy on non-existent column throws error`, () => {
+                expect(() => { table.countBy('FAKE'); }).toThrow();
+            });
+            test(`table.select() basic tests`, () => {
+                let selected = table.select('f32', 'dictionary');
+                expect(selected.schema.fields.length).toEqual(2);
+                expect(selected.schema.fields[0]).toEqual(table.schema.fields[0]);
+                expect(selected.schema.fields[1]).toEqual(table.schema.fields[2]);
+
+                expect(selected.length).toEqual(values.length);
+                let idx = 0, expected_row;
+                for (let row of selected) {
+                    expected_row = values[idx++];
+                    expect(row.get(0)).toEqual(expected_row[F32]);
+                    expect(row.get(1)).toEqual(expected_row[DICT]);
+                }
+            });
+            test(`table.toString()`, () => {
+                let selected = table.select('i32', 'dictionary');
+                let headers  = [`"row_id"`, `"i32: Int32"`, `"dictionary: Dictionary<Utf8, Int8>"`];
+                let expected = [headers.join(' | '), ...values.map((row, idx) => {
+                    return [`${idx}`, `${row[I32]}`, `"${row[DICT]}"`].map((str, col) => {
+                                return leftPad(str, ' ', headers[col].length);
+                            }).join(' | ');
+                })].join('\n') + '\n';
+                expect(selected.toString()).toEqual(expected);
+            });
+            test(`table.filter(..).count() on always false predicates returns 0`, () => {
+                expect(table.filter(col('i32').gteq(100)).count()).toEqual(0);
+                expect(table.filter(col('dictionary').eq('z')).count()).toEqual(0);
+            });
+            describe(`lit-lit comparison`, () => {
+                test(`always-false count() returns 0`, () => {
+                    expect(table.filter(lit('abc').eq('def')).count()).toEqual(0);
+                    expect(table.filter(lit(0).gteq(1)).count()).toEqual(0);
+                });
+                test(`always-true count() returns length`, () => {
+                    expect(table.filter(lit('abc').eq('abc')).count()).toEqual(table.length);
+                    expect(table.filter(lit(-100).lteq(0)).count()).toEqual(table.length);
+                });
+            });
+            describe(`col-col comparison`, () => {
+                test(`always-false count() returns 0`, () => {
+                    expect(table.filter(col('dictionary').eq(col('i32'))).count()).toEqual(0);
+                });
+                test(`always-true count() returns length`, () => {
+                    expect(table.filter(col('dictionary').eq(col('dictionary'))).count()).toEqual(table.length);
+                });
+            });
+        });
+    }
+});
+
+function leftPad(str: string, fill: string, n: number) {
+    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
+}
diff --git a/js/test/unit/vector-tests.ts b/js/test/unit/vector-tests.ts
index 75706229ab172..81676b003e399 100644
--- a/js/test/unit/vector-tests.ts
+++ b/js/test/unit/vector-tests.ts
@@ -15,41 +15,32 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import Arrow, {
-    TypedArray,
-    TypedArrayConstructor,
-    NumericVectorConstructor,
-} from '../Arrow';
+import Arrow from '../Arrow';
+import { type, TypedArray, TypedArrayConstructor } from '../../src/Arrow';
 
+const { BoolData, FlatData } = Arrow.data;
+const { IntVector, FloatVector, BoolVector } = Arrow.vector;
 const {
-    BoolVector,
-    Int64Vector,
-    Uint64Vector,
-    Int8Vector,
-    Int16Vector,
-    Int32Vector,
-    Uint8Vector,
-    Uint16Vector,
-    Uint32Vector,
-    Float16Vector,
-    Float32Vector,
-    Float64Vector,
-} = Arrow;
+    Bool,
+    Float16, Float32, Float64,
+    Int8, Int16, Int32, Int64,
+    Uint8, Uint16, Uint32, Uint64,
+} = Arrow.type;
 
 const FixedSizeVectors = {
-    Int64Vector: [Int64Vector, Int32Array] as [NumericVectorConstructor<number, any>, any],
-    Uint64Vector: [Uint64Vector, Uint32Array] as [NumericVectorConstructor<number, any>, any]
+    Int64Vector: [IntVector, Int64] as [typeof IntVector, typeof Int64],
+    Uint64Vector: [IntVector, Uint64] as [typeof IntVector, typeof Uint64],
 };
 
 const FixedWidthVectors = {
-    Int8Vector: [Int8Vector, Int8Array] as [NumericVectorConstructor<number, any>, any],
-    Int16Vector: [Int16Vector, Int16Array] as [NumericVectorConstructor<number, any>, any],
-    Int32Vector: [Int32Vector, Int32Array] as [NumericVectorConstructor<number, any>, any],
-    Uint8Vector: [Uint8Vector, Uint8Array] as [NumericVectorConstructor<number, any>, any],
-    Uint16Vector: [Uint16Vector, Uint16Array] as [NumericVectorConstructor<number, any>, any],
-    Uint32Vector: [Uint32Vector, Uint32Array] as [NumericVectorConstructor<number, any>, any],
-    Float32Vector: [Float32Vector, Float32Array] as [NumericVectorConstructor<number, any>, any],
-    Float64Vector: [Float64Vector, Float64Array] as [NumericVectorConstructor<number, any>, any]
+    Int8Vector: [IntVector, Int8] as [typeof IntVector, typeof Int8],
+    Int16Vector: [IntVector, Int16] as [typeof IntVector, typeof Int16],
+    Int32Vector: [IntVector, Int32] as [typeof IntVector, typeof Int32],
+    Uint8Vector: [IntVector, Uint8] as [typeof IntVector, typeof Uint8],
+    Uint16Vector: [IntVector, Uint16] as [typeof IntVector, typeof Uint16],
+    Uint32Vector: [IntVector, Uint32] as [typeof IntVector, typeof Uint32],
+    Float32Vector: [FloatVector, Float32] as [typeof FloatVector, typeof Float32],
+    Float64Vector: [FloatVector, Float64] as [typeof FloatVector, typeof Float64],
 };
 
 const fixedSizeVectors = toMap(FixedSizeVectors, Object.keys(FixedSizeVectors));
@@ -61,9 +52,8 @@ const bytes = Array.from(
         () => Math.random() * 255 | 0));
 
 describe(`BoolVector`, () => {
-    const vector = new BoolVector({ data: new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]) });
-    const values = [true, true, false, true, true, false, false, false];
-    const n = values.length;
+    const values = [true, true, false, true, true, false, false, false], n = values.length;
+    const vector = new BoolVector(new BoolData(new Bool(), n, null, new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0])));
     test(`gets expected values`, () => {
         let i = -1;
         while (++i < n) {
@@ -78,7 +68,7 @@ describe(`BoolVector`, () => {
         }
     });
     test(`can set values to true and false`, () => {
-        const v = new BoolVector({ data: new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]) });
+        const v = new BoolVector(new BoolData(new Bool(), n, null, new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0])));
         const expected1 = [true, true, false, true, true, false, false, false];
         const expected2 = [true, true,  true, true, true, false, false, false];
         const expected3 = [true, true, false, false, false, false, true, true];
@@ -104,40 +94,43 @@ describe(`BoolVector`, () => {
         validate(expected1);
     });
     test(`packs 0 values`, () => {
-        expect(BoolVector.pack([])).toEqual(
+        expect(BoolVector.from([]).values).toEqual(
             new Uint8Array([0, 0, 0, 0, 0, 0, 0, 0]));
     });
     test(`packs 3 values`, () => {
-        expect(BoolVector.pack([
+        expect(BoolVector.from([
             true, false, true
-        ])).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
+        ]).values).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
     });
     test(`packs 8 values`, () => {
-        expect(BoolVector.pack([
+        expect(BoolVector.from([
             true, true, false, true, true, false, false, false
-        ])).toEqual(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]));
+        ]).values).toEqual(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]));
     });
     test(`packs 25 values`, () => {
-        expect(BoolVector.pack([
+        expect(BoolVector.from([
             true, true, false, true, true, false, false, false,
             false, false, false, true, true, false, true, true,
             false
-        ])).toEqual(new Uint8Array([27, 216, 0, 0, 0, 0, 0, 0]));
+        ]).values).toEqual(new Uint8Array([27, 216, 0, 0, 0, 0, 0, 0]));
     });
     test(`from with boolean Array packs values`, () => {
-        expect(new BoolVector({
-            data: BoolVector.pack([true, false, true])
-        }).slice()).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
+        expect(BoolVector
+            .from([true, false, true])
+            .slice().values
+        ).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0]));
     });
 });
 
 describe('Float16Vector', () => {
     const values = concatTyped(Uint16Array, ...bytes);
     const vector = bytes
-        .map((b) => new Float16Vector({ data: new Uint16Array(b.buffer) }))
+        .map((b) => new Uint16Array(b.buffer))
+        .map((b) => new FloatVector<type.Float16>(new FlatData(new Float16(), b.length, null, b)))
         .reduce((v: any, v2) => v.concat(v2));
     const n = values.length;
-    const clamp = (x: number) => Math.min((x -  32767) / 32767, 1);
+    const clamp = (x: number) => (x -  32767) / 32767;
+    const float16s = new Float32Array([...values].map((x) => clamp(x)));
     test(`gets expected values`, () => {
         let i = -1;
         while (++i < n) {
@@ -153,31 +146,32 @@ describe('Float16Vector', () => {
         }
     });
     test(`slices the entire array`, () => {
-        expect(vector.slice()).toEqual(values);
+        expect(vector.slice().toArray()).toEqual(float16s);
     });
     test(`slice returns a TypedArray`, () => {
-        expect(vector.slice()).toBeInstanceOf(Uint16Array);
+        expect(vector.slice().toArray()).toBeInstanceOf(Float32Array);
     });
     test(`slices from -20 to length`, () => {
-        expect(vector.slice(-20)).toEqual(values.slice(-20));
+        expect(vector.slice(-20).toArray()).toEqual(float16s.slice(-20));
     });
     test(`slices from 0 to -20`, () => {
-        expect(vector.slice(0, -20)).toEqual(values.slice(0, -20));
+        expect(vector.slice(0, -20).toArray()).toEqual(float16s.slice(0, -20));
     });
     test(`slices the array from 0 to length - 20`, () => {
-        expect(vector.slice(0, n - 20)).toEqual(values.slice(0, n - 20));
+        expect(vector.slice(0, n - 20).toArray()).toEqual(float16s.slice(0, n - 20));
     });
     test(`slices the array from 0 to length + 20`, () => {
-        expect(vector.slice(0, n + 20)).toEqual(
-            concatTyped(Uint16Array, values, values.slice(0, 20)));
+        expect(vector.slice(0, n + 20).toArray()).toEqual(float16s.slice(0, n + 20));
     });
 });
 
-for (const [VectorName, [VectorType, ArrayType]] of fixedSizeVectors) {
+for (const [VectorName, [VectorType, DataType]] of fixedSizeVectors) {
     describe(`${VectorName}`, () => {
-        const values = concatTyped(ArrayType, ...bytes);
+        const type = new DataType();
+        const values = concatTyped(type.ArrayType as any, ...bytes);
         const vector = bytes
-            .map((b) => new VectorType({ data: new ArrayType(b.buffer) }))
+            .map((b) => new type.ArrayType(b.buffer))
+            .map((b) => new VectorType(new FlatData(type, b.length * 0.5, null, b)))
             .reduce((v: any, v2) => v.concat(v2));
         const n = values.length * 0.5;
         test(`gets expected values`, () => {
@@ -194,32 +188,33 @@ for (const [VectorName, [VectorType, ArrayType]] of fixedSizeVectors) {
             }
         });
         test(`slices the entire array`, () => {
-            expect(vector.slice()).toEqual(values);
+            expect(vector.slice().toArray()).toEqual(values);
         });
         test(`slice returns a TypedArray`, () => {
-            expect(vector.slice()).toBeInstanceOf(ArrayType);
+            expect(vector.slice().toArray()).toBeInstanceOf(type.ArrayType);
         });
         test(`slices from -20 to length`, () => {
-            expect(vector.slice(-20)).toEqual(values.slice(-40));
+            expect(vector.slice(-20).toArray()).toEqual(values.slice(-40));
         });
         test(`slices from 0 to -20`, () => {
-            expect(vector.slice(0, -20)).toEqual(values.slice(0, -40));
+            expect(vector.slice(0, -20).toArray()).toEqual(values.slice(0, -40));
         });
         test(`slices the array from 0 to length - 20`, () => {
-            expect(vector.slice(0, n - 20)).toEqual(values.slice(0, values.length - 40));
+            expect(vector.slice(0, n - 20).toArray()).toEqual(values.slice(0, values.length - 40));
         });
         test(`slices the array from 0 to length + 20`, () => {
-            expect(vector.slice(0, n + 20)).toEqual(
-                concatTyped(ArrayType, values, values.slice(0, 40)));
+            expect(vector.slice(0, n + 20).toArray()).toEqual(values.slice(0, values.length + 40));
         });
     });
 }
 
-for (const [VectorName, [VectorType, ArrayType]] of fixedWidthVectors) {
+for (const [VectorName, [VectorType, DataType]] of fixedWidthVectors) {
     describe(`${VectorName}`, () => {
-        const values = concatTyped(ArrayType, ...bytes);
+        const type = new DataType();
+        const values = concatTyped(type.ArrayType as any, ...bytes);
         const vector = bytes
-            .map((b) => new VectorType({ data: new ArrayType(b.buffer) }))
+            .map((b) => new type.ArrayType(b.buffer))
+            .map((b) => new VectorType(new FlatData<any>(type, b.length, null, b)))
             .reduce((v: any, v2) => v.concat(v2));
 
         const n = values.length;
@@ -238,23 +233,22 @@ for (const [VectorName, [VectorType, ArrayType]] of fixedWidthVectors) {
             }
         });
         test(`slices the entire array`, () => {
-            expect(vector.slice()).toEqual(values);
+            expect(vector.slice().toArray()).toEqual(values);
         });
         test(`slice returns a TypedArray`, () => {
-            expect(vector.slice()).toBeInstanceOf(ArrayType);
+            expect(vector.slice().toArray()).toBeInstanceOf(type.ArrayType);
         });
         test(`slices from -20 to length`, () => {
-            expect(vector.slice(-20)).toEqual(values.slice(-20));
+            expect(vector.slice(-20).toArray()).toEqual(values.slice(-20));
         });
         test(`slices from 0 to -20`, () => {
-            expect(vector.slice(0, -20)).toEqual(values.slice(0, -20));
+            expect(vector.slice(0, -20).toArray()).toEqual(values.slice(0, -20));
         });
         test(`slices the array from 0 to length - 20`, () => {
-            expect(vector.slice(0, n - 20)).toEqual(values.slice(0, n - 20));
+            expect(vector.slice(0, n - 20).toArray()).toEqual(values.slice(0, n - 20));
         });
         test(`slices the array from 0 to length + 20`, () => {
-            expect(vector.slice(0, n + 20)).toEqual(
-                concatTyped(ArrayType, values, values.slice(0, 20)));
+            expect(vector.slice(0, n + 20).toArray()).toEqual(values.slice(0, n + 20));
         });
     });
 }

From 3098c1411930259070efb571fb350304b18ddc70 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Fri, 2 Feb 2018 00:07:03 -0500
Subject: [PATCH 1430/1644] ARROW-2052: [C++ / Python] Rework OwnedRef, remove
 ScopedRef

OwnedRef API cleaned up:
- doesn't try to take the GIL in its destructor anymore
- reset() decrefs the underlying pointer
- detach() unowns the pointer without decrefing it

Add OwnedRefNoGIL which forcefully takes the GIL in its destructor, at the expense of runtime performance.

Author: Antoine Pitrou <antoine@python.org>

Closes #1534 from pitrou/ARROW-2052-unify-ownedref-scopedref and squashes the following commits:

772c5c19 [Antoine Pitrou] ARROW-2052: [C++ / Python] Rework OwnedRef, remove ScopedRef
---
 cpp/src/arrow/python/arrow_to_pandas.cc |  6 +--
 cpp/src/arrow/python/arrow_to_python.cc | 38 +++++++++---------
 cpp/src/arrow/python/builtin_convert.cc |  2 +-
 cpp/src/arrow/python/common.h           | 52 ++++++++-----------------
 cpp/src/arrow/python/numpy_to_arrow.cc  | 20 +++++-----
 cpp/src/arrow/python/python_to_arrow.cc | 42 ++++++++++----------
 6 files changed, 71 insertions(+), 89 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 5c8c970e1e058..fcf05f833dbdf 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -265,13 +265,13 @@ class PandasBlock {
   int64_t num_rows_;
   int num_columns_;
 
-  OwnedRef block_arr_;
+  OwnedRefNoGIL block_arr_;
   uint8_t* block_data_;
 
   PandasOptions options_;
 
   // ndarray<int32>
-  OwnedRef placement_arr_;
+  OwnedRefNoGIL placement_arr_;
   int64_t* placement_data_;
 
  private:
@@ -1140,7 +1140,7 @@ class CategoricalBlock : public PandasBlock {
   }
 
   MemoryPool* pool_;
-  OwnedRef dictionary_;
+  OwnedRefNoGIL dictionary_;
   bool ordered_;
   bool needs_copy_;
 };
diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index c67e5410eb6ee..54a71d5a391d2 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -64,8 +64,8 @@ Status DeserializeDict(PyObject* context, const Array& array, int64_t start_idx,
                        int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                        PyObject** out) {
   const auto& data = static_cast<const StructArray&>(array);
-  ScopedRef keys, vals;
-  ScopedRef result(PyDict_New());
+  OwnedRef keys, vals;
+  OwnedRef result(PyDict_New());
   RETURN_IF_PYERROR();
 
   DCHECK_EQ(2, data.num_fields());
@@ -77,16 +77,16 @@ Status DeserializeDict(PyObject* context, const Array& array, int64_t start_idx,
   for (int64_t i = start_idx; i < stop_idx; ++i) {
     // PyDict_SetItem behaves differently from PyList_SetItem and PyTuple_SetItem.
     // The latter two steal references whereas PyDict_SetItem does not. So we need
-    // to make sure the reference count is decremented by letting the ScopedRef
+    // to make sure the reference count is decremented by letting the OwnedRef
     // go out of scope at the end.
-    PyDict_SetItem(result.get(), PyList_GET_ITEM(keys.get(), i - start_idx),
-                   PyList_GET_ITEM(vals.get(), i - start_idx));
+    PyDict_SetItem(result.obj(), PyList_GET_ITEM(keys.obj(), i - start_idx),
+                   PyList_GET_ITEM(vals.obj(), i - start_idx));
   }
   static PyObject* py_type = PyUnicode_FromString("_pytype_");
-  if (PyDict_Contains(result.get(), py_type)) {
-    RETURN_NOT_OK(CallDeserializeCallback(context, result.get(), out));
+  if (PyDict_Contains(result.obj(), py_type)) {
+    RETURN_NOT_OK(CallDeserializeCallback(context, result.obj(), out));
   } else {
-    *out = result.release();
+    *out = result.detach();
   }
   return Status::OK();
 }
@@ -96,10 +96,10 @@ Status DeserializeArray(const Array& array, int64_t offset, PyObject* base,
   int32_t index = static_cast<const Int32Array&>(array).Value(offset);
   RETURN_NOT_OK(py::TensorToNdarray(*blobs.tensors[index], base, out));
   // Mark the array as immutable
-  ScopedRef flags(PyObject_GetAttrString(*out, "flags"));
-  DCHECK(flags.get() != NULL) << "Could not mark Numpy array immutable";
+  OwnedRef flags(PyObject_GetAttrString(*out, "flags"));
+  DCHECK(flags.obj() != NULL) << "Could not mark Numpy array immutable";
   Py_INCREF(Py_False);
-  int flag_set = PyObject_SetAttrString(flags.get(), "writeable", Py_False);
+  int flag_set = PyObject_SetAttrString(flags.obj(), "writeable", Py_False);
   DCHECK(flag_set == 0) << "Could not mark Numpy array immutable";
   return Status::OK();
 }
@@ -184,23 +184,23 @@ Status GetValue(PyObject* context, const UnionArray& parent, const Array& arr,
 
 #define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN)                                     \
   const auto& data = static_cast<const UnionArray&>(array);                              \
-  ScopedRef result(CREATE_FN(stop_idx - start_idx));                                     \
+  OwnedRef result(CREATE_FN(stop_idx - start_idx));                                      \
   const uint8_t* type_ids = data.raw_type_ids();                                         \
   const int32_t* value_offsets = data.raw_value_offsets();                               \
   for (int64_t i = start_idx; i < stop_idx; ++i) {                                       \
     if (data.IsNull(i)) {                                                                \
       Py_INCREF(Py_None);                                                                \
-      SET_ITEM_FN(result.get(), i - start_idx, Py_None);                                 \
+      SET_ITEM_FN(result.obj(), i - start_idx, Py_None);                                 \
     } else {                                                                             \
       int64_t offset = value_offsets[i];                                                 \
       uint8_t type = type_ids[i];                                                        \
       PyObject* value;                                                                   \
       RETURN_NOT_OK(GetValue(context, data, *data.UnsafeChild(type), offset, type, base, \
                              blobs, &value));                                            \
-      SET_ITEM_FN(result.get(), i - start_idx, value);                                   \
+      SET_ITEM_FN(result.obj(), i - start_idx, value);                                   \
     }                                                                                    \
   }                                                                                      \
-  *out = result.release();                                                               \
+  *out = result.detach();                                                                \
   return Status::OK()
 
 Status DeserializeList(PyObject* context, const Array& array, int64_t start_idx,
@@ -219,13 +219,13 @@ Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
                       int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs,
                       PyObject** out) {
   const auto& data = static_cast<const UnionArray&>(array);
-  ScopedRef result(PySet_New(nullptr));
+  OwnedRef result(PySet_New(nullptr));
   const uint8_t* type_ids = data.raw_type_ids();
   const int32_t* value_offsets = data.raw_value_offsets();
   for (int64_t i = start_idx; i < stop_idx; ++i) {
     if (data.IsNull(i)) {
       Py_INCREF(Py_None);
-      if (PySet_Add(result.get(), Py_None) < 0) {
+      if (PySet_Add(result.obj(), Py_None) < 0) {
         RETURN_IF_PYERROR();
       }
     } else {
@@ -234,12 +234,12 @@ Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx,
       PyObject* value;
       RETURN_NOT_OK(GetValue(context, data, *data.UnsafeChild(type), offset, type, base,
                              blobs, &value));
-      if (PySet_Add(result.get(), value) < 0) {
+      if (PySet_Add(result.obj(), value) < 0) {
         RETURN_IF_PYERROR();
       }
     }
   }
-  *out = result.release();
+  *out = result.detach();
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 1b3c101758eec..1e431c29f06a0 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -931,7 +931,7 @@ static Status ConvertPySequenceReal(PyObject* obj, int64_t size,
   PyAcquireGIL lock;
 
   PyObject* seq;
-  ScopedRef tmp_seq_nanny;
+  OwnedRef tmp_seq_nanny;
 
   std::shared_ptr<DataType> real_type;
 
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index 146864ffdc26e..b1e0888af63f6 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -61,59 +61,30 @@ class ARROW_EXPORT PyAcquireGIL {
 
 #define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
 
+// A RAII primitive that DECREFs the underlying PyObject* when it
+// goes out of scope.
 class ARROW_EXPORT OwnedRef {
  public:
   OwnedRef() : obj_(NULLPTR) {}
 
   explicit OwnedRef(PyObject* obj) : obj_(obj) {}
 
-  ~OwnedRef() {
-    PyAcquireGIL lock;
-    release();
-  }
+  ~OwnedRef() { reset(); }
 
   void reset(PyObject* obj) {
-    /// TODO(phillipc): Should we acquire the GIL here? It definitely needs to be
-    /// acquired,
-    /// but callers have probably already acquired it
     Py_XDECREF(obj_);
     obj_ = obj;
   }
 
-  void release() {
-    Py_XDECREF(obj_);
-    obj_ = NULLPTR;
-  }
-
-  PyObject* obj() const { return obj_; }
-
- private:
-  PyObject* obj_;
-};
-
-// This is different from OwnedRef in that it assumes that
-// the GIL is held by the caller and doesn't decrement the
-// reference count when release is called.
-class ARROW_EXPORT ScopedRef {
- public:
-  ScopedRef() : obj_(NULLPTR) {}
+  void reset() { reset(NULLPTR); }
 
-  explicit ScopedRef(PyObject* obj) : obj_(obj) {}
-
-  ~ScopedRef() { Py_XDECREF(obj_); }
-
-  void reset(PyObject* obj) {
-    Py_XDECREF(obj_);
-    obj_ = obj;
-  }
-
-  PyObject* release() {
+  PyObject* detach() {
     PyObject* result = obj_;
     obj_ = NULLPTR;
     return result;
   }
 
-  PyObject* get() const { return obj_; }
+  PyObject* obj() const { return obj_; }
 
   PyObject** ref() { return &obj_; }
 
@@ -121,6 +92,17 @@ class ARROW_EXPORT ScopedRef {
   PyObject* obj_;
 };
 
+// Same as OwnedRef, but ensures the GIL is taken when it goes out of scope.
+// This is for situations where the GIL is not always known to be held
+// (e.g. if it is released in the middle of a function for performance reasons)
+class ARROW_EXPORT OwnedRefNoGIL : public OwnedRef {
+ public:
+  ~OwnedRefNoGIL() {
+    PyAcquireGIL lock;
+    reset();
+  }
+};
+
 struct ARROW_EXPORT PyObjectStringify {
   OwnedRef tmp_obj;
   const char* bytes;
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index a1161fe32e100..1b1673bb88b4a 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -1116,10 +1116,10 @@ Status LoopPySequence(PyObject* sequence, T func) {
       }
     }
   } else if (PyObject_HasAttrString(sequence, "__iter__")) {
-    OwnedRef iter = OwnedRef(PyObject_GetIter(sequence));
+    OwnedRef iter(PyObject_GetIter(sequence));
     PyObject* item;
     while ((item = PyIter_Next(iter.obj()))) {
-      OwnedRef ref = OwnedRef(item);
+      OwnedRef ref(item);
       RETURN_NOT_OK(func(ref.obj()));
     }
   } else {
@@ -1149,11 +1149,11 @@ Status LoopPySequenceWithMasks(PyObject* sequence,
       }
     }
   } else if (PyObject_HasAttrString(sequence, "__iter__")) {
-    OwnedRef iter = OwnedRef(PyObject_GetIter(sequence));
+    OwnedRef iter(PyObject_GetIter(sequence));
     PyObject* item;
     int64_t i = 0;
     while ((item = PyIter_Next(iter.obj()))) {
-      OwnedRef ref = OwnedRef(item);
+      OwnedRef ref(item);
       RETURN_NOT_OK(func(ref.obj(), have_mask && mask_values[i]));
       i++;
     }
@@ -1476,20 +1476,20 @@ Status AppendUTF32(const char* data, int itemsize, int byteorder,
     }
   }
 
-  ScopedRef unicode_obj(PyUnicode_DecodeUTF32(data, actual_length * kNumPyUnicodeSize,
-                                              nullptr, &byteorder));
+  OwnedRef unicode_obj(PyUnicode_DecodeUTF32(data, actual_length * kNumPyUnicodeSize,
+                                             nullptr, &byteorder));
   RETURN_IF_PYERROR();
-  ScopedRef utf8_obj(PyUnicode_AsUTF8String(unicode_obj.get()));
-  if (utf8_obj.get() == NULL) {
+  OwnedRef utf8_obj(PyUnicode_AsUTF8String(unicode_obj.obj()));
+  if (utf8_obj.obj() == NULL) {
     PyErr_Clear();
     return Status::Invalid("failed converting UTF32 to UTF8");
   }
 
-  const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(utf8_obj.get()));
+  const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(utf8_obj.obj()));
   if (builder->value_data_length() + length > kBinaryMemoryLimit) {
     return Status::Invalid("Encoded string length exceeds maximum size (2GB)");
   }
-  return builder->Append(PyBytes_AS_STRING(utf8_obj.get()), length);
+  return builder->Append(PyBytes_AS_STRING(utf8_obj.obj()), length);
 }
 
 }  // namespace
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 253e9d9a7da30..6d4f64675eb9d 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -365,15 +365,15 @@ Status CallCustomCallback(PyObject* context, PyObject* method_name, PyObject* el
   *result = NULL;
   if (context == Py_None) {
     std::stringstream ss;
-    ScopedRef repr(PyObject_Repr(elem));
+    OwnedRef repr(PyObject_Repr(elem));
     RETURN_IF_PYERROR();
 #if PY_MAJOR_VERSION >= 3
-    ScopedRef ascii(PyUnicode_AsASCIIString(repr.get()));
+    OwnedRef ascii(PyUnicode_AsASCIIString(repr.obj()));
     RETURN_IF_PYERROR();
-    ss << "error while calling callback on " << PyBytes_AsString(ascii.get())
+    ss << "error while calling callback on " << PyBytes_AsString(ascii.obj())
        << ": handler not registered";
 #else
-    ss << "error while calling callback on " << PyString_AsString(repr.get())
+    ss << "error while calling callback on " << PyString_AsString(repr.obj())
        << ": handler not registered";
 #endif
     return Status::SerializationError(ss.str());
@@ -386,8 +386,8 @@ Status CallCustomCallback(PyObject* context, PyObject* method_name, PyObject* el
 
 Status CallSerializeCallback(PyObject* context, PyObject* value,
                              PyObject** serialized_object) {
-  ScopedRef method_name(PyUnicode_FromString("_serialize_callback"));
-  RETURN_NOT_OK(CallCustomCallback(context, method_name.get(), value, serialized_object));
+  OwnedRef method_name(PyUnicode_FromString("_serialize_callback"));
+  RETURN_NOT_OK(CallCustomCallback(context, method_name.obj(), value, serialized_object));
   if (!PyDict_Check(*serialized_object)) {
     return Status::TypeError("serialization callback must return a valid dictionary");
   }
@@ -396,8 +396,8 @@ Status CallSerializeCallback(PyObject* context, PyObject* value,
 
 Status CallDeserializeCallback(PyObject* context, PyObject* value,
                                PyObject** deserialized_object) {
-  ScopedRef method_name(PyUnicode_FromString("_deserialize_callback"));
-  return CallCustomCallback(context, method_name.get(), value, deserialized_object);
+  OwnedRef method_name(PyUnicode_FromString("_deserialize_callback"));
+  return CallCustomCallback(context, method_name.obj(), value, deserialized_object);
 }
 
 Status SerializeDict(PyObject* context, std::vector<PyObject*> dicts,
@@ -493,9 +493,9 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
 #if PY_MAJOR_VERSION >= 3
     char* data = PyUnicode_AsUTF8AndSize(elem, &size);
 #else
-    ScopedRef str(PyUnicode_AsUTF8String(elem));
-    char* data = PyString_AS_STRING(str.get());
-    size = PyString_GET_SIZE(str.get());
+    OwnedRef str(PyUnicode_AsUTF8String(elem));
+    char* data = PyString_AS_STRING(str.obj());
+    size = PyString_GET_SIZE(str.obj());
 #endif
     if (size > std::numeric_limits<int32_t>::max()) {
       return Status::Invalid("Cannot writes bytes over 2GB");
@@ -585,15 +585,15 @@ Status SerializeSequences(PyObject* context, std::vector<PyObject*> sequences,
   SequenceBuilder builder(nullptr);
   std::vector<PyObject*> sublists, subtuples, subdicts, subsets;
   for (const auto& sequence : sequences) {
-    ScopedRef iterator(PyObject_GetIter(sequence));
+    OwnedRef iterator(PyObject_GetIter(sequence));
     RETURN_IF_PYERROR();
-    ScopedRef item;
+    OwnedRef item;
     while (true) {
-      item.reset(PyIter_Next(iterator.get()));
-      if (!item.get()) {
+      item.reset(PyIter_Next(iterator.obj()));
+      if (!item.obj()) {
         break;
       }
-      RETURN_NOT_OK(Append(context, item.get(), &builder, &sublists, &subtuples,
+      RETURN_NOT_OK(Append(context, item.obj(), &builder, &sublists, &subtuples,
                            &subdicts, &subsets, blobs_out));
     }
   }
@@ -739,18 +739,18 @@ Status SerializedPyObject::WriteTo(io::OutputStream* dst) {
 Status SerializedPyObject::GetComponents(MemoryPool* memory_pool, PyObject** out) {
   PyAcquireGIL py_gil;
 
-  ScopedRef result(PyDict_New());
+  OwnedRef result(PyDict_New());
   PyObject* buffers = PyList_New(0);
 
   // TODO(wesm): Not sure how pedantic we need to be about checking the return
   // values of these functions. There are other places where we do not check
   // PyDict_SetItem/SetItemString return value, but these failures would be
   // quite esoteric
-  PyDict_SetItemString(result.get(), "num_tensors",
+  PyDict_SetItemString(result.obj(), "num_tensors",
                        PyLong_FromSize_t(this->tensors.size()));
-  PyDict_SetItemString(result.get(), "num_buffers",
+  PyDict_SetItemString(result.obj(), "num_buffers",
                        PyLong_FromSize_t(this->buffers.size()));
-  PyDict_SetItemString(result.get(), "data", buffers);
+  PyDict_SetItemString(result.obj(), "data", buffers);
   RETURN_IF_PYERROR();
 
   Py_DECREF(buffers);
@@ -792,7 +792,7 @@ Status SerializedPyObject::GetComponents(MemoryPool* memory_pool, PyObject** out
     RETURN_NOT_OK(PushBuffer(buf));
   }
 
-  *out = result.release();
+  *out = result.detach();
   return Status::OK();
 }
 

From a1fd79625aca5fd77eb987572bc45b8dc4a4871b Mon Sep 17 00:00:00 2001
From: moriyoshi <mozo@mozo.jp>
Date: Fri, 2 Feb 2018 10:27:37 +0100
Subject: [PATCH 1431/1644] ARROW-2050: [Python] Support `setup.py pytest`

* By using `pytest-runner`, the test dependencies can automatically be fetched before running test suite.

Author: moriyoshi <mozo@mozo.jp>

Closes #1527 from moriyoshi/moriyoshi/pytest-runner and squashes the following commits:

7b73105 [moriyoshi] Support setup.py pytest
---
 python/setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index cfc771fe870ab..726bb51f266ca 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -447,9 +447,9 @@ def parse_version(root):
         ]
     },
     use_scm_version={"root": "..", "relative_to": __file__, "parse": parse_version},
-    setup_requires=['setuptools_scm', 'cython >= 0.23'],
+    setup_requires=['setuptools_scm', 'cython >= 0.23', 'pytest-runner'],
     install_requires=install_requires,
-    tests_require=['pytest'],
+    tests_require=['pytest', 'pandas'],
     description="Python library for Apache Arrow",
     long_description=long_description,
     classifiers=[

From 5042863d88d02afa9ff791ef56aba11d90528aa5 Mon Sep 17 00:00:00 2001
From: Joris Van den Bossche <jorisvandenbossche@gmail.com>
Date: Fri, 2 Feb 2018 12:25:27 -0500
Subject: [PATCH 1432/1644] ARROW-1754: [Python] alternative fix for duplicate
 index/column name that preserves index name if available

Related to the discussion about the pandas metadata specification in https://github.com/pandas-dev/pandas/pull/18201, and an alternative to https://github.com/apache/arrow/pull/1271.

I don't open this PR because it should necessarily be merged, I just want to show that it is not that difficult to both fix [ARROW-1754](https://issues.apache.org/jira/browse/ARROW-1754) and preserve index names as field names when possible (as this was mentioned in https://github.com/pandas-dev/pandas/pull/18201 as the reason to make this change to not preserve index names).
The diff is partly a revert of https://github.com/apache/arrow/pull/1271, but then adapted to the current codebase.

Main reasons I prefer to preserve index names: 1) usability in pyarrow itself (if you would want to work with pyarrow Tables created from pandas) and 2) when interchanging parquet files with other people / other non-pandas systems, then it would be much nicer to not have `__index_level_n__` column names if possible.

Author: Joris Van den Bossche <jorisvandenbossche@gmail.com>

Closes #1408 from jorisvandenbossche/index-names and squashes the following commits:

eef1d334 [Joris Van den Bossche] alternative fix for duplicate index/column name that preserves index name if available
---
 python/pyarrow/pandas_compat.py             | 45 ++++++++++++++-------
 python/pyarrow/tests/test_convert_pandas.py |  5 ++-
 2 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 4a30fb3b44a4e..240cccdaffe56 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -179,10 +179,8 @@ def get_column_metadata(column, name, arrow_type, field_name):
     }
 
 
-index_level_name = '__index_level_{:d}__'.format
-
-
-def construct_metadata(df, column_names, index_levels, preserve_index, types):
+def construct_metadata(df, column_names, index_levels, index_column_names,
+                       preserve_index, types):
     """Returns a dictionary containing enough metadata to reconstruct a pandas
     DataFrame as an Arrow Table, including index columns.
 
@@ -197,9 +195,8 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
     -------
     dict
     """
-    ncolumns = len(column_names)
-    df_types = types[:ncolumns - len(index_levels)]
-    index_types = types[ncolumns - len(index_levels):]
+    df_types = types[:-len(index_levels)]
+    index_types = types[-len(index_levels):]
 
     column_metadata = [
         get_column_metadata(
@@ -213,9 +210,6 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types):
     ]
 
     if preserve_index:
-        index_column_names = list(map(
-            index_level_name, range(len(index_levels))
-        ))
         index_column_metadata = [
             get_column_metadata(
                 level,
@@ -294,9 +288,29 @@ def _column_name_to_strings(name):
     return str(name)
 
 
+def _index_level_name(index, i, column_names):
+    """Return the name of an index level or a default name if `index.name` is
+    None or is already a column name.
+
+    Parameters
+    ----------
+    index : pandas.Index
+    i : int
+
+    Returns
+    -------
+    name : str
+    """
+    if index.name is not None and index.name not in column_names:
+        return index.name
+    else:
+        return '__index_level_{:d}__'.format(i)
+
+
 def dataframe_to_arrays(df, schema, preserve_index, nthreads=1):
-    names = []
+    column_names = []
     index_columns = []
+    index_column_names = []
     type = None
 
     if preserve_index:
@@ -324,12 +338,13 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1):
 
         columns_to_convert.append(col)
         convert_types.append(type)
-        names.append(name)
+        column_names.append(name)
 
     for i, column in enumerate(index_columns):
         columns_to_convert.append(column)
         convert_types.append(None)
-        names.append(index_level_name(i))
+        name = _index_level_name(column, i, column_names)
+        index_column_names.append(name)
 
     # NOTE(wesm): If nthreads=None, then we use a heuristic to decide whether
     # using a thread pool is worth it. Currently the heuristic is whether the
@@ -358,8 +373,10 @@ def convert_column(col, ty):
     types = [x.type for x in arrays]
 
     metadata = construct_metadata(
-        df, names, index_columns, preserve_index, types
+        df, column_names, index_columns, index_column_names, preserve_index,
+        types
     )
+    names = column_names + index_column_names
     return names, arrays, metadata
 
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index ca2f1e3611963..f1f40a695edc1 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -191,8 +191,9 @@ def test_index_metadata_field_name(self):
         assert idx0['field_name'] == idx0_name
         assert idx0['name'] is None
 
-        assert foo_name == '__index_level_1__'
-        assert foo['name'] == 'foo'
+        assert foo_name == 'foo'
+        assert foo['field_name'] == foo_name
+        assert foo['name'] == foo_name
 
     def test_categorical_column_index(self):
         df = pd.DataFrame(

From 2e9f7dda2bf207d75a638749233b064fa049292a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 2 Feb 2018 12:27:22 -0500
Subject: [PATCH 1433/1644] ARROW-1706: [Python] Coerce array inputs to
 StructArray.from_arrays. Flip order of arguments

I flipped the argument order to be more consistent with the same methods in RecordBatch, Table. The StructArray method doesn't seem to be widely used so I'm not sure there's the need to go through a deprecation cycle

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1512 from wesm/ARROW-1706 and squashes the following commits:

786c37d0 [Wes McKinney] Raise error when names is None in StructArray.from_arrays
990dda57 [Wes McKinney] Fix API change
2053d941 [Wes McKinney] Add test case
9c229498 [Wes McKinney] Flip order of arguments to StructArray.from_arrays, try to coerce non-pyarrow data to Array
---
 python/pyarrow/array.pxi                     | 24 +++++++++++++++--
 python/pyarrow/tests/test_convert_builtin.py | 28 ++++++++++++++++++--
 python/pyarrow/tests/test_convert_pandas.py  |  4 +--
 3 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index caeefd2ff4f6a..9b6ae0fc634e1 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -828,8 +828,23 @@ cdef class DictionaryArray(Array):
 
 
 cdef class StructArray(Array):
+
     @staticmethod
-    def from_arrays(field_names, arrays):
+    def from_arrays(arrays, names=None):
+        """
+        Construct StructArray from collection of arrays representing each field
+        in the struct
+
+        Parameters
+        ----------
+        arrays : sequence of Array
+        names : List[str]
+            Field names
+
+        Returns
+        -------
+        result : StructArray
+        """
         cdef:
             Array array
             shared_ptr[CArray] c_array
@@ -839,6 +854,11 @@ cdef class StructArray(Array):
             ssize_t length
             ssize_t i
 
+        if names is None:
+            raise ValueError('Names are currently required')
+
+        arrays = [asarray(x) for x in arrays]
+
         num_arrays = len(arrays)
         if num_arrays == 0:
             raise ValueError("arrays list is empty")
@@ -855,7 +875,7 @@ cdef class StructArray(Array):
         cdef DataType struct_type = struct([
             field(name, array.type)
             for name, array in
-            zip(field_names, arrays)
+            zip(names, arrays)
         ])
 
         c_result.reset(new CStructArray(struct_type.sp_type, length, c_arrays))
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index bbdf6e71e0f1d..ce54f23eb0c6b 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -493,8 +493,8 @@ def test_structarray():
     strs = pa.array([u'a', None, u'c'], type=pa.string())
     bools = pa.array([True, False, None], type=pa.bool_())
     arr = pa.StructArray.from_arrays(
-        ['ints', 'strs', 'bools'],
-        [ints, strs, bools])
+        [ints, strs, bools],
+        ['ints', 'strs', 'bools'])
 
     expected = [
         {'ints': None, 'strs': u'a', 'bools': True},
@@ -529,3 +529,27 @@ def test_struct_from_dicts():
                 {'a': None, 'b': None, 'c': None},
                 {'a': None, 'b': 'bar', 'c': None}]
     assert arr.to_pylist() == expected
+
+
+def test_structarray_from_arrays_coerce():
+    # ARROW-1706
+    ints = [None, 2, 3]
+    strs = [u'a', None, u'c']
+    bools = [True, False, None]
+    ints_nonnull = [1, 2, 3]
+
+    arrays = [ints, strs, bools, ints_nonnull]
+    result = pa.StructArray.from_arrays(arrays,
+                                        ['ints', 'strs', 'bools',
+                                         'int_nonnull'])
+    expected = pa.StructArray.from_arrays(
+        [pa.array(ints, type='int64'),
+         pa.array(strs, type='utf8'),
+         pa.array(bools),
+         pa.array(ints_nonnull, type='int64')],
+        ['ints', 'strs', 'bools', 'int_nonnull'])
+
+    with pytest.raises(ValueError):
+        pa.StructArray.from_arrays(arrays)
+
+    assert result.equals(expected)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index f1f40a695edc1..31099072f29d2 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1154,8 +1154,8 @@ def test_structarray(self):
         strs = pa.array([u'a', None, u'c'], type=pa.string())
         bools = pa.array([True, False, None], type=pa.bool_())
         arr = pa.StructArray.from_arrays(
-            ['ints', 'strs', 'bools'],
-            [ints, strs, bools])
+            [ints, strs, bools],
+            ['ints', 'strs', 'bools'])
 
         expected = pd.Series([
             {'ints': None, 'strs': u'a', 'bools': True},

From e26f3dad3675288564ef0c0330a5c9afcac652f1 Mon Sep 17 00:00:00 2001
From: Jim Crist <jiminy.crist@gmail.com>
Date: Sat, 3 Feb 2018 16:54:23 -0500
Subject: [PATCH 1434/1644] ARROW-2085: [Python] HadoopFileSystem.isdir/.isfile
 return False on missing paths

Return `False` instead of erroring if the path is missing to match the behavior of the python standard library.

Also changes the default fixture settings to rely on the default configuration detection provided by libhdfs. This allows the tests to run fine with no extra environment variables (at least on my system).

Author: Jim Crist <jiminy.crist@gmail.com>

Closes #1552 from jcrist/isdir-isfile-dont-error and squashes the following commits:

59ab4568 [Jim Crist] HadoopFileSystem.isdir/.isfile don't error
---
 python/pyarrow/io-hdfs.pxi        | 10 ++++++++--
 python/pyarrow/tests/test_hdfs.py | 25 +++++++++++++++++++++++--
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi
index 83b14b687830d..dc6ba23abc9c1 100644
--- a/python/pyarrow/io-hdfs.pxi
+++ b/python/pyarrow/io-hdfs.pxi
@@ -137,12 +137,18 @@ cdef class HadoopFileSystem:
 
     def isdir(self, path):
         cdef HdfsPathInfo info
-        self._path_info(path, &info)
+        try:
+            self._path_info(path, &info)
+        except ArrowIOError:
+            return False
         return info.kind == ObjectType_DIRECTORY
 
     def isfile(self, path):
         cdef HdfsPathInfo info
-        self._path_info(path, &info)
+        try:
+            self._path_info(path, &info)
+        except ArrowIOError:
+            return False
         return info.kind == ObjectType_FILE
 
     def get_capacity(self):
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index b62458cd73689..885272ba8bdf5 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -36,10 +36,10 @@
 
 
 def hdfs_test_client(driver='libhdfs'):
-    host = os.environ.get('ARROW_HDFS_TEST_HOST', 'localhost')
+    host = os.environ.get('ARROW_HDFS_TEST_HOST', 'default')
     user = os.environ.get('ARROW_HDFS_TEST_USER', None)
     try:
-        port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 20500))
+        port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 0))
     except ValueError:
         raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
                          'an integer')
@@ -162,6 +162,27 @@ def test_info(self):
         assert file_path_info['kind'] == 'file'
         assert file_path_info['size'] == len(data)
 
+    def test_exists_isdir_isfile(self):
+        dir_path = pjoin(self.tmp_path, 'info-base')
+        file_path = pjoin(dir_path, 'ex')
+        missing_path = pjoin(dir_path, 'this-path-is-missing')
+
+        self.hdfs.mkdir(dir_path)
+        with self.hdfs.open(file_path, 'wb') as f:
+            f.write(b'foobarbaz')
+
+        assert self.hdfs.exists(dir_path)
+        assert self.hdfs.exists(file_path)
+        assert not self.hdfs.exists(missing_path)
+
+        assert self.hdfs.isdir(dir_path)
+        assert not self.hdfs.isdir(file_path)
+        assert not self.hdfs.isdir(missing_path)
+
+        assert not self.hdfs.isfile(dir_path)
+        assert self.hdfs.isfile(file_path)
+        assert not self.hdfs.isfile(missing_path)
+
     def test_disk_usage(self):
         path = pjoin(self.tmp_path, 'disk-usage-base')
         p1 = pjoin(path, 'p1')

From d949ee6cce7de52f0f87dc1a78e45accc4139c46 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 4 Feb 2018 10:13:46 +0100
Subject: [PATCH 1435/1644] ARROW-2084: [C++] Support newer Brotli static
 library names

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1554 from cpcloud/ARROW-2084 and squashes the following commits:

3361f13 [Uwe L. Korn] ARROW-2084: [C++] Support newer Brotli static library names
---
 cpp/CMakeLists.txt                 |  4 ++--
 cpp/apidoc/Windows.md              |  6 ++++--
 cpp/cmake_modules/FindBrotli.cmake | 33 ++++++++++++++++++++----------
 3 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0558cba495a83..38a4d772b8f39 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -204,8 +204,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     ON)
 
   if (MSVC)
-    set(BROTLI_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
-      "Brotli static lib suffix used on Windows with MSVC (default _static)")
+    set(BROTLI_MSVC_STATIC_LIB_SUFFIX "-static" CACHE STRING
+      "Brotli static lib suffix used on Windows with MSVC (default -static)")
     set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Snappy static lib suffix used on Windows with MSVC (default is empty string)")
     set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING
diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index aa3d31f1f7b90..dae5040c24d38 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -95,9 +95,11 @@ party static libs.
 build. Set `ZLIB_HOME` environment variable. Pass
 `-DZLIB_MSVC_STATIC_LIB_SUFFIX=%ZLIB_SUFFIX%` to link with z%ZLIB_SUFFIX%.lib
 
-`brotli`. Set `BROTLY_HOME` environment variable. Pass
+`brotli`. Set `BROTLI_HOME` environment variable. Pass
 `-DBROTLI_MSVC_STATIC_LIB_SUFFIX=%BROTLI_SUFFIX%` to link with
-brotli*%BROTLI_SUFFIX%.lib.
+brotli*%BROTLI_SUFFIX%.lib. For brotli versions <= 0.6.0 installed from
+conda-forge this must be set to `_static`, otherwise the default of `-static`
+is used.
 
 `snappy`. Set `SNAPPY_HOME` environment variable. Pass
 `-DSNAPPY_MSVC_STATIC_LIB_SUFFIX=%SNAPPY_SUFFIX%` to link with
diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake
index f2e714c6fc870..ea971f09ed8f4 100644
--- a/cpp/cmake_modules/FindBrotli.cmake
+++ b/cpp/cmake_modules/FindBrotli.cmake
@@ -43,17 +43,17 @@ find_path( BROTLI_INCLUDE_DIR NAMES brotli/decode.h
   NO_DEFAULT_PATH
   PATH_SUFFIXES "include" )
 
-find_library( BROTLI_LIBRARY_ENC NAMES libbrotlienc.a brotlienc
+find_library( BROTLI_LIBRARY_ENC NAMES libbrotlienc.a libbrotlienc-static.a brotlienc
   PATHS ${_brotli_roots}
   NO_DEFAULT_PATH
   PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
 
-find_library( BROTLI_LIBRARY_DEC NAMES libbrotlidec.a brotlidec
+find_library( BROTLI_LIBRARY_DEC NAMES libbrotlidec.a libbrotlidec-static.a brotlidec
   PATHS ${_brotli_roots}
   NO_DEFAULT_PATH
   PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
 
-find_library( BROTLI_LIBRARY_COMMON NAMES libbrotlicommon.a brotlicommon
+find_library( BROTLI_LIBRARY_COMMON NAMES libbrotlicommon.a libbrotlicommon-static.a brotlicommon
   PATHS ${_brotli_roots}
   NO_DEFAULT_PATH
   PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
@@ -65,16 +65,27 @@ if (BROTLI_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR BROTLI_LIBRARIES))
   set(BROTLI_FOUND TRUE)
   get_filename_component( BROTLI_LIBS ${BROTLI_LIBRARY_ENC} PATH )
   set(BROTLI_LIB_NAME brotli)
-  if (MSVC AND NOT BROTLI_MSVC_STATIC_LIB_SUFFIX)
-    set(BROTLI_MSVC_STATIC_LIB_SUFFIX _static)
+  if (BROTLI_MSVC_STATIC_LIB_SUFFIX)
+    set(BROTLI_STATIC_LIB_SUFFIX "${BROTLI_MSVC_STATIC_LIB_SUFFIX}")
+  endif()
+  if (NOT BROTLI_STATIC_LIB_SUFFIX)
+    if (EXISTS "${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc-static${CMAKE_STATIC_LIBRARY_SUFFIX}")
+      set(BROTLI_STATIC_LIB_SUFFIX -static)
+    else()
+      if (MSVC)
+        set(BROTLI_STATIC_LIB_SUFFIX _static)
+      else()
+        set(BROTLI_STATIC_LIB_SUFFIX "")
+      endif()
+    endif()
   endif()
   set(BROTLI_STATIC_LIB
-      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
-      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
-      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
-  set(BROTLI_STATIC_LIBRARY_ENC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
-  set(BROTLI_STATIC_LIBRARY_DEC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
-  set(BROTLI_STATIC_LIBRARY_COMMON ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
+      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
+      ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(BROTLI_STATIC_LIBRARY_ENC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(BROTLI_STATIC_LIBRARY_DEC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(BROTLI_STATIC_LIBRARY_COMMON ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
   set(BROTLI_SHARED_LIB
       ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${CMAKE_SHARED_LIBRARY_SUFFIX}
       ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${CMAKE_SHARED_LIBRARY_SUFFIX}

From febc19d840ba935101dbd9fccfe92ab22e27da9b Mon Sep 17 00:00:00 2001
From: Marco Neumann <marco.neumann@blue-yonder.com>
Date: Sun, 4 Feb 2018 10:18:44 +0100
Subject: [PATCH 1436/1644] ARROW-1589: [C++] Fuzzing for certain input formats

See https://issues.apache.org/jira/browse/ARROW-1589

This only implements the fuzzer for a single input path, but since it also sets up the fuzzing infrastructure, it should be easy to add more tests (feel free to propose some of them to be included in this PR as well).

Author: Marco Neumann <marco.neumann@blue-yonder.com>

Closes #1529 from crepererum/ARROW-1589 and squashes the following commits:

1f76392 [Marco Neumann] add fuzzing docs
312d8d2 [Marco Neumann] add ipc-fuzzing-test
25a9273 [Marco Neumann] add ARROW_USE_COVERAGE
---
 cpp/CMakeLists.txt                    | 11 +++++++
 cpp/README.md                         | 34 +++++++++++++++++++++
 cpp/cmake_modules/BuildUtils.cmake    | 31 +++++++++++++++++++
 cpp/cmake_modules/san-config.cmake    | 10 ++++++
 cpp/src/arrow/ipc/CMakeLists.txt      |  2 ++
 cpp/src/arrow/ipc/ipc-fuzzing-test.cc | 44 +++++++++++++++++++++++++++
 6 files changed, 132 insertions(+)
 create mode 100644 cpp/src/arrow/ipc/ipc-fuzzing-test.cc

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 38a4d772b8f39..f07ef6b23ef39 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -147,6 +147,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow CPython extensions"
     OFF)
 
+  option(ARROW_FUZZING
+    "Build Arrow Fuzzing executables"
+    OFF)
+
   option(ARROW_SSE3
     "Build Arrow with SSE3"
     ON)
@@ -249,6 +253,10 @@ if(NOT ARROW_BUILD_BENCHMARKS)
   set(NO_BENCHMARKS 1)
 endif()
 
+if (NOT ARROW_FUZZING)
+  set(NO_FUZZING 1)
+endif()
+
 if(ARROW_HDFS)
   set(ARROW_BOOST_HEADER_ONLY 0)
 else()
@@ -300,6 +308,9 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
 endif()
 
 # ASAN / TSAN / UBSAN
+if(ARROW_FUZZING)
+    set(ARROW_USE_COVERAGE ON)
+endif()
 include(san-config)
 
 # For any C code, use the same flags.
diff --git a/cpp/README.md b/cpp/README.md
index 52169974de41e..1daf863819cbc 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -99,6 +99,40 @@ and benchmarks or `make runbenchmark` to run only the benchmark tests.
 
 Benchmark logs will be placed in the build directory under `build/benchmark-logs`.
 
+## Building/Running fuzzers
+
+Fuzzers can help finding unhandled exceptions and problems with untrusted input that
+may lead to crashes, security issues and undefined behavior. They do this by
+generating random input data and observing the behavior of the executed code. To build
+the fuzzer code, LLVM is required (GCC-based compilers won't work). You can build them
+using the following code:
+
+    cmake -DARROW_FUZZING=ON -DARROW_USE_ASAN=ON ..
+
+`ARROW_FUZZING` will enable building of fuzzer executables as well as enable the
+addition of coverage helpers via `ARROW_USE_COVERAGE`, so that the fuzzer can observe
+the program execution.
+
+It is also wise to enable some sanitizers like `ARROW_USE_ASAN` (see above), which
+activates the address sanitizer. This way, we ensure that bad memory operations
+provoked by the fuzzer will be found early. You may also enable other sanitizers as
+well. Just keep in mind that some of them do not work together and some may result
+in very long execution times, which will slow down the fuzzing procedure.
+
+Now you can start one of the fuzzer, e.g.:
+
+    ./debug/debug/ipc-fuzzing-test
+
+This will try to find a malformed input that crashes the payload and will show the
+stack trace as well as the input data. After a problem was found this way, it should
+be reported and fixed. Usually, the fuzzing process cannot be continued until the
+fix is applied, since the fuzzer usually converts to the problem again.
+
+There are some problems that may occur during the compilation process:
+
+- libfuzzer was not distributed with your LLVM: `ld: file not found: .../libLLVMFuzzer.a`
+- your LLVM is too old: `clang: error: unsupported argument 'fuzzer' to option 'fsanitize='`
+
 ### Third-party environment variables
 
 To set up your own specific build toolchain, here are the relevant environment
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index 207bb9aede0c5..f8c0f5ed23774 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -347,3 +347,34 @@ function(ARROW_TEST_LINK_LIBRARIES REL_TEST_NAME)
 
   target_link_libraries(${TEST_NAME} ${ARGN})
 endfunction()
+
+
+############################################################
+# Fuzzing
+############################################################
+# Add new fuzzing test executable.
+#
+# The single source file must define a function:
+#   extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+#
+# No main function must be present within the source file!
+#
+function(ADD_ARROW_FUZZING REL_FUZZING_NAME)
+  if(NO_FUZZING)
+    return()
+  endif()
+
+  if (ARROW_BUILD_STATIC)
+    set(FUZZ_LINK_LIBS arrow_static)
+  else()
+    set(FUZZ_LINK_LIBS arrow_shared)
+  endif()
+
+  add_executable(${REL_FUZZING_NAME} "${REL_FUZZING_NAME}.cc")
+  target_link_libraries(${REL_FUZZING_NAME} ${FUZZ_LINK_LIBS})
+  target_compile_options(${REL_FUZZING_NAME}
+      PRIVATE "-fsanitize=fuzzer")
+  set_target_properties(${REL_FUZZING_NAME}
+      PROPERTIES
+      LINK_FLAGS "-fsanitize=fuzzer")
+endfunction()
diff --git a/cpp/cmake_modules/san-config.cmake b/cpp/cmake_modules/san-config.cmake
index 1917eabe8b4b2..f2de9cf1f7553 100644
--- a/cpp/cmake_modules/san-config.cmake
+++ b/cpp/cmake_modules/san-config.cmake
@@ -87,6 +87,16 @@ if (${ARROW_USE_TSAN})
 endif()
 
 
+if (${ARROW_USE_COVERAGE})
+  if(NOT ("${COMPILER_FAMILY}" STREQUAL "clang"))
+    message(SEND_ERROR "You can only enable coverage with clang")
+  endif()
+  add_definitions("-fsanitize-coverage=trace-pc-guard")
+
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize-coverage=trace-pc-guard")
+endif()
+
+
 if ("${ARROW_USE_UBSAN}" OR "${ARROW_USE_ASAN}" OR "${ARROW_USE_TSAN}")
   # GCC 4.8 and 4.9 (latest as of this writing) don't allow you to specify a
   # sanitizer blacklist.
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index a24349fd6c928..2976a15ce27b2 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -117,3 +117,5 @@ if (ARROW_BUILD_UTILITIES)
 endif()
 
 ADD_ARROW_BENCHMARK(ipc-read-write-benchmark)
+
+ADD_ARROW_FUZZING(ipc-fuzzing-test)
diff --git a/cpp/src/arrow/ipc/ipc-fuzzing-test.cc b/cpp/src/arrow/ipc/ipc-fuzzing-test.cc
new file mode 100644
index 0000000000000..83fe94b139bd3
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-fuzzing-test.cc
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+
+#include <arrow/buffer.h>
+#include <arrow/io/memory.h>
+#include <arrow/ipc/reader.h>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  arrow::Status status;
+
+  auto buffer = std::make_shared<arrow::Buffer>(data, size);
+  arrow::io::BufferReader buffer_reader(buffer);
+
+  std::shared_ptr<arrow::ipc::RecordBatchReader> batch_reader;
+  status = arrow::ipc::RecordBatchStreamReader::Open(&buffer_reader, &batch_reader);
+  if (!status.ok()) {
+    return 0;
+  }
+
+  std::shared_ptr<arrow::RecordBatch> batch;
+  do {
+    status = batch_reader->ReadNext(&batch);
+    if (!status.ok()) {
+      return 0;
+    }
+  } while (batch);
+  return 0;
+}

From 416d56593f4e324b326ba9096e86253f130125bf Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Sun, 4 Feb 2018 19:53:18 +0100
Subject: [PATCH 1437/1644] ARROW-2071: [Python] Lighten serialization tests

Attempt to reduce running times on Travis-CI.

Note this PR integrates the contents of PR #1548 to get the builds running at all.

Author: Antoine Pitrou <antoine@python.org>

Closes #1550 from pitrou/ARROW-2071-lighten-serialization-tests and squashes the following commits:

231ef3b [Antoine Pitrou] ARROW-2071: [Python] Lighten serialization tests
---
 python/manylinux1/build_arrow.sh           |  2 +-
 python/pyarrow/tests/test_serialization.py | 20 +++++++++++---------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 4d816bec9b6a4..6bed451d2d667 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -81,7 +81,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     source /venv-test-${PYTHON}/bin/activate
     pip install repaired_wheels/*.whl
 
-    py.test --parquet /venv-test-${PYTHON}/lib/*/site-packages/pyarrow -v
+    py.test -v -r sxX --durations=15 --parquet /venv-test-${PYTHON}/lib/*/site-packages/pyarrow
     deactivate
 
     mv repaired_wheels/*.whl /io/dist
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 23059e12d8be5..3b1324beb999b 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -99,7 +99,7 @@ def assert_equal(obj1, obj2):
 
 
 PRIMITIVE_OBJECTS = [
-    0, 0.0, 0.9, 1 << 62, 1 << 100, 1 << 999,
+    0, 0.0, 0.9, 1 << 62, 1 << 999,
     [1 << 100, [1 << 100]], "a", string.printable, "\u262F",
     "hello world", u"hello world", u"\xff\xfe\x9c\x001\x000\x00",
     None, True, False, [], (), {}, {(1, 2): 1}, {(): 2},
@@ -110,10 +110,11 @@ def assert_equal(obj1, obj2):
     {"hello": set([2, 3]), "world": set([42.0]), "this": None},
     np.int8(3), np.int32(4), np.int64(5),
     np.uint8(3), np.uint32(4), np.uint64(5), np.float16(1.9), np.float32(1.9),
-    np.float64(1.9), np.zeros([100, 100]),
-    np.random.normal(size=[100, 100]), np.array(["hi", 3]),
+    np.float64(1.9), np.zeros([8, 20]),
+    np.random.normal(size=[17, 10]), np.array(["hi", 3]),
     np.array(["hi", 3], dtype=object),
-    np.random.normal(size=[45, 22]).T]
+    np.random.normal(size=[15, 13]).T,
+]
 
 
 if sys.version_info >= (3, 0):
@@ -126,11 +127,12 @@ def assert_equal(obj1, obj2):
 
 COMPLEX_OBJECTS = [
     [[[[[[[[[[[[]]]]]]]]]]]],
-    {"obj{}".format(i): np.random.normal(size=[100, 100]) for i in range(10)},
+    {"obj{}".format(i): np.random.normal(size=[4, 4]) for i in range(5)},
     # {(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {
     #       (): {(): {}}}}}}}}}}}}},
     ((((((((((),),),),),),),),),),
-    {"a": {"b": {"c": {"d": {}}}}}]
+    {"a": {"b": {"c": {"d": {}}}}},
+]
 
 
 class Foo(object):
@@ -146,7 +148,7 @@ def __eq__(self, other):
 
 class Bar(object):
     def __init__(self):
-        for i, val in enumerate(PRIMITIVE_OBJECTS + COMPLEX_OBJECTS):
+        for i, val in enumerate(COMPLEX_OBJECTS):
             setattr(self, "field{}".format(i), val)
 
 
@@ -161,7 +163,7 @@ def method(self, arg):
 
 class Qux(object):
     def __init__(self):
-        self.objs = [Foo(), Bar(), Baz()]
+        self.objs = [Foo(1), Foo(42)]
 
 
 class SubQux(Qux):
@@ -231,7 +233,7 @@ def _check_component_roundtrip(value, context=global_serialization_context):
 
 
 @pytest.yield_fixture(scope='session')
-def large_buffer(size=100*1024*1024):
+def large_buffer(size=32*1024*1024):
     return pa.allocate_buffer(size)
 
 

From de850b9b86139e73f573bc61a9cb5f00a2608851 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 4 Feb 2018 19:55:35 +0100
Subject: [PATCH 1438/1644] ARROW-2089: [GLib] Rename to GARROW_TYPE_BOOLEAN
 for consistency

Array name and data type name use "boolean" not "bool".
GArrowType only uses "bool".

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1557 from kou/glib-use-boolean and squashes the following commits:

aa1316f [Kouhei Sutou] [GLib] Rename to GARROW_TYPE_BOOLEAN for consistency
---
 c_glib/arrow-glib/type.cpp            | 2 +-
 c_glib/arrow-glib/type.h              | 4 ++--
 c_glib/test/test-array.rb             | 2 +-
 c_glib/test/test-boolean-data-type.rb | 2 +-
 c_glib/test/test-chunked-array.rb     | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/c_glib/arrow-glib/type.cpp b/c_glib/arrow-glib/type.cpp
index 68ebc965440f1..0642004e2f07b 100644
--- a/c_glib/arrow-glib/type.cpp
+++ b/c_glib/arrow-glib/type.cpp
@@ -39,7 +39,7 @@ garrow_type_from_raw(arrow::Type::type type)
   case arrow::Type::type::NA:
     return GARROW_TYPE_NA;
   case arrow::Type::type::BOOL:
-    return GARROW_TYPE_BOOL;
+    return GARROW_TYPE_BOOLEAN;
   case arrow::Type::type::UINT8:
     return GARROW_TYPE_UINT8;
   case arrow::Type::type::INT8:
diff --git a/c_glib/arrow-glib/type.h b/c_glib/arrow-glib/type.h
index 378d821ef2118..2137c785515f8 100644
--- a/c_glib/arrow-glib/type.h
+++ b/c_glib/arrow-glib/type.h
@@ -26,7 +26,7 @@ G_BEGIN_DECLS
 /**
  * GArrowType:
  * @GARROW_TYPE_NA: A degenerate NULL type represented as 0 bytes/bits.
- * @GARROW_TYPE_BOOL: A boolean value represented as 1 bit.
+ * @GARROW_TYPE_BOOLEAN: A boolean value represented as 1 bit.
  * @GARROW_TYPE_UINT8: Little-endian 8bit unsigned integer.
  * @GARROW_TYPE_INT8: Little-endian 8bit signed integer.
  * @GARROW_TYPE_UINT16: Little-endian 16bit unsigned integer.
@@ -58,7 +58,7 @@ G_BEGIN_DECLS
  */
 typedef enum {
   GARROW_TYPE_NA,
-  GARROW_TYPE_BOOL,
+  GARROW_TYPE_BOOLEAN,
   GARROW_TYPE_UINT8,
   GARROW_TYPE_INT8,
   GARROW_TYPE_UINT16,
diff --git a/c_glib/test/test-array.rb b/c_glib/test/test-array.rb
index d0ade2f1e7ea2..ca9bff8c2d20e 100644
--- a/c_glib/test/test-array.rb
+++ b/c_glib/test/test-array.rb
@@ -92,7 +92,7 @@ def test_value_data_type
   def test_value_type
     builder = Arrow::BooleanArrayBuilder.new
     array = builder.finish
-    assert_equal(Arrow::Type::BOOL, array.value_type)
+    assert_equal(Arrow::Type::BOOLEAN, array.value_type)
   end
 
   def test_slice
diff --git a/c_glib/test/test-boolean-data-type.rb b/c_glib/test/test-boolean-data-type.rb
index 39b8128989de3..0c60b10874724 100644
--- a/c_glib/test/test-boolean-data-type.rb
+++ b/c_glib/test/test-boolean-data-type.rb
@@ -18,7 +18,7 @@
 class TestBooleanDataType < Test::Unit::TestCase
   def test_type
     data_type = Arrow::BooleanDataType.new
-    assert_equal(Arrow::Type::BOOL, data_type.id)
+    assert_equal(Arrow::Type::BOOLEAN, data_type.id)
   end
 
   def test_to_s
diff --git a/c_glib/test/test-chunked-array.rb b/c_glib/test/test-chunked-array.rb
index 9287058e1abc6..4f6a4fb0df0b5 100644
--- a/c_glib/test/test-chunked-array.rb
+++ b/c_glib/test/test-chunked-array.rb
@@ -45,7 +45,7 @@ def test_value_type
       build_boolean_array([true, false]),
       build_boolean_array([true]),
     ]
-    assert_equal(Arrow::Type::BOOL,
+    assert_equal(Arrow::Type::BOOLEAN,
                  Arrow::ChunkedArray.new(chunks).value_type)
   end
 

From bdedf2390aafd7809032a2c6c5946db2123be325 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 4 Feb 2018 19:56:11 +0100
Subject: [PATCH 1439/1644] ARROW-2088: [GLib] Add GArrowNumericArray

We can use the class to determine whether an array is a numeric array or not.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1556 from kou/glib-add-numeric-array and squashes the following commits:

41a2535 [Kouhei Sutou] [GLib] Add GArrowNumericArray
---
 c_glib/arrow-glib/basic-array.cpp |  45 +-
 c_glib/arrow-glib/basic-array.h   | 724 +++++-------------------------
 2 files changed, 149 insertions(+), 620 deletions(-)

diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp
index 3c027c0e11240..35a5ea884e491 100644
--- a/c_glib/arrow-glib/basic-array.cpp
+++ b/c_glib/arrow-glib/basic-array.cpp
@@ -778,9 +778,24 @@ garrow_boolean_array_get_values(GArrowBooleanArray *array,
 }
 
 
+G_DEFINE_TYPE(GArrowNumericArray,               \
+              garrow_numeric_array,             \
+              GARROW_TYPE_PRIMITIVE_ARRAY)
+
+static void
+garrow_numeric_array_init(GArrowNumericArray *object)
+{
+}
+
+static void
+garrow_numeric_array_class_init(GArrowNumericArrayClass *klass)
+{
+}
+
+
 G_DEFINE_TYPE(GArrowInt8Array,               \
               garrow_int8_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_int8_array_init(GArrowInt8Array *object)
@@ -853,7 +868,7 @@ garrow_int8_array_get_values(GArrowInt8Array *array,
 
 G_DEFINE_TYPE(GArrowUInt8Array,               \
               garrow_uint8_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_uint8_array_init(GArrowUInt8Array *object)
@@ -926,7 +941,7 @@ garrow_uint8_array_get_values(GArrowUInt8Array *array,
 
 G_DEFINE_TYPE(GArrowInt16Array,               \
               garrow_int16_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_int16_array_init(GArrowInt16Array *object)
@@ -999,7 +1014,7 @@ garrow_int16_array_get_values(GArrowInt16Array *array,
 
 G_DEFINE_TYPE(GArrowUInt16Array,               \
               garrow_uint16_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_uint16_array_init(GArrowUInt16Array *object)
@@ -1072,7 +1087,7 @@ garrow_uint16_array_get_values(GArrowUInt16Array *array,
 
 G_DEFINE_TYPE(GArrowInt32Array,               \
               garrow_int32_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_int32_array_init(GArrowInt32Array *object)
@@ -1145,7 +1160,7 @@ garrow_int32_array_get_values(GArrowInt32Array *array,
 
 G_DEFINE_TYPE(GArrowUInt32Array,               \
               garrow_uint32_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_uint32_array_init(GArrowUInt32Array *object)
@@ -1218,7 +1233,7 @@ garrow_uint32_array_get_values(GArrowUInt32Array *array,
 
 G_DEFINE_TYPE(GArrowInt64Array,               \
               garrow_int64_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_int64_array_init(GArrowInt64Array *object)
@@ -1293,7 +1308,7 @@ garrow_int64_array_get_values(GArrowInt64Array *array,
 
 G_DEFINE_TYPE(GArrowUInt64Array,               \
               garrow_uint64_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_uint64_array_init(GArrowUInt64Array *object)
@@ -1368,7 +1383,7 @@ garrow_uint64_array_get_values(GArrowUInt64Array *array,
 
 G_DEFINE_TYPE(GArrowFloatArray,               \
               garrow_float_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_float_array_init(GArrowFloatArray *object)
@@ -1441,7 +1456,7 @@ garrow_float_array_get_values(GArrowFloatArray *array,
 
 G_DEFINE_TYPE(GArrowDoubleArray,               \
               garrow_double_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_double_array_init(GArrowDoubleArray *object)
@@ -1690,7 +1705,7 @@ garrow_string_array_get_string(GArrowStringArray *array,
 
 G_DEFINE_TYPE(GArrowDate32Array,               \
               garrow_date32_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_date32_array_init(GArrowDate32Array *object)
@@ -1767,7 +1782,7 @@ garrow_date32_array_get_values(GArrowDate32Array *array,
 
 G_DEFINE_TYPE(GArrowDate64Array,               \
               garrow_date64_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_date64_array_init(GArrowDate64Array *object)
@@ -1846,7 +1861,7 @@ garrow_date64_array_get_values(GArrowDate64Array *array,
 
 G_DEFINE_TYPE(GArrowTimestampArray,             \
               garrow_timestamp_array,           \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_timestamp_array_init(GArrowTimestampArray *object)
@@ -1929,7 +1944,7 @@ garrow_timestamp_array_get_values(GArrowTimestampArray *array,
 
 G_DEFINE_TYPE(GArrowTime32Array,               \
               garrow_time32_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_time32_array_init(GArrowTime32Array *object)
@@ -2010,7 +2025,7 @@ garrow_time32_array_get_values(GArrowTime32Array *array,
 
 G_DEFINE_TYPE(GArrowTime64Array,               \
               garrow_time64_array,             \
-              GARROW_TYPE_PRIMITIVE_ARRAY)
+              GARROW_TYPE_NUMERIC_ARRAY)
 
 static void
 garrow_time64_array_init(GArrowTime64Array *object)
diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h
index 420c30ecc7161..4bf0af3aebf1c 100644
--- a/c_glib/arrow-glib/basic-array.h
+++ b/c_glib/arrow-glib/basic-array.h
@@ -116,48 +116,17 @@ GType garrow_null_array_get_type(void) G_GNUC_CONST;
 GArrowNullArray *garrow_null_array_new(gint64 length);
 
 
-#define GARROW_TYPE_PRIMITIVE_ARRAY             \
-  (garrow_primitive_array_get_type())
-#define GARROW_PRIMITIVE_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_PRIMITIVE_ARRAY,      \
-                              GArrowPrimitiveArray))
-#define GARROW_PRIMITIVE_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_PRIMITIVE_ARRAY, \
-                           GArrowPrimitiveArrayClass))
-#define GARROW_IS_PRIMITIVE_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_PRIMITIVE_ARRAY))
-#define GARROW_IS_PRIMITIVE_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_PRIMITIVE_ARRAY))
-#define GARROW_PRIMITIVE_ARRAY_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_PRIMITIVE_ARRAY,       \
-                             GArrowPrimitiveArrayClass))
-
-typedef struct _GArrowPrimitiveArray         GArrowPrimitiveArray;
-typedef struct _GArrowPrimitiveArrayClass    GArrowPrimitiveArrayClass;
-
-/**
- * GArrowPrimitiveArray:
- *
- * It wraps `arrow::PrimitiveArray`.
- */
-struct _GArrowPrimitiveArray
-{
-  /*< private >*/
-  GArrowArray parent_instance;
-};
-
+#define GARROW_TYPE_PRIMITIVE_ARRAY (garrow_primitive_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowPrimitiveArray,
+                         garrow_primitive_array,
+                         GARROW,
+                         PRIMITIVE_ARRAY,
+                         GArrowArray)
 struct _GArrowPrimitiveArrayClass
 {
   GArrowArrayClass parent_class;
 };
 
-GType garrow_primitive_array_get_type(void) G_GNUC_CONST;
-
 GArrowBuffer *garrow_primitive_array_get_buffer(GArrowPrimitiveArray *array);
 
 
@@ -213,49 +182,28 @@ gboolean       garrow_boolean_array_get_value (GArrowBooleanArray *array,
 gboolean      *garrow_boolean_array_get_values(GArrowBooleanArray *array,
                                                gint64 *length);
 
-
-#define GARROW_TYPE_INT8_ARRAY                  \
-  (garrow_int8_array_get_type())
-#define GARROW_INT8_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT8_ARRAY,   \
-                              GArrowInt8Array))
-#define GARROW_INT8_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT8_ARRAY,      \
-                           GArrowInt8ArrayClass))
-#define GARROW_IS_INT8_ARRAY(obj)                       \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT8_ARRAY))
-#define GARROW_IS_INT8_ARRAY_CLASS(klass)               \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT8_ARRAY))
-#define GARROW_INT8_ARRAY_GET_CLASS(obj)                \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT8_ARRAY,    \
-                             GArrowInt8ArrayClass))
-
-typedef struct _GArrowInt8Array         GArrowInt8Array;
-typedef struct _GArrowInt8ArrayClass    GArrowInt8ArrayClass;
-
-/**
- * GArrowInt8Array:
- *
- * It wraps `arrow::Int8Array`.
- */
-struct _GArrowInt8Array
+#define GARROW_TYPE_NUMERIC_ARRAY (garrow_numeric_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowNumericArray,
+                         garrow_numeric_array,
+                         GARROW,
+                         NUMERIC_ARRAY,
+                         GArrowPrimitiveArray)
+struct _GArrowNumericArrayClass
 {
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
+  GArrowPrimitiveArrayClass parent_class;
 };
 
+#define GARROW_TYPE_INT8_ARRAY (garrow_int8_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInt8Array,
+                         garrow_int8_array,
+                         GARROW,
+                         INT8_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowInt8ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_int8_array_get_type(void) G_GNUC_CONST;
-
 GArrowInt8Array *garrow_int8_array_new(gint64 length,
                                        GArrowBuffer *data,
                                        GArrowBuffer *null_bitmap,
@@ -267,48 +215,17 @@ const gint8 *garrow_int8_array_get_values(GArrowInt8Array *array,
                                           gint64 *length);
 
 
-#define GARROW_TYPE_UINT8_ARRAY                 \
-  (garrow_uint8_array_get_type())
-#define GARROW_UINT8_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT8_ARRAY,  \
-                              GArrowUInt8Array))
-#define GARROW_UINT8_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT8_ARRAY,     \
-                           GArrowUInt8ArrayClass))
-#define GARROW_IS_UINT8_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_UINT8_ARRAY))
-#define GARROW_IS_UINT8_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT8_ARRAY))
-#define GARROW_UINT8_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT8_ARRAY,   \
-                             GArrowUInt8ArrayClass))
-
-typedef struct _GArrowUInt8Array         GArrowUInt8Array;
-typedef struct _GArrowUInt8ArrayClass    GArrowUInt8ArrayClass;
-
-/**
- * GArrowUInt8Array:
- *
- * It wraps `arrow::UInt8Array`.
- */
-struct _GArrowUInt8Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_UINT8_ARRAY (garrow_uint8_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowUInt8Array,
+                         garrow_uint8_array,
+                         GARROW,
+                         UINT8_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowUInt8ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_uint8_array_get_type(void) G_GNUC_CONST;
-
 GArrowUInt8Array *garrow_uint8_array_new(gint64 length,
                                          GArrowBuffer *data,
                                          GArrowBuffer *null_bitmap,
@@ -320,48 +237,17 @@ const guint8 *garrow_uint8_array_get_values(GArrowUInt8Array *array,
                                             gint64 *length);
 
 
-#define GARROW_TYPE_INT16_ARRAY                  \
-  (garrow_int16_array_get_type())
-#define GARROW_INT16_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT16_ARRAY,  \
-                              GArrowInt16Array))
-#define GARROW_INT16_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT16_ARRAY,     \
-                           GArrowInt16ArrayClass))
-#define GARROW_IS_INT16_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT16_ARRAY))
-#define GARROW_IS_INT16_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT16_ARRAY))
-#define GARROW_INT16_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT16_ARRAY,   \
-                             GArrowInt16ArrayClass))
-
-typedef struct _GArrowInt16Array         GArrowInt16Array;
-typedef struct _GArrowInt16ArrayClass    GArrowInt16ArrayClass;
-
-/**
- * GArrowInt16Array:
- *
- * It wraps `arrow::Int16Array`.
- */
-struct _GArrowInt16Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_INT16_ARRAY (garrow_int16_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInt16Array,
+                         garrow_int16_array,
+                         GARROW,
+                         INT16_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowInt16ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_int16_array_get_type(void) G_GNUC_CONST;
-
 GArrowInt16Array *garrow_int16_array_new(gint64 length,
                                          GArrowBuffer *data,
                                          GArrowBuffer *null_bitmap,
@@ -373,48 +259,17 @@ const gint16 *garrow_int16_array_get_values(GArrowInt16Array *array,
                                             gint64 *length);
 
 
-#define GARROW_TYPE_UINT16_ARRAY                 \
-  (garrow_uint16_array_get_type())
-#define GARROW_UINT16_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT16_ARRAY, \
-                              GArrowUInt16Array))
-#define GARROW_UINT16_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT16_ARRAY,    \
-                           GArrowUInt16ArrayClass))
-#define GARROW_IS_UINT16_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT16_ARRAY))
-#define GARROW_IS_UINT16_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT16_ARRAY))
-#define GARROW_UINT16_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT16_ARRAY,  \
-                             GArrowUInt16ArrayClass))
-
-typedef struct _GArrowUInt16Array         GArrowUInt16Array;
-typedef struct _GArrowUInt16ArrayClass    GArrowUInt16ArrayClass;
-
-/**
- * GArrowUInt16Array:
- *
- * It wraps `arrow::UInt16Array`.
- */
-struct _GArrowUInt16Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_UINT16_ARRAY (garrow_uint16_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowUInt16Array,
+                         garrow_uint16_array,
+                         GARROW,
+                         UINT16_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowUInt16ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_uint16_array_get_type(void) G_GNUC_CONST;
-
 GArrowUInt16Array *garrow_uint16_array_new(gint64 length,
                                            GArrowBuffer *data,
                                            GArrowBuffer *null_bitmap,
@@ -426,48 +281,17 @@ const guint16 *garrow_uint16_array_get_values(GArrowUInt16Array *array,
                                               gint64 *length);
 
 
-#define GARROW_TYPE_INT32_ARRAY                 \
-  (garrow_int32_array_get_type())
-#define GARROW_INT32_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT32_ARRAY,  \
-                              GArrowInt32Array))
-#define GARROW_INT32_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT32_ARRAY,     \
-                           GArrowInt32ArrayClass))
-#define GARROW_IS_INT32_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT32_ARRAY))
-#define GARROW_IS_INT32_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT32_ARRAY))
-#define GARROW_INT32_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT32_ARRAY,   \
-                             GArrowInt32ArrayClass))
-
-typedef struct _GArrowInt32Array         GArrowInt32Array;
-typedef struct _GArrowInt32ArrayClass    GArrowInt32ArrayClass;
-
-/**
- * GArrowInt32Array:
- *
- * It wraps `arrow::Int32Array`.
- */
-struct _GArrowInt32Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_INT32_ARRAY (garrow_int32_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInt32Array,
+                         garrow_int32_array,
+                         GARROW,
+                         INT32_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowInt32ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_int32_array_get_type(void) G_GNUC_CONST;
-
 GArrowInt32Array *garrow_int32_array_new(gint64 length,
                                          GArrowBuffer *data,
                                          GArrowBuffer *null_bitmap,
@@ -479,48 +303,17 @@ const gint32 *garrow_int32_array_get_values(GArrowInt32Array *array,
                                             gint64 *length);
 
 
-#define GARROW_TYPE_UINT32_ARRAY                \
-  (garrow_uint32_array_get_type())
-#define GARROW_UINT32_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT32_ARRAY, \
-                              GArrowUInt32Array))
-#define GARROW_UINT32_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT32_ARRAY,    \
-                           GArrowUInt32ArrayClass))
-#define GARROW_IS_UINT32_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT32_ARRAY))
-#define GARROW_IS_UINT32_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT32_ARRAY))
-#define GARROW_UINT32_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT32_ARRAY,  \
-                             GArrowUInt32ArrayClass))
-
-typedef struct _GArrowUInt32Array         GArrowUInt32Array;
-typedef struct _GArrowUInt32ArrayClass    GArrowUInt32ArrayClass;
-
-/**
- * GArrowUInt32Array:
- *
- * It wraps `arrow::UInt32Array`.
- */
-struct _GArrowUInt32Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_UINT32_ARRAY (garrow_uint32_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowUInt32Array,
+                         garrow_uint32_array,
+                         GARROW,
+                         UINT32_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowUInt32ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_uint32_array_get_type(void) G_GNUC_CONST;
-
 GArrowUInt32Array *garrow_uint32_array_new(gint64 length,
                                            GArrowBuffer *data,
                                            GArrowBuffer *null_bitmap,
@@ -532,48 +325,17 @@ const guint32 *garrow_uint32_array_get_values(GArrowUInt32Array *array,
                                               gint64 *length);
 
 
-#define GARROW_TYPE_INT64_ARRAY                 \
-  (garrow_int64_array_get_type())
-#define GARROW_INT64_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_INT64_ARRAY,  \
-                              GArrowInt64Array))
-#define GARROW_INT64_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT64_ARRAY,     \
-                           GArrowInt64ArrayClass))
-#define GARROW_IS_INT64_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_INT64_ARRAY))
-#define GARROW_IS_INT64_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_INT64_ARRAY))
-#define GARROW_INT64_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_INT64_ARRAY,   \
-                             GArrowInt64ArrayClass))
-
-typedef struct _GArrowInt64Array         GArrowInt64Array;
-typedef struct _GArrowInt64ArrayClass    GArrowInt64ArrayClass;
-
-/**
- * GArrowInt64Array:
- *
- * It wraps `arrow::Int64Array`.
- */
-struct _GArrowInt64Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_INT64_ARRAY (garrow_int64_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInt64Array,
+                         garrow_int64_array,
+                         GARROW,
+                         INT64_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowInt64ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_int64_array_get_type(void) G_GNUC_CONST;
-
 GArrowInt64Array *garrow_int64_array_new(gint64 length,
                                          GArrowBuffer *data,
                                          GArrowBuffer *null_bitmap,
@@ -585,48 +347,17 @@ const gint64 *garrow_int64_array_get_values(GArrowInt64Array *array,
                                             gint64 *length);
 
 
-#define GARROW_TYPE_UINT64_ARRAY                \
-  (garrow_uint64_array_get_type())
-#define GARROW_UINT64_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_UINT64_ARRAY, \
-                              GArrowUInt64Array))
-#define GARROW_UINT64_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_UINT64_ARRAY,    \
-                           GArrowUInt64ArrayClass))
-#define GARROW_IS_UINT64_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT64_ARRAY))
-#define GARROW_IS_UINT64_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_UINT64_ARRAY))
-#define GARROW_UINT64_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_UINT64_ARRAY,  \
-                             GArrowUInt64ArrayClass))
-
-typedef struct _GArrowUInt64Array         GArrowUInt64Array;
-typedef struct _GArrowUInt64ArrayClass    GArrowUInt64ArrayClass;
-
-/**
- * GArrowUInt64Array:
- *
- * It wraps `arrow::UInt64Array`.
- */
-struct _GArrowUInt64Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_UINT64_ARRAY (garrow_uint64_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowUInt64Array,
+                         garrow_uint64_array,
+                         GARROW,
+                         UINT64_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowUInt64ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_uint64_array_get_type(void) G_GNUC_CONST;
-
 GArrowUInt64Array *garrow_uint64_array_new(gint64 length,
                                            GArrowBuffer *data,
                                            GArrowBuffer *null_bitmap,
@@ -638,48 +369,17 @@ const guint64 *garrow_uint64_array_get_values(GArrowUInt64Array *array,
                                               gint64 *length);
 
 
-#define GARROW_TYPE_FLOAT_ARRAY                 \
-  (garrow_float_array_get_type())
-#define GARROW_FLOAT_ARRAY(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_FLOAT_ARRAY,  \
-                              GArrowFloatArray))
-#define GARROW_FLOAT_ARRAY_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_FLOAT_ARRAY,     \
-                           GArrowFloatArrayClass))
-#define GARROW_IS_FLOAT_ARRAY(obj)                      \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                    \
-                              GARROW_TYPE_FLOAT_ARRAY))
-#define GARROW_IS_FLOAT_ARRAY_CLASS(klass)              \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_FLOAT_ARRAY))
-#define GARROW_FLOAT_ARRAY_GET_CLASS(obj)               \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_FLOAT_ARRAY,   \
-                             GArrowFloatArrayClass))
-
-typedef struct _GArrowFloatArray         GArrowFloatArray;
-typedef struct _GArrowFloatArrayClass    GArrowFloatArrayClass;
-
-/**
- * GArrowFloatArray:
- *
- * It wraps `arrow::FloatArray`.
- */
-struct _GArrowFloatArray
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_FLOAT_ARRAY (garrow_float_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowFloatArray,
+                         garrow_float_array,
+                         GARROW,
+                         FLOAT_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowFloatArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_float_array_get_type(void) G_GNUC_CONST;
-
 GArrowFloatArray *garrow_float_array_new(gint64 length,
                                          GArrowBuffer *data,
                                          GArrowBuffer *null_bitmap,
@@ -691,48 +391,17 @@ const gfloat *garrow_float_array_get_values(GArrowFloatArray *array,
                                             gint64 *length);
 
 
-#define GARROW_TYPE_DOUBLE_ARRAY                \
-  (garrow_double_array_get_type())
-#define GARROW_DOUBLE_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DOUBLE_ARRAY, \
-                              GArrowDoubleArray))
-#define GARROW_DOUBLE_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DOUBLE_ARRAY,    \
-                           GArrowDoubleArrayClass))
-#define GARROW_IS_DOUBLE_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DOUBLE_ARRAY))
-#define GARROW_IS_DOUBLE_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DOUBLE_ARRAY))
-#define GARROW_DOUBLE_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DOUBLE_ARRAY,  \
-                             GArrowDoubleArrayClass))
-
-typedef struct _GArrowDoubleArray         GArrowDoubleArray;
-typedef struct _GArrowDoubleArrayClass    GArrowDoubleArrayClass;
-
-/**
- * GArrowDoubleArray:
- *
- * It wraps `arrow::DoubleArray`.
- */
-struct _GArrowDoubleArray
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_DOUBLE_ARRAY (garrow_double_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDoubleArray,
+                         garrow_double_array,
+                         GARROW,
+                         DOUBLE_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowDoubleArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_double_array_get_type(void) G_GNUC_CONST;
-
 GArrowDoubleArray *garrow_double_array_new(gint64 length,
                                            GArrowBuffer *data,
                                            GArrowBuffer *null_bitmap,
@@ -849,48 +518,17 @@ gchar *garrow_string_array_get_string(GArrowStringArray *array,
                                       gint64 i);
 
 
-#define GARROW_TYPE_DATE32_ARRAY                \
-  (garrow_date32_array_get_type())
-#define GARROW_DATE32_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DATE32_ARRAY, \
-                              GArrowDate32Array))
-#define GARROW_DATE32_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DATE32_ARRAY,    \
-                           GArrowDate32ArrayClass))
-#define GARROW_IS_DATE32_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DATE32_ARRAY))
-#define GARROW_IS_DATE32_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DATE32_ARRAY))
-#define GARROW_DATE32_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DATE32_ARRAY,  \
-                             GArrowDate32ArrayClass))
-
-typedef struct _GArrowDate32Array         GArrowDate32Array;
-typedef struct _GArrowDate32ArrayClass    GArrowDate32ArrayClass;
-
-/**
- * GArrowDate32Array:
- *
- * It wraps `arrow::Date32Array`.
- */
-struct _GArrowDate32Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_DATE32_ARRAY (garrow_date32_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDate32Array,
+                         garrow_date32_array,
+                         GARROW,
+                         DATE32_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowDate32ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_date32_array_get_type(void) G_GNUC_CONST;
-
 GArrowDate32Array *garrow_date32_array_new(gint64 length,
                                            GArrowBuffer *data,
                                            GArrowBuffer *null_bitmap,
@@ -902,48 +540,17 @@ const gint32 *garrow_date32_array_get_values(GArrowDate32Array *array,
                                              gint64 *length);
 
 
-#define GARROW_TYPE_DATE64_ARRAY                \
-  (garrow_date64_array_get_type())
-#define GARROW_DATE64_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_DATE64_ARRAY, \
-                              GArrowDate64Array))
-#define GARROW_DATE64_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_DATE64_ARRAY,    \
-                           GArrowDate64ArrayClass))
-#define GARROW_IS_DATE64_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DATE64_ARRAY))
-#define GARROW_IS_DATE64_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_DATE64_ARRAY))
-#define GARROW_DATE64_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_DATE64_ARRAY,  \
-                             GArrowDate64ArrayClass))
-
-typedef struct _GArrowDate64Array         GArrowDate64Array;
-typedef struct _GArrowDate64ArrayClass    GArrowDate64ArrayClass;
-
-/**
- * GArrowDate64Array:
- *
- * It wraps `arrow::Date64Array`.
- */
-struct _GArrowDate64Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_DATE64_ARRAY (garrow_date64_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDate64Array,
+                         garrow_date64_array,
+                         GARROW,
+                         DATE64_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowDate64ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_date64_array_get_type(void) G_GNUC_CONST;
-
 GArrowDate64Array *garrow_date64_array_new(gint64 length,
                                            GArrowBuffer *data,
                                            GArrowBuffer *null_bitmap,
@@ -955,48 +562,17 @@ const gint64 *garrow_date64_array_get_values(GArrowDate64Array *array,
                                              gint64 *length);
 
 
-#define GARROW_TYPE_TIMESTAMP_ARRAY             \
-  (garrow_timestamp_array_get_type())
-#define GARROW_TIMESTAMP_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_TIMESTAMP_ARRAY,      \
-                              GArrowTimestampArray))
-#define GARROW_TIMESTAMP_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_TIMESTAMP_ARRAY, \
-                           GArrowTimestampArrayClass))
-#define GARROW_IS_TIMESTAMP_ARRAY(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIMESTAMP_ARRAY))
-#define GARROW_IS_TIMESTAMP_ARRAY_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_TIMESTAMP_ARRAY))
-#define GARROW_TIMESTAMP_ARRAY_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_TIMESTAMP_ARRAY,       \
-                             GArrowTimestampArrayClass))
-
-typedef struct _GArrowTimestampArray         GArrowTimestampArray;
-typedef struct _GArrowTimestampArrayClass    GArrowTimestampArrayClass;
-
-/**
- * GArrowTimestampArray:
- *
- * It wraps `arrow::TimestampArray`.
- */
-struct _GArrowTimestampArray
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_TIMESTAMP_ARRAY (garrow_timestamp_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowTimestampArray,
+                         garrow_timestamp_array,
+                         GARROW,
+                         TIMESTAMP_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowTimestampArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_timestamp_array_get_type(void) G_GNUC_CONST;
-
 GArrowTimestampArray *garrow_timestamp_array_new(GArrowTimestampDataType *data_type,
                                                  gint64 length,
                                                  GArrowBuffer *data,
@@ -1009,48 +585,17 @@ const gint64 *garrow_timestamp_array_get_values(GArrowTimestampArray *array,
                                                 gint64 *length);
 
 
-#define GARROW_TYPE_TIME32_ARRAY                \
-  (garrow_time32_array_get_type())
-#define GARROW_TIME32_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_TIME32_ARRAY, \
-                              GArrowTime32Array))
-#define GARROW_TIME32_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_TIME32_ARRAY,    \
-                           GArrowTime32ArrayClass))
-#define GARROW_IS_TIME32_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIME32_ARRAY))
-#define GARROW_IS_TIME32_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_TIME32_ARRAY))
-#define GARROW_TIME32_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_TIME32_ARRAY,  \
-                             GArrowTime32ArrayClass))
-
-typedef struct _GArrowTime32Array         GArrowTime32Array;
-typedef struct _GArrowTime32ArrayClass    GArrowTime32ArrayClass;
-
-/**
- * GArrowTime32Array:
- *
- * It wraps `arrow::Time32Array`.
- */
-struct _GArrowTime32Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_TIME32_ARRAY (garrow_time32_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowTime32Array,
+                         garrow_time32_array,
+                         GARROW,
+                         TIME32_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowTime32ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_time32_array_get_type(void) G_GNUC_CONST;
-
 GArrowTime32Array *garrow_time32_array_new(GArrowTime32DataType *data_type,
                                            gint64 length,
                                            GArrowBuffer *data,
@@ -1063,48 +608,17 @@ const gint32 *garrow_time32_array_get_values(GArrowTime32Array *array,
                                              gint64 *length);
 
 
-#define GARROW_TYPE_TIME64_ARRAY                \
-  (garrow_time64_array_get_type())
-#define GARROW_TIME64_ARRAY(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                    \
-                              GARROW_TYPE_TIME64_ARRAY, \
-                              GArrowTime64Array))
-#define GARROW_TIME64_ARRAY_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_TIME64_ARRAY,    \
-                           GArrowTime64ArrayClass))
-#define GARROW_IS_TIME64_ARRAY(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_TIME64_ARRAY))
-#define GARROW_IS_TIME64_ARRAY_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                     \
-                           GARROW_TYPE_TIME64_ARRAY))
-#define GARROW_TIME64_ARRAY_GET_CLASS(obj)              \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                     \
-                             GARROW_TYPE_TIME64_ARRAY,  \
-                             GArrowTime64ArrayClass))
-
-typedef struct _GArrowTime64Array         GArrowTime64Array;
-typedef struct _GArrowTime64ArrayClass    GArrowTime64ArrayClass;
-
-/**
- * GArrowTime64Array:
- *
- * It wraps `arrow::Time64Array`.
- */
-struct _GArrowTime64Array
-{
-  /*< private >*/
-  GArrowPrimitiveArray parent_instance;
-};
-
+#define GARROW_TYPE_TIME64_ARRAY (garrow_time64_array_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowTime64Array,
+                         garrow_time64_array,
+                         GARROW,
+                         TIME64_ARRAY,
+                         GArrowNumericArray)
 struct _GArrowTime64ArrayClass
 {
-  GArrowPrimitiveArrayClass parent_class;
+  GArrowNumericArrayClass parent_class;
 };
 
-GType garrow_time64_array_get_type(void) G_GNUC_CONST;
-
 GArrowTime64Array *garrow_time64_array_new(GArrowTime64DataType *data_type,
                                            gint64 length,
                                            GArrowBuffer *data,

From 30ade84465f0c3b6d21833813bcdc0d6cf352cd8 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 5 Feb 2018 16:48:52 -0500
Subject: [PATCH 1440/1644] ARROW-2071: [Python] Fix test slowness on Travis-CI

The pytorch build from the "soumith" channel can be buggy and fail importing with a symbol error.  In that case avoid trying "import torch" extremely often as this spends CPU time in the dynamic loader.

Also lighten the plasma tests when run with Valgrind.

The main offending job is down to 29 minutes, with 2 minutes being taken by the Python tests.

Author: Antoine Pitrou <antoine@python.org>

Closes #1561 from pitrou/ARROW-2071-fix-travis-ci-tests-slowness and squashes the following commits:

da67a053 [Antoine Pitrou] ARROW-2071: [Python] Fix test slowness on Travis-CI
---
 ci/travis_before_script_cpp.sh             |  5 +--
 ci/travis_build_parquet_cpp.sh             |  2 +-
 ci/travis_install_clang_tools.sh           |  4 +-
 ci/travis_install_conda.sh                 |  2 +-
 ci/travis_lint.sh                          |  2 +-
 ci/travis_script_python.sh                 |  9 ++--
 python/pyarrow/tests/test_plasma.py        | 49 ++++++++++++----------
 python/pyarrow/tests/test_serialization.py | 19 +++++----
 8 files changed, 51 insertions(+), 41 deletions(-)
 mode change 100644 => 100755 ci/travis_install_conda.sh

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 7c1d726d4d37e..7725c560cb3d6 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -52,11 +52,10 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
 
   # HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
   # updated sometime on October 2 or October 3
-  conda update -y -p $CPP_TOOLCHAIN ca-certificates -c defaults
+  conda update -y -q -p $CPP_TOOLCHAIN ca-certificates -c defaults
 fi
 
-
-mkdir $ARROW_CPP_BUILD_DIR
+mkdir -p $ARROW_CPP_BUILD_DIR
 pushd $ARROW_CPP_BUILD_DIR
 
 CMAKE_COMMON_FLAGS="\
diff --git a/ci/travis_build_parquet_cpp.sh b/ci/travis_build_parquet_cpp.sh
index 4330a31a077a5..4b6370ea729fb 100755
--- a/ci/travis_build_parquet_cpp.sh
+++ b/ci/travis_build_parquet_cpp.sh
@@ -28,7 +28,7 @@ export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
 PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
 mkdir -p $PARQUET_DIR
 
-git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
+git clone -q https://github.com/apache/parquet-cpp.git $PARQUET_DIR
 
 pushd $PARQUET_DIR
 mkdir build-dir
diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
index bad1e73d24a4a..d0108ad37ef02 100755
--- a/ci/travis_install_clang_tools.sh
+++ b/ci/travis_install_clang_tools.sh
@@ -20,5 +20,5 @@
 wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
 sudo apt-add-repository -y \
      "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-4.0 main"
-sudo apt-get update
-sudo apt-get install clang-4.0 clang-format-4.0 clang-tidy-4.0
+sudo apt-get update -q
+sudo apt-get install -q clang-4.0 clang-format-4.0 clang-tidy-4.0
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
old mode 100644
new mode 100755
index 3faa548817017..caec9bb33ebf2
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -25,7 +25,7 @@ else
   MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
 fi
 
-wget -O miniconda.sh $MINICONDA_URL
+wget --no-verbose -O miniconda.sh $MINICONDA_URL
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 mkdir -p $CONDA_PKGS_DIRS
diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
index 6a2a0be18cf9f..096170a9eb0d2 100755
--- a/ci/travis_lint.sh
+++ b/ci/travis_lint.sh
@@ -33,7 +33,7 @@ fi
 popd
 
 # Fail fast on style checks
-sudo pip install flake8
+sudo pip install -q flake8
 
 PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 7c896df9c840f..4187e7541ebd4 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -53,6 +53,7 @@ if [ "$PYTHON_VERSION" != "2.7" ] || [ $TRAVIS_OS_NAME != "osx" ]; then
 fi
 
 # Build C++ libraries
+mkdir -p $ARROW_CPP_BUILD_DIR
 pushd $ARROW_CPP_BUILD_DIR
 
 # Clear out prior build files
@@ -77,21 +78,21 @@ popd
 pushd $ARROW_PYTHON_DIR
 
 if [ "$PYTHON_VERSION" == "2.7" ]; then
-  pip install futures
+  pip install -q futures
 fi
 
 export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE
 
-pip install -r requirements.txt
+pip install -q -r requirements.txt
 python setup.py build_ext --with-parquet --with-plasma --with-orc\
-       install --single-version-externally-managed --record=record.text
+       install -q --single-version-externally-managed --record=record.text
 popd
 
 python -c "import pyarrow.parquet"
 python -c "import pyarrow.plasma"
 python -c "import pyarrow.orc"
 
-if [ $TRAVIS_OS_NAME == "linux" ]; then
+if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then
   export PLASMA_VALGRIND=1
 fi
 
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index 9ea6476670441..27556e60d3bf0 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -30,7 +30,9 @@
 import pyarrow as pa
 import pandas as pd
 
-DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9
+
+DEFAULT_PLASMA_STORE_MEMORY = 10 ** 8
+USE_VALGRIND = os.getenv("PLASMA_VALGRIND") == "1"
 
 
 def random_name():
@@ -71,7 +73,7 @@ def create_object_with_id(client, object_id, data_size, metadata_size,
     return memory_buffer, metadata
 
 
-def create_object(client, data_size, metadata_size, seal=True):
+def create_object(client, data_size, metadata_size=0, seal=True):
     object_id = random_object_id()
     memory_buffer, metadata = create_object_with_id(client, object_id,
                                                     data_size, metadata_size,
@@ -158,7 +160,7 @@ def setup_method(self, test_method):
         import pyarrow.plasma as plasma
         # Start Plasma store.
         plasma_store_name, self.p = start_plasma_store(
-            use_valgrind=os.getenv("PLASMA_VALGRIND") == "1",
+            use_valgrind=USE_VALGRIND,
             use_one_memory_mapped_file=use_one_memory_mapped_file)
         # Connect to Plasma.
         self.plasma_client = plasma.connect(plasma_store_name, "", 64)
@@ -202,7 +204,7 @@ def test_create(self):
             assert memory_buffer[i] == i % 256
 
     def test_create_with_metadata(self):
-        for length in range(1000):
+        for length in range(0, 1000, 3):
             # Create an object id string.
             object_id = random_object_id()
             # Create a random metadata string.
@@ -246,7 +248,7 @@ def test_create_existing(self):
                 assert False
 
     def test_get(self):
-        num_object_ids = 100
+        num_object_ids = 60
         # Test timing out of get with various timeouts.
         for timeout in [0, 10, 100, 1000]:
             object_ids = [random_object_id() for _ in range(num_object_ids)]
@@ -390,29 +392,30 @@ def assert_create_raises_plasma_full(unit_test, size):
                 # For some reason the above didn't throw an exception, so fail.
                 assert False
 
+        PERCENT = DEFAULT_PLASMA_STORE_MEMORY // 100
+
         # Create a list to keep some of the buffers in scope.
         memory_buffers = []
-        _, memory_buffer, _ = create_object(self.plasma_client, 5 * 10 ** 8, 0)
+        _, memory_buffer, _ = create_object(self.plasma_client, 50 * PERCENT)
         memory_buffers.append(memory_buffer)
-        # Remaining space is 5 * 10 ** 8. Make sure that we can't create an
-        # object of size 5 * 10 ** 8 + 1, but we can create one of size
-        # 2 * 10 ** 8.
-        assert_create_raises_plasma_full(self, 5 * 10 ** 8 + 1)
-        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
+        # Remaining space is 50%. Make sure that we can't create an
+        # object of size 50% + 1, but we can create one of size 20%.
+        assert_create_raises_plasma_full(self, 50 * PERCENT + 1)
+        _, memory_buffer, _ = create_object(self.plasma_client, 20 * PERCENT)
         del memory_buffer
-        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
+        _, memory_buffer, _ = create_object(self.plasma_client, 20 * PERCENT)
         del memory_buffer
-        assert_create_raises_plasma_full(self, 5 * 10 ** 8 + 1)
+        assert_create_raises_plasma_full(self, 50 * PERCENT + 1)
 
-        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
+        _, memory_buffer, _ = create_object(self.plasma_client, 20 * PERCENT)
         memory_buffers.append(memory_buffer)
-        # Remaining space is 3 * 10 ** 8.
-        assert_create_raises_plasma_full(self, 3 * 10 ** 8 + 1)
+        # Remaining space is 30%.
+        assert_create_raises_plasma_full(self, 30 * PERCENT + 1)
 
-        _, memory_buffer, _ = create_object(self.plasma_client, 10 ** 8, 0)
+        _, memory_buffer, _ = create_object(self.plasma_client, 10 * PERCENT)
         memory_buffers.append(memory_buffer)
-        # Remaining space is 2 * 10 ** 8.
-        assert_create_raises_plasma_full(self, 2 * 10 ** 8 + 1)
+        # Remaining space is 20%.
+        assert_create_raises_plasma_full(self, 20 * PERCENT + 1)
 
     def test_contains(self):
         fake_object_ids = [random_object_id() for _ in range(100)]
@@ -645,10 +648,14 @@ def test_evict(self):
         del b7
         assert client.evict(2000) == 996 + 995 + 994
 
+    # Mitigate valgrind-induced slowness
+    SUBSCRIBE_TEST_SIZES = ([1, 10, 100, 1000] if USE_VALGRIND
+                            else [1, 10, 100, 1000, 10000])
+
     def test_subscribe(self):
         # Subscribe to notifications from the Plasma Store.
         self.plasma_client.subscribe()
-        for i in [1, 10, 100, 1000, 10000]:
+        for i in self.SUBSCRIBE_TEST_SIZES:
             object_ids = [random_object_id() for _ in range(i)]
             metadata_sizes = [np.random.randint(1000) for _ in range(i)]
             data_sizes = [np.random.randint(1000) for _ in range(i)]
@@ -670,7 +677,7 @@ def test_subscribe_deletions(self):
         # plasma_client2 to make sure that all used objects will get evicted
         # properly.
         self.plasma_client2.subscribe()
-        for i in [1, 10, 100, 1000, 10000]:
+        for i in self.SUBSCRIBE_TEST_SIZES:
             object_ids = [random_object_id() for _ in range(i)]
             # Add 1 to the sizes to make sure we have nonzero object sizes.
             metadata_sizes = [np.random.randint(1000) + 1 for _ in range(i)]
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 3b1324beb999b..20c195a4b8c75 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -28,15 +28,19 @@
 import pyarrow as pa
 import numpy as np
 
+try:
+    import torch
+except ImportError:
+    torch = None
+    # Blacklist the module in case `import torch` is costly before
+    # failing (ARROW-2071)
+    sys.modules['torch'] = None
+
 
 def assert_equal(obj1, obj2):
-    try:
-        import torch
-        if torch.is_tensor(obj1) and torch.is_tensor(obj2):
-            assert torch.equal(obj1, obj2)
-            return
-    except ImportError:
-        pass
+    if torch is not None and torch.is_tensor(obj1) and torch.is_tensor(obj2):
+        assert torch.equal(obj1, obj2)
+        return
     module_numpy = (type(obj1).__module__ == np.__name__ or
                     type(obj2).__module__ == np.__name__)
     if module_numpy:
@@ -346,7 +350,6 @@ def test_datetime_serialization(large_buffer):
 
 def test_torch_serialization(large_buffer):
     pytest.importorskip("torch")
-    import torch
 
     serialization_context = pa.default_serialization_context()
     pa.register_torch_serialization_handlers(serialization_context)

From 11c9b42a9f9c8b69fa33b1972bc195ea5e5f47ae Mon Sep 17 00:00:00 2001
From: Alec Posney <aposney@atlassian.com>
Date: Mon, 5 Feb 2018 16:56:31 -0500
Subject: [PATCH 1441/1644] ARROW-2090: [Python] Add context methods to
 ParquetWriter

Added the `__enter__` and `__exit__` methods to the ParquetWriter
Class. This allows you to write code in the following style:

```
with ParquetWriter(foo, schema) as writer:
    writer.write_table(table)
```

And the ParquetWriter context object will handle calling the close
method for you when the with block exits.  It propagates errors
in line with previous behavior. I have also updated the module
level `write_table` method to use the new `with` style.
And it maintains it's existing behavior of trying to remove
a partial file if an exception is encountered.

The reason for this change, was a bug I encountered while
using the `ParquetWriter` object, where I wasn't closing
the writer correctly in the event of an exception, which resulted
in partially written parquet files. Adding the ability to use
`with` syntax will reduce the chances of a similar misuse
while still maintaing backwards compatibility.

Author: Alec Posney <aposney@atlassian.com>

Closes #1559 from Posnet/ARROW-2090 and squashes the following commits:

9a24433d [Alec Posney] ARROW-2090: [Python] Add context methods to ParquetWriter
---
 python/doc/source/parquet.rst        |  9 ++++
 python/pyarrow/parquet.py            | 33 ++++++++-------
 python/pyarrow/tests/test_parquet.py | 63 ++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+), 15 deletions(-)

diff --git a/python/doc/source/parquet.rst b/python/doc/source/parquet.rst
index d466ba128cddc..ac56520ff4cee 100644
--- a/python/doc/source/parquet.rst
+++ b/python/doc/source/parquet.rst
@@ -139,11 +139,20 @@ We can similarly write a Parquet file with multiple row groups by using
    pf2 = pq.ParquetFile('example2.parquet')
    pf2.num_row_groups
 
+Alternatively python ``with`` syntax can also be use:
+
+.. ipython:: python
+
+   with pq.ParquetWriter('example3.parquet', table.schema) as writer:
+       for i in range(3):
+           writer.write_table(table)
+
 .. ipython:: python
    :suppress:
 
    !rm example.parquet
    !rm example2.parquet
+   !rm example3.parquet
 
 Compression, Encoding, and File Compatibility
 ---------------------------------------------
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 3a0924a27ceb2..8820b6b4abe75 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -292,6 +292,14 @@ def __del__(self):
         if getattr(self, 'is_open', False):
             self.close()
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.close()
+        # return false since we want to propagate exceptions
+        return False
+
     def write_table(self, table, row_group_size=None):
         if self.schema_changed:
             table = _sanitize_table(table, self.schema, self.flavor)
@@ -932,29 +940,24 @@ def write_table(table, where, row_group_size=None, version='1.0',
                 flavor=None, **kwargs):
     row_group_size = kwargs.pop('chunk_size', row_group_size)
 
-    writer = None
     try:
-        writer = ParquetWriter(
-            where, table.schema,
-            version=version,
-            flavor=flavor,
-            use_dictionary=use_dictionary,
-            coerce_timestamps=coerce_timestamps,
-            compression=compression,
-            use_deprecated_int96_timestamps=use_deprecated_int96_timestamps,
-            **kwargs)
-        writer.write_table(table, row_group_size=row_group_size)
+        with ParquetWriter(
+                where, table.schema,
+                version=version,
+                flavor=flavor,
+                use_dictionary=use_dictionary,
+                coerce_timestamps=coerce_timestamps,
+                compression=compression,
+                use_deprecated_int96_timestamps= use_deprecated_int96_timestamps, # noqa
+                **kwargs) as writer:
+            writer.write_table(table, row_group_size=row_group_size)
     except Exception:
-        if writer is not None:
-            writer.close()
         if isinstance(where, six.string_types):
             try:
                 os.remove(where)
             except os.error:
                 pass
         raise
-    else:
-        writer.close()
 
 
 write_table.__doc__ = """
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 7c2edb378df61..c49f3d396b888 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1673,3 +1673,66 @@ def test_decimal_roundtrip_negative_scale(tmpdir):
     result_table = _read_table(string_filename)
     result = result_table.to_pandas()
     tm.assert_frame_equal(result, expected)
+
+
+@parquet
+def test_parquet_writer_context_obj(tmpdir):
+
+    import pyarrow.parquet as pq
+
+    df = _test_dataframe(100)
+    df['unique_id'] = 0
+
+    arrow_table = pa.Table.from_pandas(df, preserve_index=False)
+    out = pa.BufferOutputStream()
+
+    with pq.ParquetWriter(out, arrow_table.schema, version='2.0') as writer:
+
+        frames = []
+        for i in range(10):
+            df['unique_id'] = i
+            arrow_table = pa.Table.from_pandas(df, preserve_index=False)
+            writer.write_table(arrow_table)
+
+            frames.append(df.copy())
+
+    buf = out.get_result()
+    result = _read_table(pa.BufferReader(buf))
+
+    expected = pd.concat(frames, ignore_index=True)
+    tm.assert_frame_equal(result.to_pandas(), expected)
+
+
+@parquet
+def test_parquet_writer_context_obj_with_exception(tmpdir):
+
+    import pyarrow.parquet as pq
+
+    df = _test_dataframe(100)
+    df['unique_id'] = 0
+
+    arrow_table = pa.Table.from_pandas(df, preserve_index=False)
+    out = pa.BufferOutputStream()
+    error_text = 'Artificial Error'
+
+    try:
+        with pq.ParquetWriter(out,
+                              arrow_table.schema,
+                              version='2.0') as writer:
+
+            frames = []
+            for i in range(10):
+                df['unique_id'] = i
+                arrow_table = pa.Table.from_pandas(df, preserve_index=False)
+                writer.write_table(arrow_table)
+                frames.append(df.copy())
+                if i == 5:
+                    raise ValueError(error_text)
+    except Exception as e:
+        assert str(e) == error_text
+
+    buf = out.get_result()
+    result = _read_table(pa.BufferReader(buf))
+
+    expected = pd.concat(frames, ignore_index=True)
+    tm.assert_frame_equal(result.to_pandas(), expected)

From c49636d0b7a10badba34c9d450a542d951d340e1 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Mon, 5 Feb 2018 16:58:17 -0500
Subject: [PATCH 1442/1644] =?UTF-8?q?ARROW-2086:=20[Python]=C2=A0Shrink=20?=
 =?UTF-8?q?size=20of=20arrow=5Fmanylinux1=5Fx86=5F64=5Fbase=20docker=20ima?=
 =?UTF-8?q?ge?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

First iteration, the next step would be to go over some binaries and ensure that they get stripped or that upstream already strips them.

Image size before: 3.36G
Image size now: 2.82G

Details: https://gist.github.com/xhochy/a3db85f8509101c9762d31845ae123b1

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1555 from xhochy/ARROW-2086 and squashes the following commits:

131990c3 [Korn, Uwe] ARROW-2086: [Python] Shrink size of arrow_manylinux1_x86_64_base docker image
---
 python/manylinux1/Dockerfile-x86_64            |  2 +-
 python/manylinux1/Dockerfile-x86_64_base       |  2 +-
 python/manylinux1/scripts/build_boost.sh       | 12 +++++++++++-
 python/manylinux1/scripts/build_virtualenvs.sh |  8 ++++++++
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 9c00e7ea256c9..98b559535e001 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2048
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2086
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index ec7893080f65b..b7687533aaaa2 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -17,7 +17,7 @@
 FROM quay.io/pypa/manylinux1_x86_64:latest
 
 # Install dependencies
-RUN yum install -y flex zlib-devel
+RUN yum install -y flex zlib-devel && yum clean all
 
 ADD scripts/build_openssl.sh /
 RUN /build_openssl.sh
diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh
index 4650cde9532ea..1a6ffd7eb041f 100755
--- a/python/manylinux1/scripts/build_boost.sh
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -23,6 +23,16 @@ wget --no-check-certificate https://dl.bintray.com/boostorg/release/${BOOST_VERS
 tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
 pushd /boost_${BOOST_VERSION_UNDERSCORE}
 ./bootstrap.sh
-./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install
+./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install
 popd
 rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE}
+# Boost always install header-only parts but they also take up quite some space.
+# We don't need them in array, so don't persist them in the docker layer.
+# phoenix 18.1 MiB
+rm -r /usr/include/boost/phoenix
+# fusion 16.7 MiB
+rm -r /usr/include/boost/fusion
+# spirit 8.2 MiB
+rm -r /usr/include/boost/spirit
+# geometry 6.0 MiB
+rm -r /usr/include/boost/geometry
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
index ddedcf61fde05..e64157065ff0d 100755
--- a/python/manylinux1/scripts/build_virtualenvs.sh
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -44,3 +44,11 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     pip install pytest 'numpy==1.12.1' 'pandas==0.20.1'
     deactivate
 done
+
+# Remove pip cache again. It's useful during the virtualenv creation but we
+# don't want it persisted in the docker layer, ~264MiB
+rm -rf /root/.cache
+# Remove pandas' tests module as it includes a lot of data, ~27MiB per Python
+# venv, i.e. 216MiB in total
+rm -rf /opt/_internal/*/lib/*/site-packages/pandas/tests
+rm -rf /venv-test-*/lib/*/site-packages/pandas/tests

From 0d02a7d38a2cc43abb4a59006c19921f97e23f38 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Mon, 5 Feb 2018 17:36:49 -0500
Subject: [PATCH 1443/1644] ARROW-2080: [Python] Update documentation about
 pandas serialization context.

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1547 from robertnishihara/serializationdoc and squashes the following commits:

870ff6f8 [Robert Nishihara] Update documentation about pandas serialization context.
---
 python/doc/source/ipc.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index 6842cb5be9f43..9bf93ffe8ae8e 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -318,8 +318,8 @@ Serializing pandas Objects
 --------------------------
 
 We provide a serialization context that has optimized handling of pandas
-objects like ``DataFrame`` and ``Series``. This is the
-``pyarrow.pandas_serialization_context`` member. Combined with component-based
+objects like ``DataFrame`` and ``Series``. This can be created with
+``pyarrow.pandas_serialization_context()``. Combined with component-based
 serialization above, this enables zero-copy transport of pandas DataFrame
 objects not containing any Python objects:
 
@@ -327,7 +327,7 @@ objects not containing any Python objects:
 
    import pandas as pd
    df = pd.DataFrame({'a': [1, 2, 3, 4, 5]})
-   context = pa.pandas_serialization_context
+   context = pa.pandas_serialization_context()
    serialized_df = context.serialize(df)
    df_components = serialized_df.to_components()
    original_df = context.deserialize_components(df_components)

From 1a8c9a414e23cb52178458ec6f62b1a88e47f4b5 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 5 Feb 2018 19:27:42 -0500
Subject: [PATCH 1444/1644] ARROW-1976: [Python] Handling unicode pandas
 columns on parquet.read_table

Author: Phillip Cloud <cpcloud@gmail.com>
Author: Licht-T <licht-t@outlook.jp>
Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Simba Nyatsanga <simnyatsanga@gmail.com>

Closes #1553 from cpcloud/ARROW-1976 and squashes the following commits:

d8793f7b [Wes McKinney] Fix flakes
77cd95ba [Phillip Cloud] No need for additional function call
4f71b62c [Phillip Cloud] Make sure it's actually binary
6f8ad84a [Phillip Cloud] Fix binary on python3
e8d41545 [Phillip Cloud] Use _column_name_to_strings
210607f2 [Phillip Cloud] Add tests
40910cb0 [Phillip Cloud] Revert "Fix compat by using text_type"
b098d8a4 [Phillip Cloud] Fix compat by using text_type
a52f5c78 [Phillip Cloud] Revert
8773fadf [Phillip Cloud] Ignore pytest cache
db6176c1 [Simba Nyatsanga] Not using str with frombytes to ensure Python3 tests pass.
e9385c73 [Licht-T] BUG: Convert str by frombytes on pandas_compat.py
17f28b1a [Licht-T] TST: Add tests for Pandas data SerDe with Unicode column names
85c12315 [Licht-T] BUG: Fix Pandas data SerDe with Unicode column names in Python 2.7
---
 .gitignore                                  |  1 +
 python/pyarrow/pandas_compat.py             | 17 +++++++-----
 python/pyarrow/parquet.py                   |  4 +--
 python/pyarrow/tests/test_convert_pandas.py | 30 +++++++++++++++++++++
 4 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index e6dfe19bb9807..c38694e1f6647 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,4 @@ cpp/.idea/
 python/.eggs/
 .vscode
 .idea/
+.pytest_cache/
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 240cccdaffe56..987bb7555d972 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -170,9 +170,11 @@ def get_column_metadata(column, name, arrow_type, field_name):
             )
         )
 
+    assert field_name is None or isinstance(field_name, six.string_types), \
+        str(type(field_name))
     return {
         'name': name,
-        'field_name': str(field_name),
+        'field_name': 'None' if field_name is None else field_name,
         'pandas_type': logical_type,
         'numpy_type': string_dtype,
         'metadata': extra_metadata,
@@ -279,8 +281,11 @@ def _column_name_to_strings(name):
     """
     if isinstance(name, six.string_types):
         return name
+    elif isinstance(name, six.binary_type):
+        # XXX: should we assume that bytes in Python 3 are UTF-8?
+        return name.decode('utf8')
     elif isinstance(name, tuple):
-        return tuple(map(_column_name_to_strings, name))
+        return str(tuple(map(_column_name_to_strings, name)))
     elif isinstance(name, collections.Sequence):
         raise TypeError("Unsupported type for MultiIndex level")
     elif name is None:
@@ -327,10 +332,7 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1):
 
     for name in df.columns:
         col = df[name]
-        if not isinstance(name, six.string_types):
-            name = _column_name_to_strings(name)
-            if name is not None:
-                name = str(name)
+        name = _column_name_to_strings(name)
 
         if schema is not None:
             field = schema.field_by_name(name)
@@ -561,7 +563,8 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
     column_strings = [x.name for x in block_table.itercolumns()]
     if columns:
         columns_name_dict = {
-            c.get('field_name', str(c['name'])): c['name'] for c in columns
+            c.get('field_name', _column_name_to_strings(c['name'])): c['name']
+            for c in columns
         }
         columns_values = [
             columns_name_dict.get(name, name) for name in column_strings
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 8820b6b4abe75..494e65ebb9593 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -939,7 +939,7 @@ def write_table(table, where, row_group_size=None, version='1.0',
                 coerce_timestamps=None,
                 flavor=None, **kwargs):
     row_group_size = kwargs.pop('chunk_size', row_group_size)
-
+    use_int96 = use_deprecated_int96_timestamps
     try:
         with ParquetWriter(
                 where, table.schema,
@@ -948,7 +948,7 @@ def write_table(table, where, row_group_size=None, version='1.0',
                 use_dictionary=use_dictionary,
                 coerce_timestamps=coerce_timestamps,
                 compression=compression,
-                use_deprecated_int96_timestamps= use_deprecated_int96_timestamps, # noqa
+                use_deprecated_int96_timestamps=use_int96,
                 **kwargs) as writer:
             writer.write_table(table, row_group_size=row_group_size)
     except Exception:
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 31099072f29d2..4f0a68729bc73 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -156,6 +156,11 @@ def test_multiindex_columns_with_dtypes(self):
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
         _check_pandas_roundtrip(df, preserve_index=True)
 
+    def test_multiindex_columns_unicode(self):
+        columns = pd.MultiIndex.from_arrays([[u'あ', u'い'], ['X', 'Y']])
+        df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
+        _check_pandas_roundtrip(df, preserve_index=True)
+
     def test_integer_index_column(self):
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')])
         _check_pandas_roundtrip(df, preserve_index=True)
@@ -519,6 +524,31 @@ def test_unicode(self):
 
         _check_pandas_roundtrip(df, expected_schema=schema)
 
+    def test_unicode_with_unicode_column_and_index(self):
+        df = pd.DataFrame({u'あ': [u'い']}, index=[u'う'])
+
+        _check_pandas_roundtrip(df, preserve_index=True)
+
+    def test_mixed_unicode_column_names(self):
+        df = pd.DataFrame({u'あ': [u'い'], b'a': 1}, index=[u'う'])
+
+        # TODO(phillipc): Should this raise?
+        with pytest.raises(AssertionError):
+            _check_pandas_roundtrip(df, preserve_index=True)
+
+    def test_binary_column_name(self):
+        column_data = [u'い']
+        data = {u'あ'.encode('utf8'): column_data}
+        df = pd.DataFrame(data)
+
+        # we can't use _check_pandas_roundtrip here because our metdata
+        # is always decoded as utf8: even if binary goes in, utf8 comes out
+        t = pa.Table.from_pandas(df, preserve_index=True)
+        df2 = t.to_pandas()
+        assert df.values[0] == df2.values[0]
+        assert df.index.values[0] == df2.index.values[0]
+        assert df.columns[0] == df2.columns[0].encode('utf8')
+
     def test_bytes_to_binary(self):
         values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan]
         df = pd.DataFrame({'strings': values})

From 2a881a07e4b363a69743c5dd6853ed85060e6e19 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Tue, 6 Feb 2018 11:17:13 -0500
Subject: [PATCH 1445/1644] ARROW-2044: [JS] Typings should be a regular
 dependency

Made @types/{flatbuffers,node} regular dependencies so they will be included with release packages

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1524 from TheNeuralBit/types-deps and squashes the following commits:

e6b9ec95 [Brian Hulette] Make @types/{node,flatbuffers} regular dependencies
---
 js/package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/js/package.json b/js/package.json
index 1c8b23604ab85..e553b56cd0a10 100644
--- a/js/package.json
+++ b/js/package.json
@@ -52,6 +52,8 @@
     "npm-release.sh"
   ],
   "dependencies": {
+    "@types/flatbuffers": "1.6.5",
+    "@types/node": "9.3.0",
     "@types/text-encoding-utf-8": "1.0.1",
     "command-line-args": "5.0.1",
     "command-line-usage": "4.1.0",
@@ -62,10 +64,8 @@
   },
   "devDependencies": {
     "@std/esm": "0.19.7",
-    "@types/flatbuffers": "1.6.5",
     "@types/glob": "5.0.35",
     "@types/jest": "22.1.0",
-    "@types/node": "9.3.0",
     "ast-types": "0.10.1",
     "benchmark": "2.1.4",
     "coveralls": "3.0.0",

From 310e5b1683d91f76ba3ee7d6f2096fe07b03789a Mon Sep 17 00:00:00 2001
From: Panchen Xue <pan.panchen.xue@gmail.com>
Date: Tue, 6 Feb 2018 11:17:47 -0500
Subject: [PATCH 1446/1644] ARROW-1757: [C++] Add DictionaryArray::FromArrays
 alternate ctor that can check or sanitized "untrusted" indices

Add static member function DictionaryArray::FromArrays to create DictionaryArray from a given type and an indices array. This method calls DictionaryArray::SanityCheck to check if all indices are within valid range.

Author: Panchen Xue <pan.panchen.xue@gmail.com>

Closes #1535 from xuepanchen/ARROW-1757 and squashes the following commits:

88da7381 [Panchen Xue] Rename ValidateArray method and move to array.cc
93e26a64 [Panchen Xue] Modify code based on comments
ebb6897c [Panchen Xue] ARROW-1757: [C++] Lint format and add comment
e6801ea3 [Panchen Xue] ARROW-1757: [C++] Add test case for DictionaryArray::FromArrays()
5f4944c6 [Panchen Xue] ARROW-1757: [C++] Add DictionaryArray::FromArray and SanityCheck methods
---
 cpp/src/arrow/array-test.cc | 31 +++++++++++++++
 cpp/src/arrow/array.cc      | 75 ++++++++++++++++++++++++++++++++++++-
 cpp/src/arrow/array.h       | 13 +++++++
 3 files changed, 118 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index c53da8591e94e..fa64d467b695b 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -2384,6 +2384,37 @@ TEST(TestDictionary, Validate) {
   // ASSERT_OK(ValidateArray(*arr3));
 }
 
+TEST(TestDictionary, FromArray) {
+  std::shared_ptr<Array> dict;
+  vector<string> dict_values = {"foo", "bar", "baz"};
+  ArrayFromVector<StringType, string>(dict_values, &dict);
+  std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
+
+  std::shared_ptr<Array> indices1;
+  vector<int16_t> indices_values1 = {1, 2, 0, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(indices_values1, &indices1);
+
+  std::shared_ptr<Array> indices2;
+  vector<int16_t> indices_values2 = {1, 2, 0, 3, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(indices_values2, &indices2);
+
+  std::shared_ptr<Array> indices3;
+  vector<bool> is_valid3 = {true, true, false, true, true, true};
+  vector<int16_t> indices_values3 = {1, 2, -1, 0, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(is_valid3, indices_values3, &indices3);
+
+  std::shared_ptr<Array> indices4;
+  vector<bool> is_valid4 = {true, true, false, true, true, true};
+  vector<int16_t> indices_values4 = {1, 2, 1, 3, 2, 0};
+  ArrayFromVector<Int16Type, int16_t>(is_valid4, indices_values4, &indices4);
+
+  std::shared_ptr<Array> arr1, arr2, arr3, arr4;
+  ASSERT_OK(DictionaryArray::FromArrays(dict_type, indices1, &arr1));
+  ASSERT_RAISES(Invalid, DictionaryArray::FromArrays(dict_type, indices2, &arr2));
+  ASSERT_OK(DictionaryArray::FromArrays(dict_type, indices3, &arr3));
+  ASSERT_RAISES(Invalid, DictionaryArray::FromArrays(dict_type, indices4, &arr4));
+}
+
 // ----------------------------------------------------------------------
 // Struct tests
 
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 3d72761ed18e5..a8043d69b92f2 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -476,6 +476,39 @@ const Array* UnionArray::UnsafeChild(int i) const {
 // ----------------------------------------------------------------------
 // DictionaryArray
 
+/// \brief Perform validation check to determine if all dictionary indices
+/// are within valid range (0 <= index < upper_bound)
+///
+/// \param[in] indices array of dictionary indices
+/// \param[in] upper_bound upper bound of valid range for indices
+/// \return Status
+template <typename ArrowType>
+Status ValidateDictionaryIndices(const std::shared_ptr<Array>& indices,
+                                 const int64_t upper_bound) {
+  using ArrayType = typename TypeTraits<ArrowType>::ArrayType;
+  const auto& array = static_cast<const ArrayType&>(*indices);
+  const typename ArrowType::c_type* data = array.raw_values();
+  const int64_t size = array.length();
+
+  if (array.null_count() == 0) {
+    for (int64_t idx = 0; idx < size; ++idx) {
+      if (data[idx] < 0 || data[idx] >= upper_bound) {
+        return Status::Invalid("Dictionary has out-of-bound index [0, dict.length)");
+      }
+    }
+  } else {
+    for (int64_t idx = 0; idx < size; ++idx) {
+      if (!array.IsNull(idx)) {
+        if (data[idx] < 0 || data[idx] >= upper_bound) {
+          return Status::Invalid("Dictionary has out-of-bound index [0, dict.length)");
+        }
+      }
+    }
+  }
+
+  return Status::OK();
+}
+
 DictionaryArray::DictionaryArray(const std::shared_ptr<ArrayData>& data)
     : dict_type_(static_cast<const DictionaryType*>(data->type.get())) {
   DCHECK_EQ(data->type->id(), Type::DICTIONARY);
@@ -492,11 +525,51 @@ DictionaryArray::DictionaryArray(const std::shared_ptr<DataType>& type,
   SetData(data);
 }
 
+Status DictionaryArray::FromArrays(const std::shared_ptr<DataType>& type,
+                                   const std::shared_ptr<Array>& indices,
+                                   std::shared_ptr<Array>* out) {
+  if (indices->length() == 0) {
+    return Status::Invalid("Dictionary indices must have non-zero length");
+  }
+
+  DCHECK_EQ(type->id(), Type::DICTIONARY);
+  const auto& dict = static_cast<const DictionaryType&>(*type);
+  DCHECK_EQ(indices->type_id(), dict.index_type()->id());
+
+  int64_t upper_bound = dict.dictionary()->length();
+  Status is_valid;
+
+  switch (indices->type_id()) {
+    case Type::INT8:
+      is_valid = ValidateDictionaryIndices<Int8Type>(indices, upper_bound);
+      break;
+    case Type::INT16:
+      is_valid = ValidateDictionaryIndices<Int16Type>(indices, upper_bound);
+      break;
+    case Type::INT32:
+      is_valid = ValidateDictionaryIndices<Int32Type>(indices, upper_bound);
+      break;
+    case Type::INT64:
+      is_valid = ValidateDictionaryIndices<Int64Type>(indices, upper_bound);
+      break;
+    default:
+      std::stringstream ss;
+      ss << "Categorical index type not supported: " << indices->type()->ToString();
+      return Status::NotImplemented(ss.str());
+  }
+
+  if (!is_valid.ok()) {
+    return is_valid;
+  }
+
+  *out = std::make_shared<DictionaryArray>(type, indices);
+  return is_valid;
+}
+
 void DictionaryArray::SetData(const std::shared_ptr<ArrayData>& data) {
   this->Array::SetData(data);
   auto indices_data = data_->Copy();
   indices_data->type = dict_type_->index_type();
-  std::shared_ptr<Array> result;
   indices_ = MakeArray(indices_data);
 }
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index f0a786131b2b5..5b9ce9a01fbe4 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -726,6 +726,19 @@ class ARROW_EXPORT DictionaryArray : public Array {
   DictionaryArray(const std::shared_ptr<DataType>& type,
                   const std::shared_ptr<Array>& indices);
 
+  /// \brief Construct DictionaryArray from dictonary data type and indices array
+  ///
+  /// This function does the validation of the indices and input type. It checks if
+  /// all indices are non-negative and smaller than the size of the dictionary
+  ///
+  /// \param[in] type a data type containing a dictionary
+  /// \param[in] indices an array of non-negative signed
+  /// integers smaller than the size of the dictionary
+  /// \param[out] out the resulting DictionaryArray instance
+  static Status FromArrays(const std::shared_ptr<DataType>& type,
+                           const std::shared_ptr<Array>& indices,
+                           std::shared_ptr<Array>* out);
+
   std::shared_ptr<Array> indices() const;
   std::shared_ptr<Array> dictionary() const;
 

From 54859c57527076c375bc26b0e252f6bf1258cfbc Mon Sep 17 00:00:00 2001
From: William Paul <wapaul1@berkeley.edu>
Date: Tue, 6 Feb 2018 16:19:12 -0800
Subject: [PATCH 1447/1644] ARROW-1394: [Plasma] Add optional extension for
 allocating memory on GPUs

**Done:**
- CudaIPCMemHandles are now returned as shared pointers instead of unique pointers.
- Objects now have a device number; 0 for host memory, 1-infinity for GPU memory.
- After being allocated and exported on the store, CudaIPCMemHandles are sent using flatbuffers alongside the object metadata.
- Create and Get now return CudaBuffers for device numbers greater than zero, with the API change in #1444 .
- There is an issue with the same object on the GPU being retrieved multiples on the same process. CudaIPCMemHandles can only be mapped once per process, so to solve this, there is a process-global unordered map `gpu_object_map` of object id to a struct containing the mapped CudaBuffer and count of how many clients are using the object. Removing entries would be done when the count reaches zero on releasing the object.

**Todo:**
- The hash on the data done when the object is sealed is a constant zero for objects on the GPU.
- The eviction policy currently has no notion of total size on GPUs, so GPU objects will never be released or evicted.
- Similar to the last point, there is no configuration for how much memory to use on the GPU or what GPU's to use (though this can be resolved by `CUDA_VISIBLE_DEVICES`).

As a side note, it seems like what's currently done could be abstracted into supporting arbitrary devices that can ship memory handles, though that is out of scope for the ticket.

@pcmoritz @wesm

Author: William Paul <wapaul1@berkeley.edu>
Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1445 from Wapaul1/plasma_gpu_module and squashes the following commits:

d803e08b [Philipp Moritz] fixes
712e33b5 [Philipp Moritz] rebase and fixes
1a2b2fdd [William Paul] Changed CudaBuffer to MemHandle in store
45e7eefd [William Paul] Plasma GPU Module
8ad9949c [William Paul] Create and Get use Buffers now
---
 cpp/src/arrow/gpu/cuda-test.cc             |   4 +-
 cpp/src/arrow/gpu/cuda_context.cc          |   6 +-
 cpp/src/arrow/gpu/cuda_context.h           |   2 +-
 cpp/src/arrow/gpu/cuda_memory.cc           |   6 +-
 cpp/src/arrow/gpu/cuda_memory.h            |   4 +-
 cpp/src/plasma/CMakeLists.txt              |  19 ++-
 cpp/src/plasma/client.cc                   | 175 ++++++++++++++++-----
 cpp/src/plasma/client.h                    |  17 +-
 cpp/src/plasma/format/plasma.fbs           |  12 ++
 cpp/src/plasma/plasma.h                    |  18 +++
 cpp/src/plasma/protocol.cc                 |  80 ++++++++--
 cpp/src/plasma/protocol.h                  |   4 +-
 cpp/src/plasma/store.cc                    |  98 ++++++++----
 cpp/src/plasma/store.h                     |  12 +-
 cpp/src/plasma/test/client_tests.cc        |  78 +++++++++
 cpp/src/plasma/test/serialization_tests.cc |   9 +-
 16 files changed, 439 insertions(+), 105 deletions(-)

diff --git a/cpp/src/arrow/gpu/cuda-test.cc b/cpp/src/arrow/gpu/cuda-test.cc
index 7595f8bec7912..ae411c9b8886f 100644
--- a/cpp/src/arrow/gpu/cuda-test.cc
+++ b/cpp/src/arrow/gpu/cuda-test.cc
@@ -94,14 +94,14 @@ TEST_F(TestCudaBuffer, DISABLED_ExportForIpc) {
   ASSERT_OK(device_buffer->CopyFromHost(0, host_buffer->data(), kSize));
 
   // Export for IPC and serialize
-  std::unique_ptr<CudaIpcMemHandle> ipc_handle;
+  std::shared_ptr<CudaIpcMemHandle> ipc_handle;
   ASSERT_OK(device_buffer->ExportForIpc(&ipc_handle));
 
   std::shared_ptr<Buffer> serialized_handle;
   ASSERT_OK(ipc_handle->Serialize(default_memory_pool(), &serialized_handle));
 
   // Deserialize IPC handle and open
-  std::unique_ptr<CudaIpcMemHandle> ipc_handle2;
+  std::shared_ptr<CudaIpcMemHandle> ipc_handle2;
   ASSERT_OK(CudaIpcMemHandle::FromBuffer(serialized_handle->data(), &ipc_handle2));
 
   std::shared_ptr<CudaBuffer> ipc_buffer;
diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc
index 2f5ccb0a95063..909c98aa88ddd 100644
--- a/cpp/src/arrow/gpu/cuda_context.cc
+++ b/cpp/src/arrow/gpu/cuda_context.cc
@@ -89,11 +89,11 @@ class CudaContext::CudaContextImpl {
     return Status::OK();
   }
 
-  Status ExportIpcBuffer(void* data, std::unique_ptr<CudaIpcMemHandle>* handle) {
+  Status ExportIpcBuffer(void* data, std::shared_ptr<CudaIpcMemHandle>* handle) {
     CU_RETURN_NOT_OK(cuCtxSetCurrent(context_));
     CUipcMemHandle cu_handle;
     CU_RETURN_NOT_OK(cuIpcGetMemHandle(&cu_handle, reinterpret_cast<CUdeviceptr>(data)));
-    *handle = std::unique_ptr<CudaIpcMemHandle>(new CudaIpcMemHandle(&cu_handle));
+    *handle = std::shared_ptr<CudaIpcMemHandle>(new CudaIpcMemHandle(&cu_handle));
     return Status::OK();
   }
 
@@ -241,7 +241,7 @@ Status CudaContext::Allocate(int64_t nbytes, std::shared_ptr<CudaBuffer>* out) {
 }
 
 Status CudaContext::ExportIpcBuffer(void* data,
-                                    std::unique_ptr<CudaIpcMemHandle>* handle) {
+                                    std::shared_ptr<CudaIpcMemHandle>* handle) {
   return impl_->ExportIpcBuffer(data, handle);
 }
 
diff --git a/cpp/src/arrow/gpu/cuda_context.h b/cpp/src/arrow/gpu/cuda_context.h
index 6fc2e0d08abc9..50ea94c9b8a66 100644
--- a/cpp/src/arrow/gpu/cuda_context.h
+++ b/cpp/src/arrow/gpu/cuda_context.h
@@ -88,7 +88,7 @@ class ARROW_EXPORT CudaContext : public std::enable_shared_from_this<CudaContext
  private:
   CudaContext();
 
-  Status ExportIpcBuffer(void* data, std::unique_ptr<CudaIpcMemHandle>* handle);
+  Status ExportIpcBuffer(void* data, std::shared_ptr<CudaIpcMemHandle>* handle);
   Status CopyHostToDevice(void* dst, const void* src, int64_t nbytes);
   Status CopyDeviceToHost(void* dst, const void* src, int64_t nbytes);
   Status Free(void* device_ptr, int64_t nbytes);
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
index cbf044121a644..183cbcbc6bf2d 100644
--- a/cpp/src/arrow/gpu/cuda_memory.cc
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -54,8 +54,8 @@ CudaIpcMemHandle::CudaIpcMemHandle(const void* handle) {
 CudaIpcMemHandle::~CudaIpcMemHandle() {}
 
 Status CudaIpcMemHandle::FromBuffer(const void* opaque_handle,
-                                    std::unique_ptr<CudaIpcMemHandle>* handle) {
-  *handle = std::unique_ptr<CudaIpcMemHandle>(new CudaIpcMemHandle(opaque_handle));
+                                    std::shared_ptr<CudaIpcMemHandle>* handle) {
+  *handle = std::shared_ptr<CudaIpcMemHandle>(new CudaIpcMemHandle(opaque_handle));
   return Status::OK();
 }
 
@@ -111,7 +111,7 @@ Status CudaBuffer::CopyFromHost(const int64_t position, const void* data,
   return context_->CopyHostToDevice(mutable_data_ + position, data, nbytes);
 }
 
-Status CudaBuffer::ExportForIpc(std::unique_ptr<CudaIpcMemHandle>* handle) {
+Status CudaBuffer::ExportForIpc(std::shared_ptr<CudaIpcMemHandle>* handle) {
   if (is_ipc_) {
     return Status::Invalid("Buffer has already been exported for IPC");
   }
diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h
index 9376b4b3ffcb9..3f3dd2f6c452b 100644
--- a/cpp/src/arrow/gpu/cuda_memory.h
+++ b/cpp/src/arrow/gpu/cuda_memory.h
@@ -64,7 +64,7 @@ class ARROW_EXPORT CudaBuffer : public Buffer {
   ///
   /// \note After calling this function, this device memory will not be freed
   /// when the CudaBuffer is destructed
-  virtual Status ExportForIpc(std::unique_ptr<CudaIpcMemHandle>* handle);
+  virtual Status ExportForIpc(std::shared_ptr<CudaIpcMemHandle>* handle);
 
   std::shared_ptr<CudaContext> context() const { return context_; }
 
@@ -95,7 +95,7 @@ class ARROW_EXPORT CudaIpcMemHandle {
   /// \param[out] handle the CudaIpcMemHandle instance
   /// \return Status
   static Status FromBuffer(const void* opaque_handle,
-                           std::unique_ptr<CudaIpcMemHandle>* handle);
+                           std::shared_ptr<CudaIpcMemHandle>* handle);
 
   /// \brief Write CudaIpcMemHandle to a Buffer
   /// \param[in] pool a MemoryPool to allocate memory from
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 4b76f25ba6e32..3448d009c9c89 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -79,11 +79,20 @@ set(PLASMA_SRCS
   thirdparty/ae/ae.c
   thirdparty/xxhash.cc)
 
+set(PLASMA_LINK_LIBS arrow_static)
+
+if (ARROW_GPU)
+  set(PLASMA_LINK_LIBS ${PLASMA_LINK_LIBS} arrow_gpu_shared)
+  add_definitions(-DPLASMA_GPU)
+endif()
+
+
+
 ADD_ARROW_LIB(plasma
   SOURCES ${PLASMA_SRCS}
   DEPENDENCIES gen_plasma_fbs
-  SHARED_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static
-  STATIC_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static)
+  SHARED_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} ${PLASMA_LINK_LIBS}
+  STATIC_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} ${PLASMA_LINK_LIBS})
 
 # The optimization flag -O3 is suggested by dlmalloc.c, which is #included in
 # malloc.cc; we set it here regardless of whether we do a debug or release build.
@@ -113,7 +122,7 @@ if ("${COMPILER_FAMILY}" STREQUAL "gcc")
 endif()
 
 add_executable(plasma_store store.cc)
-target_link_libraries(plasma_store plasma_static)
+target_link_libraries(plasma_store plasma_static ${PLASMA_LINK_LIBS})
 
 # Headers: top level
 install(FILES
@@ -143,6 +152,6 @@ install(
 #######################################
 
 ADD_ARROW_TEST(test/serialization_tests)
-ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static)
+ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static ${PLASMA_LINK_LIBS})
 ADD_ARROW_TEST(test/client_tests)
-ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static)
+ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static ${PLASMA_LINK_LIBS})
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 6e9b6968a8673..679d9cecda8f1 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -37,6 +37,7 @@
 #include <unistd.h>
 
 #include <algorithm>
+#include <mutex>
 #include <thread>
 #include <vector>
 
@@ -47,6 +48,15 @@
 #include "plasma/plasma.h"
 #include "plasma/protocol.h"
 
+#ifdef PLASMA_GPU
+#include "arrow/gpu/cuda_api.h"
+
+using arrow::gpu::CudaBuffer;
+using arrow::gpu::CudaBufferWriter;
+using arrow::gpu::CudaContext;
+using arrow::gpu::CudaDeviceManager;
+#endif
+
 #define XXH_STATIC_LINKING_ONLY
 #include "thirdparty/xxhash.h"
 
@@ -75,7 +85,26 @@ struct ObjectInUseEntry {
   bool is_sealed;
 };
 
-PlasmaClient::PlasmaClient() {}
+#ifdef PLASMA_GPU
+struct GpuProcessHandle {
+  /// Pointer to CUDA buffer that is backing this GPU object.
+  std::shared_ptr<CudaBuffer> ptr;
+  /// Number of client using this GPU object.
+  int client_count;
+};
+
+// This is necessary as IPC handles can only be mapped once per process.
+// Thus if multiple clients in the same process get the same gpu object,
+// they need to access the same mapped CudaBuffer.
+static std::unordered_map<ObjectID, GpuProcessHandle*, UniqueIDHasher> gpu_object_map;
+static std::mutex gpu_mutex;
+#endif
+
+PlasmaClient::PlasmaClient() {
+#ifdef PLASMA_GPU
+  CudaDeviceManager::GetInstance(&manager_);
+#endif
+}
 
 PlasmaClient::~PlasmaClient() {}
 
@@ -127,16 +156,18 @@ void PlasmaClient::increment_object_count(const ObjectID& object_id, PlasmaObjec
     objects_in_use_[object_id]->count = 0;
     objects_in_use_[object_id]->is_sealed = is_sealed;
     object_entry = objects_in_use_[object_id].get();
-    // Increment the count of the number of objects in the memory-mapped file
-    // that are being used. The corresponding decrement should happen in
-    // PlasmaClient::Release.
-    auto entry = mmap_table_.find(object->store_fd);
-    ARROW_CHECK(entry != mmap_table_.end());
-    ARROW_CHECK(entry->second.count >= 0);
-    // Update the in_use_object_bytes_.
-    in_use_object_bytes_ +=
-        (object_entry->object.data_size + object_entry->object.metadata_size);
-    entry->second.count += 1;
+    if (object->device_num == 0) {
+      // Increment the count of the number of objects in the memory-mapped file
+      // that are being used. The corresponding decrement should happen in
+      // PlasmaClient::Release.
+      auto entry = mmap_table_.find(object->store_fd);
+      ARROW_CHECK(entry != mmap_table_.end());
+      ARROW_CHECK(entry->second.count >= 0);
+      // Update the in_use_object_bytes_.
+      in_use_object_bytes_ +=
+          (object_entry->object.data_size + object_entry->object.metadata_size);
+      entry->second.count += 1;
+    }
   } else {
     object_entry = elem->second.get();
     ARROW_CHECK(object_entry->count > 0);
@@ -149,10 +180,11 @@ void PlasmaClient::increment_object_count(const ObjectID& object_id, PlasmaObjec
 
 Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
                             uint8_t* metadata, int64_t metadata_size,
-                            std::shared_ptr<Buffer>* data) {
+                            std::shared_ptr<Buffer>* data, int device_num) {
   ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
                    << data_size << " and metadata size " << metadata_size;
-  RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
+  RETURN_NOT_OK(
+      SendCreateRequest(store_conn_, object_id, data_size, metadata_size, device_num));
   std::vector<uint8_t> buffer;
   RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaCreateReply, &buffer));
   ObjectID id;
@@ -163,21 +195,41 @@ Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
       ReadCreateReply(buffer.data(), buffer.size(), &id, &object, &store_fd, &mmap_size));
   // If the CreateReply included an error, then the store will not send a file
   // descriptor.
-  int fd = recv_fd(store_conn_);
-  ARROW_CHECK(fd >= 0) << "recv not successful";
-  ARROW_CHECK(object.data_size == data_size);
-  ARROW_CHECK(object.metadata_size == metadata_size);
-  // The metadata should come right after the data.
-  ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
-  *data = std::make_shared<MutableBuffer>(
-      lookup_or_mmap(fd, store_fd, mmap_size) + object.data_offset, data_size);
-  // If plasma_create is being called from a transfer, then we will not copy the
-  // metadata here. The metadata will be written along with the data streamed
-  // from the transfer.
-  if (metadata != NULL) {
-    // Copy the metadata to the buffer.
-    memcpy((*data)->mutable_data() + object.data_size, metadata, metadata_size);
+  if (device_num == 0) {
+    int fd = recv_fd(store_conn_);
+    ARROW_CHECK(fd >= 0) << "recv not successful";
+    ARROW_CHECK(object.data_size == data_size);
+    ARROW_CHECK(object.metadata_size == metadata_size);
+    // The metadata should come right after the data.
+    ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
+    *data = std::make_shared<MutableBuffer>(
+        lookup_or_mmap(fd, store_fd, mmap_size) + object.data_offset, data_size);
+    // If plasma_create is being called from a transfer, then we will not copy the
+    // metadata here. The metadata will be written along with the data streamed
+    // from the transfer.
+    if (metadata != NULL) {
+      // Copy the metadata to the buffer.
+      memcpy((*data)->mutable_data() + object.data_size, metadata, metadata_size);
+    }
+  } else {
+#ifdef PLASMA_GPU
+    std::lock_guard<std::mutex> lock(gpu_mutex);
+    std::shared_ptr<CudaContext> context;
+    RETURN_NOT_OK(manager_->GetContext(device_num - 1, &context));
+    GpuProcessHandle* handle = new GpuProcessHandle();
+    RETURN_NOT_OK(context->OpenIpcBuffer(*object.ipc_handle, &handle->ptr));
+    gpu_object_map[object_id] = handle;
+    *data = handle->ptr;
+    if (metadata != NULL) {
+      // Copy the metadata to the buffer.
+      CudaBufferWriter writer(std::dynamic_pointer_cast<CudaBuffer>(*data));
+      writer.WriteAt(object.data_size, metadata, metadata_size);
+    }
+#else
+    ARROW_LOG(FATAL) << "Arrow GPU library is not enabled.";
+#endif
   }
+
   // Increment the count of the number of instances of this object that this
   // client is using. A call to PlasmaClient::Release is required to decrement
   // this
@@ -210,13 +262,27 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
       ARROW_CHECK(object_entry->second->is_sealed)
           << "Plasma client called get on an unsealed object that it created";
       PlasmaObject* object = &object_entry->second->object;
-      uint8_t* data = lookup_mmapped_file(object->store_fd);
-      object_buffers[i].data =
-          std::make_shared<Buffer>(data + object->data_offset, object->data_size);
-      object_buffers[i].metadata = std::make_shared<Buffer>(
-          data + object->data_offset + object->data_size, object->metadata_size);
+      if (object->device_num == 0) {
+        uint8_t* data = lookup_mmapped_file(object->store_fd);
+        object_buffers[i].data =
+            std::make_shared<Buffer>(data + object->data_offset, object->data_size);
+        object_buffers[i].metadata = std::make_shared<Buffer>(
+            data + object->data_offset + object->data_size, object->metadata_size);
+      } else {
+#ifdef PLASMA_GPU
+        std::shared_ptr<CudaBuffer> gpu_handle =
+            gpu_object_map.find(object_ids[i])->second->ptr;
+        object_buffers[i].data =
+            std::make_shared<CudaBuffer>(gpu_handle, 0, object->data_size);
+        object_buffers[i].metadata = std::make_shared<CudaBuffer>(
+            gpu_handle, object->data_size, object->metadata_size);
+#else
+        ARROW_LOG(FATAL) << "Arrow GPU library is not enabled.";
+#endif
+      }
       object_buffers[i].data_size = object->data_size;
       object_buffers[i].metadata_size = object->metadata_size;
+      object_buffers[i].device_num = object->device_num;
       // Increment the count of the number of instances of this object that this
       // client is using. A call to PlasmaClient::Release is required to
       // decrement this
@@ -265,14 +331,40 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
     // If we are here, the object was not currently in use, so we need to
     // process the reply from the object store.
     if (object->data_size != -1) {
-      uint8_t* data = lookup_mmapped_file(object->store_fd);
-      // Finish filling out the return values.
-      object_buffers[i].data =
-          std::make_shared<Buffer>(data + object->data_offset, object->data_size);
-      object_buffers[i].metadata = std::make_shared<Buffer>(
-          data + object->data_offset + object->data_size, object->metadata_size);
+      if (object->device_num == 0) {
+        uint8_t* data = lookup_mmapped_file(object->store_fd);
+        // Finish filling out the return values.
+        object_buffers[i].data =
+            std::make_shared<Buffer>(data + object->data_offset, object->data_size);
+        object_buffers[i].metadata = std::make_shared<Buffer>(
+            data + object->data_offset + object->data_size, object->metadata_size);
+      } else {
+#ifdef PLASMA_GPU
+        std::lock_guard<std::mutex> lock(gpu_mutex);
+        auto handle = gpu_object_map.find(object_ids[i]);
+        std::shared_ptr<CudaBuffer> gpu_handle;
+        if (handle == gpu_object_map.end()) {
+          std::shared_ptr<CudaContext> context;
+          RETURN_NOT_OK(manager_->GetContext(object->device_num - 1, &context));
+          GpuProcessHandle* obj_handle = new GpuProcessHandle();
+          RETURN_NOT_OK(context->OpenIpcBuffer(*object->ipc_handle, &obj_handle->ptr));
+          gpu_object_map[object_ids[i]] = obj_handle;
+          gpu_handle = obj_handle->ptr;
+        } else {
+          handle->second->client_count += 1;
+          gpu_handle = handle->second->ptr;
+        }
+        object_buffers[i].data =
+            std::make_shared<CudaBuffer>(gpu_handle, 0, object->data_size);
+        object_buffers[i].metadata = std::make_shared<CudaBuffer>(
+            gpu_handle, object->data_size, object->metadata_size);
+#else
+        ARROW_LOG(FATAL) << "Arrow GPU library is not enabled.";
+#endif
+      }
       object_buffers[i].data_size = object->data_size;
       object_buffers[i].metadata_size = object->metadata_size;
+      object_buffers[i].device_num = object->device_num;
       // Increment the count of the number of instances of this object that this
       // client is using. A call to PlasmaClient::Release is required to
       // decrement this
@@ -358,6 +450,9 @@ Status PlasmaClient::Release(const ObjectID& object_id) {
   // If there are too many bytes in use by the client or if there are too many
   // pending release calls, and there are at least some pending release calls in
   // the release_history list, then release some objects.
+
+  // TODO(wap) Evicition policy only works on host memory, and thus objects
+  //           on the GPU cannot be released currently.
   while ((in_use_object_bytes_ > std::min(kL3CacheSizeBytes, store_capacity_ / 100) ||
           release_history_.size() > config_.release_delay) &&
          release_history_.size() > 0) {
@@ -447,6 +542,10 @@ static inline bool compute_object_hash_parallel(XXH64_state_t* hash_state,
 
 static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
   XXH64_state_t hash_state;
+  if (obj_buffer.device_num != 0) {
+    // TODO(wap): Create cuda program to hash data on gpu.
+    return 0;
+  }
   XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
   if (obj_buffer.data_size >= kBytesInMB) {
     compute_object_hash_parallel(
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
index d6372f44a7f28..7c27c474dea39 100644
--- a/cpp/src/plasma/client.h
+++ b/cpp/src/plasma/client.h
@@ -30,6 +30,9 @@
 #include "arrow/status.h"
 #include "arrow/util/visibility.h"
 #include "plasma/common.h"
+#ifdef PLASMA_GPU
+#include "arrow/gpu/cuda_api.h"
+#endif
 
 using arrow::Buffer;
 using arrow::Status;
@@ -111,11 +114,15 @@ class ARROW_EXPORT PlasmaClient {
   ///        should be NULL.
   /// \param metadata_size The size in bytes of the metadata. If there is no
   ///        metadata, this should be 0.
-  /// \param data A buffer containing the address of the newly created object
-  ///        will be written here.
+  /// \param data The address of the newly created object will be written here.
+  /// \param device_num The number of the device where the object is being
+  ///        created.
+  ///        device_num = 0 corresponds to the host,
+  ///        device_num = 1 corresponds to GPU0,
+  ///        device_num = 2 corresponds to GPU1, etc.
   /// \return The return status.
   Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
-                int64_t metadata_size, std::shared_ptr<Buffer>* data);
+                int64_t metadata_size, std::shared_ptr<Buffer>* data, int device_num = 0);
   /// Get some objects from the Plasma Store. This function will block until the
   /// objects have all been created and sealed in the Plasma Store or the
   /// timeout
@@ -368,6 +375,10 @@ class ARROW_EXPORT PlasmaClient {
   /// information to make sure that it does not delay in releasing so much
   /// memory that the store is unable to evict enough objects to free up space.
   int64_t store_capacity_;
+#ifdef PLASMA_GPU
+  /// Cuda Device Manager.
+  arrow::gpu::CudaDeviceManager* manager_;
+#endif
 };
 
 }  // namespace plasma
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
index 33803f7799ba0..66cb00f777239 100644
--- a/cpp/src/plasma/format/plasma.fbs
+++ b/cpp/src/plasma/format/plasma.fbs
@@ -97,6 +97,8 @@ struct PlasmaObjectSpec {
   metadata_offset: ulong;
   // The size in bytes of the metadata.
   metadata_size: ulong;
+  // Device to create buffer on.
+  device_num: int;
 }
 
 table PlasmaCreateRequest {
@@ -106,6 +108,12 @@ table PlasmaCreateRequest {
   data_size: ulong;
   // The size of the object's metadata in bytes.
   metadata_size: ulong;
+  // Device to create buffer on.
+  device_num: int;
+}
+
+table CudaHandle {
+  handle: [ubyte];
 }
 
 table PlasmaCreateReply {
@@ -121,6 +129,8 @@ table PlasmaCreateReply {
   // The size in bytes of the segment for the store file descriptor (needed to
   // call mmap).
   mmap_size: long;
+  // CUDA IPC Handle for objects on GPU.
+  ipc_handle: CudaHandle;
 }
 
 table PlasmaAbortRequest {
@@ -171,6 +181,8 @@ table PlasmaGetReply {
   // Size in bytes of the segment for each store file descriptor (needed to call
   // mmap). This list must have the same length as store_fds.
   mmap_sizes: [long];
+  // The number of elements in both object_ids and plasma_objects arrays must agree.
+  handles: [CudaHandle];
 }
 
 table PlasmaReleaseRequest {
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
index bb9cdae601146..901601f4747eb 100644
--- a/cpp/src/plasma/plasma.h
+++ b/cpp/src/plasma/plasma.h
@@ -39,6 +39,12 @@
 #include "plasma/common.h"
 #include "plasma/common_generated.h"
 
+#ifdef PLASMA_GPU
+#include "arrow/gpu/cuda_api.h"
+
+using arrow::gpu::CudaIpcMemHandle;
+#endif
+
 namespace plasma {
 
 #define HANDLE_SIGPIPE(s, fd_)                                              \
@@ -67,6 +73,10 @@ typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectReques
 
 // TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
 struct PlasmaObject {
+#ifdef PLASMA_GPU
+  // IPC handle for Cuda.
+  std::shared_ptr<CudaIpcMemHandle> ipc_handle;
+#endif
   /// The file descriptor of the memory mapped file in the store. It is used as
   /// a unique identifier of the file in the client to look up the corresponding
   /// file descriptor on the client's side.
@@ -79,6 +89,8 @@ struct PlasmaObject {
   int64_t data_size;
   /// The size in bytes of the metadata.
   int64_t metadata_size;
+  /// Device number object is on.
+  int device_num;
 };
 
 enum object_state {
@@ -104,12 +116,18 @@ struct ObjectTableEntry {
   ObjectInfoT info;
   /// Memory mapped file containing the object.
   int fd;
+  /// Device number.
+  int device_num;
   /// Size of the underlying map.
   int64_t map_size;
   /// Offset from the base of the mmap.
   ptrdiff_t offset;
   /// Pointer to the object data. Needed to free the object.
   uint8_t* pointer;
+#ifdef PLASMA_GPU
+  /// IPC GPU handle to share with clients.
+  std::shared_ptr<CudaIpcMemHandle> ipc_handle;
+#endif
   /// Set of clients currently using this object.
   std::unordered_set<Client*> clients;
   /// The state of the object, e.g., whether it is open or sealed.
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
index 6c0bc0cab28bb..9443762ec6f80 100644
--- a/cpp/src/plasma/protocol.cc
+++ b/cpp/src/plasma/protocol.cc
@@ -23,6 +23,10 @@
 #include "plasma/common.h"
 #include "plasma/io.h"
 
+#ifdef ARROW_GPU
+#include "arrow/gpu/cuda_api.h"
+#endif
+
 namespace plasma {
 
 using flatbuffers::uoffset_t;
@@ -55,21 +59,22 @@ Status PlasmaSend(int sock, int64_t message_type, flatbuffers::FlatBufferBuilder
 // Create messages.
 
 Status SendCreateRequest(int sock, ObjectID object_id, int64_t data_size,
-                         int64_t metadata_size) {
+                         int64_t metadata_size, int device_num) {
   flatbuffers::FlatBufferBuilder fbb;
   auto message = CreatePlasmaCreateRequest(fbb, fbb.CreateString(object_id.binary()),
-                                           data_size, metadata_size);
+                                           data_size, metadata_size, device_num);
   return PlasmaSend(sock, MessageType_PlasmaCreateRequest, &fbb, message);
 }
 
 Status ReadCreateRequest(uint8_t* data, size_t size, ObjectID* object_id,
-                         int64_t* data_size, int64_t* metadata_size) {
+                         int64_t* data_size, int64_t* metadata_size, int* device_num) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
   DCHECK(verify_flatbuffer(message, data, size));
   *data_size = message->data_size();
   *metadata_size = message->metadata_size();
   *object_id = ObjectID::from_binary(message->object_id()->str());
+  *device_num = message->device_num();
   return Status::OK();
 }
 
@@ -77,10 +82,31 @@ Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int e
                        int64_t mmap_size) {
   flatbuffers::FlatBufferBuilder fbb;
   PlasmaObjectSpec plasma_object(object->store_fd, object->data_offset, object->data_size,
-                                 object->metadata_offset, object->metadata_size);
-  auto message = CreatePlasmaCreateReply(
-      fbb, fbb.CreateString(object_id.binary()), &plasma_object,
-      static_cast<PlasmaError>(error_code), object->store_fd, mmap_size);
+                                 object->metadata_offset, object->metadata_size,
+                                 object->device_num);
+  auto object_string = fbb.CreateString(object_id.binary());
+#ifdef PLASMA_GPU
+  flatbuffers::Offset<CudaHandle> ipc_handle;
+  if (object->device_num != 0) {
+    std::shared_ptr<arrow::Buffer> handle;
+    object->ipc_handle->Serialize(arrow::default_memory_pool(), &handle);
+    ipc_handle = CreateCudaHandle(fbb, fbb.CreateVector(handle->data(), handle->size()));
+  }
+#endif
+  PlasmaCreateReplyBuilder crb(fbb);
+  crb.add_error(static_cast<PlasmaError>(error_code));
+  crb.add_plasma_object(&plasma_object);
+  crb.add_object_id(object_string);
+  crb.add_store_fd(object->store_fd);
+  crb.add_mmap_size(mmap_size);
+  if (object->device_num != 0) {
+#ifdef PLASMA_GPU
+    crb.add_ipc_handle(ipc_handle);
+#else
+    ARROW_LOG(FATAL) << "This should be unreachable.";
+#endif
+  }
+  auto message = crb.Finish();
   return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
 }
 
@@ -99,6 +125,13 @@ Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id,
   *store_fd = message->store_fd();
   *mmap_size = message->mmap_size();
 
+  object->device_num = message->plasma_object()->device_num();
+#ifdef PLASMA_GPU
+  if (object->device_num != 0) {
+    CudaIpcMemHandle::FromBuffer(message->ipc_handle()->handle()->data(),
+                                 &object->ipc_handle);
+  }
+#endif
   return plasma_error_status(message->error());
 }
 
@@ -396,18 +429,25 @@ Status SendGetReply(
   flatbuffers::FlatBufferBuilder fbb;
   std::vector<PlasmaObjectSpec> objects;
 
-  ARROW_CHECK(store_fds.size() == mmap_sizes.size());
-
+  std::vector<flatbuffers::Offset<CudaHandle>> handles;
   for (int64_t i = 0; i < num_objects; ++i) {
     const PlasmaObject& object = plasma_objects[object_ids[i]];
     objects.push_back(PlasmaObjectSpec(object.store_fd, object.data_offset,
                                        object.data_size, object.metadata_offset,
-                                       object.metadata_size));
+                                       object.metadata_size, object.device_num));
+#ifdef PLASMA_GPU
+    if (object.device_num != 0) {
+      std::shared_ptr<arrow::Buffer> handle;
+      object.ipc_handle->Serialize(arrow::default_memory_pool(), &handle);
+      handles.push_back(
+          CreateCudaHandle(fbb, fbb.CreateVector(handle->data(), handle->size())));
+    }
+#endif
   }
-  auto message =
-      CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
-                           fbb.CreateVectorOfStructs(objects.data(), num_objects),
-                           fbb.CreateVector(store_fds), fbb.CreateVector(mmap_sizes));
+  auto message = CreatePlasmaGetReply(
+      fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+      fbb.CreateVectorOfStructs(objects.data(), num_objects), fbb.CreateVector(store_fds),
+      fbb.CreateVector(mmap_sizes), fbb.CreateVector(handles));
   return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
 }
 
@@ -416,6 +456,9 @@ Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
                     std::vector<int>& store_fds, std::vector<int64_t>& mmap_sizes) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
+#ifdef PLASMA_GPU
+  int handle_pos = 0;
+#endif
   DCHECK(verify_flatbuffer(message, data, size));
   for (uoffset_t i = 0; i < num_objects; ++i) {
     object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
@@ -427,6 +470,14 @@ Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
     plasma_objects[i].data_size = object->data_size();
     plasma_objects[i].metadata_offset = object->metadata_offset();
     plasma_objects[i].metadata_size = object->metadata_size();
+    plasma_objects[i].device_num = object->device_num();
+#ifdef PLASMA_GPU
+    if (object->device_num() != 0) {
+      CudaIpcMemHandle::FromBuffer(message->handles()->Get(handle_pos)->handle()->data(),
+                                   &plasma_objects[i].ipc_handle);
+      handle_pos++;
+    }
+#endif
   }
   ARROW_CHECK(message->store_fds()->size() == message->mmap_sizes()->size());
   for (uoffset_t i = 0; i < message->store_fds()->size(); i++) {
@@ -435,7 +486,6 @@ Status ReadGetReply(uint8_t* data, size_t size, ObjectID object_ids[],
   }
   return Status::OK();
 }
-
 // Fetch messages.
 
 Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
index 101a3faa7675e..86b3577c72649 100644
--- a/cpp/src/plasma/protocol.h
+++ b/cpp/src/plasma/protocol.h
@@ -43,10 +43,10 @@ Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffe
 /* Plasma Create message functions. */
 
 Status SendCreateRequest(int sock, ObjectID object_id, int64_t data_size,
-                         int64_t metadata_size);
+                         int64_t metadata_size, int device_num);
 
 Status ReadCreateRequest(uint8_t* data, size_t size, ObjectID* object_id,
-                         int64_t* data_size, int64_t* metadata_size);
+                         int64_t* data_size, int64_t* metadata_size, int* device_num);
 
 Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error,
                        int64_t mmap_size);
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
index 316a27f63f680..5e7b452aee533 100644
--- a/cpp/src/plasma/store.cc
+++ b/cpp/src/plasma/store.cc
@@ -57,6 +57,14 @@
 #include "plasma/io.h"
 #include "plasma/malloc.h"
 
+#ifdef PLASMA_GPU
+#include "arrow/gpu/cuda_api.h"
+
+using arrow::gpu::CudaBuffer;
+using arrow::gpu::CudaContext;
+using arrow::gpu::CudaDeviceManager;
+#endif
+
 namespace plasma {
 
 extern "C" {
@@ -104,6 +112,9 @@ PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory, std::string dir
   store_info_.memory_capacity = system_memory;
   store_info_.directory = directory;
   store_info_.hugepages_enabled = hugepages_enabled;
+#ifdef PLASMA_GPU
+  CudaDeviceManager::GetInstance(&manager_);
+#endif
 }
 
 // TODO(pcm): Get rid of this destructor by using RAII to clean up data.
@@ -142,7 +153,7 @@ void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client*
 
 // Create a new object buffer in the hash table.
 int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
-                               int64_t metadata_size, Client* client,
+                               int64_t metadata_size, int device_num, Client* client,
                                PlasmaObject* result) {
   ARROW_LOG(DEBUG) << "creating object " << object_id.hex();
   if (store_info_.objects.count(object_id) != 0) {
@@ -152,7 +163,14 @@ int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
   }
   // Try to evict objects until there is enough space.
   uint8_t* pointer;
-  do {
+#ifdef PLASMA_GPU
+  std::shared_ptr<CudaBuffer> gpu_handle;
+  std::shared_ptr<CudaContext> context_;
+  if (device_num != 0) {
+    manager_->GetContext(device_num - 1, &context_);
+  }
+#endif
+  while (true) {
     // Allocate space for the new object. We use dlmemalign instead of dlmalloc
     // in order to align the allocated region to a 64-byte boundary. This is not
     // strictly necessary, but it is an optimization that could speed up the
@@ -160,27 +178,37 @@ int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
     // plasma_client.cc). Note that even though this pointer is 64-byte aligned,
     // it is not guaranteed that the corresponding pointer in the client will be
     // 64-byte aligned, but in practice it often will be.
-    pointer =
-        reinterpret_cast<uint8_t*>(dlmemalign(BLOCK_SIZE, data_size + metadata_size));
-    if (pointer == NULL) {
-      // Tell the eviction policy how much space we need to create this object.
-      std::vector<ObjectID> objects_to_evict;
-      bool success =
-          eviction_policy_.require_space(data_size + metadata_size, &objects_to_evict);
-      delete_objects(objects_to_evict);
-      // Return an error to the client if not enough space could be freed to
-      // create the object.
-      if (!success) {
-        return PlasmaError_OutOfMemory;
+    if (device_num == 0) {
+      pointer =
+          reinterpret_cast<uint8_t*>(dlmemalign(BLOCK_SIZE, data_size + metadata_size));
+      if (pointer == NULL) {
+        // Tell the eviction policy how much space we need to create this object.
+        std::vector<ObjectID> objects_to_evict;
+        bool success =
+            eviction_policy_.require_space(data_size + metadata_size, &objects_to_evict);
+        delete_objects(objects_to_evict);
+        // Return an error to the client if not enough space could be freed to
+        // create the object.
+        if (!success) {
+          return PlasmaError_OutOfMemory;
+        }
+      } else {
+        break;
       }
+    } else {
+#ifdef PLASMA_GPU
+      context_->Allocate(data_size + metadata_size, &gpu_handle);
+      break;
+#endif
     }
-  } while (pointer == NULL);
-  int fd;
-  int64_t map_size;
-  ptrdiff_t offset;
-  get_malloc_mapinfo(pointer, &fd, &map_size, &offset);
-  assert(fd != -1);
-
+  }
+  int fd = -1;
+  int64_t map_size = 0;
+  ptrdiff_t offset = 0;
+  if (device_num == 0) {
+    get_malloc_mapinfo(pointer, &fd, &map_size, &offset);
+    assert(fd != -1);
+  }
   auto entry = std::unique_ptr<ObjectTableEntry>(new ObjectTableEntry());
   entry->object_id = object_id;
   entry->info.object_id = object_id.binary();
@@ -192,13 +220,20 @@ int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
   entry->map_size = map_size;
   entry->offset = offset;
   entry->state = PLASMA_CREATED;
-
+  entry->device_num = device_num;
+#ifdef PLASMA_GPU
+  if (device_num != 0) {
+    gpu_handle->ExportForIpc(&entry->ipc_handle);
+    result->ipc_handle = entry->ipc_handle;
+  }
+#endif
   store_info_.objects[object_id] = std::move(entry);
   result->store_fd = fd;
   result->data_offset = offset;
   result->metadata_offset = offset + data_size;
   result->data_size = data_size;
   result->metadata_size = metadata_size;
+  result->device_num = device_num;
   // Notify the eviction policy that this object was created. This must be done
   // immediately before the call to add_client_to_object_clients so that the
   // eviction policy does not have an opportunity to evict the object.
@@ -212,11 +247,17 @@ void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
   DCHECK(object != NULL);
   DCHECK(entry != NULL);
   DCHECK(entry->state == PLASMA_SEALED);
+#ifdef PLASMA_GPU
+  if (entry->device_num != 0) {
+    object->ipc_handle = entry->ipc_handle;
+  }
+#endif
   object->store_fd = entry->fd;
   object->data_offset = entry->offset;
   object->metadata_offset = entry->offset + entry->info.data_size;
   object->data_size = entry->info.data_size;
   object->metadata_size = entry->info.metadata_size;
+  object->device_num = entry->device_num;
 }
 
 void PlasmaStore::return_from_get(GetRequest* get_req) {
@@ -227,7 +268,7 @@ void PlasmaStore::return_from_get(GetRequest* get_req) {
   for (const auto& object_id : get_req->object_ids) {
     PlasmaObject& object = get_req->objects[object_id];
     int fd = object.store_fd;
-    if (object.data_size != -1 && fds_to_send.count(fd) == 0) {
+    if (object.data_size != -1 && fds_to_send.count(fd) == 0 && fd != -1) {
       fds_to_send.insert(fd);
       store_fds.push_back(fd);
       mmap_sizes.push_back(get_mmap_size(fd));
@@ -646,18 +687,19 @@ Status PlasmaStore::process_message(Client* client) {
     case MessageType_PlasmaCreateRequest: {
       int64_t data_size;
       int64_t metadata_size;
-      RETURN_NOT_OK(
-          ReadCreateRequest(input, input_size, &object_id, &data_size, &metadata_size));
+      int device_num;
+      RETURN_NOT_OK(ReadCreateRequest(input, input_size, &object_id, &data_size,
+                                      &metadata_size, &device_num));
       int error_code =
-          create_object(object_id, data_size, metadata_size, client, &object);
+          create_object(object_id, data_size, metadata_size, device_num, client, &object);
       int64_t mmap_size = 0;
-      if (error_code == PlasmaError_OK) {
+      if (error_code == PlasmaError_OK && device_num == 0) {
         mmap_size = get_mmap_size(object.store_fd);
       }
       HANDLE_SIGPIPE(
           SendCreateReply(client->fd, object_id, &object, error_code, mmap_size),
           client->fd);
-      if (error_code == PlasmaError_OK) {
+      if (error_code == PlasmaError_OK && device_num == 0) {
         warn_if_sigpipe(send_fd(client->fd, object.store_fd), client->fd);
       }
     } break;
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
index 7e716d284f389..d97cdf7f8f4f3 100644
--- a/cpp/src/plasma/store.h
+++ b/cpp/src/plasma/store.h
@@ -65,6 +65,13 @@ class PlasmaStore {
   /// @param object_id Object ID of the object to be created.
   /// @param data_size Size in bytes of the object to be created.
   /// @param metadata_size Size in bytes of the object metadata.
+  /// @param device_num The number of the device where the object is being
+  ///        created.
+  ///        device_num = 0 corresponds to the host,
+  ///        device_num = 1 corresponds to GPU0,
+  ///        device_num = 2 corresponds to GPU1, etc.
+  /// @param client The client that created the object.
+  /// @param result The object that has been created.
   /// @return One of the following error codes:
   ///  - PlasmaError_OK, if the object was created successfully.
   ///  - PlasmaError_ObjectExists, if an object with this ID is already
@@ -74,7 +81,7 @@ class PlasmaStore {
   ///    cannot create the object. In this case, the client should not call
   ///    plasma_release.
   int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
-                    Client* client, PlasmaObject* result);
+                    int device_num, Client* client, PlasmaObject* result);
 
   /// Abort a created but unsealed object. If the client is not the
   /// creator, then the abort will fail.
@@ -187,6 +194,9 @@ class PlasmaStore {
   std::unordered_map<int, NotificationQueue> pending_notifications_;
 
   std::unordered_map<int, std::unique_ptr<Client>> connected_clients_;
+#ifdef PLASMA_GPU
+  arrow::gpu::CudaDeviceManager* manager_;
+#endif
 };
 
 }  // namespace plasma
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index f19c2bfbdb380..15f9e7cd111e4 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -295,6 +295,84 @@ TEST_F(TestPlasmaStore, ManyObjectTest) {
   }
 }
 
+#ifdef PLASMA_GPU
+using arrow::gpu::CudaBuffer;
+using arrow::gpu::CudaBufferReader;
+using arrow::gpu::CudaBufferWriter;
+
+TEST_F(TestPlasmaStore, GetGPUTest) {
+  ObjectID object_id = ObjectID::from_random();
+  ObjectBuffer object_buffer;
+
+  // Test for object non-existence.
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer));
+  ASSERT_EQ(object_buffer.data_size, -1);
+
+  // Test for the object being in local Plasma store.
+  // First create object.
+  uint8_t data[] = {4, 5, 3, 1};
+  int64_t data_size = sizeof(data);
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  std::shared_ptr<Buffer> data_buffer;
+  std::shared_ptr<CudaBuffer> gpu_buffer;
+  ARROW_CHECK_OK(
+      client_.Create(object_id, data_size, metadata, metadata_size, &data_buffer, 1));
+  gpu_buffer = std::dynamic_pointer_cast<CudaBuffer>(data_buffer);
+  CudaBufferWriter writer(gpu_buffer);
+  writer.Write(data, data_size);
+  ARROW_CHECK_OK(client_.Seal(object_id));
+
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  gpu_buffer = std::dynamic_pointer_cast<CudaBuffer>(object_buffer.data);
+  CudaBufferReader reader(gpu_buffer);
+  uint8_t read_data[data_size];
+  int64_t read_data_size;
+  reader.Read(data_size, &read_data_size, read_data);
+  for (int64_t i = 0; i < data_size; i++) {
+    ASSERT_EQ(data[i], read_data[i]);
+  }
+}
+
+TEST_F(TestPlasmaStore, MultipleClientGPUTest) {
+  ObjectID object_id = ObjectID::from_random();
+
+  // Test for object non-existence on the first client.
+  bool has_object;
+  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, false);
+
+  // Test for the object being in local Plasma store.
+  // First create and seal object on the second client.
+  int64_t data_size = 100;
+  uint8_t metadata[] = {5};
+  int64_t metadata_size = sizeof(metadata);
+  std::shared_ptr<Buffer> data;
+  ARROW_CHECK_OK(
+      client2_.Create(object_id, data_size, metadata, metadata_size, &data, 1));
+  ARROW_CHECK_OK(client2_.Seal(object_id));
+  // Test that the first client can get the object.
+  ObjectBuffer object_buffer;
+  ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+  ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, true);
+
+  // Test that one client disconnecting does not interfere with the other.
+  // First create object on the second client.
+  object_id = ObjectID::from_random();
+  ARROW_CHECK_OK(
+      client2_.Create(object_id, data_size, metadata, metadata_size, &data, 1));
+  // Disconnect the first client.
+  ARROW_CHECK_OK(client_.Disconnect());
+  // Test that the second client can seal and get the created object.
+  ARROW_CHECK_OK(client2_.Seal(object_id));
+  ARROW_CHECK_OK(client2_.Get(&object_id, 1, -1, &object_buffer));
+  ARROW_CHECK_OK(client2_.Contains(object_id, &has_object));
+  ASSERT_EQ(has_object, true);
+}
+
+#endif
+
 }  // namespace plasma
 
 int main(int argc, char** argv) {
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
index 656b2cc6b9bca..03b1428d8f47f 100644
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -68,6 +68,7 @@ PlasmaObject random_plasma_object(void) {
   object.metadata_offset = random + 2;
   object.data_size = random + 3;
   object.metadata_size = random + 4;
+  object.device_num = 0;
   return object;
 }
 
@@ -76,16 +77,20 @@ TEST(PlasmaSerialization, CreateRequest) {
   ObjectID object_id1 = ObjectID::from_random();
   int64_t data_size1 = 42;
   int64_t metadata_size1 = 11;
-  ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1));
+  int device_num1 = 0;
+  ARROW_CHECK_OK(
+      SendCreateRequest(fd, object_id1, data_size1, metadata_size1, device_num1));
   std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateRequest);
   ObjectID object_id2;
   int64_t data_size2;
   int64_t metadata_size2;
+  int device_num2;
   ARROW_CHECK_OK(ReadCreateRequest(data.data(), data.size(), &object_id2, &data_size2,
-                                   &metadata_size2));
+                                   &metadata_size2, &device_num2));
   ASSERT_EQ(data_size1, data_size2);
   ASSERT_EQ(metadata_size1, metadata_size2);
   ASSERT_EQ(object_id1, object_id2);
+  ASSERT_EQ(device_num1, device_num2);
   close(fd);
 }
 

From 3b6b93d2cfb61705a08a7356ac72c57367bd2fd2 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 7 Feb 2018 11:18:10 +0100
Subject: [PATCH 1448/1644] ARROW-2108: [Python] Update instructions for ASV

Author: Antoine Pitrou <antoine@python.org>

Closes #1566 from pitrou/ARROW-2108-update-asv-instructions and squashes the following commits:

a2e8bef [Antoine Pitrou] ARROW-2108: [Python] Update instructions for ASV
---
 python/README-benchmarks.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/README-benchmarks.md b/python/README-benchmarks.md
index 6389665b075d9..3fecb35cb8095 100644
--- a/python/README-benchmarks.md
+++ b/python/README-benchmarks.md
@@ -35,10 +35,10 @@ ASV allows to store results and generate graphs of the benchmarks over
 the project's evolution.  Doing this requires a bit more massaging
 currently.
 
-First you have to install our ASV fork:
+First you have to install ASV's development version:
 
 ```shell
-pip install git+https://github.com/pitrou/asv.git@issue-547-specify-project-subdir
+pip install git+https://github.com/airspeed-velocity/asv.git
 ```
 
 <!--- TODO remove the above once https://github.com/airspeed-velocity/asv/pull/611 is merged -->

From 57563cd7a85f89094405d705232693fe4a78344b Mon Sep 17 00:00:00 2001
From: Max Risuhin <risuhin.max@gmail.com>
Date: Wed, 7 Feb 2018 15:28:41 +0100
Subject: [PATCH 1449/1644] ARROW-2109: [C++] Completely disable boost autolink
 on MSVC build

Boost's autolink disabling solves linkage issues appeared with boost v.1.66.0 (in case of linkage with static boost libs)

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #1567 from MaxRis/ARROW-2109 and squashes the following commits:

bf5dcef [Max Risuhin] ARROW-2109: [C++] Completely disable boost autolink on MSVC build
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 69812b97cc770..374c652074143 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -136,6 +136,7 @@ if (MSVC AND ARROW_USE_STATIC_CRT)
   set(Boost_USE_STATIC_RUNTIME ON)
 endif()
 set(Boost_ADDITIONAL_VERSIONS
+  "1.66.0" "1.66"
   "1.65.0" "1.65"
   "1.64.0" "1.64"
   "1.63.0" "1.63"
@@ -192,14 +193,15 @@ if (ARROW_BOOST_VENDORED)
   set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}")
   add_dependencies(arrow_dependencies boost_ep)
 else()
+  if (MSVC)
+    # disable autolinking in boost
+    add_definitions(-DBOOST_ALL_NO_LIB)
+  endif()
   if (ARROW_BOOST_USE_SHARED)
     # Find shared Boost libraries.
     set(Boost_USE_STATIC_LIBS OFF)
 
-    if(MSVC)
-      # disable autolinking in boost
-      add_definitions(-DBOOST_ALL_NO_LIB)
-
+    if (MSVC)
       # force all boost libraries to dynamic link
       add_definitions(-DBOOST_ALL_DYN_LINK)
     endif()

From 5dd9bffc1d026c84d3a9d4115173c4ecd1d1a314 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 7 Feb 2018 15:29:24 +0100
Subject: [PATCH 1450/1644] ARROW-2107: [GLib] Follow
 arrow::gpu::CudaIpcMemHandle API change

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1565 from kou/glib-follow-gpu-cuda-ipc-memory-handle-change and squashes the following commits:

99ebf14 [Kouhei Sutou] [GLib] Follow arrow::gpu::CudaIpcMemHandle API change
---
 c_glib/arrow-gpu-glib/cuda.cpp | 22 +++++++++++-----------
 c_glib/arrow-gpu-glib/cuda.hpp |  4 ++--
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/c_glib/arrow-gpu-glib/cuda.cpp b/c_glib/arrow-gpu-glib/cuda.cpp
index c2a9af54dda94..7f2a050dfd7b6 100644
--- a/c_glib/arrow-gpu-glib/cuda.cpp
+++ b/c_glib/arrow-gpu-glib/cuda.cpp
@@ -238,7 +238,7 @@ garrow_gpu_cuda_context_class_init(GArrowGPUCUDAContextClass *klass)
    */
   spec = g_param_spec_pointer("context",
                               "Context",
-                              "The raw std::shared_ptr<arrow::gpu::CudaContext> *",
+                              "The raw std::shared_ptr<arrow::gpu::CudaContext>",
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property(gobject_class, PROP_CONTEXT, spec);
@@ -427,10 +427,10 @@ GArrowGPUCUDAIPCMemoryHandle *
 garrow_gpu_cuda_buffer_export(GArrowGPUCUDABuffer *buffer, GError **error)
 {
   auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer);
-  std::unique_ptr<arrow::gpu::CudaIpcMemHandle> arrow_handle;
+  std::shared_ptr<arrow::gpu::CudaIpcMemHandle> arrow_handle;
   auto status = arrow_buffer->ExportForIpc(&arrow_handle);
   if (garrow_error_check(error, status, "[gpu][cuda][buffer][export-for-ipc]")) {
-    return garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow_handle.release());
+    return garrow_gpu_cuda_ipc_memory_handle_new_raw(&arrow_handle);
   } else {
     return NULL;
   }
@@ -527,7 +527,7 @@ garrow_gpu_cuda_host_buffer_new(gint64 size, GError **error)
 
 
 typedef struct GArrowGPUCUDAIPCMemoryHandlePrivate_ {
-  arrow::gpu::CudaIpcMemHandle *ipc_memory_handle;
+  std::shared_ptr<arrow::gpu::CudaIpcMemHandle> ipc_memory_handle;
 } GArrowGPUCUDAIPCMemoryHandlePrivate;
 
 enum {
@@ -548,7 +548,7 @@ garrow_gpu_cuda_ipc_memory_handle_finalize(GObject *object)
 {
   auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object);
 
-  delete priv->ipc_memory_handle;
+  priv->ipc_memory_handle = nullptr;
 
   G_OBJECT_CLASS(garrow_gpu_cuda_ipc_memory_handle_parent_class)->finalize(object);
 }
@@ -564,7 +564,7 @@ garrow_gpu_cuda_ipc_memory_handle_set_property(GObject *object,
   switch (prop_id) {
   case PROP_IPC_MEMORY_HANDLE:
     priv->ipc_memory_handle =
-      static_cast<arrow::gpu::CudaIpcMemHandle *>(g_value_get_pointer(value));
+      *static_cast<std::shared_ptr<arrow::gpu::CudaIpcMemHandle> *>(g_value_get_pointer(value));
     break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
@@ -608,7 +608,7 @@ garrow_gpu_cuda_ipc_memory_handle_class_init(GArrowGPUCUDAIPCMemoryHandleClass *
    */
   spec = g_param_spec_pointer("ipc-memory-handle",
                               "IPC Memory Handle",
-                              "The raw arrow::gpu::CudaIpcMemHandle *",
+                              "The raw std::shared_ptr<arrow::gpu::CudaIpcMemHandle>",
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property(gobject_class, PROP_IPC_MEMORY_HANDLE, spec);
@@ -630,11 +630,11 @@ garrow_gpu_cuda_ipc_memory_handle_new(const guint8 *data,
                                       gsize size,
                                       GError **error)
 {
-  std::unique_ptr<arrow::gpu::CudaIpcMemHandle> arrow_handle;
+  std::shared_ptr<arrow::gpu::CudaIpcMemHandle> arrow_handle;
   auto status = arrow::gpu::CudaIpcMemHandle::FromBuffer(data, &arrow_handle);
   if (garrow_error_check(error, status,
                          "[gpu][cuda][ipc-memory-handle][new]")) {
-    return garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow_handle.release());
+    return garrow_gpu_cuda_ipc_memory_handle_new_raw(&arrow_handle);
   } else {
     return NULL;
   }
@@ -845,7 +845,7 @@ garrow_gpu_cuda_context_get_raw(GArrowGPUCUDAContext *context)
 }
 
 GArrowGPUCUDAIPCMemoryHandle *
-garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_handle)
+garrow_gpu_cuda_ipc_memory_handle_new_raw(std::shared_ptr<arrow::gpu::CudaIpcMemHandle> *arrow_handle)
 {
   auto handle = g_object_new(GARROW_GPU_TYPE_CUDA_IPC_MEMORY_HANDLE,
                              "ipc-memory-handle", arrow_handle,
@@ -853,7 +853,7 @@ garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_ha
   return GARROW_GPU_CUDA_IPC_MEMORY_HANDLE(handle);
 }
 
-arrow::gpu::CudaIpcMemHandle *
+std::shared_ptr<arrow::gpu::CudaIpcMemHandle>
 garrow_gpu_cuda_ipc_memory_handle_get_raw(GArrowGPUCUDAIPCMemoryHandle *handle)
 {
   if (!handle)
diff --git a/c_glib/arrow-gpu-glib/cuda.hpp b/c_glib/arrow-gpu-glib/cuda.hpp
index 3eeff8b6f18ed..4b5b03c8b4608 100644
--- a/c_glib/arrow-gpu-glib/cuda.hpp
+++ b/c_glib/arrow-gpu-glib/cuda.hpp
@@ -29,8 +29,8 @@ std::shared_ptr<arrow::gpu::CudaContext>
 garrow_gpu_cuda_context_get_raw(GArrowGPUCUDAContext *context);
 
 GArrowGPUCUDAIPCMemoryHandle *
-garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_handle);
-arrow::gpu::CudaIpcMemHandle *
+garrow_gpu_cuda_ipc_memory_handle_new_raw(std::shared_ptr<arrow::gpu::CudaIpcMemHandle> *arrow_handle);
+std::shared_ptr<arrow::gpu::CudaIpcMemHandle>
 garrow_gpu_cuda_ipc_memory_handle_get_raw(GArrowGPUCUDAIPCMemoryHandle *handle);
 
 GArrowGPUCUDABuffer *

From bd1e6e58d489b965a76d7f548042052a15c5fc2a Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 7 Feb 2018 16:28:59 +0100
Subject: [PATCH 1451/1644] ARROW-2095: [C++] Less verbose building 3rd party
 deps

Also fixes ARROW-2096.

Author: Antoine Pitrou <antoine@python.org>

Closes #1569 from pitrou/ARROW-2095-less-verbose-cpp-build and squashes the following commits:

1c8bf1e [Antoine Pitrou] ARROW-2095: [C++] Less verbose building 3rd party deps
---
 cpp/CMakeLists.txt                          |  2 +-
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 16 ++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f07ef6b23ef39..073471283c036 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -205,7 +205,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
 
   option(ARROW_VERBOSE_THIRDPARTY_BUILD
     "If off, output from ExternalProjects will be logged to files rather than shown"
-    ON)
+    OFF)
 
   if (MSVC)
     set(BROTLI_MSVC_STATIC_LIB_SUFFIX "-static" CACHE STRING
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 374c652074143..193b162d18285 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -44,8 +44,10 @@ if (NOT ARROW_VERBOSE_THIRDPARTY_BUILD)
     LOG_BUILD 1
     LOG_INSTALL 1
     LOG_DOWNLOAD 1)
+  set(Boost_DEBUG FALSE)
 else()
   set(EP_LOG_OPTIONS)
+  set(Boost_DEBUG TRUE)
 endif()
 
 if (NOT MSVC)
@@ -130,7 +132,6 @@ endif()
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
-set(Boost_DEBUG TRUE)
 set(Boost_USE_MULTITHREADED ON)
 if (MSVC AND ARROW_USE_STATIC_CRT)
   set(Boost_USE_STATIC_RUNTIME ON)
@@ -849,7 +850,8 @@ if (ARROW_WITH_GRPC)
       BUILD_BYPRODUCTS "${GRPC_STATIC_LIBRARY_GPR}" "${GRPC_STATIC_LIBRARY_GRPC}" "${GRPC_STATIC_LIBRARY_GRPCPP}"
       ${GRPC_BUILD_BYPRODUCTS}
       ${EP_LOG_OPTIONS}
-      CMAKE_ARGS ${GRPC_CMAKE_ARGS})
+      CMAKE_ARGS ${GRPC_CMAKE_ARGS}
+      ${EP_LOG_OPTIONS})
   else()
     find_package(gRPC CONFIG REQUIRED)
     set(GRPC_VENDORED 0)
@@ -884,11 +886,8 @@ if (ARROW_ORC)
       CONFIGURE_COMMAND "./configure" "--disable-shared" "--prefix=${PROTOBUF_PREFIX}" "CXXFLAGS=${EP_CXX_FLAGS}"
       BUILD_IN_SOURCE 1
       URL ${PROTOBUF_SRC_URL}
-      LOG_DOWNLOAD 1
-      LOG_CONFIGURE 1
-      LOG_BUILD 1
-      LOG_INSTALL 1
-      BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}")
+      BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}"
+      ${EP_LOG_OPTIONS})
 
     set (PROTOBUF_VENDORED 1)
   else ()
@@ -930,7 +929,8 @@ if (ARROW_ORC)
   ExternalProject_Add(orc_ep
     URL "https://github.com/apache/orc/archive/${ORC_VERSION}.tar.gz"
     BUILD_BYPRODUCTS ${ORC_STATIC_LIB}
-    CMAKE_ARGS ${ORC_CMAKE_ARGS})
+    CMAKE_ARGS ${ORC_CMAKE_ARGS}
+    ${EP_LOG_OPTIONS})
 
   include_directories(SYSTEM ${ORC_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(orc

From 90bb4bdb53486f2d86b3bd8d7105a15c7731d942 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 7 Feb 2018 15:01:11 -0500
Subject: [PATCH 1452/1644] ARROW-2111: [C++] Lint in parallel

On my 4-core machine this cuts `make lint` down from 24s. to 8s.

Note the `-P` option to `xargs` is not POSIX, though it seems supported on OS X (and of course on GNU). Not sure we are ok with this.

Author: Antoine Pitrou <antoine@python.org>

Closes #1573 from pitrou/ARROW-2111-parallel-lint and squashes the following commits:

246bd6da [Antoine Pitrou] ARROW-2111: [C++] Lint in parallel
---
 cpp/CMakeLists.txt | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 073471283c036..62c8e6590a63a 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -455,11 +455,14 @@ if (UNIX)
   message(STATUS "Found cpplint executable at ${CPPLINT_BIN}")
 
   # Full lint
-  add_custom_target(lint ${CPPLINT_BIN}
+  # Balancing act: cpplint.py takes a non-trivial time to launch,
+  # so process 12 files per invocation, while still ensuring parallelism
+  add_custom_target(lint echo ${FILTERED_LINT_FILES} | xargs -n12 -P8
+  ${CPPLINT_BIN}
   --verbose=2
   --linelength=90
   --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11,-runtime/references,-build/include_order
-  ${FILTERED_LINT_FILES})
+  )
 endif (UNIX)
 
 

From 45ad0b079d152b52162bd29a5cd9adecf3d37a4b Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Wed, 7 Feb 2018 15:35:12 -0500
Subject: [PATCH 1453/1644] =?UTF-8?q?ARROW-2087:=20[Python]=C2=A0Binaries?=
 =?UTF-8?q?=20of=203rdparty=20are=20not=20stripped=20in=20manylinux1=20bas?=
 =?UTF-8?q?e=20image?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This brings the size of the docker container down to 2.19G

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1564 from xhochy/ARROW-2087 and squashes the following commits:

ad8c16fa [Korn, Uwe] ARROW-2087: [Python] Binaries of 3rdparty are not stripped in manylinux1 base image
---
 python/manylinux1/Dockerfile-x86_64           |  2 +-
 python/manylinux1/Dockerfile-x86_64_base      | 11 +++-------
 .../manylinux1/scripts/build_virtualenvs.sh   | 14 +++++++++---
 python/manylinux1/scripts/install_cmake.sh    | 22 +++++++++++++++++++
 4 files changed, 37 insertions(+), 12 deletions(-)
 create mode 100755 python/manylinux1/scripts/install_cmake.sh

diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 98b559535e001..1ade9ab100036 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2086
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2087
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index b7687533aaaa2..1f15f77d8ddc7 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -28,11 +28,9 @@ RUN /build_boost.sh
 ADD scripts/build_jemalloc.sh /
 RUN /build_jemalloc.sh
 
-WORKDIR /
 # Install cmake manylinux1 package
-RUN /opt/python/cp35-cp35m/bin/pip install cmake ninja
-RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
-RUN ln -s /opt/python/cp35-cp35m/bin/ninja /usr/bin/ninja
+ADD scripts/install_cmake.sh /
+RUN /install_cmake.sh
 
 ADD scripts/build_gtest.sh /
 RUN /build_gtest.sh
@@ -69,10 +67,7 @@ ADD scripts/build_ccache.sh /
 RUN /build_ccache.sh
 
 WORKDIR /
-RUN git clone https://github.com/matthew-brett/multibuild.git
-WORKDIR /multibuild
-RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
-WORKDIR /
+RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
 ADD scripts/build_virtualenvs.sh /
 RUN /build_virtualenvs.sh
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
index e64157065ff0d..b548611261cd1 100755
--- a/python/manylinux1/scripts/build_virtualenvs.sh
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -33,18 +33,26 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     PATH="$PATH:$(cpython_path $PYTHON)"
 
     echo "=== (${PYTHON}) Installing build dependencies ==="
-    $PIPI_IO "numpy==1.10.1"
+    $PIPI_IO "numpy==1.10.4"
     $PIPI_IO "cython==0.25.2"
-    $PIPI_IO "pandas==0.20.1"
+    $PIPI_IO "pandas==0.20.3"
     $PIPI_IO "virtualenv==15.1.0"
 
     echo "=== (${PYTHON}) Preparing virtualenv for tests ==="
     "$(cpython_path $PYTHON)/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}
     source /venv-test-${PYTHON}/bin/activate
-    pip install pytest 'numpy==1.12.1' 'pandas==0.20.1'
+    pip install pytest 'numpy==1.14.0' 'pandas==0.20.3'
     deactivate
 done
 
+# Remove debug symbols from libraries that were installed via wheel.
+find /venv-test-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';'
+find /venv-test-*/lib/*/site-packages/numpy -name '*.so' -exec strip '{}' ';'
+find /opt/_internal/cpython-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';'
+# Only Python 3.6 packages are stripable as they are built inside of the image 
+find /opt/_internal/cpython-3.6.4/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';'
+find /opt/_internal/*/lib/*/site-packages/Cython -name '*.so' -exec strip '{}' ';'
+
 # Remove pip cache again. It's useful during the virtualenv creation but we
 # don't want it persisted in the docker layer, ~264MiB
 rm -rf /root/.cache
diff --git a/python/manylinux1/scripts/install_cmake.sh b/python/manylinux1/scripts/install_cmake.sh
new file mode 100755
index 0000000000000..864348514cfa4
--- /dev/null
+++ b/python/manylinux1/scripts/install_cmake.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+/opt/python/cp35-cp35m/bin/pip install cmake ninja
+ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
+ln -s /opt/python/cp35-cp35m/bin/ninja /usr/bin/ninja
+strip /opt/_internal/cpython-3.5.4/lib/python3.5/site-packages/cmake/data/bin/*

From ea4c7b1af62d67e0b0c4ba468ce2765bcaaa3f96 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 7 Feb 2018 15:37:15 -0500
Subject: [PATCH 1454/1644] ARROW-2110: [Python] Only require pytest-runner on
 test commands

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1570 from xhochy/ARROW-2110 and squashes the following commits:

aad82911 [Uwe L. Korn] ARROW-2110: [Python] Only require pytest-runner on test commands
---
 python/setup.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index 726bb51f266ca..849d1203be7b2 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -428,6 +428,13 @@ def parse_version(root):
     else:
         return version
 
+
+# Only include pytest-runner in setup_requires if we're invoking tests
+if {'pytest', 'test', 'ptr'}.intersection(sys.argv):
+    setup_requires = ['pytest-runner']
+else:
+    setup_requires = []
+
 setup(
     name="pyarrow",
     packages=['pyarrow', 'pyarrow.tests'],
@@ -447,7 +454,7 @@ def parse_version(root):
         ]
     },
     use_scm_version={"root": "..", "relative_to": __file__, "parse": parse_version},
-    setup_requires=['setuptools_scm', 'cython >= 0.23', 'pytest-runner'],
+    setup_requires=['setuptools_scm', 'cython >= 0.23'] + setup_requires,
     install_requires=install_requires,
     tests_require=['pytest', 'pandas'],
     description="Python library for Apache Arrow",

From f69e9dba1c610e2bce63b2b1b1cacdd7b5cd4000 Mon Sep 17 00:00:00 2001
From: Jingyuan Wang <jingyuan@live.com>
Date: Wed, 7 Feb 2018 17:35:24 -0500
Subject: [PATCH 1455/1644] ARROW-633/634: [Java] Add FixedSizeBinary support
 in Java and integration tests (Updated)

The original PR is at https://github.com/apache/arrow/pull/1012. Due to the major refactoring last year, changes are big. So I created this separate PR for easier to review and will cancel the other one later.

Changes include:
* Arrow-633: [Java] Add support for FixedWidthBinary type
* Arrow-634: Add integration tests for FixedSizeBinary

Author: Jingyuan Wang <jingyuan@live.com>
Author: jingyuan <jingyuan.nt@gmail.com>

Closes #1492 from alphalfalfa/updated-arrow-633-634 and squashes the following commits:

c994a19f [jingyuan] create data buffer layout for FixedSizeBinary directly instead of using dataBuffer() method and some other changes based on PR comments
873c5b2e [jingyuan] do not pass scale or byteWidth into writeValueToGenerator in JsonFileWriter.java
4fbb67dc [jingyuan] put arbitrary values for FixSizeBinary nulls
0c0015dc [Jingyuan Wang] add a new line
a8a95530 [Jingyuan Wang] Pass byteWidth info for FixedSizeBinary type in JsonFileReader
1a64c5cb [Jingyuan Wang] expand get() method inside getObject() method to remove duplicate check of isSet()
071fb252 [Jingyuan Wang] ARROW-633/634: Add FixedSizeBinary support in Java and integration tests
---
 integration/integration_test.py               |  62 ++-
 .../src/main/codegen/data/ArrowTypes.tdd      |   5 +
 .../main/codegen/data/ValueVectorTypes.tdd    |  15 +
 .../codegen/templates/HolderReaderImpl.java   |   4 +
 .../arrow/vector/BaseFixedWidthVector.java    |   4 +-
 .../org/apache/arrow/vector/BitVector.java    |   2 +-
 .../org/apache/arrow/vector/BufferLayout.java |   4 +-
 .../arrow/vector/FixedSizeBinaryVector.java   | 404 ++++++++++++++++++
 .../org/apache/arrow/vector/TypeLayout.java   |   6 +
 .../vector/dictionary/DictionaryEncoder.java  |   2 +-
 .../arrow/vector/ipc/JsonFileReader.java      |  23 +
 .../arrow/vector/ipc/JsonFileWriter.java      |  17 +-
 .../org/apache/arrow/vector/types/Types.java  |  19 +
 .../apache/arrow/vector/util/Validator.java   |   2 +-
 .../vector/TestFixedSizeBinaryVector.java     | 261 +++++++++++
 .../complex/writer/TestComplexWriter.java     |  51 ++-
 .../arrow/vector/ipc/TestArrowFile.java       |  59 +++
 .../arrow/vector/types/pojo/TestSchema.java   |   4 +-
 18 files changed, 926 insertions(+), 18 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeBinaryVector.java

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 79b098a614a6c..301017cac3d40 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -463,6 +463,46 @@ def generate_column(self, size, name=None):
         return self.column_class(name, size, is_valid, values)
 
 
+class FixedSizeBinaryType(PrimitiveType):
+
+    def __init__(self, name, byte_width, nullable=True):
+        super(FixedSizeBinaryType, self).__init__(name, nullable=nullable)
+        self.byte_width = byte_width
+
+    @property
+    def numpy_type(self):
+        return object
+
+    @property
+    def column_class(self):
+        return FixedSizeBinaryColumn
+
+    def _get_type(self):
+        return OrderedDict([('name', 'fixedsizebinary'), ('byteWidth', self.byte_width)])
+
+    def _get_type_layout(self):
+        return OrderedDict([
+            ('vectors',
+             [OrderedDict([('type', 'VALIDITY'),
+                           ('typeBitWidth', 1)]),
+              OrderedDict([('type', 'DATA'),
+                           ('typeBitWidth', self.byte_width)])])])
+
+    def generate_column(self, size, name=None):
+        is_valid = self._make_is_valid(size)
+        values = []
+
+        for i in range(size):
+            draw = (np.random.randint(0, 255, size=self.byte_width)
+                    .astype(np.uint8)
+                    .tostring())
+            values.append(draw)
+
+        if name is None:
+            name = self.name
+        return self.column_class(name, size, is_valid, values)
+
+
 class StringType(BinaryType):
 
     @property
@@ -525,6 +565,22 @@ def _get_buffers(self):
         ]
 
 
+class FixedSizeBinaryColumn(PrimitiveColumn):
+
+    def _encode_value(self, x):
+        return ''.join('{:02x}'.format(c).upper() for c in x)
+
+    def _get_buffers(self):
+        data = []
+        for i, v in enumerate(self.values):
+            data.append(self._encode_value(v))
+
+        return [
+            ('VALIDITY', [int(x) for x in self.is_valid]),
+            ('DATA', data)
+        ]
+
+
 class StringColumn(BinaryColumn):
 
     def _encode_value(self, x):
@@ -719,6 +775,9 @@ def get_field(name, type_, nullable=True):
         return BinaryType(name, nullable=nullable)
     elif type_ == 'utf8':
         return StringType(name, nullable=nullable)
+    elif type_.startswith('fixedsizebinary_'):
+        byte_width = int(type_.split('_')[1])
+        return FixedSizeBinaryType(name, byte_width=byte_width, nullable=nullable)
 
     dtype = np.dtype(type_)
 
@@ -751,7 +810,8 @@ def _generate_file(name, fields, batch_sizes, dictionaries=None):
 def generate_primitive_case(batch_sizes, name='primitive'):
     types = ['bool', 'int8', 'int16', 'int32', 'int64',
              'uint8', 'uint16', 'uint32', 'uint64',
-             'float32', 'float64', 'binary', 'utf8']
+             'float32', 'float64', 'binary', 'utf8',
+             'fixedsizebinary_19', 'fixedsizebinary_120']
 
     fields = []
 
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index ce92c1333a501..63b193fc66a63 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -57,6 +57,11 @@
       fields: [],
       complex: false
     },
+    {
+      name: "FixedSizeBinary",
+      fields: [{name: "byteWidth", type: int}],
+      complex: false
+    }
     {
       name: "Bool",
       fields: [],
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index 970d887c76003..f294a8e83a494 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -122,6 +122,21 @@
         }
       ]
     },
+    {
+      major: "Fixed",
+      width: -1,
+      javaType: "byte[]",
+      boxedType: "ArrowBuf",
+      minor: [
+        {
+          class: "FixedSizeBinary",
+          typeParams: [ {name: "byteWidth", type: "int"} ],
+          arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary",
+          friendlyType: "byte[]",
+          fields: [{name: "buffer", type: "ArrowBuf"}],
+        }
+      ]
+    },
     {
       major: "VarLen",
       width: 4,
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index c2aa83757b99c..f1c10d1021b1c 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -128,6 +128,10 @@ public void read(Nullable${name}Holder h) {
     holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
     ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
     return value;
+  <#elseif minor.class == "FixedSizeBinary">
+    byte[] value = new byte [holder.byteWidth];
+    holder.buffer.getBytes(0, value, 0, holder.byteWidth);
+    return value;
   <#else>
     ${friendlyType} value = new ${friendlyType}(this.holder.value);
     return value;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index 702db9f528152..cbc56fe3dde26 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -42,7 +42,7 @@
  */
 public abstract class BaseFixedWidthVector extends BaseValueVector
         implements FixedWidthVector, FieldVector, VectorDefinitionSetter {
-  private final byte typeWidth;
+  private final int typeWidth;
 
   protected int valueAllocationSizeInBytes;
   protected int validityAllocationSizeInBytes;
@@ -54,7 +54,7 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
   protected int valueCount;
 
   public BaseFixedWidthVector(final String name, final BufferAllocator allocator,
-                                      FieldType fieldType, final byte typeWidth) {
+                                      FieldType fieldType, final int typeWidth) {
     super(name, allocator);
     this.typeWidth = typeWidth;
     field = new Field(name, fieldType, null);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 3887da4a618f0..b7b7b990630bd 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -58,7 +58,7 @@ public BitVector(String name, BufferAllocator allocator) {
    * @param allocator allocator for memory management.
    */
   public BitVector(String name, FieldType fieldType, BufferAllocator allocator) {
-    super(name, allocator, fieldType, (byte) 0);
+    super(name, allocator, fieldType, 0);
     reader = new BitReaderImpl(BitVector.this);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
index f6529d8e55bba..0faab7b2b2913 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
@@ -70,7 +70,7 @@ public static BufferLayout dataBuffer(int typeBitWidth) {
       case 128:
         return VALUES_128;
       default:
-        throw new IllegalArgumentException("only 8, 16, 32, or 64 bits supported");
+        throw new IllegalArgumentException("only 8, 16, 32, 64, or 128 bits supported");
     }
   }
 
@@ -90,7 +90,7 @@ public static BufferLayout byteVector() {
 
   private final BufferType type;
 
-  private BufferLayout(BufferType type, int typeBitWidth) {
+  BufferLayout(BufferType type, int typeBitWidth) {
     super();
     this.type = Preconditions.checkNotNull(type);
     this.typeBitWidth = (short) typeBitWidth;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java
new file mode 100644
index 0000000000000..232e6564aceac
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.impl.FixedSizeBinaryReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.FixedSizeBinaryHolder;
+import org.apache.arrow.vector.holders.NullableFixedSizeBinaryHolder;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.TransferPair;
+
+
+/**
+ * FixedSizeBinaryVector implements a fixed width vector of
+ * binary values which could be null. A validity buffer (bit vector) is
+ * maintained to track which elements in the vector are null.
+ */
+public class FixedSizeBinaryVector extends BaseFixedWidthVector {
+  private final int byteWidth;
+  private final FieldReader reader;
+
+  /**
+   * Instantiate a FixedSizeBinaryVector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name      name of the vector
+   * @param allocator allocator for memory management.
+   * @param byteWidth byte width of the binary values
+   */
+  public FixedSizeBinaryVector(String name, BufferAllocator allocator, int byteWidth) {
+    this(name, FieldType.nullable(new FixedSizeBinary(byteWidth)), allocator);
+  }
+
+  /**
+   * Instantiate a FixedSizeBinaryVector. This doesn't allocate any memory for
+   * the data in vector.
+   *
+   * @param name      name of the vector
+   * @param fieldType type of Field materialized by this vector
+   * @param allocator allocator for memory management.
+   */
+  public FixedSizeBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) {
+    super(name, allocator, fieldType, ((FixedSizeBinary) fieldType.getType()).getByteWidth());
+    reader = new FixedSizeBinaryReaderImpl(FixedSizeBinaryVector.this);
+    byteWidth = ((FixedSizeBinary) fieldType.getType()).getByteWidth();
+  }
+
+  /**
+   * Get a reader that supports reading values from this vector
+   *
+   * @return Field Reader for this vector
+   */
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  /**
+   * Get minor type for this vector. The vector holds values belonging
+   * to a particular type.
+   *
+   * @return {@link org.apache.arrow.vector.types.Types.MinorType}
+   */
+  @Override
+  public Types.MinorType getMinorType() {
+    return Types.MinorType.FIXEDSIZEBINARY;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value retrieval methods                        *
+   *                                                                *
+   ******************************************************************/
+
+  /**
+   * Get the element at the given index from the vector.
+   *
+   * @param index position of element
+   * @return element at given index
+   */
+  public byte[] get(int index) {
+    assert index >= 0;
+    if (isSet(index) == 0) {
+      throw new IllegalStateException("Value at index is null");
+    }
+    final byte[] dst = new byte[byteWidth];
+    valueBuffer.getBytes(index * byteWidth, dst, 0, byteWidth);
+    return dst;
+  }
+
+  /**
+   * Get the element at the given index from the vector and
+   * sets the state in holder. If element at given index
+   * is null, holder.isSet will be zero.
+   *
+   * @param index position of element
+   * @param holder nullable holder to carry the buffer
+   */
+  public void get(int index, NullableFixedSizeBinaryHolder holder) {
+    assert index >= 0;
+    if (isSet(index) == 0) {
+      holder.isSet = 0;
+      return;
+    }
+    holder.isSet = 1;
+    holder.buffer = valueBuffer.slice(index * byteWidth, byteWidth);
+  }
+
+  /**
+   * Same as {@link #get(int)}.
+   *
+   * @param index position of element
+   * @return element at given index
+   */
+  @Override
+  public byte[] getObject(int index) {
+    assert index >= 0;
+    if (isSet(index) == 0) {
+      return null;
+    } else {
+      final byte[] dst = new byte[byteWidth];
+      valueBuffer.getBytes(index * byteWidth, dst, 0, byteWidth);
+      return dst;
+    }
+  }
+
+  /**
+   * Copy a cell value from a particular index in source vector to a particular
+   * position in this vector
+   *
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from      source vector
+   */
+  public void copyFrom(int fromIndex, int thisIndex, FixedSizeBinaryVector from) {
+    BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex));
+    from.valueBuffer.getBytes(fromIndex * byteWidth, valueBuffer,
+        thisIndex * byteWidth, byteWidth);
+  }
+
+  /**
+   * Same as {@link #copyFrom(int, int, FixedSizeBinaryVector)} except that
+   * it handles the case when the capacity of the vector needs to be expanded
+   * before copy.
+   *
+   * @param fromIndex position to copy from in source vector
+   * @param thisIndex position to copy to in this vector
+   * @param from      source vector
+   */
+  public void copyFromSafe(int fromIndex, int thisIndex, FixedSizeBinaryVector from) {
+    handleSafe(thisIndex);
+    copyFrom(fromIndex, thisIndex, from);
+  }
+
+  public int getByteWidth() {
+    return byteWidth;
+  }
+
+
+  /******************************************************************
+   *                                                                *
+   *          vector value setter methods                           *
+   *                                                                *
+   ******************************************************************/
+
+  public void set(int index, byte[] value) {
+    assert index >= 0;
+    assert byteWidth <= value.length;
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    valueBuffer.setBytes(index * byteWidth, value, 0, byteWidth);
+  }
+
+  public void setSafe(int index, byte[] value) {
+    handleSafe(index);
+    set(index, value);
+  }
+
+  public void set(int index, int isSet, byte[] value) {
+    if (isSet > 0) {
+      set(index, value);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  public void setSafe(int index, int isSet, byte[] value) {
+    handleSafe(index);
+    set(index, isSet, value);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index  position of element
+   * @param buffer ArrowBuf containing binary value.
+   */
+  public void set(int index, ArrowBuf buffer) {
+    assert index >= 0;
+    assert byteWidth <= buffer.capacity();
+    BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+    valueBuffer.setBytes(index * byteWidth, buffer, 0, byteWidth);
+  }
+
+  /**
+   * Same as {@link #set(int, ArrowBuf)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index  position of element
+   * @param buffer ArrowBuf containing binary value.
+   */
+  public void setSafe(int index, ArrowBuf buffer) {
+    handleSafe(index);
+    set(index, buffer);
+  }
+
+  /**
+   * Set the element at the given index to the given value.
+   *
+   * @param index  position of element
+   * @param buffer ArrowBuf containing binary value.
+   */
+  public void set(int index, int isSet, ArrowBuf buffer) {
+    if (isSet > 0) {
+      set(index, buffer);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, ArrowBuf)} except that it handles the
+   * case when index is greater than or equal to existing
+   * value capacity {@link #getValueCapacity()}.
+   *
+   * @param index  position of element
+   * @param buffer ArrowBuf containing binary value.
+   */
+  public void setSafe(int index, int isSet, ArrowBuf buffer) {
+    handleSafe(index);
+    set(index, isSet, buffer);
+  }
+
+  /**
+   * Set the variable length element at the specified index to the data
+   * buffer supplied in the holder
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void set(int index, FixedSizeBinaryHolder holder) {
+    assert holder.byteWidth == byteWidth;
+    set(index, holder.buffer);
+  }
+
+  /**
+   * Same as {@link #set(int, FixedSizeBinaryHolder)} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void setSafe(int index, FixedSizeBinaryHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  /**
+   * Set the variable length element at the specified index to the data
+   * buffer supplied in the holder
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void set(int index, NullableFixedSizeBinaryHolder holder) {
+    assert holder.byteWidth == byteWidth;
+    if (holder.isSet < 0) {
+      throw new IllegalArgumentException("holder has a negative isSet value");
+    } else if (holder.isSet > 0) {
+      set(index, holder.buffer);
+    } else {
+      BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+    }
+  }
+
+  /**
+   * Same as {@link #set(int, NullableFixedSizeBinaryHolder)} except that it handles the
+   * case where index and length of new element are beyond the existing
+   * capacity of the vector.
+   *
+   * @param index   position of the element to set
+   * @param holder  holder that carries data buffer.
+   */
+  public void setSafe(int index, NullableFixedSizeBinaryHolder holder) {
+    handleSafe(index);
+    set(index, holder);
+  }
+
+  public void setNull(int index) {
+    handleSafe(index);
+    BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+  }
+
+  /**
+   * Given a data buffer, get the value stored at a particular position
+   * in the vector.
+   *
+   * This method should not be used externally.
+   *
+   * @param buffer data buffer
+   * @param index position of the element.
+   * @return value stored at the index.
+   */
+  public static byte[] get(final ArrowBuf buffer, final int index, final int byteWidth) {
+    final byte[] dst = new byte[byteWidth];
+    buffer.getBytes(index * byteWidth, dst, 0, byteWidth);
+    return dst;
+  }
+
+  /******************************************************************
+   *                                                                *
+   *                      vector transfer                           *
+   *                                                                *
+   ******************************************************************/
+
+
+  /**
+   * Construct a TransferPair comprising of this and and a target vector of
+   * the same type.
+   *
+   * @param ref       name of the target vector
+   * @param allocator allocator for the target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(ref, allocator);
+  }
+
+  /**
+   * Construct a TransferPair with a desired target vector of the same type.
+   *
+   * @param to target vector
+   * @return {@link TransferPair}
+   */
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((FixedSizeBinaryVector) to);
+  }
+
+  private class TransferImpl implements TransferPair {
+    FixedSizeBinaryVector to;
+
+    public TransferImpl(String ref, BufferAllocator allocator) {
+      to = new FixedSizeBinaryVector(ref, allocator, FixedSizeBinaryVector.this.byteWidth);
+    }
+
+    public TransferImpl(FixedSizeBinaryVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public FixedSizeBinaryVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, FixedSizeBinaryVector.this);
+    }
+  }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
index d6f32b4b4b1d5..4c05b97b4ac08 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
@@ -31,6 +31,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
 import org.apache.arrow.vector.types.pojo.ArrowType.Date;
 import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary;
 import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
@@ -136,6 +137,11 @@ public TypeLayout visit(Decimal type) {
         return newFixedWidthTypeLayout(BufferLayout.dataBuffer(128));
       }
 
+      @Override
+      public TypeLayout visit(FixedSizeBinary type) {
+        return newFixedWidthTypeLayout(new BufferLayout(BufferType.DATA, type.getByteWidth() * 8));
+      }
+
       @Override
       public TypeLayout visit(Bool type) {
         return newFixedWidthTypeLayout(BufferLayout.booleanVector());
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
index 762a442c983fe..08df7b6d93176 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -136,7 +136,7 @@ public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
   private static void validateType(MinorType type) {
     // byte arrays don't work as keys in our dictionary map - we could wrap them with something to
     // implement equals and hashcode if we want that functionality
-    if (type == MinorType.VARBINARY || type == MinorType.LIST || type == MinorType.MAP || type == MinorType.UNION) {
+    if (type == MinorType.VARBINARY || type == MinorType.FIXEDSIZEBINARY || type == MinorType.LIST || type == MinorType.MAP || type == MinorType.UNION) {
       throw new IllegalArgumentException("Dictionary encoding for complex types not implemented: type " + type);
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
index d0a9b9e18b8f2..8995716461e15 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -329,6 +329,26 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException
       }
     };
 
+    BufferReader FIXEDSIZEBINARY = new BufferReader() {
+      @Override
+      protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
+        ArrayList<byte[]> values = Lists.newArrayList();
+        for (int i = 0; i < count; i++) {
+          parser.nextToken();
+          final byte[] value = decodeHexSafe(parser.readValueAs(String.class));
+          values.add(value);
+        }
+
+        int byteWidth = count > 0 ? values.get(0).length : 0;
+        ArrowBuf buf = allocator.buffer(byteWidth * count);
+        for (byte[] value : values) {
+          buf.writeBytes(value);
+        }
+
+        return buf;
+      }
+    };
+
     BufferReader VARCHAR = new BufferReader() {
       @Override
       protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
@@ -428,6 +448,9 @@ private ArrowBuf readIntoBuffer(BufferAllocator allocator, BufferType bufferType
         case DECIMAL:
           reader = helper.DECIMAL;
           break;
+        case FIXEDSIZEBINARY:
+          reader = helper.FIXEDSIZEBINARY;
+          break;
         case VARCHAR:
           reader = helper.VARCHAR;
           break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
index 6eb76a7a14723..ad56d573b918e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -163,8 +163,7 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
       generator.writeObjectField("name", field.getName());
       int valueCount = vector.getValueCount();
       generator.writeObjectField("count", valueCount);
-      final int scale = (vector instanceof DecimalVector) ?
-                            ((DecimalVector) vector).getScale() : 0;
+
       for (int v = 0; v < vectorTypes.size(); v++) {
         BufferType bufferType = vectorTypes.get(v);
         ArrowBuf vectorBuffer = vectorBuffers.get(v);
@@ -173,9 +172,9 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
         for (int i = 0; i < bufferValueCount; i++) {
           if (bufferType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR ||
                   vector.getMinorType() == Types.MinorType.VARBINARY)) {
-            writeValueToGenerator(bufferType, vectorBuffer, vectorBuffers.get(v-1), vector, i, scale);
+            writeValueToGenerator(bufferType, vectorBuffer, vectorBuffers.get(v-1), vector, i);
           } else {
-            writeValueToGenerator(bufferType, vectorBuffer, null, vector, i, scale);
+            writeValueToGenerator(bufferType, vectorBuffer, null, vector, i);
           }
         }
         generator.writeEndArray();
@@ -200,12 +199,12 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
 
   private void writeValueToGenerator(BufferType bufferType, ArrowBuf buffer,
                                      ArrowBuf offsetBuffer, FieldVector vector,
-                                     final int index, final int scale) throws IOException {
+                                     final int index) throws IOException {
     if (bufferType.equals(TYPE)) {
       generator.writeNumber(buffer.getByte(index * TinyIntVector.TYPE_WIDTH));
     } else if (bufferType.equals(OFFSET)) {
       generator.writeNumber(buffer.getInt(index * BaseVariableWidthVector.OFFSET_WIDTH));
-    } else if(bufferType.equals(VALIDITY)) {
+    } else if (bufferType.equals(VALIDITY)) {
       generator.writeNumber(vector.isNull(index) ? 0 : 1);
     } else if (bufferType.equals(DATA)) {
       switch (vector.getMinorType()) {
@@ -279,6 +278,11 @@ private void writeValueToGenerator(BufferType bufferType, ArrowBuf buffer,
           generator.writeObject(hexString);
           break;
         }
+        case FIXEDSIZEBINARY:
+          int byteWidth = ((FixedSizeBinaryVector) vector).getByteWidth();
+          String fixedSizeHexString = Hex.encodeHexString(FixedSizeBinaryVector.get(buffer, index, byteWidth));
+          generator.writeObject(fixedSizeHexString);
+          break;
         case VARCHAR: {
           assert offsetBuffer != null;
           byte[] b = (BaseVariableWidthVector.get(buffer, offsetBuffer, index));
@@ -286,6 +290,7 @@ private void writeValueToGenerator(BufferType bufferType, ArrowBuf buffer,
           break;
         }
         case DECIMAL: {
+          int scale = ((DecimalVector) vector).getScale();
           BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale);
           // We write the unscaled value, because the scale is stored in the type metadata.
           generator.writeString(decimalValue.unscaledValue().toString());
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 3c5fd81d572d0..4adbefbeb5ac4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -29,6 +29,7 @@
 import org.apache.arrow.vector.DateDayVector;
 import org.apache.arrow.vector.DateMilliVector;
 import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.FixedSizeBinaryVector;
 import org.apache.arrow.vector.Float4Vector;
 import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.IntVector;
@@ -65,6 +66,7 @@
 import org.apache.arrow.vector.complex.impl.DateDayWriterImpl;
 import org.apache.arrow.vector.complex.impl.DateMilliWriterImpl;
 import org.apache.arrow.vector.complex.impl.DecimalWriterImpl;
+import org.apache.arrow.vector.complex.impl.FixedSizeBinaryWriterImpl;
 import org.apache.arrow.vector.complex.impl.Float4WriterImpl;
 import org.apache.arrow.vector.complex.impl.Float8WriterImpl;
 import org.apache.arrow.vector.complex.impl.IntWriterImpl;
@@ -100,6 +102,7 @@
 import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
 import org.apache.arrow.vector.types.pojo.ArrowType.Date;
 import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary;
 import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
 import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
@@ -387,6 +390,17 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return new DecimalWriterImpl((DecimalVector) vector);
       }
     },
+    FIXEDSIZEBINARY(null) {
+      @Override
+      public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
+        return new FixedSizeBinaryVector(name, fieldType, allocator);
+      }
+
+      @Override
+      public FieldWriter getNewFieldWriter(ValueVector vector) {
+        return new FixedSizeBinaryWriterImpl((FixedSizeBinaryVector) vector);
+      }
+    },
     UINT1(new Int(8, false)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
@@ -607,6 +621,11 @@ public MinorType visit(Decimal type) {
         return MinorType.DECIMAL;
       }
 
+      @Override
+      public MinorType visit(FixedSizeBinary type) {
+        return MinorType.FIXEDSIZEBINARY;
+      }
+
       @Override
       public MinorType visit(Date type) {
         switch (type.getUnit()) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
index c27e5e5c85c30..c27634a7edaee 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
@@ -140,7 +140,7 @@ static boolean equals(ArrowType type, final Object o1, final Object o2) {
         default:
           throw new UnsupportedOperationException("unsupported precision: " + fpType);
       }
-    } else if (type instanceof ArrowType.Binary) {
+    } else if (type instanceof ArrowType.Binary || type instanceof ArrowType.FixedSizeBinary) {
       return Arrays.equals((byte[]) o1, (byte[]) o2);
     }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeBinaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeBinaryVector.java
new file mode 100644
index 0000000000000..90529879172c1
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeBinaryVector.java
@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.holders.FixedSizeBinaryHolder;
+import org.apache.arrow.vector.holders.NullableFixedSizeBinaryHolder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestFixedSizeBinaryVector {
+  private static final int numValues = 123;
+  private static final int typeWidth = 9;
+  private static final int smallDataSize = 6;
+  private static final int largeDataSize = 12;
+
+  private static byte[][] values;
+
+  static {
+    values = new byte[numValues][typeWidth];
+    for (int i = 0; i < numValues; i++) {
+      for (int j = 0; j < typeWidth; j++) {
+        values[i][j] = ((byte) i);
+      }
+    }
+  }
+
+  private ArrowBuf[] bufs = new ArrowBuf[numValues];
+  private FixedSizeBinaryHolder[] holders = new FixedSizeBinaryHolder[numValues];
+  private NullableFixedSizeBinaryHolder[] nullableHolders = new NullableFixedSizeBinaryHolder[numValues];
+
+  private static byte[] smallValue;
+
+  static {
+    smallValue = new byte[smallDataSize];
+    for (int i = 0; i < smallDataSize; i++) {
+      smallValue[i] = ((byte) i);
+    }
+  }
+
+  private ArrowBuf smallBuf;
+  private FixedSizeBinaryHolder smallHolder;
+  private NullableFixedSizeBinaryHolder smallNullableHolder;
+
+  private static byte[] largeValue;
+
+  static {
+    largeValue = new byte[largeDataSize];
+    for (int i = 0; i < largeDataSize; i++) {
+      largeValue[i] = ((byte) i);
+    }
+  }
+
+  private ArrowBuf largeBuf;
+  private FixedSizeBinaryHolder largeHolder;
+  private NullableFixedSizeBinaryHolder largeNullableHolder;
+
+  private BufferAllocator allocator;
+  private FixedSizeBinaryVector vector;
+
+  private static void failWithException(String message) throws Exception {
+    throw new Exception(message);
+  }
+
+
+  @Before
+  public void init() throws Exception {
+    allocator = new DirtyRootAllocator(Integer.MAX_VALUE, (byte) 100);
+    vector = new FixedSizeBinaryVector("fixedSizeBinary", allocator, typeWidth);
+    vector.allocateNew();
+
+    for (int i = 0; i < numValues; i++) {
+      bufs[i] = allocator.buffer(typeWidth);
+      bufs[i].setBytes(0, values[i]);
+
+      holders[i] = new FixedSizeBinaryHolder();
+      holders[i].byteWidth = typeWidth;
+      holders[i].buffer = bufs[i];
+
+      nullableHolders[i] = new NullableFixedSizeBinaryHolder();
+      nullableHolders[i].byteWidth = typeWidth;
+      nullableHolders[i].buffer = bufs[i];
+      nullableHolders[i].isSet = 1;
+    }
+
+    smallBuf = allocator.buffer(smallDataSize);
+    smallBuf.setBytes(0, smallValue);
+
+    smallHolder = new FixedSizeBinaryHolder();
+    smallHolder.byteWidth = smallDataSize;
+    smallHolder.buffer = smallBuf;
+
+    smallNullableHolder = new NullableFixedSizeBinaryHolder();
+    smallNullableHolder.byteWidth = smallDataSize;
+    smallNullableHolder.buffer = smallBuf;
+
+    largeBuf = allocator.buffer(largeDataSize);
+    largeBuf.setBytes(0, largeValue);
+
+    largeHolder = new FixedSizeBinaryHolder();
+    largeHolder.byteWidth = typeWidth;
+    largeHolder.buffer = largeBuf;
+
+    largeNullableHolder = new NullableFixedSizeBinaryHolder();
+    largeNullableHolder.byteWidth = typeWidth;
+    largeNullableHolder.buffer = largeBuf;
+  }
+
+  @After
+  public void terminate() throws Exception {
+    for (int i=0; i<numValues; i++) {
+      bufs[i].close();
+    }
+    smallBuf.close();
+    largeBuf.close();
+
+    vector.close();
+    allocator.close();
+  }
+
+  @Test
+  public void testSetUsingByteArray() {
+    for (int i = 0; i < numValues; i++) {
+      vector.set(i, values[i]);
+    }
+    vector.setValueCount(numValues);
+    for (int i = 0; i < numValues; i++) {
+      assertArrayEquals(values[i], vector.getObject(i));
+    }
+  }
+
+  @Test
+  public void testSetUsingHolder() {
+    for (int i = 0; i < numValues; i++) {
+      vector.set(i, holders[i]);
+    }
+    vector.setValueCount(numValues);
+    for (int i = 0; i < numValues; i++) {
+      assertArrayEquals(values[i], vector.getObject(i));
+    }
+  }
+
+  @Test
+  public void testSetUsingNullableHolder() {
+    for (int i = 0; i < numValues; i++) {
+      vector.set(i, nullableHolders[i]);
+    }
+    vector.setValueCount(numValues);
+    for (int i = 0; i < numValues; i++) {
+      assertArrayEquals(values[i], vector.getObject(i));
+    }
+  }
+
+  @Test
+  public void testGetUsingNullableHolder() {
+    for (int i = 0; i < numValues; i++) {
+      vector.set(i, holders[i]);
+    }
+    vector.setValueCount(numValues);
+    for (int i = 0; i < numValues; i++) {
+      vector.get(i, nullableHolders[i]);
+      assertEquals(typeWidth, nullableHolders[i].byteWidth);
+      assertTrue(nullableHolders[i].isSet > 0);
+      byte[] actual = new byte[typeWidth];
+      nullableHolders[i].buffer.getBytes(0, actual, 0, typeWidth);
+      assertArrayEquals(values[i], actual);
+    }
+  }
+
+  @Test
+  public void testSetWithInvalidInput() throws Exception {
+    String errorMsg = "input data needs to be at least " + typeWidth + " bytes";
+
+    // test small inputs, byteWidth matches but value or buffer is too small
+    try {
+      vector.set(0, smallValue);
+      failWithException(errorMsg);
+    } catch (AssertionError ignore) {
+    }
+
+    try {
+      vector.set(0, smallHolder);
+      failWithException(errorMsg);
+    } catch (AssertionError ignore) {
+    }
+
+    try {
+      vector.set(0, smallNullableHolder);
+      failWithException(errorMsg);
+    } catch (AssertionError ignore) {
+    }
+
+    try {
+      vector.set(0, smallBuf);
+      failWithException(errorMsg);
+    } catch (AssertionError ignore) {
+    }
+
+    // test large inputs, byteWidth matches but value or buffer is bigger than byteWidth
+    vector.set(0, largeValue);
+    vector.set(0, largeHolder);
+    vector.set(0, largeNullableHolder);
+    vector.set(0, largeBuf);
+  }
+
+  @Test
+  public void setSetSafeWithInvalidInput() throws Exception {
+    String errorMsg = "input data needs to be at least " + typeWidth + " bytes";
+
+    // test small inputs, byteWidth matches but value or buffer is too small
+    try {
+      vector.setSafe(0, smallValue);
+      failWithException(errorMsg);
+    } catch (AssertionError ignore) {
+    }
+
+    try {
+      vector.setSafe(0, smallHolder);
+      failWithException(errorMsg);
+    } catch (AssertionError ignore) {
+    }
+
+    try {
+      vector.setSafe(0, smallNullableHolder);
+      failWithException(errorMsg);
+    } catch (AssertionError ignore) {
+    }
+
+    try {
+      vector.setSafe(0, smallBuf);
+      failWithException(errorMsg);
+    } catch (AssertionError ignore) {
+    }
+
+    // test large inputs, byteWidth matches but value or buffer is bigger than byteWidth
+    vector.setSafe(0, largeValue);
+    vector.setSafe(0, largeHolder);
+    vector.setSafe(0, largeNullableHolder);
+    vector.setSafe(0, largeBuf);
+  }
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 29d39aabe6b47..b52769e44ef59 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -618,7 +618,7 @@ public void timeStampSecWriter() throws Exception {
     final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0);
 
     // write
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
 
@@ -718,7 +718,7 @@ public void timeStampMicroWriters() throws Exception {
     final LocalDateTime expectedMicroDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
 
     // write
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
 
@@ -765,7 +765,7 @@ public void timeStampNanoWriters() throws Exception {
     final LocalDateTime expectedNanoDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
 
     // write
-    MapVector parent = new MapVector("parent", allocator, null);
+    MapVector parent = MapVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();
 
@@ -806,6 +806,51 @@ public void timeStampNanoWriters() throws Exception {
     }
   }
 
+  @Test
+  public void fixedSizeBinaryWriters() throws Exception {
+    // test values
+    int numValues = 10;
+    int byteWidth = 9;
+    byte[][] values = new byte[numValues][byteWidth];
+    for (int i = 0; i < numValues; i++) {
+      for (int j = 0; j < byteWidth; j++) {
+        values[i][j] = ((byte) i);
+      }
+    }
+    ArrowBuf[] bufs = new ArrowBuf[numValues];
+    for (int i = 0; i < numValues; i++) {
+      bufs[i] = allocator.buffer(byteWidth);
+      bufs[i].setBytes(0, values[i]);
+    }
+
+    // write
+    MapVector parent = MapVector.empty("parent", allocator);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+
+    String fieldName = "fixedSizeBinary";
+    FixedSizeBinaryWriter fixedSizeBinaryWriter = rootWriter.fixedSizeBinary(fieldName, byteWidth);
+    for (int i = 0; i < numValues; i++) {
+      fixedSizeBinaryWriter.setPosition(i);
+      fixedSizeBinaryWriter.writeFixedSizeBinary(bufs[i]);
+    }
+
+    // schema
+    List<Field> children = parent.getField().getChildren().get(0).getChildren();
+    Assert.assertEquals(fieldName, children.get(0).getName());
+    Assert.assertEquals(ArrowType.FixedSizeBinary.TYPE_TYPE, children.get(0).getType().getTypeID());
+
+    // read
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+
+    FieldReader fixedSizeBinaryReader = rootReader.reader(fieldName);
+    for (int i = 0; i < numValues; i++) {
+      fixedSizeBinaryReader.setPosition(i);
+      byte[] readValues = fixedSizeBinaryReader.readByteArray();
+      Assert.assertArrayEquals(values[i], readValues);
+    }
+  }
+
   @Test
   public void complexCopierWithList() {
     MapVector parent = MapVector.empty("parent", allocator);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
index 055c34e7010f3..3b809d00c2c62 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
@@ -42,6 +42,7 @@
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.FixedSizeBinaryVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.ipc.message.ArrowBlock;
@@ -51,6 +52,7 @@
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary;
 import org.apache.arrow.vector.types.pojo.ArrowType.Int;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -549,6 +551,63 @@ public void testWriteReadNestedDictionary() throws IOException {
     }
   }
 
+  @Test
+  public void testWriteReadFixedSizeBinary() throws IOException {
+    File file = new File("target/mytest_fixed_size_binary.arrow");
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+    final int numValues = 10;
+    final int typeWidth = 11;
+    byte[][] byteValues = new byte[numValues][typeWidth];
+    for (int i=0; i<numValues; i++) {
+      for (int j=0; j<typeWidth; j++) {
+        byteValues[i][j] = ((byte) i);
+      }
+    }
+
+    // write
+    try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+      FixedSizeBinaryVector fixedSizeBinaryVector = parent.addOrGet("fixed-binary", FieldType.nullable(new FixedSizeBinary(typeWidth)), FixedSizeBinaryVector.class);
+      parent.allocateNew();
+      for (int i=0; i<numValues; i++) {
+        fixedSizeBinaryVector.set(i, byteValues[i]);
+      }
+      parent.setValueCount(numValues);
+      write(parent, file, stream);
+    }
+
+    // read
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         FileInputStream fileInputStream = new FileInputStream(file);
+         ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+
+      for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) {
+        arrowReader.loadRecordBatch(rbBlock);
+        Assert.assertEquals(numValues, root.getRowCount());
+        for (int i = 0; i < numValues; i++) {
+          Assert.assertArrayEquals(byteValues[i], ((byte[]) root.getVector("fixed-binary").getObject(i)));
+        }
+      }
+    }
+
+    // read from stream
+    try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+         ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+         ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+      VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+      Schema schema = root.getSchema();
+      LOGGER.debug("reading schema: " + schema);
+      arrowReader.loadNextBatch();
+      Assert.assertEquals(numValues, root.getRowCount());
+      for (int i = 0; i < numValues; i++) {
+        Assert.assertArrayEquals(byteValues[i], ((byte[]) root.getVector("fixed-binary").getObject(i)));
+      }
+    }
+  }
+
   @Test
   public void testWriteReadFixedSizeList() throws IOException {
     File file = new File("target/mytest_fixed_list.arrow");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 43b0907720f83..c6699fe270a8f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -30,6 +30,7 @@
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.UnionMode;
 import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
+import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary;
 import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
 import org.apache.arrow.vector.types.pojo.ArrowType.Date;
 import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
@@ -95,7 +96,8 @@ public void testAll() throws IOException {
         field("p", new Time(TimeUnit.NANOSECOND, 64)),
         field("q", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
         field("r", new Timestamp(TimeUnit.MICROSECOND, null)),
-        field("s", new Interval(IntervalUnit.DAY_TIME))
+        field("s", new Interval(IntervalUnit.DAY_TIME)),
+        field("t", new FixedSizeBinary(100))
     ));
     roundTrip(schema);
   }

From 7bf7b2e9639a3df10d9de76df6c705e4495e9e75 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Wed, 7 Feb 2018 21:37:34 -0500
Subject: [PATCH 1456/1644] ARROW-1950: [Python] pandas_type in pandas metadata
 incorrect for List types

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1571 from cpcloud/ARROW-1950 and squashes the following commits:

0b6bc1cb [Phillip Cloud] ARROW-1950: [Python] pandas_type in pandas metadata incorrect for List types
---
 cpp/src/arrow/python/arrow_to_pandas.cc     |  6 +--
 python/pyarrow/pandas_compat.py             |  2 +-
 python/pyarrow/tests/test_array.py          |  2 +-
 python/pyarrow/tests/test_convert_pandas.py | 51 +++++++++++++++++++++
 4 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index fcf05f833dbdf..a17d14bf62e25 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -56,8 +56,8 @@
 namespace arrow {
 namespace py {
 
-using internal::kPandasTimestampNull;
 using internal::kNanosecondsInDay;
+using internal::kPandasTimestampNull;
 
 using compute::Datum;
 
@@ -90,7 +90,6 @@ struct WrapBytes<FixedSizeBinaryArray> {
 
 static inline bool ListTypeSupported(const DataType& type) {
   switch (type.id()) {
-    case Type::NA:
     case Type::UINT8:
     case Type::INT8:
     case Type::UINT16:
@@ -104,6 +103,7 @@ static inline bool ListTypeSupported(const DataType& type) {
     case Type::BINARY:
     case Type::STRING:
     case Type::TIMESTAMP:
+    case Type::NA:  // empty list
       // The above types are all supported.
       return true;
     case Type::LIST: {
@@ -696,7 +696,6 @@ class ObjectBlock : public PandasBlock {
     } else if (type == Type::LIST) {
       auto list_type = std::static_pointer_cast<ListType>(col->type());
       switch (list_type->value_type()->id()) {
-        CONVERTLISTSLIKE_CASE(FloatType, NA)
         CONVERTLISTSLIKE_CASE(UInt8Type, UINT8)
         CONVERTLISTSLIKE_CASE(Int8Type, INT8)
         CONVERTLISTSLIKE_CASE(UInt16Type, UINT16)
@@ -711,6 +710,7 @@ class ObjectBlock : public PandasBlock {
         CONVERTLISTSLIKE_CASE(BinaryType, BINARY)
         CONVERTLISTSLIKE_CASE(StringType, STRING)
         CONVERTLISTSLIKE_CASE(ListType, LIST)
+        CONVERTLISTSLIKE_CASE(NullType, NA)
         default: {
           std::stringstream ss;
           ss << "Not implemented type for conversion from List to Pandas ObjectBlock: "
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 987bb7555d972..f5e56a9b28420 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -45,7 +45,7 @@ def get_logical_type_map():
 
     if not _logical_type_map:
         _logical_type_map.update({
-            pa.lib.Type_NA: 'float64',  # NaNs
+            pa.lib.Type_NA: 'empty',
             pa.lib.Type_BOOL: 'bool',
             pa.lib.Type_INT8: 'int8',
             pa.lib.Type_INT16: 'int16',
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 1d5d30071902a..efbcef5e135b6 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -455,7 +455,7 @@ def test_simple_type_construction():
 @pytest.mark.parametrize(
     ('type', 'expected'),
     [
-        (pa.null(), 'float64'),
+        (pa.null(), 'empty'),
         (pa.bool_(), 'bool'),
         (pa.int8(), 'int8'),
         (pa.int16(), 'int16'),
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 4f0a68729bc73..7dbf0d7ed8e1c 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1404,6 +1404,57 @@ def test_empty_list_roundtrip(self):
 
         tm.assert_frame_equal(result, df)
 
+    def test_empty_list_metadata(self):
+        # Create table with array of empty lists, forced to have type
+        # list(string) in pyarrow
+        c1 = [["test"], ["a", "b"], None]
+        c2 = [[], [], []]
+        arrays = OrderedDict([
+            ('c1', pa.array(c1, type=pa.list_(pa.string()))),
+            ('c2', pa.array(c2, type=pa.list_(pa.string()))),
+        ])
+        rb = pa.RecordBatch.from_arrays(
+            list(arrays.values()),
+            list(arrays.keys())
+        )
+        tbl = pa.Table.from_batches([rb])
+
+        # First roundtrip changes schema, because pandas cannot preserve the
+        # type of empty lists
+        df = tbl.to_pandas()
+        tbl2 = pa.Table.from_pandas(df, preserve_index=True)
+        md2 = json.loads(tbl2.schema.metadata[b'pandas'].decode('utf8'))
+
+        # Second roundtrip
+        df2 = tbl2.to_pandas()
+        expected = pd.DataFrame(OrderedDict([('c1', c1), ('c2', c2)]))
+
+        tm.assert_frame_equal(df2, expected)
+
+        assert md2['columns'] == [
+            {
+                'name': 'c1',
+                'field_name': 'c1',
+                'metadata': None,
+                'numpy_type': 'object',
+                'pandas_type': 'list[unicode]',
+            },
+            {
+                'name': 'c2',
+                'field_name': 'c2',
+                'metadata': None,
+                'numpy_type': 'object',
+                'pandas_type': 'list[empty]',
+            },
+            {
+                'name': None,
+                'field_name': '__index_level_0__',
+                'metadata': None,
+                'numpy_type': 'int64',
+                'pandas_type': 'int64',
+            }
+        ]
+
 
 def _fully_loaded_dataframe_example():
     from distutils.version import LooseVersion

From 6e6f9b7860dc756dd233bcac1ea3b91a154d44cc Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 8 Feb 2018 11:01:40 +0100
Subject: [PATCH 1457/1644] ARROW-2083: [CI] Detect changed components on
 Travis-CI

Hopefully this doesn't add too much maintenance burden.

Also allows to skip Travis builds by putting "[skip travis]" in the last commit message.
Also skip non-Python non-C++ changes on AppVeyor.

Author: Antoine Pitrou <antoine@python.org>

Closes #1568 from pitrou/ARROW-2083-detect-changed-components and squashes the following commits:

cdb371c [Antoine Pitrou] [skip appveyor] ARROW-2083: [CI] Detect changed components on Travis-CI
---
 .travis.yml                     |  85 +++++++++-------
 appveyor.yml                    |   8 ++
 c_glib/meson_options.txt        |   1 +
 ci/travis_before_script_cpp.sh  |  23 +----
 ci/travis_build_parquet_cpp.sh  |   3 +-
 ci/travis_detect_changes.py     | 170 ++++++++++++++++++++++++++++++++
 ci/travis_install_conda.sh      |  39 ++++----
 ci/travis_install_toolchain.sh  |  48 +++++++++
 ci/travis_lint.sh               |  33 ++++---
 ci/travis_script_integration.sh |   3 +
 ci/travis_script_java.sh        |   1 -
 ci/travis_script_javadoc.sh     |  29 ++++++
 ci/travis_script_python.sh      |   4 +-
 13 files changed, 358 insertions(+), 89 deletions(-)
 create mode 100644 ci/travis_detect_changes.py
 create mode 100755 ci/travis_install_toolchain.sh
 create mode 100755 ci/travis_script_javadoc.sh

diff --git a/.travis.yml b/.travis.yml
index 58d6786aab16a..d591a992269c0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,11 +46,13 @@ matrix:
   allow_failures:
   - jdk: oraclejdk9
   include:
+  # C++ & Python w/ clang 4.0
   - compiler: gcc
     language: cpp
     os: linux
     group: deprecated
     before_script:
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_VALGRIND=1
     - export ARROW_TRAVIS_PLASMA=1
@@ -61,12 +63,13 @@ matrix:
     - export CXX="clang++-4.0"
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
+    - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
+    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh; fi
+    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7; fi
+    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6; fi
+  # [OS X] C++ & Python w/ XCode 6.4
   - compiler: clang
     language: cpp
     osx_image: xcode6.4
@@ -74,81 +77,96 @@ matrix:
     cache:
     addons:
     before_script:
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_PLASMA=1
     - export ARROW_TRAVIS_ORC=1
     - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
+    - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
+    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh; fi
+    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7; fi
+    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6; fi
+  # [manylinux1] Python
   - language: cpp
     before_script:
-    - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest; fi
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
+    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
+  # Java w/ OpenJDK 7
   - language: java
     os: linux
     jdk: openjdk7
+    before_script:
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+    - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_java.sh; fi
+    - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh; fi
+  # Java w/ Oracle JDK 9
   - language: java
     os: linux
-    env: ARROW_TRAVIS_SKIP_SITE=yes
     jdk: oraclejdk9
+    before_script:
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+    - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_java.sh; fi
     addons:
       apt:
         packages:
           - oracle-java9-installer
+  # Integration w/ OpenJDK 8
   - language: java
     os: linux
     env: ARROW_TEST_GROUP=integration
     jdk: openjdk8
     before_script:
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - export CC="clang-4.0"
     - export CXX="clang++-4.0"
     - nvm install node
-    - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh; fi
+    - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
+    - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh; fi
+  # NodeJS
   - language: node_js
     os: linux
     node_js: node
     before_script:
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+    - if [ $ARROW_CI_JS_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh; fi
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
+    - if [ $ARROW_CI_JS_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_js.sh; fi
+  # C++ & glib w/ gcc 4.9 & autotools
   - compiler: gcc
     language: cpp
     os: linux
     env: BUILD_SYSTEM=autotools BUILD_TORCH_EXAMPLE=no
     before_script:
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
-    - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
+  # C++ & glib w/ gcc 4.9 & meson
   - compiler: gcc
     language: cpp
     os: linux
     env: BUILD_SYSTEM=meson BUILD_TORCH_EXAMPLE=no
     before_script:
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
-    - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
+  # [OS X] C++ & glib w/ XCode 8.3 & autotools
   - compiler: clang
     osx_image: xcode8.3
     os: osx
@@ -157,11 +175,12 @@ matrix:
     rvm: 2.2
     env: BUILD_SYSTEM=autotools
     before_script:
-    - brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile; fi
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
 
 before_install:
 - ulimit -c unlimited -S
diff --git a/appveyor.yml b/appveyor.yml
index ea7922bf658ef..22d6158bebbb3 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -18,6 +18,14 @@
 # Operating system (build VM template)
 os: Visual Studio 2015
 
+only_commits:
+  # Skip commits not related to Python or C++
+  files:
+    - appveyor.yml
+    - ci/
+    - cpp/
+    - python/
+
 environment:
   matrix:
     - JOB: "Build"
diff --git a/c_glib/meson_options.txt b/c_glib/meson_options.txt
index 2988e1af0acc9..859a4e620650f 100644
--- a/c_glib/meson_options.txt
+++ b/c_glib/meson_options.txt
@@ -21,3 +21,4 @@ option('enable_gtk_doc',
        type: 'boolean',
        value: false,
        description: 'Build document by GTK-Doc')
+
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 7725c560cb3d6..4ffe97f675c5f 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -31,28 +31,7 @@ fi
 
 if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
   # Set up C++ toolchain from conda-forge packages for faster builds
-  conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
-        jemalloc=4.4.0 \
-        nomkl \
-        boost-cpp \
-        rapidjson \
-        flatbuffers \
-        gflags \
-        gtest \
-        lz4-c \
-        snappy \
-        ccache \
-        zstd \
-        brotli \
-        zlib \
-        cmake \
-        curl \
-        thrift-cpp=0.11.0 \
-        ninja
-
-  # HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
-  # updated sometime on October 2 or October 3
-  conda update -y -q -p $CPP_TOOLCHAIN ca-certificates -c defaults
+  source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
 fi
 
 mkdir -p $ARROW_CPP_BUILD_DIR
diff --git a/ci/travis_build_parquet_cpp.sh b/ci/travis_build_parquet_cpp.sh
index 4b6370ea729fb..fc4ae72c15847 100755
--- a/ci/travis_build_parquet_cpp.sh
+++ b/ci/travis_build_parquet_cpp.sh
@@ -20,9 +20,10 @@ set -e
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
+source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
+
 export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
 
-# $CPP_TOOLCHAIN set up in before_script_cpp
 export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
 
 PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
diff --git a/ci/travis_detect_changes.py b/ci/travis_detect_changes.py
new file mode 100644
index 0000000000000..28426190385ff
--- /dev/null
+++ b/ci/travis_detect_changes.py
@@ -0,0 +1,170 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+
+import functools
+import os
+import pprint
+import sys
+import subprocess
+
+
+perr = functools.partial(print, file=sys.stderr)
+
+LANGUAGE_TOPICS = ['c_glib', 'cpp', 'java', 'js', 'python']
+
+ALL_TOPICS = LANGUAGE_TOPICS + ['integration', 'site']
+
+
+def run_cmd(cmdline):
+    proc = subprocess.Popen(cmdline,
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = proc.communicate()
+    if proc.returncode != 0:
+        raise RuntimeError("Command {cmdline} failed with code {returncode}, "
+                           "stderr was:\n{stderr}\n"
+                           .format(cmdline=cmdline, returncode=proc.returncode,
+                                   stderr=err.decode()))
+    return out
+
+
+def get_commit_description(commit):
+    """
+    Return the textual description (title + body) of the given git commit.
+    """
+    out = run_cmd(["git", "show", "--no-patch", "--pretty=format:%B",
+                   commit])
+    return out.decode('utf-8', 'ignore')
+
+
+def list_affected_files(commit_range):
+    """
+    Return a list of files changed by the given git commit range.
+    """
+    out = run_cmd(["git", "diff", "--name-only", commit_range])
+    return list(filter(None, (s.strip() for s in out.decode().splitlines())))
+
+
+def get_travis_head_commit():
+    return os.environ['TRAVIS_COMMIT']
+
+
+def get_travis_commit_range():
+    cr = os.environ['TRAVIS_COMMIT_RANGE']
+    # See https://github.com/travis-ci/travis-ci/issues/4596#issuecomment-139811122
+    return cr.replace('...', '..')
+
+
+def list_travis_affected_files():
+    """
+    Return a list of files affected in the current Travis build.
+    """
+    commit_range = get_travis_commit_range()
+    try:
+        return list_affected_files(commit_range)
+    except RuntimeError:
+        # TRAVIS_COMMIT_RANGE can contain invalid revisions when
+        # building a branch (not a PR) after rebasing:
+        # https://github.com/travis-ci/travis-ci/issues/2668
+        if os.environ['TRAVIS_EVENT_TYPE'] == 'pull_request':
+            raise
+        # If it's a rebase, it's probably enough to use the last commit only
+        commit_range = '{0}^..'.format(get_travis_head_commit())
+        return list_affected_files(commit_range)
+
+
+def get_affected_topics(affected_files):
+    """
+    Return a dict of topics affected by the given files.
+    Each dict value is True if affected, False otherwise.
+    """
+    affected = dict.fromkeys(ALL_TOPICS, False)
+
+    for path in affected_files:
+        parts = []
+        head = path
+        while head:
+            head, tail = os.path.split(head)
+            parts.append(tail)
+        parts.reverse()
+        assert parts
+        p = parts[0]
+        fn = parts[-1]
+        if fn.startswith('README'):
+            continue
+        if p in ('ci', 'dev', '.travis.yml'):
+            # For these changes, test everything
+            for k in ALL_TOPICS:
+                affected[k] = True
+            break
+        elif p in ('cpp', 'format'):
+            # All languages are potentially affected
+            for k in LANGUAGE_TOPICS:
+                affected[k] = True
+            affected['integration'] = True
+        elif p in ('java', 'js'):
+            affected[p] = True
+            affected['integration'] = True
+        elif p in ('c_glib', 'integration', 'python', 'site'):
+            affected[p] = True
+
+    return affected
+
+
+def make_env_for_topics(affected):
+    return {'ARROW_CI_{0}_AFFECTED'.format(k.upper()): '1' if v else '0'
+            for k, v in affected.items()}
+
+
+def get_unix_shell_eval(env):
+    """
+    Return a shell-evalable string to setup some environment variables.
+    """
+    return "; ".join(("export {0}='{1}'".format(k, v)
+                      for k, v in env.items()))
+
+
+def run_from_travis():
+    desc = get_commit_description(get_travis_head_commit())
+    if '[skip travis]' in desc:
+        # Skip everything
+        affected = dict.fromkeys(ALL_TOPICS, False)
+    elif '[force ci]' in desc or '[force travis]' in desc:
+        # Test everything
+        affected = dict.fromkeys(ALL_TOPICS, True)
+    else:
+        # Test affected topics
+        affected_files = list_travis_affected_files()
+        perr("Affected files:", affected_files)
+        affected = get_affected_topics(affected_files)
+        assert set(affected) <= set(ALL_TOPICS), affected
+
+    perr("Affected topics:")
+    perr(pprint.pformat(affected))
+    return get_unix_shell_eval(make_env_for_topics(affected))
+
+
+if __name__ == "__main__":
+    # This script should have its output evaluated by a shell,
+    # e.g. "eval `python ci/travis_detect_changes.py`"
+    try:
+        print(run_from_travis())
+    except:
+        # Make sure the enclosing eval will return an error
+        print("exit 1")
+        raise
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index caec9bb33ebf2..7896b1252c54c 100755
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -19,28 +19,31 @@
 
 set -e
 
-if [ $TRAVIS_OS_NAME == "linux" ]; then
-  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
-else
-  MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
-fi
+if (! which conda > /dev/null ); then
+  if [ $TRAVIS_OS_NAME == "linux" ]; then
+    MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
+  else
+    MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+  fi
 
-wget --no-verbose -O miniconda.sh $MINICONDA_URL
+  source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+  mkdir -p $CONDA_PKGS_DIRS
 
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-mkdir -p $CONDA_PKGS_DIRS
+  wget --no-verbose -O miniconda.sh $MINICONDA_URL
+  bash miniconda.sh -b -p $MINICONDA
+  export PATH="$MINICONDA/bin:$PATH"
 
-bash miniconda.sh -b -p $MINICONDA
-export PATH="$MINICONDA/bin:$PATH"
-conda update -y -q conda
-conda config --set auto_update_conda false
-conda info -a
+  conda update -y -q conda
+  conda config --set auto_update_conda false
+  conda info -a
 
-conda config --set show_channel_urls True
+  conda config --set show_channel_urls True
 
-# Help with SSL timeouts to S3
-conda config --set remote_connect_timeout_secs 12
+  # Help with SSL timeouts to S3
+  conda config --set remote_connect_timeout_secs 12
+
+  conda config --add channels https://repo.continuum.io/pkgs/free
+  conda config --add channels conda-forge
+fi
 
-conda config --add channels https://repo.continuum.io/pkgs/free
-conda config --add channels conda-forge
 conda info -a
diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh
new file mode 100755
index 0000000000000..e01a084daac86
--- /dev/null
+++ b/ci/travis_install_toolchain.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
+if [ ! -e $CPP_TOOLCHAIN ]; then
+    # Set up C++ toolchain from conda-forge packages for faster builds
+    conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
+        jemalloc=4.4.0 \
+        nomkl \
+        boost-cpp \
+        rapidjson \
+        flatbuffers \
+        gflags \
+        gtest \
+        lz4-c \
+        snappy \
+        ccache \
+        zstd \
+        brotli \
+        zlib \
+        cmake \
+        curl \
+        thrift-cpp=0.11.0 \
+        ninja
+
+    # HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
+    # updated sometime on October 2 or October 3
+    conda update -y -q -p $CPP_TOOLCHAIN ca-certificates -c defaults
+fi
diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
index 096170a9eb0d2..0cbdf1a5cb5e4 100755
--- a/ci/travis_lint.sh
+++ b/ci/travis_lint.sh
@@ -20,25 +20,32 @@
 set -ex
 
 # Fail fast for code linting issues
-mkdir $TRAVIS_BUILD_DIR/cpp/lint
-pushd $TRAVIS_BUILD_DIR/cpp/lint
 
-cmake ..
-make lint
+if [ "$ARROW_CI_CPP_AFFECTED" != "0" ]; then
+  mkdir $TRAVIS_BUILD_DIR/cpp/lint
+  pushd $TRAVIS_BUILD_DIR/cpp/lint
 
-if [ "$ARROW_TRAVIS_CLANG_FORMAT" == "1" ]; then
-  make check-format
+  cmake ..
+  make lint
+
+  if [ "$ARROW_TRAVIS_CLANG_FORMAT" == "1" ]; then
+    make check-format
+  fi
+
+  popd
 fi
 
-popd
 
 # Fail fast on style checks
-sudo pip install -q flake8
 
-PYTHON_DIR=$TRAVIS_BUILD_DIR/python
+if [ "$ARROW_CI_PYTHON_AFFECTED" != "0" ]; then
+  sudo pip install -q flake8
 
-flake8 --count $PYTHON_DIR/pyarrow
+  PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 
-# Check Cython files with some checks turned off
-flake8 --count --config=$PYTHON_DIR/.flake8.cython \
-       $PYTHON_DIR/pyarrow
+  flake8 --count $PYTHON_DIR/pyarrow
+
+  # Check Cython files with some checks turned off
+  flake8 --count --config=$PYTHON_DIR/.flake8.cython \
+         $PYTHON_DIR/pyarrow
+fi
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 0c415dc4865cd..8c6da38e1eefe 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -20,6 +20,9 @@
 set -e
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
 export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug
 
 pushd $ARROW_JAVA_DIR
diff --git a/ci/travis_script_java.sh b/ci/travis_script_java.sh
index 58e5d42708b30..a8ad94c5e8d66 100755
--- a/ci/travis_script_java.sh
+++ b/ci/travis_script_java.sh
@@ -25,6 +25,5 @@ pushd $JAVA_DIR
 
 export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
 mvn -B install
-[ "${ARROW_TRAVIS_SKIP_SITE}" = "yes" ] || mvn -B site
 
 popd
diff --git a/ci/travis_script_javadoc.sh b/ci/travis_script_javadoc.sh
new file mode 100755
index 0000000000000..ccfb2dab61d05
--- /dev/null
+++ b/ci/travis_script_javadoc.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+
+pushd $JAVA_DIR
+
+export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
+mvn -B site
+
+popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 4187e7541ebd4..dbaaab89e1740 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -20,6 +20,8 @@ set -e
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
 export ARROW_HOME=$ARROW_CPP_INSTALL
 export PARQUET_HOME=$ARROW_PYTHON_PARQUET_HOME
 export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH
@@ -102,6 +104,6 @@ python -m pytest -vv -r sxX --durations=15 -s $PYARROW_PATH --parquet
 if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
   # Build documentation once
   pushd $ARROW_PYTHON_DIR/doc
-  sphinx-build -b html -d _build/doctrees -W source _build/html
+  sphinx-build -q -b html -d _build/doctrees -W source _build/html
   popd
 fi

From 88b72df27514a8ec47736bca1da010d9a2f84f3c Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 8 Feb 2018 13:22:23 +0100
Subject: [PATCH 1458/1644] ARROW-2073: [Python] Create struct array from
 sequence of tuples

Author: Antoine Pitrou <antoine@python.org>

Closes #1572 from pitrou/ARROW-2073-struct-from-tuples and squashes the following commits:

0a41ccc [Antoine Pitrou] ARROW-2073: [Python] Create struct array from sequence of tuples
---
 cpp/src/arrow/python/builtin_convert.cc      | 47 +++++++++++++++-----
 python/benchmarks/convert_builtins.py        | 21 +++++++--
 python/pyarrow/tests/test_convert_builtin.py | 39 ++++++++++++++++
 3 files changed, 93 insertions(+), 14 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 1e431c29f06a0..f0e5449b63464 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -771,18 +771,21 @@ class StructConverter : public TypedConverterVisitor<StructBuilder, StructConver
   // Append a non-missing item
   Status AppendItem(PyObject* obj) {
     RETURN_NOT_OK(typed_builder_->Append());
-    if (!PyDict_Check(obj)) {
-      return Status::TypeError("dict value expected for struct type");
+    // Note heterogenous sequences are not allowed
+    if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
+      if (PyDict_Check(obj)) {
+        source_kind_ = DICTS;
+      } else if (PyTuple_Check(obj)) {
+        source_kind_ = TUPLES;
+      }
     }
-    // NOTE we're ignoring any extraneous dict items
-    for (int i = 0; i < num_fields_; i++) {
-      PyObject* nameobj = PyList_GET_ITEM(field_name_list_.obj(), i);
-      PyObject* valueobj = PyDict_GetItem(obj, nameobj);  // borrowed
-      RETURN_IF_PYERROR();
-      RETURN_NOT_OK(value_converters_[i]->AppendSingle(valueobj ? valueobj : Py_None));
+    if (PyDict_Check(obj) && source_kind_ == DICTS) {
+      return AppendDictItem(obj);
+    } else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
+      return AppendTupleItem(obj);
+    } else {
+      return Status::TypeError("Expected sequence of dicts or tuples for struct type");
     }
-
-    return Status::OK();
   }
 
   // Append a missing item
@@ -797,9 +800,33 @@ class StructConverter : public TypedConverterVisitor<StructBuilder, StructConver
   }
 
  protected:
+  Status AppendDictItem(PyObject* obj) {
+    // NOTE we're ignoring any extraneous dict items
+    for (int i = 0; i < num_fields_; i++) {
+      PyObject* nameobj = PyList_GET_ITEM(field_name_list_.obj(), i);
+      PyObject* valueobj = PyDict_GetItem(obj, nameobj);  // borrowed
+      RETURN_IF_PYERROR();
+      RETURN_NOT_OK(value_converters_[i]->AppendSingle(valueobj ? valueobj : Py_None));
+    }
+    return Status::OK();
+  }
+
+  Status AppendTupleItem(PyObject* obj) {
+    if (PyTuple_GET_SIZE(obj) != num_fields_) {
+      return Status::Invalid("Tuple size must be equal to number of struct fields");
+    }
+    for (int i = 0; i < num_fields_; i++) {
+      PyObject* valueobj = PyTuple_GET_ITEM(obj, i);
+      RETURN_NOT_OK(value_converters_[i]->AppendSingle(valueobj));
+    }
+    return Status::OK();
+  }
+
   std::vector<std::unique_ptr<SeqConverter>> value_converters_;
   OwnedRef field_name_list_;
   int num_fields_;
+  // Whether we're converting from a sequence of dicts or tuples
+  enum { UNKNOWN, DICTS, TUPLES } source_kind_ = UNKNOWN;
 };
 
 class DecimalConverter
diff --git a/python/benchmarks/convert_builtins.py b/python/benchmarks/convert_builtins.py
index 92b2b850f2a0a..a4dc9f2620c1e 100644
--- a/python/benchmarks/convert_builtins.py
+++ b/python/benchmarks/convert_builtins.py
@@ -144,11 +144,21 @@ def generate_int_list_list(self, n, min_size, max_size,
             partial(self.generate_int_list, none_prob=none_prob),
             n, min_size, max_size, none_prob)
 
+    def generate_tuple_list(self, n, none_prob=DEFAULT_NONE_PROB):
+        """
+        Generate a list of tuples with random values.
+        Each tuple has the form `(int value, float value, bool value)`
+        """
+        dicts = self.generate_dict_list(n, none_prob=none_prob)
+        tuples = [(d.get('u'), d.get('v'), d.get('w'))
+                  if d is not None else None
+                  for d in dicts]
+        assert len(tuples) == n
+        return tuples
 
     def generate_dict_list(self, n, none_prob=DEFAULT_NONE_PROB):
         """
-        Generate a list of dicts with a random size between *min_size* and
-        *max_size*.
+        Generate a list of dicts with random values.
         Each dict has the form `{'u': int value, 'v': float value, 'w': bool value}`
         """
         ints = self.generate_int_list(n, none_prob=none_prob)
@@ -179,12 +189,14 @@ def get_type_and_builtins(self, n, type_name):
         """
         size = None
 
-        if type_name in ('bool', 'ascii', 'unicode', 'int64 list', 'struct'):
+        if type_name in ('bool', 'ascii', 'unicode', 'int64 list'):
             kind = type_name
         elif type_name.startswith(('int', 'uint')):
             kind = 'int'
         elif type_name.startswith('float'):
             kind = 'float'
+        elif type_name.startswith('struct'):
+            kind = 'struct'
         elif type_name == 'binary':
             kind = 'varying binary'
         elif type_name.startswith('binary'):
@@ -226,6 +238,7 @@ def get_type_and_builtins(self, n, type_name):
             'int64 list': partial(self.generate_int_list_list,
                                   min_size=0, max_size=20),
             'struct': self.generate_dict_list,
+            'struct from tuples': self.generate_tuple_list,
         }
         data = factories[kind](n)
         return ty, data
@@ -239,7 +252,7 @@ class ConvertPyListToArray(object):
     types = ('int32', 'uint32', 'int64', 'uint64',
              'float32', 'float64', 'bool',
              'binary', 'binary10', 'ascii', 'unicode',
-             'int64 list', 'struct')
+             'int64 list', 'struct', 'struct from tuples')
 
     param_names = ['type']
     params = [types]
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index ce54f23eb0c6b..5cd4a52a21839 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -531,6 +531,45 @@ def test_struct_from_dicts():
     assert arr.to_pylist() == expected
 
 
+def test_struct_from_tuples():
+    ty = pa.struct([pa.field('a', pa.int32()),
+                    pa.field('b', pa.string()),
+                    pa.field('c', pa.bool_())])
+
+    data = [(5, 'foo', True),
+            (6, 'bar', False)]
+    expected = [{'a': 5, 'b': 'foo', 'c': True},
+                {'a': 6, 'b': 'bar', 'c': False}]
+    arr = pa.array(data, type=ty)
+    assert arr.to_pylist() == expected
+
+    # With omitted values
+    data = [(5, 'foo', None),
+            None,
+            (6, None, False)]
+    expected = [{'a': 5, 'b': 'foo', 'c': None},
+                None,
+                {'a': 6, 'b': None, 'c': False}]
+    arr = pa.array(data, type=ty)
+    assert arr.to_pylist() == expected
+
+    # Invalid tuple size
+    for tup in [(5, 'foo'), (), ('5', 'foo', True, None)]:
+        with pytest.raises(ValueError, match="(?i)tuple size"):
+            pa.array([tup], type=ty)
+
+
+def test_struct_from_mixed_sequence():
+    # It is forbidden to mix dicts and tuples when initializing a struct array
+    ty = pa.struct([pa.field('a', pa.int32()),
+                    pa.field('b', pa.string()),
+                    pa.field('c', pa.bool_())])
+    data = [(5, 'foo', True),
+            {'a': 6, 'b': 'bar', 'c': False}]
+    with pytest.raises(TypeError):
+        pa.array(data, type=ty)
+
+
 def test_structarray_from_arrays_coerce():
     # ARROW-1706
     ints = [None, 2, 3]

From 4fe3a0be8be41f6843065c5aed528bcc907070b7 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Thu, 8 Feb 2018 14:04:14 -0500
Subject: [PATCH 1459/1644] ARROW-2114: [Python] Pull latest docker manylinux1
 image [skip appveyor]

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1577 from xhochy/ARROW-2114 and squashes the following commits:

1dcadba9 [Uwe L. Korn] ARROW-2114: [Python] Pull latest docker manylinux1 image [skip appveyor]
---
 python/manylinux1/Dockerfile-x86_64 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 1ade9ab100036..919a32be715b0 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2087
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp

From 163be4204da1d1aa37c73e3f6eb3eb2fe8e9f7ff Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 8 Feb 2018 13:59:22 -0500
Subject: [PATCH 1460/1644] [Doc] Refine top of README, add GLib bindings to
 README, fix ASF link

Change-Id: I227eab4be9d8df54a71731c7b52496aff09d0f6a
---
 README.md | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 15a9ffea2a7a3..a7fed986526dd 100644
--- a/README.md
+++ b/README.md
@@ -30,20 +30,22 @@
   </tr>
 </table>
 
-### Powering Columnar In-Memory Analytics
+### Powering In-Memory Analytics
 
-Arrow is a set of technologies that enable big-data systems to process and move data fast.
+Apache Arrow is a development platform for in-memory analytics. It contains a
+set of technologies that enable big data systems to process and move data fast.
 
-Initial implementations include:
+Major components of the project include:
 
- - [The Arrow Format](https://github.com/apache/arrow/tree/master/format)
+ - [The Arrow Columnar In-Memory Format](https://github.com/apache/arrow/tree/master/format)
  - [C++ implementation](https://github.com/apache/arrow/tree/master/cpp)
+ - [C bindings using GLib](https://github.com/apache/arrow/tree/master/c_glib)
  - [Java implementation](https://github.com/apache/arrow/tree/master/java)
  - [JavaScript implementation](https://github.com/apache/arrow/tree/master/js)
- - [Python interface to C++ libraries](https://github.com/apache/arrow/tree/master/python)
+ - [Python bindings to C++](https://github.com/apache/arrow/tree/master/python)
 
-Arrow is an [Apache Software Foundation](www.apache.org) project. Learn more at
-[arrow.apache.org](http://arrow.apache.org).
+Arrow is an [Apache Software Foundation](https://www.apache.org) project. Learn more at
+[arrow.apache.org](https://arrow.apache.org).
 
 ### What's in the Arrow libraries?
 
@@ -62,8 +64,7 @@ components:
   interprocess communication (IPC)
 - Integration tests for verifying binary compatibility between the
   implementations (e.g. sending data from Java to C++)
-- Conversions to and from other in-memory data structures (e.g. Python's pandas
-  library)
+- Conversions to and from other in-memory data structures
 
 ### Getting involved
 

From 0417b1420019d5c6611bba36aaa5eb59765618ef Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Fri, 9 Feb 2018 12:30:56 -0500
Subject: [PATCH 1461/1644] ARROW-2123: [JS] Upgrade to TS 2.7.1

Bump TS version to 2.7.1 and fix some related errors

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1582 from TheNeuralBit/ts-upgrade and squashes the following commits:

df47bfa0 [Brian Hulette] linter
95aa2b7e [Brian Hulette] Upgrade to TS 2.7.1
---
 js/package.json             |  2 +-
 js/src/data.ts              |  2 +-
 js/src/vector/chunked.ts    |  1 +
 js/test/unit/table-tests.ts | 16 ++++++++--------
 4 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/js/package.json b/js/package.json
index e553b56cd0a10..d36e872638bc5 100644
--- a/js/package.json
+++ b/js/package.json
@@ -97,7 +97,7 @@
     "trash": "4.2.1",
     "ts-jest": "22.0.1",
     "tslint": "5.9.1",
-    "typescript": "2.6.2",
+    "typescript": "2.7.1",
     "uglifyjs-webpack-plugin": "1.1.6",
     "webpack": "3.10.0",
     "xml2js": "0.4.19"
diff --git a/js/src/data.ts b/js/src/data.ts
index 81d19a3cf63cd..ab2e40ae9a912 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -142,7 +142,7 @@ export class FlatListData<T extends FlatListType> extends FlatData<T> {
         this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
     }
     public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
-        return new FlatListData(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this[VectorType.DATA], offset, nullCount);
+        return new FlatListData(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this[VectorType.DATA], offset, nullCount) as FlatListData<R>;
     }
 }
 
diff --git a/js/src/vector/chunked.ts b/js/src/vector/chunked.ts
index c0087fdd75833..2eaf99c7cc131 100644
--- a/js/src/vector/chunked.ts
+++ b/js/src/vector/chunked.ts
@@ -22,6 +22,7 @@ import { DataType, TypedArray, IterableArrayLike } from '../type';
 export class ChunkedView<T extends DataType> implements View<T> {
     public chunkVectors: Vector<T>[];
     public chunkOffsets: Uint32Array;
+    // @ts-ignore
     protected _children: Vector<any>[];
     constructor(data: ChunkedData<T>) {
         this.chunkVectors = data.chunkVectors;
diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts
index 36d2ae9038147..8dd30de935e80 100644
--- a/js/test/unit/table-tests.ts
+++ b/js/test/unit/table-tests.ts
@@ -313,8 +313,8 @@ describe(`Table`, () => {
                     for (let batch of table.batches) {
                         expect(bind).toHaveBeenCalledWith(batch);
                     }
-                })
-            })
+                });
+            });
             test(`count() returns the correct length`, () => {
                 expect(table.count()).toEqual(values.length);
             });
@@ -331,7 +331,7 @@ describe(`Table`, () => {
                 }, {
                     name:     `filter on 0 <= f32`,
                     filtered: table.filter(lit(0).lteq(col('f32'))),
-                    expected: values.filter((row)=>0 <= row[F32])
+                    expected: values.filter((row) => 0 <= row[F32])
                 }, {
                     name:     `filter on i32 <= 0`,
                     filtered: table.filter(col('i32').lteq(0)),
@@ -339,11 +339,11 @@ describe(`Table`, () => {
                 }, {
                     name:     `filter on 0 >= i32`,
                     filtered: table.filter(lit(0).gteq(col('i32'))),
-                    expected: values.filter((row)=>0 >= row[I32])
+                    expected: values.filter((row) => 0 >= row[I32])
                 }, {
                     name:     `filter on f32 <= -.25 || f3 >= .25`,
                     filtered: table.filter(col('f32').lteq(-.25).or(col('f32').gteq(.25))),
-                    expected: values.filter((row)=>row[F32] <= -.25 || row[F32] >= .25)
+                    expected: values.filter((row) => row[F32] <= -.25 || row[F32] >= .25)
                 }, {
                     name:     `filter method combines predicates (f32 >= 0 && i32 <= 0)`,
                     filtered: table.filter(col('i32').lteq(0)).filter(col('f32').gteq(0)),
@@ -355,15 +355,15 @@ describe(`Table`, () => {
                 }, {
                     name:     `filter on 'a' == dictionary (commutativity)`,
                     filtered: table.filter(lit('a').eq(col('dictionary'))),
-                    expected: values.filter((row)=>row[DICT] === 'a')
+                    expected: values.filter((row) => row[DICT] === 'a')
                 }, {
                     name:     `filter on f32 >= i32`,
                     filtered: table.filter(col('f32').gteq(col('i32'))),
-                    expected: values.filter((row)=>row[F32] >= row[I32])
+                    expected: values.filter((row) => row[F32] >= row[I32])
                 }, {
                     name:     `filter on f32 <= i32`,
                     filtered: table.filter(col('f32').lteq(col('i32'))),
-                    expected: values.filter((row)=>row[F32] <= row[I32])
+                    expected: values.filter((row) => row[F32] <= row[I32])
                 }
             ];
             for (let this_test of filter_tests) {

From ddae2407fe26c97316f3b09f9b20da0e6785703c Mon Sep 17 00:00:00 2001
From: "rip.nsk" <rip.nsk@gmail.com>
Date: Fri, 9 Feb 2018 14:01:22 -0500
Subject: [PATCH 1462/1644] ARROW-2120: [C++] Add possibility to use empty
 _MSVC_STATIC_LIB_SUFFIX for Thirdparties

Author: rip.nsk <rip.nsk@gmail.com>

Closes #1580 from rip-nsk/ARROW-2120 and squashes the following commits:

7d1b1554 [rip.nsk] Add possibility to use empty _MSVC_STATIC_LIB_SUFFIX for Thirdparties
---
 cpp/cmake_modules/FindArrow.cmake   | 2 +-
 cpp/cmake_modules/FindGFlags.cmake  | 2 +-
 cpp/cmake_modules/FindLz4.cmake     | 2 +-
 cpp/cmake_modules/FindParquet.cmake | 2 +-
 cpp/cmake_modules/FindZSTD.cmake    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cpp/cmake_modules/FindArrow.cmake b/cpp/cmake_modules/FindArrow.cmake
index bce4404a47809..70defd6525c5b 100644
--- a/cpp/cmake_modules/FindArrow.cmake
+++ b/cpp/cmake_modules/FindArrow.cmake
@@ -70,7 +70,7 @@ get_filename_component(ARROW_PYTHON_LIBS ${ARROW_PYTHON_LIB_PATH} DIRECTORY)
 if (MSVC)
   SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
 
-  if (MSVC AND NOT ARROW_MSVC_STATIC_LIB_SUFFIX)
+  if (MSVC AND NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
     set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
   endif()
 
diff --git a/cpp/cmake_modules/FindGFlags.cmake b/cpp/cmake_modules/FindGFlags.cmake
index 63a55d75133b6..f71950cb530cd 100644
--- a/cpp/cmake_modules/FindGFlags.cmake
+++ b/cpp/cmake_modules/FindGFlags.cmake
@@ -29,7 +29,7 @@ elseif ( GFlags_HOME )
     list( APPEND _gflags_roots ${GFlags_HOME} )
 endif()
 
-if (MSVC AND NOT GFLAGS_MSVC_STATIC_LIB_SUFFIX)
+if (MSVC AND NOT DEFINED GFLAGS_MSVC_STATIC_LIB_SUFFIX)
   set(GFLAGS_MSVC_STATIC_LIB_SUFFIX "_static")
 endif()
 
diff --git a/cpp/cmake_modules/FindLz4.cmake b/cpp/cmake_modules/FindLz4.cmake
index 07707cf7c6fa9..3a89c8d028802 100644
--- a/cpp/cmake_modules/FindLz4.cmake
+++ b/cpp/cmake_modules/FindLz4.cmake
@@ -29,7 +29,7 @@ elseif ( Lz4_HOME )
     list( APPEND _lz4_roots ${Lz4_HOME} )
 endif()
 
-if (MSVC AND NOT LZ4_MSVC_STATIC_LIB_SUFFIX)
+if (MSVC AND NOT DEFINED LZ4_MSVC_STATIC_LIB_SUFFIX)
   set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static")
 endif()
 
diff --git a/cpp/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
index 8bbe05f127f87..68b004c9f428f 100644
--- a/cpp/cmake_modules/FindParquet.cmake
+++ b/cpp/cmake_modules/FindParquet.cmake
@@ -33,7 +33,7 @@ endif()
 if (MSVC)
   SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
 
-  if (MSVC AND NOT PARQUET_MSVC_STATIC_LIB_SUFFIX)
+  if (MSVC AND NOT DEFINED PARQUET_MSVC_STATIC_LIB_SUFFIX)
     set(PARQUET_MSVC_STATIC_LIB_SUFFIX "_static")
   endif()
 
diff --git a/cpp/cmake_modules/FindZSTD.cmake b/cpp/cmake_modules/FindZSTD.cmake
index 02a0c39bb7193..810e5c0bed72a 100644
--- a/cpp/cmake_modules/FindZSTD.cmake
+++ b/cpp/cmake_modules/FindZSTD.cmake
@@ -29,7 +29,7 @@ elseif ( ZStd_HOME )
     list( APPEND _zstd_roots ${ZStd_HOME} )
 endif()
 
-if (MSVC AND NOT ZSTD_MSVC_STATIC_LIB_SUFFIX)
+if (MSVC AND NOT DEFINED ZSTD_MSVC_STATIC_LIB_SUFFIX)
   set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static")
 endif()
 

From b18e1590ff1022d90a991a70a76d4dff40bf09a9 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 9 Feb 2018 16:06:57 -0500
Subject: [PATCH 1463/1644] ARROW-1973: [Python] Memory leak when converting
 Arrow tables with array columns to Pandas dataframes.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1578 from cpcloud/ARROW-1973 and squashes the following commits:

f1b061a0 [Phillip Cloud] ARROW-1973: [Python] Memory leak when converting Arrow tables with array columns to Pandas dataframes.
---
 cpp/src/arrow/python/arrow_to_pandas.cc | 88 ++++++++++++++++++-------
 cpp/src/arrow/python/common.cc          | 20 ++++--
 python/pyarrow/array.pxi                |  2 +-
 python/pyarrow/table.pxi                |  4 +-
 4 files changed, 82 insertions(+), 32 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index a17d14bf62e25..60a2eae5d43f9 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -502,19 +502,23 @@ template <typename ArrowType>
 inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Column>& col,
                                PyObject** out_values) {
   const ChunkedArray& data = *col->data().get();
-  auto list_type = std::static_pointer_cast<ListType>(col->type());
+  const auto& list_type = static_cast<const ListType&>(*col->type());
 
   // Get column of underlying value arrays
   std::vector<std::shared_ptr<Array>> value_arrays;
   for (int c = 0; c < data.num_chunks(); c++) {
-    auto arr = std::static_pointer_cast<ListArray>(data.chunk(c));
-    value_arrays.emplace_back(arr->values());
+    const auto& arr = static_cast<const ListArray&>(*data.chunk(c));
+    value_arrays.emplace_back(arr.values());
   }
-  auto flat_column = std::make_shared<Column>(list_type->value_field(), value_arrays);
+  auto flat_column = std::make_shared<Column>(list_type.value_field(), value_arrays);
   // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
   //    Storing a reference to the whole Array would be to expensive.
-  PyObject* numpy_array;
-  RETURN_NOT_OK(ConvertColumnToPandas(options, flat_column, nullptr, &numpy_array));
+
+  OwnedRef owned_numpy_array;
+  RETURN_NOT_OK(
+      ConvertColumnToPandas(options, flat_column, nullptr, owned_numpy_array.ref()));
+
+  PyObject* numpy_array = owned_numpy_array.obj();
 
   PyAcquireGIL lock;
 
@@ -528,24 +532,20 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
         Py_INCREF(Py_None);
         *out_values = Py_None;
       } else {
-        PyObject* start = PyLong_FromLongLong(arr->value_offset(i) + chunk_offset);
-        PyObject* end = PyLong_FromLongLong(arr->value_offset(i + 1) + chunk_offset);
-        PyObject* slice = PySlice_New(start, end, NULL);
-        Py_XDECREF(start);
-        Py_XDECREF(end);
+        OwnedRef start(PyLong_FromLongLong(arr->value_offset(i) + chunk_offset));
+        OwnedRef end(PyLong_FromLongLong(arr->value_offset(i + 1) + chunk_offset));
+        OwnedRef slice(PySlice_New(start.obj(), end.obj(), nullptr));
 
-        if (ARROW_PREDICT_FALSE(slice == nullptr)) {
+        if (ARROW_PREDICT_FALSE(slice.obj() == nullptr)) {
           // Fall out of loop, will return from RETURN_IF_PYERROR
           break;
         }
-        *out_values = PyObject_GetItem(numpy_array, slice);
+        *out_values = PyObject_GetItem(numpy_array, slice.obj());
 
         if (*out_values == nullptr) {
           // Fall out of loop, will return from RETURN_IF_PYERROR
           break;
         }
-
-        Py_XDECREF(slice);
       }
       ++out_values;
     }
@@ -554,7 +554,6 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
     chunk_offset += arr->values()->length();
   }
 
-  Py_XDECREF(numpy_array);
   return Status::OK();
 }
 
@@ -1447,7 +1446,7 @@ class ArrowDeserializer {
 
   template <int TYPE>
   Status ConvertValuesZeroCopy(PandasOptions options, int npy_type,
-                               std::shared_ptr<Array> arr) {
+                               const std::shared_ptr<Array>& arr) {
     typedef typename internal::arrow_traits<TYPE>::T T;
 
     const T* in_values = GetPrimitiveValues<T>(*arr);
@@ -1461,15 +1460,57 @@ class ArrowDeserializer {
     result_ = NewArray1DFromType(col_->type().get(), npy_type, col_->length(), data);
     arr_ = reinterpret_cast<PyArrayObject*>(result_);
 
-    if (arr_ == NULL) {
+    if (arr_ == nullptr) {
       // Error occurred, trust that error set
       return Status::OK();
     }
 
+    // See ARROW-1973 for the original memory leak report.
+    //
+    // There are two scenarios: py_ref_ is nullptr or py_ref_ is not nullptr
+    //
+    //   1. py_ref_ is nullptr (it **was not** passed in to ArrowDeserializer's
+    //      constructor)
+    //
+    //      In this case, the stolen reference must not be incremented since nothing
+    //      outside of the PyArrayObject* (the arr_ member) is holding a reference to
+    //      it. If we increment this, then we have a memory leak.
+    //
+    //
+    //      Here's an example of how memory can be leaked when converting an arrow Array
+    //      of List<Float64>.to a numpy array
+    //
+    //      1. Create a 1D numpy that is the flattened arrow array.
+    //
+    //         There's nothing outside of the serializer that owns this new numpy array.
+    //
+    //      2. Make a capsule for the base array.
+    //
+    //         The reference count of base is 1.
+    //
+    //      3. Call PyArray_SetBaseObject(arr_, base)
+    //
+    //         The reference count is still 1, because the reference is stolen.
+    //
+    //      4. Increment the reference count of base (unconditionally)
+    //
+    //         The reference count is now 2. This is okay if there's an object holding
+    //         another reference. The PyArrayObject that stole the reference will
+    //         eventually decrement the reference count, which will leaves us with a
+    //         refcount of 1, with nothing owning that 1 reference. Memory leakage
+    //         ensues.
+    //
+    //   2. py_ref_ is not nullptr (it **was** passed in to ArrowDeserializer's
+    //      constructor)
+    //
+    //      This case is simpler. We assume that the reference accounting is correct
+    //      coming in. We need to preserve that accounting knowing that the
+    //      PyArrayObject that stole the reference will eventually decref it, thus we
+    //      increment the reference count.
+
     PyObject* base;
     if (py_ref_ == nullptr) {
-      ArrowCapsule* capsule = new ArrowCapsule;
-      capsule->array = arr;
+      auto capsule = new ArrowCapsule{{arr}};
       base = PyCapsule_New(reinterpret_cast<void*>(capsule), "arrow",
                            &ArrowCapsule_Destructor);
       if (base == nullptr) {
@@ -1478,14 +1519,13 @@ class ArrowDeserializer {
       }
     } else {
       base = py_ref_;
+      Py_INCREF(base);
     }
 
     if (PyArray_SetBaseObject(arr_, base) == -1) {
       // Error occurred, trust that SetBaseObject set the error state
+      Py_XDECREF(base);
       return Status::OK();
-    } else {
-      // PyArray_SetBaseObject steals our reference to base
-      Py_INCREF(base);
     }
 
     // Arrow data is immutable.
@@ -1686,7 +1726,9 @@ class ArrowDeserializer {
     RETURN_IF_PYERROR();
 
     PyDict_SetItemString(result_, "indices", block->block_arr());
+    RETURN_IF_PYERROR();
     PyDict_SetItemString(result_, "dictionary", block->dictionary());
+    RETURN_IF_PYERROR();
 
     return Status::OK();
   }
diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
index 53bd57bf6da94..14a8ae6fd5b85 100644
--- a/cpp/src/arrow/python/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -65,16 +65,22 @@ PyBuffer::~PyBuffer() {
 
 Status CheckPyError(StatusCode code) {
   if (PyErr_Occurred()) {
-    PyObject *exc_type, *exc_value, *traceback;
+    PyObject* exc_type = nullptr;
+    PyObject* exc_value = nullptr;
+    PyObject* traceback = nullptr;
+
+    OwnedRef exc_type_ref(exc_type);
+    OwnedRef exc_value_ref(exc_value);
+    OwnedRef traceback_ref(traceback);
+
     PyErr_Fetch(&exc_type, &exc_value, &traceback);
+
     PyErr_NormalizeException(&exc_type, &exc_value, &traceback);
-    PyObject* exc_value_str = PyObject_Str(exc_value);
-    PyObjectStringify stringified(exc_value_str);
+
+    OwnedRef exc_value_str(PyObject_Str(exc_value));
+    PyObjectStringify stringified(exc_value_str.obj());
     std::string message(stringified.bytes);
-    Py_XDECREF(exc_type);
-    Py_XDECREF(exc_value);
-    Py_XDECREF(exc_value_str);
-    Py_XDECREF(traceback);
+
     PyErr_Clear();
     return Status(code, message);
   }
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 9b6ae0fc634e1..f85363cb12959 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -414,7 +414,7 @@ cdef class Array:
         return pyarrow_wrap_array(result)
 
     def to_pandas(self, c_bool strings_to_categorical=False,
-                  zero_copy_only=False):
+                  c_bool zero_copy_only=False):
         """
         Convert to an array object suitable for use in pandas
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index b03ee26702245..e14d4739f805e 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -271,7 +271,9 @@ cdef class Column:
         casted_data = pyarrow_wrap_chunked_array(out.chunked_array())
         return column(self.name, casted_data)
 
-    def to_pandas(self, strings_to_categorical=False, zero_copy_only=False):
+    def to_pandas(self,
+                  c_bool strings_to_categorical=False,
+                  c_bool zero_copy_only=False):
         """
         Convert the arrow::Column to a pandas.Series
 

From 9d9cc8f79ea869daaf41d85772bb0274b8c2ded3 Mon Sep 17 00:00:00 2001
From: moriyoshi <mozo@mozo.jp>
Date: Sat, 10 Feb 2018 11:05:06 -0500
Subject: [PATCH 1464/1644] ARROW-2049: [Python] Use python -m cython to run
 Cython, instead of CYTHON_EXECUTABLE

Author: moriyoshi <mozo@mozo.jp>

Closes #1583 from cpcloud/ARROW-2049 and squashes the following commits:

ca5dbbff [moriyoshi] ARROW-2049: [Python] Try searching for the Cython executable according to sysconfig
---
 cpp/cmake_modules/FindCython.cmake | 30 ------------------------------
 cpp/cmake_modules/UseCython.cmake  |  3 +--
 2 files changed, 1 insertion(+), 32 deletions(-)
 delete mode 100644 cpp/cmake_modules/FindCython.cmake

diff --git a/cpp/cmake_modules/FindCython.cmake b/cpp/cmake_modules/FindCython.cmake
deleted file mode 100644
index 9df3b5d59d274..0000000000000
--- a/cpp/cmake_modules/FindCython.cmake
+++ /dev/null
@@ -1,30 +0,0 @@
-# Find the Cython compiler.
-#
-# This code sets the following variables:
-#
-#  CYTHON_EXECUTABLE
-#
-# See also UseCython.cmake
-
-#=============================================================================
-# Copyright 2011 Kitware, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#=============================================================================
-
-find_program( CYTHON_EXECUTABLE NAMES cython cython.bat )
-
-include( FindPackageHandleStandardArgs )
-FIND_PACKAGE_HANDLE_STANDARD_ARGS( Cython REQUIRED_VARS CYTHON_EXECUTABLE )
-
-mark_as_advanced( CYTHON_EXECUTABLE )
diff --git a/cpp/cmake_modules/UseCython.cmake b/cpp/cmake_modules/UseCython.cmake
index 7920940e688c7..a1cfac6e99e29 100644
--- a/cpp/cmake_modules/UseCython.cmake
+++ b/cpp/cmake_modules/UseCython.cmake
@@ -66,7 +66,6 @@ set( CYTHON_FLAGS "" CACHE STRING
   "Extra flags to the cython compiler." )
 mark_as_advanced( CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS)
 
-find_package( Cython REQUIRED )
 find_package( PythonLibsNew REQUIRED )
 
 set( CYTHON_CXX_EXTENSION "cxx" )
@@ -132,7 +131,7 @@ function( compile_pyx _name pyx_target_name generated_files pyx_file)
 
   # Add the command to run the compiler.
   add_custom_target(${pyx_target_name}
-    COMMAND ${CYTHON_EXECUTABLE} ${cxx_arg} ${include_directory_arg}
+    COMMAND ${PYTHON_EXECUTABLE} -m cython ${cxx_arg} ${include_directory_arg}
     ${annotate_arg} ${no_docstrings_arg} ${cython_debug_arg}
     ${CYTHON_FLAGS}
     --output-file "${_name}.${extension}" ${pyx_location}

From 41a42de75500b2306e69dec0f9eb38c059ee808a Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 12 Feb 2018 00:39:21 -0500
Subject: [PATCH 1465/1644] ARROW-2128: [Python] Support arrays of empty lists

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1587 from xhochy/ARROW-2128 and squashes the following commits:

36015cdf [Uwe L. Korn] ARROW-2128: [Python] Support arrays of empty lists
---
 cpp/src/arrow/python/builtin_convert.cc     | 20 ++++++++------------
 python/pyarrow/tests/test_convert_pandas.py |  8 ++++++++
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index f0e5449b63464..60f58ea5b3b7b 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -200,22 +200,18 @@ class SeqVisitor {
 
   std::shared_ptr<DataType> GetType() {
     // If all the non-list inputs were null (or there were no inputs)
+    std::shared_ptr<DataType> result;
     if (scalars_.total_count() == 0) {
-      if (max_nesting_level_ == 0) {
-        // If its just a single empty list or list of nulls, return null.
-        return null();
-      } else {
-        // Error, if we have nesting but no concrete base type.
-        return nullptr;
-      }
+      // Lists of Lists of NULL
+      result = null();
     } else {
       // Lists of Lists of [X]
-      std::shared_ptr<DataType> result = scalars_.GetType();
-      for (int i = 0; i < max_nesting_level_; ++i) {
-        result = std::make_shared<ListType>(result);
-      }
-      return result;
+      result = scalars_.GetType();
     }
+    for (int i = 0; i < max_nesting_level_; ++i) {
+      result = std::make_shared<ListType>(result);
+    }
+    return result;
   }
 
   Status Validate() const {
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 7dbf0d7ed8e1c..53e37f5399900 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -851,6 +851,14 @@ def test_nested_lists_all_none(self):
         expected = pa.array([None, None, [None, None], [None, None]])
         assert arr.equals(expected)
 
+    def test_nested_lists_all_empty(self):
+        # ARROW-2128
+        data = pd.Series([[], [], []])
+        arr = pa.array(data)
+        expected = pa.array(list(data))
+        assert arr.equals(expected)
+        assert arr.type == pa.list_(pa.null())
+
     def test_threaded_conversion(self):
         df = _alltypes_example()
         _check_pandas_roundtrip(df, nthreads=2)

From a6bcaa7645ce6a71016cd0fac784d88699e3b210 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 12 Feb 2018 00:42:29 -0500
Subject: [PATCH 1466/1644] ARROW-2106: [Python] Add conversion for a series of
 datetime objects

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1585 from xhochy/ARROW-2106 and squashes the following commits:

d8be4863 [Uwe L. Korn] ARROW-2106: [Python] Add conversion for a series of datetime objects
---
 cpp/src/arrow/python/numpy_to_arrow.cc      | 32 +++++++++++++++++++++
 python/pyarrow/tests/test_convert_pandas.py | 18 +++++++++++-
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 1b1673bb88b4a..d487d9d9dbc85 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -475,6 +475,7 @@ class NumPyConverter {
   Status ConvertLists(const std::shared_ptr<DataType>& type, ListBuilder* builder,
                       PyObject* list);
   Status ConvertDecimals();
+  Status ConvertDateTimes();
   Status ConvertTimes();
   Status ConvertObjectsInfer();
   Status ConvertObjectsInferAndCast();
@@ -783,6 +784,35 @@ Status NumPyConverter::ConvertDecimals() {
   return PushBuilderResult(&builder);
 }
 
+Status NumPyConverter::ConvertDateTimes() {
+  // Convert array of datetime.datetime objects to Arrow
+  PyAcquireGIL lock;
+  PyDateTime_IMPORT;
+
+  Ndarray1DIndexer<PyObject*> objects(arr_);
+
+  // datetime.datetime stores microsecond resolution
+  TimestampBuilder builder(::arrow::timestamp(TimeUnit::MICRO), pool_);
+  RETURN_NOT_OK(builder.Resize(length_));
+
+  PyObject* obj = NULLPTR;
+  for (int64_t i = 0; i < length_; ++i) {
+    obj = objects[i];
+    if (PyDateTime_Check(obj)) {
+      RETURN_NOT_OK(
+          builder.Append(PyDateTime_to_us(reinterpret_cast<PyDateTime_DateTime*>(obj))));
+    } else if (PandasObjectIsNull(obj)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else {
+      std::stringstream ss;
+      ss << "Error converting from Python objects to Timestamp: ";
+      RETURN_NOT_OK(InvalidConversion(obj, "datetime.datetime", &ss));
+      return Status::Invalid(ss.str());
+    }
+  }
+  return PushBuilderResult(&builder);
+}
+
 Status NumPyConverter::ConvertTimes() {
   // Convert array of datetime.time objects to Arrow
   PyAcquireGIL lock;
@@ -1005,6 +1035,8 @@ Status NumPyConverter::ConvertObjectsInfer() {
     } else if (PyDate_CheckExact(obj)) {
       // We could choose Date32 or Date64
       return ConvertDates<Date32Type>();
+    } else if (PyDateTime_CheckExact(obj)) {
+      return ConvertDateTimes();
     } else if (PyTime_Check(obj)) {
       return ConvertTimes();
     } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 53e37f5399900..edb605158c6d9 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -18,7 +18,7 @@
 
 from collections import OrderedDict
 
-from datetime import date, time
+from datetime import date, datetime, time, timedelta
 import decimal
 import json
 
@@ -649,6 +649,22 @@ def test_timestamps_with_timezone(self):
 
         _check_pandas_roundtrip(df)
 
+    def test_python_datetime(self):
+        # ARROW-2106
+        date_array = [datetime.today() + timedelta(days=x) for x in range(10)]
+        df = pd.DataFrame({
+            'datetime': pd.Series(date_array, dtype=object)
+        })
+
+        table = pa.Table.from_pandas(df)
+        assert isinstance(table[0].data.chunk(0), pa.TimestampArray)
+
+        result = table.to_pandas()
+        expected_df = pd.DataFrame({
+            'datetime': date_array
+        })
+        tm.assert_frame_equal(expected_df, result)
+
     def test_datetime64_to_date32(self):
         # ARROW-1718
         arr = pa.array([date(2017, 10, 23), None])

From f7d3c64426272ff42271c8aef71f4cce4418e710 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 12 Feb 2018 09:52:48 -0500
Subject: [PATCH 1467/1644] ARROW-2134: [CI] Make Travis-CI commit inspection
 more robust

Author: Antoine Pitrou <antoine@python.org>

Closes #1590 from pitrou/ci_more_robust_commit_inspection and squashes the following commits:

9ab0d7e7 [Antoine Pitrou] ARROW-2134: [CI] [skip appveyor] Make Travis-CI commit inspection more robust
---
 ci/travis_detect_changes.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/ci/travis_detect_changes.py b/ci/travis_detect_changes.py
index 28426190385ff..2aeb34fa00af5 100644
--- a/ci/travis_detect_changes.py
+++ b/ci/travis_detect_changes.py
@@ -70,6 +70,13 @@ def get_travis_commit_range():
     return cr.replace('...', '..')
 
 
+def get_travis_commit_description():
+    # Prefer this to get_commit_description(get_travis_head_commit()),
+    # as rebasing or other repository events may make TRAVIS_COMMIT invalid
+    # at the time we inspect it
+    return os.environ['TRAVIS_COMMIT_MESSAGE']
+
+
 def list_travis_affected_files():
     """
     Return a list of files affected in the current Travis build.
@@ -140,7 +147,7 @@ def get_unix_shell_eval(env):
 
 
 def run_from_travis():
-    desc = get_commit_description(get_travis_head_commit())
+    desc = get_travis_commit_description()
     if '[skip travis]' in desc:
         # Skip everything
         affected = dict.fromkeys(ALL_TOPICS, False)

From 90d9daf1419fdff5fd295053dcad756d1cdb3931 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 12 Feb 2018 09:55:09 -0500
Subject: [PATCH 1468/1644] ARROW-2129: [Python] Handle conversion of empty
 tables to Pandas

Opened https://issues.apache.org/jira/browse/ARROW-2133 to handle the nested case.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1588 from xhochy/ARROW-2129 and squashes the following commits:

78b62d3f [Uwe L. Korn] Use Pandas' testing facilities
8287d034 [Uwe L. Korn] ARROW-2129: [Python] Handle conversion of empty tables to Pandas
---
 cpp/src/arrow/python/arrow_to_pandas.cc     | 11 ++++++++---
 python/pyarrow/tests/test_convert_pandas.py |  6 ++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 60a2eae5d43f9..048898936d710 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -280,6 +280,9 @@ class PandasBlock {
 
 template <typename T>
 inline const T* GetPrimitiveValues(const Array& arr) {
+  if (arr.length() == 0) {
+    return nullptr;
+  }
   const auto& prim_arr = static_cast<const PrimitiveArray&>(arr);
   const T* raw_values = reinterpret_cast<const T*>(prim_arr.values()->data());
   return raw_values + arr.offset();
@@ -304,9 +307,11 @@ inline void ConvertIntegerNoNullsSameType(PandasOptions options, const ChunkedAr
                                           T* out_values) {
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = *data.chunk(c);
-    const T* in_values = GetPrimitiveValues<T>(arr);
-    memcpy(out_values, in_values, sizeof(T) * arr.length());
-    out_values += arr.length();
+    if (arr.length() > 0) {
+      const T* in_values = GetPrimitiveValues<T>(arr);
+      memcpy(out_values, in_values, sizeof(T) * arr.length());
+      out_values += arr.length();
+    }
   }
 }
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index edb605158c6d9..b825dfc6686f5 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1346,6 +1346,12 @@ def test_table_batch_empty_dataframe(self):
         _check_pandas_roundtrip(df2, preserve_index=True)
         _check_pandas_roundtrip(df2, as_batch=True, preserve_index=True)
 
+    def test_convert_empty_table(self):
+        arr = pa.array([], type=pa.int64())
+        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=np.int64))
+        arr = pa.array([], type=pa.string())
+        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
+
     def test_array_from_pandas_date_with_mask(self):
         m = np.array([True, False, True])
         data = pd.Series([

From 08d664a3dd3b8eca9cd443215d917107736eb5e8 Mon Sep 17 00:00:00 2001
From: kisankansagra <kisankansagra@users.noreply.github.com>
Date: Mon, 12 Feb 2018 11:57:17 -0500
Subject: [PATCH 1469/1644] ARROW-2137: [Python] Don't print paths that are
 ignored when reading Parquet files

_should_silently_exclude should silently exclude, not print stuff

Author: kisankansagra <kisankansagra@users.noreply.github.com>

Closes #1592 from kisankansagra/patch-1 and squashes the following commits:

7e80895b [kisankansagra] Update parquet.py
---
 python/pyarrow/parquet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 494e65ebb9593..485459e0b24ca 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -594,7 +594,7 @@ def _visit_level(self, level, base_path, part_keys):
             elif path.endswith('_metadata'):
                 self.metadata_path = full_path
             elif self._should_silently_exclude(path):
-                print('Ignoring path: {0}'.format(full_path))
+                continue
             else:
                 filtered_files.append(full_path)
 

From 31a1b9b8d1529839d0b256eecfa6ee9f47f75b03 Mon Sep 17 00:00:00 2001
From: Johan <j.w.peltenburg@tudelft.nl>
Date: Mon, 12 Feb 2018 11:18:19 -0600
Subject: [PATCH 1470/1644] [Website] Add FPGA acceleration framework Fletcher
 to "Powered By" (#1593)

---
 site/powered_by.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/site/powered_by.md b/site/powered_by.md
index 4033202679817..20feea46ce45d 100644
--- a/site/powered_by.md
+++ b/site/powered_by.md
@@ -65,6 +65,11 @@ short description of your use case. See the following for some examples.
   Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow
   buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI,
   Python, R, and more (all backed by Apache Arrow).
+* **[Fletcher][20]:** Fletcher is an FPGA acceleration framework that can
+  convert an Arrow schema into an easy-to-use hardware interface. The
+  accelerator can request data from Arrow tables by supplying row indices.
+  In turn, the interface provides streams of data of the types defined
+  through the schema. Furthermore, Arrow alleviates serialization bottlenecks.
 * **[GeoMesa][8]:** A suite of tools that enables large-scale geospatial query
   and analytics on distributed computing systems. GeoMesa supports query
   results in the Arrow IPC format, which can then be used for in-browser
@@ -124,3 +129,4 @@ short description of your use case. See the following for some examples.
 [17]: https://github.com/red-data-tools/red-arrow/
 [18]: https://www.graphistry.com
 [19]: http://gpuopenanalytics.com
+[20]: https://github.com/johanpel/fletcher

From 296e7b054472be98ce539cdfee0a0981c43abadd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 12 Feb 2018 14:37:19 -0500
Subject: [PATCH 1471/1644] ARROW-1912: [Website] Add committer affiliations
 and roles to website

cc @jacques-n

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1596 from wesm/ARROW-1912 and squashes the following commits:

ed30ecd1 [Wes McKinney] Add LLC designation for Uwe
2fb8d892 [Wes McKinney] Add committer affiliations to website
---
 site/committers.html | 58 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/site/committers.html b/site/committers.html
index 990260ef70d3e..a123de74d0325 100644
--- a/site/committers.html
+++ b/site/committers.html
@@ -6,120 +6,178 @@ <h2>Committers</h2>
 <table class="table"><thead>
 <tr>
 <th>Name</th>
+<th>Role</th>
 <th>Alias (email is &lt;alias&gt;@apache.org)</th>
+<th>Affiliation</th>
 </tr>
 </thead><tbody>
 <tr>
 <td>Jacques Nadeau</td>
+<td>VP, PMC</td>
 <td>jacques</td>
+<td>Dremio</td>
 </tr>
 <tr>
 <td>Todd Lipcon</td>
+<td>PMC</td>
 <td>todd</td>
+<td>Cloudera</td>
 </tr>
 <tr>
 <td>Ted Dunning</td>
+<td>PMC</td>
 <td>tdunning</td>
+<td>MapR</td>
 </tr>
 <tr>
 <td>Michael Stack</td>
+<td>PMC</td>
 <td>stack</td>
+<td>Cloudera</td>
 </tr>
 <tr>
 <td>P. Taylor Goetz</td>
+<td>PMC</td>
 <td>ptgoetz</td>
+<td>Hortonworks</td>
 </tr>
 <tr>
 <td>Julian Hyde</td>
+<td>PMC</td>
 <td>jhyde</td>
+<td>Hortonworks</td>
 </tr>
 <tr>
 <td>Reynold Xin</td>
+<td>PMC</td>
 <td>rxin</td>
+<td>Databricks</td>
 </tr>
 <tr>
 <td>James Taylor</td>
+<td>PMC</td>
 <td>jamestaylor</td>
+<td>Salesforce</td>
 </tr>
 <tr>
 <td>Julien Le Dem</td>
+<td>PMC</td>
 <td>julien</td>
+<td>WeWork</td>
 </tr>
 <tr>
 <td>Jake Luciani</td>
+<td>PMC</td>
 <td>jake</td>
+<td>DataStax</td>
 </tr>
 <tr>
 <td>Jason Altekruse</td>
+<td>PMC</td>
 <td>json</td>
+<td>Workday</td>
 </tr>
 <tr>
 <td>Alex Levenson</td>
+<td>PMC</td>
 <td>alexlevenson</td>
+<td>Twitter</td>
 </tr>
 <tr>
 <td>Parth Chandra</td>
+<td>PMC</td>
 <td>parthc</td>
+<td>MapR</td>
 </tr>
 <tr>
 <td>Marcel Kornacker</td>
+<td>PMC</td>
 <td>marcel</td>
+<td>Independent</td>
 </tr>
 <tr>
 <td>Steven Phillips</td>
+<td>PMC</td>
 <td>smp</td>
+<td>Dremio</td>
 </tr>
 <tr>
 <td>Hanifi Gunes</td>
+<td>PMC</td>
 <td>hg</td>
+<td></td>
 </tr>
 <tr>
 <td>Abdelhakim Deneche</td>
+<td>PMC</td>
 <td>adeneche</td>
+<td>Dremio</td>
 </tr>
 <tr>
 <td>Wes McKinney</td>
+<td>PMC</td>
 <td>wesm</td>
+<td>Two Sigma</td>
 </tr>
 <tr>
 <td>David Alves</td>
+<td>PMC</td>
 <td>dralves</td>
+<td>Cloudera</td>
 </tr>
 <tr>
 <td>Ippokratis Pandis</td>
+<td>PMC</td>
 <td>ippokratis</td>
+<td>Amazon</td>
 </tr>
 <tr>
 <td>Uwe L. Korn</td>
+<td>PMC</td>
 <td>uwe</td>
+<td>Blue Yonder GmbH</td>
 </tr>
 <tr>
 <td>Kouhei Sutou</td>
+<td>PMC</td>
 <td>kou</td>
+<td>ClearCode</td>
 </tr>
 <tr>
 <td>Philipp Moritz</td>
+<td>Committer</td>
 <td>pcmoritz</td>
+<td>UC Berkeley RISELab</td>
 </tr>
 <tr>
 <td>Phillip Cloud</td>
+<td>Committer</td>
 <td>cpcloud</td>
+<td>Two Sigma</td>
 </tr>
 <tr>
 <td>Bryan Cutler</td>
+<td>Committer</td>
 <td>cutlerb</td>
+<td>IBM</td>
 </tr>
 <tr>
 <td>Li Jin</td>
+<td>Committer</td>
 <td>icexelloss</td>
+<td>Two Sigma</td>
 </tr>
 <tr>
 <td>Siddharth Teotia</td>
+<td>Committer</td>
 <td>siddteotia</td>
+<td>Dremio</td>
 </tr>
 <tr>
 <td>Paul Taylor</td>
+<td>Committer</td>
 <td>ptaylor</td>
+<td>Graphistry</td>
 </tr>
 </tbody></table>
 

From 62c55ecdb237c73a96553252937e92180dffd4eb Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 12 Feb 2018 14:45:34 -0500
Subject: [PATCH 1472/1644] ARROW-2133: [Python] Fix segfault on conversion of
 empty nested array to Pandas

Based on PR #1588

Author: Antoine Pitrou <antoine@python.org>

Closes #1591 from pitrou/ARROW-2133-segfault-nested-array-to-pandas and squashes the following commits:

3ac6e522 [Antoine Pitrou] ARROW-2133: [Python] Fix segfault on conversion of empty nested arrays to Pandas
---
 cpp/src/arrow/python/arrow_to_pandas.cc     | 2 +-
 python/pyarrow/tests/test_convert_pandas.py | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 048898936d710..125892afe59b2 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -519,7 +519,7 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr<Colu
   // TODO(ARROW-489): Currently we don't have a Python reference for single columns.
   //    Storing a reference to the whole Array would be to expensive.
 
-  OwnedRef owned_numpy_array;
+  OwnedRefNoGIL owned_numpy_array;
   RETURN_NOT_OK(
       ConvertColumnToPandas(options, flat_column, nullptr, owned_numpy_array.ref()));
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index b825dfc6686f5..026cd25076ed6 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1351,6 +1351,10 @@ def test_convert_empty_table(self):
         tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=np.int64))
         arr = pa.array([], type=pa.string())
         tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
+        arr = pa.array([], type=pa.list_(pa.int64()))
+        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
+        arr = pa.array([], type=pa.struct([pa.field('a', pa.int64())]))
+        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
 
     def test_array_from_pandas_date_with_mask(self):
         m = np.array([True, False, True])

From e8a93ca1125659e0eaff353d92d351d25f104a4b Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 12 Feb 2018 15:25:18 -0500
Subject: [PATCH 1473/1644] ARROW-1021: [Python] Add documentation for C++
 pyarrow API

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: Antoine Pitrou <antoine@python.org>

Closes #1576 from pitrou/ARROW-1021-document-pyarrow-cpp-api and squashes the following commits:

61f95852 [Wes McKinney] Include get_libraries, get_library_dirs in docs, enable test to pass on Linux in local dev setup
3fced81d [Wes McKinney] Add get_libraries, get_library_dirs functions, get Cython test case passing on Windows
d82ead24 [Antoine Pitrou] ARROW-1021: [Python] Add documentation for C++ pyarrow API

Change-Id: I0160fb23569d1e2ba549b1b75486bc41001eebbc
---
 .gitignore                                    |   1 +
 ci/msvc-build.bat                             |  12 +-
 python/doc/Makefile                           |   2 +-
 python/doc/source/api.rst                     |  12 +
 python/doc/source/extending.rst               | 327 +++++++++++++++++-
 python/pyarrow/__init__.py                    |  30 ++
 python/pyarrow/lib.pxd                        |  27 +-
 .../pyarrow/tests/pyarrow_cython_example.pyx  |  29 ++
 python/pyarrow/tests/test_cython.py           | 100 ++++++
 9 files changed, 522 insertions(+), 18 deletions(-)
 create mode 100644 python/pyarrow/tests/pyarrow_cython_example.pyx
 create mode 100644 python/pyarrow/tests/test_cython.py

diff --git a/.gitignore b/.gitignore
index c38694e1f6647..f5b235a8995e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@
 *.o
 *.py[ocd]
 *.so
+*.so.*
 *.dylib
 .build_cache_dir
 MANIFEST
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 58dfc2a146572..069ddf0fc85c4 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -137,14 +137,20 @@ cmake -G "%GENERATOR%" ^
 cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
 popd
 
-@rem Build and import pyarrow
+@rem Build and install pyarrow
 @rem parquet-cpp has some additional runtime dependencies that we need to figure out
 @rem see PARQUET-1018
 
 pushd python
 
 set PYARROW_CXXFLAGS=/WX
-python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel  || exit /B
-py.test pyarrow -r sxX --durations=15 -v -s --parquet || exit /B
+python setup.py build_ext --with-parquet --bundle-arrow-cpp ^
+    install -q --single-version-externally-managed --record=record.text ^
+    bdist_wheel || exit /B
+
+@rem Test directly from installed location
+
+SET PYARROW_PATH=%CONDA_PREFIX%\Lib\site-packages\pyarrow
+py.test -r sxX --durations=15 -v %PYARROW_PATH% --parquet || exit /B
 
 popd
diff --git a/python/doc/Makefile b/python/doc/Makefile
index 1b9f707021a1a..eacb124c3fe0d 100644
--- a/python/doc/Makefile
+++ b/python/doc/Makefile
@@ -20,7 +20,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    = -j4
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 2d3e39c6954a1..a71e92b0b634b 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -342,3 +342,15 @@ Apache Parquet
    read_schema
    write_metadata
    write_table
+
+.. currentmodule:: pyarrow
+
+Using with C extensions
+-----------------------
+
+.. autosummary::
+   :toctree: generated/
+
+   get_include
+   get_libraries
+   get_library_dirs
diff --git a/python/doc/source/extending.rst b/python/doc/source/extending.rst
index 2d6caed268ef2..4a358984fac1c 100644
--- a/python/doc/source/extending.rst
+++ b/python/doc/source/extending.rst
@@ -15,19 +15,332 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
-.. currentmodule:: pyarrow
+.. currentmodule:: pyarrow.lib
 .. _extending:
 
-Building C++ and Cython Extensions using pyarrow
-================================================
+Using pyarrow from C++ and Cython Code
+======================================
 
-pyarrow features both a Cython and C++ API. We intend to fully document the
-details of how to do this.
+pyarrow features both a Cython and C++ API.
 
-The Arrow C++ header files are bundled with a pyarrow installation. To get the
-absolute path to this directory (like ``numpy.get_include()``), use:
+C++ API
+-------
+
+.. default-domain:: cpp
+
+The Arrow C++ header files are bundled with a pyarrow installation.
+To get the absolute path to this directory (like ``numpy.get_include()``), use:
 
 .. code-block:: python
 
    import pyarrow as pa
    pa.get_include()
+
+Assuming the path above is on your compiler's include path, the pyarrow API
+can be included using the following directive:
+
+.. code-block:: cpp
+
+   #include <arrow/python/pyarrow.h>
+
+This will not include other parts of the Arrow API, which you will need
+to include yourself (for example ``arrow/api.h``).
+
+When building C extensions that use the Arrow C++ libraries, you must add
+appropriate linker flags. We have provided functions ``pyarrow.get_libraries``
+and ``pyarrow.get_library_dirs`` which return a list of library names and
+likely library install locations (if you installed pyarrow with pip or
+conda). These must be included when declaring your C extensions with distutils
+(see below).
+
+Initializing the API
+~~~~~~~~~~~~~~~~~~~~
+
+.. function:: int import_pyarrow()
+
+   Initialize inner pointers of the pyarrow API.  On success, 0 is
+   returned.  Otherwise, -1 is returned and a Python exception is set.
+
+   It is mandatory to call this function before calling any other function
+   in the pyarrow C++ API.  Failing to do so will likely lead to crashes.
+
+Wrapping and Unwrapping
+~~~~~~~~~~~~~~~~~~~~~~~
+
+pyarrow provides the following functions to go back and forth between
+Python wrappers (as exposed by the pyarrow Python API) and the underlying
+C++ objects.
+
+.. function:: bool is_array(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`Array` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.Array` instance.
+
+.. function:: bool is_buffer(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`Buffer` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.Buffer` instance.
+
+.. function:: bool is_column(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`Column` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.Column` instance.
+
+.. function:: bool is_data_type(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`DataType` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.DataType` instance.
+
+.. function:: bool is_field(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`Field` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.Field` instance.
+
+.. function:: bool is_record_batch(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`RecordBatch` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.RecordBatch` instance.
+
+.. function:: bool is_schema(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`Schema` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.Schema` instance.
+
+.. function:: bool is_table(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`Table` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.Table` instance.
+
+.. function:: bool is_tensor(PyObject* obj)
+
+   Return whether *obj* wraps an Arrow C++ :class:`Tensor` pointer;
+   in other words, whether *obj* is a :py:class:`pyarrow.Tensor` instance.
+
+The following functions expect a pyarrow object, unwrap the underlying
+Arrow C++ API pointer, and put it in the *out* parameter.  The returned
+:class:`Status` object must be inspected first to know whether any error
+occurred.  If successful, *out* is guaranteed to be non-NULL.
+
+.. function:: Status unwrap_array(PyObject* obj, std::shared_ptr<Array>* out)
+
+   Unwrap the Arrow C++ :class:`Array` pointer from *obj* and put it in *out*.
+
+.. function:: Status unwrap_buffer(PyObject* obj, std::shared_ptr<Buffer>* out)
+
+   Unwrap the Arrow C++ :class:`Buffer` pointer from *obj* and put it in *out*.
+
+.. function:: Status unwrap_column(PyObject* obj, std::shared_ptr<Column>* out)
+
+   Unwrap the Arrow C++ :class:`Column` pointer from *obj* and put it in *out*.
+
+.. function:: Status unwrap_data_type(PyObject* obj, std::shared_ptr<DataType>* out)
+
+   Unwrap the Arrow C++ :class:`DataType` pointer from *obj* and put it in *out*.
+
+.. function:: Status unwrap_field(PyObject* obj, std::shared_ptr<Field>* out)
+
+   Unwrap the Arrow C++ :class:`Field` pointer from *obj* and put it in *out*.
+
+.. function:: Status unwrap_record_batch(PyObject* obj, std::shared_ptr<RecordBatch>* out)
+
+   Unwrap the Arrow C++ :class:`RecordBatch` pointer from *obj* and put it in *out*.
+
+.. function:: Status unwrap_schema(PyObject* obj, std::shared_ptr<Schema>* out)
+
+   Unwrap the Arrow C++ :class:`Schema` pointer from *obj* and put it in *out*.
+
+.. function:: Status unwrap_table(PyObject* obj, std::shared_ptr<Table>* out)
+
+   Unwrap the Arrow C++ :class:`Table` pointer from *obj* and put it in *out*.
+
+.. function:: Status unwrap_tensor(PyObject* obj, std::shared_ptr<Tensor>* out)
+
+   Unwrap the Arrow C++ :class:`Tensor` pointer from *obj* and put it in *out*.
+
+The following functions take an Arrow C++ API pointer and wrap it in a
+pyarray object of the corresponding type.  A new reference is returned.
+On error, NULL is returned and a Python exception is set.
+
+.. function:: PyObject* wrap_array(const std::shared_ptr<Array>& array)
+
+   Wrap the Arrow C++ *array* in a :py:class:`pyarrow.Array` instance.
+
+.. function:: PyObject* wrap_buffer(const std::shared_ptr<Buffer>& buffer)
+
+   Wrap the Arrow C++ *buffer* in a :py:class:`pyarrow.Buffer` instance.
+
+.. function:: PyObject* wrap_column(const std::shared_ptr<Column>& column)
+
+   Wrap the Arrow C++ *column* in a :py:class:`pyarrow.Column` instance.
+
+.. function:: PyObject* wrap_data_type(const std::shared_ptr<DataType>& data_type)
+
+   Wrap the Arrow C++ *data_type* in a :py:class:`pyarrow.DataType` instance.
+
+.. function:: PyObject* wrap_field(const std::shared_ptr<Field>& field)
+
+   Wrap the Arrow C++ *field* in a :py:class:`pyarrow.Field` instance.
+
+.. function:: PyObject* wrap_record_batch(const std::shared_ptr<RecordBatch>& batch)
+
+   Wrap the Arrow C++ record *batch* in a :py:class:`pyarrow.RecordBatch` instance.
+
+.. function:: PyObject* wrap_schema(const std::shared_ptr<Schema>& schema)
+
+   Wrap the Arrow C++ *schema* in a :py:class:`pyarrow.Schema` instance.
+
+.. function:: PyObject* wrap_table(const std::shared_ptr<Table>& table)
+
+   Wrap the Arrow C++ *table* in a :py:class:`pyarrow.Table` instance.
+
+.. function:: PyObject* wrap_tensor(const std::shared_ptr<Tensor>& tensor)
+
+   Wrap the Arrow C++ *tensor* in a :py:class:`pyarrow.Tensor` instance.
+
+
+Cython API
+----------
+
+.. default-domain:: py
+
+The Cython API more or less mirrors the C++ API, but the calling convention
+can be different as required by Cython.  In Cython, you don't need to
+initialize the API as that will be handled automaticalled by the ``cimport``
+directive.
+
+.. note::
+   Classes from the Arrow C++ API are renamed when exposed in Cython, to
+   avoid named clashes with the corresponding Python classes.  For example,
+   C++ Arrow arrays have the ``CArray`` type and ``Array`` is the
+   corresponding Python wrapper class.
+
+Wrapping and Unwrapping
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The following functions expect a pyarrow object, unwrap the underlying
+Arrow C++ API pointer, and return it.  NULL is returned (without setting
+an exception) if the input is not of the right type.
+
+.. function:: pyarrow_unwrap_array(obj) -> shared_ptr[CArray]
+
+   Unwrap the Arrow C++ :cpp:class:`Array` pointer from *obj*.
+
+.. function:: pyarrow_unwrap_batch(obj) -> shared_ptr[CRecordBatch]
+
+   Unwrap the Arrow C++ :cpp:class:`RecordBatch` pointer from *obj*.
+
+.. function:: pyarrow_unwrap_buffer(obj) -> shared_ptr[CBuffer]
+
+   Unwrap the Arrow C++ :cpp:class:`Buffer` pointer from *obj*.
+
+.. function:: pyarrow_unwrap_column(obj) -> shared_ptr[CColumn]
+
+   Unwrap the Arrow C++ :cpp:class:`Column` pointer from *obj*.
+
+.. function:: pyarrow_unwrap_data_type(obj) -> shared_ptr[CDataType]
+
+   Unwrap the Arrow C++ :cpp:class:`CDataType` pointer from *obj*.
+
+.. function:: pyarrow_unwrap_field(obj) -> shared_ptr[CField]
+
+   Unwrap the Arrow C++ :cpp:class:`Field` pointer from *obj*.
+
+.. function:: pyarrow_unwrap_schema(obj) -> shared_ptr[CSchema]
+
+   Unwrap the Arrow C++ :cpp:class:`Schema` pointer from *obj*.
+
+.. function:: pyarrow_unwrap_table(obj) -> shared_ptr[CTable]
+
+   Unwrap the Arrow C++ :cpp:class:`Table` pointer from *obj*.
+
+.. function:: pyarrow_unwrap_tensor(obj) -> shared_ptr[CTensor]
+
+   Unwrap the Arrow C++ :cpp:class:`Tensor` pointer from *obj*.
+
+The following functions take a Arrow C++ API pointer and wrap it in a
+pyarray object of the corresponding type.  An exception is raised on error.
+
+.. function:: pyarrow_wrap_array(sp_array: const shared_ptr[CArray]& array) -> object
+
+   Wrap the Arrow C++ *array* in a Python :class:`pyarrow.Array` instance.
+
+.. function:: pyarrow_wrap_batch(sp_array: const shared_ptr[CRecordBatch]& batch) -> object
+
+   Wrap the Arrow C++ record *batch* in a Python :class:`pyarrow.RecordBatch` instance.
+
+.. function:: pyarrow_wrap_buffer(sp_array: const shared_ptr[CBuffer]& buffer) -> object
+
+   Wrap the Arrow C++ *buffer* in a Python :class:`pyarrow.Buffer` instance.
+
+.. function:: pyarrow_wrap_column(sp_array: const shared_ptr[CColumn]& column) -> object
+
+   Wrap the Arrow C++ *column* in a Python :class:`pyarrow.Column` instance.
+
+.. function:: pyarrow_wrap_data_type(sp_array: const shared_ptr[CDataType]& data_type) -> object
+
+   Wrap the Arrow C++ *data_type* in a Python :class:`pyarrow.DataType` instance.
+
+.. function:: pyarrow_wrap_field(sp_array: const shared_ptr[CField]& field) -> object
+
+   Wrap the Arrow C++ *field* in a Python :class:`pyarrow.Field` instance.
+
+.. function:: pyarrow_wrap_resizable_buffer(sp_array: const shared_ptr[CResizableBuffer]& buffer) -> object
+
+   Wrap the Arrow C++ resizable *buffer* in a Python :class:`pyarrow.ResizableBuffer` instance.
+
+.. function:: pyarrow_wrap_schema(sp_array: const shared_ptr[CSchema]& schema) -> object
+
+   Wrap the Arrow C++ *schema* in a Python :class:`pyarrow.Schema` instance.
+
+.. function:: pyarrow_wrap_table(sp_array: const shared_ptr[CTable]& table) -> object
+
+   Wrap the Arrow C++ *table* in a Python :class:`pyarrow.Table` instance.
+
+.. function:: pyarrow_wrap_tensor(sp_array: const shared_ptr[CTensor]& tensor) -> object
+
+   Wrap the Arrow C++ *tensor* in a Python :class:`pyarrow.Tensor` instance.
+
+Example
+~~~~~~~
+
+The following Cython module shows how to unwrap a Python object and call
+the underlying C++ object's API.
+
+.. code-block:: python
+
+   # distutils: language=c++
+
+   from pyarrow.lib cimport *
+
+   def get_array_length(obj):
+       # Just an example function accessing both the pyarrow Cython API
+       # and the Arrow C++ API
+       cdef shared_ptr[CArray] arr = pyarrow_unwrap_array(obj)
+       if arr.get() == NULL:
+           raise TypeError("not an array")
+       return arr.get().length()
+
+To build this module, you will need a slightly customized ``setup.py`` file
+(this is assuming the file above is named ``example.pyx``):
+
+.. code-block:: python
+
+    from distutils.core import setup
+    from Cython.Build import cythonize
+
+    import numpy as np
+
+    import pyarrow as pa
+
+    ext_modules = cythonize("example.pyx")
+
+    for ext in ext_modules:
+        # The Numpy C headers are currently required
+        ext.include_dirs.append(np.get_include())
+        ext.include_dirs.append(pa.get_include())
+        ext.libraries.extend(pa.get_libraries())
+        ext.library_dirs.append(pa.get_library_dirs())
+
+    setup(
+        ext_modules=ext_modules,
+    )
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 8b3cba92414f8..d95954ed30e42 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -165,3 +165,33 @@ def get_include():
     """
     import os
     return os.path.join(os.path.dirname(__file__), 'include')
+
+
+def get_libraries():
+    """
+    Return list of library names to include in the `libraries` argument for C
+    or Cython extensions using pyarrow
+    """
+    return ['arrow_python']
+
+
+def get_library_dirs():
+    """
+    Return lists of directories likely to contain Arrow C++ libraries for
+    linking C or Cython extensions using pyarrow
+    """
+    import os
+    import sys
+    package_cwd = os.path.dirname(__file__)
+
+    library_dirs = [package_cwd]
+
+    if sys.platform == 'win32':
+        # TODO(wesm): Is this necessary, or does setuptools within a conda
+        # installation add Library\lib to the linker path for MSVC?
+        site_packages, _ = os.path.split(package_cwd)
+        python_base_install, _ = os.path.split(site_packages)
+        library_dirs.append(os.path.join(python_base_install,
+                                         'Library', 'lib'))
+
+    return library_dirs
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 161562c040c30..b1433ecde270a 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -347,16 +347,29 @@ cdef class NativeFile:
 cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader)
 cdef get_writer(object source, shared_ptr[OutputStream]* writer)
 
+cdef dict box_metadata(const CKeyValueMetadata* sp_metadata)
+
+# Public Cython API for 3rd party code
+
+cdef public object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array)
+# XXX pyarrow.h calls it `wrap_record_batch`
+cdef public object pyarrow_wrap_batch(const shared_ptr[CRecordBatch]& cbatch)
 cdef public object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf)
-cdef public object pyarrow_wrap_resizable_buffer(
-    const shared_ptr[CResizableBuffer]& buf)
+cdef public object pyarrow_wrap_column(const shared_ptr[CColumn]& ccolumn)
 cdef public object pyarrow_wrap_data_type(const shared_ptr[CDataType]& type)
 cdef public object pyarrow_wrap_field(const shared_ptr[CField]& field)
+cdef public object pyarrow_wrap_resizable_buffer(
+    const shared_ptr[CResizableBuffer]& buf)
 cdef public object pyarrow_wrap_schema(const shared_ptr[CSchema]& type)
-cdef public object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array)
-cdef public object pyarrow_wrap_tensor(const shared_ptr[CTensor]& sp_tensor)
-cdef public object pyarrow_wrap_column(const shared_ptr[CColumn]& ccolumn)
 cdef public object pyarrow_wrap_table(const shared_ptr[CTable]& ctable)
-cdef public object pyarrow_wrap_batch(const shared_ptr[CRecordBatch]& cbatch)
+cdef public object pyarrow_wrap_tensor(const shared_ptr[CTensor]& sp_tensor)
 
-cdef dict box_metadata(const CKeyValueMetadata* sp_metadata)
+cdef public shared_ptr[CArray] pyarrow_unwrap_array(object array)
+cdef public shared_ptr[CRecordBatch] pyarrow_unwrap_batch(object batch)
+cdef public shared_ptr[CBuffer] pyarrow_unwrap_buffer(object buffer)
+cdef public shared_ptr[CColumn] pyarrow_unwrap_column(object column)
+cdef public shared_ptr[CDataType] pyarrow_unwrap_data_type(object data_type)
+cdef public shared_ptr[CField] pyarrow_unwrap_field(object field)
+cdef public shared_ptr[CSchema] pyarrow_unwrap_schema(object schema)
+cdef public shared_ptr[CTable] pyarrow_unwrap_table(object table)
+cdef public shared_ptr[CTensor] pyarrow_unwrap_tensor(object tensor)
diff --git a/python/pyarrow/tests/pyarrow_cython_example.pyx b/python/pyarrow/tests/pyarrow_cython_example.pyx
new file mode 100644
index 0000000000000..b5e54063df185
--- /dev/null
+++ b/python/pyarrow/tests/pyarrow_cython_example.pyx
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language=c++
+
+from pyarrow.lib cimport *
+
+
+def get_array_length(obj):
+    # Just an example function accessing both the pyarrow Cython API
+    # and the Arrow C++ API
+    cdef shared_ptr[CArray] arr = pyarrow_unwrap_array(obj)
+    if arr.get() == NULL:
+        raise TypeError("not an array")
+    return arr.get().length()
diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py
new file mode 100644
index 0000000000000..51b1a48ded09e
--- /dev/null
+++ b/python/pyarrow/tests/test_cython.py
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+import pyarrow as pa
+
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+setup_template = """if 1:
+    from distutils.core import setup
+    from Cython.Build import cythonize
+
+    import numpy as np
+
+    import pyarrow as pa
+
+    ext_modules = cythonize({pyx_file!r})
+    compiler_opts = {compiler_opts!r}
+    custom_ld_path = {test_ld_path!r}
+
+    for ext in ext_modules:
+        # XXX required for numpy/numpyconfig.h,
+        # included from arrow/python/api.h
+        ext.include_dirs.append(np.get_include())
+        ext.include_dirs.append(pa.get_include())
+        ext.libraries.extend(pa.get_libraries())
+        ext.library_dirs.extend(pa.get_library_dirs())
+        if custom_ld_path:
+            ext.library_dirs.append(custom_ld_path)
+        ext.extra_compile_args.extend(compiler_opts)
+
+    setup(
+        ext_modules=ext_modules,
+    )
+"""
+
+
+def test_cython_api(tmpdir):
+    """
+    Basic test for the Cython API.
+    """
+    pytest.importorskip('Cython')
+
+    if 'ARROW_HOME' in os.environ:
+        ld_path_default = os.path.join(os.environ['ARROW_HOME'], 'lib')
+
+    test_ld_path = os.environ.get('PYARROW_TEST_LD_PATH', ld_path_default)
+
+    with tmpdir.as_cwd():
+        # Set up temporary workspace
+        pyx_file = 'pyarrow_cython_example.pyx'
+        shutil.copyfile(os.path.join(here, pyx_file),
+                        os.path.join(str(tmpdir), pyx_file))
+        # Create setup.py file
+        if os.name == 'posix':
+            compiler_opts = ['-std=c++11']
+        else:
+            compiler_opts = []
+        setup_code = setup_template.format(pyx_file=pyx_file,
+                                           compiler_opts=compiler_opts,
+                                           test_ld_path=test_ld_path)
+        with open('setup.py', 'w') as f:
+            f.write(setup_code)
+
+        # Compile extension module
+        subprocess.check_call([sys.executable, 'setup.py',
+                               'build_ext', '--inplace'])
+
+        # Check basic functionality
+        orig_path = sys.path[:]
+        sys.path.insert(0, str(tmpdir))
+        try:
+            mod = __import__('pyarrow_cython_example')
+            arr = pa.array([1, 2, 3])
+            assert mod.get_array_length(arr) == 3
+            with pytest.raises(TypeError, match="not an array"):
+                mod.get_array_length(None)
+        finally:
+            sys.path = orig_path

From cf00b57c6197738f79e637a98c105e2c6d9465a3 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 12 Feb 2018 16:09:51 -0500
Subject: [PATCH 1474/1644] ARROW-2138: [C++] abort on failed debug check

Using abort() instead of exit(1) triggers debug tools such as gdb and therefore makes debugging easier.

Author: Antoine Pitrou <antoine@python.org>

Closes #1595 from pitrou/ARROW-2138-abort-on-debug-check and squashes the following commits:

c8a7e347 [Antoine Pitrou] ARROW-2138: [C++] abort on failed debug check
---
 cpp/src/arrow/allocator-test.cc   | 4 ++--
 cpp/src/arrow/memory_pool-test.cc | 4 ++--
 cpp/src/arrow/util/logging.h      | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
index 7c768be92812c..55cc203bac5f7 100644
--- a/cpp/src/arrow/allocator-test.cc
+++ b/cpp/src/arrow/allocator-test.cc
@@ -51,8 +51,8 @@ TEST(stl_allocator, FreeLargeMemory) {
   uint8_t* data = alloc.allocate(100);
 
 #ifndef NDEBUG
-  EXPECT_EXIT(alloc.deallocate(data, 120), ::testing::ExitedWithCode(1),
-              ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
+  EXPECT_DEATH(alloc.deallocate(data, 120),
+               ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
 #endif
 
   alloc.deallocate(data, 100);
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 0a4785d5229be..c5e3ef295a7f6 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -51,8 +51,8 @@ TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
   ASSERT_OK(pool->Allocate(100, &data));
 
 #ifndef NDEBUG
-  EXPECT_EXIT(pool->Free(data, 120), ::testing::ExitedWithCode(1),
-              ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
+  EXPECT_DEATH(pool->Free(data, 120),
+               ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
 #endif
 
   pool->Free(data, 100);
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 39815f303151e..4ca4d22527fc7 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -105,7 +105,7 @@ class CerrLog {
       std::cerr << std::endl;
     }
     if (severity_ == ARROW_FATAL) {
-      std::exit(1);
+      std::abort();
     }
   }
 
@@ -134,7 +134,7 @@ class FatalLog : public CerrLog {
     if (has_logged_) {
       std::cerr << std::endl;
     }
-    std::exit(1);
+    std::abort();
   }
 };
 

From 30ab43f2c0ddafec9b7e572a463681dec9ac239d Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 12 Feb 2018 16:24:52 -0500
Subject: [PATCH 1475/1644] ARROW-2124: [Python] Add test for empty item in
 array

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1586 from xhochy/ARROW-2124 and squashes the following commits:

1f93916a [Uwe L. Korn] ARROW-2124: [Python] Add test for empty item in array
---
 python/pyarrow/tests/test_convert_pandas.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 026cd25076ed6..b73522d17d6f4 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -800,6 +800,14 @@ def test_column_of_lists(self):
             field = schema.field_by_name(column)
             _check_array_roundtrip(df[column], type=field.type)
 
+    def test_column_of_lists_first_empty(self):
+        # ARROW-2124
+        num_lists = [[], [2, 3, 4], [3, 6, 7, 8], [], [2]]
+        series = pd.Series([np.array(s, dtype=float) for s in num_lists])
+        arr = pa.array(series)
+        result = pd.Series(arr.to_pandas())
+        tm.assert_series_equal(result, series)
+
     def test_column_of_lists_chunked(self):
         # ARROW-1357
         df = pd.DataFrame({

From 0ffc8822aaa7f62eda8ba44a44210e90bd0fe23c Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 12 Feb 2018 18:00:13 -0500
Subject: [PATCH 1476/1644] ARROW-1998: [Python] fix crash on empty Numpy
 arrays

This would happen with some object types (binary, string).

Author: Antoine Pitrou <antoine@python.org>

Closes #1594 from pitrou/ARROW-1998-empty-np-array-crash and squashes the following commits:

3e071148 [Antoine Pitrou] ARROW-1998: [Python] fix crash on empty Numpy arrays
---
 cpp/src/arrow/python/numpy_to_arrow.cc      | 39 ++++++++++++++-------
 python/pyarrow/tests/test_convert_pandas.py | 25 +++++++++++++
 2 files changed, 51 insertions(+), 13 deletions(-)

diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index d487d9d9dbc85..3dd5a79cb0403 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -850,16 +850,23 @@ Status NumPyConverter::ConvertObjectStrings() {
   RETURN_NOT_OK(builder.Resize(length_));
 
   bool global_have_bytes = false;
-  int64_t offset = 0;
-  while (offset < length_) {
-    bool chunk_have_bytes = false;
-    RETURN_NOT_OK(
-        AppendObjectStrings(arr_, mask_, offset, &builder, &offset, &chunk_have_bytes));
-
-    global_have_bytes = global_have_bytes | chunk_have_bytes;
+  if (length_ == 0) {
+    // Produce an empty chunk
     std::shared_ptr<Array> chunk;
     RETURN_NOT_OK(builder.Finish(&chunk));
     out_arrays_.emplace_back(std::move(chunk));
+  } else {
+    int64_t offset = 0;
+    while (offset < length_) {
+      bool chunk_have_bytes = false;
+      RETURN_NOT_OK(
+          AppendObjectStrings(arr_, mask_, offset, &builder, &offset, &chunk_have_bytes));
+
+      global_have_bytes = global_have_bytes | chunk_have_bytes;
+      std::shared_ptr<Array> chunk;
+      RETURN_NOT_OK(builder.Finish(&chunk));
+      out_arrays_.emplace_back(std::move(chunk));
+    }
   }
 
   // If we saw PyBytes, convert everything to BinaryArray
@@ -954,14 +961,21 @@ Status NumPyConverter::ConvertObjectFixedWidthBytes(
   FixedSizeBinaryBuilder builder(type, pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
-  int64_t offset = 0;
-  while (offset < length_) {
-    RETURN_NOT_OK(
-        AppendObjectFixedWidthBytes(arr_, mask_, byte_width, offset, &builder, &offset));
-
+  if (length_ == 0) {
+    // Produce an empty chunk
     std::shared_ptr<Array> chunk;
     RETURN_NOT_OK(builder.Finish(&chunk));
     out_arrays_.emplace_back(std::move(chunk));
+  } else {
+    int64_t offset = 0;
+    while (offset < length_) {
+      RETURN_NOT_OK(AppendObjectFixedWidthBytes(arr_, mask_, byte_width, offset, &builder,
+                                                &offset));
+
+      std::shared_ptr<Array> chunk;
+      RETURN_NOT_OK(builder.Finish(&chunk));
+      out_arrays_.emplace_back(std::move(chunk));
+    }
   }
   return Status::OK();
 }
@@ -1567,7 +1581,6 @@ Status NdarrayToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
   if (!PyArray_Check(ao)) {
     return Status::Invalid("Input object was not a NumPy array");
   }
-
   NumPyConverter converter(pool, ao, mo, type, use_pandas_null_sentinels);
   RETURN_NOT_OK(converter.Convert());
   const auto& output_arrays = converter.result();
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index b73522d17d6f4..987ac23ec7117 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -119,6 +119,26 @@ def _check_array_from_pandas_roundtrip(np_array):
 
 class TestPandasConversion(object):
 
+    type_pairs = [
+        (np.int8, pa.int8()),
+        (np.int16, pa.int16()),
+        (np.int32, pa.int32()),
+        (np.int64, pa.int64()),
+        (np.uint8, pa.uint8()),
+        (np.uint16, pa.uint16()),
+        (np.uint32, pa.uint32()),
+        (np.uint64, pa.uint64()),
+        # (np.float16, pa.float16()),  # XXX unsupported
+        (np.float32, pa.float32()),
+        (np.float64, pa.float64()),
+        # XXX unsupported
+        # (np.dtype([('a', 'i2')]), pa.struct([pa.field('a', pa.int16())])),
+        (np.object, pa.string()),
+        # (np.object, pa.binary()),  # XXX unsupported
+        (np.object, pa.binary(10)),
+        (np.object, pa.list_(pa.int64())),
+        ]
+
     def test_all_none_objects(self):
         df = pd.DataFrame({'a': [None, None, None]})
         _check_pandas_roundtrip(df)
@@ -128,6 +148,11 @@ def test_all_none_category(self):
         df['a'] = df['a'].astype('category')
         _check_pandas_roundtrip(df)
 
+    def test_empty_arrays(self):
+        for dtype, pa_type in self.type_pairs:
+            arr = np.array([], dtype=dtype)
+            _check_array_roundtrip(arr, type=pa_type)
+
     def test_non_string_columns(self):
         df = pd.DataFrame({0: [1, 2, 3]})
         table = pa.Table.from_pandas(df)

From 4a8a102d508fc31570584b3618291103a901e6b0 Mon Sep 17 00:00:00 2001
From: Panchen Xue <pan.panchen.xue@gmail.com>
Date: Tue, 13 Feb 2018 01:20:29 -0500
Subject: [PATCH 1477/1644] ARROW-969: [C++] Add add/remove field functions for
 RecordBatch

Add AddColumn and RemoveColumn methods for RecordBatch, as well as test cases

Author: Panchen Xue <pan.panchen.xue@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1574 from xuepanchen/ARROW-969 and squashes the following commits:

b082c46b [Wes McKinney] Add variant of RecordBatch::AddColumn that takes a field name instead of a Field
cd3a69d0 [Panchen Xue] Move index boundscheck to Schema module and add more test cases for boundscheck
68e93f49 [Panchen Xue] Add test cases for AddColumn and RemoveColumn
5ebfdfff [Panchen Xue] Add AddColumn and RemoveColumn methods for RecordBatch
---
 cpp/src/arrow/record_batch.cc |  45 ++++++++++++++
 cpp/src/arrow/record_batch.h  |  29 +++++++++
 cpp/src/arrow/table-test.cc   | 111 ++++++++++++++++++++++++++++++++++
 cpp/src/arrow/table.cc        |  10 +--
 cpp/src/arrow/type.cc         |  10 +--
 5 files changed, 193 insertions(+), 12 deletions(-)

diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc
index d418cc4a2e66c..f295b864c0066 100644
--- a/cpp/src/arrow/record_batch.cc
+++ b/cpp/src/arrow/record_batch.cc
@@ -21,15 +21,24 @@
 #include <cstdlib>
 #include <memory>
 #include <sstream>
+#include <string>
 #include <utility>
 
 #include "arrow/array.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/logging.h"
+#include "arrow/util/stl.h"
 
 namespace arrow {
 
+Status RecordBatch::AddColumn(int i, const std::string& field_name,
+                              const std::shared_ptr<Array>& column,
+                              std::shared_ptr<RecordBatch>* out) const {
+  auto field = ::arrow::field(field_name, column->type());
+  return AddColumn(i, field, column, out);
+}
+
 /// \class SimpleRecordBatch
 /// \brief A basic, non-lazy in-memory record batch
 class SimpleRecordBatch : public RecordBatch {
@@ -78,6 +87,42 @@ class SimpleRecordBatch : public RecordBatch {
 
   std::shared_ptr<ArrayData> column_data(int i) const override { return columns_[i]; }
 
+  Status AddColumn(int i, const std::shared_ptr<Field>& field,
+                   const std::shared_ptr<Array>& column,
+                   std::shared_ptr<RecordBatch>* out) const override {
+    DCHECK(field != nullptr);
+    DCHECK(column != nullptr);
+
+    if (!field->type()->Equals(column->type())) {
+      std::stringstream ss;
+      ss << "Column data type " << field->type()->name()
+         << " does not match field data type " << column->type()->name();
+      return Status::Invalid(ss.str());
+    }
+    if (column->length() != num_rows_) {
+      std::stringstream ss;
+      ss << "Added column's length must match record batch's length. Expected length "
+         << num_rows_ << " but got length " << column->length();
+      return Status::Invalid(ss.str());
+    }
+
+    std::shared_ptr<Schema> new_schema;
+    RETURN_NOT_OK(schema_->AddField(i, field, &new_schema));
+
+    *out = RecordBatch::Make(new_schema, num_rows_,
+                             internal::AddVectorElement(columns_, i, column->data()));
+    return Status::OK();
+  }
+
+  Status RemoveColumn(int i, std::shared_ptr<RecordBatch>* out) const override {
+    std::shared_ptr<Schema> new_schema;
+    RETURN_NOT_OK(schema_->RemoveField(i, &new_schema));
+
+    *out = RecordBatch::Make(new_schema, num_rows_,
+                             internal::DeleteVectorElement(columns_, i));
+    return Status::OK();
+  }
+
   std::shared_ptr<RecordBatch> ReplaceSchemaMetadata(
       const std::shared_ptr<const KeyValueMetadata>& metadata) const override {
     auto new_schema = schema_->AddMetadata(metadata);
diff --git a/cpp/src/arrow/record_batch.h b/cpp/src/arrow/record_batch.h
index b2c4c76b3f2d3..6fb747c409d15 100644
--- a/cpp/src/arrow/record_batch.h
+++ b/cpp/src/arrow/record_batch.h
@@ -96,6 +96,35 @@ class ARROW_EXPORT RecordBatch {
   /// \return an internal ArrayData object
   virtual std::shared_ptr<ArrayData> column_data(int i) const = 0;
 
+  /// \brief Add column to the record batch, producing a new RecordBatch
+  ///
+  /// \param[in] i field index, which will be boundschecked
+  /// \param[in] field field to be added
+  /// \param[in] column column to be added
+  /// \param[out] out record batch with column added
+  virtual Status AddColumn(int i, const std::shared_ptr<Field>& field,
+                           const std::shared_ptr<Array>& column,
+                           std::shared_ptr<RecordBatch>* out) const = 0;
+
+  /// \brief Add new nullable column to the record batch, producing a new
+  /// RecordBatch.
+  ///
+  /// For non-nullable columns, use the Field-based version of this method.
+  ///
+  /// \param[in] i field index, which will be boundschecked
+  /// \param[in] field_name name of field to be added
+  /// \param[in] column column to be added
+  /// \param[out] out record batch with column added
+  virtual Status AddColumn(int i, const std::string& field_name,
+                           const std::shared_ptr<Array>& column,
+                           std::shared_ptr<RecordBatch>* out) const;
+
+  /// \brief Remove column from the record batch, producing a new RecordBatch
+  ///
+  /// \param[in] i field index, does boundscheck
+  /// \param[out] out record batch with column removed
+  virtual Status RemoveColumn(int i, std::shared_ptr<RecordBatch>* out) const = 0;
+
   virtual std::shared_ptr<RecordBatch> ReplaceSchemaMetadata(
       const std::shared_ptr<const KeyValueMetadata>& metadata) const = 0;
 
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 99e4dd5db5146..af744168227cd 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -466,6 +466,8 @@ TEST_F(TestTable, AddColumn) {
   // Some negative tests with invalid index
   Status status = table.AddColumn(10, columns_[0], &result);
   ASSERT_TRUE(status.IsInvalid());
+  status = table.AddColumn(4, columns_[0], &result);
+  ASSERT_TRUE(status.IsInvalid());
   status = table.AddColumn(-1, columns_[0], &result);
   ASSERT_TRUE(status.IsInvalid());
 
@@ -588,6 +590,115 @@ TEST_F(TestRecordBatch, Slice) {
   }
 }
 
+TEST_F(TestRecordBatch, AddColumn) {
+  const int length = 10;
+
+  auto field1 = field("f1", int32());
+  auto field2 = field("f2", uint8());
+  auto field3 = field("f3", int16());
+
+  auto schema1 = ::arrow::schema({field1, field2});
+  auto schema2 = ::arrow::schema({field2, field3});
+  auto schema3 = ::arrow::schema({field2});
+
+  auto array1 = MakeRandomArray<Int32Array>(length);
+  auto array2 = MakeRandomArray<UInt8Array>(length);
+  auto array3 = MakeRandomArray<Int16Array>(length);
+
+  auto batch1 = RecordBatch::Make(schema1, length, {array1, array2});
+  auto batch2 = RecordBatch::Make(schema2, length, {array2, array3});
+  auto batch3 = RecordBatch::Make(schema3, length, {array2});
+
+  const RecordBatch& batch = *batch3;
+  std::shared_ptr<RecordBatch> result;
+
+  // Negative tests with invalid index
+  Status status = batch.AddColumn(5, field1, array1, &result);
+  ASSERT_TRUE(status.IsInvalid());
+  status = batch.AddColumn(2, field1, array1, &result);
+  ASSERT_TRUE(status.IsInvalid());
+  status = batch.AddColumn(-1, field1, array1, &result);
+  ASSERT_TRUE(status.IsInvalid());
+
+  // Negative test with wrong length
+  auto longer_col = MakeRandomArray<Int32Array>(length + 1);
+  status = batch.AddColumn(0, field1, longer_col, &result);
+  ASSERT_TRUE(status.IsInvalid());
+
+  // Negative test with mismatch type
+  status = batch.AddColumn(0, field1, array2, &result);
+  ASSERT_TRUE(status.IsInvalid());
+
+  ASSERT_OK(batch.AddColumn(0, field1, array1, &result));
+  ASSERT_TRUE(result->Equals(*batch1));
+
+  ASSERT_OK(batch.AddColumn(1, field3, array3, &result));
+  ASSERT_TRUE(result->Equals(*batch2));
+
+  std::shared_ptr<RecordBatch> result2;
+  ASSERT_OK(batch.AddColumn(1, "f3", array3, &result2));
+  ASSERT_TRUE(result2->Equals(*result));
+
+  ASSERT_TRUE(result2->schema()->field(1)->nullable());
+}
+
+TEST_F(TestRecordBatch, RemoveColumn) {
+  const int length = 10;
+
+  auto field1 = field("f1", int32());
+  auto field2 = field("f2", uint8());
+  auto field3 = field("f3", int16());
+
+  auto schema1 = ::arrow::schema({field1, field2, field3});
+  auto schema2 = ::arrow::schema({field2, field3});
+  auto schema3 = ::arrow::schema({field1, field3});
+  auto schema4 = ::arrow::schema({field1, field2});
+
+  auto array1 = MakeRandomArray<Int32Array>(length);
+  auto array2 = MakeRandomArray<UInt8Array>(length);
+  auto array3 = MakeRandomArray<Int16Array>(length);
+
+  auto batch1 = RecordBatch::Make(schema1, length, {array1, array2, array3});
+  auto batch2 = RecordBatch::Make(schema2, length, {array2, array3});
+  auto batch3 = RecordBatch::Make(schema3, length, {array1, array3});
+  auto batch4 = RecordBatch::Make(schema4, length, {array1, array2});
+
+  const RecordBatch& batch = *batch1;
+  std::shared_ptr<RecordBatch> result;
+
+  // Negative tests with invalid index
+  Status status = batch.RemoveColumn(3, &result);
+  ASSERT_TRUE(status.IsInvalid());
+  status = batch.RemoveColumn(-1, &result);
+  ASSERT_TRUE(status.IsInvalid());
+
+  ASSERT_OK(batch.RemoveColumn(0, &result));
+  ASSERT_TRUE(result->Equals(*batch2));
+
+  ASSERT_OK(batch.RemoveColumn(1, &result));
+  ASSERT_TRUE(result->Equals(*batch3));
+
+  ASSERT_OK(batch.RemoveColumn(2, &result));
+  ASSERT_TRUE(result->Equals(*batch4));
+}
+
+TEST_F(TestRecordBatch, RemoveColumnEmpty) {
+  const int length = 10;
+
+  auto field1 = field("f1", int32());
+  auto schema1 = ::arrow::schema({field1});
+  auto array1 = MakeRandomArray<Int32Array>(length);
+  auto batch1 = RecordBatch::Make(schema1, length, {array1});
+
+  std::shared_ptr<RecordBatch> empty;
+  ASSERT_OK(batch1->RemoveColumn(0, &empty));
+  ASSERT_EQ(batch1->num_rows(), empty->num_rows());
+
+  std::shared_ptr<RecordBatch> added;
+  ASSERT_OK(empty->AddColumn(0, field1, array1, &added));
+  ASSERT_TRUE(added->Equals(*batch1));
+}
+
 class TestTableBatchReader : public TestBase {};
 
 TEST_F(TestTableBatchReader, ReadNext) {
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 8cfd67faef1ee..62ea32a8d9080 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -234,14 +234,8 @@ class SimpleTable : public Table {
 
   Status AddColumn(int i, const std::shared_ptr<Column>& col,
                    std::shared_ptr<Table>* out) const override {
-    if (i < 0 || i > num_columns() + 1) {
-      return Status::Invalid("Invalid column index.");
-    }
-    if (col == nullptr) {
-      std::stringstream ss;
-      ss << "Column " << i << " was null";
-      return Status::Invalid(ss.str());
-    }
+    DCHECK(col != nullptr);
+
     if (col->length() != num_rows_) {
       std::stringstream ss;
       ss << "Added column's length must match table's length. Expected length "
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 0a2889f040026..836a2aa930f6f 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -293,8 +293,9 @@ int64_t Schema::GetFieldIndex(const std::string& name) const {
 
 Status Schema::AddField(int i, const std::shared_ptr<Field>& field,
                         std::shared_ptr<Schema>* out) const {
-  DCHECK_GE(i, 0);
-  DCHECK_LE(i, this->num_fields());
+  if (i < 0 || i > this->num_fields()) {
+    return Status::Invalid("Invalid column index to add field.");
+  }
 
   *out =
       std::make_shared<Schema>(internal::AddVectorElement(fields_, i, field), metadata_);
@@ -323,8 +324,9 @@ std::shared_ptr<Schema> Schema::RemoveMetadata() const {
 }
 
 Status Schema::RemoveField(int i, std::shared_ptr<Schema>* out) const {
-  DCHECK_GE(i, 0);
-  DCHECK_LT(i, this->num_fields());
+  if (i < 0 || i >= this->num_fields()) {
+    return Status::Invalid("Invalid column index to remove field.");
+  }
 
   *out = std::make_shared<Schema>(internal::DeleteVectorElement(fields_, i), metadata_);
   return Status::OK();

From d28f055ee20ae57b9acf16d47baa328f132d9ea3 Mon Sep 17 00:00:00 2001
From: yosuke shiro <ys2014hs@gmail.com>
Date: Tue, 13 Feb 2018 13:33:51 +0100
Subject: [PATCH 1478/1644] ARROW-2146: [GLib] Add Slice api to ChunkedArray

Add Slice api to ChunkedArray.

Author: yosuke shiro <ys2014hs@gmail.com>

Closes #1600 from shiro615/ARROW-2146 and squashes the following commits:

9b95c70 [yosuke shiro] ARROW-2146: [GLib] Add Slice api to ChunkedArray
---
 c_glib/arrow-glib/chunked-array.cpp | 20 ++++++++++++++++++++
 c_glib/arrow-glib/chunked-array.h   |  3 +++
 c_glib/test/test-chunked-array.rb   | 14 ++++++++++++++
 3 files changed, 37 insertions(+)

diff --git a/c_glib/arrow-glib/chunked-array.cpp b/c_glib/arrow-glib/chunked-array.cpp
index 69195c57b1768..a4f81755974f0 100644
--- a/c_glib/arrow-glib/chunked-array.cpp
+++ b/c_glib/arrow-glib/chunked-array.cpp
@@ -274,6 +274,26 @@ garrow_chunked_array_get_chunks(GArrowChunkedArray *chunked_array)
   return g_list_reverse(chunks);
 }
 
+/**
+ * garrow_chunked_array_slice:
+ * @chunked_array: A #GArrowChunkedArray.
+ * @offset: The offset of sub #GArrowChunkedArray.
+ * @length: The length of sub #GArrowChunkedArray.
+ *
+ * Returns: (transfer full): The sub #GArrowChunkedArray. It covers only from
+ *   `offset` to `offset + length` range. The sub #GArrowChunkedArray shares
+ *   values with the base #GArrowChunkedArray.
+ */
+GArrowChunkedArray  *
+garrow_chunked_array_slice(GArrowChunkedArray *chunked_array,
+                           guint64 offset,
+                           guint64 length)
+{
+  const auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array);
+  auto arrow_sub_chunked_array = arrow_chunked_array->Slice(offset, length);
+  return garrow_chunked_array_new_raw(&arrow_sub_chunked_array);
+}
+
 G_END_DECLS
 
 GArrowChunkedArray *
diff --git a/c_glib/arrow-glib/chunked-array.h b/c_glib/arrow-glib/chunked-array.h
index 0c3c81a744ceb..d109150d2fd46 100644
--- a/c_glib/arrow-glib/chunked-array.h
+++ b/c_glib/arrow-glib/chunked-array.h
@@ -82,5 +82,8 @@ guint   garrow_chunked_array_get_n_chunks (GArrowChunkedArray *chunked_array);
 GArrowArray *garrow_chunked_array_get_chunk(GArrowChunkedArray *chunked_array,
                                             guint i);
 GList *garrow_chunked_array_get_chunks(GArrowChunkedArray *chunked_array);
+GArrowChunkedArray *garrow_chunked_array_slice(GArrowChunkedArray *chunked_array,
+                                               guint64 offset,
+                                               guint64 length);
 
 G_END_DECLS
diff --git a/c_glib/test/test-chunked-array.rb b/c_glib/test/test-chunked-array.rb
index 4f6a4fb0df0b5..252e1e2bbd06f 100644
--- a/c_glib/test/test-chunked-array.rb
+++ b/c_glib/test/test-chunked-array.rb
@@ -95,4 +95,18 @@ def test_chunks
     assert_equal([2, 1],
                  chunked_array.chunks.collect(&:length))
   end
+
+  def test_slice
+    chunks1 = [
+      build_boolean_array([true, false, true]),
+      build_boolean_array([false, true]),
+    ]
+    chunks2 = [
+      build_boolean_array([false, true]),
+      build_boolean_array([false]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(chunks1)
+    sub_chunked_array = chunked_array.slice(1, 3)
+    assert_equal(chunks2, sub_chunked_array.chunks)
+  end
 end

From c78c92a04f3db0bd7dd9a34e6f36743050871acc Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 13 Feb 2018 11:19:44 -0500
Subject: [PATCH 1479/1644] ARROW-2149: [Python] Reorganize
 test_convert_pandas.py

This PR simply shuffles things around, reorganizing the test methods under several classes.

Author: Antoine Pitrou <antoine@python.org>

Closes #1601 from pitrou/ARROW-2149-reorganize-test-convert-pandas and squashes the following commits:

821c59be [Antoine Pitrou] ARROW-2149: [Python] Reorganize test_convert_pandas.py
---
 python/pyarrow/tests/test_convert_pandas.py | 1447 ++++++++++---------
 1 file changed, 749 insertions(+), 698 deletions(-)

diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 987ac23ec7117..5b6f6bcdfe5af 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -117,41 +117,10 @@ def _check_array_from_pandas_roundtrip(np_array):
     npt.assert_array_equal(result, np_array)
 
 
-class TestPandasConversion(object):
-
-    type_pairs = [
-        (np.int8, pa.int8()),
-        (np.int16, pa.int16()),
-        (np.int32, pa.int32()),
-        (np.int64, pa.int64()),
-        (np.uint8, pa.uint8()),
-        (np.uint16, pa.uint16()),
-        (np.uint32, pa.uint32()),
-        (np.uint64, pa.uint64()),
-        # (np.float16, pa.float16()),  # XXX unsupported
-        (np.float32, pa.float32()),
-        (np.float64, pa.float64()),
-        # XXX unsupported
-        # (np.dtype([('a', 'i2')]), pa.struct([pa.field('a', pa.int16())])),
-        (np.object, pa.string()),
-        # (np.object, pa.binary()),  # XXX unsupported
-        (np.object, pa.binary(10)),
-        (np.object, pa.list_(pa.int64())),
-        ]
-
-    def test_all_none_objects(self):
-        df = pd.DataFrame({'a': [None, None, None]})
-        _check_pandas_roundtrip(df)
-
-    def test_all_none_category(self):
-        df = pd.DataFrame({'a': [None, None, None]})
-        df['a'] = df['a'].astype('category')
-        _check_pandas_roundtrip(df)
-
-    def test_empty_arrays(self):
-        for dtype, pa_type in self.type_pairs:
-            arr = np.array([], dtype=dtype)
-            _check_array_roundtrip(arr, type=pa_type)
+class TestConvertMetadata(object):
+    """
+    Conversion tests for Pandas metadata & indices.
+    """
 
     def test_non_string_columns(self):
         df = pd.DataFrame({0: [1, 2, 3]})
@@ -302,25 +271,6 @@ def test_categorical_row_index(self):
 
         _check_pandas_roundtrip(df, preserve_index=True)
 
-    def test_float_no_nulls(self):
-        data = {}
-        fields = []
-        dtypes = [('f4', pa.float32()), ('f8', pa.float64())]
-        num_values = 100
-
-        for numpy_dtype, arrow_dtype in dtypes:
-            values = np.random.randn(num_values)
-            data[numpy_dtype] = values.astype(numpy_dtype)
-            fields.append(pa.field(numpy_dtype, arrow_dtype))
-
-        df = pd.DataFrame(data)
-        schema = pa.schema(fields)
-        _check_pandas_roundtrip(df, expected_schema=schema)
-
-    def test_zero_copy_success(self):
-        result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True)
-        npt.assert_array_equal(result, [0, 1, 2])
-
     def test_duplicate_column_names_does_not_crash(self):
         df = pd.DataFrame([(1, 'a'), (2, 'b')], columns=list('aa'))
         with pytest.raises(ValueError):
@@ -341,44 +291,187 @@ def test_dictionary_indices_boundscheck(self):
             with pytest.raises(pa.ArrowException):
                 table.to_pandas()
 
-    def test_zero_copy_dictionaries(self):
-        arr = pa.DictionaryArray.from_arrays(
-            np.array([0, 0]),
-            np.array([5]))
+    def test_unicode_with_unicode_column_and_index(self):
+        df = pd.DataFrame({u'あ': [u'い']}, index=[u'う'])
 
-        result = arr.to_pandas(zero_copy_only=True)
-        values = pd.Categorical([5, 5])
+        _check_pandas_roundtrip(df, preserve_index=True)
 
-        tm.assert_series_equal(pd.Series(result), pd.Series(values),
-                               check_names=False)
+    def test_mixed_unicode_column_names(self):
+        df = pd.DataFrame({u'あ': [u'い'], b'a': 1}, index=[u'う'])
 
-    def test_zero_copy_failure_on_object_types(self):
-        with pytest.raises(pa.ArrowException):
-            pa.array(['A', 'B', 'C']).to_pandas(zero_copy_only=True)
+        # TODO(phillipc): Should this raise?
+        with pytest.raises(AssertionError):
+            _check_pandas_roundtrip(df, preserve_index=True)
 
-    def test_zero_copy_failure_with_int_when_nulls(self):
-        with pytest.raises(pa.ArrowException):
-            pa.array([0, 1, None]).to_pandas(zero_copy_only=True)
+    def test_binary_column_name(self):
+        column_data = [u'い']
+        data = {u'あ'.encode('utf8'): column_data}
+        df = pd.DataFrame(data)
 
-    def test_zero_copy_failure_with_float_when_nulls(self):
-        with pytest.raises(pa.ArrowException):
-            pa.array([0.0, 1.0, None]).to_pandas(zero_copy_only=True)
+        # we can't use _check_pandas_roundtrip here because our metdata
+        # is always decoded as utf8: even if binary goes in, utf8 comes out
+        t = pa.Table.from_pandas(df, preserve_index=True)
+        df2 = t.to_pandas()
+        assert df.values[0] == df2.values[0]
+        assert df.index.values[0] == df2.index.values[0]
+        assert df.columns[0] == df2.columns[0].encode('utf8')
 
-    def test_zero_copy_failure_on_bool_types(self):
-        with pytest.raises(pa.ArrowException):
-            pa.array([True, False]).to_pandas(zero_copy_only=True)
+    def test_multiindex_duplicate_values(self):
+        num_rows = 3
+        numbers = list(range(num_rows))
+        index = pd.MultiIndex.from_arrays(
+            [['foo', 'foo', 'bar'], numbers],
+            names=['foobar', 'some_numbers'],
+        )
 
-    def test_zero_copy_failure_on_list_types(self):
-        arr = np.array([[1, 2], [8, 9]], dtype=object)
+        df = pd.DataFrame({'numbers': numbers}, index=index)
 
-        with pytest.raises(pa.ArrowException):
-            pa.array(arr).to_pandas(zero_copy_only=True)
+        table = pa.Table.from_pandas(df)
+        result_df = table.to_pandas()
+        tm.assert_frame_equal(result_df, df)
 
-    def test_zero_copy_failure_on_timestamp_types(self):
-        arr = np.array(['2007-07-13'], dtype='datetime64[ns]')
+    def test_metadata_with_mixed_types(self):
+        df = pd.DataFrame({'data': [b'some_bytes', u'some_unicode']})
+        table = pa.Table.from_pandas(df)
+        metadata = table.schema.metadata
+        assert b'mixed' not in metadata[b'pandas']
 
-        with pytest.raises(pa.ArrowException):
-            pa.array(arr).to_pandas(zero_copy_only=True)
+        js = json.loads(metadata[b'pandas'].decode('utf8'))
+        data_column = js['columns'][0]
+        assert data_column['pandas_type'] == 'bytes'
+        assert data_column['numpy_type'] == 'object'
+
+    def test_list_metadata(self):
+        df = pd.DataFrame({'data': [[1], [2, 3, 4], [5] * 7]})
+        schema = pa.schema([pa.field('data', type=pa.list_(pa.int64()))])
+        table = pa.Table.from_pandas(df, schema=schema)
+        metadata = table.schema.metadata
+        assert b'mixed' not in metadata[b'pandas']
+
+        js = json.loads(metadata[b'pandas'].decode('utf8'))
+        data_column = js['columns'][0]
+        assert data_column['pandas_type'] == 'list[int64]'
+        assert data_column['numpy_type'] == 'object'
+
+    def test_decimal_metadata(self):
+        expected = pd.DataFrame({
+            'decimals': [
+                decimal.Decimal('394092382910493.12341234678'),
+                -decimal.Decimal('314292388910493.12343437128'),
+            ]
+        })
+        table = pa.Table.from_pandas(expected)
+        metadata = table.schema.metadata
+        assert b'mixed' not in metadata[b'pandas']
+
+        js = json.loads(metadata[b'pandas'].decode('utf8'))
+        data_column = js['columns'][0]
+        assert data_column['pandas_type'] == 'decimal'
+        assert data_column['numpy_type'] == 'object'
+        assert data_column['metadata'] == {'precision': 26, 'scale': 11}
+
+    def test_table_column_subset_metadata(self):
+        # ARROW-1883
+        df = pd.DataFrame({
+            'a': [1, 2, 3],
+            'b': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')})
+        table = pa.Table.from_pandas(df)
+
+        table_subset = table.remove_column(1)
+        result = table_subset.to_pandas()
+        tm.assert_frame_equal(result, df[['a']])
+
+        table_subset2 = table_subset.remove_column(1)
+        result = table_subset2.to_pandas()
+        tm.assert_frame_equal(result, df[['a']])
+
+        # non-default index
+        for index in [
+                pd.Index(['a', 'b', 'c'], name='index'),
+                pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')]:
+            df = pd.DataFrame({'a': [1, 2, 3],
+                               'b': [.1, .2, .3]}, index=index)
+            table = pa.Table.from_pandas(df)
+
+            table_subset = table.remove_column(1)
+            result = table_subset.to_pandas()
+            tm.assert_frame_equal(result, df[['a']])
+
+            table_subset2 = table_subset.remove_column(1)
+            result = table_subset2.to_pandas()
+            tm.assert_frame_equal(result, df[['a']].reset_index(drop=True))
+
+    def test_empty_list_metadata(self):
+        # Create table with array of empty lists, forced to have type
+        # list(string) in pyarrow
+        c1 = [["test"], ["a", "b"], None]
+        c2 = [[], [], []]
+        arrays = OrderedDict([
+            ('c1', pa.array(c1, type=pa.list_(pa.string()))),
+            ('c2', pa.array(c2, type=pa.list_(pa.string()))),
+        ])
+        rb = pa.RecordBatch.from_arrays(
+            list(arrays.values()),
+            list(arrays.keys())
+        )
+        tbl = pa.Table.from_batches([rb])
+
+        # First roundtrip changes schema, because pandas cannot preserve the
+        # type of empty lists
+        df = tbl.to_pandas()
+        tbl2 = pa.Table.from_pandas(df, preserve_index=True)
+        md2 = json.loads(tbl2.schema.metadata[b'pandas'].decode('utf8'))
+
+        # Second roundtrip
+        df2 = tbl2.to_pandas()
+        expected = pd.DataFrame(OrderedDict([('c1', c1), ('c2', c2)]))
+
+        tm.assert_frame_equal(df2, expected)
+
+        assert md2['columns'] == [
+            {
+                'name': 'c1',
+                'field_name': 'c1',
+                'metadata': None,
+                'numpy_type': 'object',
+                'pandas_type': 'list[unicode]',
+            },
+            {
+                'name': 'c2',
+                'field_name': 'c2',
+                'metadata': None,
+                'numpy_type': 'object',
+                'pandas_type': 'list[empty]',
+            },
+            {
+                'name': None,
+                'field_name': '__index_level_0__',
+                'metadata': None,
+                'numpy_type': 'int64',
+                'pandas_type': 'int64',
+            }
+        ]
+
+
+class TestConvertPrimitiveTypes(object):
+    """
+    Conversion tests for primitive (e.g. numeric) types.
+    """
+
+    def test_float_no_nulls(self):
+        data = {}
+        fields = []
+        dtypes = [('f4', pa.float32()), ('f8', pa.float64())]
+        num_values = 100
+
+        for numpy_dtype, arrow_dtype in dtypes:
+            values = np.random.randn(num_values)
+            data[numpy_dtype] = values.astype(numpy_dtype)
+            fields.append(pa.field(numpy_dtype, arrow_dtype))
+
+        df = pd.DataFrame(data)
+        schema = pa.schema(fields)
+        _check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_float_nulls(self):
         num_values = 100
@@ -408,24 +501,6 @@ def test_float_nulls(self):
         result = table.to_pandas()
         tm.assert_frame_equal(result, ex_frame)
 
-    def test_float_object_nulls(self):
-        arr = np.array([None, 1.5, np.float64(3.5)] * 5, dtype=object)
-        df = pd.DataFrame({'floats': arr})
-        expected = pd.DataFrame({'floats': pd.to_numeric(arr)})
-        field = pa.field('floats', pa.float64())
-        schema = pa.schema([field])
-        _check_pandas_roundtrip(df, expected=expected,
-                                expected_schema=schema)
-
-    def test_int_object_nulls(self):
-        arr = np.array([None, 1, np.int64(3)] * 5, dtype=object)
-        df = pd.DataFrame({'ints': arr})
-        expected = pd.DataFrame({'ints': pd.to_numeric(arr)})
-        field = pa.field('ints', pa.int64())
-        schema = pa.schema([field])
-        _check_pandas_roundtrip(df, expected=expected,
-                                expected_schema=schema)
-
     def test_integer_no_nulls(self):
         data = OrderedDict()
         fields = []
@@ -522,6 +597,24 @@ def test_boolean_nulls(self):
 
         tm.assert_frame_equal(result, ex_frame)
 
+    def test_float_object_nulls(self):
+        arr = np.array([None, 1.5, np.float64(3.5)] * 5, dtype=object)
+        df = pd.DataFrame({'floats': arr})
+        expected = pd.DataFrame({'floats': pd.to_numeric(arr)})
+        field = pa.field('floats', pa.float64())
+        schema = pa.schema([field])
+        _check_pandas_roundtrip(df, expected=expected,
+                                expected_schema=schema)
+
+    def test_int_object_nulls(self):
+        arr = np.array([None, 1, np.int64(3)] * 5, dtype=object)
+        df = pd.DataFrame({'ints': arr})
+        expected = pd.DataFrame({'ints': pd.to_numeric(arr)})
+        field = pa.field('ints', pa.int64())
+        schema = pa.schema([field])
+        _check_pandas_roundtrip(df, expected=expected,
+                                expected_schema=schema)
+
     def test_boolean_object_nulls(self):
         arr = np.array([False, None, True] * 100, dtype=object)
         df = pd.DataFrame({'bools': arr})
@@ -540,81 +633,11 @@ def _check_type(t):
         _check_type(pa.int32())
         _check_type(pa.float64())
 
-    def test_unicode(self):
-        repeats = 1000
-        values = [u'foo', None, u'bar', u'mañana', np.nan]
-        df = pd.DataFrame({'strings': values * repeats})
-        field = pa.field('strings', pa.string())
-        schema = pa.schema([field])
-
-        _check_pandas_roundtrip(df, expected_schema=schema)
-
-    def test_unicode_with_unicode_column_and_index(self):
-        df = pd.DataFrame({u'あ': [u'い']}, index=[u'う'])
-
-        _check_pandas_roundtrip(df, preserve_index=True)
-
-    def test_mixed_unicode_column_names(self):
-        df = pd.DataFrame({u'あ': [u'い'], b'a': 1}, index=[u'う'])
-
-        # TODO(phillipc): Should this raise?
-        with pytest.raises(AssertionError):
-            _check_pandas_roundtrip(df, preserve_index=True)
-
-    def test_binary_column_name(self):
-        column_data = [u'い']
-        data = {u'あ'.encode('utf8'): column_data}
-        df = pd.DataFrame(data)
-
-        # we can't use _check_pandas_roundtrip here because our metdata
-        # is always decoded as utf8: even if binary goes in, utf8 comes out
-        t = pa.Table.from_pandas(df, preserve_index=True)
-        df2 = t.to_pandas()
-        assert df.values[0] == df2.values[0]
-        assert df.index.values[0] == df2.index.values[0]
-        assert df.columns[0] == df2.columns[0].encode('utf8')
-
-    def test_bytes_to_binary(self):
-        values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan]
-        df = pd.DataFrame({'strings': values})
-
-        table = pa.Table.from_pandas(df)
-        assert table[0].type == pa.binary()
-
-        values2 = [b'qux', b'foo', None, b'bar', b'qux', np.nan]
-        expected = pd.DataFrame({'strings': values2})
-        _check_pandas_roundtrip(df, expected)
-
-    @pytest.mark.large_memory
-    def test_bytes_exceed_2gb(self):
-        val = 'x' * (1 << 20)
-        df = pd.DataFrame({
-            'strings': np.array([val] * 4000, dtype=object)
-        })
-        arr = pa.array(df['strings'])
-        assert isinstance(arr, pa.ChunkedArray)
-        assert arr.num_chunks == 2
-        arr = None
-
-        table = pa.Table.from_pandas(df)
-        assert table[0].data.num_chunks == 2
-
-    def test_fixed_size_bytes(self):
-        values = [b'foo', None, b'bar', None, None, b'hey']
-        df = pd.DataFrame({'strings': values})
-        schema = pa.schema([pa.field('strings', pa.binary(3))])
-        table = pa.Table.from_pandas(df, schema=schema)
-        assert table.schema[0].type == schema[0].type
-        assert table.schema[0].name == schema[0].name
-        result = table.to_pandas()
-        tm.assert_frame_equal(result, df)
 
-    def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
-        values = [b'foo', None, b'ba', None, None, b'hey']
-        df = pd.DataFrame({'strings': values})
-        schema = pa.schema([pa.field('strings', pa.binary(3))])
-        with pytest.raises(pa.ArrowInvalid):
-            pa.Table.from_pandas(df, schema=schema)
+class TestConvertDateTimeLikeTypes(object):
+    """
+    Conversion tests for datetime- and timestamp-like types (date64, etc.).
+    """
 
     def test_timestamps_notimezone_no_nulls(self):
         df = pd.DataFrame({
@@ -788,205 +811,257 @@ def test_timedelta(self):
         })
         pa.Table.from_pandas(df)
 
-    def test_column_of_arrays(self):
-        df, schema = dataframe_with_arrays()
-        _check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
-        table = pa.Table.from_pandas(df, schema=schema, preserve_index=False)
-        assert table.schema.equals(schema)
+    def test_pytime_from_pandas(self):
+        pytimes = [time(1, 2, 3, 1356),
+                   time(4, 5, 6, 1356)]
 
-        for column in df.columns:
-            field = schema.field_by_name(column)
-            _check_array_roundtrip(df[column], type=field.type)
+        # microseconds
+        t1 = pa.time64('us')
 
-    def test_column_of_arrays_to_py(self):
-        # Test regression in ARROW-1199 not caught in above test
-        dtype = 'i1'
-        arr = np.array([
-            np.arange(10, dtype=dtype),
-            np.arange(5, dtype=dtype),
-            None,
-            np.arange(1, dtype=dtype)
-        ])
-        type_ = pa.list_(pa.int8())
-        parr = pa.array(arr, type=type_)
+        aobjs = np.array(pytimes + [None], dtype=object)
+        parr = pa.array(aobjs)
+        assert parr.type == t1
+        assert parr[0].as_py() == pytimes[0]
+        assert parr[1].as_py() == pytimes[1]
+        assert parr[2] is pa.NA
 
-        assert parr[0].as_py() == list(range(10))
-        assert parr[1].as_py() == list(range(5))
-        assert parr[2].as_py() is None
-        assert parr[3].as_py() == [0]
+        # DataFrame
+        df = pd.DataFrame({'times': aobjs})
+        batch = pa.RecordBatch.from_pandas(df)
+        assert batch[0].equals(parr)
 
-    def test_column_of_lists(self):
-        df, schema = dataframe_with_lists()
-        _check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
-        table = pa.Table.from_pandas(df, schema=schema, preserve_index=False)
-        assert table.schema.equals(schema)
+        # Test ndarray of int64 values
+        arr = np.array([_pytime_to_micros(v) for v in pytimes],
+                       dtype='int64')
 
-        for column in df.columns:
-            field = schema.field_by_name(column)
-            _check_array_roundtrip(df[column], type=field.type)
+        a1 = pa.array(arr, type=pa.time64('us'))
+        assert a1[0].as_py() == pytimes[0]
 
-    def test_column_of_lists_first_empty(self):
-        # ARROW-2124
-        num_lists = [[], [2, 3, 4], [3, 6, 7, 8], [], [2]]
-        series = pd.Series([np.array(s, dtype=float) for s in num_lists])
-        arr = pa.array(series)
-        result = pd.Series(arr.to_pandas())
-        tm.assert_series_equal(result, series)
+        a2 = pa.array(arr * 1000, type=pa.time64('ns'))
+        assert a2[0].as_py() == pytimes[0]
 
-    def test_column_of_lists_chunked(self):
-        # ARROW-1357
-        df = pd.DataFrame({
-            'lists': np.array([
-                [1, 2],
-                None,
-                [2, 3],
-                [4, 5],
-                [6, 7],
-                [8, 9]
-            ], dtype=object)
-        })
+        a3 = pa.array((arr / 1000).astype('i4'),
+                      type=pa.time32('ms'))
+        assert a3[0].as_py() == pytimes[0].replace(microsecond=1000)
 
-        schema = pa.schema([
-            pa.field('lists', pa.list_(pa.int64()))
-        ])
+        a4 = pa.array((arr / 1000000).astype('i4'),
+                      type=pa.time32('s'))
+        assert a4[0].as_py() == pytimes[0].replace(microsecond=0)
 
-        t1 = pa.Table.from_pandas(df[:2], schema=schema)
-        t2 = pa.Table.from_pandas(df[2:], schema=schema)
+    def test_arrow_time_to_pandas(self):
+        pytimes = [time(1, 2, 3, 1356),
+                   time(4, 5, 6, 1356),
+                   time(0, 0, 0)]
 
-        table = pa.concat_tables([t1, t2])
-        result = table.to_pandas()
+        expected = np.array(pytimes[:2] + [None])
+        expected_ms = np.array([x.replace(microsecond=1000)
+                                for x in pytimes[:2]] +
+                               [None])
+        expected_s = np.array([x.replace(microsecond=0)
+                               for x in pytimes[:2]] +
+                              [None])
 
-        tm.assert_frame_equal(result, df)
+        arr = np.array([_pytime_to_micros(v) for v in pytimes],
+                       dtype='int64')
+        arr = np.array([_pytime_to_micros(v) for v in pytimes],
+                       dtype='int64')
 
-    def test_column_of_lists_chunked2(self):
-        data1 = [[0, 1], [2, 3], [4, 5], [6, 7], [10, 11],
-                 [12, 13], [14, 15], [16, 17]]
-        data2 = [[8, 9], [18, 19]]
+        null_mask = np.array([False, False, True], dtype=bool)
 
-        a1 = pa.array(data1)
-        a2 = pa.array(data2)
+        a1 = pa.array(arr, mask=null_mask, type=pa.time64('us'))
+        a2 = pa.array(arr * 1000, mask=null_mask,
+                      type=pa.time64('ns'))
 
-        t1 = pa.Table.from_arrays([a1], names=['a'])
-        t2 = pa.Table.from_arrays([a2], names=['a'])
+        a3 = pa.array((arr / 1000).astype('i4'), mask=null_mask,
+                      type=pa.time32('ms'))
+        a4 = pa.array((arr / 1000000).astype('i4'), mask=null_mask,
+                      type=pa.time32('s'))
 
-        concatenated = pa.concat_tables([t1, t2])
+        names = ['time64[us]', 'time64[ns]', 'time32[ms]', 'time32[s]']
+        batch = pa.RecordBatch.from_arrays([a1, a2, a3, a4], names)
+        arr = a1.to_pandas()
+        assert (arr == expected).all()
 
-        result = concatenated.to_pandas()
-        expected = pd.DataFrame({'a': data1 + data2})
+        arr = a2.to_pandas()
+        assert (arr == expected).all()
 
-        tm.assert_frame_equal(result, expected)
+        arr = a3.to_pandas()
+        assert (arr == expected_ms).all()
 
-    def test_column_of_lists_strided(self):
-        df, schema = dataframe_with_lists()
-        df = pd.concat([df] * 6, ignore_index=True)
+        arr = a4.to_pandas()
+        assert (arr == expected_s).all()
 
-        arr = df['int64'].values[::3]
-        assert arr.strides[0] != 8
+        df = batch.to_pandas()
+        expected_df = pd.DataFrame({'time64[us]': expected,
+                                    'time64[ns]': expected,
+                                    'time32[ms]': expected_ms,
+                                    'time32[s]': expected_s},
+                                   columns=names)
 
-        _check_array_roundtrip(arr)
+        tm.assert_frame_equal(df, expected_df)
 
-    def test_nested_lists_all_none(self):
-        data = np.array([[None, None], None], dtype=object)
+    def test_numpy_datetime64_columns(self):
+        datetime64_ns = np.array([
+                '2007-07-13T01:23:34.123456789',
+                None,
+                '2006-01-13T12:34:56.432539784',
+                '2010-08-13T05:46:57.437699912'],
+                dtype='datetime64[ns]')
+        _check_array_from_pandas_roundtrip(datetime64_ns)
 
-        arr = pa.array(data)
-        expected = pa.array(list(data))
-        assert arr.equals(expected)
-        assert arr.type == pa.list_(pa.null())
+        datetime64_us = np.array([
+                '2007-07-13T01:23:34.123456',
+                None,
+                '2006-01-13T12:34:56.432539',
+                '2010-08-13T05:46:57.437699'],
+                dtype='datetime64[us]')
+        _check_array_from_pandas_roundtrip(datetime64_us)
 
-        data2 = np.array([None, None, [None, None],
-                          np.array([None, None], dtype=object)],
-                         dtype=object)
-        arr = pa.array(data2)
-        expected = pa.array([None, None, [None, None], [None, None]])
-        assert arr.equals(expected)
+        datetime64_ms = np.array([
+                '2007-07-13T01:23:34.123',
+                None,
+                '2006-01-13T12:34:56.432',
+                '2010-08-13T05:46:57.437'],
+                dtype='datetime64[ms]')
+        _check_array_from_pandas_roundtrip(datetime64_ms)
 
-    def test_nested_lists_all_empty(self):
-        # ARROW-2128
-        data = pd.Series([[], [], []])
-        arr = pa.array(data)
-        expected = pa.array(list(data))
-        assert arr.equals(expected)
-        assert arr.type == pa.list_(pa.null())
+        datetime64_s = np.array([
+                '2007-07-13T01:23:34',
+                None,
+                '2006-01-13T12:34:56',
+                '2010-08-13T05:46:57'],
+                dtype='datetime64[s]')
+        _check_array_from_pandas_roundtrip(datetime64_s)
 
-    def test_threaded_conversion(self):
-        df = _alltypes_example()
-        _check_pandas_roundtrip(df, nthreads=2)
-        _check_pandas_roundtrip(df, nthreads=2, as_batch=True)
+    def test_numpy_datetime64_day_unit(self):
+        datetime64_d = np.array([
+                '2007-07-13',
+                None,
+                '2006-01-15',
+                '2010-08-19'],
+                dtype='datetime64[D]')
+        _check_array_from_pandas_roundtrip(datetime64_d)
 
-    def test_category(self):
-        repeats = 5
-        v1 = ['foo', None, 'bar', 'qux', np.nan]
-        v2 = [4, 5, 6, 7, 8]
-        v3 = [b'foo', None, b'bar', b'qux', np.nan]
-        df = pd.DataFrame({'cat_strings': pd.Categorical(v1 * repeats),
-                           'cat_ints': pd.Categorical(v2 * repeats),
-                           'cat_binary': pd.Categorical(v3 * repeats),
-                           'cat_strings_ordered': pd.Categorical(
-                               v1 * repeats, categories=['bar', 'qux', 'foo'],
-                               ordered=True),
-                           'ints': v2 * repeats,
-                           'ints2': v2 * repeats,
-                           'strings': v1 * repeats,
-                           'strings2': v1 * repeats,
-                           'strings3': v3 * repeats})
-        _check_pandas_roundtrip(df)
+    def test_array_from_pandas_date_with_mask(self):
+        m = np.array([True, False, True])
+        data = pd.Series([
+            date(1990, 1, 1),
+            date(1991, 1, 1),
+            date(1992, 1, 1)
+        ])
 
-        arrays = [
-            pd.Categorical(v1 * repeats),
-            pd.Categorical(v2 * repeats),
-            pd.Categorical(v3 * repeats)
-        ]
-        for values in arrays:
-            _check_array_roundtrip(values)
+        result = pa.Array.from_pandas(data, mask=m)
 
-    def test_mixed_types_fails(self):
-        data = pd.DataFrame({'a': ['a', 1, 2.0]})
-        with pytest.raises(pa.ArrowException):
-            pa.Table.from_pandas(data)
+        expected = pd.Series([None, date(1991, 1, 1), None])
+        assert pa.Array.from_pandas(expected).equals(result)
 
-        data = pd.DataFrame({'a': [1, True]})
-        with pytest.raises(pa.ArrowException):
-            pa.Table.from_pandas(data)
 
-    def test_strided_data_import(self):
-        cases = []
+class TestConvertStringLikeTypes(object):
+    """
+    Conversion tests for string and binary types.
+    """
 
-        columns = ['a', 'b', 'c']
-        N, K = 100, 3
-        random_numbers = np.random.randn(N, K).copy() * 100
+    def test_unicode(self):
+        repeats = 1000
+        values = [u'foo', None, u'bar', u'mañana', np.nan]
+        df = pd.DataFrame({'strings': values * repeats})
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
 
-        numeric_dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8',
-                          'f4', 'f8']
+        _check_pandas_roundtrip(df, expected_schema=schema)
 
-        for type_name in numeric_dtypes:
-            cases.append(random_numbers.astype(type_name))
+    def test_bytes_to_binary(self):
+        values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan]
+        df = pd.DataFrame({'strings': values})
 
-        # strings
-        cases.append(np.array([tm.rands(10) for i in range(N * K)],
-                              dtype=object)
-                     .reshape(N, K).copy())
+        table = pa.Table.from_pandas(df)
+        assert table[0].type == pa.binary()
 
-        # booleans
-        boolean_objects = (np.array([True, False, True] * N, dtype=object)
-                           .reshape(N, K).copy())
+        values2 = [b'qux', b'foo', None, b'bar', b'qux', np.nan]
+        expected = pd.DataFrame({'strings': values2})
+        _check_pandas_roundtrip(df, expected)
 
-        # add some nulls, so dtype comes back as objects
-        boolean_objects[5] = None
-        cases.append(boolean_objects)
+    @pytest.mark.large_memory
+    def test_bytes_exceed_2gb(self):
+        val = 'x' * (1 << 20)
+        df = pd.DataFrame({
+            'strings': np.array([val] * 4000, dtype=object)
+        })
+        arr = pa.array(df['strings'])
+        assert isinstance(arr, pa.ChunkedArray)
+        assert arr.num_chunks == 2
+        arr = None
 
-        cases.append(np.arange("2016-01-01T00:00:00.001", N * K,
-                               dtype='datetime64[ms]')
-                     .reshape(N, K).copy())
+        table = pa.Table.from_pandas(df)
+        assert table[0].data.num_chunks == 2
 
-        strided_mask = (random_numbers > 0).astype(bool)[:, 0]
+    def test_fixed_size_bytes(self):
+        values = [b'foo', None, b'bar', None, None, b'hey']
+        df = pd.DataFrame({'strings': values})
+        schema = pa.schema([pa.field('strings', pa.binary(3))])
+        table = pa.Table.from_pandas(df, schema=schema)
+        assert table.schema[0].type == schema[0].type
+        assert table.schema[0].name == schema[0].name
+        result = table.to_pandas()
+        tm.assert_frame_equal(result, df)
 
-        for case in cases:
-            df = pd.DataFrame(case, columns=columns)
-            col = df['a']
+    def test_fixed_size_bytes_does_not_accept_varying_lengths(self):
+        values = [b'foo', None, b'ba', None, None, b'hey']
+        df = pd.DataFrame({'strings': values})
+        schema = pa.schema([pa.field('strings', pa.binary(3))])
+        with pytest.raises(pa.ArrowInvalid):
+            pa.Table.from_pandas(df, schema=schema)
 
-            _check_pandas_roundtrip(df)
-            _check_array_roundtrip(col)
-            _check_array_roundtrip(col, mask=strided_mask)
+    def test_table_empty_str(self):
+        values = ['', '', '', '', '']
+        df = pd.DataFrame({'strings': values})
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
+        table = pa.Table.from_pandas(df, schema=schema)
+
+        result1 = table.to_pandas(strings_to_categorical=False)
+        expected1 = pd.DataFrame({'strings': values})
+        tm.assert_frame_equal(result1, expected1, check_dtype=True)
+
+        result2 = table.to_pandas(strings_to_categorical=True)
+        expected2 = pd.DataFrame({'strings': pd.Categorical(values)})
+        tm.assert_frame_equal(result2, expected2, check_dtype=True)
+
+    def test_table_str_to_categorical_without_na(self):
+        values = ['a', 'a', 'b', 'b', 'c']
+        df = pd.DataFrame({'strings': values})
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
+        table = pa.Table.from_pandas(df, schema=schema)
+
+        result = table.to_pandas(strings_to_categorical=True)
+        expected = pd.DataFrame({'strings': pd.Categorical(values)})
+        tm.assert_frame_equal(result, expected, check_dtype=True)
+
+        with pytest.raises(pa.ArrowInvalid):
+            table.to_pandas(strings_to_categorical=True,
+                            zero_copy_only=True)
+
+    def test_table_str_to_categorical_with_na(self):
+        values = [None, 'a', 'b', np.nan]
+        df = pd.DataFrame({'strings': values})
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
+        table = pa.Table.from_pandas(df, schema=schema)
+
+        result = table.to_pandas(strings_to_categorical=True)
+        expected = pd.DataFrame({'strings': pd.Categorical(values)})
+        tm.assert_frame_equal(result, expected, check_dtype=True)
+
+        with pytest.raises(pa.ArrowInvalid):
+            table.to_pandas(strings_to_categorical=True,
+                            zero_copy_only=True)
+
+
+class TestConvertDecimalTypes(object):
+    """
+    Conversion test for decimal types.
+    """
 
     def test_decimal_32_from_pandas(self):
         expected = pd.DataFrame({
@@ -1057,201 +1132,131 @@ def test_decimal_128_to_pandas(self):
         df = converted.to_pandas()
         tm.assert_frame_equal(df, expected)
 
-    def test_pytime_from_pandas(self):
-        pytimes = [time(1, 2, 3, 1356),
-                   time(4, 5, 6, 1356)]
-
-        # microseconds
-        t1 = pa.time64('us')
 
-        aobjs = np.array(pytimes + [None], dtype=object)
-        parr = pa.array(aobjs)
-        assert parr.type == t1
-        assert parr[0].as_py() == pytimes[0]
-        assert parr[1].as_py() == pytimes[1]
-        assert parr[2] is pa.NA
+class TestListTypes(object):
+    """
+    Conversion tests for list<> types.
+    """
 
-        # DataFrame
-        df = pd.DataFrame({'times': aobjs})
-        batch = pa.RecordBatch.from_pandas(df)
-        assert batch[0].equals(parr)
+    def test_column_of_arrays(self):
+        df, schema = dataframe_with_arrays()
+        _check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
+        table = pa.Table.from_pandas(df, schema=schema, preserve_index=False)
+        assert table.schema.equals(schema)
 
-        # Test ndarray of int64 values
-        arr = np.array([_pytime_to_micros(v) for v in pytimes],
-                       dtype='int64')
+        for column in df.columns:
+            field = schema.field_by_name(column)
+            _check_array_roundtrip(df[column], type=field.type)
 
-        a1 = pa.array(arr, type=pa.time64('us'))
-        assert a1[0].as_py() == pytimes[0]
+    def test_column_of_arrays_to_py(self):
+        # Test regression in ARROW-1199 not caught in above test
+        dtype = 'i1'
+        arr = np.array([
+            np.arange(10, dtype=dtype),
+            np.arange(5, dtype=dtype),
+            None,
+            np.arange(1, dtype=dtype)
+        ])
+        type_ = pa.list_(pa.int8())
+        parr = pa.array(arr, type=type_)
 
-        a2 = pa.array(arr * 1000, type=pa.time64('ns'))
-        assert a2[0].as_py() == pytimes[0]
+        assert parr[0].as_py() == list(range(10))
+        assert parr[1].as_py() == list(range(5))
+        assert parr[2].as_py() is None
+        assert parr[3].as_py() == [0]
 
-        a3 = pa.array((arr / 1000).astype('i4'),
-                      type=pa.time32('ms'))
-        assert a3[0].as_py() == pytimes[0].replace(microsecond=1000)
-
-        a4 = pa.array((arr / 1000000).astype('i4'),
-                      type=pa.time32('s'))
-        assert a4[0].as_py() == pytimes[0].replace(microsecond=0)
-
-    def test_arrow_time_to_pandas(self):
-        pytimes = [time(1, 2, 3, 1356),
-                   time(4, 5, 6, 1356),
-                   time(0, 0, 0)]
-
-        expected = np.array(pytimes[:2] + [None])
-        expected_ms = np.array([x.replace(microsecond=1000)
-                                for x in pytimes[:2]] +
-                               [None])
-        expected_s = np.array([x.replace(microsecond=0)
-                               for x in pytimes[:2]] +
-                              [None])
-
-        arr = np.array([_pytime_to_micros(v) for v in pytimes],
-                       dtype='int64')
-        arr = np.array([_pytime_to_micros(v) for v in pytimes],
-                       dtype='int64')
-
-        null_mask = np.array([False, False, True], dtype=bool)
-
-        a1 = pa.array(arr, mask=null_mask, type=pa.time64('us'))
-        a2 = pa.array(arr * 1000, mask=null_mask,
-                      type=pa.time64('ns'))
-
-        a3 = pa.array((arr / 1000).astype('i4'), mask=null_mask,
-                      type=pa.time32('ms'))
-        a4 = pa.array((arr / 1000000).astype('i4'), mask=null_mask,
-                      type=pa.time32('s'))
-
-        names = ['time64[us]', 'time64[ns]', 'time32[ms]', 'time32[s]']
-        batch = pa.RecordBatch.from_arrays([a1, a2, a3, a4], names)
-        arr = a1.to_pandas()
-        assert (arr == expected).all()
-
-        arr = a2.to_pandas()
-        assert (arr == expected).all()
-
-        arr = a3.to_pandas()
-        assert (arr == expected_ms).all()
-
-        arr = a4.to_pandas()
-        assert (arr == expected_s).all()
+    def test_column_of_lists(self):
+        df, schema = dataframe_with_lists()
+        _check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
+        table = pa.Table.from_pandas(df, schema=schema, preserve_index=False)
+        assert table.schema.equals(schema)
 
-        df = batch.to_pandas()
-        expected_df = pd.DataFrame({'time64[us]': expected,
-                                    'time64[ns]': expected,
-                                    'time32[ms]': expected_ms,
-                                    'time32[s]': expected_s},
-                                   columns=names)
+        for column in df.columns:
+            field = schema.field_by_name(column)
+            _check_array_roundtrip(df[column], type=field.type)
 
-        tm.assert_frame_equal(df, expected_df)
+    def test_column_of_lists_first_empty(self):
+        # ARROW-2124
+        num_lists = [[], [2, 3, 4], [3, 6, 7, 8], [], [2]]
+        series = pd.Series([np.array(s, dtype=float) for s in num_lists])
+        arr = pa.array(series)
+        result = pd.Series(arr.to_pandas())
+        tm.assert_series_equal(result, series)
 
-    def test_numpy_datetime64_columns(self):
-        datetime64_ns = np.array([
-                '2007-07-13T01:23:34.123456789',
+    def test_column_of_lists_chunked(self):
+        # ARROW-1357
+        df = pd.DataFrame({
+            'lists': np.array([
+                [1, 2],
                 None,
-                '2006-01-13T12:34:56.432539784',
-                '2010-08-13T05:46:57.437699912'],
-                dtype='datetime64[ns]')
-        _check_array_from_pandas_roundtrip(datetime64_ns)
+                [2, 3],
+                [4, 5],
+                [6, 7],
+                [8, 9]
+            ], dtype=object)
+        })
 
-        datetime64_us = np.array([
-                '2007-07-13T01:23:34.123456',
-                None,
-                '2006-01-13T12:34:56.432539',
-                '2010-08-13T05:46:57.437699'],
-                dtype='datetime64[us]')
-        _check_array_from_pandas_roundtrip(datetime64_us)
+        schema = pa.schema([
+            pa.field('lists', pa.list_(pa.int64()))
+        ])
 
-        datetime64_ms = np.array([
-                '2007-07-13T01:23:34.123',
-                None,
-                '2006-01-13T12:34:56.432',
-                '2010-08-13T05:46:57.437'],
-                dtype='datetime64[ms]')
-        _check_array_from_pandas_roundtrip(datetime64_ms)
+        t1 = pa.Table.from_pandas(df[:2], schema=schema)
+        t2 = pa.Table.from_pandas(df[2:], schema=schema)
 
-        datetime64_s = np.array([
-                '2007-07-13T01:23:34',
-                None,
-                '2006-01-13T12:34:56',
-                '2010-08-13T05:46:57'],
-                dtype='datetime64[s]')
-        _check_array_from_pandas_roundtrip(datetime64_s)
+        table = pa.concat_tables([t1, t2])
+        result = table.to_pandas()
 
-    def test_numpy_datetime64_day_unit(self):
-        datetime64_d = np.array([
-                '2007-07-13',
-                None,
-                '2006-01-15',
-                '2010-08-19'],
-                dtype='datetime64[D]')
-        _check_array_from_pandas_roundtrip(datetime64_d)
+        tm.assert_frame_equal(result, df)
 
-    def test_all_nones(self):
-        def _check_series(s):
-            converted = pa.array(s)
-            assert isinstance(converted, pa.NullArray)
-            assert len(converted) == 3
-            assert converted.null_count == 3
-            assert converted[0] is pa.NA
+    def test_column_of_lists_chunked2(self):
+        data1 = [[0, 1], [2, 3], [4, 5], [6, 7], [10, 11],
+                 [12, 13], [14, 15], [16, 17]]
+        data2 = [[8, 9], [18, 19]]
 
-        _check_series(pd.Series([None] * 3, dtype=object))
-        _check_series(pd.Series([np.nan] * 3, dtype=object))
-        _check_series(pd.Series([np.sqrt(-1)] * 3, dtype=object))
+        a1 = pa.array(data1)
+        a2 = pa.array(data2)
 
-    def test_multiindex_duplicate_values(self):
-        num_rows = 3
-        numbers = list(range(num_rows))
-        index = pd.MultiIndex.from_arrays(
-            [['foo', 'foo', 'bar'], numbers],
-            names=['foobar', 'some_numbers'],
-        )
+        t1 = pa.Table.from_arrays([a1], names=['a'])
+        t2 = pa.Table.from_arrays([a2], names=['a'])
 
-        df = pd.DataFrame({'numbers': numbers}, index=index)
+        concatenated = pa.concat_tables([t1, t2])
 
-        table = pa.Table.from_pandas(df)
-        result_df = table.to_pandas()
-        tm.assert_frame_equal(result_df, df)
+        result = concatenated.to_pandas()
+        expected = pd.DataFrame({'a': data1 + data2})
 
-    def test_partial_schema(self):
-        data = OrderedDict([
-            ('a', [0, 1, 2, 3, 4]),
-            ('b', np.array([-10, -5, 0, 5, 10], dtype=np.int32)),
-            ('c', [-10, -5, 0, 5, 10])
-        ])
-        df = pd.DataFrame(data)
+        tm.assert_frame_equal(result, expected)
 
-        partial_schema = pa.schema([
-            pa.field('a', pa.int64()),
-            pa.field('b', pa.int32())
-        ])
+    def test_column_of_lists_strided(self):
+        df, schema = dataframe_with_lists()
+        df = pd.concat([df] * 6, ignore_index=True)
 
-        expected_schema = pa.schema([
-            pa.field('a', pa.int64()),
-            pa.field('b', pa.int32()),
-            pa.field('c', pa.int64())
-        ])
+        arr = df['int64'].values[::3]
+        assert arr.strides[0] != 8
 
-        _check_pandas_roundtrip(df, schema=partial_schema,
-                                expected_schema=expected_schema)
+        _check_array_roundtrip(arr)
 
-    def test_structarray(self):
-        ints = pa.array([None, 2, 3], type=pa.int64())
-        strs = pa.array([u'a', None, u'c'], type=pa.string())
-        bools = pa.array([True, False, None], type=pa.bool_())
-        arr = pa.StructArray.from_arrays(
-            [ints, strs, bools],
-            ['ints', 'strs', 'bools'])
+    def test_nested_lists_all_none(self):
+        data = np.array([[None, None], None], dtype=object)
 
-        expected = pd.Series([
-            {'ints': None, 'strs': u'a', 'bools': True},
-            {'ints': 2, 'strs': None, 'bools': False},
-            {'ints': 3, 'strs': u'c', 'bools': None},
-        ])
+        arr = pa.array(data)
+        expected = pa.array(list(data))
+        assert arr.equals(expected)
+        assert arr.type == pa.list_(pa.null())
 
-        series = pd.Series(arr.to_pandas())
-        tm.assert_series_equal(series, expected)
+        data2 = np.array([None, None, [None, None],
+                          np.array([None, None], dtype=object)],
+                         dtype=object)
+        arr = pa.array(data2)
+        expected = pa.array([None, None, [None, None], [None, None]])
+        assert arr.equals(expected)
+
+    def test_nested_lists_all_empty(self):
+        # ARROW-2128
+        data = pd.Series([[], [], []])
+        arr = pa.array(data)
+        expected = pa.array(list(data))
+        assert arr.equals(expected)
+        assert arr.type == pa.list_(pa.null())
 
     def test_infer_lists(self):
         data = OrderedDict([
@@ -1285,242 +1290,288 @@ def test_infer_numpy_array(self):
 
         _check_pandas_roundtrip(df, expected_schema=expected_schema)
 
-    def test_metadata_with_mixed_types(self):
-        df = pd.DataFrame({'data': [b'some_bytes', u'some_unicode']})
-        table = pa.Table.from_pandas(df)
-        metadata = table.schema.metadata
-        assert b'mixed' not in metadata[b'pandas']
+    @pytest.mark.parametrize('t,data,expected', [
+        (
+            pa.int64,
+            [[1, 2], [3], None],
+            [None, [3], None]
+        ),
+        (
+            pa.string,
+            [[u'aaa', u'bb'], [u'c'], None],
+            [None, [u'c'], None]
+        ),
+        (
+            pa.null,
+            [[None, None], [None], None],
+            [None, [None], None]
+        )
+    ])
+    def test_array_from_pandas_typed_array_with_mask(self, t, data, expected):
+        m = np.array([True, False, True])
 
-        js = json.loads(metadata[b'pandas'].decode('utf8'))
-        data_column = js['columns'][0]
-        assert data_column['pandas_type'] == 'bytes'
-        assert data_column['numpy_type'] == 'object'
+        s = pd.Series(data)
+        result = pa.Array.from_pandas(s, mask=m, type=pa.list_(t()))
 
-    def test_list_metadata(self):
-        df = pd.DataFrame({'data': [[1], [2, 3, 4], [5] * 7]})
-        schema = pa.schema([pa.field('data', type=pa.list_(pa.int64()))])
-        table = pa.Table.from_pandas(df, schema=schema)
-        metadata = table.schema.metadata
-        assert b'mixed' not in metadata[b'pandas']
+        assert pa.Array.from_pandas(expected,
+                                    type=pa.list_(t())).equals(result)
 
-        js = json.loads(metadata[b'pandas'].decode('utf8'))
-        data_column = js['columns'][0]
-        assert data_column['pandas_type'] == 'list[int64]'
-        assert data_column['numpy_type'] == 'object'
+    def test_empty_list_roundtrip(self):
+        empty_list_array = np.empty((3,), dtype=object)
+        empty_list_array.fill([])
 
-    def test_decimal_metadata(self):
-        expected = pd.DataFrame({
-            'decimals': [
-                decimal.Decimal('394092382910493.12341234678'),
-                -decimal.Decimal('314292388910493.12343437128'),
-            ]
-        })
-        table = pa.Table.from_pandas(expected)
-        metadata = table.schema.metadata
-        assert b'mixed' not in metadata[b'pandas']
+        df = pd.DataFrame({'a': np.array(['1', '2', '3']),
+                           'b': empty_list_array})
+        tbl = pa.Table.from_pandas(df)
 
-        js = json.loads(metadata[b'pandas'].decode('utf8'))
-        data_column = js['columns'][0]
-        assert data_column['pandas_type'] == 'decimal'
-        assert data_column['numpy_type'] == 'object'
-        assert data_column['metadata'] == {'precision': 26, 'scale': 11}
+        result = tbl.to_pandas()
 
-    def test_table_empty_str(self):
-        values = ['', '', '', '', '']
-        df = pd.DataFrame({'strings': values})
-        field = pa.field('strings', pa.string())
-        schema = pa.schema([field])
-        table = pa.Table.from_pandas(df, schema=schema)
+        tm.assert_frame_equal(result, df)
 
-        result1 = table.to_pandas(strings_to_categorical=False)
-        expected1 = pd.DataFrame({'strings': values})
-        tm.assert_frame_equal(result1, expected1, check_dtype=True)
 
-        result2 = table.to_pandas(strings_to_categorical=True)
-        expected2 = pd.DataFrame({'strings': pd.Categorical(values)})
-        tm.assert_frame_equal(result2, expected2, check_dtype=True)
+class TestConvertStructTypes(object):
+    """
+    Conversion tests for struct types.
+    """
 
-    def test_table_str_to_categorical_without_na(self):
-        values = ['a', 'a', 'b', 'b', 'c']
-        df = pd.DataFrame({'strings': values})
-        field = pa.field('strings', pa.string())
-        schema = pa.schema([field])
-        table = pa.Table.from_pandas(df, schema=schema)
+    def test_structarray(self):
+        ints = pa.array([None, 2, 3], type=pa.int64())
+        strs = pa.array([u'a', None, u'c'], type=pa.string())
+        bools = pa.array([True, False, None], type=pa.bool_())
+        arr = pa.StructArray.from_arrays(
+            [ints, strs, bools],
+            ['ints', 'strs', 'bools'])
 
-        result = table.to_pandas(strings_to_categorical=True)
-        expected = pd.DataFrame({'strings': pd.Categorical(values)})
-        tm.assert_frame_equal(result, expected, check_dtype=True)
+        expected = pd.Series([
+            {'ints': None, 'strs': u'a', 'bools': True},
+            {'ints': 2, 'strs': None, 'bools': False},
+            {'ints': 3, 'strs': u'c', 'bools': None},
+        ])
 
-        with pytest.raises(pa.ArrowInvalid):
-            table.to_pandas(strings_to_categorical=True,
-                            zero_copy_only=True)
+        series = pd.Series(arr.to_pandas())
+        tm.assert_series_equal(series, expected)
 
-    def test_table_str_to_categorical_with_na(self):
-        values = [None, 'a', 'b', np.nan]
-        df = pd.DataFrame({'strings': values})
-        field = pa.field('strings', pa.string())
-        schema = pa.schema([field])
-        table = pa.Table.from_pandas(df, schema=schema)
 
-        result = table.to_pandas(strings_to_categorical=True)
-        expected = pd.DataFrame({'strings': pd.Categorical(values)})
-        tm.assert_frame_equal(result, expected, check_dtype=True)
+class TestZeroCopyConversion(object):
+    """
+    Tests that zero-copy conversion works with some types.
+    """
 
-        with pytest.raises(pa.ArrowInvalid):
-            table.to_pandas(strings_to_categorical=True,
-                            zero_copy_only=True)
+    def test_zero_copy_success(self):
+        result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True)
+        npt.assert_array_equal(result, [0, 1, 2])
 
-    def test_table_batch_empty_dataframe(self):
-        df = pd.DataFrame({})
+    def test_zero_copy_dictionaries(self):
+        arr = pa.DictionaryArray.from_arrays(
+            np.array([0, 0]),
+            np.array([5]))
+
+        result = arr.to_pandas(zero_copy_only=True)
+        values = pd.Categorical([5, 5])
+
+        tm.assert_series_equal(pd.Series(result), pd.Series(values),
+                               check_names=False)
+
+    def test_zero_copy_failure_on_object_types(self):
+        with pytest.raises(pa.ArrowException):
+            pa.array(['A', 'B', 'C']).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_with_int_when_nulls(self):
+        with pytest.raises(pa.ArrowException):
+            pa.array([0, 1, None]).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_with_float_when_nulls(self):
+        with pytest.raises(pa.ArrowException):
+            pa.array([0.0, 1.0, None]).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_on_bool_types(self):
+        with pytest.raises(pa.ArrowException):
+            pa.array([True, False]).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_on_list_types(self):
+        arr = np.array([[1, 2], [8, 9]], dtype=object)
+
+        with pytest.raises(pa.ArrowException):
+            pa.array(arr).to_pandas(zero_copy_only=True)
+
+    def test_zero_copy_failure_on_timestamp_types(self):
+        arr = np.array(['2007-07-13'], dtype='datetime64[ns]')
+
+        with pytest.raises(pa.ArrowException):
+            pa.array(arr).to_pandas(zero_copy_only=True)
+
+
+class TestConvertMisc(object):
+    """
+    Miscellaneous conversion tests.
+    """
+
+    type_pairs = [
+        (np.int8, pa.int8()),
+        (np.int16, pa.int16()),
+        (np.int32, pa.int32()),
+        (np.int64, pa.int64()),
+        (np.uint8, pa.uint8()),
+        (np.uint16, pa.uint16()),
+        (np.uint32, pa.uint32()),
+        (np.uint64, pa.uint64()),
+        # (np.float16, pa.float16()),  # XXX unsupported
+        (np.float32, pa.float32()),
+        (np.float64, pa.float64()),
+        # XXX unsupported
+        # (np.dtype([('a', 'i2')]), pa.struct([pa.field('a', pa.int16())])),
+        (np.object, pa.string()),
+        # (np.object, pa.binary()),  # XXX unsupported
+        (np.object, pa.binary(10)),
+        (np.object, pa.list_(pa.int64())),
+        ]
+
+    def test_all_none_objects(self):
+        df = pd.DataFrame({'a': [None, None, None]})
         _check_pandas_roundtrip(df)
-        _check_pandas_roundtrip(df, as_batch=True)
 
-        df2 = pd.DataFrame({}, index=[0, 1, 2])
-        _check_pandas_roundtrip(df2, preserve_index=True)
-        _check_pandas_roundtrip(df2, as_batch=True, preserve_index=True)
+    def test_all_none_category(self):
+        df = pd.DataFrame({'a': [None, None, None]})
+        df['a'] = df['a'].astype('category')
+        _check_pandas_roundtrip(df)
 
-    def test_convert_empty_table(self):
-        arr = pa.array([], type=pa.int64())
-        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=np.int64))
-        arr = pa.array([], type=pa.string())
-        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
-        arr = pa.array([], type=pa.list_(pa.int64()))
-        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
-        arr = pa.array([], type=pa.struct([pa.field('a', pa.int64())]))
-        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
+    def test_empty_arrays(self):
+        for dtype, pa_type in self.type_pairs:
+            arr = np.array([], dtype=dtype)
+            _check_array_roundtrip(arr, type=pa_type)
 
-    def test_array_from_pandas_date_with_mask(self):
-        m = np.array([True, False, True])
-        data = pd.Series([
-            date(1990, 1, 1),
-            date(1991, 1, 1),
-            date(1992, 1, 1)
-        ])
+    def test_threaded_conversion(self):
+        df = _alltypes_example()
+        _check_pandas_roundtrip(df, nthreads=2)
+        _check_pandas_roundtrip(df, nthreads=2, as_batch=True)
 
-        result = pa.Array.from_pandas(data, mask=m)
+    def test_category(self):
+        repeats = 5
+        v1 = ['foo', None, 'bar', 'qux', np.nan]
+        v2 = [4, 5, 6, 7, 8]
+        v3 = [b'foo', None, b'bar', b'qux', np.nan]
+        df = pd.DataFrame({'cat_strings': pd.Categorical(v1 * repeats),
+                           'cat_ints': pd.Categorical(v2 * repeats),
+                           'cat_binary': pd.Categorical(v3 * repeats),
+                           'cat_strings_ordered': pd.Categorical(
+                               v1 * repeats, categories=['bar', 'qux', 'foo'],
+                               ordered=True),
+                           'ints': v2 * repeats,
+                           'ints2': v2 * repeats,
+                           'strings': v1 * repeats,
+                           'strings2': v1 * repeats,
+                           'strings3': v3 * repeats})
+        _check_pandas_roundtrip(df)
 
-        expected = pd.Series([None, date(1991, 1, 1), None])
-        assert pa.Array.from_pandas(expected).equals(result)
+        arrays = [
+            pd.Categorical(v1 * repeats),
+            pd.Categorical(v2 * repeats),
+            pd.Categorical(v3 * repeats)
+        ]
+        for values in arrays:
+            _check_array_roundtrip(values)
 
-    @pytest.mark.parametrize('t,data,expected', [
-        (
-            pa.int64,
-            [[1, 2], [3], None],
-            [None, [3], None]
-        ),
-        (
-            pa.string,
-            [[u'aaa', u'bb'], [u'c'], None],
-            [None, [u'c'], None]
-        ),
-        (
-            pa.null,
-            [[None, None], [None], None],
-            [None, [None], None]
-        )
-    ])
-    def test_array_from_pandas_typed_array_with_mask(self, t, data, expected):
-        m = np.array([True, False, True])
+    def test_mixed_types_fails(self):
+        data = pd.DataFrame({'a': ['a', 1, 2.0]})
+        with pytest.raises(pa.ArrowException):
+            pa.Table.from_pandas(data)
 
-        s = pd.Series(data)
-        result = pa.Array.from_pandas(s, mask=m, type=pa.list_(t()))
+        data = pd.DataFrame({'a': [1, True]})
+        with pytest.raises(pa.ArrowException):
+            pa.Table.from_pandas(data)
 
-        assert pa.Array.from_pandas(expected,
-                                    type=pa.list_(t())).equals(result)
+    def test_strided_data_import(self):
+        cases = []
 
-    def test_table_column_subset_metadata(self):
-        # ARROW-1883
-        df = pd.DataFrame({
-            'a': [1, 2, 3],
-            'b': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')})
-        table = pa.Table.from_pandas(df)
+        columns = ['a', 'b', 'c']
+        N, K = 100, 3
+        random_numbers = np.random.randn(N, K).copy() * 100
 
-        table_subset = table.remove_column(1)
-        result = table_subset.to_pandas()
-        tm.assert_frame_equal(result, df[['a']])
+        numeric_dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8',
+                          'f4', 'f8']
 
-        table_subset2 = table_subset.remove_column(1)
-        result = table_subset2.to_pandas()
-        tm.assert_frame_equal(result, df[['a']])
+        for type_name in numeric_dtypes:
+            cases.append(random_numbers.astype(type_name))
 
-        # non-default index
-        for index in [
-                pd.Index(['a', 'b', 'c'], name='index'),
-                pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')]:
-            df = pd.DataFrame({'a': [1, 2, 3],
-                               'b': [.1, .2, .3]}, index=index)
-            table = pa.Table.from_pandas(df)
+        # strings
+        cases.append(np.array([tm.rands(10) for i in range(N * K)],
+                              dtype=object)
+                     .reshape(N, K).copy())
 
-            table_subset = table.remove_column(1)
-            result = table_subset.to_pandas()
-            tm.assert_frame_equal(result, df[['a']])
+        # booleans
+        boolean_objects = (np.array([True, False, True] * N, dtype=object)
+                           .reshape(N, K).copy())
 
-            table_subset2 = table_subset.remove_column(1)
-            result = table_subset2.to_pandas()
-            tm.assert_frame_equal(result, df[['a']].reset_index(drop=True))
+        # add some nulls, so dtype comes back as objects
+        boolean_objects[5] = None
+        cases.append(boolean_objects)
 
-    def test_empty_list_roundtrip(self):
-        empty_list_array = np.empty((3,), dtype=object)
-        empty_list_array.fill([])
+        cases.append(np.arange("2016-01-01T00:00:00.001", N * K,
+                               dtype='datetime64[ms]')
+                     .reshape(N, K).copy())
 
-        df = pd.DataFrame({'a': np.array(['1', '2', '3']),
-                           'b': empty_list_array})
-        tbl = pa.Table.from_pandas(df)
+        strided_mask = (random_numbers > 0).astype(bool)[:, 0]
 
-        result = tbl.to_pandas()
+        for case in cases:
+            df = pd.DataFrame(case, columns=columns)
+            col = df['a']
 
-        tm.assert_frame_equal(result, df)
+            _check_pandas_roundtrip(df)
+            _check_array_roundtrip(col)
+            _check_array_roundtrip(col, mask=strided_mask)
 
-    def test_empty_list_metadata(self):
-        # Create table with array of empty lists, forced to have type
-        # list(string) in pyarrow
-        c1 = [["test"], ["a", "b"], None]
-        c2 = [[], [], []]
-        arrays = OrderedDict([
-            ('c1', pa.array(c1, type=pa.list_(pa.string()))),
-            ('c2', pa.array(c2, type=pa.list_(pa.string()))),
+    def test_all_nones(self):
+        def _check_series(s):
+            converted = pa.array(s)
+            assert isinstance(converted, pa.NullArray)
+            assert len(converted) == 3
+            assert converted.null_count == 3
+            assert converted[0] is pa.NA
+
+        _check_series(pd.Series([None] * 3, dtype=object))
+        _check_series(pd.Series([np.nan] * 3, dtype=object))
+        _check_series(pd.Series([np.sqrt(-1)] * 3, dtype=object))
+
+    def test_partial_schema(self):
+        data = OrderedDict([
+            ('a', [0, 1, 2, 3, 4]),
+            ('b', np.array([-10, -5, 0, 5, 10], dtype=np.int32)),
+            ('c', [-10, -5, 0, 5, 10])
         ])
-        rb = pa.RecordBatch.from_arrays(
-            list(arrays.values()),
-            list(arrays.keys())
-        )
-        tbl = pa.Table.from_batches([rb])
+        df = pd.DataFrame(data)
 
-        # First roundtrip changes schema, because pandas cannot preserve the
-        # type of empty lists
-        df = tbl.to_pandas()
-        tbl2 = pa.Table.from_pandas(df, preserve_index=True)
-        md2 = json.loads(tbl2.schema.metadata[b'pandas'].decode('utf8'))
+        partial_schema = pa.schema([
+            pa.field('a', pa.int64()),
+            pa.field('b', pa.int32())
+        ])
 
-        # Second roundtrip
-        df2 = tbl2.to_pandas()
-        expected = pd.DataFrame(OrderedDict([('c1', c1), ('c2', c2)]))
+        expected_schema = pa.schema([
+            pa.field('a', pa.int64()),
+            pa.field('b', pa.int32()),
+            pa.field('c', pa.int64())
+        ])
 
-        tm.assert_frame_equal(df2, expected)
+        _check_pandas_roundtrip(df, schema=partial_schema,
+                                expected_schema=expected_schema)
 
-        assert md2['columns'] == [
-            {
-                'name': 'c1',
-                'field_name': 'c1',
-                'metadata': None,
-                'numpy_type': 'object',
-                'pandas_type': 'list[unicode]',
-            },
-            {
-                'name': 'c2',
-                'field_name': 'c2',
-                'metadata': None,
-                'numpy_type': 'object',
-                'pandas_type': 'list[empty]',
-            },
-            {
-                'name': None,
-                'field_name': '__index_level_0__',
-                'metadata': None,
-                'numpy_type': 'int64',
-                'pandas_type': 'int64',
-            }
-        ]
+    def test_table_batch_empty_dataframe(self):
+        df = pd.DataFrame({})
+        _check_pandas_roundtrip(df)
+        _check_pandas_roundtrip(df, as_batch=True)
+
+        df2 = pd.DataFrame({}, index=[0, 1, 2])
+        _check_pandas_roundtrip(df2, preserve_index=True)
+        _check_pandas_roundtrip(df2, as_batch=True, preserve_index=True)
+
+    def test_convert_empty_table(self):
+        arr = pa.array([], type=pa.int64())
+        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=np.int64))
+        arr = pa.array([], type=pa.string())
+        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
+        arr = pa.array([], type=pa.list_(pa.int64()))
+        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
+        arr = pa.array([], type=pa.struct([pa.field('a', pa.int64())]))
+        tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
 
 
 def _fully_loaded_dataframe_example():

From 5dce01f76f0569d820089682aef32d87c806b61d Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 15 Feb 2018 10:09:53 -0500
Subject: [PATCH 1480/1644] ARROW-2155: [Python] frombuffer() should respect
 mutability of argument

Also raise TypeError instead of ValueError for invalid arguments to frombuffer()

Author: Antoine Pitrou <antoine@python.org>

Closes #1608 from pitrou/ARROW-2155-frombuffer-mutable2 and squashes the following commits:

b3a66364 [Antoine Pitrou] ARROW-2155: [Python] frombuffer() should respect mutability of argument
---
 cpp/src/arrow/python/common.cc       | 38 ++++++++++++++++++++--------
 cpp/src/arrow/python/common.h        | 13 ++++++----
 cpp/src/arrow/python/io.cc           | 19 +++-----------
 cpp/src/arrow/python/io.h            |  7 -----
 cpp/src/arrow/python/python-test.cc  |  9 ++++++-
 python/pyarrow/includes/libarrow.pxd |  6 ++---
 python/pyarrow/io.pxi                |  8 ++----
 python/pyarrow/tests/test_io.py      | 26 +++++++++++++++++--
 8 files changed, 76 insertions(+), 50 deletions(-)

diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
index 14a8ae6fd5b85..1ded88071fe76 100644
--- a/cpp/src/arrow/python/common.cc
+++ b/cpp/src/arrow/python/common.cc
@@ -23,6 +23,7 @@
 
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 
 namespace arrow {
 namespace py {
@@ -47,22 +48,39 @@ MemoryPool* get_memory_pool() {
 // ----------------------------------------------------------------------
 // PyBuffer
 
-PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0), obj_(nullptr) {
-  if (PyObject_CheckBuffer(obj)) {
-    obj_ = PyMemoryView_FromObject(obj);
-    Py_buffer* buffer = PyMemoryView_GET_BUFFER(obj_);
-    data_ = reinterpret_cast<const uint8_t*>(buffer->buf);
-    size_ = buffer->len;
-    capacity_ = buffer->len;
-    is_mutable_ = false;
+PyBuffer::PyBuffer() : Buffer(nullptr, 0) {}
+
+Status PyBuffer::Init(PyObject* obj) {
+  if (!PyObject_GetBuffer(obj, &py_buf_, PyBUF_ANY_CONTIGUOUS)) {
+    data_ = reinterpret_cast<const uint8_t*>(py_buf_.buf);
+    DCHECK(data_ != nullptr);
+    size_ = py_buf_.len;
+    capacity_ = py_buf_.len;
+    is_mutable_ = !py_buf_.readonly;
+    return Status::OK();
+  } else {
+    return Status(StatusCode::PythonError, "");
   }
 }
 
+Status PyBuffer::FromPyObject(PyObject* obj, std::shared_ptr<Buffer>* out) {
+  PyBuffer* buf = new PyBuffer();
+  std::shared_ptr<Buffer> res(buf);
+  RETURN_NOT_OK(buf->Init(obj));
+  *out = res;
+  return Status::OK();
+}
+
 PyBuffer::~PyBuffer() {
-  PyAcquireGIL lock;
-  Py_XDECREF(obj_);
+  if (data_ != nullptr) {
+    PyAcquireGIL lock;
+    PyBuffer_Release(&py_buf_);
+  }
 }
 
+// ----------------------------------------------------------------------
+// Python exception -> Status
+
 Status CheckPyError(StatusCode code) {
   if (PyErr_Occurred()) {
     PyObject* exc_type = nullptr;
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index b1e0888af63f6..269385c1a63b2 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -18,6 +18,7 @@
 #ifndef ARROW_PYTHON_COMMON_H
 #define ARROW_PYTHON_COMMON_H
 
+#include <memory>
 #include <string>
 
 #include "arrow/python/config.h"
@@ -140,15 +141,17 @@ ARROW_EXPORT MemoryPool* get_memory_pool();
 
 class ARROW_EXPORT PyBuffer : public Buffer {
  public:
-  /// Note that the GIL must be held when calling the PyBuffer constructor.
-  ///
-  /// While memoryview objects support multi-demensional buffers, PyBuffer only supports
+  /// While memoryview objects support multi-dimensional buffers, PyBuffer only supports
   /// one-dimensional byte buffers.
-  explicit PyBuffer(PyObject* obj);
   ~PyBuffer();
 
+  static Status FromPyObject(PyObject* obj, std::shared_ptr<Buffer>* out);
+
  private:
-  PyObject* obj_;
+  PyBuffer();
+  Status Init(PyObject*);
+
+  Py_buffer py_buf_;
 };
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index 2cff046085e69..801a325743c33 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -149,14 +149,11 @@ Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) {
 Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
   PyAcquireGIL lock;
 
-  PyObject* bytes_obj = NULL;
-  ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj));
-  DCHECK(bytes_obj != NULL);
-
-  *out = std::make_shared<PyBuffer>(bytes_obj);
-  Py_XDECREF(bytes_obj);
+  OwnedRef bytes_obj;
+  ARROW_RETURN_NOT_OK(file_->Read(nbytes, bytes_obj.ref()));
+  DCHECK(bytes_obj.obj() != NULL);
 
-  return Status::OK();
+  return PyBuffer::FromPyObject(bytes_obj.obj(), out);
 }
 
 Status PyReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
@@ -219,13 +216,5 @@ Status PyOutputStream::Write(const void* data, int64_t nbytes) {
   return file_->Write(data, nbytes);
 }
 
-// ----------------------------------------------------------------------
-// A readable file that is backed by a PyBuffer
-
-PyBytesReader::PyBytesReader(PyObject* obj)
-    : io::BufferReader(std::make_shared<PyBuffer>(obj)) {}
-
-PyBytesReader::~PyBytesReader() {}
-
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index 0632d28faf789..648f6de06a5a1 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -79,13 +79,6 @@ class ARROW_EXPORT PyOutputStream : public io::OutputStream {
   int64_t position_;
 };
 
-// A zero-copy reader backed by a PyBuffer object
-class ARROW_EXPORT PyBytesReader : public io::BufferReader {
- public:
-  explicit PyBytesReader(PyObject* obj);
-  virtual ~PyBytesReader();
-};
-
 // TODO(wesm): seekable output files
 
 }  // namespace py
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index d9919ee499188..bcf89a4f6e8bb 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -33,7 +33,14 @@
 namespace arrow {
 namespace py {
 
-TEST(PyBuffer, InvalidInputObject) { PyBuffer buffer(Py_None); }
+TEST(PyBuffer, InvalidInputObject) {
+  std::shared_ptr<Buffer> res;
+  PyObject* input = Py_None;
+  auto old_refcnt = Py_REFCNT(input);
+  ASSERT_RAISES(PythonError, PyBuffer::FromPyObject(input, &res));
+  PyErr_Clear();
+  ASSERT_EQ(old_refcnt, Py_REFCNT(input));
+}
 
 class DecimalTest : public ::testing::Test {
  public:
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 2e83f0701ce2e..81cc4d2761e2d 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -894,7 +894,8 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
         " arrow::py::get_memory_pool"()
 
     cdef cppclass PyBuffer(CBuffer):
-        PyBuffer(object o)
+        @staticmethod
+        CStatus FromPyObject(object obj, shared_ptr[CBuffer]* out)
 
     cdef cppclass PyReadableFile(RandomAccessFile):
         PyReadableFile(object fo)
@@ -902,9 +903,6 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     cdef cppclass PyOutputStream(OutputStream):
         PyOutputStream(object fo)
 
-    cdef cppclass PyBytesReader(CBufferReader):
-        PyBytesReader(object fo)
-
     cdef struct PandasOptions:
         c_bool strings_to_categorical
         c_bool zero_copy_only
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index bd508cf57ee8d..b0996f85e203e 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -802,12 +802,8 @@ def frombuffer(object obj):
     Construct an Arrow buffer from a Python bytes object
     """
     cdef shared_ptr[CBuffer] buf
-    try:
-        memoryview(obj)
-        buf.reset(new PyBuffer(obj))
-        return pyarrow_wrap_buffer(buf)
-    except TypeError:
-        raise ValueError('Must pass object that implements buffer protocol')
+    check_status(PyBuffer.FromPyObject(obj, &buf))
+    return pyarrow_wrap_buffer(buf)
 
 
 cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index da26b101db260..0947cb7c7870e 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -104,7 +104,7 @@ def test_bytes_reader():
 
 
 def test_bytes_reader_non_bytes():
-    with pytest.raises(ValueError):
+    with pytest.raises(TypeError):
         pa.BufferReader(u('some sample data'))
 
 
@@ -132,6 +132,7 @@ def test_buffer_bytes():
 
     buf = pa.frombuffer(val)
     assert isinstance(buf, pa.Buffer)
+    assert not buf.is_mutable
 
     result = buf.to_pybytes()
 
@@ -143,6 +144,7 @@ def test_buffer_memoryview():
 
     buf = pa.frombuffer(val)
     assert isinstance(buf, pa.Buffer)
+    assert not buf.is_mutable
 
     result = memoryview(buf)
 
@@ -154,13 +156,20 @@ def test_buffer_bytearray():
 
     buf = pa.frombuffer(val)
     assert isinstance(buf, pa.Buffer)
+    assert buf.is_mutable
 
     result = bytearray(buf)
 
     assert result == val
 
 
-def test_buffer_numpy():
+def test_buffer_invalid():
+    with pytest.raises(TypeError,
+                       match="(bytes-like object|buffer interface)"):
+        pa.frombuffer(None)
+
+
+def test_buffer_to_numpy():
     # Make sure creating a numpy array from an arrow buffer works
     byte_array = bytearray(20)
     byte_array[0] = 42
@@ -170,6 +179,19 @@ def test_buffer_numpy():
     assert array.base == buf
 
 
+def test_buffer_from_numpy():
+    # C-contiguous
+    arr = np.arange(12, dtype=np.int8).reshape((3, 4))
+    buf = pa.frombuffer(arr)
+    assert buf.to_pybytes() == arr.tobytes()
+    # F-contiguous; note strides informations is lost
+    buf = pa.frombuffer(arr.T)
+    assert buf.to_pybytes() == arr.tobytes()
+    # Non-contiguous
+    with pytest.raises(ValueError, match="not contiguous"):
+        buf = pa.frombuffer(arr.T[::2])
+
+
 def test_allocate_buffer():
     buf = pa.allocate_buffer(100)
     assert buf.size == 100

From 78152f113cf2a29b4c78b1c87d88a99fa4e92a29 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 15 Feb 2018 10:37:21 -0500
Subject: [PATCH 1481/1644] ARROW-1815: [Java] Rename MapVector to StructVector

This renames the MapVector to StructVector and NullableMapVector to NullableStructVector.  Related classes have also been renamed as well as APIs making reference to "Map" meaning a struct.  The MinorType.Map is also changed to MinorType.STRUCT.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1490 from BryanCutler/java-rename-MapVector-ARROW-1815 and squashes the following commits:

1540f3e2 [Bryan Cutler] rename NullableStructVector to StructVector
b63451d7 [Bryan Cutler] renamed StructVector to NonNullableStructVector
60413dd9 [Bryan Cutler] Merge remote-tracking branch 'upstream/master' into java-rename-MapVector-ARROW-1815
e5436ea3 [Bryan Cutler] changes some final usages
769a3a7e [Bryan Cutler] changed minor type to STRUCT, tests pass
8129843f [Bryan Cutler] changed some more usage
63aee073 [Bryan Cutler] renamed map based APIs, build working
760c354a [Bryan Cutler] rename Map reader writer classes, now building
8868f181 [Bryan Cutler] rename MapVector
4c50f6f2 [Bryan Cutler] renamed NullableMapVector, supporting Nullable classes
---
 .../arrow/tools/ArrowFileTestFixtures.java    |  10 +-
 .../apache/arrow/tools/TestIntegration.java   |  12 +-
 .../src/main/codegen/includes/vv_imports.ftl  |   2 +-
 .../templates/AbstractFieldReader.java        |   8 +-
 .../templates/AbstractFieldWriter.java        |  12 +-
 .../AbstractPromotableFieldWriter.java        |  18 +-
 .../main/codegen/templates/BaseReader.java    |  10 +-
 .../main/codegen/templates/BaseWriter.java    |  24 +-
 ...s.java => CaseSensitiveStructWriters.java} |  14 +-
 .../main/codegen/templates/ComplexCopier.java |  14 +-
 .../codegen/templates/ComplexReaders.java     |   4 +-
 .../main/codegen/templates/NullReader.java    |   8 +-
 .../{MapWriters.java => StructWriters.java}   |  38 +--
 .../codegen/templates/UnionListWriter.java    |  24 +-
 .../main/codegen/templates/UnionReader.java   |  22 +-
 .../main/codegen/templates/UnionVector.java   |  84 +++---
 .../main/codegen/templates/UnionWriter.java   |  66 ++---
 .../arrow/vector/BaseVariableWidthVector.java |   1 -
 .../complex/AbstractContainerVector.java      |   4 +-
 ...pVector.java => AbstractStructVector.java} |  12 +-
 ...ctor.java => NonNullableStructVector.java} |  42 ++-
 ...llableMapVector.java => StructVector.java} |  42 ++-
 .../complex/impl/ComplexWriterImpl.java       |  74 ++---
 ...mpl.java => NullableStructReaderImpl.java} |  32 +--
 ....java => NullableStructWriterFactory.java} |  20 +-
 .../vector/complex/impl/PromotableWriter.java |  40 +--
 .../complex/impl/SingleListReaderImpl.java    |   4 +-
 ...rImpl.java => SingleStructReaderImpl.java} |  20 +-
 ...rImpl.java => StructOrListWriterImpl.java} |  56 ++--
 .../vector/complex/reader/FieldReader.java    |   6 +-
 .../vector/complex/writer/FieldWriter.java    |   4 +-
 .../vector/dictionary/DictionaryEncoder.java  |   2 +-
 ...pHolder.java => RepeatedStructHolder.java} |   2 +-
 .../org/apache/arrow/vector/types/Types.java  |  12 +-
 ...stMapVector.java => TestStructVector.java} |   6 +-
 .../arrow/vector/TestVectorReAlloc.java       |   7 +-
 .../apache/arrow/vector/TestVectorReset.java  |  30 +-
 .../arrow/vector/TestVectorUnloadLoad.java    |  12 +-
 .../complex/impl/TestPromotableWriter.java    |  20 +-
 .../complex/writer/TestComplexWriter.java     | 260 +++++++++---------
 .../apache/arrow/vector/ipc/BaseFileTest.java |  50 ++--
 .../arrow/vector/ipc/TestArrowFile.java       |  22 +-
 .../apache/arrow/vector/ipc/TestJSONFile.java |  14 +-
 43 files changed, 578 insertions(+), 586 deletions(-)
 rename java/vector/src/main/codegen/templates/{CaseSensitiveMapWriters.java => CaseSensitiveStructWriters.java} (74%)
 rename java/vector/src/main/codegen/templates/{MapWriters.java => StructWriters.java} (88%)
 rename java/vector/src/main/java/org/apache/arrow/vector/complex/{AbstractMapVector.java => AbstractStructVector.java} (95%)
 rename java/vector/src/main/java/org/apache/arrow/vector/complex/{MapVector.java => NonNullableStructVector.java} (85%)
 rename java/vector/src/main/java/org/apache/arrow/vector/complex/{NullableMapVector.java => StructVector.java} (90%)
 rename java/vector/src/main/java/org/apache/arrow/vector/complex/impl/{NullableMapReaderImpl.java => NullableStructReaderImpl.java} (55%)
 rename java/vector/src/main/java/org/apache/arrow/vector/complex/impl/{NullableMapWriterFactory.java => NullableStructWriterFactory.java} (55%)
 rename java/vector/src/main/java/org/apache/arrow/vector/complex/impl/{SingleMapReaderImpl.java => SingleStructReaderImpl.java} (81%)
 rename java/vector/src/main/java/org/apache/arrow/vector/complex/impl/{MapOrListWriterImpl.java => StructOrListWriterImpl.java} (59%)
 rename java/vector/src/main/java/org/apache/arrow/vector/holders/{RepeatedMapHolder.java => RepeatedStructHolder.java} (93%)
 rename java/vector/src/test/java/org/apache/arrow/vector/{TestMapVector.java => TestStructVector.java} (90%)

diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
index eac517d96bd48..da6f0e3764f73 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -22,10 +22,10 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
 import org.apache.arrow.vector.ipc.message.ArrowBlock;
@@ -43,9 +43,9 @@
 public class ArrowFileTestFixtures {
   static final int COUNT = 10;
 
-  static void writeData(int count, MapVector parent) {
+  static void writeData(int count, NonNullableStructVector parent) {
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     IntWriter intWriter = rootWriter.integer("int");
     BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
     for (int i = 0; i < count; i++) {
@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NonNullableStructVector parent = NonNullableStructVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), testInFile);
     }
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
index 4b840df670fdc..5293eb6f67973 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -27,10 +27,10 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.tools.Integration.Command;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.Float8Writer;
 import org.apache.arrow.vector.complex.writer.IntWriter;
@@ -76,9 +76,9 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NonNullableStructVector parent = NonNullableStructVector.empty("parent", vectorAllocator)) {
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
-      MapWriter rootWriter = writer.rootAsMap();
+      StructWriter rootWriter = writer.rootAsStruct();
       Float8Writer floatWriter = rootWriter.float8("float");
       for (int i = 0; i < f.length; i++) {
         floatWriter.setPosition(i);
@@ -95,10 +95,10 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+        NonNullableStructVector parent = NonNullableStructVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
-      MapWriter rootWriter = writer.rootAsMap();
+      StructWriter rootWriter = writer.rootAsStruct();
       IntWriter intWriter = rootWriter.integer("int");
       BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
       intWriter.setPosition(5);
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
index a55304d733539..fb733fbb74cbe 100644
--- a/java/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -35,7 +35,7 @@ import org.apache.arrow.vector.complex.*;
 import org.apache.arrow.vector.complex.reader.*;
 import org.apache.arrow.vector.complex.impl.*;
 import org.apache.arrow.vector.complex.writer.*;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
index b16ee162fde9e..031efec2ae610 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -66,12 +66,12 @@ public Field getField() {
   }
 
   </#list>
-  public void copyAsValue(MapWriter writer) {
-    fail("CopyAsValue MapWriter");
+  public void copyAsValue(StructWriter writer) {
+    fail("CopyAsValue StructWriter");
   }
 
-  public void copyAsField(String name, MapWriter writer) {
-    fail("CopyAsField MapWriter");
+  public void copyAsField(String name, StructWriter writer) {
+    fail("CopyAsField StructWriter");
   }
 
   public void copyAsField(String name, ListWriter writer) {
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index fce6876025a91..f61e52e927197 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -82,17 +82,17 @@ public void writeNull() {
   /**
    * This implementation returns {@code false}.
    * <p>  
-   *   Must be overridden by map writers.
+   *   Must be overridden by struct writers.
    * </p>  
    */
   @Override
-  public boolean isEmptyMap() {
+  public boolean isEmptyStruct() {
     return false;
   }
 
   @Override
-  public MapWriter map() {
-    fail("Map");
+  public StructWriter struct() {
+    fail("Struct");
     return null;
   }
 
@@ -103,8 +103,8 @@ public ListWriter list() {
   }
 
   @Override
-  public MapWriter map(String name) {
-    fail("Map");
+  public StructWriter struct(String name) {
+    fail("Struct");
     return null;
   }
 
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 7f4a13d4f06e8..5b8cfa910ba34 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -53,12 +53,12 @@ abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
 
   @Override
   public void start() {
-    getWriter(MinorType.MAP).start();
+    getWriter(MinorType.STRUCT).start();
   }
 
   @Override
   public void end() {
-    getWriter(MinorType.MAP).end();
+    getWriter(MinorType.STRUCT).end();
     setPosition(idx() + 1);
   }
 
@@ -95,8 +95,8 @@ public void writeNull() {
   }
 
   @Override
-  public MapWriter map() {
-    return getWriter(MinorType.LIST).map();
+  public StructWriter struct() {
+    return getWriter(MinorType.LIST).struct();
   }
 
   @Override
@@ -105,13 +105,13 @@ public ListWriter list() {
   }
 
   @Override
-  public MapWriter map(String name) {
-    return getWriter(MinorType.MAP).map(name);
+  public StructWriter struct(String name) {
+    return getWriter(MinorType.STRUCT).struct(name);
   }
 
   @Override
   public ListWriter list(String name) {
-    return getWriter(MinorType.MAP).list(name);
+    return getWriter(MinorType.STRUCT).list(name);
   }
 
   <#list vv.types as type><#list type.minor as minor>
@@ -123,13 +123,13 @@ public ListWriter list(String name) {
   <#if minor.typeParams?? >
   @Override
   public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
-    return getWriter(MinorType.MAP).${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name}</#list>);
+    return getWriter(MinorType.STRUCT).${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name}</#list>);
   }
 
   </#if>
   @Override
   public ${capName}Writer ${lowerName}(String name) {
-    return getWriter(MinorType.MAP).${lowerName}(name);
+    return getWriter(MinorType.STRUCT).${lowerName}(name);
   }
 
   @Override
diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java
index ea3286e86817a..701911b645b42 100644
--- a/java/vector/src/main/codegen/templates/BaseReader.java
+++ b/java/vector/src/main/codegen/templates/BaseReader.java
@@ -39,14 +39,14 @@ public interface BaseReader extends Positionable{
   void copyAsValue(UnionWriter writer);
   boolean isSet();
 
-  public interface MapReader extends BaseReader, Iterable<String>{
+  public interface StructReader extends BaseReader, Iterable<String>{
     FieldReader reader(String name);
   }
   
-  public interface RepeatedMapReader extends MapReader{
+  public interface RepeatedStructReader extends StructReader{
     boolean next();
     int size();
-    void copyAsValue(MapWriter writer);
+    void copyAsValue(StructWriter writer);
   }
   
   public interface ListReader extends BaseReader{
@@ -64,9 +64,9 @@ public interface ScalarReader extends
   BaseReader {}
   
   interface ComplexReader{
-    MapReader rootAsMap();
+    StructReader rootAsStruct();
     ListReader rootAsList();
-    boolean rootIsMap();
+    boolean rootIsStruct();
     boolean ok();
   }
 }
diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java
index 405f466cbc708..b55d1e79df591 100644
--- a/java/vector/src/main/codegen/templates/BaseWriter.java
+++ b/java/vector/src/main/codegen/templates/BaseWriter.java
@@ -33,20 +33,20 @@
 public interface BaseWriter extends AutoCloseable, Positionable {
   int getValueCapacity();
 
-  public interface MapWriter extends BaseWriter {
+  public interface StructWriter extends BaseWriter {
 
     Field getField();
 
     /**
-     * Whether this writer is a map writer and is empty (has no children).
+     * Whether this writer is a struct writer and is empty (has no children).
      *
      * <p>
      *   Intended only for use in determining whether to add dummy vector to
      *   avoid empty (zero-column) schema, as in JsonReader.
      * </p>
-     * @return whether the map is empty
+     * @return whether the struct is empty
      */
-    boolean isEmptyMap();
+    boolean isEmptyStruct();
 
     <#list vv.types as type><#list type.minor as minor>
     <#assign lowerName = minor.class?uncap_first />
@@ -60,7 +60,7 @@ public interface MapWriter extends BaseWriter {
     </#list></#list>
 
     void copyReaderToField(String name, FieldReader reader);
-    MapWriter map(String name);
+    StructWriter struct(String name);
     ListWriter list(String name);
     void start();
     void end();
@@ -69,7 +69,7 @@ public interface MapWriter extends BaseWriter {
   public interface ListWriter extends BaseWriter {
     void startList();
     void endList();
-    MapWriter map();
+    StructWriter struct();
     ListWriter list();
     void copyReader(FieldReader reader);
 
@@ -89,7 +89,7 @@ public interface ComplexWriter {
     void allocate();
     void clear();
     void copyReader(FieldReader reader);
-    MapWriter rootAsMap();
+    StructWriter rootAsStruct();
     ListWriter rootAsList();
 
     void setPosition(int index);
@@ -97,13 +97,13 @@ public interface ComplexWriter {
     void reset();
   }
 
-  public interface MapOrListWriter {
+  public interface StructOrListWriter {
     void start();
     void end();
-    MapOrListWriter map(String name);
-    MapOrListWriter listoftmap(String name);
-    MapOrListWriter list(String name);
-    boolean isMapWriter();
+    StructOrListWriter struct(String name);
+    StructOrListWriter listoftstruct(String name);
+    StructOrListWriter list(String name);
+    boolean isStructWriter();
     boolean isListWriter();
     VarCharWriter varChar(String name);
     IntWriter integer(String name);
diff --git a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java b/java/vector/src/main/codegen/templates/CaseSensitiveStructWriters.java
similarity index 74%
rename from java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
rename to java/vector/src/main/codegen/templates/CaseSensitiveStructWriters.java
index 5357f9b8a9d3a..c5d25e96af62e 100644
--- a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
+++ b/java/vector/src/main/codegen/templates/CaseSensitiveStructWriters.java
@@ -18,12 +18,12 @@
 
 <@pp.dropOutputFile />
 <#list ["Nullable", "Single"] as mode>
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}CaseSensitiveMapWriter.java" />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}CaseSensitiveStructWriter.java" />
 <#assign index = "idx()">
 <#if mode == "Single">
-<#assign containerClass = "MapVector" />
+<#assign containerClass = "NonNullableStructVector" />
 <#else>
-<#assign containerClass = "NullableMapVector" />
+<#assign containerClass = "StructVector" />
 </#if>
 
 <#include "/@includes/license.ftl" />
@@ -35,8 +35,8 @@
  * This class is generated using FreeMarker and the ${.template_name} template.
  */
 @SuppressWarnings("unused")
-public class ${mode}CaseSensitiveMapWriter extends ${mode}MapWriter {
-  public ${mode}CaseSensitiveMapWriter(${containerClass} container) {
+public class ${mode}CaseSensitiveStructWriter extends ${mode}StructWriter {
+  public ${mode}CaseSensitiveStructWriter(${containerClass} container) {
     super(container);
   }
 
@@ -46,8 +46,8 @@ protected String handleCase(final String input){
   }
 
   @Override
-  protected NullableMapWriterFactory getNullableMapWriterFactory() {
-    return NullableMapWriterFactory.getNullableCaseSensitiveMapWriterFactoryInstance();
+  protected NullableStructWriterFactory getNullableStructWriterFactory() {
+    return NullableStructWriterFactory.getNullableCaseSensitiveStructWriterFactoryInstance();
   }
 
 }
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index 518ad5d398ddb..bbe537687e671 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -57,13 +57,13 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
         break;
       case FIXED_SIZE_LIST:
         throw new UnsupportedOperationException("Copy fixed size list");
-      case MAP:
+      case STRUCT:
         if (reader.isSet()) {
           writer.start();
           for(String name : reader){
             FieldReader childReader = reader.reader(name);
             if(childReader.isSet()){
-              writeValue(childReader, getMapWriterForReader(childReader, writer, name));
+              writeValue(childReader, getStructWriterForReader(childReader, writer, name));
             }
           }
           writer.end();
@@ -90,7 +90,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
       }
  }
 
-  private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
+  private static FieldWriter getStructWriterForReader(FieldReader reader, StructWriter writer, String name) {
     switch (reader.getMinorType()) {
     <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
     <#assign fields = minor.fields!type.fields />
@@ -100,8 +100,8 @@ private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter w
       return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name);
     </#if>
     </#list></#list>
-    case MAP:
-      return (FieldWriter) writer.map(name);
+    case STRUCT:
+      return (FieldWriter) writer.struct(name);
     case LIST:
       return (FieldWriter) writer.list(name);
     default:
@@ -119,8 +119,8 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter
     return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>();
     </#if>
     </#list></#list>
-    case MAP:
-      return (FieldWriter) writer.map();
+    case STRUCT:
+      return (FieldWriter) writer.struct();
     case LIST:
       return (FieldWriter) writer.list();
     default:
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 4863ecdb63284..29979d383b8b5 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -22,7 +22,7 @@
 import org.apache.arrow.record.TransferPair;
 import org.apache.arrow.vector.complex.IndexHolder;
 import org.apache.arrow.vector.complex.writer.IntervalWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 
 <@pp.dropOutputFile />
 <#list vv.types as type>
@@ -77,7 +77,7 @@ public void copyAsValue(${minor.class?cap_first}Writer writer){
     impl.vector.copyFromSafe(idx(), impl.idx(), vector);
   }
   
-  public void copyAsField(String name, MapWriter writer){
+  public void copyAsField(String name, StructWriter writer){
     ${minor.class?cap_first}WriterImpl impl = (${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
     impl.vector.copyFromSafe(idx(), impl.idx(), vector);
   }
diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java
index 284a8d38f23e0..fae57081ba4f2 100644
--- a/java/vector/src/main/codegen/templates/NullReader.java
+++ b/java/vector/src/main/codegen/templates/NullReader.java
@@ -37,7 +37,7 @@ public class NullReader extends AbstractBaseReader implements FieldReader{
   
   public static final NullReader INSTANCE = new NullReader();
   public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(MinorType.NULL);
-  public static final NullReader EMPTY_MAP_INSTANCE = new NullReader(MinorType.MAP);
+  public static final NullReader EMPTY_STRUCT_INSTANCE = new NullReader(MinorType.STRUCT);
   private MinorType type;
   
   private NullReader(){
@@ -60,7 +60,7 @@ public Field getField() {
     return new Field("", FieldType.nullable(new Null()), null);
   }
 
-  public void copyAsValue(MapWriter writer) {}
+  public void copyAsValue(StructWriter writer) {}
 
   public void copyAsValue(ListWriter writer) {}
 
@@ -99,7 +99,7 @@ public boolean next(){
     return false;
   }
   
-  public RepeatedMapReader map(){
+  public RepeatedStructReader struct(){
     return this;
   }
   
@@ -107,7 +107,7 @@ public RepeatedListReader list(){
     return this;
   }
   
-  public MapReader map(String name){
+  public StructReader struct(String name){
     return this;
   }
   
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/StructWriters.java
similarity index 88%
rename from java/vector/src/main/codegen/templates/MapWriters.java
rename to java/vector/src/main/codegen/templates/StructWriters.java
index ac59e59b30888..0bd2e96a5b593 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/StructWriters.java
@@ -18,12 +18,12 @@
 
 <@pp.dropOutputFile />
 <#list ["Nullable", "Single"] as mode>
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}StructWriter.java" />
 <#assign index = "idx()">
 <#if mode == "Single">
-<#assign containerClass = "MapVector" />
+<#assign containerClass = "NonNullableStructVector" />
 <#else>
-<#assign containerClass = "NullableMapVector" />
+<#assign containerClass = "StructVector" />
 </#if>
 
 <#include "/@includes/license.ftl" />
@@ -33,7 +33,7 @@
 <#include "/@includes/vv_imports.ftl" />
 import java.util.Map;
 
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
+import org.apache.arrow.vector.holders.RepeatedStructHolder;
 import org.apache.arrow.vector.AllocationHelper;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
@@ -44,14 +44,14 @@
  * This class is generated using FreeMarker and the ${.template_name} template.
  */
 @SuppressWarnings("unused")
-public class ${mode}MapWriter extends AbstractFieldWriter {
+public class ${mode}StructWriter extends AbstractFieldWriter {
 
   protected final ${containerClass} container;
   private int initialCapacity;
   private final Map<String, FieldWriter> fields = Maps.newHashMap();
-  public ${mode}MapWriter(${containerClass} container) {
+  public ${mode}StructWriter(${containerClass} container) {
     <#if mode == "Single">
-    if (container instanceof NullableMapVector) {
+    if (container instanceof StructVector) {
       throw new IllegalArgumentException("Invalid container: " + container);
     }
     </#if>
@@ -60,14 +60,14 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
     for (Field child : container.getField().getChildren()) {
       MinorType minorType = Types.getMinorTypeForArrowType(child.getType());
       switch (minorType) {
-      case MAP:
-        map(child.getName());
+      case STRUCT:
+        struct(child.getName());
         break;
       case LIST:
         list(child.getName());
         break;
       case UNION:
-        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), FieldType.nullable(MinorType.UNION.getType()), UnionVector.class), getNullableMapWriterFactory());
+        UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), FieldType.nullable(MinorType.UNION.getType()), UnionVector.class), getNullableStructWriterFactory());
         fields.put(handleCase(child.getName()), writer);
         break;
 <#list vv.types as type><#list type.minor as minor>
@@ -94,8 +94,8 @@ protected String handleCase(final String input) {
     return input.toLowerCase();
   }
 
-  protected NullableMapWriterFactory getNullableMapWriterFactory() {
-    return NullableMapWriterFactory.getNullableMapWriterFactoryInstance();
+  protected NullableStructWriterFactory getNullableStructWriterFactory() {
+    return NullableStructWriterFactory.getNullableStructWriterFactoryInstance();
   }
 
   @Override
@@ -109,7 +109,7 @@ public void setInitialCapacity(int initialCapacity) {
   }
 
   @Override
-  public boolean isEmptyMap() {
+  public boolean isEmptyStruct() {
     return 0 == container.size();
   }
 
@@ -119,13 +119,13 @@ public Field getField() {
   }
 
   @Override
-  public MapWriter map(String name) {
+  public StructWriter struct(String name) {
     String finalName = handleCase(name);
     FieldWriter writer = fields.get(finalName);
     if(writer == null){
       int vectorCount=container.size();
-      NullableMapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), NullableMapVector.class);
-      writer = new PromotableWriter(vector, container, getNullableMapWriterFactory());
+      StructVector vector = container.addOrGet(name, FieldType.nullable(MinorType.STRUCT.getType()), StructVector.class);
+      writer = new PromotableWriter(vector, container, getNullableStructWriterFactory());
       if(vectorCount != container.size()) {
         writer.allocate();
       }
@@ -134,7 +134,7 @@ public MapWriter map(String name) {
     } else {
       if (writer instanceof PromotableWriter) {
         // ensure writers are initialized
-        ((PromotableWriter)writer).getWriter(MinorType.MAP);
+        ((PromotableWriter)writer).getWriter(MinorType.STRUCT);
       }
     }
     return writer;
@@ -168,7 +168,7 @@ public ListWriter list(String name) {
     FieldWriter writer = fields.get(finalName);
     int vectorCount = container.size();
     if(writer == null) {
-      writer = new PromotableWriter(container.addOrGet(name, FieldType.nullable(MinorType.LIST.getType()), ListVector.class), container, getNullableMapWriterFactory());
+      writer = new PromotableWriter(container.addOrGet(name, FieldType.nullable(MinorType.LIST.getType()), ListVector.class), container, getNullableStructWriterFactory());
       if (container.size() > vectorCount) {
         writer.allocate();
       }
@@ -251,7 +251,7 @@ public void end() {
           </#if>
           ),
           ${vectName}Vector.class);
-      writer = new PromotableWriter(v, container, getNullableMapWriterFactory());
+      writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
       vector = v;
       if (currentVector == null || currentVector != vector) {
         if(this.initialCapacity > 0) {
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index 8ac23fe46f7d2..eebf04431cc35 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -37,18 +37,18 @@ public class UnionListWriter extends AbstractFieldWriter {
 
   private ListVector vector;
   private PromotableWriter writer;
-  private boolean inMap = false;
-  private String mapName;
+  private boolean inStruct = false;
+  private String structName;
   private int lastIndex = 0;
   private static final int OFFSET_WIDTH = 4;
 
   public UnionListWriter(ListVector vector) {
-    this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
+    this(vector, NullableStructWriterFactory.getNullableStructWriterFactoryInstance());
   }
 
-  public UnionListWriter(ListVector vector, NullableMapWriterFactory nullableMapWriterFactory) {
+  public UnionListWriter(ListVector vector, NullableStructWriterFactory nullableStructWriterFactory) {
     this.vector = vector;
-    this.writer = new PromotableWriter(vector.getDataVector(), vector, nullableMapWriterFactory);
+    this.writer = new PromotableWriter(vector.getDataVector(), vector, nullableStructWriterFactory);
   }
 
   public UnionListWriter(ListVector vector, AbstractFieldWriter parent) {
@@ -101,15 +101,15 @@ public void setPosition(int index) {
 
   @Override
   public ${name}Writer ${uncappedName}(String name) {
-    mapName = name;
+    structName = name;
     return writer.${uncappedName}(name);
   }
   </#if>
   </#list></#list>
 
   @Override
-  public MapWriter map() {
-    inMap = true;
+  public StructWriter struct() {
+    inStruct = true;
     return this;
   }
 
@@ -125,9 +125,9 @@ public ListWriter list(String name) {
   }
 
   @Override
-  public MapWriter map(String name) {
-    MapWriter mapWriter = writer.map(name);
-    return mapWriter;
+  public StructWriter struct(String name) {
+    StructWriter structWriter = writer.struct(name);
+    return structWriter;
   }
 
   @Override
@@ -150,7 +150,7 @@ public void start() {
   @Override
   public void end() {
     writer.end();
-    inMap = false;
+    inStruct = false;
   }
 
   <#list vv.types as type>
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index 98bb7c1f53d84..e63fbc8313919 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -81,8 +81,8 @@ private FieldReader getReaderForIndex(int index) {
     switch (MinorType.values()[typeValue]) {
     case NULL:
       return NullReader.INSTANCE;
-    case MAP:
-      return (FieldReader) getMap();
+    case STRUCT:
+      return (FieldReader) getStruct();
     case LIST:
       return (FieldReader) getList();
     <#list vv.types as type>
@@ -100,15 +100,15 @@ private FieldReader getReaderForIndex(int index) {
     }
   }
 
-  private SingleMapReaderImpl mapReader;
+  private SingleStructReaderImpl structReader;
 
-  private MapReader getMap() {
-    if (mapReader == null) {
-      mapReader = (SingleMapReaderImpl) data.getMap().getReader();
-      mapReader.setPosition(idx());
-      readers[MinorType.MAP.ordinal()] = mapReader;
+  private StructReader getStruct() {
+    if (structReader == null) {
+      structReader = (SingleStructReaderImpl) data.getStruct().getReader();
+      structReader.setPosition(idx());
+      readers[MinorType.STRUCT.ordinal()] = structReader;
     }
-    return mapReader;
+    return structReader;
   }
 
   private UnionListReader listReader;
@@ -124,7 +124,7 @@ private FieldReader getList() {
 
   @Override
   public java.util.Iterator<String> iterator() {
-    return getMap().iterator();
+    return getStruct().iterator();
   }
 
   @Override
@@ -198,7 +198,7 @@ public void setPosition(int index) {
   }
 
   public FieldReader reader(String name){
-    return getMap().reader(name);
+    return getStruct().reader(name);
   }
 
   public FieldReader reader() {
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index aa8178a92f80a..84450bee5026e 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -47,11 +47,11 @@
 
 
 /**
- * A vector which can hold values of different types. It does so by using a NullableMapVector which contains a vector for each
- * primitive type that is stored. NullableMapVector is used in order to take advantage of its serialization/deserialization methods,
+ * A vector which can hold values of different types. It does so by using a StructVector which contains a vector for each
+ * primitive type that is stored. StructVector is used in order to take advantage of its serialization/deserialization methods,
  * as well as the addOrGet method.
  *
- * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
+ * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the struct lookup
  * each time the vector is accessed.
  * Source code generated using FreeMarker template ${.template_name}
  */
@@ -61,10 +61,10 @@ public class UnionVector implements FieldVector {
   private BufferAllocator allocator;
   int valueCount;
 
-  MapVector internalMap;
+  NonNullableStructVector internalStruct;
   protected ArrowBuf typeBuffer;
 
-  private NullableMapVector mapVector;
+  private StructVector structVector;
   private ListVector listVector;
 
   private FieldReader reader;
@@ -79,7 +79,7 @@ public class UnionVector implements FieldVector {
   public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
     this.name = name;
     this.allocator = allocator;
-    this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
+    this.internalStruct = new NonNullableStructVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
     this.typeBuffer = allocator.getEmpty();
     this.callBack = callBack;
     this.typeBufferAllocationSizeInBytes = BaseValueVector.INITIAL_VALUE_ALLOCATION * TYPE_WIDTH;
@@ -96,12 +96,12 @@ public MinorType getMinorType() {
 
   @Override
   public void initializeChildrenFromFields(List<Field> children) {
-    internalMap.initializeChildrenFromFields(children);
+    internalStruct.initializeChildrenFromFields(children);
   }
 
   @Override
   public List<FieldVector> getChildrenFromFields() {
-    return internalMap.getChildrenFromFields();
+    return internalStruct.getChildrenFromFields();
   }
 
   @Override
@@ -146,7 +146,7 @@ private FieldType fieldType(MinorType type) {
   }
 
   private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
-    return internalMap.addOrGet(fieldName(minorType), fieldType(minorType), c);
+    return internalStruct.addOrGet(fieldName(minorType), fieldType(minorType), c);
   }
 
   @Override
@@ -173,18 +173,18 @@ public long getOffsetBufferAddress() {
   @Override
   public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); }
 
-  public NullableMapVector getMap() {
-    if (mapVector == null) {
-      int vectorCount = internalMap.size();
-      mapVector = addOrGet(MinorType.MAP, NullableMapVector.class);
-      if (internalMap.size() > vectorCount) {
-        mapVector.allocateNew();
+  public StructVector getStruct() {
+    if (structVector == null) {
+      int vectorCount = internalStruct.size();
+      structVector = addOrGet(MinorType.STRUCT, StructVector.class);
+      if (internalStruct.size() > vectorCount) {
+        structVector.allocateNew();
         if (callBack != null) {
           callBack.doWork();
         }
       }
     }
-    return mapVector;
+    return structVector;
   }
   <#list vv.types as type>
     <#list type.minor as minor>
@@ -198,9 +198,9 @@ public NullableMapVector getMap() {
 
   public ${name}Vector get${name}Vector() {
     if (${uncappedName}Vector == null) {
-      int vectorCount = internalMap.size();
+      int vectorCount = internalStruct.size();
       ${uncappedName}Vector = addOrGet(MinorType.${name?upper_case}, ${name}Vector.class);
-      if (internalMap.size() > vectorCount) {
+      if (internalStruct.size() > vectorCount) {
         ${uncappedName}Vector.allocateNew();
         if (callBack != null) {
           callBack.doWork();
@@ -215,9 +215,9 @@ public NullableMapVector getMap() {
 
   public ListVector getList() {
     if (listVector == null) {
-      int vectorCount = internalMap.size();
+      int vectorCount = internalStruct.size();
       listVector = addOrGet(MinorType.LIST, ListVector.class);
-      if (internalMap.size() > vectorCount) {
+      if (internalStruct.size() > vectorCount) {
         listVector.allocateNew();
         if (callBack != null) {
           callBack.doWork();
@@ -235,7 +235,7 @@ public int getTypeValue(int index) {
   public void allocateNew() throws OutOfMemoryException {
     /* new allocation -- clear the current buffers */
     clear();
-    internalMap.allocateNew();
+    internalStruct.allocateNew();
     try {
       allocateTypeBuffer();
     } catch (Exception e) {
@@ -248,7 +248,7 @@ public void allocateNew() throws OutOfMemoryException {
   public boolean allocateNewSafe() {
     /* new allocation -- clear the current buffers */
     clear();
-    boolean safe = internalMap.allocateNewSafe();
+    boolean safe = internalStruct.allocateNewSafe();
     if (!safe) { return false; }
     try {
       allocateTypeBuffer();
@@ -268,7 +268,7 @@ private void allocateTypeBuffer() {
 
   @Override
   public void reAlloc() {
-    internalMap.reAlloc();
+    internalStruct.reAlloc();
     reallocTypeBuffer();
   }
 
@@ -301,7 +301,7 @@ public void setInitialCapacity(int numRecords) { }
 
   @Override
   public int getValueCapacity() {
-    return Math.min(getTypeBufferValueCapacity(), internalMap.getValueCapacity());
+    return Math.min(getTypeBufferValueCapacity(), internalStruct.getValueCapacity());
   }
 
   @Override
@@ -314,20 +314,20 @@ public void clear() {
     valueCount = 0;
     typeBuffer.release();
     typeBuffer = allocator.getEmpty();
-    internalMap.clear();
+    internalStruct.clear();
   }
 
   @Override
   public void reset() {
     valueCount = 0;
     typeBuffer.setZero(0, typeBuffer.capacity());
-    internalMap.reset();
+    internalStruct.reset();
   }
 
   @Override
   public Field getField() {
     List<org.apache.arrow.vector.types.pojo.Field> childFields = new ArrayList<>();
-    List<FieldVector> children = internalMap.getChildren();
+    List<FieldVector> children = internalStruct.getChildren();
     int[] typeIds = new int[children.size()];
     for (ValueVector v : children) {
       typeIds[childFields.size()] = v.getMinorType().ordinal();
@@ -368,10 +368,10 @@ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
 
   public FieldVector addVector(FieldVector v) {
     String name = v.getMinorType().name().toLowerCase();
-    Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
-    final FieldVector newVector = internalMap.addOrGet(name, v.getField().getFieldType(), v.getClass());
+    Preconditions.checkState(internalStruct.getChild(name) == null, String.format("%s vector already exists", name));
+    final FieldVector newVector = internalStruct.addOrGet(name, v.getField().getFieldType(), v.getClass());
     v.makeTransferPair(newVector).transfer();
-    internalMap.putChild(name, newVector);
+    internalStruct.putChild(name, newVector);
     if (callBack != null) {
       callBack.doWork();
     }
@@ -379,24 +379,24 @@ public FieldVector addVector(FieldVector v) {
   }
 
   private class TransferImpl implements TransferPair {
-    private final TransferPair internalMapVectorTransferPair;
+    private final TransferPair internalStructVectorTransferPair;
     private final UnionVector to;
 
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
       to = new UnionVector(name, allocator, callBack);
-      internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap);
+      internalStructVectorTransferPair = internalStruct.makeTransferPair(to.internalStruct);
     }
 
     public TransferImpl(UnionVector to) {
       this.to = to;
-      internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap);
+      internalStructVectorTransferPair = internalStruct.makeTransferPair(to.internalStruct);
     }
 
     @Override
     public void transfer() {
       to.clear();
       to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer;
-      internalMapVectorTransferPair.transfer();
+      internalStructVectorTransferPair.transfer();
       to.valueCount = valueCount;
       clear();
     }
@@ -404,7 +404,7 @@ public void transfer() {
     @Override
     public void splitAndTransfer(int startIndex, int length) {
       to.clear();
-      internalMapVectorTransferPair.splitAndTransfer(startIndex, length);
+      internalStructVectorTransferPair.splitAndTransfer(startIndex, length);
       final int startPoint = startIndex * TYPE_WIDTH;
       final int sliceLength = length * TYPE_WIDTH;
       to.typeBuffer = typeBuffer.slice(startPoint, sliceLength).transferOwnership(to.allocator).buffer;
@@ -441,7 +441,7 @@ public FieldWriter getWriter() {
   public int getBufferSize() {
     if (valueCount == 0) { return 0; }
 
-    return (valueCount * TYPE_WIDTH) + internalMap.getBufferSize();
+    return (valueCount * TYPE_WIDTH) + internalStruct.getBufferSize();
   }
 
   @Override
@@ -464,7 +464,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
     setReaderAndWriterIndex();
     if (getBufferSize() != 0) {
       builder.add(typeBuffer);
-      builder.add(internalMap.getBuffers(clear));
+      builder.add(internalStruct.getBuffers(clear));
     }
     if (clear) {
       valueCount = 0;
@@ -478,7 +478,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
 
   @Override
   public Iterator<ValueVector> iterator() {
-    List<ValueVector> vectors = Lists.newArrayList(internalMap.iterator());
+    List<ValueVector> vectors = Lists.newArrayList(internalStruct.iterator());
     return vectors.iterator();
   }
 
@@ -499,8 +499,8 @@ public Object getObject(int index) {
           </#if>
         </#list>
       </#list>
-      case MAP:
-        return getMap().getObject(index);
+      case STRUCT:
+        return getStruct().getObject(index);
       case LIST:
         return getList().getObject(index);
       default:
@@ -551,7 +551,7 @@ public void setValueCount(int valueCount) {
       while (valueCount > getTypeBufferValueCapacity()) {
         reallocTypeBuffer();
       }
-      internalMap.setValueCount(valueCount);
+      internalStruct.setValueCount(valueCount);
     }
 
     public void setSafe(int index, UnionHolder holder) {
@@ -576,7 +576,7 @@ public void setSafe(int index, UnionHolder holder) {
           </#if>
         </#list>
       </#list>
-      case MAP: {
+      case STRUCT: {
         ComplexCopier.copy(reader, writer);
         break;
       }
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 526708a4c328b..1915c87428fb0 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-import org.apache.arrow.vector.complex.impl.NullableMapWriterFactory;
+import org.apache.arrow.vector.complex.impl.NullableStructWriterFactory;
 
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionWriter.java" />
@@ -37,18 +37,18 @@
 public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
 
   UnionVector data;
-  private MapWriter mapWriter;
+  private StructWriter structWriter;
   private UnionListWriter listWriter;
   private List<BaseWriter> writers = Lists.newArrayList();
-  private final NullableMapWriterFactory nullableMapWriterFactory;
+  private final NullableStructWriterFactory nullableStructWriterFactory;
 
   public UnionWriter(UnionVector vector) {
-    this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
+    this(vector, NullableStructWriterFactory.getNullableStructWriterFactoryInstance());
   }
 
-  public UnionWriter(UnionVector vector, NullableMapWriterFactory nullableMapWriterFactory) {
+  public UnionWriter(UnionVector vector, NullableStructWriterFactory nullableStructWriterFactory) {
     data = vector;
-    this.nullableMapWriterFactory = nullableMapWriterFactory;
+    this.nullableStructWriterFactory = nullableStructWriterFactory;
   }
 
   @Override
@@ -62,13 +62,13 @@ public void setPosition(int index) {
 
   @Override
   public void start() {
-    data.setType(idx(), MinorType.MAP);
-    getMapWriter().start();
+    data.setType(idx(), MinorType.STRUCT);
+    getStructWriter().start();
   }
 
   @Override
   public void end() {
-    getMapWriter().end();
+    getStructWriter().end();
   }
 
   @Override
@@ -82,23 +82,23 @@ public void endList() {
     getListWriter().endList();
   }
 
-  private MapWriter getMapWriter() {
-    if (mapWriter == null) {
-      mapWriter = nullableMapWriterFactory.build(data.getMap());
-      mapWriter.setPosition(idx());
-      writers.add(mapWriter);
+  private StructWriter getStructWriter() {
+    if (structWriter == null) {
+      structWriter = nullableStructWriterFactory.build(data.getStruct());
+      structWriter.setPosition(idx());
+      writers.add(structWriter);
     }
-    return mapWriter;
+    return structWriter;
   }
 
-  public MapWriter asMap() {
-    data.setType(idx(), MinorType.MAP);
-    return getMapWriter();
+  public StructWriter asStruct() {
+    data.setType(idx(), MinorType.STRUCT);
+    return getStructWriter();
   }
 
   private ListWriter getListWriter() {
     if (listWriter == null) {
-      listWriter = new UnionListWriter(data.getList(), nullableMapWriterFactory);
+      listWriter = new UnionListWriter(data.getList(), nullableStructWriterFactory);
       listWriter.setPosition(idx());
       writers.add(listWriter);
     }
@@ -112,8 +112,8 @@ public ListWriter asList() {
 
   BaseWriter getWriter(MinorType minorType) {
     switch (minorType) {
-    case MAP:
-      return getMapWriter();
+    case STRUCT:
+      return getStructWriter();
     case LIST:
       return getListWriter();
     <#list vv.types as type>
@@ -174,10 +174,10 @@ public void writeNull() {
   }
 
   @Override
-  public MapWriter map() {
+  public StructWriter struct() {
     data.setType(idx(), MinorType.LIST);
     getListWriter().setPosition(idx());
-    return getListWriter().map();
+    return getListWriter().struct();
   }
 
   @Override
@@ -189,16 +189,16 @@ public ListWriter list() {
 
   @Override
   public ListWriter list(String name) {
-    data.setType(idx(), MinorType.MAP);
-    getMapWriter().setPosition(idx());
-    return getMapWriter().list(name);
+    data.setType(idx(), MinorType.STRUCT);
+    getStructWriter().setPosition(idx());
+    return getStructWriter().list(name);
   }
 
   @Override
-  public MapWriter map(String name) {
-    data.setType(idx(), MinorType.MAP);
-    getMapWriter().setPosition(idx());
-    return getMapWriter().map(name);
+  public StructWriter struct(String name) {
+    data.setType(idx(), MinorType.STRUCT);
+    getStructWriter().setPosition(idx());
+    return getStructWriter().struct(name);
   }
 
   <#list vv.types as type><#list type.minor as minor>
@@ -209,9 +209,9 @@ public MapWriter map(String name) {
   <#if !minor.typeParams?? >
   @Override
   public ${capName}Writer ${lowerName}(String name) {
-    data.setType(idx(), MinorType.MAP);
-    getMapWriter().setPosition(idx());
-    return getMapWriter().${lowerName}(name);
+    data.setType(idx(), MinorType.STRUCT);
+    getStructWriter().setPosition(idx());
+    return getStructWriter().${lowerName}(name);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index d1190ceb7b672..c32d20f180dd8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -24,7 +24,6 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index db0ff86df47a9..a99f3c8cdb7ad 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -100,8 +100,8 @@ protected boolean supportsDirectRead() {
   // return the child vector's ordinal in the composite container
   public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
 
-  public NullableMapVector addOrGetMap(String name) {
-    return addOrGet(name, FieldType.nullable(new Struct()), NullableMapVector.class);
+  public StructVector addOrGetStruct(String name) {
+    return addOrGet(name, FieldType.nullable(new Struct()), StructVector.class);
   }
 
   public ListVector addOrGetList(String name) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java
similarity index 95%
rename from java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java
index 1b20b09d81b0b..8ea8b379f92f2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java
@@ -37,15 +37,15 @@
 import io.netty.buffer.ArrowBuf;
 
 /*
- * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
+ * Base class for StructVectors. Currently used by NonNullableStructVector
  */
-public abstract class AbstractMapVector extends AbstractContainerVector {
+public abstract class AbstractStructVector extends AbstractContainerVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
 
   // Maintains a map with key as field name and value is the vector itself
   private final MapWithOrdinal<String, FieldVector> vectors = new MapWithOrdinal<>();
 
-  protected AbstractMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+  protected AbstractStructVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator, callBack);
   }
 
@@ -257,9 +257,9 @@ public Iterator<ValueVector> iterator() {
   public List<ValueVector> getPrimitiveVectors() {
     final List<ValueVector> primitiveVectors = Lists.newArrayList();
     for (final ValueVector v : vectors.values()) {
-      if (v instanceof AbstractMapVector) {
-        AbstractMapVector mapVector = (AbstractMapVector) v;
-        primitiveVectors.addAll(mapVector.getPrimitiveVectors());
+      if (v instanceof AbstractStructVector) {
+        AbstractStructVector structVector = (AbstractStructVector) v;
+        primitiveVectors.addAll(structVector.getPrimitiveVectors());
       } else {
         primitiveVectors.add(v);
       }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
similarity index 85%
rename from java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
index 6eab6ef3dcdcc..aec06b60c1803 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
@@ -28,7 +28,6 @@
 
 import javax.annotation.Nullable;
 
-import com.google.common.base.Preconditions;
 import com.google.common.collect.Ordering;
 import com.google.common.primitives.Ints;
 
@@ -36,7 +35,7 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.SingleStructReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -47,25 +46,24 @@
 import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.TransferPair;
 
-public class MapVector extends AbstractMapVector {
-  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
+public class NonNullableStructVector extends AbstractStructVector {
 
-  public static MapVector empty(String name, BufferAllocator allocator) {
+  public static NonNullableStructVector empty(String name, BufferAllocator allocator) {
     FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
-    return new MapVector(name, allocator, fieldType, null);
+    return new NonNullableStructVector(name, allocator, fieldType, null);
   }
 
-  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
+  private final SingleStructReaderImpl reader = new SingleStructReaderImpl(this);
   protected final FieldType fieldType;
   public int valueCount;
 
   // deprecated, use FieldType or static constructor instead
   @Deprecated
-  public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
+  public NonNullableStructVector(String name, BufferAllocator allocator, CallBack callBack) {
     this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
   }
 
-  public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+  public NonNullableStructVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
     super(name, allocator, callBack);
     this.fieldType = checkNotNull(fieldType);
     this.valueCount = 0;
@@ -76,11 +74,11 @@ public FieldReader getReader() {
     return reader;
   }
 
-  transient private MapTransferPair ephPair;
+  transient private StructTransferPair ephPair;
 
-  public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, NonNullableStructVector from) {
     if (ephPair == null || ephPair.from != from) {
-      ephPair = (MapTransferPair) from.makeTransferPair(this);
+      ephPair = (StructTransferPair) from.makeTransferPair(this);
     }
     ephPair.copyValueSafe(fromIndex, thisIndex);
   }
@@ -150,29 +148,29 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false);
+    return new StructTransferPair(this, new NonNullableStructVector(name, allocator, fieldType, callBack), false);
   }
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new MapTransferPair(this, (MapVector) to);
+    return new StructTransferPair(this, (NonNullableStructVector) to);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false);
+    return new StructTransferPair(this, new NonNullableStructVector(ref, allocator, fieldType, callBack), false);
   }
 
-  protected static class MapTransferPair implements TransferPair {
+  protected static class StructTransferPair implements TransferPair {
     private final TransferPair[] pairs;
-    private final MapVector from;
-    private final MapVector to;
+    private final NonNullableStructVector from;
+    private final NonNullableStructVector to;
 
-    public MapTransferPair(MapVector from, MapVector to) {
+    public StructTransferPair(NonNullableStructVector from, NonNullableStructVector to) {
       this(from, to, true);
     }
 
-    protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
+    protected StructTransferPair(NonNullableStructVector from, NonNullableStructVector to, boolean allocate) {
       this.from = from;
       this.to = to;
       this.pairs = new TransferPair[from.size()];
@@ -291,7 +289,7 @@ public void setValueCount(int valueCount) {
     for (final ValueVector v : getChildren()) {
       v.setValueCount(valueCount);
     }
-    MapVector.this.valueCount = valueCount;
+    NonNullableStructVector.this.valueCount = valueCount;
   }
 
   @Override
@@ -321,7 +319,7 @@ public Field getField() {
 
   @Override
   public MinorType getMinorType() {
-    return MinorType.MAP;
+    return MinorType.STRUCT;
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
similarity index 90%
rename from java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
rename to java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
index fb84d23b5074e..26bb4bbfac1ee 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
@@ -21,8 +21,6 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
 import com.google.common.collect.ObjectArrays;
@@ -31,8 +29,8 @@
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
-import org.apache.arrow.vector.complex.impl.NullableMapWriter;
+import org.apache.arrow.vector.complex.impl.NullableStructReaderImpl;
+import org.apache.arrow.vector.complex.impl.NullableStructWriter;
 import org.apache.arrow.vector.holders.ComplexHolder;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -44,32 +42,32 @@
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
 
-public class NullableMapVector extends MapVector implements FieldVector {
+public class StructVector extends NonNullableStructVector implements FieldVector {
 
-  public static NullableMapVector empty(String name, BufferAllocator allocator) {
+  public static StructVector empty(String name, BufferAllocator allocator) {
     FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
-    return new NullableMapVector(name, allocator, fieldType, null);
+    return new StructVector(name, allocator, fieldType, null);
   }
 
-  private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
-  private final NullableMapWriter writer = new NullableMapWriter(this);
+  private final NullableStructReaderImpl reader = new NullableStructReaderImpl(this);
+  private final NullableStructWriter writer = new NullableStructWriter(this);
 
   protected ArrowBuf validityBuffer;
   private int validityAllocationSizeInBytes;
 
   // deprecated, use FieldType or static constructor instead
   @Deprecated
-  public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+  public StructVector(String name, BufferAllocator allocator, CallBack callBack) {
     this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack);
   }
 
   // deprecated, use FieldType or static constructor instead
   @Deprecated
-  public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
+  public StructVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
     this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack);
   }
 
-  public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+  public StructVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
     super(name, checkNotNull(allocator), fieldType, callBack);
     this.validityBuffer = allocator.getEmpty();
     this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION);
@@ -117,39 +115,39 @@ public List<BufferBacked> getFieldInnerVectors() {
   }
 
   @Override
-  public NullableMapReaderImpl getReader() {
+  public NullableStructReaderImpl getReader() {
     return reader;
   }
 
-  public NullableMapWriter getWriter() {
+  public NullableStructWriter getWriter() {
     return writer;
   }
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false);
+    return new NullableStructTransferPair(this, new StructVector(name, allocator, fieldType, null), false);
   }
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new NullableMapTransferPair(this, (NullableMapVector) to, true);
+    return new NullableStructTransferPair(this, (StructVector) to, true);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false);
+    return new NullableStructTransferPair(this, new StructVector(ref, allocator, fieldType, null), false);
   }
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
-    return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false);
+    return new NullableStructTransferPair(this, new StructVector(ref, allocator, fieldType, callBack), false);
   }
 
-  protected class NullableMapTransferPair extends MapTransferPair {
+  protected class NullableStructTransferPair extends StructTransferPair {
 
-    private NullableMapVector target;
+    private StructVector target;
 
-    protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, boolean allocate) {
+    protected NullableStructTransferPair(StructVector from, StructVector to, boolean allocate) {
       super(from, to, allocate);
       this.target = to;
     }
@@ -182,7 +180,7 @@ public void splitAndTransfer(int startIndex, int length) {
   /*
    * transfer the validity.
    */
-  private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) {
+  private void splitAndTransferValidityBuffer(int startIndex, int length, StructVector target) {
     assert startIndex + length <= valueCount;
     int firstByteSource = BitVectorHelper.byteIndex(startIndex);
     int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index a6960238b9165..e12dd143e0f9b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -19,8 +19,8 @@
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.complex.StateTool;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -30,32 +30,32 @@
 public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
 
-  private NullableMapWriter mapRoot;
+  private NullableStructWriter structRoot;
   private UnionListWriter listRoot;
-  private final MapVector container;
+  private final NonNullableStructVector container;
 
   Mode mode = Mode.INIT;
   private final String name;
   private final boolean unionEnabled;
-  private final NullableMapWriterFactory nullableMapWriterFactory;
+  private final NullableStructWriterFactory nullableStructWriterFactory;
 
-  private enum Mode {INIT, MAP, LIST}
+  private enum Mode {INIT, STRUCT, LIST}
 
   ;
 
-  public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled, boolean caseSensitive) {
+  public ComplexWriterImpl(String name, NonNullableStructVector container, boolean unionEnabled, boolean caseSensitive) {
     this.name = name;
     this.container = container;
     this.unionEnabled = unionEnabled;
-    nullableMapWriterFactory = caseSensitive ? NullableMapWriterFactory.getNullableCaseSensitiveMapWriterFactoryInstance() :
-        NullableMapWriterFactory.getNullableMapWriterFactoryInstance();
+    nullableStructWriterFactory = caseSensitive ? NullableStructWriterFactory.getNullableCaseSensitiveStructWriterFactoryInstance() :
+        NullableStructWriterFactory.getNullableStructWriterFactoryInstance();
   }
 
-  public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled) {
+  public ComplexWriterImpl(String name, NonNullableStructVector container, boolean unionEnabled) {
     this(name, container, unionEnabled, false);
   }
 
-  public ComplexWriterImpl(String name, MapVector container) {
+  public ComplexWriterImpl(String name, NonNullableStructVector container) {
     this(name, container, false);
   }
 
@@ -81,7 +81,7 @@ public void reset() {
   @Override
   public void close() throws Exception {
     clear();
-    mapRoot.close();
+    structRoot.close();
     if (listRoot != null) {
       listRoot.close();
     }
@@ -90,8 +90,8 @@ public void close() throws Exception {
   @Override
   public void clear() {
     switch (mode) {
-      case MAP:
-        mapRoot.clear();
+      case STRUCT:
+        structRoot.clear();
         break;
       case LIST:
         listRoot.clear();
@@ -102,8 +102,8 @@ public void clear() {
   @Override
   public void setValueCount(int count) {
     switch (mode) {
-      case MAP:
-        mapRoot.setValueCount(count);
+      case STRUCT:
+        structRoot.setValueCount(count);
         break;
       case LIST:
         listRoot.setValueCount(count);
@@ -115,8 +115,8 @@ public void setValueCount(int count) {
   public void setPosition(int index) {
     super.setPosition(index);
     switch (mode) {
-      case MAP:
-        mapRoot.setPosition(index);
+      case STRUCT:
+        structRoot.setPosition(index);
         break;
       case LIST:
         listRoot.setPosition(index);
@@ -125,53 +125,53 @@ public void setPosition(int index) {
   }
 
 
-  public MapWriter directMap() {
+  public StructWriter directStruct() {
     Preconditions.checkArgument(name == null);
 
     switch (mode) {
 
       case INIT:
-        mapRoot = nullableMapWriterFactory.build((NullableMapVector) container);
-        mapRoot.setPosition(idx());
-        mode = Mode.MAP;
+        structRoot = nullableStructWriterFactory.build((StructVector) container);
+        structRoot.setPosition(idx());
+        mode = Mode.STRUCT;
         break;
 
-      case MAP:
+      case STRUCT:
         break;
 
       default:
-        check(Mode.INIT, Mode.MAP);
+        check(Mode.INIT, Mode.STRUCT);
     }
 
-    return mapRoot;
+    return structRoot;
   }
 
   @Override
-  public MapWriter rootAsMap() {
+  public StructWriter rootAsStruct() {
     switch (mode) {
 
       case INIT:
         // TODO allow dictionaries in complex types
-        NullableMapVector map = container.addOrGetMap(name);
-        mapRoot = nullableMapWriterFactory.build(map);
-        mapRoot.setPosition(idx());
-        mode = Mode.MAP;
+        StructVector struct = container.addOrGetStruct(name);
+        structRoot = nullableStructWriterFactory.build(struct);
+        structRoot.setPosition(idx());
+        mode = Mode.STRUCT;
         break;
 
-      case MAP:
+      case STRUCT:
         break;
 
       default:
-        check(Mode.INIT, Mode.MAP);
+        check(Mode.INIT, Mode.STRUCT);
     }
 
-    return mapRoot;
+    return structRoot;
   }
 
   @Override
   public void allocate() {
-    if (mapRoot != null) {
-      mapRoot.allocate();
+    if (structRoot != null) {
+      structRoot.allocate();
     } else if (listRoot != null) {
       listRoot.allocate();
     }
@@ -188,7 +188,7 @@ public ListWriter rootAsList() {
         if (container.size() > vectorCount) {
           listVector.allocateNew();
         }
-        listRoot = new UnionListWriter(listVector, nullableMapWriterFactory);
+        listRoot = new UnionListWriter(listVector, nullableStructWriterFactory);
         listRoot.setPosition(idx());
         mode = Mode.LIST;
         break;
@@ -197,7 +197,7 @@ public ListWriter rootAsList() {
         break;
 
       default:
-        check(Mode.INIT, Mode.MAP);
+        check(Mode.INIT, Mode.STRUCT);
     }
 
     return listRoot;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableStructReaderImpl.java
similarity index 55%
rename from java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
rename to java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableStructReaderImpl.java
index 06b0f4d43272f..bc4f876bd41b9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableStructReaderImpl.java
@@ -19,39 +19,39 @@
 
 package org.apache.arrow.vector.complex.impl;
 
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.StructVector;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.types.pojo.Field;
 
-public class NullableMapReaderImpl extends SingleMapReaderImpl {
+public class NullableStructReaderImpl extends SingleStructReaderImpl {
 
-  private NullableMapVector nullableMapVector;
+  private StructVector nullableStructVector;
 
-  public NullableMapReaderImpl(MapVector vector) {
-    super((NullableMapVector) vector);
-    this.nullableMapVector = (NullableMapVector) vector;
+  public NullableStructReaderImpl(NonNullableStructVector vector) {
+    super(vector);
+    this.nullableStructVector = (StructVector) vector;
   }
 
   @Override
   public Field getField() {
-    return nullableMapVector.getField();
+    return nullableStructVector.getField();
   }
 
   @Override
-  public void copyAsValue(MapWriter writer) {
-    NullableMapWriter impl = (NullableMapWriter) writer;
-    impl.container.copyFromSafe(idx(), impl.idx(), nullableMapVector);
+  public void copyAsValue(StructWriter writer) {
+    NullableStructWriter impl = (NullableStructWriter) writer;
+    impl.container.copyFromSafe(idx(), impl.idx(), nullableStructVector);
   }
 
   @Override
-  public void copyAsField(String name, MapWriter writer) {
-    NullableMapWriter impl = (NullableMapWriter) writer.map(name);
-    impl.container.copyFromSafe(idx(), impl.idx(), nullableMapVector);
+  public void copyAsField(String name, StructWriter writer) {
+    NullableStructWriter impl = (NullableStructWriter) writer.struct(name);
+    impl.container.copyFromSafe(idx(), impl.idx(), nullableStructVector);
   }
 
   @Override
   public boolean isSet() {
-    return !nullableMapVector.isNull(idx());
+    return !nullableStructVector.isNull(idx());
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableStructWriterFactory.java
similarity index 55%
rename from java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
rename to java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableStructWriterFactory.java
index d2dcb2374d0e7..7499687c94729 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableStructWriterFactory.java
@@ -18,26 +18,26 @@
 
 package org.apache.arrow.vector.complex.impl;
 
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.StructVector;
 
-public class NullableMapWriterFactory {
+public class NullableStructWriterFactory {
   private final boolean caseSensitive;
-  private static final NullableMapWriterFactory nullableMapWriterFactory = new NullableMapWriterFactory(false);
-  private static final NullableMapWriterFactory nullableCaseSensitiveWriterFactory = new NullableMapWriterFactory(true);
+  private static final NullableStructWriterFactory nullableStructWriterFactory = new NullableStructWriterFactory(false);
+  private static final NullableStructWriterFactory nullableCaseSensitiveWriterFactory = new NullableStructWriterFactory(true);
 
-  public NullableMapWriterFactory(boolean caseSensitive) {
+  public NullableStructWriterFactory(boolean caseSensitive) {
     this.caseSensitive = caseSensitive;
   }
 
-  public NullableMapWriter build(NullableMapVector container) {
-    return this.caseSensitive ? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container);
+  public NullableStructWriter build(StructVector container) {
+    return this.caseSensitive ? new NullableCaseSensitiveStructWriter(container) : new NullableStructWriter(container);
   }
 
-  public static NullableMapWriterFactory getNullableMapWriterFactoryInstance() {
-    return nullableMapWriterFactory;
+  public static NullableStructWriterFactory getNullableStructWriterFactoryInstance() {
+    return nullableStructWriterFactory;
   }
 
-  public static NullableMapWriterFactory getNullableCaseSensitiveMapWriterFactoryInstance() {
+  public static NullableStructWriterFactory getNullableCaseSensitiveStructWriterFactoryInstance() {
     return nullableCaseSensitiveWriterFactory;
   }
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 5bd439cac4269..c2121f143a75f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -21,9 +21,9 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.complex.AbstractMapVector;
+import org.apache.arrow.vector.complex.AbstractStructVector;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -35,13 +35,13 @@
  * This FieldWriter implementation delegates all FieldWriter API calls to an inner FieldWriter. This inner field writer
  * can start as a specific type, and this class will promote the writer to a UnionWriter if a call is made that the specifically
  * typed writer cannot handle. A new UnionVector is created, wrapping the original vector, and replaces the original vector
- * in the parent vector, which can be either an AbstractMapVector or a ListVector.
+ * in the parent vector, which can be either an AbstractStructVector or a ListVector.
  */
 public class PromotableWriter extends AbstractPromotableFieldWriter {
 
-  private final AbstractMapVector parentContainer;
+  private final AbstractStructVector parentContainer;
   private final ListVector listVector;
-  private final NullableMapWriterFactory nullableMapWriterFactory;
+  private final NullableStructWriterFactory nullableStructWriterFactory;
   private int position;
 
   private enum State {
@@ -54,25 +54,25 @@ private enum State {
   private State state;
   private FieldWriter writer;
 
-  public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
-    this(v, parentContainer, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
+  public PromotableWriter(ValueVector v, AbstractStructVector parentContainer) {
+    this(v, parentContainer, NullableStructWriterFactory.getNullableStructWriterFactoryInstance());
   }
 
-  public PromotableWriter(ValueVector v, AbstractMapVector parentContainer, NullableMapWriterFactory nullableMapWriterFactory) {
+  public PromotableWriter(ValueVector v, AbstractStructVector parentContainer, NullableStructWriterFactory nullableStructWriterFactory) {
     this.parentContainer = parentContainer;
     this.listVector = null;
-    this.nullableMapWriterFactory = nullableMapWriterFactory;
+    this.nullableStructWriterFactory = nullableStructWriterFactory;
     init(v);
   }
 
   public PromotableWriter(ValueVector v, ListVector listVector) {
-    this(v, listVector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance());
+    this(v, listVector, NullableStructWriterFactory.getNullableStructWriterFactoryInstance());
   }
 
-  public PromotableWriter(ValueVector v, ListVector listVector, NullableMapWriterFactory nullableMapWriterFactory) {
+  public PromotableWriter(ValueVector v, ListVector listVector, NullableStructWriterFactory nullableStructWriterFactory) {
     this.listVector = listVector;
     this.parentContainer = null;
-    this.nullableMapWriterFactory = nullableMapWriterFactory;
+    this.nullableStructWriterFactory = nullableStructWriterFactory;
     init(v);
   }
 
@@ -80,7 +80,7 @@ private void init(ValueVector v) {
     if (v instanceof UnionVector) {
       state = State.UNION;
       unionVector = (UnionVector) v;
-      writer = new UnionWriter(unionVector, nullableMapWriterFactory);
+      writer = new UnionWriter(unionVector, nullableStructWriterFactory);
     } else if (v instanceof ZeroVector) {
       state = State.UNTYPED;
     } else {
@@ -93,14 +93,14 @@ private void setWriter(ValueVector v) {
     vector = v;
     type = v.getMinorType();
     switch (type) {
-      case MAP:
-        writer = nullableMapWriterFactory.build((NullableMapVector) vector);
+      case STRUCT:
+        writer = nullableStructWriterFactory.build((StructVector) vector);
         break;
       case LIST:
-        writer = new UnionListWriter((ListVector) vector, nullableMapWriterFactory);
+        writer = new UnionListWriter((ListVector) vector, nullableStructWriterFactory);
         break;
       case UNION:
-        writer = new UnionWriter((UnionVector) vector, nullableMapWriterFactory);
+        writer = new UnionWriter((UnionVector) vector, nullableStructWriterFactory);
         break;
       default:
         writer = type.getNewFieldWriter(vector);
@@ -139,8 +139,8 @@ protected FieldWriter getWriter(MinorType type) {
   }
 
   @Override
-  public boolean isEmptyMap() {
-    return writer.isEmptyMap();
+  public boolean isEmptyStruct() {
+    return writer.isEmptyStruct();
   }
 
   protected FieldWriter getWriter() {
@@ -159,7 +159,7 @@ private FieldWriter promoteToUnion() {
       unionVector = listVector.promoteToUnion();
     }
     unionVector.addVector((FieldVector) tp.getTo());
-    writer = new UnionWriter(unionVector, nullableMapWriterFactory);
+    writer = new UnionWriter(unionVector, nullableStructWriterFactory);
     writer.setPosition(idx());
     for (int i = 0; i <= idx(); i++) {
       unionVector.setType(i, vector.getMinorType());
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
index f2b46ab98db7f..e8c6ce5a409d8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
@@ -24,7 +24,7 @@
 import org.apache.arrow.vector.complex.AbstractContainerVector;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
 
 @SuppressWarnings("unused")
@@ -78,7 +78,7 @@ public void copyAsValue(ListWriter writer) {
   }
 
   @Override
-  public void copyAsField(String name, MapWriter writer) {
+  public void copyAsField(String name, StructWriter writer) {
     throw new UnsupportedOperationException("Generic list copying not yet supported.  Please resolve to typed list.");
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleStructReaderImpl.java
similarity index 81%
rename from java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
rename to java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleStructReaderImpl.java
index c77ca4e8f23ba..a5535bdf057e7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleStructReaderImpl.java
@@ -23,21 +23,21 @@
 import java.util.Map;
 
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
 import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
 
 import com.google.common.collect.Maps;
 
 @SuppressWarnings("unused")
-public class SingleMapReaderImpl extends AbstractFieldReader {
+public class SingleStructReaderImpl extends AbstractFieldReader {
 
-  private final MapVector vector;
+  private final NonNullableStructVector vector;
   private final Map<String, FieldReader> fields = Maps.newHashMap();
 
-  public SingleMapReaderImpl(MapVector vector) {
+  public SingleStructReaderImpl(NonNullableStructVector vector) {
     this.vector = vector;
   }
 
@@ -83,7 +83,7 @@ public Object readObject() {
 
   @Override
   public MinorType getMinorType() {
-    return MinorType.MAP;
+    return MinorType.STRUCT;
   }
 
   @Override
@@ -97,14 +97,14 @@ public java.util.Iterator<String> iterator() {
   }
 
   @Override
-  public void copyAsValue(MapWriter writer) {
-    SingleMapWriter impl = (SingleMapWriter) writer;
+  public void copyAsValue(StructWriter writer) {
+    SingleStructWriter impl = (SingleStructWriter) writer;
     impl.container.copyFromSafe(idx(), impl.idx(), vector);
   }
 
   @Override
-  public void copyAsField(String name, MapWriter writer) {
-    SingleMapWriter impl = (SingleMapWriter) writer.map(name);
+  public void copyAsField(String name, StructWriter writer) {
+    SingleStructWriter impl = (SingleStructWriter) writer.struct(name);
     impl.container.copyFromSafe(idx(), impl.idx(), vector);
   }
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java
similarity index 59%
rename from java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java
rename to java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java
index 0d860b6a04115..e6179fdcd6a12 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/MapOrListWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java
@@ -19,7 +19,7 @@
 package org.apache.arrow.vector.complex.impl;
 
 import org.apache.arrow.vector.complex.writer.BaseWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapOrListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructOrListWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.BitWriter;
 import org.apache.arrow.vector.complex.writer.Float4Writer;
@@ -28,54 +28,54 @@
 import org.apache.arrow.vector.complex.writer.VarBinaryWriter;
 import org.apache.arrow.vector.complex.writer.VarCharWriter;
 
-public class MapOrListWriterImpl implements MapOrListWriter {
+public class StructOrListWriterImpl implements StructOrListWriter {
 
-  public final BaseWriter.MapWriter map;
+  public final BaseWriter.StructWriter struct;
   public final BaseWriter.ListWriter list;
 
-  public MapOrListWriterImpl(final BaseWriter.MapWriter writer) {
-    this.map = writer;
+  public StructOrListWriterImpl(final BaseWriter.StructWriter writer) {
+    this.struct = writer;
     this.list = null;
   }
 
-  public MapOrListWriterImpl(final BaseWriter.ListWriter writer) {
-    this.map = null;
+  public StructOrListWriterImpl(final BaseWriter.ListWriter writer) {
+    this.struct = null;
     this.list = writer;
   }
 
   public void start() {
-    if (map != null) {
-      map.start();
+    if (struct != null) {
+      struct.start();
     } else {
       list.startList();
     }
   }
 
   public void end() {
-    if (map != null) {
-      map.end();
+    if (struct != null) {
+      struct.end();
     } else {
       list.endList();
     }
   }
 
-  public MapOrListWriter map(final String name) {
-    assert map != null;
-    return new MapOrListWriterImpl(map.map(name));
+  public StructOrListWriter struct(final String name) {
+    assert struct != null;
+    return new StructOrListWriterImpl(struct.struct(name));
   }
 
-  public MapOrListWriter listoftmap(final String name) {
+  public StructOrListWriter listoftstruct(final String name) {
     assert list != null;
-    return new MapOrListWriterImpl(list.map());
+    return new StructOrListWriterImpl(list.struct());
   }
 
-  public MapOrListWriter list(final String name) {
-    assert map != null;
-    return new MapOrListWriterImpl(map.list(name));
+  public StructOrListWriter list(final String name) {
+    assert struct != null;
+    return new StructOrListWriterImpl(struct.list(name));
   }
 
-  public boolean isMapWriter() {
-    return map != null;
+  public boolean isStructWriter() {
+    return struct != null;
   }
 
   public boolean isListWriter() {
@@ -83,31 +83,31 @@ public boolean isListWriter() {
   }
 
   public VarCharWriter varChar(final String name) {
-    return (map != null) ? map.varChar(name) : list.varChar();
+    return (struct != null) ? struct.varChar(name) : list.varChar();
   }
 
   public IntWriter integer(final String name) {
-    return (map != null) ? map.integer(name) : list.integer();
+    return (struct != null) ? struct.integer(name) : list.integer();
   }
 
   public BigIntWriter bigInt(final String name) {
-    return (map != null) ? map.bigInt(name) : list.bigInt();
+    return (struct != null) ? struct.bigInt(name) : list.bigInt();
   }
 
   public Float4Writer float4(final String name) {
-    return (map != null) ? map.float4(name) : list.float4();
+    return (struct != null) ? struct.float4(name) : list.float4();
   }
 
   public Float8Writer float8(final String name) {
-    return (map != null) ? map.float8(name) : list.float8();
+    return (struct != null) ? struct.float8(name) : list.float8();
   }
 
   public BitWriter bit(final String name) {
-    return (map != null) ? map.bit(name) : list.bit();
+    return (struct != null) ? struct.bit(name) : list.bit();
   }
 
   public VarBinaryWriter binary(final String name) {
-    return (map != null) ? map.varBinary(name) : list.varBinary();
+    return (struct != null) ? struct.varBinary(name) : list.varBinary();
   }
 
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
index df142c7819d7c..ce2f6cd2132ce 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/reader/FieldReader.java
@@ -19,11 +19,11 @@
 package org.apache.arrow.vector.complex.reader;
 
 import org.apache.arrow.vector.complex.reader.BaseReader.ListReader;
-import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.reader.BaseReader.StructReader;
 import org.apache.arrow.vector.complex.reader.BaseReader.RepeatedListReader;
-import org.apache.arrow.vector.complex.reader.BaseReader.RepeatedMapReader;
+import org.apache.arrow.vector.complex.reader.BaseReader.RepeatedStructReader;
 import org.apache.arrow.vector.complex.reader.BaseReader.ScalarReader;
 
 
-public interface FieldReader extends MapReader, ListReader, ScalarReader, RepeatedMapReader, RepeatedListReader {
+public interface FieldReader extends StructReader, ListReader, ScalarReader, RepeatedStructReader, RepeatedListReader {
 }
\ No newline at end of file
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
index a2a1f5d000a8a..2c19cbca5b0bb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java
@@ -19,10 +19,10 @@
 package org.apache.arrow.vector.complex.writer;
 
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ScalarWriter;
 
-public interface FieldWriter extends MapWriter, ListWriter, ScalarWriter {
+public interface FieldWriter extends StructWriter, ListWriter, ScalarWriter {
   void allocate();
 
   void clear();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
index 08df7b6d93176..ccd2759f7548e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -136,7 +136,7 @@ public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
   private static void validateType(MinorType type) {
     // byte arrays don't work as keys in our dictionary map - we could wrap them with something to
     // implement equals and hashcode if we want that functionality
-    if (type == MinorType.VARBINARY || type == MinorType.FIXEDSIZEBINARY || type == MinorType.LIST || type == MinorType.MAP || type == MinorType.UNION) {
+    if (type == MinorType.VARBINARY || type == MinorType.FIXEDSIZEBINARY || type == MinorType.LIST || type == MinorType.STRUCT || type == MinorType.UNION) {
       throw new IllegalArgumentException("Dictionary encoding for complex types not implemented: type " + type);
     }
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedStructHolder.java
similarity index 93%
rename from java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java
rename to java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedStructHolder.java
index c400b4dc49a80..50c2c8acb8d09 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedMapHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/RepeatedStructHolder.java
@@ -18,7 +18,7 @@
 
 package org.apache.arrow.vector.holders;
 
-public final class RepeatedMapHolder implements ValueHolder {
+public final class RepeatedStructHolder implements ValueHolder {
   public int start;
   public int end;
 }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 4adbefbeb5ac4..ba62d3fcb42c3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -59,7 +59,7 @@
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.BitWriterImpl;
@@ -72,7 +72,7 @@
 import org.apache.arrow.vector.complex.impl.IntWriterImpl;
 import org.apache.arrow.vector.complex.impl.IntervalDayWriterImpl;
 import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl;
-import org.apache.arrow.vector.complex.impl.NullableMapWriter;
+import org.apache.arrow.vector.complex.impl.NullableStructWriter;
 import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeMicroWriterImpl;
 import org.apache.arrow.vector.complex.impl.TimeMilliWriterImpl;
@@ -131,15 +131,15 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
         return null;
       }
     },
-    MAP(Struct.INSTANCE) {
+    STRUCT(Struct.INSTANCE) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
-        return new NullableMapVector(name, allocator, fieldType, schemaChangeCallback);
+        return new StructVector(name, allocator, fieldType, schemaChangeCallback);
       }
 
       @Override
       public FieldWriter getNewFieldWriter(ValueVector vector) {
-        return new NullableMapWriter((NullableMapVector) vector);
+        return new NullableStructWriter((StructVector) vector);
       }
     },
     TINYINT(new Int(8, true)) {
@@ -553,7 +553,7 @@ public MinorType visit(Null type) {
 
       @Override
       public MinorType visit(Struct type) {
-        return MinorType.MAP;
+        return MinorType.STRUCT;
       }
 
       @Override
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java
similarity index 90%
rename from java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
rename to java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java
index 357df96aa2efc..556a47269196b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java
@@ -22,7 +22,7 @@
 import java.util.Map;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.junit.After;
@@ -31,7 +31,7 @@
 import org.junit.Test;
 
 
-public class TestMapVector {
+public class TestStructVector {
 
   private BufferAllocator allocator;
 
@@ -50,7 +50,7 @@ public void testFieldMetadata() throws Exception {
     Map<String, String> metadata = new HashMap<>();
     metadata.put("k1", "v1");
     FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata);
-    try (NullableMapVector vector = new NullableMapVector("map", allocator, type, null)) {
+    try (StructVector vector = new StructVector("struct", allocator, type, null)) {
       Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata());
     }
   }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index ca039c52f9715..391083e67ceca 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -26,9 +26,8 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.junit.After;
 import org.junit.Assert;
@@ -120,8 +119,8 @@ public void testListType() {
   }
 
   @Test
-  public void testMapType() {
-    try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) {
+  public void testStructType() {
+    try (final StructVector vector = StructVector.empty("", allocator)) {
       vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), IntVector.class);
 
       vector.setInitialCapacity(512);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
index 84ea9657f8c9f..88cba6b4377b2 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java
@@ -113,22 +113,22 @@ public void testListTypeReset() {
   }
 
   @Test
-  public void testMapTypeReset() {
-    try (final MapVector mapVector = new MapVector("Map", allocator, FieldType.nullable(MinorType.INT.getType()), null);
-         final NullableMapVector nullableMapVector = new NullableMapVector("NullableMap", allocator, FieldType.nullable(MinorType.INT.getType()), null)
+  public void testStructTypeReset() {
+    try (final NonNullableStructVector nonNullableStructVector = new NonNullableStructVector("Struct", allocator, FieldType.nullable(MinorType.INT.getType()), null);
+         final StructVector structVector = new StructVector("NullableStruct", allocator, FieldType.nullable(MinorType.INT.getType()), null)
     ) {
-      // MapVector
-      mapVector.allocateNewSafe();
-      IntVector mapChild = mapVector.addOrGet("child", FieldType.nullable(new Int(32, true)), IntVector.class);
-      mapChild.setNull(0);
-      mapVector.setValueCount(1);
-      resetVectorAndVerify(mapVector, mapVector.getBuffers(false));
-
-      // NullableMapVector
-      nullableMapVector.allocateNewSafe();
-      nullableMapVector.setNull(0);
-      nullableMapVector.setValueCount(1);
-      resetVectorAndVerify(nullableMapVector, nullableMapVector.getBuffers(false));
+      // NonNullableStructVector
+      nonNullableStructVector.allocateNewSafe();
+      IntVector structChild = nonNullableStructVector.addOrGet("child", FieldType.nullable(new Int(32, true)), IntVector.class);
+      structChild.setNull(0);
+      nonNullableStructVector.setValueCount(1);
+      resetVectorAndVerify(nonNullableStructVector, nonNullableStructVector.getBuffers(false));
+
+      // StructVector
+      structVector.allocateNewSafe();
+      structVector.setNull(0);
+      structVector.setValueCount(1);
+      resetVectorAndVerify(structVector, structVector.getBuffers(false));
     }
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 439a62725e49b..a9786a6c36aec 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -31,12 +31,12 @@
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
@@ -68,11 +68,11 @@ public void testUnloadLoad() throws IOException {
 
     try (
         BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", originalVectorsAllocator)) {
+        NonNullableStructVector parent = NonNullableStructVector.empty("parent", originalVectorsAllocator)) {
 
       // write some data
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
-      MapWriter rootWriter = writer.rootAsMap();
+      StructWriter rootWriter = writer.rootAsStruct();
       IntWriter intWriter = rootWriter.integer("int");
       BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
       for (int i = 0; i < count; i++) {
@@ -116,11 +116,11 @@ public void testUnloadLoadAddPadding() throws IOException {
     Schema schema;
     try (
         BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        MapVector parent = MapVector.empty("parent", originalVectorsAllocator)) {
+        NonNullableStructVector parent = NonNullableStructVector.empty("parent", originalVectorsAllocator)) {
 
       // write some data
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
-      MapWriter rootWriter = writer.rootAsMap();
+      StructWriter rootWriter = writer.rootAsStruct();
       ListWriter list = rootWriter.list("list");
       IntWriter intWriter = list.integer();
       for (int i = 0; i < count; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index b0d6cf555e98d..40b2b0d173564 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -24,14 +24,12 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.DirtyRootAllocator;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.complex.UnionVector;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
 import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -54,8 +52,8 @@ public void terminate() throws Exception {
   @Test
   public void testPromoteToUnion() throws Exception {
 
-    try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator);
-         final NullableMapVector v = container.addOrGetMap("test");
+    try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator);
+         final StructVector v = container.addOrGetStruct("test");
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
       container.allocateNew();
@@ -103,12 +101,12 @@ public void testPromoteToUnion() throws Exception {
 
       ComplexWriterImpl newWriter = new ComplexWriterImpl(EMPTY_SCHEMA_PATH, container);
 
-      MapWriter newMapWriter = newWriter.rootAsMap();
+      StructWriter newStructWriter = newWriter.rootAsStruct();
 
-      newMapWriter.start();
+      newStructWriter.start();
 
-      newMapWriter.setPosition(2);
-      newMapWriter.integer("A").writeInt(10);
+      newStructWriter.setPosition(2);
+      newStructWriter.integer("A").writeInt(10);
 
       Field childField1 = container.getField().getChildren().get(0).getChildren().get(0);
       Field childField2 = container.getField().getChildren().get(0).getChildren().get(1);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index b52769e44ef59..e6475c390aa11 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -33,25 +33,25 @@
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
-import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.SingleStructReaderImpl;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.impl.UnionReader;
 import org.apache.arrow.vector.complex.impl.UnionWriter;
-import org.apache.arrow.vector.complex.impl.SingleMapWriter;
+import org.apache.arrow.vector.complex.impl.SingleStructWriter;
 import org.apache.arrow.vector.complex.reader.IntReader;
 import org.apache.arrow.vector.complex.reader.Float8Reader;
 import org.apache.arrow.vector.complex.reader.Float4Reader;
 import org.apache.arrow.vector.complex.reader.BigIntReader;
-import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.reader.BaseReader.StructReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.holders.IntHolder;
 import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -81,8 +81,8 @@ public class TestComplexWriter {
 
   @Test
   public void simpleNestedTypes() {
-    MapVector parent = populateMapVector(null);
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    NonNullableStructVector parent = populateStructVector(null);
+    StructReader rootReader = new SingleStructReaderImpl(parent).reader("root");
     for (int i = 0; i < COUNT; i++) {
       rootReader.setPosition(i);
       Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
@@ -96,12 +96,12 @@ public void simpleNestedTypes() {
   public void transferPairSchemaChange() {
     SchemaChangeCallBack callBack1 = new SchemaChangeCallBack();
     SchemaChangeCallBack callBack2 = new SchemaChangeCallBack();
-    MapVector parent = populateMapVector(callBack1);
+    NonNullableStructVector parent = populateStructVector(callBack1);
 
     TransferPair tp = parent.getTransferPair("newVector", allocator, callBack2);
 
     ComplexWriter writer = new ComplexWriterImpl("newWriter", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     IntWriter intWriter = rootWriter.integer("newInt");
     intWriter.writeInt(1);
     writer.setValueCount(1);
@@ -111,10 +111,10 @@ public void transferPairSchemaChange() {
     assertFalse(callBack1.getSchemaChangedAndReset());
   }
 
-  private MapVector populateMapVector(CallBack callBack) {
-    MapVector parent = new MapVector("parent", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
+  private NonNullableStructVector populateStructVector(CallBack callBack) {
+    NonNullableStructVector parent = new NonNullableStructVector("parent", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     IntWriter intWriter = rootWriter.integer("int");
     BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
     for (int i = 0; i < COUNT; i++) {
@@ -128,73 +128,73 @@ private MapVector populateMapVector(CallBack callBack) {
   }
 
   @Test
-  public void nullableMap() {
-    try (MapVector mapVector = MapVector.empty("parent", allocator)) {
-      ComplexWriter writer = new ComplexWriterImpl("root", mapVector);
-      MapWriter rootWriter = writer.rootAsMap();
+  public void nullableStruct() {
+    try (NonNullableStructVector structVector = NonNullableStructVector.empty("parent", allocator)) {
+      ComplexWriter writer = new ComplexWriterImpl("root", structVector);
+      StructWriter rootWriter = writer.rootAsStruct();
       for (int i = 0; i < COUNT; i++) {
         rootWriter.start();
         if (i % 2 == 0) {
-          MapWriter mapWriter = rootWriter.map("map");
-          mapWriter.setPosition(i);
-          mapWriter.start();
-          mapWriter.bigInt("nested").writeBigInt(i);
-          mapWriter.end();
+          StructWriter structWriter = rootWriter.struct("struct");
+          structWriter.setPosition(i);
+          structWriter.start();
+          structWriter.bigInt("nested").writeBigInt(i);
+          structWriter.end();
         }
         rootWriter.end();
       }
       writer.setValueCount(COUNT);
-      checkNullableMap(mapVector);
+      checkNullableStruct(structVector);
     }
   }
 
   /**
-   * This test is similar to {@link #nullableMap()} ()} but we get the inner map writer once at the beginning
+   * This test is similar to {@link #nullableStruct()} ()} but we get the inner struct writer once at the beginning
    */
   @Test
-  public void nullableMap2() {
-    try (MapVector mapVector = MapVector.empty("parent", allocator)) {
-      ComplexWriter writer = new ComplexWriterImpl("root", mapVector);
-      MapWriter rootWriter = writer.rootAsMap();
-      MapWriter mapWriter = rootWriter.map("map");
+  public void nullableStruct2() {
+    try (NonNullableStructVector structVector = NonNullableStructVector.empty("parent", allocator)) {
+      ComplexWriter writer = new ComplexWriterImpl("root", structVector);
+      StructWriter rootWriter = writer.rootAsStruct();
+      StructWriter structWriter = rootWriter.struct("struct");
 
       for (int i = 0; i < COUNT; i++) {
         rootWriter.start();
         if (i % 2 == 0) {
-          mapWriter.setPosition(i);
-          mapWriter.start();
-          mapWriter.bigInt("nested").writeBigInt(i);
-          mapWriter.end();
+          structWriter.setPosition(i);
+          structWriter.start();
+          structWriter.bigInt("nested").writeBigInt(i);
+          structWriter.end();
         }
         rootWriter.end();
       }
       writer.setValueCount(COUNT);
-      checkNullableMap(mapVector);
+      checkNullableStruct(structVector);
     }
   }
 
-  private void checkNullableMap(MapVector mapVector) {
-    MapReader rootReader = new SingleMapReaderImpl(mapVector).reader("root");
+  private void checkNullableStruct(NonNullableStructVector structVector) {
+    StructReader rootReader = new SingleStructReaderImpl(structVector).reader("root");
     for (int i = 0; i < COUNT; i++) {
       rootReader.setPosition(i);
       assertTrue("index is set: " + i, rootReader.isSet());
-      FieldReader map = rootReader.reader("map");
+      FieldReader struct = rootReader.reader("struct");
       if (i % 2 == 0) {
-        assertTrue("index is set: " + i, map.isSet());
-        assertNotNull("index is set: " + i, map.readObject());
-        assertEquals(i, map.reader("nested").readLong().longValue());
+        assertTrue("index is set: " + i, struct.isSet());
+        assertNotNull("index is set: " + i, struct.readObject());
+        assertEquals(i, struct.reader("nested").readLong().longValue());
       } else {
-        assertFalse("index is not set: " + i, map.isSet());
-        assertNull("index is not set: " + i, map.readObject());
+        assertFalse("index is not set: " + i, struct.isSet());
+        assertNull("index is not set: " + i, struct.readObject());
       }
     }
   }
 
   @Test
   public void testList() {
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
 
     rootWriter.start();
     rootWriter.bigInt("int").writeBigInt(0);
@@ -209,7 +209,7 @@ public void testList() {
 
     writer.setValueCount(2);
 
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    StructReader rootReader = new SingleStructReaderImpl(parent).reader("root");
 
     rootReader.setPosition(0);
     assertTrue("row 0 list is not set", rootReader.reader("list").isSet());
@@ -278,18 +278,18 @@ public void listScalarTypeNullable() {
   }
 
   @Test
-  public void listMapType() {
+  public void listStructType() {
     ListVector listVector = ListVector.empty("list", allocator);
     listVector.allocateNew();
     UnionListWriter listWriter = new UnionListWriter(listVector);
-    MapWriter mapWriter = listWriter.map();
+    StructWriter structWriter = listWriter.struct();
     for (int i = 0; i < COUNT; i++) {
       listWriter.startList();
       for (int j = 0; j < i % 7; j++) {
-        mapWriter.start();
-        mapWriter.integer("int").writeInt(j);
-        mapWriter.bigInt("bigInt").writeBigInt(j);
-        mapWriter.end();
+        structWriter.start();
+        structWriter.integer("int").writeInt(j);
+        structWriter.bigInt("bigInt").writeBigInt(j);
+        structWriter.end();
       }
       listWriter.endList();
     }
@@ -471,9 +471,9 @@ public void simpleUnion() {
 
   @Test
   public void promotableWriter() {
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     for (int i = 0; i < 100; i++) {
       BigIntWriter bigIntWriter = rootWriter.bigInt("a");
       bigIntWriter.setPosition(i);
@@ -499,7 +499,7 @@ public void promotableWriter() {
     Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeID());
     Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeID());
     Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeID());
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    StructReader rootReader = new SingleStructReaderImpl(parent).reader("root");
     for (int i = 0; i < 100; i++) {
       rootReader.setPosition(i);
       FieldReader reader = rootReader.reader("a");
@@ -520,9 +520,9 @@ public void promotableWriter() {
    */
   @Test
   public void promotableWriterSchema() {
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     rootWriter.bigInt("a");
     rootWriter.varChar("a");
 
@@ -551,22 +551,22 @@ private Set<String> getFieldNames(List<Field> fields) {
   }
 
   @Test
-  public void mapWriterMixedCaseFieldNames() {
-    // test case-sensitive MapWriter
-    MapVector parent = MapVector.empty("parent", allocator);
+  public void structWriterMixedCaseFieldNames() {
+    // test case-sensitive StructWriter
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("rootCaseSensitive", parent, false, true);
-    MapWriter rootWriterCaseSensitive = writer.rootAsMap();
+    StructWriter rootWriterCaseSensitive = writer.rootAsStruct();
     rootWriterCaseSensitive.bigInt("int_field");
     rootWriterCaseSensitive.bigInt("Int_Field");
     rootWriterCaseSensitive.float4("float_field");
     rootWriterCaseSensitive.float4("Float_Field");
-    MapWriter mapFieldWriterCaseSensitive = rootWriterCaseSensitive.map("map_field");
-    mapFieldWriterCaseSensitive.varChar("char_field");
-    mapFieldWriterCaseSensitive.varChar("Char_Field");
+    StructWriter structFieldWriterCaseSensitive = rootWriterCaseSensitive.struct("struct_field");
+    structFieldWriterCaseSensitive.varChar("char_field");
+    structFieldWriterCaseSensitive.varChar("Char_Field");
     ListWriter listFieldWriterCaseSensitive = rootWriterCaseSensitive.list("list_field");
-    MapWriter listMapFieldWriterCaseSensitive = listFieldWriterCaseSensitive.map();
-    listMapFieldWriterCaseSensitive.bit("bit_field");
-    listMapFieldWriterCaseSensitive.bit("Bit_Field");
+    StructWriter listStructFieldWriterCaseSensitive = listFieldWriterCaseSensitive.struct();
+    listStructFieldWriterCaseSensitive.bit("bit_field");
+    listStructFieldWriterCaseSensitive.bit("Bit_Field");
 
     List<Field> fieldsCaseSensitive = parent.getField().getChildren().get(0).getChildren();
     Set<String> fieldNamesCaseSensitive = getFieldNames(fieldsCaseSensitive);
@@ -575,37 +575,37 @@ public void mapWriterMixedCaseFieldNames() {
     Assert.assertTrue(fieldNamesCaseSensitive.contains("Int_Field"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("float_field"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("Float_Field"));
-    Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field"));
-    Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field::char_field"));
-    Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field::Char_Field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("struct_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("struct_field::char_field"));
+    Assert.assertTrue(fieldNamesCaseSensitive.contains("struct_field::Char_Field"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::Bit_Field"));
 
-    // test case-insensitive MapWriter
+    // test case-insensitive StructWriter
     ComplexWriter writerCaseInsensitive = new ComplexWriterImpl("rootCaseInsensitive", parent, false, false);
-    MapWriter rootWriterCaseInsensitive = writerCaseInsensitive.rootAsMap();
+    StructWriter rootWriterCaseInsensitive = writerCaseInsensitive.rootAsStruct();
 
     rootWriterCaseInsensitive.bigInt("int_field");
     rootWriterCaseInsensitive.bigInt("Int_Field");
     rootWriterCaseInsensitive.float4("float_field");
     rootWriterCaseInsensitive.float4("Float_Field");
-    MapWriter mapFieldWriterCaseInsensitive = rootWriterCaseInsensitive.map("map_field");
-    mapFieldWriterCaseInsensitive.varChar("char_field");
-    mapFieldWriterCaseInsensitive.varChar("Char_Field");
+    StructWriter structFieldWriterCaseInsensitive = rootWriterCaseInsensitive.struct("struct_field");
+    structFieldWriterCaseInsensitive.varChar("char_field");
+    structFieldWriterCaseInsensitive.varChar("Char_Field");
     ListWriter listFieldWriterCaseInsensitive = rootWriterCaseInsensitive.list("list_field");
-    MapWriter listMapFieldWriterCaseInsensitive = listFieldWriterCaseInsensitive.map();
-    listMapFieldWriterCaseInsensitive.bit("bit_field");
-    listMapFieldWriterCaseInsensitive.bit("Bit_Field");
+    StructWriter listStructFieldWriterCaseInsensitive = listFieldWriterCaseInsensitive.struct();
+    listStructFieldWriterCaseInsensitive.bit("bit_field");
+    listStructFieldWriterCaseInsensitive.bit("Bit_Field");
 
     List<Field> fieldsCaseInsensitive = parent.getField().getChildren().get(1).getChildren();
     Set<String> fieldNamesCaseInsensitive = getFieldNames(fieldsCaseInsensitive);
     Assert.assertEquals(7, fieldNamesCaseInsensitive.size());
     Assert.assertTrue(fieldNamesCaseInsensitive.contains("int_field"));
     Assert.assertTrue(fieldNamesCaseInsensitive.contains("float_field"));
-    Assert.assertTrue(fieldNamesCaseInsensitive.contains("map_field"));
-    Assert.assertTrue(fieldNamesCaseInsensitive.contains("map_field::char_field"));
+    Assert.assertTrue(fieldNamesCaseInsensitive.contains("struct_field"));
+    Assert.assertTrue(fieldNamesCaseInsensitive.contains("struct_field::char_field"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$"));
     Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field"));
@@ -618,9 +618,9 @@ public void timeStampSecWriter() throws Exception {
     final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0);
 
     // write
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
 
     {
       TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec");
@@ -638,7 +638,7 @@ public void timeStampSecWriter() throws Exception {
     checkTimestampTZField(children.get(1), "secTZ", "UTC");
 
     // read
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    StructReader rootReader = new SingleStructReaderImpl(parent).reader("root");
     {
       FieldReader secReader = rootReader.reader("sec");
       secReader.setPosition(0);
@@ -662,9 +662,9 @@ public void timeStampMilliWriters() throws Exception {
     final LocalDateTime expectedMilliDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
 
     // write
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     {
       TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli");
       timeStampWriter.setPosition(0);
@@ -682,7 +682,7 @@ public void timeStampMilliWriters() throws Exception {
     checkTimestampTZField(children.get(1), "milliTZ", tz);
 
     // read
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    StructReader rootReader = new SingleStructReaderImpl(parent).reader("root");
 
     {
       FieldReader milliReader = rootReader.reader("milli");
@@ -718,9 +718,9 @@ public void timeStampMicroWriters() throws Exception {
     final LocalDateTime expectedMicroDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
 
     // write
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
 
     {
       TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro");
@@ -740,7 +740,7 @@ public void timeStampMicroWriters() throws Exception {
     checkTimestampTZField(children.get(1), "microTZ", tz);
 
     // read
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    StructReader rootReader = new SingleStructReaderImpl(parent).reader("root");
     {
       FieldReader microReader = rootReader.reader("micro");
       microReader.setPosition(0);
@@ -765,9 +765,9 @@ public void timeStampNanoWriters() throws Exception {
     final LocalDateTime expectedNanoDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
 
     // write
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
 
     {
       TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano");
@@ -785,7 +785,7 @@ public void timeStampNanoWriters() throws Exception {
     checkTimestampField(children.get(0), "nano");
     checkTimestampTZField(children.get(1), "nanoTZ", tz);
     // read
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    StructReader rootReader = new SingleStructReaderImpl(parent).reader("root");
 
     {
       FieldReader nanoReader = rootReader.reader("nano");
@@ -824,9 +824,9 @@ public void fixedSizeBinaryWriters() throws Exception {
     }
 
     // write
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
 
     String fieldName = "fixedSizeBinary";
     FixedSizeBinaryWriter fixedSizeBinaryWriter = rootWriter.fixedSizeBinary(fieldName, byteWidth);
@@ -841,7 +841,7 @@ public void fixedSizeBinaryWriters() throws Exception {
     Assert.assertEquals(ArrowType.FixedSizeBinary.TYPE_TYPE, children.get(0).getType().getTypeID());
 
     // read
-    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    StructReader rootReader = new SingleStructReaderImpl(parent).reader("root");
 
     FieldReader fixedSizeBinaryReader = rootReader.reader(fieldName);
     for (int i = 0; i < numValues; i++) {
@@ -853,58 +853,58 @@ public void fixedSizeBinaryWriters() throws Exception {
 
   @Test
   public void complexCopierWithList() {
-    MapVector parent = MapVector.empty("parent", allocator);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     ListWriter listWriter = rootWriter.list("list");
-    MapWriter innerMapWriter = listWriter.map();
+    StructWriter innerStructWriter = listWriter.struct();
     IntWriter outerIntWriter = listWriter.integer();
     rootWriter.start();
     listWriter.startList();
     outerIntWriter.writeInt(1);
     outerIntWriter.writeInt(2);
-    innerMapWriter.start();
-    IntWriter intWriter = innerMapWriter.integer("a");
+    innerStructWriter.start();
+    IntWriter intWriter = innerStructWriter.integer("a");
     intWriter.writeInt(1);
-    innerMapWriter.end();
-    innerMapWriter.start();
-    intWriter = innerMapWriter.integer("a");
+    innerStructWriter.end();
+    innerStructWriter.start();
+    intWriter = innerStructWriter.integer("a");
     intWriter.writeInt(2);
-    innerMapWriter.end();
+    innerStructWriter.end();
     listWriter.endList();
     rootWriter.end();
     writer.setValueCount(1);
 
-    NullableMapVector mapVector = (NullableMapVector) parent.getChild("root");
-    TransferPair tp = mapVector.getTransferPair(allocator);
+    StructVector structVector = (StructVector) parent.getChild("root");
+    TransferPair tp = structVector.getTransferPair(allocator);
     tp.splitAndTransfer(0, 1);
-    MapVector toMapVector = (MapVector) tp.getTo();
-    JsonStringHashMap<?, ?> toMapValue = (JsonStringHashMap<?, ?>) toMapVector.getObject(0);
+    NonNullableStructVector toStructVector = (NonNullableStructVector) tp.getTo();
+    JsonStringHashMap<?, ?> toMapValue = (JsonStringHashMap<?, ?>) toStructVector.getObject(0);
     JsonStringArrayList<?> object = (JsonStringArrayList<?>) toMapValue.get("list");
     assertEquals(1, object.get(0));
     assertEquals(2, object.get(1));
-    JsonStringHashMap<?, ?> innerMap = (JsonStringHashMap<?, ?>) object.get(2);
-    assertEquals(1, innerMap.get("a"));
-    innerMap = (JsonStringHashMap<?, ?>) object.get(3);
-    assertEquals(2, innerMap.get("a"));
+    JsonStringHashMap<?, ?> innerStruct = (JsonStringHashMap<?, ?>) object.get(2);
+    assertEquals(1, innerStruct.get("a"));
+    innerStruct = (JsonStringHashMap<?, ?>) object.get(3);
+    assertEquals(2, innerStruct.get("a"));
   }
 
   @Test
-  public void testSingleMapWriter1() {
-    /* initialize a SingleMapWriter with empty MapVector and then lazily
+  public void testSingleStructWriter1() {
+    /* initialize a SingleStructWriter with empty StructVector and then lazily
      * create all vectors with expected initialCapacity.
      */
-    MapVector parent = MapVector.empty("parent", allocator);
-    SingleMapWriter singleMapWriter = new SingleMapWriter(parent);
+    NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator);
+    SingleStructWriter singleStructWriter = new SingleStructWriter(parent);
 
     int initialCapacity = 1024;
-    singleMapWriter.setInitialCapacity(initialCapacity);
+    singleStructWriter.setInitialCapacity(initialCapacity);
 
-    IntWriter intWriter = singleMapWriter.integer("intField");
-    BigIntWriter bigIntWriter = singleMapWriter.bigInt("bigIntField");
-    Float4Writer float4Writer = singleMapWriter.float4("float4Field");
-    Float8Writer float8Writer = singleMapWriter.float8("float8Field");
-    ListWriter listWriter = singleMapWriter.list("listField");
+    IntWriter intWriter = singleStructWriter.integer("intField");
+    BigIntWriter bigIntWriter = singleStructWriter.bigInt("bigIntField");
+    Float4Writer float4Writer = singleStructWriter.float4("float4Field");
+    Float8Writer float8Writer = singleStructWriter.float8("float8Field");
+    ListWriter listWriter = singleStructWriter.list("listField");
 
     int intValue = 100;
     long bigIntValue = 10000;
@@ -912,7 +912,7 @@ public void testSingleMapWriter1() {
     double float8Value = 100.375;
 
     for (int i = 0; i < initialCapacity; i++) {
-      singleMapWriter.start();
+      singleStructWriter.start();
 
       intWriter.writeInt(intValue + i);
       bigIntWriter.writeBigInt(bigIntValue + (long)i);
@@ -927,7 +927,7 @@ public void testSingleMapWriter1() {
       listWriter.integer().writeInt(intValue + i + 3);
       listWriter.endList();
 
-      singleMapWriter.end();
+      singleStructWriter.end();
     }
 
     IntVector intVector = (IntVector)parent.getChild("intField");
@@ -935,19 +935,19 @@ public void testSingleMapWriter1() {
     Float4Vector float4Vector = (Float4Vector)parent.getChild("float4Field");
     Float8Vector float8Vector = (Float8Vector)parent.getChild("float8Field");
 
-    assertEquals(initialCapacity, singleMapWriter.getValueCapacity());
+    assertEquals(initialCapacity, singleStructWriter.getValueCapacity());
     assertEquals(initialCapacity, intVector.getValueCapacity());
     assertEquals(initialCapacity, bigIntVector.getValueCapacity());
     assertEquals(initialCapacity, float4Vector.getValueCapacity());
     assertEquals(initialCapacity, float8Vector.getValueCapacity());
 
-    MapReader singleMapReader = new SingleMapReaderImpl(parent);
+    StructReader singleStructReader = new SingleStructReaderImpl(parent);
 
-    IntReader intReader = singleMapReader.reader("intField");
-    BigIntReader bigIntReader = singleMapReader.reader("bigIntField");
-    Float4Reader float4Reader = singleMapReader.reader("float4Field");
-    Float8Reader float8Reader = singleMapReader.reader("float8Field");
-    UnionListReader listReader = (UnionListReader)singleMapReader.reader("listField");
+    IntReader intReader = singleStructReader.reader("intField");
+    BigIntReader bigIntReader = singleStructReader.reader("bigIntField");
+    Float4Reader float4Reader = singleStructReader.reader("float4Field");
+    Float8Reader float8Reader = singleStructReader.reader("float8Field");
+    UnionListReader listReader = (UnionListReader)singleStructReader.reader("listField");
 
     for (int i = 0; i < initialCapacity; i++) {
       intReader.setPosition(i);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index 9eb55c9bbdaee..419876c33f404 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -36,13 +36,13 @@
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
 import org.apache.arrow.vector.complex.writer.DateMilliWriter;
 import org.apache.arrow.vector.complex.writer.Float4Writer;
@@ -94,9 +94,9 @@ public void tearDown() {
     DateTimeZone.setDefault(defaultTimezone);
   }
 
-  protected void writeData(int count, NullableMapVector parent) {
+  protected void writeData(int count, StructVector parent) {
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     IntWriter intWriter = rootWriter.integer("int");
     BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
     Float4Writer float4Writer = rootWriter.float4("float");
@@ -119,7 +119,7 @@ protected void validateContent(int count, VectorSchemaRoot root) {
     }
   }
 
-  protected void writeComplexData(int count, NullableMapVector parent) {
+  protected void writeComplexData(int count, StructVector parent) {
     ArrowBuf varchar = allocator.buffer(3);
     varchar.readerIndex(0);
     varchar.setByte(0, 'a');
@@ -127,11 +127,11 @@ protected void writeComplexData(int count, NullableMapVector parent) {
     varchar.setByte(2, 'c');
     varchar.writerIndex(3);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     IntWriter intWriter = rootWriter.integer("int");
     BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
     ListWriter listWriter = rootWriter.list("list");
-    MapWriter mapWriter = rootWriter.map("map");
+    StructWriter structWriter = rootWriter.struct("struct");
     for (int i = 0; i < count; i++) {
       if (i % 5 != 3) {
         intWriter.setPosition(i);
@@ -145,10 +145,10 @@ protected void writeComplexData(int count, NullableMapVector parent) {
         listWriter.varChar().writeVarChar(0, 3, varchar);
       }
       listWriter.endList();
-      mapWriter.setPosition(i);
-      mapWriter.start();
-      mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
-      mapWriter.end();
+      structWriter.setPosition(i);
+      structWriter.start();
+      structWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
+      structWriter.end();
     }
     writer.setValueCount(count);
     varchar.release();
@@ -178,9 +178,9 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) {
       Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i));
       Assert.assertEquals(i % 3, ((List<?>) root.getVector("list").getObject(i)).size());
       NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
-      FieldReader mapReader = root.getVector("map").getReader();
-      mapReader.setPosition(i);
-      mapReader.reader("timestamp").read(h);
+      FieldReader structReader = root.getVector("struct").getReader();
+      structReader.setPosition(i);
+      structReader.reader("timestamp").read(h);
       Assert.assertEquals(i, h.value);
     }
   }
@@ -189,10 +189,10 @@ private LocalDateTime makeDateTimeFromCount(int i) {
     return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i);
   }
 
-  protected void writeDateTimeData(int count, NullableMapVector parent) {
+  protected void writeDateTimeData(int count, StructVector parent) {
     Assert.assertTrue(count < 100);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     DateMilliWriter dateWriter = rootWriter.dateMilli("date");
     TimeMilliWriter timeWriter = rootWriter.timeMilli("time");
     TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli");
@@ -494,7 +494,7 @@ public void validateUnionData(int count, VectorSchemaRoot root) {
     }
   }
 
-  public void writeUnionData(int count, NullableMapVector parent) {
+  public void writeUnionData(int count, StructVector parent) {
     ArrowBuf varchar = allocator.buffer(3);
     varchar.readerIndex(0);
     varchar.setByte(0, 'a');
@@ -502,11 +502,11 @@ public void writeUnionData(int count, NullableMapVector parent) {
     varchar.setByte(2, 'c');
     varchar.writerIndex(3);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     IntWriter intWriter = rootWriter.integer("union");
     BigIntWriter bigIntWriter = rootWriter.bigInt("union");
     ListWriter listWriter = rootWriter.list("union");
-    MapWriter mapWriter = rootWriter.map("union");
+    StructWriter structWriter = rootWriter.struct("union");
     for (int i = 0; i < count; i++) {
       switch (i % 4) {
         case 0:
@@ -526,10 +526,10 @@ public void writeUnionData(int count, NullableMapVector parent) {
           listWriter.endList();
           break;
         case 3:
-          mapWriter.setPosition(i);
-          mapWriter.start();
-          mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
-          mapWriter.end();
+          structWriter.setPosition(i);
+          structWriter.start();
+          structWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
+          structWriter.end();
           break;
       }
     }
@@ -537,10 +537,10 @@ public void writeUnionData(int count, NullableMapVector parent) {
     varchar.release();
   }
 
-  protected void writeVarBinaryData(int count, NullableMapVector parent) {
+  protected void writeVarBinaryData(int count, StructVector parent) {
     Assert.assertTrue(count < 100);
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
-    MapWriter rootWriter = writer.rootAsMap();
+    StructWriter rootWriter = writer.rootAsStruct();
     ListWriter listWriter = rootWriter.list("list");
     ArrowBuf varbin = allocator.buffer(count);
     for (int i = 0; i < count; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
index 3b809d00c2c62..2434692ce2fd7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
@@ -43,7 +43,7 @@
 import org.apache.arrow.vector.VectorUnloader;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.FixedSizeBinaryVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.ipc.message.ArrowBlock;
 import org.apache.arrow.vector.ipc.message.ArrowBuffer;
@@ -71,7 +71,7 @@ public void testWrite() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        StructVector parent = StructVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), file, new ByteArrayOutputStream());
     }
@@ -83,7 +83,7 @@ public void testWriteComplex() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        StructVector parent = StructVector.empty("parent", vectorAllocator)) {
       writeComplexData(count, parent);
       FieldVector root = parent.getChild("root");
       validateComplexContent(count, new VectorSchemaRoot(root));
@@ -99,7 +99,7 @@ public void testWriteRead() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+         StructVector parent = StructVector.empty("parent", originalVectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), file, stream);
     }
@@ -155,7 +155,7 @@ public void testWriteReadComplex() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+         StructVector parent = StructVector.empty("parent", originalVectorAllocator)) {
       writeComplexData(count, parent);
       write(parent.getChild("root"), file, stream);
     }
@@ -196,7 +196,7 @@ public void testWriteReadMultipleRBs() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator);
+         StructVector parent = StructVector.empty("parent", originalVectorAllocator);
          FileOutputStream fileOutputStream = new FileOutputStream(file)) {
       writeData(counts[0], parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
@@ -269,7 +269,7 @@ public void testWriteReadUnion() throws IOException {
 
     // write
     try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+         StructVector parent = StructVector.empty("parent", vectorAllocator)) {
       writeUnionData(count, parent);
       validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
       write(parent.getChild("root"), file, stream);
@@ -383,7 +383,7 @@ public void testWriteReadMetadata() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) {
+         StructVector vector = (StructVector) field.createVector(originalVectorAllocator)) {
       vector.allocateNewSafe();
       vector.setValueCount(0);
 
@@ -566,7 +566,7 @@ public void testWriteReadFixedSizeBinary() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+         StructVector parent = StructVector.empty("parent", originalVectorAllocator)) {
       FixedSizeBinaryVector fixedSizeBinaryVector = parent.addOrGet("fixed-binary", FieldType.nullable(new FixedSizeBinary(typeWidth)), FixedSizeBinaryVector.class);
       parent.allocateNew();
       for (int i=0; i<numValues; i++) {
@@ -616,7 +616,7 @@ public void testWriteReadFixedSizeList() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+         StructVector parent = StructVector.empty("parent", originalVectorAllocator)) {
       FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class);
       Float4Vector floats = (Float4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
       IntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), IntVector.class);
@@ -676,7 +676,7 @@ public void testWriteReadVarBin() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        StructVector parent = StructVector.empty("parent", vectorAllocator)) {
       writeVarBinaryData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateVarBinary(count, root);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
index 6a23a8438de48..3545f4519e343 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
@@ -24,7 +24,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -44,7 +44,7 @@ public void testWriteRead() throws IOException {
 
     // write
     try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-         NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+         StructVector parent = StructVector.empty("parent", originalVectorAllocator)) {
       writeData(count, parent);
       writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
     }
@@ -72,7 +72,7 @@ public void testWriteReadComplexJSON() throws IOException {
     // write
     try (
         BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+        StructVector parent = StructVector.empty("parent", originalVectorAllocator)) {
       writeComplexData(count, parent);
       writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
     }
@@ -99,7 +99,7 @@ public void testWriteComplexJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        StructVector parent = StructVector.empty("parent", vectorAllocator)) {
       writeComplexData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateComplexContent(root.getRowCount(), root);
@@ -121,7 +121,7 @@ public void testWriteReadUnionJSON() throws IOException {
     int count = COUNT;
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        StructVector parent = StructVector.empty("parent", vectorAllocator)) {
       writeUnionData(count, parent);
       printVectors(parent.getChildrenFromFields());
 
@@ -153,7 +153,7 @@ public void testWriteReadDateTimeJSON() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        StructVector parent = StructVector.empty("parent", vectorAllocator)) {
 
       writeDateTimeData(count, parent);
 
@@ -318,7 +318,7 @@ public void testWriteReadVarBinJSON() throws IOException {
     // write
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
-        NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
+        StructVector parent = StructVector.empty("parent", vectorAllocator)) {
       writeVarBinaryData(count, parent);
       VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
       validateVarBinary(count, root);

From 7c8844360846f203e5cfad4af71341f3e1d8d32a Mon Sep 17 00:00:00 2001
From: rvernica <rvernica@gmail.com>
Date: Thu, 15 Feb 2018 07:39:56 -0800
Subject: [PATCH 1482/1644] [Website] Add SciDB to "Powered By" (#1604)

---
 site/powered_by.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/site/powered_by.md b/site/powered_by.md
index 20feea46ce45d..73850cab3a023 100644
--- a/site/powered_by.md
+++ b/site/powered_by.md
@@ -105,6 +105,12 @@ short description of your use case. See the following for some examples.
   of Apache Arrow based on Apache Arrow GLib. Red Arrow is a core
   library for it. It also provides many Ruby libraries to integrate
   existing Ruby libraries with Apache Arrow. They use Red Arrow.
+* **[SciDB][21]:** Paradigm4's SciDB is a scalable, scientific
+  database management system that helps researchers integrate and
+  analyze diverse, multi-dimensional, high resolution data - like
+  genomic, clinical, images, sensor, environmental, and IoT data -
+  all in one analytical platform. [SciDB streaming][22] is
+  powered by Apache Arrow.
 * **[Turbodbc][4]:** Python module to access relational databases via the Open
   Database Connectivity (ODBC) interface. It provides the ability to return
   Arrow Tables and RecordBatches in addition to the Python Database API
@@ -130,3 +136,5 @@ short description of your use case. See the following for some examples.
 [18]: https://www.graphistry.com
 [19]: http://gpuopenanalytics.com
 [20]: https://github.com/johanpel/fletcher
+[21]: https://www.paradigm4.com
+[22]: https://github.com/Paradigm4/stream

From e15f54505d4d92810c8c216a89eee3e703013d1c Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 15 Feb 2018 11:34:59 -0500
Subject: [PATCH 1483/1644] ARROW-2039: [Python] Avoid crashing on
 uninitialized Buffer

Author: Antoine Pitrou <antoine@python.org>

Closes #1605 from pitrou/ARROW-2039-uninitialized-buffer and squashes the following commits:

358443c6 [Antoine Pitrou] ARROW-2039: [Python] Avoid crashing on uninitialized Buffer
---
 python/pyarrow/io.pxi           | 16 +++++++++++
 python/pyarrow/lib.pxd          |  1 +
 python/pyarrow/tests/test_io.py | 47 +++++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+)

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index b0996f85e203e..aa2f7ed07a2bc 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -595,22 +595,30 @@ cdef class Buffer:
         self.shape[0] = self.size
         self.strides[0] = <Py_ssize_t>(1)
 
+    cdef int _check_nullptr(self) except -1:
+        if self.buffer.get() == NULL:
+            raise ReferenceError("operation on uninitialized Buffer object")
+        return 0
+
     def __len__(self):
         return self.size
 
     property size:
 
         def __get__(self):
+            self._check_nullptr()
             return self.buffer.get().size()
 
     property is_mutable:
 
         def __get__(self):
+            self._check_nullptr()
             return self.buffer.get().is_mutable()
 
     property parent:
 
         def __get__(self):
+            self._check_nullptr()
             cdef shared_ptr[CBuffer] parent_buf = self.buffer.get().parent()
 
             if parent_buf.get() == NULL:
@@ -620,6 +628,7 @@ cdef class Buffer:
 
     def __getitem__(self, key):
         # TODO(wesm): buffer slicing
+        self._check_nullptr()
         raise NotImplementedError
 
     def equals(self, Buffer other):
@@ -634,17 +643,21 @@ cdef class Buffer:
         -------
         are_equal : True if buffer contents and size are equal
         """
+        self._check_nullptr()
+        other._check_nullptr()
         cdef c_bool result = False
         with nogil:
             result = self.buffer.get().Equals(deref(other.buffer.get()))
         return result
 
     def to_pybytes(self):
+        self._check_nullptr()
         return cp.PyBytes_FromStringAndSize(
             <const char*>self.buffer.get().data(),
             self.buffer.get().size())
 
     def __getbuffer__(self, cp.Py_buffer* buffer, int flags):
+        self._check_nullptr()
 
         buffer.buf = <char *>self.buffer.get().data()
         buffer.format = 'b'
@@ -662,11 +675,13 @@ cdef class Buffer:
         buffer.suboffsets = NULL
 
     def __getsegcount__(self, Py_ssize_t *len_out):
+        self._check_nullptr()
         if len_out != NULL:
             len_out[0] = <Py_ssize_t>self.size
         return 1
 
     def __getreadbuffer__(self, Py_ssize_t idx, void **p):
+        self._check_nullptr()
         if idx != 0:
             raise SystemError("accessing non-existent buffer segment")
         if p != NULL:
@@ -674,6 +689,7 @@ cdef class Buffer:
         return self.size
 
     def __getwritebuffer__(self, Py_ssize_t idx, void **p):
+        self._check_nullptr()
         if not self.buffer.get().is_mutable():
             raise SystemError("trying to write an immutable buffer")
         if idx != 0:
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index b1433ecde270a..31732a6e07741 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -321,6 +321,7 @@ cdef class Buffer:
         Py_ssize_t strides[1]
 
     cdef void init(self, const shared_ptr[CBuffer]& buffer)
+    cdef int _check_nullptr(self) except -1
 
 
 cdef class ResizableBuffer(Buffer):
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 0947cb7c7870e..736020f60fd59 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from functools import partial
 from io import BytesIO, TextIOWrapper
 import gc
 import os
@@ -176,6 +177,8 @@ def test_buffer_to_numpy():
     buf = pa.frombuffer(byte_array)
     array = np.frombuffer(buf, dtype="uint8")
     assert array[0] == byte_array[0]
+    byte_array[0] += 1
+    assert array[0] == byte_array[0]
     assert array.base == buf
 
 
@@ -192,6 +195,25 @@ def test_buffer_from_numpy():
         buf = pa.frombuffer(arr.T[::2])
 
 
+def test_buffer_equals():
+    # Buffer.equals() returns true iff the buffers have the same contents
+    b1 = b'some data!'
+    b2 = bytearray(b1)
+    b3 = bytearray(b1)
+    b3[0] = 42
+    buf1 = pa.frombuffer(b1)
+    buf2 = pa.frombuffer(b2)
+    buf3 = pa.frombuffer(b2)
+    buf4 = pa.frombuffer(b3)
+    buf5 = pa.frombuffer(np.frombuffer(b2, dtype=np.int16))
+    assert buf1.equals(buf1)
+    assert buf1.equals(buf2)
+    assert buf2.equals(buf3)
+    assert not buf2.equals(buf4)
+    # Data type is indifferent
+    assert buf2.equals(buf5)
+
+
 def test_allocate_buffer():
     buf = pa.allocate_buffer(100)
     assert buf.size == 100
@@ -244,9 +266,11 @@ def test_buffer_memoryview_is_immutable():
     val = b'some data'
 
     buf = pa.frombuffer(val)
+    assert not buf.is_mutable
     assert isinstance(buf, pa.Buffer)
 
     result = memoryview(buf)
+    assert result.readonly
 
     with pytest.raises(TypeError) as exc:
         result[0] = b'h'
@@ -258,6 +282,29 @@ def test_buffer_memoryview_is_immutable():
         assert 'cannot modify read-only' in str(exc.value)
 
 
+def test_uninitialized_buffer():
+    # ARROW-2039: calling Buffer() directly creates an uninitialized object
+    check_uninitialized = partial(pytest.raises,
+                                  ReferenceError, match="uninitialized")
+    buf = pa.Buffer()
+    with check_uninitialized():
+        buf.size
+    with check_uninitialized():
+        len(buf)
+    with check_uninitialized():
+        buf.is_mutable
+    with check_uninitialized():
+        buf.parent
+    with check_uninitialized():
+        buf.to_pybytes()
+    with check_uninitialized():
+        memoryview(buf)
+    with check_uninitialized():
+        buf.equals(pa.frombuffer(b''))
+    with check_uninitialized():
+        pa.frombuffer(b'').equals(buf)
+
+
 def test_memory_output_stream():
     # 10 bytes
     val = b'dataabcdef'

From 0265fb4012c7ceb25b755b6995428b4d74ef9e66 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 15 Feb 2018 18:50:32 +0100
Subject: [PATCH 1484/1644] ARROW-2151: [Python] Fix conversion from np.uint64
 scalars

Also add tests for ARROW-1345, ARROW-2008.

Author: Antoine Pitrou <antoine@python.org>

Closes #1602 from pitrou/ARROW-2151-convert-np-uint64 and squashes the following commits:

613af18 [Antoine Pitrou] ARROW-2151: [Python] Fix conversion from np.uint64 scalars
---
 cpp/src/arrow/python/builtin_convert.cc      |  4 +-
 cpp/src/arrow/python/helpers.cc              | 17 ++++++
 cpp/src/arrow/python/helpers.h               |  2 +
 python/pyarrow/tests/test_convert_builtin.py | 57 ++++++++++++++++++--
 python/pyarrow/tests/test_convert_pandas.py  |  9 ++++
 5 files changed, 82 insertions(+), 7 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index 60f58ea5b3b7b..a286c6bd5e90d 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -538,8 +538,8 @@ class UInt64Converter : public TypedConverterVisitor<UInt64Builder, UInt64Conver
  public:
   // Append a non-missing item
   Status AppendItem(PyObject* obj) {
-    const auto val = static_cast<int64_t>(PyLong_AsUnsignedLongLong(obj));
-    RETURN_IF_PYERROR();
+    uint64_t val;
+    RETURN_NOT_OK(internal::UInt64FromPythonInt(obj, &val));
     return typed_builder_->Append(val);
   }
 };
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 494f929004ae9..df1db99911b63 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -176,6 +176,23 @@ bool IsPyInteger(PyObject* obj) {
 #endif
 }
 
+Status UInt64FromPythonInt(PyObject* obj, uint64_t* out) {
+  OwnedRef ref;
+  // PyLong_AsUnsignedLongLong() doesn't handle conversion from non-ints
+  // (e.g. np.uint64), so do it ourselves
+  if (!PyLong_Check(obj)) {
+    ref.reset(PyNumber_Long(obj));
+    RETURN_IF_PYERROR();
+    obj = ref.obj();
+  }
+  auto result = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(obj));
+  if (result == static_cast<uint64_t>(-1)) {
+    RETURN_IF_PYERROR();
+  }
+  *out = static_cast<uint64_t>(result);
+  return Status::OK();
+}
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index c82bdabc47614..c0171aa2f5a71 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -57,6 +57,8 @@ Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arr
                                 Decimal128* out);
 bool IsPyInteger(PyObject* obj);
 
+Status UInt64FromPythonInt(PyObject* obj, uint64_t* out);
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 5cd4a52a21839..8423ff00b6722 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -28,6 +28,20 @@
 import six
 
 
+int_type_pairs = [
+    (np.int8, pa.int8()),
+    (np.int16, pa.int64()),
+    (np.int32, pa.int32()),
+    (np.int64, pa.int64()),
+    (np.uint8, pa.uint8()),
+    (np.uint16, pa.uint64()),
+    (np.uint32, pa.uint32()),
+    (np.uint64, pa.uint64())]
+
+
+np_int_types, _ = zip(*int_type_pairs)
+
+
 class StrangeIterable:
     def __init__(self, lst):
         self.lst = lst
@@ -146,7 +160,20 @@ def test_sequence_all_none():
 
 
 @pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
-def test_sequence_integer(seq):
+@pytest.mark.parametrize("np_scalar_pa_type", int_type_pairs)
+def test_sequence_integer(seq, np_scalar_pa_type):
+    np_scalar, pa_type = np_scalar_pa_type
+    expected = [1, None, 3, None,
+                np.iinfo(np_scalar).min, np.iinfo(np_scalar).max]
+    arr = pa.array(seq(expected), type=pa_type)
+    assert len(arr) == 6
+    assert arr.null_count == 2
+    assert arr.type == pa_type
+    assert arr.to_pylist() == expected
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+def test_sequence_integer_inferred(seq):
     expected = [1, None, 3, None]
     arr = pa.array(seq(expected))
     assert len(arr) == 4
@@ -156,12 +183,32 @@ def test_sequence_integer(seq):
 
 
 @pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
-@pytest.mark.parametrize("np_scalar", [np.int16, np.int32, np.int64, np.uint16,
-                                       np.uint32, np.uint64])
-def test_sequence_numpy_integer(seq, np_scalar):
+@pytest.mark.parametrize("np_scalar_pa_type", int_type_pairs)
+def test_sequence_numpy_integer(seq, np_scalar_pa_type):
+    np_scalar, pa_type = np_scalar_pa_type
+    expected = [np_scalar(1), None, np_scalar(3), None,
+                np_scalar(np.iinfo(np_scalar).min),
+                np_scalar(np.iinfo(np_scalar).max)]
+    arr = pa.array(seq(expected), type=pa_type)
+    assert len(arr) == 6
+    assert arr.null_count == 2
+    assert arr.type == pa_type
+    assert arr.to_pylist() == expected
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+@pytest.mark.parametrize("np_scalar_pa_type", int_type_pairs)
+def test_sequence_numpy_integer_inferred(seq, np_scalar_pa_type):
+    np_scalar, pa_type = np_scalar_pa_type
     expected = [np_scalar(1), None, np_scalar(3), None]
+    if np_scalar != np.uint64:
+        expected += [np_scalar(np.iinfo(np_scalar).min),
+                     np_scalar(np.iinfo(np_scalar).max)]
+    else:
+        # max(uint64) is too large for the inferred int64 type
+        expected += [0, np.iinfo(np.int64).max]
     arr = pa.array(seq(expected))
-    assert len(arr) == 4
+    assert len(arr) == 6
     assert arr.null_count == 2
     assert arr.type == pa.int64()
     assert arr.to_pylist() == expected
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 5b6f6bcdfe5af..95137ffb26e28 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1328,6 +1328,15 @@ def test_empty_list_roundtrip(self):
 
         tm.assert_frame_equal(result, df)
 
+    def test_array_from_nested_arrays(self):
+        df, schema = dataframe_with_arrays()
+        for field in schema:
+            arr = df[field.name].values
+            expected = pa.array(list(arr), type=field.type)
+            result = pa.array(arr)
+            assert result.type == field.type  # == list<scalar>
+            assert result.equals(expected)
+
 
 class TestConvertStructTypes(object):
     """

From c0779861417f44535196d5c383d35f07ff144627 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 15 Feb 2018 18:58:34 +0100
Subject: [PATCH 1485/1644] ARROW-2068: [Python] Expose array's buffers

This recurses into child data if present (for nested types).

Author: Antoine Pitrou <antoine@python.org>

Closes #1613 from pitrou/ARROW-2068-expose-array-buffers and squashes the following commits:

0634aaf [Antoine Pitrou] ARROW-2068: [Python] Expose array's buffers
---
 python/pyarrow/array.pxi           | 25 ++++++++++++
 python/pyarrow/tests/test_array.py | 64 ++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index f85363cb12959..a43bfb93b4d0c 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -244,6 +244,22 @@ cdef wrap_datum(const CDatum& datum):
         raise ValueError("Unable to wrap Datum in a Python object")
 
 
+cdef _append_array_buffers(const CArrayData* ad, list res):
+    """
+    Recursively append Buffer wrappers from *ad* and its children.
+    """
+    cdef size_t i, n
+    assert ad != NULL
+    n = ad.buffers.size()
+    for i in range(n):
+        buf = ad.buffers[i]
+        res.append(pyarrow_wrap_buffer(buf)
+                   if buf.get() != NULL else None)
+    n = ad.child_data.size()
+    for i in range(n):
+        _append_array_buffers(ad.child_data[i].get(), res)
+
+
 cdef class Array:
 
     cdef void init(self, const shared_ptr[CArray]& sp_array):
@@ -463,6 +479,15 @@ cdef class Array:
         with nogil:
             check_status(ValidateArray(deref(self.ap)))
 
+    def buffers(self):
+        """
+        Return a list of Buffer objects pointing to this array's physical
+        storage.
+        """
+        res = []
+        _append_array_buffers(self.sp_array.get().data().get(), res)
+        return res
+
 
 cdef class Tensor:
 
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index efbcef5e135b6..197dac0d84f2c 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -17,6 +17,7 @@
 
 import datetime
 import pytest
+import struct
 import sys
 
 import numpy as np
@@ -589,3 +590,66 @@ def test_array_from_numpy_unicode():
     arrow_arr = pa.array(arr)
     expected = pa.array(['', '', ''], type='utf8')
     assert arrow_arr.equals(expected)
+
+
+def test_buffers_primitive():
+    a = pa.array([1, 2, None, 4], type=pa.int16())
+    buffers = a.buffers()
+    assert len(buffers) == 2
+    null_bitmap = buffers[0].to_pybytes()
+    assert 1 <= len(null_bitmap) <= 64  # XXX this is varying
+    assert bytearray(null_bitmap)[0] == 0b00001011
+
+    assert struct.unpack('hhxxh', buffers[1].to_pybytes()) == (1, 2, 4)
+
+    a = pa.array(np.int8([4, 5, 6]))
+    buffers = a.buffers()
+    assert len(buffers) == 2
+    # No null bitmap from Numpy int array
+    assert buffers[0] is None
+    assert struct.unpack('3b', buffers[1].to_pybytes()) == (4, 5, 6)
+
+    a = pa.array([b'foo!', None, b'bar!!'])
+    buffers = a.buffers()
+    assert len(buffers) == 3
+    null_bitmap = buffers[0].to_pybytes()
+    assert bytearray(null_bitmap)[0] == 0b00000101
+    offsets = buffers[1].to_pybytes()
+    assert struct.unpack('4i', offsets) == (0, 4, 4, 9)
+    values = buffers[2].to_pybytes()
+    assert values == b'foo!bar!!'
+
+
+def test_buffers_nested():
+    a = pa.array([[1, 2], None, [3, None, 4, 5]], type=pa.list_(pa.int64()))
+    buffers = a.buffers()
+    assert len(buffers) == 4
+    # The parent buffers
+    null_bitmap = buffers[0].to_pybytes()
+    assert bytearray(null_bitmap)[0] == 0b00000101
+    offsets = buffers[1].to_pybytes()
+    assert struct.unpack('4i', offsets) == (0, 2, 2, 6)
+    # The child buffers
+    null_bitmap = buffers[2].to_pybytes()
+    assert bytearray(null_bitmap)[0] == 0b00110111
+    values = buffers[3].to_pybytes()
+    assert struct.unpack('qqq8xqq', values) == (1, 2, 3, 4, 5)
+
+    a = pa.array([(42, None), None, (None, 43)],
+                 type=pa.struct([pa.field('a', pa.int8()),
+                                 pa.field('b', pa.int16())]))
+    buffers = a.buffers()
+    assert len(buffers) == 5
+    # The parent buffer
+    null_bitmap = buffers[0].to_pybytes()
+    assert bytearray(null_bitmap)[0] == 0b00000101
+    # The child buffers: 'a'
+    null_bitmap = buffers[1].to_pybytes()
+    assert bytearray(null_bitmap)[0] == 0b00000001
+    values = buffers[2].to_pybytes()
+    assert struct.unpack('bxx', values) == (42,)
+    # The child buffers: 'b'
+    null_bitmap = buffers[3].to_pybytes()
+    assert bytearray(null_bitmap)[0] == 0b00000100
+    values = buffers[4].to_pybytes()
+    assert struct.unpack('4xh', values) == (43,)

From 148ab9cdf9fcf3eaad521611838c2184d623303d Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 15 Feb 2018 19:00:15 +0100
Subject: [PATCH 1486/1644] ARROW-2156: [CI] Isolate Sphinx dependencies

Not sure how much time this saves in practice, since build times are quite varying, but it feels like a no-brainer.

Author: Antoine Pitrou <antoine@python.org>

Closes #1606 from pitrou/ARROW-2156-isolate-sphinx-deps and squashes the following commits:

0baadb9 [Antoine Pitrou] ARROW-2156: [CI] Isolate Sphinx dependencies
---
 ci/travis_script_python.sh | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index dbaaab89e1740..a487da596bd04 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -41,12 +41,7 @@ conda install -y -q pip \
       cloudpickle \
       numpy=1.13.1 \
       pandas \
-      cython \
-      ipython \
-      matplotlib \
-      numpydoc \
-      sphinx \
-      sphinx_bootstrap_theme
+      cython
 
 if [ "$PYTHON_VERSION" != "2.7" ] || [ $TRAVIS_OS_NAME != "osx" ]; then
   # Install pytorch for torch tensor conversion tests
@@ -103,6 +98,13 @@ python -m pytest -vv -r sxX --durations=15 -s $PYARROW_PATH --parquet
 
 if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
   # Build documentation once
+  conda install -y -q \
+        ipython \
+        matplotlib \
+        numpydoc \
+        sphinx \
+        sphinx_bootstrap_theme
+
   pushd $ARROW_PYTHON_DIR/doc
   sphinx-build -q -b html -d _build/doctrees -W source _build/html
   popd

From e8870d1cfae9ecd62c69e7b9f57da0bdfb3a8dc2 Mon Sep 17 00:00:00 2001
From: yosuke shiro <ys2014hs@gmail.com>
Date: Fri, 16 Feb 2018 14:31:29 -0500
Subject: [PATCH 1487/1644] ARROW-2166: [GLib] Add Slice api to Column

Add Slice api to Column.

Author: yosuke shiro <ys2014hs@gmail.com>

Closes #1617 from shiro615/ARROW-2166 and squashes the following commits:

bd39d756 [yosuke shiro] ARROW-2166: [GLib] Add Slice api to Column
---
 c_glib/arrow-glib/column.cpp | 20 ++++++++++++++++++++
 c_glib/arrow-glib/column.h   |  3 +++
 c_glib/test/test-column.rb   | 16 ++++++++++++++++
 3 files changed, 39 insertions(+)

diff --git a/c_glib/arrow-glib/column.cpp b/c_glib/arrow-glib/column.cpp
index a7222b17650bb..55d06ea9eea6b 100644
--- a/c_glib/arrow-glib/column.cpp
+++ b/c_glib/arrow-glib/column.cpp
@@ -160,6 +160,26 @@ garrow_column_new_chunked_array(GArrowField *field,
   return garrow_column_new_raw(&arrow_column);
 }
 
+/**
+ * garrow_column_slice:
+ * @column: A #GArrowColumn.
+ * @offset: The offset of sub #GArrowColumn.
+ * @length: The length of sub #GArrowColumn.
+ *
+ * Returns: (transfer full): The sub #GArrowColumn. It covers only from
+ *   `offset` to `offset + length` range. The sub #GArrowColumn shares
+ *   values with the base #GArrowColumn.
+ */
+GArrowColumn *
+garrow_column_slice(GArrowColumn *column,
+                    guint64 offset,
+                    guint64 length)
+{
+  const auto arrow_column = garrow_column_get_raw(column);
+  auto arrow_sub_column = arrow_column->Slice(offset, length);
+  return garrow_column_new_raw(&arrow_sub_column);
+}
+
 /**
  * garrow_column_equal:
  * @column: A #GArrowColumn.
diff --git a/c_glib/arrow-glib/column.h b/c_glib/arrow-glib/column.h
index b649c5f1e50be..f2c181d4336d5 100644
--- a/c_glib/arrow-glib/column.h
+++ b/c_glib/arrow-glib/column.h
@@ -71,6 +71,9 @@ GArrowColumn *garrow_column_new_array(GArrowField *field,
                                       GArrowArray *array);
 GArrowColumn *garrow_column_new_chunked_array(GArrowField *field,
                                               GArrowChunkedArray *chunked_array);
+GArrowColumn *garrow_column_slice(GArrowColumn *column,
+                                  guint64 offset,
+                                  guint64 length);
 
 gboolean            garrow_column_equal         (GArrowColumn *column,
                                                  GArrowColumn *other_column);
diff --git a/c_glib/test/test-column.rb b/c_glib/test/test-column.rb
index 96e02b60319fd..01127de6e02e9 100644
--- a/c_glib/test/test-column.rb
+++ b/c_glib/test/test-column.rb
@@ -96,4 +96,20 @@ def test_data
     column = Arrow::Column.new(field, chunked_array)
     assert_equal(3, column.data.length)
   end
+
+  def test_slice
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    chunks1 = [
+      build_boolean_array([true, false, true]),
+      build_boolean_array([false, true]),
+    ]
+    chunks2 = [
+      build_boolean_array([false, true]),
+      build_boolean_array([false]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(chunks1)
+    column = Arrow::Column.new(field, chunked_array)
+    sub_column = column.slice(1, 3)
+    assert_equal(chunks2, sub_column.data.chunks)
+  end
 end

From 7c37b0d3376323c7bbfa4b612fec77238dc23c9d Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 16 Feb 2018 15:34:27 -0500
Subject: [PATCH 1488/1644] ARROW-2161: [Python] Skip test_cython_api if
 ARROW_HOME isn't defined

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1615 from cpcloud/ARROW-2161 and squashes the following commits:

95adf634 [Phillip Cloud] ARROW-2161: Skip test_cython_api if ARROW_HOME isn't defined
---
 python/pyarrow/tests/test_cython.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py
index 51b1a48ded09e..df5e70ee7e3a5 100644
--- a/python/pyarrow/tests/test_cython.py
+++ b/python/pyarrow/tests/test_cython.py
@@ -56,14 +56,16 @@
 """
 
 
+@pytest.mark.skipif(
+    'ARROW_HOME' not in os.environ,
+    reason='ARROW_HOME environment variable not defined')
 def test_cython_api(tmpdir):
     """
     Basic test for the Cython API.
     """
     pytest.importorskip('Cython')
 
-    if 'ARROW_HOME' in os.environ:
-        ld_path_default = os.path.join(os.environ['ARROW_HOME'], 'lib')
+    ld_path_default = os.path.join(os.environ['ARROW_HOME'], 'lib')
 
     test_ld_path = os.environ.get('PYARROW_TEST_LD_PATH', ld_path_default)
 

From c71a56527726f3cab58b37ad61a2129662f8a4fb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 16 Feb 2018 15:53:41 -0500
Subject: [PATCH 1489/1644] ARROW-2117: [C++] Update codebase / CI toolchain
 for clang 5.0

LLVM 5.0 found a number of code tidying issues that I fixed here. It also wants us to use `override` with our virtual destructor implementations, which I wasn't aware of whether is a best practice or not.

cc @xhochy @cpcloud for review

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1597 from wesm/ARROW-2117 and squashes the following commits:

28f384c0 [Wes McKinney] Use clang-5 in integration test builds also
f76c2880 [Wes McKinney] Mark rest of virtual dtors as override
03d03cd6 [Wes McKinney] Upgrade clang and clang-format use to LLVM 5.0, account for new compiler warnings
---
 .travis.yml                                 | 10 +++---
 ci/travis_install_clang_tools.sh            |  4 +--
 cpp/CMakeLists.txt                          |  2 +-
 cpp/cmake_modules/SetupCxxFlags.cmake       |  6 +++-
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 11 ++++++-
 cpp/src/arrow/array-test.cc                 | 15 +++++----
 cpp/src/arrow/buffer.h                      |  2 +-
 cpp/src/arrow/builder.cc                    |  9 +++---
 cpp/src/arrow/builder.h                     |  4 +--
 cpp/src/arrow/compare.cc                    |  5 ++-
 cpp/src/arrow/compute/compute-test.cc       | 16 +++++-----
 cpp/src/arrow/compute/kernels/cast.cc       | 22 +++++++------
 cpp/src/arrow/compute/kernels/hash.cc       |  2 +-
 cpp/src/arrow/io/file.h                     |  6 ++--
 cpp/src/arrow/io/hdfs.h                     |  6 ++--
 cpp/src/arrow/io/io-file-test.cc            |  4 +--
 cpp/src/arrow/io/io-hdfs-test.cc            |  2 +-
 cpp/src/arrow/ipc/feather.cc                |  7 +++--
 cpp/src/arrow/ipc/reader.h                  |  2 +-
 cpp/src/arrow/ipc/writer.cc                 |  8 ++---
 cpp/src/arrow/ipc/writer.h                  |  6 ++--
 cpp/src/arrow/memory_pool.cc                |  2 +-
 cpp/src/arrow/memory_pool.h                 |  2 +-
 cpp/src/arrow/python/CMakeLists.txt         |  7 +++++
 cpp/src/arrow/python/io.h                   |  4 +--
 cpp/src/arrow/python/numpy_to_arrow.cc      | 10 +++---
 cpp/src/arrow/table.h                       |  2 +-
 cpp/src/arrow/test-common.h                 | 10 +-----
 cpp/src/arrow/type.cc                       |  4 +--
 cpp/src/arrow/util/bit-util.h               | 35 +++++++++------------
 cpp/src/arrow/util/compression_zlib.h       |  2 +-
 cpp/src/arrow/util/io-util.h                |  4 +--
 cpp/src/plasma/plasma.cc                    |  5 +--
 cpp/src/plasma/plasma.h                     |  5 +--
 34 files changed, 126 insertions(+), 115 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d591a992269c0..9ef16b5cf0b52 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,7 +46,7 @@ matrix:
   allow_failures:
   - jdk: oraclejdk9
   include:
-  # C++ & Python w/ clang 4.0
+  # C++ & Python w/ clang 5.0
   - compiler: gcc
     language: cpp
     os: linux
@@ -59,8 +59,8 @@ matrix:
     - export ARROW_TRAVIS_ORC=1
     - export ARROW_TRAVIS_CLANG_FORMAT=1
     - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    - export CC="clang-4.0"
-    - export CXX="clang++-4.0"
+    - export CC="clang-5.0"
+    - export CXX="clang++-5.0"
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
     - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
@@ -124,8 +124,8 @@ matrix:
     before_script:
     - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    - export CC="clang-4.0"
-    - export CXX="clang++-4.0"
+    - export CC="clang-5.0"
+    - export CXX="clang++-5.0"
     - nvm install node
     - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh; fi
     - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
index d0108ad37ef02..630c09e31ff91 100755
--- a/ci/travis_install_clang_tools.sh
+++ b/ci/travis_install_clang_tools.sh
@@ -19,6 +19,6 @@
 
 wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
 sudo apt-add-repository -y \
-     "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-4.0 main"
+     "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-5.0 main"
 sudo apt-get update -q
-sudo apt-get install -q clang-4.0 clang-format-4.0 clang-tidy-4.0
+sudo apt-get install -q clang-5.0 clang-format-5.0 clang-tidy-5.0
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 62c8e6590a63a..42c1ec8f0405a 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -48,7 +48,7 @@ set(ARROW_ABI_VERSION "${ARROW_SO_VERSION}.0.0")
 
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 
-set(CLANG_FORMAT_VERSION "4.0")
+set(CLANG_FORMAT_VERSION "5.0")
 find_package(ClangTools)
 if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
   # Generate a Clang compile_commands.json "compilation database" file for use
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index d901bde47c631..e21c549d90ee1 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -91,7 +91,7 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
 -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
--Wno-comma -Wno-unused-parameter -Wno-undef \
+-Wno-comma -Wno-unused-parameter -Wno-unused-template -Wno-undef \
 -Wno-shadow -Wno-switch-enum -Wno-exit-time-destructors \
 -Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \
 -Wno-implicit-fallthrough -Wno-unreachable-code-return \
@@ -117,6 +117,10 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
       set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-undefined-func-template")
     endif()
 
+    if ("${COMPILER_VERSION}" VERSION_GREATER "4.0")
+      set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-zero-as-null-pointer-constant")
+    endif()
+
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option -Werror")
   elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 193b162d18285..3511d40d42ce6 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -909,12 +909,21 @@ if (ARROW_ORC)
   set(ORC_INCLUDE_DIR "${ORC_PREFIX}/include")
   set(ORC_STATIC_LIB "${ORC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}orc${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
+  if ("${COMPILER_FAMILY}" STREQUAL "clang")
+    if ("${COMPILER_VERSION}" VERSION_GREATER "4.0")
+      set(ORC_CMAKE_CXX_FLAGS " -Wno-zero-as-null-pointer-constant \
+-Wno-inconsistent-missing-destructor-override ")
+    endif()
+  endif()
+
+  set(ORC_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${ORC_CMAKE_CXX_FLAGS}")
+
   # Since LZ4 isn't installed, the header file is in ${LZ4_HOME}/lib instead of
   # ${LZ4_HOME}/include, which forces us to specify the include directory
   # manually as well.
   set (ORC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                       -DCMAKE_INSTALL_PREFIX=${ORC_PREFIX}
-                      -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}
+                      -DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}
                       -DBUILD_LIBHDFSPP=OFF
                       -DBUILD_JAVA=OFF
                       -DBUILD_TOOLS=OFF
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index fa64d467b695b..d68b04dc38b8a 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -193,11 +193,13 @@ TEST_F(TestArray, TestCopy) {}
 // Primitive type tests
 
 TEST_F(TestBuilder, TestReserve) {
-  ASSERT_OK(builder_->Init(10));
-  ASSERT_EQ(2, builder_->null_bitmap()->size());
+  UInt8Builder builder(pool_);
 
-  ASSERT_OK(builder_->Reserve(30));
-  ASSERT_EQ(4, builder_->null_bitmap()->size());
+  ASSERT_OK(builder.Init(10));
+  ASSERT_EQ(2, builder.null_bitmap()->size());
+
+  ASSERT_OK(builder.Reserve(30));
+  ASSERT_EQ(4, builder.null_bitmap()->size());
 }
 
 template <typename Attrs>
@@ -266,7 +268,6 @@ class TestPrimitiveBuilder : public TestBuilder {
   int64_t FlipValue(int64_t value) const { return ~value; }
 
  protected:
-  std::shared_ptr<DataType> type_;
   std::unique_ptr<BuilderType> builder_;
   std::unique_ptr<BuilderType> builder_nn_;
 
@@ -1287,7 +1288,7 @@ TEST_F(TestFWBinaryArray, Builder) {
 
   std::shared_ptr<Array> result;
 
-  auto CheckResult = [&length, &is_valid, &raw_data, byte_width](const Array& result) {
+  auto CheckResult = [&length, &is_valid, &raw_data](const Array& result) {
     // Verify output
     const auto& fw_result = static_cast<const FixedSizeBinaryArray&>(result);
 
@@ -2029,7 +2030,6 @@ class TestListArray : public TestBuilder {
 
  protected:
   std::shared_ptr<DataType> value_type_;
-  std::shared_ptr<DataType> type_;
 
   std::shared_ptr<ListBuilder> builder_;
   std::shared_ptr<ListArray> result_;
@@ -2486,7 +2486,6 @@ class TestStructBuilder : public TestBuilder {
 
  protected:
   vector<std::shared_ptr<Field>> value_fields_;
-  std::shared_ptr<DataType> type_;
 
   std::shared_ptr<StructBuilder> builder_;
   std::shared_ptr<StructArray> result_;
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index d12eeb4df9eed..74a3c680d268e 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -198,7 +198,7 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
 class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
  public:
   explicit PoolBuffer(MemoryPool* pool = NULLPTR);
-  virtual ~PoolBuffer();
+  ~PoolBuffer() override;
 
   Status Resize(const int64_t new_size, bool shrink_to_fit = true) override;
   Status Reserve(const int64_t new_capacity) override;
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index a740299dfe194..6f9749d196df9 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -1131,8 +1131,9 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
 
 ListBuilder::ListBuilder(MemoryPool* pool, std::unique_ptr<ArrayBuilder> value_builder,
                          const std::shared_ptr<DataType>& type)
-    : ArrayBuilder(type ? type : std::static_pointer_cast<DataType>(
-                                     std::make_shared<ListType>(value_builder->type())),
+    : ArrayBuilder(type ? type
+                        : std::static_pointer_cast<DataType>(
+                              std::make_shared<ListType>(value_builder->type())),
                    pool),
       offsets_builder_(pool),
       value_builder_(std::move(value_builder)) {}
@@ -1373,8 +1374,8 @@ Status StructBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   return Status::OK();
 }
 
-// ----------------------------------------------------------------------
-// Helper functions
+  // ----------------------------------------------------------------------
+  // Helper functions
 
 #define BUILDER_CASE(ENUM, BuilderType)      \
   case Type::ENUM:                           \
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index d1611f60cd924..9826a6c8370fe 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -857,7 +857,7 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
  public:
   using Scalar = typename internal::DictionaryScalar<T>::type;
 
-  ~DictionaryBuilder() {}
+  ~DictionaryBuilder() override {}
 
   DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
 
@@ -907,7 +907,7 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
 template <>
 class ARROW_EXPORT DictionaryBuilder<NullType> : public ArrayBuilder {
  public:
-  ~DictionaryBuilder();
+  ~DictionaryBuilder() override;
 
   DictionaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
   explicit DictionaryBuilder(MemoryPool* pool);
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 9f07fa7eff826..9ed54ca3a4fc8 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -359,9 +359,8 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       const uint8_t* right_data = right.values()->data();
 
       for (int64_t i = 0; i < left.length(); ++i) {
-        if (left.IsValid(i) &&
-            BitUtil::GetBit(left_data, i + left.offset()) !=
-                BitUtil::GetBit(right_data, i + right.offset())) {
+        if (left.IsValid(i) && BitUtil::GetBit(left_data, i + left.offset()) !=
+                                   BitUtil::GetBit(right_data, i + right.offset())) {
           result_ = false;
           return Status::OK();
         }
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 3fc15018630da..0d27b9df1c129 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -291,14 +291,14 @@ TEST_F(TestCast, ToIntDowncastUnsafe) {
 TEST_F(TestCast, TimestampToTimestamp) {
   CastOptions options;
 
-  auto CheckTimestampCast = [this](
-      const CastOptions& options, TimeUnit::type from_unit, TimeUnit::type to_unit,
-      const vector<int64_t>& from_values, const vector<int64_t>& to_values,
-      const vector<bool>& is_valid) {
-    CheckCase<TimestampType, int64_t, TimestampType, int64_t>(
-        timestamp(from_unit), from_values, is_valid, timestamp(to_unit), to_values,
-        options);
-  };
+  auto CheckTimestampCast =
+      [this](const CastOptions& options, TimeUnit::type from_unit, TimeUnit::type to_unit,
+             const vector<int64_t>& from_values, const vector<int64_t>& to_values,
+             const vector<bool>& is_valid) {
+        CheckCase<TimestampType, int64_t, TimestampType, int64_t>(
+            timestamp(from_unit), from_values, is_valid, timestamp(to_unit), to_values,
+            options);
+      };
 
   vector<bool> is_valid = {true, false, true, true, true};
 
diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc
index afa05485f65b4..eaebd7cefc93d 100644
--- a/cpp/src/arrow/compute/kernels/cast.cc
+++ b/cpp/src/arrow/compute/kernels/cast.cc
@@ -83,8 +83,9 @@ struct is_zero_copy_cast {
 
 template <typename O, typename I>
 struct is_zero_copy_cast<
-    O, I, typename std::enable_if<std::is_same<I, O>::value &&
-                                  !std::is_base_of<ParametricType, O>::value>::type> {
+    O, I,
+    typename std::enable_if<std::is_same<I, O>::value &&
+                            !std::is_base_of<ParametricType, O>::value>::type> {
   static constexpr bool value = true;
 };
 
@@ -121,8 +122,9 @@ struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>
 // Null to other things
 
 template <typename T>
-struct CastFunctor<T, NullType, typename std::enable_if<
-                                    std::is_base_of<FixedWidthType, T>::value>::type> {
+struct CastFunctor<
+    T, NullType,
+    typename std::enable_if<std::is_base_of<FixedWidthType, T>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {}
 };
@@ -172,8 +174,9 @@ struct is_integer_downcast {
 
 template <typename O, typename I>
 struct is_integer_downcast<
-    O, I, typename std::enable_if<std::is_base_of<Integer, O>::value &&
-                                  std::is_base_of<Integer, I>::value>::type> {
+    O, I,
+    typename std::enable_if<std::is_base_of<Integer, O>::value &&
+                            std::is_base_of<Integer, I>::value>::type> {
   using O_T = typename O::c_type;
   using I_T = typename I::c_type;
 
@@ -189,9 +192,10 @@ struct is_integer_downcast<
 };
 
 template <typename O, typename I>
-struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::value &&
-                                                 std::is_base_of<Number, I>::value &&
-                                                 !std::is_same<O, I>::value>::type> {
+struct CastFunctor<O, I,
+                   typename std::enable_if<std::is_same<BooleanType, O>::value &&
+                                           std::is_base_of<Number, I>::value &&
+                                           !std::is_same<O, I>::value>::type> {
   void operator()(FunctionContext* ctx, const CastOptions& options,
                   const ArrayData& input, ArrayData* output) {
     auto in_data = GetValues<typename I::c_type>(input, 1);
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index acbf403987b40..da9797f77bbaf 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -56,7 +56,7 @@ class HashException : public std::exception {
   explicit HashException(const std::string& msg, StatusCode code = StatusCode::Invalid)
       : msg_(msg), code_(code) {}
 
-  ~HashException() throw() {}
+  ~HashException() throw() override {}
 
   const char* what() const throw() override;
 
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 265df4d65225d..3d65834640dfe 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -37,7 +37,7 @@ namespace io {
 
 class ARROW_EXPORT FileOutputStream : public OutputStream {
  public:
-  ~FileOutputStream();
+  ~FileOutputStream() override;
 
   /// \brief Open a local file for writing, truncating any existing file
   /// \param[in] path with UTF8 encoding
@@ -73,7 +73,7 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
 // Operating system file
 class ARROW_EXPORT ReadableFile : public RandomAccessFile {
  public:
-  ~ReadableFile();
+  ~ReadableFile() override;
 
   /// \brief Open a local file for reading
   /// \param[in] path with UTF8 encoding
@@ -125,7 +125,7 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile {
 // FileOutputStream
 class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
  public:
-  ~MemoryMappedFile();
+  ~MemoryMappedFile() override;
 
   /// Create new file with indicated size, return in read/write mode
   static Status Create(const std::string& path, int64_t size,
diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h
index 062473b20104d..a52ec0bc73245 100644
--- a/cpp/src/arrow/io/hdfs.h
+++ b/cpp/src/arrow/io/hdfs.h
@@ -68,7 +68,7 @@ struct HdfsConnectionConfig {
 
 class ARROW_EXPORT HadoopFileSystem : public FileSystem {
  public:
-  ~HadoopFileSystem();
+  ~HadoopFileSystem() override;
 
   // Connect to an HDFS cluster given a configuration
   //
@@ -174,7 +174,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
 
 class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
  public:
-  ~HdfsReadableFile();
+  ~HdfsReadableFile() override;
 
   Status Close() override;
 
@@ -213,7 +213,7 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
 // WriteableFile interface)
 class ARROW_EXPORT HdfsOutputStream : public OutputStream {
  public:
-  ~HdfsOutputStream();
+  ~HdfsOutputStream() override;
 
   Status Close() override;
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index e70431e69580a..a492016ebca1f 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -395,7 +395,7 @@ TEST_F(TestReadableFile, ThreadSafety) {
   std::atomic<int> correct_count(0);
   constexpr int niter = 10000;
 
-  auto ReadData = [&correct_count, &data, this, niter]() {
+  auto ReadData = [&correct_count, &data, this]() {
     std::shared_ptr<Buffer> buffer;
 
     for (int i = 0; i < niter; ++i) {
@@ -588,7 +588,7 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) {
   std::atomic<int> correct_count(0);
   constexpr int niter = 10000;
 
-  auto ReadData = [&correct_count, &data, &file, niter]() {
+  auto ReadData = [&correct_count, &data, &file]() {
     std::shared_ptr<Buffer> buffer;
 
     for (int i = 0; i < niter; ++i) {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index f2ded6ff4b945..380fb34e822c5 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -454,7 +454,7 @@ TYPED_TEST(TestHadoopFileSystem, ThreadSafety) {
   std::atomic<int> correct_count(0);
   constexpr int niter = 1000;
 
-  auto ReadData = [&file, &correct_count, &data, niter]() {
+  auto ReadData = [&file, &correct_count, &data]() {
     for (int i = 0; i < niter; ++i) {
       std::shared_ptr<Buffer> buffer;
       if (i % 2 == 0) {
diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc
index f440c19efe414..faf6a08df9aff 100644
--- a/cpp/src/arrow/ipc/feather.cc
+++ b/cpp/src/arrow/ipc/feather.cc
@@ -579,9 +579,10 @@ class TableWriter::TableWriterImpl : public ArrayVisitor {
         values_bytes = bin_values.raw_value_offsets()[values.length()];
 
         // Write the variable-length offsets
-        RETURN_NOT_OK(WritePadded(stream_.get(), reinterpret_cast<const uint8_t*>(
-                                                     bin_values.raw_value_offsets()),
-                                  offset_bytes, &bytes_written));
+        RETURN_NOT_OK(
+            WritePadded(stream_.get(),
+                        reinterpret_cast<const uint8_t*>(bin_values.raw_value_offsets()),
+                        offset_bytes, &bytes_written));
       } else {
         RETURN_NOT_OK(WritePaddedBlank(stream_.get(), offset_bytes, &bytes_written));
       }
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index 019c9bc1f32d8..1763527def6b3 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -53,7 +53,7 @@ using RecordBatchReader = ::arrow::RecordBatchReader;
 /// reads see the ReadRecordBatch functions
 class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
  public:
-  virtual ~RecordBatchStreamReader();
+  ~RecordBatchStreamReader() override;
 
   /// Create batch reader from generic MessageReader
   ///
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index c6aa770127c88..3debd767ee029 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -106,7 +106,7 @@ class RecordBatchSerializer : public ArrayVisitor {
     DCHECK_GT(max_recursion_depth, 0);
   }
 
-  virtual ~RecordBatchSerializer() = default;
+  ~RecordBatchSerializer() override = default;
 
   Status VisitArray(const Array& arr) {
     if (max_recursion_depth_ <= 0) {
@@ -956,17 +956,17 @@ Status RecordBatchFileWriter::Open(io::OutputStream* sink,
                                    std::shared_ptr<RecordBatchWriter>* out) {
   // ctor is private
   auto result = std::shared_ptr<RecordBatchFileWriter>(new RecordBatchFileWriter());
-  result->impl_.reset(new RecordBatchFileWriterImpl(sink, schema));
+  result->file_impl_.reset(new RecordBatchFileWriterImpl(sink, schema));
   *out = result;
   return Status::OK();
 }
 
 Status RecordBatchFileWriter::WriteRecordBatch(const RecordBatch& batch,
                                                bool allow_64bit) {
-  return impl_->WriteRecordBatch(batch, allow_64bit);
+  return file_impl_->WriteRecordBatch(batch, allow_64bit);
 }
 
-Status RecordBatchFileWriter::Close() { return impl_->Close(); }
+Status RecordBatchFileWriter::Close() { return file_impl_->Close(); }
 
 // ----------------------------------------------------------------------
 // Serialization public APIs
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 013783ee0a224..6dbf29da4806a 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -88,7 +88,7 @@ class ARROW_EXPORT RecordBatchWriter {
 /// format
 class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
  public:
-  virtual ~RecordBatchStreamWriter();
+  ~RecordBatchStreamWriter() override;
 
   /// Create a new writer from stream sink and schema. User is responsible for
   /// closing the actual OutputStream.
@@ -126,7 +126,7 @@ class ARROW_EXPORT RecordBatchStreamWriter : public RecordBatchWriter {
 /// numbers are written at the start and end of the file
 class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
  public:
-  virtual ~RecordBatchFileWriter();
+  ~RecordBatchFileWriter() override;
 
   /// Create a new writer from stream sink and schema
   ///
@@ -151,7 +151,7 @@ class ARROW_EXPORT RecordBatchFileWriter : public RecordBatchStreamWriter {
  private:
   RecordBatchFileWriter();
   class ARROW_NO_EXPORT RecordBatchFileWriterImpl;
-  std::unique_ptr<RecordBatchFileWriterImpl> impl_;
+  std::unique_ptr<RecordBatchFileWriterImpl> file_impl_;
 };
 
 /// \brief Low-level API for writing a record batch (without schema) to an OutputStream
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 3496636a40587..99530432c92cc 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -89,7 +89,7 @@ class DefaultMemoryPool : public MemoryPool {
  public:
   DefaultMemoryPool() : bytes_allocated_(0) { max_memory_ = 0; }
 
-  ~DefaultMemoryPool() {}
+  ~DefaultMemoryPool() override {}
 
   Status Allocate(int64_t size, uint8_t** out) override {
     RETURN_NOT_OK(AllocateAligned(size, out));
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 52ec67fee8dc9..348343b54e918 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -71,7 +71,7 @@ class ARROW_EXPORT MemoryPool {
 class ARROW_EXPORT LoggingMemoryPool : public MemoryPool {
  public:
   explicit LoggingMemoryPool(MemoryPool* pool);
-  virtual ~LoggingMemoryPool() = default;
+  ~LoggingMemoryPool() override = default;
 
   Status Allocate(int64_t size, uint8_t** out) override;
   Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index aa12baacce5fc..f931abe386808 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -62,6 +62,13 @@ set(ARROW_PYTHON_SRCS
   pyarrow.cc
 )
 
+if ("${COMPILER_FAMILY}" STREQUAL "clang")
+  set_property(SOURCE pyarrow.cc
+    APPEND_STRING
+    PROPERTY COMPILE_FLAGS
+    " -Wno-cast-qual ")
+endif()
+
 set(ARROW_PYTHON_SHARED_LINK_LIBS
   arrow_shared
 )
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index 648f6de06a5a1..6960556103b77 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -39,7 +39,7 @@ class ARROW_NO_EXPORT PythonFile;
 class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile {
  public:
   explicit PyReadableFile(PyObject* file);
-  virtual ~PyReadableFile();
+  ~PyReadableFile() override;
 
   Status Close() override;
 
@@ -68,7 +68,7 @@ class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile {
 class ARROW_EXPORT PyOutputStream : public io::OutputStream {
  public:
   explicit PyOutputStream(PyObject* file);
-  virtual ~PyOutputStream();
+  ~PyOutputStream() override;
 
   Status Close() override;
   Status Tell(int64_t* position) const override;
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 3dd5a79cb0403..522bf51740d57 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -67,7 +67,7 @@ constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max();
 
 namespace {
 
-inline bool PyFloat_isnan(const PyObject* obj) {
+inline bool PyFloat_isnan(PyObject* obj) {
   if (PyFloat_Check(obj)) {
     double val = PyFloat_AS_DOUBLE(obj);
     return val != val;
@@ -76,11 +76,11 @@ inline bool PyFloat_isnan(const PyObject* obj) {
   }
 }
 
-inline bool PandasObjectIsNull(const PyObject* obj) {
+inline bool PandasObjectIsNull(PyObject* obj) {
   return obj == Py_None || obj == numpy_nan || PyFloat_isnan(obj);
 }
 
-inline bool PyObject_is_string(const PyObject* obj) {
+inline bool PyObject_is_string(PyObject* obj) {
 #if PY_MAJOR_VERSION >= 3
   return PyUnicode_Check(obj) || PyBytes_Check(obj);
 #else
@@ -88,9 +88,9 @@ inline bool PyObject_is_string(const PyObject* obj) {
 #endif
 }
 
-inline bool PyObject_is_float(const PyObject* obj) { return PyFloat_Check(obj); }
+inline bool PyObject_is_float(PyObject* obj) { return PyFloat_Check(obj); }
 
-inline bool PyObject_is_integer(const PyObject* obj) {
+inline bool PyObject_is_integer(PyObject* obj) {
   return (!PyBool_Check(obj)) && PyArray_IsIntegerScalar(obj);
 }
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 570a650e7fa4a..6938db1be3a96 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -222,7 +222,7 @@ class ARROW_EXPORT Table {
 /// \brief Compute a sequence of record batches from a (possibly chunked) Table
 class ARROW_EXPORT TableBatchReader : public RecordBatchReader {
  public:
-  ~TableBatchReader();
+  ~TableBatchReader() override;
 
   /// \brief Read batches with the maximum possible size
   explicit TableBatchReader(const Table& table);
diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h
index 911adf7b6057a..4f7a268485d8c 100644
--- a/cpp/src/arrow/test-common.h
+++ b/cpp/src/arrow/test-common.h
@@ -119,19 +119,11 @@ std::shared_ptr<Array> TestBase::MakeRandomArray<BinaryArray>(int64_t length,
 
 class TestBuilder : public ::testing::Test {
  public:
-  void SetUp() {
-    pool_ = default_memory_pool();
-    type_ = uint8();
-    builder_.reset(new UInt8Builder(pool_));
-    builder_nn_.reset(new UInt8Builder(pool_));
-  }
+  void SetUp() { pool_ = default_memory_pool(); }
 
  protected:
   MemoryPool* pool_;
-
   std::shared_ptr<DataType> type_;
-  std::unique_ptr<ArrayBuilder> builder_;
-  std::unique_ptr<ArrayBuilder> builder_nn_;
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 836a2aa930f6f..6574cce50ee16 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -364,8 +364,8 @@ std::shared_ptr<Schema> schema(std::vector<std::shared_ptr<Field>>&& fields,
   return std::make_shared<Schema>(std::move(fields), metadata);
 }
 
-// ----------------------------------------------------------------------
-// Visitors and factory functions
+  // ----------------------------------------------------------------------
+  // Visitors and factory functions
 
 #define ACCEPT_VISITOR(TYPE) \
   Status TYPE::Accept(TypeVisitor* visitor) const { return visitor->Visit(*this); }
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 86c17d16801b9..aed3a0184a48b 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -360,23 +360,20 @@ static inline void ByteSwap(void* dst, const void* src, int len) {
 /// Converts to big endian format (if not already in big endian) from the
 /// machine's native endian format.
 #if ARROW_LITTLE_ENDIAN
-template <typename T,
-          typename =
-              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+template <typename T, typename = EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t,
+                                                 int16_t, uint16_t>>
 static inline T ToBigEndian(T value) {
   return ByteSwap(value);
 }
 
-template <typename T,
-          typename =
-              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+template <typename T, typename = EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t,
+                                                 int16_t, uint16_t>>
 static inline T ToLittleEndian(T value) {
   return value;
 }
 #else
-template <typename T,
-          typename =
-              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+template <typename T, typename = EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t,
+                                                 int16_t, uint16_t>>
 static inline T ToBigEndian(T value) {
   return value;
 }
@@ -384,30 +381,26 @@ static inline T ToBigEndian(T value) {
 
 /// Converts from big endian format to the machine's native endian format.
 #if ARROW_LITTLE_ENDIAN
-template <typename T,
-          typename =
-              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+template <typename T, typename = EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t,
+                                                 int16_t, uint16_t>>
 static inline T FromBigEndian(T value) {
   return ByteSwap(value);
 }
 
-template <typename T,
-          typename =
-              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+template <typename T, typename = EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t,
+                                                 int16_t, uint16_t>>
 static inline T FromLittleEndian(T value) {
   return value;
 }
 #else
-template <typename T,
-          typename =
-              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+template <typename T, typename = EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t,
+                                                 int16_t, uint16_t>>
 static inline T FromBigEndian(T value) {
   return value;
 }
 
-template <typename T,
-          typename =
-              EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t, int16_t, uint16_t>>
+template <typename T, typename = EnableIfIsOneOf<T, int64_t, uint64_t, int32_t, uint32_t,
+                                                 int16_t, uint16_t>>
 static inline T FromLittleEndian(T value) {
   return ByteSwap(value);
 }
diff --git a/cpp/src/arrow/util/compression_zlib.h b/cpp/src/arrow/util/compression_zlib.h
index a18af6eaa8079..0a7b0cea15fbe 100644
--- a/cpp/src/arrow/util/compression_zlib.h
+++ b/cpp/src/arrow/util/compression_zlib.h
@@ -38,7 +38,7 @@ class ARROW_EXPORT GZipCodec : public Codec {
   };
 
   explicit GZipCodec(Format format = GZIP);
-  virtual ~GZipCodec();
+  ~GZipCodec() override;
 
   Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
                     uint8_t* output_buffer) override;
diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
index d1af6c666a156..864e1bde749e5 100644
--- a/cpp/src/arrow/util/io-util.h
+++ b/cpp/src/arrow/util/io-util.h
@@ -32,7 +32,7 @@ namespace io {
 class StdoutStream : public OutputStream {
  public:
   StdoutStream() : pos_(0) { set_mode(FileMode::WRITE); }
-  virtual ~StdoutStream() {}
+  ~StdoutStream() override {}
 
   Status Close() override { return Status::OK(); }
 
@@ -55,7 +55,7 @@ class StdoutStream : public OutputStream {
 class StdinStream : public InputStream {
  public:
   StdinStream() : pos_(0) { set_mode(FileMode::READ); }
-  virtual ~StdinStream() {}
+  ~StdinStream() override {}
 
   Status Close() override { return Status::OK(); }
 
diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc
index 87082817f12e9..0a019ddf3e324 100644
--- a/cpp/src/plasma/plasma.cc
+++ b/cpp/src/plasma/plasma.cc
@@ -33,8 +33,9 @@ int warn_if_sigpipe(int status, int client_sock) {
   if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
     ARROW_LOG(WARNING) << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when "
                           "sending a message to client on fd "
-                       << client_sock << ". The client on the other end may "
-                                         "have hung up.";
+                       << client_sock
+                       << ". The client on the other end may "
+                          "have hung up.";
     return errno;
   }
   ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock << ".";
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
index 901601f4747eb..cfaa927bdcf8d 100644
--- a/cpp/src/plasma/plasma.h
+++ b/cpp/src/plasma/plasma.h
@@ -55,8 +55,9 @@ namespace plasma {
         ARROW_LOG(WARNING)                                                  \
             << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \
                "sending a message to client on fd "                         \
-            << fd_ << ". "                                                  \
-                      "The client on the other end may have hung up.";      \
+            << fd_                                                          \
+            << ". "                                                         \
+               "The client on the other end may have hung up.";             \
       } else {                                                              \
         return _s;                                                          \
       }                                                                     \

From 62b9eb29df69456778f339fb00702bc6c55e1394 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Sat, 17 Feb 2018 11:43:11 -0500
Subject: [PATCH 1490/1644] ARROW-1579: [Java] Adding containerized Spark
 Integration tests

This adds configuration files for running containerized integration tests with Spark for Java and Python.  The image is based off of maven with openJDK-8 and will include a copy of Arrow when built on the parent directory to ARROW_HOME.  Running the container will do the following:

1) Install Arrow Java artifacts to m2 repo in container
2) Checkout current Spark master branch
3) Update Spark POM to use the installed version of Arrow and package
4) Run Arrow related Java/Scala Spark tests
5) Run pyarrow related Python Spark tests

Successful completion of the running container will indicate that there are no breaking changes between current Spark code and the Arrow code

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1319 from BryanCutler/docker-spark-integration-ARROW-1579 and squashes the following commits:

3f9f483d [Bryan Cutler] now building with ARROW_BUILD_TOOLCHAIN set to conda env
f962e387 [Bryan Cutler] Revert "added fix for using setuptools_scm to get version outside of arrow python dir"
b2182d31 [Bryan Cutler] docker container now running and passing all tests
3305a352 [Bryan Cutler] added fix for using setuptools_scm to get version outside of arrow python dir
95eb22a5 [Bryan Cutler] running python tests now and building pyarrow, not yet passing pyspark tests
e38d43db [Bryan Cutler] using build/mvn script for spark builds
b04b0e57 [Bryan Cutler] docker running Scala Arrow tests
09c961ee [Bryan Cutler] added initial docker files
---
 dev/docker-compose.yml                     |  5 ++
 dev/spark_integration/Dockerfile           | 70 ++++++++++++++++
 dev/spark_integration/spark_integration.sh | 92 ++++++++++++++++++++++
 3 files changed, 167 insertions(+)
 create mode 100644 dev/spark_integration/Dockerfile
 create mode 100755 dev/spark_integration/spark_integration.sh

diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
index a73fd1bfbbaf0..b1e593cf480c1 100644
--- a/dev/docker-compose.yml
+++ b/dev/docker-compose.yml
@@ -33,3 +33,8 @@ services:
       context: dask_integration
     volumes:
      - ../..:/apache-arrow
+  spark_integration:
+    build: 
+      context: spark_integration
+    volumes:
+     - ../..:/apache-arrow
diff --git a/dev/spark_integration/Dockerfile b/dev/spark_integration/Dockerfile
new file mode 100644
index 0000000000000..d1b3cf89f0b46
--- /dev/null
+++ b/dev/spark_integration/Dockerfile
@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM maven:3.5.2-jdk-8-slim
+
+# Basic OS utilities
+RUN apt-get update && apt-get install -y \
+        wget \
+        git build-essential \
+        software-properties-common
+
+# This will install conda in /home/ubuntu/miniconda
+RUN wget -O /tmp/miniconda.sh \
+    https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
+    rm /tmp/miniconda.sh
+
+# Python dependencies
+RUN apt-get install -y \
+        pkg-config
+
+# Create Conda environment
+ENV PATH="/home/ubuntu/miniconda/bin:${PATH}"
+RUN conda create -y -q -n pyarrow-dev \
+        # Python
+        python=2.7 \
+        numpy \
+        pandas \
+        pytest \
+        cython \
+        ipython \
+        matplotlib \
+        six \
+        setuptools \
+        setuptools_scm \
+        # C++
+        boost-cpp \
+        cmake \
+        flatbuffers \
+        rapidjson \
+        thrift-cpp \
+        snappy \
+        zlib \
+        gflags \
+        brotli \
+        jemalloc \
+        lz4-c \
+        zstd \
+        -c conda-forge
+
+ADD . /apache-arrow
+WORKDIR /apache-arrow
+
+CMD arrow/dev/spark_integration/spark_integration.sh
+
+# BUILD: $ docker build -f arrow/dev/spark_integration/Dockerfile -t spark-arrow .
+# RUN:   $ docker run -v $HOME/.m2:/root/.m2 spark-arrow
diff --git a/dev/spark_integration/spark_integration.sh b/dev/spark_integration/spark_integration.sh
new file mode 100755
index 0000000000000..8ca4dc3ac97ea
--- /dev/null
+++ b/dev/spark_integration/spark_integration.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Exit on any error
+set -e
+
+# Set up environment and working directory
+cd /apache-arrow
+
+# Activate our pyarrow-dev conda env
+source activate pyarrow-dev
+
+export ARROW_HOME=$(pwd)/arrow
+export ARROW_BUILD_TYPE=release
+export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
+export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
+export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m"
+
+# Build Arrow C++
+pushd arrow/cpp
+rm -rf build/*
+mkdir -p build
+cd build/
+cmake -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" -DARROW_PYTHON=on -DARROW_HDFS=on -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE -DCMAKE_INSTALL_PREFIX=$ARROW_HOME ..
+make -j4
+make install
+popd
+
+# Build pyarrow and install inplace
+export PYARROW_CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
+pushd arrow/python
+python setup.py clean
+python setup.py build_ext --build-type=$ARROW_BUILD_TYPE install
+popd
+
+# Install Arrow to local maven repo and get the version
+pushd arrow/java
+echo "Building and installing Arrow Java"
+mvn -DskipTests -Drat.skip=true clean install
+ARROW_VERSION=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | sed -n -e '/^\[.*\]/ !{ /^[0-9]/ { p; q } }'`
+echo "Using Arrow version $ARROW_VERSION"
+popd
+
+# Build Spark with Arrow
+SPARK_REPO=git://git.apache.org/spark.git
+SPARK_BRANCH=master
+
+# Get the Spark repo if not in image already
+if [ ! -d "$(pwd)/spark" ]; then
+    export GIT_COMMITTER_NAME="Nobody"
+    export GIT_COMMITTER_EMAIL="nobody@nowhere.com"
+    git clone "$SPARK_REPO"
+fi
+
+pushd spark
+
+# Make sure branch has no modifications
+git checkout "$SPARK_BRANCH"
+git reset --hard HEAD
+
+# Update Spark pom with the Arrow version just installed and build Spark, need package phase for pyspark
+sed -i -e "s/\(.*<arrow.version>\).*\(<\/arrow.version>\)/\1$ARROW_VERSION\2/g" ./pom.xml
+echo "Building Spark with Arrow $ARROW_VERSION"
+build/mvn -DskipTests clean package
+
+# Run Arrow related Scala tests only, NOTE: -Dtest=_NonExist_ is to enable surefire test discovery without running any tests so that Scalatest can run
+SPARK_SCALA_TESTS="org.apache.spark.sql.execution.arrow,org.apache.spark.sql.execution.vectorized.ColumnarBatchSuite,org.apache.spark.sql.execution.vectorized.ArrowColumnVectorSuite"
+echo "Testing Spark: $SPARK_SCALA_TESTS"
+# TODO: should be able to only build spark-sql tests with adding "-pl sql/core" but not currently working
+build/mvn -Dtest=none -DwildcardSuites="$SPARK_SCALA_TESTS" test
+
+# Run pyarrow related Python tests only
+SPARK_PYTHON_TESTS="ArrowTests PandasUDFTests ScalarPandasUDFTests GroupedMapPandasUDFTests GroupedAggPandasUDFTests"
+echo "Testing PySpark: $SPARK_PYTHON_TESTS"
+SPARK_TESTING=1 bin/pyspark pyspark.sql.tests $SPARK_PYTHON_TESTS 
+popd
+

From a1a34b8ad1fefaefe45c97d92d471f085493776f Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Sun, 18 Feb 2018 14:29:15 -0500
Subject: [PATCH 1491/1644] ARROW-2169: [C++] MSVC is complaining about
 uncaptured variables

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1622 from cpcloud/ARROW-2169 and squashes the following commits:

fc0b02c5 [Phillip Cloud] ARROW-2169: [C++] MSVC is complaining about uncaptured variables
---
 cpp/src/arrow/array-test.cc      | 4 ++--
 cpp/src/arrow/io/io-file-test.cc | 8 ++++----
 cpp/src/arrow/io/io-hdfs-test.cc | 4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index d68b04dc38b8a..1d321e67756b1 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1273,7 +1273,7 @@ class TestFWBinaryArray : public ::testing::Test {
 };
 
 TEST_F(TestFWBinaryArray, Builder) {
-  constexpr int32_t byte_width = 10;
+  int32_t byte_width = 10;
   int64_t length = 4096;
 
   int64_t nbytes = length * byte_width;
@@ -1288,7 +1288,7 @@ TEST_F(TestFWBinaryArray, Builder) {
 
   std::shared_ptr<Array> result;
 
-  auto CheckResult = [&length, &is_valid, &raw_data](const Array& result) {
+  auto CheckResult = [&length, &is_valid, &raw_data, &byte_width](const Array& result) {
     // Verify output
     const auto& fw_result = static_cast<const FixedSizeBinaryArray&>(result);
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index a492016ebca1f..7a7f396368265 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -393,9 +393,9 @@ TEST_F(TestReadableFile, ThreadSafety) {
   ASSERT_OK(ReadableFile::Open(path_, &pool, &file_));
 
   std::atomic<int> correct_count(0);
-  constexpr int niter = 10000;
+  int niter = 10000;
 
-  auto ReadData = [&correct_count, &data, this]() {
+  auto ReadData = [&correct_count, &data, &niter, this]() {
     std::shared_ptr<Buffer> buffer;
 
     for (int i = 0; i < niter; ++i) {
@@ -586,9 +586,9 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) {
   ASSERT_OK(file->Write(data.c_str(), static_cast<int64_t>(data.size())));
 
   std::atomic<int> correct_count(0);
-  constexpr int niter = 10000;
+  int niter = 10000;
 
-  auto ReadData = [&correct_count, &data, &file]() {
+  auto ReadData = [&correct_count, &data, &file, &niter]() {
     std::shared_ptr<Buffer> buffer;
 
     for (int i = 0; i < niter; ++i) {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 380fb34e822c5..610a91fbc2c9d 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -452,9 +452,9 @@ TYPED_TEST(TestHadoopFileSystem, ThreadSafety) {
   ASSERT_OK(this->client_->OpenReadable(src_path, &file));
 
   std::atomic<int> correct_count(0);
-  constexpr int niter = 1000;
+  int niter = 1000;
 
-  auto ReadData = [&file, &correct_count, &data]() {
+  auto ReadData = [&file, &correct_count, &data, &niter]() {
     for (int i = 0; i < niter; ++i) {
       std::shared_ptr<Buffer> buffer;
       if (i % 2 == 0) {

From 864c9391d2f571e443085edcc7f0198b847d7957 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 19 Feb 2018 10:46:11 -0500
Subject: [PATCH 1492/1644] ARROW-2174: [JS] export arrow format and schema
 enums

A bit of cleanup before we finalize the 0.3 JS release. Adds exports for some of the Arrow format and schema enums, refactors the VectorVisitor method names, and adds support for partial visitors by always using `visitTypeInline` in the DataType's `acceptTypeVisitor` impl.

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1624 from trxcllnt/js-export-schema-enums and squashes the following commits:

e873eb0f [Paul Taylor] export arrow format and schema enums
---
 js/src/Arrow.externs.js     |  97 +++++++++++++++----
 js/src/Arrow.ts             |  13 +++
 js/src/type.ts              | 111 +++++-----------------
 js/src/visitor.ts           | 185 ++++++++++++++++--------------------
 js/test/unit/table-tests.ts |   2 +-
 5 files changed, 197 insertions(+), 211 deletions(-)

diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index 438ac8b736cac..21dca8be890b6 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -252,6 +252,67 @@ Type.DenseUnion = function() {};
 /** @type {?} */
 Type.SparseUnion = function() {};
 
+var DateUnit = function() {};
+/** @type {?} */
+DateUnit.DAY = function() {};
+/** @type {?} */
+DateUnit.MILLISECOND = function() {};
+var TimeUnit = function() {};
+/** @type {?} */
+TimeUnit.SECOND = function() {};
+/** @type {?} */
+TimeUnit.MILLISECOND = function() {};
+/** @type {?} */
+TimeUnit.MICROSECOND = function() {};
+/** @type {?} */
+TimeUnit.NANOSECOND = function() {};
+var Precision = function() {};
+/** @type {?} */
+Precision.HALF = function() {};
+/** @type {?} */
+Precision.SINGLE = function() {};
+/** @type {?} */
+Precision.DOUBLE = function() {};
+var UnionMode = function() {};
+/** @type {?} */
+UnionMode.Sparse = function() {};
+/** @type {?} */
+UnionMode.Dense = function() {};
+var VectorType = function() {};
+/** @type {?} */
+VectorType.OFFSET = function() {};
+/** @type {?} */
+VectorType.DATA = function() {};
+/** @type {?} */
+VectorType.VALIDITY = function() {};
+/** @type {?} */
+VectorType.TYPE = function() {};
+var IntervalUnit = function() {};
+/** @type {?} */
+IntervalUnit.YEAR_MONTH = function() {};
+/** @type {?} */
+IntervalUnit.DAY_TIME = function() {};
+var MessageHeader = function() {};
+/** @type {?} */
+MessageHeader.NONE = function() {};
+/** @type {?} */
+MessageHeader.Schema = function() {};
+/** @type {?} */
+MessageHeader.DictionaryBatch = function() {};
+/** @type {?} */
+MessageHeader.RecordBatch = function() {};
+/** @type {?} */
+MessageHeader.Tensor = function() {};
+var MetadataVersion = function() {};
+/** @type {?} */
+MetadataVersion.V1 = function() {};
+/** @type {?} */
+MetadataVersion.V2 = function() {};
+/** @type {?} */
+MetadataVersion.V3 = function() {};
+/** @type {?} */
+MetadataVersion.V4 = function() {};
+
 var DataType = function() {};
 /** @type {?} */
 DataType.isNull = function() {};
@@ -643,38 +704,38 @@ VectorVisitor.prototype.visit;
 /** @type {?} */
 VectorVisitor.prototype.visitMany;
 /** @type {?} */
-VectorVisitor.prototype.visitNullVector;
+VectorVisitor.prototype.visitNull;
 /** @type {?} */
-VectorVisitor.prototype.visitBoolVector;
+VectorVisitor.prototype.visitBool;
 /** @type {?} */
-VectorVisitor.prototype.visitIntVector;
+VectorVisitor.prototype.visitInt;
 /** @type {?} */
-VectorVisitor.prototype.visitFloatVector;
+VectorVisitor.prototype.visitFloat;
 /** @type {?} */
-VectorVisitor.prototype.visitUtf8Vector;
+VectorVisitor.prototype.visitUtf8;
 /** @type {?} */
-VectorVisitor.prototype.visitBinaryVector;
+VectorVisitor.prototype.visitBinary;
 /** @type {?} */
-VectorVisitor.prototype.visitFixedSizeBinaryVector;
+VectorVisitor.prototype.visitFixedSizeBinary;
 /** @type {?} */
-VectorVisitor.prototype.visitDateVector;
+VectorVisitor.prototype.visitDate;
 /** @type {?} */
-VectorVisitor.prototype.visitTimestampVector;
+VectorVisitor.prototype.visitTimestamp;
 /** @type {?} */
-VectorVisitor.prototype.visitTimeVector;
+VectorVisitor.prototype.visitTime;
 /** @type {?} */
-VectorVisitor.prototype.visitDecimalVector;
+VectorVisitor.prototype.visitDecimal;
 /** @type {?} */
-VectorVisitor.prototype.visitListVector;
+VectorVisitor.prototype.visitList;
 /** @type {?} */
-VectorVisitor.prototype.visitStructVector;
+VectorVisitor.prototype.visitStruct;
 /** @type {?} */
-VectorVisitor.prototype.visitUnionVector;
+VectorVisitor.prototype.visitUnion;
 /** @type {?} */
-VectorVisitor.prototype.visitDictionaryVector;
+VectorVisitor.prototype.visitDictionary;
 /** @type {?} */
-VectorVisitor.prototype.visitIntervalVector;
+VectorVisitor.prototype.visitInterval;
 /** @type {?} */
-VectorVisitor.prototype.visitFixedSizeListVector;
+VectorVisitor.prototype.visitFixedSizeList;
 /** @type {?} */
-VectorVisitor.prototype.visitMapVector;
\ No newline at end of file
+VectorVisitor.prototype.visitMap;
\ No newline at end of file
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index e58aa69865bb3..df37a8fb01324 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -59,6 +59,18 @@ export namespace data {
     export import ChunkedData = data_.ChunkedData;
 }
 
+export namespace enum_ {
+    export import Type = type_.ArrowType;
+    export import DateUnit = type_.DateUnit;
+    export import TimeUnit = type_.TimeUnit;
+    export import Precision = type_.Precision;
+    export import UnionMode = type_.UnionMode;
+    export import VectorType = type_.VectorType;
+    export import IntervalUnit = type_.IntervalUnit;
+    export import MessageHeader = type_.MessageHeader;
+    export import MetadataVersion = type_.MetadataVersion;
+}
+
 export namespace type {
     export import Schema = type_.Schema;
     export import Field = type_.Field;
@@ -179,6 +191,7 @@ try {
         Arrow['type'] = type;
         Arrow['util'] = util;
         Arrow['view'] = view;
+        Arrow['enum_'] = enum_;
         Arrow['vector'] = vector;
         Arrow['visitor'] = visitor;
         Arrow['predicate'] = predicate;
diff --git a/js/src/type.ts b/js/src/type.ts
index 6f382bd5b2b05..5e6c939ef67ad 100644
--- a/js/src/type.ts
+++ b/js/src/type.ts
@@ -146,52 +146,31 @@ export abstract class DataType<TType extends Type = any> implements Partial<Visi
     // @ts-ignore
     public [Symbol.toStringTag]: string;
 
-    static            isNull (x: DataType): x is Null            { return x.TType === Type.Null;            }
-    static             isInt (x: DataType): x is Int             { return x.TType === Type.Int;             }
-    static           isFloat (x: DataType): x is Float           { return x.TType === Type.Float;           }
-    static          isBinary (x: DataType): x is Binary          { return x.TType === Type.Binary;          }
-    static            isUtf8 (x: DataType): x is Utf8            { return x.TType === Type.Utf8;            }
-    static            isBool (x: DataType): x is Bool            { return x.TType === Type.Bool;            }
-    static         isDecimal (x: DataType): x is Decimal         { return x.TType === Type.Decimal;         }
-    static            isDate (x: DataType): x is Date_           { return x.TType === Type.Date;            }
-    static            isTime (x: DataType): x is Time            { return x.TType === Type.Time;            }
-    static       isTimestamp (x: DataType): x is Timestamp       { return x.TType === Type.Timestamp;       }
-    static        isInterval (x: DataType): x is Interval        { return x.TType === Type.Interval;        }
-    static            isList (x: DataType): x is List            { return x.TType === Type.List;            }
-    static          isStruct (x: DataType): x is Struct          { return x.TType === Type.Struct;          }
-    static           isUnion (x: DataType): x is Union           { return x.TType === Type.Union;           }
-    static      isDenseUnion (x: DataType): x is DenseUnion      { return x.TType === Type.DenseUnion;      }
-    static     isSparseUnion (x: DataType): x is SparseUnion     { return x.TType === Type.SparseUnion;     }
-    static isFixedSizeBinary (x: DataType): x is FixedSizeBinary { return x.TType === Type.FixedSizeBinary; }
-    static   isFixedSizeList (x: DataType): x is FixedSizeList   { return x.TType === Type.FixedSizeList;   }
-    static             isMap (x: DataType): x is Map_            { return x.TType === Type.Map;             }
-    static      isDictionary (x: DataType): x is Dictionary      { return x.TType === Type.Dictionary;      }
+    static            isNull (x: any): x is Null            { return x && x.TType === Type.Null;            }
+    static             isInt (x: any): x is Int             { return x && x.TType === Type.Int;             }
+    static           isFloat (x: any): x is Float           { return x && x.TType === Type.Float;           }
+    static          isBinary (x: any): x is Binary          { return x && x.TType === Type.Binary;          }
+    static            isUtf8 (x: any): x is Utf8            { return x && x.TType === Type.Utf8;            }
+    static            isBool (x: any): x is Bool            { return x && x.TType === Type.Bool;            }
+    static         isDecimal (x: any): x is Decimal         { return x && x.TType === Type.Decimal;         }
+    static            isDate (x: any): x is Date_           { return x && x.TType === Type.Date;            }
+    static            isTime (x: any): x is Time            { return x && x.TType === Type.Time;            }
+    static       isTimestamp (x: any): x is Timestamp       { return x && x.TType === Type.Timestamp;       }
+    static        isInterval (x: any): x is Interval        { return x && x.TType === Type.Interval;        }
+    static            isList (x: any): x is List            { return x && x.TType === Type.List;            }
+    static          isStruct (x: any): x is Struct          { return x && x.TType === Type.Struct;          }
+    static           isUnion (x: any): x is Union           { return x && x.TType === Type.Union;           }
+    static      isDenseUnion (x: any): x is DenseUnion      { return x && x.TType === Type.DenseUnion;      }
+    static     isSparseUnion (x: any): x is SparseUnion     { return x && x.TType === Type.SparseUnion;     }
+    static isFixedSizeBinary (x: any): x is FixedSizeBinary { return x && x.TType === Type.FixedSizeBinary; }
+    static   isFixedSizeList (x: any): x is FixedSizeList   { return x && x.TType === Type.FixedSizeList;   }
+    static             isMap (x: any): x is Map_            { return x && x.TType === Type.Map;             }
+    static      isDictionary (x: any): x is Dictionary      { return x && x.TType === Type.Dictionary;      }
 
     constructor(public readonly TType: TType,
                 public readonly children?: Field[]) {}
-
-    acceptTypeVisitor(visitor: TypeVisitor): any {
-        switch (this.TType) {
-            case Type.Null:            return DataType.isNull(this)            && visitor.visitNull(this)            || null;
-            case Type.Int:             return DataType.isInt(this)             && visitor.visitInt(this)             || null;
-            case Type.Float:           return DataType.isFloat(this)           && visitor.visitFloat(this)           || null;
-            case Type.Binary:          return DataType.isBinary(this)          && visitor.visitBinary(this)          || null;
-            case Type.Utf8:            return DataType.isUtf8(this)            && visitor.visitUtf8(this)            || null;
-            case Type.Bool:            return DataType.isBool(this)            && visitor.visitBool(this)            || null;
-            case Type.Decimal:         return DataType.isDecimal(this)         && visitor.visitDecimal(this)         || null;
-            case Type.Date:            return DataType.isDate(this)            && visitor.visitDate(this)            || null;
-            case Type.Time:            return DataType.isTime(this)            && visitor.visitTime(this)            || null;
-            case Type.Timestamp:       return DataType.isTimestamp(this)       && visitor.visitTimestamp(this)       || null;
-            case Type.Interval:        return DataType.isInterval(this)        && visitor.visitInterval(this)        || null;
-            case Type.List:            return DataType.isList(this)            && visitor.visitList(this)            || null;
-            case Type.Struct:          return DataType.isStruct(this)          && visitor.visitStruct(this)          || null;
-            case Type.Union:           return DataType.isUnion(this)           && visitor.visitUnion(this)           || null;
-            case Type.FixedSizeBinary: return DataType.isFixedSizeBinary(this) && visitor.visitFixedSizeBinary(this) || null;
-            case Type.FixedSizeList:   return DataType.isFixedSizeList(this)   && visitor.visitFixedSizeList(this)   || null;
-            case Type.Map:             return DataType.isMap(this)             && visitor.visitMap(this)             || null;
-            case Type.Dictionary:      return DataType.isDictionary(this)      && visitor.visitDictionary(this)      || null;
-            default: return null;
-        }
+    public acceptTypeVisitor(visitor: TypeVisitor): any {
+        return TypeVisitor.visitTypeInline(visitor, this);
     }
     protected static [Symbol.toStringTag] = ((proto: DataType) => {
         (<any> proto).ArrayType = Array;
@@ -205,9 +184,6 @@ export class Null extends DataType<Type.Null> {
         super(Type.Null);
     }
     public toString() { return `Null`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitNull(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Null) => {
         return proto[Symbol.toStringTag] = 'Null';
     })(Null.prototype);
@@ -221,7 +197,7 @@ export class Int<TValueType = any, TArrayType extends IntArray = IntArray> exten
     }
     public get ArrayType(): TypedArrayConstructor<TArrayType> {
         switch (this.bitWidth) {
-            case  8: return (this.isSigned ? Int8Array : Uint8Array) as any;
+            case  8: return (this.isSigned ?  Int8Array :  Uint8Array) as any;
             case 16: return (this.isSigned ? Int16Array : Uint16Array) as any;
             case 32: return (this.isSigned ? Int32Array : Uint32Array) as any;
             case 64: return (this.isSigned ? Int32Array : Uint32Array) as any;
@@ -229,7 +205,6 @@ export class Int<TValueType = any, TArrayType extends IntArray = IntArray> exten
         throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
     }
     public toString() { return `${this.isSigned ? `I` : `Ui`}nt${this.bitWidth}`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitInt(this); }
     protected static [Symbol.toStringTag] = ((proto: Int) => {
         return proto[Symbol.toStringTag] = 'Int';
     })(Int.prototype);
@@ -259,7 +234,6 @@ export class Float<TArrayType extends FloatArray = FloatArray> extends DataType<
         throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`);
     }
     public toString() { return `Float${(this.precision << 5) || 16}`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitFloat(this); }
     protected static [Symbol.toStringTag] = ((proto: Float) => {
         return proto[Symbol.toStringTag] = 'Float';
     })(Float.prototype);
@@ -275,9 +249,6 @@ export class Binary extends DataType<Type.Binary> {
         super(Type.Binary);
     }
     public toString() { return `Binary`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitBinary(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Binary) => {
         (<any> proto).ArrayType = Uint8Array;
         return proto[Symbol.toStringTag] = 'Binary';
@@ -290,9 +261,6 @@ export class Utf8 extends DataType<Type.Utf8> {
         super(Type.Utf8);
     }
     public toString() { return `Utf8`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitUtf8(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Utf8) => {
         (<any> proto).ArrayType = Uint8Array;
         return proto[Symbol.toStringTag] = 'Utf8';
@@ -305,9 +273,6 @@ export class Bool extends DataType<Type.Bool> {
         super(Type.Bool);
     }
     public toString() { return `Bool`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitBool(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Bool) => {
         (<any> proto).ArrayType = Uint8Array;
         return proto[Symbol.toStringTag] = 'Bool';
@@ -321,9 +286,6 @@ export class Decimal extends DataType<Type.Decimal> {
         super(Type.Decimal);
     }
     public toString() { return `Decimal[${this.precision}e${this.scale > 0 ? `+` : ``}${this.scale}]`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitDecimal(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Decimal) => {
         (<any> proto).ArrayType = Uint32Array;
         return proto[Symbol.toStringTag] = 'Decimal';
@@ -337,9 +299,6 @@ export class Date_ extends DataType<Type.Date> {
         super(Type.Date);
     }
     public toString() { return `Date${(this.unit + 1) * 32}<${DateUnit[this.unit]}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitDate(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Date_) => {
         (<any> proto).ArrayType = Int32Array;
         return proto[Symbol.toStringTag] = 'Date';
@@ -353,9 +312,6 @@ export class Time extends DataType<Type.Time> {
         super(Type.Time);
     }
     public toString() { return `Time${this.bitWidth}<${TimeUnit[this.unit]}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitTime(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Time) => {
         (<any> proto).ArrayType = Uint32Array;
         return proto[Symbol.toStringTag] = 'Time';
@@ -368,9 +324,6 @@ export class Timestamp extends DataType<Type.Timestamp> {
         super(Type.Timestamp);
     }
     public toString() { return `Timestamp<${TimeUnit[this.unit]}${this.timezone ? `, ${this.timezone}` : ``}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitTimestamp(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Timestamp) => {
         (<any> proto).ArrayType = Int32Array;
         return proto[Symbol.toStringTag] = 'Timestamp';
@@ -383,9 +336,6 @@ export class Interval extends DataType<Type.Interval> {
         super(Type.Interval);
     }
     public toString() { return `Interval<${IntervalUnit[this.unit]}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitInterval(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Interval) => {
         (<any> proto).ArrayType = Int32Array;
         return proto[Symbol.toStringTag] = 'Interval';
@@ -401,9 +351,6 @@ export class List<T extends DataType = any> extends DataType<Type.List> {
     public get ArrayType() { return this.valueType.ArrayType; }
     public get valueType() { return this.children[0].type as T; }
     public get valueField() { return this.children[0] as Field<T>; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitList(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: List) => {
         return proto[Symbol.toStringTag] = 'List';
     })(List.prototype);
@@ -415,9 +362,6 @@ export class Struct extends DataType<Type.Struct> {
         super(Type.Struct, children);
     }
     public toString() { return `Struct<${this.children.map((f) => f.type).join(`, `)}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitStruct(this);
-    }
     protected static [Symbol.toStringTag] = ((proto: Struct) => {
         return proto[Symbol.toStringTag] = 'Struct';
     })(Struct.prototype);
@@ -431,7 +375,6 @@ export class Union<TType extends Type = any> extends DataType<TType> {
         super(<TType> (mode === UnionMode.Sparse ? Type.SparseUnion : Type.DenseUnion), children);
     }
     public toString() { return `${this[Symbol.toStringTag]}<${this.typeIds.map((x) => Type[x]).join(` | `)}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitUnion(this); }
     protected static [Symbol.toStringTag] = ((proto: Union) => {
         (<any> proto).ArrayType = Int8Array;
         return proto[Symbol.toStringTag] = 'Union';
@@ -462,7 +405,6 @@ export class FixedSizeBinary extends DataType<Type.FixedSizeBinary> {
         super(Type.FixedSizeBinary);
     }
     public toString() { return `FixedSizeBinary[${this.byteWidth}]`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitFixedSizeBinary(this); }
     protected static [Symbol.toStringTag] = ((proto: FixedSizeBinary) => {
         (<any> proto).ArrayType = Uint8Array;
         return proto[Symbol.toStringTag] = 'FixedSizeBinary';
@@ -479,7 +421,6 @@ export class FixedSizeList<T extends DataType = any> extends DataType<Type.Fixed
     public get valueType() { return this.children[0].type as T; }
     public get valueField() { return this.children[0] as Field<T>; }
     public toString() { return `FixedSizeList[${this.listSize}]<${this.valueType}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitFixedSizeList(this); }
     protected static [Symbol.toStringTag] = ((proto: FixedSizeList) => {
         return proto[Symbol.toStringTag] = 'FixedSizeList';
     })(FixedSizeList.prototype);
@@ -493,7 +434,6 @@ export class Map_ extends DataType<Type.Map> {
         super(Type.Map, children);
     }
     public toString() { return `Map<${this.children.join(`, `)}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any { return visitor.visitMap(this); }
     protected static [Symbol.toStringTag] = ((proto: Map_) => {
         return proto[Symbol.toStringTag] = 'Map_';
     })(Map_.prototype);
@@ -513,10 +453,7 @@ export class Dictionary<T extends DataType> extends DataType<Type.Dictionary> {
         this.id = id == null ? DictionaryBatch.getId() : typeof id === 'number' ? id : id.low;
     }
     public get ArrayType() { return this.dictionary.ArrayType; }
-    public toString() { return `Dictionary<${this.dictionary}, ${this.indicies}>`; }
-    public acceptTypeVisitor(visitor: TypeVisitor): any {
-        return visitor.visitDictionary(this);
-    }
+    public toString() { return `Dictionary<${this.indicies}, ${this.dictionary}>`; }
     protected static [Symbol.toStringTag] = ((proto: Dictionary) => {
         return proto[Symbol.toStringTag] = 'Dictionary';
     })(Dictionary.prototype);
diff --git a/js/src/visitor.ts b/js/src/visitor.ts
index 3f46f6681c7e7..c88b6bd311579 100644
--- a/js/src/visitor.ts
+++ b/js/src/visitor.ts
@@ -24,131 +24,106 @@ import { Bool, Null, Int, Float, Date_, Time, Interval, Timestamp } from './type
 export interface VisitorNode {
     acceptTypeVisitor(visitor: TypeVisitor): any;
     acceptVectorVisitor(visitor: VectorVisitor): any;
-    // acceptMessageVisitor(visitor: MessageVisitor): any;
 }
 
 export abstract class TypeVisitor {
-    visit(node: Partial<VisitorNode>): any {
-        return node.acceptTypeVisitor!(this);
+    visit(type: Partial<VisitorNode>): any {
+        return type.acceptTypeVisitor && type.acceptTypeVisitor(this) || null;
     }
-    visitMany(nodes: Partial<VisitorNode>[]): any[] {
-        return nodes.map((node) => this.visit(node));
+    visitMany(types: Partial<VisitorNode>[]): any[] {
+        return types.map((type) => this.visit(type));
     }
-    abstract visitNull(node: Null): any;
-    abstract visitBool(node: Bool): any;
-    abstract visitInt(node: Int): any;
-    abstract visitFloat(node: Float): any;
-    abstract visitUtf8(node: Utf8): any;
-    abstract visitBinary(node: Binary): any;
-    abstract visitFixedSizeBinary(node: FixedSizeBinary): any;
-    abstract visitDate(node: Date_): any;
-    abstract visitTimestamp(node: Timestamp): any;
-    abstract visitTime(node: Time): any;
-    abstract visitDecimal(node: Decimal): any;
-    abstract visitList(node: List): any;
-    abstract visitStruct(node: Struct): any;
-    abstract visitUnion(node: Union<any>): any;
-    abstract visitDictionary(node: Dictionary): any;
-    abstract visitInterval(node: Interval): any;
-    abstract visitFixedSizeList(node: FixedSizeList): any;
-    abstract visitMap(node: Map_): any;
+    abstract visitNull?(type: Null): any;
+    abstract visitBool?(type: Bool): any;
+    abstract visitInt?(type: Int): any;
+    abstract visitFloat?(type: Float): any;
+    abstract visitUtf8?(type: Utf8): any;
+    abstract visitBinary?(type: Binary): any;
+    abstract visitFixedSizeBinary?(type: FixedSizeBinary): any;
+    abstract visitDate?(type: Date_): any;
+    abstract visitTimestamp?(type: Timestamp): any;
+    abstract visitTime?(type: Time): any;
+    abstract visitDecimal?(type: Decimal): any;
+    abstract visitList?(type: List): any;
+    abstract visitStruct?(type: Struct): any;
+    abstract visitUnion?(type: Union<any>): any;
+    abstract visitDictionary?(type: Dictionary): any;
+    abstract visitInterval?(type: Interval): any;
+    abstract visitFixedSizeList?(type: FixedSizeList): any;
+    abstract visitMap?(type: Map_): any;
 
     static visitTypeInline<T extends DataType>(visitor: TypeVisitor, type: T): any {
         switch (type.TType) {
-            case Type.Null:            return visitor.visitNull(type            as any as Null);
-            case Type.Int:             return visitor.visitInt(type             as any as Int);
-            case Type.Float:           return visitor.visitFloat(type           as any as Float);
-            case Type.Binary:          return visitor.visitBinary(type          as any as Binary);
-            case Type.Utf8:            return visitor.visitUtf8(type            as any as Utf8);
-            case Type.Bool:            return visitor.visitBool(type            as any as Bool);
-            case Type.Decimal:         return visitor.visitDecimal(type         as any as Decimal);
-            case Type.Date:            return visitor.visitDate(type            as any as Date_);
-            case Type.Time:            return visitor.visitTime(type            as any as Time);
-            case Type.Timestamp:       return visitor.visitTimestamp(type       as any as Timestamp);
-            case Type.Interval:        return visitor.visitInterval(type        as any as Interval);
-            case Type.List:            return visitor.visitList(type            as any as List<T>);
-            case Type.Struct:          return visitor.visitStruct(type          as any as Struct);
-            case Type.Union:           return visitor.visitUnion(type           as any as Union);
-            case Type.FixedSizeBinary: return visitor.visitFixedSizeBinary(type as any as FixedSizeBinary);
-            case Type.FixedSizeList:   return visitor.visitFixedSizeList(type   as any as FixedSizeList);
-            case Type.Map:             return visitor.visitMap(type             as any as Map_);
-            case Type.Dictionary:      return visitor.visitDictionary(type      as any as Dictionary);
+            case Type.Null:            return visitor.visitNull            && visitor.visitNull(type            as any as Null);
+            case Type.Int:             return visitor.visitInt             && visitor.visitInt(type             as any as Int);
+            case Type.Float:           return visitor.visitFloat           && visitor.visitFloat(type           as any as Float);
+            case Type.Binary:          return visitor.visitBinary          && visitor.visitBinary(type          as any as Binary);
+            case Type.Utf8:            return visitor.visitUtf8            && visitor.visitUtf8(type            as any as Utf8);
+            case Type.Bool:            return visitor.visitBool            && visitor.visitBool(type            as any as Bool);
+            case Type.Decimal:         return visitor.visitDecimal         && visitor.visitDecimal(type         as any as Decimal);
+            case Type.Date:            return visitor.visitDate            && visitor.visitDate(type            as any as Date_);
+            case Type.Time:            return visitor.visitTime            && visitor.visitTime(type            as any as Time);
+            case Type.Timestamp:       return visitor.visitTimestamp       && visitor.visitTimestamp(type       as any as Timestamp);
+            case Type.Interval:        return visitor.visitInterval        && visitor.visitInterval(type        as any as Interval);
+            case Type.List:            return visitor.visitList            && visitor.visitList(type            as any as List<T>);
+            case Type.Struct:          return visitor.visitStruct          && visitor.visitStruct(type          as any as Struct);
+            case Type.Union:           return visitor.visitUnion           && visitor.visitUnion(type           as any as Union);
+            case Type.FixedSizeBinary: return visitor.visitFixedSizeBinary && visitor.visitFixedSizeBinary(type as any as FixedSizeBinary);
+            case Type.FixedSizeList:   return visitor.visitFixedSizeList   && visitor.visitFixedSizeList(type   as any as FixedSizeList);
+            case Type.Map:             return visitor.visitMap             && visitor.visitMap(type             as any as Map_);
+            case Type.Dictionary:      return visitor.visitDictionary      && visitor.visitDictionary(type      as any as Dictionary);
             default: return null;
         }
     }
 }
 
 export abstract class VectorVisitor {
-    visit(node: Partial<VisitorNode>): any {
-        return node.acceptVectorVisitor!(this);
+    visit(vector: Partial<VisitorNode>): any {
+        return vector.acceptVectorVisitor && vector.acceptVectorVisitor(this) || null;
     }
-    visitMany(nodes: Partial<VisitorNode>[]): any[] {
-        return nodes.map((node) => this.visit(node));
+    visitMany(vectors: Partial<VisitorNode>[]): any[] {
+        return vectors.map((vector) => this.visit(vector));
     }
-    abstract visitNullVector(node: Vector<Null>): any;
-    abstract visitBoolVector(node: Vector<Bool>): any;
-    abstract visitIntVector(node: Vector<Int>): any;
-    abstract visitFloatVector(node: Vector<Float>): any;
-    abstract visitUtf8Vector(node: Vector<Utf8>): any;
-    abstract visitBinaryVector(node: Vector<Binary>): any;
-    abstract visitFixedSizeBinaryVector(node: Vector<FixedSizeBinary>): any;
-    abstract visitDateVector(node: Vector<Date_>): any;
-    abstract visitTimestampVector(node: Vector<Timestamp>): any;
-    abstract visitTimeVector(node: Vector<Time>): any;
-    abstract visitDecimalVector(node: Vector<Decimal>): any;
-    abstract visitListVector(node: Vector<List>): any;
-    abstract visitStructVector(node: Vector<Struct>): any;
-    abstract visitUnionVector(node: Vector<Union<any>>): any;
-    abstract visitDictionaryVector(node: Vector<Dictionary>): any;
-    abstract visitIntervalVector(node: Vector<Interval>): any;
-    abstract visitFixedSizeListVector(node: Vector<FixedSizeList>): any;
-    abstract visitMapVector(node: Vector<Map_>): any;
+    abstract visitNull?(vector: Vector<Null>): any;
+    abstract visitBool?(vector: Vector<Bool>): any;
+    abstract visitInt?(vector: Vector<Int>): any;
+    abstract visitFloat?(vector: Vector<Float>): any;
+    abstract visitUtf8?(vector: Vector<Utf8>): any;
+    abstract visitBinary?(vector: Vector<Binary>): any;
+    abstract visitFixedSizeBinary?(vector: Vector<FixedSizeBinary>): any;
+    abstract visitDate?(vector: Vector<Date_>): any;
+    abstract visitTimestamp?(vector: Vector<Timestamp>): any;
+    abstract visitTime?(vector: Vector<Time>): any;
+    abstract visitDecimal?(vector: Vector<Decimal>): any;
+    abstract visitList?(vector: Vector<List>): any;
+    abstract visitStruct?(vector: Vector<Struct>): any;
+    abstract visitUnion?(vector: Vector<Union<any>>): any;
+    abstract visitDictionary?(vector: Vector<Dictionary>): any;
+    abstract visitInterval?(vector: Vector<Interval>): any;
+    abstract visitFixedSizeList?(vector: Vector<FixedSizeList>): any;
+    abstract visitMap?(vector: Vector<Map_>): any;
 
     static visitTypeInline<T extends DataType>(visitor: VectorVisitor, type: T, vector: Vector<T>): any {
         switch (type.TType) {
-            case Type.Null:            return visitor.visitNullVector(vector            as any as Vector<Null>);
-            case Type.Int:             return visitor.visitIntVector(vector             as any as Vector<Int>);
-            case Type.Float:           return visitor.visitFloatVector(vector           as any as Vector<Float>);
-            case Type.Binary:          return visitor.visitBinaryVector(vector          as any as Vector<Binary>);
-            case Type.Utf8:            return visitor.visitUtf8Vector(vector            as any as Vector<Utf8>);
-            case Type.Bool:            return visitor.visitBoolVector(vector            as any as Vector<Bool>);
-            case Type.Decimal:         return visitor.visitDecimalVector(vector         as any as Vector<Decimal>);
-            case Type.Date:            return visitor.visitDateVector(vector            as any as Vector<Date_>);
-            case Type.Time:            return visitor.visitTimeVector(vector            as any as Vector<Time>);
-            case Type.Timestamp:       return visitor.visitTimestampVector(vector       as any as Vector<Timestamp>);
-            case Type.Interval:        return visitor.visitIntervalVector(vector        as any as Vector<Interval>);
-            case Type.List:            return visitor.visitListVector(vector            as any as Vector<List<T>>);
-            case Type.Struct:          return visitor.visitStructVector(vector          as any as Vector<Struct>);
-            case Type.Union:           return visitor.visitUnionVector(vector           as any as Vector<Union>);
-            case Type.FixedSizeBinary: return visitor.visitFixedSizeBinaryVector(vector as any as Vector<FixedSizeBinary>);
-            case Type.FixedSizeList:   return visitor.visitFixedSizeListVector(vector   as any as Vector<FixedSizeList>);
-            case Type.Map:             return visitor.visitMapVector(vector             as any as Vector<Map_>);
-            case Type.Dictionary:      return visitor.visitDictionaryVector(vector      as any as Vector<Dictionary>);
+            case Type.Null:            return visitor.visitNull            && visitor.visitNull(vector            as any as Vector<Null>);
+            case Type.Int:             return visitor.visitInt             && visitor.visitInt(vector             as any as Vector<Int>);
+            case Type.Float:           return visitor.visitFloat           && visitor.visitFloat(vector           as any as Vector<Float>);
+            case Type.Binary:          return visitor.visitBinary          && visitor.visitBinary(vector          as any as Vector<Binary>);
+            case Type.Utf8:            return visitor.visitUtf8            && visitor.visitUtf8(vector            as any as Vector<Utf8>);
+            case Type.Bool:            return visitor.visitBool            && visitor.visitBool(vector            as any as Vector<Bool>);
+            case Type.Decimal:         return visitor.visitDecimal         && visitor.visitDecimal(vector         as any as Vector<Decimal>);
+            case Type.Date:            return visitor.visitDate            && visitor.visitDate(vector            as any as Vector<Date_>);
+            case Type.Time:            return visitor.visitTime            && visitor.visitTime(vector            as any as Vector<Time>);
+            case Type.Timestamp:       return visitor.visitTimestamp       && visitor.visitTimestamp(vector       as any as Vector<Timestamp>);
+            case Type.Interval:        return visitor.visitInterval        && visitor.visitInterval(vector        as any as Vector<Interval>);
+            case Type.List:            return visitor.visitList            && visitor.visitList(vector            as any as Vector<List<T>>);
+            case Type.Struct:          return visitor.visitStruct          && visitor.visitStruct(vector          as any as Vector<Struct>);
+            case Type.Union:           return visitor.visitUnion           && visitor.visitUnion(vector           as any as Vector<Union>);
+            case Type.FixedSizeBinary: return visitor.visitFixedSizeBinary && visitor.visitFixedSizeBinary(vector as any as Vector<FixedSizeBinary>);
+            case Type.FixedSizeList:   return visitor.visitFixedSizeList   && visitor.visitFixedSizeList(vector   as any as Vector<FixedSizeList>);
+            case Type.Map:             return visitor.visitMap             && visitor.visitMap(vector             as any as Vector<Map_>);
+            case Type.Dictionary:      return visitor.visitDictionary      && visitor.visitDictionary(vector      as any as Vector<Dictionary>);
             default: return null;
         }
     }
 }
-
-// import { Footer, Block } from './ipc/message';
-// import { Field, FieldNode, Buffer } from './ipc/message';
-// import { Message, Schema, RecordBatch, DictionaryBatch } from './ipc/message';
-
-// export abstract class MessageVisitor {
-//     visit(node: VisitorNode): any {
-//         return node.acceptMessageVisitor(this);
-//     }
-//     visitMany(nodes: VisitorNode[]): any[] {
-//         return nodes.map((node) => this.visit(node));
-//     }
-//     abstract visitFooter(node: Footer): any;
-//     abstract visitBlock(node: Block): any;
-//     abstract visitMessage(node: Message): any;
-//     abstract visitSchema(node: Schema): any;
-//     abstract visitField<T extends DataType>(node: Field<T>): any;
-//     abstract visitBuffer(node: Buffer): any;
-//     abstract visitFieldNode(node: FieldNode): any;
-//     abstract visitDataType<T extends Type>(node: DataType<T>): any;
-//     abstract visitDictionary(node: Dictionary): any;
-//     abstract visitRecordBatch(node: RecordBatch): any;
-//     abstract visitDictionaryBatch(node: DictionaryBatch): any;
-// }
diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts
index 8dd30de935e80..ffcc8f477a29f 100644
--- a/js/test/unit/table-tests.ts
+++ b/js/test/unit/table-tests.ts
@@ -436,7 +436,7 @@ describe(`Table`, () => {
             });
             test(`table.toString()`, () => {
                 let selected = table.select('i32', 'dictionary');
-                let headers  = [`"row_id"`, `"i32: Int32"`, `"dictionary: Dictionary<Utf8, Int8>"`];
+                let headers  = [`"row_id"`, `"i32: Int32"`, `"dictionary: Dictionary<Int8, Utf8>"`];
                 let expected = [headers.join(' | '), ...values.map((row, idx) => {
                     return [`${idx}`, `${row[I32]}`, `"${row[DICT]}"`].map((str, col) => {
                                 return leftPad(str, ' ', headers[col].length);

From 2106152d050adfc04b467407c9569e9f6e8c867c Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 19 Feb 2018 10:53:49 -0500
Subject: [PATCH 1493/1644] ARROW-2163: [CI] Make apt installs explicit

Together with increasing timeouts and retries, this will hopefully make the builds less flaky. Besides, we can now skip installing for jobs that don't have related changes (e.g. Python jobs when a Java-only change is tested), speeding up builds slightly.

Author: Antoine Pitrou <antoine@python.org>

Closes #1625 from pitrou/ARROW-2163-explicit-apt-installs and squashes the following commits:

eb94e1f8 [Antoine Pitrou] Remove temporary changes [skip appveyor]
563d176f [Antoine Pitrou] glib-only change [skip appveyor]
b7781033 [Antoine Pitrou] Factor out c_glib installs [skip ci]
31923fee [Antoine Pitrou] Java-only change [skip appveyor]
f702fcc1 [Antoine Pitrou] Fix integration build [skip ci]
701fc9e6 [Antoine Pitrou] Python-only change [skip appveyor]
427a9dcb [Antoine Pitrou] Reenable all builds [skip appveyor]
4669b017 [Antoine Pitrou] Avoid travis_before_script_cpp.sh if not C++ was modified [skip appveyor]
1720020a [Antoine Pitrou] Merge branch 'master' into ARROW-2163-explicit-apt-installs
2c8b4a65 [Antoine Pitrou] Dummy change [skip appveyor]
2d95dbf0 [Antoine Pitrou] Python-only change [skip appveyor]
bdcdd3e3 [Antoine Pitrou] [skip ci]
4e08a980 [Antoine Pitrou] Python-only change [skip appveyor]
3d16e1b4 [Antoine Pitrou] glib-only change [skip appveyor]
0a8a70c7 [Antoine Pitrou] JS-only change [skip appveyor]
eb0ea328 [Antoine Pitrou] Java-only change [skip appveyor]
9a2a2fe3 [Antoine Pitrou] glib-only change [skip appveyor]
9dcc93ab [Antoine Pitrou] ARROW-2163 [skip appveyor]
---
 .travis.yml                       | 155 +++++++++++++++---------------
 ci/travis_before_script_c_glib.sh |   6 ++
 ci/travis_install_clang_tools.sh  |   2 +-
 ci/travis_install_linux.sh        |  26 +++++
 4 files changed, 112 insertions(+), 77 deletions(-)
 create mode 100755 ci/travis_install_linux.sh

diff --git a/.travis.yml b/.travis.yml
index 9ef16b5cf0b52..73a9f46427081 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,23 +17,6 @@
 
 sudo: required
 dist: trusty
-addons:
-  apt:
-    sources:
-    - ubuntu-toolchain-r-test
-    packages:
-    - gcc-4.9   # Needed for C++11
-    - g++-4.9   # Needed for C++11
-    - gdb
-    - ccache
-    - valgrind
-    - libboost-dev
-    - libboost-filesystem-dev
-    - libboost-system-dev
-    - libjemalloc-dev
-    - gtk-doc-tools
-    - autoconf-archive
-    - libgirepository1.0-dev
 
 services:
   - docker
@@ -41,6 +24,18 @@ services:
 cache:
   ccache: true
 
+before_install:
+  # Common pre-install steps for all builds
+  - ulimit -c unlimited -S
+  - |
+      if [ $TRAVIS_OS_NAME == "linux" ]; then
+        sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
+        sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+        sudo apt-get update -qq
+      fi
+  - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+
+
 matrix:
   fast_finish: true
   allow_failures:
@@ -51,24 +46,27 @@ matrix:
     language: cpp
     os: linux
     group: deprecated
+    env:
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - ARROW_TRAVIS_VALGRIND=1
+    - ARROW_TRAVIS_PLASMA=1
+    - ARROW_TRAVIS_ORC=1
+    - ARROW_TRAVIS_CLANG_FORMAT=1
+    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    - CC="clang-5.0"
+    - CXX="clang++-5.0"
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
-    - export ARROW_TRAVIS_USE_TOOLCHAIN=1
-    - export ARROW_TRAVIS_VALGRIND=1
-    - export ARROW_TRAVIS_PLASMA=1
-    - export ARROW_TRAVIS_ORC=1
-    - export ARROW_TRAVIS_CLANG_FORMAT=1
-    - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    - export CC="clang-5.0"
-    - export CXX="clang++-5.0"
+    # (ARROW_CI_CPP_AFFECTED implies ARROW_CI_PYTHON_AFFECTED)
+    - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
     - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
     script:
     - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
-    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh; fi
-    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7; fi
-    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
   # [OS X] C++ & Python w/ XCode 6.4
   - compiler: clang
     language: cpp
@@ -76,22 +74,22 @@ matrix:
     os: osx
     cache:
     addons:
+    env:
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - ARROW_TRAVIS_PLASMA=1
+    - ARROW_TRAVIS_ORC=1
+    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
-    - export ARROW_TRAVIS_USE_TOOLCHAIN=1
-    - export ARROW_TRAVIS_PLASMA=1
-    - export ARROW_TRAVIS_ORC=1
-    - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
     - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
     script:
     - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
-    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh; fi
-    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7; fi
-    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
   # [manylinux1] Python
   - language: cpp
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
     - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest; fi
     script:
     - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
@@ -100,18 +98,19 @@ matrix:
     os: linux
     jdk: openjdk7
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+    - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     script:
-    - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_java.sh; fi
-    - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh
   # Java w/ Oracle JDK 9
   - language: java
     os: linux
     jdk: oraclejdk9
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+    - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
     script:
-    - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_java.sh; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
     addons:
       apt:
         packages:
@@ -121,51 +120,58 @@ matrix:
     os: linux
     env: ARROW_TEST_GROUP=integration
     jdk: openjdk8
+    env:
+    - CC="clang-5.0"
+    - CXX="clang++-5.0"
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
-    - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    - export CC="clang-5.0"
-    - export CXX="clang++-5.0"
+    - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - nvm install node
-    - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh; fi
-    - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
-    - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
   # NodeJS
   - language: node_js
     os: linux
     node_js: node
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
-    - if [ $ARROW_CI_JS_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh; fi
+    - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
     script:
-    - if [ $ARROW_CI_JS_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_js.sh; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
   # C++ & glib w/ gcc 4.9 & autotools
   - compiler: gcc
     language: cpp
     os: linux
-    env: BUILD_SYSTEM=autotools BUILD_TORCH_EXAMPLE=no
+    env:
+    - BUILD_SYSTEM=autotools BUILD_TORCH_EXAMPLE=no
+    - CC="gcc-4.9"
+    - CXX="g++-4.9"
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
-    - export CC="gcc-4.9"
-    - export CXX="g++-4.9"
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
+    - if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
   # C++ & glib w/ gcc 4.9 & meson
   - compiler: gcc
     language: cpp
     os: linux
-    env: BUILD_SYSTEM=meson BUILD_TORCH_EXAMPLE=no
+    env:
+    - BUILD_SYSTEM=meson BUILD_TORCH_EXAMPLE=no
+    - CC="gcc-4.9"
+    - CXX="g++-4.9"
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
-    - export CC="gcc-4.9"
-    - export CXX="g++-4.9"
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
+    - if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
   # [OS X] C++ & glib w/ XCode 8.3 & autotools
   - compiler: clang
     osx_image: xcode8.3
@@ -173,17 +179,14 @@ matrix:
     cache:
     addons:
     rvm: 2.2
-    env: BUILD_SYSTEM=autotools
+    env:
+    - BUILD_SYSTEM=autotools
     before_script:
-    - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile; fi
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
+    - if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
-    - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
-
-before_install:
-- ulimit -c unlimited -S
+    - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
 
 after_failure:
 - COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1)
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 9d07e02a84302..27d1e86fd95f6 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -21,6 +21,12 @@ set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
+if [ $TRAVIS_OS_NAME = "osx" ]; then
+  brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile
+else  # Linux
+  sudo apt-get install -y -q gtk-doc-tools autoconf-archive libgirepository1.0-dev
+fi
+
 if [ $TRAVIS_OS_NAME = "osx" ]; then
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
   export XML_CATALOG_FILES=/usr/local/etc/xml/catalog
diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
index 630c09e31ff91..9c89253dfd867 100755
--- a/ci/travis_install_clang_tools.sh
+++ b/ci/travis_install_clang_tools.sh
@@ -20,5 +20,5 @@
 wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
 sudo apt-add-repository -y \
      "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-5.0 main"
-sudo apt-get update -q
+sudo apt-get update -qq
 sudo apt-get install -q clang-5.0 clang-format-5.0 clang-tidy-5.0
diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh
new file mode 100755
index 0000000000000..acee9ebcb2ac3
--- /dev/null
+++ b/ci/travis_install_linux.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+sudo apt-get install -y -q \
+    gdb ccache libboost-dev libboost-filesystem-dev \
+    libboost-system-dev libjemalloc-dev
+
+if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then
+    sudo apt-get install -y -q valgrind
+fi

From d64e144cd4de03ad517f0d38ea72c2f308a8ea35 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 19 Feb 2018 11:14:15 -0500
Subject: [PATCH 1494/1644] =?UTF-8?q?ARROW-2170:=20[Python]=C2=A0construct?=
 =?UTF-8?q?=5Fmetadata=20fails=20on=20reading=20files=20where=20no=20index?=
 =?UTF-8?q?=20was=20preserved?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

cc @cpcloud

The result was that we only persisted empty Pandas metadata for these files.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1623 from xhochy/ARROW-2170 and squashes the following commits:

ba1ce3f3 [Uwe L. Korn] ARROW-2170: [Python] construct_metadata fails on reading files where no index was preserved
---
 python/pyarrow/pandas_compat.py      | 7 +++++--
 python/pyarrow/tests/test_parquet.py | 3 +++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index f5e56a9b28420..e8fa83fe71383 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -197,8 +197,11 @@ def construct_metadata(df, column_names, index_levels, index_column_names,
     -------
     dict
     """
-    df_types = types[:-len(index_levels)]
-    index_types = types[-len(index_levels):]
+    # Use ntypes instead of Python shorthand notation [:-len(x)] as [:-0]
+    # behaves differently to what we want.
+    ntypes = len(types)
+    df_types = types[:ntypes - len(index_levels)]
+    index_types = types[ntypes - len(index_levels):]
 
     column_metadata = [
         get_column_metadata(
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index c49f3d396b888..bd76feb2e8cfd 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -215,6 +215,9 @@ def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tmpdir):
     arrow_table = pa.Table.from_pandas(df, preserve_index=False)
     js = json.loads(arrow_table.schema.metadata[b'pandas'].decode('utf8'))
     assert not js['index_columns']
+    # ARROW-2170
+    # While index_columns should be empty, columns needs to be filled still.
+    assert js['columns']
 
     _write_table(arrow_table, filename.strpath, version="2.0",
                  coerce_timestamps='ms')

From 080ce12b247d2722ae57b92c3ac89b0f558d1b0d Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 19 Feb 2018 11:20:48 -0500
Subject: [PATCH 1495/1644] ARROW-2173: [C++/Python] Hold the GIL in
 NumPyBuffer destructor

Author: Antoine Pitrou <antoine@python.org>

Closes #1627 from pitrou/ARROW-2173-numpybuffer-gil and squashes the following commits:

30ec23a0 [Antoine Pitrou] ARROW-2173: [C++/Python] Hold the GIL in NumPyBuffer destructor
---
 cpp/src/arrow/python/numpy_convert.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index c2d055fceed5a..7ba13877d52cd 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -76,7 +76,10 @@ NumPyBuffer::NumPyBuffer(PyObject* ao) : Buffer(nullptr, 0) {
   }
 }
 
-NumPyBuffer::~NumPyBuffer() { Py_XDECREF(arr_); }
+NumPyBuffer::~NumPyBuffer() {
+  PyAcquireGIL lock;
+  Py_XDECREF(arr_);
+}
 
 #define TO_ARROW_TYPE_CASE(NPY_NAME, FACTORY) \
   case NPY_##NPY_NAME:                        \

From 64d2af6053271a8eb317cd8d0e78a3ba204b65d8 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Mon, 19 Feb 2018 12:44:37 -0500
Subject: [PATCH 1496/1644] ARROW-2144: [JS] Don't repeat dictionary lookups in
 DataFrame ops

The optimized `Equals` predicate now caches its reverse dictionary lookup for subsequent `bind` calls. Also adds `DictionaryVector.reverseLookup(value)` and `Vector.indexOf(value)`.

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1599 from TheNeuralBit/cache-dictionary and squashes the following commits:

06be136 [Brian Hulette] throw not implemented error in NestedView.indexOf
e5eaae2 [Brian Hulette] Add basic typedoc script
6bb0876 [Brian Hulette] caching should check for -1
ec990e3 [Brian Hulette] indexOf now returns -1 rather than null...
e2ae901 [Brian Hulette] Rename find -> indexOf
e6f34a1 [Brian Hulette] Add basic Utf8 tests
74de6bf [Brian Hulette] Add unit tests for find in numeric vectors
15947fc [Brian Hulette] Add Vector.find, use in reverseLookup
249d070 [Brian Hulette] cache dictionary reverse lookup in member var
---
 js/package.json              |  5 +-
 js/src/predicate.ts          | 27 ++++++-----
 js/src/vector.ts             |  5 ++
 js/src/vector/chunked.ts     | 10 ++++
 js/src/vector/dictionary.ts  |  8 ++++
 js/src/vector/flat.ts        | 31 ++++++++++++
 js/src/vector/list.ts        |  9 ++++
 js/src/vector/nested.ts      |  3 ++
 js/test/unit/vector-tests.ts | 93 +++++++++++++++++++++++++++++++++---
 9 files changed, 169 insertions(+), 22 deletions(-)

diff --git a/js/package.json b/js/package.json
index d36e872638bc5..1c22fc117fab0 100644
--- a/js/package.json
+++ b/js/package.json
@@ -19,7 +19,7 @@
     "clean:testdata": "gulp clean:testdata",
     "create:testdata": "gulp create:testdata",
     "test:coverage": "gulp test -t ts --coverage",
-    "doc": "shx rm -rf ./doc && esdoc",
+    "doc": "shx rm -rf ./doc && typedoc --mode file --out doc src/Arrow.ts",
     "lint": "run-p lint:*",
     "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
     "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"",
@@ -70,8 +70,6 @@
     "benchmark": "2.1.4",
     "coveralls": "3.0.0",
     "del": "3.0.0",
-    "esdoc": "1.0.4",
-    "esdoc-standard-plugin": "1.0.0",
     "glob": "7.1.2",
     "google-closure-compiler": "20180101.0.0",
     "gulp": "github:gulpjs/gulp#6d71a658c61edb3090221579d8f97dbe086ba2ed",
@@ -97,6 +95,7 @@
     "trash": "4.2.1",
     "ts-jest": "22.0.1",
     "tslint": "5.9.1",
+    "typedoc": "0.10.0",
     "typescript": "2.7.1",
     "uglifyjs-webpack-plugin": "1.1.6",
     "webpack": "3.10.0",
diff --git a/js/src/predicate.ts b/js/src/predicate.ts
index 9d55274bd880b..981ffb1664309 100644
--- a/js/src/predicate.ts
+++ b/js/src/predicate.ts
@@ -125,6 +125,10 @@ export class Or extends CombinationPredicate {
 }
 
 export class Equals extends ComparisonPredicate {
+    // Helpers used to cache dictionary reverse lookups between calls to bind
+    private lastDictionary: Vector|undefined;
+    private lastKey: number|undefined;
+
     protected _bindLitLit(_batch: RecordBatch, left: Literal, right: Literal): PredicateFunc {
         const rtrn: boolean = left.v == right.v;
         return () => rtrn;
@@ -139,20 +143,17 @@ export class Equals extends ComparisonPredicate {
     protected _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc {
         const col_func = col.bind(batch);
         if (col.vector instanceof DictionaryVector) {
-            // Assume that there is only one key with the value `lit.v`
-            // TODO: add lazily-computed reverse dictionary lookups, associated
-            // with col.vector.data so that we only have to do this once per
-            // dictionary
-            let key = -1;
-            let dict = col.vector;
-            let data = dict.dictionary!;
-            for (let len = data.length; ++key < len;) {
-                if (data.get(key) === lit.v) {
-                    break;
-                }
+            let key: any;
+            const vector = col.vector as DictionaryVector;
+            if (vector.dictionary !== this.lastDictionary) {
+                key = vector.reverseLookup(lit.v);
+                this.lastDictionary = vector.dictionary;
+                this.lastKey = key;
+            } else {
+                key = this.lastKey;
             }
 
-            if (key == data.length) {
+            if (key === -1) {
                 // the value doesn't exist in the dictionary - always return
                 // false
                 // TODO: special-case of PredicateFunc that encapsulates this
@@ -161,7 +162,7 @@ export class Equals extends ComparisonPredicate {
                 return () => false;
             } else {
                 return (idx: number) => {
-                    return dict.getKey(idx) === key;
+                    return vector.getKey(idx) === key;
                 };
             }
         } else {
diff --git a/js/src/vector.ts b/js/src/vector.ts
index d9ca97b5fd120..fa1d16efcba72 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -28,6 +28,7 @@ export interface View<T extends DataType> {
     get(index: number): T['TValue'] | null;
     set(index: number, value: T['TValue']): void;
     toArray(): IterableArrayLike<T['TValue'] | null>;
+    indexOf(search: T['TValue']): number;
     [Symbol.iterator](): IterableIterator<T['TValue'] | null>;
 }
 
@@ -77,6 +78,9 @@ export class Vector<T extends DataType = any> implements VectorLike, View<T>, Vi
     public toArray(): IterableArrayLike<T['TValue'] | null> {
         return this.view.toArray();
     }
+    public indexOf(value: T['TValue']) {
+        return this.view.indexOf(value);
+    }
     public [Symbol.iterator](): IterableIterator<T['TValue'] | null> {
         return this.view[Symbol.iterator]();
     }
@@ -414,6 +418,7 @@ export class DictionaryVector<T extends DataType = DataType> extends Vector<Dict
     }
     public getKey(index: number) { return this.indicies.get(index); }
     public getValue(key: number) { return this.dictionary.get(key); }
+    public reverseLookup(value: T) { return this.dictionary.indexOf(value); }
 }
 
 export const createVector = ((VectorLoader: new <T extends DataType>(data: Data<T>) => TypeVisitor) => (
diff --git a/js/src/vector/chunked.ts b/js/src/vector/chunked.ts
index 2eaf99c7cc131..7876bbae5798b 100644
--- a/js/src/vector/chunked.ts
+++ b/js/src/vector/chunked.ts
@@ -103,6 +103,16 @@ export class ChunkedView<T extends DataType> implements View<T> {
         }
         return target;
     }
+    public indexOf(search: T['TValue']) {
+        let offset = 0, result;
+        for (const vector of this.chunkVectors) {
+            result = vector.indexOf(search);
+            if (result !== -1) { return result + offset; }
+            offset += vector.length;
+        }
+
+        return -1;
+    }
 }
 
 function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
diff --git a/js/src/vector/dictionary.ts b/js/src/vector/dictionary.ts
index 385729814efb9..f4de810b015db 100644
--- a/js/src/vector/dictionary.ts
+++ b/js/src/vector/dictionary.ts
@@ -47,4 +47,12 @@ export class DictionaryView<T extends DataType> implements View<T> {
             yield values.get(indicies.get(index));
         }
     }
+    public indexOf(search: T['TValue']) {
+        // First find the dictionary key for the desired value...
+        const key = this.dictionary.indexOf(search);
+        if (key === -1) { return key; }
+
+        // ... then find the first occurence of that key in indicies
+        return this.indicies.indexOf(key!);
+    }
 }
diff --git a/js/src/vector/flat.ts b/js/src/vector/flat.ts
index a32bd9d397ff0..acc2f1af9baa6 100644
--- a/js/src/vector/flat.ts
+++ b/js/src/vector/flat.ts
@@ -43,6 +43,15 @@ export class FlatView<T extends FlatType> implements View<T> {
     public toArray(): IterableArrayLike<T['TValue']> {
         return this.values.subarray(0, this.length);
     }
+    public indexOf(search: T['TValue']) {
+        let index = 0;
+        for (let value of this) {
+            if (value === search) { return index; }
+            ++index;
+        }
+
+        return -1;
+    }
     public [Symbol.iterator](): IterableIterator<T['TValue']> {
         return this.values.subarray(0, this.length)[Symbol.iterator]() as IterableIterator<T['TValue']>;
     }
@@ -64,6 +73,10 @@ export class NullView implements View<Null> {
     public toArray(): IterableArrayLike<null> {
         return [...this];
     }
+    public indexOf(search: any) {
+        // if you're looking for nulls and the view isn't empty, we've got 'em!
+        return search === null && this.length > 0 ? 0 : -1;
+    }
     public *[Symbol.iterator](): IterableIterator<null> {
         for (let index = -1, length = this.length; ++index < length;) {
             yield null;
@@ -107,6 +120,15 @@ export class ValidityView<T extends DataType> implements View<T> {
     public toArray(): IterableArrayLike<T['TValue'] | null> {
         return [...this];
     }
+    public indexOf(search: T['TValue']) {
+        let index = 0;
+        for (let value of this) {
+            if (value === search) { return index; }
+            ++index;
+        }
+
+        return -1;
+    }
     public isValid(index: number): boolean {
         const nullBitIndex = this.offset + index;
         return getBool(null, index, this.nullBitmap[nullBitIndex >> 3], nullBitIndex % 8);
@@ -169,6 +191,15 @@ export class FixedSizeView<T extends PrimitiveType> extends PrimitiveView<T> {
     public toArray(): IterableArrayLike<T['TValue']> {
         return this.values;
     }
+    public indexOf(search: T['TValue']) {
+        let index = 0;
+        for (let value of this) {
+            if (value.every((d: number, i: number) => d === search[i])) { return index; }
+            ++index;
+        }
+
+        return -1;
+    }
     protected getValue(values: T['TArray'], index: number, size: number): T['TValue'] {
         return values.subarray(index * size, index * size + size);
     }
diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts
index 3d365ceac29b2..8561c66ba4e26 100644
--- a/js/src/vector/list.ts
+++ b/js/src/vector/list.ts
@@ -59,6 +59,15 @@ export abstract class ListViewBase<T extends (ListType | FlatListType | FixedSiz
             yield get(values, index, valueOffsets);
         }
     }
+    public indexOf(search: T['TValue']) {
+        let index = 0;
+        for (let value of this) {
+            if (value === search) { return index; }
+            ++index;
+        }
+
+        return -1;
+    }
     protected abstract getList(values: T['TArray'], index: number, valueOffsets?: Int32Array): T['TValue'];
     protected abstract setList(values: T['TArray'], index: number, value: T['TValue'], valueOffsets?: Int32Array): void;
 }
diff --git a/js/src/vector/nested.ts b/js/src/vector/nested.ts
index d0fb24ca99682..a45a912c94e37 100644
--- a/js/src/vector/nested.ts
+++ b/js/src/vector/nested.ts
@@ -40,6 +40,9 @@ export abstract class NestedView<T extends NestedType> implements View<T> {
     public toArray(): IterableArrayLike<T['TValue']> {
         return [...this];
     }
+    public indexOf(_: T['TValue']): number {
+        throw new Error(`Not implemented yet`);
+    }
     public toJSON(): any { return this.toArray(); }
     public toString() {
         return [...this].map((x) => stringify(x)).join(', ');
diff --git a/js/test/unit/vector-tests.ts b/js/test/unit/vector-tests.ts
index 81676b003e399..e2be229834f8e 100644
--- a/js/test/unit/vector-tests.ts
+++ b/js/test/unit/vector-tests.ts
@@ -15,13 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { TextEncoder } from 'text-encoding-utf-8';
 import Arrow from '../Arrow';
 import { type, TypedArray, TypedArrayConstructor } from '../../src/Arrow';
 
-const { BoolData, FlatData } = Arrow.data;
-const { IntVector, FloatVector, BoolVector } = Arrow.vector;
+const utf8Encoder = new TextEncoder('utf-8');
+
+const { BoolData, FlatData, FlatListData } = Arrow.data;
+const { IntVector, FloatVector, BoolVector, Utf8Vector } = Arrow.vector;
 const {
-    Bool,
+    Utf8, Bool,
     Float16, Float32, Float64,
     Int8, Int16, Int32, Int64,
     Uint8, Uint16, Uint32, Uint64,
@@ -45,11 +48,14 @@ const FixedWidthVectors = {
 
 const fixedSizeVectors = toMap(FixedSizeVectors, Object.keys(FixedSizeVectors));
 const fixedWidthVectors = toMap(FixedWidthVectors, Object.keys(FixedWidthVectors));
+const randomBytes = (n: number) => Uint8Array.from(
+    { length: n },
+    () => Math.random() * 255 | 0
+);
 const bytes = Array.from(
     { length: 5 },
-    () => Uint8Array.from(
-        { length: 64 },
-        () => Math.random() * 255 | 0));
+    () => randomBytes(64)
+);
 
 describe(`BoolVector`, () => {
     const values = [true, true, false, true, true, false, false, false], n = values.length;
@@ -67,6 +73,16 @@ describe(`BoolVector`, () => {
             expect(v).toEqual(values[i]);
         }
     });
+    test(`indexOf returns expected values`, () => {
+        for (let test_value of [true, false]) {
+            const expected = values.indexOf(test_value);
+            expect(vector.indexOf(test_value)).toEqual(expected);
+        }
+    });
+    test(`indexOf returns -1 when value not found`, () => {
+        const v = new BoolVector(new BoolData(new Bool(), 3, null, new Uint8Array([0xFF])));
+        expect(v.indexOf(false)).toEqual(-1);
+    });
     test(`can set values to true and false`, () => {
         const v = new BoolVector(new BoolData(new Bool(), n, null, new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0])));
         const expected1 = [true, true, false, true, true, false, false, false];
@@ -145,6 +161,13 @@ describe('Float16Vector', () => {
             expect(v).toEqual(clamp(values[i]));
         }
     });
+    test(`indexOf returns expected values`, () => {
+        const randomValues = new Uint16Array(randomBytes(64).buffer);
+        for (let value of [...values, ...randomValues]) {
+            const expected = values.indexOf(value);
+            expect(vector.indexOf(clamp(value))).toEqual(expected);
+        }
+    });
     test(`slices the entire array`, () => {
         expect(vector.slice().toArray()).toEqual(float16s);
     });
@@ -187,6 +210,21 @@ for (const [VectorName, [VectorType, DataType]] of fixedSizeVectors) {
                 expect(v).toEqual(values.slice(2 * i, 2 * (i + 1)));
             }
         });
+        test(`indexOf returns expected values`, () => {
+            // Create a set of test data composed of all of the actual values
+            // and a few random values
+            let testValues = concatTyped(
+                type.ArrayType,
+                ...bytes,
+                ...[randomBytes(8 * 2 * type.ArrayType.BYTES_PER_ELEMENT)]
+            );
+
+            for (let i = -1, n = testValues.length / 2 | 0; ++i < n;) {
+                const value = testValues.slice(2 * i, 2 * (i + 1));
+                const expected = values.findIndex((d, i) => i % 2 === 0 && d === value[0] && testValues[i + 1] === value[1]);
+                expect(vector.indexOf(value)).toEqual(expected >= 0 ? expected / 2 : -1);
+            }
+        });
         test(`slices the entire array`, () => {
             expect(vector.slice().toArray()).toEqual(values);
         });
@@ -232,6 +270,20 @@ for (const [VectorName, [VectorType, DataType]] of fixedWidthVectors) {
                 expect(v).toEqual(values[i]);
             }
         });
+        test(`indexOf returns expected values`, () => {
+            // Create a set of test data composed of all of the actual values
+            // and a few random values
+            let testValues = concatTyped(
+                type.ArrayType,
+                ...bytes,
+                ...[randomBytes(8 * type.ArrayType.BYTES_PER_ELEMENT)]
+            );
+
+            for (const value of testValues) {
+                const expected = values.indexOf(value);
+                expect(vector.indexOf(value)).toEqual(expected);
+            }
+        });
         test(`slices the entire array`, () => {
             expect(vector.slice().toArray()).toEqual(values);
         });
@@ -253,6 +305,35 @@ for (const [VectorName, [VectorType, DataType]] of fixedWidthVectors) {
     });
 }
 
+describe(`Utf8Vector`, () => {
+    const values = ['foo', 'bar', 'baz', 'foo bar', 'bar'], n = values.length;
+    let offset = 0;
+    const offsets = Uint32Array.of(0, ...values.map((d) => { offset += d.length; return offset; }));
+    const vector = new Utf8Vector(new FlatListData(new Utf8(), n, null, offsets, utf8Encoder.encode(values.join(''))));
+    test(`gets expected values`, () => {
+        let i = -1;
+        while (++i < n) {
+            expect(vector.get(i)).toEqual(values[i]);
+        }
+    });
+    test(`iterates expected values`, () => {
+        expect.hasAssertions();
+        let i = -1;
+        for (let v of vector) {
+            expect(++i).toBeLessThan(n);
+            expect(v).toEqual(values[i]);
+        }
+    });
+    test(`indexOf returns expected values`, () => {
+        let testValues = values.concat(['abc', '12345']);
+
+        for (const value of testValues) {
+            const expected = values.indexOf(value);
+            expect(vector.indexOf(value)).toEqual(expected);
+        }
+    });
+});
+
 function toMap<T>(entries: Record<string, T>, keys: string[]) {
     return keys.reduce((map, key) => {
         map.set(key, entries[key] as T);

From f8445a3a4398187c2ea7dc07bb5ee1cca4b89a35 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 19 Feb 2018 12:52:46 -0500
Subject: [PATCH 1497/1644] ARROW-2171: [C++/Python] Make OwnedRef safer

Disable copy constructor and assignment operator.  Define a move constructor for proper ownership transfer.  Also inherit the constructors in OwnedRefNoGIL.

Author: Antoine Pitrou <antoine@python.org>

Closes #1626 from pitrou/ARROW-2171-copy-constructor and squashes the following commits:

6591b55a [Antoine Pitrou] ARROW-2171: [C++/Python] Make OwnedRef safer
---
 cpp/src/arrow/python/common.h       |  8 ++++++-
 cpp/src/arrow/python/python-test.cc | 34 +++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index 269385c1a63b2..b2844b18c913d 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -67,7 +67,7 @@ class ARROW_EXPORT PyAcquireGIL {
 class ARROW_EXPORT OwnedRef {
  public:
   OwnedRef() : obj_(NULLPTR) {}
-
+  OwnedRef(OwnedRef&& other) : OwnedRef(other.detach()) {}
   explicit OwnedRef(PyObject* obj) : obj_(obj) {}
 
   ~OwnedRef() { reset(); }
@@ -90,6 +90,8 @@ class ARROW_EXPORT OwnedRef {
   PyObject** ref() { return &obj_; }
 
  private:
+  ARROW_DISALLOW_COPY_AND_ASSIGN(OwnedRef);
+
   PyObject* obj_;
 };
 
@@ -98,6 +100,10 @@ class ARROW_EXPORT OwnedRef {
 // (e.g. if it is released in the middle of a function for performance reasons)
 class ARROW_EXPORT OwnedRefNoGIL : public OwnedRef {
  public:
+  OwnedRefNoGIL() : OwnedRef() {}
+  OwnedRefNoGIL(OwnedRefNoGIL&& other) : OwnedRef(other.detach()) {}
+  explicit OwnedRefNoGIL(PyObject* obj) : OwnedRef(obj) {}
+
   ~OwnedRefNoGIL() {
     PyAcquireGIL lock;
     reset();
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index bcf89a4f6e8bb..a2b832bdba74a 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -42,6 +42,40 @@ TEST(PyBuffer, InvalidInputObject) {
   ASSERT_EQ(old_refcnt, Py_REFCNT(input));
 }
 
+TEST(OwnedRef, TestMoves) {
+  PyAcquireGIL lock;
+  std::vector<OwnedRef> vec;
+  PyObject *u, *v;
+  u = PyList_New(0);
+  v = PyList_New(0);
+  {
+    OwnedRef ref(u);
+    vec.push_back(std::move(ref));
+    ASSERT_EQ(ref.obj(), nullptr);
+  }
+  vec.emplace_back(v);
+  ASSERT_EQ(Py_REFCNT(u), 1);
+  ASSERT_EQ(Py_REFCNT(v), 1);
+}
+
+TEST(OwnedRefNoGIL, TestMoves) {
+  std::vector<OwnedRefNoGIL> vec;
+  PyObject *u, *v;
+  {
+    PyAcquireGIL lock;
+    u = PyList_New(0);
+    v = PyList_New(0);
+  }
+  {
+    OwnedRefNoGIL ref(u);
+    vec.push_back(std::move(ref));
+    ASSERT_EQ(ref.obj(), nullptr);
+  }
+  vec.emplace_back(v);
+  ASSERT_EQ(Py_REFCNT(u), 1);
+  ASSERT_EQ(Py_REFCNT(v), 1);
+}
+
 class DecimalTest : public ::testing::Test {
  public:
   DecimalTest() : lock_(), decimal_module_(), decimal_constructor_() {

From d5c507ad231181bd1ee82970597dffc1ae639a4f Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 19 Feb 2018 12:55:33 -0500
Subject: [PATCH 1498/1644] =?UTF-8?q?ARROW-2143:=20[Python]=C2=A0Provide?=
 =?UTF-8?q?=20a=20manylinux1=20wheel=20for=20cp27m?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1603 from xhochy/ARROW-2143 and squashes the following commits:

730c2f88 [Uwe L. Korn] ARROW-2143: [Python] Provide a manylinux1 wheel for cp27m [skip appveyor]
---
 python/manylinux1/Dockerfile-x86_64           |  2 +-
 python/manylinux1/build_arrow.sh              | 38 ++++++++++---------
 .../manylinux1/scripts/build_virtualenvs.sh   | 38 +++++++++----------
 3 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 919a32be715b0..ec520338ff2fe 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2143
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 6bed451d2d667..5fd27c8d03a3f 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -25,10 +25,8 @@
 # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
 # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
 
-PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5 3.6}"
-
-# Package index with only manylinux1 builds
-MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
+# Build different python versions with various unicode widths
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.4,16 3.5,16 3.6,16}"
 
 source /multibuild/manylinux_utils.sh
 
@@ -48,40 +46,44 @@ export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr'
 # Ensure the target directory exists
 mkdir -p /io/dist
 
-for PYTHON in ${PYTHON_VERSIONS}; do
-    PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
-    PIP="$(cpython_path $PYTHON)/bin/pip"
-    PIPI_IO="$PIP install -f $MANYLINUX_URL"
-    PATH="$PATH:$(cpython_path $PYTHON)"
+for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
+    IFS=","
+    set -- $PYTHON_TUPLE;
+    PYTHON=$1
+    U_WIDTH=$2
+    CPYTHON_PATH="$(cpython_path $PYTHON ${U_WIDTH})"
+    PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python"
+    PIP="${CPYTHON_PATH}/bin/pip"
+    PATH="$PATH:${CPYTHON_PATH}"
 
     echo "=== (${PYTHON}) Building Arrow C++ libraries ==="
-    ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
+    ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}-${U_WIDTH}
     mkdir -p "${ARROW_BUILD_DIR}"
     pushd "${ARROW_BUILD_DIR}"
-    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON -DARROW_ORC=ON ..
+    PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON -DARROW_ORC=ON ..
     make -j5 install
     popd
 
     # Clear output directory
     rm -rf dist/
     echo "=== (${PYTHON}) Building wheel ==="
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-static-parquet --bundle-arrow-cpp
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
+    PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-static-parquet --bundle-arrow-cpp
+    PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
     echo "=== (${PYTHON}) Test the existence of optional modules ==="
-    $PIPI_IO -r requirements.txt
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
-    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER -c "import pyarrow.plasma"
+    $PIP install -r requirements.txt
+    PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
+    PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.plasma"
 
     echo "=== (${PYTHON}) Tag the wheel with manylinux1 ==="
     mkdir -p repaired_wheels/
     auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/
 
     echo "=== (${PYTHON}) Testing manylinux1 wheel ==="
-    source /venv-test-${PYTHON}/bin/activate
+    source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate
     pip install repaired_wheels/*.whl
 
-    py.test -v -r sxX --durations=15 --parquet /venv-test-${PYTHON}/lib/*/site-packages/pyarrow
+    py.test -v -r sxX --durations=15 --parquet /venv-test-${PYTHON}-${U_WIDTH}/lib/*/site-packages/pyarrow
     deactivate
 
     mv repaired_wheels/*.whl /io/dist
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
index b548611261cd1..220c26003e9f9 100755
--- a/python/manylinux1/scripts/build_virtualenvs.sh
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -19,28 +19,28 @@
 # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
 # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
 
-PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5 3.6}"
-
-# Package index with only manylinux1 builds
-MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.4,16 3.5,16 3.6,16}"
 
 source /multibuild/manylinux_utils.sh
 
-for PYTHON in ${PYTHON_VERSIONS}; do
-    PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
-    PIP="$(cpython_path $PYTHON)/bin/pip"
-    PIPI_IO="$PIP install -f $MANYLINUX_URL"
-    PATH="$PATH:$(cpython_path $PYTHON)"
-
-    echo "=== (${PYTHON}) Installing build dependencies ==="
-    $PIPI_IO "numpy==1.10.4"
-    $PIPI_IO "cython==0.25.2"
-    $PIPI_IO "pandas==0.20.3"
-    $PIPI_IO "virtualenv==15.1.0"
-
-    echo "=== (${PYTHON}) Preparing virtualenv for tests ==="
-    "$(cpython_path $PYTHON)/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}
-    source /venv-test-${PYTHON}/bin/activate
+for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
+    IFS=","
+    set -- $PYTHON_TUPLE;
+    PYTHON=$1
+    U_WIDTH=$2
+    PYTHON_INTERPRETER="$(cpython_path $PYTHON ${U_WIDTH})/bin/python"
+    PIP="$(cpython_path $PYTHON ${U_WIDTH})/bin/pip"
+    PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})"
+
+    echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ==="
+    $PIP install "numpy==1.10.4"
+    $PIP install "cython==0.25.2"
+    $PIP install "pandas==0.20.3"
+    $PIP install "virtualenv==15.1.0"
+
+    echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ==="
+    "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH}
+    source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate
     pip install pytest 'numpy==1.14.0' 'pandas==0.20.3'
     deactivate
 done

From 3da9fd3af3775ea83d3efa644eab871fd19c8ed0 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 19 Feb 2018 13:38:40 -0500
Subject: [PATCH 1499/1644] ARROW-2178: [JS] Fix JS html FileReader example

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1614 from trxcllnt/js-fix-html-example and squashes the following commits:

b2a05965 [Paul Taylor] fix JS html FileReader example
---
 js/examples/read_file.html | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/js/examples/read_file.html b/js/examples/read_file.html
index 3093622fcc1d6..3e082d9dc412f 100644
--- a/js/examples/read_file.html
+++ b/js/examples/read_file.html
@@ -29,6 +29,7 @@
 }
 table, th, td {
   border: 1px solid black;
+  white-space: nowrap;
 }
     </style>
     <script type="text/javascript">
@@ -53,8 +54,8 @@
   }
 
   var header_row = document.createElement("tr");
-  for (let column of arrowTable.columns) {
-    addCell(header_row, "th", column.name);
+  for (let field of arrowTable.schema.fields) {
+    addCell(header_row, "th", `${field}`);
   }
 
   thead.appendChild(header_row);

From 8e90836af0090c2b4f4baacb6854cdf78042d6c8 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 19 Feb 2018 13:39:48 -0500
Subject: [PATCH 1500/1644] ARROW-2168: [C++] Build toolchain on CI with
 jemalloc

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1621 from xhochy/ARROW-2168 and squashes the following commits:

30a9b979 [Uwe L. Korn] [ARROW-2168] [C++] Build toolchain on CI with jemalloc [skip appveyor]
---
 ci/travis_before_script_cpp.sh | 17 +++++++++--------
 ci/travis_install_toolchain.sh |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 4ffe97f675c5f..17b5deb36b519 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -29,14 +29,6 @@ else
   source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 fi
 
-if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
-  # Set up C++ toolchain from conda-forge packages for faster builds
-  source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
-fi
-
-mkdir -p $ARROW_CPP_BUILD_DIR
-pushd $ARROW_CPP_BUILD_DIR
-
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
 -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \
@@ -45,6 +37,15 @@ CMAKE_COMMON_FLAGS="\
 CMAKE_LINUX_FLAGS=""
 CMAKE_OSX_FLAGS=""
 
+if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
+  # Set up C++ toolchain from conda-forge packages for faster builds
+  source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
+  CMAKE_COMMON_FLAGS="${CMAKE_COMMON_FLAGS} -DARROW_JEMALLOC=ON"
+fi
+
+mkdir -p $ARROW_CPP_BUILD_DIR
+pushd $ARROW_CPP_BUILD_DIR
+
 if [ $only_library_mode == "yes" ]; then
   CMAKE_COMMON_FLAGS="\
 $CMAKE_COMMON_FLAGS \
diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh
index e01a084daac86..60cdc36a2efff 100755
--- a/ci/travis_install_toolchain.sh
+++ b/ci/travis_install_toolchain.sh
@@ -24,7 +24,7 @@ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 if [ ! -e $CPP_TOOLCHAIN ]; then
     # Set up C++ toolchain from conda-forge packages for faster builds
     conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
-        jemalloc=4.4.0 \
+        jemalloc=4.5.0.post \
         nomkl \
         boost-cpp \
         rapidjson \

From 77f2841f635403f3baf12e642fdea598542ce9aa Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Mon, 19 Feb 2018 14:20:16 -0500
Subject: [PATCH 1501/1644] ARROW-2159: [JS] Support custom predicates

Adds support for defining a custom predicate with callback functions, like so:
```js
table.filter(predicate.custom(
    (idx) => {...},
    (batch) => {...}
));
```

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1616 from TheNeuralBit/custom-predicate and squashes the following commits:

77f5441 [Brian Hulette] Add test, squash bug
29ecc80 [Brian Hulette] Add CustomPredicate
---
 js/src/Arrow.externs.js     |  3 ++-
 js/src/Arrow.ts             |  1 +
 js/src/predicate.ts         | 14 ++++++++++++++
 js/test/unit/table-tests.ts | 14 ++++++++++++--
 4 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index 21dca8be890b6..de1e6539285bc 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -70,6 +70,7 @@ CountByResult.prototype.asJSON;
 
 var col = function () {};
 var lit = function () {};
+var custom = function () {};
 
 var Value = function() {};
 /** @type {?} */
@@ -738,4 +739,4 @@ VectorVisitor.prototype.visitInterval;
 /** @type {?} */
 VectorVisitor.prototype.visitFixedSizeList;
 /** @type {?} */
-VectorVisitor.prototype.visitMap;
\ No newline at end of file
+VectorVisitor.prototype.visitMap;
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index df37a8fb01324..4a0a2ac6de10a 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -168,6 +168,7 @@ export namespace view {
 export namespace predicate {
     export import col = predicate_.col;
     export import lit = predicate_.lit;
+    export import custom = predicate_.custom;
 
     export import Or = predicate_.Or;
     export import Col = predicate_.Col;
diff --git a/js/src/predicate.ts b/js/src/predicate.ts
index 981ffb1664309..b177b4fa76459 100644
--- a/js/src/predicate.ts
+++ b/js/src/predicate.ts
@@ -222,5 +222,19 @@ export class GTeq extends ComparisonPredicate {
     }
 }
 
+export class CustomPredicate extends Predicate {
+    constructor(private next: PredicateFunc, private bind_: (batch: RecordBatch) => void) {
+        super();
+    }
+
+    bind(batch: RecordBatch) {
+        this.bind_(batch);
+        return this.next;
+    }
+}
+
 export function lit(v: any): Value<any> { return new Literal(v); }
 export function col(n: string): Col<any> { return new Col(n); }
+export function custom(next: PredicateFunc, bind: (batch: RecordBatch) => void) {
+    return new CustomPredicate(next, bind);
+}
diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts
index ffcc8f477a29f..8a433815dab50 100644
--- a/js/test/unit/table-tests.ts
+++ b/js/test/unit/table-tests.ts
@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import Arrow from '../Arrow';
+import Arrow, { RecordBatch } from '../Arrow';
 
 const { predicate, Table } = Arrow;
 
-const { col, lit } = predicate;
+const { col, lit, custom } = predicate;
 
 const F32 = 0, I32 = 1, DICT = 2;
 const test_data = [
@@ -323,6 +323,7 @@ describe(`Table`, () => {
                 expect(table.getColumnIndex('f32')).toEqual(F32);
                 expect(table.getColumnIndex('dictionary')).toEqual(DICT);
             });
+            let get_i32: (idx: number) => number, get_f32: (idx: number) => number;
             const filter_tests = [
                 {
                     name:     `filter on f32 >= 0`,
@@ -364,6 +365,15 @@ describe(`Table`, () => {
                     name:     `filter on f32 <= i32`,
                     filtered: table.filter(col('f32').lteq(col('i32'))),
                     expected: values.filter((row) => row[F32] <= row[I32])
+                }, {
+                    name:     `filter on f32*i32 > 0 (custom predicate)`,
+                    filtered: table.filter(custom(
+                        (idx: number) => (get_f32(idx) * get_i32(idx) > 0),
+                        (batch: RecordBatch) => {
+                            get_f32 = col('f32').bind(batch);
+                            get_i32 = col('i32').bind(batch);
+                        })),
+                    expected: values.filter((row) => (row[F32] as number) * (row[I32] as number) > 0)
                 }
             ];
             for (let this_test of filter_tests) {

From 7d992de1de7dd276eb9aeda349376e79b62da11c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 19 Feb 2018 15:36:56 -0500
Subject: [PATCH 1502/1644] [Release] Apache Arrow JavaScript 0.3.0

Change-Id: Ic2b2ad552a9de4e317c130126106a817ea11567c
---
 js/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/package.json b/js/package.json
index 1c22fc117fab0..507babfbbb5b1 100644
--- a/js/package.json
+++ b/js/package.json
@@ -1,5 +1,5 @@
 {
-  "version": "0.2.0",
+  "version": "0.3.0",
   "name": "apache-arrow",
   "description": "Apache Arrow columnar in-memory format",
   "bin": {

From 46aa99e9843ac0148357bb36a9235cfd48903e73 Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Mon, 19 Feb 2018 20:55:20 -0500
Subject: [PATCH 1503/1644] ARROW-2121: [Python] Handle object arrays directly
 in pandas serializer.

The goal here is to get the best of both the `pandas_serialization_context` (speed at serializing pandas dataframes containing strings and other objects) and the `default_serialization_context` (correctly serializing a large class of numpy object arrays).

This PR sort of messes up the function `pa.pandas_compat.dataframe_to_serialized_dict`. Is that function just a helper function for implementing the custom pandas serializers? Or is it intended to be used in other places.

TODO in this PR (assuming you think this approach is reasonable):

- [x] remove `pandas_serialization_context`
- [x] make sure this code path is tested
- [x] double check performance/behavior

cc @wesm @pcmoritz @devin-petersohn

Author: Robert Nishihara <robertnishihara@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1581 from robertnishihara/pandasserialization and squashes the following commits:

c551ed6c [Wes McKinney] Add benchmark for zero-copy pandas deserialization
e12cd721 [Robert Nishihara] Use highest protocol.
6e3eceb6 [Robert Nishihara] Fix bug.
31b3cbee [Robert Nishihara] Fixes.
0fc9ecfd [Robert Nishihara] Update documentation and testing.
b796fbdb [Robert Nishihara] Handle object arrays directly in pandas serializer.
---
 python/README-benchmarks.md                 |  2 --
 python/benchmarks/convert_pandas.py         | 20 ++++++++++++++++++++
 python/doc/source/ipc.rst                   |  7 +++----
 python/pyarrow/__init__.py                  |  1 -
 python/pyarrow/pandas_compat.py             | 15 +++++++++++++--
 python/pyarrow/serialization.py             | 12 ------------
 python/pyarrow/tests/test_convert_pandas.py |  2 +-
 python/pyarrow/tests/test_serialization.py  |  2 --
 8 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/python/README-benchmarks.md b/python/README-benchmarks.md
index 3fecb35cb8095..60fa88f4ac3f1 100644
--- a/python/README-benchmarks.md
+++ b/python/README-benchmarks.md
@@ -41,8 +41,6 @@ First you have to install ASV's development version:
 pip install git+https://github.com/airspeed-velocity/asv.git
 ```
 
-<!--- TODO remove the above once https://github.com/airspeed-velocity/asv/pull/611 is merged -->
-
 Then you need to set up a few environment variables:
 
 ```shell
diff --git a/python/benchmarks/convert_pandas.py b/python/benchmarks/convert_pandas.py
index c4a7a59cb77dc..244b3dcc84713 100644
--- a/python/benchmarks/convert_pandas.py
+++ b/python/benchmarks/convert_pandas.py
@@ -48,3 +48,23 @@ def setup(self, n, dtype):
 
     def time_to_series(self, n, dtype):
         self.arrow_data.to_pandas()
+
+
+class ZeroCopyPandasRead(object):
+
+    def setup(self):
+        # Transpose to make column-major
+        values = np.random.randn(10, 100000)
+
+        df = pd.DataFrame(values.T)
+        ctx = pa.default_serialization_context()
+
+        self.serialized = ctx.serialize(df)
+        self.as_buffer = self.serialized.to_buffer()
+        self.as_components = self.serialized.to_components()
+
+    def time_deserialize_from_buffer(self):
+        pa.deserialize(self.as_buffer)
+
+    def time_deserialize_from_components(self):
+        pa.deserialize_components(self.as_components)
diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index 9bf93ffe8ae8e..bce8b1ed1e951 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -317,9 +317,8 @@ An object can be reconstructed from its component-based representation using
 Serializing pandas Objects
 --------------------------
 
-We provide a serialization context that has optimized handling of pandas
-objects like ``DataFrame`` and ``Series``. This can be created with
-``pyarrow.pandas_serialization_context()``. Combined with component-based
+The default serialization context has optimized handling of pandas
+objects like ``DataFrame`` and ``Series``. Combined with component-based
 serialization above, this enables zero-copy transport of pandas DataFrame
 objects not containing any Python objects:
 
@@ -327,7 +326,7 @@ objects not containing any Python objects:
 
    import pandas as pd
    df = pd.DataFrame({'a': [1, 2, 3, 4, 5]})
-   context = pa.pandas_serialization_context()
+   context = pa.default_serialization_context()
    serialized_df = context.serialize(df)
    df_components = serialized_df.to_components()
    original_df = context.deserialize_components(df_components)
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index d95954ed30e42..15a37ca10d276 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -125,7 +125,6 @@
 localfs = LocalFileSystem.get_instance()
 
 from pyarrow.serialization import (default_serialization_context,
-                                   pandas_serialization_context,
                                    register_default_serialization_handlers,
                                    register_torch_serialization_handlers)
 
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index e8fa83fe71383..6d4bf5e787079 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -27,7 +27,7 @@
 import six
 
 import pyarrow as pa
-from pyarrow.compat import PY2, zip_longest  # noqa
+from pyarrow.compat import builtin_pickle, PY2, zip_longest  # noqa
 
 
 def infer_dtype(column):
@@ -424,11 +424,19 @@ def dataframe_to_serialized_dict(frame):
             block_data.update(dictionary=values.categories,
                               ordered=values.ordered)
             values = values.codes
-
         block_data.update(
             placement=block.mgr_locs.as_array,
             block=values
         )
+
+        # If we are dealing with an object array, pickle it instead. Note that
+        # we do not use isinstance here because _int.CategoricalBlock is a
+        # subclass of _int.ObjectBlock.
+        if type(block) == _int.ObjectBlock:
+            block_data['object'] = None
+            block_data['block'] = builtin_pickle.dumps(
+                values, protocol=builtin_pickle.HIGHEST_PROTOCOL)
+
         blocks.append(block_data)
 
     return {
@@ -463,6 +471,9 @@ def _reconstruct_block(item):
         block = _int.make_block(block_arr, placement=placement,
                                 klass=_int.DatetimeTZBlock,
                                 dtype=dtype)
+    elif 'object' in item:
+        block = _int.make_block(builtin_pickle.loads(block_arr),
+                                placement=placement, klass=_int.ObjectBlock)
     else:
         block = _int.make_block(block_arr, placement=placement)
 
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index c8b72b74896c9..bdf7535796833 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -57,10 +57,6 @@ def _load_pickle_from_buffer(data):
         return builtin_pickle.loads(as_memoryview)
 
 
-_serialize_numpy_array_pickle = _pickle_to_buffer
-_deserialize_numpy_array_pickle = _load_pickle_from_buffer
-
-
 # ----------------------------------------------------------------------
 # pandas-specific serialization matters
 
@@ -190,11 +186,3 @@ def default_serialization_context():
 
 
 register_default_serialization_handlers(_default_serialization_context)
-
-
-def pandas_serialization_context():
-    context = default_serialization_context()
-    context.register_type(np.ndarray, 'np.array',
-                          custom_serializer=_serialize_numpy_array_pickle,
-                          custom_deserializer=_deserialize_numpy_array_pickle)
-    return context
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 95137ffb26e28..f7718f06a410c 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1613,7 +1613,7 @@ def _fully_loaded_dataframe_example():
 
 
 def _check_serialize_components_roundtrip(df):
-    ctx = pa.pandas_serialization_context()
+    ctx = pa.default_serialization_context()
 
     components = ctx.serialize(df).to_components()
     deserialized = ctx.deserialize_components(components)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 20c195a4b8c75..0917172d22c1f 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -284,8 +284,6 @@ def custom_deserializer(serialized_obj):
 def test_primitive_serialization(large_buffer):
     for obj in PRIMITIVE_OBJECTS:
         serialization_roundtrip(obj, large_buffer)
-        serialization_roundtrip(obj, large_buffer,
-                                pa.pandas_serialization_context())
 
 
 def test_serialize_to_buffer():

From ac2637a84445c7e14619def47f6288988a2f48a8 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 19 Feb 2018 21:12:13 -0500
Subject: [PATCH 1504/1644] ARROW-2175: [Python] Install Arrow libraries in
 Travis CI builds when only Python directory is affected

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1630 from wesm/ARROW-2175 and squashes the following commits:

b6bdd3d2 [Wes McKinney] ARROW_HOME needed to override PARQUET_BUILD_TOOLCHAIN
119b290b [Wes McKinney] Test the behavior of a Python-only change
49e8706f [Wes McKinney] Just use PARQUET_BUILD_TOOLCHAIN [force travis] [skip appveyor]
cdd9929f [Wes McKinney] Always build Arrow libraries when either C++/Python affected. Add missing ARROW_HOME env var to Parquet build CI script. [force travis] [skip appveyor]
---
 .travis.yml                    | 6 ++++--
 ci/travis_build_parquet_cpp.sh | 3 +--
 python/README.md               | 6 +++---
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 73a9f46427081..a4c74657e5257 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -61,7 +61,8 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
-    - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
+    # If either C++ or Python changed, we must install the C++ libraries
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
     - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
@@ -81,7 +82,8 @@ matrix:
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     before_script:
     - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
-    - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
+    # If either C++ or Python changed, we must install the C++ libraries
+    - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
     - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
diff --git a/ci/travis_build_parquet_cpp.sh b/ci/travis_build_parquet_cpp.sh
index fc4ae72c15847..7d2e3ab7364fa 100755
--- a/ci/travis_build_parquet_cpp.sh
+++ b/ci/travis_build_parquet_cpp.sh
@@ -22,9 +22,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
 
-export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
-
 export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+export ARROW_HOME=$ARROW_CPP_INSTALL
 
 PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
 mkdir -p $PARQUET_DIR
diff --git a/python/README.md b/python/README.md
index 38c994013755a..e2ed9db6f50f8 100644
--- a/python/README.md
+++ b/python/README.md
@@ -19,9 +19,9 @@
 
 ## Python library for Apache Arrow
 
-This library provides a Pythonic API wrapper for the reference Arrow C++
-implementation, along with tools for interoperability with pandas, NumPy, and
-other traditional Python scientific computing packages.
+This library provides a Python API for functionality provided by the Arrow C++
+libraries, along with tools for Arrow integration and interoperability with
+pandas, NumPy, and other software in the Python ecosystem.
 
 ## Installing
 

From e06cfdd0037adba3fb5f1c775f01f465933cc71d Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 20 Feb 2018 12:38:37 -0500
Subject: [PATCH 1505/1644] ARROW-2179: [C++] Install omitted headers in
 arrow/util

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1631 from wesm/ARROW-2179 and squashes the following commits:

e416e082 [Wes McKinney] Use GLOB_RECURSE to collect headers
f20e8aac [Wes McKinney] Install omitted C++ headers in arrow/util
---
 cpp/src/arrow/util/CMakeLists.txt | 27 +++------------------------
 1 file changed, 3 insertions(+), 24 deletions(-)

diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 8b61a3acfe709..41c27a58f928d 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -19,32 +19,11 @@
 # arrow_util
 #######################################
 
+file(GLOB_RECURSE ARROW_UTIL_HEADERS "*.h")
+
 # Headers: top level
 install(FILES
-  bit-stream-utils.h
-  bit-util.h
-  bpacking.h
-  compiler-util.h
-  compression_brotli.h
-  compression.h
-  compression_lz4.h
-  compression_snappy.h
-  compression_zlib.h
-  compression_zstd.h
-  cpu-info.h
-  decimal.h
-  hash-util.h
-  hash.h
-  key_value_metadata.h
-  logging.h
-  macros.h
-  parallel.h
-  rle-encoding.h
-  sse-util.h
-  stl.h
-  type_traits.h
-  variant.h
-  visibility.h
+  ${ARROW_UTIL_HEADERS}
   DESTINATION include/arrow/util)
 
 #######################################

From f35790653c7213f9445f8525f75e42092ef46389 Mon Sep 17 00:00:00 2001
From: Panchen Xue <pan.panchen.xue@gmail.com>
Date: Tue, 20 Feb 2018 12:40:42 -0500
Subject: [PATCH 1506/1644] ARROW-1942: [C++] Hash table specializations for
 small integers

Add hash table specializations for int8_t and uint8_t

Author: Panchen Xue <pan.panchen.xue@gmail.com>

Closes #1551 from xuepanchen/ARROW-1942 and squashes the following commits:

37c42161 [Panchen Xue] Hash table specializations for small integers
---
 cpp/src/arrow/compute/compute-benchmark.cc    | 22 +++++-
 cpp/src/arrow/compute/kernels/hash.cc         | 79 ++++++++++++++++++-
 cpp/src/arrow/compute/kernels/util-internal.h | 22 ++++--
 3 files changed, 116 insertions(+), 7 deletions(-)

diff --git a/cpp/src/arrow/compute/compute-benchmark.cc b/cpp/src/arrow/compute/compute-benchmark.cc
index 44df441394ad4..646010550f47d 100644
--- a/cpp/src/arrow/compute/compute-benchmark.cc
+++ b/cpp/src/arrow/compute/compute-benchmark.cc
@@ -94,7 +94,7 @@ struct HashParams {
     std::vector<bool> is_valid;
     test::randint<int64_t>(length, 0, num_unique, &draws);
     for (int64_t draw : draws) {
-      values.push_back(draw);
+      values.push_back(static_cast<T>(draw));
     }
 
     if (this->null_percent > 0) {
@@ -170,6 +170,14 @@ void BenchDictionaryEncode(benchmark::State& state, const ParamType& params,
   state.SetBytesProcessed(state.iterations() * params.GetBytesProcessed(length));
 }
 
+static void BM_UniqueUInt8NoNulls(benchmark::State& state) {
+  BenchUnique(state, HashParams<UInt8Type>{0}, state.range(0), state.range(1));
+}
+
+static void BM_UniqueUInt8WithNulls(benchmark::State& state) {
+  BenchUnique(state, HashParams<UInt8Type>{0.05}, state.range(0), state.range(1));
+}
+
 static void BM_UniqueInt64NoNulls(benchmark::State& state) {
   BenchUnique(state, HashParams<Int64Type>{0}, state.range(0), state.range(1));
 }
@@ -207,5 +215,17 @@ ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64WithNulls));
 ADD_HASH_ARGS(BENCHMARK(BM_UniqueString10bytes));
 ADD_HASH_ARGS(BENCHMARK(BM_UniqueString100bytes));
 
+BENCHMARK(BM_UniqueUInt8NoNulls)
+    ->Args({kHashBenchmarkLength, 200})
+    ->MinTime(1.0)
+    ->Unit(benchmark::kMicrosecond)
+    ->UseRealTime();
+
+BENCHMARK(BM_UniqueUInt8WithNulls)
+    ->Args({kHashBenchmarkLength, 200})
+    ->MinTime(1.0)
+    ->Unit(benchmark::kMicrosecond)
+    ->UseRealTime();
+
 }  // namespace compute
 }  // namespace arrow
diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc
index da9797f77bbaf..dbce6e561c52e 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -221,7 +221,10 @@ struct HashDictionary<Type, enable_if_has_c_type<Type>> {
   }
 
 template <typename Type, typename Action>
-class HashTableKernel<Type, Action, enable_if_has_c_type<Type>> : public HashTable {
+class HashTableKernel<
+    Type, Action,
+    typename std::enable_if<has_c_type<Type>::value && !is_8bit_int<Type>::value>::type>
+    : public HashTable {
  public:
   using T = typename Type::c_type;
 
@@ -611,6 +614,80 @@ class HashTableKernel<Type, Action, enable_if_fixed_size_binary<Type>>
   int32_t dict_size_;
 };
 
+// ----------------------------------------------------------------------
+// Hash table pass for uint8 and int8
+
+template <typename T>
+inline int Hash8Bit(const T val) {
+  return 0;
+}
+
+template <>
+inline int Hash8Bit(const uint8_t val) {
+  return val;
+}
+
+template <>
+inline int Hash8Bit(const int8_t val) {
+  return val + 128;
+}
+
+template <typename Type, typename Action>
+class HashTableKernel<Type, Action, enable_if_8bit_int<Type>> : public HashTable {
+ public:
+  using T = typename Type::c_type;
+
+  HashTableKernel(const std::shared_ptr<DataType>& type, MemoryPool* pool)
+      : HashTable(type, pool) {
+    std::fill(table_, table_ + 256, kHashSlotEmpty);
+  }
+
+  Status Append(const ArrayData& arr) override {
+    const T* values = GetValues<T>(arr, 1);
+    auto action = static_cast<Action*>(this);
+    RETURN_NOT_OK(action->Reserve(arr.length));
+
+#define HASH_INNER_LOOP()                                      \
+  const T value = values[i];                                   \
+  const int hash = Hash8Bit<T>(value);                         \
+  hash_slot_t slot = table_[hash];                             \
+                                                               \
+  if (slot == kHashSlotEmpty) {                                \
+    if (!Action::allow_expand) {                               \
+      throw HashException("Encountered new dictionary value"); \
+    }                                                          \
+                                                               \
+    slot = static_cast<hash_slot_t>(dict_.size());             \
+    table_[hash] = slot;                                       \
+    dict_.push_back(value);                                    \
+    action->ObserveNotFound(slot);                             \
+  } else {                                                     \
+    action->ObserveFound(slot);                                \
+  }
+
+    GENERIC_HASH_PASS(HASH_INNER_LOOP);
+
+#undef HASH_INNER_LOOP
+
+    return Status::OK();
+  }
+
+  Status GetDictionary(std::shared_ptr<ArrayData>* out) override {
+    using BuilderType = typename TypeTraits<Type>::BuilderType;
+    BuilderType builder(pool_);
+
+    for (const T value : dict_) {
+      RETURN_NOT_OK(builder.Append(value));
+    }
+
+    return builder.FinishInternal(out);
+  }
+
+ private:
+  hash_slot_t table_[256];
+  std::vector<T> dict_;
+};
+
 // ----------------------------------------------------------------------
 // Unique implementation
 
diff --git a/cpp/src/arrow/compute/kernels/util-internal.h b/cpp/src/arrow/compute/kernels/util-internal.h
index 2f611320a7687..bde267629f30e 100644
--- a/cpp/src/arrow/compute/kernels/util-internal.h
+++ b/cpp/src/arrow/compute/kernels/util-internal.h
@@ -32,6 +32,22 @@ class FunctionContext;
 template <typename T>
 using is_number = std::is_base_of<Number, T>;
 
+template <typename T>
+struct has_c_type {
+  static constexpr bool value =
+      (std::is_base_of<PrimitiveCType, T>::value || std::is_base_of<DateType, T>::value ||
+       std::is_base_of<TimeType, T>::value || std::is_base_of<TimestampType, T>::value);
+};
+
+template <typename T>
+struct is_8bit_int {
+  static constexpr bool value =
+      (std::is_same<UInt8Type, T>::value || std::is_same<Int8Type, T>::value);
+};
+
+template <typename T>
+using enable_if_8bit_int = typename std::enable_if<is_8bit_int<T>::value>::type;
+
 template <typename T>
 using enable_if_primitive_ctype =
     typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value>::type;
@@ -47,11 +63,7 @@ using enable_if_timestamp =
     typename std::enable_if<std::is_base_of<TimestampType, T>::value>::type;
 
 template <typename T>
-using enable_if_has_c_type =
-    typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
-                            std::is_base_of<DateType, T>::value ||
-                            std::is_base_of<TimeType, T>::value ||
-                            std::is_base_of<TimestampType, T>::value>::type;
+using enable_if_has_c_type = typename std::enable_if<has_c_type<T>::value>::type;
 
 template <typename T>
 using enable_if_null = typename std::enable_if<std::is_same<NullType, T>::value>::type;

From 94fd3ee8a70e5fcf5b2233aa64c0985fe8478c91 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 20 Feb 2018 21:12:20 +0100
Subject: [PATCH 1507/1644] ARROW-2172: [C++/Python] Fix converting from Numpy
 array with non-natural stride

Author: Antoine Pitrou <antoine@python.org>

Closes #1628 from pitrou/ARROW-2172-non-natural-stride-array and squashes the following commits:

6ae801f [Antoine Pitrou] ARROW-2172: [C++/Python] Fix converting from Numpy array with non-natural stride
---
 cpp/src/arrow/python/numpy_to_arrow.cc      | 32 +++++++++++++++------
 python/pyarrow/tests/test_convert_pandas.py | 13 +++++++++
 2 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 522bf51740d57..23418ad920c53 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -554,12 +554,22 @@ Status StaticCastBuffer(const Buffer& input, const int64_t length, MemoryPool* p
   return Status::OK();
 }
 
-template <typename T, typename T2>
-void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) {
+template <typename T>
+void CopyStridedBytewise(int8_t* input_data, int64_t length, int64_t stride,
+                         T* output_data) {
+  // Passing input_data as non-const is a concession to PyObject*
+  for (int64_t i = 0; i < length; ++i) {
+    memcpy(output_data + i, input_data, sizeof(T));
+    input_data += stride;
+  }
+}
+
+template <typename T>
+void CopyStridedNatural(T* input_data, int64_t length, int64_t stride, T* output_data) {
   // Passing input_data as non-const is a concession to PyObject*
   int64_t j = 0;
   for (int64_t i = 0; i < length; ++i) {
-    output_data[i] = static_cast<T2>(input_data[j]);
+    output_data[i] = input_data[j];
     j += stride;
   }
 }
@@ -571,13 +581,19 @@ Status CopyStridedArray(PyArrayObject* arr, const int64_t length, MemoryPool* po
   using T = typename traits::T;
 
   // Strided, must copy into new contiguous memory
-  const int64_t stride = PyArray_STRIDES(arr)[0];
-  const int64_t stride_elements = stride / sizeof(T);
-
   auto new_buffer = std::make_shared<PoolBuffer>(pool);
   RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length));
-  CopyStrided(reinterpret_cast<T*>(PyArray_DATA(arr)), length, stride_elements,
-              reinterpret_cast<T*>(new_buffer->mutable_data()));
+
+  const int64_t stride = PyArray_STRIDES(arr)[0];
+  if (stride % sizeof(T) == 0) {
+    const int64_t stride_elements = stride / sizeof(T);
+    CopyStridedNatural(reinterpret_cast<T*>(PyArray_DATA(arr)), length, stride_elements,
+                       reinterpret_cast<T*>(new_buffer->mutable_data()));
+  } else {
+    CopyStridedBytewise(reinterpret_cast<int8_t*>(PyArray_DATA(arr)), length, stride,
+                        reinterpret_cast<T*>(new_buffer->mutable_data()));
+  }
+
   *out = new_buffer;
   return Status::OK();
 }
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index f7718f06a410c..92f6c25f1965c 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1582,6 +1582,19 @@ def test_convert_empty_table(self):
         arr = pa.array([], type=pa.struct([pa.field('a', pa.int64())]))
         tm.assert_almost_equal(arr.to_pandas(), np.array([], dtype=object))
 
+    def test_non_natural_stride(self):
+        """
+        ARROW-2172: converting from a Numpy array with a stride that's
+        not a multiple of itemsize.
+        """
+        dtype = np.dtype([('x', np.int32), ('y', np.int16)])
+        data = np.array([(42, -1), (-43, 2)], dtype=dtype)
+        assert data.strides == (6,)
+        arr = pa.array(data['x'], type=pa.int32())
+        assert arr.to_pylist() == [42, -43]
+        arr = pa.array(data['y'], type=pa.int16())
+        assert arr.to_pylist() == [-1, 2]
+
 
 def _fully_loaded_dataframe_example():
     from distutils.version import LooseVersion

From 7d11d0245f0f9afda07906839595efe1f0df6e4e Mon Sep 17 00:00:00 2001
From: yosuke shiro <ys2014hs@gmail.com>
Date: Tue, 20 Feb 2018 21:13:07 +0100
Subject: [PATCH 1508/1644] ARROW-2190: [GLib] Add add/remove field functions
 for RecordBatch

Add AddColumn and RemoveColumn api to RecordBatch.

Author: yosuke shiro <ys2014hs@gmail.com>

Closes #1632 from shiro615/ARROW-2190 and squashes the following commits:

877d42e [yosuke shiro] ARROW-2190: [GLib] Add AddColumn and RemoveColumn api to RecordBatch
---
 c_glib/arrow-glib/record-batch.cpp | 58 ++++++++++++++++++++++++++++++
 c_glib/arrow-glib/record-batch.h   |  8 +++++
 c_glib/test/test-record-batch.rb   | 14 ++++++++
 3 files changed, 80 insertions(+)

diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp
index 64f2020ad3efc..0d0777dabb7ea 100644
--- a/c_glib/arrow-glib/record-batch.cpp
+++ b/c_glib/arrow-glib/record-batch.cpp
@@ -25,6 +25,7 @@
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
+#include <arrow-glib/field.hpp>
 
 #include <sstream>
 
@@ -350,6 +351,63 @@ garrow_record_batch_to_string(GArrowRecordBatch *record_batch, GError **error)
   }
 }
 
+/**
+ * garrow_record_batch_add_column:
+ * @record_batch: A #GArrowRecordBatch.
+ * @i: The index of the new column.
+ * @field: The field to be added.
+ * @column: The column to be added.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): The newly allocated
+ *   #GArrowRecordBatch that has a new column or %NULL on error.
+ *
+ * Since: 0.9.0
+ */
+GArrowRecordBatch *
+garrow_record_batch_add_column(GArrowRecordBatch *record_batch,
+                               guint i,
+                               GArrowField *field,
+                               GArrowArray *column,
+                               GError **error)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  const auto arrow_field = garrow_field_get_raw(field);
+  const auto arrow_column = garrow_array_get_raw(column);
+  std::shared_ptr<arrow::RecordBatch> arrow_new_record_batch;
+  auto status = arrow_record_batch->AddColumn(i, arrow_field, arrow_column, &arrow_new_record_batch);
+  if (garrow_error_check(error, status, "[record-batch][add-column]")) {
+    return garrow_record_batch_new_raw(&arrow_new_record_batch);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_record_batch_remove_column:
+ * @record_batch: A #GArrowRecordBatch.
+ * @i: The index of the new column.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable) (transfer full): The newly allocated
+ *   #GArrowRecordBatch that doesn't have the column or %NULL on error.
+ *
+ * Since: 0.9.0
+ */
+GArrowRecordBatch *
+garrow_record_batch_remove_column(GArrowRecordBatch *record_batch,
+                                  guint i,
+                                  GError **error)
+{
+  const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch);
+  std::shared_ptr<arrow::RecordBatch> arrow_new_record_batch;
+  auto status = arrow_record_batch->RemoveColumn(i, &arrow_new_record_batch);
+  if (garrow_error_check(error, status, "[record-batch][remove-column]")) {
+    return garrow_record_batch_new_raw(&arrow_new_record_batch);
+  } else {
+    return NULL;
+  }
+}
 
 G_END_DECLS
 
diff --git a/c_glib/arrow-glib/record-batch.h b/c_glib/arrow-glib/record-batch.h
index d31edf43973d0..bd6a1ce2f4efe 100644
--- a/c_glib/arrow-glib/record-batch.h
+++ b/c_glib/arrow-glib/record-batch.h
@@ -88,5 +88,13 @@ GArrowRecordBatch *garrow_record_batch_slice     (GArrowRecordBatch *record_batc
 
 gchar        *garrow_record_batch_to_string      (GArrowRecordBatch *record_batch,
                                                   GError **error);
+GArrowRecordBatch *garrow_record_batch_add_column(GArrowRecordBatch *record_batch,
+                                                  guint i,
+                                                  GArrowField *field,
+                                                  GArrowArray *column,
+                                                  GError **error);
+GArrowRecordBatch *garrow_record_batch_remove_column(GArrowRecordBatch *record_batch,
+                                                     guint i,
+                                                     GError **error);
 
 G_END_DECLS
diff --git a/c_glib/test/test-record-batch.rb b/c_glib/test/test-record-batch.rb
index 365922f496fe2..b215e725a6b8e 100644
--- a/c_glib/test/test-record-batch.rb
+++ b/c_glib/test/test-record-batch.rb
@@ -135,5 +135,19 @@ def test_to_s
 valid: [false, true, false, true, false]
       PRETTY_PRINT
     end
+
+    def test_add_column
+      field = Arrow::Field.new("added", Arrow::BooleanDataType.new)
+      column = build_boolean_array([false, false, true, true, true])
+      new_record_batch = @record_batch.add_column(1, field, column)
+      assert_equal(["visible", "added", "valid"],
+                   new_record_batch.schema.fields.collect(&:name))
+    end
+
+    def test_remove_column
+      new_record_batch = @record_batch.remove_column(0)
+      assert_equal(["valid"],
+                   new_record_batch.schema.fields.collect(&:name))
+    end
   end
 end

From 9298c9397f54068b8d5cb3ad331ca0ecbd6293e3 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 21 Feb 2018 13:06:59 -0500
Subject: [PATCH 1509/1644] ARROW-2192: [CI] Always build on master branch and
 repository

Author: Antoine Pitrou <antoine@python.org>

Closes #1634 from pitrou/ARROW-2192-always-build-on-master and squashes the following commits:

c7a87805 [Antoine Pitrou] ARROW-2192: [CI] Always build on master branch and repo [skip travis] [skip appveyor]
---
 ci/travis_detect_changes.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/ci/travis_detect_changes.py b/ci/travis_detect_changes.py
index 2aeb34fa00af5..d60b13227af92 100644
--- a/ci/travis_detect_changes.py
+++ b/ci/travis_detect_changes.py
@@ -147,19 +147,25 @@ def get_unix_shell_eval(env):
 
 
 def run_from_travis():
-    desc = get_travis_commit_description()
-    if '[skip travis]' in desc:
-        # Skip everything
-        affected = dict.fromkeys(ALL_TOPICS, False)
-    elif '[force ci]' in desc or '[force travis]' in desc:
-        # Test everything
+    if (os.environ['TRAVIS_REPO_SLUG'] == 'apache/arrow' and
+        os.environ['TRAVIS_BRANCH'] == 'master' and
+        os.environ['TRAVIS_EVENT_TYPE'] != 'pull_request'):
+        # Never skip anything on master builds in the official repository
         affected = dict.fromkeys(ALL_TOPICS, True)
     else:
-        # Test affected topics
-        affected_files = list_travis_affected_files()
-        perr("Affected files:", affected_files)
-        affected = get_affected_topics(affected_files)
-        assert set(affected) <= set(ALL_TOPICS), affected
+        desc = get_travis_commit_description()
+        if '[skip travis]' in desc:
+            # Skip everything
+            affected = dict.fromkeys(ALL_TOPICS, False)
+        elif '[force ci]' in desc or '[force travis]' in desc:
+            # Test everything
+            affected = dict.fromkeys(ALL_TOPICS, True)
+        else:
+            # Test affected topics
+            affected_files = list_travis_affected_files()
+            perr("Affected files:", affected_files)
+            affected = get_affected_topics(affected_files)
+            assert set(affected) <= set(ALL_TOPICS), affected
 
     perr("Affected topics:")
     perr(pprint.pformat(affected))

From 9fefc23519d7f4dbd796807562f5302cceda44d7 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Wed, 21 Feb 2018 10:42:11 -0800
Subject: [PATCH 1510/1644] ARROW-2162: [Python/C++] Decimal Values with
 too-high precision are multiplied by 100

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1619 from cpcloud/ARROW-2162 and squashes the following commits:

57f95eee [Phillip Cloud] ARROW-2162: [Python/C++] Decimal Values with too-high precision are multiplied by 100
---
 cpp/src/arrow/python/python-test.cc         | 40 +++++++++++++++++++++
 cpp/src/arrow/util/decimal.cc               | 38 +++++++++++++++-----
 python/pyarrow/tests/test_convert_pandas.py | 11 ++++++
 3 files changed, 80 insertions(+), 9 deletions(-)

diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index a2b832bdba74a..b76caaecee6ba 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -201,5 +201,45 @@ TEST(BuiltinConversionTest, TestMixedTypeFails) {
   ASSERT_RAISES(UnknownError, ConvertPySequence(list, pool, &arr));
 }
 
+TEST_F(DecimalTest, FromPythonDecimalRescaleNotTruncateable) {
+  // We fail when truncating values that would lose data if cast to a decimal type with
+  // lower scale
+  Decimal128 value;
+  OwnedRef python_decimal(this->CreatePythonDecimal("1.001"));
+  auto type = ::arrow::decimal(10, 2);
+  const auto& decimal_type = static_cast<const DecimalType&>(*type);
+  ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(),
+                                                            decimal_type, &value));
+}
+
+TEST_F(DecimalTest, FromPythonDecimalRescaleTruncateable) {
+  // We allow truncation of values that do not lose precision when dividing by 10 * the
+  // difference between the scales, e.g., 1.000 -> 1.00
+  Decimal128 value;
+  OwnedRef python_decimal(this->CreatePythonDecimal("1.000"));
+  auto type = ::arrow::decimal(10, 2);
+  const auto& decimal_type = static_cast<const DecimalType&>(*type);
+  ASSERT_OK(
+      internal::DecimalFromPythonDecimal(python_decimal.obj(), decimal_type, &value));
+  ASSERT_EQ(100, value.low_bits());
+}
+
+TEST_F(DecimalTest, TestOverflowFails) {
+  Decimal128 value;
+  int32_t precision;
+  int32_t scale;
+  OwnedRef python_decimal(
+      this->CreatePythonDecimal("9999999999999999999999999999999999999.9"));
+  ASSERT_OK(
+      internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));
+  ASSERT_EQ(38, precision);
+  ASSERT_EQ(1, scale);
+
+  auto type = ::arrow::decimal(38, 38);
+  const auto& decimal_type = static_cast<const DecimalType&>(*type);
+  ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(),
+                                                            decimal_type, &value));
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index e999854b10860..a3c8cda7654ab 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -854,26 +854,46 @@ static const Decimal128 ScaleMultipliers[] = {
     Decimal128("10000000000000000000000000000000000000"),
     Decimal128("100000000000000000000000000000000000000")};
 
+static bool RescaleWouldCauseDataLoss(const Decimal128& value, int32_t delta_scale,
+                                      int32_t abs_delta_scale, Decimal128* result) {
+  Decimal128 multiplier(ScaleMultipliers[abs_delta_scale]);
+
+  if (delta_scale < 0) {
+    DCHECK_NE(multiplier, 0);
+    Decimal128 remainder;
+    Status status = value.Divide(multiplier, result, &remainder);
+    DCHECK(status.ok()) << status.message();
+    return remainder != 0;
+  }
+
+  *result = value * multiplier;
+  return *result < value;
+}
+
 Status Decimal128::Rescale(int32_t original_scale, int32_t new_scale,
                            Decimal128* out) const {
-  DCHECK_NE(out, NULLPTR);
-  DCHECK_NE(original_scale, new_scale);
-  const int32_t delta_scale = original_scale - new_scale;
+  DCHECK_NE(out, NULLPTR) << "out is nullptr";
+  DCHECK_NE(original_scale, new_scale) << "original_scale != new_scale";
+
+  const int32_t delta_scale = new_scale - original_scale;
   const int32_t abs_delta_scale = std::abs(delta_scale);
+
   DCHECK_GE(abs_delta_scale, 1);
   DCHECK_LE(abs_delta_scale, 38);
 
-  const Decimal128 scale_multiplier = ScaleMultipliers[abs_delta_scale];
-  const Decimal128 result = *this * scale_multiplier;
+  Decimal128 result(*this);
+  const bool rescale_would_cause_data_loss =
+      RescaleWouldCauseDataLoss(result, delta_scale, abs_delta_scale, out);
 
-  if (ARROW_PREDICT_FALSE(result < *this)) {
+  // Fail if we overflow or truncate
+  if (ARROW_PREDICT_FALSE(rescale_would_cause_data_loss)) {
     std::stringstream buf;
-    buf << "Rescaling decimal value from original scale " << original_scale
-        << " to new scale " << new_scale << " would cause overflow";
+    buf << "Rescaling decimal value " << ToString(original_scale)
+        << " from original scale of " << original_scale << " to new scale of "
+        << new_scale << " would cause data loss";
     return Status::Invalid(buf.str());
   }
 
-  *out = result;
   return Status::OK();
 }
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 92f6c25f1965c..6b62622f5b85e 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1132,6 +1132,17 @@ def test_decimal_128_to_pandas(self):
         df = converted.to_pandas()
         tm.assert_frame_equal(df, expected)
 
+    def test_decimal_fails_with_truncation(self):
+        data1 = [decimal.Decimal('1.234')]
+        type1 = pa.decimal128(10, 2)
+        with pytest.raises(pa.ArrowException):
+            pa.array(data1, type=type1)
+
+        data2 = [decimal.Decimal('1.2345')]
+        type2 = pa.decimal128(10, 3)
+        with pytest.raises(pa.ArrowException):
+            pa.array(data2, type=type2)
+
 
 class TestListTypes(object):
     """

From 897cc4d917e375b180147856baa9c5da2e6173e5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Feb 2018 10:45:45 -0500
Subject: [PATCH 1511/1644] ARROW-1345: [Python] Test conversion from nested
 NumPy arrays with smaller int, float types

This also resolves ARROW-2008

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1643 from wesm/ARROW-1345 and squashes the following commits:

a0997104 [Wes McKinney] Add another test
8af33486 [Wes McKinney] Add unit test for ARROW-1345
---
 python/pyarrow/tests/test_convert_pandas.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 6b62622f5b85e..6e68dd961bef8 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1269,6 +1269,21 @@ def test_nested_lists_all_empty(self):
         assert arr.equals(expected)
         assert arr.type == pa.list_(pa.null())
 
+    def test_nested_smaller_ints(self):
+        # ARROW-1345, ARROW-2008, there were some type inference bugs happening
+        # before
+        data = pd.Series([np.array([1, 2, 3], dtype='i1'), None])
+        result = pa.array(data)
+        result2 = pa.array(data.values)
+        expected = pa.array([[1, 2, 3], None], type=pa.list_(pa.int8()))
+        assert result.equals(expected)
+        assert result2.equals(expected)
+
+        data3 = pd.Series([np.array([1, 2, 3], dtype='f4'), None])
+        result3 = pa.array(data3)
+        expected3 = pa.array([[1, 2, 3], None], type=pa.list_(pa.float32()))
+        assert result3.equals(expected3)
+
     def test_infer_lists(self):
         data = OrderedDict([
             ('nan_ints', [[None, 1], [2, 3]]),

From 3e87f7753a252d1e76187894bb294954e45a93ee Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Feb 2018 12:09:39 -0500
Subject: [PATCH 1512/1644] ARROW-2131: [Python] Prepend module path to
 PYTHONPATH when spawning subprocess

This enables this test to pass in an in-place build without running `setup.py develop`

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1640 from wesm/ARROW-2131 and squashes the following commits:

460ded1b [Wes McKinney] Use more idiomatic option for getting module parent directory
faa247af [Wes McKinney] Prepend development module path when spawning subprocess in test_serialization.py
---
 python/pyarrow/tests/test_serialization.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 0917172d22c1f..feccebbde36e0 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -580,6 +580,22 @@ def deserialize_regex(serialized, q):
     p.join()
 
 
+def _get_modified_env_with_pythonpath():
+    # Prepend pyarrow root directory to PYTHONPATH
+    env = os.environ.copy()
+    existing_pythonpath = env.get('PYTHONPATH', '')
+    if sys.platform == 'win32':
+        sep = ';'
+    else:
+        sep = ':'
+
+    module_path = os.path.abspath(
+        os.path.dirname(os.path.dirname(pa.__file__)))
+
+    env['PYTHONPATH'] = sep.join((module_path, existing_pythonpath))
+    return env
+
+
 def test_deserialize_buffer_in_different_process():
     import tempfile
     import subprocess
@@ -589,9 +605,12 @@ def test_deserialize_buffer_in_different_process():
     f.write(b.to_pybytes())
     f.close()
 
+    subprocess_env = _get_modified_env_with_pythonpath()
+
     dir_path = os.path.dirname(os.path.realpath(__file__))
     python_file = os.path.join(dir_path, 'deserialize_buffer.py')
-    subprocess.check_call([sys.executable, python_file, f.name])
+    subprocess.check_call([sys.executable, python_file, f.name],
+                          env=subprocess_env)
 
 
 def test_set_pickle():

From bcbcf025fe30f41cc38bcec4489802064f3573d4 Mon Sep 17 00:00:00 2001
From: Jeffrey Heer <jeffrey.heer@gmail.com>
Date: Thu, 22 Feb 2018 09:10:32 -0800
Subject: [PATCH 1513/1644] [JS] Fix typo in npm target for esNext/CommonJS.
 (#1645)

---
 js/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/README.md b/js/README.md
index b427923e37ea1..a34664b27767f 100644
--- a/js/README.md
+++ b/js/README.md
@@ -186,7 +186,7 @@ npm install @apache-arrow/es5-umd # standalone es5/UMD package
 npm install @apache-arrow/es2015-cjs # standalone es2015/CommonJS package
 npm install @apache-arrow/es2015-esm # standalone es2015/ESModules package
 npm install @apache-arrow/es2015-umd # standalone es2015/UMD package
-npm install @apache-arrow/esnext-esm # standalone esNext/CommonJS package
+npm install @apache-arrow/esnext-cjs # standalone esNext/CommonJS package
 npm install @apache-arrow/esnext-esm # standalone esNext/ESModules package
 npm install @apache-arrow/esnext-umd # standalone esNext/UMD package
 ```

From 5f100674096b4dd365929a33e0a724fa432c6d36 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Feb 2018 12:11:09 -0500
Subject: [PATCH 1514/1644] ARROW-2180: [C++] Remove deprecated APIs from 0.8.0
 cycle

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1638 from wesm/ARROW-2180 and squashes the following commits:

a13931c9 [Wes McKinney] Remove deprecated C++ APIs from 0.8.0 cycle
---
 cpp/src/arrow/array.cc   | 20 --------------------
 cpp/src/arrow/array.h    | 29 -----------------------------
 cpp/src/arrow/buffer.h   | 16 ----------------
 cpp/src/arrow/compare.cc | 26 --------------------------
 cpp/src/arrow/compare.h  | 27 ---------------------------
 cpp/src/arrow/table.cc   | 26 --------------------------
 cpp/src/arrow/table.h    | 12 ------------
 cpp/src/arrow/type.cc    | 20 --------------------
 cpp/src/arrow/type.h     | 12 ------------
 9 files changed, 188 deletions(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index a8043d69b92f2..83142dfefaab9 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -140,15 +140,6 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t le
   SetData(ArrayData::Make(type, length, {null_bitmap, data}, null_count, offset));
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-const uint8_t* PrimitiveArray::raw_values() const {
-  return raw_values_ +
-         offset() * static_cast<const FixedWidthType&>(*type()).bit_width() / CHAR_BIT;
-}
-
-#endif
-
 template <typename T>
 NumericArray<T>::NumericArray(const std::shared_ptr<ArrayData>& data)
     : PrimitiveArray(data) {
@@ -752,17 +743,6 @@ class ArrayDataWrapper {
 
 }  // namespace internal
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out) {
-  internal::ArrayDataWrapper wrapper_visitor(data, out);
-  RETURN_NOT_OK(VisitTypeInline(*data->type, &wrapper_visitor));
-  DCHECK(out);
-  return Status::OK();
-}
-
-#endif
-
 std::shared_ptr<Array> MakeArray(const std::shared_ptr<ArrayData>& data) {
   std::shared_ptr<Array> out;
   internal::ArrayDataWrapper wrapper_visitor(data, &out);
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 5b9ce9a01fbe4..faa9211c674ae 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -146,13 +146,6 @@ struct ARROW_EXPORT ArrayData {
 
   std::shared_ptr<ArrayData> Copy() const { return std::make_shared<ArrayData>(*this); }
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-  // Deprecated since 0.8.0
-  std::shared_ptr<ArrayData> ShallowCopy() const { return Copy(); }
-
-#endif
-
   std::shared_ptr<DataType> type;
   int64_t length;
   int64_t null_count;
@@ -161,19 +154,6 @@ struct ARROW_EXPORT ArrayData {
   std::vector<std::shared_ptr<ArrayData>> child_data;
 };
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-/// \brief Create a strongly-typed Array instance from generic ArrayData
-/// \param[in] data the array contents
-/// \param[out] out the resulting Array instance
-/// \return Status
-///
-/// \note Deprecated since 0.8.0
-ARROW_EXPORT
-Status MakeArray(const std::shared_ptr<ArrayData>& data, std::shared_ptr<Array>* out);
-
-#endif
-
 /// \brief Create a strongly-typed Array instance from generic ArrayData
 /// \param[in] data the array contents
 /// \return the resulting Array instance
@@ -335,15 +315,6 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray {
   /// Does not account for any slice offset
   std::shared_ptr<Buffer> values() const { return data_->buffers[1]; }
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-  /// \brief Return pointer to start of raw data
-  ///
-  /// \note Deprecated since 0.8.0
-  const uint8_t* raw_values() const;
-
-#endif
-
  protected:
   PrimitiveArray() {}
 
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index 74a3c680d268e..cf25ccd036490 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -371,22 +371,6 @@ ARROW_EXPORT
 Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size,
                                std::shared_ptr<ResizableBuffer>* out);
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-/// \brief Create Buffer referencing std::string memory
-///
-/// Warning: string instance must stay alive
-///
-/// \param str std::string instance
-/// \return std::shared_ptr<Buffer>
-///
-/// \note Deprecated Since 0.8.0
-static inline std::shared_ptr<Buffer> GetBufferFromString(const std::string& str) {
-  return std::make_shared<Buffer>(str);
-}
-
-#endif  // ARROW_NO_DEPRECATED_API
-
 }  // namespace arrow
 
 #endif  // ARROW_BUFFER_H
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 9ed54ca3a4fc8..69cacbfac818f 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -783,30 +783,4 @@ bool TypeEquals(const DataType& left, const DataType& right) {
   return are_equal;
 }
 
-Status ArrayEquals(const Array& left, const Array& right, bool* are_equal) {
-  *are_equal = ArrayEquals(left, right);
-  return Status::OK();
-}
-
-Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) {
-  *are_equal = TensorEquals(left, right);
-  return Status::OK();
-}
-
-Status ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal) {
-  *are_equal = ArrayApproxEquals(left, right);
-  return Status::OK();
-}
-
-Status ArrayRangeEquals(const Array& left, const Array& right, int64_t start_idx,
-                        int64_t end_idx, int64_t other_start_idx, bool* are_equal) {
-  *are_equal = ArrayRangeEquals(left, right, start_idx, end_idx, other_start_idx);
-  return Status::OK();
-}
-
-Status TypeEquals(const DataType& left, const DataType& right, bool* are_equal) {
-  *are_equal = TypeEquals(left, right);
-  return Status::OK();
-}
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h
index df3386e4bfc19..956ae897efd9e 100644
--- a/cpp/src/arrow/compare.h
+++ b/cpp/src/arrow/compare.h
@@ -31,33 +31,6 @@ class DataType;
 class Status;
 class Tensor;
 
-#ifndef ARROW_NO_DEPRECATED_API
-/// Returns true if the arrays are exactly equal
-/// \note Deprecated since 0.8.0
-Status ARROW_EXPORT ArrayEquals(const Array& left, const Array& right, bool* are_equal);
-
-/// \note Deprecated since 0.8.0
-Status ARROW_EXPORT TensorEquals(const Tensor& left, const Tensor& right,
-                                 bool* are_equal);
-
-/// Returns true if the arrays are approximately equal. For non-floating point
-/// types, this is equivalent to ArrayEquals(left, right)
-/// \note Deprecated since 0.8.0
-Status ARROW_EXPORT ArrayApproxEquals(const Array& left, const Array& right,
-                                      bool* are_equal);
-
-/// Returns true if indicated equal-length segment of arrays is exactly equal
-/// \note Deprecated since 0.8.0
-Status ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right,
-                                     int64_t start_idx, int64_t end_idx,
-                                     int64_t other_start_idx, bool* are_equal);
-
-/// Returns true if the type metadata are exactly equal
-/// \note Deprecated since 0.8.0
-Status ARROW_EXPORT TypeEquals(const DataType& left, const DataType& right,
-                               bool* are_equal);
-#endif
-
 /// Returns true if the arrays are exactly equal
 bool ARROW_EXPORT ArrayEquals(const Array& left, const Array& right);
 
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index 62ea32a8d9080..ed5858624ae30 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -388,32 +388,6 @@ bool Table::Equals(const Table& other) const {
   return true;
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-Status MakeTable(const std::shared_ptr<Schema>& schema,
-                 const std::vector<std::shared_ptr<Array>>& arrays,
-                 std::shared_ptr<Table>* table) {
-  // Make sure the length of the schema corresponds to the length of the vector
-  if (schema->num_fields() != static_cast<int>(arrays.size())) {
-    std::stringstream ss;
-    ss << "Schema and Array vector have different lengths: " << schema->num_fields()
-       << " != " << arrays.size();
-    return Status::Invalid(ss.str());
-  }
-
-  std::vector<std::shared_ptr<Column>> columns;
-  columns.reserve(schema->num_fields());
-  for (int i = 0; i < schema->num_fields(); i++) {
-    columns.emplace_back(std::make_shared<Column>(schema->field(i), arrays[i]));
-  }
-
-  *table = Table::Make(schema, columns);
-
-  return Status::OK();
-}
-
-#endif  // ARROW_NO_DEPRECATED_API
-
 // ----------------------------------------------------------------------
 // Convert a table to a sequence of record batches
 
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 6938db1be3a96..7274fca4d23d5 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -244,18 +244,6 @@ ARROW_EXPORT
 Status ConcatenateTables(const std::vector<std::shared_ptr<Table>>& tables,
                          std::shared_ptr<Table>* table);
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-/// \brief Construct table from multiple input tables.
-/// \return Status, fails if any schemas are different
-/// \note Deprecated since 0.8.0
-ARROW_EXPORT
-Status MakeTable(const std::shared_ptr<Schema>& schema,
-                 const std::vector<std::shared_ptr<Array>>& arrays,
-                 std::shared_ptr<Table>* table);
-
-#endif
-
 }  // namespace arrow
 
 #endif  // ARROW_TABLE_H
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 6574cce50ee16..792d1bfd035a3 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -38,16 +38,6 @@ std::shared_ptr<Field> Field::AddMetadata(
   return std::make_shared<Field>(name_, type_, nullable_, metadata);
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-Status Field::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
-                          std::shared_ptr<Field>* out) const {
-  *out = AddMetadata(metadata);
-  return Status::OK();
-}
-
-#endif
-
 std::shared_ptr<Field> Field::RemoveMetadata() const {
   return std::make_shared<Field>(name_, type_, nullable_);
 }
@@ -307,16 +297,6 @@ std::shared_ptr<Schema> Schema::AddMetadata(
   return std::make_shared<Schema>(fields_, metadata);
 }
 
-#ifndef ARROW_NO_DEPRECATED_API
-
-Status Schema::AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
-                           std::shared_ptr<Schema>* out) const {
-  *out = AddMetadata(metadata);
-  return Status::OK();
-}
-
-#endif
-
 std::shared_ptr<const KeyValueMetadata> Schema::metadata() const { return metadata_; }
 
 std::shared_ptr<Schema> Schema::RemoveMetadata() const {
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index cfee6fd0e2363..95f010a8f23c3 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -232,12 +232,6 @@ class ARROW_EXPORT Field {
 
   std::shared_ptr<const KeyValueMetadata> metadata() const { return metadata_; }
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \note Deprecated since 0.8.0
-  Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
-                     std::shared_ptr<Field>* out) const;
-#endif
-
   std::shared_ptr<Field> AddMetadata(
       const std::shared_ptr<const KeyValueMetadata>& metadata) const;
   std::shared_ptr<Field> RemoveMetadata() const;
@@ -768,12 +762,6 @@ class ARROW_EXPORT Schema {
                   std::shared_ptr<Schema>* out) const;
   Status RemoveField(int i, std::shared_ptr<Schema>* out) const;
 
-#ifndef ARROW_NO_DEPRECATED_API
-  /// \note Deprecated since 0.8.0
-  Status AddMetadata(const std::shared_ptr<const KeyValueMetadata>& metadata,
-                     std::shared_ptr<Schema>* out) const;
-#endif
-
   /// \brief Replace key-value metadata with new metadata
   ///
   /// \param[in] metadata new KeyValueMetadata

From cdc347cc01dc4c3e168f59542e517b34adeeb0bd Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Feb 2018 12:11:38 -0500
Subject: [PATCH 1515/1644] ARROW-2132: Add link to Plasma in main README

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1636 from wesm/ARROW-2132 and squashes the following commits:

dbbef5d8 [Wes McKinney] Add link to Plasma in main README, tweak language
---
 README.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index a7fed986526dd..846a4030e83d1 100644
--- a/README.md
+++ b/README.md
@@ -38,10 +38,12 @@ set of technologies that enable big data systems to process and move data fast.
 Major components of the project include:
 
  - [The Arrow Columnar In-Memory Format](https://github.com/apache/arrow/tree/master/format)
- - [C++ implementation](https://github.com/apache/arrow/tree/master/cpp)
+ - [C++ libraries](https://github.com/apache/arrow/tree/master/cpp)
+ - [Plasma Object Store](https://github.com/apache/arrow/tree/master/cpp/src/plasma): a
+   shared-memory blob store, part of the C++ codebase
  - [C bindings using GLib](https://github.com/apache/arrow/tree/master/c_glib)
- - [Java implementation](https://github.com/apache/arrow/tree/master/java)
- - [JavaScript implementation](https://github.com/apache/arrow/tree/master/js)
+ - [Java libraries](https://github.com/apache/arrow/tree/master/java)
+ - [JavaScript libraries](https://github.com/apache/arrow/tree/master/js)
  - [Python bindings to C++](https://github.com/apache/arrow/tree/master/python)
 
 Arrow is an [Apache Software Foundation](https://www.apache.org) project. Learn more at
@@ -49,8 +51,7 @@ Arrow is an [Apache Software Foundation](https://www.apache.org) project. Learn
 
 ### What's in the Arrow libraries?
 
-The reference Arrow implementations contain a number of distinct software
-components:
+The reference Arrow libraries contain a number of distinct software components:
 
 - Columnar vector and table-like containers (similar to data frames) supporting
   flat or nested types

From 27f7eba92166d0790595c2d235c4597f88b71fd1 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Feb 2018 17:53:06 -0500
Subject: [PATCH 1516/1644] ARROW-2069: [Python] Add note that Plasma is not
 supported on Windows

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1641 from wesm/ARROW-2069 and squashes the following commits:

e66d1b60 [Wes McKinney] Add note that Plasma is not supported on Windows
---
 python/doc/source/plasma.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/python/doc/source/plasma.rst b/python/doc/source/plasma.rst
index 74837b96c4831..b64b4c260659f 100644
--- a/python/doc/source/plasma.rst
+++ b/python/doc/source/plasma.rst
@@ -24,6 +24,9 @@ The Plasma In-Memory Object Store
 .. contents:: Contents
   :depth: 3
 
+.. note::
+
+   As present, Plasma is only supported for use on Linux and macOS.
 
 The Plasma API
 --------------

From 81bfb38e2fbe246347822c42197d76e655d6b0a0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Feb 2018 19:47:05 -0500
Subject: [PATCH 1517/1644] ARROW-2185: Strip CI directives from commit
 messages

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1639 from wesm/ARROW-2185 and squashes the following commits:

adf038d5  Fix regex
75c3b178  Do not eat closing braces
baa7e2c6  Strip CI directives from commit messages when preparing squashed commit message
---
 dev/merge_arrow_pr.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index 6c0e663769e16..74f0762c3fe09 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -175,7 +175,8 @@ def merge_pr(pr_num, target_ref):
         "Closes #%s from %s and squashes the following commits:"
         % (pr_num, pr_repo_desc)]
     for c in commits:
-        merge_message_flags += ["-m", c]
+        stripped_message = strip_ci_directives(c).strip()
+        merge_message_flags += ["-m", stripped_message]
 
     run_cmd(['git', 'commit',
              '--no-verify',  # do not run commit hooks
@@ -199,6 +200,15 @@ def merge_pr(pr_num, target_ref):
     return merge_hash
 
 
+_REGEX_CI_DIRECTIVE = re.compile('\[[^\]]*\]')
+
+
+def strip_ci_directives(commit_message):
+    # Remove things like '[force ci]', '[skip appveyor]' from the assembled
+    # commit message
+    return _REGEX_CI_DIRECTIVE.sub('', commit_message)
+
+
 def fix_version_from_branch(branch, versions):
     # Note: Assumes this is a sorted (newest->oldest) list of un-released
     # versions
@@ -209,7 +219,7 @@ def fix_version_from_branch(branch, versions):
         return [x for x in versions if x.name.startswith(branch_ver)][-1]
 
 
-def exctract_jira_id(title):
+def extract_jira_id(title):
     m = re.search(r'^(ARROW-[0-9]+)\b.*$', title)
     if m:
         return m.group(1)
@@ -219,7 +229,7 @@ def exctract_jira_id(title):
 
 
 def check_jira(title):
-    jira_id = exctract_jira_id(title)
+    jira_id = extract_jira_id(title)
     asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
                                 basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
     try:
@@ -232,7 +242,7 @@ def resolve_jira(title, merge_branches, comment):
     asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
                                 basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
 
-    default_jira_id = exctract_jira_id(title)
+    default_jira_id = extract_jira_id(title)
 
     jira_id = input("Enter a JIRA id [%s]: " % default_jira_id)
     if jira_id == "":

From d52f2ff54ae7ef56c19b615ae247ee5aa268472a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Feb 2018 19:50:02 -0500
Subject: [PATCH 1518/1644] [Dev] Follow-up, use angle brackets for commit
 author instead of square brackets

Change-Id: Iad77d14c9116c30249e3e7f13ed40ed82e579832
---
 dev/merge_arrow_pr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py
index 74f0762c3fe09..a9c75fe618abd 100755
--- a/dev/merge_arrow_pr.py
+++ b/dev/merge_arrow_pr.py
@@ -148,7 +148,7 @@ def merge_pr(pr_num, target_ref):
                               reverse=True)
     primary_author = distinct_authors[0]
     commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,
-                      '--pretty=format:%h [%an] %s']).split("\n\n")
+                      '--pretty=format:%h <%an> %s']).split("\n\n")
 
     merge_message_flags = []
 

From c2865d064cd1d7622ca33d77a6a683dfb2903e38 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Feb 2018 19:53:33 -0500
Subject: [PATCH 1519/1644] ARROW-2093: [Python] Do not install PyTorch in
 Travis CI

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1637 from wesm/ARROW-2093 and squashes the following commits:

e3040a99 <Wes McKinney> Do not install PyTorch in Travis CI
---
 ci/travis_script_python.sh | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index a487da596bd04..9ed5825bb8ffa 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -43,11 +43,14 @@ conda install -y -q pip \
       pandas \
       cython
 
-if [ "$PYTHON_VERSION" != "2.7" ] || [ $TRAVIS_OS_NAME != "osx" ]; then
-  # Install pytorch for torch tensor conversion tests
-  # PyTorch seems to be broken on Python 2.7 on macOS so we skip it
-  conda install -y -q pytorch torchvision -c soumith
-fi
+# ARROW-2093: PyTorch increases the size of our conda dependency stack
+# significantly, and so we have disabled these tests in Travis CI for now
+
+# if [ "$PYTHON_VERSION" != "2.7" ] || [ $TRAVIS_OS_NAME != "osx" ]; then
+#   # Install pytorch for torch tensor conversion tests
+#   # PyTorch seems to be broken on Python 2.7 on macOS so we skip it
+#   conda install -y -q pytorch torchvision -c soumith
+# fi
 
 # Build C++ libraries
 mkdir -p $ARROW_CPP_BUILD_DIR

From 2f01658f9c953d39df8c09f00b8d69d8d7539901 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Fri, 23 Feb 2018 11:25:18 +0100
Subject: [PATCH 1520/1644] ARROW-2201: [Website] Publish JS API Docs

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1647 from TheNeuralBit/js-doc-site and squashes the following commits:

e5ca4df [Brian Hulette] Add JS docs to website
---
 site/README.md             | 8 ++++++++
 site/_includes/header.html | 1 +
 2 files changed, 9 insertions(+)

diff --git a/site/README.md b/site/README.md
index c1522651c506f..3da7c9764c9bf 100644
--- a/site/README.md
+++ b/site/README.md
@@ -106,4 +106,12 @@ LD_LIBRARY_PATH=$PWD/../cpp/build/debug make GTK_DOC_V_XREF=": "
 rsync -r doc/reference/html/ ../site/asf-site/docs/c_glib/
 ```
 
+#### Javascript
+
+```
+cd ../js
+npm run doc
+rsync -r doc/ ../site/asf-site/docs/js
+```
+
 Then add/commit/push from the site/asf-site git checkout.
diff --git a/site/_includes/header.html b/site/_includes/header.html
index 03b3c8750cdb8..e6d7479d3c3e8 100644
--- a/site/_includes/header.html
+++ b/site/_includes/header.html
@@ -52,6 +52,7 @@
             <li><a href="{{ site.baseurl }}/docs/cpp">C++ API</a></li>
             <li><a href="{{ site.baseurl }}/docs/java">Java API</a></li>
             <li><a href="{{ site.baseurl }}/docs/c_glib">C GLib API</a></li>
+            <li><a href="{{ site.baseurl }}/docs/js">Javascript API</a></li>
           </ul>
         </li>
         <!-- <li><a href="{{ site.baseurl }}/blog">Blog</a></li> -->

From 3e3f7c2c583054ec226cb5909a9368f920eae06c Mon Sep 17 00:00:00 2001
From: rrussell <rrussell@adobe.com>
Date: Fri, 23 Feb 2018 13:38:38 +0100
Subject: [PATCH 1521/1644] ARROW-2066: [Python] Document using pyarrow with
 Azure Blob Store

Original question:

https://github.com/apache/arrow/issues/1510

Improvement story:

https://issues.apache.org/jira/browse/ARROW-2066

Author: rrussell <rrussell@adobe.com>

Closes #1544 from rjrussell77/arrow-2066-docs-azure-parquet and squashes the following commits:

0d3972c <rrussell> Add missing byte_stream declaration/assignment
a5addb0 <rrussell> use more common 'df' instead of 'pd' for pandas dataframe variable, remove head() call and instead use comment to indicate generic fill-in code, add comment re: stream closure in finally block
f056888 <rrussell> Clean up white space
1fe9866 <rrussell> Add try/except/finally blocks to ensure closure of the byte stream
36f7378 <rrussell> Replace usage of tempfile buffer with BytesIO stream
654a6f9 <rrussell> Add back original Notes bullets
5d450fc <rrussell> fix
4770de1 <rrussell> fix
4c75824 <rrussell> Try moving the bullet to remove italics
803cbca <rrussell> Use asterisks for list
051b91d <rrussell> Fix formatting
a015deb <rrussell> Fix formatting
1815816 <rrussell> fix formatting
599e04f <rrussell> Fix formatting
34c5a16 <rrussell> Fix formatting
6fd9f70 <rrussell> remove inline edits
83a38c4 <rrussell> Try to fix italics
f130e04 <rrussell> Change wording a bit
718bd94 <rrussell> Fix unintended italics
7bab640 <rrussell> Refine indented bullet and fix title underline
26a53e4 <rrussell> Fix formatting
5fbea89 <rrussell> Add a note about keys and add polish
5365a9c <rrussell> Add helpful notes about Azure properties
6841116 <rrussell> Polish the formatting
eb643e4 <rrussell> ARROW-2066 Add documentation for Arrow/Azure/Parquet solution
---
 python/doc/source/parquet.rst | 41 +++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/python/doc/source/parquet.rst b/python/doc/source/parquet.rst
index ac56520ff4cee..3d01e1d3c1ad8 100644
--- a/python/doc/source/parquet.rst
+++ b/python/doc/source/parquet.rst
@@ -246,3 +246,44 @@ throughput:
 
    pq.read_table(where, nthreads=4)
    pq.ParquetDataset(where).read(nthreads=4)
+
+Reading a Parquet File from Azure Blob storage
+----------------------------------------------
+
+The code below shows how to use Azure's storage sdk along with pyarrow to read
+a parquet file into a Pandas dataframe.
+This is suitable for executing inside a Jupyter notebook running on a Python 3
+kernel.
+
+Dependencies: 
+
+* python 3.6.2 
+* azure-storage 0.36.0 
+* pyarrow 0.8.0 
+
+.. code-block:: python
+
+   import pyarrow.parquet as pq
+   from io import BytesIO
+   from azure.storage.blob import BlockBlobService
+
+   account_name = '...'
+   account_key = '...'
+   container_name = '...'
+   parquet_file = 'mysample.parquet'
+
+   byte_stream = io.BytesIO()
+   block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key)
+   try:
+      block_blob_service.get_blob_to_stream(container_name=container_name, blob_name=parquet_file, stream=byte_stream)
+      df = pq.read_table(source=byte_stream).to_pandas()
+      # Do work on df ...
+   finally:
+      # Add finally block to ensure closure of the stream
+      byte_stream.close()
+
+Notes:
+
+* The ``account_key`` can be found under ``Settings -> Access keys`` in the Microsoft Azure portal for a given container
+* The code above works for a container with private access, Lease State = Available, Lease Status = Unlocked
+* The parquet file was Blob Type = Block blob

From cca4a748f5fd66310a00840be8cd8fbb4606379c Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Fri, 23 Feb 2018 12:22:51 -0500
Subject: [PATCH 1522/1644] ARROW-2197: Document C++ ABI issue and workaround

Author: Antoine Pitrou <antoine@python.org>

Closes #1644 from pitrou/ARROW-2197-doc-undefined-symbol and squashes the following commits:

99db6e04 <Antoine Pitrou> ARROW-2197: Document C++ ABI issue and workaround
---
 python/doc/source/development.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index af93d8d1a52c4..e4078bdaff74b 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -246,6 +246,22 @@ To build a self-contained wheel (include Arrow C++ and Parquet C++), one can set
 Again, if you did not build parquet-cpp, you should omit ``--with-parquet`` and
 if you did not build with plasma, you should omit ``--with-plasma``.
 
+Known issues
+------------
+
+If using packages provided by conda-forge (see "Using Conda" above)
+together with a reasonably recent compiler, you may get "undefined symbol"
+errors when importing pyarrow.  In that case you'll need to force the C++
+ABI version to the older version used by conda-forge binaries:
+
+.. code-block:: shell
+
+   export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
+   export PYARROW_CXXFLAGS=$CXXFLAGS
+
+Be sure to add ``-DCMAKE_CXX_FLAGS=$CXXFLAGS`` to the cmake invocations
+when rebuilding.
+
 Developing on Windows
 =====================
 

From e2dd864ab8dac93c5b679565e3b82f929eedfc0a Mon Sep 17 00:00:00 2001
From: Panchen Xue <pan.panchen.xue@gmail.com>
Date: Sat, 24 Feb 2018 19:16:18 +0100
Subject: [PATCH 1523/1644] ARROW-2184: [C++]  Add static constructor for
 FileOutputStream returning shared_ptr to OutputStream

Add constructors to return pointers to the base interface

Author: Panchen Xue <pan.panchen.xue@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1642 from xuepanchen/ARROW-2184 and squashes the following commits:

43dc483 <Wes McKinney> Update Python bindings to use new FileOutputStream API
d093062 <Panchen Xue> Add test cases
f4e3b6a <Panchen Xue> Add static constructor for FileOutputStream returning pointer to base OutputStream
---
 cpp/src/arrow/io/file.cc             | 11 +++++++
 cpp/src/arrow/io/file.h              | 15 +++++++++
 cpp/src/arrow/io/io-file-test.cc     | 46 ++++++++++++++++++++++++++--
 python/pyarrow/_parquet.pyx          |  4 +--
 python/pyarrow/includes/libarrow.pxd |  2 +-
 python/pyarrow/io.pxi                |  5 +--
 6 files changed, 73 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 65a302c8beac2..d44d90cbe359e 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -554,6 +554,17 @@ FileOutputStream::~FileOutputStream() {
   DCHECK(impl_->Close().ok());
 }
 
+Status FileOutputStream::Open(const std::string& path,
+                              std::shared_ptr<OutputStream>* file) {
+  return Open(path, false, file);
+}
+
+Status FileOutputStream::Open(const std::string& path, bool append,
+                              std::shared_ptr<OutputStream>* out) {
+  *out = std::shared_ptr<FileOutputStream>(new FileOutputStream());
+  return std::static_pointer_cast<FileOutputStream>(*out)->impl_->Open(path, append);
+}
+
 Status FileOutputStream::Open(const std::string& path,
                               std::shared_ptr<FileOutputStream>* file) {
   return Open(path, false, file);
diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h
index 3d65834640dfe..a1f9edc1c66f9 100644
--- a/cpp/src/arrow/io/file.h
+++ b/cpp/src/arrow/io/file.h
@@ -39,6 +39,21 @@ class ARROW_EXPORT FileOutputStream : public OutputStream {
  public:
   ~FileOutputStream() override;
 
+  /// \brief Open a local file for writing, truncating any existing file
+  /// \param[in] path with UTF8 encoding
+  /// \param[out] out a base interface OutputStream instance
+  ///
+  /// When opening a new file, any existing file with the indicated path is
+  /// truncated to 0 bytes, deleting any existing memory
+  static Status Open(const std::string& path, std::shared_ptr<OutputStream>* out);
+
+  /// \brief Open a local file for writing
+  /// \param[in] path with UTF8 encoding
+  /// \param[in] append append to existing file, otherwise truncate to 0 bytes
+  /// \param[out] out a base interface OutputStream instance
+  static Status Open(const std::string& path, bool append,
+                     std::shared_ptr<OutputStream>* out);
+
   /// \brief Open a local file for writing, truncating any existing file
   /// \param[in] path with UTF8 encoding
   /// \param[out] file a FileOutputStream instance
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 7a7f396368265..2a4acab59db51 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -103,10 +103,12 @@ class TestFileOutputStream : public FileTestFixture {
  public:
   void OpenFile(bool append = false) {
     ASSERT_OK(FileOutputStream::Open(path_, append, &file_));
+    ASSERT_OK(FileOutputStream::Open(path_, append, &stream_));
   }
 
  protected:
   std::shared_ptr<FileOutputStream> file_;
+  std::shared_ptr<OutputStream> stream_;
 };
 
 #if defined(_MSC_VER)
@@ -116,6 +118,9 @@ TEST_F(TestFileOutputStream, FileNameWideCharConversionRangeException) {
   std::string file_name = "\x80";
   ASSERT_RAISES(Invalid, FileOutputStream::Open(file_name, &file));
 
+  std::shared_ptr<OutputStream> stream;
+  ASSERT_RAISES(Invalid, FileOutputStream::Open(file_name, &stream));
+
   std::shared_ptr<ReadableFile> rd_file;
   ASSERT_RAISES(Invalid, ReadableFile::Open(file_name, &rd_file));
 }
@@ -129,6 +134,12 @@ TEST_F(TestFileOutputStream, DestructorClosesFile) {
     fd = file->file_descriptor();
   }
   ASSERT_TRUE(FileIsClosed(fd));
+  {
+    std::shared_ptr<OutputStream> stream;
+    ASSERT_OK(FileOutputStream::Open(path_, &stream));
+    fd = std::static_pointer_cast<FileOutputStream>(stream)->file_descriptor();
+  }
+  ASSERT_TRUE(FileIsClosed(fd));
 }
 
 TEST_F(TestFileOutputStream, Close) {
@@ -139,7 +150,6 @@ TEST_F(TestFileOutputStream, Close) {
 
   int fd = file_->file_descriptor();
   ASSERT_OK(file_->Close());
-
   ASSERT_TRUE(FileIsClosed(fd));
 
   // Idempotent
@@ -151,6 +161,19 @@ TEST_F(TestFileOutputStream, Close) {
   int64_t size = 0;
   ASSERT_OK(rd_file->GetSize(&size));
   ASSERT_EQ(strlen(data), size);
+
+  ASSERT_OK(stream_->Write(data, strlen(data)));
+
+  fd = std::static_pointer_cast<FileOutputStream>(stream_)->file_descriptor();
+  ASSERT_OK(stream_->Close());
+  ASSERT_TRUE(FileIsClosed(fd));
+
+  // Idempotent
+  ASSERT_OK(stream_->Close());
+
+  ASSERT_OK(ReadableFile::Open(path_, &rd_file));
+  ASSERT_OK(rd_file->GetSize(&size));
+  ASSERT_EQ(strlen(data), size);
 }
 
 TEST_F(TestFileOutputStream, InvalidWrites) {
@@ -159,13 +182,13 @@ TEST_F(TestFileOutputStream, InvalidWrites) {
   const char* data = "";
 
   ASSERT_RAISES(IOError, file_->Write(data, -1));
+  ASSERT_RAISES(IOError, stream_->Write(data, -1));
 }
 
 TEST_F(TestFileOutputStream, Tell) {
   OpenFile();
 
   int64_t position;
-
   ASSERT_OK(file_->Tell(&position));
   ASSERT_EQ(0, position);
 
@@ -173,6 +196,13 @@ TEST_F(TestFileOutputStream, Tell) {
   ASSERT_OK(file_->Write(data, 8));
   ASSERT_OK(file_->Tell(&position));
   ASSERT_EQ(8, position);
+
+  ASSERT_OK(stream_->Tell(&position));
+  ASSERT_EQ(0, position);
+
+  ASSERT_OK(stream_->Write(data, 8));
+  ASSERT_OK(stream_->Tell(&position));
+  ASSERT_EQ(8, position);
 }
 
 TEST_F(TestFileOutputStream, TruncatesNewFile) {
@@ -191,6 +221,18 @@ TEST_F(TestFileOutputStream, TruncatesNewFile) {
   int64_t size;
   ASSERT_OK(rd_file->GetSize(&size));
   ASSERT_EQ(0, size);
+
+  ASSERT_OK(FileOutputStream::Open(path_, &stream_));
+
+  ASSERT_OK(stream_->Write(data, strlen(data)));
+  ASSERT_OK(stream_->Close());
+
+  ASSERT_OK(FileOutputStream::Open(path_, &stream_));
+  ASSERT_OK(stream_->Close());
+
+  ASSERT_OK(ReadableFile::Open(path_, &rd_file));
+  ASSERT_OK(rd_file->GetSize(&size));
+  ASSERT_EQ(0, size);
 }
 
 // ----------------------------------------------------------------------
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 147af217579e7..9061ed53dac2b 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -821,7 +821,6 @@ cdef class ParquetWriter:
                   use_deprecated_int96_timestamps=False,
                   coerce_timestamps=None):
         cdef:
-            shared_ptr[FileOutputStream] filestream
             shared_ptr[WriterProperties] properties
             c_string c_where
             CMemoryPool* pool
@@ -830,8 +829,7 @@ cdef class ParquetWriter:
             c_where = tobytes(where)
             with nogil:
                 check_status(FileOutputStream.Open(c_where,
-                                                   &filestream))
-            self.sink = <shared_ptr[OutputStream]> filestream
+                                                   &self.sink))
             self.own_sink = True
         else:
             get_writer(where, &self.sink)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 81cc4d2761e2d..8da126aafd586 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -539,7 +539,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil:
 
     cdef cppclass FileOutputStream(OutputStream):
         @staticmethod
-        CStatus Open(const c_string& path, shared_ptr[FileOutputStream]* file)
+        CStatus Open(const c_string& path, shared_ptr[OutputStream]* file)
 
         int file_descriptor()
 
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index aa2f7ed07a2bc..8b364dc7163a5 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -550,12 +550,9 @@ cdef class OSFile(NativeFile):
         self.rd_file = <shared_ptr[RandomAccessFile]> handle
 
     cdef _open_writable(self, c_string path):
-        cdef shared_ptr[FileOutputStream] handle
-
         with nogil:
-            check_status(FileOutputStream.Open(path, &handle))
+            check_status(FileOutputStream.Open(path, &self.wr_file))
         self.is_writable = True
-        self.wr_file = <shared_ptr[OutputStream]> handle
 
 
 cdef class FixedSizeBufferWriter(NativeFile):

From c0b0e3300ee9747e9eb781b517f0b3dc7c661ada Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sat, 24 Feb 2018 18:55:49 -0500
Subject: [PATCH 1524/1644] =?UTF-8?q?ARROW-2191:=20[C++]=C2=A0Only=20use?=
 =?UTF-8?q?=20specific=20version=20of=20jemalloc?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1633 from xhochy/ARROW-2191 and squashes the following commits:

5f67465 <Uwe L. Korn> ARROW-2191:  Only use specific version of jemalloc
---
 ci/travis_install_toolchain.sh              |  1 -
 cpp/CMakeLists.txt                          | 52 +++++++--------------
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 52 ++++++++++-----------
 cpp/src/arrow/CMakeLists.txt                |  4 ++
 cpp/src/arrow/memory_pool.cc                |  3 +-
 5 files changed, 48 insertions(+), 64 deletions(-)

diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh
index 60cdc36a2efff..b5eb137ba9168 100755
--- a/ci/travis_install_toolchain.sh
+++ b/ci/travis_install_toolchain.sh
@@ -24,7 +24,6 @@ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 if [ ! -e $CPP_TOOLCHAIN ]; then
     # Set up C++ toolchain from conda-forge packages for faster builds
     conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
-        jemalloc=4.5.0.post \
         nomkl \
         boost-cpp \
         rapidjson \
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 42c1ec8f0405a..b9221414bf657 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -127,10 +127,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the Arrow jemalloc-based allocator"
     OFF)
 
-  option(ARROW_JEMALLOC_USE_SHARED
-    "Rely on jemalloc shared libraries where relevant"
-    ON)
-
   option(ARROW_HDFS
     "Build the Arrow HDFS bridge"
     ON)
@@ -605,44 +601,28 @@ set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 if (ARROW_JEMALLOC)
   add_definitions(-DARROW_JEMALLOC)
-  # In the case that jemalloc is only available as a shared library also use it to
-  # link it in the static requirements. In contrast to other libraries we try in
-  # most cases to use the system provided version of jemalloc to better align with
-  # other potential users of jemalloc.
-  if (JEMALLOC_STATIC_LIB AND NOT ARROW_JEMALLOC_USE_SHARED)
-    set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_static)
-  else()
-    set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_shared)
-  endif()
-
-  if (NOT APPLE)
-      set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} ${PTHREAD_LIBRARY} rt)
-  endif()
-
-  if (ARROW_JEMALLOC_USE_SHARED)
-    set(ARROW_JEMALLOC_SHARED_LINK_LIBS jemalloc_shared)
+  add_definitions(-DARROW_JEMALLOC_INCLUDE_DIR=${JEMALLOC_INCLUDE_DIR})
+
+  if (CMAKE_COMPILER_IS_GNUCXX)
+    set(ARROW_JEMALLOC_LINK_LIBS
+      jemalloc_static
+      # For glibc <2.17 we need to link to librt.
+      # As we compile with --as-needed by default, the linker will omit this
+      # dependency if not required.
+      ${PTHREAD_LIBRARY}
+      rt
+      )
   else()
-    if (CMAKE_COMPILER_IS_GNUCXX)
-      set(ARROW_JEMALLOC_SHARED_LINK_LIBS
-        jemalloc_static
-        # For glibc <2.17 we need to link to librt.
-        # As we compile with --as-needed by default, the linker will omit this
-        # dependency if not required.
-        ${PTHREAD_LIBRARY}
-        rt
-        )
-    else()
-      set(ARROW_JEMALLOC_SHARED_LINK_LIBS
-        jemalloc_static
-        )
-    endif()
+    set(ARROW_JEMALLOC_LINK_LIBS
+      jemalloc_static
+      )
   endif()
   set(ARROW_SHARED_PRIVATE_LINK_LIBS
     ${ARROW_SHARED_PRIVATE_LINK_LIBS}
-    ${ARROW_JEMALLOC_SHARED_LINK_LIBS})
+    ${ARROW_JEMALLOC_LINK_LIBS})
   set(ARROW_STATIC_LINK_LIBS
     ${ARROW_STATIC_LINK_LIBS}
-    ${ARROW_JEMALLOC_STATIC_LINK_LIBS})
+    ${ARROW_JEMALLOC_LINK_LIBS})
 elseif (NOT MSVC)
   # We need to separate this as otherwise CMake would mess with the library
   # linking order.
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 3511d40d42ce6..e64284cf06479 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -82,10 +82,6 @@ if (DEFINED ENV{RAPIDJSON_HOME})
   set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}")
 endif()
 
-if (DEFINED ENV{JEMALLOC_HOME})
-  set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}")
-endif()
-
 if (DEFINED ENV{GFLAGS_HOME})
   set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
 endif()
@@ -474,33 +470,37 @@ if (MSVC)
 endif()
 
 if (ARROW_JEMALLOC)
-  find_package(jemalloc)
-
-  if(NOT JEMALLOC_FOUND)
-    set(ARROW_JEMALLOC_USE_SHARED OFF)
-    set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
-    set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
-    set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
-    set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
-    set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(JEMALLOC_VENDORED 1)
-    ExternalProject_Add(jemalloc_ep
-      URL ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
-      CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_" && touch doc/jemalloc.html && touch doc/jemalloc.3
-      ${EP_LOG_OPTIONS}
-      BUILD_IN_SOURCE 1
-      BUILD_COMMAND ${MAKE}
-      BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
-      INSTALL_COMMAND ${MAKE} -j1 install)
-  else()
-    set(JEMALLOC_VENDORED 0)
-  endif()
+  # We only use a vendored jemalloc as we want to control its version.
+  # Also our build of jemalloc is specially prefixed so that it will not 
+  # conflict with the default allocator as well as other jemalloc
+  # installations.
+  # find_package(jemalloc)
+
+  set(ARROW_JEMALLOC_USE_SHARED OFF)
+  set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
+  set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
+  set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
+  set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(JEMALLOC_VENDORED 1)
+  ExternalProject_Add(jemalloc_ep
+    URL ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
+    PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
+    CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
+    ${EP_LOG_OPTIONS}
+    BUILD_IN_SOURCE 1
+    BUILD_COMMAND ${MAKE}
+    BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
+    INSTALL_COMMAND ${MAKE} -j1 install)
 
-  include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
+  # Don't use the include directory directly so that we can point to a path
+  # that is unique to our codebase.
+  include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/")
   ADD_THIRDPARTY_LIB(jemalloc
     STATIC_LIB ${JEMALLOC_STATIC_LIB}
     SHARED_LIB ${JEMALLOC_SHARED_LIB}
     DEPS ${PTHREAD_LIBRARY})
+  add_dependencies(jemalloc_static jemalloc_ep)
 endif()
 
 ## Google PerfTools
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 74674bebb43be..5fd9256702448 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -65,6 +65,10 @@ if (ARROW_GPU)
   add_subdirectory(gpu)
 endif()
 
+if (ARROW_JEMALLOC AND JEMALLOC_VENDORED)
+  add_dependencies(arrow_dependencies jemalloc_static)
+endif()
+
 if (ARROW_WITH_BROTLI)
   add_definitions(-DARROW_WITH_BROTLI)
   SET(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS})
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 99530432c92cc..b8cd9c2fcf9a4 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -31,7 +31,8 @@
 #ifdef ARROW_JEMALLOC
 // Needed to support jemalloc 3 and 4
 #define JEMALLOC_MANGLE
-#include <jemalloc/jemalloc.h>
+// Explicitly link to our version of jemalloc
+#include "jemalloc_ep/dist/include/jemalloc/jemalloc.h"
 #endif
 
 namespace arrow {

From 2fd8f0ac00ed810fd46090dad95874e33dcb5d49 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 25 Feb 2018 12:10:04 +0100
Subject: [PATCH 1525/1644] ARROW-2204: Fix TLS errors in manylinux1 build

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1655 from xhochy/ARROW-2204 and squashes the following commits:

1a21f17 [Uwe L. Korn] ARROW-2204: Fix TLS errors in manylinux1 build
---
 python/manylinux1/Dockerfile-x86_64           |  2 +-
 python/manylinux1/Dockerfile-x86_64_base      |  3 --
 python/manylinux1/scripts/build_brotli.sh     |  2 +-
 .../manylinux1/scripts/build_flatbuffers.sh   |  2 +-
 python/manylinux1/scripts/build_gtest.sh      |  2 +-
 python/manylinux1/scripts/build_jemalloc.sh   | 31 -------------------
 python/manylinux1/scripts/build_lz4.sh        |  2 +-
 python/manylinux1/scripts/build_snappy.sh     |  2 +-
 python/manylinux1/scripts/build_zstd.sh       |  2 +-
 9 files changed, 7 insertions(+), 41 deletions(-)
 delete mode 100755 python/manylinux1/scripts/build_jemalloc.sh

diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index ec520338ff2fe..f2b64e60b0159 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2143
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2204
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index 1f15f77d8ddc7..4dfb131c73bc8 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -25,9 +25,6 @@ RUN /build_openssl.sh
 ADD scripts/build_boost.sh /
 RUN /build_boost.sh
 
-ADD scripts/build_jemalloc.sh /
-RUN /build_jemalloc.sh
-
 # Install cmake manylinux1 package
 ADD scripts/install_cmake.sh /
 RUN /install_cmake.sh
diff --git a/python/manylinux1/scripts/build_brotli.sh b/python/manylinux1/scripts/build_brotli.sh
index 9a1eca7b7809f..c2659d2201cbb 100755
--- a/python/manylinux1/scripts/build_brotli.sh
+++ b/python/manylinux1/scripts/build_brotli.sh
@@ -17,7 +17,7 @@
 # under the License.
 
 export BROTLI_VERSION="0.6.0"
-wget "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -O brotli-${BROTLI_VERSION}.tar.gz
+curl -sL "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -o brotli-${BROTLI_VERSION}.tar.gz
 tar xf brotli-${BROTLI_VERSION}.tar.gz
 pushd brotli-${BROTLI_VERSION}
 mkdir build
diff --git a/python/manylinux1/scripts/build_flatbuffers.sh b/python/manylinux1/scripts/build_flatbuffers.sh
index 683a89ce5c474..509ec41deacc3 100755
--- a/python/manylinux1/scripts/build_flatbuffers.sh
+++ b/python/manylinux1/scripts/build_flatbuffers.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-wget https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -O flatbuffers-1.6.0.tar.gz
+curl -sL https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -o flatbuffers-1.6.0.tar.gz
 tar xf flatbuffers-1.6.0.tar.gz
 pushd flatbuffers-1.6.0
 cmake "-DCMAKE_CXX_FLAGS=-fPIC" "-DCMAKE_INSTALL_PREFIX:PATH=/usr" "-DFLATBUFFERS_BUILD_TESTS=OFF"
diff --git a/python/manylinux1/scripts/build_gtest.sh b/python/manylinux1/scripts/build_gtest.sh
index 4ce20c1fb44b3..f921efd489d67 100755
--- a/python/manylinux1/scripts/build_gtest.sh
+++ b/python/manylinux1/scripts/build_gtest.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -O googletest-release-1.7.0.tar.gz
+curl -sL https://github.com/google/googletest/archive/release-1.7.0.tar.gz -o googletest-release-1.7.0.tar.gz
 tar xf googletest-release-1.7.0.tar.gz
 ls -l
 pushd googletest-release-1.7.0
diff --git a/python/manylinux1/scripts/build_jemalloc.sh b/python/manylinux1/scripts/build_jemalloc.sh
deleted file mode 100755
index 370bdcd4b4bcc..0000000000000
--- a/python/manylinux1/scripts/build_jemalloc.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash -ex
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-wget https://github.com/jemalloc/jemalloc/archive/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz -O jemalloc.tar.gz
-tar xf jemalloc.tar.gz
-mv jemalloc-* jemalloc
-pushd /jemalloc
-./autogen.sh
-./configure "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
-# Skip doc generation
-touch doc/jemalloc.html
-touch doc/jemalloc.3
-make -j5
-make install
-popd
-rm -rf jemalloc.tar.gz jemalloc
diff --git a/python/manylinux1/scripts/build_lz4.sh b/python/manylinux1/scripts/build_lz4.sh
index 8242a5fe27e53..4c77061f07c2f 100755
--- a/python/manylinux1/scripts/build_lz4.sh
+++ b/python/manylinux1/scripts/build_lz4.sh
@@ -20,7 +20,7 @@ export LZ4_VERSION="1.7.5"
 export PREFIX="/usr"
 export CFLAGS="${CFLAGS} -O3 -fPIC"
 export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib -L${PREFIX}/lib"
-wget "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -O lz4-${LZ4_VERSION}.tar.gz
+curl -sL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -o lz4-${LZ4_VERSION}.tar.gz
 tar xf lz4-${LZ4_VERSION}.tar.gz
 pushd lz4-${LZ4_VERSION}
 
diff --git a/python/manylinux1/scripts/build_snappy.sh b/python/manylinux1/scripts/build_snappy.sh
index 5392e14a33a45..bc3afe62e8e8d 100755
--- a/python/manylinux1/scripts/build_snappy.sh
+++ b/python/manylinux1/scripts/build_snappy.sh
@@ -17,7 +17,7 @@
 # under the License.
 
 export SNAPPY_VERSION="1.1.3"
-wget "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz" -O snappy-${SNAPPY_VERSION}.tar.gz
+curl -sL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz" -o snappy-${SNAPPY_VERSION}.tar.gz
 tar xf snappy-${SNAPPY_VERSION}.tar.gz
 pushd snappy-${SNAPPY_VERSION}
 ./configure --with-pic "--prefix=/usr" CXXFLAGS='-DNDEBUG -O2'
diff --git a/python/manylinux1/scripts/build_zstd.sh b/python/manylinux1/scripts/build_zstd.sh
index ef0e267757eab..ccfda3f399c31 100755
--- a/python/manylinux1/scripts/build_zstd.sh
+++ b/python/manylinux1/scripts/build_zstd.sh
@@ -20,7 +20,7 @@ export ZSTD_VERSION="1.2.0"
 export CFLAGS="${CFLAGS} -O3 -fPIC"
 export PREFIX="/usr"
 export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib"
-wget "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -O zstd-${ZSTD_VERSION}.tar.gz
+curl -sL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -o zstd-${ZSTD_VERSION}.tar.gz
 tar xf zstd-${ZSTD_VERSION}.tar.gz
 pushd zstd-${ZSTD_VERSION}
 

From 27d83392ecb763c7ae0f3775a6c6582377998522 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 14:11:07 +0100
Subject: [PATCH 1526/1644] ARROW-2214: [JS] add nullBitmap getter to
 DictionaryData that proxies to its indices' nullBitmap

Resolves https://issues.apache.org/jira/browse/ARROW-2214

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1659 from trxcllnt/ARROW-2214 and squashes the following commits:

8c4ba46 <Paul Taylor> add nullBitmap getter to DictionaryData that proxies to its indices' nullBitmap
---
 js/src/data.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/js/src/data.ts b/js/src/data.ts
index ab2e40ae9a912..d49fb790b9f26 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -158,6 +158,7 @@ export class DictionaryData<T extends DataType> extends BaseData<Dictionary<T>>
         this.length = this._indicies.length;
     }
     public get nullCount() { return this._indicies.nullCount; }
+    public get nullBitmap() { return this._indicies.nullBitmap; }
     public clone<R extends Dictionary<T>>(type: R, length = this.length, offset = this.offset) {
         const data = this._dictionary.data.clone(type.dictionary as any);
         return new DictionaryData<R>(

From 655eb74e7a275b5861246baa0641851df3ca2ffd Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 26 Feb 2018 10:54:12 -0500
Subject: [PATCH 1527/1644] =?UTF-8?q?ARROW-2212:=20[C++/Python]=C2=A0Build?=
 =?UTF-8?q?=20Protobuf=20in=20base=20manylinux=201=20docker=20image?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Saves ~6min build time.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1662 from xhochy/ARROW-2212 and squashes the following commits:

acf2319d <Uwe L. Korn> ARROW-2212:  Build Protobuf in base manylinux1 docker image
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake |  4 +++
 python/manylinux1/Dockerfile-x86_64         |  2 +-
 python/manylinux1/Dockerfile-x86_64_base    |  4 +++
 python/manylinux1/scripts/build_protobuf.sh | 28 +++++++++++++++++++++
 4 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100755 python/manylinux1/scripts/build_protobuf.sh

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index e64284cf06479..944ca1d3bb1c2 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -110,6 +110,10 @@ if (DEFINED ENV{GRPC_HOME})
   set(GRPC_HOME "$ENV{GRPC_HOME}")
 endif()
 
+if (DEFINED ENV{PROTOBUF_HOME})
+  set(PROTOBUF_HOME "$ENV{PROTOBUF_HOME}")
+endif()
+
 # Ensure that a default make is set
 if ("${MAKE}" STREQUAL "")
     if (NOT MSVC)
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index f2b64e60b0159..62a0893292cbd 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2204
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2212
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index 4dfb131c73bc8..955abdd937a0f 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -63,6 +63,10 @@ ENV ZSTD_HOME /usr
 ADD scripts/build_ccache.sh /
 RUN /build_ccache.sh
 
+ADD scripts/build_protobuf.sh /
+RUN /build_protobuf.sh
+ENV PROTOBUF_HOME /usr
+
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
diff --git a/python/manylinux1/scripts/build_protobuf.sh b/python/manylinux1/scripts/build_protobuf.sh
new file mode 100755
index 0000000000000..c3ea8007fc052
--- /dev/null
+++ b/python/manylinux1/scripts/build_protobuf.sh
@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+PROTOBUF_VERSION="2.6.0"
+
+curl -sL https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz -o protobuf-${PROTOBUF_VERSION}.tar.gz
+tar xf protobuf-${PROTOBUF_VERSION}.tar.gz
+pushd protobuf-${PROTOBUF_VERSION}
+./configure --disable-shared --prefix=/usr "CXXFLAGS=-O2 -fPIC"
+make -j5
+make install
+popd
+rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION}

From 5521bcf1af07a539ef1bf632b49c9d4089ecad63 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 26 Feb 2018 11:54:24 -0500
Subject: [PATCH 1528/1644] ARROW-2094: [C++] Install libprotobuf and set
 PROTOBUF_HOME when using toolchain

C++-only libprotobuf is now being provided by conda-forge

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1664 from wesm/ARROW-2094 and squashes the following commits:

cec6c887 <Wes McKinney> Install libprotobuf and set PROTOBUF_HOME when using toolchain
---
 ci/travis_env_common.sh        | 3 +++
 ci/travis_install_toolchain.sh | 1 +
 2 files changed, 4 insertions(+)

diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index 21b6e266ea678..416977c1301a9 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -47,6 +47,9 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
   export ARROW_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
   export BOOST_ROOT=$CPP_TOOLCHAIN
 
+  # Protocol buffers used by Apache ORC thirdparty build
+  export PROTOBUF_HOME=$CPP_TOOLCHAIN
+
   export PATH=$CPP_TOOLCHAIN/bin:$PATH
   export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
   export TRAVIS_MAKE=ninja
diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh
index b5eb137ba9168..31076b2e736f9 100755
--- a/ci/travis_install_toolchain.sh
+++ b/ci/travis_install_toolchain.sh
@@ -26,6 +26,7 @@ if [ ! -e $CPP_TOOLCHAIN ]; then
     conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
         nomkl \
         boost-cpp \
+        libprotobuf \
         rapidjson \
         flatbuffers \
         gflags \

From 564fefe68b6d7f255cc0712f446f09bf5b0908a2 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 14:14:59 -0500
Subject: [PATCH 1529/1644] ARROW-2213: [JS] fix npm release

This should resolve https://issues.apache.org/jira/browse/ARROW-2213

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1658 from trxcllnt/js-fix-npm-release and squashes the following commits:

71273fa9 <Paul Taylor> specify unpkg key in apache-arrow module's package.json
fd4d5be8 <Paul Taylor> npm publish via lerna exec
---
 js/gulp/package-task.js | 5 ++---
 js/npm-release.sh       | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index c42b3fc323321..3390da01923a8 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -47,8 +47,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
     main: mainExport,
     types: `${mainExport}.d.ts`,
     module: `${mainExport}.mjs`,
-    dist: `${mainExport}.es5.min.js`,
-    [`dist:es2015`]: `${mainExport}.es2015.min.js`,
+    unpkg: `${mainExport}.es5.min.js`,
     [`@std/esm`]: { esm: `mjs`, warnings: false, sourceMap: true }
 });
   
@@ -69,7 +68,7 @@ const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
             {
                 name: `${npmOrgName}/${packageName(target, format)}`,
                 version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`,
-                dist: undefined, [`dist:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined
+                unpkg: undefined, module: undefined, [`@std/esm`]: undefined
             }
         )
     )
diff --git a/js/npm-release.sh b/js/npm-release.sh
index 42cd73c0cfe59..3ef24d3e6f828 100755
--- a/js/npm-release.sh
+++ b/js/npm-release.sh
@@ -27,4 +27,4 @@ npx run-s clean:all lint build
 npm run test
 
 # publish the JS target modules to npm
-npx lerna publish --yes --skip-git --force-publish=*
+npx lerna exec -- npm publish

From 8c493cdcf8a3f70369d6c6ab73561bb0fed46be8 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 14:38:09 -0500
Subject: [PATCH 1530/1644] ARROW-2219: [JS] rename indicies to indices

@TheNeuralBit I think this is my bad as "indicies" isn't a real word. what do you think, is this PR worth it? should we name this "keys" instead?

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1666 from trxcllnt/js-fix-indicies-typo and squashes the following commits:

711571a <Paul Taylor> fix typo -- rename indicies to indices
---
 js/src/Arrow.externs.js     |  2 +-
 js/src/data.ts              | 22 +++++++++++-----------
 js/src/ipc/reader/vector.ts |  2 +-
 js/src/table.ts             |  4 ++--
 js/src/type.ts              | 12 ++++++------
 js/src/vector.ts            | 12 ++++++------
 js/src/vector/dictionary.ts | 24 ++++++++++++------------
 7 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index de1e6539285bc..cf4db9134b9fa 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -447,7 +447,7 @@ FlatListData.prototype.valueOffsets;
 
 var DictionaryData = function() {};
 /** @type {?} */
-DictionaryData.prototype.indicies;
+DictionaryData.prototype.indices;
 /** @type {?} */
 DictionaryData.prototype.dictionary;
 
diff --git a/js/src/data.ts b/js/src/data.ts
index d49fb790b9f26..cdd9f29a7a908 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -148,28 +148,28 @@ export class FlatListData<T extends FlatListType> extends FlatData<T> {
 
 export class DictionaryData<T extends DataType> extends BaseData<Dictionary<T>> {
     protected _dictionary: Vector<T>;
-    protected _indicies: Data<Int<any>>;
-    public get indicies() { return this._indicies; }
+    protected _indices: Data<Int<any>>;
+    public get indices() { return this._indices; }
     public get dictionary() { return this._dictionary; }
-    constructor(type: Dictionary<T>, dictionary: Vector<T>, indicies: Data<Int<any>>) {
-        super(type, indicies.length, (indicies as any)._nullCount);
-        this._indicies = indicies;
+    constructor(type: Dictionary<T>, dictionary: Vector<T>, indices: Data<Int<any>>) {
+        super(type, indices.length, (indices as any)._nullCount);
+        this._indices = indices;
         this._dictionary = dictionary;
-        this.length = this._indicies.length;
+        this.length = this._indices.length;
     }
-    public get nullCount() { return this._indicies.nullCount; }
-    public get nullBitmap() { return this._indicies.nullBitmap; }
+    public get nullCount() { return this._indices.nullCount; }
+    public get nullBitmap() { return this._indices.nullBitmap; }
     public clone<R extends Dictionary<T>>(type: R, length = this.length, offset = this.offset) {
         const data = this._dictionary.data.clone(type.dictionary as any);
         return new DictionaryData<R>(
             this.type as any,
             this._dictionary.clone(data) as any,
-            this._indicies.slice(offset - this.offset, length)
+            this._indices.slice(offset - this.offset, length)
         ) as any;
     }
     protected sliceInternal(clone: this, _offset: number, _length: number) {
-        clone.length = clone._indicies.length;
-        clone._nullCount = (clone._indicies as any)._nullCount;
+        clone.length = clone._indices.length;
+        clone._nullCount = (clone._indices as any)._nullCount;
         return clone;
     }
 }
diff --git a/js/src/ipc/reader/vector.ts b/js/src/ipc/reader/vector.ts
index 809069c6d9864..b8c4871ebac2f 100644
--- a/js/src/ipc/reader/vector.ts
+++ b/js/src/ipc/reader/vector.ts
@@ -92,7 +92,7 @@ export abstract class TypeDataLoader extends TypeVisitor {
     public visitFixedSizeList  (type: FixedSizeList)   { return this.visitFixedSizeListType(type); }
     public visitMap            (type: Map_)            { return this.visitNestedType(type); }
     public visitDictionary     (type: Dictionary)      {
-        return new DictionaryData(type, this.dictionaries.get(type.id)!, this.visit(type.indicies));
+        return new DictionaryData(type, this.dictionaries.get(type.id)!, this.visit(type.indices));
     }
     protected getFieldMetadata() { return this.nodes.next().value; }
     protected getBufferMetadata() { return this.buffers.next().value; }
diff --git a/js/src/table.ts b/js/src/table.ts
index 3e50d16e3724d..d0d699f366e44 100644
--- a/js/src/table.ts
+++ b/js/src/table.ts
@@ -160,7 +160,7 @@ export class Table implements DataFrame {
             const batch = batches[batchIndex];
             // rebind the countBy Col
             count_by.bind(batch);
-            const keys = (count_by.vector as DictionaryVector).indicies;
+            const keys = (count_by.vector as DictionaryVector).indices;
             // yield all indices
             for (let index = -1, numRows = batch.length; ++index < numRows;) {
                 let key = keys.get(index);
@@ -258,7 +258,7 @@ class FilteredDataFrame implements DataFrame {
             const predicate = this.predicate.bind(batch);
             // rebind the countBy Col
             count_by.bind(batch);
-            const keys = (count_by.vector as DictionaryVector).indicies;
+            const keys = (count_by.vector as DictionaryVector).indices;
             // yield all indices
             for (let index = -1, numRows = batch.length; ++index < numRows;) {
                 let key = keys.get(index);
diff --git a/js/src/type.ts b/js/src/type.ts
index 5e6c939ef67ad..370be0def0c99 100644
--- a/js/src/type.ts
+++ b/js/src/type.ts
@@ -83,8 +83,8 @@ export class Field<T extends DataType = DataType> {
     public toString() { return `${this.name}: ${this.type}`; }
     public get typeId(): T['TType'] { return this.type.TType; }
     public get [Symbol.toStringTag](): string { return 'Field'; }
-    public get indicies(): T | Int<any> {
-        return DataType.isDictionary(this.type) ? this.type.indicies : this.type;
+    public get indices(): T | Int<any> {
+        return DataType.isDictionary(this.type) ? this.type.indices : this.type;
     }
 }
 
@@ -443,17 +443,17 @@ export interface Dictionary<T extends DataType = any> extends DataType<Type.Dict
 export class Dictionary<T extends DataType> extends DataType<Type.Dictionary> {
     public readonly id: number;
     public readonly dictionary: T;
-    public readonly indicies: Int<any>;
+    public readonly indices: Int<any>;
     public readonly isOrdered: boolean;
-    constructor(dictionary: T, indicies: Int<any>, id?: Long | number | null, isOrdered?: boolean | null) {
+    constructor(dictionary: T, indices: Int<any>, id?: Long | number | null, isOrdered?: boolean | null) {
         super(Type.Dictionary);
-        this.indicies = indicies;
+        this.indices = indices;
         this.dictionary = dictionary;
         this.isOrdered = isOrdered || false;
         this.id = id == null ? DictionaryBatch.getId() : typeof id === 'number' ? id : id.low;
     }
     public get ArrayType() { return this.dictionary.ArrayType; }
-    public toString() { return `Dictionary<${this.indicies}, ${this.dictionary}>`; }
+    public toString() { return `Dictionary<${this.indices}, ${this.dictionary}>`; }
     protected static [Symbol.toStringTag] = ((proto: Dictionary) => {
         return proto[Symbol.toStringTag] = 'Dictionary';
     })(Dictionary.prototype);
diff --git a/js/src/vector.ts b/js/src/vector.ts
index fa1d16efcba72..f36c691e1bd27 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -394,29 +394,29 @@ export class UnionVector<T extends (SparseUnion | DenseUnion) = any> extends Nes
 
 export class DictionaryVector<T extends DataType = DataType> extends Vector<Dictionary<T>> {
     // @ts-ignore
-    public readonly indicies: Vector<Int>;
+    public readonly indices: Vector<Int>;
     // @ts-ignore
     public readonly dictionary: Vector<T>;
-    constructor(data: Data<Dictionary<T>>, view: View<Dictionary<T>> = new DictionaryView<T>(data.dictionary, new IntVector(data.indicies))) {
+    constructor(data: Data<Dictionary<T>>, view: View<Dictionary<T>> = new DictionaryView<T>(data.dictionary, new IntVector(data.indices))) {
         super(data as Data<any>, view);
         if (data instanceof DictionaryData && view instanceof DictionaryView) {
-            this.indicies = view.indicies;
+            this.indices = view.indices;
             this.dictionary = data.dictionary;
         } else if (data instanceof ChunkedData && view instanceof ChunkedView) {
             const chunks = view.chunkVectors as DictionaryVector<T>[];
             // Assume the last chunk's dictionary data is the most up-to-date,
             // including data from DictionaryBatches that were marked as deltas
             this.dictionary = chunks[chunks.length - 1].dictionary;
-            this.indicies = chunks.reduce<Vector<Int> | null>(
+            this.indices = chunks.reduce<Vector<Int> | null>(
                 (idxs: Vector<Int> | null, dict: DictionaryVector<T>) =>
-                    !idxs ? dict.indicies! : idxs.concat(dict.indicies!),
+                    !idxs ? dict.indices! : idxs.concat(dict.indices!),
                 null
             )!;
         } else {
             throw new TypeError(`Unrecognized DictionaryVector view`);
         }
     }
-    public getKey(index: number) { return this.indicies.get(index); }
+    public getKey(index: number) { return this.indices.get(index); }
     public getValue(key: number) { return this.dictionary.get(key); }
     public reverseLookup(value: T) { return this.dictionary.indexOf(value); }
 }
diff --git a/js/src/vector/dictionary.ts b/js/src/vector/dictionary.ts
index f4de810b015db..21f9bac0993e0 100644
--- a/js/src/vector/dictionary.ts
+++ b/js/src/vector/dictionary.ts
@@ -20,31 +20,31 @@ import { View, Vector } from '../vector';
 import { IterableArrayLike, DataType, Dictionary, Int } from '../type';
 
 export class DictionaryView<T extends DataType> implements View<T> {
-    public indicies: Vector<Int>;
+    public indices: Vector<Int>;
     public dictionary: Vector<T>;
-    constructor(dictionary: Vector<T>, indicies: Vector<Int>) {
-        this.indicies = indicies;
+    constructor(dictionary: Vector<T>, indices: Vector<Int>) {
+        this.indices = indices;
         this.dictionary = dictionary;
     }
     public clone(data: Data<Dictionary<T>>): this {
-        return new DictionaryView(data.dictionary, this.indicies.clone(data.indicies)) as this;
+        return new DictionaryView(data.dictionary, this.indices.clone(data.indices)) as this;
     }
     public isValid(index: number): boolean {
-        return this.indicies.isValid(index);
+        return this.indices.isValid(index);
     }
     public get(index: number): T['TValue'] {
-        return this.dictionary.get(this.indicies.get(index));
+        return this.dictionary.get(this.indices.get(index));
     }
     public set(index: number, value: T['TValue']): void {
-        this.dictionary.set(this.indicies.get(index), value);
+        this.dictionary.set(this.indices.get(index), value);
     }
     public toArray(): IterableArrayLike<T['TValue']> {
         return [...this];
     }
     public *[Symbol.iterator](): IterableIterator<T['TValue']> {
-        const values = this.dictionary, indicies = this.indicies;
-        for (let index = -1, n = indicies.length; ++index < n;) {
-            yield values.get(indicies.get(index));
+        const values = this.dictionary, indices = this.indices;
+        for (let index = -1, n = indices.length; ++index < n;) {
+            yield values.get(indices.get(index));
         }
     }
     public indexOf(search: T['TValue']) {
@@ -52,7 +52,7 @@ export class DictionaryView<T extends DataType> implements View<T> {
         const key = this.dictionary.indexOf(search);
         if (key === -1) { return key; }
 
-        // ... then find the first occurence of that key in indicies
-        return this.indicies.indexOf(key!);
+        // ... then find the first occurence of that key in indices
+        return this.indices.indexOf(key!);
     }
 }

From e50a8ec432625656c7c2158228c878ba5fb94568 Mon Sep 17 00:00:00 2001
From: lmeyerov <lmeyerov@gmail.com>
Date: Mon, 26 Feb 2018 17:05:25 -0500
Subject: [PATCH 1531/1644] ARROW-2206: [JS] Document Perspective project

See https://issues.apache.org/jira/browse/ARROW-2206

+ https://github.com/deepankarsharma

Author: lmeyerov <lmeyerov@gmail.com>

Closes #1652 from lmeyerov/patch-2 and squashes the following commits:

904e1725 <lmeyerov> fix(reduce perspective description)
91291a09 <lmeyerov> fix(Perspective description)
b8ef13ec <lmeyerov> fix(missing colon)
80a46306 <lmeyerov> Document Perspective project  (main site)
cc938c1a <lmeyerov>  ARROW-2206 - Document Perspective project
---
 js/README.md       | 1 +
 site/powered_by.md | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/js/README.md b/js/README.md
index a34664b27767f..2126c39426691 100644
--- a/js/README.md
+++ b/js/README.md
@@ -212,6 +212,7 @@ Full list of broader Apache Arrow [projects & organizations](https://github.com/
 
 * [Apache Arrow](https://arrow.apache.org) -- Parent project for Powering Columnar In-Memory Analytics, including affiliated open source projects
 * [rxjs-mapd](https://github.com/graphistry/rxjs-mapd) -- A MapD Core node-driver that returns query results as Arrow columns
+* [Perspective](https://github.com/jpmorganchase/perspective) -- Perspective is a streaming data visualization engine by J.P. Morgan for JavaScript for building real-time & user-configurable analytics entirely in the browser. 
 
 ## Companies & Organizations
 
diff --git a/site/powered_by.md b/site/powered_by.md
index 73850cab3a023..8b407f26d64a2 100644
--- a/site/powered_by.md
+++ b/site/powered_by.md
@@ -91,6 +91,7 @@ short description of your use case. See the following for some examples.
 * **[pandas][12]:** data analysis toolkit for Python programmers. pandas
   supports reading and writing Parquet files using pyarrow. Several pandas
   core developers are also contributors to Apache Arrow.
+* **[Perspective][23]:** Perspective is a streaming data visualization engine in JavaScript for building real-time & user-configurable analytics entirely in the browser.
 * **[Quilt Data][13]:** Quilt is a data package manager, designed to make
   managing data as easy as managing code. It supports Parquet format via
   pyarrow for data access.
@@ -138,3 +139,4 @@ short description of your use case. See the following for some examples.
 [20]: https://github.com/johanpel/fletcher
 [21]: https://www.paradigm4.com
 [22]: https://github.com/Paradigm4/stream
+[23]: https://github.com/jpmorganchase/perspective

From e0328b0af5ea80116d266bb270260a9f9605ebcc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 27 Feb 2018 10:38:08 +0100
Subject: [PATCH 1532/1644] ARROW-2023: [C++] Fix ASAN failure on malformed /
 empty stream input, enable ASAN builds, add more dev docs

More to do here, see discussion in https://issues.apache.org/jira/browse/ARROW-1589

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1503 from wesm/ARROW-2023 and squashes the following commits:

e3a590e <Wes McKinney> Do not invoke memcpy unless number of bytes to read greater than 0. Return more informative error message
90787ec <Wes McKinney> Add ASAN instructions, llvm-symbolizer instructions for fuzzers. Remove Apache Kudu cruft from run-test.sh
53d2c92 <Wes McKinney> Add missing lsan-suppressions.txt to enable ASAN to run with unittests
69cb5b7 <Wes McKinney> Start C++ stream reader tests for malformed inputs
---
 cpp/CMakeLists.txt                       |  7 ++++---
 cpp/README.md                            | 25 +++++++++++++++++++++++-
 cpp/build-support/lsan-suppressions.txt  | 19 ++++++++++++++++++
 cpp/build-support/run-test.sh            |  6 ------
 cpp/src/arrow/io/memory.cc               |  6 ++++--
 cpp/src/arrow/ipc/ipc-read-write-test.cc | 16 +++++++++++++++
 cpp/src/arrow/ipc/message.cc             |  5 ++++-
 7 files changed, 71 insertions(+), 13 deletions(-)
 create mode 100644 cpp/build-support/lsan-suppressions.txt

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index b9221414bf657..8c0e95634fc1f 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -291,8 +291,6 @@ include(ThirdpartyToolchain)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMMON_FLAGS}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}")
 
-message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
-
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
   # Using Clang with ccache causes a bunch of spurious warnings that are
   # purportedly fixed in the next version of ccache. See the following for details:
@@ -305,7 +303,7 @@ endif()
 
 # ASAN / TSAN / UBSAN
 if(ARROW_FUZZING)
-    set(ARROW_USE_COVERAGE ON)
+  set(ARROW_USE_COVERAGE ON)
 endif()
 include(san-config)
 
@@ -333,6 +331,9 @@ if ("${ARROW_GENERATE_COVERAGE}")
   endif()
 endif()
 
+# CMAKE_CXX_FLAGS now fully assembled
+message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
+
 # set compile output directory
 string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
 
diff --git a/cpp/README.md b/cpp/README.md
index 1daf863819cbc..daeeade723019 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -99,7 +99,14 @@ and benchmarks or `make runbenchmark` to run only the benchmark tests.
 
 Benchmark logs will be placed in the build directory under `build/benchmark-logs`.
 
-## Building/Running fuzzers
+### Testing with LLVM AddressSanitizer
+
+To use AddressSanitizer (ASAN) to find bad memory accesses or leaks with LLVM,
+pass `-DARROW_USE_ASAN=ON` when building. You must use clang to compile with
+ASAN, and `ARROW_USE_ASAN` is mutually-exclusive with the valgrind option
+`ARROW_TEST_MEMCHECK`.
+
+### Building/Running fuzzers
 
 Fuzzers can help finding unhandled exceptions and problems with untrusted input that
 may lead to crashes, security issues and undefined behavior. They do this by
@@ -128,6 +135,22 @@ stack trace as well as the input data. After a problem was found this way, it sh
 be reported and fixed. Usually, the fuzzing process cannot be continued until the
 fix is applied, since the fuzzer usually converts to the problem again.
 
+If you build fuzzers with ASAN, you need to set the `ASAN_SYMBOLIZER_PATH`
+environment variable to the absolute path of `llvm-symbolizer`, which is a tool
+that ships with LLVM.
+
+```shell
+export ASAN_SYMBOLIZER_PATH=$(type -p llvm-symbolizer)
+```
+
+Note that some fuzzer builds currently reject paths with a version qualifier
+(like `llvm-sanitizer-5.0`). To overcome this, set an appropriate symlink
+(here, when using LLVM 5.0):
+
+```shell
+ln -sf /usr/bin/llvm-sanitizer-5.0 /usr/bin/llvm-sanitizer
+```
+
 There are some problems that may occur during the compilation process:
 
 - libfuzzer was not distributed with your LLVM: `ld: file not found: .../libLLVMFuzzer.a`
diff --git a/cpp/build-support/lsan-suppressions.txt b/cpp/build-support/lsan-suppressions.txt
new file mode 100644
index 0000000000000..927afb39710bd
--- /dev/null
+++ b/cpp/build-support/lsan-suppressions.txt
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# False positive from atexit() registration in libc
+leak:*__new_exitfn*
diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh
index b4da4f3f02ee4..656ab7bd3b805 100755
--- a/cpp/build-support/run-test.sh
+++ b/cpp/build-support/run-test.sh
@@ -88,12 +88,6 @@ function setup_sanitizers() {
   # Set up suppressions for LeakSanitizer
   LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt"
   export LSAN_OPTIONS
-
-  # Suppressions require symbolization. We'll default to using the symbolizer in
-  # thirdparty.
-  if [ -z "$ASAN_SYMBOLIZER_PATH" ]; then
-    export ASAN_SYMBOLIZER_PATH=$(find $NATIVE_TOOLCHAIN/llvm-3.7.0/bin -name llvm-symbolizer)
-  fi
 }
 
 function run_test() {
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index ecdf26f0a991b..45d6bdbde3bec 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -257,9 +257,11 @@ Status BufferReader::Tell(int64_t* position) const {
 bool BufferReader::supports_zero_copy() const { return true; }
 
 Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
-  memcpy(buffer, data_ + position_, nbytes);
   *bytes_read = std::min(nbytes, size_ - position_);
-  position_ += *bytes_read;
+  if (*bytes_read) {
+    memcpy(buffer, data_ + position_, *bytes_read);
+    position_ += *bytes_read;
+  }
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 1fcbdac5ebc73..d877e9922e367 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -755,5 +755,21 @@ TEST_F(TestTensorRoundTrip, NonContiguous) {
   CheckTensorRoundTrip(tensor);
 }
 
+TEST(TestRecordBatchStreamReader, MalformedInput) {
+  const std::string empty_str = "";
+  const std::string garbage_str = "12345678";
+
+  auto empty = std::make_shared<Buffer>(empty_str);
+  auto garbage = std::make_shared<Buffer>(garbage_str);
+
+  std::shared_ptr<RecordBatchReader> batch_reader;
+
+  io::BufferReader empty_reader(empty);
+  ASSERT_RAISES(Invalid, RecordBatchStreamReader::Open(&empty_reader, &batch_reader));
+
+  io::BufferReader garbage_reader(garbage);
+  ASSERT_RAISES(Invalid, RecordBatchStreamReader::Open(&garbage_reader, &batch_reader));
+}
+
 }  // namespace ipc
 }  // namespace arrow
diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc
index 1835cefde09ee..fd747563a45b0 100644
--- a/cpp/src/arrow/ipc/message.cc
+++ b/cpp/src/arrow/ipc/message.cc
@@ -227,7 +227,10 @@ Status ReadMessage(io::InputStream* file, std::unique_ptr<Message>* message) {
   std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(file->Read(message_length, &metadata));
   if (metadata->size() != message_length) {
-    return Status::IOError("Unexpected end of stream trying to read message");
+    std::stringstream ss;
+    ss << "Expected to read " << message_length << " metadata bytes, but "
+       << "only read " << metadata->size();
+    return Status::Invalid(ss.str());
   }
 
   return Message::ReadFrom(metadata, file, message);

From c017a6346f95d58f2ea14b5c2bcd3d7fd7321316 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Feb 2018 10:38:39 +0100
Subject: [PATCH 1533/1644] ARROW-1035: [Python] Add streaming dataframe
 reconstruction benchmark

Author: Antoine Pitrou <antoine@python.org>

Closes #1665 from pitrou/ARROW-1035-streaming-benchmark and squashes the following commits:

32b1956 <Antoine Pitrou> ARROW-1035:  Add streaming dataframe reconstruction benchmark
---
 python/benchmarks/common.py    | 31 ++++++++++++----
 python/benchmarks/streaming.py | 67 ++++++++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+), 8 deletions(-)
 create mode 100644 python/benchmarks/streaming.py

diff --git a/python/benchmarks/common.py b/python/benchmarks/common.py
index 7dd42fde5abe1..b205ba5817d30 100644
--- a/python/benchmarks/common.py
+++ b/python/benchmarks/common.py
@@ -23,13 +23,21 @@
 import numpy as np
 
 
+KILOBYTE = 1 << 10
+MEGABYTE = KILOBYTE * KILOBYTE
+
+
 def _multiplicate_sequence(base, target_size):
     q, r = divmod(target_size, len(base))
     return [base] * q + [base[:r]]
 
 
-def get_random_bytes(n):
-    rnd = np.random.RandomState(42)
+def get_random_bytes(n, *, seed=42):
+    """
+    Generate a random bytes object of size *n*.
+    Note the result might be compressible.
+    """
+    rnd = np.random.RandomState(seed)
     # Computing a huge random bytestring can be costly, so we get at most
     # 100KB and duplicate the result as needed
     base_size = 100003
@@ -43,22 +51,25 @@ def get_random_bytes(n):
     return result
 
 
-def get_random_ascii(n):
-    arr = np.frombuffer(get_random_bytes(n), dtype=np.int8) & 0x7f
+def get_random_ascii(n, *, seed=42):
+    """
+    Get a random ASCII-only unicode string of size *n*.
+    """
+    arr = np.frombuffer(get_random_bytes(n, seed=seed), dtype=np.int8) & 0x7f
     result, _ = codecs.ascii_decode(arr)
     assert isinstance(result, str)
     assert len(result) == n
     return result
 
 
-def _random_unicode_letters(n):
+def _random_unicode_letters(n, *, seed=42):
     """
     Generate a string of random unicode letters (slow).
     """
     def _get_more_candidates():
         return rnd.randint(0, sys.maxunicode, size=n).tolist()
 
-    rnd = np.random.RandomState(42)
+    rnd = np.random.RandomState(seed)
     out = []
     candidates = []
 
@@ -75,8 +86,12 @@ def _get_more_candidates():
 _1024_random_unicode_letters = _random_unicode_letters(1024)
 
 
-def get_random_unicode(n):
-    indices = np.frombuffer(get_random_bytes(n * 2), dtype=np.int16) & 1023
+def get_random_unicode(n, *, seed=42):
+    """
+    Get a random non-ASCII unicode string of size *n*.
+    """
+    indices = np.frombuffer(get_random_bytes(n * 2, seed=seed),
+                            dtype=np.int16) & 1023
     unicode_arr = np.array(_1024_random_unicode_letters)[indices]
 
     result = ''.join(unicode_arr.tolist())
diff --git a/python/benchmarks/streaming.py b/python/benchmarks/streaming.py
new file mode 100644
index 0000000000000..be7fda42c673a
--- /dev/null
+++ b/python/benchmarks/streaming.py
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+
+from . import common
+from .common import KILOBYTE, MEGABYTE
+
+
+def generate_chunks(total_size, nchunks, ncols, dtype=np.dtype('int64')):
+    rowsize = total_size // nchunks // ncols
+    assert rowsize % dtype.itemsize == 0
+    return [pd.DataFrame({
+            'c' + str(col): np.frombuffer(
+                common.get_random_bytes(rowsize, seed=col + 997 * chunk)).view(dtype)
+            for col in range(ncols)
+        })
+        for chunk in range(nchunks)]
+
+
+class StreamReader(object):
+    """
+    Benchmark in-memory streaming to a Pandas dataframe.
+    """
+    total_size = 64 * MEGABYTE
+    ncols = 8
+    chunk_sizes = [16 * KILOBYTE, 256 * KILOBYTE, 8 * MEGABYTE]
+
+    param_names = ['chunk_size']
+    params = [chunk_sizes]
+
+    def setup(self, chunk_size):
+        # Note we're careful to stream different chunks instead of
+        # streaming N times the same chunk, so that we avoid operating
+        # entirely out of L1/L2.
+        chunks = generate_chunks(self.total_size,
+                                 nchunks=self.total_size // chunk_size,
+                                 ncols=self.ncols)
+        batches = [pa.RecordBatch.from_pandas(df)
+                   for df in chunks]
+        schema = batches[0].schema
+        sink = pa.BufferOutputStream()
+        stream_writer = pa.RecordBatchStreamWriter(sink, schema)
+        for batch in batches:
+            stream_writer.write_batch(batch)
+        self.source = sink.get_result()
+
+    def time_read_to_dataframe(self, *args):
+        reader = pa.RecordBatchStreamReader(self.source)
+        table = reader.read_all()
+        df = table.to_pandas()

From a5c5ad211216434f1753606b8f90573566deb00f Mon Sep 17 00:00:00 2001
From: rvernica <rvernica@gmail.com>
Date: Tue, 27 Feb 2018 10:39:08 +0100
Subject: [PATCH 1534/1644] ARROW-2203: [C++] StderrStream class

Output stream that just writes to stderr. Adapted from StdoutStream, but use `cerr` instead of `cout`.

Author: rvernica <rvernica@gmail.com>

Closes #1657 from rvernica/patch-1 and squashes the following commits:

535dd25 <rvernica> Fix lint issue - trim whitespace
4aa45b0 <rvernica> ARROW-2203:  StderrStream class
---
 cpp/src/arrow/util/io-util.h | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h
index 864e1bde749e5..2a01be1044903 100644
--- a/cpp/src/arrow/util/io-util.h
+++ b/cpp/src/arrow/util/io-util.h
@@ -51,6 +51,29 @@ class StdoutStream : public OutputStream {
   int64_t pos_;
 };
 
+// Output stream that just writes to stderr.
+class StderrStream : public OutputStream {
+ public:
+  StderrStream() : pos_(0) { set_mode(FileMode::WRITE); }
+  ~StderrStream() override {}
+
+  Status Close() override { return Status::OK(); }
+
+  Status Tell(int64_t* position) const override {
+    *position = pos_;
+    return Status::OK();
+  }
+
+  Status Write(const void* data, int64_t nbytes) override {
+    pos_ += nbytes;
+    std::cerr.write(reinterpret_cast<const char*>(data), nbytes);
+    return Status::OK();
+  }
+
+ private:
+  int64_t pos_;
+};
+
 // Input stream that just reads from stdin.
 class StdinStream : public InputStream {
  public:

From 887e89303af57ae391a1eb63ffa851d813c4d39a Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Feb 2018 10:39:36 +0100
Subject: [PATCH 1535/1644] ARROW-1937: [Python] Document nested array
 initialization

Author: Antoine Pitrou <antoine@python.org>

Closes #1663 from pitrou/ARROW-1937-nested-array-init-doc and squashes the following commits:

b85d1b2 <Antoine Pitrou> ARROW-1937:  Document nested array initialization
---
 python/doc/source/data.rst | 91 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 87 insertions(+), 4 deletions(-)

diff --git a/python/doc/source/data.rst b/python/doc/source/data.rst
index 04e74ae64d437..3a602d54c5c7f 100644
--- a/python/doc/source/data.rst
+++ b/python/doc/source/data.rst
@@ -153,13 +153,20 @@ can also be created from various types of Python sequences (lists, NumPy
 arrays, pandas data).
 
 A simple way to create arrays is with ``pyarrow.array``, which is similar to
-the ``numpy.array`` function:
+the ``numpy.array`` function.  By default PyArrow will infer the data type
+for you:
 
 .. ipython:: python
 
    arr = pa.array([1, 2, None, 3])
    arr
 
+But you may also pass a specific data type to override type inference:
+
+.. ipython:: python
+
+   pa.array([1, 2], type=pa.uint16())
+
 The array's ``type`` attribute is the corresponding piece of type metadata:
 
 .. ipython:: python
@@ -189,15 +196,91 @@ Arrays can be sliced without copying:
 
 .. ipython:: python
 
-   arr[3]
+   arr[1:3]
 
-``pyarrow.array`` can create simple nested data structures like lists:
+List arrays
+~~~~~~~~~~~
+
+``pyarrow.array`` is able to infer the type of simple nested data structures
+like lists:
 
 .. ipython:: python
 
    nested_arr = pa.array([[], None, [1, 2], [None, 1]])
    print(nested_arr.type)
 
+Struct arrays
+~~~~~~~~~~~~~
+
+For other kinds of nested arrays, such as struct arrays, you currently need
+to pass the type explicitly.  Struct arrays can be initialized from a
+sequence of Python dicts or tuples:
+
+.. ipython:: python
+
+   ty = pa.struct([
+       pa.field('x', pa.int8()),
+       pa.field('y', pa.bool_()),
+   ])
+   pa.array([{'x': 1, 'y': True}, {'x': 2, 'y': False}], type=ty)
+   pa.array([(3, True), (4, False)], type=ty)
+
+When initializing a struct array, nulls are allowed both at the struct
+level and at the individual field level.  If initializing from a sequence
+of Python dicts, a missing dict key is handled as a null value:
+
+.. ipython:: python
+
+   pa.array([{'x': 1}, None, {'y': None}], type=ty)
+
+You can also construct a struct array from existing arrays for each of the
+struct's components.  In this case, data storage will be shared with the
+individual arrays, and no copy is involved:
+
+.. ipython:: python
+
+   xs = pa.array([5, 6, 7], type=pa.int16())
+   ys = pa.array([False, True, True])
+   arr = pa.StructArray.from_arrays((xs, ys), names=('x', 'y'))
+   arr.type
+   arr
+
+Union arrays
+~~~~~~~~~~~~
+
+The union type represents a nested array type where each value can be one
+(and only one) of a set of possible types.  There are two possible
+storage types for union arrays: sparse and dense.
+
+In a sparse union array, each of the child arrays has the same length
+as the resulting union array.  They are adjuncted with a ``int8`` "types"
+array that tells, for each value, from which child array it must be
+selected:
+
+.. ipython:: python
+
+   xs = pa.array([5, 6, 7])
+   ys = pa.array([False, False, True])
+   types = pa.array([0, 1, 1], type=pa.int8())
+   union_arr = pa.UnionArray.from_sparse(types, [xs, ys])
+   union_arr.type
+   union_arr
+
+In a dense union array, you also pass, in addition to the ``int8`` "types"
+array, a ``int32`` "offsets" array that tells, for each value, at
+each offset in the selected child array it can be found:
+
+.. ipython:: python
+
+   xs = pa.array([5, 6, 7])
+   ys = pa.array([False, True])
+   types = pa.array([0, 1, 1, 0, 0], type=pa.int8())
+   offsets = pa.array([0, 0, 1, 1, 2], type=pa.int32())
+   union_arr = pa.UnionArray.from_dense(types, offsets, [xs, ys])
+   union_arr.type
+   union_arr
+
+
 Dictionary Arrays
 ~~~~~~~~~~~~~~~~~
 
@@ -303,7 +386,7 @@ for one or more arrays of the same type.
    c.data.num_chunks
    c.data.chunk(0)
 
-As you'll see in the :ref:`pandas section <pandas>`, we can convert thee
+As you'll see in the :ref:`pandas section <pandas>`, we can convert these
 objects to contiguous NumPy arrays for use in pandas:
 
 .. ipython:: python

From 482fc584dcc09ea808062e0869a8c63cf7d5eb4c Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Tue, 27 Feb 2018 10:41:39 -0500
Subject: [PATCH 1536/1644] ARROW-2210: [C++] Reset ptr on failed memory
 allocation

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1672 from xhochy/ARROW-2210 and squashes the following commits:

6e5febe9 <Uwe L. Korn> ARROW-2210:  Reset ptr on failed memory allocation
---
 cpp/src/arrow/memory_pool.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index b8cd9c2fcf9a4..dedab7ea7291d 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -107,10 +107,12 @@ class DefaultMemoryPool : public MemoryPool {
 
   Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override {
 #ifdef ARROW_JEMALLOC
+    uint8_t* previous_ptr = *ptr;
     *ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
     if (*ptr == NULL) {
       std::stringstream ss;
       ss << "realloc of size " << new_size << " failed";
+      *ptr = previous_ptr;
       return Status::OutOfMemory(ss.str());
     }
 #else

From 1a92846d0559b0a3380ef9d15e0cf3d5a072b9b4 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Tue, 27 Feb 2018 10:57:36 -0500
Subject: [PATCH 1537/1644] ARROW-2223: [JS] compile src/bin as es5-cjs to all
 output targets

This ensures the `bin` files are uniformly es5/commonjs for all target modules. Now this works:
```sh
npm i @apache-arrow/esnext-esm
npx arrow2csv -f some-file.arrow
```

Resolves https://issues.apache.org/jira/browse/ARROW-2223

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1669 from trxcllnt/js-add-bin-to-umd-targets and squashes the following commits:

9c6b7878 <Paul Taylor> compile src/bin as es5-cjs to all output targets
---
 js/gulp/arrow-task.js             |  8 ++++----
 js/gulp/closure-task.js           |  6 +++++-
 js/gulp/typescript-task.js        | 25 ++++++++++++++++++-------
 js/gulp/uglify-task.js            |  2 ++
 js/gulpfile.js                    |  8 ++++----
 js/src/bin/arrow2csv.ts           |  2 --
 js/tsconfig/tsconfig.base.json    |  2 +-
 js/tsconfig/tsconfig.bin.cjs.json | 12 ++++++++++++
 8 files changed, 46 insertions(+), 19 deletions(-)
 create mode 100644 js/tsconfig/tsconfig.bin.cjs.json

diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js
index d1e8046e67ab9..eb83a6da31028 100644
--- a/js/gulp/arrow-task.js
+++ b/js/gulp/arrow-task.js
@@ -31,10 +31,10 @@ const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, forma
     const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`;
     const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`;
     const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`;
-    const es5UmdGlob = `${targetDir(`es5`, `umd`)}/**/*.js`;
-    const es5UmdMaps = `${targetDir(`es5`, `umd`)}/**/*.map`;
-    const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/**/*.js`;
-    const es2015UmdMaps = `${targetDir(`es2015`, `umd`)}/**/*.map`;
+    const es5UmdGlob = `${targetDir(`es5`, `umd`)}/*.js`;
+    const es5UmdMaps = `${targetDir(`es5`, `umd`)}/*.map`;
+    const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/*.js`;
+    const es2015UmdMaps = `${targetDir(`es2015`, `umd`)}/*.map`;
     const ch_ext = (ext) => gulpRename((p) => { p.extname = ext; });
     const append = (ap) => gulpRename((p) => { p.basename += ap; });
     return Observable.forkJoin(
diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js
index 0b2ef1b846b81..8833c2c2228ff 100644
--- a/js/gulp/closure-task.js
+++ b/js/gulp/closure-task.js
@@ -27,6 +27,7 @@ const gulp = require('gulp');
 const path = require('path');
 const sourcemaps = require('gulp-sourcemaps');
 const { memoizeTask } = require('./memoize-task');
+const { compileBinFiles } = require('./typescript-task');
 const ASTBuilders = require('ast-types').builders;
 const transformAST = require('gulp-transform-js-ast');
 const { Observable, ReplaySubject } = require('rxjs');
@@ -55,7 +56,10 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
         // rename the sourcemaps from *.js.map files to *.min.js.map
         sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }),
         gulp.dest(out)
-    ).publish(new ReplaySubject()).refCount();
+    )
+    .merge(compileBinFiles(target, format))
+    .takeLast(1)
+    .publish(new ReplaySubject()).refCount();
 }))({});
 
 const createClosureArgs = (entry, externs) => ({
diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js
index c42357adb2f75..0fdd1c7326650 100644
--- a/js/gulp/typescript-task.js
+++ b/js/gulp/typescript-task.js
@@ -30,22 +30,33 @@ const { Observable, ReplaySubject } = require('rxjs');
 
 const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target, format) {
     const out = targetDir(target, format);
-    const tsconfigFile = `tsconfig.${tsconfigName(target, format)}.json`;
-    const tsProject = ts.createProject(path.join(`tsconfig`, tsconfigFile), { typescript: require(`typescript`) });
+    const tsconfigPath = path.join(`tsconfig`, `tsconfig.${tsconfigName(target, format)}.json`);
+    return compileTypescript(out, tsconfigPath)
+        .merge(compileBinFiles(target, format)).takeLast(1)
+        .concat(maybeCopyRawJSArrowFormatFiles(target, format))
+        .publish(new ReplaySubject()).refCount();
+}))({});
+
+function compileBinFiles(target, format) {
+    const out = targetDir(target, format);
+    const tsconfigPath = path.join(`tsconfig`, `tsconfig.${tsconfigName('bin', 'cjs')}.json`);
+    return compileTypescript(path.join(out, 'bin'), tsconfigPath);
+}
+
+function compileTypescript(out, tsconfigPath) {
+    const tsProject = ts.createProject(tsconfigPath, { typescript: require(`typescript`) });
     const { stream: { js, dts } } = observableFromStreams(
       tsProject.src(), sourcemaps.init(),
       tsProject(ts.reporter.defaultReporter())
     );
     const writeDTypes = observableFromStreams(dts, gulp.dest(out));
     const writeJS = observableFromStreams(js, sourcemaps.write(), gulp.dest(out));
-    return Observable
-        .forkJoin(writeDTypes, writeJS)
-        .concat(maybeCopyRawJSArrowFormatFiles(target, format))
-        .publish(new ReplaySubject()).refCount();
-}))({});
+    return Observable.forkJoin(writeDTypes, writeJS);
+}
 
 module.exports = typescriptTask;
 module.exports.typescriptTask = typescriptTask;
+module.exports.compileBinFiles = compileBinFiles;
 
 function maybeCopyRawJSArrowFormatFiles(target, format) {
     if (target !== `es5` || format !== `cls`) {
diff --git a/js/gulp/uglify-task.js b/js/gulp/uglify-task.js
index 9ba3e41a16f41..f8dc123cae7a4 100644
--- a/js/gulp/uglify-task.js
+++ b/js/gulp/uglify-task.js
@@ -27,6 +27,7 @@ const {
 const path = require('path');
 const webpack = require(`webpack`);
 const { memoizeTask } = require('./memoize-task');
+const { compileBinFiles } = require('./typescript-task');
 const { Observable, ReplaySubject } = require('rxjs');
 const UglifyJSPlugin = require(`uglifyjs-webpack-plugin`);
 const esmRequire = require(`@std/esm`)(module, { cjs: true, esm: `js`, warnings: false });
@@ -73,6 +74,7 @@ const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJ
     const compilers = webpack(webpackConfigs);
     return Observable
             .bindNodeCallback(compilers.run.bind(compilers))()
+            .merge(compileBinFiles(target, format)).takeLast(1)
             .multicast(new ReplaySubject()).refCount();
 }))({}, {
     resolve: { mainFields: [`module`, `main`] },
diff --git a/js/gulpfile.js b/js/gulpfile.js
index 7b82962035e46..891d6c79b3866 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -58,15 +58,15 @@ knownTargets.forEach((target) =>
     )
 );
 
-// The main "apache-arrow" module builds the es5/cjs, es5/umd,
-// es2015/esm, es2015/umd, and ts targets, then copies and
-// renames the compiled output into the apache-arrow folder
+// The main "apache-arrow" module builds the es5/umd, es2015/cjs,
+// es2015/esm, and es2015/umd targets, then copies and renames the
+// compiled output into the apache-arrow folder
 gulp.task(`build:${npmPkgName}`,
     gulp.series(
         cleanTask(npmPkgName),
         gulp.parallel(
-            `build:${taskName(`es5`, `cjs`)}`,
             `build:${taskName(`es5`, `umd`)}`,
+            `build:${taskName(`es2015`, `cjs`)}`,
             `build:${taskName(`es2015`, `esm`)}`,
             `build:${taskName(`es2015`, `umd`)}`
         ),
diff --git a/js/src/bin/arrow2csv.ts b/js/src/bin/arrow2csv.ts
index ee9561323785b..6d197c7b6b4f9 100644
--- a/js/src/bin/arrow2csv.ts
+++ b/js/src/bin/arrow2csv.ts
@@ -84,12 +84,10 @@ if (!files.length) {
 }
 
 files.forEach((source) => {
-    debugger;
     let table: Arrow.Table, input = fs.readFileSync(source);
     try {
         table = Arrow.Table.from(input);
     } catch (e) {
-        debugger;
         table = Arrow.Table.from(parse(input + ''));
     }
     if (argv.schema && argv.schema.length) {
diff --git a/js/tsconfig/tsconfig.base.json b/js/tsconfig/tsconfig.base.json
index 8b8210198960a..d0b813ef38e2a 100644
--- a/js/tsconfig/tsconfig.base.json
+++ b/js/tsconfig/tsconfig.base.json
@@ -1,5 +1,5 @@
 {
-  "exclude": ["../node_modules"],
+  "exclude": ["../node_modules", "../src/bin/*.ts"],
   "include": ["../src/**/*.ts"],
   "compileOnSave": false,
   "compilerOptions": {
diff --git a/js/tsconfig/tsconfig.bin.cjs.json b/js/tsconfig/tsconfig.bin.cjs.json
new file mode 100644
index 0000000000000..e17c1b589acc7
--- /dev/null
+++ b/js/tsconfig/tsconfig.bin.cjs.json
@@ -0,0 +1,12 @@
+//Compiler configuaration to build the ES5 CommonJS bin files
+{
+    "extends": "./tsconfig.base.json",
+    "exclude": ["../node_modules"],
+    "include": ["../src/bin/*.ts"],
+      "compilerOptions": {
+      "target": "ES5",
+      "module": "commonjs",
+      "declaration": false
+    }
+  }
+  
\ No newline at end of file

From d3fabe0f5318a7b0767bc097962372bad19edae3 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Wed, 28 Feb 2018 14:00:15 -0500
Subject: [PATCH 1538/1644] =?UTF-8?q?ARROW-2230:=20[Python]=C2=A0Strip=20c?=
 =?UTF-8?q?atch-all=20tag=20matching=20from=20git-describe?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1676 from xhochy/ARROW-2230 and squashes the following commits:

b46c1a27 <Korn, Uwe> ARROW-2230:  Strip catch-all tag matching from git-describe
---
 python/setup.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/setup.py b/python/setup.py
index 849d1203be7b2..2a6326489d3fa 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -422,6 +422,8 @@ def parse_version(root):
     from setuptools_scm import version_from_scm
     import setuptools_scm.git
     describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --match 'apache-arrow-[0-9]*'"
+    # Strip catchall from the commandline
+    describe = describe.replace("--match *.*", "")
     version = setuptools_scm.git.parse(root, describe)
     if not version:
         return version_from_scm(root)

From 524b5221d2c977f2fa8284ed72d6bbb51dedcb69 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Feb 2018 14:04:49 -0500
Subject: [PATCH 1539/1644] ARROW-2218: [Python] PythonFile should infer mode
 when not given

Author: Antoine Pitrou <antoine@python.org>

Closes #1674 from pitrou/ARROW-2218-infer-python-file-mode and squashes the following commits:

a5d704e8 <Antoine Pitrou> ARROW-2218:  PythonFile should infer mode when not given
---
 python/pyarrow/io-hdfs.pxi      |  2 --
 python/pyarrow/io.pxi           | 14 +++++++++++-
 python/pyarrow/lib.pxd          |  1 +
 python/pyarrow/tests/test_io.py | 39 +++++++++++++++++++++++++++++++++
 4 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi
index dc6ba23abc9c1..31c0437e48a28 100644
--- a/python/pyarrow/io-hdfs.pxi
+++ b/python/pyarrow/io-hdfs.pxi
@@ -480,7 +480,5 @@ cdef class HdfsFile(NativeFile):
         object mode
         object parent
 
-    cdef object __weakref__
-
     def __dealloc__(self):
         self.parent = None
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 8b364dc7163a5..0b444cdb98c85 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -410,9 +410,21 @@ cdef class PythonFile(NativeFile):
     cdef:
         object handle
 
-    def __cinit__(self, handle, mode='w'):
+    def __cinit__(self, handle, mode=None):
         self.handle = handle
 
+        if mode is None:
+            try:
+                mode = handle.mode
+            except AttributeError:
+                # Not all file-like objects have a mode attribute
+                # (e.g. BytesIO)
+                try:
+                    mode = 'w' if handle.writable() else 'r'
+                except AttributeError:
+                    raise ValueError("could not infer open mode for file-like "
+                                     "object %r, please pass it explicitly"
+                                     % (handle,))
         if mode.startswith('w'):
             self.wr_file.reset(new PyOutputStream(handle))
             self.is_writable = True
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 31732a6e07741..e4d574f18b34f 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -337,6 +337,7 @@ cdef class NativeFile:
         bint is_writable
         readonly bint closed
         bint own_file
+        object __weakref__
 
     # By implementing these "virtual" functions (all functions in Cython
     # extension classes are technically virtual in the C++ sense) we can expose
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 736020f60fd59..d269ad0e7cdae 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -21,6 +21,7 @@
 import os
 import pytest
 import sys
+import weakref
 
 import numpy as np
 
@@ -124,6 +125,44 @@ def get_buffer():
     assert buf.to_pybytes() == b'sample'
     assert buf.parent is not None
 
+
+def test_python_file_implicit_mode(tmpdir):
+    path = os.path.join(str(tmpdir), 'foo.txt')
+    with open(path, 'wb') as f:
+        pf = pa.PythonFile(f)
+        assert pf.writable()
+        assert not pf.readable()
+        assert not pf.seekable()  # PyOutputStream isn't seekable
+        f.write(b'foobar\n')
+
+    with open(path, 'rb') as f:
+        pf = pa.PythonFile(f)
+        assert pf.readable()
+        assert not pf.writable()
+        assert pf.seekable()
+        assert pf.read() == b'foobar\n'
+
+    bio = BytesIO()
+    pf = pa.PythonFile(bio)
+    assert pf.writable()
+    assert not pf.readable()
+    assert not pf.seekable()
+    pf.write(b'foobar\n')
+    assert bio.getvalue() == b'foobar\n'
+
+
+def test_python_file_closing():
+    bio = BytesIO()
+    pf = pa.PythonFile(bio)
+    wr = weakref.ref(pf)
+    del pf
+    assert wr() is None  # object was destroyed
+    assert not bio.closed
+    pf = pa.PythonFile(bio)
+    pf.close()
+    assert bio.closed
+
+
 # ----------------------------------------------------------------------
 # Buffers
 

From 0a672bcbfd05eb126cd252e5abb0fbef95c051a2 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Wed, 28 Feb 2018 14:23:56 -0500
Subject: [PATCH 1540/1644] ARROW-2226, ARROW-2233: [JS] Dictionary bugfixes

Resolves https://issues.apache.org/jira/browse/ARROW-2226

Author: Paul Taylor <paul.e.taylor@me.com>
Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1671 from trxcllnt/js-fix-dictionary-data and squashes the following commits:

ccecf55 <Paul Taylor> Merge pull request #5 from TheNeuralBit/dictionary-vector-tests
3fb9a26 <Brian Hulette> Fix bug in DictionaryVector with nullable indices
2888657 <Brian Hulette> Add dictionary vector unit tests
b0a0c08 <Paul Taylor> use indicies.offset in DictionaryData constructor
---
 js/src/data.ts               |  3 +-
 js/src/vector.ts             |  3 ++
 js/test/unit/vector-tests.ts | 61 ++++++++++++++++++++++++++++++++----
 3 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/js/src/data.ts b/js/src/data.ts
index cdd9f29a7a908..3bfb3209b696b 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -152,10 +152,9 @@ export class DictionaryData<T extends DataType> extends BaseData<Dictionary<T>>
     public get indices() { return this._indices; }
     public get dictionary() { return this._dictionary; }
     constructor(type: Dictionary<T>, dictionary: Vector<T>, indices: Data<Int<any>>) {
-        super(type, indices.length, (indices as any)._nullCount);
+        super(type, indices.length, indices.offset, (indices as any)._nullCount);
         this._indices = indices;
         this._dictionary = dictionary;
-        this.length = this._indices.length;
     }
     public get nullCount() { return this._indices.nullCount; }
     public get nullBitmap() { return this._indices.nullBitmap; }
diff --git a/js/src/vector.ts b/js/src/vector.ts
index f36c691e1bd27..6c2bbbb86a7d2 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -399,6 +399,9 @@ export class DictionaryVector<T extends DataType = DataType> extends Vector<Dict
     public readonly dictionary: Vector<T>;
     constructor(data: Data<Dictionary<T>>, view: View<Dictionary<T>> = new DictionaryView<T>(data.dictionary, new IntVector(data.indices))) {
         super(data as Data<any>, view);
+        if (view instanceof ValidityView) {
+            view = (view as any).view;
+        }
         if (data instanceof DictionaryData && view instanceof DictionaryView) {
             this.indices = view.indices;
             this.dictionary = data.dictionary;
diff --git a/js/test/unit/vector-tests.ts b/js/test/unit/vector-tests.ts
index e2be229834f8e..3eb3fbe0195b3 100644
--- a/js/test/unit/vector-tests.ts
+++ b/js/test/unit/vector-tests.ts
@@ -17,14 +17,15 @@
 
 import { TextEncoder } from 'text-encoding-utf-8';
 import Arrow from '../Arrow';
-import { type, TypedArray, TypedArrayConstructor } from '../../src/Arrow';
+import { type, TypedArray, TypedArrayConstructor, Vector } from '../../src/Arrow';
+import { packBools } from '../../src/util/bit'
 
 const utf8Encoder = new TextEncoder('utf-8');
 
-const { BoolData, FlatData, FlatListData } = Arrow.data;
-const { IntVector, FloatVector, BoolVector, Utf8Vector } = Arrow.vector;
+const { BoolData, FlatData, FlatListData, DictionaryData } = Arrow.data;
+const { IntVector, FloatVector, BoolVector, Utf8Vector, DictionaryVector } = Arrow.vector;
 const {
-    Utf8, Bool,
+    Dictionary, Utf8, Bool,
     Float16, Float32, Float64,
     Int8, Int16, Int32, Int64,
     Uint8, Uint16, Uint32, Uint64,
@@ -310,6 +311,54 @@ describe(`Utf8Vector`, () => {
     let offset = 0;
     const offsets = Uint32Array.of(0, ...values.map((d) => { offset += d.length; return offset; }));
     const vector = new Utf8Vector(new FlatListData(new Utf8(), n, null, offsets, utf8Encoder.encode(values.join(''))));
+    basicVectorTests(vector, values, ['abc', '123']);
+    describe(`sliced`, () => {
+        basicVectorTests(vector.slice(1,3), values.slice(1,3), ['foo', 'abc']);
+    });
+});
+
+describe(`DictionaryVector`, () => {
+    const dictionary = ['foo', 'bar', 'baz'];
+    const extras = ['abc', '123']; // values to search for that should NOT be found
+    let offset = 0;
+    const offsets = Uint32Array.of(0, ...dictionary.map((d) => { offset += d.length; return offset; }));
+    const dictionary_vec = new Utf8Vector(new FlatListData(new Utf8(), dictionary.length, null, offsets, utf8Encoder.encode(dictionary.join(''))));
+
+    const indices = Array.from({length: 50}, () => Math.random() * 3 | 0);
+
+    describe(`index with nullCount == 0`, () => {
+        const indices_data = new FlatData(new Int32(), indices.length, new Uint8Array(0), indices);
+
+        const values = Array.from(indices).map((d) => dictionary[d]);
+        const vector = new DictionaryVector(new DictionaryData(new Dictionary(dictionary_vec.type, indices_data.type), dictionary_vec, indices_data));
+
+        basicVectorTests(vector, values, extras);
+
+        describe(`sliced`, () => {
+            basicVectorTests(vector.slice(10, 20), values.slice(10,20), extras);
+        })
+    });
+
+    describe(`index with nullCount > 0`, () => {
+        const validity = Array.from({length: indices.length}, () => Math.random() > 0.2 ? true : false);
+        const indices_data = new FlatData(new Int32(), indices.length, packBools(validity), indices, 0, validity.reduce((acc, d) => acc + (d ? 0 : 1), 0));
+        const values = Array.from(indices).map((d, i) => validity[i] ? dictionary[d] : null);
+        const vector = new DictionaryVector(new DictionaryData(new Dictionary(dictionary_vec.type, indices_data.type), dictionary_vec, indices_data));
+
+        basicVectorTests(vector, values, ['abc', '123']);
+        describe(`sliced`, () => {
+            basicVectorTests(vector.slice(10, 20), values.slice(10,20), extras);
+        });
+    });
+});
+
+// Creates some basic tests for the given vector.
+// Verifies that:
+// - `get` and the native iterator return the same data as `values`
+// - `indexOf` returns the same indices as `values`
+function basicVectorTests(vector: Vector, values: any[], extras: any[]) {
+    const n = values.length;
+
     test(`gets expected values`, () => {
         let i = -1;
         while (++i < n) {
@@ -325,14 +374,14 @@ describe(`Utf8Vector`, () => {
         }
     });
     test(`indexOf returns expected values`, () => {
-        let testValues = values.concat(['abc', '12345']);
+        let testValues = values.concat(extras);
 
         for (const value of testValues) {
             const expected = values.indexOf(value);
             expect(vector.indexOf(value)).toEqual(expected);
         }
     });
-});
+}
 
 function toMap<T>(entries: Record<string, T>, keys: string[]) {
     return keys.reduce((map, key) => {

From 1d9b8347467d6e915977e8ecc9b0759f86fd745a Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Wed, 28 Feb 2018 14:37:07 -0500
Subject: [PATCH 1541/1644] ARROW-2225: [JS] support tables split across
 buffers

@TheNeuralBit somewhere along the way we lost this. Resolves https://issues.apache.org/jira/browse/ARROW-2225

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1670 from trxcllnt/js-fix-multiple-buffers and squashes the following commits:

869cd74 <Paul Taylor> make the reader work for tables split across buffers again
---
 js/src/ipc/reader/binary.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/js/src/ipc/reader/binary.ts b/js/src/ipc/reader/binary.ts
index 6e3c7fc5cf080..26bc10bb377f2 100644
--- a/js/src/ipc/reader/binary.ts
+++ b/js/src/ipc/reader/binary.ts
@@ -49,7 +49,7 @@ export function* readBuffers<T extends Uint8Array | Buffer | string>(sources: It
     }
     for (const source of sources) {
         const bb = toByteBuffer(source);
-        if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) {
+        if ((!schema && ({ schema, readMessages } = readSchema(bb)) || true) && schema && readMessages) {
             for (const message of readMessages(bb)) {
                 yield {
                     schema, message,
@@ -71,7 +71,7 @@ export async function* readBuffersAsync<T extends Uint8Array | Buffer | string>(
     let readMessages: MessageReader | null = null;
     for await (const source of sources) {
         const bb = toByteBuffer(source);
-        if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) {
+        if ((!schema && ({ schema, readMessages } = readSchema(bb)) || true) && schema && readMessages) {
             for (const message of readMessages(bb)) {
                 yield {
                     schema, message,

From 671b53c36bbf9d2cced39d5c8ca291adee859d68 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Feb 2018 14:49:45 -0500
Subject: [PATCH 1542/1644] ARROW-2046: [Python] Support path-like objects

IO functions accepting string filenames should also accept PEP 519 path objects such as pathlib.Path (on Python 3.6 and later).

Author: Antoine Pitrou <antoine@python.org>

Closes #1675 from pitrou/ARROW-2046-path-like-objects and squashes the following commits:

c449bfe7 <Antoine Pitrou> ARROW-2046:  Support path-like objects
---
 python/pyarrow/_parquet.pyx                | 12 ++++---
 python/pyarrow/io.pxi                      | 42 +++++++++++++++-------
 python/pyarrow/tests/test_parquet.py       | 14 ++++++++
 python/pyarrow/tests/test_serialization.py | 17 ++++++---
 4 files changed, 63 insertions(+), 22 deletions(-)

diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index 9061ed53dac2b..e513e1d924401 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -31,7 +31,7 @@ from pyarrow.lib cimport (Array, Schema,
                           NativeFile, get_reader, get_writer)
 
 from pyarrow.compat import tobytes, frombytes
-from pyarrow.lib import ArrowException, NativeFile
+from pyarrow.lib import ArrowException, NativeFile, _stringify_path
 
 import six
 
@@ -825,15 +825,17 @@ cdef class ParquetWriter:
             c_string c_where
             CMemoryPool* pool
 
-        if isinstance(where, six.string_types):
+        try:
+            where = _stringify_path(where)
+        except TypeError:
+            get_writer(where, &self.sink)
+            self.own_sink = False
+        else:
             c_where = tobytes(where)
             with nogil:
                 check_status(FileOutputStream.Open(c_where,
                                                    &self.sink))
             self.own_sink = True
-        else:
-            get_writer(where, &self.sink)
-            self.own_sink = False
 
         self.use_dictionary = use_dictionary
         self.compression = compression
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 0b444cdb98c85..325c5827f00c0 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -38,6 +38,18 @@ cdef extern from "Python.h":
         char *v, Py_ssize_t len) except NULL
 
 
+def _stringify_path(path):
+    """
+    Convert *path* to a string or unicode path if possible.
+    """
+    if isinstance(path, six.string_types):
+        return path
+    try:
+        return path.__fspath__()
+    except AttributeError:
+        raise TypeError("not a path-like object")
+
+
 cdef class NativeFile:
     def __cinit__(self):
         self.closed = True
@@ -834,13 +846,16 @@ def frombuffer(object obj):
 cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
     cdef NativeFile nf
 
-    if isinstance(source, six.string_types):
-        source = memory_map(source, mode='r')
-    elif isinstance(source, Buffer):
-        source = BufferReader(source)
-    elif not isinstance(source, NativeFile) and hasattr(source, 'read'):
-        # Optimistically hope this is file-like
-        source = PythonFile(source, mode='r')
+    try:
+        source_path = _stringify_path(source)
+    except TypeError:
+        if isinstance(source, Buffer):
+            source = BufferReader(source)
+        elif not isinstance(source, NativeFile) and hasattr(source, 'read'):
+            # Optimistically hope this is file-like
+            source = PythonFile(source, mode='r')
+    else:
+        source = memory_map(source_path, mode='r')
 
     if isinstance(source, NativeFile):
         nf = source
@@ -858,11 +873,14 @@ cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
 cdef get_writer(object source, shared_ptr[OutputStream]* writer):
     cdef NativeFile nf
 
-    if isinstance(source, six.string_types):
-        source = OSFile(source, mode='w')
-    elif not isinstance(source, NativeFile) and hasattr(source, 'write'):
-        # Optimistically hope this is file-like
-        source = PythonFile(source, mode='w')
+    try:
+        source_path = _stringify_path(source)
+    except TypeError:
+        if not isinstance(source, NativeFile) and hasattr(source, 'write'):
+            # Optimistically hope this is file-like
+            source = PythonFile(source, mode='w')
+    else:
+        source = OSFile(source_path, mode='w')
 
     if isinstance(source, NativeFile):
         nf = source
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index bd76feb2e8cfd..187971fb8564b 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -22,6 +22,7 @@
 import io
 import json
 import os
+import sys
 
 import pytest
 
@@ -264,6 +265,19 @@ def test_pandas_parquet_1_0_rountrip(tmpdir):
     tm.assert_frame_equal(df, df_read)
 
 
+@parquet
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="need Python 3.6")
+def test_path_objects(tmpdir):
+    # Test compatibility with PEP 519 path-like objects
+    import pathlib
+    p = pathlib.Path(tmpdir) / 'zzz.parquet'
+    df = pd.DataFrame({'x': np.arange(10, dtype=np.int64)})
+    _write_table(df, p)
+    table_read = _read_table(p)
+    df_read = table_read.to_pandas()
+    tm.assert_frame_equal(df, df_read)
+
+
 @parquet
 def test_pandas_column_selection(tmpdir):
     size = 10000
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index feccebbde36e0..3ee02cb8c893e 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -584,15 +584,11 @@ def _get_modified_env_with_pythonpath():
     # Prepend pyarrow root directory to PYTHONPATH
     env = os.environ.copy()
     existing_pythonpath = env.get('PYTHONPATH', '')
-    if sys.platform == 'win32':
-        sep = ';'
-    else:
-        sep = ':'
 
     module_path = os.path.abspath(
         os.path.dirname(os.path.dirname(pa.__file__)))
 
-    env['PYTHONPATH'] = sep.join((module_path, existing_pythonpath))
+    env['PYTHONPATH'] = os.pathsep.join((module_path, existing_pythonpath))
     return env
 
 
@@ -650,3 +646,14 @@ def loads2(serialized_obj):
     serialized = pa.serialize(test_object, context=context).to_buffer()
     deserialized = pa.deserialize(serialized.to_pybytes(), context=context)
     assert deserialized == b'custom serialization 2'
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="need Python 3.6")
+def test_path_objects(tmpdir):
+    # Test compatibility with PEP 519 path-like objects
+    import pathlib
+    p = pathlib.Path(tmpdir) / 'zzz.bin'
+    obj = 1234
+    pa.serialize_to(obj, p)
+    res = pa.deserialize_from(p, None)
+    assert res == obj

From 3d5880aa0fd2325f1c0a4d5557a71f57838def83 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Feb 2018 18:18:47 -0500
Subject: [PATCH 1543/1644] ARROW-2040: [Python] Deserialized Numpy array must
 keep ref to underlying tensor

Author: Antoine Pitrou <antoine@python.org>

Closes #1680 from pitrou/ARROW-2040-ndarray-from-tensor-base and squashes the following commits:

15534c41 [Antoine Pitrou] ARROW-2040: [Python] Deserialized Numpy array must keep ref to underlying tensor
---
 cpp/src/arrow/python/arrow_to_python.cc    |  2 +-
 cpp/src/arrow/python/numpy_convert.cc      | 35 ++++++++++++----------
 cpp/src/arrow/python/numpy_convert.h       |  3 +-
 python/pyarrow/array.pxi                   |  2 +-
 python/pyarrow/includes/common.pxd         |  1 +
 python/pyarrow/includes/libarrow.pxd       |  2 +-
 python/pyarrow/tests/test_serialization.py | 17 +++++++++++
 7 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc
index 54a71d5a391d2..5515d24bd119c 100644
--- a/cpp/src/arrow/python/arrow_to_python.cc
+++ b/cpp/src/arrow/python/arrow_to_python.cc
@@ -94,7 +94,7 @@ Status DeserializeDict(PyObject* context, const Array& array, int64_t start_idx,
 Status DeserializeArray(const Array& array, int64_t offset, PyObject* base,
                         const SerializedPyObject& blobs, PyObject** out) {
   int32_t index = static_cast<const Int32Array&>(array).Value(offset);
-  RETURN_NOT_OK(py::TensorToNdarray(*blobs.tensors[index], base, out));
+  RETURN_NOT_OK(py::TensorToNdarray(blobs.tensors[index], base, out));
   // Mark the array as immutable
   OwnedRef flags(PyObject_GetAttrString(*out, "flags"));
   DCHECK(flags.obj() != NULL) << "Could not mark Numpy array immutable";
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 7ba13877d52cd..0cd616aec41c8 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -30,6 +30,7 @@
 #include "arrow/type.h"
 
 #include "arrow/python/common.h"
+#include "arrow/python/pyarrow.h"
 #include "arrow/python/type_traits.h"
 
 namespace arrow {
@@ -251,50 +252,54 @@ Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>*
   return Status::OK();
 }
 
-Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out) {
+Status TensorToNdarray(const std::shared_ptr<Tensor>& tensor, PyObject* base,
+                       PyObject** out) {
   PyAcquireGIL lock;
 
   int type_num;
-  RETURN_NOT_OK(GetNumPyType(*tensor.type(), &type_num));
+  RETURN_NOT_OK(GetNumPyType(*tensor->type(), &type_num));
   PyArray_Descr* dtype = PyArray_DescrNewFromType(type_num);
   RETURN_IF_PYERROR();
 
-  std::vector<npy_intp> npy_shape(tensor.ndim());
-  std::vector<npy_intp> npy_strides(tensor.ndim());
+  const int ndim = tensor->ndim();
+  std::vector<npy_intp> npy_shape(ndim);
+  std::vector<npy_intp> npy_strides(ndim);
 
-  for (int i = 0; i < tensor.ndim(); ++i) {
-    npy_shape[i] = tensor.shape()[i];
-    npy_strides[i] = tensor.strides()[i];
+  for (int i = 0; i < ndim; ++i) {
+    npy_shape[i] = tensor->shape()[i];
+    npy_strides[i] = tensor->strides()[i];
   }
 
   const void* immutable_data = nullptr;
-  if (tensor.data()) {
-    immutable_data = tensor.data()->data();
+  if (tensor->data()) {
+    immutable_data = tensor->data()->data();
   }
 
   // Remove const =(
   void* mutable_data = const_cast<void*>(immutable_data);
 
   int array_flags = 0;
-  if (tensor.is_row_major()) {
+  if (tensor->is_row_major()) {
     array_flags |= NPY_ARRAY_C_CONTIGUOUS;
   }
-  if (tensor.is_column_major()) {
+  if (tensor->is_column_major()) {
     array_flags |= NPY_ARRAY_F_CONTIGUOUS;
   }
-  if (tensor.is_mutable()) {
+  if (tensor->is_mutable()) {
     array_flags |= NPY_ARRAY_WRITEABLE;
   }
 
   PyObject* result =
-      PyArray_NewFromDescr(&PyArray_Type, dtype, tensor.ndim(), npy_shape.data(),
+      PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, npy_shape.data(),
                            npy_strides.data(), mutable_data, array_flags, nullptr);
   RETURN_IF_PYERROR()
 
-  if (base != Py_None) {
-    PyArray_SetBaseObject(reinterpret_cast<PyArrayObject*>(result), base);
+  if (base == Py_None || base == nullptr) {
+    base = py::wrap_tensor(tensor);
+  } else {
     Py_XINCREF(base);
   }
+  PyArray_SetBaseObject(reinterpret_cast<PyArrayObject*>(result), base);
   *out = result;
   return Status::OK();
 }
diff --git a/cpp/src/arrow/python/numpy_convert.h b/cpp/src/arrow/python/numpy_convert.h
index 220e38f2e1e02..dfdb1acd1237b 100644
--- a/cpp/src/arrow/python/numpy_convert.h
+++ b/cpp/src/arrow/python/numpy_convert.h
@@ -65,7 +65,8 @@ Status GetNumPyType(const DataType& type, int* type_num);
 ARROW_EXPORT Status NdarrayToTensor(MemoryPool* pool, PyObject* ao,
                                     std::shared_ptr<Tensor>* out);
 
-ARROW_EXPORT Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out);
+ARROW_EXPORT Status TensorToNdarray(const std::shared_ptr<Tensor>& tensor, PyObject* base,
+                                    PyObject** out);
 
 }  // namespace py
 }  // namespace arrow
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index a43bfb93b4d0c..5b8621f13b3a2 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -518,7 +518,7 @@ strides: {2}""".format(self.type, self.shape, self.strides)
             PyObject* out
 
         with nogil:
-            check_status(TensorToNdarray(deref(self.tp), self, &out))
+            check_status(TensorToNdarray(self.sp_tensor, self, &out))
         return PyObject_to_object(out)
 
     def equals(self, Tensor other):
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index f323feaff9ff0..4d799ecd2b434 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -32,6 +32,7 @@ cdef extern from "arrow/python/platform.h":
 
 cdef extern from "<Python.h>":
     void Py_XDECREF(PyObject* o)
+    Py_ssize_t Py_REFCNT(PyObject* o)
 
 cdef extern from "arrow/api.h" namespace "arrow" nogil:
     # We can later add more of the common status factory methods as needed
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 8da126aafd586..900c3a597044d 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -871,7 +871,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
                             shared_ptr[CTensor]* out)
 
-    CStatus TensorToNdarray(const CTensor& tensor, object base,
+    CStatus TensorToNdarray(const shared_ptr[CTensor]& tensor, object base,
                             PyObject** out)
 
     CStatus ConvertArrayToPandas(PandasOptions options,
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 3ee02cb8c893e..72315d2dcb24c 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -372,6 +372,23 @@ def test_numpy_immutable(large_buffer):
         result[0] = 1.0
 
 
+def test_numpy_base_object(tmpdir):
+    # ARROW-2040: deserialized Numpy array should keep a reference to the
+    # owner of its memory
+    path = os.path.join(str(tmpdir), 'zzz.bin')
+    data = np.arange(12, dtype=np.int32)
+
+    with open(path, 'wb') as f:
+        f.write(pa.serialize(data).to_buffer())
+
+    serialized = pa.read_serialized(pa.OSFile(path))
+    result = serialized.deserialize()
+    assert_equal(result, data)
+    serialized = None
+    assert_equal(result, data)
+    assert result.base is not None
+
+
 # see https://issues.apache.org/jira/browse/ARROW-1695
 def test_serialization_callback_numpy():
 

From 5321582b9f9df0e47c347a193e09ee833c477d31 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Feb 2018 18:24:46 -0500
Subject: [PATCH 1544/1644] ARROW-2231: [CI] Use clcache on AppVeyor for faster
 builds

clcache is most efficient with Ninja, so this PR switches most of the builds to it.
Also the builds are upgraded to Python 3.6 from 3.5.
Finally, tests are also run on debug builds.

Example build without cache:
https://ci.appveyor.com/project/pitrou/arrow/build/1.0.113

Example build with the cache fully primed:
https://ci.appveyor.com/project/pitrou/arrow/build/1.0.143

Author: Antoine Pitrou <antoine@python.org>

Closes #1679 from pitrou/ARROW-2231-appveyor-clcache and squashes the following commits:

86f97f72 <Antoine Pitrou> ARROW-2231:  Use clcache on AppVeyor for faster builds
---
 appveyor.yml                             | 41 +++++++++++------
 ci/appveyor-setup.bat                    | 56 ++++++++++++++++++++++++
 ci/msvc-build.bat                        | 33 +++++---------
 ci/test-cmake-build-script.bat           |  7 +--
 cpp/src/arrow/ipc/ipc-read-write-test.cc |  3 ++
 5 files changed, 99 insertions(+), 41 deletions(-)
 create mode 100644 ci/appveyor-setup.bat

diff --git a/appveyor.yml b/appveyor.yml
index 22d6158bebbb3..42c0ae34c21ff 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -26,56 +26,69 @@ only_commits:
     - cpp/
     - python/
 
+cache:
+  - C:\Users\Appveyor\clcache
+
 environment:
   matrix:
+    # NOTE: clcache seems to work best with Ninja and worst with msbuild
+    # (as generated by cmake)
     - JOB: "Build"
-      GENERATOR: Visual Studio 15 2017 Win64
-      PYTHON: "3.5"
+      GENERATOR: Ninja
+      PYTHON: "3.6"
       ARCH: "64"
       CONFIGURATION: "Release"
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       BOOST_ROOT: C:\Libraries\boost_1_64_0
     - JOB: "Build_Debug"
-      GENERATOR: Visual Studio 14 2015 Win64
-      PYTHON: "3.5"
+      GENERATOR: Ninja
+      PYTHON: "3.6"
       ARCH: "64"
       CONFIGURATION: "Debug"
     - JOB: "Build"
-      GENERATOR: Visual Studio 14 2015 Win64
-      PYTHON: "3.5"
+      GENERATOR: Ninja
+      PYTHON: "3.6"
       ARCH: "64"
       CONFIGURATION: "Release"
     - JOB: "Static_Crt_Build"
-      GENERATOR: Visual Studio 14 2015 Win64
-      PYTHON: "3.5"
+      GENERATOR: Ninja
+      PYTHON: "3.6"
       ARCH: "64"
     - JOB: "Toolchain"
       GENERATOR: Visual Studio 14 2015 Win64
-      PYTHON: "3.5"
+      PYTHON: "3.6"
       ARCH: "64"
       CONFIGURATION: "Release"
     - JOB: "Cmake_Script_Tests"
       GENERATOR: NMake Makefiles
-      PYTHON: "3.5"
+      PYTHON: "3.6"
       ARCH: "64"
       CONFIGURATION: "Release"
     - JOB: "Build"
       GENERATOR: NMake Makefiles
-      PYTHON: "3.5"
+      PYTHON: "3.6"
       ARCH: "64"
       CONFIGURATION: "Release"
 
   MSVC_DEFAULT_OPTIONS: ON
   BOOST_ROOT: C:\Libraries\boost_1_63_0
   BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
+  APPVEYOR_SAVE_CACHE_ON_ERROR: true
+  USE_CLCACHE: true
 
-init:
-  - set MINICONDA=C:\Miniconda35-x64
+install:
+  - set MINICONDA=C:\Miniconda36-x64
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
-  - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
+  - call ci\appveyor-setup.bat
 
 build_script:
   - git config core.symlinks true
   - git reset --hard
   - if "%JOB%"=="Cmake_Script_Tests" call ci\test-cmake-build-script.bat
   - if NOT "%JOB%"=="Cmake_Script_Tests" call ci\msvc-build.bat
+
+# Disable test discovery
+test: off
+
+after_build:
+  - if "%USE_CLCACHE%" == "true" clcache -s
diff --git a/ci/appveyor-setup.bat b/ci/appveyor-setup.bat
new file mode 100644
index 0000000000000..34f8f1d33ecdb
--- /dev/null
+++ b/ci/appveyor-setup.bat
@@ -0,0 +1,56 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+conda update -y -q conda
+conda config --set auto_update_conda false
+conda info -a
+
+conda config --set show_channel_urls True
+
+@rem Help with SSL timeouts to S3
+conda config --set remote_connect_timeout_secs 12
+
+conda config --add channels https://repo.continuum.io/pkgs/free
+conda config --add channels conda-forge
+conda info -a
+
+if "%GENERATOR%"=="NMake Makefiles" set need_vcvarsall=1
+if "%GENERATOR%"=="Ninja" set need_vcvarsall=1
+
+if defined need_vcvarsall (
+    @rem Select desired compiler version
+    if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" (
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64
+    ) else (
+        call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
+    )
+)
+
+if "%GENERATOR%"=="Ninja" conda install -y -q -c conda-forge ninja
+
+if "%USE_CLCACHE%" == "true" (
+    @rem Use clcache for faster builds
+    pip install -q git+https://github.com/frerich/clcache.git
+    clcache -s
+    set CC=clcache
+    set CXX=clcache
+    set CLCACHE_SERVER=1
+    set CLCACHE_HARDLINK=1
+    powershell.exe -Command "Start-Process clcache-server"
+)
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 069ddf0fc85c4..b6f672da9aeb1 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -29,6 +29,7 @@ if "%JOB%" == "Static_Crt_Build" (
         ..  || exit /B
 
   cmake --build . --config Debug || exit /B
+  ctest -VV  || exit /B
   popd
 
   mkdir cpp\build-release
@@ -60,42 +61,27 @@ if "%JOB%" == "Build_Debug" (
         ..  || exit /B
 
   cmake --build . --config Debug || exit /B
+  ctest -VV  || exit /B
   popd
 
   @rem Finish Debug build successfully
   exit /B 0
 )
 
-conda update --yes --quiet conda
-conda config --set auto_update_conda false
-conda info -a
-
-conda config --set show_channel_urls True
-
-# Help with SSL timeouts to S3
-conda config --set remote_connect_timeout_secs 12
-
-conda config --add channels https://repo.continuum.io/pkgs/free
-conda config --add channels conda-forge
-conda info -a
-
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython ^
       thrift-cpp=0.11.0
 
-if "%JOB%" == "Toolchain" (
+call activate arrow
 
-  conda install -n arrow -q -y -c conda-forge ^
+if "%JOB%" == "Toolchain" (
+  @rem Install pre-built "toolchain" packages for faster builds
+  conda install -q -y -c conda-forge ^
       flatbuffers rapidjson ^
       cmake ^
       git ^
       boost-cpp ^
       snappy zlib brotli gflags lz4-c zstd
-)
-
-call activate arrow
-
-if "%JOB%" == "Toolchain" (
   set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 )
 
@@ -113,12 +99,15 @@ cmake -G "%GENERATOR%" ^
       -DARROW_CXXFLAGS="/WX /MP" ^
       -DARROW_PYTHON=ON ^
       ..  || exit /B
-cmake --build . --target INSTALL --config %CONFIGURATION%  || exit /B
+cmake --build . --target install --config %CONFIGURATION%  || exit /B
 
 @rem Needed so python-test.exe works
+set OLD_PYTHONPATH=%PYTHONPATH%
 set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
 
 ctest -VV  || exit /B
+
+set PYTHONPATH=%OLD_PYTHONPATH%
 popd
 
 @rem Build parquet-cpp
@@ -134,7 +123,7 @@ cmake -G "%GENERATOR%" ^
      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
      -DPARQUET_BOOST_USE_SHARED=OFF ^
      -DPARQUET_BUILD_TESTS=off .. || exit /B
-cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
+cmake --build . --target install --config %CONFIGURATION% || exit /B
 popd
 
 @rem Build and install pyarrow
diff --git a/ci/test-cmake-build-script.bat b/ci/test-cmake-build-script.bat
index 54d892fd40ef1..94857e8233375 100644
--- a/ci/test-cmake-build-script.bat
+++ b/ci/test-cmake-build-script.bat
@@ -17,13 +17,10 @@
 
 @echo on
 
-conda update --yes --quiet conda
-
 @rem Validate cmake script behaviour on missed lib in toolchain
 set CONDA_ENV=arrow-cmake-tests-libs
 conda create -n %CONDA_ENV% -q -y
-conda install -n %CONDA_ENV% -q -y -c conda-forge ^
-cmake git boost-cpp
+conda install -n %CONDA_ENV% -q -y -c conda-forge boost-cpp
 call activate %CONDA_ENV%
 
 set BUILD_DIR=cpp\build-cmake-test
@@ -125,7 +122,7 @@ pushd %BUILD_DIR%
 set CONDA_ENV=arrow-cmake-tests-toolchain
 conda create -n %CONDA_ENV% -q -y
 conda install -n %CONDA_ENV% -q -y -c conda-forge ^
-      flatbuffers rapidjson cmake git boost-cpp ^
+      flatbuffers rapidjson boost-cpp ^
       thrift-cpp snappy zlib brotli gflags lz4-c zstd
 call activate %CONDA_ENV%
 
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index d877e9922e367..c1ff8a448d5de 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -461,6 +461,8 @@ TEST_F(RecursionLimits, ReadLimit) {
   ASSERT_RAISES(Invalid, ReadRecordBatch(*message->metadata(), schema, &reader, &result));
 }
 
+// Test fails with a structured exception on Windows + Debug
+#if !defined(_WIN32) || defined(NDEBUG)
 TEST_F(RecursionLimits, StressLimit) {
   auto CheckDepth = [this](int recursion_depth, bool* it_works) {
     int32_t metadata_length = -1;
@@ -487,6 +489,7 @@ TEST_F(RecursionLimits, StressLimit) {
   CheckDepth(500, &it_works);
   ASSERT_TRUE(it_works);
 }
+#endif  // !defined(_WIN32) || defined(NDEBUG)
 
 class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
  public:

From af2047e5e2cccd0dec6e567e387fc85e600fb284 Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Wed, 28 Feb 2018 16:43:40 -0800
Subject: [PATCH 1545/1644] ARROW-2215: [Plasma] Hugetables munmap issue

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1660 from pcmoritz/fix-huge-pages and squashes the following commits:

4e933e1 [Philipp Moritz] linting
91900ee [Philipp Moritz] fix
aa939bd [Philipp Moritz] fix linting
03da6f0 [Philipp Moritz] introduce constant
61b705d [Philipp Moritz] only try to create huge pages on linux
2b9a294 [Philipp Moritz] fixes
6f124ce [Philipp Moritz] fix test
49cf78f [Philipp Moritz] fix
59ed734 [Philipp Moritz] add tests
4ac737b [Philipp Moritz] linting
adee028 [Philipp Moritz] test if hugepages can be activated on travis
ea7aa11 [Philipp Moritz] fix hugetables munmap issue
---
 ci/travis_script_python.sh          |  8 ++++++++
 cpp/src/plasma/client.cc            | 11 ++++++++---
 cpp/src/plasma/malloc.cc            | 10 +++++-----
 cpp/src/plasma/malloc.h             |  6 ++++++
 python/pyarrow/tests/test_plasma.py | 18 +++++++++++++++++-
 5 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 9ed5825bb8ffa..a776c42630816 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -96,6 +96,14 @@ if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then
   export PLASMA_VALGRIND=1
 fi
 
+# Set up huge pages for plasma test
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+    sudo mkdir -p /mnt/hugepages
+    sudo mount -t hugetlbfs -o uid=`id -u` -o gid=`id -g` none /mnt/hugepages
+    sudo bash -c "echo `id -g` > /proc/sys/vm/hugetlb_shm_group"
+    sudo bash -c "echo 20000 > /proc/sys/vm/nr_hugepages"
+fi
+
 PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
 python -m pytest -vv -r sxX --durations=15 -s $PYARROW_PATH --parquet
 
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
index 679d9cecda8f1..a9bbd8cc43f87 100644
--- a/cpp/src/plasma/client.cc
+++ b/cpp/src/plasma/client.cc
@@ -45,6 +45,7 @@
 #include "plasma/common.h"
 #include "plasma/fling.h"
 #include "plasma/io.h"
+#include "plasma/malloc.h"
 #include "plasma/plasma.h"
 #include "plasma/protocol.h"
 
@@ -117,8 +118,10 @@ uint8_t* PlasmaClient::lookup_or_mmap(int fd, int store_fd_val, int64_t map_size
     close(fd);
     return entry->second.pointer;
   } else {
-    uint8_t* result = reinterpret_cast<uint8_t*>(
-        mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
+    // We subtract kMmapRegionsGap from the length that was added
+    // in fake_mmap in malloc.h, to make map_size page-aligned again.
+    uint8_t* result = reinterpret_cast<uint8_t*>(mmap(
+        NULL, map_size - kMmapRegionsGap, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
     // TODO(pcm): Don't fail here, instead return a Status.
     if (result == MAP_FAILED) {
       ARROW_LOG(FATAL) << "mmap failed";
@@ -395,7 +398,9 @@ Status PlasmaClient::UnmapObject(const ObjectID& object_id) {
   ARROW_CHECK(entry->second.count >= 1);
   if (entry->second.count == 1) {
     // If no other objects are being used, then unmap the file.
-    int err = munmap(entry->second.pointer, entry->second.length);
+    // We subtract kMmapRegionsGap from the length that was added
+    // in fake_mmap in malloc.h, to make the size page-aligned again.
+    int err = munmap(entry->second.pointer, entry->second.length - kMmapRegionsGap);
     if (err == -1) {
       return Status::IOError("Error during munmap");
     }
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
index 3c5d107b2bbe3..e2403fd6177a6 100644
--- a/cpp/src/plasma/malloc.cc
+++ b/cpp/src/plasma/malloc.cc
@@ -127,10 +127,10 @@ int create_buffer(int64_t size) {
 }
 
 void* fake_mmap(size_t size) {
-  // Add sizeof(size_t) so that the returned pointer is deliberately not
+  // Add kMmapRegionsGap so that the returned pointer is deliberately not
   // page-aligned. This ensures that the segments of memory returned by
   // fake_mmap are never contiguous.
-  size += sizeof(size_t);
+  size += kMmapRegionsGap;
 
   int fd = create_buffer(size);
   ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
@@ -155,15 +155,15 @@ void* fake_mmap(size_t size) {
   record.size = size;
 
   // We lie to dlmalloc about where mapped memory actually lives.
-  pointer = pointer_advance(pointer, sizeof(size_t));
+  pointer = pointer_advance(pointer, kMmapRegionsGap);
   ARROW_LOG(DEBUG) << pointer << " = fake_mmap(" << size << ")";
   return pointer;
 }
 
 int fake_munmap(void* addr, int64_t size) {
   ARROW_LOG(DEBUG) << "fake_munmap(" << addr << ", " << size << ")";
-  addr = pointer_retreat(addr, sizeof(size_t));
-  size += sizeof(size_t);
+  addr = pointer_retreat(addr, kMmapRegionsGap);
+  size += kMmapRegionsGap;
 
   auto entry = mmap_records.find(addr);
 
diff --git a/cpp/src/plasma/malloc.h b/cpp/src/plasma/malloc.h
index cb8c600b14b3b..c24f15456d677 100644
--- a/cpp/src/plasma/malloc.h
+++ b/cpp/src/plasma/malloc.h
@@ -21,6 +21,12 @@
 #include <inttypes.h>
 #include <stddef.h>
 
+/// Gap between two consecutive mmap regions allocated by fake_mmap.
+/// This ensures that the segments of memory returned by
+/// fake_mmap are never contiguous and dlmalloc does not coalesce it
+/// (in the client we cannot guarantee that these mmaps are contiguous).
+constexpr int64_t kMmapRegionsGap = sizeof(size_t);
+
 void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_length, ptrdiff_t* offset);
 
 /// Get the mmap size corresponding to a specific file descriptor.
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index 27556e60d3bf0..0df627fe6a6cb 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -105,7 +105,8 @@ def assert_get_object_equal(unit_test, client1, client2, object_id,
 def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
                        use_valgrind=False, use_profiler=False,
                        stdout_file=None, stderr_file=None,
-                       use_one_memory_mapped_file=False):
+                       use_one_memory_mapped_file=False,
+                       plasma_directory=None, use_hugepages=False):
     """Start a plasma store process.
     Args:
         use_valgrind (bool): True if the plasma store should be started inside
@@ -131,6 +132,10 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
                "-m", str(plasma_store_memory)]
     if use_one_memory_mapped_file:
         command += ["-f"]
+    if plasma_directory:
+        command += ["-d", plasma_directory]
+    if use_hugepages:
+        command += ["-h"]
     if use_valgrind:
         pid = subprocess.Popen(["valgrind",
                                 "--track-origins=yes",
@@ -762,3 +767,14 @@ def test_object_id_size():
     with pytest.raises(ValueError):
         plasma.ObjectID("hello")
     plasma.ObjectID(20 * b"0")
+
+
+@pytest.mark.skipif(not os.path.exists("/mnt/hugepages"),
+                    reason="requires hugepage support")
+def test_use_huge_pages():
+    import pyarrow.plasma as plasma
+    plasma_store_name, p = start_plasma_store(
+        plasma_directory="/mnt/hugepages", use_hugepages=True)
+    plasma_client = plasma.connect(plasma_store_name, "", 64)
+    create_object(plasma_client, 100000000)
+    p.kill()

From 8b3bbaea8e1787184df1de22f4eb596584ea7044 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 28 Feb 2018 23:31:44 -0500
Subject: [PATCH 1546/1644] ARROW-2198: [Python] correct docstring for
 parquet.read_table

cc @wesm

Author: Wes McKinney <wes.mckinney@twosigma.com>
Author: siddharth <siddharth@dremio.com>

Closes #1654 from siddharthteotia/ARROW-2198 and squashes the following commits:

35ff194d <Wes McKinney> Refactor read_table, read_pandas docstrings, better code reuse
506f874b <siddharth> ARROW-2198:  correct docstring for parquet.read_table
---
 python/pyarrow/parquet.py | 105 +++++++++++++++++---------------------
 1 file changed, 48 insertions(+), 57 deletions(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 485459e0b24ca..f46ce9481a9ba 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -42,9 +42,9 @@ class ParquetFile(object):
 
     Parameters
     ----------
-    source : str or pyarrow.io.NativeFile
-        Readable source. For passing Python file objects or byte buffers,
-        see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
+    source : str, pyarrow.NativeFile, or file-like object
+        Readable source. For passing bytes or buffer-like file containing a
+        Parquet file, use pyarorw.BufferReader
     metadata : ParquetFileMetadata, default None
         Use existing metadata object, rather than reading from file.
     common_metadata : ParquetFileMetadata, default None
@@ -862,35 +862,34 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
     return pieces, partitions, metadata_path
 
 
-def read_table(source, columns=None, nthreads=1, metadata=None,
-               use_pandas_metadata=False):
-    """
-    Read a Table from Parquet format
+_read_table_docstring = """
+{0}
 
-    Parameters
-    ----------
-    source: str or pyarrow.io.NativeFile
-        Location of Parquet dataset. If a string passed, can be a single file
-        name or directory name. For passing Python file objects or byte
-        buffers, see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
-    columns: list
-        If not None, only these columns will be read from the file. A column
-        name may be a prefix of a nested field, e.g. 'a' will select 'a.b',
-        'a.c', and 'a.d.e'
-    nthreads : int, default 1
-        Number of columns to read in parallel. Requires that the underlying
-        file source is threadsafe
-    metadata : FileMetaData
-        If separately computed
-    use_pandas_metadata : boolean, default False
-        If True and file has custom pandas schema metadata, ensure that
-        index columns are also loaded
+Parameters
+----------
+source: str, pyarrow.NativeFile, or file-like object
+    If a string passed, can be a single file name or directory name. For
+    file-like objects, only read a single file. Use pyarrow.BufferReader to
+    read a file contained in a bytes or buffer-like object
+columns: list
+    If not None, only these columns will be read from the file. A column
+    name may be a prefix of a nested field, e.g. 'a' will select 'a.b',
+    'a.c', and 'a.d.e'
+nthreads : int, default 1
+    Number of columns to read in parallel. Requires that the underlying
+    file source is threadsafe
+metadata : FileMetaData
+    If separately computed
+{1}
+
+Returns
+-------
+{2}
+"""
 
-    Returns
-    -------
-    pyarrow.Table
-        Content of the file as a table (of columns)
-    """
+
+def read_table(source, columns=None, nthreads=1, metadata=None,
+               use_pandas_metadata=False):
     if is_string(source):
         fs = LocalFileSystem.get_instance()
         if fs.isdir(source):
@@ -902,37 +901,29 @@ def read_table(source, columns=None, nthreads=1, metadata=None,
                    use_pandas_metadata=use_pandas_metadata)
 
 
-def read_pandas(source, columns=None, nthreads=1, metadata=None):
-    """
-    Read a Table from Parquet format, also reading DataFrame index values if
-    known in the file metadata
+read_table.__doc__ = _read_table_docstring.format(
+    'Read a Table from Parquet format',
+    """use_pandas_metadata : boolean, default False
+    If True and file has custom pandas schema metadata, ensure that
+    index columns are also loaded""",
+    """pyarrow.Table
+    Content of the file as a table (of columns)""")
 
-    Parameters
-    ----------
-    source: str or pyarrow.io.NativeFile
-        Location of Parquet dataset. If a string passed, can be a single file
-        name. For passing Python file objects or byte buffers,
-        see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
-    columns: list
-        If not None, only these columns will be read from the file. A column
-        name may be a prefix of a nested field, e.g. 'a' will select 'a.b',
-        'a.c', and 'a.d.e'
-    nthreads : int, default 1
-        Number of columns to read in parallel. Requires that the underlying
-        file source is threadsafe
-    metadata : FileMetaData
-        If separately computed
 
-    Returns
-    -------
-    pyarrow.Table
-        Content of the file as a Table of Columns, including DataFrame indexes
-        as Columns.
-    """
+def read_pandas(source, columns=None, nthreads=1, metadata=None):
     return read_table(source, columns=columns, nthreads=nthreads,
                       metadata=metadata, use_pandas_metadata=True)
 
 
+read_pandas.__doc__ = _read_table_docstring.format(
+    'Read a Table from Parquet format, also reading DataFrame\n'
+    'index values if known in the file metadata',
+    '',
+    """pyarrow.Table
+    Content of the file as a Table of Columns, including DataFrame
+    indexes as columns""")
+
+
 def write_table(table, where, row_group_size=None, version='1.0',
                 use_dictionary=True, compression='snappy',
                 use_deprecated_int96_timestamps=None,
@@ -966,7 +957,7 @@ def write_table(table, where, row_group_size=None, version='1.0',
 Parameters
 ----------
 table : pyarrow.Table
-where: string or pyarrow.io.NativeFile
+where: string or pyarrow.NativeFile
 {0}
 """.format(_parquet_writer_arg_docs)
 
@@ -1064,7 +1055,7 @@ def write_metadata(schema, where, version='1.0',
     Parameters
     ----------
     schema : pyarrow.Schema
-    where: string or pyarrow.io.NativeFile
+    where: string or pyarrow.NativeFile
     version : {"1.0", "2.0"}, default "1.0"
         The Parquet format version, defaults to 1.0
     use_deprecated_int96_timestamps : boolean, default False

From b2eb6ac90f9fd09d2bd2855b9c6121ea789ea6a0 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 28 Feb 2018 23:34:41 -0500
Subject: [PATCH 1547/1644] ARROW-1632: [Python] Permit categorical conversions
 in Table.to_pandas on a per-column basis
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is failing due to the first entry of the indices being uninitialised memory in the pandas series. On the C++ side, everything looks good. Would be nice if someone could have a look, I‘m out of ideas.

Author: Uwe L. Korn <uwelk@xhochy.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1620 from xhochy/ARROW-1632 and squashes the following commits:

17ca4e18 <Uwe L. Korn> Check status codes
8d1e9995 <Wes McKinney> Fix flake
27fe924e <Uwe L. Korn> Ensure underlying index array is kept alive
7ae6a46e <Uwe L. Korn> ARROW-1632:  Permit categorical conversions in Table.to_pandas on a per-column basis
---
 cpp/src/arrow/python/arrow_to_pandas.cc     | 59 ++++++++++++++++++---
 cpp/src/arrow/python/arrow_to_pandas.h      | 12 +++++
 python/pyarrow/includes/common.pxd          |  1 +
 python/pyarrow/includes/libarrow.pxd        | 10 ++--
 python/pyarrow/pandas_compat.py             |  9 ++--
 python/pyarrow/table.pxi                    | 16 ++++--
 python/pyarrow/tests/test_convert_pandas.py | 18 +++++++
 7 files changed, 106 insertions(+), 19 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 125892afe59b2..aefd4d76dfa0f 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -60,6 +60,7 @@ using internal::kNanosecondsInDay;
 using internal::kPandasTimestampNull;
 
 using compute::Datum;
+using compute::FunctionContext;
 
 // ----------------------------------------------------------------------
 // Utility code
@@ -986,7 +987,8 @@ class CategoricalBlock : public PandasBlock {
     // Sniff the first chunk
     const std::shared_ptr<Array> arr_first = data.chunk(0);
     const auto& dict_arr_first = static_cast<const DictionaryArray&>(*arr_first);
-    const auto& indices_first = static_cast<const ArrayType&>(*dict_arr_first.indices());
+    const auto indices_first =
+        std::static_pointer_cast<ArrayType>(dict_arr_first.indices());
 
     auto CheckIndices = [](const ArrayType& arr, int64_t dict_length) {
       const T* values = arr.raw_values();
@@ -1000,8 +1002,8 @@ class CategoricalBlock : public PandasBlock {
       return Status::OK();
     };
 
-    if (!needs_copy_ && data.num_chunks() == 1 && indices_first.null_count() == 0) {
-      RETURN_NOT_OK(CheckIndices(indices_first, dict_arr_first.dictionary()->length()));
+    if (!needs_copy_ && data.num_chunks() == 1 && indices_first->null_count() == 0) {
+      RETURN_NOT_OK(CheckIndices(*indices_first, dict_arr_first.dictionary()->length()));
       RETURN_NOT_OK(AllocateNDArrayFromIndices<T>(npy_type, indices_first));
     } else {
       if (options_.zero_copy_only) {
@@ -1011,7 +1013,7 @@ class CategoricalBlock : public PandasBlock {
              << "but only zero-copy conversions allowed.";
         } else {
           ss << "Needed to copy " << data.num_chunks() << " chunks with "
-             << indices_first.null_count()
+             << indices_first->null_count()
              << " indices nulls, but zero_copy_only was True";
         }
         return Status::Invalid(ss.str());
@@ -1109,10 +1111,11 @@ class CategoricalBlock : public PandasBlock {
 
  protected:
   template <typename T>
-  Status AllocateNDArrayFromIndices(int npy_type, const PrimitiveArray& indices) {
+  Status AllocateNDArrayFromIndices(int npy_type,
+                                    const std::shared_ptr<PrimitiveArray>& indices) {
     npy_intp block_dims[1] = {num_rows_};
 
-    const T* in_values = GetPrimitiveValues<T>(indices);
+    const T* in_values = GetPrimitiveValues<T>(*indices);
     void* data = const_cast<T*>(in_values);
 
     PyAcquireGIL lock;
@@ -1127,6 +1130,22 @@ class CategoricalBlock : public PandasBlock {
                                                nullptr, data, NPY_ARRAY_CARRAY, nullptr);
     RETURN_IF_PYERROR();
 
+    // Add a reference to the underlying Array. Otherwise the array may be
+    // deleted once we leave the block conversion.
+    auto capsule = new ArrowCapsule{{indices}};
+    PyObject* base = PyCapsule_New(reinterpret_cast<void*>(capsule), "arrow",
+                                   &ArrowCapsule_Destructor);
+    if (base == nullptr) {
+      delete capsule;
+      RETURN_IF_PYERROR();
+    }
+
+    if (PyArray_SetBaseObject(reinterpret_cast<PyArrayObject*>(block_arr), base) == -1) {
+      // Error occurred, trust that SetBaseObject set the error state
+      Py_XDECREF(base);
+      return Status::OK();
+    }
+
     npy_intp placement_dims[1] = {num_columns_};
     PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64);
     RETURN_IF_PYERROR();
@@ -1771,7 +1790,33 @@ Status ConvertColumnToPandas(PandasOptions options, const std::shared_ptr<Column
 
 Status ConvertTableToPandas(PandasOptions options, const std::shared_ptr<Table>& table,
                             int nthreads, MemoryPool* pool, PyObject** out) {
-  DataFrameBlockCreator helper(options, table, pool);
+  return ConvertTableToPandas(options, std::unordered_set<std::string>(), table, nthreads,
+                              pool, out);
+}
+
+Status ConvertTableToPandas(PandasOptions options,
+                            const std::unordered_set<std::string>& categorical_columns,
+                            const std::shared_ptr<Table>& table, int nthreads,
+                            MemoryPool* pool, PyObject** out) {
+  std::shared_ptr<Table> current_table = table;
+  if (!categorical_columns.empty()) {
+    FunctionContext ctx;
+    for (int i = 0; i < table->num_columns(); i++) {
+      const Column& col = *table->column(i);
+      if (categorical_columns.count(col.name())) {
+        Datum out;
+        RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col.data()), &out));
+        std::shared_ptr<ChunkedArray> array = out.chunked_array();
+        auto field = std::make_shared<Field>(
+            col.name(), array->type(), col.field()->nullable(), col.field()->metadata());
+        auto column = std::make_shared<Column>(field, array);
+        RETURN_NOT_OK(current_table->RemoveColumn(i, &current_table));
+        RETURN_NOT_OK(current_table->AddColumn(i, column, &current_table));
+      }
+    }
+  }
+
+  DataFrameBlockCreator helper(options, current_table, pool);
   return helper.Convert(nthreads, out);
 }
 
diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h
index 1e486463708ac..0541b0f9a295b 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.h
+++ b/cpp/src/arrow/python/arrow_to_pandas.h
@@ -25,6 +25,7 @@
 
 #include <memory>
 #include <string>
+#include <unordered_set>
 
 #include "arrow/util/visibility.h"
 
@@ -40,6 +41,7 @@ class Table;
 namespace py {
 
 struct PandasOptions {
+  /// If true, we will convert all string columns to categoricals
   bool strings_to_categorical;
   bool zero_copy_only;
 
@@ -64,6 +66,16 @@ ARROW_EXPORT
 Status ConvertTableToPandas(PandasOptions options, const std::shared_ptr<Table>& table,
                             int nthreads, MemoryPool* pool, PyObject** out);
 
+/// Convert a whole table as efficiently as possible to a pandas.DataFrame.
+///
+/// Explicitly name columns that should be a categorical
+/// This option is only used on conversions that are applied to a table.
+ARROW_EXPORT
+Status ConvertTableToPandas(PandasOptions options,
+                            const std::unordered_set<std::string>& categorical_columns,
+                            const std::shared_ptr<Table>& table, int nthreads,
+                            MemoryPool* pool, PyObject** out);
+
 }  // namespace py
 }  // namespace arrow
 
diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd
index 4d799ecd2b434..5afa0753726fa 100644
--- a/python/pyarrow/includes/common.pxd
+++ b/python/pyarrow/includes/common.pxd
@@ -23,6 +23,7 @@ from libcpp.memory cimport shared_ptr, unique_ptr, make_shared
 from libcpp.string cimport string as c_string
 from libcpp.vector cimport vector
 from libcpp.unordered_map cimport unordered_map
+from libcpp.unordered_set cimport unordered_set
 
 from cpython cimport PyObject
 cimport cpython
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 900c3a597044d..b9abf2b163b40 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -882,10 +882,12 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
                                   const shared_ptr[CColumn]& arr,
                                   object py_ref, PyObject** out)
 
-    CStatus ConvertTableToPandas(PandasOptions options,
-                                 const shared_ptr[CTable]& table,
-                                 int nthreads, CMemoryPool* pool,
-                                 PyObject** out)
+    CStatus ConvertTableToPandas(
+        PandasOptions options,
+        const unordered_set[c_string]& categorical_columns,
+        const shared_ptr[CTable]& table,
+        int nthreads, CMemoryPool* pool,
+        PyObject** out)
 
     void c_set_default_memory_pool \
         " arrow::py::set_default_memory_pool"(CMemoryPool* pool)\
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 6d4bf5e787079..0bc47fc0d6090 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -490,7 +490,7 @@ def _make_datetimetz(tz):
 
 
 def table_to_blockmanager(options, table, memory_pool, nthreads=1,
-                          categoricals=None):
+                          categories=None):
     from pyarrow.compat import DatetimeTZDtype
 
     index_columns = []
@@ -564,7 +564,8 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
                 block_table.schema.get_field_index(raw_name)
             )
 
-    blocks = _table_to_blocks(options, block_table, nthreads, memory_pool)
+    blocks = _table_to_blocks(options, block_table, nthreads, memory_pool,
+                              categories)
 
     # Construct the row index
     if len(index_arrays) > 1:
@@ -651,12 +652,12 @@ def _reconstruct_columns_from_metadata(columns, column_indexes):
     )
 
 
-def _table_to_blocks(options, block_table, nthreads, memory_pool):
+def _table_to_blocks(options, block_table, nthreads, memory_pool, categories):
     # Part of table_to_blockmanager
 
     # Convert an arrow table to Block from the internal pandas API
     result = pa.lib.table_to_blocks(options, block_table, nthreads,
-                                    memory_pool)
+                                    memory_pool, categories)
 
     # Defined above
     return [_reconstruct_block(item) for item in result]
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index e14d4739f805e..178df57671da5 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -746,17 +746,22 @@ cdef class RecordBatch:
 
 
 def table_to_blocks(PandasOptions options, Table table, int nthreads,
-                    MemoryPool memory_pool):
+                    MemoryPool memory_pool, categories):
     cdef:
         PyObject* result_obj
         shared_ptr[CTable] c_table = table.sp_table
         CMemoryPool* pool
+        unordered_set[c_string] categorical_columns
+
+    if categories is not None:
+        categorical_columns = {tobytes(cat) for cat in categories}
 
     pool = maybe_unbox_memory_pool(memory_pool)
     with nogil:
         check_status(
             libarrow.ConvertTableToPandas(
-                options, c_table, nthreads, pool, &result_obj
+                options, categorical_columns, c_table, nthreads, pool,
+                &result_obj
             )
         )
 
@@ -1012,7 +1017,7 @@ cdef class Table:
         return result
 
     def to_pandas(self, nthreads=None, strings_to_categorical=False,
-                  memory_pool=None, zero_copy_only=False):
+                  memory_pool=None, zero_copy_only=False, categories=None):
         """
         Convert the arrow::Table to a pandas DataFrame
 
@@ -1029,6 +1034,8 @@ cdef class Table:
         zero_copy_only : boolean, default False
             Raise an ArrowException if this function call would require copying
             the underlying data
+        categories: list, default empty
+            List of columns that should be returned as pandas.Categorical
 
         Returns
         -------
@@ -1036,6 +1043,7 @@ cdef class Table:
         """
         cdef:
             PandasOptions options
+
         options = PandasOptions(
             strings_to_categorical=strings_to_categorical,
             zero_copy_only=zero_copy_only)
@@ -1043,7 +1051,7 @@ cdef class Table:
         if nthreads is None:
             nthreads = cpu_count()
         mgr = pdcompat.table_to_blockmanager(options, self, memory_pool,
-                                             nthreads)
+                                             nthreads, categories)
         return pd.DataFrame(mgr)
 
     def to_pydict(self):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 6e68dd961bef8..986aeffcacc9c 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1027,6 +1027,24 @@ def test_table_empty_str(self):
         expected2 = pd.DataFrame({'strings': pd.Categorical(values)})
         tm.assert_frame_equal(result2, expected2, check_dtype=True)
 
+    def test_selective_categoricals(self):
+        values = ['', '', '', '', '']
+        df = pd.DataFrame({'strings': values})
+        field = pa.field('strings', pa.string())
+        schema = pa.schema([field])
+        table = pa.Table.from_pandas(df, schema=schema)
+        expected_str = pd.DataFrame({'strings': values})
+        expected_cat = pd.DataFrame({'strings': pd.Categorical(values)})
+
+        result1 = table.to_pandas(categories=['strings'])
+        tm.assert_frame_equal(result1, expected_cat, check_dtype=True)
+        result2 = table.to_pandas(categories=[])
+        tm.assert_frame_equal(result2, expected_str, check_dtype=True)
+        result3 = table.to_pandas(categories=('strings',))
+        tm.assert_frame_equal(result3, expected_cat, check_dtype=True)
+        result4 = table.to_pandas(categories=tuple())
+        tm.assert_frame_equal(result4, expected_str, check_dtype=True)
+
     def test_table_str_to_categorical_without_na(self):
         values = ['a', 'a', 'b', 'b', 'c']
         df = pd.DataFrame({'strings': values})

From bfac60dd73bffa5f7bcefc890486268036182278 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 1 Mar 2018 17:27:14 -0500
Subject: [PATCH 1548/1644] 
 ARROW-2145/ARROW-2153/ARROW-2157/ARROW-2160/ARROW-2177: [Python] Decimal
 conversion not working for NaN values

This PR closes the following JIRAs

ARROW-2145: [Python] Decimal conversion not working for NaN values
ARROW-2153: [C++/Python] Decimal conversion not working for exponential notation
ARROW-2157: [Python] Decimal arrays cannot be constructed from Python lists
ARROW-2160: [C++/Python] Fix decimal precision inference
ARROW-2177: [C++] Remove support for specifying negative scale values in DecimalType

I originally separated these fixes into a few smaller PRs, but it turned out
that the issues were all related, so I fixed them all in one PR.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1651 from cpcloud/ARROW-2145-2153-2157-2160 and squashes the following commits:

97fcb961 <Phillip Cloud> Make sure we only install if glibc is affected
1fc2a963 <Phillip Cloud> Args must be a ruby Hash
0d45688a <Phillip Cloud> Pass version as argument
ab3e4a54 <Phillip Cloud> Brewfile
00be5781 <Phillip Cloud> Add tests to accommodate decimal values
99505a9f <Phillip Cloud> Silence cmake complaints about boost version
ae5db5fd <Phillip Cloud> Perms
03ee9995 <Phillip Cloud> Fix boost root
78cbf51c <Phillip Cloud> More script debugging
b4bcfd97 <Phillip Cloud> DCHECK_OK for release builds
29e1ebc4 <Phillip Cloud> boost osx debugging
4e6db3c7 <Phillip Cloud> Formatting
a05b3161 <Phillip Cloud> Refactor import decimal and acquire the gil before importing
3190b1a3 <Phillip Cloud> Ignore nans in decimal metadata update
b24ff259 <Phillip Cloud> Add DecimalMetadata::Update test for ignoring NaN values
418754ff <Phillip Cloud> Check return value of PyList_SetItem
092a9624 <Phillip Cloud> Fix order of operands
db664f22 <Phillip Cloud> DCHECK_Ok
1df6923d <Phillip Cloud> DCHECK_OK
281f7984 <Phillip Cloud> DCHECK_OK
d9052029 <Phillip Cloud> DCHECK_OK
4c74c63c <Phillip Cloud> NULLPTR to nullptr
77a41ee1 <Phillip Cloud> Install boost first
7c7270a0 <Phillip Cloud> Show boost install
8be22a69 <Phillip Cloud> Install boost with c++11 option
50e35d6c <Phillip Cloud> Use shared boost on parquet CI build
e6ac8646 <Phillip Cloud> Install libboost-regex-dev on travis
0665f6e9 <Phillip Cloud> Revert test change
8893a45c <Phillip Cloud> Revert header change
f562378b <Phillip Cloud> IWYU
8e816ec2 <Phillip Cloud> ARROW-2145:  Decimal conversion not working for NaN values
---
 .travis.yml                                  |   5 +-
 c_glib/Brewfile                              |   2 +-
 ci/travis_before_script_c_glib.sh            |   4 +-
 ci/travis_before_script_cpp.sh               |  24 +-
 ci/travis_build_parquet_cpp.sh               |   2 +-
 ci/travis_install_linux.sh                   |   2 +-
 ci/travis_install_osx.sh                     |  23 ++
 cpp/CMakeLists.txt                           |   6 +-
 cpp/cmake_modules/ThirdpartyToolchain.cmake  |  22 +-
 cpp/src/arrow/python/arrow_to_pandas.cc      |  13 +-
 cpp/src/arrow/python/builtin_convert.cc      |  40 ++-
 cpp/src/arrow/python/helpers.cc              |  98 ++++++-
 cpp/src/arrow/python/helpers.h               |  79 ++++-
 cpp/src/arrow/python/numpy-internal.h        |   3 +
 cpp/src/arrow/python/numpy_to_arrow.cc       |  88 ++----
 cpp/src/arrow/python/python-test.cc          | 184 ++++++++++--
 cpp/src/arrow/util/decimal-test.cc           |  73 ++---
 cpp/src/arrow/util/decimal.cc                | 290 ++++++++-----------
 cpp/src/arrow/util/decimal.h                 |   2 +-
 cpp/src/arrow/util/logging.h                 |   6 +-
 python/pyarrow/tests/test_convert_builtin.py |  10 +
 python/pyarrow/tests/test_convert_pandas.py  |  14 +
 22 files changed, 660 insertions(+), 330 deletions(-)
 create mode 100755 ci/travis_install_osx.sh

diff --git a/.travis.yml b/.travis.yml
index a4c74657e5257..b1241e793bc70 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -174,7 +174,7 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
-  # [OS X] C++ & glib w/ XCode 8.3 & autotools
+  # [OS X] C++ & glib w/ XCode 8.3 & autotools & homebrew
   - compiler: clang
     osx_image: xcode8.3
     os: osx
@@ -185,7 +185,8 @@ matrix:
     - BUILD_SYSTEM=autotools
     before_script:
     - if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+    - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
diff --git a/c_glib/Brewfile b/c_glib/Brewfile
index 9fe5c3b616317..955072e1ea767 100644
--- a/c_glib/Brewfile
+++ b/c_glib/Brewfile
@@ -16,7 +16,7 @@
 # under the License.
 
 brew "autoconf-archive"
-brew "boost"
+brew "boost", args: ["1.65.0"]
 brew "ccache"
 brew "cmake"
 brew "git"
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 27d1e86fd95f6..033fbd7c6cecf 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -21,9 +21,7 @@ set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-if [ $TRAVIS_OS_NAME = "osx" ]; then
-  brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile
-else  # Linux
+if [ $TRAVIS_OS_NAME = "linux" ]; then
   sudo apt-get install -y -q gtk-doc-tools autoconf-archive libgirepository1.0-dev
 fi
 
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 17b5deb36b519..b9afbee7863f1 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -22,10 +22,22 @@ set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-if [ "$1" == "--only-library" ]; then
-  only_library_mode=yes
-else
-  only_library_mode=no
+only_library_mode=no
+using_homebrew=no
+
+while true; do
+    case "$1" in
+	--only-library)
+	    only_library_mode=yes
+	    shift ;;
+	--homebrew)
+	    using_homebrew=yes
+	    shift ;;
+	*) break ;;
+    esac
+done
+
+if [ "$only_library_mode" == "no" ]; then
   source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 fi
 
@@ -78,6 +90,10 @@ if [ $TRAVIS_OS_NAME == "linux" ]; then
           -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \
           $ARROW_CPP_DIR
 else
+    if [ "$using_homebrew" = "yes" ]; then
+	# build against homebrew's boost if we're using it
+	export BOOST_ROOT=/usr/local/opt/boost
+    fi
     cmake $CMAKE_COMMON_FLAGS \
           $CMAKE_OSX_FLAGS \
           -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
diff --git a/ci/travis_build_parquet_cpp.sh b/ci/travis_build_parquet_cpp.sh
index 7d2e3ab7364fa..f64a85d621dcc 100755
--- a/ci/travis_build_parquet_cpp.sh
+++ b/ci/travis_build_parquet_cpp.sh
@@ -38,7 +38,7 @@ cmake \
     -GNinja \
     -DCMAKE_BUILD_TYPE=debug \
     -DCMAKE_INSTALL_PREFIX=$ARROW_PYTHON_PARQUET_HOME \
-    -DPARQUET_BOOST_USE_SHARED=off \
+    -DPARQUET_BOOST_USE_SHARED=on \
     -DPARQUET_BUILD_BENCHMARKS=off \
     -DPARQUET_BUILD_EXECUTABLES=off \
     -DPARQUET_BUILD_TESTS=off \
diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh
index acee9ebcb2ac3..74fde277413b7 100755
--- a/ci/travis_install_linux.sh
+++ b/ci/travis_install_linux.sh
@@ -19,7 +19,7 @@
 
 sudo apt-get install -y -q \
     gdb ccache libboost-dev libboost-filesystem-dev \
-    libboost-system-dev libjemalloc-dev
+    libboost-system-dev libboost-regex-dev libjemalloc-dev
 
 if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then
     sudo apt-get install -y -q valgrind
diff --git a/ci/travis_install_osx.sh b/ci/travis_install_osx.sh
new file mode 100755
index 0000000000000..b03a5f16a31fe
--- /dev/null
+++ b/ci/travis_install_osx.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ "$ARROW_CI_C_GLIB_AFFECTED" = "1" ]; then
+    brew update
+    brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile
+fi
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 8c0e95634fc1f..47692a831fa35 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -574,11 +574,13 @@ set(ARROW_LINK_LIBS
 
 set(ARROW_SHARED_PRIVATE_LINK_LIBS
   ${BOOST_SYSTEM_LIBRARY}
-  ${BOOST_FILESYSTEM_LIBRARY})
+  ${BOOST_FILESYSTEM_LIBRARY}
+  ${BOOST_REGEX_LIBRARY})
 
 set(ARROW_STATIC_PRIVATE_LINK_LIBS
   ${BOOST_SYSTEM_LIBRARY}
-  ${BOOST_FILESYSTEM_LIBRARY})
+  ${BOOST_FILESYSTEM_LIBRARY}
+  ${BOOST_REGEX_LIBRARY})
 
 if (NOT MSVC)
   set(ARROW_LINK_LIBS
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 944ca1d3bb1c2..4103af41b53ad 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -157,8 +157,11 @@ if (ARROW_BOOST_VENDORED)
     "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(BOOST_STATIC_FILESYSTEM_LIBRARY
     "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(BOOST_STATIC_REGEX_LIBRARY
+          "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
   set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
+  set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
   if (ARROW_BOOST_HEADER_ONLY)
     set(BOOST_BUILD_PRODUCTS)
     set(BOOST_CONFIGURE_COMMAND "")
@@ -166,7 +169,8 @@ if (ARROW_BOOST_VENDORED)
   else()
     set(BOOST_BUILD_PRODUCTS
       ${BOOST_SYSTEM_LIBRARY}
-      ${BOOST_FILESYSTEM_LIBRARY})
+      ${BOOST_FILESYSTEM_LIBRARY}
+      ${BOOST_REGEX_LIBRARY})
     set(BOOST_CONFIGURE_COMMAND
       "./bootstrap.sh"
       "--prefix=${BOOST_PREFIX}"
@@ -210,16 +214,19 @@ else()
     if (ARROW_BOOST_HEADER_ONLY)
       find_package(Boost REQUIRED)
     else()
-      find_package(Boost COMPONENTS system filesystem REQUIRED)
+      find_package(Boost COMPONENTS system filesystem regex REQUIRED)
       if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
         set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
         set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+        set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
       else()
         set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
         set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+        set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
       endif()
       set(BOOST_SYSTEM_LIBRARY boost_system_shared)
       set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
+      set(BOOST_REGEX_LIBRARY boost_regex_shared)
     endif()
   else()
     # Find static boost headers and libs
@@ -228,16 +235,19 @@ else()
     if (ARROW_BOOST_HEADER_ONLY)
       find_package(Boost REQUIRED)
     else()
-      find_package(Boost COMPONENTS system filesystem REQUIRED)
+      find_package(Boost COMPONENTS system filesystem regex REQUIRED)
       if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
         set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
         set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+        set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
       else()
         set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
         set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+        set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
       endif()
       set(BOOST_SYSTEM_LIBRARY boost_system_static)
       set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
+      set(BOOST_REGEX_LIBRARY boost_regex_static)
     endif()
   endif()
 endif()
@@ -254,7 +264,11 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
       STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
       SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
 
-  SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
+  ADD_THIRDPARTY_LIB(boost_regex
+      STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
+      SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")
+
+  SET(ARROW_BOOST_LIBS boost_system boost_filesystem boost_regex)
 endif()
 
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index aefd4d76dfa0f..17b87bf4eca0c 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -640,11 +640,11 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
 static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
                               PyObject** out_values) {
   PyAcquireGIL lock;
-  OwnedRef decimal_ref;
-  OwnedRef Decimal_ref;
-  RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_ref));
-  RETURN_NOT_OK(internal::ImportFromModule(decimal_ref, "Decimal", &Decimal_ref));
-  PyObject* Decimal = Decimal_ref.obj();
+  OwnedRef decimal;
+  OwnedRef Decimal;
+  RETURN_NOT_OK(internal::ImportModule("decimal", &decimal));
+  RETURN_NOT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal));
+  PyObject* decimal_constructor = Decimal.obj();
 
   for (int c = 0; c < data.num_chunks(); c++) {
     const auto& arr = static_cast<const arrow::Decimal128Array&>(*data.chunk(c));
@@ -654,7 +654,8 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data,
         Py_INCREF(Py_None);
         *out_values++ = Py_None;
       } else {
-        *out_values++ = internal::DecimalFromString(Decimal, arr.FormatValue(i));
+        *out_values++ =
+            internal::DecimalFromString(decimal_constructor, arr.FormatValue(i));
         RETURN_IF_PYERROR();
       }
     }
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index a286c6bd5e90d..d2f900f6ae859 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -76,7 +76,15 @@ class ScalarVisitor {
         timestamp_count_(0),
         float_count_(0),
         binary_count_(0),
-        unicode_count_(0) {}
+        unicode_count_(0),
+        decimal_count_(0),
+        max_decimal_metadata_(std::numeric_limits<int32_t>::min(),
+                              std::numeric_limits<int32_t>::min()),
+        decimal_type_() {
+    PyAcquireGIL lock;
+    Status status = internal::ImportDecimalType(&decimal_type_);
+    DCHECK_OK(status);
+  }
 
   Status Visit(PyObject* obj) {
     ++total_count_;
@@ -111,10 +119,13 @@ class ScalarVisitor {
         ss << type->ToString();
         return Status::Invalid(ss.str());
       }
+    } else if (PyObject_IsInstance(obj, decimal_type_.obj())) {
+      RETURN_NOT_OK(max_decimal_metadata_.Update(obj));
+      ++decimal_count_;
     } else {
       // TODO(wesm): accumulate error information somewhere
       static std::string supported_types =
-          "bool, float, integer, date, datetime, bytes, unicode";
+          "bool, float, integer, date, datetime, bytes, unicode, decimal";
       std::stringstream ss;
       ss << "Error inferring Arrow data type for collection of Python objects. ";
       RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
@@ -125,7 +136,9 @@ class ScalarVisitor {
 
   std::shared_ptr<DataType> GetType() {
     // TODO(wesm): handling mixed-type cases
-    if (float_count_) {
+    if (decimal_count_) {
+      return decimal(max_decimal_metadata_.precision(), max_decimal_metadata_.scale());
+    } else if (float_count_) {
       return float64();
     } else if (int_count_) {
       // TODO(wesm): tighter type later
@@ -157,8 +170,13 @@ class ScalarVisitor {
   int64_t float_count_;
   int64_t binary_count_;
   int64_t unicode_count_;
+  int64_t decimal_count_;
+
+  internal::DecimalMetadata max_decimal_metadata_;
+
   // Place to accumulate errors
   // std::vector<Status> errors_;
+  OwnedRefNoGIL decimal_type_;
 };
 
 static constexpr int MAX_NESTING_LEVELS = 32;
@@ -379,17 +397,14 @@ class TypedConverter : public SeqConverter {
   BuilderType* typed_builder_;
 };
 
-// We use the CRTP trick here to devirtualize the AppendItem() and AppendNull()
+// We use the CRTP trick here to devirtualize the AppendItem(), AppendNull(), and IsNull()
 // method calls.
 template <typename BuilderType, class Derived>
 class TypedConverterVisitor : public TypedConverter<BuilderType> {
  public:
   Status AppendSingle(PyObject* obj) override {
-    if (obj == Py_None) {
-      return static_cast<Derived*>(this)->AppendNull();
-    } else {
-      return static_cast<Derived*>(this)->AppendItem(obj);
-    }
+    auto self = static_cast<Derived*>(this);
+    return self->IsNull(obj) ? self->AppendNull() : self->AppendItem(obj);
   }
 
   Status AppendMultiple(PyObject* obj, int64_t size) override {
@@ -409,6 +424,7 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
 
   // Append a missing item (default implementation)
   Status AppendNull() { return this->typed_builder_->AppendNull(); }
+  bool IsNull(PyObject* obj) const { return obj == Py_None; }
 };
 
 class NullConverter : public TypedConverterVisitor<NullBuilder, NullConverter> {
@@ -830,12 +846,16 @@ class DecimalConverter
  public:
   // Append a non-missing item
   Status AppendItem(PyObject* obj) {
-    /// TODO(phillipc): Check for nan?
     Decimal128 value;
     const auto& type = static_cast<const DecimalType&>(*typed_builder_->type());
     RETURN_NOT_OK(internal::DecimalFromPythonDecimal(obj, type, &value));
     return typed_builder_->Append(value);
   }
+
+  bool IsNull(PyObject* obj) const {
+    return obj == Py_None || obj == numpy_nan || internal::PyFloat_isnan(obj) ||
+           (internal::PyDecimal_Check(obj) && internal::PyDecimal_ISNAN(obj));
+  }
 };
 
 // Dynamic constructor for sequence converters
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index df1db99911b63..429068dd1aca7 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
+#include <limits>
 #include <sstream>
 
 #include "arrow/python/common.h"
@@ -61,6 +63,7 @@ namespace internal {
 Status ImportModule(const std::string& module_name, OwnedRef* ref) {
   PyObject* module = PyImport_ImportModule(module_name.c_str());
   RETURN_IF_PYERROR();
+  DCHECK_NE(module, nullptr) << "unable to import the " << module_name << " module";
   ref->reset(module);
   return Status::OK();
 }
@@ -71,10 +74,18 @@ Status ImportFromModule(const OwnedRef& module, const std::string& name, OwnedRe
 
   PyObject* attr = PyObject_GetAttrString(module.obj(), name.c_str());
   RETURN_IF_PYERROR();
+  DCHECK_NE(attr, nullptr) << "unable to import the " << name << " object";
   ref->reset(attr);
   return Status::OK();
 }
 
+Status ImportDecimalType(OwnedRef* decimal_type) {
+  OwnedRef decimal_module;
+  RETURN_NOT_OK(ImportModule("decimal", &decimal_module));
+  RETURN_NOT_OK(ImportFromModule(decimal_module, "Decimal", decimal_type));
+  return Status::OK();
+}
+
 Status PythonDecimalToString(PyObject* python_decimal, std::string* out) {
   // Call Python's str(decimal_object)
   OwnedRef str_obj(PyObject_Str(python_decimal));
@@ -93,13 +104,19 @@ Status PythonDecimalToString(PyObject* python_decimal, std::string* out) {
   return Status::OK();
 }
 
-Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precision,
-                                     int32_t* scale) {
+// \brief Infer the precision and scale of a Python decimal.Decimal instance
+// \param python_decimal[in] An instance of decimal.Decimal
+// \param precision[out] The value of the inferred precision
+// \param scale[out] The value of the inferred scale
+// \return The status of the operation
+static Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precision,
+                                            int32_t* scale) {
   DCHECK_NE(python_decimal, NULLPTR);
   DCHECK_NE(precision, NULLPTR);
   DCHECK_NE(scale, NULLPTR);
 
-  OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()"));
+  // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
+  OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
   RETURN_IF_PYERROR();
   DCHECK(PyTuple_Check(as_tuple.obj()));
 
@@ -117,8 +134,23 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precisio
   const auto exponent = static_cast<int32_t>(PyLong_AsLong(py_exponent.obj()));
   RETURN_IF_PYERROR();
 
-  *precision = num_digits;
-  *scale = -exponent;
+  const int32_t abs_exponent = std::abs(exponent);
+
+  int32_t num_additional_zeros;
+
+  if (num_digits <= abs_exponent) {
+    DCHECK_NE(exponent, 0) << "exponent should never be zero here";
+
+    // we have leading/trailing zeros, leading if exponent is negative
+    num_additional_zeros = exponent < 0 ? abs_exponent - num_digits : exponent;
+    *scale = static_cast<int32_t>(exponent < 0) * -exponent;
+  } else {
+    // we can use the number of digits as the precision
+    num_additional_zeros = 0;
+    *scale = -exponent;
+  }
+
+  *precision = num_digits + num_additional_zeros;
   return Status::OK();
 }
 
@@ -193,6 +225,62 @@ Status UInt64FromPythonInt(PyObject* obj, uint64_t* out) {
   return Status::OK();
 }
 
+bool PyFloat_isnan(PyObject* obj) {
+  return PyFloat_Check(obj) && std::isnan(PyFloat_AS_DOUBLE(obj));
+}
+
+bool PyDecimal_Check(PyObject* obj) {
+  // TODO(phillipc): Is this expensive?
+  OwnedRef Decimal;
+  Status status = ImportDecimalType(&Decimal);
+  DCHECK_OK(status);
+  const int32_t result = PyObject_IsInstance(obj, Decimal.obj());
+  DCHECK_NE(result, -1) << " error during PyObject_IsInstance check";
+  return result == 1;
+}
+
+bool PyDecimal_ISNAN(PyObject* obj) {
+  DCHECK(PyDecimal_Check(obj)) << "obj is not an instance of decimal.Decimal";
+  OwnedRef is_nan(PyObject_CallMethod(obj, "is_nan", ""));
+  return PyObject_IsTrue(is_nan.obj()) == 1;
+}
+
+DecimalMetadata::DecimalMetadata()
+    : DecimalMetadata(std::numeric_limits<int32_t>::min(),
+                      std::numeric_limits<int32_t>::min()) {}
+
+DecimalMetadata::DecimalMetadata(int32_t precision, int32_t scale)
+    : precision_(precision), scale_(scale) {}
+
+Status DecimalMetadata::Update(int32_t suggested_precision, int32_t suggested_scale) {
+  const int32_t current_precision = precision_;
+  precision_ = std::max(current_precision, suggested_precision);
+
+  const int32_t current_scale = scale_;
+  scale_ = std::max(current_scale, suggested_scale);
+
+  // if our suggested scale is zero and we don't yet have enough precision then we need to
+  // add whatever the current scale is to the precision
+  if (suggested_scale == 0 && suggested_precision > current_precision) {
+    precision_ += scale_;
+  }
+
+  return Status::OK();
+}
+
+Status DecimalMetadata::Update(PyObject* object) {
+  DCHECK(PyDecimal_Check(object)) << "Object is not a Python Decimal";
+
+  if (ARROW_PREDICT_FALSE(PyDecimal_ISNAN(object))) {
+    return Status::OK();
+  }
+
+  int32_t precision;
+  int32_t scale;
+  RETURN_NOT_OK(InferDecimalPrecisionAndScale(object, &precision, &scale));
+  return Update(precision, scale);
+}
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index c0171aa2f5a71..6be0e49b18af3 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -36,29 +36,92 @@ namespace py {
 
 class OwnedRef;
 
-ARROW_EXPORT
-std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
+// \brief Get an arrow DataType instance from Arrow's Type::type enum
+// \param[in] type One of the values of Arrow's Type::type enum
+// \return A shared pointer to DataType
+ARROW_EXPORT std::shared_ptr<DataType> GetPrimitiveType(Type::type type);
 
 namespace internal {
 
+// \brief Import a Python module
+// \param[in] module_name The name of the module
+// \param[out] ref The OwnedRef containing the module PyObject*
 Status ImportModule(const std::string& module_name, OwnedRef* ref);
-Status ImportFromModule(const OwnedRef& module, const std::string& module_name,
-                        OwnedRef* ref);
 
-Status PythonDecimalToString(PyObject* python_decimal, std::string* out);
+// \brief Import an object from a Python module
+// \param[in] module A Python module
+// \param[in] name The name of the object to import
+// \param[out] ref The OwnedRef containing the \c name attribute of the Python module \c
+// module
+Status ImportFromModule(const OwnedRef& module, const std::string& name, OwnedRef* ref);
+
+// \brief Import
+Status ImportDecimalType(OwnedRef* decimal_type);
 
-Status InferDecimalPrecisionAndScale(PyObject* python_decimal,
-                                     int32_t* precision = NULLPTR,
-                                     int32_t* scale = NULLPTR);
+// \brief Convert a Python Decimal object to a C++ string
+// \param[in] python_decimal A Python decimal.Decimal instance
+// \param[out] The string representation of the Python Decimal instance
+// \return The status of the operation
+Status PythonDecimalToString(PyObject* python_decimal, std::string* out);
 
+// \brief Convert a C++ std::string to a Python Decimal instance
+// \param[in] decimal_constructor The decimal type object
+// \param[in] decimal_string A decimal string
+// \return An instance of decimal.Decimal
 PyObject* DecimalFromString(PyObject* decimal_constructor,
                             const std::string& decimal_string);
+
+// \brief Convert a Python decimal to an Arrow Decimal128 object
+// \param[in] python_decimal A Python decimal.Decimal instance
+// \param[in] arrow_type An instance of arrow::DecimalType
+// \param[out] out A pointer to a Decimal128
+// \return The status of the operation
 Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
                                 Decimal128* out);
+
+// \brief Check whether obj is an integer, independent of Python versions.
 bool IsPyInteger(PyObject* obj);
 
+// \brief Check whether obj is nan
+bool PyFloat_isnan(PyObject* obj);
+
+// \brief Check whether obj is an instance of Decimal
+bool PyDecimal_Check(PyObject* obj);
+
+// \brief Check whether obj is nan. This function will abort the program if the argument
+// is not a Decimal instance
+bool PyDecimal_ISNAN(PyObject* obj);
+
+// \brief Convert a Python integer into an unsigned 64-bit integer
+// \param[in] obj A Python integer
+// \param[out] out A pointer to a C uint64_t to hold the result of the conversion
+// \return The status of the operation
 Status UInt64FromPythonInt(PyObject* obj, uint64_t* out);
 
+// \brief Helper class to track and update the precision and scale of a decimal
+class DecimalMetadata {
+ public:
+  DecimalMetadata();
+  DecimalMetadata(int32_t precision, int32_t scale);
+
+  // \brief Adjust the precision and scale of a decimal type given a new precision and a
+  // new scale \param[in] suggested_precision A candidate precision \param[in]
+  // suggested_scale A candidate scale \return The status of the operation
+  Status Update(int32_t suggested_precision, int32_t suggested_scale);
+
+  // \brief A convenient interface for updating the precision and scale based on a Python
+  // Decimal object \param object A Python Decimal object \return The status of the
+  // operation
+  Status Update(PyObject* object);
+
+  int32_t precision() const { return precision_; }
+  int32_t scale() const { return scale_; }
+
+ private:
+  int32_t precision_;
+  int32_t scale_;
+};
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/numpy-internal.h b/cpp/src/arrow/python/numpy-internal.h
index 6c9c871a100a2..8d4308065c2df 100644
--- a/cpp/src/arrow/python/numpy-internal.h
+++ b/cpp/src/arrow/python/numpy-internal.h
@@ -54,6 +54,9 @@ class Ndarray1DIndexer {
 
   T* data() const { return data_; }
 
+  T* begin() const { return data(); }
+  T* end() const { return begin() + size() * stride_; }
+
   bool is_strided() const { return stride_ == 1; }
 
   T& operator[](size_type index) { return data_[index * stride_]; }
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 23418ad920c53..04a71c1f6408f 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -67,17 +67,9 @@ constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max();
 
 namespace {
 
-inline bool PyFloat_isnan(PyObject* obj) {
-  if (PyFloat_Check(obj)) {
-    double val = PyFloat_AS_DOUBLE(obj);
-    return val != val;
-  } else {
-    return false;
-  }
-}
-
 inline bool PandasObjectIsNull(PyObject* obj) {
-  return obj == Py_None || obj == numpy_nan || PyFloat_isnan(obj);
+  return obj == Py_None || obj == numpy_nan || internal::PyFloat_isnan(obj) ||
+         (internal::PyDecimal_Check(obj) && internal::PyDecimal_ISNAN(obj));
 }
 
 inline bool PyObject_is_string(PyObject* obj) {
@@ -88,10 +80,8 @@ inline bool PyObject_is_string(PyObject* obj) {
 #endif
 }
 
-inline bool PyObject_is_float(PyObject* obj) { return PyFloat_Check(obj); }
-
 inline bool PyObject_is_integer(PyObject* obj) {
-  return (!PyBool_Check(obj)) && PyArray_IsIntegerScalar(obj);
+  return !PyBool_Check(obj) && PyArray_IsIntegerScalar(obj);
 }
 
 template <int TYPE>
@@ -310,13 +300,18 @@ class NumPyConverter {
         arr_(reinterpret_cast<PyArrayObject*>(ao)),
         dtype_(PyArray_DESCR(arr_)),
         mask_(nullptr),
-        use_pandas_null_sentinels_(use_pandas_null_sentinels) {
+        use_pandas_null_sentinels_(use_pandas_null_sentinels),
+        decimal_type_() {
     if (mo != nullptr && mo != Py_None) {
       mask_ = reinterpret_cast<PyArrayObject*>(mo);
     }
     length_ = static_cast<int64_t>(PyArray_SIZE(arr_));
     itemsize_ = static_cast<int>(PyArray_DESCR(arr_)->elsize);
     stride_ = static_cast<int64_t>(PyArray_STRIDES(arr_)[0]);
+
+    PyAcquireGIL lock;
+    Status status = internal::ImportDecimalType(&decimal_type_);
+    DCHECK_OK(status);
   }
 
   bool is_strided() const { return itemsize_ != stride_; }
@@ -491,6 +486,8 @@ class NumPyConverter {
 
   bool use_pandas_null_sentinels_;
 
+  OwnedRefNoGIL decimal_type_;
+
   // Used in visitor pattern
   std::vector<std::shared_ptr<Array>> out_arrays_;
 
@@ -743,58 +740,42 @@ Status NumPyConverter::ConvertDates() {
 Status NumPyConverter::ConvertDecimals() {
   PyAcquireGIL lock;
 
-  // Import the decimal module and Decimal class
-  OwnedRef decimal;
-  OwnedRef Decimal;
-  RETURN_NOT_OK(internal::ImportModule("decimal", &decimal));
-  RETURN_NOT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal));
-
+  internal::DecimalMetadata max_decimal_metadata;
   Ndarray1DIndexer<PyObject*> objects(arr_);
-  PyObject* object = objects[0];
 
   if (type_ == NULLPTR) {
-    int32_t precision;
-    int32_t desired_scale;
-
-    int32_t tmp_precision;
-    int32_t tmp_scale;
-
-    RETURN_NOT_OK(
-        internal::InferDecimalPrecisionAndScale(objects[0], &precision, &desired_scale));
-
-    for (int64_t i = 1; i < length_; ++i) {
-      RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision,
-                                                            &tmp_scale));
-      precision = std::max(precision, tmp_precision);
-
-      if (std::abs(desired_scale) < std::abs(tmp_scale)) {
-        desired_scale = tmp_scale;
-      }
+    for (PyObject* object : objects) {
+      RETURN_NOT_OK(max_decimal_metadata.Update(object));
     }
 
-    type_ = ::arrow::decimal(precision, desired_scale);
+    type_ =
+        ::arrow::decimal(max_decimal_metadata.precision(), max_decimal_metadata.scale());
   }
 
   Decimal128Builder builder(type_, pool_);
   RETURN_NOT_OK(builder.Resize(length_));
 
   const auto& decimal_type = static_cast<const DecimalType&>(*type_);
-  PyObject* Decimal_type_object = Decimal.obj();
 
-  for (int64_t i = 0; i < length_; ++i) {
-    object = objects[i];
+  for (PyObject* object : objects) {
+    const int is_decimal = PyObject_IsInstance(object, decimal_type_.obj());
 
-    if (PyObject_IsInstance(object, Decimal_type_object)) {
-      Decimal128 value;
-      RETURN_NOT_OK(internal::DecimalFromPythonDecimal(object, decimal_type, &value));
-      RETURN_NOT_OK(builder.Append(value));
-    } else if (PandasObjectIsNull(object)) {
-      RETURN_NOT_OK(builder.AppendNull());
-    } else {
+    if (ARROW_PREDICT_FALSE(is_decimal == 0)) {
       std::stringstream ss;
       ss << "Error converting from Python objects to Decimal: ";
       RETURN_NOT_OK(InvalidConversion(object, "decimal.Decimal", &ss));
       return Status::Invalid(ss.str());
+    } else if (ARROW_PREDICT_FALSE(is_decimal == -1)) {
+      DCHECK_NE(PyErr_Occurred(), nullptr);
+      RETURN_IF_PYERROR();
+    }
+
+    if (PandasObjectIsNull(object)) {
+      RETURN_NOT_OK(builder.AppendNull());
+    } else {
+      Decimal128 value;
+      RETURN_NOT_OK(internal::DecimalFromPythonDecimal(object, decimal_type, &value));
+      RETURN_NOT_OK(builder.Append(value));
     }
   }
   return PushBuilderResult(&builder);
@@ -1045,18 +1026,13 @@ Status NumPyConverter::ConvertObjectsInfer() {
   objects.Init(arr_);
   PyDateTime_IMPORT;
 
-  OwnedRef decimal;
-  OwnedRef Decimal;
-  RETURN_NOT_OK(internal::ImportModule("decimal", &decimal));
-  RETURN_NOT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal));
-
   for (int64_t i = 0; i < length_; ++i) {
     PyObject* obj = objects[i];
     if (PandasObjectIsNull(obj)) {
       continue;
     } else if (PyObject_is_string(obj)) {
       return ConvertObjectStrings();
-    } else if (PyObject_is_float(obj)) {
+    } else if (PyFloat_Check(obj)) {
       return ConvertObjectFloats();
     } else if (PyBool_Check(obj)) {
       return ConvertBooleans();
@@ -1069,7 +1045,7 @@ Status NumPyConverter::ConvertObjectsInfer() {
       return ConvertDateTimes();
     } else if (PyTime_Check(obj)) {
       return ConvertTimes();
-    } else if (PyObject_IsInstance(const_cast<PyObject*>(obj), Decimal.obj())) {
+    } else if (PyObject_IsInstance(obj, decimal_type_.obj()) == 1) {
       return ConvertDecimals();
     } else if (PyList_Check(obj)) {
       std::shared_ptr<DataType> inferred_type;
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index b76caaecee6ba..16ac1e332b10e 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -78,15 +78,14 @@ TEST(OwnedRefNoGIL, TestMoves) {
 
 class DecimalTest : public ::testing::Test {
  public:
-  DecimalTest() : lock_(), decimal_module_(), decimal_constructor_() {
-    auto s = internal::ImportModule("decimal", &decimal_module_);
-    DCHECK(s.ok()) << s.message();
-    DCHECK_NE(decimal_module_.obj(), NULLPTR);
+  DecimalTest() : lock_(), decimal_constructor_() {
+    OwnedRef decimal_module;
 
-    s = internal::ImportFromModule(decimal_module_, "Decimal", &decimal_constructor_);
-    DCHECK(s.ok()) << s.message();
+    Status status = internal::ImportModule("decimal", &decimal_module);
+    DCHECK_OK(status);
 
-    DCHECK_NE(decimal_constructor_.obj(), NULLPTR);
+    status = internal::ImportFromModule(decimal_module, "Decimal", &decimal_constructor_);
+    DCHECK_OK(status);
   }
 
   OwnedRef CreatePythonDecimal(const std::string& string_value) {
@@ -94,16 +93,17 @@ class DecimalTest : public ::testing::Test {
     return ref;
   }
 
+  PyObject* decimal_constructor() const { return decimal_constructor_.obj(); }
+
  private:
   PyAcquireGIL lock_;
-  OwnedRef decimal_module_;
   OwnedRef decimal_constructor_;
 };
 
 TEST_F(DecimalTest, TestPythonDecimalToString) {
   std::string decimal_string("-39402950693754869342983");
 
-  OwnedRef python_object = this->CreatePythonDecimal(decimal_string);
+  OwnedRef python_object(this->CreatePythonDecimal(decimal_string));
   ASSERT_NE(python_object.obj(), nullptr);
 
   std::string string_result;
@@ -114,35 +114,57 @@ TEST_F(DecimalTest, TestInferPrecisionAndScale) {
   std::string decimal_string("-394029506937548693.42983");
   OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
 
-  int32_t precision;
-  int32_t scale;
-
-  ASSERT_OK(
-      internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));
+  internal::DecimalMetadata metadata;
+  ASSERT_OK(metadata.Update(python_decimal.obj()));
 
   const auto expected_precision =
       static_cast<int32_t>(decimal_string.size() - 2);  // 1 for -, 1 for .
   const int32_t expected_scale = 5;
 
-  ASSERT_EQ(expected_precision, precision);
-  ASSERT_EQ(expected_scale, scale);
+  ASSERT_EQ(expected_precision, metadata.precision());
+  ASSERT_EQ(expected_scale, metadata.scale());
 }
 
 TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) {
   std::string decimal_string("-3.94042983E+10");
   OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
 
-  int32_t precision;
-  int32_t scale;
-
-  ASSERT_OK(
-      internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));
+  internal::DecimalMetadata metadata;
+  ASSERT_OK(metadata.Update(python_decimal.obj()));
 
   const auto expected_precision = 9;
   const int32_t expected_scale = -2;
 
-  ASSERT_EQ(expected_precision, precision);
-  ASSERT_EQ(expected_scale, scale);
+  ASSERT_EQ(expected_precision, metadata.precision());
+  ASSERT_EQ(expected_scale, metadata.scale());
+}
+
+TEST_F(DecimalTest, TestInferAllLeadingZeros) {
+  std::string decimal_string("0.001");
+  OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
+
+  internal::DecimalMetadata metadata;
+  ASSERT_OK(metadata.Update(python_decimal.obj()));
+  ASSERT_EQ(3, metadata.precision());
+  ASSERT_EQ(3, metadata.scale());
+}
+
+TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) {
+  std::string decimal_string("0.01E5");
+  OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
+  internal::DecimalMetadata metadata;
+  ASSERT_OK(metadata.Update(python_decimal.obj()));
+  ASSERT_EQ(4, metadata.precision());
+  ASSERT_EQ(0, metadata.scale());
+}
+
+TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) {
+  std::string decimal_string("0.01E3");
+  OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string));
+  internal::DecimalMetadata metadata;
+  ASSERT_OK(metadata.Update(python_decimal.obj()));
+  ASSERT_EQ(2, metadata.precision());
+  ASSERT_EQ(0, metadata.scale());
 }
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
@@ -226,14 +248,12 @@ TEST_F(DecimalTest, FromPythonDecimalRescaleTruncateable) {
 
 TEST_F(DecimalTest, TestOverflowFails) {
   Decimal128 value;
-  int32_t precision;
-  int32_t scale;
   OwnedRef python_decimal(
       this->CreatePythonDecimal("9999999999999999999999999999999999999.9"));
-  ASSERT_OK(
-      internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale));
-  ASSERT_EQ(38, precision);
-  ASSERT_EQ(1, scale);
+  internal::DecimalMetadata metadata;
+  ASSERT_OK(metadata.Update(python_decimal.obj()));
+  ASSERT_EQ(38, metadata.precision());
+  ASSERT_EQ(1, metadata.scale());
 
   auto type = ::arrow::decimal(38, 38);
   const auto& decimal_type = static_cast<const DecimalType&>(*type);
@@ -241,5 +261,111 @@ TEST_F(DecimalTest, TestOverflowFails) {
                                                             decimal_type, &value));
 }
 
+TEST_F(DecimalTest, TestNoneAndNaN) {
+  OwnedRef list_ref(PyList_New(4));
+  PyObject* list = list_ref.obj();
+
+  ASSERT_NE(list, nullptr);
+
+  PyObject* constructor = this->decimal_constructor();
+  PyObject* decimal_value = internal::DecimalFromString(constructor, "1.234");
+  ASSERT_NE(decimal_value, nullptr);
+
+  Py_INCREF(Py_None);
+  PyObject* missing_value1 = Py_None;
+  ASSERT_NE(missing_value1, nullptr);
+
+  PyObject* missing_value2 = PyFloat_FromDouble(NPY_NAN);
+  ASSERT_NE(missing_value2, nullptr);
+
+  PyObject* missing_value3 = internal::DecimalFromString(constructor, "nan");
+  ASSERT_NE(missing_value3, nullptr);
+
+  // This steals a reference to each object, so we don't need to decref them later,
+  // just the list
+  ASSERT_EQ(0, PyList_SetItem(list, 0, decimal_value));
+  ASSERT_EQ(0, PyList_SetItem(list, 1, missing_value1));
+  ASSERT_EQ(0, PyList_SetItem(list, 2, missing_value2));
+  ASSERT_EQ(0, PyList_SetItem(list, 3, missing_value3));
+
+  MemoryPool* pool = default_memory_pool();
+  std::shared_ptr<Array> arr;
+  ASSERT_OK(ConvertPySequence(list, pool, &arr));
+  ASSERT_TRUE(arr->IsValid(0));
+  ASSERT_TRUE(arr->IsNull(1));
+  ASSERT_TRUE(arr->IsNull(2));
+  ASSERT_TRUE(arr->IsNull(3));
+}
+
+TEST_F(DecimalTest, TestMixedPrecisionAndScale) {
+  std::vector<std::string> strings{{"0.001", "1.01E5", "1.01E5"}};
+
+  OwnedRef list_ref(PyList_New(static_cast<Py_ssize_t>(strings.size())));
+  PyObject* list = list_ref.obj();
+
+  ASSERT_NE(list, nullptr);
+
+  // PyList_SetItem steals a reference to the item so we don't decref it later
+  PyObject* decimal_constructor = this->decimal_constructor();
+  for (Py_ssize_t i = 0; i < static_cast<Py_ssize_t>(strings.size()); ++i) {
+    const int result = PyList_SetItem(
+        list, i, internal::DecimalFromString(decimal_constructor, strings.at(i)));
+    ASSERT_EQ(0, result);
+  }
+
+  MemoryPool* pool = default_memory_pool();
+  std::shared_ptr<Array> arr;
+  ASSERT_OK(ConvertPySequence(list, pool, &arr));
+  const auto& type = static_cast<const DecimalType&>(*arr->type());
+
+  int32_t expected_precision = 9;
+  int32_t expected_scale = 3;
+  ASSERT_EQ(expected_precision, type.precision());
+  ASSERT_EQ(expected_scale, type.scale());
+}
+
+TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) {
+  PyAcquireGIL lock;
+  MemoryPool* pool = default_memory_pool();
+  std::shared_ptr<Array> arr;
+
+  PyObject* value1 = this->CreatePythonDecimal("0.01").detach();
+  ASSERT_NE(value1, nullptr);
+
+  PyObject* value2 = this->CreatePythonDecimal("0.001").detach();
+  ASSERT_NE(value2, nullptr);
+
+  OwnedRef list_ref(PyList_New(2));
+  PyObject* list = list_ref.obj();
+
+  // This steals a reference to each object, so we don't need to decref them later
+  // just the list
+  ASSERT_EQ(PyList_SetItem(list, 0, value1), 0);
+  ASSERT_EQ(PyList_SetItem(list, 1, value2), 0);
+
+  ASSERT_OK(ConvertPySequence(list, pool, &arr));
+
+  const auto& type = static_cast<const Decimal128Type&>(*arr->type());
+  ASSERT_EQ(3, type.precision());
+  ASSERT_EQ(3, type.scale());
+}
+
+TEST_F(DecimalTest, SimpleInference) {
+  OwnedRef value(this->CreatePythonDecimal("0.01"));
+  ASSERT_NE(value.obj(), nullptr);
+  internal::DecimalMetadata metadata;
+  ASSERT_OK(metadata.Update(value.obj()));
+  ASSERT_EQ(2, metadata.precision());
+  ASSERT_EQ(2, metadata.scale());
+}
+
+TEST_F(DecimalTest, UpdateWithNaN) {
+  internal::DecimalMetadata metadata;
+  OwnedRef nan_value(this->CreatePythonDecimal("nan"));
+  ASSERT_OK(metadata.Update(nan_value.obj()));
+  ASSERT_EQ(std::numeric_limits<int32_t>::min(), metadata.precision());
+  ASSERT_EQ(std::numeric_limits<int32_t>::min(), metadata.scale());
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index e4406747d557c..6db46d48512b0 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -14,7 +14,6 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-//
 
 #include <cstdint>
 #include <string>
@@ -37,7 +36,7 @@ class DecimalTestFixture : public ::testing::Test {
 
 TEST_F(DecimalTestFixture, TestToString) {
   Decimal128 decimal(this->integer_value_);
-  int scale = 5;
+  int32_t scale = 5;
   std::string result = decimal.ToString(scale);
   ASSERT_EQ(result, this->string_value_);
 }
@@ -45,7 +44,7 @@ TEST_F(DecimalTestFixture, TestToString) {
 TEST_F(DecimalTestFixture, TestFromString) {
   Decimal128 expected(this->integer_value_);
   Decimal128 result;
-  int precision, scale;
+  int32_t precision, scale;
   ASSERT_OK(Decimal128::FromString(this->string_value_, &result, &precision, &scale));
   ASSERT_EQ(result, expected);
   ASSERT_EQ(precision, 8);
@@ -55,8 +54,8 @@ TEST_F(DecimalTestFixture, TestFromString) {
 TEST_F(DecimalTestFixture, TestStringStartingWithPlus) {
   std::string plus_value("+234.234");
   Decimal128 out;
-  int scale;
-  int precision;
+  int32_t scale;
+  int32_t precision;
   ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
   ASSERT_EQ(234234, out);
   ASSERT_EQ(6, precision);
@@ -67,8 +66,8 @@ TEST_F(DecimalTestFixture, TestStringStartingWithPlus128) {
   std::string plus_value("+2342394230592.232349023094");
   Decimal128 expected_value("2342394230592232349023094");
   Decimal128 out;
-  int scale;
-  int precision;
+  int32_t scale;
+  int32_t precision;
   ASSERT_OK(Decimal128::FromString(plus_value, &out, &precision, &scale));
   ASSERT_EQ(expected_value, out);
   ASSERT_EQ(25, precision);
@@ -90,9 +89,7 @@ TEST(DecimalTest, TestFromDecimalString128) {
   Decimal128 result;
   ASSERT_OK(Decimal128::FromString(string_value, &result));
   Decimal128 expected(static_cast<int64_t>(-230492239423435324));
-  expected *= 100;
-  expected -= 12;
-  ASSERT_EQ(result, expected);
+  ASSERT_EQ(result, expected * 100 - 12);
 
   // Sanity check that our number is actually using more than 64 bits
   ASSERT_NE(result.high_bits(), 0);
@@ -194,36 +191,36 @@ TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
 TEST(DecimalZerosTest, LeadingZerosNoDecimalPoint) {
   std::string string_value("0000000");
   Decimal128 d;
-  int precision;
-  int scale;
+  int32_t precision;
+  int32_t scale;
   ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
-  ASSERT_EQ(precision, 7);
-  ASSERT_EQ(scale, 0);
-  ASSERT_EQ(d, 0);
+  ASSERT_EQ(0, precision);
+  ASSERT_EQ(0, scale);
+  ASSERT_EQ(0, d);
 }
 
 TEST(DecimalZerosTest, LeadingZerosDecimalPoint) {
   std::string string_value("000.0000");
   Decimal128 d;
-  int precision;
-  int scale;
+  int32_t precision;
+  int32_t scale;
   ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
   // We explicitly do not support this for now, otherwise this would be ASSERT_EQ
-  ASSERT_NE(precision, 7);
+  ASSERT_EQ(4, precision);
 
-  ASSERT_EQ(scale, 4);
-  ASSERT_EQ(d, 0);
+  ASSERT_EQ(4, scale);
+  ASSERT_EQ(0, d);
 }
 
 TEST(DecimalZerosTest, NoLeadingZerosDecimalPoint) {
   std::string string_value(".00000");
   Decimal128 d;
-  int precision;
-  int scale;
+  int32_t precision;
+  int32_t scale;
   ASSERT_OK(Decimal128::FromString(string_value, &d, &precision, &scale));
-  ASSERT_EQ(precision, 5);
-  ASSERT_EQ(scale, 5);
-  ASSERT_EQ(d, 0);
+  ASSERT_EQ(5, precision);
+  ASSERT_EQ(5, scale);
+  ASSERT_EQ(0, d);
 }
 
 template <typename T>
@@ -335,16 +332,16 @@ INSTANTIATE_TEST_CASE_P(Decimal128ParsingTest, Decimal128ParsingTest,
                                           std::make_tuple("0.00123", 123ULL, 5),
                                           std::make_tuple("1.23E-8", 123ULL, 10),
                                           std::make_tuple("-1.23E-8", -123LL, 10),
-                                          std::make_tuple("1.23E+3", 123ULL, -1),
-                                          std::make_tuple("-1.23E+3", -123LL, -1),
-                                          std::make_tuple("1.23E+5", 123ULL, -3),
-                                          std::make_tuple("1.2345E+7", 12345ULL, -3),
+                                          std::make_tuple("1.23E+3", 1230ULL, 0),
+                                          std::make_tuple("-1.23E+3", -1230LL, 0),
+                                          std::make_tuple("1.23E+5", 123000ULL, 0),
+                                          std::make_tuple("1.2345E+7", 12345000ULL, 0),
                                           std::make_tuple("1.23e-8", 123ULL, 10),
                                           std::make_tuple("-1.23e-8", -123LL, 10),
-                                          std::make_tuple("1.23e+3", 123ULL, -1),
-                                          std::make_tuple("-1.23e+3", -123LL, -1),
-                                          std::make_tuple("1.23e+5", 123ULL, -3),
-                                          std::make_tuple("1.2345e+7", 12345ULL, -3)));
+                                          std::make_tuple("1.23e+3", 1230ULL, 0),
+                                          std::make_tuple("-1.23e+3", -1230LL, 0),
+                                          std::make_tuple("1.23e+5", 123000ULL, 0),
+                                          std::make_tuple("1.2345e+7", 12345000ULL, 0)));
 
 class Decimal128ParsingTestInvalid : public ::testing::TestWithParam<std::string> {};
 
@@ -375,4 +372,14 @@ TEST(Decimal128Test, TestSmallNumberFormat) {
   ASSERT_EQ(expected, result);
 }
 
+TEST(Decimal128Test, TestNoDecimalPointExponential) {
+  Decimal128 value;
+  int32_t precision;
+  int32_t scale;
+  ASSERT_OK(Decimal128::FromString("1E1", &value, &precision, &scale));
+  ASSERT_EQ(10, value.low_bits());
+  ASSERT_EQ(2, precision);
+  ASSERT_EQ(0, scale);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index a3c8cda7654ab..48380a9c98083 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -23,12 +23,55 @@
 #include <limits>
 #include <sstream>
 
+#include <boost/regex.hpp>
+
 #include "arrow/util/bit-util.h"
 #include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
 
+static const Decimal128 ScaleMultipliers[] = {
+    Decimal128(0LL),
+    Decimal128(10LL),
+    Decimal128(100LL),
+    Decimal128(1000LL),
+    Decimal128(10000LL),
+    Decimal128(100000LL),
+    Decimal128(1000000LL),
+    Decimal128(10000000LL),
+    Decimal128(100000000LL),
+    Decimal128(1000000000LL),
+    Decimal128(10000000000LL),
+    Decimal128(100000000000LL),
+    Decimal128(1000000000000LL),
+    Decimal128(10000000000000LL),
+    Decimal128(100000000000000LL),
+    Decimal128(1000000000000000LL),
+    Decimal128(10000000000000000LL),
+    Decimal128(100000000000000000LL),
+    Decimal128(1000000000000000000LL),
+    Decimal128(0LL, 10000000000000000000ULL),
+    Decimal128(5LL, 7766279631452241920ULL),
+    Decimal128(54LL, 3875820019684212736ULL),
+    Decimal128(542LL, 1864712049423024128ULL),
+    Decimal128(5421LL, 200376420520689664ULL),
+    Decimal128(54210LL, 2003764205206896640ULL),
+    Decimal128(542101LL, 1590897978359414784ULL),
+    Decimal128(5421010LL, 15908979783594147840ULL),
+    Decimal128(54210108LL, 11515845246265065472ULL),
+    Decimal128(542101086LL, 4477988020393345024ULL),
+    Decimal128(5421010862LL, 7886392056514347008ULL),
+    Decimal128(54210108624LL, 5076944270305263616ULL),
+    Decimal128(542101086242LL, 13875954555633532928ULL),
+    Decimal128(5421010862427LL, 9632337040368467968ULL),
+    Decimal128(54210108624275LL, 4089650035136921600ULL),
+    Decimal128(542101086242752LL, 4003012203950112768ULL),
+    Decimal128(5421010862427522LL, 3136633892082024448ULL),
+    Decimal128(54210108624275221LL, 12919594847110692864ULL),
+    Decimal128(542101086242752217LL, 68739955140067328ULL),
+    Decimal128(5421010862427522170LL, 687399551400673280ULL)};
+
 static constexpr uint64_t kIntMask = 0xFFFFFFFF;
 static constexpr auto kCarryBit = static_cast<uint64_t>(1) << static_cast<uint64_t>(32);
 
@@ -49,7 +92,7 @@ std::array<uint8_t, 16> Decimal128::ToBytes() const {
 }
 
 void Decimal128::ToBytes(uint8_t* out) const {
-  DCHECK_NE(out, NULLPTR);
+  DCHECK_NE(out, nullptr);
   reinterpret_cast<uint64_t*>(out)[0] = BitUtil::ToLittleEndian(low_bits_);
   reinterpret_cast<int64_t*>(out)[1] = BitUtil::ToLittleEndian(high_bits_);
 }
@@ -187,12 +230,10 @@ static constexpr int64_t kPowersOfTen[kInt64DecimalDigits + 1] = {1LL,
                                                                   100000000000000000LL,
                                                                   1000000000000000000LL};
 
-static inline bool isdigit(char value) { return std::isdigit(value) != 0; }
-
 static void StringToInteger(const std::string& str, Decimal128* out) {
   using std::size_t;
 
-  DCHECK_NE(out, NULLPTR) << "Decimal128 output variable cannot be NULLPTR";
+  DCHECK_NE(out, nullptr) << "Decimal128 output variable cannot be nullptr";
   DCHECK_EQ(*out, 0)
       << "When converting a string to Decimal128 the initial output must be 0";
 
@@ -212,160 +253,124 @@ static void StringToInteger(const std::string& str, Decimal128* out) {
   }
 }
 
-Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precision,
-                              int* scale) {
-  // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?((E|e)(\\+|-)?\\d+)?";
-  if (s.empty()) {
-    return Status::Invalid("Empty string cannot be converted to decimal");
-  }
+static const boost::regex DECIMAL_REGEX(
+    // sign of the number
+    "(?<SIGN>[-+]?)"
 
-  std::string::const_iterator charp = s.cbegin();
-  std::string::const_iterator end = s.cend();
+    // digits around the decimal point
+    "(((?<LEFT_DIGITS>\\d+)\\.(?<FIRST_RIGHT_DIGITS>\\d*)|\\.(?<SECOND_RIGHT_DIGITS>\\d+)"
+    ")"
 
-  char first_char = *charp;
-  bool is_negative = false;
-  if (first_char == '+' || first_char == '-') {
-    is_negative = first_char == '-';
-    ++charp;
-  }
+    // optional exponent
+    "([eE](?<FIRST_EXP_VALUE>[-+]?\\d+))?"
 
-  if (charp == end) {
-    std::stringstream ss;
-    ss << "Single character: '" << first_char << "' is not a valid decimal value";
-    return Status::Invalid(ss.str());
-  }
+    // otherwise
+    "|"
 
-  std::string::const_iterator numeric_string_start = charp;
+    // we're just an integer
+    "(?<INTEGER>\\d+)"
 
-  DCHECK_LT(charp, end);
+    // or an integer with an exponent
+    "(?:[eE](?<SECOND_EXP_VALUE>[-+]?\\d+))?)");
 
-  // skip leading zeros
-  charp = std::find_if_not(charp, end, [](char value) { return value == '0'; });
+static inline bool is_zero_character(char c) { return c == '0'; }
 
-  // all zeros and no decimal point
-  if (charp == end) {
-    if (out != NULLPTR) {
-      *out = 0;
-    }
+Status Decimal128::FromString(const std::string& s, Decimal128* out, int32_t* precision,
+                              int32_t* scale) {
+  if (s.empty()) {
+    return Status::Invalid("Empty string cannot be converted to decimal");
+  }
 
-    // Not sure what other libraries assign precision to for this case (this case of
-    // a string consisting only of one or more zeros)
-    if (precision != NULLPTR) {
-      *precision = static_cast<int>(charp - numeric_string_start);
+  // case of all zeros
+  if (std::all_of(s.cbegin(), s.cend(), is_zero_character)) {
+    if (precision != nullptr) {
+      *precision = 0;
     }
 
-    if (scale != NULLPTR) {
+    if (scale != nullptr) {
       *scale = 0;
     }
 
+    *out = 0;
     return Status::OK();
   }
 
-  std::string::const_iterator whole_part_start = charp;
+  boost::smatch results;
+  const bool matches = boost::regex_match(s, results, DECIMAL_REGEX);
 
-  charp = std::find_if_not(charp, end, isdigit);
+  if (!matches) {
+    std::stringstream ss;
+    ss << "The string " << s << " is not a valid decimal number";
+    return Status::Invalid(ss.str());
+  }
 
-  std::string::const_iterator whole_part_end = charp;
-  std::string whole_part(whole_part_start, whole_part_end);
+  const std::string sign = results["SIGN"];
+  const std::string integer = results["INTEGER"];
 
-  if (charp != end && *charp == '.') {
-    ++charp;
+  const std::string left_digits = results["LEFT_DIGITS"];
+  const std::string first_right_digits = results["FIRST_RIGHT_DIGITS"];
 
-    if (charp == end) {
-      return Status::Invalid(
-          "Decimal point must be followed by at least one base ten digit. Reached the "
-          "end of the string.");
-    }
+  const std::string second_right_digits = results["SECOND_RIGHT_DIGITS"];
 
-    if (std::isdigit(*charp) == 0) {
-      std::stringstream ss;
-      ss << "Decimal point must be followed by a base ten digit. Found '" << *charp
-         << "'";
-      return Status::Invalid(ss.str());
-    }
-  } else {
-    if (charp != end) {
-      std::stringstream ss;
-      ss << "Expected base ten digit or decimal point but found '" << *charp
-         << "' instead.";
-      return Status::Invalid(ss.str());
-    }
-  }
+  const std::string first_exp_value = results["FIRST_EXP_VALUE"];
+  const std::string second_exp_value = results["SECOND_EXP_VALUE"];
 
-  std::string::const_iterator fractional_part_start = charp;
+  std::string whole_part;
+  std::string fractional_part;
+  std::string exponent_value;
 
-  // The rest must be digits or an exponent
-  if (charp != end) {
-    charp = std::find_if_not(charp, end, isdigit);
-
-    // The while loop has ended before the end of the string which means we've hit a
-    // character that isn't a base ten digit or "E" for exponent
-    if (charp != end && *charp != 'E' && *charp != 'e') {
-      std::stringstream ss;
-      ss << "Found non base ten digit character '" << *charp
-         << "' before the end of the string";
-      return Status::Invalid(ss.str());
-    }
+  if (!integer.empty()) {
+    whole_part = integer;
+  } else if (!left_digits.empty()) {
+    DCHECK(second_right_digits.empty()) << s << " " << second_right_digits;
+    whole_part = left_digits;
+    fractional_part = first_right_digits;
+  } else {
+    DCHECK(first_right_digits.empty()) << s << " " << first_right_digits;
+    fractional_part = second_right_digits;
   }
 
-  std::string::const_iterator fractional_part_end = charp;
-  std::string fractional_part(fractional_part_start, fractional_part_end);
+  // skip leading zeros before the decimal point
+  std::string::const_iterator without_leading_zeros =
+      std::find_if_not(whole_part.cbegin(), whole_part.cend(), is_zero_character);
+  whole_part = std::string(without_leading_zeros, whole_part.cend());
 
-  if (precision != NULLPTR) {
-    *precision = static_cast<int>(whole_part.size() + fractional_part.size());
+  if (!first_exp_value.empty()) {
+    exponent_value = first_exp_value;
+  } else {
+    exponent_value = second_exp_value;
   }
 
-  if (charp != end) {
-    // we must have an exponent, if this aborts then we have somehow not caught this and
-    // raised a proper error
-    DCHECK(*charp == 'E' || *charp == 'e');
-
-    ++charp;
-
-    const char value = *charp;
-    const bool starts_with_plus_or_minus = value == '+' || value == '-';
-
-    // we use this to construct the adjusted exponent integer later
-    std::string::const_iterator digit_start = charp;
-
-    // skip plus or minus
-    charp += starts_with_plus_or_minus;
-
-    // confirm that the rest of the characters are digits
-    charp = std::find_if_not(charp, end, isdigit);
-
-    if (charp != end) {
-      // we have something other than digits here
-      std::stringstream ss;
-      ss << "Found non decimal digit exponent value '" << *charp << "'";
-      return Status::Invalid(ss.str());
-    }
-
-    if (scale != NULLPTR) {
-      // compute the scale from the adjusted exponent
-      std::string adjusted_exponent_string(digit_start, end);
-      DCHECK(std::all_of(adjusted_exponent_string.cbegin() + starts_with_plus_or_minus,
-                         adjusted_exponent_string.cend(), isdigit))
-          << "Non decimal digit character found in " << adjusted_exponent_string;
-      const auto adjusted_exponent =
-          static_cast<int32_t>(std::stol(adjusted_exponent_string));
-      const auto len = static_cast<int32_t>(whole_part.size() + fractional_part.size());
+  if (precision != nullptr) {
+    *precision = static_cast<int32_t>(whole_part.size() + fractional_part.size());
+  }
 
+  if (scale != nullptr) {
+    if (!exponent_value.empty()) {
+      auto adjusted_exponent = static_cast<int32_t>(std::stol(exponent_value));
+      auto len = static_cast<int32_t>(whole_part.size() + fractional_part.size());
       *scale = -adjusted_exponent + len - 1;
-    }
-  } else {
-    if (scale != NULLPTR) {
-      *scale = static_cast<int>(fractional_part.size());
+    } else {
+      *scale = static_cast<int32_t>(fractional_part.size());
     }
   }
 
-  if (out != NULLPTR) {
-    // zero out in case we've passed in a previously used value
+  if (out != nullptr) {
     *out = 0;
     StringToInteger(whole_part + fractional_part, out);
-    if (is_negative) {
+    if (sign == "-") {
       out->Negate();
     }
+
+    if (scale != nullptr && *scale < 0) {
+      const int32_t abs_scale = std::abs(*scale);
+      *out *= ScaleMultipliers[abs_scale];
+
+      if (precision != nullptr) {
+        *precision += abs_scale;
+      }
+      *scale = 0;
+    }
   }
 
   return Status::OK();
@@ -813,47 +818,6 @@ Decimal128 operator%(const Decimal128& left, const Decimal128& right) {
   return remainder;
 }
 
-static const Decimal128 ScaleMultipliers[] = {
-    Decimal128(1),
-    Decimal128(10),
-    Decimal128(100),
-    Decimal128(1000),
-    Decimal128(10000),
-    Decimal128(100000),
-    Decimal128(1000000),
-    Decimal128(10000000),
-    Decimal128(100000000),
-    Decimal128(1000000000),
-    Decimal128(10000000000),
-    Decimal128(100000000000),
-    Decimal128(1000000000000),
-    Decimal128(10000000000000),
-    Decimal128(100000000000000),
-    Decimal128(1000000000000000),
-    Decimal128(10000000000000000),
-    Decimal128(100000000000000000),
-    Decimal128(1000000000000000000),
-    Decimal128("10000000000000000000"),
-    Decimal128("100000000000000000000"),
-    Decimal128("1000000000000000000000"),
-    Decimal128("10000000000000000000000"),
-    Decimal128("100000000000000000000000"),
-    Decimal128("1000000000000000000000000"),
-    Decimal128("10000000000000000000000000"),
-    Decimal128("100000000000000000000000000"),
-    Decimal128("1000000000000000000000000000"),
-    Decimal128("10000000000000000000000000000"),
-    Decimal128("100000000000000000000000000000"),
-    Decimal128("1000000000000000000000000000000"),
-    Decimal128("10000000000000000000000000000000"),
-    Decimal128("100000000000000000000000000000000"),
-    Decimal128("1000000000000000000000000000000000"),
-    Decimal128("10000000000000000000000000000000000"),
-    Decimal128("100000000000000000000000000000000000"),
-    Decimal128("1000000000000000000000000000000000000"),
-    Decimal128("10000000000000000000000000000000000000"),
-    Decimal128("100000000000000000000000000000000000000")};
-
 static bool RescaleWouldCauseDataLoss(const Decimal128& value, int32_t delta_scale,
                                       int32_t abs_delta_scale, Decimal128* result) {
   Decimal128 multiplier(ScaleMultipliers[abs_delta_scale]);
@@ -872,7 +836,7 @@ static bool RescaleWouldCauseDataLoss(const Decimal128& value, int32_t delta_sca
 
 Status Decimal128::Rescale(int32_t original_scale, int32_t new_scale,
                            Decimal128* out) const {
-  DCHECK_NE(out, NULLPTR) << "out is nullptr";
+  DCHECK_NE(out, nullptr) << "out is nullptr";
   DCHECK_NE(original_scale, new_scale) << "original_scale != new_scale";
 
   const int32_t delta_scale = new_scale - original_scale;
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 1594090a0d30e..79a99ba6aba81 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -124,7 +124,7 @@ class ARROW_EXPORT Decimal128 {
   /// \brief Convert a decimal string to an Decimal128 value, optionally including
   /// precision and scale if they're passed in and not null.
   static Status FromString(const std::string& s, Decimal128* out,
-                           int* precision = NULLPTR, int* scale = NULLPTR);
+                           int32_t* precision = NULLPTR, int32_t* scale = NULLPTR);
 
   /// \brief Convert Decimal128 from one scale to another
   Status Rescale(int32_t original_scale, int32_t new_scale, Decimal128* out) const;
diff --git a/cpp/src/arrow/util/logging.h b/cpp/src/arrow/util/logging.h
index 4ca4d22527fc7..c823f06bdf63b 100644
--- a/cpp/src/arrow/util/logging.h
+++ b/cpp/src/arrow/util/logging.h
@@ -30,7 +30,7 @@ namespace arrow {
 //
 // Add more as needed.
 
-// Log levels. LOG ignores them, so their values are abitrary.
+// Log levels. LOG ignores them, so their values are arbitrary.
 
 #define ARROW_DEBUG (-1)
 #define ARROW_INFO 0
@@ -53,6 +53,9 @@ namespace arrow {
 #define DCHECK(condition)      \
   ARROW_IGNORE_EXPR(condition) \
   while (false) ::arrow::internal::NullLog()
+#define DCHECK_OK(status)   \
+  ARROW_IGNORE_EXPR(status) \
+  while (false) ::arrow::internal::NullLog()
 #define DCHECK_EQ(val1, val2) \
   ARROW_IGNORE_EXPR(val1)     \
   while (false) ::arrow::internal::NullLog()
@@ -76,6 +79,7 @@ namespace arrow {
 #define ARROW_DFATAL ARROW_FATAL
 
 #define DCHECK(condition) ARROW_CHECK(condition)
+#define DCHECK_OK(status) (ARROW_CHECK((status).ok()) << (status).message())
 #define DCHECK_EQ(val1, val2) ARROW_CHECK((val1) == (val2))
 #define DCHECK_NE(val1, val2) ARROW_CHECK((val1) != (val2))
 #define DCHECK_LE(val1, val2) ARROW_CHECK((val1) <= (val2))
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 8423ff00b6722..19b59a49b6d61 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -639,3 +639,13 @@ def test_structarray_from_arrays_coerce():
         pa.StructArray.from_arrays(arrays)
 
     assert result.equals(expected)
+
+
+def test_decimal_array_with_none_and_nan():
+    values = [decimal.Decimal('1.234'), None, np.nan, decimal.Decimal('nan')]
+    array = pa.array(values)
+    assert array.type == pa.decimal128(4, 3)
+    assert array.to_pylist() == values[:2] + [None, None]
+
+    array = pa.array(values, type=pa.decimal128(10, 4))
+    assert array.to_pylist() == [decimal.Decimal('1.2340'), None, None, None]
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 986aeffcacc9c..813fbdf40bbc0 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1161,6 +1161,20 @@ def test_decimal_fails_with_truncation(self):
         with pytest.raises(pa.ArrowException):
             pa.array(data2, type=type2)
 
+    def test_decimal_with_different_precisions(self):
+        data = [
+            decimal.Decimal('0.01'),
+            decimal.Decimal('0.001'),
+        ]
+        series = pd.Series(data)
+        array = pa.array(series)
+        assert array.to_pylist() == data
+        assert array.type == pa.decimal128(3, 3)
+
+        array = pa.array(data, type=pa.decimal128(12, 5))
+        expected = [decimal.Decimal('0.01000'), decimal.Decimal('0.00100')]
+        assert array.to_pylist() == expected
+
 
 class TestListTypes(object):
     """

From 99899d6e3c761b2cdcb843dc0da7fd08aa3db06b Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 2 Mar 2018 15:16:40 +0100
Subject: [PATCH 1549/1644] ARROW-2232: [Python] pyarrow.Tensor constructor
 segfaults

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1682 from cpcloud/ARROW-2232 and squashes the following commits:

1837b02 <Phillip Cloud> ARROW-2232:  pyarrow.Tensor constructor segfaults
---
 python/pyarrow/array.pxi           | 87 +++++++++++++++---------------
 python/pyarrow/tests/test_array.py | 11 ++++
 2 files changed, 56 insertions(+), 42 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 5b8621f13b3a2..80e15f2bd158d 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -496,11 +496,19 @@ cdef class Tensor:
         self.tp = sp_tensor.get()
         self.type = pyarrow_wrap_data_type(self.tp.type())
 
+    def _validate(self):
+        if self.tp is NULL:
+            raise TypeError(
+                'pyarrow.Tensor has not been initialized correctly Please use '
+                'pyarrow.Tensor.from_numpy to construct a pyarrow.Tensor')
+
     def __repr__(self):
+        if self.tp is NULL:
+            return '<invalid pyarrow.Tensor>'
         return """<pyarrow.Tensor>
-type: {0}
-shape: {1}
-strides: {2}""".format(self.type, self.shape, self.strides)
+type: {0.type}
+shape: {0.shape}
+strides: {0.strides}""".format(self)
 
     @staticmethod
     def from_numpy(obj):
@@ -514,8 +522,9 @@ strides: {2}""".format(self.type, self.shape, self.strides)
         """
         Convert arrow::Tensor to numpy.ndarray with zero copy
         """
-        cdef:
-            PyObject* out
+        self._validate()
+
+        cdef PyObject* out
 
         with nogil:
             check_status(TensorToNdarray(self.sp_tensor, self, &out))
@@ -525,45 +534,39 @@ strides: {2}""".format(self.type, self.shape, self.strides)
         """
         Return true if the tensors contains exactly equal data
         """
+        self._validate()
         return self.tp.Equals(deref(other.tp))
 
-    property is_mutable:
-
-        def __get__(self):
-            return self.tp.is_mutable()
-
-    property is_contiguous:
-
-        def __get__(self):
-            return self.tp.is_contiguous()
-
-    property ndim:
-
-        def __get__(self):
-            return self.tp.ndim()
-
-    property size:
-
-        def __get__(self):
-            return self.tp.size()
-
-    property shape:
-
-        def __get__(self):
-            cdef size_t i
-            py_shape = []
-            for i in range(self.tp.shape().size()):
-                py_shape.append(self.tp.shape()[i])
-            return py_shape
-
-    property strides:
-
-        def __get__(self):
-            cdef size_t i
-            py_strides = []
-            for i in range(self.tp.strides().size()):
-                py_strides.append(self.tp.strides()[i])
-            return py_strides
+    @property
+    def is_mutable(self):
+        self._validate()
+        return self.tp.is_mutable()
+
+    @property
+    def is_contiguous(self):
+        self._validate()
+        return self.tp.is_contiguous()
+
+    @property
+    def ndim(self):
+        self._validate()
+        return self.tp.ndim()
+
+    @property
+    def size(self):
+        self._validate()
+        return self.tp.size()
+
+    @property
+    def shape(self):
+        # Cython knows how to convert a vector[T] to a Python list
+        self._validate()
+        return self.tp.shape()
+
+    @property
+    def strides(self):
+        self._validate()
+        return self.tp.strides()
 
 
 cdef wrap_array_output(PyObject* output):
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 197dac0d84f2c..c1131a00234d1 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -653,3 +653,14 @@ def test_buffers_nested():
     assert bytearray(null_bitmap)[0] == 0b00000100
     values = buffers[4].to_pybytes()
     assert struct.unpack('4xh', values) == (43,)
+
+
+def test_invalid_tensor_constructor_repr():
+    t = pa.Tensor([1])
+    assert repr(t) == '<invalid pyarrow.Tensor>'
+
+
+def test_invalid_tensor_operation():
+    t = pa.Tensor()
+    with pytest.raises(TypeError):
+        t.to_numpy()

From 29495ce03fe62a0c93934e768290e2e363a8acd5 Mon Sep 17 00:00:00 2001
From: Dimitri Vorona <vorona@in.tum.de>
Date: Fri, 2 Mar 2018 15:29:17 +0100
Subject: [PATCH 1550/1644] ARROW-2176: [C++] Extend DictionaryBuilder to
 support delta dictionaries

Author: Dimitri Vorona <vorona@in.tum.de>

Closes #1629 from alendit/delta_dictionary_builder and squashes the following commits:

f53acff <Dimitri Vorona> Fix typo
57282e3 <Dimitri Vorona> Add comment describing the new DictionaryBuilder behaviour
911d717 <Dimitri Vorona> Run clang format
8b41c3f <Dimitri Vorona> Fix types for DictionaryBuilder
9fd5811 <Dimitri Vorona> Change type of entry id offset
1420762 <Dimitri Vorona> Clearer SlotDifferent checks for DictionaryBuilder
01e0bf8 <Dimitri Vorona> Fix FixedSizeBinaryBuilder with delta dictionaries
1e81664 <Dimitri Vorona> Avoid duplication in DictionaryBuilder for StringType and BinaryType
908574d <Dimitri Vorona> Make lint happy
6bee0ca <Dimitri Vorona> Better test coverage for delta dictionary builder
27c5c45 <Dimitri Vorona> Fix string builder
d8f6afe <Dimitri Vorona> Add getter for the dictionary builder status
7c3a554 <Dimitri Vorona> Implement delta dictionary builder
---
 cpp/src/arrow/array-test.cc | 347 ++++++++++++++++++++++++++++++++++++
 cpp/src/arrow/builder.cc    | 231 ++++++++++++++++--------
 cpp/src/arrow/builder.h     |  24 ++-
 3 files changed, 529 insertions(+), 73 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 1d321e67756b1..4aaf182569940 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -1770,6 +1770,154 @@ TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) {
   }
 }
 
+TYPED_TEST(TestDictionaryBuilder, DeltaDictionary) {
+  DictionaryBuilder<TypeParam> builder(default_memory_pool());
+
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data for the initial dictionary
+  NumericBuilder<TypeParam> dict_builder1;
+  ASSERT_OK(dict_builder1.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(dict_builder1.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> dict_array1;
+  ASSERT_OK(dict_builder1.Finish(&dict_array1));
+  auto dtype1 = std::make_shared<DictionaryType>(int8(), dict_array1);
+
+  Int8Builder int_builder1;
+  ASSERT_OK(int_builder1.Append(0));
+  ASSERT_OK(int_builder1.Append(1));
+  ASSERT_OK(int_builder1.Append(0));
+  ASSERT_OK(int_builder1.Append(1));
+  std::shared_ptr<Array> int_array1;
+  ASSERT_OK(int_builder1.Finish(&int_array1));
+
+  DictionaryArray expected(dtype1, int_array1);
+  ASSERT_TRUE(expected.Equals(result));
+
+  // extend the dictionary builder with new data
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(3)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(3)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(3)));
+
+  std::shared_ptr<Array> result_delta;
+  ASSERT_OK(builder.Finish(&result_delta));
+
+  // Build expected data for the delta dictionary
+  NumericBuilder<TypeParam> dict_builder2;
+  ASSERT_OK(dict_builder2.Append(static_cast<typename TypeParam::c_type>(3)));
+  std::shared_ptr<Array> dict_array2;
+  ASSERT_OK(dict_builder2.Finish(&dict_array2));
+  auto dtype2 = std::make_shared<DictionaryType>(int8(), dict_array2);
+
+  Int8Builder int_builder2;
+  ASSERT_OK(int_builder2.Append(1));
+  ASSERT_OK(int_builder2.Append(2));
+  ASSERT_OK(int_builder2.Append(2));
+  ASSERT_OK(int_builder2.Append(0));
+  ASSERT_OK(int_builder2.Append(2));
+  std::shared_ptr<Array> int_array2;
+  ASSERT_OK(int_builder2.Finish(&int_array2));
+
+  DictionaryArray expected_delta(dtype2, int_array2);
+  ASSERT_TRUE(expected_delta.Equals(result_delta));
+}
+
+TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
+  DictionaryBuilder<TypeParam> builder(default_memory_pool());
+
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data for the initial dictionary
+  NumericBuilder<TypeParam> dict_builder1;
+  ASSERT_OK(dict_builder1.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(dict_builder1.Append(static_cast<typename TypeParam::c_type>(2)));
+  std::shared_ptr<Array> dict_array1;
+  ASSERT_OK(dict_builder1.Finish(&dict_array1));
+  auto dtype1 = std::make_shared<DictionaryType>(int8(), dict_array1);
+
+  Int8Builder int_builder1;
+  ASSERT_OK(int_builder1.Append(0));
+  ASSERT_OK(int_builder1.Append(1));
+  ASSERT_OK(int_builder1.Append(0));
+  ASSERT_OK(int_builder1.Append(1));
+  std::shared_ptr<Array> int_array1;
+  ASSERT_OK(int_builder1.Finish(&int_array1));
+
+  DictionaryArray expected(dtype1, int_array1);
+  ASSERT_TRUE(expected.Equals(result));
+
+  // extend the dictionary builder with new data
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(3)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(3)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(3)));
+
+  std::shared_ptr<Array> result_delta1;
+  ASSERT_OK(builder.Finish(&result_delta1));
+
+  // Build expected data for the delta dictionary
+  NumericBuilder<TypeParam> dict_builder2;
+  ASSERT_OK(dict_builder2.Append(static_cast<typename TypeParam::c_type>(3)));
+  std::shared_ptr<Array> dict_array2;
+  ASSERT_OK(dict_builder2.Finish(&dict_array2));
+  auto dtype2 = std::make_shared<DictionaryType>(int8(), dict_array2);
+
+  Int8Builder int_builder2;
+  ASSERT_OK(int_builder2.Append(1));
+  ASSERT_OK(int_builder2.Append(2));
+  ASSERT_OK(int_builder2.Append(2));
+  ASSERT_OK(int_builder2.Append(0));
+  ASSERT_OK(int_builder2.Append(2));
+  std::shared_ptr<Array> int_array2;
+  ASSERT_OK(int_builder2.Finish(&int_array2));
+
+  DictionaryArray expected_delta1(dtype2, int_array2);
+  ASSERT_TRUE(expected_delta1.Equals(result_delta1));
+
+  // extend the dictionary builder with new data again
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(1)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(2)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(3)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(4)));
+  ASSERT_OK(builder.Append(static_cast<typename TypeParam::c_type>(5)));
+
+  std::shared_ptr<Array> result_delta2;
+  ASSERT_OK(builder.Finish(&result_delta2));
+
+  // Build expected data for the delta dictionary again
+  NumericBuilder<TypeParam> dict_builder3;
+  ASSERT_OK(dict_builder3.Append(static_cast<typename TypeParam::c_type>(4)));
+  ASSERT_OK(dict_builder3.Append(static_cast<typename TypeParam::c_type>(5)));
+  std::shared_ptr<Array> dict_array3;
+  ASSERT_OK(dict_builder3.Finish(&dict_array3));
+  auto dtype3 = std::make_shared<DictionaryType>(int8(), dict_array3);
+
+  Int8Builder int_builder3;
+  ASSERT_OK(int_builder3.Append(0));
+  ASSERT_OK(int_builder3.Append(1));
+  ASSERT_OK(int_builder3.Append(2));
+  ASSERT_OK(int_builder3.Append(3));
+  ASSERT_OK(int_builder3.Append(4));
+  std::shared_ptr<Array> int_array3;
+  ASSERT_OK(int_builder3.Finish(&int_array3));
+
+  DictionaryArray expected_delta2(dtype3, int_array3);
+  ASSERT_TRUE(expected_delta2.Equals(result_delta2));
+}
+
 TEST(TestStringDictionaryBuilder, Basic) {
   // Build the dictionary Array
   StringDictionaryBuilder builder(default_memory_pool());
@@ -1835,6 +1983,146 @@ TEST(TestStringDictionaryBuilder, DoubleTableSize) {
   ASSERT_TRUE(expected.Equals(result));
 }
 
+TEST(TestStringDictionaryBuilder, DeltaDictionary) {
+  // Build the dictionary Array
+  StringDictionaryBuilder builder(default_memory_pool());
+  ASSERT_OK(builder.Append("test"));
+  ASSERT_OK(builder.Append("test2"));
+  ASSERT_OK(builder.Append("test"));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  // Build expected data
+  StringBuilder str_builder1;
+  ASSERT_OK(str_builder1.Append("test"));
+  ASSERT_OK(str_builder1.Append("test2"));
+  std::shared_ptr<Array> str_array1;
+  ASSERT_OK(str_builder1.Finish(&str_array1));
+  auto dtype1 = std::make_shared<DictionaryType>(int8(), str_array1);
+
+  Int8Builder int_builder1;
+  ASSERT_OK(int_builder1.Append(0));
+  ASSERT_OK(int_builder1.Append(1));
+  ASSERT_OK(int_builder1.Append(0));
+  std::shared_ptr<Array> int_array1;
+  ASSERT_OK(int_builder1.Finish(&int_array1));
+
+  DictionaryArray expected(dtype1, int_array1);
+  ASSERT_TRUE(expected.Equals(result));
+
+  // build a delta dictionary
+  ASSERT_OK(builder.Append("test2"));
+  ASSERT_OK(builder.Append("test3"));
+  ASSERT_OK(builder.Append("test2"));
+
+  std::shared_ptr<Array> result_delta;
+  ASSERT_OK(builder.Finish(&result_delta));
+
+  // Build expected data
+  StringBuilder str_builder2;
+  ASSERT_OK(str_builder2.Append("test3"));
+  std::shared_ptr<Array> str_array2;
+  ASSERT_OK(str_builder2.Finish(&str_array2));
+  auto dtype2 = std::make_shared<DictionaryType>(int8(), str_array2);
+
+  Int8Builder int_builder2;
+  ASSERT_OK(int_builder2.Append(1));
+  ASSERT_OK(int_builder2.Append(2));
+  ASSERT_OK(int_builder2.Append(1));
+  std::shared_ptr<Array> int_array2;
+  ASSERT_OK(int_builder2.Finish(&int_array2));
+
+  DictionaryArray expected_delta(dtype2, int_array2);
+  ASSERT_TRUE(expected_delta.Equals(result_delta));
+}
+
+TEST(TestStringDictionaryBuilder, BigDeltaDictionary) {
+  constexpr int16_t kTestLength = 2048;
+  // Build the dictionary Array
+  StringDictionaryBuilder builder(default_memory_pool());
+
+  StringBuilder str_builder1;
+  Int16Builder int_builder1;
+
+  for (int16_t idx = 0; idx < kTestLength; ++idx) {
+    std::stringstream sstream;
+    sstream << "test" << idx;
+    ASSERT_OK(builder.Append(sstream.str()));
+    ASSERT_OK(str_builder1.Append(sstream.str()));
+    ASSERT_OK(int_builder1.Append(idx));
+  }
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+
+  std::shared_ptr<Array> str_array1;
+  ASSERT_OK(str_builder1.Finish(&str_array1));
+  auto dtype1 = std::make_shared<DictionaryType>(int16(), str_array1);
+
+  std::shared_ptr<Array> int_array1;
+  ASSERT_OK(int_builder1.Finish(&int_array1));
+
+  DictionaryArray expected(dtype1, int_array1);
+  ASSERT_TRUE(expected.Equals(result));
+
+  // build delta 1
+  StringBuilder str_builder2;
+  Int16Builder int_builder2;
+
+  for (int16_t idx = 0; idx < kTestLength; ++idx) {
+    ASSERT_OK(builder.Append("test1"));
+    ASSERT_OK(int_builder2.Append(1));
+  }
+
+  for (int16_t idx = 0; idx < kTestLength; ++idx) {
+    ASSERT_OK(builder.Append("test_new_value1"));
+    ASSERT_OK(int_builder2.Append(kTestLength));
+  }
+  ASSERT_OK(str_builder2.Append("test_new_value1"));
+
+  std::shared_ptr<Array> result2;
+  ASSERT_OK(builder.Finish(&result2));
+
+  std::shared_ptr<Array> str_array2;
+  ASSERT_OK(str_builder2.Finish(&str_array2));
+  auto dtype2 = std::make_shared<DictionaryType>(int16(), str_array2);
+
+  std::shared_ptr<Array> int_array2;
+  ASSERT_OK(int_builder2.Finish(&int_array2));
+
+  DictionaryArray expected2(dtype2, int_array2);
+  ASSERT_TRUE(expected2.Equals(result2));
+
+  // build delta 2
+  StringBuilder str_builder3;
+  Int16Builder int_builder3;
+
+  for (int16_t idx = 0; idx < kTestLength; ++idx) {
+    ASSERT_OK(builder.Append("test2"));
+    ASSERT_OK(int_builder3.Append(2));
+  }
+
+  for (int16_t idx = 0; idx < kTestLength; ++idx) {
+    ASSERT_OK(builder.Append("test_new_value2"));
+    ASSERT_OK(int_builder3.Append(kTestLength + 1));
+  }
+  ASSERT_OK(str_builder3.Append("test_new_value2"));
+
+  std::shared_ptr<Array> result3;
+  ASSERT_OK(builder.Finish(&result3));
+
+  std::shared_ptr<Array> str_array3;
+  ASSERT_OK(str_builder3.Finish(&str_array3));
+  auto dtype3 = std::make_shared<DictionaryType>(int16(), str_array3);
+
+  std::shared_ptr<Array> int_array3;
+  ASSERT_OK(int_builder3.Finish(&int_array3));
+
+  DictionaryArray expected3(dtype3, int_array3);
+  ASSERT_TRUE(expected3.Equals(result3));
+}
+
 TEST(TestFixedSizeBinaryDictionaryBuilder, Basic) {
   // Build the dictionary Array
   DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
@@ -1867,6 +2155,65 @@ TEST(TestFixedSizeBinaryDictionaryBuilder, Basic) {
   ASSERT_TRUE(expected.Equals(result));
 }
 
+TEST(TestFixedSizeBinaryDictionaryBuilder, DeltaDictionary) {
+  // Build the dictionary Array
+  DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
+                                                 default_memory_pool());
+  std::vector<uint8_t> test{12, 12, 11, 12};
+  std::vector<uint8_t> test2{12, 12, 11, 11};
+  std::vector<uint8_t> test3{12, 12, 11, 10};
+
+  ASSERT_OK(builder.Append(test.data()));
+  ASSERT_OK(builder.Append(test2.data()));
+  ASSERT_OK(builder.Append(test.data()));
+
+  std::shared_ptr<Array> result1;
+  ASSERT_OK(builder.Finish(&result1));
+
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder1(arrow::fixed_size_binary(4));
+  ASSERT_OK(fsb_builder1.Append(test.data()));
+  ASSERT_OK(fsb_builder1.Append(test2.data()));
+  std::shared_ptr<Array> fsb_array1;
+  ASSERT_OK(fsb_builder1.Finish(&fsb_array1));
+  auto dtype1 = std::make_shared<DictionaryType>(int8(), fsb_array1);
+
+  Int8Builder int_builder1;
+  ASSERT_OK(int_builder1.Append(0));
+  ASSERT_OK(int_builder1.Append(1));
+  ASSERT_OK(int_builder1.Append(0));
+  std::shared_ptr<Array> int_array1;
+  ASSERT_OK(int_builder1.Finish(&int_array1));
+
+  DictionaryArray expected1(dtype1, int_array1);
+  ASSERT_TRUE(expected1.Equals(result1));
+
+  // build delta dictionary
+  ASSERT_OK(builder.Append(test.data()));
+  ASSERT_OK(builder.Append(test2.data()));
+  ASSERT_OK(builder.Append(test3.data()));
+
+  std::shared_ptr<Array> result2;
+  ASSERT_OK(builder.Finish(&result2));
+
+  // Build expected data
+  FixedSizeBinaryBuilder fsb_builder2(arrow::fixed_size_binary(4));
+  ASSERT_OK(fsb_builder2.Append(test3.data()));
+  std::shared_ptr<Array> fsb_array2;
+  ASSERT_OK(fsb_builder2.Finish(&fsb_array2));
+  auto dtype2 = std::make_shared<DictionaryType>(int8(), fsb_array2);
+
+  Int8Builder int_builder2;
+  ASSERT_OK(int_builder2.Append(0));
+  ASSERT_OK(int_builder2.Append(1));
+  ASSERT_OK(int_builder2.Append(2));
+  std::shared_ptr<Array> int_array2;
+  ASSERT_OK(int_builder2.Finish(&int_array2));
+
+  DictionaryArray expected2(dtype2, int_array2);
+  ASSERT_TRUE(expected2.Equals(result2));
+}
+
 TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) {
   // Build the dictionary Array
   DictionaryBuilder<FixedSizeBinaryType> builder(arrow::fixed_size_binary(4),
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 6f9749d196df9..ef4e7fde98d3e 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -818,6 +818,7 @@ DictionaryBuilder<T>::DictionaryBuilder(const std::shared_ptr<DataType>& type,
     : ArrayBuilder(type, pool),
       hash_slots_(nullptr),
       dict_builder_(type, pool),
+      overflow_dict_builder_(type, pool),
       values_builder_(pool),
       byte_width_(-1) {
   if (!::arrow::CpuInfo::initialized()) {
@@ -841,6 +842,7 @@ DictionaryBuilder<FixedSizeBinaryType>::DictionaryBuilder(
     : ArrayBuilder(type, pool),
       hash_slots_(nullptr),
       dict_builder_(type, pool),
+      overflow_dict_builder_(type, pool),
       values_builder_(pool),
       byte_width_(static_cast<const FixedSizeBinaryType&>(*type).byte_width()) {
   if (!::arrow::CpuInfo::initialized()) {
@@ -856,6 +858,7 @@ Status DictionaryBuilder<T>::Init(int64_t elements) {
   RETURN_NOT_OK(internal::NewHashTable(kInitialHashTableSize, pool_, &hash_table_));
   hash_slots_ = reinterpret_cast<int32_t*>(hash_table_->mutable_data());
   hash_table_size_ = kInitialHashTableSize;
+  entry_id_offset_ = 0;
   mod_bitmask_ = kInitialHashTableSize - 1;
   hash_table_load_threshold_ =
       static_cast<int64_t>(static_cast<double>(elements) * kMaxHashTableLoad);
@@ -893,24 +896,6 @@ Status DictionaryBuilder<NullType>::Resize(int64_t capacity) {
   }
 }
 
-template <typename T>
-Status DictionaryBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
-  std::shared_ptr<Array> dictionary;
-  RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
-
-  RETURN_NOT_OK(values_builder_.FinishInternal(out));
-  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
-  return Status::OK();
-}
-
-Status DictionaryBuilder<NullType>::FinishInternal(std::shared_ptr<ArrayData>* out) {
-  std::shared_ptr<Array> dictionary = std::make_shared<NullArray>(0);
-
-  RETURN_NOT_OK(values_builder_.FinishInternal(out));
-  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
-  return Status::OK();
-}
-
 template <typename T>
 Status DictionaryBuilder<T>::Append(const Scalar& value) {
   RETURN_NOT_OK(Reserve(1));
@@ -930,7 +915,7 @@ Status DictionaryBuilder<T>::Append(const Scalar& value) {
 
   if (index == kHashSlotEmpty) {
     // Not in the hash table, so we insert it now
-    index = static_cast<hash_slot_t>(dict_builder_.length());
+    index = static_cast<hash_slot_t>(dict_builder_.length() + entry_id_offset_);
     hash_slots_[j] = index;
     RETURN_NOT_OK(AppendDictionary(value));
 
@@ -991,8 +976,8 @@ Status DictionaryBuilder<NullType>::AppendNull() { return values_builder_.Append
 
 template <typename T>
 Status DictionaryBuilder<T>::DoubleTableSize() {
-#define INNER_LOOP                                                \
-  Scalar value = GetDictionaryValue(static_cast<int64_t>(index)); \
+#define INNER_LOOP                                                               \
+  Scalar value = GetDictionaryValue(dict_builder_, static_cast<int64_t>(index)); \
   int64_t j = HashValue(value) & new_mod_bitmask;
 
   DOUBLE_TABLE_SIZE(, INNER_LOOP);
@@ -1002,14 +987,64 @@ Status DictionaryBuilder<T>::DoubleTableSize() {
 
 template <typename T>
 typename DictionaryBuilder<T>::Scalar DictionaryBuilder<T>::GetDictionaryValue(
-    int64_t index) {
-  const Scalar* data = reinterpret_cast<const Scalar*>(dict_builder_.data()->data());
+    typename TypeTraits<T>::BuilderType& dictionary_builder, int64_t index) {
+  const Scalar* data = reinterpret_cast<const Scalar*>(dictionary_builder.data()->data());
   return data[index];
 }
 
+template <typename T>
+Status DictionaryBuilder<T>::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  entry_id_offset_ += dict_builder_.length();
+  RETURN_NOT_OK(overflow_dict_builder_.Append(
+      reinterpret_cast<const DictionaryBuilder<T>::Scalar*>(dict_builder_.data()->data()),
+      dict_builder_.length(), nullptr));
+
+  std::shared_ptr<Array> dictionary;
+  RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
+
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
+
+  RETURN_NOT_OK(dict_builder_.Init(capacity_));
+  RETURN_NOT_OK(values_builder_.Init(capacity_));
+  return Status::OK();
+}
+
+Status DictionaryBuilder<NullType>::FinishInternal(std::shared_ptr<ArrayData>* out) {
+  std::shared_ptr<Array> dictionary = std::make_shared<NullArray>(0);
+
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
+  return Status::OK();
+}
+
 template <>
-const uint8_t* DictionaryBuilder<FixedSizeBinaryType>::GetDictionaryValue(int64_t index) {
-  return dict_builder_.GetValue(index);
+const uint8_t* DictionaryBuilder<FixedSizeBinaryType>::GetDictionaryValue(
+    typename TypeTraits<FixedSizeBinaryType>::BuilderType& dictionary_builder,
+    int64_t index) {
+  return dictionary_builder.GetValue(index);
+}
+
+template <>
+Status DictionaryBuilder<FixedSizeBinaryType>::FinishInternal(
+    std::shared_ptr<ArrayData>* out) {
+  entry_id_offset_ += dict_builder_.length();
+
+  for (uint64_t index = 0, limit = dict_builder_.length(); index < limit; ++index) {
+    const Scalar value = GetDictionaryValue(dict_builder_, index);
+    RETURN_NOT_OK(overflow_dict_builder_.Append(value));
+  }
+
+  std::shared_ptr<Array> dictionary;
+  RETURN_NOT_OK(dict_builder_.Finish(&dictionary));
+
+  RETURN_NOT_OK(values_builder_.FinishInternal(out));
+  (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);
+
+  RETURN_NOT_OK(dict_builder_.Init(capacity_));
+  RETURN_NOT_OK(values_builder_.Init(capacity_));
+
+  return Status::OK();
 }
 
 template <typename T>
@@ -1024,16 +1059,34 @@ int64_t DictionaryBuilder<FixedSizeBinaryType>::HashValue(const Scalar& value) {
 
 template <typename T>
 bool DictionaryBuilder<T>::SlotDifferent(hash_slot_t index, const Scalar& value) {
-  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
-  return other != value;
+  const bool value_found =
+      index >= entry_id_offset_ &&
+      GetDictionaryValue(dict_builder_, static_cast<int64_t>(index - entry_id_offset_)) ==
+          value;
+  const bool value_found_overflow =
+      entry_id_offset_ > 0 &&
+      GetDictionaryValue(overflow_dict_builder_, static_cast<int64_t>(index)) == value;
+  return !(value_found || value_found_overflow);
 }
 
 template <>
 bool DictionaryBuilder<FixedSizeBinaryType>::SlotDifferent(hash_slot_t index,
                                                            const Scalar& value) {
   int32_t width = static_cast<const FixedSizeBinaryType&>(*type_).byte_width();
-  const Scalar other = GetDictionaryValue(static_cast<int64_t>(index));
-  return memcmp(other, value, width) != 0;
+  bool value_found = false;
+  if (index >= entry_id_offset_) {
+    const Scalar other =
+        GetDictionaryValue(dict_builder_, static_cast<int64_t>(index - entry_id_offset_));
+    value_found = memcmp(other, value, width) == 0;
+  }
+
+  bool value_found_overflow = false;
+  if (entry_id_offset_ > 0) {
+    const Scalar other_overflow =
+        GetDictionaryValue(overflow_dict_builder_, static_cast<int64_t>(index));
+    value_found_overflow = memcmp(other_overflow, value, width) == 0;
+  }
+  return !(value_found || value_found_overflow);
 }
 
 template <typename T>
@@ -1041,47 +1094,82 @@ Status DictionaryBuilder<T>::AppendDictionary(const Scalar& value) {
   return dict_builder_.Append(value);
 }
 
-#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                     \
-  template <>                                                                       \
-  WrappedBinary DictionaryBuilder<Type>::GetDictionaryValue(int64_t index) {        \
-    int32_t v_len;                                                                  \
-    const uint8_t* v = dict_builder_.GetValue(static_cast<int64_t>(index), &v_len); \
-    return WrappedBinary(v, v_len);                                                 \
-  }                                                                                 \
-                                                                                    \
-  template <>                                                                       \
-  Status DictionaryBuilder<Type>::AppendDictionary(const WrappedBinary& value) {    \
-    return dict_builder_.Append(value.ptr_, value.length_);                         \
-  }                                                                                 \
-                                                                                    \
-  template <>                                                                       \
-  Status DictionaryBuilder<Type>::AppendArray(const Array& array) {                 \
-    const BinaryArray& binary_array = static_cast<const BinaryArray&>(array);       \
-    WrappedBinary value(nullptr, 0);                                                \
-    for (int64_t i = 0; i < array.length(); i++) {                                  \
-      if (array.IsNull(i)) {                                                        \
-        RETURN_NOT_OK(AppendNull());                                                \
-      } else {                                                                      \
-        value.ptr_ = binary_array.GetValue(i, &value.length_);                      \
-        RETURN_NOT_OK(Append(value));                                               \
-      }                                                                             \
-    }                                                                               \
-    return Status::OK();                                                            \
-  }                                                                                 \
-                                                                                    \
-  template <>                                                                       \
-  int64_t DictionaryBuilder<Type>::HashValue(const WrappedBinary& value) {          \
-    return HashUtil::Hash(value.ptr_, value.length_, 0);                            \
-  }                                                                                 \
-                                                                                    \
-  template <>                                                                       \
-  bool DictionaryBuilder<Type>::SlotDifferent(hash_slot_t index,                    \
-                                              const WrappedBinary& value) {         \
-    int32_t other_length;                                                           \
-    const uint8_t* other_value =                                                    \
-        dict_builder_.GetValue(static_cast<int64_t>(index), &other_length);         \
-    return !(other_length == value.length_ &&                                       \
-             0 == memcmp(other_value, value.ptr_, value.length_));                  \
+#define BINARY_DICTIONARY_SPECIALIZATIONS(Type)                                        \
+  template <>                                                                          \
+  WrappedBinary DictionaryBuilder<Type>::GetDictionaryValue(                           \
+      typename TypeTraits<Type>::BuilderType& dictionary_builder, int64_t index) {     \
+    int32_t v_len;                                                                     \
+    const uint8_t* v = dictionary_builder.GetValue(                                    \
+        static_cast<int64_t>(index - entry_id_offset_), &v_len);                       \
+    return WrappedBinary(v, v_len);                                                    \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  Status DictionaryBuilder<Type>::AppendDictionary(const WrappedBinary& value) {       \
+    return dict_builder_.Append(value.ptr_, value.length_);                            \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  Status DictionaryBuilder<Type>::AppendArray(const Array& array) {                    \
+    const BinaryArray& binary_array = static_cast<const BinaryArray&>(array);          \
+    WrappedBinary value(nullptr, 0);                                                   \
+    for (int64_t i = 0; i < array.length(); i++) {                                     \
+      if (array.IsNull(i)) {                                                           \
+        RETURN_NOT_OK(AppendNull());                                                   \
+      } else {                                                                         \
+        value.ptr_ = binary_array.GetValue(i, &value.length_);                         \
+        RETURN_NOT_OK(Append(value));                                                  \
+      }                                                                                \
+    }                                                                                  \
+    return Status::OK();                                                               \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  int64_t DictionaryBuilder<Type>::HashValue(const WrappedBinary& value) {             \
+    return HashUtil::Hash(value.ptr_, value.length_, 0);                               \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  bool DictionaryBuilder<Type>::SlotDifferent(hash_slot_t index,                       \
+                                              const WrappedBinary& value) {            \
+    int32_t other_length;                                                              \
+    bool value_found = false;                                                          \
+    if (index >= entry_id_offset_) {                                                   \
+      const uint8_t* other_value = dict_builder_.GetValue(                             \
+          static_cast<int64_t>(index - entry_id_offset_), &other_length);              \
+      value_found = other_length == value.length_ &&                                   \
+                    memcmp(other_value, value.ptr_, value.length_) == 0;               \
+    }                                                                                  \
+                                                                                       \
+    bool value_found_overflow = false;                                                 \
+    if (entry_id_offset_ > 0) {                                                        \
+      const uint8_t* other_value_overflow =                                            \
+          overflow_dict_builder_.GetValue(static_cast<int64_t>(index), &other_length); \
+      value_found_overflow =                                                           \
+          other_length == value.length_ &&                                             \
+          memcmp(other_value_overflow, value.ptr_, value.length_) == 0;                \
+    }                                                                                  \
+    return !(value_found || value_found_overflow);                                     \
+  }                                                                                    \
+                                                                                       \
+  template <>                                                                          \
+  Status DictionaryBuilder<Type>::FinishInternal(std::shared_ptr<ArrayData>* out) {    \
+    entry_id_offset_ += dict_builder_.length();                                        \
+    for (uint64_t index = 0, limit = dict_builder_.length(); index < limit; ++index) { \
+      int32_t out_length;                                                              \
+      const uint8_t* value = dict_builder_.GetValue(index, &out_length);               \
+      RETURN_NOT_OK(overflow_dict_builder_.Append(value, out_length));                 \
+    }                                                                                  \
+                                                                                       \
+    std::shared_ptr<Array> dictionary;                                                 \
+    RETURN_NOT_OK(dict_builder_.Finish(&dictionary));                                  \
+                                                                                       \
+    RETURN_NOT_OK(values_builder_.FinishInternal(out));                                \
+    (*out)->type = std::make_shared<DictionaryType>((*out)->type, dictionary);         \
+                                                                                       \
+    RETURN_NOT_OK(dict_builder_.Init(capacity_));                                      \
+    RETURN_NOT_OK(values_builder_.Init(capacity_));                                    \
+    return Status::OK();                                                               \
   }
 
 BINARY_DICTIONARY_SPECIALIZATIONS(StringType);
@@ -1344,6 +1432,9 @@ Status FixedSizeBinaryBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
   RETURN_NOT_OK(byte_builder_.Finish(&data));
 
   *out = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count_);
+
+  null_bitmap_ = nullptr;
+  capacity_ = length_ = null_count_ = 0;
   return Status::OK();
 }
 
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 9826a6c8370fe..dabfb7506247a 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -109,13 +109,14 @@ class ARROW_EXPORT ArrayBuilder {
   std::shared_ptr<PoolBuffer> null_bitmap() const { return null_bitmap_; }
 
   /// \brief Return result of builder as an internal generic ArrayData
-  /// object. Resets builder
+  /// object. Resets builder except for dictionary builder
   ///
   /// \param[out] out the finalized ArrayData object
   /// \return Status
   virtual Status FinishInternal(std::shared_ptr<ArrayData>* out) = 0;
 
-  /// \brief Return result of builder as an Array object. Resets builder
+  /// \brief Return result of builder as an Array object.
+  ///        Resets the builder except for DictionaryBuilder
   ///
   /// \param[out] out the finalized Array object
   /// \return Status
@@ -851,6 +852,12 @@ struct DictionaryScalar<FixedSizeBinaryType> {
 }  // namespace internal
 
 /// \brief Array builder for created encoded DictionaryArray from dense array
+///
+/// Unlike other builders, dictionary builder does not completely reset the state
+/// on Finish calls. The arrays built after the initial Finish call will reuse
+/// the previously created encoding and build a delta dictionary when new terms
+/// occur.
+///
 /// data
 template <typename T>
 class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
@@ -879,9 +886,13 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
   Status Resize(int64_t capacity) override;
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
 
+  /// is the dictionary builder in the delta building mode
+  bool is_building_delta() { return entry_id_offset_ > 0; }
+
  protected:
   Status DoubleTableSize();
-  Scalar GetDictionaryValue(int64_t index);
+  Scalar GetDictionaryValue(typename TypeTraits<T>::BuilderType& dictionary_builder,
+                            int64_t index);
   int64_t HashValue(const Scalar& value);
   bool SlotDifferent(hash_slot_t slot, const Scalar& value);
   Status AppendDictionary(const Scalar& value);
@@ -892,11 +903,18 @@ class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder {
   /// Size of the table. Must be a power of 2.
   int64_t hash_table_size_;
 
+  // offset for the entry ids. Used to build delta dictionaries,
+  // increased on every InternalFinish by the number of current entries
+  // in the dictionary
+  int64_t entry_id_offset_;
+
   // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j %
   // hash_table_size_, but uses far fewer CPU cycles
   int64_t mod_bitmask_;
 
   typename TypeTraits<T>::BuilderType dict_builder_;
+  typename TypeTraits<T>::BuilderType overflow_dict_builder_;
+
   AdaptiveIntBuilder values_builder_;
   int32_t byte_width_;
 

From f403804d97d325ee31b753dc8aba6a9a4d650e8d Mon Sep 17 00:00:00 2001
From: Albert Shieh <ashieh@ansatzcapital.com>
Date: Fri, 2 Mar 2018 15:31:42 +0100
Subject: [PATCH 1551/1644] ARROW-2205: [Python] Option for integer object
 nulls

Fixes [ARROW-2205](https://issues.apache.org/jira/browse/ARROW-2205).

Author: Albert Shieh <ashieh@ansatzcapital.com>

Closes #1650 from adshieh/integer_object_nulls and squashes the following commits:

134eb25 <Albert Shieh> Clang format
7982d1e <Albert Shieh> Pass dtype through to array creation
3cfc876 <Albert Shieh> Add test for array case and parametrize tests
43451f2 <Albert Shieh> Array case for integer object nulls
305abf1 <Albert Shieh> Option for integer object nulls
---
 cpp/src/arrow/python/arrow_to_pandas.cc     | 84 ++++++++++++++++-----
 cpp/src/arrow/python/arrow_to_pandas.h      |  6 +-
 python/pyarrow/array.pxi                    |  8 +-
 python/pyarrow/includes/libarrow.pxd        |  1 +
 python/pyarrow/table.pxi                    | 14 +++-
 python/pyarrow/tests/test_convert_pandas.py | 45 +++++++++++
 6 files changed, 131 insertions(+), 27 deletions(-)

diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 17b87bf4eca0c..5749e4f405c49 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -362,6 +362,29 @@ static void ConvertBooleanNoNulls(PandasOptions options, const ChunkedArray& dat
   }
 }
 
+template <typename T>
+static Status ConvertIntegerObjects(PandasOptions options, const ChunkedArray& data,
+                                    PyObject** out_values) {
+  PyAcquireGIL lock;
+  for (int c = 0; c < data.num_chunks(); c++) {
+    const auto& arr = *data.chunk(c);
+    const T* in_values = GetPrimitiveValues<T>(arr);
+
+    for (int i = 0; i < arr.length(); ++i) {
+      if (arr.IsNull(i)) {
+        Py_INCREF(Py_None);
+        *out_values++ = Py_None;
+      } else {
+        *out_values++ = std::is_signed<T>::value
+                            ? PyLong_FromLongLong(in_values[i])
+                            : PyLong_FromUnsignedLongLong(in_values[i]);
+        RETURN_IF_PYERROR();
+      }
+    }
+  }
+  return Status::OK();
+}
+
 template <typename Type>
 inline Status ConvertBinaryLike(PandasOptions options, const ChunkedArray& data,
                                 PyObject** out_values) {
@@ -685,6 +708,22 @@ class ObjectBlock : public PandasBlock {
 
     if (type == Type::BOOL) {
       RETURN_NOT_OK(ConvertBooleanWithNulls(options_, data, out_buffer));
+    } else if (type == Type::UINT8) {
+      RETURN_NOT_OK(ConvertIntegerObjects<uint8_t>(options_, data, out_buffer));
+    } else if (type == Type::INT8) {
+      RETURN_NOT_OK(ConvertIntegerObjects<int8_t>(options_, data, out_buffer));
+    } else if (type == Type::UINT16) {
+      RETURN_NOT_OK(ConvertIntegerObjects<uint16_t>(options_, data, out_buffer));
+    } else if (type == Type::INT16) {
+      RETURN_NOT_OK(ConvertIntegerObjects<int16_t>(options_, data, out_buffer));
+    } else if (type == Type::UINT32) {
+      RETURN_NOT_OK(ConvertIntegerObjects<uint32_t>(options_, data, out_buffer));
+    } else if (type == Type::INT32) {
+      RETURN_NOT_OK(ConvertIntegerObjects<int32_t>(options_, data, out_buffer));
+    } else if (type == Type::UINT64) {
+      RETURN_NOT_OK(ConvertIntegerObjects<uint64_t>(options_, data, out_buffer));
+    } else if (type == Type::INT64) {
+      RETURN_NOT_OK(ConvertIntegerObjects<int64_t>(options_, data, out_buffer));
     } else if (type == Type::BINARY) {
       RETURN_NOT_OK(ConvertBinaryLike<BinaryType>(options_, data, out_buffer));
     } else if (type == Type::STRING) {
@@ -1203,34 +1242,33 @@ using BlockMap = std::unordered_map<int, std::shared_ptr<PandasBlock>>;
 
 static Status GetPandasBlockType(const Column& col, const PandasOptions& options,
                                  PandasBlock::type* output_type) {
+#define INTEGER_CASE(NAME)                                                           \
+  *output_type =                                                                     \
+      col.null_count() > 0                                                           \
+          ? options.integer_object_nulls ? PandasBlock::OBJECT : PandasBlock::DOUBLE \
+          : PandasBlock::NAME;                                                       \
+  break;
+
   switch (col.type()->id()) {
     case Type::BOOL:
       *output_type = col.null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL;
       break;
     case Type::UINT8:
-      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT8;
-      break;
+      INTEGER_CASE(UINT8);
     case Type::INT8:
-      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT8;
-      break;
+      INTEGER_CASE(INT8);
     case Type::UINT16:
-      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT16;
-      break;
+      INTEGER_CASE(UINT16);
     case Type::INT16:
-      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT16;
-      break;
+      INTEGER_CASE(INT16);
     case Type::UINT32:
-      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT32;
-      break;
+      INTEGER_CASE(UINT32);
     case Type::INT32:
-      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT32;
-      break;
-    case Type::INT64:
-      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::INT64;
-      break;
+      INTEGER_CASE(INT32);
     case Type::UINT64:
-      *output_type = col.null_count() > 0 ? PandasBlock::DOUBLE : PandasBlock::UINT64;
-      break;
+      INTEGER_CASE(UINT64);
+    case Type::INT64:
+      INTEGER_CASE(INT64);
     case Type::FLOAT:
       *output_type = PandasBlock::FLOAT;
       break;
@@ -1648,9 +1686,15 @@ class ArrowDeserializer {
     }
 
     if (data_.null_count() > 0) {
-      RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
-      auto out_values = reinterpret_cast<double*>(PyArray_DATA(arr_));
-      ConvertIntegerWithNulls<T>(options_, data_, out_values);
+      if (options_.integer_object_nulls) {
+        using c_type = typename Type::c_type;
+
+        return VisitObjects(ConvertIntegerObjects<c_type>);
+      } else {
+        RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
+        auto out_values = reinterpret_cast<double*>(PyArray_DATA(arr_));
+        ConvertIntegerWithNulls<T>(options_, data_, out_values);
+      }
     } else {
       RETURN_NOT_OK(AllocateOutput(traits::npy_type));
       auto out_values = reinterpret_cast<T*>(PyArray_DATA(arr_));
diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h
index 0541b0f9a295b..4819eb42c68f0 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.h
+++ b/cpp/src/arrow/python/arrow_to_pandas.h
@@ -44,8 +44,12 @@ struct PandasOptions {
   /// If true, we will convert all string columns to categoricals
   bool strings_to_categorical;
   bool zero_copy_only;
+  bool integer_object_nulls;
 
-  PandasOptions() : strings_to_categorical(false), zero_copy_only(false) {}
+  PandasOptions()
+      : strings_to_categorical(false),
+        zero_copy_only(false),
+        integer_object_nulls(false) {}
 };
 
 ARROW_EXPORT
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 80e15f2bd158d..d4e53ecd541bf 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -430,7 +430,8 @@ cdef class Array:
         return pyarrow_wrap_array(result)
 
     def to_pandas(self, c_bool strings_to_categorical=False,
-                  c_bool zero_copy_only=False):
+                  c_bool zero_copy_only=False,
+                  c_bool integer_object_nulls=False):
         """
         Convert to an array object suitable for use in pandas
 
@@ -441,6 +442,8 @@ cdef class Array:
         zero_copy_only : boolean, default False
             Raise an ArrowException if this function call would require copying
             the underlying data
+        integer_object_nulls : boolean, default False
+            Cast integers with nulls to objects
 
         See also
         --------
@@ -454,7 +457,8 @@ cdef class Array:
 
         options = PandasOptions(
             strings_to_categorical=strings_to_categorical,
-            zero_copy_only=zero_copy_only)
+            zero_copy_only=zero_copy_only,
+            integer_object_nulls=integer_object_nulls)
         with nogil:
             check_status(ConvertArrayToPandas(options, self.sp_array,
                                               self, &out))
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index b9abf2b163b40..233f2cb4784bb 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -908,6 +908,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     cdef struct PandasOptions:
         c_bool strings_to_categorical
         c_bool zero_copy_only
+        c_bool integer_object_nulls
 
 cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 178df57671da5..c27c0edd9bdf5 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -273,7 +273,8 @@ cdef class Column:
 
     def to_pandas(self,
                   c_bool strings_to_categorical=False,
-                  c_bool zero_copy_only=False):
+                  c_bool zero_copy_only=False,
+                  c_bool integer_object_nulls=False):
         """
         Convert the arrow::Column to a pandas.Series
 
@@ -287,7 +288,8 @@ cdef class Column:
 
         options = PandasOptions(
             strings_to_categorical=strings_to_categorical,
-            zero_copy_only=zero_copy_only)
+            zero_copy_only=zero_copy_only,
+            integer_object_nulls=integer_object_nulls)
 
         with nogil:
             check_status(libarrow.ConvertColumnToPandas(options,
@@ -1017,7 +1019,8 @@ cdef class Table:
         return result
 
     def to_pandas(self, nthreads=None, strings_to_categorical=False,
-                  memory_pool=None, zero_copy_only=False, categories=None):
+                  memory_pool=None, zero_copy_only=False, categories=None,
+                  integer_object_nulls=False):
         """
         Convert the arrow::Table to a pandas DataFrame
 
@@ -1036,6 +1039,8 @@ cdef class Table:
             the underlying data
         categories: list, default empty
             List of columns that should be returned as pandas.Categorical
+        integer_object_nulls : boolean, default False
+            Cast integers with nulls to objects
 
         Returns
         -------
@@ -1046,7 +1051,8 @@ cdef class Table:
 
         options = PandasOptions(
             strings_to_categorical=strings_to_categorical,
-            zero_copy_only=zero_copy_only)
+            zero_copy_only=zero_copy_only,
+            integer_object_nulls=integer_object_nulls)
         self._check_nullptr()
         if nthreads is None:
             nthreads = cpu_count()
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 813fbdf40bbc0..5abc026bf9b85 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -634,6 +634,51 @@ def _check_type(t):
         _check_type(pa.float64())
 
 
+@pytest.mark.parametrize('dtype',
+                         ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8'])
+def test_array_integer_object_nulls_option(dtype):
+    num_values = 100
+
+    null_mask = np.random.randint(0, 10, size=num_values) < 3
+    values = np.random.randint(0, 100, size=num_values, dtype=dtype)
+
+    array = pa.array(values, mask=null_mask)
+
+    if null_mask.any():
+        expected = values.astype('O')
+        expected[null_mask] = None
+    else:
+        expected = values
+
+    result = array.to_pandas(integer_object_nulls=True)
+
+    np.testing.assert_equal(result, expected)
+
+
+@pytest.mark.parametrize('dtype',
+                         ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8'])
+def test_table_integer_object_nulls_option(dtype):
+    num_values = 100
+
+    null_mask = np.random.randint(0, 10, size=num_values) < 3
+    values = np.random.randint(0, 100, size=num_values, dtype=dtype)
+
+    array = pa.array(values, mask=null_mask)
+
+    if null_mask.any():
+        expected = values.astype('O')
+        expected[null_mask] = None
+    else:
+        expected = values
+
+    expected = pd.DataFrame({dtype: expected})
+
+    table = pa.Table.from_arrays([array], [dtype])
+    result = table.to_pandas(integer_object_nulls=True)
+
+    tm.assert_frame_equal(result, expected)
+
+
 class TestConvertDateTimeLikeTypes(object):
     """
     Conversion tests for datetime- and timestamp-like types (date64, etc.).

From 5994094e2a963ba22abd657121935e2ddbfa8660 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 2 Mar 2018 15:32:37 +0100
Subject: [PATCH 1552/1644] =?UTF-8?q?ARROW-2209:=20[Python]=C2=A0Partition?=
 =?UTF-8?q?=20columns=20are=20not=20correctly=20loaded=20in=20schema=20of?=
 =?UTF-8?q?=20ParquetDataset?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1656 from xhochy/ARROW-2209 and squashes the following commits:

7771eb0 <Uwe L. Korn> flake8
c68c6a5 <Uwe L. Korn> Change field API on schema to be aligned with Python lists
609df73 <Uwe L. Korn> ARROW-2209:  Partition columns are not correctly loaded in schema of ParquetDataset
---
 python/pyarrow/includes/libarrow.pxd |  4 ++
 python/pyarrow/parquet.py            | 42 +++++++++++++-------
 python/pyarrow/tests/test_parquet.py | 13 +++++--
 python/pyarrow/types.pxi             | 58 ++++++++++++++++++++++++++++
 4 files changed, 100 insertions(+), 17 deletions(-)

diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 233f2cb4784bb..d95f01661cf0d 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -263,6 +263,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int num_fields()
         c_string ToString()
 
+        CStatus AddField(int i, const shared_ptr[CField]& field,
+                         shared_ptr[CSchema]* out)
+        CStatus RemoveField(int i, shared_ptr[CSchema]* out)
+
         # Removed const in Cython so don't have to cast to get code to generate
         shared_ptr[CSchema] AddMetadata(
             const shared_ptr[CKeyValueMetadata]& metadata)
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index f46ce9481a9ba..42c558b0bd9e2 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -581,6 +581,10 @@ def __init__(self, dirpath, filesystem=None, pathsep='/',
 
         self._visit_level(0, self.dirpath, [])
 
+        if self.common_metadata_path is None:
+            # _common_metadata is a subset of _metadata
+            self.common_metadata_path = self.metadata_path
+
     def _visit_level(self, level, base_path, part_keys):
         fs = self.filesystem
 
@@ -695,10 +699,10 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
         self.paths = path_or_paths
 
         (self.pieces, self.partitions,
-         self.metadata_path) = _make_manifest(path_or_paths, self.fs)
+         self.common_metadata_path) = _make_manifest(path_or_paths, self.fs)
 
-        if self.metadata_path is not None:
-            with self.fs.open(self.metadata_path) as f:
+        if self.common_metadata_path is not None:
+            with self.fs.open(self.common_metadata_path) as f:
                 self.common_metadata = ParquetFile(f).metadata
         else:
             self.common_metadata = None
@@ -718,21 +722,31 @@ def validate_schemas(self):
         open_file = self._get_open_file_func()
 
         if self.metadata is None and self.schema is None:
-            if self.metadata_path is not None:
-                self.schema = open_file(self.metadata_path).schema
+            if self.common_metadata_path is not None:
+                self.schema = open_file(self.common_metadata_path).schema
             else:
                 self.schema = self.pieces[0].get_metadata(open_file).schema
         elif self.schema is None:
             self.schema = self.metadata.schema
 
-        # Verify schemas are all equal
+        # Verify schemas are all compatible
+        dataset_schema = self.schema.to_arrow_schema()
+        # Exclude the partition columns from the schema, they are provided
+        # by the path, not the DatasetPiece
+        if self.partitions is not None:
+            for partition_name in self.partitions.partition_names:
+                if dataset_schema.get_field_index(partition_name) != -1:
+                    field_idx = dataset_schema.get_field_index(partition_name)
+                    dataset_schema = dataset_schema.remove(field_idx)
+
         for piece in self.pieces:
             file_metadata = piece.get_metadata(open_file)
-            if not self.schema.equals(file_metadata.schema):
-                raise ValueError('Schema in {0!s} was different. '
-                                 '{1!s} vs {2!s}'
-                                 .format(piece, file_metadata.schema,
-                                         self.schema))
+            file_schema = file_metadata.schema.to_arrow_schema()
+            if not dataset_schema.equals(file_schema):
+                raise ValueError('Schema in {0!s} was different. \n'
+                                 '{1!s}\n\nvs\n\n{2!s}'
+                                 .format(piece, file_schema,
+                                         dataset_schema))
 
     def read(self, columns=None, nthreads=1, use_pandas_metadata=False):
         """
@@ -831,7 +845,7 @@ def _ensure_filesystem(fs):
 
 def _make_manifest(path_or_paths, fs, pathsep='/'):
     partitions = None
-    metadata_path = None
+    common_metadata_path = None
 
     if len(path_or_paths) == 1:
         # Dask passes a directory as a list of length 1
@@ -840,7 +854,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
     if is_string(path_or_paths) and fs.isdir(path_or_paths):
         manifest = ParquetManifest(path_or_paths, filesystem=fs,
                                    pathsep=fs.pathsep)
-        metadata_path = manifest.metadata_path
+        common_metadata_path = manifest.common_metadata_path
         pieces = manifest.pieces
         partitions = manifest.partitions
     else:
@@ -859,7 +873,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
             piece = ParquetDatasetPiece(path)
             pieces.append(piece)
 
-    return pieces, partitions, metadata_path
+    return pieces, partitions, common_metadata_path
 
 
 _read_table_docstring = """
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 187971fb8564b..cec01c8598d1a 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1113,12 +1113,12 @@ def _test_read_common_metadata_files(fs, base_path):
     with fs.open(data_path, 'wb') as f:
         _write_table(table, f)
 
-    metadata_path = pjoin(base_path, '_metadata')
+    metadata_path = pjoin(base_path, '_common_metadata')
     with fs.open(metadata_path, 'wb') as f:
         pq.write_metadata(table.schema, f)
 
     dataset = pq.ParquetDataset(base_path, filesystem=fs)
-    assert dataset.metadata_path == metadata_path
+    assert dataset.common_metadata_path == metadata_path
 
     with fs.open(data_path) as f:
         common_schema = pq.read_metadata(f).schema
@@ -1431,7 +1431,14 @@ def _test_write_to_dataset_with_partitions(base_path, filesystem=None):
     output_table = pa.Table.from_pandas(output_df)
     pq.write_to_dataset(output_table, base_path, partition_by,
                         filesystem=filesystem)
-    input_table = pq.ParquetDataset(base_path, filesystem=filesystem).read()
+    pq.write_metadata(output_table.schema,
+                      os.path.join(base_path, '_common_metadata'))
+    dataset = pq.ParquetDataset(base_path, filesystem=filesystem)
+    # ARROW-2209: Ensure the dataset schema also includes the partition columns
+    dataset_cols = set(dataset.schema.to_arrow_schema().names)
+    assert dataset_cols == set(output_table.schema.names)
+
+    input_table = dataset.read()
     input_df = input_table.to_pandas()
 
     # Read data back in and compare with original DataFrame
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 849a0e016a60d..5f962901c3697 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -529,6 +529,64 @@ cdef class Schema:
     def get_field_index(self, name):
         return self.schema.GetFieldIndex(tobytes(name))
 
+    def append(self, Field field):
+        """
+        Append a field at the end of the schema.
+
+        Parameters
+        ----------
+
+        field: Field
+
+        Returns
+        -------
+        schema: Schema
+        """
+        return self.insert(self.schema.num_fields(), field)
+
+    def insert(self, int i, Field field):
+        """
+        Add a field at position i to the schema.
+
+        Parameters
+        ----------
+        i: int
+        field: Field
+
+        Returns
+        -------
+        schema: Schema
+        """
+        cdef:
+            shared_ptr[CSchema] new_schema
+            shared_ptr[CField] c_field
+
+        c_field = field.sp_field
+
+        with nogil:
+            check_status(self.schema.AddField(i, c_field, &new_schema))
+
+        return pyarrow_wrap_schema(new_schema)
+
+    def remove(self, int i):
+        """
+        Remove the field at index i from the schema.
+
+        Parameters
+        ----------
+        i: int
+
+        Returns
+        -------
+        schema: Schema
+        """
+        cdef shared_ptr[CSchema] new_schema
+
+        with nogil:
+            check_status(self.schema.RemoveField(i, &new_schema))
+
+        return pyarrow_wrap_schema(new_schema)
+
     def add_metadata(self, dict metadata):
         """
         Add metadata as dict of string keys and values to Schema

From 34c33f1f305a7f2602440fda49d28a7bbc0f9dcd Mon Sep 17 00:00:00 2001
From: Mitar <mitar.github@tnode.com>
Date: Sun, 4 Mar 2018 11:38:02 -0800
Subject: [PATCH 1553/1644] [Python] Document serialization parameter as
 "string" instead of "bytes"

---
 python/pyarrow/serialization.pxi | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index 44bf8773e6cb0..1ec607355aa92 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -105,8 +105,8 @@ cdef class SerializationContext:
         ----------
         type_ : TypeType
             The type that we can serialize.
-        type_id : bytes
-            A string of bytes used to identify the type.
+        type_id : string
+            A string used to identify the type.
         pickle : bool
             True if the serialization should be done with pickle.
             False if it should be done efficiently with Arrow.

From 8b1c8118b017a941f0102709d72df7e5a9783aa4 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Sun, 4 Mar 2018 14:40:12 -0500
Subject: [PATCH 1554/1644] ARROW-2245: ARROW-2246: [Python] Revert static
 linkage of parquet-cpp in manylinux1 wheel

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1692 from xhochy/ARROW-2245 and squashes the following commits:

fa33dd5e <Uwe L. Korn> ARROW-2245: ARROW-2246:  Revert static linkage of parquet-cpp in manylinux1 wheel
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake |  3 ++-
 cpp/src/plasma/CMakeLists.txt               | 10 +++++++
 python/CMakeLists.txt                       | 29 ++++++++++++++++++++-
 python/manylinux1/Dockerfile-x86_64         |  6 ++---
 python/manylinux1/build_arrow.sh            | 11 ++++----
 python/manylinux1/scripts/build_boost.sh    | 20 ++++++++------
 python/manylinux1/scripts/build_thrift.sh   |  8 +++---
 python/setup.py                             |  6 ++++-
 8 files changed, 70 insertions(+), 23 deletions(-)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 4103af41b53ad..c330e2ae3aefa 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -501,10 +501,11 @@ if (ARROW_JEMALLOC)
   set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
   set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(JEMALLOC_VENDORED 1)
+  # We need to disable TLS or otherwise C++ exceptions won't work anymore.
   ExternalProject_Add(jemalloc_ep
     URL ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
     PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
-    CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
+    CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_" "--disable-tls"
     ${EP_LOG_OPTIONS}
     BUILD_IN_SOURCE 1
     BUILD_COMMAND ${MAKE}
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 3448d009c9c89..bc00f980671ef 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -124,6 +124,16 @@ endif()
 add_executable(plasma_store store.cc)
 target_link_libraries(plasma_store plasma_static ${PLASMA_LINK_LIBS})
 
+if (ARROW_RPATH_ORIGIN)
+  if (APPLE)
+    set(_lib_install_rpath "@loader_path")
+  else()
+    set(_lib_install_rpath "\$ORIGIN")
+  endif()
+  set_target_properties(plasma_store PROPERTIES
+      INSTALL_RPATH ${_lib_install_rpath})
+endif()
+
 # Headers: top level
 install(FILES
   common.h
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index e9de08ba19730..72294d494e787 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -66,7 +66,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     ON)
   option(PYARROW_BOOST_USE_SHARED
     "Rely on boost shared libraries on linking static parquet"
-    OFF)
+    ON)
   option(PYARROW_BUILD_PLASMA
     "Build the PyArrow Plasma integration"
     OFF)
@@ -235,6 +235,24 @@ function(bundle_arrow_implib library_path)
       COPYONLY)
 endfunction(bundle_arrow_implib)
 
+function(bundle_boost_lib library_path)
+  get_filename_component(LIBRARY_NAME ${${library_path}} NAME)
+  get_filename_component(LIBRARY_NAME_WE ${${library_path}} NAME_WE)
+  configure_file(${${library_path}}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}
+      COPYONLY)
+  set(Boost_SO_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}")
+  if (APPLE)
+    configure_file(${${library_path}}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME_WE}.${Boost_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        COPYONLY)
+  else()
+    configure_file(${${library_path}}
+        ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME_WE}${CMAKE_SHARED_LIBRARY_SUFFIX}.${Boost_SO_VERSION}
+        COPYONLY)
+  endif()
+endfunction()
+
 # Always bundle includes
 file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
 
@@ -247,6 +265,15 @@ if (PYARROW_BUNDLE_ARROW_CPP)
     ABI_VERSION ${ARROW_ABI_VERSION}
     SO_VERSION ${ARROW_SO_VERSION})
 
+  # boost
+  if (PYARROW_BOOST_USE_SHARED)
+    set(Boost_USE_STATIC_LIBS OFF)
+    find_package(Boost COMPONENTS system filesystem regex REQUIRED)
+    bundle_boost_lib(Boost_REGEX_LIBRARY)
+    bundle_boost_lib(Boost_FILESYSTEM_LIBRARY)
+    bundle_boost_lib(Boost_SYSTEM_LIBRARY)
+  endif()
+
   if (MSVC)
     bundle_arrow_implib(ARROW_SHARED_IMP_LIB)
     bundle_arrow_implib(ARROW_PYTHON_SHARED_IMP_LIB)
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 62a0893292cbd..d48bd0d2c6efb 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,13 +14,13 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2212
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2245
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
 RUN mkdir build-plain
 WORKDIR /arrow/cpp/build-plain
-RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF ..
+RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=ON -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist ..
 RUN ninja install
 
 ADD scripts/check_arrow_visibility.sh /
@@ -29,5 +29,5 @@ RUN /check_arrow_visibility.sh
 WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BUILD_SHARED=OFF -DPARQUET_BUILD_STATIC=ON -DPARQUET_BOOST_USE_SHARED=OFF -GNinja .
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BUILD_SHARED=ON -DPARQUET_BUILD_STATIC=OFF -DPARQUET_BOOST_USE_SHARED=ON -DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist -GNinja .
 RUN ninja install
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 5fd27c8d03a3f..f83c759729c6b 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -38,11 +38,10 @@ cd /arrow/python
 # PyArrow build configuration
 export PYARROW_BUILD_TYPE='release'
 export PYARROW_WITH_PARQUET=1
-export PYARROW_WITH_STATIC_PARQUET=1
 export PYARROW_WITH_PLASMA=1
 export PYARROW_BUNDLE_ARROW_CPP=1
 export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
-export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr'
+export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist'
 # Ensure the target directory exists
 mkdir -p /io/dist
 
@@ -60,14 +59,14 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
     ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}-${U_WIDTH}
     mkdir -p "${ARROW_BUILD_DIR}"
     pushd "${ARROW_BUILD_DIR}"
-    PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON -DARROW_ORC=ON ..
-    make -j5 install
+    PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=ON -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON -DARROW_ORC=ON -DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist -GNinja ..
+    ninja install
     popd
 
     # Clear output directory
     rm -rf dist/
     echo "=== (${PYTHON}) Building wheel ==="
-    PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-static-parquet --bundle-arrow-cpp
+    PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp
     PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
     echo "=== (${PYTHON}) Test the existence of optional modules ==="
@@ -83,7 +82,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
     source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate
     pip install repaired_wheels/*.whl
 
-    py.test -v -r sxX --durations=15 --parquet /venv-test-${PYTHON}-${U_WIDTH}/lib/*/site-packages/pyarrow
+    py.test -v -r sxX --durations=15 --parquet ${VIRTUAL_ENV}/lib/*/site-packages/pyarrow
     deactivate
 
     mv repaired_wheels/*.whl /io/dist
diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh
index 1a6ffd7eb041f..47d826251dd7c 100755
--- a/python/manylinux1/scripts/build_boost.sh
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -21,18 +21,22 @@ BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_}
 
 wget --no-check-certificate https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz -O /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
 tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
+mkdir /arrow_boost
 pushd /boost_${BOOST_VERSION_UNDERSCORE}
 ./bootstrap.sh
-./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install
+./b2 tools/bcp
+./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build algorithm locale format /arrow_boost
 popd
-rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE}
+
+pushd /arrow_boost
+ls -l
+./bootstrap.sh
+./bjam cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11" variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem --with-date_time --with-system --with-regex install
+popd
+rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE} arrow_boost
 # Boost always install header-only parts but they also take up quite some space.
 # We don't need them in array, so don't persist them in the docker layer.
-# phoenix 18.1 MiB
-rm -r /usr/include/boost/phoenix
 # fusion 16.7 MiB
-rm -r /usr/include/boost/fusion
+rm -r /arrow_boost_dist/include/boost/fusion
 # spirit 8.2 MiB
-rm -r /usr/include/boost/spirit
-# geometry 6.0 MiB
-rm -r /usr/include/boost/geometry
+rm -r /arrow_boost_dist/include/boost/spirit
diff --git a/python/manylinux1/scripts/build_thrift.sh b/python/manylinux1/scripts/build_thrift.sh
index aaec4ad6bad41..451025c6646af 100755
--- a/python/manylinux1/scripts/build_thrift.sh
+++ b/python/manylinux1/scripts/build_thrift.sh
@@ -34,9 +34,11 @@ cmake -DCMAKE_BUILD_TYPE=release \
     "-DWITH_JAVA=OFF" \
     "-DWITH_PYTHON=OFF" \
     "-DWITH_CPP=ON" \
-    "-DWITH_STATIC_LIB=ON" ..
-make -j5
-make install
+    "-DWITH_STATIC_LIB=ON" \
+    -DBoost_NAMESPACE=arrow_boost \
+    -DBOOST_ROOT=/arrow_boost_dist \
+    -GNinja ..
+ninja install
 popd
 popd
 rm -rf thrift-${THRIFT_VERSION}.tar.gz thrift-${THRIFT_VERSION}
diff --git a/python/setup.py b/python/setup.py
index 2a6326489d3fa..34d2d9079e37f 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -120,7 +120,7 @@ def initialize_options(self):
         self.with_static_parquet = strtobool(
             os.environ.get('PYARROW_WITH_STATIC_PARQUET', '0'))
         self.with_static_boost = strtobool(
-            os.environ.get('PYARROW_WITH_STATIC_BOOST', '1'))
+            os.environ.get('PYARROW_WITH_STATIC_BOOST', '0'))
         self.with_plasma = strtobool(
             os.environ.get('PYARROW_WITH_PLASMA', '0'))
         self.with_orc = strtobool(
@@ -255,6 +255,10 @@ def _run_cmake(self):
                     move_shared_libs(build_prefix, build_lib, "plasma")
                 if self.with_parquet and not self.with_static_parquet:
                     move_shared_libs(build_prefix, build_lib, "parquet")
+                if not self.with_static_boost:
+                    move_shared_libs(build_prefix, build_lib, "arrow_boost_filesystem")
+                    move_shared_libs(build_prefix, build_lib, "arrow_boost_system")
+                    move_shared_libs(build_prefix, build_lib, "arrow_boost_regex")
 
             print('Bundling includes: ' + pjoin(build_prefix, 'include'))
             if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):

From 6e699d77693f22aeeb5ac47f96b6180dcc7f64c0 Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Sun, 4 Mar 2018 23:26:38 -0500
Subject: [PATCH 1555/1644] =?UTF-8?q?ARROW-2252:=20[Python]=C2=A0Create=20?=
 =?UTF-8?q?buffer=20from=20address,=20size=20and=20base?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Usage with Arrow Java vectors:

```
import jpype
import numpy as np
import pyarrow as pa
import sys

# Start JVM with Arrow and all of its dependencies.
jpype.startJVM(getDefaultJVMPath(), "-Djava.class.path=arrow-tools-0.9.0-SNAPSHOT-jar-with-dependencies.jar")

# Create vector
ra = jpype.JPackage("org").apache.arrow.memory.RootAllocator(sys.maxsize)
uint1 = jpype.JPackage("org").apache.arrow.vector.UInt1Vector("int", ra)
uint1.allocateNew(128)
for i in range(128):
    uint1.setSafe(i, i)
uint1.setValueCount(128)

# Access it in Python
addr = uint1.getDataBuffer().unwrap().memoryAddress()
size = uint1.getDataBuffer().unwrap().capacity()
fb = pa.ForeignBuffer(addr, size, n)
np.asarray(fb)
```

Output (as expected):
```
array([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,
        13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,
        26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,
        39,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,
        52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,
        65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,
        78,  79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,
        91,  92,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102, 103,
       104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,
       117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127], dtype=int8)
```

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1693 from xhochy/ARROW-2252 and squashes the following commits:

86a9a6df <Korn, Uwe> ARROW-2252:  Create buffer from address, size and base
---
 python/pyarrow/__init__.py      |  4 ++--
 python/pyarrow/io.pxi           | 12 ++++++++++++
 python/pyarrow/lib.pxd          |  5 +++++
 python/pyarrow/tests/test_io.py |  9 +++++++++
 4 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 15a37ca10d276..8cb4b3b9b7533 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -72,8 +72,8 @@
 from pyarrow.lib import TimestampType
 
 # Buffers, allocation
-from pyarrow.lib import (Buffer, ResizableBuffer, compress, decompress,
-                         allocate_buffer, frombuffer)
+from pyarrow.lib import (Buffer, ForeignBuffer, ResizableBuffer, compress,
+                         decompress, allocate_buffer, frombuffer)
 
 from pyarrow.lib import (MemoryPool, total_allocated_bytes,
                          set_memory_pool, default_memory_pool,
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 325c5827f00c0..5c8411be4cdbc 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -720,6 +720,18 @@ cdef class Buffer:
         return self.size
 
 
+cdef class ForeignBuffer(Buffer):
+
+    def __init__(self, addr, size, base):
+        cdef:
+            intptr_t c_addr = addr
+            int64_t c_size = size
+        self.base = base
+        cdef shared_ptr[CBuffer] buffer = make_shared[CBuffer](
+            <uint8_t*>c_addr, c_size)
+        self.init(<shared_ptr[CBuffer]> buffer)
+
+
 cdef class ResizableBuffer(Buffer):
 
     cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer):
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index e4d574f18b34f..c37bc2bebe7ea 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -324,6 +324,11 @@ cdef class Buffer:
     cdef int _check_nullptr(self) except -1
 
 
+cdef class ForeignBuffer(Buffer):
+    cdef:
+        object base
+
+
 cdef class ResizableBuffer(Buffer):
 
     cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index d269ad0e7cdae..17aca43331f3d 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -24,6 +24,7 @@
 import weakref
 
 import numpy as np
+import numpy.testing as npt
 
 import pandas as pd
 
@@ -253,6 +254,14 @@ def test_buffer_equals():
     assert buf2.equals(buf5)
 
 
+def test_foreign_buffer():
+    n = np.array([1, 2])
+    addr = n.__array_interface__["data"][0]
+    size = n.nbytes
+    fb = pa.ForeignBuffer(addr, size, n)
+    npt.assert_array_equal(np.asarray(fb), n.view(dtype=np.int8))
+
+
 def test_allocate_buffer():
     buf = pa.allocate_buffer(100)
     assert buf.size == 100

From 03db8a30bc68658733eb0ed26638136792f15639 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Sun, 4 Mar 2018 23:36:40 -0500
Subject: [PATCH 1556/1644] ARROW-2251: [GLib] Keep GArrowBuffer alive while
 GArrowTensor for the buffer is live

It prevents GC-related crash.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1691 from kou/glib-tensor-refer-buffer and squashes the following commits:

33ab8576 <Kouhei Sutou>  Keep GArrowBuffer alive while GArrowTensor for the buffer is live
---
 c_glib/arrow-glib/input-stream.cpp |  2 +-
 c_glib/arrow-glib/tensor.cpp       | 58 +++++++++++++++++++++++++-----
 c_glib/arrow-glib/tensor.hpp       |  3 +-
 3 files changed, 52 insertions(+), 11 deletions(-)

diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 94422241b9cfa..f602e5f7eb645 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -282,7 +282,7 @@ garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream *input_stream
                                        arrow_random_access_file.get(),
                                        &arrow_tensor);
   if (garrow_error_check(error, status, "[seekable-input-stream][read-tensor]")) {
-    return garrow_tensor_new_raw(&arrow_tensor);
+    return garrow_tensor_new_raw(&arrow_tensor, nullptr);
   } else {
     return NULL;
   }
diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp
index 3325f85113375..359831f67220d 100644
--- a/c_glib/arrow-glib/tensor.cpp
+++ b/c_glib/arrow-glib/tensor.cpp
@@ -40,11 +40,13 @@ G_BEGIN_DECLS
 
 typedef struct GArrowTensorPrivate_ {
   std::shared_ptr<arrow::Tensor> tensor;
+  GArrowBuffer *buffer;
 } GArrowTensorPrivate;
 
 enum {
   PROP_0,
-  PROP_TENSOR
+  PROP_TENSOR,
+  PROP_BUFFER
 };
 
 G_DEFINE_TYPE_WITH_PRIVATE(GArrowTensor, garrow_tensor, G_TYPE_OBJECT)
@@ -52,6 +54,19 @@ G_DEFINE_TYPE_WITH_PRIVATE(GArrowTensor, garrow_tensor, G_TYPE_OBJECT)
 #define GARROW_TENSOR_GET_PRIVATE(obj)                                   \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj), GARROW_TYPE_TENSOR, GArrowTensorPrivate))
 
+static void
+garrow_tensor_dispose(GObject *object)
+{
+  auto priv = GARROW_TENSOR_GET_PRIVATE(object);
+
+  if (priv->buffer) {
+    g_object_unref(priv->buffer);
+    priv->buffer = nullptr;
+  }
+
+  G_OBJECT_CLASS(garrow_tensor_parent_class)->dispose(object);
+}
+
 static void
 garrow_tensor_finalize(GObject *object)
 {
@@ -64,9 +79,9 @@ garrow_tensor_finalize(GObject *object)
 
 static void
 garrow_tensor_set_property(GObject *object,
-                          guint prop_id,
-                          const GValue *value,
-                          GParamSpec *pspec)
+                           guint prop_id,
+                           const GValue *value,
+                           GParamSpec *pspec)
 {
   auto priv = GARROW_TENSOR_GET_PRIVATE(object);
 
@@ -75,6 +90,9 @@ garrow_tensor_set_property(GObject *object,
     priv->tensor =
       *static_cast<std::shared_ptr<arrow::Tensor> *>(g_value_get_pointer(value));
     break;
+  case PROP_BUFFER:
+    priv->buffer = GARROW_BUFFER(g_value_dup_object(value));
+    break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
     break;
@@ -83,11 +101,16 @@ garrow_tensor_set_property(GObject *object,
 
 static void
 garrow_tensor_get_property(GObject *object,
-                          guint prop_id,
-                          GValue *value,
-                          GParamSpec *pspec)
+                           guint prop_id,
+                           GValue *value,
+                           GParamSpec *pspec)
 {
+  auto priv = GARROW_TENSOR_GET_PRIVATE(object);
+
   switch (prop_id) {
+  case PROP_BUFFER:
+    g_value_set_object(value, priv->buffer);
+    break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
     break;
@@ -106,6 +129,7 @@ garrow_tensor_class_init(GArrowTensorClass *klass)
 
   auto gobject_class = G_OBJECT_CLASS(klass);
 
+  gobject_class->dispose      = garrow_tensor_dispose;
   gobject_class->finalize     = garrow_tensor_finalize;
   gobject_class->set_property = garrow_tensor_set_property;
   gobject_class->get_property = garrow_tensor_get_property;
@@ -116,6 +140,14 @@ garrow_tensor_class_init(GArrowTensorClass *klass)
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property(gobject_class, PROP_TENSOR, spec);
+
+  spec = g_param_spec_object("buffer",
+                             "Buffer",
+                             "The data",
+                             GARROW_TYPE_BUFFER,
+                             static_cast<GParamFlags>(G_PARAM_READWRITE |
+                                                      G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_BUFFER, spec);
 }
 
 /**
@@ -166,7 +198,7 @@ garrow_tensor_new(GArrowDataType *data_type,
                                     arrow_shape,
                                     arrow_strides,
                                     arrow_dimension_names);
-  auto tensor = garrow_tensor_new_raw(&arrow_tensor);
+  auto tensor = garrow_tensor_new_raw(&arrow_tensor, data);
   return tensor;
 }
 
@@ -231,6 +263,12 @@ garrow_tensor_get_value_type(GArrowTensor *tensor)
 GArrowBuffer *
 garrow_tensor_get_buffer(GArrowTensor *tensor)
 {
+  auto priv = GARROW_TENSOR_GET_PRIVATE(tensor);
+  if (priv->buffer) {
+    g_object_ref(priv->buffer);
+    return priv->buffer;
+  }
+
   auto arrow_tensor = garrow_tensor_get_raw(tensor);
   auto arrow_buffer = arrow_tensor->data();
   return garrow_buffer_new_raw(&arrow_buffer);
@@ -398,10 +436,12 @@ garrow_tensor_is_column_major(GArrowTensor *tensor)
 G_END_DECLS
 
 GArrowTensor *
-garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor)
+garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor,
+                      GArrowBuffer *buffer)
 {
   auto tensor = GARROW_TENSOR(g_object_new(GARROW_TYPE_TENSOR,
                                            "tensor", arrow_tensor,
+                                           "buffer", buffer,
                                            NULL));
   return tensor;
 }
diff --git a/c_glib/arrow-glib/tensor.hpp b/c_glib/arrow-glib/tensor.hpp
index 392aeeebb6d2c..8e54e492aa141 100644
--- a/c_glib/arrow-glib/tensor.hpp
+++ b/c_glib/arrow-glib/tensor.hpp
@@ -23,5 +23,6 @@
 
 #include <arrow-glib/tensor.h>
 
-GArrowTensor *garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor);
+GArrowTensor *garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor,
+                                    GArrowBuffer *buffer);
 std::shared_ptr<arrow::Tensor> garrow_tensor_get_raw(GArrowTensor *tensor);

From 9ceda35a6860c837a91aeca48e7c1d3996e184b5 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 5 Mar 2018 09:25:37 +0100
Subject: [PATCH 1557/1644] ARROW-2244: [C++] Add unit test to explicitly check
 that NullArray internal data set correctly in Slice operations

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1696 from wesm/ARROW-2244 and squashes the following commits:

7cf2bed <Wes McKinney> Add test for ARROW-2244
---
 cpp/src/arrow/array-test.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 4aaf182569940..c3ac082864bb8 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -136,9 +136,15 @@ TEST_F(TestArray, SliceRecomputeNullCount) {
 
   auto arr = std::make_shared<Int32Array>(16, data, nullptr, -1);
   ASSERT_EQ(0, arr->null_count());
+}
 
+TEST_F(TestArray, NullArraySliceNullCount) {
   auto null_arr = std::make_shared<NullArray>(10);
   auto null_arr_sliced = null_arr->Slice(3, 6);
+
+  // The internal null count is 6, does not require recomputation
+  ASSERT_EQ(6, null_arr_sliced->data()->null_count);
+
   ASSERT_EQ(6, null_arr_sliced->null_count());
 }
 

From b89c1249b9ad76fa217072fefe7479b06bda593f Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 5 Mar 2018 09:26:51 +0100
Subject: [PATCH 1558/1644] =?UTF-8?q?ARROW-2253:=20[Python]=C2=A0Support?=
 =?UTF-8?q?=20=5F=5Feq=5F=5F=20on=20scalar=20values?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1695 from xhochy/ARROW-2253 and squashes the following commits:

1b1fb60 <Uwe L. Korn> Update cython version
c850c93 <Uwe L. Korn> ARROW-2253:  Support __eq__ on scalar values
---
 python/manylinux1/Dockerfile-x86_64            | 2 +-
 python/manylinux1/scripts/build_virtualenvs.sh | 2 +-
 python/pyarrow/scalar.pxi                      | 9 +++++++++
 python/pyarrow/tests/test_scalars.py           | 7 +++++++
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index d48bd0d2c6efb..d5117da8c763c 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2245
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2253
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
index 220c26003e9f9..7e0d80cc756f9 100755
--- a/python/manylinux1/scripts/build_virtualenvs.sh
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -34,7 +34,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
 
     echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ==="
     $PIP install "numpy==1.10.4"
-    $PIP install "cython==0.25.2"
+    $PIP install "cython==0.27.3"
     $PIP install "pandas==0.20.3"
     $PIP install "virtualenv==15.1.0"
 
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index 1bc5ed7a372a8..a801acd69c582 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -64,6 +64,15 @@ cdef class ArrayValue(Scalar):
         else:
             return super(Scalar, self).__repr__()
 
+    def __eq__(self, other):
+        if hasattr(self, 'as_py'):
+            if isinstance(other, ArrayValue):
+                other = other.as_py()
+            return self.as_py() == other
+        else:
+            raise NotImplementedError(
+                "Cannot compare Arrow values that don't support as_py()")
+
 
 cdef class BooleanValue(ArrayValue):
 
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 0aa9466936503..7061a0d3a175c 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -58,6 +58,7 @@ def test_int64(self):
         assert isinstance(v, pa.Int64Value)
         assert repr(v) == "1"
         assert v.as_py() == 1
+        assert v == 1
 
         assert arr[2] is pa.NA
 
@@ -68,6 +69,7 @@ def test_double(self):
         assert isinstance(v, pa.DoubleValue)
         assert repr(v) == "1.5"
         assert v.as_py() == 1.5
+        assert v == 1.5
 
         assert arr[1] is pa.NA
 
@@ -80,6 +82,10 @@ def test_string_unicode(self):
         v = arr[0]
         assert isinstance(v, pa.StringValue)
         assert v.as_py() == 'foo'
+        assert v == 'foo'
+        # Assert that newly created values are equal to the previously created
+        # one.
+        assert v == arr[0]
 
         assert arr[1] is pa.NA
 
@@ -93,6 +99,7 @@ def test_bytes(self):
         v = arr[0]
         assert isinstance(v, pa.BinaryValue)
         assert v.as_py() == b'foo'
+        assert v == b'foo'
 
         assert arr[1] is pa.NA
 

From 49f1d006b58e2ed3c041ba5697c112ee1f7e105e Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 5 Mar 2018 09:44:12 -0500
Subject: [PATCH 1559/1644] =?UTF-8?q?ARROW-2258:=20[Python]=C2=A0Add=20add?=
 =?UTF-8?q?itional=20information=20to=20find=20Boost=20on=20windows?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

cc @MaxRis

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1700 from xhochy/ARROW-2258 and squashes the following commits:

a8c00b22 <Uwe L. Korn> ARROW-2258:  Add additional information to find Boost on windows
---
 ci/msvc-build.bat     |  2 +-
 python/CMakeLists.txt | 18 ++++++++++++++++++
 python/setup.py       |  3 +++
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index b6f672da9aeb1..beefee6c0fc80 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -133,7 +133,7 @@ popd
 pushd python
 
 set PYARROW_CXXFLAGS=/WX
-python setup.py build_ext --with-parquet --bundle-arrow-cpp ^
+python setup.py build_ext --with-parquet --bundle-arrow-cpp --with-static-boost ^
     install -q --single-version-externally-managed --record=record.text ^
     bdist_wheel || exit /B
 
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 72294d494e787..d17194628480a 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -268,6 +268,24 @@ if (PYARROW_BUNDLE_ARROW_CPP)
   # boost
   if (PYARROW_BOOST_USE_SHARED)
     set(Boost_USE_STATIC_LIBS OFF)
+    set(Boost_USE_MULTITHREADED ON)
+    if (MSVC AND ARROW_USE_STATIC_CRT)
+      set(Boost_USE_STATIC_RUNTIME ON)
+    endif()
+    set(Boost_ADDITIONAL_VERSIONS
+      "1.66.0" "1.66"
+      "1.65.0" "1.65"
+      "1.64.0" "1.64"
+      "1.63.0" "1.63"
+      "1.62.0" "1.61"
+      "1.61.0" "1.62"
+      "1.60.0" "1.60")
+    list(GET Boost_ADDITIONAL_VERSIONS 0 BOOST_LATEST_VERSION)
+    string(REPLACE "." "_" BOOST_LATEST_VERSION_IN_PATH ${BOOST_LATEST_VERSION})
+    if (MSVC)
+      # disable autolinking in boost
+      add_definitions(-DBOOST_ALL_NO_LIB)
+    endif()
     find_package(Boost COMPONENTS system filesystem regex REQUIRED)
     bundle_boost_lib(Boost_REGEX_LIBRARY)
     bundle_boost_lib(Boost_FILESYSTEM_LIBRARY)
diff --git a/python/setup.py b/python/setup.py
index 34d2d9079e37f..ddf7cc55aad3b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -96,6 +96,7 @@ def run(self):
                      ('build-type=', None, 'build type (debug or release)'),
                      ('with-parquet', None, 'build the Parquet extension'),
                      ('with-static-parquet', None, 'link parquet statically'),
+                     ('with-static-boost', None, 'link boost statically'),
                      ('with-plasma', None, 'build the Plasma extension'),
                      ('with-orc', None, 'build the ORC extension'),
                      ('bundle-arrow-cpp', None,
@@ -170,6 +171,8 @@ def _run_cmake(self):
                 cmake_options.append('-DPYARROW_PARQUET_USE_SHARED=off')
             if not self.with_static_boost:
                 cmake_options.append('-DPYARROW_BOOST_USE_SHARED=on')
+            else:
+                cmake_options.append('-DPYARROW_BOOST_USE_SHARED=off')
 
             if self.with_plasma:
                 cmake_options.append('-DPYARROW_BUILD_PLASMA=on')

From 55bdae57c5f712d691e928899518fa41ec2d8acd Mon Sep 17 00:00:00 2001
From: "Korn, Uwe" <Uwe.Korn@blue-yonder.com>
Date: Mon, 5 Mar 2018 10:51:29 -0500
Subject: [PATCH 1560/1644] ARROW-2254: [Python] Ignore JS tags in local dev
 versions

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #1699 from xhochy/ARROW-2254 and squashes the following commits:

3c41b363 <Korn, Uwe> ARROW-2254:  Ignore JS tags in local dev versions
---
 python/pyarrow/__init__.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 8cb4b3b9b7533..28ac98ea06984 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -23,8 +23,22 @@
 except DistributionNotFound:
    # package is not installed
     try:
+        # This code is duplicated from setup.py to avoid a dependency on each
+        # other.
+        def parse_version(root):
+            from setuptools_scm import version_from_scm
+            import setuptools_scm.git
+            describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --match 'apache-arrow-[0-9]*'"
+            # Strip catchall from the commandline
+            describe = describe.replace("--match *.*", "")
+            version = setuptools_scm.git.parse(root, describe)
+            if not version:
+                return version_from_scm(root)
+            else:
+                return version
+
         import setuptools_scm
-        __version__ = setuptools_scm.get_version('../')
+        __version__ = setuptools_scm.get_version('../', parse=parse_version)
     except (ImportError, LookupError):
         __version__ = None
 

From c6359cbcb61b904e49efbb33b70d3be040622e9e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 5 Mar 2018 14:20:45 -0500
Subject: [PATCH 1561/1644] ARROW-1929: [C++] Copy over testing utility code
 from PARQUET-1092

This code was introduced in parquet-cpp in https://github.com/apache/parquet-cpp/pull/426

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1697 from wesm/ARROW-1929 and squashes the following commits:

3f34b389 <Wes McKinney> Change namespace to avoid conflict with parquet-cpp builds
3cccaea6 <Wes McKinney> Migrate testing utility code from PARQUET-1092
---
 cpp/src/arrow/array-test.cc |  6 +--
 cpp/src/arrow/table-test.cc |  6 +--
 cpp/src/arrow/test-util.h   | 85 ++++++++++++++++++++++++++++++-------
 3 files changed, 75 insertions(+), 22 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index c3ac082864bb8..bda1946c6a72a 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -2480,19 +2480,19 @@ TEST_F(TestListArray, TestFromArrays) {
 
   ListArray expected1(list_type, length, offsets1->data()->buffers[1], values,
                       offsets1->data()->buffers[0], 0);
-  AssertArraysEqual(expected1, *list1);
+  test::AssertArraysEqual(expected1, *list1);
 
   // Use null bitmap from offsets3, but clean offsets from non-null version
   ListArray expected3(list_type, length, offsets1->data()->buffers[1], values,
                       offsets3->data()->buffers[0], 1);
-  AssertArraysEqual(expected3, *list3);
+  test::AssertArraysEqual(expected3, *list3);
 
   // Check that the last offset bit is zero
   ASSERT_TRUE(BitUtil::BitNotSet(list3->null_bitmap()->data(), length + 1));
 
   ListArray expected4(list_type, length, offsets2->data()->buffers[1], values,
                       offsets4->data()->buffers[0], 1);
-  AssertArraysEqual(expected4, *list4);
+  test::AssertArraysEqual(expected4, *list4);
 
   // Test failure modes
 
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index af744168227cd..24c8d5e15bef3 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -116,11 +116,11 @@ TEST_F(TestChunkedArray, SliceEquals) {
 
   std::shared_ptr<ChunkedArray> slice = one_->Slice(125, 50);
   ASSERT_EQ(slice->length(), 50);
-  ASSERT_TRUE(slice->Equals(one_->Slice(125, 50)));
+  test::AssertChunkedEqual(*one_->Slice(125, 50), *slice);
 
   std::shared_ptr<ChunkedArray> slice2 = one_->Slice(75)->Slice(25)->Slice(25, 50);
   ASSERT_EQ(slice2->length(), 50);
-  ASSERT_TRUE(slice2->Equals(slice));
+  test::AssertChunkedEqual(*slice, *slice2);
 }
 
 class TestColumn : public TestChunkedArray {
@@ -390,7 +390,7 @@ TEST_F(TestTable, ConcatenateTables) {
 
   ASSERT_OK(ConcatenateTables({t1, t2}, &result));
   ASSERT_OK(Table::FromRecordBatches({batch1, batch2}, &expected));
-  ASSERT_TRUE(result->Equals(*expected));
+  test::AssertTablesEqual(*expected, *result);
 
   // Error states
   std::vector<std::shared_ptr<Table>> empty_tables;
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 1a34808488a83..ab68fd44258bc 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -35,6 +35,7 @@
 #include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
 #include "arrow/status.h"
+#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -77,6 +78,18 @@ namespace arrow {
 
 using ArrayVector = std::vector<std::shared_ptr<Array>>;
 
+#define ASSERT_ARRAYS_EQUAL(LEFT, RIGHT)                                               \
+  do {                                                                                 \
+    if (!(LEFT).Equals((RIGHT))) {                                                     \
+      std::stringstream pp_result;                                                     \
+      std::stringstream pp_expected;                                                   \
+                                                                                       \
+      EXPECT_OK(PrettyPrint(RIGHT, 0, &pp_result));                                    \
+      EXPECT_OK(PrettyPrint(LEFT, 0, &pp_expected));                                   \
+      FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str(); \
+    }                                                                                  \
+  } while (false)
+
 namespace test {
 
 template <typename T, typename U>
@@ -288,6 +301,62 @@ Status MakeRandomBytePoolBuffer(int64_t length, MemoryPool* pool,
   return Status::OK();
 }
 
+void AssertArraysEqual(const Array& expected, const Array& actual) {
+  ASSERT_ARRAYS_EQUAL(expected, actual);
+}
+
+void AssertChunkedEqual(const ChunkedArray& expected, const ChunkedArray& actual) {
+  ASSERT_EQ(expected.num_chunks(), actual.num_chunks()) << "# chunks unequal";
+  if (!actual.Equals(expected)) {
+    std::stringstream pp_result;
+    std::stringstream pp_expected;
+
+    for (int i = 0; i < actual.num_chunks(); ++i) {
+      auto c1 = actual.chunk(i);
+      auto c2 = expected.chunk(i);
+      if (!c1->Equals(*c2)) {
+        EXPECT_OK(::arrow::PrettyPrint(*c1, 0, &pp_result));
+        EXPECT_OK(::arrow::PrettyPrint(*c2, 0, &pp_expected));
+        FAIL() << "Chunk " << i << " Got: " << pp_result.str()
+               << "\nExpected: " << pp_expected.str();
+      }
+    }
+  }
+}
+
+void PrintColumn(const Column& col, std::stringstream* ss) {
+  const ChunkedArray& carr = *col.data();
+  for (int i = 0; i < carr.num_chunks(); ++i) {
+    auto c1 = carr.chunk(i);
+    *ss << "Chunk " << i << std::endl;
+    EXPECT_OK(::arrow::PrettyPrint(*c1, 0, ss));
+    *ss << std::endl;
+  }
+}
+
+void AssertTablesEqual(const Table& expected, const Table& actual,
+                       bool same_chunk_layout = true) {
+  ASSERT_EQ(expected.num_columns(), actual.num_columns());
+
+  if (same_chunk_layout) {
+    for (int i = 0; i < actual.num_columns(); ++i) {
+      AssertChunkedEqual(*expected.column(i)->data(), *actual.column(i)->data());
+    }
+  } else {
+    std::stringstream ss;
+    if (!actual.Equals(expected)) {
+      for (int i = 0; i < expected.num_columns(); ++i) {
+        ss << "Actual column " << i << std::endl;
+        PrintColumn(*actual.column(i), &ss);
+
+        ss << "Expected column " << i << std::endl;
+        PrintColumn(*expected.column(i), &ss);
+      }
+      FAIL() << ss.str();
+    }
+  }
+}
+
 }  // namespace test
 
 template <typename TYPE, typename C_TYPE>
@@ -352,26 +421,10 @@ Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>&
   return builder->Finish(out);
 }
 
-#define ASSERT_ARRAYS_EQUAL(LEFT, RIGHT)                                               \
-  do {                                                                                 \
-    if (!(LEFT).Equals((RIGHT))) {                                                     \
-      std::stringstream pp_result;                                                     \
-      std::stringstream pp_expected;                                                   \
-                                                                                       \
-      EXPECT_OK(PrettyPrint(RIGHT, 0, &pp_result));                                    \
-      EXPECT_OK(PrettyPrint(LEFT, 0, &pp_expected));                                   \
-      FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str(); \
-    }                                                                                  \
-  } while (false)
-
 #define DECL_T() typedef typename TestFixture::T T;
 
 #define DECL_TYPE() typedef typename TestFixture::Type Type;
 
-void AssertArraysEqual(const Array& expected, const Array& actual) {
-  ASSERT_ARRAYS_EQUAL(expected, actual);
-}
-
 #define ASSERT_BATCHES_EQUAL(LEFT, RIGHT)    \
   do {                                       \
     if (!(LEFT).ApproxEquals(RIGHT)) {       \

From 45f5da2f4b841c5562b14727361da421f5fb64a2 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 5 Mar 2018 14:28:35 -0500
Subject: [PATCH 1562/1644] ARROW-1982: [Python] Coerce Parquet statistics as
 bytes to more useful Python scalar types

I also changed the BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY to return bytes since decoding from binary to UTF8 unicode didn't seem correct to me as the default behavior

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1698 from wesm/ARROW-1982 and squashes the following commits:

0c6b23a7 <Wes McKinney> Update comments per code review
d0ae6f40 <Wes McKinney> Coerce Parquet statistics as bytes to more useful Python scalar types
---
 python/pyarrow/_parquet.pyx          | 29 ++++++++++++++++++++++++++--
 python/pyarrow/tests/test_parquet.py | 24 +++++++++++------------
 2 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index e513e1d924401..101fcd165e4b5 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -70,6 +70,31 @@ cdef class RowGroupStatistics:
                                self.num_values,
                                self.physical_type)
 
+    cdef inline _cast_statistic(self, object value):
+        # Input value is bytes
+        cdef ParquetType physical_type = self.statistics.get().physical_type()
+        if physical_type == ParquetType_BOOLEAN:
+            return bool(int(value))
+        elif physical_type == ParquetType_INT32:
+            return int(value)
+        elif physical_type == ParquetType_INT64:
+            return int(value)
+        elif physical_type == ParquetType_INT96:
+            # Leave as PyBytes
+            return value
+        elif physical_type == ParquetType_FLOAT:
+            return float(value)
+        elif physical_type == ParquetType_DOUBLE:
+            return float(value)
+        elif physical_type == ParquetType_BYTE_ARRAY:
+            # Leave as PyBytes
+            return value
+        elif physical_type == ParquetType_FIXED_LEN_BYTE_ARRAY:
+            # Leave as PyBytes
+            return value
+        else:
+            raise ValueError('Unknown physical ParquetType')
+
     property has_min_max:
 
         def __get__(self):
@@ -82,7 +107,7 @@ cdef class RowGroupStatistics:
             encode_min = self.statistics.get().EncodeMin()
 
             min_value = FormatStatValue(raw_physical_type, encode_min.c_str())
-            return frombytes(min_value)
+            return self._cast_statistic(min_value)
 
     property max:
 
@@ -91,7 +116,7 @@ cdef class RowGroupStatistics:
             encode_max = self.statistics.get().EncodeMax()
 
             max_value = FormatStatValue(raw_physical_type, encode_max.c_str())
-            return frombytes(max_value)
+            return self._cast_statistic(max_value)
 
     property null_count:
 
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index cec01c8598d1a..a3da05fe36a82 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -26,7 +26,7 @@
 
 import pytest
 
-from pyarrow.compat import guid, u, BytesIO, unichar, frombytes
+from pyarrow.compat import guid, u, BytesIO, unichar
 from pyarrow.tests import util
 from pyarrow.filesystem import LocalFileSystem
 import pyarrow as pa
@@ -524,20 +524,20 @@ def test_parquet_metadata_api():
 @pytest.mark.parametrize(
     'data, dtype, min_value, max_value, null_count, num_values',
     [
-        ([1, 2, 2, None, 4], np.uint8, u'1', u'4', 1, 4),
-        ([1, 2, 2, None, 4], np.uint16, u'1', u'4', 1, 4),
-        ([1, 2, 2, None, 4], np.uint32, u'1', u'4', 1, 4),
-        ([1, 2, 2, None, 4], np.uint64, u'1', u'4', 1, 4),
-        ([-1, 2, 2, None, 4], np.int16, u'-1', u'4', 1, 4),
-        ([-1, 2, 2, None, 4], np.int32, u'-1', u'4', 1, 4),
-        ([-1, 2, 2, None, 4], np.int64, u'-1', u'4', 1, 4),
-        ([-1.1, 2.2, 2.3, None, 4.4], np.float32, u'-1.1', u'4.4', 1, 4),
-        ([-1.1, 2.2, 2.3, None, 4.4], np.float64, u'-1.1', u'4.4', 1, 4),
+        ([1, 2, 2, None, 4], np.uint8, 1, 4, 1, 4),
+        ([1, 2, 2, None, 4], np.uint16, 1, 4, 1, 4),
+        ([1, 2, 2, None, 4], np.uint32, 1, 4, 1, 4),
+        ([1, 2, 2, None, 4], np.uint64, 1, 4, 1, 4),
+        ([-1, 2, 2, None, 4], np.int16, -1, 4, 1, 4),
+        ([-1, 2, 2, None, 4], np.int32, -1, 4, 1, 4),
+        ([-1, 2, 2, None, 4], np.int64, -1, 4, 1, 4),
+        ([-1.1, 2.2, 2.3, None, 4.4], np.float32, -1.1, 4.4, 1, 4),
+        ([-1.1, 2.2, 2.3, None, 4.4], np.float64, -1.1, 4.4, 1, 4),
         (
             [u'', u'b', unichar(1000), None, u'aaa'],
-            str, u' ', frombytes((unichar(1000) + u' ').encode('utf-8')), 1, 4
+            str, b' ', (unichar(1000) + u' ').encode('utf-8'), 1, 4
         ),
-        ([True, False, False, True, True], np.bool, u'0', u'1', 0, 5),
+        ([True, False, False, True, True], np.bool, False, True, 0, 5),
     ]
 )
 def test_parquet_column_statistics_api(

From 01a099c81df7a54c5b22f944a77f7f20dbebc7ee Mon Sep 17 00:00:00 2001
From: Sidd <siddharth@dremio.com>
Date: Mon, 5 Mar 2018 15:01:18 -0800
Subject: [PATCH 1563/1644] ARROW-2199: [JAVA] Control the memory allocated for
 inner vectors in containers. (#1646)

* ARROW-2199: [JAVA] Control the memory allocated for inner vector in LIST.

Use density based setInitialCapacity and propagate density down the vector
tree from complex vectors. Also ensure that density driven initial capacity
is never less than 1.

* comment cleanup

* use Integer.MAX_VALUE instead of 2billion constant

* review comments

* review comments

* address review comments for realloc and tests
---
 .../apache/arrow/memory/BaseAllocator.java    |  6 ++
 .../main/codegen/templates/UnionVector.java   |  1 +
 .../arrow/vector/BaseFixedWidthVector.java    |  1 +
 .../arrow/vector/BaseVariableWidthVector.java | 10 ++--
 .../arrow/vector/DensityAwareVector.java      | 57 +++++++++++++++++++
 .../arrow/vector/VariableWidthVector.java     |  2 +-
 .../complex/AbstractContainerVector.java      |  3 +-
 .../complex/BaseRepeatedValueVector.java      | 20 +++++--
 .../vector/complex/FixedSizeListVector.java   |  1 +
 .../arrow/vector/complex/ListVector.java      |  9 ++-
 .../complex/NonNullableStructVector.java      | 11 ++++
 .../vector/complex/RepeatedValueVector.java   |  3 +-
 .../arrow/vector/complex/StructVector.java    |  7 +++
 .../apache/arrow/vector/TestListVector.java   | 23 +++++---
 .../apache/arrow/vector/TestValueVector.java  | 12 ++--
 15 files changed, 137 insertions(+), 29 deletions(-)
 create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/DensityAwareVector.java

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 5411baf7bdfe3..2f70f737214dd 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -134,6 +134,9 @@ private static String createErrorMsg(final BufferAllocator allocator, final int
    * @return The closest power of two of that value.
    */
   public static int nextPowerOfTwo(int val) {
+    if (val == 0 || val == 1) {
+      return val + 1;
+    }
     int highestBit = Integer.highestOneBit(val);
     if (highestBit == val) {
       return val;
@@ -149,6 +152,9 @@ public static int nextPowerOfTwo(int val) {
    * @return The closest power of two of that value.
    */
   public static long nextPowerOfTwo(long val) {
+    if (val == 0 || val == 1) {
+      return val + 1;
+    }
     long highestBit = Long.highestOneBit(val);
     if (highestBit == val) {
       return val;
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 84450bee5026e..1cfa0666a3139 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -282,6 +282,7 @@ private void reallocTypeBuffer() {
 
     long newAllocationSize = baseSize * 2L;
     newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+    assert newAllocationSize >= 1;
 
     if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index cbc56fe3dde26..4b47df8a450a0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -444,6 +444,7 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer)
 
     long newAllocationSize = baseSize * 2L;
     newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+    assert newAllocationSize >= 1;
 
     if (newAllocationSize > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index c32d20f180dd8..ecb3c780efca4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -174,14 +174,14 @@ public void setInitialCapacity(int valueCount) {
    * @param valueCount desired number of elements in the vector
    * @param density average number of bytes per variable width element
    */
+  @Override
   public void setInitialCapacity(int valueCount, double density) {
-    final long size = (long) (valueCount * density);
-    if (size < 1) {
-      throw new IllegalArgumentException("With the provided density and value count, potential capacity of the data buffer is 0");
-    }
+    long size = Math.max((long)(valueCount * density), 1L);
+
     if (size > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
     }
+
     valueAllocationSizeInBytes = (int) size;
     validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
     /* to track the end offset of last data element in vector, we need
@@ -489,6 +489,7 @@ public void reallocDataBuffer() {
 
     long newAllocationSize = baseSize * 2L;
     newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+    assert newAllocationSize >= 1;
 
     if (newAllocationSize > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer");
@@ -541,6 +542,7 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer
 
     long newAllocationSize = baseSize * 2L;
     newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+    assert newAllocationSize >= 1;
 
     if (newAllocationSize > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DensityAwareVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DensityAwareVector.java
new file mode 100644
index 0000000000000..2915661a429f5
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DensityAwareVector.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+/**
+ * Vector that support density aware initial capacity settings.
+ * We use this for ListVector and VarCharVector as of now to
+ * control the memory allocated.
+ *
+ * For ListVector, we have been using a multiplier of 5
+ * to compute the initial capacity of the inner data vector.
+ * For deeply nested lists and lists with lots of NULL values,
+ * this is over-allocation upfront. So density helps to be
+ * conservative when computing the value capacity of the
+ * inner vector.
+ *
+ * For example, a density value of 10 implies each position in the
+ * list vector has a list of 10 values. So we will provision
+ * an initial capacity of (valuecount * 10) for the inner vector.
+ * A density value of 0.1 implies out of 10 positions in the list vector,
+ * 1 position has a list of size 1 and remaining positions are
+ * null (no lists) or empty lists. This helps in tightly controlling
+ * the memory we provision for inner data vector.
+ *
+ * Similar analogy is applicable for VarCharVector where the capacity
+ * of the data buffer can be controlled using density multiplier
+ * instead of default multiplier of 8 (default size of average
+ * varchar length).
+ *
+ * Also from container vectors, we propagate the density down
+ * the the inner vectors so that they can use it appropriately.
+ */
+public interface DensityAwareVector {
+
+  /**
+   * Set value with density
+   * @param valueCount
+   * @param density
+   */
+  void setInitialCapacity(int valueCount, double density);
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
index 593d4dceaf782..f91a5c8b543c2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
@@ -18,7 +18,7 @@
 
 package org.apache.arrow.vector;
 
-public interface VariableWidthVector extends ValueVector {
+public interface VariableWidthVector extends ValueVector, DensityAwareVector {
 
   /**
    * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index a99f3c8cdb7ad..d2a3c4a62ef91 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -20,6 +20,7 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.DensityAwareVector;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
@@ -33,7 +34,7 @@
  *
  * This class implements common functionality of composite vectors.
  */
-public abstract class AbstractContainerVector implements ValueVector {
+public abstract class AbstractContainerVector implements ValueVector, DensityAwareVector {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
 
   protected final String name;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 50ee3a7573efe..2dd2894ffebf9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -25,6 +25,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.DensityAwareVector;
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
@@ -108,6 +109,7 @@ protected void reallocOffsetBuffer() {
 
     long newAllocationSize = baseSize * 2L;
     newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+    assert newAllocationSize >= 1;
 
     if (newAllocationSize > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer");
@@ -139,7 +141,7 @@ public void setInitialCapacity(int numRecords) {
     if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) {
       vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
     } else {
-     vector.setInitialCapacity(numRecords);
+      vector.setInitialCapacity(numRecords);
     }
   }
 
@@ -166,13 +168,21 @@ public void setInitialCapacity(int numRecords) {
    *                This helps in tightly controlling the memory we provision
    *                for inner data vector.
    */
+  @Override
   public void setInitialCapacity(int numRecords, double density) {
+    if ((numRecords * density) >= Integer.MAX_VALUE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
+    }
+
     offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH;
-    final int innerValueCapacity = (int)(numRecords * density);
-    if (innerValueCapacity < 1) {
-      throw new IllegalArgumentException("With the provided density and value count, potential value capacity for the data vector is 0");
+
+    int innerValueCapacity = Math.max((int)(numRecords * density), 1);
+
+    if (vector instanceof DensityAwareVector) {
+      ((DensityAwareVector)vector).setInitialCapacity(innerValueCapacity, density);
+    } else {
+      vector.setInitialCapacity(innerValueCapacity);
     }
-    vector.setInitialCapacity(innerValueCapacity);
   }
 
   @Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 9314a2566b124..eadbab43619a6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -210,6 +210,7 @@ private void reallocValidityBuffer() {
 
     long newAllocationSize = baseSize * 2L;
     newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+    assert newAllocationSize >= 1;
 
     if (newAllocationSize > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index b472dae069431..d3eeaf2f0df7a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -31,7 +31,12 @@
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.BitVectorHelper;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
@@ -126,6 +131,7 @@ public void setInitialCapacity(int numRecords) {
    *                This helps in tightly controlling the memory we provision
    *                for inner data vector.
    */
+  @Override
   public void setInitialCapacity(int numRecords, double density) {
     validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
     super.setInitialCapacity(numRecords, density);
@@ -287,6 +293,7 @@ private void reallocValidityBuffer() {
 
     long newAllocationSize = baseSize * 2L;
     newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+    assert newAllocationSize >= 1;
 
     if (newAllocationSize > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
index aec06b60c1803..c41cbf262dc67 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
@@ -99,6 +99,17 @@ public void setInitialCapacity(int numRecords) {
     }
   }
 
+  @Override
+  public void setInitialCapacity(int valueCount, double density) {
+    for (final ValueVector vector : (Iterable<ValueVector>) this) {
+      if (vector instanceof DensityAwareVector) {
+        ((DensityAwareVector)vector).setInitialCapacity(valueCount, density);
+      } else {
+        vector.setInitialCapacity(valueCount);
+      }
+    }
+  }
+
   @Override
   public int getBufferSize() {
     if (valueCount == 0 || size() == 0) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index 36401172994c5..a7f6d4393796d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -18,6 +18,7 @@
 
 package org.apache.arrow.vector.complex;
 
+import org.apache.arrow.vector.DensityAwareVector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
 
@@ -28,7 +29,7 @@
  * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector
  * uses the offset vector to determine the sequence of cells pertaining to an individual value.
  */
-public interface RepeatedValueVector extends ValueVector {
+public interface RepeatedValueVector extends ValueVector, DensityAwareVector {
 
   final static int DEFAULT_REPEAT_PER_RECORD = 5;
 
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
index 26bb4bbfac1ee..05571bbb931c6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
@@ -355,6 +355,12 @@ public void setInitialCapacity(int numRecords) {
     super.setInitialCapacity(numRecords);
   }
 
+  @Override
+  public void setInitialCapacity(int numRecords, double density) {
+    validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords);
+    super.setInitialCapacity(numRecords, density);
+  }
+
   @Override
   public boolean allocateNewSafe() {
     /* Boolean to keep track if all the memory allocations were successful
@@ -401,6 +407,7 @@ private void reallocValidityBuffer() {
 
     long newAllocationSize = baseSize * 2L;
     newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+    assert newAllocationSize >= 1;
 
     if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Unable to expand the buffer");
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index d49a677f67922..d12586ecc59d1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -811,14 +811,21 @@ public void testSetInitialCapacity() {
       assertEquals(512, vector.getValueCapacity());
       assertEquals(8, vector.getDataVector().getValueCapacity());
 
-      boolean error = false;
-      try {
-        vector.setInitialCapacity(5, 0.1);
-      } catch (IllegalArgumentException e) {
-        error = true;
-      } finally {
-        assertTrue(error);
-      }
+      /**
+       * inner value capacity we pass to data vector is 5 * 0.1 => 0
+       * which is then rounded off to 1. So we pass value count as 1
+       * to the inner int vector.
+       * the offset buffer of the list vector is allocated for 6 values
+       * which is 24 bytes and then rounded off to 32 bytes (8 values)
+       * the validity buffer of the list vector is allocated for 5
+       * values which is 1 byte. This is why value capacity of the list
+       * vector is 7 as we take the min of validity buffer value capacity
+       * and offset buffer value capacity.
+       */
+      vector.setInitialCapacity(5, 0.1);
+      vector.allocateNew();
+      assertEquals(7, vector.getValueCapacity());
+      assertEquals(1, vector.getDataVector().getValueCapacity());
     }
   }
 }
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 992bb6264a1cf..5104962e9a3ed 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -1934,14 +1934,10 @@ public void testSetInitialCapacity() {
       assertEquals(4096, vector.getValueCapacity());
       assertEquals(64, vector.getDataBuffer().capacity());
 
-      boolean error = false;
-      try {
-        vector.setInitialCapacity(5, 0.1);
-      } catch (IllegalArgumentException e) {
-        error = true;
-      } finally {
-        assertTrue(error);
-      }
+      vector.setInitialCapacity(5, 0.01);
+      vector.allocateNew();
+      assertEquals(7, vector.getValueCapacity());
+      assertEquals(2, vector.getDataBuffer().capacity());
     }
   }
 }

From 06e9fb4db380585a223a83f6fc4e998241f51c44 Mon Sep 17 00:00:00 2001
From: Mitar <mitar.github@tnode.com>
Date: Mon, 5 Mar 2018 18:01:36 -0800
Subject: [PATCH 1564/1644] [Python] Add missing dependency to development.rst

---
 python/doc/source/development.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index e4078bdaff74b..42f407a292dd2 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -117,7 +117,8 @@ dependencies will be automatically built by Arrow's third-party toolchain.
 
    $ sudo apt-get install libjemalloc-dev libboost-dev \
                           libboost-filesystem-dev \
-                          libboost-system-dev
+                          libboost-system-dev \
+                          libboost-regex-dev
 
 On Arch Linux, you can get these dependencies via pacman.
 

From 57e4dd8bfb756bb3db7676c37fc05d16260bd82d Mon Sep 17 00:00:00 2001
From: Robert Nishihara <robertnishihara@gmail.com>
Date: Mon, 5 Mar 2018 21:11:32 -0500
Subject: [PATCH 1565/1644] ARROW-2265: [Python] Use CheckExact when
 serializing lists and numpy arrays.

cc @mitar

Author: Robert Nishihara <robertnishihara@gmail.com>

Closes #1704 from robertnishihara/subclassingnparray and squashes the following commits:

459e1d95 <Robert Nishihara> Remove PySet_CheckExact because it does not exist.
8b2c7dcd <Robert Nishihara> Use CheckExact when serializing lists, sets, numpy arrays.
---
 cpp/src/arrow/python/python_to_arrow.cc    |  4 ++--
 python/pyarrow/tests/test_serialization.py | 27 ++++++++++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 6d4f64675eb9d..d781d9f0ffcbb 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -501,7 +501,7 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
       return Status::Invalid("Cannot writes bytes over 2GB");
     }
     RETURN_NOT_OK(builder->AppendString(data, static_cast<int32_t>(size)));
-  } else if (PyList_Check(elem)) {
+  } else if (PyList_CheckExact(elem)) {
     RETURN_NOT_OK(builder->AppendList(PyList_Size(elem)));
     sublists->push_back(elem);
   } else if (PyDict_CheckExact(elem)) {
@@ -515,7 +515,7 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
     subsets->push_back(elem);
   } else if (PyArray_IsScalar(elem, Generic)) {
     RETURN_NOT_OK(AppendScalar(elem, builder));
-  } else if (PyArray_Check(elem)) {
+  } else if (PyArray_CheckExact(elem)) {
     RETURN_NOT_OK(SerializeArray(context, reinterpret_cast<PyArrayObject*>(elem), builder,
                                  subdicts, blobs_out));
   } else if (elem == Py_None) {
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 72315d2dcb24c..c1740845711d8 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -410,6 +410,33 @@ def deserialize_dummy_class(serialized_obj):
     pa.serialize(DummyClass(), context=context)
 
 
+def test_numpy_subclass_serialization():
+    # Check that we can properly serialize subclasses of np.ndarray.
+    class CustomNDArray(np.ndarray):
+        def __new__(cls, input_array):
+            array = np.asarray(input_array).view(cls)
+            return array
+
+    def serializer(obj):
+        return {'numpy': obj.view(np.ndarray)}
+
+    def deserializer(data):
+        array = data['numpy'].view(CustomNDArray)
+        return array
+
+    context = pa.default_serialization_context()
+
+    context.register_type(CustomNDArray, 'CustomNDArray',
+                          custom_serializer=serializer,
+                          custom_deserializer=deserializer)
+
+    x = CustomNDArray(np.zeros(3))
+    serialized = pa.serialize(x, context=context).to_buffer()
+    new_x = pa.deserialize(serialized, context=context)
+    assert type(new_x) == CustomNDArray
+    assert np.alltrue(new_x.view(np.ndarray) == np.zeros(3))
+
+
 def test_buffer_serialization():
 
     class BufferClass(object):

From 51e117d31ef4e3bf26abf3c4af71ca723de13d5c Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 6 Mar 2018 20:20:33 +0100
Subject: [PATCH 1566/1644] ARROW-2154: [Python] Implement equality on buffers

And also on tensors.

Author: Antoine Pitrou <antoine@python.org>

Closes #1716 from pitrou/ARROW-2154-buffer-equality and squashes the following commits:

b568123 <Antoine Pitrou> ARROW-2154:  Implement equality on buffers
---
 python/pyarrow/array.pxi            | 10 ++++++++--
 python/pyarrow/io.pxi               |  6 ++++++
 python/pyarrow/tests/test_io.py     | 26 +++++++++++++++++++++-----
 python/pyarrow/tests/test_tensor.py | 24 ++++++++++++++++++++----
 4 files changed, 55 insertions(+), 11 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index d4e53ecd541bf..7899d9dbbd320 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -541,6 +541,12 @@ strides: {0.strides}""".format(self)
         self._validate()
         return self.tp.Equals(deref(other.tp))
 
+    def __eq__(self, other):
+        if isinstance(other, Tensor):
+            return self.equals(other)
+        else:
+            return NotImplemented
+
     @property
     def is_mutable(self):
         self._validate()
@@ -565,12 +571,12 @@ strides: {0.strides}""".format(self)
     def shape(self):
         # Cython knows how to convert a vector[T] to a Python list
         self._validate()
-        return self.tp.shape()
+        return tuple(self.tp.shape())
 
     @property
     def strides(self):
         self._validate()
-        return self.tp.strides()
+        return tuple(self.tp.strides())
 
 
 cdef wrap_array_output(PyObject* output):
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 5c8411be4cdbc..611c8a86dadce 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -671,6 +671,12 @@ cdef class Buffer:
             result = self.buffer.get().Equals(deref(other.buffer.get()))
         return result
 
+    def __eq__(self, other):
+        if isinstance(other, Buffer):
+            return self.equals(other)
+        else:
+            return NotImplemented
+
     def to_pybytes(self):
         self._check_nullptr()
         return cp.PyBytes_FromStringAndSize(
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 17aca43331f3d..e42914ffcfbed 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -237,6 +237,16 @@ def test_buffer_from_numpy():
 
 def test_buffer_equals():
     # Buffer.equals() returns true iff the buffers have the same contents
+    def eq(a, b):
+        assert a.equals(b)
+        assert a == b
+        assert not (a != b)
+
+    def ne(a, b):
+        assert not a.equals(b)
+        assert not (a == b)
+        assert a != b
+
     b1 = b'some data!'
     b2 = bytearray(b1)
     b3 = bytearray(b1)
@@ -246,12 +256,18 @@ def test_buffer_equals():
     buf3 = pa.frombuffer(b2)
     buf4 = pa.frombuffer(b3)
     buf5 = pa.frombuffer(np.frombuffer(b2, dtype=np.int16))
-    assert buf1.equals(buf1)
-    assert buf1.equals(buf2)
-    assert buf2.equals(buf3)
-    assert not buf2.equals(buf4)
+    eq(buf1, buf1)
+    eq(buf1, buf2)
+    eq(buf2, buf3)
+    ne(buf2, buf4)
     # Data type is indifferent
-    assert buf2.equals(buf5)
+    eq(buf2, buf5)
+
+
+def test_buffer_hashing():
+    # Buffers are unhashable
+    with pytest.raises(TypeError, match="unhashable"):
+        hash(pa.frombuffer(b'123'))
 
 
 def test_foreign_buffer():
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index 1d45dc743b532..093bc86c35a79 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -30,8 +30,8 @@ def test_tensor_attrs():
 
     assert tensor.ndim == 2
     assert tensor.size == 40
-    assert tensor.shape == list(data.shape)
-    assert tensor.strides == list(data.strides)
+    assert tensor.shape == data.shape
+    assert tensor.strides == data.strides
 
     assert tensor.is_contiguous
     assert tensor.is_mutable
@@ -121,14 +121,30 @@ def test_tensor_ipc_strided(tmpdir):
 
 
 def test_tensor_equals():
+    def eq(a, b):
+        assert a.equals(b)
+        assert a == b
+        assert not (a != b)
+
+    def ne(a, b):
+        assert not a.equals(b)
+        assert not (a == b)
+        assert a != b
+
     data = np.random.randn(10, 6, 4)[::, ::2, ::]
     tensor1 = pa.Tensor.from_numpy(data)
     tensor2 = pa.Tensor.from_numpy(np.ascontiguousarray(data))
-    assert tensor1.equals(tensor2)
+    eq(tensor1, tensor2)
     data = data.copy()
     data[9, 0, 0] = 1.0
     tensor2 = pa.Tensor.from_numpy(np.ascontiguousarray(data))
-    assert not tensor1.equals(tensor2)
+    ne(tensor1, tensor2)
+
+
+def test_tensor_hashing():
+    # Tensors are unhashable
+    with pytest.raises(TypeError, match="unhashable"):
+        hash(pa.Tensor.from_numpy(np.arange(10)))
 
 
 def test_tensor_size():

From cde18a6ba525d090d11ac6f3563370dce60d03a1 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Tue, 6 Mar 2018 14:22:07 -0500
Subject: [PATCH 1567/1644] ARROW-2234: [JS] Read timestamp low bits as Uint32s

I ran into this with the epoch millisecond timestamp `1394015437000`. We need all 32 bits of precision for the low bits, but still need the underlying TypedArray to be an Int32Array so the high bits can be signed. Example:

```sh
> mult = Math.pow(2, 32)
4294967296
> hi = (1394015437000 / mult) | 0
324
> lo = (1394015437000 - (mult * hi) | 0)
-1848934200
> mult * hi + lo
1389720469704 # wrong
> mult * hi + (new Uint32Array([lo])[0])
1394015437000 # right
```

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1678 from trxcllnt/js-fix-timestamps and squashes the following commits:

bd34db52 <Paul Taylor> add more public methods to the closure-compiler mangler whitelist
22210f89 <Paul Taylor> add initial DateVector tests to validate low bits are read unsigned
56d5c297 <Paul Taylor> fix jest deprecation warning
ddb1b444 <Paul Taylor> read timestamp low bits as Uint32s
---
 js/package.json                   |   1 -
 js/src/Arrow.externs.js           |  18 +++++
 js/src/vector/flat.ts             |   6 +-
 js/test/unit/date-vector-tests.ts | 120 ++++++++++++++++++++++++++++++
 4 files changed, 141 insertions(+), 4 deletions(-)
 create mode 100644 js/test/unit/date-vector-tests.ts

diff --git a/js/package.json b/js/package.json
index 507babfbbb5b1..af3c97f5e3383 100644
--- a/js/package.json
+++ b/js/package.json
@@ -125,7 +125,6 @@
       "ts",
       "tsx"
     ],
-    "mapCoverage": true,
     "coverageReporters": [
       "lcov"
     ],
diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index cf4db9134b9fa..a14f959bb469b 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -510,24 +510,42 @@ var FloatVector = function() {};
 FloatVector.from = function() {};
 
 var DateVector = function() {};
+/** @type {?} */
+DateVector.prototype.asEpochMilliseconds;
 var DecimalVector = function() {};
 var TimeVector = function() {};
 var TimestampVector = function() {};
+/** @type {?} */
+TimestampVector.prototype.asEpochMilliseconds;
 var IntervalVector = function() {};
 var BinaryVector = function() {};
+/** @type {?} */
+BinaryVector.prototype.asUtf8;
 var FixedSizeBinaryVector = function() {};
 var Utf8Vector = function() {};
+/** @type {?} */
+Utf8Vector.prototype.asBinary;
 var ListVector = function() {};
 var FixedSizeListVector = function() {};
 var MapVector = function() {};
+/** @type {?} */
+MapVector.prototype.asStruct;
 var StructVector = function() {};
+/** @type {?} */
+StructVector.prototype.asMap;
 var UnionVector = function() {};
 
 var DictionaryVector = function() {};
 /** @type {?} */
+DictionaryVector.prototype.indices;
+/** @type {?} */
+DictionaryVector.prototype.dictionary;
+/** @type {?} */
 DictionaryVector.prototype.getKey;
 /** @type {?} */
 DictionaryVector.prototype.getValue;
+/** @type {?} */
+DictionaryVector.prototype.reverseLookup;
 
 var FlatView = function() {};
 /** @type {?} */
diff --git a/js/src/vector/flat.ts b/js/src/vector/flat.ts
index acc2f1af9baa6..06189c47d846e 100644
--- a/js/src/vector/flat.ts
+++ b/js/src/vector/flat.ts
@@ -326,9 +326,9 @@ export class IntervalMonthView extends PrimitiveView<Int32> {
 
 export function epochSecondsToMs(data: Int32Array, index: number) { return 1000 * data[index]; }
 export function epochDaysToMs(data: Int32Array, index: number) { return 86400000 * data[index]; }
-export function epochMillisecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1]) + data[index]; }
-export function epochMicrosecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1] / 1000) + (data[index] / 1000); }
-export function epochNanosecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1] / 1000000) + (data[index] / 1000000); }
+export function epochMillisecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1]) + (data[index] >>> 0); }
+export function epochMicrosecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1] / 1000) + ((data[index] >>> 0) / 1000); }
+export function epochNanosecondsLongToMs(data: Int32Array, index: number) { return 4294967296 * (data[index + 1] / 1000000) + ((data[index] >>> 0) / 1000000); }
 
 export function epochMillisecondsToDate(epochMs: number) { return new Date(epochMs); }
 export function epochDaysToDate(data: Int32Array, index: number) { return epochMillisecondsToDate(epochDaysToMs(data, index)); }
diff --git a/js/test/unit/date-vector-tests.ts b/js/test/unit/date-vector-tests.ts
new file mode 100644
index 0000000000000..b30d0498b034e
--- /dev/null
+++ b/js/test/unit/date-vector-tests.ts
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Arrow from '../Arrow';
+import { DateVector } from '../../src/vector';
+const { Table } = Arrow;
+
+describe(`DateVector`, () => {
+    it('returns days since the epoch as correct JS Dates', () => {
+        const table = Table.from(test_data);
+        const date32 = table.getColumnAt(0) as DateVector;
+        const expectedMillis = expectedMillis32();
+        for (const date of date32) {
+            const millis = expectedMillis.shift();
+            expect(date).toEqual(millis === null ? null : new Date(millis!));
+        }
+    });
+    it('returns millisecond longs since the epoch as correct JS Dates', () => {
+        const table = Table.from(test_data);
+        const date64 = table.getColumnAt(1) as DateVector;
+        const expectedMillis = expectedMillis64();
+        for (const date of date64) {
+            const millis = expectedMillis.shift();
+            expect(date).toEqual(millis === null ? null : new Date(millis!));
+        }
+    });
+    it('converts days since the epoch to milliseconds', () => {
+        const table = Table.from(test_data);
+        const date32 = table.getColumnAt(0) as DateVector;
+        const expectedMillis = expectedMillis32();
+        for (const timestamp of date32.asEpochMilliseconds()) {
+            expect(timestamp).toEqual(expectedMillis.shift());
+        }
+    });
+    it('converts millisecond longs since the epoch to millisecond ints', () => {
+        const table = Table.from(test_data);
+        const date64 = table.getColumnAt(1) as DateVector;
+        const expectedMillis = expectedMillis64();
+        for (const timestamp of date64.asEpochMilliseconds()) {
+            expect(timestamp).toEqual(expectedMillis.shift());
+        }
+    });
+});
+
+const expectedMillis32 = () => [
+    165247430400000, 34582809600000, 232604524800000, null,
+    199808812800000, 165646771200000, 209557238400000, null
+];
+
+const expectedMillis64 = () => [
+    27990830234011, -41278585914325, 12694624797111,
+    null, null, 10761360520213, null, 1394015437000
+];
+
+const test_data = {
+    'schema': {
+        'fields': [
+            {
+                'name': 'f0',
+                'type': {
+                    'name': 'date',
+                    'unit': 'DAY'
+                },
+                'nullable': true,
+                'children': []
+            },
+            {
+                'name': 'f1',
+                'type': {
+                    'name': 'date',
+                    'unit': 'MILLISECOND'
+                },
+                'nullable': true,
+                'children': []
+            }
+        ]
+    },
+    'batches': [
+        {
+            'count': 8,
+            'columns': [
+                {
+                    'name': 'f0',
+                    'count': 8,
+                    'VALIDITY': [1, 1, 1, 0, 1, 1, 1, 0],
+                    'DATA': [1912586, 400264, 2692182, 2163746, 2312602, 1917208, 2425431]
+                },
+                {
+                    'name': 'f1',
+                    'count': 8,
+                    'VALIDITY': [1, 1, 1, 0, 0, 1, 0, 1],
+                    'DATA': [
+                        27990830234011,
+                        -41278585914325,
+                        12694624797111,
+                        -38604948562547,
+                        -37802308043516,
+                        10761360520213,
+                        -25129181633384,
+                        1394015437000 // <-- the tricky one
+                    ]
+                }
+            ]
+        }
+    ]
+};

From a58bd723c3cd99eca78229491a1649f2777a45a7 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 6 Mar 2018 20:22:32 +0100
Subject: [PATCH 1568/1644] ARROW-2272: [Python] Clean up leftovers in
 test_plasma.py

Author: Antoine Pitrou <antoine@python.org>

Closes #1715 from pitrou/ARROW-2272-test-plasma-tmp and squashes the following commits:

6d6d084 <Antoine Pitrou> ARROW-2272:  Clean up leftovers in test_plasma.py
---
 python/pyarrow/tests/test_plasma.py | 114 +++++++++++++++++-----------
 1 file changed, 68 insertions(+), 46 deletions(-)

diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index 0df627fe6a6cb..b4e8649414412 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -19,14 +19,17 @@
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
+import contextlib
 import os
 import pytest
 import random
+import shutil
 import signal
 import subprocess
+import tempfile
 import time
 
+import numpy as np
 import pyarrow as pa
 import pandas as pd
 
@@ -102,9 +105,9 @@ def assert_get_object_equal(unit_test, client1, client2, object_id,
         assert plasma.buffers_equal(metadata, client1_metadata)
 
 
+@contextlib.contextmanager
 def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
                        use_valgrind=False, use_profiler=False,
-                       stdout_file=None, stderr_file=None,
                        use_one_memory_mapped_file=False,
                        plasma_directory=None, use_hugepages=False):
     """Start a plasma store process.
@@ -125,34 +128,53 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
     """
     if use_valgrind and use_profiler:
         raise Exception("Cannot use valgrind and profiler at the same time.")
-    plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store")
-    plasma_store_name = "/tmp/plasma_store{}".format(random_name())
-    command = [plasma_store_executable,
-               "-s", plasma_store_name,
-               "-m", str(plasma_store_memory)]
-    if use_one_memory_mapped_file:
-        command += ["-f"]
-    if plasma_directory:
-        command += ["-d", plasma_directory]
-    if use_hugepages:
-        command += ["-h"]
-    if use_valgrind:
-        pid = subprocess.Popen(["valgrind",
-                                "--track-origins=yes",
-                                "--leak-check=full",
-                                "--show-leak-kinds=all",
-                                "--leak-check-heuristics=stdstring",
-                                "--error-exitcode=1"] + command,
-                               stdout=stdout_file, stderr=stderr_file)
-        time.sleep(1.0)
-    elif use_profiler:
-        pid = subprocess.Popen(["valgrind", "--tool=callgrind"] + command,
-                               stdout=stdout_file, stderr=stderr_file)
-        time.sleep(1.0)
-    else:
-        pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
-        time.sleep(0.1)
-    return plasma_store_name, pid
+
+    tmpdir = tempfile.mkdtemp(prefix='test_plasma-')
+    try:
+        plasma_store_name = os.path.join(tmpdir, 'plasma.sock')
+        plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store")
+        command = [plasma_store_executable,
+                   "-s", plasma_store_name,
+                   "-m", str(plasma_store_memory)]
+        if use_one_memory_mapped_file:
+            command += ["-f"]
+        if plasma_directory:
+            command += ["-d", plasma_directory]
+        if use_hugepages:
+            command += ["-h"]
+        stdout_file = None
+        stderr_file = None
+        if use_valgrind:
+            command = ["valgrind",
+                       "--track-origins=yes",
+                       "--leak-check=full",
+                       "--show-leak-kinds=all",
+                       "--leak-check-heuristics=stdstring",
+                       "--error-exitcode=1"] + command
+            proc = subprocess.Popen(command, stdout=stdout_file,
+                                    stderr=stderr_file)
+            time.sleep(1.0)
+        elif use_profiler:
+            command = ["valgrind", "--tool=callgrind"] + command
+            proc = subprocess.Popen(command, stdout=stdout_file,
+                                    stderr=stderr_file)
+            time.sleep(1.0)
+        else:
+            proc = subprocess.Popen(command, stdout=stdout_file,
+                                    stderr=stderr_file)
+            time.sleep(0.1)
+        rc = proc.poll()
+        if rc is not None:
+            err = proc.stderr.read().decode()
+            raise RuntimeError("plasma_store exited unexpectedly with "
+                               "code %d. Error output follows:\n%s\n"
+                               % (rc, err))
+
+        yield plasma_store_name, proc
+    finally:
+        if proc.poll() is None:
+            proc.kill()
+        shutil.rmtree(tmpdir)
 
 
 @pytest.mark.plasma
@@ -164,25 +186,26 @@ def setup_method(self, test_method):
 
         import pyarrow.plasma as plasma
         # Start Plasma store.
-        plasma_store_name, self.p = start_plasma_store(
+        self.plasma_store_ctx = start_plasma_store(
             use_valgrind=USE_VALGRIND,
             use_one_memory_mapped_file=use_one_memory_mapped_file)
+        plasma_store_name, self.p = self.plasma_store_ctx.__enter__()
         # Connect to Plasma.
         self.plasma_client = plasma.connect(plasma_store_name, "", 64)
         # For the eviction test
         self.plasma_client2 = plasma.connect(plasma_store_name, "", 0)
 
     def teardown_method(self, test_method):
-        # Check that the Plasma store is still alive.
-        assert self.p.poll() is None
-        # Kill the plasma store process.
-        if os.getenv("PLASMA_VALGRIND") == "1":
-            self.p.send_signal(signal.SIGTERM)
-            self.p.wait()
-            if self.p.returncode != 0:
-                assert False
-        else:
-            self.p.kill()
+        try:
+            # Check that the Plasma store is still alive.
+            assert self.p.poll() is None
+            # Ensure Valgrind detected no issues
+            if USE_VALGRIND:
+                self.p.send_signal(signal.SIGTERM)
+                self.p.wait()
+                assert self.p.returncode == 0
+        finally:
+            self.plasma_store_ctx.__exit__(None, None, None)
 
     def test_connection_failure_raises_exception(self):
         import pyarrow.plasma as plasma
@@ -773,8 +796,7 @@ def test_object_id_size():
                     reason="requires hugepage support")
 def test_use_huge_pages():
     import pyarrow.plasma as plasma
-    plasma_store_name, p = start_plasma_store(
-        plasma_directory="/mnt/hugepages", use_hugepages=True)
-    plasma_client = plasma.connect(plasma_store_name, "", 64)
-    create_object(plasma_client, 100000000)
-    p.kill()
+    with start_plasma_store(plasma_directory="/mnt/hugepages",
+                            use_hugepages=True) as (plasma_store_name, p):
+        plasma_client = plasma.connect(plasma_store_name, "", 64)
+        create_object(plasma_client, 100000000)

From 5f8a7934dde092a67798f0185dc500923e3dc8da Mon Sep 17 00:00:00 2001
From: Mitar <mitar.github@tnode.com>
Date: Tue, 6 Mar 2018 11:25:58 -0800
Subject: [PATCH 1569/1644] ARROW-2279: [Python] Better error message if lib
 cannot be found

Change-Id: I82e6b51fa5129473815e19fd6d03bdaaef7a88ff
---
 python/setup.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/python/setup.py b/python/setup.py
index ddf7cc55aad3b..f3521f2777ace 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -381,6 +381,10 @@ def _move_shared_libs_unix(build_prefix, build_lib, lib_name):
     else:
         libs = glob.glob(pjoin(build_prefix, lib_filename) + '*')
 
+    if not libs:
+        raise Exception('Could not find library:' + lib_filename +
+                        ' in ' + build_prefix)
+
     # Longest suffix library should be copied, all others symlinked
     libs.sort(key=lambda s: -len(s))
     print(libs, libs[0])

From 60c80817fb3e45265814581d694fc107fe29cce2 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Tue, 6 Mar 2018 20:34:24 +0100
Subject: [PATCH 1570/1644] ARROW-2261: [GLib] Improve memory management for
 GArrowBuffer data

This change introduces GBytes constructors to GArrowBuffer and
GArrowMutableBuffer. GBytes has reference count feature. It means that
we can share the same memory safely.

We can't share the same memory safely with the current raw guint8
constructor.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1701 from kou/glib-buffer-accept-gbytes and squashes the following commits:

78de627 <Kouhei Sutou>  Improve memory management for GArrowBuffer data
---
 c_glib/arrow-glib/buffer.cpp       | 122 ++++++++++++++++++++++++++---
 c_glib/arrow-glib/buffer.h         |   2 +
 c_glib/arrow-glib/buffer.hpp       |   4 +
 c_glib/arrow-glib/input-stream.cpp | 104 ++++++++++++++++++++++--
 c_glib/arrow-glib/input-stream.hpp |   2 +
 c_glib/arrow-glib/tensor.cpp       |  12 ++-
 c_glib/arrow-glib/tensor.hpp       |   5 +-
 c_glib/test/test-buffer.rb         |  10 +++
 c_glib/test/test-mutable-buffer.rb |  10 +++
 9 files changed, 250 insertions(+), 21 deletions(-)

diff --git a/c_glib/arrow-glib/buffer.cpp b/c_glib/arrow-glib/buffer.cpp
index 4be8fed18ea01..4dd9ce33a1202 100644
--- a/c_glib/arrow-glib/buffer.cpp
+++ b/c_glib/arrow-glib/buffer.cpp
@@ -47,11 +47,13 @@ G_BEGIN_DECLS
 
 typedef struct GArrowBufferPrivate_ {
   std::shared_ptr<arrow::Buffer> buffer;
+  GBytes *data;
 } GArrowBufferPrivate;
 
 enum {
   PROP_0,
-  PROP_BUFFER
+  PROP_BUFFER,
+  PROP_DATA
 };
 
 G_DEFINE_TYPE_WITH_PRIVATE(GArrowBuffer, garrow_buffer, G_TYPE_OBJECT)
@@ -59,6 +61,19 @@ G_DEFINE_TYPE_WITH_PRIVATE(GArrowBuffer, garrow_buffer, G_TYPE_OBJECT)
 #define GARROW_BUFFER_GET_PRIVATE(obj) \
   (G_TYPE_INSTANCE_GET_PRIVATE((obj), GARROW_TYPE_BUFFER, GArrowBufferPrivate))
 
+static void
+garrow_buffer_dispose(GObject *object)
+{
+  auto priv = GARROW_BUFFER_GET_PRIVATE(object);
+
+  if (priv->data) {
+    g_bytes_unref(priv->data);
+    priv->data = nullptr;
+  }
+
+  G_OBJECT_CLASS(garrow_buffer_parent_class)->dispose(object);
+}
+
 static void
 garrow_buffer_finalize(GObject *object)
 {
@@ -71,9 +86,9 @@ garrow_buffer_finalize(GObject *object)
 
 static void
 garrow_buffer_set_property(GObject *object,
-                          guint prop_id,
-                          const GValue *value,
-                          GParamSpec *pspec)
+                           guint prop_id,
+                           const GValue *value,
+                           GParamSpec *pspec)
 {
   auto priv = GARROW_BUFFER_GET_PRIVATE(object);
 
@@ -82,6 +97,9 @@ garrow_buffer_set_property(GObject *object,
     priv->buffer =
       *static_cast<std::shared_ptr<arrow::Buffer> *>(g_value_get_pointer(value));
     break;
+  case PROP_DATA:
+    priv->data = static_cast<GBytes *>(g_value_dup_boxed(value));
+    break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
     break;
@@ -90,11 +108,16 @@ garrow_buffer_set_property(GObject *object,
 
 static void
 garrow_buffer_get_property(GObject *object,
-                          guint prop_id,
-                          GValue *value,
-                          GParamSpec *pspec)
+                           guint prop_id,
+                           GValue *value,
+                           GParamSpec *pspec)
 {
+  auto priv = GARROW_BUFFER_GET_PRIVATE(object);
+
   switch (prop_id) {
+  case PROP_DATA:
+    g_value_set_boxed(value, priv->data);
+    break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
     break;
@@ -113,6 +136,7 @@ garrow_buffer_class_init(GArrowBufferClass *klass)
 
   auto gobject_class = G_OBJECT_CLASS(klass);
 
+  gobject_class->dispose      = garrow_buffer_dispose;
   gobject_class->finalize     = garrow_buffer_finalize;
   gobject_class->set_property = garrow_buffer_set_property;
   gobject_class->get_property = garrow_buffer_get_property;
@@ -123,6 +147,14 @@ garrow_buffer_class_init(GArrowBufferClass *klass)
                               static_cast<GParamFlags>(G_PARAM_WRITABLE |
                                                        G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property(gobject_class, PROP_BUFFER, spec);
+
+  spec = g_param_spec_boxed("data",
+                            "Data",
+                            "The raw data passed as GBytes *",
+                            G_TYPE_BYTES,
+                            static_cast<GParamFlags>(G_PARAM_READWRITE |
+                                                     G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_DATA, spec);
 }
 
 /**
@@ -141,7 +173,25 @@ garrow_buffer_new(const guint8 *data, gint64 size)
 {
   auto arrow_buffer = std::make_shared<arrow::Buffer>(data, size);
   return garrow_buffer_new_raw(&arrow_buffer);
+}
 
+/**
+ * garrow_buffer_new_bytes:
+ * @data: Data for the buffer.
+ *
+ * Returns: A newly created #GArrowBuffer.
+ *
+ * Since: 0.9.0
+ */
+GArrowBuffer *
+garrow_buffer_new_bytes(GBytes *data)
+{
+  size_t data_size;
+  auto raw_data = g_bytes_get_data(data, &data_size);
+  auto arrow_buffer =
+    std::make_shared<arrow::Buffer>(static_cast<const uint8_t *>(raw_data),
+                                    data_size);
+  return garrow_buffer_new_raw_bytes(&arrow_buffer, data);
 }
 
 /**
@@ -226,6 +276,12 @@ garrow_buffer_get_capacity(GArrowBuffer *buffer)
 GBytes *
 garrow_buffer_get_data(GArrowBuffer *buffer)
 {
+  auto priv = GARROW_BUFFER_GET_PRIVATE(buffer);
+  if (priv->data) {
+    g_bytes_ref(priv->data);
+    return priv->data;
+  }
+
   auto arrow_buffer = garrow_buffer_get_raw(buffer);
   auto data = g_bytes_new_static(arrow_buffer->data(),
                                  arrow_buffer->size());
@@ -249,6 +305,13 @@ garrow_buffer_get_mutable_data(GArrowBuffer *buffer)
   if (!arrow_buffer->is_mutable()) {
     return NULL;
   }
+
+  auto priv = GARROW_BUFFER_GET_PRIVATE(buffer);
+  if (priv->data) {
+    g_bytes_ref(priv->data);
+    return priv->data;
+  }
+
   return g_bytes_new_static(arrow_buffer->mutable_data(),
                             arrow_buffer->size());
 }
@@ -284,7 +347,8 @@ garrow_buffer_get_parent(GArrowBuffer *buffer)
   auto arrow_parent_buffer = arrow_buffer->parent();
 
   if (arrow_parent_buffer) {
-    return garrow_buffer_new_raw(&arrow_parent_buffer);
+    auto priv = GARROW_BUFFER_GET_PRIVATE(buffer);
+    return garrow_buffer_new_raw_bytes(&arrow_parent_buffer, priv->data);
   } else {
     return NULL;
   }
@@ -338,7 +402,8 @@ garrow_buffer_slice(GArrowBuffer *buffer, gint64 offset, gint64 size)
   auto arrow_buffer = std::make_shared<arrow::Buffer>(arrow_parent_buffer,
                                                       offset,
                                                       size);
-  return garrow_buffer_new_raw(&arrow_buffer);
+  auto priv = GARROW_BUFFER_GET_PRIVATE(buffer);
+  return garrow_buffer_new_raw_bytes(&arrow_buffer, priv->data);
 }
 
 
@@ -374,6 +439,26 @@ garrow_mutable_buffer_new(guint8 *data, gint64 size)
   return garrow_mutable_buffer_new_raw(&arrow_buffer);
 }
 
+/**
+ * garrow_mutable_buffer_new_bytes:
+ * @data: Data for the buffer.
+ *
+ * Returns: A newly created #GArrowMutableBuffer.
+ *
+ * Since: 0.9.0
+ */
+GArrowMutableBuffer *
+garrow_mutable_buffer_new_bytes(GBytes *data)
+{
+  size_t data_size;
+  auto raw_data = g_bytes_get_data(data, &data_size);
+  auto mutable_raw_data = const_cast<gpointer>(raw_data);
+  auto arrow_buffer =
+    std::make_shared<arrow::MutableBuffer>(static_cast<uint8_t *>(mutable_raw_data),
+                                           data_size);
+  return garrow_mutable_buffer_new_raw_bytes(&arrow_buffer, data);
+}
+
 /**
  * garrow_mutable_buffer_slice:
  * @buffer: A #GArrowMutableBuffer.
@@ -397,7 +482,8 @@ garrow_mutable_buffer_slice(GArrowMutableBuffer *buffer,
     std::make_shared<arrow::MutableBuffer>(arrow_parent_buffer,
                                            offset,
                                            size);
-  return garrow_mutable_buffer_new_raw(&arrow_buffer);
+  auto priv = GARROW_BUFFER_GET_PRIVATE(buffer);
+  return garrow_mutable_buffer_new_raw_bytes(&arrow_buffer, priv->data);
 }
 
 
@@ -494,9 +580,17 @@ G_END_DECLS
 
 GArrowBuffer *
 garrow_buffer_new_raw(std::shared_ptr<arrow::Buffer> *arrow_buffer)
+{
+  return garrow_buffer_new_raw_bytes(arrow_buffer, nullptr);
+}
+
+GArrowBuffer *
+garrow_buffer_new_raw_bytes(std::shared_ptr<arrow::Buffer> *arrow_buffer,
+                            GBytes *data)
 {
   auto buffer = GARROW_BUFFER(g_object_new(GARROW_TYPE_BUFFER,
                                            "buffer", arrow_buffer,
+                                           "data", data,
                                            NULL));
   return buffer;
 }
@@ -513,9 +607,17 @@ garrow_buffer_get_raw(GArrowBuffer *buffer)
 
 GArrowMutableBuffer *
 garrow_mutable_buffer_new_raw(std::shared_ptr<arrow::MutableBuffer> *arrow_buffer)
+{
+  return garrow_mutable_buffer_new_raw_bytes(arrow_buffer, nullptr);
+}
+
+GArrowMutableBuffer *
+garrow_mutable_buffer_new_raw_bytes(std::shared_ptr<arrow::MutableBuffer> *arrow_buffer,
+                                    GBytes *data)
 {
   auto buffer = GARROW_MUTABLE_BUFFER(g_object_new(GARROW_TYPE_MUTABLE_BUFFER,
                                                    "buffer", arrow_buffer,
+                                                   "data", data,
                                                    NULL));
   return buffer;
 }
diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h
index 300bb4f4ea3ca..50debcc05bdf9 100644
--- a/c_glib/arrow-glib/buffer.h
+++ b/c_glib/arrow-glib/buffer.h
@@ -36,6 +36,7 @@ struct _GArrowBufferClass
 
 GArrowBuffer  *garrow_buffer_new          (const guint8 *data,
                                            gint64 size);
+GArrowBuffer  *garrow_buffer_new_bytes    (GBytes *data);
 gboolean       garrow_buffer_equal        (GArrowBuffer *buffer,
                                            GArrowBuffer *other_buffer);
 gboolean       garrow_buffer_equal_n_bytes(GArrowBuffer *buffer,
@@ -70,6 +71,7 @@ struct _GArrowMutableBufferClass
 
 GArrowMutableBuffer *garrow_mutable_buffer_new  (guint8 *data,
                                                  gint64 size);
+GArrowMutableBuffer *garrow_mutable_buffer_new_bytes(GBytes *data);
 GArrowMutableBuffer *garrow_mutable_buffer_slice(GArrowMutableBuffer *buffer,
                                                  gint64 offset,
                                                  gint64 size);
diff --git a/c_glib/arrow-glib/buffer.hpp b/c_glib/arrow-glib/buffer.hpp
index d1664b11b17c9..3dd37945a98cd 100644
--- a/c_glib/arrow-glib/buffer.hpp
+++ b/c_glib/arrow-glib/buffer.hpp
@@ -24,7 +24,11 @@
 #include <arrow-glib/buffer.h>
 
 GArrowBuffer *garrow_buffer_new_raw(std::shared_ptr<arrow::Buffer> *arrow_buffer);
+GArrowBuffer *garrow_buffer_new_raw_bytes(std::shared_ptr<arrow::Buffer> *arrow_buffer,
+                                          GBytes *data);
 std::shared_ptr<arrow::Buffer> garrow_buffer_get_raw(GArrowBuffer *buffer);
 
 GArrowMutableBuffer *garrow_mutable_buffer_new_raw(std::shared_ptr<arrow::MutableBuffer> *arrow_buffer);
+GArrowMutableBuffer *garrow_mutable_buffer_new_raw_bytes(std::shared_ptr<arrow::MutableBuffer> *arrow_buffer,
+                                                         GBytes *data);
 GArrowPoolBuffer *garrow_pool_buffer_new_raw(std::shared_ptr<arrow::PoolBuffer> *arrow_buffer);
diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index f602e5f7eb645..c643ad2a7ef01 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -282,16 +282,79 @@ garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream *input_stream
                                        arrow_random_access_file.get(),
                                        &arrow_tensor);
   if (garrow_error_check(error, status, "[seekable-input-stream][read-tensor]")) {
-    return garrow_tensor_new_raw(&arrow_tensor, nullptr);
+    return garrow_tensor_new_raw(&arrow_tensor);
   } else {
     return NULL;
   }
 }
 
 
-G_DEFINE_TYPE(GArrowBufferInputStream,                       \
-              garrow_buffer_input_stream,                     \
-              GARROW_TYPE_SEEKABLE_INPUT_STREAM);
+typedef struct GArrowBufferInputStreamPrivate_ {
+  GArrowBuffer *buffer;
+} GArrowBufferInputStreamPrivate;
+
+enum {
+  PROP_0_,
+  PROP_BUFFER
+};
+
+G_DEFINE_TYPE_WITH_PRIVATE(GArrowBufferInputStream,                     \
+                           garrow_buffer_input_stream,                  \
+                           GARROW_TYPE_SEEKABLE_INPUT_STREAM);
+
+#define GARROW_BUFFER_INPUT_STREAM_GET_PRIVATE(obj)                     \
+  (G_TYPE_INSTANCE_GET_PRIVATE((obj),                                   \
+                               GARROW_TYPE_BUFFER_INPUT_STREAM,         \
+                               GArrowBufferInputStreamPrivate))
+
+static void
+garrow_buffer_input_stream_dispose(GObject *object)
+{
+  auto priv = GARROW_BUFFER_INPUT_STREAM_GET_PRIVATE(object);
+
+  if (priv->buffer) {
+    g_object_unref(priv->buffer);
+    priv->buffer = nullptr;
+  }
+
+  G_OBJECT_CLASS(garrow_buffer_input_stream_parent_class)->dispose(object);
+}
+
+static void
+garrow_buffer_input_stream_set_property(GObject *object,
+                                        guint prop_id,
+                                        const GValue *value,
+                                        GParamSpec *pspec)
+{
+  auto priv = GARROW_BUFFER_INPUT_STREAM_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_BUFFER:
+    priv->buffer = GARROW_BUFFER(g_value_dup_object(value));
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
+
+static void
+garrow_buffer_input_stream_get_property(GObject *object,
+                                        guint prop_id,
+                                        GValue *value,
+                                        GParamSpec *pspec)
+{
+  auto priv = GARROW_BUFFER_INPUT_STREAM_GET_PRIVATE(object);
+
+  switch (prop_id) {
+  case PROP_BUFFER:
+    g_value_set_object(value, priv->buffer);
+    break;
+  default:
+    G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+    break;
+  }
+}
 
 static void
 garrow_buffer_input_stream_init(GArrowBufferInputStream *object)
@@ -301,6 +364,21 @@ garrow_buffer_input_stream_init(GArrowBufferInputStream *object)
 static void
 garrow_buffer_input_stream_class_init(GArrowBufferInputStreamClass *klass)
 {
+  GParamSpec *spec;
+
+  auto gobject_class = G_OBJECT_CLASS(klass);
+
+  gobject_class->dispose      = garrow_buffer_input_stream_dispose;
+  gobject_class->set_property = garrow_buffer_input_stream_set_property;
+  gobject_class->get_property = garrow_buffer_input_stream_get_property;
+
+  spec = g_param_spec_object("buffer",
+                             "Buffer",
+                             "The data",
+                             GARROW_TYPE_BUFFER,
+                             static_cast<GParamFlags>(G_PARAM_READWRITE |
+                                                      G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property(gobject_class, PROP_BUFFER, spec);
 }
 
 /**
@@ -315,18 +393,24 @@ garrow_buffer_input_stream_new(GArrowBuffer *buffer)
   auto arrow_buffer = garrow_buffer_get_raw(buffer);
   auto arrow_buffer_reader =
     std::make_shared<arrow::io::BufferReader>(arrow_buffer);
-  return garrow_buffer_input_stream_new_raw(&arrow_buffer_reader);
+  return garrow_buffer_input_stream_new_raw_buffer(&arrow_buffer_reader, buffer);
 }
 
 /**
  * garrow_buffer_input_stream_get_buffer:
  * @input_stream: A #GArrowBufferInputStream.
  *
- * Returns: (transfer full): The data of the array as #GArrowBuffer.
+ * Returns: (transfer full): The data of the stream as #GArrowBuffer.
  */
 GArrowBuffer *
 garrow_buffer_input_stream_get_buffer(GArrowBufferInputStream *input_stream)
 {
+  auto priv = GARROW_BUFFER_INPUT_STREAM_GET_PRIVATE(input_stream);
+  if (priv->buffer) {
+    g_object_ref(priv->buffer);
+    return priv->buffer;
+  }
+
   auto arrow_buffer_reader = garrow_buffer_input_stream_get_raw(input_stream);
   auto arrow_buffer = arrow_buffer_reader->buffer();
   return garrow_buffer_new_raw(&arrow_buffer);
@@ -626,10 +710,18 @@ garrow_seekable_input_stream_get_raw(GArrowSeekableInputStream *seekable_input_s
 
 GArrowBufferInputStream *
 garrow_buffer_input_stream_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader)
+{
+  return garrow_buffer_input_stream_new_raw_buffer(arrow_buffer_reader, nullptr);
+}
+
+GArrowBufferInputStream *
+garrow_buffer_input_stream_new_raw_buffer(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader,
+                                          GArrowBuffer *buffer)
 {
   auto buffer_input_stream =
     GARROW_BUFFER_INPUT_STREAM(g_object_new(GARROW_TYPE_BUFFER_INPUT_STREAM,
                                             "input-stream", arrow_buffer_reader,
+                                            "buffer", buffer,
                                             NULL));
   return buffer_input_stream;
 }
diff --git a/c_glib/arrow-glib/input-stream.hpp b/c_glib/arrow-glib/input-stream.hpp
index 17d2bd92422d6..1d835e80b5bfa 100644
--- a/c_glib/arrow-glib/input-stream.hpp
+++ b/c_glib/arrow-glib/input-stream.hpp
@@ -31,6 +31,8 @@ std::shared_ptr<arrow::io::InputStream> garrow_input_stream_get_raw(GArrowInputS
 std::shared_ptr<arrow::io::RandomAccessFile> garrow_seekable_input_stream_get_raw(GArrowSeekableInputStream *input_stream);
 
 GArrowBufferInputStream *garrow_buffer_input_stream_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader);
+GArrowBufferInputStream *garrow_buffer_input_stream_new_raw_buffer(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader,
+                                                                   GArrowBuffer *buffer);
 std::shared_ptr<arrow::io::BufferReader> garrow_buffer_input_stream_get_raw(GArrowBufferInputStream *input_stream);
 
 GArrowMemoryMappedInputStream *garrow_memory_mapped_input_stream_new_raw(std::shared_ptr<arrow::io::MemoryMappedFile> *arrow_memory_mapped_file);
diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp
index 359831f67220d..e7d3c38f7c671 100644
--- a/c_glib/arrow-glib/tensor.cpp
+++ b/c_glib/arrow-glib/tensor.cpp
@@ -198,7 +198,7 @@ garrow_tensor_new(GArrowDataType *data_type,
                                     arrow_shape,
                                     arrow_strides,
                                     arrow_dimension_names);
-  auto tensor = garrow_tensor_new_raw(&arrow_tensor, data);
+  auto tensor = garrow_tensor_new_raw_buffer(&arrow_tensor, data);
   return tensor;
 }
 
@@ -436,8 +436,14 @@ garrow_tensor_is_column_major(GArrowTensor *tensor)
 G_END_DECLS
 
 GArrowTensor *
-garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor,
-                      GArrowBuffer *buffer)
+garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor)
+{
+  return garrow_tensor_new_raw_buffer(arrow_tensor, nullptr);
+}
+
+GArrowTensor *
+garrow_tensor_new_raw_buffer(std::shared_ptr<arrow::Tensor> *arrow_tensor,
+                             GArrowBuffer *buffer)
 {
   auto tensor = GARROW_TENSOR(g_object_new(GARROW_TYPE_TENSOR,
                                            "tensor", arrow_tensor,
diff --git a/c_glib/arrow-glib/tensor.hpp b/c_glib/arrow-glib/tensor.hpp
index 8e54e492aa141..c90dc6d4d970f 100644
--- a/c_glib/arrow-glib/tensor.hpp
+++ b/c_glib/arrow-glib/tensor.hpp
@@ -23,6 +23,7 @@
 
 #include <arrow-glib/tensor.h>
 
-GArrowTensor *garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor,
-                                    GArrowBuffer *buffer);
+GArrowTensor *garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor);
+GArrowTensor *garrow_tensor_new_raw_buffer(std::shared_ptr<arrow::Tensor> *arrow_tensor,
+                                           GArrowBuffer *buffer);
 std::shared_ptr<arrow::Tensor> garrow_tensor_get_raw(GArrowTensor *tensor);
diff --git a/c_glib/test/test-buffer.rb b/c_glib/test/test-buffer.rb
index 3b02df7ba7e84..3b9a031ab8086 100644
--- a/c_glib/test/test-buffer.rb
+++ b/c_glib/test/test-buffer.rb
@@ -23,6 +23,16 @@ def setup
     @buffer = Arrow::Buffer.new(@data)
   end
 
+  def test_new_bytes
+    bytes_data = GLib::Bytes.new(@data)
+    buffer = Arrow::Buffer.new(bytes_data)
+    if GLib.check_binding_version?(3, 2, 2)
+      assert_equal(bytes_data.pointer, buffer.data.pointer)
+    else
+      assert_equal(@data, buffer.data.to_s)
+    end
+  end
+
   def test_equal
     assert_equal(@buffer,
                  Arrow::Buffer.new(@data.dup))
diff --git a/c_glib/test/test-mutable-buffer.rb b/c_glib/test/test-mutable-buffer.rb
index df62dcf1e8d15..f370e60265c25 100644
--- a/c_glib/test/test-mutable-buffer.rb
+++ b/c_glib/test/test-mutable-buffer.rb
@@ -21,6 +21,16 @@ def setup
     @buffer = Arrow::MutableBuffer.new(@data)
   end
 
+  def test_new_bytes
+    bytes_data = GLib::Bytes.new(@data)
+    buffer = Arrow::MutableBuffer.new(bytes_data)
+    if GLib.check_binding_version?(3, 2, 2)
+      assert_equal(bytes_data.pointer, buffer.mutable_data.pointer)
+    else
+      assert_equal(@data, buffer.mutable_data.to_s)
+    end
+  end
+
   def test_mutable?
     assert do
       @buffer.mutable?

From 9effbedfa6949b91633197716fd44424903179b9 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Wed, 7 Mar 2018 10:42:03 -0500
Subject: [PATCH 1571/1644] ARROW-2283: [C++] Support Arrow C++ installed in
 /usr detection by pkg-config

pkg-config doesn't show -I... and -L... flags when ... is the system
default path (i.e. /usr). If Arrow C++ is installed in /usr, we
couldn't detect include path and library path for Arrow C++.

It's caused when we install Arrow C++ with .rpm and .deb packages.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1721 from kou/cpp-find-arrow and squashes the following commits:

4911c8fc <Kouhei Sutou>  Add Debian based system support
0a2d9c99 <Kouhei Sutou>  Support Arrow C++ installed in /usr detection by pkg-config
---
 cpp/cmake_modules/FindArrow.cmake | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/cpp/cmake_modules/FindArrow.cmake b/cpp/cmake_modules/FindArrow.cmake
index 70defd6525c5b..0a1789a8f676d 100644
--- a/cpp/cmake_modules/FindArrow.cmake
+++ b/cpp/cmake_modules/FindArrow.cmake
@@ -25,6 +25,7 @@
 #  ARROW_FOUND, whether arrow has been found
 
 include(FindPkgConfig)
+include(GNUInstallDirs)
 
 if ("$ENV{ARROW_HOME}" STREQUAL "")
   pkg_check_modules(ARROW arrow)
@@ -33,6 +34,16 @@ if ("$ENV{ARROW_HOME}" STREQUAL "")
     message(STATUS "Arrow ABI version: ${ARROW_ABI_VERSION}")
     pkg_get_variable(ARROW_SO_VERSION arrow so_version)
     message(STATUS "Arrow SO version: ${ARROW_SO_VERSION}")
+    if ("${ARROW_INCLUDE_DIRS}" STREQUAL "")
+      set(ARROW_INCLUDE_DIRS "/usr/${CMAKE_INSTALL_INCLUDEDIR}")
+    endif()
+    if ("${ARROW_LIBRARY_DIRS}" STREQUAL "")
+      set(ARROW_LIBRARY_DIRS "/usr/${CMAKE_INSTALL_LIBDIR}")
+      if (EXISTS "/etc/debian_version" AND CMAKE_LIBRARY_ARCHITECTURE)
+        set(ARROW_LIBRARY_DIRS
+          "${ARROW_LIBRARY_DIRS}/${CMAKE_LIBRARY_ARCHITECTURE}")
+      endif()
+    endif()
     set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
     set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
     set(ARROW_SEARCH_LIB_PATH ${ARROW_LIBRARY_DIRS})

From fb2316c6bcb82cd5b5aca21f0672b2c02ae89466 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 7 Mar 2018 10:49:05 -0500
Subject: [PATCH 1572/1644] ARROW-2238: [C++] Detect and use clcache in cmake
 configuration

Author: Antoine Pitrou <antoine@python.org>

Closes #1684 from pitrou/ARROW-2238-cmake-clcache and squashes the following commits:

8539a0ec <Antoine Pitrou> ARROW-2238:  Detect and use clcache in cmake configuration
---
 .gitignore            |  5 +++++
 ci/appveyor-setup.bat |  2 --
 cpp/CMakeLists.txt    | 13 +++++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index f5b235a8995e0..c902ba39cb1f0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,6 +26,11 @@
 .build_cache_dir
 MANIFEST
 
+# Generated Visual Studio files
+*.vcxproj
+*.vcxproj.*
+*.sln
+
 cpp/.idea/
 python/.eggs/
 .vscode
diff --git a/ci/appveyor-setup.bat b/ci/appveyor-setup.bat
index 34f8f1d33ecdb..d781ceb2e1e53 100644
--- a/ci/appveyor-setup.bat
+++ b/ci/appveyor-setup.bat
@@ -48,8 +48,6 @@ if "%USE_CLCACHE%" == "true" (
     @rem Use clcache for faster builds
     pip install -q git+https://github.com/frerich/clcache.git
     clcache -s
-    set CC=clcache
-    set CXX=clcache
     set CLCACHE_SERVER=1
     set CLCACHE_HARDLINK=1
     powershell.exe -Command "Start-Process clcache-server"
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 47692a831fa35..a61bcad838789 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -204,6 +204,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     OFF)
 
   if (MSVC)
+    option(ARROW_USE_CLCACHE
+      "Use clcache if available"
+      ON)
+
     set(BROTLI_MSVC_STATIC_LIB_SUFFIX "-static" CACHE STRING
       "Brotli static lib suffix used on Windows with MSVC (default -static)")
     set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
@@ -259,6 +263,15 @@ else()
   set(ARROW_BOOST_HEADER_ONLY 1)
 endif()
 
+if (MSVC AND ARROW_USE_CLCACHE AND
+     (("${CMAKE_GENERATOR}" STREQUAL "NMake Makefiles") OR
+      ("${CMAKE_GENERATOR}" STREQUAL "Ninja")))
+  find_program(CLCACHE_FOUND clcache)
+  if(CLCACHE_FOUND)
+    set(CMAKE_CXX_COMPILER ${CLCACHE_FOUND})
+  endif(CLCACHE_FOUND)
+endif()
+
 ############################################################
 # Compiler flags
 ############################################################

From c372dfbdae95fe445c036188aca174c6ff18ecde Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Wed, 7 Mar 2018 14:29:38 -0500
Subject: [PATCH 1573/1644] ARROW-2280: [Python] Return the offset for the
 buffers in pyarrow.Array

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1719 from xhochy/ARROW-2280 and squashes the following commits:

82b50a76 <Uwe L. Korn> ARROW-2280:  Return the offset for the buffers in pyarrow.Array
---
 python/pyarrow/array.pxi             | 13 +++++++++++++
 python/pyarrow/includes/libarrow.pxd |  1 +
 python/pyarrow/tests/test_array.py   |  9 +++++++++
 3 files changed, 23 insertions(+)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 7899d9dbbd320..e785c0ec5c3de 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -483,10 +483,23 @@ cdef class Array:
         with nogil:
             check_status(ValidateArray(deref(self.ap)))
 
+    property offset:
+
+        def __get__(self):
+            """
+            A relative position into another array's data, to enable zero-copy
+            slicing. This value defaults to zero but must be applied on all
+            operations with the physical storage buffers.
+            """
+            return self.sp_array.get().offset()
+
     def buffers(self):
         """
         Return a list of Buffer objects pointing to this array's physical
         storage.
+
+        To correctly interpret these buffers, you need to also apply the offset
+        multiplied with the size of the stored data type.
         """
         res = []
         _append_array_buffers(self.sp_array.get().data().get(), res)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index d95f01661cf0d..456fcca3602db 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -103,6 +103,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         int64_t length()
         int64_t null_count()
+        int64_t offset()
         Type type_id()
 
         int num_fields()
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index c1131a00234d1..f034d78b39396 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -600,6 +600,15 @@ def test_buffers_primitive():
     assert 1 <= len(null_bitmap) <= 64  # XXX this is varying
     assert bytearray(null_bitmap)[0] == 0b00001011
 
+    # Slicing does not affect the buffers but the offset
+    a_sliced = a[1:]
+    buffers = a_sliced.buffers()
+    a_sliced.offset == 1
+    assert len(buffers) == 2
+    null_bitmap = buffers[0].to_pybytes()
+    assert 1 <= len(null_bitmap) <= 64  # XXX this is varying
+    assert bytearray(null_bitmap)[0] == 0b00001011
+
     assert struct.unpack('hhxxh', buffers[1].to_pybytes()) == (1, 2, 4)
 
     a = pa.array(np.int8([4, 5, 6]))

From 5e945a3661fcb44affb7d6b5c26219ea417167fa Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 8 Mar 2018 11:52:14 -0500
Subject: [PATCH 1574/1644] ARROW-2239: [C++] Update Windows build docs

Recommend Ninja and clcache.

Author: Antoine Pitrou <antoine@python.org>

Closes #1722 from pitrou/ARROW-2239-windows-build-docs and squashes the following commits:

a0e0288e <Antoine Pitrou> ARROW-2239:  Update Windows build docs
---
 cpp/apidoc/Windows.md | 63 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 48 insertions(+), 15 deletions(-)

diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index dae5040c24d38..965369521dd62 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -44,9 +44,8 @@ Now, you can bootstrap a build environment
 conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy zlib brotli gflags lz4-c zstd -c conda-forge
 ```
 
-***Note:***
-> *Make sure to get the `conda-forge` build of `gflags` as the
-  naming of the library differs from that in the `defaults` channel*
+> **Note:** Make sure to get the `conda-forge` build of `gflags` as the
+> naming of the library differs from that in the `defaults` channel.
 
 Activate just created conda environment with pre-installed packages from
 previous step:
@@ -116,52 +115,85 @@ zstd%ZSTD_SUFFIX%.lib.
 ### Visual Studio
 
 Microsoft provides the free Visual Studio Community edition. When doing
-development, you must launch the developer command prompt using
+development, you must launch the developer command prompt using:
 
 #### Visual Studio 2015
 
-```"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64```
+```
+"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
+```
 
 #### Visual Studio 2017
 
-```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64```
+```
+"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64
+```
 
 It's easiest to configure a console emulator like [cmder][3] to automatically
 launch this when starting a new development console.
 
+## Building with Ninja and clcache
+
+We recommend the [Ninja](https://ninja-build.org/) build system for better
+build parallelization, and the optional
+[clcache](https://github.com/frerich/clcache/) compiler cache which keeps
+track of past compilations to avoid running them over and over again
+(in a way similar to the Unix-specific "ccache").
+
+Activate your conda build environment to first install those utilities:
+
+```shell
+activate arrow-dev
+
+conda install -c conda-forge ninja
+pip install git+https://github.com/frerich/clcache.git
+```
+
+Change working directory in cmd.exe to the root directory of Arrow and
+do an out of source build by generating Ninja files:
+
+```shell
+cd cpp
+mkdir build
+cd build
+cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Release ..
+cmake --build . --config Release
+```
+
 ## Building with NMake
 
 Activate your conda build environment:
 
-```
+```shell
 activate arrow-dev
 ```
 
 Change working directory in cmd.exe to the root directory of Arrow and
 do an out of source build using `nmake`:
 
-```
+```shell
 cd cpp
 mkdir build
 cd build
 cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ..
+cmake --build . --config Release
 nmake
 ```
 
 When using conda, only release builds are currently supported.
 
-## Build using Visual Studio (MSVC) Solution Files
+## Building using Visual Studio (MSVC) Solution Files
 
 Activate your conda build environment:
 
-```
+```shell
 activate arrow-dev
 ```
 
 Change working directory in cmd.exe to the root directory of Arrow and
 do an out of source build by generating a MSVC solution:
 
-```
+```shell
 cd cpp
 mkdir build
 cd build
@@ -171,10 +203,11 @@ cmake --build . --config Release
 
 ## Debug build
 
-To build Debug version of Arrow you should have pre-insalled Debug version of
-boost libs.
+To build Debug version of Arrow you should have pre-installed a Debug version
+of boost libs.
 
-It's recommended to configure cmake build with following variables for Debug build:
+It's recommended to configure cmake build with the following variables for
+Debug build:
 
 `-DARROW_BOOST_USE_SHARED=OFF` - enables static linking with boost debug libs and
 simplifies run-time loading of 3rd parties. (Recommended)
@@ -185,7 +218,7 @@ simplifies run-time loading of 3rd parties. (Recommended)
 
 Command line to build Arrow in Debug might look as following:
 
-```
+```shell
 cd cpp
 mkdir build
 cd build

From f3f91b0cc43d2d5da1e17095aa04202e7776cf27 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 8 Mar 2018 23:04:18 -0500
Subject: [PATCH 1575/1644] ARROW-2263: [Python] Prepend local pyarrow/ path to
 PYTHONPATH in test_cython.py

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1730 from wesm/ARROW-2263 and squashes the following commits:

6b3f827c <Wes McKinney> Prepend local pyarrow/ path to PYTHONPATH in test_cython.py
---
 python/pyarrow/tests/test_cython.py        |  8 +++++++-
 python/pyarrow/tests/test_serialization.py | 16 +++-------------
 python/pyarrow/tests/util.py               | 17 ++++++++++++++++-
 3 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py
index df5e70ee7e3a5..57dbeb554ca33 100644
--- a/python/pyarrow/tests/test_cython.py
+++ b/python/pyarrow/tests/test_cython.py
@@ -24,6 +24,7 @@
 
 import pyarrow as pa
 
+import pyarrow.tests.util as test_util
 
 here = os.path.dirname(os.path.abspath(__file__))
 
@@ -85,9 +86,14 @@ def test_cython_api(tmpdir):
         with open('setup.py', 'w') as f:
             f.write(setup_code)
 
+        # ARROW-2263: Make environment with this pyarrow/ package first on the
+        # PYTHONPATH, for local dev environments
+        subprocess_env = test_util.get_modified_env_with_pythonpath()
+
         # Compile extension module
         subprocess.check_call([sys.executable, 'setup.py',
-                               'build_ext', '--inplace'])
+                               'build_ext', '--inplace'],
+                              env=subprocess_env)
 
         # Check basic functionality
         orig_path = sys.path[:]
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index c1740845711d8..64aab06712446 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -28,6 +28,8 @@
 import pyarrow as pa
 import numpy as np
 
+import pyarrow.tests.util as test_util
+
 try:
     import torch
 except ImportError:
@@ -624,18 +626,6 @@ def deserialize_regex(serialized, q):
     p.join()
 
 
-def _get_modified_env_with_pythonpath():
-    # Prepend pyarrow root directory to PYTHONPATH
-    env = os.environ.copy()
-    existing_pythonpath = env.get('PYTHONPATH', '')
-
-    module_path = os.path.abspath(
-        os.path.dirname(os.path.dirname(pa.__file__)))
-
-    env['PYTHONPATH'] = os.pathsep.join((module_path, existing_pythonpath))
-    return env
-
-
 def test_deserialize_buffer_in_different_process():
     import tempfile
     import subprocess
@@ -645,7 +635,7 @@ def test_deserialize_buffer_in_different_process():
     f.write(b.to_pybytes())
     f.close()
 
-    subprocess_env = _get_modified_env_with_pythonpath()
+    subprocess_env = test_util.get_modified_env_with_pythonpath()
 
     dir_path = os.path.dirname(os.path.realpath(__file__))
     python_file = os.path.join(dir_path, 'deserialize_buffer.py')
diff --git a/python/pyarrow/tests/util.py b/python/pyarrow/tests/util.py
index a3ba9000c2f62..8c8d23b0c3d48 100644
--- a/python/pyarrow/tests/util.py
+++ b/python/pyarrow/tests/util.py
@@ -19,9 +19,12 @@
 Utility functions for testing
 """
 
+import contextlib
 import decimal
+import os
 import random
-import contextlib
+
+import pyarrow as pa
 
 
 def randsign():
@@ -91,3 +94,15 @@ def randdecimal(precision, scale):
     return decimal.Decimal(
         '{}.{}'.format(whole, str(fractional).rjust(scale, '0'))
     )
+
+
+def get_modified_env_with_pythonpath():
+    # Prepend pyarrow root directory to PYTHONPATH
+    env = os.environ.copy()
+    existing_pythonpath = env.get('PYTHONPATH', '')
+
+    module_path = os.path.abspath(
+        os.path.dirname(os.path.dirname(pa.__file__)))
+
+    env['PYTHONPATH'] = os.pathsep.join((module_path, existing_pythonpath))
+    return env

From 34b18f711e2f566722316a62b49e7050adbd75ac Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 8 Mar 2018 23:11:46 -0500
Subject: [PATCH 1576/1644] ARROW-1940: [Python] Extra metadata gets added
 after multiple conversions between pd.DataFrame and pa.Table

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1728 from cpcloud/ARROW-1940 and squashes the following commits:

2e5b7afb <Phillip Cloud> ARROW-1940:  Extra metadata gets added after multiple conversions between pd.DataFrame and pa.Table
---
 cpp/src/arrow/python/helpers.cc             |  6 +-
 python/pyarrow/pandas_compat.py             | 99 ++++++++++++++++++---
 python/pyarrow/tests/test_convert_pandas.py | 15 +++-
 3 files changed, 103 insertions(+), 17 deletions(-)

diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 429068dd1aca7..13dcc46614534 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -116,7 +116,8 @@ static Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* p
   DCHECK_NE(scale, NULLPTR);
 
   // TODO(phillipc): Make sure we perform PyDecimal_Check(python_decimal) as a DCHECK
-  OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", ""));
+  OwnedRef as_tuple(PyObject_CallMethod(python_decimal, const_cast<char*>("as_tuple"),
+                                        const_cast<char*>("")));
   RETURN_IF_PYERROR();
   DCHECK(PyTuple_Check(as_tuple.obj()));
 
@@ -241,7 +242,8 @@ bool PyDecimal_Check(PyObject* obj) {
 
 bool PyDecimal_ISNAN(PyObject* obj) {
   DCHECK(PyDecimal_Check(obj)) << "obj is not an instance of decimal.Decimal";
-  OwnedRef is_nan(PyObject_CallMethod(obj, "is_nan", ""));
+  OwnedRef is_nan(
+      PyObject_CallMethod(obj, const_cast<char*>("is_nan"), const_cast<char*>("")));
   return PyObject_IsTrue(is_nan.obj()) == 1;
 }
 
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 0bc47fc0d6090..97ea51d7e9f81 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -18,6 +18,7 @@
 import ast
 import collections
 import json
+import operator
 import re
 
 import pandas.core.internals as _int
@@ -99,8 +100,8 @@ def get_logical_type(arrow_type):
     np.float32: 'float32',
     np.float64: 'float64',
     'datetime64[D]': 'date',
-    np.str_: 'unicode',
-    np.bytes_: 'bytes',
+    np.unicode_: 'string' if not PY2 else 'unicode',
+    np.bytes_: 'bytes' if not PY2 else 'string',
 }
 
 
@@ -615,6 +616,22 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1,
 
 
 def _backwards_compatible_index_name(raw_name, logical_name):
+    """Compute the name of an index column that is compatible with older
+    versions of :mod:`pyarrow`.
+
+    Parameters
+    ----------
+    raw_name : str
+    logical_name : str
+
+    Returns
+    -------
+    result : str
+
+    Notes
+    -----
+    * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager`
+    """
     # Part of table_to_blockmanager
     pattern = r'^__index_level_\d+__$'
     if raw_name == logical_name and re.match(pattern, raw_name) is not None:
@@ -623,8 +640,57 @@ def _backwards_compatible_index_name(raw_name, logical_name):
         return logical_name
 
 
+_pandas_logical_type_map = {
+    'date': 'datetime64[D]',
+    'unicode': np.unicode_,
+    'bytes': np.bytes_,
+    'string': np.str_,
+    'empty': np.object_,
+    'mixed': np.object_,
+}
+
+
+def _pandas_type_to_numpy_type(pandas_type):
+    """Get the numpy dtype that corresponds to a pandas type.
+
+    Parameters
+    ----------
+    pandas_type : str
+        The result of a call to pandas.lib.infer_dtype.
+
+    Returns
+    -------
+    dtype : np.dtype
+        The dtype that corresponds to `pandas_type`.
+    """
+    try:
+        return _pandas_logical_type_map[pandas_type]
+    except KeyError:
+        return np.dtype(pandas_type)
+
+
 def _reconstruct_columns_from_metadata(columns, column_indexes):
-    # Part of table_to_blockmanager
+    """Construct a pandas MultiIndex from `columns` and column index metadata
+    in `column_indexes`.
+
+    Parameters
+    ----------
+    columns : List[pd.Index]
+        The columns coming from a pyarrow.Table
+    column_indexes : List[Dict[str, str]]
+        The column index metadata deserialized from the JSON schema metadata
+        in a :class:`~pyarrow.Table`.
+
+    Returns
+    -------
+    result : MultiIndex
+        The index reconstructed using `column_indexes` metadata with levels of
+        the correct type.
+
+    Notes
+    -----
+    * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager`
+    """
 
     # Get levels and labels, and provide sane defaults if the index has a
     # single level to avoid if/else spaghetti.
@@ -635,21 +701,28 @@ def _reconstruct_columns_from_metadata(columns, column_indexes):
 
     # Convert each level to the dtype provided in the metadata
     levels_dtypes = [
-        (level, col_index.get('numpy_type', level.dtype))
+        (level, col_index.get('pandas_type', str(level.dtype)))
         for level, col_index in zip_longest(
             levels, column_indexes, fillvalue={}
         )
     ]
-    new_levels = [
-        _level if _level.dtype == _dtype else _level.astype(_dtype)
-        for _level, _dtype in levels_dtypes
-    ]
 
-    return pd.MultiIndex(
-        levels=new_levels,
-        labels=labels,
-        names=columns.names
-    )
+    new_levels = []
+    encoder = operator.methodcaller('encode', 'UTF-8')
+    for level, pandas_dtype in levels_dtypes:
+        dtype = _pandas_type_to_numpy_type(pandas_dtype)
+
+        # Since our metadata is UTF-8 encoded, Python turns things that were
+        # bytes into unicode strings when json.loads-ing them. We need to
+        # convert them back to bytes to preserve metadata.
+        if dtype == np.bytes_:
+            level = level.map(encoder)
+        elif level.dtype != dtype:
+            level = level.astype(dtype)
+
+        new_levels.append(level)
+
+    return pd.MultiIndex(levels=new_levels, labels=labels, names=columns.names)
 
 
 def _table_to_blocks(options, block_table, nthreads, memory_pool, categories):
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 5abc026bf9b85..333199ab28e54 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -305,7 +305,8 @@ def test_mixed_unicode_column_names(self):
 
     def test_binary_column_name(self):
         column_data = [u'い']
-        data = {u'あ'.encode('utf8'): column_data}
+        key = u'あ'.encode('utf8')
+        data = {key: column_data}
         df = pd.DataFrame(data)
 
         # we can't use _check_pandas_roundtrip here because our metdata
@@ -314,7 +315,7 @@ def test_binary_column_name(self):
         df2 = t.to_pandas()
         assert df.values[0] == df2.values[0]
         assert df.index.values[0] == df2.index.values[0]
-        assert df.columns[0] == df2.columns[0].encode('utf8')
+        assert df.columns[0] == key
 
     def test_multiindex_duplicate_values(self):
         num_rows = 3
@@ -1728,6 +1729,16 @@ def _fully_loaded_dataframe_example():
     return pd.DataFrame(data, index=index)
 
 
+@pytest.mark.parametrize('columns', ([b'foo'], ['foo']))
+def test_roundtrip_with_bytes_unicode(columns):
+    df = pd.DataFrame(columns=columns)
+    table1 = pa.Table.from_pandas(df)
+    table2 = pa.Table.from_pandas(table1.to_pandas())
+    assert table1.equals(table2)
+    assert table1.schema.equals(table2.schema)
+    assert table1.schema.metadata == table2.schema.metadata
+
+
 def _check_serialize_components_roundtrip(df):
     ctx = pa.default_serialization_context()
 

From 04f4e6bf88f3bb9b4b21a18d891ed0c5c497e5c8 Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 8 Mar 2018 23:21:57 -0500
Subject: [PATCH 1577/1644] ARROW-2289: [GLib] Add Numeric, Integer,
 FloatingPoint data types

They are useful to detect numeric data types.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1726 from kou/glib-numeric-data-type and squashes the following commits:

89a5a8ab <Kouhei Sutou>  Add Numeric, Integer, FloatingPoint data types
---
 c_glib/arrow-glib/basic-data-type.cpp |  65 +++-
 c_glib/arrow-glib/basic-data-type.h   | 463 ++++++--------------------
 2 files changed, 155 insertions(+), 373 deletions(-)

diff --git a/c_glib/arrow-glib/basic-data-type.cpp b/c_glib/arrow-glib/basic-data-type.cpp
index a5f7aed1b8f76..82abfa35c8a42 100644
--- a/c_glib/arrow-glib/basic-data-type.cpp
+++ b/c_glib/arrow-glib/basic-data-type.cpp
@@ -315,9 +315,39 @@ garrow_boolean_data_type_new(void)
 }
 
 
+G_DEFINE_ABSTRACT_TYPE(GArrowNumericDataType,                    \
+                       garrow_numeric_data_type,                 \
+                       GARROW_TYPE_FIXED_WIDTH_DATA_TYPE)
+
+static void
+garrow_numeric_data_type_init(GArrowNumericDataType *object)
+{
+}
+
+static void
+garrow_numeric_data_type_class_init(GArrowNumericDataTypeClass *klass)
+{
+}
+
+
+G_DEFINE_ABSTRACT_TYPE(GArrowIntegerDataType,                    \
+                       garrow_integer_data_type,                 \
+                       GARROW_TYPE_NUMERIC_DATA_TYPE)
+
+static void
+garrow_integer_data_type_init(GArrowIntegerDataType *object)
+{
+}
+
+static void
+garrow_integer_data_type_class_init(GArrowIntegerDataTypeClass *klass)
+{
+}
+
+
 G_DEFINE_TYPE(GArrowInt8DataType,                \
               garrow_int8_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_INTEGER_DATA_TYPE)
 
 static void
 garrow_int8_data_type_init(GArrowInt8DataType *object)
@@ -349,7 +379,7 @@ garrow_int8_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowUInt8DataType,                \
               garrow_uint8_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_INTEGER_DATA_TYPE)
 
 static void
 garrow_uint8_data_type_init(GArrowUInt8DataType *object)
@@ -381,7 +411,7 @@ garrow_uint8_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowInt16DataType,                \
               garrow_int16_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_INTEGER_DATA_TYPE)
 
 static void
 garrow_int16_data_type_init(GArrowInt16DataType *object)
@@ -413,7 +443,7 @@ garrow_int16_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowUInt16DataType,                \
               garrow_uint16_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_INTEGER_DATA_TYPE)
 
 static void
 garrow_uint16_data_type_init(GArrowUInt16DataType *object)
@@ -445,7 +475,7 @@ garrow_uint16_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowInt32DataType,                \
               garrow_int32_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_INTEGER_DATA_TYPE)
 
 static void
 garrow_int32_data_type_init(GArrowInt32DataType *object)
@@ -477,7 +507,7 @@ garrow_int32_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowUInt32DataType,                \
               garrow_uint32_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_INTEGER_DATA_TYPE)
 
 static void
 garrow_uint32_data_type_init(GArrowUInt32DataType *object)
@@ -509,7 +539,7 @@ garrow_uint32_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowInt64DataType,                \
               garrow_int64_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_INTEGER_DATA_TYPE)
 
 static void
 garrow_int64_data_type_init(GArrowInt64DataType *object)
@@ -541,7 +571,7 @@ garrow_int64_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowUInt64DataType,                \
               garrow_uint64_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_INTEGER_DATA_TYPE)
 
 static void
 garrow_uint64_data_type_init(GArrowUInt64DataType *object)
@@ -571,9 +601,24 @@ garrow_uint64_data_type_new(void)
 }
 
 
+G_DEFINE_ABSTRACT_TYPE(GArrowFloatingPointDataType,                    \
+                       garrow_floating_point_data_type,                \
+                       GARROW_TYPE_NUMERIC_DATA_TYPE)
+
+static void
+garrow_floating_point_data_type_init(GArrowFloatingPointDataType *object)
+{
+}
+
+static void
+garrow_floating_point_data_type_class_init(GArrowFloatingPointDataTypeClass *klass)
+{
+}
+
+
 G_DEFINE_TYPE(GArrowFloatDataType,                \
               garrow_float_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_FLOATING_POINT_DATA_TYPE)
 
 static void
 garrow_float_data_type_init(GArrowFloatDataType *object)
@@ -605,7 +650,7 @@ garrow_float_data_type_new(void)
 
 G_DEFINE_TYPE(GArrowDoubleDataType,                \
               garrow_double_data_type,             \
-              GARROW_TYPE_DATA_TYPE)
+              GARROW_TYPE_FLOATING_POINT_DATA_TYPE)
 
 static void
 garrow_double_data_type_init(GArrowDoubleDataType *object)
diff --git a/c_glib/arrow-glib/basic-data-type.h b/c_glib/arrow-glib/basic-data-type.h
index 469590191d4e1..478c905e9f925 100644
--- a/c_glib/arrow-glib/basic-data-type.h
+++ b/c_glib/arrow-glib/basic-data-type.h
@@ -116,443 +116,180 @@ struct _GArrowBooleanDataTypeClass
 GArrowBooleanDataType *garrow_boolean_data_type_new      (void);
 
 
-#define GARROW_TYPE_INT8_DATA_TYPE            \
-  (garrow_int8_data_type_get_type())
-#define GARROW_INT8_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
-                              GARROW_TYPE_INT8_DATA_TYPE,     \
-                              GArrowInt8DataType))
-#define GARROW_INT8_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
-                           GARROW_TYPE_INT8_DATA_TYPE,        \
-                           GArrowInt8DataTypeClass))
-#define GARROW_IS_INT8_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
-                              GARROW_TYPE_INT8_DATA_TYPE))
-#define GARROW_IS_INT8_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
-                           GARROW_TYPE_INT8_DATA_TYPE))
-#define GARROW_INT8_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
-                             GARROW_TYPE_INT8_DATA_TYPE,      \
-                             GArrowInt8DataTypeClass))
-
-typedef struct _GArrowInt8DataType         GArrowInt8DataType;
-typedef struct _GArrowInt8DataTypeClass    GArrowInt8DataTypeClass;
+#define GARROW_TYPE_NUMERIC_DATA_TYPE (garrow_numeric_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowNumericDataType,
+                         garrow_numeric_data_type,
+                         GARROW,
+                         NUMERIC_DATA_TYPE,
+                         GArrowFixedWidthDataType)
+struct _GArrowNumericDataTypeClass
+{
+  GArrowFixedWidthDataTypeClass parent_class;
+};
 
-/**
- * GArrowInt8DataType:
- *
- * It wraps `arrow::Int8Type`.
- */
-struct _GArrowInt8DataType
+
+#define GARROW_TYPE_INTEGER_DATA_TYPE (garrow_integer_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowIntegerDataType,
+                         garrow_integer_data_type,
+                         GARROW,
+                         INTEGER_DATA_TYPE,
+                         GArrowNumericDataType)
+struct _GArrowIntegerDataTypeClass
 {
-  /*< private >*/
-  GArrowDataType parent_instance;
+  GArrowNumericDataTypeClass parent_class;
 };
 
+
+#define GARROW_TYPE_INT8_DATA_TYPE (garrow_int8_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInt8DataType,
+                         garrow_int8_data_type,
+                         GARROW,
+                         INT8_DATA_TYPE,
+                         GArrowIntegerDataType)
 struct _GArrowInt8DataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowIntegerDataTypeClass parent_class;
 };
 
-GType                 garrow_int8_data_type_get_type (void) G_GNUC_CONST;
 GArrowInt8DataType   *garrow_int8_data_type_new      (void);
 
 
-#define GARROW_TYPE_UINT8_DATA_TYPE            \
-  (garrow_uint8_data_type_get_type())
-#define GARROW_UINT8_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
-                              GARROW_TYPE_UINT8_DATA_TYPE,     \
-                              GArrowUInt8DataType))
-#define GARROW_UINT8_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
-                           GARROW_TYPE_UINT8_DATA_TYPE,        \
-                           GArrowUInt8DataTypeClass))
-#define GARROW_IS_UINT8_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
-                              GARROW_TYPE_UINT8_DATA_TYPE))
-#define GARROW_IS_UINT8_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
-                           GARROW_TYPE_UINT8_DATA_TYPE))
-#define GARROW_UINT8_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
-                             GARROW_TYPE_UINT8_DATA_TYPE,      \
-                             GArrowUInt8DataTypeClass))
-
-typedef struct _GArrowUInt8DataType         GArrowUInt8DataType;
-typedef struct _GArrowUInt8DataTypeClass    GArrowUInt8DataTypeClass;
-
-/**
- * GArrowUInt8DataType:
- *
- * It wraps `arrow::UInt8Type`.
- */
-struct _GArrowUInt8DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_UINT8_DATA_TYPE (garrow_uint8_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowUInt8DataType,
+                         garrow_uint8_data_type,
+                         GARROW,
+                         UINT8_DATA_TYPE,
+                         GArrowIntegerDataType)
 struct _GArrowUInt8DataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowIntegerDataTypeClass parent_class;
 };
 
-GType                 garrow_uint8_data_type_get_type (void) G_GNUC_CONST;
 GArrowUInt8DataType  *garrow_uint8_data_type_new      (void);
 
 
-#define GARROW_TYPE_INT16_DATA_TYPE            \
-  (garrow_int16_data_type_get_type())
-#define GARROW_INT16_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                           \
-                              GARROW_TYPE_INT16_DATA_TYPE,     \
-                              GArrowInt16DataType))
-#define GARROW_INT16_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                            \
-                           GARROW_TYPE_INT16_DATA_TYPE,        \
-                           GArrowInt16DataTypeClass))
-#define GARROW_IS_INT16_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                           \
-                              GARROW_TYPE_INT16_DATA_TYPE))
-#define GARROW_IS_INT16_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                            \
-                           GARROW_TYPE_INT16_DATA_TYPE))
-#define GARROW_INT16_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                            \
-                             GARROW_TYPE_INT16_DATA_TYPE,      \
-                             GArrowInt16DataTypeClass))
-
-typedef struct _GArrowInt16DataType         GArrowInt16DataType;
-typedef struct _GArrowInt16DataTypeClass    GArrowInt16DataTypeClass;
-
-/**
- * GArrowInt16DataType:
- *
- * It wraps `arrow::Int16Type`.
- */
-struct _GArrowInt16DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_INT16_DATA_TYPE (garrow_int16_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInt16DataType,
+                         garrow_int16_data_type,
+                         GARROW,
+                         INT16_DATA_TYPE,
+                         GArrowIntegerDataType)
 struct _GArrowInt16DataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowIntegerDataTypeClass parent_class;
 };
 
-GType                 garrow_int16_data_type_get_type (void) G_GNUC_CONST;
 GArrowInt16DataType  *garrow_int16_data_type_new      (void);
 
 
-#define GARROW_TYPE_UINT16_DATA_TYPE            \
-  (garrow_uint16_data_type_get_type())
-#define GARROW_UINT16_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT16_DATA_TYPE,     \
-                              GArrowUInt16DataType))
-#define GARROW_UINT16_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT16_DATA_TYPE,        \
-                           GArrowUInt16DataTypeClass))
-#define GARROW_IS_UINT16_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT16_DATA_TYPE))
-#define GARROW_IS_UINT16_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT16_DATA_TYPE))
-#define GARROW_UINT16_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT16_DATA_TYPE,      \
-                             GArrowUInt16DataTypeClass))
-
-typedef struct _GArrowUInt16DataType         GArrowUInt16DataType;
-typedef struct _GArrowUInt16DataTypeClass    GArrowUInt16DataTypeClass;
-
-/**
- * GArrowUInt16DataType:
- *
- * It wraps `arrow::UInt16Type`.
- */
-struct _GArrowUInt16DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_UINT16_DATA_TYPE (garrow_uint16_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowUInt16DataType,
+                         garrow_uint16_data_type,
+                         GARROW,
+                         UINT16_DATA_TYPE,
+                         GArrowIntegerDataType)
 struct _GArrowUInt16DataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowIntegerDataTypeClass parent_class;
 };
 
-GType                 garrow_uint16_data_type_get_type (void) G_GNUC_CONST;
 GArrowUInt16DataType *garrow_uint16_data_type_new      (void);
 
 
-#define GARROW_TYPE_INT32_DATA_TYPE            \
-  (garrow_int32_data_type_get_type())
-#define GARROW_INT32_DATA_TYPE(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT32_DATA_TYPE,      \
-                              GArrowInt32DataType))
-#define GARROW_INT32_DATA_TYPE_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT32_DATA_TYPE, \
-                           GArrowInt32DataTypeClass))
-#define GARROW_IS_INT32_DATA_TYPE(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT32_DATA_TYPE))
-#define GARROW_IS_INT32_DATA_TYPE_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT32_DATA_TYPE))
-#define GARROW_INT32_DATA_TYPE_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT32_DATA_TYPE,       \
-                             GArrowInt32DataTypeClass))
-
-typedef struct _GArrowInt32DataType         GArrowInt32DataType;
-typedef struct _GArrowInt32DataTypeClass    GArrowInt32DataTypeClass;
-
-/**
- * GArrowInt32DataType:
- *
- * It wraps `arrow::Int32Type`.
- */
-struct _GArrowInt32DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_INT32_DATA_TYPE (garrow_int32_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInt32DataType,
+                         garrow_int32_data_type,
+                         GARROW,
+                         INT32_DATA_TYPE,
+                         GArrowIntegerDataType)
 struct _GArrowInt32DataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowIntegerDataTypeClass parent_class;
 };
 
-GType                 garrow_int32_data_type_get_type (void) G_GNUC_CONST;
 GArrowInt32DataType  *garrow_int32_data_type_new      (void);
 
 
-#define GARROW_TYPE_UINT32_DATA_TYPE            \
-  (garrow_uint32_data_type_get_type())
-#define GARROW_UINT32_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT32_DATA_TYPE,     \
-                              GArrowUInt32DataType))
-#define GARROW_UINT32_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT32_DATA_TYPE,        \
-                           GArrowUInt32DataTypeClass))
-#define GARROW_IS_UINT32_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT32_DATA_TYPE))
-#define GARROW_IS_UINT32_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT32_DATA_TYPE))
-#define GARROW_UINT32_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT32_DATA_TYPE,      \
-                             GArrowUInt32DataTypeClass))
-
-typedef struct _GArrowUInt32DataType         GArrowUInt32DataType;
-typedef struct _GArrowUInt32DataTypeClass    GArrowUInt32DataTypeClass;
-
-/**
- * GArrowUInt32DataType:
- *
- * It wraps `arrow::UInt32Type`.
- */
-struct _GArrowUInt32DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_UINT32_DATA_TYPE (garrow_uint32_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowUInt32DataType,
+                         garrow_uint32_data_type,
+                         GARROW,
+                         UINT32_DATA_TYPE,
+                         GArrowIntegerDataType)
 struct _GArrowUInt32DataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowIntegerDataTypeClass parent_class;
 };
 
-GType                 garrow_uint32_data_type_get_type (void) G_GNUC_CONST;
 GArrowUInt32DataType *garrow_uint32_data_type_new      (void);
 
 
-#define GARROW_TYPE_INT64_DATA_TYPE            \
-  (garrow_int64_data_type_get_type())
-#define GARROW_INT64_DATA_TYPE(obj)                             \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_INT64_DATA_TYPE,      \
-                              GArrowInt64DataType))
-#define GARROW_INT64_DATA_TYPE_CLASS(klass)             \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                     \
-                           GARROW_TYPE_INT64_DATA_TYPE, \
-                           GArrowInt64DataTypeClass))
-#define GARROW_IS_INT64_DATA_TYPE(obj)                          \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_INT64_DATA_TYPE))
-#define GARROW_IS_INT64_DATA_TYPE_CLASS(klass)                  \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_INT64_DATA_TYPE))
-#define GARROW_INT64_DATA_TYPE_GET_CLASS(obj)                   \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_INT64_DATA_TYPE,       \
-                             GArrowInt64DataTypeClass))
-
-typedef struct _GArrowInt64DataType         GArrowInt64DataType;
-typedef struct _GArrowInt64DataTypeClass    GArrowInt64DataTypeClass;
-
-/**
- * GArrowInt64DataType:
- *
- * It wraps `arrow::Int64Type`.
- */
-struct _GArrowInt64DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_INT64_DATA_TYPE (garrow_int64_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowInt64DataType,
+                         garrow_int64_data_type,
+                         GARROW,
+                         INT64_DATA_TYPE,
+                         GArrowIntegerDataType)
 struct _GArrowInt64DataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowIntegerDataTypeClass parent_class;
 };
 
-GType                 garrow_int64_data_type_get_type (void) G_GNUC_CONST;
 GArrowInt64DataType  *garrow_int64_data_type_new      (void);
 
 
-#define GARROW_TYPE_UINT64_DATA_TYPE            \
-  (garrow_uint64_data_type_get_type())
-#define GARROW_UINT64_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_UINT64_DATA_TYPE,     \
-                              GArrowUInt64DataType))
-#define GARROW_UINT64_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_UINT64_DATA_TYPE,        \
-                           GArrowUInt64DataTypeClass))
-#define GARROW_IS_UINT64_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_UINT64_DATA_TYPE))
-#define GARROW_IS_UINT64_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_UINT64_DATA_TYPE))
-#define GARROW_UINT64_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_UINT64_DATA_TYPE,      \
-                             GArrowUInt64DataTypeClass))
-
-typedef struct _GArrowUInt64DataType         GArrowUInt64DataType;
-typedef struct _GArrowUInt64DataTypeClass    GArrowUInt64DataTypeClass;
-
-/**
- * GArrowUInt64DataType:
- *
- * It wraps `arrow::UInt64Type`.
- */
-struct _GArrowUInt64DataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_UINT64_DATA_TYPE (garrow_uint64_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowUInt64DataType,
+                         garrow_uint64_data_type,
+                         GARROW,
+                         UINT64_DATA_TYPE,
+                         GArrowIntegerDataType)
 struct _GArrowUInt64DataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowIntegerDataTypeClass parent_class;
 };
 
-GType                 garrow_uint64_data_type_get_type (void) G_GNUC_CONST;
 GArrowUInt64DataType *garrow_uint64_data_type_new      (void);
 
 
-#define GARROW_TYPE_FLOAT_DATA_TYPE           \
-  (garrow_float_data_type_get_type())
-#define GARROW_FLOAT_DATA_TYPE(obj)                           \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
-                              GARROW_TYPE_FLOAT_DATA_TYPE,    \
-                              GArrowFloatDataType))
-#define GARROW_FLOAT_DATA_TYPE_CLASS(klass)                   \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                           \
-                           GARROW_TYPE_FLOAT_DATA_TYPE,       \
-                           GArrowFloatDataTypeClass))
-#define GARROW_IS_FLOAT_DATA_TYPE(obj)                        \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                          \
-                              GARROW_TYPE_FLOAT_DATA_TYPE))
-#define GARROW_IS_FLOAT_DATA_TYPE_CLASS(klass)                \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                           \
-                           GARROW_TYPE_FLOAT_DATA_TYPE))
-#define GARROW_FLOAT_DATA_TYPE_GET_CLASS(obj)                 \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
-                             GARROW_TYPE_FLOAT_DATA_TYPE,     \
-                             GArrowFloatDataTypeClass))
-
-typedef struct _GArrowFloatDataType         GArrowFloatDataType;
-typedef struct _GArrowFloatDataTypeClass    GArrowFloatDataTypeClass;
-
-/**
- * GArrowFloatDataType:
- *
- * It wraps `arrow::FloatType`.
- */
-struct _GArrowFloatDataType
+#define GARROW_TYPE_FLOATING_POINT_DATA_TYPE    \
+  (garrow_floating_point_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowFloatingPointDataType,
+                         garrow_floating_point_data_type,
+                         GARROW,
+                         FLOATING_POINT_DATA_TYPE,
+                         GArrowNumericDataType)
+struct _GArrowFloatingPointDataTypeClass
 {
-  /*< private >*/
-  GArrowDataType parent_instance;
+  GArrowNumericDataTypeClass parent_class;
 };
 
+
+#define GARROW_TYPE_FLOAT_DATA_TYPE (garrow_float_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowFloatDataType,
+                         garrow_float_data_type,
+                         GARROW,
+                         FLOAT_DATA_TYPE,
+                         GArrowFloatingPointDataType)
 struct _GArrowFloatDataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowFloatingPointDataTypeClass parent_class;
 };
 
-GType                garrow_float_data_type_get_type (void) G_GNUC_CONST;
 GArrowFloatDataType *garrow_float_data_type_new      (void);
 
 
-#define GARROW_TYPE_DOUBLE_DATA_TYPE           \
-  (garrow_double_data_type_get_type())
-#define GARROW_DOUBLE_DATA_TYPE(obj)                            \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),                            \
-                              GARROW_TYPE_DOUBLE_DATA_TYPE,     \
-                              GArrowDoubleDataType))
-#define GARROW_DOUBLE_DATA_TYPE_CLASS(klass)                    \
-  (G_TYPE_CHECK_CLASS_CAST((klass),                             \
-                           GARROW_TYPE_DOUBLE_DATA_TYPE,        \
-                           GArrowDoubleDataTypeClass))
-#define GARROW_IS_DOUBLE_DATA_TYPE(obj)                         \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),                            \
-                              GARROW_TYPE_DOUBLE_DATA_TYPE))
-#define GARROW_IS_DOUBLE_DATA_TYPE_CLASS(klass)                 \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),                             \
-                           GARROW_TYPE_DOUBLE_DATA_TYPE))
-#define GARROW_DOUBLE_DATA_TYPE_GET_CLASS(obj)                  \
-  (G_TYPE_INSTANCE_GET_CLASS((obj),                             \
-                             GARROW_TYPE_DOUBLE_DATA_TYPE,      \
-                             GArrowDoubleDataTypeClass))
-
-typedef struct _GArrowDoubleDataType         GArrowDoubleDataType;
-typedef struct _GArrowDoubleDataTypeClass    GArrowDoubleDataTypeClass;
-
-/**
- * GArrowDoubleDataType:
- *
- * It wraps `arrow::DoubleType`.
- */
-struct _GArrowDoubleDataType
-{
-  /*< private >*/
-  GArrowDataType parent_instance;
-};
-
+#define GARROW_TYPE_DOUBLE_DATA_TYPE (garrow_double_data_type_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowDoubleDataType,
+                         garrow_double_data_type,
+                         GARROW,
+                         DOUBLE_DATA_TYPE,
+                         GArrowFloatingPointDataType)
 struct _GArrowDoubleDataTypeClass
 {
-  GArrowDataTypeClass parent_class;
+  GArrowFloatingPointDataTypeClass parent_class;
 };
 
-GType                 garrow_double_data_type_get_type (void) G_GNUC_CONST;
 GArrowDoubleDataType *garrow_double_data_type_new      (void);
 
 

From f56fdc9ccd77987af22093d0d711b5c1a566a094 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 8 Mar 2018 23:32:26 -0500
Subject: [PATCH 1578/1644] ARROW-2270: [Python] Fix lifetime of ForeignBuffer
 base object

Author: Antoine Pitrou <antoine@python.org>

Closes #1714 from pitrou/ARROW-2270-pyforeignbuffer and squashes the following commits:

51f2d85e <Antoine Pitrou> ARROW-2270:  Fix lifetime of ForeignBuffer base object
---
 cpp/src/arrow/python/io.cc           | 14 ++++++++++++++
 cpp/src/arrow/python/io.h            | 21 +++++++++++++++++++++
 python/doc/source/api.rst            |  2 ++
 python/pyarrow/__init__.py           |  2 +-
 python/pyarrow/includes/libarrow.pxd |  5 +++++
 python/pyarrow/io.pxi                | 27 +++++++++++++++------------
 python/pyarrow/lib.pxd               |  5 -----
 python/pyarrow/tests/test_io.py      | 16 ++++++++++------
 8 files changed, 68 insertions(+), 24 deletions(-)

diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
index 801a325743c33..36c193dbfaeb4 100644
--- a/cpp/src/arrow/python/io.cc
+++ b/cpp/src/arrow/python/io.cc
@@ -216,5 +216,19 @@ Status PyOutputStream::Write(const void* data, int64_t nbytes) {
   return file_->Write(data, nbytes);
 }
 
+// ----------------------------------------------------------------------
+// Foreign buffer
+
+Status PyForeignBuffer::Make(const uint8_t* data, int64_t size, PyObject* base,
+                             std::shared_ptr<Buffer>* out) {
+  PyForeignBuffer* buf = new PyForeignBuffer(data, size, base);
+  if (buf == NULL) {
+    return Status::OutOfMemory("could not allocate foreign buffer object");
+  } else {
+    *out = std::shared_ptr<Buffer>(buf);
+    return Status::OK();
+  }
+}
+
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h
index 6960556103b77..5c76fe9fe71e5 100644
--- a/cpp/src/arrow/python/io.h
+++ b/cpp/src/arrow/python/io.h
@@ -81,6 +81,27 @@ class ARROW_EXPORT PyOutputStream : public io::OutputStream {
 
 // TODO(wesm): seekable output files
 
+// A Buffer subclass that keeps a PyObject reference throughout its
+// lifetime, such that the Python object is kept alive as long as the
+// C++ buffer is still needed.
+// Keeping the reference in a Python wrapper would be incorrect as
+// the Python wrapper can get destroyed even though the wrapped C++
+// buffer is still alive (ARROW-2270).
+class ARROW_EXPORT PyForeignBuffer : public Buffer {
+ public:
+  static Status Make(const uint8_t* data, int64_t size, PyObject* base,
+                     std::shared_ptr<Buffer>* out);
+
+ private:
+  PyForeignBuffer(const uint8_t* data, int64_t size, PyObject* base)
+      : Buffer(data, size) {
+    Py_INCREF(base);
+    base_.reset(base);
+  }
+
+  OwnedRefNoGIL base_;
+};
+
 }  // namespace py
 }  // namespace arrow
 
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index a71e92b0b634b..3db1a04b6d206 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -186,6 +186,7 @@ Tables and Record Batches
 
    column
    chunked_array
+   concat_tables
    ChunkedArray
    Column
    RecordBatch
@@ -213,6 +214,7 @@ Input / Output and Shared Memory
    compress
    decompress
    frombuffer
+   foreign_buffer
    Buffer
    ResizableBuffer
    BufferReader
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 28ac98ea06984..225dfd0b28aed 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -86,7 +86,7 @@ def parse_version(root):
 from pyarrow.lib import TimestampType
 
 # Buffers, allocation
-from pyarrow.lib import (Buffer, ForeignBuffer, ResizableBuffer, compress,
+from pyarrow.lib import (Buffer, ResizableBuffer, foreign_buffer, compress,
                          decompress, allocate_buffer, frombuffer)
 
 from pyarrow.lib import (MemoryPool, total_allocated_bytes,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 456fcca3602db..22c39a865b95f 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -904,6 +904,11 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
         @staticmethod
         CStatus FromPyObject(object obj, shared_ptr[CBuffer]* out)
 
+    cdef cppclass PyForeignBuffer(CBuffer):
+        @staticmethod
+        CStatus Make(const uint8_t* data, int64_t size, object base,
+                     shared_ptr[CBuffer]* out)
+
     cdef cppclass PyReadableFile(RandomAccessFile):
         PyReadableFile(object fo)
 
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 611c8a86dadce..15ecd0164e439 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -726,18 +726,6 @@ cdef class Buffer:
         return self.size
 
 
-cdef class ForeignBuffer(Buffer):
-
-    def __init__(self, addr, size, base):
-        cdef:
-            intptr_t c_addr = addr
-            int64_t c_size = size
-        self.base = base
-        cdef shared_ptr[CBuffer] buffer = make_shared[CBuffer](
-            <uint8_t*>c_addr, c_size)
-        self.init(<shared_ptr[CBuffer]> buffer)
-
-
 cdef class ResizableBuffer(Buffer):
 
     cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer):
@@ -861,6 +849,21 @@ def frombuffer(object obj):
     return pyarrow_wrap_buffer(buf)
 
 
+def foreign_buffer(address, size, base):
+    """
+    Construct an Arrow buffer with the given *address* and *size*,
+    backed by the Python *base* object.
+    """
+    cdef:
+        intptr_t c_addr = address
+        int64_t c_size = size
+        shared_ptr[CBuffer] buf
+
+    check_status(PyForeignBuffer.Make(<uint8_t*> c_addr, c_size,
+                                      base, &buf))
+    return pyarrow_wrap_buffer(buf)
+
+
 cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader):
     cdef NativeFile nf
 
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index c37bc2bebe7ea..e4d574f18b34f 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -324,11 +324,6 @@ cdef class Buffer:
     cdef int _check_nullptr(self) except -1
 
 
-cdef class ForeignBuffer(Buffer):
-    cdef:
-        object base
-
-
 cdef class ResizableBuffer(Buffer):
 
     cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index e42914ffcfbed..fe680133b4b32 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -24,7 +24,6 @@
 import weakref
 
 import numpy as np
-import numpy.testing as npt
 
 import pandas as pd
 
@@ -271,11 +270,16 @@ def test_buffer_hashing():
 
 
 def test_foreign_buffer():
-    n = np.array([1, 2])
-    addr = n.__array_interface__["data"][0]
-    size = n.nbytes
-    fb = pa.ForeignBuffer(addr, size, n)
-    npt.assert_array_equal(np.asarray(fb), n.view(dtype=np.int8))
+    obj = np.array([1, 2], dtype=np.int32)
+    addr = obj.__array_interface__["data"][0]
+    size = obj.nbytes
+    buf = pa.foreign_buffer(addr, size, obj)
+    wr = weakref.ref(obj)
+    del obj
+    assert np.frombuffer(buf, dtype=np.int32).tolist() == [1, 2]
+    assert wr() is not None
+    del buf
+    assert wr() is None
 
 
 def test_allocate_buffer():

From 40a0008509e479af7e338d16728cd80f686ec947 Mon Sep 17 00:00:00 2001
From: Mitar <mitar.github@tnode.com>
Date: Thu, 8 Mar 2018 20:33:30 -0800
Subject: [PATCH 1579/1644] [Python] Adding more missing Linux dependencies to
 developer docs

---
 python/doc/source/development.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 42f407a292dd2..3e1e72be19852 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -118,7 +118,9 @@ dependencies will be automatically built by Arrow's third-party toolchain.
    $ sudo apt-get install libjemalloc-dev libboost-dev \
                           libboost-filesystem-dev \
                           libboost-system-dev \
-                          libboost-regex-dev
+                          libboost-regex-dev \
+                          flex \
+                          bison
 
 On Arch Linux, you can get these dependencies via pacman.
 

From 23d08b7e68190159caac719d35e0e7578fdef36c Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 9 Mar 2018 10:17:31 -0500
Subject: [PATCH 1580/1644] ARROW-2150: [Python] Raise NotImplementedError when
 comparing with pyarrow.Array for now

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1729 from wesm/ARROW-2150 and squashes the following commits:

f0ddcf5e <Wes McKinney> Raise NotImplementedError when comparing with pyarrow.Array for now
---
 python/pyarrow/array.pxi           |  4 ++++
 python/pyarrow/tests/test_array.py | 10 ++++++++++
 2 files changed, 14 insertions(+)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index e785c0ec5c3de..f05806cfac551 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -267,6 +267,10 @@ cdef class Array:
         self.ap = sp_array.get()
         self.type = pyarrow_wrap_data_type(self.sp_array.get().type())
 
+    def __richcmp__(Array self, object other, int op):
+        raise NotImplementedError('Comparisons with pyarrow.Array are not '
+                                  'implemented')
+
     def _debug_print(self):
         with nogil:
             check_status(DebugPrint(deref(self.ap), 0))
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index f034d78b39396..4c14c1c61a762 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -158,6 +158,16 @@ def test_array_ref_to_ndarray_base():
     assert sys.getrefcount(arr) == (refcount + 1)
 
 
+def test_array_eq_raises():
+    # ARROW-2150: we are raising when comparing arrays until we define the
+    # behavior to either be elementwise comparisons or data equality
+    arr1 = pa.array([1, 2, 3], type=pa.int32())
+    arr2 = pa.array([1, 2, 3], type=pa.int32())
+
+    with pytest.raises(NotImplementedError):
+        arr1 == arr2
+
+
 def test_dictionary_from_numpy():
     indices = np.repeat([0, 1, 2], 2)
     dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)

From 7354a190d78a0cadb7e0db9a2014b8a5e2848410 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Fri, 9 Mar 2018 10:19:44 -0500
Subject: [PATCH 1581/1644] ARROW-2284: [Python] Fix error display on
 test_plasma error

Just a trivial fix.  stderr is captured by py.test, not by the subprocess call.

Author: Antoine Pitrou <antoine@python.org>

Closes #1724 from pitrou/ARROW-2284 and squashes the following commits:

46a692c9 <Antoine Pitrou> ARROW-2284:  Fix error display on test_plasma error
---
 python/pyarrow/tests/test_plasma.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index b4e8649414412..1df213dec4741 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -165,10 +165,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
             time.sleep(0.1)
         rc = proc.poll()
         if rc is not None:
-            err = proc.stderr.read().decode()
             raise RuntimeError("plasma_store exited unexpectedly with "
-                               "code %d. Error output follows:\n%s\n"
-                               % (rc, err))
+                               "code %d" % (rc,))
 
         yield plasma_store_name, proc
     finally:

From 816747208c31625f9b198ab3d29c2c3185ba3c25 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Fri, 9 Mar 2018 10:24:15 -0500
Subject: [PATCH 1582/1644] ARROW-2275: [C++] Guard against bad use of
 Buffer.mutable_data()

Also disambiguate the Tensor API on this front.

Author: Antoine Pitrou <antoine@python.org>

Closes #1717 from pitrou/ARROW-2275-bad-mutable-data and squashes the following commits:

fabd7b9f <Antoine Pitrou> ARROW-2275:  Guard against bad use of Buffer.mutable_data()
---
 cpp/src/arrow/buffer.cc |  8 ++++++++
 cpp/src/arrow/buffer.h  | 10 +++++++++-
 cpp/src/arrow/tensor.h  |  2 +-
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index 29e2c242a3f4a..e32e02c9ffdd7 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -70,6 +70,14 @@ Status Buffer::FromString(const std::string& data, std::shared_ptr<Buffer>* out)
   return FromString(data, default_memory_pool(), out);
 }
 
+#ifndef NDEBUG
+// DCHECK macros aren't allowed in public include files
+uint8_t* Buffer::mutable_data() {
+  DCHECK(is_mutable());
+  return mutable_data_;
+}
+#endif
+
 PoolBuffer::PoolBuffer(MemoryPool* pool) : ResizableBuffer(nullptr, 0) {
   if (pool == nullptr) {
     pool = default_memory_pool();
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index cf25ccd036490..ad11ff943d448 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -54,7 +54,11 @@ class ARROW_EXPORT Buffer {
   ///
   /// \note The passed memory must be kept alive through some other means
   Buffer(const uint8_t* data, int64_t size)
-      : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
+      : is_mutable_(false),
+        data_(data),
+        mutable_data_(NULLPTR),
+        size_(size),
+        capacity_(size) {}
 
   /// \brief Construct from std::string without copying memory
   ///
@@ -113,7 +117,11 @@ class ARROW_EXPORT Buffer {
 
   int64_t capacity() const { return capacity_; }
   const uint8_t* data() const { return data_; }
+#ifdef NDEBUG
   uint8_t* mutable_data() { return mutable_data_; }
+#else
+  uint8_t* mutable_data();
+#endif
 
   int64_t size() const { return size_; }
 
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index 4e4c6b8d57c33..699dc0393099c 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -71,7 +71,7 @@ class ARROW_EXPORT Tensor {
   std::shared_ptr<Buffer> data() const { return data_; }
 
   const uint8_t* raw_data() const { return data_->data(); }
-  uint8_t* raw_data() { return data_->mutable_data(); }
+  uint8_t* raw_mutable_data() { return data_->mutable_data(); }
 
   const std::vector<int64_t>& shape() const { return shape_; }
   const std::vector<int64_t>& strides() const { return strides_; }

From 3511c654270913c8dbf2e1c05e2bd724057f8c01 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 9 Mar 2018 10:27:11 -0500
Subject: [PATCH 1583/1644] ARROW-2268: Drop usage of md5 checksums for source
 releases, verification scripts

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1731 from wesm/ARROW-2268 and squashes the following commits:

42768eff <Wes McKinney> Drop usage of md5 checksums for source releases, verification scripts
---
 dev/release/02-source.sh                   | 1 -
 dev/release/js-source-release.sh           | 1 -
 dev/release/js-verify-release-candidate.sh | 5 +++--
 dev/release/verify-release-candidate.sh    | 5 +++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index 62478131d8c93..fa1c3e3cad693 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -97,7 +97,6 @@ ${SOURCE_DIR}/run-rat.sh ${tarball}
 
 # sign the archive
 gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
-gpg --print-md MD5 ${tarball} > ${tarball}.md5
 sha1sum $tarball > ${tarball}.sha1
 sha256sum $tarball > ${tarball}.sha256
 sha512sum $tarball > ${tarball}.sha512
diff --git a/dev/release/js-source-release.sh b/dev/release/js-source-release.sh
index bf32acd052403..53b31af62f2af 100755
--- a/dev/release/js-source-release.sh
+++ b/dev/release/js-source-release.sh
@@ -78,7 +78,6 @@ ${SOURCE_DIR}/run-rat.sh ${tarball}
 
 # sign the archive
 gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
-gpg --print-md MD5 ${tarball} > ${tarball}.md5
 sha1sum $tarball > ${tarball}.sha1
 sha256sum $tarball > ${tarball}.sha256
 sha512sum $tarball > ${tarball}.sha512
diff --git a/dev/release/js-verify-release-candidate.sh b/dev/release/js-verify-release-candidate.sh
index 5a37e10f74afb..039c94dec5349 100755
--- a/dev/release/js-verify-release-candidate.sh
+++ b/dev/release/js-verify-release-candidate.sh
@@ -54,13 +54,14 @@ fetch_archive() {
   local dist_name=$1
   download_rc_file ${dist_name}.tar.gz
   download_rc_file ${dist_name}.tar.gz.asc
-  download_rc_file ${dist_name}.tar.gz.md5
+  download_rc_file ${dist_name}.tar.gz.sha1
   download_rc_file ${dist_name}.tar.gz.sha512
   gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
-  gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
   if [ "$(uname)" == "Darwin" ]; then
+    shasum -a 1 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha1
     shasum -a 512 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
   else
+    sha1sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha1
     sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
   fi
 }
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index f33211e26a3f6..cb9b01b37e595 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -62,13 +62,14 @@ fetch_archive() {
   local dist_name=$1
   download_rc_file ${dist_name}.tar.gz
   download_rc_file ${dist_name}.tar.gz.asc
-  download_rc_file ${dist_name}.tar.gz.md5
+  download_rc_file ${dist_name}.tar.gz.sha1
   download_rc_file ${dist_name}.tar.gz.sha512
   gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
-  gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
   if [ "$(uname)" == "Darwin" ]; then
+    shasum -a 1 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha1
     shasum -a 512 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
   else
+    sha1sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha1
     sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
   fi
 }

From c7c2393bf61dc958ff35b0574696e97462401756 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 9 Mar 2018 10:29:38 -0500
Subject: [PATCH 1584/1644] ARROW-2269: [Python] Make boost namespace
 selectable in wheels

cc @mitar

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1718 from xhochy/ARROW-2269 and squashes the following commits:

edc1f3d9 <Uwe L. Korn> ARROW-2269:  Make boost namespace selectable in wheels
---
 python/CMakeLists.txt            |  5 ++++-
 python/manylinux1/build_arrow.sh |  4 +++-
 python/setup.py                  | 29 +++++++++++++++++++++++++----
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index d17194628480a..44a3c6c91a553 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -76,6 +76,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PYARROW_BUNDLE_ARROW_CPP
     "Bundle the Arrow C++ libraries"
     OFF)
+  option(PYARROW_BUNDLE_BOOST
+    "Bundle the Boost libraries when we bundle Arrow C++"
+    ON)
   set(PYARROW_CXXFLAGS "" CACHE STRING
     "Compiler flags to append when compiling Arrow")
 endif()
@@ -266,7 +269,7 @@ if (PYARROW_BUNDLE_ARROW_CPP)
     SO_VERSION ${ARROW_SO_VERSION})
 
   # boost
-  if (PYARROW_BOOST_USE_SHARED)
+  if (PYARROW_BOOST_USE_SHARED AND PYARROW_BUNDLE_BOOST)
     set(Boost_USE_STATIC_LIBS OFF)
     set(Boost_USE_MULTITHREADED ON)
     if (MSVC AND ARROW_USE_STATIC_CRT)
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index f83c759729c6b..5df55a65ca52b 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -40,6 +40,8 @@ export PYARROW_BUILD_TYPE='release'
 export PYARROW_WITH_PARQUET=1
 export PYARROW_WITH_PLASMA=1
 export PYARROW_BUNDLE_ARROW_CPP=1
+export PYARROW_BUNDLE_BOOST=1
+export PYARROW_BOOST_NAMESPACE=arrow_boost
 export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
 export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist'
 # Ensure the target directory exists
@@ -66,7 +68,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
     # Clear output directory
     rm -rf dist/
     echo "=== (${PYTHON}) Building wheel ==="
-    PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp
+    PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp --bundle-boost --boost-namespace=arrow_boost
     PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
     echo "=== (${PYTHON}) Test the existence of optional modules ==="
diff --git a/python/setup.py b/python/setup.py
index f3521f2777ace..6f0b0fa4d4850 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -94,11 +94,15 @@ def run(self):
     description = "Build the C-extensions for arrow"
     user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
                      ('build-type=', None, 'build type (debug or release)'),
+                     ('boost-namespace=', None,
+                      'namespace of boost (default: boost)'),
                      ('with-parquet', None, 'build the Parquet extension'),
                      ('with-static-parquet', None, 'link parquet statically'),
                      ('with-static-boost', None, 'link boost statically'),
                      ('with-plasma', None, 'build the Plasma extension'),
                      ('with-orc', None, 'build the ORC extension'),
+                     ('bundle-boost', None,
+                      'bundle the (shared) Boost libraries'),
                      ('bundle-arrow-cpp', None,
                       'bundle the Arrow C++ libraries')] +
                     _build_ext.user_options)
@@ -107,6 +111,8 @@ def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
+        self.boost_namespace = os.environ.get('PYARROW_BOOST_NAMESPACE',
+                                              'boost')
 
         self.cmake_cxxflags = os.environ.get('PYARROW_CXXFLAGS', '')
 
@@ -128,6 +134,10 @@ def initialize_options(self):
             os.environ.get('PYARROW_WITH_ORC', '0'))
         self.bundle_arrow_cpp = strtobool(
             os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
+        # Default is True but this only is actually bundled when
+        # we also bundle arrow-cpp.
+        self.bundle_boost = strtobool(
+            os.environ.get('PYARROW_BUNDLE_BOOST', '1'))
 
     CYTHON_MODULE_NAMES = [
         'lib',
@@ -186,15 +196,20 @@ def _run_cmake(self):
 
             if self.bundle_arrow_cpp:
                 cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
+                cmake_options.append('-DPYARROW_BUNDLE_BOOST=ON')
                 # ARROW-1090: work around CMake rough edges
                 if 'ARROW_HOME' in os.environ and sys.platform != 'win32':
                     pkg_config = pjoin(os.environ['ARROW_HOME'], 'lib',
                                        'pkgconfig')
                     os.environ['PKG_CONFIG_PATH'] = pkg_config
                     del os.environ['ARROW_HOME']
+            else:
+                cmake_options.append('-DPYARROW_BUNDLE_BOOST=OFF')
 
             cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                                  .format(self.build_type.lower()))
+            cmake_options.append('-DBoost_NAMESPACE={}'.format(
+                self.boost_namespace))
 
             extra_cmake_args = shlex.split(self.extra_cmake_args)
             if sys.platform != 'win32':
@@ -258,10 +273,16 @@ def _run_cmake(self):
                     move_shared_libs(build_prefix, build_lib, "plasma")
                 if self.with_parquet and not self.with_static_parquet:
                     move_shared_libs(build_prefix, build_lib, "parquet")
-                if not self.with_static_boost:
-                    move_shared_libs(build_prefix, build_lib, "arrow_boost_filesystem")
-                    move_shared_libs(build_prefix, build_lib, "arrow_boost_system")
-                    move_shared_libs(build_prefix, build_lib, "arrow_boost_regex")
+                if not self.with_static_boost and self.bundle_boost:
+                    move_shared_libs(
+                        build_prefix, build_lib,
+                        "{}_filesystem".format(self.boost_namespace))
+                    move_shared_libs(
+                        build_prefix, build_lib,
+                        "{}_system".format(self.boost_namespace))
+                    move_shared_libs(
+                        build_prefix, build_lib,
+                        "{}_regex".format(self.boost_namespace))
 
             print('Bundling includes: ' + pjoin(build_prefix, 'include'))
             if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):

From fc9f89adf7fc7317db785e340799829f483f5aa4 Mon Sep 17 00:00:00 2001
From: Mitar <mitar.git@tnode.com>
Date: Fri, 9 Mar 2018 10:30:32 -0500
Subject: [PATCH 1585/1644] ARROW-2250: [Python] Do not create a subprocess for
 plasma but just use existing process

Author: Mitar <mitar.git@tnode.com>

Closes #1705 from mitar/ARROW-2250 and squashes the following commits:

4e71d445 <Mitar> ARROW-2250:  Do not create a subprocess for plasma but just use existing process.
---
 python/pyarrow/__init__.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 225dfd0b28aed..965a37b4b9cf8 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -156,11 +156,9 @@ def _plasma_store_entry_point():
     """
     import os
     import pyarrow
-    import subprocess
     import sys
     plasma_store_executable = os.path.join(pyarrow.__path__[0], "plasma_store")
-    process = subprocess.Popen([plasma_store_executable] + sys.argv[1:])
-    process.wait()
+    os.execv(plasma_store_executable, sys.argv)
 
 # ----------------------------------------------------------------------
 # Deprecations

From d0284cb1857456f6ecbd1f4ae96eee8b3a259794 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Fri, 9 Mar 2018 10:32:36 -0500
Subject: [PATCH 1586/1644] ARROW-2236: [JS] Add more complete set of
 predicates

Adds not, gt, lt, and neq

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1683 from TheNeuralBit/js-more-predicates and squashes the following commits:

707de827 <Brian Hulette> Use two letter names
56ecea38 <Brian Hulette> export packBools, import compiled code in vector-test
32b26e3b <Brian Hulette> lint
57383277 <Brian Hulette> add externs
84895a00 <Brian Hulette> Add not, lt, gt, neq
---
 js/src/Arrow.externs.js      | 15 ++++++++++++--
 js/src/Arrow.ts              | 27 +++++++++++++------------
 js/src/predicate.ts          | 25 ++++++++++++++++++++++--
 js/test/unit/table-tests.ts  | 38 +++++++++++++++++++++++++-----------
 js/test/unit/vector-tests.ts | 10 +++++-----
 5 files changed, 83 insertions(+), 32 deletions(-)

diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index a14f959bb469b..4932c677507b1 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -74,17 +74,24 @@ var custom = function () {};
 
 var Value = function() {};
 /** @type {?} */
-Value.prototype.gteq;
+Value.prototype.ge;
 /** @type {?} */
-Value.prototype.lteq;
+Value.prototype.le;
 /** @type {?} */
 Value.prototype.eq;
+/** @type {?} */
+Value.prototype.lt;
+/** @type {?} */
+Value.prototype.gt;
+/** @type {?} */
+Value.prototype.ne;
 
 var Col = function() {};
 /** @type {?} */
 Col.prototype.bind;
 var Or = function() {};
 var And = function() {};
+var Not = function() {};
 var GTeq = function () {};
 /** @type {?} */
 GTeq.prototype.and;
@@ -108,6 +115,8 @@ Predicate.prototype.and;
 /** @type {?} */
 Predicate.prototype.or;
 /** @type {?} */
+Predicate.prototype.not;
+/** @type {?} */
 Predicate.prototype.ands;
 var Literal = function() {};
 
@@ -209,6 +218,8 @@ Int128.prototype.plus
 /** @type {?} */
 Int128.prototype.hex
 
+var packBools = function() {};
+
 var Type = function() {};
 /** @type {?} */
 Type.NONE = function() {};
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index 4a0a2ac6de10a..23e8b99832dae 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -18,7 +18,8 @@
 import * as type_ from './type';
 import * as data_ from './data';
 import * as vector_ from './vector';
-import * as util_ from './util/int';
+import * as util_int_ from './util/int';
+import * as util_bit_ from './util/bit';
 import * as visitor_ from './visitor';
 import * as view_ from './vector/view';
 import * as predicate_ from './predicate';
@@ -40,9 +41,10 @@ export { Table, DataFrame, NextFunc, BindFunc, CountByResult };
 export { Field, Schema, RecordBatch, Vector, Type };
 
 export namespace util {
-    export import Uint64 = util_.Uint64;
-    export import Int64 = util_.Int64;
-    export import Int128 = util_.Int128;
+    export import Uint64 = util_int_.Uint64;
+    export import Int64 = util_int_.Int64;
+    export import Int128 = util_int_.Int128;
+    export import packBools = util_bit_.packBools;
 }
 
 export namespace data {
@@ -173,6 +175,7 @@ export namespace predicate {
     export import Or = predicate_.Or;
     export import Col = predicate_.Col;
     export import And = predicate_.And;
+    export import Not = predicate_.Not;
     export import GTeq = predicate_.GTeq;
     export import LTeq = predicate_.LTeq;
     export import Value = predicate_.Value;
@@ -222,16 +225,16 @@ Table['empty'] = Table.empty;
 Vector['create'] = Vector.create;
 RecordBatch['from'] = RecordBatch.from;
 
-util_.Uint64['add'] = util_.Uint64.add;
-util_.Uint64['multiply'] = util_.Uint64.multiply;
+util_int_.Uint64['add'] = util_int_.Uint64.add;
+util_int_.Uint64['multiply'] = util_int_.Uint64.multiply;
 
-util_.Int64['add'] = util_.Int64.add;
-util_.Int64['multiply'] = util_.Int64.multiply;
-util_.Int64['fromString'] = util_.Int64.fromString;
+util_int_.Int64['add'] = util_int_.Int64.add;
+util_int_.Int64['multiply'] = util_int_.Int64.multiply;
+util_int_.Int64['fromString'] = util_int_.Int64.fromString;
 
-util_.Int128['add'] = util_.Int128.add;
-util_.Int128['multiply'] = util_.Int128.multiply;
-util_.Int128['fromString'] = util_.Int128.fromString;
+util_int_.Int128['add'] = util_int_.Int128.add;
+util_int_.Int128['multiply'] = util_int_.Int128.multiply;
+util_int_.Int128['fromString'] = util_int_.Int128.fromString;
 
 data_.ChunkedData['computeOffsets'] = data_.ChunkedData.computeOffsets;
 
diff --git a/js/src/predicate.ts b/js/src/predicate.ts
index b177b4fa76459..bff393863bfd0 100644
--- a/js/src/predicate.ts
+++ b/js/src/predicate.ts
@@ -26,14 +26,23 @@ export abstract class Value<T> {
         if (!(other instanceof Value)) { other = new Literal(other); }
         return new Equals(this, other);
     }
-    lteq(other: Value<T> | T): Predicate {
+    le(other: Value<T> | T): Predicate {
         if (!(other instanceof Value)) { other = new Literal(other); }
         return new LTeq(this, other);
     }
-    gteq(other: Value<T> | T): Predicate {
+    ge(other: Value<T> | T): Predicate {
         if (!(other instanceof Value)) { other = new Literal(other); }
         return new GTeq(this, other);
     }
+    lt(other: Value<T> | T): Predicate {
+        return new Not(this.ge(other));
+    }
+    gt(other: Value<T> | T): Predicate {
+        return new Not(this.le(other));
+    }
+    ne(other: Value<T> | T): Predicate {
+        return new Not(this.eq(other));
+    }
 }
 
 export class Literal<T= any> extends Value<T> {
@@ -70,6 +79,7 @@ export abstract class Predicate {
     abstract bind(batch: RecordBatch): PredicateFunc;
     and(expr: Predicate): Predicate { return new And(this, expr); }
     or(expr: Predicate): Predicate { return new Or(this, expr); }
+    not(): Predicate { return new Not(this); }
     ands(): Predicate[] { return [this]; }
 }
 
@@ -222,6 +232,17 @@ export class GTeq extends ComparisonPredicate {
     }
 }
 
+export class Not extends Predicate {
+    constructor(public readonly child: Predicate) {
+        super();
+    }
+
+    bind(batch: RecordBatch) {
+        const func = this.child.bind(batch);
+        return (idx: number, batch: RecordBatch) => !func(idx, batch);
+    }
+}
+
 export class CustomPredicate extends Predicate {
     constructor(private next: PredicateFunc, private bind_: (batch: RecordBatch) => void) {
         super();
diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts
index 8a433815dab50..37bbf624241d8 100644
--- a/js/test/unit/table-tests.ts
+++ b/js/test/unit/table-tests.ts
@@ -327,27 +327,39 @@ describe(`Table`, () => {
             const filter_tests = [
                 {
                     name:     `filter on f32 >= 0`,
-                    filtered: table.filter(col('f32').gteq(0)),
+                    filtered: table.filter(col('f32').ge(0)),
                     expected: values.filter((row) => row[F32] >= 0)
                 }, {
                     name:     `filter on 0 <= f32`,
-                    filtered: table.filter(lit(0).lteq(col('f32'))),
+                    filtered: table.filter(lit(0).le(col('f32'))),
                     expected: values.filter((row) => 0 <= row[F32])
                 }, {
                     name:     `filter on i32 <= 0`,
-                    filtered: table.filter(col('i32').lteq(0)),
+                    filtered: table.filter(col('i32').le(0)),
                     expected: values.filter((row) => row[I32] <= 0)
                 }, {
                     name:     `filter on 0 >= i32`,
-                    filtered: table.filter(lit(0).gteq(col('i32'))),
+                    filtered: table.filter(lit(0).ge(col('i32'))),
                     expected: values.filter((row) => 0 >= row[I32])
+                }, {
+                    name:     `filter on f32 < 0`,
+                    filtered: table.filter(col('f32').lt(0)),
+                    expected: values.filter((row) => row[F32] < 0)
+                }, {
+                    name:     `filter on i32 > 1 (empty)`,
+                    filtered: table.filter(col('i32').gt(0)),
+                    expected: values.filter((row) => row[I32] > 0)
                 }, {
                     name:     `filter on f32 <= -.25 || f3 >= .25`,
-                    filtered: table.filter(col('f32').lteq(-.25).or(col('f32').gteq(.25))),
+                    filtered: table.filter(col('f32').le(-.25).or(col('f32').ge(.25))),
                     expected: values.filter((row) => row[F32] <= -.25 || row[F32] >= .25)
+                }, {
+                    name:     `filter on !(f32 <= -.25 || f3 >= .25) (not)`,
+                    filtered: table.filter(col('f32').le(-.25).or(col('f32').ge(.25)).not()),
+                    expected: values.filter((row) => !(row[F32] <= -.25 || row[F32] >= .25))
                 }, {
                     name:     `filter method combines predicates (f32 >= 0 && i32 <= 0)`,
-                    filtered: table.filter(col('i32').lteq(0)).filter(col('f32').gteq(0)),
+                    filtered: table.filter(col('i32').le(0)).filter(col('f32').ge(0)),
                     expected: values.filter((row) => row[I32] <= 0 && row[F32] >= 0)
                 }, {
                     name:     `filter on dictionary == 'a'`,
@@ -357,13 +369,17 @@ describe(`Table`, () => {
                     name:     `filter on 'a' == dictionary (commutativity)`,
                     filtered: table.filter(lit('a').eq(col('dictionary'))),
                     expected: values.filter((row) => row[DICT] === 'a')
+                }, {
+                    name:     `filter on dictionary != 'b'`,
+                    filtered: table.filter(col('dictionary').ne('b')),
+                    expected: values.filter((row) => row[DICT] !== 'b')
                 }, {
                     name:     `filter on f32 >= i32`,
-                    filtered: table.filter(col('f32').gteq(col('i32'))),
+                    filtered: table.filter(col('f32').ge(col('i32'))),
                     expected: values.filter((row) => row[F32] >= row[I32])
                 }, {
                     name:     `filter on f32 <= i32`,
-                    filtered: table.filter(col('f32').lteq(col('i32'))),
+                    filtered: table.filter(col('f32').le(col('i32'))),
                     expected: values.filter((row) => row[F32] <= row[I32])
                 }, {
                     name:     `filter on f32*i32 > 0 (custom predicate)`,
@@ -455,17 +471,17 @@ describe(`Table`, () => {
                 expect(selected.toString()).toEqual(expected);
             });
             test(`table.filter(..).count() on always false predicates returns 0`, () => {
-                expect(table.filter(col('i32').gteq(100)).count()).toEqual(0);
+                expect(table.filter(col('i32').ge(100)).count()).toEqual(0);
                 expect(table.filter(col('dictionary').eq('z')).count()).toEqual(0);
             });
             describe(`lit-lit comparison`, () => {
                 test(`always-false count() returns 0`, () => {
                     expect(table.filter(lit('abc').eq('def')).count()).toEqual(0);
-                    expect(table.filter(lit(0).gteq(1)).count()).toEqual(0);
+                    expect(table.filter(lit(0).ge(1)).count()).toEqual(0);
                 });
                 test(`always-true count() returns length`, () => {
                     expect(table.filter(lit('abc').eq('abc')).count()).toEqual(table.length);
-                    expect(table.filter(lit(-100).lteq(0)).count()).toEqual(table.length);
+                    expect(table.filter(lit(-100).le(0)).count()).toEqual(table.length);
                 });
             });
             describe(`col-col comparison`, () => {
diff --git a/js/test/unit/vector-tests.ts b/js/test/unit/vector-tests.ts
index 3eb3fbe0195b3..d25e0e9fdeb01 100644
--- a/js/test/unit/vector-tests.ts
+++ b/js/test/unit/vector-tests.ts
@@ -17,13 +17,13 @@
 
 import { TextEncoder } from 'text-encoding-utf-8';
 import Arrow from '../Arrow';
-import { type, TypedArray, TypedArrayConstructor, Vector } from '../../src/Arrow';
-import { packBools } from '../../src/util/bit'
+import { TypedArray, TypedArrayConstructor } from '../../src/Arrow';
 
 const utf8Encoder = new TextEncoder('utf-8');
 
+const { packBools } = Arrow.util;
 const { BoolData, FlatData, FlatListData, DictionaryData } = Arrow.data;
-const { IntVector, FloatVector, BoolVector, Utf8Vector, DictionaryVector } = Arrow.vector;
+const { Vector, IntVector, FloatVector, BoolVector, Utf8Vector, DictionaryVector } = Arrow.vector;
 const {
     Dictionary, Utf8, Bool,
     Float16, Float32, Float64,
@@ -143,7 +143,7 @@ describe('Float16Vector', () => {
     const values = concatTyped(Uint16Array, ...bytes);
     const vector = bytes
         .map((b) => new Uint16Array(b.buffer))
-        .map((b) => new FloatVector<type.Float16>(new FlatData(new Float16(), b.length, null, b)))
+        .map((b) => new FloatVector<Float16>(new FlatData(new Float16(), b.length, null, b)))
         .reduce((v: any, v2) => v.concat(v2));
     const n = values.length;
     const clamp = (x: number) => (x -  32767) / 32767;
@@ -336,7 +336,7 @@ describe(`DictionaryVector`, () => {
 
         describe(`sliced`, () => {
             basicVectorTests(vector.slice(10, 20), values.slice(10,20), extras);
-        })
+        });
     });
 
     describe(`index with nullCount > 0`, () => {

From 412bb91f8ebe9b48851642c7ab08ae38fc2eea35 Mon Sep 17 00:00:00 2001
From: Andy Grove <andygrove73@gmail.com>
Date: Fri, 9 Mar 2018 13:21:11 -0500
Subject: [PATCH 1587/1644] ARROW-2291: [C++] Add additional libboost-regex-dev
 to build instructions in README

libboost-regex-dev is required

Author: Andy Grove <andygrove73@gmail.com>

Closes #1732 from andygrove/ARROW-2291 and squashes the following commits:

f82154ab <Andy Grove> update README to include instructions for installing libboost-regex-dev since this is required
---
 cpp/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cpp/README.md b/cpp/README.md
index daeeade723019..8018efd9e6d1f 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -35,6 +35,7 @@ On Ubuntu/Debian you can install the requirements with:
 ```shell
 sudo apt-get install cmake \
      libboost-dev \
+     libboost-regex-dev \
      libboost-filesystem-dev \
      libboost-system-dev
 ```

From 907a27d913bbf1d824454b6f2be719f897b61073 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Fri, 9 Mar 2018 15:04:36 -0500
Subject: [PATCH 1588/1644] ARROW-2288: [Python] Fix slicing logic

Author: Antoine Pitrou <antoine@python.org>

Closes #1723 from pitrou/ARROW-2288-py-slicing-logic and squashes the following commits:

0c5461f1 <Antoine Pitrou> ARROW-2288:  Fix slicing logic
---
 python/pyarrow/array.pxi           | 16 +++++++++++++---
 python/pyarrow/tests/test_array.py |  9 +++++----
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index f05806cfac551..321809fa2df72 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -205,15 +205,25 @@ def asarray(values, type=None):
 
 
 def _normalize_slice(object arrow_obj, slice key):
-    cdef Py_ssize_t n = len(arrow_obj)
+    cdef:
+        Py_ssize_t start, stop, step
+        Py_ssize_t n = len(arrow_obj)
 
     start = key.start or 0
-    while start < 0:
+    if start < 0:
         start += n
+        if start < 0:
+            start = 0
+    elif start >= n:
+        start = n
 
     stop = key.stop if key.stop is not None else n
-    while stop < 0:
+    if stop < 0:
         stop += n
+        if stop < 0:
+            stop = 0
+    elif stop >= n:
+        stop = n
 
     step = key.step or 1
     if step != 1:
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 4c14c1c61a762..45b3f9ec5eed4 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -132,17 +132,18 @@ def test_array_slice():
 
     # Test slice notation
     assert arr[2:].equals(arr.slice(2))
-
     assert arr[2:5].equals(arr.slice(2, 3))
-
     assert arr[-5:].equals(arr.slice(len(arr) - 5))
-
     with pytest.raises(IndexError):
         arr[::-1]
-
     with pytest.raises(IndexError):
         arr[::2]
 
+    n = len(arr)
+    for start in range(-n * 2, n * 2):
+        for stop in range(-n * 2, n * 2):
+            assert arr[start:stop].to_pylist() == arr.to_pylist()[start:stop]
+
 
 def test_array_factory_invalid_type():
     arr = np.array([datetime.timedelta(1), datetime.timedelta(2)])

From 2f718d755f8b20add501597048bf1945ee9dca4b Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Fri, 9 Mar 2018 15:27:17 -0500
Subject: [PATCH 1589/1644] =?UTF-8?q?ARROW-2262:=20[Python]=C2=A0Support?=
 =?UTF-8?q?=20slicing=20on=20pyarrow.ChunkedArray?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1702 from xhochy/ARROW-2262 and squashes the following commits:

7446414d <Uwe L. Korn> ARROW-2262:  Support slicing on pyarrow.ChunkedArray
---
 python/pyarrow/includes/libarrow.pxd |  2 ++
 python/pyarrow/table.pxi             | 46 ++++++++++++++++++++++++++++
 python/pyarrow/tests/test_table.py   | 15 +++++++++
 3 files changed, 63 insertions(+)

diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 22c39a865b95f..262230013417b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -388,6 +388,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         int num_chunks()
         shared_ptr[CArray] chunk(int i)
         shared_ptr[CDataType] type()
+        shared_ptr[CChunkedArray] Slice(int64_t offset, int64_t length) const
+        shared_ptr[CChunkedArray] Slice(int64_t offset) const
 
     cdef cppclass CColumn" arrow::Column":
         CColumn(const shared_ptr[CField]& field,
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index c27c0edd9bdf5..94041e465cfce 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -77,6 +77,52 @@ cdef class ChunkedArray:
         self._check_nullptr()
         return self.chunked_array.null_count()
 
+    def __getitem__(self, key):
+        cdef int64_t item
+        cdef int i
+        self._check_nullptr()
+        if isinstance(key, slice):
+            return _normalize_slice(self, key)
+        elif isinstance(key, six.integer_types):
+            item = key
+            if item >= self.chunked_array.length() or item < 0:
+                return IndexError("ChunkedArray selection out of bounds")
+            for i in range(self.num_chunks):
+                if item < self.chunked_array.chunk(i).get().length():
+                    return self.chunk(i)[item]
+                else:
+                    item -= self.chunked_array.chunk(i).get().length()
+        else:
+            raise TypeError("key must either be a slice or integer")
+
+    def slice(self, offset=0, length=None):
+        """
+        Compute zero-copy slice of this ChunkedArray
+
+        Parameters
+        ----------
+        offset : int, default 0
+            Offset from start of array to slice
+        length : int, default None
+            Length of slice (default is until end of batch starting from
+            offset)
+
+        Returns
+        -------
+        sliced : ChunkedArray
+        """
+        cdef shared_ptr[CChunkedArray] result
+
+        if offset < 0:
+            raise IndexError('Offset must be non-negative')
+
+        if length is None:
+            result = self.chunked_array.Slice(offset)
+        else:
+            result = self.chunked_array.Slice(offset, length)
+
+        return pyarrow_wrap_chunked_array(result)
+
     @property
     def num_chunks(self):
         """
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index e72761d32f634..356ecb7e067ce 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -24,6 +24,21 @@
 import pyarrow as pa
 
 
+def test_chunked_array_getitem():
+    data = [
+        pa.array([1, 2, 3]),
+        pa.array([4, 5, 6])
+    ]
+    data = pa.chunked_array(data)
+    assert data[1].as_py() == 2
+
+    data_slice = data[2:4]
+    assert data_slice.to_pylist() == [3, 4]
+
+    data_slice = data[4:-1]
+    assert data_slice.to_pylist() == [5]
+
+
 def test_column_basics():
     data = [
         pa.array([-10, -5, 0, 5, 10])

From d64a2318d8a7113c2aa6362b3476c68cc429679d Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Sat, 10 Mar 2018 13:34:22 -0500
Subject: [PATCH 1590/1644] ARROW-2181: [PYTHON][DOC] Add doc on usage of
 concat_tables

Adding Python API doc on usage of pa.concat_tables.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #1733 from BryanCutler/doc-python-concat_table-ARROW-2181 and squashes the following commits:

2f8d9f80 <Bryan Cutler> use full pyarrow in api ref
95087090 <Bryan Cutler> added doc on usage for concat_tables
---
 python/doc/source/data.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/python/doc/source/data.rst b/python/doc/source/data.rst
index 3a602d54c5c7f..07172604a3523 100644
--- a/python/doc/source/data.rst
+++ b/python/doc/source/data.rst
@@ -393,6 +393,22 @@ objects to contiguous NumPy arrays for use in pandas:
 
    c.to_pandas()
 
+Multiple tables can also be concatenated together to form a single table using
+``pyarrow.concat_tables``, if the schemas are equal:
+
+.. ipython:: python
+
+   tables = [table] * 2
+   table_all = pa.concat_tables(tables)
+   table_all.num_rows
+   c = table_all[0]
+   c.data.num_chunks
+
+This is similar to ``Table.from_batches``, but uses tables as input instead of
+record batches. Record batches can be made into tables, but not the other way
+around, so if your data is already in table form, then use
+``pyarrow.concat_tables``.
+
 Custom Schema and Field Metadata
 --------------------------------
 

From dc45a1a7bcfc916d5b5f98cf40c03fad68d06b9b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sun, 11 Mar 2018 23:41:38 -0400
Subject: [PATCH 1591/1644] ARROW-2099: [Python] Add safe option to
 DictionaryArray.from_arrays to do boundschecking of indices by default

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1734 from wesm/ARROW-2099 and squashes the following commits:

eabc5d19 <Wes McKinney> Add safe option to DictionaryArray.from_arrays to do boundschecking of indices by default
---
 python/pyarrow/array.pxi                    | 14 ++++++++++++--
 python/pyarrow/includes/libarrow.pxd        |  5 +++++
 python/pyarrow/tests/test_array.py          | 21 +++++++++++++++++++++
 python/pyarrow/tests/test_convert_pandas.py |  2 +-
 python/setup.py                             |  9 +++++----
 5 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 321809fa2df72..2ea131bca635d 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -831,7 +831,8 @@ cdef class DictionaryArray(Array):
 
     @staticmethod
     def from_arrays(indices, dictionary, mask=None, ordered=False,
-                    from_pandas=False, MemoryPool memory_pool=None):
+                    from_pandas=False, safe=True,
+                    MemoryPool memory_pool=None):
         """
         Construct Arrow DictionaryArray from array of indices (must be
         non-negative integers) and corresponding array of dictionary values
@@ -847,6 +848,8 @@ cdef class DictionaryArray(Array):
             a pandas.Categorical (null encoded as -1)
         ordered : boolean, default False
             Set to True if the category values are ordered
+        safe : boolean, default True
+            If True, check that the dictionary indices are in range
         memory_pool : MemoryPool, default None
             For memory allocations, if required, otherwise uses default pool
 
@@ -885,7 +888,14 @@ cdef class DictionaryArray(Array):
 
         c_type.reset(new CDictionaryType(_indices.type.sp_type,
                                          _dictionary.sp_array, c_ordered))
-        c_result.reset(new CDictionaryArray(c_type, _indices.sp_array))
+
+        if safe:
+            with nogil:
+                check_status(
+                    CDictionaryArray.FromArrays(c_type, _indices.sp_array,
+                                                &c_result))
+        else:
+            c_result.reset(new CDictionaryArray(c_type, _indices.sp_array))
 
         result = DictionaryArray()
         result.init(c_result)
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 262230013417b..503ee8888743f 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -127,6 +127,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         CDictionaryArray(const shared_ptr[CDataType]& type,
                          const shared_ptr[CArray]& indices)
 
+        @staticmethod
+        CStatus FromArrays(const shared_ptr[CDataType]& type,
+                           const shared_ptr[CArray]& indices,
+                           shared_ptr[CArray]* out)
+
         shared_ptr[CArray] indices()
         shared_ptr[CArray] dictionary()
 
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 45b3f9ec5eed4..69d6a93fe0e94 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -199,6 +199,27 @@ def test_dictionary_from_boxed_arrays():
         assert d1[i].as_py() == dictionary[indices[i]]
 
 
+def test_dictionary_from_arrays_boundscheck():
+    indices1 = pa.array([0, 1, 2, 0, 1, 2])
+    indices2 = pa.array([0, -1, 2])
+    indices3 = pa.array([0, 1, 2, 3])
+
+    dictionary = pa.array(['foo', 'bar', 'baz'])
+
+    # Works fine
+    pa.DictionaryArray.from_arrays(indices1, dictionary)
+
+    with pytest.raises(pa.ArrowException):
+        pa.DictionaryArray.from_arrays(indices2, dictionary)
+
+    with pytest.raises(pa.ArrowException):
+        pa.DictionaryArray.from_arrays(indices3, dictionary)
+
+    # If we are confident that the indices are "safe" we can pass safe=False to
+    # disable the boundschecking
+    pa.DictionaryArray.from_arrays(indices2, dictionary, safe=False)
+
+
 def test_dictionary_with_pandas():
     indices = np.repeat([0, 1, 2], 2)
     dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 333199ab28e54..79291359593c4 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -281,7 +281,7 @@ def test_dictionary_indices_boundscheck(self):
         indices = [[0, 1], [0, -1]]
 
         for inds in indices:
-            arr = pa.DictionaryArray.from_arrays(inds, ['a'])
+            arr = pa.DictionaryArray.from_arrays(inds, ['a'], safe=False)
             batch = pa.RecordBatch.from_arrays([arr], ['foo'])
             table = pa.Table.from_batches([batch, batch, batch])
 
diff --git a/python/setup.py b/python/setup.py
index 6f0b0fa4d4850..453626020fd94 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -111,8 +111,7 @@ def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
-        self.boost_namespace = os.environ.get('PYARROW_BOOST_NAMESPACE',
-                                              'boost')
+        self.boost_namespace = os.environ.get('PYARROW_BOOST_NAMESPACE')
 
         self.cmake_cxxflags = os.environ.get('PYARROW_CXXFLAGS', '')
 
@@ -208,8 +207,10 @@ def _run_cmake(self):
 
             cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                                  .format(self.build_type.lower()))
-            cmake_options.append('-DBoost_NAMESPACE={}'.format(
-                self.boost_namespace))
+
+            if self.boost_namespace is not None:
+                cmake_options.append('-DBoost_NAMESPACE={}'
+                                     .format(self.boost_namespace))
 
             extra_cmake_args = shlex.split(self.extra_cmake_args)
             if sys.platform != 'win32':

From c7b3c05adfab2d099315e55ecb389e961dd537f6 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Mon, 12 Mar 2018 13:19:59 -0400
Subject: [PATCH 1592/1644] ARROW-2297: [JS] babel-jest is not listed as a dev
 dependency

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1737 from TheNeuralBit/babel-jest-dep and squashes the following commits:

f6dea526 <Brian Hulette> Add babel-jest dev dependency
---
 js/package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/js/package.json b/js/package.json
index af3c97f5e3383..9bd9d13994e4e 100644
--- a/js/package.json
+++ b/js/package.json
@@ -67,6 +67,7 @@
     "@types/glob": "5.0.35",
     "@types/jest": "22.1.0",
     "ast-types": "0.10.1",
+    "babel-jest": "22.4.1",
     "benchmark": "2.1.4",
     "coveralls": "3.0.0",
     "del": "3.0.0",

From 8f2ff30664f3a92163801a84bf0be1f4499fcfc3 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 12 Mar 2018 14:00:14 -0400
Subject: [PATCH 1593/1644] ARROW-2240: [Python] Array initialization with
 leading numpy nan fails with exception

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1686 from cpcloud/ARROW-2240 and squashes the following commits:

2da8ed31 <Phillip Cloud> Remove duplicated isnan logic
d2f833fd <Phillip Cloud> Remove redundant test
70837fbd <Phillip Cloud> Formatting
bdc5f962 <Phillip Cloud> Resolve conflicts
0a576267 <Phillip Cloud> Python 2, gross
62018f9b <Phillip Cloud> ARROW-2240:  Array initialization with leading numpy nan fails with exception
---
 cpp/src/arrow/python/builtin_convert.cc | 17 ++++++------
 cpp/src/arrow/python/helpers.cc         | 13 ++++++---
 cpp/src/arrow/python/helpers.h          |  5 +++-
 cpp/src/arrow/python/numpy_to_arrow.cc  | 37 +++++++++++--------------
 cpp/src/arrow/python/python-test.cc     | 15 ++++++++++
 5 files changed, 52 insertions(+), 35 deletions(-)

diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index d2f900f6ae859..595499de79e28 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -88,7 +88,7 @@ class ScalarVisitor {
 
   Status Visit(PyObject* obj) {
     ++total_count_;
-    if (obj == Py_None) {
+    if (obj == Py_None || internal::PyFloat_IsNaN(obj)) {
       ++none_count_;
     } else if (PyBool_Check(obj)) {
       ++bool_count_;
@@ -412,9 +412,10 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
     RETURN_NOT_OK(this->typed_builder_->Reserve(size));
     // Iterate over the items adding each one
     if (PySequence_Check(obj)) {
+      auto self = static_cast<Derived*>(this);
       for (int64_t i = 0; i < size; ++i) {
         OwnedRef ref(PySequence_GetItem(obj, i));
-        RETURN_NOT_OK(static_cast<Derived*>(this)->AppendSingle(ref.obj()));
+        RETURN_NOT_OK(self->AppendSingle(ref.obj()));
       }
     } else {
       return Status::TypeError("Object is not a sequence");
@@ -424,7 +425,8 @@ class TypedConverterVisitor : public TypedConverter<BuilderType> {
 
   // Append a missing item (default implementation)
   Status AppendNull() { return this->typed_builder_->AppendNull(); }
-  bool IsNull(PyObject* obj) const { return obj == Py_None; }
+
+  bool IsNull(PyObject* obj) const { return internal::PandasObjectIsNull(obj); }
 };
 
 class NullConverter : public TypedConverterVisitor<NullBuilder, NullConverter> {
@@ -438,7 +440,9 @@ class NullConverter : public TypedConverterVisitor<NullBuilder, NullConverter> {
 class BoolConverter : public TypedConverterVisitor<BooleanBuilder, BoolConverter> {
  public:
   // Append a non-missing item
-  Status AppendItem(PyObject* obj) { return typed_builder_->Append(obj == Py_True); }
+  Status AppendItem(PyObject* obj) {
+    return typed_builder_->Append(PyObject_IsTrue(obj) == 1);
+  }
 };
 
 class Int8Converter : public TypedConverterVisitor<Int8Builder, Int8Converter> {
@@ -851,11 +855,6 @@ class DecimalConverter
     RETURN_NOT_OK(internal::DecimalFromPythonDecimal(obj, type, &value));
     return typed_builder_->Append(value);
   }
-
-  bool IsNull(PyObject* obj) const {
-    return obj == Py_None || obj == numpy_nan || internal::PyFloat_isnan(obj) ||
-           (internal::PyDecimal_Check(obj) && internal::PyDecimal_ISNAN(obj));
-  }
 };
 
 // Dynamic constructor for sequence converters
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 13dcc46614534..5719af6f3c699 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -226,10 +226,6 @@ Status UInt64FromPythonInt(PyObject* obj, uint64_t* out) {
   return Status::OK();
 }
 
-bool PyFloat_isnan(PyObject* obj) {
-  return PyFloat_Check(obj) && std::isnan(PyFloat_AS_DOUBLE(obj));
-}
-
 bool PyDecimal_Check(PyObject* obj) {
   // TODO(phillipc): Is this expensive?
   OwnedRef Decimal;
@@ -283,6 +279,15 @@ Status DecimalMetadata::Update(PyObject* object) {
   return Update(precision, scale);
 }
 
+bool PyFloat_IsNaN(PyObject* obj) {
+  return PyFloat_Check(obj) && std::isnan(PyFloat_AsDouble(obj));
+}
+
+bool PandasObjectIsNull(PyObject* obj) {
+  return obj == Py_None || obj == numpy_nan || PyFloat_IsNaN(obj) ||
+         (internal::PyDecimal_Check(obj) && internal::PyDecimal_ISNAN(obj));
+}
+
 }  // namespace internal
 }  // namespace py
 }  // namespace arrow
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index 6be0e49b18af3..b9f505a160c6b 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -82,8 +82,11 @@ Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arr
 // \brief Check whether obj is an integer, independent of Python versions.
 bool IsPyInteger(PyObject* obj);
 
+// \brief Use pandas missing value semantics to check if a value is null
+bool PandasObjectIsNull(PyObject* obj);
+
 // \brief Check whether obj is nan
-bool PyFloat_isnan(PyObject* obj);
+bool PyFloat_IsNaN(PyObject* obj);
 
 // \brief Check whether obj is an instance of Decimal
 bool PyDecimal_Check(PyObject* obj);
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 04a71c1f6408f..9e3534d628487 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -67,11 +67,6 @@ constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max();
 
 namespace {
 
-inline bool PandasObjectIsNull(PyObject* obj) {
-  return obj == Py_None || obj == numpy_nan || internal::PyFloat_isnan(obj) ||
-         (internal::PyDecimal_Check(obj) && internal::PyDecimal_ISNAN(obj));
-}
-
 inline bool PyObject_is_string(PyObject* obj) {
 #if PY_MAJOR_VERSION >= 3
   return PyUnicode_Check(obj) || PyBytes_Check(obj);
@@ -161,7 +156,7 @@ static Status AppendObjectBinaries(PyArrayObject* arr, PyArrayObject* mask,
   for (; offset < objects.size(); ++offset) {
     OwnedRef tmp_obj;
     obj = objects[offset];
-    if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) {
+    if ((have_mask && mask_values[offset]) || internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder->AppendNull());
       continue;
     } else if (!PyBytes_Check(obj)) {
@@ -207,7 +202,7 @@ static Status AppendObjectStrings(PyArrayObject* arr, PyArrayObject* mask, int64
   for (; offset < objects.size(); ++offset) {
     OwnedRef tmp_obj;
     obj = objects[offset];
-    if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) {
+    if ((have_mask && mask_values[offset]) || internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder->AppendNull());
       continue;
     } else if (PyUnicode_Check(obj)) {
@@ -256,7 +251,7 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
   for (; offset < objects.size(); ++offset) {
     OwnedRef tmp_obj;
     obj = objects[offset];
-    if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) {
+    if ((have_mask && mask_values[offset]) || internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder->AppendNull());
       continue;
     } else if (PyUnicode_Check(obj)) {
@@ -722,7 +717,7 @@ Status NumPyConverter::ConvertDates() {
   PyObject* obj;
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+    if ((have_mask && mask_values[i]) || internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder.AppendNull());
     } else if (PyDate_CheckExact(obj)) {
       RETURN_NOT_OK(builder.Append(UnboxDate<ArrowType>::Unbox(obj)));
@@ -770,7 +765,7 @@ Status NumPyConverter::ConvertDecimals() {
       RETURN_IF_PYERROR();
     }
 
-    if (PandasObjectIsNull(object)) {
+    if (internal::PandasObjectIsNull(object)) {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       Decimal128 value;
@@ -798,7 +793,7 @@ Status NumPyConverter::ConvertDateTimes() {
     if (PyDateTime_Check(obj)) {
       RETURN_NOT_OK(
           builder.Append(PyDateTime_to_us(reinterpret_cast<PyDateTime_DateTime*>(obj))));
-    } else if (PandasObjectIsNull(obj)) {
+    } else if (internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       std::stringstream ss;
@@ -826,7 +821,7 @@ Status NumPyConverter::ConvertTimes() {
     obj = objects[i];
     if (PyTime_Check(obj)) {
       RETURN_NOT_OK(builder.Append(PyTime_to_us(obj)));
-    } else if (PandasObjectIsNull(obj)) {
+    } else if (internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
       std::stringstream ss;
@@ -895,7 +890,7 @@ Status NumPyConverter::ConvertObjectFloats() {
   PyObject* obj;
   for (int64_t i = 0; i < objects.size(); ++i) {
     obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+    if ((have_mask && mask_values[i]) || internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder.AppendNull());
     } else if (PyFloat_Check(obj)) {
       double val = PyFloat_AsDouble(obj);
@@ -930,7 +925,7 @@ Status NumPyConverter::ConvertObjectIntegers() {
   PyObject* obj;
   for (int64_t i = 0; i < objects.size(); ++i) {
     obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+    if ((have_mask && mask_values[i]) || internal::PandasObjectIsNull(obj)) {
       RETURN_NOT_OK(builder.AppendNull());
     } else if (PyObject_is_integer(obj)) {
       const int64_t val = static_cast<int64_t>(PyLong_AsLong(obj));
@@ -999,7 +994,7 @@ Status NumPyConverter::ConvertBooleans() {
   PyObject* obj;
   for (int64_t i = 0; i < length_; ++i) {
     obj = objects[i];
-    if ((have_mask && mask_values[i]) || PandasObjectIsNull(obj)) {
+    if ((have_mask && mask_values[i]) || internal::PandasObjectIsNull(obj)) {
       ++null_count;
     } else if (obj == Py_True) {
       BitUtil::SetBit(bitmap, i);
@@ -1028,7 +1023,7 @@ Status NumPyConverter::ConvertObjectsInfer() {
 
   for (int64_t i = 0; i < length_; ++i) {
     PyObject* obj = objects[i];
-    if (PandasObjectIsNull(obj)) {
+    if (internal::PandasObjectIsNull(obj)) {
       continue;
     } else if (PyObject_is_string(obj)) {
       return ConvertObjectStrings();
@@ -1221,7 +1216,7 @@ inline Status NumPyConverter::ConvertTypedLists(const std::shared_ptr<DataType>&
   BuilderT* value_builder = static_cast<BuilderT*>(builder->value_builder());
 
   auto foreach_item = [&](PyObject* object, bool mask) {
-    if (mask || PandasObjectIsNull(object)) {
+    if (mask || internal::PandasObjectIsNull(object)) {
       return builder->AppendNull();
     } else if (PyArray_Check(object)) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
@@ -1266,7 +1261,7 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
   auto value_builder = static_cast<NullBuilder*>(builder->value_builder());
 
   auto foreach_item = [&](PyObject* object, bool mask) {
-    if (mask || PandasObjectIsNull(object)) {
+    if (mask || internal::PandasObjectIsNull(object)) {
       return builder->AppendNull();
     } else if (PyArray_Check(object)) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
@@ -1312,7 +1307,7 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, BinaryType>(
   auto value_builder = static_cast<BinaryBuilder*>(builder->value_builder());
 
   auto foreach_item = [&](PyObject* object, bool mask) {
-    if (mask || PandasObjectIsNull(object)) {
+    if (mask || internal::PandasObjectIsNull(object)) {
       return builder->AppendNull();
     } else if (PyArray_Check(object)) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
@@ -1365,7 +1360,7 @@ inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
   auto value_builder = static_cast<StringBuilder*>(builder->value_builder());
 
   auto foreach_item = [&](PyObject* object, bool mask) {
-    if (mask || PandasObjectIsNull(object)) {
+    if (mask || internal::PandasObjectIsNull(object)) {
       return builder->AppendNull();
     } else if (PyArray_Check(object)) {
       auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
@@ -1428,7 +1423,7 @@ Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type,
       auto value_builder = static_cast<ListBuilder*>(builder->value_builder());
 
       auto foreach_item = [this, &builder, &value_builder, &list_type](PyObject* object) {
-        if (PandasObjectIsNull(object)) {
+        if (internal::PandasObjectIsNull(object)) {
           return builder->AppendNull();
         } else {
           RETURN_NOT_OK(builder->Append(true));
diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc
index 16ac1e332b10e..c18b1598ff63d 100644
--- a/cpp/src/arrow/python/python-test.cc
+++ b/cpp/src/arrow/python/python-test.cc
@@ -367,5 +367,20 @@ TEST_F(DecimalTest, UpdateWithNaN) {
   ASSERT_EQ(std::numeric_limits<int32_t>::min(), metadata.scale());
 }
 
+TEST(PythonTest, ConstructStringArrayWithLeadingZeros) {
+  PyAcquireGIL lock;
+
+  OwnedRef list_ref(PyList_New(2));
+  PyObject* list = list_ref.obj();
+  std::string str("str");
+
+  ASSERT_EQ(0, PyList_SetItem(list, 0, PyFloat_FromDouble(NAN)));
+  ASSERT_EQ(0, PyList_SetItem(list, 1, PyUnicode_FromString(str.c_str())));
+
+  std::shared_ptr<Array> out;
+  auto pool = default_memory_pool();
+  ASSERT_OK(ConvertPySequence(list, pool, &out));
+}
+
 }  // namespace py
 }  // namespace arrow

From 3917e85d4ac0c9144e6112b627e44641da6a9b86 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 12 Mar 2018 14:10:29 -0400
Subject: [PATCH 1594/1644] ARROW-2292: [Python] Rename frombuffer() to
 py_buffer()

Author: Antoine Pitrou <antoine@python.org>

Closes #1736 from pitrou/ARROW-2292-py_buffer and squashes the following commits:

41bf5f8f <Antoine Pitrou> ARROW-2292:  Rename frombuffer() to py_buffer()
---
 python/doc/source/api.rst                  |  2 +-
 python/doc/source/ipc.rst                  |  5 +--
 python/doc/source/memory.rst               |  2 +-
 python/pyarrow/__init__.py                 | 11 ++++---
 python/pyarrow/io.pxi                      | 11 ++++---
 python/pyarrow/serialization.py            |  4 +--
 python/pyarrow/tests/test_io.py            | 38 +++++++++++-----------
 python/pyarrow/tests/test_serialization.py |  6 ++--
 python/pyarrow/util.py                     |  8 ++---
 9 files changed, 46 insertions(+), 41 deletions(-)

diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 3db1a04b6d206..cb9993302a7a5 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -213,7 +213,7 @@ Input / Output and Shared Memory
    allocate_buffer
    compress
    decompress
-   frombuffer
+   py_buffer
    foreign_buffer
    Buffer
    ResizableBuffer
diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst
index bce8b1ed1e951..c77888ab906bb 100644
--- a/python/doc/source/ipc.rst
+++ b/python/doc/source/ipc.rst
@@ -296,12 +296,13 @@ which are zero-copy convertible to Python ``memoryview`` objects:
 
    memoryview(components['data'][0])
 
-A memoryview can be converted back to a ``Buffer`` with ``pyarrow.frombuffer``:
+A memoryview can be converted back to a Arrow ``Buffer`` with
+``pyarrow.py_buffer``:
 
 .. ipython:: python
 
    mv = memoryview(components['data'][0])
-   buf = pa.frombuffer(mv)
+   buf = pa.py_buffer(mv)
 
 An object can be reconstructed from its component-based representation using
 ``deserialize_components``:
diff --git a/python/doc/source/memory.rst b/python/doc/source/memory.rst
index 4806bbb85722e..34664b898f9ec 100644
--- a/python/doc/source/memory.rst
+++ b/python/doc/source/memory.rst
@@ -50,7 +50,7 @@ implements the buffer protocol. Let's consider a bytes object:
    import pyarrow as pa
 
    data = b'abcdefghijklmnopqrstuvwxyz'
-   buf = pa.frombuffer(data)
+   buf = pa.py_buffer(data)
    buf
    buf.size
 
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 965a37b4b9cf8..bfd7d4db98659 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -28,7 +28,8 @@
         def parse_version(root):
             from setuptools_scm import version_from_scm
             import setuptools_scm.git
-            describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --match 'apache-arrow-[0-9]*'"
+            describe = (setuptools_scm.git.DEFAULT_DESCRIBE +
+                        " --match 'apache-arrow-[0-9]*'")
             # Strip catchall from the commandline
             describe = describe.replace("--match *.*", "")
             version = setuptools_scm.git.parse(root, describe)
@@ -86,8 +87,8 @@ def parse_version(root):
 from pyarrow.lib import TimestampType
 
 # Buffers, allocation
-from pyarrow.lib import (Buffer, ResizableBuffer, foreign_buffer, compress,
-                         decompress, allocate_buffer, frombuffer)
+from pyarrow.lib import (Buffer, ResizableBuffer, foreign_buffer, py_buffer,
+                         compress, decompress, allocate_buffer)
 
 from pyarrow.lib import (MemoryPool, total_allocated_bytes,
                          set_memory_pool, default_memory_pool,
@@ -163,7 +164,9 @@ def _plasma_store_entry_point():
 # ----------------------------------------------------------------------
 # Deprecations
 
-from pyarrow.util import _deprecate_class  # noqa
+from pyarrow.util import _deprecate_api  # noqa
+
+frombuffer = _deprecate_api('frombuffer', 'py_buffer', py_buffer, '0.9.0')
 
 # ----------------------------------------------------------------------
 # Returning absolute path to the pyarrow include directory (if bundled, e.g. in
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 15ecd0164e439..3947323233f8d 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -26,6 +26,7 @@ import six
 import sys
 import threading
 import time
+import warnings
 
 
 # 64K
@@ -211,7 +212,7 @@ cdef class NativeFile:
         if isinstance(data, six.string_types):
             data = tobytes(data)
 
-        cdef Buffer arrow_buffer = frombuffer(data)
+        cdef Buffer arrow_buffer = py_buffer(data)
 
         cdef const uint8_t* buf = arrow_buffer.buffer.get().data()
         cdef int64_t bufsize = len(arrow_buffer)
@@ -833,14 +834,14 @@ cdef class BufferReader(NativeFile):
         if isinstance(obj, Buffer):
             self.buffer = obj
         else:
-            self.buffer = frombuffer(obj)
+            self.buffer = py_buffer(obj)
 
         self.rd_file.reset(new CBufferReader(self.buffer.buffer))
         self.is_readable = True
         self.closed = False
 
 
-def frombuffer(object obj):
+def py_buffer(object obj):
     """
     Construct an Arrow buffer from a Python bytes object
     """
@@ -966,7 +967,7 @@ def compress(object buf, codec='lz4', asbytes=False, memory_pool=None):
         check_status(CCodec.Create(c_codec, &compressor))
 
     if not isinstance(buf, Buffer):
-        buf = frombuffer(buf)
+        buf = py_buffer(buf)
 
     c_buf = (<Buffer> buf).buffer.get()
 
@@ -1031,7 +1032,7 @@ def decompress(object buf, decompressed_size=None, codec='lz4',
         check_status(CCodec.Create(c_codec, &compressor))
 
     if not isinstance(buf, Buffer):
-        buf = frombuffer(buf)
+        buf = py_buffer(buf)
 
     c_buf = (<Buffer> buf).buffer.get()
 
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index bdf7535796833..6c8df350bf46d 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -23,7 +23,7 @@
 
 from pyarrow.compat import builtin_pickle
 from pyarrow.lib import (SerializationContext, _default_serialization_context,
-                         frombuffer)
+                         py_buffer)
 
 try:
     import cloudpickle
@@ -46,7 +46,7 @@ def _deserialize_numpy_array_list(data):
 
 def _pickle_to_buffer(x):
     pickled = builtin_pickle.dumps(x, protocol=builtin_pickle.HIGHEST_PROTOCOL)
-    return frombuffer(pickled)
+    return py_buffer(pickled)
 
 
 def _load_pickle_from_buffer(data):
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index fe680133b4b32..591381085c2a8 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -170,7 +170,7 @@ def test_python_file_closing():
 def test_buffer_bytes():
     val = b'some data'
 
-    buf = pa.frombuffer(val)
+    buf = pa.py_buffer(val)
     assert isinstance(buf, pa.Buffer)
     assert not buf.is_mutable
 
@@ -182,7 +182,7 @@ def test_buffer_bytes():
 def test_buffer_memoryview():
     val = b'some data'
 
-    buf = pa.frombuffer(val)
+    buf = pa.py_buffer(val)
     assert isinstance(buf, pa.Buffer)
     assert not buf.is_mutable
 
@@ -194,7 +194,7 @@ def test_buffer_memoryview():
 def test_buffer_bytearray():
     val = bytearray(b'some data')
 
-    buf = pa.frombuffer(val)
+    buf = pa.py_buffer(val)
     assert isinstance(buf, pa.Buffer)
     assert buf.is_mutable
 
@@ -206,14 +206,14 @@ def test_buffer_bytearray():
 def test_buffer_invalid():
     with pytest.raises(TypeError,
                        match="(bytes-like object|buffer interface)"):
-        pa.frombuffer(None)
+        pa.py_buffer(None)
 
 
 def test_buffer_to_numpy():
     # Make sure creating a numpy array from an arrow buffer works
     byte_array = bytearray(20)
     byte_array[0] = 42
-    buf = pa.frombuffer(byte_array)
+    buf = pa.py_buffer(byte_array)
     array = np.frombuffer(buf, dtype="uint8")
     assert array[0] == byte_array[0]
     byte_array[0] += 1
@@ -224,14 +224,14 @@ def test_buffer_to_numpy():
 def test_buffer_from_numpy():
     # C-contiguous
     arr = np.arange(12, dtype=np.int8).reshape((3, 4))
-    buf = pa.frombuffer(arr)
+    buf = pa.py_buffer(arr)
     assert buf.to_pybytes() == arr.tobytes()
     # F-contiguous; note strides informations is lost
-    buf = pa.frombuffer(arr.T)
+    buf = pa.py_buffer(arr.T)
     assert buf.to_pybytes() == arr.tobytes()
     # Non-contiguous
     with pytest.raises(ValueError, match="not contiguous"):
-        buf = pa.frombuffer(arr.T[::2])
+        buf = pa.py_buffer(arr.T[::2])
 
 
 def test_buffer_equals():
@@ -250,11 +250,11 @@ def ne(a, b):
     b2 = bytearray(b1)
     b3 = bytearray(b1)
     b3[0] = 42
-    buf1 = pa.frombuffer(b1)
-    buf2 = pa.frombuffer(b2)
-    buf3 = pa.frombuffer(b2)
-    buf4 = pa.frombuffer(b3)
-    buf5 = pa.frombuffer(np.frombuffer(b2, dtype=np.int16))
+    buf1 = pa.py_buffer(b1)
+    buf2 = pa.py_buffer(b2)
+    buf3 = pa.py_buffer(b2)
+    buf4 = pa.py_buffer(b3)
+    buf5 = pa.py_buffer(np.frombuffer(b2, dtype=np.int16))
     eq(buf1, buf1)
     eq(buf1, buf2)
     eq(buf2, buf3)
@@ -266,7 +266,7 @@ def ne(a, b):
 def test_buffer_hashing():
     # Buffers are unhashable
     with pytest.raises(TypeError, match="unhashable"):
-        hash(pa.frombuffer(b'123'))
+        hash(pa.py_buffer(b'123'))
 
 
 def test_foreign_buffer():
@@ -307,7 +307,7 @@ def test_compress_decompress():
     test_data = (np.random.randint(0, 255, size=INPUT_SIZE)
                  .astype(np.uint8)
                  .tostring())
-    test_buf = pa.frombuffer(test_data)
+    test_buf = pa.py_buffer(test_data)
 
     codecs = ['lz4', 'snappy', 'gzip', 'zstd', 'brotli']
     for codec in codecs:
@@ -333,7 +333,7 @@ def test_compress_decompress():
 def test_buffer_memoryview_is_immutable():
     val = b'some data'
 
-    buf = pa.frombuffer(val)
+    buf = pa.py_buffer(val)
     assert not buf.is_mutable
     assert isinstance(buf, pa.Buffer)
 
@@ -368,9 +368,9 @@ def test_uninitialized_buffer():
     with check_uninitialized():
         memoryview(buf)
     with check_uninitialized():
-        buf.equals(pa.frombuffer(b''))
+        buf.equals(pa.py_buffer(b''))
     with check_uninitialized():
-        pa.frombuffer(b'').equals(buf)
+        pa.py_buffer(b'').equals(buf)
 
 
 def test_memory_output_stream():
@@ -400,7 +400,7 @@ def test_inmemory_write_after_closed():
 
 def test_buffer_protocol_ref_counting():
     def make_buffer(bytes_obj):
-        return bytearray(pa.frombuffer(bytes_obj))
+        return bytearray(pa.py_buffer(bytes_obj))
 
     buf = make_buffer(b'foo')
     gc.collect()
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 64aab06712446..7ddf3958e5d8c 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -445,7 +445,7 @@ class BufferClass(object):
         pass
 
     def serialize_buffer_class(obj):
-        return pa.frombuffer(b"hello")
+        return pa.py_buffer(b"hello")
 
     def deserialize_buffer_class(serialized_obj):
         return serialized_obj
@@ -581,7 +581,7 @@ def test_serialize_subclasses():
 
 
 def test_serialize_to_components_invalid_cases():
-    buf = pa.frombuffer(b'hello')
+    buf = pa.py_buffer(b'hello')
 
     components = {
         'num_tensors': 0,
@@ -631,7 +631,7 @@ def test_deserialize_buffer_in_different_process():
     import subprocess
 
     f = tempfile.NamedTemporaryFile(delete=False)
-    b = pa.serialize(pa.frombuffer(b'hello')).to_buffer()
+    b = pa.serialize(pa.py_buffer(b'hello')).to_buffer()
     f.write(b.to_pybytes())
     f.close()
 
diff --git a/python/pyarrow/util.py b/python/pyarrow/util.py
index d984e19215b49..b8825658d88e9 100644
--- a/python/pyarrow/util.py
+++ b/python/pyarrow/util.py
@@ -27,11 +27,11 @@ def decorator(g):
     return decorator
 
 
-def _deprecate_class(old_name, new_name, klass, next_version='0.5.0'):
+def _deprecate_api(old_name, new_name, api, next_version):
     msg = ('pyarrow.{0} is deprecated as of {1}, please use {2} instead'
            .format(old_name, next_version, new_name))
 
-    def deprecated_factory(*args, **kwargs):
+    def wrapper(*args, **kwargs):
         warnings.warn(msg, FutureWarning)
-        return klass(*args)
-    return deprecated_factory
+        return api(*args)
+    return wrapper

From 58fa873f70907a44ed8c40ae3481f6a0bbf2a669 Mon Sep 17 00:00:00 2001
From: "Uwe L. Korn" <uwelk@xhochy.com>
Date: Mon, 12 Mar 2018 14:20:30 -0400
Subject: [PATCH 1595/1644] ARROW-2282: [Python] Create StringArray from
 buffers

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #1720 from xhochy/ARROW-2282 and squashes the following commits:

36dc6b86 <Uwe L. Korn> Check computed null_count
5bb257de <Uwe L. Korn> ARROW-2282:  Create StringArray from buffers
---
 python/pyarrow/array.pxi             | 35 +++++++++++++++++++++++++++-
 python/pyarrow/includes/libarrow.pxd |  6 +++++
 python/pyarrow/tests/test_array.py   | 30 ++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 2ea131bca635d..8dac57d18cced 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -788,8 +788,41 @@ cdef class UnionArray(Array):
         return pyarrow_wrap_array(out)
 
 cdef class StringArray(Array):
-    pass
 
+    @staticmethod
+    def from_buffers(int length, Buffer value_offsets, Buffer data,
+                     Buffer null_bitmap=None, int null_count=-1,
+                     int offset=0):
+        """
+        Construct a StringArray from value_offsets and data buffers.
+        If there are nulls in the data, also a null_bitmap and the matching
+        null_count must be passed.
+
+        Parameters
+        ----------
+        length : int
+        value_offsets : Buffer
+        data : Buffer
+        null_bitmap : Buffer, optional
+        null_count : int, default 0
+        offset : int, default 0
+
+        Returns
+        -------
+        string_array : StringArray
+        """
+        cdef shared_ptr[CBuffer] c_null_bitmap
+        cdef shared_ptr[CArray] out
+
+        if null_bitmap is not None:
+            c_null_bitmap = null_bitmap.buffer
+        else:
+            null_count = 0
+
+        out.reset(new CStringArray(
+            length, value_offsets.buffer, data.buffer, c_null_bitmap,
+            null_count, offset))
+        return pyarrow_wrap_array(out)
 
 cdef class BinaryArray(Array):
     pass
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 503ee8888743f..3d0c02b895859 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -372,6 +372,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         const uint8_t* GetValue(int i, int32_t* length)
 
     cdef cppclass CStringArray" arrow::StringArray"(CBinaryArray):
+        CStringArray(int64_t length, shared_ptr[CBuffer] value_offsets,
+                     shared_ptr[CBuffer] data,
+                     shared_ptr[CBuffer] null_bitmap,
+                     int64_t null_count,
+                     int64_t offset)
+
         c_string GetString(int i)
 
     cdef cppclass CStructArray" arrow::StructArray"(CArray):
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 69d6a93fe0e94..fea56862cd38d 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -290,6 +290,36 @@ def test_union_from_sparse():
     assert result.to_pylist() == [b'a', 1, b'b', b'c', 2, 3, b'd']
 
 
+def test_string_from_buffers():
+    array = pa.array(["a", None, "b", "c"])
+
+    buffers = array.buffers()
+    copied = pa.StringArray.from_buffers(
+        len(array), buffers[1], buffers[2], buffers[0], array.null_count,
+        array.offset)
+    assert copied.to_pylist() == ["a", None, "b", "c"]
+
+    copied = pa.StringArray.from_buffers(
+        len(array), buffers[1], buffers[2], buffers[0])
+    assert copied.to_pylist() == ["a", None, "b", "c"]
+
+    sliced = array[1:]
+    buffers = sliced.buffers()
+    copied = pa.StringArray.from_buffers(
+        len(sliced), buffers[1], buffers[2], buffers[0], -1, sliced.offset)
+    assert copied.to_pylist() == [None, "b", "c"]
+    assert copied.null_count == 1
+
+    # Slice but exclude all null entries so that we don't need to pass
+    # the null bitmap.
+    sliced = array[2:]
+    buffers = sliced.buffers()
+    copied = pa.StringArray.from_buffers(
+        len(sliced), buffers[1], buffers[2], None, -1, sliced.offset)
+    assert copied.to_pylist() == ["b", "c"]
+    assert copied.null_count == 0
+
+
 def _check_cast_case(case, safe=True):
     in_data, in_type, out_data, out_type = case
 

From 317b54318f468a7af1f7b08d57826ef5a3e7a151 Mon Sep 17 00:00:00 2001
From: Brian Hulette <brian.hulette@ccri.com>
Date: Mon, 12 Mar 2018 14:22:19 -0400
Subject: [PATCH 1596/1644] ARROW-2293: [JS] Print release vote e-mail template
 when making source release

Author: Brian Hulette <brian.hulette@ccri.com>

Closes #1738 from TheNeuralBit/js-email-template and squashes the following commits:

3c0fc205 <Brian Hulette> Default to 72 hour vote window
6ce42160 <Brian Hulette> Add email template to JS release script
---
 dev/release/js-source-release.sh | 118 ++++++++++++++++++++++++++++---
 1 file changed, 109 insertions(+), 9 deletions(-)

diff --git a/dev/release/js-source-release.sh b/dev/release/js-source-release.sh
index 53b31af62f2af..292869db69d09 100755
--- a/dev/release/js-source-release.sh
+++ b/dev/release/js-source-release.sh
@@ -21,9 +21,38 @@ set -e
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+function print_help_and_exit {
+cat <<EOF
+Apache Arrow JS release candidate tool.
+
+Usage: $0 [-h] [-p] <js-version> <rc-num>"
+
+  -h  Print this help message and exit
+  -p  If present, publish the release candidate (default: does not publish anything)
+EOF
+exit 0
+}
+
+publish=0
+while getopts ":hp" opt; do
+  case $opt in
+    p)
+      publish=1
+      ;;
+    h)
+      print_help_and_exit
+      ;;
+    *  )
+      echo "Unknown option: -$OPTARG"
+      print_help_and_exit
+      ;;
+  esac
+done
+
+shift $(($OPTIND - 1))
+
 if [ "$#" -ne 2 ]; then
-  echo "Usage: $0 <js-version> <rc-num>"
-  exit
+  print_help_and_exit
 fi
 
 js_version=$1
@@ -32,6 +61,17 @@ rc=$2
 tag=apache-arrow-js-${js_version}
 tagrc=${tag}-rc${rc}
 
+# Reset instructions
+current_git_rev=$(git rev-parse HEAD)
+function print_reset_instructions {
+cat <<EOF
+To roll back your local repo you will need to run:
+
+  git reset --hard ${current_git_rev}
+  git tag -d ${tag}
+EOF
+}
+
 echo "Preparing source for tag ${tag}"
 
 tarball=${tag}.tar.gz
@@ -82,14 +122,16 @@ sha1sum $tarball > ${tarball}.sha1
 sha256sum $tarball > ${tarball}.sha256
 sha512sum $tarball > ${tarball}.sha512
 
-# check out the arrow RC folder
-svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow js-rc-tmp
+if [[ $publish == 1 ]]; then
+  # check out the arrow RC folder
+  svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow js-rc-tmp
 
-# add the release candidate for the tag
-mkdir -p js-rc-tmp/${tagrc}
-cp ${tarball}* js-rc-tmp/${tagrc}
-svn add js-rc-tmp/${tagrc}
-svn ci -m 'Apache Arrow JavaScript ${version} RC${rc}' js-rc-tmp/${tagrc}
+  # add the release candidate for the tag
+  mkdir -p js-rc-tmp/${tagrc}
+  cp ${tarball}* js-rc-tmp/${tagrc}
+  svn add js-rc-tmp/${tagrc}
+  svn ci -m 'Apache Arrow JavaScript ${version} RC${rc}' js-rc-tmp/${tagrc}
+fi
 
 cd -
 
@@ -100,3 +142,61 @@ echo "Success! The release candidate is available here:"
 echo "  https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
 echo ""
 echo "Commit SHA1: ${release_hash}"
+echo ""
+echo "The following draft email has been created to send to the "
+echo "dev@arrow.apache.org mailing list"
+echo ""
+
+# Create the email template for the release candidate to be sent to the mailing lists.
+MESSAGE=$(cat <<__EOF__
+To: dev@arrow.apache.org
+Subject: [VOTE] Release Apache Arrow JS ${js_version} - RC${rc}
+
+Hello all,
+
+I\'d like to propose the following release candidate (rc${rc}) of Apache Arrow
+JavaScript version ${js_version}.
+
+The source release rc{$rc} is hosted at [1].
+
+This release candidate is based on commit
+${release_hash}
+
+Please download, verify checksums and signatures, run the unit tests, and vote
+on the release. The easiest way is to use the JavaScript-specific release
+verification script dev/release/js-verify-release-candidate.sh.
+
+The vote will be open for at least 72 hours and will close once
+enough PMCs have approved the release.
+
+[ ] +1 Release this as Apache Arrow JavaScript ${js_version}
+[ ] +0
+[ ] -1 Do not release this as Apache Arrow JavaScript ${js_version} because...
+
+
+How to validate a release signature:
+https://httpd.apache.org/dev/verification.html
+
+[1]: https://dist.apache.org/repos/dist/dev/arrow/${tagrc}/
+[2]: https://github.com/apache/arrow/tree/${release_hash}
+
+__EOF__
+)
+
+
+echo "--------------------------------------------------------------------------------"
+echo
+echo "${MESSAGE}"
+echo
+echo "--------------------------------------------------------------------------------"
+echo
+
+
+# Print reset instructions if this was a dry-run
+if [[ $publish == 0 ]]; then
+  echo
+  echo "This was a dry run, nothing has been published."
+  echo "To publish, re-run this script with the -p flag."
+  echo
+  print_reset_instructions
+fi

From 6fc9922a66017ab24f1a1676b83e4aaed0dd1930 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Mon, 12 Mar 2018 14:41:14 -0400
Subject: [PATCH 1597/1644] ARROW-2118: [C++] Fix misleading error when memory
 mapping a zero-length file

This was causing an unintuitive error to bubble up when reading a malformed, zero-length Parquet file. To improve the actual Parquet error message, we must complete PARQUET-1243

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1735 from wesm/ARROW-2118 and squashes the following commits:

dd5dcb06 <Wes McKinney> Use std::strerror to make errno more helpful
7717689d <Wes McKinney> Add Python unit test
10a368fc <Wes McKinney> Do not return error when memory-mapping length 0 file
---
 cpp/src/arrow/io/file.cc         | 20 +++++++++++++-------
 cpp/src/arrow/io/io-file-test.cc | 10 ++++++++++
 python/pyarrow/tests/test_io.py  |  8 ++++++++
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index d44d90cbe359e..02cc4dbbd661d 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -624,16 +624,22 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer {
       is_mutable_ = false;
     }
 
-    void* result = mmap(nullptr, static_cast<size_t>(file_->size()), prot_flags, map_mode,
-                        file_->fd(), 0);
-    if (result == MAP_FAILED) {
-      std::stringstream ss;
-      ss << "Memory mapping file failed, errno: " << errno;
-      return Status::IOError(ss.str());
+    size_ = file_->size();
+
+    void* result = nullptr;
+
+    // Memory mapping fails when file size is 0
+    if (size_ > 0) {
+      result =
+          mmap(nullptr, static_cast<size_t>(size_), prot_flags, map_mode, file_->fd(), 0);
+      if (result == MAP_FAILED) {
+        std::stringstream ss;
+        ss << "Memory mapping file failed: " << std::strerror(errno);
+        return Status::IOError(ss.str());
+      }
     }
 
     data_ = mutable_data_ = reinterpret_cast<uint8_t*>(result);
-    size_ = file_->size();
 
     position_ = 0;
 
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 2a4acab59db51..53218ca857dff 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -467,6 +467,16 @@ class TestMemoryMappedFile : public ::testing::Test, public MemoryMapFixture {
 
 TEST_F(TestMemoryMappedFile, InvalidUsages) {}
 
+TEST_F(TestMemoryMappedFile, ZeroSizeFlie) {
+  std::string path = "io-memory-map-zero-size";
+  std::shared_ptr<MemoryMappedFile> result;
+  ASSERT_OK(InitMemoryMap(0, path, &result));
+
+  int64_t size = 0;
+  ASSERT_OK(result->Tell(&size));
+  ASSERT_EQ(0, size);
+}
+
 TEST_F(TestMemoryMappedFile, WriteRead) {
   const int64_t buffer_size = 1024;
   std::vector<uint8_t> buffer(buffer_size);
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 591381085c2a8..b29b9f1f6ec51 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -592,6 +592,14 @@ def test_memory_map_writer(tmpdir):
     assert f.read(3) == b'foo'
 
 
+def test_memory_zero_length(tmpdir):
+    path = os.path.join(str(tmpdir), guid())
+    f = open(path, 'wb')
+    f.close()
+    with pa.memory_map(path, mode='r+b') as memory_map:
+        assert memory_map.size() == 0
+
+
 def test_os_file_writer(tmpdir):
     SIZE = 4096
     arr = np.random.randint(0, 256, size=SIZE).astype('u1')

From 171340fdb05fc1d281f50c2c49b272ab1b908f6a Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 12 Mar 2018 15:03:14 -0400
Subject: [PATCH 1598/1644] ARROW-2135: [Python] Fix NaN conversion when
 casting from Numpy array

Author: Antoine Pitrou <antoine@python.org>

Closes #1681 from pitrou/ARROW-2135-nan-conversion-when-casting and squashes the following commits:

939428dd <Antoine Pitrou> ARROW-2135:  Fix NaN conversion when casting from Numpy array
---
 cpp/src/arrow/python/numpy-internal.h       |  54 +++++++-
 cpp/src/arrow/python/numpy_interop.h        |  25 ++++
 cpp/src/arrow/python/numpy_to_arrow.cc      | 142 +++++++++++++-------
 cpp/src/arrow/python/type_traits.h          |  40 +++---
 python/pyarrow/tests/test_convert_pandas.py |  19 +++
 5 files changed, 214 insertions(+), 66 deletions(-)

diff --git a/cpp/src/arrow/python/numpy-internal.h b/cpp/src/arrow/python/numpy-internal.h
index 8d4308065c2df..7672861d44714 100644
--- a/cpp/src/arrow/python/numpy-internal.h
+++ b/cpp/src/arrow/python/numpy-internal.h
@@ -68,6 +68,9 @@ class Ndarray1DIndexer {
   int64_t stride_;
 };
 
+// Handling of Numpy Types by their static numbers
+// (the NPY_TYPES enum and related defines)
+
 static inline std::string GetNumPyTypeName(int npy_type) {
 #define TYPE_CASE(TYPE, NAME) \
   case NPY_##TYPE:            \
@@ -79,14 +82,20 @@ static inline std::string GetNumPyTypeName(int npy_type) {
     TYPE_CASE(INT16, "int16")
     TYPE_CASE(INT32, "int32")
     TYPE_CASE(INT64, "int64")
-#if (NPY_INT64 != NPY_LONGLONG)
+#if !NPY_INT32_IS_INT
+    TYPE_CASE(INT, "intc")
+#endif
+#if !NPY_INT64_IS_LONG_LONG
     TYPE_CASE(LONGLONG, "longlong")
 #endif
     TYPE_CASE(UINT8, "uint8")
     TYPE_CASE(UINT16, "uint16")
     TYPE_CASE(UINT32, "uint32")
     TYPE_CASE(UINT64, "uint64")
-#if (NPY_UINT64 != NPY_ULONGLONG)
+#if !NPY_INT32_IS_INT
+    TYPE_CASE(UINT, "uintc")
+#endif
+#if !NPY_INT64_IS_LONG_LONG
     TYPE_CASE(ULONGLONG, "ulonglong")
 #endif
     TYPE_CASE(FLOAT16, "float16")
@@ -100,9 +109,48 @@ static inline std::string GetNumPyTypeName(int npy_type) {
   }
 
 #undef TYPE_CASE
-  return "unrecognized type in GetNumPyTypeName";
+  std::stringstream ss;
+  ss << "unrecognized type (" << npy_type << ") in GetNumPyTypeName";
+  return ss.str();
 }
 
+#define TYPE_VISIT_INLINE(TYPE) \
+  case NPY_##TYPE:              \
+    return visitor->template Visit<NPY_##TYPE>(arr);
+
+template <typename VISITOR>
+inline Status VisitNumpyArrayInline(PyArrayObject* arr, VISITOR* visitor) {
+  switch (PyArray_TYPE(arr)) {
+    TYPE_VISIT_INLINE(BOOL);
+    TYPE_VISIT_INLINE(INT8);
+    TYPE_VISIT_INLINE(UINT8);
+    TYPE_VISIT_INLINE(INT16);
+    TYPE_VISIT_INLINE(UINT16);
+    TYPE_VISIT_INLINE(INT32);
+    TYPE_VISIT_INLINE(UINT32);
+    TYPE_VISIT_INLINE(INT64);
+    TYPE_VISIT_INLINE(UINT64);
+#if !NPY_INT32_IS_INT
+    TYPE_VISIT_INLINE(INT);
+    TYPE_VISIT_INLINE(UINT);
+#endif
+#if !NPY_INT64_IS_LONG_LONG
+    TYPE_VISIT_INLINE(LONGLONG);
+    TYPE_VISIT_INLINE(ULONGLONG);
+#endif
+    TYPE_VISIT_INLINE(FLOAT16);
+    TYPE_VISIT_INLINE(FLOAT32);
+    TYPE_VISIT_INLINE(FLOAT64);
+    TYPE_VISIT_INLINE(DATETIME);
+    TYPE_VISIT_INLINE(OBJECT);
+  }
+  std::stringstream ss;
+  ss << "NumPy type not implemented: " << GetNumPyTypeName(PyArray_TYPE(arr));
+  return Status::NotImplemented(ss.str());
+}
+
+#undef TYPE_VISIT_INLINE
+
 }  // namespace py
 }  // namespace arrow
 
diff --git a/cpp/src/arrow/python/numpy_interop.h b/cpp/src/arrow/python/numpy_interop.h
index 8c569e232c121..0715c66c50da0 100644
--- a/cpp/src/arrow/python/numpy_interop.h
+++ b/cpp/src/arrow/python/numpy_interop.h
@@ -43,6 +43,31 @@
 #include <numpy/arrayscalars.h>
 #include <numpy/ufuncobject.h>
 
+// A bit subtle. Numpy has 5 canonical integer types:
+// (or, rather, type pairs: signed and unsigned)
+//   NPY_BYTE, NPY_SHORT, NPY_INT, NPY_LONG, NPY_LONGLONG
+// It also has 4 fixed-width integer aliases.
+// When mapping Arrow integer types to these 4 fixed-width aliases,
+// we always miss one of the canonical types (even though it may
+// have the same width as one of the aliases).
+// Which one depends on the platform...
+// On a LP64 system, NPY_INT64 maps to NPY_LONG and
+// NPY_LONGLONG needs to be handled separately.
+// On a LLP64 system, NPY_INT32 maps to NPY_LONG and
+// NPY_INT needs to be handled separately.
+
+#if NPY_BITSOF_LONG == 32 && NPY_BITSOF_LONGLONG == 64
+#define NPY_INT64_IS_LONG_LONG 1
+#else
+#define NPY_INT64_IS_LONG_LONG 0
+#endif
+
+#if NPY_BITSOF_INT == 32 && NPY_BITSOF_LONG == 64
+#define NPY_INT32_IS_INT 1
+#else
+#define NPY_INT32_IS_INT 0
+#endif
+
 namespace arrow {
 namespace py {
 
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 9e3534d628487..c22afb7f1ffdc 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -79,6 +79,38 @@ inline bool PyObject_is_integer(PyObject* obj) {
   return !PyBool_Check(obj) && PyArray_IsIntegerScalar(obj);
 }
 
+Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
+  if (PyArray_NDIM(numpy_array) != 1) {
+    return Status::Invalid("only handle 1-dimensional arrays");
+  }
+
+  const int received_type = PyArray_DESCR(numpy_array)->type_num;
+  if (received_type != np_type) {
+    std::stringstream ss;
+    ss << "trying to convert NumPy type " << GetNumPyTypeName(np_type) << " but got "
+       << GetNumPyTypeName(received_type);
+    return Status::Invalid(ss.str());
+  }
+
+  return Status::OK();
+}
+
+Status AllocateNullBitmap(MemoryPool* pool, int64_t length,
+                          std::shared_ptr<ResizableBuffer>* out) {
+  int64_t null_bytes = BitUtil::BytesForBits(length);
+  std::shared_ptr<ResizableBuffer> null_bitmap;
+
+  null_bitmap = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(null_bitmap->Resize(null_bytes));
+
+  memset(null_bitmap->mutable_data(), 0, static_cast<size_t>(null_bytes));
+  *out = null_bitmap;
+  return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Conversion from NumPy-in-Pandas to Arrow null bitmap
+
 template <int TYPE>
 inline int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
   typedef internal::npy_traits<TYPE> traits;
@@ -98,6 +130,55 @@ inline int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* bitmap) {
   return null_count;
 }
 
+class NumPyNullsConverter {
+ public:
+  /// Convert the given array's null values to a null bitmap.
+  /// The null bitmap is only allocated if null values are ever possible.
+  static Status Convert(MemoryPool* pool, PyArrayObject* arr,
+                        bool use_pandas_null_sentinels,
+                        std::shared_ptr<ResizableBuffer>* out_null_bitmap_,
+                        int64_t* out_null_count) {
+    NumPyNullsConverter converter(pool, arr, use_pandas_null_sentinels);
+    RETURN_NOT_OK(VisitNumpyArrayInline(arr, &converter));
+    *out_null_bitmap_ = converter.null_bitmap_;
+    *out_null_count = converter.null_count_;
+    return Status::OK();
+  }
+
+  template <int TYPE>
+  Status Visit(PyArrayObject* arr) {
+    typedef internal::npy_traits<TYPE> traits;
+
+    const bool null_sentinels_possible =
+        // Always treat Numpy's NaT as null
+        TYPE == NPY_DATETIME ||
+        // Observing pandas's null sentinels
+        (use_pandas_null_sentinels_ && traits::supports_nulls);
+
+    if (null_sentinels_possible) {
+      RETURN_NOT_OK(AllocateNullBitmap(pool_, PyArray_SIZE(arr), &null_bitmap_));
+      null_count_ = ValuesToBitmap<TYPE>(arr, null_bitmap_->mutable_data());
+    }
+    return Status::OK();
+  }
+
+ protected:
+  NumPyNullsConverter(MemoryPool* pool, PyArrayObject* arr,
+                      bool use_pandas_null_sentinels)
+      : pool_(pool),
+        arr_(arr),
+        use_pandas_null_sentinels_(use_pandas_null_sentinels),
+        null_bitmap_data_(nullptr),
+        null_count_(0) {}
+
+  MemoryPool* pool_;
+  PyArrayObject* arr_;
+  bool use_pandas_null_sentinels_;
+  std::shared_ptr<ResizableBuffer> null_bitmap_;
+  uint8_t* null_bitmap_data_;
+  int64_t null_count_;
+};
+
 // Returns null count
 int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
   int64_t null_count = 0;
@@ -114,22 +195,6 @@ int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) {
   return null_count;
 }
 
-Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) {
-  if (PyArray_NDIM(numpy_array) != 1) {
-    return Status::Invalid("only handle 1-dimensional arrays");
-  }
-
-  const int received_type = PyArray_DESCR(numpy_array)->type_num;
-  if (received_type != np_type) {
-    std::stringstream ss;
-    ss << "trying to convert NumPy type " << GetNumPyTypeName(np_type) << " but got "
-       << GetNumPyTypeName(received_type);
-    return Status::Invalid(ss.str());
-  }
-
-  return Status::OK();
-}
-
 }  // namespace
 
 /// Append as many string objects from NumPy arrays to a `StringBuilder` as we
@@ -296,7 +361,9 @@ class NumPyConverter {
         dtype_(PyArray_DESCR(arr_)),
         mask_(nullptr),
         use_pandas_null_sentinels_(use_pandas_null_sentinels),
-        decimal_type_() {
+        decimal_type_(),
+        null_bitmap_data_(nullptr),
+        null_count_(0) {
     if (mo != nullptr && mo != Py_None) {
       mask_ = reinterpret_cast<PyArrayObject*>(mo);
     }
@@ -351,14 +418,8 @@ class NumPyConverter {
 
  protected:
   Status InitNullBitmap() {
-    int64_t null_bytes = BitUtil::BytesForBits(length_);
-
-    null_bitmap_ = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(null_bitmap_->Resize(null_bytes));
-
+    RETURN_NOT_OK(AllocateNullBitmap(pool_, length_, &null_bitmap_));
     null_bitmap_data_ = null_bitmap_->mutable_data();
-    memset(null_bitmap_data_, 0, static_cast<size_t>(null_bytes));
-
     return Status::OK();
   }
 
@@ -409,32 +470,18 @@ class NumPyConverter {
 
   template <typename ArrowType>
   Status VisitNative() {
-    using traits = internal::arrow_traits<ArrowType::type_id>;
-
-    const bool null_sentinels_possible =
-        // NumPy has a NaT type
-        (ArrowType::type_id == Type::TIMESTAMP || ArrowType::type_id == Type::DATE32) ||
-
-        // Observing pandas's null sentinels
-        ((use_pandas_null_sentinels_ && traits::supports_nulls));
-
-    if (mask_ != nullptr || null_sentinels_possible) {
+    if (mask_ != nullptr) {
       RETURN_NOT_OK(InitNullBitmap());
+      null_count_ = MaskToBitmap(mask_, length_, null_bitmap_data_);
+    } else {
+      RETURN_NOT_OK(NumPyNullsConverter::Convert(pool_, arr_, use_pandas_null_sentinels_,
+                                                 &null_bitmap_, &null_count_));
     }
 
     std::shared_ptr<Buffer> data;
     RETURN_NOT_OK(ConvertData<ArrowType>(&data));
 
-    int64_t null_count = 0;
-    if (mask_ != nullptr) {
-      null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
-    } else if (null_sentinels_possible) {
-      // TODO(wesm): this presumes the NumPy C type and arrow C type are the
-      // same
-      null_count = ValuesToBitmap<traits::npy_type>(arr_, null_bitmap_data_);
-    }
-
-    auto arr_data = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count, 0);
+    auto arr_data = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count_, 0);
     return PushArray(arr_data);
   }
 
@@ -488,6 +535,7 @@ class NumPyConverter {
 
   std::shared_ptr<ResizableBuffer> null_bitmap_;
   uint8_t* null_bitmap_data_;
+  int64_t null_count_;
 };
 
 Status NumPyConverter::Convert() {
@@ -654,12 +702,10 @@ inline Status NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* d
       Status s = StaticCastBuffer<int64_t, int32_t>(**data, length_, pool_, data);
       RETURN_NOT_OK(s);
     } else {
-      // TODO(wesm): This is redundant, and recomputed in VisitNative()
-      const int64_t null_count = ValuesToBitmap<NPY_DATETIME>(arr_, null_bitmap_data_);
-
       RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_), &input_type));
       if (!input_type->Equals(*type_)) {
-        RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count,
+        // The null bitmap was already computed in VisitNative()
+        RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count_,
                                  type_, pool_, data));
       }
     }
diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h
index 587b27c9a6039..ff39aad672e2e 100644
--- a/cpp/src/arrow/python/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -34,6 +34,9 @@ namespace arrow {
 namespace py {
 namespace internal {
 
+//
+// Type traits for Numpy -> Arrow equivalence
+//
 template <int TYPE>
 struct npy_traits {};
 
@@ -68,7 +71,11 @@ NPY_INT_DECL(UINT16, UInt16, uint16_t);
 NPY_INT_DECL(UINT32, UInt32, uint32_t);
 NPY_INT_DECL(UINT64, UInt64, uint64_t);
 
-#if NPY_INT64 != NPY_LONGLONG
+#if !NPY_INT32_IS_INT && NPY_BITSOF_INT == 32
+NPY_INT_DECL(INT, Int32, int32_t);
+NPY_INT_DECL(UINT, UInt32, uint32_t);
+#endif
+#if !NPY_INT64_IS_LONG_LONG && NPY_BITSOF_LONGLONG == 64
 NPY_INT_DECL(LONGLONG, Int64, int64_t);
 NPY_INT_DECL(ULONGLONG, UInt64, uint64_t);
 #endif
@@ -127,8 +134,14 @@ template <>
 struct npy_traits<NPY_OBJECT> {
   typedef PyObject* value_type;
   static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(PyObject* v) { return v == Py_None; }
 };
 
+//
+// Type traits for Arrow -> Numpy equivalence
+// Note *supports_nulls* means the equivalent Numpy type support nulls
+//
 template <int TYPE>
 struct arrow_traits {};
 
@@ -252,30 +265,27 @@ struct arrow_traits<Type::BINARY> {
 static inline int NumPyTypeSize(int npy_type) {
   switch (npy_type) {
     case NPY_BOOL:
-      return 1;
     case NPY_INT8:
-      return 1;
-    case NPY_INT16:
-      return 2;
-    case NPY_INT32:
-      return 4;
-    case NPY_INT64:
-      return 8;
-#if (NPY_INT64 != NPY_LONGLONG)
-    case NPY_LONGLONG:
-      return 8;
-#endif
     case NPY_UINT8:
       return 1;
+    case NPY_INT16:
     case NPY_UINT16:
       return 2;
+    case NPY_INT32:
     case NPY_UINT32:
       return 4;
+    case NPY_INT64:
     case NPY_UINT64:
       return 8;
-#if (NPY_UINT64 != NPY_ULONGLONG)
+#if !NPY_INT32_IS_INT
+    case NPY_INT:
+    case NPY_UINT:
+      return NPY_BITSOF_INT / 8;
+#endif
+#if !NPY_INT64_IS_LONG_LONG
+    case NPY_LONGLONG:
     case NPY_ULONGLONG:
-      return 8;
+      return NPY_BITSOF_LONGLONG / 8;
 #endif
     case NPY_FLOAT16:
       return 2;
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 79291359593c4..48db204ab96d8 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -502,6 +502,14 @@ def test_float_nulls(self):
         result = table.to_pandas()
         tm.assert_frame_equal(result, ex_frame)
 
+    def test_float_nulls_to_ints(self):
+        # ARROW-2135
+        df = pd.DataFrame({"a": [1.0, 2.0, pd.np.NaN]})
+        schema = pa.schema([pa.field("a", pa.int16(), nullable=True)])
+        table = pa.Table.from_pandas(df, schema=schema)
+        assert table[0].to_pylist() == [1, 2, None]
+        tm.assert_frame_equal(df, table.to_pandas())
+
     def test_integer_no_nulls(self):
         data = OrderedDict()
         fields = []
@@ -527,6 +535,17 @@ def test_integer_no_nulls(self):
         schema = pa.schema(fields)
         _check_pandas_roundtrip(df, expected_schema=schema)
 
+    def test_all_integer_types(self):
+        # Test all Numpy integer aliases
+        data = OrderedDict()
+        numpy_dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8',
+                        'byte', 'ubyte', 'short', 'ushort', 'intc', 'uintc',
+                        'int_', 'uint', 'longlong', 'ulonglong']
+        for dtype in numpy_dtypes:
+            data[dtype] = np.arange(12, dtype=dtype)
+        df = pd.DataFrame(data)
+        _check_pandas_roundtrip(df)
+
     def test_integer_with_nulls(self):
         # pandas requires upcast to float dtype
 

From 0b28dc50769893ff9d1608754ec499be4b3aa6bb Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Mon, 12 Mar 2018 22:38:07 -0400
Subject: [PATCH 1599/1644] ARROW-2142: [Python] Allow conversion from Numpy
 struct array

Author: Antoine Pitrou <antoine@python.org>

Closes #1635 from pitrou/ARROW-2142-convert-from-np-struct-array and squashes the following commits:

5ade2b26 <Antoine Pitrou> ARROW-2142:  Allow conversion from Numpy struct array
---
 cpp/src/arrow/array-test.cc                 |  64 +++++++++++
 cpp/src/arrow/array.cc                      |  81 +++++++++++++
 cpp/src/arrow/array.h                       |  11 ++
 cpp/src/arrow/python/numpy_to_arrow.cc      | 113 +++++++++++++++++-
 cpp/src/arrow/test-util.h                   |   4 +-
 python/pyarrow/tests/test_convert_pandas.py | 120 +++++++++++++++++++-
 6 files changed, 388 insertions(+), 5 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index bda1946c6a72a..ad2335fa11240 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -3256,4 +3256,68 @@ TEST_P(DecimalTest, WithNulls) {
 
 INSTANTIATE_TEST_CASE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38));
 
+// ----------------------------------------------------------------------
+// Test rechunking
+
+TEST(TestRechunkArraysConsistently, Trivial) {
+  std::vector<ArrayVector> groups, rechunked;
+  rechunked = internal::RechunkArraysConsistently(groups);
+  ASSERT_EQ(rechunked.size(), 0);
+
+  std::shared_ptr<Array> a1, a2, b1;
+  ArrayFromVector<Int16Type, int16_t>({}, &a1);
+  ArrayFromVector<Int16Type, int16_t>({}, &a2);
+  ArrayFromVector<Int32Type, int32_t>({}, &b1);
+
+  groups = {{a1, a2}, {}, {b1}};
+  rechunked = internal::RechunkArraysConsistently(groups);
+  ASSERT_EQ(rechunked.size(), 3);
+}
+
+TEST(TestRechunkArraysConsistently, Plain) {
+  std::shared_ptr<Array> expected;
+  std::shared_ptr<Array> a1, a2, a3, b1, b2, b3, b4;
+  ArrayFromVector<Int16Type, int16_t>({1, 2, 3}, &a1);
+  ArrayFromVector<Int16Type, int16_t>({4, 5}, &a2);
+  ArrayFromVector<Int16Type, int16_t>({6, 7, 8, 9}, &a3);
+
+  ArrayFromVector<Int32Type, int32_t>({41, 42}, &b1);
+  ArrayFromVector<Int32Type, int32_t>({43, 44, 45}, &b2);
+  ArrayFromVector<Int32Type, int32_t>({46, 47}, &b3);
+  ArrayFromVector<Int32Type, int32_t>({48, 49}, &b4);
+
+  ArrayVector a{a1, a2, a3};
+  ArrayVector b{b1, b2, b3, b4};
+
+  std::vector<ArrayVector> groups{a, b}, rechunked;
+  rechunked = internal::RechunkArraysConsistently(groups);
+  ASSERT_EQ(rechunked.size(), 2);
+  auto ra = rechunked[0];
+  auto rb = rechunked[1];
+
+  ASSERT_EQ(ra.size(), 5);
+  ArrayFromVector<Int16Type, int16_t>({1, 2}, &expected);
+  ASSERT_ARRAYS_EQUAL(*ra[0], *expected);
+  ArrayFromVector<Int16Type, int16_t>({3}, &expected);
+  ASSERT_ARRAYS_EQUAL(*ra[1], *expected);
+  ArrayFromVector<Int16Type, int16_t>({4, 5}, &expected);
+  ASSERT_ARRAYS_EQUAL(*ra[2], *expected);
+  ArrayFromVector<Int16Type, int16_t>({6, 7}, &expected);
+  ASSERT_ARRAYS_EQUAL(*ra[3], *expected);
+  ArrayFromVector<Int16Type, int16_t>({8, 9}, &expected);
+  ASSERT_ARRAYS_EQUAL(*ra[4], *expected);
+
+  ASSERT_EQ(rb.size(), 5);
+  ArrayFromVector<Int32Type, int32_t>({41, 42}, &expected);
+  ASSERT_ARRAYS_EQUAL(*rb[0], *expected);
+  ArrayFromVector<Int32Type, int32_t>({43}, &expected);
+  ASSERT_ARRAYS_EQUAL(*rb[1], *expected);
+  ArrayFromVector<Int32Type, int32_t>({44, 45}, &expected);
+  ASSERT_ARRAYS_EQUAL(*rb[2], *expected);
+  ArrayFromVector<Int32Type, int32_t>({46, 47}, &expected);
+  ASSERT_ARRAYS_EQUAL(*rb[3], *expected);
+  ArrayFromVector<Int32Type, int32_t>({48, 49}, &expected);
+  ASSERT_ARRAYS_EQUAL(*rb[4], *expected);
+}
+
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 83142dfefaab9..bd2b40c1a4923 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -20,6 +20,8 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
+#include <limits>
+#include <set>
 #include <sstream>
 #include <utility>
 
@@ -752,6 +754,85 @@ std::shared_ptr<Array> MakeArray(const std::shared_ptr<ArrayData>& data) {
   return out;
 }
 
+// ----------------------------------------------------------------------
+// Misc APIs
+
+namespace internal {
+
+std::vector<ArrayVector> RechunkArraysConsistently(
+    const std::vector<ArrayVector>& groups) {
+  if (groups.size() <= 1) {
+    return groups;
+  }
+  int64_t total_length = 0;
+  for (const auto& array : groups.front()) {
+    total_length += array->length();
+  }
+#ifndef NDEBUG
+  for (const auto& group : groups) {
+    int64_t group_length = 0;
+    for (const auto& array : group) {
+      group_length += array->length();
+    }
+    DCHECK_EQ(group_length, total_length)
+        << "Array groups should have the same total number of elements";
+  }
+#endif
+  if (total_length == 0) {
+    return groups;
+  }
+
+  // Set up result vectors
+  std::vector<ArrayVector> rechunked_groups(groups.size());
+
+  // Set up progress counters
+  std::vector<ArrayVector::const_iterator> current_arrays;
+  std::vector<int64_t> array_offsets;
+  for (const auto& group : groups) {
+    current_arrays.emplace_back(group.cbegin());
+    array_offsets.emplace_back(0);
+  }
+
+  // Scan all array vectors at once, rechunking along the way
+  int64_t start = 0;
+  while (start < total_length) {
+    // First compute max possible length for next chunk
+    int64_t chunk_length = std::numeric_limits<int64_t>::max();
+    for (size_t i = 0; i < groups.size(); i++) {
+      auto& arr_it = current_arrays[i];
+      auto& offset = array_offsets[i];
+      // Skip any done arrays (including 0-length arrays)
+      while (offset == (*arr_it)->length()) {
+        ++arr_it;
+        offset = 0;
+      }
+      const auto& array = *arr_it;
+      DCHECK_GT(array->length(), offset);
+      chunk_length = std::min(chunk_length, array->length() - offset);
+    }
+    DCHECK_GT(chunk_length, 0);
+
+    // Then slice all arrays along this chunk size
+    for (size_t i = 0; i < groups.size(); i++) {
+      const auto& array = *current_arrays[i];
+      auto& offset = array_offsets[i];
+      if (offset == 0 && array->length() == chunk_length) {
+        // Slice spans entire array
+        rechunked_groups[i].emplace_back(array);
+      } else {
+        DCHECK_LT(chunk_length - offset, array->length());
+        rechunked_groups[i].emplace_back(array->Slice(offset, chunk_length));
+      }
+      offset += chunk_length;
+    }
+    start += chunk_length;
+  }
+
+  return rechunked_groups;
+}
+
+}  // namespace internal
+
 // ----------------------------------------------------------------------
 // Instantiate templates
 
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index faa9211c674ae..04bd0126141be 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -278,6 +278,17 @@ class ARROW_EXPORT Array {
 
 using ArrayVector = std::vector<std::shared_ptr<Array>>;
 
+namespace internal {
+
+/// Given a number of ArrayVectors, treat each ArrayVector as the
+/// chunks of a chunked array.  Then rechunk each ArrayVector such that
+/// all ArrayVectors are chunked identically.  It is mandatory that
+/// all ArrayVectors contain the same total number of elements.
+ARROW_EXPORT
+std::vector<ArrayVector> RechunkArraysConsistently(const std::vector<ArrayVector>&);
+
+}  // namespace internal
+
 static inline std::ostream& operator<<(std::ostream& os, const Array& x) {
   os << x.ToString();
   return os;
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index c22afb7f1ffdc..4d91e531712a0 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -380,7 +380,7 @@ class NumPyConverter {
 
   Status Convert();
 
-  const std::vector<std::shared_ptr<Array>>& result() const { return out_arrays_; }
+  const ArrayVector& result() const { return out_arrays_; }
 
   template <typename T>
   typename std::enable_if<std::is_base_of<PrimitiveCType, T>::value ||
@@ -406,6 +406,8 @@ class NumPyConverter {
   // NumPy unicode arrays
   Status Visit(const StringType& type);
 
+  Status Visit(const StructType& type);
+
   Status Visit(const FixedSizeBinaryType& type) {
     return TypeNotImplemented(type.ToString());
   }
@@ -531,7 +533,7 @@ class NumPyConverter {
   OwnedRefNoGIL decimal_type_;
 
   // Used in visitor pattern
-  std::vector<std::shared_ptr<Array>> out_arrays_;
+  ArrayVector out_arrays_;
 
   std::shared_ptr<ResizableBuffer> null_bitmap_;
   uint8_t* null_bitmap_data_;
@@ -1607,6 +1609,113 @@ Status NumPyConverter::Visit(const StringType& type) {
   return PushArray(result->data());
 }
 
+Status NumPyConverter::Visit(const StructType& type) {
+  std::vector<NumPyConverter> sub_converters;
+  std::vector<OwnedRefNoGIL> sub_arrays;
+
+  {
+    PyAcquireGIL gil_lock;
+
+    // Create converters for each struct type field
+    if (dtype_->fields == NULL || !PyDict_Check(dtype_->fields)) {
+      return Status::TypeError("Expected struct array");
+    }
+
+    for (auto field : type.children()) {
+      PyObject* tup = PyDict_GetItemString(dtype_->fields, field->name().c_str());
+      if (tup == NULL) {
+        std::stringstream ss;
+        ss << "Missing field '" << field->name() << "' in struct array";
+        return Status::TypeError(ss.str());
+      }
+      PyArray_Descr* sub_dtype =
+          reinterpret_cast<PyArray_Descr*>(PyTuple_GET_ITEM(tup, 0));
+      DCHECK(PyArray_DescrCheck(sub_dtype));
+      int offset = static_cast<int>(PyLong_AsLong(PyTuple_GET_ITEM(tup, 1)));
+      RETURN_IF_PYERROR();
+      Py_INCREF(sub_dtype); /* PyArray_GetField() steals ref */
+      PyObject* sub_array = PyArray_GetField(arr_, sub_dtype, offset);
+      RETURN_IF_PYERROR();
+      sub_arrays.emplace_back(sub_array);
+      sub_converters.emplace_back(pool_, sub_array, nullptr /* mask */, field->type(),
+                                  use_pandas_null_sentinels_);
+    }
+  }
+
+  std::vector<ArrayVector> groups;
+  int64_t null_count = 0;
+
+  // Compute null bitmap and store it as a Boolean Array to include it
+  // in the rechunking below
+  {
+    if (mask_ != nullptr) {
+      RETURN_NOT_OK(InitNullBitmap());
+      null_count = MaskToBitmap(mask_, length_, null_bitmap_data_);
+    }
+    groups.push_back({std::make_shared<BooleanArray>(length_, null_bitmap_)});
+  }
+
+  // Convert child data
+  for (auto& converter : sub_converters) {
+    RETURN_NOT_OK(converter.Convert());
+    groups.push_back(converter.result());
+    const auto& group = groups.back();
+    int64_t n = 0;
+    for (const auto& array : group) {
+      n += array->length();
+    }
+  }
+  // Ensure the different array groups are chunked consistently
+  groups = ::arrow::internal::RechunkArraysConsistently(groups);
+  for (const auto& group : groups) {
+    int64_t n = 0;
+    for (const auto& array : group) {
+      n += array->length();
+    }
+  }
+
+  // Make struct array chunks by combining groups
+  size_t ngroups = groups.size();
+  size_t nchunks = groups[0].size();
+  for (size_t chunk = 0; chunk < nchunks; chunk++) {
+    // First group has the null bitmaps as Boolean Arrays
+    const auto& null_data = groups[0][chunk]->data();
+    DCHECK_EQ(null_data->type->id(), Type::BOOL);
+    DCHECK_EQ(null_data->buffers.size(), 2);
+    const auto& null_buffer = null_data->buffers[1];
+    // Careful: the rechunked null bitmap may have a non-zero offset
+    // to its buffer, and it may not even start on a byte boundary
+    int64_t null_offset = null_data->offset;
+    std::shared_ptr<Buffer> fixed_null_buffer;
+
+    if (!null_buffer) {
+      fixed_null_buffer = null_buffer;
+    } else if (null_offset % 8 == 0) {
+      fixed_null_buffer =
+          std::make_shared<Buffer>(null_buffer,
+                                   // byte offset
+                                   null_offset / 8,
+                                   // byte size
+                                   BitUtil::BytesForBits(null_data->length));
+    } else {
+      RETURN_NOT_OK(CopyBitmap(pool_, null_buffer->data(), null_offset, null_data->length,
+                               &fixed_null_buffer));
+    }
+
+    // Create struct array chunk and populate it
+    auto arr_data =
+        ArrayData::Make(type_, null_data->length, null_count ? kUnknownNullCount : 0, 0);
+    arr_data->buffers.push_back(fixed_null_buffer);
+    // Append child chunks
+    for (size_t i = 1; i < ngroups; i++) {
+      arr_data->child_data.push_back(groups[i][chunk]->data());
+    }
+    RETURN_NOT_OK(PushArray(arr_data));
+  }
+
+  return Status::OK();
+}
+
 Status NdarrayToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
                       bool use_pandas_null_sentinels,
                       const std::shared_ptr<DataType>& type,
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index ab68fd44258bc..fdd42a67acf0e 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -401,8 +401,8 @@ void ArrayFromVector(const std::vector<bool>& is_valid, const std::vector<C_TYPE
 template <typename TYPE, typename C_TYPE>
 void ArrayFromVector(const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
   typename TypeTraits<TYPE>::BuilderType builder;
-  for (size_t i = 0; i < values.size(); ++i) {
-    ASSERT_OK(builder.Append(values[i]));
+  for (auto& value : values) {
+    ASSERT_OK(builder.Append(value));
   }
   ASSERT_OK(builder.Finish(out));
 }
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 48db204ab96d8..dff4e1068fc03 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1466,7 +1466,7 @@ class TestConvertStructTypes(object):
     Conversion tests for struct types.
     """
 
-    def test_structarray(self):
+    def test_to_pandas(self):
         ints = pa.array([None, 2, 3], type=pa.int64())
         strs = pa.array([u'a', None, u'c'], type=pa.string())
         bools = pa.array([True, False, None], type=pa.bool_())
@@ -1483,6 +1483,124 @@ def test_structarray(self):
         series = pd.Series(arr.to_pandas())
         tm.assert_series_equal(series, expected)
 
+    def test_from_numpy(self):
+        dt = np.dtype([('x', np.int32),
+                       (('y_title', 'y'), np.bool_)])
+        ty = pa.struct([pa.field('x', pa.int32()),
+                        pa.field('y', pa.bool_())])
+
+        data = np.array([], dtype=dt)
+        arr = pa.array(data, type=ty)
+        assert arr.to_pylist() == []
+
+        data = np.array([(42, True), (43, False)], dtype=dt)
+        arr = pa.array(data, type=ty)
+        assert arr.to_pylist() == [{'x': 42, 'y': True},
+                                   {'x': 43, 'y': False}]
+
+        # With mask
+        arr = pa.array(data, mask=np.bool_([False, True]), type=ty)
+        assert arr.to_pylist() == [{'x': 42, 'y': True}, None]
+
+        # Trivial struct type
+        dt = np.dtype([])
+        ty = pa.struct([])
+
+        data = np.array([], dtype=dt)
+        arr = pa.array(data, type=ty)
+        assert arr.to_pylist() == []
+
+        data = np.array([(), ()], dtype=dt)
+        arr = pa.array(data, type=ty)
+        assert arr.to_pylist() == [{}, {}]
+
+    def test_from_numpy_nested(self):
+        dt = np.dtype([('x', np.dtype([('xx', np.int8),
+                                       ('yy', np.bool_)])),
+                       ('y', np.int16)])
+        ty = pa.struct([pa.field('x', pa.struct([pa.field('xx', pa.int8()),
+                                                 pa.field('yy', pa.bool_())])),
+                        pa.field('y', pa.int16())])
+
+        data = np.array([], dtype=dt)
+        arr = pa.array(data, type=ty)
+        assert arr.to_pylist() == []
+
+        data = np.array([((1, True), 2), ((3, False), 4)], dtype=dt)
+        arr = pa.array(data, type=ty)
+        assert arr.to_pylist() == [{'x': {'xx': 1, 'yy': True}, 'y': 2},
+                                   {'x': {'xx': 3, 'yy': False}, 'y': 4}]
+
+    @pytest.mark.large_memory
+    def test_from_numpy_large(self):
+        # Exercise rechunking + nulls
+        target_size = 3 * 1024**3  # 4GB
+        dt = np.dtype([('x', np.float64), ('y', 'object')])
+        bs = 65536 - dt.itemsize
+        block = b'.' * bs
+        n = target_size // (bs + dt.itemsize)
+        data = np.zeros(n, dtype=dt)
+        data['x'] = np.random.random_sample(n)
+        data['y'] = block
+        # Add implicit nulls
+        data['x'][data['x'] < 0.2] = np.nan
+
+        ty = pa.struct([pa.field('x', pa.float64()),
+                        pa.field('y', pa.binary(bs))])
+        arr = pa.array(data, type=ty, from_pandas=True)
+        assert arr.num_chunks == 2
+
+        def iter_chunked_array(arr):
+            for chunk in arr.iterchunks():
+                for item in chunk:
+                    yield item
+
+        def check(arr, data, mask=None):
+            assert len(arr) == len(data)
+            xs = data['x']
+            ys = data['y']
+            for i, obj in enumerate(iter_chunked_array(arr)):
+                try:
+                    d = obj.as_py()
+                    if mask is not None and mask[i]:
+                        assert d is None
+                    else:
+                        x = xs[i]
+                        if np.isnan(x):
+                            assert d['x'] is None
+                        else:
+                            assert d['x'] == x
+                        assert d['y'] == ys[i]
+                except Exception:
+                    print("Failed at index", i)
+                    raise
+
+        check(arr, data)
+        del arr
+
+        # Now with explicit mask
+        mask = np.random.random_sample(n) < 0.2
+        arr = pa.array(data, type=ty, mask=mask, from_pandas=True)
+        assert arr.num_chunks == 2
+
+        check(arr, data, mask)
+        del arr
+
+    def test_from_numpy_bad_input(self):
+        ty = pa.struct([pa.field('x', pa.int32()),
+                        pa.field('y', pa.bool_())])
+        dt = np.dtype([('x', np.int32),
+                       ('z', np.bool_)])
+
+        data = np.array([], dtype=dt)
+        with pytest.raises(TypeError,
+                           match="Missing field 'y'"):
+            pa.array(data, type=ty)
+        data = np.int32([])
+        with pytest.raises(TypeError,
+                           match="Expected struct array"):
+            pa.array(data, type=ty)
+
 
 class TestZeroCopyConversion(object):
     """

From 7c7b09f6e52d8c582ef00d201ec4e783484d3180 Mon Sep 17 00:00:00 2001
From: Ehsan Totoni <ehsan.totoni@intel.com>
Date: Tue, 13 Mar 2018 09:53:33 -0400
Subject: [PATCH 1600/1644] ARROW-1643: [Python] Accept hdfs:// prefixes in
 parquet.read_table and attempt to connect to HDFS

This patch enables pq.read_table, pq.write_table, and pq.ParquetFile to accept HDFS URI and connect to HDFS.

Author: Ehsan Totoni <ehsan.totoni@intel.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1668 from ehsantn/ARROW-1643 and squashes the following commits:

91c3c7f6 <Wes McKinney> Close automatically-opened file handle in ParquetWriter.close rather than relying on gc cycle to run
e93256e9 <Ehsan Totoni> extra blank line for flake8 fix
9a859b76 <Ehsan Totoni> add test for hdfs uri read of multiple files
72fd145e <Ehsan Totoni> remove schema check
60272283 <Ehsan Totoni> handle pathlib.Path for Parquet file names
46a4e23c <Ehsan Totoni> use IOError instead of FileNotFoundError for Python 2.7 compatibility
01d301cc <Wes McKinney> Fix unit test
9090e93c <Wes McKinney> Factor out string-to-file coercion
f12711fc <Ehsan Totoni> ARROW-1643  Accept hdfs:// prefixes in parquet.read_table and attempt to connect to HDFS
---
 python/pyarrow/parquet.py         | 87 ++++++++++++++++++++++++++-----
 python/pyarrow/tests/test_hdfs.py | 71 +++++++++++++++++++++----
 2 files changed, 135 insertions(+), 23 deletions(-)

diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index 42c558b0bd9e2..fd9c740f14e20 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -21,6 +21,13 @@
 import json
 import re
 import six
+from six.moves.urllib.parse import urlparse
+# pathlib might not be available in Python 2
+try:
+    import pathlib
+    _has_pathlib = True
+except ImportError:
+    _has_pathlib = False
 
 import numpy as np
 
@@ -53,6 +60,7 @@ class ParquetFile(object):
     """
     def __init__(self, source, metadata=None, common_metadata=None):
         self.reader = ParquetReader()
+        source = _ensure_file(source)
         self.reader.open(source, metadata=metadata)
         self.common_metadata = common_metadata
         self._nested_paths_by_prefix = self._build_nested_paths()
@@ -279,8 +287,20 @@ def __init__(self, where, schema, flavor=None,
             self.schema_changed = False
 
         self.schema = schema
+        self.where = where
+
+        # If we open a file using an implied filesystem, so it can be assured
+        # to be closed
+        self.file_handle = None
+
+        if is_path(where):
+            fs = _get_fs_from_path(where)
+            sink = self.file_handle = fs.open(where, 'wb')
+        else:
+            sink = where
+
         self.writer = _parquet.ParquetWriter(
-            where, schema,
+            sink, schema,
             version=version,
             compression=compression,
             use_dictionary=use_dictionary,
@@ -310,6 +330,8 @@ def close(self):
         if self.is_open:
             self.writer.close()
             self.is_open = False
+        if self.file_handle is not None:
+            self.file_handle.close()
 
 
 def _get_pandas_index_columns(keyvalues):
@@ -559,8 +581,9 @@ def get_index(self, level, name, key):
         return self.levels[level].get_index(key)
 
 
-def is_string(x):
-    return isinstance(x, six.string_types)
+def is_path(x):
+    return (isinstance(x, six.string_types)
+            or (_has_pathlib and isinstance(x, pathlib.Path)))
 
 
 class ParquetManifest(object):
@@ -569,7 +592,7 @@ class ParquetManifest(object):
     """
     def __init__(self, dirpath, filesystem=None, pathsep='/',
                  partition_scheme='hive'):
-        self.filesystem = filesystem or LocalFileSystem.get_instance()
+        self.filesystem = filesystem or _get_fs_from_path(dirpath)
         self.pathsep = pathsep
         self.dirpath = dirpath
         self.partition_scheme = partition_scheme
@@ -692,7 +715,10 @@ class ParquetDataset(object):
     def __init__(self, path_or_paths, filesystem=None, schema=None,
                  metadata=None, split_row_groups=False, validate_schema=True):
         if filesystem is None:
-            self.fs = LocalFileSystem.get_instance()
+            a_path = path_or_paths
+            if isinstance(a_path, list):
+                a_path = a_path[0]
+            self.fs = _get_fs_from_path(a_path)
         else:
             self.fs = _ensure_filesystem(filesystem)
 
@@ -851,7 +877,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
         # Dask passes a directory as a list of length 1
         path_or_paths = path_or_paths[0]
 
-    if is_string(path_or_paths) and fs.isdir(path_or_paths):
+    if is_path(path_or_paths) and fs.isdir(path_or_paths):
         manifest = ParquetManifest(path_or_paths, filesystem=fs,
                                    pathsep=fs.pathsep)
         common_metadata_path = manifest.common_metadata_path
@@ -904,11 +930,11 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
 
 def read_table(source, columns=None, nthreads=1, metadata=None,
                use_pandas_metadata=False):
-    if is_string(source):
-        fs = LocalFileSystem.get_instance()
+    if is_path(source):
+        fs = _get_fs_from_path(source)
+
         if fs.isdir(source):
-            return fs.read_parquet(source, columns=columns,
-                                   metadata=metadata)
+            return fs.read_parquet(source, columns=columns, metadata=metadata)
 
     pf = ParquetFile(source, metadata=metadata)
     return pf.read(columns=columns, nthreads=nthreads,
@@ -957,7 +983,7 @@ def write_table(table, where, row_group_size=None, version='1.0',
                 **kwargs) as writer:
             writer.write_table(table, row_group_size=row_group_size)
     except Exception:
-        if isinstance(where, six.string_types):
+        if is_path(where):
             try:
                 os.remove(where)
             except os.error:
@@ -1026,7 +1052,7 @@ def write_to_dataset(table, root_path, partition_cols=None,
     )
 
     if filesystem is None:
-        fs = LocalFileSystem.get_instance()
+        fs = _get_fs_from_path(root_path)
     else:
         fs = _ensure_filesystem(filesystem)
 
@@ -1113,3 +1139,40 @@ def read_schema(where):
     schema : pyarrow.Schema
     """
     return ParquetFile(where).schema.to_arrow_schema()
+
+
+def _ensure_file(source):
+    if is_path(source):
+        fs = _get_fs_from_path(source)
+        try:
+            return fs.open(source)
+        except IOError as e:
+            raise lib.ArrowIOError("failed to open file {}, {}"
+                                   .format(source, e))
+    elif not hasattr(source, 'seek'):
+        raise ValueError('Source does not appear file-like')
+    else:
+        return source
+
+
+def _get_fs_from_path(path):
+    """
+    return filesystem from path which could be an HDFS URI
+    """
+    # input can be hdfs URI such as hdfs://host:port/myfile.parquet
+    if _has_pathlib and isinstance(path, pathlib.Path):
+        path = str(path)
+    parsed_uri = urlparse(path)
+    if parsed_uri.scheme == 'hdfs':
+        netloc_split = parsed_uri.netloc.split(':')
+        host = netloc_split[0]
+        if host == '':
+            host = 'default'
+        port = 0
+        if len(netloc_split) == 2 and netloc_split[1].isnumeric():
+            port = int(netloc_split[1])
+        fs = pa.hdfs.connect(host=host, port=port)
+    else:
+        fs = LocalFileSystem.get_instance()
+
+    return fs
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 885272ba8bdf5..4840aee48ca13 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -272,19 +272,11 @@ def test_read_whole_file(self):
 
         assert result == data
 
-    @test_parquet.parquet
-    def test_read_multiple_parquet_files(self):
+    def _write_multiple_hdfs_pq_files(self, tmpdir):
         import pyarrow.parquet as pq
-
         nfiles = 10
         size = 5
-
-        tmpdir = pjoin(self.tmp_path, 'multi-parquet-' + guid())
-
-        self.hdfs.mkdir(tmpdir)
-
         test_data = []
-        paths = []
         for i in range(nfiles):
             df = test_parquet._test_dataframe(size, seed=i)
 
@@ -300,15 +292,60 @@ def test_read_multiple_parquet_files(self):
                 pq.write_table(table, f)
 
             test_data.append(table)
-            paths.append(path)
 
-        result = self.hdfs.read_parquet(tmpdir)
         expected = pa.concat_tables(test_data)
+        return expected
+
+    @test_parquet.parquet
+    def test_read_multiple_parquet_files(self):
+
+        tmpdir = pjoin(self.tmp_path, 'multi-parquet-' + guid())
+
+        self.hdfs.mkdir(tmpdir)
+
+        expected = self._write_multiple_hdfs_pq_files(tmpdir)
+        result = self.hdfs.read_parquet(tmpdir)
 
         pdt.assert_frame_equal(result.to_pandas()
                                .sort_values(by='index').reset_index(drop=True),
                                expected.to_pandas())
 
+    @test_parquet.parquet
+    def test_read_multiple_parquet_files_with_uri(self):
+        import pyarrow.parquet as pq
+
+        tmpdir = pjoin(self.tmp_path, 'multi-parquet-uri-' + guid())
+
+        self.hdfs.mkdir(tmpdir)
+
+        expected = self._write_multiple_hdfs_pq_files(tmpdir)
+        path = _get_hdfs_uri(tmpdir)
+        result = pq.read_table(path)
+
+        pdt.assert_frame_equal(result.to_pandas()
+                               .sort_values(by='index').reset_index(drop=True),
+                               expected.to_pandas())
+
+    @test_parquet.parquet
+    def test_read_write_parquet_files_with_uri(self):
+        import pyarrow.parquet as pq
+
+        tmpdir = pjoin(self.tmp_path, 'uri-parquet-' + guid())
+        self.hdfs.mkdir(tmpdir)
+        path = _get_hdfs_uri(pjoin(tmpdir, 'test.parquet'))
+
+        size = 5
+        df = test_parquet._test_dataframe(size, seed=0)
+        # Hack so that we don't have a dtype cast in v1 files
+        df['uint32'] = df['uint32'].astype(np.int64)
+        table = pa.Table.from_pandas(df, preserve_index=False)
+
+        pq.write_table(table, path)
+
+        result = pq.read_table(path).to_pandas()
+
+        pdt.assert_frame_equal(result, df)
+
     @test_parquet.parquet
     def test_read_common_metadata_files(self):
         tmpdir = pjoin(self.tmp_path, 'common-metadata-' + guid())
@@ -357,3 +394,15 @@ class TestLibHdfs3(HdfsTestCases, unittest.TestCase):
     def check_driver(cls):
         if not pa.have_libhdfs3():
             pytest.skip('No libhdfs3 available on system')
+
+
+def _get_hdfs_uri(path):
+    host = os.environ.get('ARROW_HDFS_TEST_HOST', 'localhost')
+    try:
+        port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 0))
+    except ValueError:
+        raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
+                         'an integer')
+    uri = "hdfs://{}:{}{}".format(host, port, path)
+
+    return uri

From 33d1091ee6f6a835ed058a9c368f465a9a97cbe0 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Tue, 13 Mar 2018 16:46:10 -0400
Subject: [PATCH 1601/1644] ARROW-2227: [Python] Fix off-by-one error in
 chunked binary conversions

We were already testing the chunked behavior but we did not exercise the off-by-one error edge case

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1740 from wesm/ARROW-2227 and squashes the following commits:

fd1ab178 <Wes McKinney> Move constexpr to builder.h, add constexpr for max # of list elements
ca5ecc0f <Wes McKinney> Fix off-by-one error in chunked binary conversions
---
 cpp/src/arrow/builder.cc                    | 16 ++++++++--------
 cpp/src/arrow/builder.h                     |  7 ++++---
 cpp/src/arrow/python/numpy_to_arrow.cc      |  2 --
 python/pyarrow/tests/test_convert_pandas.py |  7 +++++--
 4 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index ef4e7fde98d3e..aa9f3ce423444 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -1236,7 +1236,7 @@ Status ListBuilder::Append(const int32_t* offsets, int64_t length,
 
 Status ListBuilder::AppendNextOffset() {
   int64_t num_values = value_builder_->length();
-  if (ARROW_PREDICT_FALSE(num_values >= std::numeric_limits<int32_t>::max())) {
+  if (ARROW_PREDICT_FALSE(num_values > kListMaximumElements)) {
     std::stringstream ss;
     ss << "ListArray cannot contain more then INT32_MAX - 1 child elements,"
        << " have " << num_values;
@@ -1252,14 +1252,14 @@ Status ListBuilder::Append(bool is_valid) {
 }
 
 Status ListBuilder::Init(int64_t elements) {
-  DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
+  DCHECK_LE(elements, kListMaximumElements);
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
   // one more then requested for offsets
   return offsets_builder_.Resize((elements + 1) * sizeof(int32_t));
 }
 
 Status ListBuilder::Resize(int64_t capacity) {
-  DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
+  DCHECK_LE(capacity, kListMaximumElements);
   // one more then requested for offsets
   RETURN_NOT_OK(offsets_builder_.Resize((capacity + 1) * sizeof(int32_t)));
   return ArrayBuilder::Resize(capacity);
@@ -1303,14 +1303,14 @@ BinaryBuilder::BinaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool*
 BinaryBuilder::BinaryBuilder(MemoryPool* pool) : BinaryBuilder(binary(), pool) {}
 
 Status BinaryBuilder::Init(int64_t elements) {
-  DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
+  DCHECK_LE(elements, kListMaximumElements);
   RETURN_NOT_OK(ArrayBuilder::Init(elements));
   // one more then requested for offsets
   return offsets_builder_.Resize((elements + 1) * sizeof(int32_t));
 }
 
 Status BinaryBuilder::Resize(int64_t capacity) {
-  DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
+  DCHECK_LE(capacity, kListMaximumElements);
   // one more then requested for offsets
   RETURN_NOT_OK(offsets_builder_.Resize((capacity + 1) * sizeof(int32_t)));
   return ArrayBuilder::Resize(capacity);
@@ -1318,7 +1318,7 @@ Status BinaryBuilder::Resize(int64_t capacity) {
 
 Status BinaryBuilder::ReserveData(int64_t elements) {
   if (value_data_length() + elements > value_data_capacity()) {
-    if (value_data_length() + elements > std::numeric_limits<int32_t>::max()) {
+    if (value_data_length() + elements > kBinaryMemoryLimit) {
       return Status::Invalid("Cannot reserve capacity larger than 2^31 - 1 for binary");
     }
     RETURN_NOT_OK(value_data_builder_.Reserve(elements));
@@ -1328,9 +1328,9 @@ Status BinaryBuilder::ReserveData(int64_t elements) {
 
 Status BinaryBuilder::AppendNextOffset() {
   const int64_t num_bytes = value_data_builder_.length();
-  if (ARROW_PREDICT_FALSE(num_bytes > kMaximumCapacity)) {
+  if (ARROW_PREDICT_FALSE(num_bytes > kBinaryMemoryLimit)) {
     std::stringstream ss;
-    ss << "BinaryArray cannot contain more than " << kMaximumCapacity << " bytes, have "
+    ss << "BinaryArray cannot contain more than " << kBinaryMemoryLimit << " bytes, have "
        << num_bytes;
     return Status::Invalid(ss.str());
   }
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index dabfb7506247a..cdcee80bedbfa 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -41,13 +41,16 @@ namespace arrow {
 class Array;
 class Decimal128;
 
+constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max() - 1;
+constexpr int64_t kListMaximumElements = std::numeric_limits<int32_t>::max() - 1;
+
 namespace internal {
 
 struct ArrayData;
 
 }  // namespace internal
 
-static constexpr int64_t kMinBuilderCapacity = 1 << 5;
+constexpr int64_t kMinBuilderCapacity = 1 << 5;
 
 /// Base class for all data array builders.
 //
@@ -702,8 +705,6 @@ class ARROW_EXPORT BinaryBuilder : public ArrayBuilder {
   TypedBufferBuilder<int32_t> offsets_builder_;
   TypedBufferBuilder<uint8_t> value_data_builder_;
 
-  static constexpr int64_t kMaximumCapacity = std::numeric_limits<int32_t>::max() - 1;
-
   Status AppendNextOffset();
   void Reset();
 };
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index 4d91e531712a0..71bf69fc1ed28 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -60,8 +60,6 @@ namespace py {
 
 using internal::NumPyTypeSize;
 
-constexpr int64_t kBinaryMemoryLimit = std::numeric_limits<int32_t>::max();
-
 // ----------------------------------------------------------------------
 // Conversion utilities
 
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index dff4e1068fc03..d448de08be42f 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1048,9 +1048,12 @@ def test_bytes_to_binary(self):
 
     @pytest.mark.large_memory
     def test_bytes_exceed_2gb(self):
-        val = 'x' * (1 << 20)
+        v1 = b'x' * 100000000
+        v2 = b'x' * 147483646
+
+        # ARROW-2227, hit exactly 2GB on the nose
         df = pd.DataFrame({
-            'strings': np.array([val] * 4000, dtype=object)
+            'strings': [v1] * 20 + [v2] + ['x'] * 20
         })
         arr = pa.array(df['strings'])
         assert isinstance(arr, pa.ChunkedArray)

From a4307587b48aa8bf23e69b05b52739058302543b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 14 Mar 2018 11:19:30 +0100
Subject: [PATCH 1602/1644] ARROW-2306: [Python] Fix partitioned Parquet test
 against HDFS

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1742 from wesm/ARROW-2306 and squashes the following commits:

6ae75f1 <Wes McKinney> Fix partitioned Parquet test against HDFS
---
 python/pyarrow/hdfs.py               |  7 +++++++
 python/pyarrow/parquet.py            |  3 +++
 python/pyarrow/tests/test_parquet.py | 11 +++++++++--
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/python/pyarrow/hdfs.py b/python/pyarrow/hdfs.py
index 3f2014b65c097..34ddfaef3c47c 100644
--- a/python/pyarrow/hdfs.py
+++ b/python/pyarrow/hdfs.py
@@ -40,6 +40,13 @@ def __reduce__(self):
         return (HadoopFileSystem, (self.host, self.port, self.user,
                                    self.kerb_ticket, self.driver))
 
+    def _isfilestore(self):
+        """
+        Returns True if this FileSystem is a unix-style file store with
+        directories.
+        """
+        return True
+
     @implements(FileSystem.isdir)
     def isdir(self, path):
         return super(HadoopFileSystem, self).isdir(path)
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index fd9c740f14e20..0929a1549f5b6 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -1103,6 +1103,9 @@ def write_metadata(schema, where, version='1.0',
     coerce_timestamps : string, default None
         Cast timestamps a particular resolution.
         Valid values: {None, 'ms', 'us'}
+    filesystem : FileSystem, default None
+        If nothing passed, paths assumed to be found in the local on-disk
+        filesystem
     """
     writer = ParquetWriter(
         where, schema, version=version,
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index a3da05fe36a82..b301de6062f04 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1431,8 +1431,15 @@ def _test_write_to_dataset_with_partitions(base_path, filesystem=None):
     output_table = pa.Table.from_pandas(output_df)
     pq.write_to_dataset(output_table, base_path, partition_by,
                         filesystem=filesystem)
-    pq.write_metadata(output_table.schema,
-                      os.path.join(base_path, '_common_metadata'))
+
+    metadata_path = os.path.join(base_path, '_common_metadata')
+
+    if filesystem is not None:
+        with filesystem.open(metadata_path, 'wb') as f:
+            pq.write_metadata(output_table.schema, f)
+    else:
+        pq.write_metadata(output_table.schema, metadata_path)
+
     dataset = pq.ParquetDataset(base_path, filesystem=filesystem)
     # ARROW-2209: Ensure the dataset schema also includes the partition columns
     dataset_cols = set(dataset.schema.to_arrow_schema().names)

From 385656ca06e11f0f2e63c5f8985deaf16e7a41cb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 14 Mar 2018 11:21:15 +0100
Subject: [PATCH 1603/1644] ARROW-2304: [C++] Fix HDFS MultipleClients unit
 test

This test was failing because the `scratch_dir_` directory did not exist

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1743 from wesm/ARROW-2304 and squashes the following commits:

9804f3b <Wes McKinney> Fix MultipleClients unit test
---
 cpp/src/arrow/io/io-hdfs-test.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 610a91fbc2c9d..e02215b5efbc8 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -181,6 +181,8 @@ TYPED_TEST(TestHadoopFileSystem, ConnectsAgain) {
 TYPED_TEST(TestHadoopFileSystem, MultipleClients) {
   SKIP_IF_NO_DRIVER();
 
+  ASSERT_OK(this->MakeScratchDir());
+
   std::shared_ptr<HadoopFileSystem> client1;
   std::shared_ptr<HadoopFileSystem> client2;
   ASSERT_OK(HadoopFileSystem::Connect(&this->conf_, &client1));
@@ -189,7 +191,7 @@ TYPED_TEST(TestHadoopFileSystem, MultipleClients) {
 
   // client2 continues to function after equivalent client1 has shutdown
   std::vector<HdfsPathInfo> listing;
-  EXPECT_OK(client2->ListDirectory(this->scratch_dir_, &listing));
+  ASSERT_OK(client2->ListDirectory(this->scratch_dir_, &listing));
   ASSERT_OK(client2->Disconnect());
 }
 

From e25e3efc9f70858abcba867a82bd1201c25eea5b Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Wed, 14 Mar 2018 20:50:41 -0400
Subject: [PATCH 1604/1644] ARROW-2307: [Python] Allow reading record batch
 streams with zero record batches

This is a pretty rough edge case -- it would be good to get this fix into 0.9.0.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1747 from wesm/ARROW-2307 and squashes the following commits:

8f88c04a <Wes McKinney> Fix clang documentation warnings
5ab356a9 <Wes McKinney> Pin Cython 0.27.3
2e5774a0 <Wes McKinney> Do not fail when reading from a stream with schema but no record batches
---
 ci/msvc-build.bat                    |  3 ++-
 ci/travis_script_python.sh           |  2 +-
 cpp/src/arrow/table-test.cc          | 11 +++++++++++
 cpp/src/arrow/table.cc               | 18 +++++++++++-------
 cpp/src/arrow/table.h                | 20 ++++++++++++++++++--
 python/pyarrow/includes/libarrow.pxd |  1 +
 python/pyarrow/ipc.pxi               |  6 ++++--
 python/pyarrow/table.pxi             | 25 ++++++++++++++++++++-----
 python/pyarrow/tests/test_ipc.py     | 18 ++++++++++++++++++
 9 files changed, 86 insertions(+), 18 deletions(-)

diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index beefee6c0fc80..a29ef0bad2973 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -69,7 +69,8 @@ if "%JOB%" == "Build_Debug" (
 )
 
 conda create -n arrow -q -y python=%PYTHON% ^
-      six pytest setuptools numpy pandas cython ^
+      six pytest setuptools numpy pandas ^
+      cython=0.27.3 ^
       thrift-cpp=0.11.0
 
 call activate arrow
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index a776c42630816..247d10278047c 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -41,7 +41,7 @@ conda install -y -q pip \
       cloudpickle \
       numpy=1.13.1 \
       pandas \
-      cython
+      cython=0.27.3
 
 # ARROW-2093: PyTorch increases the size of our conda dependency stack
 # significantly, and so we have disabled these tests in Travis CI for now
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 24c8d5e15bef3..b1cf6e59a257c 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -374,6 +374,17 @@ TEST_F(TestTable, FromRecordBatches) {
   ASSERT_RAISES(Invalid, Table::FromRecordBatches({batch1, batch2}, &result));
 }
 
+TEST_F(TestTable, FromRecordBatchesZeroLength) {
+  // ARROW-2307
+  MakeExample1(10);
+
+  std::shared_ptr<Table> result;
+  ASSERT_OK(Table::FromRecordBatches(schema_, {}, &result));
+
+  ASSERT_EQ(0, result->num_rows());
+  ASSERT_TRUE(result->schema()->Equals(*schema_));
+}
+
 TEST_F(TestTable, ConcatenateTables) {
   const int64_t length = 10;
 
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index ed5858624ae30..f6ac6dd3b1ea7 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -297,14 +297,9 @@ std::shared_ptr<Table> Table::Make(const std::shared_ptr<Schema>& schema,
   return std::make_shared<SimpleTable>(schema, arrays, num_rows);
 }
 
-Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>& batches,
+Status Table::FromRecordBatches(const std::shared_ptr<Schema>& schema,
+                                const std::vector<std::shared_ptr<RecordBatch>>& batches,
                                 std::shared_ptr<Table>* table) {
-  if (batches.size() == 0) {
-    return Status::Invalid("Must pass at least one record batch");
-  }
-
-  std::shared_ptr<Schema> schema = batches[0]->schema();
-
   const int nbatches = static_cast<int>(batches.size());
   const int ncolumns = static_cast<int>(schema->num_fields());
 
@@ -332,6 +327,15 @@ Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>&
   return Status::OK();
 }
 
+Status Table::FromRecordBatches(const std::vector<std::shared_ptr<RecordBatch>>& batches,
+                                std::shared_ptr<Table>* table) {
+  if (batches.size() == 0) {
+    return Status::Invalid("Must pass at least one record batch");
+  }
+
+  return FromRecordBatches(batches[0]->schema(), batches, table);
+}
+
 Status ConcatenateTables(const std::vector<std::shared_ptr<Table>>& tables,
                          std::shared_ptr<Table>* table) {
   if (tables.size() == 0) {
diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h
index 7274fca4d23d5..20d027d6a5e96 100644
--- a/cpp/src/arrow/table.h
+++ b/cpp/src/arrow/table.h
@@ -169,9 +169,25 @@ class ARROW_EXPORT Table {
                                      const std::vector<std::shared_ptr<Array>>& arrays,
                                      int64_t num_rows = -1);
 
-  // Construct table from RecordBatch, but only if all of the batch schemas are
-  // equal. Returns Status::Invalid if there is some problem
+  /// \brief Construct table from RecordBatches, using schema supplied by the first
+  /// RecordBatch.
+  ///
+  /// \param[in] batches a std::vector of record batches
+  /// \param[out] table the returned table
+  /// \return Status Returns Status::Invalid if there is some problem
+  static Status FromRecordBatches(
+      const std::vector<std::shared_ptr<RecordBatch>>& batches,
+      std::shared_ptr<Table>* table);
+
+  /// Construct table from RecordBatches, using supplied schema. There may be
+  /// zero record batches
+  ///
+  /// \param[in] schema the arrow::Schema for each batch
+  /// \param[in] batches a std::vector of record batches
+  /// \param[out] table the returned table
+  /// \return Status
   static Status FromRecordBatches(
+      const std::shared_ptr<Schema>& schema,
       const std::vector<std::shared_ptr<RecordBatch>>& batches,
       std::shared_ptr<Table>* table);
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 3d0c02b895859..01a641896f70b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -456,6 +456,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         @staticmethod
         CStatus FromRecordBatches(
+            const shared_ptr[CSchema]& schema,
             const vector[shared_ptr[CRecordBatch]]& batches,
             shared_ptr[CTable]* table)
 
diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi
index a30a228ae878f..f2ba6b6925925 100644
--- a/python/pyarrow/ipc.pxi
+++ b/python/pyarrow/ipc.pxi
@@ -304,7 +304,8 @@ cdef class _RecordBatchReader:
                     break
                 batches.push_back(batch)
 
-            check_status(CTable.FromRecordBatches(batches, &table))
+            check_status(CTable.FromRecordBatches(self.schema.sp_schema,
+                                                  batches, &table))
 
         return pyarrow_wrap_table(table)
 
@@ -386,7 +387,8 @@ cdef class _RecordBatchFileReader:
         with nogil:
             for i in range(nbatches):
                 check_status(self.reader.get().ReadRecordBatch(i, &batches[i]))
-            check_status(CTable.FromRecordBatches(batches, &table))
+            check_status(CTable.FromRecordBatches(self.schema.sp_schema,
+                                                  batches, &table))
 
         return pyarrow_wrap_table(table)
 
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 94041e465cfce..6cfa9873bcf65 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -975,7 +975,7 @@ cdef class Table:
         columns.reserve(K)
 
         for i in range(K):
-            if isinstance(arrays[i], Array):
+            if isinstance(arrays[i], (Array, list)):
                 columns.push_back(
                     make_shared[CColumn](
                         c_schema.get().field(i),
@@ -1003,26 +1003,41 @@ cdef class Table:
         return pyarrow_wrap_table(CTable.Make(c_schema, columns))
 
     @staticmethod
-    def from_batches(batches):
+    def from_batches(batches, Schema schema=None):
         """
         Construct a Table from a list of Arrow RecordBatches
 
         Parameters
         ----------
-
         batches: list of RecordBatch
-            RecordBatch list to be converted, schemas must be equal
+            RecordBatch list to be converted, all schemas must be equal
+        schema : Schema, default None
+            If not passed, will be inferred from the first RecordBatch
+
+        Returns
+        -------
+        table : Table
         """
         cdef:
             vector[shared_ptr[CRecordBatch]] c_batches
             shared_ptr[CTable] c_table
+            shared_ptr[CSchema] c_schema
             RecordBatch batch
 
         for batch in batches:
             c_batches.push_back(batch.sp_batch)
 
+        if schema is None:
+            if len(batches) == 0:
+                raise ValueError('Must pass schema, or at least '
+                                 'one RecordBatch')
+            c_schema = c_batches[0].get().schema()
+        else:
+            c_schema = schema.sp_schema
+
         with nogil:
-            check_status(CTable.FromRecordBatches(c_batches, &c_table))
+            check_status(CTable.FromRecordBatches(c_schema, c_batches,
+                                                  &c_table))
 
         return pyarrow_wrap_table(c_table)
 
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 9cd5f807662d6..c31c32251d02b 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -379,6 +379,24 @@ def test_ipc_zero_copy_numpy():
     assert_frame_equal(df, rdf)
 
 
+def test_ipc_stream_no_batches():
+    # ARROW-2307
+    table = pa.Table.from_arrays([pa.array([1, 2, 3, 4]),
+                                  pa.array(['foo', 'bar', 'baz', 'qux'])],
+                                 names=['a', 'b'])
+
+    sink = pa.BufferOutputStream()
+    writer = pa.RecordBatchStreamWriter(sink, table.schema)
+    writer.close()
+
+    source = sink.get_result()
+    reader = pa.open_stream(source)
+    result = reader.read_all()
+
+    assert result.schema.equals(table.schema)
+    assert len(result) == 0
+
+
 def test_get_record_batch_size():
     N = 10
     itemsize = 8

From 98012cb32fd5f25af7d12d47ae29aba60878db8a Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Thu, 15 Mar 2018 00:50:01 -0400
Subject: [PATCH 1605/1644] ARROW-2312: [JS] run test_js before
 test_integration

Author: Paul Taylor <paul.e.taylor@me.com>

Closes #1751 from trxcllnt/ARROW-2312 and squashes the following commits:

ac4759e2 <Paul Taylor> run test_js before test_integration
---
 dev/release/verify-release-candidate.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index cb9b01b37e595..0b278e7cfd5ce 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -246,12 +246,11 @@ cd ${DIST_NAME}
 test_package_java
 setup_miniconda
 test_and_install_cpp
+test_js
 test_integration
 test_glib
 install_parquet_cpp
 test_python
 
-test_js
-
 echo 'Release candidate looks good!'
 exit 0

From b185951a6fd04bbd502b6cbc0ebc61574189fe9d Mon Sep 17 00:00:00 2001
From: Kouhei Sutou <kou@clear-code.com>
Date: Thu, 15 Mar 2018 10:09:08 +0100
Subject: [PATCH 1606/1644] ARROW-2313: [C++] Add -NDEBUG flag to arrow.pc

Arrow C++ users should use the same -NDEBUG flag as Arrow C++ itself.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #1752 from kou/cpp-ndebug and squashes the following commits:

6a48486 <Kouhei Sutou>  Add -NDEBUG flag to arrow.pc
---
 cpp/cmake_modules/SetupCxxFlags.cmake | 6 ++++++
 cpp/src/arrow/arrow.pc.in             | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index e21c549d90ee1..28d903f3d3825 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -236,4 +236,10 @@ else()
   message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
 endif ()
 
+if ("${CMAKE_CXX_FLAGS}" MATCHES "-DNDEBUG")
+  set(ARROW_DEFINITION_FLAGS "-DNDEBUG")
+else()
+  set(ARROW_DEFINITION_FLAGS "")
+endif()
+
 message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}")
diff --git a/cpp/src/arrow/arrow.pc.in b/cpp/src/arrow/arrow.pc.in
index fc16267c7bcc8..cae815a001923 100644
--- a/cpp/src/arrow/arrow.pc.in
+++ b/cpp/src/arrow/arrow.pc.in
@@ -25,4 +25,4 @@ Name: Apache Arrow
 Description: Arrow is a set of technologies that enable big-data systems to process and move data fast.
 Version: @ARROW_VERSION@
 Libs: -L${libdir} -larrow
-Cflags: -I${includedir}
+Cflags: -I${includedir} @ARROW_DEFINITION_FLAGS@

From 630ce5ec57fb05bc696f47f78de9c597f92214dc Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 15 Mar 2018 11:49:36 -0400
Subject: [PATCH 1607/1644] ARROW-2311: [Python/C++] Fix struct array slicing

`StructArray::field()` would not adjust the offset if the struct array was sliced, requiring error-prone fixup code in the caller.

Author: Antoine Pitrou <antoine@python.org>

Closes #1754 from pitrou/ARROW-2311-struct-array-slicing and squashes the following commits:

f41ec7e3 <Antoine Pitrou> ARROW-2311:  Fix struct array slicing
---
 cpp/src/arrow/array-test.cc          | 82 +++++++++++++++++++++++-----
 cpp/src/arrow/array.cc               |  8 ++-
 cpp/src/arrow/array.h                |  5 +-
 cpp/src/arrow/compare.cc             | 12 ++--
 cpp/src/arrow/ipc/writer.cc          |  4 --
 cpp/src/arrow/pretty_print.cc        |  5 +-
 python/pyarrow/includes/libarrow.pxd |  1 -
 python/pyarrow/scalar.pxi            | 10 +++-
 python/pyarrow/tests/test_array.py   |  9 +++
 9 files changed, 103 insertions(+), 33 deletions(-)

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index ad2335fa11240..2aa73a09ae033 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -2983,14 +2983,14 @@ TEST_F(TestStructBuilder, TestEquality) {
   std::shared_ptr<Array> unequal_bitmap_array, unequal_offsets_array,
       unequal_values_array;
 
-  vector<int32_t> int_values = {1, 2, 3, 4};
+  vector<int32_t> int_values = {101, 102, 103, 104};
   vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
   vector<int> list_lengths = {3, 0, 3, 4};
   vector<int> list_offsets = {0, 3, 3, 6};
   vector<uint8_t> list_is_valid = {1, 0, 1, 1};
   vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
 
-  vector<int32_t> unequal_int_values = {4, 2, 3, 1};
+  vector<int32_t> unequal_int_values = {104, 102, 103, 101};
   vector<char> unequal_list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'l', 'u', 'c', 'y'};
   vector<int> unequal_list_offsets = {0, 3, 4, 6};
   vector<uint8_t> unequal_list_is_valid = {1, 1, 1, 1};
@@ -3104,29 +3104,85 @@ TEST_F(TestStructBuilder, TestEquality) {
   EXPECT_FALSE(array->RangeEquals(0, 1, 0, unequal_values_array));
   EXPECT_TRUE(array->RangeEquals(1, 3, 1, unequal_values_array));
   EXPECT_FALSE(array->RangeEquals(3, 4, 3, unequal_values_array));
+}
 
-  // ARROW-33 Slice / equality
-  std::shared_ptr<Array> slice, slice2;
+TEST_F(TestStructBuilder, TestZeroLength) {
+  // All buffers are null
+  Done();
+  ASSERT_OK(ValidateArray(*result_));
+}
 
-  slice = array->Slice(2);
-  slice2 = array->Slice(2);
+TEST_F(TestStructBuilder, TestSlice) {
+  std::shared_ptr<Array> array, equal_array;
+  std::shared_ptr<Array> unequal_bitmap_array, unequal_offsets_array,
+      unequal_values_array;
+
+  vector<int32_t> int_values = {101, 102, 103, 104};
+  vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+  vector<int> list_lengths = {3, 0, 3, 4};
+  vector<int> list_offsets = {0, 3, 3, 6};
+  vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+  vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+  ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0));
+  Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder());
+  Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1));
+  ASSERT_OK(builder_->Reserve(list_lengths.size()));
+  ASSERT_OK(char_vb->Reserve(list_values.size()));
+  ASSERT_OK(int_vb->Reserve(int_values.size()));
+
+  ASSERT_OK(builder_->Append(struct_is_valid.size(), struct_is_valid.data()));
+  ASSERT_OK(
+      list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data()));
+  for (int8_t value : list_values) {
+    char_vb->UnsafeAppend(value);
+  }
+  for (int32_t value : int_values) {
+    int_vb->UnsafeAppend(value);
+  }
+  ASSERT_OK(builder_->Finish(&array));
+
+  std::shared_ptr<StructArray> slice, slice2;
+  std::shared_ptr<Int32Array> int_field;
+  std::shared_ptr<ListArray> list_field;
+
+  slice = std::dynamic_pointer_cast<StructArray>(array->Slice(2));
+  slice2 = std::dynamic_pointer_cast<StructArray>(array->Slice(2));
   ASSERT_EQ(array->length() - 2, slice->length());
 
   ASSERT_TRUE(slice->Equals(slice2));
   ASSERT_TRUE(array->RangeEquals(2, slice->length(), 0, slice));
 
-  slice = array->Slice(1, 2);
-  slice2 = array->Slice(1, 2);
+  int_field = std::dynamic_pointer_cast<Int32Array>(slice->field(1));
+  ASSERT_EQ(int_field->length(), slice->length());
+  ASSERT_EQ(int_field->Value(0), 103);
+  ASSERT_EQ(int_field->Value(1), 104);
+  ASSERT_EQ(int_field->null_count(), 0);
+  list_field = std::dynamic_pointer_cast<ListArray>(slice->field(0));
+  ASSERT_FALSE(list_field->IsNull(0));
+  ASSERT_FALSE(list_field->IsNull(1));
+  ASSERT_EQ(list_field->value_length(0), 3);
+  ASSERT_EQ(list_field->value_length(1), 4);
+  ASSERT_EQ(list_field->null_count(), 0);
+
+  slice = std::dynamic_pointer_cast<StructArray>(array->Slice(1, 2));
+  slice2 = std::dynamic_pointer_cast<StructArray>(array->Slice(1, 2));
   ASSERT_EQ(2, slice->length());
 
   ASSERT_TRUE(slice->Equals(slice2));
   ASSERT_TRUE(array->RangeEquals(1, 3, 0, slice));
-}
 
-TEST_F(TestStructBuilder, TestZeroLength) {
-  // All buffers are null
-  Done();
-  ASSERT_OK(ValidateArray(*result_));
+  int_field = std::dynamic_pointer_cast<Int32Array>(slice->field(1));
+  ASSERT_EQ(int_field->length(), slice->length());
+  ASSERT_EQ(int_field->Value(0), 102);
+  ASSERT_EQ(int_field->Value(1), 103);
+  ASSERT_EQ(int_field->null_count(), 0);
+  list_field = std::dynamic_pointer_cast<ListArray>(slice->field(0));
+  ASSERT_TRUE(list_field->IsNull(0));
+  ASSERT_FALSE(list_field->IsNull(1));
+  ASSERT_EQ(list_field->value_length(0), 0);
+  ASSERT_EQ(list_field->value_length(1), 3);
+  ASSERT_EQ(list_field->null_count(), 1);
 }
 
 // ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index bd2b40c1a4923..7e66999a379ae 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -353,7 +353,13 @@ StructArray::StructArray(const std::shared_ptr<DataType>& type, int64_t length,
 
 std::shared_ptr<Array> StructArray::field(int i) const {
   if (!boxed_fields_[i]) {
-    boxed_fields_[i] = MakeArray(data_->child_data[i]);
+    std::shared_ptr<ArrayData> field_data;
+    if (data_->offset != 0 || data_->child_data[i]->length != data_->length) {
+      field_data = SliceData(*data_->child_data[i].get(), data_->offset, data_->length);
+    } else {
+      field_data = data_->child_data[i];
+    }
+    boxed_fields_[i] = MakeArray(field_data);
   }
   DCHECK(boxed_fields_[i]);
   return boxed_fields_[i];
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 04bd0126141be..0a155af7e4a8e 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -149,6 +149,8 @@ struct ARROW_EXPORT ArrayData {
   std::shared_ptr<DataType> type;
   int64_t length;
   int64_t null_count;
+  // The logical start point into the physical buffers (in values, not bytes).
+  // Note that, for child data, this must be *added* to the child data's own offset.
   int64_t offset;
   std::vector<std::shared_ptr<Buffer>> buffers;
   std::vector<std::shared_ptr<ArrayData>> child_data;
@@ -599,7 +601,8 @@ class ARROW_EXPORT StructArray : public Array {
               int64_t offset = 0);
 
   // Return a shared pointer in case the requestor desires to share ownership
-  // with this array.
+  // with this array.  The returned array has its offset, length and null
+  // count adjusted.
   std::shared_ptr<Array> field(int pos) const;
 
  private:
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 69cacbfac818f..002d4b8520a4d 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -136,11 +136,7 @@ class RangeEqualsVisitor {
       for (int j = 0; j < left.num_fields(); ++j) {
         // TODO: really we should be comparing stretches of non-null data rather
         // than looking at one value at a time.
-        const int64_t left_abs_index = i + left.offset();
-        const int64_t right_abs_index = o_i + right.offset();
-
-        equal_fields = left.field(j)->RangeEquals(left_abs_index, left_abs_index + 1,
-                                                  right_abs_index, right.field(j));
+        equal_fields = left.field(j)->RangeEquals(i, i + 1, o_i, right.field(j));
         if (!equal_fields) {
           return false;
         }
@@ -467,9 +463,9 @@ class ArrayEqualsVisitor : public RangeEqualsVisitor {
       return Status::OK();
     }
 
-    result_ =
-        left.values()->RangeEquals(left.value_offset(0), left.value_offset(left.length()),
-                                   right.value_offset(0), right.values());
+    result_ = left.values()->RangeEquals(
+        left.value_offset(0), left.value_offset(left.length()) - left.value_offset(0),
+        right.value_offset(0), right.values());
     return Status::OK();
   }
 
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 3debd767ee029..078efe1b006a1 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -376,10 +376,6 @@ class RecordBatchSerializer : public ArrayVisitor {
     --max_recursion_depth_;
     for (int i = 0; i < array.num_fields(); ++i) {
       std::shared_ptr<Array> field = array.field(i);
-      if (array.offset() != 0 || array.length() < field->length()) {
-        // If offset is non-zero, slice the child array
-        field = field->Slice(array.offset(), array.length());
-      }
       RETURN_NOT_OK(VisitArray(*field));
     }
     ++max_recursion_depth_;
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 994f528ea4bad..38ccb7b505294 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -233,7 +233,8 @@ class ArrayPrinter : public PrettyPrinter {
     Newline();
     Write("-- values: ");
     auto values =
-        array.values()->Slice(array.value_offset(0), array.value_offset(array.length()));
+        array.values()->Slice(array.value_offset(0),
+                              array.value_offset(array.length()) - array.value_offset(0));
     RETURN_NOT_OK(PrettyPrint(*values, indent_ + 2, sink_));
 
     return Status::OK();
@@ -264,7 +265,7 @@ class ArrayPrinter : public PrettyPrinter {
     for (int i = 0; i < array.num_fields(); ++i) {
       children.emplace_back(array.field(i));
     }
-    return PrintChildren(children, array.offset(), array.length());
+    return PrintChildren(children, 0, array.length());
   }
 
   Status Visit(const UnionArray& array) {
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 01a641896f70b..589103635f8bd 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -388,7 +388,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
                      int64_t offset=0)
 
         shared_ptr[CArray] field(int pos)
-        const vector[shared_ptr[CArray]] fields()
 
     CStatus ValidateArray(const CArray& array)
 
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index a801acd69c582..2692ace40a3fb 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -342,6 +342,7 @@ cdef class UnionValue(ArrayValue):
     def as_py(self):
         return self.getitem(self.index).as_py()
 
+
 cdef class FixedSizeBinaryValue(ArrayValue):
 
     def as_py(self):
@@ -358,14 +359,16 @@ cdef class FixedSizeBinaryValue(ArrayValue):
 
 
 cdef class StructValue(ArrayValue):
+
     def as_py(self):
         cdef:
             CStructArray* ap
             vector[shared_ptr[CField]] child_fields = self.type.type.children()
+
         ap = <CStructArray*> self.sp_array.get()
-        wrapped_arrays = (pyarrow_wrap_array(ap.field(i))
-                          for i in range(ap.num_fields()))
-        child_names = (child.get().name() for child in child_fields)
+        wrapped_arrays = [pyarrow_wrap_array(ap.field(i))
+                          for i in range(ap.num_fields())]
+        child_names = [child.get().name() for child in child_fields]
         # Return the struct as a dict
         return {
             frombytes(name): child_array[self.index].as_py()
@@ -373,6 +376,7 @@ cdef class StructValue(ArrayValue):
             zip(child_names, wrapped_arrays)
         }
 
+
 cdef dict _scalar_classes = {
     _Type_BOOL: BooleanValue,
     _Type_UINT8: UInt8Value,
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index fea56862cd38d..6faf4566b1983 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -145,6 +145,15 @@ def test_array_slice():
             assert arr[start:stop].to_pylist() == arr.to_pylist()[start:stop]
 
 
+def test_struct_array_slice():
+    # ARROW-2311: slicing nested arrays needs special care
+    ty = pa.struct([pa.field('a', pa.int8()),
+                    pa.field('b', pa.float32())])
+    arr = pa.array([(1, 2.5), (3, 4.5), (5, 6.5)], type=ty)
+    assert arr[1:].to_pylist() == [{'a': 3, 'b': 4.5},
+                                   {'a': 5, 'b': 6.5}]
+
+
 def test_array_factory_invalid_type():
     arr = np.array([datetime.timedelta(1), datetime.timedelta(2)])
     with pytest.raises(ValueError):

From 019a560c992d09119ca83dd949a8fee24ec741e5 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 15 Mar 2018 11:51:08 -0400
Subject: [PATCH 1608/1644] ARROW-2309: [C++] Use std::make_unsigned

No need for our own reimplementation.

Author: Antoine Pitrou <antoine@python.org>

Closes #1748 from pitrou/ARROW-2309-std-make_unsigned and squashes the following commits:

d4aa567e <Antoine Pitrou> Merge branch 'master' into ARROW-2309-std-make_unsigned
ece4fa48 <Antoine Pitrou> ARROW-2309:  Use std::make_unsigned
---
 cpp/src/arrow/util/bit-util.h | 35 +++++++----------------------------
 1 file changed, 7 insertions(+), 28 deletions(-)

diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index aed3a0184a48b..6649d45d3de41 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -55,6 +55,7 @@
 #include <cstdint>
 #include <limits>
 #include <memory>
+#include <type_traits>
 #include <vector>
 
 #include "arrow/util/macros.h"
@@ -68,34 +69,12 @@
 
 namespace arrow {
 
-// TODO(wesm): The source from Impala was depending on boost::make_unsigned
-//
-// We add a partial stub implementation here
-
 namespace detail {
 
-template <typename T>
-struct make_unsigned {};
-
-template <>
-struct make_unsigned<int8_t> {
-  typedef uint8_t type;
-};
-
-template <>
-struct make_unsigned<int16_t> {
-  typedef uint16_t type;
-};
-
-template <>
-struct make_unsigned<int32_t> {
-  typedef uint32_t type;
-};
-
-template <>
-struct make_unsigned<int64_t> {
-  typedef uint64_t type;
-};
+template <typename Integer>
+typename std::make_unsigned<Integer>::type as_unsigned(Integer x) {
+  return static_cast<typename std::make_unsigned<Integer>::type>(x);
+}
 
 }  // namespace detail
 
@@ -268,7 +247,7 @@ static inline int Popcount(uint64_t x) {
 template <typename T>
 static inline int PopcountSigned(T v) {
   // Converting to same-width unsigned then extending preserves the bit pattern.
-  return BitUtil::Popcount(static_cast<typename detail::make_unsigned<T>::type>(v));
+  return BitUtil::Popcount(detail::as_unsigned(v));
 }
 
 /// Returns the 'num_bits' least-significant bits of 'v'.
@@ -412,7 +391,7 @@ static inline T FromLittleEndian(T value) {
 template <typename T>
 static T ShiftRightLogical(T v, int shift) {
   // Conversion to unsigned ensures most significant bits always filled with 0's
-  return static_cast<typename detail::make_unsigned<T>::type>(v) >> shift;
+  return detail::as_unsigned(v) >> shift;
 }
 
 void FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits);

From 60749b2d2b7c98559f985912936c535229f89093 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 15 Mar 2018 13:32:38 -0400
Subject: [PATCH 1609/1644] ARROW-2316: [C++] Revert Buffer::mutable_data to
 inline so that linkers do not have to remember to define NDEBUG for release
 builds

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1756 from wesm/revert-mutable-data-release-build and squashes the following commits:

722273fe <Wes McKinney> clang-format
ce1e7dd0 <Wes McKinney> Add debug-only mutability check to Buffer::mutable_data
51098b48 <Wes McKinney> Revert Buffer::mutable_data to inline so that linkers do not have to remember to build with -DNDEBUG
---
 cpp/src/arrow/buffer.cc |  8 +-------
 cpp/src/arrow/buffer.h  | 12 ++++++++----
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
index e32e02c9ffdd7..60dd154e4e23c 100644
--- a/cpp/src/arrow/buffer.cc
+++ b/cpp/src/arrow/buffer.cc
@@ -70,13 +70,7 @@ Status Buffer::FromString(const std::string& data, std::shared_ptr<Buffer>* out)
   return FromString(data, default_memory_pool(), out);
 }
 
-#ifndef NDEBUG
-// DCHECK macros aren't allowed in public include files
-uint8_t* Buffer::mutable_data() {
-  DCHECK(is_mutable());
-  return mutable_data_;
-}
-#endif
+void Buffer::CheckMutable() const { DCHECK(is_mutable()) << "buffer not mutable"; }
 
 PoolBuffer::PoolBuffer(MemoryPool* pool) : ResizableBuffer(nullptr, 0) {
   if (pool == nullptr) {
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
index ad11ff943d448..06160d7d43848 100644
--- a/cpp/src/arrow/buffer.h
+++ b/cpp/src/arrow/buffer.h
@@ -117,11 +117,13 @@ class ARROW_EXPORT Buffer {
 
   int64_t capacity() const { return capacity_; }
   const uint8_t* data() const { return data_; }
-#ifdef NDEBUG
-  uint8_t* mutable_data() { return mutable_data_; }
-#else
-  uint8_t* mutable_data();
+
+  uint8_t* mutable_data() {
+#ifndef NDEBUG
+    CheckMutable();
 #endif
+    return mutable_data_;
+  }
 
   int64_t size() const { return size_; }
 
@@ -137,6 +139,8 @@ class ARROW_EXPORT Buffer {
   // null by default, but may be set
   std::shared_ptr<Buffer> parent_;
 
+  void CheckMutable() const;
+
  private:
   ARROW_DISALLOW_COPY_AND_ASSIGN(Buffer);
 };

From 20ea7817464ca16b75addb49ec7a5fdfe625e0da Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@users.noreply.github.com>
Date: Thu, 15 Mar 2018 16:57:22 -0400
Subject: [PATCH 1610/1644] [Python] Pin Cython to 0.27.3 in
 verify-release-candidate.sh (#1758)

Change-Id: I1e6ddc19a8ca9896e5a434011bd1faba1f24be12
---
 dev/release/verify-release-candidate.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 0b278e7cfd5ce..502b45bf8bd46 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -100,7 +100,11 @@ setup_miniconda() {
   export PATH=$MINICONDA/bin:$PATH
 
   conda create -n arrow-test -y -q python=3.6 \
-        nomkl numpy pandas six cython
+        nomkl \
+        numpy \
+        pandas \
+        six \
+        cython=0.27.3
   source activate arrow-test
 }
 
@@ -121,7 +125,7 @@ test_and_install_cpp() {
   make -j$NPROC
   make install
 
-  ctest -L unittest
+  ctest -VV -L unittest
   popd
 }
 

From e29df7d3a2c68eb52b132818f8082bf543980be1 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 16 Mar 2018 15:34:00 +0100
Subject: [PATCH 1611/1644] ARROW-2320: [C++] Vendored Boost build does not
 build regex library

I can confirm that arrow failed to build with `-DARROW_BOOST_VENDORED=ON` and
builds successfully with this patch.

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1760 from cpcloud/ARROW-2320 and squashes the following commits:

3deb24e <Phillip Cloud> ARROW-2320:  Vendored Boost build does not build regex library
---
 cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index c330e2ae3aefa..be9d55c531d40 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -158,7 +158,7 @@ if (ARROW_BOOST_VENDORED)
   set(BOOST_STATIC_FILESYSTEM_LIBRARY
     "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(BOOST_STATIC_REGEX_LIBRARY
-          "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(BOOST_SYSTEM_LIBRARY "${BOOST_STATIC_SYSTEM_LIBRARY}")
   set(BOOST_FILESYSTEM_LIBRARY "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
   set(BOOST_REGEX_LIBRARY "${BOOST_STATIC_REGEX_LIBRARY}")
@@ -174,7 +174,7 @@ if (ARROW_BOOST_VENDORED)
     set(BOOST_CONFIGURE_COMMAND
       "./bootstrap.sh"
       "--prefix=${BOOST_PREFIX}"
-      "--with-libraries=filesystem,system")
+      "--with-libraries=filesystem,system,regex")
     if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
       set(BOOST_BUILD_VARIANT "debug")
     else()

From 79e19c3b858d35c6fc9eb8afacd4ea5cc086acaa Mon Sep 17 00:00:00 2001
From: Wes McKinney <wesm@users.noreply.github.com>
Date: Fri, 16 Mar 2018 12:17:53 -0400
Subject: [PATCH 1612/1644] [JS] Small fixes to source release workflow and
 e-mail template (#1750)

Change-Id: I381465e5bd5bc84851dfcb34e47ff2c284f14efc
---
 dev/release/README               | 9 +++++----
 dev/release/js-source-release.sh | 9 ++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/dev/release/README b/dev/release/README
index 8e0104ed83318..7868c9438e733 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -55,10 +55,11 @@ Once validation has passed, click the "release" button
 * Make release branch then tag the release
 
 git checkout -b release-js-X.Y.Z
-git tag -a apache-arrow-js-X.Y.Z
 
-* Build the source release (requires NodeJS)
+* Build the source release (requires NodeJS) and push tag. Omit "-p" for a dry
+  run
 
-dev/release/js-source-release.sh X.Y.Z $RC_NUM
+dev/release/js-source-release.sh -p X.Y.Z $RC_NUM
+git push apache apache-arrow-js-X.Y.Z
 
-* After release vote push tag, and rebase master on release branch
\ No newline at end of file
+* After release vote, rebase master on release branch
\ No newline at end of file
diff --git a/dev/release/js-source-release.sh b/dev/release/js-source-release.sh
index 292869db69d09..49d28582a9bf3 100755
--- a/dev/release/js-source-release.sh
+++ b/dev/release/js-source-release.sh
@@ -154,10 +154,10 @@ Subject: [VOTE] Release Apache Arrow JS ${js_version} - RC${rc}
 
 Hello all,
 
-I\'d like to propose the following release candidate (rc${rc}) of Apache Arrow
-JavaScript version ${js_version}.
+I would like to propose the following release candidate (rc${rc}) of Apache
+Arrow JavaScript version ${js_version}.
 
-The source release rc{$rc} is hosted at [1].
+The source release rc${rc} is hosted at [1].
 
 This release candidate is based on commit
 ${release_hash}
@@ -166,8 +166,7 @@ Please download, verify checksums and signatures, run the unit tests, and vote
 on the release. The easiest way is to use the JavaScript-specific release
 verification script dev/release/js-verify-release-candidate.sh.
 
-The vote will be open for at least 72 hours and will close once
-enough PMCs have approved the release.
+The vote will be open for at least 72 hours.
 
 [ ] +1 Release this as Apache Arrow JavaScript ${js_version}
 [ ] +0

From 82c8b6fc8fa1e87722fd50230b5f7589b622487c Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Fri, 16 Mar 2018 12:27:06 -0400
Subject: [PATCH 1613/1644] ARROW-2318: [Plasma] Run plasma store tests with
 unique socket

It seems like using the same store socket name for different tests introduced race conditions between connecting clients and starting/stopping the plasma store for each tests. This PR should fix it.

This should fix https://issues.apache.org/jira/browse/ARROW-2318

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #1761 from pcmoritz/fix-plasma-test and squashes the following commits:

ee53c31d <Philipp Moritz> fix linting
381410cb <Philipp Moritz> run plasma store tests with unique socket
---
 cpp/src/plasma/test/client_tests.cc | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
index 15f9e7cd111e4..07e0f9c6a6ba2 100644
--- a/cpp/src/plasma/test/client_tests.cc
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -22,6 +22,8 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <random>
+
 #include "plasma/client.h"
 #include "plasma/common.h"
 #include "plasma/plasma.h"
@@ -38,14 +40,20 @@ class TestPlasmaStore : public ::testing::Test {
   // TODO(pcm): At the moment, stdout of the test gets mixed up with
   // stdout of the object store. Consider changing that.
   void SetUp() {
+    std::mt19937 rng;
+    rng.seed(std::random_device()());
+    std::string store_index = std::to_string(rng());
+
     std::string plasma_directory =
         test_executable.substr(0, test_executable.find_last_of("/"));
-    std::string plasma_command =
-        plasma_directory +
-        "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &";
+    std::string plasma_command = plasma_directory +
+                                 "/plasma_store -m 1000000000 -s /tmp/store" +
+                                 store_index + " 1> /dev/null 2> /dev/null &";
     system(plasma_command.c_str());
-    ARROW_CHECK_OK(client_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
-    ARROW_CHECK_OK(client2_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
+    ARROW_CHECK_OK(
+        client_.Connect("/tmp/store" + store_index, "", PLASMA_DEFAULT_RELEASE_DELAY));
+    ARROW_CHECK_OK(
+        client2_.Connect("/tmp/store" + store_index, "", PLASMA_DEFAULT_RELEASE_DELAY));
   }
   virtual void Finish() {
     ARROW_CHECK_OK(client_.Disconnect());

From 95ba6ef97429eb24bcfc9a65bf78f5743d5e978e Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Fri, 16 Mar 2018 12:28:22 -0400
Subject: [PATCH 1614/1644] ARROW-2321: [C++] Release verification script fails
 with if CMAKE_INSTALL_LIBDIR is not $ARROW_HOME/lib

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1762 from cpcloud/ARROW-2321 and squashes the following commits:

e1bf3831 <Phillip Cloud> ARROW-2321:  Release verification script fails with if CMAKE_INSTALL_LIBDIR is not $ARROW_HOME/lib
---
 dev/release/verify-release-candidate.bat | 2 +-
 dev/release/verify-release-candidate.sh  | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat
index 27157da6687a2..bc05b23a9f76b 100644
--- a/dev/release/verify-release-candidate.bat
+++ b/dev/release/verify-release-candidate.bat
@@ -28,7 +28,7 @@
 @rem       cmake ^
 @rem       git ^
 @rem       boost-cpp ^
-@rem       snappy zlib brotli gflags lz4-c zstd || exit /B
+@rem       snappy zlib brotli gflags lz4-c zstd -c conda-forge || exit /B
 
 @rem Then run from the directory containing the RC tarball
 @rem
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 502b45bf8bd46..34aff209aa761 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -104,7 +104,7 @@ setup_miniconda() {
         numpy \
         pandas \
         six \
-        cython=0.27.3
+        cython=0.27.3 -c conda-forge
   source activate arrow-test
 }
 
@@ -115,6 +115,7 @@ test_and_install_cpp() {
   pushd cpp/build
 
   cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+        -DCMAKE_INSTALL_LIBDIR=$ARROW_HOME/lib \
         -DARROW_PLASMA=on \
         -DARROW_PYTHON=on \
         -DARROW_BOOST_USE_SHARED=on \
@@ -138,6 +139,7 @@ install_parquet_cpp() {
   pushd parquet-cpp/build
 
   cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+        -DCMAKE_INSTALL_LIBDIR=$PARQUET_HOME/lib \
         -DCMAKE_BUILD_TYPE=release \
         -DPARQUET_BOOST_USE_SHARED=on \
         -DPARQUET_BUILD_TESTS=off \

From 7be8d375df1a9c3988aaf2528ff8eee9677c698a Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 16 Mar 2018 12:53:24 -0400
Subject: [PATCH 1615/1644] [Release] Update CHANGELOG.md for 0.9.0

Change-Id: I8c3682711f4b8ec0106de1233d50843a40b078a5
---
 CHANGELOG.md | 269 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 269 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 153159cb779e4..b38e2b5398a2b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,274 @@
   under the License.
 -->
 
+# Apache Arrow 0.9.0 (16 March 2018)
+
+## Bug
+
+* ARROW-1345 - [Python] Conversion from nested NumPy arrays fails on integers other than int64, float32
+* ARROW-1646 - [Python] pyarrow.array cannot handle NumPy scalar types
+* ARROW-1856 - [Python] Auto-detect Parquet ABI version when using PARQUET\_HOME
+* ARROW-1909 - [C++] Bug: Build fails on windows with "-DARROW\_BUILD\_BENCHMARKS=ON"
+* ARROW-1912 - [Website] Add org affiliations to committers.html
+* ARROW-1919 - Plasma hanging if object id is not 20 bytes
+* ARROW-1924 - [Python] Bring back pickle=True option for serialization
+* ARROW-1933 - [GLib] Build failure with --with-arrow-cpp-build-dir and GPU enabled Arrow C++
+* ARROW-1940 - [Python] Extra metadata gets added after multiple conversions between pd.DataFrame and pa.Table
+* ARROW-1941 - Table <–> DataFrame roundtrip failing
+* ARROW-1943 - Handle setInitialCapacity() for deeply nested lists of lists
+* ARROW-1944 - FindArrow has wrong ARROW\_STATIC\_LIB
+* ARROW-1945 - [C++] Fix doxygen documentation of array.h
+* ARROW-1946 - Add APIs to decimal vector for writing big endian data
+* ARROW-1948 - [Java] ListVector does not handle ipc with all non-null values with none set
+* ARROW-1950 - [Python] pandas\_type in pandas metadata incorrect for List types
+* ARROW-1953 - [JS] JavaScript builds broken on master
+* ARROW-1958 - [Python] Error in pandas conversion for datetimetz row index
+* ARROW-1961 - [Python] Writing Parquet file with flavor='spark' loses pandas schema metadata
+* ARROW-1966 - [C++] Support JAVA\_HOME paths in HDFS libjvm loading that include the jre directory
+* ARROW-1971 - [Python] Add pandas serialization to the default
+* ARROW-1972 - Deserialization of buffer objects (and pandas dataframes) segfaults on different processes.
+* ARROW-1973 - [Python] Memory leak when converting Arrow tables with array columns to Pandas dataframes.
+* ARROW-1976 - [Python] Handling unicode pandas columns on parquet.read\_table
+* ARROW-1979 - [JS] JS builds handing in es2015:umd tests
+* ARROW-1980 - [Python] Race condition in `write\_to\_dataset`
+* ARROW-1982 - [Python] Return parquet statistics min/max as values instead of strings
+* ARROW-1991 - [GLib] Docker-based documentation build is broken
+* ARROW-1992 - [Python] to\_pandas crashes when using strings\_to\_categoricals on empty string cols on 0.8.0
+* ARROW-1997 - [Python] to\_pandas with strings\_to\_categorical fails
+* ARROW-1998 - [Python] Table.from\_pandas crashes when data frame is empty
+* ARROW-1999 - [Python] from\_numpy\_dtype returns wrong types
+* ARROW-2000 - Deduplicate file descriptors when plasma store replies to get request.
+* ARROW-2002 - use pyarrow download file will raise queue.Full exceptions sometimes
+* ARROW-2003 - [Python] Do not use deprecated kwarg in pandas.core.internals.make\_block
+* ARROW-2005 - [Python] pyflakes warnings on Cython files not failing build
+* ARROW-2008 - [Python] Type inference for int32 NumPy arrays (expecting list<int32>) returns int64 and then conversion fails
+* ARROW-2010 - [C++] Compiler warnings with CHECKIN warning level in ORC adapter
+* ARROW-2017 - Array initialization with  large (>2**31-1) uint64 values fails
+* ARROW-2023 - [C++] Test opening IPC stream reader or file reader on an empty InputStream
+* ARROW-2025 - [Python/C++] HDFS Client disconnect closes all open clients
+* ARROW-2029 - [Python] Program crash on `HdfsFile.tell` if file is closed
+* ARROW-2032 - [C++] ORC ep installs on each call to ninja build (even if no work to do)
+* ARROW-2033 - pa.array() doesn't work with iterators
+* ARROW-2039 - [Python] pyarrow.Buffer().to\_pybytes() segfaults
+* ARROW-2040 - [Python] Deserialized Numpy array must keep ref to underlying tensor
+* ARROW-2047 - [Python] test\_serialization.py uses a python executable in PATH rather than that used for a test run
+* ARROW-2049 - ARROW-2049: [Python] Use python -m cython to run Cython, instead of CYTHON\_EXECUTABLE
+* ARROW-2062 - [C++] Stalled builds in test\_serialization.py in Travis CI
+* ARROW-2070 - [Python] chdir logic in setup.py buggy
+* ARROW-2072 - [Python] decimal128.byte\_width crashes
+* ARROW-2080 - [Python] Update documentation after ARROW-2024
+* ARROW-2085 - HadoopFileSystem.isdir and .isfile should return False if the path doesn't exist 
+* ARROW-2106 - [Python] pyarrow.array can't take a pandas Series of python datetime objects.
+* ARROW-2109 - [C++] Boost 1.66 compilation fails on Windows on linkage stage
+* ARROW-2124 - [Python] ArrowInvalid raised if the first item of a nested list of numpy arrays is empty
+* ARROW-2128 - [Python] Cannot serialize array of empty lists
+* ARROW-2129 - [Python] Segmentation fault on conversion of empty array to Pandas
+* ARROW-2131 - [Python] Serialization test fails on Windows when library has been built in place / not installed
+* ARROW-2133 - [Python] Segmentation fault on conversion of empty nested arrays to Pandas
+* ARROW-2135 - [Python] NaN values silently casted to int64 when passing explicit schema for conversion in Table.from\_pandas
+* ARROW-2145 - [Python] Decimal conversion not working for NaN values
+* ARROW-2150 - [Python] array equality defaults to identity
+* ARROW-2151 - [Python] Error when converting from list of uint64 arrays
+* ARROW-2153 - [C++/Python] Decimal conversion not working for exponential notation
+* ARROW-2157 - [Python] Decimal arrays cannot be constructed from Python lists
+* ARROW-2160 - [C++/Python] Fix decimal precision inference
+* ARROW-2161 - [Python] Skip test\_cython\_api if ARROW\_HOME isn't defined
+* ARROW-2162 - [Python/C++] Decimal Values with too-high precision are multiplied by 100
+* ARROW-2167 - [C++] Building Orc extensions fails with the default BUILD\_WARNING\_LEVEL=Production
+* ARROW-2170 - [Python] construct\_metadata fails on reading files where no index was preserved
+* ARROW-2171 - [Python] OwnedRef is fragile
+* ARROW-2172 - [Python] Incorrect conversion from Numpy array when stride % itemsize != 0
+* ARROW-2173 - [Python] NumPyBuffer destructor should hold the GIL
+* ARROW-2175 - [Python] arrow\_ep build is triggering during parquet-cpp build in Travis CI
+* ARROW-2178 -  [JS] Fix JS html FileReader example
+* ARROW-2179 - [C++] arrow/util/io-util.h missing from libarrow-dev
+* ARROW-2192 - Commits to master should run all builds in CI matrix
+* ARROW-2209 - [Python] Partition columns are not correctly loaded in schema of ParquetDataset
+* ARROW-2210 - [C++] TestBuffer\_ResizeOOM has a memory leak with jemalloc
+* ARROW-2212 - [C++/Python] Build Protobuf in base manylinux 1 docker image
+* ARROW-2223 - [JS] installing umd release throws an error
+* ARROW-2227 - [Python] Table.from\_pandas does not create chunked\_arrays.
+* ARROW-2230 - [Python] JS version number is sometimes picked up
+* ARROW-2232 - [Python] pyarrow.Tensor constructor segfaults
+* ARROW-2234 - [JS] Read timestamp low bits as Uint32s
+* ARROW-2240 - [Python] Array initialization with leading numpy nan fails with exception
+* ARROW-2244 - [C++] Slicing NullArray should not cause the null count on the internal data to be unknown
+* ARROW-2245 - [Python] Revert static linkage of parquet-cpp in manylinux1 wheel
+* ARROW-2246 - [Python] Use namespaced boost in manylinux1 package
+* ARROW-2251 - [GLib] Destroying GArrowBuffer while GArrowTensor that uses the buffer causes a crash
+* ARROW-2254 - [Python] Local in-place dev versions picking up JS tags
+* ARROW-2258 - [C++] Appveyor builds failing on master
+* ARROW-2263 - [Python] test\_cython.py fails if pyarrow is not in import path (e.g. with inplace builds)
+* ARROW-2265 - [Python] Serializing subclasses of np.ndarray returns a np.ndarray.
+* ARROW-2268 - Remove MD5 checksums from release process
+* ARROW-2269 - [Python] Cannot build bdist\_wheel for Python
+* ARROW-2270 - [Python] ForeignBuffer doesn't tie Python object lifetime to C++ buffer lifetime
+* ARROW-2272 - [Python] test\_plasma spams /tmp
+* ARROW-2275 - [C++] Buffer::mutable\_data\_ member uninitialized
+* ARROW-2280 - [Python] pyarrow.Array.buffers should also include the offsets
+* ARROW-2284 - [Python] test\_plasma error on plasma\_store error
+* ARROW-2288 - [Python] slicing logic defective
+* ARROW-2297 - [JS] babel-jest is not listed as a dev dependency
+* ARROW-2304 - [C++] MultipleClients test in io-hdfs-test fails on trunk
+* ARROW-2306 - [Python] HDFS test failures
+* ARROW-2307 - [Python] Unable to read arrow stream containing 0 record batches
+* ARROW-2311 - [Python] Struct array slicing defective
+* ARROW-2312 - [JS] verify-release-candidate-sh must be updated to include JS in integration tests
+* ARROW-2313 - [GLib] Release builds must define NDEBUG
+* ARROW-2316 - [C++] Revert Buffer::mutable\_data member to always inline
+* ARROW-2318 - [C++] TestPlasmaStore.MultipleClientTest is flaky (hangs) in release builds
+* ARROW-2320 - [C++] Vendored Boost build does not build regex library
+
+## Improvement
+
+* ARROW-1021 - [Python] Add documentation about using pyarrow from other Cython and C++ projects
+* ARROW-1035 - [Python] Add ASV benchmarks for streaming columnar deserialization
+* ARROW-1463 - [JAVA] Restructure ValueVector hierarchy to minimize compile-time generated code
+* ARROW-1579 - [Java] Add dockerized test setup to validate Spark integration
+* ARROW-1580 - [Python] Instructions for setting up nightly builds on Linux
+* ARROW-1623 - [C++] Add convenience method to construct Buffer from a string that owns its memory
+* ARROW-1632 - [Python] Permit categorical conversions in Table.to\_pandas on a per-column basis
+* ARROW-1643 - [Python] Accept hdfs:// prefixes in parquet.read\_table and attempt to connect to HDFS
+* ARROW-1706 - [Python] StructArray.from\_arrays should handle sequences that are coercible to arrays
+* ARROW-1712 - [C++] Add method to BinaryBuilder to reserve space for value data
+* ARROW-1835 - [C++] Create Arrow schema from std::tuple types
+* ARROW-1861 - [Python] Fix up ASV setup, add developer instructions for writing new benchmarks and running benchmark suite locally
+* ARROW-1872 - [Website] Populate hard-coded fields for current release from a YAML file
+* ARROW-1927 - [Plasma] Implement delete function
+* ARROW-1929 - [C++] Move various Arrow testing utility code from Parquet to Arrow codebase
+* ARROW-1937 - [Python] Add documentation for different forms of constructing nested arrays from Python data structures 
+* ARROW-1942 - [C++] Hash table specializations for small integers
+* ARROW-1947 - [Plasma] Change Client Create and Get to use Buffers
+* ARROW-1951 - Add memcopy\_threads to serialization context
+* ARROW-1962 - [Java] Add reset() to ValueVector interface
+* ARROW-1969 - [C++] Do not build ORC adapter by default
+* ARROW-1977 - [C++] Update windows dev docs
+* ARROW-1978 - [Website] Add more visible link to "Powered By" page to front page, simplify Powered By
+* ARROW-2004 - [C++] Add shrink\_to\_fit option in BufferBuilder::Resize
+* ARROW-2007 - [Python] Sequence converter for float32 not implemented
+* ARROW-2011 - Allow setting the pickler to use in pyarrow serialization.
+* ARROW-2012 - [GLib] Support "make distclean"
+* ARROW-2018 - [C++] Build instruction on macOS and Homebrew is incomplete
+* ARROW-2019 - Control the memory allocated for inner vector in LIST
+* ARROW-2024 - [Python] Remove global SerializationContext variables
+* ARROW-2028 - [Python] extra\_cmake\_args needs to be passed through shlex.split
+* ARROW-2031 - HadoopFileSystem isn't pickleable
+* ARROW-2035 - [C++] Update vendored cpplint.py to a Py3-compatible one
+* ARROW-2036 - NativeFile should support standard IOBase methods
+* ARROW-2042 - [Plasma] Revert API change of plasma::Create to output a MutableBuffer
+* ARROW-2043 - [C++] Change description from OS X to macOS
+* ARROW-2046 - [Python] Add support for PEP519 - pathlib and similar objects
+* ARROW-2048 - [Python/C++] Upate Thrift pin to 0.11
+* ARROW-2050 - Support `setup.py pytest` to automatically fetch the test dependencies
+* ARROW-2064 - [GLib] Add common build problems link to the install section
+* ARROW-2065 - Fix bug in SerializationContext.clone().
+* ARROW-2068 - [Python] Expose Array's buffers to Python users
+* ARROW-2069 - [Python] Document that Plasma is not (yet) supported on Windows
+* ARROW-2071 - [Python] Reduce runtime of builds in Travis CI
+* ARROW-2073 - [Python] Create StructArray from sequence of tuples given a known data type
+* ARROW-2076 - [Python] Display slowest test durations
+* ARROW-2083 - Support skipping builds
+* ARROW-2084 - [C++] Support newer Brotli static library names
+* ARROW-2086 - [Python] Shrink size of arrow\_manylinux1\_x86\_64\_base docker image
+* ARROW-2087 - [Python] Binaries of 3rdparty are not stripped in manylinux1 base image
+* ARROW-2088 - [GLib] Add GArrowNumericArray
+* ARROW-2089 - [GLib] Rename to GARROW\_TYPE\_BOOLEAN for consistency
+* ARROW-2090 - [Python] Add context manager methods to ParquetWriter
+* ARROW-2093 - [Python] Possibly do not test pytorch serialization in Travis CI
+* ARROW-2094 - [Python] Use toolchain libraries and PROTOBUF\_HOME for protocol buffers
+* ARROW-2095 - [C++] Suppress ORC EP build logging by default
+* ARROW-2096 - [C++] Turn off Boost\_DEBUG to trim build output
+* ARROW-2099 - [Python] Support DictionaryArray::FromArrays in Python bindings
+* ARROW-2107 - [GLib] Follow arrow::gpu::CudaIpcMemHandle API change
+* ARROW-2110 - [Python] Only require pytest-runner on test commands
+* ARROW-2111 - [C++] Linting could be faster
+* ARROW-2117 - [C++] Pin clang to version 5.0
+* ARROW-2118 - [Python] Improve error message when calling parquet.read\_table on an empty file
+* ARROW-2120 - Add possibility to use empty \_MSVC\_STATIC\_LIB\_SUFFIX for Thirdparties
+* ARROW-2121 - [Python] Consider special casing object arrays in pandas serializers.
+* ARROW-2132 - [Doc] Add links / mentions of Plasma store to main README
+* ARROW-2137 - [Python] Don't print paths that are ignored when reading Parquet files
+* ARROW-2138 - [C++] Have FatalLog abort instead of exiting
+* ARROW-2142 - [Python] Conversion from Numpy struct array unimplemented
+* ARROW-2143 - [Python] Provide a manylinux1 wheel for cp27m
+* ARROW-2146 - [GLib] Implement Slice for ChunkedArray
+* ARROW-2154 - [Python] \_\_eq\_\_ unimplemented on Buffer
+* ARROW-2155 - [Python] pa.frombuffer(bytearray) returns immutable Buffer
+* ARROW-2163 - Install apt dependencies separate from built-in Travis commands, retry on flakiness
+* ARROW-2168 - [C++] Build toolchain builds with jemalloc
+* ARROW-2169 - [C++] MSVC is complaining about uncaptured variables
+* ARROW-2174 - [JS] Export format and schema enums
+* ARROW-2177 - [C++] Remove support for specifying negative scale values in DecimalType
+* ARROW-2180 - [C++] Remove APIs deprecated in 0.8.0 release
+* ARROW-2181 - [Python] Add concat\_tables to API reference, add documentation on use
+* ARROW-2184 - [C++] Add static constructor for FileOutputStream returning shared\_ptr to base OutputStream
+* ARROW-2185 - Remove CI directives from squashed commit messages
+* ARROW-2191 - [C++] Only use specific version of jemalloc
+* ARROW-2198 - [Python] Docstring for parquet.read\_table is misleading or incorrect
+* ARROW-2199 - [JAVA] Follow up fixes for ARROW-2019. Ensure density driven capacity is never less than 1 and propagate density throughout the vector tree
+* ARROW-2203 - [C++] StderrStream class
+* ARROW-2204 - [C++] Build fails with TLS error on parquet-cpp clone
+* ARROW-2206 - [JS] Add Perspective as a community project
+* ARROW-2218 - [Python] PythonFile should infer mode when not given
+* ARROW-2231 - [CI] Use clcache on AppVeyor
+* ARROW-2238 - [C++] Detect clcache in cmake configuration
+* ARROW-2250 - plasma\_store process should cleanup on INT and TERM signals
+* ARROW-2261 - [GLib] Can't share the same memory in GArrowBuffer safely
+* ARROW-2279 - [Python] Better error message if lib cannot be found
+* ARROW-2282 - [Python] Create StringArray from buffers
+* ARROW-2283 - [C++] Support Arrow C++ installed in /usr detection by pkg-config
+* ARROW-2289 - [GLib] Add  Numeric, Integer and FloatingPoint data types
+* ARROW-2291 - [C++] README missing instructions for libboost-regex-dev
+* ARROW-2292 - [Python] More consistent / intuitive name for pyarrow.frombuffer
+* ARROW-2321 - [C++] Release verification script fails with if CMAKE\_INSTALL\_LIBDIR is not $ARROW\_HOME/lib
+* ARROW-764 - [C++] Improve performance of CopyBitmap, add benchmarks
+
+## New Feature
+
+* ARROW-1394 - [Plasma] Add optional extension for allocating memory on GPUs
+* ARROW-1705 - [Python] Create StructArray from sequence of dicts given a known data type
+* ARROW-1757 - [C++] Add DictionaryArray::FromArrays alternate ctor that can check or sanitized "untrusted" indices
+* ARROW-1832 - [JS] Implement JSON reader for integration tests
+* ARROW-1920 - Add support for reading ORC files
+* ARROW-1926 - [GLib] Add garrow\_timestamp\_data\_type\_get\_unit()
+* ARROW-1930 - [C++] Implement Slice for ChunkedArray and Column
+* ARROW-1931 - [C++] w4996 warning due to std::tr1 failing builds on Visual Studio 2017
+* ARROW-1965 - [GLib] Add garrow\_array\_builder\_get\_value\_data\_type() and garrow\_array\_builder\_get\_value\_type()
+* ARROW-1970 - [GLib] Add garrow\_chunked\_array\_get\_value\_data\_type() and garrow\_chunked\_array\_get\_value\_type()
+* ARROW-2166 - [GLib] Implement Slice for Column
+* ARROW-2176 - [C++] Extend DictionaryBuilder to support delta dictionaries
+* ARROW-2190 - [GLib] Add add/remove field functions for RecordBatch.
+* ARROW-2205 - [Python] Option for integer object nulls
+* ARROW-2252 - [Python] Create buffer from address, size and base
+* ARROW-2253 - [Python] Support \_\_eq\_\_ on scalar values
+* ARROW-2262 - [Python] Support slicing on pyarrow.ChunkedArray
+* ARROW-232 - C++/Parquet: Support writing chunked arrays as part of a table 
+* ARROW-633 - [Java] Add support for FixedSizeBinary type
+* ARROW-634 - Add integration tests for FixedSizeBinary
+* ARROW-969 - [C++/Python] Add add/remove field functions for RecordBatch
+
+## Sub-task
+
+* ARROW-1815 - [Java] Rename MapVector to StructVector
+
+## Task
+
+* ARROW-2052 - Unify OwnedRef and ScopedRef
+* ARROW-2054 - Compilation warnings
+* ARROW-2108 - [Python] Update instructions for ASV
+* ARROW-2114 - [Python] Pull latest docker manylinux1 image
+* ARROW-2123 - [JS] Upgrade to TS 2.7.1
+* ARROW-2134 - [CI] Make Travis commit inspection more robust
+* ARROW-2149 - [Python] reorganize test\_convert\_pandas.py
+* ARROW-2156 - [CI] Isolate Sphinx dependencies
+* ARROW-2197 - Document "undefined symbol" issue and workaround
+* ARROW-2239 - [C++] Update build docs for Windows
+* ARROW-2309 - [C++] Use std::make\_unsigned
+
+## Test
+
+* ARROW-1589 - [C++] Fuzzing for certain input formats
+
 # Apache Arrow 0.8.0 (12 December 2017)
 
 ## Bug
@@ -1576,3 +1844,4 @@
 
 
 
+

From c695a5ddc8d26c977b5ecd0c55212e900726953e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 16 Mar 2018 12:56:31 -0400
Subject: [PATCH 1616/1644] [maven-release-plugin] prepare release
 apache-arrow-0.9.0

Change-Id: I825873650c8f736bdfc2533fdb380e6844323a2e
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 3e6582d3fd639..1bd0ebeb3c895 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.9.0-SNAPSHOT</version>
+  <version>0.9.0</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 74ec45c1d2f97..66a8c0056be28 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.9.0-SNAPSHOT</version>
+    <version>0.9.0</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 152deaa9c428e..4fdb56d80993f 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.9.0-SNAPSHOT</version>
+  <version>0.9.0</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -44,7 +44,7 @@
     <connection>scm:git:https://github.com/apache/arrow.git</connection>
     <developerConnection>scm:git:https://github.com/apache/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.7.1</tag>
+    <tag>apache-arrow-0.9.0</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 246f264a6def8..fa7e355a47150 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.9.0-SNAPSHOT</version>
+        <version>0.9.0</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 0f3e03e6902da..0ebff2a6181c3 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.9.0-SNAPSHOT</version>
+    <version>0.9.0</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From bb17a0d772d3d641f9d434492ff08b7ddb1230c7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 16 Mar 2018 12:56:42 -0400
Subject: [PATCH 1617/1644] [maven-release-plugin] prepare for next development
 iteration

Change-Id: I2a71d5db1bf2e68e652784aeb3aa9877422f5bbd
---
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml        | 4 ++--
 java/tools/pom.xml  | 2 +-
 java/vector/pom.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/java/format/pom.xml b/java/format/pom.xml
index 1bd0ebeb3c895..4ff5a4ee70037 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 <parent>
   <artifactId>arrow-java-root</artifactId>
   <groupId>org.apache.arrow</groupId>
-  <version>0.9.0</version>
+  <version>0.10.0-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 66a8c0056be28..59c14d4afecb7 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.9.0</version>
+    <version>0.10.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>Arrow Memory</name>
diff --git a/java/pom.xml b/java/pom.xml
index 4fdb56d80993f..6bb807f431b7c 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.9.0</version>
+  <version>0.10.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -44,7 +44,7 @@
     <connection>scm:git:https://github.com/apache/arrow.git</connection>
     <developerConnection>scm:git:https://github.com/apache/arrow.git</developerConnection>
     <url>https://github.com/apache/arrow</url>
-    <tag>apache-arrow-0.9.0</tag>
+    <tag>apache-arrow-0.7.1</tag>
   </scm>
 
   <mailingLists>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index fa7e355a47150..964f0c85b1356 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-java-root</artifactId>
-        <version>0.9.0</version>
+        <version>0.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>arrow-tools</artifactId>
     <name>Arrow Tools</name>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 0ebff2a6181c3..5a8915a90afcb 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.9.0</version>
+    <version>0.10.0-SNAPSHOT</version>
   </parent>
   <artifactId>arrow-vector</artifactId>
   <name>Arrow Vectors</name>

From a50ef9fa310da75a75101d868f8c5386e82541f1 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Tue, 20 Mar 2018 19:30:39 -0400
Subject: [PATCH 1618/1644] ARROW-2329: [Website] 0.9.0 release update

cc @wesm

Author: siddharth <siddharth@dremio.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1767 from siddharthteotia/release_website and squashes the following commits:

2e21fd0c <Wes McKinney> Update _data/versions.yml
73e008c1 <siddharth> update install.md
244da6b6 <siddharth> 0.9.0 release update for website
---
 dev/release/changelog.py |   5 +-
 site/_data/versions.yml  |  20 +--
 site/_release/0.9.0.md   | 334 +++++++++++++++++++++++++++++++++++++++
 site/_release/index.md   |   3 +
 site/install.md          |  10 +-
 5 files changed, 355 insertions(+), 17 deletions(-)
 create mode 100644 site/_release/0.9.0.md

diff --git a/dev/release/changelog.py b/dev/release/changelog.py
index 23ac4189da9d9..94c74bd6b0dca 100644
--- a/dev/release/changelog.py
+++ b/dev/release/changelog.py
@@ -43,7 +43,7 @@
                             basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
 
 
-locale.setlocale(locale.LC_ALL, 'en_US.utf8')
+locale.setlocale(locale.LC_ALL, 'en_US.utf-8')
 
 
 def get_issues_for_version(version):
@@ -84,7 +84,8 @@ def format_changelog_website(issues, out):
         'Wish': NEW_FEATURE,
         'Task': NEW_FEATURE,
         'Test': BUGFIX,
-        'Bug': BUGFIX
+        'Bug': BUGFIX,
+        'Sub-task': NEW_FEATURE
     }
 
     issues_by_category = defaultdict(list)
diff --git a/site/_data/versions.yml b/site/_data/versions.yml
index 0d04183868dcf..461480fe505ed 100644
--- a/site/_data/versions.yml
+++ b/site/_data/versions.yml
@@ -17,13 +17,13 @@
 # Blogs and other pages use this data
 #
 current:
-  number: '0.8.0'
-  date: '18 December 2017'
-  git-tag: '1d689e5'
-  github-tag-link: 'https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0'
-  release-notes: 'http://arrow.apache.org/release/0.8.0.html'
-  mirrors: 'https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/'
-  mirrors-tar: 'https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz'
-  java-artifacts: 'http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.8.0%22'
-  asc: 'https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc'
-  sha512: 'https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512'
\ No newline at end of file
+  number: '0.9.0'
+  date: '21 March 2018'
+  git-tag: 'c695a5d'
+  github-tag-link: 'https://github.com/apache/arrow/releases/tag/apache-arrow-0.9.0'
+  release-notes: 'http://arrow.apache.org/release/0.9.0.html'
+  mirrors: 'https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.9.0/'
+  mirrors-tar: 'https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.9.0/apache-arrow-0.9.0.tar.gz'
+  java-artifacts: 'http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.9.0%22'
+  asc: 'https://www.apache.org/dist/arrow/arrow-0.9.0/apache-arrow-0.9.0.tar.gz.asc'
+  sha512: 'https://www.apache.org/dist/arrow/arrow-0.9.0/apache-arrow-0.9.0.tar.gz.sha512'
diff --git a/site/_release/0.9.0.md b/site/_release/0.9.0.md
new file mode 100644
index 0000000000000..1c8f5444d7628
--- /dev/null
+++ b/site/_release/0.9.0.md
@@ -0,0 +1,334 @@
+---
+layout: default
+title: Apache Arrow 0.9.0 Release
+permalink: /release/0.9.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.9.0 (21 March 2018)
+
+This is a major release.
+
+## Download
+
+* [**Source Artifacts**][2]
+* [Git tag][1]
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.8.0..apache-arrow-0.9.0
+    52  Wes McKinney
+    52  Antoine Pitrou
+    25  Uwe L. Korn
+    14  Paul Taylor
+    13  Kouhei Sutou
+    13  Phillip Cloud
+     9  Robert Nishihara
+     9  Korn, Uwe
+     9  Jim Crist
+     8  Brian Hulette
+     7  Philipp Moritz
+     6  Panchen Xue
+     6  yosuke shiro
+     5  Mitar
+     5  Bryan Cutler
+     4  siddharth
+     3  Adam Seibert
+     3  Licht-T
+     3  moriyoshi
+     2  rvernica
+     2  Sidd
+     2  Albert Shieh
+     1  Marco Neumann
+     1  Max Risuhin
+     1  Jin Hai
+     1  Jeffrey Heer
+     1  Jacques Nadeau
+     1  Ehsan Totoni
+     1  Dimitri Vorona
+     1  Chris Bartak
+     1  Simbarashe Nyatsanga
+     1  Cheng Lian
+     1  Viktor Gal
+     1  Andy Grove
+     1  William Paul
+     1  devin-petersohn
+```
+
+# Changelog
+
+## New Features and Improvements
+
+* [ARROW-1021](https://issues.apache.org/jira/browse/ARROW-1021) - [Python] Add documentation about using pyarrow from other Cython and C++ projects
+* [ARROW-1035](https://issues.apache.org/jira/browse/ARROW-1035) - [Python] Add ASV benchmarks for streaming columnar deserialization
+* [ARROW-1394](https://issues.apache.org/jira/browse/ARROW-1394) - [Plasma] Add optional extension for allocating memory on GPUs
+* [ARROW-1463](https://issues.apache.org/jira/browse/ARROW-1463) - [JAVA] Restructure ValueVector hierarchy to minimize compile-time generated code
+* [ARROW-1579](https://issues.apache.org/jira/browse/ARROW-1579) - [Java] Add dockerized test setup to validate Spark integration
+* [ARROW-1580](https://issues.apache.org/jira/browse/ARROW-1580) - [Python] Instructions for setting up nightly builds on Linux
+* [ARROW-1623](https://issues.apache.org/jira/browse/ARROW-1623) - [C++] Add convenience method to construct Buffer from a string that owns its memory
+* [ARROW-1632](https://issues.apache.org/jira/browse/ARROW-1632) - [Python] Permit categorical conversions in Table.to_pandas on a per-column basis
+* [ARROW-1643](https://issues.apache.org/jira/browse/ARROW-1643) - [Python] Accept hdfs:// prefixes in parquet.read_table and attempt to connect to HDFS
+* [ARROW-1705](https://issues.apache.org/jira/browse/ARROW-1705) - [Python] Create StructArray from sequence of dicts given a known data type
+* [ARROW-1706](https://issues.apache.org/jira/browse/ARROW-1706) - [Python] StructArray.from_arrays should handle sequences that are coercible to arrays
+* [ARROW-1712](https://issues.apache.org/jira/browse/ARROW-1712) - [C++] Add method to BinaryBuilder to reserve space for value data
+* [ARROW-1757](https://issues.apache.org/jira/browse/ARROW-1757) - [C++] Add DictionaryArray::FromArrays alternate ctor that can check or sanitized "untrusted" indices
+* [ARROW-1815](https://issues.apache.org/jira/browse/ARROW-1815) - [Java] Rename MapVector to StructVector
+* [ARROW-1832](https://issues.apache.org/jira/browse/ARROW-1832) - [JS] Implement JSON reader for integration tests
+* [ARROW-1835](https://issues.apache.org/jira/browse/ARROW-1835) - [C++] Create Arrow schema from std::tuple types
+* [ARROW-1861](https://issues.apache.org/jira/browse/ARROW-1861) - [Python] Fix up ASV setup, add developer instructions for writing new benchmarks and running benchmark suite locally
+* [ARROW-1872](https://issues.apache.org/jira/browse/ARROW-1872) - [Website] Populate hard-coded fields for current release from a YAML file
+* [ARROW-1920](https://issues.apache.org/jira/browse/ARROW-1920) - Add support for reading ORC files
+* [ARROW-1926](https://issues.apache.org/jira/browse/ARROW-1926) - [GLib] Add garrow_timestamp_data_type_get_unit()
+* [ARROW-1927](https://issues.apache.org/jira/browse/ARROW-1927) - [Plasma] Implement delete function
+* [ARROW-1929](https://issues.apache.org/jira/browse/ARROW-1929) - [C++] Move various Arrow testing utility code from Parquet to Arrow codebase
+* [ARROW-1930](https://issues.apache.org/jira/browse/ARROW-1930) - [C++] Implement Slice for ChunkedArray and Column
+* [ARROW-1931](https://issues.apache.org/jira/browse/ARROW-1931) - [C++] w4996 warning due to std::tr1 failing builds on Visual Studio 2017
+* [ARROW-1937](https://issues.apache.org/jira/browse/ARROW-1937) - [Python] Add documentation for different forms of constructing nested arrays from Python data structures 
+* [ARROW-1942](https://issues.apache.org/jira/browse/ARROW-1942) - [C++] Hash table specializations for small integers
+* [ARROW-1947](https://issues.apache.org/jira/browse/ARROW-1947) - [Plasma] Change Client Create and Get to use Buffers
+* [ARROW-1951](https://issues.apache.org/jira/browse/ARROW-1951) - Add memcopy_threads to serialization context
+* [ARROW-1962](https://issues.apache.org/jira/browse/ARROW-1962) - [Java] Add reset() to ValueVector interface
+* [ARROW-1965](https://issues.apache.org/jira/browse/ARROW-1965) - [GLib] Add garrow_array_builder_get_value_data_type() and garrow_array_builder_get_value_type()
+* [ARROW-1969](https://issues.apache.org/jira/browse/ARROW-1969) - [C++] Do not build ORC adapter by default
+* [ARROW-1970](https://issues.apache.org/jira/browse/ARROW-1970) - [GLib] Add garrow_chunked_array_get_value_data_type() and garrow_chunked_array_get_value_type()
+* [ARROW-1977](https://issues.apache.org/jira/browse/ARROW-1977) - [C++] Update windows dev docs
+* [ARROW-1978](https://issues.apache.org/jira/browse/ARROW-1978) - [Website] Add more visible link to "Powered By" page to front page, simplify Powered By
+* [ARROW-2004](https://issues.apache.org/jira/browse/ARROW-2004) - [C++] Add shrink_to_fit option in BufferBuilder::Resize
+* [ARROW-2007](https://issues.apache.org/jira/browse/ARROW-2007) - [Python] Sequence converter for float32 not implemented
+* [ARROW-2011](https://issues.apache.org/jira/browse/ARROW-2011) - Allow setting the pickler to use in pyarrow serialization.
+* [ARROW-2012](https://issues.apache.org/jira/browse/ARROW-2012) - [GLib] Support "make distclean"
+* [ARROW-2018](https://issues.apache.org/jira/browse/ARROW-2018) - [C++] Build instruction on macOS and Homebrew is incomplete
+* [ARROW-2019](https://issues.apache.org/jira/browse/ARROW-2019) - Control the memory allocated for inner vector in LIST
+* [ARROW-2024](https://issues.apache.org/jira/browse/ARROW-2024) - [Python] Remove global SerializationContext variables
+* [ARROW-2028](https://issues.apache.org/jira/browse/ARROW-2028) - [Python] extra_cmake_args needs to be passed through shlex.split
+* [ARROW-2031](https://issues.apache.org/jira/browse/ARROW-2031) - HadoopFileSystem isn't pickleable
+* [ARROW-2035](https://issues.apache.org/jira/browse/ARROW-2035) - [C++] Update vendored cpplint.py to a Py3-compatible one
+* [ARROW-2036](https://issues.apache.org/jira/browse/ARROW-2036) - NativeFile should support standard IOBase methods
+* [ARROW-2042](https://issues.apache.org/jira/browse/ARROW-2042) - [Plasma] Revert API change of plasma::Create to output a MutableBuffer
+* [ARROW-2043](https://issues.apache.org/jira/browse/ARROW-2043) - [C++] Change description from OS X to macOS
+* [ARROW-2046](https://issues.apache.org/jira/browse/ARROW-2046) - [Python] Add support for PEP519 - pathlib and similar objects
+* [ARROW-2048](https://issues.apache.org/jira/browse/ARROW-2048) - [Python/C++] Upate Thrift pin to 0.11
+* [ARROW-2050](https://issues.apache.org/jira/browse/ARROW-2050) - Support `setup.py pytest` to automatically fetch the test dependencies
+* [ARROW-2052](https://issues.apache.org/jira/browse/ARROW-2052) - Unify OwnedRef and ScopedRef
+* [ARROW-2054](https://issues.apache.org/jira/browse/ARROW-2054) - Compilation warnings
+* [ARROW-2064](https://issues.apache.org/jira/browse/ARROW-2064) - [GLib] Add common build problems link to the install section
+* [ARROW-2065](https://issues.apache.org/jira/browse/ARROW-2065) - Fix bug in SerializationContext.clone().
+* [ARROW-2068](https://issues.apache.org/jira/browse/ARROW-2068) - [Python] Expose Array's buffers to Python users
+* [ARROW-2069](https://issues.apache.org/jira/browse/ARROW-2069) - [Python] Document that Plasma is not (yet) supported on Windows
+* [ARROW-2071](https://issues.apache.org/jira/browse/ARROW-2071) - [Python] Reduce runtime of builds in Travis CI
+* [ARROW-2073](https://issues.apache.org/jira/browse/ARROW-2073) - [Python] Create StructArray from sequence of tuples given a known data type
+* [ARROW-2076](https://issues.apache.org/jira/browse/ARROW-2076) - [Python] Display slowest test durations
+* [ARROW-2083](https://issues.apache.org/jira/browse/ARROW-2083) - Support skipping builds
+* [ARROW-2084](https://issues.apache.org/jira/browse/ARROW-2084) - [C++] Support newer Brotli static library names
+* [ARROW-2086](https://issues.apache.org/jira/browse/ARROW-2086) - [Python] Shrink size of arrow_manylinux1_x86_64_base docker image
+* [ARROW-2087](https://issues.apache.org/jira/browse/ARROW-2087) - [Python] Binaries of 3rdparty are not stripped in manylinux1 base image
+* [ARROW-2088](https://issues.apache.org/jira/browse/ARROW-2088) - [GLib] Add GArrowNumericArray
+* [ARROW-2089](https://issues.apache.org/jira/browse/ARROW-2089) - [GLib] Rename to GARROW_TYPE_BOOLEAN for consistency
+* [ARROW-2090](https://issues.apache.org/jira/browse/ARROW-2090) - [Python] Add context manager methods to ParquetWriter
+* [ARROW-2093](https://issues.apache.org/jira/browse/ARROW-2093) - [Python] Possibly do not test pytorch serialization in Travis CI
+* [ARROW-2094](https://issues.apache.org/jira/browse/ARROW-2094) - [Python] Use toolchain libraries and PROTOBUF_HOME for protocol buffers
+* [ARROW-2095](https://issues.apache.org/jira/browse/ARROW-2095) - [C++] Suppress ORC EP build logging by default
+* [ARROW-2096](https://issues.apache.org/jira/browse/ARROW-2096) - [C++] Turn off Boost_DEBUG to trim build output
+* [ARROW-2099](https://issues.apache.org/jira/browse/ARROW-2099) - [Python] Support DictionaryArray::FromArrays in Python bindings
+* [ARROW-2107](https://issues.apache.org/jira/browse/ARROW-2107) - [GLib] Follow arrow::gpu::CudaIpcMemHandle API change
+* [ARROW-2108](https://issues.apache.org/jira/browse/ARROW-2108) - [Python] Update instructions for ASV
+* [ARROW-2110](https://issues.apache.org/jira/browse/ARROW-2110) - [Python] Only require pytest-runner on test commands
+* [ARROW-2111](https://issues.apache.org/jira/browse/ARROW-2111) - [C++] Linting could be faster
+* [ARROW-2114](https://issues.apache.org/jira/browse/ARROW-2114) - [Python] Pull latest docker manylinux1 image
+* [ARROW-2117](https://issues.apache.org/jira/browse/ARROW-2117) - [C++] Pin clang to version 5.0
+* [ARROW-2118](https://issues.apache.org/jira/browse/ARROW-2118) - [Python] Improve error message when calling parquet.read_table on an empty file
+* [ARROW-2120](https://issues.apache.org/jira/browse/ARROW-2120) - Add possibility to use empty _MSVC_STATIC_LIB_SUFFIX for Thirdparties
+* [ARROW-2121](https://issues.apache.org/jira/browse/ARROW-2121) - [Python] Consider special casing object arrays in pandas serializers.
+* [ARROW-2123](https://issues.apache.org/jira/browse/ARROW-2123) - [JS] Upgrade to TS 2.7.1
+* [ARROW-2132](https://issues.apache.org/jira/browse/ARROW-2132) - [Doc] Add links / mentions of Plasma store to main README
+* [ARROW-2134](https://issues.apache.org/jira/browse/ARROW-2134) - [CI] Make Travis commit inspection more robust
+* [ARROW-2137](https://issues.apache.org/jira/browse/ARROW-2137) - [Python] Don't print paths that are ignored when reading Parquet files
+* [ARROW-2138](https://issues.apache.org/jira/browse/ARROW-2138) - [C++] Have FatalLog abort instead of exiting
+* [ARROW-2142](https://issues.apache.org/jira/browse/ARROW-2142) - [Python] Conversion from Numpy struct array unimplemented
+* [ARROW-2143](https://issues.apache.org/jira/browse/ARROW-2143) - [Python] Provide a manylinux1 wheel for cp27m
+* [ARROW-2146](https://issues.apache.org/jira/browse/ARROW-2146) - [GLib] Implement Slice for ChunkedArray
+* [ARROW-2149](https://issues.apache.org/jira/browse/ARROW-2149) - [Python] reorganize test_convert_pandas.py
+* [ARROW-2154](https://issues.apache.org/jira/browse/ARROW-2154) - [Python] __eq__ unimplemented on Buffer
+* [ARROW-2155](https://issues.apache.org/jira/browse/ARROW-2155) - [Python] pa.frombuffer(bytearray) returns immutable Buffer
+* [ARROW-2156](https://issues.apache.org/jira/browse/ARROW-2156) - [CI] Isolate Sphinx dependencies
+* [ARROW-2163](https://issues.apache.org/jira/browse/ARROW-2163) - Install apt dependencies separate from built-in Travis commands, retry on flakiness
+* [ARROW-2166](https://issues.apache.org/jira/browse/ARROW-2166) - [GLib] Implement Slice for Column
+* [ARROW-2168](https://issues.apache.org/jira/browse/ARROW-2168) - [C++] Build toolchain builds with jemalloc
+* [ARROW-2169](https://issues.apache.org/jira/browse/ARROW-2169) - [C++] MSVC is complaining about uncaptured variables
+* [ARROW-2174](https://issues.apache.org/jira/browse/ARROW-2174) - [JS] Export format and schema enums
+* [ARROW-2176](https://issues.apache.org/jira/browse/ARROW-2176) - [C++] Extend DictionaryBuilder to support delta dictionaries
+* [ARROW-2177](https://issues.apache.org/jira/browse/ARROW-2177) - [C++] Remove support for specifying negative scale values in DecimalType
+* [ARROW-2180](https://issues.apache.org/jira/browse/ARROW-2180) - [C++] Remove APIs deprecated in 0.8.0 release
+* [ARROW-2181](https://issues.apache.org/jira/browse/ARROW-2181) - [Python] Add concat_tables to API reference, add documentation on use
+* [ARROW-2184](https://issues.apache.org/jira/browse/ARROW-2184) - [C++] Add static constructor for FileOutputStream returning shared_ptr to base OutputStream
+* [ARROW-2185](https://issues.apache.org/jira/browse/ARROW-2185) - Remove CI directives from squashed commit messages
+* [ARROW-2190](https://issues.apache.org/jira/browse/ARROW-2190) - [GLib] Add add/remove field functions for RecordBatch.
+* [ARROW-2191](https://issues.apache.org/jira/browse/ARROW-2191) - [C++] Only use specific version of jemalloc
+* [ARROW-2197](https://issues.apache.org/jira/browse/ARROW-2197) - Document "undefined symbol" issue and workaround
+* [ARROW-2198](https://issues.apache.org/jira/browse/ARROW-2198) - [Python] Docstring for parquet.read_table is misleading or incorrect
+* [ARROW-2199](https://issues.apache.org/jira/browse/ARROW-2199) - [JAVA] Follow up fixes for ARROW-2019. Ensure density driven capacity is never less than 1 and propagate density throughout the vector tree
+* [ARROW-2203](https://issues.apache.org/jira/browse/ARROW-2203) - [C++] StderrStream class
+* [ARROW-2204](https://issues.apache.org/jira/browse/ARROW-2204) - [C++] Build fails with TLS error on parquet-cpp clone
+* [ARROW-2205](https://issues.apache.org/jira/browse/ARROW-2205) - [Python] Option for integer object nulls
+* [ARROW-2206](https://issues.apache.org/jira/browse/ARROW-2206) - [JS] Add Perspective as a community project
+* [ARROW-2218](https://issues.apache.org/jira/browse/ARROW-2218) - [Python] PythonFile should infer mode when not given
+* [ARROW-2231](https://issues.apache.org/jira/browse/ARROW-2231) - [CI] Use clcache on AppVeyor
+* [ARROW-2238](https://issues.apache.org/jira/browse/ARROW-2238) - [C++] Detect clcache in cmake configuration
+* [ARROW-2239](https://issues.apache.org/jira/browse/ARROW-2239) - [C++] Update build docs for Windows
+* [ARROW-2250](https://issues.apache.org/jira/browse/ARROW-2250) - plasma_store process should cleanup on INT and TERM signals
+* [ARROW-2252](https://issues.apache.org/jira/browse/ARROW-2252) - [Python] Create buffer from address, size and base
+* [ARROW-2253](https://issues.apache.org/jira/browse/ARROW-2253) - [Python] Support __eq__ on scalar values
+* [ARROW-2261](https://issues.apache.org/jira/browse/ARROW-2261) - [GLib] Can't share the same memory in GArrowBuffer safely
+* [ARROW-2262](https://issues.apache.org/jira/browse/ARROW-2262) - [Python] Support slicing on pyarrow.ChunkedArray
+* [ARROW-2279](https://issues.apache.org/jira/browse/ARROW-2279) - [Python] Better error message if lib cannot be found
+* [ARROW-2282](https://issues.apache.org/jira/browse/ARROW-2282) - [Python] Create StringArray from buffers
+* [ARROW-2283](https://issues.apache.org/jira/browse/ARROW-2283) - [C++] Support Arrow C++ installed in /usr detection by pkg-config
+* [ARROW-2289](https://issues.apache.org/jira/browse/ARROW-2289) - [GLib] Add  Numeric, Integer and FloatingPoint data types
+* [ARROW-2291](https://issues.apache.org/jira/browse/ARROW-2291) - [C++] README missing instructions for libboost-regex-dev
+* [ARROW-2292](https://issues.apache.org/jira/browse/ARROW-2292) - [Python] More consistent / intuitive name for pyarrow.frombuffer
+* [ARROW-2309](https://issues.apache.org/jira/browse/ARROW-2309) - [C++] Use std::make_unsigned
+* [ARROW-232](https://issues.apache.org/jira/browse/ARROW-232) - C++/Parquet: Support writing chunked arrays as part of a table 
+* [ARROW-2321](https://issues.apache.org/jira/browse/ARROW-2321) - [C++] Release verification script fails with if CMAKE_INSTALL_LIBDIR is not $ARROW_HOME/lib
+* [ARROW-633](https://issues.apache.org/jira/browse/ARROW-633) - [Java] Add support for FixedSizeBinary type
+* [ARROW-634](https://issues.apache.org/jira/browse/ARROW-634) - Add integration tests for FixedSizeBinary
+* [ARROW-764](https://issues.apache.org/jira/browse/ARROW-764) - [C++] Improve performance of CopyBitmap, add benchmarks
+* [ARROW-969](https://issues.apache.org/jira/browse/ARROW-969) - [C++/Python] Add add/remove field functions for RecordBatch
+
+## Bug Fixes
+
+* [ARROW-1345](https://issues.apache.org/jira/browse/ARROW-1345) - [Python] Conversion from nested NumPy arrays fails on integers other than int64, float32
+* [ARROW-1589](https://issues.apache.org/jira/browse/ARROW-1589) - [C++] Fuzzing for certain input formats
+* [ARROW-1646](https://issues.apache.org/jira/browse/ARROW-1646) - [Python] pyarrow.array cannot handle NumPy scalar types
+* [ARROW-1856](https://issues.apache.org/jira/browse/ARROW-1856) - [Python] Auto-detect Parquet ABI version when using PARQUET_HOME
+* [ARROW-1909](https://issues.apache.org/jira/browse/ARROW-1909) - [C++] Bug: Build fails on windows with "-DARROW_BUILD_BENCHMARKS=ON"
+* [ARROW-1912](https://issues.apache.org/jira/browse/ARROW-1912) - [Website] Add org affiliations to committers.html
+* [ARROW-1919](https://issues.apache.org/jira/browse/ARROW-1919) - Plasma hanging if object id is not 20 bytes
+* [ARROW-1924](https://issues.apache.org/jira/browse/ARROW-1924) - [Python] Bring back pickle=True option for serialization
+* [ARROW-1933](https://issues.apache.org/jira/browse/ARROW-1933) - [GLib] Build failure with --with-arrow-cpp-build-dir and GPU enabled Arrow C++
+* [ARROW-1940](https://issues.apache.org/jira/browse/ARROW-1940) - [Python] Extra metadata gets added after multiple conversions between pd.DataFrame and pa.Table
+* [ARROW-1941](https://issues.apache.org/jira/browse/ARROW-1941) - Table <–> DataFrame roundtrip failing
+* [ARROW-1943](https://issues.apache.org/jira/browse/ARROW-1943) - Handle setInitialCapacity() for deeply nested lists of lists
+* [ARROW-1944](https://issues.apache.org/jira/browse/ARROW-1944) - FindArrow has wrong ARROW_STATIC_LIB
+* [ARROW-1945](https://issues.apache.org/jira/browse/ARROW-1945) - [C++] Fix doxygen documentation of array.h
+* [ARROW-1946](https://issues.apache.org/jira/browse/ARROW-1946) - Add APIs to decimal vector for writing big endian data
+* [ARROW-1948](https://issues.apache.org/jira/browse/ARROW-1948) - [Java] ListVector does not handle ipc with all non-null values with none set
+* [ARROW-1950](https://issues.apache.org/jira/browse/ARROW-1950) - [Python] pandas_type in pandas metadata incorrect for List types
+* [ARROW-1953](https://issues.apache.org/jira/browse/ARROW-1953) - [JS] JavaScript builds broken on master
+* [ARROW-1958](https://issues.apache.org/jira/browse/ARROW-1958) - [Python] Error in pandas conversion for datetimetz row index
+* [ARROW-1961](https://issues.apache.org/jira/browse/ARROW-1961) - [Python] Writing Parquet file with flavor='spark' loses pandas schema metadata
+* [ARROW-1966](https://issues.apache.org/jira/browse/ARROW-1966) - [C++] Support JAVA_HOME paths in HDFS libjvm loading that include the jre directory
+* [ARROW-1971](https://issues.apache.org/jira/browse/ARROW-1971) - [Python] Add pandas serialization to the default
+* [ARROW-1972](https://issues.apache.org/jira/browse/ARROW-1972) - Deserialization of buffer objects (and pandas dataframes) segfaults on different processes.
+* [ARROW-1973](https://issues.apache.org/jira/browse/ARROW-1973) - [Python] Memory leak when converting Arrow tables with array columns to Pandas dataframes.
+* [ARROW-1976](https://issues.apache.org/jira/browse/ARROW-1976) - [Python] Handling unicode pandas columns on parquet.read_table
+* [ARROW-1979](https://issues.apache.org/jira/browse/ARROW-1979) - [JS] JS builds handing in es2015:umd tests
+* [ARROW-1980](https://issues.apache.org/jira/browse/ARROW-1980) - [Python] Race condition in `write_to_dataset`
+* [ARROW-1982](https://issues.apache.org/jira/browse/ARROW-1982) - [Python] Return parquet statistics min/max as values instead of strings
+* [ARROW-1991](https://issues.apache.org/jira/browse/ARROW-1991) - [GLib] Docker-based documentation build is broken
+* [ARROW-1992](https://issues.apache.org/jira/browse/ARROW-1992) - [Python] to_pandas crashes when using strings_to_categoricals on empty string cols on 0.8.0
+* [ARROW-1997](https://issues.apache.org/jira/browse/ARROW-1997) - [Python] to_pandas with strings_to_categorical fails
+* [ARROW-1998](https://issues.apache.org/jira/browse/ARROW-1998) - [Python] Table.from_pandas crashes when data frame is empty
+* [ARROW-1999](https://issues.apache.org/jira/browse/ARROW-1999) - [Python] from_numpy_dtype returns wrong types
+* [ARROW-2000](https://issues.apache.org/jira/browse/ARROW-2000) - Deduplicate file descriptors when plasma store replies to get request.
+* [ARROW-2002](https://issues.apache.org/jira/browse/ARROW-2002) - use pyarrow download file will raise queue.Full exceptions sometimes
+* [ARROW-2003](https://issues.apache.org/jira/browse/ARROW-2003) - [Python] Do not use deprecated kwarg in pandas.core.internals.make_block
+* [ARROW-2005](https://issues.apache.org/jira/browse/ARROW-2005) - [Python] pyflakes warnings on Cython files not failing build
+* [ARROW-2008](https://issues.apache.org/jira/browse/ARROW-2008) - [Python] Type inference for int32 NumPy arrays (expecting list<int32>) returns int64 and then conversion fails
+* [ARROW-2010](https://issues.apache.org/jira/browse/ARROW-2010) - [C++] Compiler warnings with CHECKIN warning level in ORC adapter
+* [ARROW-2017](https://issues.apache.org/jira/browse/ARROW-2017) - Array initialization with  large (>2**31-1) uint64 values fails
+* [ARROW-2023](https://issues.apache.org/jira/browse/ARROW-2023) - [C++] Test opening IPC stream reader or file reader on an empty InputStream
+* [ARROW-2025](https://issues.apache.org/jira/browse/ARROW-2025) - [Python/C++] HDFS Client disconnect closes all open clients
+* [ARROW-2029](https://issues.apache.org/jira/browse/ARROW-2029) - [Python] Program crash on `HdfsFile.tell` if file is closed
+* [ARROW-2032](https://issues.apache.org/jira/browse/ARROW-2032) - [C++] ORC ep installs on each call to ninja build (even if no work to do)
+* [ARROW-2033](https://issues.apache.org/jira/browse/ARROW-2033) - pa.array() doesn't work with iterators
+* [ARROW-2039](https://issues.apache.org/jira/browse/ARROW-2039) - [Python] pyarrow.Buffer().to_pybytes() segfaults
+* [ARROW-2040](https://issues.apache.org/jira/browse/ARROW-2040) - [Python] Deserialized Numpy array must keep ref to underlying tensor
+* [ARROW-2047](https://issues.apache.org/jira/browse/ARROW-2047) - [Python] test_serialization.py uses a python executable in PATH rather than that used for a test run
+* [ARROW-2049](https://issues.apache.org/jira/browse/ARROW-2049) - ARROW-2049: [Python] Use python -m cython to run Cython, instead of CYTHON_EXECUTABLE
+* [ARROW-2062](https://issues.apache.org/jira/browse/ARROW-2062) - [C++] Stalled builds in test_serialization.py in Travis CI
+* [ARROW-2070](https://issues.apache.org/jira/browse/ARROW-2070) - [Python] chdir logic in setup.py buggy
+* [ARROW-2072](https://issues.apache.org/jira/browse/ARROW-2072) - [Python] decimal128.byte_width crashes
+* [ARROW-2080](https://issues.apache.org/jira/browse/ARROW-2080) - [Python] Update documentation after ARROW-2024
+* [ARROW-2085](https://issues.apache.org/jira/browse/ARROW-2085) - HadoopFileSystem.isdir and .isfile should return False if the path doesn't exist 
+* [ARROW-2106](https://issues.apache.org/jira/browse/ARROW-2106) - [Python] pyarrow.array can't take a pandas Series of python datetime objects.
+* [ARROW-2109](https://issues.apache.org/jira/browse/ARROW-2109) - [C++] Boost 1.66 compilation fails on Windows on linkage stage
+* [ARROW-2124](https://issues.apache.org/jira/browse/ARROW-2124) - [Python] ArrowInvalid raised if the first item of a nested list of numpy arrays is empty
+* [ARROW-2128](https://issues.apache.org/jira/browse/ARROW-2128) - [Python] Cannot serialize array of empty lists
+* [ARROW-2129](https://issues.apache.org/jira/browse/ARROW-2129) - [Python] Segmentation fault on conversion of empty array to Pandas
+* [ARROW-2131](https://issues.apache.org/jira/browse/ARROW-2131) - [Python] Serialization test fails on Windows when library has been built in place / not installed
+* [ARROW-2133](https://issues.apache.org/jira/browse/ARROW-2133) - [Python] Segmentation fault on conversion of empty nested arrays to Pandas
+* [ARROW-2135](https://issues.apache.org/jira/browse/ARROW-2135) - [Python] NaN values silently casted to int64 when passing explicit schema for conversion in Table.from_pandas
+* [ARROW-2145](https://issues.apache.org/jira/browse/ARROW-2145) - [Python] Decimal conversion not working for NaN values
+* [ARROW-2150](https://issues.apache.org/jira/browse/ARROW-2150) - [Python] array equality defaults to identity
+* [ARROW-2151](https://issues.apache.org/jira/browse/ARROW-2151) - [Python] Error when converting from list of uint64 arrays
+* [ARROW-2153](https://issues.apache.org/jira/browse/ARROW-2153) - [C++/Python] Decimal conversion not working for exponential notation
+* [ARROW-2157](https://issues.apache.org/jira/browse/ARROW-2157) - [Python] Decimal arrays cannot be constructed from Python lists
+* [ARROW-2160](https://issues.apache.org/jira/browse/ARROW-2160) - [C++/Python] Fix decimal precision inference
+* [ARROW-2161](https://issues.apache.org/jira/browse/ARROW-2161) - [Python] Skip test_cython_api if ARROW_HOME isn't defined
+* [ARROW-2162](https://issues.apache.org/jira/browse/ARROW-2162) - [Python/C++] Decimal Values with too-high precision are multiplied by 100
+* [ARROW-2167](https://issues.apache.org/jira/browse/ARROW-2167) - [C++] Building Orc extensions fails with the default BUILD_WARNING_LEVEL=Production
+* [ARROW-2170](https://issues.apache.org/jira/browse/ARROW-2170) - [Python] construct_metadata fails on reading files where no index was preserved
+* [ARROW-2171](https://issues.apache.org/jira/browse/ARROW-2171) - [Python] OwnedRef is fragile
+* [ARROW-2172](https://issues.apache.org/jira/browse/ARROW-2172) - [Python] Incorrect conversion from Numpy array when stride % itemsize != 0
+* [ARROW-2173](https://issues.apache.org/jira/browse/ARROW-2173) - [Python] NumPyBuffer destructor should hold the GIL
+* [ARROW-2175](https://issues.apache.org/jira/browse/ARROW-2175) - [Python] arrow_ep build is triggering during parquet-cpp build in Travis CI
+* [ARROW-2178](https://issues.apache.org/jira/browse/ARROW-2178) -  [JS] Fix JS html FileReader example
+* [ARROW-2179](https://issues.apache.org/jira/browse/ARROW-2179) - [C++] arrow/util/io-util.h missing from libarrow-dev
+* [ARROW-2192](https://issues.apache.org/jira/browse/ARROW-2192) - Commits to master should run all builds in CI matrix
+* [ARROW-2209](https://issues.apache.org/jira/browse/ARROW-2209) - [Python] Partition columns are not correctly loaded in schema of ParquetDataset
+* [ARROW-2210](https://issues.apache.org/jira/browse/ARROW-2210) - [C++] TestBuffer_ResizeOOM has a memory leak with jemalloc
+* [ARROW-2212](https://issues.apache.org/jira/browse/ARROW-2212) - [C++/Python] Build Protobuf in base manylinux 1 docker image
+* [ARROW-2223](https://issues.apache.org/jira/browse/ARROW-2223) - [JS] installing umd release throws an error
+* [ARROW-2227](https://issues.apache.org/jira/browse/ARROW-2227) - [Python] Table.from_pandas does not create chunked_arrays.
+* [ARROW-2230](https://issues.apache.org/jira/browse/ARROW-2230) - [Python] JS version number is sometimes picked up
+* [ARROW-2232](https://issues.apache.org/jira/browse/ARROW-2232) - [Python] pyarrow.Tensor constructor segfaults
+* [ARROW-2234](https://issues.apache.org/jira/browse/ARROW-2234) - [JS] Read timestamp low bits as Uint32s
+* [ARROW-2240](https://issues.apache.org/jira/browse/ARROW-2240) - [Python] Array initialization with leading numpy nan fails with exception
+* [ARROW-2244](https://issues.apache.org/jira/browse/ARROW-2244) - [C++] Slicing NullArray should not cause the null count on the internal data to be unknown
+* [ARROW-2245](https://issues.apache.org/jira/browse/ARROW-2245) - [Python] Revert static linkage of parquet-cpp in manylinux1 wheel
+* [ARROW-2246](https://issues.apache.org/jira/browse/ARROW-2246) - [Python] Use namespaced boost in manylinux1 package
+* [ARROW-2251](https://issues.apache.org/jira/browse/ARROW-2251) - [GLib] Destroying GArrowBuffer while GArrowTensor that uses the buffer causes a crash
+* [ARROW-2254](https://issues.apache.org/jira/browse/ARROW-2254) - [Python] Local in-place dev versions picking up JS tags
+* [ARROW-2258](https://issues.apache.org/jira/browse/ARROW-2258) - [C++] Appveyor builds failing on master
+* [ARROW-2263](https://issues.apache.org/jira/browse/ARROW-2263) - [Python] test_cython.py fails if pyarrow is not in import path (e.g. with inplace builds)
+* [ARROW-2265](https://issues.apache.org/jira/browse/ARROW-2265) - [Python] Serializing subclasses of np.ndarray returns a np.ndarray.
+* [ARROW-2268](https://issues.apache.org/jira/browse/ARROW-2268) - Remove MD5 checksums from release process
+* [ARROW-2269](https://issues.apache.org/jira/browse/ARROW-2269) - [Python] Cannot build bdist_wheel for Python
+* [ARROW-2270](https://issues.apache.org/jira/browse/ARROW-2270) - [Python] ForeignBuffer doesn't tie Python object lifetime to C++ buffer lifetime
+* [ARROW-2272](https://issues.apache.org/jira/browse/ARROW-2272) - [Python] test_plasma spams /tmp
+* [ARROW-2275](https://issues.apache.org/jira/browse/ARROW-2275) - [C++] Buffer::mutable_data_ member uninitialized
+* [ARROW-2280](https://issues.apache.org/jira/browse/ARROW-2280) - [Python] pyarrow.Array.buffers should also include the offsets
+* [ARROW-2284](https://issues.apache.org/jira/browse/ARROW-2284) - [Python] test_plasma error on plasma_store error
+* [ARROW-2288](https://issues.apache.org/jira/browse/ARROW-2288) - [Python] slicing logic defective
+* [ARROW-2297](https://issues.apache.org/jira/browse/ARROW-2297) - [JS] babel-jest is not listed as a dev dependency
+* [ARROW-2304](https://issues.apache.org/jira/browse/ARROW-2304) - [C++] MultipleClients test in io-hdfs-test fails on trunk
+* [ARROW-2306](https://issues.apache.org/jira/browse/ARROW-2306) - [Python] HDFS test failures
+* [ARROW-2307](https://issues.apache.org/jira/browse/ARROW-2307) - [Python] Unable to read arrow stream containing 0 record batches
+* [ARROW-2311](https://issues.apache.org/jira/browse/ARROW-2311) - [Python] Struct array slicing defective
+* [ARROW-2312](https://issues.apache.org/jira/browse/ARROW-2312) - [JS] verify-release-candidate-sh must be updated to include JS in integration tests
+* [ARROW-2313](https://issues.apache.org/jira/browse/ARROW-2313) - [GLib] Release builds must define NDEBUG
+* [ARROW-2316](https://issues.apache.org/jira/browse/ARROW-2316) - [C++] Revert Buffer::mutable_data member to always inline
+* [ARROW-2318](https://issues.apache.org/jira/browse/ARROW-2318) - [C++] TestPlasmaStore.MultipleClientTest is flaky (hangs) in release builds
+* [ARROW-2320](https://issues.apache.org/jira/browse/ARROW-2320) - [C++] Vendored Boost build does not build regex library
+
+
+[1]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.9.0
+[2]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.9.0/
diff --git a/site/_release/index.md b/site/_release/index.md
index e22df92b6cb41..adf54f12de1f3 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -26,6 +26,7 @@ limitations under the License.
 
 Navigate to the release page for downloads and the changelog.
 
+* [0.9.0 (21 March 2018)][11]
 * [0.8.0 (18 December 2017)][10]
 * [0.7.1 (1 October 2017)][9]
 * [0.7.0 (17 September 2017)][8]
@@ -47,3 +48,5 @@ Navigate to the release page for downloads and the changelog.
 [8]: {{ site.baseurl }}/release/0.7.0.html
 [9]: {{ site.baseurl }}/release/0.7.1.html
 [10]: {{ site.baseurl }}/release/0.8.0.html
+[11]: {{ site.baseurl }}/release/0.9.0.html
+
diff --git a/site/install.md b/site/install.md
index f795299676eb5..2d47e210b1695 100644
--- a/site/install.md
+++ b/site/install.md
@@ -28,7 +28,7 @@ See the [release notes][10] for more about what's new.
 
 ### Source release
 
-* **Source Release**: [apache-arrow-0.8.0.tar.gz][6]
+* **Source Release**: [apache-arrow-0.9.0.tar.gz][6]
 * **Verification**: [sha512][3], [asc][7] ([verification instructions][12])
 * [Git tag {{site.data.versions['current'].git-tag}}][2]
 * [PGP keys for release signatures][11]
@@ -53,8 +53,8 @@ Install them with:
 
 
 ```shell
-conda install arrow-cpp=0.8.* -c conda-forge
-conda install pyarrow=0.8.* -c conda-forge
+conda install arrow-cpp=0.9.* -c conda-forge
+conda install pyarrow=0.9.* -c conda-forge
 ```
 
 ### Python Wheels on PyPI (Unofficial)
@@ -62,10 +62,10 @@ conda install pyarrow=0.8.* -c conda-forge
 We have provided binary wheels on PyPI for Linux, macOS, and Windows:
 
 ```shell
-pip install pyarrow==0.8.*
+pip install pyarrow==0.9.*
 ```
 
-We recommend pinning `0.8.*` in `requirements.txt` to install the latest patch
+We recommend pinning `0.9.*` in `requirements.txt` to install the latest patch
 release.
 
 These include the Apache Arrow and Apache Parquet C++ binary libraries bundled

From 60848c0e051017f24228866a58c942886dfc21ae Mon Sep 17 00:00:00 2001
From: Stuart Carnie <stuart.carnie@gmail.com>
Date: Wed, 21 Mar 2018 16:04:16 -0400
Subject: [PATCH 1619/1644] ARROW-2299: [Go] Import Go arrow implementation
 from influxdata/arrow

**NOTE:**

Some code generated files and assembler output from the LLVM compiler do not have the headers as they would be stripped each time code generation is rerun. These files are included so that the Go package is go-gettable without any additional build steps.

Author: Stuart Carnie <stuart.carnie@gmail.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1739 from stuartcarnie/sgc-go-arrow and squashes the following commits:

95b9b42d <Wes McKinney> Add new ci/travis_release_audit.sh script
2320777a <Wes McKinney> Split Apache RAT check into separate script, always run. Update rat_exclude_files.txt
f00fb6f9 <Stuart Carnie> Rename title; add Apache copyright headers to markdown files
f31d8ca2 <Stuart Carnie> Add Apache copyright headers
3e17fe4a <Stuart Carnie> Initial commit, before copyright update
---
 .travis.yml                                   |    2 +
 ci/travis_release_audit.sh                    |   24 +
 ci/travis_script_cpp.sh                       |    4 -
 dev/release/rat_exclude_files.txt             |    5 +
 go/arrow/.editorconfig                        |   21 +
 go/arrow/.gitignore                           |   35 +
 go/arrow/Gopkg.lock                           |   27 +
 go/arrow/Gopkg.toml                           |   19 +
 go/arrow/Makefile                             |   54 +
 go/arrow/README.md                            |  174 ++
 go/arrow/_examples/helloworld/main.go         |   32 +
 go/arrow/_tools/tmpl/main.go                  |  267 ++++
 go/arrow/_tools/tmpl/main_test.go             |   73 +
 go/arrow/array/array.go                       |  185 +++
 go/arrow/array/array_test.go                  |   99 ++
 go/arrow/array/binary.go                      |   69 +
 go/arrow/array/binarybuilder.go               |  184 +++
 go/arrow/array/binarybuilder_test.go          |   61 +
 go/arrow/array/boolean.go                     |   53 +
 go/arrow/array/booleanbuilder.go              |  150 ++
 go/arrow/array/booleanbuilder_test.go         |   45 +
 go/arrow/array/bufferbuilder.go               |  127 ++
 go/arrow/array/bufferbuilder_byte.go          |   30 +
 go/arrow/array/bufferbuilder_numeric.gen.go   |   58 +
 .../array/bufferbuilder_numeric.gen.go.tmpl   |   61 +
 go/arrow/array/bufferbuilder_numeric_test.go  |   81 +
 go/arrow/array/builder.go                     |  164 ++
 go/arrow/array/builder_test.go                |   57 +
 go/arrow/array/data.go                        |   81 +
 go/arrow/array/doc.go                         |   20 +
 go/arrow/array/numeric.gen.go                 |  287 ++++
 go/arrow/array/numeric.gen.go.tmpl            |   48 +
 go/arrow/array/numeric_test.go                |   36 +
 go/arrow/array/numericbuilder.gen.go          | 1415 +++++++++++++++++
 go/arrow/array/numericbuilder.gen.go.tmpl     |  165 ++
 go/arrow/array/numericbuilder_test.go         |  107 ++
 go/arrow/array/util.go                        |   24 +
 go/arrow/datatype.go                          |  133 ++
 go/arrow/datatype_binary.go                   |   39 +
 go/arrow/datatype_fixedwidth.go               |   61 +
 go/arrow/datatype_fixedwidth_test.go          |   42 +
 go/arrow/datatype_numeric.gen.go              |   96 ++
 go/arrow/datatype_numeric.gen.go.tmpl         |   38 +
 go/arrow/datatype_numeric.gen.go.tmpldata     |   32 +
 go/arrow/doc.go                               |   38 +
 go/arrow/example_test.go                      |  116 ++
 go/arrow/internal/bitutil/bitutil.go          |   95 ++
 go/arrow/internal/bitutil/bitutil_test.go     |  190 +++
 go/arrow/internal/cpu/README.md               |   42 +
 go/arrow/internal/cpu/cpu.go                  |   77 +
 go/arrow/internal/cpu/cpu_test.go             |   51 +
 go/arrow/internal/cpu/cpu_x86.go              |  107 ++
 go/arrow/internal/cpu/cpu_x86.s               |   32 +
 go/arrow/internal/debug/assert_off.go         |   24 +
 go/arrow/internal/debug/assert_on.go          |   28 +
 go/arrow/internal/debug/doc.go                |   32 +
 go/arrow/internal/debug/log_off.go            |   21 +
 go/arrow/internal/debug/log_on.go             |   32 +
 go/arrow/internal/debug/util.go               |   37 +
 go/arrow/internal/testing/tools/bits.go       |   40 +
 go/arrow/internal/testing/tools/bits_test.go  |   42 +
 go/arrow/internal/testing/tools/bool.go       |   25 +
 go/arrow/math/Makefile                        |   90 ++
 go/arrow/math/_lib/.gitignore                 |   18 +
 go/arrow/math/_lib/CMakeLists.txt             |   24 +
 go/arrow/math/_lib/arch.h                     |   27 +
 go/arrow/math/_lib/float64.c                  |   26 +
 go/arrow/math/_lib/float64_avx2.s             |  176 ++
 go/arrow/math/_lib/float64_sse4.s             |  103 ++
 go/arrow/math/_lib/int64.c                    |   27 +
 go/arrow/math/_lib/int64_avx2.s               |  181 +++
 go/arrow/math/_lib/int64_sse4.s               |  108 ++
 go/arrow/math/_lib/uint64.c                   |   27 +
 go/arrow/math/_lib/uint64_avx2.s              |  181 +++
 go/arrow/math/_lib/uint64_sse4.s              |  108 ++
 go/arrow/math/doc.go                          |   30 +
 go/arrow/math/float64.go                      |   44 +
 go/arrow/math/float64.tmpldata                |    4 +
 go/arrow/math/float64_amd64.go                |   33 +
 go/arrow/math/float64_avx2_amd64.go           |   41 +
 go/arrow/math/float64_avx2_amd64.s            |  167 ++
 go/arrow/math/float64_noasm.go                |   25 +
 go/arrow/math/float64_sse4_amd64.go           |   41 +
 go/arrow/math/float64_sse4_amd64.s            |   94 ++
 go/arrow/math/float64_test.go                 |   68 +
 go/arrow/math/int64.go                        |   44 +
 go/arrow/math/int64.tmpldata                  |    4 +
 go/arrow/math/int64_amd64.go                  |   33 +
 go/arrow/math/int64_avx2_amd64.go             |   41 +
 go/arrow/math/int64_avx2_amd64.s              |  173 ++
 go/arrow/math/int64_noasm.go                  |   25 +
 go/arrow/math/int64_sse4_amd64.go             |   41 +
 go/arrow/math/int64_sse4_amd64.s              |  100 ++
 go/arrow/math/int64_test.go                   |   68 +
 go/arrow/math/math_amd64.go                   |   51 +
 go/arrow/math/math_noasm.go                   |   29 +
 go/arrow/math/type.go.tmpl                    |   45 +
 go/arrow/math/type_amd64.go.tmpl              |   33 +
 go/arrow/math/type_noasm.go.tmpl              |   25 +
 go/arrow/math/type_simd_amd64.go.tmpl         |   42 +
 go/arrow/math/type_test.go.tmpl               |   69 +
 go/arrow/math/uint64.go                       |   44 +
 go/arrow/math/uint64.tmpldata                 |    4 +
 go/arrow/math/uint64_amd64.go                 |   33 +
 go/arrow/math/uint64_avx2_amd64.go            |   41 +
 go/arrow/math/uint64_avx2_amd64.s             |  173 ++
 go/arrow/math/uint64_noasm.go                 |   25 +
 go/arrow/math/uint64_sse4_amd64.go            |   41 +
 go/arrow/math/uint64_sse4_amd64.s             |  100 ++
 go/arrow/math/uint64_test.go                  |   68 +
 go/arrow/memory/Makefile                      |   54 +
 go/arrow/memory/_lib/.gitignore               |   18 +
 go/arrow/memory/_lib/CMakeLists.txt           |   24 +
 go/arrow/memory/_lib/arch.h                   |   27 +
 go/arrow/memory/_lib/memory.c                 |   27 +
 go/arrow/memory/_lib/memory_avx2.s            |   97 ++
 go/arrow/memory/_lib/memory_sse4.s            |   96 ++
 go/arrow/memory/allocator.go                  |   33 +
 go/arrow/memory/buffer.go                     |  108 ++
 go/arrow/memory/buffer_test.go                |   45 +
 go/arrow/memory/checked_allocator.go          |   70 +
 go/arrow/memory/doc.go                        |   20 +
 go/arrow/memory/go_allocator.go               |   44 +
 go/arrow/memory/go_allocator_test.go          |   76 +
 go/arrow/memory/memory.go                     |   33 +
 go/arrow/memory/memory_amd64.go               |   33 +
 go/arrow/memory/memory_avx2_amd64.go          |   41 +
 go/arrow/memory/memory_avx2_amd64.s           |   85 +
 go/arrow/memory/memory_noasm.go               |   23 +
 go/arrow/memory/memory_sse4_amd64.go          |   31 +
 go/arrow/memory/memory_sse4_amd64.s           |   84 +
 go/arrow/memory/memory_test.go                |  125 ++
 go/arrow/memory/util.go                       |   37 +
 go/arrow/memory/util_test.go                  |   61 +
 go/arrow/metadata/schema.go                   |   37 +
 go/arrow/numeric.schema.json                  |   15 +
 go/arrow/numeric.tmpldata                     |   87 +
 go/arrow/type_string.go                       |   16 +
 go/arrow/type_traits_boolean.go               |   28 +
 go/arrow/type_traits_numeric.gen.go           |  567 +++++++
 go/arrow/type_traits_numeric.gen.go.tmpl      |   89 ++
 141 files changed, 10936 insertions(+), 4 deletions(-)
 create mode 100755 ci/travis_release_audit.sh
 create mode 100644 go/arrow/.editorconfig
 create mode 100644 go/arrow/.gitignore
 create mode 100644 go/arrow/Gopkg.lock
 create mode 100644 go/arrow/Gopkg.toml
 create mode 100644 go/arrow/Makefile
 create mode 100644 go/arrow/README.md
 create mode 100644 go/arrow/_examples/helloworld/main.go
 create mode 100644 go/arrow/_tools/tmpl/main.go
 create mode 100644 go/arrow/_tools/tmpl/main_test.go
 create mode 100644 go/arrow/array/array.go
 create mode 100644 go/arrow/array/array_test.go
 create mode 100644 go/arrow/array/binary.go
 create mode 100644 go/arrow/array/binarybuilder.go
 create mode 100644 go/arrow/array/binarybuilder_test.go
 create mode 100644 go/arrow/array/boolean.go
 create mode 100644 go/arrow/array/booleanbuilder.go
 create mode 100644 go/arrow/array/booleanbuilder_test.go
 create mode 100644 go/arrow/array/bufferbuilder.go
 create mode 100644 go/arrow/array/bufferbuilder_byte.go
 create mode 100644 go/arrow/array/bufferbuilder_numeric.gen.go
 create mode 100644 go/arrow/array/bufferbuilder_numeric.gen.go.tmpl
 create mode 100644 go/arrow/array/bufferbuilder_numeric_test.go
 create mode 100644 go/arrow/array/builder.go
 create mode 100644 go/arrow/array/builder_test.go
 create mode 100644 go/arrow/array/data.go
 create mode 100644 go/arrow/array/doc.go
 create mode 100644 go/arrow/array/numeric.gen.go
 create mode 100644 go/arrow/array/numeric.gen.go.tmpl
 create mode 100644 go/arrow/array/numeric_test.go
 create mode 100644 go/arrow/array/numericbuilder.gen.go
 create mode 100644 go/arrow/array/numericbuilder.gen.go.tmpl
 create mode 100644 go/arrow/array/numericbuilder_test.go
 create mode 100644 go/arrow/array/util.go
 create mode 100644 go/arrow/datatype.go
 create mode 100644 go/arrow/datatype_binary.go
 create mode 100644 go/arrow/datatype_fixedwidth.go
 create mode 100644 go/arrow/datatype_fixedwidth_test.go
 create mode 100644 go/arrow/datatype_numeric.gen.go
 create mode 100644 go/arrow/datatype_numeric.gen.go.tmpl
 create mode 100644 go/arrow/datatype_numeric.gen.go.tmpldata
 create mode 100644 go/arrow/doc.go
 create mode 100644 go/arrow/example_test.go
 create mode 100644 go/arrow/internal/bitutil/bitutil.go
 create mode 100644 go/arrow/internal/bitutil/bitutil_test.go
 create mode 100644 go/arrow/internal/cpu/README.md
 create mode 100644 go/arrow/internal/cpu/cpu.go
 create mode 100644 go/arrow/internal/cpu/cpu_test.go
 create mode 100644 go/arrow/internal/cpu/cpu_x86.go
 create mode 100644 go/arrow/internal/cpu/cpu_x86.s
 create mode 100644 go/arrow/internal/debug/assert_off.go
 create mode 100644 go/arrow/internal/debug/assert_on.go
 create mode 100644 go/arrow/internal/debug/doc.go
 create mode 100644 go/arrow/internal/debug/log_off.go
 create mode 100644 go/arrow/internal/debug/log_on.go
 create mode 100644 go/arrow/internal/debug/util.go
 create mode 100644 go/arrow/internal/testing/tools/bits.go
 create mode 100644 go/arrow/internal/testing/tools/bits_test.go
 create mode 100644 go/arrow/internal/testing/tools/bool.go
 create mode 100644 go/arrow/math/Makefile
 create mode 100644 go/arrow/math/_lib/.gitignore
 create mode 100644 go/arrow/math/_lib/CMakeLists.txt
 create mode 100644 go/arrow/math/_lib/arch.h
 create mode 100644 go/arrow/math/_lib/float64.c
 create mode 100644 go/arrow/math/_lib/float64_avx2.s
 create mode 100644 go/arrow/math/_lib/float64_sse4.s
 create mode 100644 go/arrow/math/_lib/int64.c
 create mode 100644 go/arrow/math/_lib/int64_avx2.s
 create mode 100644 go/arrow/math/_lib/int64_sse4.s
 create mode 100644 go/arrow/math/_lib/uint64.c
 create mode 100644 go/arrow/math/_lib/uint64_avx2.s
 create mode 100644 go/arrow/math/_lib/uint64_sse4.s
 create mode 100644 go/arrow/math/doc.go
 create mode 100644 go/arrow/math/float64.go
 create mode 100644 go/arrow/math/float64.tmpldata
 create mode 100644 go/arrow/math/float64_amd64.go
 create mode 100644 go/arrow/math/float64_avx2_amd64.go
 create mode 100644 go/arrow/math/float64_avx2_amd64.s
 create mode 100644 go/arrow/math/float64_noasm.go
 create mode 100644 go/arrow/math/float64_sse4_amd64.go
 create mode 100644 go/arrow/math/float64_sse4_amd64.s
 create mode 100644 go/arrow/math/float64_test.go
 create mode 100644 go/arrow/math/int64.go
 create mode 100644 go/arrow/math/int64.tmpldata
 create mode 100644 go/arrow/math/int64_amd64.go
 create mode 100644 go/arrow/math/int64_avx2_amd64.go
 create mode 100644 go/arrow/math/int64_avx2_amd64.s
 create mode 100644 go/arrow/math/int64_noasm.go
 create mode 100644 go/arrow/math/int64_sse4_amd64.go
 create mode 100644 go/arrow/math/int64_sse4_amd64.s
 create mode 100644 go/arrow/math/int64_test.go
 create mode 100644 go/arrow/math/math_amd64.go
 create mode 100644 go/arrow/math/math_noasm.go
 create mode 100644 go/arrow/math/type.go.tmpl
 create mode 100644 go/arrow/math/type_amd64.go.tmpl
 create mode 100644 go/arrow/math/type_noasm.go.tmpl
 create mode 100644 go/arrow/math/type_simd_amd64.go.tmpl
 create mode 100644 go/arrow/math/type_test.go.tmpl
 create mode 100644 go/arrow/math/uint64.go
 create mode 100644 go/arrow/math/uint64.tmpldata
 create mode 100644 go/arrow/math/uint64_amd64.go
 create mode 100644 go/arrow/math/uint64_avx2_amd64.go
 create mode 100644 go/arrow/math/uint64_avx2_amd64.s
 create mode 100644 go/arrow/math/uint64_noasm.go
 create mode 100644 go/arrow/math/uint64_sse4_amd64.go
 create mode 100644 go/arrow/math/uint64_sse4_amd64.s
 create mode 100644 go/arrow/math/uint64_test.go
 create mode 100644 go/arrow/memory/Makefile
 create mode 100644 go/arrow/memory/_lib/.gitignore
 create mode 100644 go/arrow/memory/_lib/CMakeLists.txt
 create mode 100644 go/arrow/memory/_lib/arch.h
 create mode 100644 go/arrow/memory/_lib/memory.c
 create mode 100644 go/arrow/memory/_lib/memory_avx2.s
 create mode 100644 go/arrow/memory/_lib/memory_sse4.s
 create mode 100644 go/arrow/memory/allocator.go
 create mode 100644 go/arrow/memory/buffer.go
 create mode 100644 go/arrow/memory/buffer_test.go
 create mode 100644 go/arrow/memory/checked_allocator.go
 create mode 100644 go/arrow/memory/doc.go
 create mode 100644 go/arrow/memory/go_allocator.go
 create mode 100644 go/arrow/memory/go_allocator_test.go
 create mode 100644 go/arrow/memory/memory.go
 create mode 100644 go/arrow/memory/memory_amd64.go
 create mode 100644 go/arrow/memory/memory_avx2_amd64.go
 create mode 100644 go/arrow/memory/memory_avx2_amd64.s
 create mode 100644 go/arrow/memory/memory_noasm.go
 create mode 100644 go/arrow/memory/memory_sse4_amd64.go
 create mode 100644 go/arrow/memory/memory_sse4_amd64.s
 create mode 100644 go/arrow/memory/memory_test.go
 create mode 100644 go/arrow/memory/util.go
 create mode 100644 go/arrow/memory/util_test.go
 create mode 100644 go/arrow/metadata/schema.go
 create mode 100644 go/arrow/numeric.schema.json
 create mode 100644 go/arrow/numeric.tmpldata
 create mode 100644 go/arrow/type_string.go
 create mode 100644 go/arrow/type_traits_boolean.go
 create mode 100644 go/arrow/type_traits_numeric.gen.go
 create mode 100644 go/arrow/type_traits_numeric.gen.go.tmpl

diff --git a/.travis.yml b/.travis.yml
index b1241e793bc70..03bc8e26aef39 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -56,6 +56,8 @@ matrix:
     - CC="clang-5.0"
     - CXX="clang++-5.0"
     before_script:
+    # Always run RAT checks, in case another build in matrix breaks RAT
+    - $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh
     # (ARROW_CI_CPP_AFFECTED implies ARROW_CI_PYTHON_AFFECTED)
     - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
diff --git a/ci/travis_release_audit.sh b/ci/travis_release_audit.sh
new file mode 100755
index 0000000000000..8da71a09cf784
--- /dev/null
+++ b/ci/travis_release_audit.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# Check licenses according to Apache policy
+git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
+./dev/release/run-rat.sh arrow-src.tar.gz
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index 3d61bc5b89b71..27ab88ba91d6d 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -21,10 +21,6 @@ set -e
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-# Check licenses according to Apache policy
-git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
-./dev/release/run-rat.sh arrow-src.tar.gz
-
 pushd $CPP_BUILD_DIR
 
 ctest -VV -L unittest
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index bf962bcd49e42..eb735604fd763 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -39,6 +39,11 @@ cpp/src/plasma/thirdparty/dlmalloc.c
 cpp/src/plasma/thirdparty/xxhash.cc
 cpp/src/plasma/thirdparty/xxhash.h
 dev/release/rat_exclude_files.txt
+go/arrow/Gopkg.lock
+go/arrow/internal/cpu/*
+go/arrow/type_string.go
+go/*.tmpldata
+go/*.s
 js/.npmignore
 js/closure-compiler-scripts/*
 python/cmake_modules
diff --git a/go/arrow/.editorconfig b/go/arrow/.editorconfig
new file mode 100644
index 0000000000000..a7ceaf938f92c
--- /dev/null
+++ b/go/arrow/.editorconfig
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+root = true
+
+[*.tmpl]
+indent_style = tab
+indent_size = 4
\ No newline at end of file
diff --git a/go/arrow/.gitignore b/go/arrow/.gitignore
new file mode 100644
index 0000000000000..d4b831ae811da
--- /dev/null
+++ b/go/arrow/.gitignore
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### Go template
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+*.o
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+
+bin/
+vendor/
\ No newline at end of file
diff --git a/go/arrow/Gopkg.lock b/go/arrow/Gopkg.lock
new file mode 100644
index 0000000000000..30a0d82f4e2e2
--- /dev/null
+++ b/go/arrow/Gopkg.lock
@@ -0,0 +1,27 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
+  version = "v1.1.0"
+
+[[projects]]
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
+[[projects]]
+  name = "github.com/stretchr/testify"
+  packages = ["assert"]
+  revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c"
+  version = "v1.2.0"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "a9dd4821c4522b2069722c1be94b191e7aff1736aeb5c12ab0070f87cdbc5af2"
+  solver-name = "gps-cdcl"
+  solver-version = 1
diff --git a/go/arrow/Gopkg.toml b/go/arrow/Gopkg.toml
new file mode 100644
index 0000000000000..03bbeba1c0e10
--- /dev/null
+++ b/go/arrow/Gopkg.toml
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "1.2.0"
diff --git a/go/arrow/Makefile b/go/arrow/Makefile
new file mode 100644
index 0000000000000..a1fd22589317e
--- /dev/null
+++ b/go/arrow/Makefile
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GO_BUILD=go build
+GO_GEN=go generate
+GO_TEST?=go test
+GOPATH=$(realpath ../../../../../..)
+
+GO_SOURCES  := $(shell find . -path ./_lib -prune -o -name '*.go' -not -name '*_test.go')
+ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -not -name '*_test.go')
+SOURCES_NO_VENDOR := $(shell find . -path ./vendor -prune -o -name "*.go" -not -name '*_test.go' -print)
+
+.PHONEY: test bench assembly generate
+
+assembly:
+	@$(MAKE) -C memory assembly
+	@$(MAKE) -C math assembly
+
+generate: bin/tmpl
+	bin/tmpl -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
+	bin/tmpl -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl
+	@$(MAKE) -C math generate
+
+fmt: $(SOURCES_NO_VENDOR)
+	goimports -w $^
+
+bench: $(GO_SOURCES) | assembly
+	$(GO_TEST) $(GO_TEST_ARGS) -bench=. -run=- ./...
+
+bench-noasm: $(GO_SOURCES)
+	$(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' -bench=. -run=- ./...
+
+test: $(GO_SOURCES) | assembly
+	$(GO_TEST) $(GO_TEST_ARGS) ./...
+
+test-noasm: $(GO_SOURCES)
+	$(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' ./...
+
+bin/tmpl: _tools/tmpl/main.go
+	$(GO_BUILD) -o $@ ./_tools/tmpl
+
diff --git a/go/arrow/README.md b/go/arrow/README.md
new file mode 100644
index 0000000000000..e4d349de95c2d
--- /dev/null
+++ b/go/arrow/README.md
@@ -0,0 +1,174 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+Apache Arrow for Go
+===================
+
+[Apache Arrow][arrow] is a cross-language development platform for in-memory data. It specifies a 
+standardized language-independent columnar memory format for flat and hierarchical data, 
+organized for efficient analytic operations on modern hardware. It also provides computational 
+libraries and zero-copy streaming messaging and inter-process communication.
+
+
+Reference Counting
+------------------
+
+arrow makes use of reference counting so that it can track when memory buffers are no longer used. This allows 
+arrow to update resource accounting, pool memory such and track overall memory usage as objects are created 
+and released. Types expose two methods to deal with this pattern. The `Retain` method will increase the 
+reference count by 1 and `Release` method will reduce the count by 1. Once the reference count of an object 
+is zero, any associated object will be freed. `Retain` and `Release` are safe to call from multiple goroutines.
+
+### When to call `Retain` / `Release`?
+
+* If you are passed an object and wish to take ownership of it, you must call `Retain`. You must later pair this 
+  with a call to `Release` when you no longer need the object.  "Taking ownership" typically means you
+  wish to access the object outside the scope of the current function call.
+  
+* You own any object you create via functions whose name begins with `New` or `Copy` or when receiving
+  an object over a channel. Therefore you must call `Release` once you no longer need the object.
+  
+* If you send an object over a channel, you must call `Retain` before sending it as the receiver is
+  assumed to own the object and will later call `Release` when it no longer needs the object. 
+
+
+Performance
+-----------
+
+The arrow package makes extensive use of [c2goasm][] to leverage LLVM's advanced optimizer and generate PLAN9 
+assembly functions from C/C++ code. The arrow package can be compiled without these optimizations using the `noasm` 
+build tag. Alternatively, by configuring an environment variable, it is possible to dynamically configure which 
+architecture optimizations are used at runtime. 
+See the `cpu` package [README](internal/cpu/README.md) for a description of this environment variable.
+
+### Example Usage
+
+The following benchmarks demonstrate summing an array of 8192 values using various optimizations. 
+
+Disable no architecture optimizations (thus using AVX2):
+
+```sh
+$ INTEL_DISABLE_EXT=NONE go test -bench=8192 -run=. ./math
+goos: darwin
+goarch: amd64
+pkg: github.com/apache/arrow/go/arrow/math
+BenchmarkFloat64Funcs_Sum_8192-8   	 2000000	       687 ns/op	95375.41 MB/s
+BenchmarkInt64Funcs_Sum_8192-8     	 2000000	       719 ns/op	91061.06 MB/s
+BenchmarkUint64Funcs_Sum_8192-8    	 2000000	       691 ns/op	94797.29 MB/s
+PASS
+ok  	github.com/apache/arrow/go/arrow/math	6.444s
+```
+
+**NOTE:** `NONE` is simply ignored, thus enabling optimizations for AVX2 and SSE4
+
+----
+
+Disable AVX2 architecture optimizations:
+
+```sh
+$ INTEL_DISABLE_EXT=AVX2 go test -bench=8192 -run=. ./math
+goos: darwin
+goarch: amd64
+pkg: github.com/apache/arrow/go/arrow/math
+BenchmarkFloat64Funcs_Sum_8192-8   	 1000000	      1912 ns/op	34263.63 MB/s
+BenchmarkInt64Funcs_Sum_8192-8     	 1000000	      1392 ns/op	47065.57 MB/s
+BenchmarkUint64Funcs_Sum_8192-8    	 1000000	      1405 ns/op	46636.41 MB/s
+PASS
+ok  	github.com/apache/arrow/go/arrow/math	4.786s
+```
+
+----
+
+Disable ALL architecture optimizations, thus using pure Go implementation:
+
+```sh
+$ INTEL_DISABLE_EXT=ALL go test -bench=8192 -run=. ./math
+goos: darwin
+goarch: amd64
+pkg: github.com/apache/arrow/go/arrow/math
+BenchmarkFloat64Funcs_Sum_8192-8   	  200000	     10285 ns/op	6371.41 MB/s
+BenchmarkInt64Funcs_Sum_8192-8     	  500000	      3892 ns/op	16837.37 MB/s
+BenchmarkUint64Funcs_Sum_8192-8    	  500000	      3929 ns/op	16680.00 MB/s
+PASS
+ok  	github.com/apache/arrow/go/arrow/math	6.179s
+```
+
+Status
+------
+
+The first milestone was to implement the necessary Array types in order to use
+them internally in the [ifql][] execution engine and storage layers of [InfluxDB][].
+
+
+### Memory Management
+
+- [x] Allocations are 64-byte aligned and padded to 8-bytes
+
+
+### Array and builder support
+
+**Primitive types**
+
+- [x] Signed and unsigned 8, 16, 32 and 64 bit integers
+- [x] 32 and 64 bit floats
+- [x] Packed LSB booleans
+- [x] Variable-length binary
+- [ ] String (valid UTF-8)
+- [ ] Half-float (16-bit)
+- [ ] Null (no physical storage)
+
+**Parametric types**
+
+- [x] Timestamp
+- [ ] Interval (year/month or day/time)
+- [ ] Date32 (days since UNIX epoch)
+- [ ] Date64 (milliseconds since UNIX epoch)
+- [ ] Time32 (seconds or milliseconds since midnight)
+- [ ] Time64 (microseconds or nanoseconds since midnight)
+- [ ] Decimal (128-bit)
+- [ ] Fixed-sized binary
+- [ ] List
+- [ ] Struct
+- [ ] Union
+    - [ ] Dense
+    - [ ] Sparse
+- [ ] Dictionary 
+    - [ ] Dictionary encoding
+
+### Type metadata
+
+- [x] Data types (implemented arrays)
+- [ ] Field
+- [ ] Schema
+  
+
+### I/O 
+
+Serialization is planned for a future iteration.
+
+- [ ] Flat buffers for serializing metadata
+- [ ] Record Batch
+- [ ] Table
+
+
+
+[arrow]:    https://arrow.apache.org
+[ifql]:     https://github.com/influxdata/ifql
+[InfluxDB]: https://github.com/influxdata/influxdb
+[c2goasm]:  https://github.com/minio/c2goasm
\ No newline at end of file
diff --git a/go/arrow/_examples/helloworld/main.go b/go/arrow/_examples/helloworld/main.go
new file mode 100644
index 0000000000000..ab21f6b42c055
--- /dev/null
+++ b/go/arrow/_examples/helloworld/main.go
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/math"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+func main() {
+	fb := array.NewFloat64Builder(memory.DefaultAllocator)
+
+	fb.AppendValues([]float64{1, 3, 5, 7, 9, 11}, nil)
+
+	vec := fb.NewFloat64Array()
+	math.Float64.Sum(vec)
+}
diff --git a/go/arrow/_tools/tmpl/main.go b/go/arrow/_tools/tmpl/main.go
new file mode 100644
index 0000000000000..0cabef3db2c8c
--- /dev/null
+++ b/go/arrow/_tools/tmpl/main.go
@@ -0,0 +1,267 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"go/format"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"text/template"
+)
+
+const Ext = ".tmpl"
+
+type pathSpec struct {
+	in, out string
+}
+
+func (p *pathSpec) String() string { return p.in + " → " + p.out }
+func (p *pathSpec) IsGoFile() bool { return filepath.Ext(p.out) == ".go" }
+
+func parsePath(path string) (string, string) {
+	p := strings.IndexByte(path, '=')
+	if p == -1 {
+		if filepath.Ext(path) != Ext {
+			errExit("template file '%s' must have .tmpl extension", path)
+		}
+		return path, path[:len(path)-len(Ext)]
+	}
+
+	return path[:p], path[p+1:]
+}
+
+type data struct {
+	In interface{}
+	D  listValue
+}
+
+func errExit(format string, a ...interface{}) {
+	fmt.Fprintf(os.Stderr, format, a...)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+type listValue map[string]string
+
+func (l listValue) String() string {
+	res := make([]string, 0, len(l))
+	for k, v := range l {
+		res = append(res, fmt.Sprintf("%s=%s", k, v))
+	}
+	return strings.Join(res, ", ")
+}
+
+func (l listValue) Set(v string) error {
+	nv := strings.Split(v, "=")
+	if len(nv) != 2 {
+		return fmt.Errorf("expected NAME=VALUE, got %s", v)
+	}
+	l[nv[0]] = nv[1]
+	return nil
+}
+
+func main() {
+	var (
+		dataArg = flag.String("data", "", "input JSON data")
+		gi      = flag.Bool("i", false, "run goimports")
+		in      = &data{D: make(listValue)}
+	)
+
+	flag.Var(&in.D, "d", "-d NAME=VALUE")
+
+	flag.Parse()
+	if *dataArg == "" {
+		errExit("data option is required")
+	}
+
+	if *gi {
+		if _, err := exec.LookPath("goimports"); err != nil {
+			errExit("failed to find goimports: %s", err.Error())
+		}
+		formatter = formatSource
+	} else {
+		formatter = format.Source
+	}
+
+	paths := flag.Args()
+	if len(paths) == 0 {
+		errExit("no tmpl files specified")
+	}
+
+	specs := make([]pathSpec, len(paths))
+	for i, p := range paths {
+		in, out := parsePath(p)
+		specs[i] = pathSpec{in: in, out: out}
+	}
+
+	in.In = readData(*dataArg)
+	process(in, specs)
+}
+
+func mustReadAll(path string) []byte {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		errExit(err.Error())
+	}
+
+	return data
+}
+
+func readData(path string) interface{} {
+	data := mustReadAll(path)
+	var v interface{}
+	if err := json.Unmarshal(StripComments(data), &v); err != nil {
+		errExit("invalid JSON data: %s", err.Error())
+	}
+	return v
+}
+
+func fileMode(path string) os.FileMode {
+	stat, err := os.Stat(path)
+	if err != nil {
+		errExit(err.Error())
+	}
+	return stat.Mode()
+}
+
+var funcs = template.FuncMap{
+	"lower": strings.ToLower,
+	"upper": strings.ToUpper,
+}
+
+func process(data interface{}, specs []pathSpec) {
+	for _, spec := range specs {
+		var (
+			t   *template.Template
+			err error
+		)
+		t, err = template.New("gen").Funcs(funcs).Parse(string(mustReadAll(spec.in)))
+		if err != nil {
+			errExit("error processing template '%s': %s", spec.in, err.Error())
+		}
+
+		var buf bytes.Buffer
+		if spec.IsGoFile() {
+			// preamble
+			fmt.Fprintf(&buf, "// Code generated by %s. DO NOT EDIT.\n", spec.in)
+			fmt.Fprintln(&buf)
+		}
+		err = t.Execute(&buf, data)
+		if err != nil {
+			errExit("error executing template '%s': %s", spec.in, err.Error())
+		}
+
+		generated := buf.Bytes()
+		if spec.IsGoFile() {
+			generated, err = formatter(generated)
+			if err != nil {
+				errExit("error formatting '%s': %s", spec.in, err.Error())
+			}
+		}
+
+		ioutil.WriteFile(spec.out, generated, fileMode(spec.in))
+	}
+}
+
+var (
+	formatter func([]byte) ([]byte, error)
+)
+
+func formatSource(in []byte) ([]byte, error) {
+	r := bytes.NewReader(in)
+	cmd := exec.Command("goimports")
+	cmd.Stdin = r
+	out, err := cmd.Output()
+	if err != nil {
+		if ee, ok := err.(*exec.ExitError); ok {
+			return nil, fmt.Errorf("error running goimports: %s", string(ee.Stderr))
+		}
+		return nil, fmt.Errorf("error running goimports: %s", string(out))
+	}
+
+	return out, nil
+}
+
+func StripComments(raw []byte) []byte {
+	var (
+		quoted, esc bool
+		comment     bool
+	)
+
+	buf := bytes.Buffer{}
+
+	for i := 0; i < len(raw); i++ {
+		b := raw[i]
+
+		if comment {
+			switch b {
+			case '/':
+				comment = false
+				j := bytes.IndexByte(raw[i+1:], '\n')
+				if j == -1 {
+					i = len(raw)
+				} else {
+					i += j // keep new line
+				}
+			case '*':
+				j := bytes.Index(raw[i+1:], []byte("*/"))
+				if j == -1 {
+					i = len(raw)
+				} else {
+					i += j + 2
+					comment = false
+				}
+			}
+			continue
+		}
+
+		if esc {
+			esc = false
+			continue
+		}
+
+		if b == '\\' && quoted {
+			esc = true
+			continue
+		}
+
+		if b == '"' || b == '\'' {
+			quoted = !quoted
+		}
+
+		if b == '/' && !quoted {
+			comment = true
+			continue
+		}
+
+		buf.WriteByte(b)
+	}
+
+	if quoted || esc || comment {
+		// unexpected state, so return raw bytes
+		return raw
+	}
+
+	return buf.Bytes()
+}
diff --git a/go/arrow/_tools/tmpl/main_test.go b/go/arrow/_tools/tmpl/main_test.go
new file mode 100644
index 0000000000000..831cf791e3a0b
--- /dev/null
+++ b/go/arrow/_tools/tmpl/main_test.go
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"testing"
+)
+
+func TestStripComments(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		exp  string
+	}{
+		{name: "none", in: `[1,2,3]`, exp: `[1,2,3]`},
+		{name: "single-line, line comment at end", in: `[1,2,3] // foo bar`, exp: `[1,2,3] `},
+		{name: "single-line, block comment at end", in: `[1,2,3] /* foo bar */  `, exp: `[1,2,3]   `},
+		{name: "single-line, block comment at end", in: `[1,2,3] /* /* // */`, exp: `[1,2,3] `},
+		{name: "single-line, block comment in middle", in: `[1,/* foo bar */2,3]`, exp: `[1,2,3]`},
+		{name: "single-line, block comment in string", in: `[1,"/* foo bar */"]`, exp: `[1,"/* foo bar */"]`},
+		{name: "single-line, malformed block comment", in: `[1,2,/*]`, exp: `[1,2,/*]`},
+		{name: "single-line, malformed JSON", in: `[1,2,/]`, exp: `[1,2,/]`},
+
+		{
+			name: "multi-line",
+			in: `[
+  1,
+  2,
+  3
+]`,
+			exp: `[
+  1,
+  2,
+  3
+]`,
+		},
+		{
+			name: "multi-line, multiple line comments",
+			in: `[ // foo
+  1, // bar
+  2,
+  3
+] // fit`,
+			exp: `[ 
+  1, 
+  2,
+  3
+] `,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := string(StripComments([]byte(test.in)))
+			if got != test.exp {
+				t.Errorf("got:\n%s\nexp:\n%s", got, test.exp)
+			}
+		})
+	}
+}
diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go
new file mode 100644
index 0000000000000..a789d75b3ef2c
--- /dev/null
+++ b/go/arrow/array/array.go
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/internal/debug"
+)
+
+// A type which satisfies array.Interface represents an immutable sequence of values.
+type Interface interface {
+	// DataType returns the type metadata for this instance.
+	DataType() arrow.DataType
+
+	// NullN returns the number of null values in the array.
+	NullN() int
+
+	// NullBitmapBytes returns a byte slice of the validity bitmap.
+	NullBitmapBytes() []byte
+
+	// IsNull returns true if value at index is null.
+	// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
+	IsNull(i int) bool
+
+	// IsValid returns true if value at index is not null.
+	// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
+	IsValid(i int) bool
+
+	Data() *Data
+
+	// Len returns the number of elements in the array.
+	Len() int
+
+	// Retain increases the reference count by 1.
+	// Retain may be called simultaneously from multiple goroutines.
+	Retain()
+
+	// Release decreases the reference count by 1.
+	// Release may be called simultaneously from multiple goroutines.
+	// When the reference count goes to zero, the memory is freed.
+	Release()
+}
+
+const (
+	// UnknownNullCount specifies the NullN should be calculated from the null bitmap buffer.
+	UnknownNullCount = -1
+)
+
+type array struct {
+	refCount        int64
+	data            *Data
+	nullBitmapBytes []byte
+}
+
+// Retain increases the reference count by 1.
+// Retain may be called simultaneously from multiple goroutines.
+func (a *array) Retain() {
+	atomic.AddInt64(&a.refCount, 1)
+}
+
+// Release decreases the reference count by 1.
+// Release may be called simultaneously from multiple goroutines.
+// When the reference count goes to zero, the memory is freed.
+func (a *array) Release() {
+	debug.Assert(atomic.LoadInt64(&a.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&a.refCount, -1) == 0 {
+		a.data.Release()
+		a.data, a.nullBitmapBytes = nil, nil
+	}
+}
+
+// DataType returns the type metadata for this instance.
+func (a *array) DataType() arrow.DataType { return a.data.typE }
+
+// NullN returns the number of null values in the array.
+func (a *array) NullN() int {
+	if a.data.nullN < 0 {
+		a.data.nullN = a.data.length - bitutil.CountSetBits(a.nullBitmapBytes, a.data.length)
+	}
+	return a.data.nullN
+}
+
+// NullBitmapBytes returns a byte slice of the validity bitmap.
+func (a *array) NullBitmapBytes() []byte { return a.nullBitmapBytes }
+
+func (a *array) Data() *Data { return a.data }
+
+// Len returns the number of elements in the array.
+func (a *array) Len() int { return a.data.length }
+
+// IsNull returns true if value at index is null.
+// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
+func (a *array) IsNull(i int) bool {
+	return len(a.nullBitmapBytes) != 0 && bitutil.BitIsNotSet(a.nullBitmapBytes, i)
+}
+
+// IsValid returns true if value at index is not null.
+// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
+func (a *array) IsValid(i int) bool {
+	return len(a.nullBitmapBytes) == 0 || bitutil.BitIsSet(a.nullBitmapBytes, i)
+}
+
+func (a *array) setData(data *Data) {
+	if a.data != nil {
+		a.data.Release()
+	}
+
+	data.Retain()
+	if len(data.buffers) > 0 && data.buffers[0] != nil {
+		a.nullBitmapBytes = data.buffers[0].Bytes()
+	}
+	a.data = data
+}
+
+type arrayConstructorFn func(*Data) Interface
+
+var (
+	makeArrayFn = [...]arrayConstructorFn{
+		arrow.NULL:              unsupportedArrayType,
+		arrow.BOOL:              func(data *Data) Interface { return NewBooleanData(data) },
+		arrow.UINT8:             unsupportedArrayType,
+		arrow.INT8:              unsupportedArrayType,
+		arrow.UINT16:            unsupportedArrayType,
+		arrow.INT16:             unsupportedArrayType,
+		arrow.UINT32:            unsupportedArrayType,
+		arrow.INT32:             func(data *Data) Interface { return NewInt32Data(data) },
+		arrow.UINT64:            func(data *Data) Interface { return NewUint64Data(data) },
+		arrow.INT64:             func(data *Data) Interface { return NewInt64Data(data) },
+		arrow.HALF_FLOAT:        unsupportedArrayType,
+		arrow.FLOAT32:           unsupportedArrayType,
+		arrow.FLOAT64:           func(data *Data) Interface { return NewFloat64Data(data) },
+		arrow.STRING:            unsupportedArrayType,
+		arrow.BINARY:            func(data *Data) Interface { return NewBinaryData(data) },
+		arrow.FIXED_SIZE_BINARY: unsupportedArrayType,
+		arrow.DATE32:            unsupportedArrayType,
+		arrow.DATE64:            unsupportedArrayType,
+		arrow.TIMESTAMP:         func(data *Data) Interface { return NewTimestampData(data) },
+		arrow.TIME32:            unsupportedArrayType,
+		arrow.TIME64:            unsupportedArrayType,
+		arrow.INTERVAL:          unsupportedArrayType,
+		arrow.DECIMAL:           unsupportedArrayType,
+		arrow.LIST:              unsupportedArrayType,
+		arrow.STRUCT:            unsupportedArrayType,
+		arrow.UNION:             unsupportedArrayType,
+		arrow.DICTIONARY:        unsupportedArrayType,
+		arrow.MAP:               unsupportedArrayType,
+
+		// invalid data types to fill out array size 2⁵-1
+		28: invalidDataType,
+		29: invalidDataType,
+		30: invalidDataType,
+		31: invalidDataType,
+	}
+)
+
+func unsupportedArrayType(data *Data) Interface {
+	panic("unsupported data type: " + data.typE.ID().String())
+}
+
+func invalidDataType(data *Data) Interface {
+	panic("invalid data type: " + data.typE.ID().String())
+}
+
+// MakeFromData constructs a strongly-typed array instance from generic Data.
+func MakeFromData(data *Data) Interface {
+	return makeArrayFn[byte(data.typE.ID()&0x1f)](data)
+}
diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go
new file mode 100644
index 0000000000000..6d505cf9f189e
--- /dev/null
+++ b/go/arrow/array/array_test.go
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/internal/testing/tools"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+type testDataType struct {
+	id arrow.Type
+}
+
+func (d *testDataType) ID() arrow.Type { return d.id }
+func (d *testDataType) Name() string   { panic("implement me") }
+
+func TestMakeFromData(t *testing.T) {
+	tests := []struct {
+		name     string
+		d        arrow.DataType
+		expPanic bool
+		expError string
+	}{
+		// unsupported types
+		{name: "null", d: &testDataType{arrow.NULL}, expPanic: true, expError: "unsupported data type: NULL"},
+		{name: "map", d: &testDataType{arrow.MAP}, expPanic: true, expError: "unsupported data type: MAP"},
+
+		// supported types
+		{name: "bool", d: &testDataType{arrow.BOOL}},
+
+		// invalid types
+		{name: "invalid(-1)", d: &testDataType{arrow.Type(-1)}, expPanic: true, expError: "invalid data type: Type(-1)"},
+		{name: "invalid(28)", d: &testDataType{arrow.Type(28)}, expPanic: true, expError: "invalid data type: Type(28)"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var b [4]*memory.Buffer
+			data := array.NewData(test.d, 0, b[:], 0)
+
+			if test.expPanic {
+				assert.PanicsWithValue(t, test.expError, func() {
+					array.MakeFromData(data)
+				})
+			} else {
+				assert.NotNil(t, array.MakeFromData(data))
+			}
+		})
+	}
+}
+
+func bbits(v ...int32) []byte {
+	return tools.IntsToBitsLSB(v...)
+}
+
+func TestArray_NullN(t *testing.T) {
+	tests := []struct {
+		name string
+		l    int
+		bm   []byte
+		n    int
+		exp  int
+	}{
+		{name: "unknown,l16", l: 16, bm: bbits(0x11001010, 0x00110011), n: array.UnknownNullCount, exp: 8},
+		{name: "unknown,l12,ignores last nibble", l: 12, bm: bbits(0x11001010, 0x00111111), n: array.UnknownNullCount, exp: 6},
+		{name: "unknown,l12,12 nulls", l: 12, bm: bbits(0x00000000, 0x00000000), n: array.UnknownNullCount, exp: 12},
+		{name: "unknown,l12,00 nulls", l: 12, bm: bbits(0x11111111, 0x11111111), n: array.UnknownNullCount, exp: 0},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			buf := memory.NewBufferBytes(test.bm)
+			data := array.NewData(arrow.FixedWidthTypes.Boolean, test.l, []*memory.Buffer{buf, nil}, test.n)
+			buf.Release()
+			ar := array.MakeFromData(data)
+			data.Release()
+			got := ar.NullN()
+			ar.Release()
+			assert.Equal(t, test.exp, got)
+		})
+	}
+}
diff --git a/go/arrow/array/binary.go b/go/arrow/array/binary.go
new file mode 100644
index 0000000000000..0b89b7e5817cc
--- /dev/null
+++ b/go/arrow/array/binary.go
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow"
+)
+
+// A type which represents an immutable sequence of variable-length binary strings.
+type Binary struct {
+	array
+	valueOffsets []int32
+	valueBytes   []byte
+}
+
+// NewBinaryData constructs a new Binary array from data.
+func NewBinaryData(data *Data) *Binary {
+	a := &Binary{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+// Value returns the slice at index i. This value should not be mutated.
+func (a *Binary) Value(i int) []byte { return a.valueBytes[a.valueOffsets[i]:a.valueOffsets[i+1]] }
+
+// ValueString returns the string at index i without performing additional allocations.
+// The string is only valid for the lifetime of the Binary array.
+func (a *Binary) ValueString(i int) string {
+	b := a.Value(i)
+	return *(*string)(unsafe.Pointer(&b))
+}
+
+func (a *Binary) ValueOffset(i int) int { return int(a.valueOffsets[i]) }
+func (a *Binary) ValueLen(i int) int    { return int(a.valueOffsets[i+1] - a.valueOffsets[i]) }
+func (a *Binary) ValueOffsets() []int32 { return a.valueOffsets }
+func (a *Binary) ValueBytes() []byte    { return a.valueBytes }
+
+func (a *Binary) setData(data *Data) {
+	if len(data.buffers) != 3 {
+		panic("len(data.buffers) != 3")
+	}
+
+	a.array.setData(data)
+
+	if valueData := data.buffers[2]; valueData != nil {
+		a.valueBytes = valueData.Bytes()
+	}
+
+	if valueOffsets := data.buffers[1]; valueOffsets != nil {
+		a.valueOffsets = arrow.Int32Traits.CastFromBytes(valueOffsets.Bytes())
+	}
+}
diff --git a/go/arrow/array/binarybuilder.go b/go/arrow/array/binarybuilder.go
new file mode 100644
index 0000000000000..71aa4ccd51f5d
--- /dev/null
+++ b/go/arrow/array/binarybuilder.go
@@ -0,0 +1,184 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"math"
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/debug"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+const (
+	binaryArrayMaximumCapacity = math.MaxInt32
+)
+
+// A BinaryBuilder is used to build a Binary array using the Append methods.
+type BinaryBuilder struct {
+	builder
+
+	typE    arrow.BinaryDataType
+	offsets *int32BufferBuilder
+	values  *byteBufferBuilder
+}
+
+func NewBinaryBuilder(mem memory.Allocator, typE arrow.BinaryDataType) *BinaryBuilder {
+	b := &BinaryBuilder{
+		builder: builder{refCount: 1, mem: mem},
+		typE:    typE,
+		offsets: newInt32BufferBuilder(mem),
+		values:  newByteBufferBuilder(mem),
+	}
+	return b
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+// Release may be called simultaneously from multiple goroutines.
+func (b *BinaryBuilder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.offsets != nil {
+			b.offsets.Release()
+			b.offsets = nil
+		}
+		if b.values != nil {
+			b.values.Release()
+			b.values = nil
+		}
+	}
+}
+
+func (b *BinaryBuilder) Append(v []byte) {
+	b.Reserve(1)
+	b.appendNextOffset()
+	b.values.Append(v)
+	b.UnsafeAppendBoolToBitmap(true)
+}
+
+func (b *BinaryBuilder) AppendString(v string) {
+	b.Append([]byte(v))
+}
+
+func (b *BinaryBuilder) AppendNull() {
+	b.Reserve(1)
+	b.appendNextOffset()
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *BinaryBuilder) AppendValues(v [][]byte, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	for _, vv := range v {
+		b.appendNextOffset()
+		b.values.Append(vv)
+	}
+
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+// AppendStringValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *BinaryBuilder) AppendStringValues(v []string, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	for _, vv := range v {
+		b.appendNextOffset()
+		b.values.Append([]byte(vv))
+	}
+
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *BinaryBuilder) Value(i int) []byte {
+	offsets := b.offsets.Values()
+	start := int(offsets[i])
+	var end int
+	if i == (b.length - 1) {
+		end = b.values.Len()
+	} else {
+		end = int(offsets[i+1])
+	}
+	return b.values.Bytes()[start:end]
+}
+
+func (b *BinaryBuilder) init(capacity int) {
+	b.builder.init(capacity)
+	b.offsets.resize((capacity + 1) * arrow.Int32SizeBytes)
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *BinaryBuilder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *BinaryBuilder) Resize(n int) {
+	b.offsets.resize((n + 1) * arrow.Int32SizeBytes)
+	b.builder.resize(n, b.init)
+}
+
+// NewBinaryArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder
+// so it can be used to build a new array.
+func (b *BinaryBuilder) NewBinaryArray() (a *Binary) {
+	data := b.newData()
+	a = NewBinaryData(data)
+	data.Release()
+	return
+}
+
+func (b *BinaryBuilder) newData() (data *Data) {
+	b.appendNextOffset()
+	offsets, values := b.offsets.Finish(), b.values.Finish()
+	data = NewData(b.typE, b.length, []*memory.Buffer{b.nullBitmap, offsets, values}, b.nullN)
+	if offsets != nil {
+		offsets.Release()
+	}
+
+	if values != nil {
+		values.Release()
+	}
+
+	b.builder.reset()
+
+	return
+}
+
+func (b *BinaryBuilder) appendNextOffset() {
+	numBytes := b.values.Len()
+	// TODO(sgc): check binaryArrayMaximumCapacity?
+	b.offsets.AppendValue(int32(numBytes))
+}
diff --git a/go/arrow/array/binarybuilder_test.go b/go/arrow/array/binarybuilder_test.go
new file mode 100644
index 0000000000000..4de512499d34e
--- /dev/null
+++ b/go/arrow/array/binarybuilder_test.go
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBinaryBuilder(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	ab := array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
+
+	exp := [][]byte{[]byte("foo"), []byte("bar"), nil, []byte("sydney"), []byte("cameron")}
+	for _, v := range exp {
+		if v == nil {
+			ab.AppendNull()
+		} else {
+			ab.Append(v)
+		}
+	}
+
+	assert.Equal(t, len(exp), ab.Len(), "unexpected Len()")
+	assert.Equal(t, 1, ab.NullN(), "unexpected NullN()")
+
+	for i, v := range exp {
+		if v == nil {
+			v = []byte{}
+		}
+		assert.Equal(t, v, ab.Value(i), "unexpected BinaryArrayBuilder.Value(%d)", i)
+	}
+
+	ar := ab.NewBinaryArray()
+	ab.Release()
+	ar.Release()
+
+	// check state of builder after NewBinaryArray
+	assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewBinaryArray did not reset state")
+	assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewBinaryArray did not reset state")
+	assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewBinaryArray did not reset state")
+}
diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go
new file mode 100644
index 0000000000000..0d41381b0415d
--- /dev/null
+++ b/go/arrow/array/boolean.go
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+// A type which represents an immutable sequence of boolean values.
+type Boolean struct {
+	array
+	values []byte
+}
+
+// NewBoolean creates a boolean array from the data memory.Buffer and contains length elements.
+// The nullBitmap buffer can be nil of there are no null values.
+// If nullN is not known, use UnknownNullCount to calculate the value of NullN at runtime from the nullBitmap buffer.
+func NewBoolean(length int, data *memory.Buffer, nullBitmap *memory.Buffer, nullN int) *Boolean {
+	return NewBooleanData(NewData(arrow.FixedWidthTypes.Boolean, length, []*memory.Buffer{nullBitmap, data}, nullN))
+}
+
+func NewBooleanData(data *Data) *Boolean {
+	a := &Boolean{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Boolean) Value(i int) bool { return bitutil.BitIsSet(a.values, i) }
+
+func (a *Boolean) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = vals.Bytes()
+	}
+}
diff --git a/go/arrow/array/booleanbuilder.go b/go/arrow/array/booleanbuilder.go
new file mode 100644
index 0000000000000..05e506fb2b7f8
--- /dev/null
+++ b/go/arrow/array/booleanbuilder.go
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/internal/debug"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+type BooleanBuilder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []byte
+}
+
+func NewBooleanBuilder(mem memory.Allocator) *BooleanBuilder {
+	return &BooleanBuilder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+// Release may be called simultaneously from multiple goroutines.
+func (b *BooleanBuilder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *BooleanBuilder) Append(v bool) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *BooleanBuilder) AppendByte(v byte) {
+	b.Reserve(1)
+	b.UnsafeAppend(v != 0)
+}
+
+func (b *BooleanBuilder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *BooleanBuilder) UnsafeAppend(v bool) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	if v {
+		bitutil.SetBit(b.rawData, b.length)
+	} else {
+		bitutil.ClearBit(b.rawData, b.length)
+	}
+	b.length++
+}
+
+func (b *BooleanBuilder) AppendValues(v []bool, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	for i, vv := range v {
+		bitutil.SetBitTo(b.rawData, b.length+i, vv)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *BooleanBuilder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.BooleanTraits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = b.data.Bytes()
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *BooleanBuilder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *BooleanBuilder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.BooleanTraits.BytesRequired(n))
+		b.rawData = b.data.Bytes()
+	}
+}
+
+// NewBooleanArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder
+// so it can be used to build a new array.
+func (b *BooleanBuilder) NewBooleanArray() (a *Boolean) {
+	data := b.newData()
+	a = NewBooleanData(data)
+	data.Release()
+	return
+}
+
+func (b *BooleanBuilder) newData() *Data {
+	bytesRequired := arrow.BooleanTraits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	res := NewData(arrow.FixedWidthTypes.Boolean, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return res
+}
diff --git a/go/arrow/array/booleanbuilder_test.go b/go/arrow/array/booleanbuilder_test.go
new file mode 100644
index 0000000000000..5540c9b337712
--- /dev/null
+++ b/go/arrow/array/booleanbuilder_test.go
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/internal/testing/tools"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBooleanBuilder_AppendValues(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	b := array.NewBooleanBuilder(mem)
+
+	exp := tools.Bools(1, 1, 0, 1, 1, 0, 1, 0)
+	got := make([]bool, len(exp))
+
+	b.AppendValues(exp, nil)
+	a := b.NewBooleanArray()
+	b.Release()
+	for i := 0; i < a.Len(); i++ {
+		got[i] = a.Value(i)
+	}
+	assert.Equal(t, exp, got)
+	a.Release()
+}
diff --git a/go/arrow/array/bufferbuilder.go b/go/arrow/array/bufferbuilder.go
new file mode 100644
index 0000000000000..29d97fc8d41f9
--- /dev/null
+++ b/go/arrow/array/bufferbuilder.go
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/internal/debug"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+// A bufferBuilder provides common functionality for populating memory with a sequence of type-specific values.
+// Specialized implementations provide type-safe APIs for appending and accessing the memory.
+type bufferBuilder struct {
+	refCount int64
+	mem      memory.Allocator
+	buffer   *memory.Buffer
+	length   int
+	capacity int
+
+	bytes []byte
+}
+
+// Retain increases the reference count by 1.
+// Retain may be called simultaneously from multiple goroutines.
+func (b *bufferBuilder) Retain() {
+	atomic.AddInt64(&b.refCount, 1)
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+// Release may be called simultaneously from multiple goroutines.
+func (b *bufferBuilder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.buffer != nil {
+			b.buffer.Release()
+			b.buffer, b.bytes = nil, nil
+		}
+	}
+}
+
+// Len returns the length of the memory buffer in bytes.
+func (b *bufferBuilder) Len() int { return b.length }
+
+// Cap returns the total number of bytes that can be stored without allocating additional memory.
+func (b *bufferBuilder) Cap() int { return b.capacity }
+
+// Bytes returns a slice of length b.Len().
+// The slice is only valid for use until the next buffer modification. That is, until the next call
+// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
+// buffer modification.
+func (b *bufferBuilder) Bytes() []byte { return b.bytes[:b.length] }
+
+func (b *bufferBuilder) resize(elements int) {
+	if b.buffer == nil {
+		b.buffer = memory.NewResizableBuffer(b.mem)
+	}
+
+	b.buffer.Resize(elements)
+	oldCapacity := b.capacity
+	b.capacity = b.buffer.Cap()
+	b.bytes = b.buffer.Buf()
+
+	if b.capacity > oldCapacity {
+		memory.Set(b.bytes[oldCapacity:], 0)
+	}
+}
+
+// Advance increases the buffer by length and initializes the skipped bytes to zero.
+func (b *bufferBuilder) Advance(length int) {
+	if b.capacity < b.length+length {
+		newCapacity := bitutil.NextPowerOf2(b.length + length)
+		b.resize(newCapacity)
+	}
+	b.length += length
+}
+
+// Append appends the contents of v to the buffer, resizing it if necessary.
+func (b *bufferBuilder) Append(v []byte) {
+	if b.capacity < b.length+len(v) {
+		newCapacity := bitutil.NextPowerOf2(b.length + len(v))
+		b.resize(newCapacity)
+	}
+	b.unsafeAppend(v)
+}
+
+// Reset returns the buffer to an empty state. Reset releases the memory and sets the length and capacity to zero.
+func (b *bufferBuilder) Reset() {
+	if b.buffer != nil {
+		b.buffer.Release()
+	}
+	b.buffer, b.bytes = nil, nil
+	b.capacity, b.length = 0, 0
+}
+
+// Finish TODO(sgc)
+func (b *bufferBuilder) Finish() (buffer *memory.Buffer) {
+	if b.length > 0 {
+		b.buffer.ResizeNoShrink(b.length)
+	}
+	buffer = b.buffer
+	b.buffer = nil
+	b.Reset()
+	return
+}
+
+func (b *bufferBuilder) unsafeAppend(data []byte) {
+	copy(b.bytes[b.length:], data)
+	b.length += len(data)
+}
diff --git a/go/arrow/array/bufferbuilder_byte.go b/go/arrow/array/bufferbuilder_byte.go
new file mode 100644
index 0000000000000..f5f5445b7ac44
--- /dev/null
+++ b/go/arrow/array/bufferbuilder_byte.go
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import "github.com/apache/arrow/go/arrow/memory"
+
+type byteBufferBuilder struct {
+	bufferBuilder
+}
+
+func newByteBufferBuilder(mem memory.Allocator) *byteBufferBuilder {
+	return &byteBufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}}
+}
+
+func (b *byteBufferBuilder) Values() []byte   { return b.Bytes() }
+func (b *byteBufferBuilder) Value(i int) byte { return b.bytes[i] }
diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go b/go/arrow/array/bufferbuilder_numeric.gen.go
new file mode 100644
index 0000000000000..e39b2528a9eff
--- /dev/null
+++ b/go/arrow/array/bufferbuilder_numeric.gen.go
@@ -0,0 +1,58 @@
+// Code generated by array/bufferbuilder_numeric.gen.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+type int32BufferBuilder struct {
+	bufferBuilder
+}
+
+func newInt32BufferBuilder(mem memory.Allocator) *int32BufferBuilder {
+	return &int32BufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}}
+}
+
+// AppendValues appends the contents of v to the buffer, growing the buffer as needed.
+func (b *int32BufferBuilder) AppendValues(v []int32) { b.Append(arrow.Int32Traits.CastToBytes(v)) }
+
+// Values returns a slice of length b.Len().
+// The slice is only valid for use until the next buffer modification. That is, until the next call
+// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
+// buffer modification.
+func (b *int32BufferBuilder) Values() []int32 { return arrow.Int32Traits.CastFromBytes(b.Bytes()) }
+
+// Value returns the int32 element at the index i. Value will panic if i is negative or ≥ Len.
+func (b *int32BufferBuilder) Value(i int) int32 { return b.Values()[i] }
+
+// Len returns the number of int32 elements in the buffer.
+func (b *int32BufferBuilder) Len() int { return b.length / arrow.Int32SizeBytes }
+
+// AppendValue appends v to the buffer, growing the buffer as needed.
+func (b *int32BufferBuilder) AppendValue(v int32) {
+	if b.capacity < b.length+arrow.Int32SizeBytes {
+		newCapacity := bitutil.NextPowerOf2(b.length + arrow.Int32SizeBytes)
+		b.resize(newCapacity)
+	}
+	arrow.Int32Traits.PutValue(b.bytes[b.length:], v)
+	b.length += arrow.Int32SizeBytes
+}
diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl b/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl
new file mode 100644
index 0000000000000..d8b03b06ce21f
--- /dev/null
+++ b/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+{{range .In}}
+{{$TypeNamePrefix := .name}}
+{{if .Opt.BufferBuilder}}
+type {{$TypeNamePrefix}}BufferBuilder struct {
+	bufferBuilder
+}
+
+func new{{.Name}}BufferBuilder(mem memory.Allocator) *{{$TypeNamePrefix}}BufferBuilder {
+	return &{{$TypeNamePrefix}}BufferBuilder{bufferBuilder:bufferBuilder{refCount: 1, mem:mem}}
+}
+
+// AppendValues appends the contents of v to the buffer, growing the buffer as needed.
+func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValues(v []{{.Type}}) { b.Append(arrow.{{.Name}}Traits.CastToBytes(v)) }
+
+// Values returns a slice of length b.Len().
+// The slice is only valid for use until the next buffer modification. That is, until the next call
+// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
+// buffer modification.
+func (b *{{$TypeNamePrefix}}BufferBuilder) Values() []{{.Type}}           { return arrow.{{.Name}}Traits.CastFromBytes(b.Bytes()) }
+
+// Value returns the {{.Type}} element at the index i. Value will panic if i is negative or ≥ Len.
+func (b *{{$TypeNamePrefix}}BufferBuilder) Value(i int) {{.Type}}         { return b.Values()[i] }
+
+// Len returns the number of {{.Type}} elements in the buffer.
+func (b *{{$TypeNamePrefix}}BufferBuilder) Len() int                      { return b.length/arrow.{{.Name}}SizeBytes }
+
+// AppendValue appends v to the buffer, growing the buffer as needed.
+func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValue(v {{.Type}}) {
+	if b.capacity < b.length+arrow.{{.Name}}SizeBytes {
+		newCapacity := bitutil.NextPowerOf2(b.length + arrow.{{.Name}}SizeBytes)
+		b.resize(newCapacity)
+	}
+	arrow.{{.Name}}Traits.PutValue(b.bytes[b.length:], v)
+	b.length+=arrow.{{.Name}}SizeBytes
+}
+{{end}}
+{{end}}
\ No newline at end of file
diff --git a/go/arrow/array/bufferbuilder_numeric_test.go b/go/arrow/array/bufferbuilder_numeric_test.go
new file mode 100644
index 0000000000000..a2205ea48156f
--- /dev/null
+++ b/go/arrow/array/bufferbuilder_numeric_test.go
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"testing"
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInt32BufferBuilder(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	bb := newInt32BufferBuilder(mem)
+	exp := []int32{0x01020304, 0x05060708, 0x090a0b0c, 0x0d0e0f01, 0x02030405, 0x06070809}
+	bb.AppendValues(exp[:3])
+	bb.AppendValues(exp[3:])
+
+	expBuf := []byte{
+		0x04, 0x03, 0x02, 0x01,
+		0x08, 0x07, 0x06, 0x05,
+		0x0c, 0x0b, 0x0a, 0x09,
+		0x01, 0x0f, 0x0e, 0x0d,
+		0x05, 0x04, 0x03, 0x02,
+		0x09, 0x08, 0x07, 0x06,
+	}
+	assert.Equal(t, expBuf, bb.Bytes(), "unexpected byte values")
+	assert.Equal(t, exp, bb.Values(), "unexpected int32 values")
+	assert.Equal(t, len(exp), bb.Len(), "unexpected Len()")
+
+	buflen := bb.Len()
+	bfr := bb.Finish()
+	assert.Equal(t, buflen*int(unsafe.Sizeof(int32(0))), bfr.Len(), "Buffer was not resized")
+	assert.Len(t, bfr.Bytes(), bfr.Len(), "Buffer.Bytes() != Buffer.Len()")
+	bfr.Release()
+
+	assert.Len(t, bb.Bytes(), 0, "BufferBuilder was not reset after Finish")
+	assert.Zero(t, bb.Len(), "BufferBuilder was not reset after Finish")
+	bb.Release()
+}
+
+func TestInt32BufferBuilder_AppendValue(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	bb := newInt32BufferBuilder(mem)
+	exp := []int32{0x01020304, 0x05060708, 0x090a0b0c, 0x0d0e0f01, 0x02030405, 0x06070809}
+	for _, v := range exp {
+		bb.AppendValue(v)
+	}
+
+	expBuf := []byte{
+		0x04, 0x03, 0x02, 0x01,
+		0x08, 0x07, 0x06, 0x05,
+		0x0c, 0x0b, 0x0a, 0x09,
+		0x01, 0x0f, 0x0e, 0x0d,
+		0x05, 0x04, 0x03, 0x02,
+		0x09, 0x08, 0x07, 0x06,
+	}
+	assert.Equal(t, expBuf, bb.Bytes(), "unexpected byte values")
+	assert.Equal(t, exp, bb.Values(), "unexpected int32 values")
+	assert.Equal(t, len(exp), bb.Len(), "unexpected Len()")
+	bb.Release()
+}
diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go
new file mode 100644
index 0000000000000..bde12fbc88e35
--- /dev/null
+++ b/go/arrow/array/builder.go
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+const (
+	minBuilderCapacity = 1 << 5
+)
+
+// builder provides common functionality for managing the validity bitmap (nulls) when building arrays.
+type builder struct {
+	refCount   int64
+	mem        memory.Allocator
+	nullBitmap *memory.Buffer
+	nullN      int
+	length     int
+	capacity   int
+}
+
+// Retain increases the reference count by 1.
+// Retain may be called simultaneously from multiple goroutines.
+func (b *builder) Retain() {
+	atomic.AddInt64(&b.refCount, 1)
+}
+
+// Len returns the number of elements in the array builder.
+func (b *builder) Len() int { return b.length }
+
+// Cap returns the total number of elements that can be stored without allocating additional memory.
+func (b *builder) Cap() int { return b.capacity }
+
+// NullN returns the number of null values in the array builder.
+func (b *builder) NullN() int { return b.nullN }
+
+func (b *builder) init(capacity int) {
+	toAlloc := bitutil.CeilByte(capacity) / 8
+	b.nullBitmap = memory.NewResizableBuffer(b.mem)
+	b.nullBitmap.Resize(toAlloc)
+	b.capacity = capacity
+	memory.Set(b.nullBitmap.Buf(), 0)
+}
+
+func (b *builder) reset() {
+	if b.nullBitmap != nil {
+		b.nullBitmap.Release()
+		b.nullBitmap = nil
+	}
+
+	b.nullN = 0
+	b.length = 0
+	b.capacity = 0
+}
+
+func (b *builder) resize(newBits int, init func(int)) {
+	if b.nullBitmap == nil {
+		init(newBits)
+		return
+	}
+
+	newBytesN := bitutil.CeilByte(newBits) / 8
+	oldBytesN := b.nullBitmap.Len()
+	b.nullBitmap.Resize(newBytesN)
+	b.capacity = newBits
+	if oldBytesN < newBytesN {
+		// TODO(sgc): necessary?
+		memory.Set(b.nullBitmap.Buf()[oldBytesN:], 0)
+	}
+}
+
+func (b *builder) reserve(elements int, resize func(int)) {
+	if b.length+elements > b.capacity {
+		newCap := bitutil.NextPowerOf2(b.length + elements)
+		resize(newCap)
+	}
+}
+
+// unsafeAppendBoolsToBitmap appends the contents of valid to the validity bitmap.
+// As an optimization, if the valid slice is empty, the next length bits will be set to valid (not null).
+func (b *builder) unsafeAppendBoolsToBitmap(valid []bool, length int) {
+	if len(valid) == 0 {
+		b.unsafeSetValid(length)
+		return
+	}
+
+	byteOffset := b.length / 8
+	bitOffset := byte(b.length % 8)
+	nullBitmap := b.nullBitmap.Bytes()
+	bitSet := nullBitmap[byteOffset]
+
+	for _, v := range valid {
+		if bitOffset == 8 {
+			bitOffset = 0
+			nullBitmap[byteOffset] = bitSet
+			byteOffset++
+			bitSet = nullBitmap[byteOffset]
+		}
+
+		if v {
+			bitSet |= bitutil.BitMask[bitOffset]
+		} else {
+			bitSet &= bitutil.FlippedBitMask[bitOffset]
+			b.nullN++
+		}
+		bitOffset++
+	}
+
+	if bitOffset != 0 {
+		nullBitmap[byteOffset] = bitSet
+	}
+	b.length += len(valid)
+}
+
+// unsafeSetValid sets the next length bits to valid in the validity bitmap.
+func (b *builder) unsafeSetValid(length int) {
+	padToByte := min(8-(b.length%8), length)
+	if padToByte == 8 {
+		padToByte = 0
+	}
+	bits := b.nullBitmap.Bytes()
+	for i := b.length; i < b.length+padToByte; i++ {
+		bitutil.SetBit(bits, i)
+	}
+
+	start := (b.length + padToByte) / 8
+	fastLength := (length - padToByte) / 8
+	memory.Set(bits[start:start+fastLength], 0xff)
+
+	newLength := b.length + length
+	// trailing bytes
+	for i := b.length + padToByte + (fastLength * 8); i < newLength; i++ {
+		bitutil.SetBit(bits, i)
+	}
+
+	b.length = newLength
+}
+
+func (b *builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
diff --git a/go/arrow/array/builder_test.go b/go/arrow/array/builder_test.go
new file mode 100644
index 0000000000000..1ac8f5aa7bc45
--- /dev/null
+++ b/go/arrow/array/builder_test.go
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/internal/testing/tools"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuilder_Init(t *testing.T) {
+	type exp struct{ size int }
+	tests := []struct {
+		name string
+		cap  int
+
+		exp exp
+	}{
+		{"07 bits", 07, exp{size: 1}},
+		{"19 bits", 19, exp{size: 3}},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ab := &builder{mem: memory.NewGoAllocator()}
+			ab.init(test.cap)
+			assert.Equal(t, test.cap, ab.Cap(), "invalid capacity")
+			assert.Equal(t, test.exp.size, ab.nullBitmap.Len(), "invalid length")
+		})
+	}
+}
+
+func TestBuilder_UnsafeSetValid(t *testing.T) {
+	ab := &builder{mem: memory.NewGoAllocator()}
+	ab.init(32)
+	ab.unsafeAppendBoolsToBitmap(tools.Bools(0, 0, 0, 0, 0), 5)
+	assert.Equal(t, 5, ab.Len())
+	assert.Equal(t, []byte{0, 0, 0, 0}, ab.nullBitmap.Bytes())
+
+	ab.unsafeSetValid(17)
+	assert.Equal(t, []byte{0xe0, 0xff, 0x3f, 0}, ab.nullBitmap.Bytes())
+}
diff --git a/go/arrow/array/data.go b/go/arrow/array/data.go
new file mode 100644
index 0000000000000..e689f0fe49530
--- /dev/null
+++ b/go/arrow/array/data.go
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/debug"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+// A type which represents the memory and metadata for an Arrow array.
+type Data struct {
+	refCount  int64
+	typE      arrow.DataType
+	nullN     int
+	length    int
+	buffers   []*memory.Buffer // TODO(sgc): should this be an interface?
+	childData []*Data          // TODO(sgc): managed by ListArray, StructArray and UnionArray types
+}
+
+func NewData(typE arrow.DataType, length int, buffers []*memory.Buffer, nullN int) *Data {
+	for _, b := range buffers {
+		if b != nil {
+			b.Retain()
+		}
+	}
+
+	return &Data{
+		refCount: 1,
+		typE:     typE,
+		nullN:    nullN,
+		length:   length,
+		buffers:  buffers,
+	}
+}
+
+// Retain increases the reference count by 1.
+// Retain may be called simultaneously from multiple goroutines.
+func (d *Data) Retain() {
+	atomic.AddInt64(&d.refCount, 1)
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+// Release may be called simultaneously from multiple goroutines.
+func (d *Data) Release() {
+	debug.Assert(atomic.LoadInt64(&d.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&d.refCount, -1) == 0 {
+		for _, b := range d.buffers {
+			if b != nil {
+				b.Release()
+			}
+		}
+
+		for _, b := range d.childData {
+			b.Release()
+		}
+		d.buffers, d.childData = nil, nil
+	}
+}
+
+func (d *Data) DataType() arrow.DataType { return d.typE }
+func (d *Data) NullN() int               { return d.nullN }
+func (d *Data) Len() int                 { return d.length }
diff --git a/go/arrow/array/doc.go b/go/arrow/array/doc.go
new file mode 100644
index 0000000000000..5cf85408626ac
--- /dev/null
+++ b/go/arrow/array/doc.go
@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+Package array provides implementations of various Arrow array types.
+*/
+package array
diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go
new file mode 100644
index 0000000000000..02903f9e3a5d3
--- /dev/null
+++ b/go/arrow/array/numeric.gen.go
@@ -0,0 +1,287 @@
+// Code generated by array/numeric.gen.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"github.com/apache/arrow/go/arrow"
+)
+
+// A type which represents an immutable sequence of int64 values.
+type Int64 struct {
+	array
+	values []int64
+}
+
+func NewInt64Data(data *Data) *Int64 {
+	a := &Int64{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Int64) Value(i int) int64    { return a.values[i] }
+func (a *Int64) Int64Values() []int64 { return a.values }
+
+func (a *Int64) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Int64Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of uint64 values.
+type Uint64 struct {
+	array
+	values []uint64
+}
+
+func NewUint64Data(data *Data) *Uint64 {
+	a := &Uint64{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Uint64) Value(i int) uint64     { return a.values[i] }
+func (a *Uint64) Uint64Values() []uint64 { return a.values }
+
+func (a *Uint64) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Uint64Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of float64 values.
+type Float64 struct {
+	array
+	values []float64
+}
+
+func NewFloat64Data(data *Data) *Float64 {
+	a := &Float64{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Float64) Value(i int) float64      { return a.values[i] }
+func (a *Float64) Float64Values() []float64 { return a.values }
+
+func (a *Float64) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Float64Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of int32 values.
+type Int32 struct {
+	array
+	values []int32
+}
+
+func NewInt32Data(data *Data) *Int32 {
+	a := &Int32{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Int32) Value(i int) int32    { return a.values[i] }
+func (a *Int32) Int32Values() []int32 { return a.values }
+
+func (a *Int32) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Int32Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of uint32 values.
+type Uint32 struct {
+	array
+	values []uint32
+}
+
+func NewUint32Data(data *Data) *Uint32 {
+	a := &Uint32{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Uint32) Value(i int) uint32     { return a.values[i] }
+func (a *Uint32) Uint32Values() []uint32 { return a.values }
+
+func (a *Uint32) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Uint32Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of float32 values.
+type Float32 struct {
+	array
+	values []float32
+}
+
+func NewFloat32Data(data *Data) *Float32 {
+	a := &Float32{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Float32) Value(i int) float32      { return a.values[i] }
+func (a *Float32) Float32Values() []float32 { return a.values }
+
+func (a *Float32) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Float32Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of int16 values.
+type Int16 struct {
+	array
+	values []int16
+}
+
+func NewInt16Data(data *Data) *Int16 {
+	a := &Int16{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Int16) Value(i int) int16    { return a.values[i] }
+func (a *Int16) Int16Values() []int16 { return a.values }
+
+func (a *Int16) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Int16Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of uint16 values.
+type Uint16 struct {
+	array
+	values []uint16
+}
+
+func NewUint16Data(data *Data) *Uint16 {
+	a := &Uint16{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Uint16) Value(i int) uint16     { return a.values[i] }
+func (a *Uint16) Uint16Values() []uint16 { return a.values }
+
+func (a *Uint16) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Uint16Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of int8 values.
+type Int8 struct {
+	array
+	values []int8
+}
+
+func NewInt8Data(data *Data) *Int8 {
+	a := &Int8{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Int8) Value(i int) int8   { return a.values[i] }
+func (a *Int8) Int8Values() []int8 { return a.values }
+
+func (a *Int8) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Int8Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of uint8 values.
+type Uint8 struct {
+	array
+	values []uint8
+}
+
+func NewUint8Data(data *Data) *Uint8 {
+	a := &Uint8{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Uint8) Value(i int) uint8    { return a.values[i] }
+func (a *Uint8) Uint8Values() []uint8 { return a.values }
+
+func (a *Uint8) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.Uint8Traits.CastFromBytes(vals.Bytes())
+	}
+}
+
+// A type which represents an immutable sequence of arrow.Timestamp values.
+type Timestamp struct {
+	array
+	values []arrow.Timestamp
+}
+
+func NewTimestampData(data *Data) *Timestamp {
+	a := &Timestamp{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *Timestamp) Value(i int) arrow.Timestamp        { return a.values[i] }
+func (a *Timestamp) TimestampValues() []arrow.Timestamp { return a.values }
+
+func (a *Timestamp) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.TimestampTraits.CastFromBytes(vals.Bytes())
+	}
+}
diff --git a/go/arrow/array/numeric.gen.go.tmpl b/go/arrow/array/numeric.gen.go.tmpl
new file mode 100644
index 0000000000000..5c33a36d57b97
--- /dev/null
+++ b/go/arrow/array/numeric.gen.go.tmpl
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"github.com/apache/arrow/go/arrow"
+)
+
+{{range .In}}
+
+// A type which represents an immutable sequence of {{or .QualifiedType .Type}} values.
+type {{.Name}} struct {
+	array
+	values []{{or .QualifiedType .Type}}
+}
+
+func New{{.Name}}Data(data *Data) *{{.Name}} {
+	a := &{{.Name}}{}
+	a.refCount = 1
+	a.setData(data)
+	return a
+}
+
+func (a *{{.Name}}) Value(i int)      {{or .QualifiedType .Type}} { return a.values[i] }
+func (a *{{.Name}}) {{.Name}}Values() []{{or .QualifiedType .Type}} { return a.values }
+
+func (a *{{.Name}}) setData(data *Data) {
+	a.array.setData(data)
+	vals := data.buffers[1]
+	if vals != nil {
+		a.values = arrow.{{.Name}}Traits.CastFromBytes(vals.Bytes())
+	}
+}
+{{end}}
\ No newline at end of file
diff --git a/go/arrow/array/numeric_test.go b/go/arrow/array/numeric_test.go
new file mode 100644
index 0000000000000..d5b798f3d9110
--- /dev/null
+++ b/go/arrow/array/numeric_test.go
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewFloat64Data(t *testing.T) {
+	exp := []float64{1.0, 2.0, 4.0, 8.0, 16.0}
+
+	ad := array.NewData(arrow.PrimitiveTypes.Float64, len(exp), []*memory.Buffer{nil, memory.NewBufferBytes(arrow.Float64Traits.CastToBytes(exp))}, 0)
+	fa := array.NewFloat64Data(ad)
+
+	assert.Equal(t, len(exp), fa.Len(), "unexpected Len()")
+	assert.Equal(t, exp, fa.Float64Values(), "unexpected Float64Values()")
+}
diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go
new file mode 100644
index 0000000000000..c4e3508bb1cf9
--- /dev/null
+++ b/go/arrow/array/numericbuilder.gen.go
@@ -0,0 +1,1415 @@
+// Code generated by array/numericbuilder.gen.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/internal/debug"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+type Int64Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []int64
+}
+
+func NewInt64Builder(mem memory.Allocator) *Int64Builder {
+	return &Int64Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Int64Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Int64Builder) Append(v int64) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Int64Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Int64Builder) UnsafeAppend(v int64) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Int64Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Int64Builder) AppendValues(v []int64, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Int64Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Int64Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Int64Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Int64Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Int64Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Int64Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Int64Traits.BytesRequired(n))
+		b.rawData = arrow.Int64Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewInt64Array creates a Int64 array from the memory buffers used by the builder and resets the Int64Builder
+// so it can be used to build a new array.
+func (b *Int64Builder) NewInt64Array() (a *Int64) {
+	data := b.newData()
+	a = NewInt64Data(data)
+	data.Release()
+	return
+}
+
+func (b *Int64Builder) newData() (data *Data) {
+	bytesRequired := arrow.Int64Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Int64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Uint64Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []uint64
+}
+
+func NewUint64Builder(mem memory.Allocator) *Uint64Builder {
+	return &Uint64Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Uint64Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Uint64Builder) Append(v uint64) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Uint64Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Uint64Builder) UnsafeAppend(v uint64) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Uint64Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Uint64Builder) AppendValues(v []uint64, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Uint64Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Uint64Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Uint64Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Uint64Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Uint64Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Uint64Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Uint64Traits.BytesRequired(n))
+		b.rawData = arrow.Uint64Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewUint64Array creates a Uint64 array from the memory buffers used by the builder and resets the Uint64Builder
+// so it can be used to build a new array.
+func (b *Uint64Builder) NewUint64Array() (a *Uint64) {
+	data := b.newData()
+	a = NewUint64Data(data)
+	data.Release()
+	return
+}
+
+func (b *Uint64Builder) newData() (data *Data) {
+	bytesRequired := arrow.Uint64Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Uint64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Float64Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []float64
+}
+
+func NewFloat64Builder(mem memory.Allocator) *Float64Builder {
+	return &Float64Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Float64Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Float64Builder) Append(v float64) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Float64Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Float64Builder) UnsafeAppend(v float64) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Float64Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Float64Builder) AppendValues(v []float64, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Float64Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Float64Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Float64Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Float64Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Float64Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Float64Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Float64Traits.BytesRequired(n))
+		b.rawData = arrow.Float64Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewFloat64Array creates a Float64 array from the memory buffers used by the builder and resets the Float64Builder
+// so it can be used to build a new array.
+func (b *Float64Builder) NewFloat64Array() (a *Float64) {
+	data := b.newData()
+	a = NewFloat64Data(data)
+	data.Release()
+	return
+}
+
+func (b *Float64Builder) newData() (data *Data) {
+	bytesRequired := arrow.Float64Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Float64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Int32Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []int32
+}
+
+func NewInt32Builder(mem memory.Allocator) *Int32Builder {
+	return &Int32Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Int32Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Int32Builder) Append(v int32) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Int32Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Int32Builder) UnsafeAppend(v int32) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Int32Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Int32Builder) AppendValues(v []int32, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Int32Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Int32Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Int32Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Int32Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Int32Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Int32Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Int32Traits.BytesRequired(n))
+		b.rawData = arrow.Int32Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewInt32Array creates a Int32 array from the memory buffers used by the builder and resets the Int32Builder
+// so it can be used to build a new array.
+func (b *Int32Builder) NewInt32Array() (a *Int32) {
+	data := b.newData()
+	a = NewInt32Data(data)
+	data.Release()
+	return
+}
+
+func (b *Int32Builder) newData() (data *Data) {
+	bytesRequired := arrow.Int32Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Int32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Uint32Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []uint32
+}
+
+func NewUint32Builder(mem memory.Allocator) *Uint32Builder {
+	return &Uint32Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Uint32Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Uint32Builder) Append(v uint32) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Uint32Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Uint32Builder) UnsafeAppend(v uint32) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Uint32Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Uint32Builder) AppendValues(v []uint32, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Uint32Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Uint32Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Uint32Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Uint32Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Uint32Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Uint32Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Uint32Traits.BytesRequired(n))
+		b.rawData = arrow.Uint32Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewUint32Array creates a Uint32 array from the memory buffers used by the builder and resets the Uint32Builder
+// so it can be used to build a new array.
+func (b *Uint32Builder) NewUint32Array() (a *Uint32) {
+	data := b.newData()
+	a = NewUint32Data(data)
+	data.Release()
+	return
+}
+
+func (b *Uint32Builder) newData() (data *Data) {
+	bytesRequired := arrow.Uint32Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Uint32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Float32Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []float32
+}
+
+func NewFloat32Builder(mem memory.Allocator) *Float32Builder {
+	return &Float32Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Float32Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Float32Builder) Append(v float32) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Float32Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Float32Builder) UnsafeAppend(v float32) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Float32Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Float32Builder) AppendValues(v []float32, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Float32Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Float32Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Float32Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Float32Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Float32Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Float32Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Float32Traits.BytesRequired(n))
+		b.rawData = arrow.Float32Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewFloat32Array creates a Float32 array from the memory buffers used by the builder and resets the Float32Builder
+// so it can be used to build a new array.
+func (b *Float32Builder) NewFloat32Array() (a *Float32) {
+	data := b.newData()
+	a = NewFloat32Data(data)
+	data.Release()
+	return
+}
+
+func (b *Float32Builder) newData() (data *Data) {
+	bytesRequired := arrow.Float32Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Float32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Int16Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []int16
+}
+
+func NewInt16Builder(mem memory.Allocator) *Int16Builder {
+	return &Int16Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Int16Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Int16Builder) Append(v int16) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Int16Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Int16Builder) UnsafeAppend(v int16) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Int16Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Int16Builder) AppendValues(v []int16, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Int16Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Int16Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Int16Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Int16Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Int16Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Int16Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Int16Traits.BytesRequired(n))
+		b.rawData = arrow.Int16Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewInt16Array creates a Int16 array from the memory buffers used by the builder and resets the Int16Builder
+// so it can be used to build a new array.
+func (b *Int16Builder) NewInt16Array() (a *Int16) {
+	data := b.newData()
+	a = NewInt16Data(data)
+	data.Release()
+	return
+}
+
+func (b *Int16Builder) newData() (data *Data) {
+	bytesRequired := arrow.Int16Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Int16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Uint16Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []uint16
+}
+
+func NewUint16Builder(mem memory.Allocator) *Uint16Builder {
+	return &Uint16Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Uint16Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Uint16Builder) Append(v uint16) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Uint16Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Uint16Builder) UnsafeAppend(v uint16) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Uint16Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Uint16Builder) AppendValues(v []uint16, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Uint16Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Uint16Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Uint16Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Uint16Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Uint16Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Uint16Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Uint16Traits.BytesRequired(n))
+		b.rawData = arrow.Uint16Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewUint16Array creates a Uint16 array from the memory buffers used by the builder and resets the Uint16Builder
+// so it can be used to build a new array.
+func (b *Uint16Builder) NewUint16Array() (a *Uint16) {
+	data := b.newData()
+	a = NewUint16Data(data)
+	data.Release()
+	return
+}
+
+func (b *Uint16Builder) newData() (data *Data) {
+	bytesRequired := arrow.Uint16Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Uint16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Int8Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []int8
+}
+
+func NewInt8Builder(mem memory.Allocator) *Int8Builder {
+	return &Int8Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Int8Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Int8Builder) Append(v int8) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Int8Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Int8Builder) UnsafeAppend(v int8) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Int8Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Int8Builder) AppendValues(v []int8, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Int8Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Int8Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Int8Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Int8Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Int8Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Int8Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Int8Traits.BytesRequired(n))
+		b.rawData = arrow.Int8Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewInt8Array creates a Int8 array from the memory buffers used by the builder and resets the Int8Builder
+// so it can be used to build a new array.
+func (b *Int8Builder) NewInt8Array() (a *Int8) {
+	data := b.newData()
+	a = NewInt8Data(data)
+	data.Release()
+	return
+}
+
+func (b *Int8Builder) newData() (data *Data) {
+	bytesRequired := arrow.Int8Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Int8, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type Uint8Builder struct {
+	builder
+
+	data    *memory.Buffer
+	rawData []uint8
+}
+
+func NewUint8Builder(mem memory.Allocator) *Uint8Builder {
+	return &Uint8Builder{builder: builder{refCount: 1, mem: mem}}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Uint8Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *Uint8Builder) Append(v uint8) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *Uint8Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Uint8Builder) UnsafeAppend(v uint8) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *Uint8Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Uint8Builder) AppendValues(v []uint8, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.Uint8Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Uint8Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.Uint8Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.Uint8Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Uint8Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Uint8Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.Uint8Traits.BytesRequired(n))
+		b.rawData = arrow.Uint8Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewUint8Array creates a Uint8 array from the memory buffers used by the builder and resets the Uint8Builder
+// so it can be used to build a new array.
+func (b *Uint8Builder) NewUint8Array() (a *Uint8) {
+	data := b.newData()
+	a = NewUint8Data(data)
+	data.Release()
+	return
+}
+
+func (b *Uint8Builder) newData() (data *Data) {
+	bytesRequired := arrow.Uint8Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(arrow.PrimitiveTypes.Uint8, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+
+type TimestampBuilder struct {
+	builder
+
+	typE    *arrow.TimestampType
+	data    *memory.Buffer
+	rawData []arrow.Timestamp
+}
+
+func NewTimestampBuilder(mem memory.Allocator, typE *arrow.TimestampType) *TimestampBuilder {
+	return &TimestampBuilder{builder: builder{refCount: 1, mem: mem}, typE: typE}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *TimestampBuilder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *TimestampBuilder) Append(v arrow.Timestamp) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *TimestampBuilder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *TimestampBuilder) UnsafeAppend(v arrow.Timestamp) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *TimestampBuilder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *TimestampBuilder) AppendValues(v []arrow.Timestamp, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.TimestampTraits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *TimestampBuilder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.TimestampTraits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.TimestampTraits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *TimestampBuilder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *TimestampBuilder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.TimestampTraits.BytesRequired(n))
+		b.rawData = arrow.TimestampTraits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// NewTimestampArray creates a Timestamp array from the memory buffers used by the builder and resets the TimestampBuilder
+// so it can be used to build a new array.
+func (b *TimestampBuilder) NewTimestampArray() (a *Timestamp) {
+	data := b.newData()
+	a = NewTimestampData(data)
+	data.Release()
+	return
+}
+
+func (b *TimestampBuilder) newData() (data *Data) {
+	bytesRequired := arrow.TimestampTraits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+	data = NewData(b.typE, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
diff --git a/go/arrow/array/numericbuilder.gen.go.tmpl b/go/arrow/array/numericbuilder.gen.go.tmpl
new file mode 100644
index 0000000000000..1b2aaf08a3c5a
--- /dev/null
+++ b/go/arrow/array/numericbuilder.gen.go.tmpl
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+	"github.com/apache/arrow/go/arrow"
+	"github.com/apache/arrow/go/arrow/internal/debug"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+{{range .In}}
+
+type {{.Name}}Builder struct {
+	builder
+
+{{if .Opt.Parametric -}}
+	typE    *arrow.{{.Name}}Type
+{{end -}}
+	data    *memory.Buffer
+	rawData []{{or .QualifiedType .Type}}
+}
+
+{{if .Opt.Parametric}}
+func New{{.Name}}Builder(mem memory.Allocator, typE *arrow.{{.Name}}Type) *{{.Name}}Builder {
+	return &{{.Name}}Builder{builder: builder{refCount:1, mem: mem}, typE: typE}
+}
+{{else}}
+func New{{.Name}}Builder(mem memory.Allocator) *{{.Name}}Builder {
+	return &{{.Name}}Builder{builder: builder{refCount:1, mem: mem}}
+}
+{{end}}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *{{.Name}}Builder) Release() {
+	debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+	if atomic.AddInt64(&b.refCount, -1) == 0 {
+		if b.nullBitmap != nil {
+			b.nullBitmap.Release()
+			b.nullBitmap = nil
+		}
+		if b.data != nil {
+			b.data.Release()
+			b.data = nil
+		}
+	}
+}
+
+func (b *{{.Name}}Builder) Append(v {{or .QualifiedType .Type}}) {
+	b.Reserve(1)
+	b.UnsafeAppend(v)
+}
+
+func (b *{{.Name}}Builder) AppendNull() {
+	b.Reserve(1)
+	b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *{{.Name}}Builder) UnsafeAppend(v {{or .QualifiedType .Type}}) {
+	bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	b.rawData[b.length] = v
+	b.length++
+}
+
+func (b *{{.Name}}Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+	if isValid {
+		bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+	} else {
+		b.nullN++
+	}
+	b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *{{.Name}}Builder) AppendValues(v []{{or .QualifiedType .Type}}, valid []bool) {
+	if len(v) != len(valid) && len(valid) != 0 {
+		panic("len(v) != len(valid) && len(valid) != 0")
+	}
+
+	b.Reserve(len(v))
+	if len(v) > 0 {
+		arrow.{{.Name}}Traits.Copy(b.rawData[b.length:], v)
+	}
+	b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *{{.Name}}Builder) init(capacity int) {
+	b.builder.init(capacity)
+
+	b.data = memory.NewResizableBuffer(b.mem)
+	bytesN := arrow.{{.Name}}Traits.BytesRequired(capacity)
+	b.data.Resize(bytesN)
+	b.rawData = arrow.{{.Name}}Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *{{.Name}}Builder) Reserve(n int) {
+	b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *{{.Name}}Builder) Resize(n int) {
+	if n < minBuilderCapacity {
+		n = minBuilderCapacity
+	}
+
+	if b.capacity == 0 {
+		b.init(n)
+	} else {
+		b.builder.resize(n, b.init)
+		b.data.Resize(arrow.{{.Name}}Traits.BytesRequired(n))
+		b.rawData = arrow.{{.Name}}Traits.CastFromBytes(b.data.Bytes())
+	}
+}
+
+// New{{.Name}}Array creates a {{.Name}} array from the memory buffers used by the builder and resets the {{.Name}}Builder
+// so it can be used to build a new array.
+func (b *{{.Name}}Builder) New{{.Name}}Array() (a *{{.Name}}) {
+	data := b.newData()
+	a = New{{.Name}}Data(data)
+	data.Release()
+	return
+}
+
+func (b *{{.Name}}Builder) newData() (data *Data) {
+	bytesRequired := arrow.{{.Name}}Traits.BytesRequired(b.length)
+	if bytesRequired > 0 && bytesRequired < b.data.Len() {
+		// trim buffers
+		b.data.Resize(bytesRequired)
+	}
+{{if .Opt.Parametric -}}
+	data = NewData(b.typE, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+{{else -}}
+	data = NewData(arrow.PrimitiveTypes.{{.Name}}, b.length, []*memory.Buffer{b.nullBitmap, b.data}, b.nullN)
+{{end -}}
+	b.reset()
+
+	if b.data != nil {
+		b.data.Release()
+		b.data = nil
+		b.rawData = nil
+	}
+
+	return
+}
+{{end}}
\ No newline at end of file
diff --git a/go/arrow/array/numericbuilder_test.go b/go/arrow/array/numericbuilder_test.go
new file mode 100644
index 0000000000000..c0cd7adc2a3d8
--- /dev/null
+++ b/go/arrow/array/numericbuilder_test.go
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewFloat64Builder(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	ab := array.NewFloat64Builder(mem)
+
+	ab.Append(1)
+	ab.Append(2)
+	ab.Append(3)
+	ab.AppendNull()
+	ab.Append(5)
+	ab.Append(6)
+	ab.AppendNull()
+	ab.Append(8)
+	ab.Append(9)
+	ab.Append(10)
+
+	// check state of builder before NewFloat64Array
+	assert.Equal(t, 10, ab.Len(), "unexpected Len()")
+	assert.Equal(t, 2, ab.NullN(), "unexpected NullN()")
+
+	a := ab.NewFloat64Array()
+
+	// check state of builder after NewFloat64Array
+	assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewFloat64Array did not reset state")
+	assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewFloat64Array did not reset state")
+	assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewFloat64Array did not reset state")
+
+	// check state of array
+	assert.Equal(t, 2, a.NullN(), "unexpected null count")
+	assert.Equal(t, []float64{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.Float64Values(), "unexpected Float64Values")
+	assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity
+	assert.Len(t, a.Float64Values(), 10, "unexpected length of Float64Values")
+
+	a.Release()
+
+	ab.Append(7)
+	ab.Append(8)
+
+	a = ab.NewFloat64Array()
+
+	assert.Equal(t, 0, a.NullN())
+	assert.Equal(t, []float64{7, 8}, a.Float64Values())
+	assert.Len(t, a.Float64Values(), 2)
+
+	a.Release()
+}
+
+func TestFloat32Builder_AppendValues(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	ab := array.NewFloat64Builder(mem)
+
+	exp := []float64{1.0, 1.1, 1.2, 1.3}
+	ab.AppendValues(exp, nil)
+	a := ab.NewFloat64Array()
+	assert.Equal(t, exp, a.Float64Values())
+
+	a.Release()
+	ab.Release()
+}
+
+func TestFloat32Builder_Empty(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	ab := array.NewFloat64Builder(mem)
+
+	exp := []float64{1.0, 1.1, 1.2, 1.3}
+	ab.AppendValues(exp, nil)
+	a := ab.NewFloat64Array()
+	assert.Equal(t, exp, a.Float64Values())
+	a.Release()
+
+	a = ab.NewFloat64Array()
+	assert.Zero(t, a.Len())
+	a.Release()
+
+	ab.Release()
+}
diff --git a/go/arrow/array/util.go b/go/arrow/array/util.go
new file mode 100644
index 0000000000000..c8d7e17c2bd8b
--- /dev/null
+++ b/go/arrow/array/util.go
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
diff --git a/go/arrow/datatype.go b/go/arrow/datatype.go
new file mode 100644
index 0000000000000..3329e50859f1f
--- /dev/null
+++ b/go/arrow/datatype.go
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow
+
+// Type is a logical type. They can be expressed as
+// either a primitive physical type (bytes or bits of some fixed size), a
+// nested type consisting of other data types, or another data type (e.g. a
+// timestamp encoded as an int64)
+type Type int
+
+const (
+	// NULL type having no physical storage
+	NULL Type = iota
+
+	// BOOL is a 1 bit, LSB bit-packed ordering
+	BOOL
+
+	// UINT8 is an Unsigned 8-bit little-endian integer
+	UINT8
+
+	// INT8 is a Signed 8-bit little-endian integer
+	INT8
+
+	// UINT16 is an Unsigned 16-bit little-endian integer
+	UINT16
+
+	// INT16 is a Signed 16-bit little-endian integer
+	INT16
+
+	// UINT32 is an Unsigned 32-bit little-endian integer
+	UINT32
+
+	// INT32 is a Signed 32-bit little-endian integer
+	INT32
+
+	// UINT64 is an Unsigned 64-bit little-endian integer
+	UINT64
+
+	// INT64 is a Signed 64-bit little-endian integer
+	INT64
+
+	// HALF_FLOAT is a 2-byte floating point value
+	HALF_FLOAT
+
+	// FLOAT32 is a 4-byte floating point value
+	FLOAT32
+
+	// FLOAT64 is an 8-byte floating point value
+	FLOAT64
+
+	// STRING is a UTF8 variable-length string
+	STRING
+
+	// BINARY is a Variable-length byte type (no guarantee of UTF8-ness)
+	BINARY
+
+	// FIXED_SIZE_BINARY is a binary where each value occupies the same number of bytes
+	FIXED_SIZE_BINARY
+
+	// DATE32 is int32 days since the UNIX epoch
+	DATE32
+
+	// DATE64 is int64 milliseconds since the UNIX epoch
+	DATE64
+
+	// TIMESTAMP is an exact timestamp encoded with int64 since UNIX epoch
+	// Default unit millisecond
+	TIMESTAMP
+
+	// TIME32 is a signed 32-bit integer, representing either seconds or
+	// milliseconds since midnight
+	TIME32
+
+	// TIME64 is a signed 64-bit integer, representing either microseconds or
+	// nanoseconds since midnight
+	TIME64
+
+	// INTERVAL is YEAR_MONTH or DAY_TIME interval in SQL style
+	INTERVAL
+
+	// DECIMAL is a precision- and scale-based decimal type. Storage type depends on the
+	// parameters.
+	DECIMAL
+
+	// LIST is a list of some logical data type
+	LIST
+
+	// STRUCT of logical types
+	STRUCT
+
+	// UNION of logical types
+	UNION
+
+	// DICTIONARY aka Category type
+	DICTIONARY
+
+	// MAP is a repeated struct logical type
+	MAP
+)
+
+// DataType is the representation of an Arrow type.
+type DataType interface {
+	ID() Type
+	// Name is name of the data type.
+	Name() string
+}
+
+// FixedWidthDataType is the representation of an Arrow type that
+// requires a fixed number of bits in memory for each element.
+type FixedWidthDataType interface {
+	DataType
+	// BitWidth returns the number of bits required to store a single element of this data type in memory.
+	BitWidth() int
+}
+
+type BinaryDataType interface {
+	DataType
+	binary()
+}
diff --git a/go/arrow/datatype_binary.go b/go/arrow/datatype_binary.go
new file mode 100644
index 0000000000000..40ab860def53f
--- /dev/null
+++ b/go/arrow/datatype_binary.go
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow
+
+type BinaryType struct{}
+
+func (t *BinaryType) ID() Type     { return BINARY }
+func (t *BinaryType) Name() string { return "binary" }
+func (t *BinaryType) binary()      {}
+
+type StringType struct{}
+
+func (t *StringType) ID() Type     { return STRING }
+func (t *StringType) Name() string { return "utf8" }
+func (t *StringType) binary()      {}
+
+var (
+	BinaryTypes = struct {
+		Binary BinaryDataType
+		String BinaryDataType
+	}{
+		Binary: &BinaryType{},
+		String: &StringType{},
+	}
+)
diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go
new file mode 100644
index 0000000000000..de96ccba4f44d
--- /dev/null
+++ b/go/arrow/datatype_fixedwidth.go
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow
+
+type BooleanType struct{}
+
+func (t *BooleanType) ID() Type     { return BOOL }
+func (t *BooleanType) Name() string { return "bool" }
+
+// BitWidth returns the number of bits required to store a single element of this data type in memory.
+func (t *BooleanType) BitWidth() int { return 1 }
+
+type (
+	Timestamp int64
+	TimeUnit  int
+)
+
+const (
+	Nanosecond TimeUnit = iota
+	Microsecond
+	Millisecond
+	Second
+)
+
+func (u TimeUnit) String() string { return [...]string{"ns", "µs", "ms", "s"}[uint(u)&3] }
+
+// TimestampType is encoded as a 64-bit signed integer since the UNIX epoch (2017-01-01T00:00:00Z).
+// The zero-value is a nanosecond and time zone neutral. Time zone neutral can be
+// considered UTC without having "UTC" as a time zone.
+type TimestampType struct {
+	Unit     TimeUnit
+	TimeZone string
+}
+
+func (*TimestampType) ID() Type     { return TIMESTAMP }
+func (*TimestampType) Name() string { return "timestamp" }
+
+// BitWidth returns the number of bits required to store a single element of this data type in memory.
+func (*TimestampType) BitWidth() int { return 64 }
+
+var (
+	FixedWidthTypes = struct {
+		Boolean FixedWidthDataType
+	}{
+		Boolean: &BooleanType{},
+	}
+)
diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go
new file mode 100644
index 0000000000000..44fbdd1357f2b
--- /dev/null
+++ b/go/arrow/datatype_fixedwidth_test.go
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestTimeUnit_String verifies each time unit matches its string representation.
+func TestTimeUnit_String(t *testing.T) {
+	tests := []struct {
+		u   arrow.TimeUnit
+		exp string
+	}{
+		{arrow.Nanosecond, "ns"},
+		{arrow.Microsecond, "µs"},
+		{arrow.Millisecond, "ms"},
+		{arrow.Second, "s"},
+	}
+	for _, test := range tests {
+		t.Run(test.exp, func(t *testing.T) {
+			assert.Equal(t, test.exp, test.u.String())
+		})
+	}
+}
diff --git a/go/arrow/datatype_numeric.gen.go b/go/arrow/datatype_numeric.gen.go
new file mode 100644
index 0000000000000..30537be749bd0
--- /dev/null
+++ b/go/arrow/datatype_numeric.gen.go
@@ -0,0 +1,96 @@
+// Code generated by datatype_numeric.gen.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow
+
+type Int8Type struct{}
+
+func (t *Int8Type) ID() Type     { return INT8 }
+func (t *Int8Type) Name() string { return "int8" }
+
+type Int16Type struct{}
+
+func (t *Int16Type) ID() Type     { return INT16 }
+func (t *Int16Type) Name() string { return "int16" }
+
+type Int32Type struct{}
+
+func (t *Int32Type) ID() Type     { return INT32 }
+func (t *Int32Type) Name() string { return "int32" }
+
+type Int64Type struct{}
+
+func (t *Int64Type) ID() Type     { return INT64 }
+func (t *Int64Type) Name() string { return "int64" }
+
+type Uint8Type struct{}
+
+func (t *Uint8Type) ID() Type     { return UINT8 }
+func (t *Uint8Type) Name() string { return "uint8" }
+
+type Uint16Type struct{}
+
+func (t *Uint16Type) ID() Type     { return UINT16 }
+func (t *Uint16Type) Name() string { return "uint16" }
+
+type Uint32Type struct{}
+
+func (t *Uint32Type) ID() Type     { return UINT32 }
+func (t *Uint32Type) Name() string { return "uint32" }
+
+type Uint64Type struct{}
+
+func (t *Uint64Type) ID() Type     { return UINT64 }
+func (t *Uint64Type) Name() string { return "uint64" }
+
+type Float32Type struct{}
+
+func (t *Float32Type) ID() Type     { return FLOAT32 }
+func (t *Float32Type) Name() string { return "float32" }
+
+type Float64Type struct{}
+
+func (t *Float64Type) ID() Type     { return FLOAT64 }
+func (t *Float64Type) Name() string { return "float64" }
+
+var (
+	PrimitiveTypes = struct {
+		Int8    DataType
+		Int16   DataType
+		Int32   DataType
+		Int64   DataType
+		Uint8   DataType
+		Uint16  DataType
+		Uint32  DataType
+		Uint64  DataType
+		Float32 DataType
+		Float64 DataType
+	}{
+
+		Int8:    &Int8Type{},
+		Int16:   &Int16Type{},
+		Int32:   &Int32Type{},
+		Int64:   &Int64Type{},
+		Uint8:   &Uint8Type{},
+		Uint16:  &Uint16Type{},
+		Uint32:  &Uint32Type{},
+		Uint64:  &Uint64Type{},
+		Float32: &Float32Type{},
+		Float64: &Float64Type{},
+	}
+)
diff --git a/go/arrow/datatype_numeric.gen.go.tmpl b/go/arrow/datatype_numeric.gen.go.tmpl
new file mode 100644
index 0000000000000..582c895256392
--- /dev/null
+++ b/go/arrow/datatype_numeric.gen.go.tmpl
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow
+
+{{range .In}}
+type {{.Name}}Type struct {}
+
+func (t *{{.Name}}Type) ID() Type { return {{.Name|upper}} }
+func (t *{{.Name}}Type) Name() string { return "{{.Name|lower}}" }
+
+
+{{end}}
+
+var (
+        PrimitiveTypes = struct {
+{{range .In}}
+                {{.Name}} DataType
+{{- end}}
+        }{
+{{range .In}}
+                {{.Name}}: &{{.Name}}Type{},
+{{- end}}
+        }
+)
\ No newline at end of file
diff --git a/go/arrow/datatype_numeric.gen.go.tmpldata b/go/arrow/datatype_numeric.gen.go.tmpldata
new file mode 100644
index 0000000000000..1f40fd502cdc0
--- /dev/null
+++ b/go/arrow/datatype_numeric.gen.go.tmpldata
@@ -0,0 +1,32 @@
+[
+  {
+    "Name": "Int8"
+  },
+  {
+    "Name": "Int16"
+  },
+  {
+    "Name": "Int32"
+  },
+  {
+    "Name": "Int64"
+  },
+  {
+    "Name": "Uint8"
+  },
+  {
+    "Name": "Uint16"
+  },
+  {
+    "Name": "Uint32"
+  },
+  {
+    "Name": "Uint64"
+  },
+  {
+    "Name": "Float32"
+  },
+  {
+    "Name": "Float64"
+  }
+]
\ No newline at end of file
diff --git a/go/arrow/doc.go b/go/arrow/doc.go
new file mode 100644
index 0000000000000..6ee0c2a4301d1
--- /dev/null
+++ b/go/arrow/doc.go
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+Package arrow provides an implementation of Apache Arrow.
+
+Apache Arrow is a cross-language development platform for in-memory data. It specifies a standardized
+language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic
+operations on modern hardware. It also provides computational libraries and zero-copy streaming
+messaging and inter-process communication.
+
+Basics
+
+The fundamental data structure in Arrow is an Array, which holds a sequence of values of the same type. An array
+consists of memory holding the data and an additional validity bitmap that indicates if the corresponding entry in the
+array is valid (not null). If the array has no null entries, it is possible to omit this bitmap.
+
+*/
+package arrow
+
+//go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
+//go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl
+
+// stringer
+//go:generate stringer -type=Type
diff --git a/go/arrow/example_test.go b/go/arrow/example_test.go
new file mode 100644
index 0000000000000..96fd3e55fd298
--- /dev/null
+++ b/go/arrow/example_test.go
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow_test
+
+import (
+	"fmt"
+
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/memory"
+)
+
+// This example demonstrates how to build an array of int64 values using a builder and Append.
+// Whilst convenient for small arrays,
+func Example_minimal() {
+	// Create an allocator.
+	pool := memory.NewGoAllocator()
+
+	// Create an int64 array builder.
+	builder := array.NewInt64Builder(pool)
+
+	builder.Append(1)
+	builder.Append(2)
+	builder.Append(3)
+	builder.AppendNull()
+	builder.Append(5)
+	builder.Append(6)
+	builder.Append(7)
+	builder.Append(8)
+
+	// Finish building the int64 array and reset the builder.
+	ints := builder.NewInt64Array()
+
+	// Enumerate the values.
+	for i, v := range ints.Int64Values() {
+		fmt.Printf("ints[%d] = ", i)
+		if ints.IsNull(i) {
+			fmt.Println("(null)")
+		} else {
+			fmt.Println(v)
+		}
+	}
+
+	// Output:
+	// ints[0] = 1
+	// ints[1] = 2
+	// ints[2] = 3
+	// ints[3] = (null)
+	// ints[4] = 5
+	// ints[5] = 6
+	// ints[6] = 7
+	// ints[7] = 8
+}
+
+// This example demonstrates creating an array, sourcing the values and
+// null bitmaps directly from byte slices. The null count is set to
+// UnknownNullCount, instructing the array to calculate the
+// null count from the bitmap when NullN is called.
+func Example_fromMemory() {
+	// create LSB packed bits with the following pattern:
+	// 01010011 11000101
+	data := memory.NewBufferBytes([]byte{0xca, 0xa3})
+
+	// create LSB packed validity (null) bitmap, where every 4th element is null:
+	// 11101110 11101110
+	nullBitmap := memory.NewBufferBytes([]byte{0x77, 0x77})
+
+	// Create a boolean array and lazily determine NullN using UnknownNullCount
+	bools := array.NewBoolean(16, data, nullBitmap, array.UnknownNullCount)
+
+	// Show the null count
+	fmt.Printf("NullN()  = %d\n", bools.NullN())
+
+	// Enumerate the values.
+	n := bools.Len()
+	for i := 0; i < n; i++ {
+		fmt.Printf("bools[%d] = ", i)
+		if bools.IsNull(i) {
+			fmt.Println("(null)")
+		} else {
+			fmt.Printf("%t\n", bools.Value(i))
+		}
+	}
+
+	// Output:
+	// NullN()  = 4
+	// bools[0] = false
+	// bools[1] = true
+	// bools[2] = false
+	// bools[3] = (null)
+	// bools[4] = false
+	// bools[5] = false
+	// bools[6] = true
+	// bools[7] = (null)
+	// bools[8] = true
+	// bools[9] = true
+	// bools[10] = false
+	// bools[11] = (null)
+	// bools[12] = false
+	// bools[13] = true
+	// bools[14] = false
+	// bools[15] = (null)
+}
diff --git a/go/arrow/internal/bitutil/bitutil.go b/go/arrow/internal/bitutil/bitutil.go
new file mode 100644
index 0000000000000..00f1693362052
--- /dev/null
+++ b/go/arrow/internal/bitutil/bitutil.go
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bitutil
+
+import (
+	"math/bits"
+	"reflect"
+	"unsafe"
+)
+
+var (
+	BitMask        = [8]byte{1, 2, 4, 8, 16, 32, 64, 128}
+	FlippedBitMask = [8]byte{254, 253, 251, 247, 239, 223, 191, 127}
+)
+
+// NextPowerOf2 rounds x to the next power of two.
+func NextPowerOf2(x int) int { return 1 << uint(bits.Len(uint(x))) }
+
+// CeilByte rounds size to the next multiple of 8.
+func CeilByte(size int) int { return (size + 7) &^ 7 }
+
+// BitIsSet returns true if the bit at index i in buf is set (1).
+func BitIsSet(buf []byte, i int) bool { return (buf[uint(i)/8] & BitMask[byte(i)%8]) != 0 }
+
+// BitIsNotSet returns true if the bit at index i in buf is not set (0).
+func BitIsNotSet(buf []byte, i int) bool { return (buf[uint(i)/8] & BitMask[byte(i)%8]) == 0 }
+
+// SetBit sets the bit at index i in buf to 1.
+func SetBit(buf []byte, i int) { buf[uint(i)/8] |= BitMask[byte(i)%8] }
+
+// ClearBit sets the bit at index i in buf to 0.
+func ClearBit(buf []byte, i int) { buf[uint(i)/8] &= FlippedBitMask[byte(i)%8] }
+
+// SetBitTo sets the bit at index i in buf to val.
+func SetBitTo(buf []byte, i int, val bool) {
+	if val {
+		SetBit(buf, i)
+	} else {
+		ClearBit(buf, i)
+	}
+}
+
+// CountSetBits counts the number of 1's in buf up to n bits.
+func CountSetBits(buf []byte, n int) int {
+	count := 0
+
+	uint64Bytes := n / uint64SizeBits * 8
+	for _, v := range bytesToUint64(buf[:uint64Bytes]) {
+		count += bits.OnesCount64(v)
+	}
+
+	for _, v := range buf[uint64Bytes : n/8] {
+		count += bits.OnesCount8(v)
+	}
+
+	// tail bits
+	for i := n &^ 0x7; i < n; i++ {
+		if BitIsSet(buf, i) {
+			count++
+		}
+	}
+
+	return count
+}
+
+const (
+	uint64SizeBytes = int(unsafe.Sizeof(uint64(0)))
+	uint64SizeBits  = uint64SizeBytes * 8
+)
+
+func bytesToUint64(b []byte) []uint64 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []uint64
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / uint64SizeBytes
+	s.Cap = h.Cap / uint64SizeBytes
+
+	return res
+}
diff --git a/go/arrow/internal/bitutil/bitutil_test.go b/go/arrow/internal/bitutil/bitutil_test.go
new file mode 100644
index 0000000000000..b1890ec4f2a50
--- /dev/null
+++ b/go/arrow/internal/bitutil/bitutil_test.go
@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bitutil_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+	"github.com/apache/arrow/go/arrow/internal/testing/tools"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCeilByte(t *testing.T) {
+	tests := []struct {
+		name    string
+		in, exp int
+	}{
+		{"zero", 0, 0},
+		{"five", 5, 8},
+		{"sixteen", 16, 16},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := bitutil.CeilByte(test.in)
+			assert.Equal(t, test.exp, got)
+		})
+	}
+}
+
+func TestBitIsSet(t *testing.T) {
+	buf := make([]byte, 2)
+	buf[0] = 0xa1
+	buf[1] = 0xc2
+	exp := []bool{true, false, false, false, false, true, false, true, false, true, false, false, false, false, true, true}
+	var got []bool
+	for i := 0; i < 0x10; i++ {
+		got = append(got, bitutil.BitIsSet(buf, i))
+	}
+	assert.Equal(t, exp, got)
+}
+
+func TestBitIsNotSet(t *testing.T) {
+	buf := make([]byte, 2)
+	buf[0] = 0xa1
+	buf[1] = 0xc2
+	exp := []bool{false, true, true, true, true, false, true, false, true, false, true, true, true, true, false, false}
+	var got []bool
+	for i := 0; i < 0x10; i++ {
+		got = append(got, bitutil.BitIsNotSet(buf, i))
+	}
+	assert.Equal(t, exp, got)
+}
+
+func TestClearBit(t *testing.T) {
+	buf := make([]byte, 2)
+	buf[0] = 0xff
+	buf[1] = 0xff
+	for i, v := range []bool{false, true, true, true, true, false, true, false, true, false, true, true, true, true, false, false} {
+		if v {
+			bitutil.ClearBit(buf, i)
+		}
+	}
+	assert.Equal(t, []byte{0xa1, 0xc2}, buf)
+}
+
+func TestSetBit(t *testing.T) {
+	buf := make([]byte, 2)
+	for i, v := range []bool{true, false, false, false, false, true, false, true, false, true, false, false, false, false, true, true} {
+		if v {
+			bitutil.SetBit(buf, i)
+		}
+	}
+	assert.Equal(t, []byte{0xa1, 0xc2}, buf)
+}
+
+func TestSetBitTo(t *testing.T) {
+	buf := make([]byte, 2)
+	for i, v := range []bool{true, false, false, false, false, true, false, true, false, true, false, false, false, false, true, true} {
+		bitutil.SetBitTo(buf, i, v)
+	}
+	assert.Equal(t, []byte{0xa1, 0xc2}, buf)
+}
+
+func TestCountSetBits(t *testing.T) {
+	tests := []struct {
+		name string
+		buf  []byte
+		n    int
+		exp  int
+	}{
+		{"some 03 bits", bbits(0x11000000), 3, 2},
+		{"some 11 bits", bbits(0x11000011, 0x01000000), 11, 5},
+		{"some 72 bits", bbits(0x11001010, 0x11110000, 0x00001111, 0x11000011, 0x11001010, 0x11110000, 0x00001111, 0x11000011, 0x10001001), 9 * 8, 35},
+		{"all  03 bits", bbits(0x11100001), 3, 3},
+		{"all  11 bits", bbits(0x11111111, 0x11111111), 11, 11},
+		{"all  72 bits", bbits(0x11111111, 0x11111111, 0x11111111, 0x11111111, 0x11111111, 0x11111111, 0x11111111, 0x11111111, 0x11111111), 9 * 8, 72},
+		{"none 03 bits", bbits(0x00000001), 3, 0},
+		{"none 11 bits", bbits(0x00000000, 0x00000000), 11, 0},
+		{"none 72 bits", bbits(0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000), 9 * 8, 0},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := bitutil.CountSetBits(test.buf, test.n)
+			assert.Equal(t, test.exp, got)
+		})
+	}
+}
+
+func bbits(v ...int32) []byte {
+	return tools.IntsToBitsLSB(v...)
+}
+
+func BenchmarkBitIsSet(b *testing.B) {
+	buf := make([]byte, 32)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bitutil.BitIsSet(buf, (i%32)&0x1a)
+	}
+}
+
+func BenchmarkSetBit(b *testing.B) {
+	buf := make([]byte, 32)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bitutil.SetBit(buf, (i%32)&0x1a)
+	}
+}
+
+func BenchmarkSetBitTo(b *testing.B) {
+	vals := []bool{true, false, false, false, false, true, false, true, false, true, false, false, false, false, true, true}
+	buf := make([]byte, 32)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bitutil.SetBitTo(buf, i%32, vals[i%len(vals)])
+	}
+}
+
+var (
+	intval int
+)
+
+func benchmarkCountSetBitsN(b *testing.B, n int) {
+	nn := n/8 + 1
+	buf := make([]byte, nn)
+	//src := [4]byte{0x1f, 0xaa, 0xba, 0x11}
+	src := [4]byte{0x01, 0x01, 0x01, 0x01}
+	for i := 0; i < nn; i++ {
+		buf[i] = src[i&0x3]
+	}
+	b.ResetTimer()
+	var res int
+	for i := 0; i < b.N; i++ {
+		res = bitutil.CountSetBits(buf, n)
+	}
+	intval = res
+}
+
+func BenchmarkCountSetBits_3(b *testing.B) {
+	benchmarkCountSetBitsN(b, 3)
+}
+
+func BenchmarkCountSetBits_32(b *testing.B) {
+	benchmarkCountSetBitsN(b, 32)
+}
+
+func BenchmarkCountSetBits_128(b *testing.B) {
+	benchmarkCountSetBitsN(b, 128)
+}
+
+func BenchmarkCountSetBits_1000(b *testing.B) {
+	benchmarkCountSetBitsN(b, 1000)
+}
+
+func BenchmarkCountSetBits_1024(b *testing.B) {
+	benchmarkCountSetBitsN(b, 1024)
+}
diff --git a/go/arrow/internal/cpu/README.md b/go/arrow/internal/cpu/README.md
new file mode 100644
index 0000000000000..2619b38c11ec0
--- /dev/null
+++ b/go/arrow/internal/cpu/README.md
@@ -0,0 +1,42 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Package cpu
+
+Copied from Go src/internal/cpu
+
+## Extras
+
+### Intel
+
+The `INTEL_DISABLE_EXT` environment variable can control which CPU extensions are available for
+the running process. It should be a comma-separate list of upper-case strings as follows
+
+|   Flag   | Description |
+| -------- | ----------- |
+| `ALL`    | Disable all CPU extensions and fall back to Go implementation |
+| `AVX2`   | Disable AVX2 optimizations |  
+| `AVX`    | Disable AVX optimizations |  
+| `SSE`    | Disable all SSE optimizations |  
+| `SSE4`   | Disable SSE42, SSE41 optimizations |  
+| `SSSE3`  | Disable supplemental SSE3 optimizations |  
+| `SSE3`   | Disable SSE3 optimizations |  
+| `SSE2`   | Disable SSE2 optimizations |
+
+Any unrecognized flags will be ignored and therefore it is possible to leave the environment variable with a bogus value such as `NONE` when experimenting.
\ No newline at end of file
diff --git a/go/arrow/internal/cpu/cpu.go b/go/arrow/internal/cpu/cpu.go
new file mode 100644
index 0000000000000..22fc561002d86
--- /dev/null
+++ b/go/arrow/internal/cpu/cpu.go
@@ -0,0 +1,77 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cpu implements processor feature detection
+// used by the Go standard library.
+package cpu
+
+var X86 x86
+
+// The booleans in x86 contain the correspondingly named cpuid feature bit.
+// HasAVX and HasAVX2 are only set if the OS does support XMM and YMM registers
+// in addition to the cpuid feature bit being set.
+// The struct is padded to avoid false sharing.
+type x86 struct {
+	_            [CacheLineSize]byte
+	HasAES       bool
+	HasADX       bool
+	HasAVX       bool
+	HasAVX2      bool
+	HasBMI1      bool
+	HasBMI2      bool
+	HasERMS      bool
+	HasFMA       bool
+	HasOSXSAVE   bool
+	HasPCLMULQDQ bool
+	HasPOPCNT    bool
+	HasSSE2      bool
+	HasSSE3      bool
+	HasSSSE3     bool
+	HasSSE41     bool
+	HasSSE42     bool
+	_            [CacheLineSize]byte
+}
+
+var PPC64 ppc64
+
+// For ppc64x, it is safe to check only for ISA level starting on ISA v3.00,
+// since there are no optional categories. There are some exceptions that also
+// require kernel support to work (darn, scv), so there are capability bits for
+// those as well. The minimum processor requirement is POWER8 (ISA 2.07), so we
+// maintain some of the old capability checks for optional categories for
+// safety.
+// The struct is padded to avoid false sharing.
+type ppc64 struct {
+	_          [CacheLineSize]byte
+	HasVMX     bool // Vector unit (Altivec)
+	HasDFP     bool // Decimal Floating Point unit
+	HasVSX     bool // Vector-scalar unit
+	HasHTM     bool // Hardware Transactional Memory
+	HasISEL    bool // Integer select
+	HasVCRYPTO bool // Vector cryptography
+	HasHTMNOSC bool // HTM: kernel-aborted transaction in syscalls
+	HasDARN    bool // Hardware random number generator (requires kernel enablement)
+	HasSCV     bool // Syscall vectored (requires kernel enablement)
+	IsPOWER8   bool // ISA v2.07 (POWER8)
+	IsPOWER9   bool // ISA v3.00 (POWER9)
+	_          [CacheLineSize]byte
+}
+
+var ARM64 arm64
+
+// The booleans in arm64 contain the correspondingly named cpu feature bit.
+// The struct is padded to avoid false sharing.
+type arm64 struct {
+	_          [CacheLineSize]byte
+	HasFP      bool
+	HasASIMD   bool
+	HasEVTSTRM bool
+	HasAES     bool
+	HasPMULL   bool
+	HasSHA1    bool
+	HasSHA2    bool
+	HasCRC32   bool
+	HasATOMICS bool
+	_          [CacheLineSize]byte
+}
diff --git a/go/arrow/internal/cpu/cpu_test.go b/go/arrow/internal/cpu/cpu_test.go
new file mode 100644
index 0000000000000..02f09d3d54cef
--- /dev/null
+++ b/go/arrow/internal/cpu/cpu_test.go
@@ -0,0 +1,51 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu_test
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/internal/cpu"
+)
+
+func TestAMD64minimalFeatures(t *testing.T) {
+	if runtime.GOARCH == "amd64" {
+		if !cpu.X86.HasSSE2 {
+			t.Fatalf("HasSSE2 expected true, got false")
+		}
+	}
+}
+
+func TestAVX2hasAVX(t *testing.T) {
+	if runtime.GOARCH == "amd64" {
+		if cpu.X86.HasAVX2 && !cpu.X86.HasAVX {
+			t.Fatalf("HasAVX expected true, got false")
+		}
+	}
+}
+
+func TestPPC64minimalFeatures(t *testing.T) {
+	if runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" {
+		if !cpu.PPC64.IsPOWER8 {
+			t.Fatalf("IsPOWER8 expected true, got false")
+		}
+		if !cpu.PPC64.HasVMX {
+			t.Fatalf("HasVMX expected true, got false")
+		}
+		if !cpu.PPC64.HasDFP {
+			t.Fatalf("HasDFP expected true, got false")
+		}
+		if !cpu.PPC64.HasVSX {
+			t.Fatalf("HasVSX expected true, got false")
+		}
+		if !cpu.PPC64.HasISEL {
+			t.Fatalf("HasISEL expected true, got false")
+		}
+		if !cpu.PPC64.HasVCRYPTO {
+			t.Fatalf("HasVCRYPTO expected true, got false")
+		}
+	}
+}
diff --git a/go/arrow/internal/cpu/cpu_x86.go b/go/arrow/internal/cpu/cpu_x86.go
new file mode 100644
index 0000000000000..777269f1a2523
--- /dev/null
+++ b/go/arrow/internal/cpu/cpu_x86.go
@@ -0,0 +1,107 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build 386 amd64 amd64p32
+
+package cpu
+
+import (
+	"os"
+	"strings"
+)
+
+const CacheLineSize = 64
+
+// cpuid is implemented in cpu_x86.s.
+func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
+
+// xgetbv with ecx = 0 is implemented in cpu_x86.s.
+func xgetbv() (eax, edx uint32)
+
+func init() {
+	maxID, _, _, _ := cpuid(0, 0)
+
+	if maxID < 1 {
+		return
+	}
+
+	_, _, ecx1, edx1 := cpuid(1, 0)
+	X86.HasSSE2 = isSet(26, edx1)
+
+	X86.HasSSE3 = isSet(0, ecx1)
+	X86.HasPCLMULQDQ = isSet(1, ecx1)
+	X86.HasSSSE3 = isSet(9, ecx1)
+	X86.HasFMA = isSet(12, ecx1)
+	X86.HasSSE41 = isSet(19, ecx1)
+	X86.HasSSE42 = isSet(20, ecx1)
+	X86.HasPOPCNT = isSet(23, ecx1)
+	X86.HasAES = isSet(25, ecx1)
+	X86.HasOSXSAVE = isSet(27, ecx1)
+
+	osSupportsAVX := false
+	// For XGETBV, OSXSAVE bit is required and sufficient.
+	if X86.HasOSXSAVE {
+		eax, _ := xgetbv()
+		// Check if XMM and YMM registers have OS support.
+		osSupportsAVX = isSet(1, eax) && isSet(2, eax)
+	}
+
+	X86.HasAVX = isSet(28, ecx1) && osSupportsAVX
+
+	if maxID < 7 {
+		return
+	}
+
+	_, ebx7, _, _ := cpuid(7, 0)
+	X86.HasBMI1 = isSet(3, ebx7)
+	X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
+	X86.HasBMI2 = isSet(8, ebx7)
+	X86.HasERMS = isSet(9, ebx7)
+	X86.HasADX = isSet(19, ebx7)
+
+	// NOTE(sgc): added ability to disable extension via environment
+	checkEnvironment()
+}
+func checkEnvironment() {
+	if ext, ok := os.LookupEnv("INTEL_DISABLE_EXT"); ok {
+		exts := strings.Split(ext, ",")
+
+		for _, x := range exts {
+			switch x {
+			case "ALL":
+				X86.HasAVX2 = false
+				X86.HasAVX = false
+				X86.HasSSE42 = false
+				X86.HasSSE41 = false
+				X86.HasSSSE3 = false
+				X86.HasSSE3 = false
+				X86.HasSSE2 = false
+
+			case "AVX2":
+				X86.HasAVX2 = false
+			case "AVX":
+				X86.HasAVX = false
+			case "SSE":
+				X86.HasSSE42 = false
+				X86.HasSSE41 = false
+				X86.HasSSSE3 = false
+				X86.HasSSE3 = false
+				X86.HasSSE2 = false
+			case "SSE4":
+				X86.HasSSE42 = false
+				X86.HasSSE41 = false
+			case "SSSE3":
+				X86.HasSSSE3 = false
+			case "SSE3":
+				X86.HasSSE3 = false
+			case "SSE2":
+				X86.HasSSE2 = false
+			}
+		}
+	}
+}
+
+func isSet(bitpos uint, value uint32) bool {
+	return value&(1<<bitpos) != 0
+}
diff --git a/go/arrow/internal/cpu/cpu_x86.s b/go/arrow/internal/cpu/cpu_x86.s
new file mode 100644
index 0000000000000..228fbcf6d6601
--- /dev/null
+++ b/go/arrow/internal/cpu/cpu_x86.s
@@ -0,0 +1,32 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build 386 amd64 amd64p32
+
+#include "textflag.h"
+
+// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
+TEXT ·cpuid(SB), NOSPLIT, $0-24
+	MOVL eaxArg+0(FP), AX
+	MOVL ecxArg+4(FP), CX
+	CPUID
+	MOVL AX, eax+8(FP)
+	MOVL BX, ebx+12(FP)
+	MOVL CX, ecx+16(FP)
+	MOVL DX, edx+20(FP)
+	RET
+
+// func xgetbv() (eax, edx uint32)
+TEXT ·xgetbv(SB),NOSPLIT,$0-8
+#ifdef GOOS_nacl
+	// nacl does not support XGETBV.
+	MOVL $0, eax+0(FP)
+	MOVL $0, edx+4(FP)
+#else
+	MOVL $0, CX
+	XGETBV
+	MOVL AX, eax+0(FP)
+	MOVL DX, edx+4(FP)
+#endif
+	RET
diff --git a/go/arrow/internal/debug/assert_off.go b/go/arrow/internal/debug/assert_off.go
new file mode 100644
index 0000000000000..52b9a233169d2
--- /dev/null
+++ b/go/arrow/internal/debug/assert_off.go
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !assert
+
+package debug
+
+// Assert will panic with msg if cond is false.
+//
+// msg must be a string, func() string or fmt.Stringer.
+func Assert(cond bool, msg interface{}) {}
diff --git a/go/arrow/internal/debug/assert_on.go b/go/arrow/internal/debug/assert_on.go
new file mode 100644
index 0000000000000..2aa5d6ace4cf0
--- /dev/null
+++ b/go/arrow/internal/debug/assert_on.go
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build assert
+
+package debug
+
+// Assert will panic with msg if cond is false.
+//
+// msg must be a string, func() string or fmt.Stringer.
+func Assert(cond bool, msg interface{}) {
+	if !cond {
+		panic(getStringValue(msg))
+	}
+}
diff --git a/go/arrow/internal/debug/doc.go b/go/arrow/internal/debug/doc.go
new file mode 100644
index 0000000000000..3ee1783ca4bda
--- /dev/null
+++ b/go/arrow/internal/debug/doc.go
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+Package debug provides APIs for conditional runtime assertions and debug logging.
+
+
+Using Assert
+
+To enable runtime assertions, build with the assert tag. When the assert tag is omitted,
+the code for the assertion will be omitted from the binary.
+
+
+Using Log
+
+To enable runtime debug logs, build with the debug tag. When the debug tag is omitted,
+the code for logging will be omitted from the binary.
+*/
+package debug
diff --git a/go/arrow/internal/debug/log_off.go b/go/arrow/internal/debug/log_off.go
new file mode 100644
index 0000000000000..48da8e1ee94c7
--- /dev/null
+++ b/go/arrow/internal/debug/log_off.go
@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !debug
+
+package debug
+
+func Log(interface{}) {}
diff --git a/go/arrow/internal/debug/log_on.go b/go/arrow/internal/debug/log_on.go
new file mode 100644
index 0000000000000..99d0c8ae33fef
--- /dev/null
+++ b/go/arrow/internal/debug/log_on.go
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build debug
+
+package debug
+
+import (
+	"log"
+	"os"
+)
+
+var (
+	debug = log.New(os.Stderr, "[D] ", log.LstdFlags)
+)
+
+func Log(msg interface{}) {
+	debug.Output(1, getStringValue(msg))
+}
diff --git a/go/arrow/internal/debug/util.go b/go/arrow/internal/debug/util.go
new file mode 100644
index 0000000000000..7bd3d5389e669
--- /dev/null
+++ b/go/arrow/internal/debug/util.go
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build debug assert
+
+package debug
+
+import "fmt"
+
+func getStringValue(v interface{}) string {
+	switch a := v.(type) {
+	case func() string:
+		return a()
+
+	case string:
+		return a
+
+	case fmt.Stringer:
+		return a.String()
+
+	default:
+		panic(fmt.Sprintf("unexpected type, %t", v))
+	}
+}
diff --git a/go/arrow/internal/testing/tools/bits.go b/go/arrow/internal/testing/tools/bits.go
new file mode 100644
index 0000000000000..c123573e2fa13
--- /dev/null
+++ b/go/arrow/internal/testing/tools/bits.go
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tools
+
+import "math/bits"
+
+// IntsToBitsLSB encodes ints as LSB 0 bit numbering per https://en.wikipedia.org/wiki/Bit_numbering#LSB_0_bit_numbering
+// The low bit of each nibble is tested, therefore integers should be written as 8-digit
+// hex numbers consisting of 1s or 0s.
+//
+//     IntsToBitsLSB(0x11001010) -> 0x35
+func IntsToBitsLSB(v ...int32) []byte {
+	res := make([]byte, 0, len(v))
+	for _, b := range v {
+		c := uint8(0)
+		for i := uint(0); i < 8; i++ {
+			if b&1 == 1 {
+				c |= 1 << i
+			}
+			b >>= 4
+		}
+		c = bits.Reverse8(c)
+		res = append(res, c)
+	}
+	return res
+}
diff --git a/go/arrow/internal/testing/tools/bits_test.go b/go/arrow/internal/testing/tools/bits_test.go
new file mode 100644
index 0000000000000..ba790eabc5283
--- /dev/null
+++ b/go/arrow/internal/testing/tools/bits_test.go
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tools_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/internal/testing/tools"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIntsToBitsLSB(t *testing.T) {
+	tests := []struct {
+		in  int32
+		exp byte
+	}{
+		{0x11001010, 0x53},
+		{0x00001111, 0xf0},
+		{0x11110000, 0x0f},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%08x", test.in), func(t *testing.T) {
+			got := tools.IntsToBitsLSB(test.in)
+			assert.Equal(t, []byte{test.exp}, got)
+		})
+	}
+}
diff --git a/go/arrow/internal/testing/tools/bool.go b/go/arrow/internal/testing/tools/bool.go
new file mode 100644
index 0000000000000..757a8f52811ab
--- /dev/null
+++ b/go/arrow/internal/testing/tools/bool.go
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tools
+
+func Bools(v ...int) []bool {
+	res := make([]bool, len(v))
+	for i, b := range v {
+		res[i] = b != 0
+	}
+	return res
+}
diff --git a/go/arrow/math/Makefile b/go/arrow/math/Makefile
new file mode 100644
index 0000000000000..89f5d2fa5e959
--- /dev/null
+++ b/go/arrow/math/Makefile
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GO_BUILD=go build
+GO_GEN=go generate
+GO_TEST=go test
+GOPATH=$(realpath ../../../../../..)
+
+# this converts rotate instructions from "ro[lr] <reg>" -> "ro[lr] <reg>, 1" for yasm compatibility
+PERL_FIXUP_ROTATE=perl -i -pe 's/(ro[rl]\s+\w{2,3})$$/\1, 1/'
+
+C2GOASM=c2goasm -a -f
+CC=clang
+C_FLAGS=-target x86_64-unknown-none -masm=intel -mno-red-zone -mstackrealign -mllvm -inline-threshold=1000 -fno-asynchronous-unwind-tables \
+	-fno-exceptions -fno-rtti -O3 -fno-builtin -ffast-math -fno-jump-tables -I_lib
+ASM_FLAGS_AVX2=-mavx2 -mfma -mllvm -force-vector-width=32
+ASM_FLAGS_SSE4=-msse4
+
+GO_SOURCES  := $(shell find . -path ./_lib -prune -o -name '*.go' -not -name '*_test.go')
+ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -not -name '*_test.go')
+
+INTEL_SOURCES := \
+	float64_avx2_amd64.s float64_sse4_amd64.s \
+	int64_avx2_amd64.s int64_sse4_amd64.s \
+	uint64_avx2_amd64.s uint64_sse4_amd64.s
+
+.PHONEY: assembly
+
+assembly: $(INTEL_SOURCES)
+
+generate: ../bin/tmpl
+	../bin/tmpl -i -data=float64.tmpldata type.go.tmpl=float64.go type_amd64.go.tmpl=float64_amd64.go type_noasm.go.tmpl=float64_noasm.go type_test.go.tmpl=float64_test.go
+	../bin/tmpl -i -data=float64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=float64_avx2_amd64.go
+	../bin/tmpl -i -data=float64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=float64_sse4_amd64.go
+	../bin/tmpl -i -data=int64.tmpldata type.go.tmpl=int64.go type_amd64.go.tmpl=int64_amd64.go type_noasm.go.tmpl=int64_noasm.go type_test.go.tmpl=int64_test.go
+	../bin/tmpl -i -data=int64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=int64_avx2_amd64.go
+	../bin/tmpl -i -data=int64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=int64_sse4_amd64.go
+	../bin/tmpl -i -data=uint64.tmpldata type.go.tmpl=uint64.go type_amd64.go.tmpl=uint64_amd64.go type_noasm.go.tmpl=uint64_noasm.go type_test.go.tmpl=uint64_test.go
+	../bin/tmpl -i -data=uint64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=uint64_avx2_amd64.go
+	../bin/tmpl -i -data=uint64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=uint64_sse4_amd64.go
+
+_lib/float64_avx2.s: _lib/float64.c
+	$(CC) -S $(C_FLAGS) $(ASM_FLAGS_AVX2) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+_lib/float64_sse4.s: _lib/float64.c
+	$(CC) -S $(C_FLAGS) $(ASM_FLAGS_SSE4) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+float64_avx2_amd64.s: _lib/float64_avx2.s
+	$(C2GOASM) -a -f $^ $@
+
+float64_sse4_amd64.s: _lib/float64_sse4.s
+	$(C2GOASM) -a -f $^ $@
+
+_lib/int64_avx2.s: _lib/int64.c
+	$(CC) -S $(C_FLAGS) $(ASM_FLAGS_AVX2) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+_lib/int64_sse4.s: _lib/int64.c
+	$(CC) -S $(C_FLAGS) $(ASM_FLAGS_SSE4) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+int64_avx2_amd64.s: _lib/int64_avx2.s
+	$(C2GOASM) -a -f $^ $@
+
+int64_sse4_amd64.s: _lib/int64_sse4.s
+	$(C2GOASM) -a -f $^ $@
+
+_lib/uint64_avx2.s: _lib/uint64.c
+	$(CC) -S $(C_FLAGS) $(ASM_FLAGS_AVX2) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+_lib/uint64_sse4.s: _lib/uint64.c
+	$(CC) -S $(C_FLAGS) $(ASM_FLAGS_SSE4) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+uint64_avx2_amd64.s: _lib/uint64_avx2.s
+	$(C2GOASM) -a -f $^ $@
+
+uint64_sse4_amd64.s: _lib/uint64_sse4.s
+	$(C2GOASM) -a -f $^ $@
+
diff --git a/go/arrow/math/_lib/.gitignore b/go/arrow/math/_lib/.gitignore
new file mode 100644
index 0000000000000..72757d396c0a0
--- /dev/null
+++ b/go/arrow/math/_lib/.gitignore
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.idea
+cmake-build-*
\ No newline at end of file
diff --git a/go/arrow/math/_lib/CMakeLists.txt b/go/arrow/math/_lib/CMakeLists.txt
new file mode 100644
index 0000000000000..ec1558b25fbed
--- /dev/null
+++ b/go/arrow/math/_lib/CMakeLists.txt
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.6)
+
+project(math-func)
+set(CMAKE_C_STANDARD 99)
+
+add_library(memory STATIC float64.c int64.c uint64.c)
+
+
diff --git a/go/arrow/math/_lib/arch.h b/go/arrow/math/_lib/arch.h
new file mode 100644
index 0000000000000..d9c7d98bf6e1b
--- /dev/null
+++ b/go/arrow/math/_lib/arch.h
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#undef FULL_NAME
+
+#if  defined(__AVX2__)
+    #define FULL_NAME(x) x##_avx2
+#elif __SSE4_2__ == 1
+    #define FULL_NAME(x) x##_sse4
+#elif __SSE3__ == 1
+    #define FULL_NAME(x) x##_sse3
+#else
+    #define FULL_NAME(x) x##_x86
+#endif
\ No newline at end of file
diff --git a/go/arrow/math/_lib/float64.c b/go/arrow/math/_lib/float64.c
new file mode 100644
index 0000000000000..24f8c6105717a
--- /dev/null
+++ b/go/arrow/math/_lib/float64.c
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <arch.h>
+#include <memory.h>
+
+void FULL_NAME(sum_float64)(double buf[], size_t len, double *res) {
+    double acc = 0.0;
+    for(int i = 0; i < len; i++) {
+        acc += buf[i];
+    }
+    *res = acc;
+}
\ No newline at end of file
diff --git a/go/arrow/math/_lib/float64_avx2.s b/go/arrow/math/_lib/float64_avx2.s
new file mode 100644
index 0000000000000..8d90a949303ff
--- /dev/null
+++ b/go/arrow/math/_lib/float64_avx2.s
@@ -0,0 +1,176 @@
+	.text
+	.intel_syntax noprefix
+	.file	"_lib/float64.c"
+	.globl	sum_float64_avx2
+	.p2align	4, 0x90
+	.type	sum_float64_avx2,@function
+sum_float64_avx2:                       # @sum_float64_avx2
+# BB#0:
+	push	rbp
+	mov	rbp, rsp
+	and	rsp, -8
+	vxorpd	xmm0, xmm0, xmm0
+	test	rsi, rsi
+	je	.LBB0_14
+# BB#1:
+	cmp	rsi, 31
+	jbe	.LBB0_2
+# BB#5:
+	mov	r9, rsi
+	and	r9, -32
+	je	.LBB0_2
+# BB#6:
+	lea	r8, [r9 - 32]
+	mov	eax, r8d
+	shr	eax, 5
+	inc	eax
+	and	rax, 7
+	je	.LBB0_7
+# BB#8:
+	neg	rax
+	vxorpd	ymm0, ymm0, ymm0
+	xor	ecx, ecx
+	vxorpd	ymm1, ymm1, ymm1
+	vxorpd	ymm2, ymm2, ymm2
+	vxorpd	ymm3, ymm3, ymm3
+	vxorpd	ymm4, ymm4, ymm4
+	vxorpd	ymm5, ymm5, ymm5
+	vxorpd	ymm6, ymm6, ymm6
+	vxorpd	ymm7, ymm7, ymm7
+	.p2align	4, 0x90
+.LBB0_9:                                # =>This Inner Loop Header: Depth=1
+	vaddpd	ymm0, ymm0, ymmword ptr [rdi + 8*rcx]
+	vaddpd	ymm1, ymm1, ymmword ptr [rdi + 8*rcx + 32]
+	vaddpd	ymm2, ymm2, ymmword ptr [rdi + 8*rcx + 64]
+	vaddpd	ymm3, ymm3, ymmword ptr [rdi + 8*rcx + 96]
+	vaddpd	ymm4, ymm4, ymmword ptr [rdi + 8*rcx + 128]
+	vaddpd	ymm5, ymm5, ymmword ptr [rdi + 8*rcx + 160]
+	vaddpd	ymm6, ymm6, ymmword ptr [rdi + 8*rcx + 192]
+	vaddpd	ymm7, ymm7, ymmword ptr [rdi + 8*rcx + 224]
+	add	rcx, 32
+	inc	rax
+	jne	.LBB0_9
+	jmp	.LBB0_10
+.LBB0_2:
+	xor	r9d, r9d
+.LBB0_3:
+	lea	rax, [rdi + 8*r9]
+	sub	rsi, r9
+	.p2align	4, 0x90
+.LBB0_4:                                # =>This Inner Loop Header: Depth=1
+	vaddsd	xmm0, xmm0, qword ptr [rax]
+	add	rax, 8
+	dec	rsi
+	jne	.LBB0_4
+.LBB0_14:
+	vmovsd	qword ptr [rdx], xmm0
+	mov	rsp, rbp
+	pop	rbp
+	vzeroupper
+	ret
+.LBB0_7:
+	xor	ecx, ecx
+	vxorpd	ymm0, ymm0, ymm0
+	vxorpd	ymm1, ymm1, ymm1
+	vxorpd	ymm2, ymm2, ymm2
+	vxorpd	ymm3, ymm3, ymm3
+	vxorpd	ymm4, ymm4, ymm4
+	vxorpd	ymm5, ymm5, ymm5
+	vxorpd	ymm6, ymm6, ymm6
+	vxorpd	ymm7, ymm7, ymm7
+.LBB0_10:
+	cmp	r8, 224
+	jb	.LBB0_13
+# BB#11:
+	mov	rax, r9
+	sub	rax, rcx
+	lea	rcx, [rdi + 8*rcx + 1792]
+	.p2align	4, 0x90
+.LBB0_12:                               # =>This Inner Loop Header: Depth=1
+	vaddpd	ymm7, ymm7, ymmword ptr [rcx - 1568]
+	vaddpd	ymm6, ymm6, ymmword ptr [rcx - 1600]
+	vaddpd	ymm5, ymm5, ymmword ptr [rcx - 1632]
+	vaddpd	ymm4, ymm4, ymmword ptr [rcx - 1664]
+	vaddpd	ymm3, ymm3, ymmword ptr [rcx - 1696]
+	vaddpd	ymm2, ymm2, ymmword ptr [rcx - 1728]
+	vaddpd	ymm1, ymm1, ymmword ptr [rcx - 1760]
+	vaddpd	ymm0, ymm0, ymmword ptr [rcx - 1792]
+	vaddpd	ymm0, ymm0, ymmword ptr [rcx - 1536]
+	vaddpd	ymm1, ymm1, ymmword ptr [rcx - 1504]
+	vaddpd	ymm2, ymm2, ymmword ptr [rcx - 1472]
+	vaddpd	ymm3, ymm3, ymmword ptr [rcx - 1440]
+	vaddpd	ymm4, ymm4, ymmword ptr [rcx - 1408]
+	vaddpd	ymm5, ymm5, ymmword ptr [rcx - 1376]
+	vaddpd	ymm6, ymm6, ymmword ptr [rcx - 1344]
+	vaddpd	ymm7, ymm7, ymmword ptr [rcx - 1312]
+	vaddpd	ymm7, ymm7, ymmword ptr [rcx - 1056]
+	vaddpd	ymm6, ymm6, ymmword ptr [rcx - 1088]
+	vaddpd	ymm5, ymm5, ymmword ptr [rcx - 1120]
+	vaddpd	ymm4, ymm4, ymmword ptr [rcx - 1152]
+	vaddpd	ymm3, ymm3, ymmword ptr [rcx - 1184]
+	vaddpd	ymm2, ymm2, ymmword ptr [rcx - 1216]
+	vaddpd	ymm1, ymm1, ymmword ptr [rcx - 1248]
+	vaddpd	ymm0, ymm0, ymmword ptr [rcx - 1280]
+	vaddpd	ymm0, ymm0, ymmword ptr [rcx - 1024]
+	vaddpd	ymm1, ymm1, ymmword ptr [rcx - 992]
+	vaddpd	ymm2, ymm2, ymmword ptr [rcx - 960]
+	vaddpd	ymm3, ymm3, ymmword ptr [rcx - 928]
+	vaddpd	ymm4, ymm4, ymmword ptr [rcx - 896]
+	vaddpd	ymm5, ymm5, ymmword ptr [rcx - 864]
+	vaddpd	ymm6, ymm6, ymmword ptr [rcx - 832]
+	vaddpd	ymm7, ymm7, ymmword ptr [rcx - 800]
+	vaddpd	ymm7, ymm7, ymmword ptr [rcx - 544]
+	vaddpd	ymm6, ymm6, ymmword ptr [rcx - 576]
+	vaddpd	ymm5, ymm5, ymmword ptr [rcx - 608]
+	vaddpd	ymm4, ymm4, ymmword ptr [rcx - 640]
+	vaddpd	ymm3, ymm3, ymmword ptr [rcx - 672]
+	vaddpd	ymm2, ymm2, ymmword ptr [rcx - 704]
+	vaddpd	ymm1, ymm1, ymmword ptr [rcx - 736]
+	vaddpd	ymm0, ymm0, ymmword ptr [rcx - 768]
+	vaddpd	ymm0, ymm0, ymmword ptr [rcx - 512]
+	vaddpd	ymm1, ymm1, ymmword ptr [rcx - 480]
+	vaddpd	ymm2, ymm2, ymmword ptr [rcx - 448]
+	vaddpd	ymm3, ymm3, ymmword ptr [rcx - 416]
+	vaddpd	ymm4, ymm4, ymmword ptr [rcx - 384]
+	vaddpd	ymm5, ymm5, ymmword ptr [rcx - 352]
+	vaddpd	ymm6, ymm6, ymmword ptr [rcx - 320]
+	vaddpd	ymm7, ymm7, ymmword ptr [rcx - 288]
+	vaddpd	ymm7, ymm7, ymmword ptr [rcx - 32]
+	vaddpd	ymm6, ymm6, ymmword ptr [rcx - 64]
+	vaddpd	ymm5, ymm5, ymmword ptr [rcx - 96]
+	vaddpd	ymm4, ymm4, ymmword ptr [rcx - 128]
+	vaddpd	ymm3, ymm3, ymmword ptr [rcx - 160]
+	vaddpd	ymm2, ymm2, ymmword ptr [rcx - 192]
+	vaddpd	ymm1, ymm1, ymmword ptr [rcx - 224]
+	vaddpd	ymm0, ymm0, ymmword ptr [rcx - 256]
+	vaddpd	ymm0, ymm0, ymmword ptr [rcx]
+	vaddpd	ymm1, ymm1, ymmword ptr [rcx + 32]
+	vaddpd	ymm2, ymm2, ymmword ptr [rcx + 64]
+	vaddpd	ymm3, ymm3, ymmword ptr [rcx + 96]
+	vaddpd	ymm4, ymm4, ymmword ptr [rcx + 128]
+	vaddpd	ymm5, ymm5, ymmword ptr [rcx + 160]
+	vaddpd	ymm6, ymm6, ymmword ptr [rcx + 192]
+	vaddpd	ymm7, ymm7, ymmword ptr [rcx + 224]
+	add	rcx, 2048
+	add	rax, -256
+	jne	.LBB0_12
+.LBB0_13:
+	vaddpd	ymm1, ymm1, ymm5
+	vaddpd	ymm3, ymm3, ymm7
+	vaddpd	ymm0, ymm0, ymm4
+	vaddpd	ymm2, ymm2, ymm6
+	vaddpd	ymm0, ymm0, ymm2
+	vaddpd	ymm1, ymm1, ymm3
+	vaddpd	ymm0, ymm0, ymm1
+	vextractf128	xmm1, ymm0, 1
+	vaddpd	ymm0, ymm0, ymm1
+	vhaddpd	ymm0, ymm0, ymm0
+	cmp	r9, rsi
+	jne	.LBB0_3
+	jmp	.LBB0_14
+.Lfunc_end0:
+	.size	sum_float64_avx2, .Lfunc_end0-sum_float64_avx2
+
+
+	.ident	"Apple LLVM version 9.0.0 (clang-900.0.39.2)"
+	.section	".note.GNU-stack","",@progbits
diff --git a/go/arrow/math/_lib/float64_sse4.s b/go/arrow/math/_lib/float64_sse4.s
new file mode 100644
index 0000000000000..7a454ee2de37a
--- /dev/null
+++ b/go/arrow/math/_lib/float64_sse4.s
@@ -0,0 +1,103 @@
+	.text
+	.intel_syntax noprefix
+	.file	"_lib/float64.c"
+	.globl	sum_float64_sse4
+	.p2align	4, 0x90
+	.type	sum_float64_sse4,@function
+sum_float64_sse4:                       # @sum_float64_sse4
+# BB#0:
+	push	rbp
+	mov	rbp, rsp
+	and	rsp, -8
+	xorpd	xmm0, xmm0
+	test	rsi, rsi
+	je	.LBB0_14
+# BB#1:
+	cmp	rsi, 3
+	jbe	.LBB0_2
+# BB#5:
+	mov	r9, rsi
+	and	r9, -4
+	je	.LBB0_2
+# BB#6:
+	lea	r8, [r9 - 4]
+	mov	eax, r8d
+	shr	eax, 2
+	inc	eax
+	and	rax, 3
+	je	.LBB0_7
+# BB#8:
+	neg	rax
+	xorpd	xmm0, xmm0
+	xor	ecx, ecx
+	xorpd	xmm1, xmm1
+	.p2align	4, 0x90
+.LBB0_9:                                # =>This Inner Loop Header: Depth=1
+	movupd	xmm2, xmmword ptr [rdi + 8*rcx]
+	movupd	xmm3, xmmword ptr [rdi + 8*rcx + 16]
+	addpd	xmm0, xmm2
+	addpd	xmm1, xmm3
+	add	rcx, 4
+	inc	rax
+	jne	.LBB0_9
+	jmp	.LBB0_10
+.LBB0_2:
+	xor	r9d, r9d
+.LBB0_3:
+	lea	rax, [rdi + 8*r9]
+	sub	rsi, r9
+	.p2align	4, 0x90
+.LBB0_4:                                # =>This Inner Loop Header: Depth=1
+	addsd	xmm0, qword ptr [rax]
+	add	rax, 8
+	dec	rsi
+	jne	.LBB0_4
+.LBB0_14:
+	movsd	qword ptr [rdx], xmm0
+	mov	rsp, rbp
+	pop	rbp
+	ret
+.LBB0_7:
+	xor	ecx, ecx
+	xorpd	xmm0, xmm0
+	xorpd	xmm1, xmm1
+.LBB0_10:
+	cmp	r8, 12
+	jb	.LBB0_13
+# BB#11:
+	mov	rax, r9
+	sub	rax, rcx
+	lea	rcx, [rdi + 8*rcx + 112]
+	.p2align	4, 0x90
+.LBB0_12:                               # =>This Inner Loop Header: Depth=1
+	movupd	xmm2, xmmword ptr [rcx - 112]
+	movupd	xmm3, xmmword ptr [rcx - 96]
+	movupd	xmm4, xmmword ptr [rcx - 80]
+	movupd	xmm5, xmmword ptr [rcx - 64]
+	addpd	xmm2, xmm0
+	addpd	xmm3, xmm1
+	movupd	xmm6, xmmword ptr [rcx - 48]
+	movupd	xmm7, xmmword ptr [rcx - 32]
+	addpd	xmm6, xmm4
+	addpd	xmm6, xmm2
+	addpd	xmm7, xmm5
+	addpd	xmm7, xmm3
+	movupd	xmm0, xmmword ptr [rcx - 16]
+	movupd	xmm1, xmmword ptr [rcx]
+	addpd	xmm0, xmm6
+	addpd	xmm1, xmm7
+	sub	rcx, -128
+	add	rax, -16
+	jne	.LBB0_12
+.LBB0_13:
+	addpd	xmm0, xmm1
+	haddpd	xmm0, xmm0
+	cmp	r9, rsi
+	jne	.LBB0_3
+	jmp	.LBB0_14
+.Lfunc_end0:
+	.size	sum_float64_sse4, .Lfunc_end0-sum_float64_sse4
+
+
+	.ident	"Apple LLVM version 9.0.0 (clang-900.0.39.2)"
+	.section	".note.GNU-stack","",@progbits
diff --git a/go/arrow/math/_lib/int64.c b/go/arrow/math/_lib/int64.c
new file mode 100644
index 0000000000000..5afe505f44fe8
--- /dev/null
+++ b/go/arrow/math/_lib/int64.c
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <arch.h>
+#include <memory.h>
+#include <stdint.h>
+
+void FULL_NAME(sum_int64)(int64_t buf[], size_t len, int64_t *res) {
+    int64_t acc = 0;
+    for(int i = 0; i < len; i++) {
+        acc += buf[i];
+    }
+    *res = acc;
+}
\ No newline at end of file
diff --git a/go/arrow/math/_lib/int64_avx2.s b/go/arrow/math/_lib/int64_avx2.s
new file mode 100644
index 0000000000000..f088c49e8b09a
--- /dev/null
+++ b/go/arrow/math/_lib/int64_avx2.s
@@ -0,0 +1,181 @@
+	.text
+	.intel_syntax noprefix
+	.file	"_lib/int64.c"
+	.globl	sum_int64_avx2
+	.p2align	4, 0x90
+	.type	sum_int64_avx2,@function
+sum_int64_avx2:                         # @sum_int64_avx2
+# BB#0:
+	push	rbp
+	mov	rbp, rsp
+	and	rsp, -8
+	test	rsi, rsi
+	je	.LBB0_1
+# BB#2:
+	cmp	rsi, 31
+	jbe	.LBB0_3
+# BB#6:
+	mov	r9, rsi
+	and	r9, -32
+	je	.LBB0_3
+# BB#7:
+	lea	r8, [r9 - 32]
+	mov	eax, r8d
+	shr	eax, 5
+	inc	eax
+	and	rax, 7
+	je	.LBB0_8
+# BB#9:
+	neg	rax
+	vpxor	ymm0, ymm0, ymm0
+	xor	ecx, ecx
+	vpxor	ymm1, ymm1, ymm1
+	vpxor	ymm2, ymm2, ymm2
+	vpxor	ymm3, ymm3, ymm3
+	vpxor	ymm4, ymm4, ymm4
+	vpxor	ymm5, ymm5, ymm5
+	vpxor	ymm6, ymm6, ymm6
+	vpxor	ymm7, ymm7, ymm7
+	.p2align	4, 0x90
+.LBB0_10:                               # =>This Inner Loop Header: Depth=1
+	vpaddq	ymm0, ymm0, ymmword ptr [rdi + 8*rcx]
+	vpaddq	ymm1, ymm1, ymmword ptr [rdi + 8*rcx + 32]
+	vpaddq	ymm2, ymm2, ymmword ptr [rdi + 8*rcx + 64]
+	vpaddq	ymm3, ymm3, ymmword ptr [rdi + 8*rcx + 96]
+	vpaddq	ymm4, ymm4, ymmword ptr [rdi + 8*rcx + 128]
+	vpaddq	ymm5, ymm5, ymmword ptr [rdi + 8*rcx + 160]
+	vpaddq	ymm6, ymm6, ymmword ptr [rdi + 8*rcx + 192]
+	vpaddq	ymm7, ymm7, ymmword ptr [rdi + 8*rcx + 224]
+	add	rcx, 32
+	inc	rax
+	jne	.LBB0_10
+	jmp	.LBB0_11
+.LBB0_3:
+	xor	r9d, r9d
+	xor	eax, eax
+.LBB0_4:
+	lea	rcx, [rdi + 8*r9]
+	sub	rsi, r9
+	.p2align	4, 0x90
+.LBB0_5:                                # =>This Inner Loop Header: Depth=1
+	add	rax, qword ptr [rcx]
+	add	rcx, 8
+	dec	rsi
+	jne	.LBB0_5
+	jmp	.LBB0_15
+.LBB0_1:
+	xor	eax, eax
+.LBB0_15:
+	mov	qword ptr [rdx], rax
+	mov	rsp, rbp
+	pop	rbp
+	vzeroupper
+	ret
+.LBB0_8:
+	xor	ecx, ecx
+	vpxor	ymm0, ymm0, ymm0
+	vpxor	ymm1, ymm1, ymm1
+	vpxor	ymm2, ymm2, ymm2
+	vpxor	ymm3, ymm3, ymm3
+	vpxor	ymm4, ymm4, ymm4
+	vpxor	ymm5, ymm5, ymm5
+	vpxor	ymm6, ymm6, ymm6
+	vpxor	ymm7, ymm7, ymm7
+.LBB0_11:
+	cmp	r8, 224
+	jb	.LBB0_14
+# BB#12:
+	mov	rax, r9
+	sub	rax, rcx
+	lea	rcx, [rdi + 8*rcx + 1792]
+	.p2align	4, 0x90
+.LBB0_13:                               # =>This Inner Loop Header: Depth=1
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 1568]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 1600]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 1632]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 1664]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 1696]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 1728]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 1760]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 1792]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 1536]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 1504]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 1472]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 1440]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 1408]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 1376]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 1344]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 1312]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 1056]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 1088]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 1120]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 1152]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 1184]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 1216]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 1248]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 1280]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 1024]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 992]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 960]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 928]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 896]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 864]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 832]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 800]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 544]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 576]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 608]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 640]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 672]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 704]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 736]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 768]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 512]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 480]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 448]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 416]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 384]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 352]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 320]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 288]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 32]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 64]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 96]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 128]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 160]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 192]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 224]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 256]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx + 32]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx + 64]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx + 96]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx + 128]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx + 160]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx + 192]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx + 224]
+	add	rcx, 2048
+	add	rax, -256
+	jne	.LBB0_13
+.LBB0_14:
+	vpaddq	ymm1, ymm1, ymm5
+	vpaddq	ymm3, ymm3, ymm7
+	vpaddq	ymm0, ymm0, ymm4
+	vpaddq	ymm2, ymm2, ymm6
+	vpaddq	ymm0, ymm0, ymm2
+	vpaddq	ymm1, ymm1, ymm3
+	vpaddq	ymm0, ymm0, ymm1
+	vextracti128	xmm1, ymm0, 1
+	vpaddq	ymm0, ymm0, ymm1
+	vpshufd	xmm1, xmm0, 78          # xmm1 = xmm0[2,3,0,1]
+	vpaddq	ymm0, ymm0, ymm1
+	vmovq	rax, xmm0
+	cmp	r9, rsi
+	jne	.LBB0_4
+	jmp	.LBB0_15
+.Lfunc_end0:
+	.size	sum_int64_avx2, .Lfunc_end0-sum_int64_avx2
+
+
+	.ident	"Apple LLVM version 9.0.0 (clang-900.0.39.2)"
+	.section	".note.GNU-stack","",@progbits
diff --git a/go/arrow/math/_lib/int64_sse4.s b/go/arrow/math/_lib/int64_sse4.s
new file mode 100644
index 0000000000000..d433260f7fb42
--- /dev/null
+++ b/go/arrow/math/_lib/int64_sse4.s
@@ -0,0 +1,108 @@
+	.text
+	.intel_syntax noprefix
+	.file	"_lib/int64.c"
+	.globl	sum_int64_sse4
+	.p2align	4, 0x90
+	.type	sum_int64_sse4,@function
+sum_int64_sse4:                         # @sum_int64_sse4
+# BB#0:
+	push	rbp
+	mov	rbp, rsp
+	and	rsp, -8
+	test	rsi, rsi
+	je	.LBB0_1
+# BB#2:
+	cmp	rsi, 3
+	jbe	.LBB0_3
+# BB#6:
+	mov	r9, rsi
+	and	r9, -4
+	je	.LBB0_3
+# BB#7:
+	lea	r8, [r9 - 4]
+	mov	eax, r8d
+	shr	eax, 2
+	inc	eax
+	and	rax, 3
+	je	.LBB0_8
+# BB#9:
+	neg	rax
+	pxor	xmm0, xmm0
+	xor	ecx, ecx
+	pxor	xmm1, xmm1
+	.p2align	4, 0x90
+.LBB0_10:                               # =>This Inner Loop Header: Depth=1
+	movdqu	xmm2, xmmword ptr [rdi + 8*rcx]
+	movdqu	xmm3, xmmword ptr [rdi + 8*rcx + 16]
+	paddq	xmm0, xmm2
+	paddq	xmm1, xmm3
+	add	rcx, 4
+	inc	rax
+	jne	.LBB0_10
+	jmp	.LBB0_11
+.LBB0_3:
+	xor	r9d, r9d
+	xor	eax, eax
+.LBB0_4:
+	lea	rcx, [rdi + 8*r9]
+	sub	rsi, r9
+	.p2align	4, 0x90
+.LBB0_5:                                # =>This Inner Loop Header: Depth=1
+	add	rax, qword ptr [rcx]
+	add	rcx, 8
+	dec	rsi
+	jne	.LBB0_5
+	jmp	.LBB0_15
+.LBB0_1:
+	xor	eax, eax
+.LBB0_15:
+	mov	qword ptr [rdx], rax
+	mov	rsp, rbp
+	pop	rbp
+	ret
+.LBB0_8:
+	xor	ecx, ecx
+	pxor	xmm0, xmm0
+	pxor	xmm1, xmm1
+.LBB0_11:
+	cmp	r8, 12
+	jb	.LBB0_14
+# BB#12:
+	mov	rax, r9
+	sub	rax, rcx
+	lea	rcx, [rdi + 8*rcx + 112]
+	.p2align	4, 0x90
+.LBB0_13:                               # =>This Inner Loop Header: Depth=1
+	movdqu	xmm2, xmmword ptr [rcx - 112]
+	movdqu	xmm3, xmmword ptr [rcx - 96]
+	movdqu	xmm4, xmmword ptr [rcx - 80]
+	movdqu	xmm5, xmmword ptr [rcx - 64]
+	paddq	xmm2, xmm0
+	paddq	xmm3, xmm1
+	movdqu	xmm6, xmmword ptr [rcx - 48]
+	movdqu	xmm7, xmmword ptr [rcx - 32]
+	paddq	xmm6, xmm4
+	paddq	xmm6, xmm2
+	paddq	xmm7, xmm5
+	paddq	xmm7, xmm3
+	movdqu	xmm0, xmmword ptr [rcx - 16]
+	movdqu	xmm1, xmmword ptr [rcx]
+	paddq	xmm0, xmm6
+	paddq	xmm1, xmm7
+	sub	rcx, -128
+	add	rax, -16
+	jne	.LBB0_13
+.LBB0_14:
+	paddq	xmm0, xmm1
+	pshufd	xmm1, xmm0, 78          # xmm1 = xmm0[2,3,0,1]
+	paddq	xmm1, xmm0
+	movq	rax, xmm1
+	cmp	r9, rsi
+	jne	.LBB0_4
+	jmp	.LBB0_15
+.Lfunc_end0:
+	.size	sum_int64_sse4, .Lfunc_end0-sum_int64_sse4
+
+
+	.ident	"Apple LLVM version 9.0.0 (clang-900.0.39.2)"
+	.section	".note.GNU-stack","",@progbits
diff --git a/go/arrow/math/_lib/uint64.c b/go/arrow/math/_lib/uint64.c
new file mode 100644
index 0000000000000..91d3d9add86aa
--- /dev/null
+++ b/go/arrow/math/_lib/uint64.c
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <arch.h>
+#include <memory.h>
+#include <stdint.h>
+
+void FULL_NAME(sum_uint64)(uint64_t buf[], size_t len, uint64_t *res) {
+    uint64_t acc = 0;
+    for(int i = 0; i < len; i++) {
+        acc += buf[i];
+    }
+    *res = acc;
+}
\ No newline at end of file
diff --git a/go/arrow/math/_lib/uint64_avx2.s b/go/arrow/math/_lib/uint64_avx2.s
new file mode 100644
index 0000000000000..e4502ee06ca40
--- /dev/null
+++ b/go/arrow/math/_lib/uint64_avx2.s
@@ -0,0 +1,181 @@
+	.text
+	.intel_syntax noprefix
+	.file	"_lib/uint64.c"
+	.globl	sum_uint64_avx2
+	.p2align	4, 0x90
+	.type	sum_uint64_avx2,@function
+sum_uint64_avx2:                        # @sum_uint64_avx2
+# BB#0:
+	push	rbp
+	mov	rbp, rsp
+	and	rsp, -8
+	test	rsi, rsi
+	je	.LBB0_1
+# BB#2:
+	cmp	rsi, 31
+	jbe	.LBB0_3
+# BB#6:
+	mov	r9, rsi
+	and	r9, -32
+	je	.LBB0_3
+# BB#7:
+	lea	r8, [r9 - 32]
+	mov	eax, r8d
+	shr	eax, 5
+	inc	eax
+	and	rax, 7
+	je	.LBB0_8
+# BB#9:
+	neg	rax
+	vpxor	ymm0, ymm0, ymm0
+	xor	ecx, ecx
+	vpxor	ymm1, ymm1, ymm1
+	vpxor	ymm2, ymm2, ymm2
+	vpxor	ymm3, ymm3, ymm3
+	vpxor	ymm4, ymm4, ymm4
+	vpxor	ymm5, ymm5, ymm5
+	vpxor	ymm6, ymm6, ymm6
+	vpxor	ymm7, ymm7, ymm7
+	.p2align	4, 0x90
+.LBB0_10:                               # =>This Inner Loop Header: Depth=1
+	vpaddq	ymm0, ymm0, ymmword ptr [rdi + 8*rcx]
+	vpaddq	ymm1, ymm1, ymmword ptr [rdi + 8*rcx + 32]
+	vpaddq	ymm2, ymm2, ymmword ptr [rdi + 8*rcx + 64]
+	vpaddq	ymm3, ymm3, ymmword ptr [rdi + 8*rcx + 96]
+	vpaddq	ymm4, ymm4, ymmword ptr [rdi + 8*rcx + 128]
+	vpaddq	ymm5, ymm5, ymmword ptr [rdi + 8*rcx + 160]
+	vpaddq	ymm6, ymm6, ymmword ptr [rdi + 8*rcx + 192]
+	vpaddq	ymm7, ymm7, ymmword ptr [rdi + 8*rcx + 224]
+	add	rcx, 32
+	inc	rax
+	jne	.LBB0_10
+	jmp	.LBB0_11
+.LBB0_3:
+	xor	r9d, r9d
+	xor	eax, eax
+.LBB0_4:
+	lea	rcx, [rdi + 8*r9]
+	sub	rsi, r9
+	.p2align	4, 0x90
+.LBB0_5:                                # =>This Inner Loop Header: Depth=1
+	add	rax, qword ptr [rcx]
+	add	rcx, 8
+	dec	rsi
+	jne	.LBB0_5
+	jmp	.LBB0_15
+.LBB0_1:
+	xor	eax, eax
+.LBB0_15:
+	mov	qword ptr [rdx], rax
+	mov	rsp, rbp
+	pop	rbp
+	vzeroupper
+	ret
+.LBB0_8:
+	xor	ecx, ecx
+	vpxor	ymm0, ymm0, ymm0
+	vpxor	ymm1, ymm1, ymm1
+	vpxor	ymm2, ymm2, ymm2
+	vpxor	ymm3, ymm3, ymm3
+	vpxor	ymm4, ymm4, ymm4
+	vpxor	ymm5, ymm5, ymm5
+	vpxor	ymm6, ymm6, ymm6
+	vpxor	ymm7, ymm7, ymm7
+.LBB0_11:
+	cmp	r8, 224
+	jb	.LBB0_14
+# BB#12:
+	mov	rax, r9
+	sub	rax, rcx
+	lea	rcx, [rdi + 8*rcx + 1792]
+	.p2align	4, 0x90
+.LBB0_13:                               # =>This Inner Loop Header: Depth=1
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 1568]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 1600]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 1632]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 1664]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 1696]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 1728]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 1760]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 1792]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 1536]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 1504]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 1472]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 1440]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 1408]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 1376]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 1344]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 1312]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 1056]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 1088]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 1120]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 1152]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 1184]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 1216]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 1248]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 1280]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 1024]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 992]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 960]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 928]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 896]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 864]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 832]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 800]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 544]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 576]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 608]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 640]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 672]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 704]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 736]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 768]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 512]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 480]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 448]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 416]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 384]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 352]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 320]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 288]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx - 32]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx - 64]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx - 96]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx - 128]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx - 160]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx - 192]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx - 224]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx - 256]
+	vpaddq	ymm0, ymm0, ymmword ptr [rcx]
+	vpaddq	ymm1, ymm1, ymmword ptr [rcx + 32]
+	vpaddq	ymm2, ymm2, ymmword ptr [rcx + 64]
+	vpaddq	ymm3, ymm3, ymmword ptr [rcx + 96]
+	vpaddq	ymm4, ymm4, ymmword ptr [rcx + 128]
+	vpaddq	ymm5, ymm5, ymmword ptr [rcx + 160]
+	vpaddq	ymm6, ymm6, ymmword ptr [rcx + 192]
+	vpaddq	ymm7, ymm7, ymmword ptr [rcx + 224]
+	add	rcx, 2048
+	add	rax, -256
+	jne	.LBB0_13
+.LBB0_14:
+	vpaddq	ymm1, ymm1, ymm5
+	vpaddq	ymm3, ymm3, ymm7
+	vpaddq	ymm0, ymm0, ymm4
+	vpaddq	ymm2, ymm2, ymm6
+	vpaddq	ymm0, ymm0, ymm2
+	vpaddq	ymm1, ymm1, ymm3
+	vpaddq	ymm0, ymm0, ymm1
+	vextracti128	xmm1, ymm0, 1
+	vpaddq	ymm0, ymm0, ymm1
+	vpshufd	xmm1, xmm0, 78          # xmm1 = xmm0[2,3,0,1]
+	vpaddq	ymm0, ymm0, ymm1
+	vmovq	rax, xmm0
+	cmp	r9, rsi
+	jne	.LBB0_4
+	jmp	.LBB0_15
+.Lfunc_end0:
+	.size	sum_uint64_avx2, .Lfunc_end0-sum_uint64_avx2
+
+
+	.ident	"Apple LLVM version 9.0.0 (clang-900.0.39.2)"
+	.section	".note.GNU-stack","",@progbits
diff --git a/go/arrow/math/_lib/uint64_sse4.s b/go/arrow/math/_lib/uint64_sse4.s
new file mode 100644
index 0000000000000..04b4cc8536189
--- /dev/null
+++ b/go/arrow/math/_lib/uint64_sse4.s
@@ -0,0 +1,108 @@
+	.text
+	.intel_syntax noprefix
+	.file	"_lib/uint64.c"
+	.globl	sum_uint64_sse4
+	.p2align	4, 0x90
+	.type	sum_uint64_sse4,@function
+sum_uint64_sse4:                        # @sum_uint64_sse4
+# BB#0:
+	push	rbp
+	mov	rbp, rsp
+	and	rsp, -8
+	test	rsi, rsi
+	je	.LBB0_1
+# BB#2:
+	cmp	rsi, 3
+	jbe	.LBB0_3
+# BB#6:
+	mov	r9, rsi
+	and	r9, -4
+	je	.LBB0_3
+# BB#7:
+	lea	r8, [r9 - 4]
+	mov	eax, r8d
+	shr	eax, 2
+	inc	eax
+	and	rax, 3
+	je	.LBB0_8
+# BB#9:
+	neg	rax
+	pxor	xmm0, xmm0
+	xor	ecx, ecx
+	pxor	xmm1, xmm1
+	.p2align	4, 0x90
+.LBB0_10:                               # =>This Inner Loop Header: Depth=1
+	movdqu	xmm2, xmmword ptr [rdi + 8*rcx]
+	movdqu	xmm3, xmmword ptr [rdi + 8*rcx + 16]
+	paddq	xmm0, xmm2
+	paddq	xmm1, xmm3
+	add	rcx, 4
+	inc	rax
+	jne	.LBB0_10
+	jmp	.LBB0_11
+.LBB0_3:
+	xor	r9d, r9d
+	xor	eax, eax
+.LBB0_4:
+	lea	rcx, [rdi + 8*r9]
+	sub	rsi, r9
+	.p2align	4, 0x90
+.LBB0_5:                                # =>This Inner Loop Header: Depth=1
+	add	rax, qword ptr [rcx]
+	add	rcx, 8
+	dec	rsi
+	jne	.LBB0_5
+	jmp	.LBB0_15
+.LBB0_1:
+	xor	eax, eax
+.LBB0_15:
+	mov	qword ptr [rdx], rax
+	mov	rsp, rbp
+	pop	rbp
+	ret
+.LBB0_8:
+	xor	ecx, ecx
+	pxor	xmm0, xmm0
+	pxor	xmm1, xmm1
+.LBB0_11:
+	cmp	r8, 12
+	jb	.LBB0_14
+# BB#12:
+	mov	rax, r9
+	sub	rax, rcx
+	lea	rcx, [rdi + 8*rcx + 112]
+	.p2align	4, 0x90
+.LBB0_13:                               # =>This Inner Loop Header: Depth=1
+	movdqu	xmm2, xmmword ptr [rcx - 112]
+	movdqu	xmm3, xmmword ptr [rcx - 96]
+	movdqu	xmm4, xmmword ptr [rcx - 80]
+	movdqu	xmm5, xmmword ptr [rcx - 64]
+	paddq	xmm2, xmm0
+	paddq	xmm3, xmm1
+	movdqu	xmm6, xmmword ptr [rcx - 48]
+	movdqu	xmm7, xmmword ptr [rcx - 32]
+	paddq	xmm6, xmm4
+	paddq	xmm6, xmm2
+	paddq	xmm7, xmm5
+	paddq	xmm7, xmm3
+	movdqu	xmm0, xmmword ptr [rcx - 16]
+	movdqu	xmm1, xmmword ptr [rcx]
+	paddq	xmm0, xmm6
+	paddq	xmm1, xmm7
+	sub	rcx, -128
+	add	rax, -16
+	jne	.LBB0_13
+.LBB0_14:
+	paddq	xmm0, xmm1
+	pshufd	xmm1, xmm0, 78          # xmm1 = xmm0[2,3,0,1]
+	paddq	xmm1, xmm0
+	movq	rax, xmm1
+	cmp	r9, rsi
+	jne	.LBB0_4
+	jmp	.LBB0_15
+.Lfunc_end0:
+	.size	sum_uint64_sse4, .Lfunc_end0-sum_uint64_sse4
+
+
+	.ident	"Apple LLVM version 9.0.0 (clang-900.0.39.2)"
+	.section	".note.GNU-stack","",@progbits
diff --git a/go/arrow/math/doc.go b/go/arrow/math/doc.go
new file mode 100644
index 0000000000000..30d50a2f16e8f
--- /dev/null
+++ b/go/arrow/math/doc.go
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+Package math provides optimized mathematical functions for processing Arrow arrays.
+*/
+package math
+
+//go:generate go run ../_tools/tmpl/main.go -i -data=float64.tmpldata type.go.tmpl=float64.go type_amd64.go.tmpl=float64_amd64.go type_test.go.tmpl=float64_test.go
+//go:generate go run ../_tools/tmpl/main.go -i -data=float64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=float64_avx2_amd64.go
+//go:generate go run ../_tools/tmpl/main.go -i -data=float64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=float64_sse4_amd64.go
+//go:generate go run ../_tools/tmpl/main.go -i -data=int64.tmpldata type.go.tmpl=int64.go type_amd64.go.tmpl=int64_amd64.go type_test.go.tmpl=int64_test.go
+//go:generate go run ../_tools/tmpl/main.go -i -data=int64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=int64_avx2_amd64.go
+//go:generate go run ../_tools/tmpl/main.go -i -data=int64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=int64_sse4_amd64.go
+//go:generate go run ../_tools/tmpl/main.go -i -data=uint64.tmpldata type.go.tmpl=uint64.go type_amd64.go.tmpl=uint64_amd64.go type_test.go.tmpl=uint64_test.go
+//go:generate go run ../_tools/tmpl/main.go -i -data=uint64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=uint64_avx2_amd64.go
+//go:generate go run ../_tools/tmpl/main.go -i -data=uint64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=uint64_sse4_amd64.go
diff --git a/go/arrow/math/float64.go b/go/arrow/math/float64.go
new file mode 100644
index 0000000000000..a9c54665e5404
--- /dev/null
+++ b/go/arrow/math/float64.go
@@ -0,0 +1,44 @@
+// Code generated by type.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math
+
+import (
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+type Float64Funcs struct {
+	sum func(a *array.Float64) float64
+}
+
+var (
+	Float64 Float64Funcs
+)
+
+// Sum returns the summation of all elements in a.
+func (f Float64Funcs) Sum(a *array.Float64) float64 {
+	return f.sum(a)
+}
+
+func sum_float64_go(a *array.Float64) float64 {
+	acc := float64(0)
+	for _, v := range a.Float64Values() {
+		acc += v
+	}
+	return acc
+}
diff --git a/go/arrow/math/float64.tmpldata b/go/arrow/math/float64.tmpldata
new file mode 100644
index 0000000000000..de54eea2a1056
--- /dev/null
+++ b/go/arrow/math/float64.tmpldata
@@ -0,0 +1,4 @@
+{
+  "Name": "Float64",
+  "Type": "float64"
+}
\ No newline at end of file
diff --git a/go/arrow/math/float64_amd64.go b/go/arrow/math/float64_amd64.go
new file mode 100644
index 0000000000000..87227d5fa822f
--- /dev/null
+++ b/go/arrow/math/float64_amd64.go
@@ -0,0 +1,33 @@
+// Code generated by type_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+func initFloat64AVX2() {
+	Float64.sum = sum_float64_avx2
+}
+
+func initFloat64SSE4() {
+	Float64.sum = sum_float64_sse4
+}
+
+func initFloat64Go() {
+	Float64.sum = sum_float64_go
+}
diff --git a/go/arrow/math/float64_avx2_amd64.go b/go/arrow/math/float64_avx2_amd64.go
new file mode 100644
index 0000000000000..9301b3284978e
--- /dev/null
+++ b/go/arrow/math/float64_avx2_amd64.go
@@ -0,0 +1,41 @@
+// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+import (
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+//go:noescape
+func _sum_float64_avx2(buf, len, res unsafe.Pointer)
+
+func sum_float64_avx2(a *array.Float64) float64 {
+	buf := a.Float64Values()
+	var (
+		p1  = unsafe.Pointer(&buf[0])
+		p2  = unsafe.Pointer(uintptr(len(buf)))
+		res float64
+	)
+	_sum_float64_avx2(p1, p2, unsafe.Pointer(&res))
+	return res
+}
diff --git a/go/arrow/math/float64_avx2_amd64.s b/go/arrow/math/float64_avx2_amd64.s
new file mode 100644
index 0000000000000..e5fe247ec158c
--- /dev/null
+++ b/go/arrow/math/float64_avx2_amd64.s
@@ -0,0 +1,167 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_sum_float64_avx2(SB), $0-24
+
+	MOVQ buf+0(FP), DI
+	MOVQ len+8(FP), SI
+	MOVQ res+16(FP), DX
+
+	LONG $0xc057f9c5         // vxorpd    xmm0, xmm0, xmm0
+	WORD $0x8548; BYTE $0xf6 // test    rsi, rsi
+	JE   LBB0_14
+	LONG $0x1ffe8348         // cmp    rsi, 31
+	JBE  LBB0_2
+	WORD $0x8949; BYTE $0xf1 // mov    r9, rsi
+	LONG $0xe0e18349         // and    r9, -32
+	JE   LBB0_2
+	LONG $0xe0418d4d         // lea    r8, [r9 - 32]
+	WORD $0x8944; BYTE $0xc0 // mov    eax, r8d
+	WORD $0xe8c1; BYTE $0x05 // shr    eax, 5
+	WORD $0xc0ff             // inc    eax
+	LONG $0x07e08348         // and    rax, 7
+	JE   LBB0_7
+	WORD $0xf748; BYTE $0xd8 // neg    rax
+	LONG $0xc057fdc5         // vxorpd    ymm0, ymm0, ymm0
+	WORD $0xc931             // xor    ecx, ecx
+	LONG $0xc957f5c5         // vxorpd    ymm1, ymm1, ymm1
+	LONG $0xd257edc5         // vxorpd    ymm2, ymm2, ymm2
+	LONG $0xdb57e5c5         // vxorpd    ymm3, ymm3, ymm3
+	LONG $0xe457ddc5         // vxorpd    ymm4, ymm4, ymm4
+	LONG $0xed57d5c5         // vxorpd    ymm5, ymm5, ymm5
+	LONG $0xf657cdc5         // vxorpd    ymm6, ymm6, ymm6
+	LONG $0xff57c5c5         // vxorpd    ymm7, ymm7, ymm7
+
+LBB0_9:
+	LONG $0x0458fdc5; BYTE $0xcf         // vaddpd    ymm0, ymm0, yword [rdi + 8*rcx]
+	LONG $0x4c58f5c5; WORD $0x20cf       // vaddpd    ymm1, ymm1, yword [rdi + 8*rcx + 32]
+	LONG $0x5458edc5; WORD $0x40cf       // vaddpd    ymm2, ymm2, yword [rdi + 8*rcx + 64]
+	LONG $0x5c58e5c5; WORD $0x60cf       // vaddpd    ymm3, ymm3, yword [rdi + 8*rcx + 96]
+	QUAD $0x000080cfa458ddc5; BYTE $0x00 // vaddpd    ymm4, ymm4, yword [rdi + 8*rcx + 128]
+	QUAD $0x0000a0cfac58d5c5; BYTE $0x00 // vaddpd    ymm5, ymm5, yword [rdi + 8*rcx + 160]
+	QUAD $0x0000c0cfb458cdc5; BYTE $0x00 // vaddpd    ymm6, ymm6, yword [rdi + 8*rcx + 192]
+	QUAD $0x0000e0cfbc58c5c5; BYTE $0x00 // vaddpd    ymm7, ymm7, yword [rdi + 8*rcx + 224]
+	LONG $0x20c18348                     // add    rcx, 32
+	WORD $0xff48; BYTE $0xc0             // inc    rax
+	JNE  LBB0_9
+	JMP  LBB0_10
+
+LBB0_2:
+	WORD $0x3145; BYTE $0xc9 // xor    r9d, r9d
+
+LBB0_3:
+	LONG $0xcf048d4a         // lea    rax, [rdi + 8*r9]
+	WORD $0x294c; BYTE $0xce // sub    rsi, r9
+
+LBB0_4:
+	LONG $0x0058fbc5         // vaddsd    xmm0, xmm0, qword [rax]
+	LONG $0x08c08348         // add    rax, 8
+	WORD $0xff48; BYTE $0xce // dec    rsi
+	JNE  LBB0_4
+
+LBB0_14:
+	LONG $0x0211fbc5 // vmovsd    qword [rdx], xmm0
+	VZEROUPPER
+	RET
+
+LBB0_7:
+	WORD $0xc931     // xor    ecx, ecx
+	LONG $0xc057fdc5 // vxorpd    ymm0, ymm0, ymm0
+	LONG $0xc957f5c5 // vxorpd    ymm1, ymm1, ymm1
+	LONG $0xd257edc5 // vxorpd    ymm2, ymm2, ymm2
+	LONG $0xdb57e5c5 // vxorpd    ymm3, ymm3, ymm3
+	LONG $0xe457ddc5 // vxorpd    ymm4, ymm4, ymm4
+	LONG $0xed57d5c5 // vxorpd    ymm5, ymm5, ymm5
+	LONG $0xf657cdc5 // vxorpd    ymm6, ymm6, ymm6
+	LONG $0xff57c5c5 // vxorpd    ymm7, ymm7, ymm7
+
+LBB0_10:
+	LONG $0xe0f88149; WORD $0x0000; BYTE $0x00 // cmp    r8, 224
+	JB   LBB0_13
+	WORD $0x894c; BYTE $0xc8                   // mov    rax, r9
+	WORD $0x2948; BYTE $0xc8                   // sub    rax, rcx
+	QUAD $0x00000700cf8c8d48                   // lea    rcx, [rdi + 8*rcx + 1792]
+
+LBB0_12:
+	QUAD $0xfffff9e0b958c5c5                   // vaddpd    ymm7, ymm7, yword [rcx - 1568]
+	QUAD $0xfffff9c0b158cdc5                   // vaddpd    ymm6, ymm6, yword [rcx - 1600]
+	QUAD $0xfffff9a0a958d5c5                   // vaddpd    ymm5, ymm5, yword [rcx - 1632]
+	QUAD $0xfffff980a158ddc5                   // vaddpd    ymm4, ymm4, yword [rcx - 1664]
+	QUAD $0xfffff9609958e5c5                   // vaddpd    ymm3, ymm3, yword [rcx - 1696]
+	QUAD $0xfffff9409158edc5                   // vaddpd    ymm2, ymm2, yword [rcx - 1728]
+	QUAD $0xfffff9208958f5c5                   // vaddpd    ymm1, ymm1, yword [rcx - 1760]
+	QUAD $0xfffff9008158fdc5                   // vaddpd    ymm0, ymm0, yword [rcx - 1792]
+	QUAD $0xfffffa008158fdc5                   // vaddpd    ymm0, ymm0, yword [rcx - 1536]
+	QUAD $0xfffffa208958f5c5                   // vaddpd    ymm1, ymm1, yword [rcx - 1504]
+	QUAD $0xfffffa409158edc5                   // vaddpd    ymm2, ymm2, yword [rcx - 1472]
+	QUAD $0xfffffa609958e5c5                   // vaddpd    ymm3, ymm3, yword [rcx - 1440]
+	QUAD $0xfffffa80a158ddc5                   // vaddpd    ymm4, ymm4, yword [rcx - 1408]
+	QUAD $0xfffffaa0a958d5c5                   // vaddpd    ymm5, ymm5, yword [rcx - 1376]
+	QUAD $0xfffffac0b158cdc5                   // vaddpd    ymm6, ymm6, yword [rcx - 1344]
+	QUAD $0xfffffae0b958c5c5                   // vaddpd    ymm7, ymm7, yword [rcx - 1312]
+	QUAD $0xfffffbe0b958c5c5                   // vaddpd    ymm7, ymm7, yword [rcx - 1056]
+	QUAD $0xfffffbc0b158cdc5                   // vaddpd    ymm6, ymm6, yword [rcx - 1088]
+	QUAD $0xfffffba0a958d5c5                   // vaddpd    ymm5, ymm5, yword [rcx - 1120]
+	QUAD $0xfffffb80a158ddc5                   // vaddpd    ymm4, ymm4, yword [rcx - 1152]
+	QUAD $0xfffffb609958e5c5                   // vaddpd    ymm3, ymm3, yword [rcx - 1184]
+	QUAD $0xfffffb409158edc5                   // vaddpd    ymm2, ymm2, yword [rcx - 1216]
+	QUAD $0xfffffb208958f5c5                   // vaddpd    ymm1, ymm1, yword [rcx - 1248]
+	QUAD $0xfffffb008158fdc5                   // vaddpd    ymm0, ymm0, yword [rcx - 1280]
+	QUAD $0xfffffc008158fdc5                   // vaddpd    ymm0, ymm0, yword [rcx - 1024]
+	QUAD $0xfffffc208958f5c5                   // vaddpd    ymm1, ymm1, yword [rcx - 992]
+	QUAD $0xfffffc409158edc5                   // vaddpd    ymm2, ymm2, yword [rcx - 960]
+	QUAD $0xfffffc609958e5c5                   // vaddpd    ymm3, ymm3, yword [rcx - 928]
+	QUAD $0xfffffc80a158ddc5                   // vaddpd    ymm4, ymm4, yword [rcx - 896]
+	QUAD $0xfffffca0a958d5c5                   // vaddpd    ymm5, ymm5, yword [rcx - 864]
+	QUAD $0xfffffcc0b158cdc5                   // vaddpd    ymm6, ymm6, yword [rcx - 832]
+	QUAD $0xfffffce0b958c5c5                   // vaddpd    ymm7, ymm7, yword [rcx - 800]
+	QUAD $0xfffffde0b958c5c5                   // vaddpd    ymm7, ymm7, yword [rcx - 544]
+	QUAD $0xfffffdc0b158cdc5                   // vaddpd    ymm6, ymm6, yword [rcx - 576]
+	QUAD $0xfffffda0a958d5c5                   // vaddpd    ymm5, ymm5, yword [rcx - 608]
+	QUAD $0xfffffd80a158ddc5                   // vaddpd    ymm4, ymm4, yword [rcx - 640]
+	QUAD $0xfffffd609958e5c5                   // vaddpd    ymm3, ymm3, yword [rcx - 672]
+	QUAD $0xfffffd409158edc5                   // vaddpd    ymm2, ymm2, yword [rcx - 704]
+	QUAD $0xfffffd208958f5c5                   // vaddpd    ymm1, ymm1, yword [rcx - 736]
+	QUAD $0xfffffd008158fdc5                   // vaddpd    ymm0, ymm0, yword [rcx - 768]
+	QUAD $0xfffffe008158fdc5                   // vaddpd    ymm0, ymm0, yword [rcx - 512]
+	QUAD $0xfffffe208958f5c5                   // vaddpd    ymm1, ymm1, yword [rcx - 480]
+	QUAD $0xfffffe409158edc5                   // vaddpd    ymm2, ymm2, yword [rcx - 448]
+	QUAD $0xfffffe609958e5c5                   // vaddpd    ymm3, ymm3, yword [rcx - 416]
+	QUAD $0xfffffe80a158ddc5                   // vaddpd    ymm4, ymm4, yword [rcx - 384]
+	QUAD $0xfffffea0a958d5c5                   // vaddpd    ymm5, ymm5, yword [rcx - 352]
+	QUAD $0xfffffec0b158cdc5                   // vaddpd    ymm6, ymm6, yword [rcx - 320]
+	QUAD $0xfffffee0b958c5c5                   // vaddpd    ymm7, ymm7, yword [rcx - 288]
+	LONG $0x7958c5c5; BYTE $0xe0               // vaddpd    ymm7, ymm7, yword [rcx - 32]
+	LONG $0x7158cdc5; BYTE $0xc0               // vaddpd    ymm6, ymm6, yword [rcx - 64]
+	LONG $0x6958d5c5; BYTE $0xa0               // vaddpd    ymm5, ymm5, yword [rcx - 96]
+	LONG $0x6158ddc5; BYTE $0x80               // vaddpd    ymm4, ymm4, yword [rcx - 128]
+	QUAD $0xffffff609958e5c5                   // vaddpd    ymm3, ymm3, yword [rcx - 160]
+	QUAD $0xffffff409158edc5                   // vaddpd    ymm2, ymm2, yword [rcx - 192]
+	QUAD $0xffffff208958f5c5                   // vaddpd    ymm1, ymm1, yword [rcx - 224]
+	QUAD $0xffffff008158fdc5                   // vaddpd    ymm0, ymm0, yword [rcx - 256]
+	LONG $0x0158fdc5                           // vaddpd    ymm0, ymm0, yword [rcx]
+	LONG $0x4958f5c5; BYTE $0x20               // vaddpd    ymm1, ymm1, yword [rcx + 32]
+	LONG $0x5158edc5; BYTE $0x40               // vaddpd    ymm2, ymm2, yword [rcx + 64]
+	LONG $0x5958e5c5; BYTE $0x60               // vaddpd    ymm3, ymm3, yword [rcx + 96]
+	QUAD $0x00000080a158ddc5                   // vaddpd    ymm4, ymm4, yword [rcx + 128]
+	QUAD $0x000000a0a958d5c5                   // vaddpd    ymm5, ymm5, yword [rcx + 160]
+	QUAD $0x000000c0b158cdc5                   // vaddpd    ymm6, ymm6, yword [rcx + 192]
+	QUAD $0x000000e0b958c5c5                   // vaddpd    ymm7, ymm7, yword [rcx + 224]
+	LONG $0x00c18148; WORD $0x0008; BYTE $0x00 // add    rcx, 2048
+	LONG $0xff000548; WORD $0xffff             // add    rax, -256
+	JNE  LBB0_12
+
+LBB0_13:
+	LONG $0xcd58f5c5               // vaddpd    ymm1, ymm1, ymm5
+	LONG $0xdf58e5c5               // vaddpd    ymm3, ymm3, ymm7
+	LONG $0xc458fdc5               // vaddpd    ymm0, ymm0, ymm4
+	LONG $0xd658edc5               // vaddpd    ymm2, ymm2, ymm6
+	LONG $0xc258fdc5               // vaddpd    ymm0, ymm0, ymm2
+	LONG $0xcb58f5c5               // vaddpd    ymm1, ymm1, ymm3
+	LONG $0xc158fdc5               // vaddpd    ymm0, ymm0, ymm1
+	LONG $0x197de3c4; WORD $0x01c1 // vextractf128    xmm1, ymm0, 1
+	LONG $0xc158fdc5               // vaddpd    ymm0, ymm0, ymm1
+	LONG $0xc07cfdc5               // vhaddpd    ymm0, ymm0, ymm0
+	WORD $0x3949; BYTE $0xf1       // cmp    r9, rsi
+	JNE  LBB0_3
+	JMP  LBB0_14
diff --git a/go/arrow/math/float64_noasm.go b/go/arrow/math/float64_noasm.go
new file mode 100644
index 0000000000000..b480ad34f3c4d
--- /dev/null
+++ b/go/arrow/math/float64_noasm.go
@@ -0,0 +1,25 @@
+// Code generated by type_noasm.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build noasm
+
+package math
+
+func initFloat64Go() {
+	Float64.sum = sum_float64_go
+}
diff --git a/go/arrow/math/float64_sse4_amd64.go b/go/arrow/math/float64_sse4_amd64.go
new file mode 100644
index 0000000000000..30a79f48a415a
--- /dev/null
+++ b/go/arrow/math/float64_sse4_amd64.go
@@ -0,0 +1,41 @@
+// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+import (
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+//go:noescape
+func _sum_float64_sse4(buf, len, res unsafe.Pointer)
+
+func sum_float64_sse4(a *array.Float64) float64 {
+	buf := a.Float64Values()
+	var (
+		p1  = unsafe.Pointer(&buf[0])
+		p2  = unsafe.Pointer(uintptr(len(buf)))
+		res float64
+	)
+	_sum_float64_sse4(p1, p2, unsafe.Pointer(&res))
+	return res
+}
diff --git a/go/arrow/math/float64_sse4_amd64.s b/go/arrow/math/float64_sse4_amd64.s
new file mode 100644
index 0000000000000..957efa427cad2
--- /dev/null
+++ b/go/arrow/math/float64_sse4_amd64.s
@@ -0,0 +1,94 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_sum_float64_sse4(SB), $0-24
+
+	MOVQ buf+0(FP), DI
+	MOVQ len+8(FP), SI
+	MOVQ res+16(FP), DX
+
+	LONG $0xc0570f66         // xorpd    xmm0, xmm0
+	WORD $0x8548; BYTE $0xf6 // test    rsi, rsi
+	JE   LBB0_14
+	LONG $0x03fe8348         // cmp    rsi, 3
+	JBE  LBB0_2
+	WORD $0x8949; BYTE $0xf1 // mov    r9, rsi
+	LONG $0xfce18349         // and    r9, -4
+	JE   LBB0_2
+	LONG $0xfc418d4d         // lea    r8, [r9 - 4]
+	WORD $0x8944; BYTE $0xc0 // mov    eax, r8d
+	WORD $0xe8c1; BYTE $0x02 // shr    eax, 2
+	WORD $0xc0ff             // inc    eax
+	LONG $0x03e08348         // and    rax, 3
+	JE   LBB0_7
+	WORD $0xf748; BYTE $0xd8 // neg    rax
+	LONG $0xc0570f66         // xorpd    xmm0, xmm0
+	WORD $0xc931             // xor    ecx, ecx
+	LONG $0xc9570f66         // xorpd    xmm1, xmm1
+
+LBB0_9:
+	LONG $0x14100f66; BYTE $0xcf   // movupd    xmm2, oword [rdi + 8*rcx]
+	LONG $0x5c100f66; WORD $0x10cf // movupd    xmm3, oword [rdi + 8*rcx + 16]
+	LONG $0xc2580f66               // addpd    xmm0, xmm2
+	LONG $0xcb580f66               // addpd    xmm1, xmm3
+	LONG $0x04c18348               // add    rcx, 4
+	WORD $0xff48; BYTE $0xc0       // inc    rax
+	JNE  LBB0_9
+	JMP  LBB0_10
+
+LBB0_2:
+	WORD $0x3145; BYTE $0xc9 // xor    r9d, r9d
+
+LBB0_3:
+	LONG $0xcf048d4a         // lea    rax, [rdi + 8*r9]
+	WORD $0x294c; BYTE $0xce // sub    rsi, r9
+
+LBB0_4:
+	LONG $0x00580ff2         // addsd    xmm0, qword [rax]
+	LONG $0x08c08348         // add    rax, 8
+	WORD $0xff48; BYTE $0xce // dec    rsi
+	JNE  LBB0_4
+
+LBB0_14:
+	LONG $0x02110ff2 // movsd    qword [rdx], xmm0
+	RET
+
+LBB0_7:
+	WORD $0xc931     // xor    ecx, ecx
+	LONG $0xc0570f66 // xorpd    xmm0, xmm0
+	LONG $0xc9570f66 // xorpd    xmm1, xmm1
+
+LBB0_10:
+	LONG $0x0cf88349             // cmp    r8, 12
+	JB   LBB0_13
+	WORD $0x894c; BYTE $0xc8     // mov    rax, r9
+	WORD $0x2948; BYTE $0xc8     // sub    rax, rcx
+	LONG $0xcf4c8d48; BYTE $0x70 // lea    rcx, [rdi + 8*rcx + 112]
+
+LBB0_12:
+	LONG $0x51100f66; BYTE $0x90 // movupd    xmm2, oword [rcx - 112]
+	LONG $0x59100f66; BYTE $0xa0 // movupd    xmm3, oword [rcx - 96]
+	LONG $0x61100f66; BYTE $0xb0 // movupd    xmm4, oword [rcx - 80]
+	LONG $0x69100f66; BYTE $0xc0 // movupd    xmm5, oword [rcx - 64]
+	LONG $0xd0580f66             // addpd    xmm2, xmm0
+	LONG $0xd9580f66             // addpd    xmm3, xmm1
+	LONG $0x71100f66; BYTE $0xd0 // movupd    xmm6, oword [rcx - 48]
+	LONG $0x79100f66; BYTE $0xe0 // movupd    xmm7, oword [rcx - 32]
+	LONG $0xf4580f66             // addpd    xmm6, xmm4
+	LONG $0xf2580f66             // addpd    xmm6, xmm2
+	LONG $0xfd580f66             // addpd    xmm7, xmm5
+	LONG $0xfb580f66             // addpd    xmm7, xmm3
+	LONG $0x41100f66; BYTE $0xf0 // movupd    xmm0, oword [rcx - 16]
+	LONG $0x09100f66             // movupd    xmm1, oword [rcx]
+	LONG $0xc6580f66             // addpd    xmm0, xmm6
+	LONG $0xcf580f66             // addpd    xmm1, xmm7
+	LONG $0x80e98348             // sub    rcx, -128
+	LONG $0xf0c08348             // add    rax, -16
+	JNE  LBB0_12
+
+LBB0_13:
+	LONG $0xc1580f66         // addpd    xmm0, xmm1
+	LONG $0xc07c0f66         // haddpd    xmm0, xmm0
+	WORD $0x3949; BYTE $0xf1 // cmp    r9, rsi
+	JNE  LBB0_3
+	JMP  LBB0_14
diff --git a/go/arrow/math/float64_test.go b/go/arrow/math/float64_test.go
new file mode 100644
index 0000000000000..ad33e0f77e01a
--- /dev/null
+++ b/go/arrow/math/float64_test.go
@@ -0,0 +1,68 @@
+// Code generated by type_test.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/math"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFloat64Funcs_Sum(t *testing.T) {
+	vec := makeArrayFloat64(10000)
+	res := math.Float64.Sum(vec)
+	assert.Equal(t, res, float64(49995000))
+}
+
+func makeArrayFloat64(l int) *array.Float64 {
+	fb := array.NewFloat64Builder(memory.NewGoAllocator())
+	fb.Reserve(l)
+	for i := 0; i < l; i++ {
+		fb.Append(float64(i))
+	}
+	return fb.NewFloat64Array()
+}
+
+func benchmarkFloat64Funcs_Sum(b *testing.B, n int) {
+	vec := makeArrayFloat64(n)
+	b.SetBytes(int64(vec.Len() * 8))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		math.Float64.Sum(vec)
+	}
+}
+
+func BenchmarkFloat64Funcs_Sum_256(b *testing.B) {
+	benchmarkFloat64Funcs_Sum(b, 256)
+}
+
+func BenchmarkFloat64Funcs_Sum_1024(b *testing.B) {
+	benchmarkFloat64Funcs_Sum(b, 1024)
+}
+
+func BenchmarkFloat64Funcs_Sum_8192(b *testing.B) {
+	benchmarkFloat64Funcs_Sum(b, 8192)
+}
+
+func BenchmarkFloat64Funcs_Sum_1000000(b *testing.B) {
+	benchmarkFloat64Funcs_Sum(b, 1e6)
+}
diff --git a/go/arrow/math/int64.go b/go/arrow/math/int64.go
new file mode 100644
index 0000000000000..4f70d2e32c4c8
--- /dev/null
+++ b/go/arrow/math/int64.go
@@ -0,0 +1,44 @@
+// Code generated by type.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math
+
+import (
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+type Int64Funcs struct {
+	sum func(a *array.Int64) int64
+}
+
+var (
+	Int64 Int64Funcs
+)
+
+// Sum returns the summation of all elements in a.
+func (f Int64Funcs) Sum(a *array.Int64) int64 {
+	return f.sum(a)
+}
+
+func sum_int64_go(a *array.Int64) int64 {
+	acc := int64(0)
+	for _, v := range a.Int64Values() {
+		acc += v
+	}
+	return acc
+}
diff --git a/go/arrow/math/int64.tmpldata b/go/arrow/math/int64.tmpldata
new file mode 100644
index 0000000000000..94669f9d3897e
--- /dev/null
+++ b/go/arrow/math/int64.tmpldata
@@ -0,0 +1,4 @@
+{
+  "Name": "Int64",
+  "Type": "int64"
+}
\ No newline at end of file
diff --git a/go/arrow/math/int64_amd64.go b/go/arrow/math/int64_amd64.go
new file mode 100644
index 0000000000000..2703bebd00a99
--- /dev/null
+++ b/go/arrow/math/int64_amd64.go
@@ -0,0 +1,33 @@
+// Code generated by type_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+func initInt64AVX2() {
+	Int64.sum = sum_int64_avx2
+}
+
+func initInt64SSE4() {
+	Int64.sum = sum_int64_sse4
+}
+
+func initInt64Go() {
+	Int64.sum = sum_int64_go
+}
diff --git a/go/arrow/math/int64_avx2_amd64.go b/go/arrow/math/int64_avx2_amd64.go
new file mode 100644
index 0000000000000..55e03cd631187
--- /dev/null
+++ b/go/arrow/math/int64_avx2_amd64.go
@@ -0,0 +1,41 @@
+// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+import (
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+//go:noescape
+func _sum_int64_avx2(buf, len, res unsafe.Pointer)
+
+func sum_int64_avx2(a *array.Int64) int64 {
+	buf := a.Int64Values()
+	var (
+		p1  = unsafe.Pointer(&buf[0])
+		p2  = unsafe.Pointer(uintptr(len(buf)))
+		res int64
+	)
+	_sum_int64_avx2(p1, p2, unsafe.Pointer(&res))
+	return res
+}
diff --git a/go/arrow/math/int64_avx2_amd64.s b/go/arrow/math/int64_avx2_amd64.s
new file mode 100644
index 0000000000000..10d234cc4ac73
--- /dev/null
+++ b/go/arrow/math/int64_avx2_amd64.s
@@ -0,0 +1,173 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_sum_int64_avx2(SB), $0-24
+
+	MOVQ buf+0(FP), DI
+	MOVQ len+8(FP), SI
+	MOVQ res+16(FP), DX
+
+	WORD $0x8548; BYTE $0xf6 // test    rsi, rsi
+	JE   LBB0_1
+	LONG $0x1ffe8348         // cmp    rsi, 31
+	JBE  LBB0_3
+	WORD $0x8949; BYTE $0xf1 // mov    r9, rsi
+	LONG $0xe0e18349         // and    r9, -32
+	JE   LBB0_3
+	LONG $0xe0418d4d         // lea    r8, [r9 - 32]
+	WORD $0x8944; BYTE $0xc0 // mov    eax, r8d
+	WORD $0xe8c1; BYTE $0x05 // shr    eax, 5
+	WORD $0xc0ff             // inc    eax
+	LONG $0x07e08348         // and    rax, 7
+	JE   LBB0_8
+	WORD $0xf748; BYTE $0xd8 // neg    rax
+	LONG $0xc0effdc5         // vpxor    ymm0, ymm0, ymm0
+	WORD $0xc931             // xor    ecx, ecx
+	LONG $0xc9eff5c5         // vpxor    ymm1, ymm1, ymm1
+	LONG $0xd2efedc5         // vpxor    ymm2, ymm2, ymm2
+	LONG $0xdbefe5c5         // vpxor    ymm3, ymm3, ymm3
+	LONG $0xe4efddc5         // vpxor    ymm4, ymm4, ymm4
+	LONG $0xedefd5c5         // vpxor    ymm5, ymm5, ymm5
+	LONG $0xf6efcdc5         // vpxor    ymm6, ymm6, ymm6
+	LONG $0xffefc5c5         // vpxor    ymm7, ymm7, ymm7
+
+LBB0_10:
+	LONG $0x04d4fdc5; BYTE $0xcf         // vpaddq    ymm0, ymm0, yword [rdi + 8*rcx]
+	LONG $0x4cd4f5c5; WORD $0x20cf       // vpaddq    ymm1, ymm1, yword [rdi + 8*rcx + 32]
+	LONG $0x54d4edc5; WORD $0x40cf       // vpaddq    ymm2, ymm2, yword [rdi + 8*rcx + 64]
+	LONG $0x5cd4e5c5; WORD $0x60cf       // vpaddq    ymm3, ymm3, yword [rdi + 8*rcx + 96]
+	QUAD $0x000080cfa4d4ddc5; BYTE $0x00 // vpaddq    ymm4, ymm4, yword [rdi + 8*rcx + 128]
+	QUAD $0x0000a0cfacd4d5c5; BYTE $0x00 // vpaddq    ymm5, ymm5, yword [rdi + 8*rcx + 160]
+	QUAD $0x0000c0cfb4d4cdc5; BYTE $0x00 // vpaddq    ymm6, ymm6, yword [rdi + 8*rcx + 192]
+	QUAD $0x0000e0cfbcd4c5c5; BYTE $0x00 // vpaddq    ymm7, ymm7, yword [rdi + 8*rcx + 224]
+	LONG $0x20c18348                     // add    rcx, 32
+	WORD $0xff48; BYTE $0xc0             // inc    rax
+	JNE  LBB0_10
+	JMP  LBB0_11
+
+LBB0_3:
+	WORD $0x3145; BYTE $0xc9 // xor    r9d, r9d
+	WORD $0xc031             // xor    eax, eax
+
+LBB0_4:
+	LONG $0xcf0c8d4a         // lea    rcx, [rdi + 8*r9]
+	WORD $0x294c; BYTE $0xce // sub    rsi, r9
+
+LBB0_5:
+	WORD $0x0348; BYTE $0x01 // add    rax, qword [rcx]
+	LONG $0x08c18348         // add    rcx, 8
+	WORD $0xff48; BYTE $0xce // dec    rsi
+	JNE  LBB0_5
+	JMP  LBB0_15
+
+LBB0_1:
+	WORD $0xc031 // xor    eax, eax
+
+LBB0_15:
+	WORD $0x8948; BYTE $0x02 // mov    qword [rdx], rax
+	VZEROUPPER
+	RET
+
+LBB0_8:
+	WORD $0xc931     // xor    ecx, ecx
+	LONG $0xc0effdc5 // vpxor    ymm0, ymm0, ymm0
+	LONG $0xc9eff5c5 // vpxor    ymm1, ymm1, ymm1
+	LONG $0xd2efedc5 // vpxor    ymm2, ymm2, ymm2
+	LONG $0xdbefe5c5 // vpxor    ymm3, ymm3, ymm3
+	LONG $0xe4efddc5 // vpxor    ymm4, ymm4, ymm4
+	LONG $0xedefd5c5 // vpxor    ymm5, ymm5, ymm5
+	LONG $0xf6efcdc5 // vpxor    ymm6, ymm6, ymm6
+	LONG $0xffefc5c5 // vpxor    ymm7, ymm7, ymm7
+
+LBB0_11:
+	LONG $0xe0f88149; WORD $0x0000; BYTE $0x00 // cmp    r8, 224
+	JB   LBB0_14
+	WORD $0x894c; BYTE $0xc8                   // mov    rax, r9
+	WORD $0x2948; BYTE $0xc8                   // sub    rax, rcx
+	QUAD $0x00000700cf8c8d48                   // lea    rcx, [rdi + 8*rcx + 1792]
+
+LBB0_13:
+	QUAD $0xfffff9e0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 1568]
+	QUAD $0xfffff9c0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 1600]
+	QUAD $0xfffff9a0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 1632]
+	QUAD $0xfffff980a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 1664]
+	QUAD $0xfffff96099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 1696]
+	QUAD $0xfffff94091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 1728]
+	QUAD $0xfffff92089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 1760]
+	QUAD $0xfffff90081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 1792]
+	QUAD $0xfffffa0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 1536]
+	QUAD $0xfffffa2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 1504]
+	QUAD $0xfffffa4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 1472]
+	QUAD $0xfffffa6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 1440]
+	QUAD $0xfffffa80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 1408]
+	QUAD $0xfffffaa0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 1376]
+	QUAD $0xfffffac0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 1344]
+	QUAD $0xfffffae0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 1312]
+	QUAD $0xfffffbe0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 1056]
+	QUAD $0xfffffbc0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 1088]
+	QUAD $0xfffffba0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 1120]
+	QUAD $0xfffffb80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 1152]
+	QUAD $0xfffffb6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 1184]
+	QUAD $0xfffffb4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 1216]
+	QUAD $0xfffffb2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 1248]
+	QUAD $0xfffffb0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 1280]
+	QUAD $0xfffffc0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 1024]
+	QUAD $0xfffffc2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 992]
+	QUAD $0xfffffc4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 960]
+	QUAD $0xfffffc6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 928]
+	QUAD $0xfffffc80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 896]
+	QUAD $0xfffffca0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 864]
+	QUAD $0xfffffcc0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 832]
+	QUAD $0xfffffce0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 800]
+	QUAD $0xfffffde0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 544]
+	QUAD $0xfffffdc0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 576]
+	QUAD $0xfffffda0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 608]
+	QUAD $0xfffffd80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 640]
+	QUAD $0xfffffd6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 672]
+	QUAD $0xfffffd4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 704]
+	QUAD $0xfffffd2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 736]
+	QUAD $0xfffffd0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 768]
+	QUAD $0xfffffe0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 512]
+	QUAD $0xfffffe2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 480]
+	QUAD $0xfffffe4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 448]
+	QUAD $0xfffffe6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 416]
+	QUAD $0xfffffe80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 384]
+	QUAD $0xfffffea0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 352]
+	QUAD $0xfffffec0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 320]
+	QUAD $0xfffffee0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 288]
+	LONG $0x79d4c5c5; BYTE $0xe0               // vpaddq    ymm7, ymm7, yword [rcx - 32]
+	LONG $0x71d4cdc5; BYTE $0xc0               // vpaddq    ymm6, ymm6, yword [rcx - 64]
+	LONG $0x69d4d5c5; BYTE $0xa0               // vpaddq    ymm5, ymm5, yword [rcx - 96]
+	LONG $0x61d4ddc5; BYTE $0x80               // vpaddq    ymm4, ymm4, yword [rcx - 128]
+	QUAD $0xffffff6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 160]
+	QUAD $0xffffff4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 192]
+	QUAD $0xffffff2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 224]
+	QUAD $0xffffff0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 256]
+	LONG $0x01d4fdc5                           // vpaddq    ymm0, ymm0, yword [rcx]
+	LONG $0x49d4f5c5; BYTE $0x20               // vpaddq    ymm1, ymm1, yword [rcx + 32]
+	LONG $0x51d4edc5; BYTE $0x40               // vpaddq    ymm2, ymm2, yword [rcx + 64]
+	LONG $0x59d4e5c5; BYTE $0x60               // vpaddq    ymm3, ymm3, yword [rcx + 96]
+	QUAD $0x00000080a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx + 128]
+	QUAD $0x000000a0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx + 160]
+	QUAD $0x000000c0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx + 192]
+	QUAD $0x000000e0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx + 224]
+	LONG $0x00c18148; WORD $0x0008; BYTE $0x00 // add    rcx, 2048
+	LONG $0xff000548; WORD $0xffff             // add    rax, -256
+	JNE  LBB0_13
+
+LBB0_14:
+	LONG $0xcdd4f5c5               // vpaddq    ymm1, ymm1, ymm5
+	LONG $0xdfd4e5c5               // vpaddq    ymm3, ymm3, ymm7
+	LONG $0xc4d4fdc5               // vpaddq    ymm0, ymm0, ymm4
+	LONG $0xd6d4edc5               // vpaddq    ymm2, ymm2, ymm6
+	LONG $0xc2d4fdc5               // vpaddq    ymm0, ymm0, ymm2
+	LONG $0xcbd4f5c5               // vpaddq    ymm1, ymm1, ymm3
+	LONG $0xc1d4fdc5               // vpaddq    ymm0, ymm0, ymm1
+	LONG $0x397de3c4; WORD $0x01c1 // vextracti128    xmm1, ymm0, 1
+	LONG $0xc1d4fdc5               // vpaddq    ymm0, ymm0, ymm1
+	LONG $0xc870f9c5; BYTE $0x4e   // vpshufd    xmm1, xmm0, 78
+	LONG $0xc1d4fdc5               // vpaddq    ymm0, ymm0, ymm1
+	LONG $0x7ef9e1c4; BYTE $0xc0   // vmovq    rax, xmm0
+	WORD $0x3949; BYTE $0xf1       // cmp    r9, rsi
+	JNE  LBB0_4
+	JMP  LBB0_15
diff --git a/go/arrow/math/int64_noasm.go b/go/arrow/math/int64_noasm.go
new file mode 100644
index 0000000000000..5a5efc31bb9d6
--- /dev/null
+++ b/go/arrow/math/int64_noasm.go
@@ -0,0 +1,25 @@
+// Code generated by type_noasm.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build noasm
+
+package math
+
+func initInt64Go() {
+	Int64.sum = sum_int64_go
+}
diff --git a/go/arrow/math/int64_sse4_amd64.go b/go/arrow/math/int64_sse4_amd64.go
new file mode 100644
index 0000000000000..c13c8de14422e
--- /dev/null
+++ b/go/arrow/math/int64_sse4_amd64.go
@@ -0,0 +1,41 @@
+// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+import (
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+//go:noescape
+func _sum_int64_sse4(buf, len, res unsafe.Pointer)
+
+func sum_int64_sse4(a *array.Int64) int64 {
+	buf := a.Int64Values()
+	var (
+		p1  = unsafe.Pointer(&buf[0])
+		p2  = unsafe.Pointer(uintptr(len(buf)))
+		res int64
+	)
+	_sum_int64_sse4(p1, p2, unsafe.Pointer(&res))
+	return res
+}
diff --git a/go/arrow/math/int64_sse4_amd64.s b/go/arrow/math/int64_sse4_amd64.s
new file mode 100644
index 0000000000000..ef27eee1649f1
--- /dev/null
+++ b/go/arrow/math/int64_sse4_amd64.s
@@ -0,0 +1,100 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_sum_int64_sse4(SB), $0-24
+
+	MOVQ buf+0(FP), DI
+	MOVQ len+8(FP), SI
+	MOVQ res+16(FP), DX
+
+	WORD $0x8548; BYTE $0xf6 // test    rsi, rsi
+	JE   LBB0_1
+	LONG $0x03fe8348         // cmp    rsi, 3
+	JBE  LBB0_3
+	WORD $0x8949; BYTE $0xf1 // mov    r9, rsi
+	LONG $0xfce18349         // and    r9, -4
+	JE   LBB0_3
+	LONG $0xfc418d4d         // lea    r8, [r9 - 4]
+	WORD $0x8944; BYTE $0xc0 // mov    eax, r8d
+	WORD $0xe8c1; BYTE $0x02 // shr    eax, 2
+	WORD $0xc0ff             // inc    eax
+	LONG $0x03e08348         // and    rax, 3
+	JE   LBB0_8
+	WORD $0xf748; BYTE $0xd8 // neg    rax
+	LONG $0xc0ef0f66         // pxor    xmm0, xmm0
+	WORD $0xc931             // xor    ecx, ecx
+	LONG $0xc9ef0f66         // pxor    xmm1, xmm1
+
+LBB0_10:
+	LONG $0x146f0ff3; BYTE $0xcf   // movdqu    xmm2, oword [rdi + 8*rcx]
+	LONG $0x5c6f0ff3; WORD $0x10cf // movdqu    xmm3, oword [rdi + 8*rcx + 16]
+	LONG $0xc2d40f66               // paddq    xmm0, xmm2
+	LONG $0xcbd40f66               // paddq    xmm1, xmm3
+	LONG $0x04c18348               // add    rcx, 4
+	WORD $0xff48; BYTE $0xc0       // inc    rax
+	JNE  LBB0_10
+	JMP  LBB0_11
+
+LBB0_3:
+	WORD $0x3145; BYTE $0xc9 // xor    r9d, r9d
+	WORD $0xc031             // xor    eax, eax
+
+LBB0_4:
+	LONG $0xcf0c8d4a         // lea    rcx, [rdi + 8*r9]
+	WORD $0x294c; BYTE $0xce // sub    rsi, r9
+
+LBB0_5:
+	WORD $0x0348; BYTE $0x01 // add    rax, qword [rcx]
+	LONG $0x08c18348         // add    rcx, 8
+	WORD $0xff48; BYTE $0xce // dec    rsi
+	JNE  LBB0_5
+	JMP  LBB0_15
+
+LBB0_1:
+	WORD $0xc031 // xor    eax, eax
+
+LBB0_15:
+	WORD $0x8948; BYTE $0x02 // mov    qword [rdx], rax
+	RET
+
+LBB0_8:
+	WORD $0xc931     // xor    ecx, ecx
+	LONG $0xc0ef0f66 // pxor    xmm0, xmm0
+	LONG $0xc9ef0f66 // pxor    xmm1, xmm1
+
+LBB0_11:
+	LONG $0x0cf88349             // cmp    r8, 12
+	JB   LBB0_14
+	WORD $0x894c; BYTE $0xc8     // mov    rax, r9
+	WORD $0x2948; BYTE $0xc8     // sub    rax, rcx
+	LONG $0xcf4c8d48; BYTE $0x70 // lea    rcx, [rdi + 8*rcx + 112]
+
+LBB0_13:
+	LONG $0x516f0ff3; BYTE $0x90 // movdqu    xmm2, oword [rcx - 112]
+	LONG $0x596f0ff3; BYTE $0xa0 // movdqu    xmm3, oword [rcx - 96]
+	LONG $0x616f0ff3; BYTE $0xb0 // movdqu    xmm4, oword [rcx - 80]
+	LONG $0x696f0ff3; BYTE $0xc0 // movdqu    xmm5, oword [rcx - 64]
+	LONG $0xd0d40f66             // paddq    xmm2, xmm0
+	LONG $0xd9d40f66             // paddq    xmm3, xmm1
+	LONG $0x716f0ff3; BYTE $0xd0 // movdqu    xmm6, oword [rcx - 48]
+	LONG $0x796f0ff3; BYTE $0xe0 // movdqu    xmm7, oword [rcx - 32]
+	LONG $0xf4d40f66             // paddq    xmm6, xmm4
+	LONG $0xf2d40f66             // paddq    xmm6, xmm2
+	LONG $0xfdd40f66             // paddq    xmm7, xmm5
+	LONG $0xfbd40f66             // paddq    xmm7, xmm3
+	LONG $0x416f0ff3; BYTE $0xf0 // movdqu    xmm0, oword [rcx - 16]
+	LONG $0x096f0ff3             // movdqu    xmm1, oword [rcx]
+	LONG $0xc6d40f66             // paddq    xmm0, xmm6
+	LONG $0xcfd40f66             // paddq    xmm1, xmm7
+	LONG $0x80e98348             // sub    rcx, -128
+	LONG $0xf0c08348             // add    rax, -16
+	JNE  LBB0_13
+
+LBB0_14:
+	LONG $0xc1d40f66             // paddq    xmm0, xmm1
+	LONG $0xc8700f66; BYTE $0x4e // pshufd    xmm1, xmm0, 78
+	LONG $0xc8d40f66             // paddq    xmm1, xmm0
+	LONG $0x7e0f4866; BYTE $0xc8 // movq    rax, xmm1
+	WORD $0x3949; BYTE $0xf1     // cmp    r9, rsi
+	JNE  LBB0_4
+	JMP  LBB0_15
diff --git a/go/arrow/math/int64_test.go b/go/arrow/math/int64_test.go
new file mode 100644
index 0000000000000..19722828ef237
--- /dev/null
+++ b/go/arrow/math/int64_test.go
@@ -0,0 +1,68 @@
+// Code generated by type_test.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/math"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInt64Funcs_Sum(t *testing.T) {
+	vec := makeArrayInt64(10000)
+	res := math.Int64.Sum(vec)
+	assert.Equal(t, res, int64(49995000))
+}
+
+func makeArrayInt64(l int) *array.Int64 {
+	fb := array.NewInt64Builder(memory.NewGoAllocator())
+	fb.Reserve(l)
+	for i := 0; i < l; i++ {
+		fb.Append(int64(i))
+	}
+	return fb.NewInt64Array()
+}
+
+func benchmarkInt64Funcs_Sum(b *testing.B, n int) {
+	vec := makeArrayInt64(n)
+	b.SetBytes(int64(vec.Len() * 8))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		math.Int64.Sum(vec)
+	}
+}
+
+func BenchmarkInt64Funcs_Sum_256(b *testing.B) {
+	benchmarkInt64Funcs_Sum(b, 256)
+}
+
+func BenchmarkInt64Funcs_Sum_1024(b *testing.B) {
+	benchmarkInt64Funcs_Sum(b, 1024)
+}
+
+func BenchmarkInt64Funcs_Sum_8192(b *testing.B) {
+	benchmarkInt64Funcs_Sum(b, 8192)
+}
+
+func BenchmarkInt64Funcs_Sum_1000000(b *testing.B) {
+	benchmarkInt64Funcs_Sum(b, 1e6)
+}
diff --git a/go/arrow/math/math_amd64.go b/go/arrow/math/math_amd64.go
new file mode 100644
index 0000000000000..73789de6d0ffe
--- /dev/null
+++ b/go/arrow/math/math_amd64.go
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+import (
+	"github.com/apache/arrow/go/arrow/internal/cpu"
+)
+
+func init() {
+	if cpu.X86.HasAVX2 {
+		initAVX2()
+	} else if cpu.X86.HasSSE42 {
+		initSSE4()
+	} else {
+		initGo()
+	}
+}
+
+func initAVX2() {
+	initFloat64AVX2()
+	initInt64AVX2()
+	initUint64AVX2()
+}
+
+func initSSE4() {
+	initFloat64SSE4()
+	initInt64SSE4()
+	initUint64SSE4()
+}
+
+func initGo() {
+	initFloat64Go()
+	initInt64Go()
+	initUint64Go()
+}
diff --git a/go/arrow/math/math_noasm.go b/go/arrow/math/math_noasm.go
new file mode 100644
index 0000000000000..0fa924d90aa88
--- /dev/null
+++ b/go/arrow/math/math_noasm.go
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build noasm
+
+package math
+
+func init() {
+	initGo()
+}
+
+func initGo() {
+	initFloat64Go()
+	initInt64Go()
+	initUint64Go()
+}
diff --git a/go/arrow/math/type.go.tmpl b/go/arrow/math/type.go.tmpl
new file mode 100644
index 0000000000000..1b2dd6966cfcd
--- /dev/null
+++ b/go/arrow/math/type.go.tmpl
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math
+
+import (
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+{{$def := .D}}
+{{with .In}}
+type {{.Name}}Funcs struct {
+	sum func(a *array.{{.Name}}) {{.Type}}
+}
+
+var (
+	{{.Name}} {{.Name}}Funcs
+)
+
+// Sum returns the summation of all elements in a.
+func (f {{.Name}}Funcs) Sum(a *array.{{.Name}}) {{.Type}} {
+	return f.sum(a)
+}
+
+func sum_{{.Type}}_go(a *array.{{.Name}}) {{.Type}} {
+	acc := {{.Type}}(0)
+	for _, v := range a.{{.Name}}Values() {
+		acc += v
+	}
+	return acc
+}
+{{end}}
\ No newline at end of file
diff --git a/go/arrow/math/type_amd64.go.tmpl b/go/arrow/math/type_amd64.go.tmpl
new file mode 100644
index 0000000000000..c3c27498be5dc
--- /dev/null
+++ b/go/arrow/math/type_amd64.go.tmpl
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+{{with .In}}
+func init{{.Name}}AVX2() {
+	{{.Name}}.sum = sum_{{.Type}}_avx2
+}
+
+func init{{.Name}}SSE4() {
+	{{.Name}}.sum = sum_{{.Type}}_sse4
+}
+
+func init{{.Name}}Go() {
+	{{.Name}}.sum = sum_{{.Type}}_go
+}
+{{end}}
\ No newline at end of file
diff --git a/go/arrow/math/type_noasm.go.tmpl b/go/arrow/math/type_noasm.go.tmpl
new file mode 100644
index 0000000000000..66802f0fe50ac
--- /dev/null
+++ b/go/arrow/math/type_noasm.go.tmpl
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build noasm
+
+package math
+
+{{with .In}}
+func init{{.Name}}Go() {
+	{{.Name}}.sum = sum_{{.Type}}_go
+}
+{{end}}
\ No newline at end of file
diff --git a/go/arrow/math/type_simd_amd64.go.tmpl b/go/arrow/math/type_simd_amd64.go.tmpl
new file mode 100644
index 0000000000000..b572290cc5b61
--- /dev/null
+++ b/go/arrow/math/type_simd_amd64.go.tmpl
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+import (
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+{{$name := printf "%s_%s" .In.Type .D.arch}}
+{{with .In}}
+//go:noescape
+func _sum_{{$name}}(buf, len, res unsafe.Pointer)
+
+func sum_{{$name}}(a *array.{{.Name}}) {{.Type}} {
+	buf := a.{{.Name}}Values()
+	var (
+		p1  = unsafe.Pointer(&buf[0])
+		p2  = unsafe.Pointer(uintptr(len(buf)))
+		res {{.Type}}
+	)
+	_sum_{{$name}}(p1, p2, unsafe.Pointer(&res))
+	return res
+}
+{{end}}
\ No newline at end of file
diff --git a/go/arrow/math/type_test.go.tmpl b/go/arrow/math/type_test.go.tmpl
new file mode 100644
index 0000000000000..f1b744ca4f607
--- /dev/null
+++ b/go/arrow/math/type_test.go.tmpl
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/math"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+{{$name := printf "%s_%s" .In.Type .D.arch}}
+{{with .In}}
+func Test{{.Name}}Funcs_Sum(t *testing.T) {
+	vec := makeArray{{.Name}}(10000)
+	res := math.{{.Name}}.Sum(vec)
+	assert.Equal(t, res, {{.Type}}(49995000))
+}
+
+func makeArray{{.Name}}(l int) *array.{{.Name}} {
+	fb := array.New{{.Name}}Builder(memory.NewGoAllocator())
+	fb.Reserve(l)
+	for i := 0; i < l; i++ {
+		fb.Append({{.Type}}(i))
+	}
+	return fb.New{{.Name}}Array()
+}
+
+func benchmark{{.Name}}Funcs_Sum(b *testing.B, n int) {
+	vec := makeArray{{.Name}}(n)
+	b.SetBytes(int64(vec.Len() * 8))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		math.{{.Name}}.Sum(vec)
+	}
+}
+
+func Benchmark{{.Name}}Funcs_Sum_256(b *testing.B) {
+	benchmark{{.Name}}Funcs_Sum(b, 256)
+}
+
+func Benchmark{{.Name}}Funcs_Sum_1024(b *testing.B) {
+	benchmark{{.Name}}Funcs_Sum(b, 1024)
+}
+
+func Benchmark{{.Name}}Funcs_Sum_8192(b *testing.B) {
+	benchmark{{.Name}}Funcs_Sum(b, 8192)
+}
+
+func Benchmark{{.Name}}Funcs_Sum_1000000(b *testing.B) {
+	benchmark{{.Name}}Funcs_Sum(b, 1e6)
+}
+{{end}}
\ No newline at end of file
diff --git a/go/arrow/math/uint64.go b/go/arrow/math/uint64.go
new file mode 100644
index 0000000000000..875e82143b9ab
--- /dev/null
+++ b/go/arrow/math/uint64.go
@@ -0,0 +1,44 @@
+// Code generated by type.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math
+
+import (
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+type Uint64Funcs struct {
+	sum func(a *array.Uint64) uint64
+}
+
+var (
+	Uint64 Uint64Funcs
+)
+
+// Sum returns the summation of all elements in a.
+func (f Uint64Funcs) Sum(a *array.Uint64) uint64 {
+	return f.sum(a)
+}
+
+func sum_uint64_go(a *array.Uint64) uint64 {
+	acc := uint64(0)
+	for _, v := range a.Uint64Values() {
+		acc += v
+	}
+	return acc
+}
diff --git a/go/arrow/math/uint64.tmpldata b/go/arrow/math/uint64.tmpldata
new file mode 100644
index 0000000000000..22165eb1d00ed
--- /dev/null
+++ b/go/arrow/math/uint64.tmpldata
@@ -0,0 +1,4 @@
+{
+  "Name": "Uint64",
+  "Type": "uint64"
+}
\ No newline at end of file
diff --git a/go/arrow/math/uint64_amd64.go b/go/arrow/math/uint64_amd64.go
new file mode 100644
index 0000000000000..c6bc29c4e3e32
--- /dev/null
+++ b/go/arrow/math/uint64_amd64.go
@@ -0,0 +1,33 @@
+// Code generated by type_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+func initUint64AVX2() {
+	Uint64.sum = sum_uint64_avx2
+}
+
+func initUint64SSE4() {
+	Uint64.sum = sum_uint64_sse4
+}
+
+func initUint64Go() {
+	Uint64.sum = sum_uint64_go
+}
diff --git a/go/arrow/math/uint64_avx2_amd64.go b/go/arrow/math/uint64_avx2_amd64.go
new file mode 100644
index 0000000000000..ff0500e7c8a35
--- /dev/null
+++ b/go/arrow/math/uint64_avx2_amd64.go
@@ -0,0 +1,41 @@
+// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+import (
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+//go:noescape
+func _sum_uint64_avx2(buf, len, res unsafe.Pointer)
+
+func sum_uint64_avx2(a *array.Uint64) uint64 {
+	buf := a.Uint64Values()
+	var (
+		p1  = unsafe.Pointer(&buf[0])
+		p2  = unsafe.Pointer(uintptr(len(buf)))
+		res uint64
+	)
+	_sum_uint64_avx2(p1, p2, unsafe.Pointer(&res))
+	return res
+}
diff --git a/go/arrow/math/uint64_avx2_amd64.s b/go/arrow/math/uint64_avx2_amd64.s
new file mode 100644
index 0000000000000..7edfd476aac25
--- /dev/null
+++ b/go/arrow/math/uint64_avx2_amd64.s
@@ -0,0 +1,173 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_sum_uint64_avx2(SB), $0-24
+
+	MOVQ buf+0(FP), DI
+	MOVQ len+8(FP), SI
+	MOVQ res+16(FP), DX
+
+	WORD $0x8548; BYTE $0xf6 // test    rsi, rsi
+	JE   LBB0_1
+	LONG $0x1ffe8348         // cmp    rsi, 31
+	JBE  LBB0_3
+	WORD $0x8949; BYTE $0xf1 // mov    r9, rsi
+	LONG $0xe0e18349         // and    r9, -32
+	JE   LBB0_3
+	LONG $0xe0418d4d         // lea    r8, [r9 - 32]
+	WORD $0x8944; BYTE $0xc0 // mov    eax, r8d
+	WORD $0xe8c1; BYTE $0x05 // shr    eax, 5
+	WORD $0xc0ff             // inc    eax
+	LONG $0x07e08348         // and    rax, 7
+	JE   LBB0_8
+	WORD $0xf748; BYTE $0xd8 // neg    rax
+	LONG $0xc0effdc5         // vpxor    ymm0, ymm0, ymm0
+	WORD $0xc931             // xor    ecx, ecx
+	LONG $0xc9eff5c5         // vpxor    ymm1, ymm1, ymm1
+	LONG $0xd2efedc5         // vpxor    ymm2, ymm2, ymm2
+	LONG $0xdbefe5c5         // vpxor    ymm3, ymm3, ymm3
+	LONG $0xe4efddc5         // vpxor    ymm4, ymm4, ymm4
+	LONG $0xedefd5c5         // vpxor    ymm5, ymm5, ymm5
+	LONG $0xf6efcdc5         // vpxor    ymm6, ymm6, ymm6
+	LONG $0xffefc5c5         // vpxor    ymm7, ymm7, ymm7
+
+LBB0_10:
+	LONG $0x04d4fdc5; BYTE $0xcf         // vpaddq    ymm0, ymm0, yword [rdi + 8*rcx]
+	LONG $0x4cd4f5c5; WORD $0x20cf       // vpaddq    ymm1, ymm1, yword [rdi + 8*rcx + 32]
+	LONG $0x54d4edc5; WORD $0x40cf       // vpaddq    ymm2, ymm2, yword [rdi + 8*rcx + 64]
+	LONG $0x5cd4e5c5; WORD $0x60cf       // vpaddq    ymm3, ymm3, yword [rdi + 8*rcx + 96]
+	QUAD $0x000080cfa4d4ddc5; BYTE $0x00 // vpaddq    ymm4, ymm4, yword [rdi + 8*rcx + 128]
+	QUAD $0x0000a0cfacd4d5c5; BYTE $0x00 // vpaddq    ymm5, ymm5, yword [rdi + 8*rcx + 160]
+	QUAD $0x0000c0cfb4d4cdc5; BYTE $0x00 // vpaddq    ymm6, ymm6, yword [rdi + 8*rcx + 192]
+	QUAD $0x0000e0cfbcd4c5c5; BYTE $0x00 // vpaddq    ymm7, ymm7, yword [rdi + 8*rcx + 224]
+	LONG $0x20c18348                     // add    rcx, 32
+	WORD $0xff48; BYTE $0xc0             // inc    rax
+	JNE  LBB0_10
+	JMP  LBB0_11
+
+LBB0_3:
+	WORD $0x3145; BYTE $0xc9 // xor    r9d, r9d
+	WORD $0xc031             // xor    eax, eax
+
+LBB0_4:
+	LONG $0xcf0c8d4a         // lea    rcx, [rdi + 8*r9]
+	WORD $0x294c; BYTE $0xce // sub    rsi, r9
+
+LBB0_5:
+	WORD $0x0348; BYTE $0x01 // add    rax, qword [rcx]
+	LONG $0x08c18348         // add    rcx, 8
+	WORD $0xff48; BYTE $0xce // dec    rsi
+	JNE  LBB0_5
+	JMP  LBB0_15
+
+LBB0_1:
+	WORD $0xc031 // xor    eax, eax
+
+LBB0_15:
+	WORD $0x8948; BYTE $0x02 // mov    qword [rdx], rax
+	VZEROUPPER
+	RET
+
+LBB0_8:
+	WORD $0xc931     // xor    ecx, ecx
+	LONG $0xc0effdc5 // vpxor    ymm0, ymm0, ymm0
+	LONG $0xc9eff5c5 // vpxor    ymm1, ymm1, ymm1
+	LONG $0xd2efedc5 // vpxor    ymm2, ymm2, ymm2
+	LONG $0xdbefe5c5 // vpxor    ymm3, ymm3, ymm3
+	LONG $0xe4efddc5 // vpxor    ymm4, ymm4, ymm4
+	LONG $0xedefd5c5 // vpxor    ymm5, ymm5, ymm5
+	LONG $0xf6efcdc5 // vpxor    ymm6, ymm6, ymm6
+	LONG $0xffefc5c5 // vpxor    ymm7, ymm7, ymm7
+
+LBB0_11:
+	LONG $0xe0f88149; WORD $0x0000; BYTE $0x00 // cmp    r8, 224
+	JB   LBB0_14
+	WORD $0x894c; BYTE $0xc8                   // mov    rax, r9
+	WORD $0x2948; BYTE $0xc8                   // sub    rax, rcx
+	QUAD $0x00000700cf8c8d48                   // lea    rcx, [rdi + 8*rcx + 1792]
+
+LBB0_13:
+	QUAD $0xfffff9e0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 1568]
+	QUAD $0xfffff9c0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 1600]
+	QUAD $0xfffff9a0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 1632]
+	QUAD $0xfffff980a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 1664]
+	QUAD $0xfffff96099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 1696]
+	QUAD $0xfffff94091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 1728]
+	QUAD $0xfffff92089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 1760]
+	QUAD $0xfffff90081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 1792]
+	QUAD $0xfffffa0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 1536]
+	QUAD $0xfffffa2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 1504]
+	QUAD $0xfffffa4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 1472]
+	QUAD $0xfffffa6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 1440]
+	QUAD $0xfffffa80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 1408]
+	QUAD $0xfffffaa0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 1376]
+	QUAD $0xfffffac0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 1344]
+	QUAD $0xfffffae0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 1312]
+	QUAD $0xfffffbe0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 1056]
+	QUAD $0xfffffbc0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 1088]
+	QUAD $0xfffffba0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 1120]
+	QUAD $0xfffffb80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 1152]
+	QUAD $0xfffffb6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 1184]
+	QUAD $0xfffffb4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 1216]
+	QUAD $0xfffffb2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 1248]
+	QUAD $0xfffffb0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 1280]
+	QUAD $0xfffffc0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 1024]
+	QUAD $0xfffffc2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 992]
+	QUAD $0xfffffc4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 960]
+	QUAD $0xfffffc6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 928]
+	QUAD $0xfffffc80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 896]
+	QUAD $0xfffffca0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 864]
+	QUAD $0xfffffcc0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 832]
+	QUAD $0xfffffce0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 800]
+	QUAD $0xfffffde0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 544]
+	QUAD $0xfffffdc0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 576]
+	QUAD $0xfffffda0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 608]
+	QUAD $0xfffffd80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 640]
+	QUAD $0xfffffd6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 672]
+	QUAD $0xfffffd4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 704]
+	QUAD $0xfffffd2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 736]
+	QUAD $0xfffffd0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 768]
+	QUAD $0xfffffe0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 512]
+	QUAD $0xfffffe2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 480]
+	QUAD $0xfffffe4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 448]
+	QUAD $0xfffffe6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 416]
+	QUAD $0xfffffe80a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx - 384]
+	QUAD $0xfffffea0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx - 352]
+	QUAD $0xfffffec0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx - 320]
+	QUAD $0xfffffee0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx - 288]
+	LONG $0x79d4c5c5; BYTE $0xe0               // vpaddq    ymm7, ymm7, yword [rcx - 32]
+	LONG $0x71d4cdc5; BYTE $0xc0               // vpaddq    ymm6, ymm6, yword [rcx - 64]
+	LONG $0x69d4d5c5; BYTE $0xa0               // vpaddq    ymm5, ymm5, yword [rcx - 96]
+	LONG $0x61d4ddc5; BYTE $0x80               // vpaddq    ymm4, ymm4, yword [rcx - 128]
+	QUAD $0xffffff6099d4e5c5                   // vpaddq    ymm3, ymm3, yword [rcx - 160]
+	QUAD $0xffffff4091d4edc5                   // vpaddq    ymm2, ymm2, yword [rcx - 192]
+	QUAD $0xffffff2089d4f5c5                   // vpaddq    ymm1, ymm1, yword [rcx - 224]
+	QUAD $0xffffff0081d4fdc5                   // vpaddq    ymm0, ymm0, yword [rcx - 256]
+	LONG $0x01d4fdc5                           // vpaddq    ymm0, ymm0, yword [rcx]
+	LONG $0x49d4f5c5; BYTE $0x20               // vpaddq    ymm1, ymm1, yword [rcx + 32]
+	LONG $0x51d4edc5; BYTE $0x40               // vpaddq    ymm2, ymm2, yword [rcx + 64]
+	LONG $0x59d4e5c5; BYTE $0x60               // vpaddq    ymm3, ymm3, yword [rcx + 96]
+	QUAD $0x00000080a1d4ddc5                   // vpaddq    ymm4, ymm4, yword [rcx + 128]
+	QUAD $0x000000a0a9d4d5c5                   // vpaddq    ymm5, ymm5, yword [rcx + 160]
+	QUAD $0x000000c0b1d4cdc5                   // vpaddq    ymm6, ymm6, yword [rcx + 192]
+	QUAD $0x000000e0b9d4c5c5                   // vpaddq    ymm7, ymm7, yword [rcx + 224]
+	LONG $0x00c18148; WORD $0x0008; BYTE $0x00 // add    rcx, 2048
+	LONG $0xff000548; WORD $0xffff             // add    rax, -256
+	JNE  LBB0_13
+
+LBB0_14:
+	LONG $0xcdd4f5c5               // vpaddq    ymm1, ymm1, ymm5
+	LONG $0xdfd4e5c5               // vpaddq    ymm3, ymm3, ymm7
+	LONG $0xc4d4fdc5               // vpaddq    ymm0, ymm0, ymm4
+	LONG $0xd6d4edc5               // vpaddq    ymm2, ymm2, ymm6
+	LONG $0xc2d4fdc5               // vpaddq    ymm0, ymm0, ymm2
+	LONG $0xcbd4f5c5               // vpaddq    ymm1, ymm1, ymm3
+	LONG $0xc1d4fdc5               // vpaddq    ymm0, ymm0, ymm1
+	LONG $0x397de3c4; WORD $0x01c1 // vextracti128    xmm1, ymm0, 1
+	LONG $0xc1d4fdc5               // vpaddq    ymm0, ymm0, ymm1
+	LONG $0xc870f9c5; BYTE $0x4e   // vpshufd    xmm1, xmm0, 78
+	LONG $0xc1d4fdc5               // vpaddq    ymm0, ymm0, ymm1
+	LONG $0x7ef9e1c4; BYTE $0xc0   // vmovq    rax, xmm0
+	WORD $0x3949; BYTE $0xf1       // cmp    r9, rsi
+	JNE  LBB0_4
+	JMP  LBB0_15
diff --git a/go/arrow/math/uint64_noasm.go b/go/arrow/math/uint64_noasm.go
new file mode 100644
index 0000000000000..b7174b3d745c3
--- /dev/null
+++ b/go/arrow/math/uint64_noasm.go
@@ -0,0 +1,25 @@
+// Code generated by type_noasm.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build noasm
+
+package math
+
+func initUint64Go() {
+	Uint64.sum = sum_uint64_go
+}
diff --git a/go/arrow/math/uint64_sse4_amd64.go b/go/arrow/math/uint64_sse4_amd64.go
new file mode 100644
index 0000000000000..428bbe69fb6cb
--- /dev/null
+++ b/go/arrow/math/uint64_sse4_amd64.go
@@ -0,0 +1,41 @@
+// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package math
+
+import (
+	"unsafe"
+
+	"github.com/apache/arrow/go/arrow/array"
+)
+
+//go:noescape
+func _sum_uint64_sse4(buf, len, res unsafe.Pointer)
+
+func sum_uint64_sse4(a *array.Uint64) uint64 {
+	buf := a.Uint64Values()
+	var (
+		p1  = unsafe.Pointer(&buf[0])
+		p2  = unsafe.Pointer(uintptr(len(buf)))
+		res uint64
+	)
+	_sum_uint64_sse4(p1, p2, unsafe.Pointer(&res))
+	return res
+}
diff --git a/go/arrow/math/uint64_sse4_amd64.s b/go/arrow/math/uint64_sse4_amd64.s
new file mode 100644
index 0000000000000..1af9cfeb582a9
--- /dev/null
+++ b/go/arrow/math/uint64_sse4_amd64.s
@@ -0,0 +1,100 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_sum_uint64_sse4(SB), $0-24
+
+	MOVQ buf+0(FP), DI
+	MOVQ len+8(FP), SI
+	MOVQ res+16(FP), DX
+
+	WORD $0x8548; BYTE $0xf6 // test    rsi, rsi
+	JE   LBB0_1
+	LONG $0x03fe8348         // cmp    rsi, 3
+	JBE  LBB0_3
+	WORD $0x8949; BYTE $0xf1 // mov    r9, rsi
+	LONG $0xfce18349         // and    r9, -4
+	JE   LBB0_3
+	LONG $0xfc418d4d         // lea    r8, [r9 - 4]
+	WORD $0x8944; BYTE $0xc0 // mov    eax, r8d
+	WORD $0xe8c1; BYTE $0x02 // shr    eax, 2
+	WORD $0xc0ff             // inc    eax
+	LONG $0x03e08348         // and    rax, 3
+	JE   LBB0_8
+	WORD $0xf748; BYTE $0xd8 // neg    rax
+	LONG $0xc0ef0f66         // pxor    xmm0, xmm0
+	WORD $0xc931             // xor    ecx, ecx
+	LONG $0xc9ef0f66         // pxor    xmm1, xmm1
+
+LBB0_10:
+	LONG $0x146f0ff3; BYTE $0xcf   // movdqu    xmm2, oword [rdi + 8*rcx]
+	LONG $0x5c6f0ff3; WORD $0x10cf // movdqu    xmm3, oword [rdi + 8*rcx + 16]
+	LONG $0xc2d40f66               // paddq    xmm0, xmm2
+	LONG $0xcbd40f66               // paddq    xmm1, xmm3
+	LONG $0x04c18348               // add    rcx, 4
+	WORD $0xff48; BYTE $0xc0       // inc    rax
+	JNE  LBB0_10
+	JMP  LBB0_11
+
+LBB0_3:
+	WORD $0x3145; BYTE $0xc9 // xor    r9d, r9d
+	WORD $0xc031             // xor    eax, eax
+
+LBB0_4:
+	LONG $0xcf0c8d4a         // lea    rcx, [rdi + 8*r9]
+	WORD $0x294c; BYTE $0xce // sub    rsi, r9
+
+LBB0_5:
+	WORD $0x0348; BYTE $0x01 // add    rax, qword [rcx]
+	LONG $0x08c18348         // add    rcx, 8
+	WORD $0xff48; BYTE $0xce // dec    rsi
+	JNE  LBB0_5
+	JMP  LBB0_15
+
+LBB0_1:
+	WORD $0xc031 // xor    eax, eax
+
+LBB0_15:
+	WORD $0x8948; BYTE $0x02 // mov    qword [rdx], rax
+	RET
+
+LBB0_8:
+	WORD $0xc931     // xor    ecx, ecx
+	LONG $0xc0ef0f66 // pxor    xmm0, xmm0
+	LONG $0xc9ef0f66 // pxor    xmm1, xmm1
+
+LBB0_11:
+	LONG $0x0cf88349             // cmp    r8, 12
+	JB   LBB0_14
+	WORD $0x894c; BYTE $0xc8     // mov    rax, r9
+	WORD $0x2948; BYTE $0xc8     // sub    rax, rcx
+	LONG $0xcf4c8d48; BYTE $0x70 // lea    rcx, [rdi + 8*rcx + 112]
+
+LBB0_13:
+	LONG $0x516f0ff3; BYTE $0x90 // movdqu    xmm2, oword [rcx - 112]
+	LONG $0x596f0ff3; BYTE $0xa0 // movdqu    xmm3, oword [rcx - 96]
+	LONG $0x616f0ff3; BYTE $0xb0 // movdqu    xmm4, oword [rcx - 80]
+	LONG $0x696f0ff3; BYTE $0xc0 // movdqu    xmm5, oword [rcx - 64]
+	LONG $0xd0d40f66             // paddq    xmm2, xmm0
+	LONG $0xd9d40f66             // paddq    xmm3, xmm1
+	LONG $0x716f0ff3; BYTE $0xd0 // movdqu    xmm6, oword [rcx - 48]
+	LONG $0x796f0ff3; BYTE $0xe0 // movdqu    xmm7, oword [rcx - 32]
+	LONG $0xf4d40f66             // paddq    xmm6, xmm4
+	LONG $0xf2d40f66             // paddq    xmm6, xmm2
+	LONG $0xfdd40f66             // paddq    xmm7, xmm5
+	LONG $0xfbd40f66             // paddq    xmm7, xmm3
+	LONG $0x416f0ff3; BYTE $0xf0 // movdqu    xmm0, oword [rcx - 16]
+	LONG $0x096f0ff3             // movdqu    xmm1, oword [rcx]
+	LONG $0xc6d40f66             // paddq    xmm0, xmm6
+	LONG $0xcfd40f66             // paddq    xmm1, xmm7
+	LONG $0x80e98348             // sub    rcx, -128
+	LONG $0xf0c08348             // add    rax, -16
+	JNE  LBB0_13
+
+LBB0_14:
+	LONG $0xc1d40f66             // paddq    xmm0, xmm1
+	LONG $0xc8700f66; BYTE $0x4e // pshufd    xmm1, xmm0, 78
+	LONG $0xc8d40f66             // paddq    xmm1, xmm0
+	LONG $0x7e0f4866; BYTE $0xc8 // movq    rax, xmm1
+	WORD $0x3949; BYTE $0xf1     // cmp    r9, rsi
+	JNE  LBB0_4
+	JMP  LBB0_15
diff --git a/go/arrow/math/uint64_test.go b/go/arrow/math/uint64_test.go
new file mode 100644
index 0000000000000..7b8a165797940
--- /dev/null
+++ b/go/arrow/math/uint64_test.go
@@ -0,0 +1,68 @@
+// Code generated by type_test.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/array"
+	"github.com/apache/arrow/go/arrow/math"
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUint64Funcs_Sum(t *testing.T) {
+	vec := makeArrayUint64(10000)
+	res := math.Uint64.Sum(vec)
+	assert.Equal(t, res, uint64(49995000))
+}
+
+func makeArrayUint64(l int) *array.Uint64 {
+	fb := array.NewUint64Builder(memory.NewGoAllocator())
+	fb.Reserve(l)
+	for i := 0; i < l; i++ {
+		fb.Append(uint64(i))
+	}
+	return fb.NewUint64Array()
+}
+
+func benchmarkUint64Funcs_Sum(b *testing.B, n int) {
+	vec := makeArrayUint64(n)
+	b.SetBytes(int64(vec.Len() * 8))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		math.Uint64.Sum(vec)
+	}
+}
+
+func BenchmarkUint64Funcs_Sum_256(b *testing.B) {
+	benchmarkUint64Funcs_Sum(b, 256)
+}
+
+func BenchmarkUint64Funcs_Sum_1024(b *testing.B) {
+	benchmarkUint64Funcs_Sum(b, 1024)
+}
+
+func BenchmarkUint64Funcs_Sum_8192(b *testing.B) {
+	benchmarkUint64Funcs_Sum(b, 8192)
+}
+
+func BenchmarkUint64Funcs_Sum_1000000(b *testing.B) {
+	benchmarkUint64Funcs_Sum(b, 1e6)
+}
diff --git a/go/arrow/memory/Makefile b/go/arrow/memory/Makefile
new file mode 100644
index 0000000000000..e50d94e603123
--- /dev/null
+++ b/go/arrow/memory/Makefile
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GO_BUILD=go build
+GO_GEN=go generate
+GO_TEST=go test
+GOPATH=$(realpath ../../../..)
+
+# this converts rotate instructions from "ro[lr] <reg>" -> "ro[lr] <reg>, 1" for yasm compatibility
+PERL_FIXUP_ROTATE=perl -i -pe 's/(ro[rl]\s+\w{2,3})$$/\1, 1/'
+
+C2GOASM=c2goasm -a -f
+CC=clang
+C_FLAGS=-target x86_64-unknown-none -masm=intel -mno-red-zone -mstackrealign -mllvm -inline-threshold=1000 -fno-asynchronous-unwind-tables \
+	-fno-exceptions -fno-rtti -O3 -fno-builtin -ffast-math -fno-jump-tables -I_lib
+ASM_FLAGS_AVX2=-mavx2 -mfma -mllvm -force-vector-width=32
+ASM_FLAGS_SSE3=-msse3
+ASM_FLAGS_SSE4=-msse4
+
+GO_SOURCES  := $(shell find . -path ./_lib -prune -o -name '*.go' -not -name '*_test.go')
+ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -not -name '*_test.go')
+
+INTEL_SOURCES := \
+	memory_avx2_amd64.s memory_sse4_amd64.s
+
+.PHONEY: assembly
+
+assembly: $(INTEL_SOURCES)
+
+_lib/memory_avx2.s: _lib/memory.c
+	$(CC) -S $(C_FLAGS) $(ASM_FLAGS_AVX2) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+_lib/memory_sse4.s: _lib/memory.c
+	$(CC) -S $(C_FLAGS) $(ASM_FLAGS_SSE4) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@
+
+memory_avx2_amd64.s: _lib/memory_avx2.s
+	$(C2GOASM) -a -f $^ $@
+
+memory_sse4_amd64.s: _lib/memory_sse4.s
+	$(C2GOASM) -a -f $^ $@
+
diff --git a/go/arrow/memory/_lib/.gitignore b/go/arrow/memory/_lib/.gitignore
new file mode 100644
index 0000000000000..72757d396c0a0
--- /dev/null
+++ b/go/arrow/memory/_lib/.gitignore
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.idea
+cmake-build-*
\ No newline at end of file
diff --git a/go/arrow/memory/_lib/CMakeLists.txt b/go/arrow/memory/_lib/CMakeLists.txt
new file mode 100644
index 0000000000000..f6815302de1a3
--- /dev/null
+++ b/go/arrow/memory/_lib/CMakeLists.txt
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.6)
+
+project(memory-func)
+set(CMAKE_C_STANDARD 99)
+
+add_library(memory STATIC memory.c)
+
+
diff --git a/go/arrow/memory/_lib/arch.h b/go/arrow/memory/_lib/arch.h
new file mode 100644
index 0000000000000..d9c7d98bf6e1b
--- /dev/null
+++ b/go/arrow/memory/_lib/arch.h
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#undef FULL_NAME
+
+#if  defined(__AVX2__)
+    #define FULL_NAME(x) x##_avx2
+#elif __SSE4_2__ == 1
+    #define FULL_NAME(x) x##_sse4
+#elif __SSE3__ == 1
+    #define FULL_NAME(x) x##_sse3
+#else
+    #define FULL_NAME(x) x##_x86
+#endif
\ No newline at end of file
diff --git a/go/arrow/memory/_lib/memory.c b/go/arrow/memory/_lib/memory.c
new file mode 100644
index 0000000000000..0e57237c31d80
--- /dev/null
+++ b/go/arrow/memory/_lib/memory.c
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "arch.h"
+#include <memory.h>
+
+void FULL_NAME(memset)(void *buf, size_t len, int v) {
+    char *bytes = buf;
+    char *end   = buf+len;
+    const char val = v;
+    while (bytes < end) {
+        *bytes++ = val;
+    }
+}
\ No newline at end of file
diff --git a/go/arrow/memory/_lib/memory_avx2.s b/go/arrow/memory/_lib/memory_avx2.s
new file mode 100644
index 0000000000000..f53fa15c5eff1
--- /dev/null
+++ b/go/arrow/memory/_lib/memory_avx2.s
@@ -0,0 +1,97 @@
+	.text
+	.intel_syntax noprefix
+	.file	"_lib/memory.c"
+	.globl	memset_avx2
+	.p2align	4, 0x90
+	.type	memset_avx2,@function
+memset_avx2:                            # @memset_avx2
+# BB#0:
+	push	rbp
+	mov	rbp, rsp
+	and	rsp, -8
+	lea	r11, [rdi + rsi]
+	cmp	r11, rdi
+	jbe	.LBB0_13
+# BB#1:
+	cmp	rsi, 128
+	jb	.LBB0_12
+# BB#2:
+	mov	r8, rsi
+	and	r8, -128
+	mov	r10, rsi
+	and	r10, -128
+	je	.LBB0_12
+# BB#3:
+	vmovd	xmm0, edx
+	vpbroadcastb	ymm0, xmm0
+	lea	r9, [r10 - 128]
+	mov	eax, r9d
+	shr	eax, 7
+	inc	eax
+	and	rax, 3
+	je	.LBB0_4
+# BB#5:
+	neg	rax
+	xor	ecx, ecx
+	.p2align	4, 0x90
+.LBB0_6:                                # =>This Inner Loop Header: Depth=1
+	vmovdqu	ymmword ptr [rdi + rcx], ymm0
+	vmovdqu	ymmword ptr [rdi + rcx + 32], ymm0
+	vmovdqu	ymmword ptr [rdi + rcx + 64], ymm0
+	vmovdqu	ymmword ptr [rdi + rcx + 96], ymm0
+	sub	rcx, -128
+	inc	rax
+	jne	.LBB0_6
+	jmp	.LBB0_7
+.LBB0_4:
+	xor	ecx, ecx
+.LBB0_7:
+	cmp	r9, 384
+	jb	.LBB0_10
+# BB#8:
+	mov	rax, r10
+	sub	rax, rcx
+	lea	rcx, [rdi + rcx + 480]
+	.p2align	4, 0x90
+.LBB0_9:                                # =>This Inner Loop Header: Depth=1
+	vmovdqu	ymmword ptr [rcx - 480], ymm0
+	vmovdqu	ymmword ptr [rcx - 448], ymm0
+	vmovdqu	ymmword ptr [rcx - 416], ymm0
+	vmovdqu	ymmword ptr [rcx - 384], ymm0
+	vmovdqu	ymmword ptr [rcx - 352], ymm0
+	vmovdqu	ymmword ptr [rcx - 320], ymm0
+	vmovdqu	ymmword ptr [rcx - 288], ymm0
+	vmovdqu	ymmword ptr [rcx - 256], ymm0
+	vmovdqu	ymmword ptr [rcx - 224], ymm0
+	vmovdqu	ymmword ptr [rcx - 192], ymm0
+	vmovdqu	ymmword ptr [rcx - 160], ymm0
+	vmovdqu	ymmword ptr [rcx - 128], ymm0
+	vmovdqu	ymmword ptr [rcx - 96], ymm0
+	vmovdqu	ymmword ptr [rcx - 64], ymm0
+	vmovdqu	ymmword ptr [rcx - 32], ymm0
+	vmovdqu	ymmword ptr [rcx], ymm0
+	add	rcx, 512
+	add	rax, -512
+	jne	.LBB0_9
+.LBB0_10:
+	cmp	r10, rsi
+	je	.LBB0_13
+# BB#11:
+	add	rdi, r8
+	.p2align	4, 0x90
+.LBB0_12:                               # =>This Inner Loop Header: Depth=1
+	mov	byte ptr [rdi], dl
+	inc	rdi
+	cmp	r11, rdi
+	jne	.LBB0_12
+.LBB0_13:
+	mov	rsp, rbp
+	pop	rbp
+	vzeroupper
+	ret
+.Lfunc_end0:
+	.size	memset_avx2, .Lfunc_end0-memset_avx2
+
+
+	.ident	"Apple LLVM version 9.0.0 (clang-900.0.39.2)"
+	.section	".note.GNU-stack","",@progbits
diff --git a/go/arrow/memory/_lib/memory_sse4.s b/go/arrow/memory/_lib/memory_sse4.s
new file mode 100644
index 0000000000000..4b44cd31c315f
--- /dev/null
+++ b/go/arrow/memory/_lib/memory_sse4.s
@@ -0,0 +1,96 @@
+	.text
+	.intel_syntax noprefix
+	.file	"_lib/memory.c"
+	.globl	memset_sse4
+	.p2align	4, 0x90
+	.type	memset_sse4,@function
+memset_sse4:                            # @memset_sse4
+# BB#0:
+	push	rbp
+	mov	rbp, rsp
+	and	rsp, -8
+	lea	r11, [rdi + rsi]
+	cmp	r11, rdi
+	jbe	.LBB0_13
+# BB#1:
+	cmp	rsi, 32
+	jb	.LBB0_12
+# BB#2:
+	mov	r8, rsi
+	and	r8, -32
+	mov	r10, rsi
+	and	r10, -32
+	je	.LBB0_12
+# BB#3:
+	movzx	eax, dl
+	movd	xmm0, eax
+	pxor	xmm1, xmm1
+	pshufb	xmm0, xmm1
+	lea	r9, [r10 - 32]
+	mov	ecx, r9d
+	shr	ecx, 5
+	inc	ecx
+	and	rcx, 7
+	je	.LBB0_4
+# BB#5:
+	neg	rcx
+	xor	eax, eax
+	.p2align	4, 0x90
+.LBB0_6:                                # =>This Inner Loop Header: Depth=1
+	movdqu	xmmword ptr [rdi + rax], xmm0
+	movdqu	xmmword ptr [rdi + rax + 16], xmm0
+	add	rax, 32
+	inc	rcx
+	jne	.LBB0_6
+	jmp	.LBB0_7
+.LBB0_4:
+	xor	eax, eax
+.LBB0_7:
+	cmp	r9, 224
+	jb	.LBB0_10
+# BB#8:
+	mov	rcx, r10
+	sub	rcx, rax
+	lea	rax, [rdi + rax + 240]
+	.p2align	4, 0x90
+.LBB0_9:                                # =>This Inner Loop Header: Depth=1
+	movdqu	xmmword ptr [rax - 240], xmm0
+	movdqu	xmmword ptr [rax - 224], xmm0
+	movdqu	xmmword ptr [rax - 208], xmm0
+	movdqu	xmmword ptr [rax - 192], xmm0
+	movdqu	xmmword ptr [rax - 176], xmm0
+	movdqu	xmmword ptr [rax - 160], xmm0
+	movdqu	xmmword ptr [rax - 144], xmm0
+	movdqu	xmmword ptr [rax - 128], xmm0
+	movdqu	xmmword ptr [rax - 112], xmm0
+	movdqu	xmmword ptr [rax - 96], xmm0
+	movdqu	xmmword ptr [rax - 80], xmm0
+	movdqu	xmmword ptr [rax - 64], xmm0
+	movdqu	xmmword ptr [rax - 48], xmm0
+	movdqu	xmmword ptr [rax - 32], xmm0
+	movdqu	xmmword ptr [rax - 16], xmm0
+	movdqu	xmmword ptr [rax], xmm0
+	add	rax, 256
+	add	rcx, -256
+	jne	.LBB0_9
+.LBB0_10:
+	cmp	r10, rsi
+	je	.LBB0_13
+# BB#11:
+	add	rdi, r8
+	.p2align	4, 0x90
+.LBB0_12:                               # =>This Inner Loop Header: Depth=1
+	mov	byte ptr [rdi], dl
+	inc	rdi
+	cmp	r11, rdi
+	jne	.LBB0_12
+.LBB0_13:
+	mov	rsp, rbp
+	pop	rbp
+	ret
+.Lfunc_end0:
+	.size	memset_sse4, .Lfunc_end0-memset_sse4
+
+
+	.ident	"Apple LLVM version 9.0.0 (clang-900.0.39.2)"
+	.section	".note.GNU-stack","",@progbits
diff --git a/go/arrow/memory/allocator.go b/go/arrow/memory/allocator.go
new file mode 100644
index 0000000000000..da6be44e37a80
--- /dev/null
+++ b/go/arrow/memory/allocator.go
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+const (
+	alignment = 64
+)
+
+type Allocator interface {
+	Allocate(size int) []byte
+	Reallocate(size int, b []byte) []byte
+	Free(b []byte)
+}
+
+// DefaultAllocator is a default implementation of Allocator and can be used anywhere
+// an Allocator is required.
+//
+// DefaultAllocator is safe to use from multiple goroutines.
+var DefaultAllocator Allocator = NewGoAllocator()
diff --git a/go/arrow/memory/buffer.go b/go/arrow/memory/buffer.go
new file mode 100644
index 0000000000000..234f5d4337fab
--- /dev/null
+++ b/go/arrow/memory/buffer.go
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+import (
+	"sync/atomic"
+
+	"github.com/apache/arrow/go/arrow/internal/debug"
+)
+
+type Buffer struct {
+	refCount int64
+	buf      []byte
+	length   int
+	mutable  bool
+	mem      Allocator
+}
+
+// NewBufferBytes creates a fixed-size buffer from the specified data.
+func NewBufferBytes(data []byte) *Buffer {
+	return &Buffer{refCount: 0, buf: data, length: len(data)}
+}
+
+// NewBuffer creates a mutable, resizable buffer with an Allocator for managing memory.
+func NewResizableBuffer(mem Allocator) *Buffer {
+	return &Buffer{refCount: 1, mutable: true, mem: mem}
+}
+
+// Retain increases the reference count by 1.
+func (b *Buffer) Retain() {
+	if b.mem != nil {
+		atomic.AddInt64(&b.refCount, 1)
+	}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Buffer) Release() {
+	if b.mem != nil {
+		debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+		if atomic.AddInt64(&b.refCount, -1) == 0 {
+			b.mem.Free(b.buf)
+			b.buf, b.length = nil, 0
+		}
+	}
+}
+
+// Buf returns the slice of memory allocated by the Buffer, which is adjusted by calling Reserve.
+func (b *Buffer) Buf() []byte { return b.buf }
+
+// Bytes returns a slice of size Len, which is adjusted by calling Resize.
+func (b *Buffer) Bytes() []byte { return b.buf[:b.length] }
+func (b *Buffer) Mutable() bool { return b.mutable }
+func (b *Buffer) Len() int      { return b.length }
+func (b *Buffer) Cap() int      { return len(b.buf) }
+
+func (b *Buffer) Reserve(capacity int) {
+	if capacity > len(b.buf) {
+		newCap := roundUpToMultipleOf64(capacity)
+		if len(b.buf) == 0 {
+			b.buf = b.mem.Allocate(newCap)
+		} else {
+			b.buf = b.mem.Reallocate(newCap, b.buf)
+		}
+	}
+}
+
+func (b *Buffer) Resize(newSize int) {
+	b.resize(newSize, true)
+}
+
+func (b *Buffer) ResizeNoShrink(newSize int) {
+	b.resize(newSize, false)
+}
+
+func (b *Buffer) resize(newSize int, shrink bool) {
+	if !shrink || newSize > b.length {
+		b.Reserve(newSize)
+	} else {
+		// Buffer is not growing, so shrink to the requested size without
+		// excess space.
+		newCap := roundUpToMultipleOf64(newSize)
+		if len(b.buf) != newCap {
+			if newSize == 0 {
+				b.mem.Free(b.buf)
+				b.buf = nil
+			} else {
+				b.buf = b.mem.Reallocate(newCap, b.buf)
+			}
+		}
+	}
+	b.length = newSize
+}
diff --git a/go/arrow/memory/buffer_test.go b/go/arrow/memory/buffer_test.go
new file mode 100644
index 0000000000000..918175cb6e269
--- /dev/null
+++ b/go/arrow/memory/buffer_test.go
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewResizableBuffer(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	buf := memory.NewResizableBuffer(mem)
+	buf.Retain() // refCount == 2
+
+	exp := 10
+	buf.Resize(exp)
+	assert.NotNil(t, buf.Bytes())
+	assert.Equal(t, exp, len(buf.Bytes()))
+	assert.Equal(t, exp, buf.Len())
+
+	buf.Release() // refCount == 1
+	assert.NotNil(t, buf.Bytes())
+
+	buf.Release() // refCount == 0
+	assert.Nil(t, buf.Bytes())
+	assert.Zero(t, buf.Len())
+}
diff --git a/go/arrow/memory/checked_allocator.go b/go/arrow/memory/checked_allocator.go
new file mode 100644
index 0000000000000..4cce89f2444bb
--- /dev/null
+++ b/go/arrow/memory/checked_allocator.go
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+type CheckedAllocator struct {
+	mem  Allocator
+	base int
+	sz   int
+}
+
+func NewCheckedAllocator(mem Allocator) *CheckedAllocator {
+	return &CheckedAllocator{mem: mem}
+}
+
+func (a *CheckedAllocator) Allocate(size int) []byte {
+	a.sz += size
+	return a.mem.Allocate(size)
+}
+
+func (a *CheckedAllocator) Reallocate(size int, b []byte) []byte {
+	a.sz += size - len(b)
+	return a.mem.Reallocate(size, b)
+}
+
+func (a *CheckedAllocator) Free(b []byte) {
+	a.sz -= len(b)
+	a.mem.Free(b)
+}
+
+type TestingT interface {
+	Errorf(format string, args ...interface{})
+	Helper()
+}
+
+func (a *CheckedAllocator) AssertSize(t TestingT, sz int) {
+	if a.sz != sz {
+		t.Helper()
+		t.Errorf("invalid memory size exp=%d, got=%d", sz, a.sz)
+	}
+}
+
+type CheckedAllocatorScope struct {
+	alloc *CheckedAllocator
+	sz    int
+}
+
+func NewCheckedAllocatorScope(alloc *CheckedAllocator) *CheckedAllocatorScope {
+	return &CheckedAllocatorScope{alloc: alloc, sz: alloc.sz}
+}
+
+func (c *CheckedAllocatorScope) CheckSize(t TestingT) {
+	if c.sz != c.alloc.sz {
+		t.Helper()
+		t.Errorf("invalid memory size exp=%d, got=%d", c.sz, c.alloc.sz)
+	}
+}
diff --git a/go/arrow/memory/doc.go b/go/arrow/memory/doc.go
new file mode 100644
index 0000000000000..959f88b4f2225
--- /dev/null
+++ b/go/arrow/memory/doc.go
@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+Package memory provides support for allocating and manipulating memory at a low level.
+*/
+package memory
diff --git a/go/arrow/memory/go_allocator.go b/go/arrow/memory/go_allocator.go
new file mode 100644
index 0000000000000..a85baee713bb5
--- /dev/null
+++ b/go/arrow/memory/go_allocator.go
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+type GoAllocator struct{}
+
+func NewGoAllocator() *GoAllocator { return &GoAllocator{} }
+
+func (a *GoAllocator) Allocate(size int) []byte {
+	buf := make([]byte, size+alignment) // padding for 64-byte alignment
+	addr := int(addressOf(buf))
+	next := roundUpToMultipleOf64(addr)
+	if addr != next {
+		shift := next - addr
+		return buf[shift : size+shift : size+shift]
+	}
+	return buf[:size:size]
+}
+
+func (a *GoAllocator) Reallocate(size int, b []byte) []byte {
+	if size == len(b) {
+		return b
+	}
+
+	newBuf := a.Allocate(size)
+	copy(newBuf, b)
+	return newBuf
+}
+
+func (a *GoAllocator) Free(b []byte) {}
diff --git a/go/arrow/memory/go_allocator_test.go b/go/arrow/memory/go_allocator_test.go
new file mode 100644
index 0000000000000..d146e301ae575
--- /dev/null
+++ b/go/arrow/memory/go_allocator_test.go
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func isAlignedTo(addr, alignment int) bool {
+	return addr&(alignment-1) == 0
+}
+
+func TestGoAllocator_Allocate(t *testing.T) {
+	tests := []struct {
+		name string
+		sz   int
+	}{
+		{"lt alignment", 33},
+		{"gt alignment unaligned", 65},
+		{"eq alignment", 64},
+		{"large unaligned", 4097},
+		{"large aligned", 8192},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			a := &GoAllocator{}
+			buf := a.Allocate(test.sz)
+			addr := addressOf(buf)
+			assert.True(t, isAlignedTo(int(addr), alignment))
+			assert.Equal(t, test.sz, len(buf), "invalid len")
+			assert.Equal(t, test.sz, cap(buf), "invalid cap")
+		})
+	}
+}
+
+func TestGoAllocator_Reallocate(t *testing.T) {
+	tests := []struct {
+		name     string
+		sz1, sz2 int
+	}{
+		{"smaller", 200, 100},
+		{"same", 200, 200},
+		{"larger", 200, 300},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			a := &GoAllocator{}
+			buf := a.Allocate(test.sz1)
+			for i := range buf {
+				buf[i] = byte(i & 0xff)
+			}
+
+			exp := make([]byte, test.sz2)
+			copy(exp, buf)
+
+			newBuf := a.Reallocate(test.sz2, buf)
+			assert.Equal(t, exp, newBuf)
+		})
+	}
+}
diff --git a/go/arrow/memory/memory.go b/go/arrow/memory/memory.go
new file mode 100644
index 0000000000000..de2a3a66394d9
--- /dev/null
+++ b/go/arrow/memory/memory.go
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+var (
+	memset func(b []byte, c byte)
+)
+
+// Set assigns the value c to every element of the slice buf.
+func Set(buf []byte, c byte) {
+	memset(buf, c)
+}
+
+// memory_memset_go reference implementation
+func memory_memset_go(buf []byte, c byte) {
+	for i := 0; i < len(buf); i++ {
+		buf[i] = c
+	}
+}
diff --git a/go/arrow/memory/memory_amd64.go b/go/arrow/memory/memory_amd64.go
new file mode 100644
index 0000000000000..5ea4a3fe1a046
--- /dev/null
+++ b/go/arrow/memory/memory_amd64.go
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package memory
+
+import (
+	"github.com/apache/arrow/go/arrow/internal/cpu"
+)
+
+func init() {
+	if cpu.X86.HasAVX2 {
+		memset = memory_memset_avx2
+	} else if cpu.X86.HasSSE42 {
+		memset = memory_memset_sse4
+	} else {
+		memset = memory_memset_go
+	}
+}
diff --git a/go/arrow/memory/memory_avx2_amd64.go b/go/arrow/memory/memory_avx2_amd64.go
new file mode 100644
index 0000000000000..1f9d176cce5aa
--- /dev/null
+++ b/go/arrow/memory/memory_avx2_amd64.go
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package memory
+
+import "unsafe"
+
+//go:noescape
+func _memset_avx2(buf, len, c unsafe.Pointer)
+
+func memory_memset_avx2(buf []byte, c byte) {
+	if len(buf) == 0 {
+		return
+	}
+
+	var (
+		p1 = unsafe.Pointer(&buf[0])
+		p2 = unsafe.Pointer(uintptr(len(buf)))
+		p3 = unsafe.Pointer(uintptr(c))
+	)
+	if len(buf) > 2000 || isMultipleOfPowerOf2(len(buf), 256) {
+		_memset_avx2(p1, p2, p3)
+	} else {
+		_memset_sse4(p1, p2, p3)
+	}
+}
diff --git a/go/arrow/memory/memory_avx2_amd64.s b/go/arrow/memory/memory_avx2_amd64.s
new file mode 100644
index 0000000000000..2a77807cb27c3
--- /dev/null
+++ b/go/arrow/memory/memory_avx2_amd64.s
@@ -0,0 +1,85 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_memset_avx2(SB), $0-24
+
+	MOVQ buf+0(FP), DI
+	MOVQ len+8(FP), SI
+	MOVQ c+16(FP), DX
+
+	LONG $0x371c8d4c                           // lea    r11, [rdi + rsi]
+	WORD $0x3949; BYTE $0xfb                   // cmp    r11, rdi
+	JBE  LBB0_13
+	LONG $0x80fe8148; WORD $0x0000; BYTE $0x00 // cmp    rsi, 128
+	JB   LBB0_12
+	WORD $0x8949; BYTE $0xf0                   // mov    r8, rsi
+	LONG $0x80e08349                           // and    r8, -128
+	WORD $0x8949; BYTE $0xf2                   // mov    r10, rsi
+	LONG $0x80e28349                           // and    r10, -128
+	JE   LBB0_12
+	LONG $0xc26ef9c5                           // vmovd    xmm0, edx
+	LONG $0x787de2c4; BYTE $0xc0               // vpbroadcastb    ymm0, xmm0
+	LONG $0x804a8d4d                           // lea    r9, [r10 - 128]
+	WORD $0x8944; BYTE $0xc8                   // mov    eax, r9d
+	WORD $0xe8c1; BYTE $0x07                   // shr    eax, 7
+	WORD $0xc0ff                               // inc    eax
+	LONG $0x03e08348                           // and    rax, 3
+	JE   LBB0_4
+	WORD $0xf748; BYTE $0xd8                   // neg    rax
+	WORD $0xc931                               // xor    ecx, ecx
+
+LBB0_6:
+	LONG $0x047ffec5; BYTE $0x0f   // vmovdqu    yword [rdi + rcx], ymm0
+	LONG $0x447ffec5; WORD $0x200f // vmovdqu    yword [rdi + rcx + 32], ymm0
+	LONG $0x447ffec5; WORD $0x400f // vmovdqu    yword [rdi + rcx + 64], ymm0
+	LONG $0x447ffec5; WORD $0x600f // vmovdqu    yword [rdi + rcx + 96], ymm0
+	LONG $0x80e98348               // sub    rcx, -128
+	WORD $0xff48; BYTE $0xc0       // inc    rax
+	JNE  LBB0_6
+	JMP  LBB0_7
+
+LBB0_4:
+	WORD $0xc931 // xor    ecx, ecx
+
+LBB0_7:
+	LONG $0x80f98149; WORD $0x0001; BYTE $0x00 // cmp    r9, 384
+	JB   LBB0_10
+	WORD $0x894c; BYTE $0xd0                   // mov    rax, r10
+	WORD $0x2948; BYTE $0xc8                   // sub    rax, rcx
+	QUAD $0x000001e00f8c8d48                   // lea    rcx, [rdi + rcx + 480]
+
+LBB0_9:
+	QUAD $0xfffffe20817ffec5                   // vmovdqu    yword [rcx - 480], ymm0
+	QUAD $0xfffffe40817ffec5                   // vmovdqu    yword [rcx - 448], ymm0
+	QUAD $0xfffffe60817ffec5                   // vmovdqu    yword [rcx - 416], ymm0
+	QUAD $0xfffffe80817ffec5                   // vmovdqu    yword [rcx - 384], ymm0
+	QUAD $0xfffffea0817ffec5                   // vmovdqu    yword [rcx - 352], ymm0
+	QUAD $0xfffffec0817ffec5                   // vmovdqu    yword [rcx - 320], ymm0
+	QUAD $0xfffffee0817ffec5                   // vmovdqu    yword [rcx - 288], ymm0
+	QUAD $0xffffff00817ffec5                   // vmovdqu    yword [rcx - 256], ymm0
+	QUAD $0xffffff20817ffec5                   // vmovdqu    yword [rcx - 224], ymm0
+	QUAD $0xffffff40817ffec5                   // vmovdqu    yword [rcx - 192], ymm0
+	QUAD $0xffffff60817ffec5                   // vmovdqu    yword [rcx - 160], ymm0
+	LONG $0x417ffec5; BYTE $0x80               // vmovdqu    yword [rcx - 128], ymm0
+	LONG $0x417ffec5; BYTE $0xa0               // vmovdqu    yword [rcx - 96], ymm0
+	LONG $0x417ffec5; BYTE $0xc0               // vmovdqu    yword [rcx - 64], ymm0
+	LONG $0x417ffec5; BYTE $0xe0               // vmovdqu    yword [rcx - 32], ymm0
+	LONG $0x017ffec5                           // vmovdqu    yword [rcx], ymm0
+	LONG $0x00c18148; WORD $0x0002; BYTE $0x00 // add    rcx, 512
+	LONG $0xfe000548; WORD $0xffff             // add    rax, -512
+	JNE  LBB0_9
+
+LBB0_10:
+	WORD $0x3949; BYTE $0xf2 // cmp    r10, rsi
+	JE   LBB0_13
+	WORD $0x014c; BYTE $0xc7 // add    rdi, r8
+
+LBB0_12:
+	WORD $0x1788             // mov    byte [rdi], dl
+	WORD $0xff48; BYTE $0xc7 // inc    rdi
+	WORD $0x3949; BYTE $0xfb // cmp    r11, rdi
+	JNE  LBB0_12
+
+LBB0_13:
+	VZEROUPPER
+	RET
diff --git a/go/arrow/memory/memory_noasm.go b/go/arrow/memory/memory_noasm.go
new file mode 100644
index 0000000000000..bf8846fa2e059
--- /dev/null
+++ b/go/arrow/memory/memory_noasm.go
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build noasm
+
+package memory
+
+func init() {
+	memset = memory_memset_go
+}
diff --git a/go/arrow/memory/memory_sse4_amd64.go b/go/arrow/memory/memory_sse4_amd64.go
new file mode 100644
index 0000000000000..61da4665adf2b
--- /dev/null
+++ b/go/arrow/memory/memory_sse4_amd64.go
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noasm
+
+package memory
+
+import "unsafe"
+
+//go:noescape
+func _memset_sse4(buf, len, c unsafe.Pointer)
+
+func memory_memset_sse4(buf []byte, c byte) {
+	if len(buf) == 0 {
+		return
+	}
+	_memset_sse4(unsafe.Pointer(&buf[0]), unsafe.Pointer(uintptr(len(buf))), unsafe.Pointer(uintptr(c)))
+}
diff --git a/go/arrow/memory/memory_sse4_amd64.s b/go/arrow/memory/memory_sse4_amd64.s
new file mode 100644
index 0000000000000..b1906f99b7161
--- /dev/null
+++ b/go/arrow/memory/memory_sse4_amd64.s
@@ -0,0 +1,84 @@
+//+build !noasm !appengine
+// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
+
+TEXT ·_memset_sse4(SB), $0-24
+
+	MOVQ buf+0(FP), DI
+	MOVQ len+8(FP), SI
+	MOVQ c+16(FP), DX
+
+	LONG $0x371c8d4c             // lea    r11, [rdi + rsi]
+	WORD $0x3949; BYTE $0xfb     // cmp    r11, rdi
+	JBE  LBB0_13
+	LONG $0x20fe8348             // cmp    rsi, 32
+	JB   LBB0_12
+	WORD $0x8949; BYTE $0xf0     // mov    r8, rsi
+	LONG $0xe0e08349             // and    r8, -32
+	WORD $0x8949; BYTE $0xf2     // mov    r10, rsi
+	LONG $0xe0e28349             // and    r10, -32
+	JE   LBB0_12
+	WORD $0xb60f; BYTE $0xc2     // movzx    eax, dl
+	LONG $0xc06e0f66             // movd    xmm0, eax
+	LONG $0xc9ef0f66             // pxor    xmm1, xmm1
+	LONG $0x00380f66; BYTE $0xc1 // pshufb    xmm0, xmm1
+	LONG $0xe04a8d4d             // lea    r9, [r10 - 32]
+	WORD $0x8944; BYTE $0xc9     // mov    ecx, r9d
+	WORD $0xe9c1; BYTE $0x05     // shr    ecx, 5
+	WORD $0xc1ff                 // inc    ecx
+	LONG $0x07e18348             // and    rcx, 7
+	JE   LBB0_4
+	WORD $0xf748; BYTE $0xd9     // neg    rcx
+	WORD $0xc031                 // xor    eax, eax
+
+LBB0_6:
+	LONG $0x047f0ff3; BYTE $0x07   // movdqu    oword [rdi + rax], xmm0
+	LONG $0x447f0ff3; WORD $0x1007 // movdqu    oword [rdi + rax + 16], xmm0
+	LONG $0x20c08348               // add    rax, 32
+	WORD $0xff48; BYTE $0xc1       // inc    rcx
+	JNE  LBB0_6
+	JMP  LBB0_7
+
+LBB0_4:
+	WORD $0xc031 // xor    eax, eax
+
+LBB0_7:
+	LONG $0xe0f98149; WORD $0x0000; BYTE $0x00 // cmp    r9, 224
+	JB   LBB0_10
+	WORD $0x894c; BYTE $0xd1                   // mov    rcx, r10
+	WORD $0x2948; BYTE $0xc1                   // sub    rcx, rax
+	QUAD $0x000000f007848d48                   // lea    rax, [rdi + rax + 240]
+
+LBB0_9:
+	QUAD $0xffffff10807f0ff3                   // movdqu    oword [rax - 240], xmm0
+	QUAD $0xffffff20807f0ff3                   // movdqu    oword [rax - 224], xmm0
+	QUAD $0xffffff30807f0ff3                   // movdqu    oword [rax - 208], xmm0
+	QUAD $0xffffff40807f0ff3                   // movdqu    oword [rax - 192], xmm0
+	QUAD $0xffffff50807f0ff3                   // movdqu    oword [rax - 176], xmm0
+	QUAD $0xffffff60807f0ff3                   // movdqu    oword [rax - 160], xmm0
+	QUAD $0xffffff70807f0ff3                   // movdqu    oword [rax - 144], xmm0
+	LONG $0x407f0ff3; BYTE $0x80               // movdqu    oword [rax - 128], xmm0
+	LONG $0x407f0ff3; BYTE $0x90               // movdqu    oword [rax - 112], xmm0
+	LONG $0x407f0ff3; BYTE $0xa0               // movdqu    oword [rax - 96], xmm0
+	LONG $0x407f0ff3; BYTE $0xb0               // movdqu    oword [rax - 80], xmm0
+	LONG $0x407f0ff3; BYTE $0xc0               // movdqu    oword [rax - 64], xmm0
+	LONG $0x407f0ff3; BYTE $0xd0               // movdqu    oword [rax - 48], xmm0
+	LONG $0x407f0ff3; BYTE $0xe0               // movdqu    oword [rax - 32], xmm0
+	LONG $0x407f0ff3; BYTE $0xf0               // movdqu    oword [rax - 16], xmm0
+	LONG $0x007f0ff3                           // movdqu    oword [rax], xmm0
+	LONG $0x01000548; WORD $0x0000             // add    rax, 256
+	LONG $0x00c18148; WORD $0xffff; BYTE $0xff // add    rcx, -256
+	JNE  LBB0_9
+
+LBB0_10:
+	WORD $0x3949; BYTE $0xf2 // cmp    r10, rsi
+	JE   LBB0_13
+	WORD $0x014c; BYTE $0xc7 // add    rdi, r8
+
+LBB0_12:
+	WORD $0x1788             // mov    byte [rdi], dl
+	WORD $0xff48; BYTE $0xc7 // inc    rdi
+	WORD $0x3949; BYTE $0xfb // cmp    r11, rdi
+	JNE  LBB0_12
+
+LBB0_13:
+	RET
diff --git a/go/arrow/memory/memory_test.go b/go/arrow/memory/memory_test.go
new file mode 100644
index 0000000000000..2ee5bfbe89282
--- /dev/null
+++ b/go/arrow/memory/memory_test.go
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory_test
+
+import (
+	"testing"
+
+	"github.com/apache/arrow/go/arrow/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func makeExpectedBuf(sz, lo, hi int, c byte) []byte {
+	buf := make([]byte, sz)
+	for i := lo; i < hi; i++ {
+		buf[i] = c
+	}
+	return buf
+}
+
+func TestSet(t *testing.T) {
+	tests := []struct {
+		name   string
+		sz     int
+		lo, hi int
+		c      byte
+	}{
+		{"sz=0", 0, 0, 0, 0x00},
+		{"all,sz=7", 7, 0, 7, 0x1f},
+		{"part,sz=7", 7, 3, 4, 0x1f},
+		{"last,sz=7", 7, 6, 7, 0x1f},
+		{"all,sz=25", 25, 0, 25, 0x1f},
+		{"part,sz=25", 25, 13, 19, 0x1f},
+		{"last,sz=25", 25, 24, 25, 0x1f},
+		{"all,sz=4096", 4096, 0, 4096, 0x1f},
+		{"part,sz=4096", 4096, 1000, 3000, 0x1f},
+		{"last,sz=4096", 4096, 4095, 4096, 0x1f},
+		{"all,sz=16384", 16384, 0, 16384, 0x1f},
+		{"part,sz=16384", 16384, 3333, 10000, 0x1f},
+		{"last,sz=16384", 16384, 16383, 16384, 0x1f},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			buf := make([]byte, test.sz)
+			memory.Set(buf[test.lo:test.hi], test.c)
+			exp := makeExpectedBuf(test.sz, test.lo, test.hi, test.c)
+			assert.Equal(t, exp, buf)
+		})
+	}
+}
+
+func benchmarkSet(b *testing.B, n int64) {
+	buf := make([]byte, n)
+	b.SetBytes(n)
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		memory.Set(buf, 0x1f)
+	}
+}
+
+func BenchmarkSet_8(b *testing.B) {
+	benchmarkSet(b, 8)
+}
+
+func BenchmarkSet_32(b *testing.B) {
+	benchmarkSet(b, 32)
+}
+
+func BenchmarkSet_64(b *testing.B) {
+	benchmarkSet(b, 64)
+}
+
+func BenchmarkSet_500(b *testing.B) {
+	benchmarkSet(b, 500)
+}
+
+func BenchmarkSet_512(b *testing.B) {
+	benchmarkSet(b, 512)
+}
+
+func BenchmarkSet_1000(b *testing.B) {
+	benchmarkSet(b, 1000)
+}
+
+func BenchmarkSet_1024(b *testing.B) {
+	benchmarkSet(b, 1024)
+}
+
+func BenchmarkSet_2000(b *testing.B) {
+	benchmarkSet(b, 2000)
+}
+
+func BenchmarkSet_2048(b *testing.B) {
+	benchmarkSet(b, 2048)
+}
+
+func BenchmarkSet_4000(b *testing.B) {
+	benchmarkSet(b, 4000)
+}
+
+func BenchmarkSet_4096(b *testing.B) {
+	benchmarkSet(b, 4096)
+}
+
+func BenchmarkSet_8000(b *testing.B) {
+	benchmarkSet(b, 8000)
+}
+
+func BenchmarkSet_8192(b *testing.B) {
+	benchmarkSet(b, 8192)
+}
diff --git a/go/arrow/memory/util.go b/go/arrow/memory/util.go
new file mode 100644
index 0000000000000..3b0d3a5cb9ef1
--- /dev/null
+++ b/go/arrow/memory/util.go
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+import "unsafe"
+
+func roundToPowerOf2(v, round int) int {
+	forceCarry := round - 1
+	truncateMask := ^forceCarry
+	return (v + forceCarry) & truncateMask
+}
+
+func roundUpToMultipleOf64(v int) int {
+	return roundToPowerOf2(v, 64)
+}
+
+func isMultipleOfPowerOf2(v int, d int) bool {
+	return (v & (d - 1)) == 0
+}
+
+func addressOf(b []byte) uintptr {
+	return uintptr(unsafe.Pointer(&b[0]))
+}
diff --git a/go/arrow/memory/util_test.go b/go/arrow/memory/util_test.go
new file mode 100644
index 0000000000000..d8afe80a069e3
--- /dev/null
+++ b/go/arrow/memory/util_test.go
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRoundToPowerOf2(t *testing.T) {
+	tests := []struct {
+		v, round int
+		exp      int
+	}{
+		{60, 64, 64},
+		{122, 64, 128},
+		{16, 64, 64},
+		{64, 64, 64},
+		{13, 8, 16},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("v%d_r%d", test.v, test.round), func(t *testing.T) {
+			a := roundToPowerOf2(test.v, test.round)
+			assert.Equal(t, test.exp, a)
+		})
+	}
+}
+
+func TestIsMultipleOfPowerOf2(t *testing.T) {
+	tests := []struct {
+		v, d int
+		exp  bool
+	}{
+		{200, 256, false},
+		{256, 256, true},
+		{500, 256, false},
+		{512, 256, true},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%d_%d_%t", test.v, test.d, test.exp), func(t *testing.T) {
+			got := isMultipleOfPowerOf2(test.v, test.d)
+			assert.Equal(t, test.exp, got)
+		})
+	}
+}
diff --git a/go/arrow/metadata/schema.go b/go/arrow/metadata/schema.go
new file mode 100644
index 0000000000000..021b6ee536023
--- /dev/null
+++ b/go/arrow/metadata/schema.go
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadata
+
+import "github.com/apache/arrow/go/arrow"
+
+type Schema struct {
+	fields      []Field
+	nameToIndex map[string]int
+	metadata    KeyValueMetadata
+}
+
+type Field struct {
+	name     string           // Field name
+	typ      arrow.DataType   // The field's data type
+	nullable bool             // Fields can be nullable
+	metadata KeyValueMetadata // The field's metadata, if any
+}
+
+type KeyValueMetadata struct {
+	keys   []string
+	values []string
+}
diff --git a/go/arrow/numeric.schema.json b/go/arrow/numeric.schema.json
new file mode 100644
index 0000000000000..7fa2800a57a37
--- /dev/null
+++ b/go/arrow/numeric.schema.json
@@ -0,0 +1,15 @@
+{
+  "title": "templates",
+  "type": "array",
+  "items": {
+    "title": "template",
+    "type": "object",
+    "properties": {
+      "Name": {
+        "type": "string",
+        "description": "The name of the template type"
+      }
+    },
+    "required": ["Name"]
+  }
+}
\ No newline at end of file
diff --git a/go/arrow/numeric.tmpldata b/go/arrow/numeric.tmpldata
new file mode 100644
index 0000000000000..ea08d8bded0d2
--- /dev/null
+++ b/go/arrow/numeric.tmpldata
@@ -0,0 +1,87 @@
+[
+  {
+    "Name": "Int64",
+    "name": "int64",
+    "Type": "int64",
+    "Default": "0",
+    "Size": "8"
+  },
+  {
+    "Name": "Uint64",
+    "name": "uint64",
+    "Type": "uint64",
+    "Default": "0",
+    "Size": "8"
+  },
+  {
+    "Name": "Float64",
+    "name": "float64",
+    "Type": "float64",
+    "Default": "0",
+    "Size": "8"
+  },
+  {
+    "Name": "Int32",
+    "name": "int32",
+    "Type": "int32",
+    "Default": "0",
+    "Size": "4",
+    "Opt": {
+      "BufferBuilder": true
+    }
+  },
+  {
+    "Name": "Uint32",
+    "name": "uint32",
+    "Type": "uint32",
+    "Default": "0",
+    "Size": "4"
+  },
+  {
+    "Name": "Float32",
+    "name": "float32",
+    "Type": "float32",
+    "Default": "0",
+    "Size": "4"
+  },
+  {
+    "Name": "Int16",
+    "name": "int16",
+    "Type": "int16",
+    "Default": "0",
+    "Size": "2"
+  },
+  {
+    "Name": "Uint16",
+    "name": "uint16",
+    "Type": "uint16",
+    "Default": "0",
+    "Size": "2"
+  },
+  {
+    "Name": "Int8",
+    "name": "int8",
+    "Type": "int8",
+    "Default": "0",
+    "Size": "1"
+  },
+  {
+    "Name": "Uint8",
+    "name": "uint8",
+    "Type": "uint8",
+    "Default": "0",
+    "Size": "1"
+  },
+  {
+    "Name": "Timestamp",
+    "name": "timestamp",
+    "Type": "Timestamp",
+    "QualifiedType": "arrow.Timestamp",
+    "InternalType": "int64",
+    "Default": "0",
+    "Size": "8",
+    "Opt": {
+      "Parametric": true
+    }
+  }
+]
\ No newline at end of file
diff --git a/go/arrow/type_string.go b/go/arrow/type_string.go
new file mode 100644
index 0000000000000..9c82dbad41b7f
--- /dev/null
+++ b/go/arrow/type_string.go
@@ -0,0 +1,16 @@
+// Code generated by "stringer -type=Type"; DO NOT EDIT.
+
+package arrow
+
+import "fmt"
+
+const _Type_name = "NULLBOOLUINT8INT8UINT16INT16UINT32INT32UINT64INT64HALF_FLOATFLOAT32FLOAT64STRINGBINARYFIXED_SIZE_BINARYDATE32DATE64TIMESTAMPTIME32TIME64INTERVALDECIMALLISTSTRUCTUNIONDICTIONARYMAP"
+
+var _Type_index = [...]uint8{0, 4, 8, 13, 17, 23, 28, 34, 39, 45, 50, 60, 67, 74, 80, 86, 103, 109, 115, 124, 130, 136, 144, 151, 155, 161, 166, 176, 179}
+
+func (i Type) String() string {
+	if i < 0 || i >= Type(len(_Type_index)-1) {
+		return fmt.Sprintf("Type(%d)", i)
+	}
+	return _Type_name[_Type_index[i]:_Type_index[i+1]]
+}
diff --git a/go/arrow/type_traits_boolean.go b/go/arrow/type_traits_boolean.go
new file mode 100644
index 0000000000000..c30ae0fec4fec
--- /dev/null
+++ b/go/arrow/type_traits_boolean.go
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow
+
+import (
+	"github.com/apache/arrow/go/arrow/internal/bitutil"
+)
+
+type booleanTraits struct{}
+
+var BooleanTraits booleanTraits
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (booleanTraits) BytesRequired(n int) int { return bitutil.CeilByte(n) / 8 }
diff --git a/go/arrow/type_traits_numeric.gen.go b/go/arrow/type_traits_numeric.gen.go
new file mode 100644
index 0000000000000..b51960b6dd4e8
--- /dev/null
+++ b/go/arrow/type_traits_numeric.gen.go
@@ -0,0 +1,567 @@
+// Code generated by type_traits_numeric.gen.go.tmpl. DO NOT EDIT.
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow
+
+import (
+	"encoding/binary"
+	"reflect"
+	"unsafe"
+)
+
+var (
+	Int64Traits     int64Traits
+	Uint64Traits    uint64Traits
+	Float64Traits   float64Traits
+	Int32Traits     int32Traits
+	Uint32Traits    uint32Traits
+	Float32Traits   float32Traits
+	Int16Traits     int16Traits
+	Uint16Traits    uint16Traits
+	Int8Traits      int8Traits
+	Uint8Traits     uint8Traits
+	TimestampTraits timestampTraits
+)
+
+// Int64 traits
+
+const (
+	// Int64SizeBytes specifies the number of bytes required to store a single int64 in memory
+	Int64SizeBytes = int(unsafe.Sizeof(int64(0)))
+)
+
+type int64Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (int64Traits) BytesRequired(n int) int { return Int64SizeBytes * n }
+
+// PutValue
+func (int64Traits) PutValue(b []byte, v int64) {
+	binary.LittleEndian.PutUint64(b, uint64(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type int64.
+//
+// NOTE: len(b) must be a multiple of Int64SizeBytes.
+func (int64Traits) CastFromBytes(b []byte) []int64 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []int64
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Int64SizeBytes
+	s.Cap = h.Cap / Int64SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (int64Traits) CastToBytes(b []int64) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Int64SizeBytes
+	s.Cap = h.Cap * Int64SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (int64Traits) Copy(dst, src []int64) { copy(dst, src) }
+
+// Uint64 traits
+
+const (
+	// Uint64SizeBytes specifies the number of bytes required to store a single uint64 in memory
+	Uint64SizeBytes = int(unsafe.Sizeof(uint64(0)))
+)
+
+type uint64Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (uint64Traits) BytesRequired(n int) int { return Uint64SizeBytes * n }
+
+// PutValue
+func (uint64Traits) PutValue(b []byte, v uint64) {
+	binary.LittleEndian.PutUint64(b, uint64(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type uint64.
+//
+// NOTE: len(b) must be a multiple of Uint64SizeBytes.
+func (uint64Traits) CastFromBytes(b []byte) []uint64 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []uint64
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Uint64SizeBytes
+	s.Cap = h.Cap / Uint64SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (uint64Traits) CastToBytes(b []uint64) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Uint64SizeBytes
+	s.Cap = h.Cap * Uint64SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (uint64Traits) Copy(dst, src []uint64) { copy(dst, src) }
+
+// Float64 traits
+
+const (
+	// Float64SizeBytes specifies the number of bytes required to store a single float64 in memory
+	Float64SizeBytes = int(unsafe.Sizeof(float64(0)))
+)
+
+type float64Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (float64Traits) BytesRequired(n int) int { return Float64SizeBytes * n }
+
+// PutValue
+func (float64Traits) PutValue(b []byte, v float64) {
+	binary.LittleEndian.PutUint64(b, uint64(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type float64.
+//
+// NOTE: len(b) must be a multiple of Float64SizeBytes.
+func (float64Traits) CastFromBytes(b []byte) []float64 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []float64
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Float64SizeBytes
+	s.Cap = h.Cap / Float64SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (float64Traits) CastToBytes(b []float64) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Float64SizeBytes
+	s.Cap = h.Cap * Float64SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (float64Traits) Copy(dst, src []float64) { copy(dst, src) }
+
+// Int32 traits
+
+const (
+	// Int32SizeBytes specifies the number of bytes required to store a single int32 in memory
+	Int32SizeBytes = int(unsafe.Sizeof(int32(0)))
+)
+
+type int32Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (int32Traits) BytesRequired(n int) int { return Int32SizeBytes * n }
+
+// PutValue
+func (int32Traits) PutValue(b []byte, v int32) {
+	binary.LittleEndian.PutUint32(b, uint32(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type int32.
+//
+// NOTE: len(b) must be a multiple of Int32SizeBytes.
+func (int32Traits) CastFromBytes(b []byte) []int32 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []int32
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Int32SizeBytes
+	s.Cap = h.Cap / Int32SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (int32Traits) CastToBytes(b []int32) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Int32SizeBytes
+	s.Cap = h.Cap * Int32SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (int32Traits) Copy(dst, src []int32) { copy(dst, src) }
+
+// Uint32 traits
+
+const (
+	// Uint32SizeBytes specifies the number of bytes required to store a single uint32 in memory
+	Uint32SizeBytes = int(unsafe.Sizeof(uint32(0)))
+)
+
+type uint32Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (uint32Traits) BytesRequired(n int) int { return Uint32SizeBytes * n }
+
+// PutValue
+func (uint32Traits) PutValue(b []byte, v uint32) {
+	binary.LittleEndian.PutUint32(b, uint32(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type uint32.
+//
+// NOTE: len(b) must be a multiple of Uint32SizeBytes.
+func (uint32Traits) CastFromBytes(b []byte) []uint32 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []uint32
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Uint32SizeBytes
+	s.Cap = h.Cap / Uint32SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (uint32Traits) CastToBytes(b []uint32) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Uint32SizeBytes
+	s.Cap = h.Cap * Uint32SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (uint32Traits) Copy(dst, src []uint32) { copy(dst, src) }
+
+// Float32 traits
+
+const (
+	// Float32SizeBytes specifies the number of bytes required to store a single float32 in memory
+	Float32SizeBytes = int(unsafe.Sizeof(float32(0)))
+)
+
+type float32Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (float32Traits) BytesRequired(n int) int { return Float32SizeBytes * n }
+
+// PutValue
+func (float32Traits) PutValue(b []byte, v float32) {
+	binary.LittleEndian.PutUint32(b, uint32(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type float32.
+//
+// NOTE: len(b) must be a multiple of Float32SizeBytes.
+func (float32Traits) CastFromBytes(b []byte) []float32 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []float32
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Float32SizeBytes
+	s.Cap = h.Cap / Float32SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (float32Traits) CastToBytes(b []float32) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Float32SizeBytes
+	s.Cap = h.Cap * Float32SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (float32Traits) Copy(dst, src []float32) { copy(dst, src) }
+
+// Int16 traits
+
+const (
+	// Int16SizeBytes specifies the number of bytes required to store a single int16 in memory
+	Int16SizeBytes = int(unsafe.Sizeof(int16(0)))
+)
+
+type int16Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (int16Traits) BytesRequired(n int) int { return Int16SizeBytes * n }
+
+// PutValue
+func (int16Traits) PutValue(b []byte, v int16) {
+	binary.LittleEndian.PutUint16(b, uint16(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type int16.
+//
+// NOTE: len(b) must be a multiple of Int16SizeBytes.
+func (int16Traits) CastFromBytes(b []byte) []int16 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []int16
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Int16SizeBytes
+	s.Cap = h.Cap / Int16SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (int16Traits) CastToBytes(b []int16) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Int16SizeBytes
+	s.Cap = h.Cap * Int16SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (int16Traits) Copy(dst, src []int16) { copy(dst, src) }
+
+// Uint16 traits
+
+const (
+	// Uint16SizeBytes specifies the number of bytes required to store a single uint16 in memory
+	Uint16SizeBytes = int(unsafe.Sizeof(uint16(0)))
+)
+
+type uint16Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (uint16Traits) BytesRequired(n int) int { return Uint16SizeBytes * n }
+
+// PutValue
+func (uint16Traits) PutValue(b []byte, v uint16) {
+	binary.LittleEndian.PutUint16(b, uint16(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type uint16.
+//
+// NOTE: len(b) must be a multiple of Uint16SizeBytes.
+func (uint16Traits) CastFromBytes(b []byte) []uint16 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []uint16
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Uint16SizeBytes
+	s.Cap = h.Cap / Uint16SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (uint16Traits) CastToBytes(b []uint16) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Uint16SizeBytes
+	s.Cap = h.Cap * Uint16SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (uint16Traits) Copy(dst, src []uint16) { copy(dst, src) }
+
+// Int8 traits
+
+const (
+	// Int8SizeBytes specifies the number of bytes required to store a single int8 in memory
+	Int8SizeBytes = int(unsafe.Sizeof(int8(0)))
+)
+
+type int8Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (int8Traits) BytesRequired(n int) int { return Int8SizeBytes * n }
+
+// PutValue
+func (int8Traits) PutValue(b []byte, v int8) {
+	b[0] = byte(v)
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type int8.
+//
+// NOTE: len(b) must be a multiple of Int8SizeBytes.
+func (int8Traits) CastFromBytes(b []byte) []int8 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []int8
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Int8SizeBytes
+	s.Cap = h.Cap / Int8SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (int8Traits) CastToBytes(b []int8) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Int8SizeBytes
+	s.Cap = h.Cap * Int8SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (int8Traits) Copy(dst, src []int8) { copy(dst, src) }
+
+// Uint8 traits
+
+const (
+	// Uint8SizeBytes specifies the number of bytes required to store a single uint8 in memory
+	Uint8SizeBytes = int(unsafe.Sizeof(uint8(0)))
+)
+
+type uint8Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (uint8Traits) BytesRequired(n int) int { return Uint8SizeBytes * n }
+
+// PutValue
+func (uint8Traits) PutValue(b []byte, v uint8) {
+	b[0] = byte(v)
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type uint8.
+//
+// NOTE: len(b) must be a multiple of Uint8SizeBytes.
+func (uint8Traits) CastFromBytes(b []byte) []uint8 {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []uint8
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / Uint8SizeBytes
+	s.Cap = h.Cap / Uint8SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (uint8Traits) CastToBytes(b []uint8) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * Uint8SizeBytes
+	s.Cap = h.Cap * Uint8SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (uint8Traits) Copy(dst, src []uint8) { copy(dst, src) }
+
+// Timestamp traits
+
+const (
+	// TimestampSizeBytes specifies the number of bytes required to store a single Timestamp in memory
+	TimestampSizeBytes = int(unsafe.Sizeof(Timestamp(0)))
+)
+
+type timestampTraits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (timestampTraits) BytesRequired(n int) int { return TimestampSizeBytes * n }
+
+// PutValue
+func (timestampTraits) PutValue(b []byte, v Timestamp) {
+	binary.LittleEndian.PutUint64(b, uint64(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type Timestamp.
+//
+// NOTE: len(b) must be a multiple of TimestampSizeBytes.
+func (timestampTraits) CastFromBytes(b []byte) []Timestamp {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []Timestamp
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len / TimestampSizeBytes
+	s.Cap = h.Cap / TimestampSizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (timestampTraits) CastToBytes(b []Timestamp) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len * TimestampSizeBytes
+	s.Cap = h.Cap * TimestampSizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func (timestampTraits) Copy(dst, src []Timestamp) { copy(dst, src) }
diff --git a/go/arrow/type_traits_numeric.gen.go.tmpl b/go/arrow/type_traits_numeric.gen.go.tmpl
new file mode 100644
index 0000000000000..362d2d850f1cb
--- /dev/null
+++ b/go/arrow/type_traits_numeric.gen.go.tmpl
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package arrow
+
+import (
+	"encoding/binary"
+	"reflect"
+	"unsafe"
+)
+
+var (
+{{range .In}}
+	{{.Name}}Traits {{.name}}Traits
+{{- end}}
+)
+
+{{range .In}}
+// {{.Name}} traits
+
+const (
+	// {{.Name}}SizeBytes specifies the number of bytes required to store a single {{.Type}} in memory
+	{{.Name}}SizeBytes = int(unsafe.Sizeof({{.Type}}({{.Default}})))
+)
+
+type {{.name}}Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func ({{.name}}Traits) BytesRequired(n int) int { return {{.Name}}SizeBytes * n }
+
+// PutValue
+func ({{.name}}Traits) PutValue(b []byte, v {{.Type}}) {
+{{- if eq .Size "1" -}}
+	b[0] = byte(v)
+{{- else if eq .Size "2" -}}
+	binary.LittleEndian.PutUint16(b, uint16(v))
+{{- else if eq .Size "4" -}}
+	binary.LittleEndian.PutUint32(b, uint32(v))
+{{- else if eq .Size "8" -}}
+	binary.LittleEndian.PutUint64(b, uint64(v))
+{{- else -}}
+	panic("invalid type {{.Type}}")
+{{end}}
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type {{.Type}}.
+//
+// NOTE: len(b) must be a multiple of {{.Name}}SizeBytes.
+func ({{.name}}Traits) CastFromBytes(b []byte) []{{.Type}} {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []{{.Type}}
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len/{{.Name}}SizeBytes
+	s.Cap = h.Cap/{{.Name}}SizeBytes
+
+	return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func ({{.name}}Traits) CastToBytes(b []{{.Type}}) []byte {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+	var res []byte
+	s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+	s.Data = h.Data
+	s.Len = h.Len*{{.Name}}SizeBytes
+	s.Cap = h.Cap*{{.Name}}SizeBytes
+
+	return res
+}
+
+// Copy copies src to dst.
+func ({{.name}}Traits) Copy(dst, src []{{.Type}}) { copy(dst, src) }
+{{end}}
\ No newline at end of file

From 607c7fa686ade7a406f1393a756fdf56dd8ae8cb Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Mar 2018 08:58:35 -0400
Subject: [PATCH 1620/1644] ARROW-2340: [Website] Add blog post about Go code
 donation

This to go live tomorrow morning alongside the 0.9.0 release announcement

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1777 from wesm/ARROW-2340 and squashes the following commits:

3718bc1c <Wes McKinney> Update publication date to 3/22
416be489 <Wes McKinney> Add missing link
3f083e39 <Wes McKinney> Add blog post about Go code donation
---
 site/_data/contributors.yml                |   3 +
 site/_posts/2018-03-22-go-code-donation.md |  81 +++++++++++++++++++++
 site/img/native_go_implementation.png      | Bin 0 -> 56186 bytes
 3 files changed, 84 insertions(+)
 create mode 100644 site/_posts/2018-03-22-go-code-donation.md
 create mode 100644 site/img/native_go_implementation.png

diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml
index 19ca53ad48599..9289542fd8d68 100644
--- a/site/_data/contributors.yml
+++ b/site/_data/contributors.yml
@@ -35,4 +35,7 @@
 - name: Siddharth Teotia
   apacheId: siddharthteotia
   githubId: siddharthteotia
+- name: The Apache Arrow PMC
+  apacheId: pmc
+  githubId: pmc
 # End contributors.yml
diff --git a/site/_posts/2018-03-22-go-code-donation.md b/site/_posts/2018-03-22-go-code-donation.md
new file mode 100644
index 0000000000000..e446fd9958837
--- /dev/null
+++ b/site/_posts/2018-03-22-go-code-donation.md
@@ -0,0 +1,81 @@
+---
+layout: post
+title: "A Native Go Library for Apache Arrow"
+date: "2018-03-22 00:00:00 -0400"
+author: pmc
+categories: [application]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+Since launching in early 2016, Apache Arrow has been growing fast. We have made
+nine major releases through the efforts of over 120 distinct contributors. The
+project’s scope has also expanded. We began by focusing on the development of
+the standardized in-memory columnar data format, which now serves as a pillar
+of the project. Since then, we have been growing into a more general
+cross-language platform for in-memory data analysis through new additions to
+the project like the [Plasma shared memory object store][1]. A primary goal of
+the project is to enable data system developers to process and move data fast.
+
+So far, we officially have developed native Arrow implementations in C++, Java,
+and JavaScript. We have created binding layers for the C++ libraries in C
+(using the GLib libraries) and Python. We have also seen efforts to develop
+interfaces to the Arrow C++ libraries in Go, Lua, Ruby, and Rust. While binding
+layers serve many purposes, there can be benefits to native implementations,
+and so we’ve been keen to see future work on native implementations in growing
+systems languages like Go and Rust.
+
+This past October, engineers [Stuart Carnie][2], [Nathaniel Cook][3], and
+[Chris Goller][4], employees of [InfluxData][5], began developing a native [Go
+language implementation of the [Apache Arrow][6] in-memory columnar format for
+use in Go-based database systems like InfluxDB. We are excited to announce that
+InfluxData has donated this native Go implementation to the Apache Arrow
+project, where it will continue to be developed. This work features low-level
+integration with the Go runtime and native support for SIMD instruction
+sets. We are looking forward to working more closely with the Go community on
+solving in-memory analytics and data interoperability problems.
+
+<div align="center">
+<img src="{{ site.base-url }}/img/native_go_implementation.png"
+     alt="Apache Arrow implementations and bindings"
+     width="60%" class="img-responsive">
+</div>
+
+One of the mantras in [The Apache Software Foundation][7] is "Community over
+Code". By building an open and collaborative development community across many
+programming language ecosystems, we will be able to development better and
+longer-lived solutions to the systems problems faced by data developers.
+
+We are excited for what the future holds for the Apache Arrow project. Adding
+first-class support for a popular systems programming language like Go is an
+important step along the way. We welcome others from the Go community to get
+involved in the project. We also welcome others who wish to explore building
+Arrow support for other programming languages not yet represented. Learn more
+at [https://arrow.apache.org][9] and join the mailing list
+[dev@arrow.apache.org][8].
+
+[1]: http://arrow.apache.org/blog/2017/08/16/0.6.0-release/
+[2]: https://github.com/stuartcarnie
+[3]: https://github.com/nathanielc
+[4]: https://github.com/goller
+[5]: https://influxdata.com
+[6]: https://github.com/influxdata/arrow
+[7]: https://www.apache.org
+[8]: https://lists.apache.org/list.html?dev@arrow.apache.org
+[9]: https://arrow.apache.org
\ No newline at end of file
diff --git a/site/img/native_go_implementation.png b/site/img/native_go_implementation.png
new file mode 100644
index 0000000000000000000000000000000000000000..39f09524321b1b5c4861a91cb6e895a8a795ff30
GIT binary patch
literal 56186
zcmb@uc|4Wv+CMDOfHH<;D1>AxiVzVQG8D>?%rwayiVVq=5JEB~Ns_T-%1jax4V2k3
zB$)}B-tX1D@BMzB=kxpPeSdqm_kE|;TGw@*$9Wvz>0H65v{bfHu~Ctck!@2`Rnj3N
zBVWY-xVLP=Ps#${JjXvaS)5c+B3mc^`=~5E3P0KEplaw$Mn=s@{BHx<!vt3RaI=e=
zrt;>2jm+d!JEiXYP9h`QO{S)#py&SmSDBlO@lfLWdb!)5ixR1Er*~fZY<(!RRK1!~
zpW^K{OUt{CTlU(%OFAC8m;b^`-gxU+|ChqKx#l&S6!SK0eB04?>#FTZr$Q^<Z$Bfy
zG^(bKmUZ`Xt#yxN=J86}YBbGC*N;>-C636oIOj~<lusVfWep=qof2T8@>fhbC2(Jc
zJVtB7-+v{3LMiv(e@!7zCSLH*>wLBU`>)R`)BpXc*TpTPOrDYyehTigoQtboWMnmB
z_tewqLh8&|hwoHelTMsk@-$=JL{oE&mq9@=`j)l*<e$4Fsg7~i+lnbCGnHff4TJ*p
z1k+d&{_{~(D>CsD6L#!!o~$D+$@~^l;^K@aV}um6v_9-mp8J}0uC=3M3)wwY-gk|S
z$CZ^SU8K8(Cl@+8SZxpQYHMrb^Sn@CH2r>mPNmhs4P<(PJ8V8A1P?XF9v6xI6r%N?
zTcBJ^+*<GWV{)?oL!wNceXrci(yL3KD)WURqoUH8YEw>aNxu1e=XQ1}GBUbgC4+*?
z#JgNSNm>4Bw6-pU#c}S(>0F_Z3rW{)s<r&O?dp@)=J;u7X*ZI+G7>&)-*cCen)%ef
z&&w(Pk=$Z0JsyAnQ^2bCg$w-KIizm~1O)8QseZevCB%GOfXU=b*2syioBUWgIF!$v
z*<s>2tt`8~GTTvT&-L_qARlu`&Dy!$bRR=B|G9prT`9~fofM)EGM_)ccmMwF$B((+
zS><>3<n;HO+jf`iurHfu-n4mhzf*CgW7iY%WOLRn6y*72^jj#*41dQ6%l*6ZdVSuL
z2jVoOyjG;j|NPvqr>EETo9s+!ci8Z-&D_F5&A2P$9{HVu(Nq<GzT}t{7*&bUFi}xH
z)lOb3GkErI1^8+g3*T1U^Qy4Wvh(#-0YO2<%a?y@zVr!H+=F^>U{}eXG!~u>JGR@W
zWrW(fz}(PPr+t^dqOS_)b&4vpdsd6Z9EL7kc9I4$WlyJ1|F_tRhYYgL>gnr~-bL^j
zl)4IkzC6FJ(E4E9$4{U9T3b)EhMsUQGr{NP3Q-3uk%ykxm~}Sf{(Yv}w{IhRi;YF*
z=H~ipgBe*lIZueO^7e*?g$<15RBUR@4+;%cwy@wvVR*`MJ$(4E&$Se{#mlhO!qPG#
zGLpQaH#|FAgl5||`qrasg;D)3(x**LX~XL`l07MV_3Blw6a`+<(_=zILu2Od&U!e0
z8@Vr;MILKjUY@F|YSF;;|J(ugNSBLyh@?Y9n;i=BgUsaa{rjoY($dUt{Wj{8+I}LH
zetBWCzMjkb?#l};)m8`hFj3J{%QfcXxeDytR~4{rm&5w%0?(d38)#~XPx4-yP0qVE
zc2+7;3wPe;di?mtB=>=>sKXum&-UfjbJjG)i61z8IKXsmCichpIAvr0$59u9vKxoI
ze!onv8h0g@#~p83l`muvQ@64-J3SbsZLzkp#Q2A63&qE-uDhly11zlU>=ZR(su~)#
zezbBy=}9-JA|fJiy;|vv&CSh=Q#G{2esQ^WZ4cd+dq*@?RV!~%Qs>^7zbs_*YRlz<
zheq@J{_Ph_cBcfRQF#th>{B<&o@Hg#`EJ~N@7_HhEY8^F#(sxB`Sn@(=?4B}vx^tE
z$R&p}J+ffklyWLah*v8?%J;(uEff{0uwqqsG_b+yR_jGx>P?)5ouO-MURd?02BlhW
z0jBDyDKWLE21j-wW@@Rh<F>X!F47z;Uh;QO@UR9{;RD6oCc4xp*WSI`vYkurF4i_u
z-!!K^^~n<ohG2@ywDSQQH*fDti#4gn#x^hP3@#ek$<AJjzHs{7xsWrNXZvF-R;)X;
zDF5k-?-REly4sT+e8I5Gdwuo6QIEs)^>&53WL$q8yLN5<TEUQ0aqfi{PIhURE#2MS
z4yD~&{1t;jLTVC@I6EwKyQMyQL`F|fPevAA?^j+f$1dYa!>^shD{|nC46XhANWawf
zDl<B-iez^2wb|+Q#a=23a*bp;4sn~N-4z^Y4!XLGn>KCYyIW-@*7q^xcy8zTmQ8+W
zQK~0T9#>P_s>DrC);HB(<GwoI-um$)SyTh6l{QT$_|`4*rg%x(hK2_8Vl3tO_;^RL
zGr!#OFL73yO*d-(*>_7-$AlLPw+5KwnHgjIVU<(T(>IfS{Q5QEc;NPw++4c7d-t|8
zIgGiM5<SrPjXOPhrD6)5va)i=t4oaNzq_N=&GQUS;n@}TdhN1#xf3tg<hSA4wQB;R
zqAD&f5{K=&wqUQmf3N=P_3P@9*$rg*`T0}tc*5yglO8_2p=sIF*jR(=be!r_pdjCX
z$%1Y7C_8)mjj|;6k=Z4O=>g@>?d>&_3meIFbafAS{I*q6R^Cvgw{@|v%1`S04DYDE
zg2ILvQPZ$*FE60&TVnH+x{g<8=%$Ndd!Pb{9_jDzk9A3v<oIV@1*i5Ep6$#0a=9tt
z2;<SCN8Pbl{R_)znGOTBTdl0Dh~3K&Joud3l$T)-56{N^XJ7bKnH^J7qF@#^xtpHs
zCH;c#3s#v}eLOmJrbSOVbSYf_MG3YSMg>&G#l>QA8b5yi<W`iHk=a$T+&!7?aaTHV
zdtF@}T1RDNB^g@D_wU~w{>)uMFTU3#TT@f>EGH)rm0!B?%^A+3SGo>AKkxhe^=mb0
z+Q*#roz=k?=S!Kx<=3Qibabjej5-xN48EhsQ%gAN!K|sNDIg;9^vkvXZph+m4(yxJ
zz6J&?UYL}zsf4jBxVcH<LrY3Z9QrCZ2%URPgHkJcUa2n_?LJXFLPzC~N4BZ7TT7@H
zRW6rIYm&pl971_8jz*}FVb9GY_$^AYNjcE1qBugZ0QL1i(w@_G{5xe}jW71;=PzHX
z(AuP>*=TAWWn^qk*GlB3=E^g;`{s=-(~cbx#m_S{Yq0S)7^@HMWMuTq&p-IXr6kv<
zZ)tXf^rJQP*^3vvB1*UuG&E%`t?gyg|L=e(2vXu^VrCv_RpF}3%qgdO`Hd!w^zr-m
zJCePgjG`4QGl#5w`!nBPmSBsl&QDF<$RcW@bn+y%h;fn6pFcNV6%<4;J@EAZ-thc$
zkI(>#<k)g_f!I?H(urty8#iv0avr9^+S0^n1bqE^77snr5g*4bDk^Hbkdl)_o0^(R
zb}WCx{O{k7v$MmB=BH*y=WpzZP*qLI27Un?(ES$zP<g)3@+V25CVsd`zc75cDR4W7
zqNe6HOuZLp_lI%Gho!27TiDwEdiWjNktku}XRlu$exSkEVS5<A9yHq+WPP3VysMtZ
z;5^)<prgaE^)3e+TXla;;As<+?WwInnAwpx1I=0Q$Hwx=%gfs)E%u%3(@}56ta|h3
zXC}sqc`3b6qlLTsgCdp<v6nYWv2Xu(;{)h9jcxT;1af=V)KvRaGhR$q_U`ZBuBcv`
z?b`>s-FlVv_1kxJqMKj5dbN%}+4B}Qu*9K5=2lj>J#%z2?qPI>p75h)5vjC29JnLD
zglPP=b#*B(UeMgUc~gv)Ml8Nr%5i`~gO8DI*RIyDUr*d|R6cRS;zPm_RV?h43!dJ`
zQ|NxV%3vmE-&ivf3yO%K$DQAJ`5#O`o;=fyr<-fj%$6W~odHij*Fb2>T~^9z@ZFiZ
zqB@JJ&0L;7MAN~pF*9T~%3?M&G*nerSH#0Yi+Q9cI6c!ST4VF_<^}R(uFj(iU4Eqc
z@ES9*O49VB($Zb%i6}QE6_w3o$Ap;i47u?48Obf`OZQF7Eq?p<4b#lLewwnDHDFJ+
z%GRx0gTlkpC3pHO7N?4_g|f1;?w$zJ#Z=tQM1@&S{7Y<OO6x^FS$WYwx6rvIPhL4e
z9v;&3isfwd0aiA)Titg2E$ef0E)2bOEGoYyCIY(cHrI$@(_#GHKAX9B--CTjL}Q|f
zv8HEX2|RS=>!XSaE|qZ3y}m{I+^jT@an+rL_WVmdsgED$x=&lkF8_K7@ZhV#w~-7G
zrvHn+X_;N}^7I)udkT~>cGr`qPY2Lkd6U=LO}#^jvU6~tBg2(CPL?fb(UAaCQ%(V?
za$~ytSDq9tyS-D~<~B#8Bl{Z@QFIDSnCmI6mtQkc4QZvme$6~tzASRX!y{+WI98Q6
zC@9EBF~vtuP+3ut{7y*7X=7uWG86er0SCfJxj+rSx=UGjczAGwKw)+AD>d?uo;(@!
zIIvbyBJKHSH2qm-CXeIAinaNiJlk&vgX%0IUkwzrn;Zw!F^Eyt(_`ZF_;o<&d<hd<
zXh4tB-$%n4V1DZKX$7ac*$svI)t3t%=jGA6mL{^tCnVf$j+ab*_DtYc)%fg^+2MG8
zJ?}(~rf`-yNgg`=G#&G=FAOhUym$=jjjEGd?AsWbA7##}gqm!9tQ>VVNmETtK|_NE
zNWE`ix%Jz(t(jk>6^c5?T@z&7wwINascLE}U%o5|+^BZQ7{sYqmvwU3J?(<Ty2b^=
zm@VePO1BJO+Vu0wFKvwxK6g0d9TzL>v70xK;tf$EDeYsy1uL#Kj_ge~;zmYB4ijBR
z$YNau@1C&iFb2k((V`>+s`aaKI732>QOXTH<&J8|bs5#cIP6<p=&krU?R#vu5ZR6*
zt1k=gvgVJ!+8!oh0-A>eG}OvA<}*4?dX$;m4GrCM_;8FYQ;ype-Q9F%(*t!<9>0a8
zrKRa;HXV?a)nKMQefso+1D2aGOZi_li_zS<bLU=6Oh8J?o|(=vj|Ih?_CV|UBRBs@
z^jsg|Aa32Mn4QX>oK}8CY@%X)EqA~OcyFd%ON{k)%LvbxcR8`op9`Y5U?`hob6XY0
zvV6qg!1yr9iCtOH+ZSWszPQ9;zP5Tuu30?!-7T(-4*RaT?4~P9`$iHV7Bwm|K0ai5
zvVx0WD`9u>bh@HoG`EO|h{iqI?b|Jue~;5`p&&L$v(53VSA~gL^YWWU*zEEu(2zN;
zZJq&5n$Av}os++Q)uMhWvd&VMnS`SO>{361+i+oGV(P2(-RMwkY(5ei99-3wc9MU7
z+=_LRwYBxL?CiI3;ur7%pas))ET)y?uBoqHF`?@M=$m>i?G@a=U+tcwl@%`mV>0yA
z?S-iPom39W4GZkuOL^(ir6?*#Ak?t1?fK3>9i$w7|1$JknXdmhHvi!8RU?D=>FH^L
zwxX;sqybd9xw*^RXC0Q-Rg{zngdq|eQuL#?w)WJz2i1<n`;(j?N<cA|^FP11k7X5_
z<rzGcJ<QEbj&20V=htlG*D_KA?)&@KH^02R1HAnFH5Vlwr=)Da{5qBS1+xtF8~^`f
zY>fKcv3EmGctzk*0goFuroLav@wGN2P~m|C2PAo?urVWIV*{I-G=Q$6S`1^9(Knez
zjHS}(v^6#B@~s;`e*Eb3DP5a~Zp)*xG7juYsT*?w5^UQtDjUzee2fm{G^8$MkV|Hi
z71LvY1`&2)qN}*0$kDPRTcm;vEZ|v2M%CKta%*?Drcj^9!bDxkwK2XeD{XB$D3+>G
zm&iXny|IZOuLM(*SoV&6%{qSl`VoT|fAkY`^F3L%TEv)|<vf8lMDFXOG&8$)FDe4`
zm)Pg|`3x)~#y-81{1v3mJ}$F;`}Vm{Rc-E{ztP&-x&ibpr#%F9DMqs~LiP5$cdDp@
z-4GTI9Xf=@4Q6`W+glz20yZ<)AjlyVH8njKm-hGm{ng|?jYHga(xm#@g&}oIa-WqI
zPa)$XI)D;jdq4&7V>}fVJG+S;e&(^l=K?@=V@+;sMT@{aqs}hc*=;{3q?)EPFf;q>
zWec~scvO!Mp3mm0xzSH~Hq8(BRhSmqae$A$w(r$C_<S1;&F%4t83Js`$i&yG<q63s
zu+C1%tb}316Nr;Fz_)#D?<7t6hhT^&8hoWsi=LT@ZHaB946i@_orTIDbNpU|mA)n!
zxZdN`)P_>a6&VZ9J<49w;$AeMJ=_sNzgSIsHpgjf%Fw&<g<`wsz~=Jjvh|KnZWbGF
z2V&1@KiN~T;^FoD^>zJ>Y3C1)>{tg?-djrC#ZSFX6Wo^9Q8Zeeo{@25nTfX2fpxls
z_cKJ>sOR*c!AMfk)}}?v#<k+(YdE#Afwbk8cn1asrl#w;NMNeD-)!;hK%7y^26+~g
z6Qz@K=o)W+f4?&5yF04kV^7caM6Ez96;N(d($QV=>q{-uJsy)O<>j1!ofuTy5y#R$
z(Mz)V9`m_<n*vZJl2SG6tm$-BWOTH@KQ+q$P~oliF=dd%yF0~mFSb6;l)Hx==YUbi
zedy4RWUoK_!q|`8!fw^6z0>kKDmghkdcXdF?`F<pkO;c(q@5Cwf40yZtz?fIadL8^
z<Kp5v%Vl*i7DR<DvQ}X?9fswj$B)U?)YO!0O^O@{nuFPn0Y^~pVyC@;d>@jIhSUVo
zrk0f@>aixv-IT_j3c{WaEUOpc#~i>62OO*}b{@fu!g>LU9svCkGRz-*K27!yd&0<9
zMuA=*nYMenVo^8hGLRZB$6Y!E(Lq31SXo`2s-(2^^qDjM-@h9HGg|n>1BJhT|NhaZ
z(WeRaVI;GIai>g8!vWb2673!$O*Psch`ldv#2D9&k@<u>QN~S#Sg!9*rQMi}mD8m9
z*0whC_~sKd)@b?}85!UcCoWyuhsL#INv}`0oI3yI%jwzVb@p(%KLXduwBrwt-x&};
zc#whEym+Cp2ut3BD-sWLuj&MtKWe>5C>jLw-sI=27kVZ`j}T)#J!SEJD1HkI3usKi
zhxQ&i6iuOjfW{BAo1iyfB$XdO(lwlA1VMQ4fK}(VdTMJQt@q+a$KVKnj^N<p$Kg*;
z3DiP}nEHCoK4P;jpaKH}L-uGLR`8Uu@m>5a*~LCT)?K^&>MXEqK1$pQcnx}2^G!n5
zjDqxsG29S<-bS};SIDzx&$bTCVRtUhebP1;V-0c_6%wLhT%4bu?<l#p|Hh3QyO~2G
zFr_|!*8A02NV|DE`|j&Mmn;J6!#HZ7IpkV3Fs#g_%Wq@MlieP;vD9tyZ6KFd5P|5M
zZ6ZTV1)~QbYe~6J?UrC`-k~g)e9u=qZ7Uh%lcL8n<e*w440si+xX(#P2aqkie<gv}
zfbrWl_eJg>OF15R%E;&rA<uw4MwK^?{rs7GbJ>+$)}0Q>*ow2(;s8d~GytIar>B}1
zZEcmVT{{Hn07x$nK+(#^CbGEyllgk@`YKl#=g~U^*dBL<V0X|fLX`qdpui|}^i#Se
z{&Ep8-fkwHI2B-!n8$j82X6kE8}p(YbCCvByCQ9p`GvuB7I@uEOsYzen>pmPo}SB@
zJ<TE4*Y<oU{c~4^JHia(sr@Jm5c$y1kfdxZ+Ru|pcKNeC<G~Ppy`JG?F%AbJtIv6L
z+exBmzz`f)79A}@nW4Uxxa^}JCsCHSUR2W8XUWJLn0D`TDt>nWTLV-SD$9?F2?%?b
za_-{=?GE^RbH9IU{Yd_1d&1sc^n{8^L{yZYbz_Xbu*j{d-L8{8p#(v!59dmjXsCsH
zBx3R==*bga@T+|zw@6aZvFGOJBkoM`MFq3^yDxe!y{U97?Y44TuJ)&9WTroxxdWq*
zkjWbJ7gr~}xqc^Y-{kiLXrpwpJfuZ#nczE&e{3Y!2+Rml8r$d(J_t=9zp+Y`Z&9vd
zX*eFmNUO?KA94eeskD20RKsnMWfvEqpe@IsSd6x%&4u3rZA`3n?_IQSjDCH3@>QOW
z0MmO@@on3-p$4xQ2&x9wY{0Mh<^b^v9|(bjGki0E8H5e@A1Q%+-OAR6F|y}G!`IpD
z=?22hSvNs)O!l1Firx0Uu`%V#W#}8Not+`LxFTZ_2lhbxizh(OyOtJya(S?etgLNZ
zUW<DOEe>;MWepn}9gr5!J*4hdOboMxq@<~CN#pe7=9E)X87G?;EkW)eZc`+;1cKlH
zn7@GmiGl}Bj1W??w6yFfbrXd=&VF`hd%uGj_6ngv(Jx@0qql$s27o+tj=zNvf~I+L
zIBc@l{djcySRHyfQQVNE{mfZGD$U>C-3jug0vLv3ftUiV%Lk%xQMVm7z`Y_iC?0<T
z75?L7+^;xWzBoYbxil23>(xSS5n~+zaxg#EPRcHwcqpMfJ<k9_a^=THT3H`Ll*Wbv
zlI$qB!eVlLXotk%!;vJYfhw*HJ}1kku$50hgv-l4nWl3;CT8F-x)(CM6%@3E=%d~%
zB(6NSNfWUD-nk=l%d3Ergh-fleRwCKu0Z>Ppi@uio~b1z4}Y=ak`v^;IWYv$#}qKB
z(COrs1{onQLl1*iN^EhWn}TU%49)x*%SkOLxc0vK93H0>Ksl(3AqAmX<y5RmM@B}X
zfGQo?e~yh!o378BK70QBR_|mW26yLpz&G0fW0CaIPBiYnBFfU0A*21_jYCcs3ykms
zi}8w>&r$j^g&KS{cXsU0wQghvNCFc+p4w`*`ez>RMM$I&#Dgh4JtIT_b1_e#9=0WS
zbQrAsX|EtggPO+aQ>T1-*M1km%cx5_>S3ogorOOq><oeZ`*}tnBG+Mf-GMA2At6Cl
z^lD5@LV}{Oa~l~D0SI6XcI2qOu`&DC9Mf=OIRKe?73v-yy3*$$HT8Cv{iD=W%Qi2F
zYX(Bh3@_h`(Lh4^v%IVp?HXWyFD@?FboHlx(MXcVBZv|YAKqD9bO*RR?Wp~4HE@vw
zJpjD9lR_m4wTO`JglN}7xuexjpFJCZjy}-SAV$N@!vl(zq9+(!S!k4H9&;uAQ!o^1
z?wDxxz3S@fVrYV5l8jKY1_!MOcml6V#m#L&GjSJ)_T9TQgTuoCiTZp|dw}XOXw5^|
zfX#H&&Sjk?>cp-vUg#CTW*+#Qju;J@nsFW^^j|$?9_qfL5vmxKL8>EC+W~1Som-9u
z;xnu+T;Rc5#l^*`s;O1IkKPY0Bzxbseg0IQ|JZomlz<ErAuOUvQ}7^eH4YJ_MC@G%
zo?wCIwzf3TVS!W(3hlZJMXdHR1m8<alG|hIg;yJ8oxo-SGp%$qHeh~mN`L}WzgFI1
zUz9O$?Gxn;)&K=3r-Qg+`;y8LT}4w9_Xu-Vf#~<Sw;9>%>BpoUF0nvK5PR{_0Pk~9
zT%6zP6*>-3I7YnxDS-ruM+TtW#J&v+<D1Yx<4)I3J^>93a0kyh;@&+nur%Vi4Gl%u
zIukVTuZ^%0{SxD9WmvKzZcYXm9zE(;N8U!Rn{)IZlI3a1fZi*!&zFi_{r<{ccnYM&
z%4+b*=uS~lv1?1@K80Q3EZ>c?Ty<C&1I+2@=?_@X&)S!&o)(Ob+Qby1^mu>C8^Fe*
z?g_ENJ>ZSe>W4-rF^LQ%H29{5oQ`0f3AyRlx0jnCzQe;9G+td@HIKXY`SPLR4~ZeL
z6)hVg_X-FIEPl<_|L-+-op?(1b&%58)%EYbhX{-&!qDh^{(Sd+JMQ+4ll*dV&)$4p
z^l9{8*E7y~)*cGAMK!If;g5IPfvFwdIR&P%lVlnJCsM13#YY8ut}TsRw6_oJ9Ov8N
zP(R6x()XPIqE9G=Y;0^(10kZ^a&k#`R=9Rc%-kY;)cag?pe2ruj^<xoLh79A5@y<K
zLbuqjNs_!r=5Lro4B4^7USBmbd)`rD_Uf`X&*KNyhN@|8TS<_zqQU@!V4u=wuIR0{
zUM_GS2<D&q-Dyv1mh7z+Z;|zby}4UXj*}ty_LBocBX0p$K4<6>y*%T$){?n`@Zvv9
z=L3K*61!i6Jy0<v-e#zb{^U_E7wMj%w>B?Rva*Dm6pxzB{a#qGba9!BX=(|*yP;@M
zr)H#?tHS*<@XF=Omkr8S!KL25J;56KHZv*2?dE{gb{G(N-acq%UoO{SQhe|1B#au^
z6IDY_(A}!QfB!NBNg=TZ|C-tNE4W2+q`Z#6j)3T0)eyxW>|n#XNjfo1PqBhyD$W(g
zg_x8S!Jsg$pFZWo$Vz$jO2XFG&(CjhINsh)jdR7`54*(Ny#IE&7nTDTaqR5bvmGNK
z6Py|OGCRSz3BMM21O5jTqXW*v8nQF*cre%as44xbe1Q2=)6yst6BD0Qm)7(b?Y*S5
zkLbnOuNpi5%>ob)ZYiwDp`RQ$<>;szS`{j85ep1Q<)j>10%ko8V{+lf481#8hDQYj
zJHRucwSwW-yuRAmH|?&<#|AkV^MLKbhXmhlyLFT13dNLTUS4u-pFWuv7=aU;AB=N=
zz49bIolrMWDXyg)pu&qV1ksHptUVwGEH0A59pQaIiz2H8`@aWLb?MX72=-MBu~Qj(
z=63oz8Ta+p)Bbh3hjQ+N25bTQs~9*y>Q(?_)XAuw8EV8ta5%Rt64gKuug75O5NHWm
zLW+H{(RA$=x+?e}0bR1PvN~Q~pv6p6;%*h5uBkYUwgY}}Hp_?-w}jrt%|vDH;=%-V
z{~+`#`W-tc0Y#FMIAH|@)eYp{Pa&Po{PN(4GvCjjKLHXk3NK%|GL6S)(Oq&4S}=Ky
z7z~(dKy<=KHWO35b16DHx;s+i_<b3rzVPs=VV=SI<%n)I<B73)m0Dea*b6d6QR~|Y
z<TdI1)5x=a$t@mtMPLQx5M~$~e_sDS&&!YkUvIyDcI}TJ0%%LxGecZ(_Aay}QPzlI
zZE&;i-oJ<33tHT=@RLm-T3z;chb6J2TQ5S#;vp5ePjef@P}Vzwxf>KY>;Y2$2=o}=
zd^jZw*wps>tF3sqI#^h^E2uz(TLbg(uEEGgUou%v+Q0s}GR#kAAp;*G?Z6k+Do-Og
z#XsO8op+^O1Xf#QSN7!sHW0%TIy!9}d%c#j8i6?hy`i%*v9MU;0ZG(-zUtF*4;-Q5
z<q|EfdwG?SGYA9*w005Dqu)UaPZSS64@wU&AD>PN$dg^j1H4p{iAFH0;gpWftvh$9
z2y+R8im_(|X6~R3{x~Y>%F2NiN8R}mqp%$zb%ZptCt97L^Zkp{B*G5ei18Sf@$6aU
zW91MBc#GUqepP)G`=#VnNG(Us9FEWAs1fg>llqqDsZk^9HBC(j*)Ln<0L70Rm=c$C
z`HSES(+W-kfwlJZ$c+}GkrU2hHMOW2obN?w|3JIcg`G!J?8vlDOgN$71#R1P_*jNs
zn>>0qe?4q~=g+CYwf(<b#!?Un6ZsMbM#jl!4QPvtKQnVq>FaOKX{Uy60{F9;)7^(^
zW%3?U7kX#TRM{T3gmA2?tz9uBcUvTuniz19K>*4i%@5>WnQ$#_g~<-Ljuq5m@dZF7
z3tR-E1>B!ZfbJ$c_whIa3rTu<vn^tvV8KcZd=rP*wjDfp@RWf;(Ea=Ow^mpiGQ<Dz
zZ5aWP1uFMJOHx!+e838RT3bUaPvcib))#qF&Dgl>b{%<RKCpZ}k_2L6V#jLta|uW%
zK`41)T+9H;kFaAAMuBbK@G1%n>H6&Oo+C$gvW0Gfy8cUbaB%S6f<Hb%E4}7>!Ijen
z23tVkL8nihqC3l>pK{6!)DTYvelj5-qb*UiGsjqC_7E~44AX!7Nnm6xx;B`9*dQoB
zN=iz?KY|$#Dh8Piw<c!!mIfczhvD+Z;-Lmq^O+eZB_*XtMMbQ_!ompY1mad9EthV<
zRYUkyLMMZ^0q~-=uKKL9euw^ZzNoFJ1FPjAYeOg9Znb}$R~T2-lMBZMqpNKX_d1eR
z%8edF;WIjS?zEv{Fu~K{TQ4mwLB=mjt1i#Xq$YM4umrZ(UnSG%m4HUy?6)mQ(x9W0
z8B=+cF#gjCHebJf9Rd%whc1<;$!8Ohi#gT{H>++@LLUj}o&3kc!3)f*oxyzJs6iRu
z=+My2Cnx9C_WMHfL!yz=A7~V7x*zRKB9Zc3$BhU{5!e_{v^GbUpMNt17vE!KK#mZ2
z_Kt&-(EEj8HB55qVI~9>AP#ga^9#hRTM7H2x5%%KY6pu}sPZZT|4{h0h|%0v-^=N>
zXwEPt2)R}#gAT2-uT2D%t$gLmegq|;XhGvVaOJB|Tar!&FLm~@-DH_x)IT(jv?e!Q
zg#ApUil8OAxVl=nxZMABnGlb8d3Os7hd!-+VjN$|wSKRtOVEM;V9&_f*)t8P8nvY9
zK3GfcppcF;gKF^Zni6FeVp_aSO}PjrMSvJ6Fvm?ydY+V03N<Q1zf04J5dI2{4^8q$
zIABXBas9$OoQ<-6-^mcaennE$E+;@e?Us64Nwyq(^sZ~|=cA}uQ|JI~VBg2jEa+q~
z0n_R;D=O~65YN_>*y^t<qV8VR)7t9W3)~2Y5Gz5Ds~b0@iELI;k<?7sw7cw&(b28@
z#omglzZHGQ7ta`a;`ln5K(7gQV<8(NPyqp(h?Id{p?9VtPy2Bw37}<Bxz1XA9eWwJ
zVv?fj3GRJL;P$UywxCe|AXf6e$j(S(5f!{1qy#1)lqH115V-mIC6mY(fkw=Zv=CVw
zY=phke5m)8m6a<d5~l?SMXenQH6k0yEh^+Oi$aGA+t%fP33ib8yImFOH93oH5cIyY
zXPL2mq{93(_y{YZ`G5wI3%hkTQ~C61`V%Kks3v)#JxTVsZ%p=HJ2L2W1g|Do(w{#;
zo#_TbZ;^E)b{$Bcbj1Bm(W&zyEb#D%>=0CBTo4c^;Uw6+?8kayUfzPw00jUa1yc{?
zS)-QTG%p(-0PqwU9S27!*gmj?_QT-@cu<f@YG>H+Q&1^lAH#Y_VgDnu^HeFIlqCy{
zOd;i9D;|)LP;#0Pj8@0z@gNidK~s0_H9S0sSnzNm4`Yseyj(!kjin_It`g7%5fg$k
z#K-Sqmar@^YW@8AHZTNgRh5?k-U;EOV1w!x2*Lk?M!{$*C+9&^k@z4wdI$4^L!3w^
z;N?)e0R8a@AXC7VN42(ztoOBDl)$7`B!HdYen=olzd-YYxI@!uW%lwLVbVb>d8Q}G
zyLYeV1MQdJ?jjlhxv1)LL3O`_kF_Co2^>V&Rol^87~3kgp$VCZJwn#UHYq`ipFpdI
zhFc*)08OD1k;ee_MU-jfJa{ECZuif6lR-hgc;yOFa%j#N?g*!$gfK_$#m9Tp32lz*
zCV^OC#UVl|L5#ptLDcWvyeJ^4y88PaESyk+;ARsm2Ym5q^fqjfIvt4bQ0QGxlDKf!
zLt08%XGy{QNj~)?1#9cQxHc3Cav)D%80}{W7DHc<o&T&0@cJ}(mq0JAul>QeT<>+S
zf3hdL5$Hga?EwU11UMWQr_8}{&}NVCrrUks06hx4{^mk=Ae<6BX1%j#3AK~p_UQZj
zm<e^e3&aMMoVdFUTIv)bhtxinCRA_OM;M)(X*p|wbqELq&iCWTMa+{vI0SD24(>%o
z&1T(^ho1KN^XER|jWaS7P)#Ush@SK>iR+U0mYbQK<rg|76;?CRU3ywqcOyoVPR3L!
z*XkDdA4I2bO;u_A{+$ZI9NG<mgx|bjN2uo*wh*rA)cNy-J5E?>V9k(VZ<KKoLNsw(
z$fjstrbMa<!sA<rq{KWyyCCMx$m|}tb9gOQpGd+CH^uqkGPlz1!sl?vxDuNIW+;q8
z;!j~IoPuZ%Uy9HKQ1ZM}8JKz@%)*LrzNX#gkeY)dNn`}8suUQ+YLiKywB`Lzq!Rg$
zui58MQg~qupaO_RKrjfI7J}wOp%iN5##BIyf>@wnXt<N`9B`#PxH$HD!a*TuP%tBZ
zwN65u1|QIiIbs0N;k;)hy$OvSpr5dMk%BdR5b`11YmNsiph5QuE-0iELU;l_gL*=h
z+6vO7{G8gx#^y1o1d&ZSd-m%SMelr%6)f0^-E@_B&;*ab#Z}oA0+{YWg}`ey$hX>x
zRPB)}ZE?CSX88CoU%n7YROCNE#Z(1FPADr!)TzMvBQPzT^wx0;!eu{!?SN<!C-x!2
zo{teK!t_VY{EZ^f{gX51i&Dt^V)6aeB?$SIf9A(7_1t*h>|06&-=Q<td}F#!h7=h)
znaG|!zA}iUj4Ui42_qFDyW+4*(ryqD*LpFU*ar{Z9feHW)w|l=>j+EJk~Lr@#+)_G
z+7Q3&fR?*Jw~Ao&1)oG?lnIl^AKD>J*#)Iv^+Wwj<Q=ql86rr6f`UaA>&q1*5M7VN
zX&g=wj<Lo+-(LNe?=N%hClkvSit6t&c%qW_J+fNd5e{Th_|v0nizLRdUAuOXTz_>`
z?O<VXW(c-YtXP`O&=bsl{P>oP)8Lj77wMYsD8nwub*jAuM#2lgx(-*`vxnf~@F#9g
z(`h)Pr{{ppG`RNLem&--s_KI|sXT)w;_X0t66#4?W$Xo*2GEF$3E9jTT&q6+*OOJ`
z7e0;6cjb7mUdb*fh^F>eG#^K{Eg&8ni$7ZZNPu|}_Vz@&{OZ2q;izT+9%CCDz6H_s
z8PSAKqk{le7xD~>lGj&82A3NrcdV5R4G%j_R;<g_JF??x7lUEj%?>vw7-c2BS|{Zc
ztMU#Uw(Y1qqpu&ew!YxKE*YDgEI)J<%fQbNTvV|#B%07WIYP5(;{t-+4sdh_mth)D
zfI1`~R2**_Qdd3RRUCB=CB+}rz;}0SX4w&vWf%0cBY`ys`X?u6JXSz9hSo-t*MdtI
zdOVI48_Qja7ix3@sT{^5rgxnTP~4N9o*w8inigAx3E1}Z*zP{m8gakA$BxOxs;70~
z85Fh1uX|Rno+Lz;OiMA=BEQYFPV3WL>s6~Wu@!?vLS>>qkZTxFqv`1C+^4Qd?{yEm
z_U!Y<fI*2u?=_X(qnNjvufKi!)&YGJGiV^PQa_nQeGUpd;S)Z6N(3}P$ng|uYJ5mi
z7I}kG+X*Ufl=b$!TQ?;P*OgRJ+S(UcS<1+>;i+KbB9{ZMjNHx+)uWuWxS<VboZicY
z-hL)I71>#0zyo-5l3T?x2bz4}?2_squ^vr7`DCQm;qy(0uP=tXLm@wHWTYNe{ALn8
z#QV>uWN5BPLV&9AH^tTTA44`B`9%1XP=Om*myVz?0D_4&h`j)_5gLK=hf9b%gDMR~
zY0K?d>N1Q$5cvs5`8r6y`$(c2_$xulfc!MSdxw#*)q{{SskX>N)I$3Qi?=aa`R~cz
za5Py<#0Li7cv@F9`dwZspRV8FgOC?e++nzrs9(5ipB0kumzC?F!1x=qaIB6v58o@F
z9uLdWqF$-cQ1=XA_hOGVec3?no2bQ)CON$_+Y;5CI!)^9AL$@Xo3$hztu4EVy=-2u
z=^8#OBLm27k@w)0q*s!5g471YRAD_tvbQ3<a^}pL#Tu?PAEYeug_|btL2-Z|VQFPW
zfyRShuw@8Ed3S^=kw+4SP~q>yfiPLr0>^sTaJTf5lz5(jM$!<|`{scdrOh(~fbW0Z
z1Hy~Z6LdsU<PD4=$T)sb?7?{OOHZ&4LT{c$9Tnh43bdnhg?4o?ua1BJi8v&pM6gVs
zx?~~3pKFu_VfF2Bb3B3QVY_(FHcKwzZV3YycoL`+)-a)*!iFEbmbNn!Q4C_P!26CY
zbVU@(z0Y?uLEnLAqxiK4W?SUzu>@fJM3i@Rd4cEfVdkAXcSgP5^*1+k7wQ;6^H0W#
z5dJ!5%k<jvWGYw`LHVzaG=}6L@(RWWe~x%Ass*yR1}vN<_PtVHsj<kf?*-Jbfcy|{
z^?i&TDSGyWfiDvLckkXMrg}&DO&QQNpjn4s9S4Zy*p;s@;tpPm1r|jmB1oL%IirTK
z<MA{dsEL1_D!4|}2v&n8`TB0aUfl$&i5!H%tBdsTiHKw^hGb-p6tqh;d!>JzXB=nP
zj!Z0Y9J`bQ8T39fk);JulQ*n{>jFgun~Ts`kuD?xtXOHxK&ZjMv&W!_Ln59cajmmr
zcoB|KhQ_lmm%){TA|n~VX^`DsEqQ*Z5Q!}!#|FCXD5(R13T>1aY5+FyPcVJ^k=w#X
zLRN&3bYVRZNt_oiUhG>Psek(xJ1`<KF;w{6>w8x`a0hQ@Kga<3@$PLzj^|@XM-X};
zL>w@+fx(6-_%4)F>vu|RyNc+k{BH@76WJg_n8QjCaY+ok7#LPCL{C?oA_nUAMW#mG
zI>ao!&ydR_xCrc;sh{%}H1l3^vNpvzXJ>Kr69w`e)(3Y2|Nnbx;9lbp)P+E-#kq64
z@_b5g=3;){pVC~5FLM6IQQ~NVZAb0~z;{?+L~a0)z_r!hbq<89F|%q1-gvG4$)sAD
zj#+XZ`H%wP1$&RCX7|yf9GGBr_;gkOii8AqAY@2<P^vKRi8Dbjk(bfw+Zem@8xz9^
z&tx|;GOw-;H|@l!g4@<N46)c=bE#bB5atn#M@AIDlSuc#o1||;zUq~IVP~H6h!(=_
zz9cE&b>wUYa0Y>h55OYWfLcHm!oY(;#|EDQvEf=^$UM)bnSGSNTLc9^PJ8q=k}h{C
zmH=QF!BRkQl%}K;wepV*gxp675cLYtOsTKFw$^OueKa9iE2iw;x9^PL4gfd8#=s{W
z&uJ%(rYHaH>Tuo3-ZeM-LnL%_bL%*2CBYW);6V_W74|*>D&aixk&x+vi&n!EE=!Ko
z3kZZoRRtS-UF52(p=ztOvLG{)b~1JVDVuua7tmRt_(mir)1sFFf8OJGfy4)4Xaolb
z>u&1V<fj1UMY<rryfb5N3mNj;gl&gruBspn5)7W?v<<Ehk-0y}8Dp)M(CD75jkiKH
z1Ck;)dIH2_fH?^cfxkI=c`IL9GQ2f$Vkb|G{1`$O0OCZF?p_Ph7U)lbP<OaF`S&w6
zmoDF>4SK^43k^zxEbtDr9p2=0oRC37MiTcn&V1Ba>|wS9bY5Is+;7Bp-CP_T6|rQU
zNa!Dr5G`(u6{X$Zn~2~MM9-+cFX&#d3t*&c+}v|p5fUY0X}5wr$e_xpAku?z4yAN!
z6LiL7K-l!JfRG8k2LR_-Hod%RRldkUF4rO}zW`W3kd1f7AxhkYy_b15OA&PrLWZUU
zahZt4hn^5mQ<g#QFaGdVMLRo@xR-b=Lg*Z5dr<g_=nVg!0}!J*uJjY}p6YCZuNxV$
zIXOEwjIh@erwAZJ2|-_K{q$)g#uowv@j}&*zwt4UcKC|wrlKz_4G4klnOC3r8+Y0x
zDY;NGxrBvDkiJHxl^)gP0mAay_S-GU(f*-VzU5m(o0oE%U<as!!GAzn`p(Ve-$V!m
zS{0r(a2pYYF|WtE6Xq4~7FIGGm)W{ySXhnJ-RVBIa?ZZYACd;D7{;kW)>)!s;~)z%
zj1qkze-G&iPKg5k1W_icfPM%@2Ly}Yh)F_1LT<|<DVqg1($&=^me<S!N7ck641*`K
zq;+JLI7)&<wf?Jt*)=D)AV@$zbkO>H5T-o0rOd?MU=t&@I)bO}Wj$(zKj8s9S;Z*B
z{UBJpo7_|+mIuv4qi>pdDPy>K$f<vzE~Hla1GOd!a$;g4H@_R*nebXyUwyNMyaUH_
z=Kd}1G@U*C{1>V%)C8j=XQ$QzW6#yrFY|?Q>58!`;cVE|gEYtUDfdF!LgE{P9zr7Y
zde)sg70|BX9w1gl8Cc_3=Mk`Qj>}uCM}F;wv9r&~o%SFqs`5Jn6u?k0!}-g4VB!<j
zLqK4lxddB3)LtYHp%C1@eOpa<*t6tg<4YbjUIt(S?FY-#mH^l&z3M8z%~gNH)!R8`
zgJ7r=0tPOGNEG4}3rYo2NB`^+@<tg~w*TE@dVPL_eq5rFPjvQS{#VyxWb>|7$i9&(
zF^MH2!--mwqmzp(%Opy%<bxl_T}`97*5FGRvQG_M8u^e2Mj4>Qy~pB?IV)N!06;*n
zAyO*XWKdzZUY)I8b?UuRp3lMMQ<E=rUw!Dwh9lS|y3F;c0b=Tu50gPo_94_IA{|25
zrSGQdk{6Y!MPI3?sPs>}^X*vllutJ@Gv4PgcLEED<Uiq|5y>3L1dm}^A-n)L6_0ta
z*UkT%?OS;MOY5NF2o&w>eh4g!69}996%{PF7d;O0@L0B8k7*o2{GM^bu{@@FXuxYl
z_JWNKadvGHfl63bO1BDc%5kAnnMtq_$svrx{9~qw6=u9~FQexC$02unR#{<OG#jVp
z4xs>zp;~D@l<F9vvZ_j4>faM!&i9LR>ef$nJ9!fHLEhY$^;H;*#Ca=r8N=SAoZ1FS
z3ysT?a=J~6u5U$MpGt=%cV;LV0CeB7d-DT%<Ak}z#^p(1>xlFQx{$gksuuf~X7N;r
z!&ugbMI5sQS%$=F9PV>6#tv?Gd3pJy#^AE29M1DOH6tq6&>{ZiP<T+#1Bc1Z@!$I_
zH6h(GQPth{STP#L#|$8^QZ2vwNxr{qp@(NfGs#VTaoHJ!AIbsYY7roez#?dh>Dn59
z@7e1ed9t<P0GHR^ei0l<h-$FAvpBotSsQ9C7~xS|8L7(K{>y7OLogqEJP-$B280Lv
z*xgee!U+J>(^t8smrgque7@VBFTj@kJ3=Kx51G_mIJBgg;_!ZL_RKk5UEOqzp*qoQ
zzWC*M^E@u^(D?ZHX@c-dIrNdKMg8Yg&y%z?g`D=|8G06GW@N-s#tLy?QHJ2^u7{p-
zrX67odZgl&^84y(x;;_kzGiJSGVRj!(0C1A+H6B!2@RKsY5=Ta$eWp&?S3H&#*ZF?
zb0tqorlDv4(2e@X_kLbkaaKVxZtBfx5x0mU<E~i|K2l3o>&5IN>o3&Ps2G3X2n>cc
z5jn-lVG7fU>9g|)uXS{EY>if~*w5E_y7i(2AsYY=;CWU5;lgo@-s@Lf2%9BmO^<xm
z#_%WqOZ9jSJ{<BB(CC^~_>5t30{tI{htYAThhj|!h<tvEbI%sR=sQoU25)u=GdJFM
zWbsrsPye*@b>HVv7u=_$WeSc!Dc^cUpg0&q2>1OXwll$fC3YH#a7eLnreKZi=Hc<P
z`9#RPR6~J+Q#o#X9K>IgV)DZ?B|M|PkVzM5$F-H~1IJrJGSEaaZMODzwvNq1JT>p#
zN<mI2Tt!9hil*R;*t4iC`VL73Sx<4`IKrY<u6N(VNl)(!p+zB0$A4=J0y^jq*hmO2
zLs-2i@o?YlOK<(DBZKv&R6Br~mPV3k&s9R;L%PJjxWqVQ7d4le2M_nz^iO+)g|~%<
zhSrX!VY34pA#h4{1u_qDd<8!9U=kCKEW*iARIC<`yMU+(Gy9u!->{6tnfA7H7;1$2
zghQ2;fFkd<-v$i$mWo?P|MRRUPJKKK0|T&>IFW#S8TVHVbsV8M1!Xn0)%}w7?E1MU
zu(Wb6SB<c6L2H8uwl%ZJ13Uuf(frTQZ`<TYMA0~J&YdQNNJP|mk~}7kK&b3_U5rTS
zyLaz^^7scFadMD2qzi+>z;zcwdh#p%w1ju**n4xqyfH?2vE6&M{hZ(wA7ihl(aDNl
z?O`|1JDQvks!XoimbxE;afmG>G%!cSCby5xSC6^`oT1~acvs4RYr-SpmeGM22jojA
zBghS?UF+_c_-Oxc764U=G5==%>pAGpkZ_^05GP4-7VBnXn~oPi6_N8JqNW7YCy)e2
zi0Uvr0)#<0ab&&<ZDQA98@VJvPZZAk1+M<Y!;b)^2?7C;nFyu<W_)aGyLHs#_oK|r
z5CdMqjk2h_vu`$3I{<DH9vl&LVdU3>5CO#8KVH}y0LJEDLip-9c}F+|AUfEG0Nc8o
zo_Xm*9DiYuM~*RyGkg9}m%&p_-1{ksBalaq905}$l4LBVp1$zyf$lTE+<Efk$+^pA
zBi(UtKee|95=Xd@rV2bIpahI&b#SKyn`TEItiX=fS9k5Jy@nzuxFb$uSR16|z_yBQ
z-0@lp{g&_va7I>)wHC>z^0hx#a49GcA{GUE^3Br&@UIBa2u?!i>yRd)N6W3w=|Z`E
z4D}bV4O%bilZeg1l~+!OK${DivX3NDF3cT498tDXIT)uwWPpL02vHRj4ykoGMV9Mp
ztLD@x81}gjng3Smsb`hzTG(B1NUvVKN+doVV2fcZ;bc|{(%?ZJ#l~USZsyGo59vM?
zV{dU!5yxRx*{RBnL>vIa3}KE_y1IVI=CUuZca+ZlDEhG6G~4T8e<p=W;8E`6{tc0R
z<P%Q>PTnst$Q1q-Y<s<Mq{FdoR8H=Ll(W{2gE#pS4L{5Yb&K<lj#`y?$nu9JhRN8~
z#q)~hGpjSPIBsye*TG^TOm>FsIaNWI$4EH&9Wu$&udRItW{;@6R$``N5&e1g=t5b=
z!b15HA3PN>JV4c`{j|W!1d1%?xY^q>y;RFDM}Vb%hX?MmAAbE3l3!}jhHsmWW0CH%
zS~y|~?WZ`o6u*U%oLcxreNKPJ77FP0hVvKjmy%b1T}Y6-`49^ZsXAPCW(yAAa^;l&
zxjf;BRL#n~>H4+oIuaHY@C9ihiNfM2k`!;0T6O=_b03^t$I)_Y8=Fm?uY>DJI2#z5
zk+E-JS_)a?%)=8UJCa+7Z+TdHuU}qmx>9FKpas4t^NSaQ9Hg26%R$X{vxOd%mVVfI
zkBU%)Jcq@T&4GC*zxjWL9qj%yBOR5zmn}3>EB*TNcp=KCf4<!`JvaAWU7a%EL6PT@
zlU)MjrDDh!-%74!{+z_g2RVTay|2g56g0KJKKG{8((da~tKvgL@9Vo?I5!pa&L)00
zjERnhcU3zuzzsjb8Y!rfdf4Upa9QN$GqlgM7{oMQiK&=$Qma!3mn7^MaK_oTlcGb1
z!%JL78Dd4w8_gRu<|Exb2zg@g+l6HG+xe0M=ay!>ZJT1w8H>nIoFG-qEZYC}n7L7g
zW6#kDMsBZnl=55@Lz*5y3vpeS8tPlfK}?|)65Kw!gta6wujJbp=|uSXH4`Y}R$g97
zRtMFj6K$NFA{!diU=*QE6yICN5u#}%GU-B37#bK5U$>AT<;cC@iOw(k=SQk>xn~?o
zcX#KX>8@huOs^O)Kj?JY_)m5Mblt(91A&2oammU2qN4U+Ci`u2x_|z>EVulv1^L0h
z84nTDat#T#y`16BPy)x^roDNSfU`ccOD*0>`0fqlE7R>3WZl#2Ul$Zi0|#Oa-|w=2
zQw7jAJ3EV6+V<_+_3qiYr)roLT&q7+uHz_{v&{C8prCggoKwP$?}a&yC&W!)!SqhH
z_*=qsMMxg9Ph+$aOq(tT@gyXKoSmJsX;QMYCrS>-H69%Ux^bR=2NMLDnO>)ti4b}S
z%@+);+I01cspHbjkg-}oJqgw?5z;B2OB0>OmnN9HdE!_<&N<=>OjHm_m|V~`G`!wT
zf+4sRCBMY@{P}bDet#CIO79AkjI*sxIHL%GA?N1!mG-N4)~uV*mJmuCBA*B?Nr%!4
zB4sB8ZxFgu`<F|*0u;nppCIVDkiZ!}QX<0?U_Ldf?fnosd--hQ<P=o%K^y0*oo~hN
z8kg2<#vh(8Zn+tT<m;ajou{1~nL{LqBnTEqyGUz-&Jb@p?gSFo#8R=S>|=cO&y2sC
zmzP&Iqo#+44B@Ibw6u)ics4IX@R@oOUELa8?W1+Drke3(4dY$@#}pJ4?EKQLVV%k@
zbU9;gIF7dMjcl;W_WL+9VA1&+@EVWrQ(Ig0uV243O-mbdRta32mYV7}lI)#?v1gHI
zV0SlGGEGoOXs~>Ff`p*hL66_Riu=2tfWd;>I@fn~bf`KyPHQe)cTab=!MVVdrR4P^
zgo9NRB3k~Sli%z*fB#UJ8oD+dwj{4Vm&&L7MRV=CrOVFU2@2}kL$ko~y}jdgb6?+X
zU_QyQpR-Hw<eLjLjE#-0&}J4`+pmFo7iV91O_FMAZayF-^?=$Q5f-o*lXkBI%pnLA
z#mApH-$Pjl34_JJ<i_Y;)McFUBD%nS)8+5&i$C=%-p|cRAPY1MUtCB?2!<b_uNYK#
z9feaVX?u<}ynZJqrwtyvK1-RWexFeo3A5}$Qqt4XQsRqno~Ea(rGL5~9i8X3dIN|8
z{H@$hJk=Uhxaq@(;h!xyHeFCq(0=XvP7aQ}?AxQ%)13CaPD>jqiH!U9&5)0u-<icC
zz?_(4yRyXr=5P^WM~*b$oFf=GH};`)F#tpa;HUb@llH6@z0C$`pDw{v4z9B(%8p+)
zjK_)W&cS`yhD$TnL^2<0TJ{(+SI1Ee2$X$j=H`!5!*!1S{7D=x*3i^M<nI_Z9scks
zMAM4imcc<@adGkDJZcJZ$bx(HH@$olHHO1hgq3*y{CN;EeE20>TdT>QGECEr-y&0@
zS++=ejEkJ)Wxy3X487lhgBn0)TX2A{x>^x<4?{dPBV+Wb4CC?Fvzk!NCbV$22j{VH
zHvRn=r-#f))BAQig!AC)Kp!wx)A=hJ-Nn%bY`cBJ!eygl`ba2DeDtYcpUD}QbsFEi
zHZE&(<W!bWOVKw{_RSANTKxO{-WP1}2H0Dv;#%2Yy8fqqgMop;yK$<Av9T%wOeN;I
z6<=ENjB9T0Z&a|m{)ro1M2xj5YwO~*x7Uu`>%QohyKDSL=}O1BqZ8kA;^O0ZB2))U
zh;Z7isy=&{7j_F5&X?~d-f=!~VR^NfRQ&q_RD$Bjq#+q*e7C|vmn2(M{FkDZmKJe&
z`S7cq#ig6}*Ni*qKkZ!FnZ5dLWMstI^36<fZ${7BiIK)o*MsHV&cj9_AtA->c6?Dd
zPzHiEv54Co{NQ}<&|)E@g`Hb~eKAF!<or#0rl1&)H=F&c+`1zM^_P--kM-$azUp4u
zJC-OS)^oM*&F?FUDIcH9El$=Ml*QNIC^LcHjRWRO1@&v)chwMv6=q@R{ZQA_?DZR8
zDgkx4cXh{Nsg$W#bbC(kq<8Oc8%U}V3js?CD=A)#@ixIwa)Jc<(1Fo|mw$#$_}M=k
zem;Y9i_VRItPm`vr>9&w^rE;p&fb%>t62FyTG6krV%-g<xB;aJqrxxB7qafuSr`{P
zQNH`yJu)=3`aWbQrmd0^)70O8SZTI$pZn*HJpcaVbS2CG;kW<ipVCwQkH7PO;YI)R
z4gc5s|NGnj|1bLga;4YT-M<bfAk2go(O`A3C_A$^@*P7XDdk!2J$zHZwg9YA*O3kD
zP>WX<ynkP4Ix*~khn#O5Yi3TbeRQ6}dqt(UQsUZ1En;cz`EZ5QZD~<j9FSk)8;U+7
z^*XEf{tv5p_odPB@)l_PE*DM`uY2J$St1fMWWJnz^YdrP$c88L@sXuTEhD7bR>_^^
z5-C+g!N~0I6Fr%(`$R|Wm&S>Tee_73($suBMih-^Mi8-Qm6ZM|iGP><fBp79#qhs=
z`oDhrf8j;{^CGN5r!H=p@XQ_)S*K<_G0_sbV_gIUi<@fEDe$&Oy`#Z{<Qhj?Sx(XG
z*S+4zC2zu^Z{j{CB=G$XiHZH^))y-H*y2+1ilc9C=Co*heg0t;QM;!rV`V}wJ**`i
zFmYd-jEhSm{^+xWL|(CF`rr2$v}+1vyCs9~`C6QrA)ET+X%Lh4&4Z}=|5R4-rwpYL
zkwV!|$yaMh<;g0MSIF*<QStug+Q!7}jhJIDMQH4e3qKJgDn!;tOCcac)lWM}k~;kQ
z>PYo#bJhn>fvqWPDqo9t#5D<5#9n&%AjbdF!=kX0(uqblvWLX%%^rTMFx9AyOAg6@
zph|X+i$;f4@a@QKf5$gVN5=!FzC{pwp1gceNQ94d=dTMZ@5OYZ7@KaV8$1ZNIeyJ+
zLtUD+-r07Y_6|kMllvj$)`+R5>uP9gws|aZH`GytHfKo`WFw>7ll!&z+IhbE`l5pp
zmA^C|n4PBQkonz*i!H&I1Wfz0)@&@h8@in_iPh58=An|%_DxL8J9LdstDU(fd~KWN
z=fe{Hd9qU3sRfk(+`e1kkW*~O3Oh?n;_!(4^(3CnrsftaRGT)%SmW!MMqQ}t9iL^`
z_xXm)*42A0IKR5urR1fsVN~x<PviNOP8_m2D1ES2Jb6VndG)7!TFL9RryJG>^Nz1f
z$$JO(oMidYQ5GBXbsNPD^*bwD?w95tlB7R0imvh4b4kCO`%J+Z&hjZgS9~WTm?k;l
z!(X?_PDSu9cDzhZD^H;eZ^<-$_V=-^b>o{E+A?_4d*qrbmfP3GYgUhf84EtY{4C=N
zw;nq;eYp6+&)V)z2^rG|Ogz^ncXDtJ(pD^`yZq`>@lv?=^jent&Fex<EskqVY9Es2
z6s}!e-Bwd}wKL(!^}EjLJ$&*8s+|GC?l<M|Wn&`JjaTG7eAgFa*LOvH+bO1TneE{-
zd)uc1deu#zd22S_M4)wavRKLe@-27Sw-?G+mL&?W)*1?OF#IjKmD0#dhQ?Bz4cgvw
zH~)y3*3KAs{;BwLbHd|?<NNo5xw6c0cQu_3e2DPNnfk=F&T^yVgY(ahb92{BExH3V
zDae;7sE^xr7KHCVzH!dDr1G0?{q@0yaMAP|0Ix;KtQ;u|Gs?pwk3UNmIJB&C@wFs}
zn?FoAI)7Mnb&Bgj0&D7%R|zBKbIZR<CpS=qD7wp<;MwLaUH4jQqa}I~9?~s+lRpP&
zWoIaaukM<s-tl*fRg@NIH!gkRzfOvk-<zoAIJr@KYAUKF*K)y2meW4_cD>jmys;9u
zmUjDf=3xX`cXCV1oZOAGZd_*TzJcz`S?$H1)9g39S$1-jDOz9K7E|x1;A(!An{(t>
zMj?K7t9~pkBQ5mnd_1IYVkLeLb|%PqQ8d1CA8fd=_zs`tHT%^zBYcc}Q=f$qpLqX1
zla;Ck+>y1bv<4`HgyxriVsLid`qStAabr2@l!;+ow#Ni1f<LK0qjyO{Q+Uid<ib@M
zzpI@cuaBSXJn?$GE98=mDYJ9ymgx@J?6HCQrH;>eekB1mU=E~I9%CuzNEgB2JP8kb
z)=k)B_FP_maT!|$ul&3?*`c6dPXF{0K>M@oD}Begyl!uXw5;v9qKh3g&~Z4<g{||o
zBm1s)|F7HPQRJd5D<i{?c?S;o6OAXr?a|H^+8dQ_=Q*zr?EHQ6M@^GTDpPFqX*!`=
zNA_PD_|gI-!9Kjsb1}BLP+jaA6Q{u@6ICT<EY$e8=vBh+tWGKKVO~4}EzX-!M;AEX
z$t9bcbsmm*M>C?GY$n2HK5}#+?`nH_S006*zvr)SEi?TpNxPYo)1SVkH#O;bZmE{8
zYt?oM3#D3XA=ENj&pfQgJHW*3|C#?UlGw05VPnLsoHAfb=f;8h$f#4s*D`2mhKY+G
zZFtv5wpteZL)uMRa+fy`N9jcWTyQG8+#*$a7}uV|@+xYVjs#pf$Q9Q=<1c=jzaA&m
zuZ?wH>dn9Ur5$3v`uqI?@9&d|goJDzpr&rp&^~bVP*q1XJzstH$M){Jx|fOiW6K;D
zu1ks#yXACcMdQZuOOdJ~Z5|mRg(mAqVnq}7${uR4;MNk1<Be8FGrbp7Z(yu3SpGrm
zZrY>dtPn{WKjBWd*0!%7&gHB}PSvu;mVP*GY;@w;ll!9?YCT?KyF-F0G_!VHyws`c
zB^H!eTyiO9X!>5zi6d@*C{%hqL(3<qo3sKAHH2)!Ee|MMc8qZ6)^fT;${b?Y-P}c#
zm_T1|Ud)A7;c))KT`!Pw%bpdfDXSY?+!0p3#8$D`Am4U#RMwK`IN9RN%KeyVU1OsU
zCG+Fx&-@2DhWXcr<h!Sek@+urb@$q??-^P4ylkN#udi$w>nzkUGEu+=WK^MzP#4L+
z+AjC~uq}CPS2s?8G@6-8+ON!{zge3(IcHL?>h8qubIXi=uQlev8C{bz$IobT{6B2H
z1yq&U_s6XmC?N)&Dy6h=6_8R8MLGqfTco7xQX(LrARt|Wgn(X3Ixa}Jc%?fpNO!}#
zuQT&o@B3fxteG{V1M=MSoU_k9`|R)Mt1@MDIN883-6+1b^Kbb6YI(9xI--M(y{K@7
zgDthVI&d=lm)7}_nb_qSFWy+v#3quBqZMQ@hBf3e2SL95Silzx%Z#s;;RQj`(rs*6
z+24#j4#?#h#99fvPL7DVnWE4(3;Ry@ot9@JlaFGqJsF*zU`<LX$d0N#rbiJG>ub^%
zFAd7TCKWO@9b)0h^<%;pJ$*2CMD8)7qpI4Iw^EItW%VL8wIJ@-IzcNtdsa3o^LuKz
zkxg}L8}|{9RuXh!;&K9KUJN3TvwfrP<gg?`*zRrbHmC4LOIN=8$x+#-D%wf9q0Y!$
z*mq4d%L=04Ek7A`c@G}~b~^hpiO}_73Up&HUB%kWo_bG_yD&YiP@VErI}Lrb(GUV<
zVYJX$Rk@B~==PYPHDavO$Y}7xu-fWG$N4|*EwV~0O?&q{h)YN$n)}u6d-b`6<y0Qk
z_G?|p@KBKh!CtEuO;cO5Aq_l4re?T(f4rRX#jNh%WaZ?{^+b7${rt_U5sEt;!TIcU
zIe&geo9$s!yN6|6ucN*74Q}nkB*{>~?tJP&Uf$a-<X&=KnGwEju8mqL%-q;{o+%^s
zQQJEfu99~bhWJ%<H9LO86y{Da_=LS`ra;Ha-325zF)2++=FYp$r`@=c%pB%5*m7R(
zCUzkMGkS^H-NyCqQp7R~--%jE{fy*>!L%FtjqlaL%flJ`tH7>Om!a;c%0W*!nsDmF
zQV5?I!_2QeOq$B=Q{v3&s?T~(4Nx6J9A_)L0R7q@Bh?|>L9uV-3)7c#Ct>?j`mFO}
zw*Gi=_tmLC>nXsF;;+u`mA#r}dO}jwsP9EmT`9ahvUAyl3^LrRKh_dAudv3-;*D?3
zDB;qbHZ4aERJ>J->)xd1sOaJOQZAI~-*XA+>WTwu0s8Lxhw~#+C$G+8Zzc#ky`@M#
zarfVM2W2tO@3FdCxH<CkI&`1RKPRlqUGz6unITV)ic-JK!GWiLCu|%Z?`?bM2kg$&
z*50a{M?Y2kRKVCcP{immL#iSP2j?AvWLbO{bcy@@Azo6ovgMTS_&kA&R_*&<QAwx`
z0Vn^#eU|6^-&HPiR4$9Bq<wv@ue7}^?%)*1#~N}aKDmu9cxLTtXoJg^8TaZBG0)It
zR5~H`@bF##`}4R=WH?URq1Up;84AFQqh4Z9Mv;E2^#UR~*gb`lF+KaT1>pwf^hb=C
zgazT2tdcQ23a7;Zg{F`Z2))h5zfK{FzIPW?Cr&*XH3kt+=<!4a?1bIr2G?|xL!p?O
z(fCC0u#tV!L8m4fn|VvM{ZRYk3*#skq(PSd@t=Ib5Q)57^^KfEU%qCiV1J#{?o0pv
z^);RQa!ToUQVb$ry?va<zt&$)OS8n2k)kW%rm$cw33{hQHh?wD=4CC9?A0|&M&Bda
z!x@=(p&^R8N)5m3q1iUB{|9Yw;3HJh->m`ORH^sM)=a*wJwC&6vr79D?A$_ib2_0}
zsSAp4M*xGkcs}^cSJcc@^6}N7?ZqxX_d1?0*;!^?RrqsN$S}yJW)EW{w1w@13bJ2{
zrx+TpQFafQuOmJ0J1wQBltsd8fYEtq>=sw{g0v2yFDY1s%dek4)f61sS!BH$%Y2_t
zPREGvfJ9hUH;JLi_F3d8RQ!e-JBIDYd^dHM4fAs2`SlLYv5t8L|5}$m^f?9T&1@>)
zS%TNY?&DWl*>hZHGQ|w-kgy$Gyjh4reA5bD7<BW&y+4TTDMY@RVtcNCXyW$<LX)x6
zVf+^ZSYao}6u`BoVbiF)>tU=yzR3_*P*AYCN12sbzt^}yD|@!fJpS0!Tzu{wCzF`6
zW)Aq71QIu#7R#a!7~I~uFMC~p5T4TK8MgHDjIr`O4R^Vt4xSHE^l(mvDI%C?XQ#kp
z2Zms|%?Vo;h|h3vEAp;m<!0y@9Q-68&F2-l8)D*mF*3?-%r|sxqDizB<-uTR;CboQ
z{C%(UnDq~D8#dkGY<R^rb9W9^>lUD{>*{y1t#?Rug3&_;|0zGeC_9TfI#-5@w}=(I
zmSo7qD<e$L7^_{su%)D-`p7`(jtN=h?$I7)Mwr%xiuHwv7D>|#mcYlRPgyAJb(~?r
zojdn1NT%%;TdOYZBgR7byEH_?-1QtBO!$TGfgZ4lyg;a%i{%12xssYnNj3-eG5NKN
z8JR3G!H=cV!w04sYNj@nH7HZlz9#4!+=zAXkDb452unFi($-uMa*c}ZqVcW0U9Xd+
zliC+ug4G9!=!OE0DqkpyG&>@E(Y?7fQ7=0QU;B~z2i+mQ)|3yYauCtZtc?rEqbQDB
zU!f1TLZ-&sHdNq{Bw@es2X(i!G$eSczwD=;uBM!cBPZoR^$xqmd~Vh1Ee^I$tLhm6
zuHvIk3L~FGBz0er0hC(kci=ddM~B~M3&+IYz5IqTETvq<OXC8?PT&7cgg44i!HF&U
zde?>%)3;;T&m6tyx>|5r(AR@zW?>-D2EAb)zO)_DB7pAgvSTl{DqR0Wwu~#iSj$=Q
z{=X|R|L3&*-!K2aN9O;2_y3&5|Np<plbD{^=WTOho1>2Bt}j;5as6yy{Y`&^7mCwS
zw0$WEQsb`W5hNTI7VZ{4rV~WI)jK>igW}<-eYzH*<UYT1=YFfHul9%?`r}o@A49*Y
z=eEXN9&(gG$7#}!`us3Vi`SdK5gQXnIW^Ig+*rUt+)C{L-6Wp7$<+<jTj?<wc-6yk
z=9KT+(|27-;cOj7B3={9&Eo2y&Ui9n<`f}2L`_=8oZK|6=dmJSR;NVVNL?Hwi+Of!
z((7dKG{-rlE5564aB?E$MDlUZjR(y4Z{B41l$U(;WI9ooHFe{;sCRAz*~vG_V%XE1
zZ}f}?BVTn{{Bwp6Bf0$Fe|)&B=?e)vMEE$<<uS=+r#7rlj-f29jWP^g^+LHovD{61
zM*7eRW8Qta79H+H;3KEqksEQjhAXXbj0=U-P1jSNw4ry64t>0zg$AGUKV671*j^m`
zU-hM=q+|novfd??j*+QKQB7q8@zBN$=w-4saTT(%LOf5n*2R&gw}#-?uqvYw#J6yH
zO{zC;L&oJdfB*G=e))fX*TVaW#B~^hv#9uywE<}Vtl_cVq@?IOXZ3;0js_dPwp>i@
zsE$0c*>TZ<|A~0U5bREoJcs*J?3(ya<u+XpE0|{p{l!OZ!h4l9seQbZE5qPJGB%}{
zL&{y^;?|6x2Dw0I4-ex_&5Vg~siA}=5<Uu<iQ!3FO;Io5sK7BZCCXJLdICi%g7JJ=
z+P((cg42gphOy@(!p<gos0=8si1Uo6qlBeABrk7p3D=UrUOpZ2-7e}*So=?Y@`*=X
zq?zt@+SIWo$a_)5yCHDeI^!uJc_W^tt$*NIZHVK{*PM$6vFEqlX&*dV{z7P4cfC4h
zmVspJmj`pWh?q5R^h)@VP-qj=c;T(q9VMj%1r?JUT5-~vdRpSPZ+p$I|JMs(Kp(O(
zDwnAhRXaXS>vi-)la|#z8kK*u*{CA^3p;lyd+zu@&qllxvT!vxb8eA%qsljWZ!OF-
zBQ-GIb7ZfU%F_+`VfK%>=tj+B(XAS{=N3WD8?K`pnvcVT{LCpiGAu9YQBEx_U!=v3
zh&|yMpJ=J#L@w99A4V5N9SV23J9%nmw5TF#Wo$^MTkY=kQWRFBDEq2}y3BF&vTVxO
z{<RG9PRD}TlRdFyv6hyk+sah#4FT0*Rd;QSxi45YwRr53$Z@TnRJuehvlkw&WjYn+
zd=sD0oqmiRB>r(i%(^kMXnDLYNG@Z%J24@en$Hxnu-=b)EE}=2Da`kCpZf35@hyWV
z?Y#76kH0H9t<X8hc>iT7$@xl4J#~2)c=|RbY8kr%_wcm=f#Qg77n5|P*6px7@p00(
zxtc$xm(QVeKe86=-yCc{Dx{6y*x;aM4_?)r&Az?_*HT<6-DfAt8fGdL*sL~s_wKNX
zpI#nOl~Ag-X*jp<ZeQ8Py4EwLjw>f?9(0eMOKwUPqMfnv8jv(wAq~#5EZuSEy#Kt_
zWxK|$nB$yt8{#PwhDg4B!?(_Z_P$u_Ci32!M@F|j9nwOH9htxPzG)T+F<qCmu0edN
z7H;-Bsu0S$a4s)5E6bTSx>-EDj-tusSdnC74Z)R{7n5P(;*Xl3aw>lI0bBTL#B#OM
zsp^g$#l_*&x}KuYs>I;6nI=kabFOp8>AB7cmVsmM$p_4AlquP{^hV1VyDIiH{|4C7
z?YchZ;+R=WubAjj`sR*dcM%q?IGMdF){^zbqlJrPyHTPa3mFP`tZcKmIBT3}b?=LQ
z$YB@7@OPB?n2VNk%O&-x)=-owdEX_LCOi_(&kSsA6}LRt-~A)N_3iGB{l-+<Dug_5
zqDaf^q&e67NBzH4{l&V@l&0}s+E{Z~KKw2lA)M?tW--(G&^c?7H8DSs+f>MqQ|#Uc
zvxyCj;jdwt3*|AU_gq-{?p-Ui&A?K$7}F9Q>cY=beS_yBV!CYD;HdM9Dxl$`+@NkX
z<BJd2qSXTq4vfau@i-TqLZg$WhU^Ua)p&oOE{iLxuA?Ot7ybqdvvb6ZLwBZjXD+gs
zT`SWphzmz14P)BPL@Q6{Hr-6LMIF?58JnGvh_K%beG*u&d=^Q+z9CFHbD$xWwt9d^
ztsfTuN=M3!7JhU)F<!L^_3udhfj(V1Grbt&`{#_&ECnCFHSkAuPP9lRL@Qjvird-p
zgc5HtTfV&2;`gG?TXoRD;eKN`%Jymt5tkx$yU{^8!5{Y5E+V~hW30+l@nNfYBA=B)
z^pZYV82dM*$*DaRU04VjsHzSx9=x1_CzcWB5&D)hLlDD|;mi>?r1H7=t9OT7`tN88
zUA_VR53**I7r9?8{yH1nLUc;<oU7$aj`X5?k4EJ}6mUDKaZdo-=KC(&a+{Yu$<E(D
zZj2Y-m))4fn1n18I2)0T?E;bna|srSc{WEH(;RD;h50K<&u@gMoFz4bD%f@pHVT}Z
zv?(feBTy5X$9pB0W=y;Ie^YGCJh|6Bs%W>RjZdyZ)w$`nx3NOo(X&&+|8cbmT@le5
zei(lqN#vW-;(B}(Y_=EREAj?=wLhc+QD~zdd!AEG5_ufvBziPKtm=<f795kDOgk-B
z?T|FXmKHM0TYbE?gis?7%g>Eki#^}o&gweRf68mmSN>-H;<=9AS>@wBd-oS8wzIgf
z5aw)3ecp)g&VJ+Uv!2p>YUBm!Ws4ci_Xb`~{8FY%El)cIShJ1s9aoL~dkT+6C-8>a
zd%AM>X^74=ni|rU^TcUUoKYO#`K3+u!^nhvoVf0~kx+YsN?B9=RdWbqYFaWz=Fdc|
zggBUf)sth&nlh}+%<+e47_gD6!eWAh3cO3h-yN;{nmJC6{p{O2S@&#2%Yu>U=wGSA
zf7%FdUvQyH4m|U7fM0Uw#dNtNew6v;5EcOk&yTCahA3?w%GK^}<>Ts(QFgc)8=l~3
z`+?}F;LxXR@8ssa5}5mzT=6;Or|&<#>VEl{pe#5dXI!F>g@mT&O8Il9kH$5*0>KWU
z?Uzn)QR~7MDr?G?7911=T}(d)O)gLjG=`f$t|O$Kr^X)EIG`SN@)G}PD`e&p4UHsR
z^*5Nl7wbzFtp9$Y%eju{*+^`~et&#&5kwvPHH2tr%M+)2KUFo~^zqIm$4ve7yF-4}
zoVQr!q@ETnAXtwomZl%*dC;Ql+JWko)=i)~rGLi+BSf8I!y#?&sN|FayZR)%aP+4_
zXSSV<XXN<(y~^^Ew-_*p(UQah+<vvJ2$#apeRcIxfH^eiWhBGEhqZo<yRqpihv!Pq
zfvK-<KneTTDnEJGGv&z7nbYBa)A#uJVKuPxM;$5NItM{#+Sjb4Wz+&M{nJ%`Zat<C
zX5Y%EX37V=#jfh=BPt)fY}k+Hlrs^ou81A{h5TwUXi=TnL`X_8y%t7CEW9)BHuy4l
z?Kz9c=PTLL1iQ6bRb@gKJdBi*#D3RRZ?lQ-iQ;xTFO|DuVt1rW=kd~<5v^IHx8t%2
z4CW^dO+}xMX?$FZb+JCZ>R)cSZS}R>`XLIE%eTWHtcD=-gw~kY#6%N)eQyxCfWZEI
z<eqbnhIYkL)!0HU_Kup_2G@_S@$t(zVA>tIjdDVI*Awm^AbRRoT}EZ~$s(eri})B<
zb~SRFnC5nrwVu~t8Ah86@dxcF!qMNFEvWO#vOd}zc7|A%z3cmNuD9pSi)^;Z$)7}-
zZmcc+T-`1~CERj6d?iw9wxW(RwTm1`TW{DJQG`d=HWHTOCu5H|WBh|n;vnh?_Rz2x
z36-ss<yXHMv5H)AKk<N-6?T`-n9g#D1z4|_0t7t|1#p}>XvzW|IId9%F52_jr$^{C
zMiwUpiIz-UwN77+2DRAw%1K>`YDVGXqZz$pW;4Cxio4t8r$bG+mM$fZ@GQq!SmBA1
zEW9Krvyq`gyX&zF^{QPa+Vxy$dVYDeAS>k!XNQT4i24d`jQ>NETgmwMUs7G{;wx4)
zW01+9N=V4e%6=QJPw!}Q4*9~Kn46vk(!ki5_qGPF;`SO44SS-pXWW{eac`5%VDnQ@
zF&V8thTGA&x1vq)+}+*X6|`}<MlR@Y0G9UOE}h*5Zg5Z#2PuUZ^dG$%39E4XYS<!M
zj4fW1H*zbY-?pwI)kylbvxfu<%iJTa&v-sXJq10dT7&R}zgN-cHW{*A{p^ltt*IaG
zQ8cG=`2KP2K*8GNB!l{;4B_g**PZkWRbPK97@6GA?NZ)JN>4t`sP(JwnU<_lO%vsS
zcXB=*#380|^B}_YFC3lN+7LJ0CE6Qd1-9ChiGjA+W<exY*^NfAo9`t{c@Poq3<E{g
z;oFS`ck_L5v#psLTGOEUBDJtE0@`*hhcdo=c?)VE(DR7{I~`htf#)$@-MFwds>V!?
zTDVH&($ckYS?`aKl@OQx8q-=sJEw&nah=JHFxbe>GFI$%Ve|9ocFHAv8C2Ezuh`{V
zY4Pn^e6$ZPVp_q+Aii*2Ty1zVn04zC4}(HC|GlQRM2W-cxl_Se;K0CppKdA=!>Aff
zuBb^{Q&4Gaxl=;#ax|I~JRi-ic)h|Fbkd}|RRa}Q*hke3g9rRdLKc*t&zW^Kf_2bo
zQ;P7?2Lf^9%?)=T+2J93`je0PM*b%K*}UAUp!ZwM7L!%DBjdEkJ%0WskPSeaDYTnh
zfn}DI#6lkm-RL;M{{V;wQxxaQbBnAFP&vs?I=8q96jkCBR{DDe>RMjB?w&vQhn>LY
zewa<%`8pNCN>8UZ>=Tp=*}*TqT==D|BWtDURIDU45HvjZgJ)>D;B)%BjBIh*KK-#q
z(vBWK+4e@kZjq<Lu_K%F2R06T$x;(rF~f!V)ML4oP4&6QqqF0qcC}UJuR7?&&e%6J
z<aSvotJ531Tbwg0yaRcW>1+gT1)?*WVdRn77-`k#?&1EA=Z`F{*gm7a>Pwb=!o5))
z0K<c>toy~svNAzv1r_i*7KZLekksICPB`f!AlyNJ*XOMx<cX58GCyrq)R2Sc3E>Fi
zVSxK^o+jhe+LQ^iLg>Xw<1SLsHe-90+j@u1ETXPnG_Ja^9-2A(Lu4AAo|;*%SCUr@
zvhDq-TY5wup1_b$YpbJ76+jU}tf)ty#uU@kJ6n*xE@0F!KZlA-DBT|`Go;q;Ywp=c
z4+Xw^#M^<qS@ma=pNOYf6l*xB>v$SEUj0sUC#Ac2^x3YtbBY!e`oKfYP8QZjP?h_e
zmGRO|_0L?~k8X`X(nrvJn}Kb9Vn;L0a>rw(Jsr-E>ssUGT)<cwDu%ZT2nD{%<NXD|
zWF~<=1}FVLo(nz3`CY89uY;n|Mc6I4dO)MNm%Om>TX?hrG1fXDndZXAde#lm%5O=Z
z#^R57^zJZXOSv%JlJX7;ugbZJ9<(~aLaJ;2efY~?onYr?i3b<nIZu-0T}5`Y{knHu
zjrQ@<f+1}?x@cKPbJ2xtiQNrhb%|W}BJ`0RHs{^ahjxJ+mL+1w1MEiN?}|&iD&3Oj
zS!W?ZlwAob5TSrHaWwzd###==Df+fg<=?%%yCMz-4n>2{Xf5n6=R}tZ^9B<_E>Srs
z*dA~81d@Z(1AYzo_y2t1xRm#d?PzoU1`GlZ^fLnJ7l#QhEGz^r+t2RqT2|4ui_j`{
z8=PGL6_MPPS?h)$vhYhovw4(Q?RON}D-*ZVO+GI;8rTyVRU;E-b3$!0%G31cB}N)f
z-`se5TqlFwZOQjhyIb%+((ijuq~gbwwygs4FfFZ7szDy2*QdGlkZFmp5Vbp(oUrtK
zLdnxJ5p~~tqE^?A779k3qmDb7hQgYh3Jx?M89x1CAE*||8D)MoFkN7Tf+aS8tY?i`
z<zDn-Q^CQzoyT%LyB<l}6eBN$TeM^<y0{3>BN5NYmB)8DM&x+%WFxY}@rv(7+Is58
ztq$MZ2qg8|X=P@)xuzWh2Rhn*Wto{@=k+ZN;vV1@eAR`N5D91{2D>DXG6X=e81(U=
zhh`=vP*PWa0@^%if}{TG+~P=FN$(5$2XeNS_^^T@`9^U}t*yZeyCk%GQE*C&U~!SE
zqTZEyUSf#24%_l;`?2gqsJZ>qtNqK5w@cKGxEu#BhYgYJ$(S+6$G5r|+VF`ODZ#Yi
zI=Y=OPqHz^=&?AYDCziAG|GIubbRCaN3uPt_&{hrfQ%`#oK?`p>7)B#&Z-+nd2ajq
zXU3yS<jc2@)<@1EDN2Q}PUL#p`R_f7OSB@}ts;k{!_%cSbi<dA2{VA@Xx+)L_!bTu
zTZVzD#J+MHU2>msmp>bv3a$&6NHsdm(}m7*k;AzJm)@tOUPgBnZWgV0)#Hw&d&F>0
z1MR_l!ouzK^=F|+?w=-T=rIJ<w=eYf11~zecE3nma=X|=4w@qG=Orz(U(pp25l2Mb
zv=Z8U-_b25`b1`na%<+Ha7IgiJ6<%Zy}+&`(S&+&OtnMBUF$^-vB`q1S}Nl@`#BFN
zavgIPy_4<0@KGCZCH-(LbnXq&ro}cG9U-@C-;<IVuYB}-?P_Utn~_%TLu+Xo-z78r
zjJ*Yl&R+vUfn651d(v-8xZd3n5DYMh6F|8SxevH<`nvGeDTI;~#1zcGxZSj}@zdTU
zO1<kHPnF@er;g(lhLfN7U!Pw2r4!f_P2kS2y7t`Y_7iS74mn=!qoIPEVmnr-j;(z?
zb`wwBp$~2PB^1P)&^O1+3%!>VII1Fy3y6s5J$EPKood4Lbezb?3_Ywao-EEU#bngN
zQjAT~t9L56@d;AcF^x@sJ%&y9fqunyF?Tl_p&_<4S58_A(iO?5uKdOF$Qhl_h4Ifv
zkoh@F0uaL1!xD*Um#L>BJFk$d3{b6ytqPAF&q!`te>Id*5%lC%jh^1<?|j_3qri9n
zTAibT(~XwUkj=Hvj2>lJPLjQW!s;M6DP^QywA}e)t6d@GJaefpl6mhV#d%!vjb#nk
z-D^o%2pK>Szm!>bec>(=x@>v<1Gd3dy{m?X@_Ube|8K>|UuvEpqMmQj_Au`aqtnUJ
zUC-IpRA7FTF^X+M8W+*R1N$amd|d&Rjk9mS&5xmy&|KsUJJ9ozKyF;Ao4fxGjUOfE
z<)Ir<MU(yI!bqJM?cyxM=$z?`m!6W8nrjBua9{CIsiv(e-eR|PDt-W+<^RK*d&oed
zjxF3hnUBa<I9Kv|mqeq}n#RD1E~*NBoPfxqi!LrVJZ^bX(k*h;;KfUW+e~$luijCN
z(SQ2R^FYS&idw@SaxsZqeoKCBJ<GrpOHnvt7u9Kl;kk9YeJW6-<FGT}`JTGh$vwhA
zg!xeIN>N^WI`P-x#kJ=d53hGcXz4*Y=`{q{xkbW_!YSYOCY79|F<!ZBnT@^SI-0%7
zx+O?ED2H9nkMignN!)=A&vk3FNA4UaUxmNfLAf1lf)?{Sf3pjn3juXF;SDIf-ma}d
z*Gel+dxBFYfT*CXp}-TZ=ev~@@FD%zsa#1J$G5p6Wqtuph*udlLH52FB83h<zGNmV
zA<}8rFKnG=MiLi=wAcrRRiz;dul08BF?@LCbbw{oK^M~H4CJ=?RupZ#UpP6hQ?8DF
zS9!NkK8<x97<pDJcCx8^o)v57xEy7otnwVnJ4<X9l)t0S?P0nWXHu57Xcywfe>HP-
zyM(X*IJNYic^ISHF}kHTjbDkJV2r0s)Pz&4Dj7vyN;Ol7QlwP{LY-pm3A&JxVwJx)
zI<N5C>n3OZHK?$SLV2oTLUS^hN;-JGe^|8<MzKF1l@K&}n4{xi{c2*e{HL~BeeW!<
zt`J{%yub?-L4HnAcxCz}ZHrGwRk(a*iJFpf94N5b<qm<6v4BJXTI4OqsuQ85ADWhN
zb^-`F;$EBXRubjhrnF*VDM?#VtD+mrL&`EtQA}a&`a*Cdm4s?*s=x!er>Fh)bS**H
zUqXBDk+F&5&gyOwOSrjEU#w`CMObo4rTz96_MHVF7{1hWO}WQGm8RwE;$4!D=QpDE
z2hA$xzUFdfK;E+LI51*BfuSN;FTlR^aQ!f;Pv7#r?wWht(+oZVluS}zYH3YqAFu6*
zrOk19!S!UXQjrS9Q9mR)Ap8>px+^t1&71Rmy6od?5WReaNz|D45YI?q|EiwqEQDR}
zJu{p-a$ioB!6BoHCdZz}_*;5O*j=jC%!~}&;#zY#L=B}VO+Q;-Km1;d0-2_S?;5={
zO9Zs9`T^0Yv8gE#B2;^OUS1yb%%29V1Sp$=K_x)kOC8>6Y&MZ2L-2)@y{1NE%Lk^#
z%OJku3O0wg&8_wgHAYlW{TW@QEgz6A%6F)>j5}{4>B%PU)6+#*CYKY%oy`{_^E4%S
zJ@u5o`E-&|Sv!}NS)0wDHeWd6X-5?z$%SN`FTvs0#^q}Y9z*&98%p#fXy>&>-O>6@
ztu~(guZtu8SyHbdM%x~)KU#Ko6JeqGBhx>qUCh!qo7>{;1ugx@hhDo3Yv%~^I6}p%
zn}5;JhjjRo4C4re2(3e!C@H3x_;@30>kwE&xaMsj;8M}h;G8hPb1~0-#~O6gcQq`2
zuLpdcDmT`$SSXa3?h}c&cAy?%`$^4?SL_jS=(I$~*XJjD&v9&X&)SKDJ$c2HZ>_}b
zm`0>eTexL&_~u#zQSkzld|y+9Y+?94iyEw<h7LRTmkZ~lhC=X;tvY_BVPfbRyw=5z
zX`r<Un(Y#;?D{tLkw~oAvDW^9JAx7&NSHwuY(ha!UJAOJ01}4TZzZGPv4?v`Q4llz
z>h6Zl@}Hdra4mBh;|*1?>LWV}w7?yyK_@87JxxQIDutS?hUvUtwEX7Mv`&BS&_dMZ
z^t!Jf+<QaiH43k~Z-FtQWe(MA13tJDAeF&^>VXOeEz=;`C&aatXQ5Cm-~meD2a2Sh
z`hjnUIRORO<!k;!%4MQF{ycq~{?5L%XnAhzwtx8Dii54Vq!2qge6x9mu!_Vffj^sT
z7GRrzBQfJL<rfHPXm&3s2;W-hQ&CgHfr4N`0vMAx3Jw>|jylq>T@fGgZ?C#T#q`X8
zWCbyq-=ZZa_S#NyLC&{uh~JG)z0=r&W&Yr1{3xp-(0JbA;i&<uAq)odHv%SuW(YSv
zAZW>~y?P}~yXNHQ?DOOC&*r8S<j@nP=o8u)1VK(mCVuK$&#PR>>lp;#Mzci2>E+8H
z5H^4c7YBO>VGB<3HtfDq1$M6%dC;gH=WPSkfKo3RPW<BDZyI+Kj*@4Js@o1;eN}^v
z-7Wx4RRID}Ac@7w0*Em}i$e%DHPCpuZgxMJpPK`BEl{CDmmReLTujKL$gsb#ZHa^L
zWFU`RenCh`2=0hcIBXNp{D6UJZexR$(c8ZY`D0n4iEr(vUT(OM)p>~fY5_>RK}ci=
z!adM8K+^-rRyO87hbjZ(2lNdvq<_4jod=h0SoFZ)0IV4|QsD42q|-aK2D09EU{DQ|
z+J3%w3U1%dA>sb=t0b_S_u$5Ub{41dfMgc*E^x#Hr99446bWvv>BYs*Qg977@C?1F
zGl?cvLm9cbp#Y=<v%4#Rj>0Jr8TNk$?AD1LN&sQZPN<Cy*SQ6)_081ZOG~AIqJ*o|
z0AL4L-g&lDk44u46@V2Z1v*g9D77zei2*(1VRv5`82A8d05dSqvb!q~WdfFM?Laqq
z1f(aBapCBR1bzsOf9~@N$o=XJfNEK0H$wok0MsYELPEjjaUiP2c^-fSDphyuzb^$?
z%q$ig`1mu3%m6%zGaUn83^3PNuG-NR-OdliT_!4?Sn!;}@bL63=#BcAQ_iW`u^v_b
zbmc5@)}?_lU8gUw5-QPm0{$on=-0s81=NRiz_Z0k%k_4DP=FSulQXJ_L*~p*j=y`|
z^;p{&nY5?rR>W_I!~2ZF3v^`be$U!i999<7MejX2-}ZausWoS;tyEgVu;4eCkZ2`%
z6)+V?-4-};?yy!vxFjfm-tBF!=T;8PhTxze(m!4jv`rHS0sDfZLcvqz)R$cY#xGzt
zZg11|qiQwkwEJdUx=8ho(Yk&5%|g7yz76jlD%r-urg&%yyb)a71JEAmx$Aj|Ks`|k
z8bQF1d`e020d>~j0EnLFcf?HZc&_YFdLG*9@U&F8jeJM&XEF9rZv%uImT&sTbL)5$
z&9jG|P?Fp0k^HO#uYro1`gu{|*M)LiEe?P-pelO&@S)H71rabaBN5&GbY6V=Oa@4*
zAmb%|b0`gQXu245{Xc%-(%=_}R?s~GRt#rARL9c~^md$t7*HkvF*L7%xd&fw%;~=m
zv#)M~2LLmZKorG^gaPnO?=pTm3D|%*5%3=JIS2bc_r4a93*`XCvInjij3(@BL?C<x
zQ-gQ~k3T9PZN!z>02VdXt%cKpzzBU6oOg(W@duj%oD(UCid0qW;d6k*Bn1WtggT(A
zy!*fD*Dz%Za!XK_nn47CECVFD@cocs;C!>d#}7a>)EsabeGa*2RZ(spXuEY1wQpGQ
zCzMkr*EcfQq^YS%5XdO-p>HJ07rJhk{#EY5_d;y<hDW+J-ga=M|JBIwa!QC1_D5v}
z>AoDMzJP<H4SYuit5&(D1mvyF1|YVA+7*Y@gJZ=s=x@N)Xrsf%GXiWy4?VrCoi*VZ
zplN^1E>G{$pL{=8u7suO$pICj?Hyhr5kKgs43SW>y$Buc-<=or50>38kVR|nY^~Qi
zfyfEWcv^v&2$HZ2P{H8jj`^V;?_kW~xC`uiKt(_V#|>Vk4O%!fdSITX=@PoL+c)>u
zn0sJgK09>Ozil!q`;|~BEXHgyql#W5;9MAj4r;UhY6}L*hIE0DZ$5|Kx`$ivqM)|{
z!e)Tg5P%54lCu%~I&eZj_`yI4Dk>uV`!R|xj{-9x4;>(pdz=uhcX*mLg!YFcT`L=#
zx%tq8yjyR~pO=QzX(N90N8Y#}$}q9nVg!w}S3)`SPL2a^6Oo1G^r0Jev^0A+7Jjc|
zm#HZqJGnmCCfSlXB9>x_fa*UlY%ITjlft9~y&BLt0ksAMADrWhW6eJ!dH3U@!EIxO
zlas{}=MUOg*Ok4Mr*xThZ;}!i&&Rx}f7tmhRaW_?q>PE?`c_e2&iV+AQ(IrjAL~d9
z51VgG70!J#J*_i3Em0gwu{Y&{41r~9d|5Ccwstx}rOCeYxLbsn2=Aiu^>c5#G6{%$
z=lL?nl1_WB40Rn;GD=i#v_<dj?VN=9R9oZ4YO1RC{i=LPOx(fV!hTD<k60FC{HY{f
zrHRCTu|~99Xh<OMz{;@u)}??5M=hsM3oQljDk2wd1-&-WtWQ;fn;jk^hbdnMdw#+`
z>H4XdjM!m{PYYkN*=EEU42Xa<6HfOE<T$Yj3C7^l3LI7Yi67@o&CF6fE8YI<1%T+G
z4<c2Zo<6yY^$hEq%i>Zs6eRiu!2iD*C}4B4PEolr*hWOMJwal+KF!Cy+8$jkZ6Ze(
zl$!eGrJEnJrGSNgpwv%pC~17<c%Qq}X`U8dY0c8MVwLf2=yzf$O;J`NHO#+-RbXJH
zP0IX|mH?fsh>=}Ng@-qU#h&Q(IW{&uX=!i0fiC^{;QFoH6_Jj$@c1n3?Ce_*V8iAL
zHvmAs)`V%ZG+2Z@TF}b`mR4>bJ8K)kjW?ICt~Q*ZTLup<c{6vK17~!@R&kE;+1Hk%
zYfN1yqYTQ=G~XKts4ye8Qx)lWcr|+q!o9ZMpVWvQE}h5^MFu-)SGA9pjE%=qpo6*?
zN-gH!XJ^YQ$gl659HDu55zQEGq+jk~?aAAw9It6LXSE(mI_l}3<YVt^PqqtF9dS%#
zcH`o&u&!Oh#Ek8XhwiE8>Q11K`%a`m&j|KOAnz=I-~j9$;AQ}zQJX88DCV;Gg&U{!
zgy&Qx!8qmBbk&HqO@%ZEc{-{x%uIJzq?IlE4iSEo$tZa_XPdYZDKFCVS?*AMwGKKp
zH#4}bjB4Ml0;K=*t1cz_u~PN()uWwr&6lAwHzv!%GvNbuup|}z+?3maT96F;TYqf2
zFjfMX(SeNkWp>9L1O(u(4fQ{uz)JxV0e%6@0jgUtl<?-gH2RC*XS1|vg)DkS2j(kP
zQhB~ypfP`;8JMv!f88rnk+`Mf+p#`NJcCEjVuCiJP9NSstvoisIzf$;S;slod@z@+
znN0E_#u4DD7{`yQB4xXIwr3>OnmgL2vNDsvu|fH~<b}U4j$!mwrheST4A!QyQ71ns
zm!!D;^M9i5rqDEsoL#0tqiu(-S-LLT3(#^|4*e)zs*t}_@B2_q?b&w}b~?X~=zDTz
zWu^B2{;+LthNNVpPbC2w2)qaK)}jsRWJrzpHZ8#fd2dOPRU5AEGRM1PxcK{g^MH_{
z!cjSI^@OnR;FAUC;q%@8Z+Cb3^<+MbYT0JmlELeF^TGS%a?qQ)cSOYDJl9U`o_Sax
z;mY8e`l@90g(i8v)SJE4;ZzmtyK-hP?QYW5e4gNmm@;CT%$KZMBOa-l>Uh9U1SJyc
z_m^Xno4&ViZXtBc++GH0AcCwUIH*lb-GbKgyEos!Dszx)nzx&<bD#D@*I9g#){MYS
z4yJLj!<0_MwKpmgG&KOHT60-xBPdC|)!r#<KiTX#>Ul)b9~HS=mV1j}qD&B44^Io&
zPM?anX4G=LN2oVceEW9p^gOp`>|}v7zzSJIlk~fWa`|`>G0PM1Eqp2w4%!v6!sIip
z4TY7Pi6;SiRr?s4oL|Lqx(}~>qbbg_Zo5P(IVjA<bL}GRLjzz%0?aZ%2}=BLxPQG(
z)-^$E-R{VxS!JAXMQIB(3FV>>eUgVp2IQU14y+h*zv%uL>=?Su!6ngyC>Ijy-)dwh
zBDplY^<3hN2NCh+$oAUd55ofc4UG!diptFr$X{mxb(0Avx_@XLK$X3^?e}-BoER7y
zea`-qN?Zg)@fQd;FO<y?rN3|GYw?#+95PW;9AC%JthppqFE2orgjdT^U?<+f8y5d;
zHD@7eqGL7X3tKTzvJ>mV{Y{ZR?KJGW4-nJ_hL#`82B?H}8EV@(hUvTBcODhX*=jPU
z3?sA9Xuphk@U;UoF9C<S%aF=yX~_T^87elgh5;#Ho3X8lo5kXI)h>!>s;#D8-G>+^
zpDYW`Om8I_ga6E|%-rWc(Rv!$acbB1VX?@uYn*znj<#>^AEHZ}>*(VlF=Gegm@O@a
znTFgW5*J=x@}BXE1Zz9-8xA_G;7mOk=BV2H`$}+=&F^uG`+d1OpeNhC#2P}MX5R2p
zbjNSZ#2h2daqyV98Z%K(aY3pE?K?cIt&3cf#8Co(7FJvBaj<*5gYG&!6kh1dHG?RR
zgF`x1%rI&F`{3T!MCGI76?Jc9#}QeoL`1v`AyV48!cBHDAT~bnF~$K#aXv5C{Qhw}
z2QOZ2!x&R}2+HDxTLDQm_vN&-%5b7&FQroo*2k(Ew^-Nr58*?UmW5085vNYL3Xc-0
zs?|<)Tu0}{tY^Q31`(9#r(0|cMG`-4M3`Nv^V()0utCb#($+Qw=oj3rHXI#rna?$E
zNY!vjn1@Hz8|1%_y1HKVzHZ{?$9M%E_Tc=m8~*qQxUjd+8S0s~;NYljveL<4=DEac
zqIOq!Yccv38)p*_t@Gl~)CITo2idAI>S|<iecd+u{d9$2QcpRTMrEXAzudPlkK0b^
zyBe$~(N^#iK^#yA&|B%gNl*d<<+~12TBx!d&OH%>M}^}s;8lnrE9CXYg@?1qwd)!e
z>O{S~2lpTQ@4Hb^En)}LH{M{et1f(^JlDBDHh;-(EyJ+7V7H=i@hyB^`re8QThVha
zdech*zRWUFINA;zYj9a47zk#Tltcrp5*T~ZPEPj#HVOw=FcIbUW%*}MMn5gKns6x7
zC^Vo^?@UhlyhNbRl{9lWKZVGB9^5wboOISE@j8V0`u<yH^0X#-Xo|X5ii)bz5DNY4
zd`#jP23Z+us<R@xn%wDwNlL;c;+Mt1CIt)};7S*O1h#M(1fQUeFvcf9*vv>z$Dzc5
zS%c%st?aaYy7)I{H)JxbS|xB%e51JsCbGHRj#ckbA@_9mo6xL69~CrB)Utoi^!B*W
zwE1TR+CrEz=}bf>Lud~#V7n)_&qSZQbB6!Vpi_ZwXwy`~%=Y-Ja#1J2jvqzZT>SJl
z=eeS3<S<OrUa!F)9OV9{fL)fAm&YBLaZiRLhvTZJ`9}D-Xat~dO8<{C|2-Ggs+t-j
zJ8Sf}P_Lx0l!sgif-Un#0eihCYthIFXLRFw@7C_&Y0DvLF?l7@^KRt>du{42djThp
zylg1sql42s+6Ip6SAdc;aR#s0L~|GK^zrW4v0z~#-|yetbu)GzY0Dz~D3kDr=q^l`
zSFD-^f4I7;tbw`f{>bUIi$|X_w`IX@EFVtCKzOBOV95OT?LE{5)<;%6qoD`~2a!hL
z@3yqJe|&iDZ-ft47!;4`CH8gHqBnVoOeh=Byg!s{XM94>()jp`$BtEN&dcT&_L>=o
zJA5LN@Q_yOT(&bmG|O{KPUkscoRU0`+25=aDv{1esDtSO$wG-}ka?Vdj`h2Eh5Vzp
z5g*?Xi4k+=e9`KCXFw?+I&1t8S-Elp@Nr#O@M-!8)B`A2%r7n)TUx$@=PZDmU^o@P
z3^W1i2sjB+3H-D4cGD!Ddp^9QMejx3Ry=Sto1<Ojkc4RpeYHJ(hagD$!VDSM0s+P*
z6<reFvUcZ`-3Fog^R){KB0D$PwfeaK_?|pW(Z(Pk+ig*_8DgkN1H3arhqVsNI)I0X
zT2C1J1aS+}BTaBEhQBILjt_7ZO>kfY0#!4Psl&mM2KI(vCB7+2f_o-NP<J$i=|6uh
z@Hf*t^dJb5>ECN)cPP>WjjsPIm!uGj=)AASRTUGj6jh(gJIQmgKkwW4U8t9uT};14
z@BVoV8AFCPrPOoaWa<}_0%y~#uV3pCa&QNZGmOG9x@PRUp}GO_9uhd5yD2yU8c_1S
z_-EbQqM0ftV_<$13vUbdS`W+jZCQNRs}T26V1*Q*e#$mMo+5TY@S}Q9L0U>+f+D?q
zG>v3ram-{VLhf^Rb`Z$ntM_|y?V#id6mbz8Q{{scQ~!7KYdFFMAf@Vfc!;Ko3$MXP
z#ifZzB0@TI+Doi(3V=`}&7K|DkQx*CC>y0C$?uRL&u<<2L>@u5!?foyQ#0}E=g*%H
zKq3ufW+v#Y!R62By?8YEVHbe;+&!y7D8@uCru>s*1nc#E^J&Fp9~BRk6g755XlU{{
z-IdJfZY)P)=9##;L*WVsstN#T$%F?7{tgtG_m-W}!3LDL8|1-3X<OTTqPTyabC{%<
zljd0iE0{nk>FFszOE>gm;T}LGQ@(p_tP3uR<`)(i;fhsPS030nDT-3(<^CpG+;3bc
zp_&LLja|UkkBx~L|Cu6zJL&`C8CP$F7ePiw224XiPWAu3AqMtgdapxU*eE)I;{mlP
zBjBP}ZvK*ixLF9tJ@EOGeEj%47*5Gy(oFUK`##0nLE)mDCI9ra;H-I|0EVm6RyZw9
zCZFsjYZhBlf$LaiR#qSw2>!PSE>Awp%w_{hE}ZrND+SmVV5k7Fr48y_Le9(6xM~I<
zobFom0s!siNFwglUECFa8{m_A3y{FdV0cthWYH(!v`7hF8o=zzhkhp<{R9g2K&60k
z#^fsj<8jecsLK5UteklqFXdzstU{Qb<}Y7{*Wwb5%i>5R6gYt<1oNfTX^9#rzOz7Z
zYrt-RXXyhNaCk0b6O%BQuBbK!B?b^3c&_>~yjrYEoZcnkFi&F^{V4a_DbMpj1zunC
zLr2_F5`TNJiXVznfqQSL)8RNXxVufj<4nYl9kTcv;EvAVZUuClJLRZ0^D+iQH@7&}
zxWYZX>Yw6>8#%j^zJiEIw8s`gU1-pl2l8g_H<x$1Ix=U9zkgSJ_N)<%wPe7$id8;g
zocnl^8_?7MZSjF(go?8A%|z#A!1BRSHidn6_<Ts|yRh94-nluHFV{yH1cfAVgLq*r
z5Lnh=9|~?!56sPPgBu1sJ<iqvOijl2drRQ|!ud4_pxc5E7UUl?<XLJ-JJbghEKctP
z|2uAM;0*=N>9{HgTp!3M-fsZDB_M8Q044<#NU#CH*~7!&!mOe~{1f;tPa1rPjBMCm
z8p3q|K_;*`hE4|jEzY3?7?MzgODipnNl2i@8H2f1{S6OH<87BO+@L=X^j2Zu^$UZ`
z54>#Lcj$T@Il@%az)i1WtIyy{1F%{HO1tN%f2PWXwd*RR;PA#n<O}c<9Pt`B>UDrx
z1!VMeq`K~e*4REdxWXAj9tY;Hvd=alGyePf0(>&e;jv%W>bt2}!{h`s_q)*04WQ<)
z2kRZ;46vZ8-UOz5)8Dn>lU-f)gHQG;n+;M7oK>49Is63K<i)|hz9#5)a)tU6aEekg
zGW>xAe(~V%Rz-NDzV7Na4K+1fy<MlhgsUTLFOReV&<yI#K$6ObOa}UAAXT_YNhIA*
z@i%Va#sREJfu)RrU@Aa63+xZvjTA8FBffs+D7GGH1tZqxwl<l*?Y4~nJ=*CEbLzMZ
z@C48-dPxRr13n98WTXQeTyW@j_z{MHQv(mXK-f0^zN&vd5NzDgzJpt9P#1(%1qglU
zV8B2f3JRvc2#4mpJaBabk8zWq|Lh^KI^iBhA9gzqz3K!&{cf6<;UboR?Q8JXhK~aZ
zwok>y<WLv+n}Y#y97ZiW+b}1)QhAb3K~0TRvbNZY>fHgrB-i?lLUI4#YRYi=&;%r2
zLExeRv<l33Iqq}zO|3QKQYbi8+5#o2eUnr-nSn3tgSs?w7O=k@<dN5&i{b2ykn=Kn
zlknbdf)gWvzoT)UAi%wY`vI+D%M_S4(QT2U|As-Y(8mGzdpReEE8WES%|YpxPsu7M
z6xfbDr+TZ<3=1$SI(h>@Z24A0_n_2?>-yrZ+5`)_YhSfWLU1j8zyqwC_~3)fM^2yA
zap;tPL?7p%e;`a|bl=8j#XXjEJ(vCQo5#ABz@PgtrNK5<MO#}IMi*8J)Mf#o-3Zp{
z5~^zHusuM0Q&w01Z>&b3mbvkRu&XVH@r}!MC^7tRpM@e}Q@&9Xj_C*274!4vA+&-x
zHY8k`K;=+G)kGE6$1Q@m>`!b2w`%Y!ZC{RU>J<}5S&8x3d>_1bi{MMp>uffTN}^)o
z4D%Z+_=Ip;CcN=R;}Hgd<2ad{8^9$YbjaqLb<xA8eC+8d3S$XT^D=>7b7v>Z<qN`~
z%DfdM&SAhSNMrJn%im<6<%3sK*Et<EC9-pJDzAhbc(u=SWgg!iv*-QJDoZG*_Nz-m
z$x^n>6v@>%t^HAB_-xHjRf0mkSAqydQe%pLqs-eOD5yTm)$IVA%8LPYv|^qK&ZDjY
z(R!Y-uV3TgHY%{~y67iP0uwD8&j@p6RKO|GS>E9ejr^-j!9Xit3H6IEi2{yf$xRPU
zJiTXIX``Y{rZy;FWZ`=%cVXnKv@XlUv`bDtnqnzC&0n50y!!Ri<v`|s=i+ER`U?56
zIoeb3`zP}aYQIU%QZ25o`2Ah%;=>Iv7htVM!BHN_ptvh4K>On?<<nu$S{f=T1e_QC
z`YBGyr~Z?u6siy%DriP-OpIQokaks7J%>=Mo&VX~`6@r7+S_YKo6yA7Q&ny6><Zaq
zmbyyM&HHBeA8Tkvv{Ce`PxKWT@4d&%Fvp0AHqCEeL6OLx`H(zsz;ZSEUg6&z@xqPe
zL@?oA*n;BxBj}KYDm27jz$n&3KpsGQiGkw-7&X5EnwKaK{)^iL8c}i_*1QZ)FNef3
z%&(o9+IS}$@X*HwSsiGSKETAWQ2lLkC~LlPQ~TYBnOnbob^v2EejZ2sI#MjdB(v+S
zqSUcaQeI3b)7!M9HYAge`9Od1x`7rEgaF)z`gUjrtO+9`>fp^laxw)bFyNl#1Cw&p
z>%;@4TO}<oe+ee$KPP9ex*K{ApIXW4jN8tU&brtNZEiMkwy|85;XzQDx`{>|52v*7
za5Gqb&c7Kj>>Xf<Ua0naagK*G-X~D78oh9JC2_$>H0~2eU$p3{<+>ypySk<KawY!;
z%lf9`ywsC+zAjv2-tp0aAl&%_&l2u+VC~{;)S-g`!lpJ?-x=Gnluz+yeb1EQuuVV1
zi?%XM?%buAG29VoLS5?S6OPbjPL2s&Uyh~!Bf|@wA_>nm(etmj19B7C=e^SC!v|L^
zv_KJ~Bws|##~@d%_)I0<gp7HQFuo#syuU>L1v$lyf#IPr_HDV6l2^GUx>(&LtX0=m
zR1<Re07AvzE%`P(H+N=Q+8YX87a!c6m&cCIG5FS8k;7<IBL(5&#}IJv^KxbA7K~lt
zAy+Xsw~3_Og+B53?rfGUSINbO-ThW79eoP>o?t=+#{?f^e<I?aTE8jc5TbJf`^%rd
z9p=!kGb4~146zst2AA(r#s%R$nq=uNEqn1pKV91PNB3EI5ufba0**^4^FevJQ<E1X
zV$9Q5<W3{h8h)yM|1EtYUeO`sx9;cuo|QwEeQ%8Lw`zrwgbHTmI)`UcOD#;2H<50r
ztOqLy#Dm)JUw;?beBb&_dQL|jbB$B5<2f$%hAr%Jc6Mquo4)~|_9S60fZ?V9Ts;C&
zIK;;5U<lfU^UX0VxM~F?Pvyt)d!L`HkrL@NWLXFhTXho@T75Z`w_wJiLb(Wbr!ZGc
z89|3+RO#jTwzGELSfj{loloRzRu^m2ihN0Nr6w$4tWeV&RidPv-)Q9BvQyei@za(~
zQf^!H`uDD4jfXDF>oiRK)KH6Pieq+V*iA^i7_pIPROyTgO`6b@9r}@y^P9}9z*F;v
zT%;hwuFJW%vG<OQ3Tev)YG#sb8#5Szvy_qj29GN$$~3?HUUCrL37z`LwG7FfRiezH
zCsc_S&x|_@?H<Mv@D5)>ZOXqIM~X$eHwHG!JxF2t_;4W{+4~ziWjFV0R^{K}dE{@9
zX0Zj=;F!#r?d@$|pv3~V(Fz>kVCw@{2W1_dBbqXWgABd-&;i8{OE-qF<(!qzpA}Ge
zjLmV34LOF*?-}pJ1qq84TW$2WD)VZG456Ioew{-1k3Bx9Aw4B8k&ftl0!`eRD>^fF
zJN2pFVj>Jvs|}~Q92J<<hmv1A$`CH^(q4^{>uM%C=-kdbK{6yt@X2;H<mGp(Msg%a
z_)_e<u7}wr>1g?NP1Q(SjmnKCzei<#-;yiXP$hAfcgEZmuA6gsBZcrW8G^l~DLrDz
zSt*u5-@^V**Vk`1+g8M@=5>5Z&AUZ;8WeW)(!beU^&b8d0P#6Qn|iuqD$#*R_Rc-J
z8wQ%*ug9kJ$I3+Saj;L5Fz89^M)9j9F)6`3`a74Tz%c|a7~vWcI(2}E?hhBycZ7vy
z;idp6-H^k<SxXaary#?9P#}lRBwPPggw6V~Qyb{oXnHdk8Y&|*ToUfP#?qGum{C$D
z<LkdL4EaIJ9Zwy+T0wTT_kC*e8xz~jU#C_QS7dBPzqRj7*;2ie4=DS2_wml^Vrnj#
zf(DVCKUJJ8>h(@&$4HSR7Rl7fjz1(|>-cswzii0U%a~2JsRmIOVZgR=h9mdVge|2;
z7WUf*4l%baMWI|;ODwXlpAOyGPqg{2-LBl@V~$4z47IDR3@1Dl;x1J)XPc=VS8LDc
zMbswC8PC2|qOmcLSdmP3fj-xRq9(JO4wvLGf&y|+<<Bb=MMcnTO6fot#)V>G4g)km
zLm`XLfQBb9mT})%o>`r!gM$_{i#&h>BV-^jMzWYk-78+rB~~<;?@}@3ws!|Se|@(!
zycit&>K$yR_Qm!N-9~nJTu5ttDz3zIW=&WTyrS2bsrh4LL+#fGFR0kxu{%ULyst1H
zBvN=&*2^DzX-7A@b2h_}ps-4|OERmS<g$#$J6ZobD%Z~FsH~RJ?G9feB@;j0r^7$c
z`Q+MR+r%YtPl~N{O_#*`876%9=}e&^SY!C6_49-(APuO9kYbG$!&YkUadOa@*4^<e
zKA{#v?yRQ<J`Nd^%;@ry=zGg7+n7-&+tC1f1FYY}k?GsKd<nu;8Lx*w(X<s~=sX>k
z8qEzSG+x`F9qBVuRMakCA9uuSdEanm%6|TRyYDsoLzgi%YW9NL?XB0XpRb=`{O8<7
zat-j9V2lFA6rJ)0plu65!;z;rcrxP3X)j+AFs~frA4+jN>{~_XM*GzLgc026J}UtN
zo)y}eNZ%#xg=AT;>g)&hCU1jOZQMfJ^zBvdES<r?>;8~ux!*KfIw<te`se$uDJhRc
zfhH~zUc{Z`>fRql_J=KT+hT61K75Iv59_$Q?nK6&JzFVXj36t_Pq~~a@wx0~z#G>c
zW-D~lcmrpVFgo9#nOU>Fy~*Y11-{owo4vM7WNZCZ@+K6~+Fma~Ho<F+AaMja7Vd$a
zk``0VjPD(3+aFr%_N#V;keN)Ixltv=Ol>jJReDqg`>eTg44GQq|8a6OXfD0M+dr!8
z*5|BcDYO?-|G6!AcfI#{Zx4n3z`d2Er5PJN!H$8DD;z?uUJ&vq<3m14JBIu_1(BqZ
znhwvG!7u_2i>6Q*gM$D-jUPb+jD?-=&XGk4iR=ct%7G|aLdH2N-!lHukDIhIJc2ZG
zxfy4SOX(~vZl~l62fQ(va+(7P#_W5ezIm$H)7NQ(?KfMmZ$Z;V<t0m+n-?SMqVcH`
zFJ;!*>;9Bbk-X;_dZt98X~l7J<Nofb5JjwX$C*lbUlWr7;j-*^-hF;`iWnVp>lxcW
z!dp`3xGl2{T-7c}^fkYt5k{xPF-D({SQ4FLxrj-1Huf)jAw1$~&lNr%DPg6}kRTsj
zZ@i~{*hHbeHW#D}6MQaC+R?x}Jzdk-RA6uK_+GhxL`2-S&+QOx`m}N$Daq=SkR7u*
zwhpx$)!TDd;q^^9%(Wa5u(t~I-1{+JPwShdoaram9#(q$E*#DO<^p*E9{-dT!9u9&
z&JIW{^t(oPaw*=&Z-$HxIl__Z@HoV+mz%SG#Syq${wQVjt7#cL%?&8q+0QKzCM4Q(
z*!f<9mju4`a=~R%2~&ONp%s!PcgB8AN{3$1lRrCk@+a9LJ&2_9zclK~^z7sFB^aBQ
zhFdYOD5n<k41D3bFo{6OvJWud8d}M?-WJ@Kel0M*-)a4Nu3zBm_Z^$ak4EbLF>%4^
ztc35$1TFP+#T^Z<SNZgszks4c*SDOEDz6~a#B>@<RGt7vv~}MEqe^ym*~Lp<Vk&G9
z`6q~P&6T$PYD<Lo8D0DtSJ47q?RjsTfqSrW^4|nvKA;=_$+Ra+9s@6*lmHC3@Y_BA
z=#8c17$1zNP9JEf>orH0^eRU!$j}L-f65-;5YSLcY|AP(;M$SDs-U60NS*0y9QC5;
zDU<21UNPKuod2jGNtl1!R7T~Bb!O6~#TNSQk}+RmjVxz!%kY$=5U&AqHb}&Wvjz|R
zIP0RUrg#wVR8NpvPnYlJsT8eu$F$V!9;Oa@p_?A6m|RNnrjb*6%0W99Ssv=YZuv6U
z_lxxU=qVIQPs4cFf4u<md<apmJ~C$sh3Yfowp2{k**PUk)0A$B`>BCRv42#V{w*Gq
zyfcr}h&uP@2wmKlu5xvK!jUTYbs?Sr9d#^~JR2e2*KgeV_C{AkLm3B{O}lR>5qy(b
z7j(%_ZmWvuubH;|*d}uAYZgfa4=1$vYdhSxGH^PdW3gdN{k;5WI#))i25{=xL5A%8
z&Slf$3r{7L6v_NvwHvz_dUhK;vks8=%%IpR#>OEn{fDme)dkim2G)lgc);r4L@uIE
z>&blUrHOMJdeoS*Fjqo5Ui2#TzXV$(=XUEfH@Y}#M(u`n39UoX<+upelI*#-g1ISv
z{8?x)jDmC1|I^u5M@990eWR!tgi45{N;lFSDj-M*%+L)(cQ=Xxf&#)2(xJ@I-6<d-
zAUX8V-CaYz2fx4HTF-i)=eOQxy?=O_HO@Wv-gEca@j0KpcUF6}8M(m3^QMDFrdsw7
zH0rm&_}9djT?&8D!ket^P9USFo5>s&$V-p4teSRxH}_YoM-YXUUY=*P*uJzetXY_!
z4PT*jx<?!v9RH5M-0tyiKDNcKNOPzN{6bA4EvopTdX{zdO>zJEoRSXtqTJSvmZ#K6
zbYO4`T)lv;dp?5BBSok3GSg2M(K@{-{M<%`;m@Rx$Rf?1#b^BZ3mu0H=%7iL?g*_2
zbrMUT`CP51zX~Lyp8_mtN79gYRCKA;Z|Y?217p_0G96u<h)Qhi#qsnnksuB{`}zJS
zb0rb`HP~e>ak=^JUQd#1VCYJf1%W_^xq{9W3}dVpLm?|UbfG-74wxHIb%2d|t-WVA
za>0%5WCZ6+y!YIle?o3|d(CnMsNp+kSyd%vJAW7<q9yG!Hqgj~EC)jprh)LW*(VuQ
z^=P#%yQr3Nim&hlZQd*29>Sk+s*W~7-wn)<AhcJGS!>Q>xa>&8YZF92Z;A_h(A4Li
zU9K;wO8gMfl8h2fw58T`&O7OsmviztI=J|Ff7bTLE1BC*e=$&O6BnPeugVo?vTi$N
zm_3`IzE~t}$Yo8ao|UJlmYzl6KflMj-A)q6PxT-;VL!a9pPQ?B+vMB3VPM0YrZ$BI
z+X~z5$*pbJ5mx4b^+kg6od_rMK<f1>`KHoLmiAY}Jn1_(l3t>&tZjEGgVHhdb@{oD
zZi13j@@2ly34q}KMv*~*=@Lz)?RcIbDOnyYUhWz9;$I90g4x$|tempg?ix)RMr)&p
z#_=gP5M8x2svz?BOmc5BGU56I50BbA6<#f>@_RZ~4EMU7tgYP@@#ZD2Qebcb$&9&w
z%9h>Si(yhWXcdu($KwS2y-AE1sJYnBdm<Lz2+!%%P^Nq~?J{W`3~bmH9C&-@T?uPK
zZE6CW0^OPW7Syz9M+%hvH!p=Os6<d#=H*Y_!c};>>=L=*_MDS12*%Lq4V+2L&SVa)
z{O6D}ET@QLpSrC8>>B2K*xoK}51$p?z~R8hy3fzXejwk=%kv=({4ink^_P!v-q&q|
zT0>TKE)L*$5Rw~(tf%#gV*d`TYjEt*^vQ+&8_d^?9Jn9LZ8c|?*f6}&1<Evwt;2|3
z<P~j<&H|rvG@|1^%tLG(KS`rW2lAod>-Zn?Z2@;%8{+Tq2PbVy{ZGASZj8^gd(fNG
z2)KlPv|w9xCamjpDzfqFThfY2y3ncYqe3n8jt;5#clI)M?v*S#6MP`t`$b5i8a!gt
zQ}L{)?K_K{hpP{9=)!0vg=aCcA%2{}K&xU>e>RL^XN>P#jMO@!r;HW&lRuLG<*NVE
zZK{LK9SOMqU=KHcCMn>u4DqN>8BfJgfckE@m}x8b5(gLr0t>CFvqGta<6Qyj>=u2h
zl_yYJy)i1RAR)q@WHzjBsKwV-fXB^8NpCw=i5r?a+};tdLs1?0a@d@)XAy%p_(eY8
z+HLowkN%q`KkHYjL(&V=g<c70bq?qd0me*+ePqf)BhMrLTU|i1iqbzeS#um@v@db;
zJTP_SlM^vCT)yiY4)<r}=!zD!wr2d1!5qcjsZ4Yet(csx9r0*-)CIa9O0V?f(1QXp
zk7K_)mhQ$$)Pi_6)uWoXFhGI>r&cAl4EbOlzu*2Z;c>u0z~BpRJI+>uX!WQc>VYl_
zEo+XcRviTsaXx_DX)&5Q@;N_x#-Y&F;+mkZ`YvQr*gGR2jL)Sd?kvGft8z$O*6WhC
zuTnO!Q_|tL46HtdHRT?v6r?@<UL6zp1Nlq^h3-^+**AgBuw;fb&E<4}sviC@Z9r4c
z_V2N|&z~m+$3vCw2;iv=FBE0-g-2w0jNu@bDdS*b@jvGVe)n{w<_o~z{)`qIJX%~b
zMz<5XBrrQ>GT}yQREkTrto1v9$3!_0OQmH!3j+FQ=VLTk=e$K#;0-_}xcYbB4R<QU
z|CYPXE!$WJZO$d*kEa2${;<&%XKF<a;4zs_jrvL>X(Wov>v@LT`XgD-sBjQ26<)Ou
zzH*P>>LM28i7`VqRr_bts+Vs-1YCukb@0!VlRpnzOgl`9J*_%O`k1f<aN*)}TP|*$
z+}v&jB{Gug^YJ`S(-{gDWZDd8sqoYk8~?Fn3d=tR^~E^v{M6R;H%>NBY18M<(Ules
zt7DwvlWx(;5pm9O@_N0^hcOR9L~bptnQ|eCGr~TOM9?wNRVcBFjX7Z|d$g5_h$IH1
zl$>+LgNFgR>MGd$c>EgzHMC`K>><#Zd$B~yJtNQz-_u!3FcI@Krm>UF5AP9G2}5Tg
zSGW8of<wO4{aXU2m_lBRmsSq(w{wKMAow>YbGgKS0!5R_(FNdztU)i?<$oJg?6z_n
zT!l2?HfrXFPezPF#Jz|sQ~{VV3nUM?rniFfWeb5yy7CgAQmwyBS*L8s7fWd$Wwbem
zk4W^c{m6dub7;|M^-T$gvbZ$<J#O>YZOXvbka!W0qk;+&vd9iW^=?<;B?jh>1EBZs
zOC*q~S0{o*vay|0wJAtWwwA*;xECnYz+>ayJOd3|52*!y4$PZm#}?;q-!??^O@6@z
zjY^xcqghNOeqpmwF4=I}kkfX9jSVy5>58(l9ngGfYWtYG50A#jAmo0^!Ed5FJw+Dh
zT57CzlF$XBH&=T?BDB}UBVMNpC!S&SFWh$Te&!@ZTx|vP*pqs1QD1~5cXx7;f`|C%
zUU~hZDWndy(6n%9K>u)8?#`WKe4KcMp=L~N4jmZjQxMDa6vh4{-#a^+728fvbY;Mo
zz;yrkbhW|xuN|TP`NJnK8ivukPk-4pQ!mE*ulCxvQ(5mkCDN4|CbBU2_v~c9de)eB
zc<o}QZ10!Cn_I8EpC~{DCdJ-Y<jPe$-uWvazdlGS6M^KK))@eHZn}27`{Z@nDL(MC
z1DZnz@boJg0zZL5FV=f5a8W#TxxwK8VaFst_{ZeoM1Y+GNpPnT^t#ShcYL+C0e1fW
zA0Pg{ysO{*FFyQ*^{)n`BCE!IU#T9Fp2{e9atVkOVAqB8U$a9XhAvkoB8EaGlz)~N
zv9cUEEN~OAUXw@B0&=McRa-eL1lls=JnLN5$HOLSO3=C%llY0zau<)elSjO4c%$U-
z{NE!%3q#w|Up+JY^Xj@3bjO_=?80+%7AY6162`l;%9;2k2M0S;=j$6AS%1`E4eVTb
z#=xJF-tTxlZnScO3O5KF3PpuqD%{YOQs(Ts0g~-CXs^HHZw{KV_cj6&s^}XIHZ|zQ
zRsNZwO9>U;Q0}(yZ4U8su~_y*I$?%i4VZAh#@#_HkbRpH{G4=sYoWJZmttfOKWRh#
zlhpTbrq|KM$`E4RVTWo_Kpif2Sewh8C=IP`nw>iSl?$k$Ne$is`gOa6L^m%`ewYT0
z5(n}f_jST}Vo5-Cb+3IKr3U#B#dgJ*0sV77$S4y5Ht_Pwdg^(H6oP*DTS|hIvai<>
z9Y?NQn~^eml5iJ!!}0UBaEjE5oDc4-kHdI%+j{Y4g9S{VUHd6v_h^FB29yE~VMsaG
zC5u-#5J(&;KgcSWWdVWvLqqe_SU|+M_AMz+8ajA-;qq|j#)tE|b>B58pyss1w28?`
zx#)W8{FN9$cQk-NrTB4q6aa1ONR(FW6P&EGn3R%gwk)OY*CzjEKvwXb5V=l`vC_m^
z|7U<*23T?rlsNgh75wxGmU<|V=c{zmfjFO}jORZ(GO;qf8D8`ogA)azWEm^o^@lKn
zt-6Gn=HlE_F}0a(3Q!<+<^T-bD*-4nEXu^hs=oioFd*eJUi0&{4eMM|EU{ZkkXQJk
zT4(Y&T=e`VmCuPFD1rJMSH>}8D)2SWi_^h3XRFaEg-091E1T;EQ{DPSXAv%d7s6HI
zD0UE9QEoGLgkzyoPo^-eOcYSt^o+H$w#PW*;hi4T6uJe}g+^GUukFr(sOm~X64VLX
z&?~s1W$7EyV{d#Dt$fG5w2xglj?%NqjA=elh*rkFcE{J{A36NBgjsF4P{l<OTEAvp
z7cZ1V@no;9Vm{cNc{%GYonB6X5}Zr!M;rRZCYpqTRTzG;OLo>yxuM)CXZ@9~oFh(r
z{)$x{ciYBXYqf%@Uma!SSiL-ydoP7nM%q7T<ioH(x2-CM13GV2mOxq2G2EN&F9(x$
z`Vw(noKsX(mWM)G8Hzjc<ZE1Tx_9*gbGZ%tQ(Ih894MKQo=K({1jqH8&VR3}Oa!Vb
ztgM=Y;(?$p?@LV>o?Z$8isun~qBVK;yBEb)@}{nz=<wJE*la*T1N=&oR<eS<ZZEZ;
z9KSZ9(&gSZIkp!_@{SUq2>l55(?|o~w<Sd2FT~saXsFuC(r>UK{45RXJT=RItJJZr
zef+q6iTSTIUAyGwc~{-|>zFaNCuQbOFwMTJ+jHLMiq9U%K#2hL=>DG{wbfxzD?XKq
z(g)`VwFI2Q+UED7c5_|74GyZiSgkZ=pM3MsexeWv`JDD%Ob@do&i$P4#a;++?r3G`
z`uI5Hvvg?Mg&2_I#sDWID0pn*YD(701BZi(L*JrbLTyCDbEo7LCb)7F_?wvD!H^xA
z>yQxz15$i>?pn@!l|DvTkint7swKT2wiYycDJhBL-mIuvms{^G=$E%vW=9%*bZWe>
zSZsj&pPfDH>cy0oneNi87Q@3cvxn2a75IwA5Wltt4wpGpg=jS{)gQwYsYHLz9Jz7}
ziF`5?U}Vpm-kHKYd!K8~o+#EB{2d3x&`b4aE`ZGaTmk;tB-sfW=CJ!)l4l~-1IE$<
zBVPAe!edWXP?WdsY+7*ygLZfDlABv<N~zgWnLdlCkA3OoR^+xszVE7U=Icv^<^oJG
zt9=)j*+b9KodKsP_2Fs9s)e9eSJvy{HnlT<tS+D!|Cu?j5v;Z2f7G35U~NLGfN~I^
z{AE<+EEC-nsgkcN)Kkvj8>AqdEP>iLc|yGmyG*Y5^s!l9LGg;XXzGe@`Ybu5TQ)nJ
zix-gj=43q1%{ZP5alH!U_}RJ<j<mhoog&oTSSRdp*(=7bFVeF=FMy9ER2UBa*gTRI
zihxD)I=hq>&1IQsRGEr~q-UsC=~8jr44_LCc1bMiGwe<j=;%i*M5zzWItv!r6xvwz
zWmm#9=O2-jtn?u@CabBCC|O&{zEJ4pz&Fq(K)`92)XS~DvAr3`yvFkZy4ay=9MFwH
zngm!%WBEEJYja8^rgMBRsywiM%{p{bh<9-K?u!CS`@`RVjF!j?9JeRjIJ@Ram*y8s
zW+Td$13#27gza{O*9T=Y=AvsqoCK1vc7pB;P8<C<Cp|Y{5CvlsbFA>;w&V&uV?gLc
zb_QGr)oo|HwjCB`H9afD+X4TTUfaM3v^o~>oyRJ;JPT@%W#oMXk{6#agj3N1I^<Y}
z0#S6#jH}mtss5GT2OR%iHs=J;I|v5uXnU;3SHe%}QPoYUT)m3mT7RQ)J%X7kkHA|e
zq{|yd=BeR;8LkS-(q?7Zr50x1Xy#<h%S4#AbK!4Jv}73vxu5OzsA_5l*|4m%IHJq@
zTs{;<7>2D+H4n_+h|?5|ih_QH$0`BS38q``5<}o|dJucX6)en*f^~JabFj9~*n?ud
zExi&Pb-I-Mt}dKBy=`aXth{exK)wmscmuTUxM+pFY&HB0wN`=+bU-$v^~m83kDCF<
z*ClHwamT4R6k}O@Itut{Xf*pH<!UQpS^&QS^j<h#al6z%-1H?3G6cO1-grRqMg^-t
z+DvuL*ovjy+(d=GnR_ZVy4+(oZmG{1+d9ErL>iF7-R<A0`<{v3@~pYDKs<fu&Z$6{
zBL0r!{;Oe^*&Wh|ZSU1-7H(*D$=m9vCuG6*@p~Nhf9kbL2i`U{EwSCh2lTv1!U|B7
z#T9eC4SzkO0hxfQ($qUX7jIhY0tpeohG`DfMh5RFgG${+HVOXe4*_AaeH#aypJowX
z`s<22^YO-65Z!c}@pJYR;jT%y$<%@xcyWlO-tidq-m+@XG=T~IOEx}kRn>Jg=9c(b
z_Y|tnHvn+o6Gc4Vt@nrqHLv_guZ~%SS_xFTG3X|3IJ@c?TkJV<d0!)m350KvjC{c*
za9ACA3LXehG*zKBM4iQ-UNr$vgD>c4%h1v>5g-iphi#XHAQxbE$CS32kB{&>P3;M4
zYj3Pa&j{owt=Ys{)Hp`8F61~=PiVwQnY!}jkyq?aZdM4mtltN1J<ev^rsncM&pn_9
z!3t2;o_Y;)p}Z;8A^?0PP@0v(Jd2&HBH6@Blca=O^{i9rBJP8i+pKf7IUtYfpMDpI
zS#-*Q^piiYijimGmn+mG=R*zlD`!ksq++##iPww`_I-iFMSTzG0juibHyNL`mp|ZG
zn<9~%g!BO|us_|3Nb$=dg_e7l1=)82GxLw4QiQg~A8PM8@j^qivB|>6LYsv4K4rc0
zQ3CgKGx5}lQz5TO&rmwNu3VjxYX$rtrp`}N3K#m2?n|NS$D0)wN}v%W$Jqt-j+fst
zlJ=p4bMxymX!OUwUN5lVFMlySyyLY0vj!wL%<j4a1ro;KyX0}b@sHH`6kc=VAu@Sk
zJZo<M2teYZ3ZTqPb&b59?}M^8<iG#mKNsU`3gBiiH`kMF-i2xmh1v@|7n3$SCBAj)
z$*BNGMCU;U2Cm%?{wU~ZzyT5{<`*9o8?G7WqKmQZ0{xV?F2>JZ)Nd~oIL-W#1;Ps!
zSz1m>(n%|cPGylwnn;xTPyUOg%Rd~x$4a%Q-5igpC=5&;BlnQuC?KgdItA1Al{+~8
z;2K4Mv>Kg9o|&&5GGjl^r`7jb0P5ryp}{JrYJAUcck3c??Btfn<NEm;oIpus3qgHM
zvt94haLdJnUEeUOz3;=R`@T~T`)aW<nq-03<!qDs>`u<G^$TTUdG68BHuwz8({neO
z-U7?xrj+*%%VkI?nwo(jDk$D_qj+x@(-hm#y}bRzuI|vuvqoriZ_$d?M(cgsoD`H-
z$Cf|P@CO54pdnb7ATM#njpN5>0DJvSm$_k$Ze4LM@|>XezId|Z9tED9tEBi*P&H(^
z6MsPkbbR1s2?9iP4c16@!#<v*Gk>Fa^>Ro1Ek$4k<kkYa0FC|wj7}*W3Jsmi$7fF#
z4S<aT&8e!a`{v2F09btGgr2L&0c$kxn{CWFPl2^U9k%_P9`Gye1==~7pRx$!K%vj)
z92U*Y;D$U0g5@^Gq>9<AF}t(F7vEZmbE=#-abvqv5Wr7G(6<^g3*!5T_M!&6F{n1d
z!X?TW-U5T>p*C}^(GWv{LRLa3>XY>ML?PF2GilfS<3@SPbANyyKCx}swCcx?t~)AQ
zJVqxUi(qa_nJj)}WM+~=Ip?~wie3PYaLk|;XhXpOsEdiLY^5XeqDurmc&eA|%y|ZA
zxDq@X{y9B+1-l3Wr4x>Nvv$G;JdO7t4sir@Zm@QKcXZ3fo{q10%`Jw%sr<JxdN!>s
zkl6lJv*gFq$HjGe`vaD?&!`wg#M{>QaZxD3FX5rRXrCl^*2s0vtjv{On|S)sG?5|O
z$;Q*5$?pY7c=w?d=v_j`?~U-x({FH%E@=y=Z`jP{ohK$rvmo1X$BjjIwc3Q2cps4Q
zC^Rn-AhzhJK0oK@?gHHoQL2O>FXeeuwp~S0k=^R3-4xMlUWM<7`iR-$yJT8wDBlA>
zxZS`#IC#z{^atd@(Y(aQJvi#~sSDj)vjJrFFpz5rs-J~+s)LT&$gQ-E4k^%y0u<Rt
zX4^91$F7Av)qp~WtVcSE7o!D|*ssUMk>cYNY4Ds21~yCD_|Sk>O0Tj@x?U8W;vD>F
z;Qdr1N(Hn$BRi8){b&15%NiTa0XD1j(%KN!w!f>mSArUdb|q*?^{>FP;U-B_6>6nv
z!I7I^xO4uPv2NnZMfua!%_%6>`n&h7qp_6|gE~ht%-Z?edg8d$a4{>PpDD@J9Y+%y
zA&f)7{9R=TP83UHEt;BDPWlF{ZK$_*bk9V%B(!(n)A5HRUgV+IS~hk#xkXz*V+l}c
z0Jw<YF-fT@uS{Cwg<iE#exEC5Ha*&hu1xt5!EjsR*@HHF0j7fNv~&di&2I@wAZH{N
zjBDQD7+l`@2*ia3sSQCO3DC6xL$;%V+aUh4e~~z|I?Mv}+#-AOK*_cJ&cdSK^au%2
ztctzreXt{cikQx`5URr5tkl_bHcJf`ku6~b*;u=uLr{eI!sEEWY~N7b`UC8RpB+E2
z@xZqHCh-5)3ckImM(BfHW&14rb6m%>Xxk3O6PKJgz21H#xIf7boYSFlnZ>1|Yf$K0
z2p$v~+-8&+;)+gwBc<HS8@ir&|4o8A$k>!HYlDKtE4k5s3k&#U^PZ=?L{d+=g<rkw
zWIk)mHL2-<2q^tB{t}K466v1eZ=u@<J0Fp*ZZVMD!)|xGc5!*Q$pc!{)NdVcvAQ*k
z90kzcet1LH9$5*0$P^p}Ymc<n41u*dz$@Xdk#XInzNk2!xNeg%A(X4pa8j!SrEtuw
zhATufgsr`par5)55Mjx$H?G~k5lnex)TY}l=;QF7#s3DARs?1LnXl<4E{@PqvQx4X
z@19rksmP_7<?vu|QBhXH*)|p4%jCJ=4ioZ}-b$kZ>B&p&3FCM8?&F$D*Dd`ST(l9M
z7YJid7XF^nuuHSyy~cZR;A}OOuWvcAn3tWO)y1wwNlhM2i^2B2`7$lsTbduXv36k-
zADAv!1wXw*dg6Xh6AyEC5uU3`eOCtbPcFIUXU>TjkXi4(hlM5avjL|n?0bDO`}u_a
zW%S|JNyaVXb24h`&MzvDh_Zg<>VS?V@w&qLPD^aPm{ycf<x-M$Z{5>AX41P>8?8ak
zhre(vYn+uqi=#o+9v|754OASQI&;AY?znfi)M(?@0x?ksPnZ0Ce%uTcQ<;uP0k!vO
zf%AT^SZLT6b0Z_2<h88S!q*wZ(b<`a8JcB#P;G4_Xi^azMlt4wf_(`M&#K5#ok&uT
zRz#MXx8Eg>jxxnQg5a22j=iO)eRGdIhK0XC!XqHyB*D&GxBb^g_C(<vSk7=(<_~MF
zrRCOaKI*oE1`&Ci+|wMl{rnn|x$iwr#xmhm2Uu9|-ucCH;;I&1=;>Mu<qUmRxGVEa
ztHvXNIolNWNJB1i=CK_8LogL^i#X7mDQb%Pn+~a!g^Bs=g3I}gc1w;}D6dyoviITV
z>^$wv^hlKe8fB?x%2Io~Q3-J5h=d=#j3;+M*Pq|Vs|N}Di#Z%?j-662I<`=+HM(f^
zF15L=xz+W`+>TX2tqO-6&}#kg(Jd4w4^j9E1Bb5+j_ak$%bm(a4(nbZ0X$}#-V2XN
zm?_$oY2&|x9DOOLUFZF6F_`6`pF1AH&@#0`#%sz$b?ag_W!@wI;@HX$(KOqSzM-YG
z8^v}sTId~p_ihLeDfL_HRHg6D&7<8v2N>PC;vhywvwV%{PS$Fu=_7+qu4uUo+ba9R
zxkc;d?I=ZtXgPX24EDPv4>Gb4D{CGUQwi%EV2E~aUboh}iG_9h*iGpt{{VU->x0*1
zU!$&FPHDQ_-YPjnw1%GhSw4YG7Q9{lC%?Sv%<C{#bPQ4+D->_vq6cl4xi$ucLc02C
z!o~B<e3w|_b-9%q?HS4Mi}t`1TtOandc@3A%@`ja|36e_nlYd=32%XKM0b*48W2=U
zugiC<B={?GXgY;2;W_D^F)_K`Pslm(n%{CgPh*MI<{~#}G14k@&jRn}rs(c5Dz2;a
z`ujhEI!eh)%+Yd8(TZk5+^t<nDek%YAashMO%lbtUmWI8bFdgK?d}QH8HRF_sxT78
zz~bUUL^?M6={mWf8amlI>g-mGRKpT}TQ=Cfoh|gd$wFyqjn}&J<B+@wRPkz?d+-w~
zkoL+vG{s=NTVfZFSnya~S%ruUrC?H8mQ<w%tpi>>xGn}Z;isoWuj$8=_N;Sytr)=r
zgMz?oSGYB@;<Vk?D$T>i!}v1r#f-fx4Ff~cP_KdfgzoUXfJK*nrG_@uMU>^@aQj<?
z<sXQ~zKBxu%MYvduF&r^G#%|-s-BLoJS{KIAelT0Z#vcTP#}^XbY1ZWx-X-ClIq~-
zkTE_1>2dH`AZ6JW3yZ8l_LU?As-_%$rzV-9xfo1*Hj51#*<&y#Y+A)S1mcH3m~+hD
z^p{#WP|Aqv4il|yC7q!U8poi@7KlW%CqycaMlWOtHc;c$o;G9EFMa(aRD}m7M^a-f
znD}~;EH-&$R8)~`0?h18s<yuGb$X(0PI=ufuIpC2D~jqSS=Ko}FL^io=yyy)t(XS2
zV4;lo$wkE=kS7_j^Qf3y9r!5k5#c%2H!><Rh>TaxY|*uarANOj(;~y3W?{x3>)O~w
z!!()M*;O^A%C5gRFg^dU^kDxQ)=%BG+9}(SJ&}d^g`CP#R^b=ues4V=QBgT0rC`{1
z2y>Q(Ux5&NMOFn;yWA$na)=_JIW9um&5|rwP*bJ96sCDt8p5tR<7?vn8bG2dYXS5?
zd1WLy)8DAsR8VaIlGeOC7++x!AuSW=a&oLp>9zH47W+I~dn9!sXGmcy%RO1A4a&7+
z80wBHHPJfUUy%-cI_$zr8g8-GT`}O2XPo_fUNNRU>PKRtPFGeQ>g?p!XnEcd!Fs>2
z_+=KE5+!Q;tqvjKoE#G6NMW`ctp+3;#6NWneig3unE1%xkN%BcS4r5EVF#>&VKjbE
zQZsU>-h%QmCtfwqi^n*tmE+kaT+j1pq7;$D)XT$hgw&aPmNlSFO{w{c9OnE&U;AtQ
zxTA8=QGL$LOq#{;tUy@cEn}fdH)J<|vKn|-YPp&PiQ|dw_&*QUu(%|}lfb%>>u^Z4
zuIayqg@x5%aRehy+U>CH(k^^m-ctbKN*ngujig}6>mm?G7%=9}xXU*jkFty%9QM%m
zbPgqRFtWz1RqkTEJG#2m&5Oz8Rp(?WCol6!*fqh=6O-Ji!ts`WzZdmfskzk7V|&E4
z<Tmjqv)WXTQcJW(dXI!v1>DrhsV9#JsXA9&Z*8@dR=H)DLR(PN187%vgs9-blJ^`_
zU4>t4^4a)3E(U%y7w*gRI?=JcZt|0$EkAAb0-YGz3>79k0zA{^%sGL0Rc$1|$%0N6
z{m!}Z_75zN(BkTy0raAUhM(2)&_4vr_K3q4e}9d7Ex#<MofAeimUDEtr3ThL2K$=L
z_(>)Sn{K{2k9z(Z^L<>4^B9>@yCt-xosHVuit)+XB}RWg$ZwZ-aEL8R{QlRq>eaQ4
zA@TZS-6R3e*DKCi&=RXsi#6AqFAaeX^GTojkcDU%_mt)EInur>)udJD`x#`V0Gl_6
zMrEk$ooQ1}p60ho%Rsa%-U^HD<u*lp66uLHQE<)V@|h3bdCr?+2LbkknU(#@KD-s+
zvQak%Z|V9)0n>C4jq)IyPWpj%`O5UB+kkhyasf!e7pgp^94%LBpZ~~S!IOimQEhyy
z0W=1*xfmL<IN!0S^EfbT-2b}G<l(o;vz73{2@eIkx{L7Djn#qmsy(9HfjlBa>Rwi#
z*^}%b>SjTUf07Cs8^4`+RReMBkuQpP?j!ff@OQCKc3#eKQy@&J)-IMl(SbM?npy4}
z0pt)FdRvLDJBhbKty8UFaK@)*z_CbH{G!oqz<Du>*{mzZB0l8c=*nc4@udteZx>!&
zVMHROm7$fGd>OtbG^%_tUhAEQu7&C!pZk3u_B)FNOL}!Yj&OBjWdLW3@~!RUJiS&&
z)E!DHD#-94K5(1bL5ZS+KsL{dql>b|fuVL((Aqt@j*f%49zSKEK|L>jvQv(nzX7#K
zuT1)x{|h>L`hiJM=~#(M-OVhBgiSlEEd@gfRqKg>Y4S%iby?fGteKt7>`bz`CU#kW
zTR1v(*%^QAvqY{xoLn*W*1@8xKWl!zj6&6I2?{jo3nlTR8eo}%tyBDV<ErikG%@F7
z#u%FBZ`GaG40;@Z#Md|b^CkNIyEsEk{2VFD-d7&lrAvIg-t!uST8h_<emfUil9=zS
zSnULtsxptUqUgbrGRH1y8^7HW=7M;-PEc86mMI>d)0X0)*Z%57(9wp9*PD_t?Rcn)
z+S3N{)Zky0^RmRyLC@8_hMbU9O`%dZ<e}U7iw{M<SLUH1RU<~!=i{(Te>@O1mSNk0
zq9X#?WF%duy2Ye_KymILgz_+sCW;3NQOQQ8TozlWD7E2J_k!$q=lY#eIk`^QHw4H#
z+Qs<9b5|&)c&-Y@A<_WcvpT*)xLWHeHD_$>$bemB5zow)(@U6g?*A-;f>N+j#-{7h
zfX7D2iD$K$HR`QE`bpq|rAED6Ld8aJ2roGiZeR%|4pWKXU_2f{heLk{2Wv0TY0JMG
z>x2#`uSul~%F)4rtETEJ8NnxV{E5nEwq&f-rRGeP*b28D>>&m|ZJ0`l<FGTieikZ^
zih`kcajwG~5E}de);)V*bjVbpycAsq7)(;v+A?ZtD|Ss_)Jn6$BZumIpWD;?6iFOB
zFYo6kkn3@e#PU2QPsr0e#j`p;3%M3=jaKt%zB|aFmtA{y{U7swSzVR?ef;dY&3LtL
zgn8Iu-RV7DbCJA0KyesB?B2aw>byW^GkW-qOfC1K(EGdL?>!yN`877(D$sjXRTCDi
z=)0YK%NW~{LLFhq7zzI#KPDtvI3t-iwfjk@C!%L7){vB$m06iN4#J!i1q-sR<WtXB
zi1svSCef|#rS(WIw??PMSGmBUz6dup;Lkj{KM*4$mSInu9nzs`DdZ(b19$3BKtX&B
zBS={F(s72wGeMSnHh36s--g}Ydk`(X=qluTod=h~8z5l^XInl{mf_stzL%#iGvkk4
zTuWIZzWwgr;>UeCWB+Q<Y0(lBkEd1V`rWiEiNVehR2O^HO%|ADZku0j>-WORa>=TP
zW2zT;BgAW4P@$w43oW2z%=LUeQ2DOkVbiKsi7;2>qd=D{x!fb)pZ*>WthT&|2ZjsR
ziECP^z?(35F*CnjQoBMt!eT43KOq~-Bbfocgi5DlIfz@T&lD^ZA6zk(;o%{TooAxx
z>Pp;-Xd5WBj&`&q4emMPGR!IVEKfe&RR?PzZ4n_&_mB`=J{x!jHH_p~SkwIN+nY@@
zEpAgj>X$o8mpXnsT@6r=K~}+){elG%)wBs0iqJ58D`%5eib8BDQ%JCfbJJf^6G7L^
zV^?*Ww^Z$@BNH5a2alj7S`IP4cS;=miWnD^CsuQ%DxWhU$uxB>D6>1Q%3m|xLYJ~?
z!fSdbSH%!QIQettM&3F*$M0&}F<UdQA&mxa5inDk67GAF3<WKR6Q$Rnu0mhdOkFEn
zC^91NfILT2%eFVg*4hAD-Bs9hw~8N<u{DW~6Ph!9wAzYu!u$L=J6Pi5<02SrNzL-4
zQJ14?JOtN9tmcSUj#YQ^T(wmUE!xbCggOBccx%q{fcir5%H!e{<4ZTI-zGjsUv48K
zW3%FUgt#JCXy@mdiEv?Vdv$}Nu$1UIdG_M&+`K|o?HHI(I=K<%%oiA|OWoo4p5-ll
z&j`#p7M6kZ(Z>C`fpNFaVw2JGUGKQJ;?VN`QdXFBpxwmfbKIAKu=Y|`jQ53gG-Wd<
z*NjhbBCNc~CdFIqavw{ynO;S87^W14HMKdCHAe^AYFRpBp$IkIY~d-X=~kTR9C&8X
zylr;D^^%&ShTUNIV*ZHbJ}xs+FsSb2%KU56Li{;l2U{G1u3_qH!#KzA`L^bt{V$08
zmU0`6cfExT*PIit!YG1iuG-ivIcN)pZYs83IPnIe026}pe>{mByJ+3|`urvGv#uTR
zajsqR|N5qP!?*RFM~%HW;a=;})HBHBuF%Efwn_JH;i_2um9_+J-+Zw{ioL8AiHfo2
zRka4&@fD}KixITT7sQy^;?_^f`l&tBv-10!z$>+jmn8&78S94kbh7h#B4&;M$8{{M
zV;jjhf^qekNly8jFP|<vA_DFNBT*P<=q)U)``homc&Bk*J{3J)p3An=pdWUDsHiFa
z#Q8gj9^P>bEUUCug)U=Zmo^R|`S0(YeFT*!Bp*XSi0k#>v$X>?%{Vr220-BPKO7~l
z&H`5Ee?HzmzB=hY9Dmv|oaS%(XvDB~Mv8<p1e_+rG`Vm;1cEb;OhFH4nVkzUpw<Kx
z<FEd&XKDD?G3aVXyNnI)iv_gYOmp{3slQ0MsPh&}5s7`^EfLw;CS3;GlO_Z6GA{JS
zYjT4BGNYnGT_%iTW(A9%Q$ZCJ3ep03)uG`1Y$m-U09TrxOAEmVfB(Op0P7kxIQ2gr
zpG<>k1Hsz=eY^q8^3@yoACK?;-qn9P-rv6Rp8wnN-D!*uzuo|5Z|R_J%SQ>6b8ys+
zMc#P5E2328f$_BOG!@<Z!wu$E8k~M4{c~$|wVz|sSB7QCPFWz5Cj2YGZ%#bkByl8&
zZICIx@mKc%=4VSF?5y#t?^gIl5&$(iWsx2#m0OtWzUN*1_4$WRZ3`Y$cwnl*FG26}
zD+mAK_BB~9+F=Kic-ACvr3z`4t7rTVum3+y+dtp`_i6jD-~6v9ZyIijx^ZM!(9HuJ
z3*6?fPhSQa{)RL3BzRZlgFdcle{ZkS>Tw{hY!*bW`GW%<UJLJYF?NO|6|RV{8++|b
z1dOD(Obq-=0FeY`LCH_>np5$>v)w+fBzI3ng1{vr?Wv3I+S|F6IU%>jj{rP12XP-j
zg`}MJAa6BtcRQGYZO>mI$W;fH`B@(Y##@Z+!4-TIUh{P1jf#=ci%?|vnpwfwTM!Bl
z|K3go;TZ)3+px=mP}rC5oMKS53TAbLwqtB4d`%#FQc2~tx~ElXQl&EtBLd<F#__Y5
zn=>B6v|4#?v~jHVW29S*vAGmfdG|;%T^pYf6Oq6cr@fMm&V4LxZ2L3rDUdC14T9@<
zdR6cLTB=Vy(5C8kCA)nB=HiC;*)4_8ab%=y-RaPp=R(jFs{MNqY_wc0ZzA&w7qjhq
z2!(X3DB9+mkVY<9xSPLK7iP~{*_Wjp+Y1}hiZ63p(%!|K#dEvW%XPBAM!jKwXZSc2
zgj4M%Dv!F8`m8t_GT|}=EJR`EE%65@>s#JZ<W!!oZc5QPCYJJyKK|zYUCm1ygg5xn
zG3v{M)aMd5vFyvsGd^0yMrGjP1*M!Eizz>8CwW;)+967wdh8-nuC&h$@A1bydnWM+
z>A+dfbLvYywny<Mn&qpHI8fAQ?Za0F{Y6@#O8YOisgzj#)Sk;t>Qyb7niDUt0T7)C
zPkHJlP-CJT9i7Z@j{%74qMH#Zn}Nz&PxR?5<|6j=zE~fiLSVp!?r%E;;Woczj#dF~
zd7Ym@#cYu;v?n>Ii@;93xj6dnt(O&eqpE)QZoC{TwW%3n*ByxX;02=gm(4rhLf&+k
z#4nZ_aL6hO<t1+_@+OO=g_V=wwc?ndW8^w=cp7|*z#T&c9wxb7siphdI}hYh0|3M^
zQ>0SOZBWWzvuvEr7~7uye)m@M=*_Z<d!vC7qi2>o{ZP-tH-4!K+7(v7(`L@gthOMU
z<WVTot<Dr}IIF_nn#!o(WXB_bW$%+B((DDY{nL%QKy-D;AeOU`GtA6TA(7lBt@Y1D
zD*vSZa<I1EP7ozW4?~L${?Sjx9DGLJJPtP^4ZIr?Qf9lS=8>NRP|FO|ip7jrl490I
zzts8J<@xKa`h)5*Z4M@Y6pdAT0#GwUmFd9MZDoIlM#%3l%2TMof{_XZKuMymE@Rq%
z1X%daB29C%W!=aRXXcDr|Lf7w>dX(jl$oQ?GWnHGDO6STB-3zZ!{+2q?UI+6#Ci+(
z^?WqNy%vH(2ym}N95}XxYWbt7<7eJ<@H@H|Hho^FZY#&9LC)q|iW&V(z`-9o+s2n&
zq;wMi4+v0eCG7ev5cs;d>A~lIK(+L;YR_I&amxqC#fxuM1I<w7<3a&L%zcZYL;Xpd
zGeT+B?W}gLuC4T)ok*Rm-_!3~(mVD6K!~+j<HWjU^M~-|%d_x?-bT@_<zEpm0j4y#
zJGd4Fk7YoWx|{zqr;5)a-ii!y3?C5T1}qTVFlAM>APr1();jm8Tix1F17LQbVfV;`
zLhCmHbQ#Rd>`)MjU*VPGC8{*dFN;}K6R3UVmBdb^Zu&P`(Ci{CvePYN>h}l5)y!mS
z6jKDUhZgUhOxIzLZTHfE&`^XET=u${EzI&n-n8BuVS+Fu-Te|5u0Tt_-#BwVhFMe^
zL=6hgIfF$*bXTU-<rI0aJ*ULN(eQR><Xv~N5CX!L74LDc)S&MnF%dCKlHA0;FF=Yg
z_sAit6>lHKcUu~ysw<c&t-i<8Cc>A|wEw#w(3a;8eCu-JuvY+w`~lRXGc#LrLKQ7y
zPN3R#p*XD0O`_=i1@L=%wuaAcevXot|IMQi8XmT~x}YYgs8zDmt!m-XV)MrC)`zFi
zuK4dDCIS_qv)3$x`d9W%f<7(*zeeI`8ymLo6H<tSY+S313^6rn@A7gXF6wJyoG`Z2
z;-JM|b9hz(axKunHoNX1QfN$j;RwfM)V2JhDZqXVye0D2vI{F;^(E?fY;HtW1BYA(
z$SEFPD{x#{dt6#hu2un<I`VDZfaWWFsIQD=V|8T{;-?=@(N<;gabl0lX`lLH06tbd
z7o4!Ta(aHvXN}{sq<*jA!m%~%6?0soNH+)i@LVqGew*xGAsEgVPd<LSf^FKXqpNe+
zKQLn#b1o<UWzEtnr@BBdZfE3=DKy3!9ka1FvCqY3)q2#<2|GQD{PF5p7DUJe-RIuZ
zS<B(1s9N8C#3c%}9nkt{dkPt#G_^24rGdZhk&JAGw>i6CZd%mz%3ao;?7f$e0FFMF
zyehU&9Y8>%Y-@z`0fgWC`}bE6r8V_Y#L?@E|MQ?$*|S{$g9VY)i!}RJHf&vEr`_b$
z_J<I}d<IO3(z}z#Rx2j5-Ncowdtmu=faPJRrK>_72wVZ2<)UB1jN-A&&e-xu78xE~
zUe3%fuds0E%m-0;$LeUY^=Wz85ipow(QPTP`Li*y4jHF-sFhe+^709#6Uq>*4aWMV
z!eypZtyA@Z-Ywmbf2@G|V{HU8<tvh-d$?+D)mQK4m$rB>bTn1$U^IV?u^{#J8})n;
z$EpKVmrRLT-Au--oONlSvzQeqn<O#US8QF6qU32IZy~vXFctu-Fh$(K!rH!L9I;`S
z_GfnVYx$<TN|!=t2*G|%L4~%s59aSKRJizFZA_llcfcFfgbSk-*}N6x<*$Ix!wqlk
zty)ZcekmWg#=_!4PGRONu@wEVxlABOR{S=D-TeBf=yEk+4Z@k3a}EFq1L35Ifobi|
zGbOn?=hZ<AE}D-9`-Yx@738tER%Sh@1zs8z37G$D)NT*A0VXW)u<u^OrOe%cmnSQo
z99O|~7`0&LUMLbo(^;dudk{@g5V_HT@`B1r!@O5JK{8$m(g6GC<6~A*8BDA?*+=zQ
zwV8<_XZsxmS4x4gTjbEm-B9vYyi%2xb{0e7e%pS7L!Pp0{B}@)_V%wtvGXJi0=|l@
zLMW0)Oemd~oO-}mNi@fvhoOckRx(fiANjCh=1f3j6?a&Qo}a}+MUiSXlNVKC6ytds
zeks7Ms2=aneNCG!!I|kSl((~^4EE-JyKvpyvjR(TV$nvc!Xx3zrfI^;SUWocchAmV
zh#ri`nrOyMdT%fQ%yVeE$4JOy{_FL|uR6nnZ$%KB-T03YSwXHMX$&bok^n}E1PQTT
zfA2+1v9};U3knQE2aB0S3-;Q8>q>fcetAwqg*vD@N=8l@(Q^o159G@G`H}kqzz@8B
zjYhn=8hHQ(0droJPot#gIzLft)0LG}(3aw>2uO`W2aSfh7FK-{eg#I@KE<rsm;k9U
zNQ17+=UB@LMBbIV1z?wwq}$*gfJtVNL#o`BN!O}bk<6PoP8BN9Q$}apt9C8Z{jl>b
zFlcu7^!gP-Q1}A^#2gBc;>7XVcB!&_9e%>!c#ly+1V8~H^7Wv+PG!EGd`aS7DquSU
z2xI3V1_WsNf=)|HYWu;aizi6bmq`#lfJ-_k1X!3k4%C*WtI#bFM=mj`{^*AnH~Mh3
zD&0C2zB8M=*1^Sgr7*#oL&@or!KsCyK4#g~S5Cz3<KvJ17onFIwBiZd;EQdJ&q99#
zv;Ppj{8vx=zkmJ@zV|-?&HpBF8K+yXIqJs+2mpY>1!G#^O+GzKN9~|6a<Yu^RnW;X
z%|PGkyg#kC@QAq6g(8_~(<{aXsLBNY##G$qR7otXpLN9B#^*i&d_Q%HqnEt(l@@q@
z=<;Om**j!lzQ!YGcD$~1+qU)xsdDi5zsDLslQ5U_k7DPKcuA|SejqEQ_$vQ}f&c#j
D)wWVN

literal 0
HcmV?d00001


From 948cb4a05a455559a7cfcb6ca5f378fa684de81e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Mar 2018 08:59:52 -0400
Subject: [PATCH 1621/1644] ARROW-2336: [Website] Add 0.9.0 release blog post

Short and sweet. I know it's late, but if anyone would like to add anything please push directly to this branch or post a commit for me to cherry-pick

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1776 from wesm/ARROW-2336 and squashes the following commits:

c843d7cd <Wes McKinney> Update pub date
212cf218 <Wes McKinney> Add 0.9.0 release blog post
---
 site/_posts/2017-12-19-0.8.0-release.md |   2 +-
 site/_posts/2018-03-22-0.9.0-release.md | 100 ++++++++++++++++++++++++
 2 files changed, 101 insertions(+), 1 deletion(-)
 create mode 100644 site/_posts/2018-03-22-0.9.0-release.md

diff --git a/site/_posts/2017-12-19-0.8.0-release.md b/site/_posts/2017-12-19-0.8.0-release.md
index 8c1f464ed7200..1ecc93999f708 100644
--- a/site/_posts/2017-12-19-0.8.0-release.md
+++ b/site/_posts/2017-12-19-0.8.0-release.md
@@ -25,7 +25,7 @@ limitations under the License.
 -->
 
 The Apache Arrow team is pleased to announce the 0.8.0 release. It is the
-product of 10 weeks of development andincludes [**286 resolved JIRAs**][1] with
+product of 10 weeks of development and includes [**286 resolved JIRAs**][1] with
 many new features and bug fixes to the various language implementations. This
 is the largest release since 0.3.0 earlier this year.
 
diff --git a/site/_posts/2018-03-22-0.9.0-release.md b/site/_posts/2018-03-22-0.9.0-release.md
new file mode 100644
index 0000000000000..e8778d55e4405
--- /dev/null
+++ b/site/_posts/2018-03-22-0.9.0-release.md
@@ -0,0 +1,100 @@
+---
+layout: post
+title: "Apache Arrow 0.9.0 Release"
+date: "2018-03-22 00:00:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.9.0 release. It is the
+product of over 3 months of development and includes [**260 resolved
+JIRAs**][1].
+
+While we made some of backwards-incompatible columnar binary format changes in
+last December's 0.8.0 release, the 0.9.0 release is backwards-compatible with
+0.8.0. We will be working toward a 1.0.0 release this year, which will mark
+longer-term binary stability for the Arrow columnar format and metadata.
+
+See the [Install Page][2] to learn how to get the libraries for your
+platform. The [complete changelog][3] is also available.
+
+We discuss some highlights from the release and other project news in this
+post. This release has been overall focused more on bug fixes, compatibility,
+and stability compared with previous releases which have pushed more on new and
+expanded features.
+
+## New Arrow committers and PMC members
+
+Since the last release, we have added 2 new Arrow committers: [Brian
+Hulette][4] and [Robert Nishihara][5]. Additionally, [Phillip Cloud][6] and
+[Philipp Moritz][7] have been promoted from committer to PMC
+member. Congratulations and thank you for your contributions!
+
+## Plasma Object Store Improvements
+
+The Plasma Object Store now supports managing interprocess shared memory on
+CUDA-enabled GPUs. We are excited to see more GPU-related functionality develop
+in Apache Arrow, as this has become a key computing environment for scalable
+machine learning.
+
+## Python Improvements
+
+[Antoine Pitrou][8] has joined the Python development efforts and helped
+significantly this release with interoperability with built-in CPython data
+structures and NumPy structured data types.
+
+* New experimental support for reading Apache ORC files
+* `pyarrow.array` now accepts lists of tuples or Python dicts for creating
+  Arrow struct type arrays.
+* NumPy structured dtypes (which are row/record-oriented) can be directly
+  converted to Arrow struct (column-oriented) arrays
+* Python 3.6 `pathlib` objects for file paths are now accepted in many file
+  APIs, including for Parquet files
+* Arrow integer arrays with nulls can now be converted to NumPy object arrays
+  with `None` values
+* New `pyarrow.foreign_buffer` API for interacting with memory blocks located
+  at particular memory addresses
+
+## Java Improvements
+
+Java now fully supports the `FixedSizeBinary` data type.
+
+## JavaScript Improvements
+
+The JavaScript library has been significantly refactored and expanded. We are
+making separate Apache releases (most recently `JS-0.3.1`) for JavaScript,
+which are being [published to NPM][9].
+
+## Upcoming Roadmap
+
+In the coming months, we will be working to move Apache Arrow closer to a 1.0.0
+release. We will also be discussing plans to develop native Arrow-based
+computational libraries within the project.
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.9.0
+[2]: https://arrow.apache.org/install
+[3]: https://arrow.apache.org/release/0.8.0.html
+[4]: https://github.com/theneuralbit
+[5]: https://github.com/robertnishihara
+[6]: https://github.com/cpcloud
+[7]: https://github.com/pcmoritz
+[8]: https://github.com/pitrou
+[9]: https://www.npmjs.com/package/apache-arrow
\ No newline at end of file

From f45abf0660db196e8685bca44dc0c9e70f25fe4e Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 22 Mar 2018 09:10:58 -0400
Subject: [PATCH 1622/1644] [Website] Add link to press release

Change-Id: I81118996633814122c8ff606bc80aa9325832ece
---
 site/_posts/2018-03-22-go-code-donation.md | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/site/_posts/2018-03-22-go-code-donation.md b/site/_posts/2018-03-22-go-code-donation.md
index e446fd9958837..41ddace9e51c4 100644
--- a/site/_posts/2018-03-22-go-code-donation.md
+++ b/site/_posts/2018-03-22-go-code-donation.md
@@ -45,11 +45,11 @@ This past October, engineers [Stuart Carnie][2], [Nathaniel Cook][3], and
 [Chris Goller][4], employees of [InfluxData][5], began developing a native [Go
 language implementation of the [Apache Arrow][6] in-memory columnar format for
 use in Go-based database systems like InfluxDB. We are excited to announce that
-InfluxData has donated this native Go implementation to the Apache Arrow
-project, where it will continue to be developed. This work features low-level
-integration with the Go runtime and native support for SIMD instruction
-sets. We are looking forward to working more closely with the Go community on
-solving in-memory analytics and data interoperability problems.
+InfluxData has [donated this native Go implementation to the Apache Arrow
+project][10], where it will continue to be developed. This work features
+low-level integration with the Go runtime and native support for SIMD
+instruction sets. We are looking forward to working more closely with the Go
+community on solving in-memory analytics and data interoperability problems.
 
 <div align="center">
 <img src="{{ site.base-url }}/img/native_go_implementation.png"
@@ -78,4 +78,5 @@ at [https://arrow.apache.org][9] and join the mailing list
 [6]: https://github.com/influxdata/arrow
 [7]: https://www.apache.org
 [8]: https://lists.apache.org/list.html?dev@arrow.apache.org
-[9]: https://arrow.apache.org
\ No newline at end of file
+[9]: https://arrow.apache.org
+[10]: https://www.businesswire.com/news/home/20180322005393/en/InfluxData-Announces-Language-Implementation-Contribution-Apache-Arrow
\ No newline at end of file

From 07beb518ebbfa9dc4e673b3a3fef781886ff4159 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 22 Mar 2018 18:33:52 +0100
Subject: [PATCH 1623/1644] ARROW-2333: [Python] Fix bundling boost with
 default namespace

Author: Antoine Pitrou <antoine@python.org>

Closes #1773 from pitrou/ARROW-2333-boost-bundling and squashes the following commits:

ad79a38 <Antoine Pitrou> ARROW-2333:  Fix bundling boost with default namespace
---
 python/setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index 453626020fd94..cf44b1e43a8dc 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -111,7 +111,7 @@ def initialize_options(self):
         _build_ext.initialize_options(self)
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
-        self.boost_namespace = os.environ.get('PYARROW_BOOST_NAMESPACE')
+        self.boost_namespace = os.environ.get('PYARROW_BOOST_NAMESPACE', 'boost')
 
         self.cmake_cxxflags = os.environ.get('PYARROW_CXXFLAGS', '')
 
@@ -208,7 +208,7 @@ def _run_cmake(self):
             cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                                  .format(self.build_type.lower()))
 
-            if self.boost_namespace is not None:
+            if self.boost_namespace != 'boost':
                 cmake_options.append('-DBoost_NAMESPACE={}'
                                      .format(self.boost_namespace))
 

From 47fcef369b8f2b1645f591cf332374745f0acfa7 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 22 Mar 2018 18:34:47 +0100
Subject: [PATCH 1624/1644] ARROW-2334: [C++] Update boost to 1.66.0

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1774 from cpcloud/ARROW-2334 and squashes the following commits:

de93189 <Phillip Cloud> ARROW-2334:  Update boost to 1.66.0
---
 c_glib/Brewfile                          | 2 +-
 python/manylinux1/scripts/build_boost.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/c_glib/Brewfile b/c_glib/Brewfile
index 955072e1ea767..e4ad9880ba7ec 100644
--- a/c_glib/Brewfile
+++ b/c_glib/Brewfile
@@ -16,7 +16,7 @@
 # under the License.
 
 brew "autoconf-archive"
-brew "boost", args: ["1.65.0"]
+brew "boost", args: ["1.66.0"]
 brew "ccache"
 brew "cmake"
 brew "git"
diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh
index 47d826251dd7c..80b79ab802273 100755
--- a/python/manylinux1/scripts/build_boost.sh
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-BOOST_VERSION=1.65.1
+BOOST_VERSION=1.66.0
 BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_}
 
 wget --no-check-certificate https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz -O /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz

From d623567da78a11572b3992b26788f76c5c241434 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 22 Mar 2018 13:49:13 -0400
Subject: [PATCH 1625/1644] ARROW-2341: [Python] Improve pa.union() mode
 argument behaviour

Also:
- make UnionType.mode return a string ('sparse' or 'dense')
- make UnionType indexing return fields, not types (like StructType)

Author: Antoine Pitrou <antoine@python.org>

Closes #1778 from pitrou/ARROW-2341-python-union and squashes the following commits:

c215f9bd <Antoine Pitrou> ARROW-2341:  Improve pa.union() mode argument behaviour
---
 python/pyarrow/lib.pxd             |  5 ---
 python/pyarrow/scalar.pxi          |  4 +--
 python/pyarrow/tests/test_types.py | 29 +++++++++++++++---
 python/pyarrow/types.pxi           | 49 ++++++++++++++++++++++--------
 4 files changed, 63 insertions(+), 24 deletions(-)

diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index e4d574f18b34f..be103b354a541 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -56,11 +56,6 @@ cdef class DictionaryType(DataType):
         const CDictionaryType* dict_type
 
 
-cdef class UnionType(DataType):
-    cdef:
-        list child_types
-
-
 cdef class TimestampType(DataType):
     cdef:
         const CTimestampType* ts_type
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index 2692ace40a3fb..a0f8480db99c5 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -334,9 +334,9 @@ cdef class UnionValue(ArrayValue):
         cdef int8_t type_id = self.ap.raw_type_ids()[i]
         cdef shared_ptr[CArray] child = self.ap.child(type_id)
         if self.ap.mode() == _UnionMode_SPARSE:
-            return box_scalar(self.type[type_id], child, i)
+            return box_scalar(self.type[type_id].type, child, i)
         else:
-            return box_scalar(self.type[type_id], child,
+            return box_scalar(self.type[type_id].type, child,
                               self.ap.value_offset(i))
 
     def as_py(self):
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index ad683e9a2ea00..6459496f64060 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -88,10 +88,11 @@ def test_is_nested_or_struct():
 
 
 def test_is_union():
-    assert types.is_union(pa.union([pa.field('a', pa.int32()),
-                                    pa.field('b', pa.int8()),
-                                    pa.field('c', pa.string())],
-                                   pa.lib.UnionMode_SPARSE))
+    for mode in [pa.lib.UnionMode_SPARSE, pa.lib.UnionMode_DENSE]:
+        assert types.is_union(pa.union([pa.field('a', pa.int32()),
+                                        pa.field('b', pa.int8()),
+                                        pa.field('c', pa.string())],
+                                       mode=mode))
     assert not types.is_union(pa.list_(pa.int32()))
 
 
@@ -141,6 +142,26 @@ def test_timestamp_type():
     assert isinstance(pa.timestamp('ns'), pa.TimestampType)
 
 
+def test_union_type():
+    def check_fields(ty, fields):
+        assert ty.num_children == len(fields)
+        assert [ty[i] for i in range(ty.num_children)] == fields
+
+    fields = [pa.field('x', pa.list_(pa.int32())),
+              pa.field('y', pa.binary())]
+    for mode in ('sparse', pa.lib.UnionMode_SPARSE):
+        ty = pa.union(fields, mode=mode)
+        assert ty.mode == 'sparse'
+        check_fields(ty, fields)
+    for mode in ('dense', pa.lib.UnionMode_DENSE):
+        ty = pa.union(fields, mode=mode)
+        assert ty.mode == 'dense'
+        check_fields(ty, fields)
+    for mode in ('unknown', 2):
+        with pytest.raises(ValueError, match='Invalid union mode'):
+            pa.union(fields, mode=mode)
+
+
 def test_types_hashable():
     types = [
         pa.null(),
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 5f962901c3697..12948500807ca 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -189,9 +189,6 @@ cdef class UnionType(DataType):
 
     cdef void init(self, const shared_ptr[CDataType]& type):
         DataType.init(self, type)
-        self.child_types = [
-            pyarrow_wrap_data_type(type.get().child(i).get().type())
-            for i in range(self.num_children)]
 
     property num_children:
 
@@ -202,20 +199,25 @@ cdef class UnionType(DataType):
 
         def __get__(self):
             cdef CUnionType* type = <CUnionType*> self.sp_type.get()
-            return type.mode()
+            cdef int mode = type.mode()
+            if mode == _UnionMode_DENSE:
+                return 'dense'
+            if mode == _UnionMode_SPARSE:
+                return 'sparse'
+            assert 0
 
     def __getitem__(self, i):
-        return self.child_types[i]
+        return pyarrow_wrap_field(self.type.child(i))
 
     def __getstate__(self):
-        children = [pyarrow_wrap_field(self.type.child(i))
-                    for i in range(self.num_children)]
+        children = [self[i] for i in range(self.num_children)]
         return children, self.mode
 
     def __setstate__(self, state):
         cdef DataType reconstituted = union(*state)
         self.init(reconstituted.sp_type)
 
+
 cdef class TimestampType(DataType):
 
     cdef void init(self, const shared_ptr[CDataType]& type):
@@ -1145,6 +1147,16 @@ def struct(fields):
 def union(children_fields, mode):
     """
     Create UnionType from children fields.
+
+    Parameters
+    ----------
+    fields : sequence of Field values
+    mode : str
+        'dense' or 'sparse'
+
+    Returns
+    -------
+    type : DataType
     """
     cdef:
         Field child_field
@@ -1153,16 +1165,27 @@ def union(children_fields, mode):
         shared_ptr[CDataType] union_type
         int i
 
+    if isinstance(mode, int):
+        if mode not in (_UnionMode_SPARSE, _UnionMode_DENSE):
+            raise ValueError("Invalid union mode {0!r}".format(mode))
+    else:
+        if mode == 'sparse':
+            mode = _UnionMode_SPARSE
+        elif mode == 'dense':
+            mode = _UnionMode_DENSE
+        else:
+            raise ValueError("Invalid union mode {0!r}".format(mode))
+
     for i, child_field in enumerate(children_fields):
         type_codes.push_back(i)
         c_fields.push_back(child_field.sp_field)
 
-        if mode == UnionMode_SPARSE:
-            union_type.reset(new CUnionType(c_fields, type_codes,
-                                            _UnionMode_SPARSE))
-        else:
-            union_type.reset(new CUnionType(c_fields, type_codes,
-                                            _UnionMode_DENSE))
+    if mode == UnionMode_SPARSE:
+        union_type.reset(new CUnionType(c_fields, type_codes,
+                                        _UnionMode_SPARSE))
+    else:
+        union_type.reset(new CUnionType(c_fields, type_codes,
+                                        _UnionMode_DENSE))
 
     return pyarrow_wrap_data_type(union_type)
 

From eecb1bc023d2a62536bf1fe2a3edb11673fa06c3 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 22 Mar 2018 13:56:33 -0400
Subject: [PATCH 1626/1644] ARROW-2281: [Python] Add Array.from_buffers()

Note this is shadowed by the specialized StringArray.from_buffers().

Author: Antoine Pitrou <antoine@python.org>

Closes #1772 from pitrou/ARROW-2281-python-array-from-buffers and squashes the following commits:

c6bf3730 <Antoine Pitrou> Try to fix crashes
a7f658e2 <Antoine Pitrou> ARROW-2281:  Add Array.from_buffers()
---
 cpp/src/arrow/array.cc               |  7 +++++
 cpp/src/arrow/array.h                |  5 ++++
 python/pyarrow/array.pxi             | 39 +++++++++++++++++++++++++++-
 python/pyarrow/includes/libarrow.pxd |  7 +++++
 python/pyarrow/tests/test_array.py   | 20 ++++++++++++++
 5 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 7e66999a379ae..80d64c8712513 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -47,6 +47,13 @@ std::shared_ptr<ArrayData> ArrayData::Make(const std::shared_ptr<DataType>& type
                                      offset);
 }
 
+std::shared_ptr<ArrayData> ArrayData::Make(
+    const std::shared_ptr<DataType>& type, int64_t length,
+    const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t null_count,
+    int64_t offset) {
+  return std::make_shared<ArrayData>(type, length, buffers, null_count, offset);
+}
+
 std::shared_ptr<ArrayData> ArrayData::Make(const std::shared_ptr<DataType>& type,
                                            int64_t length, int64_t null_count,
                                            int64_t offset) {
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 0a155af7e4a8e..660d0c3e56a73 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -111,6 +111,11 @@ struct ARROW_EXPORT ArrayData {
                                          int64_t null_count = kUnknownNullCount,
                                          int64_t offset = 0);
 
+  static std::shared_ptr<ArrayData> Make(
+      const std::shared_ptr<DataType>& type, int64_t length,
+      const std::vector<std::shared_ptr<Buffer>>& buffers,
+      int64_t null_count = kUnknownNullCount, int64_t offset = 0);
+
   static std::shared_ptr<ArrayData> Make(const std::shared_ptr<DataType>& type,
                                          int64_t length,
                                          int64_t null_count = kUnknownNullCount,
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 8dac57d18cced..afb68a2fb2a90 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -344,7 +344,7 @@ cdef class Array:
         """
         Convert pandas.Series to an Arrow Array, using pandas's semantics about
         what values indicate nulls. See pyarrow.array for more general
-        conversion from arrays or sequences to Arrow arrays
+        conversion from arrays or sequences to Arrow arrays.
 
         Parameters
         ----------
@@ -372,6 +372,41 @@ cdef class Array:
         return array(obj, mask=mask, type=type, memory_pool=memory_pool,
                      from_pandas=True)
 
+    @staticmethod
+    def from_buffers(DataType type, length, buffers, null_count=-1, offset=0):
+        """
+        Construct an Array from a sequence of buffers.  The concrete type
+        returned depends on the datatype.
+
+        Parameters
+        ----------
+        type : DataType
+            The value type of the array
+        length : int
+            The number of values in the array
+        buffers: List[Buffer]
+            The buffers backing this array
+        null_count : int, default -1
+        offset : int, default 0
+            The array's logical offset (in values, not in bytes) from the
+            start of each buffer
+
+        Returns
+        -------
+        array : Array
+        """
+        cdef:
+            Buffer buf
+            vector[shared_ptr[CBuffer]] c_buffers
+            shared_ptr[CArrayData] ad
+
+        for buf in buffers:
+            # None will produce a null buffer pointer
+            c_buffers.push_back(pyarrow_unwrap_buffer(buf))
+        ad = CArrayData.Make(type.sp_type, length, c_buffers,
+                             null_count, offset)
+        return pyarrow_wrap_array(MakeArray(ad))
+
     property null_count:
 
         def __get__(self):
@@ -787,6 +822,7 @@ cdef class UnionArray(Array):
             check_status(CUnionArray.MakeSparse(deref(types.ap), c, &out))
         return pyarrow_wrap_array(out)
 
+
 cdef class StringArray(Array):
 
     @staticmethod
@@ -824,6 +860,7 @@ cdef class StringArray(Array):
             null_count, offset))
         return pyarrow_wrap_array(out)
 
+
 cdef class BinaryArray(Array):
     pass
 
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 589103635f8bd..f1032495e384e 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -98,6 +98,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         vector[shared_ptr[CBuffer]] buffers
         vector[shared_ptr[CArrayData]] child_data
 
+        @staticmethod
+        shared_ptr[CArrayData] Make(const shared_ptr[CDataType]& type,
+                                    int64_t length,
+                                    vector[shared_ptr[CBuffer]]& buffers,
+                                    int64_t null_count,
+                                    int64_t offset)
+
     cdef cppclass CArray" arrow::Array":
         shared_ptr[CDataType] type()
 
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 6faf4566b1983..d126db37375da 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -178,6 +178,26 @@ def test_array_eq_raises():
         arr1 == arr2
 
 
+def test_array_from_buffers():
+    values_buf = pa.py_buffer(np.int16([4, 5, 6, 7]))
+    nulls_buf = pa.py_buffer(np.uint8([0b00001101]))
+    arr = pa.Array.from_buffers(pa.int16(), 4, [nulls_buf, values_buf])
+    assert arr.type == pa.int16()
+    assert arr.to_pylist() == [4, None, 6, 7]
+
+    arr = pa.Array.from_buffers(pa.int16(), 4, [None, values_buf])
+    assert arr.type == pa.int16()
+    assert arr.to_pylist() == [4, 5, 6, 7]
+
+    arr = pa.Array.from_buffers(pa.int16(), 3, [nulls_buf, values_buf],
+                                offset=1)
+    assert arr.type == pa.int16()
+    assert arr.to_pylist() == [None, 6, 7]
+
+    with pytest.raises(TypeError):
+        pa.Array.from_buffers(pa.int16(), 3, [u'', u''], offset=1)
+
+
 def test_dictionary_from_numpy():
     indices = np.repeat([0, 1, 2], 2)
     dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)

From f50d8587d5945c66c1fd4d9f23f1deb7cc5dacc2 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 22 Mar 2018 19:36:53 +0100
Subject: [PATCH 1627/1644] ARROW-2343: [Java/Packaging] Run mvn clean in API
 doc builds

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1779 from cpcloud/ARROW-2343 and squashes the following commits:

4cd9a7e <Phillip Cloud> ARROW-2343:  Run mvn clean in API doc builds
---
 dev/gen_apidocs/create_documents.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh
index 3100d3b984b3a..6332ac9894e80 100755
--- a/dev/gen_apidocs/create_documents.sh
+++ b/dev/gen_apidocs/create_documents.sh
@@ -47,6 +47,7 @@ export PATH=$(pwd)/apache-maven-3.5.2/bin:$PATH
 
 pushd arrow/java
 rm -rf target/site/apidocs/*
+mvn -Drat.skip=true clean
 mvn -Drat.skip=true install
 mvn -Drat.skip=true site
 mkdir -p ../site/asf-site/docs/java/

From 29268ec5a0a10da639d0dd5684bc57fcd1c8f7e3 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Thu, 22 Mar 2018 16:58:44 -0400
Subject: [PATCH 1628/1644] ARROW-2342: [Python] Allow pickling more types

Several types (bool, timestamp with a non-None timezone, floats) were unpicklable.

Based on PR #1778.

Author: Antoine Pitrou <antoine@python.org>

Closes #1780 from pitrou/ARROW-2342-types-pickling and squashes the following commits:

4cda526d <Antoine Pitrou> ARROW-2342:  Allow pickling more types
---
 python/pyarrow/tests/test_types.py | 57 +++++++++++++++++++++---------
 python/pyarrow/types.pxi           | 13 +++++++
 2 files changed, 53 insertions(+), 17 deletions(-)

diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index 6459496f64060..b5170203ca579 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -15,12 +15,44 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import pickle
+
 import pytest
 
 import pyarrow as pa
 import pyarrow.types as types
 
 
+MANY_TYPES = [
+    pa.null(),
+    pa.bool_(),
+    pa.int32(),
+    pa.time32('s'),
+    pa.time64('us'),
+    pa.date32(),
+    pa.timestamp('us'),
+    pa.timestamp('us', tz='UTC'),
+    pa.timestamp('us', tz='Europe/Paris'),
+    pa.float16(),
+    pa.float32(),
+    pa.float64(),
+    pa.decimal128(19, 4),
+    pa.string(),
+    pa.binary(),
+    pa.binary(10),
+    pa.list_(pa.int32()),
+    pa.struct([pa.field('a', pa.int32()),
+               pa.field('b', pa.int8()),
+               pa.field('c', pa.string())]),
+    pa.union([pa.field('a', pa.binary(10)),
+              pa.field('b', pa.string())], mode=pa.lib.UnionMode_DENSE),
+    pa.union([pa.field('a', pa.binary(10)),
+              pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE),
+    # XXX Needs array pickling
+    # pa.dictionary(pa.int32(), pa.array(['a', 'b', 'c'])),
+]
+
+
 def test_is_boolean():
     assert types.is_boolean(pa.bool_())
     assert not types.is_boolean(pa.int8())
@@ -163,27 +195,18 @@ def check_fields(ty, fields):
 
 
 def test_types_hashable():
-    types = [
-        pa.null(),
-        pa.int32(),
-        pa.time32('s'),
-        pa.time64('us'),
-        pa.date32(),
-        pa.timestamp('us'),
-        pa.string(),
-        pa.binary(),
-        pa.binary(10),
-        pa.list_(pa.int32()),
-        pa.struct([pa.field('a', pa.int32()),
-                   pa.field('b', pa.int8()),
-                   pa.field('c', pa.string())])
-    ]
-
     in_dict = {}
-    for i, type_ in enumerate(types):
+    for i, type_ in enumerate(MANY_TYPES):
         assert hash(type_) == hash(type_)
         in_dict[type_] = i
         assert in_dict[type_] == i
+    assert len(in_dict) == len(MANY_TYPES)
+
+
+def test_types_picklable():
+    for ty in MANY_TYPES:
+        data = pickle.dumps(ty)
+        assert pickle.loads(data) == ty
 
 
 @pytest.mark.parametrize('t,check_func', [
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 12948500807ca..a4391c7f9587c 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -247,6 +247,13 @@ cdef class TimestampType(DataType):
             # Return DatetimeTZ
             return pdcompat.make_datetimetz(self.tz)
 
+    def __getstate__(self):
+        return self.unit, self.tz
+
+    def __setstate__(self, state):
+        cdef DataType reconstituted = timestamp(*state)
+        self.init(reconstituted.sp_type)
+
 
 cdef class Time32Type(DataType):
 
@@ -1192,6 +1199,7 @@ def union(children_fields, mode):
 
 cdef dict _type_aliases = {
     'null': null,
+    'bool': bool_,
     'i1': int8,
     'int8': int8,
     'i2': int16,
@@ -1208,9 +1216,14 @@ cdef dict _type_aliases = {
     'uint32': uint32,
     'u8': uint64,
     'uint64': uint64,
+    'f2': float16,
+    'halffloat': float16,
+    'float16': float16,
     'f4': float32,
+    'float': float32,
     'float32': float32,
     'f8': float64,
+    'double': float64,
     'float64': float64,
     'string': string,
     'str': string,

From 0c8d16405ebff643ede08b550605b41bc307701e Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Thu, 22 Mar 2018 20:14:21 -0400
Subject: [PATCH 1629/1644] ARROW-2345: [Documentation] Fix bundle exec and set
 sphinx nosidebar to True

Author: Phillip Cloud <cpcloud@gmail.com>

Closes #1781 from cpcloud/ARROW-2345 and squashes the following commits:

a0ce689f <Phillip Cloud> ARROW-2345:  Fix bundle exec and set sphinx nosidebar to True
---
 dev/run_site/run_site.sh  | 4 +++-
 python/doc/source/conf.py | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/dev/run_site/run_site.sh b/dev/run_site/run_site.sh
index 8527b2b16d222..088eabdda8a17 100755
--- a/dev/run_site/run_site.sh
+++ b/dev/run_site/run_site.sh
@@ -21,4 +21,6 @@ export BUNDLE_PATH=$(pwd)
 export HOME=$(pwd)
 bundle install --path .
 HOST_IP=$(grep $(hostname) /etc/hosts | cut -f 1)
-bundle exec jekyll serve -c _config.yml -s asf-site --host ${HOST_IP} --port 4000
+pushd asf-site
+bundle exec jekyll serve -c ../_config.yml -s . --host ${HOST_IP} --port 4000
+popd
diff --git a/python/doc/source/conf.py b/python/doc/source/conf.py
index 25e6d5e2d44cf..1e35ef9359019 100644
--- a/python/doc/source/conf.py
+++ b/python/doc/source/conf.py
@@ -165,7 +165,9 @@
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-# html_theme_options = {}
+html_theme_options = {
+    'nosidebar': True
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
 html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()

From e6d8eed29c7d120a1ecff9d59fd9abe64840bac7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 23 Mar 2018 10:05:50 +0100
Subject: [PATCH 1630/1644] ARROW-2322: [Java] Document dev environment
 requirements for publishing Java release artifacts

This is a one-time setup, but may be a stumbling block for new release managers (it was for me after a clean install)

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1782 from wesm/ARROW-2322 and squashes the following commits:

7992832 <Wes McKinney> Add links about Maven configuration for Java release process
---
 dev/release/README                | 4 +++-
 dev/release/RELEASE_MANAGEMENT.md | 5 ++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/dev/release/README b/dev/release/README
index 7868c9438e733..dceb484ef620e 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -2,7 +2,9 @@ requirements:
 - being a committer to be able to push to dist and maven repository
 - a gpg key to sign the artifacts
 - use java 7. check your JAVA_HOME environment variable (at least for now. See ARROW-930)
-- have the build requirements for cpp and c_glibg installed (see their README)
+- Maven configured to publish artifacts to Apache repositories (see
+  http://www.apache.org/dev/publishing-maven-artifacts.html)
+- have the build requirements for cpp and c_glib installed (see their README)
 
 to release, run the following (replace 0.1.0 with version to release):
 
diff --git a/dev/release/RELEASE_MANAGEMENT.md b/dev/release/RELEASE_MANAGEMENT.md
index 0f8c2202fef51..0069a2af89e3f 100644
--- a/dev/release/RELEASE_MANAGEMENT.md
+++ b/dev/release/RELEASE_MANAGEMENT.md
@@ -180,7 +180,10 @@ same time because they are interdependent.
 
 ### Updating Java Maven artifacts in Maven central
 
-See instructions at end of https://github.com/apache/arrow/blob/master/dev/release/README
+See instructions at end of
+https://github.com/apache/arrow/blob/master/dev/release/README. You must set up
+Maven to be able to publish to Apache's repositories. Read more at
+http://www.apache.org/dev/publishing-maven-artifacts.html.
 
 [1]: https://github.com/apache/arrow/blob/master/dev/release/README
 [2]: https://github.com/apache/arrow-dist

From a0ca9b452041e3917f69d1189747bb70cd1ef42e Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Fri, 23 Mar 2018 14:20:27 +0100
Subject: [PATCH 1631/1644] ARROW-2346: [Python] Fix PYARROW_CXX_FLAGS with
 multiple options

Quoting seems necessary only when going through the shell.

Author: Antoine Pitrou <antoine@python.org>

Closes #1783 from pitrou/ARROW-2346-multiple-pyarrow-cxxflags and squashes the following commits:

6599ab0 <Antoine Pitrou> ARROW-2346:  Fix PYARROW_CXX_FLAGS with multiple options
---
 python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index cf44b1e43a8dc..47246905385db 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -190,7 +190,7 @@ def _run_cmake(self):
                 cmake_options.append('-DPYARROW_BUILD_ORC=on')
 
             if len(self.cmake_cxxflags) > 0:
-                cmake_options.append('-DPYARROW_CXXFLAGS="{0}"'
+                cmake_options.append('-DPYARROW_CXXFLAGS={0}'
                                      .format(self.cmake_cxxflags))
 
             if self.bundle_arrow_cpp:

From 777f986129594dd8859d1215dbf89ec2495b6904 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Fri, 23 Mar 2018 17:03:29 +0100
Subject: [PATCH 1632/1644] ARROW-2331: [Python] Fix indexing for negative or
 out-of-bounds indices

Author: Antoine Pitrou <antoine@python.org>

Closes #1770 from pitrou/ARROW-2331-python-indexing and squashes the following commits:

aec1ef0 <Antoine Pitrou> Try to fix downcast errors
1a38451 <Antoine Pitrou> ARROW-2331:  Fix indexing for negative or out-of-bounds indices
---
 python/pyarrow/array.pxi             | 19 ++++++++++++++----
 python/pyarrow/lib.pxd               |  2 ++
 python/pyarrow/scalar.pxi            |  7 +++++--
 python/pyarrow/table.pxi             | 19 ++++++++----------
 python/pyarrow/tests/test_array.py   | 14 +++++++++++++
 python/pyarrow/tests/test_scalars.py |  6 ++++++
 python/pyarrow/tests/test_table.py   |  6 ++++++
 python/pyarrow/types.pxi             | 30 +++++++---------------------
 8 files changed, 63 insertions(+), 40 deletions(-)

diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index afb68a2fb2a90..c40d7b5548b95 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -232,6 +232,17 @@ def _normalize_slice(object arrow_obj, slice key):
         return arrow_obj.slice(start, stop - start)
 
 
+cdef Py_ssize_t _normalize_index(Py_ssize_t index,
+                                 Py_ssize_t length) except -1:
+    if index < 0:
+        index += length
+        if index < 0:
+            raise IndexError("index out of bounds")
+    elif index >= length:
+        raise IndexError("index out of bounds")
+    return index
+
+
 cdef class _FunctionContext:
     cdef:
         unique_ptr[CFunctionContext] ctx
@@ -427,6 +438,9 @@ cdef class Array:
         return self.ap.Equals(deref(other.ap))
 
     def __len__(self):
+        return self.length()
+
+    cdef int64_t length(self):
         if self.sp_array.get():
             return self.sp_array.get().length()
         else:
@@ -441,10 +455,7 @@ cdef class Array:
         if PySlice_Check(key):
             return _normalize_slice(self, key)
 
-        while key < 0:
-            key += len(self)
-
-        return self.getitem(key)
+        return self.getitem(_normalize_index(key, self.length()))
 
     cdef getitem(self, int64_t i):
         return box_scalar(self.type, self.sp_array, i)
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index be103b354a541..7a1b221db2679 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -137,6 +137,7 @@ cdef class ListValue(ArrayValue):
         CListArray* ap
 
     cdef getitem(self, int64_t i)
+    cdef int64_t length(self)
 
 
 cdef class UnionValue(ArrayValue):
@@ -164,6 +165,7 @@ cdef class Array:
 
     cdef void init(self, const shared_ptr[CArray]& sp_array)
     cdef getitem(self, int64_t i)
+    cdef int64_t length(self)
 
 
 cdef class Tensor:
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index a0f8480db99c5..e9c2c7c845b0e 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -294,10 +294,10 @@ cdef class BinaryValue(ArrayValue):
 cdef class ListValue(ArrayValue):
 
     def __len__(self):
-        return self.ap.value_length(self.index)
+        return self.length()
 
     def __getitem__(self, i):
-        return self.getitem(i)
+        return self.getitem(_normalize_index(i, self.length()))
 
     def __iter__(self):
         for i in range(len(self)):
@@ -313,6 +313,9 @@ cdef class ListValue(ArrayValue):
         cdef int64_t j = self.ap.value_offset(self.index) + i
         return box_scalar(self.value_type, self.ap.values(), j)
 
+    cdef int64_t length(self):
+        return self.ap.value_length(self.index)
+
     def as_py(self):
         cdef:
             int64_t j
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 6cfa9873bcf65..672b9fb7d6a6f 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -84,14 +84,12 @@ cdef class ChunkedArray:
         if isinstance(key, slice):
             return _normalize_slice(self, key)
         elif isinstance(key, six.integer_types):
-            item = key
-            if item >= self.chunked_array.length() or item < 0:
-                return IndexError("ChunkedArray selection out of bounds")
+            index = _normalize_index(key, self.chunked_array.length())
             for i in range(self.num_chunks):
-                if item < self.chunked_array.chunk(i).get().length():
-                    return self.chunk(i)[item]
+                if index < self.chunked_array.chunk(i).get().length():
+                    return self.chunk(i)[index]
                 else:
-                    item -= self.chunked_array.chunk(i).get().length()
+                    index -= self.chunked_array.chunk(i).get().length()
         else:
             raise TypeError("key must either be a slice or integer")
 
@@ -630,12 +628,10 @@ cdef class RecordBatch:
         return pyarrow_wrap_array(self.batch.column(i))
 
     def __getitem__(self, key):
-        cdef:
-            Py_ssize_t start, stop
         if isinstance(key, slice):
             return _normalize_slice(self, key)
         else:
-            return self.column(key)
+            return self.column(_normalize_index(key, self.num_columns))
 
     def serialize(self, memory_pool=None):
         """
@@ -1183,8 +1179,9 @@ cdef class Table:
         column.init(self.table.column(index))
         return column
 
-    def __getitem__(self, int64_t i):
-        return self.column(i)
+    def __getitem__(self, key):
+        cdef int index = <int> _normalize_index(key, self.num_columns)
+        return self.column(index)
 
     def itercolumns(self):
         """
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index d126db37375da..4a441fb97f1e7 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -110,6 +110,20 @@ def test_to_pandas_zero_copy():
         np_arr.sum()
 
 
+def test_array_getitem():
+    arr = pa.array(range(10, 15))
+    lst = arr.to_pylist()
+
+    for idx in range(-len(arr), len(arr)):
+        assert arr[idx].as_py() == lst[idx]
+    for idx in range(-2 * len(arr), -len(arr)):
+        with pytest.raises(IndexError):
+            arr[idx]
+    for idx in range(len(arr), 2 * len(arr)):
+        with pytest.raises(IndexError):
+            arr[idx]
+
+
 def test_array_slice():
     arr = pa.array(range(10))
 
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 7061a0d3a175c..c63be02030a33 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -131,6 +131,12 @@ def test_list(self):
         assert v.as_py() == ['foo', None]
         assert v[0].as_py() == 'foo'
         assert v[1] is pa.NA
+        assert v[-1] == v[1]
+        assert v[-2] == v[0]
+        with pytest.raises(IndexError):
+            v[-3]
+        with pytest.raises(IndexError):
+            v[2]
 
         assert arr[1] is pa.NA
 
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 356ecb7e067ce..81564352b1acb 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -31,6 +31,12 @@ def test_chunked_array_getitem():
     ]
     data = pa.chunked_array(data)
     assert data[1].as_py() == 2
+    assert data[-1].as_py() == 6
+    assert data[-6].as_py() == 1
+    with pytest.raises(IndexError):
+        data[6]
+    with pytest.raises(IndexError):
+        data[-7]
 
     data_slice = data[2:4]
     assert data_slice.to_pylist() == [3, 4]
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index a4391c7f9587c..b0557eb57b546 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -166,10 +166,8 @@ cdef class StructType(DataType):
         DataType.init(self, type)
 
     def __getitem__(self, i):
-        if i < 0 or i >= self.num_children:
-            raise IndexError(i)
-
-        return pyarrow_wrap_field(self.type.child(i))
+        cdef int index = <int> _normalize_index(i, self.num_children)
+        return pyarrow_wrap_field(self.type.child(index))
 
     property num_children:
 
@@ -207,7 +205,8 @@ cdef class UnionType(DataType):
             assert 0
 
     def __getitem__(self, i):
-        return pyarrow_wrap_field(self.type.child(i))
+        cdef int index = <int> _normalize_index(i, self.num_children)
+        return pyarrow_wrap_field(self.type.child(index))
 
     def __getstate__(self):
         children = [self[i] for i in range(self.num_children)]
@@ -440,24 +439,9 @@ cdef class Schema:
     def __len__(self):
         return self.schema.num_fields()
 
-    def __getitem__(self, int i):
-        cdef:
-            Field result = Field()
-            int num_fields = self.schema.num_fields()
-            int index
-
-        if not -num_fields <= i < num_fields:
-            raise IndexError(
-                'Schema field index {:d} is out of range'.format(i)
-            )
-
-        index = i if i >= 0 else num_fields + i
-        assert index >= 0
-
-        result.init(self.schema.field(index))
-        result.type = pyarrow_wrap_data_type(result.field.type())
-
-        return result
+    def __getitem__(self, key):
+        cdef int index = <int> _normalize_index(key, self.schema.num_fields())
+        return pyarrow_wrap_field(self.schema.field(index))
 
     def __iter__(self):
         for i in range(len(self)):

From 7b2c79765cf92760e1f8cca079159d9613b86412 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Sat, 24 Mar 2018 00:55:39 -0400
Subject: [PATCH 1633/1644] ARROW-2349: [Python] Opt in to bundling Boost
 shared libraries separately

In the scenario that the following are true, this prevents issues in the following case:

* Arrow libraries built with static Boost linking, and we pass `--with-static-boost` to setup.py
* Arrow libraries being bundled `--bundle-arrow-cpp`
* Boost libraries cannot be found when building pyarrow

Right now if `--bundle-arrow-cpp` is passed, then `-DPYARROW_BUNDLE_BOOST=ON` is passed to CMake.

Related to issues in https://github.com/apache/arrow-dist/pull/23

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1786 from wesm/ARROW-2349 and squashes the following commits:

4b5d13e <Wes McKinney> Opt in to bundling Boost separately
---
 python/CMakeLists.txt | 2 +-
 python/setup.py       | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 44a3c6c91a553..cb3cd7023d9b9 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -78,7 +78,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     OFF)
   option(PYARROW_BUNDLE_BOOST
     "Bundle the Boost libraries when we bundle Arrow C++"
-    ON)
+    OFF)
   set(PYARROW_CXXFLAGS "" CACHE STRING
     "Compiler flags to append when compiling Arrow")
 endif()
diff --git a/python/setup.py b/python/setup.py
index 47246905385db..91e0bb93de33b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -136,7 +136,7 @@ def initialize_options(self):
         # Default is True but this only is actually bundled when
         # we also bundle arrow-cpp.
         self.bundle_boost = strtobool(
-            os.environ.get('PYARROW_BUNDLE_BOOST', '1'))
+            os.environ.get('PYARROW_BUNDLE_BOOST', '0'))
 
     CYTHON_MODULE_NAMES = [
         'lib',
@@ -195,15 +195,15 @@ def _run_cmake(self):
 
             if self.bundle_arrow_cpp:
                 cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON')
-                cmake_options.append('-DPYARROW_BUNDLE_BOOST=ON')
                 # ARROW-1090: work around CMake rough edges
                 if 'ARROW_HOME' in os.environ and sys.platform != 'win32':
                     pkg_config = pjoin(os.environ['ARROW_HOME'], 'lib',
                                        'pkgconfig')
                     os.environ['PKG_CONFIG_PATH'] = pkg_config
                     del os.environ['ARROW_HOME']
-            else:
-                cmake_options.append('-DPYARROW_BUNDLE_BOOST=OFF')
+
+            if self.bundle_boost:
+                cmake_options.append('-DPYARROW_BUNDLE_BOOST=ON')
 
             cmake_options.append('-DCMAKE_BUILD_TYPE={0}'
                                  .format(self.build_type.lower()))

From fda4e6dd0b01b1898070cc151ca3189c83afe4b5 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Thu, 7 Dec 2017 13:57:30 -0800
Subject: [PATCH 1634/1644] Add Decimal Vector APIs to write Big Endian data

---
 .../src/main/java/org/apache/arrow/vector/DecimalVector.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index a043575081fb3..13f25774e218b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -18,6 +18,7 @@
 
 package org.apache.arrow.vector;
 
+import com.google.common.base.Preconditions;
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.impl.DecimalReaderImpl;

From a812ad65e67e32a7ea55f21ca476a3219f0d8f0d Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Mon, 15 Jan 2018 13:52:39 -0800
Subject: [PATCH 1635/1644] Add getReservation to Accountant

---
 .../main/java/org/apache/arrow/memory/Accountant.java    | 9 +++++++++
 .../java/org/apache/arrow/memory/BufferAllocator.java    | 7 +++++++
 2 files changed, 16 insertions(+)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
index 5bd6b9fe37956..ed2184aadab16 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -219,6 +219,15 @@ public long getLimit() {
     return allocationLimit.get();
   }
 
+  /**
+   * Return the initial reservation.
+   *
+   * @return reservation in bytes.
+   */
+  public long getInitReservation() {
+    return reservation;
+  }
+
   /**
    * Set the maximum amount of memory that can be allocated in the this Accountant before failing
    * an allocation.
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
index b23a6e4bd8507..a5da50e627874 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -91,6 +91,13 @@ public interface BufferAllocator extends AutoCloseable {
    */
   public long getLimit();
 
+  /**
+   * Return the initial reservation.
+   *
+   * @return reservation in bytes.
+   */
+  public long getInitReservation();
+
   /**
    * Set the maximum amount of memory this allocator is allowed to allocate.
    *

From a97b757712e5e436a0ba4a578f794deca3835f0d Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Tue, 23 Jan 2018 15:13:13 -0800
Subject: [PATCH 1636/1644] ARROW-2019: [JAVA] Control the memory allocated for
 inner vector in LIST

---
 .../java/org/apache/arrow/vector/complex/ListVector.java   | 7 +------
 .../test/java/org/apache/arrow/vector/TestValueVector.java | 1 -
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index d3eeaf2f0df7a..e82a5fd79fadf 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -31,12 +31,7 @@
 import org.apache.arrow.memory.BaseAllocator;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.BufferBacked;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.BitVectorHelper;
+import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 5104962e9a3ed..cac5c59560f4c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -1912,7 +1912,6 @@ public static void setBytes(int index, byte[] bytes, VarCharVector vector) {
   @Test /* VarCharVector */
   public void testSetInitialCapacity() {
     try (final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
-
       /* use the default 8 data bytes on average per element */
       vector.setInitialCapacity(4096);
       vector.allocateNew();

From 58683e425834e69463ad0cc4b675b4e311c998d9 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Fri, 2 Mar 2018 12:02:39 -0800
Subject: [PATCH 1637/1644] Fix setValueCount in splitAndTransfer of Variable
 Width Vector.

We need to use the split length as the value count of the target
vector. We are incorrectly using the value count of the current
vector for the target vector. Thus the latter ends up asking
for a realloc when it didn't really need extra memory.
---
 .../java/org/apache/arrow/vector/BaseVariableWidthVector.java | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index ecb3c780efca4..d56da1bdd2cda 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -711,9 +711,7 @@ public void splitAndTransferTo(int startIndex, int length,
     splitAndTransferValidityBuffer(startIndex, length, target);
     splitAndTransferOffsetBuffer(startIndex, length, target);
     target.setLastSet(length - 1);
-    if (this.valueCount > 0) {
-      target.setValueCount(this.valueCount);
-    }
+    target.setValueCount(length);
   }
 
   /*

From d6bad111cfddc6fbcb8009d49d586dd6e6476018 Mon Sep 17 00:00:00 2001
From: siddharth <siddharth@dremio.com>
Date: Wed, 14 Mar 2018 23:38:40 -0700
Subject: [PATCH 1638/1644] Do not allocate memory for inner vectors in
 nullable map when creating transfer pair

---
 .../main/java/org/apache/arrow/vector/complex/StructVector.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
index 05571bbb931c6..a296fbf1adb2b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
@@ -130,7 +130,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new NullableStructTransferPair(this, (StructVector) to, true);
+    return new NullableStructTransferPair(this, (StructVector) to, false);
   }
 
   @Override

From 073378a680b8e1fa8092d72b252818e89cebfd78 Mon Sep 17 00:00:00 2001
From: vkorukanti <venki@dremio.com>
Date: Thu, 29 Mar 2018 13:39:56 -0700
Subject: [PATCH 1639/1644] ARROW-2368: Correctly pad negative values in
 DecimalVector#setBigEndian

---
 .../apache/arrow/vector/DecimalVector.java    | 26 +++++++--
 .../arrow/vector/TestDecimalVector.java       | 54 +++++++++++++++++++
 2 files changed, 76 insertions(+), 4 deletions(-)

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index 13f25774e218b..db1ad0b31b3a8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -212,7 +212,6 @@ public void set(int index, ArrowBuf buffer) {
    * @param value array of bytes containing decimal in big endian byte order.
    */
   public void setBigEndian(int index, byte[] value) {
-    assert value.length <= TYPE_WIDTH;
     BitVectorHelper.setValidityBitToOne(validityBuffer, index);
     final int length = value.length;
     int startIndex = index * TYPE_WIDTH;
@@ -224,13 +223,32 @@ public void setBigEndian(int index, byte[] value) {
         valueBuffer.setByte(startIndex + 3, value[i-3]);
         startIndex += 4;
       }
-    } else {
+
+      return;
+    }
+
+    if (length == 0) {
+      valueBuffer.setZero(startIndex, TYPE_WIDTH);
+      return;
+    }
+
+    if (length < 16) {
       for (int i = length - 1; i >= 0; i--) {
         valueBuffer.setByte(startIndex, value[i]);
         startIndex++;
       }
-      valueBuffer.setZero(startIndex, TYPE_WIDTH - length);
+
+      final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00);
+      final int maxStartIndex = (index + 1) * TYPE_WIDTH;
+      while (startIndex < maxStartIndex) {
+        valueBuffer.setByte(startIndex, pad);
+        startIndex++;
+      }
+
+      return;
     }
+
+    throw new IllegalArgumentException("Invalid decimal value length. Valid length in [1 - 16], got " + length);
   }
 
   /**
@@ -468,4 +486,4 @@ public void copyValueSafe(int fromIndex, int toIndex) {
       to.copyFromSafe(fromIndex, toIndex, DecimalVector.this);
     }
   }
-}
\ No newline at end of file
+}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index 8c86452fcc3bf..15c56ae2bc382 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -20,6 +20,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -190,4 +191,57 @@ public void testBigDecimalReadWrite() {
       assertEquals(decimal8, decimalVector.getObject(7));
     }
   }
+
+  /**
+   * Test {@link DecimalVector#setBigEndian(int, byte[])} which takes BE layout input and stores in LE layout.
+   * Cases to cover: value given in byte array in different lengths in range [1-16] and negative values.
+   */
+  @Test
+  public void decimalBE2LE() {
+    try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(21, 2), allocator);) {
+      decimalVector.allocateNew();
+
+      BigInteger[] testBigInts = new BigInteger[] {
+          new BigInteger("0"),
+          new BigInteger("-1"),
+          new BigInteger("23"),
+          new BigInteger("234234"),
+          new BigInteger("-234234234"),
+          new BigInteger("234234234234"),
+          new BigInteger("-56345345345345"),
+          new BigInteger("29823462983462893462934679234653456345"), // converts to 16 byte array
+          new BigInteger("-3894572983475982374598324598234346536"), // converts to 16 byte array
+          new BigInteger("-345345"),
+          new BigInteger("754533")
+      };
+
+      int insertionIdx = 0;
+      insertionIdx++; // insert a null
+      for (BigInteger val : testBigInts) {
+        decimalVector.setBigEndian(insertionIdx++, val.toByteArray());
+      }
+      insertionIdx++; // insert a null
+      // insert a zero length buffer
+      decimalVector.setBigEndian(insertionIdx++, new byte[0]);
+
+      // Try inserting a buffer larger than 16bytes and expect a failure
+      try {
+        decimalVector.setBigEndian(insertionIdx, new byte[17]);
+        fail("above statement should have failed");
+      } catch (IllegalArgumentException ex) {
+        assertTrue(ex.getMessage().equals("Invalid decimal value length. Valid length in [1 - 16], got 17"));
+      }
+      decimalVector.setValueCount(insertionIdx);
+
+      // retrieve values and check if they are correct
+      int outputIdx = 0;
+      assertTrue(decimalVector.isNull(outputIdx++));
+      for (BigInteger expected : testBigInts) {
+        final BigDecimal actual = decimalVector.getObject(outputIdx++);
+        assertEquals(expected, actual.unscaledValue());
+      }
+      assertTrue(decimalVector.isNull(outputIdx++));
+      assertEquals(BigInteger.valueOf(0), decimalVector.getObject(outputIdx).unscaledValue());
+    }
+  }
 }

From d9a52c359866e6d97f522fc88bad4f1ad7eb0e5e Mon Sep 17 00:00:00 2001
From: vkorukanti <venki@dremio.com>
Date: Wed, 25 Apr 2018 11:13:04 -0700
Subject: [PATCH 1640/1644] ARROW-2511: [Java] Fix
 BaseVariableWidthVector.allocateNew to not swallow exception

+ Also remove the e.printStackTrace() calls
---
 .../arrow/vector/BaseFixedWidthVector.java    |  4 +-
 .../arrow/vector/BaseVariableWidthVector.java |  3 +-
 .../vector/TestOutOfMemoryForValueVector.java | 73 +++++++++++++++++++
 3 files changed, 75 insertions(+), 5 deletions(-)
 create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestOutOfMemoryForValueVector.java

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index 4b47df8a450a0..b275ab22db89d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -281,7 +281,6 @@ public boolean allocateNewSafe() {
     try {
       allocateBytes(curAllocationSizeValue, curAllocationSizeValidity);
     } catch (Exception e) {
-      e.printStackTrace();
       clear();
       return false;
     }
@@ -314,7 +313,6 @@ public void allocateNew(int valueCount) {
     try {
       allocateBytes(valueBufferSize, validityBufferSize);
     } catch (Exception e) {
-      e.printStackTrace();
       clear();
       throw e;
     }
@@ -841,4 +839,4 @@ protected void handleSafe(int index) {
       reAlloc();
     }
   }
-}
\ No newline at end of file
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index d56da1bdd2cda..44df96b82827d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -394,7 +394,6 @@ public boolean allocateNewSafe() {
     try {
       allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset);
     } catch (Exception e) {
-      e.printStackTrace();
       clear();
       return false;
     }
@@ -427,8 +426,8 @@ public void allocateNew(int totalBytes, int valueCount) {
     try {
       allocateBytes(totalBytes, validityBufferSize, offsetBufferSize);
     } catch (Exception e) {
-      e.printStackTrace();
       clear();
+      throw e;
     }
   }
 
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOutOfMemoryForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOutOfMemoryForValueVector.java
new file mode 100644
index 0000000000000..84c0fc33d9d4d
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOutOfMemoryForValueVector.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.memory.RootAllocator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * This class tests cases where we expect to receive {@link OutOfMemoryException}.
+ */
+public class TestOutOfMemoryForValueVector {
+
+  private final static String EMPTY_SCHEMA_PATH = "";
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(200); // Start with low memory limit
+  }
+
+  @Test(expected = OutOfMemoryException.class)
+  public void variableWidthVectorAllocateNew() {
+    try (VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
+      vector.allocateNew();
+    }
+  }
+
+  @Test(expected = OutOfMemoryException.class)
+  public void variableWidthVectorAllocateNewCustom() {
+    try (VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
+      vector.allocateNew(2342, 234);
+    }
+  }
+
+  @Test(expected = OutOfMemoryException.class)
+  public void fixedWidthVectorAllocateNew() {
+    try (IntVector vector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) {
+      vector.allocateNew();
+    }
+  }
+
+  @Test(expected = OutOfMemoryException.class)
+  public void fixedWidthVectorAllocateNewCustom() {
+    try (IntVector vector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) {
+      vector.allocateNew(2342);
+    }
+  }
+
+  @After
+  public void terminate() {
+    allocator.close();
+  }
+}

From 2033ac148a1028db6eaffbecea3feb518744a094 Mon Sep 17 00:00:00 2001
From: Vanco Buca <vanco@dremio.com>
Date: Tue, 12 Jun 2018 13:40:11 -0700
Subject: [PATCH 1641/1644] ARROW-2696: [JAVA] enhance AllocationListener with
 an onFailedAllocation() call

---
 .../arrow/memory/AllocationListener.java      |  20 ++-
 .../apache/arrow/memory/BaseAllocator.java    |  54 ++++----
 .../apache/arrow/memory/BufferAllocator.java  |  11 ++
 .../apache/arrow/memory/ChildAllocator.java   |   4 +-
 .../apache/arrow/memory/RootAllocator.java    |   2 +-
 .../arrow/memory/TestBaseAllocator.java       | 117 +++++++++++++++++-
 6 files changed, 180 insertions(+), 28 deletions(-)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
index d36cb37fc2e24..de181df247c83 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
@@ -21,8 +21,8 @@
 /**
  * An allocation listener being notified for allocation/deallocation
  * <p>
- * It is expected to be called from multiple threads and as such,
- * provider should take care of making the implementation thread-safe
+ * It might be called from multiple threads if the allocator hierarchy shares a listener, in which
+ * case, the provider should take care of making the implementation thread-safe.
  */
 public interface AllocationListener {
 
@@ -30,6 +30,11 @@ public interface AllocationListener {
     @Override
     public void onAllocation(long size) {
     }
+
+    @Override
+    public boolean onFailedAllocation(long size, Accountant.AllocationOutcome outcome) {
+      return false;
+    }
   };
 
   /**
@@ -39,4 +44,15 @@ public void onAllocation(long size) {
    */
   void onAllocation(long size);
 
+  /**
+   * Called whenever an allocation failed, giving the caller a chance to create some space in the allocator
+   * (either by freeing some resource, or by changing the limit), and, if successful, allowing the allocator
+   * to retry the allocation.
+   *
+   * @param size     the buffer size that was being allocated
+   * @param outcome  the outcome of the failed allocation. Carries information of what failed
+   * @return true, if the allocation can be retried; false if the allocation should fail
+   */
+  boolean onFailedAllocation(long size, Accountant.AllocationOutcome outcome);
+
 }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 2f70f737214dd..5ae79b488d4ef 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -56,28 +56,21 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato
   private final HistoricalLog historicalLog;
   private volatile boolean isClosed = false; // the allocator has been closed
 
+  /**
+   * Initialize an allocator
+   * @param parentAllocator   parent allocator. null if defining a root allocator
+   * @param listener          listener callback. Must be non-null -- use {@link AllocationListener#NOOP} if no listener
+   *                          desired
+   * @param name              name of this allocator
+   * @param initReservation   initial reservation. Cannot be modified after construction
+   * @param maxAllocation     limit. Allocations past the limit fail. Can be modified after construction
+   */
   protected BaseAllocator(
-      final AllocationListener listener,
-      final String name,
-      final long initReservation,
-      final long maxAllocation) throws OutOfMemoryException {
-    this(listener, null, name, initReservation, maxAllocation);
-  }
-
-  protected BaseAllocator(
-      final BaseAllocator parentAllocator,
-      final String name,
-      final long initReservation,
-      final long maxAllocation) throws OutOfMemoryException {
-    this(parentAllocator.listener, parentAllocator, name, initReservation, maxAllocation);
-  }
-
-  private BaseAllocator(
-      final AllocationListener listener,
-      final BaseAllocator parentAllocator,
-      final String name,
-      final long initReservation,
-      final long maxAllocation) throws OutOfMemoryException {
+          final BaseAllocator parentAllocator,
+          final AllocationListener listener,
+          final String name,
+          final long initReservation,
+          final long maxAllocation) throws OutOfMemoryException {
     super(parentAllocator, initReservation, maxAllocation);
 
     this.listener = listener;
@@ -276,7 +269,13 @@ public ArrowBuf buffer(final int initialRequestSize, BufferManager manager) {
         : initialRequestSize;
     AllocationOutcome outcome = this.allocateBytes(actualRequestSize);
     if (!outcome.isOk()) {
-      throw new OutOfMemoryException(createErrorMsg(this, actualRequestSize, initialRequestSize));
+      if (listener.onFailedAllocation(actualRequestSize, outcome)) {
+        // Second try, in case the listener can do something about it
+        outcome = this.allocateBytes(actualRequestSize);
+      }
+      if (!outcome.isOk()) {
+        throw new OutOfMemoryException(createErrorMsg(this, actualRequestSize, initialRequestSize));
+      }
     }
 
     boolean success = false;
@@ -333,9 +332,18 @@ public BufferAllocator newChildAllocator(
       final String name,
       final long initReservation,
       final long maxAllocation) {
+    return newChildAllocator(name, this.listener, initReservation, maxAllocation);
+  }
+
+  @Override
+  public BufferAllocator newChildAllocator(
+      final String name,
+      final AllocationListener listener,
+      final long initReservation,
+      final long maxAllocation) {
     assertOpen();
 
-    final ChildAllocator childAllocator = new ChildAllocator(this, name, initReservation,
+    final ChildAllocator childAllocator = new ChildAllocator(listener, this, name, initReservation,
         maxAllocation);
 
     if (DEBUG) {
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
index a5da50e627874..11ca38e089d18 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java
@@ -68,6 +68,17 @@ public interface BufferAllocator extends AutoCloseable {
    */
   public BufferAllocator newChildAllocator(String name, long initReservation, long maxAllocation);
 
+  /**
+   * Create a new child allocator.
+   *
+   * @param name            the name of the allocator.
+   * @param listener        allocation listener for the newly created child
+   * @param initReservation the initial space reservation (obtained from this allocator)
+   * @param maxAllocation   maximum amount of space the new allocator can allocate
+   * @return the new allocator, or null if it can't be created
+   */
+  public BufferAllocator newChildAllocator(String name, AllocationListener listener, long initReservation, long maxAllocation);
+
   /**
    * Close and release all buffers generated from this buffer pool.
    *
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
index f9a6dc72ece8c..03ec268d35cb1 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java
@@ -32,6 +32,7 @@ class ChildAllocator extends BaseAllocator {
   /**
    * Constructor.
    *
+   * @param listener        Allocation listener to be used in this child
    * @param parentAllocator parent allocator -- the one creating this child
    * @param name            the name of this child allocator
    * @param initReservation initial amount of space to reserve (obtained from the parent)
@@ -41,11 +42,12 @@ class ChildAllocator extends BaseAllocator {
    *                        allocation policy in force, even less memory may be available
    */
   ChildAllocator(
+      AllocationListener listener,
       BaseAllocator parentAllocator,
       String name,
       long initReservation,
       long maxAllocation) {
-    super(parentAllocator, name, initReservation, maxAllocation);
+    super(parentAllocator, listener, name, initReservation, maxAllocation);
   }
 
 
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
index 1dc6bf0c92fa0..161b81a58b5be 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
@@ -31,7 +31,7 @@ public RootAllocator(final long limit) {
   }
 
   public RootAllocator(final AllocationListener listener, final long limit) {
-    super(listener, "ROOT", 0, limit);
+    super(null, listener, "ROOT", 0, limit);
   }
 
   /**
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index 76f2c501cf4c7..62b0046b2ae9f 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -222,7 +222,122 @@ public void testRootAllocator_createChildDontClose() throws Exception {
     }
   }
 
-  private static void allocateAndFree(final BufferAllocator allocator) {
+  // Allocation listener
+  // It counts the number of times it has been invoked, and how much memory allocation it has seen
+  // When set to 'expand on fail', it attempts to expand the associated allocator's limit
+  private static final class TestAllocationListener implements AllocationListener {
+    private int numCalls;
+    private long totalMem;
+    private boolean expandOnFail;
+    BufferAllocator expandAlloc;
+    long expandLimit;
+
+    TestAllocationListener() {
+      this.numCalls = 0;
+      this.totalMem = 0;
+      this.expandOnFail = false;
+      this.expandAlloc = null;
+      this.expandLimit = 0;
+    }
+
+    @Override
+    public void onAllocation(long size) {
+      numCalls++;
+      totalMem += size;
+    }
+
+    @Override
+    public boolean onFailedAllocation(long size,  Accountant.AllocationOutcome outcome) {
+      if (expandOnFail) {
+        expandAlloc.setLimit(expandLimit);
+        return true;
+      }
+      return false;
+    }
+
+    void setExpandOnFail(BufferAllocator expandAlloc, long expandLimit) {
+      this.expandOnFail = true;
+      this.expandAlloc = expandAlloc;
+      this.expandLimit = expandLimit;
+    }
+
+    int getNumCalls() {
+      return numCalls;
+    }
+
+    long getTotalMem() {
+      return totalMem;
+    }
+  }
+
+  @Test
+  public void testRootAllocator_listeners() throws Exception {
+    TestAllocationListener l1 = new TestAllocationListener();
+    assertEquals(0, l1.getNumCalls());
+    assertEquals(0, l1.getTotalMem());
+    TestAllocationListener l2 = new TestAllocationListener();
+    assertEquals(0, l2.getNumCalls());
+    assertEquals(0, l2.getTotalMem());
+    // root and first-level child share the first listener
+    // second-level and third-level child share the second listener
+    try (final RootAllocator rootAllocator = new RootAllocator(l1, MAX_ALLOCATION)) {
+      try (final BufferAllocator c1 = rootAllocator.newChildAllocator("c1", 0, MAX_ALLOCATION)) {
+        final ArrowBuf buf1 = c1.buffer(16);
+        assertNotNull("allocation failed", buf1);
+        assertEquals(1, l1.getNumCalls());
+        assertEquals(16, l1.getTotalMem());
+        buf1.release();
+        try (final BufferAllocator c2 = c1.newChildAllocator("c2", l2, 0, MAX_ALLOCATION)) {
+          final ArrowBuf buf2 = c2.buffer(32);
+          assertNotNull("allocation failed", buf2);
+          assertEquals(1, l1.getNumCalls());
+          assertEquals(16, l1.getTotalMem());
+          assertEquals(1, l2.getNumCalls());
+          assertEquals(32, l2.getTotalMem());
+          buf2.release();
+          try (final BufferAllocator c3 = c2.newChildAllocator("c3", 0, MAX_ALLOCATION)) {
+            final ArrowBuf buf3 = c3.buffer(64);
+            assertNotNull("allocation failed", buf3);
+            assertEquals(1, l1.getNumCalls());
+            assertEquals(16, l1.getTotalMem());
+            assertEquals(2, l2.getNumCalls());
+            assertEquals(32 + 64, l2.getTotalMem());
+            buf3.release();
+          }
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testRootAllocator_listenerAllocationFail() throws Exception {
+    TestAllocationListener l1 = new TestAllocationListener();
+    assertEquals(0, l1.getNumCalls());
+    assertEquals(0, l1.getTotalMem());
+    // Test attempts to allocate too much from a child whose limit is set to half of the max allocation
+    // The listener's callback triggers, expanding the child allocator's limit, so then the allocation succeeds
+    try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) {
+      try (final BufferAllocator c1 = rootAllocator.newChildAllocator("c1", l1,0, MAX_ALLOCATION / 2)) {
+        try {
+          c1.buffer(MAX_ALLOCATION);
+          fail("allocated memory beyond max allowed");
+        } catch (OutOfMemoryException e) {
+          // expected
+        }
+        assertEquals(0, l1.getNumCalls());
+        assertEquals(0, l1.getTotalMem());
+
+        l1.setExpandOnFail(c1, MAX_ALLOCATION);
+        ArrowBuf arrowBuf = c1.buffer(MAX_ALLOCATION);
+        assertNotNull("allocation failed", arrowBuf);
+        assertEquals(1, l1.getNumCalls());
+        assertEquals(MAX_ALLOCATION, l1.getTotalMem());
+        arrowBuf.release();
+      }
+    }
+  }
+
+    private static void allocateAndFree(final BufferAllocator allocator) {
     final ArrowBuf arrowBuf = allocator.buffer(512);
     assertNotNull("allocation failed", arrowBuf);
     arrowBuf.release();

From 1f84eefdd815f93f2fa4440e4484ea30a57c1a01 Mon Sep 17 00:00:00 2001
From: Vanco Buca <vanco@dremio.com>
Date: Wed, 20 Jun 2018 17:46:07 -0700
Subject: [PATCH 1642/1644] ARROW-2725: [Java] make
 Accountant.AllocationOutcome publicly visible

---
 .../org/apache/arrow/memory/Accountant.java   | 37 +------------
 .../arrow/memory/AllocationListener.java      |  4 +-
 .../arrow/memory/AllocationOutcome.java       | 55 +++++++++++++++++++
 .../apache/arrow/memory/TestAccountant.java   |  1 -
 .../arrow/memory/TestBaseAllocator.java       |  2 +-
 5 files changed, 59 insertions(+), 40 deletions(-)
 create mode 100644 java/memory/src/main/java/org/apache/arrow/memory/AllocationOutcome.java

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
index ed2184aadab16..e3ef2cca8144b 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -170,7 +170,7 @@ private AllocationOutcome allocate(final long size, final boolean incomingUpdate
     }
 
     final AllocationOutcome finalOutcome = beyondLimit ? AllocationOutcome.FAILED_LOCAL :
-        parentOutcome.ok ? AllocationOutcome.SUCCESS : AllocationOutcome.FAILED_PARENT;
+        parentOutcome.isOk() ? AllocationOutcome.SUCCESS : AllocationOutcome.FAILED_PARENT;
 
     if (updatePeak) {
       updatePeak();
@@ -267,39 +267,4 @@ public long getHeadroom() {
     return Math.min(localHeadroom, parent.getHeadroom());
   }
 
-  /**
-   * Describes the type of outcome that occurred when trying to account for allocation of memory.
-   */
-  public static enum AllocationOutcome {
-
-    /**
-     * Allocation succeeded.
-     */
-    SUCCESS(true),
-
-    /**
-     * Allocation succeeded but only because the allocator was forced to move beyond a limit.
-     */
-    FORCED_SUCCESS(true),
-
-    /**
-     * Allocation failed because the local allocator's limits were exceeded.
-     */
-    FAILED_LOCAL(false),
-
-    /**
-     * Allocation failed because a parent allocator's limits were exceeded.
-     */
-    FAILED_PARENT(false);
-
-    private final boolean ok;
-
-    AllocationOutcome(boolean ok) {
-      this.ok = ok;
-    }
-
-    public boolean isOk() {
-      return ok;
-    }
-  }
 }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
index de181df247c83..67a7b9c63c9e2 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
@@ -32,7 +32,7 @@ public void onAllocation(long size) {
     }
 
     @Override
-    public boolean onFailedAllocation(long size, Accountant.AllocationOutcome outcome) {
+    public boolean onFailedAllocation(long size, AllocationOutcome outcome) {
       return false;
     }
   };
@@ -53,6 +53,6 @@ public boolean onFailedAllocation(long size, Accountant.AllocationOutcome outcom
    * @param outcome  the outcome of the failed allocation. Carries information of what failed
    * @return true, if the allocation can be retried; false if the allocation should fail
    */
-  boolean onFailedAllocation(long size, Accountant.AllocationOutcome outcome);
+  boolean onFailedAllocation(long size, AllocationOutcome outcome);
 
 }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationOutcome.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationOutcome.java
new file mode 100644
index 0000000000000..9f2daa7aa4d08
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationOutcome.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.memory;
+
+/**
+ * Describes the type of outcome that occurred when trying to account for allocation of memory.
+ */
+public enum AllocationOutcome {
+
+  /**
+   * Allocation succeeded.
+   */
+  SUCCESS(true),
+
+  /**
+   * Allocation succeeded but only because the allocator was forced to move beyond a limit.
+   */
+  FORCED_SUCCESS(true),
+
+  /**
+   * Allocation failed because the local allocator's limits were exceeded.
+   */
+  FAILED_LOCAL(false),
+
+  /**
+   * Allocation failed because a parent allocator's limits were exceeded.
+   */
+  FAILED_PARENT(false);
+
+  private final boolean ok;
+
+  AllocationOutcome(boolean ok) {
+    this.ok = ok;
+  }
+
+  public boolean isOk() {
+    return ok;
+  }
+}
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java b/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
index 100be069fe6d4..a45de487625e8 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java
@@ -20,7 +20,6 @@
 
 import static org.junit.Assert.assertEquals;
 
-import org.apache.arrow.memory.Accountant.AllocationOutcome;
 import org.junit.Assert;
 import org.junit.Test;
 
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index 62b0046b2ae9f..b826d168a6421 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -247,7 +247,7 @@ public void onAllocation(long size) {
     }
 
     @Override
-    public boolean onFailedAllocation(long size,  Accountant.AllocationOutcome outcome) {
+    public boolean onFailedAllocation(long size,  AllocationOutcome outcome) {
       if (expandOnFail) {
         expandAlloc.setLimit(expandLimit);
         return true;

From 96f8a832025066d6559b551336397002329607a2 Mon Sep 17 00:00:00 2001
From: Vanco Buca <vanco@dremio.com>
Date: Wed, 3 Oct 2018 16:13:24 -0700
Subject: [PATCH 1643/1644] ARROW-2165: [JAVA] enhance AllocatorListener to
 listen for child allocator addition and removal

* AllocationListener.onChildAdded() and onChildRemoved() methods added, with default no-op implementation
* Calls to onChildAdded() and onChildRemoved() implemented
* Both parent and child allocator passed to the listener
---
 .../arrow/memory/AllocationListener.java      | 21 ++++++++++++++
 .../apache/arrow/memory/BaseAllocator.java    |  2 ++
 .../arrow/memory/TestBaseAllocator.java       | 28 +++++++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
index 67a7b9c63c9e2..d6a97efb40839 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java
@@ -35,6 +35,14 @@ public void onAllocation(long size) {
     public boolean onFailedAllocation(long size, AllocationOutcome outcome) {
       return false;
     }
+
+    @Override
+    public void onChildAdded(BufferAllocator parentAllocator, BufferAllocator childAllocator) {
+    }
+
+    @Override
+    public void onChildRemoved(BufferAllocator parentAllocator, BufferAllocator childAllocator) {
+    }
   };
 
   /**
@@ -55,4 +63,17 @@ public boolean onFailedAllocation(long size, AllocationOutcome outcome) {
    */
   boolean onFailedAllocation(long size, AllocationOutcome outcome);
 
+  /**
+   * Called immediately after a child allocator was added to the parent allocator
+   * @param parentAllocator The parent allocator to which a child was added
+   * @param childAllocator  The child allocator that was just added
+   */
+  void onChildAdded(BufferAllocator parentAllocator, BufferAllocator childAllocator);
+
+  /**
+   * Called immediately after a child allocator was removed from the parent allocator
+   * @param parentAllocator The parent allocator from which a child was removed
+   * @param childAllocator The child allocator that was just removed
+   */
+  void onChildRemoved(BufferAllocator parentAllocator, BufferAllocator childAllocator);
 }
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 5ae79b488d4ef..e7cb80a608857 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -239,6 +239,7 @@ private void childClosed(final BaseAllocator childAllocator) {
         }
       }
     }
+    listener.onChildRemoved(this, childAllocator);
   }
 
   @Override
@@ -353,6 +354,7 @@ public BufferAllocator newChildAllocator(
             childAllocator.name);
       }
     }
+    this.listener.onChildAdded(this, childAllocator);
 
     return childAllocator;
   }
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index b826d168a6421..4ac35d132367b 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -227,6 +227,7 @@ public void testRootAllocator_createChildDontClose() throws Exception {
   // When set to 'expand on fail', it attempts to expand the associated allocator's limit
   private static final class TestAllocationListener implements AllocationListener {
     private int numCalls;
+    private int numChildren;
     private long totalMem;
     private boolean expandOnFail;
     BufferAllocator expandAlloc;
@@ -234,6 +235,7 @@ private static final class TestAllocationListener implements AllocationListener
 
     TestAllocationListener() {
       this.numCalls = 0;
+      this.numChildren = 0;
       this.totalMem = 0;
       this.expandOnFail = false;
       this.expandAlloc = null;
@@ -255,6 +257,16 @@ public boolean onFailedAllocation(long size,  AllocationOutcome outcome) {
       return false;
     }
 
+    @Override
+    public void onChildAdded(BufferAllocator parentAllocator, BufferAllocator childAllocator) {
+      ++numChildren;
+    }
+
+    @Override
+    public void onChildRemoved(BufferAllocator parentAllocator, BufferAllocator childAllocator) {
+      --numChildren;
+    }
+
     void setExpandOnFail(BufferAllocator expandAlloc, long expandLimit) {
       this.expandOnFail = true;
       this.expandAlloc = expandAlloc;
@@ -265,6 +277,10 @@ int getNumCalls() {
       return numCalls;
     }
 
+    int getNumChildren() {
+      return numChildren;
+    }
+
     long getTotalMem() {
       return totalMem;
     }
@@ -274,20 +290,25 @@ long getTotalMem() {
   public void testRootAllocator_listeners() throws Exception {
     TestAllocationListener l1 = new TestAllocationListener();
     assertEquals(0, l1.getNumCalls());
+    assertEquals(0, l1.getNumChildren());
     assertEquals(0, l1.getTotalMem());
     TestAllocationListener l2 = new TestAllocationListener();
     assertEquals(0, l2.getNumCalls());
+    assertEquals(0, l2.getNumChildren());
     assertEquals(0, l2.getTotalMem());
     // root and first-level child share the first listener
     // second-level and third-level child share the second listener
     try (final RootAllocator rootAllocator = new RootAllocator(l1, MAX_ALLOCATION)) {
       try (final BufferAllocator c1 = rootAllocator.newChildAllocator("c1", 0, MAX_ALLOCATION)) {
+        assertEquals(1, l1.getNumChildren());
         final ArrowBuf buf1 = c1.buffer(16);
         assertNotNull("allocation failed", buf1);
         assertEquals(1, l1.getNumCalls());
         assertEquals(16, l1.getTotalMem());
         buf1.release();
         try (final BufferAllocator c2 = c1.newChildAllocator("c2", l2, 0, MAX_ALLOCATION)) {
+          assertEquals(2, l1.getNumChildren());  // c1 got a new child, so c1's listener (l1) is notified
+          assertEquals(0, l2.getNumChildren());
           final ArrowBuf buf2 = c2.buffer(32);
           assertNotNull("allocation failed", buf2);
           assertEquals(1, l1.getNumCalls());
@@ -296,6 +317,8 @@ public void testRootAllocator_listeners() throws Exception {
           assertEquals(32, l2.getTotalMem());
           buf2.release();
           try (final BufferAllocator c3 = c2.newChildAllocator("c3", 0, MAX_ALLOCATION)) {
+            assertEquals(2, l1.getNumChildren());
+            assertEquals(1, l2.getNumChildren());
             final ArrowBuf buf3 = c3.buffer(64);
             assertNotNull("allocation failed", buf3);
             assertEquals(1, l1.getNumCalls());
@@ -304,8 +327,13 @@ public void testRootAllocator_listeners() throws Exception {
             assertEquals(32 + 64, l2.getTotalMem());
             buf3.release();
           }
+          assertEquals(2, l1.getNumChildren());
+          assertEquals(0, l2.getNumChildren()); // third-level child removed
         }
+        assertEquals(1, l1.getNumChildren()); // second-level child removed
+        assertEquals(0, l2.getNumChildren());
       }
+      assertEquals(0, l1.getNumChildren()); // first-level child removed
     }
   }
 

From d55177b7d840dc661516dc8561e9e51d5ccca2f0 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Thu, 17 May 2018 11:20:45 -0700
Subject: [PATCH 1644/1644] ARROW-2594: [Java] When realloc Vectors, zero out
 all unfilled bytes of new buffer

Currently when reallocating vectors, only the second half of the new buffer will be zeroed out assuming that it is doubled from the previous buffer and the first half is already populated or cleaned.  This isn't the case if the vector had been cleared and the buffer is empty causing incorrect values in the new buffer if it was recycled from an old one.

Added a new test with a ListVector that should reuse a previous buffer after being cleared.

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #2054 from BryanCutler/java-vector-realloc-clear-buffer-ARROW-2594 and squashes the following commits:

28b8095 <Bryan Cutler> added a comment about clear
be3ee8f <Bryan Cutler> remove extra spaces
5a39790 <Bryan Cutler> zero out any newly allocated buffer bytes
---
 .../main/codegen/templates/UnionVector.java   |  3 +-
 .../arrow/vector/BaseFixedWidthVector.java    |  3 +-
 .../arrow/vector/BaseVariableWidthVector.java |  3 +-
 .../complex/BaseRepeatedValueVector.java      |  3 +-
 .../vector/complex/FixedSizeListVector.java   |  2 +-
 .../arrow/vector/complex/ListVector.java      |  3 +-
 .../arrow/vector/complex/StructVector.java    |  2 +-
 .../apache/arrow/vector/TestListVector.java   | 45 +++++++++++++++++++
 8 files changed, 52 insertions(+), 12 deletions(-)

diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 1cfa0666a3139..8b27f39bfb215 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -290,8 +290,7 @@ private void reallocTypeBuffer() {
 
     final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
     newBuf.setBytes(0, typeBuffer, 0, currentBufferCapacity);
-    final int halfNewCapacity = newBuf.capacity() / 2;
-    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity);
     typeBuffer.release(1);
     typeBuffer = newBuf;
     typeBufferAllocationSizeInBytes = (int)newAllocationSize;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index b275ab22db89d..5c456368c4961 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -450,8 +450,7 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer)
 
     final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
     newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
-    final int halfNewCapacity = newBuf.capacity() / 2;
-    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity);
     buffer.release(1);
     buffer = newBuf;
     if (dataBuffer) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index 44df96b82827d..3ebbf81fba160 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -549,8 +549,7 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer
 
     final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
     newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
-    final int halfNewCapacity = newBuf.capacity() / 2;
-    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity);
     buffer.release(1);
     buffer = newBuf;
     if (offsetBuffer) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 2dd2894ffebf9..bd4f7aac7b117 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -117,8 +117,7 @@ protected void reallocOffsetBuffer() {
 
     final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
     newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity);
-    final int halfNewCapacity = newBuf.capacity() / 2;
-    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity);
     offsetBuffer.release(1);
     offsetBuffer = newBuf;
     offsetAllocationSizeInBytes = (int) newAllocationSize;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index eadbab43619a6..f863bb6791856 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -217,8 +217,8 @@ private void reallocValidityBuffer() {
     }
 
     final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
-    newBuf.setZero(0, newBuf.capacity());
     newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+    newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity);
     validityBuffer.release(1);
     validityBuffer = newBuf;
     validityAllocationSizeInBytes = (int) newAllocationSize;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index e82a5fd79fadf..9666d800e4fe2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -296,8 +296,7 @@ private void reallocValidityBuffer() {
 
     final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
     newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
-    final int halfNewCapacity = newBuf.capacity() / 2;
-    newBuf.setZero(halfNewCapacity, halfNewCapacity);
+    newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity);
     validityBuffer.release(1);
     validityBuffer = newBuf;
     validityAllocationSizeInBytes = (int) newAllocationSize;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
index a296fbf1adb2b..a90902fc86187 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
@@ -414,8 +414,8 @@ private void reallocValidityBuffer() {
     }
 
     final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
-    newBuf.setZero(0, newBuf.capacity());
     newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+    newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity);
     validityBuffer.release(1);
     validityBuffer = newBuf;
     validityAllocationSizeInBytes = (int) newAllocationSize;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index d12586ecc59d1..5150273aab97d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -828,4 +828,49 @@ public void testSetInitialCapacity() {
       assertEquals(1, vector.getDataVector().getValueCapacity());
     }
   }
+
+  @Test
+  public void testClearAndReuse() {
+    try (final ListVector vector = ListVector.empty("list", allocator)) {
+      BigIntVector bigIntVector = (BigIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.BIGINT.getType())).getVector();
+      vector.setInitialCapacity(10);
+      vector.allocateNew();
+
+      vector.startNewValue(0);
+      bigIntVector.setSafe(0, 7);
+      vector.endValue(0, 1);
+      vector.startNewValue(1);
+      bigIntVector.setSafe(1, 8);
+      vector.endValue(1, 1);
+      vector.setValueCount(2);
+
+      Object result = vector.getObject(0);
+      ArrayList<Long> resultSet = (ArrayList<Long>) result;
+      assertEquals(new Long(7), resultSet.get(0));
+
+      result = vector.getObject(1);
+      resultSet = (ArrayList<Long>) result;
+      assertEquals(new Long(8), resultSet.get(0));
+
+      // Clear and release the buffers to trigger a realloc when adding next value
+      vector.clear();
+
+      // The list vector should reuse a buffer when reallocating the offset buffer
+      vector.startNewValue(0);
+      bigIntVector.setSafe(0, 7);
+      vector.endValue(0, 1);
+      vector.startNewValue(1);
+      bigIntVector.setSafe(1, 8);
+      vector.endValue(1, 1);
+      vector.setValueCount(2);
+
+      result = vector.getObject(0);
+      resultSet = (ArrayList<Long>) result;
+      assertEquals(new Long(7), resultSet.get(0));
+
+      result = vector.getObject(1);
+      resultSet = (ArrayList<Long>) result;
+      assertEquals(new Long(8), resultSet.get(0));
+    }
+  }
 }